diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 64c85c1101e6e61..fb11c82bb96d9cd 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,13 +1,10 @@ { - "image": "ghcr.io/python/devcontainer:2024.09.25.11038928730", + "image": "ghcr.io/python/devcontainer:latest", "onCreateCommand": [ // Install common tooling. "dnf", "install", "-y", - "which", - "zsh", - "fish", // For umask fix below. "/usr/bin/setfacl" ], diff --git a/.editorconfig b/.editorconfig index 5b04b32a89e3d22..ab1f7ce8425769e 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,11 +1,11 @@ root = true -[*.{py,c,cpp,h,js,rst,md,yml,yaml}] +[*.{py,c,cpp,h,js,rst,md,yml,yaml,toml,gram}] trim_trailing_whitespace = true insert_final_newline = true indent_style = space -[*.{py,c,cpp,h}] +[*.{py,c,cpp,h,toml,gram}] indent_size = 4 [*.rst] diff --git a/.gitattributes b/.gitattributes index 2f5a030981fb946..70aaadf5e3b5207 100644 --- a/.gitattributes +++ b/.gitattributes @@ -10,6 +10,7 @@ *.ico binary *.jpg binary *.pck binary +*.pdf binary *.png binary *.psd binary *.tar binary @@ -33,6 +34,9 @@ Lib/test/xmltestdata/* noeol Lib/venv/scripts/common/activate text eol=lf Lib/venv/scripts/posix/* text eol=lf +# Prevent GitHub's web conflict editor from converting LF to CRLF +*.rst text eol=lf + # CRLF files [attr]dos text eol=crlf @@ -67,6 +71,7 @@ PCbuild/readme.txt dos **/clinic/*.cpp.h generated **/clinic/*.h.h generated *_db.h generated +Doc/c-api/lifecycle.dot.svg generated Doc/data/stable_abi.dat generated Doc/library/token-list.inc generated Include/internal/pycore_ast.h generated @@ -80,7 +85,10 @@ Include/opcode.h generated Include/opcode_ids.h generated Include/token.h generated Lib/_opcode_metadata.py generated +Lib/idlelib/help.html generated Lib/keyword.py generated +Lib/pydoc_data/topics.py generated +Lib/pydoc_data/module_docs.py generated Lib/test/certdata/*.pem generated Lib/test/certdata/*.0 generated Lib/test/levenshtein_examples.json generated @@ -101,3 +109,5 @@ Python/stdlib_module_names.h generated Tools/peg_generator/pegen/grammar_parser.py generated aclocal.m4 generated configure generated +*.min.js generated +package-lock.json generated diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 16331b65e52979f..b54e5791367ca73 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,10 +5,13 @@ # https://git-scm.com/docs/gitignore#_pattern_format # GitHub -.github/** @ezio-melotti @hugovk @AA-Turner +.github/** @ezio-melotti @hugovk @AA-Turner @webknjaz +.github/workflows/jit.yml @savannahostrowski +Tools/build/compute-changes.py @AA-Turner @hugovk @webknjaz +Lib/test/test_tools/test_compute_changes.py @AA-Turner @hugovk @webknjaz # pre-commit -.pre-commit-config.yaml @hugovk @AlexWaygood +.pre-commit-config.yaml @hugovk .ruff.toml @hugovk @AlexWaygood @AA-Turner # Build system @@ -16,6 +19,10 @@ configure* @erlend-aasland @corona10 Makefile.pre.in @erlend-aasland Modules/Setup* @erlend-aasland +# generate-build-details +Tools/build/generate-build-details.py @FFY00 +Lib/test/test_build_details.py @FFY00 + # argparse **/*argparse* @savannahostrowski @@ -27,6 +34,7 @@ Modules/Setup* @erlend-aasland **/*genobject* @markshannon **/*hamt* @1st1 **/*jit* @brandtbucher @savannahostrowski +Python/perf_jit_trampoline.c # Exclude the owners of "**/*jit*", above. Objects/set* @rhettinger Objects/dict* @methane @markshannon Objects/typevarobject.c @JelleZijlstra @@ -53,6 +61,7 @@ Lib/test/test_type_*.py @JelleZijlstra Lib/test/test_capi/test_misc.py @markshannon Lib/test/test_pyrepl/* @pablogsal @lysnikolaou @ambv Tools/c-analyzer/ @ericsnowcurrently +Tools/check-c-api-docs/ @ZeroIntensity # dbm **/*dbm* @corona10 @erlend-aasland @serhiy-storchaka @@ -155,6 +164,7 @@ Doc/c-api/module.rst @ericsnowcurrently # Dates and times **/*datetime* @pganssle @abalkin **/*str*time* @pganssle @abalkin +Doc/library/datetime-* @pganssle Doc/library/time.rst @pganssle @abalkin Lib/test/test_time.py @pganssle @abalkin Modules/timemodule.c @pganssle @abalkin @@ -203,7 +213,7 @@ Lib/test/test_ast/ @eclips4 **/*multiprocessing* @gpshead # SQLite 3 -**/*sqlite* @berkerpeksag @erlend-aasland +**/*sqlite* @erlend-aasland # subprocess /Lib/subprocess.py @gpshead @@ -281,14 +291,21 @@ Doc/howto/clinic.rst @erlend-aasland # Subinterpreters **/*interpreteridobject.* @ericsnowcurrently **/*crossinterp* @ericsnowcurrently -Lib/test/support/interpreters/ @ericsnowcurrently Modules/_interp*module.c @ericsnowcurrently +Lib/test/test__interp*.py @ericsnowcurrently +Lib/concurrent/interpreters/ @ericsnowcurrently +Lib/test/support/channels.py @ericsnowcurrently +Doc/library/concurrent.interpreters.rst @ericsnowcurrently Lib/test/test_interpreters/ @ericsnowcurrently +Lib/concurrent/futures/interpreter.py @ericsnowcurrently # Android **/*Android* @mhsmith @freakboy3742 **/*android* @mhsmith @freakboy3742 +# Apple +/Apple @freakboy3742 + # iOS (but not termios) **/iOS* @freakboy3742 **/ios* @freakboy3742 @@ -298,7 +315,12 @@ Lib/test/test_interpreters/ @ericsnowcurrently **/*-ios* @freakboy3742 # WebAssembly -/Tools/wasm/ @brettcannon @freakboy3742 +/Platforms/emscripten @freakboy3742 +/Tools/wasm/README.md @brettcannon @freakboy3742 +/Tools/wasm/wasi-env @brettcannon +/Tools/wasm/wasi_build.py @brettcannon +/Tools/wasm/emscripten @freakboy3742 +/Tools/wasm/wasi @brettcannon # SBOM /Misc/externals.spdx.json @sethmlarson @@ -326,3 +348,8 @@ Modules/_xxtestfuzz/ @ammaraskar **/*templateobject* @lysnikolaou **/*templatelib* @lysnikolaou **/*tstring* @lysnikolaou + +# Remote debugging +Python/remote_debug.h @pablogsal +Python/remote_debugging.c @pablogsal +Modules/_remote_debugging_module.c @pablogsal @ambv @1st1 diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index 68aae1963574141..61ee08db9919bb9 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,9 +1,12 @@ -self-hosted-runner: - # Pending https://github.com/rhysd/actionlint/issues/533 - labels: ["windows-11-arm"] - config-variables: null +# Pending release of actionlint > 1.7.12 for ubuntu-26.04* support: +# https://github.com/rhysd/actionlint/pull/683 +self-hosted-runner: + labels: + - ubuntu-26.04 + - ubuntu-26.04-arm + paths: .github/workflows/**/*.yml: ignore: diff --git a/.github/dependabot.yml b/.github/dependabot.yml index c8a3165d6903645..7f3376f8ddb1e25 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -12,6 +12,11 @@ updates: update-types: - "version-update:semver-minor" - "version-update:semver-patch" + cooldown: + # https://blog.yossarian.net/2025/11/21/We-should-all-be-using-dependency-cooldowns + # Cooldowns protect against supply chain attacks by avoiding the + # highest-risk window immediately after new releases. + default-days: 14 - package-ecosystem: "pip" directory: "/Tools/" schedule: @@ -19,3 +24,5 @@ updates: labels: - "skip issue" - "skip news" + cooldown: + default-days: 14 diff --git a/.github/workflows/add-issue-header.yml b/.github/workflows/add-issue-header.yml index 3cbc23af578d106..55eb6bf33b81286 100644 --- a/.github/workflows/add-issue-header.yml +++ b/.github/workflows/add-issue-header.yml @@ -12,15 +12,17 @@ on: # Only ever run once - opened +permissions: + contents: read jobs: add-header: - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: issues: write timeout-minutes: 5 steps: - - uses: actions/github-script@v7 + - uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 with: # language=JavaScript script: | diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b192508c78685c4..287dccdbaad26ec 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,13 @@ permissions: contents: read concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-reusable + # https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#concurrency + # 'group' must be a key uniquely representing a PR or push event. + # github.workflow is the workflow name + # github.actor is the user invoking the workflow + # github.head_ref is the source branch of the PR or otherwise blank + # github.run_id is a unique number for the current run + group: ${{ github.workflow }}-${{ github.actor }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true env: @@ -43,11 +49,58 @@ jobs: if: fromJSON(needs.build-context.outputs.run-docs) uses: ./.github/workflows/reusable-docs.yml + check-abi: + name: 'Check if the ABI has changed' + runs-on: ubuntu-22.04 # 24.04 causes spurious errors + needs: build-context + if: needs.build-context.outputs.run-tests == 'true' + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.x' + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + sudo apt-get install -yq --no-install-recommends abigail-tools + - name: Build CPython + env: + CFLAGS: -g3 -O0 + run: | + # Build Python with the libpython dynamic library + ./configure --enable-shared + make -j4 + - name: Check for changes in the ABI + id: check + run: | + if ! make check-abidump; then + echo "Generated ABI file is not up to date." + echo "Please add the release manager of this branch as a reviewer of this PR." + echo "" + echo "The up to date ABI file should be attached to this build as an artifact." + echo "" + echo "To learn more about this check: https://devguide.python.org/getting-started/setup-building/index.html#regenerate-the-abi-dump" + echo "" + exit 1 + fi + - name: Generate updated ABI files + if: ${{ failure() && steps.check.conclusion == 'failure' }} + run: | + make regen-abidump + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + name: Publish updated ABI files + if: ${{ failure() && steps.check.conclusion == 'failure' }} + with: + name: abi-data + path: ./Doc/data/*.abi + check-autoconf-regen: name: 'Check if Autoconf files are up to date' # Don't use ubuntu-latest but a specific version to make the job # reproducible: to get the same tools versions (autoconf, aclocal, ...) - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 container: image: ghcr.io/python/autoconf:2025.01.02.12581854023 timeout-minutes: 60 @@ -58,7 +111,7 @@ jobs: run: | apt update && apt install git -yq git config --global --add safe.directory "$GITHUB_WORKSPACE" - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 1 persist-credentials: false @@ -90,33 +143,21 @@ jobs: name: 'Check if generated files are up to date' # Don't use ubuntu-latest but a specific version to make the job # reproducible: to get the same tools versions (autoconf, aclocal, ...) - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 60 needs: build-context if: needs.build-context.outputs.run-tests == 'true' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.x' - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }}-${{ env.pythonLocation }} - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Add ccache to PATH - run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Configure CPython run: | # Build Python with the libpython dynamic library @@ -124,7 +165,7 @@ jobs: - name: Build CPython run: | make -j4 regen-all - make regen-stdlib-module-names regen-sbom regen-unicodedata + make regen-stdlib-module-names regen-sbom - name: Check for changes run: | git add -u @@ -147,6 +188,14 @@ jobs: if: github.event_name == 'pull_request' # $GITHUB_EVENT_NAME run: make check-c-globals + check-c-api-docs: + name: C API Docs + needs: build-context + if: >- + needs.build-context.outputs.run-tests == 'true' + || needs.build-context.outputs.run-docs == 'true' + uses: ./.github/workflows/reusable-check-c-api-docs.yml + build-windows: name: >- Windows @@ -172,8 +221,8 @@ jobs: free-threading: ${{ matrix.free-threading }} build-windows-msi: - name: >- # ${{ '' } is a hack to nest jobs under the same sidebar category - Windows MSI${{ '' }} + # ${{ '' } is a hack to nest jobs under the same sidebar category. + name: Windows MSI${{ '' }} # zizmor: ignore[obfuscation] needs: build-context if: fromJSON(needs.build-context.outputs.run-windows-msi) strategy: @@ -192,32 +241,23 @@ jobs: macOS ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-macos == 'true' strategy: fail-fast: false matrix: - # Cirrus and macos-14 are M1, macos-13 is default GHA Intel. - # macOS 13 only runs tests against the GIL-enabled CPython. - # Cirrus used for upstream, macos-14 for forks. + # macos-26 is Apple Silicon, macos-26-intel is Intel. + # macos-26-intel only runs tests against the GIL-enabled CPython. os: - - ghcr.io/cirruslabs/macos-runner:sonoma - - macos-14 - - macos-13 - is-fork: # only used for the exclusion trick - - ${{ github.repository_owner != 'python' }} + - macos-26 + - macos-26-intel free-threading: - false - true exclude: - - os: ghcr.io/cirruslabs/macos-runner:sonoma - is-fork: true - - os: macos-14 - is-fork: false - - os: macos-13 + - os: macos-26-intel free-threading: true uses: ./.github/workflows/reusable-macos.yml with: - config_hash: ${{ needs.build-context.outputs.config-hash }} free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} @@ -227,7 +267,7 @@ jobs: ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} ${{ fromJSON(matrix.bolt) && '(bolt)' || '' }} needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' strategy: fail-fast: false matrix: @@ -237,6 +277,8 @@ jobs: free-threading: - false - true + # For BOLT jobs, https://apt.llvm.org/llvm.sh doesn't support LLVM 19 + # on Ubuntu 26.04, so stick to Ubuntu 24.04 until LLVM is upgraded. os: - ubuntu-24.04 - ubuntu-24.04-arm @@ -249,92 +291,139 @@ jobs: bolt: true uses: ./.github/workflows/reusable-ubuntu.yml with: - config_hash: ${{ needs.build-context.outputs.config-hash }} bolt-optimizations: ${{ matrix.bolt }} free-threading: ${{ matrix.free-threading }} os: ${{ matrix.os }} build-ubuntu-ssltests: - name: 'Ubuntu SSL tests with OpenSSL' + name: 'Ubuntu SSL tests' runs-on: ${{ matrix.os }} timeout-minutes: 60 needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' strategy: fail-fast: false matrix: - os: [ubuntu-24.04] - openssl_ver: [3.0.16, 3.1.8, 3.2.4, 3.3.3, 3.4.1] - # See Tools/ssl/make_ssl_data.py for notes on adding a new version + os: [ubuntu-26.04] + ssllib: + # See Tools/ssl/make_ssl_data.py for notes on adding a new version + ## OpenSSL + # Keep 1.1.1w in our list despite it being upstream EOL and otherwise + # unsupported as it most resembles other 1.1.1-work-a-like ssl APIs + # supported by important vendors such as AWS-LC. + - { name: openssl, version: 1.1.1w } + - { name: openssl, version: 3.0.21 } + - { name: openssl, version: 3.4.6 } + - { name: openssl, version: 3.5.7 } + - { name: openssl, version: 3.6.3 } env: - OPENSSL_VER: ${{ matrix.openssl_ver }} + SSLLIB_VER: ${{ matrix.ssllib.version }} MULTISSL_DIR: ${{ github.workspace }}/multissl - OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }} - LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib + SSLLIB_DIR: ${{ github.workspace }}/multissl/${{ matrix.ssllib.name }}/${{ matrix.ssllib.version }} + LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/${{ matrix.ssllib.name }}/${{ matrix.ssllib.version }}/lib steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v4 + - name: 'Restore SSL library build' + id: cache-ssl-lib + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH + path: ./multissl/${{ matrix.ssllib.name }}/${{ matrix.ssllib.version }} + key: ${{ matrix.os }}-multissl-${{ matrix.ssllib.name }}-${{ matrix.ssllib.version }} + - name: Install SSL Library + if: steps.cache-ssl-lib.outputs.cache-hit != 'true' run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false + python3 Tools/ssl/multissltests.py \ + --steps=library \ + --base-directory "$MULTISSL_DIR" \ + '--${{ matrix.ssllib.name }}' '${{ matrix.ssllib.version }}' \ + --system Linux - name: Configure CPython - run: ./configure CFLAGS="-fdiagnostics-format=json" --config-cache --enable-slower-safety --with-pydebug --with-openssl="$OPENSSL_DIR" + run: | + ./configure CFLAGS="-fdiagnostics-format=json" \ + --config-cache \ + --enable-slower-safety \ + --with-pydebug \ + --with-openssl="$SSLLIB_DIR" \ + --with-builtin-hashlib-hashes=blake2 \ + --with-ssl-default-suites=openssl - name: Build CPython run: make -j4 - name: Display build info run: make pythoninfo + - name: Verify python is linked to the right lib + run: | + ./python -c 'import ssl; print(ssl.OPENSSL_VERSION)' \ + | grep -iE '${{ matrix.ssllib.name }}.*${{ matrix.ssllib.version }}' - name: SSL tests run: ./python Lib/test/ssltests.py + build-android: + name: Android (${{ matrix.arch }}) + needs: build-context + if: needs.build-context.outputs.run-android == 'true' + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - arch: aarch64 + runs-on: macos-26 + - arch: x86_64 + runs-on: ubuntu-26.04 + + runs-on: ${{ matrix.runs-on }} + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Build and test + run: JAVA_HOME="${JAVA_HOME_21_X64:-$JAVA_HOME_21_arm64}" ./Android/android.py ci --fast-ci ${{ matrix.arch }}-linux-android + + build-ios: + name: iOS + needs: build-context + if: needs.build-context.outputs.run-ios == 'true' + timeout-minutes: 60 + runs-on: macos-26 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Build and test + run: python3 Apple ci iOS --fast-ci + + build-emscripten: + name: 'Emscripten' + needs: build-context + if: needs.build-context.outputs.run-emscripten == 'true' + uses: ./.github/workflows/reusable-emscripten.yml + build-wasi: name: 'WASI' needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-wasi == 'true' uses: ./.github/workflows/reusable-wasi.yml - with: - config_hash: ${{ needs.build-context.outputs.config-hash }} test-hypothesis: name: "Hypothesis tests on Ubuntu" - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 60 needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' env: - OPENSSL_VER: 3.0.16 + OPENSSL_VER: 3.5.7 PYTHONSTRICTEXTENSIONBUILD: 1 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Register gcc problem matcher @@ -348,20 +437,13 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v4 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: false - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" @@ -372,11 +454,6 @@ jobs: run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} run: | @@ -407,7 +484,7 @@ jobs: ./python -m venv "$VENV_LOC" && "$VENV_PYTHON" -m pip install -r "${GITHUB_WORKSPACE}/Tools/requirements-hypothesis.txt" - name: 'Restore Hypothesis database' id: cache-hypothesis-database - uses: actions/cache@v4 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ${{ env.CPYTHON_BUILDDIR }}/.hypothesis/ key: hypothesis-database-${{ github.head_ref || github.run_id }} @@ -434,7 +511,7 @@ jobs: -x test_subprocess \ -x test_signal \ -x test_sysconfig - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 if: always() with: name: hypothesis-example-db @@ -445,34 +522,25 @@ jobs: runs-on: ${{ matrix.os }} timeout-minutes: 60 needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' strategy: fail-fast: false matrix: - os: [ubuntu-24.04] + os: [ubuntu-26.04] env: - OPENSSL_VER: 3.0.16 + OPENSSL_VER: 3.5.7 PYTHONSTRICTEXTENSIONBUILD: 1 ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Install dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Set up GCC-10 for ASAN - uses: egor-tensin/setup-gcc@v1 - with: - version: 10 - name: Configure OpenSSL env vars run: | echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" @@ -480,21 +548,13 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v4 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ matrix.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Configure CPython run: ./configure --config-cache --with-address-sanitizer --without-pymalloc - name: Build CPython @@ -504,40 +564,42 @@ jobs: - name: Tests run: xvfb-run make ci - build-tsan: - name: >- - Thread sanitizer - ${{ fromJSON(matrix.free-threading) && '(free-threading)' || '' }} + build-san: + # ${{ '' } is a hack to nest jobs under the same sidebar category. + name: Sanitizers${{ '' }} # zizmor: ignore[obfuscation] needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' strategy: fail-fast: false matrix: + check-name: + - Thread free-threading: - false - true - uses: ./.github/workflows/reusable-tsan.yml + sanitizer: + - TSan + include: + - check-name: Undefined behavior + sanitizer: UBSan + free-threading: false + uses: ./.github/workflows/reusable-san.yml with: - config_hash: ${{ needs.build-context.outputs.config-hash }} + sanitizer: ${{ matrix.sanitizer }} free-threading: ${{ matrix.free-threading }} cross-build-linux: name: Cross build Linux - runs-on: ubuntu-latest + runs-on: ubuntu-26.04 timeout-minutes: 60 needs: build-context - if: needs.build-context.outputs.run-tests == 'true' + if: needs.build-context.outputs.run-ubuntu == 'true' steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ needs.build-context.outputs.config-hash }} - name: Register gcc problem matcher run: echo "::add-matcher::.github/problem-matchers/gcc.json" - name: Set build dir @@ -557,49 +619,56 @@ jobs: run: ./configure --prefix="$BUILD_DIR/cross-python" --with-build-python="$BUILD_DIR/host-python/bin/python3" - name: Install cross Python run: make -j8 install + - name: Display build info + run: | + "$BUILD_DIR/cross-python/bin/python3" -m test.pythoninfo - name: Run test subset with host build run: | "$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed - # CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ cifuzz: - name: CIFuzz - runs-on: ubuntu-latest - timeout-minutes: 60 + # ${{ '' } is a hack to nest jobs under the same sidebar category. + name: CIFuzz${{ '' }} # zizmor: ignore[obfuscation] needs: build-context - if: needs.build-context.outputs.run-ci-fuzz == 'true' + if: >- + needs.build-context.outputs.run-ci-fuzz == 'true' + || needs.build-context.outputs.run-ci-fuzz-stdlib == 'true' permissions: + contents: read security-events: write strategy: fail-fast: false matrix: - sanitizer: [address, undefined, memory] - steps: - - name: Build fuzzers (${{ matrix.sanitizer }}) - id: build - uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master - with: - oss-fuzz-project-name: cpython3 - sanitizer: ${{ matrix.sanitizer }} - - name: Run fuzzers (${{ matrix.sanitizer }}) - uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master - with: - fuzz-seconds: 600 - oss-fuzz-project-name: cpython3 - output-sarif: true - sanitizer: ${{ matrix.sanitizer }} - - name: Upload crash - if: failure() && steps.build.outcome == 'success' - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.sanitizer }}-artifacts - path: ./out/artifacts - - name: Upload SARIF - if: always() && steps.build.outcome == 'success' - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: cifuzz-sarif/results.sarif - checkout_path: cifuzz-sarif + sanitizer: + - address + oss-fuzz-project-name: + - cpython3 + - python3-libraries + include: + - sanitizer: undefined + oss-fuzz-project-name: cpython3 + - sanitizer: memory + oss-fuzz-project-name: cpython3 + exclude: + # Note that the 'no-exclude' sentinel below is to prevent + # an empty string value from excluding all jobs and causing + # GHA to create a 'default' matrix entry with all empty values. + - oss-fuzz-project-name: >- + ${{ + needs.build-context.outputs.run-ci-fuzz == 'true' + && 'no-exclude' + || 'cpython3' + }} + - oss-fuzz-project-name: >- + ${{ + needs.build-context.outputs.run-ci-fuzz-stdlib == 'true' + && 'no-exclude' + || 'python3-libraries' + }} + uses: ./.github/workflows/reusable-cifuzz.yml + with: + oss-fuzz-project-name: ${{ matrix.oss-fuzz-project-name }} + sanitizer: ${{ matrix.sanitizer }} all-required-green: # This job does nothing and is only used for the branch protection name: All required checks pass @@ -610,15 +679,18 @@ jobs: - check-docs - check-autoconf-regen - check-generated-files + - check-c-api-docs - build-windows - build-windows-msi - build-macos - build-ubuntu - build-ubuntu-ssltests + - build-ios + - build-emscripten - build-wasi - test-hypothesis - build-asan - - build-tsan + - build-san - cross-build-linux - cifuzz if: always() @@ -628,46 +700,50 @@ jobs: uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe with: allowed-failures: >- + build-android, + build-emscripten, build-windows-msi, build-ubuntu-ssltests, test-hypothesis, cifuzz, allowed-skips: >- - ${{ - !fromJSON(needs.build-context.outputs.run-docs) - && ' - check-docs, - ' - || '' - }} + ${{ !fromJSON(needs.build-context.outputs.run-docs) && 'check-docs,' || '' }} ${{ needs.build-context.outputs.run-tests != 'true' && ' check-autoconf-regen, check-generated-files, - build-macos, - build-ubuntu, - build-ubuntu-ssltests, - build-wasi, - test-hypothesis, - build-asan, - build-tsan, - cross-build-linux, ' || '' }} ${{ - !fromJSON(needs.build-context.outputs.run-windows-tests) - && ' - build-windows, - ' + !fromJSON(needs.build-context.outputs.run-tests) + && !fromJSON(needs.build-context.outputs.run-docs) + && 'check-c-api-docs,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-windows-tests) && 'build-windows,' || '' }} ${{ !fromJSON(needs.build-context.outputs.run-ci-fuzz) + && !fromJSON(needs.build-context.outputs.run-ci-fuzz-stdlib) + && 'cifuzz,' || + '' + }} + ${{ !fromJSON(needs.build-context.outputs.run-macos) && 'build-macos,' || '' }} + ${{ + !fromJSON(needs.build-context.outputs.run-ubuntu) && ' - cifuzz, + build-ubuntu, + build-ubuntu-ssltests, + test-hypothesis, + build-asan, + build-san, + cross-build-linux, ' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-android) && 'build-android,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-ios) && 'build-ios,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-emscripten) && 'build-emscripten,' || '' }} + ${{ !fromJSON(needs.build-context.outputs.run-wasi) && 'build-wasi,' || '' }} jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/documentation-links.yml b/.github/workflows/documentation-links.yml deleted file mode 100644 index a09a30587b35ebb..000000000000000 --- a/.github/workflows/documentation-links.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: Read the Docs PR preview -# Automatically edits a pull request's descriptions with a link -# to the documentation's preview on Read the Docs. - -on: - pull_request_target: - types: - - opened - paths: - - 'Doc/**' - - '.github/workflows/doc.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - documentation-links: - runs-on: ubuntu-latest - permissions: - pull-requests: write - timeout-minutes: 5 - - steps: - - uses: readthedocs/actions/preview@v1 - with: - project-slug: "cpython-previews" - single-version: "true" diff --git a/.github/workflows/jit.yml b/.github/workflows/jit.yml index 116e0c1e945e380..27846f8843e792d 100644 --- a/.github/workflows/jit.yml +++ b/.github/workflows/jit.yml @@ -1,7 +1,7 @@ name: JIT on: pull_request: - paths: + paths: &paths - '**jit**' - 'Python/bytecodes.c' - 'Python/optimizer*.c' @@ -9,13 +9,7 @@ on: - '!**/*.md' - '!**/*.ini' push: - paths: - - '**jit**' - - 'Python/bytecodes.c' - - 'Python/optimizer*.c' - - '!Python/perf_jit_trampoline.c' - - '!**/*.md' - - '!**/*.ini' + paths: *paths workflow_dispatch: permissions: @@ -27,14 +21,15 @@ concurrency: env: FORCE_COLOR: 1 + LLVM_VERSION: 19 jobs: interpreter: name: Interpreter (Debug) - runs-on: ubuntu-24.04 - timeout-minutes: 90 + runs-on: ubuntu-26.04 + timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Build tier two interpreter @@ -44,11 +39,12 @@ jobs: - name: Test tier two interpreter run: | ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - jit: + + windows: name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) - needs: interpreter + runs-on: ${{ matrix.runner }} - timeout-minutes: 90 + timeout-minutes: 60 strategy: fail-fast: false matrix: @@ -56,100 +52,151 @@ jobs: - i686-pc-windows-msvc/msvc - x86_64-pc-windows-msvc/msvc - aarch64-pc-windows-msvc/msvc - - x86_64-apple-darwin/clang - - aarch64-apple-darwin/clang - - x86_64-unknown-linux-gnu/gcc - - aarch64-unknown-linux-gnu/gcc debug: - true - false - llvm: - - 19 include: - target: i686-pc-windows-msvc/msvc architecture: Win32 - runner: windows-latest + runner: windows-2022 - target: x86_64-pc-windows-msvc/msvc architecture: x64 - runner: windows-latest + runner: windows-2022 - target: aarch64-pc-windows-msvc/msvc architecture: ARM64 runner: windows-11-arm - - target: x86_64-apple-darwin/clang - architecture: x86_64 - runner: macos-13 - - target: aarch64-apple-darwin/clang - architecture: aarch64 - runner: macos-14 - - target: x86_64-unknown-linux-gnu/gcc - architecture: x86_64 - runner: ubuntu-24.04 - - target: aarch64-unknown-linux-gnu/gcc - architecture: aarch64 - runner: ubuntu-24.04-arm steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' - # PCbuild downloads LLVM automatically: - - name: Windows - if: runner.os == 'Windows' + - name: Build run: | ./PCbuild/build.bat --experimental-jit ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} + - name: Test + run: | ./PCbuild/rt.bat ${{ matrix.debug && '-d' || '' }} -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. - # This is a bug in the macOS runner image where the pre-installed Python is installed in the same - # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes - # the symlink to the pre-installed Python so that the Homebrew Python is used instead. - - name: macOS - if: runner.os == 'macOS' + macos: + name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) + + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + target: + - x86_64-apple-darwin/clang + - aarch64-apple-darwin/clang + debug: + - true + - false + include: + - target: x86_64-apple-darwin/clang + runner: macos-26-intel + - target: aarch64-apple-darwin/clang + runner: macos-26 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - name: Install LLVM run: | brew update - find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete - brew install llvm@${{ matrix.llvm }} + brew install llvm@${{ env.LLVM_VERSION }} + - name: Build + run: | export SDKROOT="$(xcrun --show-sdk-path)" + # Set MACOSX_DEPLOYMENT_TARGET and -Werror=unguarded-availability to + # make sure we don't break downstream distributors (like uv): + export CFLAGS_JIT='-Werror=unguarded-availability' + export MACOSX_DEPLOYMENT_TARGET=10.15 ./configure --enable-experimental-jit --enable-universalsdk --with-universal-archs=universal2 ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 + - name: Test + run: | ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - name: Linux - if: runner.os == 'Linux' + linux: + name: ${{ matrix.target }} (${{ matrix.debug && 'Debug' || 'Release' }}) + + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + target: + - x86_64-unknown-linux-gnu/gcc + - aarch64-unknown-linux-gnu/gcc + debug: + - true + - false + include: + # https://apt.llvm.org/llvm.sh doesn't support LLVM 19 + # on Ubuntu 26.04, so stick to Ubuntu 24.04 until LLVM is upgraded. + - target: x86_64-unknown-linux-gnu/gcc + runner: ubuntu-24.04 + - target: aarch64-unknown-linux-gnu/gcc + runner: ubuntu-24.04-arm + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - name: Build run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ env.LLVM_VERSION }} + export PATH="$(llvm-config-${{ env.LLVM_VERSION }} --bindir):$PATH" ./configure --enable-experimental-jit ${{ matrix.debug && '--with-pydebug' || '' }} make all --jobs 4 + - name: Test + run: | ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - # XXX: GH-133171 - # jit-with-disabled-gil: - # name: Free-Threaded (Debug) - # needs: interpreter - # runs-on: ubuntu-24.04 - # timeout-minutes: 90 - # strategy: - # fail-fast: false - # matrix: - # llvm: - # - 19 - # steps: - # - uses: actions/checkout@v4 - # with: - # persist-credentials: false - # - uses: actions/setup-python@v5 - # with: - # python-version: '3.11' - # - name: Build with JIT enabled and GIL disabled - # run: | - # sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - # export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - # ./configure --enable-experimental-jit --with-pydebug --disable-gil - # make all --jobs 4 - # - name: Run tests - # run: | - # ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 + linux-extras: + name: ${{ matrix.name }} + + # https://apt.llvm.org/llvm.sh doesn't support LLVM 19 on Ubuntu 26.04, + # so stick to Ubuntu 24.04 until LLVM is upgraded. + runs-on: ubuntu-24.04 + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + + - name: JIT without optimizations (Debug) + configure_flags: --enable-experimental-jit --with-pydebug + test_env: "PYTHON_UOPS_OPTIMIZE=0" + - name: JIT with tail calling interpreter + configure_flags: --enable-experimental-jit --with-tail-call-interp --with-pydebug + use_clang: true + run_tests: false + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - name: Build + run: | + sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ env.LLVM_VERSION }} + export PATH="$(llvm-config-${{ env.LLVM_VERSION }} --bindir):$PATH" + if [ "${{ matrix.use_clang }}" = "true" ]; then + export CC=clang-${{ env.LLVM_VERSION }} + fi + ./configure ${{ matrix.configure_flags }} + make all --jobs 4 + - name: Test + if: matrix.run_tests != false + run: | + ${{ matrix.test_env }} ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d74ce8fcc256dc8..e9a4eb2b0808cb7 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -19,10 +19,7 @@ jobs: timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v5 - with: - python-version: "3.x" - - uses: pre-commit/action@v3.0.1 + - uses: j178/prek-action@0bb87d7f00b0c99306c8bcb8b8beba1eb581c037 # v1.1.1 diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml index 908daaf3a6019a5..bbcd57eedf26436 100644 --- a/.github/workflows/mypy.yml +++ b/.github/workflows/mypy.yml @@ -13,13 +13,24 @@ on: - "Lib/test/libregrtest/**" - "Lib/tomllib/**" - "Misc/mypy/**" + - "Tools/build/check_extension_modules.py" + - "Tools/build/check_warnings.py" + - "Tools/build/compute-changes.py" + - "Tools/build/deepfreeze.py" + - "Tools/build/generate-build-details.py" + - "Tools/build/generate_levenshtein_examples.py" - "Tools/build/generate_sbom.py" + - "Tools/build/generate_stdlib_module_names.py" + - "Tools/build/mypy.ini" + - "Tools/build/umarshal.py" + - "Tools/build/update_file.py" + - "Tools/build/verify_ensurepip_wheels.py" - "Tools/cases_generator/**" + - "Tools/check-c-api-docs/**" - "Tools/clinic/**" - "Tools/jit/**" - "Tools/peg_generator/**" - "Tools/requirements-dev.txt" - - "Tools/wasm/**" workflow_dispatch: permissions: @@ -48,20 +59,20 @@ jobs: "Lib/tomllib", "Tools/build", "Tools/cases_generator", + "Tools/check-c-api-docs", "Tools/clinic", "Tools/jit", "Tools/peg_generator", - "Tools/wasm", ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 with: python-version: "3.13" - cache: pip - cache-dependency-path: Tools/requirements-dev.txt - - run: pip install -r Tools/requirements-dev.txt + activate-environment: true + cache-dependency-glob: Tools/requirements-dev.txt + - run: uv pip install -r Tools/requirements-dev.txt - run: python3 Misc/mypy/make_symlinks.py --symlink - - run: mypy --config-file ${{ matrix.target }}/mypy.ini + - run: mypy --num-workers 4 --config-file ${{ matrix.target }}/mypy.ini diff --git a/.github/workflows/new-bugs-announce-notifier.yml b/.github/workflows/new-bugs-announce-notifier.yml index 9f1a8a824e5f19a..ea175ed20d1e7ba 100644 --- a/.github/workflows/new-bugs-announce-notifier.yml +++ b/.github/workflows/new-bugs-announce-notifier.yml @@ -6,19 +6,21 @@ on: - opened permissions: - issues: read + contents: read jobs: notify-new-bugs-announce: - runs-on: ubuntu-latest + runs-on: ubuntu-slim + permissions: + issues: read timeout-minutes: 10 steps: - - uses: actions/setup-node@v4 + - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 with: node-version: 20 - run: npm install mailgun.js form-data - name: Send notification - uses: actions/github-script@v7 + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0 env: MAILGUN_API_KEY: ${{ secrets.MAILGUN_PYTHON_ORG_MAILGUN_KEY }} with: diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index d5538cd9367ec60..3b3d90e198360ea 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -1,24 +1,23 @@ #!/bin/sh apt-get update -apt-get -yq install \ +apt-get -yq --no-install-recommends install \ build-essential \ pkg-config \ - ccache \ + curl \ gdb \ lcov \ libb2-dev \ libbz2-dev \ libffi-dev \ - libgdbm-dev \ libgdbm-compat-dev \ + libgdbm-dev \ liblzma-dev \ libncurses5-dev \ libreadline6-dev \ libsqlite3-dev \ libssl-dev \ - lzma \ - lzma-dev \ + libzstd-dev \ strace \ tk-dev \ uuid-dev \ diff --git a/.github/workflows/project-updater.yml b/.github/workflows/project-updater.yml index 1d9d637ec848a61..710424a28f2824b 100644 --- a/.github/workflows/project-updater.yml +++ b/.github/workflows/project-updater.yml @@ -24,7 +24,7 @@ jobs: - { project: 32, label: sprint } steps: - - uses: actions/add-to-project@v1.0.0 + - uses: actions/add-to-project@244f685bbc3b7adfa8466e08b698b5577571133e # v1.0.2 with: project-url: https://github.com/orgs/python/projects/${{ matrix.project }} github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} diff --git a/.github/workflows/regen-abidump.sh b/.github/workflows/regen-abidump.sh index 251bb3857ecfcb6..75a1a72e3702024 100644 --- a/.github/workflows/regen-abidump.sh +++ b/.github/workflows/regen-abidump.sh @@ -2,7 +2,7 @@ set -ex export DEBIAN_FRONTEND=noninteractive ./.github/workflows/posix-deps-apt.sh -apt-get install -yq abigail-tools python3 +apt-get install -yq --no-install-recommends abigail-tools python3 export CFLAGS="-g3 -O0" ./configure --enable-shared && make make regen-abidump diff --git a/.github/workflows/require-pr-label.yml b/.github/workflows/require-pr-label.yml index 7e534c58c798d1c..8af254c10786e1b 100644 --- a/.github/workflows/require-pr-label.yml +++ b/.github/workflows/require-pr-label.yml @@ -4,18 +4,21 @@ on: pull_request: types: [opened, reopened, labeled, unlabeled, synchronize] +permissions: + contents: read + jobs: label-dnm: name: DO-NOT-MERGE if: github.repository_owner == 'python' - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: pull-requests: read timeout-minutes: 10 steps: - name: Check there's no DO-NOT-MERGE - uses: mheap/github-action-required-labels@v5 + uses: mheap/github-action-required-labels@0ac283b4e65c1fb28ce6079dea5546ceca98ccbe # v5.5.2 with: mode: exactly count: 0 @@ -25,7 +28,7 @@ jobs: label-reviews: name: Unresolved review if: github.repository_owner == 'python' - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: pull-requests: read timeout-minutes: 10 @@ -33,7 +36,7 @@ jobs: steps: # Check that the PR is not awaiting changes from the author due to previous review. - name: Check there's no required changes - uses: mheap/github-action-required-labels@v5 + uses: mheap/github-action-required-labels@0ac283b4e65c1fb28ce6079dea5546ceca98ccbe # v5.5.2 with: mode: exactly count: 0 @@ -42,7 +45,7 @@ jobs: awaiting change review - id: is-feature name: Check whether this PR is a feature (contains a "type-feature" label) - uses: mheap/github-action-required-labels@v5 + uses: mheap/github-action-required-labels@0ac283b4e65c1fb28ce6079dea5546ceca98ccbe # v5.5.2 with: mode: exactly count: 1 @@ -53,7 +56,7 @@ jobs: - id: awaiting-merge if: steps.is-feature.outputs.status == 'success' name: Check for complete review - uses: mheap/github-action-required-labels@v5 + uses: mheap/github-action-required-labels@0ac283b4e65c1fb28ce6079dea5546ceca98ccbe # v5.5.2 with: mode: exactly count: 1 diff --git a/.github/workflows/reusable-check-c-api-docs.yml b/.github/workflows/reusable-check-c-api-docs.yml new file mode 100644 index 000000000000000..db030c80008b1ed --- /dev/null +++ b/.github/workflows/reusable-check-c-api-docs.yml @@ -0,0 +1,25 @@ +name: Reusable C API Docs Check + +on: + workflow_call: + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + +jobs: + check-c-api-docs: + name: 'Check if all C APIs are documented' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.x' + - name: Check for undocumented C APIs + run: python Tools/check-c-api-docs/main.py diff --git a/.github/workflows/reusable-cifuzz.yml b/.github/workflows/reusable-cifuzz.yml new file mode 100644 index 000000000000000..9b49e7fd26f0078 --- /dev/null +++ b/.github/workflows/reusable-cifuzz.yml @@ -0,0 +1,49 @@ +# CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/ +name: Reusable CIFuzz + +on: + workflow_call: + inputs: + oss-fuzz-project-name: + description: OSS-Fuzz project name + required: true + type: string + sanitizer: + description: OSS-Fuzz sanitizer + required: true + type: string + +permissions: + contents: read + +jobs: + cifuzz: + name: ${{ inputs.oss-fuzz-project-name }} (${{ inputs.sanitizer }}) + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Build fuzzers (${{ inputs.sanitizer }}) + id: build + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@ed23f8af80ff82b25ca67cd9b101e690b8897b3f # master + with: + oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }} + sanitizer: ${{ inputs.sanitizer }} + - name: Run fuzzers (${{ inputs.sanitizer }}) + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@ed23f8af80ff82b25ca67cd9b101e690b8897b3f # master + with: + fuzz-seconds: 600 + oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }} + output-sarif: true + sanitizer: ${{ inputs.sanitizer }} + - name: Upload crash + if: failure() && steps.build.outcome == 'success' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ${{ inputs.sanitizer }}-artifacts + path: ./out/artifacts + - name: Upload SARIF + if: always() && steps.build.outcome == 'success' + uses: github/codeql-action/upload-sarif@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1 + with: + sarif_file: cifuzz-sarif/results.sarif + checkout_path: cifuzz-sarif diff --git a/.github/workflows/reusable-context.yml b/.github/workflows/reusable-context.yml index 73dc254edc5fbc7..b8a9e2960eca591 100644 --- a/.github/workflows/reusable-context.yml +++ b/.github/workflows/reusable-context.yml @@ -17,24 +17,45 @@ on: # yamllint disable-line rule:truthy # || 'falsy-branch' # }} # - config-hash: - description: Config hash value for use in cache keys - value: ${{ jobs.compute-changes.outputs.config-hash }} # str + run-android: + description: Whether to run the Android tests + value: ${{ jobs.compute-changes.outputs.run-android }} # bool + run-ci-fuzz: + description: Whether to run the CIFuzz job for 'cpython' fuzzer + value: ${{ jobs.compute-changes.outputs.run-ci-fuzz }} # bool + run-ci-fuzz-stdlib: + description: Whether to run the CIFuzz job for 'python3-libraries' fuzzer + value: ${{ jobs.compute-changes.outputs.run-ci-fuzz-stdlib }} # bool run-docs: description: Whether to build the docs value: ${{ jobs.compute-changes.outputs.run-docs }} # bool + run-ios: + description: Whether to run the iOS tests + value: ${{ jobs.compute-changes.outputs.run-ios }} # bool + run-macos: + description: Whether to run the macOS tests + value: ${{ jobs.compute-changes.outputs.run-macos }} # bool run-tests: description: Whether to run the regular tests value: ${{ jobs.compute-changes.outputs.run-tests }} # bool - run-windows-tests: - description: Whether to run the Windows tests - value: ${{ jobs.compute-changes.outputs.run-windows-tests }} # bool + run-ubuntu: + description: Whether to run the Ubuntu tests + value: ${{ jobs.compute-changes.outputs.run-ubuntu }} # bool + run-emscripten: + description: Whether to run the Emscripten tests + value: ${{ jobs.compute-changes.outputs.run-emscripten }} # bool + run-wasi: + description: Whether to run the WASI tests + value: ${{ jobs.compute-changes.outputs.run-wasi }} # bool run-windows-msi: description: Whether to run the MSI installer smoke tests value: ${{ jobs.compute-changes.outputs.run-windows-msi }} # bool - run-ci-fuzz: - description: Whether to run the CIFuzz job - value: ${{ jobs.compute-changes.outputs.run-ci-fuzz }} # bool + run-windows-tests: + description: Whether to run the Windows tests + value: ${{ jobs.compute-changes.outputs.run-windows-tests }} # bool + +permissions: + contents: read jobs: compute-changes: @@ -42,22 +63,28 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 outputs: - config-hash: ${{ steps.config-hash.outputs.hash }} + run-android: ${{ steps.changes.outputs.run-android }} run-ci-fuzz: ${{ steps.changes.outputs.run-ci-fuzz }} + run-ci-fuzz-stdlib: ${{ steps.changes.outputs.run-ci-fuzz-stdlib }} run-docs: ${{ steps.changes.outputs.run-docs }} + run-ios: ${{ steps.changes.outputs.run-ios }} + run-macos: ${{ steps.changes.outputs.run-macos }} run-tests: ${{ steps.changes.outputs.run-tests }} + run-ubuntu: ${{ steps.changes.outputs.run-ubuntu }} + run-emscripten: ${{ steps.changes.outputs.run-emscripten }} + run-wasi: ${{ steps.changes.outputs.run-wasi }} run-windows-msi: ${{ steps.changes.outputs.run-windows-msi }} run-windows-tests: ${{ steps.changes.outputs.run-windows-tests }} steps: - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3" - run: >- echo '${{ github.event_name }}' - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false ref: >- @@ -97,10 +124,6 @@ jobs: run: python Tools/build/compute-changes.py env: GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + GITHUB_EVENT_NAME: ${{ github.event_name }} CCF_TARGET_REF: ${{ github.base_ref || github.event.repository.default_branch }} CCF_HEAD_REF: ${{ github.event.pull_request.head.sha || github.sha }} - - - name: Compute hash for config cache key - id: config-hash - run: | - echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/reusable-docs.yml b/.github/workflows/reusable-docs.yml index 657e0a6bf662f75..2e54cd9e23cab76 100644 --- a/.github/workflows/reusable-docs.yml +++ b/.github/workflows/reusable-docs.yml @@ -3,6 +3,17 @@ name: Reusable Docs on: workflow_call: workflow_dispatch: + # Pushes to CPython branches seed the pip caches under the exact keys every + # docs PR restores from. Without a branch-scoped copy, each PR saves a + # duplicate into its own refs/pull/N/merge scope that nothing else can read. + push: + branches: + - main + - '3.*' + paths: + - 'Doc/pylock.toml' + - 'Doc/requirements.txt' + - '.github/workflows/reusable-docs.yml' permissions: contents: read @@ -27,7 +38,7 @@ jobs: refspec_pr: '+${{ github.event.pull_request.head.sha }}:remotes/origin/${{ github.event.pull_request.head.ref }}' steps: - name: 'Check out latest PR branch commit' - uses: actions/checkout@v4 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false ref: >- @@ -52,7 +63,7 @@ jobs: git fetch origin "${refspec_base}" --shallow-since="${DATE}" \ --no-tags --prune --no-recurse-submodules - name: 'Set up Python' - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3' cache: 'pip' @@ -66,7 +77,7 @@ jobs: run: | set -Eeuo pipefail # Build docs with the nit-picky option; write warnings to file - make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet --nitpicky --fail-on-warning --warning-file sphinx-warnings.txt" html + make -C Doc/ PYTHON=../python SPHINXOPTS="--quiet --nitpicky --warning-file sphinx-warnings.txt" html - name: 'Check warnings' if: github.event_name == 'pull_request' run: | @@ -79,20 +90,20 @@ jobs: # Run "doctest" on HEAD as new syntax doesn't exist in the latest stable release doctest: name: 'Doctest' - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 60 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/cache@v4 + - uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ~/.cache/pip key: ubuntu-doc-${{ hashFiles('Doc/requirements.txt') }} restore-keys: | ubuntu-doc- - name: 'Install Dependencies' - run: sudo ./.github/workflows/posix-deps-apt.sh && sudo apt-get install wamerican + run: sudo ./.github/workflows/posix-deps-apt.sh && sudo apt-get install --no-install-recommends wamerican - name: 'Configure CPython' run: ./configure --with-pydebug - name: 'Build CPython' @@ -102,3 +113,30 @@ jobs: # Use "xvfb-run" since some doctest tests open GUI windows - name: 'Run documentation doctest' run: xvfb-run make -C Doc/ PYTHON=../python SPHINXERRORHANDLING="--fail-on-warning" doctest + + check-epub: + name: 'Check EPUB' + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: 'Set up Python' + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3' + cache: 'pip' + cache-dependency-path: 'Doc/requirements.txt' + - name: 'Install build dependencies' + run: | + make -C Doc/ venv + python -m pip install epubcheck + - name: 'Build EPUB documentation' + run: make -C Doc/ PYTHON=../python epub + - name: 'Run epubcheck' + continue-on-error: true + run: epubcheck Doc/build/epub/Python.epub &> Doc/epubcheck.txt + - run: cat Doc/epubcheck.txt + - name: 'Check for fatal errors in EPUB' + run: python Doc/tools/check-epub.py diff --git a/.github/workflows/reusable-emscripten.yml b/.github/workflows/reusable-emscripten.yml new file mode 100644 index 000000000000000..b15169164572b4b --- /dev/null +++ b/.github/workflows/reusable-emscripten.yml @@ -0,0 +1,81 @@ +name: Reusable Emscripten + +on: + workflow_call: + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + +jobs: + build-emscripten-reusable: + name: 'build and test' + runs-on: ubuntu-26.04 + timeout-minutes: 40 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: "Read Emscripten config" + id: emscripten-config + shell: python + run: | + import hashlib + import json + import os + import tomllib + from pathlib import Path + + config = tomllib.loads(Path("Platforms/emscripten/config.toml").read_text()) + h = hashlib.sha256() + h.update(json.dumps(config["dependencies"], sort_keys=True).encode()) + h.update(Path("Platforms/emscripten/make_libffi.sh").read_bytes()) + h.update(b'1') # Update to explicitly bust cache + emsdk_cache = Path(os.environ["RUNNER_TEMP"]) / "emsdk-cache" + with open(os.environ["GITHUB_OUTPUT"], "a") as f: + f.write(f"emscripten-version={config['emscripten-version']}\n") + f.write(f"node-version={config['node-version']}\n") + f.write(f"deps-hash={h.hexdigest()}\n") + with open(os.environ["GITHUB_ENV"], "a") as f: + f.write(f"EMSDK_CACHE={emsdk_cache}\n") + - name: "Install Node.js" + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: ${{ steps.emscripten-config.outputs.node-version }} + - name: "Cache Emscripten SDK" + id: emsdk-cache + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: ${{ env.EMSDK_CACHE }} + key: emsdk-${{ steps.emscripten-config.outputs.emscripten-version }}-${{ steps.emscripten-config.outputs.deps-hash }} + restore-keys: emsdk-${{ steps.emscripten-config.outputs.emscripten-version }} + - name: "Install Python" + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.x' + - name: "Runner image version" + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: "Install Emscripten" + run: python3 Platforms/emscripten install-emscripten + - name: "Configure build Python" + run: python3 Platforms/emscripten configure-build-python -- --config-cache --with-pydebug + - name: "Make build Python" + run: python3 Platforms/emscripten make-build-python + - name: "Display build info of the build Python" + run: python3 Platforms/emscripten pythoninfo-build + - name: "Make dependencies" + run: >- + python3 Platforms/emscripten make-dependencies + ${{ steps.emsdk-cache.outputs.cache-hit == 'true' && '--check-up-to-date' || '' }} + - name: "Configure host/Emscripten Python" + run: python3 Platforms/emscripten configure-host --host-runner node -- --config-cache + - name: "Make host/Emscripten Python" + run: python3 Platforms/emscripten make-host + - name: "Display build info of the host/Emscripten Python" + run: python3 Platforms/emscripten pythoninfo-host + - name: "Test" + run: python3 Platforms/emscripten run --test + - name: "Test Repl" + run: Platforms/emscripten/browser_test/run_test.sh diff --git a/.github/workflows/reusable-macos.yml b/.github/workflows/reusable-macos.yml index de0c40221364ad9..6cb427a03713cf7 100644 --- a/.github/workflows/reusable-macos.yml +++ b/.github/workflows/reusable-macos.yml @@ -3,9 +3,6 @@ name: Reusable macOS on: workflow_call: inputs: - config_hash: - required: true - type: string free-threading: required: false type: boolean @@ -15,6 +12,9 @@ on: required: true type: string +permissions: + contents: read + env: FORCE_COLOR: 1 @@ -31,19 +31,15 @@ jobs: PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} - name: Install Homebrew dependencies run: | - brew install pkg-config openssl@3.0 xz gdbm tcl-tk@8 make + brew bundle --file=Misc/Brewfile + brew install make # Because alternate versions are not symlinked into place by default: brew link --overwrite tcl-tk@8 - name: Configure CPython @@ -60,15 +56,15 @@ jobs: --prefix=/opt/python-dev \ --with-openssl="$(brew --prefix openssl@3.0)" - name: Build CPython - if : ${{ inputs.free-threading || inputs.os != 'macos-13' }} + if : ${{ inputs.free-threading || inputs.os != 'macos-26-intel' }} run: gmake -j8 - name: Build CPython for compiler warning check - if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + if : ${{ !inputs.free-threading && inputs.os == 'macos-26-intel' }} run: set -o pipefail; gmake -j8 --output-sync 2>&1 | tee compiler_output_macos.txt - name: Display build info run: make pythoninfo - name: Check compiler warnings - if : ${{ !inputs.free-threading && inputs.os == 'macos-13' }} + if : ${{ !inputs.free-threading && inputs.os == 'macos-26-intel' }} run: >- python3 Tools/build/check_warnings.py --compiler-output-file-path=compiler_output_macos.txt diff --git a/.github/workflows/reusable-san.yml b/.github/workflows/reusable-san.yml new file mode 100644 index 000000000000000..47e41c90a1658b5 --- /dev/null +++ b/.github/workflows/reusable-san.yml @@ -0,0 +1,138 @@ +name: Reusable Sanitizer + +on: + workflow_call: + inputs: + sanitizer: + required: true + type: string + free-threading: + description: Whether to use free-threaded mode + required: false + type: boolean + default: false + +permissions: + contents: read + +env: + FORCE_COLOR: 1 + OPENSSL_VER: 3.5.7 + +jobs: + build-san-reusable: + name: >- + ${{ inputs.sanitizer }}${{ + inputs.free-threading + && ' (free-threading)' + || '' + }} + # https://apt.llvm.org/llvm.sh doesn't support LLVM 20 on Ubuntu 26.04, + # so stick to Ubuntu 24.04 until LLVM is upgraded. + runs-on: ubuntu-24.04 + timeout-minutes: 60 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Runner image version + run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" + - name: Install dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + # Install clang + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + sudo ./llvm.sh 20 + sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-20 100 + sudo update-alternatives --set clang /usr/bin/clang-20 + sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-20 100 + sudo update-alternatives --set clang++ /usr/bin/clang++-20 + echo "CC=clang" >> "$GITHUB_ENV" + echo "CXX=clang++" >> "$GITHUB_ENV" + - name: TSan option setup + if: inputs.sanitizer == 'TSan' + run: | + sudo sysctl -w vm.mmap_rnd_bits=28 # Reduce ASLR to avoid TSan crashing + + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> "$GITHUB_ENV" + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> "$GITHUB_ENV" + echo "TSAN_OPTIONS=${SAN_LOG_OPTION} suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${SUPPRESSIONS_SUFFIX}.txt handle_segv=0" >> "$GITHUB_ENV" + env: + SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log + SUPPRESSIONS_SUFFIX: >- + ${{ + fromJSON(inputs.free-threading) + && '_free_threading' + || '' + }} + - name: UBSan option setup + if: inputs.sanitizer != 'TSan' + run: >- + echo + "UBSAN_OPTIONS=${SAN_LOG_OPTION}" + >> "$GITHUB_ENV" + env: + SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" + - name: 'Restore OpenSSL build (TSan)' + id: cache-openssl + if: inputs.sanitizer == 'TSan' + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ env.IMAGE_OS_VERSION }}-multissl-openssl-tsan-${{ env.OPENSSL_VER }} + - name: Install OpenSSL (TSan) + if: >- + inputs.sanitizer == 'TSan' + && steps.cache-openssl.outputs.cache-hit != 'true' + run: >- + python3 Tools/ssl/multissltests.py + --steps=library + --base-directory="${MULTISSL_DIR}" + --openssl="${OPENSSL_VER}" + --system=Linux + --tsan + - name: Configure CPython + run: >- + ./configure + --config-cache + ${{ + inputs.sanitizer == 'TSan' + && '--with-thread-sanitizer' + || '--with-undefined-behavior-sanitizer' + }} + --with-pydebug + ${{ inputs.sanitizer == 'TSan' && '--with-openssl="$OPENSSL_DIR" --with-openssl-rpath=auto' || '' }} + ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: Tests + run: >- + ./python -m test + ${{ inputs.sanitizer == 'TSan' && '--tsan' || '' }} + -j4 -W --timeout=900 --slowest + - name: Parallel tests + if: >- + inputs.sanitizer == 'TSan' + && fromJSON(inputs.free-threading) + run: ./python -m test --tsan-parallel --parallel-threads=4 -j4 -W --timeout=600 --slowest + - name: Display logs + if: always() + run: find "${GITHUB_WORKSPACE}" -name 'san_log.*' | xargs head -n 1000 + - name: Archive logs + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: >- + ${{ inputs.sanitizer }}-logs-${{ + fromJSON(inputs.free-threading) + && 'free-threading' + || 'default' + }} + path: san_log.* + if-no-files-found: ignore diff --git a/.github/workflows/reusable-tsan.yml b/.github/workflows/reusable-tsan.yml deleted file mode 100644 index 6a58e5305f8e09b..000000000000000 --- a/.github/workflows/reusable-tsan.yml +++ /dev/null @@ -1,94 +0,0 @@ -name: Reusable Thread Sanitizer - -on: - workflow_call: - inputs: - config_hash: - required: true - type: string - free-threading: - description: Whether to use free-threaded mode - required: false - type: boolean - default: false - -env: - FORCE_COLOR: 1 - -jobs: - build-tsan-reusable: - name: 'Thread sanitizer' - runs-on: ubuntu-24.04 - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Runner image version - run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} - - name: Install dependencies - run: | - sudo ./.github/workflows/posix-deps-apt.sh - # Install clang-18 - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - sudo ./llvm.sh 17 # gh-121946: llvm-18 package is temporarily broken - sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-17 100 - sudo update-alternatives --set clang /usr/bin/clang-17 - sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-17 100 - sudo update-alternatives --set clang++ /usr/bin/clang++-17 - # Reduce ASLR to avoid TSAN crashing - sudo sysctl -w vm.mmap_rnd_bits=28 - - name: TSAN option setup - run: | - echo "TSAN_OPTIONS=log_path=${GITHUB_WORKSPACE}/tsan_log suppressions=${GITHUB_WORKSPACE}/Tools/tsan/suppressions${{ - fromJSON(inputs.free-threading) - && '_free_threading' - || '' - }}.txt handle_segv=0" >> "$GITHUB_ENV" - echo "CC=clang" >> "$GITHUB_ENV" - echo "CXX=clang++" >> "$GITHUB_ENV" - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - - name: Configure CPython - run: >- - ./configure - --config-cache - --with-thread-sanitizer - --with-pydebug - ${{ fromJSON(inputs.free-threading) && '--disable-gil' || '' }} - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: Tests - run: ./python -m test --tsan -j4 - - name: Parallel tests - if: fromJSON(inputs.free-threading) - run: ./python -m test --tsan-parallel --parallel-threads=4 -j4 - - name: Display TSAN logs - if: always() - run: find "${GITHUB_WORKSPACE}" -name 'tsan_log.*' | xargs head -n 1000 - - name: Archive TSAN logs - if: always() - uses: actions/upload-artifact@v4 - with: - name: >- - tsan-logs-${{ - fromJSON(inputs.free-threading) - && 'free-threading' - || 'default' - }} - path: tsan_log.* - if-no-files-found: ignore diff --git a/.github/workflows/reusable-ubuntu.yml b/.github/workflows/reusable-ubuntu.yml index 76b19fd5d1a72e2..6cd480b0285346d 100644 --- a/.github/workflows/reusable-ubuntu.yml +++ b/.github/workflows/reusable-ubuntu.yml @@ -3,9 +3,6 @@ name: Reusable Ubuntu on: workflow_call: inputs: - config_hash: - required: true - type: string bolt-optimizations: description: Whether to enable BOLT optimizations required: false @@ -21,6 +18,9 @@ on: required: true type: string +permissions: + contents: read + env: FORCE_COLOR: 1 @@ -30,11 +30,11 @@ jobs: runs-on: ${{ inputs.os }} timeout-minutes: 60 env: - OPENSSL_VER: 3.0.15 + OPENSSL_VER: 3.5.7 PYTHONSTRICTEXTENSIONBUILD: 1 TERM: linux steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Register gcc problem matcher @@ -43,9 +43,11 @@ jobs: run: sudo ./.github/workflows/posix-deps-apt.sh - name: Install Clang and BOLT if: ${{ fromJSON(inputs.bolt-optimizations) }} + # For BOLT jobs, https://apt.llvm.org/llvm.sh doesn't support LLVM 19 + # on Ubuntu 26.04, so stick to Ubuntu 24.04 until LLVM is upgraded. run: | sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh 19 - sudo apt-get install bolt-19 + sudo apt-get install --no-install-recommends bolt-19 echo PATH="$(llvm-config-19 --bindir):$PATH" >> $GITHUB_ENV - name: Configure OpenSSL env vars run: | @@ -54,21 +56,13 @@ jobs: echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> "$GITHUB_ENV" - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v4 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ inputs.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' run: python3 Tools/ssl/multissltests.py --steps=library --base-directory "$MULTISSL_DIR" --openssl "$OPENSSL_VER" --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - name: Setup directory envs for out-of-tree builds run: | echo "CPYTHON_RO_SRCDIR=$(realpath -m "${GITHUB_WORKSPACE}"/../cpython-ro-srcdir)" >> "$GITHUB_ENV" @@ -79,11 +73,6 @@ jobs: run: sudo mount --bind -o ro "$GITHUB_WORKSPACE" "$CPYTHON_RO_SRCDIR" - name: Runner image version run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: Restore config.cache - uses: actions/cache@v4 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ inputs.config_hash }} - name: Configure CPython out-of-tree working-directory: ${{ env.CPYTHON_BUILDDIR }} # `test_unpickle_module_race` writes to the source directory, which is diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 6beb91e66d40273..bd08c120fd00ee2 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -2,10 +2,9 @@ name: Reusable WASI on: workflow_call: - inputs: - config_hash: - required: true - type: string + +permissions: + contents: read env: FORCE_COLOR: 1 @@ -13,26 +12,25 @@ env: jobs: build-wasi-reusable: name: 'build and test' - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 60 env: - WASMTIME_VERSION: 22.0.0 + WASMTIME_VERSION: 38.0.3 WASI_SDK_VERSION: 24 WASI_SDK_PATH: /opt/wasi-sdk - CROSS_BUILD_PYTHON: cross-build/build CROSS_BUILD_WASI: cross-build/wasm32-wasip1 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false # No problem resolver registered as one doesn't currently exist for Clang. - name: "Install wasmtime" - uses: bytecodealliance/actions/wasmtime/setup@v1 + uses: bytecodealliance/actions/wasmtime/setup@9152e710e9f7182e4c29ad218e4f335a7b203613 # v1.1.3 with: version: ${{ env.WASMTIME_VERSION }} - name: "Restore WASI SDK" id: cache-wasi-sdk - uses: actions/cache@v4 + uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: path: ${{ env.WASI_SDK_PATH }} key: ${{ runner.os }}-wasi-sdk-${{ env.WASI_SDK_VERSION }} @@ -42,43 +40,24 @@ jobs: mkdir "${WASI_SDK_PATH}" && \ curl -s -S --location "https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz" | \ tar --strip-components 1 --directory "${WASI_SDK_PATH}" --extract --gunzip - - name: "Configure ccache action" - uses: hendrikmuhs/ccache-action@v1.2 - with: - save: ${{ github.event_name == 'push' }} - max-size: "200M" - - name: "Add ccache to PATH" - run: echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV" - name: "Install Python" - uses: actions/setup-python@v5 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.x' - name: "Runner image version" run: echo "IMAGE_OS_VERSION=${ImageOS}-${ImageVersion}" >> "$GITHUB_ENV" - - name: "Restore Python build config.cache" - uses: actions/cache@v4 - with: - path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache - # Include env.pythonLocation in key to avoid changes in environment when setup-python updates Python. - # Include the hash of `Tools/wasm/wasi.py` as it may change the environment variables. - # (Make sure to keep the key in sync with the other config.cache step below.) - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} - name: "Configure build Python" - run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug + run: python3 Tools/wasm/wasi configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" - run: python3 Tools/wasm/wasi.py make-build-python - - name: "Restore host config.cache" - uses: actions/cache@v4 - with: - path: ${{ env.CROSS_BUILD_WASI }}/config.cache - # Should be kept in sync with the other config.cache step above. - key: ${{ github.job }}-${{ env.IMAGE_OS_VERSION }}-${{ env.WASI_SDK_VERSION }}-${{ env.WASMTIME_VERSION }}-${{ inputs.config_hash }}-${{ hashFiles('Tools/wasm/wasi.py') }}-${{ env.pythonLocation }} - - name: "Configure host" + run: python3 Tools/wasm/wasi make-build-python + - name: "Display build info of the build Python" + run: python3 Tools/wasm/wasi pythoninfo-build + - name: "Configure host/WASI Python" # `--with-pydebug` inferred from configure-build-python - run: python3 Tools/wasm/wasi.py configure-host -- --config-cache - - name: "Make host" - run: python3 Tools/wasm/wasi.py make-host - - name: "Display build info" - run: make --directory "${CROSS_BUILD_WASI}" pythoninfo + run: python3 Tools/wasm/wasi configure-host -- --config-cache + - name: "Make host/WASI Python" + run: python3 Tools/wasm/wasi make-host + - name: "Display build info of the host/WASI Python" + run: python3 Tools/wasm/wasi pythoninfo-host - name: "Test" run: make --directory "${CROSS_BUILD_WASI}" test diff --git a/.github/workflows/reusable-windows-msi.yml b/.github/workflows/reusable-windows-msi.yml index a50de344bba4dac..420c9cd909a5e99 100644 --- a/.github/workflows/reusable-windows-msi.yml +++ b/.github/workflows/reusable-windows-msi.yml @@ -17,13 +17,13 @@ env: jobs: build: name: installer for ${{ inputs.arch }} - runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-latest' }} + runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2022' }} timeout-minutes: 60 env: ARCH: ${{ inputs.arch }} IncludeFreethreaded: true steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Build CPython installer diff --git a/.github/workflows/reusable-windows.yml b/.github/workflows/reusable-windows.yml index 37c802095b0f2fa..138e6846cb90391 100644 --- a/.github/workflows/reusable-windows.yml +++ b/.github/workflows/reusable-windows.yml @@ -13,6 +13,9 @@ on: type: boolean default: false +permissions: + contents: read + env: FORCE_COLOR: 1 IncludeUwp: >- @@ -21,12 +24,12 @@ env: jobs: build: name: Build and test (${{ inputs.arch }}) - runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-latest' }} + runs-on: ${{ inputs.arch == 'arm64' && 'windows-11-arm' || 'windows-2022' }} timeout-minutes: 60 env: ARCH: ${{ inputs.arch }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - name: Register MSVC problem matcher diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index febb2dd823a8fe2..56193a901ce0762 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -4,17 +4,20 @@ on: schedule: - cron: "0 */6 * * *" +permissions: + contents: read + jobs: stale: if: github.repository_owner == 'python' - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: pull-requests: write timeout-minutes: 10 steps: - name: "Check PRs" - uses: actions/stale@v9 + uses: actions/stale@5bef64f19d7facfb25b37b414482c7164d639639 # v9.1.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} stale-pr-message: 'This PR is stale because it has been open for 30 days with no activity.' diff --git a/.github/workflows/tail-call.yml b/.github/workflows/tail-call.yml index 4636372e26c41be..8fd63d7a6f89882 100644 --- a/.github/workflows/tail-call.yml +++ b/.github/workflows/tail-call.yml @@ -1,19 +1,14 @@ name: Tail calling interpreter on: pull_request: - paths: + paths: &paths - '.github/workflows/tail-call.yml' - 'Python/bytecodes.c' - 'Python/ceval.c' - 'Python/ceval_macros.h' - 'Python/generated_cases.c.h' push: - paths: - - '.github/workflows/tail-call.yml' - - 'Python/bytecodes.c' - - 'Python/ceval.c' - - 'Python/ceval_macros.h' - - 'Python/generated_cases.c.h' + paths: *paths workflow_dispatch: permissions: @@ -25,116 +20,106 @@ concurrency: env: FORCE_COLOR: 1 + LLVM_VERSION: 20 jobs: - tail-call: + windows: name: ${{ matrix.target }} runs-on: ${{ matrix.runner }} - timeout-minutes: 90 + timeout-minutes: 60 strategy: fail-fast: false matrix: - target: -# Un-comment as we add support for more platforms for tail-calling interpreters. -# - i686-pc-windows-msvc/msvc - - x86_64-pc-windows-msvc/msvc -# - aarch64-pc-windows-msvc/msvc - - x86_64-apple-darwin/clang - - aarch64-apple-darwin/clang - - x86_64-unknown-linux-gnu/gcc - - aarch64-unknown-linux-gnu/gcc - - free-threading - llvm: - - 20 include: -# - target: i686-pc-windows-msvc/msvc -# architecture: Win32 -# runner: windows-latest - target: x86_64-pc-windows-msvc/msvc architecture: x64 - runner: windows-latest -# - target: aarch64-pc-windows-msvc/msvc -# architecture: ARM64 -# runner: windows-latest - - target: x86_64-apple-darwin/clang - architecture: x86_64 - runner: macos-13 - - target: aarch64-apple-darwin/clang - architecture: aarch64 - runner: macos-14 - - target: x86_64-unknown-linux-gnu/gcc - architecture: x86_64 - runner: ubuntu-24.04 - - target: aarch64-unknown-linux-gnu/gcc - architecture: aarch64 - runner: ubuntu-24.04-arm - - target: free-threading - architecture: x86_64 - runner: ubuntu-24.04 + runner: windows-2022 + build_flags: "" + run_tests: true steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.11' - - - name: Native Windows (debug) - if: runner.os == 'Windows' && matrix.architecture != 'ARM64' - shell: cmd + - name: Build + shell: pwsh run: | - choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 - set PlatformToolset=clangcl - set LLVMToolsVersion=${{ matrix.llvm }}.1.0 - set LLVMInstallDir=C:\Program Files\LLVM - call ./PCbuild/build.bat --tail-call-interp -d -p ${{ matrix.architecture }} - call ./PCbuild/rt.bat -d -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - # No tests (yet): - - name: Emulated Windows (release) - if: runner.os == 'Windows' && matrix.architecture == 'ARM64' - shell: cmd + choco install llvm --allow-downgrade --no-progress --version ${{ env.LLVM_VERSION }}.1.0 + $env:PlatformToolset = "clangcl" + $env:LLVMToolsVersion = "${{ env.LLVM_VERSION }}.1.0" + $env:LLVMInstallDir = "C:\Program Files\LLVM" + ./PCbuild/build.bat --tail-call-interp ${{ matrix.build_flags }} -c Release -p ${{ matrix.architecture }} + - name: Test + if: matrix.run_tests + shell: pwsh run: | - choco install llvm --allow-downgrade --no-progress --version ${{ matrix.llvm }}.1.0 - set PlatformToolset=clangcl - set LLVMToolsVersion=${{ matrix.llvm }}.1.0 - set LLVMInstallDir=C:\Program Files\LLVM - ./PCbuild/build.bat --tail-call-interp -p ${{ matrix.architecture }} + ./PCbuild/rt.bat -p ${{ matrix.architecture }} -q --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - # The `find` line is required as a result of https://github.com/actions/runner-images/issues/9966. - # This is a bug in the macOS runner image where the pre-installed Python is installed in the same - # directory as the Homebrew Python, which causes the build to fail for macos-13. This line removes - # the symlink to the pre-installed Python so that the Homebrew Python is used instead. - # Note: when a new LLVM is released, the homebrew installation directory changes, so the builds will fail. - # We either need to upgrade LLVM or change the directory being pointed to. - - name: Native macOS (release) - if: runner.os == 'macOS' + macos: + name: ${{ matrix.target }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-apple-darwin/clang + runner: macos-26-intel + - target: aarch64-apple-darwin/clang + runner: macos-26 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - name: Install dependencies run: | brew update - find /usr/local/bin -lname '*/Library/Frameworks/Python.framework/*' -delete - brew install llvm@${{ matrix.llvm }} + brew install llvm@${{ env.LLVM_VERSION }} + - name: Build + run: | export SDKROOT="$(xcrun --show-sdk-path)" - export PATH="/usr/local/opt/llvm/bin:$PATH" - export PATH="/opt/homebrew/opt/llvm/bin:$PATH" - CC=clang-20 ./configure --with-tail-call-interp + export PATH="/usr/local/opt/llvm@${{ env.LLVM_VERSION }}/bin:$PATH" + export PATH="/opt/homebrew/opt/llvm@${{ env.LLVM_VERSION }}/bin:$PATH" + CC=clang-${{ env.LLVM_VERSION }} ./configure --with-tail-call-interp make all --jobs 4 + - name: Test + run: | ./python.exe -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - name: Native Linux (debug) - if: runner.os == 'Linux' && matrix.target != 'free-threading' + linux: + name: ${{ matrix.target }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-unknown-linux-gnu/gcc + runner: ubuntu-26.04 + configure_flags: --with-pydebug + - target: x86_64-unknown-linux-gnu/gcc-free-threading + runner: ubuntu-26.04 + configure_flags: --disable-gil + - target: aarch64-unknown-linux-gnu/gcc + runner: ubuntu-26.04-arm + configure_flags: --with-pydebug + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.11' + - name: Build run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - CC=clang-20 ./configure --with-tail-call-interp --with-pydebug + # On ubuntu-26.04 image, clang is clang-21 by default + CC=clang-${{ env.LLVM_VERSION }} ./configure --with-tail-call-interp ${{ matrix.configure_flags }} make all --jobs 4 - ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - - - name: Native Linux with free-threading (release) - if: matrix.target == 'free-threading' + - name: Test run: | - sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)" ./llvm.sh ${{ matrix.llvm }} - export PATH="$(llvm-config-${{ matrix.llvm }} --bindir):$PATH" - CC=clang-20 ./configure --with-tail-call-interp --disable-gil - make all --jobs 4 ./python -m test --multiprocess 0 --timeout 4500 --verbose2 --verbose3 - diff --git a/.github/workflows/verify-ensurepip-wheels.yml b/.github/workflows/verify-ensurepip-wheels.yml index 463e7bf3355cc39..cb40f6abc0b3b75 100644 --- a/.github/workflows/verify-ensurepip-wheels.yml +++ b/.github/workflows/verify-ensurepip-wheels.yml @@ -25,10 +25,10 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - uses: actions/setup-python@v5 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3' - name: Compare checksum of bundled wheels to the ones published on PyPI diff --git a/.github/workflows/verify-expat.yml b/.github/workflows/verify-expat.yml new file mode 100644 index 000000000000000..158e8f10377cadc --- /dev/null +++ b/.github/workflows/verify-expat.yml @@ -0,0 +1,32 @@ +name: Verify bundled libexpat + +on: + workflow_dispatch: + push: + paths: + - 'Modules/expat/**' + - '.github/workflows/verify-expat.yml' + pull_request: + paths: + - 'Modules/expat/**' + - '.github/workflows/verify-expat.yml' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + verify: + runs-on: ubuntu-slim + timeout-minutes: 5 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Download and verify bundled libexpat files + run: | + ./Modules/expat/refresh.sh + git diff --exit-code Modules/expat/ diff --git a/.github/zizmor.yml b/.github/zizmor.yml index 9b42b47cc855456..7c776d5ea1f941a 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -1,10 +1,6 @@ -# Configuration for the zizmor static analysis tool, run via pre-commit in CI -# https://woodruffw.github.io/zizmor/configuration/ +# Configuration for the zizmor static analysis tool, run via prek in CI +# https://docs.zizmor.sh/configuration/ rules: dangerous-triggers: ignore: - documentation-links.yml - unpinned-uses: - config: - policies: - "*": ref-pin diff --git a/.gitignore b/.gitignore index 2a6f249275c32e2..67043c33d1b6e58 100644 --- a/.gitignore +++ b/.gitignore @@ -71,16 +71,15 @@ Lib/test/data/* /Makefile /Makefile.pre /iOSTestbed.* -iOS/Frameworks/ -iOS/Resources/Info.plist -iOS/testbed/build -iOS/testbed/Python.xcframework/ios-*/bin -iOS/testbed/Python.xcframework/ios-*/include -iOS/testbed/Python.xcframework/ios-*/lib -iOS/testbed/Python.xcframework/ios-*/Python.framework -iOS/testbed/iOSTestbed.xcodeproj/project.xcworkspace -iOS/testbed/iOSTestbed.xcodeproj/xcuserdata -iOS/testbed/iOSTestbed.xcodeproj/xcshareddata +Apple/iOS/Frameworks/ +Apple/iOS/Resources/Info.plist +Apple/testbed/build +Apple/testbed/Python.xcframework/*/bin +Apple/testbed/Python.xcframework/*/include +Apple/testbed/Python.xcframework/*/lib +Apple/testbed/Python.xcframework/*/Python.framework +Apple/testbed/*Testbed.xcodeproj/project.xcworkspace +Apple/testbed/*Testbed.xcodeproj/xcuserdata Mac/Makefile Mac/PythonLauncher/Info.plist Mac/PythonLauncher/Makefile @@ -135,10 +134,10 @@ Tools/unicode/data/ /config.log /config.status /config.status.lineno -# hendrikmuhs/ccache-action@v1 /.ccache -/cross-build/ +/cross-build*/ /jit_stencils*.h +/jit_unwind_info*.h /platform /profile-clean-stamp /profile-run-stamp @@ -171,5 +170,6 @@ Python/frozen_modules/MANIFEST /python !/Python/ -# main branch only: ABI files are not checked/maintained. -Doc/data/python*.abi +# People's custom https://docs.anthropic.com/en/docs/claude-code/memory configs. +/.claude/ +CLAUDE.local.md diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7ad829c94d50f37..8d65323022554d4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,41 +1,71 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.11.8 + rev: a27a2e47c7751b639d2b5badf0ef6ff11fee893f # frozen: v0.15.4 hooks: - - id: ruff + - id: ruff-check + name: Run Ruff (lint) on Apple/ + args: [--exit-non-zero-on-fix, --config=Apple/.ruff.toml] + files: ^Apple/ + - id: ruff-check name: Run Ruff (lint) on Doc/ args: [--exit-non-zero-on-fix] files: ^Doc/ - - id: ruff + - id: ruff-check name: Run Ruff (lint) on Lib/test/ args: [--exit-non-zero-on-fix] files: ^Lib/test/ - - id: ruff + - id: ruff-check name: Run Ruff (lint) on Tools/build/ args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml] files: ^Tools/build/ - - id: ruff + - id: ruff-check + name: Run Ruff (lint) on Tools/i18n/ + args: [--exit-non-zero-on-fix, --config=Tools/i18n/.ruff.toml] + files: ^Tools/i18n/ + - id: ruff-check name: Run Ruff (lint) on Argument Clinic args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml] files: ^Tools/clinic/|Lib/test/test_clinic.py + - id: ruff-check + name: Run Ruff (lint) on Tools/peg_generator/ + args: [--exit-non-zero-on-fix, --config=Tools/peg_generator/.ruff.toml] + files: ^Tools/peg_generator/ + - id: ruff-check + name: Run Ruff (lint) on Tools/wasm/ + args: [--exit-non-zero-on-fix, --config=Tools/wasm/.ruff.toml] + files: ^Tools/wasm/ + - id: ruff-format + name: Run Ruff (format) on Apple/ + args: [--exit-non-zero-on-fix, --config=Apple/.ruff.toml] + files: ^Apple - id: ruff-format name: Run Ruff (format) on Doc/ - args: [--check] + args: [--exit-non-zero-on-fix] files: ^Doc/ - id: ruff-format name: Run Ruff (format) on Tools/build/check_warnings.py - args: [--check, --config=Tools/build/.ruff.toml] + args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml] files: ^Tools/build/check_warnings.py + - id: ruff-format + name: Run Ruff (format) on Tools/wasm/ + args: [--exit-non-zero-on-fix, --config=Tools/wasm/.ruff.toml] + files: ^Tools/wasm/ - repo: https://github.com/psf/black-pre-commit-mirror - rev: 25.1.0 + rev: ea488cebbfd88a5f50b8bd95d5c829d0bb76feb8 # frozen: 26.1.0 hooks: - id: black name: Run Black on Tools/jit/ files: ^Tools/jit/ + - repo: https://github.com/Lucas-C/pre-commit-hooks + rev: ad1b27d73581aa16cca06fc4a0761fc563ffe8e8 # frozen: v1.5.6 + hooks: + - id: remove-tabs + types: [python] + - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: 3e8a8703264a2f4a69428a0aa4dcb512790b2c8c # frozen: v6.0.0 hooks: - id: check-case-conflict - id: check-merge-conflict @@ -43,30 +73,37 @@ repos: exclude: ^Lib/test/test_tomllib/ - id: check-yaml - id: end-of-file-fixer - types: [python] + types_or: [python, yaml] exclude: Lib/test/tokenizedata/coding20731.py + - id: end-of-file-fixer + files: '^\.github/CODEOWNERS$' + - id: mixed-line-ending + args: [--fix=auto] + exclude: '^Lib/test/.*data/' + - id: trailing-whitespace + types_or: [c, inc, python, rst, yaml] - id: trailing-whitespace - types_or: [c, inc, python, rst] + files: '^\.github/CODEOWNERS|\.(gram)$' - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.33.0 + rev: 9f48a48aa91a6040d749ad68ec70907d907a5a7f # frozen: 0.37.0 hooks: - id: check-dependabot - id: check-github-workflows - id: check-readthedocs - repo: https://github.com/rhysd/actionlint - rev: v1.7.7 + rev: 914e7df21a07ef503a81201c76d2b11c789d3fca # frozen: v1.7.12 hooks: - id: actionlint - - repo: https://github.com/woodruffw/zizmor-pre-commit - rev: v1.6.0 + - repo: https://github.com/zizmorcore/zizmor-pre-commit + rev: b546b77c44c466a54a42af5499dcc0dcc1a3193f # frozen: v1.22.0 hooks: - id: zizmor - repo: https://github.com/sphinx-contrib/sphinx-lint - rev: v1.0.0 + rev: c883505f64b59c3c5c9375191e4ad9f98e727ccd # frozen: v1.0.2 hooks: - id: sphinx-lint args: [--enable=default-role] diff --git a/.readthedocs.yml b/.readthedocs.yml index a57de00544e4e3e..038417e4bb34385 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -11,25 +11,50 @@ build: os: ubuntu-24.04 tools: python: "3" + apt_packages: + - jq - commands: - # https://docs.readthedocs.io/en/stable/build-customization.html#cancel-build-based-on-a-condition - # - # Cancel building pull requests when there aren't changes in the Doc directory. - # - # If there are no changes (git diff exits with 0) we force the command to return with 183. - # This is a special exit code on Read the Docs that will cancel the build immediately. - - | - if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && [ "$(git diff --quiet origin/main -- Doc/ .readthedocs.yml; echo $?)" -eq 0 ]; - then - echo "No changes to Doc/ - exiting the build."; - exit 183; - fi - - - asdf plugin add uv - - asdf install uv latest - - asdf global uv latest - - make -C Doc venv html - - mkdir _readthedocs - - mv Doc/build/html _readthedocs/html - + jobs: + post_system_dependencies: + # https://docs.readthedocs.com/platform/stable/guides/build/skip-build.html#skip-builds-based-on-conditions + # + # Cancel building pull requests when there are no changes in the Doc + # directory or RTD configuration, or if we can't cleanly merge the base + # branch. + - | + set -eEux; + if [ "$READTHEDOCS_VERSION_TYPE" = "external" ]; + then + base_branch=$(wget -qO- "https://api.github.com/repos/python/cpython/pulls/$READTHEDOCS_VERSION" | jq -er ".base.ref"); + git fetch --depth=50 origin $base_branch:origin-$base_branch; + for attempt in $(seq 10); + do + if ! git merge-base HEAD origin-$base_branch; + then + git fetch --deepen=50 origin $base_branch; + else + break; + fi; + done; + if ! git -c "user.name=rtd" -c "user.email=no-reply@readthedocs.org" merge --no-stat --no-edit origin-$base_branch; + then + echo "Unsuccessful merge with '$base_branch' branch, skipping the build"; + exit 183; + fi; + if git diff --exit-code --stat origin-$base_branch -- Doc/ .readthedocs.yml; + then + echo "No changes to Doc/ - skipping the build."; + exit 183; + fi; + fi; + create_environment: + - echo "Skipping default environment creation" + install: + - asdf plugin add uv + - asdf install uv latest + - asdf global uv latest + build: + html: + - make -C Doc venv html + - mkdir -p "$READTHEDOCS_OUTPUT" + - mv Doc/build/html "$READTHEDOCS_OUTPUT/" diff --git a/Android/README.md b/Android/README.md index 6cabd6ba5d6844b..0474bdf1a79147c 100644 --- a/Android/README.md +++ b/Android/README.md @@ -31,7 +31,8 @@ any packages it needs. The script also requires the following commands to be on the `PATH`: * `curl` -* `java` (or set the `JAVA_HOME` environment variable) +* `java` (or set the `JAVA_HOME` environment variable). + Java versions 17 and 21 are supported. ## Building @@ -96,18 +97,12 @@ similar to the `Android` directory of the CPython source tree. ## Testing -The Python test suite can be run on Linux, macOS, or Windows: +The Python test suite can be run on Linux, macOS, or Windows. -* On Linux, the emulator needs access to the KVM virtualization interface, and - a DISPLAY environment variable pointing at an X server. Xvfb is acceptable. - -The test suite can usually be run on a device with 2 GB of RAM, but this is -borderline, so you may need to increase it to 4 GB. As of Android -Studio Koala, 2 GB is the default for all emulators, although the user interface -may indicate otherwise. Locate the emulator's directory under `~/.android/avd`, -and find `hw.ramSize` in both config.ini and hardware-qemu.ini. Either set these -manually to the same value, or use the Android Studio Device Manager, which will -update both files. +On Linux, the emulator needs access to the KVM virtualization interface. This may +require adding your user to a group, or changing your udev rules. On GitHub +Actions, the test script will do this automatically using the commands shown +[here](https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). You can run the test suite either: @@ -156,6 +151,10 @@ repository's `Lib` directory will be picked up immediately. Changes in C files, and architecture-specific files such as sysconfigdata, will not take effect until you re-run `android.py make-host` or `build`. +The testbed app can also be used to test third-party packages. For more details, +run `android.py test --help`, paying attention to the options `--site-packages`, +`--cwd`, `-c` and `-m`. + ## Using in your own app diff --git a/Android/android-env.sh b/Android/android-env.sh index bab4130c9e92d05..5859c0eac4a88fb 100644 --- a/Android/android-env.sh +++ b/Android/android-env.sh @@ -3,7 +3,7 @@ : "${HOST:?}" # GNU target triplet # You may also override the following: -: "${api_level:=24}" # Minimum Android API level the build will run on +: "${ANDROID_API_LEVEL:=24}" # Minimum Android API level the build will run on : "${PREFIX:-}" # Path in which to find required libraries @@ -24,7 +24,7 @@ fail() { # * https://android.googlesource.com/platform/ndk/+/ndk-rXX-release/docs/BuildSystemMaintainers.md # where XX is the NDK version. Do a diff against the version you're upgrading from, e.g.: # https://android.googlesource.com/platform/ndk/+/ndk-r25-release..ndk-r26-release/docs/BuildSystemMaintainers.md -ndk_version=27.1.12297006 +ndk_version=27.3.13750724 ndk=$ANDROID_HOME/ndk/$ndk_version if ! [ -e "$ndk" ]; then @@ -43,7 +43,7 @@ fi toolchain=$(echo "$ndk"/toolchains/llvm/prebuilt/*) export AR="$toolchain/bin/llvm-ar" export AS="$toolchain/bin/llvm-as" -export CC="$toolchain/bin/${clang_triplet}${api_level}-clang" +export CC="$toolchain/bin/${clang_triplet}${ANDROID_API_LEVEL}-clang" export CXX="${CC}++" export LD="$toolchain/bin/ld" export NM="$toolchain/bin/llvm-nm" diff --git a/Android/android.py b/Android/android.py index 3f48b42aa175717..3784886bff4ce79 100755 --- a/Android/android.py +++ b/Android/android.py @@ -2,7 +2,9 @@ import asyncio import argparse +import json import os +import platform import re import shlex import shutil @@ -13,8 +15,9 @@ from asyncio import wait_for from contextlib import asynccontextmanager from datetime import datetime, timezone +from enum import IntEnum, auto from glob import glob -from os.path import basename, relpath +from os.path import abspath, basename, relpath from pathlib import Path from subprocess import CalledProcessError from tempfile import TemporaryDirectory @@ -22,11 +25,21 @@ SCRIPT_NAME = Path(__file__).name ANDROID_DIR = Path(__file__).resolve().parent -CHECKOUT = ANDROID_DIR.parent -TESTBED_DIR = ANDROID_DIR / "testbed" -CROSS_BUILD_DIR = CHECKOUT / "cross-build" +PYTHON_DIR = ANDROID_DIR.parent +in_source_tree = ( + ANDROID_DIR.name == "Android" and (PYTHON_DIR / "pyconfig.h.in").exists() +) -HOSTS = ["aarch64-linux-android", "x86_64-linux-android"] +ENV_SCRIPT = ANDROID_DIR / "android-env.sh" +TESTBED_DIR = ANDROID_DIR / "testbed" +CROSS_BUILD_DIR = PYTHON_DIR / "cross-build" + +HOSTS = [ + "aarch64-linux-android", + "arm-linux-androideabi", + "i686-linux-android", + "x86_64-linux-android", +] APP_ID = "org.python.testbed" DECODE_ARGS = ("UTF-8", "backslashreplace") @@ -46,7 +59,32 @@ + (".bat" if os.name == "nt" else "") ) -logcat_started = False +# Whether we've seen any output from Python yet. +python_started = False + +# Buffer for verbose output which will be displayed only if a test fails and +# there has been no output from Python. +hidden_output = [] + + +# Based on android/log.h in the NDK. +class LogPriority(IntEnum): + UNKNOWN = 0 + DEFAULT = auto() + VERBOSE = auto() + DEBUG = auto() + INFO = auto() + WARN = auto() + ERROR = auto() + FATAL = auto() + SILENT = auto() + + +def log_verbose(context, line, stream=sys.stdout): + if context.verbose: + stream.write(line) + else: + hidden_output.append((stream, line)) def delete_glob(pattern): @@ -76,39 +114,67 @@ def run(command, *, host=None, env=None, log=True, **kwargs): kwargs.setdefault("check", True) if env is None: env = os.environ.copy() - original_env = env.copy() if host: - env_script = ANDROID_DIR / "android-env.sh" - env_output = subprocess.run( - f"set -eu; " - f"HOST={host}; " - f"PREFIX={subdir(host)}/prefix; " - f". {env_script}; " - f"export", - check=True, shell=True, text=True, stdout=subprocess.PIPE - ).stdout - - for line in env_output.splitlines(): - # We don't require every line to match, as there may be some other - # output from installing the NDK. - if match := re.search( - "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line - ): - key, value = match[2], match[3] - if env.get(key) != value: - print(line) - env[key] = value - - if env == original_env: - raise ValueError(f"Found no variables in {env_script.name} output:\n" - + env_output) + host_env = android_env(host) + print_env(host_env) + env.update(host_env) if log: - print(">", " ".join(map(str, command))) + print(">", join_command(command)) return subprocess.run(command, env=env, **kwargs) +# Format a command so it can be copied into a shell. Like shlex.join, but also +# accepts arguments which are Paths, or a single string/Path outside of a list. +def join_command(args): + if isinstance(args, (str, Path)): + return str(args) + else: + return shlex.join(map(str, args)) + + +# Format the environment so it can be pasted into a shell. +def print_env(env): + for key, value in sorted(env.items()): + print(f"export {key}={shlex.quote(value)}") + + +def android_env(host): + if host: + prefix = subdir(host) / "prefix" + else: + prefix = ANDROID_DIR / "prefix" + sysconfig_files = prefix.glob("lib/python*/_sysconfigdata__android_*.py") + sysconfig_filename = next(sysconfig_files).name + host = re.fullmatch(r"_sysconfigdata__android_(.+).py", sysconfig_filename)[1] + + env_output = subprocess.run( + f"set -eu; " + f"HOST={host}; " + f"PREFIX={prefix}; " + f". {ENV_SCRIPT}; " + f"export", + check=True, shell=True, capture_output=True, encoding='utf-8', + ).stdout + + env = {} + for line in env_output.splitlines(): + # We don't require every line to match, as there may be some other + # output from installing the NDK. + if match := re.search( + "^(declare -x |export )?(\\w+)=['\"]?(.*?)['\"]?$", line + ): + key, value = match[2], match[3] + if os.environ.get(key) != value: + env[key] = value + + if not env: + raise ValueError(f"Found no variables in {ENV_SCRIPT.name} output:\n" + + env_output) + return env + + def build_python_path(): """The path to the build Python binary.""" build_dir = subdir("build") @@ -127,7 +193,7 @@ def configure_build_python(context): clean("build") os.chdir(subdir("build", create=True)) - command = [relpath(CHECKOUT / "configure")] + command = [relpath(PYTHON_DIR / "configure")] if context.args: command.extend(context.args) run(command) @@ -138,37 +204,65 @@ def make_build_python(context): run(["make", "-j", str(os.cpu_count())]) -def unpack_deps(host, prefix_dir): +def pythoninfo_build_python(context): + os.chdir(subdir("build")) + run(["make", "pythoninfo"]) + + +# To create new builds of these dependencies, usually all that's necessary is to +# push a tag to the cpython-android-source-deps repository, and GitHub Actions +# will do the rest. +# +# If you're a member of the Python core team, and you'd like to be able to push +# these tags yourself, please contact Malcolm Smith or Russell Keith-Magee. +def unpack_deps(host, prefix_dir, cache_dir): + os.chdir(prefix_dir) deps_url = "https://github.com/beeware/cpython-android-source-deps/releases/download" - for name_ver in ["bzip2-1.0.8-2", "libffi-3.4.4-3", "openssl-3.0.15-4", - "sqlite-3.49.1-0", "xz-5.4.6-1"]: + for name_ver in [ + "bzip2-1.0.8-3", + "libffi-3.4.4-3", + "openssl-3.5.7-0", + "sqlite-3.50.4-0", + "xz-5.4.6-1", + "zstd-1.5.7-2" + ]: filename = f"{name_ver}-{host}.tar.gz" - download(f"{deps_url}/{name_ver}/{filename}") - shutil.unpack_archive(filename, prefix_dir) - os.remove(filename) + out_path = download(f"{deps_url}/{name_ver}/{filename}", cache_dir) + shutil.unpack_archive(out_path) -def download(url, target_dir="."): - out_path = f"{target_dir}/{basename(url)}" - run(["curl", "-Lf", "--retry", "5", "--retry-all-errors", "-o", out_path, url]) +def download(url, cache_dir): + out_path = cache_dir / basename(url) + cache_dir.mkdir(parents=True, exist_ok=True) + if not out_path.is_file(): + run(["curl", "-Lf", "--retry", "5", "--retry-all-errors", "-o", out_path, url]) + else: + print(f"Using cached version of {basename(url)}") return out_path -def configure_host_python(context): +def configure_host_python(context, host=None): + if host is None: + host = context.host if context.clean: - clean(context.host) + clean(host) - host_dir = subdir(context.host, create=True) + host_dir = subdir(host, create=True) prefix_dir = host_dir / "prefix" if not prefix_dir.exists(): prefix_dir.mkdir() - unpack_deps(context.host, prefix_dir) + cache_dir = ( + Path(context.cache_dir).resolve() + if context.cache_dir + else CROSS_BUILD_DIR / "downloads" + ) + unpack_deps(host, prefix_dir, cache_dir) os.chdir(host_dir) command = [ # Basic cross-compiling configuration - relpath(CHECKOUT / "configure"), - f"--host={context.host}", + relpath(PYTHON_DIR / "configure"), + f"--host={host}", f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", f"--with-build-python={build_python_path()}", "--without-ensurepip", @@ -184,37 +278,84 @@ def configure_host_python(context): if context.args: command.extend(context.args) - run(command, host=context.host) + run(command, host=host) -def make_host_python(context): +def make_host_python(context, host=None): + if host is None: + host = context.host # The CFLAGS and LDFLAGS set in android-env include the prefix dir, so # delete any previous Python installation to prevent it being used during # the build. - host_dir = subdir(context.host) + host_dir = subdir(host) prefix_dir = host_dir / "prefix" for pattern in ("include/python*", "lib/libpython*", "lib/python*"): delete_glob(f"{prefix_dir}/{pattern}") + # The Android environment variables were already captured in the Makefile by + # `configure`, and passing them again when running `make` may cause some + # flags to be duplicated. So we don't use the `host` argument here. os.chdir(host_dir) - run(["make", "-j", str(os.cpu_count())], host=context.host) - run(["make", "install", f"prefix={prefix_dir}"], host=context.host) + run(["make", "-j", str(os.cpu_count())]) + + # The `make install` output is very verbose and rarely useful, so + # suppress it by default. + run( + ["make", "install", f"prefix={prefix_dir}"], + capture_output=not context.verbose, + ) + +def build_targets(context): + if context.target in {"all", "build"}: + configure_build_python(context) + make_build_python(context) + pythoninfo_build_python(context) -def build_all(context): - steps = [configure_build_python, make_build_python, configure_host_python, - make_host_python] - for step in steps: - step(context) + for host in HOSTS: + if context.target in {"all", "hosts", host}: + configure_host_python(context, host) + make_host_python(context, host) def clean(host): delete_glob(CROSS_BUILD_DIR / host) -def clean_all(context): - for host in HOSTS + ["build"]: - clean(host) +def clean_targets(context): + if context.target in {"all", "build"}: + clean("build") + + for host in HOSTS: + if context.target in {"all", "hosts", host}: + clean(host) + + +def setup_ci(): + if "GITHUB_ACTIONS" in os.environ: + # Enable emulator hardware acceleration + # (https://github.blog/changelog/2024-04-02-github-actions-hardware-accelerated-android-virtualization-now-available/). + if platform.system() == "Linux": + run( + ["sudo", "tee", "/etc/udev/rules.d/99-kvm4all.rules"], + input='KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"\n', + text=True, + ) + run(["sudo", "udevadm", "control", "--reload-rules"]) + run(["sudo", "udevadm", "trigger", "--name-match=kvm"]) + + # Free up disk space by deleting unused versions of the NDK + # (https://github.com/freakboy3742/pyspamsum/pull/108). + for line in ENV_SCRIPT.read_text().splitlines(): + if match := re.fullmatch(r"ndk_version=(.+)", line): + ndk_version = match[1] + break + else: + raise ValueError(f"Failed to find NDK version in {ENV_SCRIPT.name}") + + for item in (android_home / "ndk").iterdir(): + if item.name[0].isdigit() and item.name != ndk_version: + delete_glob(item) def setup_sdk(): @@ -228,7 +369,12 @@ def setup_sdk(): if not all((android_home / "licenses" / path).exists() for path in [ "android-sdk-arm-dbt-license", "android-sdk-license" ]): - run([sdkmanager, "--licenses"], text=True, input="y\n" * 100) + run( + [sdkmanager, "--licenses"], + text=True, + capture_output=True, + input="y\n" * 100, + ) # Gradle may install this automatically, but we can't rely on that because # we need to run adb within the logcat task. @@ -259,17 +405,6 @@ def setup_testbed(): os.chmod(out_path, 0o755) -# run_testbed will build the app automatically, but it's useful to have this as -# a separate command to allow running the app outside of this script. -def build_testbed(context): - setup_sdk() - setup_testbed() - run( - [gradlew, "--console", "plain", "packageDebug", "packageDebugAndroidTest"], - cwd=TESTBED_DIR, - ) - - # Work around a bug involving sys.exit and TaskGroups # (https://github.com/python/cpython/issues/101515). def exit(*args): @@ -410,19 +545,23 @@ async def logcat_task(context, initial_devices): pid = await wait_for(find_pid(serial), startup_timeout) # `--pid` requires API level 24 or higher. - args = [adb, "-s", serial, "logcat", "--pid", pid, "--format", "tag"] - hidden_output = [] + # + # `--binary` mode is used in order to detect which messages end with a + # newline, which most of the other modes don't indicate (except `--format + # long`). For example, every time pytest runs a test, it prints a "." and + # flushes the stream. Each "." becomes a separate log message, but we should + # show them all on the same line. + args = [adb, "-s", serial, "logcat", "--pid", pid, "--binary"] + logcat_started = False async with async_process( - *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + *args, stdout=subprocess.PIPE, stderr=None ) as process: - while line := (await process.stdout.readline()).decode(*DECODE_ARGS): - if match := re.fullmatch(r"([A-Z])/(.*)", line, re.DOTALL): - level, message = match.groups() - else: - # If the regex doesn't match, this is probably the second or - # subsequent line of a multi-line message. Python won't produce - # such messages, but other components might. - level, message = None, line + while True: + try: + priority, tag, message = await read_logcat(process.stdout) + logcat_started = True + except asyncio.IncompleteReadError: + break # Exclude high-volume messages which are rarely useful. if context.verbose < 2 and "from python test_syslog" in message: @@ -430,36 +569,69 @@ async def logcat_task(context, initial_devices): # Put high-level messages on stderr so they're highlighted in the # buildbot logs. This will include Python's own stderr. - stream = ( - sys.stderr - if level in ["W", "E", "F"] # WARNING, ERROR, FATAL (aka ASSERT) - else sys.stdout - ) - - # To simplify automated processing of the output, e.g. a buildbot - # posting a failure notice on a GitHub PR, we strip the level and - # tag indicators from Python's stdout and stderr. - for prefix in ["python.stdout: ", "python.stderr: "]: - if message.startswith(prefix): - global logcat_started - logcat_started = True - stream.write(message.removeprefix(prefix)) - break + stream = sys.stderr if priority >= LogPriority.WARN else sys.stdout + + # The app's stdout and stderr should be passed through transparently + # to our own corresponding streams. + if tag in ["python.stdout", "python.stderr"]: + global python_started + python_started = True + stream.write(message) + stream.flush() else: - if context.verbose: - # Non-Python messages add a lot of noise, but they may - # sometimes help explain a failure. - stream.write(line) - else: - hidden_output.append(line) + # Non-Python messages add a lot of noise, but they may + # sometimes help explain a failure. Format them in the same way + # as `logcat --format tag`. + formatted = f"{priority.name[0]}/{tag}: {message}" + if not formatted.endswith("\n"): + formatted += "\n" + log_verbose(context, formatted, stream) # If the device disconnects while logcat is running, which always # happens in --managed mode, some versions of adb return non-zero. # Distinguish this from a logcat startup error by checking whether we've - # received a message from Python yet. + # received any logcat messages yet. status = await wait_for(process.wait(), timeout=1) if status != 0 and not logcat_started: - raise CalledProcessError(status, args, "".join(hidden_output)) + raise CalledProcessError(status, args) + + +# Read one binary log message from the given StreamReader. The message format is +# described at https://android.stackexchange.com/a/74660. All supported versions +# of Android use format version 2 or later. +async def read_logcat(stream): + async def read_bytes(size): + return await stream.readexactly(size) + + async def read_int(size): + return int.from_bytes(await read_bytes(size), "little") + + payload_len = await read_int(2) + if payload_len < 2: + # 1 byte for priority, 1 byte for null terminator of tag. + raise ValueError(f"payload length {payload_len} is too short") + + header_len = await read_int(2) + if header_len < 4: + raise ValueError(f"header length {header_len} is too short") + await read_bytes(header_len - 4) # Ignore other header fields. + + priority_int = await read_int(1) + try: + priority = LogPriority(priority_int) + except ValueError: + priority = LogPriority.UNKNOWN + + payload_fields = (await read_bytes(payload_len - 1)).split(b"\0") + if len(payload_fields) < 2: + raise ValueError( + f"payload {payload!r} does not contain at least 2 " + f"null-separated fields" + ) + tag, message, *_ = [ + field.decode(*DECODE_ARGS) for field in payload_fields + ] + return priority, tag, message def stop_app(serial): @@ -474,12 +646,42 @@ async def gradle_task(context): task_prefix = "connected" env["ANDROID_SERIAL"] = context.connected + # Ensure that CROSS_BUILD_DIR is in the Gradle environment, regardless + # of whether it was set by environment variable or `--cross-build-dir`. + env["CROSS_BUILD_DIR"] = CROSS_BUILD_DIR + + if context.ci_mode: + context.args[0:0] = [ + # See _add_ci_python_opts in libregrtest/main.py. + "-W", "error", "-bb", "-E", + + # Randomization is disabled because order-dependent failures are + # much less likely to pass on a rerun in single-process mode. + "-m", "test", + f"--{context.ci_mode}-ci", "--single-process", "--no-randomize" + ] + + if not any(arg in context.args for arg in ["-c", "-m"]): + context.args[0:0] = ["-m", "test"] + args = [ gradlew, "--console", "plain", f"{task_prefix}DebugAndroidTest", - "-Pandroid.testInstrumentationRunnerArguments.pythonArgs=" - + shlex.join(context.args), + ] + [ + f"-P{name}={value}" + for name, value in [ + ("python.sitePackages", context.site_packages), + ("python.cwd", context.cwd), + ( + "android.testInstrumentationRunnerArguments.pythonArgs", + json.dumps(context.args), + ), + ] + if value ] - hidden_output = [] + if context.verbose >= 2: + args.append("--info") + log_verbose(context, f"> {join_command(args)}\n") + try: async with async_process( *args, cwd=TESTBED_DIR, env=env, @@ -488,10 +690,10 @@ async def gradle_task(context): while line := (await process.stdout.readline()).decode(*DECODE_ARGS): # Gradle may take several minutes to install SDK packages, so # it's worth showing those messages even in non-verbose mode. - if context.verbose or line.startswith('Preparing "Install'): + if line.startswith('Preparing "Install'): sys.stdout.write(line) else: - hidden_output.append(line) + log_verbose(context, line) status = await wait_for(process.wait(), timeout=1) if status == 0: @@ -499,17 +701,13 @@ async def gradle_task(context): else: raise CalledProcessError(status, args) finally: - # If logcat never started, then something has gone badly wrong, so the - # user probably wants to see the Gradle output even in non-verbose mode. - if hidden_output and not logcat_started: - sys.stdout.write("".join(hidden_output)) - # Gradle does not stop the tests when interrupted. if context.connected: stop_app(context.connected) async def run_testbed(context): + setup_ci() setup_sdk() setup_testbed() @@ -533,6 +731,12 @@ async def run_testbed(context): except* MySystemExit as e: raise SystemExit(*e.exceptions[0].args) from None except* CalledProcessError as e: + # If Python produced no output, then the user probably wants to see the + # verbose output to explain why the test failed. + if not python_started: + for stream, line in hidden_output: + stream.write(line) + # Extract it from the ExceptionGroup so it can be handled by `main`. raise e.exceptions[0] @@ -597,11 +801,65 @@ def package(context): else: shutil.copy2(src, dst, follow_symlinks=False) + # Strip debug information. + if not context.debug: + so_files = glob(f"{temp_dir}/**/*.so", recursive=True) + run([android_env(context.host)["STRIP"], *so_files], log=False) + dist_dir = subdir(context.host, "dist", create=True) package_path = shutil.make_archive( f"{dist_dir}/python-{version}-{context.host}", "gztar", temp_dir ) print(f"Wrote {package_path}") + return package_path + + +def ci(context): + for step in [ + configure_build_python, + make_build_python, + pythoninfo_build_python, + configure_host_python, + make_host_python, + package, + ]: + caption = ( + step.__name__.replace("_", " ") + .capitalize() + .replace("python", "Python") + ) + print(f"::group::{caption}") + result = step(context) + if step is package: + package_path = result + print("::endgroup::") + + if ( + "GITHUB_ACTIONS" in os.environ + and (platform.system(), platform.machine()) != ("Linux", "x86_64") + ): + print( + "Skipping tests: GitHub Actions does not support the Android " + "emulator on this platform." + ) + else: + with TemporaryDirectory(prefix=SCRIPT_NAME) as temp_dir: + print("::group::Tests") + + # Prove the package is self-contained by using it to run the tests. + shutil.unpack_archive(package_path, temp_dir) + launcher_args = [ + "--managed", "maxVersion", "-v", f"--{context.ci_mode}-ci" + ] + run( + ["./android.py", "test", *launcher_args], + cwd=temp_dir + ) + print("::endgroup::") + + +def env(context): + print_env(android_env(getattr(context, "host", None))) # Handle SIGTERM the same way as SIGINT. This ensures that if we're terminated @@ -615,45 +873,96 @@ def signal_handler(*args): def parse_args(): parser = argparse.ArgumentParser() - subcommands = parser.add_subparsers(dest="subcommand") + subcommands = parser.add_subparsers(dest="subcommand", required=True) + + def add_parser(*args, **kwargs): + parser = subcommands.add_parser(*args, **kwargs) + parser.add_argument( + "--cross-build-dir", + action="store", + default=os.environ.get("CROSS_BUILD_DIR"), + dest="cross_build_dir", + type=Path, + help=( + "Path to the cross-build directory " + f"(default: {CROSS_BUILD_DIR}). Can also be set " + "with the CROSS_BUILD_DIR environment variable." + ), + ) + parser.add_argument( + "-v", "--verbose", action="count", default=0, + help="Show verbose output. Use twice to be even more verbose.") + return parser # Subcommands - build = subcommands.add_parser("build", help="Build everything") - configure_build = subcommands.add_parser("configure-build", - help="Run `configure` for the " - "build Python") - make_build = subcommands.add_parser("make-build", - help="Run `make` for the build Python") - configure_host = subcommands.add_parser("configure-host", - help="Run `configure` for Android") - make_host = subcommands.add_parser("make-host", - help="Run `make` for Android") - subcommands.add_parser( - "clean", help="Delete all build and prefix directories") - subcommands.add_parser( - "build-testbed", help="Build the testbed app") - test = subcommands.add_parser( - "test", help="Run the test suite") - package = subcommands.add_parser("package", help="Make a release package") + build = add_parser( + "build", + help="Run configure and make for the selected target" + ) + configure_build = add_parser( + "configure-build", help="Run `configure` for the build Python") + add_parser( + "make-build", help="Run `make` for the build Python") + add_parser( + "pythoninfo-build", help="Display build info of the build Python") + configure_host = add_parser( + "configure-host", help="Run `configure` for Android") + make_host = add_parser( + "make-host", help="Run `make` for Android") + + clean = add_parser( + "clean", + help="Delete build directories for the selected target" + ) + + test = add_parser("test", help="Run the testbed app") + package = add_parser("package", help="Make a release package") + ci = add_parser("ci", help="Run build, package and test") + env = add_parser("env", help="Print environment variables") # Common arguments - for subcommand in build, configure_build, configure_host: + # --cache-dir option + for cmd in [configure_host, build, ci]: + cmd.add_argument( + "--cache-dir", + default=os.environ.get("CACHE_DIR"), + help="The directory to store cached downloads.", + ) + + # --clean option + for subcommand in [build, configure_build, configure_host, ci]: subcommand.add_argument( "--clean", action="store_true", default=False, dest="clean", - help="Delete the relevant build and prefix directories first") - for subcommand in [build, configure_host, make_host, package]: + help="Delete the relevant build directories first") + + # Allow "all", "build" and "hosts" targets for some commands + for subcommand in [clean, build]: + subcommand.add_argument( + "target", + nargs="?", + default="all", + choices=["all", "build", "hosts"] + HOSTS, + help=( + "The host triplet (e.g., aarch64-linux-android), " + "or 'build' for just the build platform, or 'hosts' for all " + "host platforms, or 'all' for the build platform and all " + "hosts. Defaults to 'all'" + ), + ) + + host_commands = [configure_host, make_host, package, ci] + if in_source_tree: + host_commands.append(env) + for subcommand in host_commands: subcommand.add_argument( "host", metavar="HOST", choices=HOSTS, help="Host triplet: choices=[%(choices)s]") - for subcommand in build, configure_build, configure_host: + + for subcommand in [build, configure_build, configure_host, ci]: subcommand.add_argument("args", nargs="*", help="Extra arguments to pass to `configure`") # Test arguments - test.add_argument( - "-v", "--verbose", action="count", default=0, - help="Show Gradle output, and non-Python logcat messages. " - "Use twice to include high-volume messages which are rarely useful.") device_group = test.add_mutually_exclusive_group(required=True) device_group.add_argument( "--connected", metavar="SERIAL", help="Run on a connected device. " @@ -661,9 +970,34 @@ def parse_args(): device_group.add_argument( "--managed", metavar="NAME", help="Run on a Gradle-managed device. " "These are defined in `managedDevices` in testbed/app/build.gradle.kts.") + + test.add_argument( + "--site-packages", metavar="DIR", type=abspath, + help="Directory to copy as the app's site-packages.") + test.add_argument( + "--cwd", metavar="DIR", type=abspath, + help="Directory to copy as the app's working directory.") test.add_argument( - "args", nargs="*", help=f"Arguments for `python -m test`. " - f"Separate them from {SCRIPT_NAME}'s own arguments with `--`.") + "args", nargs="*", help=f"Python command-line arguments. " + f"Separate them from {SCRIPT_NAME}'s own arguments with `--`. " + f"If neither -c nor -m are included, `-m test` will be prepended, " + f"which will run Python's own test suite.") + + # Package arguments. + for subcommand in [package, ci]: + subcommand.add_argument( + "-g", action="store_true", default=False, dest="debug", + help="Include debug information in package") + + # CI arguments + for subcommand in [test, ci]: + group = subcommand.add_mutually_exclusive_group(required=subcommand is ci) + group.add_argument( + "--fast-ci", action="store_const", dest="ci_mode", const="fast", + help="Add test arguments for GitHub Actions") + group.add_argument( + "--slow-ci", action="store_const", dest="ci_mode", const="slow", + help="Add test arguments for buildbots") return parser.parse_args() @@ -678,16 +1012,24 @@ def main(): stream.reconfigure(line_buffering=True) context = parse_args() + + # Set the CROSS_BUILD_DIR if an argument was provided + if context.cross_build_dir: + global CROSS_BUILD_DIR + CROSS_BUILD_DIR = context.cross_build_dir.resolve() + dispatch = { "configure-build": configure_build_python, "make-build": make_build_python, + "pythoninfo-build": pythoninfo_build_python, "configure-host": configure_host_python, "make-host": make_host_python, - "build": build_all, - "clean": clean_all, - "build-testbed": build_testbed, + "build": build_targets, + "clean": clean_targets, "test": run_testbed, "package": package, + "ci": ci, + "env": env, } try: @@ -702,20 +1044,17 @@ def main(): def print_called_process_error(e): for stream_name in ["stdout", "stderr"]: content = getattr(e, stream_name) + if isinstance(content, bytes): + content = content.decode(*DECODE_ARGS) stream = getattr(sys, stream_name) if content: stream.write(content) if not content.endswith("\n"): stream.write("\n") - # Format the command so it can be copied into a shell. shlex uses single - # quotes, so we surround the whole command with double quotes. - args_joined = ( - e.cmd if isinstance(e.cmd, str) - else " ".join(shlex.quote(str(arg)) for arg in e.cmd) - ) + # shlex uses single quotes, so we surround the command with double quotes. print( - f'Command "{args_joined}" returned exit status {e.returncode}' + f'Command "{join_command(e.cmd)}" returned exit status {e.returncode}' ) diff --git a/Android/testbed/app/build.gradle.kts b/Android/testbed/app/build.gradle.kts index c627cb1b0e0b223..f9e8d7431de2a82 100644 --- a/Android/testbed/app/build.gradle.kts +++ b/Android/testbed/app/build.gradle.kts @@ -8,16 +8,26 @@ plugins { val ANDROID_DIR = file("../..") val PYTHON_DIR = ANDROID_DIR.parentFile!! -val PYTHON_CROSS_DIR = file("$PYTHON_DIR/cross-build") +val PYTHON_CROSS_DIR = file(System.getenv("CROSS_BUILD_DIR") ?: "$PYTHON_DIR/cross-build") val inSourceTree = ( ANDROID_DIR.name == "Android" && file("$PYTHON_DIR/pyconfig.h.in").exists() ) val KNOWN_ABIS = mapOf( "aarch64-linux-android" to "arm64-v8a", + "arm-linux-androideabi" to "armeabi-v7a", + "i686-linux-android" to "x86", "x86_64-linux-android" to "x86_64", ) +val osArch = System.getProperty("os.arch") +val NATIVE_ABI = mapOf( + "aarch64" to "arm64-v8a", + "amd64" to "x86_64", + "arm64" to "arm64-v8a", + "x86_64" to "x86_64", +)[osArch] ?: throw GradleException("Unknown os.arch '$osArch'") + // Discover prefixes. val prefixes = ArrayList() if (inSourceTree) { @@ -47,7 +57,7 @@ for ((i, prefix) in prefixes.withIndex()) { val libDir = file("$prefix/lib") val version = run { for (filename in libDir.list()!!) { - """python(\d+\.\d+)""".toRegex().matchEntire(filename)?.let { + """python(\d+\.\d+[a-z]*)""".toRegex().matchEntire(filename)?.let { return@run it.groupValues[1] } } @@ -64,9 +74,10 @@ for ((i, prefix) in prefixes.withIndex()) { val libPythonDir = file("$libDir/python$pythonVersion") val triplet = run { for (filename in libPythonDir.list()!!) { - """_sysconfigdata__android_(.+).py""".toRegex().matchEntire(filename)?.let { - return@run it.groupValues[1] - } + """_sysconfigdata_[a-z]*_android_(.+).py""".toRegex() + .matchEntire(filename)?.let { + return@run it.groupValues[1] + } } throw GradleException("Failed to find Python triplet in $libPythonDir") } @@ -78,20 +89,29 @@ android { val androidEnvFile = file("../../android-env.sh").absoluteFile namespace = "org.python.testbed" - compileSdk = 34 + compileSdk = 35 defaultConfig { applicationId = "org.python.testbed" minSdk = androidEnvFile.useLines { for (line in it) { - """api_level:=(\d+)""".toRegex().find(line)?.let { + """ANDROID_API_LEVEL:=(\d+)""".toRegex().find(line)?.let { return@useLines it.groupValues[1].toInt() } } throw GradleException("Failed to find API level in $androidEnvFile") } - targetSdk = 34 + + // This controls the API level of the maxVersion managed emulator, which is used + // by CI and cibuildwheel. + // * 32 has intermittent failures accessing the internet (#142387). + // * 33 has excessive buffering in the logcat client + // (https://cs.android.com/android/_/android/platform/system/logging/+/d340721894f223327339010df59b0ac514308826). + // * 34 consumes too much disk space on GitHub Actions (#142289), though switching to the + // "default" image may be a workaround. + // * 36 and later are not available as aosp_atd images yet. + targetSdk = 35 versionCode = 1 versionName = "1.0" @@ -124,9 +144,10 @@ android { path("src/main/c/CMakeLists.txt") } - // Set this property to something non-empty, otherwise it'll use the default - // list, which ignores asset directories beginning with an underscore. - aaptOptions.ignoreAssetsPattern = ".git" + // Set this property to something nonexistent but non-empty. Otherwise it'll use the + // default list, which ignores asset directories beginning with an underscore, and + // maybe also other files required by tests. + aaptOptions.ignoreAssetsPattern = "android-testbed-dont-ignore-anything" compileOptions { sourceCompatibility = JavaVersion.VERSION_1_8 @@ -139,6 +160,9 @@ android { testOptions { managedDevices { localDevices { + // systemImageSource should use what its documentation calls an + // "explicit source", i.e. the sdkmanager package name format, because + // that will be required in CreateEmulatorTask below. create("minVersion") { device = "Small Phone" @@ -147,13 +171,13 @@ android { // ATD devices are smaller and faster, but have a minimum // API level of 30. - systemImageSource = if (apiLevel >= 30) "aosp-atd" else "aosp" + systemImageSource = if (apiLevel >= 30) "aosp_atd" else "default" } create("maxVersion") { device = "Small Phone" apiLevel = defaultConfig.targetSdk!! - systemImageSource = "aosp-atd" + systemImageSource = "aosp_atd" } } @@ -179,6 +203,138 @@ dependencies { } +afterEvaluate { + // Every new emulator has a maximum of 2 GB RAM, regardless of its hardware profile + // (https://cs.android.com/android-studio/platform/tools/base/+/refs/tags/studio-2025.3.2:sdklib/src/main/java/com/android/sdklib/internal/avd/EmulatedProperties.java;l=68). + // This is barely enough to test Python, and not enough to test Pandas + // (https://github.com/python/cpython/pull/137186#issuecomment-3136301023, + // https://github.com/pandas-dev/pandas/pull/63405#issuecomment-3667846159). + // So we'll increase it by editing the emulator configuration files. + // + // If the emulator doesn't exist yet, we want to edit it after it's created, but + // before it starts for the first time. Otherwise it'll need to be cold-booted + // again, which would slow down the first run, which is likely the only run in CI + // environments. But the Setup task both creates and starts the emulator if it + // doesn't already exist. So we create it ourselves before the Setup task runs. + for (device in android.testOptions.managedDevices.localDevices) { + val createTask = tasks.register("${device.name}Create") { + this.device = device.device + apiLevel = device.apiLevel + systemImageSource = device.systemImageSource + abi = NATIVE_ABI + } + tasks.named("${device.name}Setup") { + dependsOn(createTask) + } + } +} + +abstract class CreateEmulatorTask : DefaultTask() { + @get:Input abstract val device: Property + @get:Input abstract val apiLevel: Property + @get:Input abstract val systemImageSource: Property + @get:Input abstract val abi: Property + @get:Inject abstract val execOps: ExecOperations + + private val avdName by lazy { + listOf( + "dev${apiLevel.get()}", + systemImageSource.get(), + abi.get(), + device.get().replace(' ', '_'), + ).joinToString("_") + } + + private val avdDir by lazy { + // XDG_CONFIG_HOME is respected by both avdmanager and Gradle. + val userHome = System.getenv("ANDROID_USER_HOME") ?: ( + (System.getenv("XDG_CONFIG_HOME") ?: System.getProperty("user.home")!!) + + "/.android" + ) + File("$userHome/avd/gradle-managed", "$avdName.avd") + } + + @TaskAction + fun run() { + if (!avdDir.exists()) { + createAvd() + } + updateAvd() + } + + fun createAvd() { + val systemImage = listOf( + "system-images", + "android-${apiLevel.get()}", + systemImageSource.get(), + abi.get(), + ).joinToString(";") + + runCmdlineTool("sdkmanager", systemImage) + runCmdlineTool( + "avdmanager", "create", "avd", + "--name", avdName, + "--path", avdDir, + "--device", device.get().lowercase().replace(" ", "_"), + "--package", systemImage, + ) + + val iniName = "$avdName.ini" + if (!File(avdDir.parentFile.parentFile, iniName).renameTo( + File(avdDir.parentFile, iniName) + )) { + throw GradleException("Failed to rename $iniName") + } + } + + fun updateAvd() { + for (filename in listOf( + "config.ini", // Created by avdmanager; always exists + "hardware-qemu.ini", // Created on first run; might not exist + )) { + val iniFile = File(avdDir, filename) + if (!iniFile.exists()) { + if (filename == "config.ini") { + throw GradleException("$iniFile does not exist") + } + continue + } + + val iniText = iniFile.readText() + val pattern = Regex( + """^\s*hw.ramSize\s*=\s*(.+?)\s*$""", RegexOption.MULTILINE + ) + val matches = pattern.findAll(iniText).toList() + if (matches.size != 1) { + throw GradleException( + "Found ${matches.size} instances of $pattern in $iniFile; expected 1" + ) + } + + val expectedRam = "4096" + if (matches[0].groupValues[1] != expectedRam) { + iniFile.writeText( + iniText.replace(pattern, "hw.ramSize = $expectedRam") + ) + } + } + } + + fun runCmdlineTool(tool: String, vararg args: Any) { + val androidHome = System.getenv("ANDROID_HOME")!! + val exeSuffix = + if (System.getProperty("os.name").lowercase().startsWith("win")) ".exe" + else "" + val command = + listOf("$androidHome/cmdline-tools/latest/bin/$tool$exeSuffix", *args) + println(command.joinToString(" ")) + execOps.exec { + commandLine(command) + } + } +} + + // Create some custom tasks to copy Python and its standard library from // elsewhere in the repository. androidComponents.onVariants { variant -> @@ -205,11 +361,35 @@ androidComponents.onVariants { variant -> into("site-packages") { from("$projectDir/src/main/python") + + val sitePackages = findProperty("python.sitePackages") as String? + if (!sitePackages.isNullOrEmpty()) { + if (!file(sitePackages).exists()) { + throw GradleException("$sitePackages does not exist") + } + from(sitePackages) + } } duplicatesStrategy = DuplicatesStrategy.EXCLUDE exclude("**/__pycache__") } + + into("cwd") { + val cwd = findProperty("python.cwd") as String? + if (!cwd.isNullOrEmpty()) { + if (!file(cwd).exists()) { + throw GradleException("$cwd does not exist") + } + from(cwd) + } + } + + // A filename ending with .gz will be automatically decompressed + // while building the APK. Avoid this by adding a dash to the end, + // and add an extra dash to any filenames that already end with one. + // This will be undone in MainActivity.kt. + rename(""".*(\.gz|-)""", "$0-") } } diff --git a/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt index 0e888ab71d87da1..e57243566f91dca 100644 --- a/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt +++ b/Android/testbed/app/src/androidTest/java/org/python/testbed/PythonSuite.kt @@ -17,11 +17,11 @@ class PythonSuite { fun testPython() { val start = System.currentTimeMillis() try { - val context = + val status = PythonTestRunner( InstrumentationRegistry.getInstrumentation().targetContext - val args = - InstrumentationRegistry.getArguments().getString("pythonArgs", "") - val status = PythonTestRunner(context).run(args) + ).run( + InstrumentationRegistry.getArguments().getString("pythonArgs")!!, + ) assertEquals(0, status) } finally { // Make sure the process lives long enough for the test script to diff --git a/Android/testbed/app/src/main/c/main_activity.c b/Android/testbed/app/src/main/c/main_activity.c index ec7f93a3e5ee13a..7f024f0a348b61c 100644 --- a/Android/testbed/app/src/main/c/main_activity.c +++ b/Android/testbed/app/src/main/c/main_activity.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -15,6 +16,13 @@ static void throw_runtime_exception(JNIEnv *env, const char *message) { message); } +static void throw_errno(JNIEnv *env, const char *error_prefix) { + char error_message[1024]; + snprintf(error_message, sizeof(error_message), + "%s: %s", error_prefix, strerror(errno)); + throw_runtime_exception(env, error_message); +} + // --- Stdio redirection ------------------------------------------------------ @@ -95,10 +103,7 @@ JNIEXPORT void JNICALL Java_org_python_testbed_PythonTestRunner_redirectStdioToL for (StreamInfo *si = STREAMS; si->file; si++) { char *error_prefix; if ((error_prefix = redirect_stream(si))) { - char error_message[1024]; - snprintf(error_message, sizeof(error_message), - "%s: %s", error_prefix, strerror(errno)); - throw_runtime_exception(env, error_message); + throw_errno(env, error_prefix); return; } } @@ -107,13 +112,38 @@ JNIEXPORT void JNICALL Java_org_python_testbed_PythonTestRunner_redirectStdioToL // --- Python initialization --------------------------------------------------- -static PyStatus set_config_string( - JNIEnv *env, PyConfig *config, wchar_t **config_str, jstring value -) { - const char *value_utf8 = (*env)->GetStringUTFChars(env, value, NULL); - PyStatus status = PyConfig_SetBytesString(config, config_str, value_utf8); - (*env)->ReleaseStringUTFChars(env, value, value_utf8); - return status; +static char *init_signals() { + // Some tests use SIGUSR1, but that's blocked by default in an Android app in + // order to make it available to `sigwait` in the Signal Catcher thread. + // (https://cs.android.com/android/platform/superproject/+/android14-qpr3-release:art/runtime/signal_catcher.cc). + // That thread's functionality is only useful for debugging the JVM, so disabling + // it should not weaken the tests. + // + // There's no safe way of stopping the thread completely (#123982), but simply + // unblocking SIGUSR1 is enough to fix most tests. + // + // However, in tests that generate multiple different signals in quick + // succession, it's possible for SIGUSR1 to arrive while the main thread is busy + // running the C-level handler for a different signal. In that case, the SIGUSR1 + // may be sent to the Signal Catcher thread instead, which will generate a log + // message containing the text "reacting to signal". + // + // Such tests may need to be changed in one of the following ways: + // * Use a signal other than SIGUSR1 (e.g. test_stress_delivery_simultaneous in + // test_signal.py). + // * Send the signal to a specific thread rather than the whole process (e.g. + // test_signals in test_threadsignals.py. + sigset_t set; + if (sigemptyset(&set)) { + return "sigemptyset"; + } + if (sigaddset(&set, SIGUSR1)) { + return "sigaddset"; + } + if ((errno = pthread_sigmask(SIG_UNBLOCK, &set, NULL))) { + return "pthread_sigmask"; + } + return NULL; } static void throw_status(JNIEnv *env, PyStatus status) { @@ -121,27 +151,47 @@ static void throw_status(JNIEnv *env, PyStatus status) { } JNIEXPORT int JNICALL Java_org_python_testbed_PythonTestRunner_runPython( - JNIEnv *env, jobject obj, jstring home, jstring runModule + JNIEnv *env, jobject obj, jstring home, jarray args ) { + const char *home_utf8 = (*env)->GetStringUTFChars(env, home, NULL); + char cwd[PATH_MAX]; + snprintf(cwd, sizeof(cwd), "%s/%s", home_utf8, "cwd"); + if (chdir(cwd)) { + throw_errno(env, "chdir"); + return 1; + } + + char *error_prefix; + if ((error_prefix = init_signals())) { + throw_errno(env, error_prefix); + return 1; + } + PyConfig config; PyStatus status; - PyConfig_InitIsolatedConfig(&config); + PyConfig_InitPythonConfig(&config); - status = set_config_string(env, &config, &config.home, home); - if (PyStatus_Exception(status)) { + jsize argc = (*env)->GetArrayLength(env, args); + const char *argv[argc + 1]; + for (int i = 0; i < argc; i++) { + jobject arg = (*env)->GetObjectArrayElement(env, args, i); + argv[i] = (*env)->GetStringUTFChars(env, arg, NULL); + } + argv[argc] = NULL; + + // PyConfig_SetBytesArgv "must be called before other methods, since the + // preinitialization configuration depends on command line arguments" + if (PyStatus_Exception(status = PyConfig_SetBytesArgv(&config, argc, (char**)argv))) { throw_status(env, status); return 1; } - status = set_config_string(env, &config, &config.run_module, runModule); + status = PyConfig_SetBytesString(&config, &config.home, home_utf8); if (PyStatus_Exception(status)) { throw_status(env, status); return 1; } - // Some tests generate SIGPIPE and SIGXFSZ, which should be ignored. - config.install_signal_handlers = 1; - status = Py_InitializeFromConfig(&config); if (PyStatus_Exception(status)) { throw_status(env, status); diff --git a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt index c4bf6cbe83d8cd8..dc49cdb9a9f7395 100644 --- a/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt +++ b/Android/testbed/app/src/main/java/org/python/testbed/MainActivity.kt @@ -5,6 +5,7 @@ import android.os.* import android.system.Os import android.widget.TextView import androidx.appcompat.app.* +import org.json.JSONArray import java.io.* @@ -15,18 +16,25 @@ class MainActivity : AppCompatActivity() { override fun onCreate(savedInstanceState: Bundle?) { super.onCreate(savedInstanceState) setContentView(R.layout.activity_main) - val status = PythonTestRunner(this).run("-W -uall") + val status = PythonTestRunner(this).run("""["-m", "test", "-W", "-uall"]""") findViewById(R.id.tvHello).text = "Exit status $status" } } class PythonTestRunner(val context: Context) { - /** @param args Extra arguments for `python -m test`. - * @return The Python exit status: zero if the tests passed, nonzero if - * they failed. */ - fun run(args: String = "") : Int { - Os.setenv("PYTHON_ARGS", args, true) + /** Run Python. + * + * @param args Python command-line, encoded as JSON. + * @return The Python exit status: zero on success, nonzero on failure. */ + fun run(args: String) : Int { + // We leave argument 0 as an empty string, which is a placeholder for the + // executable name in embedded mode. + val argsJsonArray = JSONArray(args) + val argsStringArray = Array(argsJsonArray.length() + 1) { it -> ""} + for (i in 0.. + // Undo the .gz workaround from build.gradle.kts. + val outputName = name.replace(Regex("""(.*)-"""), "$1") + File(targetSubdir, outputName).outputStream().use { output -> input.copyTo(output) } } @@ -75,5 +90,5 @@ class PythonTestRunner(val context: Context) { } private external fun redirectStdioToLogcat() - private external fun runPython(home: String, runModule: String) : Int + private external fun runPython(home: String, args: Array) : Int } diff --git a/Android/testbed/app/src/main/python/main.py b/Android/testbed/app/src/main/python/main.py deleted file mode 100644 index d6941b14412fcc2..000000000000000 --- a/Android/testbed/app/src/main/python/main.py +++ /dev/null @@ -1,32 +0,0 @@ -import os -import runpy -import shlex -import signal -import sys - -# Some tests use SIGUSR1, but that's blocked by default in an Android app in -# order to make it available to `sigwait` in the Signal Catcher thread. -# (https://cs.android.com/android/platform/superproject/+/android14-qpr3-release:art/runtime/signal_catcher.cc). -# That thread's functionality is only useful for debugging the JVM, so disabling -# it should not weaken the tests. -# -# There's no safe way of stopping the thread completely (#123982), but simply -# unblocking SIGUSR1 is enough to fix most tests. -# -# However, in tests that generate multiple different signals in quick -# succession, it's possible for SIGUSR1 to arrive while the main thread is busy -# running the C-level handler for a different signal. In that case, the SIGUSR1 -# may be sent to the Signal Catcher thread instead, which will generate a log -# message containing the text "reacting to signal". -# -# Such tests may need to be changed in one of the following ways: -# * Use a signal other than SIGUSR1 (e.g. test_stress_delivery_simultaneous in -# test_signal.py). -# * Send the signal to a specific thread rather than the whole process (e.g. -# test_signals in test_threadsignals.py. -signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGUSR1]) - -sys.argv[1:] = shlex.split(os.environ["PYTHON_ARGS"]) - -# The test module will call sys.exit to indicate whether the tests passed. -runpy.run_module("test") diff --git a/Android/testbed/build.gradle.kts b/Android/testbed/build.gradle.kts index 4d1d6f87594da32..451517b3f1aeab6 100644 --- a/Android/testbed/build.gradle.kts +++ b/Android/testbed/build.gradle.kts @@ -1,5 +1,5 @@ // Top-level build file where you can add configuration options common to all sub-projects/modules. plugins { - id("com.android.application") version "8.6.1" apply false + id("com.android.application") version "8.10.0" apply false id("org.jetbrains.kotlin.android") version "1.9.22" apply false } diff --git a/Android/testbed/gradle/wrapper/gradle-wrapper.properties b/Android/testbed/gradle/wrapper/gradle-wrapper.properties index 36529c896426b0b..5d42fbae084da1f 100644 --- a/Android/testbed/gradle/wrapper/gradle-wrapper.properties +++ b/Android/testbed/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ #Mon Feb 19 20:29:06 GMT 2024 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/Apple/.ruff.toml b/Apple/.ruff.toml new file mode 100644 index 000000000000000..4cdc39ebee4be9b --- /dev/null +++ b/Apple/.ruff.toml @@ -0,0 +1,22 @@ +extend = "../.ruff.toml" # Inherit the project-wide settings + +[format] +preview = true +docstring-code-format = true + +[lint] +select = [ + "C4", # flake8-comprehensions + "E", # pycodestyle + "F", # pyflakes + "I", # isort + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "PYI", # flake8-pyi + "RUF100", # Ban unused `# noqa` comments + "UP", # pyupgrade + "W", # pycodestyle + "YTT", # flake8-2020 +] diff --git a/Apple/__main__.py b/Apple/__main__.py new file mode 100644 index 000000000000000..ec67265e02a081f --- /dev/null +++ b/Apple/__main__.py @@ -0,0 +1,1119 @@ +#!/usr/bin/env python3 +########################################################################## +# Apple XCframework build script +# +# This script simplifies the process of configuring, compiling and packaging an +# XCframework for an Apple platform. +# +# At present, it only supports iOS, but it has been constructed so that it +# could be used on any Apple platform. +# +# The simplest entry point is: +# +# $ python Apple ci iOS +# +# which will: +# * Clean any pre-existing build artefacts +# * Configure and make a Python that can be used for the build +# * Configure and make a Python for each supported iOS architecture and ABI +# * Combine the outputs of the builds from the previous step into a single +# XCframework, merging binaries into a "fat" binary if necessary +# * Clone a copy of the testbed, configured to use the XCframework +# * Construct a tarball containing the release artefacts +# * Run the test suite using the generated XCframework. +# +# This is the complete sequence that would be needed in CI to build and test +# a candidate release artefact. +# +# Each individual step can be invoked individually - there are commands to +# clean, configure-build, make-build, configure-host, make-host, package, and +# test. +# +# There is also a build command that can be used to combine the configure and +# make steps for the build Python, an individual host, all hosts, or all +# builds. +########################################################################## +from __future__ import annotations + +import argparse +import os +import platform +import re +import shlex +import shutil +import signal +import subprocess +import sys +import sysconfig +import time +from collections.abc import Callable, Sequence +from contextlib import contextmanager +from datetime import datetime, timezone +from os.path import basename, relpath +from pathlib import Path +from subprocess import CalledProcessError + +EnvironmentT = dict[str, str] +ArgsT = Sequence[str | Path] + +SCRIPT_NAME = Path(__file__).name +PYTHON_DIR = Path(__file__).resolve().parent.parent + +CROSS_BUILD_DIR = PYTHON_DIR / "cross-build" + +HOSTS: dict[str, dict[str, dict[str, str]]] = { + # Structure of this data: + # * Platform identifier + # * an XCframework slice that must exist for that platform + # * a host triple: the multiarch spec for that host + "iOS": { + "ios-arm64": { + "arm64-apple-ios": "arm64-iphoneos", + }, + "ios-arm64_x86_64-simulator": { + "arm64-apple-ios-simulator": "arm64-iphonesimulator", + "x86_64-apple-ios-simulator": "x86_64-iphonesimulator", + }, + }, +} + + +def subdir(name: str, create: bool = False) -> Path: + """Ensure that a cross-build directory for the given name exists.""" + path = CROSS_BUILD_DIR / name + if not path.exists(): + if not create: + sys.exit( + f"{path} does not exist. Create it by running the appropriate " + f"`configure` subcommand of {SCRIPT_NAME}." + ) + else: + path.mkdir(parents=True) + return path + + +def run( + command: ArgsT, + *, + host: str | None = None, + env: EnvironmentT | None = None, + log: bool | None = True, + **kwargs, +) -> subprocess.CompletedProcess: + """Run a command in an Apple development environment. + + Optionally logs the executed command to the console. + """ + kwargs.setdefault("check", True) + if env is None: + env = os.environ.copy() + + if host: + host_env = apple_env(host) + print_env(host_env) + env.update(host_env) + + if log: + print(">", join_command(command)) + return subprocess.run(command, env=env, **kwargs) + + +def join_command(args: str | Path | ArgsT) -> str: + """Format a command so it can be copied into a shell. + + Similar to `shlex.join`, but also accepts arguments which are Paths, or a + single string/Path outside of a list. + """ + if isinstance(args, (str, Path)): + return str(args) + else: + return shlex.join(map(str, args)) + + +def print_env(env: EnvironmentT) -> None: + """Format the environment so it can be pasted into a shell.""" + for key, value in sorted(env.items()): + print(f"export {key}={shlex.quote(value)}") + + +def apple_env(host: str) -> EnvironmentT: + """Construct an Apple development environment for the given host.""" + env = { + "PATH": ":".join([ + str(PYTHON_DIR / "Apple/iOS/Resources/bin"), + str(subdir(host) / "prefix"), + "/usr/bin", + "/bin", + "/usr/sbin", + "/sbin", + "/Library/Apple/usr/bin", + ]), + } + + return env + + +def delete_path(name: str) -> None: + """Delete the named cross-build directory, if it exists.""" + path = CROSS_BUILD_DIR / name + if path.exists(): + print(f"Deleting {path} ...") + shutil.rmtree(path) + + +def all_host_triples(platform: str) -> list[str]: + """Return all host triples for the given platform. + + The host triples are the platform definitions used as input to configure + (e.g., "arm64-apple-ios-simulator"). + """ + triples = [] + for slice_name, slice_parts in HOSTS[platform].items(): + triples.extend(list(slice_parts)) + return triples + + +def clean(context: argparse.Namespace, target: str | None = None) -> None: + """The implementation of the "clean" command.""" + if target is None: + target = context.host + + # If we're explicitly targeting the build, there's no platform or + # distribution artefacts. If we're cleaning tests, we keep all built + # artefacts. Otherwise, the built artefacts must be dirty, so we remove + # them. + if target not in {"build", "test"}: + paths = ["dist", context.platform] + list(HOSTS[context.platform]) + else: + paths = [] + + if target in {"all", "build"}: + paths.append("build") + + if target in {"all", "hosts"}: + paths.extend(all_host_triples(context.platform)) + elif target not in {"build", "test", "package"}: + paths.append(target) + + if target in {"all", "hosts", "test"}: + paths.extend([ + path.name + for path in CROSS_BUILD_DIR.glob(f"{context.platform}-testbed.*") + ]) + + for path in paths: + delete_path(path) + + +def build_python_path() -> Path: + """The path to the build Python binary.""" + build_dir = subdir("build") + binary = build_dir / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError( + f"Unable to find `python(.exe)` in {build_dir}" + ) + + return binary + + +@contextmanager +def group(text: str): + """A context manager that outputs a log marker around a section of a build. + + If running in a GitHub Actions environment, the GitHub syntax for + collapsible log sections is used. + """ + if "GITHUB_ACTIONS" in os.environ: + print(f"::group::{text}") + else: + print(f"===== {text} " + "=" * (70 - len(text))) + + yield + + if "GITHUB_ACTIONS" in os.environ: + print("::endgroup::") + else: + print() + + +@contextmanager +def cwd(subdir: Path): + """A context manager that sets the current working directory.""" + orig = os.getcwd() + os.chdir(subdir) + yield + os.chdir(orig) + + +def configure_build_python(context: argparse.Namespace) -> None: + """The implementation of the "configure-build" command.""" + if context.clean: + clean(context, "build") + + with ( + group("Configuring build Python"), + cwd(subdir("build", create=True)), + ): + command = [relpath(PYTHON_DIR / "configure")] + if context.args: + command.extend(context.args) + run(command) + + +def make_build_python(context: argparse.Namespace) -> None: + """The implementation of the "make-build" command.""" + with ( + group("Compiling build Python"), + cwd(subdir("build")), + ): + run(["make", "-j", str(os.cpu_count())]) + + +def pythoninfo_build_python(context: argparse.Namespace) -> None: + """The implementation of the "pythoninfo-build" command.""" + with ( + group("Display build info of the build Python"), + cwd(subdir("build")), + ): + run(["make", "pythoninfo"]) + + +def apple_target(host: str) -> str: + """Return the Apple platform identifier for a given host triple.""" + for _, platform_slices in HOSTS.items(): + for slice_name, slice_parts in platform_slices.items(): + for host_triple, multiarch in slice_parts.items(): + if host == host_triple: + return ".".join(multiarch.split("-")[::-1]) + + raise KeyError(host) + + +def apple_multiarch(host: str) -> str: + """Return the multiarch descriptor for a given host triple.""" + for _, platform_slices in HOSTS.items(): + for slice_name, slice_parts in platform_slices.items(): + for host_triple, multiarch in slice_parts.items(): + if host == host_triple: + return multiarch + + raise KeyError(host) + + +def unpack_deps( + platform: str, + host: str, + prefix_dir: Path, + cache_dir: Path, +) -> None: + """Unpack binary dependencies into a provided directory. + + Downloads binaries if they aren't already present. Downloads will be stored + in provided cache directory. + + On iOS, as a safety mechanism, any dynamic libraries will be purged from + the unpacked dependencies. + """ + # To create new builds of these dependencies, usually all that's necessary + # is to push a tag to the cpython-apple-source-deps repository, and GitHub + # Actions will do the rest. + # + # If you're a member of the Python core team, and you'd like to be able to + # push these tags yourself, please contact Malcolm Smith or Russell + # Keith-Magee. + deps_url = "https://github.com/beeware/cpython-apple-source-deps/releases/download" + for name_ver in [ + "BZip2-1.0.8-2", + "libFFI-3.4.7-2", + "OpenSSL-3.5.7-1", + "XZ-5.6.4-2", + "mpdecimal-4.0.0-2", + "zstd-1.5.7-1", + ]: + filename = f"{name_ver.lower()}-{apple_target(host)}.tar.gz" + archive_path = download( + f"{deps_url}/{name_ver}/{filename}", + target_dir=cache_dir, + ) + shutil.unpack_archive(archive_path, prefix_dir) + + # Dynamic libraries will be preferentially linked over static; + # On iOS, ensure that no dylibs are available in the prefix folder. + if platform == "iOS": + for dylib in prefix_dir.glob("**/*.dylib"): + dylib.unlink() + + +def download(url: str, target_dir: Path) -> Path: + """Download the specified URL into the given directory. + + :return: The path to the downloaded archive. + """ + target_path = Path(target_dir).resolve() + target_path.mkdir(exist_ok=True, parents=True) + + out_path = target_path / basename(url) + if not Path(out_path).is_file(): + run([ + "curl", + "-Lf", + "--retry", + "5", + "--retry-all-errors", + "-o", + out_path, + url, + ]) + else: + print(f"Using cached version of {basename(url)}") + return out_path + + +def configure_host_python( + context: argparse.Namespace, + host: str | None = None, +) -> None: + """The implementation of the "configure-host" command.""" + if host is None: + host = context.host + + if context.clean: + clean(context, host) + + host_dir = subdir(host, create=True) + prefix_dir = host_dir / "prefix" + + with group(f"Downloading dependencies ({host})"): + if not prefix_dir.exists(): + prefix_dir.mkdir() + cache_dir = ( + Path(context.cache_dir).resolve() + if context.cache_dir + else CROSS_BUILD_DIR / "downloads" + ) + unpack_deps(context.platform, host, prefix_dir, cache_dir) + else: + print("Dependencies already installed") + + with ( + group(f"Configuring host Python ({host})"), + cwd(host_dir), + ): + command = [ + # Basic cross-compiling configuration + relpath(PYTHON_DIR / "configure"), + f"--host={host}", + f"--build={sysconfig.get_config_var('BUILD_GNU_TYPE')}", + f"--with-build-python={build_python_path()}", + "--with-system-libmpdec", + "--enable-framework", + # Dependent libraries. + f"--with-openssl={prefix_dir}", + f"LIBLZMA_CFLAGS=-I{prefix_dir}/include", + f"LIBLZMA_LIBS=-L{prefix_dir}/lib -llzma", + f"LIBFFI_CFLAGS=-I{prefix_dir}/include", + f"LIBFFI_LIBS=-L{prefix_dir}/lib -lffi", + f"LIBMPDEC_CFLAGS=-I{prefix_dir}/include", + f"LIBMPDEC_LIBS=-L{prefix_dir}/lib -lmpdec", + f"LIBZSTD_CFLAGS=-I{prefix_dir}/include", + f"LIBZSTD_LIBS=-L{prefix_dir}/lib -lzstd", + ] + + if context.args: + command.extend(context.args) + run(command, host=host) + + +def make_host_python( + context: argparse.Namespace, + host: str | None = None, +) -> None: + """The implementation of the "make-host" command.""" + if host is None: + host = context.host + + with ( + group(f"Compiling host Python ({host})"), + cwd(subdir(host)), + ): + run(["make", "-j", str(os.cpu_count())], host=host) + run(["make", "install"], host=host) + + +def framework_path(host_triple: str, multiarch: str) -> Path: + """The path to a built single-architecture framework product. + + :param host_triple: The host triple (e.g., arm64-apple-ios-simulator) + :param multiarch: The multiarch identifier (e.g., arm64-simulator) + """ + return CROSS_BUILD_DIR / f"{host_triple}/Apple/iOS/Frameworks/{multiarch}" + + +def package_version(prefix_path: Path) -> str: + """Extract the Python version being built from patchlevel.h.""" + for path in prefix_path.glob("**/patchlevel.h"): + text = path.read_text(encoding="utf-8") + if match := re.search( + r'\n\s*#define\s+PY_VERSION\s+"(.+)"\s*\n', text + ): + version = match[1] + # If not building against a tagged commit, add a timestamp to the + # version. Follow the PyPA version number rules, as this will make + # it easier to process with other tools. The version will have a + # `+` suffix once any official release has been made; a freshly + # forked main branch will have a version of 3.X.0a0. + if version.endswith("a0"): + version += "+" + if version.endswith("+"): + version += datetime.now(timezone.utc).strftime("%Y%m%d.%H%M%S") + + return version + + sys.exit("Unable to determine Python version being packaged.") + + +def lib_platform_files(dirname, names): + """A file filter that ignores platform-specific files in lib.""" + path = Path(dirname) + if ( + path.parts[-3] == "lib" + and path.parts[-2].startswith("python") + and path.parts[-1] == "lib-dynload" + ): + return names + elif path.parts[-2] == "lib" and path.parts[-1].startswith("python"): + ignored_names = { + name + for name in names + if ( + name.startswith("_sysconfigdata_") + or name.startswith("_sysconfig_vars_") + or name == "build-details.json" + ) + } + elif path.parts[-1] == "lib": + ignored_names = { + name + for name in names + if name.startswith("libpython") and name.endswith(".dylib") + } + else: + ignored_names = set() + + return ignored_names + + +def lib_non_platform_files(dirname, names): + """A file filter that ignores anything *except* platform-specific files + in the lib directory. + """ + path = Path(dirname) + if path.parts[-2] == "lib" and path.parts[-1].startswith("python"): + return ( + set(names) - lib_platform_files(dirname, names) - {"lib-dynload"} + ) + else: + return set() + + +def create_xcframework(platform: str) -> str: + """Build an XCframework from the component parts for the platform. + + :return: The version number of the Python version that was packaged. + """ + package_path = CROSS_BUILD_DIR / platform + try: + package_path.mkdir() + except FileExistsError: + raise RuntimeError( + f"{platform} XCframework already exists; do you need to run " + "with --clean?" + ) from None + + frameworks = [] + # Merge Frameworks for each component SDK. If there's only one architecture + # for the SDK, we can use the compiled Python.framework as-is. However, if + # there's more than architecture, we need to merge the individual built + # frameworks into a merged "fat" framework. + for slice_name, slice_parts in HOSTS[platform].items(): + # Some parts are the same across all slices, so we use can any of the + # host frameworks as the source for the merged version. Use the first + # one on the list, as it's as representative as any other. + first_host_triple, first_multiarch = next(iter(slice_parts.items())) + first_framework = ( + framework_path(first_host_triple, first_multiarch) + / "Python.framework" + ) + + if len(slice_parts) == 1: + # The first framework is the only framework, so copy it. + print(f"Copying framework for {slice_name}...") + frameworks.append(first_framework) + else: + print(f"Merging framework for {slice_name}...") + slice_path = CROSS_BUILD_DIR / slice_name + slice_framework = slice_path / "Python.framework" + slice_framework.mkdir(exist_ok=True, parents=True) + + # Copy the Info.plist + shutil.copy( + first_framework / "Info.plist", + slice_framework / "Info.plist", + ) + + # Copy the headers + shutil.copytree( + first_framework / "Headers", + slice_framework / "Headers", + ) + + # Create the "fat" library binary for the slice + run( + ["lipo", "-create", "-output", slice_framework / "Python"] + + [ + ( + framework_path(host_triple, multiarch) + / "Python.framework/Python" + ) + for host_triple, multiarch in slice_parts.items() + ] + ) + + # Add this merged slice to the list to be added to the XCframework + frameworks.append(slice_framework) + + print() + print("Build XCframework...") + cmd = [ + "xcodebuild", + "-create-xcframework", + "-output", + package_path / "Python.xcframework", + ] + for framework in frameworks: + cmd.extend(["-framework", framework]) + + run(cmd) + + # Extract the package version from the merged framework + version = package_version(package_path / "Python.xcframework") + version_tag = ".".join(version.split(".")[:2]) + + # On non-macOS platforms, each framework in XCframework only contains the + # headers, libPython, plus an Info.plist. Other resources like the standard + # library and binary shims aren't allowed to live in framework; they need + # to be copied in separately. + print() + print("Copy additional resources...") + has_common_stdlib = False + for slice_name, slice_parts in HOSTS[platform].items(): + # Some parts are the same across all slices, so we can any of the + # host frameworks as the source for the merged version. + first_host_triple, first_multiarch = next(iter(slice_parts.items())) + first_path = framework_path(first_host_triple, first_multiarch) + first_framework = first_path / "Python.framework" + + slice_path = package_path / f"Python.xcframework/{slice_name}" + slice_framework = slice_path / "Python.framework" + + # Copy the binary helpers + print(f" - {slice_name} binaries") + shutil.copytree(first_path / "bin", slice_path / "bin") + + # Copy the include path (a symlink to the framework headers) + print(f" - {slice_name} include files") + shutil.copytree( + first_path / "include", + slice_path / "include", + symlinks=True, + ) + + # Copy in the cross-architecture pyconfig.h + shutil.copy( + PYTHON_DIR / f"Apple/{platform}/Resources/pyconfig.h", + slice_framework / "Headers/pyconfig.h", + ) + + print(f" - {slice_name} shared library") + # Create a simlink for the fat library + shared_lib = slice_path / f"lib/libpython{version_tag}.dylib" + shared_lib.parent.mkdir() + shared_lib.symlink_to("../Python.framework/Python") + + print(f" - {slice_name} architecture-specific files") + for host_triple, multiarch in slice_parts.items(): + print(f" - {multiarch} standard library") + arch, _ = multiarch.split("-", 1) + + if not has_common_stdlib: + print(" - using this architecture as the common stdlib") + shutil.copytree( + framework_path(host_triple, multiarch) / "lib", + package_path / "Python.xcframework/lib", + ignore=lib_platform_files, + symlinks=True, + ) + has_common_stdlib = True + + shutil.copytree( + framework_path(host_triple, multiarch) / "lib", + slice_path / f"lib-{arch}", + ignore=lib_non_platform_files, + symlinks=True, + ) + + # Copy the host's pyconfig.h to an architecture-specific name. + arch = multiarch.split("-")[0] + host_path = ( + CROSS_BUILD_DIR + / host_triple + / "Apple/iOS/Frameworks" + / multiarch + ) + host_framework = host_path / "Python.framework" + shutil.copy( + host_framework / "Headers/pyconfig.h", + slice_framework / f"Headers/pyconfig-{arch}.h", + ) + + # Apple identifies certain libraries as "security risks"; if you + # statically link those libraries into a Framework, you become + # responsible for providing a privacy manifest for that framework. + xcprivacy_file = { + "OpenSSL": subdir(host_triple) + / "prefix/share/OpenSSL.xcprivacy" + } + print(f" - {multiarch} xcprivacy files") + for module, lib in [ + ("_hashlib", "OpenSSL"), + ("_ssl", "OpenSSL"), + ]: + shutil.copy( + xcprivacy_file[lib], + slice_path + / f"lib-{arch}/python{version_tag}" + / f"lib-dynload/{module}.xcprivacy", + ) + + print(" - build tools") + shutil.copytree( + PYTHON_DIR / "Apple/testbed/Python.xcframework/build", + package_path / "Python.xcframework/build", + ) + + return version + + +def package(context: argparse.Namespace) -> None: + """The implementation of the "package" command.""" + if context.clean: + clean(context, "package") + + with group("Building package"): + # Create an XCframework + version = create_xcframework(context.platform) + + # Clone testbed + print() + run([ + sys.executable, + "Apple/testbed", + "clone", + "--platform", + context.platform, + "--framework", + CROSS_BUILD_DIR / context.platform / "Python.xcframework", + CROSS_BUILD_DIR / context.platform / "testbed", + ]) + + # Build the final archive + archive_name = ( + CROSS_BUILD_DIR + / "dist" + / f"python-{version}-{context.platform}-XCframework" + ) + + print() + print("Create package archive...") + shutil.make_archive( + str(CROSS_BUILD_DIR / archive_name), + format="gztar", + root_dir=CROSS_BUILD_DIR / context.platform, + base_dir=".", + ) + print() + print(f"{archive_name.relative_to(PYTHON_DIR)}.tar.gz created.") + + +def build(context: argparse.Namespace, host: str | None = None) -> None: + """The implementation of the "build" command.""" + if host is None: + host = context.host + + if context.clean: + clean(context, host) + + if host in {"all", "build"}: + for step in [ + configure_build_python, + make_build_python, + pythoninfo_build_python, + ]: + step(context) + + if host == "build": + hosts = [] + elif host in {"all", "hosts"}: + hosts = all_host_triples(context.platform) + else: + hosts = [host] + + for step_host in hosts: + for step in [ + configure_host_python, + make_host_python, + ]: + step(context, host=step_host) + + if host == "all": + package(context) + + +def test(context: argparse.Namespace, host: str | None = None) -> None: # noqa: PT028 + """The implementation of the "test" command.""" + if host is None: + host = context.host + + if context.clean: + clean(context, "test") + + with group(f"Test {'XCframework' if host in {'all', 'hosts'} else host}"): + timestamp = str(time.time_ns())[:-6] + testbed_dir = ( + CROSS_BUILD_DIR / f"{context.platform}-testbed.{timestamp}" + ) + if host in {"all", "hosts"}: + framework_path = ( + CROSS_BUILD_DIR / context.platform / "Python.xcframework" + ) + else: + build_arch = platform.machine() + host_arch = host.split("-")[0] + + if not host.endswith("-simulator"): + print("Skipping test suite non-simulator build.") + return + elif build_arch != host_arch: + print( + f"Skipping test suite for an {host_arch} build " + f"on an {build_arch} machine." + ) + return + else: + framework_path = ( + CROSS_BUILD_DIR + / host + / f"Apple/{context.platform}" + / f"Frameworks/{apple_multiarch(host)}" + ) + + run([ + sys.executable, + "Apple/testbed", + "clone", + "--platform", + context.platform, + "--framework", + framework_path, + testbed_dir, + ]) + + run( + [ + sys.executable, + testbed_dir, + "run", + "--verbose", + ] + + ( + ["--simulator", str(context.simulator)] + if context.simulator + else [] + ) + + [ + "--", + "test", + f"--{context.ci_mode or 'fast'}-ci", + "--single-process", + "--no-randomize", + # Timeout handling requires subprocesses; explicitly setting + # the timeout to -1 disables the faulthandler. + "--timeout=-1", + # Adding Python options requires the use of a subprocess to + # start a new Python interpreter. + "--dont-add-python-opts", + ] + ) + + +def apple_sim_host(platform_name: str) -> str: + """Determine the native simulator target for this platform.""" + for _, slice_parts in HOSTS[platform_name].items(): + for host_triple in slice_parts: + parts = host_triple.split("-") + if parts[0] == platform.machine() and parts[-1] == "simulator": + return host_triple + + raise KeyError(platform_name) + + +def ci(context: argparse.Namespace) -> None: + """The implementation of the "ci" command. + + In "Fast" mode, this compiles the build python, and the simulator for the + build machine's architecture; and runs the test suite with `--fast-ci` + configuration. + + In "Slow" mode, it compiles the build python, plus all candidate + architectures (both device and simulator); then runs the test suite with + `--slow-ci` configuration. + """ + clean(context, "all") + if context.ci_mode == "slow": + # In slow mode, build and test the full XCframework + build(context, host="all") + test(context, host="all") + else: + # In fast mode, just build the simulator platform. + sim_host = apple_sim_host(context.platform) + build(context, host="build") + build(context, host=sim_host) + test(context, host=sim_host) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "A tool for managing the build, package and test process of " + "CPython on Apple platforms." + ), + ) + parser.suggest_on_error = True + subcommands = parser.add_subparsers(dest="subcommand", required=True) + + clean = subcommands.add_parser( + "clean", + help="Delete all build directories", + ) + + configure_build = subcommands.add_parser( + "configure-build", help="Run `configure` for the build Python" + ) + make_build = subcommands.add_parser( + "make-build", help="Run `make` for the build Python" + ) + pythoninfo_build = subcommands.add_parser( + "pythoninfo-build", help="Display build info of the build Python" + ) + configure_host = subcommands.add_parser( + "configure-host", + help="Run `configure` for a specific platform and target", + ) + make_host = subcommands.add_parser( + "make-host", + help="Run `make` for a specific platform and target", + ) + package = subcommands.add_parser( + "package", + help="Create a release package for the platform", + ) + build = subcommands.add_parser( + "build", + help="Build all platform targets and create the XCframework", + ) + test = subcommands.add_parser( + "test", + help="Run the testbed for a specific platform", + ) + ci = subcommands.add_parser( + "ci", + help="Run build, package, and test", + ) + + # platform argument + for cmd in [clean, configure_host, make_host, package, build, test, ci]: + cmd.add_argument( + "platform", + choices=HOSTS.keys(), + help="The target platform to build", + ) + + # host triple argument + for cmd in [configure_host, make_host]: + cmd.add_argument( + "host", + help="The host triple to build (e.g., arm64-apple-ios-simulator)", + ) + # optional host triple argument + for cmd in [clean, build, test]: + cmd.add_argument( + "host", + nargs="?", + default="all", + help=( + "The host triple to build (e.g., arm64-apple-ios-simulator), " + "or 'build' for just the build platform, or 'hosts' for all " + "host platforms, or 'all' for the build platform and all " + "hosts. Defaults to 'all'" + ), + ) + + # --cross-build-dir argument + for cmd in [ + clean, + configure_build, + make_build, + pythoninfo_build, + configure_host, + make_host, + build, + package, + test, + ci, + ]: + cmd.add_argument( + "--cross-build-dir", + action="store", + default=os.environ.get("CROSS_BUILD_DIR"), + dest="cross_build_dir", + type=Path, + help=( + "Path to the cross-build directory " + f"(default: {CROSS_BUILD_DIR}). Can also be set " + "with the CROSS_BUILD_DIR environment variable." + ), + ) + + # --clean option + for cmd in [configure_build, configure_host, build, package, test, ci]: + cmd.add_argument( + "--clean", + action="store_true", + default=False, + dest="clean", + help="Delete the relevant build directories first", + ) + + # --cache-dir option + for cmd in [configure_host, build, ci]: + cmd.add_argument( + "--cache-dir", + default=os.environ.get("CACHE_DIR"), + help="The directory to store cached downloads.", + ) + + # --simulator option + for cmd in [test, ci]: + cmd.add_argument( + "--simulator", + help=( + "The name of the simulator to use (eg: 'iPhone 16e'). " + "Defaults to the most recently released 'entry level' " + "iPhone device. Device architecture and OS version can also " + "be specified; e.g., " + "`--simulator 'iPhone 16 Pro,arch=arm64,OS=26.0'` would " + "run on an ARM64 iPhone 16 Pro simulator running iOS 26.0." + ), + ) + group = cmd.add_mutually_exclusive_group() + group.add_argument( + "--fast-ci", + action="store_const", + dest="ci_mode", + const="fast", + help="Add test arguments for GitHub Actions", + ) + group.add_argument( + "--slow-ci", + action="store_const", + dest="ci_mode", + const="slow", + help="Add test arguments for buildbots", + ) + + for subcommand in [configure_build, configure_host, build, ci]: + subcommand.add_argument( + "args", nargs="*", help="Extra arguments to pass to `configure`" + ) + + return parser.parse_args() + + +def print_called_process_error(e: subprocess.CalledProcessError) -> None: + for stream_name in ["stdout", "stderr"]: + content = getattr(e, stream_name) + stream = getattr(sys, stream_name) + if content: + stream.write(content) + if not content.endswith("\n"): + stream.write("\n") + + # shlex uses single quotes, so we surround the command with double quotes. + print( + f'Command "{join_command(e.cmd)}" returned exit status {e.returncode}' + ) + + +def main() -> None: + # Handle SIGTERM the same way as SIGINT. This ensures that if we're + # terminated by the buildbot worker, we'll make an attempt to clean up our + # subprocesses. + def signal_handler(*args): + os.kill(os.getpid(), signal.SIGINT) + + signal.signal(signal.SIGTERM, signal_handler) + + # Process command line arguments + context = parse_args() + + # Set the CROSS_BUILD_DIR if an argument was provided + if context.cross_build_dir: + global CROSS_BUILD_DIR + CROSS_BUILD_DIR = context.cross_build_dir.resolve() + + dispatch: dict[str, Callable] = { + "clean": clean, + "configure-build": configure_build_python, + "make-build": make_build_python, + "pythoninfo-build": pythoninfo_build_python, + "configure-host": configure_host_python, + "make-host": make_host_python, + "package": package, + "build": build, + "test": test, + "ci": ci, + } + + try: + dispatch[context.subcommand](context) + except CalledProcessError as e: + print() + print_called_process_error(e) + sys.exit(1) + except RuntimeError as e: + print() + print(e) + sys.exit(2) + + +if __name__ == "__main__": + # Under the buildbot, stdout is not a TTY, but we must still flush after + # every line to make sure our output appears in the correct order relative + # to the output of our subprocesses. + for stream in [sys.stdout, sys.stderr]: + stream.reconfigure(line_buffering=True) + + main() diff --git a/Apple/iOS/README.md b/Apple/iOS/README.md new file mode 100644 index 000000000000000..7ee257b5d648f46 --- /dev/null +++ b/Apple/iOS/README.md @@ -0,0 +1,339 @@ +# Python on iOS README + +**iOS support is [tier 3](https://peps.python.org/pep-0011/#tier-3).** + +This document provides a quick overview of some iOS specific features in the +Python distribution. + +These instructions are only needed if you're planning to compile Python for iOS +yourself. Most users should *not* need to do this. If you're looking to +experiment with writing an iOS app in Python, tools such as [BeeWare's +Briefcase](https://briefcase.readthedocs.io) and [Kivy's +Buildozer](https://buildozer.readthedocs.io) will provide a much more +approachable user experience. + +## Compilers for building on iOS + +Building for iOS requires the use of Apple's Xcode tooling. It is strongly +recommended that you use the most recent stable release of Xcode. This will +require the use of the most (or second-most) recently released macOS version, +as Apple does not maintain Xcode for older macOS versions. The Xcode Command +Line Tools are not sufficient for iOS development; you need a *full* Xcode +install. + +If you want to run your code on the iOS simulator, you'll also need to install +an iOS Simulator Platform. You should be prompted to select an iOS Simulator +Platform when you first run Xcode. Alternatively, you can add an iOS Simulator +Platform by selecting an open the Platforms tab of the Xcode Settings panel. + +## Building Python on iOS + +### ABIs and Architectures + +iOS apps can be deployed on physical devices, and on the iOS simulator. Although +the API used on these devices is identical, the ABI is different - you need to +link against different libraries for an iOS device build (`iphoneos`) or an +iOS simulator build (`iphonesimulator`). + +Apple uses the `XCframework` format to allow specifying a single dependency +that supports multiple ABIs. An `XCframework` is a wrapper around multiple +ABI-specific frameworks that share a common API. + +iOS can also support different CPU architectures within each ABI. At present, +there is only a single supported architecture on physical devices - ARM64. +However, the *simulator* supports 2 architectures - ARM64 (for running on Apple +Silicon machines), and x86_64 (for running on older Intel-based machines). + +To support multiple CPU architectures on a single platform, Apple uses a "fat +binary" format - a single physical file that contains support for multiple +architectures. It is possible to compile and use a "thin" single architecture +version of a binary for testing purposes; however, the "thin" binary will not be +portable to machines using other architectures. + +### Building a multi-architecture iOS XCframework + +The `Apple` subfolder of the Python repository acts as a build script that +can be used to coordinate the compilation of a complete iOS XCframework. To use +it, run:: + + python Apple build iOS + +This will: + +* Configure and compile a version of Python to run on the build machine +* Download pre-compiled binary dependencies for each platform +* Configure and build a `Python.framework` for each required architecture and + iOS SDK +* Merge the multiple `Python.framework` folders into a single `Python.xcframework` +* Produce a `.tar.gz` archive in the `cross-build/dist` folder containing + the `Python.xcframework`, plus a copy of the Testbed app pre-configured to + use the XCframework. + +The `Apple` build script has other entry points that will perform the +individual parts of the overall `build` target, plus targets to test the +build, clean the `cross-build` folder of iOS build products, and perform a +complete "build and test" CI run. The `--clean` flag can also be used on +individual commands to ensure that a stale build product are removed before +building. + +### Building a single-architecture framework + +If you're using the `Apple` build script, you won't need to build +individual frameworks. However, if you do need to manually configure an iOS +Python build for a single framework, the following options are available. + +#### iOS specific arguments to configure + +* `--enable-framework[=DIR]` + + This argument specifies the location where the Python.framework will be + installed. If `DIR` is not specified, the framework will be installed into + a subdirectory of the `iOS/Frameworks` folder. + + This argument *must* be provided when configuring iOS builds. iOS does not + support non-framework builds. + +* `--with-framework-name=NAME` + + Specify the name for the Python framework; defaults to `Python`. + + > [!NOTE] + > Unless you know what you're doing, changing the name of the Python + > framework on iOS is not advised. If you use this option, you won't be able + > to run the `Apple` build script without making significant manual + > alterations, and you won't be able to use any binary packages unless you + > compile them yourself using your own framework name. + +#### Building Python for iOS + +The Python build system will create a `Python.framework` that supports a +*single* ABI with a *single* architecture. Unlike macOS, iOS does not allow a +framework to contain non-library content, so the iOS build will produce a +`bin` and `lib` folder in the same output folder as `Python.framework`. +The `lib` folder will be needed at runtime to support the Python library. + +If you want to use Python in a real iOS project, you need to produce multiple +`Python.framework` builds, one for each ABI and architecture. iOS builds of +Python *must* be constructed as framework builds. To support this, you must +provide the `--enable-framework` flag when configuring the build. The build +also requires the use of cross-compilation. The minimal commands for building +Python for the ARM64 iOS simulator will look something like: +``` +export PATH="$(pwd)/Apple/iOS/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" +./configure \ + --enable-framework \ + --host=arm64-apple-ios-simulator \ + --build=arm64-apple-darwin \ + --with-build-python=/path/to/python.exe +make +make install +``` + +In this invocation: + +* `Apple/iOS/Resources/bin` has been added to the path, providing some shims for the + compilers and linkers needed by the build. Xcode requires the use of `xcrun` + to invoke compiler tooling. However, if `xcrun` is pre-evaluated and the + result passed to `configure`, these results can embed user- and + version-specific paths into the sysconfig data, which limits the portability + of the compiled Python. Alternatively, if `xcrun` is used *as* the compiler, + it requires that compiler variables like `CC` include spaces, which can + cause significant problems with many C configuration systems which assume that + `CC` will be a single executable. + + To work around this problem, the `Apple/iOS/Resources/bin` folder contains some + wrapper scripts that present as simple compilers and linkers, but wrap + underlying calls to `xcrun`. This allows configure to use a `CC` + definition without spaces, and without user- or version-specific paths, while + retaining the ability to adapt to the local Xcode install. These scripts are + included in the `bin` directory of an iOS install. + + These scripts will, by default, use the currently active Xcode installation. + If you want to use a different Xcode installation, you can use + `xcode-select` to set a new default Xcode globally, or you can use the + `DEVELOPER_DIR` environment variable to specify an Xcode install. The + scripts will use the default `iphoneos`/`iphonesimulator` SDK version for + the select Xcode install; if you want to use a different SDK, you can set the + `IOS_SDK_VERSION` environment variable. (e.g, setting + `IOS_SDK_VERSION=17.1` would cause the scripts to use the `iphoneos17.1` + and `iphonesimulator17.1` SDKs, regardless of the Xcode default.) + + The path has also been cleared of any user customizations. A common source of + bugs is for tools like Homebrew to accidentally leak macOS binaries into an iOS + build. Resetting the path to a known "bare bones" value is the easiest way to + avoid these problems. + +* `--host` is the architecture and ABI that you want to build, in GNU compiler + triple format. This will be one of: + + - `arm64-apple-ios` for ARM64 iOS devices. + - `arm64-apple-ios-simulator` for the iOS simulator running on Apple + Silicon devices. + - `x86_64-apple-ios-simulator` for the iOS simulator running on Intel + devices. + +* `--build` is the GNU compiler triple for the machine that will be running + the compiler. This is one of: + + - `arm64-apple-darwin` for Apple Silicon devices. + - `x86_64-apple-darwin` for Intel devices. + +* `/path/to/python.exe` is the path to a Python binary on the machine that + will be running the compiler. This is needed because the Python compilation + process involves running some Python code. On a normal desktop build of + Python, you can compile a python interpreter and then use that interpreter to + run Python code. However, the binaries produced for iOS won't run on macOS, so + you need to provide an external Python interpreter. This interpreter must be + the same version as the Python that is being compiled. To be completely safe, + this should be the *exact* same commit hash. However, the longer a Python + release has been stable, the more likely it is that this constraint can be + relaxed - the same micro version will often be sufficient. + +* The `install` target for iOS builds is slightly different to other + platforms. On most platforms, `make install` will install the build into + the final runtime location. This won't be the case for iOS, as the final + runtime location will be on a physical device. + + However, you still need to run the `install` target for iOS builds, as it + performs some final framework assembly steps. The location specified with + `--enable-framework` will be the location where `make install` will + assemble the complete iOS framework. This completed framework can then + be copied and relocated as required. + +For a full CPython build, you also need to specify the paths to iOS builds of +the binary libraries that CPython depends on (such as XZ, LibFFI and OpenSSL). +This can be done by defining library specific environment variables (such as +`LIBLZMA_CFLAGS`, `LIBLZMA_LIBS`), and the `--with-openssl` configure +option. Versions of these libraries pre-compiled for iOS can be found in [this +repository](https://github.com/beeware/cpython-apple-source-deps/releases). +LibFFI is especially important, as many parts of the standard library +(including the `platform`, `sysconfig` and `webbrowser` modules) require +the use of the `ctypes` module at runtime. + +By default, Python will be compiled with an iOS deployment target (i.e., the +minimum supported iOS version) of 13.0. To specify a different deployment +target, provide the version number as part of the `--host` argument - for +example, `--host=arm64-apple-ios15.4-simulator` would compile an ARM64 +simulator build with a deployment target of 15.4. + +## Testing Python on iOS + +### Testing a multi-architecture framework + +Once you have a built an XCframework, you can test that framework by running: + + $ python Apple test iOS + +This test will attempt to find an "SE-class" simulator (i.e., an iPhone SE, or +iPhone 16e, or similar), and run the test suite on the most recent version of +iOS that is available. You can specify a simulator using the `--simulator` +command line argument, providing the name of the simulator (e.g., `--simulator +'iPhone 16 Pro'`). You can also use this argument to control the OS version used +for testing; `--simulator 'iPhone 16 Pro,OS=18.2'` would attempt to run the +tests on an iPhone 16 Pro running iOS 18.2. + +If the test runner is executed on GitHub Actions, the `GITHUB_ACTIONS` +environment variable will be exposed to the iOS process at runtime. + +### Testing a single-architecture framework + +The `Apple/testbed` folder that contains an Xcode project that is able to run +the Python test suite on Apple platforms. This project converts the Python test +suite into a single test case in Xcode's XCTest framework. The single XCTest +passes if the test suite passes. + +To run the test suite, configure a Python build for an iOS simulator (i.e., +`--host=arm64-apple-ios-simulator` or `--host=x86_64-apple-ios-simulator` +), specifying a framework build (i.e. `--enable-framework`). Ensure that your +`PATH` has been configured to include the `Apple/iOS/Resources/bin` folder and +exclude any non-iOS tools, then run: +``` +make all +make install +make testios +``` + +This will: + +* Build an iOS framework for your chosen architecture; +* Finalize the single-platform framework; +* Make a clean copy of the testbed project; +* Install the Python iOS framework into the copy of the testbed project; and +* Run the test suite on an "entry-level device" simulator (i.e., an iPhone SE, + iPhone 16e, or a similar). + +On success, the test suite will exit and report successful completion of the +test suite. On a 2022 M1 MacBook Pro, the test suite takes approximately 15 +minutes to run; a couple of extra minutes is required to compile the testbed +project, and then boot and prepare the iOS simulator. + +### Debugging test failures + +Running `python Apple test iOS` generates a standalone version of the +`Apple/testbed` project, and runs the full test suite. It does this using +`Apple/testbed` itself - the folder is an executable module that can be used +to create and run a clone of the testbed project. The standalone version of the +testbed will be created in a directory named +`cross-build/iOS-testbed.`. + +You can generate your own standalone testbed instance by running: +``` +python cross-build/iOS/testbed clone my-testbed +``` + +In this invocation, `my-testbed` is the name of the folder for the new +testbed clone. + +If you've built your own XCframework, or you only want to test a single architecture, +you can construct a standalone testbed instance by running: +``` +python Apple/testbed clone --platform iOS --framework my-testbed +``` + +The framework path can be the path path to a `Python.xcframework`, or the +path to a folder that contains a single-platform `Python.framework`. + +You can then use the `my-testbed` folder to run the Python test suite, +passing in any command line arguments you may require. For example, if you're +trying to diagnose a failure in the `os` module, you might run: +``` +python my-testbed run -- test -W test_os +``` + +This is the equivalent of running `python -m test -W test_os` on a desktop +Python build. Any arguments after the `--` will be passed to testbed as if +they were arguments to `python -m` on a desktop machine. + +### Testing in Xcode + +You can also open the testbed project in Xcode by running: +``` +open my-testbed/iOSTestbed.xcodeproj +``` + +This will allow you to use the full Xcode suite of tools for debugging. + +The arguments used to run the test suite are defined as part of the test plan. +To modify the test plan, select the test plan node of the project tree (it +should be the first child of the root node), and select the "Configurations" +tab. Modify the "Arguments Passed On Launch" value to change the testing +arguments. + +The test plan also disables parallel testing, and specifies the use of the +`Testbed.lldbinit` file for providing configuration of the debugger. The +default debugger configuration disables automatic breakpoints on the +`SIGINT`, `SIGUSR1`, `SIGUSR2`, and `SIGXFSZ` signals. + +### Testing on an iOS device + +To test on an iOS device, the app needs to be signed with known developer +credentials. To obtain these credentials, you must have an iOS Developer +account, and your Xcode install will need to be logged into your account (see +the Accounts tab of the Preferences dialog). + +Once the project is open, and you're signed into your Apple Developer account, +select the root node of the project tree (labeled "iOSTestbed"), then the +"Signing & Capabilities" tab in the details page. Select a development team +(this will likely be your own name), and plug in a physical device to your +macOS machine with a USB cable. You should then be able to select your physical +device from the list of targets in the pulldown in the Xcode titlebar. diff --git a/iOS/Resources/Info.plist.in b/Apple/iOS/Resources/Info.plist.in similarity index 100% rename from iOS/Resources/Info.plist.in rename to Apple/iOS/Resources/Info.plist.in diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-ar b/Apple/iOS/Resources/bin/arm64-apple-ios-ar new file mode 100755 index 000000000000000..3cf3eb218741fae --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} ar "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-clang b/Apple/iOS/Resources/bin/arm64-apple-ios-clang new file mode 100755 index 000000000000000..f50d5b5142fc764 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET} "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-clang++ b/Apple/iOS/Resources/bin/arm64-apple-ios-clang++ new file mode 100755 index 000000000000000..0794731d7dcbda0 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang++ -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET} "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-cpp b/Apple/iOS/Resources/bin/arm64-apple-ios-cpp new file mode 100755 index 000000000000000..24fa1506bab8270 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET} -E "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-ar b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-ar new file mode 100755 index 000000000000000..b836b6db9025bb8 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang new file mode 100755 index 000000000000000..4891a00876e0bdd --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ new file mode 100755 index 000000000000000..58b2a5f6f18c2b7 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-cpp b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-cpp new file mode 100755 index 000000000000000..c9df94e8b7c8371 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target arm64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator -E "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-strip b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-strip new file mode 100755 index 000000000000000..fd59d309b73a202 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-simulator-strip @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} strip -arch arm64 "$@" diff --git a/Apple/iOS/Resources/bin/arm64-apple-ios-strip b/Apple/iOS/Resources/bin/arm64-apple-ios-strip new file mode 100755 index 000000000000000..75e823a3d02d615 --- /dev/null +++ b/Apple/iOS/Resources/bin/arm64-apple-ios-strip @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} strip -arch arm64 "$@" diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-ar b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-ar new file mode 100755 index 000000000000000..b836b6db9025bb8 --- /dev/null +++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-ar @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} ar "$@" diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang new file mode 100755 index 000000000000000..f4739a7b945d010 --- /dev/null +++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@" diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ new file mode 100755 index 000000000000000..c348ae4c10395bd --- /dev/null +++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-clang++ @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang++ -target x86_64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator "$@" diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp new file mode 100755 index 000000000000000..6d7f8084c9fdccf --- /dev/null +++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-cpp @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} clang -target x86_64-apple-ios${IPHONEOS_DEPLOYMENT_TARGET}-simulator -E "$@" diff --git a/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-strip b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-strip new file mode 100755 index 000000000000000..c5cfb28929195aa --- /dev/null +++ b/Apple/iOS/Resources/bin/x86_64-apple-ios-simulator-strip @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} strip -arch x86_64 "$@" diff --git a/iOS/Resources/pyconfig.h b/Apple/iOS/Resources/pyconfig.h similarity index 100% rename from iOS/Resources/pyconfig.h rename to Apple/iOS/Resources/pyconfig.h diff --git a/iOS/testbed/Python.xcframework/Info.plist b/Apple/testbed/Python.xcframework/Info.plist similarity index 100% rename from iOS/testbed/Python.xcframework/Info.plist rename to Apple/testbed/Python.xcframework/Info.plist diff --git a/iOS/Resources/dylib-Info-template.plist b/Apple/testbed/Python.xcframework/build/iOS-dylib-Info-template.plist similarity index 96% rename from iOS/Resources/dylib-Info-template.plist rename to Apple/testbed/Python.xcframework/build/iOS-dylib-Info-template.plist index f652e272f71c88a..d6caa01c1e44b97 100644 --- a/iOS/Resources/dylib-Info-template.plist +++ b/Apple/testbed/Python.xcframework/build/iOS-dylib-Info-template.plist @@ -19,7 +19,7 @@ iPhoneOS MinimumOSVersion - 12.0 + 13.0 CFBundleVersion 1 diff --git a/Apple/testbed/Python.xcframework/build/utils.sh b/Apple/testbed/Python.xcframework/build/utils.sh new file mode 100755 index 000000000000000..e54471f68b7cb2c --- /dev/null +++ b/Apple/testbed/Python.xcframework/build/utils.sh @@ -0,0 +1,151 @@ +# Utility methods for use in an Xcode project. +# +# An iOS XCframework cannot include any content other than the library binary +# and relevant metadata. However, Python requires a standard library at runtime. +# Therefore, it is necessary to add a build step to an Xcode app target that +# processes the standard library and puts the content into the final app. +# +# In general, these tools will be invoked after bundle resources have been +# copied into the app, but before framework embedding (and signing). +# +# The following is an example script, assuming that: +# * Python.xcframework is in the root of the project +# * There is an `app` folder that contains the app code +# * There is an `app_packages` folder that contains installed Python packages. +# ----- +# set -e +# source $PROJECT_DIR/Python.xcframework/build/build_utils.sh +# install_python Python.xcframework app app_packages +# ----- + +# Copy the standard library from the XCframework into the app bundle. +# +# Accepts one argument: +# 1. The path, relative to the root of the Xcode project, where the Python +# XCframework can be found. +install_stdlib() { + PYTHON_XCFRAMEWORK_PATH=$1 + + mkdir -p "$CODESIGNING_FOLDER_PATH/python/lib" + if [ "$EFFECTIVE_PLATFORM_NAME" = "-iphonesimulator" ]; then + echo "Installing Python modules for iOS Simulator" + if [ -d "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/ios-arm64-simulator" ]; then + SLICE_FOLDER="ios-arm64-simulator" + else + SLICE_FOLDER="ios-arm64_x86_64-simulator" + fi + else + echo "Installing Python modules for iOS Device" + SLICE_FOLDER="ios-arm64" + fi + + # If the XCframework has a shared lib folder, then it's a full framework. + # Copy both the common and slice-specific part of the lib directory. + # Otherwise, it's a single-arch framework; use the "full" lib folder. + # Don't include any libpython symlink; that can't be included at runtime + if [ -d "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/lib" ]; then + rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" --exclude 'libpython*.dylib' + rsync -au "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib-$ARCHS/" "$CODESIGNING_FOLDER_PATH/python/lib/" --exclude 'libpython*.dylib' + else + rsync -au --delete "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/$SLICE_FOLDER/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" --exclude 'libpython*.dylib' + fi +} + +# Convert a single .so library into a framework that iOS can load. +# +# Accepts three arguments: +# 1. The path, relative to the root of the Xcode project, where the Python +# XCframework can be found. +# 2. The base path, relative to the installed location in the app bundle, that +# needs to be processed. Any .so file found in this path (or a subdirectory +# of it) will be processed. +# 2. The full path to a single .so file to process. This path should include +# the base path. +install_dylib () { + PYTHON_XCFRAMEWORK_PATH=$1 + INSTALL_BASE=$2 + FULL_EXT=$3 + + # The name of the extension file + EXT=$(basename "$FULL_EXT") + # The name and location of the module + MODULE_PATH=$(dirname "$FULL_EXT") + MODULE_NAME=$(echo $EXT | cut -d "." -f 1) + # The location of the extension file, relative to the bundle + RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} + # The path to the extension file, relative to the install base + PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/} + # The full dotted name of the extension module, constructed from the file path. + FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d "." -f 1 | tr "/" "."); + # A bundle identifier; not actually used, but required by Xcode framework packaging + FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr "_" "-") + # The name of the framework folder. + FRAMEWORK_FOLDER="Frameworks/$FULL_MODULE_NAME.framework" + + # If the framework folder doesn't exist, create it. + if [ ! -d "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" ]; then + echo "Creating framework for $RELATIVE_EXT" + mkdir -p "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" + cp "$PROJECT_DIR/$PYTHON_XCFRAMEWORK_PATH/build/$PLATFORM_FAMILY_NAME-dylib-Info-template.plist" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" + plutil -replace CFBundleExecutable -string "$FULL_MODULE_NAME" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" + plutil -replace CFBundleIdentifier -string "$FRAMEWORK_BUNDLE_ID" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" + fi + + echo "Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME" + mv "$FULL_EXT" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" + # Create a placeholder .fwork file where the .so was + echo "$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" > ${FULL_EXT%.so}.fwork + # Create a back reference to the .so file location in the framework + echo "${RELATIVE_EXT%.so}.fwork" > "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin" + + # If the framework provides an xcprivacy file, install it. + if [ -e "$MODULE_PATH/$MODULE_NAME.xcprivacy" ]; then + echo "Installing XCPrivacy file for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME" + XCPRIVACY_FILE="$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/PrivacyInfo.xcprivacy" + if [ -e "$XCPRIVACY_FILE" ]; then + rm -rf "$XCPRIVACY_FILE" + fi + mv "$MODULE_PATH/$MODULE_NAME.xcprivacy" "$XCPRIVACY_FILE" + fi + + echo "Signing framework as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)..." + /usr/bin/codesign --force --sign "$EXPANDED_CODE_SIGN_IDENTITY" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" +} + +# Process all the dynamic libraries in a path into Framework format. +# +# Accepts two arguments: +# 1. The path, relative to the root of the Xcode project, where the Python +# XCframework can be found. +# 2. The base path, relative to the installed location in the app bundle, that +# needs to be processed. Any .so file found in this path (or a subdirectory +# of it) will be processed. +process_dylibs () { + PYTHON_XCFRAMEWORK_PATH=$1 + LIB_PATH=$2 + find "$CODESIGNING_FOLDER_PATH/$LIB_PATH" -name "*.so" | while read FULL_EXT; do + install_dylib $PYTHON_XCFRAMEWORK_PATH "$LIB_PATH/" "$FULL_EXT" + done +} + +# The entry point for post-processing a Python XCframework. +# +# Accepts 1 or more arguments: +# 1. The path, relative to the root of the Xcode project, where the Python +# XCframework can be found. If the XCframework is in the root of the project, +# 2+. The path of a package, relative to the root of the packaged app, that contains +# library content that should be processed for binary libraries. +install_python() { + PYTHON_XCFRAMEWORK_PATH=$1 + shift + + install_stdlib $PYTHON_XCFRAMEWORK_PATH + PYTHON_VER=$(ls -1 "$CODESIGNING_FOLDER_PATH/python/lib" | grep -E "^python3\.\d+$") + echo "Install Python $PYTHON_VER standard library extension modules..." + process_dylibs $PYTHON_XCFRAMEWORK_PATH python/lib/$PYTHON_VER/lib-dynload + + for package_path in $@; do + echo "Installing $package_path extension modules ..." + process_dylibs $PYTHON_XCFRAMEWORK_PATH $package_path + done +} diff --git a/iOS/testbed/Python.xcframework/ios-arm64/README b/Apple/testbed/Python.xcframework/ios-arm64/README similarity index 100% rename from iOS/testbed/Python.xcframework/ios-arm64/README rename to Apple/testbed/Python.xcframework/ios-arm64/README diff --git a/iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README b/Apple/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README similarity index 100% rename from iOS/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README rename to Apple/testbed/Python.xcframework/ios-arm64_x86_64-simulator/README diff --git a/Apple/testbed/Testbed.lldbinit b/Apple/testbed/Testbed.lldbinit new file mode 100644 index 000000000000000..4cf00dd0f9de1d7 --- /dev/null +++ b/Apple/testbed/Testbed.lldbinit @@ -0,0 +1,4 @@ +process handle SIGINT -n true -p true -s false +process handle SIGUSR1 -n true -p true -s false +process handle SIGUSR2 -n true -p true -s false +process handle SIGXFSZ -n true -p true -s false diff --git a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m b/Apple/testbed/TestbedTests/TestbedTests.m similarity index 69% rename from iOS/testbed/iOSTestbedTests/iOSTestbedTests.m rename to Apple/testbed/TestbedTests/TestbedTests.m index dd6e76f9496fe08..f7788c47f2c2296 100644 --- a/iOS/testbed/iOSTestbedTests/iOSTestbedTests.m +++ b/Apple/testbed/TestbedTests/TestbedTests.m @@ -1,11 +1,11 @@ #import #import -@interface iOSTestbedTests : XCTestCase +@interface TestbedTests : XCTestCase @end -@implementation iOSTestbedTests +@implementation TestbedTests - (void)testPython { @@ -15,6 +15,11 @@ - (void)testPython { PyStatus status; PyPreConfig preconfig; PyConfig config; + PyObject *app_packages_path; + PyObject *method_args; + PyObject *result; + PyObject *site_module; + PyObject *site_addsitedir_attr; PyObject *sys_module; PyObject *sys_path_attr; NSArray *test_args; @@ -30,19 +35,26 @@ - (void)testPython { setenv("NO_COLOR", "1", true); setenv("PYTHON_COLORS", "0", true); + if (getenv("GITHUB_ACTIONS")) { + NSLog(@"Running in a GitHub Actions environment"); + } // Arguments to pass into the test suite runner. // argv[0] must identify the process; any subsequent arg // will be handled as if it were an argument to `python -m test` - test_args = [[NSBundle mainBundle] objectForInfoDictionaryKey:@"TestArgs"]; + // The processInfo arguments contain the binary that is running, + // followed by the arguments defined in the test plan. This means: + // run_module = test_args[1] + // argv = ["Testbed"] + test_args[2:] + test_args = [[NSProcessInfo processInfo] arguments]; if (test_args == NULL) { NSLog(@"Unable to identify test arguments."); } - argv = malloc(sizeof(char *) * ([test_args count] + 1)); - argv[0] = "iOSTestbed"; - for (int i = 1; i < [test_args count]; i++) { - argv[i] = [[test_args objectAtIndex:i] UTF8String]; + NSLog(@"Test arguments: %@", test_args); + argv = malloc(sizeof(char *) * ([test_args count] - 1)); + argv[0] = "Testbed"; + for (int i = 1; i < [test_args count] - 1; i++) { + argv[i] = [[test_args objectAtIndex:i+1] UTF8String]; } - NSLog(@"Test command: %@", test_args); // Generate an isolated Python configuration. NSLog(@"Configuring isolated Python..."); @@ -63,7 +75,7 @@ - (void)testPython { // Ensure that signal handlers are installed config.install_signal_handlers = 1; // Run the test module. - config.run_module = Py_DecodeLocale([[test_args objectAtIndex:0] UTF8String], NULL); + config.run_module = Py_DecodeLocale([[test_args objectAtIndex:1] UTF8String], NULL); // For debugging - enable verbose mode. // config.verbose = 1; @@ -96,7 +108,7 @@ - (void)testPython { } NSLog(@"Configure argc/argv..."); - status = PyConfig_SetBytesArgv(&config, [test_args count], (char**) argv); + status = PyConfig_SetBytesArgv(&config, [test_args count] - 1, (char**) argv); if (PyStatus_Exception(status)) { XCTFail(@"Unable to configure argc/argv: %s", status.err_msg); PyConfig_Clear(&config); @@ -111,29 +123,55 @@ - (void)testPython { return; } - sys_module = PyImport_ImportModule("sys"); - if (sys_module == NULL) { - XCTFail(@"Could not import sys module"); + // Add app_packages as a site directory. This both adds to sys.path, + // and ensures that any .pth files in that directory will be executed. + site_module = PyImport_ImportModule("site"); + if (site_module == NULL) { + XCTFail(@"Could not import site module"); return; } - sys_path_attr = PyObject_GetAttrString(sys_module, "path"); - if (sys_path_attr == NULL) { - XCTFail(@"Could not access sys.path"); + site_addsitedir_attr = PyObject_GetAttrString(site_module, "addsitedir"); + if (site_addsitedir_attr == NULL || !PyCallable_Check(site_addsitedir_attr)) { + XCTFail(@"Could not access site.addsitedir"); return; } - // Add the app packages path path = [NSString stringWithFormat:@"%@/app_packages", resourcePath, nil]; NSLog(@"App packages path: %@", path); wtmp_str = Py_DecodeLocale([path UTF8String], NULL); - failed = PyList_Insert(sys_path_attr, 0, PyUnicode_FromString([path UTF8String])); - if (failed) { - XCTFail(@"Unable to add app packages to sys.path"); + app_packages_path = PyUnicode_FromWideChar(wtmp_str, wcslen(wtmp_str)); + if (app_packages_path == NULL) { + XCTFail(@"Could not convert app_packages path to unicode"); return; } PyMem_RawFree(wtmp_str); + method_args = Py_BuildValue("(O)", app_packages_path); + if (method_args == NULL) { + XCTFail(@"Could not create arguments for site.addsitedir"); + return; + } + + result = PyObject_CallObject(site_addsitedir_attr, method_args); + if (result == NULL) { + XCTFail(@"Could not add app_packages directory using site.addsitedir"); + return; + } + + // Add test code to sys.path + sys_module = PyImport_ImportModule("sys"); + if (sys_module == NULL) { + XCTFail(@"Could not import sys module"); + return; + } + + sys_path_attr = PyObject_GetAttrString(sys_module, "path"); + if (sys_path_attr == NULL) { + XCTFail(@"Could not access sys.path"); + return; + } + path = [NSString stringWithFormat:@"%@/app", resourcePath, nil]; NSLog(@"App path: %@", path); wtmp_str = Py_DecodeLocale([path UTF8String], NULL); diff --git a/Apple/testbed/__main__.py b/Apple/testbed/__main__.py new file mode 100644 index 000000000000000..797b522678f9d74 --- /dev/null +++ b/Apple/testbed/__main__.py @@ -0,0 +1,441 @@ +import argparse +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +from pathlib import Path + +TEST_SLICES = { + "iOS": "ios-arm64_x86_64-simulator", +} + +DECODE_ARGS = ("UTF-8", "backslashreplace") + +# The system log prefixes each line: +# 2025-01-17 16:14:29.093742+0800 iOSTestbed[23987:1fd393b4] ... +# 2025-01-17 16:14:29.093742+0800 iOSTestbed[23987:1fd393b4] ... + +LOG_PREFIX_REGEX = re.compile( + r"^\d{4}-\d{2}-\d{2}" # YYYY-MM-DD + r"\s+\d+:\d{2}:\d{2}\.\d+\+\d{4}" # HH:MM:SS.ssssss+ZZZZ + r"\s+iOSTestbed\[\d+:\w+\] " # Process/thread ID +) + + +# Select a simulator device to use. +def select_simulator_device(platform): + # List the testing simulators, in JSON format + raw_json = subprocess.check_output(["xcrun", "simctl", "list", "-j"]) + json_data = json.loads(raw_json) + + if platform == "iOS": + # Any iOS device will do; we'll look for "SE" devices - but the name + # isn't consistent over time. Older Xcode versions will use "iPhone SE + # (Nth generation)"; As of 2025, they've started using "iPhone 16e". + # + # When Xcode is updated after a new release, new devices will be + # available and old ones will be dropped from the set available on the + # latest iOS version. Select the one with the highest minimum runtime + # version - this is an indicator of the "newest" released device, which + # should always be supported on the "most recent" iOS version. + se_simulators = sorted( + (devicetype["minRuntimeVersion"], devicetype["name"]) + for devicetype in json_data["devicetypes"] + if devicetype["productFamily"] == "iPhone" + and ( + ( + "iPhone " in devicetype["name"] + and devicetype["name"].endswith("e") + ) + or "iPhone SE " in devicetype["name"] + ) + ) + simulator = se_simulators[-1][1] + else: + raise ValueError(f"Unknown platform {platform}") + + return simulator + + +def xcode_test(location: Path, platform: str, simulator: str, verbose: bool): + # Build and run the test suite on the named simulator. + args = [ + "-project", + str(location / f"{platform}Testbed.xcodeproj"), + "-scheme", + f"{platform}Testbed", + "-destination", + f"platform={platform} Simulator,name={simulator}", + "-derivedDataPath", + str(location / "DerivedData"), + ] + verbosity_args = [] if verbose else ["-quiet"] + + print("Building test project...") + subprocess.run( + ["xcodebuild", "build-for-testing"] + args + verbosity_args, + check=True, + ) + + # Any environment variable prefixed with TEST_RUNNER_ is exposed into the + # test runner environment. There are some variables (like those identifying + # CI platforms) that can be useful to have access to. + test_env = os.environ.copy() + if "GITHUB_ACTIONS" in os.environ: + test_env["TEST_RUNNER_GITHUB_ACTIONS"] = os.environ["GITHUB_ACTIONS"] + + print("Running test project...") + # Test execution *can't* be run -quiet; verbose mode + # is how we see the output of the test output. + process = subprocess.Popen( + ["xcodebuild", "test-without-building"] + args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=test_env, + ) + while line := (process.stdout.readline()).decode(*DECODE_ARGS): + # Strip the timestamp/process prefix from each log line + line = LOG_PREFIX_REGEX.sub("", line) + sys.stdout.write(line) + sys.stdout.flush() + + status = process.wait(timeout=5) + exit(status) + + +def copy(src, tgt): + """An all-purpose copy. + + If src is a file, it is copied. If src is a symlink, it is copied *as a + symlink*. If src is a directory, the full tree is duplicated, with symlinks + being preserved. + """ + if src.is_file() or src.is_symlink(): + shutil.copyfile(src, tgt, follow_symlinks=False) + else: + shutil.copytree(src, tgt, symlinks=True) + + +def clone_testbed( + source: Path, + target: Path, + framework: Path, + platform: str, + apps: list[Path], +) -> None: + if target.exists(): + print(f"{target} already exists; aborting without creating project.") + sys.exit(10) + + if framework is None: + if not ( + source / "Python.xcframework" / TEST_SLICES[platform] / "bin" + ).is_dir(): + print( + f"The testbed being cloned ({source}) does not contain " + "a framework with slices. Re-run with --framework" + ) + sys.exit(11) + else: + if not framework.is_dir(): + print(f"{framework} does not exist.") + sys.exit(12) + elif not ( + framework.suffix == ".xcframework" + or (framework / "Python.framework").is_dir() + ): + print( + f"{framework} is not an XCframework, " + f"or a simulator slice of a framework build." + ) + sys.exit(13) + + print("Cloning testbed project:") + print(f" Cloning {source}...", end="") + # Only copy the files for the platform being cloned plus the files common + # to all platforms. The XCframework will be copied later, if needed. + target.mkdir(parents=True) + + for name in [ + "__main__.py", + "TestbedTests", + "Testbed.lldbinit", + f"{platform}Testbed", + f"{platform}Testbed.xcodeproj", + f"{platform}Testbed.xctestplan", + ]: + copy(source / name, target / name) + + print(" done") + + orig_xc_framework_path = source / "Python.xcframework" + xc_framework_path = target / "Python.xcframework" + test_framework_path = xc_framework_path / TEST_SLICES[platform] + if framework is not None: + if framework.suffix == ".xcframework": + print(" Installing XCFramework...", end="") + xc_framework_path.symlink_to( + framework.relative_to(xc_framework_path.parent, walk_up=True) + ) + print(" done") + else: + print(" Installing simulator framework...", end="") + # We're only installing a slice of a framework; we need + # to do a full tree copy to make sure we don't damage + # symlinked content. + shutil.copytree(orig_xc_framework_path, xc_framework_path) + if test_framework_path.is_dir(): + shutil.rmtree(test_framework_path) + else: + test_framework_path.unlink(missing_ok=True) + test_framework_path.symlink_to( + framework.relative_to(test_framework_path.parent, walk_up=True) + ) + print(" done") + else: + copy(orig_xc_framework_path, xc_framework_path) + + if ( + xc_framework_path.is_symlink() + and not xc_framework_path.readlink().is_absolute() + ): + # XCFramework is a relative symlink. Rewrite the symlink relative + # to the new location. + print(" Rewriting symlink to XCframework...", end="") + resolved_xc_framework_path = ( + source / xc_framework_path.readlink() + ).resolve() + xc_framework_path.unlink() + xc_framework_path.symlink_to( + resolved_xc_framework_path.relative_to( + xc_framework_path.parent, walk_up=True + ) + ) + print(" done") + elif ( + test_framework_path.is_symlink() + and not test_framework_path.readlink().is_absolute() + ): + print(" Rewriting symlink to simulator framework...", end="") + # Simulator framework is a relative symlink. Rewrite the symlink + # relative to the new location. + orig_test_framework_path = ( + source / "Python.XCframework" / test_framework_path.readlink() + ).resolve() + test_framework_path.unlink() + test_framework_path.symlink_to( + orig_test_framework_path.relative_to( + test_framework_path.parent, walk_up=True + ) + ) + print(" done") + else: + print(" Using pre-existing Python framework.") + + for app_src in apps: + print(f" Installing app {app_src.name!r}...", end="") + app_target = target / f"Testbed/app/{app_src.name}" + if app_target.is_dir(): + shutil.rmtree(app_target) + shutil.copytree(app_src, app_target) + print(" done") + + print(f"Successfully cloned testbed: {target.resolve()}") + + +def update_test_plan(testbed_path, platform, args): + # Modify the test plan to use the requested test arguments. + test_plan_path = testbed_path / f"{platform}Testbed.xctestplan" + with test_plan_path.open("r", encoding="utf-8") as f: + test_plan = json.load(f) + + test_plan["defaultOptions"]["commandLineArgumentEntries"] = [ + {"argument": shlex.quote(arg)} for arg in args + ] + + with test_plan_path.open("w", encoding="utf-8") as f: + json.dump(test_plan, f, indent=2) + + +def run_testbed( + platform: str, + simulator: str | None, + args: list[str], + verbose: bool = False, +): + location = Path(__file__).parent + print("Updating test plan...", end="") + update_test_plan(location, platform, args) + print(" done.") + + # xcodebuild doesn't guarantee that the CoreSimulatorService daemon is + # running prior to using a simulator, but calling `xcrun simctl list` does. + # Determining the default simulator that *would* be used is a cheap action; + # so use that as a way to ensure that the CoreSimulatorService daemon is + # running. + default_simulator = select_simulator_device(platform) + if simulator is None: + simulator = default_simulator + print(f"Running test on {simulator}") + + xcode_test( + location, + platform=platform, + simulator=simulator, + verbose=verbose, + ) + + +def main(): + # Look for directories like `iOSTestbed` as an indicator of the platforms + # that the testbed folder supports. The original source testbed can support + # many platforms, but when cloned, only one platform is preserved. + available_platforms = [ + platform + for platform in ["iOS"] + if (Path(__file__).parent / f"{platform}Testbed").is_dir() + ] + + parser = argparse.ArgumentParser( + description=( + "Manages the process of testing an Apple Python project " + "through Xcode." + ), + ) + + subcommands = parser.add_subparsers(dest="subcommand") + clone = subcommands.add_parser( + "clone", + description=( + "Clone the testbed project, copying in a Python framework and" + "any specified application code." + ), + help="Clone a testbed project to a new location.", + ) + clone.add_argument( + "--framework", + help=( + "The location of the XCFramework (or simulator-only slice of an " + "XCFramework) to use when running the testbed" + ), + ) + clone.add_argument( + "--platform", + dest="platform", + choices=available_platforms, + default=available_platforms[0], + help=f"The platform to target (default: {available_platforms[0]})", + ) + clone.add_argument( + "--app", + dest="apps", + action="append", + default=[], + help="The location of any code to include in the testbed project", + ) + clone.add_argument( + "location", + help="The path where the testbed will be cloned.", + ) + + run = subcommands.add_parser( + "run", + usage=( + "%(prog)s [-h] [--simulator SIMULATOR] -- " + " [ ...]" + ), + description=( + "Run a testbed project. The arguments provided after `--` will be " + "passed to the running iOS process as if they were arguments to " + "`python -m`." + ), + help="Run a testbed project", + ) + run.add_argument( + "--platform", + dest="platform", + choices=available_platforms, + default=available_platforms[0], + help=f"The platform to target (default: {available_platforms[0]})", + ) + run.add_argument( + "--simulator", + help=( + "The name of the simulator to use (eg: 'iPhone 16e'). Defaults to " + "the most recently released 'entry level' iPhone device. Device " + "architecture and OS version can also be specified; e.g., " + "`--simulator 'iPhone 16 Pro,arch=arm64,OS=26.0'` would run on " + "an ARM64 iPhone 16 Pro simulator running iOS 26.0." + ), + ) + run.add_argument( + "-v", + "--verbose", + action="store_true", + help="Enable verbose output", + ) + + try: + pos = sys.argv.index("--") + testbed_args = sys.argv[1:pos] + test_args = sys.argv[pos + 1 :] + except ValueError: + testbed_args = sys.argv[1:] + test_args = [] + + context = parser.parse_args(testbed_args) + + if context.subcommand == "clone": + clone_testbed( + source=Path(__file__).parent.resolve(), + target=Path(context.location).resolve(), + framework=Path(context.framework).resolve() + if context.framework + else None, + platform=context.platform, + apps=[Path(app) for app in context.apps], + ) + elif context.subcommand == "run": + if test_args: + if not ( + Path(__file__).parent + / "Python.xcframework" + / TEST_SLICES[context.platform] + / "bin" + ).is_dir(): + print( + "Testbed does not contain a compiled Python framework. " + f"Use `python {sys.argv[0]} clone ...` to create a " + "runnable clone of this testbed." + ) + sys.exit(20) + + run_testbed( + platform=context.platform, + simulator=context.simulator, + verbose=context.verbose, + args=test_args, + ) + else: + print( + "Must specify test arguments " + f"(e.g., {sys.argv[0]} run -- test)" + ) + print() + parser.print_help(sys.stderr) + sys.exit(21) + else: + parser.print_help(sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + # Under the buildbot, stdout is not a TTY, but we must still flush after + # every line to make sure our output appears in the correct order relative + # to the output of our subprocesses. + for stream in [sys.stdout, sys.stderr]: + stream.reconfigure(line_buffering=True) + main() diff --git a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj b/Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj similarity index 78% rename from iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj rename to Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj index c7d63909ee2453c..f8835a3bc587df5 100644 --- a/iOS/testbed/iOSTestbed.xcodeproj/project.pbxproj +++ b/Apple/testbed/iOSTestbed.xcodeproj/project.pbxproj @@ -11,12 +11,11 @@ 607A66222B0EFA390010BFC8 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 607A66212B0EFA390010BFC8 /* Assets.xcassets */; }; 607A66252B0EFA390010BFC8 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 607A66232B0EFA390010BFC8 /* LaunchScreen.storyboard */; }; 607A66282B0EFA390010BFC8 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66272B0EFA390010BFC8 /* main.m */; }; - 607A66322B0EFA3A0010BFC8 /* iOSTestbedTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */; }; + 607A66322B0EFA3A0010BFC8 /* TestbedTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 607A66312B0EFA3A0010BFC8 /* TestbedTests.m */; }; 607A664C2B0EFC080010BFC8 /* Python.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; }; 607A664D2B0EFC080010BFC8 /* Python.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; 607A66502B0EFFE00010BFC8 /* Python.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; }; 607A66512B0EFFE00010BFC8 /* Python.xcframework in Embed Frameworks */ = {isa = PBXBuildFile; fileRef = 607A664A2B0EFB310010BFC8 /* Python.xcframework */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; - 607A66582B0F079F0010BFC8 /* dylib-Info-template.plist in Resources */ = {isa = PBXBuildFile; fileRef = 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */; }; 608619542CB77BA900F46182 /* app_packages in Resources */ = {isa = PBXBuildFile; fileRef = 608619532CB77BA900F46182 /* app_packages */; }; 608619562CB7819B00F46182 /* app in Resources */ = {isa = PBXBuildFile; fileRef = 608619552CB7819B00F46182 /* app */; }; /* End PBXBuildFile section */ @@ -64,12 +63,12 @@ 607A66242B0EFA390010BFC8 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; 607A66272B0EFA390010BFC8 /* main.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = main.m; sourceTree = ""; }; 607A662D2B0EFA3A0010BFC8 /* iOSTestbedTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = iOSTestbedTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; - 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = iOSTestbedTests.m; sourceTree = ""; }; + 607A66312B0EFA3A0010BFC8 /* TestbedTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = TestbedTests.m; sourceTree = ""; }; 607A664A2B0EFB310010BFC8 /* Python.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; path = Python.xcframework; sourceTree = ""; }; - 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = "dylib-Info-template.plist"; sourceTree = ""; }; 607A66592B0F08600010BFC8 /* iOSTestbed-Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = "iOSTestbed-Info.plist"; sourceTree = ""; }; 608619532CB77BA900F46182 /* app_packages */ = {isa = PBXFileReference; lastKnownFileType = folder; path = app_packages; sourceTree = ""; }; 608619552CB7819B00F46182 /* app */ = {isa = PBXFileReference; lastKnownFileType = folder; path = app; sourceTree = ""; }; + 60FE0EFB2E56BB6D00524F87 /* iOSTestbed.xctestplan */ = {isa = PBXFileReference; lastKnownFileType = text; path = iOSTestbed.xctestplan; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -95,9 +94,10 @@ 607A66092B0EFA380010BFC8 = { isa = PBXGroup; children = ( + 60FE0EFB2E56BB6D00524F87 /* iOSTestbed.xctestplan */, 607A664A2B0EFB310010BFC8 /* Python.xcframework */, 607A66142B0EFA380010BFC8 /* iOSTestbed */, - 607A66302B0EFA3A0010BFC8 /* iOSTestbedTests */, + 607A66302B0EFA3A0010BFC8 /* TestbedTests */, 607A66132B0EFA380010BFC8 /* Products */, 607A664F2B0EFFE00010BFC8 /* Frameworks */, ); @@ -118,7 +118,6 @@ 608619552CB7819B00F46182 /* app */, 608619532CB77BA900F46182 /* app_packages */, 607A66592B0F08600010BFC8 /* iOSTestbed-Info.plist */, - 607A66572B0F079F0010BFC8 /* dylib-Info-template.plist */, 607A66152B0EFA380010BFC8 /* AppDelegate.h */, 607A66162B0EFA380010BFC8 /* AppDelegate.m */, 607A66212B0EFA390010BFC8 /* Assets.xcassets */, @@ -128,12 +127,12 @@ path = iOSTestbed; sourceTree = ""; }; - 607A66302B0EFA3A0010BFC8 /* iOSTestbedTests */ = { + 607A66302B0EFA3A0010BFC8 /* TestbedTests */ = { isa = PBXGroup; children = ( - 607A66312B0EFA3A0010BFC8 /* iOSTestbedTests.m */, + 607A66312B0EFA3A0010BFC8 /* TestbedTests.m */, ); - path = iOSTestbedTests; + path = TestbedTests; sourceTree = ""; }; 607A664F2B0EFFE00010BFC8 /* Frameworks */ = { @@ -153,8 +152,7 @@ 607A660E2B0EFA380010BFC8 /* Sources */, 607A660F2B0EFA380010BFC8 /* Frameworks */, 607A66102B0EFA380010BFC8 /* Resources */, - 607A66552B0F061D0010BFC8 /* Install Target Specific Python Standard Library */, - 607A66562B0F06200010BFC8 /* Prepare Python Binary Modules */, + 607A66552B0F061D0010BFC8 /* Process Python libraries */, 607A664E2B0EFC080010BFC8 /* Embed Frameworks */, ); buildRules = ( @@ -228,7 +226,6 @@ buildActionMask = 2147483647; files = ( 607A66252B0EFA390010BFC8 /* LaunchScreen.storyboard in Resources */, - 607A66582B0F079F0010BFC8 /* dylib-Info-template.plist in Resources */, 608619562CB7819B00F46182 /* app in Resources */, 607A66222B0EFA390010BFC8 /* Assets.xcassets in Resources */, 608619542CB77BA900F46182 /* app_packages in Resources */, @@ -245,7 +242,7 @@ /* End PBXResourcesBuildPhase section */ /* Begin PBXShellScriptBuildPhase section */ - 607A66552B0F061D0010BFC8 /* Install Target Specific Python Standard Library */ = { + 607A66552B0F061D0010BFC8 /* Process Python libraries */ = { isa = PBXShellScriptBuildPhase; alwaysOutOfDate = 1; buildActionMask = 2147483647; @@ -255,34 +252,14 @@ ); inputPaths = ( ); - name = "Install Target Specific Python Standard Library"; + name = "Process Python libraries"; outputFileListPaths = ( ); outputPaths = ( ); runOnlyForDeploymentPostprocessing = 0; shellPath = /bin/sh; - shellScript = "set -e\n\nmkdir -p \"$CODESIGNING_FOLDER_PATH/python/lib\"\nif [ \"$EFFECTIVE_PLATFORM_NAME\" = \"-iphonesimulator\" ]; then\n echo \"Installing Python modules for iOS Simulator\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64_x86_64-simulator/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nelse\n echo \"Installing Python modules for iOS Device\"\n rsync -au --delete \"$PROJECT_DIR/Python.xcframework/ios-arm64/lib/\" \"$CODESIGNING_FOLDER_PATH/python/lib/\" \nfi\n"; - showEnvVarsInLog = 0; - }; - 607A66562B0F06200010BFC8 /* Prepare Python Binary Modules */ = { - isa = PBXShellScriptBuildPhase; - alwaysOutOfDate = 1; - buildActionMask = 2147483647; - files = ( - ); - inputFileListPaths = ( - ); - inputPaths = ( - ); - name = "Prepare Python Binary Modules"; - outputFileListPaths = ( - ); - outputPaths = ( - ); - runOnlyForDeploymentPostprocessing = 0; - shellPath = /bin/sh; - shellScript = "set -e\n\ninstall_dylib () {\n INSTALL_BASE=$1\n FULL_EXT=$2\n\n # The name of the extension file\n EXT=$(basename \"$FULL_EXT\")\n # The location of the extension file, relative to the bundle\n RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} \n # The path to the extension file, relative to the install base\n PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/}\n # The full dotted name of the extension module, constructed from the file path.\n FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d \".\" -f 1 | tr \"/\" \".\"); \n # A bundle identifier; not actually used, but required by Xcode framework packaging\n FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr \"_\" \"-\")\n # The name of the framework folder.\n FRAMEWORK_FOLDER=\"Frameworks/$FULL_MODULE_NAME.framework\"\n\n # If the framework folder doesn't exist, create it.\n if [ ! -d \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\" ]; then\n echo \"Creating framework for $RELATIVE_EXT\" \n mkdir -p \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER\"\n cp \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleExecutable -string \"$FULL_MODULE_NAME\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n plutil -replace CFBundleIdentifier -string \"$FRAMEWORK_BUNDLE_ID\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist\"\n fi\n \n echo \"Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" \n mv \"$FULL_EXT\" \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\"\n # Create a placeholder .fwork file where the .so was\n echo \"$FRAMEWORK_FOLDER/$FULL_MODULE_NAME\" > ${FULL_EXT%.so}.fwork\n # Create a back reference to the .so file location in the framework\n echo \"${RELATIVE_EXT%.so}.fwork\" > \"$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin\" \n}\n\nPYTHON_VER=$(ls -1 \"$CODESIGNING_FOLDER_PATH/python/lib\")\necho \"Install Python $PYTHON_VER standard library extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib python/lib/$PYTHON_VER/lib-dynload/ \"$FULL_EXT\"\ndone\necho \"Install app package extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/app_packages\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib app_packages/ \"$FULL_EXT\"\ndone\necho \"Install app extension modules...\"\nfind \"$CODESIGNING_FOLDER_PATH/app\" -name \"*.so\" | while read FULL_EXT; do\n install_dylib app/ \"$FULL_EXT\"\ndone\n\n# Clean up dylib template \nrm -f \"$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist\"\necho \"Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)...\"\nfind \"$CODESIGNING_FOLDER_PATH/Frameworks\" -name \"*.framework\" -exec /usr/bin/codesign --force --sign \"$EXPANDED_CODE_SIGN_IDENTITY\" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der \"{}\" \\; \n"; + shellScript = "set -e\nsource $PROJECT_DIR/Python.xcframework/build/utils.sh\ninstall_python Python.xcframework app app_packages\n"; showEnvVarsInLog = 0; }; /* End PBXShellScriptBuildPhase section */ @@ -301,7 +278,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( - 607A66322B0EFA3A0010BFC8 /* iOSTestbedTests.m in Sources */, + 607A66322B0EFA3A0010BFC8 /* TestbedTests.m in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -379,7 +356,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 12.0; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MTL_ENABLE_DEBUG_INFO = INCLUDE_SOURCE; MTL_FAST_MATH = YES; @@ -434,7 +411,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 12.0; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; LOCALIZATION_PREFERS_STRING_CATALOGS = YES; MTL_ENABLE_DEBUG_INFO = NO; MTL_FAST_MATH = YES; @@ -460,7 +437,7 @@ INFOPLIST_KEY_UIMainStoryboardFile = Main; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; - IPHONEOS_DEPLOYMENT_TARGET = 12.0; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", @@ -491,7 +468,7 @@ INFOPLIST_KEY_UIMainStoryboardFile = Main; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPad = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; INFOPLIST_KEY_UISupportedInterfaceOrientations_iPhone = "UIInterfaceOrientationPortrait UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight"; - IPHONEOS_DEPLOYMENT_TARGET = 12.0; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", @@ -514,7 +491,7 @@ DEVELOPMENT_TEAM = 3HEZE76D99; GENERATE_INFOPLIST_FILE = YES; HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\""; - IPHONEOS_DEPLOYMENT_TARGET = 12.0; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = org.python.iOSTestbedTests; PRODUCT_NAME = "$(TARGET_NAME)"; @@ -534,7 +511,7 @@ DEVELOPMENT_TEAM = 3HEZE76D99; GENERATE_INFOPLIST_FILE = YES; HEADER_SEARCH_PATHS = "\"$(BUILT_PRODUCTS_DIR)/Python.framework/Headers\""; - IPHONEOS_DEPLOYMENT_TARGET = 12.0; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; MARKETING_VERSION = 1.0; PRODUCT_BUNDLE_IDENTIFIER = org.python.iOSTestbedTests; PRODUCT_NAME = "$(TARGET_NAME)"; diff --git a/Apple/testbed/iOSTestbed.xcodeproj/xcshareddata/xcschemes/iOSTestbed.xcscheme b/Apple/testbed/iOSTestbed.xcodeproj/xcshareddata/xcschemes/iOSTestbed.xcscheme new file mode 100644 index 000000000000000..3c330a4152bf920 --- /dev/null +++ b/Apple/testbed/iOSTestbed.xcodeproj/xcshareddata/xcschemes/iOSTestbed.xcscheme @@ -0,0 +1,97 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Apple/testbed/iOSTestbed.xctestplan b/Apple/testbed/iOSTestbed.xctestplan new file mode 100644 index 000000000000000..0c4ab9eb2bad305 --- /dev/null +++ b/Apple/testbed/iOSTestbed.xctestplan @@ -0,0 +1,46 @@ +{ + "configurations" : [ + { + "id" : "F5A95CE4-1ADE-4A6E-A0E1-CDBAE26DF0C5", + "name" : "Test Scheme Action", + "options" : { + + } + } + ], + "defaultOptions" : { + "commandLineArgumentEntries" : [ + { + "argument" : "test" + }, + { + "argument" : "-uall" + }, + { + "argument" : "--single-process" + }, + { + "argument" : "--rerun" + }, + { + "argument" : "-W" + } + ], + "targetForVariableExpansion" : { + "containerPath" : "container:iOSTestbed.xcodeproj", + "identifier" : "607A66112B0EFA380010BFC8", + "name" : "iOSTestbed" + } + }, + "testTargets" : [ + { + "parallelizable" : false, + "target" : { + "containerPath" : "container:iOSTestbed.xcodeproj", + "identifier" : "607A662C2B0EFA3A0010BFC8", + "name" : "iOSTestbedTests" + } + } + ], + "version" : 1 +} diff --git a/iOS/testbed/iOSTestbed/AppDelegate.h b/Apple/testbed/iOSTestbed/AppDelegate.h similarity index 100% rename from iOS/testbed/iOSTestbed/AppDelegate.h rename to Apple/testbed/iOSTestbed/AppDelegate.h diff --git a/iOS/testbed/iOSTestbed/AppDelegate.m b/Apple/testbed/iOSTestbed/AppDelegate.m similarity index 100% rename from iOS/testbed/iOSTestbed/AppDelegate.m rename to Apple/testbed/iOSTestbed/AppDelegate.m diff --git a/iOS/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json b/Apple/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json similarity index 100% rename from iOS/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json rename to Apple/testbed/iOSTestbed/Assets.xcassets/AccentColor.colorset/Contents.json diff --git a/iOS/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json b/Apple/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json similarity index 100% rename from iOS/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json rename to Apple/testbed/iOSTestbed/Assets.xcassets/AppIcon.appiconset/Contents.json diff --git a/iOS/testbed/iOSTestbed/Assets.xcassets/Contents.json b/Apple/testbed/iOSTestbed/Assets.xcassets/Contents.json similarity index 100% rename from iOS/testbed/iOSTestbed/Assets.xcassets/Contents.json rename to Apple/testbed/iOSTestbed/Assets.xcassets/Contents.json diff --git a/iOS/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard b/Apple/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard similarity index 100% rename from iOS/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard rename to Apple/testbed/iOSTestbed/Base.lproj/LaunchScreen.storyboard diff --git a/iOS/testbed/iOSTestbed/app/README b/Apple/testbed/iOSTestbed/app/README similarity index 92% rename from iOS/testbed/iOSTestbed/app/README rename to Apple/testbed/iOSTestbed/app/README index af22c685f87976a..46c0e8e2a29a1cc 100644 --- a/iOS/testbed/iOSTestbed/app/README +++ b/Apple/testbed/iOSTestbed/app/README @@ -1,7 +1,7 @@ This folder can contain any Python application code. During the build, any binary modules found in this folder will be processed into -iOS Framework form. +Framework form. When the test suite runs, this folder will be on the PYTHONPATH, and will be the working directory for the test suite. diff --git a/iOS/testbed/iOSTestbed/app_packages/README b/Apple/testbed/iOSTestbed/app_packages/README similarity index 92% rename from iOS/testbed/iOSTestbed/app_packages/README rename to Apple/testbed/iOSTestbed/app_packages/README index 42d7fdeb813250f..02c2beccfbdaed2 100644 --- a/iOS/testbed/iOSTestbed/app_packages/README +++ b/Apple/testbed/iOSTestbed/app_packages/README @@ -2,6 +2,6 @@ This folder can be a target for installing any Python dependencies needed by the test suite. During the build, any binary modules found in this folder will be processed into -iOS Framework form. +Framework form. When the test suite runs, this folder will be on the PYTHONPATH. diff --git a/iOS/testbed/iOSTestbed/iOSTestbed-Info.plist b/Apple/testbed/iOSTestbed/iOSTestbed-Info.plist similarity index 74% rename from iOS/testbed/iOSTestbed/iOSTestbed-Info.plist rename to Apple/testbed/iOSTestbed/iOSTestbed-Info.plist index a582f42a2127838..fea45e1fad6f6f6 100644 --- a/iOS/testbed/iOSTestbed/iOSTestbed-Info.plist +++ b/Apple/testbed/iOSTestbed/iOSTestbed-Info.plist @@ -41,18 +41,6 @@ UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight - TestArgs - - test - -uall - --single-process - --rerun - -W - - UIApplicationSceneManifest UIApplicationSupportsMultipleScenes diff --git a/iOS/testbed/iOSTestbed/main.m b/Apple/testbed/iOSTestbed/main.m similarity index 100% rename from iOS/testbed/iOSTestbed/main.m rename to Apple/testbed/iOSTestbed/main.m diff --git a/Doc/.ruff.toml b/Doc/.ruff.toml index 3e676e13c3f41ac..6b573fd58d089bc 100644 --- a/Doc/.ruff.toml +++ b/Doc/.ruff.toml @@ -32,6 +32,9 @@ ignore = [ "E501", # Ignore line length errors (we use auto-formatting) ] +[lint.per-file-ignores] +"tools/check-html-ids.py" = ["I001"] # Unsorted imports + [format] preview = true quote-style = "preserve" diff --git a/Doc/Makefile b/Doc/Makefile index c8a749a02a89eca..6eb466a34176266 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -58,7 +58,7 @@ build: @if [ -f ../Misc/NEWS ] ; then \ echo "Using existing Misc/NEWS file"; \ cp ../Misc/NEWS build/NEWS; \ - elif $(BLURB) help >/dev/null 2>&1 && $(SPHINXBUILD) --version >/dev/null 2>&1; then \ + elif $(BLURB) --version && $(SPHINXBUILD) --version ; then \ if [ -d ../Misc/NEWS.d ]; then \ echo "Building NEWS from Misc/NEWS.d with blurb"; \ $(BLURB) merge -f build/NEWS; \ @@ -89,7 +89,8 @@ htmlhelp: build .PHONY: latex latex: BUILDER = latex -latex: build +latex: _ensure-sphinxcontrib-svg2pdfconverter + $(MAKE) build BUILDER=$(BUILDER) @echo "Build finished; the LaTeX files are in build/latex." @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ "run these through (pdf)latex." @@ -140,7 +141,8 @@ doctest: pydoc-topics: BUILDER = pydoc-topics pydoc-topics: build @echo "Building finished; now run this:" \ - "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py" + "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py" \ + "&& cp build/pydoc-topics/module_docs.py ../Lib/pydoc_data/module_docs.py" .PHONY: gettext gettext: BUILDER = gettext @@ -170,6 +172,7 @@ venv: echo "venv already exists."; \ echo "To recreate it, remove it first with \`make clean-venv'."; \ else \ + set -e; \ echo "Creating venv in $(VENVDIR)"; \ if $(UV) --version >/dev/null 2>&1; then \ $(UV) venv --python=$(PYTHON) $(VENVDIR); \ @@ -183,7 +186,7 @@ venv: fi .PHONY: dist-no-html -dist-no-html: dist-text dist-pdf dist-epub dist-texinfo +dist-no-html: dist-text dist-epub dist-texinfo .PHONY: dist dist: @@ -229,7 +232,7 @@ dist-text: @echo "Build finished and archived!" .PHONY: dist-pdf -dist-pdf: +dist-pdf: _ensure-sphinxcontrib-svg2pdfconverter # archive the A4 latex @echo "Building LaTeX (A4 paper)..." mkdir -p dist @@ -240,7 +243,8 @@ dist-pdf: # as otherwise the full latexmk process is run twice. # ($$ is needed to escape the $; https://www.gnu.org/software/make/manual/make.html#Basics-of-Variable-References) -sed -i 's/: all-$$(FMT)/:/' build/latex/Makefile - (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`nproc`+1)) --output-sync LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) + if [ -n "$(filter output-sync,$(value .FEATURES))" ]; then OUTPUTSYNC=--output-sync; else OUTPUTSYNC=; fi && \ + (cd build/latex; $(MAKE) clean && $(MAKE) --jobs=$$((`getconf _NPROCESSORS_ONLN`+1)) $$OUTPUTSYNC LATEXMKOPTS='-quiet' all-pdf && $(MAKE) FMT=pdf zip bz2) cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-a4.zip cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-a4.tar.bz2 @echo "Build finished and archived!" @@ -289,6 +293,10 @@ _ensure-pre-commit: _ensure-sphinx-autobuild: $(MAKE) _ensure-package PACKAGE=sphinx-autobuild +.PHONY: _ensure-sphinxcontrib-svg2pdfconverter +_ensure-sphinxcontrib-svg2pdfconverter: + $(MAKE) _ensure-package PACKAGE=sphinxcontrib-svg2pdfconverter + .PHONY: check check: _ensure-pre-commit $(VENVDIR)/bin/python3 -m pre_commit run --all-files @@ -333,3 +341,9 @@ autobuild-stable-html: exit 1;; \ esac @$(MAKE) autobuild-dev-html + +# Collect HTML IDs to a JSON document +.PHONY: html-ids +html-ids: + $(PYTHON) tools/check-html-ids.py collect build/html \ + -o build/html/html-ids.json.gz diff --git a/Doc/bugs.rst b/Doc/bugs.rst index 5d0f68ca69675e5..9f2b9876ba51dc9 100644 --- a/Doc/bugs.rst +++ b/Doc/bugs.rst @@ -9,7 +9,7 @@ stability. In order to maintain this reputation, the developers would like to know of any deficiencies you find in Python. It can be sometimes faster to fix bugs yourself and contribute patches to -Python as it streamlines the process and involves less people. Learn how to +Python as it streamlines the process and involves fewer people. Learn how to :ref:`contribute `. Documentation bugs @@ -19,6 +19,12 @@ If you find a bug in this documentation or would like to propose an improvement, please submit a bug report on the :ref:`issue tracker `. If you have a suggestion on how to fix it, include that as well. +.. only:: translation + + If the bug or suggested improvement concerns the translation of this + documentation, submit the report to the + `translation’s repository `_ instead. + You can also open a discussion item on our `Documentation Discourse forum `_. diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst index 7cbc99ad145ad46..5f7a9152eccfc0a 100644 --- a/Doc/c-api/allocation.rst +++ b/Doc/c-api/allocation.rst @@ -2,7 +2,7 @@ .. _allocating-objects: -Allocating Objects on the Heap +Allocating objects on the heap ============================== @@ -16,7 +16,20 @@ Allocating Objects on the Heap Initialize a newly allocated object *op* with its type and initial reference. Returns the initialized object. Other fields of the object are - not affected. + not initialized. Despite its name, this function is unrelated to the + object's :meth:`~object.__init__` method (:c:member:`~PyTypeObject.tp_init` + slot). Specifically, this function does **not** call the object's + :meth:`!__init__` method. + + In general, consider this function to be a low-level routine. Use + :c:member:`~PyTypeObject.tp_alloc` where possible. + For implementing :c:member:`!tp_alloc` for your type, prefer + :c:func:`PyType_GenericAlloc` or :c:func:`PyObject_New`. + + .. note:: + + This function only initializes the object's memory corresponding to the + initial :c:type:`PyObject` structure. It does not zero the rest. .. c:function:: PyVarObject* PyObject_InitVar(PyVarObject *op, PyTypeObject *type, Py_ssize_t size) @@ -24,43 +37,108 @@ Allocating Objects on the Heap This does everything :c:func:`PyObject_Init` does, and also initializes the length information for a variable-size object. + .. note:: + + This function only initializes some of the object's memory. It does not + zero the rest. + .. c:macro:: PyObject_New(TYPE, typeobj) - Allocate a new Python object using the C structure type *TYPE* - and the Python type object *typeobj* (``PyTypeObject*``). - Fields not defined by the Python object header are not initialized. - The caller will own the only reference to the object - (i.e. its reference count will be one). - The size of the memory allocation is determined from the - :c:member:`~PyTypeObject.tp_basicsize` field of the type object. + Allocates a new Python object using the C structure type *TYPE* and the + Python type object *typeobj* (``PyTypeObject*``) by calling + :c:func:`PyObject_Malloc` to allocate memory and initializing it like + :c:func:`PyObject_Init`. The caller will own the only reference to the + object (i.e. its reference count will be one). + + Avoid calling this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot, + :c:func:`PyType_GenericAlloc` is preferred over a custom function that + simply calls this macro. + + This macro does not call :c:member:`~PyTypeObject.tp_alloc`, + :c:member:`~PyTypeObject.tp_new` (:meth:`~object.__new__`), or + :c:member:`~PyTypeObject.tp_init` (:meth:`~object.__init__`). + + This cannot be used for objects with :c:macro:`Py_TPFLAGS_HAVE_GC` set in + :c:member:`~PyTypeObject.tp_flags`; use :c:macro:`PyObject_GC_New` instead. + + Memory allocated by this macro must be freed with :c:func:`PyObject_Free` + (usually called via the object's :c:member:`~PyTypeObject.tp_free` slot). + + .. note:: + + The returned memory is not guaranteed to have been completely zeroed + before it was initialized. + + .. note:: + + This macro does not construct a fully initialized object of the given + type; it merely allocates memory and prepares it for further + initialization by :c:member:`~PyTypeObject.tp_init`. To construct a + fully initialized object, call *typeobj* instead. For example:: - Note that this function is unsuitable if *typeobj* has - :c:macro:`Py_TPFLAGS_HAVE_GC` set. For such objects, - use :c:func:`PyObject_GC_New` instead. + PyObject *foo = PyObject_CallNoArgs((PyObject *)&PyFoo_Type); + + .. seealso:: + + * :c:func:`PyObject_Free` + * :c:macro:`PyObject_GC_New` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_alloc` .. c:macro:: PyObject_NewVar(TYPE, typeobj, size) - Allocate a new Python object using the C structure type *TYPE* and the - Python type object *typeobj* (``PyTypeObject*``). - Fields not defined by the Python object header - are not initialized. The allocated memory allows for the *TYPE* structure - plus *size* (``Py_ssize_t``) fields of the size - given by the :c:member:`~PyTypeObject.tp_itemsize` field of - *typeobj*. This is useful for implementing objects like tuples, which are - able to determine their size at construction time. Embedding the array of - fields into the same allocation decreases the number of allocations, - improving the memory management efficiency. + Like :c:macro:`PyObject_New` except: + + * It allocates enough memory for the *TYPE* structure plus *size* + (``Py_ssize_t``) fields of the size given by the + :c:member:`~PyTypeObject.tp_itemsize` field of *typeobj*. + * The memory is initialized like :c:func:`PyObject_InitVar`. + + This is useful for implementing objects like tuples, which are able to + determine their size at construction time. Embedding the array of fields + into the same allocation decreases the number of allocations, improving the + memory management efficiency. + + Avoid calling this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot, + :c:func:`PyType_GenericAlloc` is preferred over a custom function that + simply calls this macro. + + This cannot be used for objects with :c:macro:`Py_TPFLAGS_HAVE_GC` set in + :c:member:`~PyTypeObject.tp_flags`; use :c:macro:`PyObject_GC_NewVar` + instead. - Note that this function is unsuitable if *typeobj* has - :c:macro:`Py_TPFLAGS_HAVE_GC` set. For such objects, - use :c:func:`PyObject_GC_NewVar` instead. + Memory allocated by this function must be freed with :c:func:`PyObject_Free` + (usually called via the object's :c:member:`~PyTypeObject.tp_free` slot). + .. note:: -.. c:function:: void PyObject_Del(void *op) + The returned memory is not guaranteed to have been completely zeroed + before it was initialized. + + .. note:: + + This macro does not construct a fully initialized object of the given + type; it merely allocates memory and prepares it for further + initialization by :c:member:`~PyTypeObject.tp_init`. To construct a + fully initialized object, call *typeobj* instead. For example:: + + PyObject *list_instance = PyObject_CallNoArgs((PyObject *)&PyList_Type); + + .. seealso:: + + * :c:func:`PyObject_Free` + * :c:macro:`PyObject_GC_NewVar` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_alloc` - Same as :c:func:`PyObject_Free`. .. c:var:: PyObject _Py_NoneStruct @@ -71,6 +149,40 @@ Allocating Objects on the Heap .. seealso:: - :c:func:`PyModule_Create` + :ref:`moduleobjects` To allocate and create extension modules. + +Soft-deprecated aliases +^^^^^^^^^^^^^^^^^^^^^^^ + +.. soft-deprecated:: 3.10 + +These are aliases to existing functions and macros. +They exist solely for backwards compatibility. + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * Soft-deprecated alias + * Function + * * .. c:macro:: PyObject_NEW(type, typeobj) + * :c:macro:`PyObject_New` + * * .. c:macro:: PyObject_NEW_VAR(type, typeobj, n) + * :c:macro:`PyObject_NewVar` + * * .. c:macro:: PyObject_INIT(op, typeobj) + * :c:func:`PyObject_Init` + * * .. c:macro:: PyObject_INIT_VAR(op, typeobj, n) + * :c:func:`PyObject_InitVar` + * * .. c:macro:: PyObject_MALLOC(n) + * :c:func:`PyObject_Malloc` + * * .. c:macro:: PyObject_REALLOC(p, n) + * :c:func:`PyObject_Realloc` + * * .. c:macro:: PyObject_FREE(p) + * :c:func:`PyObject_Free` + * * .. c:macro:: PyObject_DEL(p) + * :c:func:`PyObject_Free` + * * .. c:macro:: PyObject_Del(p) + * :c:func:`PyObject_Free` diff --git a/Doc/c-api/apiabiversion.rst b/Doc/c-api/apiabiversion.rst index 96050f59bd52508..6396a424f44d141 100644 --- a/Doc/c-api/apiabiversion.rst +++ b/Doc/c-api/apiabiversion.rst @@ -46,6 +46,8 @@ See :ref:`stable` for a discussion of API and ABI stability across versions. Use this for numeric comparisons, for example, ``#if PY_VERSION_HEX >= ...``. +These macros are defined in :source:`Include/patchlevel.h`. + Run-time version ---------------- diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index 3bbc990b6329c0d..005a46a85caf82d 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -113,18 +113,14 @@ There are three ways strings and buffers can be converted to C: ``z`` (:class:`str` or ``None``) [const char \*] Like ``s``, but the Python object may also be ``None``, in which case the C pointer is set to ``NULL``. - It is the same as ``s?`` with the C pointer was initialized to ``NULL``. ``z*`` (:class:`str`, :term:`bytes-like object` or ``None``) [Py_buffer] Like ``s*``, but the Python object may also be ``None``, in which case the ``buf`` member of the :c:type:`Py_buffer` structure is set to ``NULL``. - It is the same as ``s*?`` with the ``buf`` member of the :c:type:`Py_buffer` - structure was initialized to ``NULL``. ``z#`` (:class:`str`, read-only :term:`bytes-like object` or ``None``) [const char \*, :c:type:`Py_ssize_t`] Like ``s#``, but the Python object may also be ``None``, in which case the C pointer is set to ``NULL``. - It is the same as ``s#?`` with the C pointer was initialized to ``NULL``. ``y`` (read-only :term:`bytes-like object`) [const char \*] This format converts a bytes-like object to a C pointer to a @@ -164,7 +160,7 @@ There are three ways strings and buffers can be converted to C: ``w*`` (read-write :term:`bytes-like object`) [Py_buffer] This format accepts any object which implements the read-write buffer interface. It fills a :c:type:`Py_buffer` structure provided by the caller. - The buffer may contain embedded null bytes. The caller have to call + The buffer may contain embedded null bytes. The caller has to call :c:func:`PyBuffer_Release` when it is done with the buffer. ``es`` (:class:`str`) [const char \*encoding, char \*\*buffer] @@ -387,17 +383,6 @@ Other objects Non-tuple sequences are deprecated if *items* contains format units which store a borrowed buffer or a borrowed reference. -``unit?`` (anything or ``None``) [*matching-variable(s)*] - ``?`` modifies the behavior of the preceding format unit. - The C variable(s) corresponding to that parameter should be initialized - to their default value --- when the argument is ``None``, - :c:func:`PyArg_ParseTuple` does not touch the contents of the corresponding - C variable(s). - If the argument is not ``None``, it is parsed according to the specified - format unit. - - .. versionadded:: 3.14 - A few other characters have a meaning in a format string. These may not occur inside nested parentheses. They are: @@ -685,6 +670,13 @@ Building values ``p`` (:class:`bool`) [int] Convert a C :c:expr:`int` to a Python :class:`bool` object. + + Be aware that this format requires an ``int`` argument. + Unlike most other contexts in C, variadic arguments are not coerced to + a suitable type automatically. + You can convert another type (for example, a pointer or a float) to a + suitable ``int`` value using ``(x) ? 1 : 0`` or ``!!x``. + .. versionadded:: 3.14 ``c`` (:class:`bytes` of length 1) [char] diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index d3081894eadaf5e..22f54cfee81a99c 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -261,6 +261,12 @@ readonly, format MUST be consistent for all consumers. For example, :c:expr:`PyBUF_SIMPLE | PyBUF_WRITABLE` can be used to request a simple writable buffer. + .. c:macro:: PyBUF_WRITEABLE + + This is an alias to :c:macro:`PyBUF_WRITABLE`. + + .. soft-deprecated:: 3.13 + .. c:macro:: PyBUF_FORMAT Controls the :c:member:`~Py_buffer.format` field. If set, this field MUST diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index e2b22ec3c794ae4..2b36da997d42956 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -44,6 +44,10 @@ Direct API functions On failure, return ``NULL`` with an exception set. + .. note:: + If the object implements the buffer protocol, then the buffer + must not be mutated while the bytearray object is being created. + .. c:function:: PyObject* PyByteArray_FromStringAndSize(const char *string, Py_ssize_t len) @@ -58,6 +62,10 @@ Direct API functions On failure, return ``NULL`` with an exception set. + .. note:: + If the object implements the buffer protocol, then the buffer + must not be mutated while the bytearray object is being created. + .. c:function:: Py_ssize_t PyByteArray_Size(PyObject *bytearray) @@ -70,6 +78,9 @@ Direct API functions ``NULL`` pointer. The returned array always has an extra null byte appended. + .. note:: + It is not thread-safe to mutate the bytearray object while using the returned char array. + .. c:function:: int PyByteArray_Resize(PyObject *bytearray, Py_ssize_t len) @@ -89,6 +100,9 @@ These macros trade safety for speed and they don't check pointers. Similar to :c:func:`PyByteArray_AsString`, but without error checking. + .. note:: + It is not thread-safe to mutate the bytearray object while using the returned char array. + .. c:function:: Py_ssize_t PyByteArray_GET_SIZE(PyObject *bytearray) diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index d47beee68eaa338..b90d030719eca5b 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -123,6 +123,10 @@ called with a non-bytes parameter. Return the bytes representation of object *o* that implements the buffer protocol. + .. note:: + If the object implements the buffer protocol, then the buffer + must not be mutated while the bytes object is being created. + .. c:function:: Py_ssize_t PyBytes_Size(PyObject *o) @@ -176,11 +180,15 @@ called with a non-bytes parameter. .. c:function:: void PyBytes_Concat(PyObject **bytes, PyObject *newpart) Create a new bytes object in *\*bytes* containing the contents of *newpart* - appended to *bytes*; the caller will own the new reference. The reference to - the old value of *bytes* will be stolen. If the new object cannot be - created, the old reference to *bytes* will still be discarded and the value - of *\*bytes* will be set to ``NULL``; the appropriate exception will be set. + appended to *bytes*; the caller will own the new reference. + The reference to the old value of *bytes* will be ":term:`stolen `". + If the new object cannot be created, the old reference to *bytes* will still + be "stolen", the value of *\*bytes* will be set to ``NULL``, and + the appropriate exception will be set. + .. note:: + If *newpart* implements the buffer protocol, then the buffer + must not be mutated while the new bytes object is being created. .. c:function:: void PyBytes_ConcatAndDel(PyObject **bytes, PyObject *newpart) @@ -188,6 +196,10 @@ called with a non-bytes parameter. appended to *bytes*. This version releases the :term:`strong reference` to *newpart* (i.e. decrements its reference count). + .. note:: + If *newpart* implements the buffer protocol, then the buffer + must not be mutated while the new bytes object is being created. + .. c:function:: PyObject* PyBytes_Join(PyObject *sep, PyObject *iterable) @@ -206,6 +218,9 @@ called with a non-bytes parameter. .. versionadded:: 3.14 + .. note:: + If *iterable* objects implement the buffer protocol, then the buffers + must not be mutated while the new bytes object is being created. .. c:function:: int _PyBytes_Resize(PyObject **bytes, Py_ssize_t newsize) @@ -219,3 +234,38 @@ called with a non-bytes parameter. reallocation fails, the original bytes object at *\*bytes* is deallocated, *\*bytes* is set to ``NULL``, :exc:`MemoryError` is set, and ``-1`` is returned. + + +.. c:function:: PyObject *PyBytes_Repr(PyObject *bytes, int smartquotes) + + Get the string representation of *bytes*. This function is currently used to + implement :meth:`!bytes.__repr__` in Python. + + This function does not do type checking; it is undefined behavior to pass + *bytes* as a non-bytes object or ``NULL``. + + If *smartquotes* is true, the representation will use a double-quoted string + instead of single-quoted string when single-quotes are present in *bytes*. + For example, the byte string ``'Python'`` would be represented as + ``b"'Python'"`` when *smartquotes* is true, or ``b'\'Python\''`` when it is + false. + + On success, this function returns a :term:`strong reference` to a + :class:`str` object containing the representation. On failure, this + returns ``NULL`` with an exception set. + + +.. c:function:: PyObject *PyBytes_DecodeEscape(const char *s, Py_ssize_t len, const char *errors, Py_ssize_t unicode, const char *recode_encoding) + + Unescape a backslash-escaped string *s*. *s* must not be ``NULL``. + *len* must be the size of *s*. + + *errors* must be one of ``"strict"``, ``"replace"``, or ``"ignore"``. If + *errors* is ``NULL``, then ``"strict"`` is used by default. + + On success, this function returns a :term:`strong reference` to a Python + :class:`bytes` object containing the unescaped string. On failure, this + function returns ``NULL`` with an exception set. + + .. versionchanged:: 3.9 + *unicode* and *recode_encoding* are now unused. diff --git a/Doc/c-api/call.rst b/Doc/c-api/call.rst index 7198d6bc056eb43..9838879a528934a 100644 --- a/Doc/c-api/call.rst +++ b/Doc/c-api/call.rst @@ -347,6 +347,8 @@ please see individual documentation for details. .. versionadded:: 3.9 +.. c:function:: PyObject* _PyObject_Vectorcall(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) + :no-typesetting: .. c:function:: PyObject* PyObject_Vectorcall(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) @@ -358,7 +360,12 @@ please see individual documentation for details. Return the result of the call on success, or raise an exception and return *NULL* on failure. - .. versionadded:: 3.9 + .. versionadded:: 3.8 as ``_PyObject_Vectorcall`` + + .. versionchanged:: 3.9 + + Renamed to the current name, without the leading underscore. + The old provisional name is :term:`soft deprecated`. .. c:function:: PyObject* PyObject_VectorcallDict(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwdict) diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst index cdb8aa33e9fd325..03a848d68ed7aba 100644 --- a/Doc/c-api/capsule.rst +++ b/Doc/c-api/capsule.rst @@ -15,13 +15,19 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:type:: PyCapsule This subtype of :c:type:`PyObject` represents an opaque value, useful for C - extension modules who need to pass an opaque value (as a :c:expr:`void*` + extension modules which need to pass an opaque value (as a :c:expr:`void*` pointer) through Python code to other C code. It is often used to make a C function pointer defined in one module available to other modules, so the regular import mechanism can be used to access C APIs defined in dynamically loaded modules. +.. c:var:: PyTypeObject PyCapsule_Type + + The type object corresponding to capsule objects. This is the same object + as :class:`types.CapsuleType` in the Python layer. + + .. c:type:: PyCapsule_Destructor The type of a destructor callback for a capsule. Defined as:: @@ -105,9 +111,19 @@ Refer to :ref:`using-capsules` for more information on using these objects. ``module.attribute``. The *name* stored in the capsule must match this string exactly. + This function splits *name* on the ``.`` character, and imports the first + element. It then processes further elements using attribute lookups. + Return the capsule's internal *pointer* on success. On failure, set an exception and return ``NULL``. + .. note:: + + If *name* points to an attribute of some submodule or subpackage, this + submodule or subpackage must be previously imported using other means + (for example, by using :c:func:`PyImport_ImportModule`) for the + attribute lookups to succeed. + .. versionchanged:: 3.3 *no_block* has no effect anymore. diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst index 61eb994c3709460..2501ed9580df898 100644 --- a/Doc/c-api/cell.rst +++ b/Doc/c-api/cell.rst @@ -7,7 +7,7 @@ Cell Objects "Cell" objects are used to implement variables referenced by multiple scopes. For each such variable, a cell object is created to store the value; the local -variables of each stack frame that references the value contains a reference to +variables of each stack frame that references the value contain a reference to the cells from outer scopes which also use that variable. When the value is accessed, the value contained in the cell is used instead of the cell object itself. This de-referencing of the cell object requires support from the diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 6eae24b38fae48a..3ffbd9dc1709391 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -69,13 +69,14 @@ bound into a function. The old name is deprecated, but will remain available until the signature changes again. +.. c:function:: PyCodeObject* PyCode_NewWithPosOnlyArgs(...) + :no-typesetting: + .. c:function:: PyCodeObject* PyUnstable_Code_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable) Similar to :c:func:`PyUnstable_Code_New`, but with an extra "posonlyargcount" for positional-only arguments. The same caveats that apply to ``PyUnstable_Code_New`` also apply to this function. - .. index:: single: PyCode_NewWithPosOnlyArgs (C function) - .. versionadded:: 3.8 as ``PyCode_NewWithPosOnlyArgs`` .. versionchanged:: 3.11 @@ -182,7 +183,7 @@ bound into a function. Type of a code object watcher callback function. If *event* is ``PY_CODE_EVENT_CREATE``, then the callback is invoked - after `co` has been fully initialized. Otherwise, the callback is invoked + after *co* has been fully initialized. Otherwise, the callback is invoked before the destruction of *co* takes place, so the prior state of *co* can be inspected. @@ -211,6 +212,84 @@ bound into a function. .. versionadded:: 3.12 +.. c:function:: PyObject *PyCode_Optimize(PyObject *code, PyObject *consts, PyObject *names, PyObject *lnotab_obj) + + This is a function that does nothing. + + Prior to Python 3.10, this function would perform basic optimizations to a + code object. + + .. versionchanged:: 3.10 + This function now does nothing. + + .. soft-deprecated:: 3.13 + + +.. _c_codeobject_flags: + +Code Object Flags +----------------- + +Code objects contain a bit-field of flags, which can be retrieved as the +:attr:`~codeobject.co_flags` Python attribute (for example using +:c:func:`PyObject_GetAttrString`), and set using a *flags* argument to +:c:func:`PyUnstable_Code_New` and similar functions. + +Flags whose names start with ``CO_FUTURE_`` correspond to features normally +selectable by :ref:`future statements `. These flags can be used in +:c:member:`PyCompilerFlags.cf_flags`. +Note that many ``CO_FUTURE_`` flags are mandatory in current versions of +Python, and setting them has no effect. + +The following flags are available. +For their meaning, see the linked documentation of their Python equivalents. + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * Flag + * Meaning + * * .. c:macro:: CO_OPTIMIZED + * :py:data:`inspect.CO_OPTIMIZED` + * * .. c:macro:: CO_NEWLOCALS + * :py:data:`inspect.CO_NEWLOCALS` + * * .. c:macro:: CO_VARARGS + * :py:data:`inspect.CO_VARARGS` + * * .. c:macro:: CO_VARKEYWORDS + * :py:data:`inspect.CO_VARKEYWORDS` + * * .. c:macro:: CO_NESTED + * :py:data:`inspect.CO_NESTED` + * * .. c:macro:: CO_GENERATOR + * :py:data:`inspect.CO_GENERATOR` + * * .. c:macro:: CO_COROUTINE + * :py:data:`inspect.CO_COROUTINE` + * * .. c:macro:: CO_ITERABLE_COROUTINE + * :py:data:`inspect.CO_ITERABLE_COROUTINE` + * * .. c:macro:: CO_ASYNC_GENERATOR + * :py:data:`inspect.CO_ASYNC_GENERATOR` + * * .. c:macro:: CO_HAS_DOCSTRING + * :py:data:`inspect.CO_HAS_DOCSTRING` + * * .. c:macro:: CO_METHOD + * :py:data:`inspect.CO_METHOD` + + * * .. c:macro:: CO_FUTURE_DIVISION + * no effect (:py:data:`__future__.division`) + * * .. c:macro:: CO_FUTURE_ABSOLUTE_IMPORT + * no effect (:py:data:`__future__.absolute_import`) + * * .. c:macro:: CO_FUTURE_WITH_STATEMENT + * no effect (:py:data:`__future__.with_statement`) + * * .. c:macro:: CO_FUTURE_PRINT_FUNCTION + * no effect (:py:data:`__future__.print_function`) + * * .. c:macro:: CO_FUTURE_UNICODE_LITERALS + * no effect (:py:data:`__future__.unicode_literals`) + * * .. c:macro:: CO_FUTURE_GENERATOR_STOP + * no effect (:py:data:`__future__.generator_stop`) + * * .. c:macro:: CO_FUTURE_ANNOTATIONS + * :py:data:`__future__.annotations` + + Extra information ----------------- @@ -222,9 +301,12 @@ These functions are part of the unstable C API tier: this functionality is a CPython implementation detail, and the API may change without deprecation warnings. +.. c:function:: Py_ssize_t _PyEval_RequestCodeExtraIndex(freefunc free) + :no-typesetting: + .. c:function:: Py_ssize_t PyUnstable_Eval_RequestCodeExtraIndex(freefunc free) - Return a new an opaque index value used to adding data to code objects. + Return a new opaque index value used to adding data to code objects. You generally call this function once (per interpreter) and use the result with ``PyCode_GetExtra`` and ``PyCode_SetExtra`` to manipulate @@ -234,8 +316,6 @@ may change without deprecation warnings. *free* will be called on non-``NULL`` data stored under the new index. Use :c:func:`Py_DecRef` when storing :c:type:`PyObject`. - .. index:: single: _PyEval_RequestCodeExtraIndex (C function) - .. versionadded:: 3.6 as ``_PyEval_RequestCodeExtraIndex`` .. versionchanged:: 3.12 @@ -244,6 +324,9 @@ may change without deprecation warnings. The old private name is deprecated, but will be available until the API changes. +.. c:function:: int _PyCode_GetExtra(PyObject *code, Py_ssize_t index, void **extra) + :no-typesetting: + .. c:function:: int PyUnstable_Code_GetExtra(PyObject *code, Py_ssize_t index, void **extra) Set *extra* to the extra data stored under the given index. @@ -252,8 +335,6 @@ may change without deprecation warnings. If no data was set under the index, set *extra* to ``NULL`` and return 0 without setting an exception. - .. index:: single: _PyCode_GetExtra (C function) - .. versionadded:: 3.6 as ``_PyCode_GetExtra`` .. versionchanged:: 3.12 @@ -262,13 +343,14 @@ may change without deprecation warnings. The old private name is deprecated, but will be available until the API changes. +.. c:function:: int _PyCode_SetExtra(PyObject *code, Py_ssize_t index, void *extra) + :no-typesetting: + .. c:function:: int PyUnstable_Code_SetExtra(PyObject *code, Py_ssize_t index, void *extra) Set the extra data stored under the given index to *extra*. Return 0 on success. Set an exception and return -1 on failure. - .. index:: single: _PyCode_SetExtra (C function) - .. versionadded:: 3.6 as ``_PyCode_SetExtra`` .. versionchanged:: 3.12 diff --git a/Doc/c-api/codec.rst b/Doc/c-api/codec.rst index 8ae5c4fecd6248f..35ee048bd5fa9f5 100644 --- a/Doc/c-api/codec.rst +++ b/Doc/c-api/codec.rst @@ -7,7 +7,7 @@ Codec registry and support functions Register a new codec search function. - As side effect, this tries to load the :mod:`!encodings` package, if not yet + As a side effect, this tries to load the :mod:`!encodings` package, if not yet done, to make sure that it is always first in the list of search functions. .. c:function:: int PyCodec_Unregister(PyObject *search_function) @@ -39,7 +39,7 @@ Codec registry and support functions *object* is passed through the decoder function found for the given *encoding* using the error handling method defined by *errors*. *errors* may be ``NULL`` to use the default method defined for the codec. Raises a - :exc:`LookupError` if no encoder can be found. + :exc:`LookupError` if no decoder can be found. Codec lookup API @@ -129,3 +129,13 @@ Registry API for Unicode encoding error handlers Replace the unicode encode error with ``\N{...}`` escapes. .. versionadded:: 3.5 + + +Codec utility variables +----------------------- + +.. c:var:: const char *Py_hexdigits + + A string constant containing the lowercase hexadecimal digits: ``"0123456789abcdef"``. + + .. versionadded:: 3.3 diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index 880f7b15ce68e82..1746fe95eaaca93 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -109,11 +109,20 @@ Other Objects descriptor.rst slice.rst memoryview.rst + picklebuffer.rst weakref.rst capsule.rst frame.rst gen.rst coro.rst contextvars.rst - datetime.rst typehints.rst + + +C API for extension modules +=========================== + +.. toctree:: + + curses.rst + datetime.rst diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index c92ef4c653a675f..794e45af66316f7 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -41,7 +41,7 @@ The return value (*rv*) for these functions should be interpreted as follows: ``rv + 1`` bytes would have been needed to succeed. ``str[size-1]`` is ``'\0'`` in this case. -* When ``rv < 0``, "something bad happened." ``str[size-1]`` is ``'\0'`` in +* When ``rv < 0``, the output conversion failed and ``str[size-1]`` is ``'\0'`` in this case too, but the rest of *str* is undefined. The exact cause of the error depends on the underlying platform. @@ -128,22 +128,46 @@ The following functions provide locale-independent string to number conversions. must be 0 and is ignored. The ``'r'`` format code specifies the standard :func:`repr` format. - *flags* can be zero or more of the values ``Py_DTSF_SIGN``, - ``Py_DTSF_ADD_DOT_0``, or ``Py_DTSF_ALT``, or-ed together: + *flags* can be zero or more of the following values or-ed together: - * ``Py_DTSF_SIGN`` means to always precede the returned string with a sign - character, even if *val* is non-negative. + .. c:namespace:: NULL - * ``Py_DTSF_ADD_DOT_0`` means to ensure that the returned string will not look - like an integer. + .. c:macro:: Py_DTSF_SIGN - * ``Py_DTSF_ALT`` means to apply "alternate" formatting rules. See the - documentation for the :c:func:`PyOS_snprintf` ``'#'`` specifier for - details. + Always precede the returned string with a sign + character, even if *val* is non-negative. - If *ptype* is non-``NULL``, then the value it points to will be set to one of - ``Py_DTST_FINITE``, ``Py_DTST_INFINITE``, or ``Py_DTST_NAN``, signifying that - *val* is a finite number, an infinite number, or not a number, respectively. + .. c:macro:: Py_DTSF_ADD_DOT_0 + + Ensure that the returned string will not look like an integer. + + .. c:macro:: Py_DTSF_ALT + + Apply "alternate" formatting rules. + See the documentation for the :c:func:`PyOS_snprintf` ``'#'`` specifier for + details. + + .. c:macro:: Py_DTSF_NO_NEG_0 + + Negative zero is converted to positive zero. + + .. versionadded:: 3.11 + + If *ptype* is non-``NULL``, then the value it points to will be set to one + of the following constants depending on the type of *val*: + + .. list-table:: + :header-rows: 1 + :align: left + + * - *\*ptype* + - type of *val* + * - .. c:macro:: Py_DTST_FINITE + - finite number + * - .. c:macro:: Py_DTST_INFINITE + - infinite number + * - .. c:macro:: Py_DTST_NAN + - not a number The return value is a pointer to *buffer* with the converted string or ``NULL`` if the conversion failed. The caller is responsible for freeing the @@ -152,13 +176,85 @@ The following functions provide locale-independent string to number conversions. .. versionadded:: 3.1 -.. c:function:: int PyOS_stricmp(const char *s1, const char *s2) +.. c:function:: int PyOS_mystricmp(const char *str1, const char *str2) + int PyOS_mystrnicmp(const char *str1, const char *str2, Py_ssize_t size) + + Case insensitive comparison of strings. These functions work almost + identically to :c:func:`!strcmp` and :c:func:`!strncmp` (respectively), + except that they ignore the case of ASCII characters. + + Return ``0`` if the strings are equal, a negative value if *str1* sorts + lexicographically before *str2*, or a positive value if it sorts after. + + In the *str1* or *str2* arguments, a NUL byte marks the end of the string. + For :c:func:`!PyOS_mystrnicmp`, the *size* argument gives the maximum size + of the string, as if NUL was present at the index given by *size*. + + These functions do not use the locale. + + +.. c:function:: int PyOS_stricmp(const char *str1, const char *str2) + int PyOS_strnicmp(const char *str1, const char *str2, Py_ssize_t size) + + Case insensitive comparison of strings. + + On Windows, these are aliases of :c:func:`!stricmp` and :c:func:`!strnicmp`, + respectively. + + On other platforms, they are aliases of :c:func:`PyOS_mystricmp` and + :c:func:`PyOS_mystrnicmp`, respectively. + + +Character classification and conversion +======================================= + +The following macros provide locale-independent (unlike the C standard library +``ctype.h``) character classification and conversion. +The argument must be a signed or unsigned :c:expr:`char`. + + +.. c:macro:: Py_ISALNUM(c) + + Return true if the character *c* is an alphanumeric character. + + +.. c:macro:: Py_ISALPHA(c) + + Return true if the character *c* is an alphabetic character (``a-z`` and ``A-Z``). + + +.. c:macro:: Py_ISDIGIT(c) + + Return true if the character *c* is a decimal digit (``0-9``). + + +.. c:macro:: Py_ISLOWER(c) + + Return true if the character *c* is a lowercase ASCII letter (``a-z``). + + +.. c:macro:: Py_ISUPPER(c) + + Return true if the character *c* is an uppercase ASCII letter (``A-Z``). + + +.. c:macro:: Py_ISSPACE(c) + + Return true if the character *c* is a whitespace character (space, tab, + carriage return, newline, vertical tab, or form feed). + + +.. c:macro:: Py_ISXDIGIT(c) + + Return true if the character *c* is a hexadecimal digit (``0-9``, ``a-f``, and + ``A-F``). + + +.. c:macro:: Py_TOLOWER(c) - Case insensitive comparison of strings. The function works almost - identically to :c:func:`!strcmp` except that it ignores the case. + Return the lowercase equivalent of the character *c*. -.. c:function:: int PyOS_strnicmp(const char *s1, const char *s2, Py_ssize_t size) +.. c:macro:: Py_TOUPPER(c) - Case insensitive comparison of strings. The function works almost - identically to :c:func:`!strncmp` except that it ignores the case. + Return the uppercase equivalent of the character *c*. diff --git a/Doc/c-api/curses.rst b/Doc/c-api/curses.rst new file mode 100644 index 000000000000000..5a1697c43cc969d --- /dev/null +++ b/Doc/c-api/curses.rst @@ -0,0 +1,138 @@ +.. highlight:: c + +Curses C API +------------ + +:mod:`curses` exposes a small C interface for extension modules. +Consumers must include the header file :file:`py_curses.h` (which is not +included by default by :file:`Python.h`) and :c:func:`import_curses` must +be invoked, usually as part of the module initialisation function, to populate +:c:var:`PyCurses_API`. + +.. warning:: + + Neither the C API nor the pure Python :mod:`curses` module are compatible + with subinterpreters. + +.. c:macro:: import_curses() + + Import the curses C API. The macro does not need a semi-colon to be called. + + On success, populate the :c:var:`PyCurses_API` pointer. + + On failure, set :c:var:`PyCurses_API` to NULL and set an exception. + The caller must check if an error occurred via :c:func:`PyErr_Occurred`: + + .. code-block:: + + import_curses(); // semi-colon is optional but recommended + if (PyErr_Occurred()) { /* cleanup */ } + + +.. c:var:: void **PyCurses_API + + Dynamically allocated object containing the curses C API. + This variable is only available once :c:macro:`import_curses` succeeds. + + ``PyCurses_API[0]`` corresponds to :c:data:`PyCursesWindow_Type`. + + ``PyCurses_API[1]``, ``PyCurses_API[2]``, and ``PyCurses_API[3]`` + are pointers to predicate functions of type ``int (*)(void)``. + + When called, these predicates return whether :func:`curses.setupterm`, + :func:`curses.initscr`, and :func:`curses.start_color` have been called + respectively. + + See also the convenience macros :c:macro:`PyCursesSetupTermCalled`, + :c:macro:`PyCursesInitialised`, and :c:macro:`PyCursesInitialisedColor`. + + .. note:: + + The number of entries in this structure is subject to changes. + Consider using :c:macro:`PyCurses_API_pointers` to check if + new fields are available or not. + + +.. c:macro:: PyCurses_API_pointers + + The number of accessible fields (``4``) in :c:var:`PyCurses_API`. + This number is incremented whenever new fields are added. + + +.. c:var:: PyTypeObject PyCursesWindow_Type + + The :ref:`heap type ` corresponding to :class:`curses.window`. + + +.. c:function:: int PyCursesWindow_Check(PyObject *op) + + Return true if *op* is a :class:`curses.window` instance, false otherwise. + + +The following macros are convenience macros expanding into C statements. +In particular, they can only be used as ``macro;`` or ``macro``, but not +``macro()`` or ``macro();``. + +.. c:macro:: PyCursesSetupTermCalled + + Macro checking if :func:`curses.setupterm` has been called. + + The macro expansion is roughly equivalent to: + + .. code-block:: + + { + typedef int (*predicate_t)(void); + predicate_t was_setupterm_called = (predicate_t)PyCurses_API[1]; + if (!was_setupterm_called()) { + return NULL; + } + } + + +.. c:macro:: PyCursesInitialised + + Macro checking if :func:`curses.initscr` has been called. + + The macro expansion is roughly equivalent to: + + .. code-block:: + + { + typedef int (*predicate_t)(void); + predicate_t was_initscr_called = (predicate_t)PyCurses_API[2]; + if (!was_initscr_called()) { + return NULL; + } + } + + +.. c:macro:: PyCursesInitialisedColor + + Macro checking if :func:`curses.start_color` has been called. + + The macro expansion is roughly equivalent to: + + .. code-block:: + + { + typedef int (*predicate_t)(void); + predicate_t was_start_color_called = (predicate_t)PyCurses_API[3]; + if (!was_start_color_called()) { + return NULL; + } + } + + +Internal data +------------- + +The following objects are exposed by the C API but should be considered +internal-only. + +.. c:macro:: PyCurses_CAPSULE_NAME + + Name of the curses capsule to pass to :c:func:`PyCapsule_Import`. + + Internal usage only. Use :c:macro:`import_curses` instead. + diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index d2d4d5309c70984..127d7c9c91a3d5f 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -8,11 +8,42 @@ DateTime Objects Various date and time objects are supplied by the :mod:`datetime` module. Before using any of these functions, the header file :file:`datetime.h` must be included in your source (note that this is not included by :file:`Python.h`), -and the macro :c:macro:`!PyDateTime_IMPORT` must be invoked, usually as part of +and the macro :c:macro:`PyDateTime_IMPORT` must be invoked, usually as part of the module initialisation function. The macro puts a pointer to a C structure -into a static variable, :c:data:`!PyDateTimeAPI`, that is used by the following +into a static variable, :c:data:`PyDateTimeAPI`, that is used by the following macros. +.. c:macro:: PyDateTime_IMPORT() + + Import the datetime C API. + + On success, populate the :c:var:`PyDateTimeAPI` pointer. + On failure, set :c:var:`PyDateTimeAPI` to ``NULL`` and set an exception. + The caller must check if an error occurred via :c:func:`PyErr_Occurred`: + + .. code-block:: + + PyDateTime_IMPORT; + if (PyErr_Occurred()) { /* cleanup */ } + + .. warning:: + + This is not compatible with subinterpreters. + +.. c:type:: PyDateTime_CAPI + + Structure containing the fields for the datetime C API. + + The fields of this structure are private and subject to change. + + Do not use this directly; prefer ``PyDateTime_*`` APIs instead. + +.. c:var:: PyDateTime_CAPI *PyDateTimeAPI + + Dynamically allocated object containing the datetime C API. + + This variable is only available once :c:macro:`PyDateTime_IMPORT` succeeds. + .. c:type:: PyDateTime_Date This subtype of :c:type:`PyObject` represents a Python date object. @@ -46,7 +77,7 @@ macros. .. c:var:: PyTypeObject PyDateTime_DeltaType - This instance of :c:type:`PyTypeObject` represents Python type for + This instance of :c:type:`PyTypeObject` represents the Python type for the difference between two datetime values; it is the same object as :class:`datetime.timedelta` in the Python layer. @@ -325,3 +356,16 @@ Macros for the convenience of modules implementing the DB API: Create and return a new :class:`datetime.date` object given an argument tuple suitable for passing to :meth:`datetime.date.fromtimestamp`. + + +Internal data +------------- + +The following symbols are exposed by the C API but should be considered +internal-only. + +.. c:macro:: PyDateTime_CAPSULE_NAME + + Name of the datetime capsule to pass to :c:func:`PyCapsule_Import`. + + Internal usage only. Use :c:macro:`PyDateTime_IMPORT` instead. diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst index b32c113e5f04573..813741f3eda51a5 100644 --- a/Doc/c-api/descriptor.rst +++ b/Doc/c-api/descriptor.rst @@ -8,8 +8,6 @@ Descriptor Objects "Descriptors" are objects that describe some attribute of an object. They are found in the dictionary of type objects. -.. XXX document these! - .. c:var:: PyTypeObject PyProperty_Type The type object for the built-in descriptor types. @@ -17,24 +15,178 @@ found in the dictionary of type objects. .. c:function:: PyObject* PyDescr_NewGetSet(PyTypeObject *type, struct PyGetSetDef *getset) + Create a new get-set descriptor for extension type *type* from the + :c:type:`PyGetSetDef` structure *getset*. + + Get-set descriptors expose attributes implemented by C getter and setter + functions rather than stored directly in the instance. This is the same kind + of descriptor created for entries in :c:member:`~PyTypeObject.tp_getset`, and + it appears in Python as a :class:`types.GetSetDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. + +.. c:function:: PyObject* PyDescr_NewMember(PyTypeObject *type, struct PyMemberDef *member) + + Create a new member descriptor for extension type *type* from the + :c:type:`PyMemberDef` structure *member*. + + Member descriptors expose fields in the type's C struct as Python + attributes. This is the same kind of descriptor created for entries in + :c:member:`~PyTypeObject.tp_members`, and it appears in Python as a + :class:`types.MemberDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. -.. c:function:: PyObject* PyDescr_NewMember(PyTypeObject *type, struct PyMemberDef *meth) +.. c:var:: PyTypeObject PyMemberDescr_Type + + The type object for member descriptor objects created from + :c:type:`PyMemberDef` structures. These descriptors expose fields of a + C struct as attributes on a type, and correspond + to :class:`types.MemberDescriptorType` objects in Python. + + + +.. c:var:: PyTypeObject PyGetSetDescr_Type + + The type object for get/set descriptor objects created from + :c:type:`PyGetSetDef` structures. These descriptors implement attributes + whose value is computed by C getter and setter functions, and are used + for many built-in type attributes. They correspond to + :class:`types.GetSetDescriptorType` objects in Python. .. c:function:: PyObject* PyDescr_NewMethod(PyTypeObject *type, struct PyMethodDef *meth) + Create a new method descriptor for extension type *type* from the + :c:type:`PyMethodDef` structure *meth*. + + Method descriptors expose C functions as methods on a type. This is the same + kind of descriptor created for entries in + :c:member:`~PyTypeObject.tp_methods`, and it appears in Python as a + :class:`types.MethodDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. + +.. c:var:: PyTypeObject PyMethodDescr_Type + + The type object for method descriptor objects created from + :c:type:`PyMethodDef` structures. These descriptors expose C functions as + methods on a type, and correspond to :class:`types.MethodDescriptorType` + objects in Python. + + +.. c:struct:: wrapperbase + + Describes a slot wrapper used by :c:func:`PyDescr_NewWrapper`. + + Each ``wrapperbase`` record stores the Python-visible name and metadata for a + special method implemented by a type slot, together with the wrapper + function used to adapt that slot to Python's calling convention. + +.. c:function:: PyObject* PyDescr_NewWrapper(PyTypeObject *type, struct wrapperbase *base, void *wrapped) + + Create a new wrapper descriptor for extension type *type* from the + :c:struct:`wrapperbase` structure *base* and the wrapped slot function + pointer + *wrapped*. + + Wrapper descriptors expose special methods implemented by type slots. This + is the same kind of descriptor that CPython creates for slot-based special + methods such as ``__repr__`` or ``__add__``, and it appears in Python as a + :class:`types.WrapperDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. + +.. c:var:: PyTypeObject PyWrapperDescr_Type -.. c:function:: PyObject* PyDescr_NewWrapper(PyTypeObject *type, struct wrapperbase *wrapper, void *wrapped) + The type object for wrapper descriptor objects created by + :c:func:`PyDescr_NewWrapper` and :c:func:`PyWrapper_New`. Wrapper + descriptors are used internally to expose special methods implemented + via wrapper structures, and appear in Python as + :class:`types.WrapperDescriptorType` objects. .. c:function:: PyObject* PyDescr_NewClassMethod(PyTypeObject *type, PyMethodDef *method) + Create a new class method descriptor for extension type *type* from the + :c:type:`PyMethodDef` structure *method*. + + Class method descriptors expose C methods that receive the class rather than + an instance when accessed. This is the same kind of descriptor created for + ``METH_CLASS`` entries in :c:member:`~PyTypeObject.tp_methods`, and it + appears in Python as a :class:`types.ClassMethodDescriptorType` object. + + On success, return a :term:`strong reference` to the descriptor. Return + ``NULL`` with an exception set on failure. .. c:function:: int PyDescr_IsData(PyObject *descr) - Return non-zero if the descriptor objects *descr* describes a data attribute, or + Return non-zero if the descriptor object *descr* describes a data attribute, or ``0`` if it describes a method. *descr* must be a descriptor object; there is no error checking. -.. c:function:: PyObject* PyWrapper_New(PyObject *, PyObject *) +.. c:function:: PyObject* PyWrapper_New(PyObject *d, PyObject *self) + + Create a new bound wrapper object from the wrapper descriptor *d* and the + instance *self*. + + This is the bound form of a wrapper descriptor created by + :c:func:`PyDescr_NewWrapper`. CPython creates these objects when a slot + wrapper is accessed through an instance, and they appear in Python as + :class:`types.MethodWrapperType` objects. + + On success, return a :term:`strong reference` to the wrapper object. Return + ``NULL`` with an exception set on failure. + +Built-in descriptors +^^^^^^^^^^^^^^^^^^^^ + +.. c:var:: PyTypeObject PySuper_Type + + The type object for super objects. This is the same object as + :class:`super` in the Python layer. + + +.. c:var:: PyTypeObject PyClassMethod_Type + + The type of class method objects. This is the same object as + :class:`classmethod` in the Python layer. + + +.. c:var:: PyTypeObject PyClassMethodDescr_Type + + The type object for C-level class method descriptor objects. + This is the type of the descriptors created for :func:`classmethod` defined + in C extension types, and corresponds to + :class:`types.ClassMethodDescriptorType` objects in Python. + + +.. c:function:: PyObject *PyClassMethod_New(PyObject *callable) + + Create a new :class:`classmethod` object wrapping *callable*. + *callable* must be a callable object and must not be ``NULL``. + + On success, this function returns a :term:`strong reference` to a new class + method descriptor. On failure, this function returns ``NULL`` with an + exception set. + + +.. c:var:: PyTypeObject PyStaticMethod_Type + + The type of static method objects. This is the same object as + :class:`staticmethod` in the Python layer. + + +.. c:function:: PyObject *PyStaticMethod_New(PyObject *callable) + + Create a new :class:`staticmethod` object wrapping *callable*. + *callable* must be a callable object and must not be ``NULL``. + + On success, this function returns a :term:`strong reference` to a new static + method descriptor. On failure, this function returns ``NULL`` with an + exception set. diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index e55c5c80cb83c0e..4c19af0d3a806e4 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -43,6 +43,17 @@ Dictionary Objects prevent modification of the dictionary for non-dynamic class types. +.. c:var:: PyTypeObject PyDictProxy_Type + + The type object for mapping proxy objects created by + :c:func:`PyDictProxy_New` and for the read-only ``__dict__`` attribute + of many built-in types. A :c:type:`PyDictProxy_Type` instance provides a + dynamic, read-only view of an underlying dictionary: changes to the + underlying dictionary are reflected in the proxy, but the proxy itself + does not support mutation operations. This corresponds to + :class:`types.MappingProxyType` in Python. + + .. c:function:: void PyDict_Clear(PyObject *p) Empty an existing dictionary of all key-value pairs. @@ -50,10 +61,15 @@ Dictionary Objects .. c:function:: int PyDict_Contains(PyObject *p, PyObject *key) - Determine if dictionary *p* contains *key*. If an item in *p* is matches + Determine if dictionary *p* contains *key*. If an item in *p* matches *key*, return ``1``, otherwise return ``0``. On error, return ``-1``. This is equivalent to the Python expression ``key in p``. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. c:function:: int PyDict_ContainsString(PyObject *p, const char *key) @@ -73,8 +89,13 @@ Dictionary Objects Insert *val* into the dictionary *p* with a key of *key*. *key* must be :term:`hashable`; if it isn't, :exc:`TypeError` will be raised. Return - ``0`` on success or ``-1`` on failure. This function *does not* steal a - reference to *val*. + ``0`` on success or ``-1`` on failure. + This function *does not* ":term:`steal`" a reference to *val*. + + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. .. c:function:: int PyDict_SetItemString(PyObject *p, const char *key, PyObject *val) @@ -91,6 +112,11 @@ Dictionary Objects If *key* is not in the dictionary, :exc:`KeyError` is raised. Return ``0`` on success or ``-1`` on failure. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. c:function:: int PyDict_DelItemString(PyObject *p, const char *key) @@ -107,7 +133,12 @@ Dictionary Objects * If the key is present, set *\*result* to a new :term:`strong reference` to the value and return ``1``. * If the key is missing, set *\*result* to ``NULL`` and return ``0``. - * On error, raise an exception and return ``-1``. + * On error, raise an exception, set *\*result* to ``NULL`` and return ``-1``. + + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. .. versionadded:: 3.13 @@ -126,6 +157,13 @@ Dictionary Objects :meth:`~object.__eq__` methods are silently ignored. Prefer the :c:func:`PyDict_GetItemWithError` function instead. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the dictionary concurrently. Prefer :c:func:`PyDict_GetItemRef`, which + returns a :term:`strong reference`. + .. versionchanged:: 3.10 Calling this API without an :term:`attached thread state` had been allowed for historical reason. It is no longer allowed. @@ -138,6 +176,13 @@ Dictionary Objects occurred. Return ``NULL`` **without** an exception set if the key wasn't present. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the dictionary concurrently. Prefer :c:func:`PyDict_GetItemRef`, which + returns a :term:`strong reference`. + .. c:function:: PyObject* PyDict_GetItemString(PyObject *p, const char *key) @@ -153,6 +198,13 @@ Dictionary Objects Prefer using the :c:func:`PyDict_GetItemWithError` function with your own :c:func:`PyUnicode_FromString` *key* instead. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the dictionary concurrently. Prefer :c:func:`PyDict_GetItemStringRef`, + which returns a :term:`strong reference`. + .. c:function:: int PyDict_GetItemStringRef(PyObject *p, const char *key, PyObject **result) @@ -173,6 +225,14 @@ Dictionary Objects .. versionadded:: 3.4 + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the dictionary concurrently. Prefer :c:func:`PyDict_SetDefaultRef`, + which returns a :term:`strong reference`. + + .. c:function:: int PyDict_SetDefaultRef(PyObject *p, PyObject *key, PyObject *default_value, PyObject **result) @@ -192,13 +252,18 @@ Dictionary Objects These may refer to the same object: in that case you hold two separate references to it. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. versionadded:: 3.13 .. c:function:: int PyDict_Pop(PyObject *p, PyObject *key, PyObject **result) Remove *key* from dictionary *p* and optionally return the removed value. - Do not raise :exc:`KeyError` if the key missing. + Do not raise :exc:`KeyError` if the key is missing. - If the key is present, set *\*result* to a new reference to the removed value if *result* is not ``NULL``, and return ``1``. @@ -207,7 +272,12 @@ Dictionary Objects - On error, raise an exception and return ``-1``. Similar to :meth:`dict.pop`, but without the default value and - not raising :exc:`KeyError` if the key missing. + not raising :exc:`KeyError` if the key is missing. + + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. .. versionadded:: 3.13 @@ -245,6 +315,11 @@ Dictionary Objects ``len(p)`` on a dictionary. +.. c:function:: Py_ssize_t PyDict_GET_SIZE(PyObject *p) + + Similar to :c:func:`PyDict_Size`, but without error checking. + + .. c:function:: int PyDict_Next(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) Iterate over all key-value pairs in the dictionary *p*. The @@ -301,6 +376,15 @@ Dictionary Objects } Py_END_CRITICAL_SECTION(); + .. note:: + + On the free-threaded build, this function can be used safely inside a + critical section. However, the references returned for *pkey* and *pvalue* + are :term:`borrowed ` and are only valid while the + critical section is held. If you need to use these objects outside the + critical section or when the critical section can be suspended, create a + :term:`strong reference ` (for example, using + :c:func:`Py_NewRef`). .. c:function:: int PyDict_Merge(PyObject *a, PyObject *b, int override) @@ -311,6 +395,13 @@ Dictionary Objects only be added if there is not a matching key in *a*. Return ``0`` on success or ``-1`` if an exception was raised. + .. note:: + + In the :term:`free-threaded build`, when *b* is a + :class:`dict` (with the standard iterator), both *a* and *b* are locked + for the duration of the operation. When *b* is a non-dict mapping, only + *a* is locked; *b* may be concurrently modified by another thread. + .. c:function:: int PyDict_Update(PyObject *a, PyObject *b) @@ -320,6 +411,13 @@ Dictionary Objects argument has no "keys" attribute. Return ``0`` on success or ``-1`` if an exception was raised. + .. note:: + + In the :term:`free-threaded build`, when *b* is a + :class:`dict` (with the standard iterator), both *a* and *b* are locked + for the duration of the operation. When *b* is a non-dict mapping, only + *a* is locked; *b* may be concurrently modified by another thread. + .. c:function:: int PyDict_MergeFromSeq2(PyObject *a, PyObject *seq2, int override) @@ -335,6 +433,13 @@ Dictionary Objects if override or key not in a: a[key] = value + .. note:: + + In the :term:`free-threaded ` build, only *a* is locked. + The iteration over *seq2* is not synchronized; *seq2* may be concurrently + modified by another thread. + + .. c:function:: int PyDict_AddWatcher(PyDict_WatchCallback callback) Register *callback* as a dictionary watcher. Return a non-negative integer @@ -342,6 +447,13 @@ Dictionary Objects of error (e.g. no more watcher IDs available), return ``-1`` and set an exception. + .. note:: + + This function is not internally synchronized. In the + :term:`free-threaded ` build, callers should ensure no + concurrent calls to :c:func:`PyDict_AddWatcher` or + :c:func:`PyDict_ClearWatcher` are in progress. + .. versionadded:: 3.12 .. c:function:: int PyDict_ClearWatcher(int watcher_id) @@ -350,6 +462,13 @@ Dictionary Objects :c:func:`PyDict_AddWatcher`. Return ``0`` on success, ``-1`` on error (e.g. if the given *watcher_id* was never registered.) + .. note:: + + This function is not internally synchronized. In the + :term:`free-threaded ` build, callers should ensure no + concurrent calls to :c:func:`PyDict_AddWatcher` or + :c:func:`PyDict_ClearWatcher` are in progress. + .. versionadded:: 3.12 .. c:function:: int PyDict_Watch(int watcher_id, PyObject *dict) @@ -417,3 +536,138 @@ Dictionary Objects it before returning. .. versionadded:: 3.12 + + +Dictionary View Objects +^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:function:: int PyDictViewSet_Check(PyObject *op) + + Return true if *op* is a view of a set inside a dictionary. This is currently + equivalent to :c:expr:`PyDictKeys_Check(op) || PyDictItems_Check(op)`. This + function always succeeds. + + +.. c:var:: PyTypeObject PyDictKeys_Type + + Type object for a view of dictionary keys. In Python, this is the type of + the object returned by :meth:`dict.keys`. + + +.. c:function:: int PyDictKeys_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary keys view. This function + always succeeds. + + +.. c:var:: PyTypeObject PyDictValues_Type + + Type object for a view of dictionary values. In Python, this is the type of + the object returned by :meth:`dict.values`. + + +.. c:function:: int PyDictValues_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary values view. This function + always succeeds. + + +.. c:var:: PyTypeObject PyDictItems_Type + + Type object for a view of dictionary items. In Python, this is the type of + the object returned by :meth:`dict.items`. + + +.. c:function:: int PyDictItems_Check(PyObject *op) + + Return true if *op* is an instance of a dictionary items view. This function + always succeeds. + + +Ordered Dictionaries +^^^^^^^^^^^^^^^^^^^^ + +Python's C API provides interface for :class:`collections.OrderedDict` from C. +Since Python 3.7, dictionaries are ordered by default, so there is usually +little need for these functions; prefer ``PyDict*`` where possible. + + +.. c:var:: PyTypeObject PyODict_Type + + Type object for ordered dictionaries. This is the same object as + :class:`collections.OrderedDict` in the Python layer. + + +.. c:function:: int PyODict_Check(PyObject *od) + + Return true if *od* is an ordered dictionary object or an instance of a + subtype of the :class:`~collections.OrderedDict` type. This function + always succeeds. + + +.. c:function:: int PyODict_CheckExact(PyObject *od) + + Return true if *od* is an ordered dictionary object, but not an instance of + a subtype of the :class:`~collections.OrderedDict` type. + This function always succeeds. + + +.. c:var:: PyTypeObject PyODictKeys_Type + + Analogous to :c:type:`PyDictKeys_Type` for ordered dictionaries. + + +.. c:var:: PyTypeObject PyODictValues_Type + + Analogous to :c:type:`PyDictValues_Type` for ordered dictionaries. + + +.. c:var:: PyTypeObject PyODictItems_Type + + Analogous to :c:type:`PyDictItems_Type` for ordered dictionaries. + + +.. c:function:: PyObject *PyODict_New(void) + + Return a new empty ordered dictionary, or ``NULL`` on failure. + + This is analogous to :c:func:`PyDict_New`. + + +.. c:function:: int PyODict_SetItem(PyObject *od, PyObject *key, PyObject *value) + + Insert *value* into the ordered dictionary *od* with a key of *key*. + Return ``0`` on success or ``-1`` with an exception set on failure. + + This is analogous to :c:func:`PyDict_SetItem`. + + +.. c:function:: int PyODict_DelItem(PyObject *od, PyObject *key) + + Remove the entry in the ordered dictionary *od* with key *key*. + Return ``0`` on success or ``-1`` with an exception set on failure. + + This is analogous to :c:func:`PyDict_DelItem`. + + +These are :term:`soft deprecated` aliases to ``PyDict`` APIs: + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * ``PyODict`` + * ``PyDict`` + * * .. c:macro:: PyODict_GetItem(od, key) + * :c:func:`PyDict_GetItem` + * * .. c:macro:: PyODict_GetItemWithError(od, key) + * :c:func:`PyDict_GetItemWithError` + * * .. c:macro:: PyODict_GetItemString(od, key) + * :c:func:`PyDict_GetItemString` + * * .. c:macro:: PyODict_Contains(od, key) + * :c:func:`PyDict_Contains` + * * .. c:macro:: PyODict_Size(od) + * :c:func:`PyDict_Size` + * * .. c:macro:: PyODict_SIZE(od) + * :c:func:`PyDict_GET_SIZE` diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index c8e1b5c24617382..4ff7122fe3f93d6 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -309,6 +309,14 @@ For convenience, some of these functions will always return a .. versionadded:: 3.4 +.. c:function:: void PyErr_RangedSyntaxLocationObject(PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset) + + Similar to :c:func:`PyErr_SyntaxLocationObject`, but also sets the + *end_lineno* and *end_col_offset* information for the current exception. + + .. versionadded:: 3.10 + + .. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string @@ -331,6 +339,23 @@ For convenience, some of these functions will always return a use. +.. c:function:: PyObject *PyErr_ProgramTextObject(PyObject *filename, int lineno) + + Get the source line in *filename* at line *lineno*. *filename* should be a + Python :class:`str` object. + + On success, this function returns a Python string object with the found line. + On failure, this function returns ``NULL`` without an exception set. + + +.. c:function:: PyObject *PyErr_ProgramText(const char *filename, int lineno) + + Similar to :c:func:`PyErr_ProgramTextObject`, but *filename* is a + :c:expr:`const char *`, which is decoded with the + :term:`filesystem encoding and error handler`, instead of a + Python object reference. + + Issuing warnings ================ @@ -387,13 +412,22 @@ an error value). .. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) - Function similar to :c:func:`PyErr_WarnEx`, but use + Function similar to :c:func:`PyErr_WarnEx`, but uses :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is an ASCII-encoded string. .. versionadded:: 3.2 +.. c:function:: int PyErr_WarnExplicitFormat(PyObject *category, const char *filename, int lineno, const char *module, PyObject *registry, const char *format, ...) + + Similar to :c:func:`PyErr_WarnExplicit`, but uses + :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is + an ASCII-encoded string. + + .. versionadded:: 3.2 + + .. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) Function similar to :c:func:`PyErr_WarnFormat`, but *category* is @@ -465,11 +499,12 @@ Querying the error indicator .. c:function:: void PyErr_SetRaisedException(PyObject *exc) Set *exc* as the exception currently being raised, - clearing the existing exception if one is set. + clearing the existing exception if one is set. If *exc* is ``NULL``, + just clear the existing exception. - .. warning:: + *exc* must be a valid exception or ``NULL``. - This call steals a reference to *exc*, which must be a valid exception. + This call ":term:`steals `" a reference to *exc*. .. versionadded:: 3.12 @@ -607,7 +642,8 @@ Querying the error indicator Set the exception info, as known from ``sys.exc_info()``. This refers to an exception that was *already caught*, not to an exception that was - freshly raised. This function steals the references of the arguments. + freshly raised. This function ":term:`steals `" the references + of the arguments. To clear the exception state, pass ``NULL`` for all three arguments. This function is kept for backwards compatibility. Prefer using :c:func:`PyErr_SetHandledException`. @@ -624,8 +660,8 @@ Querying the error indicator .. versionchanged:: 3.11 The ``type`` and ``traceback`` arguments are no longer used and can be NULL. The interpreter now derives them from the exception - instance (the ``value`` argument). The function still steals - references of all three arguments. + instance (the ``value`` argument). The function still + ":term:`steals `" references of all three arguments. Signal Handling @@ -639,28 +675,46 @@ Signal Handling single: SIGINT (C macro) single: KeyboardInterrupt (built-in exception) - This function interacts with Python's signal handling. + Handle external interruptions, such as signals or activating a debugger, + whose processing has been delayed until it is safe + to run Python code and/or raise exceptions. - If the function is called from the main thread and under the main Python - interpreter, it checks whether a signal has been sent to the processes - and if so, invokes the corresponding signal handler. If the :mod:`signal` - module is supported, this can invoke a signal handler written in Python. + For example, pressing :kbd:`Ctrl-C` causes a terminal to send the + :py:data:`signal.SIGINT` signal. + This function executes the corresponding Python signal handler, which, + by default, raises the :exc:`KeyboardInterrupt` exception. - The function attempts to handle all pending signals, and then returns ``0``. - However, if a Python signal handler raises an exception, the error - indicator is set and the function returns ``-1`` immediately (such that - other pending signals may not have been handled yet: they will be on the - next :c:func:`PyErr_CheckSignals()` invocation). + :c:func:`!PyErr_CheckSignals` should be called by long-running C code + frequently enough so that the response appears immediate to humans. - If the function is called from a non-main thread, or under a non-main - Python interpreter, it does nothing and returns ``0``. + Handlers invoked by this function currently include: - This function can be called by long-running C code that wants to - be interruptible by user requests (such as by pressing Ctrl-C). + - Signal handlers, including Python functions registered using + the :mod:`signal` module. - .. note:: - The default Python signal handler for :c:macro:`!SIGINT` raises the - :exc:`KeyboardInterrupt` exception. + Signal handlers are only run in the main thread of the main interpreter. + + (This is where the function got the name: originally, signals + were the only way to interrupt the interpreter.) + + - Running the garbage collector, if necessary. + + - Executing a pending :ref:`remote debugger ` script. + + If any handler raises an exception, immediately return ``-1`` with that + exception set. + Any remaining interruptions are left to be processed on the next + :c:func:`PyErr_CheckSignals()` invocation, if appropriate. + + If all handlers finish successfully, or there are no handlers to run, + return ``0``. + + .. versionchanged:: 3.12 + This function may now invoke the garbage collector. + + .. versionchanged:: 3.14 + This function may now execute a remote debugger script, if remote + debugging is enabled. .. c:function:: void PyErr_SetInterrupt() @@ -749,9 +803,30 @@ Exception Classes .. versionadded:: 3.2 +.. c:function:: int PyExceptionClass_Check(PyObject *ob) + + Return non-zero if *ob* is an exception class, zero otherwise. This function always succeeds. + + +.. c:function:: const char *PyExceptionClass_Name(PyObject *ob) + + Return :c:member:`~PyTypeObject.tp_name` of the exception class *ob*. + + Exception Objects ================= +.. c:function:: int PyExceptionInstance_Check(PyObject *op) + + Return true if *op* is an instance of :class:`BaseException`, false + otherwise. This function always succeeds. + + +.. c:macro:: PyExceptionInstance_Class(op) + + Equivalent to :c:func:`Py_TYPE(op) `. + + .. c:function:: PyObject* PyException_GetTraceback(PyObject *ex) Return the traceback associated with the exception as a new reference, as @@ -778,7 +853,7 @@ Exception Objects Set the context associated with the exception to *ctx*. Use ``NULL`` to clear it. There is no type check to make sure that *ctx* is an exception instance. - This steals a reference to *ctx*. + This ":term:`steals `" a reference to *ctx*. .. c:function:: PyObject* PyException_GetCause(PyObject *ex) @@ -793,7 +868,8 @@ Exception Objects Set the cause associated with the exception to *cause*. Use ``NULL`` to clear it. There is no type check to make sure that *cause* is either an exception - instance or ``None``. This steals a reference to *cause*. + instance or ``None``. + This ":term:`steals `" a reference to *cause*. The :attr:`~BaseException.__suppress_context__` attribute is implicitly set to ``True`` by this function. @@ -929,6 +1005,9 @@ because the :ref:`call protocol ` takes care of recursion handling. be concatenated to the :exc:`RecursionError` message caused by the recursion depth limit. + .. seealso:: + The :c:func:`PyUnstable_ThreadState_SetStackProtection` function. + .. versionchanged:: 3.9 This function is now also available in the :ref:`limited API `. @@ -969,184 +1048,159 @@ these are the C equivalent to :func:`reprlib.recursive_repr`. Ends a :c:func:`Py_ReprEnter`. Must be called once for each invocation of :c:func:`Py_ReprEnter` that returns zero. +.. c:function:: int Py_GetRecursionLimit(void) + + Get the recursion limit for the current interpreter. It can be set with + :c:func:`Py_SetRecursionLimit`. The recursion limit prevents the + Python interpreter stack from growing infinitely. + + This function cannot fail, and the caller must hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`sys.getrecursionlimit` + +.. c:function:: void Py_SetRecursionLimit(int new_limit) + + Set the recursion limit for the current interpreter. + + This function cannot fail, and the caller must hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`sys.setrecursionlimit` .. _standardexceptions: -Standard Exceptions -=================== - -All standard Python exceptions are available as global variables whose names are -``PyExc_`` followed by the Python exception name. These have the type -:c:expr:`PyObject*`; they are all class objects. For completeness, here are all -the variables: - -.. index:: - single: PyExc_BaseException (C var) - single: PyExc_Exception (C var) - single: PyExc_ArithmeticError (C var) - single: PyExc_AssertionError (C var) - single: PyExc_AttributeError (C var) - single: PyExc_BlockingIOError (C var) - single: PyExc_BrokenPipeError (C var) - single: PyExc_BufferError (C var) - single: PyExc_ChildProcessError (C var) - single: PyExc_ConnectionAbortedError (C var) - single: PyExc_ConnectionError (C var) - single: PyExc_ConnectionRefusedError (C var) - single: PyExc_ConnectionResetError (C var) - single: PyExc_EOFError (C var) - single: PyExc_FileExistsError (C var) - single: PyExc_FileNotFoundError (C var) - single: PyExc_FloatingPointError (C var) - single: PyExc_GeneratorExit (C var) - single: PyExc_ImportError (C var) - single: PyExc_IndentationError (C var) - single: PyExc_IndexError (C var) - single: PyExc_InterruptedError (C var) - single: PyExc_IsADirectoryError (C var) - single: PyExc_KeyError (C var) - single: PyExc_KeyboardInterrupt (C var) - single: PyExc_LookupError (C var) - single: PyExc_MemoryError (C var) - single: PyExc_ModuleNotFoundError (C var) - single: PyExc_NameError (C var) - single: PyExc_NotADirectoryError (C var) - single: PyExc_NotImplementedError (C var) - single: PyExc_OSError (C var) - single: PyExc_OverflowError (C var) - single: PyExc_PermissionError (C var) - single: PyExc_ProcessLookupError (C var) - single: PyExc_PythonFinalizationError (C var) - single: PyExc_RecursionError (C var) - single: PyExc_ReferenceError (C var) - single: PyExc_RuntimeError (C var) - single: PyExc_StopAsyncIteration (C var) - single: PyExc_StopIteration (C var) - single: PyExc_SyntaxError (C var) - single: PyExc_SystemError (C var) - single: PyExc_SystemExit (C var) - single: PyExc_TabError (C var) - single: PyExc_TimeoutError (C var) - single: PyExc_TypeError (C var) - single: PyExc_UnboundLocalError (C var) - single: PyExc_UnicodeDecodeError (C var) - single: PyExc_UnicodeEncodeError (C var) - single: PyExc_UnicodeError (C var) - single: PyExc_UnicodeTranslateError (C var) - single: PyExc_ValueError (C var) - single: PyExc_ZeroDivisionError (C var) - -+-----------------------------------------+---------------------------------+----------+ -| C Name | Python Name | Notes | -+=========================================+=================================+==========+ -| :c:data:`PyExc_BaseException` | :exc:`BaseException` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_Exception` | :exc:`Exception` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BlockingIOError` | :exc:`BlockingIOError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BufferError` | :exc:`BufferError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_GeneratorExit` | :exc:`GeneratorExit` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ImportError` | :exc:`ImportError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IndentationError` | :exc:`IndentationError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IndexError` | :exc:`IndexError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_InterruptedError` | :exc:`InterruptedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IsADirectoryError` | :exc:`IsADirectoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_KeyError` | :exc:`KeyError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_LookupError` | :exc:`LookupError` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NameError` | :exc:`NameError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NotImplementedError` | :exc:`NotImplementedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_OSError` | :exc:`OSError` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_OverflowError` | :exc:`OverflowError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PermissionError` | :exc:`PermissionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PythonFinalizationError` | :exc:`PythonFinalizationError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RecursionError` | :exc:`RecursionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_StopAsyncIteration` | :exc:`StopAsyncIteration` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_StopIteration` | :exc:`StopIteration` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SystemError` | :exc:`SystemError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TabError` | :exc:`TabError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TypeError` | :exc:`TypeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnboundLocalError` | :exc:`UnboundLocalError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeDecodeError` | :exc:`UnicodeDecodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeEncodeError` | :exc:`UnicodeEncodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeError` | :exc:`UnicodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeTranslateError` | :exc:`UnicodeTranslateError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ValueError` | :exc:`ValueError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | | -+-----------------------------------------+---------------------------------+----------+ +Exception and warning types +=========================== + +All standard Python exceptions and warning categories are available as global +variables whose names are ``PyExc_`` followed by the Python exception name. +These have the type :c:expr:`PyObject*`; they are all class objects. + +For completeness, here are all the variables: + +Exception types +--------------- + +.. list-table:: + :align: left + :widths: auto + :header-rows: 1 + + * * C name + * Python name + * * .. c:var:: PyObject *PyExc_BaseException + * :exc:`BaseException` + * * .. c:var:: PyObject *PyExc_BaseExceptionGroup + * :exc:`BaseExceptionGroup` + * * .. c:var:: PyObject *PyExc_Exception + * :exc:`Exception` + * * .. c:var:: PyObject *PyExc_ArithmeticError + * :exc:`ArithmeticError` + * * .. c:var:: PyObject *PyExc_AssertionError + * :exc:`AssertionError` + * * .. c:var:: PyObject *PyExc_AttributeError + * :exc:`AttributeError` + * * .. c:var:: PyObject *PyExc_BlockingIOError + * :exc:`BlockingIOError` + * * .. c:var:: PyObject *PyExc_BrokenPipeError + * :exc:`BrokenPipeError` + * * .. c:var:: PyObject *PyExc_BufferError + * :exc:`BufferError` + * * .. c:var:: PyObject *PyExc_ChildProcessError + * :exc:`ChildProcessError` + * * .. c:var:: PyObject *PyExc_ConnectionAbortedError + * :exc:`ConnectionAbortedError` + * * .. c:var:: PyObject *PyExc_ConnectionError + * :exc:`ConnectionError` + * * .. c:var:: PyObject *PyExc_ConnectionRefusedError + * :exc:`ConnectionRefusedError` + * * .. c:var:: PyObject *PyExc_ConnectionResetError + * :exc:`ConnectionResetError` + * * .. c:var:: PyObject *PyExc_EOFError + * :exc:`EOFError` + * * .. c:var:: PyObject *PyExc_FileExistsError + * :exc:`FileExistsError` + * * .. c:var:: PyObject *PyExc_FileNotFoundError + * :exc:`FileNotFoundError` + * * .. c:var:: PyObject *PyExc_FloatingPointError + * :exc:`FloatingPointError` + * * .. c:var:: PyObject *PyExc_GeneratorExit + * :exc:`GeneratorExit` + * * .. c:var:: PyObject *PyExc_ImportError + * :exc:`ImportError` + * * .. c:var:: PyObject *PyExc_IndentationError + * :exc:`IndentationError` + * * .. c:var:: PyObject *PyExc_IndexError + * :exc:`IndexError` + * * .. c:var:: PyObject *PyExc_InterruptedError + * :exc:`InterruptedError` + * * .. c:var:: PyObject *PyExc_IsADirectoryError + * :exc:`IsADirectoryError` + * * .. c:var:: PyObject *PyExc_KeyError + * :exc:`KeyError` + * * .. c:var:: PyObject *PyExc_KeyboardInterrupt + * :exc:`KeyboardInterrupt` + * * .. c:var:: PyObject *PyExc_LookupError + * :exc:`LookupError` + * * .. c:var:: PyObject *PyExc_MemoryError + * :exc:`MemoryError` + * * .. c:var:: PyObject *PyExc_ModuleNotFoundError + * :exc:`ModuleNotFoundError` + * * .. c:var:: PyObject *PyExc_NameError + * :exc:`NameError` + * * .. c:var:: PyObject *PyExc_NotADirectoryError + * :exc:`NotADirectoryError` + * * .. c:var:: PyObject *PyExc_NotImplementedError + * :exc:`NotImplementedError` + * * .. c:var:: PyObject *PyExc_OSError + * :exc:`OSError` + * * .. c:var:: PyObject *PyExc_OverflowError + * :exc:`OverflowError` + * * .. c:var:: PyObject *PyExc_PermissionError + * :exc:`PermissionError` + * * .. c:var:: PyObject *PyExc_ProcessLookupError + * :exc:`ProcessLookupError` + * * .. c:var:: PyObject *PyExc_PythonFinalizationError + * :exc:`PythonFinalizationError` + * * .. c:var:: PyObject *PyExc_RecursionError + * :exc:`RecursionError` + * * .. c:var:: PyObject *PyExc_ReferenceError + * :exc:`ReferenceError` + * * .. c:var:: PyObject *PyExc_RuntimeError + * :exc:`RuntimeError` + * * .. c:var:: PyObject *PyExc_StopAsyncIteration + * :exc:`StopAsyncIteration` + * * .. c:var:: PyObject *PyExc_StopIteration + * :exc:`StopIteration` + * * .. c:var:: PyObject *PyExc_SyntaxError + * :exc:`SyntaxError` + * * .. c:var:: PyObject *PyExc_SystemError + * :exc:`SystemError` + * * .. c:var:: PyObject *PyExc_SystemExit + * :exc:`SystemExit` + * * .. c:var:: PyObject *PyExc_TabError + * :exc:`TabError` + * * .. c:var:: PyObject *PyExc_TimeoutError + * :exc:`TimeoutError` + * * .. c:var:: PyObject *PyExc_TypeError + * :exc:`TypeError` + * * .. c:var:: PyObject *PyExc_UnboundLocalError + * :exc:`UnboundLocalError` + * * .. c:var:: PyObject *PyExc_UnicodeDecodeError + * :exc:`UnicodeDecodeError` + * * .. c:var:: PyObject *PyExc_UnicodeEncodeError + * :exc:`UnicodeEncodeError` + * * .. c:var:: PyObject *PyExc_UnicodeError + * :exc:`UnicodeError` + * * .. c:var:: PyObject *PyExc_UnicodeTranslateError + * :exc:`UnicodeTranslateError` + * * .. c:var:: PyObject *PyExc_ValueError + * :exc:`ValueError` + * * .. c:var:: PyObject *PyExc_ZeroDivisionError + * :exc:`ZeroDivisionError` .. versionadded:: 3.3 :c:data:`PyExc_BlockingIOError`, :c:data:`PyExc_BrokenPipeError`, @@ -1164,88 +1218,116 @@ the variables: .. versionadded:: 3.6 :c:data:`PyExc_ModuleNotFoundError`. -These are compatibility aliases to :c:data:`PyExc_OSError`: +.. versionadded:: 3.11 + :c:data:`PyExc_BaseExceptionGroup`. + -.. index:: - single: PyExc_EnvironmentError (C var) - single: PyExc_IOError (C var) - single: PyExc_WindowsError (C var) +OSError aliases +--------------- -+-------------------------------------+----------+ -| C Name | Notes | -+=====================================+==========+ -| :c:data:`!PyExc_EnvironmentError` | | -+-------------------------------------+----------+ -| :c:data:`!PyExc_IOError` | | -+-------------------------------------+----------+ -| :c:data:`!PyExc_WindowsError` | [2]_ | -+-------------------------------------+----------+ +The following are a compatibility aliases to :c:data:`PyExc_OSError`. .. versionchanged:: 3.3 These aliases used to be separate exception types. +.. list-table:: + :align: left + :widths: auto + :header-rows: 1 + + * * C name + * Python name + * Notes + * * .. c:var:: PyObject *PyExc_EnvironmentError + * :exc:`OSError` + * + * * .. c:var:: PyObject *PyExc_IOError + * :exc:`OSError` + * + * * .. c:var:: PyObject *PyExc_WindowsError + * :exc:`OSError` + * [win]_ + Notes: -.. [1] - This is a base class for other standard exceptions. +.. [win] + :c:var:`!PyExc_WindowsError` is only defined on Windows; protect code that + uses this by testing that the preprocessor macro ``MS_WINDOWS`` is defined. -.. [2] - Only defined on Windows; protect code that uses this by testing that the - preprocessor macro ``MS_WINDOWS`` is defined. .. _standardwarningcategories: -Standard Warning Categories -=========================== - -All standard Python warning categories are available as global variables whose -names are ``PyExc_`` followed by the Python exception name. These have the type -:c:expr:`PyObject*`; they are all class objects. For completeness, here are all -the variables: - -.. index:: - single: PyExc_Warning (C var) - single: PyExc_BytesWarning (C var) - single: PyExc_DeprecationWarning (C var) - single: PyExc_FutureWarning (C var) - single: PyExc_ImportWarning (C var) - single: PyExc_PendingDeprecationWarning (C var) - single: PyExc_ResourceWarning (C var) - single: PyExc_RuntimeWarning (C var) - single: PyExc_SyntaxWarning (C var) - single: PyExc_UnicodeWarning (C var) - single: PyExc_UserWarning (C var) - -+------------------------------------------+---------------------------------+----------+ -| C Name | Python Name | Notes | -+==========================================+=================================+==========+ -| :c:data:`PyExc_Warning` | :exc:`Warning` | [3]_ | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BytesWarning` | :exc:`BytesWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PendingDeprecationWarning`| :exc:`PendingDeprecationWarning`| | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ResourceWarning` | :exc:`ResourceWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RuntimeWarning` | :exc:`RuntimeWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SyntaxWarning` | :exc:`SyntaxWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeWarning` | :exc:`UnicodeWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UserWarning` | :exc:`UserWarning` | | -+------------------------------------------+---------------------------------+----------+ +Warning types +------------- + +.. list-table:: + :align: left + :widths: auto + :header-rows: 1 + + * * C name + * Python name + * * .. c:var:: PyObject *PyExc_Warning + * :exc:`Warning` + * * .. c:var:: PyObject *PyExc_BytesWarning + * :exc:`BytesWarning` + * * .. c:var:: PyObject *PyExc_DeprecationWarning + * :exc:`DeprecationWarning` + * * .. c:var:: PyObject *PyExc_EncodingWarning + * :exc:`EncodingWarning` + * * .. c:var:: PyObject *PyExc_FutureWarning + * :exc:`FutureWarning` + * * .. c:var:: PyObject *PyExc_ImportWarning + * :exc:`ImportWarning` + * * .. c:var:: PyObject *PyExc_PendingDeprecationWarning + * :exc:`PendingDeprecationWarning` + * * .. c:var:: PyObject *PyExc_ResourceWarning + * :exc:`ResourceWarning` + * * .. c:var:: PyObject *PyExc_RuntimeWarning + * :exc:`RuntimeWarning` + * * .. c:var:: PyObject *PyExc_SyntaxWarning + * :exc:`SyntaxWarning` + * * .. c:var:: PyObject *PyExc_UnicodeWarning + * :exc:`UnicodeWarning` + * * .. c:var:: PyObject *PyExc_UserWarning + * :exc:`UserWarning` .. versionadded:: 3.2 :c:data:`PyExc_ResourceWarning`. -Notes: +.. versionadded:: 3.10 + :c:data:`PyExc_EncodingWarning`. + + +Tracebacks +========== + +.. c:var:: PyTypeObject PyTraceBack_Type + + Type object for traceback objects. This is available as + :class:`types.TracebackType` in the Python layer. + + +.. c:function:: int PyTraceBack_Check(PyObject *op) + + Return true if *op* is a traceback object, false otherwise. This function + does not account for subtypes. + + +.. c:function:: int PyTraceBack_Here(PyFrameObject *f) + + Replace the :attr:`~BaseException.__traceback__` attribute on the current + exception with a new traceback prepending *f* to the existing chain. + + Calling this function without an exception set is undefined behavior. + + This function returns ``0`` on success, and returns ``-1`` with an + exception set on failure. + + +.. c:function:: int PyTraceBack_Print(PyObject *tb, PyObject *f) + + Write the traceback *tb* into the file *f*. -.. [3] - This is a base class for other standard warning categories. + This function returns ``0`` on success, and returns ``-1`` with an + exception set on failure. diff --git a/Doc/c-api/extension-modules.rst b/Doc/c-api/extension-modules.rst new file mode 100644 index 000000000000000..3d331e6ec12f76e --- /dev/null +++ b/Doc/c-api/extension-modules.rst @@ -0,0 +1,247 @@ +.. highlight:: c + +.. _extension-modules: + +Defining extension modules +-------------------------- + +A C extension for CPython is a shared library (for example, a ``.so`` file +on Linux, ``.pyd`` DLL on Windows), which is loadable into the Python process +(for example, it is compiled with compatible compiler settings), and which +exports an :ref:`initialization function `. + +To be importable by default (that is, by +:py:class:`importlib.machinery.ExtensionFileLoader`), +the shared library must be available on :py:attr:`sys.path`, +and must be named after the module name plus an extension listed in +:py:attr:`importlib.machinery.EXTENSION_SUFFIXES`. + +.. note:: + + Building, packaging and distributing extension modules is best done with + third-party tools, and is out of scope of this document. + One suitable tool is Setuptools, whose documentation can be found at + https://setuptools.pypa.io/en/latest/setuptools.html. + +Normally, the initialization function returns a module definition initialized +using :c:func:`PyModuleDef_Init`. +This allows splitting the creation process into several phases: + +- Before any substantial code is executed, Python can determine which + capabilities the module supports, and it can adjust the environment or + refuse loading an incompatible extension. +- By default, Python itself creates the module object -- that is, it does + the equivalent of :py:meth:`object.__new__` for classes. + It also sets initial attributes like :attr:`~module.__package__` and + :attr:`~module.__loader__`. +- Afterwards, the module object is initialized using extension-specific + code -- the equivalent of :py:meth:`~object.__init__` on classes. + +This is called *multi-phase initialization* to distinguish it from the legacy +(but still supported) *single-phase initialization* scheme, +where the initialization function returns a fully constructed module. +See the :ref:`single-phase-initialization section below ` +for details. + +.. versionchanged:: 3.5 + + Added support for multi-phase initialization (:pep:`489`). + + +Multiple module instances +......................... + +By default, extension modules are not singletons. +For example, if the :py:attr:`sys.modules` entry is removed and the module +is re-imported, a new module object is created, and typically populated with +fresh method and type objects. +The old module is subject to normal garbage collection. +This mirrors the behavior of pure-Python modules. + +Additional module instances may be created in +:ref:`sub-interpreters ` +or after Python runtime reinitialization +(:c:func:`Py_Finalize` and :c:func:`Py_Initialize`). +In these cases, sharing Python objects between module instances would likely +cause crashes or undefined behavior. + +To avoid such issues, each instance of an extension module should +be *isolated*: changes to one instance should not implicitly affect the others, +and all state owned by the module, including references to Python objects, +should be specific to a particular module instance. +See :ref:`isolating-extensions-howto` for more details and a practical guide. + +A simpler way to avoid these issues is +:ref:`raising an error on repeated initialization `. + +All modules are expected to support +:ref:`sub-interpreters `, or otherwise explicitly +signal a lack of support. +This is usually achieved by isolation or blocking repeated initialization, +as above. +A module may also be limited to the main interpreter using +the :c:data:`Py_mod_multiple_interpreters` slot. + + +.. _extension-export-hook: + +Initialization function +....................... + +The initialization function defined by an extension module has the +following signature: + +.. c:function:: PyObject* PyInit_modulename(void) + +Its name should be :samp:`PyInit_{}`, with ```` replaced by the +name of the module. + +For modules with ASCII-only names, the function must instead be named +:samp:`PyInit_{}`, with ```` replaced by the name of the module. +When using :ref:`multi-phase-initialization`, non-ASCII module names +are allowed. In this case, the initialization function name is +:samp:`PyInitU_{}`, with ```` encoded using Python's +*punycode* encoding with hyphens replaced by underscores. In Python: + +.. code-block:: python + + def initfunc_name(name): + try: + suffix = b'_' + name.encode('ascii') + except UnicodeEncodeError: + suffix = b'U_' + name.encode('punycode').replace(b'-', b'_') + return b'PyInit' + suffix + +It is recommended to define the initialization function using a helper macro: + +.. c:macro:: PyMODINIT_FUNC + + Declare an extension module initialization function. + This macro: + + * specifies the :c:expr:`PyObject*` return type, + * adds any special linkage declarations required by the platform, and + * for C++, declares the function as ``extern "C"``. + +For example, a module called ``spam`` would be defined like this:: + + static struct PyModuleDef spam_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "spam", + ... + }; + + PyMODINIT_FUNC + PyInit_spam(void) + { + return PyModuleDef_Init(&spam_module); + } + +It is possible to export multiple modules from a single shared library by +defining multiple initialization functions. However, importing them requires +using symbolic links or a custom importer, because by default only the +function corresponding to the filename is found. +See the `Multiple modules in one library `__ +section in :pep:`489` for details. + +The initialization function is typically the only non-\ ``static`` +item defined in the module's C source. + + +.. _multi-phase-initialization: + +Multi-phase initialization +.......................... + +Normally, the :ref:`initialization function ` +(``PyInit_modulename``) returns a :c:type:`PyModuleDef` instance with +non-``NULL`` :c:member:`~PyModuleDef.m_slots`. +Before it is returned, the ``PyModuleDef`` instance must be initialized +using the following function: + + +.. c:function:: PyObject* PyModuleDef_Init(PyModuleDef *def) + + Ensure a module definition is a properly initialized Python object that + correctly reports its type and a reference count. + + Return *def* cast to ``PyObject*``, or ``NULL`` if an error occurred. + + Calling this function is required for :ref:`multi-phase-initialization`. + It should not be used in other contexts. + + Note that Python assumes that ``PyModuleDef`` structures are statically + allocated. + This function may return either a new reference or a borrowed one; + this reference must not be released. + + .. versionadded:: 3.5 + + +.. _single-phase-initialization: + +Legacy single-phase initialization +.................................. + +.. attention:: + Single-phase initialization is a legacy mechanism to initialize extension + modules, with known drawbacks and design flaws. Extension module authors + are encouraged to use multi-phase initialization instead. + +In single-phase initialization, the +:ref:`initialization function ` (``PyInit_modulename``) +should create, populate and return a module object. +This is typically done using :c:func:`PyModule_Create` and functions like +:c:func:`PyModule_AddObjectRef`. + +Single-phase initialization differs from the :ref:`default ` +in the following ways: + +* Single-phase modules are, or rather *contain*, “singletons”. + + When the module is first initialized, Python saves the contents of + the module's ``__dict__`` (that is, typically, the module's functions and + types). + + For subsequent imports, Python does not call the initialization function + again. + Instead, it creates a new module object with a new ``__dict__``, and copies + the saved contents to it. + For example, given a single-phase module ``_testsinglephase`` + [#testsinglephase]_ that defines a function ``sum`` and an exception class + ``error``: + + .. code-block:: python + + >>> import sys + >>> import _testsinglephase as one + >>> del sys.modules['_testsinglephase'] + >>> import _testsinglephase as two + >>> one is two + False + >>> one.__dict__ is two.__dict__ + False + >>> one.sum is two.sum + True + >>> one.error is two.error + True + + The exact behavior should be considered a CPython implementation detail. + +* To work around the fact that ``PyInit_modulename`` does not take a *spec* + argument, some state of the import machinery is saved and applied to the + first suitable module created during the ``PyInit_modulename`` call. + Specifically, when a sub-module is imported, this mechanism prepends the + parent package name to the name of the module. + + A single-phase ``PyInit_modulename`` function should create “its” module + object as soon as possible, before any other module objects can be created. + +* Non-ASCII module names (``PyInitU_modulename``) are not supported. + +* Single-phase modules support module lookup functions like + :c:func:`PyState_FindModule`. + +.. [#testsinglephase] ``_testsinglephase`` is an internal module used + in CPython's self-test suite; your installation may or may not + include it. diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index e9019a0d500f7e5..dcafefdc0458722 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -2,7 +2,7 @@ .. _fileobjects: -File Objects +File objects ------------ .. index:: pair: object; file @@ -93,6 +93,29 @@ the :mod:`io` APIs instead. .. versionadded:: 3.8 +.. c:function:: PyObject *PyFile_OpenCodeObject(PyObject *path) + + Open *path* with the mode ``'rb'``. *path* must be a Python :class:`str` + object. The behavior of this function may be overridden by + :c:func:`PyFile_SetOpenCodeHook` to allow for some preprocessing of the + text. + + This is analogous to :func:`io.open_code` in Python. + + On success, this function returns a :term:`strong reference` to a Python + file object. On failure, this function returns ``NULL`` with an exception + set. + + .. versionadded:: 3.8 + + +.. c:function:: PyObject *PyFile_OpenCode(const char *path) + + Similar to :c:func:`PyFile_OpenCodeObject`, but *path* is a + UTF-8 encoded :c:expr:`const char*`. + + .. versionadded:: 3.8 + .. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags) @@ -100,11 +123,34 @@ the :mod:`io` APIs instead. Write object *obj* to file object *p*. The only supported flag for *flags* is :c:macro:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written - instead of the :func:`repr`. Return ``0`` on success or ``-1`` on failure; the - appropriate exception will be set. + instead of the :func:`repr`. + + If *obj* is ``NULL``, write the string ``""``. + Return ``0`` on success or ``-1`` on failure; the + appropriate exception will be set. .. c:function:: int PyFile_WriteString(const char *s, PyObject *p) Write string *s* to file object *p*. Return ``0`` on success or ``-1`` on failure; the appropriate exception will be set. + + +Soft-deprecated API +^^^^^^^^^^^^^^^^^^^ + +.. soft-deprecated:: 3.15 + +These are APIs that were included in Python's C API +by mistake. They are documented solely for completeness; use other +``PyFile*`` APIs instead. + +.. c:function:: PyObject *PyFile_NewStdPrinter(int fd) + + Use :c:func:`PyFile_FromFd` with defaults (``fd, NULL, "w", -1, NULL, NULL, NULL, 0``) instead. + +.. c:var:: PyTypeObject PyStdPrinter_Type + + Type of file-like objects used internally at Python startup when :py:mod:`io` is + not yet available. + Use Python :py:func:`open` or :c:func:`PyFile_FromFd` to create file objects instead. diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index c5a7653efca26bc..420f7f9401fcc4c 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -78,6 +78,104 @@ Floating-Point Objects Return the minimum normalized positive float *DBL_MIN* as C :c:expr:`double`. +.. c:macro:: Py_INFINITY + + This macro expands to a constant expression of type :c:expr:`double`, that + represents the positive infinity. + + On most platforms, this is equivalent to the :c:macro:`!INFINITY` macro from + the C11 standard ```` header. + + +.. c:macro:: Py_NAN + + This macro expands to a constant expression of type :c:expr:`double`, that + represents a quiet not-a-number (qNaN) value. + + On most platforms, this is equivalent to the :c:macro:`!NAN` macro from + the C11 standard ```` header. + + +.. c:macro:: Py_HUGE_VAL + + Equivalent to :c:macro:`!INFINITY`. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. + + +.. c:macro:: Py_MATH_E + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.e` constant. + + +.. c:macro:: Py_MATH_El + + High precision (long double) definition of :data:`~math.e` constant. + + +.. c:macro:: Py_MATH_PI + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.pi` constant. + + +.. c:macro:: Py_MATH_PIl + + High precision (long double) definition of :data:`~math.pi` constant. + + +.. c:macro:: Py_MATH_TAU + + The definition (accurate for a :c:expr:`double` type) of the :data:`math.tau` constant. + + .. versionadded:: 3.6 + + +.. c:macro:: Py_RETURN_NAN + + Return :data:`math.nan` from a function. + + On most platforms, this is equivalent to ``return PyFloat_FromDouble(NAN)``. + + +.. c:macro:: Py_RETURN_INF(sign) + + Return :data:`math.inf` or :data:`-math.inf ` from a function, + depending on the sign of *sign*. + + On most platforms, this is equivalent to the following:: + + return PyFloat_FromDouble(copysign(INFINITY, sign)); + + +.. c:macro:: Py_IS_FINITE(X) + + Return ``1`` if the given floating-point number *X* is finite, + that is, it is normal, subnormal or zero, but not infinite or NaN. + Return ``0`` otherwise. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. Use :c:macro:`!isfinite` instead. + + +.. c:macro:: Py_IS_INFINITY(X) + + Return ``1`` if the given floating-point number *X* is positive or negative + infinity. Return ``0`` otherwise. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. Use :c:macro:`!isinf` instead. + + +.. c:macro:: Py_IS_NAN(X) + + Return ``1`` if the given floating-point number *X* is a not-a-number (NaN) + value. Return ``0`` otherwise. + + .. deprecated:: 3.14 + The macro is :term:`soft deprecated`. Use :c:macro:`!isnan` instead. + + Pack and Unpack functions ------------------------- @@ -96,8 +194,8 @@ NaNs (if such things exist on the platform) isn't handled correctly, and attempting to unpack a bytes string containing an IEEE INF or NaN will raise an exception. -Note that NaNs type may not be preserved on IEEE platforms (silent NaN become -quiet), for example on x86 systems in 32-bit mode. +Note that NaN type may not be preserved on IEEE platforms (signaling NaNs become +quiet NaNs), for example on x86 systems in 32-bit mode. On non-IEEE platforms with more precision, or larger dynamic range, than IEEE 754 supports, not all values can be packed; on non-IEEE platforms with less @@ -111,7 +209,7 @@ Pack functions The pack routines write 2, 4 or 8 bytes, starting at *p*. *le* is an :c:expr:`int` argument, non-zero if you want the bytes string in little-endian -format (exponent last, at ``p+1``, ``p+3``, or ``p+6`` ``p+7``), zero if you +format (exponent last, at ``p+1``, ``p+3``, or ``p+6`` and ``p+7``), zero if you want big-endian format (exponent first, at *p*). The :c:macro:`PY_BIG_ENDIAN` constant can be used to use the native endian: it is equal to ``1`` on big endian processor, or ``0`` on little endian processor. @@ -124,15 +222,15 @@ There are two problems on non-IEEE platforms: * What this does is undefined if *x* is a NaN or infinity. * ``-0.0`` and ``+0.0`` produce the same bytes string. -.. c:function:: int PyFloat_Pack2(double x, unsigned char *p, int le) +.. c:function:: int PyFloat_Pack2(double x, char *p, int le) Pack a C double as the IEEE 754 binary16 half-precision format. -.. c:function:: int PyFloat_Pack4(double x, unsigned char *p, int le) +.. c:function:: int PyFloat_Pack4(double x, char *p, int le) Pack a C double as the IEEE 754 binary32 single precision format. -.. c:function:: int PyFloat_Pack8(double x, unsigned char *p, int le) +.. c:function:: int PyFloat_Pack8(double x, char *p, int le) Pack a C double as the IEEE 754 binary64 double precision format. @@ -154,14 +252,14 @@ Return value: The unpacked double. On error, this is ``-1.0`` and Note that on a non-IEEE platform this will refuse to unpack a bytes string that represents a NaN or infinity. -.. c:function:: double PyFloat_Unpack2(const unsigned char *p, int le) +.. c:function:: double PyFloat_Unpack2(const char *p, int le) Unpack the IEEE 754 binary16 half-precision format as a C double. -.. c:function:: double PyFloat_Unpack4(const unsigned char *p, int le) +.. c:function:: double PyFloat_Unpack4(const char *p, int le) Unpack the IEEE 754 binary32 single precision format as a C double. -.. c:function:: double PyFloat_Unpack8(const unsigned char *p, int le) +.. c:function:: double PyFloat_Unpack8(const char *p, int le) Unpack the IEEE 754 binary64 double precision format as a C double. diff --git a/Doc/c-api/frame.rst b/Doc/c-api/frame.rst index 1a52e146a697510..a04ef4e422949b9 100644 --- a/Doc/c-api/frame.rst +++ b/Doc/c-api/frame.rst @@ -1,6 +1,6 @@ .. highlight:: c -Frame Objects +Frame objects ------------- .. c:type:: PyFrameObject @@ -29,6 +29,12 @@ See also :ref:`Reflection `. Previously, this type was only available after including ````. +.. c:function:: PyFrameObject *PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals, PyObject *locals) + + Create a new frame object. This function returns a :term:`strong reference` + to the new frame object on success, and returns ``NULL`` with an exception + set on failure. + .. c:function:: int PyFrame_Check(PyObject *obj) Return non-zero if *obj* is a frame object. @@ -44,6 +50,7 @@ See also :ref:`Reflection `. Return a :term:`strong reference`, or ``NULL`` if *frame* has no outer frame. + This raises no exceptions. .. versionadded:: 3.9 @@ -140,7 +147,7 @@ See also :ref:`Reflection `. Return the line number that *frame* is currently executing. -Frame Locals Proxies +Frame locals proxies ^^^^^^^^^^^^^^^^^^^^ .. versionadded:: 3.13 @@ -161,7 +168,52 @@ See :pep:`667` for more information. Return non-zero if *obj* is a frame :func:`locals` proxy. -Internal Frames + +Legacy local variable APIs +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These APIs are :term:`soft deprecated`. As of Python 3.13, they do nothing. +They exist solely for backwards compatibility. + + +.. c:function:: void PyFrame_LocalsToFast(PyFrameObject *f, int clear) + + Prior to Python 3.13, this function would copy the :attr:`~frame.f_locals` + attribute of *f* to the internal "fast" array of local variables, allowing + changes in frame objects to be visible to the interpreter. If *clear* was + true, this function would process variables that were unset in the locals + dictionary. + + .. soft-deprecated:: 3.13 + This function now does nothing. + + +.. c:function:: void PyFrame_FastToLocals(PyFrameObject *f) + + Prior to Python 3.13, this function would copy the internal "fast" array + of local variables (which is used by the interpreter) to the + :attr:`~frame.f_locals` attribute of *f*, allowing changes in local + variables to be visible to frame objects. + + .. soft-deprecated:: 3.13 + This function now does nothing. + + +.. c:function:: int PyFrame_FastToLocalsWithError(PyFrameObject *f) + + Prior to Python 3.13, this function was similar to + :c:func:`PyFrame_FastToLocals`, but would return ``0`` on success, and + ``-1`` with an exception set on failure. + + .. soft-deprecated:: 3.13 + This function now does nothing. + + +.. seealso:: + :pep:`667` + + +Internal frames ^^^^^^^^^^^^^^^ Unless using :pep:`523`, you will not need this. @@ -193,3 +245,60 @@ Unless using :pep:`523`, you will not need this. .. versionadded:: 3.12 +.. c:var:: const PyTypeObject *PyUnstable_ExecutableKinds + + An array of executable kinds (executor types) for frames, used for internal + debugging and tracing. + + Tools like debuggers and profilers can use this to identify the type of execution + context associated with a frame (such as to filter out internal frames). + The entries are indexed by the following constants: + + .. list-table:: + :header-rows: 1 + :widths: auto + + * - Constant + - Description + * - .. c:macro:: PyUnstable_EXECUTABLE_KIND_SKIP + - The frame is internal (For example: inlined) and should be skipped by tools. + * - .. c:macro:: PyUnstable_EXECUTABLE_KIND_PY_FUNCTION + - The frame corresponds to a standard Python function. + * - .. c:macro:: PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION + - The frame corresponds to a function defined in native code. + * - .. c:macro:: PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR + - The frame corresponds to a method on a class instance. + + However, Python's C API lacks a function to read the executable kind from + a frame. Instead, use this recipe: + + .. code-block:: c + + int + get_executable_kind(PyFrameObject *frame) + { + _PyInterpreterFrame *f = frame->f_frame; + PyObject *exec = PyStackRef_AsPyObjectBorrow(f->f_executable); + + if (PyCode_Check(exec)) { + return PyUnstable_EXECUTABLE_KIND_PY_FUNCTION; + } + if (PyMethod_Check(exec)) { + return PyUnstable_EXECUTABLE_KIND_BUILTIN_FUNCTION; + } + if (Py_IS_TYPE(exec, &PyMethodDescr_Type)) { + return PyUnstable_EXECUTABLE_KIND_METHOD_DESCRIPTOR; + } + + return PyUnstable_EXECUTABLE_KIND_SKIP; + } + + .. versionadded:: 3.13 + + +.. c:macro:: PyUnstable_EXECUTABLE_KINDS + + The number of entries in :c:data:`PyUnstable_ExecutableKinds`. + + .. versionadded:: 3.13 + diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index 58792edeed25e3d..7908e4f8561aebe 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -95,6 +95,22 @@ There are a few functions specific to Python functions. .. versionadded:: 3.12 + +.. c:function:: PyObject* PyFunction_GetKwDefaults(PyObject *op) + + Return the keyword-only argument default values of the function object *op*. This can be a + dictionary of arguments or ``NULL``. + + +.. c:function:: int PyFunction_SetKwDefaults(PyObject *op, PyObject *defaults) + + Set the keyword-only argument default values of the function object *op*. + *defaults* must be a dictionary of keyword-only arguments or ``Py_None``. + + This function returns ``0`` on success, and returns ``-1`` with an exception + set on failure. + + .. c:function:: PyObject* PyFunction_GetClosure(PyObject *op) Return the closure associated with the function object *op*. This can be ``NULL`` @@ -123,6 +139,19 @@ There are a few functions specific to Python functions. Raises :exc:`SystemError` and returns ``-1`` on failure. +.. c:function:: PyObject *PyFunction_GET_CODE(PyObject *op) + PyObject *PyFunction_GET_GLOBALS(PyObject *op) + PyObject *PyFunction_GET_MODULE(PyObject *op) + PyObject *PyFunction_GET_DEFAULTS(PyObject *op) + PyObject *PyFunction_GET_KW_DEFAULTS(PyObject *op) + PyObject *PyFunction_GET_CLOSURE(PyObject *op) + PyObject *PyFunction_GET_ANNOTATIONS(PyObject *op) + + These functions are similar to their ``PyFunction_Get*`` counterparts, but + do not do type checking. Passing anything other than an instance of + :c:data:`PyFunction_Type` is undefined behavior. + + .. c:function:: int PyFunction_AddWatcher(PyFunction_WatchCallback callback) Register *callback* as a function watcher for the current interpreter. @@ -169,7 +198,7 @@ There are a few functions specific to Python functions. unpredictable effects, including infinite recursion. If *event* is ``PyFunction_EVENT_CREATE``, then the callback is invoked - after `func` has been fully initialized. Otherwise, the callback is invoked + after *func* has been fully initialized. Otherwise, the callback is invoked before the modification to *func* takes place, so the prior state of *func* can be inspected. The runtime is permitted to optimize away the creation of function objects when possible. In such cases no event will be emitted. diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index d1f0982b8189311..fed795b1e8c9637 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -57,11 +57,49 @@ rules: Analogous to :c:macro:`PyObject_New` but for container objects with the :c:macro:`Py_TPFLAGS_HAVE_GC` flag set. + Do not call this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot, + :c:func:`PyType_GenericAlloc` is preferred over a custom function that + simply calls this macro. + + Memory allocated by this macro must be freed with + :c:func:`PyObject_GC_Del` (usually called via the object's + :c:member:`~PyTypeObject.tp_free` slot). + + .. seealso:: + + * :c:func:`PyObject_GC_Del` + * :c:macro:`PyObject_New` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_alloc` + + .. c:macro:: PyObject_GC_NewVar(TYPE, typeobj, size) Analogous to :c:macro:`PyObject_NewVar` but for container objects with the :c:macro:`Py_TPFLAGS_HAVE_GC` flag set. + Do not call this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + When populating a type's :c:member:`~PyTypeObject.tp_alloc` slot, + :c:func:`PyType_GenericAlloc` is preferred over a custom function that + simply calls this macro. + + Memory allocated by this macro must be freed with + :c:func:`PyObject_GC_Del` (usually called via the object's + :c:member:`~PyTypeObject.tp_free` slot). + + .. seealso:: + + * :c:func:`PyObject_GC_Del` + * :c:macro:`PyObject_NewVar` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_alloc` + + .. c:function:: PyObject* PyUnstable_Object_GC_NewWithExtraData(PyTypeObject *type, size_t extra_size) Analogous to :c:macro:`PyObject_GC_New` but allocates *extra_size* @@ -73,6 +111,10 @@ rules: The extra data will be deallocated with the object, but otherwise it is not managed by Python. + Memory allocated by this function must be freed with + :c:func:`PyObject_GC_Del` (usually called via the object's + :c:member:`~PyTypeObject.tp_free` slot). + .. warning:: The function is marked as unstable because the final mechanism for reserving extra data after an instance is not yet decided. @@ -136,6 +178,21 @@ rules: Releases memory allocated to an object using :c:macro:`PyObject_GC_New` or :c:macro:`PyObject_GC_NewVar`. + Do not call this directly to free an object's memory; call the type's + :c:member:`~PyTypeObject.tp_free` slot instead. + + Do not use this for memory allocated by :c:macro:`PyObject_New`, + :c:macro:`PyObject_NewVar`, or related allocation functions; use + :c:func:`PyObject_Free` instead. + + .. seealso:: + + * :c:func:`PyObject_Free` is the non-GC equivalent of this function. + * :c:macro:`PyObject_GC_New` + * :c:macro:`PyObject_GC_NewVar` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_free` + .. c:function:: void PyObject_GC_UnTrack(void *op) @@ -175,14 +232,18 @@ The :c:member:`~PyTypeObject.tp_traverse` handler must have the following type: object argument. If *visit* returns a non-zero value that value should be returned immediately. + The traversal function must not have any side effects. Implementations + may not modify the reference counts of any Python objects nor create or + destroy any Python objects. + To simplify writing :c:member:`~PyTypeObject.tp_traverse` handlers, a :c:func:`Py_VISIT` macro is provided. In order to use this macro, the :c:member:`~PyTypeObject.tp_traverse` implementation must name its arguments exactly *visit* and *arg*: -.. c:function:: void Py_VISIT(PyObject *o) +.. c:macro:: Py_VISIT(o) - If *o* is not ``NULL``, call the *visit* callback, with arguments *o* + If the :c:expr:`PyObject *` *o* is not ``NULL``, call the *visit* callback, with arguments *o* and *arg*. If *visit* returns a non-zero value, then return it. Using this macro, :c:member:`~PyTypeObject.tp_traverse` handlers look like:: diff --git a/Doc/c-api/gen.rst b/Doc/c-api/gen.rst index 0eb5922f6da75f2..8648d529170c82b 100644 --- a/Doc/c-api/gen.rst +++ b/Doc/c-api/gen.rst @@ -35,12 +35,65 @@ than explicitly calling :c:func:`PyGen_New` or :c:func:`PyGen_NewWithQualName`. .. c:function:: PyObject* PyGen_New(PyFrameObject *frame) Create and return a new generator object based on the *frame* object. - A reference to *frame* is stolen by this function. The argument must not be - ``NULL``. + A reference to *frame* is ":term:`stolen `" by this function (even + on error). The argument must not be ``NULL``. .. c:function:: PyObject* PyGen_NewWithQualName(PyFrameObject *frame, PyObject *name, PyObject *qualname) Create and return a new generator object based on the *frame* object, with ``__name__`` and ``__qualname__`` set to *name* and *qualname*. - A reference to *frame* is stolen by this function. The *frame* argument + A reference to *frame* is ":term:`stolen `" by this function (even + on error). The *frame* argument must not be ``NULL``. + + +.. c:function:: PyCodeObject* PyGen_GetCode(PyGenObject *gen) + + Return a new :term:`strong reference` to the code object wrapped by *gen*. + This function always succeeds. + + +Asynchronous Generator Objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. seealso:: + :pep:`525` + +.. c:var:: PyTypeObject PyAsyncGen_Type + + The type object corresponding to asynchronous generator objects. This is + available as :class:`types.AsyncGeneratorType` in the Python layer. + + .. versionadded:: 3.6 + +.. c:function:: PyObject *PyAsyncGen_New(PyFrameObject *frame, PyObject *name, PyObject *qualname) + + Create a new asynchronous generator wrapping *frame*, with ``__name__`` and + ``__qualname__`` set to *name* and *qualname*. + *frame* is ":term:`stolen `" by this function (even on error) and must not be ``NULL``. + + On success, this function returns a :term:`strong reference` to the + new asynchronous generator. On failure, this function returns ``NULL`` + with an exception set. + + .. versionadded:: 3.6 + +.. c:function:: int PyAsyncGen_CheckExact(PyObject *op) + + Return true if *op* is an asynchronous generator object, false otherwise. + This function always succeeds. + + .. versionadded:: 3.6 + + +Deprecated API +^^^^^^^^^^^^^^ + +.. c:macro:: PyAsyncGenASend_CheckExact(op) + + This is an API that was included in Python's C API + by mistake. + + It is solely here for completeness; do not use this API. + + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/hash.rst b/Doc/c-api/hash.rst index 00f8cb887dc7ebf..1ad712b0ce4f2b1 100644 --- a/Doc/c-api/hash.rst +++ b/Doc/c-api/hash.rst @@ -11,47 +11,103 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. .. versionadded:: 3.2 + .. c:type:: Py_uhash_t Hash value type: unsigned integer. .. versionadded:: 3.2 + +.. c:macro:: Py_HASH_ALGORITHM + + A numerical value indicating the algorithm for hashing of :class:`str`, + :class:`bytes`, and :class:`memoryview`. + + The algorithm name is exposed by :data:`sys.hash_info.algorithm`. + + .. versionadded:: 3.4 + + +.. c:macro:: Py_HASH_FNV + Py_HASH_SIPHASH24 + Py_HASH_SIPHASH13 + + Numerical values to compare to :c:macro:`Py_HASH_ALGORITHM` to determine + which algorithm is used for hashing. The hash algorithm can be configured + via the configure :option:`--with-hash-algorithm` option. + + .. versionadded:: 3.4 + Add :c:macro:`!Py_HASH_FNV` and :c:macro:`!Py_HASH_SIPHASH24`. + + .. versionadded:: 3.11 + Add :c:macro:`!Py_HASH_SIPHASH13`. + + +.. c:macro:: Py_HASH_CUTOFF + + Buffers of length in range ``[1, Py_HASH_CUTOFF)`` are hashed using DJBX33A + instead of the algorithm described by :c:macro:`Py_HASH_ALGORITHM`. + + - A :c:macro:`!Py_HASH_CUTOFF` of 0 disables the optimization. + - :c:macro:`!Py_HASH_CUTOFF` must be non-negative and less or equal than 7. + + 32-bit platforms should use a cutoff smaller than 64-bit platforms because + it is easier to create colliding strings. A cutoff of 7 on 64-bit platforms + and 5 on 32-bit platforms should provide a decent safety margin. + + This corresponds to the :data:`sys.hash_info.cutoff` constant. + + .. versionadded:: 3.4 + + .. c:macro:: PyHASH_MODULUS - The `Mersenne prime `_ ``P = 2**n -1``, used for numeric hash scheme. + The `Mersenne prime `_ ``P = 2**n -1``, + used for numeric hash scheme. + + This corresponds to the :data:`sys.hash_info.modulus` constant. .. versionadded:: 3.13 + .. c:macro:: PyHASH_BITS The exponent ``n`` of ``P`` in :c:macro:`PyHASH_MODULUS`. .. versionadded:: 3.13 + .. c:macro:: PyHASH_MULTIPLIER Prime multiplier used in string and various other hashes. .. versionadded:: 3.13 + .. c:macro:: PyHASH_INF The hash value returned for a positive infinity. + This corresponds to the :data:`sys.hash_info.inf` constant. + .. versionadded:: 3.13 + .. c:macro:: PyHASH_IMAG The multiplier used for the imaginary part of a complex number. + This corresponds to the :data:`sys.hash_info.imag` constant. + .. versionadded:: 3.13 + .. c:type:: PyHash_FuncDef Hash function definition used by :c:func:`PyHash_GetFuncDef`. - .. c::member:: Py_hash_t (*const hash)(const void *, Py_ssize_t) + .. c:member:: Py_hash_t (*const hash)(const void *, Py_ssize_t) Hash function. @@ -59,14 +115,20 @@ See also the :c:member:`PyTypeObject.tp_hash` member and :ref:`numeric-hash`. Hash function name (UTF-8 encoded string). + This corresponds to the :data:`sys.hash_info.algorithm` constant. + .. c:member:: const int hash_bits Internal size of the hash value in bits. + This corresponds to the :data:`sys.hash_info.hash_bits` constant. + .. c:member:: const int seed_bits Size of seed input in bits. + This corresponds to the :data:`sys.hash_info.seed_bits` constant. + .. versionadded:: 3.4 diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 1cab3ce3061ec90..369e95dd02251b4 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -327,6 +327,13 @@ Importing Modules initialization. +.. c:var:: struct _inittab *PyImport_Inittab + + The table of built-in modules used by Python initialization. Do not use this directly; + use :c:func:`PyImport_AppendInittab` and :c:func:`PyImport_ExtendInittab` + instead. + + .. c:function:: PyObject* PyImport_ImportModuleAttr(PyObject *mod_name, PyObject *attr_name) Import the module *mod_name* and get its attribute *attr_name*. diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst index ba56b03c6ac8e7c..eabe00f4004001f 100644 --- a/Doc/c-api/index.rst +++ b/Doc/c-api/index.rst @@ -1,7 +1,7 @@ .. _c-api-index: ################################## - Python/C API Reference Manual + Python/C API reference manual ################################## This manual documents the API used by C and C++ programmers who want to write @@ -17,10 +17,16 @@ document the API functions in detail. veryhigh.rst refcounting.rst exceptions.rst + extension-modules.rst utilities.rst abstract.rst concrete.rst - init.rst + interp-lifecycle.rst + threads.rst + synchronization.rst + tls.rst + subinterpreters.rst + profiling.rst init_config.rst memory.rst objimpl.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 52f64a61006b74d..e56c67f95348c78 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1,2519 +1,13 @@ -.. highlight:: c +:orphan: +Initialization, finalization, and threads +========================================= -.. _initialization: +This page has been split up into the following: -***************************************** -Initialization, Finalization, and Threads -***************************************** - -See :ref:`Python Initialization Configuration ` for details -on how to configure the interpreter prior to initialization. - -.. _pre-init-safe: - -Before Python Initialization -============================ - -In an application embedding Python, the :c:func:`Py_Initialize` function must -be called before using any other Python/C API functions; with the exception of -a few functions and the :ref:`global configuration variables -`. - -The following functions can be safely called before Python is initialized: - -* Functions that initialize the interpreter: - - * :c:func:`Py_Initialize` - * :c:func:`Py_InitializeEx` - * :c:func:`Py_InitializeFromConfig` - * :c:func:`Py_BytesMain` - * :c:func:`Py_Main` - * the runtime pre-initialization functions covered in :ref:`init-config` - -* Configuration functions: - - * :c:func:`PyImport_AppendInittab` - * :c:func:`PyImport_ExtendInittab` - * :c:func:`!PyInitFrozenExtensions` - * :c:func:`PyMem_SetAllocator` - * :c:func:`PyMem_SetupDebugHooks` - * :c:func:`PyObject_SetArenaAllocator` - * :c:func:`Py_SetProgramName` - * :c:func:`Py_SetPythonHome` - * :c:func:`PySys_ResetWarnOptions` - * the configuration functions covered in :ref:`init-config` - -* Informative functions: - - * :c:func:`Py_IsInitialized` - * :c:func:`PyMem_GetAllocator` - * :c:func:`PyObject_GetArenaAllocator` - * :c:func:`Py_GetBuildInfo` - * :c:func:`Py_GetCompiler` - * :c:func:`Py_GetCopyright` - * :c:func:`Py_GetPlatform` - * :c:func:`Py_GetVersion` - * :c:func:`Py_IsInitialized` - -* Utilities: - - * :c:func:`Py_DecodeLocale` - * the status reporting and utility functions covered in :ref:`init-config` - -* Memory allocators: - - * :c:func:`PyMem_RawMalloc` - * :c:func:`PyMem_RawRealloc` - * :c:func:`PyMem_RawCalloc` - * :c:func:`PyMem_RawFree` - -* Synchronization: - - * :c:func:`PyMutex_Lock` - * :c:func:`PyMutex_Unlock` - -.. note:: - - Despite their apparent similarity to some of the functions listed above, - the following functions **should not be called** before the interpreter has - been initialized: :c:func:`Py_EncodeLocale`, :c:func:`Py_GetPath`, - :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, - :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome`, - :c:func:`Py_GetProgramName`, :c:func:`PyEval_InitThreads`, and - :c:func:`Py_RunMain`. - - -.. _global-conf-vars: - -Global configuration variables -============================== - -Python has variables for the global configuration to control different features -and options. By default, these flags are controlled by :ref:`command line -options `. - -When a flag is set by an option, the value of the flag is the number of times -that the option was set. For example, ``-b`` sets :c:data:`Py_BytesWarningFlag` -to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. - -.. c:var:: int Py_BytesWarningFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.bytes_warning` should be used instead, see :ref:`Python - Initialization Configuration `. - - Issue a warning when comparing :class:`bytes` or :class:`bytearray` with - :class:`str` or :class:`bytes` with :class:`int`. Issue an error if greater - or equal to ``2``. - - Set by the :option:`-b` option. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_DebugFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.parser_debug` should be used instead, see :ref:`Python - Initialization Configuration `. - - Turn on parser debugging output (for expert only, depending on compilation - options). - - Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment - variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_DontWriteBytecodeFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.write_bytecode` should be used instead, see :ref:`Python - Initialization Configuration `. - - If set to non-zero, Python won't try to write ``.pyc`` files on the - import of source modules. - - Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` - environment variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_FrozenFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.pathconfig_warnings` should be used instead, see - :ref:`Python Initialization Configuration `. - - Suppress error messages when calculating the module search path in - :c:func:`Py_GetPath`. - - Private flag used by ``_freeze_module`` and ``frozenmain`` programs. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_HashRandomizationFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.hash_seed` and :c:member:`PyConfig.use_hash_seed` should - be used instead, see :ref:`Python Initialization Configuration - `. - - Set to ``1`` if the :envvar:`PYTHONHASHSEED` environment variable is set to - a non-empty string. - - If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment - variable to initialize the secret hash seed. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_IgnoreEnvironmentFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.use_environment` should be used instead, see - :ref:`Python Initialization Configuration `. - - Ignore all :envvar:`!PYTHON*` environment variables, e.g. - :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. - - Set by the :option:`-E` and :option:`-I` options. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_InspectFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.inspect` should be used instead, see - :ref:`Python Initialization Configuration `. - - When a script is passed as first argument or the :option:`-c` option is used, - enter interactive mode after executing the script or the command, even when - :data:`sys.stdin` does not appear to be a terminal. - - Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment - variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_InteractiveFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.interactive` should be used instead, see - :ref:`Python Initialization Configuration `. - - Set by the :option:`-i` option. - - .. deprecated:: 3.12 - -.. c:var:: int Py_IsolatedFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.isolated` should be used instead, see - :ref:`Python Initialization Configuration `. - - Run Python in isolated mode. In isolated mode :data:`sys.path` contains - neither the script's directory nor the user's site-packages directory. - - Set by the :option:`-I` option. - - .. versionadded:: 3.4 - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_LegacyWindowsFSEncodingFlag - - This API is kept for backward compatibility: setting - :c:member:`PyPreConfig.legacy_windows_fs_encoding` should be used instead, see - :ref:`Python Initialization Configuration `. - - If the flag is non-zero, use the ``mbcs`` encoding with ``replace`` error - handler, instead of the UTF-8 encoding with ``surrogatepass`` error handler, - for the :term:`filesystem encoding and error handler`. - - Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment - variable is set to a non-empty string. - - See :pep:`529` for more details. - - .. availability:: Windows. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_LegacyWindowsStdioFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.legacy_windows_stdio` should be used instead, see - :ref:`Python Initialization Configuration `. - - If the flag is non-zero, use :class:`io.FileIO` instead of - :class:`!io._WindowsConsoleIO` for :mod:`sys` standard streams. - - Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment - variable is set to a non-empty string. - - See :pep:`528` for more details. - - .. availability:: Windows. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_NoSiteFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.site_import` should be used instead, see - :ref:`Python Initialization Configuration `. - - Disable the import of the module :mod:`site` and the site-dependent - manipulations of :data:`sys.path` that it entails. Also disable these - manipulations if :mod:`site` is explicitly imported later (call - :func:`site.main` if you want them to be triggered). - - Set by the :option:`-S` option. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_NoUserSiteDirectory - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.user_site_directory` should be used instead, see - :ref:`Python Initialization Configuration `. - - Don't add the :data:`user site-packages directory ` to - :data:`sys.path`. - - Set by the :option:`-s` and :option:`-I` options, and the - :envvar:`PYTHONNOUSERSITE` environment variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_OptimizeFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.optimization_level` should be used instead, see - :ref:`Python Initialization Configuration `. - - Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment - variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_QuietFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.quiet` should be used instead, see :ref:`Python - Initialization Configuration `. - - Don't display the copyright and version messages even in interactive mode. - - Set by the :option:`-q` option. - - .. versionadded:: 3.2 - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_UnbufferedStdioFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.buffered_stdio` should be used instead, see :ref:`Python - Initialization Configuration `. - - Force the stdout and stderr streams to be unbuffered. - - Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` - environment variable. - - .. deprecated-removed:: 3.12 3.15 - -.. c:var:: int Py_VerboseFlag - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.verbose` should be used instead, see :ref:`Python - Initialization Configuration `. - - Print a message each time a module is initialized, showing the place - (filename or built-in module) from which it is loaded. If greater or equal - to ``2``, print a message for each file that is checked for when - searching for a module. Also provides information on module cleanup at exit. - - Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment - variable. - - .. deprecated-removed:: 3.12 3.15 - - -Initializing and finalizing the interpreter -=========================================== - - -.. c:function:: void Py_Initialize() - - .. index:: - single: PyEval_InitThreads() - single: modules (in module sys) - single: path (in module sys) - pair: module; builtins - pair: module; __main__ - pair: module; sys - triple: module; search; path - single: Py_FinalizeEx (C function) - - Initialize the Python interpreter. In an application embedding Python, - this should be called before using any other Python/C API functions; see - :ref:`Before Python Initialization ` for the few exceptions. - - This initializes the table of loaded modules (``sys.modules``), and creates - the fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. - It also initializes the module search path (``sys.path``). It does not set - ``sys.argv``; use the :ref:`Python Initialization Configuration ` - API for that. This is a no-op when called for a second time (without calling - :c:func:`Py_FinalizeEx` first). There is no return value; it is a fatal - error if the initialization fails. - - Use :c:func:`Py_InitializeFromConfig` to customize the - :ref:`Python Initialization Configuration `. - - .. note:: - On Windows, changes the console mode from ``O_TEXT`` to ``O_BINARY``, - which will also affect non-Python uses of the console using the C Runtime. - - -.. c:function:: void Py_InitializeEx(int initsigs) - - This function works like :c:func:`Py_Initialize` if *initsigs* is ``1``. If - *initsigs* is ``0``, it skips initialization registration of signal handlers, - which may be useful when CPython is embedded as part of a larger application. - - Use :c:func:`Py_InitializeFromConfig` to customize the - :ref:`Python Initialization Configuration `. - - -.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) - - Initialize Python from *config* configuration, as described in - :ref:`init-from-config`. - - See the :ref:`init-config` section for details on pre-initializing the - interpreter, populating the runtime configuration structure, and querying - the returned status structure. - - -.. c:function:: int Py_IsInitialized() - - Return true (nonzero) when the Python interpreter has been initialized, false - (zero) if not. After :c:func:`Py_FinalizeEx` is called, this returns false until - :c:func:`Py_Initialize` is called again. - - -.. c:function:: int Py_IsFinalizing() - - Return true (non-zero) if the main Python interpreter is - :term:`shutting down `. Return false (zero) otherwise. - - .. versionadded:: 3.13 - - -.. c:function:: int Py_FinalizeEx() - - Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of - Python/C API functions, and destroy all sub-interpreters (see - :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since - the last call to :c:func:`Py_Initialize`. This is a no-op when called for a second - time (without calling :c:func:`Py_Initialize` again first). - - Since this is the reverse of :c:func:`Py_Initialize`, it should be called - in the same thread with the same interpreter active. That means - the main thread and the main interpreter. - This should never be called while :c:func:`Py_RunMain` is running. - - Normally the return value is ``0``. - If there were errors during finalization (flushing buffered data), - ``-1`` is returned. - - Note that Python will do a best effort at freeing all memory allocated by the Python - interpreter. Therefore, any C-Extension should make sure to correctly clean up all - of the preveiously allocated PyObjects before using them in subsequent calls to - :c:func:`Py_Initialize`. Otherwise it could introduce vulnerabilities and incorrect - behavior. - - This function is provided for a number of reasons. An embedding application - might want to restart Python without having to restart the application itself. - An application that has loaded the Python interpreter from a dynamically - loadable library (or DLL) might want to free all memory allocated by Python - before unloading the DLL. During a hunt for memory leaks in an application a - developer might want to free all memory allocated by Python before exiting from - the application. - - **Bugs and caveats:** The destruction of modules and objects in modules is done - in random order; this may cause destructors (:meth:`~object.__del__` methods) to fail - when they depend on other objects (even functions) or modules. Dynamically - loaded extension modules loaded by Python are not unloaded. Small amounts of - memory allocated by the Python interpreter may not be freed (if you find a leak, - please report it). Memory tied up in circular references between objects is not - freed. Interned strings will all be deallocated regardless of their reference count. - Some memory allocated by extension modules may not be freed. Some extensions may not - work properly if their initialization routine is called more than once; this can - happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx` - more than once. :c:func:`Py_FinalizeEx` must not be called recursively from - within itself. Therefore, it must not be called by any code that may be run - as part of the interpreter shutdown process, such as :py:mod:`atexit` - handlers, object finalizers, or any code that may be run while flushing the - stdout and stderr files. - - .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx - - .. versionadded:: 3.6 - - -.. c:function:: void Py_Finalize() - - This is a backwards-compatible version of :c:func:`Py_FinalizeEx` that - disregards the return value. - - -.. c:function:: int Py_BytesMain(int argc, char **argv) - - Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings, - allowing the calling application to delegate the text decoding step to - the CPython runtime. - - .. versionadded:: 3.8 - - -.. c:function:: int Py_Main(int argc, wchar_t **argv) - - The main program for the standard interpreter, encapsulating a full - initialization/finalization cycle, as well as additional - behaviour to implement reading configurations settings from the environment - and command line, and then executing ``__main__`` in accordance with - :ref:`using-on-cmdline`. - - This is made available for programs which wish to support the full CPython - command line interface, rather than just embedding a Python runtime in a - larger application. - - The *argc* and *argv* parameters are similar to those which are passed to a - C program's :c:func:`main` function, except that the *argv* entries are first - converted to ``wchar_t`` using :c:func:`Py_DecodeLocale`. It is also - important to note that the argument list entries may be modified to point to - strings other than those passed in (however, the contents of the strings - pointed to by the argument list are not modified). - - The return value will be ``0`` if the interpreter exits normally (i.e., - without an exception), ``1`` if the interpreter exits due to an exception, - or ``2`` if the argument list does not represent a valid Python command - line. - - Note that if an otherwise unhandled :exc:`SystemExit` is raised, this - function will not return ``1``, but exit the process, as long as - ``Py_InspectFlag`` is not set. If ``Py_InspectFlag`` is set, execution will - drop into the interactive Python prompt, at which point a second otherwise - unhandled :exc:`SystemExit` will still exit the process, while any other - means of exiting will set the return value as described above. - - In terms of the CPython runtime configuration APIs documented in the - :ref:`runtime configuration ` section (and without accounting - for error handling), ``Py_Main`` is approximately equivalent to:: - - PyConfig config; - PyConfig_InitPythonConfig(&config); - PyConfig_SetArgv(&config, argc, argv); - Py_InitializeFromConfig(&config); - PyConfig_Clear(&config); - - Py_RunMain(); - - In normal usage, an embedding application will call this function - *instead* of calling :c:func:`Py_Initialize`, :c:func:`Py_InitializeEx` or - :c:func:`Py_InitializeFromConfig` directly, and all settings will be applied - as described elsewhere in this documentation. If this function is instead - called *after* a preceding runtime initialization API call, then exactly - which environmental and command line configuration settings will be updated - is version dependent (as it depends on which settings correctly support - being modified after they have already been set once when the runtime was - first initialized). - - -.. c:function:: int Py_RunMain(void) - - Executes the main module in a fully configured CPython runtime. - - Executes the command (:c:member:`PyConfig.run_command`), the script - (:c:member:`PyConfig.run_filename`) or the module - (:c:member:`PyConfig.run_module`) specified on the command line or in the - configuration. If none of these values are set, runs the interactive Python - prompt (REPL) using the ``__main__`` module's global namespace. - - If :c:member:`PyConfig.inspect` is not set (the default), the return value - will be ``0`` if the interpreter exits normally (that is, without raising - an exception), or ``1`` if the interpreter exits due to an exception. If an - otherwise unhandled :exc:`SystemExit` is raised, the function will immediately - exit the process instead of returning ``1``. - - If :c:member:`PyConfig.inspect` is set (such as when the :option:`-i` option - is used), rather than returning when the interpreter exits, execution will - instead resume in an interactive Python prompt (REPL) using the ``__main__`` - module's global namespace. If the interpreter exited with an exception, it - is immediately raised in the REPL session. The function return value is - then determined by the way the *REPL session* terminates: returning ``0`` - if the session terminates without raising an unhandled exception, exiting - immediately for an unhandled :exc:`SystemExit`, and returning ``1`` for - any other unhandled exception. - - This function always finalizes the Python interpreter regardless of whether - it returns a value or immediately exits the process due to an unhandled - :exc:`SystemExit` exception. - - See :ref:`Python Configuration ` for an example of a - customized Python that always runs in isolated mode using - :c:func:`Py_RunMain`. - -.. c:function:: int PyUnstable_AtExit(PyInterpreterState *interp, void (*func)(void *), void *data) - - Register an :mod:`atexit` callback for the target interpreter *interp*. - This is similar to :c:func:`Py_AtExit`, but takes an explicit interpreter and - data pointer for the callback. - - There must be an :term:`attached thread state` for *interp*. - - .. versionadded:: 3.13 - -Process-wide parameters -======================= - - -.. c:function:: void Py_SetProgramName(const wchar_t *name) - - .. index:: - single: Py_Initialize() - single: main() - single: Py_GetPath() - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.program_name` should be used instead, see :ref:`Python - Initialization Configuration `. - - This function should be called before :c:func:`Py_Initialize` is called for - the first time, if it is called at all. It tells the interpreter the value - of the ``argv[0]`` argument to the :c:func:`main` function of the program - (converted to wide characters). - This is used by :c:func:`Py_GetPath` and some other functions below to find - the Python run-time libraries relative to the interpreter executable. The - default value is ``'python'``. The argument should point to a - zero-terminated wide character string in static storage whose contents will not - change for the duration of the program's execution. No code in the Python - interpreter will change the contents of this storage. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_t*` string. - - .. deprecated-removed:: 3.11 3.15 - - -.. c:function:: wchar_t* Py_GetProgramName() - - Return the program name set with :c:member:`PyConfig.program_name`, or the default. - The returned string points into static storage; the caller should not modify its - value. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyConfig_Get("executable") ` - (:data:`sys.executable`) instead. - - -.. c:function:: wchar_t* Py_GetPrefix() - - Return the *prefix* for installed platform-independent files. This is derived - through a number of complicated rules from the program name set with - :c:member:`PyConfig.program_name` and some environment variables; for example, if the - program name is ``'/usr/local/bin/python'``, the prefix is ``'/usr/local'``. The - returned string points into static storage; the caller should not modify its - value. This corresponds to the :makevar:`prefix` variable in the top-level - :file:`Makefile` and the :option:`--prefix` argument to the :program:`configure` - script at build time. The value is available to Python code as ``sys.base_prefix``. - It is only useful on Unix. See also the next function. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyConfig_Get("base_prefix") ` - (:data:`sys.base_prefix`) instead. Use :c:func:`PyConfig_Get("prefix") - ` (:data:`sys.prefix`) if :ref:`virtual environments - ` need to be handled. - - -.. c:function:: wchar_t* Py_GetExecPrefix() - - Return the *exec-prefix* for installed platform-*dependent* files. This is - derived through a number of complicated rules from the program name set with - :c:member:`PyConfig.program_name` and some environment variables; for example, if the - program name is ``'/usr/local/bin/python'``, the exec-prefix is - ``'/usr/local'``. The returned string points into static storage; the caller - should not modify its value. This corresponds to the :makevar:`exec_prefix` - variable in the top-level :file:`Makefile` and the ``--exec-prefix`` - argument to the :program:`configure` script at build time. The value is - available to Python code as ``sys.base_exec_prefix``. It is only useful on - Unix. - - Background: The exec-prefix differs from the prefix when platform dependent - files (such as executables and shared libraries) are installed in a different - directory tree. In a typical installation, platform dependent files may be - installed in the :file:`/usr/local/plat` subtree while platform independent may - be installed in :file:`/usr/local`. - - Generally speaking, a platform is a combination of hardware and software - families, e.g. Sparc machines running the Solaris 2.x operating system are - considered the same platform, but Intel machines running Solaris 2.x are another - platform, and Intel machines running Linux are yet another platform. Different - major revisions of the same operating system generally also form different - platforms. Non-Unix operating systems are a different story; the installation - strategies on those systems are so different that the prefix and exec-prefix are - meaningless, and set to the empty string. Note that compiled Python bytecode - files are platform independent (but not independent from the Python version by - which they were compiled!). - - System administrators will know how to configure the :program:`mount` or - :program:`automount` programs to share :file:`/usr/local` between platforms - while having :file:`/usr/local/plat` be a different filesystem for each - platform. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyConfig_Get("base_exec_prefix") ` - (:data:`sys.base_exec_prefix`) instead. Use - :c:func:`PyConfig_Get("exec_prefix") ` - (:data:`sys.exec_prefix`) if :ref:`virtual environments ` need - to be handled. - -.. c:function:: wchar_t* Py_GetProgramFullPath() - - .. index:: - single: executable (in module sys) - - Return the full program name of the Python executable; this is computed as a - side-effect of deriving the default module search path from the program name - (set by :c:member:`PyConfig.program_name`). The returned string points into - static storage; the caller should not modify its value. The value is available - to Python code as ``sys.executable``. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyConfig_Get("executable") ` - (:data:`sys.executable`) instead. - - -.. c:function:: wchar_t* Py_GetPath() - - .. index:: - triple: module; search; path - single: path (in module sys) - - Return the default module search path; this is computed from the program name - (set by :c:member:`PyConfig.program_name`) and some environment variables. - The returned string consists of a series of directory names separated by a - platform dependent delimiter character. The delimiter character is ``':'`` - on Unix and macOS, ``';'`` on Windows. The returned string points into - static storage; the caller should not modify its value. The list - :data:`sys.path` is initialized with this value on interpreter startup; it - can be (and usually is) modified later to change the search path for loading - modules. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. XXX should give the exact rules - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyConfig_Get("module_search_paths") ` - (:data:`sys.path`) instead. - -.. c:function:: const char* Py_GetVersion() - - Return the version of this Python interpreter. This is a string that looks - something like :: - - "3.0a5+ (py3k:63103M, May 12 2008, 00:53:55) \n[GCC 4.2.3]" - - .. index:: single: version (in module sys) - - The first word (up to the first space character) is the current Python version; - the first characters are the major and minor version separated by a - period. The returned string points into static storage; the caller should not - modify its value. The value is available to Python code as :data:`sys.version`. - - See also the :c:var:`Py_Version` constant. - - -.. c:function:: const char* Py_GetPlatform() - - .. index:: single: platform (in module sys) - - Return the platform identifier for the current platform. On Unix, this is - formed from the "official" name of the operating system, converted to lower - case, followed by the major revision number; e.g., for Solaris 2.x, which is - also known as SunOS 5.x, the value is ``'sunos5'``. On macOS, it is - ``'darwin'``. On Windows, it is ``'win'``. The returned string points into - static storage; the caller should not modify its value. The value is available - to Python code as ``sys.platform``. - - -.. c:function:: const char* Py_GetCopyright() - - Return the official copyright string for the current Python version, for example - - ``'Copyright 1991-1995 Stichting Mathematisch Centrum, Amsterdam'`` - - .. index:: single: copyright (in module sys) - - The returned string points into static storage; the caller should not modify its - value. The value is available to Python code as ``sys.copyright``. - - -.. c:function:: const char* Py_GetCompiler() - - Return an indication of the compiler used to build the current Python version, - in square brackets, for example:: - - "[GCC 2.7.2.2]" - - .. index:: single: version (in module sys) - - The returned string points into static storage; the caller should not modify its - value. The value is available to Python code as part of the variable - ``sys.version``. - - -.. c:function:: const char* Py_GetBuildInfo() - - Return information about the sequence number and build date and time of the - current Python interpreter instance, for example :: - - "#67, Aug 1 1997, 22:34:28" - - .. index:: single: version (in module sys) - - The returned string points into static storage; the caller should not modify its - value. The value is available to Python code as part of the variable - ``sys.version``. - - -.. c:function:: void PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) - - .. index:: - single: main() - single: Py_FatalError() - single: argv (in module sys) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.argv`, :c:member:`PyConfig.parse_argv` and - :c:member:`PyConfig.safe_path` should be used instead, see :ref:`Python - Initialization Configuration `. - - Set :data:`sys.argv` based on *argc* and *argv*. These parameters are - similar to those passed to the program's :c:func:`main` function with the - difference that the first entry should refer to the script file to be - executed rather than the executable hosting the Python interpreter. If there - isn't a script that will be run, the first entry in *argv* can be an empty - string. If this function fails to initialize :data:`sys.argv`, a fatal - condition is signalled using :c:func:`Py_FatalError`. - - If *updatepath* is zero, this is all the function does. If *updatepath* - is non-zero, the function also modifies :data:`sys.path` according to the - following algorithm: - - - If the name of an existing script is passed in ``argv[0]``, the absolute - path of the directory where the script is located is prepended to - :data:`sys.path`. - - Otherwise (that is, if *argc* is ``0`` or ``argv[0]`` doesn't point - to an existing file name), an empty string is prepended to - :data:`sys.path`, which is the same as prepending the current working - directory (``"."``). - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_t*` string. - - See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` - members of the :ref:`Python Initialization Configuration `. - - .. note:: - It is recommended that applications embedding the Python interpreter - for purposes other than executing a single script pass ``0`` as *updatepath*, - and update :data:`sys.path` themselves if desired. - See :cve:`2008-5983`. - - On versions before 3.1.3, you can achieve the same effect by manually - popping the first :data:`sys.path` element after having called - :c:func:`PySys_SetArgv`, for example using:: - - PyRun_SimpleString("import sys; sys.path.pop(0)\n"); - - .. versionadded:: 3.1.3 - - .. XXX impl. doesn't seem consistent in allowing ``0``/``NULL`` for the params; - check w/ Guido. - - .. deprecated-removed:: 3.11 3.15 - - -.. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.argv` and :c:member:`PyConfig.parse_argv` should be used - instead, see :ref:`Python Initialization Configuration `. - - This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set - to ``1`` unless the :program:`python` interpreter was started with the - :option:`-I`. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_t*` string. - - See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` - members of the :ref:`Python Initialization Configuration `. - - .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. - - .. deprecated-removed:: 3.11 3.15 - - -.. c:function:: void Py_SetPythonHome(const wchar_t *home) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.home` should be used instead, see :ref:`Python - Initialization Configuration `. - - Set the default "home" directory, that is, the location of the standard - Python libraries. See :envvar:`PYTHONHOME` for the meaning of the - argument string. - - The argument should point to a zero-terminated character string in static - storage whose contents will not change for the duration of the program's - execution. No code in the Python interpreter will change the contents of - this storage. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_t*` string. - - .. deprecated-removed:: 3.11 3.15 - - -.. c:function:: wchar_t* Py_GetPythonHome() - - Return the default "home", that is, the value set by - :c:member:`PyConfig.home`, or the value of the :envvar:`PYTHONHOME` - environment variable if it is set. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - .. deprecated-removed:: 3.13 3.15 - Use :c:func:`PyConfig_Get("home") ` or the - :envvar:`PYTHONHOME` environment variable instead. - - -.. _threads: - -Thread State and the Global Interpreter Lock -============================================ - -.. index:: - single: global interpreter lock - single: interpreter lock - single: lock, interpreter - -Unless on a :term:`free-threaded ` build of :term:`CPython`, -the Python interpreter is not fully thread-safe. In order to support -multi-threaded Python programs, there's a global lock, called the :term:`global -interpreter lock` or :term:`GIL`, that must be held by the current thread before -it can safely access Python objects. Without the lock, even the simplest -operations could cause problems in a multi-threaded program: for example, when -two threads simultaneously increment the reference count of the same object, the -reference count could end up being incremented only once instead of twice. - -.. index:: single: setswitchinterval (in module sys) - -Therefore, the rule exists that only the thread that has acquired the -:term:`GIL` may operate on Python objects or call Python/C API functions. -In order to emulate concurrency of execution, the interpreter regularly -tries to switch threads (see :func:`sys.setswitchinterval`). The lock is also -released around potentially blocking I/O operations like reading or writing -a file, so that other Python threads can run in the meantime. - -.. index:: - single: PyThreadState (C type) - -The Python interpreter keeps some thread-specific bookkeeping information -inside a data structure called :c:type:`PyThreadState`, known as a :term:`thread state`. -Each OS thread has a thread-local pointer to a :c:type:`PyThreadState`; a thread state -referenced by this pointer is considered to be :term:`attached `. - -A thread can only have one :term:`attached thread state` at a time. An attached -thread state is typically analogous with holding the :term:`GIL`, except on -:term:`free-threaded ` builds. On builds with the :term:`GIL` enabled, -:term:`attaching ` a thread state will block until the :term:`GIL` -can be acquired. However, even on builds with the :term:`GIL` disabled, it is still required -to have an attached thread state to call most of the C API. - -In general, there will always be an :term:`attached thread state` when using Python's C API. -Only in some specific cases (such as in a :c:macro:`Py_BEGIN_ALLOW_THREADS` block) will the -thread not have an attached thread state. If uncertain, check if :c:func:`PyThreadState_GetUnchecked` returns -``NULL``. - -Detaching the thread state from extension code ----------------------------------------------- - -Most extension code manipulating the :term:`thread state` has the following simple -structure:: - - Save the thread state in a local variable. - ... Do some blocking I/O operation ... - Restore the thread state from the local variable. - -This is so common that a pair of macros exists to simplify it:: - - Py_BEGIN_ALLOW_THREADS - ... Do some blocking I/O operation ... - Py_END_ALLOW_THREADS - -.. index:: - single: Py_BEGIN_ALLOW_THREADS (C macro) - single: Py_END_ALLOW_THREADS (C macro) - -The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a -hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the -block. - -The block above expands to the following code:: - - PyThreadState *_save; - - _save = PyEval_SaveThread(); - ... Do some blocking I/O operation ... - PyEval_RestoreThread(_save); - -.. index:: - single: PyEval_RestoreThread (C function) - single: PyEval_SaveThread (C function) - -Here is how these functions work: - -The :term:`attached thread state` holds the :term:`GIL` for the entire interpreter. When detaching -the :term:`attached thread state`, the :term:`GIL` is released, allowing other threads to attach -a thread state to their own thread, thus getting the :term:`GIL` and can start executing. -The pointer to the prior :term:`attached thread state` is stored as a local variable. -Upon reaching :c:macro:`Py_END_ALLOW_THREADS`, the thread state that was -previously :term:`attached ` is passed to :c:func:`PyEval_RestoreThread`. -This function will block until another releases its :term:`thread state `, -thus allowing the old :term:`thread state ` to get re-attached and the -C API can be called again. - -For :term:`free-threaded ` builds, the :term:`GIL` is normally -out of the question, but detaching the :term:`thread state ` is still required -for blocking I/O and long operations. The difference is that threads don't have to wait for the :term:`GIL` -to be released to attach their thread state, allowing true multi-core parallelism. - -.. note:: - Calling system I/O functions is the most common use case for detaching - the :term:`thread state `, but it can also be useful before calling - long-running computations which don't need access to Python objects, such - as compression or cryptographic functions operating over memory buffers. - For example, the standard :mod:`zlib` and :mod:`hashlib` modules detach the - :term:`thread state ` when compressing or hashing data. - - -.. _gilstate: - -Non-Python created threads --------------------------- - -When threads are created using the dedicated Python APIs (such as the -:mod:`threading` module), a thread state is automatically associated to them -and the code showed above is therefore correct. However, when threads are -created from C (for example by a third-party library with its own thread -management), they don't hold the :term:`GIL`, because they don't have an -:term:`attached thread state`. - -If you need to call Python code from these threads (often this will be part -of a callback API provided by the aforementioned third-party library), -you must first register these threads with the interpreter by -creating an :term:`attached thread state` before you can start using the Python/C -API. When you are done, you should detach the :term:`thread state `, and -finally free it. - -The :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` functions do -all of the above automatically. The typical idiom for calling into Python -from a C thread is:: - - PyGILState_STATE gstate; - gstate = PyGILState_Ensure(); - - /* Perform Python actions here. */ - result = CallSomeFunction(); - /* evaluate result or handle exception */ - - /* Release the thread. No Python API allowed beyond this point. */ - PyGILState_Release(gstate); - -Note that the ``PyGILState_*`` functions assume there is only one global -interpreter (created automatically by :c:func:`Py_Initialize`). Python -supports the creation of additional interpreters (using -:c:func:`Py_NewInterpreter`), but mixing multiple interpreters and the -``PyGILState_*`` API is unsupported. - - -.. _fork-and-threads: - -Cautions about fork() ---------------------- - -Another important thing to note about threads is their behaviour in the face -of the C :c:func:`fork` call. On most systems with :c:func:`fork`, after a -process forks only the thread that issued the fork will exist. This has a -concrete impact both on how locks must be handled and on all stored state -in CPython's runtime. - -The fact that only the "current" thread remains -means any locks held by other threads will never be released. Python solves -this for :func:`os.fork` by acquiring the locks it uses internally before -the fork, and releasing them afterwards. In addition, it resets any -:ref:`lock-objects` in the child. When extending or embedding Python, there -is no way to inform Python of additional (non-Python) locks that need to be -acquired before or reset after a fork. OS facilities such as -:c:func:`!pthread_atfork` would need to be used to accomplish the same thing. -Additionally, when extending or embedding Python, calling :c:func:`fork` -directly rather than through :func:`os.fork` (and returning to or calling -into Python) may result in a deadlock by one of Python's internal locks -being held by a thread that is defunct after the fork. -:c:func:`PyOS_AfterFork_Child` tries to reset the necessary locks, but is not -always able to. - -The fact that all other threads go away also means that CPython's -runtime state there must be cleaned up properly, which :func:`os.fork` -does. This means finalizing all other :c:type:`PyThreadState` objects -belonging to the current interpreter and all other -:c:type:`PyInterpreterState` objects. Due to this and the special -nature of the :ref:`"main" interpreter `, -:c:func:`fork` should only be called in that interpreter's "main" -thread, where the CPython global runtime was originally initialized. -The only exception is if :c:func:`exec` will be called immediately -after. - -.. _cautions-regarding-runtime-finalization: - -Cautions regarding runtime finalization ---------------------------------------- - -In the late stage of :term:`interpreter shutdown`, after attempting to wait for -non-daemon threads to exit (though this can be interrupted by -:class:`KeyboardInterrupt`) and running the :mod:`atexit` functions, the runtime -is marked as *finalizing*: :c:func:`Py_IsFinalizing` and -:func:`sys.is_finalizing` return true. At this point, only the *finalization -thread* that initiated finalization (typically the main thread) is allowed to -acquire the :term:`GIL`. - -If any thread, other than the finalization thread, attempts to attach a :term:`thread state` -during finalization, either explicitly or -implicitly, the thread enters **a permanently blocked state** -where it remains until the program exits. In most cases this is harmless, but this can result -in deadlock if a later stage of finalization attempts to acquire a lock owned by the -blocked thread, or otherwise waits on the blocked thread. - -Gross? Yes. This prevents random crashes and/or unexpectedly skipped C++ -finalizations further up the call stack when such threads were forcibly exited -here in CPython 3.13 and earlier. The CPython runtime :term:`thread state` C APIs -have never had any error reporting or handling expectations at :term:`thread state` -attachment time that would've allowed for graceful exit from this situation. Changing that -would require new stable C APIs and rewriting the majority of C code in the -CPython ecosystem to use those with error handling. - - -High-level API --------------- - -These are the most commonly used types and functions when writing C extension -code, or when embedding the Python interpreter: - -.. c:type:: PyInterpreterState - - This data structure represents the state shared by a number of cooperating - threads. Threads belonging to the same interpreter share their module - administration and a few other internal items. There are no public members in - this structure. - - Threads belonging to different interpreters initially share nothing, except - process state like available memory, open file descriptors and such. The global - interpreter lock is also shared by all threads, regardless of to which - interpreter they belong. - - -.. c:type:: PyThreadState - - This data structure represents the state of a single thread. The only public - data member is: - - .. c:member:: PyInterpreterState *interp - - This thread's interpreter state. - - -.. c:function:: void PyEval_InitThreads() - - .. index:: - single: PyEval_AcquireThread() - single: PyEval_ReleaseThread() - single: PyEval_SaveThread() - single: PyEval_RestoreThread() - - Deprecated function which does nothing. - - In Python 3.6 and older, this function created the GIL if it didn't exist. - - .. versionchanged:: 3.9 - The function now does nothing. - - .. versionchanged:: 3.7 - This function is now called by :c:func:`Py_Initialize()`, so you don't - have to call it yourself anymore. - - .. versionchanged:: 3.2 - This function cannot be called before :c:func:`Py_Initialize()` anymore. - - .. deprecated:: 3.9 - - .. index:: pair: module; _thread - - -.. c:function:: PyThreadState* PyEval_SaveThread() - - Detach the :term:`attached thread state` and return it. - The thread will have no :term:`thread state` upon returning. - - -.. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) - - Set the :term:`attached thread state` to *tstate*. - The passed :term:`thread state` **should not** be :term:`attached `, - otherwise deadlock ensues. *tstate* will be attached upon returning. - - .. note:: - Calling this function from a thread when the runtime is finalizing will - hang the thread until the program exits, even if the thread was not - created by Python. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - -.. c:function:: PyThreadState* PyThreadState_Get() - - Return the :term:`attached thread state`. If the thread has no attached - thread state, (such as when inside of :c:macro:`Py_BEGIN_ALLOW_THREADS` - block), then this issues a fatal error (so that the caller needn't check - for ``NULL``). - - See also :c:func:`PyThreadState_GetUnchecked`. - -.. c:function:: PyThreadState* PyThreadState_GetUnchecked() - - Similar to :c:func:`PyThreadState_Get`, but don't kill the process with a - fatal error if it is NULL. The caller is responsible to check if the result - is NULL. - - .. versionadded:: 3.13 - In Python 3.5 to 3.12, the function was private and known as - ``_PyThreadState_UncheckedGet()``. - - -.. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) - - Set the :term:`attached thread state` to *tstate*, and return the - :term:`thread state` that was attached prior to calling. - - This function is safe to call without an :term:`attached thread state`; it - will simply return ``NULL`` indicating that there was no prior thread state. - - .. seealso: - :c:func:`PyEval_ReleaseThread` - - -The following functions use thread-local storage, and are not compatible -with sub-interpreters: - -.. c:function:: PyGILState_STATE PyGILState_Ensure() - - Ensure that the current thread is ready to call the Python C API regardless - of the current state of Python, or of the :term:`attached thread state`. This may - be called as many times as desired by a thread as long as each call is - matched with a call to :c:func:`PyGILState_Release`. In general, other - thread-related APIs may be used between :c:func:`PyGILState_Ensure` and - :c:func:`PyGILState_Release` calls as long as the thread state is restored to - its previous state before the Release(). For example, normal usage of the - :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is - acceptable. - - The return value is an opaque "handle" to the :term:`attached thread state` when - :c:func:`PyGILState_Ensure` was called, and must be passed to - :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even - though recursive calls are allowed, these handles *cannot* be shared - each - unique call to :c:func:`PyGILState_Ensure` must save the handle for its call - to :c:func:`PyGILState_Release`. - - When the function returns, there will be an :term:`attached thread state` - and the thread will be able to call arbitrary Python code. Failure is a fatal error. - - .. note:: - Calling this function from a thread when the runtime is finalizing will - hang the thread until the program exits, even if the thread was not - created by Python. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - -.. c:function:: void PyGILState_Release(PyGILState_STATE) - - Release any resources previously acquired. After this call, Python's state will - be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call - (but generally this state will be unknown to the caller, hence the use of the - GILState API). - - Every call to :c:func:`PyGILState_Ensure` must be matched by a call to - :c:func:`PyGILState_Release` on the same thread. - - -.. c:function:: PyThreadState* PyGILState_GetThisThreadState() - - Get the :term:`attached thread state` for this thread. May return ``NULL`` if no - GILState API has been used on the current thread. Note that the main thread - always has such a thread-state, even if no auto-thread-state call has been - made on the main thread. This is mainly a helper/diagnostic function. - - .. seealso: :c:func:`PyThreadState_Get`` - - -.. c:function:: int PyGILState_Check() - - Return ``1`` if the current thread is holding the :term:`GIL` and ``0`` otherwise. - This function can be called from any thread at any time. - Only if it has had its Python thread state initialized and currently is - holding the :term:`GIL` will it return ``1``. - This is mainly a helper/diagnostic function. It can be useful - for example in callback contexts or memory allocation functions when - knowing that the :term:`GIL` is locked can allow the caller to perform sensitive - actions or otherwise behave differently. - - .. versionadded:: 3.4 - - -The following macros are normally used without a trailing semicolon; look for -example usage in the Python source distribution. - - -.. c:macro:: Py_BEGIN_ALLOW_THREADS - - This macro expands to ``{ PyThreadState *_save; _save = PyEval_SaveThread();``. - Note that it contains an opening brace; it must be matched with a following - :c:macro:`Py_END_ALLOW_THREADS` macro. See above for further discussion of this - macro. - - -.. c:macro:: Py_END_ALLOW_THREADS - - This macro expands to ``PyEval_RestoreThread(_save); }``. Note that it contains - a closing brace; it must be matched with an earlier - :c:macro:`Py_BEGIN_ALLOW_THREADS` macro. See above for further discussion of - this macro. - - -.. c:macro:: Py_BLOCK_THREADS - - This macro expands to ``PyEval_RestoreThread(_save);``: it is equivalent to - :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. - - -.. c:macro:: Py_UNBLOCK_THREADS - - This macro expands to ``_save = PyEval_SaveThread();``: it is equivalent to - :c:macro:`Py_BEGIN_ALLOW_THREADS` without the opening brace and variable - declaration. - - -Low-level API -------------- - -All of the following functions must be called after :c:func:`Py_Initialize`. - -.. versionchanged:: 3.7 - :c:func:`Py_Initialize()` now initializes the :term:`GIL` - and sets an :term:`attached thread state`. - - -.. c:function:: PyInterpreterState* PyInterpreterState_New() - - Create a new interpreter state object. An :term:`attached thread state` is not needed, - but may optionally exist if it is necessary to serialize calls to this - function. - - .. audit-event:: cpython.PyInterpreterState_New "" c.PyInterpreterState_New - - -.. c:function:: void PyInterpreterState_Clear(PyInterpreterState *interp) - - Reset all information in an interpreter state object. There must be - an :term:`attached thread state` for the the interpreter. - - .. audit-event:: cpython.PyInterpreterState_Clear "" c.PyInterpreterState_Clear - - -.. c:function:: void PyInterpreterState_Delete(PyInterpreterState *interp) - - Destroy an interpreter state object. There **should not** be an - :term:`attached thread state` for the target interpreter. The interpreter - state must have been reset with a previous call to :c:func:`PyInterpreterState_Clear`. - - -.. c:function:: PyThreadState* PyThreadState_New(PyInterpreterState *interp) - - Create a new thread state object belonging to the given interpreter object. - An :term:`attached thread state` is not needed. - -.. c:function:: void PyThreadState_Clear(PyThreadState *tstate) - - Reset all information in a :term:`thread state` object. *tstate* - must be :term:`attached ` - - .. versionchanged:: 3.9 - This function now calls the :c:member:`PyThreadState.on_delete` callback. - Previously, that happened in :c:func:`PyThreadState_Delete`. - - .. versionchanged:: 3.13 - The :c:member:`PyThreadState.on_delete` callback was removed. - - -.. c:function:: void PyThreadState_Delete(PyThreadState *tstate) - - Destroy a :term:`thread state` object. *tstate* should not - be :term:`attached ` to any thread. - *tstate* must have been reset with a previous call to - :c:func:`PyThreadState_Clear`. - - -.. c:function:: void PyThreadState_DeleteCurrent(void) - - Detach the :term:`attached thread state` (which must have been reset - with a previous call to :c:func:`PyThreadState_Clear`) and then destroy it. - - No :term:`thread state` will be :term:`attached ` upon - returning. - -.. c:function:: PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) - - Get the current frame of the Python thread state *tstate*. - - Return a :term:`strong reference`. Return ``NULL`` if no frame is currently - executing. - - See also :c:func:`PyEval_GetFrame`. - - *tstate* must not be ``NULL``, and must be :term:`attached `. - - .. versionadded:: 3.9 - - -.. c:function:: uint64_t PyThreadState_GetID(PyThreadState *tstate) - - Get the unique :term:`thread state` identifier of the Python thread state *tstate*. - - *tstate* must not be ``NULL``, and must be :term:`attached `. - - .. versionadded:: 3.9 - - -.. c:function:: PyInterpreterState* PyThreadState_GetInterpreter(PyThreadState *tstate) - - Get the interpreter of the Python thread state *tstate*. - - *tstate* must not be ``NULL``, and must be :term:`attached `. - - .. versionadded:: 3.9 - - -.. c:function:: void PyThreadState_EnterTracing(PyThreadState *tstate) - - Suspend tracing and profiling in the Python thread state *tstate*. - - Resume them using the :c:func:`PyThreadState_LeaveTracing` function. - - .. versionadded:: 3.11 - - -.. c:function:: void PyThreadState_LeaveTracing(PyThreadState *tstate) - - Resume tracing and profiling in the Python thread state *tstate* suspended - by the :c:func:`PyThreadState_EnterTracing` function. - - See also :c:func:`PyEval_SetTrace` and :c:func:`PyEval_SetProfile` - functions. - - .. versionadded:: 3.11 - - -.. c:function:: PyInterpreterState* PyInterpreterState_Get(void) - - Get the current interpreter. - - Issue a fatal error if there no :term:`attached thread state`. - It cannot return NULL. - - .. versionadded:: 3.9 - - -.. c:function:: int64_t PyInterpreterState_GetID(PyInterpreterState *interp) - - Return the interpreter's unique ID. If there was any error in doing - so then ``-1`` is returned and an error is set. - - The caller must have an :term:`attached thread state`. - - .. versionadded:: 3.7 - - -.. c:function:: PyObject* PyInterpreterState_GetDict(PyInterpreterState *interp) - - Return a dictionary in which interpreter-specific data may be stored. - If this function returns ``NULL`` then no exception has been raised and - the caller should assume no interpreter-specific dict is available. - - This is not a replacement for :c:func:`PyModule_GetState()`, which - extensions should use to store interpreter-specific state information. - - .. versionadded:: 3.8 - - -.. c:type:: PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) - - Type of a frame evaluation function. - - The *throwflag* parameter is used by the ``throw()`` method of generators: - if non-zero, handle the current exception. - - .. versionchanged:: 3.9 - The function now takes a *tstate* parameter. - - .. versionchanged:: 3.11 - The *frame* parameter changed from ``PyFrameObject*`` to ``_PyInterpreterFrame*``. - -.. c:function:: _PyFrameEvalFunction _PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp) - - Get the frame evaluation function. - - See the :pep:`523` "Adding a frame evaluation API to CPython". - - .. versionadded:: 3.9 - -.. c:function:: void _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame) - - Set the frame evaluation function. - - See the :pep:`523` "Adding a frame evaluation API to CPython". - - .. versionadded:: 3.9 - - -.. c:function:: PyObject* PyThreadState_GetDict() - - Return a dictionary in which extensions can store thread-specific state - information. Each extension should use a unique key to use to store state in - the dictionary. It is okay to call this function when no :term:`thread state` - is :term:`attached `. If this function returns - ``NULL``, no exception has been raised and the caller should assume no - thread state is attached. - - -.. c:function:: int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) - - Asynchronously raise an exception in a thread. The *id* argument is the thread - id of the target thread; *exc* is the exception object to be raised. This - function does not steal any references to *exc*. To prevent naive misuse, you - must write your own C extension to call this. Must be called with an :term:`attached thread state`. - Returns the number of thread states modified; this is normally one, but will be - zero if the thread id isn't found. If *exc* is ``NULL``, the pending - exception (if any) for the thread is cleared. This raises no exceptions. - - .. versionchanged:: 3.7 - The type of the *id* parameter changed from :c:expr:`long` to - :c:expr:`unsigned long`. - -.. c:function:: void PyEval_AcquireThread(PyThreadState *tstate) - - :term:`Attach ` *tstate* to the current thread, - which must not be ``NULL`` or already :term:`attached `. - - The calling thread must not already have an :term:`attached thread state`. - - .. note:: - Calling this function from a thread when the runtime is finalizing will - hang the thread until the program exits, even if the thread was not - created by Python. Refer to - :ref:`cautions-regarding-runtime-finalization` for more details. - - .. versionchanged:: 3.8 - Updated to be consistent with :c:func:`PyEval_RestoreThread`, - :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, - and terminate the current thread if called while the interpreter is finalizing. - - .. versionchanged:: 3.14 - Hangs the current thread, rather than terminating it, if called while the - interpreter is finalizing. - - :c:func:`PyEval_RestoreThread` is a higher-level function which is always - available (even when threads have not been initialized). - - -.. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) - - Detach the :term:`attached thread state`. - The *tstate* argument, which must not be ``NULL``, is only used to check - that it represents the :term:`attached thread state` --- if it isn't, a fatal error is - reported. - - :c:func:`PyEval_SaveThread` is a higher-level function which is always - available (even when threads have not been initialized). - - -.. _sub-interpreter-support: - -Sub-interpreter support -======================= - -While in most uses, you will only embed a single Python interpreter, there -are cases where you need to create several independent interpreters in the -same process and perhaps even in the same thread. Sub-interpreters allow -you to do that. - -The "main" interpreter is the first one created when the runtime initializes. -It is usually the only Python interpreter in a process. Unlike sub-interpreters, -the main interpreter has unique process-global responsibilities like signal -handling. It is also responsible for execution during runtime initialization and -is usually the active interpreter during runtime finalization. The -:c:func:`PyInterpreterState_Main` function returns a pointer to its state. - -You can switch between sub-interpreters using the :c:func:`PyThreadState_Swap` -function. You can create and destroy them using the following functions: - - -.. c:type:: PyInterpreterConfig - - Structure containing most parameters to configure a sub-interpreter. - Its values are used only in :c:func:`Py_NewInterpreterFromConfig` and - never modified by the runtime. - - .. versionadded:: 3.12 - - Structure fields: - - .. c:member:: int use_main_obmalloc - - If this is ``0`` then the sub-interpreter will use its own - "object" allocator state. - Otherwise it will use (share) the main interpreter's. - - If this is ``0`` then - :c:member:`~PyInterpreterConfig.check_multi_interp_extensions` - must be ``1`` (non-zero). - If this is ``1`` then :c:member:`~PyInterpreterConfig.gil` - must not be :c:macro:`PyInterpreterConfig_OWN_GIL`. - - .. c:member:: int allow_fork - - If this is ``0`` then the runtime will not support forking the - process in any thread where the sub-interpreter is currently active. - Otherwise fork is unrestricted. - - Note that the :mod:`subprocess` module still works - when fork is disallowed. - - .. c:member:: int allow_exec - - If this is ``0`` then the runtime will not support replacing the - current process via exec (e.g. :func:`os.execv`) in any thread - where the sub-interpreter is currently active. - Otherwise exec is unrestricted. - - Note that the :mod:`subprocess` module still works - when exec is disallowed. - - .. c:member:: int allow_threads - - If this is ``0`` then the sub-interpreter's :mod:`threading` module - won't create threads. - Otherwise threads are allowed. - - .. c:member:: int allow_daemon_threads - - If this is ``0`` then the sub-interpreter's :mod:`threading` module - won't create daemon threads. - Otherwise daemon threads are allowed (as long as - :c:member:`~PyInterpreterConfig.allow_threads` is non-zero). - - .. c:member:: int check_multi_interp_extensions - - If this is ``0`` then all extension modules may be imported, - including legacy (single-phase init) modules, - in any thread where the sub-interpreter is currently active. - Otherwise only multi-phase init extension modules - (see :pep:`489`) may be imported. - (Also see :c:macro:`Py_mod_multiple_interpreters`.) - - This must be ``1`` (non-zero) if - :c:member:`~PyInterpreterConfig.use_main_obmalloc` is ``0``. - - .. c:member:: int gil - - This determines the operation of the GIL for the sub-interpreter. - It may be one of the following: - - .. c:namespace:: NULL - - .. c:macro:: PyInterpreterConfig_DEFAULT_GIL - - Use the default selection (:c:macro:`PyInterpreterConfig_SHARED_GIL`). - - .. c:macro:: PyInterpreterConfig_SHARED_GIL - - Use (share) the main interpreter's GIL. - - .. c:macro:: PyInterpreterConfig_OWN_GIL - - Use the sub-interpreter's own GIL. - - If this is :c:macro:`PyInterpreterConfig_OWN_GIL` then - :c:member:`PyInterpreterConfig.use_main_obmalloc` must be ``0``. - - -.. c:function:: PyStatus Py_NewInterpreterFromConfig(PyThreadState **tstate_p, const PyInterpreterConfig *config) - - .. index:: - pair: module; builtins - pair: module; __main__ - pair: module; sys - single: stdout (in module sys) - single: stderr (in module sys) - single: stdin (in module sys) - - Create a new sub-interpreter. This is an (almost) totally separate environment - for the execution of Python code. In particular, the new interpreter has - separate, independent versions of all imported modules, including the - fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. The - table of loaded modules (``sys.modules``) and the module search path - (``sys.path``) are also separate. The new environment has no ``sys.argv`` - variable. It has new standard I/O stream file objects ``sys.stdin``, - ``sys.stdout`` and ``sys.stderr`` (however these refer to the same underlying - file descriptors). - - The given *config* controls the options with which the interpreter - is initialized. - - Upon success, *tstate_p* will be set to the first :term:`thread state` - created in the new sub-interpreter. This thread state is - :term:`attached `. - Note that no actual thread is created; see the discussion of thread states - below. If creation of the new interpreter is unsuccessful, - *tstate_p* is set to ``NULL``; - no exception is set since the exception state is stored in the - :term:`attached thread state`, which might not exist. - - Like all other Python/C API functions, an :term:`attached thread state` - must be present before calling this function, but it might be detached upon - returning. On success, the returned thread state will be :term:`attached `. - If the sub-interpreter is created with its own :term:`GIL` then the - :term:`attached thread state` of the calling interpreter will be detached. - When the function returns, the new interpreter's :term:`thread state` - will be :term:`attached ` to the current thread and - the previous interpreter's :term:`attached thread state` will remain detached. - - .. versionadded:: 3.12 - - Sub-interpreters are most effective when isolated from each other, - with certain functionality restricted:: - - PyInterpreterConfig config = { - .use_main_obmalloc = 0, - .allow_fork = 0, - .allow_exec = 0, - .allow_threads = 1, - .allow_daemon_threads = 0, - .check_multi_interp_extensions = 1, - .gil = PyInterpreterConfig_OWN_GIL, - }; - PyThreadState *tstate = NULL; - PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); - } - - Note that the config is used only briefly and does not get modified. - During initialization the config's values are converted into various - :c:type:`PyInterpreterState` values. A read-only copy of the config - may be stored internally on the :c:type:`PyInterpreterState`. - - .. index:: - single: Py_FinalizeEx (C function) - single: Py_Initialize (C function) - - Extension modules are shared between (sub-)interpreters as follows: - - * For modules using multi-phase initialization, - e.g. :c:func:`PyModule_FromDefAndSpec`, a separate module object is - created and initialized for each interpreter. - Only C-level static and global variables are shared between these - module objects. - - * For modules using single-phase initialization, - e.g. :c:func:`PyModule_Create`, the first time a particular extension - is imported, it is initialized normally, and a (shallow) copy of its - module's dictionary is squirreled away. - When the same extension is imported by another (sub-)interpreter, a new - module is initialized and filled with the contents of this copy; the - extension's ``init`` function is not called. - Objects in the module's dictionary thus end up shared across - (sub-)interpreters, which might cause unwanted behavior (see - `Bugs and caveats`_ below). - - Note that this is different from what happens when an extension is - imported after the interpreter has been completely re-initialized by - calling :c:func:`Py_FinalizeEx` and :c:func:`Py_Initialize`; in that - case, the extension's ``initmodule`` function *is* called again. - As with multi-phase initialization, this means that only C-level static - and global variables are shared between these modules. - - .. index:: single: close (in module os) - - -.. c:function:: PyThreadState* Py_NewInterpreter(void) - - .. index:: - pair: module; builtins - pair: module; __main__ - pair: module; sys - single: stdout (in module sys) - single: stderr (in module sys) - single: stdin (in module sys) - - Create a new sub-interpreter. This is essentially just a wrapper - around :c:func:`Py_NewInterpreterFromConfig` with a config that - preserves the existing behavior. The result is an unisolated - sub-interpreter that shares the main interpreter's GIL, allows - fork/exec, allows daemon threads, and allows single-phase init - modules. - - -.. c:function:: void Py_EndInterpreter(PyThreadState *tstate) - - .. index:: single: Py_FinalizeEx (C function) - - Destroy the (sub-)interpreter represented by the given :term:`thread state`. - The given thread state must be :term:`attached `. - When the call returns, there will be no :term:`attached thread state`. - All thread states associated with this interpreter are destroyed. - - :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that - haven't been explicitly destroyed at that point. - - -A Per-Interpreter GIL ---------------------- - -Using :c:func:`Py_NewInterpreterFromConfig` you can create -a sub-interpreter that is completely isolated from other interpreters, -including having its own GIL. The most important benefit of this -isolation is that such an interpreter can execute Python code without -being blocked by other interpreters or blocking any others. Thus a -single Python process can truly take advantage of multiple CPU cores -when running Python code. The isolation also encourages a different -approach to concurrency than that of just using threads. -(See :pep:`554`.) - -Using an isolated interpreter requires vigilance in preserving that -isolation. That especially means not sharing any objects or mutable -state without guarantees about thread-safety. Even objects that are -otherwise immutable (e.g. ``None``, ``(1, 5)``) can't normally be shared -because of the refcount. One simple but less-efficient approach around -this is to use a global lock around all use of some state (or object). -Alternately, effectively immutable objects (like integers or strings) -can be made safe in spite of their refcounts by making them :term:`immortal`. -In fact, this has been done for the builtin singletons, small integers, -and a number of other builtin objects. - -If you preserve isolation then you will have access to proper multi-core -computing without the complications that come with free-threading. -Failure to preserve isolation will expose you to the full consequences -of free-threading, including races and hard-to-debug crashes. - -Aside from that, one of the main challenges of using multiple isolated -interpreters is how to communicate between them safely (not break -isolation) and efficiently. The runtime and stdlib do not provide -any standard approach to this yet. A future stdlib module would help -mitigate the effort of preserving isolation and expose effective tools -for communicating (and sharing) data between interpreters. - -.. versionadded:: 3.12 - - -Bugs and caveats ----------------- - -Because sub-interpreters (and the main interpreter) are part of the same -process, the insulation between them isn't perfect --- for example, using -low-level file operations like :func:`os.close` they can -(accidentally or maliciously) affect each other's open files. Because of the -way extensions are shared between (sub-)interpreters, some extensions may not -work properly; this is especially likely when using single-phase initialization -or (static) global variables. -It is possible to insert objects created in one sub-interpreter into -a namespace of another (sub-)interpreter; this should be avoided if possible. - -Special care should be taken to avoid sharing user-defined functions, -methods, instances or classes between sub-interpreters, since import -operations executed by such objects may affect the wrong (sub-)interpreter's -dictionary of loaded modules. It is equally important to avoid sharing -objects from which the above are reachable. - -Also note that combining this functionality with ``PyGILState_*`` APIs -is delicate, because these APIs assume a bijection between Python thread states -and OS-level threads, an assumption broken by the presence of sub-interpreters. -It is highly recommended that you don't switch sub-interpreters between a pair -of matching :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` calls. -Furthermore, extensions (such as :mod:`ctypes`) using these APIs to allow calling -of Python code from non-Python created threads will probably be broken when using -sub-interpreters. - - -Asynchronous Notifications -========================== - -A mechanism is provided to make asynchronous notifications to the main -interpreter thread. These notifications take the form of a function -pointer and a void pointer argument. - - -.. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) - - Schedule a function to be called from the main interpreter thread. On - success, ``0`` is returned and *func* is queued for being called in the - main thread. On failure, ``-1`` is returned without setting any exception. - - When successfully queued, *func* will be *eventually* called from the - main interpreter thread with the argument *arg*. It will be called - asynchronously with respect to normally running Python code, but with - both these conditions met: - - * on a :term:`bytecode` boundary; - * with the main thread holding an :term:`attached thread state` - (*func* can therefore use the full C API). - - *func* must return ``0`` on success, or ``-1`` on failure with an exception - set. *func* won't be interrupted to perform another asynchronous - notification recursively, but it can still be interrupted to switch - threads if the :term:`thread state ` is detached. - - This function doesn't need an :term:`attached thread state`. However, to call this - function in a subinterpreter, the caller must have an :term:`attached thread state`. - Otherwise, the function *func* can be scheduled to be called from the wrong interpreter. - - .. warning:: - This is a low-level function, only useful for very special cases. - There is no guarantee that *func* will be called as quick as - possible. If the main thread is busy executing a system call, - *func* won't be called before the system call returns. This - function is generally **not** suitable for calling Python code from - arbitrary C threads. Instead, use the :ref:`PyGILState API`. - - .. versionadded:: 3.1 - - .. versionchanged:: 3.9 - If this function is called in a subinterpreter, the function *func* is - now scheduled to be called from the subinterpreter, rather than being - called from the main interpreter. Each subinterpreter now has its own - list of scheduled calls. - -.. _profiling: - -Profiling and Tracing -===================== - -.. sectionauthor:: Fred L. Drake, Jr. - - -The Python interpreter provides some low-level support for attaching profiling -and execution tracing facilities. These are used for profiling, debugging, and -coverage analysis tools. - -This C interface allows the profiling or tracing code to avoid the overhead of -calling through Python-level callable objects, making a direct C function call -instead. The essential attributes of the facility have not changed; the -interface allows trace functions to be installed per-thread, and the basic -events reported to the trace function are the same as had been reported to the -Python-level trace functions in previous versions. - - -.. c:type:: int (*Py_tracefunc)(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg) - - The type of the trace function registered using :c:func:`PyEval_SetProfile` and - :c:func:`PyEval_SetTrace`. The first parameter is the object passed to the - registration function as *obj*, *frame* is the frame object to which the event - pertains, *what* is one of the constants :c:data:`PyTrace_CALL`, - :c:data:`PyTrace_EXCEPTION`, :c:data:`PyTrace_LINE`, :c:data:`PyTrace_RETURN`, - :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION`, :c:data:`PyTrace_C_RETURN`, - or :c:data:`PyTrace_OPCODE`, and *arg* depends on the value of *what*: - - +-------------------------------+----------------------------------------+ - | Value of *what* | Meaning of *arg* | - +===============================+========================================+ - | :c:data:`PyTrace_CALL` | Always :c:data:`Py_None`. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_EXCEPTION` | Exception information as returned by | - | | :func:`sys.exc_info`. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_LINE` | Always :c:data:`Py_None`. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_RETURN` | Value being returned to the caller, | - | | or ``NULL`` if caused by an exception. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_C_CALL` | Function object being called. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_C_EXCEPTION` | Function object being called. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_C_RETURN` | Function object being called. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_OPCODE` | Always :c:data:`Py_None`. | - +-------------------------------+----------------------------------------+ - -.. c:var:: int PyTrace_CALL - - The value of the *what* parameter to a :c:type:`Py_tracefunc` function when a new - call to a function or method is being reported, or a new entry into a generator. - Note that the creation of the iterator for a generator function is not reported - as there is no control transfer to the Python bytecode in the corresponding - frame. - - -.. c:var:: int PyTrace_EXCEPTION - - The value of the *what* parameter to a :c:type:`Py_tracefunc` function when an - exception has been raised. The callback function is called with this value for - *what* when after any bytecode is processed after which the exception becomes - set within the frame being executed. The effect of this is that as exception - propagation causes the Python stack to unwind, the callback is called upon - return to each frame as the exception propagates. Only trace functions receives - these events; they are not needed by the profiler. - - -.. c:var:: int PyTrace_LINE - - The value passed as the *what* parameter to a :c:type:`Py_tracefunc` function - (but not a profiling function) when a line-number event is being reported. - It may be disabled for a frame by setting :attr:`~frame.f_trace_lines` to - *0* on that frame. - - -.. c:var:: int PyTrace_RETURN - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a - call is about to return. - - -.. c:var:: int PyTrace_C_CALL - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C - function is about to be called. - - -.. c:var:: int PyTrace_C_EXCEPTION - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C - function has raised an exception. - - -.. c:var:: int PyTrace_C_RETURN - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C - function has returned. - - -.. c:var:: int PyTrace_OPCODE - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions (but not - profiling functions) when a new opcode is about to be executed. This event is - not emitted by default: it must be explicitly requested by setting - :attr:`~frame.f_trace_opcodes` to *1* on the frame. - - -.. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj) - - Set the profiler function to *func*. The *obj* parameter is passed to the - function as its first parameter, and may be any Python object, or ``NULL``. If - the profile function needs to maintain state, using a different value for *obj* - for each thread provides a convenient and thread-safe place to store it. The - profile function is called for all monitored events except :c:data:`PyTrace_LINE` - :c:data:`PyTrace_OPCODE` and :c:data:`PyTrace_EXCEPTION`. - - See also the :func:`sys.setprofile` function. - - The caller must have an :term:`attached thread state`. - -.. c:function:: void PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *obj) - - Like :c:func:`PyEval_SetProfile` but sets the profile function in all running threads - belonging to the current interpreter instead of the setting it only on the current thread. - - The caller must have an :term:`attached thread state`. - - As :c:func:`PyEval_SetProfile`, this function ignores any exceptions raised while - setting the profile functions in all threads. - -.. versionadded:: 3.12 - - -.. c:function:: void PyEval_SetTrace(Py_tracefunc func, PyObject *obj) - - Set the tracing function to *func*. This is similar to - :c:func:`PyEval_SetProfile`, except the tracing function does receive line-number - events and per-opcode events, but does not receive any event related to C function - objects being called. Any trace function registered using :c:func:`PyEval_SetTrace` - will not receive :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION` or - :c:data:`PyTrace_C_RETURN` as a value for the *what* parameter. - - See also the :func:`sys.settrace` function. - - The caller must have an :term:`attached thread state`. - -.. c:function:: void PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject *obj) - - Like :c:func:`PyEval_SetTrace` but sets the tracing function in all running threads - belonging to the current interpreter instead of the setting it only on the current thread. - - The caller must have an :term:`attached thread state`. - - As :c:func:`PyEval_SetTrace`, this function ignores any exceptions raised while - setting the trace functions in all threads. - -.. versionadded:: 3.12 - -Reference tracing -================= - -.. versionadded:: 3.13 - -.. c:type:: int (*PyRefTracer)(PyObject *, int event, void* data) - - The type of the trace function registered using :c:func:`PyRefTracer_SetTracer`. - The first parameter is a Python object that has been just created (when **event** - is set to :c:data:`PyRefTracer_CREATE`) or about to be destroyed (when **event** - is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer - that was provided when :c:func:`PyRefTracer_SetTracer` was called. - -.. versionadded:: 3.13 - -.. c:var:: int PyRefTracer_CREATE - - The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python - object has been created. - -.. c:var:: int PyRefTracer_DESTROY - - The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python - object has been destroyed. - -.. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) - - Register a reference tracer function. The function will be called when a new - Python has been created or when an object is going to be destroyed. If - **data** is provided it must be an opaque pointer that will be provided when - the tracer function is called. Return ``0`` on success. Set an exception and - return ``-1`` on error. - - Not that tracer functions **must not** create Python objects inside or - otherwise the call will be re-entrant. The tracer also **must not** clear - any existing exception or set an exception. A :term:`thread state` will be active - every time the tracer function is called. - - There must be an :term:`attached thread state` when calling this function. - -.. versionadded:: 3.13 - -.. c:function:: PyRefTracer PyRefTracer_GetTracer(void** data) - - Get the registered reference tracer function and the value of the opaque data - pointer that was registered when :c:func:`PyRefTracer_SetTracer` was called. - If no tracer was registered this function will return NULL and will set the - **data** pointer to NULL. - - There must be an :term:`attached thread state` when calling this function. - -.. versionadded:: 3.13 - -.. _advanced-debugging: - -Advanced Debugger Support -========================= - -.. sectionauthor:: Fred L. Drake, Jr. - - -These functions are only intended to be used by advanced debugging tools. - - -.. c:function:: PyInterpreterState* PyInterpreterState_Head() - - Return the interpreter state object at the head of the list of all such objects. - - -.. c:function:: PyInterpreterState* PyInterpreterState_Main() - - Return the main interpreter state object. - - -.. c:function:: PyInterpreterState* PyInterpreterState_Next(PyInterpreterState *interp) - - Return the next interpreter state object after *interp* from the list of all - such objects. - - -.. c:function:: PyThreadState * PyInterpreterState_ThreadHead(PyInterpreterState *interp) - - Return the pointer to the first :c:type:`PyThreadState` object in the list of - threads associated with the interpreter *interp*. - - -.. c:function:: PyThreadState* PyThreadState_Next(PyThreadState *tstate) - - Return the next thread state object after *tstate* from the list of all such - objects belonging to the same :c:type:`PyInterpreterState` object. - - -.. _thread-local-storage: - -Thread Local Storage Support -============================ - -.. sectionauthor:: Masayuki Yamamoto - -The Python interpreter provides low-level support for thread-local storage -(TLS) which wraps the underlying native TLS implementation to support the -Python-level thread local storage API (:class:`threading.local`). The -CPython C level APIs are similar to those offered by pthreads and Windows: -use a thread key and functions to associate a :c:expr:`void*` value per -thread. - -A :term:`thread state` does *not* need to be :term:`attached ` -when calling these functions; they suppl their own locking. - -Note that :file:`Python.h` does not include the declaration of the TLS APIs, -you need to include :file:`pythread.h` to use thread-local storage. - -.. note:: - None of these API functions handle memory management on behalf of the - :c:expr:`void*` values. You need to allocate and deallocate them yourself. - If the :c:expr:`void*` values happen to be :c:expr:`PyObject*`, these - functions don't do refcount operations on them either. - -.. _thread-specific-storage-api: - -Thread Specific Storage (TSS) API ---------------------------------- - -TSS API is introduced to supersede the use of the existing TLS API within the -CPython interpreter. This API uses a new type :c:type:`Py_tss_t` instead of -:c:expr:`int` to represent thread keys. - -.. versionadded:: 3.7 - -.. seealso:: "A New C-API for Thread-Local Storage in CPython" (:pep:`539`) - - -.. c:type:: Py_tss_t - - This data structure represents the state of a thread key, the definition of - which may depend on the underlying TLS implementation, and it has an - internal field representing the key's initialization state. There are no - public members in this structure. - - When :ref:`Py_LIMITED_API ` is not defined, static allocation of - this type by :c:macro:`Py_tss_NEEDS_INIT` is allowed. - - -.. c:macro:: Py_tss_NEEDS_INIT - - This macro expands to the initializer for :c:type:`Py_tss_t` variables. - Note that this macro won't be defined with :ref:`Py_LIMITED_API `. - - -Dynamic Allocation -~~~~~~~~~~~~~~~~~~ - -Dynamic allocation of the :c:type:`Py_tss_t`, required in extension modules -built with :ref:`Py_LIMITED_API `, where static allocation of this type -is not possible due to its implementation being opaque at build time. - - -.. c:function:: Py_tss_t* PyThread_tss_alloc() - - Return a value which is the same state as a value initialized with - :c:macro:`Py_tss_NEEDS_INIT`, or ``NULL`` in the case of dynamic allocation - failure. - - -.. c:function:: void PyThread_tss_free(Py_tss_t *key) - - Free the given *key* allocated by :c:func:`PyThread_tss_alloc`, after - first calling :c:func:`PyThread_tss_delete` to ensure any associated - thread locals have been unassigned. This is a no-op if the *key* - argument is ``NULL``. - - .. note:: - A freed key becomes a dangling pointer. You should reset the key to - ``NULL``. - - -Methods -~~~~~~~ - -The parameter *key* of these functions must not be ``NULL``. Moreover, the -behaviors of :c:func:`PyThread_tss_set` and :c:func:`PyThread_tss_get` are -undefined if the given :c:type:`Py_tss_t` has not been initialized by -:c:func:`PyThread_tss_create`. - - -.. c:function:: int PyThread_tss_is_created(Py_tss_t *key) - - Return a non-zero value if the given :c:type:`Py_tss_t` has been initialized - by :c:func:`PyThread_tss_create`. - - -.. c:function:: int PyThread_tss_create(Py_tss_t *key) - - Return a zero value on successful initialization of a TSS key. The behavior - is undefined if the value pointed to by the *key* argument is not - initialized by :c:macro:`Py_tss_NEEDS_INIT`. This function can be called - repeatedly on the same key -- calling it on an already initialized key is a - no-op and immediately returns success. - - -.. c:function:: void PyThread_tss_delete(Py_tss_t *key) - - Destroy a TSS key to forget the values associated with the key across all - threads, and change the key's initialization state to uninitialized. A - destroyed key is able to be initialized again by - :c:func:`PyThread_tss_create`. This function can be called repeatedly on - the same key -- calling it on an already destroyed key is a no-op. - - -.. c:function:: int PyThread_tss_set(Py_tss_t *key, void *value) - - Return a zero value to indicate successfully associating a :c:expr:`void*` - value with a TSS key in the current thread. Each thread has a distinct - mapping of the key to a :c:expr:`void*` value. - - -.. c:function:: void* PyThread_tss_get(Py_tss_t *key) - - Return the :c:expr:`void*` value associated with a TSS key in the current - thread. This returns ``NULL`` if no value is associated with the key in the - current thread. - - -.. _thread-local-storage-api: - -Thread Local Storage (TLS) API ------------------------------- - -.. deprecated:: 3.7 - This API is superseded by - :ref:`Thread Specific Storage (TSS) API `. - -.. note:: - This version of the API does not support platforms where the native TLS key - is defined in a way that cannot be safely cast to ``int``. On such platforms, - :c:func:`PyThread_create_key` will return immediately with a failure status, - and the other TLS functions will all be no-ops on such platforms. - -Due to the compatibility problem noted above, this version of the API should not -be used in new code. - -.. c:function:: int PyThread_create_key() -.. c:function:: void PyThread_delete_key(int key) -.. c:function:: int PyThread_set_key_value(int key, void *value) -.. c:function:: void* PyThread_get_key_value(int key) -.. c:function:: void PyThread_delete_key_value(int key) -.. c:function:: void PyThread_ReInitTLS() - -Synchronization Primitives -========================== - -The C-API provides a basic mutual exclusion lock. - -.. c:type:: PyMutex - - A mutual exclusion lock. The :c:type:`!PyMutex` should be initialized to - zero to represent the unlocked state. For example:: - - PyMutex mutex = {0}; - - Instances of :c:type:`!PyMutex` should not be copied or moved. Both the - contents and address of a :c:type:`!PyMutex` are meaningful, and it must - remain at a fixed, writable location in memory. - - .. note:: - - A :c:type:`!PyMutex` currently occupies one byte, but the size should be - considered unstable. The size may change in future Python releases - without a deprecation period. - - .. versionadded:: 3.13 - -.. c:function:: void PyMutex_Lock(PyMutex *m) - - Lock mutex *m*. If another thread has already locked it, the calling - thread will block until the mutex is unlocked. While blocked, the thread - will temporarily detach the :term:`thread state ` if one exists. - - .. versionadded:: 3.13 - -.. c:function:: void PyMutex_Unlock(PyMutex *m) - - Unlock mutex *m*. The mutex must be locked --- otherwise, the function will - issue a fatal error. - - .. versionadded:: 3.13 - -.. _python-critical-section-api: - -Python Critical Section API ---------------------------- - -The critical section API provides a deadlock avoidance layer on top of -per-object locks for :term:`free-threaded ` CPython. They are -intended to replace reliance on the :term:`global interpreter lock`, and are -no-ops in versions of Python with the global interpreter lock. - -Critical sections avoid deadlocks by implicitly suspending active critical -sections and releasing the locks during calls to :c:func:`PyEval_SaveThread`. -When :c:func:`PyEval_RestoreThread` is called, the most recent critical section -is resumed, and its locks reacquired. This means the critical section API -provides weaker guarantees than traditional locks -- they are useful because -their behavior is similar to the :term:`GIL`. - -The functions and structs used by the macros are exposed for cases -where C macros are not available. They should only be used as in the -given macro expansions. Note that the sizes and contents of the structures may -change in future Python versions. - -.. note:: - - Operations that need to lock two objects at once must use - :c:macro:`Py_BEGIN_CRITICAL_SECTION2`. You *cannot* use nested critical - sections to lock more than one object at once, because the inner critical - section may suspend the outer critical sections. This API does not provide - a way to lock more than two objects at once. - -Example usage:: - - static PyObject * - set_field(MyObject *self, PyObject *value) - { - Py_BEGIN_CRITICAL_SECTION(self); - Py_SETREF(self->field, Py_XNewRef(value)); - Py_END_CRITICAL_SECTION(); - Py_RETURN_NONE; - } - -In the above example, :c:macro:`Py_SETREF` calls :c:macro:`Py_DECREF`, which -can call arbitrary code through an object's deallocation function. The critical -section API avoids potential deadlocks due to reentrancy and lock ordering -by allowing the runtime to temporarily suspend the critical section if the -code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. - -.. c:macro:: Py_BEGIN_CRITICAL_SECTION(op) - - Acquires the per-object lock for the object *op* and begins a - critical section. - - In the free-threaded build, this macro expands to:: - - { - PyCriticalSection _py_cs; - PyCriticalSection_Begin(&_py_cs, (PyObject*)(op)) - - In the default build, this macro expands to ``{``. - - .. versionadded:: 3.13 - -.. c:macro:: Py_END_CRITICAL_SECTION() - - Ends the critical section and releases the per-object lock. - - In the free-threaded build, this macro expands to:: - - PyCriticalSection_End(&_py_cs); - } - - In the default build, this macro expands to ``}``. - - .. versionadded:: 3.13 - -.. c:macro:: Py_BEGIN_CRITICAL_SECTION2(a, b) - - Acquires the per-objects locks for the objects *a* and *b* and begins a - critical section. The locks are acquired in a consistent order (lowest - address first) to avoid lock ordering deadlocks. - - In the free-threaded build, this macro expands to:: - - { - PyCriticalSection2 _py_cs2; - PyCriticalSection2_Begin(&_py_cs2, (PyObject*)(a), (PyObject*)(b)) - - In the default build, this macro expands to ``{``. - - .. versionadded:: 3.13 - -.. c:macro:: Py_END_CRITICAL_SECTION2() - - Ends the critical section and releases the per-object locks. - - In the free-threaded build, this macro expands to:: - - PyCriticalSection2_End(&_py_cs2); - } - - In the default build, this macro expands to ``}``. - - .. versionadded:: 3.13 +- :ref:`initialization` +- :ref:`threads` +- :ref:`synchronization` +- :ref:`thread-local-storage` +- :ref:`sub-interpreter-support` +- :ref:`profiling` diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index e1931655618b1c9..487e8491f1153c2 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -102,7 +102,7 @@ Error Handling * Set *\*err_msg* and return ``1`` if an error is set. * Set *\*err_msg* to ``NULL`` and return ``0`` otherwise. - An error message is an UTF-8 encoded string. + An error message is a UTF-8 encoded string. If *config* has an exit code, format the exit code as an error message. @@ -544,9 +544,9 @@ Configuration Options Visibility: -* Public: Can by get by :c:func:`PyConfig_Get` and set by +* Public: Can be retrieved by :c:func:`PyConfig_Get` and set by :c:func:`PyConfig_Set`. -* Read-only: Can by get by :c:func:`PyConfig_Get`, but cannot be set by +* Read-only: Can be retrieved by :c:func:`PyConfig_Get`, but cannot be set by :c:func:`PyConfig_Set`. @@ -623,6 +623,10 @@ Some options are read from the :mod:`sys` attributes. For example, the option .. versionadded:: 3.14 + .. versionchanged:: next + The function now replaces :data:`sys.flags` (create a new object), + instead of modifying :data:`sys.flags` in-place. + .. _pyconfig_api: @@ -1155,7 +1159,7 @@ PyConfig Most ``PyConfig`` methods :ref:`preinitialize Python ` if needed. In that case, the Python preinitialization configuration - (:c:type:`PyPreConfig`) in based on the :c:type:`PyConfig`. If configuration + (:c:type:`PyPreConfig`) is based on the :c:type:`PyConfig`. If configuration fields which are in common with :c:type:`PyPreConfig` are tuned, they must be set before calling a :c:type:`PyConfig` method: diff --git a/Doc/c-api/interp-lifecycle.rst b/Doc/c-api/interp-lifecycle.rst new file mode 100644 index 000000000000000..f8c8fdaa05b46fc --- /dev/null +++ b/Doc/c-api/interp-lifecycle.rst @@ -0,0 +1,956 @@ +.. highlight:: c + +.. _initialization: + +Interpreter initialization and finalization +=========================================== + +See :ref:`Python Initialization Configuration ` for details +on how to configure the interpreter prior to initialization. + +.. _pre-init-safe: + +Before Python initialization +---------------------------- + +In an application embedding Python, the :c:func:`Py_Initialize` function must +be called before using any other Python/C API functions; with the exception of +a few functions and the :ref:`global configuration variables +`. + +The following functions can be safely called before Python is initialized: + +* Functions that initialize the interpreter: + + * :c:func:`Py_Initialize` + * :c:func:`Py_InitializeEx` + * :c:func:`Py_InitializeFromConfig` + * :c:func:`Py_BytesMain` + * :c:func:`Py_Main` + * the runtime pre-initialization functions covered in :ref:`init-config` + +* Configuration functions: + + * :c:func:`PyImport_AppendInittab` + * :c:func:`PyImport_ExtendInittab` + * :c:func:`!PyInitFrozenExtensions` + * :c:func:`PyMem_SetAllocator` + * :c:func:`PyMem_SetupDebugHooks` + * :c:func:`PyObject_SetArenaAllocator` + * :c:func:`Py_SetProgramName` + * :c:func:`Py_SetPythonHome` + * the configuration functions covered in :ref:`init-config` + +* Informative functions: + + * :c:func:`Py_IsInitialized` + * :c:func:`PyMem_GetAllocator` + * :c:func:`PyObject_GetArenaAllocator` + * :c:func:`Py_GetBuildInfo` + * :c:func:`Py_GetCompiler` + * :c:func:`Py_GetCopyright` + * :c:func:`Py_GetPlatform` + * :c:func:`Py_GetVersion` + * :c:func:`Py_IsInitialized` + +* Utilities: + + * :c:func:`Py_DecodeLocale` + * the status reporting and utility functions covered in :ref:`init-config` + +* Memory allocators: + + * :c:func:`PyMem_RawMalloc` + * :c:func:`PyMem_RawRealloc` + * :c:func:`PyMem_RawCalloc` + * :c:func:`PyMem_RawFree` + +* Synchronization: + + * :c:func:`PyMutex_Lock` + * :c:func:`PyMutex_Unlock` + +.. note:: + + Despite their apparent similarity to some of the functions listed above, + the following functions **should not be called** before the interpreter has + been initialized: :c:func:`Py_EncodeLocale`, :c:func:`PyEval_InitThreads`, and + :c:func:`Py_RunMain`. + + +.. _global-conf-vars: + +Global configuration variables +------------------------------ + +Python has variables for the global configuration to control different features +and options. By default, these flags are controlled by :ref:`command line +options `. + +When a flag is set by an option, the value of the flag is the number of times +that the option was set. For example, ``-b`` sets :c:data:`Py_BytesWarningFlag` +to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. + + +.. c:var:: int Py_BytesWarningFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.bytes_warning` should be used instead, see :ref:`Python + Initialization Configuration `. + + Issue a warning when comparing :class:`bytes` or :class:`bytearray` with + :class:`str` or :class:`bytes` with :class:`int`. Issue an error if greater + or equal to ``2``. + + Set by the :option:`-b` option. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_DebugFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.parser_debug` should be used instead, see :ref:`Python + Initialization Configuration `. + + Turn on parser debugging output (for expert only, depending on compilation + options). + + Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment + variable. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_DontWriteBytecodeFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.write_bytecode` should be used instead, see :ref:`Python + Initialization Configuration `. + + If set to non-zero, Python won't try to write ``.pyc`` files on the + import of source modules. + + Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` + environment variable. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_FrozenFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.pathconfig_warnings` should be used instead, see + :ref:`Python Initialization Configuration `. + + Private flag used by ``_freeze_module`` and ``frozenmain`` programs. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_HashRandomizationFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.hash_seed` and :c:member:`PyConfig.use_hash_seed` should + be used instead, see :ref:`Python Initialization Configuration + `. + + Set to ``1`` if the :envvar:`PYTHONHASHSEED` environment variable is set to + a non-empty string. + + If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment + variable to initialize the secret hash seed. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_IgnoreEnvironmentFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.use_environment` should be used instead, see + :ref:`Python Initialization Configuration `. + + Ignore all :envvar:`!PYTHON*` environment variables, e.g. + :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. + + Set by the :option:`-E` and :option:`-I` options. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_InspectFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.inspect` should be used instead, see + :ref:`Python Initialization Configuration `. + + When a script is passed as first argument or the :option:`-c` option is used, + enter interactive mode after executing the script or the command, even when + :data:`sys.stdin` does not appear to be a terminal. + + Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment + variable. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_InteractiveFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.interactive` should be used instead, see + :ref:`Python Initialization Configuration `. + + Set by the :option:`-i` option. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_IsolatedFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.isolated` should be used instead, see + :ref:`Python Initialization Configuration `. + + Run Python in isolated mode. In isolated mode :data:`sys.path` contains + neither the script's directory nor the user's site-packages directory. + + Set by the :option:`-I` option. + + .. versionadded:: 3.4 + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_LegacyWindowsFSEncodingFlag + + This API is kept for backward compatibility: setting + :c:member:`PyPreConfig.legacy_windows_fs_encoding` should be used instead, see + :ref:`Python Initialization Configuration `. + + If the flag is non-zero, use the ``mbcs`` encoding with ``replace`` error + handler, instead of the UTF-8 encoding with ``surrogatepass`` error handler, + for the :term:`filesystem encoding and error handler`. + + Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment + variable is set to a non-empty string. + + See :pep:`529` for more details. + + .. availability:: Windows. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_LegacyWindowsStdioFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.legacy_windows_stdio` should be used instead, see + :ref:`Python Initialization Configuration `. + + If the flag is non-zero, use :class:`io.FileIO` instead of + :class:`!io._WindowsConsoleIO` for :mod:`sys` standard streams. + + Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment + variable is set to a non-empty string. + + See :pep:`528` for more details. + + .. availability:: Windows. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_NoSiteFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.site_import` should be used instead, see + :ref:`Python Initialization Configuration `. + + Disable the import of the module :mod:`site` and the site-dependent + manipulations of :data:`sys.path` that it entails. Also disable these + manipulations if :mod:`site` is explicitly imported later (call + :func:`site.main` if you want them to be triggered). + + Set by the :option:`-S` option. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_NoUserSiteDirectory + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.user_site_directory` should be used instead, see + :ref:`Python Initialization Configuration `. + + Don't add the :data:`user site-packages directory ` to + :data:`sys.path`. + + Set by the :option:`-s` and :option:`-I` options, and the + :envvar:`PYTHONNOUSERSITE` environment variable. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_OptimizeFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.optimization_level` should be used instead, see + :ref:`Python Initialization Configuration `. + + Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment + variable. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_QuietFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.quiet` should be used instead, see :ref:`Python + Initialization Configuration `. + + Don't display the copyright and version messages even in interactive mode. + + Set by the :option:`-q` option. + + .. versionadded:: 3.2 + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_UnbufferedStdioFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.buffered_stdio` should be used instead, see :ref:`Python + Initialization Configuration `. + + Force the stdout and stderr streams to be unbuffered. + + Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` + environment variable. + + .. deprecated-removed:: 3.12 3.15 + + +.. c:var:: int Py_VerboseFlag + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.verbose` should be used instead, see :ref:`Python + Initialization Configuration `. + + Print a message each time a module is initialized, showing the place + (filename or built-in module) from which it is loaded. If greater or equal + to ``2``, print a message for each file that is checked for when + searching for a module. Also provides information on module cleanup at exit. + + Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment + variable. + + .. deprecated-removed:: 3.12 3.15 + + +Initializing and finalizing the interpreter +------------------------------------------- + +.. c:function:: void Py_Initialize() + + .. index:: + single: PyEval_InitThreads() + single: modules (in module sys) + single: path (in module sys) + pair: module; builtins + pair: module; __main__ + pair: module; sys + triple: module; search; path + single: Py_FinalizeEx (C function) + + Initialize the Python interpreter. In an application embedding Python, + this should be called before using any other Python/C API functions; see + :ref:`Before Python Initialization ` for the few exceptions. + + This initializes the table of loaded modules (``sys.modules``), and creates + the fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. + It also initializes the module search path (``sys.path``). It does not set + ``sys.argv``; use the :ref:`Python Initialization Configuration ` + API for that. This is a no-op when called for a second time (without calling + :c:func:`Py_FinalizeEx` first). There is no return value; it is a fatal + error if the initialization fails. + + Use :c:func:`Py_InitializeFromConfig` to customize the + :ref:`Python Initialization Configuration `. + + .. note:: + On Windows, changes the console mode from ``O_TEXT`` to ``O_BINARY``, + which will also affect non-Python uses of the console using the C Runtime. + + +.. c:function:: void Py_InitializeEx(int initsigs) + + This function works like :c:func:`Py_Initialize` if *initsigs* is ``1``. If + *initsigs* is ``0``, it skips initialization registration of signal handlers, + which may be useful when CPython is embedded as part of a larger application. + + Use :c:func:`Py_InitializeFromConfig` to customize the + :ref:`Python Initialization Configuration `. + + +.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) + + Initialize Python from *config* configuration, as described in + :ref:`init-from-config`. + + See the :ref:`init-config` section for details on pre-initializing the + interpreter, populating the runtime configuration structure, and querying + the returned status structure. + + +.. c:function:: int Py_IsInitialized() + + Return true (nonzero) when the Python interpreter has been initialized, false + (zero) if not. After :c:func:`Py_FinalizeEx` is called, this returns false until + :c:func:`Py_Initialize` is called again. + + +.. c:function:: int Py_IsFinalizing() + + Return true (non-zero) if the main Python interpreter is + :term:`shutting down `. Return false (zero) otherwise. + + .. versionadded:: 3.13 + + +.. c:function:: int Py_FinalizeEx() + + Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of + Python/C API functions, and destroy all sub-interpreters (see + :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since + the last call to :c:func:`Py_Initialize`. This is a no-op when called for a second + time (without calling :c:func:`Py_Initialize` again first). + + Since this is the reverse of :c:func:`Py_Initialize`, it should be called + in the same thread with the same interpreter active. That means + the main thread and the main interpreter. + This should never be called while :c:func:`Py_RunMain` is running. + + Normally the return value is ``0``. + If there were errors during finalization (flushing buffered data), + ``-1`` is returned. + + Note that Python will do a best effort at freeing all memory allocated by the Python + interpreter. Therefore, any C-Extension should make sure to correctly clean up all + of the previously allocated PyObjects before using them in subsequent calls to + :c:func:`Py_Initialize`. Otherwise it could introduce vulnerabilities and incorrect + behavior. + + This function is provided for a number of reasons. An embedding application + might want to restart Python without having to restart the application itself. + An application that has loaded the Python interpreter from a dynamically + loadable library (or DLL) might want to free all memory allocated by Python + before unloading the DLL. During a hunt for memory leaks in an application a + developer might want to free all memory allocated by Python before exiting from + the application. + + **Bugs and caveats:** The destruction of modules and objects in modules is done + in random order; this may cause destructors (:meth:`~object.__del__` methods) to fail + when they depend on other objects (even functions) or modules. Dynamically + loaded extension modules loaded by Python are not unloaded. Small amounts of + memory allocated by the Python interpreter may not be freed (if you find a leak, + please report it). Memory tied up in circular references between objects is not + freed. Interned strings will all be deallocated regardless of their reference count. + Some memory allocated by extension modules may not be freed. Some extensions may not + work properly if their initialization routine is called more than once; this can + happen if an application calls :c:func:`Py_Initialize` and :c:func:`Py_FinalizeEx` + more than once. :c:func:`Py_FinalizeEx` must not be called recursively from + within itself. Therefore, it must not be called by any code that may be run + as part of the interpreter shutdown process, such as :py:mod:`atexit` + handlers, object finalizers, or any code that may be run while flushing the + stdout and stderr files. + + .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx + + .. versionadded:: 3.6 + + +.. c:function:: void Py_Finalize() + + This is a backwards-compatible version of :c:func:`Py_FinalizeEx` that + disregards the return value. + + +.. c:function:: int Py_BytesMain(int argc, char **argv) + + Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings, + allowing the calling application to delegate the text decoding step to + the CPython runtime. + + .. versionadded:: 3.8 + + +.. c:function:: int Py_Main(int argc, wchar_t **argv) + + The main program for the standard interpreter, encapsulating a full + initialization/finalization cycle, as well as additional + behaviour to implement reading configurations settings from the environment + and command line, and then executing ``__main__`` in accordance with + :ref:`using-on-cmdline`. + + This is made available for programs which wish to support the full CPython + command line interface, rather than just embedding a Python runtime in a + larger application. + + The *argc* and *argv* parameters are similar to those which are passed to a + C program's :c:func:`main` function, except that the *argv* entries are first + converted to ``wchar_t`` using :c:func:`Py_DecodeLocale`. It is also + important to note that the argument list entries may be modified to point to + strings other than those passed in (however, the contents of the strings + pointed to by the argument list are not modified). + + The return value is ``2`` if the argument list does not represent a valid + Python command line, and otherwise the same as :c:func:`Py_RunMain`. + + In terms of the CPython runtime configuration APIs documented in the + :ref:`runtime configuration ` section (and without accounting + for error handling), ``Py_Main`` is approximately equivalent to:: + + PyConfig config; + PyConfig_InitPythonConfig(&config); + PyConfig_SetArgv(&config, argc, argv); + Py_InitializeFromConfig(&config); + PyConfig_Clear(&config); + + Py_RunMain(); + + In normal usage, an embedding application will call this function + *instead* of calling :c:func:`Py_Initialize`, :c:func:`Py_InitializeEx` or + :c:func:`Py_InitializeFromConfig` directly, and all settings will be applied + as described elsewhere in this documentation. If this function is instead + called *after* a preceding runtime initialization API call, then exactly + which environmental and command line configuration settings will be updated + is version dependent (as it depends on which settings correctly support + being modified after they have already been set once when the runtime was + first initialized). + + +.. c:function:: int Py_RunMain(void) + + Executes the main module in a fully configured CPython runtime. + + Executes the command (:c:member:`PyConfig.run_command`), the script + (:c:member:`PyConfig.run_filename`) or the module + (:c:member:`PyConfig.run_module`) specified on the command line or in the + configuration. If none of these values are set, runs the interactive Python + prompt (REPL) using the ``__main__`` module's global namespace. + + If :c:member:`PyConfig.inspect` is not set (the default), the return value + will be ``0`` if the interpreter exits normally (that is, without raising + an exception), the exit status of an unhandled :exc:`SystemExit`, or ``1`` + for any other unhandled exception. + + If :c:member:`PyConfig.inspect` is set (such as when the :option:`-i` option + is used), rather than returning when the interpreter exits, execution will + instead resume in an interactive Python prompt (REPL) using the ``__main__`` + module's global namespace. If the interpreter exited with an exception, it + is immediately raised in the REPL session. The function return value is + then determined by the way the *REPL session* terminates: ``0``, ``1``, or + the status of a :exc:`SystemExit`, as specified above. + + This function always finalizes the Python interpreter before it returns. + + See :ref:`Python Configuration ` for an example of a + customized Python that always runs in isolated mode using + :c:func:`Py_RunMain`. + +.. c:function:: int PyUnstable_AtExit(PyInterpreterState *interp, void (*func)(void *), void *data) + + Register an :mod:`atexit` callback for the target interpreter *interp*. + This is similar to :c:func:`Py_AtExit`, but takes an explicit interpreter and + data pointer for the callback. + + There must be an :term:`attached thread state` for *interp*. + + .. versionadded:: 3.13 + + +.. _cautions-regarding-runtime-finalization: + +Cautions regarding runtime finalization +--------------------------------------- + +In the late stage of :term:`interpreter shutdown`, after attempting to wait for +non-daemon threads to exit (though this can be interrupted by +:class:`KeyboardInterrupt`) and running the :mod:`atexit` functions, the runtime +is marked as *finalizing*: :c:func:`Py_IsFinalizing` and +:func:`sys.is_finalizing` return true. At this point, only the *finalization +thread* that initiated finalization (typically the main thread) is allowed to +acquire the :term:`GIL`. + +If any thread, other than the finalization thread, attempts to attach a :term:`thread state` +during finalization, either explicitly or +implicitly, the thread enters **a permanently blocked state** +where it remains until the program exits. In most cases this is harmless, but this can result +in deadlock if a later stage of finalization attempts to acquire a lock owned by the +blocked thread, or otherwise waits on the blocked thread. + +Gross? Yes. This prevents random crashes and/or unexpectedly skipped C++ +finalizations further up the call stack when such threads were forcibly exited +here in CPython 3.13 and earlier. The CPython runtime :term:`thread state` C APIs +have never had any error reporting or handling expectations at :term:`thread state` +attachment time that would've allowed for graceful exit from this situation. Changing that +would require new stable C APIs and rewriting the majority of C code in the +CPython ecosystem to use those with error handling. + + +Process-wide parameters +----------------------- + +.. c:function:: void Py_SetProgramName(const wchar_t *name) + + .. index:: + single: Py_Initialize() + single: main() + single: Py_GetPath() + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.program_name` should be used instead, see :ref:`Python + Initialization Configuration `. + + This function should be called before :c:func:`Py_Initialize` is called for + the first time, if it is called at all. It tells the interpreter the value + of the ``argv[0]`` argument to the :c:func:`main` function of the program + (converted to wide characters). + This is used by :c:func:`Py_GetPath` and some other functions below to find + the Python run-time libraries relative to the interpreter executable. The + default value is ``'python'``. The argument should point to a + zero-terminated wide character string in static storage whose contents will not + change for the duration of the program's execution. No code in the Python + interpreter will change the contents of this storage. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + .. deprecated-removed:: 3.11 3.15 + + +.. c:function:: wchar_t* Py_GetProgramName() + + Return the program name set with :c:member:`PyConfig.program_name`, or the default. + The returned string points into static storage; the caller should not modify its + value. + + This function should not be called before :c:func:`Py_Initialize`, otherwise + it returns ``NULL``. + + .. versionchanged:: 3.10 + It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + + .. deprecated-removed:: 3.13 3.15 + Use :c:func:`PyConfig_Get("executable") ` + (:data:`sys.executable`) instead. + + +.. c:function:: wchar_t* Py_GetPrefix() + + Return the *prefix* for installed platform-independent files. This is derived + through a number of complicated rules from the program name set with + :c:member:`PyConfig.program_name` and some environment variables; for example, if the + program name is ``'/usr/local/bin/python'``, the prefix is ``'/usr/local'``. The + returned string points into static storage; the caller should not modify its + value. This corresponds to the :makevar:`prefix` variable in the top-level + :file:`Makefile` and the :option:`--prefix` argument to the :program:`configure` + script at build time. The value is available to Python code as ``sys.base_prefix``. + It is only useful on Unix. See also the next function. + + This function should not be called before :c:func:`Py_Initialize`, otherwise + it returns ``NULL``. + + .. versionchanged:: 3.10 + It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + + .. deprecated-removed:: 3.13 3.15 + Use :c:func:`PyConfig_Get("base_prefix") ` + (:data:`sys.base_prefix`) instead. Use :c:func:`PyConfig_Get("prefix") + ` (:data:`sys.prefix`) if :ref:`virtual environments + ` need to be handled. + + +.. c:function:: wchar_t* Py_GetExecPrefix() + + Return the *exec-prefix* for installed platform-*dependent* files. This is + derived through a number of complicated rules from the program name set with + :c:member:`PyConfig.program_name` and some environment variables; for example, if the + program name is ``'/usr/local/bin/python'``, the exec-prefix is + ``'/usr/local'``. The returned string points into static storage; the caller + should not modify its value. This corresponds to the :makevar:`exec_prefix` + variable in the top-level :file:`Makefile` and the ``--exec-prefix`` + argument to the :program:`configure` script at build time. The value is + available to Python code as ``sys.base_exec_prefix``. It is only useful on + Unix. + + Background: The exec-prefix differs from the prefix when platform dependent + files (such as executables and shared libraries) are installed in a different + directory tree. In a typical installation, platform dependent files may be + installed in the :file:`/usr/local/plat` subtree while platform independent may + be installed in :file:`/usr/local`. + + Generally speaking, a platform is a combination of hardware and software + families, e.g. Sparc machines running the Solaris 2.x operating system are + considered the same platform, but Intel machines running Solaris 2.x are another + platform, and Intel machines running Linux are yet another platform. Different + major revisions of the same operating system generally also form different + platforms. Non-Unix operating systems are a different story; the installation + strategies on those systems are so different that the prefix and exec-prefix are + meaningless, and set to the empty string. Note that compiled Python bytecode + files are platform independent (but not independent from the Python version by + which they were compiled!). + + System administrators will know how to configure the :program:`mount` or + :program:`automount` programs to share :file:`/usr/local` between platforms + while having :file:`/usr/local/plat` be a different filesystem for each + platform. + + This function should not be called before :c:func:`Py_Initialize`, otherwise + it returns ``NULL``. + + .. versionchanged:: 3.10 + It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + + .. deprecated-removed:: 3.13 3.15 + Use :c:func:`PyConfig_Get("base_exec_prefix") ` + (:data:`sys.base_exec_prefix`) instead. Use + :c:func:`PyConfig_Get("exec_prefix") ` + (:data:`sys.exec_prefix`) if :ref:`virtual environments ` need + to be handled. + + +.. c:function:: wchar_t* Py_GetProgramFullPath() + + .. index:: + single: executable (in module sys) + + Return the full program name of the Python executable; this is computed as a + side-effect of deriving the default module search path from the program name + (set by :c:member:`PyConfig.program_name`). The returned string points into + static storage; the caller should not modify its value. The value is available + to Python code as ``sys.executable``. + + This function should not be called before :c:func:`Py_Initialize`, otherwise + it returns ``NULL``. + + .. versionchanged:: 3.10 + It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + + .. deprecated-removed:: 3.13 3.15 + Use :c:func:`PyConfig_Get("executable") ` + (:data:`sys.executable`) instead. + + +.. c:function:: wchar_t* Py_GetPath() + + .. index:: + triple: module; search; path + single: path (in module sys) + + Return the default module search path; this is computed from the program name + (set by :c:member:`PyConfig.program_name`) and some environment variables. + The returned string consists of a series of directory names separated by a + platform dependent delimiter character. The delimiter character is ``':'`` + on Unix and macOS, ``';'`` on Windows. The returned string points into + static storage; the caller should not modify its value. The list + :data:`sys.path` is initialized with this value on interpreter startup; it + can be (and usually is) modified later to change the search path for loading + modules. + + This function should not be called before :c:func:`Py_Initialize`, otherwise + it returns ``NULL``. + + .. XXX should give the exact rules + + .. versionchanged:: 3.10 + It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + + .. deprecated-removed:: 3.13 3.15 + Use :c:func:`PyConfig_Get("module_search_paths") ` + (:data:`sys.path`) instead. + +.. c:function:: const char* Py_GetVersion() + + Return the version of this Python interpreter. This is a string that looks + something like :: + + "3.0a5+ (py3k:63103M, May 12 2008, 00:53:55) \n[GCC 4.2.3]" + + .. index:: single: version (in module sys) + + The first word (up to the first space character) is the current Python version; + the first characters are the major and minor version separated by a + period. The returned string points into static storage; the caller should not + modify its value. The value is available to Python code as :data:`sys.version`. + + See also the :c:var:`Py_Version` constant. + + +.. c:function:: const char* Py_GetPlatform() + + .. index:: single: platform (in module sys) + + Return the platform identifier for the current platform. On Unix, this is + formed from the "official" name of the operating system, converted to lower + case, followed by the major revision number; e.g., for Solaris 2.x, which is + also known as SunOS 5.x, the value is ``'sunos5'``. On macOS, it is + ``'darwin'``. On Windows, it is ``'win'``. The returned string points into + static storage; the caller should not modify its value. The value is available + to Python code as ``sys.platform``. + + +.. c:function:: const char* Py_GetCopyright() + + Return the official copyright string for the current Python version, for example + + ``'Copyright 1991-1995 Stichting Mathematisch Centrum, Amsterdam'`` + + .. index:: single: copyright (in module sys) + + The returned string points into static storage; the caller should not modify its + value. The value is available to Python code as ``sys.copyright``. + + +.. c:function:: const char* Py_GetCompiler() + + Return an indication of the compiler used to build the current Python version, + in square brackets, for example:: + + "[GCC 2.7.2.2]" + + .. index:: single: version (in module sys) + + The returned string points into static storage; the caller should not modify its + value. The value is available to Python code as part of the variable + ``sys.version``. + + +.. c:function:: const char* Py_GetBuildInfo() + + Return information about the sequence number and build date and time of the + current Python interpreter instance, for example :: + + "#67, Aug 1 1997, 22:34:28" + + .. index:: single: version (in module sys) + + The returned string points into static storage; the caller should not modify its + value. The value is available to Python code as part of the variable + ``sys.version``. + + +.. c:function:: void PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) + + .. index:: + single: main() + single: Py_FatalError() + single: argv (in module sys) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.argv`, :c:member:`PyConfig.parse_argv` and + :c:member:`PyConfig.safe_path` should be used instead, see :ref:`Python + Initialization Configuration `. + + Set :data:`sys.argv` based on *argc* and *argv*. These parameters are + similar to those passed to the program's :c:func:`main` function with the + difference that the first entry should refer to the script file to be + executed rather than the executable hosting the Python interpreter. If there + isn't a script that will be run, the first entry in *argv* can be an empty + string. If this function fails to initialize :data:`sys.argv`, a fatal + condition is signalled using :c:func:`Py_FatalError`. + + If *updatepath* is zero, this is all the function does. If *updatepath* + is non-zero, the function also modifies :data:`sys.path` according to the + following algorithm: + + - If the name of an existing script is passed in ``argv[0]``, the absolute + path of the directory where the script is located is prepended to + :data:`sys.path`. + - Otherwise (that is, if *argc* is ``0`` or ``argv[0]`` doesn't point + to an existing file name), an empty string is prepended to + :data:`sys.path`, which is the same as prepending the current working + directory (``"."``). + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` + members of the :ref:`Python Initialization Configuration `. + + .. note:: + It is recommended that applications embedding the Python interpreter + for purposes other than executing a single script pass ``0`` as *updatepath*, + and update :data:`sys.path` themselves if desired. + See :cve:`2008-5983`. + + On versions before 3.1.3, you can achieve the same effect by manually + popping the first :data:`sys.path` element after having called + :c:func:`PySys_SetArgv`, for example using:: + + PyRun_SimpleString("import sys; sys.path.pop(0)\n"); + + .. versionadded:: 3.1.3 + + .. deprecated-removed:: 3.11 3.15 + + +.. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.argv` and :c:member:`PyConfig.parse_argv` should be used + instead, see :ref:`Python Initialization Configuration `. + + This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set + to ``1`` unless the :program:`python` interpreter was started with the + :option:`-I`. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` + members of the :ref:`Python Initialization Configuration `. + + .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. + + .. deprecated-removed:: 3.11 3.15 + + +.. c:function:: void Py_SetPythonHome(const wchar_t *home) + + This API is kept for backward compatibility: setting + :c:member:`PyConfig.home` should be used instead, see :ref:`Python + Initialization Configuration `. + + Set the default "home" directory, that is, the location of the standard + Python libraries. See :envvar:`PYTHONHOME` for the meaning of the + argument string. + + The argument should point to a zero-terminated character string in static + storage whose contents will not change for the duration of the program's + execution. No code in the Python interpreter will change the contents of + this storage. + + Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a + :c:expr:`wchar_t*` string. + + .. deprecated-removed:: 3.11 3.15 + + +.. c:function:: wchar_t* Py_GetPythonHome() + + Return the default "home", that is, the value set by + :c:member:`PyConfig.home`, or the value of the :envvar:`PYTHONHOME` + environment variable if it is set. + + This function should not be called before :c:func:`Py_Initialize`, otherwise + it returns ``NULL``. + + .. versionchanged:: 3.10 + It now returns ``NULL`` if called before :c:func:`Py_Initialize`. + + .. deprecated-removed:: 3.13 3.15 + Use :c:func:`PyConfig_Get("home") ` or the + :envvar:`PYTHONHOME` environment variable instead. diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index c8c60eb9f48f45a..94abb9fdb50ac2d 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -111,45 +111,284 @@ Useful macros ============= Several useful macros are defined in the Python header files. Many are -defined closer to where they are useful (e.g. :c:macro:`Py_RETURN_NONE`). +defined closer to where they are useful (for example, :c:macro:`Py_RETURN_NONE`, +:c:macro:`PyMODINIT_FUNC`). Others of a more general utility are defined here. This is not necessarily a complete listing. -.. c:macro:: PyMODINIT_FUNC +.. c:macro:: Py_CAN_START_THREADS - Declare an extension module ``PyInit`` initialization function. The function - return type is :c:expr:`PyObject*`. The macro declares any special linkage - declarations required by the platform, and for C++ declares the function as - ``extern "C"``. + If this macro is defined, then the current system is able to start threads. - The initialization function must be named :samp:`PyInit_{name}`, where - *name* is the name of the module, and should be the only non-\ ``static`` - item defined in the module file. Example:: + Currently, all systems supported by CPython (per :pep:`11`), with the + exception of some WebAssembly platforms, support starting threads. - static struct PyModuleDef spam_module = { - PyModuleDef_HEAD_INIT, - .m_name = "spam", - ... - }; + .. versionadded:: 3.13 - PyMODINIT_FUNC - PyInit_spam(void) - { - return PyModule_Create(&spam_module); - } +.. c:macro:: Py_GETENV(s) + + Like :samp:`getenv({s})`, but returns ``NULL`` if :option:`-E` was passed + on the command line (see :c:member:`PyConfig.use_environment`). + + +Docstring macros +---------------- + +.. c:macro:: PyDoc_STRVAR(name, str) + + Creates a variable with name *name* that can be used in docstrings. + If Python is built without docstrings (:option:`--without-doc-strings`), + the value will be an empty string. + + Example:: + + PyDoc_STRVAR(pop_doc, "Remove and return the rightmost element."); + + static PyMethodDef deque_methods[] = { + // ... + {"pop", (PyCFunction)deque_pop, METH_NOARGS, pop_doc}, + // ... + } + + Expands to :samp:`PyDoc_VAR({name}) = PyDoc_STR({str})`. + +.. c:macro:: PyDoc_STR(str) + + Expands to the given input string, or an empty string + if docstrings are disabled (:option:`--without-doc-strings`). + + Example:: + + static PyMethodDef pysqlite_row_methods[] = { + {"keys", (PyCFunction)pysqlite_row_keys, METH_NOARGS, + PyDoc_STR("Returns the keys of the row.")}, + {NULL, NULL} + }; + +.. c:macro:: PyDoc_VAR(name) + + Declares a static character array variable with the given *name*. + Expands to :samp:`static const char {name}[]` + + For example:: + PyDoc_VAR(python_doc) = PyDoc_STR( + "A genus of constricting snakes in the Pythonidae family native " + "to the tropics and subtropics of the Eastern Hemisphere."); + + +General utility macros +---------------------- + +The following macros are for common tasks not specific to Python. + +.. c:macro:: Py_UNUSED(arg) + + Use this for unused arguments in a function definition to silence compiler + warnings. Example: ``int func(int a, int Py_UNUSED(b)) { return a; }``. + + .. versionadded:: 3.4 + +.. c:macro:: Py_GCC_ATTRIBUTE(name) + + Use a GCC attribute *name*, hiding it from compilers that don't support GCC + attributes (such as MSVC). + + This expands to :samp:`__attribute__(({name)})` on a GCC compiler, + and expands to nothing on compilers that don't support GCC attributes. + + +Numeric utilities +^^^^^^^^^^^^^^^^^ .. c:macro:: Py_ABS(x) Return the absolute value of ``x``. + The argument may be evaluated more than once. + Consequently, do not pass an expression with side-effects directly + to this macro. + + If the result cannot be represented (for example, if ``x`` has + :c:macro:`!INT_MIN` value for :c:expr:`int` type), the behavior is + undefined. + + Corresponds roughly to :samp:`(({x}) < 0 ? -({x}) : ({x}))` + + .. versionadded:: 3.3 + +.. c:macro:: Py_MAX(x, y) + Py_MIN(x, y) + + Return the larger or smaller of the arguments, respectively. + + Any arguments may be evaluated more than once. + Consequently, do not pass an expression with side-effects directly + to this macro. + + :c:macro:`!Py_MAX` corresponds roughly to + :samp:`((({x}) > ({y})) ? ({x}) : ({y}))`. + + .. versionadded:: 3.3 + +.. c:macro:: Py_ARITHMETIC_RIGHT_SHIFT(type, integer, positions) + + Similar to :samp:`{integer} >> {positions}`, but forces sign extension, + as the C standard does not define whether a right-shift of a signed + integer will perform sign extension or a zero-fill. + + *integer* should be any signed integer type. + *positions* is the number of positions to shift to the right. + + Both *integer* and *positions* can be evaluated more than once; + consequently, avoid directly passing a function call or some other + operation with side-effects to this macro. Instead, store the result as a + variable and then pass it. + + *type* is unused and only kept for backwards compatibility. Historically, + *type* was used to cast *integer*. + + .. versionchanged:: 3.1 + + This macro is now valid for all signed integer types, not just those for + which ``unsigned type`` is legal. As a result, *type* is no longer + used. + +.. c:macro:: Py_CHARMASK(c) + + Argument must be a character or an integer in the range [-128, 127] or [0, + 255]. This macro returns ``c`` cast to an ``unsigned char``. + + +Assertion utilities +^^^^^^^^^^^^^^^^^^^ + +.. c:macro:: Py_UNREACHABLE() + + Use this when you have a code path that cannot be reached by design. + For example, in the ``default:`` clause in a ``switch`` statement for which + all possible values are covered in ``case`` statements. Use this in places + where you might be tempted to put an ``assert(0)`` or ``abort()`` call. + + In release mode, the macro helps the compiler to optimize the code, and + avoids a warning about unreachable code. For example, the macro is + implemented with ``__builtin_unreachable()`` on GCC in release mode. + + In debug mode, and on unsupported compilers, the macro expands to a call to + :c:func:`Py_FatalError`. + + A use for ``Py_UNREACHABLE()`` is following a call to a function that + never returns but that is not declared ``_Noreturn``. + + If a code path is very unlikely code but can be reached under exceptional + case, this macro must not be used. For example, under low memory condition + or if a system call returns a value out of the expected range. In this + case, it's better to report the error to the caller. If the error cannot + be reported to caller, :c:func:`Py_FatalError` can be used. + + .. versionadded:: 3.7 + +.. c:macro:: Py_SAFE_DOWNCAST(value, larger, smaller) + + Cast *value* to type *smaller* from type *larger*, validating that no + information was lost. + + On release builds of Python, this is roughly equivalent to + :samp:`(({smaller}) {value})` + (in C++, :samp:`static_cast<{smaller}>({value})` will be used instead). + + On debug builds (implying that :c:macro:`Py_DEBUG` is defined), this asserts + that no information was lost with the cast from *larger* to *smaller*. + + *value*, *larger*, and *smaller* may all be evaluated more than once in the + expression; consequently, do not pass an expression with side-effects + directly to this macro. + +.. c:macro:: Py_BUILD_ASSERT(cond) + + Asserts a compile-time condition *cond*, as a statement. + The build will fail if the condition is false or cannot be evaluated at compile time. + + Corresponds roughly to :samp:`static_assert({cond})` on C23 and above. + + For example:: + + Py_BUILD_ASSERT(sizeof(PyTime_t) == sizeof(int64_t)); + .. versionadded:: 3.3 +.. c:macro:: Py_BUILD_ASSERT_EXPR(cond) + + Asserts a compile-time condition *cond*, as an expression that evaluates to ``0``. + The build will fail if the condition is false or cannot be evaluated at compile time. + + For example:: + + #define foo_to_char(foo) \ + ((char *)(foo) + Py_BUILD_ASSERT_EXPR(offsetof(struct foo, string) == 0)) + + .. versionadded:: 3.3 + + +Type size utilities +^^^^^^^^^^^^^^^^^^^ + +.. c:macro:: Py_ARRAY_LENGTH(array) + + Compute the length of a statically allocated C array at compile time. + + The *array* argument must be a C array with a size known at compile time. + Passing an array with an unknown size, such as a heap-allocated array, + will result in a compilation error on some compilers, or otherwise produce + incorrect results. + + This is roughly equivalent to:: + + sizeof(array) / sizeof((array)[0]) + +.. c:macro:: Py_MEMBER_SIZE(type, member) + + Return the size of a structure (*type*) *member* in bytes. + + Corresponds roughly to :samp:`sizeof((({type} *)NULL)->{member})`. + + .. versionadded:: 3.6 + + +Macro definition utilities +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. c:macro:: Py_FORCE_EXPANSION(X) + + This is equivalent to :samp:`{X}`, which is useful for token-pasting in + macros, as macro expansions in *X* are forcefully evaluated by the + preprocessor. + +.. c:macro:: Py_STRINGIFY(x) + + Convert ``x`` to a C string. For example, ``Py_STRINGIFY(123)`` returns + ``"123"``. + + .. versionadded:: 3.4 + + +Declaration utilities +--------------------- + +The following macros can be used in declarations. +They are most useful for defining the C API itself, and have limited use +for extension authors. +Most of them expand to compiler-specific spellings of common extensions +to the C language. + .. c:macro:: Py_ALWAYS_INLINE Ask the compiler to always inline a static inline function. The compiler can ignore it and decide to not inline the function. + Corresponds to ``always_inline`` attribute in GCC and ``__forceinline`` + in MSVC. + It can be used to inline performance critical static inline functions when building Python in debug mode with function inlining disabled. For example, MSC disables function inlining when building in debug mode. @@ -167,15 +406,24 @@ complete listing. .. versionadded:: 3.11 -.. c:macro:: Py_CHARMASK(c) +.. c:macro:: Py_NO_INLINE - Argument must be a character or an integer in the range [-128, 127] or [0, - 255]. This macro returns ``c`` cast to an ``unsigned char``. + Disable inlining on a function. For example, it reduces the C stack + consumption: useful on LTO+PGO builds which heavily inline code (see + :issue:`33720`). + + Corresponds to the ``noinline`` attribute/specification on GCC and MSVC. + + Usage:: + + Py_NO_INLINE static int random(void) { return 4; } + + .. versionadded:: 3.11 .. c:macro:: Py_DEPRECATED(version) - Use this for deprecated declarations. The macro must be placed before the - symbol name. + Use this to declare APIs that were deprecated in a specific CPython version. + The macro must be placed before the symbol name. Example:: @@ -184,110 +432,97 @@ complete listing. .. versionchanged:: 3.8 MSVC support was added. -.. c:macro:: Py_GETENV(s) +.. c:macro:: Py_LOCAL(type) - Like ``getenv(s)``, but returns ``NULL`` if :option:`-E` was passed on the - command line (see :c:member:`PyConfig.use_environment`). + Declare a function returning the specified *type* using a fast-calling + qualifier for functions that are local to the current file. + Semantically, this is equivalent to :samp:`static {type}`. -.. c:macro:: Py_MAX(x, y) +.. c:macro:: Py_LOCAL_INLINE(type) - Return the maximum value between ``x`` and ``y``. + Equivalent to :c:macro:`Py_LOCAL` but additionally requests the function + be inlined. - .. versionadded:: 3.3 +.. c:macro:: Py_LOCAL_SYMBOL -.. c:macro:: Py_MEMBER_SIZE(type, member) + Macro used to declare a symbol as local to the shared library (hidden). + On supported platforms, it ensures the symbol is not exported. - Return the size of a structure (``type``) ``member`` in bytes. + On compatible versions of GCC/Clang, it + expands to ``__attribute__((visibility("hidden")))``. - .. versionadded:: 3.6 +.. c:macro:: Py_EXPORTED_SYMBOL -.. c:macro:: Py_MIN(x, y) + Macro used to declare a symbol (function or data) as exported. + On Windows, this expands to ``__declspec(dllexport)``. + On compatible versions of GCC/Clang, it + expands to ``__attribute__((visibility("default")))``. + This macro is for defining the C API itself; extension modules should not use it. - Return the minimum value between ``x`` and ``y``. - .. versionadded:: 3.3 +.. c:macro:: Py_IMPORTED_SYMBOL -.. c:macro:: Py_NO_INLINE + Macro used to declare a symbol as imported. + On Windows, this expands to ``__declspec(dllimport)``. + This macro is for defining the C API itself; extension modules should not use it. - Disable inlining on a function. For example, it reduces the C stack - consumption: useful on LTO+PGO builds which heavily inline code (see - :issue:`33720`). - Usage:: +.. c:macro:: PyAPI_FUNC(type) - Py_NO_INLINE static int random(void) { return 4; } + Macro used by CPython to declare a function as part of the C API. + Its expansion depends on the platform and build configuration. + This macro is intended for defining CPython's C API itself; + extension modules should not use it for their own symbols. - .. versionadded:: 3.11 -.. c:macro:: Py_STRINGIFY(x) +.. c:macro:: PyAPI_DATA(type) - Convert ``x`` to a C string. E.g. ``Py_STRINGIFY(123)`` returns - ``"123"``. + Macro used by CPython to declare a public global variable as part of the C API. + Its expansion depends on the platform and build configuration. + This macro is intended for defining CPython's C API itself; + extension modules should not use it for their own symbols. - .. versionadded:: 3.4 -.. c:macro:: Py_UNREACHABLE() +Outdated macros +--------------- - Use this when you have a code path that cannot be reached by design. - For example, in the ``default:`` clause in a ``switch`` statement for which - all possible values are covered in ``case`` statements. Use this in places - where you might be tempted to put an ``assert(0)`` or ``abort()`` call. +The following macros have been used to features that have been standardized +in C11. - In release mode, the macro helps the compiler to optimize the code, and - avoids a warning about unreachable code. For example, the macro is - implemented with ``__builtin_unreachable()`` on GCC in release mode. +.. c:macro:: Py_ALIGNED(num) - A use for ``Py_UNREACHABLE()`` is following a call a function that - never returns but that is not declared :c:macro:`_Py_NO_RETURN`. + Specify alignment to *num* bytes on compilers that support it. - If a code path is very unlikely code but can be reached under exceptional - case, this macro must not be used. For example, under low memory condition - or if a system call returns a value out of the expected range. In this - case, it's better to report the error to the caller. If the error cannot - be reported to caller, :c:func:`Py_FatalError` can be used. + Consider using the C11 standard ``_Alignas`` specifier over this macro. - .. versionadded:: 3.7 +.. c:macro:: Py_LL(number) + Py_ULL(number) -.. c:macro:: Py_UNUSED(arg) + Use *number* as a ``long long`` or ``unsigned long long`` integer literal, + respectively. - Use this for unused arguments in a function definition to silence compiler - warnings. Example: ``int func(int a, int Py_UNUSED(b)) { return a; }``. + Expands to *number* followed by ``LL`` or ``LLU``, respectively, but will + expand to some compiler-specific suffixes on some older compilers. - .. versionadded:: 3.4 + Consider using the C99 standard suffixes ``LL`` and ``LLU`` directly. -.. c:macro:: PyDoc_STRVAR(name, str) +.. c:macro:: Py_MEMCPY(dest, src, n) - Creates a variable with name ``name`` that can be used in docstrings. - If Python is built without docstrings, the value will be empty. + This is an alias to :c:func:`!memcpy`. - Use :c:macro:`PyDoc_STRVAR` for docstrings to support building - Python without docstrings, as specified in :pep:`7`. + .. soft-deprecated:: 3.14 + Use :c:func:`!memcpy` directly instead. - Example:: +.. c:macro:: Py_VA_COPY - PyDoc_STRVAR(pop_doc, "Remove and return the rightmost element."); + This is an alias to the C99-standard ``va_copy`` function. - static PyMethodDef deque_methods[] = { - // ... - {"pop", (PyCFunction)deque_pop, METH_NOARGS, pop_doc}, - // ... - } + Historically, this would use a compiler-specific method to copy a ``va_list``. -.. c:macro:: PyDoc_STR(str) + .. versionchanged:: 3.6 + This is now an alias to ``va_copy``. - Creates a docstring for the given input string or an empty string - if docstrings are disabled. - - Use :c:macro:`PyDoc_STR` in specifying docstrings to support - building Python without docstrings, as specified in :pep:`7`. - - Example:: - - static PyMethodDef pysqlite_row_methods[] = { - {"keys", (PyCFunction)pysqlite_row_keys, METH_NOARGS, - PyDoc_STR("Returns the keys of the row.")}, - {NULL, NULL} - }; + .. soft-deprecated:: 3.14 .. _api-objects: @@ -406,9 +641,12 @@ the caller is said to *borrow* the reference. Nothing needs to be done for a Conversely, when a calling function passes in a reference to an object, there are two possibilities: the function *steals* a reference to the object, or it -does not. *Stealing a reference* means that when you pass a reference to a -function, that function assumes that it now owns that reference, and you are not -responsible for it any longer. +does not. + +*Stealing a reference* means that when you pass a reference to a +function, that function assumes that it now owns that reference. +Since the new owner can use :c:func:`!Py_DECREF` at its discretion, +you (the caller) must not use that reference after the call. .. index:: single: PyList_SetItem (C function) @@ -844,3 +1082,41 @@ after every statement run by the interpreter.) Please refer to :file:`Misc/SpecialBuilds.txt` in the Python source distribution for more detailed information. + + +.. _c-api-tools: + +Recommended third party tools +============================= + +The following third party tools offer both simpler and more sophisticated +approaches to creating C, C++ and Rust extensions for Python: + +* `Cython `_ +* `cffi `_ +* `HPy `_ +* `nanobind `_ (C++) +* `Numba `_ +* `pybind11 `_ (C++) +* `PyO3 `_ (Rust) +* `SWIG `_ + +Using tools such as these can help avoid writing code that is tightly bound to +a particular version of CPython, avoid reference counting errors, and focus +more on your own code than on using the CPython API. In general, new versions +of Python can be supported by updating the tool, and your code will often use +newer and more efficient APIs automatically. Some tools also support compiling +for other implementations of Python from a single set of sources. + +These projects are not supported by the same people who maintain Python, and +issues need to be raised with the projects directly. Remember to check that the +project is still maintained and supported, as the list above may become +outdated. + +.. seealso:: + + `Python Packaging User Guide: Binary Extensions `_ + The Python Packaging User Guide not only covers several available + tools that simplify the creation of binary extensions, but also + discusses the various reasons why creating an extension module may be + desirable in the first place. diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst index 6b7ba8c99791634..bfbfe3c92799809 100644 --- a/Doc/c-api/iterator.rst +++ b/Doc/c-api/iterator.rst @@ -50,3 +50,72 @@ sentinel value is returned. callable object that can be called with no parameters; each call to it should return the next item in the iteration. When *callable* returns a value equal to *sentinel*, the iteration will be terminated. + + +Range Objects +^^^^^^^^^^^^^ + +.. c:var:: PyTypeObject PyRange_Type + + The type object for :class:`range` objects. + + +.. c:function:: int PyRange_Check(PyObject *o) + + Return true if the object *o* is an instance of a :class:`range` object. + This function always succeeds. + + +Builtin Iterator Types +^^^^^^^^^^^^^^^^^^^^^^ + +These are built-in iteration types that are included in Python's C API, but +provide no additional functions. They are here for completeness. + + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * C type + * Python type + * * .. c:var:: PyTypeObject PyEnum_Type + * :py:class:`enumerate` + * * .. c:var:: PyTypeObject PyFilter_Type + * :py:class:`filter` + * * .. c:var:: PyTypeObject PyMap_Type + * :py:class:`map` + * * .. c:var:: PyTypeObject PyReversed_Type + * :py:class:`reversed` + * * .. c:var:: PyTypeObject PyZip_Type + * :py:class:`zip` + + +Other Iterator Objects +^^^^^^^^^^^^^^^^^^^^^^ + +.. c:var:: PyTypeObject PyByteArrayIter_Type +.. c:var:: PyTypeObject PyBytesIter_Type +.. c:var:: PyTypeObject PyListIter_Type +.. c:var:: PyTypeObject PyListRevIter_Type +.. c:var:: PyTypeObject PySetIter_Type +.. c:var:: PyTypeObject PyTupleIter_Type +.. c:var:: PyTypeObject PyRangeIter_Type +.. c:var:: PyTypeObject PyLongRangeIter_Type +.. c:var:: PyTypeObject PyDictIterKey_Type +.. c:var:: PyTypeObject PyDictRevIterKey_Type +.. c:var:: PyTypeObject PyDictIterValue_Type +.. c:var:: PyTypeObject PyDictRevIterValue_Type +.. c:var:: PyTypeObject PyDictIterItem_Type +.. c:var:: PyTypeObject PyDictRevIterItem_Type +.. c:var:: PyTypeObject PyODictIter_Type + + Type objects for iterators of various built-in objects. + + Do not create instances of these directly; prefer calling + :c:func:`PyObject_GetIter` instead. + + Note that there is no guarantee that a given built-in type uses a given iterator + type. For example, iterating over :class:`range` will use one of two iterator + types depending on the size of the range. Other types may start using a + similar scheme in the future, without warning. diff --git a/Doc/c-api/lifecycle.dot b/Doc/c-api/lifecycle.dot new file mode 100644 index 000000000000000..dca9f87e9e0acaa --- /dev/null +++ b/Doc/c-api/lifecycle.dot @@ -0,0 +1,156 @@ +digraph "Life Events" { + graph [ + fontnames="svg" + fontsize=12.0 + id="life_events_graph" + layout="dot" + margin="0,0" + ranksep=0.25 + stylesheet="lifecycle.dot.css" + ] + node [ + fontname="Courier" + fontsize=12.0 + ] + edge [ + fontname="Times-Italic" + fontsize=12.0 + ] + + "start" [fontname="Times-Italic" shape=plain label=< start > style=invis] + { + rank="same" + "tp_new" [href="typeobj.html#c.PyTypeObject.tp_new" target="_top"] + "tp_alloc" [href="typeobj.html#c.PyTypeObject.tp_alloc" target="_top"] + } + "tp_init" [href="typeobj.html#c.PyTypeObject.tp_init" target="_top"] + "reachable" [fontname="Times-Italic" shape=box] + "tp_traverse" [ + href="typeobj.html#c.PyTypeObject.tp_traverse" + ordering="in" + target="_top" + ] + "finalized?" [ + fontname="Times-Italic" + label=finalized?> + ordering="in" + shape=diamond + tooltip="marked as finalized?" + ] + "tp_finalize" [ + href="typeobj.html#c.PyTypeObject.tp_finalize" + ordering="in" + target="_top" + ] + "tp_clear" [href="typeobj.html#c.PyTypeObject.tp_clear" target="_top"] + "uncollectable" [ + fontname="Times-Italic" + label=(leaked)> + shape=box + tooltip="uncollectable (leaked)" + ] + "tp_dealloc" [ + href="typeobj.html#c.PyTypeObject.tp_dealloc" + ordering="in" + target="_top" + ] + "tp_free" [href="typeobj.html#c.PyTypeObject.tp_free" target="_top"] + + "start" -> "tp_new" [ + label=< type call > + ] + "tp_new" -> "tp_alloc" [ + label=< direct call > arrowhead=empty + labeltooltip="tp_new to tp_alloc: direct call" + tooltip="tp_new to tp_alloc: direct call" + ] + "tp_new" -> "tp_init" [tooltip="tp_new to tp_init"] + "tp_init" -> "reachable" [tooltip="tp_init to reachable"] + "reachable" -> "tp_traverse" [ + dir="back" + label=< not in a
cyclic
isolate > + labeltooltip="tp_traverse to reachable: not in a cyclic isolate" + tooltip="tp_traverse to reachable: not in a cyclic isolate" + ] + "reachable" -> "tp_traverse" [ + label=< periodic
cyclic isolate
detection > + labeltooltip="reachable to tp_traverse: periodic cyclic isolate detection" + tooltip="reachable to tp_traverse: periodic cyclic isolate detection" + ] + "reachable" -> "tp_init" [tooltip="reachable to tp_init"] + "reachable" -> "tp_finalize" [ + dir="back" + label=< resurrected
(maybe remove
finalized mark) > + labeltooltip="tp_finalize to reachable: resurrected (maybe remove finalized mark)" + tooltip="tp_finalize to reachable: resurrected (maybe remove finalized mark)" + ] + "tp_traverse" -> "finalized?" [ + label=< cyclic
isolate > + labeltooltip="tp_traverse to finalized?: cyclic isolate" + tooltip="tp_traverse to finalized?: cyclic isolate" + ] + "reachable" -> "finalized?" [ + label=< no refs > + labeltooltip="reachable to finalized?: no refs" + tooltip="reachable to finalized?: no refs" + ] + "finalized?" -> "tp_finalize" [ + label=< no (mark
as finalized) > + labeltooltip="finalized? to tp_finalize: no (mark as finalized)" + tooltip="finalized? to tp_finalize: no (mark as finalized)" + ] + "finalized?" -> "tp_clear" [ + label=< yes > + labeltooltip="finalized? to tp_clear: yes" + tooltip="finalized? to tp_clear: yes" + ] + "tp_finalize" -> "tp_clear" [ + label=< no refs or
cyclic isolate > + labeltooltip="tp_finalize to tp_clear: no refs or cyclic isolate" + tooltip="tp_finalize to tp_clear: no refs or cyclic isolate" + ] + "tp_finalize" -> "tp_dealloc" [ + arrowtail=empty + dir="back" + href="lifecycle.html#c.PyObject_CallFinalizerFromDealloc" + style=dashed + label=< recommended
call (see
explanation)> + labeltooltip="tp_dealloc to tp_finalize: recommended call (see explanation)" + target="_top" + tooltip="tp_dealloc to tp_finalize: recommended call (see explanation)" + ] + "tp_finalize" -> "tp_dealloc" [ + label=< no refs > + labeltooltip="tp_finalize to tp_dealloc: no refs" + tooltip="tp_finalize to tp_dealloc: no refs" + ] + "tp_clear" -> "tp_dealloc" [ + label=< no refs > + labeltooltip="tp_clear to tp_dealloc: no refs" + tooltip="tp_clear to tp_dealloc: no refs" + ] + "tp_clear" -> "uncollectable" [ + label=< cyclic
isolate > + labeltooltip="tp_clear to uncollectable: cyclic isolate" + tooltip="tp_clear to uncollectable: cyclic isolate" + ] + "uncollectable" -> "tp_dealloc" [ + style=invis + tooltip="uncollectable to tp_dealloc" + ] + "reachable" -> "uncollectable" [ + label=< cyclic
isolate
(no GC
support) > + labeltooltip="reachable to uncollectable: cyclic isolate (no GC support)" + tooltip="reachable to uncollectable: cyclic isolate (no GC support)" + ] + "reachable" -> "tp_dealloc" [ + label=< no refs> + labeltooltip="reachable to tp_dealloc: no refs" + ] + "tp_dealloc" -> "tp_free" [ + arrowhead=empty + label=< direct call > + labeltooltip="tp_dealloc to tp_free: direct call" + tooltip="tp_dealloc to tp_free: direct call" + ] +} diff --git a/Doc/c-api/lifecycle.dot.css b/Doc/c-api/lifecycle.dot.css new file mode 100644 index 000000000000000..3abf95b74da6ba8 --- /dev/null +++ b/Doc/c-api/lifecycle.dot.css @@ -0,0 +1,21 @@ +#life_events_graph { + --svg-fgcolor: currentcolor; + --svg-bgcolor: transparent; +} +#life_events_graph a { + color: inherit; +} +#life_events_graph [stroke="black"] { + stroke: var(--svg-fgcolor); +} +#life_events_graph text, +#life_events_graph [fill="black"] { + fill: var(--svg-fgcolor); +} +#life_events_graph [fill="white"] { + fill: var(--svg-bgcolor); +} +#life_events_graph [fill="none"] { + /* On links, setting fill will make the entire shape clickable */ + fill: var(--svg-bgcolor); +} diff --git a/Doc/c-api/lifecycle.dot.svg b/Doc/c-api/lifecycle.dot.svg new file mode 100644 index 000000000000000..7ace27dfcba113f --- /dev/null +++ b/Doc/c-api/lifecycle.dot.svg @@ -0,0 +1,374 @@ + + + + + + + +Life Events + + + + +tp_new + + +tp_new + + + + + +start->tp_new + + + + + + +    type call   + + + + + +tp_alloc + + +tp_alloc + + + + + +tp_new->tp_alloc + + + + + + +  direct call   + + + + + +tp_init + + +tp_init + + + + + +tp_new->tp_init + + + + + + + + +reachable + +reachable + + + +tp_init->reachable + + + + + + + + +reachable->tp_init + + + + + + + + +tp_traverse + + +tp_traverse + + + + + +reachable->tp_traverse + + + + + + +  not in a   +  cyclic   +  isolate   + + + + + +reachable->tp_traverse + + + + + + +  periodic   +  cyclic isolate    +  detection   + + + + + +finalized? + + +marked as +finalized? + + + + + +reachable->finalized? + + + + + + +  no refs   + + + + + +tp_finalize + + +tp_finalize + + + + + +reachable->tp_finalize + + + + + + +  resurrected   +  (maybe remove   +  finalized mark)   + + + + + +uncollectable + + +uncollectable +(leaked) + + + + + +reachable->uncollectable + + + + + + +  cyclic   +  isolate   +  (no GC   +  support)   + + + + + +tp_dealloc + + +tp_dealloc + + + + + +reachable->tp_dealloc + + + +  no refs + + + + + +tp_traverse->finalized? + + + + + + +  cyclic   +  isolate   + + + + + +finalized?->tp_finalize + + + + + + +  no (mark   +  as finalized)   + + + + + +tp_clear + + +tp_clear + + + + + +finalized?->tp_clear + + + + + + +  yes   + + + + + +tp_finalize->tp_clear + + + + + + +  no refs or    +  cyclic isolate   + + + + + +tp_finalize->tp_dealloc + + + + + + +  recommended +  call (see +  explanation) + + + + + +tp_finalize->tp_dealloc + + + + + + +   no refs   + + + + + +tp_clear->uncollectable + + + + + + +  cyclic   +  isolate   + + + + + +tp_clear->tp_dealloc + + + + + + +  no refs   + + + + + + +tp_free + + +tp_free + + + + + +tp_dealloc->tp_free + + + + + + +    direct call   + + + + + diff --git a/Doc/c-api/lifecycle.rst b/Doc/c-api/lifecycle.rst new file mode 100644 index 000000000000000..5e3dca29f2ad901 --- /dev/null +++ b/Doc/c-api/lifecycle.rst @@ -0,0 +1,268 @@ +.. highlight:: c + +.. _life-cycle: + +Object Life Cycle +================= + +This section explains how a type's slots relate to each other throughout the +life of an object. It is not intended to be a complete canonical reference for +the slots; instead, refer to the slot-specific documentation in +:ref:`type-structs` for details about a particular slot. + + +Life Events +----------- + +The figure below illustrates the order of events that can occur throughout an +object's life. An arrow from *A* to *B* indicates that event *B* can occur +after event *A* has occurred, with the arrow's label indicating the condition +that must be true for *B* to occur after *A*. + +.. only:: builder_html + + .. raw:: html + + + + .. raw:: html + :file: lifecycle.dot.svg + + .. raw:: html + + + +.. only:: not builder_html + + .. image:: lifecycle.dot.svg + :align: center + :class: invert-in-dark-mode + :alt: Diagram showing events in an object's life. Explained in detail below. + +.. container:: + :name: life-events-graph-description + + Explanation: + + * When a new object is constructed by calling its type: + + #. :c:member:`~PyTypeObject.tp_new` is called to create a new object. + #. :c:member:`~PyTypeObject.tp_alloc` is directly called by + :c:member:`~PyTypeObject.tp_new` to allocate the memory for the new + object. + #. :c:member:`~PyTypeObject.tp_init` initializes the newly created object. + :c:member:`!tp_init` can be called again to re-initialize an object, if + desired. The :c:member:`!tp_init` call can also be skipped entirely, + for example by Python code calling :py:meth:`~object.__new__`. + + * After :c:member:`!tp_init` completes, the object is ready to use. + * Some time after the last reference to an object is removed: + + #. If an object is not marked as *finalized*, it might be finalized by + marking it as *finalized* and calling its + :c:member:`~PyTypeObject.tp_finalize` function. Python does + *not* finalize an object when the last reference to it is deleted; use + :c:func:`PyObject_CallFinalizerFromDealloc` to ensure that + :c:member:`~PyTypeObject.tp_finalize` is always called. + #. If the object is marked as finalized, + :c:member:`~PyTypeObject.tp_clear` might be called by the garbage collector + to clear references held by the object. It is *not* called when the + object's reference count reaches zero. + #. :c:member:`~PyTypeObject.tp_dealloc` is called to destroy the object. + To avoid code duplication, :c:member:`~PyTypeObject.tp_dealloc` typically + calls into :c:member:`~PyTypeObject.tp_clear` to free up the object's + references. + #. When :c:member:`~PyTypeObject.tp_dealloc` finishes object destruction, + it directly calls :c:member:`~PyTypeObject.tp_free` (usually set to + :c:func:`PyObject_Free` or :c:func:`PyObject_GC_Del` automatically as + appropriate for the type) to deallocate the memory. + + * The :c:member:`~PyTypeObject.tp_finalize` function is permitted to add a + reference to the object if desired. If it does, the object is + *resurrected*, preventing its pending destruction. (Only + :c:member:`!tp_finalize` is allowed to resurrect an object; + :c:member:`~PyTypeObject.tp_clear` and + :c:member:`~PyTypeObject.tp_dealloc` cannot without calling into + :c:member:`!tp_finalize`.) Resurrecting an object may + or may not cause the object's *finalized* mark to be removed. Currently, + Python does not remove the *finalized* mark from a resurrected object if + it supports garbage collection (i.e., the :c:macro:`Py_TPFLAGS_HAVE_GC` + flag is set) but does remove the mark if the object does not support + garbage collection; either or both of these behaviors may change in the + future. + * :c:member:`~PyTypeObject.tp_dealloc` can optionally call + :c:member:`~PyTypeObject.tp_finalize` via + :c:func:`PyObject_CallFinalizerFromDealloc` if it wishes to reuse that + code to help with object destruction. This is recommended because it + guarantees that :c:member:`!tp_finalize` is always called before + destruction. See the :c:member:`~PyTypeObject.tp_dealloc` documentation + for example code. + * If the object is a member of a :term:`cyclic isolate` and either + :c:member:`~PyTypeObject.tp_clear` fails to break the reference cycle or + the cyclic isolate is not detected (perhaps :func:`gc.disable` was called, + or the :c:macro:`Py_TPFLAGS_HAVE_GC` flag was erroneously omitted in one + of the involved types), the objects remain indefinitely uncollectable + (they "leak"). See :data:`gc.garbage`. + + If the object is marked as supporting garbage collection (the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set in + :c:member:`~PyTypeObject.tp_flags`), the following events are also possible: + + * The garbage collector occasionally calls + :c:member:`~PyTypeObject.tp_traverse` to identify :term:`cyclic isolates + `. + * When the garbage collector discovers a :term:`cyclic isolate`, it + finalizes one of the objects in the group by marking it as *finalized* and + calling its :c:member:`~PyTypeObject.tp_finalize` function, if it has one. + This repeats until the cyclic isolate doesn't exist or all of the objects + have been finalized. + * :c:member:`~PyTypeObject.tp_finalize` is permitted to resurrect the object + by adding a reference from outside the :term:`cyclic isolate`. The new + reference causes the group of objects to no longer form a cyclic isolate + (the reference cycle may still exist, but if it does the objects are no + longer isolated). + * When the garbage collector discovers a :term:`cyclic isolate` and all of + the objects in the group have already been marked as *finalized*, the + garbage collector clears one or more of the uncleared objects in the group + (possibly concurrently) by calling each's + :c:member:`~PyTypeObject.tp_clear` function. This repeats as long as the + cyclic isolate still exists and not all of the objects have been cleared. + + +Cyclic Isolate Destruction +-------------------------- + +Listed below are the stages of life of a hypothetical :term:`cyclic isolate` +that continues to exist after each member object is finalized or cleared. It +is a memory leak if a cyclic isolate progresses through all of these stages; it should +vanish once all objects are cleared, if not sooner. A cyclic isolate can +vanish either because the reference cycle is broken or because the objects are +no longer isolated due to finalizer resurrection (see +:c:member:`~PyTypeObject.tp_finalize`). + +0. **Reachable** (not yet a cyclic isolate): All objects are in their normal, + reachable state. A reference cycle could exist, but an external reference + means the objects are not yet isolated. +#. **Unreachable but consistent:** The final reference from outside the cyclic + group of objects has been removed, causing the objects to become isolated + (thus a cyclic isolate is born). None of the group's objects have been + finalized or cleared yet. The cyclic isolate remains at this stage until + some future run of the garbage collector (not necessarily the next run + because the next run might not scan every object). +#. **Mix of finalized and not finalized:** Objects in a cyclic isolate are + finalized one at a time, which means that there is a period of time when the + cyclic isolate is composed of a mix of finalized and non-finalized objects. + Finalization order is unspecified, so it can appear random. A finalized + object must behave in a sane manner when non-finalized objects interact with + it, and a non-finalized object must be able to tolerate the finalization of + an arbitrary subset of its referents. +#. **All finalized:** All objects in a cyclic isolate are finalized before any + of them are cleared. +#. **Mix of finalized and cleared:** The objects can be cleared serially or + concurrently (but with the :term:`GIL` held); either way, some will finish + before others. A finalized object must be able to tolerate the clearing of + a subset of its referents. :pep:`442` calls this stage "cyclic trash". +#. **Leaked:** If a cyclic isolate still exists after all objects in the group + have been finalized and cleared, then the objects remain indefinitely + uncollectable (see :data:`gc.garbage`). It is a bug if a cyclic isolate + reaches this stage---it means the :c:member:`~PyTypeObject.tp_clear` methods + of the participating objects have failed to break the reference cycle as + required. + +If :c:member:`~PyTypeObject.tp_clear` did not exist, then Python would have no +way to safely break a reference cycle. Simply destroying an object in a cyclic +isolate would result in a dangling pointer, triggering undefined behavior when +an object referencing the destroyed object is itself destroyed. The clearing +step makes object destruction a two-phase process: first +:c:member:`~PyTypeObject.tp_clear` is called to partially destroy the objects +enough to detangle them from each other, then +:c:member:`~PyTypeObject.tp_dealloc` is called to complete the destruction. + +Unlike clearing, finalization is not a phase of destruction. A finalized +object must still behave properly by continuing to fulfill its design +contracts. An object's finalizer is allowed to execute arbitrary Python code, +and is even allowed to prevent the impending destruction by adding a reference. +The finalizer is only related to destruction by call order---if it runs, it runs +before destruction, which starts with :c:member:`~PyTypeObject.tp_clear` (if +called) and concludes with :c:member:`~PyTypeObject.tp_dealloc`. + +The finalization step is not necessary to safely reclaim the objects in a +cyclic isolate, but its existence makes it easier to design types that behave +in a sane manner when objects are cleared. Clearing an object might +necessarily leave it in a broken, partially destroyed state---it might be +unsafe to call any of the cleared object's methods or access any of its +attributes. With finalization, only finalized objects can possibly interact +with cleared objects; non-finalized objects are guaranteed to interact with +only non-cleared (but potentially finalized) objects. + +To summarize the possible interactions: + +* A non-finalized object might have references to or from non-finalized and + finalized objects, but not to or from cleared objects. +* A finalized object might have references to or from non-finalized, finalized, + and cleared objects. +* A cleared object might have references to or from finalized and cleared + objects, but not to or from non-finalized objects. + +Without any reference cycles, an object can be simply destroyed once its last +reference is deleted; the finalization and clearing steps are not necessary to +safely reclaim unused objects. However, it can be useful to automatically call +:c:member:`~PyTypeObject.tp_finalize` and :c:member:`~PyTypeObject.tp_clear` +before destruction anyway because type design is simplified when all objects +always experience the same series of events regardless of whether they +participated in a cyclic isolate. Python currently only calls +:c:member:`~PyTypeObject.tp_finalize` and :c:member:`~PyTypeObject.tp_clear` as +needed to destroy a cyclic isolate; this may change in a future version. + + +Functions +--------- + +To allocate and free memory, see :ref:`allocating-objects`. + + +.. c:function:: void PyObject_CallFinalizer(PyObject *op) + + Finalizes the object as described in :c:member:`~PyTypeObject.tp_finalize`. + Call this function (or :c:func:`PyObject_CallFinalizerFromDealloc`) instead + of calling :c:member:`~PyTypeObject.tp_finalize` directly because this + function may deduplicate multiple calls to :c:member:`!tp_finalize`. + Currently, calls are only deduplicated if the type supports garbage + collection (i.e., the :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set); this may + change in the future. + + .. versionadded:: 3.4 + + +.. c:function:: int PyObject_CallFinalizerFromDealloc(PyObject *op) + + Same as :c:func:`PyObject_CallFinalizer` but meant to be called at the + beginning of the object's destructor (:c:member:`~PyTypeObject.tp_dealloc`). + There must not be any references to the object. If the object's finalizer + resurrects the object, this function returns -1; no further destruction + should happen. Otherwise, this function returns 0 and destruction can + continue normally. + + .. versionadded:: 3.4 + + .. seealso:: + + :c:member:`~PyTypeObject.tp_dealloc` for example code. diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index 758415a76e5cb41..21769c4c5c869c8 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -74,11 +74,25 @@ List Objects Like :c:func:`PyList_GetItemRef`, but returns a :term:`borrowed reference` instead of a :term:`strong reference`. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the list concurrently. Prefer :c:func:`PyList_GetItemRef`, which returns + a :term:`strong reference`. + .. c:function:: PyObject* PyList_GET_ITEM(PyObject *list, Py_ssize_t i) Similar to :c:func:`PyList_GetItem`, but without error checking. + .. note:: + + In the :term:`free-threaded build`, the returned + :term:`borrowed reference` may become invalid if another thread modifies + the list concurrently. Prefer :c:func:`PyList_GetItemRef`, which returns + a :term:`strong reference`. + .. c:function:: int PyList_SetItem(PyObject *list, Py_ssize_t index, PyObject *item) @@ -88,8 +102,10 @@ List Objects .. note:: - This function "steals" a reference to *item* and discards a reference to - an item already in the list at the affected position. + This function ":term:`steals `" a reference to *item*, + even on error. + On success, it discards a reference to an item already in the list + at the affected position (unless it was ``NULL``). .. c:function:: void PyList_SET_ITEM(PyObject *list, Py_ssize_t i, PyObject *o) @@ -103,11 +119,19 @@ List Objects .. note:: - This macro "steals" a reference to *item*, and, unlike + This macro ":term:`steals `" a reference to *item*, and, unlike :c:func:`PyList_SetItem`, does *not* discard a reference to any item that is being replaced; any reference in *list* at position *i* will be leaked. + .. note:: + + In the :term:`free-threaded build`, this macro has no internal + synchronization. It is normally only used to fill in new lists where no + other thread has a reference to the list. If the list may be shared, + use :c:func:`PyList_SetItem` instead, which uses a :term:`per-object + lock`. + .. c:function:: int PyList_Insert(PyObject *list, Py_ssize_t index, PyObject *item) @@ -138,6 +162,12 @@ List Objects Return ``0`` on success, ``-1`` on failure. Indexing from the end of the list is not supported. + .. note:: + + In the :term:`free-threaded build`, when *itemlist* is a :class:`list`, + both *list* and *itemlist* are locked for the duration of the operation. + For other iterables (or ``NULL``), only *list* is locked. + .. c:function:: int PyList_Extend(PyObject *list, PyObject *iterable) @@ -150,6 +180,14 @@ List Objects .. versionadded:: 3.13 + .. note:: + + In the :term:`free-threaded build`, when *iterable* is a :class:`list`, + :class:`set`, :class:`dict`, or dict view, both *list* and *iterable* + (or its underlying dict) are locked for the duration of the operation. + For other iterables, only *list* is locked; *iterable* may be + concurrently modified by another thread. + .. c:function:: int PyList_Clear(PyObject *list) @@ -168,6 +206,14 @@ List Objects Sort the items of *list* in place. Return ``0`` on success, ``-1`` on failure. This is equivalent to ``list.sort()``. + .. note:: + + In the :term:`free-threaded build`, element comparison via + :meth:`~object.__lt__` can execute arbitrary Python code, during which + the :term:`per-object lock` may be temporarily released. For built-in + types (:class:`str`, :class:`int`, :class:`float`), the lock is not + released during comparison. + .. c:function:: int PyList_Reverse(PyObject *list) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 25d9e62e3872797..f026fd6561d8dad 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -40,9 +40,11 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Return a new :c:type:`PyLongObject` object from *v*, or ``NULL`` on failure. - The current implementation keeps an array of integer objects for all integers - between ``-5`` and ``256``. When you create an int in that range you actually - just get back a reference to the existing object. + .. impl-detail:: + + CPython keeps an array of integer objects for all integers + between ``-5`` and ``256``. When you create an int in that range + you actually just get back a reference to the existing object. .. c:function:: PyObject* PyLong_FromUnsignedLong(unsigned long v) @@ -69,6 +71,12 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. on failure. +.. c:function:: PyObject* PyLong_FromUnsignedLongLong(unsigned long long v) + + Return a new :c:type:`PyLongObject` object from a C :c:expr:`unsigned long long`, + or ``NULL`` on failure. + + .. c:function:: PyObject* PyLong_FromInt32(int32_t value) PyObject* PyLong_FromInt64(int64_t value) @@ -79,12 +87,6 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.14 -.. c:function:: PyObject* PyLong_FromUnsignedLongLong(unsigned long long v) - - Return a new :c:type:`PyLongObject` object from a C :c:expr:`unsigned long long`, - or ``NULL`` on failure. - - .. c:function:: PyObject* PyLong_FromUInt32(uint32_t value) PyObject* PyLong_FromUInt64(uint64_t value) @@ -159,6 +161,17 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.13 +.. c:macro:: PyLong_FromPid(pid) + + Macro for creating a Python integer from a process identifier. + + This can be defined as an alias to :c:func:`PyLong_FromLong` or + :c:func:`PyLong_FromLongLong`, depending on the size of the system's + PID type. + + .. versionadded:: 3.2 + + .. c:function:: long PyLong_AsLong(PyObject *obj) .. index:: @@ -184,12 +197,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: long PyLong_AS_LONG(PyObject *obj) - A :term:`soft deprecated` alias. Exactly equivalent to the preferred ``PyLong_AsLong``. In particular, it can fail with :exc:`OverflowError` or another exception. - .. deprecated:: 3.14 - The function is soft deprecated. + .. soft-deprecated:: 3.14 .. c:function:: int PyLong_AsInt(PyObject *obj) @@ -372,6 +383,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Set *\*value* to a signed C :c:expr:`int32_t` or :c:expr:`int64_t` representation of *obj*. + If *obj* is not an instance of :c:type:`PyLongObject`, first call its + :meth:`~object.__index__` method (if present) to convert it to a + :c:type:`PyLongObject`. + If the *obj* value is out of range, raise an :exc:`OverflowError`. Set *\*value* and return ``0`` on success. @@ -436,10 +451,10 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Otherwise, returns the number of bytes required to store the value. If this is equal to or less than *n_bytes*, the entire value was copied. - All *n_bytes* of the buffer are written: large buffers are padded with - zeroes. + All *n_bytes* of the buffer are written: remaining bytes filled by + copies of the sign bit. - If the returned value is greater than than *n_bytes*, the value was + If the returned value is greater than *n_bytes*, the value was truncated: as many of the lowest bits of the value as could fit are written, and the higher bits are ignored. This matches the typical behavior of a C-style downcast. @@ -569,6 +584,17 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.13 +.. c:macro:: PyLong_AsPid(pid) + + Macro for converting a Python integer into a process identifier. + + This can be defined as an alias to :c:func:`PyLong_AsLong`, + :c:func:`PyLong_FromLongLong`, or :c:func:`PyLong_AsInt`, depending on the + size of the system's PID type. + + .. versionadded:: 3.2 + + .. c:function:: int PyLong_GetSign(PyObject *obj, int *sign) Get the sign of the integer object *obj*. @@ -827,3 +853,31 @@ The :c:type:`PyLongWriter` API can be used to import an integer. If *writer* is ``NULL``, no operation is performed. The writer instance and the *digits* array are invalid after the call. + + +Deprecated API +^^^^^^^^^^^^^^ + +These macros are :term:`soft deprecated`. They describe parameters +of the internal representation of :c:type:`PyLongObject` instances. + +Use :c:func:`PyLong_GetNativeLayout` instead, along with :c:func:`PyLong_Export` +to read integer data or :c:type:`PyLongWriter` to write it. +These currently use the same layout, but are designed to continue working correctly +even if CPython's internal integer representation changes. + + +.. c:macro:: PyLong_SHIFT + + This is equivalent to :c:member:`~PyLongLayout.bits_per_digit` in + the output of :c:func:`PyLong_GetNativeLayout`. + + +.. c:macro:: PyLong_BASE + + This is currently equivalent to :c:expr:`1 << PyLong_SHIFT`. + + +.. c:macro:: PyLong_MASK + + This is currently equivalent to :c:expr:`(1 << PyLong_SHIFT) - 1` diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst index 1f55c0aa955c75b..2476ebb9b69dce0 100644 --- a/Doc/c-api/mapping.rst +++ b/Doc/c-api/mapping.rst @@ -102,7 +102,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. note:: - Exceptions which occur when this calls :meth:`~object.__getitem__` + Exceptions which occur when this calls the :meth:`~object.__getitem__` method are silently ignored. For proper error handling, use :c:func:`PyMapping_HasKeyWithError`, :c:func:`PyMapping_GetOptionalItem` or :c:func:`PyObject_GetItem()` instead. @@ -116,7 +116,7 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and .. note:: - Exceptions that occur when this calls :meth:`~object.__getitem__` + Exceptions that occur when this calls the :meth:`~object.__getitem__` method or while creating the temporary :class:`str` object are silently ignored. For proper error handling, use :c:func:`PyMapping_HasKeyStringWithError`, diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index 61218a1bf6f1716..668a163b2df5a11 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -82,7 +82,7 @@ The following functions allow marshalled values to be read back in. assumes that no further objects will be read from the file, allowing it to aggressively load file data into memory so that the de-serialization can operate from data in memory rather than reading a byte at a time from the - file. Only use these variant if you are certain that you won't be reading + file. Only use this variant if you are certain that you won't be reading anything else from the file. On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 64ae35daa703b88..af77528f3695624 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -81,7 +81,7 @@ memory footprint as a whole. Consequently, under certain circumstances, the Python memory manager may or may not trigger appropriate actions, like garbage collection, memory compaction or other preventive procedures. Note that by using the C library allocator as shown in the previous example, the allocated memory -for the I/O buffer escapes completely the Python memory manager. +for the I/O buffer completely escapes the Python memory manager. .. seealso:: @@ -102,7 +102,7 @@ All allocating functions belong to one of three different "domains" (see also strategies and are optimized for different purposes. The specific details on how every domain allocates memory or what internal functions each domain calls is considered an implementation detail, but for debugging purposes a simplified -table can be found at :ref:`here `. +table can be found at :ref:`default-memory-allocators`. The APIs used to allocate and free a block of memory must be from the same domain. For example, :c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. @@ -161,7 +161,7 @@ zero bytes. .. c:function:: void* PyMem_RawCalloc(size_t nelem, size_t elsize) - Allocates *nelem* elements each whose size in bytes is *elsize* and returns + Allocates *nelem* elements each of size *elsize* bytes and returns a pointer of type :c:expr:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. @@ -208,8 +208,11 @@ The following function sets, modeled after the ANSI C standard, but specifying behavior when requesting zero bytes, are available for allocating and releasing memory from the Python heap. -The :ref:`default memory allocator ` uses the -:ref:`pymalloc memory allocator `. +In the GIL-enabled build (default build) the +:ref:`default memory allocator ` uses the +:ref:`pymalloc memory allocator `, whereas in the +:term:`free-threaded build`, the default is the +:ref:`mimalloc memory allocator ` instead. .. warning:: @@ -219,6 +222,11 @@ The :ref:`default memory allocator ` uses the The default allocator is now pymalloc instead of system :c:func:`malloc`. +.. versionchanged:: 3.13 + + In the :term:`free-threaded ` build, the default allocator + is now :ref:`mimalloc `. + .. c:function:: void* PyMem_Malloc(size_t n) Allocates *n* bytes and returns a pointer of type :c:expr:`void*` to the @@ -231,7 +239,7 @@ The :ref:`default memory allocator ` uses the .. c:function:: void* PyMem_Calloc(size_t nelem, size_t elsize) - Allocates *nelem* elements each whose size in bytes is *elsize* and returns + Allocates *nelem* elements each of size *elsize* bytes and returns a pointer of type :c:expr:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. @@ -293,17 +301,39 @@ The following type-oriented macros are provided for convenience. Note that Same as :c:func:`PyMem_Free`. -In addition, the following macro sets are provided for calling the Python memory -allocator directly, without involving the C API functions listed above. However, -note that their use does not preserve binary compatibility across Python -versions and is therefore deprecated in extension modules. -* ``PyMem_MALLOC(size)`` -* ``PyMem_NEW(type, size)`` -* ``PyMem_REALLOC(ptr, size)`` -* ``PyMem_RESIZE(ptr, type, size)`` -* ``PyMem_FREE(ptr)`` -* ``PyMem_DEL(ptr)`` +Deprecated aliases +------------------ + +These are :term:`soft deprecated` aliases to existing functions and macros. +They exist solely for backwards compatibility. + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * Deprecated alias + * Corresponding function or macro + * * .. c:macro:: PyMem_MALLOC(size) + * :c:func:`PyMem_Malloc` + * * .. c:macro:: PyMem_NEW(type, size) + * :c:macro:`PyMem_New` + * * .. c:macro:: PyMem_REALLOC(ptr, size) + * :c:func:`PyMem_Realloc` + * * .. c:macro:: PyMem_RESIZE(ptr, type, size) + * :c:macro:`PyMem_Resize` + * * .. c:macro:: PyMem_FREE(ptr) + * :c:func:`PyMem_Free` + * * .. c:macro:: PyMem_DEL(ptr) + * :c:func:`PyMem_Free` + +.. versionchanged:: 3.4 + + The macros are now aliases of the corresponding functions and macros. + Previously, their behavior was the same, but their use did not necessarily + preserve binary compatibility across Python versions. + +.. deprecated:: 2.0 .. _objectinterface: @@ -322,7 +352,9 @@ memory from the Python heap. the :ref:`Customize Memory Allocators ` section. The :ref:`default object allocator ` uses the -:ref:`pymalloc memory allocator `. +:ref:`pymalloc memory allocator `. In the +:term:`free-threaded ` build, the default is the +:ref:`mimalloc memory allocator ` instead. .. warning:: @@ -340,7 +372,7 @@ The :ref:`default object allocator ` uses the .. c:function:: void* PyObject_Calloc(size_t nelem, size_t elsize) - Allocates *nelem* elements each whose size in bytes is *elsize* and returns + Allocates *nelem* elements each of size *elsize* bytes and returns a pointer of type :c:expr:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. @@ -376,6 +408,24 @@ The :ref:`default object allocator ` uses the If *p* is ``NULL``, no operation is performed. + Do not call this directly to free an object's memory; call the type's + :c:member:`~PyTypeObject.tp_free` slot instead. + + Do not use this for memory allocated by :c:macro:`PyObject_GC_New` or + :c:macro:`PyObject_GC_NewVar`; use :c:func:`PyObject_GC_Del` instead. + + .. seealso:: + + * :c:func:`PyObject_GC_Del` is the equivalent of this function for memory + allocated by types that support garbage collection. + * :c:func:`PyObject_Malloc` + * :c:func:`PyObject_Realloc` + * :c:func:`PyObject_Calloc` + * :c:macro:`PyObject_New` + * :c:macro:`PyObject_NewVar` + * :c:func:`PyType_GenericAlloc` + * :c:member:`~PyTypeObject.tp_free` + .. _default-memory-allocators: @@ -384,14 +434,16 @@ Default Memory Allocators Default memory allocators: -=============================== ==================== ================== ===================== ==================== -Configuration Name PyMem_RawMalloc PyMem_Malloc PyObject_Malloc -=============================== ==================== ================== ===================== ==================== -Release build ``"pymalloc"`` ``malloc`` ``pymalloc`` ``pymalloc`` -Debug build ``"pymalloc_debug"`` ``malloc`` + debug ``pymalloc`` + debug ``pymalloc`` + debug -Release build, without pymalloc ``"malloc"`` ``malloc`` ``malloc`` ``malloc`` -Debug build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug -=============================== ==================== ================== ===================== ==================== +=================================== ======================= ==================== ====================== ====================== +Configuration Name PyMem_RawMalloc PyMem_Malloc PyObject_Malloc +=================================== ======================= ==================== ====================== ====================== +Release build ``"pymalloc"`` ``malloc`` ``pymalloc`` ``pymalloc`` +Debug build ``"pymalloc_debug"`` ``malloc`` + debug ``pymalloc`` + debug ``pymalloc`` + debug +Release build, without pymalloc ``"malloc"`` ``malloc`` ``malloc`` ``malloc`` +Debug build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug +Free-threaded build ``"mimalloc"`` ``mimalloc`` ``mimalloc`` ``mimalloc`` +Free-threaded debug build ``"mimalloc_debug"`` ``mimalloc`` + debug ``mimalloc`` + debug ``mimalloc`` + debug +=================================== ======================= ==================== ====================== ====================== Legend: @@ -399,8 +451,7 @@ Legend: * ``malloc``: system allocators from the standard C library, C functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`. * ``pymalloc``: :ref:`pymalloc memory allocator `. -* ``mimalloc``: :ref:`mimalloc memory allocator `. The pymalloc - allocator will be used if mimalloc support isn't available. +* ``mimalloc``: :ref:`mimalloc memory allocator `. * "+ debug": with :ref:`debug hooks on the Python memory allocators `. * "Debug build": :ref:`Python build in debug mode `. @@ -654,6 +705,10 @@ This allocator is disabled if Python is configured with the :option:`--without-pymalloc` option. It can also be disabled at runtime using the :envvar:`PYTHONMALLOC` environment variable (ex: ``PYTHONMALLOC=malloc``). +Typically, it makes sense to disable the pymalloc allocator when building +Python with AddressSanitizer (:option:`--with-address-sanitizer`) which helps +uncover low level bugs within the C code. + Customize pymalloc Arena Allocator ---------------------------------- @@ -689,9 +744,27 @@ The mimalloc allocator .. versionadded:: 3.13 -Python supports the mimalloc allocator when the underlying platform support is available. -mimalloc "is a general purpose allocator with excellent performance characteristics. -Initially developed by Daan Leijen for the runtime systems of the Koka and Lean languages." +Python supports the `mimalloc `__ +allocator when the underlying platform support is available. +mimalloc is a general purpose allocator with excellent performance +characteristics, initially developed by Daan Leijen for the runtime systems +of the Koka and Lean languages. + +Unlike :ref:`pymalloc `, which is optimized for small objects (512 +bytes or fewer), mimalloc handles allocations of any size. + +In the :term:`free-threaded ` build, mimalloc is the default +and **required** allocator for the :c:macro:`PYMEM_DOMAIN_MEM` and +:c:macro:`PYMEM_DOMAIN_OBJ` domains. It cannot be disabled in free-threaded +builds. The free-threaded build uses per-thread mimalloc heaps, which allows +allocation and deallocation to proceed without locking in most cases. + +In the default (non-free-threaded) build, mimalloc is available but not the +default allocator. It can be selected at runtime using +:envvar:`PYTHONMALLOC`\ ``=mimalloc`` (or ``mimalloc_debug`` to include +:ref:`debug hooks `). It can be disabled at build time +using the :option:`--without-mimalloc` configure option, but this option +cannot be combined with :option:`--disable-gil`. tracemalloc C API ================= diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst index f6038032805259f..e4ac8b57673407c 100644 --- a/Doc/c-api/memoryview.rst +++ b/Doc/c-api/memoryview.rst @@ -13,6 +13,12 @@ A :class:`memoryview` object exposes the C level :ref:`buffer interface any other object. +.. c:var:: PyTypeObject PyMemoryView_Type + + This instance of :c:type:`PyTypeObject` represents the Python memoryview + type. This is the same object as :class:`memoryview` in the Python layer. + + .. c:function:: PyObject *PyMemoryView_FromObject(PyObject *obj) Create a memoryview object from an object that provides the buffer interface. diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index f7f4d37d4c721fb..51e81917b2cfebf 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -13,7 +13,7 @@ Module Objects .. index:: single: ModuleType (in module types) This instance of :c:type:`PyTypeObject` represents the Python module type. This - is exposed to Python programs as ``types.ModuleType``. + is exposed to Python programs as :py:class:`types.ModuleType`. .. c:function:: int PyModule_Check(PyObject *p) @@ -71,6 +71,9 @@ Module Objects ``PyObject_*`` functions rather than directly manipulate a module's :attr:`~object.__dict__`. + The returned reference is borrowed from the module; it is valid until + the module is destroyed. + .. c:function:: PyObject* PyModule_GetNameObject(PyObject *module) @@ -90,6 +93,10 @@ Module Objects Similar to :c:func:`PyModule_GetNameObject` but return the name encoded to ``'utf-8'``. + The returned buffer is only valid until the module is renamed or destroyed. + Note that Python code may rename a module by setting its :py:attr:`~module.__name__` + attribute. + .. c:function:: void* PyModule_GetState(PyObject *module) Return the "state" of the module, that is, a pointer to the block of memory @@ -102,6 +109,10 @@ Module Objects Return a pointer to the :c:type:`PyModuleDef` struct from which the module was created, or ``NULL`` if the module wasn't created from a definition. + On error, return ``NULL`` with an exception set. + Use :c:func:`PyErr_Occurred` to tell this case apart from a missing + :c:type:`!PyModuleDef`. + .. c:function:: PyObject* PyModule_GetFilenameObject(PyObject *module) @@ -122,30 +133,44 @@ Module Objects Similar to :c:func:`PyModule_GetFilenameObject` but return the filename encoded to 'utf-8'. + The returned buffer is only valid until the module's :py:attr:`~module.__file__` attribute + is reassigned or the module is destroyed. + .. deprecated:: 3.2 :c:func:`PyModule_GetFilename` raises :exc:`UnicodeEncodeError` on unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead. -.. _initializing-modules: +.. _pymoduledef: -Initializing C modules -^^^^^^^^^^^^^^^^^^^^^^ +Module definitions +------------------ -Modules objects are usually created from extension modules (shared libraries -which export an initialization function), or compiled-in modules -(where the initialization function is added using :c:func:`PyImport_AppendInittab`). -See :ref:`building` or :ref:`extending-with-embedding` for details. +The functions in the previous section work on any module object, including +modules imported from Python code. -The initialization function can either pass a module definition instance -to :c:func:`PyModule_Create`, and return the resulting module object, -or request "multi-phase initialization" by returning the definition struct itself. +Modules defined using the C API typically use a *module definition*, +:c:type:`PyModuleDef` -- a statically allocated, constant “description" of +how a module should be created. + +The definition is usually used to define an extension's “main” module object +(see :ref:`extension-modules` for details). +It is also used to +:ref:`create extension modules dynamically `. + +Unlike :c:func:`PyModule_New`, the definition allows management of +*module state* -- a piece of memory that is allocated and cleared together +with the module object. +Unlike the module's Python attributes, Python code cannot replace or delete +data stored in module state. .. c:type:: PyModuleDef The module definition struct, which holds all information needed to create - a module object. There is usually only one statically initialized variable - of this type for each module. + a module object. + This structure must be statically allocated (or be otherwise guaranteed + to be valid while any modules created from it exist). + Usually, there is only one variable of this type for each extension module. .. c:member:: PyModuleDef_Base m_base @@ -170,13 +195,15 @@ or request "multi-phase initialization" by returning the definition struct itsel and freed when the module object is deallocated, after the :c:member:`~PyModuleDef.m_free` function has been called, if present. - Setting ``m_size`` to ``-1`` means that the module does not support - sub-interpreters, because it has global state. - Setting it to a non-negative value means that the module can be re-initialized and specifies the additional amount of memory it requires - for its state. Non-negative ``m_size`` is required for multi-phase - initialization. + for its state. + + Setting ``m_size`` to ``-1`` means that the module does not support + sub-interpreters, because it has global state. + Negative ``m_size`` is only allowed when using + :ref:`legacy single-phase initialization ` + or when :ref:`creating modules dynamically `. See :PEP:`3121` for more details. @@ -189,7 +216,7 @@ or request "multi-phase initialization" by returning the definition struct itsel An array of slot definitions for multi-phase initialization, terminated by a ``{0, NULL}`` entry. - When using single-phase initialization, *m_slots* must be ``NULL``. + When using legacy single-phase initialization, *m_slots* must be ``NULL``. .. versionchanged:: 3.5 @@ -249,78 +276,14 @@ or request "multi-phase initialization" by returning the definition struct itsel .. versionchanged:: 3.9 No longer called before the module state is allocated. -Single-phase initialization -........................... - -The module initialization function may create and return the module object -directly. This is referred to as "single-phase initialization", and uses one -of the following two module creation functions: -.. c:function:: PyObject* PyModule_Create(PyModuleDef *def) - - Create a new module object, given the definition in *def*. This behaves - like :c:func:`PyModule_Create2` with *module_api_version* set to - :c:macro:`PYTHON_API_VERSION`. +.. c:var:: PyTypeObject PyModuleDef_Type + The type of ``PyModuleDef`` objects. -.. c:function:: PyObject* PyModule_Create2(PyModuleDef *def, int module_api_version) - - Create a new module object, given the definition in *def*, assuming the - API version *module_api_version*. If that version does not match the version - of the running interpreter, a :exc:`RuntimeWarning` is emitted. - Return ``NULL`` with an exception set on error. - - .. note:: - - Most uses of this function should be using :c:func:`PyModule_Create` - instead; only use this if you are sure you need it. - -Before it is returned from in the initialization function, the resulting module -object is typically populated using functions like :c:func:`PyModule_AddObjectRef`. - -.. _multi-phase-initialization: - -Multi-phase initialization -.......................... - -An alternate way to specify extensions is to request "multi-phase initialization". -Extension modules created this way behave more like Python modules: the -initialization is split between the *creation phase*, when the module object -is created, and the *execution phase*, when it is populated. -The distinction is similar to the :py:meth:`!__new__` and :py:meth:`!__init__` methods -of classes. - -Unlike modules created using single-phase initialization, these modules are not -singletons: if the *sys.modules* entry is removed and the module is re-imported, -a new module object is created, and the old module is subject to normal garbage -collection -- as with Python modules. -By default, multiple modules created from the same definition should be -independent: changes to one should not affect the others. -This means that all state should be specific to the module object (using e.g. -using :c:func:`PyModule_GetState`), or its contents (such as the module's -:attr:`~object.__dict__` or individual classes created with :c:func:`PyType_FromSpec`). - -All modules created using multi-phase initialization are expected to support -:ref:`sub-interpreters `. Making sure multiple modules -are independent is typically enough to achieve this. - -To request multi-phase initialization, the initialization function -(PyInit_modulename) returns a :c:type:`PyModuleDef` instance with non-empty -:c:member:`~PyModuleDef.m_slots`. Before it is returned, the ``PyModuleDef`` -instance must be initialized with the following function: - -.. c:function:: PyObject* PyModuleDef_Init(PyModuleDef *def) - - Ensures a module definition is a properly initialized Python object that - correctly reports its type and reference count. - - Returns *def* cast to ``PyObject*``, or ``NULL`` if an error occurred. - - .. versionadded:: 3.5 - -The *m_slots* member of the module definition must point to an array of -``PyModuleDef_Slot`` structures: +Module slots +............ .. c:type:: PyModuleDef_Slot @@ -334,8 +297,6 @@ The *m_slots* member of the module definition must point to an array of .. versionadded:: 3.5 -The *m_slots* array must be terminated by a slot with id 0. - The available slot types are: .. c:macro:: Py_mod_create @@ -372,6 +333,8 @@ The available slot types are: ``PyModuleDef`` has non-``NULL`` ``m_traverse``, ``m_clear``, ``m_free``; non-zero ``m_size``; or slots other than ``Py_mod_create``. + .. versionadded:: 3.5 + .. c:macro:: Py_mod_exec Specifies a function that is called to *execute* the module. @@ -386,6 +349,8 @@ The available slot types are: If multiple ``Py_mod_exec`` slots are specified, they are processed in the order they appear in the *m_slots* array. + .. versionadded:: 3.5 + .. c:macro:: Py_mod_multiple_interpreters Specifies one of the following values: @@ -446,21 +411,48 @@ The available slot types are: .. versionadded:: 3.13 -See :PEP:`489` for more details on multi-phase initialization. -Low-level module creation functions -................................... +.. _moduledef-dynamic: + +Creating extension modules dynamically +-------------------------------------- -The following functions are called under the hood when using multi-phase -initialization. They can be used directly, for example when creating module -objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and -``PyModule_ExecDef`` must be called to fully initialize a module. +The following functions may be used to create a module outside of an +extension's :ref:`initialization function `. +They are also used in +:ref:`single-phase initialization `. + +.. c:function:: PyObject* PyModule_Create(PyModuleDef *def) + + Create a new module object, given the definition in *def*. + This is a macro that calls :c:func:`PyModule_Create2` with + *module_api_version* set to :c:macro:`PYTHON_API_VERSION`, or + to :c:macro:`PYTHON_ABI_VERSION` if using the + :ref:`limited API `. + +.. c:function:: PyObject* PyModule_Create2(PyModuleDef *def, int module_api_version) + + Create a new module object, given the definition in *def*, assuming the + API version *module_api_version*. If that version does not match the version + of the running interpreter, a :exc:`RuntimeWarning` is emitted. + + Return ``NULL`` with an exception set on error. + + This function does not support slots. + The :c:member:`~PyModuleDef.m_slots` member of *def* must be ``NULL``. + + + .. note:: + + Most uses of this function should be using :c:func:`PyModule_Create` + instead; only use this if you are sure you need it. .. c:function:: PyObject * PyModule_FromDefAndSpec(PyModuleDef *def, PyObject *spec) - Create a new module object, given the definition in *def* and the - ModuleSpec *spec*. This behaves like :c:func:`PyModule_FromDefAndSpec2` - with *module_api_version* set to :c:macro:`PYTHON_API_VERSION`. + This macro calls :c:func:`PyModule_FromDefAndSpec2` with + *module_api_version* set to :c:macro:`PYTHON_API_VERSION`, or + to :c:macro:`PYTHON_ABI_VERSION` if using the + :ref:`limited API `. .. versionadded:: 3.5 @@ -473,6 +465,10 @@ objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and Return ``NULL`` with an exception set on error. + Note that this does not process execution slots (:c:data:`Py_mod_exec`). + Both ``PyModule_FromDefAndSpec`` and ``PyModule_ExecDef`` must be called + to fully initialize a module. + .. note:: Most uses of this function should be using :c:func:`PyModule_FromDefAndSpec` @@ -486,35 +482,29 @@ objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and .. versionadded:: 3.5 -.. c:function:: int PyModule_SetDocString(PyObject *module, const char *docstring) +.. c:macro:: PYTHON_API_VERSION - Set the docstring for *module* to *docstring*. - This function is called automatically when creating a module from - ``PyModuleDef``, using either ``PyModule_Create`` or - ``PyModule_FromDefAndSpec``. + The C API version. Defined for backwards compatibility. - .. versionadded:: 3.5 + Currently, this constant is not updated in new Python versions, and is not + useful for versioning. This may change in the future. -.. c:function:: int PyModule_AddFunctions(PyObject *module, PyMethodDef *functions) +.. c:macro:: PYTHON_ABI_VERSION - Add the functions from the ``NULL`` terminated *functions* array to *module*. - Refer to the :c:type:`PyMethodDef` documentation for details on individual - entries (due to the lack of a shared module namespace, module level - "functions" implemented in C typically receive the module as their first - parameter, making them similar to instance methods on Python classes). - This function is called automatically when creating a module from - ``PyModuleDef``, using either ``PyModule_Create`` or - ``PyModule_FromDefAndSpec``. + Defined as ``3`` for backwards compatibility. + + Currently, this constant is not updated in new Python versions, and is not + useful for versioning. This may change in the future. - .. versionadded:: 3.5 Support functions -................. +----------------- -The module initialization function (if using single phase initialization) or -a function called from a module execution slot (if using multi-phase -initialization), can use the following functions to help initialize the module -state: +The following functions are provided to help initialize a module +state. +They are intended for a module's execution slots (:c:data:`Py_mod_exec`), +the initialization function for legacy :ref:`single-phase initialization `, +or code that creates modules dynamically. .. c:function:: int PyModule_AddObjectRef(PyObject *module, const char *name, PyObject *value) @@ -569,8 +559,8 @@ state: .. c:function:: int PyModule_Add(PyObject *module, const char *name, PyObject *value) - Similar to :c:func:`PyModule_AddObjectRef`, but "steals" a reference - to *value*. + Similar to :c:func:`PyModule_AddObjectRef`, but ":term:`steals `" + a reference to *value* (even on error). It can be called with a result of function that returns a new reference without bothering to check its result or even saving it to a variable. @@ -585,8 +575,8 @@ state: .. c:function:: int PyModule_AddObject(PyObject *module, const char *name, PyObject *value) - Similar to :c:func:`PyModule_AddObjectRef`, but steals a reference to - *value* on success (if it returns ``0``). + Similar to :c:func:`PyModule_AddObjectRef`, but :term:`steals ` + a reference to *value* on success (if it returns ``0``). The new :c:func:`PyModule_Add` or :c:func:`PyModule_AddObjectRef` functions are recommended, since it is @@ -614,9 +604,7 @@ state: // PyModule_AddObject() stole a reference to obj: // Py_XDECREF(obj) is not needed here. - .. deprecated:: 3.13 - - :c:func:`PyModule_AddObject` is :term:`soft deprecated`. + .. soft-deprecated:: 3.13 .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) @@ -663,12 +651,45 @@ state: .. versionadded:: 3.9 +.. c:function:: int PyModule_AddFunctions(PyObject *module, PyMethodDef *functions) + + Add the functions from the ``NULL`` terminated *functions* array to *module*. + Refer to the :c:type:`PyMethodDef` documentation for details on individual + entries (due to the lack of a shared module namespace, module level + "functions" implemented in C typically receive the module as their first + parameter, making them similar to instance methods on Python classes). + + This function is called automatically when creating a module from + ``PyModuleDef`` (such as when using :ref:`multi-phase-initialization`, + ``PyModule_Create``, or ``PyModule_FromDefAndSpec``). + Some module authors may prefer defining functions in multiple + :c:type:`PyMethodDef` arrays; in that case they should call this function + directly. + + The *functions* array must be statically allocated (or otherwise guaranteed + to outlive the module object). + + .. versionadded:: 3.5 + +.. c:function:: int PyModule_SetDocString(PyObject *module, const char *docstring) + + Set the docstring for *module* to *docstring*. + This function is called automatically when creating a module from + ``PyModuleDef`` (such as when using :ref:`multi-phase-initialization`, + ``PyModule_Create``, or ``PyModule_FromDefAndSpec``). + + Return ``0`` on success. + Return ``-1`` with an exception set on error. + + .. versionadded:: 3.5 + .. c:function:: int PyUnstable_Module_SetGIL(PyObject *module, void *gil) Indicate that *module* does or does not support running without the global interpreter lock (GIL), using one of the values from :c:macro:`Py_mod_gil`. It must be called during *module*'s initialization - function. If this function is not called during module initialization, the + function when using :ref:`single-phase-initialization`. + If this function is not called during module initialization, the import machinery assumes the module does not support running without the GIL. This function is only available in Python builds configured with :option:`--disable-gil`. @@ -677,10 +698,11 @@ state: .. versionadded:: 3.13 -Module lookup -^^^^^^^^^^^^^ +Module lookup (single-phase initialization) +........................................... -Single-phase initialization creates singleton modules that can be looked up +The legacy :ref:`single-phase initialization ` +initialization scheme creates singleton modules that can be looked up in the context of the current interpreter. This allows the module object to be retrieved later with only a reference to the module definition. @@ -701,7 +723,8 @@ since multiple such modules can be created from a single definition. Only effective on modules created using single-phase initialization. - Python calls ``PyState_AddModule`` automatically after importing a module, + Python calls ``PyState_AddModule`` automatically after importing a module + that uses :ref:`single-phase initialization `, so it is unnecessary (but harmless) to call it from module initialization code. An explicit call is needed only if the module's own init code subsequently calls ``PyState_FindModule``. @@ -709,6 +732,9 @@ since multiple such modules can be created from a single definition. mechanisms (either by calling it directly, or by referring to its implementation for details of the required state updates). + If a module was attached previously using the same *def*, it is replaced + by the new *module*. + The caller must have an :term:`attached thread state`. Return ``-1`` with an exception set on error, ``0`` on success. diff --git a/Doc/c-api/monitoring.rst b/Doc/c-api/monitoring.rst index 7926148302af0b5..4bfcb86abf58ed1 100644 --- a/Doc/c-api/monitoring.rst +++ b/Doc/c-api/monitoring.rst @@ -136,7 +136,7 @@ Managing the Monitoring State ----------------------------- Monitoring states can be managed with the help of monitoring scopes. A scope -would typically correspond to a python function. +would typically correspond to a Python function. .. c:function:: int PyMonitoring_EnterScope(PyMonitoringState *state_array, uint64_t *version, const uint8_t *event_types, Py_ssize_t length) @@ -205,6 +205,4 @@ would typically correspond to a python function. .. versionadded:: 3.13 - .. deprecated:: 3.14 - - This function is :term:`soft deprecated`. + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index 0fd159f1eb87f80..44e1220d109d051 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -73,7 +73,7 @@ Object Protocol Flag to be used with multiple functions that print the object (like :c:func:`PyObject_Print` and :c:func:`PyFile_WriteObject`). - If passed, these function would use the :func:`str` of the object + If passed, these functions use the :func:`str` of the object instead of the :func:`repr`. @@ -319,6 +319,8 @@ Object Protocol representation on success, ``NULL`` on failure. This is the equivalent of the Python expression ``repr(o)``. Called by the :func:`repr` built-in function. + If argument is ``NULL``, return the string ``''``. + .. versionchanged:: 3.4 This function now includes a debug assertion to help ensure that it does not silently discard an active exception. @@ -333,6 +335,8 @@ Object Protocol a string similar to that returned by :c:func:`PyObject_Repr` in Python 2. Called by the :func:`ascii` built-in function. + If argument is ``NULL``, return the string ``''``. + .. index:: string; PyObject_Str (C function) @@ -343,6 +347,8 @@ Object Protocol Python expression ``str(o)``. Called by the :func:`str` built-in function and, therefore, by the :func:`print` function. + If argument is ``NULL``, return the string ``''``. + .. versionchanged:: 3.4 This function now includes a debug assertion to help ensure that it does not silently discard an active exception. @@ -358,6 +364,8 @@ Object Protocol a TypeError is raised when *o* is an integer instead of a zero-initialized bytes object. + If argument is ``NULL``, return the :class:`bytes` object ``b''``. + .. c:function:: int PyObject_IsSubclass(PyObject *derived, PyObject *cls) @@ -585,7 +593,7 @@ Object Protocol Clear the managed dictionary of *obj*. - This function must only be called in a traverse function of the type which + This function must only be called in a clear function of the type which has the :c:macro:`Py_TPFLAGS_MANAGED_DICT` flag set. .. versionadded:: 3.13 @@ -596,12 +604,13 @@ Object Protocol if supported by the runtime. In the :term:`free-threaded ` build, this allows the interpreter to avoid reference count adjustments to *obj*, which may improve multi-threaded performance. The tradeoff is - that *obj* will only be deallocated by the tracing garbage collector. + that *obj* will only be deallocated by the tracing garbage collector, and + not when the interpreter no longer has any references to it. - This function returns ``1`` if deferred reference counting is enabled on *obj* - (including when it was enabled before the call), + This function returns ``1`` if deferred reference counting is enabled on *obj*, and ``0`` if deferred reference counting is not supported or if the hint was - ignored by the runtime. This function is thread-safe, and cannot fail. + ignored by the interpreter, such as when deferred reference counting is already + enabled on *obj*. This function is thread-safe, and cannot fail. This function does nothing on builds with the :term:`GIL` enabled, which do not support deferred reference counting. This also does nothing if *obj* is not @@ -609,7 +618,8 @@ Object Protocol :c:func:`PyObject_GC_IsTracked`). This function is intended to be used soon after *obj* is created, - by the code that creates it. + by the code that creates it, such as in the object's :c:member:`~PyTypeObject.tp_new` + slot. .. versionadded:: 3.14 @@ -665,10 +675,10 @@ Object Protocol :c:func:`PyUnstable_EnableTryIncRef` must have been called earlier on *obj* or this function may spuriously return ``0`` in the - :term:`free threading` build. + :term:`free-threaded build`. This function is logically equivalent to the following C code, except that - it behaves atomically in the :term:`free threading` build:: + it behaves atomically in the :term:`free-threaded build`:: if (Py_REFCNT(op) > 0) { Py_INCREF(op); @@ -745,10 +755,10 @@ Object Protocol On GIL-enabled builds, this function is equivalent to :c:expr:`Py_REFCNT(op) == 1`. - On a :term:`free threaded ` build, this checks if *op*'s + On a :term:`free-threaded build`, this checks if *op*'s :term:`reference count` is equal to one and additionally checks if *op* is only used by this thread. :c:expr:`Py_REFCNT(op) == 1` is **not** - thread-safe on free threaded builds; prefer this function. + thread-safe on free-threaded builds; prefer this function. The caller must hold an :term:`attached thread state`, despite the fact that this function doesn't call into the Python interpreter. This function diff --git a/Doc/c-api/objimpl.rst b/Doc/c-api/objimpl.rst index 8bd8c107c98bdfc..83de4248039949f 100644 --- a/Doc/c-api/objimpl.rst +++ b/Doc/c-api/objimpl.rst @@ -12,6 +12,7 @@ object types. .. toctree:: allocation.rst + lifecycle.rst structures.rst typeobj.rst gcsupport.rst diff --git a/Doc/c-api/perfmaps.rst b/Doc/c-api/perfmaps.rst index 77b5e3c0876bbb7..68c2578906ff094 100644 --- a/Doc/c-api/perfmaps.rst +++ b/Doc/c-api/perfmaps.rst @@ -48,3 +48,43 @@ Note that holding an :term:`attached thread state` is not required for these API This is called by the runtime itself during interpreter shut-down. In general, there shouldn't be a reason to explicitly call this, except to handle specific scenarios such as forking. + +.. c:function:: int PyUnstable_CopyPerfMapFile(const char *parent_filename) + + Open the ``/tmp/perf-$pid.map`` file and append the content of *parent_filename* + to it. + + This function is available on all platforms but only generates output on platforms + that support perf maps (currently only Linux). On other platforms, it does nothing. + + .. versionadded:: 3.13 + +.. c:function:: int PyUnstable_PerfTrampoline_CompileCode(PyCodeObject *code) + + Compile the given code object using the current perf trampoline. + + The "current" trampoline is the one set by the runtime or the most recent + :c:func:`PyUnstable_PerfTrampoline_SetPersistAfterFork` call. + + If no trampoline is set, falls back to normal compilation (no perf map entry). + + :param code: The code object to compile. + :return: 0 on success, -1 on failure. + + .. versionadded:: 3.13 + +.. c:function:: int PyUnstable_PerfTrampoline_SetPersistAfterFork(int enable) + + Set whether the perf trampoline should persist after a fork. + + * If ``enable`` is true (non-zero): perf map file remains open/valid post-fork. + Child process inherits all existing perf map entries. + * If ``enable`` is false (zero): perf map closes post-fork. + Child process gets empty perf map. + + Default: false (clears on fork). + + :param enable: 1 to enable, 0 to disable. + :return: 0 on success, -1 on failure. + + .. versionadded:: 3.13 diff --git a/Doc/c-api/picklebuffer.rst b/Doc/c-api/picklebuffer.rst new file mode 100644 index 000000000000000..9e2d92341b0f93e --- /dev/null +++ b/Doc/c-api/picklebuffer.rst @@ -0,0 +1,59 @@ +.. highlight:: c + +.. _picklebuffer-objects: + +.. index:: + pair: object; PickleBuffer + +Pickle buffer objects +--------------------- + +.. versionadded:: 3.8 + +A :class:`pickle.PickleBuffer` object wraps a :ref:`buffer-providing object +` for out-of-band data transfer with the :mod:`pickle` module. + + +.. c:var:: PyTypeObject PyPickleBuffer_Type + + This instance of :c:type:`PyTypeObject` represents the Python pickle buffer type. + This is the same object as :class:`pickle.PickleBuffer` in the Python layer. + + +.. c:function:: int PyPickleBuffer_Check(PyObject *op) + + Return true if *op* is a pickle buffer instance. + This function always succeeds. + + +.. c:function:: PyObject *PyPickleBuffer_FromObject(PyObject *obj) + + Create a pickle buffer from the object *obj*. + + This function will fail if *obj* doesn't support the :ref:`buffer protocol `. + + On success, return a new pickle buffer instance. + On failure, set an exception and return ``NULL``. + + Analogous to calling :class:`pickle.PickleBuffer` with *obj* in Python. + + +.. c:function:: const Py_buffer *PyPickleBuffer_GetBuffer(PyObject *picklebuf) + + Get a pointer to the underlying :c:type:`Py_buffer` that the pickle buffer wraps. + + The returned pointer is valid as long as *picklebuf* is alive and has not been + released. The caller must not modify or free the returned :c:type:`Py_buffer`. + If the pickle buffer has been released, raise :exc:`ValueError`. + + On success, return a pointer to the buffer view. + On failure, set an exception and return ``NULL``. + + +.. c:function:: int PyPickleBuffer_Release(PyObject *picklebuf) + + Release the underlying buffer held by the pickle buffer. + + Return ``0`` on success. On failure, set an exception and return ``-1``. + + Analogous to calling :meth:`pickle.PickleBuffer.release` in Python. diff --git a/Doc/c-api/profiling.rst b/Doc/c-api/profiling.rst new file mode 100644 index 000000000000000..0200f2eac6d9086 --- /dev/null +++ b/Doc/c-api/profiling.rst @@ -0,0 +1,239 @@ +.. highlight:: c + +.. _profiling: + +Profiling and tracing +===================== + +The Python interpreter provides some low-level support for attaching profiling +and execution tracing facilities. These are used for profiling, debugging, and +coverage analysis tools. + +This C interface allows the profiling or tracing code to avoid the overhead of +calling through Python-level callable objects, making a direct C function call +instead. The essential attributes of the facility have not changed; the +interface allows trace functions to be installed per-thread, and the basic +events reported to the trace function are the same as had been reported to the +Python-level trace functions in previous versions. + + +.. c:type:: int (*Py_tracefunc)(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg) + + The type of the trace function registered using :c:func:`PyEval_SetProfile` and + :c:func:`PyEval_SetTrace`. The first parameter is the object passed to the + registration function as *obj*, *frame* is the frame object to which the event + pertains, *what* is one of the constants :c:data:`PyTrace_CALL`, + :c:data:`PyTrace_EXCEPTION`, :c:data:`PyTrace_LINE`, :c:data:`PyTrace_RETURN`, + :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION`, :c:data:`PyTrace_C_RETURN`, + or :c:data:`PyTrace_OPCODE`, and *arg* depends on the value of *what*: + + +-------------------------------+----------------------------------------+ + | Value of *what* | Meaning of *arg* | + +===============================+========================================+ + | :c:data:`PyTrace_CALL` | Always :c:data:`Py_None`. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_EXCEPTION` | Exception information as returned by | + | | :func:`sys.exc_info`. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_LINE` | Always :c:data:`Py_None`. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_RETURN` | Value being returned to the caller, | + | | or ``NULL`` if caused by an exception. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_C_CALL` | Function object being called. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_C_EXCEPTION` | Function object being called. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_C_RETURN` | Function object being called. | + +-------------------------------+----------------------------------------+ + | :c:data:`PyTrace_OPCODE` | Always :c:data:`Py_None`. | + +-------------------------------+----------------------------------------+ + +.. c:var:: int PyTrace_CALL + + The value of the *what* parameter to a :c:type:`Py_tracefunc` function when a new + call to a function or method is being reported, or a new entry into a generator. + Note that the creation of the iterator for a generator function is not reported + as there is no control transfer to the Python bytecode in the corresponding + frame. + + +.. c:var:: int PyTrace_EXCEPTION + + The value of the *what* parameter to a :c:type:`Py_tracefunc` function when an + exception has been raised. The callback function is called with this value for + *what* when after any bytecode is processed after which the exception becomes + set within the frame being executed. The effect of this is that as exception + propagation causes the Python stack to unwind, the callback is called upon + return to each frame as the exception propagates. Only trace functions receive + these events; they are not needed by the profiler. + + +.. c:var:: int PyTrace_LINE + + The value passed as the *what* parameter to a :c:type:`Py_tracefunc` function + (but not a profiling function) when a line-number event is being reported. + It may be disabled for a frame by setting :attr:`~frame.f_trace_lines` to + *0* on that frame. + + +.. c:var:: int PyTrace_RETURN + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a + call is about to return. + + +.. c:var:: int PyTrace_C_CALL + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C + function is about to be called. + + +.. c:var:: int PyTrace_C_EXCEPTION + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C + function has raised an exception. + + +.. c:var:: int PyTrace_C_RETURN + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C + function has returned. + + +.. c:var:: int PyTrace_OPCODE + + The value for the *what* parameter to :c:type:`Py_tracefunc` functions (but not + profiling functions) when a new opcode is about to be executed. This event is + not emitted by default: it must be explicitly requested by setting + :attr:`~frame.f_trace_opcodes` to *1* on the frame. + + +.. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj) + + Set the profiler function to *func*. The *obj* parameter is passed to the + function as its first parameter, and may be any Python object, or ``NULL``. If + the profile function needs to maintain state, using a different value for *obj* + for each thread provides a convenient and thread-safe place to store it. The + profile function is called for all monitored events except :c:data:`PyTrace_LINE` + :c:data:`PyTrace_OPCODE` and :c:data:`PyTrace_EXCEPTION`. + + See also the :func:`sys.setprofile` function. + + The caller must have an :term:`attached thread state`. + + +.. c:function:: void PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *obj) + + Like :c:func:`PyEval_SetProfile` but sets the profile function in all running threads + belonging to the current interpreter instead of the setting it only on the current thread. + + The caller must have an :term:`attached thread state`. + + As :c:func:`PyEval_SetProfile`, this function ignores any exceptions raised while + setting the profile functions in all threads. + +.. versionadded:: 3.12 + + +.. c:function:: void PyEval_SetTrace(Py_tracefunc func, PyObject *obj) + + Set the tracing function to *func*. This is similar to + :c:func:`PyEval_SetProfile`, except the tracing function does receive line-number + events and per-opcode events, but does not receive any event related to C function + objects being called. Any trace function registered using :c:func:`PyEval_SetTrace` + will not receive :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION` or + :c:data:`PyTrace_C_RETURN` as a value for the *what* parameter. + + See also the :func:`sys.settrace` function. + + The caller must have an :term:`attached thread state`. + + +.. c:function:: void PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject *obj) + + Like :c:func:`PyEval_SetTrace` but sets the tracing function in all running threads + belonging to the current interpreter instead of the setting it only on the current thread. + + The caller must have an :term:`attached thread state`. + + As :c:func:`PyEval_SetTrace`, this function ignores any exceptions raised while + setting the trace functions in all threads. + +.. versionadded:: 3.12 + + +Reference tracing +================= + +.. versionadded:: 3.13 + + +.. c:type:: int (*PyRefTracer)(PyObject *, int event, void* data) + + The type of the trace function registered using :c:func:`PyRefTracer_SetTracer`. + The first parameter is a Python object that has been just created (when **event** + is set to :c:data:`PyRefTracer_CREATE`) or about to be destroyed (when **event** + is set to :c:data:`PyRefTracer_DESTROY`). The **data** argument is the opaque pointer + that was provided when :c:func:`PyRefTracer_SetTracer` was called. + + If a new tracing function is registered replacing the current one, a call to the + trace function will be made with the object set to **NULL** and **event** set to + :c:data:`PyRefTracer_TRACKER_REMOVED`. This will happen just before the new + function is registered. + +.. versionadded:: 3.13 + + +.. c:var:: int PyRefTracer_CREATE + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been created. + + +.. c:var:: int PyRefTracer_DESTROY + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when a Python + object has been destroyed. + + +.. c:var:: int PyRefTracer_TRACKER_REMOVED + + The value for the *event* parameter to :c:type:`PyRefTracer` functions when the + current tracer is about to be replaced by a new one. + + .. versionadded:: 3.14 + + +.. c:function:: int PyRefTracer_SetTracer(PyRefTracer tracer, void *data) + + Register a reference tracer function. The function will be called when a new + Python object has been created or when an object is going to be destroyed. If + **data** is provided it must be an opaque pointer that will be provided when + the tracer function is called. Return ``0`` on success. Set an exception and + return ``-1`` on error. + + Note that tracer functions **must not** create Python objects inside or + otherwise the call will be re-entrant. The tracer also **must not** clear + any existing exception or set an exception. A :term:`thread state` will be active + every time the tracer function is called. + + There must be an :term:`attached thread state` when calling this function. + + If another tracer function was already registered, the old function will be + called with **event** set to :c:data:`PyRefTracer_TRACKER_REMOVED` just before + the new function is registered. + +.. versionadded:: 3.13 + + +.. c:function:: PyRefTracer PyRefTracer_GetTracer(void** data) + + Get the registered reference tracer function and the value of the opaque data + pointer that was registered when :c:func:`PyRefTracer_SetTracer` was called. + If no tracer was registered this function will return NULL and will set the + **data** pointer to NULL. + + There must be an :term:`attached thread state` when calling this function. + +.. versionadded:: 3.13 diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index b23f016f9b0a063..4d56a92bf2af797 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -25,7 +25,7 @@ of Python objects. .. note:: - On :term:`free threaded ` builds of Python, returning 1 + On :term:`free-threaded builds ` of Python, returning 1 isn't sufficient to determine if it's safe to treat *o* as having no access by other threads. Use :c:func:`PyUnstable_Object_IsUniquelyReferenced` for that instead. @@ -210,7 +210,7 @@ of Python objects. Py_SETREF(dst, src); - That arranges to set *dst* to *src* _before_ releasing the reference + That arranges to set *dst* to *src* *before* releasing the reference to the old value of *dst*, so that any code triggered as a side-effect of *dst* getting torn down no longer believes *dst* points to a valid object. diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index df5bf6b64a93a00..90490cf6749b59c 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -67,7 +67,7 @@ Sequence Protocol Assign object *v* to the *i*\ th element of *o*. Raise an exception and return ``-1`` on failure; return ``0`` on success. This is the equivalent of the Python statement ``o[i] = v``. This function *does - not* steal a reference to *v*. + not* ":term:`steal`" a reference to *v*. If *v* is ``NULL``, the element is deleted, but this feature is deprecated in favour of using :c:func:`PySequence_DelItem`. @@ -109,9 +109,8 @@ Sequence Protocol Alias for :c:func:`PySequence_Contains`. - .. deprecated:: 3.14 - The function is :term:`soft deprecated` and should no longer be used to - write new code. + .. soft-deprecated:: 3.14 + The function should no longer be used to write new code. .. c:function:: Py_ssize_t PySequence_Index(PyObject *o, PyObject *value) diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index cba823aa027bd6d..a44d49ecd1857ec 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -92,6 +92,11 @@ the constructor functions work with any iterable Python object. actually iterable. The constructor is also useful for copying a set (``c=set(s)``). + .. note:: + + The operation is atomic on :term:`free threading ` + when *iterable* is a :class:`set`, :class:`frozenset` or :class:`dict`. + .. c:function:: PyObject* PyFrozenSet_New(PyObject *iterable) @@ -100,6 +105,11 @@ the constructor functions work with any iterable Python object. set on success or ``NULL`` on failure. Raise :exc:`TypeError` if *iterable* is not actually iterable. + .. note:: + + The operation is atomic on :term:`free threading ` + when *iterable* is a :class:`set`, :class:`frozenset` or :class:`dict`. + The following functions and macros are available for instances of :class:`set` or :class:`frozenset` or instances of their subtypes. @@ -127,6 +137,10 @@ or :class:`frozenset` or instances of their subtypes. the *key* is unhashable. Raise :exc:`SystemError` if *anyset* is not a :class:`set`, :class:`frozenset`, or an instance of a subtype. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. .. c:function:: int PySet_Add(PyObject *set, PyObject *key) @@ -138,6 +152,12 @@ or :class:`frozenset` or instances of their subtypes. :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + + The following functions are available for instances of :class:`set` or its subtypes but not for instances of :class:`frozenset` or its subtypes. @@ -147,11 +167,16 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. Return ``1`` if found and removed, ``0`` if not found (no action taken), and ``-1`` if an error is encountered. Does not raise :exc:`KeyError` for missing keys. Raise a - :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~frozenset.discard` + :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~set.discard` method, this function does not automatically convert unhashable sets into temporary frozensets. Raise :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. + .. note:: + + The operation is atomic on :term:`free threading ` + when *key* is :class:`str`, :class:`int`, :class:`float`, :class:`bool` or :class:`bytes`. + .. c:function:: PyObject* PySet_Pop(PyObject *set) @@ -166,3 +191,28 @@ subtypes but not for instances of :class:`frozenset` or its subtypes. Empty an existing set of all elements. Return ``0`` on success. Return ``-1`` and raise :exc:`SystemError` if *set* is not an instance of :class:`set` or its subtype. + + .. note:: + + In the :term:`free-threaded build`, the set is emptied before its entries + are cleared, so other threads will observe an empty set rather than + intermediate states. + + +Deprecated API +^^^^^^^^^^^^^^ + +.. c:macro:: PySet_MINSIZE + + A constant representing the size of an internal + preallocated table inside :c:type:`PySetObject` instances. + + This is documented solely for completeness, as there are no guarantees + that a given version of CPython uses preallocated tables with a fixed + size. + In code that does not deal with unstable set internals, + :c:macro:`!PySet_MINSIZE` can be replaced with a small constant like ``8``. + + If looking for the size of a set, use :c:func:`PySet_Size` instead. + + .. soft-deprecated:: 3.14 diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst index 124e58cf950b7a7..9b65e0b8d23d93f 100644 --- a/Doc/c-api/stable.rst +++ b/Doc/c-api/stable.rst @@ -51,6 +51,7 @@ It is generally intended for specialized, low-level tools like debuggers. Projects that use this API are expected to follow CPython development and spend extra effort adjusting to changes. +.. _stable-application-binary-interface: Stable Application Binary Interface =================================== diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 987bc167c68d6cd..a3e164011a825bf 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -28,18 +28,52 @@ under :ref:`reference counting `. object. In a normal "release" build, it contains only the object's reference count and a pointer to the corresponding type object. Nothing is actually declared to be a :c:type:`PyObject`, but every pointer - to a Python object can be cast to a :c:expr:`PyObject*`. Access to the - members must be done by using the macros :c:macro:`Py_REFCNT` and - :c:macro:`Py_TYPE`. + to a Python object can be cast to a :c:expr:`PyObject*`. + + The members must not be accessed directly; instead use macros such as + :c:macro:`Py_REFCNT` and :c:macro:`Py_TYPE`. + + .. c:member:: Py_ssize_t ob_refcnt + + The object's reference count, as returned by :c:macro:`Py_REFCNT`. + Do not use this field directly; instead use functions and macros such as + :c:macro:`!Py_REFCNT`, :c:func:`Py_INCREF` and :c:func:`Py_DecRef`. + + The field type may be different from ``Py_ssize_t``, depending on + build configuration and platform. + + .. c:member:: PyTypeObject* ob_type + + The object's type. + Do not use this field directly; use :c:macro:`Py_TYPE` and + :c:func:`Py_SET_TYPE` instead. .. c:type:: PyVarObject - This is an extension of :c:type:`PyObject` that adds the :c:member:`~PyVarObject.ob_size` - field. This is only used for objects that have some notion of *length*. - This type does not often appear in the Python/C API. - Access to the members must be done by using the macros - :c:macro:`Py_REFCNT`, :c:macro:`Py_TYPE`, and :c:macro:`Py_SIZE`. + An extension of :c:type:`PyObject` that adds the + :c:member:`~PyVarObject.ob_size` field. + This is intended for objects that have some notion of *length*. + + As with :c:type:`!PyObject`, the members must not be accessed directly; + instead use macros such as :c:macro:`Py_SIZE`, :c:macro:`Py_REFCNT` and + :c:macro:`Py_TYPE`. + + .. c:member:: Py_ssize_t ob_size + + A size field, whose contents should be considered an object's internal + implementation detail. + + Do not use this field directly; use :c:macro:`Py_SIZE` instead. + + Object creation functions such as :c:func:`PyObject_NewVar` will + generally set this field to the requested size (number of items). + After creation, arbitrary values can be stored in :c:member:`!ob_size` + using :c:macro:`Py_SET_SIZE`. + + To get an object's publicly exposed length, as returned by + the Python function :py:func:`len`, use :c:func:`PyObject_Length` + instead. .. c:macro:: PyObject_HEAD @@ -103,9 +137,8 @@ under :ref:`reference counting `. Get the type of the Python object *o*. - Return a :term:`borrowed reference`. - - Use the :c:func:`Py_SET_TYPE` function to set an object type. + The returned reference is :term:`borrowed ` from *o*. + Do not release it with :c:func:`Py_DECREF` or similar. .. versionchanged:: 3.11 :c:func:`Py_TYPE()` is changed to an inline static function. @@ -122,16 +155,26 @@ under :ref:`reference counting `. .. c:function:: void Py_SET_TYPE(PyObject *o, PyTypeObject *type) - Set the object *o* type to *type*. + Set the type of object *o* to *type*, without any checking or reference + counting. + + This is a very low-level operation. + Consider instead setting the Python attribute :attr:`~object.__class__` + using :c:func:`PyObject_SetAttrString` or similar. + + Note that assigning an incompatible type can lead to undefined behavior. + + If *type* is a :ref:`heap type `, the caller must create a + new reference to it. + Similarly, if the old type of *o* is a heap type, the caller must release + a reference to that type. .. versionadded:: 3.9 .. c:function:: Py_ssize_t Py_SIZE(PyVarObject *o) - Get the size of the Python object *o*. - - Use the :c:func:`Py_SET_SIZE` function to set an object size. + Get the :c:member:`~PyVarObject.ob_size` field of *o*. .. versionchanged:: 3.11 :c:func:`Py_SIZE()` is changed to an inline static function. @@ -140,7 +183,7 @@ under :ref:`reference counting `. .. c:function:: void Py_SET_SIZE(PyVarObject *o, Py_ssize_t size) - Set the object *o* size to *size*. + Set the :c:member:`~PyVarObject.ob_size` field of *o* to *size*. .. versionadded:: 3.9 @@ -365,7 +408,7 @@ There are these calling conventions: These two constants are not used to indicate the calling convention but the -binding when use with methods of classes. These may not be used for functions +binding when used with methods of classes. These may not be used for functions defined for modules. At most one of these flags may be set for any given method. @@ -404,6 +447,25 @@ definition with the same method name. slot. This is helpful because calls to PyCFunctions are optimized more than wrapper object calls. + +.. c:var:: PyTypeObject PyCMethod_Type + + The type object corresponding to Python C method objects. This is + available as :class:`types.BuiltinMethodType` in the Python layer. + + +.. c:function:: int PyCMethod_Check(PyObject *op) + + Return true if *op* is an instance of the :c:type:`PyCMethod_Type` type + or a subtype of it. This function always succeeds. + + +.. c:function:: int PyCMethod_CheckExact(PyObject *op) + + This is the same as :c:func:`PyCMethod_Check`, but does not account for + subtypes. + + .. c:function:: PyObject * PyCMethod_New(PyMethodDef *ml, PyObject *self, PyObject *module, PyTypeObject *cls) Turn *ml* into a Python :term:`callable` object. @@ -429,6 +491,24 @@ definition with the same method name. .. versionadded:: 3.9 +.. c:var:: PyTypeObject PyCFunction_Type + + The type object corresponding to Python C function objects. This is + available as :class:`types.BuiltinFunctionType` in the Python layer. + + +.. c:function:: int PyCFunction_Check(PyObject *op) + + Return true if *op* is an instance of the :c:type:`PyCFunction_Type` type + or a subtype of it. This function always succeeds. + + +.. c:function:: int PyCFunction_CheckExact(PyObject *op) + + This is the same as :c:func:`PyCFunction_Check`, but does not account for + subtypes. + + .. c:function:: PyObject * PyCFunction_NewEx(PyMethodDef *ml, PyObject *self, PyObject *module) Equivalent to ``PyCMethod_New(ml, self, module, NULL)``. @@ -439,6 +519,62 @@ definition with the same method name. Equivalent to ``PyCMethod_New(ml, self, NULL, NULL)``. +.. c:function:: int PyCFunction_GetFlags(PyObject *func) + + Get the function's flags on *func* as they were passed to + :c:member:`~PyMethodDef.ml_flags`. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns the function's flags on success, and ``-1`` with an + exception set on failure. + + +.. c:function:: int PyCFunction_GET_FLAGS(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetFlags`, but without error + or type checking. + + +.. c:function:: PyCFunction PyCFunction_GetFunction(PyObject *func) + + Get the function pointer on *func* as it was passed to + :c:member:`~PyMethodDef.ml_meth`. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns the function pointer on success, and ``NULL`` with an + exception set on failure. + + +.. c:function:: int PyCFunction_GET_FUNCTION(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetFunction`, but without error + or type checking. + + +.. c:function:: PyObject *PyCFunction_GetSelf(PyObject *func) + + Get the "self" object on *func*. This is the object that would be passed + to the first argument of a :c:type:`PyCFunction`. For C function objects + created through a :c:type:`PyMethodDef` on a :c:type:`PyModuleDef`, this + is the resulting module object. + + If *func* is not a C function object, this fails with an exception. + *func* must not be ``NULL``. + + This function returns a :term:`borrowed reference` to the "self" object + on success, and ``NULL`` with an exception set on failure. + + +.. c:function:: PyObject *PyCFunction_GET_SELF(PyObject *func) + + This is the same as :c:func:`PyCFunction_GetSelf`, but without error or + type checking. + + Accessing attributes of extension types --------------------------------------- @@ -562,14 +698,12 @@ The following flags can be used with :c:member:`PyMemberDef.flags`: entry indicates an offset from the subclass-specific data, rather than from ``PyObject``. - Can only be used as part of :c:member:`Py_tp_members ` + Can only be used as part of the :c:data:`Py_tp_members` :c:type:`slot ` when creating a class using negative :c:member:`~PyType_Spec.basicsize`. It is mandatory in that case. - - This flag is only used in :c:type:`PyType_Slot`. - When setting :c:member:`~PyTypeObject.tp_members` during - class creation, Python clears it and sets + When setting :c:member:`~PyTypeObject.tp_members` from the slot during + class creation, Python clears the flag and sets :c:member:`PyMemberDef.offset` to the offset from the ``PyObject`` struct. .. index:: diff --git a/Doc/c-api/subinterpreters.rst b/Doc/c-api/subinterpreters.rst new file mode 100644 index 000000000000000..44e3fc96841aacb --- /dev/null +++ b/Doc/c-api/subinterpreters.rst @@ -0,0 +1,470 @@ +.. highlight:: c + +.. _sub-interpreter-support: + +Multiple interpreters in a Python process +========================================= + +While in most uses, you will only embed a single Python interpreter, there +are cases where you need to create several independent interpreters in the +same process and perhaps even in the same thread. Sub-interpreters allow +you to do that. + +The "main" interpreter is the first one created when the runtime initializes. +It is usually the only Python interpreter in a process. Unlike sub-interpreters, +the main interpreter has unique process-global responsibilities like signal +handling. It is also responsible for execution during runtime initialization and +is usually the active interpreter during runtime finalization. The +:c:func:`PyInterpreterState_Main` function returns a pointer to its state. + +You can switch between sub-interpreters using the :c:func:`PyThreadState_Swap` +function. You can create and destroy them using the following functions: + + +.. c:type:: PyInterpreterConfig + + Structure containing most parameters to configure a sub-interpreter. + Its values are used only in :c:func:`Py_NewInterpreterFromConfig` and + never modified by the runtime. + + .. versionadded:: 3.12 + + Structure fields: + + .. c:member:: int use_main_obmalloc + + If this is ``0`` then the sub-interpreter will use its own + "object" allocator state. + Otherwise it will use (share) the main interpreter's. + + If this is ``0`` then + :c:member:`~PyInterpreterConfig.check_multi_interp_extensions` + must be ``1`` (non-zero). + If this is ``1`` then :c:member:`~PyInterpreterConfig.gil` + must not be :c:macro:`PyInterpreterConfig_OWN_GIL`. + + .. c:member:: int allow_fork + + If this is ``0`` then the runtime will not support forking the + process in any thread where the sub-interpreter is currently active. + Otherwise fork is unrestricted. + + Note that the :mod:`subprocess` module still works + when fork is disallowed. + + .. c:member:: int allow_exec + + If this is ``0`` then the runtime will not support replacing the + current process via exec (e.g. :func:`os.execv`) in any thread + where the sub-interpreter is currently active. + Otherwise exec is unrestricted. + + Note that the :mod:`subprocess` module still works + when exec is disallowed. + + .. c:member:: int allow_threads + + If this is ``0`` then the sub-interpreter's :mod:`threading` module + won't create threads. + Otherwise threads are allowed. + + .. c:member:: int allow_daemon_threads + + If this is ``0`` then the sub-interpreter's :mod:`threading` module + won't create daemon threads. + Otherwise daemon threads are allowed (as long as + :c:member:`~PyInterpreterConfig.allow_threads` is non-zero). + + .. c:member:: int check_multi_interp_extensions + + If this is ``0`` then all extension modules may be imported, + including legacy (single-phase init) modules, + in any thread where the sub-interpreter is currently active. + Otherwise only multi-phase init extension modules + (see :pep:`489`) may be imported. + (Also see :c:macro:`Py_mod_multiple_interpreters`.) + + This must be ``1`` (non-zero) if + :c:member:`~PyInterpreterConfig.use_main_obmalloc` is ``0``. + + .. c:member:: int gil + + This determines the operation of the GIL for the sub-interpreter. + It may be one of the following: + + .. c:namespace:: NULL + + .. c:macro:: PyInterpreterConfig_DEFAULT_GIL + + Use the default selection (:c:macro:`PyInterpreterConfig_SHARED_GIL`). + + .. c:macro:: PyInterpreterConfig_SHARED_GIL + + Use (share) the main interpreter's GIL. + + .. c:macro:: PyInterpreterConfig_OWN_GIL + + Use the sub-interpreter's own GIL. + + If this is :c:macro:`PyInterpreterConfig_OWN_GIL` then + :c:member:`PyInterpreterConfig.use_main_obmalloc` must be ``0``. + + +.. c:function:: PyStatus Py_NewInterpreterFromConfig(PyThreadState **tstate_p, const PyInterpreterConfig *config) + + .. index:: + pair: module; builtins + pair: module; __main__ + pair: module; sys + single: stdout (in module sys) + single: stderr (in module sys) + single: stdin (in module sys) + + Create a new sub-interpreter. This is an (almost) totally separate environment + for the execution of Python code. In particular, the new interpreter has + separate, independent versions of all imported modules, including the + fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. The + table of loaded modules (``sys.modules``) and the module search path + (``sys.path``) are also separate. The new environment has no ``sys.argv`` + variable. It has new standard I/O stream file objects ``sys.stdin``, + ``sys.stdout`` and ``sys.stderr`` (however these refer to the same underlying + file descriptors). + + The given *config* controls the options with which the interpreter + is initialized. + + Upon success, *tstate_p* will be set to the first :term:`thread state` + created in the new sub-interpreter. This thread state is + :term:`attached `. + Note that no actual thread is created; see the discussion of thread states + below. If creation of the new interpreter is unsuccessful, + *tstate_p* is set to ``NULL``; + no exception is set since the exception state is stored in the + :term:`attached thread state`, which might not exist. + + Like all other Python/C API functions, an :term:`attached thread state` + must be present before calling this function, but it might be detached upon + returning. On success, the returned thread state will be :term:`attached `. + If the sub-interpreter is created with its own :term:`GIL` then the + :term:`attached thread state` of the calling interpreter will be detached. + When the function returns, the new interpreter's :term:`thread state` + will be :term:`attached ` to the current thread and + the previous interpreter's :term:`attached thread state` will remain detached. + + .. versionadded:: 3.12 + + Sub-interpreters are most effective when isolated from each other, + with certain functionality restricted:: + + PyInterpreterConfig config = { + .use_main_obmalloc = 0, + .allow_fork = 0, + .allow_exec = 0, + .allow_threads = 1, + .allow_daemon_threads = 0, + .check_multi_interp_extensions = 1, + .gil = PyInterpreterConfig_OWN_GIL, + }; + PyThreadState *tstate = NULL; + PyStatus status = Py_NewInterpreterFromConfig(&tstate, &config); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + + Note that the config is used only briefly and does not get modified. + During initialization the config's values are converted into various + :c:type:`PyInterpreterState` values. A read-only copy of the config + may be stored internally on the :c:type:`PyInterpreterState`. + + .. index:: + single: Py_FinalizeEx (C function) + single: Py_Initialize (C function) + + Extension modules are shared between (sub-)interpreters as follows: + + * For modules using multi-phase initialization, + e.g. :c:func:`PyModule_FromDefAndSpec`, a separate module object is + created and initialized for each interpreter. + Only C-level static and global variables are shared between these + module objects. + + * For modules using legacy + :ref:`single-phase initialization `, + e.g. :c:func:`PyModule_Create`, the first time a particular extension + is imported, it is initialized normally, and a (shallow) copy of its + module's dictionary is squirreled away. + When the same extension is imported by another (sub-)interpreter, a new + module is initialized and filled with the contents of this copy; the + extension's ``init`` function is not called. + Objects in the module's dictionary thus end up shared across + (sub-)interpreters, which might cause unwanted behavior (see + `Bugs and caveats`_ below). + + Note that this is different from what happens when an extension is + imported after the interpreter has been completely re-initialized by + calling :c:func:`Py_FinalizeEx` and :c:func:`Py_Initialize`; in that + case, the extension's ``initmodule`` function *is* called again. + As with multi-phase initialization, this means that only C-level static + and global variables are shared between these modules. + + .. index:: single: close (in module os) + + +.. c:function:: PyThreadState* Py_NewInterpreter(void) + + .. index:: + pair: module; builtins + pair: module; __main__ + pair: module; sys + single: stdout (in module sys) + single: stderr (in module sys) + single: stdin (in module sys) + + Create a new sub-interpreter. This is essentially just a wrapper + around :c:func:`Py_NewInterpreterFromConfig` with a config that + preserves the existing behavior. The result is an unisolated + sub-interpreter that shares the main interpreter's GIL, allows + fork/exec, allows daemon threads, and allows single-phase init + modules. + + +.. c:function:: void Py_EndInterpreter(PyThreadState *tstate) + + .. index:: single: Py_FinalizeEx (C function) + + Destroy the (sub-)interpreter represented by the given :term:`thread state`. + The given thread state must be :term:`attached `. + When the call returns, there will be no :term:`attached thread state`. + All thread states associated with this interpreter are destroyed. + + :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that + haven't been explicitly destroyed at that point. + + +.. _per-interpreter-gil: + +A per-interpreter GIL +--------------------- + +.. versionadded:: 3.12 + +Using :c:func:`Py_NewInterpreterFromConfig` you can create +a sub-interpreter that is completely isolated from other interpreters, +including having its own GIL. The most important benefit of this +isolation is that such an interpreter can execute Python code without +being blocked by other interpreters or blocking any others. Thus a +single Python process can truly take advantage of multiple CPU cores +when running Python code. The isolation also encourages a different +approach to concurrency than that of just using threads. +(See :pep:`554` and :pep:`684`.) + +Using an isolated interpreter requires vigilance in preserving that +isolation. That especially means not sharing any objects or mutable +state without guarantees about thread-safety. Even objects that are +otherwise immutable (e.g. ``None``, ``(1, 5)``) can't normally be shared +because of the refcount. One simple but less-efficient approach around +this is to use a global lock around all use of some state (or object). +Alternately, effectively immutable objects (like integers or strings) +can be made safe in spite of their refcounts by making them :term:`immortal`. +In fact, this has been done for the builtin singletons, small integers, +and a number of other builtin objects. + +If you preserve isolation then you will have access to proper multi-core +computing without the complications that come with free-threading. +Failure to preserve isolation will expose you to the full consequences +of free-threading, including races and hard-to-debug crashes. + +Aside from that, one of the main challenges of using multiple isolated +interpreters is how to communicate between them safely (not break +isolation) and efficiently. The runtime and stdlib do not provide +any standard approach to this yet. A future stdlib module would help +mitigate the effort of preserving isolation and expose effective tools +for communicating (and sharing) data between interpreters. + + +Bugs and caveats +---------------- + +Because sub-interpreters (and the main interpreter) are part of the same +process, the insulation between them isn't perfect --- for example, using +low-level file operations like :func:`os.close` they can +(accidentally or maliciously) affect each other's open files. Because of the +way extensions are shared between (sub-)interpreters, some extensions may not +work properly; this is especially likely when using single-phase initialization +or (static) global variables. +It is possible to insert objects created in one sub-interpreter into +a namespace of another (sub-)interpreter; this should be avoided if possible. + +Special care should be taken to avoid sharing user-defined functions, +methods, instances or classes between sub-interpreters, since import +operations executed by such objects may affect the wrong (sub-)interpreter's +dictionary of loaded modules. It is equally important to avoid sharing +objects from which the above are reachable. + +Also note that combining this functionality with ``PyGILState_*`` APIs +is delicate, because these APIs assume a bijection between Python thread states +and OS-level threads, an assumption broken by the presence of sub-interpreters. +It is highly recommended that you don't switch sub-interpreters between a pair +of matching :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` calls. +Furthermore, extensions (such as :mod:`ctypes`) using these APIs to allow calling +of Python code from non-Python created threads will probably be broken when using +sub-interpreters. + + +High-level APIs +--------------- + +.. c:type:: PyInterpreterState + + This data structure represents the state shared by a number of cooperating + threads. Threads belonging to the same interpreter share their module + administration and a few other internal items. There are no public members in + this structure. + + Threads belonging to different interpreters initially share nothing, except + process state like available memory, open file descriptors and such. The global + interpreter lock is also shared by all threads, regardless of to which + interpreter they belong. + + .. versionchanged:: 3.12 + + :pep:`684` introduced the possibility + of a :ref:`per-interpreter GIL `. + See :c:func:`Py_NewInterpreterFromConfig`. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Get(void) + + Get the current interpreter. + + Issue a fatal error if there is no :term:`attached thread state`. + It cannot return NULL. + + .. versionadded:: 3.9 + + +.. c:function:: int64_t PyInterpreterState_GetID(PyInterpreterState *interp) + + Return the interpreter's unique ID. If there was any error in doing + so then ``-1`` is returned and an error is set. + + The caller must have an :term:`attached thread state`. + + .. versionadded:: 3.7 + + +.. c:function:: PyObject* PyInterpreterState_GetDict(PyInterpreterState *interp) + + Return a dictionary in which interpreter-specific data may be stored. + If this function returns ``NULL`` then no exception has been raised and + the caller should assume no interpreter-specific dict is available. + + This is not a replacement for :c:func:`PyModule_GetState()`, which + extensions should use to store interpreter-specific state information. + + The returned dictionary is borrowed from the interpreter and is valid until + interpreter shutdown. + + .. versionadded:: 3.8 + + +.. c:type:: PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) + + Type of a frame evaluation function. + + The *throwflag* parameter is used by the ``throw()`` method of generators: + if non-zero, handle the current exception. + + .. versionchanged:: 3.9 + The function now takes a *tstate* parameter. + + .. versionchanged:: 3.11 + The *frame* parameter changed from ``PyFrameObject*`` to ``_PyInterpreterFrame*``. + + +.. c:function:: _PyFrameEvalFunction _PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp) + + Get the frame evaluation function. + + See the :pep:`523` "Adding a frame evaluation API to CPython". + + .. versionadded:: 3.9 + + +.. c:function:: void _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame) + + Set the frame evaluation function. + + See the :pep:`523` "Adding a frame evaluation API to CPython". + + .. versionadded:: 3.9 + + +Low-level APIs +-------------- + +All of the following functions must be called after :c:func:`Py_Initialize`. + +.. versionchanged:: 3.7 + :c:func:`Py_Initialize()` now initializes the :term:`GIL` + and sets an :term:`attached thread state`. + + +.. c:function:: PyInterpreterState* PyInterpreterState_New() + + Create a new interpreter state object. An :term:`attached thread state` is not needed, + but may optionally exist if it is necessary to serialize calls to this + function. + + .. audit-event:: cpython.PyInterpreterState_New "" c.PyInterpreterState_New + + +.. c:function:: void PyInterpreterState_Clear(PyInterpreterState *interp) + + Reset all information in an interpreter state object. There must be + an :term:`attached thread state` for the interpreter. + + .. audit-event:: cpython.PyInterpreterState_Clear "" c.PyInterpreterState_Clear + + +.. c:function:: void PyInterpreterState_Delete(PyInterpreterState *interp) + + Destroy an interpreter state object. There **should not** be an + :term:`attached thread state` for the target interpreter. The interpreter + state must have been reset with a previous call to :c:func:`PyInterpreterState_Clear`. + + +.. _advanced-debugging: + +Advanced debugger support +------------------------- + +These functions are only intended to be used by advanced debugging tools. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Head() + + Return the interpreter state object at the head of the list of all such objects. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Main() + + Return the main interpreter state object. + + +.. c:function:: PyInterpreterState* PyInterpreterState_Next(PyInterpreterState *interp) + + Return the next interpreter state object after *interp* from the list of all + such objects. + + +.. c:function:: PyThreadState * PyInterpreterState_ThreadHead(PyInterpreterState *interp) + + Return the pointer to the first :c:type:`PyThreadState` object in the list of + threads associated with the interpreter *interp*. + + +.. c:function:: PyThreadState* PyThreadState_Next(PyThreadState *tstate) + + Return the next thread state object after *tstate* from the list of all such + objects belonging to the same :c:type:`PyInterpreterState` object. diff --git a/Doc/c-api/synchronization.rst b/Doc/c-api/synchronization.rst new file mode 100644 index 000000000000000..954ac55e53813f8 --- /dev/null +++ b/Doc/c-api/synchronization.rst @@ -0,0 +1,301 @@ +.. highlight:: c + +.. _synchronization: + +Synchronization primitives +========================== + +The C-API provides a basic mutual exclusion lock. + +.. c:type:: PyMutex + + A mutual exclusion lock. The :c:type:`!PyMutex` should be initialized to + zero to represent the unlocked state. For example:: + + PyMutex mutex = {0}; + + Instances of :c:type:`!PyMutex` should not be copied or moved. Both the + contents and address of a :c:type:`!PyMutex` are meaningful, and it must + remain at a fixed, writable location in memory. + + .. note:: + + A :c:type:`!PyMutex` currently occupies one byte, but the size should be + considered unstable. The size may change in future Python releases + without a deprecation period. + + .. versionadded:: 3.13 + +.. c:function:: void PyMutex_Lock(PyMutex *m) + + Lock mutex *m*. If another thread has already locked it, the calling + thread will block until the mutex is unlocked. While blocked, the thread + will temporarily detach the :term:`thread state ` if one exists. + + .. versionadded:: 3.13 + +.. c:function:: void PyMutex_Unlock(PyMutex *m) + + Unlock mutex *m*. The mutex must be locked --- otherwise, the function will + issue a fatal error. + + .. versionadded:: 3.13 + +.. c:function:: int PyMutex_IsLocked(PyMutex *m) + + Returns non-zero if the mutex *m* is currently locked, zero otherwise. + + .. note:: + + This function is intended for use in assertions and debugging only and + should not be used to make concurrency control decisions, as the lock + state may change immediately after the check. + + .. versionadded:: 3.14 + +.. _python-critical-section-api: + +Python critical section API +--------------------------- + +The critical section API provides a deadlock avoidance layer on top of +per-object locks for :term:`free-threaded ` CPython. They are +intended to replace reliance on the :term:`global interpreter lock`, and are +no-ops in versions of Python with the global interpreter lock. + +Critical sections are intended to be used for custom types implemented +in C-API extensions. They should generally not be used with built-in types like +:class:`list` and :class:`dict` because their public C-APIs +already use critical sections internally, with the notable +exception of :c:func:`PyDict_Next`, which requires critical section +to be acquired externally. + +Critical sections avoid deadlocks by implicitly suspending active critical +sections, hence, they do not provide exclusive access such as provided by +traditional locks like :c:type:`PyMutex`. When a critical section is started, +the per-object lock for the object is acquired. If the code executed inside the +critical section calls C-API functions then it can suspend the critical section thereby +releasing the per-object lock, so other threads can acquire the per-object lock +for the same object. + +Variants that accept :c:type:`PyMutex` pointers rather than Python objects are also +available. Use these variants to start a critical section in a situation where +there is no :c:type:`PyObject` -- for example, when working with a C type that +does not extend or wrap :c:type:`PyObject` but still needs to call into the C +API in a manner that might lead to deadlocks. + +The functions and structs used by the macros are exposed for cases +where C macros are not available. They should only be used as in the +given macro expansions. Note that the sizes and contents of the structures may +change in future Python versions. + +.. note:: + + Operations that need to lock two objects at once must use + :c:macro:`Py_BEGIN_CRITICAL_SECTION2`. You *cannot* use nested critical + sections to lock more than one object at once, because the inner critical + section may suspend the outer critical sections. This API does not provide + a way to lock more than two objects at once. + +Example usage:: + + static PyObject * + set_field(MyObject *self, PyObject *value) + { + Py_BEGIN_CRITICAL_SECTION(self); + Py_SETREF(self->field, Py_XNewRef(value)); + Py_END_CRITICAL_SECTION(); + Py_RETURN_NONE; + } + +In the above example, :c:macro:`Py_SETREF` calls :c:macro:`Py_DECREF`, which +can call arbitrary code through an object's deallocation function. The critical +section API avoids potential deadlocks due to reentrancy and lock ordering +by allowing the runtime to temporarily suspend the critical section if the +code triggered by the finalizer blocks and calls :c:func:`PyEval_SaveThread`. + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION(op) + + Acquires the per-object lock for the object *op* and begins a + critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection _py_cs; + PyCriticalSection_Begin(&_py_cs, (PyObject*)(op)) + + In the default build, this macro expands to ``{``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION_MUTEX(m) + + Locks the mutex *m* and begins a critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection _py_cs; + PyCriticalSection_BeginMutex(&_py_cs, m) + + Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION`, there is no cast for + the argument of the macro - it must be a :c:type:`PyMutex` pointer. + + On the default build, this macro expands to ``{``. + + .. versionadded:: 3.14 + +.. c:macro:: Py_END_CRITICAL_SECTION() + + Ends the critical section and releases the per-object lock. + + In the free-threaded build, this macro expands to:: + + PyCriticalSection_End(&_py_cs); + } + + In the default build, this macro expands to ``}``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION2(a, b) + + Acquires the per-object locks for the objects *a* and *b* and begins a + critical section. The locks are acquired in a consistent order (lowest + address first) to avoid lock ordering deadlocks. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection2 _py_cs2; + PyCriticalSection2_Begin(&_py_cs2, (PyObject*)(a), (PyObject*)(b)) + + In the default build, this macro expands to ``{``. + + .. versionadded:: 3.13 + +.. c:macro:: Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) + + Locks the mutexes *m1* and *m2* and begins a critical section. + + In the free-threaded build, this macro expands to:: + + { + PyCriticalSection2 _py_cs2; + PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) + + Note that unlike :c:macro:`Py_BEGIN_CRITICAL_SECTION2`, there is no cast for + the arguments of the macro - they must be :c:type:`PyMutex` pointers. + + On the default build, this macro expands to ``{``. + + .. versionadded:: 3.14 + +.. c:macro:: Py_END_CRITICAL_SECTION2() + + Ends the critical section and releases the per-object locks. + + In the free-threaded build, this macro expands to:: + + PyCriticalSection2_End(&_py_cs2); + } + + In the default build, this macro expands to ``}``. + + .. versionadded:: 3.13 + + +Legacy locking APIs +------------------- + +These APIs are obsolete since Python 3.13 with the introduction of +:c:type:`PyMutex`. + + +.. c:type:: PyThread_type_lock + + A pointer to a mutual exclusion lock. + + +.. c:type:: PyLockStatus + + The result of acquiring a lock with a timeout. + + .. c:namespace:: NULL + + .. c:enumerator:: PY_LOCK_FAILURE + + Failed to acquire the lock. + + .. c:enumerator:: PY_LOCK_ACQUIRED + + The lock was successfully acquired. + + .. c:enumerator:: PY_LOCK_INTR + + The lock was interrupted by a signal. + + +.. c:function:: PyThread_type_lock PyThread_allocate_lock(void) + + Allocate a new lock. + + On success, this function returns a lock; on failure, this + function returns ``0`` without an exception set. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: void PyThread_free_lock(PyThread_type_lock lock) + + Destroy *lock*. The lock should not be held by any thread when calling + this. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: PyLockStatus PyThread_acquire_lock_timed(PyThread_type_lock lock, long long microseconds, int intr_flag) + + Acquire *lock* with a timeout. + + This will wait for *microseconds* microseconds to acquire the lock. If the + timeout expires, this function returns :c:enumerator:`PY_LOCK_FAILURE`. + If *microseconds* is ``-1``, this will wait indefinitely until the lock has + been released. + + If *intr_flag* is ``1``, acquiring the lock may be interrupted by a signal, + in which case this function returns :c:enumerator:`PY_LOCK_INTR`. Upon + interruption, it's generally expected that the caller makes a call to + :c:func:`Py_MakePendingCalls` to propagate an exception to Python code. + + If the lock is successfully acquired, this function returns + :c:enumerator:`PY_LOCK_ACQUIRED`. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: int PyThread_acquire_lock(PyThread_type_lock lock, int waitflag) + + Acquire *lock*. + + If *waitflag* is ``1`` and another thread currently holds the lock, this + function will wait until the lock can be acquired and will always return + ``1``. + + If *waitflag* is ``0`` and another thread holds the lock, this function will + not wait and instead return ``0``. If the lock is not held by any other + thread, then this function will acquire it and return ``1``. + + Unlike :c:func:`PyThread_acquire_lock_timed`, acquiring the lock cannot be + interrupted by a signal. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: int PyThread_release_lock(PyThread_type_lock lock) + + Release *lock*. If *lock* is not held, then this function issues a + fatal error. + + The caller does not need to hold an :term:`attached thread state`. diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index b3c89800e386ff1..3a8cfd95b03513b 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -123,6 +123,24 @@ Operating System Utilities This is a thin wrapper around either :c:func:`!sigaction` or :c:func:`!signal`. Do not call those functions directly! + +.. c:function:: int PyOS_InterruptOccurred(void) + + Check if a :c:macro:`!SIGINT` signal has been received. + + Returns ``1`` if a :c:macro:`!SIGINT` has occurred and clears the signal flag, + or ``0`` otherwise. + + In most cases, you should prefer :c:func:`PyErr_CheckSignals` over this function. + :c:func:`!PyErr_CheckSignals` invokes the appropriate signal handlers + for all pending signals, allowing Python code to handle the signal properly. + This function only detects :c:macro:`!SIGINT` and does not invoke any Python + signal handlers. + + This function is async-signal-safe and this function cannot fail. + The caller must hold an :term:`attached thread state`. + + .. c:function:: wchar_t* Py_DecodeLocale(const char* arg, size_t *size) .. warning:: diff --git a/Doc/c-api/threads.rst b/Doc/c-api/threads.rst new file mode 100644 index 000000000000000..7a528d712a3edd1 --- /dev/null +++ b/Doc/c-api/threads.rst @@ -0,0 +1,867 @@ +.. highlight:: c + +.. _threads: + +Thread states and the global interpreter lock +============================================= + +.. index:: + single: global interpreter lock + single: interpreter lock + single: lock, interpreter + +Unless on a :term:`free-threaded build` of :term:`CPython`, +the Python interpreter is generally not thread-safe. In order to support +multi-threaded Python programs, there's a global lock, called the :term:`global +interpreter lock` or :term:`GIL`, that must be held by a thread before +accessing Python objects. Without the lock, even the simplest operations +could cause problems in a multi-threaded program: for example, when +two threads simultaneously increment the reference count of the same object, the +reference count could end up being incremented only once instead of twice. + +As such, only a thread that holds the GIL may operate on Python objects or +invoke Python's C API. + +.. index:: single: setswitchinterval (in module sys) + +In order to emulate concurrency, the interpreter regularly tries to switch +threads between bytecode instructions (see :func:`sys.setswitchinterval`). +This is why locks are also necessary for thread-safety in pure-Python code. + +Additionally, the global interpreter lock is released around blocking I/O +operations, such as reading or writing to a file. From the C API, this is done +by :ref:`detaching the thread state `. + + +.. index:: + single: PyThreadState (C type) + +The Python interpreter keeps some thread-local information inside +a data structure called :c:type:`PyThreadState`, known as a :term:`thread state`. +Each thread has a thread-local pointer to a :c:type:`PyThreadState`; a thread state +referenced by this pointer is considered to be :term:`attached `. + +A thread can only have one :term:`attached thread state` at a time. An attached +thread state is typically analogous with holding the GIL, except on +free-threaded builds. On builds with the GIL enabled, attaching a thread state +will block until the GIL can be acquired. However, even on builds with the GIL +disabled, it is still required to have an attached thread state, as the interpreter +needs to keep track of which threads may access Python objects. + +.. note:: + + Even on the free-threaded build, attaching a thread state may block, as the + GIL can be re-enabled or threads might be temporarily suspended (such as during + a garbage collection). + +Generally, there will always be an attached thread state when using Python's +C API, including during embedding and when implementing methods, so it's uncommon +to need to set up a thread state on your own. Only in some specific cases, such +as in a :c:macro:`Py_BEGIN_ALLOW_THREADS` block or in a fresh thread, will the +thread not have an attached thread state. +If uncertain, check if :c:func:`PyThreadState_GetUnchecked` returns ``NULL``. + +If it turns out that you do need to create a thread state, call :c:func:`PyThreadState_New` +followed by :c:func:`PyThreadState_Swap`, or use the dangerous +:c:func:`PyGILState_Ensure` function. + + +.. _detaching-thread-state: + +Detaching the thread state from extension code +---------------------------------------------- + +Most extension code manipulating the :term:`thread state` has the following simple +structure:: + + Save the thread state in a local variable. + ... Do some blocking I/O operation ... + Restore the thread state from the local variable. + +This is so common that a pair of macros exists to simplify it:: + + Py_BEGIN_ALLOW_THREADS + ... Do some blocking I/O operation ... + Py_END_ALLOW_THREADS + +.. index:: + single: Py_BEGIN_ALLOW_THREADS (C macro) + single: Py_END_ALLOW_THREADS (C macro) + +The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a +hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the +block. + +The block above expands to the following code:: + + PyThreadState *_save; + + _save = PyEval_SaveThread(); + ... Do some blocking I/O operation ... + PyEval_RestoreThread(_save); + +.. index:: + single: PyEval_RestoreThread (C function) + single: PyEval_SaveThread (C function) + +Here is how these functions work: + +The attached thread state implies that the GIL is held for the interpreter. +To detach it, :c:func:`PyEval_SaveThread` is called and the result is stored +in a local variable. + +By detaching the thread state, the GIL is released, which allows other threads +to attach to the interpreter and execute while the current thread performs +blocking I/O. When the I/O operation is complete, the old thread state is +reattached by calling :c:func:`PyEval_RestoreThread`, which will wait until +the GIL can be acquired. + +.. note:: + Performing blocking I/O is the most common use case for detaching + the thread state, but it is also useful to call it over long-running + native code that doesn't need access to Python objects or Python's C API. + For example, the standard :mod:`zlib` and :mod:`hashlib` modules detach the + :term:`thread state ` when compressing or hashing + data. + +On a :term:`free-threaded build`, the :term:`GIL` is usually out of the question, +but **detaching the thread state is still required**, because the interpreter +periodically needs to block all threads to get a consistent view of Python objects +without the risk of race conditions. +For example, CPython currently suspends all threads for a short period of time +while running the garbage collector. + +.. warning:: + + Detaching the thread state can lead to unexpected behavior during interpreter + finalization. See :ref:`cautions-regarding-runtime-finalization` for more + details. + + +APIs +^^^^ + +The following macros are normally used without a trailing semicolon; look for +example usage in the Python source distribution. + +.. note:: + + These macros are still necessary on the :term:`free-threaded build` to prevent + deadlocks. + +.. c:macro:: Py_BEGIN_ALLOW_THREADS + + This macro expands to ``{ PyThreadState *_save; _save = PyEval_SaveThread();``. + Note that it contains an opening brace; it must be matched with a following + :c:macro:`Py_END_ALLOW_THREADS` macro. See above for further discussion of this + macro. + + +.. c:macro:: Py_END_ALLOW_THREADS + + This macro expands to ``PyEval_RestoreThread(_save); }``. Note that it contains + a closing brace; it must be matched with an earlier + :c:macro:`Py_BEGIN_ALLOW_THREADS` macro. See above for further discussion of + this macro. + + +.. c:macro:: Py_BLOCK_THREADS + + This macro expands to ``PyEval_RestoreThread(_save);``: it is equivalent to + :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. + + +.. c:macro:: Py_UNBLOCK_THREADS + + This macro expands to ``_save = PyEval_SaveThread();``: it is equivalent to + :c:macro:`Py_BEGIN_ALLOW_THREADS` without the opening brace and variable + declaration. + + +Non-Python created threads +-------------------------- + +When threads are created using the dedicated Python APIs (such as the +:mod:`threading` module), a thread state is automatically associated with them, +However, when a thread is created from native code (for example, by a +third-party library with its own thread management), it doesn't hold an +attached thread state. + +If you need to call Python code from these threads (often this will be part +of a callback API provided by the aforementioned third-party library), +you must first register these threads with the interpreter by +creating a new thread state and attaching it. + +The most robust way to do this is through :c:func:`PyThreadState_New` followed +by :c:func:`PyThreadState_Swap`. + +.. note:: + ``PyThreadState_New`` requires an argument pointing to the desired + interpreter; such a pointer can be acquired via a call to + :c:func:`PyInterpreterState_Get` from the code where the thread was + created. + +For example:: + + /* The return value of PyInterpreterState_Get() from the + function that created this thread. */ + PyInterpreterState *interp = thread_data->interp; + + /* Create a new thread state for the interpreter. It does not start out + attached. */ + PyThreadState *tstate = PyThreadState_New(interp); + + /* Attach the thread state, which will acquire the GIL. */ + PyThreadState_Swap(tstate); + + /* Perform Python actions here. */ + result = CallSomeFunction(); + /* evaluate result or handle exception */ + + /* Destroy the thread state. No Python API allowed beyond this point. */ + PyThreadState_Clear(tstate); + PyThreadState_DeleteCurrent(); + +.. warning:: + + If the interpreter finalized before ``PyThreadState_Swap`` was called, then + ``interp`` will be a dangling pointer! + +.. _gilstate: + +Legacy API +---------- + +Another common pattern to call Python code from a non-Python thread is to use +:c:func:`PyGILState_Ensure` followed by a call to :c:func:`PyGILState_Release`. + +These functions do not work well when multiple interpreters exist in the Python +process. If no Python interpreter has ever been used in the current thread (which +is common for threads created outside Python), ``PyGILState_Ensure`` will create +and attach a thread state for the "main" interpreter (the first interpreter in +the Python process). + +Additionally, these functions have thread-safety issues during interpreter +finalization. Using ``PyGILState_Ensure`` during finalization will likely +crash the process. + +Usage of these functions look like such:: + + PyGILState_STATE gstate; + gstate = PyGILState_Ensure(); + + /* Perform Python actions here. */ + result = CallSomeFunction(); + /* evaluate result or handle exception */ + + /* Release the thread. No Python API allowed beyond this point. */ + PyGILState_Release(gstate); + + +.. _fork-and-threads: + +Cautions about fork() +--------------------- + +Another important thing to note about threads is their behaviour in the face +of the C :c:func:`fork` call. On most systems with :c:func:`fork`, after a +process forks only the thread that issued the fork will exist. This has a +concrete impact both on how locks must be handled and on all stored state +in CPython's runtime. + +The fact that only the "current" thread remains +means any locks held by other threads will never be released. Python solves +this for :func:`os.fork` by acquiring the locks it uses internally before +the fork, and releasing them afterwards. In addition, it resets any +:ref:`lock-objects` in the child. When extending or embedding Python, there +is no way to inform Python of additional (non-Python) locks that need to be +acquired before or reset after a fork. OS facilities such as +:c:func:`!pthread_atfork` would need to be used to accomplish the same thing. +Additionally, when extending or embedding Python, calling :c:func:`fork` +directly rather than through :func:`os.fork` (and returning to or calling +into Python) may result in a deadlock by one of Python's internal locks +being held by a thread that is defunct after the fork. +:c:func:`PyOS_AfterFork_Child` tries to reset the necessary locks, but is not +always able to. + +The fact that all other threads go away also means that CPython's +runtime state there must be cleaned up properly, which :func:`os.fork` +does. This means finalizing all other :c:type:`PyThreadState` objects +belonging to the current interpreter and all other +:c:type:`PyInterpreterState` objects. Due to this and the special +nature of the :ref:`"main" interpreter `, +:c:func:`fork` should only be called in that interpreter's "main" +thread, where the CPython global runtime was originally initialized. +The only exception is if :c:func:`exec` will be called immediately +after. + + +High-level APIs +--------------- + +These are the most commonly used types and functions when writing multi-threaded +C extensions. + + +.. c:type:: PyThreadState + + This data structure represents the state of a single thread. The only public + data member is: + + .. c:member:: PyInterpreterState *interp + + This thread's interpreter state. + + +.. c:function:: void PyEval_InitThreads() + + .. index:: + single: PyEval_AcquireThread() + single: PyEval_ReleaseThread() + single: PyEval_SaveThread() + single: PyEval_RestoreThread() + + Deprecated function which does nothing. + + In Python 3.6 and older, this function created the GIL if it didn't exist. + + .. versionchanged:: 3.9 + The function now does nothing. + + .. versionchanged:: 3.7 + This function is now called by :c:func:`Py_Initialize()`, so you don't + have to call it yourself anymore. + + .. versionchanged:: 3.2 + This function cannot be called before :c:func:`Py_Initialize()` anymore. + + .. deprecated:: 3.9 + + .. index:: pair: module; _thread + + +.. c:function:: PyThreadState* PyEval_SaveThread() + + Detach the :term:`attached thread state` and return it. + The thread will have no :term:`thread state` upon returning. + + +.. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) + + Set the :term:`attached thread state` to *tstate*. + The passed :term:`thread state` **should not** be :term:`attached `, + otherwise deadlock ensues. *tstate* will be attached upon returning. + + .. note:: + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + +.. c:function:: PyThreadState* PyThreadState_Get() + + Return the :term:`attached thread state`. If the thread has no attached + thread state, (such as when inside of :c:macro:`Py_BEGIN_ALLOW_THREADS` + block), then this issues a fatal error (so that the caller needn't check + for ``NULL``). + + See also :c:func:`PyThreadState_GetUnchecked`. + +.. c:function:: PyThreadState* PyThreadState_GetUnchecked() + + Similar to :c:func:`PyThreadState_Get`, but don't kill the process with a + fatal error if it is NULL. The caller is responsible to check if the result + is NULL. + + .. versionadded:: 3.13 + In Python 3.5 to 3.12, the function was private and known as + ``_PyThreadState_UncheckedGet()``. + + +.. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) + + Set the :term:`attached thread state` to *tstate*, and return the + :term:`thread state` that was attached prior to calling. + + This function is safe to call without an :term:`attached thread state`; it + will simply return ``NULL`` indicating that there was no prior thread state. + + .. seealso:: + :c:func:`PyEval_ReleaseThread` + + .. note:: + Similar to :c:func:`PyGILState_Ensure`, this function will hang the + thread if the runtime is finalizing. + + +GIL-state APIs +-------------- + +The following functions use thread-local storage, and are not compatible +with sub-interpreters: + +.. c:type:: PyGILState_STATE + + The type of the value returned by :c:func:`PyGILState_Ensure` and passed to + :c:func:`PyGILState_Release`. + + .. c:enumerator:: PyGILState_LOCKED + + The GIL was already held when :c:func:`PyGILState_Ensure` was called. + + .. c:enumerator:: PyGILState_UNLOCKED + + The GIL was not held when :c:func:`PyGILState_Ensure` was called. + +.. c:function:: PyGILState_STATE PyGILState_Ensure() + + Ensure that the current thread is ready to call the Python C API regardless + of the current state of Python, or of the :term:`attached thread state`. This may + be called as many times as desired by a thread as long as each call is + matched with a call to :c:func:`PyGILState_Release`. In general, other + thread-related APIs may be used between :c:func:`PyGILState_Ensure` and + :c:func:`PyGILState_Release` calls as long as the thread state is restored to + its previous state before the Release(). For example, normal usage of the + :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is + acceptable. + + The return value is an opaque "handle" to the :term:`attached thread state` when + :c:func:`PyGILState_Ensure` was called, and must be passed to + :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even + though recursive calls are allowed, these handles *cannot* be shared - each + unique call to :c:func:`PyGILState_Ensure` must save the handle for its call + to :c:func:`PyGILState_Release`. + + When the function returns, there will be an :term:`attached thread state` + and the thread will be able to call arbitrary Python code. Failure is a fatal error. + + .. warning:: + Calling this function when the runtime is finalizing is unsafe. Doing + so will either hang the thread until the program ends, or fully crash + the interpreter in rare cases. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + +.. c:function:: void PyGILState_Release(PyGILState_STATE) + + Release any resources previously acquired. After this call, Python's state will + be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call + (but generally this state will be unknown to the caller, hence the use of the + GILState API). + + Every call to :c:func:`PyGILState_Ensure` must be matched by a call to + :c:func:`PyGILState_Release` on the same thread. + +.. c:function:: PyThreadState* PyGILState_GetThisThreadState() + + Get the :term:`attached thread state` for this thread. May return ``NULL`` if no + GILState API has been used on the current thread. Note that the main thread + always has such a thread-state, even if no auto-thread-state call has been + made on the main thread. This is mainly a helper/diagnostic function. + + .. note:: + This function may return non-``NULL`` even when the :term:`thread state` + is detached. + Prefer :c:func:`PyThreadState_Get` or :c:func:`PyThreadState_GetUnchecked` + for most cases. + + .. seealso:: :c:func:`PyThreadState_Get` + +.. c:function:: int PyGILState_Check() + + Return ``1`` if the current thread is holding the :term:`GIL` and ``0`` otherwise. + This function can be called from any thread at any time. + Only if it has had its :term:`thread state ` initialized + via :c:func:`PyGILState_Ensure` will it return ``1``. + This is mainly a helper/diagnostic function. It can be useful + for example in callback contexts or memory allocation functions when + knowing that the :term:`GIL` is locked can allow the caller to perform sensitive + actions or otherwise behave differently. + + .. note:: + If the current Python process has ever created a subinterpreter, this + function will *always* return ``1``. Prefer :c:func:`PyThreadState_GetUnchecked` + for most cases. + + .. versionadded:: 3.4 + + +Low-level APIs +-------------- + +.. c:function:: PyThreadState* PyThreadState_New(PyInterpreterState *interp) + + Create a new thread state object belonging to the given interpreter object. + An :term:`attached thread state` is not needed. + +.. c:function:: void PyThreadState_Clear(PyThreadState *tstate) + + Reset all information in a :term:`thread state` object. *tstate* + must be :term:`attached ` + + .. versionchanged:: 3.9 + This function now calls the :c:member:`!PyThreadState.on_delete` callback. + Previously, that happened in :c:func:`PyThreadState_Delete`. + + .. versionchanged:: 3.13 + The :c:member:`!PyThreadState.on_delete` callback was removed. + + +.. c:function:: void PyThreadState_Delete(PyThreadState *tstate) + + Destroy a :term:`thread state` object. *tstate* should not + be :term:`attached ` to any thread. + *tstate* must have been reset with a previous call to + :c:func:`PyThreadState_Clear`. + + +.. c:function:: void PyThreadState_DeleteCurrent(void) + + Detach the :term:`attached thread state` (which must have been reset + with a previous call to :c:func:`PyThreadState_Clear`) and then destroy it. + + No :term:`thread state` will be :term:`attached ` upon + returning. + +.. c:function:: PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) + + Get the current frame of the Python thread state *tstate*. + + Return a :term:`strong reference`. Return ``NULL`` if no frame is currently + executing. + + See also :c:func:`PyEval_GetFrame`. + + *tstate* must not be ``NULL``, and must be :term:`attached `. + + .. versionadded:: 3.9 + + +.. c:function:: uint64_t PyThreadState_GetID(PyThreadState *tstate) + + Get the unique :term:`thread state` identifier of the Python thread state *tstate*. + + *tstate* must not be ``NULL``, and must be :term:`attached `. + + .. versionadded:: 3.9 + + +.. c:function:: PyInterpreterState* PyThreadState_GetInterpreter(PyThreadState *tstate) + + Get the interpreter of the Python thread state *tstate*. + + *tstate* must not be ``NULL``, and must be :term:`attached `. + + .. versionadded:: 3.9 + + +.. c:function:: void PyThreadState_EnterTracing(PyThreadState *tstate) + + Suspend tracing and profiling in the Python thread state *tstate*. + + Resume them using the :c:func:`PyThreadState_LeaveTracing` function. + + .. versionadded:: 3.11 + + +.. c:function:: void PyThreadState_LeaveTracing(PyThreadState *tstate) + + Resume tracing and profiling in the Python thread state *tstate* suspended + by the :c:func:`PyThreadState_EnterTracing` function. + + See also :c:func:`PyEval_SetTrace` and :c:func:`PyEval_SetProfile` + functions. + + .. versionadded:: 3.11 + + +.. c:function:: int PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate, void *stack_start_addr, size_t stack_size) + + Set the stack protection start address and stack protection size + of a Python thread state. + + On success, return ``0``. + On failure, set an exception and return ``-1``. + + CPython implements :ref:`recursion control ` for C code by raising + :py:exc:`RecursionError` when it notices that the machine execution stack is close + to overflow. See for example the :c:func:`Py_EnterRecursiveCall` function. + For this, it needs to know the location of the current thread's stack, which it + normally gets from the operating system. + When the stack is changed, for example using context switching techniques like the + Boost library's ``boost::context``, you must call + :c:func:`~PyUnstable_ThreadState_SetStackProtection` to inform CPython of the change. + + Call :c:func:`~PyUnstable_ThreadState_SetStackProtection` either before + or after changing the stack. + Do not call any other Python C API between the call and the stack + change. + + See :c:func:`PyUnstable_ThreadState_ResetStackProtection` for undoing this operation. + + .. versionadded:: 3.15 + + +.. c:function:: void PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) + + Reset the stack protection start address and stack protection size + of a Python thread state to the operating system defaults. + + See :c:func:`PyUnstable_ThreadState_SetStackProtection` for an explanation. + + .. versionadded:: 3.15 + + +.. c:function:: PyObject* PyThreadState_GetDict() + + Return a dictionary in which extensions can store thread-specific state + information. Each extension should use a unique key to use to store state in + the dictionary. It is okay to call this function when no :term:`thread state` + is :term:`attached `. If this function returns + ``NULL``, no exception has been raised and the caller should assume no + thread state is attached. + + +.. c:function:: void PyEval_AcquireThread(PyThreadState *tstate) + + :term:`Attach ` *tstate* to the current thread, + which must not be ``NULL`` or already :term:`attached `. + + The calling thread must not already have an :term:`attached thread state`. + + .. note:: + Calling this function from a thread when the runtime is finalizing will + hang the thread until the program exits, even if the thread was not + created by Python. Refer to + :ref:`cautions-regarding-runtime-finalization` for more details. + + .. versionchanged:: 3.8 + Updated to be consistent with :c:func:`PyEval_RestoreThread`, + :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, + and terminate the current thread if called while the interpreter is finalizing. + + .. versionchanged:: 3.14 + Hangs the current thread, rather than terminating it, if called while the + interpreter is finalizing. + + :c:func:`PyEval_RestoreThread` is a higher-level function which is always + available (even when threads have not been initialized). + + +.. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) + + Detach the :term:`attached thread state`. + The *tstate* argument, which must not be ``NULL``, is only used to check + that it represents the :term:`attached thread state` --- if it isn't, a fatal error is + reported. + + :c:func:`PyEval_SaveThread` is a higher-level function which is always + available (even when threads have not been initialized). + + +Asynchronous notifications +========================== + +A mechanism is provided to make asynchronous notifications to the main +interpreter thread. These notifications take the form of a function +pointer and a void pointer argument. + + +.. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) + + Schedule a function to be called from the main interpreter thread. On + success, ``0`` is returned and *func* is queued for being called in the + main thread. On failure, ``-1`` is returned without setting any exception. + + When successfully queued, *func* will be *eventually* called from the + main interpreter thread with the argument *arg*. It will be called + asynchronously with respect to normally running Python code, but with + both these conditions met: + + * on a :term:`bytecode` boundary; + * with the main thread holding an :term:`attached thread state` + (*func* can therefore use the full C API). + + *func* must return ``0`` on success, or ``-1`` on failure with an exception + set. *func* won't be interrupted to perform another asynchronous + notification recursively, but it can still be interrupted to switch + threads if the :term:`thread state ` is detached. + + This function doesn't need an :term:`attached thread state`. However, to call this + function in a subinterpreter, the caller must have an :term:`attached thread state`. + Otherwise, the function *func* can be scheduled to be called from the wrong interpreter. + + .. warning:: + This is a low-level function, only useful for very special cases. + There is no guarantee that *func* will be called as quick as + possible. If the main thread is busy executing a system call, + *func* won't be called before the system call returns. This + function is generally **not** suitable for calling Python code from + arbitrary C threads. Instead, use the :ref:`PyGILState API`. + + .. versionadded:: 3.1 + + .. versionchanged:: 3.9 + If this function is called in a subinterpreter, the function *func* is + now scheduled to be called from the subinterpreter, rather than being + called from the main interpreter. Each subinterpreter now has its own + list of scheduled calls. + + .. versionchanged:: 3.12 + This function now always schedules *func* to be run in the main + interpreter. + + +.. c:function:: int Py_MakePendingCalls(void) + + Execute all pending calls. This is usually executed automatically by the + interpreter. + + This function returns ``0`` on success, and returns ``-1`` with an exception + set on failure. + + If this is not called in the main thread of the main + interpreter, this function does nothing and returns ``0``. + The caller must hold an :term:`attached thread state`. + + .. versionadded:: 3.1 + + .. versionchanged:: 3.12 + This function only runs pending calls in the main interpreter. + + +.. c:function:: int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) + + Asynchronously raise an exception in a thread. The *id* argument is the thread + id of the target thread; *exc* is the exception object to be raised. This + function does not :term:`steal` any references to *exc*. To prevent naive misuse, you + must write your own C extension to call this. Must be called with an :term:`attached thread state`. + Returns the number of thread states modified; this is normally one, but will be + zero if the thread id isn't found. If *exc* is ``NULL``, the pending + exception (if any) for the thread is cleared. This raises no exceptions. + + .. versionchanged:: 3.7 + The type of the *id* parameter changed from :c:expr:`long` to + :c:expr:`unsigned long`. + + +Operating system thread APIs +============================ + +.. c:macro:: PYTHREAD_INVALID_THREAD_ID + + Sentinel value for an invalid thread ID. + + This is currently equivalent to ``(unsigned long)-1``. + + +.. c:function:: unsigned long PyThread_start_new_thread(void (*func)(void *), void *arg) + + Start function *func* in a new thread with argument *arg*. + The resulting thread is not intended to be joined. + + *func* must not be ``NULL``, but *arg* may be ``NULL``. + + On success, this function returns the identifier of the new thread; on failure, + this returns :c:macro:`PYTHREAD_INVALID_THREAD_ID`. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: unsigned long PyThread_get_thread_ident(void) + + Return the identifier of the current thread, which will never be zero. + + This function cannot fail, and the caller does not need to hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`threading.get_ident` + + +.. c:function:: PyObject *PyThread_GetInfo(void) + + Get general information about the current thread in the form of a + :ref:`struct sequence ` object. This information is + accessible as :py:attr:`sys.thread_info` in Python. + + On success, this returns a new :term:`strong reference` to the thread + information; on failure, this returns ``NULL`` with an exception set. + + The caller must hold an :term:`attached thread state`. + + +.. c:macro:: PY_HAVE_THREAD_NATIVE_ID + + This macro is defined when the system supports native thread IDs. + + +.. c:function:: unsigned long PyThread_get_thread_native_id(void) + + Get the native identifier of the current thread as it was assigned by the operating + system's kernel, which will never be less than zero. + + This function is only available when :c:macro:`PY_HAVE_THREAD_NATIVE_ID` is + defined. + + This function cannot fail, and the caller does not need to hold an + :term:`attached thread state`. + + .. seealso:: + :py:func:`threading.get_native_id` + + +.. c:function:: void PyThread_exit_thread(void) + + Terminate the current thread. This function is generally considered unsafe + and should be avoided. It is kept solely for backwards compatibility. + + This function is only safe to call if all functions in the full call + stack are written to safely allow it. + + .. warning:: + + If the current system uses POSIX threads (also known as "pthreads"), + this calls :manpage:`pthread_exit(3)`, which attempts to unwind the stack + and call C++ destructors on some libc implementations. However, if a + ``noexcept`` function is reached, it may terminate the process. + Other systems, such as macOS, do unwinding. + + On Windows, this function calls ``_endthreadex()``, which kills the thread + without calling C++ destructors. + + In any case, there is a risk of corruption on the thread's stack. + + .. deprecated:: 3.14 + + +.. c:function:: void PyThread_init_thread(void) + + Initialize ``PyThread*`` APIs. Python executes this function automatically, + so there's little need to call it from an extension module. + + +.. c:function:: int PyThread_set_stacksize(size_t size) + + Set the stack size of the current thread to *size* bytes. + + This function returns ``0`` on success, ``-1`` if *size* is invalid, or + ``-2`` if the system does not support changing the stack size. This function + does not set exceptions. + + The caller does not need to hold an :term:`attached thread state`. + + +.. c:function:: size_t PyThread_get_stacksize(void) + + Return the stack size of the current thread in bytes, or ``0`` if the system's + default stack size is in use. + + The caller does not need to hold an :term:`attached thread state`. diff --git a/Doc/c-api/tls.rst b/Doc/c-api/tls.rst new file mode 100644 index 000000000000000..93ac5557141e258 --- /dev/null +++ b/Doc/c-api/tls.rst @@ -0,0 +1,155 @@ +.. highlight:: c + +.. _thread-local-storage: + +Thread-local storage support +============================ + +The Python interpreter provides low-level support for thread-local storage +(TLS) which wraps the underlying native TLS implementation to support the +Python-level thread-local storage API (:class:`threading.local`). The +CPython C level APIs are similar to those offered by pthreads and Windows: +use a thread key and functions to associate a :c:expr:`void*` value per +thread. + +A :term:`thread state` does *not* need to be :term:`attached ` +when calling these functions; they supply their own locking. + +Note that :file:`Python.h` does not include the declaration of the TLS APIs, +you need to include :file:`pythread.h` to use thread-local storage. + +.. note:: + None of these API functions handle memory management on behalf of the + :c:expr:`void*` values. You need to allocate and deallocate them yourself. + If the :c:expr:`void*` values happen to be :c:expr:`PyObject*`, these + functions don't do refcount operations on them either. + +.. _thread-specific-storage-api: + +Thread-specific storage API +--------------------------- + +The thread-specific storage (TSS) API was introduced to supersede the use of the existing TLS API within the +CPython interpreter. This API uses a new type :c:type:`Py_tss_t` instead of +:c:expr:`int` to represent thread keys. + +.. versionadded:: 3.7 + +.. seealso:: "A New C-API for Thread-Local Storage in CPython" (:pep:`539`) + + +.. c:type:: Py_tss_t + + This data structure represents the state of a thread key, the definition of + which may depend on the underlying TLS implementation, and it has an + internal field representing the key's initialization state. There are no + public members in this structure. + + When :ref:`Py_LIMITED_API ` is not defined, static allocation of + this type by :c:macro:`Py_tss_NEEDS_INIT` is allowed. + + +.. c:macro:: Py_tss_NEEDS_INIT + + This macro expands to the initializer for :c:type:`Py_tss_t` variables. + Note that this macro won't be defined with :ref:`Py_LIMITED_API `. + + +Dynamic allocation +------------------ + +Dynamic allocation of the :c:type:`Py_tss_t`, required in extension modules +built with :ref:`Py_LIMITED_API `, where static allocation of this type +is not possible due to its implementation being opaque at build time. + + +.. c:function:: Py_tss_t* PyThread_tss_alloc() + + Return a value which is the same state as a value initialized with + :c:macro:`Py_tss_NEEDS_INIT`, or ``NULL`` in the case of dynamic allocation + failure. + + +.. c:function:: void PyThread_tss_free(Py_tss_t *key) + + Free the given *key* allocated by :c:func:`PyThread_tss_alloc`, after + first calling :c:func:`PyThread_tss_delete` to ensure any associated + thread locals have been unassigned. This is a no-op if the *key* + argument is ``NULL``. + + .. note:: + A freed key becomes a dangling pointer. You should reset the key to + ``NULL``. + + +Methods +------- + +The parameter *key* of these functions must not be ``NULL``. Moreover, the +behaviors of :c:func:`PyThread_tss_set` and :c:func:`PyThread_tss_get` are +undefined if the given :c:type:`Py_tss_t` has not been initialized by +:c:func:`PyThread_tss_create`. + + +.. c:function:: int PyThread_tss_is_created(Py_tss_t *key) + + Return a non-zero value if the given :c:type:`Py_tss_t` has been initialized + by :c:func:`PyThread_tss_create`. + + +.. c:function:: int PyThread_tss_create(Py_tss_t *key) + + Return a zero value on successful initialization of a TSS key. The behavior + is undefined if the value pointed to by the *key* argument is not + initialized by :c:macro:`Py_tss_NEEDS_INIT`. This function can be called + repeatedly on the same key -- calling it on an already initialized key is a + no-op and immediately returns success. + + +.. c:function:: void PyThread_tss_delete(Py_tss_t *key) + + Destroy a TSS key to forget the values associated with the key across all + threads, and change the key's initialization state to uninitialized. A + destroyed key is able to be initialized again by + :c:func:`PyThread_tss_create`. This function can be called repeatedly on + the same key -- calling it on an already destroyed key is a no-op. + + +.. c:function:: int PyThread_tss_set(Py_tss_t *key, void *value) + + Return a zero value to indicate successfully associating a :c:expr:`void*` + value with a TSS key in the current thread. Each thread has a distinct + mapping of the key to a :c:expr:`void*` value. + + +.. c:function:: void* PyThread_tss_get(Py_tss_t *key) + + Return the :c:expr:`void*` value associated with a TSS key in the current + thread. This returns ``NULL`` if no value is associated with the key in the + current thread. + + +.. _thread-local-storage-api: + +Legacy APIs +----------- + +.. deprecated:: 3.7 + This API is superseded by the + :ref:`thread-specific storage (TSS) API `. + +.. note:: + This version of the API does not support platforms where the native TLS key + is defined in a way that cannot be safely cast to ``int``. On such platforms, + :c:func:`PyThread_create_key` will return immediately with a failure status, + and the other TLS functions will all be no-ops on such platforms. + +Due to the compatibility problem noted above, this version of the API should not +be used in new code. + +.. c:function:: int PyThread_create_key() +.. c:function:: void PyThread_delete_key(int key) +.. c:function:: int PyThread_set_key_value(int key, void *value) +.. c:function:: void* PyThread_get_key_value(int key) +.. c:function:: void PyThread_delete_key_value(int key) +.. c:function:: void PyThread_ReInitTLS() diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index 815afddad19df1a..157b303d0b903fc 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -48,7 +48,7 @@ Tuple Objects .. c:function:: Py_ssize_t PyTuple_Size(PyObject *p) Take a pointer to a tuple object, and return the size of that tuple. - On error, return ``-1`` and with an exception set. + On error, return ``-1`` with an exception set. .. c:function:: Py_ssize_t PyTuple_GET_SIZE(PyObject *p) @@ -90,8 +90,9 @@ Tuple Objects .. note:: - This function "steals" a reference to *o* and discards a reference to - an item already in the tuple at the affected position. + This function ":term:`steals `" a reference to *o* and discards + a reference to an item already in the tuple at the affected position + (unless it was NULL). .. c:function:: void PyTuple_SET_ITEM(PyObject *p, Py_ssize_t pos, PyObject *o) @@ -104,7 +105,7 @@ Tuple Objects .. note:: - This function "steals" a reference to *o*, and, unlike + This function ":term:`steals `" a reference to *o*, and, unlike :c:func:`PyTuple_SetItem`, does *not* discard a reference to any item that is being replaced; any reference in the tuple at position *pos* will be leaked. @@ -244,7 +245,7 @@ type. .. note:: - This function "steals" a reference to *o*. + This function ":term:`steals `" a reference to *o*. .. c:function:: void PyStructSequence_SET_ITEM(PyObject *p, Py_ssize_t *pos, PyObject *o) diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index ec2867b0ce09ba1..1eb2f1f3dd07da1 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -116,6 +116,20 @@ Type Objects .. versionadded:: 3.12 +.. c:function:: int PyType_Unwatch(int watcher_id, PyObject *type) + + Mark *type* as not watched. This undoes a previous call to + :c:func:`PyType_Watch`. *type* must not be ``NULL``. + + An extension should never call this function with a *watcher_id* that was + not returned to it by a previous call to :c:func:`PyType_AddWatcher`. + + On success, this function returns ``0``. On failure, this function returns + ``-1`` with an exception set. + + .. versionadded:: 3.12 + + .. c:type:: int (*PyType_WatchCallback)(PyObject *type) Type of a type-watcher callback function. @@ -133,6 +147,18 @@ Type Objects Type features are denoted by single bit flags. +.. c:function:: int PyType_FastSubclass(PyTypeObject *type, int flag) + + Return non-zero if the type object *type* sets the subclass flag *flag*. + Subclass flags are denoted by + :c:macro:`Py_TPFLAGS_*_SUBCLASS `. + This function is used by many ``_Check`` functions for common types. + + .. seealso:: + :c:func:`PyObject_TypeCheck`, which is used as a slower alternative in + ``_Check`` functions for types that don't come with subclass flags. + + .. c:function:: int PyType_IS_GC(PyTypeObject *o) Return true if the type object includes support for the cycle detector; this @@ -151,14 +177,31 @@ Type Objects .. c:function:: PyObject* PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems) - Generic handler for the :c:member:`~PyTypeObject.tp_alloc` slot of a type object. Use - Python's default memory allocation mechanism to allocate a new instance and - initialize all its contents to ``NULL``. + Generic handler for the :c:member:`~PyTypeObject.tp_alloc` slot of a type + object. Uses Python's default memory allocation mechanism to allocate memory + for a new instance, zeros the memory, then initializes the memory as if by + calling :c:func:`PyObject_Init` or :c:func:`PyObject_InitVar`. + + Do not call this directly to allocate memory for an object; call the type's + :c:member:`~PyTypeObject.tp_alloc` slot instead. + + For types that support garbage collection (i.e., the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set), this function behaves like + :c:macro:`PyObject_GC_New` or :c:macro:`PyObject_GC_NewVar` (except the + memory is guaranteed to be zeroed before initialization), and should be + paired with :c:func:`PyObject_GC_Del` in :c:member:`~PyTypeObject.tp_free`. + Otherwise, it behaves like :c:macro:`PyObject_New` or + :c:macro:`PyObject_NewVar` (except the memory is guaranteed to be zeroed + before initialization) and should be paired with :c:func:`PyObject_Free` in + :c:member:`~PyTypeObject.tp_free`. + .. c:function:: PyObject* PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds) - Generic handler for the :c:member:`~PyTypeObject.tp_new` slot of a type object. Create a - new instance using the type's :c:member:`~PyTypeObject.tp_alloc` slot. + Generic handler for the :c:member:`~PyTypeObject.tp_new` slot of a type + object. Creates a new instance using the type's + :c:member:`~PyTypeObject.tp_alloc` slot and returns the resulting object. + .. c:function:: int PyType_Ready(PyTypeObject *type) @@ -176,6 +219,7 @@ Type Objects GC protocol itself by at least implementing the :c:member:`~PyTypeObject.tp_traverse` handle. + .. c:function:: PyObject* PyType_GetName(PyTypeObject *type) Return the type's name. Equivalent to getting the type's @@ -183,6 +227,7 @@ Type Objects .. versionadded:: 3.11 + .. c:function:: PyObject* PyType_GetQualName(PyTypeObject *type) Return the type's qualified name. Equivalent to getting the @@ -198,6 +243,7 @@ Type Objects .. versionadded:: 3.13 + .. c:function:: PyObject* PyType_GetModuleName(PyTypeObject *type) Return the type's module name. Equivalent to getting the @@ -205,6 +251,7 @@ Type Objects .. versionadded:: 3.13 + .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) Return the function pointer stored in the given slot. If the @@ -221,11 +268,16 @@ Type Objects :c:func:`PyType_GetSlot` can now accept all types. Previously, it was limited to :ref:`heap types `. + .. c:function:: PyObject* PyType_GetModule(PyTypeObject *type) Return the module object associated with the given type when the type was created using :c:func:`PyType_FromModuleAndSpec`. + The returned reference is :term:`borrowed ` from *type*, + and will be valid as long as you hold a reference to *type*. + Do not release it with :c:func:`Py_DECREF` or similar. + If no module is associated with the given type, sets :py:class:`TypeError` and returns ``NULL``. @@ -240,6 +292,7 @@ Type Objects .. versionadded:: 3.9 + .. c:function:: void* PyType_GetModuleState(PyTypeObject *type) Return the state of the module object associated with the given type. @@ -254,6 +307,7 @@ Type Objects .. versionadded:: 3.9 + .. c:function:: PyObject* PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def) Find the first superclass whose module was created from @@ -267,8 +321,13 @@ Type Objects and other places where a method's defining class cannot be passed using the :c:type:`PyCMethod` calling convention. + The returned reference is :term:`borrowed ` from *type*, + and will be valid as long as you hold a reference to *type*. + Do not release it with :c:func:`Py_DECREF` or similar. + .. versionadded:: 3.11 + .. c:function:: int PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) Find the first superclass in *type*'s :term:`method resolution order` whose @@ -287,6 +346,7 @@ Type Objects .. versionadded:: 3.14 + .. c:function:: int PyUnstable_Type_AssignVersionTag(PyTypeObject *type) Attempt to assign a version tag to the given type. @@ -297,6 +357,16 @@ Type Objects .. versionadded:: 3.12 +.. c:function:: int PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) + + Return true if instances of *type* support creating weak references, false + otherwise. This function always succeeds. *type* must not be ``NULL``. + + .. seealso:: + * :ref:`weakrefobjects` + * :py:mod:`weakref` + + Creating Heap-Allocated Types ............................. @@ -317,8 +387,8 @@ The following functions and structs are used to create The *bases* argument can be used to specify base classes; it can either be only one class or a tuple of classes. - If *bases* is ``NULL``, the *Py_tp_bases* slot is used instead. - If that also is ``NULL``, the *Py_tp_base* slot is used instead. + If *bases* is ``NULL``, the :c:data:`Py_tp_bases` slot is used instead. + If that also is ``NULL``, the :c:data:`Py_tp_base` slot is used instead. If that also is ``NULL``, the new type derives from :class:`object`. The *module* argument can be used to record the module in which the new @@ -345,6 +415,7 @@ The following functions and structs are used to create .. versionadded:: 3.12 + .. c:function:: PyObject* PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) Equivalent to ``PyType_FromMetaclass(NULL, module, spec, bases)``. @@ -371,6 +442,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, bases)``. @@ -392,6 +464,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) Equivalent to ``PyType_FromMetaclass(NULL, NULL, spec, NULL)``. @@ -412,6 +485,7 @@ The following functions and structs are used to create Creating classes whose metaclass overrides :c:member:`~PyTypeObject.tp_new` is no longer allowed. + .. c:function:: int PyType_Freeze(PyTypeObject *type) Make a type immutable: set the :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag. @@ -477,7 +551,7 @@ The following functions and structs are used to create in the following situations: - The base is not variable-sized (its - :c:member:`~PyTypeObject.tp_itemsize`). + :c:member:`~PyTypeObject.tp_itemsize` is zero). - The requested :c:member:`PyType_Spec.basicsize` is positive, suggesting that the memory layout of the base class is known. - The requested :c:member:`PyType_Spec.basicsize` is zero, @@ -520,9 +594,9 @@ The following functions and structs are used to create :c:type:`PyAsyncMethods` with an added ``Py_`` prefix. For example, use: - * ``Py_tp_dealloc`` to set :c:member:`PyTypeObject.tp_dealloc` - * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add` - * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length` + * :c:data:`Py_tp_dealloc` to set :c:member:`PyTypeObject.tp_dealloc` + * :c:data:`Py_nb_add` to set :c:member:`PyNumberMethods.nb_add` + * :c:data:`Py_sq_length` to set :c:member:`PySequenceMethods.sq_length` An additional slot is supported that does not correspond to a :c:type:`!PyTypeObject` struct field: @@ -541,7 +615,7 @@ The following functions and structs are used to create If it is not possible to switch to a ``MANAGED`` flag (for example, for vectorcall or to support Python older than 3.12), specify the - offset in :c:member:`Py_tp_members `. + offset in :c:data:`Py_tp_members`. See :ref:`PyMemberDef documentation ` for details. @@ -568,8 +642,8 @@ The following functions and structs are used to create under the :ref:`limited API `. .. versionchanged:: 3.14 - The field :c:member:`~PyTypeObject.tp_vectorcall` can now set - using ``Py_tp_vectorcall``. See the field's documentation + The field :c:member:`~PyTypeObject.tp_vectorcall` can now be set + using :c:data:`Py_tp_vectorcall`. See the field's documentation for details. .. c:member:: void *pfunc @@ -579,10 +653,11 @@ The following functions and structs are used to create *pfunc* values may not be ``NULL``, except for the following slots: - * ``Py_tp_doc`` + * :c:data:`Py_tp_doc` * :c:data:`Py_tp_token` (for clarity, prefer :c:data:`Py_TP_USE_SPEC` rather than ``NULL``) + .. c:macro:: Py_tp_token A :c:member:`~PyType_Slot.slot` that records a static memory layout ID diff --git a/Doc/c-api/typehints.rst b/Doc/c-api/typehints.rst index 98fe68737deb81c..ec2fba6da8b0438 100644 --- a/Doc/c-api/typehints.rst +++ b/Doc/c-api/typehints.rst @@ -31,7 +31,7 @@ two types exist -- :ref:`GenericAlias ` and static PyMethodDef my_obj_methods[] = { // Other methods. ... - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, "See PEP 585"} + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, "my_obj is generic over its contained type"} ... } diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index 3b9f07778d5ace9..4d69821ba9dfae3 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -79,7 +79,7 @@ Quick Reference | :c:member:`~PyTypeObject.tp_setattro` | :c:type:`setattrofunc` | __setattr__, | X | X | | G | | | | __delattr__ | | | | | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_buffer` | :c:type:`PyBufferProcs` * | | | | | % | + | :c:member:`~PyTypeObject.tp_as_buffer` | :c:type:`PyBufferProcs` * | :ref:`sub-slots` | | | | % | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ | :c:member:`~PyTypeObject.tp_flags` | unsigned long | | X | X | | ? | +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ @@ -325,9 +325,10 @@ sub-slots +---------------------------------------------------------+-----------------------------------+---------------+ | | +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyBufferProcs.bf_getbuffer` | :c:func:`getbufferproc` | | + | :c:member:`~PyBufferProcs.bf_getbuffer` | :c:func:`getbufferproc` | __buffer__ | +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyBufferProcs.bf_releasebuffer` | :c:func:`releasebufferproc` | | + | :c:member:`~PyBufferProcs.bf_releasebuffer` | :c:func:`releasebufferproc` | __release_\ | + | | | buffer\__ | +---------------------------------------------------------+-----------------------------------+---------------+ .. _slot-typedefs-table: @@ -491,9 +492,9 @@ metatype) initializes :c:member:`~PyTypeObject.tp_itemsize`, which means that it type objects) *must* have the :c:member:`~PyVarObject.ob_size` field. -.. c:member:: Py_ssize_t PyObject.ob_refcnt +:c:member:`PyObject.ob_refcnt` - This is the type object's reference count, initialized to ``1`` by the + The type object's reference count is initialized to ``1`` by the ``PyObject_HEAD_INIT`` macro. Note that for :ref:`statically allocated type objects `, the type's instances (objects whose :c:member:`~PyObject.ob_type` points back to the type) do *not* count as references. But for @@ -505,7 +506,7 @@ type objects) *must* have the :c:member:`~PyVarObject.ob_size` field. This field is not inherited by subtypes. -.. c:member:: PyTypeObject* PyObject.ob_type +:c:member:`PyObject.ob_type` This is the type's type, in other words its metatype. It is initialized by the argument to the ``PyObject_HEAD_INIT`` macro, and its value should normally be @@ -531,14 +532,13 @@ type objects) *must* have the :c:member:`~PyVarObject.ob_size` field. PyVarObject Slots ----------------- -.. c:member:: Py_ssize_t PyVarObject.ob_size +:c:member:`PyVarObject.ob_size` For :ref:`statically allocated type objects `, this should be initialized to zero. For :ref:`dynamically allocated type objects `, this field has a special internal meaning. - This field should be accessed using the :c:func:`Py_SIZE()` and - :c:func:`Py_SET_SIZE()` macros. + This field should be accessed using the :c:func:`Py_SIZE()` macro. **Inheritance:** @@ -676,77 +676,144 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_dealloc - A pointer to the instance destructor function. This function must be defined - unless the type guarantees that its instances will never be deallocated (as is - the case for the singletons ``None`` and ``Ellipsis``). The function signature is:: + .. corresponding-type-slot:: Py_tp_dealloc + + A pointer to the instance destructor function. The function signature is:: void tp_dealloc(PyObject *self); - The destructor function is called by the :c:func:`Py_DECREF` and - :c:func:`Py_XDECREF` macros when the new reference count is zero. At this point, - the instance is still in existence, but there are no references to it. The - destructor function should free all references which the instance owns, free all - memory buffers owned by the instance (using the freeing function corresponding - to the allocation function used to allocate the buffer), and call the type's - :c:member:`~PyTypeObject.tp_free` function. If the type is not subtypable - (doesn't have the :c:macro:`Py_TPFLAGS_BASETYPE` flag bit set), it is - permissible to call the object deallocator directly instead of via - :c:member:`~PyTypeObject.tp_free`. The object deallocator should be the one used to allocate the - instance; this is normally :c:func:`PyObject_Free` if the instance was allocated - using :c:macro:`PyObject_New` or :c:macro:`PyObject_NewVar`, or - :c:func:`PyObject_GC_Del` if the instance was allocated using - :c:macro:`PyObject_GC_New` or :c:macro:`PyObject_GC_NewVar`. - - If the type supports garbage collection (has the :c:macro:`Py_TPFLAGS_HAVE_GC` - flag bit set), the destructor should call :c:func:`PyObject_GC_UnTrack` - before clearing any member fields. + The destructor function should remove all references which the instance owns + (e.g., call :c:func:`Py_CLEAR`), free all memory buffers owned by the + instance, and call the type's :c:member:`~PyTypeObject.tp_free` function to + free the object itself. + + If you may call functions that may set the error indicator, you must use + :c:func:`PyErr_GetRaisedException` and :c:func:`PyErr_SetRaisedException` + to ensure you don't clobber a preexisting error indicator (the deallocation + could have occurred while processing a different error): .. code-block:: c static void - foo_dealloc(PyObject *op) + foo_dealloc(foo_object *self) { + PyObject *et, *ev, *etb; + PyObject *exc = PyErr_GetRaisedException(); + ... + PyErr_SetRaisedException(exc); + } + + The dealloc handler itself must not raise an exception; if it hits an error + case it should call :c:func:`PyErr_FormatUnraisable` to log (and clear) an + unraisable exception. + + No guarantees are made about when an object is destroyed, except: + + * Python will destroy an object immediately or some time after the final + reference to the object is deleted, unless its finalizer + (:c:member:`~PyTypeObject.tp_finalize`) subsequently resurrects the + object. + * An object will not be destroyed while it is being automatically finalized + (:c:member:`~PyTypeObject.tp_finalize`) or automatically cleared + (:c:member:`~PyTypeObject.tp_clear`). + + CPython currently destroys an object immediately from :c:func:`Py_DECREF` + when the new reference count is zero, but this may change in a future + version. + + It is recommended to call :c:func:`PyObject_CallFinalizerFromDealloc` at the + beginning of :c:member:`!tp_dealloc` to guarantee that the object is always + finalized before destruction. + + If the type supports garbage collection (the :c:macro:`Py_TPFLAGS_HAVE_GC` + flag is set), the destructor should call :c:func:`PyObject_GC_UnTrack` + before clearing any member fields. + + It is permissible to call :c:member:`~PyTypeObject.tp_clear` from + :c:member:`!tp_dealloc` to reduce code duplication and to guarantee that the + object is always cleared before destruction. Beware that + :c:member:`!tp_clear` might have already been called. + + If the type is heap allocated (:c:macro:`Py_TPFLAGS_HEAPTYPE`), the + deallocator should release the owned reference to its type object (via + :c:func:`Py_DECREF`) after calling the type deallocator. See the example + code below.:: + + static void + foo_dealloc(PyObject *op) + { foo_object *self = (foo_object *) op; PyObject_GC_UnTrack(self); Py_CLEAR(self->ref); Py_TYPE(self)->tp_free(self); - } + } - Finally, if the type is heap allocated (:c:macro:`Py_TPFLAGS_HEAPTYPE`), the - deallocator should release the owned reference to its type object - (via :c:func:`Py_DECREF`) after - calling the type deallocator. In order to avoid dangling pointers, the - recommended way to achieve this is: + :c:member:`!tp_dealloc` must leave the exception status unchanged. If it + needs to call something that might raise an exception, the exception state + must be backed up first and restored later (after logging any exceptions + with :c:func:`PyErr_WriteUnraisable`). - .. code-block:: c + Example:: - static void - foo_dealloc(PyObject *op) - { - PyTypeObject *tp = Py_TYPE(op); - // free references and buffers here - tp->tp_free(op); - Py_DECREF(tp); - } + static void + foo_dealloc(PyObject *self) + { + PyObject *exc = PyErr_GetRaisedException(); - .. warning:: + if (PyObject_CallFinalizerFromDealloc(self) < 0) { + // self was resurrected. + goto done; + } + + PyTypeObject *tp = Py_TYPE(self); + + if (tp->tp_flags & Py_TPFLAGS_HAVE_GC) { + PyObject_GC_UnTrack(self); + } + + // Optional, but convenient to avoid code duplication. + if (tp->tp_clear && tp->tp_clear(self) < 0) { + PyErr_WriteUnraisable(self); + } - In a garbage collected Python, :c:member:`!tp_dealloc` may be called from - any Python thread, not just the thread which created the object (if the - object becomes part of a refcount cycle, that cycle might be collected by - a garbage collection on any thread). This is not a problem for Python - API calls, since the thread on which :c:member:`!tp_dealloc` is called - with an :term:`attached thread state`. However, if the object being - destroyed in turn destroys objects from some other C or C++ library, care - should be taken to ensure that destroying those objects on the thread - which called :c:member:`!tp_dealloc` will not violate any assumptions of - the library. + // Any additional destruction goes here. + + tp->tp_free(self); + self = NULL; // In case PyErr_WriteUnraisable() is called below. + + if (tp->tp_flags & Py_TPFLAGS_HEAPTYPE) { + Py_CLEAR(tp); + } + + done: + // Optional, if something was called that might have raised an + // exception. + if (PyErr_Occurred()) { + PyErr_WriteUnraisable(self); + } + PyErr_SetRaisedException(exc); + } + + :c:member:`!tp_dealloc` may be called from + any Python thread, not just the thread which created the object (if the + object becomes part of a refcount cycle, that cycle might be collected by + a garbage collection on any thread). This is not a problem for Python + API calls, since the thread on which :c:member:`!tp_dealloc` is called + with an :term:`attached thread state`. However, if the object being + destroyed in turn destroys objects from some other C library, care + should be taken to ensure that destroying those objects on the thread + which called :c:member:`!tp_dealloc` will not violate any assumptions of + the library. **Inheritance:** This field is inherited by subtypes. + .. seealso:: + + :ref:`life-cycle` for details about how this slot relates to other slots. + .. c:member:: Py_ssize_t PyTypeObject.tp_vectorcall_offset @@ -795,6 +862,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: getattrfunc PyTypeObject.tp_getattr + .. corresponding-type-slot:: Py_tp_getattr + An optional pointer to the get-attribute-string function. This field is deprecated. When it is defined, it should point to a function @@ -812,6 +881,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: setattrfunc PyTypeObject.tp_setattr + .. corresponding-type-slot:: Py_tp_setattr + An optional pointer to the function for setting and deleting attributes. This field is deprecated. When it is defined, it should point to a function @@ -844,6 +915,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: reprfunc PyTypeObject.tp_repr + .. corresponding-type-slot:: Py_tp_repr + .. index:: pair: built-in function; repr An optional pointer to a function that implements the built-in function @@ -909,6 +982,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: hashfunc PyTypeObject.tp_hash + .. corresponding-type-slot:: Py_tp_hash + .. index:: pair: built-in function; hash An optional pointer to a function that implements the built-in function @@ -950,6 +1025,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: ternaryfunc PyTypeObject.tp_call + .. corresponding-type-slot:: Py_tp_call + An optional pointer to a function that implements calling the object. This should be ``NULL`` if the object is not callable. The signature is the same as for :c:func:`PyObject_Call`:: @@ -963,6 +1040,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: reprfunc PyTypeObject.tp_str + .. corresponding-type-slot:: Py_tp_str + An optional pointer to a function that implements the built-in operation :func:`str`. (Note that :class:`str` is a type now, and :func:`str` calls the constructor for that type. This constructor calls :c:func:`PyObject_Str` to do @@ -988,6 +1067,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: getattrofunc PyTypeObject.tp_getattro + .. corresponding-type-slot:: Py_tp_getattro + An optional pointer to the get-attribute function. The signature is the same as for :c:func:`PyObject_GetAttr`:: @@ -1012,6 +1093,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: setattrofunc PyTypeObject.tp_setattro + .. corresponding-type-slot:: Py_tp_setattro + An optional pointer to the function for setting and deleting attributes. The signature is the same as for :c:func:`PyObject_SetAttr`:: @@ -1137,11 +1220,11 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_HAVE_GC This bit is set when the object supports garbage collection. If this bit - is set, instances must be created using :c:macro:`PyObject_GC_New` and - destroyed using :c:func:`PyObject_GC_Del`. More information in section - :ref:`supporting-cycle-detection`. This bit also implies that the - GC-related fields :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` are present in - the type object. + is set, memory for new instances (see :c:member:`~PyTypeObject.tp_alloc`) + must be allocated using :c:macro:`PyObject_GC_New` or + :c:func:`PyType_GenericAlloc` and deallocated (see + :c:member:`~PyTypeObject.tp_free`) using :c:func:`PyObject_GC_Del`. More + information in section :ref:`supporting-cycle-detection`. **Inheritance:** @@ -1192,7 +1275,7 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_MANAGED_DICT - This bit indicates that instances of the class have a `~object.__dict__` + This bit indicates that instances of the class have a :attr:`~object.__dict__` attribute, and that the space for the dictionary is managed by the VM. If this flag is set, :c:macro:`Py_TPFLAGS_HAVE_GC` should also be set. @@ -1253,8 +1336,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:macro:: Py_TPFLAGS_BASE_EXC_SUBCLASS .. c:macro:: Py_TPFLAGS_TYPE_SUBCLASS - These flags are used by functions such as - :c:func:`PyLong_Check` to quickly determine if a type is a subclass + Functions such as :c:func:`PyLong_Check` will call :c:func:`PyType_FastSubclass` + with one of these flags to quickly determine if a type is a subclass of a built-in type; such specific checks are faster than a generic check, like :c:func:`PyObject_IsInstance`. Custom types that inherit from built-ins should have their :c:member:`~PyTypeObject.tp_flags` @@ -1275,6 +1358,9 @@ and :c:data:`PyType_Type` effectively act as defaults.) type structure. + .. c:macro:: _Py_TPFLAGS_HAVE_VECTORCALL + :no-typesetting: + .. c:macro:: Py_TPFLAGS_HAVE_VECTORCALL This bit is set when the class implements @@ -1286,7 +1372,12 @@ and :c:data:`PyType_Type` effectively act as defaults.) This bit is inherited if :c:member:`~PyTypeObject.tp_call` is also inherited. - .. versionadded:: 3.9 + .. versionadded:: 3.8 as ``_Py_TPFLAGS_HAVE_VECTORCALL`` + + .. versionchanged:: 3.9 + + Renamed to the current name, without the leading underscore. + The old provisional name is :term:`soft deprecated`. .. versionchanged:: 3.12 @@ -1393,8 +1484,58 @@ and :c:data:`PyType_Type` effectively act as defaults.) It will be removed in a future version of CPython + .. c:macro:: Py_TPFLAGS_HAVE_VERSION_TAG + + This macro does nothing. + Historically, this would indicate that the + :c:member:`~PyTypeObject.tp_version_tag` field was available and + initialized. + + .. soft-deprecated:: 3.13 + + + .. c:macro:: Py_TPFLAGS_INLINE_VALUES + + This bit indicates that instances of this type will have an "inline values" + array (containing the object's attributes) placed directly after the end + of the object. + + This requires that :c:macro:`Py_TPFLAGS_HAVE_GC` is set. + + **Inheritance:** + + This flag is not inherited. + + .. versionadded:: 3.13 + + + .. c:macro:: Py_TPFLAGS_IS_ABSTRACT + + This bit indicates that this is an abstract type and therefore cannot + be instantiated. + + **Inheritance:** + + This flag is not inherited. + + .. seealso:: + :mod:`abc` + + + .. c:macro:: Py_TPFLAGS_HAVE_STACKLESS_EXTENSION + + Internal. Do not set or unset this flag. + Historically, this was a reserved flag for use in Stackless Python. + + .. warning:: + This flag is present in header files, but is not be used. + This may be removed in a future version of CPython. + + .. c:member:: const char* PyTypeObject.tp_doc + .. corresponding-type-slot:: Py_tp_doc + An optional pointer to a NUL-terminated C string giving the docstring for this type object. This is exposed as the :attr:`~type.__doc__` attribute on the type and instances of the type. @@ -1406,6 +1547,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: traverseproc PyTypeObject.tp_traverse + .. corresponding-type-slot:: Py_tp_traverse + An optional pointer to a traversal function for the garbage collector. This is only used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is:: @@ -1467,6 +1610,11 @@ and :c:data:`PyType_Type` effectively act as defaults.) but the instance has no strong reference to the elements inside it, as they are allowed to be removed even if the instance is still alive). + .. warning:: + The traversal function must not have any side effects. It must not + modify the reference counts of any Python objects nor create or destroy + any Python objects. + Note that :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to :c:func:`!local_traverse` to have these specific names; don't name them just anything. @@ -1478,6 +1626,11 @@ and :c:data:`PyType_Type` effectively act as defaults.) heap-allocated superclass). If they do not, the type object may not be garbage-collected. + .. note:: + + The :c:member:`~PyTypeObject.tp_traverse` function can be called from any + thread. + .. versionchanged:: 3.9 Heap-allocated types are expected to visit ``Py_TYPE(self)`` in @@ -1497,20 +1650,103 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: inquiry PyTypeObject.tp_clear - An optional pointer to a clear function for the garbage collector. This is only - used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is:: + .. corresponding-type-slot:: Py_tp_clear + + An optional pointer to a clear function. The signature is:: int tp_clear(PyObject *); - The :c:member:`~PyTypeObject.tp_clear` member function is used to break reference cycles in cyclic - garbage detected by the garbage collector. Taken together, all :c:member:`~PyTypeObject.tp_clear` - functions in the system must combine to break all reference cycles. This is - subtle, and if in any doubt supply a :c:member:`~PyTypeObject.tp_clear` function. For example, - the tuple type does not implement a :c:member:`~PyTypeObject.tp_clear` function, because it's - possible to prove that no reference cycle can be composed entirely of tuples. - Therefore the :c:member:`~PyTypeObject.tp_clear` functions of other types must be sufficient to - break any cycle containing a tuple. This isn't immediately obvious, and there's - rarely a good reason to avoid implementing :c:member:`~PyTypeObject.tp_clear`. + The purpose of this function is to break reference cycles that are causing a + :term:`cyclic isolate` so that the objects can be safely destroyed. A + cleared object is a partially destroyed object; the object is not obligated + to satisfy design invariants held during normal use. + + :c:member:`!tp_clear` does not need to delete references to objects that + can't participate in reference cycles, such as Python strings or Python + integers. However, it may be convenient to clear all references, and write + the type's :c:member:`~PyTypeObject.tp_dealloc` function to invoke + :c:member:`!tp_clear` to avoid code duplication. (Beware that + :c:member:`!tp_clear` might have already been called. Prefer calling + idempotent functions like :c:func:`Py_CLEAR`.) + + Any non-trivial cleanup should be performed in + :c:member:`~PyTypeObject.tp_finalize` instead of :c:member:`!tp_clear`. + + .. note:: + + If :c:member:`!tp_clear` fails to break a reference cycle then the + objects in the :term:`cyclic isolate` may remain indefinitely + uncollectable ("leak"). See :data:`gc.garbage`. + + .. note:: + + Referents (direct and indirect) might have already been cleared; they are + not guaranteed to be in a consistent state. + + .. note:: + + The :c:member:`~PyTypeObject.tp_clear` function can be called from any + thread. + + .. note:: + + An object is not guaranteed to be automatically cleared before its + destructor (:c:member:`~PyTypeObject.tp_dealloc`) is called. + + This function differs from the destructor + (:c:member:`~PyTypeObject.tp_dealloc`) in the following ways: + + * The purpose of clearing an object is to remove references to other objects + that might participate in a reference cycle. The purpose of the + destructor, on the other hand, is a superset: it must release *all* + resources it owns, including references to objects that cannot participate + in a reference cycle (e.g., integers) as well as the object's own memory + (by calling :c:member:`~PyTypeObject.tp_free`). + * When :c:member:`!tp_clear` is called, other objects might still hold + references to the object being cleared. Because of this, + :c:member:`!tp_clear` must not deallocate the object's own memory + (:c:member:`~PyTypeObject.tp_free`). The destructor, on the other hand, + is only called when no (strong) references exist, and as such, must + safely destroy the object itself by deallocating it. + * :c:member:`!tp_clear` might never be automatically called. An object's + destructor, on the other hand, will be automatically called some time + after the object becomes unreachable (i.e., either there are no references + to the object or the object is a member of a :term:`cyclic isolate`). + + No guarantees are made about when, if, or how often Python automatically + clears an object, except: + + * Python will not automatically clear an object if it is reachable, i.e., + there is a reference to it and it is not a member of a :term:`cyclic + isolate`. + * Python will not automatically clear an object if it has not been + automatically finalized (see :c:member:`~PyTypeObject.tp_finalize`). (If + the finalizer resurrected the object, the object may or may not be + automatically finalized again before it is cleared.) + * If an object is a member of a :term:`cyclic isolate`, Python will not + automatically clear it if any member of the cyclic isolate has not yet + been automatically finalized (:c:member:`~PyTypeObject.tp_finalize`). + * Python will not destroy an object until after any automatic calls to its + :c:member:`!tp_clear` function have returned. This ensures that the act + of breaking a reference cycle does not invalidate the ``self`` pointer + while :c:member:`!tp_clear` is still executing. + * Python will not automatically call :c:member:`!tp_clear` multiple times + concurrently. + + CPython currently only automatically clears objects as needed to break + reference cycles in a :term:`cyclic isolate`, but future versions might + clear objects regularly before their destruction. + + Taken together, all :c:member:`~PyTypeObject.tp_clear` functions in the + system must combine to break all reference cycles. This is subtle, and if + in any doubt supply a :c:member:`~PyTypeObject.tp_clear` function. For + example, the tuple type does not implement a + :c:member:`~PyTypeObject.tp_clear` function, because it's possible to prove + that no reference cycle can be composed entirely of tuples. Therefore the + :c:member:`~PyTypeObject.tp_clear` functions of other types are responsible + for breaking any cycle containing a tuple. This isn't immediately obvious, + and there's rarely a good reason to avoid implementing + :c:member:`~PyTypeObject.tp_clear`. Implementations of :c:member:`~PyTypeObject.tp_clear` should drop the instance's references to those of its members that may be Python objects, and set its pointers to those @@ -1540,23 +1776,11 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:func:`Py_CLEAR` macro performs the operations in a safe order. If the :c:macro:`Py_TPFLAGS_MANAGED_DICT` bit is set in the - :c:member:`~PyTypeObject.tp_flags` field, the traverse function must call + :c:member:`~PyTypeObject.tp_flags` field, the clear function must call :c:func:`PyObject_ClearManagedDict` like this:: PyObject_ClearManagedDict((PyObject*)self); - Note that :c:member:`~PyTypeObject.tp_clear` is not *always* called - before an instance is deallocated. For example, when reference counting - is enough to determine that an object is no longer used, the cyclic garbage - collector is not involved and :c:member:`~PyTypeObject.tp_dealloc` is - called directly. - - Because the goal of :c:member:`~PyTypeObject.tp_clear` functions is to break reference cycles, - it's not necessary to clear contained objects like Python strings or Python - integers, which can't participate in reference cycles. On the other hand, it may - be convenient to clear all contained Python objects, and write the type's - :c:member:`~PyTypeObject.tp_dealloc` function to invoke :c:member:`~PyTypeObject.tp_clear`. - More information about Python's garbage collection scheme can be found in section :ref:`supporting-cycle-detection`. @@ -1569,9 +1793,15 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:member:`~PyTypeObject.tp_clear` are all inherited from the base type if they are all zero in the subtype. + .. seealso:: + + :ref:`life-cycle` for details about how this slot relates to other slots. + .. c:member:: richcmpfunc PyTypeObject.tp_richcompare + .. corresponding-type-slot:: Py_tp_richcompare + An optional pointer to the rich comparison function, whose signature is:: PyObject *tp_richcompare(PyObject *self, PyObject *other, int op); @@ -1674,6 +1904,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: getiterfunc PyTypeObject.tp_iter + .. corresponding-type-slot:: Py_tp_iter + An optional pointer to a function that returns an :term:`iterator` for the object. Its presence normally signals that the instances of this type are :term:`iterable` (although sequences may be iterable without this function). @@ -1689,6 +1921,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: iternextfunc PyTypeObject.tp_iternext + .. corresponding-type-slot:: Py_tp_iternext + An optional pointer to a function that returns the next item in an :term:`iterator`. The signature is:: @@ -1712,6 +1946,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyMethodDef* PyTypeObject.tp_methods + .. corresponding-type-slot:: Py_tp_methods + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyMethodDef` structures, declaring regular methods of this type. @@ -1726,6 +1962,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyMemberDef* PyTypeObject.tp_members + .. corresponding-type-slot:: Py_tp_members + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyMemberDef` structures, declaring regular data members (fields or slots) of instances of this type. @@ -1741,6 +1979,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: struct PyGetSetDef* PyTypeObject.tp_getset + .. corresponding-type-slot:: Py_tp_getset + An optional pointer to a static ``NULL``-terminated array of :c:type:`PyGetSetDef` structures, declaring computed attributes of instances of this type. @@ -1755,6 +1995,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: PyTypeObject* PyTypeObject.tp_base + .. corresponding-type-slot:: Py_tp_base + An optional pointer to a base type from which type properties are inherited. At this level, only single inheritance is supported; multiple inheritance require dynamically creating a type object by calling the metatype. @@ -1827,6 +2069,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: descrgetfunc PyTypeObject.tp_descr_get + .. corresponding-type-slot:: Py_tp_descr_get + An optional pointer to a "descriptor get" function. The function signature is:: @@ -1842,6 +2086,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: descrsetfunc PyTypeObject.tp_descr_set + .. corresponding-type-slot:: Py_tp_descr_set + An optional pointer to a function for setting and deleting a descriptor's value. @@ -1902,6 +2148,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: initproc PyTypeObject.tp_init + .. corresponding-type-slot:: Py_tp_init + An optional pointer to an instance initialization function. This function corresponds to the :meth:`~object.__init__` method of classes. Like @@ -1937,6 +2185,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: allocfunc PyTypeObject.tp_alloc + .. corresponding-type-slot:: Py_tp_alloc + An optional pointer to an instance allocation function. The function signature is:: @@ -1945,22 +2195,23 @@ and :c:data:`PyType_Type` effectively act as defaults.) **Inheritance:** - This field is inherited by static subtypes, but not by dynamic - subtypes (subtypes created by a class statement). + Static subtypes inherit this slot, which will be + :c:func:`PyType_GenericAlloc` if inherited from :class:`object`. + + :ref:`Heap subtypes ` do not inherit this slot. **Default:** - For dynamic subtypes, this field is always set to - :c:func:`PyType_GenericAlloc`, to force a standard heap - allocation strategy. + For heap subtypes, this field is always set to + :c:func:`PyType_GenericAlloc`. - For static subtypes, :c:data:`PyBaseObject_Type` uses - :c:func:`PyType_GenericAlloc`. That is the recommended value - for all statically defined types. + For static subtypes, this slot is inherited (see above). .. c:member:: newfunc PyTypeObject.tp_new + .. corresponding-type-slot:: Py_tp_new + An optional pointer to an instance creation function. The function signature is:: @@ -2000,28 +2251,39 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: freefunc PyTypeObject.tp_free + .. corresponding-type-slot:: Py_tp_free + An optional pointer to an instance deallocation function. Its signature is:: void tp_free(void *self); - An initializer that is compatible with this signature is :c:func:`PyObject_Free`. + This function must free the memory allocated by + :c:member:`~PyTypeObject.tp_alloc`. **Inheritance:** - This field is inherited by static subtypes, but not by dynamic - subtypes (subtypes created by a class statement) + Static subtypes inherit this slot, which will be :c:func:`PyObject_Free` if + inherited from :class:`object`. Exception: If the type supports garbage + collection (i.e., the :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set in + :c:member:`~PyTypeObject.tp_flags`) and it would inherit + :c:func:`PyObject_Free`, then this slot is not inherited but instead defaults + to :c:func:`PyObject_GC_Del`. + + :ref:`Heap subtypes ` do not inherit this slot. **Default:** - In dynamic subtypes, this field is set to a deallocator suitable to - match :c:func:`PyType_GenericAlloc` and the value of the - :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit. + For :ref:`heap subtypes `, this slot defaults to a deallocator suitable to match + :c:func:`PyType_GenericAlloc` and the value of the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag. - For static subtypes, :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_Free`. + For static subtypes, this slot is inherited (see above). .. c:member:: inquiry PyTypeObject.tp_is_gc + .. corresponding-type-slot:: Py_tp_is_gc + An optional pointer to a function called by the garbage collector. The garbage collector needs to know whether a particular object is collectible @@ -2050,12 +2312,14 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: PyObject* PyTypeObject.tp_bases + .. corresponding-type-slot:: Py_tp_bases + Tuple of base types. This field should be set to ``NULL`` and treated as read-only. Python will fill it in when the type is :c:func:`initialized `. - For dynamically created classes, the ``Py_tp_bases`` + For dynamically created classes, the :c:data:`Py_tp_bases` :c:type:`slot ` can be used instead of the *bases* argument of :c:func:`PyType_FromSpecWithBases`. The argument form is preferred. @@ -2130,6 +2394,8 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_del + .. corresponding-type-slot:: Py_tp_del + This field is deprecated. Use :c:member:`~PyTypeObject.tp_finalize` instead. @@ -2144,29 +2410,140 @@ and :c:data:`PyType_Type` effectively act as defaults.) .. c:member:: destructor PyTypeObject.tp_finalize - An optional pointer to an instance finalization function. Its signature is:: + .. corresponding-type-slot:: Py_tp_finalize + + An optional pointer to an instance finalization function. This is the C + implementation of the :meth:`~object.__del__` special method. Its signature + is:: void tp_finalize(PyObject *self); - If :c:member:`~PyTypeObject.tp_finalize` is set, the interpreter calls it once when - finalizing an instance. It is called either from the garbage - collector (if the instance is part of an isolated reference cycle) or - just before the object is deallocated. Either way, it is guaranteed - to be called before attempting to break reference cycles, ensuring - that it finds the object in a sane state. + The primary purpose of finalization is to perform any non-trivial cleanup + that must be performed before the object is destroyed, while the object and + any other objects it directly or indirectly references are still in a + consistent state. The finalizer is allowed to execute + arbitrary Python code. + + Before Python automatically finalizes an object, some of the object's direct + or indirect referents might have themselves been automatically finalized. + However, none of the referents will have been automatically cleared + (:c:member:`~PyTypeObject.tp_clear`) yet. + + Other non-finalized objects might still be using a finalized object, so the + finalizer must leave the object in a sane state (e.g., invariants are still + met). + + .. note:: + + After Python automatically finalizes an object, Python might start + automatically clearing (:c:member:`~PyTypeObject.tp_clear`) the object + and its referents (direct and indirect). Cleared objects are not + guaranteed to be in a consistent state; a finalized object must be able + to tolerate cleared referents. + + .. note:: + + An object is not guaranteed to be automatically finalized before its + destructor (:c:member:`~PyTypeObject.tp_dealloc`) is called. It is + recommended to call :c:func:`PyObject_CallFinalizerFromDealloc` at the + beginning of :c:member:`!tp_dealloc` to guarantee that the object is + always finalized before destruction. + + .. note:: + + The :c:member:`~PyTypeObject.tp_finalize` function can be called from any + thread, although the :term:`GIL` will be held. - :c:member:`~PyTypeObject.tp_finalize` should not mutate the current exception status; - therefore, a recommended way to write a non-trivial finalizer is:: + .. note:: + + The :c:member:`!tp_finalize` function can be called during shutdown, + after some global variables have been deleted. See the documentation of + the :meth:`~object.__del__` method for details. + + When Python finalizes an object, it behaves like the following algorithm: + + #. Python might mark the object as *finalized*. Currently, Python always + marks objects whose type supports garbage collection (i.e., the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag is set in + :c:member:`~PyTypeObject.tp_flags`) and never marks other types of + objects; this might change in a future version. + #. If the object is not marked as *finalized* and its + :c:member:`!tp_finalize` finalizer function is non-``NULL``, the + finalizer function is called. + #. If the finalizer function was called and the finalizer made the object + reachable (i.e., there is a reference to the object and it is not a + member of a :term:`cyclic isolate`), then the finalizer is said to have + *resurrected* the object. It is unspecified whether the finalizer can + also resurrect the object by adding a new reference to the object that + does not make it reachable, i.e., the object is (still) a member of a + cyclic isolate. + #. If the finalizer resurrected the object, the object's pending destruction + is canceled and the object's *finalized* mark might be removed if + present. Currently, Python never removes the *finalized* mark; this + might change in a future version. + + *Automatic finalization* refers to any finalization performed by Python + except via calls to :c:func:`PyObject_CallFinalizer` or + :c:func:`PyObject_CallFinalizerFromDealloc`. No guarantees are made about + when, if, or how often an object is automatically finalized, except: + + * Python will not automatically finalize an object if it is reachable, i.e., + there is a reference to it and it is not a member of a :term:`cyclic + isolate`. + * Python will not automatically finalize an object if finalizing it would + not mark the object as *finalized*. Currently, this applies to objects + whose type does not support garbage collection, i.e., the + :c:macro:`Py_TPFLAGS_HAVE_GC` flag is not set. Such objects can still be + manually finalized by calling :c:func:`PyObject_CallFinalizer` or + :c:func:`PyObject_CallFinalizerFromDealloc`. + * Python will not automatically finalize any two members of a :term:`cyclic + isolate` concurrently. + * Python will not automatically finalize an object after it has + automatically cleared (:c:member:`~PyTypeObject.tp_clear`) the object. + * If an object is a member of a :term:`cyclic isolate`, Python will not + automatically finalize it after automatically clearing (see + :c:member:`~PyTypeObject.tp_clear`) any other member. + * Python will automatically finalize every member of a :term:`cyclic + isolate` before it automatically clears (see + :c:member:`~PyTypeObject.tp_clear`) any of them. + * If Python is going to automatically clear an object + (:c:member:`~PyTypeObject.tp_clear`), it will automatically finalize the + object first. + + Python currently only automatically finalizes objects that are members of a + :term:`cyclic isolate`, but future versions might finalize objects regularly + before their destruction. + + To manually finalize an object, do not call this function directly; call + :c:func:`PyObject_CallFinalizer` or + :c:func:`PyObject_CallFinalizerFromDealloc` instead. + + :c:member:`~PyTypeObject.tp_finalize` should leave the current exception + status unchanged. The recommended way to write a non-trivial finalizer is + to back up the exception at the beginning by calling + :c:func:`PyErr_GetRaisedException` and restore the exception at the end by + calling :c:func:`PyErr_SetRaisedException`. If an exception is encountered + in the middle of the finalizer, log and clear it with + :c:func:`PyErr_WriteUnraisable` or :c:func:`PyErr_FormatUnraisable`. For + example:: static void - local_finalize(PyObject *self) + foo_finalize(PyObject *self) { - /* Save the current exception, if any. */ + // Save the current exception, if any. PyObject *exc = PyErr_GetRaisedException(); - /* ... */ + // ... - /* Restore the saved exception. */ + if (do_something_that_might_raise() != success_indicator) { + PyErr_WriteUnraisable(self); + goto done; + } + + done: + // Restore the saved exception. This silently discards any exception + // raised above, so be sure to call PyErr_WriteUnraisable first if + // necessary. PyErr_SetRaisedException(exc); } @@ -2182,11 +2559,19 @@ and :c:data:`PyType_Type` effectively act as defaults.) :c:macro:`Py_TPFLAGS_HAVE_FINALIZE` flags bit in order for this field to be used. This is no longer required. - .. seealso:: "Safe object finalization" (:pep:`442`) + .. seealso:: + + * :pep:`442`: "Safe object finalization" + * :ref:`life-cycle` for details about how this slot relates to other + slots. + * :c:func:`PyObject_CallFinalizer` + * :c:func:`PyObject_CallFinalizerFromDealloc` .. c:member:: vectorcallfunc PyTypeObject.tp_vectorcall + .. corresponding-type-slot:: Py_tp_vectorcall + A :ref:`vectorcall function ` to use for calls of this type object (rather than instances). In other words, ``tp_vectorcall`` can be used to optimize ``type.__call__``, @@ -2352,42 +2737,148 @@ Number Object Structures Python 3.0.1. .. c:member:: binaryfunc PyNumberMethods.nb_add + + .. corresponding-type-slot:: Py_nb_add + .. c:member:: binaryfunc PyNumberMethods.nb_subtract + + .. corresponding-type-slot:: Py_nb_subtract + .. c:member:: binaryfunc PyNumberMethods.nb_multiply + + .. corresponding-type-slot:: Py_nb_multiply + .. c:member:: binaryfunc PyNumberMethods.nb_remainder + + .. corresponding-type-slot:: Py_nb_remainder + .. c:member:: binaryfunc PyNumberMethods.nb_divmod + + .. corresponding-type-slot:: Py_nb_divmod + .. c:member:: ternaryfunc PyNumberMethods.nb_power + + .. corresponding-type-slot:: Py_nb_power + .. c:member:: unaryfunc PyNumberMethods.nb_negative + + .. corresponding-type-slot:: Py_nb_negative + .. c:member:: unaryfunc PyNumberMethods.nb_positive + + .. corresponding-type-slot:: Py_nb_positive + .. c:member:: unaryfunc PyNumberMethods.nb_absolute + + .. corresponding-type-slot:: Py_nb_absolute + .. c:member:: inquiry PyNumberMethods.nb_bool + + .. corresponding-type-slot:: Py_nb_bool + .. c:member:: unaryfunc PyNumberMethods.nb_invert + + .. corresponding-type-slot:: Py_nb_invert + .. c:member:: binaryfunc PyNumberMethods.nb_lshift + + .. corresponding-type-slot:: Py_nb_lshift + .. c:member:: binaryfunc PyNumberMethods.nb_rshift + + .. corresponding-type-slot:: Py_nb_rshift + .. c:member:: binaryfunc PyNumberMethods.nb_and + + .. corresponding-type-slot:: Py_nb_and + .. c:member:: binaryfunc PyNumberMethods.nb_xor + + .. corresponding-type-slot:: Py_nb_xor + .. c:member:: binaryfunc PyNumberMethods.nb_or + + .. corresponding-type-slot:: Py_nb_or + .. c:member:: unaryfunc PyNumberMethods.nb_int + + .. corresponding-type-slot:: Py_nb_int + .. c:member:: void *PyNumberMethods.nb_reserved + .. c:member:: unaryfunc PyNumberMethods.nb_float + + .. corresponding-type-slot:: Py_nb_float + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_add + + .. corresponding-type-slot:: Py_nb_inplace_add + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_subtract + + .. corresponding-type-slot:: Py_nb_inplace_subtract + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_multiply + + .. corresponding-type-slot:: Py_nb_inplace_multiply + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_remainder + + .. corresponding-type-slot:: Py_nb_inplace_remainder + .. c:member:: ternaryfunc PyNumberMethods.nb_inplace_power + + .. corresponding-type-slot:: Py_nb_inplace_power + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_lshift + + .. corresponding-type-slot:: Py_nb_inplace_lshift + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_rshift + + .. corresponding-type-slot:: Py_nb_inplace_rshift + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_and + + .. corresponding-type-slot:: Py_nb_inplace_and + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_xor + + .. corresponding-type-slot:: Py_nb_inplace_xor + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_or + + .. corresponding-type-slot:: Py_nb_inplace_or + .. c:member:: binaryfunc PyNumberMethods.nb_floor_divide + + .. corresponding-type-slot:: Py_nb_floor_divide + .. c:member:: binaryfunc PyNumberMethods.nb_true_divide + + .. corresponding-type-slot:: Py_nb_true_divide + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_floor_divide + + .. corresponding-type-slot:: Py_nb_inplace_floor_divide + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_true_divide + + .. corresponding-type-slot:: Py_nb_inplace_true_divide + .. c:member:: unaryfunc PyNumberMethods.nb_index + + .. corresponding-type-slot:: Py_nb_index + .. c:member:: binaryfunc PyNumberMethods.nb_matrix_multiply + + .. corresponding-type-slot:: Py_nb_matrix_multiply + .. c:member:: binaryfunc PyNumberMethods.nb_inplace_matrix_multiply + .. corresponding-type-slot:: Py_nb_inplace_matrix_multiply + + .. _mapping-structs: @@ -2404,12 +2895,16 @@ Mapping Object Structures .. c:member:: lenfunc PyMappingMethods.mp_length + .. corresponding-type-slot:: Py_mp_length + This function is used by :c:func:`PyMapping_Size` and :c:func:`PyObject_Size`, and has the same signature. This slot may be set to ``NULL`` if the object has no defined length. .. c:member:: binaryfunc PyMappingMethods.mp_subscript + .. corresponding-type-slot:: Py_mp_subscript + This function is used by :c:func:`PyObject_GetItem` and :c:func:`PySequence_GetSlice`, and has the same signature as :c:func:`!PyObject_GetItem`. This slot must be filled for the @@ -2418,6 +2913,8 @@ Mapping Object Structures .. c:member:: objobjargproc PyMappingMethods.mp_ass_subscript + .. corresponding-type-slot:: Py_mp_ass_subscript + This function is used by :c:func:`PyObject_SetItem`, :c:func:`PyObject_DelItem`, :c:func:`PySequence_SetSlice` and :c:func:`PySequence_DelSlice`. It has the same signature as @@ -2441,6 +2938,8 @@ Sequence Object Structures .. c:member:: lenfunc PySequenceMethods.sq_length + .. corresponding-type-slot:: Py_sq_length + This function is used by :c:func:`PySequence_Size` and :c:func:`PyObject_Size`, and has the same signature. It is also used for handling negative indices via the :c:member:`~PySequenceMethods.sq_item` @@ -2448,18 +2947,24 @@ Sequence Object Structures .. c:member:: binaryfunc PySequenceMethods.sq_concat + .. corresponding-type-slot:: Py_sq_concat + This function is used by :c:func:`PySequence_Concat` and has the same signature. It is also used by the ``+`` operator, after trying the numeric addition via the :c:member:`~PyNumberMethods.nb_add` slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_repeat + .. corresponding-type-slot:: Py_sq_repeat + This function is used by :c:func:`PySequence_Repeat` and has the same signature. It is also used by the ``*`` operator, after trying numeric multiplication via the :c:member:`~PyNumberMethods.nb_multiply` slot. .. c:member:: ssizeargfunc PySequenceMethods.sq_item + .. corresponding-type-slot:: Py_sq_item + This function is used by :c:func:`PySequence_GetItem` and has the same signature. It is also used by :c:func:`PyObject_GetItem`, after trying the subscription via the :c:member:`~PyMappingMethods.mp_subscript` slot. @@ -2473,6 +2978,8 @@ Sequence Object Structures .. c:member:: ssizeobjargproc PySequenceMethods.sq_ass_item + .. corresponding-type-slot:: Py_sq_ass_item + This function is used by :c:func:`PySequence_SetItem` and has the same signature. It is also used by :c:func:`PyObject_SetItem` and :c:func:`PyObject_DelItem`, after trying the item assignment and deletion @@ -2482,6 +2989,8 @@ Sequence Object Structures .. c:member:: objobjproc PySequenceMethods.sq_contains + .. corresponding-type-slot:: Py_sq_contains + This function may be used by :c:func:`PySequence_Contains` and has the same signature. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_Contains` simply traverses the sequence until it @@ -2489,6 +2998,8 @@ Sequence Object Structures .. c:member:: binaryfunc PySequenceMethods.sq_inplace_concat + .. corresponding-type-slot:: Py_sq_inplace_concat + This function is used by :c:func:`PySequence_InPlaceConcat` and has the same signature. It should modify its first operand, and return it. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_InPlaceConcat` @@ -2498,6 +3009,8 @@ Sequence Object Structures .. c:member:: ssizeargfunc PySequenceMethods.sq_inplace_repeat + .. corresponding-type-slot:: Py_sq_inplace_repeat + This function is used by :c:func:`PySequence_InPlaceRepeat` and has the same signature. It should modify its first operand, and return it. This slot may be left to ``NULL``, in this case :c:func:`!PySequence_InPlaceRepeat` @@ -2523,6 +3036,8 @@ Buffer Object Structures .. c:member:: getbufferproc PyBufferProcs.bf_getbuffer + .. corresponding-type-slot:: Py_bf_getbuffer + The signature of this function is:: int (PyObject *exporter, Py_buffer *view, int flags); @@ -2532,24 +3047,42 @@ Buffer Object Structures steps: (1) Check if the request can be met. If not, raise :exc:`BufferError`, - set :c:expr:`view->obj` to ``NULL`` and return ``-1``. + set ``view->obj`` to ``NULL`` and return ``-1``. (2) Fill in the requested fields. (3) Increment an internal counter for the number of exports. - (4) Set :c:expr:`view->obj` to *exporter* and increment :c:expr:`view->obj`. + (4) Set ``view->obj`` to *exporter* and increment ``view->obj``. (5) Return ``0``. + **Thread safety:** + + In the :term:`free-threaded build`, implementations must ensure: + + * The export counter increment in step (3) is atomic. + + * The underlying buffer data remains valid and at a stable memory + location for the lifetime of all exports. + + * For objects that support resizing or reallocation (such as + :class:`bytearray`), the export counter is checked atomically before + such operations, and :exc:`BufferError` is raised if exports exist. + + * The function is safe to call concurrently from multiple threads. + + See also :ref:`thread-safety-memoryview` for the Python-level + thread safety guarantees of :class:`memoryview` objects. + If *exporter* is part of a chain or tree of buffer providers, two main schemes can be used: * Re-export: Each member of the tree acts as the exporting object and - sets :c:expr:`view->obj` to a new reference to itself. + sets ``view->obj`` to a new reference to itself. * Redirect: The buffer request is redirected to the root object of the - tree. Here, :c:expr:`view->obj` will be a new reference to the root + tree. Here, ``view->obj`` will be a new reference to the root object. The individual fields of *view* are described in section @@ -2572,6 +3105,8 @@ Buffer Object Structures .. c:member:: releasebufferproc PyBufferProcs.bf_releasebuffer + .. corresponding-type-slot:: Py_bf_releasebuffer + The signature of this function is:: void (PyObject *exporter, Py_buffer *view); @@ -2585,13 +3120,23 @@ Buffer Object Structures (2) If the counter is ``0``, free all memory associated with *view*. + **Thread safety:** + + In the :term:`free-threaded build`: + + * The export counter decrement in step (1) must be atomic. + + * Resource cleanup when the counter reaches zero must be done atomically, + as the final release may race with concurrent releases from other + threads and deallocation must only happen once. + The exporter MUST use the :c:member:`~Py_buffer.internal` field to keep track of buffer-specific resources. This field is guaranteed to remain constant, while a consumer MAY pass a copy of the original buffer as the *view* argument. - This function MUST NOT decrement :c:expr:`view->obj`, since that is + This function MUST NOT decrement ``view->obj``, since that is done automatically in :c:func:`PyBuffer_Release` (this scheme is useful for breaking reference cycles). @@ -2626,6 +3171,8 @@ Async Object Structures .. c:member:: unaryfunc PyAsyncMethods.am_await + .. corresponding-type-slot:: Py_am_await + The signature of this function is:: PyObject *am_await(PyObject *self); @@ -2637,6 +3184,8 @@ Async Object Structures .. c:member:: unaryfunc PyAsyncMethods.am_aiter + .. corresponding-type-slot:: Py_am_aiter + The signature of this function is:: PyObject *am_aiter(PyObject *self); @@ -2649,6 +3198,8 @@ Async Object Structures .. c:member:: unaryfunc PyAsyncMethods.am_anext + .. corresponding-type-slot:: Py_am_anext + The signature of this function is:: PyObject *am_anext(PyObject *self); @@ -2659,6 +3210,8 @@ Async Object Structures .. c:member:: sendfunc PyAsyncMethods.am_send + .. corresponding-type-slot:: Py_am_send + The signature of this function is:: PySendResult am_send(PyObject *self, PyObject *arg, PyObject **result); diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 95987e872ce639d..aa2288fb77dc383 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -65,6 +65,27 @@ Python: .. versionadded:: 3.3 + The structure of a particular object can be determined using the following + macros. + The macros cannot fail; their behavior is undefined if their argument + is not a Python Unicode object. + + .. c:namespace:: NULL + + .. c:macro:: PyUnicode_IS_COMPACT(o) + + True if *o* uses the :c:struct:`PyCompactUnicodeObject` structure. + + .. versionadded:: 3.3 + + + .. c:macro:: PyUnicode_IS_COMPACT_ASCII(o) + + True if *o* uses the :c:struct:`PyASCIIObject` structure. + + .. versionadded:: 3.3 + + The following APIs are C macros and static inlined functions for fast checks and access to internal read-only data of Unicode objects: @@ -305,12 +326,22 @@ These APIs can be used to work with surrogates: Check if *ch* is a low surrogate (``0xDC00 <= ch <= 0xDFFF``). +.. c:function:: Py_UCS4 Py_UNICODE_HIGH_SURROGATE(Py_UCS4 ch) + + Return the high UTF-16 surrogate (``0xD800`` to ``0xDBFF``) for a Unicode + code point in the range ``[0x10000; 0x10FFFF]``. + +.. c:function:: Py_UCS4 Py_UNICODE_LOW_SURROGATE(Py_UCS4 ch) + + Return the low UTF-16 surrogate (``0xDC00`` to ``0xDFFF``) for a Unicode + code point in the range ``[0x10000; 0x10FFFF]``. + .. c:function:: Py_UCS4 Py_UNICODE_JOIN_SURROGATES(Py_UCS4 high, Py_UCS4 low) Join two surrogate code points and return a single :c:type:`Py_UCS4` value. *high* and *low* are respectively the leading and trailing surrogates in a - surrogate pair. *high* must be in the range [0xD800; 0xDBFF] and *low* must - be in the range [0xDC00; 0xDFFF]. + surrogate pair. *high* must be in the range ``[0xD800; 0xDBFF]`` and *low* must + be in the range ``[0xDC00; 0xDFFF]``. Creating and accessing Unicode strings @@ -632,7 +663,8 @@ APIs: Append the string *right* to the end of *p_left*. *p_left* must point to a :term:`strong reference` to a Unicode object; - :c:func:`!PyUnicode_Append` releases ("steals") this reference. + :c:func:`!PyUnicode_Append` releases (":term:`steals `") + this reference. On error, set *\*p_left* to ``NULL`` and set an exception. @@ -645,6 +677,17 @@ APIs: difference being that it decrements the reference count of *right* by one. +.. c:function:: PyObject* PyUnicode_BuildEncodingMap(PyObject* string) + + Return a mapping suitable for decoding a custom single-byte encoding. + Given a Unicode string *string* of up to 256 characters representing an encoding + table, returns either a compact internal mapping object or a dictionary + mapping character ordinals to byte values. Raises a :exc:`TypeError` and + return ``NULL`` on invalid input. + + .. versionadded:: 3.2 + + .. c:function:: const char* PyUnicode_GetDefaultEncoding(void) Return the name of the default string encoding, ``"utf-8"``. @@ -707,7 +750,7 @@ APIs: The string must not have been “used” yet. See :c:func:`PyUnicode_New` for details. - Return the number of written character, or return ``-1`` and raise an + Return the number of written characters, or return ``-1`` and raise an exception on error. .. versionadded:: 3.3 @@ -720,7 +763,7 @@ APIs: Return ``0`` on success, ``-1`` on error with an exception set. This function checks that *unicode* is a Unicode object, that the index is - not out of bounds, and that the object's reference count is one). + not out of bounds, and that the object's reference count is one. See :c:func:`PyUnicode_WRITE` for a version that skips these checks, making them your responsibility. @@ -1119,7 +1162,7 @@ These are the UTF-8 codec APIs: .. versionadded:: 3.3 .. versionchanged:: 3.7 - The return type is now ``const char *`` rather of ``char *``. + The return type is now ``const char *`` rather than ``char *``. .. versionchanged:: 3.10 This function is a part of the :ref:`limited API `. @@ -1141,7 +1184,7 @@ These are the UTF-8 codec APIs: .. versionadded:: 3.3 .. versionchanged:: 3.7 - The return type is now ``const char *`` rather of ``char *``. + The return type is now ``const char *`` rather than ``char *``. UTF-32 Codecs @@ -1450,10 +1493,6 @@ the user settings on the machine running the codec. .. versionadded:: 3.3 -Methods & Slots -""""""""""""""" - - .. _unicodemethodsandslots: Methods and Slot Functions @@ -1715,10 +1754,6 @@ They all return ``NULL`` or ``-1`` if an exception occurs. from user input, prefer calling :c:func:`PyUnicode_FromString` and :c:func:`PyUnicode_InternInPlace` directly. - .. impl-detail:: - - Strings interned this way are made :term:`immortal`. - .. c:function:: unsigned int PyUnicode_CHECK_INTERNED(PyObject *str) @@ -1795,9 +1830,22 @@ object. See also :c:func:`PyUnicodeWriter_DecodeUTF8Stateful`. +.. c:function:: int PyUnicodeWriter_WriteASCII(PyUnicodeWriter *writer, const char *str, Py_ssize_t size) + + Write the ASCII string *str* into *writer*. + + *size* is the string length in bytes. If *size* is equal to ``-1``, call + ``strlen(str)`` to get the string length. + + *str* must only contain ASCII characters. The behavior is undefined if + *str* contains non-ASCII characters. + + On success, return ``0``. + On error, set an exception, leave the writer unchanged, and return ``-1``. + .. c:function:: int PyUnicodeWriter_WriteWideChar(PyUnicodeWriter *writer, const wchar_t *str, Py_ssize_t size) - Writer the wide string *str* into *writer*. + Write the wide string *str* into *writer*. *size* is a number of wide characters. If *size* is equal to ``-1``, call ``wcslen(str)`` to get the string length. @@ -1821,13 +1869,23 @@ object. On success, return ``0``. On error, set an exception, leave the writer unchanged, and return ``-1``. + To write a :class:`str` subclass which overrides the :meth:`~object.__str__` + method, :c:func:`PyUnicode_FromObject` can be used to get the original + string. + .. c:function:: int PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) Call :c:func:`PyObject_Repr` on *obj* and write the output into *writer*. + If *obj* is ``NULL``, write the string ``""`` into *writer*. + On success, return ``0``. On error, set an exception, leave the writer unchanged, and return ``-1``. + .. versionchanged:: 3.14.4 + + Added support for ``NULL``. + .. c:function:: int PyUnicodeWriter_WriteSubstring(PyUnicodeWriter *writer, PyObject *str, Py_ssize_t start, Py_ssize_t end) Write the substring ``str[start:end]`` into *writer*. diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index 1ef4181d52eb101..bba5c7f8ecf7751 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -13,8 +13,9 @@ the interpreter. Several of these functions accept a start symbol from the grammar as a parameter. The available start symbols are :c:data:`Py_eval_input`, -:c:data:`Py_file_input`, and :c:data:`Py_single_input`. These are described -following the functions which accept them as parameters. +:c:data:`Py_file_input`, :c:data:`Py_single_input`, and +:c:data:`Py_func_type_input`. These are described following the functions +which accept them as parameters. Note also that several of these functions take :c:expr:`FILE*` parameters. One particular issue which needs to be handled carefully is that the :c:type:`FILE` @@ -99,18 +100,12 @@ the same library that the Python runtime is using. Otherwise, Python may not handle script file with LF line ending correctly. -.. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) - - This is a simplified interface to :c:func:`PyRun_InteractiveOneFlags` below, - leaving *flags* set to ``NULL``. - - -.. c:function:: int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) +.. c:function:: int PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags) Read and execute a single statement from a file associated with an interactive device according to the *flags* argument. The user will be - prompted using ``sys.ps1`` and ``sys.ps2``. *filename* is decoded from the - :term:`filesystem encoding and error handler`. + prompted using ``sys.ps1`` and ``sys.ps2``. *filename* must be a Python + :class:`str` object. Returns ``0`` when the input was executed successfully, ``-1`` if there was an exception, or an error code @@ -119,6 +114,19 @@ the same library that the Python runtime is using. :file:`Python.h`, so must be included specifically if needed.) +.. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) + + This is a simplified interface to :c:func:`PyRun_InteractiveOneFlags` below, + leaving *flags* set to ``NULL``. + + +.. c:function:: int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) + + Similar to :c:func:`PyRun_InteractiveOneObject`, but *filename* is a + :c:expr:`const char*`, which is decoded from the + :term:`filesystem encoding and error handler`. + + .. c:function:: int PyRun_InteractiveLoop(FILE *fp, const char *filename) This is a simplified interface to :c:func:`PyRun_InteractiveLoopFlags` below, @@ -140,7 +148,7 @@ the same library that the Python runtime is using. interpreter prompt is about to become idle and wait for user input from the terminal. The return value is ignored. Overriding this hook can be used to integrate the interpreter's prompt with other - event loops, as done in the :file:`Modules/_tkinter.c` in the + event loops, as done in :file:`Modules/_tkinter.c` in the Python source code. .. versionchanged:: 3.12 @@ -183,7 +191,7 @@ the same library that the Python runtime is using. objects *globals* and *locals* with the compiler flags specified by *flags*. *globals* must be a dictionary; *locals* can be any object that implements the mapping protocol. The parameter *start* specifies - the start token that should be used to parse the source code. + the start symbol and must be one of the :ref:`available start symbols `. Returns the result of executing the code as a Python object, or ``NULL`` if an exception was raised. @@ -231,9 +239,9 @@ the same library that the Python runtime is using. .. c:function:: PyObject* Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize) Parse and compile the Python source code in *str*, returning the resulting code - object. The start token is given by *start*; this can be used to constrain the - code which can be compiled and should be :c:data:`Py_eval_input`, - :c:data:`Py_file_input`, or :c:data:`Py_single_input`. The filename specified by + object. The start symbol is given by *start*; this can be used to constrain the + code which can be compiled and should be :ref:`available start symbols + `. The filename specified by *filename* is used to construct the code object and may appear in tracebacks or :exc:`SyntaxError` exception messages. This returns ``NULL`` if the code cannot be parsed or compiled. @@ -296,32 +304,6 @@ the same library that the Python runtime is using. true on success, false on failure. -.. c:var:: int Py_eval_input - - .. index:: single: Py_CompileString (C function) - - The start symbol from the Python grammar for isolated expressions; for use with - :c:func:`Py_CompileString`. - - -.. c:var:: int Py_file_input - - .. index:: single: Py_CompileString (C function) - - The start symbol from the Python grammar for sequences of statements as read - from a file or other source; for use with :c:func:`Py_CompileString`. This is - the symbol to use when compiling arbitrarily long Python source code. - - -.. c:var:: int Py_single_input - - .. index:: single: Py_CompileString (C function) - - The start symbol from the Python grammar for a single statement; for use with - :c:func:`Py_CompileString`. This is the symbol used for the interactive - interpreter loop. - - .. c:struct:: PyCompilerFlags This is the structure used to hold compiler flags. In cases where code is only @@ -361,7 +343,210 @@ the same library that the Python runtime is using. :py:mod:`!ast` Python module, which exports these constants under the same names. - .. c:var:: int CO_FUTURE_DIVISION + .. rubric:: Low-level flags + + The following flags and masks serve narrow needs of the standard + library and interactive interpreters. Code outside the standard + library rarely has a reason to use them. They are considered + implementation details and may change at any time. + + .. c:macro:: PyCF_ALLOW_INCOMPLETE_INPUT + + This flag is a private interface between the compiler and the + :mod:`codeop` module. Do not use it; its behavior is unsupported + and may change without warning. + + With this flag set, when compilation fails because the source text + ends where more input is expected, for example in the middle of an + indented block or an unterminated string literal, the error raised + is the undocumented ``_IncompleteInputError``, a subclass of + :exc:`SyntaxError`. The :mod:`codeop` module sets this flag, + together with :c:macro:`PyCF_DONT_IMPLY_DEDENT`, to tell input + that is incomplete apart from input with a real syntax error, so + that interactive interpreters know when to prompt for another + line instead of reporting an error. + + .. versionadded:: 3.11 + + .. c:macro:: PyCF_DONT_IMPLY_DEDENT + + By default, when compiling with the :c:var:`Py_single_input` start + symbol, reaching the end of the source text implicitly closes any + open indented blocks. With this flag set, open blocks are only + closed if the last line of the source ends with a newline; otherwise, + compilation fails with a :exc:`SyntaxError`: + + .. code-block:: c + + PyCompilerFlags flags = { + .cf_flags = 0, + .cf_feature_version = PY_MINOR_VERSION, + }; + const char *source = "if a:\n pass"; + + /* The "if" block is closed implicitly; + this returns a code object: */ + Py_CompileStringFlags(source, "", Py_single_input, &flags); + + /* With the flag, this fails with a SyntaxError, + because the last line does not end with a newline: */ + flags.cf_flags = PyCF_DONT_IMPLY_DEDENT; + Py_CompileStringFlags(source, "", Py_single_input, &flags); + + The :mod:`codeop` module uses this flag to detect incomplete + interactive input. While the user is still typing inside an + indented block, the source does not yet end with a newline, so it + fails to compile and the user is prompted for another line. + + .. c:macro:: PyCF_IGNORE_COOKIE + + Read the source text as UTF-8, ignoring its :pep:`263` encoding + declaration ("coding cookie"), if any: + + .. code-block:: c + + PyCompilerFlags flags = { + .cf_flags = 0, + .cf_feature_version = PY_MINOR_VERSION, + }; + const char *source = "# coding: latin-1\ns = '\xe9'\n"; + + /* The coding cookie is honored: byte 0xE9 is decoded as + Latin-1, and this returns a code object that sets s to "é": */ + Py_CompileStringFlags(source, "", Py_file_input, &flags); + + /* With the flag, the cookie is ignored and compilation fails + with a SyntaxError, because 0xE9 is not valid UTF-8: */ + flags.cf_flags = PyCF_IGNORE_COOKIE; + Py_CompileStringFlags(source, "", Py_file_input, &flags); + + The :func:`compile`, :func:`eval` and :func:`exec` built-in functions + set this flag when the source is a :class:`str` object, because they + pass the text to the parser encoded as UTF-8. + + .. c:macro:: PyCF_SOURCE_IS_UTF8 + + Mark the source text as known to be UTF-8 encoded. + The :func:`compile`, :func:`eval` and :func:`exec` built-in functions + set this flag, but it currently has no effect. + + The "``PyCF``" flags above can be combined with "``CO_FUTURE``" flags such + as :c:macro:`CO_FUTURE_ANNOTATIONS` to enable features normally + selectable using :ref:`future statements `. + See :ref:`c_codeobject_flags` for a complete list. + + The following masks combine several flags: + + .. c:macro:: PyCF_MASK + + Bitmask of all ``CO_FUTURE`` flags (see :ref:`c_codeobject_flags`), + which select features normally enabled by + :ref:`future statements `. + When code compiled with a ``PyCompilerFlags *flags`` argument + contains a ``from __future__ import`` statement, the flag for the + imported feature is added to *flags*, so that code executed later + in the same context inherits it. + + .. c:macro:: PyCF_MASK_OBSOLETE + + Do not use this mask in new code. It is kept only so that old + code passing its flags to :func:`compile` keeps working. + + Bitmask of flags for obsolete future features that no longer + have any effect. + + .. c:macro:: PyCF_COMPILE_MASK + + Bitmask of all ``PyCF`` flags that change how the source is + compiled, such as :c:macro:`PyCF_ONLY_AST`. + The :func:`compile` built-in function uses this mask to validate + its *flags* argument. + + +.. _start-symbols: + +Available start symbols +^^^^^^^^^^^^^^^^^^^^^^^ + + +.. c:var:: int Py_eval_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for isolated expressions; for use with + :c:func:`Py_CompileString`. + + +.. c:var:: int Py_file_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for sequences of statements as read + from a file or other source; for use with :c:func:`Py_CompileString`. This is + the symbol to use when compiling arbitrarily long Python source code. + + +.. c:var:: int Py_single_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for a single statement; for use with + :c:func:`Py_CompileString`. This is the symbol used for the interactive + interpreter loop. + + +.. c:var:: int Py_func_type_input + + .. index:: single: Py_CompileString (C function) + + The start symbol from the Python grammar for a function type; for use with + :c:func:`Py_CompileString`. This is used to parse "signature type comments" + from :pep:`484`. + + This requires the :c:macro:`PyCF_ONLY_AST` flag to be set. + + .. seealso:: + * :py:class:`ast.FunctionType` + * :pep:`484` + + .. versionadded:: 3.8 + + +Stack Effects +^^^^^^^^^^^^^ + +.. seealso:: + :py:func:`dis.stack_effect` + + +.. c:macro:: PY_INVALID_STACK_EFFECT + + Sentinel value representing an invalid stack effect. + + This is currently equivalent to ``INT_MAX``. + + .. versionadded:: 3.8 + + +.. c:function:: int PyCompile_OpcodeStackEffect(int opcode, int oparg) + + Compute the stack effect of *opcode* with argument *oparg*. + + On success, this function returns the stack effect; on failure, this + returns :c:macro:`PY_INVALID_STACK_EFFECT`. + + .. versionadded:: 3.4 + + +.. c:function:: int PyCompile_OpcodeStackEffectWithJump(int opcode, int oparg, int jump) + + Similar to :c:func:`PyCompile_OpcodeStackEffect`, but don't include the + stack effect of jumping if *jump* is zero. + + If *jump* is ``0``, this will not include the stack effect of jumping, but + if *jump* is ``1`` or ``-1``, this will include it. + + On success, this function returns the stack effect; on failure, this + returns :c:macro:`PY_INVALID_STACK_EFFECT`. - This bit can be set in *flags* to cause division operator ``/`` to be - interpreted as "true division" according to :pep:`238`. + .. versionadded:: 3.8 diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index c3c6cf413dcef58..28b93add912b3af 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -19,7 +19,14 @@ as much as it can. .. c:function:: int PyWeakref_CheckRef(PyObject *ob) - Return non-zero if *ob* is a reference object. This function always succeeds. + Return non-zero if *ob* is a reference object or a subclass of the reference + type. This function always succeeds. + + +.. c:function:: int PyWeakref_CheckRefExact(PyObject *ob) + + Return non-zero if *ob* is a reference object, but not a subclass of the + reference type. This function always succeeds. .. c:function:: int PyWeakref_CheckProxy(PyObject *ob) @@ -35,8 +42,12 @@ as much as it can. callable object that receives notification when *ob* is garbage collected; it should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* is not a - weakly referenceable object, or if *callback* is not callable, ``None``, or - ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + weakly referenceable object, this will raise :exc:`TypeError` and return + ``NULL``. + + .. seealso:: + :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly + referenceable. .. c:function:: PyObject* PyWeakref_NewProxy(PyObject *ob, PyObject *callback) @@ -47,8 +58,12 @@ as much as it can. be a callable object that receives notification when *ob* is garbage collected; it should accept a single parameter, which will be the weak reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* - is not a weakly referenceable object, or if *callback* is not callable, - ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. + weakly referenceable object, this will raise :exc:`TypeError` and return + ``NULL``. + + .. seealso:: + :c:func:`PyType_SUPPORTS_WEAKREFS` for checking if *ob* is weakly + referenceable. .. c:function:: int PyWeakref_GetRef(PyObject *ref, PyObject **pobj) diff --git a/Doc/conf.py b/Doc/conf.py index 467961dd5e2bff2..98e5531585d3f65 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -8,15 +8,13 @@ import os import sys -from importlib import import_module from importlib.util import find_spec # Make our custom extensions available to Sphinx sys.path.append(os.path.abspath('tools/extensions')) sys.path.append(os.path.abspath('includes')) -# Python specific content from Doc/Tools/extensions/pyspecific.py -from pyspecific import SOURCE_URI +from patchlevel import get_header_version_info, get_version_info # General configuration # --------------------- @@ -42,8 +40,10 @@ # Skip if downstream redistributors haven't installed them _OPTIONAL_EXTENSIONS = ( + 'linklint.ext', 'notfound.extension', 'sphinxext.opengraph', + 'sphinxcontrib.rsvgconverter', ) for optional_ext in _OPTIONAL_EXTENSIONS: try: @@ -70,15 +70,20 @@ # General substitutions. project = 'Python' copyright = "2001 Python Software Foundation" +_doc_authors = 'Python documentation authors' # We look for the Include/patchlevel.h file in the current Python source tree # and replace the values accordingly. # See Doc/tools/extensions/patchlevel.py -version, release = import_module('patchlevel').get_version_info() +version, release = get_version_info() rst_epilog = f""" .. |python_version_literal| replace:: ``Python {version}`` .. |python_x_dot_y_literal| replace:: ``python{version}`` +.. |python_x_dot_y_t_literal| replace:: ``python{version}t`` +.. |python_x_dot_y_t_literal_config| replace:: ``python{version}t-config`` +.. |x_dot_y_b2_literal| replace:: ``{version}.0b2`` +.. |applications_python_version_literal| replace:: ``/Applications/Python {version}/`` .. |usr_local_bin_python_x_dot_y_literal| replace:: ``/usr/local/bin/python{version}`` .. Apparently this how you hack together a formatted link: @@ -170,6 +175,7 @@ ('c:type', '__int64'), ('c:type', 'unsigned __int64'), ('c:type', 'double'), + ('c:type', '_Float16'), # Standard C structures ('c:struct', 'in6_addr'), ('c:struct', 'in_addr'), @@ -217,99 +223,18 @@ ('envvar', 'USER'), ('envvar', 'USERNAME'), ('envvar', 'USERPROFILE'), - # Deprecated function that was never documented: - ('py:func', 'getargspec'), - ('py:func', 'inspect.getargspec'), - # Undocumented modules that users shouldn't have to worry about - # (implementation details of `os.path`): - ('py:mod', 'ntpath'), - ('py:mod', 'posixpath'), ] # Temporary undocumented names. # In future this list must be empty. nitpick_ignore += [ - # C API: Standard Python exception classes - ('c:data', 'PyExc_ArithmeticError'), - ('c:data', 'PyExc_AssertionError'), - ('c:data', 'PyExc_AttributeError'), - ('c:data', 'PyExc_BaseException'), - ('c:data', 'PyExc_BlockingIOError'), - ('c:data', 'PyExc_BrokenPipeError'), - ('c:data', 'PyExc_BufferError'), - ('c:data', 'PyExc_ChildProcessError'), - ('c:data', 'PyExc_ConnectionAbortedError'), - ('c:data', 'PyExc_ConnectionError'), - ('c:data', 'PyExc_ConnectionRefusedError'), - ('c:data', 'PyExc_ConnectionResetError'), - ('c:data', 'PyExc_EOFError'), - ('c:data', 'PyExc_Exception'), - ('c:data', 'PyExc_FileExistsError'), - ('c:data', 'PyExc_FileNotFoundError'), - ('c:data', 'PyExc_FloatingPointError'), - ('c:data', 'PyExc_GeneratorExit'), - ('c:data', 'PyExc_ImportError'), - ('c:data', 'PyExc_IndentationError'), - ('c:data', 'PyExc_IndexError'), - ('c:data', 'PyExc_InterruptedError'), - ('c:data', 'PyExc_IsADirectoryError'), - ('c:data', 'PyExc_KeyboardInterrupt'), - ('c:data', 'PyExc_KeyError'), - ('c:data', 'PyExc_LookupError'), - ('c:data', 'PyExc_MemoryError'), - ('c:data', 'PyExc_ModuleNotFoundError'), - ('c:data', 'PyExc_NameError'), - ('c:data', 'PyExc_NotADirectoryError'), - ('c:data', 'PyExc_NotImplementedError'), - ('c:data', 'PyExc_OSError'), - ('c:data', 'PyExc_OverflowError'), - ('c:data', 'PyExc_PermissionError'), - ('c:data', 'PyExc_ProcessLookupError'), - ('c:data', 'PyExc_PythonFinalizationError'), - ('c:data', 'PyExc_RecursionError'), - ('c:data', 'PyExc_ReferenceError'), - ('c:data', 'PyExc_RuntimeError'), - ('c:data', 'PyExc_StopAsyncIteration'), - ('c:data', 'PyExc_StopIteration'), - ('c:data', 'PyExc_SyntaxError'), - ('c:data', 'PyExc_SystemError'), - ('c:data', 'PyExc_SystemExit'), - ('c:data', 'PyExc_TabError'), - ('c:data', 'PyExc_TimeoutError'), - ('c:data', 'PyExc_TypeError'), - ('c:data', 'PyExc_UnboundLocalError'), - ('c:data', 'PyExc_UnicodeDecodeError'), - ('c:data', 'PyExc_UnicodeEncodeError'), - ('c:data', 'PyExc_UnicodeError'), - ('c:data', 'PyExc_UnicodeTranslateError'), - ('c:data', 'PyExc_ValueError'), - ('c:data', 'PyExc_ZeroDivisionError'), - # C API: Standard Python warning classes - ('c:data', 'PyExc_BytesWarning'), - ('c:data', 'PyExc_DeprecationWarning'), - ('c:data', 'PyExc_FutureWarning'), - ('c:data', 'PyExc_ImportWarning'), - ('c:data', 'PyExc_PendingDeprecationWarning'), - ('c:data', 'PyExc_ResourceWarning'), - ('c:data', 'PyExc_RuntimeWarning'), - ('c:data', 'PyExc_SyntaxWarning'), - ('c:data', 'PyExc_UnicodeWarning'), - ('c:data', 'PyExc_UserWarning'), - ('c:data', 'PyExc_Warning'), - # Undocumented public C macros - ('c:macro', 'Py_BUILD_ASSERT'), - ('c:macro', 'Py_BUILD_ASSERT_EXPR'), # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot # be resolved, as the method is currently undocumented. For context, see # https://github.com/python/cpython/pull/103289. ('py:meth', '_SubParsersAction.add_parser'), # Attributes/methods/etc. that definitely should be documented better, # but are deferred for now: - ('py:attr', '__annotations__'), - ('py:meth', '__missing__'), ('py:attr', '__wrapped__'), - ('py:attr', 'decimal.Context.clamp'), - ('py:meth', 'index'), # list.index, tuple.index, etc. ] # gh-106948: Copy standard C types declared in the "c:type" domain and C @@ -433,73 +358,79 @@ 'papersize': 'a4paper', # The font size ('10pt', '11pt' or '12pt'). 'pointsize': '10pt', + 'maxlistdepth': '8', # See https://github.com/python/cpython/issues/139588 } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, document class [howto/manual]). -_stdauthor = 'Guido van Rossum and the Python development team' latex_documents = [ - ('c-api/index', 'c-api.tex', 'The Python/C API', _stdauthor, 'manual'), + ('c-api/index', 'c-api.tex', 'The Python/C API', _doc_authors, 'manual'), ( 'extending/index', 'extending.tex', 'Extending and Embedding Python', - _stdauthor, + _doc_authors, 'manual', ), ( 'installing/index', 'installing.tex', 'Installing Python Modules', - _stdauthor, + _doc_authors, 'manual', ), ( 'library/index', 'library.tex', 'The Python Library Reference', - _stdauthor, + _doc_authors, 'manual', ), ( 'reference/index', 'reference.tex', 'The Python Language Reference', - _stdauthor, + _doc_authors, 'manual', ), ( 'tutorial/index', 'tutorial.tex', 'Python Tutorial', - _stdauthor, + _doc_authors, 'manual', ), ( 'using/index', 'using.tex', 'Python Setup and Usage', - _stdauthor, + _doc_authors, 'manual', ), ( 'faq/index', 'faq.tex', 'Python Frequently Asked Questions', - _stdauthor, + _doc_authors, 'manual', ), ( 'whatsnew/' + version, 'whatsnew.tex', 'What\'s New in Python', - 'A. M. Kuchling', + _doc_authors, 'howto', ), ] # Collect all HOWTOs individually latex_documents.extend( - ('howto/' + fn[:-4], 'howto-' + fn[:-4] + '.tex', '', _stdauthor, 'howto') + ( + 'howto/' + fn[:-4], + 'howto-' + fn[:-4] + '.tex', + '', + _doc_authors, + 'howto', + ) for fn in os.listdir('howto') if fn.endswith('.rst') and fn != 'index.rst' ) @@ -510,13 +441,33 @@ # Options for Epub output # ----------------------- -epub_author = 'Python Documentation Authors' +epub_author = _doc_authors epub_publisher = 'Python Software Foundation' +epub_exclude_files = ('index.xhtml', 'download.xhtml') # index pages are not valid xhtml # https://github.com/sphinx-doc/sphinx/issues/12359 epub_use_index = False +# translation tag +# --------------- + +language_code = None +for arg in sys.argv: + if arg.startswith('language='): + language_code = arg.split('=', 1)[1] + +if language_code: + tags.add('translation') # noqa: F821 + + rst_epilog += f"""\ +.. _TRANSLATION_REPO: https://github.com/python/python-docs-{language_code.replace("_", "-").lower()} +""" # noqa: F821 +else: + rst_epilog += """\ +.. _TRANSLATION_REPO: https://github.com/python +""" + # Options for the coverage checker # -------------------------------- @@ -602,15 +553,20 @@ r'https://unix.org/version2/whatsnew/lp64_wp.html', ] + # Options for sphinx.ext.extlinks # ------------------------------- +v = get_header_version_info() +branch = "main" if v.releaselevel == "alpha" else f"{v.major}.{v.minor}" + # This config is a dictionary of external sites, # mapping unique short aliases to a base URL and a prefix. # https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html extlinks = { + "oss-fuzz": ("https://issues.oss-fuzz.com/issues/%s", "#%s"), "pypi": ("https://pypi.org/project/%s/", "%s"), - "source": (SOURCE_URI, "%s"), + "source": (f"https://github.com/python/cpython/tree/{branch}/%s", "%s"), } extlinks_detect_hardcoded_links = True @@ -620,6 +576,18 @@ # Relative filename of the data files refcount_file = 'data/refcounts.dat' stable_abi_file = 'data/stable_abi.dat' +threadsafety_file = 'data/threadsafety.dat' + +# Options for notfound.extension +# ------------------------------- + +if not os.getenv("READTHEDOCS"): + if language_code: + notfound_urls_prefix = ( + f'/{language_code.replace("_", "-").lower()}/{version}/' + ) + else: + notfound_urls_prefix = f'/{version}/' # Options for sphinxext-opengraph # ------------------------------- @@ -630,13 +598,11 @@ 'image': '_static/og-image.png', 'line_color': '#3776ab', } -ogp_custom_meta_tags = [ - '', -] +ogp_custom_meta_tags = ('',) if 'create-social-cards' not in tags: # noqa: F821 # Define a static preview image when not creating social cards ogp_image = '_static/og-image.png' - ogp_custom_meta_tags += [ + ogp_custom_meta_tags += ( '', '', - ] + ) diff --git a/Doc/data/python3.14.abi b/Doc/data/python3.14.abi new file mode 100644 index 000000000000000..52432e8ea6ef045 --- /dev/null +++ b/Doc/data/python3.14.abi @@ -0,0 +1,31729 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index ca99b9e6d371418..48b800fdf9a5336 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -963,21 +963,45 @@ PyFunction_Check:PyObject*:o:0: PyFunction_GetAnnotations:PyObject*::0: PyFunction_GetAnnotations:PyObject*:op:0: +PyFunction_GET_ANNOTATIONS:PyObject*::0: +PyFunction_GET_ANNOTATIONS:PyObject*:op:0: + PyFunction_GetClosure:PyObject*::0: PyFunction_GetClosure:PyObject*:op:0: +PyFunction_GET_CLOSURE:PyObject*::0: +PyFunction_GET_CLOSURE:PyObject*:op:0: + PyFunction_GetCode:PyObject*::0: PyFunction_GetCode:PyObject*:op:0: +PyFunction_GET_CODE:PyObject*::0: +PyFunction_GET_CODE:PyObject*:op:0: + PyFunction_GetDefaults:PyObject*::0: PyFunction_GetDefaults:PyObject*:op:0: +PyFunction_GET_DEFAULTS:PyObject*::0: +PyFunction_GET_DEFAULTS:PyObject*:op:0: + +PyFunction_GetKwDefaults:PyObject*::0: +PyFunction_GetKwDefaults:PyObject*:op:0: + +PyFunction_GET_KW_DEFAULTS:PyObject*::0: +PyFunction_GET_KW_DEFAULTS:PyObject*:op:0: + PyFunction_GetGlobals:PyObject*::0: PyFunction_GetGlobals:PyObject*:op:0: +PyFunction_GET_GLOBALS:PyObject*::0: +PyFunction_GET_GLOBALS:PyObject*:op:0: + PyFunction_GetModule:PyObject*::0: PyFunction_GetModule:PyObject*:op:0: +PyFunction_GET_MODULE:PyObject*::0: +PyFunction_GET_MODULE:PyObject*:op:0: + PyFunction_New:PyObject*::+1: PyFunction_New:PyObject*:code:+1: PyFunction_New:PyObject*:globals:+1: @@ -1120,6 +1144,9 @@ PyInterpreterState_Clear:PyInterpreterState*:interp:: PyInterpreterState_Delete:void::: PyInterpreterState_Delete:PyInterpreterState*:interp:: +PyInterpreterState_GetDict:PyObject*::0: +PyInterpreterState_GetDict:PyInterpreterState*:interp:: + PyInterpreterState_GetID:int64_t::: PyInterpreterState_GetID:PyInterpreterState*:interp:: @@ -1492,9 +1519,6 @@ PyModule_SetDocString:int::: PyModule_SetDocString:PyObject*:module:0: PyModule_SetDocString:const char*:docstring:: -PyModuleDef_Init:PyObject*::0: -PyModuleDef_Init:PyModuleDef*:def:: - PyNumber_Absolute:PyObject*::+1: PyNumber_Absolute:PyObject*:o:0: @@ -2391,6 +2415,13 @@ PyType_GetFlags:PyTypeObject*:type:0: PyType_GetName:PyObject*::+1: PyType_GetName:PyTypeObject*:type:0: +PyType_GetModule:PyObject*::0: +PyType_GetModule:PyTypeObject*:type:0: + +PyType_GetModuleByDef:PyObject*::0: +PyType_GetModuleByDef:PyTypeObject*:type:0: +PyType_GetModuleByDef:PyModuleDef*:def:: + PyType_GetQualName:PyObject*::+1: PyType_GetQualName:PyTypeObject*:type:0: @@ -2781,6 +2812,9 @@ PyUnicode_AppendAndDel:void::: PyUnicode_AppendAndDel:PyObject**:p_left:0: PyUnicode_AppendAndDel:PyObject*:right:-1: +PyUnicode_BuildEncodingMap:PyObject*::+1: +PyUnicode_BuildEncodingMap:PyObject*:string::: + PyUnicode_GetDefaultEncoding:const char*::: PyUnicode_GetDefaultEncoding::void:: diff --git a/Doc/data/stable_abi.dat b/Doc/data/stable_abi.dat index 3d68487d07baf27..5fd69346ffb5e03 100644 --- a/Doc/data/stable_abi.dat +++ b/Doc/data/stable_abi.dat @@ -1,4 +1,12 @@ role,name,added,ifdef_note,struct_abi_kind +macro,METH_CLASS,3.2,, +macro,METH_COEXIST,3.2,, +macro,METH_FASTCALL,3.10,, +macro,METH_METHOD,3.7,, +macro,METH_NOARGS,3.2,, +macro,METH_O,3.2,, +macro,METH_STATIC,3.2,, +macro,METH_VARARGS,3.2,, macro,PY_VECTORCALL_ARGUMENTS_OFFSET,3.12,, func,PyAIter_Check,3.10,, func,PyArg_Parse,3.2,, @@ -8,6 +16,26 @@ func,PyArg_UnpackTuple,3.2,, func,PyArg_VaParse,3.2,, func,PyArg_VaParseTupleAndKeywords,3.2,, func,PyArg_ValidateKeywordArguments,3.2,, +macro,PyBUF_ANY_CONTIGUOUS,3.11,, +macro,PyBUF_CONTIG,3.11,, +macro,PyBUF_CONTIG_RO,3.11,, +macro,PyBUF_C_CONTIGUOUS,3.11,, +macro,PyBUF_FORMAT,3.11,, +macro,PyBUF_FULL,3.11,, +macro,PyBUF_FULL_RO,3.11,, +macro,PyBUF_F_CONTIGUOUS,3.11,, +macro,PyBUF_INDIRECT,3.11,, +macro,PyBUF_MAX_NDIM,3.11,, +macro,PyBUF_ND,3.11,, +macro,PyBUF_READ,3.11,, +macro,PyBUF_RECORDS,3.11,, +macro,PyBUF_RECORDS_RO,3.11,, +macro,PyBUF_SIMPLE,3.11,, +macro,PyBUF_STRIDED,3.11,, +macro,PyBUF_STRIDED_RO,3.11,, +macro,PyBUF_STRIDES,3.11,, +macro,PyBUF_WRITABLE,3.11,, +macro,PyBUF_WRITE,3.11,, data,PyBaseObject_Type,3.2,, func,PyBool_FromLong,3.2,, data,PyBool_Type,3.2,, @@ -426,6 +454,7 @@ data,PyMethodDescr_Type,3.2,, type,PyModuleDef,3.2,,full-abi type,PyModuleDef_Base,3.2,,full-abi func,PyModuleDef_Init,3.5,, +type,PyModuleDef_Slot,3.5,,full-abi data,PyModuleDef_Type,3.5,, func,PyModule_Add,3.13,, func,PyModule_AddFunctions,3.7,, @@ -836,6 +865,14 @@ func,PyWeakref_NewRef,3.2,, data,PyWrapperDescr_Type,3.2,, func,PyWrapper_New,3.2,, data,PyZip_Type,3.2,, +macro,Py_ASNATIVEBYTES_ALLOW_INDEX,3.14,, +macro,Py_ASNATIVEBYTES_BIG_ENDIAN,3.14,, +macro,Py_ASNATIVEBYTES_DEFAULTS,3.14,, +macro,Py_ASNATIVEBYTES_LITTLE_ENDIAN,3.14,, +macro,Py_ASNATIVEBYTES_NATIVE_ENDIAN,3.14,, +macro,Py_ASNATIVEBYTES_REJECT_NEGATIVE,3.14,, +macro,Py_ASNATIVEBYTES_UNSIGNED_BUFFER,3.14,, +macro,Py_AUDIT_READ,3.12,, func,Py_AddPendingCall,3.2,, func,Py_AtExit,3.2,, macro,Py_BEGIN_ALLOW_THREADS,3.2,, @@ -888,22 +925,136 @@ func,Py_NewInterpreter,3.2,, func,Py_NewRef,3.10,, func,Py_PACK_FULL_VERSION,3.14,, func,Py_PACK_VERSION,3.14,, +macro,Py_READONLY,3.12,, func,Py_REFCNT,3.14,, +macro,Py_RELATIVE_OFFSET,3.12,, func,Py_ReprEnter,3.2,, func,Py_ReprLeave,3.2,, func,Py_SetProgramName,3.2,, func,Py_SetPythonHome,3.2,, func,Py_SetRecursionLimit,3.2,, +macro,Py_TPFLAGS_BASETYPE,3.2,, +macro,Py_TPFLAGS_DEFAULT,3.2,, +macro,Py_TPFLAGS_HAVE_GC,3.2,, +macro,Py_TPFLAGS_HAVE_VECTORCALL,3.12,, +macro,Py_TPFLAGS_ITEMS_AT_END,3.12,, +macro,Py_TPFLAGS_METHOD_DESCRIPTOR,3.8,, +macro,Py_TP_USE_SPEC,3.14,, func,Py_TYPE,3.14,, +macro,Py_T_BOOL,3.12,, +macro,Py_T_BYTE,3.12,, +macro,Py_T_CHAR,3.12,, +macro,Py_T_DOUBLE,3.12,, +macro,Py_T_FLOAT,3.12,, +macro,Py_T_INT,3.12,, +macro,Py_T_LONG,3.12,, +macro,Py_T_LONGLONG,3.12,, +macro,Py_T_OBJECT_EX,3.12,, +macro,Py_T_PYSSIZET,3.12,, +macro,Py_T_SHORT,3.12,, +macro,Py_T_STRING,3.12,, +macro,Py_T_STRING_INPLACE,3.12,, +macro,Py_T_UBYTE,3.12,, +macro,Py_T_UINT,3.12,, +macro,Py_T_ULONG,3.12,, +macro,Py_T_ULONGLONG,3.12,, +macro,Py_T_USHORT,3.12,, type,Py_UCS4,3.2,, macro,Py_UNBLOCK_THREADS,3.2,, data,Py_UTF8Mode,3.8,, func,Py_VaBuildValue,3.2,, data,Py_Version,3.11,, func,Py_XNewRef,3.10,, +macro,Py_am_aiter,3.5,, +macro,Py_am_anext,3.5,, +macro,Py_am_await,3.5,, +macro,Py_am_send,3.10,, +macro,Py_bf_getbuffer,3.11,, +macro,Py_bf_releasebuffer,3.11,, type,Py_buffer,3.11,,full-abi type,Py_intptr_t,3.2,, +macro,Py_mod_create,3.5,, +macro,Py_mod_exec,3.5,, +macro,Py_mod_gil,3.13,, +macro,Py_mod_multiple_interpreters,3.12,, +macro,Py_mp_ass_subscript,3.2,, +macro,Py_mp_length,3.2,, +macro,Py_mp_subscript,3.2,, +macro,Py_nb_absolute,3.2,, +macro,Py_nb_add,3.2,, +macro,Py_nb_and,3.2,, +macro,Py_nb_bool,3.2,, +macro,Py_nb_divmod,3.2,, +macro,Py_nb_float,3.2,, +macro,Py_nb_floor_divide,3.2,, +macro,Py_nb_index,3.2,, +macro,Py_nb_inplace_add,3.2,, +macro,Py_nb_inplace_and,3.2,, +macro,Py_nb_inplace_floor_divide,3.2,, +macro,Py_nb_inplace_lshift,3.2,, +macro,Py_nb_inplace_matrix_multiply,3.5,, +macro,Py_nb_inplace_multiply,3.2,, +macro,Py_nb_inplace_or,3.2,, +macro,Py_nb_inplace_power,3.2,, +macro,Py_nb_inplace_remainder,3.2,, +macro,Py_nb_inplace_rshift,3.2,, +macro,Py_nb_inplace_subtract,3.2,, +macro,Py_nb_inplace_true_divide,3.2,, +macro,Py_nb_inplace_xor,3.2,, +macro,Py_nb_int,3.2,, +macro,Py_nb_invert,3.2,, +macro,Py_nb_lshift,3.2,, +macro,Py_nb_matrix_multiply,3.5,, +macro,Py_nb_multiply,3.2,, +macro,Py_nb_negative,3.2,, +macro,Py_nb_or,3.2,, +macro,Py_nb_positive,3.2,, +macro,Py_nb_power,3.2,, +macro,Py_nb_remainder,3.2,, +macro,Py_nb_rshift,3.2,, +macro,Py_nb_subtract,3.2,, +macro,Py_nb_true_divide,3.2,, +macro,Py_nb_xor,3.2,, +macro,Py_sq_ass_item,3.2,, +macro,Py_sq_concat,3.2,, +macro,Py_sq_contains,3.2,, +macro,Py_sq_inplace_concat,3.2,, +macro,Py_sq_inplace_repeat,3.2,, +macro,Py_sq_item,3.2,, +macro,Py_sq_length,3.2,, +macro,Py_sq_repeat,3.2,, type,Py_ssize_t,3.2,, +macro,Py_tp_alloc,3.2,, +macro,Py_tp_base,3.2,, +macro,Py_tp_bases,3.2,, +macro,Py_tp_call,3.2,, +macro,Py_tp_clear,3.2,, +macro,Py_tp_dealloc,3.2,, +macro,Py_tp_del,3.2,, +macro,Py_tp_descr_get,3.2,, +macro,Py_tp_descr_set,3.2,, +macro,Py_tp_doc,3.2,, +macro,Py_tp_finalize,3.5,, +macro,Py_tp_free,3.2,, +macro,Py_tp_getattr,3.2,, +macro,Py_tp_getattro,3.2,, +macro,Py_tp_getset,3.2,, +macro,Py_tp_hash,3.2,, +macro,Py_tp_init,3.2,, +macro,Py_tp_is_gc,3.2,, +macro,Py_tp_iter,3.2,, +macro,Py_tp_iternext,3.2,, +macro,Py_tp_members,3.2,, +macro,Py_tp_methods,3.2,, +macro,Py_tp_new,3.2,, +macro,Py_tp_repr,3.2,, +macro,Py_tp_richcompare,3.2,, +macro,Py_tp_setattr,3.2,, +macro,Py_tp_setattro,3.2,, +macro,Py_tp_str,3.2,, +macro,Py_tp_token,3.14,, +macro,Py_tp_traverse,3.2,, +macro,Py_tp_vectorcall,3.14,, type,Py_uintptr_t,3.2,, type,allocfunc,3.2,, type,binaryfunc,3.2,, diff --git a/Doc/data/threadsafety.dat b/Doc/data/threadsafety.dat new file mode 100644 index 000000000000000..7f9110620db4f37 --- /dev/null +++ b/Doc/data/threadsafety.dat @@ -0,0 +1,239 @@ +# Thread safety annotations for C API functions. +# +# Each line has the form: +# function_name : level +# +# Where level is one of: +# incompatible -- not safe even with external locking +# compatible -- safe if the caller serializes all access with external locks +# distinct -- safe on distinct objects without external synchronization +# shared -- safe for concurrent use on the same object +# atomic -- atomic +# +# Lines beginning with '#' are ignored. +# The function name must match the C domain identifier used in the documentation. + +# Synchronization primitives (Doc/c-api/synchronization.rst) +PyMutex_Lock:atomic: +PyMutex_Unlock:atomic: +PyMutex_IsLocked:atomic: + + +# Dictionary objects (Doc/c-api/dict.rst) + +# Type checks - read ob_type pointer, always safe +PyDict_Check:atomic: +PyDict_CheckExact:atomic: + +# Creation - pure allocation, no shared state +PyDict_New:atomic: + +# Lock-free lookups - use _Py_dict_lookup_threadsafe(), no locking. +# Atomic with simple types. +PyDict_Contains:shared: +PyDict_ContainsString:atomic: +PyDict_GetItemRef:shared: +PyDict_GetItemStringRef:atomic: +PyDict_Size:atomic: +PyDict_GET_SIZE:atomic: + +# Borrowed-reference lookups - lock-free dict access but returned +# borrowed reference is unsafe in free-threaded builds without +# external synchronization +PyDict_GetItem:compatible: +PyDict_GetItemWithError:compatible: +PyDict_GetItemString:compatible: +PyDict_SetDefault:compatible: + +# Iteration - no locking; returns borrowed refs +PyDict_Next:compatible: + +# Single-item mutations - protected by per-object critical section +PyDict_SetItem:shared: +PyDict_SetItemString:atomic: +PyDict_DelItem:shared: +PyDict_DelItemString:atomic: +PyDict_SetDefaultRef:shared: +PyDict_Pop:shared: +PyDict_PopString:atomic: + +# Bulk reads - hold per-object lock for duration +PyDict_Clear:atomic: +PyDict_Copy:atomic: +PyDict_Keys:atomic: +PyDict_Values:atomic: +PyDict_Items:atomic: + +# Merge/update - lock target dict; also lock source when it is a dict +PyDict_Update:shared: +PyDict_Merge:shared: +PyDict_MergeFromSeq2:shared: + +# Watcher registration - no synchronization on interpreter state +PyDict_AddWatcher:compatible: +PyDict_ClearWatcher:compatible: + +# Per-dict watcher tags - non-atomic RMW on _ma_watcher_tag; +# safe on distinct dicts only +PyDict_Watch:distinct: +PyDict_Unwatch:distinct: + + +# List objects (Doc/c-api/list.rst) + +# Type checks - read ob_type pointer, always safe +PyList_Check:atomic: +PyList_CheckExact:atomic: + +# Creation - pure allocation, no shared state +PyList_New:atomic: + +# Size - uses atomic load on free-threaded builds +PyList_Size:atomic: +PyList_GET_SIZE:atomic: + +# Strong-reference lookup - lock-free with atomic ops +PyList_GetItemRef:atomic: + +# Borrowed-reference lookups - no locking; returned borrowed +# reference is unsafe in free-threaded builds without +# external synchronization +PyList_GetItem:compatible: +PyList_GET_ITEM:compatible: + +# Single-item mutations - hold per-object lock for duration; +# appear atomic to lock-free readers +PyList_SetItem:atomic: +PyList_Append:atomic: + +# Insert - protected by per-object critical section; shifts +# elements so lock-free readers may observe intermediate states +PyList_Insert:shared: + +# Initialization macro - no synchronization; normally only used +# to fill in new lists where there is no previous content +PyList_SET_ITEM:compatible: + +# Bulk operations - hold per-object lock for duration +PyList_GetSlice:atomic: +PyList_AsTuple:atomic: +PyList_Clear:atomic: + +# Reverse - protected by per-object critical section; swaps +# elements so lock-free readers may observe intermediate states +PyList_Reverse:shared: + +# Slice assignment - lock target list; also lock source when it +# is a list +PyList_SetSlice:shared: + +# Sort - per-object lock held; the list is emptied before sorting +# so other threads may observe an empty list, but they won't see the +# intermediate states of the sort +PyList_Sort:shared: + +# Extend - lock target list; also lock source when it is a +# list, set, or dict +PyList_Extend:shared: + +# Creation - pure allocation, no shared state +PyBytes_FromString:atomic: +PyBytes_FromStringAndSize:atomic: +PyBytes_DecodeEscape:atomic: + +# Creation from formatting C primitives - pure allocation, no shared state +PyBytes_FromFormat:atomic: +PyBytes_FromFormatV:atomic: + +# Creation from object - uses buffer protocol so may call arbitrary code; +# safe as long as the buffer is not mutated by another thread during the operation +PyBytes_FromObject:shared: + +# Size - uses atomic load on free-threaded builds +PyBytes_Size:atomic: +PyBytes_GET_SIZE:atomic: + +# Raw data - no locking; mutating it is unsafe if the bytes object is shared between threads +PyBytes_AsString:compatible: +PyBytes_AS_STRING:compatible: +PyBytes_AsStringAndSize:compatible: + +# Concatenation - uses buffer protocol; safe as long as buffer is not mutated by another thread during the operation +PyBytes_Concat:shared: +PyBytes_ConcatAndDel:shared: +PyBytes_Join:shared: + +# Resizing - safe if the object is unique +_PyBytes_Resize:distinct: + +# Repr - atomic as bytes are immutable +PyBytes_Repr:atomic: + +# Creation from object - may call arbitrary code +PyByteArray_FromObject:shared: + +# Creation - pure allocation, no shared state +PyByteArray_FromStringAndSize:atomic: + +# Concatenation - uses buffer protocol; safe as long as buffer is not mutated by another thread during the operation +PyByteArray_Concat:shared: + +# Size - uses atomic load on free-threaded builds +PyByteArray_Size:atomic: +PyByteArray_GET_SIZE:atomic: + +# Raw data - no locking; mutating it is unsafe if the bytearray object is shared between threads +PyByteArray_AsString:compatible: +PyByteArray_AS_STRING:compatible: + +# Creation - may iterate the iterable argument, calling arbitrary code. +# Atomic for sets, frozensets, dicts, and frozendicts. +PySet_New:shared: +PyFrozenSet_New:shared: + +# Size - uses atomic load on free-threaded builds +PySet_Size:atomic: +PySet_GET_SIZE:atomic: + +# Contains - lock-free, atomic with simple types +PySet_Contains:shared: + +# Mutations - hold per-object lock for duration +# atomic with simple types +PySet_Add:shared: +PySet_Discard:shared: + +# Pop - hold per-object lock for duration +PySet_Pop:atomic: + +# Clear - empties the set before clearing +PySet_Clear:atomic: + +# Capsule objects (Doc/c-api/capsule.rst) + +# Type check - read ob_type pointer, always safe +PyCapsule_CheckExact:atomic: + +# Creation - pure allocation, no shared state +PyCapsule_New:atomic: + +# Validation - reads pointer and name fields; safe on distinct objects +PyCapsule_IsValid:distinct: + +# Getters - read struct fields; safe on distinct objects but +# concurrent access to the same capsule requires external synchronization +PyCapsule_GetPointer:distinct: +PyCapsule_GetName:distinct: +PyCapsule_GetDestructor:distinct: +PyCapsule_GetContext:distinct: + +# Setters - write struct fields; safe on distinct objects but +# concurrent access to the same capsule requires external synchronization +PyCapsule_SetPointer:distinct: +PyCapsule_SetName:distinct: +PyCapsule_SetDestructor:distinct: +PyCapsule_SetContext:distinct: + +# Import - looks up a capsule from a module attribute and +# calls PyCapsule_GetPointer; may call arbitrary code +PyCapsule_Import:compatible: diff --git a/Doc/deprecations/c-api-pending-removal-in-3.15.rst b/Doc/deprecations/c-api-pending-removal-in-3.15.rst index a5cc8f1d5b34753..bac80289cdfb3d7 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.15.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.15.rst @@ -1,7 +1,6 @@ Pending removal in Python 3.15 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* The bundled copy of ``libmpdecimal``. * The :c:func:`PyImport_ImportModuleNoBlock`: Use :c:func:`PyImport_ImportModule` instead. * :c:func:`PyWeakref_GetObject` and :c:func:`PyWeakref_GET_OBJECT`: diff --git a/Doc/deprecations/c-api-pending-removal-in-3.16.rst b/Doc/deprecations/c-api-pending-removal-in-3.16.rst new file mode 100644 index 000000000000000..9453f83799c43d7 --- /dev/null +++ b/Doc/deprecations/c-api-pending-removal-in-3.16.rst @@ -0,0 +1,4 @@ +Pending removal in Python 3.16 +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +* The bundled copy of ``libmpdec``. diff --git a/Doc/deprecations/c-api-pending-removal-in-3.18.rst b/Doc/deprecations/c-api-pending-removal-in-3.18.rst index 564cfb79a0b5134..022aee93aa70c47 100644 --- a/Doc/deprecations/c-api-pending-removal-in-3.18.rst +++ b/Doc/deprecations/c-api-pending-removal-in-3.18.rst @@ -1,11 +1,12 @@ Pending removal in Python 3.18 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -* Deprecated private functions (:gh:`128863`): +* The following private functions are deprecated + and planned for removal in Python 3.18: * :c:func:`!_PyBytes_Join`: use :c:func:`PyBytes_Join`. * :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`. - * :c:func:`!_PyDict_Pop()`: :c:func:`PyDict_Pop`. + * :c:func:`!_PyDict_Pop()`: use :c:func:`PyDict_Pop`. * :c:func:`!_PyLong_Sign()`: use :c:func:`PyLong_GetSign`. * :c:func:`!_PyLong_FromDigits` and :c:func:`!_PyLong_New`: use :c:func:`PyLongWriter_Create`. @@ -31,7 +32,7 @@ Pending removal in Python 3.18 :c:func:`PyUnicodeWriter_WriteSubstring(writer, str, start, end) `. * :c:func:`!_PyUnicodeWriter_WriteASCIIString`: replace ``_PyUnicodeWriter_WriteASCIIString(&writer, str)`` with - :c:func:`PyUnicodeWriter_WriteUTF8(writer, str) `. + :c:func:`PyUnicodeWriter_WriteASCII(writer, str) `. * :c:func:`!_PyUnicodeWriter_WriteLatin1String`: replace ``_PyUnicodeWriter_WriteLatin1String(&writer, str)`` with :c:func:`PyUnicodeWriter_WriteUTF8(writer, str) `. @@ -41,5 +42,6 @@ Pending removal in Python 3.18 * :c:func:`!_Py_fopen_obj`: use :c:func:`Py_fopen`. The `pythoncapi-compat project - `__ can be used to get these - new public functions on Python 3.13 and older. + `__ can be used to get + these new public functions on Python 3.13 and older. + (Contributed by Victor Stinner in :gh:`128863`.) diff --git a/Doc/deprecations/index.rst b/Doc/deprecations/index.rst index d064f2bec42c229..c891b0339916cb4 100644 --- a/Doc/deprecations/index.rst +++ b/Doc/deprecations/index.rst @@ -7,6 +7,8 @@ Deprecations .. include:: pending-removal-in-3.17.rst +.. include:: pending-removal-in-3.18.rst + .. include:: pending-removal-in-3.19.rst .. include:: pending-removal-in-future.rst diff --git a/Doc/deprecations/pending-removal-in-3.13.rst b/Doc/deprecations/pending-removal-in-3.13.rst index 2fd2f12cc6a2c42..d5b8c80e8f9aa0f 100644 --- a/Doc/deprecations/pending-removal-in-3.13.rst +++ b/Doc/deprecations/pending-removal-in-3.13.rst @@ -38,15 +38,3 @@ APIs: * :meth:`!unittest.TestProgram.usageExit` (:gh:`67048`) * :class:`!webbrowser.MacOSX` (:gh:`86421`) * :class:`classmethod` descriptor chaining (:gh:`89519`) -* :mod:`importlib.resources` deprecated methods: - - * ``contents()`` - * ``is_resource()`` - * ``open_binary()`` - * ``open_text()`` - * ``path()`` - * ``read_binary()`` - * ``read_text()`` - - Use :func:`importlib.resources.files` instead. Refer to `importlib-resources: Migrating from Legacy - `_ (:gh:`106531`) diff --git a/Doc/deprecations/pending-removal-in-3.14.rst b/Doc/deprecations/pending-removal-in-3.14.rst index 6159fa488482856..171758156ab1176 100644 --- a/Doc/deprecations/pending-removal-in-3.14.rst +++ b/Doc/deprecations/pending-removal-in-3.14.rst @@ -38,12 +38,6 @@ Pending removal in Python 3.14 is no current event loop set and it decides to create one. (Contributed by Serhiy Storchaka and Guido van Rossum in :gh:`100160`.) -* :mod:`collections.abc`: Deprecated :class:`!collections.abc.ByteString`. - Prefer :class:`!Sequence` or :class:`~collections.abc.Buffer`. - For use in typing, prefer a union, like ``bytes | bytearray``, - or :class:`collections.abc.Buffer`. - (Contributed by Shantanu Jain in :gh:`91896`.) - * :mod:`email`: Deprecated the *isdst* parameter in :func:`email.utils.localtime`. (Contributed by Alan Williams in :gh:`72346`.) @@ -78,7 +72,7 @@ Pending removal in Python 3.14 :meth:`~pathlib.PurePath.relative_to`: passing additional arguments is deprecated. -* :mod:`pkgutil`: :func:`!pkgutil.find_loader` and :func:!pkgutil.get_loader` +* :mod:`pkgutil`: :func:`!pkgutil.find_loader` and :func:`!pkgutil.get_loader` now raise :exc:`DeprecationWarning`; use :func:`importlib.util.find_spec` instead. (Contributed by Nikita Sobolev in :gh:`97850`.) @@ -96,9 +90,6 @@ Pending removal in Python 3.14 if :ref:`named placeholders ` are used and *parameters* is a sequence instead of a :class:`dict`. -* :mod:`typing`: :class:`!typing.ByteString`, deprecated since Python 3.9, - now causes a :exc:`DeprecationWarning` to be emitted when it is used. - * :mod:`urllib`: :class:`!urllib.parse.Quoter` is deprecated: it was not intended to be a public API. diff --git a/Doc/deprecations/pending-removal-in-3.15.rst b/Doc/deprecations/pending-removal-in-3.15.rst index 7b32275ad867608..814c6904d3e2ab9 100644 --- a/Doc/deprecations/pending-removal-in-3.15.rst +++ b/Doc/deprecations/pending-removal-in-3.15.rst @@ -33,16 +33,6 @@ Pending removal in Python 3.15 * ``load_module()`` method: use ``exec_module()`` instead. -* :class:`locale`: - - * The :func:`~locale.getdefaultlocale` function - has been deprecated since Python 3.11. - Its removal was originally planned for Python 3.13 (:gh:`90817`), - but has been postponed to Python 3.15. - Use :func:`~locale.getlocale`, :func:`~locale.setlocale`, - and :func:`~locale.getencoding` instead. - (Contributed by Hugo van Kemenade in :gh:`111187`.) - * :mod:`pathlib`: * :meth:`.PurePath.is_reserved` @@ -64,7 +54,7 @@ Pending removal in Python 3.15 * :func:`~threading.RLock` will take no arguments in Python 3.15. Passing any arguments has been deprecated since Python 3.14, - as the Python version does not permit any arguments, + as the Python version does not permit any arguments, but the C version allows any number of positional or keyword arguments, ignoring every argument. @@ -85,6 +75,13 @@ Pending removal in Python 3.15 has been deprecated since Python 3.13. Use the class-based syntax or the functional syntax instead. + * When using the functional syntax of :class:`~typing.TypedDict`\s, failing + to pass a value to the *fields* parameter (``TD = TypedDict("TD")``) or + passing ``None`` (``TD = TypedDict("TD", None)``) has been deprecated + since Python 3.13. + Use ``class TD(TypedDict): pass`` or ``TD = TypedDict("TD", {})`` + to create a TypedDict with zero field. + * The :func:`typing.no_type_check_decorator` decorator function has been deprecated since Python 3.13. After eight years in the :mod:`typing` module, diff --git a/Doc/deprecations/pending-removal-in-3.17.rst b/Doc/deprecations/pending-removal-in-3.17.rst index 370b98307e52280..0a1c2f08cab3bd7 100644 --- a/Doc/deprecations/pending-removal-in-3.17.rst +++ b/Doc/deprecations/pending-removal-in-3.17.rst @@ -1,6 +1,28 @@ Pending removal in Python 3.17 ------------------------------ +* :mod:`collections.abc`: + + - :class:`collections.abc.ByteString` is scheduled for removal in Python 3.17. + + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the :ref:`buffer protocol ` at runtime. For use + in type annotations, either use :class:`~collections.abc.Buffer` or a union + that explicitly specifies the types your code supports (e.g., + ``bytes | bytearray | memoryview``). + + :class:`!ByteString` was originally intended to be an abstract class that + would serve as a supertype of both :class:`bytes` and :class:`bytearray`. + However, since the ABC never had any methods, knowing that an object was an + instance of :class:`!ByteString` never actually told you anything useful + about the object. Other common buffer types such as :class:`memoryview` + were also never understood as subtypes of :class:`!ByteString` (either at + runtime or by static type checkers). + + See :pep:`PEP 688 <688#current-options>` for more details. + (Contributed by Shantanu Jain in :gh:`91896`.) + + * :mod:`typing`: - Before Python 3.14, old-style unions were implemented using the private class @@ -8,3 +30,22 @@ Pending removal in Python 3.17 but it has been retained for backward compatibility, with removal scheduled for Python 3.17. Users should use documented introspection helpers like :func:`typing.get_origin` and :func:`typing.get_args` instead of relying on private implementation details. + - :class:`typing.ByteString`, deprecated since Python 3.9, is scheduled for removal in + Python 3.17. + + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the :ref:`buffer protocol ` at runtime. For use + in type annotations, either use :class:`~collections.abc.Buffer` or a union + that explicitly specifies the types your code supports (e.g., + ``bytes | bytearray | memoryview``). + + :class:`!ByteString` was originally intended to be an abstract class that + would serve as a supertype of both :class:`bytes` and :class:`bytearray`. + However, since the ABC never had any methods, knowing that an object was an + instance of :class:`!ByteString` never actually told you anything useful + about the object. Other common buffer types such as :class:`memoryview` + were also never understood as subtypes of :class:`!ByteString` (either at + runtime or by static type checkers). + + See :pep:`PEP 688 <688#current-options>` for more details. + (Contributed by Shantanu Jain in :gh:`91896`.) diff --git a/Doc/deprecations/pending-removal-in-3.18.rst b/Doc/deprecations/pending-removal-in-3.18.rst new file mode 100644 index 000000000000000..3e7992194784240 --- /dev/null +++ b/Doc/deprecations/pending-removal-in-3.18.rst @@ -0,0 +1,9 @@ +Pending removal in Python 3.18 +------------------------------ + +* :mod:`decimal`: + + * The non-standard and undocumented :class:`~decimal.Decimal` format + specifier ``'N'``, which is only supported in the :mod:`!decimal` module's + C implementation, has been deprecated since Python 3.13. + (Contributed by Serhiy Storchaka in :gh:`89902`.) diff --git a/Doc/deprecations/pending-removal-in-future.rst b/Doc/deprecations/pending-removal-in-future.rst index 4c4a368baca9557..d9a2d50378e92b8 100644 --- a/Doc/deprecations/pending-removal-in-future.rst +++ b/Doc/deprecations/pending-removal-in-future.rst @@ -15,7 +15,6 @@ although there is currently no date scheduled for their removal. * :mod:`builtins`: - * ``bool(NotImplemented)``. * Generators: ``throw(type, exc, tb)`` and ``athrow(type, exc, tb)`` signature is deprecated: use ``throw(exc)`` and ``athrow(exc)`` instead, the single argument signature. @@ -36,7 +35,6 @@ although there is currently no date scheduled for their removal. * Support for ``__complex__()`` method returning a strict subclass of :class:`complex`: these methods will be required to return an instance of :class:`complex`. - * Delegation of ``int()`` to ``__trunc__()`` method. * Passing a complex number as the *real* or *imag* argument in the :func:`complex` constructor is now deprecated; it should only be passed as a single positional argument. diff --git a/Doc/extending/building.rst b/Doc/extending/building.rst index ddde567f6f3efa5..098dde39ea597e9 100644 --- a/Doc/extending/building.rst +++ b/Doc/extending/building.rst @@ -6,41 +6,10 @@ Building C and C++ Extensions ***************************** -A C extension for CPython is a shared library (e.g. a ``.so`` file on Linux, -``.pyd`` on Windows), which exports an *initialization function*. +A C extension for CPython is a shared library (for example, a ``.so`` file on +Linux, ``.pyd`` on Windows), which exports an *initialization function*. -To be importable, the shared library must be available on :envvar:`PYTHONPATH`, -and must be named after the module name, with an appropriate extension. -When using setuptools, the correct filename is generated automatically. - -The initialization function has the signature: - -.. c:function:: PyObject* PyInit_modulename(void) - -It returns either a fully initialized module, or a :c:type:`PyModuleDef` -instance. See :ref:`initializing-modules` for details. - -.. highlight:: python - -For modules with ASCII-only names, the function must be named -``PyInit_``, with ```` replaced by the name of the -module. When using :ref:`multi-phase-initialization`, non-ASCII module names -are allowed. In this case, the initialization function name is -``PyInitU_``, with ```` encoded using Python's -*punycode* encoding with hyphens replaced by underscores. In Python:: - - def initfunc_name(name): - try: - suffix = b'_' + name.encode('ascii') - except UnicodeEncodeError: - suffix = b'U_' + name.encode('punycode').replace(b'-', b'_') - return b'PyInit' + suffix - -It is possible to export multiple modules from a single shared library by -defining multiple initialization functions. However, importing them requires -using symbolic links or a custom importer, because by default only the -function corresponding to the filename is found. -See the *"Multiple modules in one library"* section in :pep:`489` for details. +See :ref:`extension-modules` for details. .. highlight:: c @@ -51,7 +20,11 @@ See the *"Multiple modules in one library"* section in :pep:`489` for details. Building C and C++ Extensions with setuptools ============================================= -Python 3.12 and newer no longer come with distutils. Please refer to the -``setuptools`` documentation at -https://setuptools.readthedocs.io/en/latest/setuptools.html -to learn more about how build and distribute C/C++ extensions with setuptools. + +Building, packaging and distributing extension modules is best done with +third-party tools, and is out of scope of this document. +One suitable tool is Setuptools, whose documentation can be found at +https://setuptools.pypa.io/en/latest/setuptools.html. + +The :mod:`distutils` module, which was included in the standard library +until Python 3.12, is now maintained as part of Setuptools. diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index b777862da79f148..cb41889437c8b0d 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -245,21 +245,23 @@ Python extension. For example:: return PyLong_FromLong(numargs); } - static PyMethodDef EmbMethods[] = { + static PyMethodDef emb_module_methods[] = { {"numargs", emb_numargs, METH_VARARGS, "Return the number of arguments received by the process."}, {NULL, NULL, 0, NULL} }; - static PyModuleDef EmbModule = { - PyModuleDef_HEAD_INIT, "emb", NULL, -1, EmbMethods, - NULL, NULL, NULL, NULL + static struct PyModuleDef emb_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "emb", + .m_size = 0, + .m_methods = emb_module_methods, }; static PyObject* PyInit_emb(void) { - return PyModule_Create(&EmbModule); + return PyModuleDef_Init(&emb_module); } Insert the above code just above the :c:func:`main` function. Also, insert the diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index b0493bed75b1513..23e0064c6c93609 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -75,12 +75,37 @@ the module and a copyright notice if you like). See :ref:`arg-parsing-string-and-buffers` for a description of this macro. All user-visible symbols defined by :file:`Python.h` have a prefix of ``Py`` or -``PY``, except those defined in standard header files. For convenience, and -since they are used extensively by the Python interpreter, ``"Python.h"`` -includes a few standard header files: ````, ````, -````, and ````. If the latter header file does not exist on -your system, it declares the functions :c:func:`malloc`, :c:func:`free` and -:c:func:`realloc` directly. +``PY``, except those defined in standard header files. + +.. tip:: + + For backward compatibility, :file:`Python.h` includes several standard header files. + C extensions should include the standard headers that they use, + and should not rely on these implicit includes. + If using the limited C API version 3.13 or newer, the implicit includes are: + + * ```` + * ```` (on Windows) + * ```` + * ```` + * ```` + * ```` + * ```` + * ```` (if present) + + If :c:macro:`Py_LIMITED_API` is not defined, or is set to version 3.12 or older, + the headers below are also included: + + * ```` + * ```` (on POSIX) + + If :c:macro:`Py_LIMITED_API` is not defined, or is set to version 3.10 or older, + the headers below are also included: + + * ```` + * ```` + * ```` + * ```` The next thing we add to our module file is the C function that will be called when the Python expression ``spam.system(string)`` is evaluated (we'll see @@ -203,31 +228,57 @@ function usually raises :c:data:`PyExc_TypeError`. If you have an argument whos value must be in a particular range or must satisfy other conditions, :c:data:`PyExc_ValueError` is appropriate. -You can also define a new exception that is unique to your module. For this, you -usually declare a static object variable at the beginning of your file:: +You can also define a new exception that is unique to your module. +The simplest way to do this is to declare a static global object variable at +the beginning of the file:: - static PyObject *SpamError; + static PyObject *SpamError = NULL; -and initialize it in your module's initialization function (:c:func:`!PyInit_spam`) -with an exception object:: +and initialize it by calling :c:func:`PyErr_NewException` in the module's +:c:data:`Py_mod_exec` function (:c:func:`!spam_module_exec`):: - PyMODINIT_FUNC - PyInit_spam(void) - { - PyObject *m; + SpamError = PyErr_NewException("spam.error", NULL, NULL); - m = PyModule_Create(&spammodule); - if (m == NULL) - return NULL; +Since :c:data:`!SpamError` is a global variable, it will be overwritten every time +the module is reinitialized, when the :c:data:`Py_mod_exec` function is called. + +For now, let's avoid the issue: we will block repeated initialization by raising an +:py:exc:`ImportError`:: + static PyObject *SpamError = NULL; + + static int + spam_module_exec(PyObject *m) + { + if (SpamError != NULL) { + PyErr_SetString(PyExc_ImportError, + "cannot initialize spam module more than once"); + return -1; + } SpamError = PyErr_NewException("spam.error", NULL, NULL); - if (PyModule_AddObjectRef(m, "error", SpamError) < 0) { - Py_CLEAR(SpamError); - Py_DECREF(m); - return NULL; + if (PyModule_AddObjectRef(m, "SpamError", SpamError) < 0) { + return -1; } - return m; + return 0; + } + + static PyModuleDef_Slot spam_module_slots[] = { + {Py_mod_exec, spam_module_exec}, + {0, NULL} + }; + + static struct PyModuleDef spam_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "spam", + .m_size = 0, // non-negative + .m_slots = spam_module_slots, + }; + + PyMODINIT_FUNC + PyInit_spam(void) + { + return PyModuleDef_Init(&spam_module); } Note that the Python name for the exception object is :exc:`!spam.error`. The @@ -242,6 +293,11 @@ needed to ensure that it will not be discarded, causing :c:data:`!SpamError` to become a dangling pointer. Should it become a dangling pointer, C code which raises the exception could cause a core dump or other unintended side effects. +For now, the :c:func:`Py_DECREF` call to remove this reference is missing. +Even when the Python interpreter shuts down, the global :c:data:`!SpamError` +variable will not be garbage-collected. It will "leak". +We did, however, ensure that this will happen at most once per process. + We discuss the use of :c:macro:`PyMODINIT_FUNC` as a function return type later in this sample. @@ -318,7 +374,7 @@ The Module's Method Table and Initialization Function I promised to show how :c:func:`!spam_system` is called from Python programs. First, we need to list its name and address in a "method table":: - static PyMethodDef SpamMethods[] = { + static PyMethodDef spam_methods[] = { ... {"system", spam_system, METH_VARARGS, "Execute a shell command."}, @@ -343,13 +399,10 @@ function. The method table must be referenced in the module definition structure:: - static struct PyModuleDef spammodule = { - PyModuleDef_HEAD_INIT, - "spam", /* name of module */ - spam_doc, /* module documentation, may be NULL */ - -1, /* size of per-interpreter state of the module, - or -1 if the module keeps state in global variables. */ - SpamMethods + static struct PyModuleDef spam_module = { + ... + .m_methods = spam_methods, + ... }; This structure, in turn, must be passed to the interpreter in the module's @@ -360,23 +413,17 @@ only non-\ ``static`` item defined in the module file:: PyMODINIT_FUNC PyInit_spam(void) { - return PyModule_Create(&spammodule); + return PyModuleDef_Init(&spam_module); } Note that :c:macro:`PyMODINIT_FUNC` declares the function as ``PyObject *`` return type, declares any special linkage declarations required by the platform, and for C++ declares the function as ``extern "C"``. -When the Python program imports module :mod:`!spam` for the first time, -:c:func:`!PyInit_spam` is called. (See below for comments about embedding Python.) -It calls :c:func:`PyModule_Create`, which returns a module object, and -inserts built-in function objects into the newly created module based upon the -table (an array of :c:type:`PyMethodDef` structures) found in the module definition. -:c:func:`PyModule_Create` returns a pointer to the module object -that it creates. It may abort with a fatal error for -certain errors, or return ``NULL`` if the module could not be initialized -satisfactorily. The init function must return the module object to its caller, -so that it then gets inserted into ``sys.modules``. +:c:func:`!PyInit_spam` is called when each interpreter imports its module +:mod:`!spam` for the first time. (See below for comments about embedding Python.) +A pointer to the module definition must be returned via :c:func:`PyModuleDef_Init`, +so that the import machinery can create the module and store it in ``sys.modules``. When embedding Python, the :c:func:`!PyInit_spam` function is not called automatically unless there's an entry in the :c:data:`PyImport_Inittab` table. @@ -433,23 +480,19 @@ optionally followed by an import of the module:: .. note:: - Removing entries from ``sys.modules`` or importing compiled modules into - multiple interpreters within a process (or following a :c:func:`fork` without an - intervening :c:func:`exec`) can create problems for some extension modules. - Extension module authors should exercise caution when initializing internal data - structures. + If you declare a global variable or a local static one, the module may + experience unintended side-effects on re-initialisation, for example when + removing entries from ``sys.modules`` or importing compiled modules into + multiple interpreters within a process + (or following a :c:func:`fork` without an intervening :c:func:`exec`). + If module state is not yet fully :ref:`isolated `, + authors should consider marking the module as having no support for subinterpreters + (via :c:macro:`Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED`). A more substantial example module is included in the Python source distribution -as :file:`Modules/xxmodule.c`. This file may be used as a template or simply +as :file:`Modules/xxlimited.c`. This file may be used as a template or simply read as an example. -.. note:: - - Unlike our ``spam`` example, ``xxmodule`` uses *multi-phase initialization* - (new in Python 3.5), where a PyModuleDef structure is returned from - ``PyInit_spam``, and creation of the module is left to the import machinery. - For details on multi-phase initialization, see :PEP:`489`. - .. _compilation: @@ -502,10 +545,8 @@ calling the Python callback functions from a C callback. Other uses are also imaginable. Fortunately, the Python interpreter is easily called recursively, and there is a -standard interface to call a Python function. (I won't dwell on how to call the -Python parser with a particular string as input --- if you're interested, have a -look at the implementation of the :option:`-c` command line option in -:file:`Modules/main.c` from the Python source code.) +standard interface to call a Python function. (If you're interested in how to call the +Python parser with a particular string as input, see :ref:`veryhigh`.) Calling a Python function is easy. First, the Python program must somehow pass you the Python function object. You should provide a function (or some other @@ -790,18 +831,17 @@ Philbrick (philbrick@hks.com):: {NULL, NULL, 0, NULL} /* sentinel */ }; - static struct PyModuleDef keywdargmodule = { - PyModuleDef_HEAD_INIT, - "keywdarg", - NULL, - -1, - keywdarg_methods + static struct PyModuleDef keywdarg_module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "keywdarg", + .m_size = 0, + .m_methods = keywdarg_methods, }; PyMODINIT_FUNC PyInit_keywdarg(void) { - return PyModule_Create(&keywdargmodule); + return PyModuleDef_Init(&keywdarg_module); } @@ -896,7 +936,7 @@ and the object is freed. An alternative strategy is called :dfn:`automatic garbage collection`. (Sometimes, reference counting is also referred to as a garbage collection -strategy, hence my use of "automatic" to distinguish the two.) The big +strategy, hence the use of "automatic" to distinguish the two.) The big advantage of automatic garbage collection is that the user doesn't need to call :c:func:`free` explicitly. (Another claimed advantage is an improvement in speed or memory usage --- this is no hard fact however.) The disadvantage is that for @@ -1042,7 +1082,14 @@ references to all its items, so when item 1 is replaced, it has to dispose of the original item 1. Now let's suppose the original item 1 was an instance of a user-defined class, and let's further suppose that the class defined a :meth:`!__del__` method. If this class instance has a reference count of 1, -disposing of it will call its :meth:`!__del__` method. +disposing of it will call its :meth:`!__del__` method. Internally, +:c:func:`PyList_SetItem` calls :c:func:`Py_DECREF` on the replaced item, +which invokes replaced item's corresponding +:c:member:`~PyTypeObject.tp_dealloc` function. During +deallocation, :c:member:`~PyTypeObject.tp_dealloc` calls +:c:member:`~PyTypeObject.tp_finalize`, which is mapped to the +:meth:`!__del__` method for class instances (see :pep:`442`). This entire +sequence happens synchronously within the :c:func:`PyList_SetItem` call. Since it is written in Python, the :meth:`!__del__` method can execute arbitrary Python code. Could it perhaps do something to invalidate the reference to @@ -1072,8 +1119,9 @@ why his :meth:`!__del__` methods would fail... The second case of problems with a borrowed reference is a variant involving threads. Normally, multiple threads in the Python interpreter can't get in each -other's way, because there is a global lock protecting Python's entire object -space. However, it is possible to temporarily release this lock using the macro +other's way, because there is a :term:`global lock ` +protecting Python's entire object space. +However, it is possible to temporarily release this lock using the macro :c:macro:`Py_BEGIN_ALLOW_THREADS`, and to re-acquire it using :c:macro:`Py_END_ALLOW_THREADS`. This is common around blocking I/O calls, to let other threads use the processor while waiting for the I/O to complete. @@ -1259,20 +1307,15 @@ two more lines must be added:: #include "spammodule.h" The ``#define`` is used to tell the header file that it is being included in the -exporting module, not a client module. Finally, the module's initialization -function must take care of initializing the C API pointer array:: +exporting module, not a client module. Finally, the module's :c:data:`mod_exec +` function must take care of initializing the C API pointer array:: - PyMODINIT_FUNC - PyInit_spam(void) + static int + spam_module_exec(PyObject *m) { - PyObject *m; static void *PySpam_API[PySpam_API_pointers]; PyObject *c_api_object; - m = PyModule_Create(&spammodule); - if (m == NULL) - return NULL; - /* Initialize the C API pointer array */ PySpam_API[PySpam_System_NUM] = (void *)PySpam_System; @@ -1280,11 +1323,10 @@ function must take care of initializing the C API pointer array:: c_api_object = PyCapsule_New((void *)PySpam_API, "spam._C_API", NULL); if (PyModule_Add(m, "_C_API", c_api_object) < 0) { - Py_DECREF(m); - return NULL; + return -1; } - return m; + return 0; } Note that ``PySpam_API`` is declared ``static``; otherwise the pointer @@ -1343,20 +1385,16 @@ like this:: All that a client module must do in order to have access to the function :c:func:`!PySpam_System` is to call the function (or rather macro) -:c:func:`!import_spam` in its initialization function:: +:c:func:`!import_spam` in its :c:data:`mod_exec ` function:: - PyMODINIT_FUNC - PyInit_client(void) + static int + client_module_exec(PyObject *m) { - PyObject *m; - - m = PyModule_Create(&clientmodule); - if (m == NULL) - return NULL; - if (import_spam() < 0) - return NULL; + if (import_spam() < 0) { + return -1; + } /* additional initialization can happen here */ - return m; + return 0; } The main disadvantage of this approach is that the file :file:`spammodule.h` is diff --git a/Doc/extending/index.rst b/Doc/extending/index.rst index 01b4df6d44acff5..4cc2c96d8d5b47e 100644 --- a/Doc/extending/index.rst +++ b/Doc/extending/index.rst @@ -26,19 +26,9 @@ Recommended third party tools ============================= This guide only covers the basic tools for creating extensions provided -as part of this version of CPython. Third party tools like -`Cython `_, `cffi `_, -`SWIG `_ and `Numba `_ -offer both simpler and more sophisticated approaches to creating C and C++ -extensions for Python. - -.. seealso:: - - `Python Packaging User Guide: Binary Extensions `_ - The Python Packaging User Guide not only covers several available - tools that simplify the creation of binary extensions, but also - discusses the various reasons why creating an extension module may be - desirable in the first place. +as part of this version of CPython. Some :ref:`third party tools +` offer both simpler and more sophisticated approaches to creating +C and C++ extensions for Python. Creating extensions without third party tools @@ -49,6 +39,10 @@ assistance from third party tools. It is intended primarily for creators of those tools, rather than being a recommended way to create your own C extensions. +.. seealso:: + + :pep:`489` -- Multi-phase extension module initialization + .. toctree:: :maxdepth: 2 :numbered: diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst index 3fc91841416d71d..3bbee33bd50698f 100644 --- a/Doc/extending/newtypes_tutorial.rst +++ b/Doc/extending/newtypes_tutorial.rst @@ -55,8 +55,10 @@ from the previous chapter. This file defines three things: #. How the :class:`!Custom` **type** behaves: this is the ``CustomType`` struct, which defines a set of flags and function pointers that the interpreter inspects when specific operations are requested. -#. How to initialize the :mod:`!custom` module: this is the ``PyInit_custom`` - function and the associated ``custommodule`` struct. +#. How to define and execute the :mod:`!custom` module: this is the + ``PyInit_custom`` function and the associated ``custom_module`` struct for + defining the module, and the ``custom_module_exec`` function to set up + a fresh module object. The first bit is:: @@ -171,18 +173,18 @@ implementation provided by the API function :c:func:`PyType_GenericNew`. :: .tp_new = PyType_GenericNew, Everything else in the file should be familiar, except for some code in -:c:func:`!PyInit_custom`:: +:c:func:`!custom_module_exec`:: - if (PyType_Ready(&CustomType) < 0) - return; + if (PyType_Ready(&CustomType) < 0) { + return -1; + } This initializes the :class:`!Custom` type, filling in a number of members to the appropriate default values, including :c:member:`~PyObject.ob_type` that we initially set to ``NULL``. :: if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(m); - return NULL; + return -1; } This adds the type to the module dictionary. This allows us to create @@ -275,7 +277,7 @@ be an instance of a subclass. The explicit cast to ``CustomObject *`` above is needed because we defined ``Custom_dealloc`` to take a ``PyObject *`` argument, as the ``tp_dealloc`` function pointer expects to receive a ``PyObject *`` argument. - By assigning to the the ``tp_dealloc`` slot of a type, we declare + By assigning to the ``tp_dealloc`` slot of a type, we declare that it can only be called with instances of our ``CustomObject`` class, so the cast to ``(CustomObject *)`` is safe. This is object-oriented polymorphism, in C! @@ -875,27 +877,22 @@ but let the base class handle it by calling its own :c:member:`~PyTypeObject.tp_ The :c:type:`PyTypeObject` struct supports a :c:member:`~PyTypeObject.tp_base` specifying the type's concrete base class. Due to cross-platform compiler issues, you can't fill that field directly with a reference to -:c:type:`PyList_Type`; it should be done later in the module initialization +:c:type:`PyList_Type`; it should be done in the :c:data:`Py_mod_exec` function:: - PyMODINIT_FUNC - PyInit_sublist(void) + static int + sublist_module_exec(PyObject *m) { - PyObject* m; SubListType.tp_base = &PyList_Type; - if (PyType_Ready(&SubListType) < 0) - return NULL; - - m = PyModule_Create(&sublistmodule); - if (m == NULL) - return NULL; + if (PyType_Ready(&SubListType) < 0) { + return -1; + } if (PyModule_AddObjectRef(m, "SubList", (PyObject *) &SubListType) < 0) { - Py_DECREF(m); - return NULL; + return -1; } - return m; + return 0; } Before calling :c:func:`PyType_Ready`, the type structure must have the diff --git a/Doc/extending/windows.rst b/Doc/extending/windows.rst index 56aa44e4e58c83a..a97c6182553c302 100644 --- a/Doc/extending/windows.rst +++ b/Doc/extending/windows.rst @@ -121,7 +121,7 @@ When creating DLLs in Windows, you can use the CPython library in two ways: :file:`Python.h` triggers an implicit, configure-aware link with the library. The header file chooses :file:`pythonXY_d.lib` for Debug, :file:`pythonXY.lib` for Release, and :file:`pythonX.lib` for Release with - the `Limited API `_ enabled. + the :ref:`Limited API ` enabled. To build two DLLs, spam and ni (which uses C functions found in spam), you could use these commands:: diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index e2710fab9cf8002..3872a9ab9364f99 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -263,6 +263,8 @@ In general, structured switch statements execute one block of code when an expression has a particular value or set of values. Since Python 3.10 one can easily match literal values, or constants within a namespace, with a ``match ... case`` statement. +See :ref:`the specification ` and :ref:`the tutorial ` +for more information about :keyword:`match` statements. An older alternative is a sequence of ``if... elif... elif... else``. For cases where you need to choose from a very large number of possibilities, @@ -589,9 +591,9 @@ exhaustive test suites that exercise every line of code in a module. An appropriate testing discipline can help build large complex applications in Python as well as having interface specifications would. In fact, it can be better because an interface specification cannot test certain properties of a -program. For example, the :meth:`!list.append` method is expected to add new elements +program. For example, the :meth:`list.append` method is expected to add new elements to the end of some internal list; an interface specification cannot test that -your :meth:`!list.append` implementation will actually do this correctly, but it's +your :meth:`list.append` implementation will actually do this correctly, but it's trivial to check this property in a test suite. Writing test suites is very helpful, and you might want to design your code to diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 3147fda7c371249..1d5abed2317b0ce 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -37,24 +37,9 @@ Writing C is hard; are there any alternatives? ---------------------------------------------- There are a number of alternatives to writing your own C extensions, depending -on what you're trying to do. - -.. XXX make sure these all work - -`Cython `_ and its relative `Pyrex -`_ are compilers -that accept a slightly modified form of Python and generate the corresponding -C code. Cython and Pyrex make it possible to write an extension without having -to learn Python's C API. - -If you need to interface to some C or C++ library for which no Python extension -currently exists, you can try wrapping the library's data types and functions -with a tool such as `SWIG `_. `SIP -`__, `CXX -`_ `Boost -`_, or `Weave -`_ are also -alternatives for wrapping C++ libraries. +on what you're trying to do. :ref:`Recommended third party tools ` +offer both simpler and more sophisticated approaches to creating C and C++ +extensions for Python. How can I execute arbitrary Python statements from C? diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 578777d7f236211..698f2117ff5c649 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -186,7 +186,7 @@ How do I get documentation on Python? ------------------------------------- The standard documentation for the current stable version of Python is available -at https://docs.python.org/3/. PDF, plain text, and downloadable HTML versions are +at https://docs.python.org/3/. EPUB, plain text, and downloadable HTML versions are also available at https://docs.python.org/3/download.html. The documentation is written in reStructuredText and processed by `the Sphinx diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 9f9e4fab685b19f..ff34bb5d71c22ba 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -8,11 +8,11 @@ Programming FAQ .. contents:: -General Questions +General questions ================= -Is there a source code level debugger with breakpoints, single-stepping, etc.? ------------------------------------------------------------------------------- +Is there a source code-level debugger with breakpoints and single-stepping? +--------------------------------------------------------------------------- Yes. @@ -25,8 +25,7 @@ Reference Manual `. You can also write your own debugger by using the code for pdb as an example. The IDLE interactive development environment, which is part of the standard -Python distribution (normally available as -`Tools/scripts/idle3 `_), +Python distribution (normally available as :mod:`idlelib`), includes a graphical debugger. PythonWin is a Python IDE that includes a GUI debugger based on pdb. The @@ -48,7 +47,6 @@ There are a number of commercial Python IDEs that include graphical debuggers. They include: * `Wing IDE `_ -* `Komodo IDE `_ * `PyCharm `_ @@ -57,13 +55,15 @@ Are there tools to help find bugs or perform static analysis? Yes. -`Pylint `_ and -`Pyflakes `_ do basic checking that will +`Ruff `__, +`Pylint `__ and +`Pyflakes `__ do basic checking that will help you catch bugs sooner. -Static type checkers such as `Mypy `_, -`Pyre `_, and -`Pytype `_ can check type hints in Python +Static type checkers such as `mypy `__, +`ty `__, +`Pyrefly `__, and +`pytype `__ can check type hints in Python source code. @@ -79,7 +79,7 @@ set of modules required by a program and bind these modules together with a Python binary to produce a single executable. One is to use the freeze tool, which is included in the Python source tree as -`Tools/freeze `_. +:source:`Tools/freeze`. It converts Python byte code to C arrays; with a C compiler you can embed all your modules into a new program, which is then linked with the standard Python modules. @@ -103,6 +103,7 @@ executables: * `py2app `_ (macOS only) * `py2exe `_ (Windows only) + Are there coding standards or a style guide for Python programs? ---------------------------------------------------------------- @@ -110,7 +111,7 @@ Yes. The coding style required for standard library modules is documented as :pep:`8`. -Core Language +Core language ============= .. _faq-unboundlocalerror: @@ -143,7 +144,7 @@ results in an :exc:`!UnboundLocalError`: >>> foo() Traceback (most recent call last): ... - UnboundLocalError: local variable 'x' referenced before assignment + UnboundLocalError: cannot access local variable 'x' where it is not associated with a value This is because when you make an assignment to a variable in a scope, that variable becomes local to that scope and shadows any similarly named variable @@ -208,7 +209,7 @@ Why do lambdas defined in a loop with different values all return the same resul ---------------------------------------------------------------------------------- Assume you use a for loop to define a few different lambdas (or even plain -functions), e.g.:: +functions), for example:: >>> squares = [] >>> for x in range(5): @@ -227,7 +228,7 @@ they all return ``16``:: This happens because ``x`` is not local to the lambdas, but is defined in the outer scope, and it is accessed when the lambda is called --- not when it is defined. At the end of the loop, the value of ``x`` is ``4``, so all the -functions now return ``4**2``, i.e. ``16``. You can also verify this by +functions now return ``4**2``, that is ``16``. You can also verify this by changing the value of ``x`` and see how the results of the lambdas change:: >>> x = 8 @@ -298,9 +299,9 @@ using multiple imports per line uses less screen space. It's good practice if you import modules in the following order: -1. standard library modules -- e.g. :mod:`sys`, :mod:`os`, :mod:`argparse`, :mod:`re` +1. standard library modules -- such as :mod:`sys`, :mod:`os`, :mod:`argparse`, :mod:`re` 2. third-party library modules (anything installed in Python's site-packages - directory) -- e.g. :mod:`!dateutil`, :mod:`!requests`, :mod:`!PIL.Image` + directory) -- such as :pypi:`dateutil`, :pypi:`requests`, :pypi:`tzdata` 3. locally developed modules It is sometimes necessary to move imports to a function or class to avoid @@ -454,7 +455,7 @@ There are two factors that produce this result: (the list), and both ``x`` and ``y`` refer to it. 2) Lists are :term:`mutable`, which means that you can change their content. -After the call to :meth:`!append`, the content of the mutable object has +After the call to :meth:`~sequence.append`, the content of the mutable object has changed from ``[]`` to ``[10]``. Since both the variables refer to the same object, using either name accesses the modified value ``[10]``. @@ -494,11 +495,11 @@ new objects). In other words: -* If we have a mutable object (:class:`list`, :class:`dict`, :class:`set`, - etc.), we can use some specific operations to mutate it and all the variables +* If we have a mutable object (such as :class:`list`, :class:`dict`, :class:`set`), + we can use some specific operations to mutate it and all the variables that refer to it will see the change. -* If we have an immutable object (:class:`str`, :class:`int`, :class:`tuple`, - etc.), all the variables that refer to it will always see the same value, +* If we have an immutable object (such as :class:`str`, :class:`int`, :class:`tuple`), + all the variables that refer to it will always see the same value, but operations that transform that value into a new value always return a new object. @@ -511,7 +512,7 @@ How do I write a function with output parameters (call by reference)? Remember that arguments are passed by assignment in Python. Since assignment just creates references to objects, there's no alias between an argument name in -the caller and callee, and so no call-by-reference per se. You can achieve the +the caller and callee, and consequently no call-by-reference. You can achieve the desired effect in a number of ways. 1) By returning a tuple of the results:: @@ -714,8 +715,8 @@ not:: "a" in ("b", "a") -The same is true of the various assignment operators (``=``, ``+=`` etc). They -are not truly operators but syntactic delimiters in assignment statements. +The same is true of the various assignment operators (``=``, ``+=``, and so on). +They are not truly operators but syntactic delimiters in assignment statements. Is there an equivalent of C's "?:" ternary operator? @@ -868,9 +869,9 @@ with either a space or parentheses. How do I convert a string to a number? -------------------------------------- -For integers, use the built-in :func:`int` type constructor, e.g. ``int('144') +For integers, use the built-in :func:`int` type constructor, for example, ``int('144') == 144``. Similarly, :func:`float` converts to a floating-point number, -e.g. ``float('144') == 144.0``. +for example, ``float('144') == 144.0``. By default, these interpret the number as decimal, so that ``int('0144') == 144`` holds true, and ``int('0x144')`` raises :exc:`ValueError`. ``int(string, @@ -887,18 +888,18 @@ unwanted side effects. For example, someone could pass directory. :func:`eval` also has the effect of interpreting numbers as Python expressions, -so that e.g. ``eval('09')`` gives a syntax error because Python does not allow +so that, for example, ``eval('09')`` gives a syntax error because Python does not allow leading '0' in a decimal number (except '0'). How do I convert a number to a string? -------------------------------------- -To convert, e.g., the number ``144`` to the string ``'144'``, use the built-in type +For example, to convert the number ``144`` to the string ``'144'``, use the built-in type constructor :func:`str`. If you want a hexadecimal or octal representation, use the built-in functions :func:`hex` or :func:`oct`. For fancy formatting, see -the :ref:`f-strings` and :ref:`formatstrings` sections, -e.g. ``"{:04d}".format(144)`` yields +the :ref:`f-strings` and :ref:`formatstrings` sections. +For example, ``"{:04d}".format(144)`` yields ``'0144'`` and ``"{:.3f}".format(1.0/3.0)`` yields ``'0.333'``. @@ -908,7 +909,7 @@ How do I modify a string in place? You can't, because strings are immutable. In most situations, you should simply construct a new string from the various parts you want to assemble it from. However, if you need an object with the ability to modify in-place -unicode data, try using an :class:`io.StringIO` object or the :mod:`array` +Unicode data, try using an :class:`io.StringIO` object or the :mod:`array` module:: >>> import io @@ -1066,13 +1067,14 @@ the raw string:: Also see the specification in the :ref:`language reference `. + Performance =========== My program is too slow. How do I speed it up? --------------------------------------------- -That's a tough one, in general. First, here are a list of things to +That's a tough one, in general. First, here is a list of things to remember before diving further: * Performance characteristics vary across Python implementations. This FAQ @@ -1125,6 +1127,7 @@ yourself. The wiki page devoted to `performance tips `_. + .. _efficient_string_concatenation: What is the most efficient way to concatenate many strings together? @@ -1143,7 +1146,7 @@ them into a list and call :meth:`str.join` at the end:: chunks.append(s) result = ''.join(chunks) -(another reasonably efficient idiom is to use :class:`io.StringIO`) +(Another reasonably efficient idiom is to use :class:`io.StringIO`.) To accumulate many :class:`bytes` objects, the recommended idiom is to extend a :class:`bytearray` object using in-place concatenation (the ``+=`` operator):: @@ -1153,7 +1156,7 @@ a :class:`bytearray` object using in-place concatenation (the ``+=`` operator):: result += b -Sequences (Tuples/Lists) +Sequences (tuples/lists) ======================== How do I convert between tuples and lists? @@ -1217,8 +1220,8 @@ list, deleting duplicates as you go:: else: last = mylist[i] -If all elements of the list may be used as set keys (i.e. they are all -:term:`hashable`) this is often faster :: +If all elements of the list may be used as set keys (that is, they are all +:term:`hashable`) this is often faster:: mylist = list(set(mylist)) @@ -1226,13 +1229,13 @@ This converts the list into a set, thereby removing duplicates, and then back into a list. -How do you remove multiple items from a list --------------------------------------------- +How do you remove multiple items from a list? +--------------------------------------------- As with removing duplicates, explicitly iterating in reverse with a delete condition is one possibility. However, it is easier and faster to use slice replacement with an implicit or explicit forward iteration. -Here are three variations.:: +Here are three variations:: mylist[:] = filter(keep_function, mylist) mylist[:] = (x for x in mylist if keep_condition) @@ -1254,7 +1257,7 @@ difference is that a Python list can contain objects of many different types. The ``array`` module also provides methods for creating arrays of fixed types with compact representations, but they are slower to index than lists. Also note that `NumPy `_ -and other third party packages define array-like structures with +and other third-party packages define array-like structures with various characteristics as well. To get Lisp-style linked lists, you can emulate *cons cells* using tuples:: @@ -1324,7 +1327,7 @@ Or, you can use an extension that provides a matrix datatype; `NumPy How do I apply a method or function to a sequence of objects? ------------------------------------------------------------- -To call a method or function and accumulate the return values is a list, +To call a method or function and accumulate the return values in a list, a :term:`list comprehension` is an elegant solution:: result = [obj.method() for obj in mylist] @@ -1340,6 +1343,7 @@ a plain :keyword:`for` loop will suffice:: for obj in mylist: function(obj) + .. _faq-augmented-assignment-tuple-error: Why does a_tuple[i] += ['item'] raise an exception when the addition works? @@ -1397,9 +1401,9 @@ To see why this happens, you need to know that (a) if an object implements an :meth:`~object.__iadd__` magic method, it gets called when the ``+=`` augmented assignment is executed, and its return value is what gets used in the assignment statement; -and (b) for lists, :meth:`!__iadd__` is equivalent to calling :meth:`!extend` on the list -and returning the list. That's why we say that for lists, ``+=`` is a -"shorthand" for :meth:`!list.extend`:: +and (b) for lists, :meth:`!__iadd__` is equivalent to calling +:meth:`~sequence.extend` on the list and returning the list. +That's why we say that for lists, ``+=`` is a "shorthand" for :meth:`list.extend`:: >>> a_list = [] >>> a_list += [1] @@ -1444,7 +1448,7 @@ How can I sort one list by values from another list? ---------------------------------------------------- Merge them into an iterator of tuples, sort the resulting list, and then pick -out the element you want. :: +out the element you want. >>> list1 = ["what", "I'm", "sorting", "by"] >>> list2 = ["something", "else", "to", "sort"] @@ -1504,14 +1508,15 @@ How do I check if an object is an instance of a given class or of a subclass of Use the built-in function :func:`isinstance(obj, cls) `. You can check if an object is an instance of any of a number of classes by providing a tuple instead of a -single class, e.g. ``isinstance(obj, (class1, class2, ...))``, and can also -check whether an object is one of Python's built-in types, e.g. +single class, for example, ``isinstance(obj, (class1, class2, ...))``, and can also +check whether an object is one of Python's built-in types, for example, ``isinstance(obj, str)`` or ``isinstance(obj, (int, float, complex))``. Note that :func:`isinstance` also checks for virtual inheritance from an :term:`abstract base class`. So, the test will return ``True`` for a registered class even if hasn't directly or indirectly inherited from it. To -test for "true inheritance", scan the :term:`MRO` of the class: +test for "true inheritance", scan the :term:`method resolution order` (MRO) of +the class: .. testcode:: @@ -1574,7 +1579,7 @@ call it:: What is delegation? ------------------- -Delegation is an object oriented technique (also called a design pattern). +Delegation is an object-oriented technique (also called a design pattern). Let's say you have an object ``x`` and want to change the behaviour of just one of its methods. You can create a new class that provides a new implementation of the method you're interested in changing and delegates all other methods to @@ -1645,7 +1650,7 @@ How can I organize my code to make it easier to change the base class? You could assign the base class to an alias and derive from the alias. Then all you have to change is the value assigned to the alias. Incidentally, this trick -is also handy if you want to decide dynamically (e.g. depending on availability +is also handy if you want to decide dynamically (such as depending on availability of resources) which base class to use. Example:: class Base: @@ -1710,9 +1715,9 @@ How can I overload constructors (or methods) in Python? This answer actually applies to all methods, but the question usually comes up first in the context of constructors. -In C++ you'd write +In C++ you'd write: -.. code-block:: c +.. code-block:: c++ class C { C() { cout << "No arguments\n"; } @@ -1731,7 +1736,7 @@ default arguments. For example:: This is not entirely equivalent, but close enough in practice. -You could also try a variable-length argument list, e.g. :: +You could also try a variable-length argument list, for example:: def __init__(self, *args): ... @@ -1774,6 +1779,7 @@ to use private variable names at all. The :ref:`private name mangling specifications ` for details and special cases. + My class defines __del__ but it is not called when I delete the object. ----------------------------------------------------------------------- @@ -1783,7 +1789,7 @@ The :keyword:`del` statement does not necessarily call :meth:`~object.__del__` - decrements the object's reference count, and if this reaches zero :meth:`!__del__` is called. -If your data structures contain circular links (e.g. a tree where each child has +If your data structures contain circular links (for example, a tree where each child has a parent reference and each parent has a list of children) the reference counts will never go back to zero. Once in a while Python runs an algorithm to detect such cycles, but the garbage collector might run some time after the last @@ -1852,6 +1858,8 @@ to the object: 13891296 +.. _faq-identity-with-is: + When can I rely on identity tests with the *is* operator? --------------------------------------------------------- @@ -1883,9 +1891,9 @@ are preferred. In particular, identity tests should not be used to check constants such as :class:`int` and :class:`str` which aren't guaranteed to be singletons:: - >>> a = 1000 - >>> b = 500 - >>> c = b + 500 + >>> a = 10_000_000 + >>> b = 5_000_000 + >>> c = b + 5_000_000 >>> a is c False @@ -1954,9 +1962,9 @@ parent class: .. testcode:: - from datetime import date + import datetime as dt - class FirstOfMonthDate(date): + class FirstOfMonthDate(dt.date): "Always choose the first day of the month" def __new__(cls, year, month, day): return super().__new__(cls, year, month, 1) @@ -1999,7 +2007,7 @@ The two principal tools for caching methods are former stores results at the instance level and the latter at the class level. -The *cached_property* approach only works with methods that do not take +The ``cached_property`` approach only works with methods that do not take any arguments. It does not create a reference to the instance. The cached method result will be kept only as long as the instance is alive. @@ -2008,7 +2016,7 @@ method result will be released right away. The disadvantage is that if instances accumulate, so too will the accumulated method results. They can grow without bound. -The *lru_cache* approach works with methods that have :term:`hashable` +The ``lru_cache`` approach works with methods that have :term:`hashable` arguments. It creates a reference to the instance unless special efforts are made to pass in weak references. @@ -2042,11 +2050,11 @@ This example shows the various techniques:: # Depends on the station_id, date, and units. The above example assumes that the *station_id* never changes. If the -relevant instance attributes are mutable, the *cached_property* approach +relevant instance attributes are mutable, the ``cached_property`` approach can't be made to work because it cannot detect changes to the attributes. -To make the *lru_cache* approach work when the *station_id* is mutable, +To make the ``lru_cache`` approach work when the *station_id* is mutable, the class needs to define the :meth:`~object.__eq__` and :meth:`~object.__hash__` methods so that the cache can detect relevant attribute updates:: @@ -2092,10 +2100,10 @@ one user but run as another, such as if you are testing with a web server. Unless the :envvar:`PYTHONDONTWRITEBYTECODE` environment variable is set, creation of a .pyc file is automatic if you're importing a module and Python -has the ability (permissions, free space, etc...) to create a ``__pycache__`` +has the ability (permissions, free space, and so on) to create a ``__pycache__`` subdirectory and write the compiled module to that subdirectory. -Running Python on a top level script is not considered an import and no +Running Python on a top-level script is not considered an import and no ``.pyc`` will be created. For example, if you have a top-level module ``foo.py`` that imports another module ``xyz.py``, when you run ``foo`` (by typing ``python foo.py`` as a shell command), a ``.pyc`` will be created for @@ -2114,7 +2122,7 @@ the ``compile()`` function in that module interactively:: This will write the ``.pyc`` to a ``__pycache__`` subdirectory in the same location as ``foo.py`` (or you can override that with the optional parameter -``cfile``). +*cfile*). You can also automatically compile all files in a directory or directories using the :mod:`compileall` module. You can do it from the shell prompt by running @@ -2219,7 +2227,7 @@ changed module, do this:: importlib.reload(modname) Warning: this technique is not 100% fool-proof. In particular, modules -containing statements like :: +containing statements like:: from modname import some_objects diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 0b26e18efd7f1b6..515fa77dd038df2 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -21,7 +21,9 @@ Glossary right delimiters (parentheses, square brackets, curly braces or triple quotes), or after specifying a decorator. - * The :const:`Ellipsis` built-in constant. + .. index:: single: ...; ellipsis literal + + * The three dots form of the :ref:`Ellipsis ` object. abstract base class Abstract base classes complement :term:`duck-typing` by @@ -37,10 +39,11 @@ Glossary ABCs with the :mod:`abc` module. annotate function - A function that can be called to retrieve the :term:`annotations ` - of an object. This function is accessible as the :attr:`~object.__annotate__` - attribute of functions, classes, and modules. Annotate functions are a - subset of :term:`evaluate functions `. + A callable that can be called to retrieve the :term:`annotations ` of + an object. Annotate functions are usually :term:`functions `, + automatically generated as the :attr:`~object.__annotate__` attribute of functions, + classes, and modules. Annotate functions are a subset of + :term:`evaluate functions `. annotation A label associated with a variable, a class @@ -93,21 +96,24 @@ Glossary :meth:`~object.__aexit__` methods. Introduced by :pep:`492`. asynchronous generator - A function which returns an :term:`asynchronous generator iterator`. It - looks like a coroutine function defined with :keyword:`async def` except - that it contains :keyword:`yield` expressions for producing a series of - values usable in an :keyword:`async for` loop. - - Usually refers to an asynchronous generator function, but may refer to an - *asynchronous generator iterator* in some contexts. In cases where the - intended meaning isn't clear, using the full terms avoids ambiguity. + Informally used to mean either an :term:`asynchronous generator + function` or an :term:`asynchronous generator iterator`, depending on + context. The formal terms :term:`asynchronous generator function` and + :term:`asynchronous generator iterator` are uncommon in practice; + "asynchronous generator" alone is almost always sufficient. + + asynchronous generator function + A function which returns an :term:`asynchronous generator iterator`. + It looks like a coroutine function defined with :keyword:`async def` + except that it contains :keyword:`yield` expressions for producing a + series of values usable in an :keyword:`async for` loop. See :pep:`525`. An asynchronous generator function may contain :keyword:`await` expressions as well as :keyword:`async for`, and :keyword:`async with` statements. asynchronous generator iterator - An object created by a :term:`asynchronous generator` function. + An object created by an :term:`asynchronous generator function`. This is an :term:`asynchronous iterator` which when called using the :meth:`~object.__anext__` method returns an awaitable object which will execute @@ -132,6 +138,14 @@ Glossary iterator's :meth:`~object.__anext__` method until it raises a :exc:`StopAsyncIteration` exception. Introduced by :pep:`492`. + atomic operation + An operation that appears to execute as a single, indivisible step: no + other thread can observe it half-done, and its effects become visible all + at once. Python does not guarantee that high-level statements are atomic + (for example, ``x += 1`` performs multiple bytecode operations and is not + atomic). Atomicity is only guaranteed where explicitly documented. See + also :term:`race condition` and :term:`data race`. + attached thread state A :term:`thread state` that is active for the current OS thread. @@ -150,9 +164,9 @@ Glossary On most builds of Python, having an attached thread state implies that the caller holds the :term:`GIL` for the current interpreter, so only one OS thread can have an attached thread state at a given moment. In - :term:`free-threaded ` builds of Python, threads can concurrently - hold an attached thread state, allowing for true parallelism of the bytecode - interpreter. + :term:`free-threaded builds ` of Python, threads can + concurrently hold an attached thread state, allowing for true parallelism of + the bytecode interpreter. attribute A value associated with an object which is usually referenced by name @@ -287,6 +301,22 @@ Glossary advanced mathematical feature. If you're not aware of a need for them, it's almost certain you can safely ignore them. + concurrency + The ability of a computer program to perform multiple tasks at the same + time. Python provides libraries for writing programs that make use of + different forms of concurrency. :mod:`asyncio` is a library for dealing + with asynchronous tasks and coroutines. :mod:`threading` provides + access to operating system threads and :mod:`multiprocessing` to + operating system processes. Multi-core processors can execute threads and + processes on different CPU cores at the same time (see + :term:`parallelism`). + + concurrent modification + When multiple threads modify shared data at the same time. Concurrent + modification without proper synchronization can cause + :term:`race conditions `, and might also trigger a + :term:`data race `, data corruption, or both. + context This term has different meanings depending on where and how it is used. Some common meanings: @@ -355,6 +385,34 @@ Glossary tasks (see :mod:`asyncio`) associate each task with a context which becomes the current context whenever the task starts or resumes execution. + cyclic isolate + A subgroup of one or more objects that reference each other in a reference + cycle, but are not referenced by objects outside the group. The goal of + the :term:`cyclic garbage collector ` is to identify these groups and break the reference + cycles so that the memory can be reclaimed. + + data race + A situation where multiple threads access the same memory location + concurrently, at least one of the accesses is a write, and the threads + do not use any synchronization to control their access. Data races + lead to :term:`non-deterministic` behavior and can cause data corruption. + Proper use of :term:`locks ` and other :term:`synchronization primitives + ` prevents data races. Note that data races + can only happen in native code, but that :term:`native code` might be + exposed in a Python API. See also :term:`race condition` and + :term:`thread-safe`. + + deadlock + A situation in which two or more tasks (threads, processes, or coroutines) + wait indefinitely for each other to release resources or complete actions, + preventing any from making progress. For example, if thread A holds lock + 1 and waits for lock 2, while thread B holds lock 2 and waits for lock 1, + both threads will wait indefinitely. In Python this often arises from + acquiring multiple locks in conflicting orders or from circular + join/await dependencies. Deadlocks can be avoided by always acquiring + multiple :term:`locks ` in a consistent order. See also + :term:`lock` and :term:`reentrant`. + decorator A function returning another function, usually applied as a function transformation using the ``@wrapper`` syntax. Common examples for @@ -429,6 +487,11 @@ Glossary with :term:`abstract base classes `.) Instead, it typically employs :func:`hasattr` tests or :term:`EAFP` programming. + dunder + An informal short-hand for "double underscore", used when talking about a + :term:`special method`. For example, ``__init__`` is often pronounced + "dunder init". + EAFP Easier to ask for forgiveness than permission. This common Python coding style assumes the existence of valid keys or attributes and catches @@ -456,7 +519,8 @@ Glossary core and with user code. f-string - String literals prefixed with ``'f'`` or ``'F'`` are commonly called + f-strings + String literals prefixed with ``f`` or ``F`` are commonly called "f-strings" which is short for :ref:`formatted string literals `. See also :pep:`498`. @@ -520,6 +584,13 @@ Glossary the :term:`global interpreter lock` which allows only one thread to execute Python bytecode at a time. See :pep:`703`. + free-threaded build + + A build of :term:`CPython` that supports :term:`free threading`, + configured using the :option:`--disable-gil` option before compilation. + + See :ref:`freethreading-python-howto`. + free variable Formally, as defined in the :ref:`language execution model `, a free variable is any variable used in a namespace which is not a local variable in that @@ -573,23 +644,33 @@ Glossary .. index:: single: generator generator - A function which returns a :term:`generator iterator`. It looks like a - normal function except that it contains :keyword:`yield` expressions - for producing a series of values usable in a for-loop or that can be - retrieved one at a time with the :func:`next` function. + Informally used to mean either a :term:`generator function` or a + :term:`generator iterator`, depending on context. The formal terms + :term:`generator function` and :term:`generator iterator` are uncommon + in practice; "generator" alone is almost always sufficient. - Usually refers to a generator function, but may refer to a - *generator iterator* in some contexts. In cases where the intended - meaning isn't clear, using the full terms avoids ambiguity. + .. index:: single: generator function + + generator function + A function which returns a :term:`generator` object. It looks like a + normal function except that it contains :keyword:`yield` expressions + for producing a series of values usable in a :keyword:`for`\-loop or + that can be retrieved one at a time with the :func:`next` function. + See :ref:`yieldexpr`. generator iterator - An object created by a :term:`generator` function. + An object created by a :term:`generator function` or a + :term:`generator expression`. Each :keyword:`yield` temporarily suspends processing, remembering the - execution state (including local variables and pending - try-statements). When the *generator iterator* resumes, it picks up where - it left off (in contrast to functions which start fresh on every - invocation). + execution state (including local variables and pending try-statements). + When the *generator iterator* resumes, it picks up where it left off + (in contrast to functions which start fresh on every invocation). + + Generator iterators also implement the :meth:`~generator.send` method + to send a value into the suspended generator, and the + :meth:`~generator.throw` method to raise an exception at the point + where the generator was paused. See :ref:`generator-methods`. .. index:: single: generator expression @@ -648,6 +729,14 @@ Glossary requires the GIL to be held in order to use it. This refers to having an :term:`attached thread state`. + global state + Data that is accessible throughout a program, such as module-level + variables, class variables, or C static variables in :term:`extension modules + `. In multi-threaded programs, global state shared + between threads typically requires synchronization to avoid + :term:`race conditions ` and + :term:`data races `. + hash-based pyc A bytecode cache file that uses the hash rather than the last-modified time of the corresponding source file to determine its validity. See @@ -692,7 +781,9 @@ Glossary tuples. Such an object cannot be altered. A new object has to be created if a different value has to be stored. They play an important role in places where a constant hash value is needed, for example as a key - in a dictionary. + in a dictionary. Immutable objects are inherently :term:`thread-safe` + because their state cannot be modified after creation, eliminating concerns + about improperly synchronized :term:`concurrent modification`. import path A list of locations (or :term:`path entries `) that are @@ -709,6 +800,19 @@ Glossary An object that both finds and loads a module; both a :term:`finder` and :term:`loader` object. + index + A numeric value that represents the position of an element in + a :term:`sequence`. + + In Python, indexing starts at zero. + For example, ``things[0]`` names the *first* element of ``things``; + ``things[1]`` names the second one. + + In some contexts, Python allows negative indexes for counting from the + end of a sequence, and indexing using :term:`slices `. + + See also :term:`subscript`. + interactive Python has an interactive interpreter which means you can enter statements and expressions at the interpreter prompt, immediately @@ -782,9 +886,13 @@ Glossary CPython does not consistently apply the requirement that an iterator define :meth:`~iterator.__iter__`. - And also please note that the free-threading CPython does not guarantee - the thread-safety of iterator operations. + And also please note that :term:`free-threaded ` + CPython does not guarantee :term:`thread-safe` behavior of iterator + operations. + key + A value that identifies an entry in a :term:`mapping`. + See also :term:`subscript`. key function A key function or collation function is a callable that returns a value @@ -799,7 +907,7 @@ Glossary :func:`itertools.groupby`. There are several ways to create a key function. For example. the - :meth:`str.lower` method can serve as a key function for case insensitive + :meth:`str.casefold` method can serve as a key function for case insensitive sorts. Alternatively, a key function can be built from a :keyword:`lambda` expression such as ``lambda r: (r[0], r[2])``. Also, :func:`operator.attrgetter`, :func:`operator.itemgetter`, and @@ -821,10 +929,11 @@ Glossary :keyword:`if` statements. In a multi-threaded environment, the LBYL approach can risk introducing a - race condition between "the looking" and "the leaping". For example, the - code, ``if key in mapping: return mapping[key]`` can fail if another + :term:`race condition` between "the looking" and "the leaping". For example, + the code, ``if key in mapping: return mapping[key]`` can fail if another thread removes *key* from *mapping* after the test, but before the lookup. - This issue can be solved with locks or by using the EAFP approach. + This issue can be solved with :term:`locks ` or by using the + :term:`EAFP` approach. See also :term:`thread-safe`. lexical analyzer @@ -843,6 +952,29 @@ Glossary clause is optional. If omitted, all elements in ``range(256)`` are processed. + lock + A :term:`synchronization primitive` that allows only one thread at a + time to access a shared resource. A thread must acquire a lock before + accessing the protected resource and release it afterward. If a thread + attempts to acquire a lock that is already held by another thread, it + will block until the lock becomes available. Python's :mod:`threading` + module provides :class:`~threading.Lock` (a basic lock) and + :class:`~threading.RLock` (a :term:`reentrant` lock). Locks are used + to prevent :term:`race conditions ` and ensure + :term:`thread-safe` access to shared data. Alternative design patterns + to locks exist such as queues, producer/consumer patterns, and + thread-local state. See also :term:`deadlock`, and :term:`reentrant`. + + lock-free + An operation that does not acquire any :term:`lock` and uses atomic CPU + instructions to ensure correctness. Lock-free operations can execute + concurrently without blocking each other and cannot be blocked by + operations that hold locks. In :term:`free-threaded ` + Python, built-in types like :class:`dict` and :class:`list` provide + lock-free read operations, which means other threads may observe + intermediate states during multi-step modifications even when those + modifications hold the :term:`per-object lock`. + loader An object that loads a module. It must define the :meth:`!exec_module` and :meth:`!create_module` methods @@ -928,8 +1060,11 @@ Glossary See :term:`method resolution order`. mutable - Mutable objects can change their value but keep their :func:`id`. See - also :term:`immutable`. + An :term:`object` with state that is allowed to change during the course + of the program. In multi-threaded programs, mutable objects that are + shared between threads require careful synchronization to avoid + :term:`race conditions `. See also :term:`immutable`, + :term:`thread-safe`, and :term:`concurrent modification`. named tuple The term "named tuple" applies to any type or class that inherits from @@ -981,6 +1116,13 @@ Glossary See also :term:`module`. + native code + Code that is compiled to machine instructions and runs directly on the + processor, as opposed to code that is interpreted or runs in a virtual + machine. In the context of Python, native code typically refers to + C, C++, Rust or Fortran code in :term:`extension modules ` + that can be called from Python. See also :term:`extension module`. + nested scope The ability to refer to a variable in an enclosing definition. For instance, a function defined inside another function can refer to @@ -997,6 +1139,15 @@ Glossary properties, :meth:`~object.__getattribute__`, class methods, and static methods. + non-deterministic + Behavior where the outcome of a program can vary between executions with + the same inputs. In multi-threaded programs, non-deterministic behavior + often results from :term:`race conditions ` where the + relative timing or interleaving of threads affects the result. + Proper synchronization using :term:`locks ` and other + :term:`synchronization primitives ` helps + ensure deterministic behavior. + object Any data with state (attributes or value) and defined behavior (methods). Also the ultimate base class of any :term:`new-style @@ -1011,6 +1162,15 @@ Glossary applied to all scopes, only those relying on a known set of local and nonlocal variable names are restricted to optimized scopes. + optional module + An :term:`extension module` that is part of the :term:`standard library`, + but may be absent in some builds of :term:`CPython`, + usually due to missing third-party libraries or because the module + is not available for a given platform. + + See :ref:`optional-module-requirements` for a list of optional modules + that require third-party libraries. + package A Python :term:`module` which can contain submodules or recursively, subpackages. Technically, a package is a Python module with a @@ -1018,6 +1178,16 @@ Glossary See also :term:`regular package` and :term:`namespace package`. + parallelism + Executing multiple operations at the same time (e.g. on multiple CPU + cores). In Python builds with the + :term:`global interpreter lock (GIL) `, only one + thread runs Python bytecode at a time, so taking advantage of multiple + CPU cores typically involves multiple processes + (e.g. :mod:`multiprocessing`) or native extensions that release the GIL. + In :term:`free-threaded ` Python, multiple Python threads + can run Python code simultaneously on different cores. + parameter A named entity in a :term:`function` (or method) definition that specifies an :term:`argument` (or in some cases, arguments) that the @@ -1071,6 +1241,16 @@ Glossary `, the :class:`inspect.Parameter` class, the :ref:`function` section, and :pep:`362`. + per-object lock + A :term:`lock` associated with an individual object instance rather than + a global lock shared across all objects. In :term:`free-threaded + ` Python, built-in types like :class:`dict` and + :class:`list` use per-object locks to allow concurrent operations on + different objects while serializing operations on the same object. + Operations that hold the per-object lock prevent other locking operations + on the same object from proceeding, but do not block :term:`lock-free` + operations. + path entry A single location on the :term:`import path` which the :term:`path based finder` consults to find modules for importing. @@ -1192,6 +1372,18 @@ Glossary >>> email.mime.text.__name__ 'email.mime.text' + race condition + A condition of a program where the behavior + depends on the relative timing or ordering of events, particularly in + multi-threaded programs. Race conditions can lead to + :term:`non-deterministic` behavior and bugs that are difficult to + reproduce. A :term:`data race` is a specific type of race condition + involving unsynchronized access to shared memory. The :term:`LBYL` + coding style is particularly susceptible to race conditions in + multi-threaded code. Using :term:`locks ` and other + :term:`synchronization primitives ` + helps prevent race conditions. + reference count The number of references to an object. When the reference count of an object drops to zero, it is deallocated. Some objects are @@ -1202,12 +1394,36 @@ Glossary :func:`sys.getrefcount` function to return the reference count for a particular object. + In :term:`CPython`, reference counts are not considered to be stable + or well-defined values; the number of references to an object, and how + that number is affected by Python code, may be different between + versions. + regular package A traditional :term:`package`, such as a directory containing an ``__init__.py`` file. See also :term:`namespace package`. + reentrant + A property of a function or :term:`lock` that allows it to be called or + acquired multiple times by the same thread without causing errors or a + :term:`deadlock`. + + For functions, reentrancy means the function can be safely called again + before a previous invocation has completed, which is important when + functions may be called recursively or from signal handlers. Thread-unsafe + functions may be :term:`non-deterministic` if they're called reentrantly in a + multithreaded program. + + For locks, Python's :class:`threading.RLock` (reentrant lock) is + reentrant, meaning a thread that already holds the lock can acquire it + again without blocking. In contrast, :class:`threading.Lock` is not + reentrant - attempting to acquire it twice from the same thread will cause + a deadlock. + + See also :term:`lock` and :term:`deadlock`. + REPL An acronym for the "read–eval–print loop", another name for the :term:`interactive` interpreter shell. @@ -1232,8 +1448,9 @@ Glossary The :class:`collections.abc.Sequence` abstract base class defines a much richer interface that goes beyond just :meth:`~object.__getitem__` and :meth:`~object.__len__`, adding - :meth:`!count`, :meth:`!index`, :meth:`~object.__contains__`, and - :meth:`~object.__reversed__`. Types that implement this expanded + :meth:`~sequence.count`, :meth:`~sequence.index`, + :meth:`~object.__contains__`, and :meth:`~object.__reversed__`. + Types that implement this expanded interface can be registered explicitly using :func:`~abc.ABCMeta.register`. For more documentation on sequence methods generally, see @@ -1250,10 +1467,11 @@ Glossary chosen based on the type of a single argument. slice - An object usually containing a portion of a :term:`sequence`. A slice is - created using the subscript notation, ``[]`` with colons between numbers - when several are given, such as in ``variable_name[1:3:5]``. The bracket - (subscript) notation uses :class:`slice` objects internally. + An object of type :class:`slice`, used to describe a portion of + a :term:`sequence`. + A slice object is created when using the :ref:`slicing ` form + of :ref:`subscript notation `, with colons inside square + brackets, such as in ``variable_name[1:3:5]``. soft deprecated A soft deprecated API should not be used in new code, @@ -1274,6 +1492,16 @@ Glossary and ending with double underscores. Special methods are documented in :ref:`specialnames`. + standard library + The collection of :term:`packages `, :term:`modules ` + and :term:`extension modules ` distributed as a part + of the official Python interpreter package. The exact membership of the + collection may vary based on platform, available system libraries, or + other criteria. Documentation can be found at :ref:`library-index`. + + See also :data:`sys.stdlib_module_names` for a list of all possible + standard library module names. + statement A statement is part of a suite (a "block" of code). A statement is either an :term:`expression` or one of several constructs with a keyword, such @@ -1284,6 +1512,17 @@ Glossary issues such as incorrect types. See also :term:`type hints ` and the :mod:`typing` module. + stdlib + An abbreviation of :term:`standard library`. + + steal + In Python's C API, "*stealing*" an argument means that ownership of the + argument is transferred to the called function. + The caller must not use that reference after the call. + Generally, functions that "steal" an argument do so even if they fail. + + See :ref:`api-refcountdetails` for a full explanation. + strong reference In Python's C API, a strong reference is a reference to an object which is owned by the code holding the reference. The strong @@ -1298,6 +1537,32 @@ Glossary See also :term:`borrowed reference`. + subscript + The expression in square brackets of a + :ref:`subscription expression `, for example, + the ``3`` in ``items[3]``. + Usually used to select an element of a container. + Also called a :term:`key` when subscripting a :term:`mapping`, + or an :term:`index` when subscripting a :term:`sequence`. + + synchronization primitive + A basic building block for coordinating (synchronizing) the execution of + multiple threads to ensure :term:`thread-safe` access to shared resources. + Python's :mod:`threading` module provides several synchronization primitives + including :class:`~threading.Lock`, :class:`~threading.RLock`, + :class:`~threading.Semaphore`, :class:`~threading.Condition`, + :class:`~threading.Event`, and :class:`~threading.Barrier`. Additionally, + the :mod:`queue` module provides multi-producer, multi-consumer queues + that are especially useful in multithreaded programs. These + primitives help prevent :term:`race conditions ` and + coordinate thread execution. See also :term:`lock`. + + t-string + t-strings + String literals prefixed with ``t`` or ``T`` are commonly called + "t-strings" which is short for + :ref:`template string literals `. + text encoding A string in Python is a sequence of Unicode code points (in range ``U+0000``--``U+10FFFF``). To store or transfer a string, it needs to be @@ -1344,6 +1609,19 @@ Glossary See :ref:`Thread State and the Global Interpreter Lock ` for more information. + thread-safe + A module, function, or class that behaves correctly when used by multiple + threads concurrently. Thread-safe code uses appropriate + :term:`synchronization primitives ` like + :term:`locks ` to protect shared mutable state, or is designed + to avoid shared mutable state entirely. In the + :term:`free-threaded ` build, built-in types like + :class:`dict`, :class:`list`, and :class:`set` use internal locking + to make many operations thread-safe, although thread safety is not + necessarily guaranteed. Code that is not thread-safe may experience + :term:`race conditions ` and :term:`data races ` + when used in multi-threaded programs. + token A small unit of source code, generated by the @@ -1443,6 +1721,11 @@ Glossary A computer defined entirely in software. Python's virtual machine executes the :term:`bytecode` emitted by the bytecode compiler. + walrus operator + A light-hearted way to refer to the :ref:`assignment expression + ` operator ``:=`` because it looks a bit like a + walrus if you turn your head. + Zen of Python Listing of Python design principles and philosophies that are helpful in understanding and using the language. The listing can be found by typing diff --git a/Doc/howto/a-conceptual-overview-of-asyncio.rst b/Doc/howto/a-conceptual-overview-of-asyncio.rst new file mode 100644 index 000000000000000..7c5f8df4558e6a6 --- /dev/null +++ b/Doc/howto/a-conceptual-overview-of-asyncio.rst @@ -0,0 +1,608 @@ +.. _a-conceptual-overview-of-asyncio: + +**************************************** +A Conceptual Overview of :mod:`!asyncio` +**************************************** + +This :ref:`HOWTO ` article seeks to help you build a sturdy mental +model of how :mod:`asyncio` fundamentally works, helping you understand the +how and why behind the recommended patterns. + +You might be curious about some key :mod:`!asyncio` concepts. +By the end of this article, you'll be able to comfortably answer these questions: + +- What's happening behind the scenes when an object is awaited? +- How does :mod:`!asyncio` differentiate between a task which doesn't need + CPU time (such as a network request or file read) as opposed to a task that + does (such as computing n-factorial)? +- How to write an asynchronous variant of an operation, such as + an async sleep or database request. + +.. seealso:: + + * The `guide `_ that inspired this HOWTO article, by Alexander Nordin. + * This in-depth `YouTube tutorial series `_ on + ``asyncio`` created by Python core team member, Łukasz Langa. + * `500 Lines or Less: A Web Crawler With asyncio Coroutines `_ by A. + Jesse Jiryu Davis and Guido van Rossum. + +-------------------------------------------- +A conceptual overview part 1: the high-level +-------------------------------------------- + +In part 1, we'll cover the main, high-level building blocks of :mod:`!asyncio`: +the event loop, coroutine functions, coroutine objects, tasks, and ``await``. + +========== +Event Loop +========== + +Everything in :mod:`!asyncio` happens relative to the event loop. +It's the star of the show. +It's like an orchestra conductor. +It's behind the scenes managing resources. +Some power is explicitly granted to it, but a lot of its ability to get things +done comes from the respect and cooperation of its worker bees. + +In more technical terms, the event loop contains a collection of jobs to be run. +Some jobs are added directly by you, and some indirectly by :mod:`!asyncio`. +The event loop takes a job from its backlog of work and invokes it (or "gives +it control"), similar to calling a function, and then that job runs. +Once it pauses or completes, it returns control to the event loop. +The event loop will then select another job from its pool and invoke it. +You can *roughly* think of the collection of jobs as a queue: jobs are added and +then processed one at a time, generally (but not always) in order. +This process repeats indefinitely, with the event loop cycling endlessly +onwards. +If there are no more jobs pending execution, the event loop is smart enough to +rest and avoid needlessly wasting CPU cycles, and will come back when there's +more work to be done. + +Effective execution relies on jobs sharing well and cooperating; a greedy job +could hog control and leave the other jobs to starve, rendering the overall +event loop approach rather useless. + +:: + + import asyncio + + # This creates an event loop and indefinitely cycles through + # its collection of jobs. + event_loop = asyncio.new_event_loop() + event_loop.run_forever() + +===================================== +Asynchronous functions and coroutines +===================================== + +This is a basic, boring Python function:: + + def hello_printer(): + print( + "Hi, I am a lowly, simple printer, though I have all I " + "need in life -- \nfresh paper and my dearly beloved octopus " + "partner in crime." + ) + +Calling a regular function invokes its logic or body:: + + >>> hello_printer() + Hi, I am a lowly, simple printer, though I have all I need in life -- + fresh paper and my dearly beloved octopus partner in crime. + +The :ref:`async def `, as opposed to just a plain ``def``, makes +this an asynchronous function (or "coroutine function"). +Calling it creates and returns a :ref:`coroutine ` object. + +:: + + async def loudmouth_penguin(magic_number: int): + print( + "I am a super special talking penguin. Far cooler than that printer. " + f"By the way, my lucky number is: {magic_number}." + ) + +Calling the async function, ``loudmouth_penguin``, does not execute the print statement; +instead, it creates a coroutine object:: + + >>> loudmouth_penguin(magic_number=3) + + +The terms "coroutine function" and "coroutine object" are often conflated +as coroutine. +That can be confusing! +In this article, coroutine specifically refers to a coroutine object, or more +precisely, an instance of :class:`types.CoroutineType` (native coroutine). +Note that coroutines can also exist as instances of +:class:`collections.abc.Coroutine` -- a distinction that matters for type +checking. + +A coroutine represents the function's body or logic. +A coroutine has to be explicitly started; again, merely creating the coroutine +does not start it. +Notably, the coroutine can be paused and resumed at various points within the +function's body. +That pausing and resuming ability is what allows for asynchronous behavior! + +Coroutines and coroutine functions were built by leveraging the functionality +of :term:`generators ` and +:term:`generator functions `. +Recall, a generator function is a function that :keyword:`yield`\s, like this +one:: + + def get_random_number(): + # This would be a bad random number generator! + print("Hi") + yield 1 + print("Hello") + yield 7 + print("Howdy") + yield 4 + ... + +Similar to a coroutine function, calling a generator function does not run it. +Instead, it creates a generator object:: + + >>> get_random_number() + + +You can proceed to the next ``yield`` of a generator by using the +built-in function :func:`next`. +In other words, the generator runs, then pauses. +For example:: + + >>> generator = get_random_number() + >>> next(generator) + Hi + 1 + >>> next(generator) + Hello + 7 + +===== +Tasks +===== + +Roughly speaking, :ref:`tasks ` are coroutines (not coroutine +functions) tied to an event loop. +A task also maintains a list of callback functions whose importance will become +clear in a moment when we discuss :keyword:`await`. +The recommended way to create tasks is via :func:`asyncio.create_task`. + +Creating a task automatically schedules it for execution (by adding a +callback to run it in the event loop's to-do list, that is, collection of jobs). + +:mod:`!asyncio` automatically associates tasks with the event loop for you. +This automatic association was purposely designed into :mod:`!asyncio` for +the sake of simplicity. +Without it, you'd have to keep track of the event loop object and pass it to +any coroutine function that wants to create tasks, adding redundant clutter +to your code. + +:: + + coroutine = loudmouth_penguin(magic_number=5) + # This creates a Task object and schedules its execution via the event loop. + task = asyncio.create_task(coroutine) + +Earlier, we manually created the event loop and set it to run forever. +In practice, it's recommended to use (and common to see) :func:`asyncio.run`, +which takes care of managing the event loop and ensuring the provided +coroutine finishes before advancing. +For example, many async programs follow this setup:: + + import asyncio + + async def main(): + # Perform all sorts of wacky, wild asynchronous things... + ... + + if __name__ == "__main__": + asyncio.run(main()) + # The program will not reach the following print statement until the + # coroutine main() finishes. + print("coroutine main() is done!") + +It's important to be aware that the task itself is not added to the event loop, +only a callback to the task is. +This matters if the task object you created is garbage collected before it's +called by the event loop. +For example, consider this program: + +.. code-block:: + :linenos: + + async def hello(): + print("hello!") + + async def main(): + asyncio.create_task(hello()) + # Other asynchronous instructions which run for a while + # and cede control to the event loop... + ... + + asyncio.run(main()) + +Because there's no reference to the task object created on line 5, it *might* +be garbage collected before the event loop invokes it. +Later instructions in the coroutine ``main()`` hand control back to the event +loop so it can invoke other jobs. +When the event loop eventually tries to run the task, it might fail and +discover the task object does not exist! +This can also happen even if a coroutine keeps a reference to a task but +completes before that task finishes. +When the coroutine exits, local variables go out of scope and may be subject +to garbage collection. +In practice, ``asyncio`` and Python's garbage collector work pretty hard to +ensure this sort of thing doesn't happen. +But that's no reason to be reckless! + +===== +await +===== + +:keyword:`await` is a Python keyword that's commonly used in one of two +different ways:: + + await task + await coroutine + +In a crucial way, the behavior of ``await`` depends on the type of object +being awaited. + +Awaiting a task will cede control from the current task or coroutine to +the event loop. +In the process of relinquishing control, a few important things happen. +We'll use the following code example to illustrate:: + + async def plant_a_tree(): + dig_the_hole_task = asyncio.create_task(dig_the_hole()) + await dig_the_hole_task + + # Other instructions associated with planting a tree. + ... + +In this example, imagine the event loop has passed control to the start of the +coroutine ``plant_a_tree()``. +As seen above, the coroutine creates a task and then awaits it. +The ``await dig_the_hole_task`` instruction adds a callback (which will resume +``plant_a_tree()``) to the ``dig_the_hole_task`` object's list of callbacks. +And then, the instruction cedes control to the event loop. +Some time later, the event loop will pass control to ``dig_the_hole_task`` +and the task will finish whatever it needs to do. +Once the task finishes, it will add its various callbacks to the event loop, +in this case, a call to resume ``plant_a_tree()``. + +Generally speaking, when the awaited task finishes (``dig_the_hole_task``), +the original task or coroutine (``plant_a_tree()``) is added back to the event +loop's to-do list to be resumed. + +This is a basic, yet reliable mental model. +In practice, the control handoffs are slightly more complex, but not by much. +In part 2, we'll walk through the details that make this possible. + +**Unlike tasks, awaiting a coroutine does not hand control back to the event +loop!** +Wrapping a coroutine in a task first, then awaiting that would cede +control. +The behavior of ``await coroutine`` is effectively the same as invoking a +regular, synchronous Python function. +Consider this program:: + + import asyncio + + async def coro_a(): + print("I am coro_a(). Hi!") + + async def coro_b(): + print("I am coro_b(). I sure hope no one hogs the event loop...") + + async def main(): + task_b = asyncio.create_task(coro_b()) + num_repeats = 3 + for _ in range(num_repeats): + await coro_a() + await task_b + + asyncio.run(main()) + +The first statement in the coroutine ``main()`` creates ``task_b`` and schedules +it for execution via the event loop. +Then, ``coro_a()`` is repeatedly awaited. Control never cedes to the +event loop, which is why we see the output of all three ``coro_a()`` +invocations before ``coro_b()``'s output: + +.. code-block:: none + + I am coro_a(). Hi! + I am coro_a(). Hi! + I am coro_a(). Hi! + I am coro_b(). I sure hope no one hogs the event loop... + +If we change ``await coro_a()`` to ``await asyncio.create_task(coro_a())``, the +behavior changes. +The coroutine ``main()`` cedes control to the event loop with that statement. +The event loop then proceeds through its backlog of work, calling ``task_b`` +and then the task which wraps ``coro_a()`` before resuming the coroutine +``main()``. + +.. code-block:: none + + I am coro_b(). I sure hope no one hogs the event loop... + I am coro_a(). Hi! + I am coro_a(). Hi! + I am coro_a(). Hi! + +This behavior of ``await coroutine`` can trip a lot of people up! +That example highlights how using only ``await coroutine`` could +unintentionally hog control from other tasks and effectively stall the event +loop. +:func:`asyncio.run` can help you detect such occurrences via the +``debug=True`` flag, which enables +:ref:`debug mode `. +Among other things, it will log any coroutines that monopolize execution for +100ms or longer. + +The design intentionally trades off some conceptual clarity around usage of +``await`` for improved performance. +Each time a task is awaited, control needs to be passed all the way up the +call stack to the event loop. +That might sound minor, but in a large program with many ``await`` statements and a deep +call stack, that overhead can add up to a meaningful performance drag. + +------------------------------------------------ +A conceptual overview part 2: the nuts and bolts +------------------------------------------------ + +Part 2 goes into detail on the mechanisms :mod:`!asyncio` uses to manage +control flow. +This is where the magic happens. +You'll come away from this section knowing what ``await`` does behind the scenes +and how to make your own asynchronous operators. + +================================ +The inner workings of coroutines +================================ + +:mod:`!asyncio` leverages four components to pass around control. + +:meth:`coroutine.send(arg) ` is the method used to start or +resume a coroutine. +If the coroutine was paused and is now being resumed, the argument ``arg`` +will be sent in as the return value of the ``yield`` statement which originally +paused it. +If the coroutine is being used for the first time (as opposed to being resumed), +``arg`` must be ``None``. + +.. code-block:: + :linenos: + + class Rock: + def __await__(self): + value_sent_in = yield 7 + print(f"Rock.__await__ resuming with value: {value_sent_in}.") + return value_sent_in + + async def main(): + print("Beginning coroutine main().") + rock = Rock() + print("Awaiting rock...") + value_from_rock = await rock + print(f"Coroutine received value: {value_from_rock} from rock.") + return 23 + + coroutine = main() + intermediate_result = coroutine.send(None) + print(f"Coroutine paused and returned intermediate value: {intermediate_result}.") + + print(f"Resuming coroutine and sending in value: 42.") + try: + coroutine.send(42) + except StopIteration as e: + returned_value = e.value + print(f"Coroutine main() finished and provided value: {returned_value}.") + +:ref:`yield `, as usual, pauses execution and returns control +to the caller. +In the example above, the ``yield``, on line 3, is called by +``... = await rock`` on line 11. +More broadly speaking, ``await`` calls the :meth:`~object.__await__` method of +the given object. +``await`` also does one more very special thing: it propagates (or "passes +along") any ``yield``\ s it receives up the call chain. +In this case, that's back to ``... = coroutine.send(None)`` on line 16. + +The coroutine is resumed via the ``coroutine.send(42)`` call on line 21. +The coroutine picks back up from where it ``yield``\ ed (or paused) on line 3 +and executes the remaining statements in its body. +When a coroutine finishes, it raises a :exc:`StopIteration` exception with the +return value attached in the :attr:`~StopIteration.value` attribute. + +That snippet produces this output: + +.. code-block:: none + + Beginning coroutine main(). + Awaiting rock... + Coroutine paused and returned intermediate value: 7. + Resuming coroutine and sending in value: 42. + Rock.__await__ resuming with value: 42. + Coroutine received value: 42 from rock. + Coroutine main() finished and provided value: 23. + +It's worth pausing for a moment here and making sure you followed the various +ways that control flow and values were passed. A lot of important ideas were +covered and it's worth ensuring your understanding is firm. + +The only way to yield (or effectively cede control) from a coroutine is to +``await`` an object that ``yield``\ s in its ``__await__`` method. +That might sound odd to you. You might be thinking: + + 1. What about a ``yield`` directly within the coroutine function? The + coroutine function becomes an + :ref:`async generator function `, a + different beast entirely. + + 2. What about a :ref:`yield from ` within the coroutine function to a (plain) + generator? + That causes the error: ``SyntaxError: yield from not allowed in a coroutine.`` + This was intentionally designed for the sake of simplicity -- mandating only + one way of using coroutines. + Initially ``yield`` was barred as well, but was re-accepted to allow for + async generators. + Despite that, ``yield from`` and ``await`` effectively do the same thing. + +======= +Futures +======= + +A :ref:`future ` is an object meant to represent a +computation's status and result. +The term is a nod to the idea of something still to come or not yet happened, +and the object is a way to keep an eye on that something. + +A future has a few important attributes. One is its state, which can be either +"pending", "cancelled", or "done". +Another is its result, which is set when the state transitions to done. +Unlike a coroutine, a future does not represent the actual computation to be +done; instead, it represents the status and result of that computation, kind of +like a status light (red, yellow, or green) or indicator. + +:class:`asyncio.Task` subclasses :class:`asyncio.Future` in order to gain +these various capabilities. +The prior section said tasks store a list of callbacks, which wasn't entirely +accurate. +It's actually the ``Future`` class that implements this logic, which ``Task`` +inherits. + +Futures may also be used directly (not via tasks). +Tasks mark themselves as done when their coroutine is complete. +Futures are much more versatile and will be marked as done when you say so. +In this way, they're the flexible interface for you to make your own conditions +for waiting and resuming. + +======================== +A homemade asyncio.sleep +======================== + +We'll go through an example of how you could leverage a future to create your +own variant of asynchronous sleep (``async_sleep``) which mimics +:func:`asyncio.sleep`. + +This snippet registers a few tasks with the event loop and then awaits the task +created by ``asyncio.create_task``, which wraps the ``async_sleep(3)`` coroutine. +We want that task to finish only after three seconds have elapsed, but without +preventing other tasks from running. + +:: + + async def other_work(): + print("I like work. Work work.") + + async def main(): + # Add a few other tasks to the event loop, so there's something + # to do while asynchronously sleeping. + work_tasks = [ + asyncio.create_task(other_work()), + asyncio.create_task(other_work()), + asyncio.create_task(other_work()) + ] + print( + "Beginning asynchronous sleep at time: " + f"{datetime.datetime.now().strftime("%H:%M:%S")}." + ) + await asyncio.create_task(async_sleep(3)) + print( + "Done asynchronous sleep at time: " + f"{datetime.datetime.now().strftime("%H:%M:%S")}." + ) + # asyncio.gather effectively awaits each task in the collection. + await asyncio.gather(*work_tasks) + + +Below, we use a future to enable custom control over when that task will be +marked as done. +If :meth:`future.set_result() ` (the method +responsible for marking that future as done) is never called, then this task +will never finish. +We've also enlisted the help of another task, which we'll see in a moment, that +will monitor how much time has elapsed and, accordingly, call +``future.set_result()``. + +:: + + async def async_sleep(seconds: float): + future = asyncio.Future() + time_to_wake = time.time() + seconds + # Add the watcher-task to the event loop. + watcher_task = asyncio.create_task(_sleep_watcher(future, time_to_wake)) + # Block until the future is marked as done. + await future + +Below, we use a rather bare ``YieldToEventLoop()`` object to ``yield`` +from its ``__await__`` method, ceding control to the event loop. +This is effectively the same as calling ``asyncio.sleep(0)``, but this approach +offers more clarity, not to mention it's somewhat cheating to use +``asyncio.sleep`` when showcasing how to implement it! + +As usual, the event loop cycles through its tasks, giving them control +and receiving control back when they pause or finish. +The ``watcher_task``, which runs the coroutine ``_sleep_watcher(...)``, will +be invoked once per full cycle of the event loop. +On each resumption, it'll check the time and if not enough has elapsed, then +it'll pause once again and hand control back to the event loop. +Once enough time has elapsed, ``_sleep_watcher(...)`` +marks the future as done and completes by exiting its +infinite ``while`` loop. +Given this helper task is only invoked once per cycle of the event loop, +you'd be correct to note that this asynchronous sleep will sleep *at least* +three seconds, rather than exactly three seconds. +Note this is also true of ``asyncio.sleep``. + +:: + + class YieldToEventLoop: + def __await__(self): + yield + + async def _sleep_watcher(future, time_to_wake): + while True: + if time.time() >= time_to_wake: + # This marks the future as done. + future.set_result(None) + break + else: + await YieldToEventLoop() + +Here is the full program's output: + +.. code-block:: none + + $ python custom-async-sleep.py + Beginning asynchronous sleep at time: 14:52:22. + I like work. Work work. + I like work. Work work. + I like work. Work work. + Done asynchronous sleep at time: 14:52:25. + +You might feel this implementation of asynchronous sleep was unnecessarily +convoluted. +And, well, it was. +The example was meant to showcase the versatility of futures with a simple +example that could be mimicked for more complex needs. +For reference, you could implement it without futures, like so:: + + async def simpler_async_sleep(seconds): + time_to_wake = time.time() + seconds + while True: + if time.time() >= time_to_wake: + return + else: + await YieldToEventLoop() + +But that's all for now. Hopefully you're ready to more confidently dive into +some async programming or check out advanced topics in the +:mod:`rest of the documentation `. diff --git a/Doc/howto/annotations.rst b/Doc/howto/annotations.rst index 78f3704ba5d000f..d7deb6c6bc1768f 100644 --- a/Doc/howto/annotations.rst +++ b/Doc/howto/annotations.rst @@ -248,4 +248,9 @@ quirks by using :func:`annotationlib.get_annotations` on Python 3.14+ or :func:`inspect.get_annotations` on Python 3.10+. On earlier versions of Python, you can avoid these bugs by accessing the annotations from the class's :attr:`~type.__dict__` -(e.g., ``cls.__dict__.get('__annotations__', None)``). +(for example, ``cls.__dict__.get('__annotations__', None)``). + +In some versions of Python, instances of classes may have an ``__annotations__`` +attribute. However, this is not supported functionality. If you need the +annotations of an instance, you can use :func:`type` to access its class +(for example, ``annotationlib.get_annotations(type(myinstance))`` on Python 3.14+). diff --git a/Doc/howto/cporting.rst b/Doc/howto/cporting.rst index 7773620b40b973b..cf857aed0425ec4 100644 --- a/Doc/howto/cporting.rst +++ b/Doc/howto/cporting.rst @@ -14,13 +14,11 @@ We recommend the following resources for porting extension modules to Python 3: module. * The `Porting guide`_ from the *py3c* project provides opinionated suggestions with supporting code. -* The `Cython`_ and `CFFI`_ libraries offer abstractions over - Python's C API. +* :ref:`Recommended third party tools ` offer abstractions over + the Python's C API. Extensions generally need to be re-written to use one of them, but the library then handles differences between various Python versions and implementations. .. _Migrating C extensions: http://python3porting.com/cextensions.html .. _Porting guide: https://py3c.readthedocs.io/en/latest/guide.html -.. _Cython: https://cython.org/ -.. _CFFI: https://cffi.readthedocs.io/en/latest/ diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst index 6994a5328e81496..7e8834161bc4849 100644 --- a/Doc/howto/curses.rst +++ b/Doc/howto/curses.rst @@ -55,8 +55,8 @@ code, all the functions described here will probably be available. The older versions of curses carried by some proprietary Unixes may not support everything, though. -The Windows version of Python doesn't include the :mod:`curses` -module. A ported version called :pypi:`UniCurses` is available. +The Windows version of Python doesn't include the :mod:`curses` module. +The third-party :pypi:`windows-curses` package provides the same interface on Windows. The Python curses module diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index f6c3e473f1c36d3..07a405837d92297 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -420,7 +420,7 @@ Here are three practical data validation utilities: def validate(self, value): if not isinstance(value, str): - raise TypeError(f'Expected {value!r} to be an str') + raise TypeError(f'Expected {value!r} to be a str') if self.minsize is not None and len(value) < self.minsize: raise ValueError( f'Expected {value!r} to be no smaller than {self.minsize!r}' @@ -1640,7 +1640,7 @@ by member descriptors: class Member: def __init__(self, name, clsname, offset): - 'Emulate PyMemberDef in Include/structmember.h' + 'Emulate PyMemberDef in Include/descrobject.h' # Also see descr_new() in Objects/descrobject.c self.name = name self.clsname = clsname diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 6441b7aed1eda8c..5260c2ca4add471 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -105,8 +105,8 @@ The complete :class:`!Weekday` enum now looks like this:: Now we can find out what today is! Observe:: - >>> from datetime import date - >>> Weekday.from_date(date.today()) # doctest: +SKIP + >>> import datetime as dt + >>> Weekday.from_date(dt.date.today()) # doctest: +SKIP Of course, if you're reading this on some other day, you'll see that day instead. @@ -965,75 +965,16 @@ want one of them to be the value:: Finer Points -^^^^^^^^^^^^ - -Supported ``__dunder__`` names -"""""""""""""""""""""""""""""" - -:attr:`~enum.EnumType.__members__` is a read-only ordered mapping of ``member_name``:``member`` -items. It is only available on the class. - -:meth:`~object.__new__`, if specified, must create and return the enum members; it is -also a very good idea to set the member's :attr:`~Enum._value_` appropriately. Once -all the members are created it is no longer used. - - -Supported ``_sunder_`` names -"""""""""""""""""""""""""""" +------------ -- :attr:`~Enum._name_` -- name of the member -- :attr:`~Enum._value_` -- value of the member; can be set in ``__new__`` -- :meth:`~Enum._missing_` -- a lookup function used when a value is not found; - may be overridden -- :attr:`~Enum._ignore_` -- a list of names, either as a :class:`list` or a - :class:`str`, that will not be transformed into members, and will be removed - from the final class -- :meth:`~Enum._generate_next_value_` -- used to get an appropriate value for - an enum member; may be overridden -- :meth:`~EnumType._add_alias_` -- adds a new name as an alias to an existing - member. -- :meth:`~EnumType._add_value_alias_` -- adds a new value as an alias to an - existing member. See `MultiValueEnum`_ for an example. +Supported ``__dunder__`` and ``_sunder_`` names +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - .. note:: - - For standard :class:`Enum` classes the next value chosen is the highest - value seen incremented by one. - - For :class:`Flag` classes the next value chosen will be the next highest - power-of-two. - - .. versionchanged:: 3.13 - Prior versions would use the last seen value instead of the highest value. - -.. versionadded:: 3.6 ``_missing_``, ``_order_``, ``_generate_next_value_`` -.. versionadded:: 3.7 ``_ignore_`` -.. versionadded:: 3.13 ``_add_alias_``, ``_add_value_alias_`` - -To help keep Python 2 / Python 3 code in sync an :attr:`~Enum._order_` attribute can -be provided. It will be checked against the actual order of the enumeration -and raise an error if the two do not match:: - - >>> class Color(Enum): - ... _order_ = 'RED GREEN BLUE' - ... RED = 1 - ... BLUE = 3 - ... GREEN = 2 - ... - Traceback (most recent call last): - ... - TypeError: member order does not match _order_: - ['RED', 'BLUE', 'GREEN'] - ['RED', 'GREEN', 'BLUE'] - -.. note:: - - In Python 2 code the :attr:`~Enum._order_` attribute is necessary as definition - order is lost before it can be recorded. +The supported ``__dunder__`` and ``_sunder_`` names can be found in the :ref:`Enum API documentation `. _Private__names -""""""""""""""" +^^^^^^^^^^^^^^^ :ref:`Private names ` are not converted to enum members, but remain normal attributes. @@ -1042,7 +983,7 @@ but remain normal attributes. ``Enum`` member type -"""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^ Enum members are instances of their enum class, and are normally accessed as ``EnumClass.member``. In certain situations, such as writing custom enum @@ -1055,7 +996,7 @@ recommended. Creating members that are mixed with other data types -""""""""""""""""""""""""""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When subclassing other data types, such as :class:`int` or :class:`str`, with an :class:`Enum`, all values after the ``=`` are passed to that data type's @@ -1069,7 +1010,7 @@ constructor. For example:: Boolean value of ``Enum`` classes and members -""""""""""""""""""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Enum classes that are mixed with non-:class:`Enum` types (such as :class:`int`, :class:`str`, etc.) are evaluated according to the mixed-in @@ -1084,7 +1025,7 @@ Plain :class:`Enum` classes always evaluate as :data:`True`. ``Enum`` classes with methods -""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you give your enum subclass extra methods, like the `Planet`_ class below, those methods will show up in a :func:`dir` of the member, @@ -1097,7 +1038,7 @@ but not of the class:: Combining members of ``Flag`` -""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Iterating over a combination of :class:`Flag` members will only return the members that are comprised of a single bit:: @@ -1117,7 +1058,7 @@ are comprised of a single bit:: ``Flag`` and ``IntFlag`` minutia -"""""""""""""""""""""""""""""""" +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Using the following snippet for our examples:: @@ -1478,6 +1419,7 @@ alias:: behaviors as well as disallowing aliases. If the only desired change is disallowing aliases, the :func:`unique` decorator can be used instead. +.. _multi-value-enum: MultiValueEnum ^^^^^^^^^^^^^^^^^ @@ -1538,8 +1480,8 @@ TimePeriod An example to show the :attr:`~Enum._ignore_` attribute in use:: - >>> from datetime import timedelta - >>> class Period(timedelta, Enum): + >>> import datetime as dt + >>> class Period(dt.timedelta, Enum): ... "different lengths of time" ... _ignore_ = 'Period i' ... Period = vars() diff --git a/Doc/howto/free-threading-extensions.rst b/Doc/howto/free-threading-extensions.rst index 3f6ee517050bd85..43b8ff26ebde1ad 100644 --- a/Doc/howto/free-threading-extensions.rst +++ b/Doc/howto/free-threading-extensions.rst @@ -6,8 +6,8 @@ C API Extension Support for Free Threading ****************************************** -Starting with the 3.13 release, CPython has experimental support for running -with the :term:`global interpreter lock` (GIL) disabled in a configuration +Starting with the 3.13 release, CPython has support for running with +the :term:`global interpreter lock` (GIL) disabled in a configuration called :term:`free threading`. This document describes how to adapt C API extensions to support free threading. @@ -23,6 +23,14 @@ You can use it to enable code that only runs under the free-threaded build:: /* code that only runs in the free-threaded build */ #endif +.. note:: + + On Windows, this macro is not defined automatically, but must be specified + to the compiler when building. The :func:`sysconfig.get_config_var` function + can be used to determine whether the current running interpreter had the + macro defined. + + Module Initialization ===================== @@ -153,6 +161,8 @@ that return :term:`strong references `. +===================================+===================================+ | :c:func:`PyList_GetItem` | :c:func:`PyList_GetItemRef` | +-----------------------------------+-----------------------------------+ +| :c:func:`PyList_GET_ITEM` | :c:func:`PyList_GetItemRef` | ++-----------------------------------+-----------------------------------+ | :c:func:`PyDict_GetItem` | :c:func:`PyDict_GetItemRef` | +-----------------------------------+-----------------------------------+ | :c:func:`PyDict_GetItemWithError` | :c:func:`PyDict_GetItemRef` | @@ -193,7 +203,7 @@ Memory Allocation APIs Python's memory management C API provides functions in three different :ref:`allocation domains `: "raw", "mem", and "object". For thread-safety, the free-threaded build requires that only Python objects -are allocated using the object domain, and that all Python object are +are allocated using the object domain, and that all Python objects are allocated using that domain. This differs from the prior Python versions, where this was only a best practice and not a hard requirement. @@ -334,12 +344,12 @@ This means you cannot rely on nested critical sections to lock multiple objects at once, as the inner critical section may suspend the outer ones. Instead, use :c:macro:`Py_BEGIN_CRITICAL_SECTION2` to lock two objects simultaneously. -Note that the locks described above are only :c:type:`!PyMutex` based locks. +Note that the locks described above are only :c:type:`PyMutex` based locks. The critical section implementation does not know about or affect other locking mechanisms that might be in use, like POSIX mutexes. Also note that while -blocking on any :c:type:`!PyMutex` causes the critical sections to be +blocking on any :c:type:`PyMutex` causes the critical sections to be suspended, only the mutexes that are part of the critical sections are -released. If :c:type:`!PyMutex` is used without a critical section, it will +released. If :c:type:`PyMutex` is used without a critical section, it will not be released and therefore does not get the same deadlock avoidance. Important Considerations @@ -385,10 +395,9 @@ C API extensions need to be built specifically for the free-threaded build. The wheels, shared libraries, and binaries are indicated by a ``t`` suffix. * `pypa/manylinux `_ supports the - free-threaded build, with the ``t`` suffix, such as ``python3.13t``. -* `pypa/cibuildwheel `_ supports the - free-threaded build if you set - `CIBW_FREE_THREADED_SUPPORT `_. + free-threaded build, with the ``t`` suffix, such as ``python3.14t``. +* `pypa/cibuildwheel `_ supports + building wheels for the free-threaded build of Python 3.14 and newer. Limited C API and Stable ABI ............................ diff --git a/Doc/howto/free-threading-python.rst b/Doc/howto/free-threading-python.rst index f7a894ac2cd78e1..53bea1db191d76f 100644 --- a/Doc/howto/free-threading-python.rst +++ b/Doc/howto/free-threading-python.rst @@ -1,18 +1,19 @@ .. _freethreading-python-howto: -********************************************** -Python experimental support for free threading -********************************************** +********************************* +Python support for free threading +********************************* -Starting with the 3.13 release, CPython has experimental support for a build of +Starting with the 3.13 release, CPython has support for a build of Python called :term:`free threading` where the :term:`global interpreter lock` (GIL) is disabled. Free-threaded execution allows for full utilization of the available processing power by running threads in parallel on available CPU cores. While not all software will benefit from this automatically, programs designed with threading in mind will run faster on multi-core hardware. -**The free-threaded mode is experimental** and work is ongoing to improve it: -expect some bugs and a substantial single-threaded performance hit. +Some third-party packages, in particular ones +with an :term:`extension module`, may not be ready for use in a +free-threaded build, and will re-enable the :term:`GIL`. This document describes the implications of free threading for Python code. See :ref:`freethreading-extensions-howto` for information on @@ -32,7 +33,7 @@ optionally support installing free-threaded Python binaries. The installers are available at https://www.python.org/downloads/. For information on other platforms, see the `Installing a Free-Threaded Python -`_, a +`_, a community-maintained installation guide for installing free-threaded Python. When building CPython from source, the :option:`--disable-gil` configure option @@ -43,7 +44,7 @@ Identifying free-threaded Python ================================ To check if the current interpreter supports free-threading, :option:`python -VV <-V>` -and :data:`sys.version` contain "experimental free-threading build". +and :data:`sys.version` contain "free-threading build". The new :func:`sys._is_gil_enabled` function can be used to check whether the GIL is actually disabled in the running process. @@ -98,60 +99,42 @@ This section describes known limitations of the free-threaded CPython build. Immortalization --------------- -The free-threaded build of the 3.13 release makes some objects :term:`immortal`. +In the free-threaded build, some objects are :term:`immortal`. Immortal objects are not deallocated and have reference counts that are never modified. This is done to avoid reference count contention that would prevent efficient multi-threaded scaling. -An object will be made immortal when a new thread is started for the first time -after the main thread is running. The following objects are immortalized: +As of the 3.14 release, immortalization is limited to: -* :ref:`function ` objects declared at the module level -* :ref:`method ` descriptors -* :ref:`code ` objects -* :term:`module` objects and their dictionaries -* :ref:`classes ` (type objects) - -Because immortal objects are never deallocated, applications that create many -objects of these types may see increased memory usage. This is expected to be -addressed in the 3.14 release. - -Additionally, numeric and string literals in the code as well as strings -returned by :func:`sys.intern` are also immortalized. This behavior is -expected to remain in the 3.14 free-threaded build. +* Code constants: numeric literals, string literals, and tuple literals + composed of other constants. +* Strings interned by :func:`sys.intern`. Frame objects ------------- -It is not safe to access :ref:`frame ` objects from other -threads and doing so may cause your program to crash . This means that -:func:`sys._current_frames` is generally not safe to use in a free-threaded -build. Functions like :func:`inspect.currentframe` and :func:`sys._getframe` -are generally safe as long as the resulting frame object is not passed to -another thread. +It is not safe to access :attr:`frame.f_locals` from a :ref:`frame ` +object if that frame is currently executing in another thread, and doing so may +crash the interpreter. + Iterators --------- -Sharing the same iterator object between multiple threads is generally not -safe and threads may see duplicate or missing elements when iterating or crash -the interpreter. +It is generally not thread-safe to access the same iterator object from +multiple threads concurrently, and threads may see duplicate or missing +elements. Single-threaded performance --------------------------- The free-threaded build has additional overhead when executing Python code -compared to the default GIL-enabled build. In 3.13, this overhead is about -40% on the `pyperformance `_ suite. -Programs that spend most of their time in C extensions or I/O will see -less of an impact. The largest impact is because the specializing adaptive -interpreter (:pep:`659`) is disabled in the free-threaded build. We expect -to re-enable it in a thread-safe way in the 3.14 release. This overhead is -expected to be reduced in upcoming Python release. We are aiming for an -overhead of 10% or less on the pyperformance suite compared to the default -GIL-enabled build. +compared to the default GIL-enabled build. The amount of overhead depends +on the workload and hardware. On the pyperformance benchmark suite, the +average overhead ranges from about 1% on macOS aarch64 to 8% on x86-64 Linux +systems. Behavioral changes @@ -182,3 +165,132 @@ to false. If the flag is true then the :class:`warnings.catch_warnings` context manager uses a context variable for warning filters. If the flag is false then :class:`~warnings.catch_warnings` modifies the global filters list, which is not thread-safe. See the :mod:`warnings` module for more details. + + +Increased memory usage +---------------------- + +The free-threaded build will typically use more memory compared to the default +build. There are multiple reasons for this, mostly due to design decisions. + + +All interned strings are immortal +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For modern Python versions (since version 2.3), interning a string (e.g. with +:func:`sys.intern`) does not cause it to become immortal. Instead, if the last +reference to that string disappears, it will be removed from the interned +string table. This is not the case for the free-threaded build and any interned +string will become immortal, surviving until interpreter shutdown. + + +Non-GC objects have a larger object header +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The free-threaded build uses a different :c:type:`PyObject` structure. Instead +of having the GC related information allocated before the :c:type:`PyObject` +structure, like in the default build, the GC related info is part of the normal +object header. For example, on the AMD64 platform, ``None`` uses 32 bytes on +the free-threaded build vs 16 bytes for the default build. GC objects (such as +dicts and lists) are the same size for both builds since the free-threaded +build does not use additional space for the GC info. + + +QSBR can delay freeing of memory +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In order to safely implement lock-free data structures, a safe memory +reclamation (SMR) scheme is used, known as quiescent state-based reclamation +(QSBR). This means that the memory backing data structures allowing lock-free +access will use QSBR, which defers the free operation, rather than immediately +freeing the memory. Two examples of these data structures are the list object +and the dictionary keys object. See ``InternalDocs/qsbr.md`` in the CPython +source tree for more details on how QSBR is implemented. Running +:func:`gc.collect` should cause all memory being held by QSBR to be actually +freed. Note that even when QSBR frees the memory, the underlying memory +allocator may not immediately return that memory to the OS and so the resident +set size (RSS) of the process might not decrease. + + +mimalloc allocator vs pymalloc +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The default build will normally use the "pymalloc" memory allocator for small +allocations (512 bytes or smaller). The free-threaded build does not use +pymalloc and allocates all Python objects using the "mimalloc" allocator. The +pymalloc allocator has the following properties that help keep memory usage +low: small per-allocated-block overhead, effective memory fragmentation +prevention, and quick return of free memory to the operating system. The +mimalloc allocator does quite well in these respects as well but can have some +more overhead. + +In the free-threaded build, mimalloc manages memory in a number of separate +heaps (currently four). For example, all GC supporting objects are allocated +from their own heap. Using separate heaps means that free memory in one heap +cannot be used for an allocation that uses another heap. Also, some heaps are +configured to use QSBR (quiescent-state based reclamation) when freeing the +memory that backs up the heap (known as "pages" in mimalloc terminology). The +use of QSBR creates a delay between all memory blocks for a page being freed +and the memory page being released, either for new allocations or back to the +OS. + +The mimalloc allocator also defers returning freed memory back to the OS. You +can reduce that delay by setting the environment variable +:envvar:`!MIMALLOC_PURGE_DELAY` to ``0``. Note that this will likely reduce +the performance of the allocator. + + +Free-threaded reference counting can cause objects to live longer +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In the default build, when an object's reference count reaches zero, it is +normally deallocated. The free-threaded build uses "biased reference +counting", with a fast-path for objects "owned" by the current thread and a +slow path for other objects. See :pep:`703` for additional details. Any time +an object's reference count ends up in a "queued" state, deallocation can be +deferred. The queued state is cleared from the "eval breaker" section of the +bytecode evaluator. + +The free-threaded build also allows a different mode of reference counting, +known as "deferred reference counting". This mode is enabled by setting a flag +on a per-object basis. Deferred reference counting is enabled for the +following types: + +* module objects +* module top-level functions +* class methods defined in the class scope +* descriptor objects +* thread-local objects, created by :class:`threading.local` + +When deferred reference counting is enabled, references from Python function +stacks are not added to the reference count. This scheme reduces the overhead +of reference counting, especially for objects used from multiple threads. +Because the stack references are not counted, objects with deferred reference +counting are not immediately freed when their internal reference count goes to +zero. Instead, they are examined by the next GC run and, if no stack +references to them are found, they are freed. This means these objects are +freed by the GC and not when their reference count goes to zero, as is typical. + + +Per-thread reference counting can delay freeing objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To avoid contention on the reference count fields of frequently shared +objects, the free-threaded build also uses "per-thread reference counting" +for a few selected object types. Rather than updating a single shared +reference count, each thread maintains its own local reference count array, +indexed by a unique id assigned to the object. The true reference count is +only computed by summing the per-thread counts when the object's local +count drops to zero. Per-thread reference counting is currently used for: + +* heap type objects (classes created in Python) +* code objects +* the ``__dict__`` of module objects + +Because the per-thread counts must be merged back to the object before it +can be deallocated, objects using per-thread reference counting are +typically freed later than they would be in the default build. In +particular, such an object is usually not freed until the thread that +referenced it reaches a safe point (for example, in the "eval breaker" +section of the bytecode evaluator) or exits. Running :func:`gc.collect` +will merge the per-thread counts and allow these objects to be freed. diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst index 1f0608fb0fc53f0..b78be3bbfbfed97 100644 --- a/Doc/howto/functional.rst +++ b/Doc/howto/functional.rst @@ -4,7 +4,7 @@ Functional Programming HOWTO ******************************** -:Author: A. M. Kuchling +:Author: \A. M. Kuchling :Release: 0.32 In this document, we'll take a tour of Python's features suitable for @@ -602,7 +602,7 @@ generators: raise an exception inside the generator; the exception is raised by the ``yield`` expression where the generator's execution is paused. -* :meth:`~generator.close` raises a :exc:`GeneratorExit` exception inside the +* :meth:`~generator.close` sends a :exc:`GeneratorExit` exception to the generator to terminate the iteration. On receiving this exception, the generator's code must either raise :exc:`GeneratorExit` or :exc:`StopIteration`; catching the exception and doing anything else is @@ -1042,7 +1042,7 @@ first calculation. :: >>> functools.reduce(operator.concat, []) Traceback (most recent call last): ... - TypeError: reduce() of empty sequence with no initial value + TypeError: reduce() of empty iterable with no initial value >>> functools.reduce(operator.mul, [1, 2, 3], 1) 6 >>> functools.reduce(operator.mul, [], 1) @@ -1217,7 +1217,7 @@ flow inside a program. The book uses Scheme for its examples, but many of the design approaches described in these chapters are applicable to functional-style Python code. -https://www.defmacro.org/ramblings/fp.html: A general introduction to functional +https://defmacro.org/2006/06/19/fp.html: A general introduction to functional programming that uses Java examples and has a lengthy historical introduction. https://en.wikipedia.org/wiki/Functional_programming: General Wikipedia entry diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst index f350141004c2db4..81fc7e63f35bd79 100644 --- a/Doc/howto/index.rst +++ b/Doc/howto/index.rst @@ -1,3 +1,5 @@ +.. _how-tos: + *************** Python HOWTOs *************** @@ -11,6 +13,7 @@ Python Library Reference. :maxdepth: 1 :hidden: + a-conceptual-overview-of-asyncio.rst cporting.rst curses.rst descriptor.rst @@ -38,6 +41,7 @@ Python Library Reference. General: +* :ref:`a-conceptual-overview-of-asyncio` * :ref:`annotations-howto` * :ref:`argparse-tutorial` * :ref:`descriptorhowto` diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst index 6e03ef20a21fa3f..06c1ae40da5e67e 100644 --- a/Doc/howto/instrumentation.rst +++ b/Doc/howto/instrumentation.rst @@ -269,6 +269,8 @@ should instead read: (assuming a :ref:`debug build ` of CPython 3.6) +.. _static-markers: + Available static markers ------------------------ @@ -339,6 +341,84 @@ Available static markers .. versionadded:: 3.8 +C Entry Points +^^^^^^^^^^^^^^ + +To simplify triggering of DTrace markers, Python's C API comes with a number +of helper functions that mirror each static marker. On builds of Python without +DTrace enabled, these do nothing. + +In general, it is not necessary to call these yourself, as Python will do +it for you. + +.. list-table:: + :widths: 50 25 25 + :header-rows: 1 + + * * C API Function + * Static Marker + * Notes + * * .. c:function:: void PyDTrace_LINE(const char *arg0, const char *arg1, int arg2) + * :c:func:`!line` + * + * * .. c:function:: void PyDTrace_FUNCTION_ENTRY(const char *arg0, const char *arg1, int arg2) + * :c:func:`!function__entry` + * + * * .. c:function:: void PyDTrace_FUNCTION_RETURN(const char *arg0, const char *arg1, int arg2) + * :c:func:`!function__return` + * + * * .. c:function:: void PyDTrace_GC_START(int arg0) + * :c:func:`!gc__start` + * + * * .. c:function:: void PyDTrace_GC_DONE(Py_ssize_t arg0) + * :c:func:`!gc__done` + * + * * .. c:function:: void PyDTrace_INSTANCE_NEW_START(int arg0) + * :c:func:`!instance__new__start` + * Not used by Python + * * .. c:function:: void PyDTrace_INSTANCE_NEW_DONE(int arg0) + * :c:func:`!instance__new__done` + * Not used by Python + * * .. c:function:: void PyDTrace_INSTANCE_DELETE_START(int arg0) + * :c:func:`!instance__delete__start` + * Not used by Python + * * .. c:function:: void PyDTrace_INSTANCE_DELETE_DONE(int arg0) + * :c:func:`!instance__delete__done` + * Not used by Python + * * .. c:function:: void PyDTrace_IMPORT_FIND_LOAD_START(const char *arg0) + * :c:func:`!import__find__load__start` + * + * * .. c:function:: void PyDTrace_IMPORT_FIND_LOAD_DONE(const char *arg0, int arg1) + * :c:func:`!import__find__load__done` + * + * * .. c:function:: void PyDTrace_AUDIT(const char *arg0, void *arg1) + * :c:func:`!audit` + * + + +C Probing Checks +^^^^^^^^^^^^^^^^ + +.. c:function:: int PyDTrace_LINE_ENABLED(void) +.. c:function:: int PyDTrace_FUNCTION_ENTRY_ENABLED(void) +.. c:function:: int PyDTrace_FUNCTION_RETURN_ENABLED(void) +.. c:function:: int PyDTrace_GC_START_ENABLED(void) +.. c:function:: int PyDTrace_GC_DONE_ENABLED(void) +.. c:function:: int PyDTrace_INSTANCE_NEW_START_ENABLED(void) +.. c:function:: int PyDTrace_INSTANCE_NEW_DONE_ENABLED(void) +.. c:function:: int PyDTrace_INSTANCE_DELETE_START_ENABLED(void) +.. c:function:: int PyDTrace_INSTANCE_DELETE_DONE_ENABLED(void) +.. c:function:: int PyDTrace_IMPORT_FIND_LOAD_START_ENABLED(void) +.. c:function:: int PyDTrace_IMPORT_FIND_LOAD_DONE_ENABLED(void) +.. c:function:: int PyDTrace_AUDIT_ENABLED(void) + + All calls to ``PyDTrace`` functions must be guarded by a call to one + of these functions. This allows Python to minimize performance impact + when probing is disabled. + + On builds without DTrace enabled, these functions do nothing and return + ``0``. + SystemTap Tapsets ----------------- diff --git a/Doc/howto/isolating-extensions.rst b/Doc/howto/isolating-extensions.rst index a636e06bda83441..6092c75f48fdef0 100644 --- a/Doc/howto/isolating-extensions.rst +++ b/Doc/howto/isolating-extensions.rst @@ -168,7 +168,7 @@ possible, consider explicit locking. If it is necessary to use process-global state, the simplest way to avoid issues with multiple interpreters is to explicitly prevent a module from being loaded more than once per process—see -`Opt-Out: Limiting to One Module Object per Process`_. +:ref:`isolating-extensions-optout`. Managing Per-Module State @@ -207,6 +207,8 @@ An example of a module with per-module state is currently available as example module initialization shown at the bottom of the file. +.. _isolating-extensions-optout: + Opt-Out: Limiting to One Module Object per Process -------------------------------------------------- @@ -215,21 +217,36 @@ multiple interpreters correctly. If this is not yet the case for your module, you can explicitly make your module loadable only once per process. For example:: + // A process-wide flag static int loaded = 0; + // Mutex to provide thread safety (only needed for free-threaded Python) + static PyMutex modinit_mutex = {0}; + static int exec_module(PyObject* module) { + PyMutex_Lock(&modinit_mutex); if (loaded) { + PyMutex_Unlock(&modinit_mutex); PyErr_SetString(PyExc_ImportError, "cannot load module more than once per process"); return -1; } loaded = 1; + PyMutex_Unlock(&modinit_mutex); // ... rest of initialization } +If your module's :c:member:`PyModuleDef.m_clear` function is able to prepare +for future re-initialization, it should clear the ``loaded`` flag. +In this case, your module won't support multiple instances existing +*concurrently*, but it will, for example, support being loaded after +Python runtime shutdown (:c:func:`Py_FinalizeEx`) and re-initialization +(:c:func:`Py_Initialize`). + + Module State Access from Functions ---------------------------------- @@ -336,7 +353,7 @@ garbage collection protocol. That is, heap types should: - Have the :c:macro:`Py_TPFLAGS_HAVE_GC` flag. -- Define a traverse function using ``Py_tp_traverse``, which +- Define a traverse function using :c:data:`Py_tp_traverse`, which visits the type (e.g. using ``Py_VISIT(Py_TYPE(self))``). Please refer to the documentation of @@ -436,7 +453,7 @@ Avoiding ``PyObject_New`` GC-tracked objects need to be allocated using GC-aware functions. -If you use use :c:func:`PyObject_New` or :c:func:`PyObject_NewVar`: +If you use :c:func:`PyObject_New` or :c:func:`PyObject_NewVar`: - Get and call type's :c:member:`~PyTypeObject.tp_alloc` slot, if possible. That is, replace ``TYPE *o = PyObject_New(TYPE, typeobj)`` with:: @@ -609,8 +626,7 @@ Open Issues Several issues around per-module state and heap types are still open. -Discussions about improving the situation are best held on the `capi-sig -mailing list `__. +Discussions about improving the situation are best held on the `discuss forum under c-api tag `__. Per-Class Scope diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 7d64a02358adb32..39a7b41c755097e 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -229,7 +229,7 @@ messages should not. Here's how you can achieve this:: # tell the handler to use this format console.setFormatter(formatter) # add the handler to the root logger - logging.getLogger('').addHandler(console) + logging.getLogger().addHandler(console) # Now, we can log to the root logger, or any other logger. First the root... logging.info('Jackdaws love my big sphinx of quartz.') @@ -650,7 +650,7 @@ the receiving end. A simple way of doing this is attaching a import logging, logging.handlers - rootLogger = logging.getLogger('') + rootLogger = logging.getLogger() rootLogger.setLevel(logging.DEBUG) socketHandler = logging.handlers.SocketHandler('localhost', logging.handlers.DEFAULT_TCP_LOGGING_PORT) @@ -1549,10 +1549,10 @@ to this (remembering to first import :mod:`concurrent.futures`):: for i in range(10): executor.submit(worker_process, queue, worker_configurer) -Deploying Web applications using Gunicorn and uWSGI +Deploying web applications using Gunicorn and uWSGI ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -When deploying Web applications using `Gunicorn `_ or `uWSGI +When deploying web applications using `Gunicorn `_ or `uWSGI `_ (or similar), multiple worker processes are created to handle client requests. In such environments, avoid creating file-based handlers directly in your web application. Instead, use a @@ -1960,7 +1960,7 @@ Subclass ``QueueListener`` class NNGSocketListener(logging.handlers.QueueListener): def __init__(self, uri, /, *handlers, **kwargs): - # Have a timeout for interruptability, and open a + # Have a timeout for interruptibility, and open a # subscriber socket socket = pynng.Sub0(listen=uri, recv_timeout=500) # The b'' subscription matches all topics @@ -3619,7 +3619,6 @@ detailed information. .. code-block:: python3 - import datetime import logging import random import sys @@ -3854,7 +3853,7 @@ Logging to syslog with RFC5424 support Although :rfc:`5424` dates from 2009, most syslog servers are configured by default to use the older :rfc:`3164`, which hails from 2001. When ``logging`` was added to Python in 2003, it supported the earlier (and only existing) protocol at the time. Since -RFC5424 came out, as there has not been widespread deployment of it in syslog +RFC 5424 came out, as there has not been widespread deployment of it in syslog servers, the :class:`~logging.handlers.SysLogHandler` functionality has not been updated. @@ -3862,7 +3861,7 @@ RFC 5424 contains some useful features such as support for structured data, and need to be able to log to a syslog server with support for it, you can do so with a subclassed handler which looks something like this:: - import datetime + import datetime as dt import logging.handlers import re import socket @@ -3880,7 +3879,7 @@ subclassed handler which looks something like this:: def format(self, record): version = 1 - asctime = datetime.datetime.fromtimestamp(record.created).isoformat() + asctime = dt.datetime.fromtimestamp(record.created).isoformat() m = self.tz_offset.match(time.strftime('%z')) has_offset = False if m and time.timezone: @@ -4078,6 +4077,104 @@ lines. With this approach, you get better output: WARNING:demo: 1/0 WARNING:demo:ZeroDivisionError: division by zero +How to uniformly handle newlines in logging output +-------------------------------------------------- + +Usually, messages that are logged (say to console or file) consist of a single +line of text. However, sometimes there is a need to handle messages with +multiple lines - whether because a logging format string contains newlines, or +logged data contains newlines. If you want to handle such messages uniformly, so +that each line in the logged message appears uniformly formatted as if it was +logged separately, you can do this using a handler mixin, as in the following +snippet: + +.. code-block:: python + + # Assume this is in a module mymixins.py + import copy + + class MultilineMixin: + def emit(self, record): + s = record.getMessage() + if '\n' not in s: + super().emit(record) + else: + lines = s.splitlines() + rec = copy.copy(record) + rec.args = None + for line in lines: + rec.msg = line + super().emit(rec) + +You can use the mixin as in the following script: + +.. code-block:: python + + import logging + + from mymixins import MultilineMixin + + logger = logging.getLogger(__name__) + + class StreamHandler(MultilineMixin, logging.StreamHandler): + pass + + if __name__ == '__main__': + logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(levelname)-9s %(message)s', + handlers = [StreamHandler()]) + logger.debug('Single line') + logger.debug('Multiple lines:\nfool me once ...') + logger.debug('Another single line') + logger.debug('Multiple lines:\n%s', 'fool me ...\ncan\'t get fooled again') + +The script, when run, prints something like: + +.. code-block:: text + + 2025-07-02 13:54:47,234 DEBUG Single line + 2025-07-02 13:54:47,234 DEBUG Multiple lines: + 2025-07-02 13:54:47,234 DEBUG fool me once ... + 2025-07-02 13:54:47,234 DEBUG Another single line + 2025-07-02 13:54:47,234 DEBUG Multiple lines: + 2025-07-02 13:54:47,234 DEBUG fool me ... + 2025-07-02 13:54:47,234 DEBUG can't get fooled again + +If, on the other hand, you are concerned about `log injection +`_, you can use a +formatter which escapes newlines, as per the following example: + +.. code-block:: python + + import logging + + logger = logging.getLogger(__name__) + + class EscapingFormatter(logging.Formatter): + def format(self, record): + s = super().format(record) + return s.replace('\n', r'\n') + + if __name__ == '__main__': + h = logging.StreamHandler() + h.setFormatter(EscapingFormatter('%(asctime)s %(levelname)-9s %(message)s')) + logging.basicConfig(level=logging.DEBUG, handlers = [h]) + logger.debug('Single line') + logger.debug('Multiple lines:\nfool me once ...') + logger.debug('Another single line') + logger.debug('Multiple lines:\n%s', 'fool me ...\ncan\'t get fooled again') + +You can, of course, use whatever escaping scheme makes the most sense for you. +The script, when run, should produce output like this: + +.. code-block:: text + + 2025-07-09 06:47:33,783 DEBUG Single line + 2025-07-09 06:47:33,783 DEBUG Multiple lines:\nfool me once ... + 2025-07-09 06:47:33,783 DEBUG Another single line + 2025-07-09 06:47:33,783 DEBUG Multiple lines:\nfool me ...\ncan't get fooled again + +Escaping behaviour can't be the stdlib default , as it would break backwards +compatibility. .. patterns-to-avoid: diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 2982cf88bf97b4d..454e2f4930e724d 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -28,7 +28,7 @@ When to use logging ^^^^^^^^^^^^^^^^^^^ You can access logging functionality by creating a logger via ``logger = -getLogger(__name__)``, and then calling the logger's :meth:`~Logger.debug`, +logging.getLogger(__name__)``, and then calling the logger's :meth:`~Logger.debug`, :meth:`~Logger.info`, :meth:`~Logger.warning`, :meth:`~Logger.error` and :meth:`~Logger.critical` methods. To determine when to use logging, and to see which logger methods to use when, see the table below. It states, for each of a @@ -302,10 +302,10 @@ reading the following sections. If you're ready for that, grab some of your favourite beverage and carry on. If your logging needs are simple, then use the above examples to incorporate -logging into your own scripts, and if you run into problems or don't -understand something, please post a question on the comp.lang.python Usenet -group (available at https://groups.google.com/g/comp.lang.python) and you -should receive help before too long. +logging into your own scripts, and if you run into problems or don't understand +something, please post a question in the Help category of the `Python +discussion forum `_ and you should receive +help before too long. Still here? You can carry on reading the next few sections, which provide a slightly more advanced/in-depth tutorial than the basic one above. After that, diff --git a/Doc/howto/mro.rst b/Doc/howto/mro.rst index 0872bedcd3a2d34..72cc311e30d669c 100644 --- a/Doc/howto/mro.rst +++ b/Doc/howto/mro.rst @@ -10,7 +10,7 @@ The Python 2.3 Method Resolution Order The Method Resolution Order discussed here was *introduced* in Python 2.3, but it is still used in later versions -- including Python 3. -By `Michele Simionato `__. +By `Michele Simionato `__. :Abstract: diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst index e543f6d5657d798..84ec535ca98e973 100644 --- a/Doc/howto/regex.rst +++ b/Doc/howto/regex.rst @@ -1,7 +1,7 @@ .. _regex-howto: **************************** - Regular Expression HOWTO + Regular expression HOWTO **************************** :Author: A.M. Kuchling @@ -47,7 +47,7 @@ Python code to do the processing; while Python code will be slower than an elaborate regular expression, it will also probably be more understandable. -Simple Patterns +Simple patterns =============== We'll start by learning about the simplest possible regular expressions. Since @@ -59,7 +59,7 @@ expressions (deterministic and non-deterministic finite automata), you can refer to almost any textbook on writing compilers. -Matching Characters +Matching characters ------------------- Most letters and characters will simply match themselves. For example, the @@ -159,7 +159,7 @@ match even a newline. ``.`` is often used where you want to match "any character". -Repeating Things +Repeating things ---------------- Being able to match varying sets of characters is the first thing regular @@ -210,7 +210,7 @@ this RE against the string ``'abcbd'``. | | | ``[bcd]*`` is only matching | | | | ``bc``. | +------+-----------+---------------------------------+ -| 6 | ``abcb`` | Try ``b`` again. This time | +| 7 | ``abcb`` | Try ``b`` again. This time | | | | the character at the | | | | current position is ``'b'``, so | | | | it succeeds. | @@ -255,7 +255,7 @@ is equivalent to ``+``, and ``{0,1}`` is the same as ``?``. It's better to use to read. -Using Regular Expressions +Using regular expressions ========================= Now that we've looked at some simple regular expressions, how do we actually use @@ -264,7 +264,7 @@ expression engine, allowing you to compile REs into objects and then perform matches with them. -Compiling Regular Expressions +Compiling regular expressions ----------------------------- Regular expressions are compiled into pattern objects, which have @@ -295,7 +295,7 @@ disadvantage which is the topic of the next section. .. _the-backslash-plague: -The Backslash Plague +The backslash plague -------------------- As stated earlier, regular expressions use the backslash character (``'\'``) to @@ -335,7 +335,7 @@ expressions will often be written in Python code using this raw string notation. In addition, special escape sequences that are valid in regular expressions, but not valid as Python string literals, now result in a -:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`, +:exc:`SyntaxWarning` and will eventually become a :exc:`SyntaxError`, which means the sequences will be invalid if raw string notation or escaping the backslashes isn't used. @@ -351,7 +351,7 @@ the backslashes isn't used. +-------------------+------------------+ -Performing Matches +Performing matches ------------------ Once you have an object representing a compiled regular expression, what do you @@ -369,10 +369,10 @@ for a complete listing. | | location where this RE matches. | +------------------+-----------------------------------------------+ | ``findall()`` | Find all substrings where the RE matches, and | -| | returns them as a list. | +| | return them as a list. | +------------------+-----------------------------------------------+ | ``finditer()`` | Find all substrings where the RE matches, and | -| | returns them as an :term:`iterator`. | +| | return them as an :term:`iterator`. | +------------------+-----------------------------------------------+ :meth:`~re.Pattern.match` and :meth:`~re.Pattern.search` return ``None`` if no match can be found. If @@ -473,7 +473,7 @@ Two pattern methods return all of the matches for a pattern. The ``r`` prefix, making the literal a raw string literal, is needed in this example because escape sequences in a normal "cooked" string literal that are not recognized by Python, as opposed to regular expressions, now result in a -:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`. See +:exc:`SyntaxWarning` and will eventually become a :exc:`SyntaxError`. See :ref:`the-backslash-plague`. :meth:`~re.Pattern.findall` has to create the entire list before it can be returned as the @@ -491,7 +491,7 @@ result. The :meth:`~re.Pattern.finditer` method returns a sequence of (29, 31) -Module-Level Functions +Module-level functions ---------------------- You don't have to create a pattern object and call its methods; the @@ -518,7 +518,7 @@ Outside of loops, there's not much difference thanks to the internal cache. -Compilation Flags +Compilation flags ----------------- .. currentmodule:: re @@ -642,7 +642,7 @@ of each one. whitespace is in a character class or preceded by an unescaped backslash; this lets you organize and indent the RE more clearly. This flag also lets you put comments within a RE that will be ignored by the engine; comments are marked by - a ``'#'`` that's neither in a character class or preceded by an unescaped + a ``'#'`` that's neither in a character class nor preceded by an unescaped backslash. For example, here's a RE that uses :const:`re.VERBOSE`; see how much easier it @@ -669,7 +669,7 @@ of each one. to understand than the version using :const:`re.VERBOSE`. -More Pattern Power +More pattern power ================== So far we've only covered a part of the features of regular expressions. In @@ -679,7 +679,7 @@ retrieve portions of the text that was matched. .. _more-metacharacters: -More Metacharacters +More metacharacters ------------------- There are some metacharacters that we haven't covered yet. Most of them will be @@ -875,7 +875,7 @@ Backreferences like this aren't often useful for just searching through a string find out that they're *very* useful when performing string substitutions. -Non-capturing and Named Groups +Non-capturing and named groups ------------------------------ Elaborate REs may use many groups, both to capture substrings of interest, and @@ -979,7 +979,7 @@ current point. The regular expression for finding doubled words, 'the the' -Lookahead Assertions +Lookahead assertions -------------------- Another zero-width assertion is the lookahead assertion. Lookahead assertions @@ -1016,7 +1016,9 @@ extension. This regular expression matches ``foo.bar`` and Now, consider complicating the problem a bit; what if you want to match filenames where the extension is not ``bat``? Some incorrect attempts: -``.*[.][^b].*$`` The first attempt above tries to exclude ``bat`` by requiring +``.*[.][^b].*$`` + +The first attempt above tries to exclude ``bat`` by requiring that the first character of the extension is not a ``b``. This is wrong, because the pattern also doesn't match ``foo.bar``. @@ -1043,7 +1045,9 @@ confusing. A negative lookahead cuts through all this confusion: -``.*[.](?!bat$)[^.]*$`` The negative lookahead means: if the expression ``bat`` +``.*[.](?!bat$)[^.]*$`` + +The negative lookahead means: if the expression ``bat`` doesn't match at this point, try the rest of the pattern; if ``bat$`` does match, the whole pattern will fail. The trailing ``$`` is required to ensure that something like ``sample.batch``, where the extension only starts with @@ -1057,7 +1061,7 @@ end in either ``bat`` or ``exe``: ``.*[.](?!bat$|exe$)[^.]*$`` -Modifying Strings +Modifying strings ================= Up to this point, we've simply performed searches against a static string. @@ -1079,7 +1083,7 @@ using the following pattern methods: +------------------+-----------------------------------------------+ -Splitting Strings +Splitting strings ----------------- The :meth:`~re.Pattern.split` method of a pattern splits a string apart @@ -1133,7 +1137,7 @@ argument, but is otherwise the same. :: ['Words', 'words, words.'] -Search and Replace +Search and replace ------------------ Another common task is to find all the matches for a pattern, and replace them @@ -1232,7 +1236,7 @@ pattern object as the first parameter, or use embedded modifiers in the pattern string, e.g. ``sub("(?i)b+", "x", "bbbb BBBB")`` returns ``'x x'``. -Common Problems +Common problems =============== Regular expressions are a powerful tool for some applications, but in some ways @@ -1240,7 +1244,7 @@ their behaviour isn't intuitive and at times they don't behave the way you may expect them to. This section will point out some of the most common pitfalls. -Use String Methods +Use string methods ------------------ Sometimes using the :mod:`re` module is a mistake. If you're matching a fixed @@ -1306,7 +1310,7 @@ string and then backtracking to find a match for the rest of the RE. Use :func:`re.search` instead. -Greedy versus Non-Greedy +Greedy versus non-greedy ------------------------ When repeating a regular expression, as in ``a*``, the resulting action is to @@ -1384,9 +1388,9 @@ Feedback ======== Regular expressions are a complicated topic. Did this document help you -understand them? Were there parts that were unclear, or Problems you +understand them? Were there parts that were unclear, or problems you encountered that weren't covered here? If so, please send suggestions for -improvements to the author. +improvements to the :ref:`issue tracker `. The most complete book on regular expressions is almost certainly Jeffrey Friedl's Mastering Regular Expressions, published by O'Reilly. Unfortunately, diff --git a/Doc/howto/remote_debugging.rst b/Doc/howto/remote_debugging.rst index 3adb6ad03e54452..1d5cf24d0628432 100644 --- a/Doc/howto/remote_debugging.rst +++ b/Doc/howto/remote_debugging.rst @@ -3,6 +3,88 @@ Remote debugging attachment protocol ==================================== +This protocol enables external tools to attach to a running CPython process and +execute Python code remotely. + +Most platforms require elevated privileges to attach to another Python process. + +Disabling remote debugging +-------------------------- + +To disable remote debugging support, use any of the following: + +* Set the :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` environment variable to ``1`` before + starting the interpreter. +* Use the :option:`-X disable_remote_debug` command-line option. +* Compile Python with the :option:`--without-remote-debug` build flag. + +.. _permission-requirements: + +Permission requirements +======================= + +Attaching to a running Python process for remote debugging requires elevated +privileges on most platforms. The specific requirements and troubleshooting +steps depend on your operating system: + +.. rubric:: Linux + +The tracer process must have the ``CAP_SYS_PTRACE`` capability or equivalent +privileges. You can only trace processes you own and can signal. Tracing may +fail if the process is already being traced, or if it is running with +set-user-ID or set-group-ID. Security modules like Yama may further restrict +tracing. + +To temporarily relax ptrace restrictions (until reboot), run: + + ``echo 0 | sudo tee /proc/sys/kernel/yama/ptrace_scope`` + +.. note:: + + Disabling ``ptrace_scope`` reduces system hardening and should only be done + in trusted environments. + +If running inside a container, use ``--cap-add=SYS_PTRACE`` or +``--privileged``, and run as root if needed. + +Try re-running the command with elevated privileges: + + ``sudo -E !!`` + + +.. rubric:: macOS + +To attach to another process, you typically need to run your debugging tool +with elevated privileges. This can be done by using ``sudo`` or running as +root. + +Even when attaching to processes you own, macOS may block debugging unless +the debugger is run with root privileges due to system security restrictions. + + +.. rubric:: Windows + +To attach to another process, you usually need to run your debugging tool +with administrative privileges. Start the command prompt or terminal as +Administrator. + +Some processes may still be inaccessible even with Administrator rights, +unless you have the ``SeDebugPrivilege`` privilege enabled. + +To resolve file or folder access issues, adjust the security permissions: + + 1. Right-click the file or folder and select **Properties**. + 2. Go to the **Security** tab to view users and groups with access. + 3. Click **Edit** to modify permissions. + 4. Select your user account. + 5. In **Permissions**, check **Read** or **Full control** as needed. + 6. Click **Apply**, then **OK** to confirm. + + +.. note:: + + Ensure you've satisfied all :ref:`permission-requirements` before proceeding. + This section describes the low-level protocol that enables external tools to inject and execute a Python script within a running CPython process. @@ -374,13 +456,13 @@ To locate a thread: reliable thread to target. 3. Optionally, use the offset ``interpreter_state.threads_head`` to iterate -through the linked list of all thread states. Each ``PyThreadState`` structure -contains a ``native_thread_id`` field, which may be compared to a target thread -ID to find a specific thread. + through the linked list of all thread states. Each ``PyThreadState`` + structure contains a ``native_thread_id`` field, which may be compared to + a target thread ID to find a specific thread. -1. Once a valid ``PyThreadState`` has been found, its address can be used in -later steps of the protocol, such as writing debugger control fields and -scheduling execution. +4. Once a valid ``PyThreadState`` has been found, its address can be used in + later steps of the protocol, such as writing debugger control fields and + scheduling execution. The following is an example implementation that locates the main thread state:: @@ -454,15 +536,15 @@ its fields are defined by the ``_Py_DebugOffsets`` structure and include the following: - ``debugger_script_path``: A fixed-size buffer that holds the full path to a - Python source file (``.py``). This file must be accessible and readable by - the target process when execution is triggered. + Python source file (``.py``). This file must be accessible and readable by + the target process when execution is triggered. - ``debugger_pending_call``: An integer flag. Setting this to ``1`` tells the - interpreter that a script is ready to be executed. + interpreter that a script is ready to be executed. - ``eval_breaker``: A field checked by the interpreter during execution. - Setting bit 5 (``_PY_EVAL_PLEASE_STOP_BIT``, value ``1U << 5``) in this - field causes the interpreter to pause and check for debugger activity. + Setting bit 5 (``_PY_EVAL_PLEASE_STOP_BIT``, value ``1U << 5``) in this + field causes the interpreter to pause and check for debugger activity. To complete the injection, the debugger must perform the following steps: @@ -543,3 +625,57 @@ To inject and execute a Python script in a remote process: 7. Resume the process (if suspended). The script will execute at the next safe evaluation point. +.. _remote-debugging-threat-model: + +Security and threat model +========================= + +The remote debugging protocol relies on the same operating system primitives +used by native debuggers such as GDB and LLDB. Attaching to a process +requires the **same privileges** that those debuggers require, for example +``ptrace`` / Yama LSM on Linux, ``task_for_pid`` on macOS, and +``SeDebugPrivilege`` on Windows. Python does not introduce any new privilege +escalation path; if an attacker already possesses the permissions needed to +attach to a process, they could equally use GDB to read memory or inject +code. + +The following principles define what is, and is not, considered a security +vulnerability in this feature: + +Attaching requires OS-level privileges + On every supported platform the operating system gates cross-process + memory access behind privilege checks (``CAP_SYS_PTRACE``, root, or + administrator rights). A report that demonstrates an issue only after + these privileges have already been obtained is **not** a vulnerability in + CPython, since the OS security boundary was already crossed. + +Crashes or memory errors when reading a compromised process are not vulnerabilities + A tool that reads internal interpreter state from a target process must + trust that memory to be well-formed. If the target process has been + corrupted or is controlled by an attacker, the debugger or profiler may + crash, produce garbage output, or behave unpredictably. This is the same + risk accepted by every ``ptrace``-based debugger. Bugs in this category + (buffer overflows, segmentation faults, or undefined behaviour triggered + by reading corrupted state) are **not** treated as security issues, though + fixes that improve robustness are welcome. + +Vulnerabilities in the target process are not in scope + If the Python process being debugged has already been compromised, the + attacker already controls execution in that process. Demonstrating further + impact from that starting point does not constitute a vulnerability in the + remote debugging protocol. + +When to use ``PYTHON_DISABLE_REMOTE_DEBUG`` +------------------------------------------- + +The environment variable :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` (and the +equivalent :option:`-X disable_remote_debug` flag) allows operators to disable +the in-process side of the protocol as a **defence-in-depth** measure. This +may be useful in hardened or sandboxed deployment environments where no +debugging or profiling of the process is expected and reducing attack surface +is a priority, even though the OS-level privilege checks already prevent +unprivileged access. + +Setting this variable does **not** affect other OS-level debugging interfaces +(``ptrace``, ``/proc``, ``task_for_pid``, etc.), which remain available +according to their own permission models. diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index 254fe7293553535..243cc27bac7025d 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -352,6 +352,8 @@ If you don't include such a comment, the default encoding used will be UTF-8 as already mentioned. See also :pep:`263` for more information. +.. _unicode-properties: + Unicode Properties ------------------ diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 33a2a7ea89ea07a..4e77d2cb407f726 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -15,7 +15,7 @@ Introduction You may also find useful the following article on fetching web resources with Python: - * `Basic Authentication `_ + * `Basic Authentication `__ A tutorial on *Basic Authentication*, with examples in Python. @@ -245,75 +245,27 @@ codes in the 100--299 range indicate success, you will usually only see error codes in the 400--599 range. :attr:`http.server.BaseHTTPRequestHandler.responses` is a useful dictionary of -response codes in that shows all the response codes used by :rfc:`2616`. The -dictionary is reproduced here for convenience :: +response codes that shows all the response codes used by :rfc:`2616`. +An excerpt from the dictionary is shown below :: - # Table mapping response codes to messages; entries have the - # form {code: (shortmessage, longmessage)}. responses = { - 100: ('Continue', 'Request received, please continue'), - 101: ('Switching Protocols', - 'Switching to new protocol; obey Upgrade header'), - - 200: ('OK', 'Request fulfilled, document follows'), - 201: ('Created', 'Document created, URL follows'), - 202: ('Accepted', - 'Request accepted, processing continues off-line'), - 203: ('Non-Authoritative Information', 'Request fulfilled from cache'), - 204: ('No Content', 'Request fulfilled, nothing follows'), - 205: ('Reset Content', 'Clear input form for further input.'), - 206: ('Partial Content', 'Partial content follows.'), - - 300: ('Multiple Choices', - 'Object has several resources -- see URI list'), - 301: ('Moved Permanently', 'Object moved permanently -- see URI list'), - 302: ('Found', 'Object moved temporarily -- see URI list'), - 303: ('See Other', 'Object moved -- see Method and URL list'), - 304: ('Not Modified', - 'Document has not changed since given time'), - 305: ('Use Proxy', - 'You must use proxy specified in Location to access this ' - 'resource.'), - 307: ('Temporary Redirect', - 'Object moved temporarily -- see URI list'), - - 400: ('Bad Request', - 'Bad request syntax or unsupported method'), - 401: ('Unauthorized', - 'No permission -- see authorization schemes'), - 402: ('Payment Required', - 'No payment -- see charging schemes'), - 403: ('Forbidden', - 'Request forbidden -- authorization will not help'), - 404: ('Not Found', 'Nothing matches the given URI'), - 405: ('Method Not Allowed', - 'Specified method is invalid for this server.'), - 406: ('Not Acceptable', 'URI not available in preferred format.'), - 407: ('Proxy Authentication Required', 'You must authenticate with ' - 'this proxy before proceeding.'), - 408: ('Request Timeout', 'Request timed out; try again later.'), - 409: ('Conflict', 'Request conflict.'), - 410: ('Gone', - 'URI no longer exists and has been permanently removed.'), - 411: ('Length Required', 'Client must specify Content-Length.'), - 412: ('Precondition Failed', 'Precondition in headers is false.'), - 413: ('Request Entity Too Large', 'Entity is too large.'), - 414: ('Request-URI Too Long', 'URI is too long.'), - 415: ('Unsupported Media Type', 'Entity body in unsupported format.'), - 416: ('Requested Range Not Satisfiable', - 'Cannot satisfy request range.'), - 417: ('Expectation Failed', - 'Expect condition could not be satisfied.'), - - 500: ('Internal Server Error', 'Server got itself in trouble'), - 501: ('Not Implemented', - 'Server does not support this operation'), - 502: ('Bad Gateway', 'Invalid responses from another server/proxy.'), - 503: ('Service Unavailable', - 'The server cannot process the request due to a high load'), - 504: ('Gateway Timeout', - 'The gateway server did not receive a timely response'), - 505: ('HTTP Version Not Supported', 'Cannot fulfill request.'), + ... + : ('OK', 'Request fulfilled, document follows'), + ... + : ('Forbidden', + 'Request forbidden -- authorization will ' + 'not help'), + : ('Not Found', + 'Nothing matches the given URI'), + ... + : ("I'm a Teapot", + 'Server refuses to brew coffee because ' + 'it is a teapot'), + ... + : ('Service Unavailable', + 'The server cannot process the ' + 'request due to a high load'), + ... } When an error is raised the server responds by returning an HTTP error code diff --git a/Doc/improve-page-nojs.rst b/Doc/improve-page-nojs.rst new file mode 100644 index 000000000000000..91b3a88b95d38b2 --- /dev/null +++ b/Doc/improve-page-nojs.rst @@ -0,0 +1,29 @@ +:orphan: + +**************************** +Improve a documentation page +**************************** + +.. This is the no-javascript version of this page. The one most people + will see (with JavaScript enabled) is improve-page.rst. If you edit + this page, please also edit that one, and vice versa. + +.. only:: html and not epub + +We are always interested to hear ideas about improvements to the documentation. + +.. only:: translation + + If the bug or suggested improvement concerns the translation of this + documentation, open an issue or edit the page in + `translation's repository `_ instead. + +You have a few ways to ask questions or suggest changes: + +- You can start a discussion about the page on the Python discussion forum. + This link will start a topic in the Documentation category: + `New Documentation topic `_. + +- You can open an issue on the Python GitHub issue tracker. This link will + create a new issue with the "docs" label: + `New docs issue `_. diff --git a/Doc/improve-page.rst b/Doc/improve-page.rst new file mode 100644 index 000000000000000..dc89fcb22fbb59c --- /dev/null +++ b/Doc/improve-page.rst @@ -0,0 +1,65 @@ +:orphan: + +**************************** +Improve a documentation page +**************************** + +.. This is the JavaScript-enabled version of this page. Another version + (for those with JavaScript disabled) is improve-page-nojs.rst. If you + edit this page, please also edit that one, and vice versa. + +.. only:: html and not epub + + .. raw:: html + + + +We are always interested to hear ideas about improvements to the documentation. + +You were reading "PAGETITLE" at ``_. The source for that page is on +`GitHub `_. + +.. only:: translation + + If the bug or suggested improvement concerns the translation of this + documentation, open an issue or edit the page in + `translation's repository `_ instead. + +You have a few ways to ask questions or suggest changes: + +- You can start a discussion about the page on the Python discussion forum. + This link will start a pre-populated topic: + `Question about page "PAGETITLE" `_. + +- You can open an issue on the Python GitHub issue tracker. This link will + create a new pre-populated issue: + `Docs: problem with page "PAGETITLE" `_. + +- You can `edit the page on GitHub `_ + to open a pull request and begin the contribution process. diff --git a/Doc/includes/diff.py b/Doc/includes/diff.py index 001619f5f83fc08..bc4bd58ff3e3f11 100644 --- a/Doc/includes/diff.py +++ b/Doc/includes/diff.py @@ -1,4 +1,4 @@ -""" Command line interface to difflib.py providing diffs in four formats: +""" Command-line interface to difflib.py providing diffs in four formats: * ndiff: lists every line and highlights interline changes. * context: highlights clusters of changes in a before/after format. @@ -8,11 +8,11 @@ """ import sys, os, difflib, argparse -from datetime import datetime, timezone +import datetime as dt def file_mtime(path): - t = datetime.fromtimestamp(os.stat(path).st_mtime, - timezone.utc) + t = dt.datetime.fromtimestamp(os.stat(path).st_mtime, + dt.timezone.utc) return t.astimezone().isoformat() def main(): diff --git a/Doc/includes/newtypes/custom.c b/Doc/includes/newtypes/custom.c index 5253f879360210e..039a1a7219349c3 100644 --- a/Doc/includes/newtypes/custom.c +++ b/Doc/includes/newtypes/custom.c @@ -16,28 +16,37 @@ static PyTypeObject CustomType = { .tp_new = PyType_GenericNew, }; -static PyModuleDef custommodule = { +static int +custom_module_exec(PyObject *m) +{ + if (PyType_Ready(&CustomType) < 0) { + return -1; + } + + if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { + return -1; + } + + return 0; +} + +static PyModuleDef_Slot custom_module_slots[] = { + {Py_mod_exec, custom_module_exec}, + // Just use this while using static types + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {0, NULL} +}; + +static PyModuleDef custom_module = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "custom", .m_doc = "Example module that creates an extension type.", - .m_size = -1, + .m_size = 0, + .m_slots = custom_module_slots, }; PyMODINIT_FUNC PyInit_custom(void) { - PyObject *m; - if (PyType_Ready(&CustomType) < 0) - return NULL; - - m = PyModule_Create(&custommodule); - if (m == NULL) - return NULL; - - if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(m); - return NULL; - } - - return m; + return PyModuleDef_Init(&custom_module); } diff --git a/Doc/includes/newtypes/custom2.c b/Doc/includes/newtypes/custom2.c index a87917583ca495f..1ff8e707d1b0a04 100644 --- a/Doc/includes/newtypes/custom2.c +++ b/Doc/includes/newtypes/custom2.c @@ -106,28 +106,36 @@ static PyTypeObject CustomType = { .tp_methods = Custom_methods, }; -static PyModuleDef custommodule = { - .m_base =PyModuleDef_HEAD_INIT, +static int +custom_module_exec(PyObject *m) +{ + if (PyType_Ready(&CustomType) < 0) { + return -1; + } + + if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { + return -1; + } + + return 0; +} + +static PyModuleDef_Slot custom_module_slots[] = { + {Py_mod_exec, custom_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {0, NULL} +}; + +static PyModuleDef custom_module = { + .m_base = PyModuleDef_HEAD_INIT, .m_name = "custom2", .m_doc = "Example module that creates an extension type.", - .m_size = -1, + .m_size = 0, + .m_slots = custom_module_slots, }; PyMODINIT_FUNC PyInit_custom2(void) { - PyObject *m; - if (PyType_Ready(&CustomType) < 0) - return NULL; - - m = PyModule_Create(&custommodule); - if (m == NULL) - return NULL; - - if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(m); - return NULL; - } - - return m; + return PyModuleDef_Init(&custom_module); } diff --git a/Doc/includes/newtypes/custom3.c b/Doc/includes/newtypes/custom3.c index 854034d4066d204..22f50eb0e1de89c 100644 --- a/Doc/includes/newtypes/custom3.c +++ b/Doc/includes/newtypes/custom3.c @@ -151,28 +151,36 @@ static PyTypeObject CustomType = { .tp_getset = Custom_getsetters, }; -static PyModuleDef custommodule = { +static int +custom_module_exec(PyObject *m) +{ + if (PyType_Ready(&CustomType) < 0) { + return -1; + } + + if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { + return -1; + } + + return 0; +} + +static PyModuleDef_Slot custom_module_slots[] = { + {Py_mod_exec, custom_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {0, NULL} +}; + +static PyModuleDef custom_module = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "custom3", .m_doc = "Example module that creates an extension type.", - .m_size = -1, + .m_size = 0, + .m_slots = custom_module_slots, }; PyMODINIT_FUNC PyInit_custom3(void) { - PyObject *m; - if (PyType_Ready(&CustomType) < 0) - return NULL; - - m = PyModule_Create(&custommodule); - if (m == NULL) - return NULL; - - if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(m); - return NULL; - } - - return m; + return PyModuleDef_Init(&custom_module); } diff --git a/Doc/includes/newtypes/custom4.c b/Doc/includes/newtypes/custom4.c index a0a1eeb289190b9..07585aff5987f3d 100644 --- a/Doc/includes/newtypes/custom4.c +++ b/Doc/includes/newtypes/custom4.c @@ -170,28 +170,36 @@ static PyTypeObject CustomType = { .tp_getset = Custom_getsetters, }; -static PyModuleDef custommodule = { +static int +custom_module_exec(PyObject *m) +{ + if (PyType_Ready(&CustomType) < 0) { + return -1; + } + + if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { + return -1; + } + + return 0; +} + +static PyModuleDef_Slot custom_module_slots[] = { + {Py_mod_exec, custom_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {0, NULL} +}; + +static PyModuleDef custom_module = { .m_base = PyModuleDef_HEAD_INIT, .m_name = "custom4", .m_doc = "Example module that creates an extension type.", - .m_size = -1, + .m_size = 0, + .m_slots = custom_module_slots, }; PyMODINIT_FUNC PyInit_custom4(void) { - PyObject *m; - if (PyType_Ready(&CustomType) < 0) - return NULL; - - m = PyModule_Create(&custommodule); - if (m == NULL) - return NULL; - - if (PyModule_AddObjectRef(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(m); - return NULL; - } - - return m; + return PyModuleDef_Init(&custom_module); } diff --git a/Doc/includes/newtypes/sublist.c b/Doc/includes/newtypes/sublist.c index 00664f3454156fb..b784456a4ef6677 100644 --- a/Doc/includes/newtypes/sublist.c +++ b/Doc/includes/newtypes/sublist.c @@ -31,7 +31,7 @@ SubList_init(PyObject *op, PyObject *args, PyObject *kwds) } static PyTypeObject SubListType = { - PyVarObject_HEAD_INIT(NULL, 0) + .ob_base = PyVarObject_HEAD_INIT(NULL, 0) .tp_name = "sublist.SubList", .tp_doc = PyDoc_STR("SubList objects"), .tp_basicsize = sizeof(SubListObject), @@ -41,29 +41,37 @@ static PyTypeObject SubListType = { .tp_methods = SubList_methods, }; -static PyModuleDef sublistmodule = { - PyModuleDef_HEAD_INIT, +static int +sublist_module_exec(PyObject *m) +{ + SubListType.tp_base = &PyList_Type; + if (PyType_Ready(&SubListType) < 0) { + return -1; + } + + if (PyModule_AddObjectRef(m, "SubList", (PyObject *) &SubListType) < 0) { + return -1; + } + + return 0; +} + +static PyModuleDef_Slot sublist_module_slots[] = { + {Py_mod_exec, sublist_module_exec}, + {Py_mod_multiple_interpreters, Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED}, + {0, NULL} +}; + +static PyModuleDef sublist_module = { + .m_base = PyModuleDef_HEAD_INIT, .m_name = "sublist", .m_doc = "Example module that creates an extension type.", - .m_size = -1, + .m_size = 0, + .m_slots = sublist_module_slots, }; PyMODINIT_FUNC PyInit_sublist(void) { - PyObject *m; - SubListType.tp_base = &PyList_Type; - if (PyType_Ready(&SubListType) < 0) - return NULL; - - m = PyModule_Create(&sublistmodule); - if (m == NULL) - return NULL; - - if (PyModule_AddObjectRef(m, "SubList", (PyObject *) &SubListType) < 0) { - Py_DECREF(m); - return NULL; - } - - return m; + return PyModuleDef_Init(&sublist_module); } diff --git a/Doc/includes/optional-module.rst b/Doc/includes/optional-module.rst new file mode 100644 index 000000000000000..262e73f2eaa09fb --- /dev/null +++ b/Doc/includes/optional-module.rst @@ -0,0 +1,9 @@ +This is an :term:`optional module`. +If it is missing from your copy of CPython, +look for documentation from your distributor (that is, +whoever provided Python to you). +If you are the distributor, see :ref:`optional-module-requirements`. + +.. Similar notes appear in the docs of the modules: + - zipfile + - tarfile diff --git a/Doc/includes/tzinfo_examples.py b/Doc/includes/tzinfo_examples.py index 1fa6e615e46a763..762b1b62fc871d4 100644 --- a/Doc/includes/tzinfo_examples.py +++ b/Doc/includes/tzinfo_examples.py @@ -1,68 +1,70 @@ -from datetime import tzinfo, timedelta, datetime - -ZERO = timedelta(0) -HOUR = timedelta(hours=1) -SECOND = timedelta(seconds=1) +import datetime as dt # A class capturing the platform's idea of local time. # (May result in wrong values on historical times in # timezones where UTC offset and/or the DST rules had # changed in the past.) -import time as _time +import time + +ZERO = dt.timedelta(0) +HOUR = dt.timedelta(hours=1) +SECOND = dt.timedelta(seconds=1) -STDOFFSET = timedelta(seconds = -_time.timezone) -if _time.daylight: - DSTOFFSET = timedelta(seconds = -_time.altzone) +STDOFFSET = dt.timedelta(seconds=-time.timezone) +if time.daylight: + DSTOFFSET = dt.timedelta(seconds=-time.altzone) else: DSTOFFSET = STDOFFSET DSTDIFF = DSTOFFSET - STDOFFSET -class LocalTimezone(tzinfo): - def fromutc(self, dt): - assert dt.tzinfo is self - stamp = (dt - datetime(1970, 1, 1, tzinfo=self)) // SECOND - args = _time.localtime(stamp)[:6] +class LocalTimezone(dt.tzinfo): + + def fromutc(self, when): + assert when.tzinfo is self + stamp = (when - dt.datetime(1970, 1, 1, tzinfo=self)) // SECOND + args = time.localtime(stamp)[:6] dst_diff = DSTDIFF // SECOND # Detect fold - fold = (args == _time.localtime(stamp - dst_diff)) - return datetime(*args, microsecond=dt.microsecond, - tzinfo=self, fold=fold) + fold = (args == time.localtime(stamp - dst_diff)) + return dt.datetime(*args, microsecond=when.microsecond, + tzinfo=self, fold=fold) - def utcoffset(self, dt): - if self._isdst(dt): + def utcoffset(self, when): + if self._isdst(when): return DSTOFFSET else: return STDOFFSET - def dst(self, dt): - if self._isdst(dt): + def dst(self, when): + if self._isdst(when): return DSTDIFF else: return ZERO - def tzname(self, dt): - return _time.tzname[self._isdst(dt)] + def tzname(self, when): + return time.tzname[self._isdst(when)] - def _isdst(self, dt): - tt = (dt.year, dt.month, dt.day, - dt.hour, dt.minute, dt.second, - dt.weekday(), 0, 0) - stamp = _time.mktime(tt) - tt = _time.localtime(stamp) + def _isdst(self, when): + tt = (when.year, when.month, when.day, + when.hour, when.minute, when.second, + when.weekday(), 0, 0) + stamp = time.mktime(tt) + tt = time.localtime(stamp) return tt.tm_isdst > 0 + Local = LocalTimezone() # A complete implementation of current DST rules for major US time zones. -def first_sunday_on_or_after(dt): - days_to_go = 6 - dt.weekday() +def first_sunday_on_or_after(when): + days_to_go = 6 - when.weekday() if days_to_go: - dt += timedelta(days_to_go) - return dt + when += dt.timedelta(days_to_go) + return when # US DST Rules @@ -75,21 +77,22 @@ def first_sunday_on_or_after(dt): # # In the US, since 2007, DST starts at 2am (standard time) on the second # Sunday in March, which is the first Sunday on or after Mar 8. -DSTSTART_2007 = datetime(1, 3, 8, 2) +DSTSTART_2007 = dt.datetime(1, 3, 8, 2) # and ends at 2am (DST time) on the first Sunday of Nov. -DSTEND_2007 = datetime(1, 11, 1, 2) +DSTEND_2007 = dt.datetime(1, 11, 1, 2) # From 1987 to 2006, DST used to start at 2am (standard time) on the first # Sunday in April and to end at 2am (DST time) on the last # Sunday of October, which is the first Sunday on or after Oct 25. -DSTSTART_1987_2006 = datetime(1, 4, 1, 2) -DSTEND_1987_2006 = datetime(1, 10, 25, 2) +DSTSTART_1987_2006 = dt.datetime(1, 4, 1, 2) +DSTEND_1987_2006 = dt.datetime(1, 10, 25, 2) # From 1967 to 1986, DST used to start at 2am (standard time) on the last # Sunday in April (the one on or after April 24) and to end at 2am (DST time) # on the last Sunday of October, which is the first Sunday # on or after Oct 25. -DSTSTART_1967_1986 = datetime(1, 4, 24, 2) +DSTSTART_1967_1986 = dt.datetime(1, 4, 24, 2) DSTEND_1967_1986 = DSTEND_1987_2006 + def us_dst_range(year): # Find start and end times for US DST. For years before 1967, return # start = end for no DST. @@ -100,17 +103,17 @@ def us_dst_range(year): elif 1966 < year < 1987: dststart, dstend = DSTSTART_1967_1986, DSTEND_1967_1986 else: - return (datetime(year, 1, 1), ) * 2 + return (dt.datetime(year, 1, 1), ) * 2 start = first_sunday_on_or_after(dststart.replace(year=year)) end = first_sunday_on_or_after(dstend.replace(year=year)) return start, end -class USTimeZone(tzinfo): +class USTimeZone(dt.tzinfo): def __init__(self, hours, reprname, stdname, dstname): - self.stdoffset = timedelta(hours=hours) + self.stdoffset = dt.timedelta(hours=hours) self.reprname = reprname self.stdname = stdname self.dstname = dstname @@ -118,45 +121,45 @@ def __init__(self, hours, reprname, stdname, dstname): def __repr__(self): return self.reprname - def tzname(self, dt): - if self.dst(dt): + def tzname(self, when): + if self.dst(when): return self.dstname else: return self.stdname - def utcoffset(self, dt): - return self.stdoffset + self.dst(dt) + def utcoffset(self, when): + return self.stdoffset + self.dst(when) - def dst(self, dt): - if dt is None or dt.tzinfo is None: + def dst(self, when): + if when is None or when.tzinfo is None: # An exception may be sensible here, in one or both cases. # It depends on how you want to treat them. The default # fromutc() implementation (called by the default astimezone() - # implementation) passes a datetime with dt.tzinfo is self. + # implementation) passes a datetime with when.tzinfo is self. return ZERO - assert dt.tzinfo is self - start, end = us_dst_range(dt.year) + assert when.tzinfo is self + start, end = us_dst_range(when.year) # Can't compare naive to aware objects, so strip the timezone from - # dt first. - dt = dt.replace(tzinfo=None) - if start + HOUR <= dt < end - HOUR: + # when first. + when = when.replace(tzinfo=None) + if start + HOUR <= when < end - HOUR: # DST is in effect. return HOUR - if end - HOUR <= dt < end: - # Fold (an ambiguous hour): use dt.fold to disambiguate. - return ZERO if dt.fold else HOUR - if start <= dt < start + HOUR: + if end - HOUR <= when < end: + # Fold (an ambiguous hour): use when.fold to disambiguate. + return ZERO if when.fold else HOUR + if start <= when < start + HOUR: # Gap (a non-existent hour): reverse the fold rule. - return HOUR if dt.fold else ZERO + return HOUR if when.fold else ZERO # DST is off. return ZERO - def fromutc(self, dt): - assert dt.tzinfo is self - start, end = us_dst_range(dt.year) + def fromutc(self, when): + assert when.tzinfo is self + start, end = us_dst_range(when.year) start = start.replace(tzinfo=self) end = end.replace(tzinfo=self) - std_time = dt + self.stdoffset + std_time = when + self.stdoffset dst_time = std_time + HOUR if end <= dst_time < end + HOUR: # Repeated hour diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst index a46c1caefe4d8ac..c372d9f47418001 100644 --- a/Doc/installing/index.rst +++ b/Doc/installing/index.rst @@ -1,16 +1,14 @@ -.. highlight:: none +.. highlight:: shell .. _installing-index: ************************* -Installing Python Modules +Installing Python modules ************************* -:Email: distutils-sig@python.org - As a popular open source development project, Python has an active supporting community of contributors and users that also make their software -available for other Python developers to use under open source license terms. +available for other Python developers to use under open-source license terms. This allows Python users to share and collaborate effectively, benefiting from the solutions others have already created to common (and sometimes @@ -34,34 +32,24 @@ creating and sharing your own Python projects, refer to the Key terms ========= -* ``pip`` is the preferred installer program. Starting with Python 3.4, it +* :program:`pip` is the preferred installer program. It is included by default with the Python binary installers. * A *virtual environment* is a semi-isolated Python environment that allows packages to be installed for use by a particular application, rather than being installed system wide. -* ``venv`` is the standard tool for creating virtual environments, and has - been part of Python since Python 3.3. Starting with Python 3.4, it - defaults to installing ``pip`` into all created virtual environments. -* ``virtualenv`` is a third party alternative (and predecessor) to - ``venv``. It allows virtual environments to be used on versions of - Python prior to 3.4, which either don't provide ``venv`` at all, or - aren't able to automatically install ``pip`` into created environments. -* The `Python Package Index `__ is a public +* ``venv`` is the standard tool for creating virtual environments. + It defaults to installing :program:`pip` into all created virtual environments. +* ``virtualenv`` is a third-party alternative (and predecessor) to + ``venv``. +* The `Python Package Index (PyPI) `__ is a public repository of open source licensed packages made available for use by other Python users. -* the `Python Packaging Authority +* The `Python Packaging Authority `__ is the group of developers and documentation authors responsible for the maintenance and evolution of the standard packaging tools and the associated metadata and file format standards. They maintain a variety of tools, documentation, and issue trackers on `GitHub `__. -* ``distutils`` is the original build and distribution system first added to - the Python standard library in 1998. While direct use of ``distutils`` is - being phased out, it still laid the foundation for the current packaging - and distribution infrastructure, and it not only remains part of the - standard library, but its name lives on in other ways (such as the name - of the mailing list used to coordinate Python packaging standards - development). .. versionchanged:: 3.5 The use of ``venv`` is now recommended for creating virtual environments. @@ -79,7 +67,7 @@ The standard packaging tools are all designed to be used from the command line. The following command will install the latest version of a module and its -dependencies from the Python Package Index:: +dependencies from PyPI:: python -m pip install SomePackage @@ -106,7 +94,7 @@ explicitly:: python -m pip install --upgrade SomePackage -More information and resources regarding ``pip`` and its capabilities can be +More information and resources regarding :program:`pip` and its capabilities can be found in the `Python Packaging User Guide `__. Creation of virtual environments is done through the :mod:`venv` module. @@ -124,19 +112,6 @@ How do I ...? These are quick answers or links for some common tasks. -... install ``pip`` in versions of Python prior to Python 3.4? --------------------------------------------------------------- - -Python only started bundling ``pip`` with Python 3.4. For earlier versions, -``pip`` needs to be "bootstrapped" as described in the Python Packaging -User Guide. - -.. seealso:: - - `Python Packaging User Guide: Requirements for Installing Packages - `__ - - .. installing-per-user-installation: ... install packages just for the current user? @@ -150,10 +125,10 @@ package just for the current user, rather than for all users of the system. --------------------------------------- A number of scientific Python packages have complex binary dependencies, and -aren't currently easy to install using ``pip`` directly. At this point in -time, it will often be easier for users to install these packages by +aren't currently easy to install using :program:`pip` directly. +It will often be easier for users to install these packages by `other means `__ -rather than attempting to install them with ``pip``. +rather than attempting to install them with :program:`pip`. .. seealso:: @@ -166,29 +141,25 @@ rather than attempting to install them with ``pip``. On Linux, macOS, and other POSIX systems, use the versioned Python commands in combination with the ``-m`` switch to run the appropriate copy of -``pip``:: +:program:`pip`:: - python2 -m pip install SomePackage # default Python 2 - python2.7 -m pip install SomePackage # specifically Python 2.7 - python3 -m pip install SomePackage # default Python 3 - python3.4 -m pip install SomePackage # specifically Python 3.4 + python3 -m pip install SomePackage # default Python 3 + python3.14 -m pip install SomePackage # specifically Python 3.14 -Appropriately versioned ``pip`` commands may also be available. +Appropriately versioned :program:`pip` commands may also be available. -On Windows, use the ``py`` Python launcher in combination with the ``-m`` +On Windows, use the :program:`py` Python launcher in combination with the ``-m`` switch:: - py -2 -m pip install SomePackage # default Python 2 - py -2.7 -m pip install SomePackage # specifically Python 2.7 - py -3 -m pip install SomePackage # default Python 3 - py -3.4 -m pip install SomePackage # specifically Python 3.4 + py -3 -m pip install SomePackage # default Python 3 + py -3.14 -m pip install SomePackage # specifically Python 3.14 .. other questions: Once the Development & Deployment part of PPUG is fleshed out, some of those sections should be linked from new questions here (most notably, we should have a question about avoiding depending on PyPI that links to - https://packaging.python.org/en/latest/mirrors/) + https://packaging.python.org/en/latest/guides/index-mirrors-and-caches/) Common installation issues @@ -201,39 +172,38 @@ On Linux systems, a Python installation will typically be included as part of the distribution. Installing into this Python installation requires root access to the system, and may interfere with the operation of the system package manager and other components of the system if a component -is unexpectedly upgraded using ``pip``. +is unexpectedly upgraded using :program:`pip`. On such systems, it is often better to use a virtual environment or a -per-user installation when installing packages with ``pip``. +per-user installation when installing packages with :program:`pip`. Pip not installed ----------------- -It is possible that ``pip`` does not get installed by default. One potential fix is:: +It is possible that :program:`pip` does not get installed by default. One potential fix is:: python -m ensurepip --default-pip -There are also additional resources for `installing pip. -`__ +There are also additional resources for `installing pip +`__. Installing binary extensions ---------------------------- -Python has typically relied heavily on source based distribution, with end +Python once relied heavily on source-based distribution, with end users being expected to compile extension modules from source as part of the installation process. -With the introduction of support for the binary ``wheel`` format, and the -ability to publish wheels for at least Windows and macOS through the -Python Package Index, this problem is expected to diminish over time, +With the introduction of the binary wheel format, and the +ability to publish wheels through PyPI, this problem is diminishing, as users are more regularly able to install pre-built extensions rather than needing to build them themselves. Some of the solutions for installing `scientific software `__ -that are not yet available as pre-built ``wheel`` files may also help with +that are not yet available as pre-built wheel files may also help with obtaining other binary extensions without needing to build them locally. .. seealso:: diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index 4f3b663006fb28b..749e4543c5b8235 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -15,7 +15,7 @@ before the release in which the feature becomes standard. While these future statements are given additional special meaning by the Python compiler, they are still executed like any other import statement and -the :mod:`__future__` exists and is handled by the import system the same way +the :mod:`!__future__` exists and is handled by the import system the same way any other Python module would be. This design serves three purposes: * To avoid confusing existing tools that analyze import statements and expect to @@ -23,52 +23,66 @@ any other Python module would be. This design serves three purposes: * To document when incompatible changes were introduced, and when they will be --- or were --- made mandatory. This is a form of executable documentation, and - can be inspected programmatically via importing :mod:`__future__` and examining + can be inspected programmatically via importing :mod:`!__future__` and examining its contents. * To ensure that :ref:`future statements ` run under releases prior to - Python 2.1 at least yield runtime exceptions (the import of :mod:`__future__` + Python 2.1 at least yield runtime exceptions (the import of :mod:`!__future__` will fail, because there was no module of that name prior to 2.1). Module Contents --------------- -No feature description will ever be deleted from :mod:`__future__`. Since its +No feature description will ever be deleted from :mod:`!__future__`. Since its introduction in Python 2.1 the following features have found their way into the language using this mechanism: -+------------------+-------------+--------------+---------------------------------------------+ -| feature | optional in | mandatory in | effect | -+==================+=============+==============+=============================================+ -| nested_scopes | 2.1.0b1 | 2.2 | :pep:`227`: | -| | | | *Statically Nested Scopes* | -+------------------+-------------+--------------+---------------------------------------------+ -| generators | 2.2.0a1 | 2.3 | :pep:`255`: | -| | | | *Simple Generators* | -+------------------+-------------+--------------+---------------------------------------------+ -| division | 2.2.0a2 | 3.0 | :pep:`238`: | -| | | | *Changing the Division Operator* | -+------------------+-------------+--------------+---------------------------------------------+ -| absolute_import | 2.5.0a1 | 3.0 | :pep:`328`: | -| | | | *Imports: Multi-Line and Absolute/Relative* | -+------------------+-------------+--------------+---------------------------------------------+ -| with_statement | 2.5.0a1 | 2.6 | :pep:`343`: | -| | | | *The "with" Statement* | -+------------------+-------------+--------------+---------------------------------------------+ -| print_function | 2.6.0a2 | 3.0 | :pep:`3105`: | -| | | | *Make print a function* | -+------------------+-------------+--------------+---------------------------------------------+ -| unicode_literals | 2.6.0a2 | 3.0 | :pep:`3112`: | -| | | | *Bytes literals in Python 3000* | -+------------------+-------------+--------------+---------------------------------------------+ -| generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | -| | | | *StopIteration handling inside generators* | -+------------------+-------------+--------------+---------------------------------------------+ -| annotations | 3.7.0b1 | Never [1]_ | :pep:`563`: | -| | | | *Postponed evaluation of annotations*, | -| | | | :pep:`649`: *Deferred evaluation of | -| | | | annotations using descriptors* | -+------------------+-------------+--------------+---------------------------------------------+ + +.. list-table:: + :widths: auto + :header-rows: 1 + + * * feature + * optional in + * mandatory in + * effect + * * .. data:: nested_scopes + * 2.1.0b1 + * 2.2 + * :pep:`227`: *Statically Nested Scopes* + * * .. data:: generators + * 2.2.0a1 + * 2.3 + * :pep:`255`: *Simple Generators* + * * .. data:: division + * 2.2.0a2 + * 3.0 + * :pep:`238`: *Changing the Division Operator* + * * .. data:: absolute_import + * 2.5.0a1 + * 3.0 + * :pep:`328`: *Imports: Multi-Line and Absolute/Relative* + * * .. data:: with_statement + * 2.5.0a1 + * 2.6 + * :pep:`343`: *The “with” Statement* + * * .. data:: print_function + * 2.6.0a2 + * 3.0 + * :pep:`3105`: *Make print a function* + * * .. data:: unicode_literals + * 2.6.0a2 + * 3.0 + * :pep:`3112`: *Bytes literals in Python 3000* + * * .. data:: generator_stop + * 3.5.0b1 + * 3.7 + * :pep:`479`: *StopIteration handling inside generators* + * * .. data:: annotations + * 3.7.0b1 + * Never [1]_ + * :pep:`563`: *Postponed evaluation of annotations*, + :pep:`649`: *Deferred evaluation of annotations using descriptors* .. XXX Adding a new entry? Remember to update simple_stmts.rst, too. diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst index ed29ac70035597c..c48e6437ec21ad3 100644 --- a/Doc/library/_thread.rst +++ b/Doc/library/_thread.rst @@ -36,11 +36,6 @@ This module defines the following constants and functions: This is now a synonym of the built-in :exc:`RuntimeError`. -.. data:: LockType - - This is the type of lock objects. - - .. function:: start_new_thread(function, args[, kwargs]) Start a new thread and return its identifier. The thread executes the @@ -159,58 +154,66 @@ This module defines the following constants and functions: .. versionadded:: 3.2 -Lock objects have the following methods: +.. raw:: html + + + + + +.. class:: LockType -.. method:: lock.acquire(blocking=True, timeout=-1) + This is the type of lock objects. - Without any optional argument, this method acquires the lock unconditionally, if - necessary waiting until it is released by another thread (only one thread at a - time can acquire a lock --- that's their reason for existence). + Lock objects have the following methods: - If the *blocking* argument is present, the action depends on its - value: if it is false, the lock is only acquired if it can be acquired - immediately without waiting, while if it is true, the lock is acquired - unconditionally as above. + .. method:: acquire(blocking=True, timeout=-1) - If the floating-point *timeout* argument is present and positive, it - specifies the maximum wait time in seconds before returning. A negative - *timeout* argument specifies an unbounded wait. You cannot specify - a *timeout* if *blocking* is false. + Without any optional argument, this method acquires the lock unconditionally, if + necessary waiting until it is released by another thread (only one thread at a + time can acquire a lock --- that's their reason for existence). - The return value is ``True`` if the lock is acquired successfully, - ``False`` if not. + If the *blocking* argument is present, the action depends on its + value: if it is false, the lock is only acquired if it can be acquired + immediately without waiting, while if it is true, the lock is acquired + unconditionally as above. - .. versionchanged:: 3.2 - The *timeout* parameter is new. + If the floating-point *timeout* argument is present and positive, it + specifies the maximum wait time in seconds before returning. A negative + *timeout* argument specifies an unbounded wait. You cannot specify + a *timeout* if *blocking* is false. - .. versionchanged:: 3.2 - Lock acquires can now be interrupted by signals on POSIX. + The return value is ``True`` if the lock is acquired successfully, + ``False`` if not. - .. versionchanged:: 3.14 - Lock acquires can now be interrupted by signals on Windows. + .. versionchanged:: 3.2 + The *timeout* parameter is new. + .. versionchanged:: 3.2 + Lock acquires can now be interrupted by signals on POSIX. -.. method:: lock.release() + .. versionchanged:: 3.14 + Lock acquires can now be interrupted by signals on Windows. - Releases the lock. The lock must have been acquired earlier, but not - necessarily by the same thread. + .. method:: release() + Releases the lock. The lock must have been acquired earlier, but not + necessarily by the same thread. -.. method:: lock.locked() + .. method:: locked() - Return the status of the lock: ``True`` if it has been acquired by some thread, - ``False`` if not. + Return the status of the lock: ``True`` if it has been acquired by some thread, + ``False`` if not. -In addition to these methods, lock objects can also be used via the -:keyword:`with` statement, e.g.:: + In addition to these methods, lock objects can also be used via the + :keyword:`with` statement, e.g.:: - import _thread + import _thread - a_lock = _thread.allocate_lock() + a_lock = _thread.allocate_lock() - with a_lock: - print("a_lock is locked while this executes") + with a_lock: + print("a_lock is locked while this executes") **Caveats:** diff --git a/Doc/library/annotationlib.rst b/Doc/library/annotationlib.rst index ff9578b6088f282..af28fe0e2fde2f8 100644 --- a/Doc/library/annotationlib.rst +++ b/Doc/library/annotationlib.rst @@ -4,6 +4,7 @@ .. module:: annotationlib :synopsis: Functionality for introspecting annotations +.. versionadded:: 3.14 **Source code:** :source:`Lib/annotationlib.py` @@ -45,6 +46,10 @@ and :func:`call_annotate_function`, as well as the :func:`call_evaluate_function` function for working with :term:`evaluate functions `. +.. caution:: + + Most functionality in this module can execute arbitrary code; see + :ref:`the security section ` for more information. .. seealso:: @@ -127,16 +132,27 @@ Classes Values are the result of evaluating the annotation expressions. - .. attribute:: FORWARDREF + .. attribute:: VALUE_WITH_FAKE_GLOBALS :value: 2 + Special value used to signal that an annotate function is being + evaluated in a special environment with fake globals. When passed this + value, annotate functions should either return the same value as for + the :attr:`Format.VALUE` format, or raise :exc:`NotImplementedError` + to signal that they do not support execution in this environment. + This format is only used internally and should not be passed to + the functions in this module. + + .. attribute:: FORWARDREF + :value: 3 + Values are real annotation values (as per :attr:`Format.VALUE` format) for defined values, and :class:`ForwardRef` proxies for undefined values. Real objects may contain references to :class:`ForwardRef` proxy objects. .. attribute:: STRING - :value: 3 + :value: 4 Values are the text string of the annotation as it appears in the source code, up to modifications including, but not restricted to, @@ -144,17 +160,6 @@ Classes The exact values of these strings may change in future versions of Python. - .. attribute:: VALUE_WITH_FAKE_GLOBALS - :value: 4 - - Special value used to signal that an annotate function is being - evaluated in a special environment with fake globals. When passed this - value, annotate functions should either return the same value as for - the :attr:`Format.VALUE` format, or raise :exc:`NotImplementedError` - to signal that they do not support execution in this environment. - This format is only used internally and should not be passed to - the functions in this module. - .. versionadded:: 3.14 .. class:: ForwardRef @@ -211,6 +216,10 @@ Classes means may not have any information about their scope, so passing arguments to this method may be necessary to evaluate them successfully. + If no *owner*, *globals*, *locals*, or *type_params* are provided and the + :class:`~ForwardRef` does not contain information about its origin, + empty globals and locals dictionaries are used. + .. versionadded:: 3.14 @@ -331,14 +340,29 @@ Functions * VALUE: :attr:`!object.__annotations__` is tried first; if that does not exist, the :attr:`!object.__annotate__` function is called if it exists. + * FORWARDREF: If :attr:`!object.__annotations__` exists and can be evaluated successfully, it is used; otherwise, the :attr:`!object.__annotate__` function is called. If it does not exist either, :attr:`!object.__annotations__` is tried again and any error from accessing it is re-raised. + + * When calling :attr:`!object.__annotate__` it is first called with :attr:`~Format.FORWARDREF`. + If this is not implemented, it will then check if :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` + is supported and use that in the fake globals environment. + If neither of these formats are supported, it will fall back to using :attr:`~Format.VALUE`. + If :attr:`~Format.VALUE` fails, the error from this call will be raised. + * STRING: If :attr:`!object.__annotate__` exists, it is called first; otherwise, :attr:`!object.__annotations__` is used and stringified using :func:`annotations_to_string`. + * When calling :attr:`!object.__annotate__` it is first called with :attr:`~Format.STRING`. + If this is not implemented, it will then check if :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` + is supported and use that in the fake globals environment. + If neither of these formats are supported, it will fall back to using :attr:`~Format.VALUE` + with the result converted using :func:`annotations_to_string`. + If :attr:`~Format.VALUE` fails, the error from this call will be raised. + Returns a dict. :func:`!get_annotations` returns a new dict every time it's called; calling it twice on the same object will return two different but equivalent dicts. @@ -485,3 +509,212 @@ annotations from the class and puts them in a separate attribute: typ.classvars = classvars # Store the ClassVars in a separate attribute return typ + +Creating a custom callable annotate function +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Custom :term:`annotate functions ` may be literal functions like those +automatically generated for functions, classes, and modules. Or, they may wish to utilise +the encapsulation provided by classes, in which case any :term:`callable` can be used as +an :term:`annotate function`. + +To provide the :attr:`~Format.VALUE`, :attr:`~Format.STRING`, or +:attr:`~Format.FORWARDREF` formats directly, an :term:`annotate function` must provide +the following attribute: + +* A callable ``__call__`` with signature ``__call__(format, /) -> dict``, that does not + raise a :exc:`NotImplementedError` when called with a supported format. + +To provide the :attr:`~Format.VALUE_WITH_FAKE_GLOBALS` format, which is used to +automatically generate :attr:`~Format.STRING` or :attr:`~Format.FORWARDREF` if they are +not supported directly, :term:`annotate functions ` must provide the +following attributes: + +* A callable ``__call__`` with signature ``__call__(format, /) -> dict``, that does not + raise a :exc:`NotImplementedError` when called with + :attr:`~Format.VALUE_WITH_FAKE_GLOBALS`. +* A :ref:`code object ` ``__code__`` containing the compiled code for the + annotate function. +* Optional: A tuple of the function's positional defaults ``__kwdefaults__``, if the + function represented by ``__code__`` uses any positional defaults. +* Optional: A dict of the function's keyword defaults ``__defaults__``, if the function + represented by ``__code__`` uses any keyword defaults. +* Optional: All other :ref:`function attributes `. + +.. code-block:: python + + class Annotate: + called_formats = [] + + def __call__(self, format=None, /, *, _self=None): + # When called with fake globals, `_self` will be the + # actual self value, and `self` will be the format. + if _self is not None: + self, format = _self, self + + self.called_formats.append(format) + if format <= 2: # VALUE or VALUE_WITH_FAKE_GLOBALS + return {"x": MyType} + raise NotImplementedError + + __code__ = __call__.__code__ + __defaults__ = (None,) + __kwdefaults__ = property(lambda self: dict(_self=self)) + + __globals__ = {} + __builtins__ = {} + __closure__ = None + +This can then be called with: + +.. code-block:: pycon + + >>> from annotationlib import call_annotate_function, Format + >>> call_annotate_function(Annotate(), format=Format.STRING) + {'x': 'MyType'} + +Or used as the annotate function for an object: + +.. code-block:: pycon + + >>> from annotationlib import get_annotations, Format + >>> class C: + ... pass + >>> C.__annotate__ = Annotate() + >>> get_annotations(Annotate(), format=Format.STRING) + {'x': 'MyType'} + + +Limitations of the ``STRING`` format +------------------------------------ + +The :attr:`~Format.STRING` format is meant to approximate the source code +of the annotation, but the implementation strategy used means that it is not +always possible to recover the exact source code. + +First, the stringifier of course cannot recover any information that is not present in +the compiled code, including comments, whitespace, parenthesization, and operations that +get simplified by the compiler. + +Second, the stringifier can intercept almost all operations that involve names looked +up in some scope, but it cannot intercept operations that operate fully on constants. +As a corollary, this also means it is not safe to request the ``STRING`` format on +untrusted code: Python is powerful enough that it is possible to achieve arbitrary +code execution even with no access to any globals or builtins. For example: + +.. code-block:: pycon + + >>> def f(x: (1).__class__.__base__.__subclasses__()[-1].__init__.__builtins__["print"]("Hello world")): pass + ... + >>> annotationlib.get_annotations(f, format=annotationlib.Format.STRING) + Hello world + {'x': 'None'} + +.. note:: + This particular example works as of the time of writing, but it relies on + implementation details and is not guaranteed to work in the future. + +Among the different kinds of expressions that exist in Python, +as represented by the :mod:`ast` module, some expressions are supported, +meaning that the ``STRING`` format can generally recover the original source code; +others are unsupported, meaning that they may result in incorrect output or an error. + +The following are supported (sometimes with caveats): + +* :class:`ast.BinOp` +* :class:`ast.UnaryOp` + + * :class:`ast.Invert` (``~``), :class:`ast.UAdd` (``+``), and :class:`ast.USub` (``-``) are supported + * :class:`ast.Not` (``not``) is not supported + +* :class:`ast.Dict` (except when using ``**`` unpacking) +* :class:`ast.Set` +* :class:`ast.Compare` + + * :class:`ast.Eq` and :class:`ast.NotEq` are supported + * :class:`ast.Lt`, :class:`ast.LtE`, :class:`ast.Gt`, and :class:`ast.GtE` are supported, but the operand may be flipped + * :class:`ast.Is`, :class:`ast.IsNot`, :class:`ast.In`, and :class:`ast.NotIn` are not supported + +* :class:`ast.Call` (except when using ``**`` unpacking) +* :class:`ast.Constant` (though not the exact representation of the constant; for example, escape + sequences in strings are lost; hexadecimal numbers are converted to decimal) +* :class:`ast.Attribute` (assuming the value is not a constant) +* :class:`ast.Subscript` (assuming the value is not a constant) +* :class:`ast.Starred` (``*`` unpacking) +* :class:`ast.Name` +* :class:`ast.List` +* :class:`ast.Tuple` +* :class:`ast.Slice` + +The following are unsupported, but throw an informative error when encountered by the +stringifier: + +* :class:`ast.FormattedValue` (f-strings; error is not detected if conversion specifiers like ``!r`` + are used) +* :class:`ast.JoinedStr` (f-strings) + +The following are unsupported and result in incorrect output: + +* :class:`ast.BoolOp` (``and`` and ``or``) +* :class:`ast.IfExp` +* :class:`ast.Lambda` +* :class:`ast.ListComp` +* :class:`ast.SetComp` +* :class:`ast.DictComp` +* :class:`ast.GeneratorExp` + +The following are disallowed in annotation scopes and therefore not relevant: + +* :class:`ast.NamedExpr` (``:=``) +* :class:`ast.Await` +* :class:`ast.Yield` +* :class:`ast.YieldFrom` + + +Limitations of the ``FORWARDREF`` format +---------------------------------------- + +The :attr:`~Format.FORWARDREF` format aims to produce real values as much +as possible, with anything that cannot be resolved replaced with +:class:`ForwardRef` objects. It is affected by broadly the same Limitations +as the :attr:`~Format.STRING` format: annotations that perform operations on +literals or that use unsupported expression types may raise exceptions when +evaluated using the :attr:`~Format.FORWARDREF` format. + +Below are a few examples of the behavior with unsupported expressions: + +.. code-block:: pycon + + >>> from annotationlib import get_annotations, Format + >>> def zerodiv(x: 1 / 0): ... + >>> get_annotations(zerodiv, format=Format.STRING) + Traceback (most recent call last): + ... + ZeroDivisionError: division by zero + >>> get_annotations(zerodiv, format=Format.FORWARDREF) + Traceback (most recent call last): + ... + ZeroDivisionError: division by zero + >>> def ifexp(x: 1 if y else 0): ... + >>> get_annotations(ifexp, format=Format.STRING) + {'x': '1'} + +.. _annotationlib-security: + +Security implications of introspecting annotations +-------------------------------------------------- + +Much of the functionality in this module involves executing code related to annotations, +which can then do arbitrary things. For example, +:func:`get_annotations` may call an arbitrary :term:`annotate function`, and +:meth:`ForwardRef.evaluate` may call :func:`eval` on an arbitrary string. Code contained +in an annotation might make arbitrary system calls, enter an infinite loop, or perform any +other operation. This is also true for any access of the :attr:`~object.__annotations__` attribute, +and for various functions in the :mod:`typing` module that work with annotations, such as +:func:`typing.get_type_hints`. + +Any security issue arising from this also applies immediately after importing +code that may contain untrusted annotations: importing code can always cause arbitrary operations +to be performed. However, it is unsafe to accept strings or other input from an untrusted source and +pass them to any of the APIs for introspecting annotations, for example by editing an +``__annotations__`` dictionary or directly creating a :class:`ForwardRef` object. diff --git a/Doc/library/archiving.rst b/Doc/library/archiving.rst index c9284949af49726..da0b3f8c3e76934 100644 --- a/Doc/library/archiving.rst +++ b/Doc/library/archiving.rst @@ -5,13 +5,15 @@ Data Compression and Archiving ****************************** The modules described in this chapter support data compression with the zlib, -gzip, bzip2 and lzma algorithms, and the creation of ZIP- and tar-format +gzip, bzip2, lzma, and zstd algorithms, and the creation of ZIP- and tar-format archives. See also :ref:`archiving-operations` provided by the :mod:`shutil` module. .. toctree:: + compression.rst + compression.zstd.rst zlib.rst gzip.rst bz2.rst diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 29396c7a0366a11..7508aea2aed0840 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -13,7 +13,7 @@ .. note:: - While :mod:`argparse` is the default recommended standard library module + While :mod:`!argparse` is the default recommended standard library module for implementing basic command line applications, authors with more exacting requirements for exactly how their command line applications behave may find it doesn't provide the necessary level of control. @@ -74,7 +74,7 @@ ArgumentParser objects prefix_chars='-', fromfile_prefix_chars=None, \ argument_default=None, conflict_handler='error', \ add_help=True, allow_abbrev=True, exit_on_error=True, \ - *, suggest_on_error=False, color=False) + *, suggest_on_error=False, color=True) Create a new :class:`ArgumentParser` object. All parameters should be passed as keyword arguments. Each parameter has its own more detailed description @@ -119,7 +119,7 @@ ArgumentParser objects * suggest_on_error_ - Enables suggestions for mistyped argument choices and subparser names (default: ``False``) - * color_ - Allow color output (default: ``False``) + * color_ - Allow color output (default: ``True``) .. versionchanged:: 3.5 *allow_abbrev* parameter was added. @@ -434,12 +434,17 @@ arguments they contain. For example:: >>> parser.parse_args(['-f', 'foo', '@args.txt']) Namespace(f='bar') -Arguments read from a file must by default be one per line (but see also +Arguments read from a file must be one per line by default (but see also :meth:`~ArgumentParser.convert_arg_line_to_args`) and are treated as if they were in the same place as the original file referencing argument on the command line. So in the example above, the expression ``['-f', 'foo', '@args.txt']`` is considered equivalent to the expression ``['-f', 'foo', '-f', 'bar']``. +.. note:: + + Each line is treated as a single argument, so an empty line is read as an + empty string (``''``). + :class:`ArgumentParser` uses :term:`filesystem encoding and error handler` to read the file containing arguments. @@ -598,13 +603,11 @@ subparser names, the feature can be enabled by setting ``suggest_on_error`` to ``True``. Note that this only applies for arguments when the choices specified are strings:: - >>> parser = argparse.ArgumentParser(description='Process some integers.', - suggest_on_error=True) - >>> parser.add_argument('--action', choices=['sum', 'max']) - >>> parser.add_argument('integers', metavar='N', type=int, nargs='+', - ... help='an integer for the accumulator') - >>> parser.parse_args(['--action', 'sumn', 1, 2, 3]) - tester.py: error: argument --action: invalid choice: 'sumn', maybe you meant 'sum'? (choose from 'sum', 'max') + >>> parser = argparse.ArgumentParser(suggest_on_error=True) + >>> parser.add_argument('--action', choices=['debug', 'dryrun']) + >>> parser.parse_args(['--action', 'debugg']) + usage: tester.py [-h] [--action {debug,dryrun}] + tester.py: error: argument --action: invalid choice: 'debugg', maybe you meant 'debug'? (choose from debug, dryrun) If you're writing code that needs to be compatible with older Python versions and want to opportunistically use ``suggest_on_error`` when it's available, you @@ -620,26 +623,30 @@ keyword argument:: color ^^^^^ -By default, the help message is printed in plain text. If you want to allow -color in help messages, you can enable it by setting ``color`` to ``True``:: +By default, the help message is printed in color using `ANSI escape sequences +`__. +If you want plain text help messages, you can disable this :ref:`in your local +environment `, or in the argument parser itself +by setting ``color`` to ``False``:: >>> parser = argparse.ArgumentParser(description='Process some integers.', - ... color=True) + ... color=False) >>> parser.add_argument('--action', choices=['sum', 'max']) >>> parser.add_argument('integers', metavar='N', type=int, nargs='+', ... help='an integer for the accumulator') >>> parser.parse_args(['--help']) -Even if a CLI author has enabled color, it can be -:ref:`controlled using environment variables `. +Note that when ``color=True``, colored output depends on both environment +variables and terminal capabilities. However, if ``color=False``, colored +output is always disabled, even if environment variables like ``FORCE_COLOR`` +are set. -If you're writing code that needs to be compatible with older Python versions -and want to opportunistically use ``color`` when it's available, you -can set it as an attribute after initializing the parser instead of using the -keyword argument:: +.. note:: - >>> parser = argparse.ArgumentParser(description='Process some integers.') - >>> parser.color = True + Error messages will include color codes when redirecting stderr to a + file. To avoid this, set the |NO_COLOR|_ or :envvar:`PYTHON_COLORS` + environment variable (for example, + ``NO_COLOR=1 python script.py 2> errors.txt``). .. versionadded:: 3.14 @@ -683,6 +690,8 @@ The add_argument() method * deprecated_ - Whether or not use of the argument is deprecated. + The method returns an :class:`Action` object representing the argument. + The following sections describe how each of these are used. @@ -724,13 +733,13 @@ By default, :mod:`!argparse` automatically handles the internal naming and display names of arguments, simplifying the process without requiring additional configuration. As such, you do not need to specify the dest_ and metavar_ parameters. -The dest_ parameter defaults to the argument name with underscores ``_`` -replacing hyphens ``-`` . The metavar_ parameter defaults to the -upper-cased name. For example:: +For optional arguments, the dest_ parameter defaults to the argument name, with +underscores ``_`` replacing hyphens ``-``. The metavar_ parameter defaults to +the upper-cased name. For example:: >>> parser = argparse.ArgumentParser(prog='PROG') >>> parser.add_argument('--foo-bar') - >>> parser.parse_args(['--foo-bar', 'FOO-BAR'] + >>> parser.parse_args(['--foo-bar', 'FOO-BAR']) Namespace(foo_bar='FOO-BAR') >>> parser.print_help() usage: [-h] [--foo-bar FOO-BAR] @@ -765,9 +774,9 @@ how the command-line arguments should be handled. The supplied actions are: Namespace(foo=42) * ``'store_true'`` and ``'store_false'`` - These are special cases of - ``'store_const'`` used for storing the values ``True`` and ``False`` - respectively. In addition, they create default values of ``False`` and - ``True`` respectively:: + ``'store_const'`` that respectively store the values ``True`` and ``False`` + with default values of ``False`` and + ``True``:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', action='store_true') @@ -776,19 +785,19 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args('--foo --bar'.split()) Namespace(foo=True, bar=False, baz=True) -* ``'append'`` - This stores a list, and appends each argument value to the - list. It is useful to allow an option to be specified multiple times. - If the default value is non-empty, the default elements will be present - in the parsed value for the option, with any values from the - command line appended after those default values. Example usage:: +* ``'append'`` - This appends each argument value to a list. + It is useful for allowing an option to be specified multiple times. + If the default value is a non-empty list, the parsed value will start + with the default list's elements and any values from the command line + will be appended after those default values. Example usage:: >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action='append') + >>> parser.add_argument('--foo', action='append', default=['0']) >>> parser.parse_args('--foo 1 --foo 2'.split()) - Namespace(foo=['1', '2']) + Namespace(foo=['0', '1', '2']) -* ``'append_const'`` - This stores a list, and appends the value specified by - the const_ keyword argument to the list; note that the const_ keyword +* ``'append_const'`` - This appends the value specified by + the const_ keyword argument to a list; note that the const_ keyword argument defaults to ``None``. The ``'append_const'`` action is typically useful when multiple arguments need to store constants to the same list. For example:: @@ -799,8 +808,8 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args('--str --int'.split()) Namespace(types=[, ]) -* ``'extend'`` - This stores a list and appends each item from the multi-value - argument list to it. +* ``'extend'`` - This appends each item from a multi-value + argument to a list. The ``'extend'`` action is typically used with the nargs_ keyword argument value ``'+'`` or ``'*'``. Note that when nargs_ is ``None`` (the default) or ``'?'``, each @@ -814,7 +823,7 @@ how the command-line arguments should be handled. The supplied actions are: .. versionadded:: 3.8 -* ``'count'`` - This counts the number of times a keyword argument occurs. For +* ``'count'`` - This counts the number of times an argument occurs. For example, this is useful for increasing verbosity levels:: >>> parser = argparse.ArgumentParser() @@ -839,23 +848,11 @@ how the command-line arguments should be handled. The supplied actions are: >>> parser.parse_args(['--version']) PROG 2.0 -Only actions that consume command-line arguments (e.g. ``'store'``, -``'append'`` or ``'extend'``) can be used with positional arguments. - -.. class:: BooleanOptionalAction - - You may also specify an arbitrary action by passing an :class:`Action` subclass or - other object that implements the same interface. The :class:`!BooleanOptionalAction` - is available in :mod:`!argparse` and adds support for boolean actions such as - ``--foo`` and ``--no-foo``:: - - >>> import argparse - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action=argparse.BooleanOptionalAction) - >>> parser.parse_args(['--no-foo']) - Namespace(foo=False) - - .. versionadded:: 3.9 +You may also specify an arbitrary action by passing an :class:`Action` subclass +(e.g. :class:`BooleanOptionalAction`) or other object that implements the same +interface. Only actions that consume command-line arguments (e.g. ``'store'``, +``'append'``, ``'extend'``, or custom actions with non-zero ``nargs``) can be used +with positional arguments. The recommended way to create a custom action is to extend :class:`Action`, overriding the :meth:`!__call__` method and optionally the :meth:`!__init__` and @@ -955,7 +952,7 @@ See also :ref:`specifying-ambiguous-arguments`. The supported values are: .. index:: single: + (plus); in argparse module -* ``'+'``. Just like ``'*'``, all command-line args present are gathered into a +* ``'+'``. Just like ``'*'``, all command-line arguments present are gathered into a list. Additionally, an error message will be generated if there wasn't at least one command-line argument present. For example:: @@ -995,8 +992,8 @@ the various :class:`ArgumentParser` actions. The two most common uses of it are (like ``-f`` or ``--foo``) and ``nargs='?'``. This creates an optional argument that can be followed by zero or one command-line arguments. When parsing the command line, if the option string is encountered with no - command-line argument following it, the value of ``const`` will be assumed to - be ``None`` instead. See the nargs_ description for examples. + command-line argument following it, the value from ``const`` will be used. + See the nargs_ description for examples. .. versionchanged:: 3.11 ``const=None`` by default, including when ``action='append_const'`` or @@ -1050,6 +1047,10 @@ is used when no command-line argument was present:: >>> parser.parse_args([]) Namespace(foo=42) +Because ``nargs='*'`` gathers any supplied values into a list, an absent +positional argument yields an empty list (``[]``). Only a non-``None`` +*default* overrides this (so ``default=None`` still gives ``[]``). + For required_ arguments, the ``default`` value is ignored. For example, this applies to positional arguments with nargs_ values other than ``?`` or ``*``, or optional arguments marked as ``required=True``. @@ -1113,7 +1114,15 @@ User defined functions can be used as well: The :func:`bool` function is not recommended as a type converter. All it does is convert empty strings to ``False`` and non-empty strings to ``True``. -This is usually not what is desired. +This is usually not what is desired:: + + >>> parser = argparse.ArgumentParser() + >>> _ = parser.add_argument('--verbose', type=bool) + >>> parser.parse_args(['--verbose', 'False']) + Namespace(verbose=True) + +See :class:`BooleanOptionalAction` or ``action='store_true'`` for common +alternatives. In general, the ``type`` keyword is a convenience that should only be used for simple conversions that can only raise one of the three supported exceptions. @@ -1156,16 +1165,21 @@ if the argument was not one of the acceptable values:: game.py: error: argument move: invalid choice: 'fire' (choose from 'rock', 'paper', 'scissors') -Note that inclusion in the *choices* sequence is checked after any type_ -conversions have been performed, so the type of the objects in the *choices* -sequence should match the type_ specified. - Any sequence can be passed as the *choices* value, so :class:`list` objects, :class:`tuple` objects, and custom sequences are all supported. Use of :class:`enum.Enum` is not recommended because it is difficult to control its appearance in usage, help, and error messages. +Note that *choices* are checked after any type_ +conversions have been performed, so objects in *choices* +should match the type_ specified. This can make *choices* +appear unfamiliar in usage, help, or error messages. + +To keep *choices* user-friendly, consider a custom type wrapper that +converts and formats values, or omit type_ and handle conversion in +your application code. + Formatted choices override the default *metavar* which is normally derived from *dest*. This is usually what you want because the user never sees the *dest* parameter. If this display isn't desirable (perhaps because there are @@ -1349,6 +1363,10 @@ behavior:: >>> parser.parse_args('--foo XXX'.split()) Namespace(bar='XXX') +Multiple arguments may share the same ``dest``. By default, the value from the +last such argument given on the command line wins. Use ``action='append'`` to +collect values from all of them into a list instead. For conflicting *option +strings* rather than ``dest`` names, see conflict_handler_. .. _deprecated: @@ -1429,6 +1447,21 @@ this API may be passed as the ``action`` parameter to and return a string which will be used when printing the usage of the program. If such method is not provided, a sensible default will be used. +.. class:: BooleanOptionalAction + + A subclass of :class:`Action` for handling boolean flags with positive + and negative options. Adding a single argument such as ``--foo`` automatically + creates both ``--foo`` and ``--no-foo`` options, storing ``True`` and ``False`` + respectively:: + + >>> import argparse + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo', action=argparse.BooleanOptionalAction) + >>> parser.parse_args(['--no-foo']) + Namespace(foo=False) + + .. versionadded:: 3.9 + The parse_args() method ----------------------- @@ -1651,7 +1684,7 @@ The Namespace object Other utilities --------------- -Sub-commands +Subcommands ^^^^^^^^^^^^ .. method:: ArgumentParser.add_subparsers(*, [title], [description], [prog], \ @@ -1680,7 +1713,7 @@ Sub-commands * *description* - description for the sub-parser group in help output, by default ``None`` - * *prog* - usage information that will be displayed with sub-command help, + * *prog* - usage information that will be displayed with subcommand help, by default the name of the program and any positional arguments before the subparser argument @@ -1690,7 +1723,7 @@ Sub-commands * action_ - the basic type of action to be taken when this argument is encountered at the command line - * dest_ - name of the attribute under which sub-command name will be + * dest_ - name of the attribute under which subcommand name will be stored; by default ``None`` and no value is stored * required_ - Whether or not a subcommand must be provided, by default @@ -1729,6 +1762,11 @@ Sub-commands present, and when the ``b`` command is specified, only the ``foo`` and ``baz`` attributes are present. + If a subparser defines an argument with the same ``dest`` as the parent + parser, the two share a single namespace attribute, so the parent's value + won't be retained. Users should give them distinct ``dest`` values to + keep both. + Similarly, when a help message is requested from a subparser, only the help for that particular parser will be printed. The help message will not include parent parser or sibling parser messages. (A help message for each @@ -1908,7 +1946,7 @@ FileType objects run and then use the :keyword:`with`-statement to manage the files. .. versionchanged:: 3.4 - Added the *encodings* and *errors* parameters. + Added the *encoding* and *errors* parameters. .. deprecated:: 3.14 @@ -1970,6 +2008,9 @@ Argument groups Note that any arguments not in your user-defined groups will end up back in the usual "positional arguments" and "optional arguments" sections. + Within each argument group, arguments are displayed in help output in the + order in which they are added. + .. deprecated-removed:: 3.11 3.14 Calling :meth:`add_argument_group` on an argument group now raises an exception. This nesting was never supported, often failed to work @@ -2060,7 +2101,9 @@ Parser defaults >>> parser.parse_args(['736']) Namespace(bar=42, baz='badger', foo=736) - Note that parser-level defaults always override argument-level defaults:: + Note that defaults can be set at both the parser level using :meth:`set_defaults` + and at the argument level using :meth:`add_argument`. If both are called for the + same argument, the last default set for an argument is used:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', default='bar') @@ -2122,12 +2165,15 @@ Partial parsing .. method:: ArgumentParser.parse_known_args(args=None, namespace=None) - Sometimes a script may only parse a few of the command-line arguments, passing - the remaining arguments on to another script or program. In these cases, the - :meth:`~ArgumentParser.parse_known_args` method can be useful. It works much like - :meth:`~ArgumentParser.parse_args` except that it does not produce an error when - extra arguments are present. Instead, it returns a two item tuple containing - the populated namespace and the list of remaining argument strings. + Sometimes a script only needs to handle a specific set of command-line + arguments, leaving any unrecognized arguments for another script or program. + In these cases, the :meth:`~ArgumentParser.parse_known_args` method can be + useful. + + This method works similarly to :meth:`~ArgumentParser.parse_args`, but it does + not raise an error for extra, unrecognized arguments. Instead, it parses the + known arguments and returns a two item tuple that contains the populated + namespace and the list of any unrecognized arguments. :: @@ -2165,6 +2211,9 @@ Customizing file parsing def convert_arg_line_to_args(self, arg_line): return arg_line.split() + Note that with this override an argument can no longer contain spaces, since + each space-separated word becomes a separate argument. + Exiting methods ^^^^^^^^^^^^^^^ diff --git a/Doc/library/array.rst b/Doc/library/array.rst index e0b1eb89cf6c05a..547bdfde87dbb29 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -9,7 +9,7 @@ -------------- This module defines an object type which can compactly represent an array of -basic values: characters, integers, floating-point numbers. Arrays are sequence +basic values: characters, integers, floating-point numbers. Arrays are mutable :term:`sequence` types and behave very much like lists, except that the type of objects stored in them is constrained. The type is specified at object creation time by using a :dfn:`type code`, which is a single character. The following type codes are @@ -24,7 +24,7 @@ defined: +-----------+--------------------+-------------------+-----------------------+-------+ | ``'u'`` | wchar_t | Unicode character | 2 | \(1) | +-----------+--------------------+-------------------+-----------------------+-------+ -| ``'w'`` | Py_UCS4 | Unicode character | 4 | | +| ``'w'`` | Py_UCS4 | Unicode character | 4 | \(2) | +-----------+--------------------+-------------------+-----------------------+-------+ | ``'h'`` | signed short | int | 2 | | +-----------+--------------------+-------------------+-----------------------+-------+ @@ -60,6 +60,16 @@ Notes: .. deprecated-removed:: 3.3 3.16 Please migrate to ``'w'`` typecode. +(2) + .. versionadded:: 3.13 + +.. seealso:: + + The :ref:`ctypes ` and + :ref:`struct ` modules, + as well as third-party modules like `numpy `__, + use similar -- but slightly different -- type codes. + The actual representation of values is determined by the machine architecture (strictly speaking, by the C implementation). The actual size can be accessed @@ -90,12 +100,14 @@ The module defines the following type: otherwise, the initializer's iterator is passed to the :meth:`extend` method to add initial items to the array. - Array objects support the ordinary sequence operations of indexing, slicing, + Array objects support the ordinary :ref:`mutable ` :term:`sequence` operations of indexing, slicing, concatenation, and multiplication. When using slice assignment, the assigned value must be an array object with the same type code; in all other cases, :exc:`TypeError` is raised. Array objects also implement the buffer interface, and may be used wherever :term:`bytes-like objects ` are supported. + Arrays are :ref:`generic ` over the type of their contents. + .. audit-event:: array.__new__ typecode,initializer array.array @@ -109,9 +121,9 @@ The module defines the following type: The length in bytes of one array item in the internal representation. - .. method:: append(x) + .. method:: append(value, /) - Append a new item with value *x* to the end of the array. + Append a new item with the specified value to the end of the array. .. method:: buffer_info() @@ -141,12 +153,12 @@ The module defines the following type: different byte order. - .. method:: count(x) + .. method:: count(value, /) - Return the number of occurrences of *x* in the array. + Return the number of occurrences of *value* in the array. - .. method:: extend(iterable) + .. method:: extend(iterable, /) Append items from *iterable* to the end of the array. If *iterable* is another array, it must have *exactly* the same type code; if not, :exc:`TypeError` will @@ -154,7 +166,7 @@ The module defines the following type: must be the right type to be appended to the array. - .. method:: frombytes(buffer) + .. method:: frombytes(buffer, /) Appends items from the :term:`bytes-like object`, interpreting its content as an array of machine values (as if it had been read @@ -164,7 +176,7 @@ The module defines the following type: :meth:`!fromstring` is renamed to :meth:`frombytes` for clarity. - .. method:: fromfile(f, n) + .. method:: fromfile(f, n, /) Read *n* items (as machine values) from the :term:`file object` *f* and append them to the end of the array. If less than *n* items are available, @@ -172,13 +184,13 @@ The module defines the following type: inserted into the array. - .. method:: fromlist(list) + .. method:: fromlist(list, /) Append items from the list. This is equivalent to ``for x in list: a.append(x)`` except that if there is a type error, the array is unchanged. - .. method:: fromunicode(s) + .. method:: fromunicode(ustr, /) Extends this array with data from the given Unicode string. The array must have type code ``'u'`` or ``'w'``; otherwise a :exc:`ValueError` is raised. @@ -186,33 +198,33 @@ The module defines the following type: array of some other type. - .. method:: index(x[, start[, stop]]) + .. method:: index(value[, start[, stop]]) Return the smallest *i* such that *i* is the index of the first occurrence of - *x* in the array. The optional arguments *start* and *stop* can be - specified to search for *x* within a subsection of the array. Raise - :exc:`ValueError` if *x* is not found. + *value* in the array. The optional arguments *start* and *stop* can be + specified to search for *value* within a subsection of the array. Raise + :exc:`ValueError` if *value* is not found. .. versionchanged:: 3.10 Added optional *start* and *stop* parameters. - .. method:: insert(i, x) + .. method:: insert(index, value, /) - Insert a new item with value *x* in the array before position *i*. Negative + Insert a new item *value* in the array before position *index*. Negative values are treated as being relative to the end of the array. - .. method:: pop([i]) + .. method:: pop(index=-1, /) Removes the item with the index *i* from the array and returns it. The optional argument defaults to ``-1``, so that by default the last item is removed and returned. - .. method:: remove(x) + .. method:: remove(value, /) - Remove the first occurrence of *x* from the array. + Remove the first occurrence of *value* from the array. .. method:: clear() @@ -237,7 +249,7 @@ The module defines the following type: :meth:`!tostring` is renamed to :meth:`tobytes` for clarity. - .. method:: tofile(f) + .. method:: tofile(f, /) Write all items (as machine values) to the :term:`file object` *f*. diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index ca9a6b0712c9a28..5338d407ca22596 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -15,7 +15,7 @@ -------------- -The :mod:`ast` module helps Python applications to process trees of the Python +The :mod:`!ast` module helps Python applications to process trees of the Python abstract syntax grammar. The abstract syntax itself might change with each Python release; this module helps to find out programmatically what the current grammar looks like. @@ -46,7 +46,7 @@ Node classes This is the base of all AST node classes. The actual node classes are derived from the :file:`Parser/Python.asdl` file, which is reproduced :ref:`above `. They are defined in the :mod:`!_ast` C - module and re-exported in :mod:`ast`. + module and re-exported in :mod:`!ast`. There is one class defined for each left-hand side symbol in the abstract grammar (for example, :class:`ast.stmt` or :class:`ast.expr`). In addition, @@ -134,17 +134,26 @@ Node classes Simple indices are represented by their value, extended slices are represented as tuples. +.. versionchanged:: 3.13 + + AST node constructors were changed to provide sensible defaults for omitted + fields: optional fields now default to ``None``, list fields default to an + empty list, and fields of type :class:`!ast.expr_context` default to + :class:`Load() `. Previously, omitted attributes would not exist on constructed + nodes (accessing them raised :exc:`AttributeError`). + .. versionchanged:: 3.14 The :meth:`~object.__repr__` output of :class:`~ast.AST` nodes includes the values of the node fields. -.. deprecated:: 3.8 +.. deprecated-removed:: 3.8 3.14 - Old classes :class:`!ast.Num`, :class:`!ast.Str`, :class:`!ast.Bytes`, - :class:`!ast.NameConstant` and :class:`!ast.Ellipsis` are still available, - but they will be removed in future Python releases. In the meantime, - instantiating them will return an instance of a different class. + Previous versions of Python provided the AST classes :class:`!ast.Num`, + :class:`!ast.Str`, :class:`!ast.Bytes`, :class:`!ast.NameConstant` and + :class:`!ast.Ellipsis`, which were deprecated in Python 3.8. These classes + were removed in Python 3.14, and their functionality has been replaced with + :class:`ast.Constant`. .. deprecated:: 3.9 @@ -265,18 +274,25 @@ Root nodes Literals ^^^^^^^^ -.. class:: Constant(value) +.. class:: Constant(value, kind) A constant value. The ``value`` attribute of the ``Constant`` literal contains the - Python object it represents. The values represented can be simple types - such as a number, string or ``None``, but also immutable container types - (tuples and frozensets) if all of their elements are constant. + Python object it represents. The values represented can be instances of :class:`str`, + :class:`bytes`, :class:`int`, :class:`float`, :class:`complex`, and :class:`bool`, + and the constants :data:`None` and :data:`Ellipsis`. + + The ``kind`` attribute is an optional string. For string literals with a + ``u`` prefix, ``kind`` is set to ``'u'``. For all other + constants, ``kind`` is ``None``. .. doctest:: >>> print(ast.dump(ast.parse('123', mode='eval'), indent=4)) Expression( body=Constant(value=123)) + >>> print(ast.dump(ast.parse("u'hello'", mode='eval'), indent=4)) + Expression( + body=Constant(value='hello', kind='u')) .. class:: FormattedValue(value, conversion, format_spec) @@ -290,9 +306,9 @@ Literals * ``conversion`` is an integer: * -1: no formatting - * 115: ``!s`` string formatting - * 114: ``!r`` repr formatting - * 97: ``!a`` ascii formatting + * 97 (``ord('a')``): ``!a`` :func:`ASCII ` formatting + * 114 (``ord('r')``): ``!r`` :func:`repr` formatting + * 115 (``ord('s')``): ``!s`` :func:`string ` formatting * ``format_spec`` is a :class:`JoinedStr` node representing the formatting of the value, or ``None`` if no format was specified. Both @@ -326,6 +342,63 @@ Literals Constant(value='.3')]))])) +.. class:: TemplateStr(values, /) + + .. versionadded:: 3.14 + + Node representing a template string literal, comprising a series of + :class:`Interpolation` and :class:`Constant` nodes. + These nodes may be any order, and do not need to be interleaved. + + .. doctest:: + + >>> expr = ast.parse('t"{name} finished {place:ordinal}"', mode='eval') + >>> print(ast.dump(expr, indent=4)) + Expression( + body=TemplateStr( + values=[ + Interpolation( + value=Name(id='name', ctx=Load()), + str='name', + conversion=-1), + Constant(value=' finished '), + Interpolation( + value=Name(id='place', ctx=Load()), + str='place', + conversion=-1, + format_spec=JoinedStr( + values=[ + Constant(value='ordinal')]))])) + +.. class:: Interpolation(value, str, conversion, format_spec=None) + + .. versionadded:: 3.14 + + Node representing a single interpolation field in a template string literal. + + * ``value`` is any expression node (such as a literal, a variable, or a + function call). + This has the same meaning as ``FormattedValue.value``. + * ``str`` is a constant containing the text of the interpolation expression. + + If ``str`` is set to ``None``, then ``value`` is used to generate code + when calling :func:`ast.unparse`. This no longer guarantees that the + generated code is identical to the original and is intended for code + generation. + * ``conversion`` is an integer: + + * -1: no conversion + * 97 (``ord('a')``): ``!a`` :func:`ASCII ` conversion + * 114 (``ord('r')``): ``!r`` :func:`repr` conversion + * 115 (``ord('s')``): ``!s`` :func:`string ` conversion + + This has the same meaning as ``FormattedValue.conversion``. + * ``format_spec`` is a :class:`JoinedStr` node representing the formatting + of the value, or ``None`` if no format was specified. Both + ``conversion`` and ``format_spec`` can be set at the same time. + This has the same meaning as ``FormattedValue.format_spec``. + + .. class:: List(elts, ctx) Tuple(elts, ctx) @@ -1291,7 +1364,7 @@ Control flow ``try`` blocks which are followed by ``except*`` clauses. The attributes are the same as for :class:`Try` but the :class:`ExceptHandler` nodes in ``handlers`` - are interpreted as ``except*`` blocks rather then ``except``. + are interpreted as ``except*`` blocks rather than ``except``. .. doctest:: @@ -2159,10 +2232,10 @@ Async and await occurrences of the same value (for example, :class:`ast.Add`). -:mod:`ast` helpers ------------------- +:mod:`!ast` helpers +------------------- -Apart from the node classes, the :mod:`ast` module defines these utility functions +Apart from the node classes, the :mod:`!ast` module defines these utility functions and classes for traversing abstract syntax trees: .. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None, optimize=-1) @@ -2376,12 +2449,12 @@ and classes for traversing abstract syntax trees: during traversal. For this a special visitor exists (:class:`NodeTransformer`) that allows modifications. - .. deprecated:: 3.8 + .. deprecated-removed:: 3.8 3.14 Methods :meth:`!visit_Num`, :meth:`!visit_Str`, :meth:`!visit_Bytes`, - :meth:`!visit_NameConstant` and :meth:`!visit_Ellipsis` are deprecated - now and will not be called in future Python versions. Add the - :meth:`visit_Constant` method to handle all constant nodes. + :meth:`!visit_NameConstant` and :meth:`!visit_Ellipsis` will not be called + in Python 3.14+. Add the :meth:`visit_Constant` method instead to handle + all constant nodes. .. class:: NodeTransformer() @@ -2445,8 +2518,9 @@ and classes for traversing abstract syntax trees: indents that many spaces per level. If *indent* is a string (such as ``"\t"``), that string is used to indent each level. - If *show_empty* is ``False`` (the default), empty lists and fields that are ``None`` - will be omitted from the output. + If *show_empty* is false (the default), optional empty lists will be + omitted from the output. + Optional ``None`` values are always omitted. .. versionchanged:: 3.9 Added the *indent* option. @@ -2482,6 +2556,20 @@ and classes for traversing abstract syntax trees: type_ignores=[]) +.. function:: compare(a, b, /, *, compare_attributes=False) + + Recursively compares two ASTs. + + *compare_attributes* affects whether AST attributes are considered + in the comparison. If *compare_attributes* is ``False`` (default), then + attributes are ignored. Otherwise they must all be equal. This + option is useful to check whether the ASTs are structurally equal but + differ in whitespace or similar details. Attributes include line numbers + and column offsets. + + .. versionadded:: 3.14 + + .. _ast-compiler-flags: Compiler flags @@ -2517,20 +2605,6 @@ effects on the compilation of a program: .. versionadded:: 3.8 -.. function:: compare(a, b, /, *, compare_attributes=False) - - Recursively compares two ASTs. - - *compare_attributes* affects whether AST attributes are considered - in the comparison. If *compare_attributes* is ``False`` (default), then - attributes are ignored. Otherwise they must all be equal. This - option is useful to check whether the ASTs are structurally equal but - differ in whitespace or similar details. Attributes include line numbers - and column offsets. - - .. versionadded:: 3.14 - - .. _ast-cli: Command-line usage @@ -2538,7 +2612,7 @@ Command-line usage .. versionadded:: 3.9 -The :mod:`ast` module can be executed as a script from the command line. +The :mod:`!ast` module can be executed as a script from the command line. It is as simple as: .. code-block:: sh diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index 44b507a98111167..f3409bcd2df6489 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -46,10 +46,6 @@ In addition to enabling the debug mode, consider also: When the debug mode is enabled: -* asyncio checks for :ref:`coroutines that were not awaited - ` and logs them; this mitigates - the "forgotten await" pitfall. - * Many non-threadsafe asyncio APIs (such as :meth:`loop.call_soon` and :meth:`loop.call_at` methods) raise an exception if they are called from a wrong thread. @@ -252,3 +248,225 @@ Output in debug mode:: File "../t.py", line 4, in bug raise Exception("not consumed") Exception: not consumed + + +Asynchronous generators best practices +====================================== + +Writing correct and efficient asyncio code requires awareness of certain pitfalls. +This section outlines essential best practices that can save you hours of debugging. + + +Close asynchronous generators explicitly +---------------------------------------- + +It is recommended to manually close the +:term:`asynchronous generator `. If a generator +exits early - for example, due to an exception raised in the body of +an ``async for`` loop - its asynchronous cleanup code may run in an +unexpected context. This can occur after the tasks it depends on have completed, +or during the event loop shutdown when the async-generator's garbage collection +hook is called. + +To avoid this, explicitly close the generator by calling its +:meth:`~agen.aclose` method, or use the :func:`contextlib.aclosing` +context manager:: + + import asyncio + import contextlib + + async def gen(): + yield 1 + yield 2 + + async def func(): + async with contextlib.aclosing(gen()) as g: + async for x in g: + break # Don't iterate until the end + + asyncio.run(func()) + +As noted above, the cleanup code for these asynchronous generators is deferred. +The following example demonstrates that the finalization of an asynchronous +generator can occur in an unexpected order:: + + import asyncio + work_done = False + + async def cursor(): + try: + yield 1 + finally: + assert work_done + + async def rows(): + global work_done + try: + yield 2 + finally: + await asyncio.sleep(0.1) # immitate some async work + work_done = True + + + async def main(): + async for c in cursor(): + async for r in rows(): + break + break + + asyncio.run(main()) + +For this example, we get the following output:: + + unhandled exception during asyncio.run() shutdown + task: ()> exception=AssertionError()> + Traceback (most recent call last): + File "example.py", line 6, in cursor + yield 1 + asyncio.exceptions.CancelledError + + During handling of the above exception, another exception occurred: + + Traceback (most recent call last): + File "example.py", line 8, in cursor + assert work_done + ^^^^^^^^^ + AssertionError + +The ``cursor()`` asynchronous generator was finalized before the ``rows`` +generator - an unexpected behavior. + +The example can be fixed by explicitly closing the +``cursor`` and ``rows`` async-generators:: + + async def main(): + async with contextlib.aclosing(cursor()) as cursor_gen: + async for c in cursor_gen: + async with contextlib.aclosing(rows()) as rows_gen: + async for r in rows_gen: + break + break + + +Create asynchronous generators only when the event loop is running +------------------------------------------------------------------ + +It is recommended to create +:term:`asynchronous generators ` only after +the event loop has been created. + +To ensure that asynchronous generators close reliably, the event loop uses the +:func:`sys.set_asyncgen_hooks` function to register callback functions. These +callbacks update the list of running asynchronous generators to keep it in a +consistent state. + +When the :meth:`loop.shutdown_asyncgens() ` +function is called, the running generators are stopped gracefully and the +list is cleared. + +The asynchronous generator invokes the corresponding system hook during its +first iteration. At the same time, the generator records that the hook has +been called and does not call it again. + +Therefore, if iteration begins before the event loop is created, +the event loop will not be able to add the generator to its list of active +generators because the hooks are set after the generator attempts to call them. +Consequently, the event loop will not be able to terminate the generator +if necessary. + +Consider the following example:: + + import asyncio + + async def agenfn(): + try: + yield 10 + finally: + await asyncio.sleep(0) + + + with asyncio.Runner() as runner: + agen = agenfn() + print(runner.run(anext(agen))) + del agen + +Output:: + + 10 + Exception ignored while closing generator : + Traceback (most recent call last): + File "example.py", line 13, in + del agen + ^^^^ + RuntimeError: async generator ignored GeneratorExit + +This example can be fixed as follows:: + + import asyncio + + async def agenfn(): + try: + yield 10 + finally: + await asyncio.sleep(0) + + async def main(): + agen = agenfn() + print(await anext(agen)) + del agen + + asyncio.run(main()) + + +Avoid concurrent iteration and closure of the same generator +------------------------------------------------------------ + +Async generators may be reentered while another +:meth:`~agen.__anext__` / :meth:`~agen.athrow` / :meth:`~agen.aclose` call is in +progress. This may lead to an inconsistent state of the async generator and can +cause errors. + +Let's consider the following example:: + + import asyncio + + async def consumer(): + for idx in range(100): + await asyncio.sleep(0) + message = yield idx + print('received', message) + + async def amain(): + agenerator = consumer() + await agenerator.asend(None) + + fa = asyncio.create_task(agenerator.asend('A')) + fb = asyncio.create_task(agenerator.asend('B')) + await fa + await fb + + asyncio.run(amain()) + +Output:: + + received A + Traceback (most recent call last): + File "test.py", line 38, in + asyncio.run(amain()) + ~~~~~~~~~~~^^^^^^^^^ + File "Lib/asyncio/runners.py", line 204, in run + return runner.run(main) + ~~~~~~~~~~^^^^^^ + File "Lib/asyncio/runners.py", line 127, in run + return self._loop.run_until_complete(task) + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^ + File "Lib/asyncio/base_events.py", line 719, in run_until_complete + return future.result() + ~~~~~~~~~~~~~^^ + File "test.py", line 36, in amain + await fb + RuntimeError: anext(): asynchronous generator is already running + + +Therefore, it is recommended to avoid using asynchronous generators in parallel +tasks or across multiple event loops. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 21f7d0547af1dd5..1caecb1c143a238 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -4,7 +4,7 @@ .. _asyncio-event-loop: ========== -Event Loop +Event loop ========== **Source code:** :source:`Lib/asyncio/events.py`, @@ -105,7 +105,7 @@ This documentation page contains the following sections: .. _asyncio-event-loop-methods: -Event Loop Methods +Event loop methods ================== Event loops have **low-level** APIs for the following: @@ -297,13 +297,20 @@ clocks to track time. are called is undefined. The optional positional *args* will be passed to the callback when - it is called. If you want the callback to be called with keyword - arguments use :func:`functools.partial`. + it is called. Use :func:`functools.partial` + :ref:`to pass keyword arguments ` to + *callback*. An optional keyword-only *context* argument allows specifying a custom :class:`contextvars.Context` for the *callback* to run in. The current context is used when no *context* is provided. + .. note:: + + For performance, callbacks scheduled with :meth:`loop.call_later` + may run up to one clock-resolution early (see + ``time.get_clock_info('monotonic').resolution``). + .. versionchanged:: 3.7 The *context* keyword-only parameter was added. See :pep:`567` for more details. @@ -324,6 +331,12 @@ clocks to track time. An instance of :class:`asyncio.TimerHandle` is returned which can be used to cancel the callback. + .. note:: + + For performance, callbacks scheduled with :meth:`loop.call_at` + may run up to one clock-resolution early (see + ``time.get_clock_info('monotonic').resolution``). + .. versionchanged:: 3.7 The *context* keyword-only parameter was added. See :pep:`567` for more details. @@ -348,7 +361,7 @@ clocks to track time. The :func:`asyncio.sleep` function. -Creating Futures and Tasks +Creating futures and tasks ^^^^^^^^^^^^^^^^^^^^^^^^^^ .. method:: loop.create_future() @@ -361,7 +374,7 @@ Creating Futures and Tasks .. versionadded:: 3.5.2 -.. method:: loop.create_task(coro, *, name=None, context=None, eager_start=None) +.. method:: loop.create_task(coro, *, name=None, context=None, eager_start=None, **kwargs) Schedule the execution of :ref:`coroutine ` *coro*. Return a :class:`Task` object. @@ -370,6 +383,10 @@ Creating Futures and Tasks for interoperability. In this case, the result type is a subclass of :class:`Task`. + The full function signature is largely the same as that of the + :class:`Task` constructor (or factory) - all of the keyword arguments to + this function are passed through to that interface. + If the *name* argument is provided and not ``None``, it is set as the name of the task using :meth:`Task.set_name`. @@ -388,8 +405,15 @@ Creating Futures and Tasks .. versionchanged:: 3.11 Added the *context* parameter. + .. versionchanged:: 3.13.3 + Added ``kwargs`` which passes on arbitrary extra parameters, including ``name`` and ``context``. + + .. versionchanged:: 3.13.4 + Rolled back the change that passes on *name* and *context* (if it is None), + while still passing on other arbitrary keyword arguments (to avoid breaking backwards compatibility with 3.13.3). + .. versionchanged:: 3.14 - Added the *eager_start* parameter. + All *kwargs* are now passed on. The *eager_start* parameter works with eager task factories. .. method:: loop.set_task_factory(factory) @@ -402,6 +426,16 @@ Creating Futures and Tasks event loop, and *coro* is a coroutine object. The callable must pass on all *kwargs*, and return a :class:`asyncio.Task`-compatible object. + .. versionchanged:: 3.13.3 + Required that all *kwargs* are passed on to :class:`asyncio.Task`. + + .. versionchanged:: 3.13.4 + *name* is no longer passed to task factories. *context* is no longer passed + to task factories if it is ``None``. + + .. versionchanged:: 3.14 + *name* and *context* are now unconditionally passed on to task factories again. + .. method:: loop.get_task_factory() Return a task factory or ``None`` if the default one is in use. @@ -590,6 +624,12 @@ Opening network connections to bind the socket locally. The *local_host* and *local_port* are looked up using :meth:`getaddrinfo`. + .. note:: + + On Windows, when using the proactor event loop with ``local_addr=None``, + an :exc:`OSError` with :attr:`!errno.WSAEINVAL` will be raised + when running it. + * *remote_addr*, if given, is a ``(remote_host, remote_port)`` tuple used to connect the socket to a remote address. The *remote_host* and *remote_port* are looked up using :meth:`getaddrinfo`. @@ -922,7 +962,7 @@ Transferring files .. versionadded:: 3.7 -TLS Upgrade +TLS upgrade ^^^^^^^^^^^ .. method:: loop.start_tls(transport, protocol, \ @@ -995,8 +1035,8 @@ Watching file descriptors .. method:: loop.add_writer(fd, callback, *args) Start monitoring the *fd* file descriptor for write availability and - invoke *callback* with the specified arguments once *fd* is available for - writing. + invoke *callback* with the specified arguments *args* once *fd* is + available for writing. Any preexisting callback registered for *fd* is cancelled and replaced by *callback*. @@ -1116,12 +1156,21 @@ convenient. *sock* must be a non-blocking socket. + With :class:`SelectorEventLoop`, *address* does not need to be resolved: + for :const:`~socket.AF_INET` and :const:`~socket.AF_INET6` sockets, + ``sock_connect`` first checks whether *address* is already resolved by + calling :func:`socket.inet_pton`, and uses :meth:`loop.getaddrinfo` to + resolve it if it is not. + + :class:`ProactorEventLoop`, the default event loop on Windows, does not + resolve *address*. The host must already be a numeric IP address; passing + a host name raises :exc:`OSError`. Resolve the address with + :meth:`loop.getaddrinfo` first, or use :meth:`loop.create_connection`, + which resolves the address on every platform. + .. versionchanged:: 3.5.2 - ``address`` no longer needs to be resolved. ``sock_connect`` - will try to check if the *address* is already resolved by calling - :func:`socket.inet_pton`. If not, - :meth:`loop.getaddrinfo` will be used to resolve the - *address*. + With :class:`SelectorEventLoop`, ``address`` no longer needs to be + resolved. .. seealso:: @@ -1225,7 +1274,9 @@ Working with pipes *protocol_factory* must be a callable returning an :ref:`asyncio protocol ` implementation. - *pipe* is a :term:`file-like object `. + *pipe* is a :term:`file-like object `. See + :ref:`Supported pipe objects ` for the objects + supported as *pipe*. Return pair ``(transport, protocol)``, where *transport* supports the :class:`ReadTransport` interface and *protocol* is an object @@ -1242,7 +1293,9 @@ Working with pipes *protocol_factory* must be a callable returning an :ref:`asyncio protocol ` implementation. - *pipe* is :term:`file-like object `. + *pipe* is a :term:`file-like object `. See + :ref:`Supported pipe objects ` for the objects + supported as *pipe*. Return pair ``(transport, protocol)``, where *transport* supports :class:`WriteTransport` interface and *protocol* is an object @@ -1251,6 +1304,33 @@ Working with pipes With :class:`SelectorEventLoop` event loop, the *pipe* is set to non-blocking mode. +.. _asyncio-pipe-objects: + +.. rubric:: Supported pipe objects + +These methods only work with objects the operating system can poll for +readiness or perform overlapped I/O on. Regular files on disk are **not** +supported on any platform. There is no asynchronous file I/O in asyncio; +use :meth:`loop.run_in_executor` to read and write regular files without +blocking the event loop. + +On Unix, with :class:`SelectorEventLoop`, *pipe* must wrap one of the +following: + +* a pipe, such as an end of an :func:`os.pipe` pair or a FIFO created with + :func:`os.mkfifo`; +* a socket; +* a character device, such as a terminal. + +On Windows, where only :class:`ProactorEventLoop` implements these methods, +*pipe* must wrap a handle opened for overlapped I/O (that is, created with the +``FILE_FLAG_OVERLAPPED`` flag), since the handle has to be associated with an +I/O completion port. Handles that were not opened for overlapped I/O are +rejected. In particular, the standard streams (:data:`sys.stdin`, +:data:`sys.stdout` and :data:`sys.stderr`), console handles, and the pipes +created by :func:`os.pipe` are **not** opened for overlapped I/O and therefore +cannot be used with these methods. + .. note:: :class:`SelectorEventLoop` does not support the above methods on @@ -1269,7 +1349,8 @@ Unix signals .. method:: loop.add_signal_handler(signum, callback, *args) - Set *callback* as the handler for the *signum* signal. + Set *callback* as the handler for the *signum* signal, + passing *args* as positional arguments. The callback will be invoked by *loop*, along with other queued callbacks and runnable coroutines of that event loop. Unlike signal handlers @@ -1304,7 +1385,8 @@ Executing code in thread or process pools .. awaitablemethod:: loop.run_in_executor(executor, func, *args) - Arrange for *func* to be called in the specified executor. + Arrange for *func* to be called in the specified executor + passing *args* as positional arguments. The *executor* argument should be an :class:`concurrent.futures.Executor` instance. The default executor is used if *executor* is ``None``. @@ -1389,7 +1471,7 @@ Executing code in thread or process pools :class:`~concurrent.futures.ThreadPoolExecutor`. -Error Handling API +Error handling API ^^^^^^^^^^^^^^^^^^ Allows customizing how exceptions are handled in the event loop. @@ -1492,7 +1574,7 @@ Enabling debug mode The :ref:`debug mode of asyncio `. -Running Subprocesses +Running subprocesses ^^^^^^^^^^^^^^^^^^^^ Methods described in this subsections are low-level. In regular @@ -1592,6 +1674,9 @@ async/await code consider using the high-level conforms to the :class:`asyncio.SubprocessTransport` base class and *protocol* is an object instantiated by the *protocol_factory*. + If the transport is closed or is garbage collected, the child process + is killed if it is still running. + .. method:: loop.subprocess_shell(protocol_factory, cmd, *, \ stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ stderr=subprocess.PIPE, **kwargs) @@ -1615,6 +1700,9 @@ async/await code consider using the high-level conforms to the :class:`SubprocessTransport` base class and *protocol* is an object instantiated by the *protocol_factory*. + If the transport is closed or is garbage collected, the child process + is killed if it is still running. + .. note:: It is the application's responsibility to ensure that all whitespace and special characters are quoted appropriately to avoid `shell injection @@ -1624,7 +1712,7 @@ async/await code consider using the high-level are going to be used to construct shell commands. -Callback Handles +Callback handles ================ .. class:: Handle @@ -1667,7 +1755,7 @@ Callback Handles .. versionadded:: 3.7 -Server Objects +Server objects ============== Server objects are created by :meth:`loop.create_server`, @@ -1810,7 +1898,7 @@ Do not instantiate the :class:`Server` class directly. .. _asyncio-event-loops: .. _asyncio-event-loop-implementations: -Event Loop Implementations +Event loop implementations ========================== asyncio ships with two different event loop implementations: @@ -1923,10 +2011,10 @@ callback uses the :meth:`loop.call_later` method to reschedule itself after 5 seconds, and then stops the event loop:: import asyncio - import datetime + import datetime as dt def display_date(end_time, loop): - print(datetime.datetime.now()) + print(dt.datetime.now()) if (loop.time() + 1.0) < end_time: loop.call_later(1, display_date, end_time, loop) else: @@ -2007,7 +2095,7 @@ Wait until a file descriptor received some data using the Set signal handlers for SIGINT and SIGTERM ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -(This ``signals`` example only works on Unix.) +(This ``signal`` example only works on Unix.) Register handlers for signals :const:`~signal.SIGINT` and :const:`~signal.SIGTERM` using the :meth:`loop.add_signal_handler` method:: diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index 32771ba72e00026..195d99123dbd367 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -75,6 +75,7 @@ Future Functions Deprecation warning is emitted if *future* is not a Future-like object and *loop* is not specified and there is no running event loop. +.. _asyncio-future-obj: Future Object ============= @@ -100,6 +101,8 @@ Future Object implementations can inject their own optimized implementations of a Future object. + Futures are :ref:`generic ` over the type of their results. + .. versionchanged:: 3.7 Added support for the :mod:`contextvars` module. @@ -195,6 +198,10 @@ Future Object Otherwise, change the Future's state to *cancelled*, schedule the callbacks, and return ``True``. + The optional string argument *msg* is passed as the argument to the + :exc:`CancelledError` exception raised when a cancelled Future + is awaited. + .. versionchanged:: 3.9 Added the *msg* parameter. diff --git a/Doc/library/asyncio-graph.rst b/Doc/library/asyncio-graph.rst index 5f642a32bf75c24..1a8cdcf6a9e8cba 100644 --- a/Doc/library/asyncio-graph.rst +++ b/Doc/library/asyncio-graph.rst @@ -4,7 +4,7 @@ .. _asyncio-graph: ======================== -Call Graph Introspection +Call graph introspection ======================== **Source code:** :source:`Lib/asyncio/graph.py` @@ -17,6 +17,12 @@ a suspended *future*. These utilities and the underlying machinery can be used from within a Python program or by external profilers and debuggers. +.. seealso:: + + :ref:`asyncio-introspection-tools` + Command-line tools for inspecting tasks in another running Python + process. + .. versionadded:: 3.14 diff --git a/Doc/library/asyncio-platforms.rst b/Doc/library/asyncio-platforms.rst index a2a3114ad6e4c5d..0b7c17a2c584795 100644 --- a/Doc/library/asyncio-platforms.rst +++ b/Doc/library/asyncio-platforms.rst @@ -19,6 +19,11 @@ All Platforms * :meth:`loop.add_reader` and :meth:`loop.add_writer` cannot be used to monitor file I/O. +* :meth:`loop.connect_read_pipe` and :meth:`loop.connect_write_pipe` + cannot be used with regular files. See :ref:`Supported pipe objects + ` for the objects that are accepted on each + platform. + Windows ======= @@ -62,6 +67,10 @@ All event loops on Windows do not support the following methods: * The :meth:`loop.add_reader` and :meth:`loop.add_writer` methods are not supported. +* :meth:`loop.connect_read_pipe` and :meth:`loop.connect_write_pipe` only + accept a handle opened for overlapped I/O. + See :ref:`Supported pipe objects ` for which objects are supported. + The resolution of the monotonic clock on Windows is usually around 15.6 milliseconds. The best resolution is 0.5 milliseconds. The resolution depends on the hardware (availability of `HPET diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 7c08d65f26bc27e..58f77feb3119841 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -390,11 +390,11 @@ Subprocess Transports Return the transport for the communication pipe corresponding to the integer file descriptor *fd*: - * ``0``: readable streaming transport of the standard input (*stdin*), + * ``0``: writable streaming transport of the standard input (*stdin*), or :const:`None` if the subprocess was not created with ``stdin=PIPE`` - * ``1``: writable streaming transport of the standard output (*stdout*), + * ``1``: readable streaming transport of the standard output (*stdout*), or :const:`None` if the subprocess was not created with ``stdout=PIPE`` - * ``2``: writable streaming transport of the standard error (*stderr*), + * ``2``: readable streaming transport of the standard error (*stderr*), or :const:`None` if the subprocess was not created with ``stderr=PIPE`` * other *fd*: :const:`None` @@ -1037,7 +1037,7 @@ The subprocess is created by the :meth:`loop.subprocess_exec` method:: # low-level APIs. loop = asyncio.get_running_loop() - code = 'import datetime; print(datetime.datetime.now())' + code = 'import datetime as dt; print(dt.datetime.now())' exit_future = asyncio.Future(loop=loop) # Create the subprocess controlled by DateProtocol; diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst index d99213aa81d53e0..5e3f531f4e482e4 100644 --- a/Doc/library/asyncio-queue.rst +++ b/Doc/library/asyncio-queue.rst @@ -71,6 +71,8 @@ Queue Return an item if one is immediately available, else raise :exc:`QueueEmpty`. + Raises :exc:`QueueShutDown` if the queue has been shut down and is empty. + .. method:: join() :async: @@ -96,23 +98,42 @@ Queue If no free slot is immediately available, raise :exc:`QueueFull`. + Raises :exc:`QueueShutDown` if the queue has been shut down. + .. method:: qsize() Return the number of items in the queue. .. method:: shutdown(immediate=False) - Shut down the queue, making :meth:`~Queue.get` and :meth:`~Queue.put` + Put a :class:`Queue` instance into a shutdown mode. + + The queue can no longer grow. + Future calls to :meth:`~Queue.put` raise :exc:`QueueShutDown`. + Currently blocked callers of :meth:`~Queue.put` will be unblocked + and will raise :exc:`QueueShutDown` in the formerly awaiting task. + + If *immediate* is false (the default), the queue can be wound + down normally with :meth:`~Queue.get` calls to extract tasks + that have already been loaded. + + And if :meth:`~Queue.task_done` is called for each remaining task, a + pending :meth:`~Queue.join` will be unblocked normally. + + Once the queue is empty, future calls to :meth:`~Queue.get` will raise :exc:`QueueShutDown`. - By default, :meth:`~Queue.get` on a shut down queue will only - raise once the queue is empty. Set *immediate* to true to make - :meth:`~Queue.get` raise immediately instead. + If *immediate* is true, the queue is terminated immediately. + The queue is drained to be completely empty and the count + of unfinished tasks is reduced by the number of tasks drained. + If unfinished tasks is zero, callers of :meth:`~Queue.join` + are unblocked. Also, blocked callers of :meth:`~Queue.get` + are unblocked and will raise :exc:`QueueShutDown` because the + queue is empty. - All blocked callers of :meth:`~Queue.put` and :meth:`~Queue.get` - will be unblocked. If *immediate* is true, a task will be marked - as done for each remaining item in the queue, which may unblock - callers of :meth:`~Queue.join`. + Use caution when using :meth:`~Queue.join` with *immediate* set + to true. This unblocks the join even when no work has been done + on the tasks, violating the usual invariant for joining a queue. .. versionadded:: 3.13 @@ -129,9 +150,6 @@ Queue call was received for every item that had been :meth:`~Queue.put` into the queue). - ``shutdown(immediate=True)`` calls :meth:`task_done` for each - remaining item in the queue. - Raises :exc:`ValueError` if called more times than there were items placed in the queue. @@ -174,8 +192,9 @@ Exceptions .. exception:: QueueShutDown - Exception raised when :meth:`~Queue.put` or :meth:`~Queue.get` is - called on a queue which has been shut down. + Exception raised when :meth:`~Queue.put`, :meth:`~Queue.put_nowait`, + :meth:`~Queue.get` or :meth:`~Queue.get_nowait` is called + on a queue which has been shut down. .. versionadded:: 3.13 diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index c56166cabb90936..05445219510ca54 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -171,13 +171,17 @@ and work with streams: .. function:: start_unix_server(client_connected_cb, path=None, \ *, limit=None, sock=None, backlog=100, ssl=None, \ ssl_handshake_timeout=None, \ - ssl_shutdown_timeout=None, start_serving=True) + ssl_shutdown_timeout=None, start_serving=True, cleanup_socket=True) :async: Start a Unix socket server. Similar to :func:`start_server` but works with Unix sockets. + If *cleanup_socket* is true then the Unix socket will automatically + be removed from the filesystem when the server is closed, unless the + socket has been replaced after the server has been created. + See also the documentation of :meth:`loop.create_unix_server`. .. note:: @@ -198,6 +202,9 @@ and work with streams: .. versionchanged:: 3.11 Added the *ssl_shutdown_timeout* parameter. + .. versionchanged:: 3.13 + Added the *cleanup_socket* parameter. + StreamReader ============ @@ -309,11 +316,15 @@ StreamWriter If that fails, the data is queued in an internal write buffer until it can be sent. + The *data* buffer should be a bytes, bytearray, or C-contiguous one-dimensional + memoryview object. + The method should be used along with the ``drain()`` method:: stream.write(data) await stream.drain() + .. method:: writelines(data) The method writes a list (or any iterable) of bytes to the underlying socket diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index 03e76bc868905ee..a6514649bf9a0a8 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -76,6 +76,9 @@ Creating Subprocesses See the documentation of :meth:`loop.subprocess_exec` for other parameters. + If the process object is garbage collected while the process is still + running, the child process will be killed. + .. versionchanged:: 3.10 Removed the *loop* parameter. @@ -95,6 +98,9 @@ Creating Subprocesses See the documentation of :meth:`loop.subprocess_shell` for other parameters. + If the process object is garbage collected while the process is still + running, the child process will be killed. + .. important:: It is the application's responsibility to ensure that all whitespace and @@ -305,8 +311,16 @@ their completion. A ``None`` value indicates that the process has not terminated yet. - A negative value ``-N`` indicates that the child was terminated - by signal ``N`` (POSIX only). + For processes created with :func:`~asyncio.create_subprocess_exec`, a negative + value ``-N`` indicates that the child was terminated by signal ``N`` + (POSIX only). + + For processes created with :func:`~asyncio.create_subprocess_shell`, the + return code reflects the exit status of the shell itself (e.g. ``/bin/sh``), + which may map signals to codes such as ``128+N``. See the + documentation of the shell (for example, the Bash manual's Exit Status) + for details. + .. _asyncio-subprocess-threads: @@ -345,7 +359,7 @@ function:: import sys async def get_date(): - code = 'import datetime; print(datetime.datetime.now())' + code = 'import datetime as dt; print(dt.datetime.now())' # Create the subprocess; redirect the standard output # into a pipe. diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 968c812ee3c8e6f..f9e98e05cab7aca 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -157,7 +157,7 @@ Event Clear (unset) the event. - Tasks awaiting on :meth:`~Event.wait` will now block until the + Subsequent tasks awaiting on :meth:`~Event.wait` will now block until the :meth:`~Event.set` method is called again. .. method:: is_set() diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 59acce1990ae047..42af2424e19bb0a 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -2,7 +2,7 @@ ==================== -Coroutines and Tasks +Coroutines and tasks ==================== This section outlines high-level asyncio APIs to work with coroutines @@ -231,25 +231,31 @@ A good example of a low-level function that returns a Future object is :meth:`loop.run_in_executor`. -Creating Tasks +Creating tasks ============== **Source code:** :source:`Lib/asyncio/tasks.py` ----------------------------------------------- -.. function:: create_task(coro, *, name=None, context=None) +.. function:: create_task(coro, *, name=None, context=None, eager_start=None, **kwargs) Wrap the *coro* :ref:`coroutine ` into a :class:`Task` and schedule its execution. Return the Task object. - If *name* is not ``None``, it is set as the name of the task using - :meth:`Task.set_name`. + The full function signature is largely the same as that of the + :class:`Task` constructor (or factory) - all of the keyword arguments to + this function are passed through to that interface. An optional keyword-only *context* argument allows specifying a custom :class:`contextvars.Context` for the *coro* to run in. The current context copy is created when no *context* is provided. + An optional keyword-only *eager_start* argument allows specifying + if the task should execute eagerly during the call to create_task, + or be scheduled later. If *eager_start* is not passed the mode set + by :meth:`loop.set_task_factory` will be used. + The task is executed in the loop returned by :func:`get_running_loop`, :exc:`RuntimeError` is raised if there is no running loop in current thread. @@ -290,8 +296,11 @@ Creating Tasks .. versionchanged:: 3.11 Added the *context* parameter. + .. versionchanged:: 3.14 + Added the *eager_start* parameter by passing on all *kwargs*. + -Task Cancellation +Task cancellation ================= Tasks can easily and safely be cancelled. @@ -315,7 +324,7 @@ remove the cancellation state. .. _taskgroups: -Task Groups +Task groups =========== Task groups combine a task creation API with a convenient @@ -330,7 +339,7 @@ and reliable way to wait for all tasks in the group to finish. .. versionadded:: 3.11 - .. method:: create_task(coro, *, name=None, context=None) + .. method:: create_task(coro, *, name=None, context=None, eager_start=None, **kwargs) Create a task in this task group. The signature matches that of :func:`asyncio.create_task`. @@ -342,6 +351,10 @@ and reliable way to wait for all tasks in the group to finish. Close the given coroutine if the task group is not active. + .. versionchanged:: 3.14 + + Passes on all *kwargs* to :meth:`loop.create_task` + Example:: async def main(): @@ -414,7 +427,7 @@ reported by :meth:`asyncio.Task.cancelling`. Improved handling of simultaneous internal and external cancellations and correct preservation of cancellation counts. -Terminating a Task Group +Terminating a task group ------------------------ While terminating a task group is not natively supported by the standard @@ -485,13 +498,13 @@ Sleeping for 5 seconds:: import asyncio - import datetime + import datetime as dt async def display_date(): loop = asyncio.get_running_loop() end_time = loop.time() + 5.0 while True: - print(datetime.datetime.now()) + print(dt.datetime.now()) if (loop.time() + 1.0) >= end_time: break await asyncio.sleep(1) @@ -506,7 +519,7 @@ Sleeping Raises :exc:`ValueError` if *delay* is :data:`~math.nan`. -Running Tasks Concurrently +Running tasks concurrently ========================== .. awaitablefunction:: gather(*aws, return_exceptions=False) @@ -544,7 +557,7 @@ Running Tasks Concurrently provides stronger safety guarantees than *gather* for scheduling a nesting of subtasks: if a task (or a subtask, a task scheduled by a task) raises an exception, *TaskGroup* will, while *gather* will not, - cancel the remaining scheduled tasks). + cancel the remaining scheduled tasks. .. _asyncio_example_gather: @@ -608,7 +621,7 @@ Running Tasks Concurrently .. _eager-task-factory: -Eager Task Factory +Eager task factory ================== .. function:: eager_task_factory(loop, coro, *, name=None, context=None) @@ -651,7 +664,7 @@ Eager Task Factory .. versionadded:: 3.12 -Shielding From Cancellation +Shielding from cancellation =========================== .. awaitablefunction:: shield(aw) @@ -758,6 +771,9 @@ Timeouts An :ref:`asynchronous context manager ` for cancelling overdue coroutines. + Prefer using :func:`asyncio.timeout` or :func:`asyncio.timeout_at` + rather than instantiating :class:`!Timeout` directly. + ``when`` should be an absolute time at which the context should time out, as measured by the event loop's clock: @@ -878,7 +894,7 @@ Timeouts Raises :exc:`TimeoutError` instead of :exc:`asyncio.TimeoutError`. -Waiting Primitives +Waiting primitives ================== .. function:: wait(aws, *, timeout=None, return_when=ALL_COMPLETED) @@ -926,6 +942,9 @@ Waiting Primitives Unlike :func:`~asyncio.wait_for`, ``wait()`` does not cancel the futures when a timeout occurs. + If ``wait()`` is cancelled, the futures in *aws* are not cancelled + and continue to run. + .. versionchanged:: 3.10 Removed the *loop* parameter. @@ -983,6 +1002,10 @@ Waiting Primitives are done. This is raised by the ``async for`` loop during asynchronous iteration or by the coroutines yielded during plain iteration. + ``as_completed()`` does not cancel the tasks running the supplied + awaitables: if a timeout occurs or the iteration is cancelled, the + remaining tasks continue to run. + .. versionchanged:: 3.10 Removed the *loop* parameter. @@ -998,7 +1021,7 @@ Waiting Primitives or as a plain :term:`iterator` (previously it was only a plain iterator). -Running in Threads +Running in threads ================== .. function:: to_thread(func, /, *args, **kwargs) @@ -1058,7 +1081,7 @@ Running in Threads .. versionadded:: 3.9 -Scheduling From Other Threads +Scheduling from other threads ============================= .. function:: run_coroutine_threadsafe(coro, loop) @@ -1180,8 +1203,9 @@ Introspection .. versionadded:: 3.4 +.. _asyncio-task-obj: -Task Object +Task object =========== .. class:: Task(coro, *, loop=None, name=None, context=None, eager_start=False) @@ -1231,6 +1255,9 @@ Task Object blocks. If the coroutine returns or raises without blocking, the task will be finished eagerly and will skip scheduling to the event loop. + Tasks are :ref:`generic ` over the return type of their wrapped + coroutines. + .. versionchanged:: 3.7 Added support for the :mod:`contextvars` module. diff --git a/Doc/library/asyncio-threading.rst b/Doc/library/asyncio-threading.rst new file mode 100644 index 000000000000000..526901a2e7eb206 --- /dev/null +++ b/Doc/library/asyncio-threading.rst @@ -0,0 +1,154 @@ +.. currentmodule:: asyncio + +.. _asyncio-threading: + +asyncio and free-threaded Python +================================ + +asyncio uses an event loop as a scheduler to enable highly efficient +concurrency by switching between tasks to allow non-blocking I/O +operations. This results in better performance for I/O-bound use +cases. It also allows off-loading CPU-bound work to a thread or +process pool, but that is still limited by the :term:`global +interpreter lock` in CPython. + +However, in :ref:`free-threaded Python `, +the GIL is disabled and Python can run true multi-threaded code. This +means that asyncio can now take advantage of multiple CPU cores without +the limitations imposed by the GIL. + +Since Python 3.14, asyncio has first-class support for free-threaded +Python, and the implementation of asyncio is safe to use in a +multi-threaded environment. + +A single event loop on one core can handle many connections +concurrently, but the Python code that runs to handle each one still +executes serially. Once requests involve a non-trivial amount of +per-request computation, that handling becomes the bottleneck, and a +single core can no longer keep up. Combining asyncio with threads is +most useful here: by running an event loop per thread, the handling of +different requests can run in parallel across multiple CPU cores. It is +also useful when you need to run blocking or CPU-bound code from an +asyncio application. + + +.. seealso:: + + `Scaling asyncio on Free-Threaded Python + `__, + a blog post by Kumar Aditya which explains the internal changes + that make asyncio safe and efficient under free-threaded Python, + together with benchmarks of the resulting improvements. + + +Thread safety considerations +---------------------------- + +While asyncio is designed to be thread-safe in a free-threaded Python +environment, there are still some considerations to keep in mind when +using asyncio with threads: + +1. **Event loop**: Each thread should have its own event loop which + should not be shared across threads. This ensures that the event loop + can manage its own tasks and callbacks without interference from + other threads. + +2. **Task management**: Tasks and futures created in one thread should + not be awaited or manipulated from another thread. + +3. **Thread-safe APIs**: When interacting with asyncio from multiple + threads, it's important to use thread-safe APIs provided by asyncio, + such as :func:`asyncio.run_coroutine_threadsafe` for submitting + coroutines to an event loop from another thread. If you need to + call a callback from a different thread, you can use + :meth:`loop.call_soon_threadsafe` to schedule it safely. + +4. **Synchronization**: The synchronization primitives provided by + asyncio (like :class:`asyncio.Lock` and :class:`asyncio.Event`) + are not designed to be used across threads. If you need to + synchronize between threads, you should use the synchronization + primitives from the :mod:`threading` module instead. + + +Using asyncio with threads +-------------------------- + +asyncio supports running one event loop per thread, which allows you to +take advantage of multiple CPU cores in a free-threaded Python +environment. Each thread can run its own event loop, and tasks can be +scheduled on those loops independently. + +Here's an example of how to use asyncio with threads:: + + import asyncio + import threading + + async def worker(name: str) -> None: + print(f"Worker {name} starting") + await asyncio.sleep(1) + print(f"Worker {name} done") + + def run_loop(name: str) -> None: + asyncio.run(worker(name)) + + threads = [ + threading.Thread(target=run_loop, args=(f"T{i}",)) + for i in range(4) + ] + for t in threads: + t.start() + for t in threads: + t.join() + +In this example, each thread creates its own event loop with +:func:`asyncio.run` and runs a coroutine on it. The threads execute +concurrently, and in a free-threaded build they can run on separate +CPU cores in parallel. + + +Producer/consumer across threads +-------------------------------- + +When a regular (non-asyncio) thread needs to hand work to an asyncio +event loop running in another thread, use a thread-safe primitive such +as :class:`queue.Queue` rather than :class:`asyncio.Queue`, which is +only safe within a single event loop.:: + + import asyncio + import queue + import threading + + def producer(q: queue.Queue[int]) -> None: + for i in range(5): + print(f"Producing {i}") + q.put(i) + q.shutdown() + + async def consumer(q: queue.Queue[int]) -> None: + while True: + try: + item = q.get_nowait() + except queue.Empty: + await asyncio.sleep(0.1) + continue + except queue.ShutDown: + break + print(f"Consumed {item}") + await asyncio.sleep(item) + + q: queue.Queue[int] = queue.Queue() + consumer_thread = threading.Thread( + target=lambda: asyncio.run(consumer(q)) + ) + consumer_thread.start() + producer(q) + consumer_thread.join() + +The producer runs on the main thread while the consumer runs inside an +event loop on its own thread, yet they communicate safely through +``queue.Queue``. When the queue is empty the consumer sleeps briefly +and tries again. When the producer is done it calls +:meth:`~queue.Queue.shutdown`, which causes subsequent +:meth:`~queue.Queue.get_nowait` calls to raise :exc:`queue.ShutDown` +so the consumer can exit cleanly. + diff --git a/Doc/library/asyncio-tools.rst b/Doc/library/asyncio-tools.rst new file mode 100644 index 000000000000000..5ffb9bb414e8422 --- /dev/null +++ b/Doc/library/asyncio-tools.rst @@ -0,0 +1,150 @@ +.. currentmodule:: asyncio + +.. _asyncio-introspection-tools: + +================================ +Command-line introspection tools +================================ + +**Source code:** :source:`Lib/asyncio/tools.py` + +------------------------------------- + +The :mod:`!asyncio` module can be invoked as a script via ``python -m +asyncio`` to inspect the task graph of another running Python process without +modifying it or restarting it. The :mod:`!asyncio.tools` submodule implements +this interface. + +The following commands inspect the process identified by ``PID``: + +.. code-block:: shell-session + + $ python -m asyncio pstree PID + $ python -m asyncio ps PID + +The commands read the target process state without executing any code in it. +They are only available on supported platforms and may require permission to +inspect another process. See the :ref:`permission requirements ` for details. + +.. seealso:: + + :ref:`asyncio-graph` + Programmatic APIs for inspecting the async call graph of a task or + future in the current process. + +The command examples below use this program, which creates a task hierarchy +suitable for inspection and prints its process ID: + +.. code-block:: python + :caption: example.py + + import asyncio + import os + + async def play(track): + await asyncio.sleep(3600) + print(f"🎵 Finished: {track}") + + async def album(name, tracks): + async with asyncio.TaskGroup() as tg: + for track in tracks: + tg.create_task(play(track), name=track) + + async def main(): + print(f"PID: {os.getpid()}") + async with asyncio.TaskGroup() as tg: + tg.create_task( + album("Sundowning", ["TNDNBTG", "Levitate"]), + name="Sundowning", + ) + tg.create_task( + album("TMBTE", ["DYWTYLM", "Aqua Regia"]), + name="TMBTE", + ) + + asyncio.run(main()) + +Run the program in one terminal and leave it running: + +.. code-block:: shell-session + + $ python example.py + PID: 12345 + +Then pass the printed process ID to the commands from another terminal. +Thread IDs, task IDs, file paths, and line numbers vary between runs and +source layouts. + +.. versionadded:: 3.14 + +Command-line options +==================== + +.. option:: pstree PID + + Display task and coroutine relationships as a tree. Each task is shown + with its full coroutine stack, nested under the task (if any) that is + awaiting it. This subcommand is useful for quickly identifying which branch + of a task hierarchy is blocked and where in its coroutine stack execution + has paused: + + .. code-block:: shell-session + + $ python -m asyncio pstree 12345 + └── (T) Task-1 + └── main example.py:12 + └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:75 + └── TaskGroup._aexit Lib/asyncio/taskgroups.py:124 + ├── (T) Sundowning + │ └── album example.py:7 + │ └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:75 + │ └── TaskGroup._aexit Lib/asyncio/taskgroups.py:124 + │ ├── (T) TNDNBTG + │ │ └── play example.py:4 + │ │ └── sleep Lib/asyncio/tasks.py:702 + │ └── (T) Levitate + │ └── play example.py:4 + │ └── sleep Lib/asyncio/tasks.py:702 + └── (T) TMBTE + └── album example.py:7 + └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:75 + └── TaskGroup._aexit Lib/asyncio/taskgroups.py:124 + ├── (T) DYWTYLM + │ └── play example.py:4 + │ └── sleep Lib/asyncio/tasks.py:702 + └── (T) Aqua Regia + └── play example.py:4 + └── sleep Lib/asyncio/tasks.py:702 + + If the await graph contains a cycle, ``pstree`` reports an error instead + of printing a tree. A cycle in the await graph is unusual and typically + indicates a programming error: + + .. code-block:: shell-session + + $ python -m asyncio pstree 12345 + ERROR: await-graph contains cycles - cannot print a tree! + + cycle: Task-2 → Task-3 → Task-2 + +.. option:: ps PID + + Display a flat table of all pending tasks in the process *PID*. Each row + shows the event-loop thread ID, task ID and name, coroutine stack, and the + awaiting task's stack, name, and ID, if any. + + This subcommand prints all tasks regardless of whether the await graph + contains cycles: + + .. code-block:: shell-session + + $ python -m asyncio ps 12345 + tid task id task name coroutine stack awaiter chain awaiter name awaiter id + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + 18445801 0x10a456060 Task-1 TaskGroup._aexit -> TaskGroup.__aexit__ -> main 0x0 + 18445801 0x10a439f60 Sundowning TaskGroup._aexit -> TaskGroup.__aexit__ -> album TaskGroup._aexit -> TaskGroup.__aexit__ -> main Task-1 0x10a456060 + 18445801 0x10a439d70 TMBTE TaskGroup._aexit -> TaskGroup.__aexit__ -> album TaskGroup._aexit -> TaskGroup.__aexit__ -> main Task-1 0x10a456060 + 18445801 0x10a2a3a80 TNDNBTG sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album Sundowning 0x10a439f60 + 18445801 0x10a2a38a0 Levitate sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album Sundowning 0x10a439f60 + 18445801 0x10a2d7150 DYWTYLM sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album TMBTE 0x10a439d70 + 18445801 0x10a6bdaa0 Aqua Regia sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album TMBTE 0x10a439d70 diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 7d368dae49dc1d7..8d19f49f6a0621f 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -29,6 +29,11 @@ database connection libraries, distributed task queues, etc. asyncio is often a perfect fit for IO-bound and high-level **structured** network code. +.. seealso:: + + :ref:`a-conceptual-overview-of-asyncio` + Explanation of the fundamentals of asyncio. + asyncio provides a set of **high-level** APIs to: * :ref:`run Python coroutines ` concurrently and @@ -42,6 +47,13 @@ asyncio provides a set of **high-level** APIs to: * :ref:`synchronize ` concurrent code; +For **introspection**, asyncio provides APIs and tools for: + +* inspecting the :ref:`async call graph ` of tasks and futures; + +* inspecting tasks in another running Python process with + :ref:`command-line tools `; + Additionally, there are **low-level** APIs for *library and framework developers* to: @@ -74,6 +86,10 @@ You can experiment with an ``asyncio`` concurrent context in the :term:`REPL`: >>> await asyncio.sleep(10, result='hello') 'hello' +This REPL provides limited compatibility with :envvar:`PYTHON_BASIC_REPL`. +It is recommended that the default REPL is used +for full functionality and the latest features. + .. audit-event:: cpython.run_stdin "" "" .. versionchanged:: 3.12.5 (also 3.11.10, 3.10.15, 3.9.20, and 3.8.20) @@ -99,7 +115,13 @@ You can experiment with an ``asyncio`` concurrent context in the :term:`REPL`: asyncio-subprocess.rst asyncio-queue.rst asyncio-exceptions.rst + +.. toctree:: + :caption: Introspection APIs + :maxdepth: 1 + asyncio-graph.rst + asyncio-tools.rst .. toctree:: :caption: Low-level APIs @@ -119,6 +141,7 @@ You can experiment with an ``asyncio`` concurrent context in the :term:`REPL`: asyncio-api-index.rst asyncio-llapi-index.rst asyncio-dev.rst + asyncio-threading.rst .. note:: The source code for asyncio can be found in :source:`Lib/asyncio/`. diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index 02d2f0807df8f69..24a3492ba10c916 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -9,9 +9,9 @@ -------------- -The :mod:`atexit` module defines functions to register and unregister cleanup +The :mod:`!atexit` module defines functions to register and unregister cleanup functions. Functions thus registered are automatically executed upon normal -interpreter termination. :mod:`atexit` runs these functions in the *reverse* +interpreter termination. :mod:`!atexit` runs these functions in the *reverse* order in which they were registered; if you register ``A``, ``B``, and ``C``, at interpreter termination time they will be run in the order ``C``, ``B``, ``A``. @@ -64,7 +64,7 @@ a cleanup function is undefined. Remove *func* from the list of functions to be run at interpreter shutdown. :func:`unregister` silently does nothing if *func* was not previously registered. If *func* has been registered more than once, every occurrence - of that function in the :mod:`atexit` call stack will be removed. Equality + of that function in the :mod:`!atexit` call stack will be removed. Equality comparisons (``==``) are used internally during unregistration, so function references do not need to have matching identities. @@ -72,14 +72,14 @@ a cleanup function is undefined. .. seealso:: Module :mod:`readline` - Useful example of :mod:`atexit` to read and write :mod:`readline` history + Useful example of :mod:`!atexit` to read and write :mod:`readline` history files. .. _atexit-example: -:mod:`atexit` Example ---------------------- +:mod:`!atexit` Example +---------------------- The following simple example demonstrates how a module can initialize a counter from a file when it is imported and save the counter's updated value diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index 834ab2536e6c14e..0248fa2f45f2f73 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -15,14 +15,11 @@ This module provides functions for encoding binary data to printable ASCII characters and decoding such encodings back to binary data. -It provides encoding and decoding functions for the encodings specified in -:rfc:`4648`, which defines the Base16, Base32, and Base64 algorithms, -and for the de-facto standard Ascii85 and Base85 encodings. - -The :rfc:`4648` encodings are suitable for encoding binary data so that it can be -safely sent by email, used as parts of URLs, or included as part of an HTTP -POST request. The encoding algorithm is not the same as the -:program:`uuencode` program. +This includes the :ref:`encodings specified in ` +:rfc:`4648` (Base64, Base32 and Base16), the :ref:`Base85 encoding +` specified in `PDF 2.0 +`_, and non-standard variants +of Base85 used elsewhere. There are two interfaces provided by this module. The modern interface supports encoding :term:`bytes-like objects ` to ASCII @@ -30,7 +27,7 @@ supports encoding :term:`bytes-like objects ` to ASCII strings containing ASCII to :class:`bytes`. Both base-64 alphabets defined in :rfc:`4648` (normal, and URL- and filesystem-safe) are supported. -The legacy interface does not support decoding from strings, but it does +The :ref:`legacy interface ` does not support decoding from strings, but it does provide functions for encoding and decoding to and from :term:`file objects `. It only supports the Base64 standard alphabet, and it adds newlines every 76 characters as per :rfc:`2045`. Note that if you are looking @@ -46,7 +43,15 @@ package instead. Any :term:`bytes-like objects ` are now accepted by all encoding and decoding functions in this module. Ascii85/Base85 support added. -The modern interface provides: + +.. _base64-rfc-4648: + +RFC 4648 Encodings +------------------ + +The :rfc:`4648` encodings are suitable for encoding binary data so that it can be +safely sent by email, used as parts of URLs, or included as part of an HTTP +POST request. .. function:: b64encode(s, altchars=None) @@ -181,6 +186,35 @@ The modern interface provides: incorrectly padded or if there are non-alphabet characters present in the input. +.. _base64-base-85: + +Base85 Encodings +----------------- + +Base85 encoding is a family of algorithms which represent four bytes +using five ASCII characters. Originally implemented in the Unix +``btoa(1)`` utility, a version of it was later adopted by Adobe in the +PostScript language and is standardized in PDF 2.0 (ISO 32000-2). +This version, in both its ``btoa`` and PDF variants, is implemented by +:func:`a85encode`. + +A separate version, using a different output character set, was +defined as an April Fool's joke in :rfc:`1924` but is now used by Git +and other software. This version is implemented by :func:`b85encode`. + +Finally, a third version, using yet another output character set +designed for safe inclusion in programming language strings, is +defined by ZeroMQ and implemented here by :func:`z85encode`. + +The functions present in this module differ in how they handle the following: + +* Whether to include and expect enclosing ``<~`` and ``~>`` markers. +* Whether to fold the input into multiple lines. +* The set of ASCII characters used for encoding. +* Compact encodings of sequences of spaces and null bytes. +* The encoding of zero-padding bytes applied to the input. + +Refer to the documentation of the individual functions for more information. .. function:: a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False) @@ -189,17 +223,22 @@ The modern interface provides: *foldspaces* is an optional flag that uses the special short sequence 'y' instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This - feature is not supported by the "standard" Ascii85 encoding. + feature is not supported by the standard encoding used in PDF. *wrapcol* controls whether the output should have newline (``b'\n'``) characters added to it. If this is non-zero, each output line will be at most this many characters long, excluding the trailing newline. - *pad* controls whether the input is padded to a multiple of 4 - before encoding. Note that the ``btoa`` implementation always pads. + *pad* controls whether zero-padding applied to the end of the input + is fully retained in the output encoding, as done by ``btoa``, + producing an exact multiple of 5 bytes of output. This is not part + of the standard encoding used in PDF, as it does not preserve the + length of the data. - *adobe* controls whether the encoded byte sequence is framed with ``<~`` - and ``~>``, which is used by the Adobe implementation. + *adobe* controls whether the encoded byte sequence is framed with + ``<~`` and ``~>``, as in a PostScript base-85 string literal. Note + that while ASCII85Decode streams in PDF documents *must* be + terminated with ``~>``, they *must not* use a leading ``<~``. .. versionadded:: 3.4 @@ -211,13 +250,14 @@ The modern interface provides: *foldspaces* is a flag that specifies whether the 'y' short sequence should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). - This feature is not supported by the "standard" Ascii85 encoding. + This feature is not supported by the standard Ascii85 encoding used in + PDF and PostScript. - *adobe* controls whether the input sequence is in Adobe Ascii85 format - (i.e. is framed with <~ and ~>). + *adobe* controls whether the ``<~`` and ``~>`` markers are + present. While the leading ``<~`` is not required, the input must + end with ``~>``, or a :exc:`ValueError` is raised. - *ignorechars* should be a :term:`bytes-like object` or ASCII string - containing characters to ignore + *ignorechars* should be a byte string containing characters to ignore from the input. This should only contain whitespace characters, and by default contains all whitespace characters in ASCII. @@ -229,8 +269,11 @@ The modern interface provides: Encode the :term:`bytes-like object` *b* using base85 (as used in e.g. git-style binary diffs) and return the encoded :class:`bytes`. - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. + The input is padded with ``b'\0'`` so its length is a multiple of 4 + bytes before encoding. If *pad* is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes, and thus the length of the data may not be + preserved on decoding. .. versionadded:: 3.4 @@ -238,8 +281,7 @@ The modern interface provides: .. function:: b85decode(b) Decode the base85-encoded :term:`bytes-like object` or ASCII string *b* and - return the decoded :class:`bytes`. Padding is implicitly removed, if - necessary. + return the decoded :class:`bytes`. .. versionadded:: 3.4 @@ -247,8 +289,12 @@ The modern interface provides: .. function:: z85encode(s) Encode the :term:`bytes-like object` *s* using Z85 (as used in ZeroMQ) - and return the encoded :class:`bytes`. See `Z85 specification - `_ for more information. + and return the encoded :class:`bytes`. + + The `ZeroMQ specification `_ + requires the length of Z85-encoded data to be a multiple of 5 + bytes. To produce compliant data frames, you must pad the input + data to this function to a multiple of 4 bytes. .. versionadded:: 3.13 @@ -256,13 +302,15 @@ The modern interface provides: .. function:: z85decode(s) Decode the Z85-encoded :term:`bytes-like object` or ASCII string *s* and - return the decoded :class:`bytes`. See `Z85 specification - `_ for more information. + return the decoded :class:`bytes`. .. versionadded:: 3.13 -The legacy interface: +.. _base64-legacy: + +Legacy Interface +---------------- .. function:: decode(input, output) @@ -327,3 +375,11 @@ recommended to review the security section for any code deployed to production. Section 5.2, "Base64 Content-Transfer-Encoding," provides the definition of the base64 encoding. + `ISO 32000-2 Portable document format - Part 2: PDF 2.0 `_ + Section 7.4.3, "ASCII85Decode Filter," provides the definition + of the Ascii85 encoding used in PDF and PostScript, including + the output character set and the details of data length preservation + using zero-padding and partial output groups. + + `ZeroMQ RFC 32/Z85 `_ + The "Formal Specification" section provides the character set used in Z85. diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst index 90f042aa377711a..8d357f1895290a4 100644 --- a/Doc/library/bdb.rst +++ b/Doc/library/bdb.rst @@ -8,7 +8,7 @@ -------------- -The :mod:`bdb` module handles basic debugger functions, like setting breakpoints +The :mod:`!bdb` module handles basic debugger functions, like setting breakpoints or managing execution via the debugger. The following exception is defined: @@ -18,7 +18,7 @@ The following exception is defined: Exception raised by the :class:`Bdb` class for quitting the debugger. -The :mod:`bdb` module also defines two classes: +The :mod:`!bdb` module also defines two classes: .. class:: Breakpoint(self, file, line, temporary=False, cond=None, funcname=None) @@ -240,7 +240,7 @@ The :mod:`bdb` module also defines two classes: Normally derived classes don't override the following methods, but they may if they want to redefine the definition of stopping and breakpoints. - .. method:: is_skipped_line(module_name) + .. method:: is_skipped_module(module_name) Return ``True`` if *module_name* matches any skip pattern. diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst index 1bab785684bbabe..4cd390d27e44e65 100644 --- a/Doc/library/binascii.rst +++ b/Doc/library/binascii.rst @@ -10,10 +10,10 @@ -------------- -The :mod:`binascii` module contains a number of methods to convert between +The :mod:`!binascii` module contains a number of methods to convert between binary and various ASCII-encoded binary representations. Normally, you will not use these functions directly but use wrapper modules like -:mod:`base64` instead. The :mod:`binascii` module contains +:mod:`base64` instead. The :mod:`!binascii` module contains low-level functions written in C for greater speed that are used by the higher-level modules. @@ -28,7 +28,7 @@ higher-level modules. ASCII-only unicode strings are now accepted by the ``a2b_*`` functions. -The :mod:`binascii` module defines the following functions: +The :mod:`!binascii` module defines the following functions: .. function:: a2b_uu(string) @@ -158,9 +158,8 @@ The :mod:`binascii` module defines the following functions: of hexadecimal digits (which can be upper or lower case), otherwise an :exc:`Error` exception is raised. - Similar functionality (accepting only text string arguments, but more - liberal towards whitespace) is also accessible using the - :meth:`bytes.fromhex` class method. + Similar functionality (but more liberal towards whitespace) is also accessible + using the :meth:`bytes.fromhex` class method. .. exception:: Error diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst index 78da563397b6255..f79569e1a98a7db 100644 --- a/Doc/library/bisect.rst +++ b/Doc/library/bisect.rst @@ -16,7 +16,7 @@ having to sort the list after each insertion. For long lists of items with expensive comparison operations, this can be an improvement over linear searches or frequent resorting. -The module is called :mod:`bisect` because it uses a basic bisection +The module is called :mod:`!bisect` because it uses a basic bisection algorithm to do its work. Unlike other bisection tools that search for a specific value, the functions in this module are designed to locate an insertion point. Accordingly, the functions never call an :meth:`~object.__eq__` @@ -24,6 +24,16 @@ method to determine whether a value has been found. Instead, the functions only call the :meth:`~object.__lt__` method and will return an insertion point between values in an array. +.. note:: + + The functions in this module are not thread-safe. If multiple threads + concurrently use :mod:`!bisect` functions on the same sequence, this + may result in undefined behaviour. Likewise, if the provided sequence + is mutated by a different thread while a :mod:`!bisect` function + is operating on it, the result is undefined. For example, using + :py:func:`~bisect.insort_left` on the same list from multiple threads + may result in the list becoming unsorted. + .. _bisect functions: The following functions are provided: @@ -73,7 +83,7 @@ The following functions are provided: Insert *x* in *a* in sorted order. This function first runs :py:func:`~bisect.bisect_left` to locate an insertion point. - Next, it runs the :meth:`!insert` method on *a* to insert *x* at the + Next, it runs the :meth:`~sequence.insert` method on *a* to insert *x* at the appropriate position to maintain sort order. To support inserting records in a table, the *key* function (if any) is @@ -93,7 +103,7 @@ The following functions are provided: entries of *x*. This function first runs :py:func:`~bisect.bisect_right` to locate an insertion point. - Next, it runs the :meth:`!insert` method on *a* to insert *x* at the + Next, it runs the :meth:`~sequence.insert` method on *a* to insert *x* at the appropriate position to maintain sort order. To support inserting records in a table, the *key* function (if any) is @@ -193,9 +203,9 @@ example uses :py:func:`~bisect.bisect` to look up a letter grade for an exam sco based on a set of ordered numeric breakpoints: 90 and up is an 'A', 80 to 89 is a 'B', and so on:: - >>> def grade(score, breakpoints=[60, 70, 80, 90], grades='FDCBA'): - ... i = bisect(breakpoints, score) - ... return grades[i] + >>> def grade(score): + ... i = bisect([60, 70, 80, 90], score) + ... return "FDCBA"[i] ... >>> [grade(score) for score in [33, 99, 77, 70, 89, 90, 100]] ['F', 'A', 'C', 'C', 'B', 'A', 'A'] diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index ebe2e43febaefa0..32e223ddbdd8a29 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -16,7 +16,7 @@ This module provides a comprehensive interface for compressing and decompressing data using the bzip2 compression algorithm. -The :mod:`bz2` module contains: +The :mod:`!bz2` module contains: * The :func:`.open` function and :class:`BZ2File` class for reading and writing compressed files. @@ -25,6 +25,8 @@ The :mod:`bz2` module contains: * The :func:`compress` and :func:`decompress` functions for one-shot (de)compression. +.. include:: ../includes/optional-module.rst + (De)compression of files ------------------------ @@ -315,7 +317,7 @@ One-shot (de)compression Examples of usage ----------------- -Below are some examples of typical usage of the :mod:`bz2` module. +Below are some examples of typical usage of the :mod:`!bz2` module. Using :func:`compress` and :func:`decompress` to demonstrate round-trip compression: diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 39090e36ed9c0d9..5c3b0afc15fe525 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -56,13 +56,13 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is .. method:: setfirstweekday(firstweekday) - Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6) + Set the first weekday to *firstweekday*, passed as an :class:`int` (0--6). Identical to setting the :attr:`~Calendar.firstweekday` property. .. method:: iterweekdays() - Return an iterator for the week day numbers that will be used for one + Return an iterator for the weekday numbers that will be used for one week. The first value from the iterator will be the same as the value of the :attr:`~Calendar.firstweekday` property. @@ -88,7 +88,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is Return an iterator for the month *month* in the year *year* similar to :meth:`itermonthdates`, but not restricted by the :class:`datetime.date` range. Days returned will be tuples consisting of a day of the month - number and a week day number. + number and a weekday number. .. method:: itermonthdays3(year, month) @@ -362,7 +362,7 @@ interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is For simple text calendars this module provides the following functions. -.. function:: setfirstweekday(weekday) +.. function:: setfirstweekday(firstweekday) Sets the weekday (``0`` is Monday, ``6`` is Sunday) to start each week. The values :const:`MONDAY`, :const:`TUESDAY`, :const:`WEDNESDAY`, :const:`THURSDAY`, @@ -397,15 +397,15 @@ For simple text calendars this module provides the following functions. *month* (``1``--``12``), *day* (``1``--``31``). -.. function:: weekheader(n) +.. function:: weekheader(width) - Return a header containing abbreviated weekday names. *n* specifies the width in + Return a header containing abbreviated weekday names. *width* specifies the width in characters for one weekday. .. function:: monthrange(year, month) - Returns weekday of first day of the month and number of days in month, for the + Returns weekday of first day of the month and number of days in month, for the specified *year* and *month*. @@ -427,12 +427,12 @@ For simple text calendars this module provides the following functions. of the :class:`TextCalendar` class. -.. function:: prcal(year, w=0, l=0, c=6, m=3) +.. function:: prcal(theyear, w=0, l=0, c=6, m=3) Prints the calendar for an entire year as returned by :func:`calendar`. -.. function:: calendar(year, w=2, l=1, c=6, m=3) +.. function:: calendar(theyear, w=2, l=1, c=6, m=3) Returns a 3-column calendar for an entire year as a multi-line string using the :meth:`~TextCalendar.formatyear` of the :class:`TextCalendar` class. @@ -443,11 +443,11 @@ For simple text calendars this module provides the following functions. An unrelated but handy function that takes a time tuple such as returned by the :func:`~time.gmtime` function in the :mod:`time` module, and returns the corresponding Unix timestamp value, assuming an epoch of 1970, and the POSIX - encoding. In fact, :func:`time.gmtime` and :func:`timegm` are each others' + encoding. In fact, :func:`time.gmtime` and :func:`timegm` are each other's inverse. -The :mod:`calendar` module exports the following data attributes: +The :mod:`!calendar` module exports the following data attributes: .. data:: day_name @@ -540,13 +540,18 @@ The :mod:`calendar` module exports the following data attributes: .. versionadded:: 3.12 -The :mod:`calendar` module defines the following exceptions: +The :mod:`!calendar` module defines the following exceptions: .. exception:: IllegalMonthError(month) - A subclass of :exc:`ValueError`, + A subclass of :exc:`ValueError` and :exc:`IndexError`, raised when the given month number is outside of the range 1-12 (inclusive). + .. versionchanged:: 3.12 + :exc:`IllegalMonthError` is now also a subclass of + :exc:`ValueError`. New code should avoid catching + :exc:`IndexError`. + .. attribute:: month The invalid month number. @@ -579,7 +584,7 @@ Command-line usage .. versionadded:: 2.5 -The :mod:`calendar` module can be executed as a script from the command line +The :mod:`!calendar` module can be executed as a script from the command line to interactively print a calendar. .. code-block:: shell diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst index 26518a0458fd813..f602003e49b8218 100644 --- a/Doc/library/cmath.rst +++ b/Doc/library/cmath.rst @@ -124,7 +124,7 @@ rectangular coordinates to polar coordinates and back. The modulus (absolute value) of a complex number *z* can be computed using the built-in :func:`abs` function. There is no - separate :mod:`cmath` module function for this operation. + separate :mod:`!cmath` module function for this operation. .. function:: polar(z) @@ -338,7 +338,7 @@ Constants .. data:: nan A floating-point "not a number" (NaN) value. Equivalent to - ``float('nan')``. + ``float('nan')``. See also :data:`math.nan`. .. versionadded:: 3.6 @@ -357,7 +357,7 @@ Note that the selection of functions is similar, but not identical, to that in module :mod:`math`. The reason for having two modules is that some users aren't interested in complex numbers, and perhaps don't even know what they are. They would rather have ``math.sqrt(-1)`` raise an exception than return a complex -number. Also note that the functions defined in :mod:`cmath` always return a +number. Also note that the functions defined in :mod:`!cmath` always return a complex number, even if the answer can be expressed as a real number (in which case the complex number has an imaginary part of zero). diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst index 66544f82f6ff3f2..1757dedabbf633f 100644 --- a/Doc/library/cmd.rst +++ b/Doc/library/cmd.rst @@ -245,7 +245,7 @@ Cmd Example .. sectionauthor:: Raymond Hettinger -The :mod:`cmd` module is mainly useful for building custom shells that let a +The :mod:`!cmd` module is mainly useful for building custom shells that let a user work with a program interactively. This section presents a simple example of how to build a shell around a few of diff --git a/Doc/library/cmdline.rst b/Doc/library/cmdline.rst index 16c67ddbf7cec28..c43b10157f9aea0 100644 --- a/Doc/library/cmdline.rst +++ b/Doc/library/cmdline.rst @@ -16,17 +16,17 @@ The following modules have a command-line interface. * :ref:`dis ` * :ref:`doctest ` * :mod:`!encodings.rot_13` -* :mod:`ensurepip` +* :ref:`ensurepip ` * :mod:`filecmp` * :mod:`fileinput` * :mod:`ftplib` * :ref:`gzip ` * :ref:`http.server ` -* :mod:`!idlelib` +* :ref:`idlelib ` * :ref:`inspect ` * :ref:`json ` * :ref:`mimetypes ` -* :mod:`pdb` +* :ref:`pdb ` * :ref:`pickle ` * :ref:`pickletools ` * :ref:`platform ` @@ -52,8 +52,8 @@ The following modules have a command-line interface. * :mod:`turtledemo` * :ref:`unittest ` * :ref:`uuid ` -* :mod:`venv` -* :mod:`webbrowser` +* :ref:`venv ` +* :ref:`webbrowser ` * :ref:`zipapp ` * :ref:`zipfile ` diff --git a/Doc/library/cmdlinelibs.rst b/Doc/library/cmdlinelibs.rst index 085d31af7bca1ff..32f8c2c9f4ae326 100644 --- a/Doc/library/cmdlinelibs.rst +++ b/Doc/library/cmdlinelibs.rst @@ -1,7 +1,7 @@ .. _cmdlinelibs: ******************************** -Command Line Interface Libraries +Command-line interface libraries ******************************** The modules described in this chapter assist with implementing @@ -19,3 +19,4 @@ Here's an overview: curses.rst curses.ascii.rst curses.panel.rst + cmd.rst diff --git a/Doc/library/code.rst b/Doc/library/code.rst index 8f7692df9fb22d1..52587c4dd8f8e87 100644 --- a/Doc/library/code.rst +++ b/Doc/library/code.rst @@ -22,6 +22,12 @@ build applications which provide an interactive interpreter prompt. it defaults to a newly created dictionary with key ``'__name__'`` set to ``'__console__'`` and key ``'__doc__'`` set to ``None``. + Note that functions and classes objects created under an + :class:`!InteractiveInterpreter` instance will belong to the namespace + specified by *locals*. + They are only pickleable if *locals* is the namespace of an existing + module. + .. class:: InteractiveConsole(locals=None, filename="", local_exit=False) diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index 86511602fa5a60d..cf17e0d34dc2a65 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -53,6 +53,14 @@ any codec: :exc:`UnicodeDecodeError`). Refer to :ref:`codec-base-classes` for more information on codec error handling. +.. function:: charmap_build(string) + + Return a mapping suitable for encoding with a custom single-byte encoding. + Given a :class:`str` *string* of up to 256 characters representing a + decoding table, returns either a compact internal mapping object + ``EncodingMap`` or a :class:`dictionary ` mapping character ordinals + to byte values. Raises a :exc:`TypeError` on invalid input. + The full details for each codec can also be looked up directly: .. function:: lookup(encoding, /) @@ -235,8 +243,8 @@ wider range of codecs when working with binary files: .. function:: iterencode(iterator, encoding, errors='strict', **kwargs) Uses an incremental encoder to iteratively encode the input provided by - *iterator*. This function is a :term:`generator`. - The *errors* argument (as well as any + *iterator*. *iterator* must yield :class:`str` objects. + This function is a :term:`generator`. The *errors* argument (as well as any other keyword argument) is passed through to the incremental encoder. This function requires that the codec accept text :class:`str` objects @@ -247,8 +255,8 @@ wider range of codecs when working with binary files: .. function:: iterdecode(iterator, encoding, errors='strict', **kwargs) Uses an incremental decoder to iteratively decode the input provided by - *iterator*. This function is a :term:`generator`. - The *errors* argument (as well as any + *iterator*. *iterator* must yield :class:`bytes` objects. + This function is a :term:`generator`. The *errors* argument (as well as any other keyword argument) is passed through to the incremental decoder. This function requires that the codec accept :class:`bytes` objects @@ -257,6 +265,20 @@ wider range of codecs when working with binary files: :func:`iterencode`. +.. function:: readbuffer_encode(buffer, errors=None, /) + + Return a :class:`tuple` containing the raw bytes of *buffer*, a + :ref:`buffer-compatible object ` or :class:`str` + (encoded to UTF-8 before processing), and their length in bytes. + + The *errors* argument is ignored. + + .. code-block:: pycon + + >>> codecs.readbuffer_encode(b"Zito") + (b'Zito', 4) + + The module also provides the following constants which are useful for reading and writing to platform dependent files: @@ -288,7 +310,7 @@ and writing to platform dependent files: Codec Base Classes ------------------ -The :mod:`codecs` module defines a set of base classes which define the +The :mod:`!codecs` module defines a set of base classes which define the interfaces for working with codec objects, and can also be used as the basis for custom codec implementations. @@ -960,17 +982,22 @@ defined in Unicode. A simple and straightforward way that can store each Unicode code point, is to store each code point as four consecutive bytes. There are two possibilities: store the bytes in big endian or in little endian order. These two encodings are called ``UTF-32-BE`` and ``UTF-32-LE`` respectively. Their -disadvantage is that if e.g. you use ``UTF-32-BE`` on a little endian machine you -will always have to swap bytes on encoding and decoding. ``UTF-32`` avoids this -problem: bytes will always be in natural endianness. When these bytes are read -by a CPU with a different endianness, then bytes have to be swapped though. To -be able to detect the endianness of a ``UTF-16`` or ``UTF-32`` byte sequence, -there's the so called BOM ("Byte Order Mark"). This is the Unicode character -``U+FEFF``. This character can be prepended to every ``UTF-16`` or ``UTF-32`` -byte sequence. The byte swapped version of this character (``0xFFFE``) is an -illegal character that may not appear in a Unicode text. So when the -first character in a ``UTF-16`` or ``UTF-32`` byte sequence -appears to be a ``U+FFFE`` the bytes have to be swapped on decoding. +disadvantage is that if, for example, you use ``UTF-32-BE`` on a little endian +machine you will always have to swap bytes on encoding and decoding. +Python's ``UTF-16`` and ``UTF-32`` codecs avoid this problem by using the +platform's native byte order when no BOM is present. +Python follows prevailing platform +practice, so native-endian data round-trips without redundant byte swapping, +even though the Unicode Standard defaults to big-endian when the byte order is +unspecified. When these bytes are read by a CPU with a different endianness, +the bytes have to be swapped. To be able to detect the endianness of a +``UTF-16`` or ``UTF-32`` byte sequence, a BOM ("Byte Order Mark") is used. +This is the Unicode character ``U+FEFF``. This character can be prepended to every +``UTF-16`` or ``UTF-32`` byte sequence. The byte swapped version of this character +(``0xFFFE``) is an illegal character that may not appear in a Unicode text. +When the first character of a ``UTF-16`` or ``UTF-32`` byte sequence is +``U+FFFE``, the bytes have to be swapped on decoding. + Unfortunately the character ``U+FEFF`` had a second purpose as a ``ZERO WIDTH NO-BREAK SPACE``: a character that has no width and doesn't allow a word to be split. It can e.g. be used to give hints to a ligature algorithm. @@ -1043,8 +1070,15 @@ or with dictionaries as mapping tables. The following table lists the codecs by name, together with a few common aliases, and the languages for which the encoding is likely used. Neither the list of aliases nor the list of languages is meant to be exhaustive. Notice that spelling alternatives that only differ in -case or use a hyphen instead of an underscore are also valid aliases; therefore, -e.g. ``'utf-8'`` is a valid alias for the ``'utf_8'`` codec. +case or use a hyphen instead of an underscore are also valid aliases +because they are equivalent when normalized by +:func:`~encodings.normalize_encoding`. For example, ``'utf-8'`` is a valid +alias for the ``'utf_8'`` codec. + +.. note:: + + The below table lists the most common aliases, for a complete list + refer to the source :source:`aliases.py ` file. On Windows, ``cpXXX`` codecs are available for all code pages. But only codecs listed in the following table are guarantead to exist on @@ -1118,7 +1152,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | cp857 | 857, IBM857 | Turkish | +-----------------+--------------------------------+--------------------------------+ -| cp858 | 858, IBM858 | Western Europe | +| cp858 | 858, IBM00858 | Western Europe | +-----------------+--------------------------------+--------------------------------+ | cp860 | 860, IBM860 | Portuguese | +-----------------+--------------------------------+--------------------------------+ @@ -1155,7 +1189,7 @@ particular, the following variants typically exist: | | | | | | | .. versionadded:: 3.4 | +-----------------+--------------------------------+--------------------------------+ -| cp1140 | ibm1140 | Western Europe | +| cp1140 | IBM01140 | Western Europe | +-----------------+--------------------------------+--------------------------------+ | cp1250 | windows-1250 | Central and Eastern Europe | +-----------------+--------------------------------+--------------------------------+ @@ -1222,7 +1256,7 @@ particular, the following variants typically exist: +-----------------+--------------------------------+--------------------------------+ | iso8859_3 | iso-8859-3, latin3, L3 | Esperanto, Maltese | +-----------------+--------------------------------+--------------------------------+ -| iso8859_4 | iso-8859-4, latin4, L4 | Baltic languages | +| iso8859_4 | iso-8859-4, latin4, L4 | Northern Europe | +-----------------+--------------------------------+--------------------------------+ | iso8859_5 | iso-8859-5, cyrillic | Belarusian, Bulgarian, | | | | Macedonian, Russian, Serbian | @@ -1362,6 +1396,14 @@ encodings. | punycode | | Implement :rfc:`3492`. | | | | Stateful codecs are not | | | | supported. | +| | | | +| | | .. warning:: | +| | | | +| | | The decoding and | +| | | encoding algorithms | +| | | scale poorly, so | +| | | limit the length of | +| | | untrusted input. | +--------------------+---------+---------------------------+ | raw_unicode_escape | | Latin-1 encoding with | | | | :samp:`\\u{XXXX}` and | @@ -1373,7 +1415,11 @@ encodings. | | | It is used in the Python | | | | pickle protocol. | +--------------------+---------+---------------------------+ -| undefined | | Raise an exception for | +| undefined | | This Codec should only | +| | | be used for testing | +| | | purposes. | +| | | | +| | | Raise an exception for | | | | all conversions, even | | | | empty strings. The error | | | | handler is ignored. | @@ -1450,6 +1496,36 @@ to :class:`bytes` mappings. They are not supported by :meth:`bytes.decode` Restoration of the aliases for the binary transforms. +.. _standalone-codec-functions: + +Standalone Codec Functions +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The following functions provide encoding and decoding functionality similar to codecs, +but are not available as named codecs through :func:`codecs.encode` or :func:`codecs.decode`. +They are used internally (for example, by :mod:`pickle`) and behave similarly to the +``string_escape`` codec that was removed in Python 3. + +.. function:: codecs.escape_encode(input, errors=None) + + Encode *input* using escape sequences. Similar to how :func:`repr` on bytes + produces escaped byte values. + + *input* must be a :class:`bytes` object. + + Returns a tuple ``(output, length)`` where *output* is a :class:`bytes` + object and *length* is the number of bytes consumed. + +.. function:: codecs.escape_decode(input, errors=None) + + Decode *input* from escape sequences back to the original bytes. + + *input* must be a :term:`bytes-like object`. + + Returns a tuple ``(output, length)`` where *output* is a :class:`bytes` + object and *length* is the number of bytes consumed. + + .. _text-transforms: Text Transforms @@ -1476,8 +1552,68 @@ mapping. It is not supported by :meth:`str.encode` (which only produces Restoration of the ``rot13`` alias. -:mod:`encodings.idna` --- Internationalized Domain Names in Applications ------------------------------------------------------------------------- +:mod:`!encodings` --- Encodings package +--------------------------------------- + +.. module:: encodings + :synopsis: Encodings package + +This module implements the following functions: + +.. function:: normalize_encoding(encoding) + + Normalize encoding name *encoding*. + + Normalization works as follows: all non-alphanumeric characters except the + dot used for Python package names are collapsed and replaced with a single + underscore, leading and trailing underscores are removed. + For example, ``' -;#'`` becomes ``'_'``. + + Note that *encoding* should be ASCII only. + + +.. note:: + The following functions should not be used directly, except for testing + purposes; :func:`codecs.lookup` should be used instead. + + +.. function:: search_function(encoding) + + Search for the codec module corresponding to the given encoding name + *encoding*. + + This function first normalizes the *encoding* using + :func:`normalize_encoding`, then looks for a corresponding alias. + It attempts to import a codec module from the encodings package using either + the alias or the normalized name. If the module is found and defines a valid + ``getregentry()`` function that returns a :class:`codecs.CodecInfo` object, + the codec is cached and returned. + + If the codec module defines a ``getaliases()`` function any returned aliases + are registered for future use. + + +.. function:: win32_code_page_search_function(encoding) + + Search for a Windows code page encoding *encoding* of the form ``cpXXXX``. + + If the code page is valid and supported, return a :class:`codecs.CodecInfo` + object for it. + + .. availability:: Windows. + + .. versionadded:: 3.14 + + +This module implements the following exception: + +.. exception:: CodecRegistryError + + Raised when a codec is invalid or incompatible. + + +:mod:`!encodings.idna` --- Internationalized Domain Names in Applications +------------------------------------------------------------------------- .. module:: encodings.idna :synopsis: Internationalized Domain Names implementation @@ -1519,7 +1655,7 @@ When receiving host names from the wire (such as in reverse name lookup), no automatic conversion to Unicode is performed: applications wishing to present such host names to the user should decode them to Unicode. -The module :mod:`encodings.idna` also implements the nameprep procedure, which +The module :mod:`!encodings.idna` also implements the nameprep procedure, which performs certain normalizations on host names, to achieve case-insensitivity of international domain names, and to unify similar characters. The nameprep functions can be used directly if desired. @@ -1542,8 +1678,8 @@ functions can be used directly if desired. Convert a label to Unicode, as specified in :rfc:`3490`. -:mod:`encodings.mbcs` --- Windows ANSI codepage ------------------------------------------------ +:mod:`!encodings.mbcs` --- Windows ANSI codepage +------------------------------------------------ .. module:: encodings.mbcs :synopsis: Windows ANSI codepage @@ -1560,8 +1696,8 @@ This module implements the ANSI codepage (CP_ACP). Support any error handler. -:mod:`encodings.utf_8_sig` --- UTF-8 codec with BOM signature -------------------------------------------------------------- +:mod:`!encodings.utf_8_sig` --- UTF-8 codec with BOM signature +-------------------------------------------------------------- .. module:: encodings.utf_8_sig :synopsis: UTF-8 codec with BOM signature diff --git a/Doc/library/codeop.rst b/Doc/library/codeop.rst index 16f674adb4b22bb..2e6d65980381ad6 100644 --- a/Doc/library/codeop.rst +++ b/Doc/library/codeop.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`codeop` module provides utilities upon which the Python +The :mod:`!codeop` module provides utilities upon which the Python read-eval-print loop can be emulated, as is done in the :mod:`code` module. As a result, you probably don't want to use the module directly; if you want to include such a loop in your program you probably want to use the :mod:`code` @@ -25,7 +25,7 @@ There are two parts to this job: #. Remembering which future statements the user has entered, so subsequent input can be compiled with these in effect. -The :mod:`codeop` module provides a way of doing each of these things, and a way +The :mod:`!codeop` module provides a way of doing each of these things, and a way of doing them both. To do just the former: diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index daa9af6d1dd9c95..213c79cc41494c5 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -140,6 +140,9 @@ ABC Inherits from Abstract Methods Mi ``__len__``, ``insert`` +:class:`ByteString` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods + ``__len__`` + :class:`Set` :class:`Collection` ``__contains__``, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``, ``__iter__``, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``, ``__len__`` ``__sub__``, ``__rsub__``, ``__xor__``, ``__rxor__`` @@ -260,21 +263,50 @@ Collections Abstract Base Classes -- Detailed Descriptions .. class:: Sequence MutableSequence + ByteString ABCs for read-only and mutable :term:`sequences `. Implementation note: Some of the mixin methods, such as - :meth:`~container.__iter__`, :meth:`~object.__reversed__` and :meth:`index`, make - repeated calls to the underlying :meth:`~object.__getitem__` method. + :meth:`~container.__iter__`, :meth:`~object.__reversed__`, + and :meth:`~sequence.index` make repeated calls to the underlying + :meth:`~object.__getitem__` method. Consequently, if :meth:`~object.__getitem__` is implemented with constant access speed, the mixin methods will have linear performance; however, if the underlying method is linear (as it would be with a linked list), the mixins will have quadratic performance and will likely need to be overridden. - .. versionchanged:: 3.5 - The index() method added support for *stop* and *start* - arguments. + .. method:: index(value, start=0, stop=None) + + Return first index of *value*. + + Raises :exc:`ValueError` if the value is not present. + + Supporting the *start* and *stop* arguments is optional, but recommended. + + .. versionchanged:: 3.5 + The :meth:`~sequence.index` method gained support for + the *stop* and *start* arguments. + + .. deprecated-removed:: 3.12 3.17 + The :class:`ByteString` ABC has been deprecated. + + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the :ref:`buffer protocol ` at runtime. For use + in type annotations, either use :class:`Buffer` or a union that + explicitly specifies the types your code supports (e.g., + ``bytes | bytearray | memoryview``). + + :class:`!ByteString` was originally intended to be an abstract class that + would serve as a supertype of both :class:`bytes` and :class:`bytearray`. + However, since the ABC never had any methods, knowing that an object was + an instance of :class:`!ByteString` never actually told you anything + useful about the object. Other common buffer types such as + :class:`memoryview` were also never understood as subtypes of + :class:`!ByteString` (either at runtime or by static type checkers). + + See :pep:`PEP 688 <688#current-options>` for more details. .. class:: Set MutableSet @@ -304,7 +336,7 @@ Collections Abstract Base Classes -- Detailed Descriptions .. note:: In CPython, generator-based coroutines (:term:`generators ` - decorated with :func:`@types.coroutine `) are + decorated with :deco:`types.coroutine`) are *awaitables*, even though they do not have an :meth:`~object.__await__` method. Using ``isinstance(gencoro, Awaitable)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. @@ -322,7 +354,7 @@ Collections Abstract Base Classes -- Detailed Descriptions .. note:: In CPython, generator-based coroutines (:term:`generators ` - decorated with :func:`@types.coroutine `) are + decorated with :deco:`types.coroutine`) are *awaitables*, even though they do not have an :meth:`~object.__await__` method. Using ``isinstance(gencoro, Coroutine)`` for them will return ``False``. Use :func:`inspect.isawaitable` to detect them. @@ -427,7 +459,7 @@ Notes on using :class:`Set` and :class:`MutableSet` as a mixin: The :class:`Set` mixin provides a :meth:`!_hash` method to compute a hash value for the set; however, :meth:`~object.__hash__` is not defined because not all sets are :term:`hashable` or immutable. To add set hashability using mixins, - inherit from both :meth:`Set` and :meth:`Hashable`, then define + inherit from both :class:`Set` and :class:`Hashable`, then define ``__hash__ = Set._hash``. .. seealso:: diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 5fbdb12f40cafa5..cfae81040624181 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -240,7 +240,9 @@ For example:: [('the', 1143), ('and', 966), ('to', 762), ('of', 669), ('i', 631), ('you', 554), ('a', 546), ('my', 514), ('hamlet', 471), ('in', 451)] -.. class:: Counter([iterable-or-mapping]) +.. class:: Counter(**kwargs) + Counter(iterable, /, **kwargs) + Counter(mapping, /, **kwargs) A :class:`Counter` is a :class:`dict` subclass for counting :term:`hashable` objects. It is a collection where elements are stored as dictionary keys @@ -290,7 +292,7 @@ For example:: >>> sorted(c.elements()) ['a', 'a', 'a', 'a', 'b', 'b'] - .. method:: most_common([n]) + .. method:: most_common(n=None) Return a list of the *n* most common elements and their counts from the most common to the least. If *n* is omitted or ``None``, @@ -300,7 +302,9 @@ For example:: >>> Counter('abracadabra').most_common(3) [('a', 5), ('b', 2), ('r', 2)] - .. method:: subtract([iterable-or-mapping]) + .. method:: subtract(**kwargs) + subtract(iterable, /, **kwargs) + subtract(mapping, /, **kwargs) Elements are subtracted from an *iterable* or from another *mapping* (or counter). Like :meth:`dict.update` but subtracts counts instead @@ -331,7 +335,9 @@ For example:: This class method is not implemented for :class:`Counter` objects. - .. method:: update([iterable-or-mapping]) + .. method:: update(**kwargs) + update(iterable, /, **kwargs) + update(mapping, /, **kwargs) Elements are counted from an *iterable* or added-in from another *mapping* (or counter). Like :meth:`dict.update` but adds counts @@ -474,17 +480,19 @@ or subtracting from an empty counter. Unix. They are also useful for tracking transactions and other pools of data where only the most recent activity is of interest. + Deques are :ref:`generic ` over the type of their contents. + Deque objects support the following methods: - .. method:: append(x) + .. method:: append(item, /) - Add *x* to the right side of the deque. + Add *item* to the right side of the deque. - .. method:: appendleft(x) + .. method:: appendleft(item, /) - Add *x* to the left side of the deque. + Add *item* to the left side of the deque. .. method:: clear() @@ -499,38 +507,38 @@ or subtracting from an empty counter. .. versionadded:: 3.5 - .. method:: count(x) + .. method:: count(value, /) - Count the number of deque elements equal to *x*. + Count the number of deque elements equal to *value*. .. versionadded:: 3.2 - .. method:: extend(iterable) + .. method:: extend(iterable, /) Extend the right side of the deque by appending elements from the iterable argument. - .. method:: extendleft(iterable) + .. method:: extendleft(iterable, /) Extend the left side of the deque by appending elements from *iterable*. Note, the series of left appends results in reversing the order of elements in the iterable argument. - .. method:: index(x[, start[, stop]]) + .. method:: index(value[, start[, stop]]) - Return the position of *x* in the deque (at or after index *start* + Return the position of *value* in the deque (at or after index *start* and before index *stop*). Returns the first match or raises :exc:`ValueError` if not found. .. versionadded:: 3.5 - .. method:: insert(i, x) + .. method:: insert(index, value, /) - Insert *x* into the deque at position *i*. + Insert *value* into the deque at position *index*. If the insertion would cause a bounded deque to grow beyond *maxlen*, an :exc:`IndexError` is raised. @@ -550,7 +558,7 @@ or subtracting from an empty counter. elements are present, raises an :exc:`IndexError`. - .. method:: remove(value) + .. method:: remove(value, /) Remove the first occurrence of *value*. If not found, raises a :exc:`ValueError`. @@ -563,7 +571,7 @@ or subtracting from an empty counter. .. versionadded:: 3.2 - .. method:: rotate(n=1) + .. method:: rotate(n=1, /) Rotate the deque *n* steps to the right. If *n* is negative, rotate to the left. @@ -715,7 +723,9 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, :class:`defaultdict` objects ---------------------------- -.. class:: defaultdict(default_factory=None, /, [...]) +.. class:: defaultdict(default_factory=None, /, **kwargs) + defaultdict(default_factory, mapping, /, **kwargs) + defaultdict(default_factory, iterable, /, **kwargs) Return a new dictionary-like object. :class:`defaultdict` is a subclass of the built-in :class:`dict` class. It overrides one method and adds one writable @@ -727,11 +737,14 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, as if they were passed to the :class:`dict` constructor, including keyword arguments. + :class:`!defaultdict`\s are :ref:`generic ` over two types, + signifying (respectively) the types of the dictionary's keys and values. + :class:`defaultdict` objects support the following method in addition to the standard :class:`dict` operations: - .. method:: __missing__(key) + .. method:: __missing__(key, /) If the :attr:`default_factory` attribute is ``None``, this raises a :exc:`KeyError` exception with the *key* as argument. @@ -758,9 +771,9 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, .. attribute:: default_factory - This attribute is used by the :meth:`__missing__` method; it is - initialized from the first argument to the constructor, if present, or to - ``None``, if absent. + This attribute is used by the :meth:`~defaultdict.__missing__` method; + it is initialized from the first argument to the constructor, if present, + or to ``None``, if absent. .. versionchanged:: 3.9 Added merge (``|``) and update (``|=``) operators, specified in @@ -783,10 +796,10 @@ sequence of key-value pairs into a dictionary of lists: When each key is encountered for the first time, it is not already in the mapping; so an entry is automatically created using the :attr:`~defaultdict.default_factory` -function which returns an empty :class:`list`. The :meth:`!list.append` +function which returns an empty :class:`list`. The :meth:`list.append` operation then attaches the value to the new list. When keys are encountered again, the look-up proceeds normally (returning the list for that key) and the -:meth:`!list.append` operation adds another value to the list. This technique is +:meth:`list.append` operation adds another value to the list. This technique is simpler and faster than an equivalent technique using :meth:`dict.setdefault`: >>> d = {} @@ -937,7 +950,7 @@ In addition to the methods inherited from tuples, named tuples support three additional methods and two attributes. To prevent conflicts with field names, the method and attribute names start with an underscore. -.. classmethod:: somenamedtuple._make(iterable) +.. classmethod:: somenamedtuple._make(iterable, /) Class method that makes a new instance from an existing sequence or iterable. @@ -1134,7 +1147,9 @@ Some differences from :class:`dict` still remain: * Until Python 3.8, :class:`dict` lacked a :meth:`~object.__reversed__` method. -.. class:: OrderedDict([items]) +.. class:: OrderedDict(**kwargs) + OrderedDict(mapping, /, **kwargs) + OrderedDict(iterable, /, **kwargs) Return an instance of a :class:`dict` subclass that has methods specialized for rearranging dictionary order. @@ -1214,7 +1229,7 @@ variants of :func:`functools.lru_cache`: .. testcode:: from collections import OrderedDict - from time import time + from time import monotonic class TimeBoundedLRU: "LRU Cache that invalidates and refreshes old entries." @@ -1229,10 +1244,10 @@ variants of :func:`functools.lru_cache`: if args in self.cache: self.cache.move_to_end(args) timestamp, result = self.cache[args] - if time() - timestamp <= self.maxage: + if monotonic() - timestamp <= self.maxage: return result result = self.func(*args) - self.cache[args] = time(), result + self.cache[args] = monotonic(), result if len(self.cache) > self.maxsize: self.cache.popitem(last=False) return result @@ -1315,16 +1330,17 @@ subclass directly from :class:`dict`; however, this class can be easier to work with because the underlying dictionary is accessible as an attribute. -.. class:: UserDict([initialdata]) +.. class:: UserDict(**kwargs) + UserDict(mapping, /, **kwargs) + UserDict(iterable, /, **kwargs) Class that simulates a dictionary. The instance's contents are kept in a regular dictionary, which is accessible via the :attr:`data` attribute of - :class:`UserDict` instances. If *initialdata* is provided, :attr:`data` is - initialized with its contents; note that a reference to *initialdata* will not - be kept, allowing it to be used for other purposes. + :class:`!UserDict` instances. If arguments are provided, they are used to + initialize :attr:`data`, like a regular dictionary. In addition to supporting the methods and operations of mappings, - :class:`UserDict` instances provide the following attribute: + :class:`!UserDict` instances provide the following attribute: .. attribute:: data diff --git a/Doc/library/colorsys.rst b/Doc/library/colorsys.rst index ffebf4e40dd609a..2d3dc2b8b57935a 100644 --- a/Doc/library/colorsys.rst +++ b/Doc/library/colorsys.rst @@ -10,7 +10,7 @@ -------------- -The :mod:`colorsys` module defines bidirectional conversions of color values +The :mod:`!colorsys` module defines bidirectional conversions of color values between colors expressed in the RGB (Red Green Blue) color space used in computer monitors and three other coordinate systems: YIQ, HLS (Hue Lightness Saturation) and HSV (Hue Saturation Value). Coordinates in all of these color @@ -24,7 +24,7 @@ spaces, the coordinates are all between 0 and 1. https://poynton.ca/ColorFAQ.html and https://www.cambridgeincolour.com/tutorials/color-spaces.htm. -The :mod:`colorsys` module defines the following functions: +The :mod:`!colorsys` module defines the following functions: .. function:: rgb_to_yiq(r, g, b) diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index c42288419c4d2dc..ebbbf857e717a4f 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -56,11 +56,18 @@ compile Python sources. executed. .. option:: -s strip_prefix + + Remove the given prefix from paths recorded in the ``.pyc`` files. + Paths are made relative to the prefix. + + This option can be used with ``-p`` but not with ``-d``. + .. option:: -p prepend_prefix - Remove (``-s``) or append (``-p``) the given prefix of paths - recorded in the ``.pyc`` files. - Cannot be combined with ``-d``. + Prepend the given prefix to paths recorded in the ``.pyc`` files. + Use ``-p /`` to make the paths absolute. + + This option can be used with ``-s`` but not with ``-d``. .. option:: -x regex diff --git a/Doc/library/compression.rst b/Doc/library/compression.rst new file mode 100644 index 000000000000000..98719be9992acc4 --- /dev/null +++ b/Doc/library/compression.rst @@ -0,0 +1,20 @@ +The :mod:`!compression` package +=============================== + +.. module:: compression + +.. versionadded:: 3.14 + +The :mod:`!compression` package contains the canonical compression modules +containing interfaces to several different compression algorithms. Some of +these modules have historically been available as separate modules; those will +continue to be available under their original names for compatibility reasons, +and will not be removed without a deprecation cycle. The use of modules in +:mod:`!compression` is encouraged where practical. + +* :mod:`!compression.bz2` -- Re-exports :mod:`bz2` +* :mod:`!compression.gzip` -- Re-exports :mod:`gzip` +* :mod:`!compression.lzma` -- Re-exports :mod:`lzma` +* :mod:`!compression.zlib` -- Re-exports :mod:`zlib` +* :mod:`compression.zstd` -- Wrapper for the Zstandard compression library + diff --git a/Doc/library/compression.zstd.rst b/Doc/library/compression.zstd.rst new file mode 100644 index 000000000000000..25459640f8e2e92 --- /dev/null +++ b/Doc/library/compression.zstd.rst @@ -0,0 +1,899 @@ +:mod:`!compression.zstd` --- Compression compatible with the Zstandard format +============================================================================= + +.. module:: compression.zstd + :synopsis: Low-level interface to compression and decompression routines in + the zstd library. + +.. versionadded:: 3.14 + +**Source code:** :source:`Lib/compression/zstd/__init__.py` + +-------------- + +This module provides classes and functions for compressing and decompressing +data using the Zstandard (or *zstd*) compression algorithm. The +`zstd manual `__ +describes Zstandard as "a fast lossless compression algorithm, targeting +real-time compression scenarios at zlib-level and better compression ratios." +Also included is a file interface that supports reading and writing the +contents of ``.zst`` files created by the :program:`zstd` utility, as well as +raw zstd compressed streams. + +The :mod:`!compression.zstd` module contains: + +* The :func:`.open` function and :class:`ZstdFile` class for reading and + writing compressed files. +* The :class:`ZstdCompressor` and :class:`ZstdDecompressor` classes for + incremental (de)compression. +* The :func:`compress` and :func:`decompress` functions for one-shot + (de)compression. +* The :func:`train_dict` and :func:`finalize_dict` functions and the + :class:`ZstdDict` class to train and manage Zstandard dictionaries. +* The :class:`CompressionParameter`, :class:`DecompressionParameter`, and + :class:`Strategy` classes for setting advanced (de)compression parameters. + +.. include:: ../includes/optional-module.rst + + +Exceptions +---------- + +.. exception:: ZstdError + + This exception is raised when an error occurs during compression or + decompression, or while initializing the (de)compressor state. + + +Reading and writing compressed files +------------------------------------ + +.. function:: open(file, /, mode='rb', *, level=None, options=None, \ + zstd_dict=None, encoding=None, errors=None, newline=None) + + Open a Zstandard-compressed file in binary or text mode, returning a + :term:`file object`. + + The *file* argument can be either a file name (given as a + :class:`str`, :class:`bytes` or :term:`path-like ` + object), in which case the named file is opened, or it can be an existing + file object to read from or write to. + + The mode argument can be either ``'rb'`` for reading (default), ``'wb'`` for + overwriting, ``'ab'`` for appending, or ``'xb'`` for exclusive creation. + These can equivalently be given as ``'r'``, ``'w'``, ``'a'``, and ``'x'`` + respectively. You may also open in text mode with ``'rt'``, ``'wt'``, + ``'at'``, and ``'xt'`` respectively. + + When reading, the *options* argument can be a dictionary providing advanced + decompression parameters; see :class:`DecompressionParameter` for detailed + information about supported + parameters. The *zstd_dict* argument is a :class:`ZstdDict` instance to be + used during decompression. When reading, if the *level* + argument is not None, a :exc:`!TypeError` will be raised. + + When writing, the *options* argument can be a dictionary + providing advanced compression parameters; see + :class:`CompressionParameter` for detailed information about supported + parameters. The *level* argument is the compression level to use when + writing compressed data. Only one of *level* or *options* may be non-None. + The *zstd_dict* argument is a :class:`ZstdDict` instance to be used during + compression. + + In binary mode, this function is equivalent to the :class:`ZstdFile` + constructor: ``ZstdFile(file, mode, ...)``. In this case, the + *encoding*, *errors*, and *newline* parameters must not be provided. + + In text mode, a :class:`ZstdFile` object is created, and wrapped in an + :class:`io.TextIOWrapper` instance with the specified encoding, error + handling behavior, and line endings. + + +.. class:: ZstdFile(file, /, mode='rb', *, level=None, options=None, \ + zstd_dict=None) + + Open a Zstandard-compressed file in binary mode. + + A :class:`ZstdFile` can wrap an already-open :term:`file object`, or operate + directly on a named file. The *file* argument specifies either the file + object to wrap, or the name of the file to open (as a :class:`str`, + :class:`bytes` or :term:`path-like ` object). If + wrapping an existing file object, the wrapped file will not be closed when + the :class:`ZstdFile` is closed. + + The *mode* argument can be either ``'rb'`` for reading (default), ``'wb'`` + for overwriting, ``'xb'`` for exclusive creation, or ``'ab'`` for appending. + These can equivalently be given as ``'r'``, ``'w'``, ``'x'`` and ``'a'`` + respectively. + + If *file* is a file object (rather than an actual file name), a mode of + ``'w'`` does not truncate the file, and is instead equivalent to ``'a'``. + + When reading, the *options* argument can be a dictionary + providing advanced decompression parameters; see + :class:`DecompressionParameter` for detailed information about supported + parameters. The *zstd_dict* argument is a :class:`ZstdDict` instance to be + used during decompression. When reading, if the *level* + argument is not None, a :exc:`!TypeError` will be raised. + + When writing, the *options* argument can be a dictionary + providing advanced compression parameters; see + :class:`CompressionParameter` for detailed information about supported + parameters. The *level* argument is the compression level to use when + writing compressed data. Only one of *level* or *options* may be passed. The + *zstd_dict* argument is a :class:`ZstdDict` instance to be used during + compression. + + :class:`!ZstdFile` supports all the members specified by + :class:`io.BufferedIOBase`, except for :meth:`~io.BufferedIOBase.detach` + and :meth:`~io.IOBase.truncate`. + Iteration and the :keyword:`with` statement are supported. + + The following method and attributes are also provided: + + .. method:: peek(size=-1) + + Return buffered data without advancing the file position. At least one + byte of data will be returned, unless EOF has been reached. The exact + number of bytes returned is unspecified (the *size* argument is ignored). + + .. note:: While calling :meth:`peek` does not change the file position of + the :class:`ZstdFile`, it may change the position of the underlying + file object (for example, if the :class:`ZstdFile` was constructed by + passing a file object for *file*). + + .. attribute:: mode + + ``'rb'`` for reading and ``'wb'`` for writing. + + .. attribute:: name + + The name of the Zstandard file. Equivalent to the :attr:`~io.FileIO.name` + attribute of the underlying :term:`file object`. + + +Compressing and decompressing data in memory +-------------------------------------------- + +.. function:: compress(data, level=None, options=None, zstd_dict=None) + + Compress *data* (a :term:`bytes-like object`), returning the compressed + data as a :class:`bytes` object. + + The *level* argument is an integer controlling the level of + compression. *level* is an alternative to setting + :attr:`CompressionParameter.compression_level` in *options*. Use + :meth:`~CompressionParameter.bounds` on + :attr:`~CompressionParameter.compression_level` to get the values that can + be passed for *level*. If advanced compression options are needed, the + *level* argument must be omitted and in the *options* dictionary the + :attr:`!CompressionParameter.compression_level` parameter should be set. + + The *options* argument is a Python dictionary containing advanced + compression parameters. The valid keys and values for compression parameters + are documented as part of the :class:`CompressionParameter` documentation. + + The *zstd_dict* argument is an instance of :class:`ZstdDict` + containing trained data to improve compression efficiency. The + function :func:`train_dict` can be used to generate a Zstandard dictionary. + + +.. function:: decompress(data, zstd_dict=None, options=None) + + Decompress *data* (a :term:`bytes-like object`), returning the uncompressed + data as a :class:`bytes` object. + + The *options* argument is a Python dictionary containing advanced + decompression parameters. The valid keys and values for compression + parameters are documented as part of the :class:`DecompressionParameter` + documentation. + + The *zstd_dict* argument is an instance of :class:`ZstdDict` + containing trained data used during compression. This must be + the same Zstandard dictionary used during compression. + + If *data* is the concatenation of multiple distinct compressed frames, + decompress all of these frames, and return the concatenation of the results. + + +.. class:: ZstdCompressor(level=None, options=None, zstd_dict=None) + + Create a compressor object, which can be used to compress data + incrementally. + + For a more convenient way of compressing a single chunk of data, see the + module-level function :func:`compress`. + + The *level* argument is an integer controlling the level of + compression. *level* is an alternative to setting + :attr:`CompressionParameter.compression_level` in *options*. Use + :meth:`~CompressionParameter.bounds` on + :attr:`~CompressionParameter.compression_level` to get the values that can + be passed for *level*. If advanced compression options are needed, the + *level* argument must be omitted and in the *options* dictionary the + :attr:`!CompressionParameter.compression_level` parameter should be set. + + The *options* argument is a Python dictionary containing advanced + compression parameters. The valid keys and values for compression parameters + are documented as part of the :class:`CompressionParameter` documentation. + + The *zstd_dict* argument is an optional instance of :class:`ZstdDict` + containing trained data to improve compression efficiency. The + function :func:`train_dict` can be used to generate a Zstandard dictionary. + + + .. method:: compress(data, mode=ZstdCompressor.CONTINUE) + + Compress *data* (a :term:`bytes-like object`), returning a :class:`bytes` + object with compressed data if possible, or otherwise an empty + :class:`!bytes` object. Some of *data* may be buffered internally, for + use in later calls to :meth:`!compress` and :meth:`~.flush`. The returned + data should be concatenated with the output of any previous calls to + :meth:`~.compress`. + + The *mode* argument is a :class:`ZstdCompressor` attribute, either + :attr:`~.CONTINUE`, :attr:`~.FLUSH_BLOCK`, + or :attr:`~.FLUSH_FRAME`. + + When all data has been provided to the compressor, call the + :meth:`~.flush` method to finish the compression process. If + :meth:`~.compress` is called with *mode* set to :attr:`~.FLUSH_FRAME`, + :meth:`~.flush` should not be called, as it would write out a new empty + frame. + + .. method:: flush(mode=ZstdCompressor.FLUSH_FRAME) + + Finish the compression process, returning a :class:`bytes` object + containing any data stored in the compressor's internal buffers. + + The *mode* argument is a :class:`ZstdCompressor` attribute, either + :attr:`~.FLUSH_BLOCK`, or :attr:`~.FLUSH_FRAME`. + + .. method:: set_pledged_input_size(size) + + Specify the amount of uncompressed data *size* that will be provided for + the next frame. *size* will be written into the frame header of the next + frame unless :attr:`CompressionParameter.content_size_flag` is ``False`` + or ``0``. A size of ``0`` means that the frame is empty. If *size* is + ``None``, the frame header will omit the frame size. Frames that include + the uncompressed data size require less memory to decompress, especially + at higher compression levels. + + If :attr:`last_mode` is not :attr:`FLUSH_FRAME`, a + :exc:`ValueError` is raised as the compressor is not at the start of + a frame. If the pledged size does not match the actual size of data + provided to :meth:`.compress`, future calls to :meth:`!compress` or + :meth:`flush` may raise :exc:`ZstdError` and the last chunk of data may + be lost. + + After :meth:`flush` or :meth:`.compress` are called with mode + :attr:`FLUSH_FRAME`, the next frame will not include the frame size into + the header unless :meth:`!set_pledged_input_size` is called again. + + .. attribute:: CONTINUE + + Collect more data for compression, which may or may not generate output + immediately. This mode optimizes the compression ratio by maximizing the + amount of data per block and frame. + + .. attribute:: FLUSH_BLOCK + + Complete and write a block to the data stream. The data returned so far + can be immediately decompressed. Past data can still be referenced in + future blocks generated by calls to :meth:`~.compress`, + improving compression. + + .. attribute:: FLUSH_FRAME + + Complete and write out a frame. Future data provided to + :meth:`~.compress` will be written into a new frame and + *cannot* reference past data. + + .. attribute:: last_mode + + The last mode passed to either :meth:`~.compress` or :meth:`~.flush`. + The value can be one of :attr:`~.CONTINUE`, :attr:`~.FLUSH_BLOCK`, or + :attr:`~.FLUSH_FRAME`. The initial value is :attr:`~.FLUSH_FRAME`, + signifying that the compressor is at the start of a new frame. + + +.. class:: ZstdDecompressor(zstd_dict=None, options=None) + + Create a decompressor object, which can be used to decompress data + incrementally. + + For a more convenient way of decompressing an entire compressed stream at + once, see the module-level function :func:`decompress`. + + The *options* argument is a Python dictionary containing advanced + decompression parameters. The valid keys and values for compression + parameters are documented as part of the :class:`DecompressionParameter` + documentation. + + The *zstd_dict* argument is an instance of :class:`ZstdDict` + containing trained data used during compression. This must be + the same Zstandard dictionary used during compression. + + .. note:: + This class does not transparently handle inputs containing multiple + compressed frames, unlike the :func:`decompress` function and + :class:`ZstdFile` class. To decompress a multi-frame input, you should + use :func:`decompress`, :class:`ZstdFile` if working with a + :term:`file object`, or multiple :class:`!ZstdDecompressor` instances. + + .. method:: decompress(data, max_length=-1) + + Decompress *data* (a :term:`bytes-like object`), returning + uncompressed data as bytes. Some of *data* may be buffered + internally, for use in later calls to :meth:`!decompress`. + The returned data should be concatenated with the output of any previous + calls to :meth:`!decompress`. + + If *max_length* is non-negative, the method returns at most *max_length* + bytes of decompressed data. If this limit is reached and further + output can be produced, the :attr:`~.needs_input` attribute will + be set to ``False``. In this case, the next call to + :meth:`~.decompress` may provide *data* as ``b''`` to obtain + more of the output. + + If all of the input data was decompressed and returned (either + because this was less than *max_length* bytes, or because + *max_length* was negative), the :attr:`~.needs_input` attribute + will be set to ``True``. + + Attempting to decompress data after the end of a frame will raise a + :exc:`ZstdError`. Any data found after the end of the frame is ignored + and saved in the :attr:`~.unused_data` attribute. + + .. attribute:: eof + + ``True`` if the end-of-stream marker has been reached. + + .. attribute:: unused_data + + Data found after the end of the compressed stream. + + Before the end of the stream is reached, this will be ``b''``. + + .. attribute:: needs_input + + ``False`` if the :meth:`.decompress` method can provide more + decompressed data before requiring new compressed input. + + +Zstandard dictionaries +---------------------- + + +.. function:: train_dict(samples, dict_size) + + Train a Zstandard dictionary, returning a :class:`ZstdDict` instance. + Zstandard dictionaries enable more efficient compression of smaller sizes + of data, which is traditionally difficult to compress due to less + repetition. If you are compressing multiple similar groups of data (such as + similar files), Zstandard dictionaries can improve compression ratios and + speed significantly. + + The *samples* argument (an iterable of :class:`bytes` objects), is the + population of samples used to train the Zstandard dictionary. + + The *dict_size* argument, an integer, is the maximum size (in bytes) the + Zstandard dictionary should be. The Zstandard documentation suggests an + absolute maximum of no more than 100 KB, but the maximum can often be smaller + depending on the data. Larger dictionaries generally slow down compression, + but improve compression ratios. Smaller dictionaries lead to faster + compression, but reduce the compression ratio. + + +.. function:: finalize_dict(zstd_dict, /, samples, dict_size, level) + + An advanced function for converting a "raw content" Zstandard dictionary into + a regular Zstandard dictionary. "Raw content" dictionaries are a sequence of + bytes that do not need to follow the structure of a normal Zstandard + dictionary. + + The *zstd_dict* argument is a :class:`ZstdDict` instance with + the :attr:`~ZstdDict.dict_content` containing the raw dictionary contents. + + The *samples* argument (an iterable of :class:`bytes` objects), contains + sample data for generating the Zstandard dictionary. + + The *dict_size* argument, an integer, is the maximum size (in bytes) the + Zstandard dictionary should be. See :func:`train_dict` for + suggestions on the maximum dictionary size. + + The *level* argument (an integer) is the compression level expected to be + passed to the compressors using this dictionary. The dictionary information + varies for each compression level, so tuning for the proper compression + level can make compression more efficient. + + +.. class:: ZstdDict(dict_content, /, *, is_raw=False) + + A wrapper around Zstandard dictionaries. Dictionaries can be used to improve + the compression of many small chunks of data. Use :func:`train_dict` if you + need to train a new dictionary from sample data. + + The *dict_content* argument (a :term:`bytes-like object`), is the already + trained dictionary information. + + The *is_raw* argument, a boolean, is an advanced parameter controlling the + meaning of *dict_content*. ``True`` means *dict_content* is a "raw content" + dictionary, without any format restrictions. ``False`` means *dict_content* + is an ordinary Zstandard dictionary, created from Zstandard functions, + for example, :func:`train_dict` or the external :program:`zstd` CLI. + + When passing a :class:`!ZstdDict` to a function, the + :attr:`!as_digested_dict` and :attr:`!as_undigested_dict` attributes can + control how the dictionary is loaded by passing them as the ``zstd_dict`` + argument, for example, ``compress(data, zstd_dict=zd.as_digested_dict)``. + Digesting a dictionary is a costly operation that occurs when loading a + Zstandard dictionary. When making multiple calls to compression or + decompression, passing a digested dictionary will reduce the overhead of + loading the dictionary. + + .. list-table:: Difference for compression + :widths: 10 14 10 + :header-rows: 1 + + * - + - Digested dictionary + - Undigested dictionary + * - Advanced parameters of the compressor which may be overridden by + the dictionary's parameters + - ``window_log``, ``hash_log``, ``chain_log``, ``search_log``, + ``min_match``, ``target_length``, ``strategy``, + ``enable_long_distance_matching``, ``ldm_hash_log``, + ``ldm_min_match``, ``ldm_bucket_size_log``, ``ldm_hash_rate_log``, + and some non-public parameters. + - None + * - :class:`!ZstdDict` internally caches the dictionary + - Yes. It's faster when loading a digested dictionary again with the + same compression level. + - No. If you wish to load an undigested dictionary multiple times, + consider reusing a compressor object. + + If passing a :class:`!ZstdDict` without any attribute, an undigested + dictionary is passed by default when compressing and a digested dictionary + is generated if necessary and passed by default when decompressing. + + .. attribute:: dict_content + + The content of the Zstandard dictionary, a ``bytes`` object. It's the + same as the *dict_content* argument in the ``__init__`` method. It can + be used with other programs, such as the ``zstd`` CLI program. + + .. attribute:: dict_id + + Identifier of the Zstandard dictionary, a non-negative int value. + + Non-zero means the dictionary is ordinary, created by Zstandard + functions and following the Zstandard format. + + ``0`` means a "raw content" dictionary, free of any format restriction, + used for advanced users. + + .. note:: + + The meaning of ``0`` for :attr:`!ZstdDict.dict_id` is different + from the ``dictionary_id`` attribute to the :func:`get_frame_info` + function. + + .. attribute:: as_digested_dict + + Load as a digested dictionary. + + .. attribute:: as_undigested_dict + + Load as an undigested dictionary. + + +Advanced parameter control +-------------------------- + +.. class:: CompressionParameter() + + An :class:`~enum.IntEnum` containing the advanced compression parameter + keys that can be used when compressing data. + + The :meth:`~.bounds` method can be used on any attribute to get the valid + values for that parameter. + + Parameters are optional; any omitted parameter will have its value selected + automatically. + + Example getting the lower and upper bound of :attr:`~.compression_level`:: + + lower, upper = CompressionParameter.compression_level.bounds() + + Example setting the :attr:`~.window_log` to the maximum size:: + + _lower, upper = CompressionParameter.window_log.bounds() + options = {CompressionParameter.window_log: upper} + compress(b'venezuelan beaver cheese', options=options) + + .. method:: bounds() + + Return the tuple of int bounds, ``(lower, upper)``, of a compression + parameter. This method should be called on the attribute you wish to + retrieve the bounds of. For example, to get the valid values for + :attr:`~.compression_level`, one may check the result of + ``CompressionParameter.compression_level.bounds()``. + + Both the lower and upper bounds are inclusive. + + .. attribute:: compression_level + + A high-level means of setting other compression parameters that affect + the speed and ratio of compressing data. + + Regular compression levels are greater than ``0``. Values greater than + ``20`` are considered "ultra" compression and require more memory than + other levels. Negative values can be used to trade off faster compression + for worse compression ratios. + + Setting the level to zero uses :attr:`COMPRESSION_LEVEL_DEFAULT`. + + .. attribute:: window_log + + Maximum allowed back-reference distance the compressor can use when + compressing data, expressed as power of two, ``1 << window_log`` bytes. + This parameter greatly influences the memory usage of compression. Higher + values require more memory but gain better compression values. + + A value of zero causes the value to be selected automatically. + + .. attribute:: hash_log + + Size of the initial probe table, as a power of two. The resulting memory + usage is ``1 << (hash_log+2)`` bytes. Larger tables improve compression + ratio of strategies <= :attr:`~Strategy.dfast`, and improve compression + speed of strategies > :attr:`~Strategy.dfast`. + + A value of zero causes the value to be selected automatically. + + .. attribute:: chain_log + + Size of the multi-probe search table, as a power of two. The resulting + memory usage is ``1 << (chain_log+2)`` bytes. Larger tables result in + better and slower compression. This parameter has no effect for the + :attr:`~Strategy.fast` strategy. It's still useful when using + :attr:`~Strategy.dfast` strategy, in which case it defines a secondary + probe table. + + A value of zero causes the value to be selected automatically. + + .. attribute:: search_log + + Number of search attempts, as a power of two. More attempts result in + better and slower compression. This parameter is useless for + :attr:`~Strategy.fast` and :attr:`~Strategy.dfast` strategies. + + A value of zero causes the value to be selected automatically. + + .. attribute:: min_match + + Minimum size of searched matches. Larger values increase compression and + decompression speed, but decrease ratio. Note that Zstandard can still + find matches of smaller size, it just tweaks its search algorithm to look + for this size and larger. For all strategies < :attr:`~Strategy.btopt`, + the effective minimum is ``4``; for all strategies + > :attr:`~Strategy.fast`, the effective maximum is ``6``. + + A value of zero causes the value to be selected automatically. + + .. attribute:: target_length + + The impact of this field depends on the selected :class:`Strategy`. + + For strategies :attr:`~Strategy.btopt`, :attr:`~Strategy.btultra` and + :attr:`~Strategy.btultra2`, the value is the length of a match + considered "good enough" to stop searching. Larger values make + compression ratios better, but compresses slower. + + For strategy :attr:`~Strategy.fast`, it is the distance between match + sampling. Larger values make compression faster, but with a worse + compression ratio. + + A value of zero causes the value to be selected automatically. + + .. attribute:: strategy + + The higher the value of selected strategy, the more complex the + compression technique used by zstd, resulting in higher compression + ratios but slower compression. + + .. seealso:: :class:`Strategy` + + .. attribute:: enable_long_distance_matching + + Long distance matching can be used to improve compression for large + inputs by finding large matches at greater distances. It increases memory + usage and window size. + + ``True`` or ``1`` enable long distance matching while ``False`` or ``0`` + disable it. + + Enabling this parameter increases default + :attr:`~CompressionParameter.window_log` to 128 MiB except when expressly + set to a different value. This setting is enabled by default if + :attr:`!window_log` >= 128 MiB and the compression + strategy >= :attr:`~Strategy.btopt` (compression level 16+). + + .. attribute:: ldm_hash_log + + Size of the table for long distance matching, as a power of two. Larger + values increase memory usage and compression ratio, but decrease + compression speed. + + A value of zero causes the value to be selected automatically. + + .. attribute:: ldm_min_match + + Minimum match size for long distance matcher. Larger or too small values + can often decrease the compression ratio. + + A value of zero causes the value to be selected automatically. + + .. attribute:: ldm_bucket_size_log + + Log size of each bucket in the long distance matcher hash table for + collision resolution. Larger values improve collision resolution but + decrease compression speed. + + A value of zero causes the value to be selected automatically. + + .. attribute:: ldm_hash_rate_log + + Frequency of inserting/looking up entries into the long distance matcher + hash table. Larger values improve compression speed. Deviating far from + the default value will likely result in a compression ratio decrease. + + A value of zero causes the value to be selected automatically. + + .. attribute:: content_size_flag + + Write the size of the data to be compressed into the Zstandard frame + header when known prior to compressing. + + This flag only takes effect under the following scenarios: + + * Calling :func:`compress` for one-shot compression + * Providing all of the data to be compressed in the frame in a single + :meth:`ZstdCompressor.compress` call, with the + :attr:`ZstdCompressor.FLUSH_FRAME` mode. + * Calling :meth:`ZstdCompressor.set_pledged_input_size` with the exact + amount of data that will be provided to the compressor prior to any + calls to :meth:`ZstdCompressor.compress` for the current frame. + :meth:`!ZstdCompressor.set_pledged_input_size` must be called for each + new frame. + + All other compression calls may not write the size information into the + frame header. + + ``True`` or ``1`` enable the content size flag while ``False`` or ``0`` + disable it. + + .. attribute:: checksum_flag + + A four-byte checksum using XXHash64 of the uncompressed content is + written at the end of each frame. Zstandard's decompression code verifies + the checksum. If there is a mismatch a :class:`ZstdError` exception is + raised. + + ``True`` or ``1`` enable checksum generation while ``False`` or ``0`` + disable it. + + .. attribute:: dict_id_flag + + When compressing with a :class:`ZstdDict`, the dictionary's ID is written + into the frame header. + + ``True`` or ``1`` enable storing the dictionary ID while ``False`` or + ``0`` disable it. + + .. attribute:: nb_workers + + Select how many threads will be spawned to compress in parallel. When + :attr:`!nb_workers` > 0, enables multi-threaded compression, a value of + ``1`` means "one-thread multi-threaded mode". More workers improve speed, + but also increase memory usage and slightly reduce compression ratio. + + A value of zero disables multi-threading. + + .. attribute:: job_size + + Size of a compression job, in bytes. This value is enforced only when + :attr:`~CompressionParameter.nb_workers` >= 1. Each compression job is + completed in parallel, so this value can indirectly impact the number of + active threads. + + A value of zero causes the value to be selected automatically. + + .. attribute:: overlap_log + + Sets how much data is reloaded from previous jobs (threads) for new jobs + to be used by the look behind window during compression. This value is + only used when :attr:`~CompressionParameter.nb_workers` >= 1. Acceptable + values vary from 0 to 9. + + * 0 means dynamically set the overlap amount + * 1 means no overlap + * 9 means use a full window size from the previous job + + Each increment halves/doubles the overlap size. "8" means an overlap of + ``window_size/2``, "7" means an overlap of ``window_size/4``, etc. + +.. class:: DecompressionParameter() + + An :class:`~enum.IntEnum` containing the advanced decompression parameter + keys that can be used when decompressing data. Parameters are optional; any + omitted parameter will have its value selected automatically. + + The :meth:`~.bounds` method can be used on any attribute to get the valid + values for that parameter. + + Example setting the :attr:`~.window_log_max` to the maximum size:: + + data = compress(b'Some very long buffer of bytes...') + + _lower, upper = DecompressionParameter.window_log_max.bounds() + + options = {DecompressionParameter.window_log_max: upper} + decompress(data, options=options) + + .. method:: bounds() + + Return the tuple of int bounds, ``(lower, upper)``, of a decompression + parameter. This method should be called on the attribute you wish to + retrieve the bounds of. + + Both the lower and upper bounds are inclusive. + + .. attribute:: window_log_max + + The base-two logarithm of the maximum size of the window used during + decompression. This can be useful to limit the amount of memory used when + decompressing data. A larger maximum window size leads to faster + decompression. + + A value of zero causes the value to be selected automatically. + + +.. class:: Strategy() + + An :class:`~enum.IntEnum` containing strategies for compression. + Higher-numbered strategies correspond to more complex and slower + compression. + + .. note:: + + The values of attributes of :class:`!Strategy` are not necessarily stable + across zstd versions. Only the ordering of the attributes may be relied + upon. The attributes are listed below in order. + + The following strategies are available: + + .. attribute:: fast + + .. attribute:: dfast + + .. attribute:: greedy + + .. attribute:: lazy + + .. attribute:: lazy2 + + .. attribute:: btlazy2 + + .. attribute:: btopt + + .. attribute:: btultra + + .. attribute:: btultra2 + + +Miscellaneous +------------- + +.. function:: get_frame_info(frame_buffer) + + Retrieve a :class:`FrameInfo` object containing metadata about a Zstandard + frame. Frames contain metadata related to the compressed data they hold. + + +.. class:: FrameInfo + + Metadata related to a Zstandard frame. + + .. attribute:: decompressed_size + + The size of the decompressed contents of the frame. + + .. attribute:: dictionary_id + + An integer representing the Zstandard dictionary ID needed for + decompressing the frame. ``0`` means the dictionary ID was not + recorded in the frame header. This may mean that a Zstandard dictionary + is not needed, or that the ID of a required dictionary was not recorded. + + +.. attribute:: COMPRESSION_LEVEL_DEFAULT + + The default compression level for Zstandard: ``3``. + + +.. attribute:: zstd_version_info + + Version number of the runtime zstd library as a tuple of integers + (major, minor, release). + + +Examples +-------- + +Reading in a compressed file: + +.. code-block:: python + + from compression import zstd + + with zstd.open("file.zst") as f: + file_content = f.read() + +Creating a compressed file: + +.. code-block:: python + + from compression import zstd + + data = b"Insert Data Here" + with zstd.open("file.zst", "w") as f: + f.write(data) + +Compressing data in memory: + +.. code-block:: python + + from compression import zstd + + data_in = b"Insert Data Here" + data_out = zstd.compress(data_in) + +Incremental compression: + +.. code-block:: python + + from compression import zstd + + comp = zstd.ZstdCompressor() + out1 = comp.compress(b"Some data\n") + out2 = comp.compress(b"Another piece of data\n") + out3 = comp.compress(b"Even more data\n") + out4 = comp.flush() + # Concatenate all the partial results: + result = b"".join([out1, out2, out3, out4]) + +Writing compressed data to an already-open file: + +.. code-block:: python + + from compression import zstd + + with open("myfile", "wb") as f: + f.write(b"This data will not be compressed\n") + with zstd.open(f, "w") as zstf: + zstf.write(b"This *will* be compressed\n") + f.write(b"Not compressed\n") + +Creating a compressed file using compression parameters: + +.. code-block:: python + + from compression import zstd + + options = { + zstd.CompressionParameter.checksum_flag: 1 + } + with zstd.open("file.zst", "w", options=options) as f: + f.write(b"Mind if I squeeze in?") diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst index 5be1a1106b09a0d..18f9443cbfea20f 100644 --- a/Doc/library/concurrency.rst +++ b/Doc/library/concurrency.rst @@ -18,6 +18,7 @@ multitasking). Here's an overview: multiprocessing.shared_memory.rst concurrent.rst concurrent.futures.rst + concurrent.interpreters.rst subprocess.rst sched.rst queue.rst diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index 7efae9e628b8286..cadf841b43537e5 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -6,12 +6,13 @@ .. versionadded:: 3.2 -**Source code:** :source:`Lib/concurrent/futures/thread.py` -and :source:`Lib/concurrent/futures/process.py` +**Source code:** :source:`Lib/concurrent/futures/thread.py`, +:source:`Lib/concurrent/futures/process.py`, +and :source:`Lib/concurrent/futures/interpreter.py` -------------- -The :mod:`concurrent.futures` module provides a high-level interface for +The :mod:`!concurrent.futures` module provides a high-level interface for asynchronously executing callables. The asynchronous execution can be performed with threads, using @@ -20,6 +21,11 @@ or separate processes, using :class:`ProcessPoolExecutor`. Each implements the same interface, which is defined by the abstract :class:`Executor` class. +:class:`concurrent.futures.Future` must not be confused with +:class:`asyncio.Future`, which is designed for use with :mod:`asyncio` +tasks and coroutines. See the :doc:`asyncio's Future ` +documentation for a detailed comparison of the two. + .. include:: ../includes/wasm-notavail.rst Executor Objects @@ -100,10 +106,10 @@ Executor Objects executor has started running will be completed prior to this method returning. The remaining futures are cancelled. - You can avoid having to call this method explicitly if you use the - :keyword:`with` statement, which will shutdown the :class:`Executor` - (waiting as if :meth:`Executor.shutdown` were called with *wait* set to - ``True``):: + You can avoid having to call this method explicitly if you use the executor + as a :term:`context manager` via the :keyword:`with` statement, which + will shutdown the :class:`Executor` (waiting as if :meth:`Executor.shutdown` + were called with *wait* set to ``True``):: import shutil with ThreadPoolExecutor(max_workers=4) as e: @@ -150,7 +156,9 @@ And:: print(f.result()) executor = ThreadPoolExecutor(max_workers=1) - executor.submit(wait_on_future) + future = executor.submit(wait_on_future) + # Note: calling future.result() would also cause a deadlock because + # the single worker thread is already waiting for wait_on_future(). .. class:: ThreadPoolExecutor(max_workers=None, thread_name_prefix='', initializer=None, initargs=()) @@ -238,6 +246,8 @@ ThreadPoolExecutor Example InterpreterPoolExecutor ----------------------- +.. versionadded:: 3.14 + The :class:`InterpreterPoolExecutor` class uses a pool of interpreters to execute calls asynchronously. It is a :class:`ThreadPoolExecutor` subclass, which means each worker is running in its own thread. @@ -264,7 +274,7 @@ Each worker's interpreter is isolated from all the other interpreters. "Isolated" means each interpreter has its own runtime state and operates completely independently. For example, if you redirect :data:`sys.stdout` in one interpreter, it will not be automatically -redirected any other interpreter. If you import a module in one +redirected to any other interpreter. If you import a module in one interpreter, it is not automatically imported in any other. You would need to import the module separately in interpreter where you need it. In fact, each module imported in an interpreter is @@ -286,7 +296,7 @@ efficient alternative is to serialize with :mod:`pickle` and then send the bytes over a shared :mod:`socket ` or :func:`pipe `. -.. class:: InterpreterPoolExecutor(max_workers=None, thread_name_prefix='', initializer=None, initargs=(), shared=None) +.. class:: InterpreterPoolExecutor(max_workers=None, thread_name_prefix='', initializer=None, initargs=()) A :class:`ThreadPoolExecutor` subclass that executes calls asynchronously using a pool of at most *max_workers* threads. Each thread runs @@ -303,20 +313,9 @@ the bytes over a shared :mod:`socket ` or and *initargs* using :mod:`pickle` when sending them to the worker's interpreter. - .. note:: - Functions defined in the ``__main__`` module cannot be pickled - and thus cannot be used. - .. note:: The executor may replace uncaught exceptions from *initializer* - with :class:`~concurrent.futures.interpreter.ExecutionFailed`. - - The optional *shared* argument is a :class:`dict` of objects that all - interpreters in the pool share. The *shared* items are added to each - interpreter's ``__main__`` module. Not all objects are shareable. - Shareable objects include the builtin singletons, :class:`str` - and :class:`bytes`, and :class:`memoryview`. See :pep:`734` - for more info. + with :class:`~concurrent.interpreters.ExecutionFailed`. Other caveats from parent :class:`ThreadPoolExecutor` apply here. @@ -325,18 +324,14 @@ except the worker serializes the callable and arguments using :mod:`pickle` when sending them to its interpreter. The worker likewise serializes the return value when sending it back. -.. note:: - Functions defined in the ``__main__`` module cannot be pickled - and thus cannot be used. - When a worker's current task raises an uncaught exception, the worker always tries to preserve the exception as-is. If that is successful then it also sets the ``__cause__`` to a corresponding -:class:`~concurrent.futures.interpreter.ExecutionFailed` +:class:`~concurrent.interpreters.ExecutionFailed` instance, which contains a summary of the original exception. In the uncommon case that the worker is not able to preserve the original as-is then it directly preserves the corresponding -:class:`~concurrent.futures.interpreter.ExecutionFailed` +:class:`~concurrent.interpreters.ExecutionFailed` instance instead. @@ -356,6 +351,11 @@ that :class:`ProcessPoolExecutor` will not work in the interactive interpreter. Calling :class:`Executor` or :class:`Future` methods from a callable submitted to a :class:`ProcessPoolExecutor` will result in deadlock. +Note that the restrictions on functions and arguments needing to picklable as +per :class:`multiprocessing.Process` apply when using :meth:`~Executor.submit` +and :meth:`~Executor.map` on a :class:`ProcessPoolExecutor`. A function defined +in a REPL or a lambda should not be expected to work. + .. class:: ProcessPoolExecutor(max_workers=None, mp_context=None, initializer=None, initargs=(), max_tasks_per_child=None) An :class:`Executor` subclass that executes calls asynchronously using a pool @@ -421,6 +421,11 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. require the *fork* start method for :class:`ProcessPoolExecutor` you must explicitly pass ``mp_context=multiprocessing.get_context("fork")``. + .. versionchanged:: next + Fixed a deadlock (:gh:`115634`) where the executor could hang after + a worker process exited upon reaching its *max_tasks_per_child* + limit while tasks remained queued. + .. method:: terminate_workers() Attempt to terminate all living worker processes immediately by calling @@ -722,15 +727,6 @@ Exception classes .. versionadded:: 3.14 -.. exception:: ExecutionFailed - - Raised from :class:`~concurrent.futures.InterpreterPoolExecutor` when - the given initializer fails or from - :meth:`~concurrent.futures.Executor.submit` when there's an uncaught - exception from the submitted task. - - .. versionadded:: 3.14 - .. currentmodule:: concurrent.futures.process .. exception:: BrokenProcessPool diff --git a/Doc/library/concurrent.interpreters.rst b/Doc/library/concurrent.interpreters.rst new file mode 100644 index 000000000000000..ee3f0f04a065566 --- /dev/null +++ b/Doc/library/concurrent.interpreters.rst @@ -0,0 +1,387 @@ +:mod:`!concurrent.interpreters` --- Multiple interpreters in the same process +============================================================================= + +.. module:: concurrent.interpreters + :synopsis: Multiple interpreters in the same process + +.. moduleauthor:: Eric Snow +.. sectionauthor:: Eric Snow + +.. versionadded:: 3.14 + +**Source code:** :source:`Lib/concurrent/interpreters` + +-------------- + +The :mod:`!concurrent.interpreters` module constructs higher-level +interfaces on top of the lower level :mod:`!_interpreters` module. + +The module is primarily meant to provide a basic API for managing +interpreters (AKA "subinterpreters") and running things in them. +Running mostly involves switching to an interpreter (in the current +thread) and calling a function in that execution context. + +For concurrency, interpreters themselves (and this module) don't +provide much more than isolation, which on its own isn't useful. +Actual concurrency is available separately through +:mod:`threads ` -- see `below `_. + +.. seealso:: + + :class:`~concurrent.futures.InterpreterPoolExecutor` + Combines threads with interpreters in a familiar interface. + + .. XXX Add references to the upcoming HOWTO docs in the seealso block. + + :ref:`isolating-extensions-howto` + How to update an extension module to support multiple interpreters. + + :pep:`554` + + :pep:`734` + + :pep:`684` + +.. XXX Why do we disallow multiple interpreters on WASM? + +.. include:: ../includes/wasm-notavail.rst + + +Key details +----------- + +Before we dive in further, there are a small number of details +to keep in mind about using multiple interpreters: + +* `isolated `_, by default +* no implicit threads +* not all PyPI packages support use in multiple interpreters yet + +.. XXX Are there other relevant details to list? + + +.. _interpreters-intro: + +Introduction +------------ + +An "interpreter" is effectively the execution context of the Python +runtime. It contains all of the state the runtime needs to execute +a program. This includes things like the import state and builtins. +(Each thread, even if there's only the main thread, has some extra +runtime state, in addition to the current interpreter, related to +the current exception and the bytecode eval loop.) + +The concept and functionality of the interpreter have been a part of +Python since version 2.2, but the feature was only available through +the C-API and not well known, and the `isolation `_ +was relatively incomplete until version 3.12. + +.. _interp-isolation: + +Multiple Interpreters and Isolation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A Python implementation may support using multiple interpreters in the +same process. CPython has this support. Each interpreter is +effectively isolated from the others (with a limited number of +carefully managed process-global exceptions to the rule). + +That isolation is primarily useful as a strong separation between +distinct logical components of a program, where you want to have +careful control of how those components interact. + +.. note:: + + Interpreters in the same process can technically never be strictly + isolated from one another since there are few restrictions on memory + access within the same process. The Python runtime makes a best + effort at isolation but extension modules may easily violate that. + Therefore, do not use multiple interpreters in security-sensitive + situations, where they shouldn't have access to each other's data. + +Running in an Interpreter +^^^^^^^^^^^^^^^^^^^^^^^^^ + +Running in a different interpreter involves switching to it in the +current thread and then calling some function. The runtime will +execute the function using the current interpreter's state. The +:mod:`!concurrent.interpreters` module provides a basic API for +creating and managing interpreters, as well as the switch-and-call +operation. + +No other threads are automatically started for the operation. +There is `a helper `_ for that though. +There is another dedicated helper for calling the builtin +:func:`exec` in an interpreter. + +When :func:`exec` (or :func:`eval`) are called in an interpreter, +they run using the interpreter's :mod:`!__main__` module as the +"globals" namespace. The same is true for functions that aren't +associated with any module. This is the same as how scripts invoked +from the command-line run in the :mod:`!__main__` module. + + +.. _interp-concurrency: + +Concurrency and Parallelism +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As noted earlier, interpreters do not provide any concurrency +on their own. They strictly represent the isolated execution +context the runtime will use *in the current thread*. That isolation +makes them similar to processes, but they still enjoy in-process +efficiency, like threads. + +All that said, interpreters do naturally support certain flavors of +concurrency. +There's a powerful side effect of that isolation. It enables a +different approach to concurrency than you can take with async or +threads. It's a similar concurrency model to CSP or the actor model, +a model which is relatively easy to reason about. + +You can take advantage of that concurrency model in a single thread, +switching back and forth between interpreters, Stackless-style. +However, this model is more useful when you combine interpreters +with multiple threads. This mostly involves starting a new thread, +where you switch to another interpreter and run what you want there. + +Each actual thread in Python, even if you're only running in the main +thread, has its own *current* execution context. Multiple threads can +use the same interpreter or different ones. + +At a high level, you can think of the combination of threads and +interpreters as threads with opt-in sharing. + +As a significant bonus, interpreters are sufficiently isolated that +they do not share the :term:`GIL`, which means combining threads with +multiple interpreters enables full multi-core parallelism. +(This has been the case since Python 3.12.) + +Communication Between Interpreters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In practice, multiple interpreters are useful only if we have a way +to communicate between them. This usually involves some form of +message passing, but can even mean sharing data in some carefully +managed way. + +With this in mind, the :mod:`!concurrent.interpreters` module provides +a :class:`queue.Queue` implementation, available through +:func:`create_queue`. + +.. _interp-object-sharing: + +"Sharing" Objects +^^^^^^^^^^^^^^^^^ + +Any data actually shared between interpreters loses the thread-safety +provided by the :term:`GIL`. There are various options for dealing with +this in extension modules. However, from Python code the lack of +thread-safety means objects can't actually be shared, with a few +exceptions. Instead, a copy must be created, which means mutable +objects won't stay in sync. + +By default, most objects are copied with :mod:`pickle` when they are +passed to another interpreter. Nearly all of the immutable builtin +objects are either directly shared or copied efficiently. For example: + +* :const:`None` +* :class:`bool` (:const:`True` and :const:`False`) +* :class:`bytes` +* :class:`str` +* :class:`int` +* :class:`float` +* :class:`tuple` (of similarly supported objects) + +There is a small number of Python types that actually share mutable +data between interpreters: + +* :class:`memoryview` +* :class:`Queue` + + +Reference +--------- + +This module defines the following functions: + +.. function:: list_all() + + Return a :class:`list` of :class:`Interpreter` objects, + one for each existing interpreter. + +.. function:: get_current() + + Return an :class:`Interpreter` object for the currently running + interpreter. + +.. function:: get_main() + + Return an :class:`Interpreter` object for the main interpreter. + This is the interpreter the runtime created to run the :term:`REPL` + or the script given at the command-line. It is usually the only one. + +.. function:: create() + + Initialize a new (idle) Python interpreter + and return a :class:`Interpreter` object for it. + +.. function:: create_queue() + + Initialize a new cross-interpreter queue and return a :class:`Queue` + object for it. + + +Interpreter objects +^^^^^^^^^^^^^^^^^^^ + +.. class:: Interpreter(id) + + A single interpreter in the current process. + + Generally, :class:`Interpreter` shouldn't be called directly. + Instead, use :func:`create` or one of the other module functions. + + .. attribute:: id + + (read-only) + + The underlying interpreter's ID. + + .. attribute:: whence + + (read-only) + + A string describing where the interpreter came from. + + .. method:: is_running() + + Return ``True`` if the interpreter is currently executing code + in its :mod:`!__main__` module and ``False`` otherwise. + + .. method:: close() + + Finalize and destroy the interpreter. + + .. method:: prepare_main(ns=None, **kwargs) + + Bind objects in the interpreter's :mod:`!__main__` module. + + Some objects are actually shared and some are copied efficiently, + but most are copied via :mod:`pickle`. See :ref:`interp-object-sharing`. + + .. method:: exec(code, /, dedent=True) + + Run the given source code in the interpreter (in the current thread). + + .. method:: call(callable, /, *args, **kwargs) + + Return the result of calling running the given function in the + interpreter (in the current thread). + + .. _interp-call-in-thread: + + .. method:: call_in_thread(callable, /, *args, **kwargs) + + Run the given function in the interpreter (in a new thread). + +Exceptions +^^^^^^^^^^ + +.. exception:: InterpreterError + + This exception, a subclass of :exc:`Exception`, is raised when + an interpreter-related error happens. + +.. exception:: InterpreterNotFoundError + + This exception, a subclass of :exc:`InterpreterError`, is raised when + the targeted interpreter no longer exists. + +.. exception:: ExecutionFailed + + This exception, a subclass of :exc:`InterpreterError`, is raised when + the running code raised an uncaught exception. + + .. attribute:: excinfo + + A basic snapshot of the exception raised in the other interpreter. + +.. XXX Document the excinfoattrs? + +.. exception:: NotShareableError + + This exception, a subclass of :exc:`TypeError`, is raised when + an object cannot be sent to another interpreter. + + +Communicating Between Interpreters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. class:: Queue(id) + + A wrapper around a low-level, cross-interpreter queue, which + implements the :class:`queue.Queue` interface. The underlying queue + can only be created through :func:`create_queue`. + + Some objects are actually shared and some are copied efficiently, + but most are copied via :mod:`pickle`. See :ref:`interp-object-sharing`. + + .. attribute:: id + + (read-only) + + The queue's ID. + + +.. exception:: QueueEmptyError + + This exception, a subclass of :exc:`queue.Empty`, is raised from + :meth:`!Queue.get` and :meth:`!Queue.get_nowait` when the queue + is empty. + +.. exception:: QueueFullError + + This exception, a subclass of :exc:`queue.Full`, is raised from + :meth:`!Queue.put` and :meth:`!Queue.put_nowait` when the queue + is full. + + +Basic usage +----------- + +Creating an interpreter and running code in it:: + + from concurrent import interpreters + + interp = interpreters.create() + + # Run in the current OS thread. + + interp.exec('print("spam!")') + + interp.exec("""if True: + print('spam!') + """) + + from textwrap import dedent + interp.exec(dedent(""" + print('spam!') + """)) + + def run(arg): + return arg + + res = interp.call(run, 'spam!') + print(res) + + def run(): + print('spam!') + + interp.call(run) + + # Run in new OS thread. + + t = interp.call_in_thread(run) + t.join() diff --git a/Doc/library/concurrent.rst b/Doc/library/concurrent.rst index 8caea78bbb57e82..748c72c733bba26 100644 --- a/Doc/library/concurrent.rst +++ b/Doc/library/concurrent.rst @@ -1,6 +1,7 @@ The :mod:`!concurrent` package ============================== -Currently, there is only one module in this package: +This package contains the following modules: * :mod:`concurrent.futures` -- Launching parallel tasks +* :mod:`concurrent.interpreters` -- Multiple interpreters in the same process diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index bb109a9b742cb77..f73252a90265cf8 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -80,7 +80,7 @@ Let's take a very basic configuration file that looks like this: The structure of INI files is described `in the following section <#supported-ini-file-structure>`_. Essentially, the file consists of sections, each of which contains keys with values. -:mod:`configparser` classes can read and write such files. Let's start by +:mod:`!configparser` classes can read and write such files. Let's start by creating the above configuration file programmatically. .. doctest:: @@ -449,7 +449,7 @@ Mapping Protocol Access .. versionadded:: 3.2 Mapping protocol access is a generic name for functionality that enables using -custom objects as if they were dictionaries. In case of :mod:`configparser`, +custom objects as if they were dictionaries. In case of :mod:`!configparser`, the mapping interface implementation is using the ``parser['section']['option']`` notation. @@ -459,7 +459,7 @@ the original parser on demand. What's even more important is that when values are changed on a section proxy, they are actually mutated in the original parser. -:mod:`configparser` objects behave as close to actual dictionaries as possible. +:mod:`!configparser` objects behave as close to actual dictionaries as possible. The mapping interface is complete and adheres to the :class:`~collections.abc.MutableMapping` ABC. However, there are a few differences that should be taken into account: @@ -507,7 +507,7 @@ Customizing Parser Behaviour ---------------------------- There are nearly as many INI format variants as there are applications using it. -:mod:`configparser` goes a long way to provide support for the largest sensible +:mod:`!configparser` goes a long way to provide support for the largest sensible set of INI styles available. The default functionality is mainly dictated by historical background and it's very likely that you will want to customize some of the features. @@ -560,7 +560,7 @@ the :meth:`!__init__` options: * *allow_no_value*, default value: ``False`` Some configuration files are known to include settings without values, but - which otherwise conform to the syntax supported by :mod:`configparser`. The + which otherwise conform to the syntax supported by :mod:`!configparser`. The *allow_no_value* parameter to the constructor can be used to indicate that such values should be accepted: @@ -615,7 +615,7 @@ the :meth:`!__init__` options: prefixes for whole line comments. .. versionchanged:: 3.2 - In previous versions of :mod:`configparser` behaviour matched + In previous versions of :mod:`!configparser` behaviour matched ``comment_prefixes=('#',';')`` and ``inline_comment_prefixes=(';',)``. Please note that config parsers don't support escaping of comment prefixes so @@ -672,7 +672,7 @@ the :meth:`!__init__` options: parsers in new applications. .. versionchanged:: 3.2 - In previous versions of :mod:`configparser` behaviour matched + In previous versions of :mod:`!configparser` behaviour matched ``strict=False``. * *empty_lines_in_values*, default value: ``True`` @@ -842,7 +842,7 @@ be overridden by subclasses or by attribute assignment. Legacy API Examples ------------------- -Mainly because of backwards compatibility concerns, :mod:`configparser` +Mainly because of backwards compatibility concerns, :mod:`!configparser` provides also a legacy API with explicit ``get``/``set`` methods. While there are valid use cases for the methods outlined below, mapping protocol access is preferred for new projects. The legacy API is at times more advanced, @@ -1378,7 +1378,7 @@ Exceptions .. exception:: Error - Base class for all other :mod:`configparser` exceptions. + Base class for all other :mod:`!configparser` exceptions. .. exception:: NoSectionError diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index c0ac4ea8412ebdd..6f005f98bd3ede5 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -22,7 +22,7 @@ A small number of constants live in the built-in namespace. They are: An object frequently used to represent the absence of a value, as when default arguments are not passed to a function. Assignments to ``None`` are illegal and raise a :exc:`SyntaxError`. - ``None`` is the sole instance of the :data:`~types.NoneType` type. + ``None`` is the sole instance of the :class:`~types.NoneType` type. .. data:: NotImplemented @@ -33,7 +33,7 @@ A small number of constants live in the built-in namespace. They are: the other type; may be returned by the in-place binary special methods (e.g. :meth:`~object.__imul__`, :meth:`~object.__iand__`, etc.) for the same purpose. It should not be evaluated in a boolean context. - :data:`!NotImplemented` is the sole instance of the :data:`types.NotImplementedType` type. + :data:`!NotImplemented` is the sole instance of the :class:`types.NotImplementedType` type. .. note:: @@ -65,9 +65,10 @@ A small number of constants live in the built-in namespace. They are: .. index:: single: ...; ellipsis literal .. data:: Ellipsis - The same as the ellipsis literal "``...``". Special value used mostly in conjunction - with extended slicing syntax for user-defined container data types. - ``Ellipsis`` is the sole instance of the :data:`types.EllipsisType` type. + The same as the ellipsis literal "``...``", an object frequently used to + indicate that something is omitted. Assignment to ``Ellipsis`` is possible, but + assignment to ``...`` raises a :exc:`SyntaxError`. + ``Ellipsis`` is the sole instance of the :class:`types.EllipsisType` type. .. data:: __debug__ @@ -97,15 +98,17 @@ should not be used in programs. exit(code=None) Objects that when printed, print a message like "Use quit() or Ctrl-D - (i.e. EOF) to exit", and when called, raise :exc:`SystemExit` with the + (i.e. EOF) to exit", and when accessed directly in the interactive + interpreter or called as functions, raise :exc:`SystemExit` with the specified exit code. .. data:: help :noindex: Object that when printed, prints the message "Type help() for interactive - help, or help(object) for help about object.", and when called, - acts as described :func:`elsewhere `. + help, or help(object) for help about object.", and when accessed directly + in the interactive interpreter, invokes the built-in help system + (see :func:`help`). .. data:: copyright credits diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 176be4ff3339555..7e221c9cabb365c 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -21,9 +21,9 @@ Functions and classes provided: .. class:: AbstractContextManager An :term:`abstract base class` for classes that implement - :meth:`object.__enter__` and :meth:`object.__exit__`. A default - implementation for :meth:`object.__enter__` is provided which returns - ``self`` while :meth:`object.__exit__` is an abstract method which by default + :meth:`~object.__enter__` and :meth:`~object.__exit__`. A default + implementation for :meth:`~object.__enter__` is provided which returns + ``self`` while :meth:`~object.__exit__` is an abstract method which by default returns ``None``. See also the definition of :ref:`typecontextmanager`. .. versionadded:: 3.6 @@ -32,9 +32,9 @@ Functions and classes provided: .. class:: AbstractAsyncContextManager An :term:`abstract base class` for classes that implement - :meth:`object.__aenter__` and :meth:`object.__aexit__`. A default - implementation for :meth:`object.__aenter__` is provided which returns - ``self`` while :meth:`object.__aexit__` is an abstract method which by default + :meth:`~object.__aenter__` and :meth:`~object.__aexit__`. A default + implementation for :meth:`~object.__aenter__` is provided which returns + ``self`` while :meth:`~object.__aexit__` is an abstract method which by default returns ``None``. See also the definition of :ref:`async-context-managers`. @@ -228,7 +228,7 @@ Functions and classes provided: .. function:: nullcontext(enter_result=None) - Return a context manager that returns *enter_result* from ``__enter__``, but + Return a context manager that returns *enter_result* from :meth:`~object.__enter__`, but otherwise does nothing. It is intended to be used as a stand-in for an optional context manager, for example:: @@ -335,7 +335,7 @@ Functions and classes provided: For example, the output of :func:`help` normally is sent to *sys.stdout*. You can capture that output in a string by redirecting the output to an :class:`io.StringIO` object. The replacement stream is returned from the - ``__enter__`` method and so is available as the target of the + :meth:`~object.__enter__` method and so is available as the target of the :keyword:`with` statement:: with redirect_stdout(io.StringIO()) as f: @@ -396,7 +396,8 @@ Functions and classes provided: A base class that enables a context manager to also be used as a decorator. Context managers inheriting from ``ContextDecorator`` have to implement - ``__enter__`` and ``__exit__`` as normal. ``__exit__`` retains its optional + :meth:`~object.__enter__` and :meth:`~object.__exit__` as normal. + ``__exit__`` retains its optional exception handling even when used as a decorator. ``ContextDecorator`` is used by :func:`contextmanager`, so you get this @@ -668,7 +669,7 @@ Examples and Recipes -------------------- This section describes some examples and recipes for making effective use of -the tools provided by :mod:`contextlib`. +the tools provided by :mod:`!contextlib`. Supporting a variable number of context managers @@ -697,9 +698,9 @@ context management protocol. Catching exceptions from ``__enter__`` methods ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -It is occasionally desirable to catch exceptions from an ``__enter__`` +It is occasionally desirable to catch exceptions from an :meth:`~object.__enter__` method implementation, *without* inadvertently catching exceptions from -the :keyword:`with` statement body or the context manager's ``__exit__`` +the :keyword:`with` statement body or the context manager's :meth:`~object.__exit__` method. By using :class:`ExitStack` the steps in the context management protocol can be separated slightly in order to allow this:: diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 57580ce026e96ac..fffb9f664c0cd70 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -44,6 +44,9 @@ Context Variables references to context variables which prevents context variables from being properly garbage collected. + :class:`!ContextVar`\s are :ref:`generic ` over the type of + their contained value. + .. attribute:: ContextVar.name The name of the variable. This is a read-only property. @@ -77,6 +80,32 @@ Context Variables to restore the variable to its previous value via the :meth:`ContextVar.reset` method. + For convenience, the token object can be used as a context manager + to avoid calling :meth:`ContextVar.reset` manually:: + + var = ContextVar('var', default='default value') + + with var.set('new value'): + assert var.get() == 'new value' + + assert var.get() == 'default value' + + It is a shorthand for:: + + var = ContextVar('var', default='default value') + + token = var.set('new value') + try: + assert var.get() == 'new value' + finally: + var.reset(token) + + assert var.get() == 'default value' + + .. versionadded:: 3.14 + + Added support for using tokens as context managers. + .. method:: reset(token) Reset the context variable to the value it had before the @@ -93,24 +122,21 @@ Context Variables # After the reset call the var has no value again, so # var.get() would raise a LookupError. + The same *token* cannot be used twice. + .. class:: Token *Token* objects are returned by the :meth:`ContextVar.set` method. They can be passed to the :meth:`ContextVar.reset` method to revert the value of the variable to what it was before the corresponding - *set*. - - The token supports :ref:`context manager protocol ` - to restore the corresponding context variable value at the exit from - :keyword:`with` block:: - - var = ContextVar('var', default='default value') + *set*. A single token cannot reset a context variable more than once. - with var.set('new value'): - assert var.get() == 'new value' + Tokens support the :ref:`context manager protocol ` + to automatically reset context variables. See :meth:`ContextVar.set`. - assert var.get() == 'default value' + Tokens are :ref:`generic ` over the same type as the + :class:`ContextVar` which created them. .. versionadded:: 3.14 diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index 95b41f988a035b8..39fc7800d03a916 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -72,15 +72,19 @@ file, socket, window, or any similar types. It does "copy" functions and classes (shallow and deeply), by returning the original object unchanged; this is compatible with the way these are treated by the :mod:`pickle` module. -Shallow copies of dictionaries can be made using :meth:`dict.copy`, and -of lists by assigning a slice of the entire list, for example, -``copied_list = original_list[:]``. +Shallow copies of many collections can be made using the corresponding +:meth:`!copy` method (such as :meth:`list.copy`, :meth:`dict.copy` or +:meth:`set.copy`), and of sequences (such as lists or bytearrays) by making +a slice of the entire sequence (``sequence[:]``). +However, these methods and slicing can create an instance of the base type +when copying an instance of a subclass, whereas :func:`copy.copy` normally +returns an instance of the same type. .. index:: pair: module; pickle Classes can use the same interfaces to control copying that they use to control pickling. See the description of module :mod:`pickle` for information on these -methods. In fact, the :mod:`copy` module uses the registered +methods. In fact, the :mod:`!copy` module uses the registered pickle functions from the :mod:`copyreg` module. .. index:: @@ -122,6 +126,8 @@ and only supports named tuples created by :func:`~collections.namedtuple`, This method should create a new object of the same type, replacing fields with values from *changes*. + .. versionadded:: 3.13 + .. seealso:: diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst index 6e3144824ebe91f..d59936029da69df 100644 --- a/Doc/library/copyreg.rst +++ b/Doc/library/copyreg.rst @@ -12,7 +12,7 @@ -------------- -The :mod:`copyreg` module offers a way to define functions used while pickling +The :mod:`!copyreg` module offers a way to define functions used while pickling specific objects. The :mod:`pickle` and :mod:`copy` modules use those functions when pickling/copying those objects. The module provides configuration information about object constructors which are not classes. diff --git a/Doc/library/crypt.rst b/Doc/library/crypt.rst index 9ff37196ccf69ff..647cb4925f31dad 100644 --- a/Doc/library/crypt.rst +++ b/Doc/library/crypt.rst @@ -13,7 +13,7 @@ being deprecated in Python 3.11. The removal was decided in :pep:`594`. Applications can use the :mod:`hashlib` module from the standard library. Other possible replacements are third-party libraries from PyPI: -:pypi:`legacycrypt`, :pypi:`bcrypt`, :pypi:`argon2-cffi`, or :pypi:`passlib`. +:pypi:`legacycrypt`, :pypi:`bcrypt`, or :pypi:`argon2-cffi`. These are not supported or maintained by the Python core team. The last version of Python that provided the :mod:`!crypt` module was diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 533cdf13974be61..1eb3fe0d4903539 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -25,14 +25,14 @@ similar enough that it is possible to write a single module which can efficiently manipulate such data, hiding the details of reading and writing the data from the programmer. -The :mod:`csv` module implements classes to read and write tabular data in CSV +The :mod:`!csv` module implements classes to read and write tabular data in CSV format. It allows programmers to say, "write this data in the format preferred by Excel," or "read data from this file which was generated by Excel," without knowing the precise details of the CSV format used by Excel. Programmers can also describe the CSV formats understood by other applications or define their own special-purpose CSV formats. -The :mod:`csv` module's :class:`reader` and :class:`writer` objects read and +The :mod:`!csv` module's :class:`reader` and :class:`writer` objects read and write sequences. Programmers can also read and write data in dictionary form using the :class:`DictReader` and :class:`DictWriter` classes. @@ -47,13 +47,13 @@ using the :class:`DictReader` and :class:`DictWriter` classes. Module Contents --------------- -The :mod:`csv` module defines the following functions: +The :mod:`!csv` module defines the following functions: .. index:: single: universal newlines; csv.reader function -.. function:: reader(csvfile, dialect='excel', **fmtparams) +.. function:: reader(csvfile, /, dialect='excel', **fmtparams) Return a :ref:`reader object ` that will process lines from the given *csvfile*. A csvfile must be an iterable of @@ -70,7 +70,7 @@ The :mod:`csv` module defines the following functions: section :ref:`csv-fmt-params`. Each row read from the csv file is returned as a list of strings. No - automatic data type conversion is performed unless the ``QUOTE_NONNUMERIC`` format + automatic data type conversion is performed unless the :data:`QUOTE_NONNUMERIC` format option is specified (in which case unquoted fields are transformed into floats). A short usage example:: @@ -83,8 +83,15 @@ The :mod:`csv` module defines the following functions: Spam, Spam, Spam, Spam, Spam, Baked Beans Spam, Lovely Spam, Wonderful Spam + where :file:`eggs.csv` contains: -.. function:: writer(csvfile, dialect='excel', **fmtparams) + .. code-block:: text + + Spam Spam Spam Spam Spam |Baked Beans| + Spam |Lovely Spam| |Wonderful Spam| + + +.. function:: writer(csvfile, /, dialect='excel', **fmtparams) Return a writer object responsible for converting the user's data into delimited strings on the given file-like object. *csvfile* can be any object with a @@ -112,8 +119,15 @@ The :mod:`csv` module defines the following functions: spamwriter.writerow(['Spam'] * 5 + ['Baked Beans']) spamwriter.writerow(['Spam', 'Lovely Spam', 'Wonderful Spam']) + which writes :file:`eggs.csv` containing: + + .. code-block:: text -.. function:: register_dialect(name[, dialect[, **fmtparams]]) + Spam Spam Spam Spam Spam |Baked Beans| + Spam |Lovely Spam| |Wonderful Spam| + + +.. function:: register_dialect(name, /, dialect='excel', **fmtparams) Associate *dialect* with *name*. *name* must be a string. The dialect can be specified either by passing a sub-class of :class:`Dialect`, or @@ -139,13 +153,14 @@ The :mod:`csv` module defines the following functions: Return the names of all registered dialects. -.. function:: field_size_limit([new_limit]) +.. function:: field_size_limit() + field_size_limit(new_limit) Returns the current maximum field size allowed by the parser. If *new_limit* is given, this becomes the new limit. -The :mod:`csv` module defines the following classes: +The :mod:`!csv` module defines the following classes: .. class:: DictReader(f, fieldnames=None, restkey=None, restval=None, \ dialect='excel', *args, **kwds) @@ -192,6 +207,14 @@ The :mod:`csv` module defines the following classes: >>> print(row) {'first_name': 'John', 'last_name': 'Cleese'} + where :file:`names.csv` contains: + + .. code-block:: text + + first_name,last_name + Eric,Idle + John,Cleese + .. class:: DictWriter(f, fieldnames, restval='', extrasaction='raise', \ dialect='excel', *args, **kwds) @@ -229,6 +252,15 @@ The :mod:`csv` module defines the following classes: writer.writerow({'first_name': 'Lovely', 'last_name': 'Spam'}) writer.writerow({'first_name': 'Wonderful', 'last_name': 'Spam'}) + which writes :file:`names.csv` containing: + + .. code-block:: text + + first_name,last_name + Baked,Beans + Lovely,Spam + Wonderful,Spam + .. class:: Dialect @@ -294,8 +326,8 @@ The :mod:`csv` module defines the following classes: - the second through n-th rows contain strings where at least one value's length differs from that of the putative header of that column. - Twenty rows after the first row are sampled; if more than half of columns + - rows meet the criteria, :const:`True` is returned. + Twenty-one rows after the header are sampled; if more than half of the + columns + rows meet the criteria, :const:`True` is returned. .. note:: @@ -313,7 +345,7 @@ An example for :class:`Sniffer` use:: .. _csv-constants: -The :mod:`csv` module defines the following constants: +The :mod:`!csv` module defines the following constants: .. data:: QUOTE_ALL @@ -323,23 +355,32 @@ The :mod:`csv` module defines the following constants: .. data:: QUOTE_MINIMAL Instructs :class:`writer` objects to only quote those fields which contain - special characters such as *delimiter*, *quotechar* or any of the characters in - *lineterminator*. + special characters such as *delimiter*, *quotechar*, ``'\r'``, ``'\n'`` + or any of the characters in *lineterminator*. .. data:: QUOTE_NONNUMERIC Instructs :class:`writer` objects to quote all non-numeric fields. - Instructs :class:`reader` objects to convert all non-quoted fields to type *float*. + Instructs :class:`reader` objects to convert all non-quoted fields to type :class:`float`. + .. note:: + Some numeric types, such as :class:`bool`, :class:`~fractions.Fraction`, + or :class:`~enum.IntEnum`, have a string representation that cannot be + converted to :class:`float`. + They cannot be read in the :data:`QUOTE_NONNUMERIC` and + :data:`QUOTE_STRINGS` modes. .. data:: QUOTE_NONE - Instructs :class:`writer` objects to never quote fields. When the current - *delimiter* occurs in output data it is preceded by the current *escapechar* - character. If *escapechar* is not set, the writer will raise :exc:`Error` if + Instructs :class:`writer` objects to never quote fields. + When the current *delimiter*, *quotechar*, *escapechar*, ``'\r'``, ``'\n'`` + or any of the characters in *lineterminator* occurs in output data + it is preceded by the current *escapechar* character. + If *escapechar* is not set, the writer will raise :exc:`Error` if any characters that require escaping are encountered. + Set *quotechar* to ``None`` to prevent its escaping. Instructs :class:`reader` objects to perform no special processing of quote characters. @@ -365,7 +406,7 @@ The :mod:`csv` module defines the following constants: .. versionadded:: 3.12 -The :mod:`csv` module defines the following exception: +The :mod:`!csv` module defines the following exception: .. exception:: Error @@ -408,9 +449,16 @@ Dialects support the following attributes: .. attribute:: Dialect.escapechar - A one-character string used by the writer to escape the *delimiter* if *quoting* - is set to :const:`QUOTE_NONE` and the *quotechar* if *doublequote* is - :const:`False`. On reading, the *escapechar* removes any special meaning from + A one-character string used by the writer to escape characters that + require escaping: + + * the *delimiter*, the *quotechar*, ``'\r'``, ``'\n'`` and any of the + characters in *lineterminator* are escaped if *quoting* is set to + :const:`QUOTE_NONE`; + * the *quotechar* is escaped if *doublequote* is :const:`False`; + * the *escapechar* itself. + + On reading, the *escapechar* removes any special meaning from the following character. It defaults to :const:`None`, which disables escaping. .. versionchanged:: 3.11 @@ -430,9 +478,12 @@ Dialects support the following attributes: .. attribute:: Dialect.quotechar - A one-character string used to quote fields containing special characters, such - as the *delimiter* or *quotechar*, or which contain new-line characters. It - defaults to ``'"'``. + A one-character string used to quote fields containing special characters, + such as the *delimiter* or the *quotechar*, or which contain new-line + characters (``'\r'``, ``'\n'`` or any of the characters in *lineterminator*). + It defaults to ``'"'``. + Can be set to ``None`` to prevent escaping ``'"'`` if *quoting* is set + to :const:`QUOTE_NONE`. .. versionchanged:: 3.11 An empty *quotechar* is not allowed. @@ -441,13 +492,15 @@ Dialects support the following attributes: Controls when quotes should be generated by the writer and recognised by the reader. It can take on any of the :ref:`QUOTE_\* constants ` - and defaults to :const:`QUOTE_MINIMAL`. + and defaults to :const:`QUOTE_MINIMAL` if *quotechar* is not ``None``, + and :const:`QUOTE_NONE` otherwise. .. attribute:: Dialect.skipinitialspace When :const:`True`, spaces immediately following the *delimiter* are ignored. - The default is :const:`False`. + The default is :const:`False`. When combining ``delimiter=' '`` with + ``skipinitialspace=True``, unquoted empty fields are not allowed. .. attribute:: Dialect.strict @@ -506,7 +559,7 @@ out surrounded by parens. This may cause some problems for other programs which read CSV files (assuming they support complex numbers at all). -.. method:: csvwriter.writerow(row) +.. method:: csvwriter.writerow(row, /) Write the *row* parameter to the writer's file object, formatted according to the current :class:`Dialect`. Return the return value of the call to the @@ -515,7 +568,7 @@ read CSV files (assuming they support complex numbers at all). .. versionchanged:: 3.5 Added support of arbitrary iterables. -.. method:: csvwriter.writerows(rows) +.. method:: csvwriter.writerows(rows, /) Write all elements in *rows* (an iterable of *row* objects as described above) to the writer's file object, formatted according to the current @@ -603,7 +656,7 @@ A slightly more advanced use of the reader --- catching and reporting errors:: for row in reader: print(row) except csv.Error as e: - sys.exit('file {}, line {}: {}'.format(filename, reader.line_num, e)) + sys.exit(f'file {filename}, line {reader.line_num}: {e}') And while the module doesn't directly support parsing strings, it can easily be done:: @@ -616,7 +669,7 @@ done:: .. rubric:: Footnotes .. [1] If ``newline=''`` is not specified, newlines embedded inside quoted fields - will not be interpreted correctly, and on platforms that use ``\r\n`` linendings + will not be interpreted correctly, and on platforms that use ``\r\n`` line endings on write an extra ``\r`` will be added. It should always be safe to specify ``newline=''``, since the csv module does its own (:term:`universal `) newline handling. diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 5b733d5321e9070..25b27743f6996cf 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -10,20 +10,26 @@ -------------- -:mod:`ctypes` is a foreign function library for Python. It provides C compatible +:mod:`!ctypes` is a foreign function library for Python. It provides C compatible data types, and allows calling functions in DLLs or shared libraries. It can be used to wrap these libraries in pure Python. +.. include:: ../includes/optional-module.rst + +.. warning:: + + :mod:`!ctypes` provides low-level access to native libraries and the + process's memory, bypassing Python's safety mechanisms and allowing + execution of arbitrary native code. + Incorrect use can corrupt data and objects, reveal sensitive information, + cause crashes, or otherwise compromise the running process. + .. _ctypes-ctypes-tutorial: ctypes tutorial --------------- -Note: The code samples in this tutorial use :mod:`doctest` to make sure that -they actually work. Since some code samples behave differently under Linux, -Windows, or macOS, they contain doctest directives in comments. - Note: Some code samples reference the ctypes :class:`c_int` type. On platforms where ``sizeof(long) == sizeof(int)`` it is an alias to :class:`c_long`. So, you should not be confused if :class:`c_long` is printed if you would expect @@ -34,13 +40,16 @@ So, you should not be confused if :class:`c_long` is printed if you would expect Loading dynamic link libraries ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:mod:`ctypes` exports the *cdll*, and on Windows *windll* and *oledll* +:mod:`!ctypes` exports the :py:data:`~ctypes.cdll`, and on Windows +:py:data:`~ctypes.windll` and :py:data:`~ctypes.oledll` objects, for loading dynamic link libraries. -You load libraries by accessing them as attributes of these objects. *cdll* -loads libraries which export functions using the standard ``cdecl`` calling -convention, while *windll* libraries call functions using the ``stdcall`` -calling convention. *oledll* also uses the ``stdcall`` calling convention, and +You load libraries by accessing them as attributes of these objects. +:py:data:`!cdll` loads libraries which export functions using the +standard ``cdecl`` calling convention, while :py:data:`!windll` +libraries call functions using the ``stdcall`` +calling convention. +:py:data:`~oledll` also uses the ``stdcall`` calling convention, and assumes the functions return a Windows :c:type:`!HRESULT` error code. The error code is used to automatically raise an :class:`OSError` exception when the function call fails. @@ -70,11 +79,13 @@ Windows appends the usual ``.dll`` file suffix automatically. being used by Python. Where possible, use native Python functionality, or else import and use the ``msvcrt`` module. -On Linux, it is required to specify the filename *including* the extension to +Other systems require the filename *including* the extension to load a library, so attribute access can not be used to load libraries. Either the :meth:`~LibraryLoader.LoadLibrary` method of the dll loaders should be used, -or you should load the library by creating an instance of CDLL by calling -the constructor:: +or you should load the library by creating an instance of :py:class:`CDLL` +by calling the constructor. + +For example, on Linux:: >>> cdll.LoadLibrary("libc.so.6") # doctest: +LINUX @@ -83,7 +94,14 @@ the constructor:: >>> -.. XXX Add section for macOS. +On macOS:: + + >>> cdll.LoadLibrary("libc.dylib") # doctest: +MACOS + + >>> libc = CDLL("libc.dylib") # doctest: +MACOS + >>> libc # doctest: +MACOS + + .. _ctypes-accessing-functions-from-loaded-dlls: @@ -180,7 +198,7 @@ handle (passing ``None`` as single argument to call it with a ``NULL`` pointer): To find out the correct calling convention you have to look into the C header file or the documentation for the function you want to call. -On Windows, :mod:`ctypes` uses win32 structured exception handling to prevent +On Windows, :mod:`!ctypes` uses win32 structured exception handling to prevent crashes from general protection faults when functions are called with invalid argument values:: @@ -190,10 +208,8 @@ argument values:: OSError: exception: access violation reading 0x00000020 >>> -There are, however, enough ways to crash Python with :mod:`ctypes`, so you -should be careful anyway. The :mod:`faulthandler` module can be helpful in -debugging crashes (e.g. from segmentation faults produced by erroneous C library -calls). +The :mod:`faulthandler` module can help debug crashes, +such as segmentation faults produced by erroneous C library calls. ``None``, integers, bytes objects and (unicode) strings are the only native Python objects that can directly be used as parameters in these function calls. @@ -203,7 +219,7 @@ as pointer to the memory block that contains their data (:c:expr:`char *` or :c:expr:`int` type, their value is masked to fit into the C type. Before we move on calling functions with other parameter types, we have to learn -more about :mod:`ctypes` data types. +more about :mod:`!ctypes` data types. .. _ctypes-fundamental-data-types: @@ -211,73 +227,166 @@ more about :mod:`ctypes` data types. Fundamental data types ^^^^^^^^^^^^^^^^^^^^^^ -:mod:`ctypes` defines a number of primitive C compatible data types: - -+----------------------+------------------------------------------+----------------------------+ -| ctypes type | C type | Python type | -+======================+==========================================+============================+ -| :class:`c_bool` | :c:expr:`_Bool` | bool (1) | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_char` | :c:expr:`char` | 1-character bytes object | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_wchar` | :c:type:`wchar_t` | 1-character string | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_byte` | :c:expr:`char` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ubyte` | :c:expr:`unsigned char` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_short` | :c:expr:`short` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ushort` | :c:expr:`unsigned short` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int` | :c:expr:`int` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint` | :c:expr:`unsigned int` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_long` | :c:expr:`long` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ulong` | :c:expr:`unsigned long` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_longlong` | :c:expr:`__int64` or :c:expr:`long long` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ulonglong` | :c:expr:`unsigned __int64` or | int | -| | :c:expr:`unsigned long long` | | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_size_t` | :c:type:`size_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ssize_t` | :c:type:`ssize_t` or | int | -| | :c:expr:`Py_ssize_t` | | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_time_t` | :c:type:`time_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_float` | :c:expr:`float` | float | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_double` | :c:expr:`double` | float | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_longdouble`| :c:expr:`long double` | float | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_char_p` | :c:expr:`char *` (NUL terminated) | bytes object or ``None`` | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_wchar_p` | :c:expr:`wchar_t *` (NUL terminated) | string or ``None`` | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_void_p` | :c:expr:`void *` | int or ``None`` | -+----------------------+------------------------------------------+----------------------------+ - -(1) - The constructor accepts any object with a truth value. +:mod:`!ctypes` defines a number of primitive C compatible data types: + +.. list-table:: + :header-rows: 1 + + * - ctypes type + - C type + - Python type + - :py:attr:`~_SimpleCData._type_` + * - :class:`c_bool` + - :c:expr:`_Bool` + - :py:class:`bool` + - ``'?'`` + * - :class:`c_char` + - :c:expr:`char` + - 1-character :py:class:`bytes` + - ``'c'`` + * - :class:`c_wchar` + - :c:type:`wchar_t` + - 1-character :py:class:`str` + - ``'u'`` + * - :class:`c_byte` + - :c:expr:`char` + - :py:class:`int` + - ``'b'`` + * - :class:`c_ubyte` + - :c:expr:`unsigned char` + - :py:class:`int` + - ``'B'`` + * - :class:`c_short` + - :c:expr:`short` + - :py:class:`int` + - ``'h'`` + * - :class:`c_ushort` + - :c:expr:`unsigned short` + - :py:class:`int` + - ``'H'`` + * - :class:`c_int` + - :c:expr:`int` + - :py:class:`int` + - ``'i'`` \* + * - :class:`c_int8` + - :c:type:`int8_t` + - :py:class:`int` + - \* + * - :class:`c_int16` + - :c:type:`int16_t` + - :py:class:`int` + - \* + * - :class:`c_int32` + - :c:type:`int32_t` + - :py:class:`int` + - \* + * - :class:`c_int64` + - :c:type:`int64_t` + - :py:class:`int` + - \* + * - :class:`c_uint` + - :c:expr:`unsigned int` + - :py:class:`int` + - ``'I'`` \* + * - :class:`c_uint8` + - :c:type:`uint8_t` + - :py:class:`int` + - \* + * - :class:`c_uint16` + - :c:type:`uint16_t` + - :py:class:`int` + - \* + * - :class:`c_uint32` + - :c:type:`uint32_t` + - :py:class:`int` + - \* + * - :class:`c_uint64` + - :c:type:`uint64_t` + - :py:class:`int` + - \* + * - :class:`c_long` + - :c:expr:`long` + - :py:class:`int` + - ``'l'`` + * - :class:`c_ulong` + - :c:expr:`unsigned long` + - :py:class:`int` + - ``'L'`` + * - :class:`c_longlong` + - :c:expr:`long long` + - :py:class:`int` + - ``'q'`` \* + * - :class:`c_ulonglong` + - :c:expr:`unsigned long long` + - :py:class:`int` + - ``'Q'`` \* + * - :class:`c_size_t` + - :c:type:`size_t` + - :py:class:`int` + - \* + * - :class:`c_ssize_t` + - :c:type:`Py_ssize_t` + - :py:class:`int` + - \* + * - :class:`c_time_t` + - :c:type:`time_t` + - :py:class:`int` + - \* + * - :class:`c_float` + - :c:expr:`float` + - :py:class:`float` + - ``'f'`` + * - :class:`c_double` + - :c:expr:`double` + - :py:class:`float` + - ``'d'`` + * - :class:`c_longdouble` + - :c:expr:`long double` + - :py:class:`float` + - ``'g'`` \* + * - :class:`c_char_p` + - :c:expr:`char *` (NUL terminated) + - :py:class:`bytes` or ``None`` + - ``'z'`` + * - :class:`c_wchar_p` + - :c:expr:`wchar_t *` (NUL terminated) + - :py:class:`str` or ``None`` + - ``'Z'`` + * - :class:`c_void_p` + - :c:expr:`void *` + - :py:class:`int` or ``None`` + - ``'P'`` + * - :class:`py_object` + - :c:expr:`PyObject *` + - :py:class:`object` + - ``'O'`` + * - :ref:`VARIANT_BOOL ` + - :c:expr:`short int` + - :py:class:`bool` + - ``'v'`` Additionally, if IEC 60559 compatible complex arithmetic (Annex G) is supported in both C and ``libffi``, the following complex types are available: -+----------------------------------+---------------------------------+-----------------+ -| ctypes type | C type | Python type | -+==================================+=================================+=================+ -| :class:`c_float_complex` | :c:expr:`float complex` | complex | -+----------------------------------+---------------------------------+-----------------+ -| :class:`c_double_complex` | :c:expr:`double complex` | complex | -+----------------------------------+---------------------------------+-----------------+ -| :class:`c_longdouble_complex` | :c:expr:`long double complex` | complex | -+----------------------------------+---------------------------------+-----------------+ +.. list-table:: + :header-rows: 1 + + * - ctypes type + - C type + - Python type + - :py:attr:`~_SimpleCData._type_` + * - :class:`c_float_complex` + - :c:expr:`float complex` + - :py:class:`complex` + - ``'F'`` + * - :class:`c_double_complex` + - :c:expr:`double complex` + - :py:class:`complex` + - ``'D'`` + * - :class:`c_longdouble_complex` + - :c:expr:`long double complex` + - :py:class:`complex` + - ``'G'`` All these types can be created by calling them with an optional initializer of @@ -291,6 +400,16 @@ the correct type and value:: c_ushort(65533) >>> +The constructors for numeric types will convert input using +:py:meth:`~object.__bool__`, +:py:meth:`~object.__index__` (for ``int``), +:py:meth:`~object.__float__` or :py:meth:`~object.__complex__`. +This means :py:class:`~ctypes.c_bool` accepts any object with a truth value:: + + >>> empty_list = [] + >>> c_bool(empty_list) + c_bool(False) + Since these types are mutable, their value can also be changed afterwards:: >>> i = c_int(42) @@ -379,7 +498,7 @@ from within *IDLE* or *PythonWin*:: >>> As has been mentioned before, all Python types except integers, strings, and -bytes objects have to be wrapped in their corresponding :mod:`ctypes` type, so +bytes objects have to be wrapped in their corresponding :mod:`!ctypes` type, so that they can be converted to the required C data type:: >>> printf(b"An int %d, a double %f\n", 1234, c_double(3.14)) @@ -413,10 +532,10 @@ specify :attr:`~_CFuncPtr.argtypes` for all variadic functions. Calling functions with your own custom data types ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -You can also customize :mod:`ctypes` argument conversion to allow instances of -your own classes be used as function arguments. :mod:`ctypes` looks for an +You can also customize :mod:`!ctypes` argument conversion to allow instances of +your own classes be used as function arguments. :mod:`!ctypes` looks for an :attr:`!_as_parameter_` attribute and uses this as the function argument. The -attribute must be an integer, string, bytes, a :mod:`ctypes` instance, or an +attribute must be an integer, string, bytes, a :mod:`!ctypes` instance, or an object with an :attr:`!_as_parameter_` attribute:: >>> class Bottles: @@ -472,7 +591,7 @@ the Python object passed to the function call, it should do a typecheck or whatever is needed to make sure this object is acceptable, and then return the object itself, its :attr:`!_as_parameter_` attribute, or whatever you want to pass as the C function argument in this case. Again, the result should be an -integer, string, bytes, a :mod:`ctypes` instance, or an object with an +integer, string, bytes, a :mod:`!ctypes` instance, or an object with an :attr:`!_as_parameter_` attribute. @@ -582,7 +701,7 @@ Sometimes a C api function expects a *pointer* to a data type as parameter, probably to write into the corresponding location, or if the data is too large to be passed by value. This is also known as *passing parameters by reference*. -:mod:`ctypes` exports the :func:`byref` function which is used to pass parameters +:mod:`!ctypes` exports the :func:`byref` function which is used to pass parameters by reference. The same effect can be achieved with the :func:`pointer` function, although :func:`pointer` does a lot more work since it constructs a real pointer object, so it is faster to use :func:`byref` if you don't need the pointer @@ -607,12 +726,12 @@ Structures and unions ^^^^^^^^^^^^^^^^^^^^^ Structures and unions must derive from the :class:`Structure` and :class:`Union` -base classes which are defined in the :mod:`ctypes` module. Each subclass must +base classes which are defined in the :mod:`!ctypes` module. Each subclass must define a :attr:`~Structure._fields_` attribute. :attr:`!_fields_` must be a list of *2-tuples*, containing a *field name* and a *field type*. -The field type must be a :mod:`ctypes` type like :class:`c_int`, or any other -derived :mod:`ctypes` type: structure, union, array, pointer. +The field type must be a :mod:`!ctypes` type like :class:`c_int`, or any other +derived :mod:`!ctypes` type: structure, union, array, pointer. Here is a simple example of a POINT structure, which contains two integers named *x* and *y*, and also shows how to initialize a structure in the constructor:: @@ -671,7 +790,7 @@ See :class:`CField`:: .. warning:: - :mod:`ctypes` does not support passing unions or structures with bit-fields + :mod:`!ctypes` does not support passing unions or structures with bit-fields to functions by value. While this may work on 32-bit x86, it's not guaranteed by the library to work in the general case. Unions and structures with bit-fields should always be passed to functions by pointer. @@ -684,16 +803,12 @@ compiler does it. It is possible to override this behavior entirely by specifyi :attr:`~Structure._layout_` class attribute in the subclass definition; see the attribute documentation for details. -It is possible to specify the maximum alignment for the fields by setting -the :attr:`~Structure._pack_` class attribute to a positive integer. -This matches what ``#pragma pack(n)`` does in MSVC. +It is possible to specify the maximum alignment for the fields and/or for the +structure itself by setting the class attributes :attr:`~Structure._pack_` +and/or :attr:`~Structure._align_`, respectively. +See the attribute documentation for details. -It is also possible to set a minimum alignment for how the subclass itself is packed in the -same way ``#pragma align(n)`` works in MSVC. -This can be achieved by specifying a :attr:`~Structure._align_` class attribute -in the subclass definition. - -:mod:`ctypes` uses the native byte order for Structures and Unions. To build +:mod:`!ctypes` uses the native byte order for Structures and Unions. To build structures with non-native byte order, you can use one of the :class:`BigEndianStructure`, :class:`LittleEndianStructure`, :class:`BigEndianUnion`, and :class:`LittleEndianUnion` base classes. These @@ -714,10 +829,16 @@ item in the :attr:`~Structure._fields_` tuples:: ... ("second_16", c_int, 16)] ... >>> print(Int.first_16) - + >>> print(Int.second_16) - - >>> + + +It is important to note that bit field allocation and layout in memory are not +defined as a C standard; their implementation is compiler-specific. +By default, Python will attempt to match the behavior of a "native" compiler +for the current platform. +See the :attr:`~Structure._layout_` attribute for details on the default +behavior and how to change it. .. _ctypes-arrays: @@ -776,7 +897,7 @@ Pointers ^^^^^^^^ Pointer instances are created by calling the :func:`pointer` function on a -:mod:`ctypes` type:: +:mod:`!ctypes` type:: >>> from ctypes import * >>> i = c_int(42) @@ -790,7 +911,7 @@ returns the object to which the pointer points, the ``i`` object above:: c_long(42) >>> -Note that :mod:`ctypes` does not have OOR (original object return), it constructs a +Note that :mod:`!ctypes` does not have OOR (original object return), it constructs a new, equivalent object each time you retrieve an attribute:: >>> pi.contents is i @@ -834,7 +955,7 @@ item. Behind the scenes, the :func:`pointer` function does more than simply create pointer instances, it has to create pointer *types* first. This is done with the -:func:`POINTER` function, which accepts any :mod:`ctypes` type, and returns a +:func:`POINTER` function, which accepts any :mod:`!ctypes` type, and returns a new type:: >>> PI = POINTER(c_int) @@ -856,7 +977,7 @@ Calling the pointer type without an argument creates a ``NULL`` pointer. False >>> -:mod:`ctypes` checks for ``NULL`` when dereferencing pointers (but dereferencing +:mod:`!ctypes` checks for ``NULL`` when dereferencing pointers (but dereferencing invalid non-\ ``NULL`` pointers would crash Python):: >>> null_ptr[0] @@ -876,7 +997,7 @@ invalid non-\ ``NULL`` pointers would crash Python):: Thread safety without the GIL ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -In Python 3.13, the :term:`GIL` may be disabled on :term:`experimental free threaded ` builds. +From Python 3.13 onward, the :term:`GIL` can be disabled on the :term:`free-threaded build`. In ctypes, reads and writes to a single object concurrently is safe, but not across multiple objects: .. code-block:: pycon @@ -941,7 +1062,7 @@ To set a POINTER type field to ``NULL``, you can assign ``None``:: .. XXX list other conversions... Sometimes you have instances of incompatible types. In C, you can cast one type -into another type. :mod:`ctypes` provides a :func:`cast` function which can be +into another type. :mod:`!ctypes` provides a :func:`cast` function which can be used in the same way. The ``Bar`` structure defined above accepts ``POINTER(c_int)`` pointers or :class:`c_int` arrays for its ``values`` field, but not instances of other types:: @@ -1005,7 +1126,7 @@ work:: >>> because the new ``class cell`` is not available in the class statement itself. -In :mod:`ctypes`, we can define the ``cell`` class and set the +In :mod:`!ctypes`, we can define the ``cell`` class and set the :attr:`~Structure._fields_` attribute later, after the class statement:: >>> from ctypes import * @@ -1039,7 +1160,7 @@ other, and finally follow the pointer chain a few times:: Callback functions ^^^^^^^^^^^^^^^^^^ -:mod:`ctypes` allows creating C callable function pointers from Python callables. +:mod:`!ctypes` allows creating C callable function pointers from Python callables. These are sometimes called *callback functions*. First, you must create a class for the callback function. The class knows the @@ -1138,7 +1259,7 @@ write:: .. note:: Make sure you keep references to :func:`CFUNCTYPE` objects as long as they - are used from C code. :mod:`ctypes` doesn't, and if you don't, they may be + are used from C code. :mod:`!ctypes` doesn't, and if you don't, they may be garbage collected, crashing your program when a callback is made. Also, note that if the callback function is called in a thread created @@ -1157,7 +1278,7 @@ Some shared libraries not only export functions, they also export variables. An example in the Python library itself is the :c:data:`Py_Version`, Python runtime version number encoded in a single constant integer. -:mod:`ctypes` can access values like this with the :meth:`~_CData.in_dll` class methods of +:mod:`!ctypes` can access values like this with the :meth:`~_CData.in_dll` class methods of the type. *pythonapi* is a predefined symbol giving access to the Python C api:: @@ -1176,7 +1297,7 @@ Quoting the docs for that value: tricks with this to provide a dynamically created collection of frozen modules. So manipulating this pointer could even prove useful. To restrict the example -size, we show only how this table can be read with :mod:`ctypes`:: +size, we show only how this table can be read with :mod:`!ctypes`:: >>> from ctypes import * >>> @@ -1222,7 +1343,7 @@ for testing. Try it out with ``import __hello__`` for example. Surprises ^^^^^^^^^ -There are some edges in :mod:`ctypes` where you might expect something other +There are some edges in :mod:`!ctypes` where you might expect something other than what actually happens. Consider the following example:: @@ -1290,7 +1411,7 @@ constructs a new Python object each time! Variable-sized data types ^^^^^^^^^^^^^^^^^^^^^^^^^ -:mod:`ctypes` provides some support for variable-sized arrays and structures. +:mod:`!ctypes` provides some support for variable-sized arrays and structures. The :func:`resize` function can be used to resize the memory buffer of an existing ctypes object. The function takes the object as first argument, and @@ -1324,7 +1445,7 @@ get errors accessing other elements:: IndexError: invalid index >>> -Another way to use variable-sized data types with :mod:`ctypes` is to use the +Another way to use variable-sized data types with :mod:`!ctypes` is to use the dynamic nature of Python, and (re-)define the data type after the required size is already known, on a case by case basis. @@ -1368,6 +1489,9 @@ On Linux, :func:`~ctypes.util.find_library` tries to run external programs (``/sbin/ldconfig``, ``gcc``, ``objdump`` and ``ld``) to find the library file. It returns the filename of the library file. +Note that if the output of these programs does not correspond to the dynamic +linker used by Python, the result of this function may be misleading. + .. versionchanged:: 3.6 On Linux, the value of the environment variable ``LD_LIBRARY_PATH`` is used when searching for libraries, if a library cannot be found by any other means. @@ -1402,7 +1526,7 @@ On Windows, :func:`~ctypes.util.find_library` searches along the system search p returns the full pathname, but since there is no predefined naming scheme a call like ``find_library("c")`` will fail and return ``None``. -If wrapping a shared library with :mod:`ctypes`, it *may* be better to determine +If wrapping a shared library with :mod:`!ctypes`, it *may* be better to determine the shared library name at development time, and hardcode that into the wrapper module instead of using :func:`~ctypes.util.find_library` to locate the library at runtime. @@ -1435,14 +1559,82 @@ Loading shared libraries ^^^^^^^^^^^^^^^^^^^^^^^^ There are several ways to load shared libraries into the Python process. One -way is to instantiate one of the following classes: +way is to instantiate :py:class:`CDLL` or one of its subclasses: .. class:: CDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) - Instances of this class represent loaded shared libraries. Functions in these - libraries use the standard C calling convention, and are assumed to return - :c:expr:`int`. + Represents a loaded shared library. + + Functions in this library use the standard C calling convention, and are + assumed to return :c:expr:`int`. + The Python :term:`global interpreter lock` is released before calling any + function exported by these libraries, and reacquired afterwards. + For different function behavior, use a subclass: :py:class:`~ctypes.OleDLL`, + :py:class:`~ctypes.WinDLL`, or :py:class:`~ctypes.PyDLL`. + + If you have an existing :py:attr:`handle ` to an already + loaded shared library, it can be passed as the *handle* argument to wrap + the opened library in a new :py:class:`!CDLL` object. + In this case, *name* is only used to set the :py:attr:`~ctypes.CDLL._name` + attribute, but it may be adjusted and/or validated. + + If *handle* is ``None``, the underlying platform's :manpage:`dlopen(3)` or + :c:func:`!LoadLibrary` function is used to load the library into + the process, and to get a handle to it. + + *name* is the pathname of the shared library to open. + If *name* does not contain a path separator, the library is found + in a platform-specific way. + + On non-Windows systems, *name* can be ``None``. In this case, + :c:func:`!dlopen` is called with ``NULL``, which opens the main program + as a "library". + (Some systems do the same is *name* is empty; ``None``/``NULL`` is more + portable.) + + .. admonition:: CPython implementation detail + + Since CPython is linked to ``libc``, a ``None`` *name* is often used + to access the C standard library:: + + >>> printf = ctypes.CDLL(None).printf + >>> printf.argtypes = [ctypes.c_char_p] + >>> printf(b"hello\n") + hello + 6 + + To access the Python C API, prefer :py:data:`ctypes.pythonapi` which + works across platforms. + + The *mode* parameter can be used to specify how the library is loaded. For + details, consult the :manpage:`dlopen(3)` manpage. On Windows, *mode* is + ignored. On posix systems, RTLD_NOW is always added, and is not + configurable. + + The *use_errno* parameter, when set to true, enables a ctypes mechanism that + allows accessing the system :data:`errno` error number in a safe way. + :mod:`!ctypes` maintains a thread-local copy of the system's :data:`errno` + variable; if you call foreign functions created with ``use_errno=True`` then the + :data:`errno` value before the function call is swapped with the ctypes private + copy, the same happens immediately after the function call. + + The function :func:`ctypes.get_errno` returns the value of the ctypes private + copy, and the function :func:`ctypes.set_errno` changes the ctypes private copy + to a new value and returns the former value. + + The *use_last_error* parameter, when set to true, enables the same mechanism for + the Windows error code which is managed by the :func:`GetLastError` and + :func:`!SetLastError` Windows API functions; :func:`ctypes.get_last_error` and + :func:`ctypes.set_last_error` are used to request and change the ctypes private + copy of the windows error code. + + The *winmode* parameter is used on Windows to specify how the library is loaded + (since *mode* is ignored). It takes any value that is valid for the Win32 API + ``LoadLibraryEx`` flags parameter. When omitted, the default is to use the + flags that result in the most secure DLL load, which avoids issues such as DLL + hijacking. Passing the full path to the DLL is the safest way to ensure the + correct library and dependencies are loaded. On Windows creating a :class:`CDLL` instance may fail even if the DLL name exists. When a dependent DLL of the loaded DLL is not found, a @@ -1454,20 +1646,47 @@ way is to instantiate one of the following classes: DLLs and determine which one is not found using Windows debugging and tracing tools. + .. seealso:: + + `Microsoft DUMPBIN tool `_ + -- A tool to find DLL dependents. + + .. versionchanged:: 3.8 + Added *winmode* parameter. + .. versionchanged:: 3.12 The *name* parameter can now be a :term:`path-like object`. -.. seealso:: + Instances of this class have no public methods. Functions exported by the + shared library can be accessed as attributes or by index. Please note that + accessing the function through an attribute caches the result and therefore + accessing it repeatedly returns the same object each time. On the other hand, + accessing it through an index returns a new object each time:: + + >>> from ctypes import CDLL + >>> libc = CDLL("libc.so.6") # On Linux + >>> libc.time == libc.time + True + >>> libc['time'] == libc['time'] + False + + The following public attributes are available. Their name starts with an + underscore to not clash with exported function names: + + .. attribute:: _handle - `Microsoft DUMPBIN tool `_ - -- A tool to find DLL dependents. + The system handle used to access the library. + .. attribute:: _name -.. class:: OleDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) + The name of the library passed in the constructor. - Instances of this class represent loaded shared libraries, - functions in these libraries use the ``stdcall`` calling convention, and are +.. class:: OleDLL + + See :py:class:`~ctypes.CDLL`, the superclass, for common information. + + Functions in this library use the ``stdcall`` calling convention, and are assumed to return the windows specific :class:`HRESULT` code. :class:`HRESULT` values contain information specifying whether the function call failed or succeeded, together with additional error code. If the return value signals a @@ -1479,133 +1698,51 @@ way is to instantiate one of the following classes: :exc:`WindowsError` used to be raised, which is now an alias of :exc:`OSError`. - .. versionchanged:: 3.12 - - The *name* parameter can now be a :term:`path-like object`. +.. class:: WinDLL -.. class:: WinDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) + See :py:class:`~ctypes.CDLL`, the superclass, for common information. - Instances of this class represent loaded shared libraries, - functions in these libraries use the ``stdcall`` calling convention, and are + Functions in these libraries use the ``stdcall`` calling convention, and are assumed to return :c:expr:`int` by default. .. availability:: Windows - .. versionchanged:: 3.12 - - The *name* parameter can now be a :term:`path-like object`. - - -The Python :term:`global interpreter lock` is released before calling any -function exported by these libraries, and reacquired afterwards. - +.. class:: PyDLL -.. class:: PyDLL(name, mode=DEFAULT_MODE, handle=None) + See :py:class:`~ctypes.CDLL`, the superclass, for common information. - Instances of this class behave like :class:`CDLL` instances, except that the + When functions in this library are called, the Python GIL is *not* released during the function call, and after the function execution the Python error flag is checked. If the error flag is set, a Python exception is raised. - Thus, this is only useful to call Python C api functions directly. - - .. versionchanged:: 3.12 - - The *name* parameter can now be a :term:`path-like object`. - -All these classes can be instantiated by calling them with at least one -argument, the pathname of the shared library. If you have an existing handle to -an already loaded shared library, it can be passed as the ``handle`` named -parameter, otherwise the underlying platform's :c:func:`!dlopen` or -:c:func:`!LoadLibrary` function is used to load the library into -the process, and to get a handle to it. - -The *mode* parameter can be used to specify how the library is loaded. For -details, consult the :manpage:`dlopen(3)` manpage. On Windows, *mode* is -ignored. On posix systems, RTLD_NOW is always added, and is not -configurable. - -The *use_errno* parameter, when set to true, enables a ctypes mechanism that -allows accessing the system :data:`errno` error number in a safe way. -:mod:`ctypes` maintains a thread-local copy of the system's :data:`errno` -variable; if you call foreign functions created with ``use_errno=True`` then the -:data:`errno` value before the function call is swapped with the ctypes private -copy, the same happens immediately after the function call. - -The function :func:`ctypes.get_errno` returns the value of the ctypes private -copy, and the function :func:`ctypes.set_errno` changes the ctypes private copy -to a new value and returns the former value. - -The *use_last_error* parameter, when set to true, enables the same mechanism for -the Windows error code which is managed by the :func:`GetLastError` and -:func:`!SetLastError` Windows API functions; :func:`ctypes.get_last_error` and -:func:`ctypes.set_last_error` are used to request and change the ctypes private -copy of the windows error code. - -The *winmode* parameter is used on Windows to specify how the library is loaded -(since *mode* is ignored). It takes any value that is valid for the Win32 API -``LoadLibraryEx`` flags parameter. When omitted, the default is to use the -flags that result in the most secure DLL load, which avoids issues such as DLL -hijacking. Passing the full path to the DLL is the safest way to ensure the -correct library and dependencies are loaded. - -.. versionchanged:: 3.8 - Added *winmode* parameter. + Thus, this is only useful to call Python C API functions directly. .. data:: RTLD_GLOBAL - :noindex: Flag to use as *mode* parameter. On platforms where this flag is not available, it is defined as the integer zero. .. data:: RTLD_LOCAL - :noindex: Flag to use as *mode* parameter. On platforms where this is not available, it is the same as *RTLD_GLOBAL*. .. data:: DEFAULT_MODE - :noindex: The default mode which is used to load shared libraries. On OSX 10.3, this is *RTLD_GLOBAL*, otherwise it is the same as *RTLD_LOCAL*. -Instances of these classes have no public methods. Functions exported by the -shared library can be accessed as attributes or by index. Please note that -accessing the function through an attribute caches the result and therefore -accessing it repeatedly returns the same object each time. On the other hand, -accessing it through an index returns a new object each time:: - - >>> from ctypes import CDLL - >>> libc = CDLL("libc.so.6") # On Linux - >>> libc.time == libc.time - True - >>> libc['time'] == libc['time'] - False - -The following public attributes are available, their name starts with an -underscore to not clash with exported function names: - - -.. attribute:: PyDLL._handle - - The system handle used to access the library. - - -.. attribute:: PyDLL._name - - The name of the library passed in the constructor. Shared libraries can also be loaded by using one of the prefabricated objects, which are instances of the :class:`LibraryLoader` class, either by calling the :meth:`~LibraryLoader.LoadLibrary` method, or by retrieving the library as attribute of the loader instance. - .. class:: LibraryLoader(dlltype) Class which loads shared libraries. *dlltype* should be one of the @@ -1624,13 +1761,11 @@ attribute of the loader instance. These prefabricated library loaders are available: .. data:: cdll - :noindex: Creates :class:`CDLL` instances. .. data:: windll - :noindex: Creates :class:`WinDLL` instances. @@ -1638,7 +1773,6 @@ These prefabricated library loaders are available: .. data:: oledll - :noindex: Creates :class:`OleDLL` instances. @@ -1646,7 +1780,6 @@ These prefabricated library loaders are available: .. data:: pydll - :noindex: Creates :class:`PyDLL` instances. @@ -1655,7 +1788,6 @@ For accessing the C Python api directly, a ready-to-use Python shared library object is available: .. data:: pythonapi - :noindex: An instance of :class:`PyDLL` that exposes Python C API functions as attributes. Note that all these functions are assumed to return C @@ -1906,7 +2038,7 @@ the windows header file is this:: LPCWSTR lpCaption, UINT uType); -Here is the wrapping with :mod:`ctypes`:: +Here is the wrapping with :mod:`!ctypes`:: >>> from ctypes import c_int, WINFUNCTYPE, windll >>> from ctypes.wintypes import HWND, LPCWSTR, UINT @@ -1929,7 +2061,7 @@ function retrieves the dimensions of a specified window by copying them into HWND hWnd, LPRECT lpRect); -Here is the wrapping with :mod:`ctypes`:: +Here is the wrapping with :mod:`!ctypes`:: >>> from ctypes import POINTER, WINFUNCTYPE, windll, WinError >>> from ctypes.wintypes import BOOL, HWND, RECT @@ -1957,7 +2089,7 @@ do the error checking, and raises an exception when the api call failed:: >>> If the :attr:`~_CFuncPtr.errcheck` function returns the argument tuple it receives -unchanged, :mod:`ctypes` continues the normal processing it does on the output +unchanged, :mod:`!ctypes` continues the normal processing it does on the output parameters. If you want to return a tuple of window coordinates instead of a ``RECT`` instance, you can retrieve the fields in the function and return them instead, the normal processing will no longer take place:: @@ -2031,35 +2163,55 @@ Utility functions pointer. -.. function:: create_string_buffer(init_or_size, size=None) +.. function:: create_string_buffer(init, size=None) + create_string_buffer(size) This function creates a mutable character buffer. The returned object is a ctypes array of :class:`c_char`. - *init_or_size* must be an integer which specifies the size of the array, or a - bytes object which will be used to initialize the array items. + If *size* is given (and not ``None``), it must be an :class:`int`. + It specifies the size of the returned array. + + If the *init* argument is given, it must be :class:`bytes`. It is used + to initialize the array items. Bytes not initialized this way are + set to zero (NUL). + + If *size* is not given (or if it is ``None``), the buffer is made one element + larger than *init*, effectively adding a NUL terminator. + + If both arguments are given, *size* must not be less than ``len(init)``. + + .. warning:: + + If *size* is equal to ``len(init)``, a NUL terminator is + not added. Do not treat such a buffer as a C string. + + For example:: - If a bytes object is specified as first argument, the buffer is made one item - larger than its length so that the last element in the array is a NUL - termination character. An integer can be passed as second argument which allows - specifying the size of the array if the length of the bytes should not be used. + >>> bytes(create_string_buffer(2)) + b'\x00\x00' + >>> bytes(create_string_buffer(b'ab')) + b'ab\x00' + >>> bytes(create_string_buffer(b'ab', 2)) + b'ab' + >>> bytes(create_string_buffer(b'ab', 4)) + b'ab\x00\x00' + >>> bytes(create_string_buffer(b'abcdef', 2)) + Traceback (most recent call last): + ... + ValueError: byte string too long .. audit-event:: ctypes.create_string_buffer init,size ctypes.create_string_buffer -.. function:: create_unicode_buffer(init_or_size, size=None) +.. function:: create_unicode_buffer(init, size=None) + create_unicode_buffer(size) This function creates a mutable unicode character buffer. The returned object is a ctypes array of :class:`c_wchar`. - *init_or_size* must be an integer which specifies the size of the array, or a - string which will be used to initialize the array items. - - If a string is specified as first argument, the buffer is made one item - larger than the length of the string so that the last element in the array is a - NUL termination character. An integer can be passed as second argument which - allows specifying the size of the array if the length of the string should not - be used. + The function takes the same arguments as :func:`~create_string_buffer` except + *init* must be a string and *size* counts :class:`c_wchar`. .. audit-event:: ctypes.create_unicode_buffer init,size ctypes.create_unicode_buffer @@ -2092,6 +2244,8 @@ Utility functions The exact functionality is system dependent. + See :ref:`ctypes-finding-shared-libraries` for complete documentation. + .. function:: find_msvcrt() :module: ctypes.util @@ -2405,10 +2559,33 @@ Fundamental data types Python bytes object or string. When the ``value`` attribute is retrieved from a ctypes instance, usually - a new object is returned each time. :mod:`ctypes` does *not* implement + a new object is returned each time. :mod:`!ctypes` does *not* implement original object return, always a new object is constructed. The same is true for all other ctypes object instances. + Each subclass has a class attribute: + + .. attribute:: _type_ + + Class attribute that contains an internal type code, as a + single-character string. + See :ref:`ctypes-fundamental-data-types` for a summary. + + Types marked \* in the summary may be (or always are) aliases of a + different :class:`_SimpleCData` subclass, and will not necessarily + use the listed type code. + For example, if the platform's :c:expr:`long`, :c:expr:`long long` + and :c:expr:`time_t` C types are the same, then :class:`c_long`, + :class:`c_longlong` and :class:`c_time_t` all refer to a single class, + :class:`c_long`, whose :attr:`_type_` code is ``'l'``. + The ``'L'`` code will be unused. + + .. seealso:: + + The :mod:`array` and :ref:`struct ` modules, + as well as third-party modules like `numpy `__, + use similar -- but slightly different -- type codes. + Fundamental data types, when returned as foreign function call results, or, for example, by retrieving structure field members or array items, are transparently @@ -2498,7 +2675,7 @@ These are the fundamental ctypes data types: .. class:: c_int8 - Represents the C 8-bit :c:expr:`signed int` datatype. Usually an alias for + Represents the C 8-bit :c:expr:`signed int` datatype. It is an alias for :class:`c_byte`. @@ -2530,6 +2707,8 @@ These are the fundamental ctypes data types: Represents the C :c:expr:`signed long long` datatype. The constructor accepts an optional integer initializer; no overflow checking is done. + On platforms where ``sizeof(long long) == sizeof(long)`` it is an alias + to :class:`c_long`. .. class:: c_short @@ -2541,11 +2720,15 @@ These are the fundamental ctypes data types: .. class:: c_size_t Represents the C :c:type:`size_t` datatype. + Usually an alias for another unsigned integer type. .. class:: c_ssize_t - Represents the C :c:type:`ssize_t` datatype. + Represents the :c:type:`Py_ssize_t` datatype. + This is a signed version of :c:type:`size_t`; + that is, the POSIX :c:type:`ssize_t` type. + Usually an alias for another integer type. .. versionadded:: 3.2 @@ -2553,6 +2736,7 @@ These are the fundamental ctypes data types: .. class:: c_time_t Represents the C :c:type:`time_t` datatype. + Usually an alias for another integer type. .. versionadded:: 3.12 @@ -2573,7 +2757,7 @@ These are the fundamental ctypes data types: .. class:: c_uint8 - Represents the C 8-bit :c:expr:`unsigned int` datatype. Usually an alias for + Represents the C 8-bit :c:expr:`unsigned int` datatype. It is an alias for :class:`c_ubyte`. @@ -2605,6 +2789,8 @@ These are the fundamental ctypes data types: Represents the C :c:expr:`unsigned long long` datatype. The constructor accepts an optional integer initializer; no overflow checking is done. + On platforms where ``sizeof(long long) == sizeof(long)`` it is an alias + to :class:`c_long`. .. class:: c_ushort @@ -2656,8 +2842,11 @@ These are the fundamental ctypes data types: .. versionchanged:: 3.14 :class:`!py_object` is now a :term:`generic type`. +.. _ctypes-wintypes: + The :mod:`!ctypes.wintypes` module provides quite some other Windows specific -data types, for example :c:type:`!HWND`, :c:type:`!WPARAM`, or :c:type:`!DWORD`. +data types, for example :c:type:`!HWND`, :c:type:`!WPARAM`, +:c:type:`!VARIANT_BOOL` or :c:type:`!DWORD`. Some useful structures like :c:type:`!MSG` or :c:type:`!RECT` are also defined. @@ -2704,7 +2893,7 @@ fields, or any other data types containing pointer type fields. Abstract base class for structures in *native* byte order. Concrete structure and union types must be created by subclassing one of these - types, and at least define a :attr:`_fields_` class variable. :mod:`ctypes` will + types, and at least define a :attr:`_fields_` class variable. :mod:`!ctypes` will create :term:`descriptor`\s which allow reading and writing the fields by direct attribute accesses. These are the @@ -2750,11 +2939,18 @@ fields, or any other data types containing pointer type fields. .. attribute:: _pack_ An optional small integer that allows overriding the alignment of - structure fields in the instance. :attr:`_pack_` must already be defined - when :attr:`_fields_` is assigned, otherwise it will have no effect. - Setting this attribute to 0 is the same as not setting it at all. + structure fields in the instance. - This is only implemented for the MSVC-compatible memory layout. + This is only implemented for the MSVC-compatible memory layout + (see :attr:`_layout_`). + + Setting :attr:`!_pack_` to 0 is the same as not setting it at all. + Otherwise, the value must be a positive power of two. + The effect is equivalent to ``#pragma pack(N)`` in C, except + :mod:`!ctypes` may allow larger *n* than what the compiler accepts. + + :attr:`!_pack_` must already be defined + when :attr:`_fields_` is assigned, otherwise it will have no effect. .. deprecated-removed:: 3.14 3.19 @@ -2767,9 +2963,22 @@ fields, or any other data types containing pointer type fields. .. attribute:: _align_ - An optional small integer that allows overriding the alignment of + An optional small integer that allows increasing the alignment of the structure when being packed or unpacked to/from memory. - Setting this attribute to 0 is the same as not setting it at all. + + The value must not be negative. + The effect is equivalent to ``__attribute__((aligned(N)))`` on GCC + or ``#pragma align(N)`` on MSVC, except :mod:`!ctypes` may allow + values that the compiler would reject. + + :attr:`!_align_` can only *increase* a structure's alignment + requirements. Setting it to 0 or 1 has no effect. + + Using values that are not powers of two is discouraged and may lead to + surprising behavior. + + :attr:`!_align_` must already be defined + when :attr:`_fields_` is assigned, otherwise it will have no effect. .. versionadded:: 3.13 @@ -2808,7 +3017,7 @@ fields, or any other data types containing pointer type fields. assigned, otherwise it will have no effect. The fields listed in this variable must be structure or union type fields. - :mod:`ctypes` will create descriptors in the structure type that allows + :mod:`!ctypes` will create descriptors in the structure type that allows accessing the nested fields directly, without the need to create the structure or union field. @@ -2939,7 +3148,7 @@ fields, or any other data types containing pointer type fields. .. attribute:: is_anonymous True if this field is anonymous, that is, it contains nested sub-fields - that should be be merged into a containing structure or union. + that should be merged into a containing structure or union. .. _ctypes-arrays-pointers: @@ -2952,12 +3161,14 @@ Arrays and pointers Abstract base class for arrays. The recommended way to create concrete array types is by multiplying any - :mod:`ctypes` data type with a non-negative integer. Alternatively, you can subclass + :mod:`!ctypes` data type with a non-negative integer. Alternatively, you can subclass this type and define :attr:`_length_` and :attr:`_type_` class variables. Array elements can be read and written using standard subscript and slice accesses; for slice reads, the resulting object is *not* itself an :class:`Array`. + Arrays are :ref:`generic ` over the type of their elements. + .. attribute:: _length_ @@ -2978,10 +3189,10 @@ Arrays and pointers Create an array. Equivalent to ``type * length``, where *type* is a - :mod:`ctypes` data type and *length* an integer. + :mod:`!ctypes` data type and *length* an integer. - This function is :term:`soft deprecated` in favor of multiplication. - There are no plans to remove it. + .. soft-deprecated:: 3.14 + In favor of multiplication. .. class:: _Pointer diff --git a/Doc/library/curses.ascii.rst b/Doc/library/curses.ascii.rst index cb895664ff1b11c..86cbbb5a4b7c031 100644 --- a/Doc/library/curses.ascii.rst +++ b/Doc/library/curses.ascii.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`curses.ascii` module supplies name constants for ASCII characters and +The :mod:`!curses.ascii` module supplies name constants for ASCII characters and functions to test membership in various ASCII character classes. The constants supplied are names for control characters as follows: @@ -117,7 +117,7 @@ C library: .. function:: isblank(c) - Checks for an ASCII whitespace character; space or horizontal tab. + Checks for an ASCII blank character; space or horizontal tab. .. function:: iscntrl(c) @@ -133,7 +133,7 @@ C library: .. function:: isgraph(c) - Checks for ASCII any printable character except space. + Checks for any ASCII printable character except space. .. function:: islower(c) @@ -148,7 +148,7 @@ C library: .. function:: ispunct(c) - Checks for any printable ASCII character which is not a space or an alphanumeric + Checks for any ASCII printable character which is not a space or an alphanumeric character. @@ -171,7 +171,8 @@ C library: .. function:: isctrl(c) - Checks for an ASCII control character (ordinal values 0 to 31). + Checks for an ASCII control character (ordinal values 0 to 31). Unlike + :func:`iscntrl`, this does not include the delete character (0x7f). .. function:: ismeta(c) diff --git a/Doc/library/curses.panel.rst b/Doc/library/curses.panel.rst index 11fd841d381f69f..58505827e33cb5b 100644 --- a/Doc/library/curses.panel.rst +++ b/Doc/library/curses.panel.rst @@ -18,7 +18,15 @@ displayed. Panels can be added, moved up or down in the stack, and removed. Functions --------- -The module :mod:`curses.panel` defines the following functions: +The module :mod:`!curses.panel` defines the following exception: + + +.. exception:: error + + Exception raised when a curses panel library function returns an error. + + +The module :mod:`!curses.panel` defines the following functions: .. function:: bottom_panel() @@ -28,7 +36,8 @@ The module :mod:`curses.panel` defines the following functions: .. function:: new_panel(win) - Returns a panel object, associating it with the given window *win*. Be aware + Returns a panel object, associating it with the given window *win* and + placing the new panel on top of the panel stack. Be aware that you need to keep the returned panel object referenced explicitly. If you don't, the panel object is garbage collected and removed from the panel stack. @@ -46,75 +55,94 @@ The module :mod:`curses.panel` defines the following functions: .. _curses-panel-objects: -Panel Objects +Panel objects ------------- -Panel objects, as returned by :func:`new_panel` above, are windows with a -stacking order. There's always a window associated with a panel which determines -the content, while the panel methods are responsible for the window's depth in -the panel stack. +.. raw:: html + + + + + + + + + + + + + + + +.. class:: panel + + Panel objects, as returned by :func:`new_panel` above, are windows with a + stacking order. There's always a window associated with a panel which + determines the content, while the panel methods are responsible for the + window's depth in the panel stack. -Panel objects have the following methods: + Panel objects have the following methods: -.. method:: Panel.above() +.. method:: panel.above() Returns the panel above the current panel. -.. method:: Panel.below() +.. method:: panel.below() Returns the panel below the current panel. -.. method:: Panel.bottom() +.. method:: panel.bottom() Push the panel to the bottom of the stack. -.. method:: Panel.hidden() +.. method:: panel.hidden() Returns ``True`` if the panel is hidden (not visible), ``False`` otherwise. -.. method:: Panel.hide() +.. method:: panel.hide() Hide the panel. This does not delete the object, it just makes the window on screen invisible. -.. method:: Panel.move(y, x) +.. method:: panel.move(y, x) Move the panel to the screen coordinates ``(y, x)``. -.. method:: Panel.replace(win) +.. method:: panel.replace(win) Change the window associated with the panel to the window *win*. -.. method:: Panel.set_userptr(obj) +.. method:: panel.set_userptr(obj) Set the panel's user pointer to *obj*. This is used to associate an arbitrary piece of data with the panel, and can be any Python object. -.. method:: Panel.show() +.. method:: panel.show() - Display the panel (which might have been hidden). + Display the panel (which might have been hidden), placing it on top of + the panel stack. -.. method:: Panel.top() +.. method:: panel.top() Push panel to the top of the stack. -.. method:: Panel.userptr() +.. method:: panel.userptr() Returns the user pointer for the panel. This might be any Python object. -.. method:: Panel.window() +.. method:: panel.window() Returns the window object associated with the panel. diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 137504c51b43588..853ff6ab7332a6a 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -4,7 +4,6 @@ .. module:: curses :synopsis: An interface to the curses library, providing portable terminal handling. - :platform: Unix .. sectionauthor:: Moshe Zadka .. sectionauthor:: Eric Raymond @@ -13,7 +12,7 @@ -------------- -The :mod:`curses` module provides an interface to the curses library, the +The :mod:`!curses` module provides an interface to the curses library, the de-facto standard for portable advanced terminal handling. While curses is most widely used in the Unix environment, versions are available @@ -23,6 +22,10 @@ Linux and the BSD variants of Unix. .. include:: ../includes/wasm-mobile-notavail.rst +.. include:: ../includes/optional-module.rst + +.. availability:: Unix. + .. note:: Whenever the documentation mentions a *character* it can be specified @@ -52,7 +55,7 @@ Linux and the BSD variants of Unix. Functions --------- -The module :mod:`curses` defines the following exception: +The module :mod:`!curses` defines the following exception: .. exception:: error @@ -65,10 +68,10 @@ The module :mod:`curses` defines the following exception: default to the current cursor location. Whenever *attr* is optional, it defaults to :const:`A_NORMAL`. -The module :mod:`curses` defines the following functions: +The module :mod:`!curses` defines the following functions: -.. function:: assume_default_colors(fg, bg) +.. function:: assume_default_colors(fg, bg, /) Allow use of default values for colors on terminals supporting this feature. Use this to support transparency in your application. @@ -80,6 +83,8 @@ The module :mod:`curses` defines the following functions: * Change the definition of the color-pair ``0`` to ``(fg, bg)``. + This is an ncurses extension. + .. versionadded:: 3.14 @@ -116,7 +121,8 @@ The module :mod:`curses` defines the following functions: Return the intensity of the red, green, and blue (RGB) components in the color *color_number*, which must be between ``0`` and ``COLORS - 1``. Return a 3-tuple, containing the R,G,B values for the given color, which will be between - ``0`` (no component) and ``1000`` (maximum amount of component). + ``0`` (no component) and ``1000`` (maximum amount of component). Raise an + exception if the color is not supported. .. function:: color_pair(pair_number) @@ -162,7 +168,7 @@ The module :mod:`curses` defines the following functions: Update the physical screen. The curses library keeps two data structures, one representing the current physical screen contents and a virtual screen - representing the desired next state. The :func:`doupdate` ground updates the + representing the desired next state. The :func:`doupdate` function updates the physical screen to match the virtual screen. The virtual screen may be updated by a :meth:`~window.noutrefresh` call after write @@ -194,7 +200,7 @@ The module :mod:`curses` defines the following functions: .. function:: filter() The :func:`.filter` routine, if used, must be called before :func:`initscr` is - called. The effect is that, during those calls, :envvar:`LINES` is set to ``1``; the + called. The effect is that, during the initialization, :envvar:`LINES` is set to ``1``; the capabilities ``clear``, ``cup``, ``cud``, ``cud1``, ``cuu1``, ``cuu``, ``vpa`` are disabled; and the ``home`` string is set to the value of ``cr``. The effect is that the cursor is confined to the current line, and so are screen updates. This may be used for enabling @@ -240,9 +246,10 @@ The module :mod:`curses` defines the following functions: .. function:: getwin(file) - Read window related data stored in the file by an earlier :func:`window.putwin` call. + Read window-related data stored in the file by an earlier :meth:`window.putwin` call. The routine then creates and initializes a new window using that data, returning - the new window object. + the new window object. The *file* argument must be a file object opened for + reading in binary mode. .. function:: has_colors() @@ -253,7 +260,7 @@ The module :mod:`curses` defines the following functions: Return ``True`` if the module supports extended colors; otherwise, return ``False``. Extended color support allows more than 256 color pairs for - terminals that support more than 16 colors (e.g. xterm-256color). + terminals that support more than 16 colors (for example, xterm-256color). Extended color support requires ncurses version 6.1 or later. @@ -320,12 +327,21 @@ The module :mod:`curses` defines the following functions: Initialize the library. Return a :ref:`window ` object which represents the whole screen. + See :func:`setupterm` for a caveat about calling it before this function. + .. note:: If there is an error opening the terminal, the underlying curses library may cause the interpreter to exit. +.. function:: intrflush(flag) + + If *flag* is ``True``, pressing an interrupt key (interrupt, break, or quit) + will flush all output in the terminal driver queue. If *flag* is ``False``, + no flushing is done. + + .. function:: is_term_resized(nlines, ncols) Return ``True`` if :func:`resize_term` would modify the window structure, @@ -347,6 +363,8 @@ The module :mod:`curses` defines the following functions: bytes object consisting of the prefix ``b'M-'`` followed by the name of the corresponding ASCII character. + Raise a :exc:`ValueError` if *k* is negative. + .. function:: killchar() @@ -372,7 +390,8 @@ The module :mod:`curses` defines the following functions: Set the maximum time in milliseconds that can elapse between press and release events in order for them to be recognized as a click, and return the previous - interval value. The default value is 200 milliseconds, or one fifth of a second. + interval value. The default value is 166 milliseconds, or one sixth of a second. + Use a negative *interval* to obtain the interval value without changing it. .. function:: mousemask(mousemask) @@ -380,7 +399,7 @@ The module :mod:`curses` defines the following functions: Set the mouse events to be reported, and return a tuple ``(availmask, oldmask)``. *availmask* indicates which of the specified mouse events can be reported; on complete failure it returns ``0``. *oldmask* is the previous value of - the given window's mouse event mask. If this function is never called, no mouse + the mouse event mask. If this function is never called, no mouse events are ever reported. @@ -402,7 +421,7 @@ The module :mod:`curses` defines the following functions: methods of a pad require 6 arguments to specify the part of the pad to be displayed and the location on the screen to be used for the display. The arguments are *pminrow*, *pmincol*, *sminrow*, *smincol*, *smaxrow*, *smaxcol*; the *p* - arguments refer to the upper left corner of the pad region to be displayed and + arguments refer to the upper-left corner of the pad region to be displayed and the *s* arguments define a clipping box on the screen within which the pad region is to be displayed. @@ -417,12 +436,14 @@ The module :mod:`curses` defines the following functions: right corner of the screen. -.. function:: nl() +.. function:: nl(flag=True) Enter newline mode. This mode translates the return key into newline on input, and translates newline into return and line-feed on output. Newline mode is initially on. + If *flag* is ``False``, the effect is the same as calling :func:`nonl`. + .. function:: nocbreak() @@ -475,6 +496,8 @@ The module :mod:`curses` defines the following functions: terminfo capability for the current terminal. Note that the output of :func:`putp` always goes to standard output. + :func:`setupterm` (or :func:`initscr`) must be called first. + .. function:: qiflush([flag]) @@ -571,6 +594,17 @@ The module :mod:`curses` defines the following functions: file descriptor to which any initialization sequences will be sent; if not supplied or ``-1``, the file descriptor for ``sys.stdout`` will be used. + Raise a :exc:`curses.error` if the terminal could not be found or its + terminfo database entry could not be read. If the terminal has already + been initialized, this function has no effect. + + .. note:: + + Calling :func:`initscr` after :func:`setupterm` + leaks the terminal that :func:`setupterm` allocated: + the curses library keeps only a single current terminal + and does not free the previously allocated one. + .. function:: start_color() @@ -579,7 +613,7 @@ The module :mod:`curses` defines the following functions: after :func:`initscr`. :func:`start_color` initializes eight basic colors (black, red, green, yellow, - blue, magenta, cyan, and white), and two global variables in the :mod:`curses` + blue, magenta, cyan, and white), and two global variables in the :mod:`!curses` module, :const:`COLORS` and :const:`COLOR_PAIRS`, containing the maximum number of colors and color-pairs the terminal can support. It also restores the colors on the terminal to the values they had when the terminal was just turned on. @@ -605,6 +639,8 @@ The module :mod:`curses` defines the following functions: Boolean capability, or ``0`` if it is canceled or absent from the terminal description. + :func:`setupterm` (or :func:`initscr`) must be called first. + .. function:: tigetnum(capname) @@ -613,6 +649,8 @@ The module :mod:`curses` defines the following functions: numeric capability, or ``-1`` if it is canceled or absent from the terminal description. + :func:`setupterm` (or :func:`initscr`) must be called first. + .. function:: tigetstr(capname) @@ -621,13 +659,17 @@ The module :mod:`curses` defines the following functions: is not a terminfo "string capability", or is canceled or absent from the terminal description. + :func:`setupterm` (or :func:`initscr`) must be called first. + .. function:: tparm(str[, ...]) Instantiate the bytes object *str* with the supplied parameters, where *str* should - be a parameterized string obtained from the terminfo database. E.g. + be a parameterized string obtained from the terminfo database. For example, ``tparm(tigetstr("cup"), 5, 3)`` could result in ``b'\033[6;4H'``, the exact - result depending on terminal type. + result depending on terminal type. Up to nine integer parameters may be supplied. + + :func:`setupterm` (or :func:`initscr`) must be called first. .. function:: typeahead(fd) @@ -713,11 +755,13 @@ The module :mod:`curses` defines the following functions: .. _curses-window-objects: -Window Objects +Window objects -------------- -Window objects, as returned by :func:`initscr` and :func:`newwin` above, have -the following methods and attributes: +.. class:: window + + Window objects, as returned by :func:`initscr` and :func:`newwin` above, have + the following methods and attributes: .. method:: window.addch(ch[, attr]) @@ -730,7 +774,7 @@ the following methods and attributes: .. note:: Writing outside the window, subwindow, or pad raises a :exc:`curses.error`. - Attempting to write to the lower right corner of a window, subwindow, + Attempting to write to the lower-right corner of a window, subwindow, or pad will cause an exception to be raised after the character is printed. @@ -751,15 +795,14 @@ the following methods and attributes: .. note:: * Writing outside the window, subwindow, or pad raises :exc:`curses.error`. - Attempting to write to the lower right corner of a window, subwindow, + Attempting to write to the lower-right corner of a window, subwindow, or pad will cause an exception to be raised after the string is printed. - * A `bug in ncurses `_, the backend - for this Python module, can cause SegFaults when resizing windows. This - is fixed in ncurses-6.1-20190511. If you are stuck with an earlier - ncurses, you can avoid triggering this if you do not call :func:`addstr` - with a *str* that has embedded newlines. Instead, call :func:`addstr` - separately for each line. + * A bug in ncurses, the backend for this Python module, could cause + segfaults when resizing windows. This was fixed in ncurses-6.1-20190511. + If you are stuck with an earlier ncurses, you can avoid triggering it by + not calling :meth:`!addstr` with a *str* that has embedded newlines; + instead, call :meth:`!addstr` separately for each line. .. method:: window.attroff(attr) @@ -770,7 +813,7 @@ the following methods and attributes: .. method:: window.attron(attr) - Add attribute *attr* from the "background" set applied to all writes to the + Add attribute *attr* to the "background" set applied to all writes to the current window. @@ -886,7 +929,8 @@ the following methods and attributes: .. method:: window.delch([y, x]) - Delete any character at ``(y, x)``. + Delete the character under the cursor, or at ``(y, x)`` if specified. All + characters to the right on the same line are shifted one position left. .. method:: window.deleteln() @@ -943,6 +987,7 @@ the following methods and attributes: .. method:: window.getbkgd() Return the given window's current background character/attribute pair. + Its components can be extracted like those of :meth:`inch`. .. method:: window.getch([y, x]) @@ -988,6 +1033,11 @@ the following methods and attributes: window.getstr(y, x, n) Read a bytes object from the user, with primitive line editing capacity. + At most *n* characters are read; + *n* defaults to and cannot exceed 2047. + + .. versionchanged:: 3.14 + The maximum value for *n* was increased from 1023 to 2047. .. method:: window.getyx() @@ -996,11 +1046,12 @@ the following methods and attributes: upper-left corner. -.. method:: window.hline(ch, n) - window.hline(y, x, ch, n) +.. method:: window.hline(ch, n[, attr]) + window.hline(y, x, ch, n[, attr]) Display a horizontal line starting at ``(y, x)`` with length *n* consisting of - the character *ch*. + the character *ch* with attributes *attr*. The line stops at the right edge + of the window if fewer than *n* cells are available. .. method:: window.idcok(flag) @@ -1013,8 +1064,8 @@ the following methods and attributes: .. method:: window.idlok(flag) - If *flag* is ``True``, :mod:`curses` will try and use hardware line - editing facilities. Otherwise, line insertion/deletion are disabled. + If *flag* is ``True``, :mod:`!curses` will try to use hardware line + editing facilities. Otherwise, curses will not use them. .. method:: window.immedok(flag) @@ -1027,15 +1078,21 @@ the following methods and attributes: .. method:: window.inch([y, x]) - Return the character at the given position in the window. The bottom 8 bits are - the character proper, and upper bits are the attributes. + Return the character at the given position in the window. + The bottom 8 bits are the character proper and the upper bits are the attributes; + extract them with the :data:`A_CHARTEXT` and :data:`A_ATTRIBUTES` bit-masks, + and the color pair with :func:`pair_number`. + The character byte is the locale-encoded byte of the cell's character, + consistent with :meth:`instr`. .. method:: window.insch(ch[, attr]) window.insch(y, x, ch[, attr]) - Paint character *ch* at ``(y, x)`` with attributes *attr*, moving the line from - position *x* right by one character. + Insert character *ch* with attributes *attr* before the character under the + cursor, or at ``(y, x)`` if specified. All characters to the right of the + cursor are shifted one position right, with the rightmost character on the + line being lost. The cursor position does not change. .. method:: window.insdelln(nlines) @@ -1076,9 +1133,14 @@ the following methods and attributes: window.instr(y, x[, n]) Return a bytes object of characters, extracted from the window starting at the - current cursor position, or at *y*, *x* if specified. Attributes are stripped + current cursor position, or at *y*, *x* if specified, and stopping at the end + of the line. Attributes and color information are stripped from the characters. If *n* is specified, :meth:`instr` returns a string at most *n* characters long (exclusive of the trailing NUL). + The maximum value for *n* is 2047. + + .. versionchanged:: 3.14 + The maximum value for *n* was increased from 1023 to 2047. .. method:: window.is_linetouched(line) @@ -1097,15 +1159,14 @@ the following methods and attributes: .. method:: window.keypad(flag) If *flag* is ``True``, escape sequences generated by some keys (keypad, function keys) - will be interpreted by :mod:`curses`. If *flag* is ``False``, escape sequences will be + will be interpreted by :mod:`!curses`. If *flag* is ``False``, escape sequences will be left as is in the input stream. .. method:: window.leaveok(flag) If *flag* is ``True``, cursor is left where it is on update, instead of being at "cursor - position." This reduces cursor movement where possible. If possible the cursor - will be made invisible. + position." This reduces cursor movement where possible. If *flag* is ``False``, cursor will always be at "cursor position" after an update. @@ -1126,6 +1187,9 @@ the following methods and attributes: Move the window so its upper-left corner is at ``(new_y, new_x)``. + Moving the window so that any part of it would be off the screen is an error: + the window is not moved and :exc:`curses.error` is raised. + .. method:: window.nodelay(flag) @@ -1141,11 +1205,16 @@ the following methods and attributes: .. method:: window.noutrefresh() + window.noutrefresh(pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol) Mark for refresh but wait. This function updates the data structure representing the desired state of the window, but does not force an update of the physical screen. To accomplish that, call :func:`doupdate`. + The 6 arguments can only be specified, and are then required, when the window + is a pad created with :func:`newpad`; they have the same meaning as for + :meth:`refresh`. + .. method:: window.overlay(destwin[, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol]) @@ -1196,12 +1265,12 @@ the following methods and attributes: Update the display immediately (sync actual screen with previous drawing/deleting methods). - The 6 optional arguments can only be specified when the window is a pad created - with :func:`newpad`. The additional parameters are needed to indicate what part - of the pad and screen are involved. *pminrow* and *pmincol* specify the upper - left-hand corner of the rectangle to be displayed in the pad. *sminrow*, + The 6 arguments can only be specified, and are then required, when the window + is a pad created with :func:`newpad`. The additional parameters are needed to indicate what part + of the pad and screen are involved. *pminrow* and *pmincol* specify the + upper-left corner of the rectangle to be displayed in the pad. *sminrow*, *smincol*, *smaxrow*, and *smaxcol* specify the edges of the rectangle to be - displayed on the screen. The lower right-hand corner of the rectangle to be + displayed on the screen. The lower-right corner of the rectangle to be displayed in the pad is calculated from the screen coordinates, since the rectangles must be the same size. Both rectangles must be entirely contained within their respective structures. Negative values of *pminrow*, *pmincol*, @@ -1218,7 +1287,9 @@ the following methods and attributes: .. method:: window.scroll([lines=1]) - Scroll the screen or scrolling region upward by *lines* lines. + Scroll the screen or scrolling region. Scroll upward by *lines* lines if + *lines* is positive, or downward if it is negative. Scrolling has no effect + unless it has been enabled for the window with :meth:`scrollok`. .. method:: window.scrollok(flag) @@ -1251,15 +1322,17 @@ the following methods and attributes: .. method:: window.subpad(begin_y, begin_x) window.subpad(nlines, ncols, begin_y, begin_x) - Return a sub-window, whose upper-left corner is at ``(begin_y, begin_x)``, and - whose width/height is *ncols*/*nlines*. + Return a sub-pad, whose upper-left corner is at ``(begin_y, begin_x)``, and + whose width/height is *ncols*/*nlines*. The coordinates are relative to the + parent pad (unlike :meth:`subwin`, which uses screen coordinates). This + method is only available for pads created with :func:`newpad`. .. method:: window.subwin(begin_y, begin_x) window.subwin(nlines, ncols, begin_y, begin_x) - Return a sub-window, whose upper-left corner is at ``(begin_y, begin_x)``, and - whose width/height is *ncols*/*nlines*. + Return a sub-window, whose upper-left corner is at the screen-relative + coordinates ``(begin_y, begin_x)``, and whose width/height is *ncols*/*nlines*. By default, the sub-window will extend from the specified position to the lower right corner of the window. @@ -1323,7 +1396,7 @@ the following methods and attributes: Constants --------- -The :mod:`curses` module defines the following data members: +The :mod:`!curses` module defines the following data members: .. data:: ERR @@ -1339,7 +1412,6 @@ The :mod:`curses` module defines the following data members: .. data:: version -.. data:: __version__ A bytes object representing the current version of the module. @@ -1367,14 +1439,14 @@ The :mod:`curses` module defines the following data members: .. data:: COLS - The width of the screen, i.e., the number of columns. + The width of the screen, that is, the number of columns. It is defined only after the call to :func:`initscr`. Updated by :func:`update_lines_cols`, :func:`resizeterm` and :func:`resize_term`. .. data:: LINES - The height of the screen, i.e., the number of lines. + The height of the screen, that is, the number of lines. It is defined only after the call to :func:`initscr`. Updated by :func:`update_lines_cols`, :func:`resizeterm` and :func:`resize_term`. @@ -1677,7 +1749,7 @@ falls back on a crude printable ASCII approximation. +------------------------+------------------------------------------+ | ACS code | Meaning | +========================+==========================================+ -| .. data:: ACS_BBSS | alternate name for upper right corner | +| .. data:: ACS_BBSS | alternate name for upper-right corner | +------------------------+------------------------------------------+ | .. data:: ACS_BLOCK | solid square block | +------------------------+------------------------------------------+ @@ -1685,7 +1757,7 @@ falls back on a crude printable ASCII approximation. +------------------------+------------------------------------------+ | .. data:: ACS_BSBS | alternate name for horizontal line | +------------------------+------------------------------------------+ -| .. data:: ACS_BSSB | alternate name for upper left corner | +| .. data:: ACS_BSSB | alternate name for upper-left corner | +------------------------+------------------------------------------+ | .. data:: ACS_BSSS | alternate name for top tee | +------------------------+------------------------------------------+ @@ -1711,9 +1783,9 @@ falls back on a crude printable ASCII approximation. +------------------------+------------------------------------------+ | .. data:: ACS_LEQUAL | less-than-or-equal-to | +------------------------+------------------------------------------+ -| .. data:: ACS_LLCORNER | lower left-hand corner | +| .. data:: ACS_LLCORNER | lower-left corner | +------------------------+------------------------------------------+ -| .. data:: ACS_LRCORNER | lower right-hand corner | +| .. data:: ACS_LRCORNER | lower-right corner | +------------------------+------------------------------------------+ | .. data:: ACS_LTEE | left tee | +------------------------+------------------------------------------+ @@ -1737,13 +1809,13 @@ falls back on a crude printable ASCII approximation. +------------------------+------------------------------------------+ | .. data:: ACS_S9 | scan line 9 | +------------------------+------------------------------------------+ -| .. data:: ACS_SBBS | alternate name for lower right corner | +| .. data:: ACS_SBBS | alternate name for lower-right corner | +------------------------+------------------------------------------+ | .. data:: ACS_SBSB | alternate name for vertical line | +------------------------+------------------------------------------+ | .. data:: ACS_SBSS | alternate name for right tee | +------------------------+------------------------------------------+ -| .. data:: ACS_SSBB | alternate name for lower left corner | +| .. data:: ACS_SSBB | alternate name for lower-left corner | +------------------------+------------------------------------------+ | .. data:: ACS_SSBS | alternate name for bottom tee | +------------------------+------------------------------------------+ @@ -1757,9 +1829,9 @@ falls back on a crude printable ASCII approximation. +------------------------+------------------------------------------+ | .. data:: ACS_UARROW | up arrow | +------------------------+------------------------------------------+ -| .. data:: ACS_ULCORNER | upper left corner | +| .. data:: ACS_ULCORNER | upper-left corner | +------------------------+------------------------------------------+ -| .. data:: ACS_URCORNER | upper right corner | +| .. data:: ACS_URCORNER | upper-right corner | +------------------------+------------------------------------------+ | .. data:: ACS_VLINE | vertical line | +------------------------+------------------------------------------+ @@ -1783,7 +1855,7 @@ The following table lists mouse button constants used by :meth:`getmouse`: +----------------------------------+---------------------------------------------+ | .. data:: BUTTON_CTRL | Control was down during button state change | +----------------------------------+---------------------------------------------+ -| .. data:: BUTTON_ALT | Control was down during button state change | +| .. data:: BUTTON_ALT | Alt was down during button state change | +----------------------------------+---------------------------------------------+ .. versionchanged:: 3.10 @@ -1813,8 +1885,8 @@ The following table lists the predefined colors: +-------------------------+----------------------------+ -:mod:`curses.textpad` --- Text input widget for curses programs -=============================================================== +:mod:`!curses.textpad` --- Text input widget for curses programs +================================================================ .. module:: curses.textpad :synopsis: Emacs-like input editing in a curses window. @@ -1822,22 +1894,22 @@ The following table lists the predefined colors: .. sectionauthor:: Eric Raymond -The :mod:`curses.textpad` module provides a :class:`Textbox` class that handles +The :mod:`!curses.textpad` module provides a :class:`Textbox` class that handles elementary text editing in a curses window, supporting a set of keybindings resembling those of Emacs (thus, also of Netscape Navigator, BBedit 6.x, FrameMaker, and many other programs). The module also provides a rectangle-drawing function useful for framing text boxes or for other purposes. -The module :mod:`curses.textpad` defines the following function: +The module :mod:`!curses.textpad` defines the following function: .. function:: rectangle(win, uly, ulx, lry, lrx) Draw a rectangle. The first argument must be a window object; the remaining arguments are coordinates relative to that window. The second and third - arguments are the y and x coordinates of the upper left hand corner of the + arguments are the y and x coordinates of the upper-left corner of the rectangle to be drawn; the fourth and fifth arguments are the y and x - coordinates of the lower right hand corner. The rectangle will be drawn using + coordinates of the lower-right corner. The rectangle will be drawn using VT100/IBM PC forms characters on terminals that make this possible (including xterm and most other software terminal emulators). Otherwise it will be drawn with ASCII dashes, vertical bars, and plus signs. @@ -1851,32 +1923,36 @@ Textbox objects You can instantiate a :class:`Textbox` object as follows: -.. class:: Textbox(win) +.. class:: Textbox(win, insert_mode=False) Return a textbox widget object. The *win* argument should be a curses :ref:`window ` object in which the textbox is to - be contained. The edit cursor of the textbox is initially located at the - upper left hand corner of the containing window, with coordinates ``(0, 0)``. + be contained. If *insert_mode* is true, the textbox inserts typed + characters, shifting existing text to the right, rather than overwriting it. + The edit cursor of the textbox is initially located at the + upper-left corner of the containing window, with coordinates ``(0, 0)``. The instance's :attr:`stripspaces` flag is initially on. :class:`Textbox` objects have the following methods: - .. method:: edit([validator]) + .. method:: edit(validate=None) This is the entry point you will normally use. It accepts editing keystrokes until one of the termination keystrokes is entered. If - *validator* is supplied, it must be a function. It will be called for + *validate* is supplied, it must be a function. It will be called for each keystroke entered with the keystroke as a parameter; command dispatch - is done on the result. This method returns the window contents as a + is done on the result. If it returns a false value, the keystroke is + ignored. This method returns the window contents as a string; whether blanks in the window are included is affected by the :attr:`stripspaces` attribute. .. method:: do_command(ch) - Process a single command keystroke. Here are the supported special - keystrokes: + Process a single command keystroke. Returns ``1`` to continue editing, + or ``0`` if a termination keystroke was processed. Here are the supported + special keystrokes: +------------------+-------------------------------------------+ | Keystroke | Action | @@ -1899,7 +1975,8 @@ You can instantiate a :class:`Textbox` object as follows: | :kbd:`Control-H` | Delete character backward. | +------------------+-------------------------------------------+ | :kbd:`Control-J` | Terminate if the window is 1 line, | - | | otherwise insert newline. | + | | otherwise move to the start of the next | + | | line. | +------------------+-------------------------------------------+ | :kbd:`Control-K` | If line is blank, delete it, otherwise | | | clear to end of line. | diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index 72612211b43d5e7..67b20b8a7938555 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -103,12 +103,25 @@ Module contents ignored. - *eq*: If true (the default), an :meth:`~object.__eq__` method will be - generated. This method compares the class as if it were a tuple - of its fields, in order. Both instances in the comparison must - be of the identical type. + generated. - If the class already defines :meth:`!__eq__`, this parameter is - ignored. + This method compares the class by comparing each field in order. Both + instances in the comparison must be of the identical type. + + If the class already defines :meth:`!__eq__`, this parameter is ignored. + + .. versionchanged:: 3.13 + The generated ``__eq__`` method now compares each field individually + (for example, ``self.a == other.a and self.b == other.b``), rather than + comparing tuples of fields as in previous versions. + + This change makes the comparison faster but it may alter results in cases + where attributes compare equal by identity but not by value (such as + ``float('nan')``). + + In Python 3.12 and earlier, the comparison was performed by creating + tuples of the fields and comparing them (for example, + ``(self.a, self.b) == (other.a, other.b)``). - *order*: If true (the default is ``False``), :meth:`~object.__lt__`, :meth:`~object.__le__`, :meth:`~object.__gt__`, and :meth:`~object.__ge__` methods will be @@ -121,8 +134,11 @@ Module contents :meth:`!__le__`, :meth:`!__gt__`, or :meth:`!__ge__`, then :exc:`TypeError` is raised. - - *unsafe_hash*: If ``False`` (the default), a :meth:`~object.__hash__` method - is generated according to how *eq* and *frozen* are set. + - *unsafe_hash*: If true, force ``dataclasses`` to create a + :meth:`~object.__hash__` method, even though it may not be safe to do so. + Otherwise, generate a :meth:`~object.__hash__` method according to how + *eq* and *frozen* are set. + The default value is ``False``. :meth:`!__hash__` is used by built-in :meth:`hash`, and when objects are added to hashed collections such as dictionaries and sets. Having a @@ -158,9 +174,11 @@ Module contents :class:`object`, this means it will fall back to id-based hashing). - *frozen*: If true (the default is ``False``), assigning to fields will - generate an exception. This emulates read-only frozen instances. If - :meth:`~object.__setattr__` or :meth:`~object.__delattr__` is defined in the class, then - :exc:`TypeError` is raised. See the discussion below. + generate an exception. This emulates read-only frozen instances. + See the :ref:`discussion ` below. + + If :meth:`~object.__setattr__` or :meth:`~object.__delattr__` is defined in the class + and *frozen* is true, then :exc:`TypeError` is raised. - *match_args*: If true (the default is ``True``), the :attr:`~object.__match_args__` tuple will be created from the list of @@ -304,15 +322,15 @@ Module contents .. versionadded:: 3.10 - - ``doc``: optional docstring for this field. + - *doc*: optional docstring for this field. - .. versionadded:: 3.13 + .. versionadded:: 3.14 If the default value of a field is specified by a call to :func:`!field`, then the class attribute for this field will be replaced by the specified *default* value. If *default* is not provided, then the class attribute will be deleted. The intent is - that after the :func:`@dataclass ` decorator runs, the class + that after the :deco:`dataclass` decorator runs, the class attributes will all contain the default values for the fields, just as if the default value itself were specified. For example, after:: @@ -422,7 +440,7 @@ Module contents :data:`typing.Any` is used for ``type``. The values of *init*, *repr*, *eq*, *order*, *unsafe_hash*, *frozen*, *match_args*, *kw_only*, *slots*, and *weakref_slot* have - the same meaning as they do in :func:`@dataclass `. + the same meaning as they do in :deco:`dataclass`. If *module* is defined, the :attr:`!__module__` attribute of the dataclass is set to that value. @@ -430,12 +448,12 @@ Module contents The *decorator* parameter is a callable that will be used to create the dataclass. It should take the class object as a first argument and the same keyword arguments - as :func:`@dataclass `. By default, the :func:`@dataclass ` + as :deco:`dataclass`. By default, the :deco:`dataclass` function is used. This function is not strictly required, because any Python - mechanism for creating a new class with :attr:`!__annotations__` can - then apply the :func:`@dataclass ` function to convert that class to + mechanism for creating a new class with :attr:`~object.__annotations__` can + then apply the :deco:`dataclass` function to convert that class to a dataclass. This function is provided as a convenience. For example:: @@ -492,7 +510,8 @@ Module contents .. function:: is_dataclass(obj) Return ``True`` if its parameter is a dataclass (including subclasses of a - dataclass) or an instance of one, otherwise return ``False``. + dataclass, but not including :ref:`generic aliases `) + or an instance of one, otherwise return ``False``. If you need to know if a class is an instance of a dataclass (and not a dataclass itself), then add a further check for ``not @@ -564,7 +583,7 @@ Post-init processing def __post_init__(self): self.c = self.a + self.b -The :meth:`~object.__init__` method generated by :func:`@dataclass ` does not call base +The :meth:`~object.__init__` method generated by :deco:`dataclass` does not call base class :meth:`!__init__` methods. If the base class has an :meth:`!__init__` method that has to be called, it is common to call this method in a :meth:`__post_init__` method:: @@ -594,7 +613,7 @@ parameters to :meth:`!__post_init__`. Also see the warning about how Class variables --------------- -One of the few places where :func:`@dataclass ` actually inspects the type +One of the few places where :deco:`dataclass` actually inspects the type of a field is to determine if a field is a class variable as defined in :pep:`526`. It does this by checking if the type of the field is :data:`typing.ClassVar`. If a field is a ``ClassVar``, it is excluded @@ -607,7 +626,7 @@ module-level :func:`fields` function. Init-only variables ------------------- -Another place where :func:`@dataclass ` inspects a type annotation is to +Another place where :deco:`dataclass` inspects a type annotation is to determine if a field is an init-only variable. It does this by seeing if the type of a field is of type :class:`InitVar`. If a field is an :class:`InitVar`, it is considered a pseudo-field called an init-only @@ -641,7 +660,7 @@ Frozen instances ---------------- It is not possible to create truly immutable Python objects. However, -by passing ``frozen=True`` to the :func:`@dataclass ` decorator you can +by passing ``frozen=True`` to the :deco:`dataclass` decorator you can emulate immutability. In that case, dataclasses will add :meth:`~object.__setattr__` and :meth:`~object.__delattr__` methods to the class. These methods will raise a :exc:`FrozenInstanceError` when invoked. @@ -657,7 +676,7 @@ must use :meth:`!object.__setattr__`. Inheritance ----------- -When the dataclass is being created by the :func:`@dataclass ` decorator, +When the dataclass is being created by the :deco:`dataclass` decorator, it looks through all of the class's base classes in reverse MRO (that is, starting at :class:`object`) and, for each dataclass that it finds, adds the fields from that base class to an ordered mapping of fields. @@ -781,7 +800,7 @@ for :attr:`!x` when creating a class instance will share the same copy of :attr:`!x`. Because dataclasses just use normal Python class creation they also share this behavior. There is no general way for Data Classes to detect this condition. Instead, the -:func:`@dataclass ` decorator will raise a :exc:`ValueError` if it +:deco:`dataclass` decorator will raise a :exc:`ValueError` if it detects an unhashable default parameter. The assumption is that if a value is unhashable, it is mutable. This is a partial solution, but it does protect against many common errors. @@ -815,7 +834,7 @@ default value have the following special behaviors: :meth:`~object.__get__` or :meth:`!__set__` method is called rather than returning or overwriting the descriptor object. -* To determine whether a field contains a default value, :func:`@dataclass ` +* To determine whether a field contains a default value, :deco:`dataclass` will call the descriptor's :meth:`!__get__` method using its class access form: ``descriptor.__get__(obj=None, type=cls)``. If the descriptor returns a value in this case, it will be used as the diff --git a/Doc/library/datetime-inheritance.dot b/Doc/library/datetime-inheritance.dot new file mode 100644 index 000000000000000..3c6b9b4beb7ab1d --- /dev/null +++ b/Doc/library/datetime-inheritance.dot @@ -0,0 +1,31 @@ +// Used to generate datetime-inheritance.svg with Graphviz +// (https://graphviz.org/) for the datetime documentation. + +digraph { + comment="Generated with datetime-inheritance.dot" + graph [ + bgcolor="transparent" + fontnames="svg" + layout="dot" + ranksep=0.5 + nodesep=0.5 + splines=line + ] + node [ + fontname="Courier" + fontsize=14.0 + shape=box + style=rounded + margin="0.15,0.07" + ] + edge [ + arrowhead=none + ] + + object -> tzinfo + object -> timedelta + object -> time + object -> date + tzinfo -> timezone + date -> datetime +} diff --git a/Doc/library/datetime-inheritance.svg b/Doc/library/datetime-inheritance.svg new file mode 100644 index 000000000000000..e6b1cf877a574f2 --- /dev/null +++ b/Doc/library/datetime-inheritance.svg @@ -0,0 +1,84 @@ + + + + + + +datetime class hierarchy + + +object + +object + + + +tzinfo + +tzinfo + + + +object->tzinfo + + + + +timedelta + +timedelta + + + +object->timedelta + + + + +time + +time + + + +object->time + + + + +date + +date + + + +object->date + + + + +timezone + +timezone + + + +tzinfo->timezone + + + + +datetime + +datetime + + + +date->datetime + + + + diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 1ce2013f05da2eb..beef4f09f02f382 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -12,8 +12,6 @@ -------------- -.. XXX what order should the types be discussed in? - The :mod:`!datetime` module supplies classes for manipulating dates and times. While date and time arithmetic is supported, the focus of the implementation is @@ -38,13 +36,14 @@ on efficient attribute extraction for output formatting and manipulation. Third-party library with expanded time zone and parsing support. Package :pypi:`DateType` - Third-party library that introduces distinct static types to e.g. allow - :term:`static type checkers ` + Third-party library that introduces distinct static types to for example, + allow :term:`static type checkers ` to differentiate between naive and aware datetimes. + .. _datetime-naive-aware: -Aware and Naive Objects +Aware and naive objects ----------------------- Date and time objects may be categorized as "aware" or "naive" depending on @@ -65,7 +64,7 @@ understand and to work with, at the cost of ignoring some aspects of reality. For applications requiring aware objects, :class:`.datetime` and :class:`.time` objects have an optional time zone information attribute, :attr:`!tzinfo`, that -can be set to an instance of a subclass of the abstract :class:`tzinfo` class. +can be set to an instance of a subclass of the abstract :class:`!tzinfo` class. These :class:`tzinfo` objects capture information about the offset from UTC time, the time zone name, and whether daylight saving time is in effect. @@ -77,6 +76,7 @@ detail is up to the application. The rules for time adjustment across the world are more political than rational, change frequently, and there is no standard suitable for every application aside from UTC. + Constants --------- @@ -93,13 +93,15 @@ The :mod:`!datetime` module exports the following constants: The largest year number allowed in a :class:`date` or :class:`.datetime` object. :const:`MAXYEAR` is 9999. + .. data:: UTC Alias for the UTC time zone singleton :attr:`datetime.timezone.utc`. .. versionadded:: 3.11 -Available Types + +Available types --------------- .. class:: date @@ -142,6 +144,7 @@ Available Types time adjustment (for example, to account for time zone and/or daylight saving time). + .. class:: timezone :noindex: @@ -150,19 +153,19 @@ Available Types .. versionadded:: 3.2 + Objects of these types are immutable. -Subclass relationships:: +Subclass relationships: + +.. figure:: datetime-inheritance.svg + :class: invert-in-dark-mode + :align: center + :alt: timedelta, tzinfo, time, and date inherit from object; timezone inherits + from tzinfo; and datetime inherits from date. - object - timedelta - tzinfo - timezone - time - date - datetime -Common Properties +Common properties ^^^^^^^^^^^^^^^^^ The :class:`date`, :class:`.datetime`, :class:`.time`, and :class:`timezone` types @@ -173,7 +176,8 @@ share these common features: dictionary keys. - Objects of these types support efficient pickling via the :mod:`pickle` module. -Determining if an Object is Aware or Naive + +Determining if an object is aware or naive ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Objects of the :class:`date` type are always naive. @@ -197,10 +201,11 @@ Otherwise, ``t`` is naive. The distinction between aware and naive doesn't apply to :class:`timedelta` objects. + .. _datetime-timedelta: -:class:`timedelta` Objects --------------------------- +:class:`!timedelta` objects +--------------------------- A :class:`timedelta` object represents a duration, the difference between two :class:`.datetime` or :class:`date` instances. @@ -229,8 +234,8 @@ A :class:`timedelta` object represents a duration, the difference between two *days*, *seconds* and *microseconds* are "merged" and normalized into those three resulting attributes:: - >>> from datetime import timedelta - >>> delta = timedelta( + >>> import datetime as dt + >>> delta = dt.timedelta( ... days=50, ... seconds=27, ... microseconds=10, @@ -243,6 +248,12 @@ A :class:`timedelta` object represents a duration, the difference between two >>> delta datetime.timedelta(days=64, seconds=29156, microseconds=10) + .. tip:: + ``import datetime as dt`` instead of ``import datetime`` or + ``from datetime import datetime`` to avoid confusion between the module + and the class. See `How I Import Python’s datetime Module + `__. + If any argument is a float and there are fractional microseconds, the fractional microseconds left over from all arguments are combined and their sum is rounded to the nearest microsecond using @@ -256,11 +267,27 @@ A :class:`timedelta` object represents a duration, the difference between two Note that normalization of negative values may be surprising at first. For example:: - >>> from datetime import timedelta - >>> d = timedelta(microseconds=-1) + >>> import datetime as dt + >>> d = dt.timedelta(microseconds=-1) >>> (d.days, d.seconds, d.microseconds) (-1, 86399, 999999) + Since the string representation of :class:`!timedelta` objects can be confusing, + use the following recipe to produce a more readable format: + + .. code-block:: pycon + + >>> def pretty_timedelta(td): + ... if td.days >= 0: + ... return str(td) + ... return f'-({-td!s})' + ... + >>> d = timedelta(hours=-1) + >>> str(d) # not human-friendly + '-1 day, 23:00:00' + >>> pretty_timedelta(d) + '-(1:00:00)' + Class attributes: @@ -280,6 +307,7 @@ Class attributes: The smallest possible difference between non-equal :class:`timedelta` objects, ``timedelta(microseconds=1)``. + Note that, because of normalization, ``timedelta.max`` is greater than ``-timedelta.min``. ``-timedelta.max`` is not representable as a :class:`timedelta` object. @@ -303,13 +331,14 @@ Instance attributes (read-only): .. doctest:: - >>> from datetime import timedelta - >>> duration = timedelta(seconds=11235813) + >>> import datetime as dt + >>> duration = dt.timedelta(seconds=11235813) >>> duration.days, duration.seconds (130, 3813) >>> duration.total_seconds() 11235813.0 + .. attribute:: timedelta.microseconds Between 0 and 999,999 inclusive. @@ -317,8 +346,6 @@ Instance attributes (read-only): Supported operations: -.. XXX this table is too wide! - +--------------------------------+-----------------------------------------------+ | Operation | Result | +================================+===============================================+ @@ -380,7 +407,6 @@ Supported operations: | | call with canonical attribute values. | +--------------------------------+-----------------------------------------------+ - Notes: (1) @@ -416,9 +442,9 @@ objects (see below). .. versionchanged:: 3.2 Floor division and true division of a :class:`timedelta` object by another - :class:`timedelta` object are now supported, as are remainder operations and + :class:`!timedelta` object are now supported, as are remainder operations and the :func:`divmod` function. True division and multiplication of a - :class:`timedelta` object by a :class:`float` object are now supported. + :class:`!timedelta` object by a :class:`float` object are now supported. :class:`timedelta` objects support equality and order comparisons. @@ -431,23 +457,24 @@ Instance methods: Return the total number of seconds contained in the duration. Equivalent to ``td / timedelta(seconds=1)``. For interval units other than seconds, use the - division form directly (e.g. ``td / timedelta(microseconds=1)``). + division form directly (for example, ``td / timedelta(microseconds=1)``). Note that for very large time intervals (greater than 270 years on most platforms) this method will lose microsecond accuracy. .. versionadded:: 3.2 -Examples of usage: :class:`timedelta` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Examples of usage: :class:`!timedelta` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ An additional example of normalization:: >>> # Components of another_year add up to exactly 365 days - >>> from datetime import timedelta - >>> year = timedelta(days=365) - >>> another_year = timedelta(weeks=40, days=84, hours=23, - ... minutes=50, seconds=600) + >>> import datetime as dt + >>> year = dt.timedelta(days=365) + >>> another_year = dt.timedelta(weeks=40, days=84, hours=23, + ... minutes=50, seconds=600) >>> year == another_year True >>> year.total_seconds() @@ -455,8 +482,8 @@ An additional example of normalization:: Examples of :class:`timedelta` arithmetic:: - >>> from datetime import timedelta - >>> year = timedelta(days=365) + >>> import datetime as dt + >>> year = dt.timedelta(days=365) >>> ten_years = 10 * year >>> ten_years datetime.timedelta(days=3650) @@ -469,10 +496,11 @@ Examples of :class:`timedelta` arithmetic:: >>> three_years, three_years.days // 365 (datetime.timedelta(days=1095), 3) + .. _datetime-date: -:class:`date` Objects ---------------------- +:class:`!date` objects +---------------------- A :class:`date` object represents a date (year, month and day) in an idealized calendar, the current Gregorian calendar indefinitely extended in both @@ -501,9 +529,10 @@ Other constructors, all class methods: This is equivalent to ``date.fromtimestamp(time.time())``. + .. classmethod:: date.fromtimestamp(timestamp) - Return the local date corresponding to the POSIX timestamp, such as is + Return the local date corresponding to the POSIX *timestamp*, such as is returned by :func:`time.time`. This may raise :exc:`OverflowError`, if the timestamp is out @@ -522,7 +551,7 @@ Other constructors, all class methods: .. classmethod:: date.fromordinal(ordinal) - Return the date corresponding to the proleptic Gregorian ordinal, where + Return the date corresponding to the proleptic Gregorian *ordinal*, where January 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless ``1 <= ordinal <= @@ -543,25 +572,27 @@ Other constructors, all class methods: Examples:: - >>> from datetime import date - >>> date.fromisoformat('2019-12-04') + >>> import datetime as dt + >>> dt.date.fromisoformat('2019-12-04') datetime.date(2019, 12, 4) - >>> date.fromisoformat('20191204') + >>> dt.date.fromisoformat('20191204') datetime.date(2019, 12, 4) - >>> date.fromisoformat('2021-W01-1') + >>> dt.date.fromisoformat('2021-W01-1') datetime.date(2021, 1, 4) .. versionadded:: 3.7 .. versionchanged:: 3.11 Previously, this method only supported the format ``YYYY-MM-DD``. + .. classmethod:: date.fromisocalendar(year, week, day) Return a :class:`date` corresponding to the ISO calendar date specified by - year, week and day. This is the inverse of the function :meth:`date.isocalendar`. + *year*, *week* and *day*. This is the inverse of the function :meth:`date.isocalendar`. .. versionadded:: 3.8 + .. classmethod:: date.strptime(date_string, format) Return a :class:`.date` corresponding to *date_string*, parsed according to @@ -587,9 +618,9 @@ Other constructors, all class methods: .. doctest:: - >>> from datetime import date + >>> import datetime as dt >>> date_string = "02/29" - >>> when = date.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. + >>> when = dt.date.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. >>> when.strftime("%B %d") # doctest: +SKIP 'February 29' @@ -681,7 +712,7 @@ Notes: In other words, ``date1 < date2`` if and only if ``date1.toordinal() < date2.toordinal()``. - Order comparison between a :class:`!date` object that is not also a + Order comparison between a :class:`date` object that is not also a :class:`.datetime` instance and a :class:`!datetime` object raises :exc:`TypeError`. @@ -704,8 +735,8 @@ Instance methods: Example:: - >>> from datetime import date - >>> d = date(2002, 12, 31) + >>> import datetime as dt + >>> d = dt.date(2002, 12, 31) >>> d.replace(day=26) datetime.date(2002, 12, 26) @@ -763,23 +794,25 @@ Instance methods: For example, 2004 begins on a Thursday, so the first week of ISO year 2004 begins on Monday, 29 Dec 2003 and ends on Sunday, 4 Jan 2004:: - >>> from datetime import date - >>> date(2003, 12, 29).isocalendar() + >>> import datetime as dt + >>> dt.date(2003, 12, 29).isocalendar() datetime.IsoCalendarDate(year=2004, week=1, weekday=1) - >>> date(2004, 1, 4).isocalendar() + >>> dt.date(2004, 1, 4).isocalendar() datetime.IsoCalendarDate(year=2004, week=1, weekday=7) .. versionchanged:: 3.9 Result changed from a tuple to a :term:`named tuple`. + .. method:: date.isoformat() Return a string representing the date in ISO 8601 format, ``YYYY-MM-DD``:: - >>> from datetime import date - >>> date(2002, 12, 4).isoformat() + >>> import datetime as dt + >>> dt.date(2002, 12, 4).isoformat() '2002-12-04' + .. method:: date.__str__() For a date ``d``, ``str(d)`` is equivalent to ``d.isoformat()``. @@ -789,8 +822,8 @@ Instance methods: Return a string representing the date:: - >>> from datetime import date - >>> date(2002, 12, 4).ctime() + >>> import datetime as dt + >>> dt.date(2002, 12, 4).ctime() 'Wed Dec 4 00:00:00 2002' ``d.ctime()`` is equivalent to:: @@ -816,19 +849,20 @@ Instance methods: literals ` and when using :meth:`str.format`. See also :ref:`strftime-strptime-behavior` and :meth:`date.isoformat`. -Examples of Usage: :class:`date` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Examples of usage: :class:`!date` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Example of counting days to an event:: >>> import time - >>> from datetime import date - >>> today = date.today() + >>> import datetime as dt + >>> today = dt.date.today() >>> today datetime.date(2007, 12, 5) - >>> today == date.fromtimestamp(time.time()) + >>> today == dt.date.fromtimestamp(time.time()) True - >>> my_birthday = date(today.year, 6, 24) + >>> my_birthday = dt.date(today.year, 6, 24) >>> if my_birthday < today: ... my_birthday = my_birthday.replace(year=today.year + 1) ... @@ -842,8 +876,8 @@ More examples of working with :class:`date`: .. doctest:: - >>> from datetime import date - >>> d = date.fromordinal(730920) # 730920th day after 1. 1. 0001 + >>> import datetime as dt + >>> d = dt.date.fromordinal(730920) # 730920th day after 1. 1. 0001 >>> d datetime.date(2002, 3, 11) @@ -859,7 +893,7 @@ More examples of working with :class:`date`: >>> 'The {1} is {0:%d}, the {2} is {0:%B}.'.format(d, "day", "month") 'The day is 11, the month is March.' - >>> # Methods for to extracting 'components' under different calendars + >>> # Methods for extracting 'components' under different calendars >>> t = d.timetuple() >>> for i in t: # doctest: +SKIP ... print(i) @@ -886,7 +920,7 @@ More examples of working with :class:`date`: .. _datetime-datetime: -:class:`.datetime` Objects +:class:`!datetime` objects -------------------------- A :class:`.datetime` object is a single object containing all the information @@ -894,7 +928,7 @@ from a :class:`date` object and a :class:`.time` object. Like a :class:`date` object, :class:`.datetime` assumes the current Gregorian calendar extended in both directions; like a :class:`.time` object, -:class:`.datetime` assumes there are exactly 3600\*24 seconds in every day. +:class:`!datetime` assumes there are exactly 3600\*24 seconds in every day. Constructor: @@ -918,6 +952,7 @@ Constructor: .. versionchanged:: 3.6 Added the *fold* parameter. + Other constructors, all class methods: .. classmethod:: datetime.today() @@ -933,6 +968,7 @@ Other constructors, all class methods: This method is functionally equivalent to :meth:`now`, but without a ``tz`` parameter. + .. classmethod:: datetime.now(tz=None) Return the current local date and time. @@ -953,6 +989,7 @@ Other constructors, all class methods: Subsequent calls to :meth:`!datetime.now` may return the same instant depending on the precision of the underlying clock. + .. classmethod:: datetime.utcnow() Return the current UTC date and time, with :attr:`.tzinfo` ``None``. @@ -1040,6 +1077,9 @@ Other constructors, all class methods: :c:func:`gmtime` function. Raise :exc:`OSError` instead of :exc:`ValueError` on :c:func:`gmtime` failure. + .. versionchanged:: 3.15 + Accepts any real number as *timestamp*, not only integer or float. + .. deprecated:: 3.12 Use :meth:`datetime.fromtimestamp` with :const:`UTC` instead. @@ -1060,7 +1100,7 @@ Other constructors, all class methods: are equal to the given :class:`.time` object's. If the *tzinfo* argument is provided, its value is used to set the :attr:`.tzinfo` attribute of the result, otherwise the :attr:`~.time.tzinfo` attribute of the *time* argument - is used. If the *date* argument is a :class:`.datetime` object, its time components + is used. If the *date* argument is a :class:`!datetime` object, its time components and :attr:`.tzinfo` attributes are ignored. For any :class:`.datetime` object ``d``, @@ -1086,24 +1126,24 @@ Other constructors, all class methods: Examples:: - >>> from datetime import datetime - >>> datetime.fromisoformat('2011-11-04') + >>> import datetime as dt + >>> dt.datetime.fromisoformat('2011-11-04') datetime.datetime(2011, 11, 4, 0, 0) - >>> datetime.fromisoformat('20111104') + >>> dt.datetime.fromisoformat('20111104') datetime.datetime(2011, 11, 4, 0, 0) - >>> datetime.fromisoformat('2011-11-04T00:05:23') + >>> dt.datetime.fromisoformat('2011-11-04T00:05:23') datetime.datetime(2011, 11, 4, 0, 5, 23) - >>> datetime.fromisoformat('2011-11-04T00:05:23Z') + >>> dt.datetime.fromisoformat('2011-11-04T00:05:23Z') datetime.datetime(2011, 11, 4, 0, 5, 23, tzinfo=datetime.timezone.utc) - >>> datetime.fromisoformat('20111104T000523') + >>> dt.datetime.fromisoformat('20111104T000523') datetime.datetime(2011, 11, 4, 0, 5, 23) - >>> datetime.fromisoformat('2011-W01-2T00:05:23.283') + >>> dt.datetime.fromisoformat('2011-W01-2T00:05:23.283') datetime.datetime(2011, 1, 4, 0, 5, 23, 283000) - >>> datetime.fromisoformat('2011-11-04 00:05:23.283') + >>> dt.datetime.fromisoformat('2011-11-04 00:05:23.283') datetime.datetime(2011, 11, 4, 0, 5, 23, 283000) - >>> datetime.fromisoformat('2011-11-04 00:05:23.283+00:00') + >>> dt.datetime.fromisoformat('2011-11-04 00:05:23.283+00:00') datetime.datetime(2011, 11, 4, 0, 5, 23, 283000, tzinfo=datetime.timezone.utc) - >>> datetime.fromisoformat('2011-11-04T00:05:23+04:00') # doctest: +NORMALIZE_WHITESPACE + >>> dt.datetime.fromisoformat('2011-11-04T00:05:23+04:00') # doctest: +NORMALIZE_WHITESPACE datetime.datetime(2011, 11, 4, 0, 5, 23, tzinfo=datetime.timezone(datetime.timedelta(seconds=14400))) @@ -1116,12 +1156,13 @@ Other constructors, all class methods: .. classmethod:: datetime.fromisocalendar(year, week, day) Return a :class:`.datetime` corresponding to the ISO calendar date specified - by year, week and day. The non-date components of the datetime are populated + by *year*, *week* and *day*. The non-date components of the datetime are populated with their normal default values. This is the inverse of the function :meth:`datetime.isocalendar`. .. versionadded:: 3.8 + .. classmethod:: datetime.strptime(date_string, format) Return a :class:`.datetime` corresponding to *date_string*, parsed according to @@ -1149,9 +1190,9 @@ Other constructors, all class methods: .. doctest:: - >>> from datetime import datetime + >>> import datetime as dt >>> date_string = "02/29" - >>> when = datetime.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. + >>> when = dt.datetime.strptime(f"{date_string};1984", "%m/%d;%Y") # Avoids leap year bug. >>> when.strftime("%B %d") # doctest: +SKIP 'February 29' @@ -1229,6 +1270,7 @@ Instance attributes (read-only): .. versionadded:: 3.6 + Supported operations: +---------------------------------------+--------------------------------+ @@ -1264,7 +1306,7 @@ Supported operations: datetime, and no time zone adjustments are done even if the input is aware. (3) - Subtraction of a :class:`.datetime` from a :class:`.datetime` is defined only if + Subtraction of a :class:`.datetime` from a :class:`!datetime` is defined only if both operands are naive, or if both are aware. If one is aware and the other is naive, :exc:`TypeError` is raised. @@ -1282,7 +1324,7 @@ Supported operations: :class:`.datetime` objects are equal if they represent the same date and time, taking into account the time zone. - Naive and aware :class:`!datetime` objects are never equal. + Naive and aware :class:`.datetime` objects are never equal. If both comparands are aware, and have the same :attr:`!tzinfo` attribute, the :attr:`!tzinfo` and :attr:`~.datetime.fold` attributes are ignored and @@ -1290,7 +1332,7 @@ Supported operations: If both comparands are aware and have different :attr:`~.datetime.tzinfo` attributes, the comparison acts as comparands were first converted to UTC datetimes except that the implementation never overflows. - :class:`!datetime` instances in a repeated interval are never equal to + :class:`.datetime` instances in a repeated interval are never equal to :class:`!datetime` instances in other time zone. (5) @@ -1319,6 +1361,7 @@ Supported operations: The default behavior can be changed by overriding the special comparison methods in subclasses. + Instance methods: .. method:: datetime.date() @@ -1474,11 +1517,13 @@ Instance methods: ``datetime.replace(tzinfo=timezone.utc)`` to make it aware, at which point you can use :meth:`.datetime.timetuple`. + .. method:: datetime.toordinal() Return the proleptic Gregorian ordinal of the date. The same as ``self.date().toordinal()``. + .. method:: datetime.timestamp() Return POSIX timestamp corresponding to the :class:`.datetime` @@ -1487,7 +1532,7 @@ Instance methods: Naive :class:`.datetime` instances are assumed to represent local time and this method relies on the platform C :c:func:`mktime` - function to perform the conversion. Since :class:`.datetime` + function to perform the conversion. Since :class:`!datetime` supports wider range of values than :c:func:`mktime` on many platforms, this method may raise :exc:`OverflowError` or :exc:`OSError` for times far in the past or far in the future. @@ -1497,12 +1542,6 @@ Instance methods: (dt - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() - .. versionadded:: 3.3 - - .. versionchanged:: 3.6 - The :meth:`timestamp` method uses the :attr:`.fold` attribute to - disambiguate the times during a repeated interval. - .. note:: There is no method to obtain the POSIX timestamp directly from a @@ -1517,6 +1556,13 @@ Instance methods: timestamp = (dt - datetime(1970, 1, 1)) / timedelta(seconds=1) + .. versionadded:: 3.3 + + .. versionchanged:: 3.6 + The :meth:`timestamp` method uses the :attr:`.fold` attribute to + disambiguate the times during a repeated interval. + + .. method:: datetime.weekday() Return the day of the week as an integer, where Monday is 0 and Sunday is 6. @@ -1552,24 +1598,24 @@ Instance methods: Examples:: - >>> from datetime import datetime, timezone - >>> datetime(2019, 5, 18, 15, 17, 8, 132263).isoformat() + >>> import datetime as dt + >>> dt.datetime(2019, 5, 18, 15, 17, 8, 132263).isoformat() '2019-05-18T15:17:08.132263' - >>> datetime(2019, 5, 18, 15, 17, tzinfo=timezone.utc).isoformat() + >>> dt.datetime(2019, 5, 18, 15, 17, tzinfo=dt.timezone.utc).isoformat() '2019-05-18T15:17:00+00:00' The optional argument *sep* (default ``'T'``) is a one-character separator, placed between the date and time portions of the result. For example:: - >>> from datetime import tzinfo, timedelta, datetime - >>> class TZ(tzinfo): + >>> import datetime as dt + >>> class TZ(dt.tzinfo): ... """A time zone with an arbitrary, constant -06:39 offset.""" - ... def utcoffset(self, dt): - ... return timedelta(hours=-6, minutes=-39) + ... def utcoffset(self, when): + ... return dt.timedelta(hours=-6, minutes=-39) ... - >>> datetime(2002, 12, 25, tzinfo=TZ()).isoformat(' ') + >>> dt.datetime(2002, 12, 25, tzinfo=TZ()).isoformat(' ') '2002-12-25 00:00:00-06:39' - >>> datetime(2009, 11, 27, microsecond=100, tzinfo=TZ()).isoformat() + >>> dt.datetime(2009, 11, 27, microsecond=100, tzinfo=TZ()).isoformat() '2009-11-27T00:00:00.000100-06:39' The optional argument *timespec* specifies the number of additional @@ -1593,11 +1639,11 @@ Instance methods: :exc:`ValueError` will be raised on an invalid *timespec* argument:: - >>> from datetime import datetime - >>> datetime.now().isoformat(timespec='minutes') # doctest: +SKIP + >>> import datetime as dt + >>> dt.datetime.now().isoformat(timespec='minutes') # doctest: +SKIP '2002-12-25T00:00' - >>> dt = datetime(2015, 1, 1, 12, 30, 59, 0) - >>> dt.isoformat(timespec='microseconds') + >>> my_datetime = dt.datetime(2015, 1, 1, 12, 30, 59, 0) + >>> my_datetime.isoformat(timespec='microseconds') '2015-01-01T12:30:59.000000' .. versionchanged:: 3.6 @@ -1614,8 +1660,8 @@ Instance methods: Return a string representing the date and time:: - >>> from datetime import datetime - >>> datetime(2002, 12, 4, 20, 30, 40).ctime() + >>> import datetime as dt + >>> dt.datetime(2002, 12, 4, 20, 30, 40).ctime() 'Wed Dec 4 20:30:40 2002' The output string will *not* include time zone information, regardless @@ -1645,34 +1691,34 @@ Instance methods: See also :ref:`strftime-strptime-behavior` and :meth:`datetime.isoformat`. -Examples of Usage: :class:`.datetime` +Examples of usage: :class:`!datetime` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Examples of working with :class:`.datetime` objects: .. doctest:: - >>> from datetime import datetime, date, time, timezone + >>> import datetime as dt >>> # Using datetime.combine() - >>> d = date(2005, 7, 14) - >>> t = time(12, 30) - >>> datetime.combine(d, t) + >>> d = dt.date(2005, 7, 14) + >>> t = dt.time(12, 30) + >>> dt.datetime.combine(d, t) datetime.datetime(2005, 7, 14, 12, 30) >>> # Using datetime.now() - >>> datetime.now() # doctest: +SKIP + >>> dt.datetime.now() # doctest: +SKIP datetime.datetime(2007, 12, 6, 16, 29, 43, 79043) # GMT +1 - >>> datetime.now(timezone.utc) # doctest: +SKIP + >>> dt.datetime.now(dt.timezone.utc) # doctest: +SKIP datetime.datetime(2007, 12, 6, 15, 29, 43, 79060, tzinfo=datetime.timezone.utc) >>> # Using datetime.strptime() - >>> dt = datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M") - >>> dt + >>> my_datetime = dt.datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M") + >>> my_datetime datetime.datetime(2006, 11, 21, 16, 30) >>> # Using datetime.timetuple() to get tuple of all attributes - >>> tt = dt.timetuple() + >>> tt = my_datetime.timetuple() >>> for it in tt: # doctest: +SKIP ... print(it) ... @@ -1687,7 +1733,7 @@ Examples of working with :class:`.datetime` objects: -1 # dst - method tzinfo.dst() returned None >>> # Date in ISO format - >>> ic = dt.isocalendar() + >>> ic = my_datetime.isocalendar() >>> for it in ic: # doctest: +SKIP ... print(it) ... @@ -1696,55 +1742,55 @@ Examples of working with :class:`.datetime` objects: 2 # ISO weekday >>> # Formatting a datetime - >>> dt.strftime("%A, %d. %B %Y %I:%M%p") + >>> my_datetime.strftime("%A, %d. %B %Y %I:%M%p") 'Tuesday, 21. November 2006 04:30PM' - >>> 'The {1} is {0:%d}, the {2} is {0:%B}, the {3} is {0:%I:%M%p}.'.format(dt, "day", "month", "time") + >>> 'The {1} is {0:%d}, the {2} is {0:%B}, the {3} is {0:%I:%M%p}.'.format(my_datetime, "day", "month", "time") 'The day is 21, the month is November, the time is 04:30PM.' The example below defines a :class:`tzinfo` subclass capturing time zone information for Kabul, Afghanistan, which used +4 UTC until 1945 and then +4:30 UTC thereafter:: - from datetime import timedelta, datetime, tzinfo, timezone + import datetime as dt - class KabulTz(tzinfo): + class KabulTz(dt.tzinfo): # Kabul used +4 until 1945, when they moved to +4:30 - UTC_MOVE_DATE = datetime(1944, 12, 31, 20, tzinfo=timezone.utc) + UTC_MOVE_DATE = dt.datetime(1944, 12, 31, 20, tzinfo=dt.timezone.utc) - def utcoffset(self, dt): - if dt.year < 1945: - return timedelta(hours=4) - elif (1945, 1, 1, 0, 0) <= dt.timetuple()[:5] < (1945, 1, 1, 0, 30): + def utcoffset(self, when): + if when.year < 1945: + return dt.timedelta(hours=4) + elif (1945, 1, 1, 0, 0) <= when.timetuple()[:5] < (1945, 1, 1, 0, 30): # An ambiguous ("imaginary") half-hour range representing # a 'fold' in time due to the shift from +4 to +4:30. - # If dt falls in the imaginary range, use fold to decide how - # to resolve. See PEP495. - return timedelta(hours=4, minutes=(30 if dt.fold else 0)) + # If when falls in the imaginary range, use fold to decide how + # to resolve. See PEP 495. + return dt.timedelta(hours=4, minutes=(30 if when.fold else 0)) else: - return timedelta(hours=4, minutes=30) + return dt.timedelta(hours=4, minutes=30) - def fromutc(self, dt): + def fromutc(self, when): # Follow same validations as in datetime.tzinfo - if not isinstance(dt, datetime): + if not isinstance(when, dt.datetime): raise TypeError("fromutc() requires a datetime argument") - if dt.tzinfo is not self: - raise ValueError("dt.tzinfo is not self") + if when.tzinfo is not self: + raise ValueError("when.tzinfo is not self") # A custom implementation is required for fromutc as # the input to this function is a datetime with utc values # but with a tzinfo set to self. # See datetime.astimezone or fromtimestamp. - if dt.replace(tzinfo=timezone.utc) >= self.UTC_MOVE_DATE: - return dt + timedelta(hours=4, minutes=30) + if when.replace(tzinfo=dt.timezone.utc) >= self.UTC_MOVE_DATE: + return when + dt.timedelta(hours=4, minutes=30) else: - return dt + timedelta(hours=4) + return when + dt.timedelta(hours=4) - def dst(self, dt): + def dst(self, when): # Kabul does not observe daylight saving time. - return timedelta(0) + return dt.timedelta(0) - def tzname(self, dt): - if dt >= self.UTC_MOVE_DATE: + def tzname(self, when): + if when >= self.UTC_MOVE_DATE: return "+04:30" return "+04" @@ -1753,17 +1799,17 @@ Usage of ``KabulTz`` from above:: >>> tz1 = KabulTz() >>> # Datetime before the change - >>> dt1 = datetime(1900, 11, 21, 16, 30, tzinfo=tz1) + >>> dt1 = dt.datetime(1900, 11, 21, 16, 30, tzinfo=tz1) >>> print(dt1.utcoffset()) 4:00:00 >>> # Datetime after the change - >>> dt2 = datetime(2006, 6, 14, 13, 0, tzinfo=tz1) + >>> dt2 = dt.datetime(2006, 6, 14, 13, 0, tzinfo=tz1) >>> print(dt2.utcoffset()) 4:30:00 >>> # Convert datetime to another time zone - >>> dt3 = dt2.astimezone(timezone.utc) + >>> dt3 = dt2.astimezone(dt.timezone.utc) >>> dt3 datetime.datetime(2006, 6, 14, 8, 30, tzinfo=datetime.timezone.utc) >>> dt2 @@ -1771,9 +1817,10 @@ Usage of ``KabulTz`` from above:: >>> dt2 == dt3 True + .. _datetime-time: -:class:`.time` Objects +:class:`!time` objects ---------------------- A :class:`.time` object represents a (local) time of day, independent of any particular @@ -1794,6 +1841,7 @@ day, and subject to adjustment via a :class:`tzinfo` object. If an argument outside those ranges is given, :exc:`ValueError` is raised. All default to 0 except *tzinfo*, which defaults to ``None``. + Class attributes: @@ -1852,6 +1900,7 @@ Instance attributes (read-only): .. versionadded:: 3.6 + :class:`.time` objects support equality and order comparisons, where ``a`` is considered less than ``b`` when ``a`` precedes ``b`` in time. @@ -1874,8 +1923,8 @@ In Boolean contexts, a :class:`.time` object is always considered to be true. .. versionchanged:: 3.5 Before Python 3.5, a :class:`.time` object was considered to be false if it represented midnight in UTC. This behavior was considered obscure and - error-prone and has been removed in Python 3.5. See :issue:`13936` for full - details. + error-prone and has been removed in Python 3.5. See :issue:`13936` for more + information. Other constructors: @@ -1896,22 +1945,22 @@ Other constructors: .. doctest:: - >>> from datetime import time - >>> time.fromisoformat('04:23:01') + >>> import datetime as dt + >>> dt.time.fromisoformat('04:23:01') datetime.time(4, 23, 1) - >>> time.fromisoformat('T04:23:01') + >>> dt.time.fromisoformat('T04:23:01') datetime.time(4, 23, 1) - >>> time.fromisoformat('T042301') + >>> dt.time.fromisoformat('T042301') datetime.time(4, 23, 1) - >>> time.fromisoformat('04:23:01.000384') + >>> dt.time.fromisoformat('04:23:01.000384') datetime.time(4, 23, 1, 384) - >>> time.fromisoformat('04:23:01,000384') + >>> dt.time.fromisoformat('04:23:01,000384') datetime.time(4, 23, 1, 384) - >>> time.fromisoformat('04:23:01+04:00') + >>> dt.time.fromisoformat('04:23:01+04:00') datetime.time(4, 23, 1, tzinfo=datetime.timezone(datetime.timedelta(seconds=14400))) - >>> time.fromisoformat('04:23:01Z') + >>> dt.time.fromisoformat('04:23:01Z') datetime.time(4, 23, 1, tzinfo=datetime.timezone.utc) - >>> time.fromisoformat('04:23:01+00:00') + >>> dt.time.fromisoformat('04:23:01+00:00') datetime.time(4, 23, 1, tzinfo=datetime.timezone.utc) @@ -1920,6 +1969,7 @@ Other constructors: Previously, this method only supported formats that could be emitted by :meth:`time.isoformat`. + .. classmethod:: time.strptime(date_string, format) Return a :class:`.time` corresponding to *date_string*, parsed according to @@ -1944,7 +1994,7 @@ Instance methods: Return a new :class:`.time` with the same values, but with specified parameters updated. Note that ``tzinfo=None`` can be specified to create a - naive :class:`.time` from an aware :class:`.time`, without conversion of the + naive :class:`!time` from an aware :class:`!time`, without conversion of the time data. :class:`.time` objects are also supported by generic function @@ -1985,13 +2035,13 @@ Instance methods: Example:: - >>> from datetime import time - >>> time(hour=12, minute=34, second=56, microsecond=123456).isoformat(timespec='minutes') + >>> import datetime as dt + >>> dt.time(hour=12, minute=34, second=56, microsecond=123456).isoformat(timespec='minutes') '12:34' - >>> dt = time(hour=12, minute=34, second=56, microsecond=0) - >>> dt.isoformat(timespec='microseconds') + >>> my_time = dt.time(hour=12, minute=34, second=56, microsecond=0) + >>> my_time.isoformat(timespec='microseconds') '12:34:56.000000' - >>> dt.isoformat(timespec='auto') + >>> my_time.isoformat(timespec='auto') '12:34:56' .. versionchanged:: 3.6 @@ -2036,29 +2086,31 @@ Instance methods: .. versionchanged:: 3.7 The DST offset is not restricted to a whole number of minutes. + .. method:: time.tzname() If :attr:`.tzinfo` is ``None``, returns ``None``, else returns ``self.tzinfo.tzname(None)``, or raises an exception if the latter doesn't return ``None`` or a string object. -Examples of Usage: :class:`.time` + +Examples of usage: :class:`!time` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Examples of working with a :class:`.time` object:: - >>> from datetime import time, tzinfo, timedelta - >>> class TZ1(tzinfo): - ... def utcoffset(self, dt): - ... return timedelta(hours=1) - ... def dst(self, dt): - ... return timedelta(0) - ... def tzname(self,dt): + >>> import datetime as dt + >>> class TZ1(dt.tzinfo): + ... def utcoffset(self, when): + ... return dt.timedelta(hours=1) + ... def dst(self, when): + ... return dt.timedelta(0) + ... def tzname(self, when): ... return "+01:00" ... def __repr__(self): ... return f"{self.__class__.__name__}()" ... - >>> t = time(12, 10, 30, tzinfo=TZ1()) + >>> t = dt.time(12, 10, 30, tzinfo=TZ1()) >>> t datetime.time(12, 10, 30, tzinfo=TZ1()) >>> t.isoformat() @@ -2075,25 +2127,25 @@ Examples of working with a :class:`.time` object:: .. _datetime-tzinfo: -:class:`tzinfo` Objects ------------------------ +:class:`!tzinfo` objects +------------------------ .. class:: tzinfo() - This is an abstract base class, meaning that this class should not be + This is an :term:`abstract base class`, meaning that this class should not be instantiated directly. Define a subclass of :class:`tzinfo` to capture information about a particular time zone. An instance of (a concrete subclass of) :class:`tzinfo` can be passed to the constructors for :class:`.datetime` and :class:`.time` objects. The latter objects - view their attributes as being in local time, and the :class:`tzinfo` object + view their attributes as being in local time, and the :class:`!tzinfo` object supports methods revealing offset of local time from UTC, the name of the time zone, and DST offset, all relative to a date or time object passed to them. You need to derive a concrete subclass, and (at least) supply implementations of the standard :class:`tzinfo` methods needed by the :class:`.datetime` methods you use. The :mod:`!datetime` module provides - :class:`timezone`, a simple concrete subclass of :class:`tzinfo` which can + :class:`timezone`, a simple concrete subclass of :class:`!tzinfo` which can represent time zones with fixed offset from UTC such as UTC itself or North American EST and EDT. @@ -2156,31 +2208,35 @@ Examples of working with a :class:`.time` object:: ``tz.utcoffset(dt) - tz.dst(dt)`` must return the same result for every :class:`.datetime` *dt* with ``dt.tzinfo == - tz``. For sane :class:`tzinfo` subclasses, this expression yields the time + tz``. For sane :class:`!tzinfo` subclasses, this expression yields the time zone's "standard offset", which should not depend on the date or the time, but only on geographic location. The implementation of :meth:`datetime.astimezone` relies on this, but cannot detect violations; it's the programmer's - responsibility to ensure it. If a :class:`tzinfo` subclass cannot guarantee + responsibility to ensure it. If a :class:`!tzinfo` subclass cannot guarantee this, it may be able to override the default implementation of :meth:`tzinfo.fromutc` to work correctly with :meth:`~.datetime.astimezone` regardless. Most implementations of :meth:`dst` will probably look like one of these two:: - def dst(self, dt): + import datetime as dt + + def dst(self, when): # a fixed-offset class: doesn't account for DST - return timedelta(0) + return dt.timedelta(0) or:: - def dst(self, dt): + import datetime as dt + + def dst(self, when): # Code to set dston and dstoff to the time zone's DST - # transition times based on the input dt.year, and expressed + # transition times based on the input when.year, and expressed # in standard local time. - if dston <= dt.replace(tzinfo=None) < dstoff: - return timedelta(hours=1) + if dston <= when.replace(tzinfo=None) < dstoff: + return dt.timedelta(hours=1) else: - return timedelta(0) + return dt.timedelta(0) The default implementation of :meth:`dst` raises :exc:`NotImplementedError`. @@ -2197,17 +2253,17 @@ Examples of working with a :class:`.time` object:: valid replies. Return ``None`` if a string name isn't known. Note that this is a method rather than a fixed string primarily because some :class:`tzinfo` subclasses will wish to return different names depending on the specific value - of *dt* passed, especially if the :class:`tzinfo` class is accounting for + of *dt* passed, especially if the :class:`!tzinfo` class is accounting for daylight time. The default implementation of :meth:`tzname` raises :exc:`NotImplementedError`. These methods are called by a :class:`.datetime` or :class:`.time` object, in -response to their methods of the same names. A :class:`.datetime` object passes -itself as the argument, and a :class:`.time` object passes ``None`` as the +response to their methods of the same names. A :class:`!datetime` object passes +itself as the argument, and a :class:`!time` object passes ``None`` as the argument. A :class:`tzinfo` subclass's methods should therefore be prepared to -accept a *dt* argument of ``None``, or of class :class:`.datetime`. +accept a *dt* argument of ``None``, or of class :class:`!datetime`. When ``None`` is passed, it's up to the class designer to decide the best response. For example, returning ``None`` is appropriate if the class wishes to @@ -2215,10 +2271,10 @@ say that time objects don't participate in the :class:`tzinfo` protocols. It may be more useful for ``utcoffset(None)`` to return the standard UTC offset, as there is no other convention for discovering the standard offset. -When a :class:`.datetime` object is passed in response to a :class:`.datetime` +When a :class:`.datetime` object is passed in response to a :class:`!datetime` method, ``dt.tzinfo`` is the same object as *self*. :class:`tzinfo` methods can -rely on this, unless user code calls :class:`tzinfo` methods directly. The -intent is that the :class:`tzinfo` methods interpret *dt* as being in local +rely on this, unless user code calls :class:`!tzinfo` methods directly. The +intent is that the :class:`!tzinfo` methods interpret *dt* as being in local time, and not need worry about objects in other time zones. There is one more :class:`tzinfo` method that a subclass may wish to override: @@ -2246,20 +2302,22 @@ There is one more :class:`tzinfo` method that a subclass may wish to override: Skipping code for error cases, the default :meth:`fromutc` implementation acts like:: - def fromutc(self, dt): - # raise ValueError error if dt.tzinfo is not self - dtoff = dt.utcoffset() - dtdst = dt.dst() + import datetime as dt + + def fromutc(self, when): + # raise ValueError error if when.tzinfo is not self + dtoff = when.utcoffset() + dtdst = when.dst() # raise ValueError if dtoff is None or dtdst is None delta = dtoff - dtdst # this is self's standard offset if delta: - dt += delta # convert to standard local time - dtdst = dt.dst() + when += delta # convert to standard local time + dtdst = when.dst() # raise ValueError if dtdst is None if dtdst: - return dt + dtdst + return when + dtdst else: - return dt + return when In the following :download:`tzinfo_examples.py <../includes/tzinfo_examples.py>` file there are some examples of @@ -2286,9 +2344,9 @@ When DST starts (the "start" line), the local wall clock leaps from 1:59 to ``astimezone(Eastern)`` won't deliver a result with ``hour == 2`` on the day DST begins. For example, at the Spring forward transition of 2016, we get:: - >>> from datetime import datetime, timezone + >>> import datetime as dt >>> from tzinfo_examples import HOUR, Eastern - >>> u0 = datetime(2016, 3, 13, 5, tzinfo=timezone.utc) + >>> u0 = dt.datetime(2016, 3, 13, 5, tzinfo=dt.timezone.utc) >>> for i in range(4): ... u = u0 + i*HOUR ... t = u.astimezone(Eastern) @@ -2311,7 +2369,9 @@ form 5:MM and 6:MM both map to 1:MM when converted to Eastern, but earlier times have the :attr:`~.datetime.fold` attribute set to 0 and the later times have it set to 1. For example, at the Fall back transition of 2016, we get:: - >>> u0 = datetime(2016, 11, 6, 4, tzinfo=timezone.utc) + >>> import datetime as dt + >>> from tzinfo_examples import HOUR, Eastern + >>> u0 = dt.datetime(2016, 11, 6, 4, tzinfo=dt.timezone.utc) >>> for i in range(4): ... u = u0 + i*HOUR ... t = u.astimezone(Eastern) @@ -2328,7 +2388,7 @@ Note that the :class:`.datetime` instances that differ only by the value of the Applications that can't bear wall-time ambiguities should explicitly check the value of the :attr:`~.datetime.fold` attribute or avoid using hybrid :class:`tzinfo` subclasses; there are no ambiguities when using :class:`timezone`, -or any other fixed-offset :class:`tzinfo` subclass (such as a class representing +or any other fixed-offset :class:`!tzinfo` subclass (such as a class representing only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. seealso:: @@ -2351,8 +2411,8 @@ only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). .. _datetime-timezone: -:class:`timezone` Objects -------------------------- +:class:`!timezone` objects +-------------------------- The :class:`timezone` class is a subclass of :class:`tzinfo`, each instance of which represents a time zone defined by a fixed offset from @@ -2390,6 +2450,7 @@ where historical changes have been made to civil time. .. versionchanged:: 3.7 The UTC offset is not restricted to a whole number of minutes. + .. method:: timezone.tzname(dt) Return the fixed value specified when the :class:`timezone` instance @@ -2410,11 +2471,13 @@ where historical changes have been made to civil time. Always returns ``None``. + .. method:: timezone.fromutc(dt) Return ``dt + offset``. The *dt* argument must be an aware :class:`.datetime` instance, with ``tzinfo`` set to ``self``. + Class attributes: .. attribute:: timezone.utc @@ -2427,8 +2490,8 @@ Class attributes: .. _strftime-strptime-behavior: -:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Behavior --------------------------------------------------------------------- +:meth:`!strftime` and :meth:`!strptime` behavior +------------------------------------------------ :class:`date`, :class:`.datetime`, and :class:`.time` objects all support a ``strftime(format)`` method, to create a string representing the time under the @@ -2454,13 +2517,14 @@ versus :meth:`~.datetime.strptime`: .. _format-codes: -:meth:`~.datetime.strftime` and :meth:`~.datetime.strptime` Format Codes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:meth:`!strftime` and :meth:`!strptime` format codes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ These methods accept format codes that can be used to parse and format dates:: - >>> datetime.strptime('31/01/22 23:59:59.999999', - ... '%d/%m/%y %H:%M:%S.%f') + >>> import datetime as dt + >>> dt.datetime.strptime('31/01/22 23:59:59.999999', + ... '%d/%m/%y %H:%M:%S.%f') datetime.datetime(2022, 1, 31, 23, 59, 59, 999999) >>> _.strftime('%a %d %b %Y, %I:%M%p') 'Mon 31 Jan 2022, 11:59PM' @@ -2611,16 +2675,55 @@ differences between platforms in handling of unsupported format specifiers. .. versionadded:: 3.12 ``%:z`` was added. -Technical Detail + +Technical detail ^^^^^^^^^^^^^^^^ Broadly speaking, ``d.strftime(fmt)`` acts like the :mod:`time` module's ``time.strftime(fmt, d.timetuple())`` although not all objects support a :meth:`~date.timetuple` method. -For the :meth:`.datetime.strptime` class method, the default value is -``1900-01-01T00:00:00.000``: any components not specified in the format string -will be pulled from the default value. [#]_ +For the :meth:`.datetime.strptime` and :meth:`.date.strptime` class methods, +the default value is ``1900-01-01T00:00:00.000``: any components not specified +in the format string will be pulled from the default value. + +.. note:: + Format strings without separators can be ambiguous for parsing. For + example, with ``%Y%m%d``, the string ``2026111`` may be parsed either as + ``2026-11-01`` or as ``2026-01-11``. + Use separators to ensure the input is parsed as intended. + +.. note:: + When used to parse partial dates lacking a year, :meth:`.datetime.strptime` + and :meth:`.date.strptime` will raise when encountering February 29 because + the default year of 1900 is *not* a leap year. Always add a default leap + year to partial date strings before parsing. + +.. testsetup:: + + # doctest seems to turn the warning into an error which makes it + # show up and require matching and prevents the actual interesting + # exception from being raised. + # Manually apply the catch_warnings context manager + import warnings + catch_warnings = warnings.catch_warnings() + catch_warnings.__enter__() + warnings.simplefilter("ignore") + +.. testcleanup:: + + catch_warnings.__exit__() + +.. doctest:: + + >>> import datetime as dt + >>> value = "2/29" + >>> dt.datetime.strptime(value, "%m/%d") + Traceback (most recent call last): + ... + ValueError: day 29 must be in range 1..28 for month 2 in year 1900 + >>> dt.datetime.strptime(f"1904 {value}", "%Y %m/%d") + datetime.datetime(1904, 2, 29, 0, 0) Using ``datetime.strptime(date_string, format)`` is equivalent to:: @@ -2751,12 +2854,12 @@ Notes: include a year in the format. If the value you need to parse lacks a year, append an explicit dummy leap year. Otherwise your code will raise an exception when it encounters leap day because the default year used by the - parser is not a leap year. Users run into this bug every four years... + parser (1900) is not a leap year. Users run into that bug every leap year. .. doctest:: >>> month_day = "02/29" - >>> datetime.strptime(f"{month_day};1984", "%m/%d;%Y") # No leap year bug. + >>> dt.datetime.strptime(f"{month_day};1984", "%m/%d;%Y") # No leap year bug. datetime.datetime(1984, 2, 29, 0, 0) .. deprecated-removed:: 3.13 3.15 @@ -2767,7 +2870,7 @@ Notes: .. rubric:: Footnotes -.. [#] If, that is, we ignore the effects of Relativity +.. [#] If, that is, we ignore the effects of relativity. .. [#] This matches the definition of the "proleptic Gregorian" calendar in Dershowitz and Reingold's book *Calendrical Calculations*, @@ -2778,5 +2881,3 @@ Notes: .. [#] See R. H. van Gent's `guide to the mathematics of the ISO 8601 calendar `_ for a good explanation. - -.. [#] Passing ``datetime.strptime('Feb 29', '%b %d')`` will fail since 1900 is not a leap year. diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 36221c026d6d4b0..c736ee0d705acc4 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -8,7 +8,7 @@ -------------- -:mod:`dbm` is a generic interface to variants of the DBM database: +:mod:`!dbm` is a generic interface to variants of the DBM database: * :mod:`dbm.sqlite3` * :mod:`dbm.gnu` @@ -84,10 +84,13 @@ the Oracle Berkeley DB. .. versionchanged:: 3.11 *file* accepts a :term:`path-like object`. -The object returned by :func:`~dbm.open` supports the same basic functionality as a -:class:`dict`; keys and their corresponding values can be stored, retrieved, and -deleted, and the :keyword:`in` operator and the :meth:`!keys` method are -available, as well as :meth:`!get` and :meth:`!setdefault` methods. +The object returned by :func:`~dbm.open` supports the basic +functionality of mutable :term:`mappings `; +keys and their corresponding values can be stored, retrieved, and +deleted, and iteration, the :keyword:`in` operator and methods :meth:`!keys`, +:meth:`!get`, :meth:`!setdefault` and :meth:`!clear` are available. +The :meth:`!keys` method returns a list instead of a view object. +The :meth:`!setdefault` method requires two arguments. Key and values are always stored as :class:`bytes`. This means that when strings are used they are implicitly converted to the default encoding before @@ -98,7 +101,7 @@ will automatically close them when done. .. versionchanged:: 3.2 :meth:`!get` and :meth:`!setdefault` methods are now available for all - :mod:`dbm` backends. + :mod:`!dbm` backends. .. versionchanged:: 3.4 Added native support for the context management protocol to the objects @@ -108,6 +111,10 @@ will automatically close them when done. Deleting a key from a read-only database raises a database module specific exception instead of :exc:`KeyError`. +.. versionchanged:: 3.13 + :meth:`!clear` methods are now available for all :mod:`!dbm` backends. + + The following example records some hostnames and a corresponding title, and then prints out the contents of the database:: @@ -144,11 +151,10 @@ then prints out the contents of the database:: The individual submodules are described in the following sections. -:mod:`dbm.sqlite3` --- SQLite backend for dbm ---------------------------------------------- +:mod:`!dbm.sqlite3` --- SQLite backend for dbm +---------------------------------------------- .. module:: dbm.sqlite3 - :platform: All :synopsis: SQLite backend for dbm .. versionadded:: 3.13 @@ -158,8 +164,8 @@ The individual submodules are described in the following sections. -------------- This module uses the standard library :mod:`sqlite3` module to provide an -SQLite backend for the :mod:`dbm` module. -The files created by :mod:`dbm.sqlite3` can thus be opened by :mod:`sqlite3`, +SQLite backend for the :mod:`!dbm` module. +The files created by :mod:`!dbm.sqlite3` can thus be opened by :mod:`sqlite3`, or any other SQLite browser, including the SQLite CLI. .. include:: ../includes/wasm-notavail.rst @@ -167,9 +173,6 @@ or any other SQLite browser, including the SQLite CLI. .. function:: open(filename, /, flag="r", mode=0o666) Open an SQLite database. - The returned object behaves like a :term:`mapping`, - implements a :meth:`!close` method, - and supports a "closing" context manager via the :keyword:`with` keyword. :param filename: The path to the database to be opened. @@ -186,35 +189,52 @@ or any other SQLite browser, including the SQLite CLI. The Unix file access mode of the file (default: octal ``0o666``), used only when the database has to be created. + The returned database object behaves similar to a mutable :term:`mapping`, + but the :meth:`!keys` method returns a list, and + the :meth:`!setdefault` method requires two arguments. + It also supports a "closing" context manager via the :keyword:`with` keyword. + + The following method is also provided: -:mod:`dbm.gnu` --- GNU database manager ---------------------------------------- + .. method:: sqlite3.close() + + Close the SQLite database. + + +:mod:`!dbm.gnu` --- GNU database manager +---------------------------------------- .. module:: dbm.gnu - :platform: Unix :synopsis: GNU database manager **Source code:** :source:`Lib/dbm/gnu.py` -------------- -The :mod:`dbm.gnu` module provides an interface to the :abbr:`GDBM (GNU dbm)` +The :mod:`!dbm.gnu` module provides an interface to the :abbr:`GDBM (GNU dbm)` library, similar to the :mod:`dbm.ndbm` module, but with additional functionality like crash tolerance. .. note:: - The file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible + The file formats created by :mod:`!dbm.gnu` and :mod:`dbm.ndbm` are incompatible and can not be used interchangeably. .. include:: ../includes/wasm-mobile-notavail.rst +.. availability:: Unix. + .. exception:: error - Raised on :mod:`dbm.gnu`-specific errors, such as I/O errors. :exc:`KeyError` is + Raised on :mod:`!dbm.gnu`-specific errors, such as I/O errors. :exc:`KeyError` is raised for general mapping errors like specifying an incorrect key. +.. data:: open_flags + + A string of characters the *flag* parameter of :meth:`~dbm.gnu.open` supports. + + .. function:: open(filename, flag="r", mode=0o666, /) Open a GDBM database and return a :class:`!gdbm` object. @@ -250,14 +270,25 @@ functionality like crash tolerance. .. versionchanged:: 3.11 *filename* accepts a :term:`path-like object`. - .. data:: open_flags + :class:`!gdbm` objects behave similar to mutable :term:`mappings `, + but methods :meth:`!items`, :meth:`!values`, :meth:`!pop`, :meth:`!popitem`, + and :meth:`!update` are not supported, + the :meth:`!keys` method returns a list, and + the :meth:`!setdefault` method requires two arguments. + It also supports a "closing" context manager via the :keyword:`with` keyword. - A string of characters the *flag* parameter of :meth:`~dbm.gnu.open` supports. + .. versionchanged:: 3.2 + Added the :meth:`!get` and :meth:`!setdefault` methods. + + .. versionchanged:: 3.13 + Added the :meth:`!clear` method. - :class:`!gdbm` objects behave similar to :term:`mappings `, - but :meth:`!items` and :meth:`!values` methods are not supported. The following methods are also provided: + .. method:: gdbm.close() + + Close the GDBM database. + .. method:: gdbm.firstkey() It's possible to loop over every key in the database using this method and the @@ -289,36 +320,25 @@ functionality like crash tolerance. When the database has been opened in fast mode, this method forces any unwritten data to be written to the disk. - .. method:: gdbm.close() - - Close the GDBM database. - - .. method:: gdbm.clear() - - Remove all items from the GDBM database. - .. versionadded:: 3.13 - - -:mod:`dbm.ndbm` --- New Database Manager ----------------------------------------- +:mod:`!dbm.ndbm` --- New Database Manager +----------------------------------------- .. module:: dbm.ndbm - :platform: Unix :synopsis: The New Database Manager **Source code:** :source:`Lib/dbm/ndbm.py` -------------- -The :mod:`dbm.ndbm` module provides an interface to the +The :mod:`!dbm.ndbm` module provides an interface to the :abbr:`NDBM (New Database Manager)` library. This module can be used with the "classic" NDBM interface or the :abbr:`GDBM (GNU dbm)` compatibility interface. .. note:: - The file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible + The file formats created by :mod:`dbm.gnu` and :mod:`!dbm.ndbm` are incompatible and can not be used interchangeably. .. warning:: @@ -330,9 +350,11 @@ This module can be used with the "classic" NDBM interface or the .. include:: ../includes/wasm-mobile-notavail.rst +.. availability:: Unix. + .. exception:: error - Raised on :mod:`dbm.ndbm`-specific errors, such as I/O errors. :exc:`KeyError` is raised + Raised on :mod:`!dbm.ndbm`-specific errors, such as I/O errors. :exc:`KeyError` is raised for general mapping errors like specifying an incorrect key. @@ -359,26 +381,31 @@ This module can be used with the "classic" NDBM interface or the :param int mode: |mode_param_doc| - :class:`!ndbm` objects behave similar to :term:`mappings `, - but :meth:`!items` and :meth:`!values` methods are not supported. - The following methods are also provided: - .. versionchanged:: 3.11 Accepts :term:`path-like object` for filename. - .. method:: ndbm.close() + :class:`!ndbm` objects behave similar to mutable :term:`mappings `, + but methods :meth:`!items`, :meth:`!values`, :meth:`!pop`, :meth:`!popitem`, + and :meth:`!update` are not supported, + the :meth:`!keys` method returns a list, and + the :meth:`!setdefault` method requires two arguments. + It also supports a "closing" context manager via the :keyword:`with` keyword. - Close the NDBM database. + .. versionchanged:: 3.2 + Added the :meth:`!get` and :meth:`!setdefault` methods. - .. method:: ndbm.clear() + .. versionchanged:: 3.13 + Added the :meth:`!clear` method. - Remove all items from the NDBM database. + The following method is also provided: - .. versionadded:: 3.13 + .. method:: ndbm.close() + + Close the NDBM database. -:mod:`dbm.dumb` --- Portable DBM implementation ------------------------------------------------ +:mod:`!dbm.dumb` --- Portable DBM implementation +------------------------------------------------ .. module:: dbm.dumb :synopsis: Portable implementation of the simple DBM interface. @@ -389,32 +416,29 @@ This module can be used with the "classic" NDBM interface or the .. note:: - The :mod:`dbm.dumb` module is intended as a last resort fallback for the - :mod:`dbm` module when a more robust module is not available. The :mod:`dbm.dumb` + The :mod:`!dbm.dumb` module is intended as a last resort fallback for the + :mod:`!dbm` module when a more robust module is not available. The :mod:`!dbm.dumb` module is not written for speed and is not nearly as heavily used as the other database modules. -------------- -The :mod:`dbm.dumb` module provides a persistent :class:`dict`-like +The :mod:`!dbm.dumb` module provides a persistent :class:`dict`-like interface which is written entirely in Python. -Unlike other :mod:`dbm` backends, such as :mod:`dbm.gnu`, no +Unlike other :mod:`!dbm` backends, such as :mod:`dbm.gnu`, no external library is required. The :mod:`!dbm.dumb` module defines the following: .. exception:: error - Raised on :mod:`dbm.dumb`-specific errors, such as I/O errors. :exc:`KeyError` is + Raised on :mod:`!dbm.dumb`-specific errors, such as I/O errors. :exc:`KeyError` is raised for general mapping errors like specifying an incorrect key. .. function:: open(filename, flag="c", mode=0o666) Open a :mod:`!dbm.dumb` database. - The returned database object behaves similar to a :term:`mapping`, - in addition to providing :meth:`~dumbdbm.sync` and :meth:`~dumbdbm.close` - methods. :param filename: The basename of the database file (without extensions). @@ -448,15 +472,18 @@ The :mod:`!dbm.dumb` module defines the following: .. versionchanged:: 3.11 *filename* accepts a :term:`path-like object`. - In addition to the methods provided by the - :class:`collections.abc.MutableMapping` class, - the following methods are provided: + The returned database object behaves similar to a mutable :term:`mapping`, + but the :meth:`!keys` and :meth:`!items` methods return lists, and + the :meth:`!setdefault` method requires two arguments. + It also supports a "closing" context manager via the :keyword:`with` keyword. - .. method:: dumbdbm.sync() - - Synchronize the on-disk directory and data files. This method is called - by the :meth:`shelve.Shelf.sync` method. + The following methods are also provided: .. method:: dumbdbm.close() Close the database. + + .. method:: dumbdbm.sync() + + Synchronize the on-disk directory and data files. This method is called + by the :meth:`shelve.Shelf.sync` method. diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index f53c1f3467034ec..d4c089f5a144145 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -2,7 +2,7 @@ ===================================================================== .. module:: decimal - :synopsis: Implementation of the General Decimal Arithmetic Specification. + :synopsis: Implementation of the General Decimal Arithmetic Specification. .. moduleauthor:: Eric Price .. moduleauthor:: Facundo Batista @@ -30,14 +30,16 @@ -------------- -The :mod:`decimal` module provides support for fast correctly rounded +The :mod:`!decimal` module provides support for fast correctly rounded decimal floating-point arithmetic. It offers several advantages over the :class:`float` datatype: -* Decimal "is based on a floating-point model which was designed with people - in mind, and necessarily has a paramount guiding principle -- computers must - provide an arithmetic that works in the same way as the arithmetic that - people learn at school." -- excerpt from the decimal arithmetic specification. +* Decimal "is based on a `floating-point model + `__ which was designed + with people in mind, and necessarily has a paramount guiding principle -- + computers must provide an arithmetic that works in the same way as the + arithmetic that people learn at school." -- excerpt from the decimal + arithmetic specification. * Decimal numbers can be represented exactly. In contrast, numbers like ``1.1`` and ``2.2`` do not have exact representations in binary @@ -121,7 +123,7 @@ reset them before monitoring a calculation. .. _decimal-tutorial: -Quick-start Tutorial +Quick-start tutorial -------------------- The usual start to using decimals is importing the module, viewing the current @@ -238,6 +240,26 @@ floating-point flying circus: >>> c % a Decimal('0.77') +Decimals can be formatted (with :func:`format` built-in or :ref:`f-strings`) in +fixed-point or scientific notation, using the same formatting syntax (see +:ref:`formatspec`) as builtin :class:`float` type: + +.. doctest:: + + >>> format(Decimal('2.675'), "f") + '2.675' + >>> format(Decimal('2.675'), ".2f") + '2.68' + >>> f"{Decimal('2.675'):.2f}" + '2.68' + >>> format(Decimal('2.675'), ".2e") + '2.68e+0' + >>> with localcontext() as ctx: + ... ctx.rounding = ROUND_DOWN + ... print(format(Decimal('2.675'), ".2f")) + ... + 2.67 + And some mathematical functions are also available to Decimal: >>> getcontext().prec = 28 @@ -264,10 +286,10 @@ allows the settings to be changed. This approach meets the needs of most applications. For more advanced work, it may be useful to create alternate contexts using the -Context() constructor. To make an alternate active, use the :func:`setcontext` +:meth:`Context` constructor. To make an alternate active, use the :func:`setcontext` function. -In accordance with the standard, the :mod:`decimal` module provides two ready to +In accordance with the standard, the :mod:`!decimal` module provides two ready to use standard contexts, :const:`BasicContext` and :const:`ExtendedContext`. The former is especially useful for debugging because many of the traps are enabled: @@ -573,7 +595,7 @@ Decimal objects >>> Decimal(321).exp() Decimal('2.561702493119680037517373933E+139') - .. classmethod:: from_float(f) + .. classmethod:: from_float(f, /) Alternative constructor that only accepts instances of :class:`float` or :class:`int`. @@ -600,7 +622,7 @@ Decimal objects .. versionadded:: 3.1 - .. classmethod:: from_number(number) + .. classmethod:: from_number(number, /) Alternative constructor that only accepts instances of :class:`float`, :class:`int` or :class:`Decimal`, but not strings @@ -991,7 +1013,7 @@ Each thread has its own current context which is accessed or changed using the Return the current context for the active thread. -.. function:: setcontext(c) +.. function:: setcontext(c, /) Set the current context for the active thread to *c*. @@ -1096,40 +1118,52 @@ In addition to the three supplied contexts, new contexts can be created with the default values are copied from the :const:`DefaultContext`. If the *flags* field is not specified or is :const:`None`, all flags are cleared. - *prec* is an integer in the range [``1``, :const:`MAX_PREC`] that sets - the precision for arithmetic operations in the context. + .. attribute:: prec + + An integer in the range [``1``, :const:`MAX_PREC`] that sets + the precision for arithmetic operations in the context. + + .. attribute:: rounding + + One of the constants listed in the section `Rounding Modes`_. + + .. attribute:: traps + flags + + Lists of any signals to be set. Generally, new contexts should only set + traps and leave the flags clear. + + .. attribute:: Emin + Emax - The *rounding* option is one of the constants listed in the section - `Rounding Modes`_. + Integers specifying the outer limits allowable for exponents. *Emin* must + be in the range [:const:`MIN_EMIN`, ``0``], *Emax* in the range + [``0``, :const:`MAX_EMAX`]. - The *traps* and *flags* fields list any signals to be set. Generally, new - contexts should only set traps and leave the flags clear. + .. attribute:: capitals - The *Emin* and *Emax* fields are integers specifying the outer limits allowable - for exponents. *Emin* must be in the range [:const:`MIN_EMIN`, ``0``], - *Emax* in the range [``0``, :const:`MAX_EMAX`]. + Either ``0`` or ``1`` (the default). If set to + ``1``, exponents are printed with a capital ``E``; otherwise, a + lowercase ``e`` is used: ``Decimal('6.02e+23')``. - The *capitals* field is either ``0`` or ``1`` (the default). If set to - ``1``, exponents are printed with a capital ``E``; otherwise, a - lowercase ``e`` is used: ``Decimal('6.02e+23')``. + .. attribute:: clamp - The *clamp* field is either ``0`` (the default) or ``1``. - If set to ``1``, the exponent ``e`` of a :class:`Decimal` - instance representable in this context is strictly limited to the - range ``Emin - prec + 1 <= e <= Emax - prec + 1``. If *clamp* is - ``0`` then a weaker condition holds: the adjusted exponent of - the :class:`Decimal` instance is at most :attr:`~Context.Emax`. When *clamp* is - ``1``, a large normal number will, where possible, have its - exponent reduced and a corresponding number of zeros added to its - coefficient, in order to fit the exponent constraints; this - preserves the value of the number but loses information about - significant trailing zeros. For example:: + Either ``0`` (the default) or ``1``. If set to ``1``, the exponent ``e`` + of a :class:`Decimal` instance representable in this context is strictly + limited to the range ``Emin - prec + 1 <= e <= Emax - prec + 1``. + If *clamp* is ``0`` then a weaker condition holds: the adjusted exponent of + the :class:`Decimal` instance is at most :attr:`~Context.Emax`. When *clamp* is + ``1``, a large normal number will, where possible, have its + exponent reduced and a corresponding number of zeros added to its + coefficient, in order to fit the exponent constraints; this + preserves the value of the number but loses information about + significant trailing zeros. For example:: - >>> Context(prec=6, Emax=999, clamp=1).create_decimal('1.23e999') - Decimal('1.23000E+999') + >>> Context(prec=6, Emax=999, clamp=1).create_decimal('1.23e999') + Decimal('1.23000E+999') - A *clamp* value of ``1`` allows compatibility with the - fixed-width decimal interchange formats specified in IEEE 754. + A *clamp* value of ``1`` allows compatibility with the + fixed-width decimal interchange formats specified in IEEE 754. The :class:`Context` class defines several general purpose methods as well as a large number of methods for doing arithmetic directly in a given context. @@ -1156,11 +1190,11 @@ In addition to the three supplied contexts, new contexts can be created with the Return a duplicate of the context. - .. method:: copy_decimal(num) + .. method:: copy_decimal(num, /) Return a copy of the Decimal instance num. - .. method:: create_decimal(num) + .. method:: create_decimal(num='0', /) Creates a new Decimal instance from *num* but using *self* as context. Unlike the :class:`Decimal` constructor, the context precision, @@ -1184,7 +1218,7 @@ In addition to the three supplied contexts, new contexts can be created with the If the argument is a string, no leading or trailing whitespace or underscores are permitted. - .. method:: create_decimal_from_float(f) + .. method:: create_decimal_from_float(f, /) Creates a new Decimal instance from a float *f* but rounding using *self* as the context. Unlike the :meth:`Decimal.from_float` class method, @@ -1222,222 +1256,222 @@ In addition to the three supplied contexts, new contexts can be created with the recounted here. - .. method:: abs(x) + .. method:: abs(x, /) Returns the absolute value of *x*. - .. method:: add(x, y) + .. method:: add(x, y, /) Return the sum of *x* and *y*. - .. method:: canonical(x) + .. method:: canonical(x, /) Returns the same Decimal object *x*. - .. method:: compare(x, y) + .. method:: compare(x, y, /) Compares *x* and *y* numerically. - .. method:: compare_signal(x, y) + .. method:: compare_signal(x, y, /) Compares the values of the two operands numerically. - .. method:: compare_total(x, y) + .. method:: compare_total(x, y, /) Compares two operands using their abstract representation. - .. method:: compare_total_mag(x, y) + .. method:: compare_total_mag(x, y, /) Compares two operands using their abstract representation, ignoring sign. - .. method:: copy_abs(x) + .. method:: copy_abs(x, /) Returns a copy of *x* with the sign set to 0. - .. method:: copy_negate(x) + .. method:: copy_negate(x, /) Returns a copy of *x* with the sign inverted. - .. method:: copy_sign(x, y) + .. method:: copy_sign(x, y, /) Copies the sign from *y* to *x*. - .. method:: divide(x, y) + .. method:: divide(x, y, /) Return *x* divided by *y*. - .. method:: divide_int(x, y) + .. method:: divide_int(x, y, /) Return *x* divided by *y*, truncated to an integer. - .. method:: divmod(x, y) + .. method:: divmod(x, y, /) Divides two numbers and returns the integer part of the result. - .. method:: exp(x) + .. method:: exp(x, /) Returns ``e ** x``. - .. method:: fma(x, y, z) + .. method:: fma(x, y, z, /) Returns *x* multiplied by *y*, plus *z*. - .. method:: is_canonical(x) + .. method:: is_canonical(x, /) Returns ``True`` if *x* is canonical; otherwise returns ``False``. - .. method:: is_finite(x) + .. method:: is_finite(x, /) Returns ``True`` if *x* is finite; otherwise returns ``False``. - .. method:: is_infinite(x) + .. method:: is_infinite(x, /) Returns ``True`` if *x* is infinite; otherwise returns ``False``. - .. method:: is_nan(x) + .. method:: is_nan(x, /) Returns ``True`` if *x* is a qNaN or sNaN; otherwise returns ``False``. - .. method:: is_normal(x) + .. method:: is_normal(x, /) Returns ``True`` if *x* is a normal number; otherwise returns ``False``. - .. method:: is_qnan(x) + .. method:: is_qnan(x, /) Returns ``True`` if *x* is a quiet NaN; otherwise returns ``False``. - .. method:: is_signed(x) + .. method:: is_signed(x, /) Returns ``True`` if *x* is negative; otherwise returns ``False``. - .. method:: is_snan(x) + .. method:: is_snan(x, /) Returns ``True`` if *x* is a signaling NaN; otherwise returns ``False``. - .. method:: is_subnormal(x) + .. method:: is_subnormal(x, /) Returns ``True`` if *x* is subnormal; otherwise returns ``False``. - .. method:: is_zero(x) + .. method:: is_zero(x, /) Returns ``True`` if *x* is a zero; otherwise returns ``False``. - .. method:: ln(x) + .. method:: ln(x, /) Returns the natural (base e) logarithm of *x*. - .. method:: log10(x) + .. method:: log10(x, /) Returns the base 10 logarithm of *x*. - .. method:: logb(x) + .. method:: logb(x, /) Returns the exponent of the magnitude of the operand's MSD. - .. method:: logical_and(x, y) + .. method:: logical_and(x, y, /) Applies the logical operation *and* between each operand's digits. - .. method:: logical_invert(x) + .. method:: logical_invert(x, /) Invert all the digits in *x*. - .. method:: logical_or(x, y) + .. method:: logical_or(x, y, /) Applies the logical operation *or* between each operand's digits. - .. method:: logical_xor(x, y) + .. method:: logical_xor(x, y, /) Applies the logical operation *xor* between each operand's digits. - .. method:: max(x, y) + .. method:: max(x, y, /) Compares two values numerically and returns the maximum. - .. method:: max_mag(x, y) + .. method:: max_mag(x, y, /) Compares the values numerically with their sign ignored. - .. method:: min(x, y) + .. method:: min(x, y, /) Compares two values numerically and returns the minimum. - .. method:: min_mag(x, y) + .. method:: min_mag(x, y, /) Compares the values numerically with their sign ignored. - .. method:: minus(x) + .. method:: minus(x, /) Minus corresponds to the unary prefix minus operator in Python. - .. method:: multiply(x, y) + .. method:: multiply(x, y, /) Return the product of *x* and *y*. - .. method:: next_minus(x) + .. method:: next_minus(x, /) Returns the largest representable number smaller than *x*. - .. method:: next_plus(x) + .. method:: next_plus(x, /) Returns the smallest representable number larger than *x*. - .. method:: next_toward(x, y) + .. method:: next_toward(x, y, /) Returns the number closest to *x*, in direction towards *y*. - .. method:: normalize(x) + .. method:: normalize(x, /) Reduces *x* to its simplest form. - .. method:: number_class(x) + .. method:: number_class(x, /) Returns an indication of the class of *x*. - .. method:: plus(x) + .. method:: plus(x, /) Plus corresponds to the unary prefix plus operator in Python. This operation applies the context precision and rounding, so it is *not* an @@ -1478,7 +1512,7 @@ In addition to the three supplied contexts, new contexts can be created with the always exact. - .. method:: quantize(x, y) + .. method:: quantize(x, y, /) Returns a value equal to *x* (rounded), having the exponent of *y*. @@ -1488,7 +1522,7 @@ In addition to the three supplied contexts, new contexts can be created with the Just returns 10, as this is Decimal, :) - .. method:: remainder(x, y) + .. method:: remainder(x, y, /) Returns the remainder from integer division. @@ -1496,43 +1530,43 @@ In addition to the three supplied contexts, new contexts can be created with the dividend. - .. method:: remainder_near(x, y) + .. method:: remainder_near(x, y, /) Returns ``x - y * n``, where *n* is the integer nearest the exact value of ``x / y`` (if the result is 0 then its sign will be the sign of *x*). - .. method:: rotate(x, y) + .. method:: rotate(x, y, /) Returns a rotated copy of *x*, *y* times. - .. method:: same_quantum(x, y) + .. method:: same_quantum(x, y, /) Returns ``True`` if the two operands have the same exponent. - .. method:: scaleb (x, y) + .. method:: scaleb (x, y, /) Returns the first operand after adding the second value its exp. - .. method:: shift(x, y) + .. method:: shift(x, y, /) Returns a shifted copy of *x*, *y* times. - .. method:: sqrt(x) + .. method:: sqrt(x, /) Square root of a non-negative number to context precision. - .. method:: subtract(x, y) + .. method:: subtract(x, y, /) Return the difference between *x* and *y*. - .. method:: to_eng_string(x) + .. method:: to_eng_string(x, /) Convert to a string, using engineering notation if an exponent is needed. @@ -1541,12 +1575,12 @@ In addition to the three supplied contexts, new contexts can be created with the require the addition of either one or two trailing zeros. - .. method:: to_integral_exact(x) + .. method:: to_integral_exact(x, /) Rounds to an integer. - .. method:: to_sci_string(x) + .. method:: to_sci_string(x, /) Converts a number to a string using scientific notation. @@ -1769,7 +1803,7 @@ The following table summarizes the hierarchy of signals:: .. _decimal-notes: -Floating-Point Notes +Floating-point notes -------------------- @@ -1804,7 +1838,7 @@ properties of addition: >>> u * (v+w) Decimal('0.0060000') -The :mod:`decimal` module makes it possible to restore the identities by +The :mod:`!decimal` module makes it possible to restore the identities by expanding the precision sufficiently to avoid loss of significance: .. doctest:: newcontext @@ -1826,7 +1860,7 @@ expanding the precision sufficiently to avoid loss of significance: Special values ^^^^^^^^^^^^^^ -The number system for the :mod:`decimal` module provides special values +The number system for the :mod:`!decimal` module provides special values including ``NaN``, ``sNaN``, ``-Infinity``, ``Infinity``, and two zeros, ``+0`` and ``-0``. @@ -2088,20 +2122,20 @@ to work with the :class:`Decimal` class:: Decimal FAQ ----------- -Q. It is cumbersome to type ``decimal.Decimal('1234.5')``. Is there a way to +Q: It is cumbersome to type ``decimal.Decimal('1234.5')``. Is there a way to minimize typing when using the interactive interpreter? -A. Some users abbreviate the constructor to just a single letter: +A: Some users abbreviate the constructor to just a single letter: >>> D = decimal.Decimal >>> D('1.23') + D('3.45') Decimal('4.68') -Q. In a fixed-point application with two decimal places, some inputs have many +Q: In a fixed-point application with two decimal places, some inputs have many places and need to be rounded. Others are not supposed to have excess digits and need to be validated. What methods should be used? -A. The :meth:`~Decimal.quantize` method rounds to a fixed number of decimal places. If +A: The :meth:`~Decimal.quantize` method rounds to a fixed number of decimal places. If the :const:`Inexact` trap is set, it is also useful for validation: >>> TWOPLACES = Decimal(10) ** -2 # same as Decimal('0.01') @@ -2119,10 +2153,10 @@ the :const:`Inexact` trap is set, it is also useful for validation: ... Inexact: None -Q. Once I have valid two place inputs, how do I maintain that invariant +Q: Once I have valid two place inputs, how do I maintain that invariant throughout an application? -A. Some operations like addition, subtraction, and multiplication by an integer +A: Some operations like addition, subtraction, and multiplication by an integer will automatically preserve fixed point. Others operations, like division and non-integer multiplication, will change the number of decimal places and need to be followed-up with a :meth:`~Decimal.quantize` step: @@ -2154,21 +2188,21 @@ to handle the :meth:`~Decimal.quantize` step: >>> div(b, a) Decimal('0.03') -Q. There are many ways to express the same value. The numbers ``200``, +Q: There are many ways to express the same value. The numbers ``200``, ``200.000``, ``2E2``, and ``.02E+4`` all have the same value at various precisions. Is there a way to transform them to a single recognizable canonical value? -A. The :meth:`~Decimal.normalize` method maps all equivalent values to a single +A: The :meth:`~Decimal.normalize` method maps all equivalent values to a single representative: >>> values = map(Decimal, '200 200.000 2E2 .02E+4'.split()) >>> [v.normalize() for v in values] [Decimal('2E+2'), Decimal('2E+2'), Decimal('2E+2'), Decimal('2E+2')] -Q. When does rounding occur in a computation? +Q: When does rounding occur in a computation? -A. It occurs *after* the computation. The philosophy of the decimal +A: It occurs *after* the computation. The philosophy of the decimal specification is that numbers are considered exact and are created independent of the current context. They can even have greater precision than current context. Computations process with those @@ -2186,10 +2220,10 @@ applied to the *result* of the computation:: >>> pi + 0 - Decimal('0.00005'). # Intermediate values are rounded Decimal('3.1416') -Q. Some decimal values always print with exponential notation. Is there a way +Q: Some decimal values always print with exponential notation. Is there a way to get a non-exponential representation? -A. For some values, exponential notation is the only way to express the number +A: For some values, exponential notation is the only way to express the number of significant places in the coefficient. For example, expressing ``5.0E+3`` as ``5000`` keeps the value constant but cannot show the original's two-place significance. @@ -2204,9 +2238,9 @@ value unchanged: >>> remove_exponent(Decimal('5E+3')) Decimal('5000') -Q. Is there a way to convert a regular float to a :class:`Decimal`? +Q: Is there a way to convert a regular float to a :class:`Decimal`? -A. Yes, any binary floating-point number can be exactly expressed as a +A: Yes, any binary floating-point number can be exactly expressed as a Decimal though an exact conversion may take more precision than intuition would suggest: @@ -2215,19 +2249,19 @@ suggest: >>> Decimal(math.pi) Decimal('3.141592653589793115997963468544185161590576171875') -Q. Within a complex calculation, how can I make sure that I haven't gotten a +Q: Within a complex calculation, how can I make sure that I haven't gotten a spurious result because of insufficient precision or rounding anomalies. -A. The decimal module makes it easy to test results. A best practice is to +A: The decimal module makes it easy to test results. A best practice is to re-run calculations using greater precision and with various rounding modes. Widely differing results indicate insufficient precision, rounding mode issues, ill-conditioned inputs, or a numerically unstable algorithm. -Q. I noticed that context precision is applied to the results of operations but +Q: I noticed that context precision is applied to the results of operations but not to the inputs. Is there anything to watch out for when mixing values of different precisions? -A. Yes. The principle is that all values are considered to be exact and so is +A: Yes. The principle is that all values are considered to be exact and so is the arithmetic on those values. Only the results are rounded. The advantage for inputs is that "what you type is what you get". A disadvantage is that the results can look odd if you forget that the inputs haven't been rounded: @@ -2255,9 +2289,9 @@ Alternatively, inputs can be rounded upon creation using the >>> Context(prec=5, rounding=ROUND_DOWN).create_decimal('1.2345678') Decimal('1.2345') -Q. Is the CPython implementation fast for large numbers? +Q: Is the CPython implementation fast for large numbers? -A. Yes. In the CPython and PyPy3 implementations, the C/CFFI versions of +A: Yes. In the CPython and PyPy3 implementations, the C/CFFI versions of the decimal module integrate the high speed `libmpdec `_ library for arbitrary precision correctly rounded decimal floating-point arithmetic [#]_. diff --git a/Doc/library/dialog.rst b/Doc/library/dialog.rst index 191e0da12103fa3..c50232f1d029cda 100644 --- a/Doc/library/dialog.rst +++ b/Doc/library/dialog.rst @@ -1,31 +1,45 @@ -Tkinter Dialogs +Tkinter dialogs =============== -:mod:`tkinter.simpledialog` --- Standard Tkinter input dialogs -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:mod:`!tkinter.simpledialog` --- Standard Tkinter input dialogs +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: tkinter.simpledialog - :platform: Tk :synopsis: Simple dialog windows **Source code:** :source:`Lib/tkinter/simpledialog.py` -------------- -The :mod:`tkinter.simpledialog` module contains convenience classes and +The :mod:`!tkinter.simpledialog` module contains convenience classes and functions for creating simple modal dialogs to get a value from the user. -.. function:: askfloat(title, prompt, **kw) - askinteger(title, prompt, **kw) - askstring(title, prompt, **kw) +.. function:: askfloat(title, prompt, *, initialvalue=None, minvalue=None, maxvalue=None, parent=None) + askinteger(title, prompt, *, initialvalue=None, minvalue=None, maxvalue=None, parent=None) + askstring(title, prompt, *, initialvalue=None, show=None, parent=None) - The above three functions provide dialogs that prompt the user to enter a value - of the desired type. + Prompt the user to enter a value of the desired type and return it, or + ``None`` if the dialog is cancelled. + + *title* is the dialog title and *prompt* the message shown above the entry. + *initialvalue* is the value initially placed in the entry. + *parent* is the window over which the dialog is shown. + :func:`askinteger` and :func:`askfloat` also accept *minvalue* and + *maxvalue*, which bound the accepted value. + :func:`askstring` also accepts *show*, a character used to mask the entered + text, for example ``'*'`` to hide a password. .. class:: Dialog(parent, title=None) The base class for custom dialogs. + Instantiating it shows the dialog modally and returns once the user closes + it; the entered value is then available in the :attr:`!result` attribute. + + .. attribute:: result + + The value produced by :meth:`apply`, or ``None`` if the dialog was + cancelled. .. method:: body(master) @@ -37,23 +51,59 @@ functions for creating simple modal dialogs to get a value from the user. Default behaviour adds OK and Cancel buttons. Override for custom button layouts. + .. method:: validate() + + Validate the data entered by the user. + Return true if it is valid, in which case the dialog proceeds to + :meth:`apply`; return false to keep the dialog open. + The default implementation always returns true; override it to check the + input. + + .. method:: apply() + + Process the data entered by the user, for example by storing it in the + :attr:`!result` attribute. + Called after :meth:`validate` succeeds and just before the dialog is + destroyed. + The default implementation does nothing; override it to act on or store + the result. + + .. method:: destroy() + + Destroy the dialog window, clearing the reference to the widget that had + the initial focus. -:mod:`tkinter.filedialog` --- File selection dialogs -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. class:: SimpleDialog(master, text='', buttons=[], default=None, cancel=None, title=None, class_=None) + + A simple modal dialog that displays the message *text* above a row of push + buttons whose labels are given by *buttons*, and returns the index of the + button the user presses. + *default* is the index of the button activated by the Return key, *cancel* + the index returned when the window is closed through the window manager, + *title* the window title, and *class_* the Tk class name of the window. + + .. method:: go() + + Display the dialog, wait until the user presses a button or closes the + window, and return the index of the chosen button. + + + +:mod:`!tkinter.filedialog` --- File selection dialogs +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: tkinter.filedialog - :platform: Tk :synopsis: Dialog classes for file selection **Source code:** :source:`Lib/tkinter/filedialog.py` -------------- -The :mod:`tkinter.filedialog` module provides classes and factory functions for +The :mod:`!tkinter.filedialog` module provides classes and factory functions for creating file/directory selection windows. -Native Load/Save Dialogs +Native load/save dialogs ------------------------ The following classes and functions provide file dialog windows that combine a @@ -79,50 +129,62 @@ listed below: **Static factory functions** The below functions when called create a modal, native look-and-feel dialog, -wait for the user's selection, then return the selected value(s) or ``None`` to the -caller. +wait for the user's selection, and return it. +The exact return value depends on the function (see below); when the dialog is +cancelled it is an empty string, an empty tuple or ``None``. +The precise type of this empty value may vary between platforms and Tk +versions, so test the result for truth rather than comparing it with a +specific value. .. function:: askopenfile(mode="r", **options) askopenfiles(mode="r", **options) - The above two functions create an :class:`Open` dialog and return the opened - file object(s) in read-only mode. + Create an :class:`Open` dialog. + :func:`askopenfile` returns the opened file object, or ``None`` if the + dialog is cancelled. + :func:`askopenfiles` returns a list of the opened file objects, or an empty + tuple if cancelled. + The files are opened in mode *mode* (read-only ``'r'`` by default). .. function:: asksaveasfile(mode="w", **options) - Create a :class:`SaveAs` dialog and return a file object opened in write-only mode. + Create a :class:`SaveAs` dialog and return the opened file object, or + ``None`` if the dialog is cancelled. + The file is opened in mode *mode* (``'w'`` by default). .. function:: askopenfilename(**options) askopenfilenames(**options) - The above two functions create an :class:`Open` dialog and return the - selected filename(s) that correspond to existing file(s). + Create an :class:`Open` dialog. + :func:`askopenfilename` returns the selected filename as a string, or an + empty string if the dialog is cancelled. + :func:`askopenfilenames` returns a tuple of the selected filenames, or an + empty tuple if cancelled. .. function:: asksaveasfilename(**options) - Create a :class:`SaveAs` dialog and return the selected filename. + Create a :class:`SaveAs` dialog and return the selected filename as a + string, or an empty string if the dialog is cancelled. .. function:: askdirectory(**options) - | Prompt user to select a directory. - | Additional keyword option: - | *mustexist* - determines if selection must be an existing directory. + Prompt the user to select a directory, and return its path as a string, or + an empty string if the dialog is cancelled. + Additional keyword option: *mustexist* - if true, the user may only select + an existing directory (false by default). .. class:: Open(master=None, **options) SaveAs(master=None, **options) + Directory(master=None, **options) - The above two classes provide native dialog windows for saving and loading - files. + The above three classes provide native dialog windows for loading and saving + files and for selecting a directory. **Convenience classes** The below classes are used for creating file/directory windows from scratch. These do not emulate the native look-and-feel of the platform. -.. class:: Directory(master=None, **options) - - Create a dialog prompting the user to select a directory. - .. note:: The *FileDialog* class should be subclassed for custom event handling and behaviour. @@ -170,6 +232,13 @@ These do not emulate the native look-and-feel of the platform. Exit dialog returning current selection. + .. method:: ok_command() + + Called when the user confirms the current selection. + The base implementation accepts the selection and closes the dialog; + :class:`LoadFileDialog` and :class:`SaveFileDialog` override it to check + the selection first. + .. method:: quit(how=None) Exit dialog returning filename, if any. @@ -204,27 +273,62 @@ These do not emulate the native look-and-feel of the platform. directory. Confirmation is required if an already existing file is selected. -:mod:`tkinter.commondialog` --- Dialog window templates -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +:mod:`!tkinter.commondialog` --- Dialog window templates +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: tkinter.commondialog - :platform: Tk :synopsis: Tkinter base class for dialogs **Source code:** :source:`Lib/tkinter/commondialog.py` -------------- -The :mod:`tkinter.commondialog` module provides the :class:`Dialog` class that +The :mod:`!tkinter.commondialog` module provides the :class:`Dialog` class that is the base class for dialogs defined in other supporting modules. .. class:: Dialog(master=None, **options) - .. method:: show(color=None, **options) + .. method:: show(**options) Render the Dialog window. +:mod:`!tkinter.dialog` --- Classic Tk dialog boxes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. module:: tkinter.dialog + :synopsis: A simple dialog box built on the classic Tk widgets. + +**Source code:** :source:`Lib/tkinter/dialog.py` + +-------------- + +The :mod:`!tkinter.dialog` module provides a simple modal dialog box built on +the classic (non-themed) Tk widgets. + +.. data:: DIALOG_ICON + + The name of a bitmap (``'questhead'``) suitable for use as the *bitmap* + of a :class:`Dialog`. + +.. class:: Dialog(master=None, cnf={}, **kw) + + Display a modal dialog box built from the classic (non-themed) Tk widgets + and wait for the user to press one of its buttons. + The options, given through *cnf* or as keyword arguments, are all required: + *title* (the window title), *text* (the message), *bitmap* (the name of a + bitmap icon, such as :data:`DIALOG_ICON`), *default* (the index of the + default button) and *strings* (the sequence of button labels). + After construction, the :attr:`!num` attribute holds the index of the button + the user pressed. + + .. method:: destroy() + + Do nothing. + The dialog window is destroyed automatically before the constructor + returns, so there is nothing left for this method to do. + + .. seealso:: Modules :mod:`tkinter.messagebox`, :ref:`tut-files` diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index ce948a6860f02c1..a45056e743389d0 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -44,13 +44,18 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. complicated way on how many elements the sequences have in common; best case time is linear. - **Automatic junk heuristic:** :class:`SequenceMatcher` supports a heuristic that - automatically treats certain sequence items as junk. The heuristic counts how many - times each individual item appears in the sequence. If an item's duplicates (after - the first one) account for more than 1% of the sequence and the sequence is at least - 200 items long, this item is marked as "popular" and is treated as junk for - the purpose of sequence matching. This heuristic can be turned off by setting - the ``autojunk`` argument to ``False`` when creating the :class:`SequenceMatcher`. + **Junk**: :class:`SequenceMatcher` accepts an ``isjunk`` predicate and an + ``autojunk`` flag. Items that are considered as junk will not be considered + to find similar content blocks. This can produce better results for humans + (typically breaking on whitespace) and faster (because it reduces the number + of possible combinations). But it can also cause pathological cases where + too many items considered junk cause an unexpectedly large (but correct) + diff result. + You should consider tuning them or turning them off depending on your data. + Moreover, only the second sequence is inspected for junk. This causes the diff + output to not be symmetrical. + When ``autojunk=True``, it will consider as junk the items that account for more + than 1% of the sequence, if it is at least 200 items long. .. versionchanged:: 3.2 Added the *autojunk* parameter. @@ -89,6 +94,11 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. with inter-line and intra-line change highlights. The table can be generated in either full or contextual difference mode. + .. warning:: + + The trailing newlines get stripped before the diff, so the result can be + incomplete. See :gh:`71896` for details. + The constructor for this class is: @@ -231,7 +241,7 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. *linejunk*: A function that accepts a single string argument, and returns true if the string is junk, or false if not. The default is ``None``. There is also a module-level function :func:`IS_LINE_JUNK`, which filters out lines - without visible characters, except for at most one pound character (``'#'``) + without visible characters, except for at most one hash character (``'#'``) -- however the underlying :class:`SequenceMatcher` class does a dynamic analysis of which lines are so frequent as to constitute noise, and this usually works better than using this function. @@ -351,14 +361,14 @@ diffs. For comparing directories and files, see also, the :mod:`filecmp` module. .. seealso:: - `Pattern Matching: The Gestalt Approach `_ + `Pattern Matching: The Gestalt Approach `_ Discussion of a similar algorithm by John W. Ratcliff and D. E. Metzener. This - was published in `Dr. Dobb's Journal `_ in July, 1988. + was published in Dr. Dobb's Journal in July, 1988. .. _sequence-matcher: -SequenceMatcher Objects +SequenceMatcher objects ----------------------- The :class:`SequenceMatcher` class has this constructor: @@ -549,16 +559,6 @@ The :class:`SequenceMatcher` class has this constructor: to try :meth:`quick_ratio` or :meth:`real_quick_ratio` first to get an upper bound. - .. note:: - - Caution: The result of a :meth:`ratio` call may depend on the order of - the arguments. For instance:: - - >>> SequenceMatcher(None, 'tide', 'diet').ratio() - 0.25 - >>> SequenceMatcher(None, 'diet', 'tide').ratio() - 0.5 - .. method:: quick_ratio() @@ -586,7 +586,7 @@ are always at least as large as :meth:`~SequenceMatcher.ratio`: .. _sequencematcher-examples: -SequenceMatcher Examples +SequenceMatcher examples ------------------------ This example compares two strings, considering blanks to be "junk": @@ -637,7 +637,7 @@ If you want to know how to change the first sequence into the second, use .. _differ-objects: -Differ Objects +Differ objects -------------- Note that :class:`Differ`\ -generated deltas make no claim to be **minimal** @@ -686,7 +686,7 @@ The :class:`Differ` class has this constructor: .. _differ-examples: -Differ Example +Differ example -------------- This example compares two texts. First we set up the texts, sequences of diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 44767b5dd2d5c93..d6b728b0848dff2 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -14,7 +14,7 @@ -------------- -The :mod:`dis` module supports the analysis of CPython :term:`bytecode` by +The :mod:`!dis` module supports the analysis of CPython :term:`bytecode` by disassembling it. The CPython bytecode which this module takes as an input is defined in the file :file:`Include/opcode.h` and used by the compiler and the interpreter. @@ -38,7 +38,7 @@ interpreter. Some instructions are accompanied by one or more inline cache entries, which take the form of :opcode:`CACHE` instructions. These instructions are hidden by default, but can be shown by passing ``show_caches=True`` to - any :mod:`dis` utility. Furthermore, the interpreter now adapts the + any :mod:`!dis` utility. Furthermore, the interpreter now adapts the bytecode to specialize it for different runtime conditions. The adaptive bytecode can be shown by passing ``adaptive=True``. @@ -87,7 +87,7 @@ the following command can be used to display the disassembly of Command-line interface ---------------------- -The :mod:`dis` module can be invoked as a script from the command line: +The :mod:`!dis` module can be invoked as a script from the command line: .. code-block:: sh @@ -223,7 +223,7 @@ Example: Analysis functions ------------------ -The :mod:`dis` module also defines the following analysis functions that convert +The :mod:`!dis` module also defines the following analysis functions that convert the input directly to the desired output. They can be useful if only a single operation is being performed, so the intermediate analysis object isn't useful: @@ -585,6 +585,22 @@ operations on it as if it was a Python list. The top of the stack corresponds to generate line tracing events. +.. opcode:: NOT_TAKEN + + Do nothing code. + Used by the interpreter to record :monitoring-event:`BRANCH_LEFT` + and :monitoring-event:`BRANCH_RIGHT` events for :mod:`sys.monitoring`. + + .. versionadded:: 3.14 + + +.. opcode:: POP_ITER + + Removes the iterator from the top of the stack. + + .. versionadded:: 3.14 + + .. opcode:: POP_TOP Removes the top-of-stack item:: @@ -752,7 +768,7 @@ not have to be) the original ``STACK[-2]``. end = STACK.pop() start = STACK.pop() container = STACK.pop() - values = STACK.pop() + value = STACK.pop() container[start:end] = value .. versionadded:: 3.12 @@ -1094,14 +1110,6 @@ iterations of the loop. .. versionadded:: 3.14 -.. opcode:: LOAD_CONST_IMMORTAL (consti) - - Pushes ``co_consts[consti]`` onto the stack. - Can be used when the constant value is known to be immortal. - - .. versionadded:: 3.14 - - .. opcode:: LOAD_NAME (namei) Pushes the value associated with ``co_names[namei]`` onto the stack. @@ -1128,6 +1136,48 @@ iterations of the loop. .. versionadded:: 3.12 +.. opcode:: BUILD_TEMPLATE + + Constructs a new :class:`~string.templatelib.Template` instance from a tuple + of strings and a tuple of interpolations and pushes the resulting object + onto the stack:: + + interpolations = STACK.pop() + strings = STACK.pop() + STACK.append(_build_template(strings, interpolations)) + + .. versionadded:: 3.14 + + +.. opcode:: BUILD_INTERPOLATION (format) + + Constructs a new :class:`~string.templatelib.Interpolation` instance from a + value and its source expression and pushes the resulting object onto the + stack. + + If no conversion or format specification is present, ``format`` is set to + ``2``. + + If the low bit of ``format`` is set, it indicates that the interpolation + contains a format specification. + + If ``format >> 2`` is non-zero, it indicates that the interpolation + contains a conversion. The value of ``format >> 2`` is the conversion type + (``0`` for no conversion, ``1`` for ``!s``, ``2`` for ``!r``, and + ``3`` for ``!a``):: + + conversion = format >> 2 + if format & 1: + format_spec = STACK.pop() + else: + format_spec = None + expression = STACK.pop() + value = STACK.pop() + STACK.append(_build_interpolation(value, expression, conversion, format_spec)) + + .. versionadded:: 3.14 + + .. opcode:: BUILD_TUPLE (count) Creates a tuple consuming *count* items from the stack, and pushes the @@ -1592,7 +1642,7 @@ iterations of the loop. Pushes a ``NULL`` to the stack. Used in the call sequence to match the ``NULL`` pushed by - :opcode:`LOAD_METHOD` for non-method calls. + :opcode:`!LOAD_METHOD` for non-method calls. .. versionadded:: 3.11 @@ -1623,9 +1673,13 @@ iterations of the loop. * ``0x02`` a dictionary of keyword-only parameters' default values * ``0x04`` a tuple of strings containing parameters' annotations * ``0x08`` a tuple containing cells for free variables, making a closure + * ``0x10`` the :term:`annotate function` for the function object .. versionadded:: 3.13 + .. versionchanged:: 3.14 + Added ``0x10`` to indicate the annotate function for the function object. + .. opcode:: BUILD_SLICE (argc) @@ -1773,7 +1827,7 @@ iterations of the loop. ignore it. Before, only opcodes ``>= HAVE_ARGUMENT`` had an argument. .. versionchanged:: 3.12 - Pseudo instructions were added to the :mod:`dis` module, and for them + Pseudo instructions were added to the :mod:`!dis` module, and for them it is not true that comparison with ``HAVE_ARGUMENT`` indicates whether they use their arg. @@ -1913,14 +1967,20 @@ but are replaced by real opcodes or removed before bytecode is generated. Marks the end of the code block associated with the last ``SETUP_FINALLY``, ``SETUP_CLEANUP`` or ``SETUP_WITH``. + +.. opcode:: LOAD_CONST_IMMORTAL (consti) + + Works as :opcode:`LOAD_CONST`, but is more efficient for immortal objects. + + .. opcode:: JUMP -.. opcode:: JUMP_NO_INTERRUPT + JUMP_NO_INTERRUPT Undirected relative jump instructions which are replaced by their directed (forward/backward) counterparts by the assembler. .. opcode:: JUMP_IF_TRUE -.. opcode:: JUMP_IF_FALSE + JUMP_IF_FALSE Conditional jumps which do not impact the stack. Replaced by the sequence ``COPY 1``, ``TO_BOOL``, ``POP_JUMP_IF_TRUE/FALSE``. @@ -1936,12 +1996,6 @@ but are replaced by real opcodes or removed before bytecode is generated. This opcode is now a pseudo-instruction. -.. opcode:: LOAD_METHOD - - Optimized unbound method lookup. Emitted as a ``LOAD_ATTR`` opcode - with a flag set in the arg. - - .. _opcode_collections: Opcode collections diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst index b86fef9fd6f3106..a303afe60b7a43b 100644 --- a/Doc/library/doctest.rst +++ b/Doc/library/doctest.rst @@ -13,7 +13,7 @@ -------------- -The :mod:`doctest` module searches for pieces of text that look like interactive +The :mod:`!doctest` module searches for pieces of text that look like interactive Python sessions, and then executes those sessions to verify that they work exactly as shown. There are several common ways to use doctest: @@ -85,7 +85,7 @@ Here's a complete but small example module:: import doctest doctest.testmod() -If you run :file:`example.py` directly from the command line, :mod:`doctest` +If you run :file:`example.py` directly from the command line, :mod:`!doctest` works its magic: .. code-block:: shell-session @@ -94,7 +94,7 @@ works its magic: $ There's no output! That's normal, and it means all the examples worked. Pass -``-v`` to the script, and :mod:`doctest` prints a detailed log of what +``-v`` to the script, and :mod:`!doctest` prints a detailed log of what it's trying, and prints a summary at the end: .. code-block:: shell-session @@ -130,7 +130,7 @@ And so on, eventually ending with: Test passed. $ -That's all you need to know to start making productive use of :mod:`doctest`! +That's all you need to know to start making productive use of :mod:`!doctest`! Jump in. The following sections provide full details. Note that there are many examples of doctests in the standard Python test suite and libraries. Especially useful examples can be found in the standard test file @@ -174,7 +174,7 @@ with assorted summaries at the end. You can force verbose mode by passing ``verbose=True`` to :func:`testmod`, or prohibit it by passing ``verbose=False``. In either of those cases, -``sys.argv`` is not examined by :func:`testmod` (so passing ``-v`` or not +:data:`sys.argv` is not examined by :func:`testmod` (so passing ``-v`` or not has no effect). There is also a command line shortcut for running :func:`testmod`, see section @@ -231,7 +231,7 @@ documentation:: As with :func:`testmod`, :func:`testfile` won't display anything unless an example fails. If an example does fail, then the failing example(s) and the cause(s) of the failure(s) are printed to stdout, using the same format as -:func:`testmod`. +:func:`!testmod`. By default, :func:`testfile` looks for files in the calling module's directory. See section :ref:`doctest-basic-api` for a description of the optional arguments @@ -252,7 +252,7 @@ For more information on :func:`testfile`, see section :ref:`doctest-basic-api`. Command-line Usage ------------------ -The :mod:`doctest` module can be invoked as a script from the command line: +The :mod:`!doctest` module can be invoked as a script from the command line: .. code-block:: bash @@ -311,6 +311,13 @@ Which Docstrings Are Examined? The module docstring, and all function, class and method docstrings are searched. Objects imported into the module are not searched. +.. currentmodule:: None + +.. attribute:: module.__test__ + :no-typesetting: + +.. currentmodule:: doctest + In addition, there are cases when you want tests to be part of a module but not part of the help text, which requires that the tests not be included in the docstring. Doctest looks for a module-level variable called ``__test__`` and uses it to locate other @@ -343,6 +350,13 @@ searches them recursively for docstrings, which are then scanned for tests. Any classes found are recursively searched similarly, to test docstrings in their contained methods and nested classes. +.. note:: + + ``doctest`` can only automatically discover classes and functions that are + defined at the module level or inside other classes. + + Since nested classes and functions only exist when an outer function + is called, they cannot be discovered. Define them outside to make them visible. .. _doctest-finding-examples: @@ -436,7 +450,7 @@ The fine print: What's the Execution Context? ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -By default, each time :mod:`doctest` finds a docstring to test, it uses a +By default, each time :mod:`!doctest` finds a docstring to test, it uses a *shallow copy* of :mod:`!M`'s globals, so that running tests doesn't change the module's real globals, and so that one test in :mod:`!M` can't leave behind crumbs that accidentally allow another test to work. This means examples can @@ -533,7 +547,7 @@ Some details you should read once, but won't need to remember: * The interactive shell omits the traceback header line for some :exc:`SyntaxError`\ s. But doctest uses the traceback header line to distinguish exceptions from non-exceptions. So in the rare case where you need - to test a :exc:`SyntaxError` that omits the traceback header, you will need to + to test a :exc:`!SyntaxError` that omits the traceback header, you will need to manually add the traceback header line to your test example. .. index:: single: ^ (caret); marker @@ -716,7 +730,7 @@ The second group of options controls how test failures are reported: There is also a way to register new option flag names, though this isn't -useful unless you intend to extend :mod:`doctest` internals via subclassing: +useful unless you intend to extend :mod:`!doctest` internals via subclassing: .. function:: register_optionflag(name) @@ -819,7 +833,7 @@ disabling an option via ``-`` in a directive can be useful. Warnings ^^^^^^^^ -:mod:`doctest` is serious about requiring exact matches in expected output. If +:mod:`!doctest` is serious about requiring exact matches in expected output. If even a single character doesn't match, the test fails. This will probably surprise you a few times, as you learn exactly what Python does and doesn't guarantee about output. For example, when printing a set, Python doesn't @@ -860,15 +874,15 @@ The :const:`ELLIPSIS` directive gives a nice approach for the last example: Floating-point numbers are also subject to small output variations across -platforms, because Python defers to the platform C library for float formatting, -and C libraries vary widely in quality here. :: +platforms, because Python defers to the platform C library for some +floating-point calculations, and C libraries vary widely in quality here. :: - >>> 1./7 # risky - 0.14285714285714285 - >>> print(1./7) # safer - 0.142857142857 - >>> print(round(1./7, 6)) # much safer - 0.142857 + >>> 1000**0.1 # risky + 1.9952623149688797 + >>> round(1000**0.1, 9) # safer + 1.995262315 + >>> print(f'{1000**0.1:.4f}') # much safer + 1.9953 Numbers of the form ``I/2.**J`` are safe across all platforms, and I often contrive doctest examples to produce numbers of that form:: @@ -938,13 +952,13 @@ and :ref:`doctest-simple-testfile`. Optional argument *verbose* prints lots of stuff if true, and prints only failures if false; by default, or if ``None``, it's true if and only if ``'-v'`` - is in ``sys.argv``. + is in :data:`sys.argv`. Optional argument *report* prints a summary at the end when true, else prints nothing at the end. In verbose mode, the summary is detailed, else the summary is very brief (in fact, empty if all tests passed). - Optional argument *optionflags* (default value 0) takes the + Optional argument *optionflags* (default value ``0``) takes the :ref:`bitwise OR ` of option flags. See section :ref:`doctest-options`. @@ -1021,7 +1035,7 @@ Unittest API ------------ As your collection of doctest'ed modules grows, you'll want a way to run all -their doctests systematically. :mod:`doctest` provides two functions that can +their doctests systematically. :mod:`!doctest` provides two functions that can be used to create :mod:`unittest` test suites from modules and text files containing doctests. To integrate with :mod:`unittest` test discovery, include a :ref:`load_tests ` function in your test module:: @@ -1045,7 +1059,7 @@ from text files and modules with doctests: The returned :class:`unittest.TestSuite` is to be run by the unittest framework and runs the interactive examples in each file. If an example in any file - fails, then the synthesized unit test fails, and a :exc:`failureException` + fails, then the synthesized unit test fails, and a :exc:`~unittest.TestCase.failureException` exception is raised showing the name of the file containing the test and a (sometimes approximate) line number. If all the examples in a file are skipped, then the synthesized unit test is also marked as skipped. @@ -1078,13 +1092,14 @@ from text files and modules with doctests: Optional argument *setUp* specifies a set-up function for the test suite. This is called before running the tests in each file. The *setUp* function - will be passed a :class:`DocTest` object. The setUp function can access the - test globals as the *globs* attribute of the test passed. + will be passed a :class:`DocTest` object. The *setUp* function can access the + test globals as the :attr:`~DocTest.globs` attribute of the test passed. Optional argument *tearDown* specifies a tear-down function for the test suite. This is called after running the tests in each file. The *tearDown* - function will be passed a :class:`DocTest` object. The setUp function can - access the test globals as the *globs* attribute of the test passed. + function will be passed a :class:`DocTest` object. The *tearDown* function can + access the test globals as the :attr:`~DocTest.globs` attribute of the test + passed. Optional argument *globs* is a dictionary containing the initial global variables for the tests. A new copy of this dictionary is created for each @@ -1111,11 +1126,12 @@ from text files and modules with doctests: Convert doctest tests for a module to a :class:`unittest.TestSuite`. The returned :class:`unittest.TestSuite` is to be run by the unittest framework - and runs each doctest in the module. If any of the doctests fail, then the - synthesized unit test fails, and a :exc:`failureException` exception is raised + and runs each doctest in the module. + Each docstring is run as a separate unit test. + If any of the doctests fail, then the synthesized unit test fails, + and a :exc:`unittest.TestCase.failureException` exception is raised showing the name of the file containing the test and a (sometimes approximate) line number. If all the examples in a docstring are skipped, then the - synthesized unit test is also marked as skipped. Optional argument *module* provides the module to be tested. It can be a module object or a (possibly dotted) module name. If not specified, the module calling @@ -1123,7 +1139,7 @@ from text files and modules with doctests: Optional argument *globs* is a dictionary containing the initial global variables for the tests. A new copy of this dictionary is created for each - test. By default, *globs* is a new empty dictionary. + test. By default, *globs* is the module's :attr:`~module.__dict__`. Optional argument *extraglobs* specifies an extra set of global variables, which is merged into *globs*. By default, no extra globals are used. @@ -1132,7 +1148,7 @@ from text files and modules with doctests: drop-in replacement) that is used to extract doctests from the module. Optional arguments *setUp*, *tearDown*, and *optionflags* are the same as for - function :func:`DocFileSuite` above. + function :func:`DocFileSuite` above, but they are called for each docstring. This function uses the same search technique as :func:`testmod`. @@ -1140,12 +1156,6 @@ from text files and modules with doctests: :func:`DocTestSuite` returns an empty :class:`unittest.TestSuite` if *module* contains no docstrings instead of raising :exc:`ValueError`. -.. exception:: failureException - - When doctests which have been converted to unit tests by :func:`DocFileSuite` - or :func:`DocTestSuite` fail, this exception is raised showing the name of - the file containing the test and a (sometimes approximate) line number. - Under the covers, :func:`DocTestSuite` creates a :class:`unittest.TestSuite` out of :class:`!doctest.DocTestCase` instances, and :class:`!DocTestCase` is a subclass of :class:`unittest.TestCase`. :class:`!DocTestCase` isn't documented @@ -1158,21 +1168,21 @@ of :class:`!DocTestCase`. So both ways of creating a :class:`unittest.TestSuite` run instances of :class:`!DocTestCase`. This is important for a subtle reason: when you run -:mod:`doctest` functions yourself, you can control the :mod:`doctest` options in -use directly, by passing option flags to :mod:`doctest` functions. However, if -you're writing a :mod:`unittest` framework, :mod:`unittest` ultimately controls +:mod:`!doctest` functions yourself, you can control the :mod:`!doctest` options in +use directly, by passing option flags to :mod:`!doctest` functions. However, if +you're writing a :mod:`unittest` framework, :mod:`!unittest` ultimately controls when and how tests get run. The framework author typically wants to control -:mod:`doctest` reporting options (perhaps, e.g., specified by command line -options), but there's no way to pass options through :mod:`unittest` to -:mod:`doctest` test runners. +:mod:`!doctest` reporting options (perhaps, e.g., specified by command line +options), but there's no way to pass options through :mod:`!unittest` to +:mod:`!doctest` test runners. -For this reason, :mod:`doctest` also supports a notion of :mod:`doctest` +For this reason, :mod:`!doctest` also supports a notion of :mod:`!doctest` reporting flags specific to :mod:`unittest` support, via this function: .. function:: set_unittest_reportflags(flags) - Set the :mod:`doctest` reporting flags to use. + Set the :mod:`!doctest` reporting flags to use. Argument *flags* takes the :ref:`bitwise OR ` of option flags. See section :ref:`doctest-options`. Only "reporting flags" can be used. @@ -1181,12 +1191,12 @@ reporting flags specific to :mod:`unittest` support, via this function: :mod:`unittest`: the :meth:`!runTest` method of :class:`!DocTestCase` looks at the option flags specified for the test case when the :class:`!DocTestCase` instance was constructed. If no reporting flags were specified (which is the - typical and expected case), :mod:`!doctest`'s :mod:`unittest` reporting flags are + typical and expected case), :mod:`!doctest`'s :mod:`!unittest` reporting flags are :ref:`bitwise ORed ` into the option flags, and the option flags so augmented are passed to the :class:`DocTestRunner` instance created to run the doctest. If any reporting flags were specified when the :class:`!DocTestCase` instance was constructed, :mod:`!doctest`'s - :mod:`unittest` reporting flags are ignored. + :mod:`!unittest` reporting flags are ignored. The value of the :mod:`unittest` reporting flags in effect before the function was called is returned by the function. @@ -1279,7 +1289,7 @@ DocTest Objects .. attribute:: filename The name of the file that this :class:`DocTest` was extracted from; or - ``None`` if the filename is unknown, or if the :class:`DocTest` was not + ``None`` if the filename is unknown, or if the :class:`!DocTest` was not extracted from a file. @@ -1419,10 +1429,10 @@ DocTestFinder objects The globals for each :class:`DocTest` is formed by combining *globs* and *extraglobs* (bindings in *extraglobs* override bindings in *globs*). A new - shallow copy of the globals dictionary is created for each :class:`DocTest`. - If *globs* is not specified, then it defaults to the module's *__dict__*, if - specified, or ``{}`` otherwise. If *extraglobs* is not specified, then it - defaults to ``{}``. + shallow copy of the globals dictionary is created for each :class:`!DocTest`. + If *globs* is not specified, then it defaults to the module's + :attr:`~module.__dict__`, if specified, or ``{}`` otherwise. + If *extraglobs* is not specified, then it defaults to ``{}``. .. _doctest-doctestparser: @@ -1446,7 +1456,7 @@ DocTestParser objects :class:`DocTest` object. *globs*, *name*, *filename*, and *lineno* are attributes for the new - :class:`DocTest` object. See the documentation for :class:`DocTest` for more + :class:`!DocTest` object. See the documentation for :class:`DocTest` for more information. @@ -1461,7 +1471,7 @@ DocTestParser objects Divide the given string into examples and intervening text, and return them as a list of alternating :class:`Example`\ s and strings. Line numbers for the - :class:`Example`\ s are 0-based. The optional argument *name* is a name + :class:`!Example`\ s are 0-based. The optional argument *name* is a name identifying this string, and is only used for error messages. @@ -1501,7 +1511,7 @@ DocTestRunner objects :class:`OutputChecker`. This comparison may be customized with a number of option flags; see section :ref:`doctest-options` for more information. If the option flags are insufficient, then the comparison may also be customized by - passing a subclass of :class:`OutputChecker` to the constructor. + passing a subclass of :class:`!OutputChecker` to the constructor. The test runner's display output can be controlled in two ways. First, an output function can be passed to :meth:`run`; this function will be called @@ -1540,7 +1550,7 @@ DocTestRunner objects output; it should not be called directly. *example* is the example about to be processed. *test* is the test - *containing example*. *out* is the output function that was passed to + containing *example*. *out* is the output function that was passed to :meth:`DocTestRunner.run`. @@ -1889,7 +1899,7 @@ There are two exceptions that may be raised by :class:`DebugRunner` instances: Soapbox ------- -As mentioned in the introduction, :mod:`doctest` has grown to have three primary +As mentioned in the introduction, :mod:`!doctest` has grown to have three primary uses: #. Checking examples in docstrings. @@ -1907,7 +1917,7 @@ this that needs to be learned---it may not be natural at first. Examples should add genuine value to the documentation. A good example can often be worth many words. If done with care, the examples will be invaluable for your users, and will pay back the time it takes to collect them many times over as the years go -by and things change. I'm still amazed at how often one of my :mod:`doctest` +by and things change. I'm still amazed at how often one of my :mod:`!doctest` examples stops working after a "harmless" change. Doctest also makes an excellent tool for regression testing, especially if you @@ -1940,7 +1950,7 @@ several options for organizing tests: containing test cases for the named topics. These functions can be included in the same file as the module, or separated out into a separate test file. -* Define a ``__test__`` dictionary mapping from regression test topics to +* Define a :attr:`~module.__test__` dictionary mapping from regression test topics to docstrings containing test cases. When you have placed your tests in a module, the module can itself be the test diff --git a/Doc/library/email.charset.rst b/Doc/library/email.charset.rst index 6875af2be49d7a9..76a57031862c85b 100644 --- a/Doc/library/email.charset.rst +++ b/Doc/library/email.charset.rst @@ -19,7 +19,7 @@ registry and several convenience methods for manipulating this registry. Instances of :class:`Charset` are used in several other modules within the :mod:`email` package. -Import this class from the :mod:`email.charset` module. +Import this class from the :mod:`!email.charset` module. .. class:: Charset(input_charset=DEFAULT_CHARSET) @@ -164,7 +164,7 @@ Import this class from the :mod:`email.charset` module. This method allows you to compare two :class:`Charset` instances for inequality. -The :mod:`email.charset` module also provides the following functions for adding +The :mod:`!email.charset` module also provides the following functions for adding new entries to the global character set, alias, and codec registries: diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index 4285c436e8da80e..5754c2b65b239f9 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -181,7 +181,7 @@ Here are the methods of the :class:`Message` class: :meth:`set_payload` instead. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by :meth:`~email.message.EmailMessage.set_content` and the related ``make`` and ``add`` methods. @@ -224,7 +224,7 @@ Here are the methods of the :class:`Message` class: ASCII charset. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by :meth:`~email.message.EmailMessage.get_content` and :meth:`~email.message.EmailMessage.iter_parts`. @@ -236,7 +236,7 @@ Here are the methods of the :class:`Message` class: the message's default character set; see :meth:`set_charset` for details. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by :meth:`~email.message.EmailMessage.set_content`. @@ -265,9 +265,9 @@ Here are the methods of the :class:`Message` class: using that :mailheader:`Content-Transfer-Encoding` and is not modified. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by the *charset* parameter of the - :meth:`email.emailmessage.EmailMessage.set_content` method. + :meth:`email.message.EmailMessage.set_content` method. .. method:: get_charset() @@ -276,7 +276,7 @@ Here are the methods of the :class:`Message` class: message's payload. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class it always returns + :class:`~email.message.EmailMessage` class it always returns ``None``. @@ -486,7 +486,7 @@ Here are the methods of the :class:`Message` class: search instead of :mailheader:`Content-Type`. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by the *params* property of the individual header objects returned by the header access methods. @@ -524,7 +524,7 @@ Here are the methods of the :class:`Message` class: to ``False``. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by the *params* property of the individual header objects returned by the header access methods. @@ -579,7 +579,7 @@ Here are the methods of the :class:`Message` class: header is also added. This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is + :class:`~email.message.EmailMessage` class its functionality is replaced by the ``make_`` and ``add_`` methods. diff --git a/Doc/library/email.encoders.rst b/Doc/library/email.encoders.rst index 9c8c8c9234ed7a7..1a9a1cad3a619e8 100644 --- a/Doc/library/email.encoders.rst +++ b/Doc/library/email.encoders.rst @@ -25,7 +25,7 @@ is especially true for :mimetype:`image/\*` and :mimetype:`text/\*` type message containing binary data. The :mod:`email` package provides some convenient encoders in its -:mod:`~email.encoders` module. These encoders are actually used by the +:mod:`!encoders` module. These encoders are actually used by the :class:`~email.mime.audio.MIMEAudio` and :class:`~email.mime.image.MIMEImage` class constructors to provide default encodings. All encoder functions take exactly one argument, the message object to encode. They usually extract the diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst index 689e7397cbcf1f2..2f7c9140cfcbe55 100644 --- a/Doc/library/email.errors.rst +++ b/Doc/library/email.errors.rst @@ -8,7 +8,7 @@ -------------- -The following exception classes are defined in the :mod:`email.errors` module: +The following exception classes are defined in the :mod:`!email.errors` module: .. exception:: MessageError() diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst index a3132d02687bc9e..6f4f813a0f84d8c 100644 --- a/Doc/library/email.generator.rst +++ b/Doc/library/email.generator.rst @@ -232,7 +232,7 @@ a formatted string representation of a message object. For more detail, see :mod:`email.message`. -The :mod:`email.generator` module also provides a derived class, +The :mod:`!email.generator` module also provides a derived class, :class:`DecodedGenerator`, which is like the :class:`Generator` base class, except that non-\ :mimetype:`text` parts are not serialized, but are instead represented in the output stream by a string derived from a template filled diff --git a/Doc/library/email.header.rst b/Doc/library/email.header.rst index 219fad0d2f6745a..e7e21d036e07deb 100644 --- a/Doc/library/email.header.rst +++ b/Doc/library/email.header.rst @@ -28,13 +28,13 @@ transferred using only 7-bit ASCII characters, so a slew of RFCs have been written describing how to encode email containing non-ASCII characters into :rfc:`2822`\ -compliant format. These RFCs include :rfc:`2045`, :rfc:`2046`, :rfc:`2047`, and :rfc:`2231`. The :mod:`email` package supports these standards -in its :mod:`email.header` and :mod:`email.charset` modules. +in its :mod:`!email.header` and :mod:`email.charset` modules. If you want to include non-ASCII characters in your email headers, say in the :mailheader:`Subject` or :mailheader:`To` fields, you should use the :class:`Header` class and assign the field in the :class:`~email.message.Message` object to an instance of :class:`Header` instead of using a string for the header -value. Import the :class:`Header` class from the :mod:`email.header` module. +value. Import the :class:`Header` class from the :mod:`!email.header` module. For example:: >>> from email.message import Message @@ -170,7 +170,7 @@ Here is the :class:`Header` class description: This method allows you to compare two :class:`Header` instances for inequality. -The :mod:`email.header` module also provides the following convenient functions. +The :mod:`!email.header` module also provides the following convenient functions. .. function:: decode_header(header) @@ -178,16 +178,36 @@ The :mod:`email.header` module also provides the following convenient functions. Decode a message header value without converting the character set. The header value is in *header*. - This function returns a list of ``(decoded_string, charset)`` pairs containing - each of the decoded parts of the header. *charset* is ``None`` for non-encoded - parts of the header, otherwise a lower case string containing the name of the - character set specified in the encoded string. + For historical reasons, this function may return either: - Here's an example:: + 1. A list of pairs containing each of the decoded parts of the header, + ``(decoded_bytes, charset)``, where *decoded_bytes* is always an instance of + :class:`bytes`, and *charset* is either: + + - A lower case string containing the name of the character set specified. + + - ``None`` for non-encoded parts of the header. + + 2. A list of length 1 containing a pair ``(string, None)``, where + *string* is always an instance of :class:`str`. + + An :exc:`email.errors.HeaderParseError` may be raised when certain decoding + errors occur (e.g. a base64 decoding exception). + + Here are examples: >>> from email.header import decode_header >>> decode_header('=?iso-8859-1?q?p=F6stal?=') [(b'p\xf6stal', 'iso-8859-1')] + >>> decode_header('unencoded_string') + [('unencoded_string', None)] + >>> decode_header('bar =?utf-8?B?ZsOzbw==?=') + [(b'bar ', None), (b'f\xc3\xb3o', 'utf-8')] + + .. note:: + + This function exists for backwards compatibility only. For + new code, we recommend using :class:`email.headerregistry.HeaderRegistry`. .. function:: make_header(decoded_seq, maxlinelen=None, header_name=None, continuation_ws=' ') @@ -203,3 +223,7 @@ The :mod:`email.header` module also provides the following convenient functions. :class:`Header` instance. Optional *maxlinelen*, *header_name*, and *continuation_ws* are as in the :class:`Header` constructor. + .. note:: + + This function exists for backwards compatibility only, and is + not recommended for use in new code. diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index 7f8044932fae993..c4570b16c3c3db5 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -96,9 +96,10 @@ headers. ``kwds`` is a dictionary containing one pre-initialized key, ``defects``. ``defects`` is an empty list. The parse method should append any detected defects to this list. On return, the ``kwds`` dictionary *must* contain - values for at least the keys ``decoded`` and ``defects``. ``decoded`` - should be the string value for the header (that is, the header value fully - decoded to unicode). The parse method should assume that *string* may + values for at least the keys ``decoded``, ``defects`` and ``parse_tree``. + ``decoded`` should be the string value for the header (that is, the header + value fully decoded to unicode). ``parse_tree`` is set to the parse tree obtained + from parsing the header. The parse method should assume that *string* may contain content-transfer-encoded parts, but should correctly handle all valid unicode characters as well so that it can parse un-encoded header values. @@ -294,7 +295,7 @@ variant, :attr:`~.BaseHeader.max_count` is set to 1. ``inline`` and ``attachment`` are the only valid values in common use. -.. class:: ContentTransferEncoding +.. class:: ContentTransferEncodingHeader Handles the :mailheader:`Content-Transfer-Encoding` header. diff --git a/Doc/library/email.iterators.rst b/Doc/library/email.iterators.rst index 090981d84b4de3d..ed300cdb30fdd69 100644 --- a/Doc/library/email.iterators.rst +++ b/Doc/library/email.iterators.rst @@ -10,7 +10,7 @@ Iterating over a message object tree is fairly easy with the :meth:`Message.walk ` method. The -:mod:`email.iterators` module provides some useful higher level iterations over +:mod:`!email.iterators` module provides some useful higher level iterations over message object trees. diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst index 71d6e321f387bcb..f6908d2e6e9748e 100644 --- a/Doc/library/email.message.rst +++ b/Doc/library/email.message.rst @@ -14,7 +14,7 @@ .. versionadded:: 3.6 [1]_ The central class in the :mod:`email` package is the :class:`EmailMessage` -class, imported from the :mod:`email.message` module. It is the base class for +class, imported from the :mod:`!email.message` module. It is the base class for the :mod:`email` object model. :class:`EmailMessage` provides the core functionality for setting and querying header fields, for accessing message bodies, and for creating or modifying structured messages. @@ -57,7 +57,7 @@ message objects. :class:`~email.policy.default` policy, which follows the rules of the email RFCs except for line endings (instead of the RFC mandated ``\r\n``, it uses the Python standard ``\n`` line endings). For more information see the - :mod:`~email.policy` documentation. + :mod:`~email.policy` documentation. [2]_ .. method:: as_string(unixfrom=False, maxheaderlen=None, policy=None) @@ -749,3 +749,9 @@ message objects. .. [1] Originally added in 3.4 as a :term:`provisional module `. Docs for legacy message class moved to :ref:`compat32_message`. + +.. [2] The :class:`EmailMessage` class requires a policy that provides a + ``content_manager`` attribute for content management methods like + ``set_content()`` and ``get_content()`` to work. The legacy + :const:`~email.policy.compat32` policy does not support these methods + and should not be used with :class:`EmailMessage`. diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst index 439b5c8f34b65a3..6a67bf7c8e555dd 100644 --- a/Doc/library/email.parser.rst +++ b/Doc/library/email.parser.rst @@ -116,7 +116,7 @@ Here is the API for the :class:`BytesFeedParser`: Works like :class:`BytesFeedParser` except that the input to the :meth:`~BytesFeedParser.feed` method must be a string. This is of limited utility, since the only way for such a message to be valid is for it to - contain only ASCII text or, if :attr:`~email.policy.Policy.utf8` is + contain only ASCII text or, if :attr:`~email.policy.EmailPolicy.utf8` is ``True``, no binary attachments. .. versionchanged:: 3.3 Added the *policy* keyword. @@ -125,10 +125,10 @@ Here is the API for the :class:`BytesFeedParser`: Parser API ^^^^^^^^^^ -The :class:`BytesParser` class, imported from the :mod:`email.parser` module, +The :class:`BytesParser` class, imported from the :mod:`!email.parser` module, provides an API that can be used to parse a message when the complete contents of the message are available in a :term:`bytes-like object` or file. The -:mod:`email.parser` module also provides :class:`Parser` for parsing strings, +:mod:`!email.parser` module also provides :class:`Parser` for parsing strings, and header-only parsers, :class:`BytesHeaderParser` and :class:`HeaderParser`, which can be used if you're only interested in the headers of the message. :class:`BytesHeaderParser` and :class:`HeaderParser` @@ -155,11 +155,11 @@ message body, instead setting the payload to the raw body. Read all the data from the binary file-like object *fp*, parse the resulting bytes, and return the message object. *fp* must support - both the :meth:`~io.IOBase.readline` and the :meth:`~io.IOBase.read` + both the :meth:`~io.IOBase.readline` and the :meth:`~io.BufferedIOBase.read` methods. The bytes contained in *fp* must be formatted as a block of :rfc:`5322` - (or, if :attr:`~email.policy.Policy.utf8` is ``True``, :rfc:`6532`) + (or, if :attr:`~email.policy.EmailPolicy.utf8` is ``True``, :rfc:`6532`) style headers and header continuation lines, optionally preceded by an envelope header. The header block is terminated either by the end of the data or by a blank line. Following the header block is the body of the diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index 6b997ee784f6e44..fef064114ecf1ba 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -602,7 +602,7 @@ The header objects and their attributes are described in This concrete :class:`Policy` is the backward compatibility policy. It replicates the behavior of the email package in Python 3.2. The - :mod:`~email.policy` module also defines an instance of this class, + :mod:`!policy` module also defines an instance of this class, :const:`compat32`, that is used as the default policy. Thus the default behavior of the email package is to maintain compatibility with Python 3.2. @@ -662,6 +662,13 @@ The header objects and their attributes are described in An instance of :class:`Compat32`, providing backward compatibility with the behavior of the email package in Python 3.2. + .. note:: + + The :const:`compat32` policy should not be used as a policy for + :class:`~email.message.EmailMessage` objects, and should only be used + to serialize messages that were created using the :const:`compat32` + policy. + .. rubric:: Footnotes diff --git a/Doc/library/email.rst b/Doc/library/email.rst index 66c42e4a5008eef..03ac1783be08bdb 100644 --- a/Doc/library/email.rst +++ b/Doc/library/email.rst @@ -12,10 +12,10 @@ -------------- -The :mod:`email` package is a library for managing email messages. It is +The :mod:`!email` package is a library for managing email messages. It is specifically *not* designed to do any sending of email messages to SMTP (:rfc:`2821`), NNTP, or other servers; those are functions of modules such as -:mod:`smtplib`. The :mod:`email` package attempts to be as +:mod:`smtplib`. The :mod:`!email` package attempts to be as RFC-compliant as possible, supporting :rfc:`5322` and :rfc:`6532`, as well as such MIME-related RFCs as :rfc:`2045`, :rfc:`2046`, :rfc:`2047`, :rfc:`2183`, and :rfc:`2231`. @@ -68,7 +68,7 @@ high level structure in question, and not the details of how those structures are represented. Since MIME content types are used widely in modern internet software (not just email), this will be a familiar concept to many programmers. -The following sections describe the functionality of the :mod:`email` package. +The following sections describe the functionality of the :mod:`!email` package. We start with the :mod:`~email.message` object model, which is the primary interface an application will use, and follow that with the :mod:`~email.parser` and :mod:`~email.generator` components. Then we cover the @@ -102,7 +102,7 @@ compatibility reasons. :class:`~email.message.EmailMessage`/:class:`~email.policy.EmailPolicy` API. -Contents of the :mod:`email` package documentation: +Contents of the :mod:`!email` package documentation: .. toctree:: diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst index 611549604fda15e..e0d2c19a3b0737a 100644 --- a/Doc/library/email.utils.rst +++ b/Doc/library/email.utils.rst @@ -8,7 +8,7 @@ -------------- -There are a couple of useful utilities provided in the :mod:`email.utils` +There are a couple of useful utilities provided in the :mod:`!email.utils` module: .. function:: localtime(dt=None) diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst index fa102c4a0801036..e0d77229b11802f 100644 --- a/Doc/library/ensurepip.rst +++ b/Doc/library/ensurepip.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`ensurepip` package provides support for bootstrapping the ``pip`` +The :mod:`!ensurepip` package provides support for bootstrapping the ``pip`` installer into an existing Python installation or virtual environment. This bootstrapping approach reflects the fact that ``pip`` is an independent project with its own release cycle, and the latest available stable version @@ -30,6 +30,8 @@ when creating a virtual environment) or after explicitly uninstalling needed to bootstrap ``pip`` are included as internal parts of the package. +.. include:: ../includes/optional-module.rst + .. seealso:: :ref:`installing-index` @@ -40,7 +42,9 @@ when creating a virtual environment) or after explicitly uninstalling .. include:: ../includes/wasm-mobile-notavail.rst -Command line interface +.. _ensurepip-cli: + +Command-line interface ---------------------- .. program:: ensurepip @@ -95,7 +99,7 @@ Providing both of the script selection options will trigger an exception. Module API ---------- -:mod:`ensurepip` exposes two functions for programmatic use: +:mod:`!ensurepip` exposes two functions for programmatic use: .. function:: version() diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index c9b2c7d76b67466..88f2254061af2d6 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -61,7 +61,7 @@ are not normal Python classes. See --------------- -Module Contents +Module contents --------------- :class:`EnumType` @@ -153,6 +153,12 @@ Module Contents Return a list of all power-of-two integers contained in a flag. + :func:`enum.bin` + + Like built-in :func:`bin`, except negative values are represented in + two's complement, and the leading bit always indicates sign + (``0`` implies positive, ``1`` implies negative). + .. versionadded:: 3.6 ``Flag``, ``IntFlag``, ``auto`` .. versionadded:: 3.11 ``StrEnum``, ``EnumCheck``, ``ReprEnum``, ``FlagBoundary``, ``property``, ``member``, ``nonmember``, ``global_enum``, ``show_flag_values`` @@ -160,7 +166,7 @@ Module Contents --------------- -Data Types +Data types ---------- @@ -175,6 +181,10 @@ Data Types final *enum*, as well as creating the enum members, properly handling duplicates, providing iteration over the enum class, etc. + .. versionadded:: 3.11 + + Before 3.11 ``EnumType`` was called ``EnumMeta``, which is still available as an alias. + .. method:: EnumType.__call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) This method is called in two different ways: @@ -206,7 +216,7 @@ Data Types >>> Color.RED.value in Color True - .. versionchanged:: 3.12 + .. versionchanged:: 3.12 Before Python 3.12, a ``TypeError`` is raised if a non-Enum-member is used in a containment check. @@ -235,7 +245,7 @@ Data Types .. method:: EnumType.__len__(cls) - Returns the number of member in *cls*:: + Returns the number of members in *cls*:: >>> len(Color) 3 @@ -251,20 +261,6 @@ Data Types >>> list(reversed(Color)) [, , ] - .. method:: EnumType._add_alias_ - - Adds a new name as an alias to an existing member. Raises a - :exc:`NameError` if the name is already assigned to a different member. - - .. method:: EnumType._add_value_alias_ - - Adds a new value as an alias to an existing member. Raises a - :exc:`ValueError` if the value is already linked with a different member. - - .. versionadded:: 3.11 - - Before 3.11 ``EnumType`` was called ``EnumMeta``, which is still available as an alias. - .. class:: Enum @@ -311,6 +307,28 @@ Data Types No longer used, kept for backward compatibility. (class attribute, removed during class creation). + The :attr:`~Enum._order_` attribute can be provided to help keep Python 2 / Python 3 code in sync. + It will be checked against the actual order of the enumeration and raise an error if the two do not match:: + + >>> class Color(Enum): + ... _order_ = 'RED GREEN BLUE' + ... RED = 1 + ... BLUE = 3 + ... GREEN = 2 + ... + Traceback (most recent call last): + ... + TypeError: member order does not match _order_: + ['RED', 'BLUE', 'GREEN'] + ['RED', 'GREEN', 'BLUE'] + + .. note:: + + In Python 2 code the :attr:`~Enum._order_` attribute is necessary as definition + order is lost before it can be recorded. + + .. versionadded:: 3.6 + .. attribute:: Enum._ignore_ ``_ignore_`` is only used during creation and is removed from the @@ -320,12 +338,15 @@ Data Types names will also be removed from the completed enumeration. See :ref:`TimePeriod ` for an example. + .. versionadded:: 3.7 + .. method:: Enum.__dir__(self) Returns ``['__class__', '__doc__', '__module__', 'name', 'value']`` and any public methods defined on *self.__class__*:: - >>> from datetime import date + >>> from enum import Enum + >>> import datetime as dt >>> class Weekday(Enum): ... MONDAY = 1 ... TUESDAY = 2 @@ -336,7 +357,7 @@ Data Types ... SUNDAY = 7 ... @classmethod ... def today(cls): - ... print('today is %s' % cls(date.today().isoweekday()).name) + ... print(f'today is {cls(dt.date.today().isoweekday()).name}') ... >>> dir(Weekday.SATURDAY) ['__class__', '__doc__', '__eq__', '__hash__', '__module__', 'name', 'today', 'value'] @@ -349,9 +370,18 @@ Data Types :last_values: A list of the previous values. A *staticmethod* that is used to determine the next value returned by - :class:`auto`:: + :class:`auto`. - >>> from enum import auto + .. note:: + For standard :class:`Enum` classes the next value chosen is the highest + value seen incremented by one. + + For :class:`Flag` classes the next value chosen will be the next highest + power-of-two. + + This method may be overridden, e.g.:: + + >>> from enum import auto, Enum >>> class PowersOfThree(Enum): ... @staticmethod ... def _generate_next_value_(name, start, count, last_values): @@ -362,6 +392,10 @@ Data Types >>> PowersOfThree.SECOND.value 9 + .. versionadded:: 3.6 + .. versionchanged:: 3.13 + Prior versions would use the last seen value instead of the highest value. + .. method:: Enum.__init__(self, *args, **kwds) By default, does nothing. If multiple values are given in the member @@ -383,7 +417,7 @@ Data Types A *classmethod* for looking up values not found in *cls*. By default it does nothing, but can be overridden to implement custom search behavior:: - >>> from enum import StrEnum + >>> from enum import auto, StrEnum >>> class Build(StrEnum): ... DEBUG = auto() ... OPTIMIZED = auto() @@ -400,6 +434,8 @@ Data Types >>> Build('deBUG') + .. versionadded:: 3.6 + .. method:: Enum.__new__(cls, *args, **kwds) By default, doesn't exist. If specified, either in the enum class @@ -422,6 +458,7 @@ Data Types Returns the string used for *repr()* calls. By default, returns the *Enum* name, member name, and value, but can be overridden:: + >>> from enum import auto, Enum >>> class OtherStyle(Enum): ... ALTERNATE = auto() ... OTHER = auto() @@ -438,6 +475,7 @@ Data Types Returns the string used for *str()* calls. By default, returns the *Enum* name and member name, but can be overridden:: + >>> from enum import auto, Enum >>> class OtherStyle(Enum): ... ALTERNATE = auto() ... OTHER = auto() @@ -453,6 +491,7 @@ Data Types Returns the string used for *format()* and *f-string* calls. By default, returns :meth:`__str__` return value, but can be overridden:: + >>> from enum import auto, Enum >>> class OtherStyle(Enum): ... ALTERNATE = auto() ... OTHER = auto() @@ -470,6 +509,31 @@ Data Types .. versionchanged:: 3.12 Added :ref:`enum-dataclass-support` + .. method:: Enum._add_alias_ + + Adds a new name as an alias to an existing member:: + + >>> Color.RED._add_alias_("ERROR") + >>> Color.ERROR + + + Raises a :exc:`NameError` if the name is already assigned to a different member. + + .. versionadded:: 3.13 + + .. method:: Enum._add_value_alias_ + + Adds a new value as an alias to an existing member:: + + >>> Color.RED._add_value_alias_(42) + >>> Color(42) + + + | Raises a :exc:`ValueError` if the value is already linked with a different member. + | See :ref:`multi-value-enum` for an example. + + .. versionadded:: 3.13 + .. class:: IntEnum @@ -504,16 +568,31 @@ Data Types .. class:: StrEnum - ``StrEnum`` is the same as :class:`Enum`, but its members are also strings and can be used - in most of the same places that a string can be used. The result of any string - operation performed on or with a *StrEnum* member is not part of the enumeration. + *StrEnum* is the same as :class:`Enum`, but its members are also strings and + can be used in most of the same places that a string can be used. The result + of any string operation performed on or with a *StrEnum* member is not part + of the enumeration. + + >>> from enum import StrEnum, auto + >>> class Color(StrEnum): + ... RED = 'r' + ... GREEN = 'g' + ... BLUE = 'b' + ... UNKNOWN = auto() + ... + >>> Color.RED + + >>> Color.UNKNOWN + + >>> str(Color.UNKNOWN) + 'unknown' .. note:: There are places in the stdlib that check for an exact :class:`str` instead of a :class:`str` subclass (i.e. ``type(unknown) == str`` instead of ``isinstance(unknown, str)``), and in those locations you - will need to use ``str(StrEnum.member)``. + will need to use ``str(MyStrEnum.MY_MEMBER)``. .. note:: @@ -555,7 +634,7 @@ Data Types >>> white in purple False - .. method:: __iter__(self): + .. method:: __iter__(self) Returns all contained non-alias members:: @@ -566,7 +645,7 @@ Data Types .. versionadded:: 3.11 - .. method:: __len__(self): + .. method:: __len__(self) Returns number of members in flag:: @@ -577,7 +656,7 @@ Data Types .. versionadded:: 3.11 - .. method:: __bool__(self): + .. method:: __bool__(self) Returns *True* if any members in flag, *False* otherwise:: @@ -614,7 +693,7 @@ Data Types >>> purple ^ Color.GREEN - .. method:: __invert__(self): + .. method:: __invert__(self) Returns all the flags in *type(self)* that are not in *self*:: @@ -850,6 +929,8 @@ Data Types --------------- +.. _enum-dunder-sunder: + Supported ``__dunder__`` names """""""""""""""""""""""""""""" @@ -857,17 +938,13 @@ Supported ``__dunder__`` names items. It is only available on the class. :meth:`~Enum.__new__`, if specified, must create and return the enum members; -it is also a very good idea to set the member's :attr:`!_value_` appropriately. +it is also a very good idea to set the member's :attr:`~Enum._value_` appropriately. Once all the members are created it is no longer used. Supported ``_sunder_`` names """""""""""""""""""""""""""" -- :meth:`~EnumType._add_alias_` -- adds a new name as an alias to an existing - member. -- :meth:`~EnumType._add_value_alias_` -- adds a new value as an alias to an - existing member. - :attr:`~Enum._name_` -- name of the member - :attr:`~Enum._value_` -- value of the member; can be set in ``__new__`` - :meth:`~Enum._missing_` -- a lookup function used when a value is not found; @@ -877,16 +954,14 @@ Supported ``_sunder_`` names from the final class - :attr:`~Enum._order_` -- no longer used, kept for backward compatibility (class attribute, removed during class creation) + - :meth:`~Enum._generate_next_value_` -- used to get an appropriate value for an enum member; may be overridden - .. note:: - - For standard :class:`Enum` classes the next value chosen is the highest - value seen incremented by one. - - For :class:`Flag` classes the next value chosen will be the next highest - power-of-two. +- :meth:`~Enum._add_alias_` -- adds a new name as an alias to an existing + member. +- :meth:`~Enum._add_value_alias_` -- adds a new value as an alias to an + existing member. - While ``_sunder_`` names are generally reserved for the further development of the :class:`Enum` class and can not be used, some are explicitly allowed: @@ -900,7 +975,7 @@ Supported ``_sunder_`` names --------------- -Utilities and Decorators +Utilities and decorators ------------------------ .. class:: auto @@ -913,12 +988,13 @@ Utilities and Decorators the member's name. Care must be taken if mixing *auto()* with manually specified values. - *auto* instances are only resolved when at the top level of an assignment: + *auto* instances are only resolved when at the top level of an assignment, either by + itself or as part of a tuple: * ``FIRST = auto()`` will work (auto() is replaced with ``1``); * ``SECOND = auto(), -2`` will work (auto is replaced with ``2``, so ``2, -2`` is used to create the ``SECOND`` enum member; - * ``THREE = [auto(), -3]`` will *not* work (``, -3`` is used to + * ``THREE = [auto(), -3]`` will *not* work (``[, -3]`` is used to create the ``THREE`` enum member) .. versionchanged:: 3.11.1 @@ -1000,6 +1076,20 @@ Utilities and Decorators .. versionadded:: 3.11 +.. function:: bin(num, max_bits=None) + + Like built-in :func:`bin`, except negative values are represented in + two's complement, and the leading bit always indicates sign + (``0`` implies positive, ``1`` implies negative). + + >>> import enum + >>> enum.bin(10) + '0b0 1010' + >>> enum.bin(~10) # ~10 is -11 + '0b1 0101' + + .. versionadded:: 3.11 + --------------- Notes diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index bb72032891ea987..aae9bad1a9f3e86 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -204,10 +204,16 @@ The following exceptions are the exceptions that are usually raised. assignment fails. (When an object does not support attribute references or attribute assignments at all, :exc:`TypeError` is raised.) - The :attr:`name` and :attr:`obj` attributes can be set using keyword-only - arguments to the constructor. When set they represent the name of the attribute - that was attempted to be accessed and the object that was accessed for said - attribute, respectively. + The optional *name* and *obj* keyword-only arguments + set the corresponding attributes: + + .. attribute:: name + + The name of the attribute that was attempted to be accessed. + + .. attribute:: obj + + The object that was accessed for the named attribute. .. versionchanged:: 3.10 Added the :attr:`name` and :attr:`obj` attributes. @@ -215,7 +221,7 @@ The following exceptions are the exceptions that are usually raised. .. exception:: EOFError Raised when the :func:`input` function hits an end-of-file condition (EOF) - without reading any data. (N.B.: the :meth:`io.IOBase.read` and + without reading any data. (Note: the :meth:`io.TextIOBase.read` and :meth:`io.IOBase.readline` methods return an empty string when they hit EOF.) @@ -312,9 +318,11 @@ The following exceptions are the exceptions that are usually raised. unqualified names. The associated value is an error message that includes the name that could not be found. - The :attr:`name` attribute can be set using a keyword-only argument to the - constructor. When set it represent the name of the variable that was attempted - to be accessed. + The optional *name* keyword-only argument sets the attribute: + + .. attribute:: name + + The name of the variable that was attempted to be accessed. .. versionchanged:: 3.10 Added the :attr:`name` attribute. @@ -382,7 +390,7 @@ The following exceptions are the exceptions that are usually raised. The corresponding error message, as provided by the operating system. It is formatted by the C - functions :c:func:`perror` under POSIX, and :c:func:`FormatMessage` + functions :c:func:`!perror` under POSIX, and :c:func:`!FormatMessage` under Windows. .. attribute:: filename @@ -398,7 +406,7 @@ The following exceptions are the exceptions that are usually raised. .. versionchanged:: 3.3 :exc:`EnvironmentError`, :exc:`IOError`, :exc:`WindowsError`, :exc:`socket.error`, :exc:`select.error` and - :exc:`mmap.error` have been merged into :exc:`OSError`, and the + :exc:`!mmap.error` have been merged into :exc:`OSError`, and the constructor may return a subclass. .. versionchanged:: 3.4 @@ -590,7 +598,7 @@ The following exceptions are the exceptions that are usually raised. handled, the Python interpreter exits; no stack traceback is printed. The constructor accepts the same optional argument passed to :func:`sys.exit`. If the value is an integer, it specifies the system exit status (passed to - C's :c:func:`exit` function); if it is ``None``, the exit status is zero; if + C's :c:func:`!exit` function); if it is ``None``, the exit status is zero; if it has another type (such as a string), the object's value is printed and the exit status is one. @@ -727,8 +735,8 @@ depending on the system error code. .. attribute:: characters_written - An integer containing the number of characters written to the stream - before it blocked. This attribute is available when using the + An integer containing the number of **bytes** written to the stream + before it blocked. This attribute is available when using the buffered I/O classes from the :mod:`io` module. .. exception:: ChildProcessError @@ -882,6 +890,9 @@ The following exceptions are used as warning categories; see the Base class for warnings about dubious syntax. + This warning is typically emitted when compiling Python source code, and usually won't be reported + when running already compiled code. + .. exception:: RuntimeWarning @@ -960,6 +971,15 @@ their subgroups based on the types of the contained exceptions. raises a :exc:`TypeError` if any contained exception is not an :exc:`Exception` subclass. + Exception groups are :ref:`generic ` over the type of their + contained exceptions. + + .. impl-detail:: + + The ``excs`` parameter may be any sequence, but lists and tuples are + specifically processed more efficiently here. For optimal performance, + pass a tuple as ``excs``. + .. attribute:: message The ``msg`` argument to the constructor. This is a read-only attribute. @@ -1048,7 +1068,7 @@ their subgroups based on the types of the contained exceptions. subclasses that need a different constructor signature need to override that rather than :meth:`~object.__init__`. For example, the following defines an exception group subclass which accepts an exit_code and - and constructs the group's message from it. :: + constructs the group's message from it. :: class Errors(ExceptionGroup): def __new__(cls, errors, exit_code): diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index 5058b85bffb15c6..f81e6bf5810f86a 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -90,7 +90,7 @@ An error will be printed instead of the stack. Additionally, some compilers do not support :term:`CPython's ` implementation of C stack dumps. As a result, a different error may be printed -instead of the stack, even if the the operating system supports dumping stacks. +instead of the stack, even if the operating system supports dumping stacks. .. note:: @@ -228,6 +228,41 @@ handler: Fatal Python error: Segmentation fault Current thread 0x00007fb899f39700 (most recent call first): - File "/home/python/cpython/Lib/ctypes/__init__.py", line 486 in string_at + File "/opt/python/Lib/ctypes/__init__.py", line 486 in string_at File "", line 1 in + + Current thread's C stack trace (most recent call first): + Binary file "/opt/python/python", at _Py_DumpStack+0x42 [0x5b27f7d7147e] + Binary file "/opt/python/python", at +0x32dcbd [0x5b27f7d85cbd] + Binary file "/opt/python/python", at +0x32df8a [0x5b27f7d85f8a] + Binary file "/usr/lib/libc.so.6", at +0x3def0 [0x77b73226bef0] + Binary file "/usr/lib/libc.so.6", at +0x17ef9c [0x77b7323acf9c] + Binary file "/opt/python/build/lib.linux-x86_64-3.14/_ctypes.cpython-314d-x86_64-linux-gnu.so", at +0xcdf6 [0x77b7315dddf6] + Binary file "/usr/lib/libffi.so.8", at +0x7976 [0x77b73158f976] + Binary file "/usr/lib/libffi.so.8", at +0x413c [0x77b73158c13c] + Binary file "/usr/lib/libffi.so.8", at ffi_call+0x12e [0x77b73158ef0e] + Binary file "/opt/python/build/lib.linux-x86_64-3.14/_ctypes.cpython-314d-x86_64-linux-gnu.so", at +0x15a33 [0x77b7315e6a33] + Binary file "/opt/python/build/lib.linux-x86_64-3.14/_ctypes.cpython-314d-x86_64-linux-gnu.so", at +0x164fa [0x77b7315e74fa] + Binary file "/opt/python/build/lib.linux-x86_64-3.14/_ctypes.cpython-314d-x86_64-linux-gnu.so", at +0xc624 [0x77b7315dd624] + Binary file "/opt/python/python", at _PyObject_MakeTpCall+0xce [0x5b27f7b73883] + Binary file "/opt/python/python", at +0x11bab6 [0x5b27f7b73ab6] + Binary file "/opt/python/python", at PyObject_Vectorcall+0x23 [0x5b27f7b73b04] + Binary file "/opt/python/python", at _PyEval_EvalFrameDefault+0x490c [0x5b27f7cbb302] + Binary file "/opt/python/python", at +0x2818e6 [0x5b27f7cd98e6] + Binary file "/opt/python/python", at +0x281aab [0x5b27f7cd9aab] + Binary file "/opt/python/python", at PyEval_EvalCode+0xc5 [0x5b27f7cd9ba3] + Binary file "/opt/python/python", at +0x255957 [0x5b27f7cad957] + Binary file "/opt/python/python", at +0x255ab4 [0x5b27f7cadab4] + Binary file "/opt/python/python", at _PyEval_EvalFrameDefault+0x6c3e [0x5b27f7cbd634] + Binary file "/opt/python/python", at +0x2818e6 [0x5b27f7cd98e6] + Binary file "/opt/python/python", at +0x281aab [0x5b27f7cd9aab] + Binary file "/opt/python/python", at +0x11b6e1 [0x5b27f7b736e1] + Binary file "/opt/python/python", at +0x11d348 [0x5b27f7b75348] + Binary file "/opt/python/python", at +0x11d626 [0x5b27f7b75626] + Binary file "/opt/python/python", at PyObject_Call+0x20 [0x5b27f7b7565e] + Binary file "/opt/python/python", at +0x32a67a [0x5b27f7d8267a] + Binary file "/opt/python/python", at +0x32a7f8 [0x5b27f7d827f8] + Binary file "/opt/python/python", at +0x32ac1b [0x5b27f7d82c1b] + Binary file "/opt/python/python", at Py_RunMain+0x31 [0x5b27f7d82ebe] + Segmentation fault diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst index c8ce86cc7af92c7..4a08af4f90d419a 100644 --- a/Doc/library/fcntl.rst +++ b/Doc/library/fcntl.rst @@ -2,7 +2,6 @@ ========================================================== .. module:: fcntl - :platform: Unix :synopsis: The fcntl() and ioctl() system calls. .. sectionauthor:: Jaap Vermeulen @@ -53,7 +52,7 @@ descriptor. the latter setting ``FD_CLOEXEC`` flag in addition. .. versionchanged:: 3.12 - On Linux >= 4.5, the :mod:`fcntl` module exposes the ``FICLONE`` and + On Linux >= 4.5, the :mod:`!fcntl` module exposes the ``FICLONE`` and ``FICLONERANGE`` constants, which allow to share some data of one file with another file by reflinking on some filesystems (e.g., btrfs, OCFS2, and XFS). This behavior is commonly referred to as "copy-on-write". @@ -91,7 +90,7 @@ The module defines the following functions: Perform the operation *cmd* on file descriptor *fd* (file objects providing a :meth:`~io.IOBase.fileno` method are accepted as well). The values used for *cmd* are operating system dependent, and are available as constants - in the :mod:`fcntl` module, using the same names as used in the relevant C + in the :mod:`!fcntl` module, using the same names as used in the relevant C header files. The argument *arg* can either be an integer value, a :term:`bytes-like object`, or a string. The type and size of *arg* must match the type and size of diff --git a/Doc/library/filecmp.rst b/Doc/library/filecmp.rst index abd1b8c826d1705..105aaa23071806b 100644 --- a/Doc/library/filecmp.rst +++ b/Doc/library/filecmp.rst @@ -10,11 +10,11 @@ -------------- -The :mod:`filecmp` module defines functions to compare files and directories, +The :mod:`!filecmp` module defines functions to compare files and directories, with various optional time/correctness trade-offs. For comparing files, see also the :mod:`difflib` module. -The :mod:`filecmp` module defines the following functions: +The :mod:`!filecmp` module defines the following functions: .. function:: cmp(f1, f2, shallow=True) @@ -36,9 +36,9 @@ The :mod:`filecmp` module defines the following functions: file changes. The entire cache may be cleared using :func:`clear_cache`. -.. function:: cmpfiles(dir1, dir2, common, shallow=True) +.. function:: cmpfiles(a, b, common, shallow=True) - Compare the files in the two directories *dir1* and *dir2* whose names are + Compare the files in the two directories *a* and *b* whose names are given by *common*. Returns three lists of file names: *match*, *mismatch*, diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index 12e61bc36f5db0c..ee654b7a83e203b 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -53,7 +53,7 @@ a :class:`!str` filename, and vice-versa. Finally, note that :func:`functools.lru_cache` with a *maxsize* of 32768 is used to cache the (typed) compiled regex patterns in the following -functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`. +functions: :func:`fnmatch`, :func:`fnmatchcase`, :func:`.filter`, :func:`.filterfalse`. .. function:: fnmatch(name, pat) diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst index fc7f9a6301a9153..575e90942d48b0c 100644 --- a/Doc/library/fractions.rst +++ b/Doc/library/fractions.rst @@ -11,11 +11,11 @@ -------------- -The :mod:`fractions` module provides support for rational number arithmetic. +The :mod:`!fractions` module provides support for rational number arithmetic. -A Fraction instance can be constructed from a pair of integers, from -another rational number, or from a string. +A Fraction instance can be constructed from a pair of rational numbers, from +a single number, or from a string. .. index:: single: as_integer_ratio() @@ -25,8 +25,8 @@ another rational number, or from a string. The first version requires that *numerator* and *denominator* are instances of :class:`numbers.Rational` and returns a new :class:`Fraction` instance - with value ``numerator/denominator``. If *denominator* is ``0``, it - raises a :exc:`ZeroDivisionError`. + with a value equal to ``numerator/denominator``. + If *denominator* is zero, it raises a :exc:`ZeroDivisionError`. The second version requires that *number* is an instance of :class:`numbers.Rational` or has the :meth:`!as_integer_ratio` method @@ -125,7 +125,8 @@ another rational number, or from a string. .. attribute:: denominator - Denominator of the Fraction in lowest term. + Denominator of the Fraction in lowest terms. + Guaranteed to be positive. .. method:: as_integer_ratio() @@ -142,7 +143,7 @@ another rational number, or from a string. .. versionadded:: 3.12 - .. classmethod:: from_float(flt) + .. classmethod:: from_float(f) Alternative constructor which only accepts instances of :class:`float` or :class:`numbers.Integral`. Beware that diff --git a/Doc/library/frameworks.rst b/Doc/library/frameworks.rst index 15ceeec9c255edb..f8e2f6bb18cb1cb 100644 --- a/Doc/library/frameworks.rst +++ b/Doc/library/frameworks.rst @@ -1,18 +1,13 @@ +:orphan: + .. _frameworks: ****************** -Program Frameworks +Program frameworks ****************** -The modules described in this chapter are frameworks that will largely dictate -the structure of your program. Currently the modules described here are all -oriented toward writing command-line interfaces. - -The full list of modules described in this chapter is: - - -.. toctree:: +This chapter is no longer maintained, and the modules it contained have been moved to their respective topical documentation. - turtle.rst - cmd.rst - shlex.rst +* :mod:`cmd` — :doc:`Command Line Interface Libraries <./cmdlinelibs>` +* :mod:`shlex` — :doc:`Unix Specific Services <./unix>` +* :mod:`turtle` — :doc:`Graphical User Interfaces with Tk <./tk>` diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index bb15322067245e8..9cbb387f40fb4c3 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -23,7 +23,7 @@ The default encoding is UTF-8, following :rfc:`2640`. .. include:: ../includes/wasm-notavail.rst -Here's a sample session using the :mod:`ftplib` module:: +Here's a sample session using the :mod:`!ftplib` module:: >>> from ftplib import FTP >>> ftp = FTP('ftp.us.debian.org') # connect to host, default port diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 7e367a0f2b6b253..7953375ee150df8 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -54,7 +54,7 @@ are always available. They are listed here in alphabetical order. .. |func-bytearray| replace:: ``bytearray()`` .. |func-bytes| replace:: ``bytes()`` -.. function:: abs(x) +.. function:: abs(number, /) Return the absolute value of a number. The argument may be an integer, a floating-point number, or an object implementing @@ -62,7 +62,7 @@ are always available. They are listed here in alphabetical order. If the argument is a complex number, its magnitude is returned. -.. function:: aiter(async_iterable) +.. function:: aiter(async_iterable, /) Return an :term:`asynchronous iterator` for an :term:`asynchronous iterable`. Equivalent to calling ``x.__aiter__()``. @@ -71,7 +71,7 @@ are always available. They are listed here in alphabetical order. .. versionadded:: 3.10 -.. function:: all(iterable) +.. function:: all(iterable, /) Return ``True`` if all elements of the *iterable* are true (or if the iterable is empty). Equivalent to:: @@ -83,8 +83,8 @@ are always available. They are listed here in alphabetical order. return True -.. awaitablefunction:: anext(async_iterator) - anext(async_iterator, default) +.. awaitablefunction:: anext(async_iterator, /) + anext(async_iterator, default, /) When awaited, return the next item from the given :term:`asynchronous iterator`, or *default* if given and the iterator is exhausted. @@ -99,7 +99,7 @@ are always available. They are listed here in alphabetical order. .. versionadded:: 3.10 -.. function:: any(iterable) +.. function:: any(iterable, /) Return ``True`` if any element of the *iterable* is true. If the iterable is empty, return ``False``. Equivalent to:: @@ -111,7 +111,7 @@ are always available. They are listed here in alphabetical order. return False -.. function:: ascii(object) +.. function:: ascii(object, /) As :func:`repr`, return a string containing a printable representation of an object, but escape the non-ASCII characters in the string returned by @@ -119,10 +119,10 @@ are always available. They are listed here in alphabetical order. similar to that returned by :func:`repr` in Python 2. -.. function:: bin(x) +.. function:: bin(integer, /) Convert an integer number to a binary string prefixed with "0b". The result - is a valid Python expression. If *x* is not a Python :class:`int` object, it + is a valid Python expression. If *integer* is not a Python :class:`int` object, it has to define an :meth:`~object.__index__` method that returns an integer. Some examples: @@ -138,6 +138,8 @@ are always available. They are listed here in alphabetical order. >>> f'{14:#b}', f'{14:b}' ('0b1110', '1110') + See also :func:`enum.bin` to represent negative values as twos-complement. + See also :func:`format` for more information. @@ -183,8 +185,7 @@ are always available. They are listed here in alphabetical order. .. _func-bytearray: .. class:: bytearray(source=b'') - bytearray(source, encoding) - bytearray(source, encoding, errors) + bytearray(source, encoding, errors='strict') :noindex: Return a new array of bytes. The :class:`bytearray` class is a mutable @@ -215,8 +216,7 @@ are always available. They are listed here in alphabetical order. .. _func-bytes: .. class:: bytes(source=b'') - bytes(source, encoding) - bytes(source, encoding, errors) + bytes(source, encoding, errors='strict') :noindex: Return a new "bytes" object which is an immutable sequence of integers in @@ -231,7 +231,7 @@ are always available. They are listed here in alphabetical order. See also :ref:`binaryseq`, :ref:`typebytes`, and :ref:`bytes-methods`. -.. function:: callable(object) +.. function:: callable(object, /) Return :const:`True` if the *object* argument appears callable, :const:`False` if not. If this returns ``True``, it is still possible that a @@ -244,14 +244,14 @@ are always available. They are listed here in alphabetical order. in Python 3.2. -.. function:: chr(i) +.. function:: chr(codepoint, /) - Return the string representing a character whose Unicode code point is the - integer *i*. For example, ``chr(97)`` returns the string ``'a'``, while + Return the string representing a character with the specified Unicode code point. + For example, ``chr(97)`` returns the string ``'a'``, while ``chr(8364)`` returns the string ``'€'``. This is the inverse of :func:`ord`. The valid range for the argument is from 0 through 1,114,111 (0x10FFFF in - base 16). :exc:`ValueError` will be raised if *i* is outside that range. + base 16). :exc:`ValueError` will be raised if it is outside that range. .. decorator:: classmethod @@ -336,8 +336,8 @@ are always available. They are listed here in alphabetical order. ``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false) or ``2`` (docstrings are removed too). - This function raises :exc:`SyntaxError` if the compiled source is invalid, - and :exc:`ValueError` if the source contains null bytes. + This function raises :exc:`SyntaxError` or :exc:`ValueError` if the compiled + source is invalid. If you want to parse Python code into its AST representation, see :func:`ast.parse`. @@ -458,7 +458,7 @@ are always available. They are listed here in alphabetical order. deprecated; it should only be passed as a single positional argument. -.. function:: delattr(object, name) +.. function:: delattr(object, name, /) This is a relative of :func:`setattr`. The arguments are an object and a string. The string must be the name of one of the object's attributes. The @@ -468,20 +468,20 @@ are always available. They are listed here in alphabetical order. .. _func-dict: -.. class:: dict(**kwarg) - dict(mapping, **kwarg) - dict(iterable, **kwarg) +.. class:: dict(**kwargs) + dict(mapping, /, **kwargs) + dict(iterable, /, **kwargs) :noindex: Create a new dictionary. The :class:`dict` object is the dictionary class. - See :class:`dict` and :ref:`typesmapping` for documentation about this class. + See also :ref:`typesmapping` for documentation about this class. For other containers see the built-in :class:`list`, :class:`set`, and :class:`tuple` classes, as well as the :mod:`collections` module. .. function:: dir() - dir(object) + dir(object, /) Without arguments, return the list of names in the current local scope. With an argument, attempt to return a list of valid attributes for that object. @@ -541,7 +541,7 @@ are always available. They are listed here in alphabetical order. class. -.. function:: divmod(a, b) +.. function:: divmod(a, b, /) Take two (non-complex) numbers as arguments and return a pair of numbers consisting of their quotient and remainder when using integer division. With @@ -597,18 +597,20 @@ are always available. They are listed here in alphabetical order. .. warning:: This function executes arbitrary code. Calling it with - user-supplied input may lead to security vulnerabilities. + untrusted user-supplied input will lead to security vulnerabilities. - The *expression* argument is parsed and evaluated as a Python expression - (technically speaking, a condition list) using the *globals* and *locals* - mappings as global and local namespace. If the *globals* dictionary is - present and does not contain a value for the key ``__builtins__``, a + The *source* argument is parsed and evaluated as a Python expression + (technically speaking, an :ref:`expression list `) + using the *globals* and *locals* mappings as global and local namespace. + If the *globals* dictionary is present and does not contain a value for the + key ``__builtins__``, a reference to the dictionary of the built-in module :mod:`builtins` is - inserted under that key before *expression* is parsed. That way you can - control what builtins are available to the executed code by inserting your - own ``__builtins__`` dictionary into *globals* before passing it to - :func:`eval`. If the *locals* mapping is omitted it defaults to the - *globals* dictionary. If both mappings are omitted, the expression is + inserted under that key before *source* is parsed. + Overriding ``__builtins__`` can be used to restrict or change the available + names, but this is **not** a security mechanism: the executed code can + still access all builtins. + If the *locals* mapping is omitted it defaults to the + *globals* dictionary. If both mappings are omitted, the source is executed with the *globals* and *locals* in the environment where :func:`eval` is called. Note, *eval()* will only have access to the :term:`nested scopes ` (non-locals) in the enclosing @@ -621,6 +623,9 @@ are always available. They are listed here in alphabetical order. >>> eval('x+1') 2 + >>> eval("1, 2") + (1, 2) + This function can also be used to execute arbitrary code objects (such as those created by :func:`compile`). In this case, pass a code object instead of a string. If the code object has been compiled with ``'exec'`` as the @@ -634,7 +639,7 @@ are always available. They are listed here in alphabetical order. If the given source is a string, then leading and trailing spaces and tabs are stripped. - See :func:`ast.literal_eval` for a function that can safely evaluate strings + See :func:`ast.literal_eval` for a function to evaluate strings with expressions containing only literals. .. audit-event:: exec code_object eval @@ -658,7 +663,7 @@ are always available. They are listed here in alphabetical order. .. warning:: This function executes arbitrary code. Calling it with - user-supplied input may lead to security vulnerabilities. + untrusted user-supplied input will lead to security vulnerabilities. This function supports dynamic execution of Python code. *source* must be either a string or a code object. If it is a string, the string is parsed as @@ -689,9 +694,10 @@ are always available. They are listed here in alphabetical order. If the *globals* dictionary does not contain a value for the key ``__builtins__``, a reference to the dictionary of the built-in module - :mod:`builtins` is inserted under that key. That way you can control what - builtins are available to the executed code by inserting your own - ``__builtins__`` dictionary into *globals* before passing it to :func:`exec`. + :mod:`builtins` is inserted under that key. + Overriding ``__builtins__`` can be used to restrict or change the available + names, but this is **not** a security mechanism: the executed code can + still access all builtins. The *closure* argument specifies a closure--a tuple of cellvars. It's only valid when the *object* is a code object containing @@ -729,7 +735,7 @@ are always available. They are listed here in alphabetical order. described for the :func:`locals` builtin. -.. function:: filter(function, iterable) +.. function:: filter(function, iterable, /) Construct an iterator from those elements of *iterable* for which *function* is true. *iterable* may be either a sequence, a container which @@ -823,7 +829,7 @@ are always available. They are listed here in alphabetical order. single: __format__ single: string; format() (built-in function) -.. function:: format(value, format_spec="") +.. function:: format(value, format_spec="", /) Convert a *value* to a "formatted" representation, as controlled by *format_spec*. The interpretation of *format_spec* will depend on the type @@ -846,11 +852,11 @@ are always available. They are listed here in alphabetical order. .. _func-frozenset: -.. class:: frozenset(iterable=set()) +.. class:: frozenset(iterable=(), /) :noindex: Return a new :class:`frozenset` object, optionally with elements taken from - *iterable*. ``frozenset`` is a built-in class. See :class:`frozenset` and + *iterable*. :class:`frozenset` is a built-in class. See also :ref:`types-set` for documentation about this class. For other containers see the built-in :class:`set`, :class:`list`, @@ -858,8 +864,8 @@ are always available. They are listed here in alphabetical order. module. -.. function:: getattr(object, name) - getattr(object, name, default) +.. function:: getattr(object, name, /) + getattr(object, name, default, /) Return the value of the named attribute of *object*. *name* must be a string. If the string is the name of one of the object's attributes, the result is the @@ -883,7 +889,7 @@ are always available. They are listed here in alphabetical order. regardless of where the function is called. -.. function:: hasattr(object, name) +.. function:: hasattr(object, name, /) The arguments are an object and a string. The result is ``True`` if the string is the name of one of the object's attributes, ``False`` if not. (This @@ -891,7 +897,7 @@ are always available. They are listed here in alphabetical order. raises an :exc:`AttributeError` or not.) -.. function:: hash(object) +.. function:: hash(object, /) Return the hash value of the object (if it has one). Hash values are integers. They are used to quickly compare dictionary keys during a @@ -926,10 +932,10 @@ are always available. They are listed here in alphabetical order. signatures for callables are now more comprehensive and consistent. -.. function:: hex(x) +.. function:: hex(integer, /) Convert an integer number to a lowercase hexadecimal string prefixed with - "0x". If *x* is not a Python :class:`int` object, it has to define an + "0x". If *integer* is not a Python :class:`int` object, it has to define an :meth:`~object.__index__` method that returns an integer. Some examples: >>> hex(255) @@ -958,7 +964,7 @@ are always available. They are listed here in alphabetical order. :meth:`float.hex` method. -.. function:: id(object) +.. function:: id(object, /) Return the "identity" of an object. This is an integer which is guaranteed to be unique and constant for this object during its lifetime. @@ -971,7 +977,7 @@ are always available. They are listed here in alphabetical order. .. function:: input() - input(prompt) + input(prompt, /) If the *prompt* argument is present, it is written to standard output without a trailing newline. The function then reads a line from input, converts it @@ -1071,7 +1077,7 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.14 :func:`int` no longer delegates to the :meth:`~object.__trunc__` method. -.. function:: isinstance(object, classinfo) +.. function:: isinstance(object, classinfo, /) Return ``True`` if the *object* argument is an instance of the *classinfo* argument, or of a (direct, indirect, or :term:`virtual `) of *classinfo*. A @@ -1102,19 +1108,19 @@ are always available. They are listed here in alphabetical order. *classinfo* can be a :ref:`types-union`. -.. function:: iter(object) - iter(object, sentinel) +.. function:: iter(iterable, /) + iter(callable, sentinel, /) Return an :term:`iterator` object. The first argument is interpreted very differently depending on the presence of the second argument. Without a - second argument, *object* must be a collection object which supports the + second argument, the single argument must be a collection object which supports the :term:`iterable` protocol (the :meth:`~object.__iter__` method), or it must support the sequence protocol (the :meth:`~object.__getitem__` method with integer arguments starting at ``0``). If it does not support either of those protocols, :exc:`TypeError` is raised. If the second argument, *sentinel*, is given, - then *object* must be a callable object. The iterator created in this case - will call *object* with no arguments for each call to its + then the first argument must be a callable object. The iterator created in this case + will call *callable* with no arguments for each call to its :meth:`~iterator.__next__` method; if the value returned is equal to *sentinel*, :exc:`StopIteration` will be raised, otherwise the value will be returned. @@ -1131,7 +1137,7 @@ are always available. They are listed here in alphabetical order. process_block(block) -.. function:: len(s) +.. function:: len(object, /) Return the length (the number of items) of an object. The argument may be a sequence (such as a string, bytes, tuple, list, or range) or a collection @@ -1144,8 +1150,7 @@ are always available. They are listed here in alphabetical order. .. _func-list: -.. class:: list() - list(iterable) +.. class:: list(iterable=(), /) :noindex: Rather than being a function, :class:`list` is actually a mutable @@ -1154,44 +1159,44 @@ are always available. They are listed here in alphabetical order. .. function:: locals() - Return a mapping object representing the current local symbol table, with - variable names as the keys, and their currently bound references as the - values. - - At module scope, as well as when using :func:`exec` or :func:`eval` with - a single namespace, this function returns the same namespace as - :func:`globals`. - - At class scope, it returns the namespace that will be passed to the - metaclass constructor. - - When using ``exec()`` or ``eval()`` with separate local and global - arguments, it returns the local namespace passed in to the function call. - - In all of the above cases, each call to ``locals()`` in a given frame of - execution will return the *same* mapping object. Changes made through - the mapping object returned from ``locals()`` will be visible as assigned, - reassigned, or deleted local variables, and assigning, reassigning, or - deleting local variables will immediately affect the contents of the - returned mapping object. - - In an :term:`optimized scope` (including functions, generators, and - coroutines), each call to ``locals()`` instead returns a fresh dictionary - containing the current bindings of the function's local variables and any - nonlocal cell references. In this case, name binding changes made via the - returned dict are *not* written back to the corresponding local variables - or nonlocal cell references, and assigning, reassigning, or deleting local - variables and nonlocal cell references does *not* affect the contents - of previously returned dictionaries. - - Calling ``locals()`` as part of a comprehension in a function, generator, or - coroutine is equivalent to calling it in the containing scope, except that - the comprehension's initialised iteration variables will be included. In - other scopes, it behaves as if the comprehension were running as a nested - function. - - Calling ``locals()`` as part of a generator expression is equivalent to - calling it in a nested generator function. + Return a mapping object representing the current local symbol table, with + variable names as the keys, and their currently bound references as the + values. + + At module scope, as well as when using :func:`exec` or :func:`eval` with + a single namespace, this function returns the same namespace as + :func:`globals`. + + At class scope, it returns the namespace that will be passed to the + metaclass constructor. + + When using ``exec()`` or ``eval()`` with separate local and global + arguments, it returns the local namespace passed in to the function call. + + In all of the above cases, each call to ``locals()`` in a given frame of + execution will return the *same* mapping object. Changes made through + the mapping object returned from ``locals()`` will be visible as assigned, + reassigned, or deleted local variables, and assigning, reassigning, or + deleting local variables will immediately affect the contents of the + returned mapping object. + + In an :term:`optimized scope` (including functions, generators, and + coroutines), each call to ``locals()`` instead returns a fresh dictionary + containing the current bindings of the function's local variables and any + nonlocal cell references. In this case, name binding changes made via the + returned dict are *not* written back to the corresponding local variables + or nonlocal cell references, and assigning, reassigning, or deleting local + variables and nonlocal cell references does *not* affect the contents + of previously returned dictionaries. + + Calling ``locals()`` as part of a comprehension in a function, generator, or + coroutine is equivalent to calling it in the containing scope, except that + the comprehension's initialised iteration variables will be included. In + other scopes, it behaves as if the comprehension were running as a nested + function. + + Calling ``locals()`` as part of a generator expression is equivalent to + calling it in a nested generator function. .. versionchanged:: 3.12 The behaviour of ``locals()`` in a comprehension has been updated as @@ -1220,9 +1225,9 @@ are always available. They are listed here in alphabetical order. Added the *strict* parameter. -.. function:: max(iterable, *, key=None) - max(iterable, *, default, key=None) - max(arg1, arg2, *args, key=None) +.. function:: max(iterable, /, *, key=None) + max(iterable, /, *, default, key=None) + max(arg1, arg2, /, *args, key=None) Return the largest item in an iterable or the largest of two or more arguments. @@ -1258,9 +1263,9 @@ are always available. They are listed here in alphabetical order. :ref:`typememoryview` for more information. -.. function:: min(iterable, *, key=None) - min(iterable, *, default, key=None) - min(arg1, arg2, *args, key=None) +.. function:: min(iterable, /, *, key=None) + min(iterable, /, *, default, key=None) + min(arg1, arg2, /, *args, key=None) Return the smallest item in an iterable or the smallest of two or more arguments. @@ -1288,8 +1293,8 @@ are always available. They are listed here in alphabetical order. The *key* can be ``None``. -.. function:: next(iterator) - next(iterator, default) +.. function:: next(iterator, /) + next(iterator, default, /) Retrieve the next item from the :term:`iterator` by calling its :meth:`~iterator.__next__` method. If *default* is given, it is returned @@ -1310,10 +1315,10 @@ are always available. They are listed here in alphabetical order. :class:`object`. -.. function:: oct(x) +.. function:: oct(integer, /) Convert an integer number to an octal string prefixed with "0o". The result - is a valid Python expression. If *x* is not a Python :class:`int` object, it + is a valid Python expression. If *integer* is not a Python :class:`int` object, it has to define an :meth:`~object.__index__` method that returns an integer. For example: @@ -1562,13 +1567,19 @@ are always available. They are listed here in alphabetical order. .. versionchanged:: 3.11 The ``'U'`` mode has been removed. -.. function:: ord(c) +.. function:: ord(character, /) + + Return the ordinal value of a character. - Given a string representing one Unicode character, return an integer - representing the Unicode code point of that character. For example, + If the argument is a one-character string, return the Unicode code point + of that character. For example, ``ord('a')`` returns the integer ``97`` and ``ord('€')`` (Euro sign) returns ``8364``. This is the inverse of :func:`chr`. + If the argument is a :class:`bytes` or :class:`bytearray` object of + length 1, return its single byte value. + For example, ``ord(b'a')`` returns the integer ``97``. + .. function:: pow(base, exp, mod=None) @@ -1577,7 +1588,7 @@ are always available. They are listed here in alphabetical order. ``pow(base, exp) % mod``). The two-argument form ``pow(base, exp)`` is equivalent to using the power operator: ``base**exp``. - The arguments must have numeric types. With mixed operand types, the + When arguments are builtin numeric types with mixed operand types, the coercion rules for binary arithmetic operators apply. For :class:`int` operands, the result has the same type as the operands (after coercion) unless the second argument is negative; in that case, all arguments are @@ -1729,15 +1740,15 @@ are always available. They are listed here in alphabetical order. .. _func-range: -.. class:: range(stop) - range(start, stop, step=1) +.. class:: range(stop, /) + range(start, stop, step=1, /) :noindex: Rather than being a function, :class:`range` is actually an immutable sequence type, as documented in :ref:`typesseq-range` and :ref:`typesseq`. -.. function:: repr(object) +.. function:: repr(object, /) Return a string containing a printable representation of an object. For many types, this function makes an attempt to return a string that would yield an @@ -1761,9 +1772,9 @@ are always available. They are listed here in alphabetical order. return f"Person('{self.name}', {self.age})" -.. function:: reversed(seq) +.. function:: reversed(object, /) - Return a reverse :term:`iterator`. *seq* must be an object which has + Return a reverse :term:`iterator`. The argument must be an object which has a :meth:`~object.__reversed__` method or supports the sequence protocol (the :meth:`~object.__len__` method and the :meth:`~object.__getitem__` method with integer arguments starting at ``0``). @@ -1797,12 +1808,11 @@ are always available. They are listed here in alphabetical order. .. _func-set: -.. class:: set() - set(iterable) +.. class:: set(iterable=(), /) :noindex: Return a new :class:`set` object, optionally with elements taken from - *iterable*. ``set`` is a built-in class. See :class:`set` and + *iterable*. :class:`set` is a built-in class. See also :ref:`types-set` for documentation about this class. For other containers see the built-in :class:`frozenset`, :class:`list`, @@ -1810,7 +1820,7 @@ are always available. They are listed here in alphabetical order. module. -.. function:: setattr(object, name, value) +.. function:: setattr(object, name, value, /) This is the counterpart of :func:`getattr`. The arguments are an object, a string, and an arbitrary value. The string may name an existing attribute or a @@ -1832,27 +1842,27 @@ are always available. They are listed here in alphabetical order. :func:`setattr`. -.. class:: slice(stop) - slice(start, stop, step=None) +.. class:: slice(stop, /) + slice(start, stop, step=None, /) Return a :term:`slice` object representing the set of indices specified by ``range(start, stop, step)``. The *start* and *step* arguments default to ``None``. + Slice objects are also generated when :ref:`slicing syntax ` + is used. For example: ``a[start:stop:step]`` or ``a[start:stop, i]``. + + See :func:`itertools.islice` for an alternate version that returns an + :term:`iterator`. + .. attribute:: slice.start - .. attribute:: slice.stop - .. attribute:: slice.step + slice.stop + slice.step - Slice objects have read-only data attributes :attr:`!start`, - :attr:`!stop`, and :attr:`!step` which merely return the argument - values (or their default). They have no other explicit functionality; + These read-only attributes are set to the argument values + (or their default). They have no other explicit functionality; however, they are used by NumPy and other third-party packages. - Slice objects are also generated when extended indexing syntax is used. For - example: ``a[start:stop:step]`` or ``a[start:stop, i]``. See - :func:`itertools.islice` for an alternate version that returns an - :term:`iterator`. - .. versionchanged:: 3.12 Slice objects are now :term:`hashable` (provided :attr:`~slice.start`, :attr:`~slice.stop`, and :attr:`~slice.step` are hashable). @@ -1885,7 +1895,7 @@ are always available. They are listed here in alphabetical order. the same data with other ordering tools such as :func:`max` that rely on a different underlying method. Implementing all six comparisons also helps avoid confusion for mixed type comparisons which can call - reflected the :meth:`~object.__gt__` method. + the reflected :meth:`~object.__gt__` method. For sorting examples and a brief sorting tutorial, see :ref:`sortinghowto`. @@ -1938,8 +1948,10 @@ are always available. They are listed here in alphabetical order. single: string; str() (built-in function) .. _func-str: -.. class:: str(object='') - str(object=b'', encoding='utf-8', errors='strict') +.. class:: str(*, encoding='utf-8', errors='strict') + str(object) + str(object, encoding, errors='strict') + str(object, *, errors) :noindex: Return a :class:`str` version of *object*. See :func:`str` for details. @@ -1972,7 +1984,7 @@ are always available. They are listed here in alphabetical order. .. class:: super() - super(type, object_or_type=None) + super(type, object_or_type=None, /) Return a proxy object that delegates method calls to a parent or sibling class of *type*. This is useful for accessing inherited methods that have @@ -2054,16 +2066,15 @@ are always available. They are listed here in alphabetical order. .. _func-tuple: -.. class:: tuple() - tuple(iterable) +.. class:: tuple(iterable=(), /) :noindex: Rather than being a function, :class:`tuple` is actually an immutable sequence type, as documented in :ref:`typesseq-tuple` and :ref:`typesseq`. -.. class:: type(object) - type(name, bases, dict, **kwds) +.. class:: type(object, /) + type(name, bases, dict, /, **kwargs) .. index:: pair: object; type @@ -2106,7 +2117,7 @@ are always available. They are listed here in alphabetical order. longer use the one-argument form to get the type of an object. .. function:: vars() - vars(object) + vars(object, /) Return the :attr:`~object.__dict__` attribute for a module, class, instance, or any other object with a :attr:`!__dict__` attribute. diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index 3a933dff057bbba..4babc246ea9d143 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -20,11 +20,11 @@ -------------- -The :mod:`functools` module is for higher-order functions: functions that act on +The :mod:`!functools` module is for higher-order functions: functions that act on or return other functions. In general, any callable object can be treated as a function for the purposes of this module. -The :mod:`functools` module defines the following functions: +The :mod:`!functools` module defines the following functions: .. decorator:: cache(user_function) @@ -42,11 +42,11 @@ The :mod:`functools` module defines the following functions: def factorial(n): return n * factorial(n-1) if n else 1 - >>> factorial(10) # no previously cached result, makes 11 recursive calls + >>> factorial(10) # no previously cached result, makes 11 recursive calls 3628800 - >>> factorial(5) # just looks up cached value result + >>> factorial(5) # no new calls, just returns the cached result 120 - >>> factorial(12) # makes two new recursive calls, the other 10 are cached + >>> factorial(12) # two new recursive calls, factorial(10) is cached 479001600 The cache is threadsafe so that the wrapped function can be used in @@ -190,7 +190,7 @@ The :mod:`functools` module defines the following functions: Note, type specificity applies only to the function's immediate arguments rather than their contents. The scalar arguments, ``Decimal(42)`` and - ``Fraction(42)`` are be treated as distinct calls with distinct results. + ``Fraction(42)`` are treated as distinct calls with distinct results. In contrast, the tuple arguments ``('answer', Decimal(42))`` and ``('answer', Fraction(42))`` are treated as equivalent. @@ -199,12 +199,18 @@ The :mod:`functools` module defines the following functions: and *typed*. This is for information purposes only. Mutating the values has no effect. + .. method:: lru_cache.cache_info() + :no-typesetting: + To help measure the effectiveness of the cache and tune the *maxsize* - parameter, the wrapped function is instrumented with a :func:`cache_info` + parameter, the wrapped function is instrumented with a :func:`!cache_info` function that returns a :term:`named tuple` showing *hits*, *misses*, *maxsize* and *currsize*. - The decorator also provides a :func:`cache_clear` function for clearing or + .. method:: lru_cache.cache_clear() + :no-typesetting: + + The decorator also provides a :func:`!cache_clear` function for clearing or invalidating the cache. The original underlying function is accessible through the @@ -284,9 +290,9 @@ The :mod:`functools` module defines the following functions: class decorator supplies the rest. This simplifies the effort involved in specifying all of the possible rich comparison operations: - The class must define one of :meth:`__lt__`, :meth:`__le__`, - :meth:`__gt__`, or :meth:`__ge__`. - In addition, the class should supply an :meth:`__eq__` method. + The class must define one of :meth:`~object.__lt__`, :meth:`~object.__le__`, + :meth:`~object.__gt__`, or :meth:`~object.__ge__`. + In addition, the class should supply an :meth:`~object.__eq__` method. For example:: @@ -403,8 +409,7 @@ The :mod:`functools` module defines the following functions: >>> remove_first_dear(message) 'Hello, dear world!' - :data:`!Placeholder` has no special treatment when used in a keyword - argument to :func:`!partial`. + :data:`!Placeholder` cannot be passed to :func:`!partial` as a keyword argument. .. versionchanged:: 3.14 Added support for :data:`Placeholder` in positional arguments. @@ -419,7 +424,7 @@ The :mod:`functools` module defines the following functions: like normal functions, are handled as descriptors). When *func* is a descriptor (such as a normal Python function, - :func:`classmethod`, :func:`staticmethod`, :func:`abstractmethod` or + :func:`classmethod`, :func:`staticmethod`, :func:`~abc.abstractmethod` or another instance of :class:`partialmethod`), calls to ``__get__`` are delegated to the underlying descriptor, and an appropriate :ref:`partial object` returned as the result. @@ -500,7 +505,10 @@ The :mod:`functools` module defines the following functions: ... print("Let me just say,", end=" ") ... print(arg) - To add overloaded implementations to the function, use the :func:`register` + .. method:: singledispatch.register() + :no-typesetting: + + To add overloaded implementations to the function, use the :func:`!register` attribute of the generic function, which can be used as a decorator. For functions annotated with types, the decorator will infer the type of the first argument automatically:: @@ -566,14 +574,14 @@ The :mod:`functools` module defines the following functions: runtime impact. To enable registering :term:`lambdas` and pre-existing functions, - the :func:`register` attribute can also be used in a functional form:: + the :func:`~singledispatch.register` attribute can also be used in a functional form:: >>> def nothing(arg, verbose=False): ... print("Nothing.") ... >>> fun.register(type(None), nothing) - The :func:`register` attribute returns the undecorated function. This + The :func:`~singledispatch.register` attribute returns the undecorated function. This enables decorator stacking, :mod:`pickling`, and the creation of unit tests for each variant independently:: @@ -651,10 +659,10 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.4 .. versionchanged:: 3.7 - The :func:`register` attribute now supports using type annotations. + The :func:`~singledispatch.register` attribute now supports using type annotations. .. versionchanged:: 3.11 - The :func:`register` attribute now supports + The :func:`~singledispatch.register` attribute now supports :class:`typing.Union` as a type annotation. @@ -664,7 +672,7 @@ The :mod:`functools` module defines the following functions: dispatch>` :term:`generic function`. To define a generic method, decorate it with the ``@singledispatchmethod`` - decorator. When defining a function using ``@singledispatchmethod``, note + decorator. When defining a method using ``@singledispatchmethod``, note that the dispatch happens on the type of the first non-*self* or non-*cls* argument:: @@ -682,7 +690,7 @@ The :mod:`functools` module defines the following functions: return not arg ``@singledispatchmethod`` supports nesting with other decorators such as - :func:`@classmethod`. Note that to allow for + :deco:`classmethod`. Note that to allow for ``dispatcher.register``, ``singledispatchmethod`` must be the *outer most* decorator. Here is the ``Negator`` class with the ``neg`` methods bound to the class, rather than an instance of the class:: @@ -704,8 +712,7 @@ The :mod:`functools` module defines the following functions: return not arg The same pattern can be used for other similar decorators: - :func:`@staticmethod`, - :func:`@abstractmethod`, and others. + :deco:`staticmethod`, :deco:`~abc.abstractmethod`, and others. .. versionadded:: 3.8 @@ -784,7 +791,7 @@ The :mod:`functools` module defines the following functions: 'Docstring' Without the use of this decorator factory, the name of the example function - would have been ``'wrapper'``, and the docstring of the original :func:`example` + would have been ``'wrapper'``, and the docstring of the original :func:`!example` would have been lost. diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 7ccb0e6bdf9406e..3b28408b03a657c 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -20,7 +20,7 @@ can be disabled by calling ``gc.disable()``. To debug a leaking program call ``gc.DEBUG_SAVEALL``, causing garbage-collected objects to be saved in gc.garbage for inspection. -The :mod:`gc` module provides the following functions: +The :mod:`!gc` module provides the following functions: .. function:: enable() @@ -40,18 +40,11 @@ The :mod:`gc` module provides the following functions: .. function:: collect(generation=2) - Perform a collection. The optional argument *generation* + With no arguments, run a full collection. The optional argument *generation* may be an integer specifying which generation to collect (from 0 to 2). A :exc:`ValueError` is raised if the generation number is invalid. The sum of collected objects and uncollectable objects is returned. - Calling ``gc.collect(0)`` will perform a GC collection on the young generation. - - Calling ``gc.collect(1)`` will perform a GC collection on the young generation - and an increment of the old generation. - - Calling ``gc.collect(2)`` or ``gc.collect()`` performs a full collection - The free lists maintained for a number of built-in types are cleared whenever a full collection or collection of the highest generation (2) is run. Not all items in some free lists may be freed due to the @@ -60,9 +53,12 @@ The :mod:`gc` module provides the following functions: The effect of calling ``gc.collect()`` while the interpreter is already performing a collection is undefined. - .. versionchanged:: 3.13 + .. versionchanged:: 3.14 ``generation=1`` performs an increment of collection. + .. versionchanged:: 3.14.5 + ``generation=1`` performs collection of the middle generation. + .. function:: set_debug(flags) @@ -78,20 +74,19 @@ The :mod:`gc` module provides the following functions: .. function:: get_objects(generation=None) - Returns a list of all objects tracked by the collector, excluding the list - returned. If *generation* is not ``None``, return only the objects as follows: - - * 0: All objects in the young generation - * 1: No objects, as there is no generation 1 (as of Python 3.13) - * 2: All objects in the old generation + returned. If *generation* is not ``None``, return only the objects tracked by + the collector that are in that generation. .. versionchanged:: 3.8 New *generation* parameter. - .. versionchanged:: 3.13 + .. versionchanged:: 3.14 Generation 1 is removed + .. versionchanged:: 3.14.5 + Generation 1 is reintroduced to maintain GC behavior from 3.13. + .. audit-event:: gc.get_objects generation gc.get_objects .. function:: get_stats() @@ -118,33 +113,33 @@ The :mod:`gc` module provides the following functions: Set the garbage collection thresholds (the collection frequency). Setting *threshold0* to zero disables collection. - The GC classifies objects into two generations depending on whether they have - survived a collection. New objects are placed in the young generation. If an - object survives a collection it is moved into the old generation. - - In order to decide when to run, the collector keeps track of the number of object + The GC classifies objects into three generations depending on how many + collection sweeps they have survived. New objects are placed in the youngest + generation (generation ``0``). If an object survives a collection it is moved + into the next older generation. Since generation ``2`` is the oldest + generation, objects in that generation remain there after a collection. In + order to decide when to run, the collector keeps track of the number object allocations and deallocations since the last collection. When the number of allocations minus the number of deallocations exceeds *threshold0*, collection - starts. For each collection, all the objects in the young generation and some - fraction of the old generation is collected. + starts. Initially only generation ``0`` is examined. If generation ``0`` has + been examined more than *threshold1* times since generation ``1`` has been + examined, then generation ``1`` is examined as well. + With the third generation, things are a bit more complicated, + see `Collecting the oldest generation `_ for more information. In the free-threaded build, the increase in process memory usage is also checked before running the collector. If the memory usage has not increased by 10% since the last collection and the net number of object allocations has not exceeded 40 times *threshold0*, the collection is not run. - The fraction of the old generation that is collected is **inversely** proportional - to *threshold1*. The larger *threshold1* is, the slower objects in the old generation - are collected. - For the default value of 10, 1% of the old generation is scanned during each collection. - - *threshold2* is ignored. + See `Garbage collector design `_ for more information. - See `Garbage collector design `_ for more information. - - .. versionchanged:: 3.13 + .. versionchanged:: 3.14 *threshold2* is ignored + .. versionchanged:: 3.14.5 + *threshold2* is restored to match Python 3.13 behavior. + .. function:: get_count() diff --git a/Doc/library/getopt.rst b/Doc/library/getopt.rst index 5c63009e22d58c4..9995f114d063e82 100644 --- a/Doc/library/getopt.rst +++ b/Doc/library/getopt.rst @@ -62,6 +62,8 @@ exception: option ``--fo`` will match as ``--foo``, but ``--f`` will not match uniquely, so :exc:`GetoptError` will be raised. + If *longopts* is a string it gets treated as a list of a single element. + The return value consists of two elements: the first is a list of ``(option, value)`` pairs; the second is the list of program arguments left after the option list was stripped (this is a trailing slice of *args*). Each diff --git a/Doc/library/getpass.rst b/Doc/library/getpass.rst index 0fb0fc88683c032..37ffbe1be55a73e 100644 --- a/Doc/library/getpass.rst +++ b/Doc/library/getpass.rst @@ -14,7 +14,7 @@ .. include:: ../includes/wasm-notavail.rst -The :mod:`getpass` module provides two functions: +The :mod:`!getpass` module provides two functions: .. function:: getpass(prompt='Password: ', stream=None, *, echo_char=None) @@ -27,9 +27,9 @@ The :mod:`getpass` module provides two functions: The *echo_char* argument controls how user input is displayed while typing. If *echo_char* is ``None`` (default), input remains hidden. Otherwise, - *echo_char* must be a printable ASCII string and each typed character - is replaced by it. For example, ``echo_char='*'`` will display - asterisks instead of the actual input. + *echo_char* must be a single printable ASCII character and each + typed character is replaced by it. For example, ``echo_char='*'`` will + display asterisks instead of the actual input. If echo free input is unavailable getpass() falls back to printing a warning message to *stream* and reading from ``sys.stdin`` and @@ -39,6 +39,14 @@ The :mod:`getpass` module provides two functions: If you call getpass from within IDLE, the input may be done in the terminal you launched IDLE from rather than the idle window itself. + .. note:: + On Unix systems, when *echo_char* is set, the terminal will be + configured to operate in + :manpage:`noncanonical mode `. + In particular, this means that line editing shortcuts such as + :kbd:`Ctrl+U` will not work and may insert unexpected characters into + the input. + .. versionchanged:: 3.14 Added the *echo_char* parameter for keyboard feedback. diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst index d0de83907eb2979..e6b3565173fe1c4 100644 --- a/Doc/library/gettext.rst +++ b/Doc/library/gettext.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`gettext` module provides internationalization (I18N) and localization +The :mod:`!gettext` module provides internationalization (I18N) and localization (L10N) services for your Python modules and applications. It supports both the GNU :program:`gettext` message catalog API and a higher level, class-based API that may be more appropriate for Python files. The interface described below allows you @@ -25,7 +25,7 @@ Some hints on localizing your Python modules and applications are also given. GNU :program:`gettext` API -------------------------- -The :mod:`gettext` module defines the following API, which is very similar to +The :mod:`!gettext` module defines the following API, which is very similar to the GNU :program:`gettext` API. If you use this API you will affect the translation of your entire application globally. Often this is what you want if your application is monolingual, with the choice of language dependent on the @@ -37,7 +37,7 @@ class-based API instead. .. function:: bindtextdomain(domain, localedir=None) Bind the *domain* to the locale directory *localedir*. More concretely, - :mod:`gettext` will look for binary :file:`.mo` files for the given domain using + :mod:`!gettext` will look for binary :file:`.mo` files for the given domain using the path (on Unix): :file:`{localedir}/{language}/LC_MESSAGES/{domain}.mo`, where *language* is searched for in the environment variables :envvar:`LANGUAGE`, :envvar:`LC_ALL`, :envvar:`LC_MESSAGES`, and :envvar:`LANG` respectively. @@ -54,19 +54,19 @@ class-based API instead. .. index:: single: _ (underscore); gettext -.. function:: gettext(message) +.. function:: gettext(message, /) Return the localized translation of *message*, based on the current global domain, language, and locale directory. This function is usually aliased as :func:`!_` in the local namespace (see examples below). -.. function:: dgettext(domain, message) +.. function:: dgettext(domain, message, /) Like :func:`.gettext`, but look the message up in the specified *domain*. -.. function:: ngettext(singular, plural, n) +.. function:: ngettext(singular, plural, n, /) Like :func:`.gettext`, but consider plural forms. If a translation is found, apply the plural formula to *n*, and return the resulting message (some @@ -81,15 +81,15 @@ class-based API instead. formulas for a variety of languages. -.. function:: dngettext(domain, singular, plural, n) +.. function:: dngettext(domain, singular, plural, n, /) Like :func:`ngettext`, but look the message up in the specified *domain*. -.. function:: pgettext(context, message) -.. function:: dpgettext(domain, context, message) -.. function:: npgettext(context, singular, plural, n) -.. function:: dnpgettext(domain, context, singular, plural, n) +.. function:: pgettext(context, message, /) +.. function:: dpgettext(domain, context, message, /) +.. function:: npgettext(context, singular, plural, n, /) +.. function:: dnpgettext(domain, context, singular, plural, n, /) Similar to the corresponding functions without the ``p`` in the prefix (that is, :func:`gettext`, :func:`dgettext`, :func:`ngettext`, :func:`dngettext`), @@ -114,7 +114,7 @@ Here's an example of typical usage for this API:: Class-based API --------------- -The class-based API of the :mod:`gettext` module gives you more flexibility and +The class-based API of the :mod:`!gettext` module gives you more flexibility and greater convenience than the GNU :program:`gettext` API. It is the recommended way of localizing your Python applications and modules. :mod:`!gettext` defines a :class:`GNUTranslations` class which implements the parsing of GNU :file:`.mo` format @@ -226,20 +226,20 @@ are the methods of :class:`!NullTranslations`: translation for a given message. - .. method:: gettext(message) + .. method:: gettext(message, /) If a fallback has been set, forward :meth:`!gettext` to the fallback. Otherwise, return *message*. Overridden in derived classes. - .. method:: ngettext(singular, plural, n) + .. method:: ngettext(singular, plural, n, /) If a fallback has been set, forward :meth:`!ngettext` to the fallback. Otherwise, return *singular* if *n* is 1; return *plural* otherwise. Overridden in derived classes. - .. method:: pgettext(context, message) + .. method:: pgettext(context, message, /) If a fallback has been set, forward :meth:`pgettext` to the fallback. Otherwise, return the translated message. Overridden in derived classes. @@ -247,7 +247,7 @@ are the methods of :class:`!NullTranslations`: .. versionadded:: 3.8 - .. method:: npgettext(context, singular, plural, n) + .. method:: npgettext(context, singular, plural, n, /) If a fallback has been set, forward :meth:`npgettext` to the fallback. Otherwise, return the translated message. Overridden in derived classes. @@ -325,7 +325,7 @@ unexpected, or if other problems occur while reading the file, instantiating a The following methods are overridden from the base class implementation: - .. method:: gettext(message) + .. method:: gettext(message, /) Look up the *message* id in the catalog and return the corresponding message string, as a Unicode string. If there is no entry in the catalog for the @@ -334,7 +334,7 @@ unexpected, or if other problems occur while reading the file, instantiating a *message* id is returned. - .. method:: ngettext(singular, plural, n) + .. method:: ngettext(singular, plural, n, /) Do a plural-forms lookup of a message id. *singular* is used as the message id for purposes of lookup in the catalog, while *n* is used to determine which @@ -355,7 +355,7 @@ unexpected, or if other problems occur while reading the file, instantiating a n) % {'num': n} - .. method:: pgettext(context, message) + .. method:: pgettext(context, message, /) Look up the *context* and *message* id in the catalog and return the corresponding message string, as a Unicode string. If there is no @@ -366,7 +366,7 @@ unexpected, or if other problems occur while reading the file, instantiating a .. versionadded:: 3.8 - .. method:: npgettext(context, singular, plural, n) + .. method:: npgettext(context, singular, plural, n, /) Do a plural-forms lookup of a message id. *singular* is used as the message id for purposes of lookup in the catalog, while *n* is used to @@ -393,7 +393,7 @@ The Catalog constructor .. index:: single: GNOME -GNOME uses a version of the :mod:`gettext` module by James Henstridge, but this +GNOME uses a version of the :mod:`!gettext` module by James Henstridge, but this version has a slightly different API. Its documented usage was:: import gettext @@ -425,7 +425,7 @@ take the following steps: #. create language-specific translations of the message catalogs -#. use the :mod:`gettext` module so that message strings are properly translated +#. use the :mod:`!gettext` module so that message strings are properly translated In order to prepare your code for I18N, you need to look at all the strings in your files. Any string that needs to be translated should be marked by wrapping @@ -473,10 +473,10 @@ supported natural language. They send back the completed language-specific versions as a :file:`.po` file that's compiled into a machine-readable :file:`.mo` binary catalog file using the :program:`msgfmt` program. The :file:`.mo` files are used by the -:mod:`gettext` module for the actual translation processing at +:mod:`!gettext` module for the actual translation processing at run-time. -How you use the :mod:`gettext` module in your code depends on whether you are +How you use the :mod:`!gettext` module in your code depends on whether you are internationalizing a single module or your entire application. The next two sections will discuss each case. diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 59ad1b07f273380..49fb430e7ae709e 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -18,23 +18,27 @@ single: - (minus); in glob-style wildcards single: . (dot); in glob-style wildcards -The :mod:`glob` module finds all the pathnames matching a specified pattern -according to the rules used by the Unix shell, although results are returned in -arbitrary order. No tilde expansion is done, but ``*``, ``?``, and character +The :mod:`!glob` module finds pathnames +using pattern matching rules similar to the Unix shell. +No tilde expansion is done, but ``*``, ``?``, and character ranges expressed with ``[]`` will be correctly matched. This is done by using the :func:`os.scandir` and :func:`fnmatch.fnmatch` functions in concert, and not by actually invoking a subshell. -Note that files beginning with a dot (``.``) can only be matched by +.. note:: + The pathnames are returned in no particular order. If you need a specific + order, sort the results. + +By default, files beginning with a dot (``.``) can only be matched by patterns that also start with a dot, unlike :func:`fnmatch.fnmatch` or :func:`pathlib.Path.glob`. -(For tilde and shell variable expansion, use :func:`os.path.expanduser` and -:func:`os.path.expandvars`.) +For tilde and shell variable expansion, use :func:`os.path.expanduser` and +:func:`os.path.expandvars`. For a literal match, wrap the meta-characters in brackets. For example, ``'[?]'`` matches the character ``'?'``. -The :mod:`glob` module defines the following functions: +The :mod:`!glob` module defines the following functions: .. function:: glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, \ @@ -51,7 +55,7 @@ The :mod:`glob` module defines the following functions: If *root_dir* is not ``None``, it should be a :term:`path-like object` specifying the root directory for searching. It has the same effect on - :func:`glob` as changing the current directory before calling it. If + :func:`!glob` as changing the current directory before calling it. If *pathname* is relative, the result will contain paths relative to *root_dir*. @@ -66,7 +70,8 @@ The :mod:`glob` module defines the following functions: pattern is followed by an :data:`os.sep` or :data:`os.altsep` then files will not match. - If *include_hidden* is true, "``**``" pattern will match hidden directories. + If *include_hidden* is true, wildcards can match path segments that + begin with a dot (``.``). .. audit-event:: glob.glob pathname,recursive glob.glob .. audit-event:: glob.glob/2 pathname,recursive,root_dir,dir_fd glob.glob @@ -79,6 +84,11 @@ The :mod:`glob` module defines the following functions: This function may return duplicate path names if *pathname* contains multiple "``**``" patterns and *recursive* is true. + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. versionchanged:: 3.5 Support for recursive globs using "``**``". @@ -102,6 +112,11 @@ The :mod:`glob` module defines the following functions: This function may return duplicate path names if *pathname* contains multiple "``**``" patterns and *recursive* is true. + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. versionchanged:: 3.5 Support for recursive globs using "``**``". @@ -117,7 +132,7 @@ The :mod:`glob` module defines the following functions: Escape all special characters (``'?'``, ``'*'`` and ``'['``). This is useful if you want to match an arbitrary literal string that may have special characters in it. Special characters in drive/UNC - sharepoints are not escaped, e.g. on Windows + sharepoints are not escaped, for example on Windows ``escape('//?/c:/Quo vadis?.txt')`` returns ``'//?/c:/Quo vadis[?].txt'``. .. versionadded:: 3.4 diff --git a/Doc/library/graphlib.rst b/Doc/library/graphlib.rst index 053d5f8231ba0e3..21f4d1fb938038e 100644 --- a/Doc/library/graphlib.rst +++ b/Doc/library/graphlib.rst @@ -204,7 +204,7 @@ Exceptions ---------- -The :mod:`graphlib` module defines the following exception classes: +The :mod:`!graphlib` module defines the following exception classes: .. exception:: CycleError diff --git a/Doc/library/grp.rst b/Doc/library/grp.rst index d1c7f22a2097803..f436970e791ace1 100644 --- a/Doc/library/grp.rst +++ b/Doc/library/grp.rst @@ -2,7 +2,6 @@ ================================== .. module:: grp - :platform: Unix :synopsis: The group database (getgrnam() and friends). -------------- diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst index c9d96085ef739d7..a3c98d2f42b0b14 100644 --- a/Doc/library/gzip.rst +++ b/Doc/library/gzip.rst @@ -11,9 +11,11 @@ This module provides a simple interface to compress and decompress files just like the GNU programs :program:`gzip` and :program:`gunzip` would. +.. include:: ../includes/optional-module.rst + The data compression is provided by the :mod:`zlib` module. -The :mod:`gzip` module provides the :class:`GzipFile` class, as well as the +The :mod:`!gzip` module provides the :class:`GzipFile` class, as well as the :func:`.open`, :func:`compress` and :func:`decompress` convenience functions. The :class:`GzipFile` class reads and writes :program:`gzip`\ -format files, automatically compressing or decompressing the data so that it looks like an @@ -267,20 +269,20 @@ Example of how to GZIP compress a binary string:: .. _gzip-cli: -Command Line Interface +Command-line interface ---------------------- -The :mod:`gzip` module provides a simple command line interface to compress or +The :mod:`!gzip` module provides a simple command line interface to compress or decompress files. -Once executed the :mod:`gzip` module keeps the input file(s). +Once executed the :mod:`!gzip` module keeps the input file(s). .. versionchanged:: 3.8 Add a new command line interface with a usage. By default, when you will execute the CLI, the default compression level is 6. -Command line options +Command-line options ^^^^^^^^^^^^^^^^^^^^ .. option:: file diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index bb2d2fad23bdb8a..84039ad28182d52 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -61,7 +61,7 @@ if you are using a rare "FIPS compliant" build of Python. These correspond to :data:`algorithms_guaranteed`. Additional algorithms may also be available if your Python distribution's -:mod:`hashlib` was linked against a build of OpenSSL that provides others. +:mod:`!hashlib` was linked against a build of OpenSSL that provides others. Others *are not guaranteed available* on all installations and will only be accessible by name via :func:`new`. See :data:`algorithms_available`. @@ -284,7 +284,7 @@ a file or file-like object. Example: >>> import io, hashlib, hmac - >>> with open(hashlib.__file__, "rb") as f: + >>> with open("library/hashlib.rst", "rb") as f: ... digest = hashlib.file_digest(f, "sha256") ... >>> digest.hexdigest() # doctest: +ELLIPSIS @@ -303,7 +303,7 @@ a file or file-like object. .. versionadded:: 3.11 .. versionchanged:: 3.14 - Now raises a :exc:`BlockingIOError` if the file is opened in blocking + Now raises a :exc:`BlockingIOError` if the file is opened in non-blocking mode. Previously, spurious null bytes were added to the digest. @@ -390,7 +390,7 @@ BLAKE2 supports **keyed mode** (a faster and simpler replacement for HMAC_), **salted hashing**, **personalization**, and **tree hashing**. Hash objects from this module follow the API of standard library's -:mod:`hashlib` objects. +:mod:`!hashlib` objects. Creating hash objects diff --git a/Doc/library/heapq.rst b/Doc/library/heapq.rst index 922ba0c8aa4214b..a21dc4885e4744f 100644 --- a/Doc/library/heapq.rst +++ b/Doc/library/heapq.rst @@ -58,6 +58,11 @@ functions, respectively. The following functions are provided for min-heaps: +.. function:: heapify(x) + + Transform list *x* into a min-heap, in-place, in linear time. + + .. function:: heappush(heap, item) Push the value *item* onto the *heap*, maintaining the min-heap invariant. @@ -77,11 +82,6 @@ The following functions are provided for min-heaps: followed by a separate call to :func:`heappop`. -.. function:: heapify(x) - - Transform list *x* into a min-heap, in-place, in linear time. - - .. function:: heapreplace(heap, item) Pop and return the smallest item from the *heap*, and also push the new *item*. @@ -231,6 +231,42 @@ Heap elements can be tuples. This is useful for assigning comparison values (1, 'write spec') +Other Applications +------------------ + +`Medians `_ are a measure of +central tendency for a set of numbers. In distributions skewed by +outliers, the median provides a more stable estimate than an average +(arithmetic mean). A running median is an `online algorithm +`_ that updates +continuously as new data arrives. + +A running median can be efficiently implemented by balancing two heaps, +a max-heap for values at or below the midpoint and a min-heap for values +above the midpoint. When the two heaps have the same size, the new +median is the average of the tops of the two heaps; otherwise, the +median is at the top of the larger heap:: + + def running_median(iterable): + "Yields the cumulative median of values seen so far." + + lo = [] # max-heap + hi = [] # min-heap (same size as or one smaller than lo) + + for x in iterable: + if len(lo) == len(hi): + heappush_max(lo, heappushpop(hi, x)) + yield lo[0] + else: + heappush(hi, heappushpop_max(lo, x)) + yield (lo[0] + hi[0]) / 2 + +For example:: + + >>> list(running_median([5.0, 9.0, 4.0, 12.0, 8.0, 9.0])) + [5.0, 7.0, 5.0, 7.0, 8.0, 8.5] + + Priority Queue Implementation Notes ----------------------------------- diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index d6692033b2d4c3a..d5608bd7543eb18 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -12,6 +12,9 @@ -------------- This module implements the HMAC algorithm as described by :rfc:`2104`. +The interface allows to use any hash function with a *fixed* digest size. +In particular, extendable output functions such as SHAKE-128 or SHAKE-256 +cannot be used with HMAC. .. function:: new(key, msg=None, digestmod) @@ -47,7 +50,9 @@ This module implements the HMAC algorithm as described by :rfc:`2104`. .. versionadded:: 3.7 -An HMAC object has the following methods: +.. class:: HMAC + + An HMAC object has the following methods: .. method:: HMAC.update(msg) diff --git a/Doc/library/html.parser.rst b/Doc/library/html.parser.rst index 6d433b5a04fc4a9..11f851d4f6c4b74 100644 --- a/Doc/library/html.parser.rst +++ b/Doc/library/html.parser.rst @@ -15,14 +15,18 @@ This module defines a class :class:`HTMLParser` which serves as the basis for parsing text files formatted in HTML (HyperText Mark-up Language) and XHTML. -.. class:: HTMLParser(*, convert_charrefs=True) +.. class:: HTMLParser(*, convert_charrefs=True, scripting=False) Create a parser instance able to parse invalid markup. - If *convert_charrefs* is ``True`` (the default), all character - references (except the ones in ``script``/``style`` elements) are + If *convert_charrefs* is true (the default), all character + references (except the ones in elements like ``script`` and ``style``) are automatically converted to the corresponding Unicode characters. + If *scripting* is false (the default), the content of the ``noscript`` + element is parsed normally; if it's true, it's returned as is without + being parsed. + An :class:`.HTMLParser` instance is fed HTML data and calls handler methods when start tags, end tags, text, comments, and other markup elements are encountered. The user should subclass :class:`.HTMLParser` and override its @@ -37,13 +41,18 @@ parsing text files formatted in HTML (HyperText Mark-up Language) and XHTML. .. versionchanged:: 3.5 The default value for argument *convert_charrefs* is now ``True``. + .. versionchanged:: 3.14.1 + Added the *scripting* parameter. + Example HTML Parser Application ------------------------------- As a basic example, below is a simple HTML parser that uses the :class:`HTMLParser` class to print out start tags, end tags, and data -as they are encountered:: +as they are encountered: + +.. testcode:: from html.parser import HTMLParser @@ -63,7 +72,7 @@ as they are encountered:: The output will then be: -.. code-block:: none +.. testoutput:: Encountered a start tag: html Encountered a start tag: head @@ -132,7 +141,7 @@ implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`): argument is a list of ``(name, value)`` pairs containing the attributes found inside the tag's ``<>`` brackets. The *name* will be translated to lower case, and quotes in the *value* have been removed, and character and entity references - have been replaced. + have been replaced. For empty attributes, *value* is ``None``. For instance, for the tag ````, this method would be called as ``handle_starttag('a', [('href', 'https://www.cwi.nl/')])``. @@ -159,15 +168,15 @@ implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`): .. method:: HTMLParser.handle_data(data) This method is called to process arbitrary data (e.g. text nodes and the - content of ```` and ````). + content of elements like ``script`` and ``style``). .. method:: HTMLParser.handle_entityref(name) This method is called to process a named character reference of the form ``&name;`` (e.g. ``>``), where *name* is a general entity reference - (e.g. ``'gt'``). This method is never called if *convert_charrefs* is - ``True``. + (e.g. ``'gt'``). + This method is only called if *convert_charrefs* is false. .. method:: HTMLParser.handle_charref(name) @@ -175,8 +184,8 @@ implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`): This method is called to process decimal and hexadecimal numeric character references of the form :samp:`&#{NNN};` and :samp:`&#x{NNN};`. For example, the decimal equivalent for ``>`` is ``>``, whereas the hexadecimal is ``>``; - in this case the method will receive ``'62'`` or ``'x3E'``. This method - is never called if *convert_charrefs* is ``True``. + in this case the method will receive ``'62'`` or ``'x3E'``. + This method is only called if *convert_charrefs* is false. .. method:: HTMLParser.handle_comment(data) @@ -230,7 +239,9 @@ Examples -------- The following class implements a parser that will be used to illustrate more -examples:: +examples: + +.. testcode:: from html.parser import HTMLParser from html.entities import name2codepoint @@ -266,13 +277,17 @@ examples:: parser = MyHTMLParser() -Parsing a doctype:: +Parsing a doctype: + +.. doctest:: >>> parser.feed('') Decl : DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd" -Parsing an element with a few attributes and a title:: +Parsing an element with a few attributes and a title: + +.. doctest:: >>> parser.feed('The Python logo') Start tag: img @@ -284,8 +299,10 @@ Parsing an element with a few attributes and a title:: Data : Python End tag : h1 -The content of ``script`` and ``style`` elements is returned as is, without -further parsing:: +The content of elements like ``script`` and ``style`` is returned as is, +without further parsing: + +.. doctest:: >>> parser.feed('') Start tag: style @@ -294,22 +311,43 @@ further parsing:: End tag : style >>> parser.feed('') + ... 'alert("hello! ☺");') Start tag: script attr: ('type', 'text/javascript') - Data : alert("hello!"); + Data : alert("hello! ☺"); End tag : script -Parsing comments:: +Attribute names are converted to lowercase, quotes from attribute values removed, +and ``None`` is returned as *value* for empty attributes (such as ``checked``): + +.. doctest:: - >>> parser.feed('' + >>> parser.feed("") + Start tag: input + attr: ('type', 'checkbox') + attr: ('checked', None) + attr: ('required', '') + attr: ('disabled', 'disabled') + +Parsing comments: + +.. doctest:: + + >>> parser.feed('' ... '') - Comment : a comment + Comment : a comment Comment : [if IE 9]>IE-specific content'``):: +correct char (note: these 3 references are all equivalent to ``'>'``): + +.. doctest:: + >>> parser = MyHTMLParser() + >>> parser.feed('>>>') + Data : >>> + + >>> parser = MyHTMLParser(convert_charrefs=False) >>> parser.feed('>>>') Named ent: > Num ent : > @@ -317,18 +355,22 @@ correct char (note: these 3 references are all equivalent to ``'>'``):: Feeding incomplete chunks to :meth:`~HTMLParser.feed` works, but :meth:`~HTMLParser.handle_data` might be called more than once -(unless *convert_charrefs* is set to ``True``):: +if *convert_charrefs* is false: - >>> for chunk in ['buff', 'ered ', 'text']: +.. doctest:: + + >>> for chunk in ['buff', 'ered', ' text']: ... parser.feed(chunk) ... Start tag: span Data : buff Data : ered - Data : text + Data : text End tag : span -Parsing invalid HTML (e.g. unquoted attributes) also works:: +Parsing invalid HTML (e.g. unquoted attributes) also works: + +.. doctest:: >>> parser.feed('

tag soup

') Start tag: p diff --git a/Doc/library/html.rst b/Doc/library/html.rst index 9aa39ba9a42b0fd..65c49a4107a0487 100644 --- a/Doc/library/html.rst +++ b/Doc/library/html.rst @@ -14,9 +14,12 @@ This module defines utilities to manipulate HTML. Convert the characters ``&``, ``<`` and ``>`` in string *s* to HTML-safe sequences. Use this if you need to display text that might contain such - characters in HTML. If the optional flag *quote* is true, the characters - (``"``) and (``'``) are also translated; this helps for inclusion in an HTML - attribute value delimited by quotes, as in ````. + characters in HTML. If the optional flag *quote* is true (the default), the + characters (``"``) and (``'``) are also translated; this helps for inclusion + in an HTML attribute value delimited by quotes, as in ````. + If *quote* is set to false, the characters (``"``) and (``'``) are not + translated. + .. versionadded:: 3.2 diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst index 2835c8d0eb711e3..14c67a0600f7287 100644 --- a/Doc/library/http.client.rst +++ b/Doc/library/http.client.rst @@ -125,7 +125,7 @@ This module provides the following function: Parse the headers from a file pointer *fp* representing a HTTP request/response. The file has to be a :class:`~io.BufferedIOBase` reader - (i.e. not text) and must provide a valid :rfc:`2822` style header. + (i.e. not text) and must provide a valid :rfc:`5322` style header. This function returns an instance of :class:`http.client.HTTPMessage` that holds the header fields, but no payload @@ -311,6 +311,12 @@ HTTPConnection Objects :class:`str` or bytes-like object that is not also a file as the body representation. + .. note:: + + Note that you must have read the whole response or call :meth:`close` + if :meth:`getresponse` raised an non-:exc:`ConnectionError` exception + before you can send a new request to the server. + .. versionchanged:: 3.2 *body* can now be an iterable. @@ -326,16 +332,15 @@ HTTPConnection Objects Should be called after a request is sent to get the response from the server. Returns an :class:`HTTPResponse` instance. - .. note:: - - Note that you must have read the whole response before you can send a new - request to the server. - .. versionchanged:: 3.5 If a :exc:`ConnectionError` or subclass is raised, the :class:`HTTPConnection` object will be ready to reconnect when a new request is sent. + Note that this does not apply to :exc:`OSError`\s raised by the underlying + socket. Instead the caller is responsible to call :meth:`close` on the + existing connection. + .. method:: HTTPConnection.set_debuglevel(level) diff --git a/Doc/library/http.cookiejar.rst b/Doc/library/http.cookiejar.rst index 23ddecf873876d4..644ddffa05052b4 100644 --- a/Doc/library/http.cookiejar.rst +++ b/Doc/library/http.cookiejar.rst @@ -11,8 +11,8 @@ -------------- -The :mod:`http.cookiejar` module defines classes for automatic handling of HTTP -cookies. It is useful for accessing web sites that require small pieces of data +The :mod:`!http.cookiejar` module defines classes for automatic handling of HTTP +cookies. It is useful for accessing websites that require small pieces of data -- :dfn:`cookies` -- to be set on the client machine by an HTTP response from a web server, and then returned to the server in later HTTP requests. @@ -21,7 +21,7 @@ Both the regular Netscape cookie protocol and the protocol defined by :rfc:`2109` cookies are parsed as Netscape cookies and subsequently treated either as Netscape or RFC 2965 cookies according to the 'policy' in effect. Note that the great majority of cookies on the internet are Netscape cookies. -:mod:`http.cookiejar` attempts to follow the de-facto Netscape cookie protocol (which +:mod:`!http.cookiejar` attempts to follow the de-facto Netscape cookie protocol (which differs substantially from that set out in the original Netscape specification), including taking note of the ``max-age`` and ``port`` cookie-attributes introduced with RFC 2965. @@ -29,7 +29,7 @@ introduced with RFC 2965. .. note:: The various named parameters found in :mailheader:`Set-Cookie` and - :mailheader:`Set-Cookie2` headers (eg. ``domain`` and ``expires``) are + :mailheader:`Set-Cookie2` headers (for example, ``domain`` and ``expires``) are conventionally referred to as :dfn:`attributes`. To distinguish them from Python attributes, the documentation for this module uses the term :dfn:`cookie-attribute` instead. @@ -88,7 +88,7 @@ The following classes are provided: Constructor arguments should be passed as keyword arguments only. *blocked_domains* is a sequence of domain names that we never accept cookies - from, nor return cookies to. *allowed_domains* if not :const:`None`, this is a + from, nor return cookies to. *allowed_domains* if not ``None``, this is a sequence of the only domains for which we accept and return cookies. *secure_protocols* is a sequence of protocols for which secure cookies can be added to. By default *https* and *wss* (secure websocket) are considered @@ -96,12 +96,12 @@ The following classes are provided: :class:`CookiePolicy` and :class:`DefaultCookiePolicy` objects. :class:`DefaultCookiePolicy` implements the standard accept / reject rules for - Netscape and :rfc:`2965` cookies. By default, :rfc:`2109` cookies (ie. cookies + Netscape and :rfc:`2965` cookies. By default, :rfc:`2109` cookies (that is, cookies received in a :mailheader:`Set-Cookie` header with a version cookie-attribute of 1) are treated according to the RFC 2965 rules. However, if RFC 2965 handling is turned off or :attr:`rfc2109_as_netscape` is ``True``, RFC 2109 cookies are 'downgraded' by the :class:`CookieJar` instance to Netscape cookies, by - setting the :attr:`version` attribute of the :class:`Cookie` instance to 0. + setting the :attr:`~Cookie.version` attribute of the :class:`Cookie` instance to 0. :class:`DefaultCookiePolicy` also provides some parameters to allow some fine-tuning of policy. @@ -109,8 +109,8 @@ The following classes are provided: .. class:: Cookie() This class represents Netscape, :rfc:`2109` and :rfc:`2965` cookies. It is not - expected that users of :mod:`http.cookiejar` construct their own :class:`Cookie` - instances. Instead, if necessary, call :meth:`make_cookies` on a + expected that users of :mod:`!http.cookiejar` construct their own :class:`Cookie` + instances. Instead, if necessary, call :meth:`~CookieJar.make_cookies` on a :class:`CookieJar` instance. @@ -121,13 +121,13 @@ The following classes are provided: Module :mod:`http.cookies` HTTP cookie classes, principally useful for server-side code. The - :mod:`http.cookiejar` and :mod:`http.cookies` modules do not depend on each + :mod:`!http.cookiejar` and :mod:`http.cookies` modules do not depend on each other. https://curl.se/rfc/cookie_spec.html The specification of the original Netscape cookie protocol. Though this is still the dominant protocol, the 'Netscape cookie protocol' implemented by all - the major browsers (and :mod:`http.cookiejar`) only bears a passing resemblance to + the major browsers (and :mod:`!http.cookiejar`) only bears a passing resemblance to the one sketched out in ``cookie_spec.html``. :rfc:`2109` - HTTP State Management Mechanism @@ -144,7 +144,7 @@ The following classes are provided: .. _cookie-jar-objects: -CookieJar and FileCookieJar Objects +CookieJar and FileCookieJar objects ----------------------------------- :class:`CookieJar` objects support the :term:`iterator` protocol for iterating over @@ -157,20 +157,27 @@ contained :class:`Cookie` objects. Add correct :mailheader:`Cookie` header to *request*. - If policy allows (ie. the :attr:`rfc2965` and :attr:`hide_cookie2` attributes of + If policy allows (that is, the :attr:`~CookiePolicy.rfc2965` and + :attr:`~CookiePolicy.hide_cookie2` attributes of the :class:`CookieJar`'s :class:`CookiePolicy` instance are true and false respectively), the :mailheader:`Cookie2` header is also added when appropriate. The *request* object (usually a :class:`urllib.request.Request` instance) - must support the methods :meth:`get_full_url`, :meth:`has_header`, - :meth:`get_header`, :meth:`header_items`, :meth:`add_unredirected_header` - and the attributes :attr:`host`, :attr:`!type`, :attr:`unverifiable` - and :attr:`origin_req_host` as documented by :mod:`urllib.request`. + must support the methods :meth:`~urllib.request.Request.get_full_url`, + :meth:`~urllib.request.Request.has_header`, + :meth:`~urllib.request.Request.get_header`, + :meth:`~urllib.request.Request.header_items`, + :meth:`~urllib.request.Request.add_unredirected_header` + and the attributes :attr:`~urllib.request.Request.host`, + :attr:`~urllib.request.Request.type`, :attr:`~urllib.request.Request.unverifiable` + and :attr:`~urllib.request.Request.origin_req_host` as documented by + :mod:`urllib.request`. .. versionchanged:: 3.3 - *request* object needs :attr:`origin_req_host` attribute. Dependency on a - deprecated method :meth:`get_origin_req_host` has been removed. + *request* object needs :attr:`~urllib.request.Request.origin_req_host` + attribute. Dependency on a deprecated method + :meth:`!get_origin_req_host` has been removed. .. method:: CookieJar.extract_cookies(response, request) @@ -183,20 +190,24 @@ contained :class:`Cookie` objects. as appropriate (subject to the :meth:`CookiePolicy.set_ok` method's approval). The *response* object (usually the result of a call to - :meth:`urllib.request.urlopen`, or similar) should support an :meth:`info` - method, which returns an :class:`email.message.Message` instance. + :meth:`urllib.request.urlopen`, or similar) should support an + :meth:`~http.client.HTTPResponse.info` method, which returns an + :class:`email.message.Message` instance. The *request* object (usually a :class:`urllib.request.Request` instance) - must support the method :meth:`get_full_url` and the attributes - :attr:`host`, :attr:`unverifiable` and :attr:`origin_req_host`, + must support the method :meth:`~urllib.request.Request.get_full_url` and + the attributes :attr:`~urllib.request.Request.host`, + :attr:`~urllib.request.Request.unverifiable` + and :attr:`~urllib.request.Request.origin_req_host`, as documented by :mod:`urllib.request`. The request is used to set default values for cookie-attributes as well as for checking that the cookie is allowed to be set. .. versionchanged:: 3.3 - *request* object needs :attr:`origin_req_host` attribute. Dependency on a - deprecated method :meth:`get_origin_req_host` has been removed. + *request* object needs :attr:`~urllib.request.Request.origin_req_host` + attribute. Dependency on a deprecated method + :meth:`!get_origin_req_host` has been removed. .. method:: CookieJar.set_policy(policy) @@ -239,13 +250,13 @@ contained :class:`Cookie` objects. Discard all session cookies. - Discards all contained cookies that have a true :attr:`discard` attribute + Discards all contained cookies that have a true :attr:`~Cookie.discard` attribute (usually because they had either no ``max-age`` or ``expires`` cookie-attribute, or an explicit ``discard`` cookie-attribute). For interactive browsers, the end of a session usually corresponds to closing the browser window. - Note that the :meth:`save` method won't save session cookies anyway, unless you - ask otherwise by passing a true *ignore_discard* argument. + Note that the :meth:`~FileCookieJar.save` method won't save session cookies + anyway, unless you ask otherwise by passing a true *ignore_discard* argument. :class:`FileCookieJar` implements the following additional methods: @@ -258,8 +269,9 @@ contained :class:`Cookie` objects. method unimplemented. *filename* is the name of file in which to save cookies. If *filename* is not - specified, :attr:`self.filename` is used (whose default is the value passed to - the constructor, if any); if :attr:`self.filename` is :const:`None`, + specified, :attr:`self.filename ` is used (whose + default is the value passed to the constructor, if any); if + :attr:`self.filename ` is ``None``, :exc:`ValueError` is raised. *ignore_discard*: save even cookies set to be discarded. *ignore_expires*: save @@ -352,7 +364,7 @@ writing. .. _cookie-policy-objects: -CookiePolicy Objects +CookiePolicy objects -------------------- Objects implementing the :class:`CookiePolicy` interface have the following @@ -436,7 +448,7 @@ setting and receiving any and all cookies (this is unlikely to be useful). .. _default-cookie-policy-objects: -DefaultCookiePolicy Objects +DefaultCookiePolicy objects --------------------------- Implements the standard rules for accepting and returning cookies. @@ -466,10 +478,10 @@ blocking some benign cookies). A domain blocklist and allowlist is provided (both off by default). Only domains not in the blocklist and present in the allowlist (if the allowlist is active) participate in cookie setting and returning. Use the *blocked_domains* -constructor argument, and :meth:`blocked_domains` and -:meth:`set_blocked_domains` methods (and the corresponding argument and methods -for *allowed_domains*). If you set an allowlist, you can turn it off again by -setting it to :const:`None`. +constructor argument, and :meth:`~DefaultCookiePolicy.blocked_domains` and +:meth:`~DefaultCookiePolicy.set_blocked_domains` methods (and the corresponding +argument and methods for *allowed_domains*). If you set an allowlist, you can +turn it off again by setting it to ``None``. Domains in block or allow lists that do not start with a dot must equal the cookie domain to be matched. For example, ``"example.com"`` matches a blocklist @@ -501,12 +513,12 @@ and ``".168.1.2"``, 192.168.1.2 is blocked, but 193.168.1.2 is not. .. method:: DefaultCookiePolicy.allowed_domains() - Return :const:`None`, or the sequence of allowed domains (as a tuple). + Return ``None``, or the sequence of allowed domains (as a tuple). .. method:: DefaultCookiePolicy.set_allowed_domains(allowed_domains) - Set the sequence of allowed domains, or :const:`None`. + Set the sequence of allowed domains, or ``None``. .. method:: DefaultCookiePolicy.is_not_allowed(domain) @@ -522,9 +534,9 @@ all be assigned to. .. attribute:: DefaultCookiePolicy.rfc2109_as_netscape If true, request that the :class:`CookieJar` instance downgrade :rfc:`2109` cookies - (ie. cookies received in a :mailheader:`Set-Cookie` header with a version + (that is, cookies received in a :mailheader:`Set-Cookie` header with a version cookie-attribute of 1) to Netscape cookies by setting the version attribute of - the :class:`Cookie` instance to 0. The default value is :const:`None`, in which + the :class:`Cookie` instance to 0. The default value is ``None``, in which case RFC 2109 cookies are downgraded if and only if :rfc:`2965` handling is turned off. Therefore, RFC 2109 cookies are downgraded by default. @@ -570,14 +582,14 @@ Netscape protocol strictness switches: Don't allow setting cookies whose path doesn't path-match request URI. -:attr:`strict_ns_domain` is a collection of flags. Its value is constructed by +:attr:`~DefaultCookiePolicy.strict_ns_domain` is a collection of flags. Its value is constructed by or-ing together (for example, ``DomainStrictNoDots|DomainStrictNonDomain`` means both flags are set). .. attribute:: DefaultCookiePolicy.DomainStrictNoDots - When setting cookies, the 'host prefix' must not contain a dot (eg. + When setting cookies, the 'host prefix' must not contain a dot (for example, ``www.foo.bar.com`` can't set a cookie for ``.bar.com``, because ``www.foo`` contains a dot). @@ -585,7 +597,7 @@ both flags are set). .. attribute:: DefaultCookiePolicy.DomainStrictNonDomain Cookies that did not explicitly specify a ``domain`` cookie-attribute can only - be returned to a domain equal to the domain that set the cookie (eg. + be returned to a domain equal to the domain that set the cookie (for example, ``spam.example.com`` won't be returned cookies from ``example.com`` that had no ``domain`` cookie-attribute). @@ -600,7 +612,7 @@ combinations of the above flags: .. attribute:: DefaultCookiePolicy.DomainLiberal - Equivalent to 0 (ie. all of the above Netscape domain strictness flags switched + Equivalent to 0 (that is, all of the above Netscape domain strictness flags switched off). @@ -609,7 +621,7 @@ combinations of the above flags: Equivalent to ``DomainStrictNoDots|DomainStrictNonDomain``. -Cookie Objects +Cookie objects -------------- :class:`Cookie` instances have Python attributes roughly corresponding to the @@ -617,7 +629,7 @@ standard cookie-attributes specified in the various cookie standards. The correspondence is not one-to-one, because there are complicated rules for assigning default values, because the ``max-age`` and ``expires`` cookie-attributes contain equivalent information, and because :rfc:`2109` cookies -may be 'downgraded' by :mod:`http.cookiejar` from version 1 to version 0 (Netscape) +may be 'downgraded' by :mod:`!http.cookiejar` from version 1 to version 0 (Netscape) cookies. Assignment to these attributes should not be necessary other than in rare @@ -627,9 +639,9 @@ internal consistency, so you should know what you're doing if you do that. .. attribute:: Cookie.version - Integer or :const:`None`. Netscape cookies have :attr:`version` 0. :rfc:`2965` and + Integer or ``None``. Netscape cookies have :attr:`version` 0. :rfc:`2965` and :rfc:`2109` cookies have a ``version`` cookie-attribute of 1. However, note that - :mod:`http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in which + :mod:`!http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in which case :attr:`version` is 0. @@ -640,13 +652,13 @@ internal consistency, so you should know what you're doing if you do that. .. attribute:: Cookie.value - Cookie value (a string), or :const:`None`. + Cookie value (a string), or ``None``. .. attribute:: Cookie.port - String representing a port or a set of ports (eg. '80', or '80,8080'), or - :const:`None`. + String representing a port or a set of ports (for example, '80', or '80,8080'), or + ``None``. .. attribute:: Cookie.domain @@ -656,7 +668,7 @@ internal consistency, so you should know what you're doing if you do that. .. attribute:: Cookie.path - Cookie path (a string, eg. ``'/acme/rocket_launchers'``). + Cookie path (a string, for example, ``'/acme/rocket_launchers'``). .. attribute:: Cookie.secure @@ -666,7 +678,7 @@ internal consistency, so you should know what you're doing if you do that. .. attribute:: Cookie.expires - Integer expiry date in seconds since epoch, or :const:`None`. See also the + Integer expiry date in seconds since epoch, or ``None``. See also the :meth:`is_expired` method. @@ -678,21 +690,21 @@ internal consistency, so you should know what you're doing if you do that. .. attribute:: Cookie.comment String comment from the server explaining the function of this cookie, or - :const:`None`. + ``None``. .. attribute:: Cookie.comment_url URL linking to a comment from the server explaining the function of this cookie, - or :const:`None`. + or ``None``. .. attribute:: Cookie.rfc2109 - ``True`` if this cookie was received as an :rfc:`2109` cookie (ie. the cookie + ``True`` if this cookie was received as an :rfc:`2109` cookie (that is, the cookie arrived in a :mailheader:`Set-Cookie` header, and the value of the Version cookie-attribute in that header was 1). This attribute is provided because - :mod:`http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in + :mod:`!http.cookiejar` may 'downgrade' RFC 2109 cookies to Netscape cookies, in which case :attr:`version` is 0. @@ -744,7 +756,7 @@ The :class:`Cookie` class also defines the following method: Examples -------- -The first example shows the most common usage of :mod:`http.cookiejar`:: +The first example shows the most common usage of :mod:`!http.cookiejar`:: import http.cookiejar, urllib.request cj = http.cookiejar.CookieJar() diff --git a/Doc/library/http.cookies.rst b/Doc/library/http.cookies.rst index eb1963207211945..1b2d17975a5edbf 100644 --- a/Doc/library/http.cookies.rst +++ b/Doc/library/http.cookies.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`http.cookies` module defines classes for abstracting the concept of +The :mod:`!http.cookies` module defines classes for abstracting the concept of cookies, an HTTP state management mechanism. It supports both simple string-only cookies, and provides an abstraction for having any serializable data-type as cookie value. @@ -65,7 +65,7 @@ in a cookie name (as :attr:`~Morsel.key`). Module :mod:`http.cookiejar` HTTP cookie handling for web *clients*. The :mod:`http.cookiejar` and - :mod:`http.cookies` modules do not depend on each other. + :mod:`!http.cookies` modules do not depend on each other. :rfc:`2109` - HTTP State Management Mechanism This is the state management specification implemented by this module. @@ -148,9 +148,12 @@ Morsel Objects in HTTP requests, and is not accessible through JavaScript. This is intended to mitigate some forms of cross-site scripting. - The attribute :attr:`samesite` specifies that the browser is not allowed to - send the cookie along with cross-site requests. This helps to mitigate CSRF - attacks. Valid values for this attribute are "Strict" and "Lax". + The attribute :attr:`samesite` controls when the browser sends the cookie with + cross-site requests. This helps to mitigate CSRF attacks. Valid values are + "Strict" (only sent with same-site requests), "Lax" (sent with same-site + requests and top-level navigations), and "None" (sent with same-site and + cross-site requests). When using "None", the "secure" attribute must also + be set, as required by modern browsers. The attribute :attr:`partitioned` indicates to user agents that these cross-site cookies *should* only be available in the same top-level context @@ -261,7 +264,7 @@ Morsel Objects Example ------- -The following example demonstrates how to use the :mod:`http.cookies` module. +The following example demonstrates how to use the :mod:`!http.cookies` module. .. doctest:: :options: +NORMALIZE_WHITESPACE @@ -289,9 +292,9 @@ The following example demonstrates how to use the :mod:`http.cookies` module. Set-Cookie: chips=ahoy Set-Cookie: vienna=finger >>> C = cookies.SimpleCookie() - >>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=\\012;";') + >>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=;";') >>> print(C) - Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=\012;" + Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=;" >>> C = cookies.SimpleCookie() >>> C["oreo"] = "doublestuff" >>> C["oreo"]["path"] = "/" diff --git a/Doc/library/http.rst b/Doc/library/http.rst index ce3fb9f81205026..43a801416e24f99 100644 --- a/Doc/library/http.rst +++ b/Doc/library/http.rst @@ -12,7 +12,7 @@ -------------- -:mod:`http` is a package that collects several modules for working with the +:mod:`!http` is a package that collects several modules for working with the HyperText Transfer Protocol: * :mod:`http.client` is a low-level HTTP protocol client; for high-level URL @@ -22,7 +22,7 @@ HyperText Transfer Protocol: * :mod:`http.cookiejar` provides persistence of cookies -The :mod:`http` module also defines the following enums that help you work with http related code: +The :mod:`!http` module also defines the following enums that help you work with http related code: .. class:: HTTPStatus @@ -139,7 +139,8 @@ equal to the constant name (i.e. ``http.HTTPStatus.OK`` is also available as .. versionchanged:: 3.13 Implemented RFC9110 naming for status constants. Old constant names are preserved for - backwards compatibility. + backwards compatibility: ``413 REQUEST_ENTITY_TOO_LARGE``, ``414 REQUEST_URI_TOO_LONG``, + ``416 REQUESTED_RANGE_NOT_SATISFIABLE`` and ``422 UNPROCESSABLE_ENTITY``. HTTP status category -------------------- diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 54df4a7e804d508..a9242558c494af9 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -19,7 +19,7 @@ This module defines classes for implementing HTTP servers. .. warning:: - :mod:`http.server` is not recommended for production. It only implements + :mod:`!http.server` is not recommended for production. It only implements :ref:`basic security checks `. .. include:: ../includes/wasm-notavail.rst @@ -39,7 +39,17 @@ handler. Code to create and run the server looks like this:: This class builds on the :class:`~socketserver.TCPServer` class by storing the server address as instance variables named :attr:`server_name` and :attr:`server_port`. The server is accessible by the handler, typically - through the handler's :attr:`server` instance variable. + through the handler's :attr:`~socketserver.BaseRequestHandler.server` + instance variable. + + .. attribute:: server_name + + The HTTP server's fully qualified domain name. + + .. attribute:: server_port + + The HTTP server's port number obtained from *server_address*. + .. class:: ThreadingHTTPServer(server_address, RequestHandlerClass) @@ -60,7 +70,7 @@ handler. Code to create and run the server looks like this:: object fails with a :exc:`RuntimeError`. The *certfile* argument is the path to the SSL certificate chain file, - and the *keyfile* is the path to file containing the private key. + and the *keyfile* is the path to the file containing the private key. A *password* can be specified for files protected and wrapped with PKCS#8, but beware that this could possibly expose hardcoded passwords in clear. @@ -99,7 +109,7 @@ instantiation, of which this module provides three different variants: This class is used to handle the HTTP requests that arrive at the server. By itself, it cannot respond to any actual HTTP requests; it must be subclassed - to handle each request method (e.g. GET or POST). + to handle each request method (for example, ``'GET'`` or ``'POST'``). :class:`BaseHTTPRequestHandler` provides a number of class and instance variables, and methods for use by subclasses. @@ -140,7 +150,7 @@ instantiation, of which this module provides three different variants: .. attribute:: path - Contains the request path. If query component of the URL is present, + Contains the request path. If the query component of the URL is present, then ``path`` includes the query. Using the terminology of :rfc:`3986`, ``path`` here includes ``hier-part`` and the ``query``. @@ -154,7 +164,7 @@ instantiation, of which this module provides three different variants: variable. This instance parses and manages the headers in the HTTP request. The :func:`~http.client.parse_headers` function from :mod:`http.client` is used to parse the headers and it requires that the - HTTP request provide a valid :rfc:`2822` style header. + HTTP request provide a valid :rfc:`5322` style header. .. attribute:: rfile @@ -190,7 +200,7 @@ instantiation, of which this module provides three different variants: Specifies a format string that should be used by :meth:`send_error` method for building an error response to the client. The string is filled by default with variables from :attr:`responses` based on the status code - that passed to :meth:`send_error`. + passed to :meth:`send_error`. .. attribute:: error_content_type @@ -238,10 +248,10 @@ instantiation, of which this module provides three different variants: .. method:: handle_expect_100() When an HTTP/1.1 conformant server receives an ``Expect: 100-continue`` - request header it responds back with a ``100 Continue`` followed by ``200 - OK`` headers. + request header it responds with a ``100 Continue`` followed by ``200 OK`` + headers. This method can be overridden to raise an error if the server does not - want the client to continue. For e.g. server can choose to send ``417 + want the client to continue. For example, the server can choose to send ``417 Expectation Failed`` as a response header and ``return False``. .. versionadded:: 3.2 @@ -287,16 +297,20 @@ instantiation, of which this module provides three different variants: specifying its value. Note that, after the send_header calls are done, :meth:`end_headers` MUST BE called in order to complete the operation. + This method does not reject input containing CRLF sequences. + .. versionchanged:: 3.2 Headers are stored in an internal buffer. .. method:: send_response_only(code, message=None) Sends the response header only, used for the purposes when ``100 - Continue`` response is sent by the server to the client. The headers not - buffered and sent directly the output stream.If the *message* is not + Continue`` response is sent by the server to the client. The headers are + not buffered and sent directly the output stream. If the *message* is not specified, the HTTP message corresponding the response *code* is sent. + This method does not reject *message* containing CRLF sequences. + .. versionadded:: 3.2 .. method:: end_headers() @@ -334,7 +348,7 @@ instantiation, of which this module provides three different variants: to create custom error logging mechanisms. The *format* argument is a standard printf-style format string, where the additional arguments to :meth:`log_message` are applied as inputs to the formatting. The client - ip address and current date and time are prefixed to every message logged. + IP address and current date and time are prefixed to every message logged. .. method:: version_string() @@ -386,6 +400,14 @@ instantiation, of which this module provides three different variants: This will be ``"SimpleHTTP/" + __version__``, where ``__version__`` is defined at the module level. + .. attribute:: index_pages + + Specifies the filenames that are treated as directory index pages. + + Defaults to ``("index.html", "index.htm")``. + + .. versionadded:: 3.12 + .. attribute:: extensions_map A dictionary mapping suffixes into MIME types, contains custom overrides @@ -409,8 +431,8 @@ instantiation, of which this module provides three different variants: The request is mapped to a local file by interpreting the request as a path relative to the current working directory. - If the request was mapped to a directory, the directory is checked for a - file named ``index.html`` or ``index.htm`` (in that order). If found, the + If the request was mapped to a directory, the directory is checked for + an index page as specified by :attr:`index_pages`. If found, the file's contents are returned; otherwise a directory listing is generated by calling the :meth:`list_directory` method. This method uses :func:`os.listdir` to scan the directory, and returns a ``404`` error @@ -429,8 +451,7 @@ instantiation, of which this module provides three different variants: ``'Last-Modified:'`` header with the file's modification time. Then follows a blank line signifying the end of the headers, and then the - contents of the file are output. If the file's MIME type starts with - ``text/`` the file is opened in text mode; otherwise binary mode is used. + contents of the file are output. For example usage, see the implementation of the ``test`` function in :source:`Lib/http/server.py`. @@ -438,9 +459,32 @@ instantiation, of which this module provides three different variants: .. versionchanged:: 3.7 Support of the ``'If-Modified-Since'`` header. -The :class:`SimpleHTTPRequestHandler` class can be used in the following -manner in order to create a very basic webserver serving files relative to -the current directory:: + .. method:: list_directory(path) + + Helper to list the contents of *path* when no index page is present. + + This returns either a :term:`file-like object` (which must be closed + by the caller) or ``None`` to indicate an error, in which case the + caller has nothing further to do. In either case, the headers are sent. + + .. method:: guess_type(path) + + Guess the type of the file at the given *path*. + + This returns a string of the form ``type/subtype``, usable for + a MIME Content-type header. + + The default implementation looks the file's extension up in + :attr:`extensions_map`, falling back to + :func:`mimetypes.guess_file_type` and then to + ``'application/octet-stream'``. + + .. versionchanged:: 3.13 + Add :func:`mimetypes.guess_file_type` as a fallback. + + +The :class:`SimpleHTTPRequestHandler` class can be used to create a very basic +webserver serving files relative to the current directory as follows:: import http.server import socketserver @@ -456,7 +500,7 @@ the current directory:: :class:`SimpleHTTPRequestHandler` can also be subclassed to enhance behavior, such as using different index file names by overriding the class attribute -:attr:`index_pages`. +:attr:`~SimpleHTTPRequestHandler.index_pages`. .. class:: CGIHTTPRequestHandler(request, client_address, server) @@ -477,7 +521,9 @@ such as using different index file names by overriding the class attribute the other common server configuration is to treat special extensions as denoting CGI scripts. - The :func:`do_GET` and :func:`do_HEAD` functions are modified to run CGI scripts + The :func:`~SimpleHTTPRequestHandler.do_GET` and + :func:`~SimpleHTTPRequestHandler.do_HEAD` functions + are modified to run CGI scripts and serve the output, instead of serving files, if the request leads to somewhere below the ``cgi_directories`` path. @@ -513,9 +559,11 @@ such as using different index file names by overriding the class attribute Command-line interface ---------------------- -:mod:`http.server` can also be invoked directly using the :option:`-m` +:mod:`!http.server` can also be invoked directly using the :option:`-m` switch of the interpreter. The following example illustrates how to serve -files relative to the current directory:: +files relative to the current directory: + +.. code-block:: bash python -m http.server [OPTIONS] [port] @@ -526,7 +574,9 @@ The following options are accepted: .. option:: port The server listens to port 8000 by default. The default can be overridden - by passing the desired port number as an argument:: + by passing the desired port number as an argument: + + .. code-block:: bash python -m http.server 9000 @@ -535,7 +585,9 @@ The following options are accepted: Specifies a specific address to which it should bind. Both IPv4 and IPv6 addresses are supported. By default, the server binds itself to all interfaces. For example, the following command causes the server to bind - to localhost only:: + to localhost only: + + .. code-block:: bash python -m http.server --bind 127.0.0.1 @@ -548,7 +600,9 @@ The following options are accepted: Specifies a directory to which it should serve the files. By default, the server uses the current directory. For example, the following command - uses a specific directory:: + uses a specific directory: + + .. code-block:: bash python -m http.server --directory /tmp/ @@ -558,7 +612,9 @@ The following options are accepted: Specifies the HTTP version to which the server is conformant. By default, the server is conformant to HTTP/1.0. For example, the following command - runs an HTTP/1.1 conformant server:: + runs an HTTP/1.1 conformant server: + + .. code-block:: bash python -m http.server --protocol HTTP/1.1 @@ -573,7 +629,7 @@ The following options are accepted: .. deprecated-removed:: 3.13 3.15 - :mod:`http.server` command line ``--cgi`` support is being removed + :mod:`!http.server` command line ``--cgi`` support is being removed because :class:`CGIHTTPRequestHandler` is being removed. .. warning:: @@ -584,7 +640,9 @@ The following options are accepted: .. option:: --tls-cert - Specifies a TLS certificate chain for HTTPS connections:: + Specifies a TLS certificate chain for HTTPS connections: + + .. code-block:: bash python -m http.server --tls-cert fullchain.pem @@ -600,14 +658,16 @@ The following options are accepted: .. option:: --tls-password-file - Specifies the password file for password-protected private keys:: + Specifies the password file for password-protected private keys: + + .. code-block:: bash python -m http.server \ --tls-cert cert.pem \ --tls-key key.pem \ --tls-password-file password.txt - This option requires `--tls-cert`` to be specified. + This option requires ``--tls-cert`` to be specified. .. versionadded:: 3.14 @@ -620,9 +680,14 @@ Security considerations .. index:: pair: http.server; security :class:`SimpleHTTPRequestHandler` will follow symbolic links when handling -requests, this makes it possible for files outside of the specified directory +requests which makes it possible for files outside of the specified directory to be served. +Methods :meth:`BaseHTTPRequestHandler.send_header` and +:meth:`BaseHTTPRequestHandler.send_response_only` assume sanitized input +and do not perform input validation such as checking for the presence of CRLF +sequences. Untrusted input may result in HTTP header injection attacks. + Earlier versions of Python did not scrub control characters from the log messages emitted to stderr from ``python -m http.server`` or the default :class:`BaseHTTPRequestHandler` ``.log_message`` diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index fabea611e0ebcd9..89be225b6baae4c 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -13,7 +13,7 @@ IDLE --- Python editor and shell single: Integrated Development Environment .. - Remember to update Lib/idlelib/help.html with idlelib.help.copy_source() when modifying this file. + Remember to update Lib/idlelib/help.html with idlelib.help.copy_strip() when modifying this file. -------------- @@ -37,6 +37,10 @@ IDLE has the following features: * configuration, browsers, and other dialogs +The IDLE application is implemented in the :mod:`idlelib` package. + +.. include:: ../includes/optional-module.rst + Menus ----- @@ -88,7 +92,7 @@ Save Save As... Save the current window with a Save As dialog. The file saved becomes the - new associated file for the window. (If your file namager is set to hide + new associated file for the window. (If your file manager is set to hide extensions, the current extension will be omitted in the file name box. If the new filename has no '.', '.py' and '.txt' will be added for Python and text files, except that on macOS Aqua,'.py' is added for all files.) @@ -154,7 +158,7 @@ Go to Line Show Completions Open a scrollable list allowing selection of existing names. See - :ref:`Completions ` in the Editing and navigation section below. + :ref:`Completions ` in the Editing and Navigation section below. Expand Word Expand a prefix you have typed to match a full word in the same window; @@ -163,7 +167,7 @@ Expand Word Show Call Tip After an unclosed parenthesis for a function, open a small window with function parameter hints. See :ref:`Calltips ` in the - Editing and navigation section below. + Editing and Navigation section below. Show Surrounding Parens Highlight the surrounding parenthesis. @@ -174,9 +178,9 @@ Format menu (Editor window only) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Format Paragraph - Reformat the current blank-line-delimited paragraph in comment block or - multiline string or selected line in a string. All lines in the - paragraph will be formatted to less than N columns, where N defaults to 72. + Rewrap the text block containing the text insert cursor. + Avoid code lines. See :ref:`Format block` in the + Editing and Navigation section below. Indent Region Shift selected lines right by the indent width (default 4 spaces). @@ -204,9 +208,9 @@ New Indent Width Open a dialog to change indent width. The accepted default by the Python community is 4 spaces. -Strip Trailing Chitespace +Strip Trailing Whitespace Remove trailing space and other whitespace characters after the last - non-whitespace character of a line by applying str.rstrip to each line, + non-whitespace character of a line by applying :meth:`str.rstrip` to each line, including lines within multiline strings. Except for Shell windows, remove extra newlines at the end of the file. @@ -562,6 +566,20 @@ In an editor, import statements have no effect until one runs the file. One might want to run a file after writing import statements, after adding function definitions, or after opening an existing file. +.. _format-block: + +Format block +^^^^^^^^^^^^ + +Reformat Paragraph rewraps a block ('paragraph') of contiguous equally +indented non-blank comments, a similar block of text within a multiline +string, or a selected subset of either. +If needed, add a blank line to separate string from code. +Partial lines in a selection expand to complete lines. +The resulting lines have the same indent as before +but have maximum total length of N columns (characters). +Change the default N of 72 on the Window tab of IDLE Settings. + .. _code-context: Code Context @@ -657,7 +675,9 @@ looked for in the user's home directory. Statements in this file will be executed in the Tk namespace, so this file is not useful for importing functions to be used from IDLE's Python shell. -Command line usage +.. _idlelib-cli: + +Command-line usage ^^^^^^^^^^^^^^^^^^ .. program:: idle diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 9f198aebcb66b0b..131bc0101fb5a50 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -24,12 +24,12 @@ This module defines three classes, :class:`IMAP4`, :class:`IMAP4_SSL` and :class:`IMAP4_stream`, which encapsulate a connection to an IMAP4 server and implement a large subset of the IMAP4rev1 client protocol as defined in -:rfc:`2060`. It is backward compatible with IMAP4 (:rfc:`1730`) servers, but +:rfc:`3501`. It is backward compatible with IMAP4 (:rfc:`1730`) servers, but note that the ``STATUS`` command is not supported in IMAP4. .. include:: ../includes/wasm-notavail.rst -Three classes are provided by the :mod:`imaplib` module, :class:`IMAP4` is the +Three classes are provided by the :mod:`!imaplib` module, :class:`IMAP4` is the base class: @@ -97,6 +97,13 @@ There's also a subclass for secure connections: (potentially long-lived) structure. Please read :ref:`ssl-security` for best practices. + .. note:: + + With the default *ssl_context*, the connection is encrypted but the + server certificate and hostname are not verified. + To verify them, pass a context created by + :func:`ssl.create_default_context`. + The optional *timeout* parameter specifies a timeout in seconds for the connection attempt. If timeout is not given or is ``None``, the global default socket timeout is used. @@ -128,11 +135,11 @@ The second subclass allows for connections created by a child process: The following utility functions are defined: -.. function:: Internaldate2tuple(datestr) +.. function:: Internaldate2tuple(resp) - Parse an IMAP4 ``INTERNALDATE`` string and return corresponding local - time. The return value is a :class:`time.struct_time` tuple or - ``None`` if the string has wrong format. + Parse a :term:`bytes-like object` containing an IMAP4 ``INTERNALDATE`` + response and return the corresponding local time. The return value is a + :class:`time.struct_time` tuple or ``None`` if the input has wrong format. .. function:: Int2AP(num) @@ -140,9 +147,11 @@ The following utility functions are defined: [``A`` .. ``P``]. -.. function:: ParseFlags(flagstr) +.. function:: ParseFlags(resp) - Converts an IMAP4 ``FLAGS`` response to a tuple of individual flags. + Converts a :term:`bytes-like object` containing an IMAP4 ``FLAGS`` response + to a tuple of individual flags as :class:`bytes`. The return value is an + empty tuple if the input has wrong format. .. function:: Time2Internaldate(date_time) @@ -187,6 +196,9 @@ enclosed with either parentheses or double quotes) each string is quoted. However, the *password* argument to the ``LOGIN`` command is always quoted. If you want to avoid having an argument string quoted (eg: the *flags* argument to ``STORE``) then enclose the string in parentheses (eg: ``r'(\Deleted)'``). +In general, pass arguments unquoted and let the module quote them as needed. +An argument that is already enclosed in double quotes is left unchanged, +so that code which quotes arguments itself keeps working. Most commands return a tuple: ``(type, [data, ...])`` where *type* is usually ``'OK'`` or ``'NO'``, and *data* is either the text from the command response, @@ -207,6 +219,11 @@ An :class:`IMAP4` instance has the following methods: Append *message* to named mailbox. + *flags* may be ``None`` or a string of IMAP flag tokens. Multiple + flags are separated by spaces, for example ``r'\Seen \Answered'``. + If *flags* is not already enclosed in parentheses, parentheses are + added automatically. + .. method:: IMAP4.authenticate(mechanism, authobject) @@ -395,7 +412,7 @@ An :class:`IMAP4` instance has the following methods: .. versionadded:: 3.14 -.. method:: IMAP4.list([directory[, pattern]]) +.. method:: IMAP4.list(directory='', pattern='*') List mailbox names in *directory* matching *pattern*. *directory* defaults to the top-level mail folder, and *pattern* defaults to match anything. Returned @@ -413,6 +430,9 @@ An :class:`IMAP4` instance has the following methods: the password. Will only work if the server ``CAPABILITY`` response includes the phrase ``AUTH=CRAM-MD5``. + .. versionchanged:: 3.14 + An :exc:`IMAP4.error` is raised if MD5 support is not available. + .. method:: IMAP4.logout() @@ -422,7 +442,7 @@ An :class:`IMAP4` instance has the following methods: The method no longer ignores silently arbitrary exceptions. -.. method:: IMAP4.lsub(directory='""', pattern='*') +.. method:: IMAP4.lsub(directory='', pattern='*') List subscribed mailbox names in directory matching pattern. *directory* defaults to the top level directory and *pattern* defaults to match any mailbox. @@ -586,6 +606,13 @@ An :class:`IMAP4` instance has the following methods: encryption on the IMAP connection. Please read :ref:`ssl-security` for best practices. + .. note:: + + With the default *ssl_context*, the connection is encrypted but the + server certificate and hostname are not verified. + To verify them, pass a context created by + :func:`ssl.create_default_context`. + .. versionadded:: 3.2 .. versionchanged:: 3.4 @@ -602,7 +629,7 @@ An :class:`IMAP4` instance has the following methods: .. method:: IMAP4.store(message_set, command, flag_list) Alters flag dispositions for messages in mailbox. *command* is specified by - section 6.4.6 of :rfc:`2060` as being one of "FLAGS", "+FLAGS", or "-FLAGS", + section 6.4.6 of :rfc:`3501` as being one of "FLAGS", "+FLAGS", or "-FLAGS", optionally with a suffix of ".SILENT". For example, to set the delete flag on all messages:: @@ -616,11 +643,11 @@ An :class:`IMAP4` instance has the following methods: Creating flags containing ']' (for example: "[test]") violates :rfc:`3501` (the IMAP protocol). However, imaplib has historically - allowed creation of such tags, and popular IMAP servers, such as Gmail, + allowed creation of such flags, and popular IMAP servers, such as Gmail, accept and produce such flags. There are non-Python programs which also - create such tags. Although it is an RFC violation and IMAP clients and + create such flags. Although it is an RFC violation and IMAP clients and servers are supposed to be strict, imaplib still continues to allow - such tags to be created for backward compatibility reasons, and as of + such flags to be created for backward compatibility reasons, and as of Python 3.6, handles them if they are sent from the server, since this improves real-world compatibility. @@ -720,3 +747,10 @@ retrieves and prints all messages:: M.close() M.logout() +.. note:: + + A ``FETCH`` response may contain additional or unsolicited data + (see :rfc:`3501`, section 7.4.2), + so production code should inspect the whole response + rather than rely on ``data[0][1]``. + diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 12014309e26ec93..3d5f4420a4f3823 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -18,11 +18,9 @@ the metadata of an installed `Distribution Package `_\s, modules, if any). Built in part on Python's import system, this library -intends to replace similar functionality in the `entry point -API`_ and `metadata API`_ of ``pkg_resources``. Along with -:mod:`importlib.resources`, -this package can eliminate the need to use the older and less efficient -``pkg_resources`` package. +provides the entry point and metadata APIs that were previously +exposed by the now-removed ``pkg_resources`` package. Along with +:mod:`importlib.resources`, it supersedes ``pkg_resources``. ``importlib.metadata`` operates on third-party *distribution packages* installed into Python's ``site-packages`` directory via tools such as @@ -588,7 +586,3 @@ packages served by the ``DatabaseImporter``, assuming that the The ``DatabaseDistribution`` may also provide other metadata files, like ``RECORD`` (required for ``Distribution.files``) or override the implementation of ``Distribution.files``. See the source for more inspiration. - - -.. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points -.. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api diff --git a/Doc/library/importlib.resources.abc.rst b/Doc/library/importlib.resources.abc.rst index 7a77466bcbaf27d..45979c5691270aa 100644 --- a/Doc/library/importlib.resources.abc.rst +++ b/Doc/library/importlib.resources.abc.rst @@ -49,44 +49,47 @@ .. method:: open_resource(resource) :abstractmethod: - Returns an opened, :term:`file-like object` for binary reading - of the *resource*. + Returns an opened, :term:`file-like object` for binary reading + of the *resource*. - If the resource cannot be found, :exc:`FileNotFoundError` is - raised. + If the resource cannot be found, :exc:`FileNotFoundError` is + raised. .. method:: resource_path(resource) :abstractmethod: - Returns the file system path to the *resource*. + Returns the file system path to the *resource*. - If the resource does not concretely exist on the file system, - raise :exc:`FileNotFoundError`. + If the resource does not concretely exist on the file system, + raise :exc:`FileNotFoundError`. - .. method:: is_resource(name) + .. method:: is_resource(path) :abstractmethod: - Returns ``True`` if the named *name* is considered a resource. - :exc:`FileNotFoundError` is raised if *name* does not exist. + Returns ``True`` if the named *path* is considered a resource. + :exc:`FileNotFoundError` is raised if *path* does not exist. + + .. versionchanged:: 3.10 + The argument *name* was renamed to *path*. .. method:: contents() :abstractmethod: - Returns an :term:`iterable` of strings over the contents of - the package. Do note that it is not required that all names - returned by the iterator be actual resources, e.g. it is - acceptable to return names for which :meth:`is_resource` would - be false. - - Allowing non-resource names to be returned is to allow for - situations where how a package and its resources are stored - are known a priori and the non-resource names would be useful. - For instance, returning subdirectory names is allowed so that - when it is known that the package and resources are stored on - the file system then those subdirectory names can be used - directly. - - The abstract method returns an iterable of no items. + Returns an :term:`iterable` of strings over the contents of + the package. Do note that it is not required that all names + returned by the iterator be actual resources, e.g. it is + acceptable to return names for which :meth:`is_resource` would + be false. + + Allowing non-resource names to be returned is to allow for + situations where how a package and its resources are stored + are known a priori and the non-resource names would be useful. + For instance, returning subdirectory names is allowed so that + when it is known that the package and resources are stored on + the file system then those subdirectory names can be used + directly. + + The abstract method returns an iterable of no items. .. class:: Traversable diff --git a/Doc/library/importlib.resources.rst b/Doc/library/importlib.resources.rst index e002198899c8b8b..0bebd974d3ad308 100644 --- a/Doc/library/importlib.resources.rst +++ b/Doc/library/importlib.resources.rst @@ -16,11 +16,12 @@ within *packages*. "Resources" are file-like resources associated with a module or package in Python. The resources may be contained directly in a package, within a subdirectory contained in that package, or adjacent to modules outside a -package. Resources may be text or binary. As a result, Python module sources -(.py) of a package and compilation artifacts (pycache) are technically -de-facto resources of that package. In practice, however, resources are -primarily those non-Python artifacts exposed specifically by the package -author. +package. Resources may be text or binary. As a result, a package's Python +module sources (.py), compilation artifacts (pycache), and installation +artifacts (like :func:`reserved filenames ` +in directories) are technically de-facto resources of that package. +In practice, however, resources are primarily those non-Python artifacts +exposed specifically by the package author. Resources can be opened or read in either binary or text mode. @@ -30,15 +31,13 @@ not** have to exist as physical files and directories on the file system: for example, a package and its resources can be imported from a zip file using :py:mod:`zipimport`. -.. note:: +.. warning:: + + :mod:`importlib.resources` follows the same security model as the built-in + :func:`open` function. Passing untrusted inputs to the functions + in this module is unsafe. - This module provides functionality similar to `pkg_resources - `_ `Basic - Resource Access - `_ - without the performance overhead of that package. This makes reading - resources included in packages easier, with more stable and consistent - semantics. +.. note:: The standalone backport of this module provides more information on `using importlib.resources @@ -238,7 +237,6 @@ For all the following functions: .. versionchanged:: 3.13 Multiple *path_names* are accepted. - *encoding* and *errors* must be given as keyword arguments. .. function:: is_resource(anchor, *path_names) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index ea5a77028683b36..2e65eff83f51a41 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -17,7 +17,7 @@ Introduction ------------ -The purpose of the :mod:`importlib` package is three-fold. +The purpose of the :mod:`!importlib` package is three-fold. One is to provide the implementation of the :keyword:`import` statement (and thus, by extension, the @@ -206,9 +206,13 @@ Functions :exc:`ModuleNotFoundError` is raised when the module being reloaded lacks a :class:`~importlib.machinery.ModuleSpec`. + .. warning:: + This function is not thread-safe. Calling it from multiple threads can result + in unexpected behavior. It's recommended to use the :class:`threading.Lock` + or other synchronization primitives for thread-safe module reloading. -:mod:`importlib.abc` -- Abstract base classes related to import ---------------------------------------------------------------- +:mod:`!importlib.abc` -- Abstract base classes related to import +---------------------------------------------------------------- .. module:: importlib.abc :synopsis: Abstract base classes related to import @@ -218,7 +222,7 @@ Functions -------------- -The :mod:`importlib.abc` module contains all of the core abstract base classes +The :mod:`!importlib.abc` module contains all of the core abstract base classes used by :keyword:`import`. Some subclasses of the core abstract base classes are also provided to help in implementing the core ABCs. @@ -389,6 +393,8 @@ ABC hierarchy:: .. deprecated:: 3.7 This ABC is deprecated in favour of supporting resource loading through :class:`importlib.resources.abc.TraversableResources`. + This class exists for backwards compatibility only with other ABCs in + this module. .. method:: get_data(path) :abstractmethod: @@ -631,174 +637,8 @@ ABC hierarchy:: itself does not end in ``__init__``. -.. class:: ResourceReader - - *Superseded by TraversableResources* - - An :term:`abstract base class` to provide the ability to read - *resources*. - - From the perspective of this ABC, a *resource* is a binary - artifact that is shipped within a package. Typically this is - something like a data file that lives next to the ``__init__.py`` - file of the package. The purpose of this class is to help abstract - out the accessing of such data files so that it does not matter if - the package and its data file(s) are stored e.g. in a zip file - versus on the file system. - - For any of methods of this class, a *resource* argument is - expected to be a :term:`path-like object` which represents - conceptually just a file name. This means that no subdirectory - paths should be included in the *resource* argument. This is - because the location of the package the reader is for, acts as the - "directory". Hence the metaphor for directories and file - names is packages and resources, respectively. This is also why - instances of this class are expected to directly correlate to - a specific package (instead of potentially representing multiple - packages or a module). - - Loaders that wish to support resource reading are expected to - provide a method called ``get_resource_reader(fullname)`` which - returns an object implementing this ABC's interface. If the module - specified by fullname is not a package, this method should return - :const:`None`. An object compatible with this ABC should only be - returned when the specified module is a package. - - .. versionadded:: 3.7 - - .. deprecated-removed:: 3.12 3.14 - Use :class:`importlib.resources.abc.TraversableResources` instead. - - .. method:: open_resource(resource) - :abstractmethod: - - Returns an opened, :term:`file-like object` for binary reading - of the *resource*. - - If the resource cannot be found, :exc:`FileNotFoundError` is - raised. - - .. method:: resource_path(resource) - :abstractmethod: - - Returns the file system path to the *resource*. - - If the resource does not concretely exist on the file system, - raise :exc:`FileNotFoundError`. - - .. method:: is_resource(name) - :abstractmethod: - - Returns ``True`` if the named *name* is considered a resource. - :exc:`FileNotFoundError` is raised if *name* does not exist. - - .. method:: contents() - :abstractmethod: - - Returns an :term:`iterable` of strings over the contents of - the package. Do note that it is not required that all names - returned by the iterator be actual resources, e.g. it is - acceptable to return names for which :meth:`is_resource` would - be false. - - Allowing non-resource names to be returned is to allow for - situations where how a package and its resources are stored - are known a priori and the non-resource names would be useful. - For instance, returning subdirectory names is allowed so that - when it is known that the package and resources are stored on - the file system then those subdirectory names can be used - directly. - - The abstract method returns an iterable of no items. - - -.. class:: Traversable - - An object with a subset of :class:`pathlib.Path` methods suitable for - traversing directories and opening files. - - For a representation of the object on the file-system, use - :meth:`importlib.resources.as_file`. - - .. versionadded:: 3.9 - - .. deprecated-removed:: 3.12 3.14 - Use :class:`importlib.resources.abc.Traversable` instead. - - .. attribute:: name - - Abstract. The base name of this object without any parent references. - - .. method:: iterdir() - :abstractmethod: - - Yield ``Traversable`` objects in ``self``. - - .. method:: is_dir() - :abstractmethod: - - Return ``True`` if ``self`` is a directory. - - .. method:: is_file() - :abstractmethod: - - Return ``True`` if ``self`` is a file. - - .. method:: joinpath(child) - :abstractmethod: - - Return Traversable child in ``self``. - - .. method:: __truediv__(child) - :abstractmethod: - - Return ``Traversable`` child in ``self``. - - .. method:: open(mode='r', *args, **kwargs) - :abstractmethod: - - *mode* may be 'r' or 'rb' to open as text or binary. Return a handle - suitable for reading (same as :attr:`pathlib.Path.open`). - - When opening as text, accepts encoding parameters such as those - accepted by :class:`io.TextIOWrapper`. - - .. method:: read_bytes() - - Read contents of ``self`` as bytes. - - .. method:: read_text(encoding=None) - - Read contents of ``self`` as text. - - -.. class:: TraversableResources - - An abstract base class for resource readers capable of serving - the :meth:`importlib.resources.files` interface. Subclasses - :class:`importlib.resources.abc.ResourceReader` and provides - concrete implementations of the :class:`importlib.resources.abc.ResourceReader`'s - abstract methods. Therefore, any loader supplying - :class:`importlib.abc.TraversableResources` also supplies ResourceReader. - - Loaders that wish to support resource reading are expected to - implement this interface. - - .. versionadded:: 3.9 - - .. deprecated-removed:: 3.12 3.14 - Use :class:`importlib.resources.abc.TraversableResources` instead. - - .. method:: files() - :abstractmethod: - - Returns a :class:`importlib.resources.abc.Traversable` object for the loaded - package. - - - -:mod:`importlib.machinery` -- Importers and path hooks ------------------------------------------------------- +:mod:`!importlib.machinery` -- Importers and path hooks +------------------------------------------------------- .. module:: importlib.machinery :synopsis: Importers and path hooks @@ -1247,7 +1087,7 @@ find and load modules. To accommodate this requirement, when running on iOS, extension module binaries are *not* packaged as ``.so`` files on ``sys.path``, but as individual standalone frameworks. To discover those frameworks, this loader - is be registered against the ``.fwork`` file extension, with a ``.fwork`` + is registered against the ``.fwork`` file extension, with a ``.fwork`` file acting as a placeholder in the original location of the binary on ``sys.path``. The ``.fwork`` file contains the path of the actual binary in the ``Frameworks`` folder, relative to the app bundle. To allow for @@ -1296,8 +1136,8 @@ find and load modules. Path to the ``.fwork`` file for the extension module. -:mod:`importlib.util` -- Utility code for importers ---------------------------------------------------- +:mod:`!importlib.util` -- Utility code for importers +---------------------------------------------------- .. module:: importlib.util :synopsis: Utility code for importers diff --git a/Doc/library/index.rst b/Doc/library/index.rst index 44b218948d07e1a..8fc77be520d4268 100644 --- a/Doc/library/index.rst +++ b/Doc/library/index.rst @@ -43,6 +43,7 @@ the `Python Package Index `_. constants.rst stdtypes.rst exceptions.rst + threadsafety.rst text.rst binary.rst @@ -63,7 +64,6 @@ the `Python Package Index `_. internet.rst mm.rst i18n.rst - frameworks.rst tk.rst development.rst debug.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index e8d1176f4778667..a83a1328e1fb17f 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -16,7 +16,7 @@ -------------- -The :mod:`inspect` module provides several useful functions to help get +The :mod:`!inspect` module provides several useful functions to help get information about live objects such as modules, classes, methods, functions, tracebacks, frame objects, and code objects. For example, it can help you examine the contents of a class, retrieve the source code of a method, extract @@ -253,6 +253,9 @@ attributes (see :ref:`import-mod-attrs` for module attributes): +-----------------+-------------------+---------------------------+ | | gi_running | is the generator running? | +-----------------+-------------------+---------------------------+ +| | gi_suspended | is the generator | +| | | suspended? | ++-----------------+-------------------+---------------------------+ | | gi_code | code | +-----------------+-------------------+---------------------------+ | | gi_yieldfrom | object being iterated by | @@ -270,6 +273,9 @@ attributes (see :ref:`import-mod-attrs` for module attributes): +-----------------+-------------------+---------------------------+ | | ag_running | is the generator running? | +-----------------+-------------------+---------------------------+ +| | ag_suspended | is the generator | +| | | suspended? | ++-----------------+-------------------+---------------------------+ | | ag_code | code | +-----------------+-------------------+---------------------------+ | coroutine | __name__ | name | @@ -283,6 +289,9 @@ attributes (see :ref:`import-mod-attrs` for module attributes): +-----------------+-------------------+---------------------------+ | | cr_running | is the coroutine running? | +-----------------+-------------------+---------------------------+ +| | cr_suspended | is the coroutine | +| | | suspended? | ++-----------------+-------------------+---------------------------+ | | cr_code | code | +-----------------+-------------------+---------------------------+ | | cr_origin | where coroutine was | @@ -316,6 +325,18 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Add ``__builtins__`` attribute to functions. +.. versionchanged:: 3.11 + + Add ``gi_suspended`` attribute to generators. + +.. versionchanged:: 3.11 + + Add ``cr_suspended`` attribute to coroutines. + +.. versionchanged:: 3.12 + + Add ``ag_suspended`` attribute to async generators. + .. versionchanged:: 3.14 Add ``f_generator`` attribute to frames. @@ -378,17 +399,47 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object is a class, whether built-in or created in Python code. + This function returns ``False`` for :ref:`generic aliases ` of classes, + such as ``list[int]``. + .. function:: ismethod(object) Return ``True`` if the object is a bound method written in Python. + .. note:: -.. function:: ispackage(object) + For example, given this class:: - Return ``True`` if the object is a :term:`package`. + >>> class Greeter: + ... def say_hello(self): + ... print('hello!') - .. versionadded:: 3.14 + A bound method (also known as an *instance method*) is created when + accessing ``say_hello`` (a :term:`function` defined in the + ``Greeter`` namespace) through an instance of the ``Greeter`` class:: + + >>> instance = Greeter() + + >>> instance.say_hello + > + >>> ismethod(instance.say_hello) + True + >>> isfunction(instance.say_hello) + False + + Accessing ``say_hello`` through the ``Greeter`` class will return the + function itself. For this function, :func:`ismethod` will return + ``False``, but :func:`isfunction` will return ``True``:: + + >>> Greeter.say_hello + + >>> ismethod(Greeter.say_hello) + False + >>> isfunction(Greeter.say_hello) + True + + See :ref:`typesmethods` for details. .. function:: isfunction(object) @@ -396,11 +447,23 @@ attributes (see :ref:`import-mod-attrs` for module attributes): Return ``True`` if the object is a Python function, which includes functions created by a :term:`lambda` expression. + See the note for :func:`~inspect.ismethod` for an example. + + +.. function:: ispackage(object) + + Return ``True`` if the object is a :term:`package`. + + .. versionadded:: 3.14 + .. function:: isgeneratorfunction(object) Return ``True`` if the object is a Python generator function. + It also returns ``True`` for bound methods created from Python generator functions + (see :ref:`typesmethods` for more information). + .. versionchanged:: 3.8 Functions wrapped in :func:`functools.partial` now return ``True`` if the wrapped function is a Python generator function. @@ -503,7 +566,7 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. versionchanged:: 3.13 Functions wrapped in :func:`functools.partialmethod` now return ``True`` - if the wrapped function is a :term:`coroutine function`. + if the wrapped function is a :term:`asynchronous generator` function. .. function:: isasyncgen(object) @@ -555,8 +618,7 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. function:: ismethoddescriptor(object) Return ``True`` if the object is a method descriptor, but not if - :func:`ismethod`, :func:`isclass`, :func:`isfunction` or :func:`isbuiltin` - are true. + :func:`isclass`, :func:`ismethod` or :func:`isfunction` is true. This, for example, is true of ``int.__add__``. An object passing this test has a :meth:`~object.__get__` method, but not a :meth:`~object.__set__` @@ -564,10 +626,10 @@ attributes (see :ref:`import-mod-attrs` for module attributes): attributes varies. A :attr:`~definition.__name__` attribute is usually sensible, and :attr:`~definition.__doc__` often is. - Methods implemented via descriptors that also pass one of the other tests - return ``False`` from the :func:`ismethoddescriptor` test, simply because the - other tests promise more -- you can, e.g., count on having the - :attr:`~method.__func__` attribute (etc) when an object passes + Method descriptors that also pass any of the other tests (:func:`!isclass`, + :func:`!ismethod` or :func:`!isfunction`) make this function return ``False``, + simply because those other tests promise more -- you can, for example, count + on having the :attr:`~method.__func__` attribute when an object passes :func:`ismethod`. .. versionchanged:: 3.13 @@ -578,16 +640,28 @@ attributes (see :ref:`import-mod-attrs` for module attributes): .. function:: isdatadescriptor(object) - Return ``True`` if the object is a data descriptor. + Return ``True`` if the object is a data descriptor, but not if + :func:`isclass`, :func:`ismethod` or :func:`isfunction` is true. + + Data descriptors always have a :meth:`~object.__set__` method and/or + a :meth:`~object.__delete__` method. Optionally, they may also have a + :meth:`~object.__get__` method. - Data descriptors have a :attr:`~object.__set__` or a :attr:`~object.__delete__` method. - Examples are properties (defined in Python), getsets, and members. The - latter two are defined in C and there are more specific tests available for - those types, which is robust across Python implementations. Typically, data - descriptors will also have :attr:`~definition.__name__` and :attr:`!__doc__` attributes - (properties, getsets, and members have both of these attributes), but this is - not guaranteed. + Examples of data descriptors are :func:`properties `, getsets and + member descriptors. Note that for the latter two (defined only in C extension + modules), more specific tests are available: :func:`isgetsetdescriptor` and + :func:`ismemberdescriptor`, respectively. + While data descriptors may also have :attr:`~definition.__name__` and + :attr:`!__doc__` attributes (as properties, getsets and member descriptors + do), this is not necessarily the case in general. + + .. versionchanged:: 3.8 + This function now reports objects with only a :meth:`~object.__set__` method + as being data descriptors (the presence of :meth:`~object.__get__` is no + longer required for that). Moreover, objects with :meth:`~object.__delete__`, + but not :meth:`~object.__set__`, are now properly recognized as data + descriptors as well, which was not the case previously. .. function:: isgetsetdescriptor(object) @@ -1179,7 +1253,7 @@ Classes and functions :func:`signature` in Python 3.5, but that decision has been reversed in order to restore a clearly supported standard interface for single-source Python 2/3 code migrating away from the legacy - :func:`getargspec` API. + :func:`!getargspec` API. .. versionchanged:: 3.7 Python only explicitly guaranteed that it preserved the declaration @@ -1289,6 +1363,11 @@ Classes and functions This is an alias for :func:`annotationlib.get_annotations`; see the documentation of that function for more information. + .. caution:: + + This function may execute arbitrary code contained in annotations. + See :ref:`annotationlib-security` for more information. + .. versionadded:: 3.10 .. versionchanged:: 3.14 @@ -1506,10 +1585,11 @@ properties, will be invoked and :meth:`~object.__getattr__` and may be called. For cases where you want passive introspection, like documentation tools, this -can be inconvenient. :func:`getattr_static` has the same signature as :func:`getattr` +can be inconvenient. :func:`getattr_static` has a similar signature as :func:`getattr` but avoids executing code when it fetches attributes. -.. function:: getattr_static(obj, attr, default=None) +.. function:: getattr_static(obj, attr) + getattr_static(obj, attr, default) Retrieve attributes without triggering dynamic lookup via the descriptor protocol, :meth:`~object.__getattr__` @@ -1729,7 +1809,7 @@ which is a bitmap of the following flags: The flags are specific to CPython, and may not be defined in other Python implementations. Furthermore, the flags are an implementation detail, and can be removed or deprecated in future Python releases. - It's recommended to use public APIs from the :mod:`inspect` module + It's recommended to use public APIs from the :mod:`!inspect` module for any introspection needs. @@ -1768,10 +1848,10 @@ Buffer flags .. _inspect-module-cli: -Command Line Interface +Command-line interface ---------------------- -The :mod:`inspect` module also provides a basic introspection capability +The :mod:`!inspect` module also provides a basic introspection capability from the command line. .. program:: inspect diff --git a/Doc/library/io.rst b/Doc/library/io.rst index 3aa2f35f05eba0d..7f72e72b3c86f96 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -24,7 +24,7 @@ Overview .. index:: single: file object; io module -The :mod:`io` module provides Python's main facilities for dealing with various +The :mod:`!io` module provides Python's main facilities for dealing with various types of I/O. There are three main types of I/O: *text I/O*, *binary I/O* and *raw I/O*. These are generic categories, and various backing stores can be used for each of them. A concrete object belonging to any of these @@ -46,6 +46,7 @@ will raise a :exc:`TypeError`. So will giving a :class:`bytes` object to the Operations that used to raise :exc:`IOError` now raise :exc:`OSError`, since :exc:`IOError` is now an alias of :exc:`OSError`. +.. _text-io: Text I/O ^^^^^^^^ @@ -73,6 +74,7 @@ In-memory text streams are also available as :class:`StringIO` objects:: The text stream API is described in detail in the documentation of :class:`TextIOBase`. +.. _binary-io: Binary I/O ^^^^^^^^^^ @@ -111,6 +113,13 @@ stream by opening a file in binary mode with buffering disabled:: The raw stream API is described in detail in the docs of :class:`RawIOBase`. +.. warning:: + Raw I/O is a low-level interface and methods generally must have their return + values checked and be explicitly retried to ensure an operation completes. + For instance :meth:`~RawIOBase.write` returns the number of bytes written + which may be less than the number of bytes provided (a partial write). + High-level I/O objects like :ref:`binary-io` and :ref:`text-io` implement + retry behavior. .. _io-text-encoding: @@ -292,7 +301,7 @@ interface to a buffered raw stream (:class:`BufferedIOBase`). Finally, Argument names are not part of the specification, and only the arguments of :func:`open` are intended to be used as keyword arguments. -The following table summarizes the ABCs provided by the :mod:`io` module: +The following table summarizes the ABCs provided by the :mod:`!io` module: .. tabularcolumns:: |l|l|L|L| @@ -486,8 +495,11 @@ I/O Base Classes Read up to *size* bytes from the object and return them. As a convenience, if *size* is unspecified or -1, all bytes until EOF are returned. - Otherwise, only one system call is ever made. Fewer than *size* bytes may - be returned if the operating system call returns fewer than *size* bytes. + + Attempts to make only one system call but will retry if interrupted and + the signal handler does not raise an exception (see :pep:`475` for the + rationale). This means fewer than *size* bytes may be returned if the + operating system call returns fewer than *size* bytes. If 0 bytes are returned, and *size* was not 0, this indicates end of file. If the object is in non-blocking mode and no bytes are available, @@ -501,13 +513,19 @@ I/O Base Classes Read and return all the bytes from the stream until EOF, using multiple calls to the stream if necessary. + If ``0`` bytes are returned this indicates end of file. If the object is in + non-blocking mode and the underlying :meth:`read` returns ``None`` + indicating no bytes are available, ``None`` is returned. + .. method:: readinto(b, /) Read bytes into a pre-allocated, writable :term:`bytes-like object` *b*, and return the number of bytes read. For example, *b* might be a :class:`bytearray`. - If the object is in non-blocking mode and no bytes - are available, ``None`` is returned. + + If ``0`` is returned and ``len(b)`` is not ``0``, this indicates end of file. If + the object is in non-blocking mode and no bytes are available, ``None`` is + returned. .. method:: write(b, /) @@ -521,6 +539,13 @@ I/O Base Classes this method returns, so the implementation should only access *b* during the method call. + .. warning:: + + This function does not ensure all bytes are written or an exception is + thrown. Callers may implement that behavior by checking the return + value and, if it is less than the length of *b*, looping with additional + write calls until all unwritten bytes are written. High-level I/O + objects like :ref:`binary-io` and :ref:`text-io` implement retry behavior. .. class:: BufferedIOBase @@ -528,14 +553,13 @@ I/O Base Classes It inherits from :class:`IOBase`. The main difference with :class:`RawIOBase` is that methods :meth:`read`, - :meth:`readinto` and :meth:`write` will try (respectively) to read as much - input as requested or to consume all given output, at the expense of - making perhaps more than one system call. + :meth:`readinto` and :meth:`write` will try (respectively) to read + as much input as requested or to emit all provided data. - In addition, those methods can raise :exc:`BlockingIOError` if the - underlying raw stream is in non-blocking mode and cannot take or give - enough data; unlike their :class:`RawIOBase` counterparts, they will - never return ``None``. + In addition, if the underlying raw stream is in non-blocking mode, when the + system returns would block :meth:`write` will raise :exc:`BlockingIOError` + with :attr:`BlockingIOError.characters_written` and :meth:`read` will return + data read so far or ``None`` if no data is available. Besides, the :meth:`read` method does not have a default implementation that defers to :meth:`readinto`. @@ -568,29 +592,40 @@ I/O Base Classes .. method:: read(size=-1, /) - Read and return up to *size* bytes. If the argument is omitted, ``None``, - or negative, data is read and returned until EOF is reached. An empty - :class:`bytes` object is returned if the stream is already at EOF. + Read and return up to *size* bytes. If the argument is omitted, ``None``, + or negative read as much as possible. - If the argument is positive, and the underlying raw stream is not - interactive, multiple raw reads may be issued to satisfy the byte count - (unless EOF is reached first). But for interactive raw streams, at most - one raw read will be issued, and a short result does not imply that EOF is - imminent. + Fewer bytes may be returned than requested. An empty :class:`bytes` object + is returned if the stream is already at EOF. More than one read may be + made and calls may be retried if specific errors are encountered, see + :meth:`os.read` and :pep:`475` for more details. Less than size bytes + being returned does not imply that EOF is imminent. - A :exc:`BlockingIOError` is raised if the underlying raw stream is in - non blocking-mode, and has no data available at the moment. + When reading as much as possible the default implementation will use + ``raw.readall`` if available (which should implement + :meth:`RawIOBase.readall`), otherwise will read in a loop until read + returns ``None``, an empty :class:`bytes`, or a non-retryable error. For + most streams this is to EOF, but for non-blocking streams more data may + become available. + + .. note:: + + When the underlying raw stream is non-blocking, implementations may + either raise :exc:`BlockingIOError` or return ``None`` if no data is + available. :mod:`!io` implementations return ``None``. .. method:: read1(size=-1, /) - Read and return up to *size* bytes, with at most one call to the - underlying raw stream's :meth:`~RawIOBase.read` (or - :meth:`~RawIOBase.readinto`) method. This can be useful if you are - implementing your own buffering on top of a :class:`BufferedIOBase` - object. + Read and return up to *size* bytes, calling :meth:`~RawIOBase.readinto` + which may retry if :py:const:`~errno.EINTR` is encountered per + :pep:`475`. If *size* is ``-1`` or not provided, the implementation will + choose an arbitrary value for *size*. + + .. note:: - If *size* is ``-1`` (the default), an arbitrary number of bytes are - returned (more than zero unless EOF is reached). + When the underlying raw stream is non-blocking, implementations may + either raise :exc:`BlockingIOError` or return ``None`` if no data is + available. :mod:`!io` implementations return ``None``. .. method:: readinto(b, /) @@ -639,7 +674,11 @@ Raw File I/O .. class:: FileIO(name, mode='r', closefd=True, opener=None) A raw binary stream representing an OS-level file containing bytes data. It - inherits from :class:`RawIOBase`. + inherits from :class:`RawIOBase` and implements its low-level access design. + This means :meth:`~RawIOBase.write` does not guarantee all bytes are written + and :meth:`~RawIOBase.read` may read less bytes than requested even when more + bytes may be present in the underlying file. To get "write all" and + "read at least" behavior, use :ref:`binary-io`. The *name* can be one of two things: @@ -659,10 +698,6 @@ Raw File I/O implies writing, so this mode behaves in a similar way to ``'w'``. Add a ``'+'`` to the mode to allow simultaneous reading and writing. - The :meth:`~RawIOBase.read` (when called with a positive argument), - :meth:`~RawIOBase.readinto` and :meth:`~RawIOBase.write` methods on this - class will only make one system call. - A custom opener can be used by passing a callable as *opener*. The underlying file descriptor for the file object is then obtained by calling *opener* with (*name*, *flags*). *opener* must return an open file descriptor (passing @@ -674,6 +709,13 @@ Raw File I/O See the :func:`open` built-in function for examples on using the *opener* parameter. + .. warning:: + :class:`FileIO` is a low-level I/O object and members, such as + :meth:`~RawIOBase.read` and :meth:`~RawIOBase.write`, need to have their + return values checked explicitly in a retry loop to implement "write all" + and "read at least" behavior. High-level I/O objects :ref:`binary-io` and + :ref:`text-io` implement retry behavior. + .. versionchanged:: 3.3 The *opener* parameter was added. The ``'x'`` mode was added. @@ -767,34 +809,21 @@ than raw I/O does. .. method:: peek(size=0, /) - Return bytes from the stream without advancing the position. At most one - single read on the raw stream is done to satisfy the call. The number of - bytes returned may be less or more than requested. + Return bytes from the stream without advancing the position. The number of + bytes returned may be less or more than requested. If the underlying raw + stream is non-blocking and the operation would block, returns empty bytes. .. method:: read(size=-1, /) - Read and return *size* bytes, or if *size* is not given or negative, until - EOF or if the read call would block in non-blocking mode. - - .. note:: - - When the underlying raw stream is non-blocking, a :exc:`BlockingIOError` - may be raised if a read operation cannot be completed immediately. + In :class:`BufferedReader` this is the same as :meth:`io.BufferedIOBase.read` .. method:: read1(size=-1, /) - Read and return up to *size* bytes with only one call on the raw stream. - If at least one byte is buffered, only buffered bytes are returned. - Otherwise, one raw stream read call is made. + In :class:`BufferedReader` this is the same as :meth:`io.BufferedIOBase.read1` .. versionchanged:: 3.7 The *size* argument is now optional. - .. note:: - - When the underlying raw stream is non-blocking, a :exc:`BlockingIOError` - may be raised if a read operation cannot be completed immediately. - .. class:: BufferedWriter(raw, buffer_size=DEFAULT_BUFFER_SIZE) A buffered binary stream providing higher-level access to a writeable, non @@ -826,15 +855,15 @@ than raw I/O does. Write the :term:`bytes-like object`, *b*, and return the number of bytes written. When in non-blocking mode, a - :exc:`BlockingIOError` is raised if the buffer needs to be written out but - the raw stream blocks. + :exc:`BlockingIOError` with :attr:`BlockingIOError.characters_written` set + is raised if the buffer needs to be written out but the raw stream blocks. .. class:: BufferedRandom(raw, buffer_size=DEFAULT_BUFFER_SIZE) - A buffered binary stream providing higher-level access to a seekable - :class:`RawIOBase` raw binary stream. It inherits from :class:`BufferedReader` - and :class:`BufferedWriter`. + A buffered binary stream implementing :class:`BufferedIOBase` interfaces + providing higher-level access to a seekable :class:`RawIOBase` raw binary + stream. The constructor creates a reader and writer for a seekable raw stream, given in the first argument. If the *buffer_size* is omitted it defaults to @@ -894,9 +923,10 @@ Text I/O .. attribute:: buffer - The underlying binary buffer (a :class:`BufferedIOBase` instance) that - :class:`TextIOBase` deals with. This is not part of the - :class:`TextIOBase` API and may not exist in some implementations. + The underlying binary buffer (a :class:`BufferedIOBase` + or :class:`RawIOBase` instance) that :class:`TextIOBase` deals with. + This is not part of the :class:`TextIOBase` API and may not exist + in some implementations. .. method:: detach() diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index e5bdfbb144b65a6..45256d29e854d09 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -10,7 +10,7 @@ -------------- -:mod:`ipaddress` provides the capabilities to create, manipulate and +:mod:`!ipaddress` provides the capabilities to create, manipulate and operate on IPv4 and IPv6 addresses and networks. The functions and classes in this module make it straightforward to handle @@ -34,7 +34,7 @@ This is the full module API reference—for an overview and introduction, see Convenience factory functions ----------------------------- -The :mod:`ipaddress` module provides factory functions to conveniently create +The :mod:`!ipaddress` module provides factory functions to conveniently create IP addresses, networks and interfaces: .. function:: ip_address(address) @@ -240,7 +240,16 @@ write code that handles both IP versions correctly. Address objects are .. attribute:: is_reserved - ``True`` if the address is otherwise IETF reserved. + ``True`` if the address is noted as reserved by the IETF. + For IPv4, this is only ``240.0.0.0/4``, the ``Reserved`` address block. + For IPv6, this is all addresses `allocated `__ as + ``Reserved by IETF`` for future use. + + .. note:: For IPv4, ``is_reserved`` is not related to the address block value of the + ``Reserved-by-Protocol`` column in iana-ipv4-special-registry_. + + .. caution:: For IPv6, ``fec0::/10`` a former Site-Local scoped address prefix is + currently excluded from that list (see :attr:`~IPv6Address.is_site_local` & :rfc:`3879`). .. attribute:: is_loopback @@ -261,6 +270,7 @@ write code that handles both IP versions correctly. Address objects are .. _iana-ipv4-special-registry: https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml .. _iana-ipv6-special-registry: https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml +.. _iana-ipv6-address-space: https://www.iana.org/assignments/ipv6-address-space/ipv6-address-space.xhtml .. method:: IPv4Address.__format__(fmt) @@ -359,9 +369,9 @@ write code that handles both IP versions correctly. Address objects are .. attribute:: ipv4_mapped - For addresses that appear to be IPv4 mapped addresses (starting with - ``::FFFF/96``), this property will report the embedded IPv4 address. - For any other address, this property will be ``None``. + For addresses that appear to be IPv4 mapped addresses in the range + ``::FFFF:0:0/96`` as defined by :RFC:`4291`, this property reports the + embedded IPv4 address. For any other address, this property will be ``None``. .. attribute:: scope_id @@ -1017,7 +1027,7 @@ The module also provides the following module level functions: IPv4Address('192.0.2.0') <= IPv4Network('192.0.2.0/24') doesn't make sense. There are some times however, where you may wish to - have :mod:`ipaddress` sort these anyway. If you need to do this, you can use + have :mod:`!ipaddress` sort these anyway. If you need to do this, you can use this function as the *key* argument to :func:`sorted`. *obj* is either a network or address object. diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 00925ae920aad9f..8bfe5ac31e8990d 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -30,18 +30,7 @@ For instance, SML provides a tabulation tool: ``tabulate(f)`` which produces a sequence ``f(0), f(1), ...``. The same effect can be achieved in Python by combining :func:`map` and :func:`count` to form ``map(f, count())``. - -**Infinite iterators:** - -================== ================= ================================================= ========================================= -Iterator Arguments Results Example -================== ================= ================================================= ========================================= -:func:`count` [start[, step]] start, start+step, start+2*step, ... ``count(10) → 10 11 12 13 14 ...`` -:func:`cycle` p p0, p1, ... plast, p0, p1, ... ``cycle('ABCD') → A B C D A B C D ...`` -:func:`repeat` elem [,n] elem, elem, elem, ... endlessly or up to n times ``repeat(10, 3) → 10 10 10`` -================== ================= ================================================= ========================================= - -**Iterators terminating on the shortest input sequence:** +**General iterators:** ============================ ============================ ================================================= ============================================================= Iterator Arguments Results Example @@ -51,11 +40,14 @@ Iterator Arguments Results :func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') → A B C D E F`` :func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) → A B C D E F`` :func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) → A C E F`` +:func:`count` [start[, step]] start, start+step, start+2*step, ... ``count(10) → 10 11 12 13 14 ...`` +:func:`cycle` p p0, p1, ... plast, p0, p1, ... ``cycle('ABCD') → A B C D A B C D ...`` :func:`dropwhile` predicate, seq seq[n], seq[n+1], starting when predicate fails ``dropwhile(lambda x: x<5, [1,4,6,3,8]) → 6 3 8`` :func:`filterfalse` predicate, seq elements of seq where predicate(elem) fails ``filterfalse(lambda x: x<5, [1,4,6,3,8]) → 6 8`` :func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) ``groupby(['A','B','DEF'], len) → (1, A B) (3, DEF)`` :func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) → C D E F G`` :func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') → AB BC CD DE EF FG`` +:func:`repeat` elem [,n] elem, elem, elem, ... endlessly or up to n times ``repeat(10, 3) → 10 10 10`` :func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) → 32 9 1000`` :func:`takewhile` predicate, seq seq[0], seq[1], until predicate fails ``takewhile(lambda x: x<5, [1,4,6,3,8]) → 1 4`` :func:`tee` it, n it1, it2, ... itn splits one iterator into n ``tee('ABC', 2) → A B C, A B C`` @@ -819,7 +811,7 @@ well as with the built-in itertools such as ``map()``, ``filter()``, A secondary purpose of the recipes is to serve as an incubator. The ``accumulate()``, ``compress()``, and ``pairwise()`` itertools started out as -recipes. Currently, the ``sliding_window()``, ``iter_index()``, and ``sieve()`` +recipes. Currently, the ``sliding_window()``, ``derangements()``, and ``sieve()`` recipes are being tested to see whether they prove their worth. Substantially all of these recipes and many, many others can be installed from @@ -838,11 +830,18 @@ and :term:`generators ` which incur interpreter overhead. .. testcode:: + from itertools import (accumulate, batched, chain, combinations, compress, + count, cycle, filterfalse, groupby, islice, permutations, product, + repeat, starmap, tee, zip_longest) from collections import Counter, deque from contextlib import suppress from functools import reduce - from math import comb, prod, sumprod, isqrt - from operator import itemgetter, getitem, mul, neg + from heapq import heappush, heappushpop, heappush_max, heappushpop_max + from math import comb, isqrt, prod, sumprod + from operator import getitem, is_not, itemgetter, mul, neg, truediv + + + # ==== Basic one liners ==== def take(n, iterable): "Return first n items of the iterable as a list." @@ -853,10 +852,6 @@ and :term:`generators ` which incur interpreter overhead. # prepend(1, [2, 3, 4]) → 1 2 3 4 return chain([value], iterable) - def tabulate(function, start=0): - "Return function(0), function(1), ..." - return map(function, count(start)) - def repeatfunc(function, times=None, *args): "Repeat calls to a function with specified arguments." if times is None: @@ -899,8 +894,8 @@ and :term:`generators ` which incur interpreter overhead. def first_true(iterable, default=False, predicate=None): "Returns the first true value or the *default* if there is no true value." - # first_true([a,b,c], x) → a or b or c or x - # first_true([a,b], x, f) → a if f(a) else b if f(b) else x + # first_true([a, b, c], x) → a or b or c or x + # first_true([a, b], x, f) → a if f(a) else b if f(b) else x return next(filter(predicate, iterable), default) def all_equal(iterable, key=None): @@ -908,6 +903,9 @@ and :term:`generators ` which incur interpreter overhead. # all_equal('4٤௪౪໔', key=int) → True return len(take(2, groupby(iterable, key))) <= 1 + + # ==== Data pipelines ==== + def unique_justseen(iterable, key=None): "Yield unique elements, preserving order. Remember only the element just seen." # unique_justseen('AAAABBBCCDAABBB') → A B C D A B @@ -933,14 +931,14 @@ and :term:`generators ` which incur interpreter overhead. yield element def unique(iterable, key=None, reverse=False): - "Yield unique elements in sorted order. Supports unhashable inputs." - # unique([[1, 2], [3, 4], [1, 2]]) → [1, 2] [3, 4] - sequenced = sorted(iterable, key=key, reverse=reverse) - return unique_justseen(sequenced, key=key) + "Yield unique elements in sorted order. Supports unhashable inputs." + # unique([[1, 2], [3, 4], [1, 2]]) → [1, 2] [3, 4] + sequenced = sorted(iterable, key=key, reverse=reverse) + return unique_justseen(sequenced, key=key) def sliding_window(iterable, n): "Collect data into overlapping fixed-length chunks or blocks." - # sliding_window('ABCDEFG', 4) → ABCD BCDE CDEF DEFG + # sliding_window('ABCDEFG', 3) → ABC BCD CDE DEF EFG iterator = iter(iterable) window = deque(islice(iterator, n - 1), maxlen=n) for x in iterator: @@ -949,7 +947,7 @@ and :term:`generators ` which incur interpreter overhead. def grouper(iterable, n, *, incomplete='fill', fillvalue=None): "Collect data into non-overlapping fixed-length chunks or blocks." - # grouper('ABCDEFG', 3, fillvalue='x') → ABC DEF Gxx + # grouper('ABCDEFG', 3, fillvalue='x') → ABC DEF Gxx # grouper('ABCDEFG', 3, incomplete='strict') → ABC DEF ValueError # grouper('ABCDEFG', 3, incomplete='ignore') → ABC DEF iterators = [iter(iterable)] * n @@ -978,6 +976,16 @@ and :term:`generators ` which incur interpreter overhead. slices = starmap(slice, combinations(range(len(seq) + 1), 2)) return map(getitem, repeat(seq), slices) + def derangements(iterable, r=None): + "Produce r length permutations without fixed points." + # derangements('ABCD') → BADC BCDA BDAC CADB CDAB CDBA DABC DCAB DCBA + # Algorithm credited to Stefan Pochmann + seq = tuple(iterable) + pos = tuple(range(len(seq))) + have_moved = map(map, repeat(is_not), repeat(pos), permutations(pos, r=r)) + valid_derangements = map(all, have_moved) + return compress(permutations(seq, r=r), valid_derangements) + def iter_index(iterable, value, start=0, stop=None): "Return indices where a value occurs in a sequence or iterable." # iter_index('AABCADEAF', 'A') → 0 1 4 7 @@ -1005,9 +1013,7 @@ and :term:`generators ` which incur interpreter overhead. yield function() -The following recipes have a more mathematical flavor: - -.. testcode:: + # ==== Mathematical operations ==== def multinomial(*counts): "Number of distinct arrangements of a multiset." @@ -1026,9 +1032,12 @@ The following recipes have a more mathematical flavor: # sum_of_squares([10, 20, 30]) → 1400 return sumprod(*tee(iterable)) + + # ==== Matrix operations ==== + def reshape(matrix, columns): "Reshape a 2-D matrix to have a given number of columns." - # reshape([(0, 1), (2, 3), (4, 5)], 3) → (0, 1, 2), (3, 4, 5) + # reshape([(0, 1), (2, 3), (4, 5)], 3) → (0, 1, 2) (3, 4, 5) return batched(chain.from_iterable(matrix), columns, strict=True) def transpose(matrix): @@ -1038,10 +1047,13 @@ The following recipes have a more mathematical flavor: def matmul(m1, m2): "Multiply two matrices." - # matmul([(7, 5), (3, 5)], [(2, 5), (7, 9)]) → (49, 80), (41, 60) + # matmul([(7, 5), (3, 5)], [(2, 5), (7, 9)]) → (49, 80) (41, 60) n = len(m2[0]) return batched(starmap(sumprod, product(m1, transpose(m2))), n) + + # ==== Polynomial arithmetic ==== + def convolve(signal, kernel): """Discrete linear convolution of two iterables. Equivalent to polynomial multiplication. @@ -1096,6 +1108,9 @@ The following recipes have a more mathematical flavor: powers = reversed(range(1, n)) return list(map(mul, coefficients, powers)) + + # ==== Number theory ==== + def sieve(n): "Primes less than n." # sieve(30) → 2 3 5 7 11 13 17 19 23 29 @@ -1134,6 +1149,49 @@ The following recipes have a more mathematical flavor: return n + # ==== Running statistics ==== + + def running_mean(iterable): + "Average of values seen so far." + # running_mean([37, 33, 38, 28]) → 37 35 36 34 + return map(truediv, accumulate(iterable), count(1)) + + def running_min(iterable): + "Smallest of values seen so far." + # running_min([37, 33, 38, 28]) → 37 33 33 28 + return accumulate(iterable, func=min) + + def running_max(iterable): + "Largest of values seen so far." + # running_max([37, 33, 38, 28]) → 37 37 38 38 + return accumulate(iterable, func=max) + + def running_median(iterable): + "Median of values seen so far." + # running_median([37, 33, 38, 28]) → 37 35 37 35 + read = iter(iterable).__next__ + lo = [] # max-heap + hi = [] # min-heap the same size as or one smaller than lo + with suppress(StopIteration): + while True: + heappush_max(lo, heappushpop(hi, read())) + yield lo[0] + heappush(hi, heappushpop_max(lo, read())) + yield (lo[0] + hi[0]) / 2 + + def running_statistics(iterable): + "Aggregate statistics for values seen so far." + # Generate tuples: (size, minimum, median, maximum, mean) + t0, t1, t2, t3 = tee(iterable, 4) + return zip( + count(1), + running_min(t0), + running_median(t1), + running_max(t2), + running_mean(t3), + ) + + .. doctest:: :hide: @@ -1210,10 +1268,6 @@ The following recipes have a more mathematical flavor: [(0, 'a'), (1, 'b'), (2, 'c')] - >>> list(islice(tabulate(lambda x: 2*x), 4)) - [0, 2, 4, 6] - - >>> for _ in loops(5): ... print('hi') ... @@ -1663,6 +1717,36 @@ The following recipes have a more mathematical flavor: ['A', 'AB', 'ABC', 'ABCD', 'B', 'BC', 'BCD', 'C', 'CD', 'D'] + >>> ' '.join(map(''.join, derangements('ABCD'))) + 'BADC BCDA BDAC CADB CDAB CDBA DABC DCAB DCBA' + >>> ' '.join(map(''.join, derangements('ABCD', 3))) + 'BAD BCA BCD BDA CAB CAD CDA CDB DAB DCA DCB' + >>> ' '.join(map(''.join, derangements('ABCD', 2))) + 'BA BC BD CA CD DA DC' + >>> ' '.join(map(''.join, derangements('ABCD', 1))) + 'B C D' + >>> ' '.join(map(''.join, derangements('ABCD', 0))) + '' + >>> # Compare number of derangements to https://oeis.org/A000166 + >>> [len(list(derangements(range(n)))) for n in range(10)] + [1, 0, 1, 2, 9, 44, 265, 1854, 14833, 133496] + >>> # Verify that identical objects are treated as unique by position + >>> identical = 'X' + >>> distinct = 'x' + >>> seq1 = ('A', identical, 'B', identical) + >>> result1 = ' '.join(map(''.join, derangements(seq1))) + >>> result1 + 'XAXB XBXA XXAB BAXX BXAX BXXA XAXB XBAX XBXA' + >>> seq2 = ('A', identical, 'B', distinct) + >>> result2 = ' '.join(map(''.join, derangements(seq2))) + >>> result2 + 'XAxB XBxA XxAB BAxX BxAX BxXA xAXB xBAX xBXA' + >>> result1 == result2 + False + >>> result1.casefold() == result2.casefold() + True + + >>> list(powerset([1,2,3])) [(), (1,), (2,), (3,), (1, 2), (1, 3), (2, 3), (1, 2, 3)] >>> all(len(list(powerset(range(n)))) == 2**n for n in range(18)) @@ -1743,11 +1827,37 @@ The following recipes have a more mathematical flavor: True + >>> list(running_mean([8.5, 9.5, 7.5, 6.5])) + [8.5, 9.0, 8.5, 8.0] + >>> list(running_mean([37, 33, 38, 28])) + [37.0, 35.0, 36.0, 34.0] + + + >>> list(running_min([37, 33, 38, 28])) + [37, 33, 33, 28] + + + >>> list(running_max([37, 33, 38, 28])) + [37, 37, 38, 38] + + + >>> list(running_median([37, 33, 38, 28])) + [37, 35.0, 37, 35.0] + + + >>> list(running_statistics([37, 33, 38, 28])) + [(1, 37, 37, 37, 37.0), (2, 33, 35.0, 37, 35.0), (3, 33, 37, 38, 36.0), (4, 28, 35.0, 38, 34.0)] + + .. testcode:: :hide: # Old recipes and their tests which are guaranteed to continue to work. + def tabulate(function, start=0): + "Return function(0), function(1), ..." + return map(function, count(start)) + def old_sumprod_recipe(vec1, vec2): "Compute a sum of products." return sum(starmap(operator.mul, zip(vec1, vec2, strict=True))) @@ -1827,6 +1937,10 @@ The following recipes have a more mathematical flavor: .. doctest:: :hide: + >>> list(islice(tabulate(lambda x: 2*x), 4)) + [0, 2, 4, 6] + + >>> dotproduct([1,2,3], [4,5,6]) 32 diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 26579ec63288602..231654a3a9315f3 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -18,12 +18,17 @@ is a lightweight data interchange format inspired by `JavaScript `_ object literal syntax (although it is not a strict subset of JavaScript [#rfc-errata]_ ). +.. note:: + The term "object" in the context of JSON processing in Python can be + ambiguous. All values in Python are objects. In JSON, an object refers to + any data wrapped in curly braces, similar to a Python dictionary. + .. warning:: Be cautious when parsing JSON data from untrusted sources. A malicious JSON string may cause the decoder to consume considerable CPU and memory resources. Limiting the size of data to be parsed is recommended. -:mod:`json` exposes an API familiar to users of the standard library +This module exposes an API familiar to users of the standard library :mod:`marshal` and :mod:`pickle` modules. Encoding basic Python object hierarchies:: @@ -60,7 +65,7 @@ Pretty printing:: "6": 7 } -Specializing JSON object encoding:: +Customizing JSON object encoding:: >>> import json >>> def custom_json(obj): @@ -83,7 +88,7 @@ Decoding JSON:: >>> json.load(io) ['streaming API'] -Specializing JSON object decoding:: +Customizing JSON object decoding:: >>> import json >>> def as_complex(dct): @@ -116,7 +121,7 @@ Extending :class:`JSONEncoder`:: ['[2.0', ', 1.0', ']'] -Using :mod:`json` from the shell to validate and pretty-print: +Using :mod:`!json` from the shell to validate and pretty-print: .. code-block:: shell-session @@ -178,8 +183,10 @@ Basic Usage :param bool ensure_ascii: If ``True`` (the default), the output is guaranteed to - have all incoming non-ASCII characters escaped. - If ``False``, these characters will be outputted as-is. + have all incoming non-ASCII and non-printable characters escaped. + If ``False``, all characters will be outputted as-is, except for + the characters that must be escaped: quotation mark, reverse solidus, + and the control characters U+0000 through U+001F. :param bool check_circular: If ``False``, the circular reference check for container types is skipped @@ -207,7 +214,7 @@ Basic Usage a string (such as ``"\t"``) is used to indent each level. If zero, negative, or ``""`` (the empty string), only newlines are inserted. - If ``None`` (the default), the most compact representation is used. + If ``None`` (the default), no newlines are inserted. :type indent: int | str | None :param separators: @@ -279,7 +286,7 @@ Basic Usage :param object_hook: If set, a function that is called with the result of - any object literal decoded (a :class:`dict`). + any JSON object literal decoded (a :class:`dict`). The return value of this function will be used instead of the :class:`dict`. This feature can be used to implement custom decoders, @@ -289,7 +296,7 @@ Basic Usage :param object_pairs_hook: If set, a function that is called with the result of - any object literal decoded with an ordered list of pairs. + any JSON object literal decoded with an ordered list of pairs. The return value of this function will be used instead of the :class:`dict`. This feature can be used to implement custom decoders. @@ -490,8 +497,10 @@ Encoders and Decoders :class:`bool` or ``None``. If *skipkeys* is true, such items are simply skipped. If *ensure_ascii* is true (the default), the output is guaranteed to - have all incoming non-ASCII characters escaped. If *ensure_ascii* is - false, these characters will be output as-is. + have all incoming non-ASCII and non-printable characters escaped. + If *ensure_ascii* is false, all characters will be output as-is, except for + the characters that must be escaped: quotation mark, reverse solidus, + and the control characters U+0000 through U+001F. If *check_circular* is true (the default), then lists, dicts, and custom encoded objects will be checked for circular references during encoding to @@ -631,7 +640,7 @@ UTF-32, with UTF-8 being the recommended default for maximum interoperability. As permitted, though not required, by the RFC, this module's serializer sets *ensure_ascii=True* by default, thus escaping the output so that the resulting -strings only contain ASCII characters. +strings only contain printable ASCII characters. Other than the *ensure_ascii* parameter, this module is defined strictly in terms of conversion between Python objects and @@ -738,8 +747,8 @@ Command-line interface -------------- -The :mod:`json` module can be invoked as a script via ``python -m json`` -to validate and pretty-print JSON objects. The :mod:`json.tool` submodule +The :mod:`!json` module can be invoked as a script via ``python -m json`` +to validate and pretty-print JSON objects. The :mod:`!json.tool` submodule implements this interface. If the optional ``infile`` and ``outfile`` arguments are not @@ -760,7 +769,7 @@ specified, :data:`sys.stdin` and :data:`sys.stdout` will be used respectively: alphabetically by key. .. versionchanged:: 3.14 - The :mod:`json` module may now be directly executed as + The :mod:`!json` module may now be directly executed as ``python -m json``. For backwards compatibility, invoking the CLI as ``python -m json.tool`` remains supported. diff --git a/Doc/library/linecache.rst b/Doc/library/linecache.rst index e766a9280946d30..0a5373ec9763719 100644 --- a/Doc/library/linecache.rst +++ b/Doc/library/linecache.rst @@ -10,7 +10,7 @@ -------------- -The :mod:`linecache` module allows one to get any line from a Python source file, while +The :mod:`!linecache` module allows one to get any line from a Python source file, while attempting to optimize internally, using a cache, the common case where many lines are read from a single file. This is used by the :mod:`traceback` module to retrieve source lines for inclusion in the formatted traceback. @@ -19,7 +19,7 @@ The :func:`tokenize.open` function is used to open files. This function uses :func:`tokenize.detect_encoding` to get the encoding of the file; in the absence of an encoding token, the file encoding defaults to UTF-8. -The :mod:`linecache` module defines the following functions: +The :mod:`!linecache` module defines the following functions: .. function:: getline(filename, lineno, module_globals=None) @@ -31,7 +31,7 @@ The :mod:`linecache` module defines the following functions: .. index:: triple: module; search; path If *filename* indicates a frozen module (starting with ``'`, or a pair, + language code and encoding. An empty string specifies the user's default settings. If the modification of the locale fails, the exception :exc:`Error` is raised. If successful, the new locale setting is returned. + If *locale* is a pair, it is converted to a locale name using + the locale aliasing engine. + The language code has the same format as a :ref:`locale name `, + but without encoding and ``@``-modifier. + The language code and encoding can be ``None``. + If *locale* is omitted or ``None``, the current setting for *category* is returned. + Example:: + + >>> import locale + >>> loc = locale.setlocale(locale.LC_ALL) # get current locale + # use German locale; name and availability varies with platform + >>> locale.setlocale(locale.LC_ALL, 'de_DE.UTF-8') + >>> locale.strcoll('f\xe4n', 'foo') # compare a string containing an umlaut + >>> locale.setlocale(locale.LC_ALL, '') # use user's preferred locale + >>> locale.setlocale(locale.LC_ALL, 'C') # use default (C) locale + >>> locale.setlocale(locale.LC_ALL, loc) # restore saved locale + :func:`setlocale` is not thread-safe on most systems. Applications typically - start with a call of :: + start with a call of:: import locale locale.setlocale(locale.LC_ALL, '') @@ -345,22 +361,24 @@ The :mod:`locale` module defines the following exception and functions: ``'LANG'``. The GNU gettext search path contains ``'LC_ALL'``, ``'LC_CTYPE'``, ``'LANG'`` and ``'LANGUAGE'``, in that order. - Except for the code ``'C'``, the language code corresponds to :rfc:`1766`. - *language code* and *encoding* may be ``None`` if their values cannot be + The language code has the same format as a :ref:`locale name `, + but without encoding and ``@``-modifier. + The language code and encoding may be ``None`` if their values cannot be determined. - - .. deprecated-removed:: 3.11 3.15 + The "C" locale is represented as ``(None, None)``. .. function:: getlocale(category=LC_CTYPE) - Returns the current setting for the given locale category as sequence containing - *language code*, *encoding*. *category* may be one of the :const:`!LC_\*` values - except :const:`LC_ALL`. It defaults to :const:`LC_CTYPE`. + Returns the current setting for the given locale category as a tuple containing + the language code and encoding. *category* may be one of the :const:`!LC_\*` + values except :const:`LC_ALL`. It defaults to :const:`LC_CTYPE`. - Except for the code ``'C'``, the language code corresponds to :rfc:`1766`. - *language code* and *encoding* may be ``None`` if their values cannot be + The language code has the same format as a :ref:`locale name `, + but without encoding and ``@``-modifier. + The language code and encoding may be ``None`` if their values cannot be determined. + The "C" locale is represented as ``(None, None)``. .. function:: getpreferredencoding(do_setlocale=True) @@ -508,14 +526,14 @@ The :mod:`locale` module defines the following exception and functions: SSH connections. Python doesn't internally use locale-dependent character transformation functions - from ``ctype.h``. Instead, an internal ``pyctype.h`` provides locale-independent - equivalents like :c:macro:`!Py_TOLOWER`. + from ``ctype.h``. Instead, ``pyctype.h`` provides locale-independent + equivalents like :c:macro:`Py_TOLOWER`. .. data:: LC_COLLATE Locale category for sorting strings. The functions :func:`strcoll` and - :func:`strxfrm` of the :mod:`locale` module are affected. + :func:`strxfrm` of the :mod:`!locale` module are affected. .. data:: LC_TIME @@ -544,7 +562,7 @@ The :mod:`locale` module defines the following exception and functions: .. data:: LC_NUMERIC Locale category for formatting numbers. The functions :func:`format_string`, - :func:`atoi`, :func:`atof` and :func:`.str` of the :mod:`locale` module are + :func:`atoi`, :func:`atof` and :func:`.str` of the :mod:`!locale` module are affected by that category. All other numeric formatting operations are not affected. @@ -564,18 +582,6 @@ The :mod:`locale` module defines the following exception and functions: :func:`localeconv`. -Example:: - - >>> import locale - >>> loc = locale.getlocale() # get current locale - # use German locale; name might vary with platform - >>> locale.setlocale(locale.LC_ALL, 'de_DE') - >>> locale.strcoll('f\xe4n', 'foo') # compare a string containing an umlaut - >>> locale.setlocale(locale.LC_ALL, '') # use user's preferred locale - >>> locale.setlocale(locale.LC_ALL, 'C') # use default (C) locale - >>> locale.setlocale(locale.LC_ALL, loc) # restore saved locale - - Background, details, hints, tips and caveats -------------------------------------------- @@ -615,6 +621,61 @@ whose high bit is set (i.e., non-ASCII bytes) are never converted or considered part of a character class such as letter or whitespace. +.. _locale_name: + +Locale names +------------ + +The format of the locale name is platform dependent, and the set of supported +locales can depend on the system configuration. + +On Posix platforms, it usually has the format [1]_: + +.. productionlist:: locale_name + : language ["_" territory] ["." charset] ["@" modifier] + +where *language* is a two- or three-letter language code from `ISO 639`_, +*territory* is a two-letter country or region code from `ISO 3166`_, +*charset* is a locale encoding, and *modifier* is a script name, +a language subtag, a sort order identifier, or other locale modifier +(for example, "latin", "valencia", "stroke" and "euro"). + +On Windows, several formats are supported. [2]_ [3]_ +A subset of `IETF BCP 47`_ tags: + +.. productionlist:: locale_name + : language ["-" script] ["-" territory] ["." charset] + : language ["-" script] "-" territory "-" modifier + +where *language* and *territory* have the same meaning as in Posix, +*script* is a four-letter script code from `ISO 15924`_, +and *modifier* is a language subtag, a sort order identifier +or custom modifier (for example, "valencia", "stroke" or "x-python"). +Both hyphen (``'-'``) and underscore (``'_'``) separators are supported. +Only UTF-8 encoding is allowed for BCP 47 tags. + +Windows also supports locale names in the format: + +.. productionlist:: locale_name + : language ["_" territory] ["." charset] + +where *language* and *territory* are full names, such as "English" and +"United States", and *charset* is either a code page number (for example, "1252") +or UTF-8. +Only the underscore separator is supported in this format. + +The "C" locale is supported on all platforms. + +.. _ISO 639: https://www.iso.org/iso-639-language-code +.. _ISO 3166: https://www.iso.org/iso-3166-country-codes.html +.. _IETF BCP 47: https://www.rfc-editor.org/info/bcp47 +.. _ISO 15924: https://www.unicode.org/iso15924/ + +.. [1] `IEEE Std 1003.1-2024; 8.2 Internationalization Variables `_ +.. [2] `UCRT Locale names, Languages, and Country/Region strings `_ +.. [3] `Locale Names `_ + + .. _embedding-locale: For extension writers and programs that embed Python @@ -625,7 +686,7 @@ the current locale is. But since the return value can only be used portably to restore it, that is not very useful (except perhaps to find out whether or not the locale is ``C``). -When Python code uses the :mod:`locale` module to change the locale, this also +When Python code uses the :mod:`!locale` module to change the locale, this also affects the embedding application. If the embedding application doesn't want this to happen, it should remove the :mod:`!_locale` extension module (which does all the work) from the table of built-in modules in the :file:`config.c` file, diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 0e9dc33ae2123ae..6709062dfca72b9 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -28,7 +28,7 @@ Configuration functions ^^^^^^^^^^^^^^^^^^^^^^^ The following functions configure the logging module. They are located in the -:mod:`logging.config` module. Their use is optional --- you can configure the +:mod:`!logging.config` module. Their use is optional --- you can configure the logging module using these functions or by making calls to the main API (defined in :mod:`logging` itself) and defining handlers which are declared either in :mod:`logging` or :mod:`logging.handlers`. @@ -55,7 +55,7 @@ in :mod:`logging` itself) and defining handlers which are declared either in Parsing is performed by the :class:`DictConfigurator` class, whose constructor is passed the dictionary used for configuration, and - has a :meth:`configure` method. The :mod:`logging.config` module + has a :meth:`configure` method. The :mod:`!logging.config` module has a callable attribute :attr:`dictConfigClass` which is initially set to :class:`DictConfigurator`. You can replace the value of :attr:`dictConfigClass` with a @@ -548,7 +548,7 @@ mnemonic that the corresponding value is a callable. The ``filters`` member of ``handlers`` and ``loggers`` can take filter instances in addition to ids. -You can also specify a special key ``'.'`` whose value is a dictionary is a +You can also specify a special key ``'.'`` whose value is a mapping of attribute names to values. If found, the specified attributes will be set on the user-defined object before it is returned. Thus, with the following configuration:: @@ -586,7 +586,7 @@ configuration dictionary for the handler named ``foo``, and later (once that handler has been configured) it points to the configured handler instance. Thus, ``cfg://handlers.foo`` could resolve to either a dictionary or a handler instance. In general, it is wise to name handlers in a way such that dependent -handlers are configured _after_ any handlers they depend on; that allows +handlers are configured *after* any handlers they depend on; that allows something like ``cfg://handlers.foo`` to be used in configuring a handler that depends on handler ``foo``. If that dependent handler were named ``bar``, problems would result, because the configuration of ``bar`` would be attempted diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 63ef533e82c658f..d128f64aae72361 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -160,7 +160,7 @@ WatchedFileHandler .. currentmodule:: logging.handlers -The :class:`WatchedFileHandler` class, located in the :mod:`logging.handlers` +The :class:`WatchedFileHandler` class, located in the :mod:`!logging.handlers` module, is a :class:`FileHandler` which watches the file it is logging to. If the file changes, it is closed and reopened using the file name. @@ -213,7 +213,7 @@ for this value. BaseRotatingHandler ^^^^^^^^^^^^^^^^^^^ -The :class:`BaseRotatingHandler` class, located in the :mod:`logging.handlers` +The :class:`BaseRotatingHandler` class, located in the :mod:`!logging.handlers` module, is the base class for the rotating file handlers, :class:`RotatingFileHandler` and :class:`TimedRotatingFileHandler`. You should not need to instantiate this class, but it has attributes and methods you may @@ -307,7 +307,7 @@ For an example, see :ref:`cookbook-rotator-namer`. RotatingFileHandler ^^^^^^^^^^^^^^^^^^^ -The :class:`RotatingFileHandler` class, located in the :mod:`logging.handlers` +The :class:`RotatingFileHandler` class, located in the :mod:`!logging.handlers` module, supports rotation of disk log files. @@ -352,13 +352,17 @@ module, supports rotation of disk log files. Outputs the record to the file, catering for rollover as described previously. + .. method:: shouldRollover(record) + + See if the supplied record would cause the file to exceed the configured size limit. + .. _timed-rotating-file-handler: TimedRotatingFileHandler ^^^^^^^^^^^^^^^^^^^^^^^^ The :class:`TimedRotatingFileHandler` class, located in the -:mod:`logging.handlers` module, supports rotation of disk log files at certain +:mod:`!logging.handlers` module, supports rotation of disk log files at certain timed intervals. @@ -461,12 +465,17 @@ timed intervals. Returns a list of filenames which should be deleted as part of rollover. These are the absolute paths of the oldest backup log files written by the handler. + .. method:: shouldRollover(record) + + See if enough time has passed for a rollover to occur and if it has, compute + the next rollover time. + .. _socket-handler: SocketHandler ^^^^^^^^^^^^^ -The :class:`SocketHandler` class, located in the :mod:`logging.handlers` module, +The :class:`SocketHandler` class, located in the :mod:`!logging.handlers` module, sends logging output to a network socket. The base class uses a TCP socket. @@ -562,7 +571,7 @@ sends logging output to a network socket. The base class uses a TCP socket. DatagramHandler ^^^^^^^^^^^^^^^ -The :class:`DatagramHandler` class, located in the :mod:`logging.handlers` +The :class:`DatagramHandler` class, located in the :mod:`!logging.handlers` module, inherits from :class:`SocketHandler` to support sending logging messages over UDP sockets. @@ -609,7 +618,7 @@ over UDP sockets. SysLogHandler ^^^^^^^^^^^^^ -The :class:`SysLogHandler` class, located in the :mod:`logging.handlers` module, +The :class:`SysLogHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to a remote or local Unix syslog. @@ -788,7 +797,7 @@ supports sending logging messages to a remote or local Unix syslog. NTEventLogHandler ^^^^^^^^^^^^^^^^^ -The :class:`NTEventLogHandler` class, located in the :mod:`logging.handlers` +The :class:`NTEventLogHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to a local Windows NT, Windows 2000 or Windows XP event log. Before you can use it, you need Mark Hammond's Win32 extensions for Python installed. @@ -855,7 +864,7 @@ extensions for Python installed. SMTPHandler ^^^^^^^^^^^ -The :class:`SMTPHandler` class, located in the :mod:`logging.handlers` module, +The :class:`SMTPHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to an email address via SMTP. @@ -896,7 +905,7 @@ supports sending logging messages to an email address via SMTP. MemoryHandler ^^^^^^^^^^^^^ -The :class:`MemoryHandler` class, located in the :mod:`logging.handlers` module, +The :class:`MemoryHandler` class, located in the :mod:`!logging.handlers` module, supports buffering of logging records in memory, periodically flushing them to a :dfn:`target` handler. Flushing occurs whenever the buffer is full, or when an event of a certain severity or greater is seen. @@ -976,7 +985,7 @@ should, then :meth:`flush` is expected to do the flushing. HTTPHandler ^^^^^^^^^^^ -The :class:`HTTPHandler` class, located in the :mod:`logging.handlers` module, +The :class:`HTTPHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to a web server, using either ``GET`` or ``POST`` semantics. @@ -1028,7 +1037,7 @@ QueueHandler .. versionadded:: 3.2 -The :class:`QueueHandler` class, located in the :mod:`logging.handlers` module, +The :class:`QueueHandler` class, located in the :mod:`!logging.handlers` module, supports sending logging messages to a queue, such as those implemented in the :mod:`queue` or :mod:`multiprocessing` modules. @@ -1051,6 +1060,15 @@ possible, while any potentially slow operations (such as sending an email via .. note:: If you are using :mod:`multiprocessing`, you should avoid using :class:`~queue.SimpleQueue` and instead use :class:`multiprocessing.Queue`. + .. warning:: + + The :mod:`multiprocessing` module uses an internal logger created and + accessed via :meth:`~multiprocessing.get_logger`. + :class:`multiprocessing.Queue` will log ``DEBUG`` level messages upon + items being queued. If those log messages are processed by a + :class:`QueueHandler` using the same :class:`multiprocessing.Queue` instance, + it will cause a deadlock or infinite recursion. + .. method:: emit(record) Enqueues the result of preparing the LogRecord. Should an exception @@ -1112,7 +1130,7 @@ QueueListener .. versionadded:: 3.2 -The :class:`QueueListener` class, located in the :mod:`logging.handlers` +The :class:`QueueListener` class, located in the :mod:`!logging.handlers` module, supports receiving logging messages from a queue, such as those implemented in the :mod:`queue` or :mod:`multiprocessing` modules. The messages are received from a queue in an internal thread and passed, on diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 72190e972405140..6e38d45504c4db8 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -671,8 +671,7 @@ Formatter Objects which is just the logged message. :type fmt: str - :param datefmt: A format string in the given *style* for - the date/time portion of the logged output. + :param datefmt: A format string for the date/time portion of the logged output. If not specified, the default described in :meth:`formatTime` is used. :type datefmt: str @@ -680,7 +679,7 @@ Formatter Objects how the format string will be merged with its data: using one of :ref:`old-string-formatting` (``%``), :meth:`str.format` (``{``) or :class:`string.Template` (``$``). This only applies to - *fmt* and *datefmt* (e.g. ``'%(message)s'`` versus ``'{message}'``), + *fmt* (e.g. ``'%(message)s'`` versus ``'{message}'``), not to the actual log messages passed to the logging methods. However, there are :ref:`other ways ` to use ``{``- and ``$``-formatting for log messages. @@ -1012,6 +1011,11 @@ the options available to you. | exc_info | You shouldn't need to | Exception tuple (à la ``sys.exc_info``) or, | | | format this yourself. | if no exception has occurred, ``None``. | +----------------+-------------------------+-----------------------------------------------+ +| exc_text | You shouldn't need to | Exception information formatted as a string. | +| | format this yourself. | This is set when :meth:`Formatter.format` is | +| | | invoked, or ``None`` if no exception has | +| | | occurred. | ++----------------+-------------------------+-----------------------------------------------+ | filename | ``%(filename)s`` | Filename portion of ``pathname``. | +----------------+-------------------------+-----------------------------------------------+ | funcName | ``%(funcName)s`` | Name of function containing the logging call. | @@ -1083,12 +1087,13 @@ LoggerAdapter Objects information into logging calls. For a usage example, see the section on :ref:`adding contextual information to your logging output `. -.. class:: LoggerAdapter(logger, extra, merge_extra=False) +.. class:: LoggerAdapter(logger, extra=None, merge_extra=False) Returns an instance of :class:`LoggerAdapter` initialized with an - underlying :class:`Logger` instance, a dict-like object (*extra*), and a - boolean (*merge_extra*) indicating whether or not the *extra* argument of - individual log calls should be merged with the :class:`LoggerAdapter` extra. + underlying :class:`Logger` instance, an optional dict-like object (*extra*), + and an optional boolean (*merge_extra*) indicating whether or not + the *extra* argument of individual log calls should be merged with + the :class:`LoggerAdapter` extra. The default behavior is to ignore the *extra* argument of individual log calls and only use the one of the :class:`LoggerAdapter` instance @@ -1128,16 +1133,20 @@ information into logging calls. For a usage example, see the section on Attribute :attr:`!manager` and method :meth:`!_log` were added, which delegate to the underlying logger and allow adapters to be nested. + .. versionchanged:: 3.10 + + The *extra* argument is now optional. + .. versionchanged:: 3.13 - The *merge_extra* argument was added. + The *merge_extra* parameter was added. Thread Safety ------------- The logging module is intended to be thread-safe without any special work -needing to be done by its clients. It achieves this though using threading +needing to be done by its clients. It achieves this through using threading locks; there is one lock to serialize access to the module's shared data, and each handler also creates a lock to serialize access to its underlying I/O. @@ -1525,7 +1534,7 @@ Module-Level Attributes Integration with the warnings module ------------------------------------ -The :func:`captureWarnings` function can be used to integrate :mod:`logging` +The :func:`captureWarnings` function can be used to integrate :mod:`!logging` with the :mod:`warnings` module. .. function:: captureWarnings(capture) @@ -1556,7 +1565,7 @@ with the :mod:`warnings` module. library. `Original Python logging package `_ - This is the original source for the :mod:`logging` package. The version of the + This is the original source for the :mod:`!logging` package. The version of the package available from this site is suitable for use with Python 1.5.2, 2.1.x - and 2.2.x, which do not include the :mod:`logging` package in the standard + and 2.2.x, which do not include the :mod:`!logging` package in the standard library. diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 69f7cb8d48d7aee..6742cbf701b67ac 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -23,6 +23,8 @@ module. Note that :class:`LZMAFile` and :class:`bz2.BZ2File` are *not* thread-safe, so if you need to use a single :class:`LZMAFile` instance from multiple threads, it is necessary to protect it with a lock. +.. include:: ../includes/optional-module.rst + .. exception:: LZMAError @@ -153,35 +155,14 @@ Compressing and decompressing data in memory :func:`compress`. The *format* argument specifies what container format should be used. - Possible values are: - - * :const:`FORMAT_XZ`: The ``.xz`` container format. - This is the default format. - - * :const:`FORMAT_ALONE`: The legacy ``.lzma`` container format. - This format is more limited than ``.xz`` -- it does not support integrity - checks or multiple filters. - - * :const:`FORMAT_RAW`: A raw data stream, not using any container format. - This format specifier does not support integrity checks, and requires that - you always specify a custom filter chain (for both compression and - decompression). Additionally, data compressed in this manner cannot be - decompressed using :const:`FORMAT_AUTO` (see :class:`LZMADecompressor`). + Possible values are :const:`FORMAT_XZ` (the default), + :const:`FORMAT_ALONE` and :const:`FORMAT_RAW`. The *check* argument specifies the type of integrity check to include in the compressed data. This check is used when decompressing, to ensure that the - data has not been corrupted. Possible values are: - - * :const:`CHECK_NONE`: No integrity check. - This is the default (and the only acceptable value) for - :const:`FORMAT_ALONE` and :const:`FORMAT_RAW`. - - * :const:`CHECK_CRC32`: 32-bit Cyclic Redundancy Check. - - * :const:`CHECK_CRC64`: 64-bit Cyclic Redundancy Check. - This is the default for :const:`FORMAT_XZ`. - - * :const:`CHECK_SHA256`: 256-bit Secure Hash Algorithm. + data has not been corrupted. Possible values are :const:`CHECK_NONE`, + :const:`CHECK_CRC32`, :const:`CHECK_CRC64` (the default for + :const:`FORMAT_XZ`) and :const:`CHECK_SHA256`. If the specified check is not supported, an :class:`LZMAError` is raised. @@ -357,12 +338,12 @@ options. Valid filter IDs are as follows: * Branch-Call-Jump (BCJ) filters: - * :const:`FILTER_X86` - * :const:`FILTER_IA64` - * :const:`FILTER_ARM` - * :const:`FILTER_ARMTHUMB` - * :const:`FILTER_POWERPC` - * :const:`FILTER_SPARC` + * :const:`!FILTER_X86` + * :const:`!FILTER_IA64` + * :const:`!FILTER_ARM` + * :const:`!FILTER_ARMTHUMB` + * :const:`!FILTER_POWERPC` + * :const:`!FILTER_SPARC` A filter chain can consist of up to 4 filters, and cannot be empty. The last filter in the chain must be a compression filter, and any other filters must be @@ -399,6 +380,106 @@ These filters support one option, ``start_offset``. This specifies the address that should be mapped to the beginning of the input data. The default is 0. +Constants +--------- + +The following module-level constants are provided for use as the *format*, +*check*, *preset* and *filters* arguments of the classes and functions above. + +Container formats: + +.. data:: FORMAT_XZ + + The ``.xz`` container format. + +.. data:: FORMAT_ALONE + + The legacy ``.lzma`` container format. This format is more limited than + ``.xz`` -- it does not support integrity checks or multiple filters. + +.. data:: FORMAT_RAW + + A raw data stream, not using any container format. This format specifier + does not support integrity checks, and requires that you always specify a + custom filter chain (for both compression and decompression). Additionally, + data compressed in this manner cannot be decompressed using + :const:`FORMAT_AUTO`. + +.. data:: FORMAT_AUTO + + Used for decompression only. The container format is detected + automatically, so that both ``.xz`` and ``.lzma`` files can be decompressed. + +Integrity checks: + +.. data:: CHECK_NONE + + No integrity check. This is the default (and the only acceptable value) for + :const:`FORMAT_ALONE` and :const:`FORMAT_RAW`. + +.. data:: CHECK_CRC32 + + A 32-bit Cyclic Redundancy Check. + +.. data:: CHECK_CRC64 + + A 64-bit Cyclic Redundancy Check. This is the default for + :const:`FORMAT_XZ`. + +.. data:: CHECK_SHA256 + + A 256-bit Secure Hash Algorithm. + +.. data:: CHECK_UNKNOWN + + The integrity check used by a stream could not yet be determined. This may + be the value of the :attr:`LZMADecompressor.check` attribute until enough of + the input has been decoded. + +.. data:: CHECK_ID_MAX + + The largest supported integrity-check ID. + +Compression presets: + +.. data:: PRESET_DEFAULT + + The default compression preset, equivalent to preset level ``6``. + +.. data:: PRESET_EXTREME + + A flag that may be bitwise OR-ed with a preset level (``0`` to ``9``) to + select a slower but more thorough variant of that preset. + +Filter IDs and options: + +.. data:: FILTER_LZMA1 + FILTER_LZMA2 + + The LZMA1 and LZMA2 compression filters. :const:`FILTER_LZMA1` is for use + with :const:`FORMAT_ALONE`, while :const:`FILTER_LZMA2` is for use with + :const:`FORMAT_XZ` and :const:`FORMAT_RAW`. + +.. data:: FILTER_DELTA + + The delta filter. + +.. data:: MODE_FAST + MODE_NORMAL + + Compression modes that may be used as the ``mode`` option of a filter + specifier (see :ref:`filter-chain-specs`). + +.. data:: MF_HC3 + MF_HC4 + MF_BT2 + MF_BT3 + MF_BT4 + + Match finders that may be used as the ``mf`` option of a filter specifier + (see :ref:`filter-chain-specs`). + + Examples -------- diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index e8a96f29ea185e6..ed135bf02cb968d 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -917,7 +917,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. copied; furthermore, any format-specific information is converted insofar as possible if *message* is a :class:`!Message` instance. If *message* is a string, a byte string, - or a file, it should contain an :rfc:`2822`\ -compliant message, which is read + or a file, it should contain an :rfc:`5322`\ -compliant message, which is read and parsed. Files should be open in binary mode, but text mode files are accepted for backward compatibility. @@ -1025,7 +1025,7 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. method:: remove_flag(flag) Unset the flag(s) specified by *flag* without changing other flags. To - remove more than one flag at a time, *flag* maybe a string of more than + remove more than one flag at a time, *flag* may be a string of more than one character. If "info" contains experimental information rather than flags, the current "info" is not modified. @@ -1190,7 +1190,7 @@ When a :class:`!MaildirMessage` instance is created based upon a .. method:: remove_flag(flag) Unset the flag(s) specified by *flag* without changing other flags. To - remove more than one flag at a time, *flag* maybe a string of more than + remove more than one flag at a time, *flag* may be a string of more than one character. When an :class:`!mboxMessage` instance is created based upon a @@ -1562,7 +1562,7 @@ When a :class:`!BabylMessage` instance is created based upon an .. method:: remove_flag(flag) Unset the flag(s) specified by *flag* without changing other flags. To - remove more than one flag at a time, *flag* maybe a string of more than + remove more than one flag at a time, *flag* may be a string of more than one character. When an :class:`!MMDFMessage` instance is created based upon a @@ -1641,7 +1641,7 @@ The following exception classes are defined in the :mod:`!mailbox` module: .. exception:: Error() - The based class for all other module-specific exceptions. + The base class for all other module-specific exceptions. .. exception:: NoSuchMailboxError() @@ -1661,7 +1661,7 @@ The following exception classes are defined in the :mod:`!mailbox` module: Raised when some mailbox-related condition beyond the control of the program causes it to be unable to proceed, such as when failing to acquire a lock that - another program already holds a lock, or when a uniquely generated file name + another program already holds, or when a uniquely generated file name already exists. diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index e8e9071a5c9ef49..ed182ea24e8f3ca 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -20,7 +20,7 @@ rarely does). [#]_ This is not a general "persistence" module. For general persistence and transfer of Python objects through RPC calls, see the modules :mod:`pickle` and -:mod:`shelve`. The :mod:`marshal` module exists mainly to support reading and +:mod:`shelve`. The :mod:`!marshal` module exists mainly to support reading and writing the "pseudo-compiled" code for Python modules of :file:`.pyc` files. Therefore, the Python maintainers reserve the right to modify the marshal format in backward incompatible ways should the need arise. @@ -34,7 +34,7 @@ supports a substantially wider range of objects than marshal. .. warning:: - The :mod:`marshal` module is not intended to be secure against erroneous or + The :mod:`!marshal` module is not intended to be secure against erroneous or maliciously constructed data. Never unmarshal data received from an untrusted or unauthenticated source. diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 0749367045dfa9e..241f043ede7a9fa 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -10,8 +10,8 @@ -------------- -This module provides access to the mathematical functions defined by the C -standard. +This module provides access to common mathematical functions and constants, +including those defined by the C standard. These functions cannot be used with complex numbers; use the functions of the same name from the :mod:`cmath` module if you require support for complex @@ -144,8 +144,7 @@ Number-theoretic functions .. function:: factorial(n) - Return *n* factorial as an integer. Raises :exc:`ValueError` if *n* is not integral or - is negative. + Return factorial of the nonnegative integer *n*. .. versionchanged:: 3.10 Floats with integral values (like ``5.0``) are no longer accepted. @@ -322,10 +321,12 @@ Floating point manipulation functions .. function:: frexp(x) - Return the mantissa and exponent of *x* as the pair ``(m, e)``. *m* is a float - and *e* is an integer such that ``x == m * 2**e`` exactly. If *x* is zero, - returns ``(0.0, 0)``, otherwise ``0.5 <= abs(m) < 1``. This is used to "pick - apart" the internal representation of a float in a portable way. + Return the mantissa and exponent of *x* as the pair ``(m, e)``. + If *x* is a finite nonzero number, then *m* is a float with + ``0.5 <= abs(m) < 1.0`` and an integer *e* is such that + ``x == m * 2**e`` exactly. Else, return ``(x, 0)``. + This is used to "pick apart" the internal representation of + a float in a portable way. Note that :func:`frexp` has a different call/return pattern than its C equivalents: it takes a single argument and return a pair of @@ -775,7 +776,7 @@ Constants The mathematical constant *τ* = 6.283185..., to available precision. Tau is a circle constant equal to 2\ *π*, the ratio of a circle's circumference to its radius. To learn more about Tau, check out Vi Hart's video `Pi is (still) - Wrong `_, and start celebrating + Wrong `_, and start celebrating `Tau day `_ by eating twice as much pie! .. versionadded:: 3.6 @@ -817,7 +818,7 @@ Constants .. impl-detail:: - The :mod:`math` module consists mostly of thin wrappers around the platform C + The :mod:`!math` module consists mostly of thin wrappers around the platform C math library functions. Behavior in exceptional cases follows Annex F of the C99 standard where appropriate. The current implementation will raise :exc:`ValueError` for invalid operations like ``sqrt(-1.0)`` or ``log(0.0)`` diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst index 13511b16a0ed8ce..647f9d05c4db23b 100644 --- a/Doc/library/mimetypes.rst +++ b/Doc/library/mimetypes.rst @@ -12,7 +12,7 @@ -------------- -The :mod:`mimetypes` module converts between a filename or URL and the MIME type +The :mod:`!mimetypes` module converts between a filename or URL and the MIME type associated with the filename extension. Conversions are provided from filename to MIME type and from MIME type to filename extension; encodings are not supported for the latter conversion. @@ -56,8 +56,8 @@ the information :func:`init` sets up. .. versionchanged:: 3.8 Added support for *url* being a :term:`path-like object`. - .. deprecated:: 3.13 - Passing a file path instead of URL is :term:`soft deprecated`. + .. soft-deprecated:: 3.13 + Passing a file path instead of URL. Use :func:`guess_file_type` for this. @@ -118,12 +118,13 @@ behavior of the module. Previously, Windows registry settings were ignored. -.. function:: read_mime_types(filename) +.. function:: read_mime_types(file) - Load the type map given in the file *filename*, if it exists. The type map is - returned as a dictionary mapping filename extensions, including the leading dot - (``'.'``), to strings of the form ``'type/subtype'``. If the file *filename* - does not exist or cannot be read, ``None`` is returned. + Load the type map given in the file named by *file*, if it exists. *file* + must be a string specifying the name of the file to read. The type map is + returned as a dictionary mapping file extensions, including the leading dot + (``'.'``), to strings of the form ``'type/subtype'``. If the file does not + exist or cannot be read, ``None`` is returned. .. function:: add_type(type, ext, strict=True) @@ -196,7 +197,7 @@ MimeTypes objects The :class:`MimeTypes` class may be useful for applications which may want more than one MIME-type database; it provides an interface similar to the one of the -:mod:`mimetypes` module. +:mod:`!mimetypes` module. .. class:: MimeTypes(filenames=(), strict=True) @@ -350,7 +351,7 @@ it converts file extensions to MIME types. For each ``type`` entry, the script writes a line into the standard output stream. If an unknown type occurs, it writes an error message into the -standard error stream and exits with the return code ``1``. +standard output stream and exits with the return code ``1``. .. mimetypes-cli-example: @@ -377,7 +378,7 @@ interface: $ # get a MIME type for a rare file extension $ python -m mimetypes filename.pict - error: unknown extension of filename.pict + error: media type unknown for filename.pict $ # now look in the extended database built into Python $ python -m mimetypes --lenient filename.pict @@ -399,7 +400,8 @@ interface: $ python -m mimetypes filename.sh filename.nc filename.xxx filename.txt type: application/x-sh encoding: None type: application/x-netcdf encoding: None - error: unknown extension of filename.xxx + error: media type unknown for filename.xxx + type: text/plain encoding: None $ # try to feed an unknown MIME type $ python -m mimetypes --extension audio/aac audio/opus audio/future audio/x-wav diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 4e20c07331a2207..f78518ac2232a0e 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -200,7 +200,8 @@ To map anonymous memory, -1 should be passed as the fileno along with the length Writable :term:`bytes-like object` is now accepted. - .. method:: flush([offset[, size]]) + .. method:: flush() + flush(offset, size, /) Flushes changes made to the in-memory copy of a file back to disk. Without use of this call there is no guarantee that changes are written back before @@ -269,7 +270,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length Resizing a map created with *access* of :const:`ACCESS_READ` or :const:`ACCESS_COPY`, will raise a :exc:`TypeError` exception. - Resizing a map created with with *trackfd* set to ``False``, + Resizing a map created with *trackfd* set to ``False``, will raise a :exc:`ValueError` exception. **On Windows**: Resizing the map will raise an :exc:`OSError` if there are other diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst index 327cc3602b1a773..79069c136c28416 100644 --- a/Doc/library/msvcrt.rst +++ b/Doc/library/msvcrt.rst @@ -2,7 +2,6 @@ =========================================================== .. module:: msvcrt - :platform: Windows :synopsis: Miscellaneous useful routines from the MS VC++ runtime. .. sectionauthor:: Fred L. Drake, Jr. @@ -22,6 +21,8 @@ api. The normal API deals only with ASCII characters and is of limited use for internationalized applications. The wide char API should be used where ever possible. +.. availability:: Windows. + .. versionchanged:: 3.3 Operations in this module now raise :exc:`OSError` where :exc:`IOError` was raised. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 6c43d5fe166e2fa..b4d5eca4e91a8e9 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -13,17 +13,16 @@ Introduction ------------ -:mod:`multiprocessing` is a package that supports spawning processes using an -API similar to the :mod:`threading` module. The :mod:`multiprocessing` package +:mod:`!multiprocessing` is a package that supports spawning processes using an +API similar to the :mod:`threading` module. The :mod:`!multiprocessing` package offers both local and remote concurrency, effectively side-stepping the :term:`Global Interpreter Lock ` by using subprocesses instead of threads. Due -to this, the :mod:`multiprocessing` module allows the programmer to fully +to this, the :mod:`!multiprocessing` module allows the programmer to fully leverage multiple processors on a given machine. It runs on both POSIX and Windows. -The :mod:`multiprocessing` module also introduces APIs which do not have -analogs in the :mod:`threading` module. A prime example of this is the +The :mod:`!multiprocessing` module also introduces the :class:`~multiprocessing.pool.Pool` object which offers a convenient means of parallelizing the execution of a function across multiple input values, distributing the input data across processes (data parallelism). The following @@ -44,6 +43,10 @@ will print to standard output :: [1, 4, 9] +The :mod:`!multiprocessing` module also introduces APIs which do not have +analogs in the :mod:`threading` module, like the ability to :meth:`terminate +`, :meth:`interrupt ` or :meth:`kill +` a running process. .. seealso:: @@ -58,7 +61,7 @@ will print to standard output :: The :class:`Process` class ^^^^^^^^^^^^^^^^^^^^^^^^^^ -In :mod:`multiprocessing`, processes are spawned by creating a :class:`Process` +In :mod:`!multiprocessing`, processes are spawned by creating a :class:`Process` object and then calling its :meth:`~Process.start` method. :class:`Process` follows the API of :class:`threading.Thread`. A trivial example of a multiprocess program is :: @@ -97,6 +100,10 @@ To show the individual process IDs involved, here is an expanded example:: For an explanation of why the ``if __name__ == '__main__'`` part is necessary, see :ref:`multiprocessing-programming`. +The arguments to :class:`Process` usually need to be picklable so they can be +passed to the child process. If you tried typing the above example directly +into a REPL it could lead to an :exc:`AttributeError` in the child process +trying to locate the *f* function in the ``__main__`` module. .. _multiprocessing-start-methods: @@ -104,7 +111,7 @@ necessary, see :ref:`multiprocessing-programming`. Contexts and start methods ^^^^^^^^^^^^^^^^^^^^^^^^^^ -Depending on the platform, :mod:`multiprocessing` supports three ways +Depending on the platform, :mod:`!multiprocessing` supports three ways to start a process. These *start methods* are .. _multiprocessing-start-method-spawn: @@ -233,9 +240,12 @@ processes for a different context. In particular, locks created using the *fork* context cannot be passed to processes started using the *spawn* or *forkserver* start methods. -A library which wants to use a particular start method should probably -use :func:`get_context` to avoid interfering with the choice of the -library user. +Libraries using :mod:`!multiprocessing` or +:class:`~concurrent.futures.ProcessPoolExecutor` should be designed to allow +their users to provide their own multiprocessing context. Using a specific +context of your own within a library can lead to incompatibilities with the +rest of the library user's application. Always document if your library +requires a specific start method. .. warning:: @@ -248,7 +258,7 @@ library user. Exchanging objects between processes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:mod:`multiprocessing` supports two types of communication channel between +:mod:`!multiprocessing` supports two types of communication channel between processes: **Queues** @@ -269,7 +279,7 @@ processes: p.join() Queues are thread and process safe. - Any object put into a :mod:`~multiprocessing` queue will be serialized. + Any object put into a :mod:`!multiprocessing` queue will be serialized. **Pipes** @@ -303,7 +313,7 @@ processes: Synchronization between processes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:mod:`multiprocessing` contains equivalents of all the synchronization +:mod:`!multiprocessing` contains equivalents of all the synchronization primitives from :mod:`threading`. For instance one can use a lock to ensure that only one process prints to standard output at a time:: @@ -334,7 +344,7 @@ avoid using shared state as far as possible. This is particularly true when using multiple processes. However, if you really do need to use some shared data then -:mod:`multiprocessing` provides a couple of ways of doing so. +:mod:`!multiprocessing` provides a couple of ways of doing so. **Shared memory** @@ -508,9 +518,24 @@ process which created it. Reference --------- -The :mod:`multiprocessing` package mostly replicates the API of the +The :mod:`!multiprocessing` package mostly replicates the API of the :mod:`threading` module. +.. _global-start-method: + +Global start method +^^^^^^^^^^^^^^^^^^^ + +Python supports several ways to create and initialize a process. +The global start method sets the default mechanism for creating a process. + +Several multiprocessing functions and methods that may also instantiate +certain objects will implicitly set the global start method to the system's default, +if it hasn’t been set already. The global start method can only be set once. +If you need to change the start method from the system default, you must +proactively set the global start method before calling functions or methods, +or creating these objects. + :class:`Process` and exceptions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -538,9 +563,42 @@ The :mod:`multiprocessing` package mostly replicates the API of the to pass to *target*. If a subclass overrides the constructor, it must make sure it invokes the - base class constructor (:meth:`Process.__init__`) before doing anything else + base class constructor (``super().__init__()``) before doing anything else to the process. + .. note:: + + In general, all arguments to :class:`Process` must be picklable. This is + frequently observed when trying to create a :class:`Process` or use a + :class:`concurrent.futures.ProcessPoolExecutor` from a REPL with a + locally defined *target* function. + + Passing a callable object defined in the current REPL session causes the + child process to die via an uncaught :exc:`AttributeError` exception when + starting as *target* must have been defined within an importable module + in order to be loaded during unpickling. + + Example of this uncatchable error from the child:: + + >>> import multiprocessing as mp + >>> def knigit(): + ... print("Ni!") + ... + >>> process = mp.Process(target=knigit) + >>> process.start() + >>> Traceback (most recent call last): + File ".../multiprocessing/spawn.py", line ..., in spawn_main + File ".../multiprocessing/spawn.py", line ..., in _main + AttributeError: module '__main__' has no attribute 'knigit' + >>> process + + + See :ref:`multiprocessing-programming-spawn`. While this restriction is + not true if using the ``"fork"`` start method, as of Python ``3.14`` that + is no longer the default on any platform. See + :ref:`multiprocessing-start-methods`. + See also :gh:`132898`. + .. versionchanged:: 3.3 Added the *daemon* parameter. @@ -646,7 +704,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the The process's authentication key (a byte string). - When :mod:`multiprocessing` is initialized the main process is assigned a + When :mod:`!multiprocessing` is initialized the main process is assigned a random string using :func:`os.urandom`. When a :class:`Process` object is created, it will inherit the @@ -747,7 +805,7 @@ The :mod:`multiprocessing` package mostly replicates the API of the .. exception:: ProcessError - The base class of all :mod:`multiprocessing` exceptions. + The base class of all :mod:`!multiprocessing` exceptions. .. exception:: BufferTooShort @@ -787,19 +845,19 @@ If you use :class:`JoinableQueue` then you **must** call semaphore used to count the number of unfinished tasks may eventually overflow, raising an exception. -One difference from other Python queue implementations, is that :mod:`multiprocessing` +One difference from other Python queue implementations, is that :mod:`!multiprocessing` queues serializes all objects that are put into them using :mod:`pickle`. -The object return by the get method is a re-created object that does not share memory -with the original object. +The object returned by the get method is a re-created object that does not share +memory with the original object. Note that one can also create a shared queue by using a manager object -- see :ref:`multiprocessing-managers`. .. note:: - :mod:`multiprocessing` uses the usual :exc:`queue.Empty` and + :mod:`!multiprocessing` uses the usual :exc:`queue.Empty` and :exc:`queue.Full` exceptions to signal a timeout. They are not available in - the :mod:`multiprocessing` namespace so you need to import them from + the :mod:`!multiprocessing` namespace so you need to import them from :mod:`queue`. .. note:: @@ -847,7 +905,7 @@ For an example of the usage of queues for interprocess communication see :ref:`multiprocessing-examples`. -.. function:: Pipe([duplex]) +.. function:: Pipe(duplex=True) Returns a pair ``(conn1, conn2)`` of :class:`~multiprocessing.connection.Connection` objects representing the @@ -867,11 +925,15 @@ For an example of the usage of queues for interprocess communication see locks/semaphores. When a process first puts an item on the queue a feeder thread is started which transfers objects from a buffer into the pipe. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + The usual :exc:`queue.Empty` and :exc:`queue.Full` exceptions from the standard library's :mod:`queue` module are raised to signal timeouts. :class:`Queue` implements all the methods of :class:`queue.Queue` except for - :meth:`~queue.Queue.task_done` and :meth:`~queue.Queue.join`. + :meth:`~queue.Queue.task_done`, :meth:`~queue.Queue.join`, and + :meth:`~queue.Queue.shutdown`. .. method:: qsize() @@ -936,8 +998,13 @@ For an example of the usage of queues for interprocess communication see .. method:: close() - Indicate that no more data will be put on this queue by the current - process. The background thread will quit once it has flushed all buffered + Close the queue: release internal resources. + + A queue must not be used anymore after it is closed. For example, + :meth:`~Queue.get`, :meth:`~Queue.put` and :meth:`~Queue.empty` + methods must no longer be called. + + The background thread will quit once it has flushed all buffered data to the pipe. This is called automatically when the queue is garbage collected. @@ -977,6 +1044,9 @@ For an example of the usage of queues for interprocess communication see It is a simplified :class:`Queue` type, very close to a locked :class:`Pipe`. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + .. method:: close() Close the queue: release internal resources. @@ -1007,6 +1077,9 @@ For an example of the usage of queues for interprocess communication see :class:`JoinableQueue`, a :class:`Queue` subclass, is a queue which additionally has :meth:`task_done` and :meth:`join` methods. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + .. method:: task_done() Indicate that a formerly enqueued task is complete. Used by queue @@ -1080,8 +1153,8 @@ Miscellaneous .. function:: freeze_support() - Add support for when a program which uses :mod:`multiprocessing` has been - frozen to produce a Windows executable. (Has been tested with **py2exe**, + Add support for when a program which uses :mod:`!multiprocessing` has been + frozen to produce an executable. (Has been tested with **py2exe**, **PyInstaller** and **cx_Freeze**.) One needs to call this function straight after the ``if __name__ == @@ -1099,10 +1172,10 @@ Miscellaneous If the ``freeze_support()`` line is omitted then trying to run the frozen executable will raise :exc:`RuntimeError`. - Calling ``freeze_support()`` has no effect when invoked on any operating - system other than Windows. In addition, if the module is being run - normally by the Python interpreter on Windows (the program has not been - frozen), then ``freeze_support()`` has no effect. + Calling ``freeze_support()`` has no effect when the start method is not + *spawn*. In addition, if the module is being run normally by the Python + interpreter (the program has not been frozen), then ``freeze_support()`` + has no effect. .. function:: get_all_start_methods() @@ -1116,9 +1189,11 @@ Miscellaneous .. function:: get_context(method=None) Return a context object which has the same attributes as the - :mod:`multiprocessing` module. + :mod:`!multiprocessing` module. - If *method* is ``None`` then the default context is returned. + If *method* is ``None`` then the default context is returned. Note that if + the global start method has not been set, this will set it to the system default + See :ref:`global-start-method` for more details. Otherwise *method* should be ``'fork'``, ``'spawn'``, ``'forkserver'``. :exc:`ValueError` is raised if the specified start method is not available. See :ref:`multiprocessing-start-methods`. @@ -1129,10 +1204,9 @@ Miscellaneous Return the name of start method used for starting processes. - If the start method has not been fixed and *allow_none* is false, - then the start method is fixed to the default and the name is - returned. If the start method has not been fixed and *allow_none* - is true then ``None`` is returned. + If the global start method is not set and *allow_none* is ``False``, the global start + method is set to the default, and its name is returned. See + :ref:`global-start-method` for more details. The return value can be ``'fork'``, ``'spawn'``, ``'forkserver'`` or ``None``. See :ref:`multiprocessing-start-methods`. @@ -1149,7 +1223,7 @@ Miscellaneous Set the path of the Python interpreter to use when starting a child process. (By default :data:`sys.executable` is used). Embedders will probably need to - do some thing like :: + do something like :: set_executable(os.path.join(sys.exec_prefix, 'pythonw.exe')) @@ -1196,7 +1270,7 @@ Miscellaneous .. note:: - :mod:`multiprocessing` contains no analogues of + :mod:`!multiprocessing` contains no analogues of :func:`threading.active_count`, :func:`threading.enumerate`, :func:`threading.settrace`, :func:`threading.setprofile`, :class:`threading.Timer`, or :class:`threading.local`. @@ -1252,12 +1326,12 @@ Connection objects are usually created using Note that multiple connection objects may be polled at once by using :func:`multiprocessing.connection.wait`. - .. method:: send_bytes(buffer[, offset[, size]]) + .. method:: send_bytes(buf[, offset[, size]]) Send byte data from a :term:`bytes-like object` as a complete message. - If *offset* is given then data is read from that position in *buffer*. If - *size* is given then that many bytes will be read from buffer. Very large + If *offset* is given then data is read from that position in *buf*. If + *size* is given then that many bytes will be read from *buf*. Very large buffers (approximately 32 MiB+, though it depends on the OS) may raise a :exc:`ValueError` exception @@ -1277,18 +1351,18 @@ Connection objects are usually created using alias of :exc:`OSError`. - .. method:: recv_bytes_into(buffer[, offset]) + .. method:: recv_bytes_into(buf[, offset]) - Read into *buffer* a complete message of byte data sent from the other end + Read into *buf* a complete message of byte data sent from the other end of the connection and return the number of bytes in the message. Blocks until there is something to receive. Raises :exc:`EOFError` if there is nothing left to receive and the other end was closed. - *buffer* must be a writable :term:`bytes-like object`. If + *buf* must be a writable :term:`bytes-like object`. If *offset* is given then the message will be written into the buffer from that position. Offset must be a non-negative integer less than the - length of *buffer* (in bytes). + length of *buf* (in bytes). If the buffer is too short then a :exc:`BufferTooShort` exception is raised and the complete message is available as ``e.args[0]`` where ``e`` @@ -1359,6 +1433,9 @@ object -- see :ref:`multiprocessing-managers`. A barrier object: a clone of :class:`threading.Barrier`. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + .. versionadded:: 3.3 .. class:: BoundedSemaphore([value]) @@ -1366,9 +1443,18 @@ object -- see :ref:`multiprocessing-managers`. A bounded semaphore object: a close analog of :class:`threading.BoundedSemaphore`. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + A solitary difference from its close analog exists: its ``acquire`` method's first argument is named *block*, as is consistent with :meth:`Lock.acquire`. + .. method:: locked() + + Return a boolean indicating whether this object is locked right now. + + .. versionadded:: 3.14 + .. note:: On macOS, this is indistinguishable from :class:`Semaphore` because ``sem_getvalue()`` is not implemented on that platform. @@ -1378,7 +1464,10 @@ object -- see :ref:`multiprocessing-managers`. A condition variable: an alias for :class:`threading.Condition`. If *lock* is specified then it should be a :class:`Lock` or :class:`RLock` - object from :mod:`multiprocessing`. + object from :mod:`!multiprocessing`. + + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. .. versionchanged:: 3.3 The :meth:`~threading.Condition.wait_for` method was added. @@ -1387,6 +1476,8 @@ object -- see :ref:`multiprocessing-managers`. A clone of :class:`threading.Event`. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. .. class:: Lock() @@ -1402,6 +1493,9 @@ object -- see :ref:`multiprocessing-managers`. instance of ``multiprocessing.synchronize.Lock`` initialized with a default context. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + :class:`Lock` supports the :term:`context manager` protocol and thus may be used in :keyword:`with` statements. @@ -1459,6 +1553,9 @@ object -- see :ref:`multiprocessing-managers`. instance of ``multiprocessing.synchronize.RLock`` initialized with a default context. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + :class:`RLock` supports the :term:`context manager` protocol and thus may be used in :keyword:`with` statements. @@ -1518,9 +1615,28 @@ object -- see :ref:`multiprocessing-managers`. A semaphore object: a close analog of :class:`threading.Semaphore`. + Instantiating this class may set the global start method. See + :ref:`global-start-method` for more details. + A solitary difference from its close analog exists: its ``acquire`` method's first argument is named *block*, as is consistent with :meth:`Lock.acquire`. + + .. method:: get_value() + + Return the current value of semaphore. + + Note that this may raise :exc:`NotImplementedError` on platforms like + macOS where ``sem_getvalue()`` is not implemented. + + + .. method:: locked() + + Return a boolean indicating whether this object is locked right now. + + .. versionadded:: 3.14 + + .. note:: On macOS, ``sem_timedwait`` is unsupported, so calling ``acquire()`` with @@ -1578,11 +1694,14 @@ inherited by child processes. value is actually a synchronized wrapper for the array. *typecode_or_type* determines the type of the elements of the returned array: - it is either a ctypes type or a one character typecode of the kind used by - the :mod:`array` module. If *size_or_initializer* is an integer, then it - determines the length of the array, and the array will be initially zeroed. - Otherwise, *size_or_initializer* is a sequence which is used to initialize - the array and whose length determines the length of the array. + it is either a :ref:`ctypes type ` or a one + character typecode of the kind used by the :mod:`array` module with the + exception of ``'w'``, which is not supported. In addition, the ``'c'`` + typecode is an alias for :class:`ctypes.c_char`. If *size_or_initializer* + is an integer, then it determines the length of the array, and the array + will be initially zeroed. Otherwise, *size_or_initializer* is a sequence + which is used to initialize the array and whose length determines the length + of the array. If *lock* is ``True`` (the default) then a new lock object is created to synchronize access to the value. If *lock* is a :class:`Lock` or @@ -1594,16 +1713,19 @@ inherited by child processes. Note that *lock* is a keyword only argument. Note that an array of :data:`ctypes.c_char` has *value* and *raw* - attributes which allow one to use it to store and retrieve strings. + attributes which can both be used to store and retrieve byte strings. + While *raw* allows interaction with a :class:`bytes` object the full size of + the array, reading *value* will terminate after a null byte, like most + programming languages handle strings. -The :mod:`multiprocessing.sharedctypes` module -"""""""""""""""""""""""""""""""""""""""""""""" +The :mod:`!multiprocessing.sharedctypes` module +""""""""""""""""""""""""""""""""""""""""""""""" .. module:: multiprocessing.sharedctypes :synopsis: Allocate ctypes objects from shared memory. -The :mod:`multiprocessing.sharedctypes` module provides functions for allocating +The :mod:`!multiprocessing.sharedctypes` module provides functions for allocating :mod:`ctypes` objects from shared memory which can be inherited by child processes. @@ -1646,7 +1768,7 @@ processes. attributes which allow one to use it to store and retrieve strings -- see documentation for :mod:`ctypes`. -.. function:: Array(typecode_or_type, size_or_initializer, *, lock=True) +.. function:: Array(typecode_or_type, size_or_initializer, *, lock=True, ctx=None) The same as :func:`RawArray` except that depending on the value of *lock* a process-safe synchronization wrapper may be returned instead of a raw ctypes @@ -1660,9 +1782,13 @@ processes. automatically protected by a lock, so it will not necessarily be "process-safe". - Note that *lock* is a keyword-only argument. + *ctx* is a context object, or ``None`` (use the current context). If ``None``, + calling this may set the global start method. See + :ref:`global-start-method` for more details. -.. function:: Value(typecode_or_type, *args, lock=True) + Note that *lock* and *ctx* are keyword-only parameters. + +.. function:: Value(typecode_or_type, *args, lock=True, ctx=None) The same as :func:`RawValue` except that depending on the value of *lock* a process-safe synchronization wrapper may be returned instead of a raw ctypes @@ -1675,19 +1801,27 @@ processes. automatically protected by a lock, so it will not necessarily be "process-safe". - Note that *lock* is a keyword-only argument. + *ctx* is a context object, or ``None`` (use the current context). If ``None``, + calling this may set the global start method. See + :ref:`global-start-method` for more details. + + Note that *lock* and *ctx* are keyword-only parameters. .. function:: copy(obj) Return a ctypes object allocated from shared memory which is a copy of the ctypes object *obj*. -.. function:: synchronized(obj[, lock]) +.. function:: synchronized(obj, lock=None, ctx=None) Return a process-safe wrapper object for a ctypes object which uses *lock* to synchronize access. If *lock* is ``None`` (the default) then a :class:`multiprocessing.RLock` object is created automatically. + *ctx* is a context object, or ``None`` (use the current context). If ``None``, + calling this may set the global start method. See + :ref:`global-start-method` for more details. + A synchronized wrapper will have two methods in addition to those of the object it wraps: :meth:`get_obj` returns the wrapped object and :meth:`get_lock` returns the lock object used for synchronization. @@ -1805,8 +1939,9 @@ their parent process exits. The manager classes are defined in the *serializer* must be ``'pickle'`` (use :mod:`pickle` serialization) or ``'xmlrpclib'`` (use :mod:`xmlrpc.client` serialization). - *ctx* is a context object, or ``None`` (use the current context). See the - :func:`get_context` function. + *ctx* is a context object, or ``None`` (use the current context). If ``None``, + calling this may set the global start method. See + :ref:`global-start-method` for more details. *shutdown_timeout* is a timeout in seconds used to wait until the process used by the manager completes in the :meth:`shutdown` method. If the @@ -2190,7 +2325,7 @@ demonstrates a level of control over the synchronization. .. note:: - The proxy types in :mod:`multiprocessing` do nothing to support comparisons + The proxy types in :mod:`!multiprocessing` do nothing to support comparisons by value. So, for instance, we have: .. doctest:: @@ -2299,7 +2434,9 @@ with the :class:`Pool` class. the worker processes. Usually a pool is created using the function :func:`multiprocessing.Pool` or the :meth:`Pool` method of a context object. In both cases *context* is set - appropriately. + appropriately. If ``None``, calling this function will have the side effect + of setting the current global start method if it has not been set already. + See the :func:`get_context` function. Note that the methods of the pool object should only be called by the process which created the pool. @@ -2330,7 +2467,7 @@ with the :class:`Pool` class. duration of the Pool's work queue. A frequent pattern found in other systems (such as Apache, mod_wsgi, etc) to free resources held by workers is to allow a worker within a pool to complete only a set - amount of work before being exiting, being cleaned up and a new + amount of work before exiting, being cleaned up and a new process spawned to replace the old one. The *maxtasksperchild* argument to the :class:`Pool` exposes this ability to the end user. @@ -2515,7 +2652,7 @@ Usually message passing between processes is done using queues or by using :class:`~Connection` objects returned by :func:`~multiprocessing.Pipe`. -However, the :mod:`multiprocessing.connection` module allows some extra +However, the :mod:`!multiprocessing.connection` module allows some extra flexibility. It basically gives a high level message oriented API for dealing with sockets or Windows named pipes. It also has support for *digest authentication* using the :mod:`hmac` module, and for polling @@ -2773,6 +2910,16 @@ between themselves. Suitable authentication keys can also be generated by using :func:`os.urandom`. +This authentication protects :class:`Listener` and :func:`Client` connections, +which are reachable by address. It is not applied to the anonymous pipes +created by :func:`~multiprocessing.Pipe` or used internally by +:class:`~multiprocessing.Queue`. +:mod:`multiprocessing` treats all local processes running as the same user as +trusted; on most operating systems such processes can access each other's pipe +file descriptors regardless. Applications that require isolation between +processes of the same user must arrange it at the operating-system level -- +for example, by running workers under a different user account or in a sandbox. + Logging ^^^^^^^ @@ -2784,7 +2931,7 @@ handler type) for messages from different processes to get mixed up. .. currentmodule:: multiprocessing .. function:: get_logger() - Returns the logger used by :mod:`multiprocessing`. If necessary, a new one + Returns the logger used by :mod:`!multiprocessing`. If necessary, a new one will be created. When first created the logger has level :const:`logging.NOTSET` and no @@ -2822,18 +2969,18 @@ Below is an example session with logging turned on:: For a full table of logging levels, see the :mod:`logging` module. -The :mod:`multiprocessing.dummy` module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +The :mod:`!multiprocessing.dummy` module +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. module:: multiprocessing.dummy :synopsis: Dumb wrapper around threading. -:mod:`multiprocessing.dummy` replicates the API of :mod:`multiprocessing` but is +:mod:`!multiprocessing.dummy` replicates the API of :mod:`!multiprocessing` but is no more than a wrapper around the :mod:`threading` module. .. currentmodule:: multiprocessing.pool -In particular, the ``Pool`` function provided by :mod:`multiprocessing.dummy` +In particular, the ``Pool`` function provided by :mod:`!multiprocessing.dummy` returns an instance of :class:`ThreadPool`, which is a subclass of :class:`Pool` that supports all the same method calls but uses a pool of worker threads rather than worker processes. @@ -2878,7 +3025,7 @@ Programming guidelines ---------------------- There are certain guidelines and idioms which should be adhered to when using -:mod:`multiprocessing`. +:mod:`!multiprocessing`. All start methods @@ -2919,7 +3066,7 @@ Joining zombie processes Better to inherit than pickle/unpickle When using the *spawn* or *forkserver* start methods many types - from :mod:`multiprocessing` need to be picklable so that child + from :mod:`!multiprocessing` need to be picklable so that child processes can use them. However, one should generally avoid sending shared objects to other processes using pipes or queues. Instead you should arrange the program so that a process which @@ -3009,7 +3156,7 @@ Explicitly pass resources to child processes Beware of replacing :data:`sys.stdin` with a "file like object" - :mod:`multiprocessing` originally unconditionally called:: + :mod:`!multiprocessing` originally unconditionally called:: os.close(sys.stdin.fileno()) @@ -3051,10 +3198,10 @@ start method. More picklability - Ensure that all arguments to :meth:`Process.__init__` are picklable. - Also, if you subclass :class:`~multiprocessing.Process` then make sure that - instances will be picklable when the :meth:`Process.start - ` method is called. + Ensure that all arguments to :class:`~multiprocessing.Process` are + picklable. Also, if you subclass ``Process.__init__``, you must make sure + that instances will be picklable when the + :meth:`Process.start ` method is called. Global variables diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst index f6260383b2b057a..74c97e8c9a97598 100644 --- a/Doc/library/netrc.rst +++ b/Doc/library/netrc.rst @@ -24,12 +24,14 @@ the Unix :program:`ftp` program and other FTP clients. a :exc:`FileNotFoundError` exception will be raised. Parse errors will raise :exc:`NetrcParseError` with diagnostic information including the file name, line number, and terminating token. + If no argument is specified on a POSIX system, the presence of passwords in the :file:`.netrc` file will raise a :exc:`NetrcParseError` if the file ownership or permissions are insecure (owned by a user other than the user running the process, or accessible for read or write by any other user). This implements security behavior equivalent to that of ftp and other - programs that use :file:`.netrc`. + programs that use :file:`.netrc`. Such security checks are not available + on platforms that do not support :func:`os.getuid`. .. versionchanged:: 3.4 Added the POSIX permission check. diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index 681d0b76f2a14b7..0ed24b4e6c5225d 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -69,11 +69,11 @@ The numeric tower .. attribute:: numerator - Abstract. + Abstract. The numerator of this rational number. .. attribute:: denominator - Abstract. + Abstract. The denominator of this rational number. .. class:: Integral @@ -90,20 +90,7 @@ Notes for type implementers Implementers should be careful to make equal numbers equal and hash them to the same values. This may be subtle if there are two different -extensions of the real numbers. For example, :class:`fractions.Fraction` -implements :func:`hash` as follows:: - - def __hash__(self): - if self.denominator == 1: - # Get integers right. - return hash(self.numerator) - # Expensive check, but definitely correct. - if self == float(self): - return hash(float(self)) - else: - # Use tuple's hash to avoid a high collision rate on - # simple fractions. - return hash((self.numerator, self.denominator)) +extensions of the real numbers. See also :ref:`numeric-hash`. Adding More Numeric ABCs diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index e8e71068dd99ebc..d88e008fa926f13 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -15,7 +15,7 @@ -------------- -The :mod:`operator` module exports a set of efficient functions corresponding to +The :mod:`!operator` module exports a set of efficient functions corresponding to the intrinsic operators of Python. For example, ``operator.add(x, y)`` is equivalent to the expression ``x+y``. Many function names are those used for special methods, without the double underscores. For backward compatibility, @@ -112,7 +112,7 @@ The mathematical and bitwise operations are the most numerous: .. function:: and_(a, b) __and__(a, b) - Return the bitwise and of *a* and *b*. + Return ``a & b``. .. function:: floordiv(a, b) @@ -136,13 +136,13 @@ The mathematical and bitwise operations are the most numerous: __inv__(obj) __invert__(obj) - Return the bitwise inverse of the number *obj*. This is equivalent to ``~obj``. + Return ``~obj``. .. function:: lshift(a, b) __lshift__(a, b) - Return *a* shifted left by *b*. + Return ``a << b``. .. function:: mod(a, b) @@ -154,7 +154,7 @@ The mathematical and bitwise operations are the most numerous: .. function:: mul(a, b) __mul__(a, b) - Return ``a * b``, for *a* and *b* numbers. + Return ``a * b``. .. function:: matmul(a, b) @@ -174,25 +174,25 @@ The mathematical and bitwise operations are the most numerous: .. function:: or_(a, b) __or__(a, b) - Return the bitwise or of *a* and *b*. + Return ``a | b``. .. function:: pos(obj) __pos__(obj) - Return *obj* positive (``+obj``). + Return ``+obj``. .. function:: pow(a, b) __pow__(a, b) - Return ``a ** b``, for *a* and *b* numbers. + Return ``a ** b``. .. function:: rshift(a, b) __rshift__(a, b) - Return *a* shifted right by *b*. + Return ``a >> b``. .. function:: sub(a, b) @@ -211,7 +211,7 @@ The mathematical and bitwise operations are the most numerous: .. function:: xor(a, b) __xor__(a, b) - Return the bitwise exclusive or of *a* and *b*. + Return ``a ^ b``. Operations which work with sequences (some of them with mappings too) include: @@ -275,7 +275,7 @@ The following operation works with callables: .. versionadded:: 3.11 -The :mod:`operator` module also defines tools for generalized attribute and item +The :mod:`!operator` module also defines tools for generalized attribute and item lookups. These are useful for making fast field extractors as arguments for :func:`map`, :func:`sorted`, :meth:`itertools.groupby`, or other functions that expect a function argument. @@ -390,7 +390,7 @@ Mapping Operators to Functions ------------------------------ This table shows how abstract operations correspond to operator symbols in the -Python syntax and the functions in the :mod:`operator` module. +Python syntax and the functions in the :mod:`!operator` module. +-----------------------+-------------------------+---------------------------------------+ | Operation | Syntax | Function | @@ -405,13 +405,18 @@ Python syntax and the functions in the :mod:`operator` module. +-----------------------+-------------------------+---------------------------------------+ | Division | ``a // b`` | ``floordiv(a, b)`` | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise And | ``a & b`` | ``and_(a, b)`` | +| Bitwise And, or | ``a & b`` | ``and_(a, b)`` | +| Intersection | | | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise Exclusive Or | ``a ^ b`` | ``xor(a, b)`` | +| Bitwise Exclusive Or, | ``a ^ b`` | ``xor(a, b)`` | +| or Symmetric | | | +| Difference | | | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise Inversion | ``~ a`` | ``invert(a)`` | +| Bitwise Inversion, or | ``~ a`` | ``invert(a)`` | +| Complement | | | +-----------------------+-------------------------+---------------------------------------+ -| Bitwise Or | ``a | b`` | ``or_(a, b)`` | +| Bitwise Or, or | ``a | b`` | ``or_(a, b)`` | +| Union | | | +-----------------------+-------------------------+---------------------------------------+ | Exponentiation | ``a ** b`` | ``pow(a, b)`` | +-----------------------+-------------------------+---------------------------------------+ diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index ff327cf9162a8cc..51827e1f8da5342 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -20,7 +20,7 @@ The standard library includes three argument parsing libraries: * :mod:`getopt`: a module that closely mirrors the procedural C ``getopt`` API. Included in the standard library since before the initial Python 1.0 release. -* :mod:`optparse`: a declarative replacement for ``getopt`` that +* :mod:`!optparse`: a declarative replacement for ``getopt`` that provides equivalent functionality without requiring each application to implement its own procedural option parsing logic. Included in the standard library since the Python 2.3 release. @@ -37,10 +37,10 @@ the highest level of baseline functionality with the least application level cod However, it also serves a niche use case as a tool for prototyping and testing command line argument handling in ``getopt``-based C applications. -:mod:`optparse` should be considered as an alternative to :mod:`argparse` in the +:mod:`!optparse` should be considered as an alternative to :mod:`argparse` in the following cases: -* an application is already using :mod:`optparse` and doesn't want to risk the +* an application is already using :mod:`!optparse` and doesn't want to risk the subtle behavioural changes that may arise when migrating to :mod:`argparse` * the application requires additional control over the way options and positional parameters are interleaved on the command line (including @@ -55,7 +55,7 @@ following cases: behavior which ``argparse`` does not support, but which can be implemented in terms of the lower level interface offered by ``optparse`` -These considerations also mean that :mod:`optparse` is likely to provide a +These considerations also mean that :mod:`!optparse` is likely to provide a better foundation for library authors writing third party command line argument processing libraries. @@ -126,15 +126,15 @@ application use case. Introduction ------------ -:mod:`optparse` is a more convenient, flexible, and powerful library for parsing +:mod:`!optparse` is a more convenient, flexible, and powerful library for parsing command-line options than the minimalist :mod:`getopt` module. -:mod:`optparse` uses a more declarative style of command-line parsing: +:mod:`!optparse` uses a more declarative style of command-line parsing: you create an instance of :class:`OptionParser`, populate it with options, and parse the command line. -:mod:`optparse` allows users to specify options in the conventional +:mod:`!optparse` allows users to specify options in the conventional GNU/POSIX syntax, and additionally generates usage and help messages for you. -Here's an example of using :mod:`optparse` in a simple script:: +Here's an example of using :mod:`!optparse` in a simple script:: from optparse import OptionParser ... @@ -152,11 +152,11 @@ on the command-line, for example:: --file=outfile -q -As it parses the command line, :mod:`optparse` sets attributes of the +As it parses the command line, :mod:`!optparse` sets attributes of the ``options`` object returned by :meth:`~OptionParser.parse_args` based on user-supplied command-line values. When :meth:`~OptionParser.parse_args` returns from parsing this command line, ``options.filename`` will be ``"outfile"`` and ``options.verbose`` will be -``False``. :mod:`optparse` supports both long and short options, allows short +``False``. :mod:`!optparse` supports both long and short options, allows short options to be merged together, and allows options to be associated with their arguments in a variety of ways. Thus, the following command lines are all equivalent to the above example:: @@ -171,7 +171,7 @@ Additionally, users can run one of the following :: -h --help -and :mod:`optparse` will print out a brief summary of your script's options: +and :mod:`!optparse` will print out a brief summary of your script's options: .. code-block:: text @@ -191,7 +191,7 @@ where the value of *yourscript* is determined at runtime (normally from Background ---------- -:mod:`optparse` was explicitly designed to encourage the creation of programs +:mod:`!optparse` was explicitly designed to encourage the creation of programs with straightforward command-line interfaces that follow the conventions established by the :c:func:`!getopt` family of functions available to C developers. To that end, it supports only the most common command-line syntax and semantics @@ -223,7 +223,7 @@ option options to be merged into a single argument, e.g. ``-x -F`` is equivalent to ``-xF``. The GNU project introduced ``--`` followed by a series of hyphen-separated words, e.g. ``--file`` or ``--dry-run``. These are the - only two option syntaxes provided by :mod:`optparse`. + only two option syntaxes provided by :mod:`!optparse`. Some other option syntaxes that the world has seen include: @@ -240,7 +240,7 @@ option * a slash followed by a letter, or a few letters, or a word, e.g. ``/f``, ``/file`` - These option syntaxes are not supported by :mod:`optparse`, and they never + These option syntaxes are not supported by :mod:`!optparse`, and they never will be. This is deliberate: the first three are non-standard on any environment, and the last only makes sense if you're exclusively targeting Windows or certain legacy platforms (e.g. VMS, MS-DOS). @@ -248,7 +248,7 @@ option option argument an argument that follows an option, is closely associated with that option, and is consumed from the argument list when that option is. With - :mod:`optparse`, option arguments may either be in a separate argument from + :mod:`!optparse`, option arguments may either be in a separate argument from their option: .. code-block:: text @@ -268,7 +268,7 @@ option argument will take an argument if they see it, and won't if they don't. This is somewhat controversial, because it makes parsing ambiguous: if ``-a`` takes an optional argument and ``-b`` is another option entirely, how do we - interpret ``-ab``? Because of this ambiguity, :mod:`optparse` does not + interpret ``-ab``? Because of this ambiguity, :mod:`!optparse` does not support this feature. positional argument @@ -278,7 +278,7 @@ positional argument required option an option that must be supplied on the command-line; note that the phrase - "required option" is self-contradictory in English. :mod:`optparse` doesn't + "required option" is self-contradictory in English. :mod:`!optparse` doesn't prevent you from implementing required options, but doesn't give you much help at it either. @@ -357,9 +357,9 @@ too many options can overwhelm users and make your code much harder to maintain. Tutorial -------- -While :mod:`optparse` is quite flexible and powerful, it's also straightforward +While :mod:`!optparse` is quite flexible and powerful, it's also straightforward to use in most cases. This section covers the code patterns that are common to -any :mod:`optparse`\ -based program. +any :mod:`!optparse`\ -based program. First, you need to import the OptionParser class; then, early in the main program, create an OptionParser instance:: @@ -374,7 +374,7 @@ Then you can start defining options. The basic syntax is:: attr=value, ...) Each option has one or more option strings, such as ``-f`` or ``--file``, -and several option attributes that tell :mod:`optparse` what to expect and what +and several option attributes that tell :mod:`!optparse` what to expect and what to do when it encounters that option on the command line. Typically, each option will have one short option string and one long option @@ -389,10 +389,10 @@ string overall. The option strings passed to :meth:`OptionParser.add_option` are effectively labels for the option defined by that call. For brevity, we will frequently refer to -*encountering an option* on the command line; in reality, :mod:`optparse` +*encountering an option* on the command line; in reality, :mod:`!optparse` encounters *option strings* and looks up options from them. -Once all of your options are defined, instruct :mod:`optparse` to parse your +Once all of your options are defined, instruct :mod:`!optparse` to parse your program's command line:: (options, args) = parser.parse_args() @@ -420,14 +420,14 @@ most fundamental. Understanding option actions ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Actions tell :mod:`optparse` what to do when it encounters an option on the -command line. There is a fixed set of actions hard-coded into :mod:`optparse`; +Actions tell :mod:`!optparse` what to do when it encounters an option on the +command line. There is a fixed set of actions hard-coded into :mod:`!optparse`; adding new actions is an advanced topic covered in section -:ref:`optparse-extending-optparse`. Most actions tell :mod:`optparse` to store +:ref:`optparse-extending-optparse`. Most actions tell :mod:`!optparse` to store a value in some variable---for example, take a string from the command line and store it in an attribute of ``options``. -If you don't specify an option action, :mod:`optparse` defaults to ``store``. +If you don't specify an option action, :mod:`!optparse` defaults to ``store``. .. _optparse-store-action: @@ -435,7 +435,7 @@ If you don't specify an option action, :mod:`optparse` defaults to ``store``. The store action ^^^^^^^^^^^^^^^^ -The most common option action is ``store``, which tells :mod:`optparse` to take +The most common option action is ``store``, which tells :mod:`!optparse` to take the next argument (or the remainder of the current argument), ensure that it is of the correct type, and store it to your chosen destination. @@ -444,16 +444,16 @@ For example:: parser.add_option("-f", "--file", action="store", type="string", dest="filename") -Now let's make up a fake command line and ask :mod:`optparse` to parse it:: +Now let's make up a fake command line and ask :mod:`!optparse` to parse it:: args = ["-f", "foo.txt"] (options, args) = parser.parse_args(args) -When :mod:`optparse` sees the option string ``-f``, it consumes the next +When :mod:`!optparse` sees the option string ``-f``, it consumes the next argument, ``foo.txt``, and stores it in ``options.filename``. So, after this call to :meth:`~OptionParser.parse_args`, ``options.filename`` is ``"foo.txt"``. -Some other option types supported by :mod:`optparse` are ``int`` and ``float``. +Some other option types supported by :mod:`!optparse` are ``int`` and ``float``. Here's an option that expects an integer argument:: parser.add_option("-n", type="int", dest="num") @@ -470,19 +470,19 @@ right up against the option: since ``-n42`` (one argument) is equivalent to will print ``42``. -If you don't specify a type, :mod:`optparse` assumes ``string``. Combined with +If you don't specify a type, :mod:`!optparse` assumes ``string``. Combined with the fact that the default action is ``store``, that means our first example can be a lot shorter:: parser.add_option("-f", "--file", dest="filename") -If you don't supply a destination, :mod:`optparse` figures out a sensible +If you don't supply a destination, :mod:`!optparse` figures out a sensible default from the option strings: if the first long option string is ``--foo-bar``, then the default destination is ``foo_bar``. If there are no -long option strings, :mod:`optparse` looks at the first short option string: the +long option strings, :mod:`!optparse` looks at the first short option string: the default destination for ``-f`` is ``f``. -:mod:`optparse` also includes the built-in ``complex`` type. Adding +:mod:`!optparse` also includes the built-in ``complex`` type. Adding types is covered in section :ref:`optparse-extending-optparse`. @@ -492,7 +492,7 @@ Handling boolean (flag) options ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Flag options---set a variable to true or false when a particular option is -seen---are quite common. :mod:`optparse` supports them with two separate actions, +seen---are quite common. :mod:`!optparse` supports them with two separate actions, ``store_true`` and ``store_false``. For example, you might have a ``verbose`` flag that is turned on with ``-v`` and off with ``-q``:: @@ -503,7 +503,7 @@ Here we have two different options with the same destination, which is perfectly OK. (It just means you have to be a bit careful when setting default values---see below.) -When :mod:`optparse` encounters ``-v`` on the command line, it sets +When :mod:`!optparse` encounters ``-v`` on the command line, it sets ``options.verbose`` to ``True``; when it encounters ``-q``, ``options.verbose`` is set to ``False``. @@ -513,7 +513,7 @@ When :mod:`optparse` encounters ``-v`` on the command line, it sets Other actions ^^^^^^^^^^^^^ -Some other actions supported by :mod:`optparse` are: +Some other actions supported by :mod:`!optparse` are: ``"store_const"`` store a constant value, pre-set via :attr:`Option.const` @@ -539,11 +539,11 @@ Default values All of the above examples involve setting some variable (the "destination") when certain command-line options are seen. What happens if those options are never seen? Since we didn't supply any defaults, they are all set to ``None``. This -is usually fine, but sometimes you want more control. :mod:`optparse` lets you +is usually fine, but sometimes you want more control. :mod:`!optparse` lets you supply a default value for each destination, which is assigned before the command line is parsed. -First, consider the verbose/quiet example. If we want :mod:`optparse` to set +First, consider the verbose/quiet example. If we want :mod:`!optparse` to set ``verbose`` to ``True`` unless ``-q`` is seen, then we can do this:: parser.add_option("-v", action="store_true", dest="verbose", default=True) @@ -582,7 +582,7 @@ values, not both. Generating help ^^^^^^^^^^^^^^^ -:mod:`optparse`'s ability to generate help and usage text automatically is +:mod:`!optparse`'s ability to generate help and usage text automatically is useful for creating user-friendly command-line interfaces. All you have to do is supply a :attr:`~Option.help` value for each option, and optionally a short usage message for your whole program. Here's an OptionParser populated with @@ -603,7 +603,7 @@ user-friendly (documented) options:: help="interaction mode: novice, intermediate, " "or expert [default: %default]") -If :mod:`optparse` encounters either ``-h`` or ``--help`` on the +If :mod:`!optparse` encounters either ``-h`` or ``--help`` on the command-line, or if you just call :meth:`parser.print_help`, it prints the following to standard output: @@ -620,26 +620,26 @@ following to standard output: -m MODE, --mode=MODE interaction mode: novice, intermediate, or expert [default: intermediate] -(If the help output is triggered by a help option, :mod:`optparse` exits after +(If the help output is triggered by a help option, :mod:`!optparse` exits after printing the help text.) -There's a lot going on here to help :mod:`optparse` generate the best possible +There's a lot going on here to help :mod:`!optparse` generate the best possible help message: * the script defines its own usage message:: usage = "usage: %prog [options] arg1 arg2" - :mod:`optparse` expands ``%prog`` in the usage string to the name of the + :mod:`!optparse` expands ``%prog`` in the usage string to the name of the current program, i.e. ``os.path.basename(sys.argv[0])``. The expanded string is then printed before the detailed option help. - If you don't supply a usage string, :mod:`optparse` uses a bland but sensible + If you don't supply a usage string, :mod:`!optparse` uses a bland but sensible default: ``"Usage: %prog [options]"``, which is fine if your script doesn't take any positional arguments. * every option defines a help string, and doesn't worry about - line-wrapping---\ :mod:`optparse` takes care of wrapping lines and making + line-wrapping---\ :mod:`!optparse` takes care of wrapping lines and making the help output look good. * options that take a value indicate this fact in their automatically generated @@ -649,7 +649,7 @@ help message: Here, "MODE" is called the meta-variable: it stands for the argument that the user is expected to supply to ``-m``/``--mode``. By default, - :mod:`optparse` converts the destination variable name to uppercase and uses + :mod:`!optparse` converts the destination variable name to uppercase and uses that for the meta-variable. Sometimes, that's not what you want---for example, the ``--filename`` option explicitly sets ``metavar="FILE"``, resulting in this automatically generated option description:: @@ -663,7 +663,7 @@ help message: way to make your help text a lot clearer and more useful for end users. * options that have a default value can include ``%default`` in the help - string---\ :mod:`optparse` will replace it with :func:`str` of the option's + string---\ :mod:`!optparse` will replace it with :func:`str` of the option's default value. If an option has no default value (or the default value is ``None``), ``%default`` expands to ``none``. @@ -779,14 +779,14 @@ option groups is: Printing a version string ^^^^^^^^^^^^^^^^^^^^^^^^^ -Similar to the brief usage string, :mod:`optparse` can also print a version +Similar to the brief usage string, :mod:`!optparse` can also print a version string for your program. You have to supply the string as the ``version`` argument to OptionParser:: parser = OptionParser(usage="%prog [-f] [-q]", version="%prog 1.0") ``%prog`` is expanded just like it is in ``usage``. Apart from that, -``version`` can contain anything you like. When you supply it, :mod:`optparse` +``version`` can contain anything you like. When you supply it, :mod:`!optparse` automatically adds a ``--version`` option to your parser. If it encounters this option on the command line, it expands your ``version`` string (by replacing ``%prog``), prints it to stdout, and exits. @@ -815,10 +815,10 @@ The following two methods can be used to print and get the ``version`` string: .. _optparse-how-optparse-handles-errors: -How :mod:`optparse` handles errors -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +How :mod:`!optparse` handles errors +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -There are two broad classes of errors that :mod:`optparse` has to worry about: +There are two broad classes of errors that :mod:`!optparse` has to worry about: programmer errors and user errors. Programmer errors are usually erroneous calls to :func:`OptionParser.add_option`, e.g. invalid option strings, unknown option attributes, missing option attributes, etc. These are dealt with in the @@ -826,7 +826,7 @@ usual way: raise an exception (either :exc:`optparse.OptionError` or :exc:`TypeError`) and let the program crash. Handling user errors is much more important, since they are guaranteed to happen -no matter how stable your code is. :mod:`optparse` can automatically detect +no matter how stable your code is. :mod:`!optparse` can automatically detect some user errors, such as bad option arguments (passing ``-n 4x`` where ``-n`` takes an integer argument), missing arguments (``-n`` at the end of the command line, where ``-n`` takes an argument of any type). Also, @@ -838,7 +838,7 @@ condition:: if options.a and options.b: parser.error("options -a and -b are mutually exclusive") -In either case, :mod:`optparse` handles the error the same way: it prints the +In either case, :mod:`!optparse` handles the error the same way: it prints the program's usage message and an error message to standard error and exits with error status 2. @@ -861,11 +861,11 @@ Or, where the user fails to pass a value at all: foo: error: -n option requires an argument -:mod:`optparse`\ -generated error messages take care always to mention the +:mod:`!optparse`\ -generated error messages take care always to mention the option involved in the error; be sure to do the same when calling :func:`OptionParser.error` from your application code. -If :mod:`optparse`'s default error-handling behaviour does not suit your needs, +If :mod:`!optparse`'s default error-handling behaviour does not suit your needs, you'll need to subclass OptionParser and override its :meth:`~OptionParser.exit` and/or :meth:`~OptionParser.error` methods. @@ -875,7 +875,7 @@ and/or :meth:`~OptionParser.error` methods. Putting it all together ^^^^^^^^^^^^^^^^^^^^^^^ -Here's what :mod:`optparse`\ -based scripts usually look like:: +Here's what :mod:`!optparse`\ -based scripts usually look like:: from optparse import OptionParser ... @@ -911,7 +911,7 @@ Reference Guide Creating the parser ^^^^^^^^^^^^^^^^^^^ -The first step in using :mod:`optparse` is to create an OptionParser instance. +The first step in using :mod:`!optparse` is to create an OptionParser instance. .. class:: OptionParser(...) @@ -921,7 +921,7 @@ The first step in using :mod:`optparse` is to create an OptionParser instance. ``usage`` (default: ``"%prog [options]"``) The usage summary to print when your program is run incorrectly or with a - help option. When :mod:`optparse` prints the usage string, it expands + help option. When :mod:`!optparse` prints the usage string, it expands ``%prog`` to ``os.path.basename(sys.argv[0])`` (or to ``prog`` if you passed that keyword argument). To suppress a usage message, pass the special value :const:`optparse.SUPPRESS_USAGE`. @@ -938,7 +938,7 @@ The first step in using :mod:`optparse` is to create an OptionParser instance. ``version`` (default: ``None``) A version string to print when the user supplies a version option. If you - supply a true value for ``version``, :mod:`optparse` automatically adds a + supply a true value for ``version``, :mod:`!optparse` automatically adds a version option with the single option string ``--version``. The substring ``%prog`` is expanded the same as for ``usage``. @@ -949,17 +949,17 @@ The first step in using :mod:`optparse` is to create an OptionParser instance. ``description`` (default: ``None``) A paragraph of text giving a brief overview of your program. - :mod:`optparse` reformats this paragraph to fit the current terminal width + :mod:`!optparse` reformats this paragraph to fit the current terminal width and prints it when the user requests help (after ``usage``, but before the list of options). ``formatter`` (default: a new :class:`IndentedHelpFormatter`) An instance of optparse.HelpFormatter that will be used for printing help - text. :mod:`optparse` provides two concrete classes for this purpose: + text. :mod:`!optparse` provides two concrete classes for this purpose: IndentedHelpFormatter and TitledHelpFormatter. ``add_help_option`` (default: ``True``) - If true, :mod:`optparse` will add a help option (with option strings ``-h`` + If true, :mod:`!optparse` will add a help option (with option strings ``-h`` and ``--help``) to the parser. ``prog`` @@ -997,7 +997,7 @@ the OptionParser constructor, as in:: (:func:`make_option` is a factory function for creating Option instances; currently it is an alias for the Option constructor. A future version of -:mod:`optparse` may split Option into several classes, and :func:`make_option` +:mod:`!optparse` may split Option into several classes, and :func:`make_option` will pick the right class to instantiate. Do not instantiate Option directly.) @@ -1027,12 +1027,12 @@ The canonical way to create an :class:`Option` instance is with the The keyword arguments define attributes of the new Option object. The most important option attribute is :attr:`~Option.action`, and it largely determines which other attributes are relevant or required. If you pass - irrelevant option attributes, or fail to pass required ones, :mod:`optparse` + irrelevant option attributes, or fail to pass required ones, :mod:`!optparse` raises an :exc:`OptionError` exception explaining your mistake. - An option's *action* determines what :mod:`optparse` does when it encounters + An option's *action* determines what :mod:`!optparse` does when it encounters this option on the command-line. The standard option actions hard-coded into - :mod:`optparse` are: + :mod:`!optparse` are: ``"store"`` store this option's argument (default) @@ -1066,7 +1066,7 @@ The canonical way to create an :class:`Option` instance is with the attributes; see :ref:`optparse-standard-option-actions`.) As you can see, most actions involve storing or updating a value somewhere. -:mod:`optparse` always creates a special object for this, conventionally called +:mod:`!optparse` always creates a special object for this, conventionally called ``options``, which is an instance of :class:`optparse.Values`. .. class:: Values @@ -1084,7 +1084,7 @@ For example, when you call :: parser.parse_args() -one of the first things :mod:`optparse` does is create the ``options`` object:: +one of the first things :mod:`!optparse` does is create the ``options`` object:: options = Values() @@ -1099,7 +1099,7 @@ and the command-line being parsed includes any of the following:: --file=foo --file foo -then :mod:`optparse`, on seeing this option, will do the equivalent of :: +then :mod:`!optparse`, on seeing this option, will do the equivalent of :: options.filename = "foo" @@ -1124,13 +1124,13 @@ Option attributes The following option attributes may be passed as keyword arguments to :meth:`OptionParser.add_option`. If you pass an option attribute that is not relevant to a particular option, or fail to pass a required option attribute, -:mod:`optparse` raises :exc:`OptionError`. +:mod:`!optparse` raises :exc:`OptionError`. .. attribute:: Option.action (default: ``"store"``) - Determines :mod:`optparse`'s behaviour when this option is seen on the + Determines :mod:`!optparse`'s behaviour when this option is seen on the command line; the available options are documented :ref:`here `. @@ -1147,8 +1147,8 @@ relevant to a particular option, or fail to pass a required option attribute, (default: derived from option strings) If the option's action implies writing or modifying a value somewhere, this - tells :mod:`optparse` where to write it: :attr:`~Option.dest` names an - attribute of the ``options`` object that :mod:`optparse` builds as it parses + tells :mod:`!optparse` where to write it: :attr:`~Option.dest` names an + attribute of the ``options`` object that :mod:`!optparse` builds as it parses the command line. .. attribute:: Option.default @@ -1161,7 +1161,7 @@ relevant to a particular option, or fail to pass a required option attribute, (default: 1) How many arguments of type :attr:`~Option.type` should be consumed when this - option is seen. If > 1, :mod:`optparse` will store a tuple of values to + option is seen. If > 1, :mod:`!optparse` will store a tuple of values to :attr:`~Option.dest`. .. attribute:: Option.const @@ -1207,7 +1207,7 @@ Standard option actions The various option actions all have slightly different requirements and effects. Most actions have several relevant option attributes which you may specify to -guide :mod:`optparse`'s behaviour; a few have required attributes, which you +guide :mod:`!optparse`'s behaviour; a few have required attributes, which you must specify for any option using that action. * ``"store"`` [relevant: :attr:`~Option.type`, :attr:`~Option.dest`, @@ -1225,9 +1225,9 @@ must specify for any option using that action. If :attr:`~Option.type` is not supplied, it defaults to ``"string"``. - If :attr:`~Option.dest` is not supplied, :mod:`optparse` derives a destination + If :attr:`~Option.dest` is not supplied, :mod:`!optparse` derives a destination from the first long option string (e.g., ``--foo-bar`` implies - ``foo_bar``). If there are no long option strings, :mod:`optparse` derives a + ``foo_bar``). If there are no long option strings, :mod:`!optparse` derives a destination from the first short option string (e.g., ``-f`` implies ``f``). Example:: @@ -1239,7 +1239,7 @@ must specify for any option using that action. -f foo.txt -p 1 -3.5 4 -fbar.txt - :mod:`optparse` will set :: + :mod:`!optparse` will set :: options.f = "foo.txt" options.point = (1.0, -3.5, 4.0) @@ -1259,7 +1259,7 @@ must specify for any option using that action. parser.add_option("--noisy", action="store_const", const=2, dest="verbose") - If ``--noisy`` is seen, :mod:`optparse` will set :: + If ``--noisy`` is seen, :mod:`!optparse` will set :: options.verbose = 2 @@ -1282,7 +1282,7 @@ must specify for any option using that action. The option must be followed by an argument, which is appended to the list in :attr:`~Option.dest`. If no default value for :attr:`~Option.dest` is - supplied, an empty list is automatically created when :mod:`optparse` first + supplied, an empty list is automatically created when :mod:`!optparse` first encounters this option on the command-line. If :attr:`~Option.nargs` > 1, multiple arguments are consumed, and a tuple of length :attr:`~Option.nargs` is appended to :attr:`~Option.dest`. @@ -1294,7 +1294,7 @@ must specify for any option using that action. parser.add_option("-t", "--tracks", action="append", type="int") - If ``-t3`` is seen on the command-line, :mod:`optparse` does the equivalent + If ``-t3`` is seen on the command-line, :mod:`!optparse` does the equivalent of:: options.tracks = [] @@ -1333,7 +1333,7 @@ must specify for any option using that action. parser.add_option("-v", action="count", dest="verbosity") - The first time ``-v`` is seen on the command line, :mod:`optparse` does the + The first time ``-v`` is seen on the command line, :mod:`!optparse` does the equivalent of:: options.verbosity = 0 @@ -1364,7 +1364,7 @@ must specify for any option using that action. listed in the help message. To omit an option entirely, use the special value :const:`optparse.SUPPRESS_HELP`. - :mod:`optparse` automatically adds a :attr:`~Option.help` option to all + :mod:`!optparse` automatically adds a :attr:`~Option.help` option to all OptionParsers, so you do not normally need to create one. Example:: @@ -1382,7 +1382,7 @@ must specify for any option using that action. help="Input file to read data from") parser.add_option("--secret", help=SUPPRESS_HELP) - If :mod:`optparse` sees either ``-h`` or ``--help`` on the command line, + If :mod:`!optparse` sees either ``-h`` or ``--help`` on the command line, it will print something like the following help message to stdout (assuming ``sys.argv[0]`` is ``"foo.py"``): @@ -1395,7 +1395,7 @@ must specify for any option using that action. -v Be moderately verbose --file=FILENAME Input file to read data from - After printing the help message, :mod:`optparse` terminates your process with + After printing the help message, :mod:`!optparse` terminates your process with ``sys.exit(0)``. * ``"version"`` @@ -1405,7 +1405,7 @@ must specify for any option using that action. ``print_version()`` method of OptionParser. Generally only relevant if the ``version`` argument is supplied to the OptionParser constructor. As with :attr:`~Option.help` options, you will rarely create ``version`` options, - since :mod:`optparse` automatically adds them when needed. + since :mod:`!optparse` automatically adds them when needed. .. _optparse-standard-option-types: @@ -1413,7 +1413,7 @@ must specify for any option using that action. Standard option types ^^^^^^^^^^^^^^^^^^^^^ -:mod:`optparse` has five built-in option types: ``"string"``, ``"int"``, +:mod:`!optparse` has five built-in option types: ``"string"``, ``"int"``, ``"choice"``, ``"float"`` and ``"complex"``. If you need to add new option types, see section :ref:`optparse-extending-optparse`. @@ -1432,7 +1432,7 @@ Integer arguments (type ``"int"``) are parsed as follows: The conversion is done by calling :func:`int` with the appropriate base (2, 8, -10, or 16). If this fails, so will :mod:`optparse`, although with a more useful +10, or 16). If this fails, so will :mod:`!optparse`, although with a more useful error message. ``"float"`` and ``"complex"`` option arguments are converted directly with @@ -1471,7 +1471,7 @@ The whole point of creating and populating an OptionParser is to call its ``options`` the same object that was passed in as *values*, or the ``optparse.Values`` - instance created by :mod:`optparse` + instance created by :mod:`!optparse` ``args`` the leftover positional arguments after all options have been processed @@ -1499,7 +1499,7 @@ provides several methods to help you out: .. method:: OptionParser.disable_interspersed_args() Set parsing to stop on the first non-option. For example, if ``-a`` and - ``-b`` are both simple options that take no arguments, :mod:`optparse` + ``-b`` are both simple options that take no arguments, :mod:`!optparse` normally accepts this syntax:: prog -a arg1 -b arg2 @@ -1554,7 +1554,7 @@ strings:: (This is particularly true if you've defined your own OptionParser subclass with some standard options.) -Every time you add an option, :mod:`optparse` checks for conflicts with existing +Every time you add an option, :mod:`!optparse` checks for conflicts with existing options. If it finds any, it invokes the current conflict-handling mechanism. You can set the conflict-handling mechanism either in the constructor:: @@ -1581,7 +1581,7 @@ intelligently and add conflicting options to it:: parser.add_option("-n", "--dry-run", ..., help="do no harm") parser.add_option("-n", "--noisy", ..., help="be noisy") -At this point, :mod:`optparse` detects that a previously added option is already +At this point, :mod:`!optparse` detects that a previously added option is already using the ``-n`` option string. Since ``conflict_handler`` is ``"resolve"``, it resolves the situation by removing ``-n`` from the earlier option's list of option strings. Now ``--dry-run`` is the only way for the user to activate @@ -1594,14 +1594,14 @@ that option. If the user asks for help, the help message will reflect that:: It's possible to whittle away the option strings for a previously added option until there are none left, and the user has no way of invoking that option from -the command-line. In that case, :mod:`optparse` removes that option completely, +the command-line. In that case, :mod:`!optparse` removes that option completely, so it doesn't show up in help text or anywhere else. Carrying on with our existing OptionParser:: parser.add_option("--dry-run", ..., help="new dry-run option") At this point, the original ``-n``/``--dry-run`` option is no longer -accessible, so :mod:`optparse` removes it, leaving this help text:: +accessible, so :mod:`!optparse` removes it, leaving this help text:: Options: ... @@ -1676,9 +1676,9 @@ OptionParser supports several other public methods: Option Callbacks ---------------- -When :mod:`optparse`'s built-in actions and types aren't quite enough for your -needs, you have two choices: extend :mod:`optparse` or define a callback option. -Extending :mod:`optparse` is more general, but overkill for a lot of simple +When :mod:`!optparse`'s built-in actions and types aren't quite enough for your +needs, you have two choices: extend :mod:`!optparse` or define a callback option. +Extending :mod:`!optparse` is more general, but overkill for a lot of simple cases. Quite often a simple callback is all you need. There are two steps to defining a callback option: @@ -1702,14 +1702,14 @@ only option attribute you must specify is ``callback``, the function to call:: ``callback`` is a function (or other callable object), so you must have already defined ``my_callback()`` when you create this callback option. In this simple -case, :mod:`optparse` doesn't even know if ``-c`` takes any arguments, +case, :mod:`!optparse` doesn't even know if ``-c`` takes any arguments, which usually means that the option takes no arguments---the mere presence of ``-c`` on the command-line is all it needs to know. In some circumstances, though, you might want your callback to consume an arbitrary number of command-line arguments. This is where writing callbacks gets tricky; it's covered later in this section. -:mod:`optparse` always passes four particular arguments to your callback, and it +:mod:`!optparse` always passes four particular arguments to your callback, and it will only pass additional arguments if you specify them via :attr:`~Option.callback_args` and :attr:`~Option.callback_kwargs`. Thus, the minimal callback function signature is:: @@ -1723,12 +1723,12 @@ callback option: :attr:`~Option.type` has its usual meaning: as with the ``"store"`` or ``"append"`` actions, it - instructs :mod:`optparse` to consume one argument and convert it to + instructs :mod:`!optparse` to consume one argument and convert it to :attr:`~Option.type`. Rather than storing the converted value(s) anywhere, - though, :mod:`optparse` passes it to your callback function. + though, :mod:`!optparse` passes it to your callback function. :attr:`~Option.nargs` - also has its usual meaning: if it is supplied and > 1, :mod:`optparse` will + also has its usual meaning: if it is supplied and > 1, :mod:`!optparse` will consume :attr:`~Option.nargs` arguments, each of which must be convertible to :attr:`~Option.type`. It then passes a tuple of converted values to your callback. @@ -1762,7 +1762,7 @@ where ``"--foobar"``.) ``value`` - is the argument to this option seen on the command-line. :mod:`optparse` will + is the argument to this option seen on the command-line. :mod:`!optparse` will only expect an argument if :attr:`~Option.type` is set; the type of ``value`` will be the type implied by the option's type. If :attr:`~Option.type` for this option is ``None`` (no argument expected), then ``value`` will be ``None``. If :attr:`~Option.nargs` @@ -1787,7 +1787,7 @@ where ``parser.values`` the object where option values are by default stored (an instance of optparse.OptionValues). This lets callbacks use the same mechanism as the - rest of :mod:`optparse` for storing option values; you don't need to mess + rest of :mod:`!optparse` for storing option values; you don't need to mess around with globals or closures. You can also access or modify the value(s) of any options already encountered on the command-line. @@ -1806,7 +1806,7 @@ Raising errors in a callback ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ The callback function should raise :exc:`OptionValueError` if there are any -problems with the option or its argument(s). :mod:`optparse` catches this and +problems with the option or its argument(s). :mod:`!optparse` catches this and terminates the program, printing the error message you supply to stderr. Your message should be clear, concise, accurate, and mention the option at fault. Otherwise, the user will have a hard time figuring out what they did wrong. @@ -1906,7 +1906,7 @@ Here's an example that just emulates the standard ``"store"`` action:: action="callback", callback=store_value, type="int", nargs=3, dest="foo") -Note that :mod:`optparse` takes care of consuming 3 arguments and converting +Note that :mod:`!optparse` takes care of consuming 3 arguments and converting them to integers for you; all you have to do is store them. (Or whatever; obviously you don't need a callback for this example.) @@ -1917,9 +1917,9 @@ Callback example 6: variable arguments ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Things get hairy when you want an option to take a variable number of arguments. -For this case, you must write a callback, as :mod:`optparse` doesn't provide any +For this case, you must write a callback, as :mod:`!optparse` doesn't provide any built-in capabilities for it. And you have to deal with certain intricacies of -conventional Unix command-line parsing that :mod:`optparse` normally handles for +conventional Unix command-line parsing that :mod:`!optparse` normally handles for you. In particular, callbacks should implement the conventional rules for bare ``--`` and ``-`` arguments: @@ -1934,7 +1934,7 @@ you. In particular, callbacks should implement the conventional rules for bare If you want an option that takes a variable number of arguments, there are several subtle, tricky issues to worry about. The exact implementation you choose will be based on which trade-offs you're willing to make for your -application (which is why :mod:`optparse` doesn't support this sort of thing +application (which is why :mod:`!optparse` doesn't support this sort of thing directly). Nevertheless, here's a stab at a callback for an option with variable @@ -1970,10 +1970,10 @@ arguments:: .. _optparse-extending-optparse: -Extending :mod:`optparse` -------------------------- +Extending :mod:`!optparse` +-------------------------- -Since the two major controlling factors in how :mod:`optparse` interprets +Since the two major controlling factors in how :mod:`!optparse` interprets command-line options are the action and type of each option, the most likely direction of extension is to add new actions and new types. @@ -1983,9 +1983,9 @@ direction of extension is to add new actions and new types. Adding new types ^^^^^^^^^^^^^^^^ -To add new types, you need to define your own subclass of :mod:`optparse`'s +To add new types, you need to define your own subclass of :mod:`!optparse`'s :class:`Option` class. This class has a couple of attributes that define -:mod:`optparse`'s types: :attr:`~Option.TYPES` and :attr:`~Option.TYPE_CHECKER`. +:mod:`!optparse`'s types: :attr:`~Option.TYPES` and :attr:`~Option.TYPE_CHECKER`. .. attribute:: Option.TYPES @@ -2015,7 +2015,7 @@ To add new types, you need to define your own subclass of :mod:`optparse`'s Here's a silly example that demonstrates adding a ``"complex"`` option type to parse Python-style complex numbers on the command line. (This is even sillier -than it used to be, because :mod:`optparse` 1.3 added built-in support for +than it used to be, because :mod:`!optparse` 1.3 added built-in support for complex numbers, but never mind.) First, the necessary imports:: @@ -2041,12 +2041,12 @@ Finally, the Option subclass:: TYPE_CHECKER["complex"] = check_complex (If we didn't make a :func:`copy` of :attr:`Option.TYPE_CHECKER`, we would end -up modifying the :attr:`~Option.TYPE_CHECKER` attribute of :mod:`optparse`'s +up modifying the :attr:`~Option.TYPE_CHECKER` attribute of :mod:`!optparse`'s Option class. This being Python, nothing stops you from doing that except good manners and common sense.) That's it! Now you can write a script that uses the new option type just like -any other :mod:`optparse`\ -based script, except you have to instruct your +any other :mod:`!optparse`\ -based script, except you have to instruct your OptionParser to use MyOption instead of Option:: parser = OptionParser(option_class=MyOption) @@ -2066,10 +2066,10 @@ Adding new actions ^^^^^^^^^^^^^^^^^^ Adding new actions is a bit trickier, because you have to understand that -:mod:`optparse` has a couple of classifications for actions: +:mod:`!optparse` has a couple of classifications for actions: "store" actions - actions that result in :mod:`optparse` storing a value to an attribute of the + actions that result in :mod:`!optparse` storing a value to an attribute of the current OptionValues instance; these options require a :attr:`~Option.dest` attribute to be supplied to the Option constructor. @@ -2101,7 +2101,7 @@ of the following class attributes of Option (all are lists of strings): .. attribute:: Option.ALWAYS_TYPED_ACTIONS Actions that always take a type (i.e. whose options always take a value) are - additionally listed here. The only effect of this is that :mod:`optparse` + additionally listed here. The only effect of this is that :mod:`!optparse` assigns the default type, ``"string"``, to options with no explicit type whose action is listed in :attr:`ALWAYS_TYPED_ACTIONS`. @@ -2144,12 +2144,12 @@ Features of note: somewhere, so it goes in both :attr:`~Option.STORE_ACTIONS` and :attr:`~Option.TYPED_ACTIONS`. -* to ensure that :mod:`optparse` assigns the default type of ``"string"`` to +* to ensure that :mod:`!optparse` assigns the default type of ``"string"`` to ``"extend"`` actions, we put the ``"extend"`` action in :attr:`~Option.ALWAYS_TYPED_ACTIONS` as well. * :meth:`MyOption.take_action` implements just this one new action, and passes - control back to :meth:`Option.take_action` for the standard :mod:`optparse` + control back to :meth:`Option.take_action` for the standard :mod:`!optparse` actions. * ``values`` is an instance of the optparse_parser.Values class, which provides diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index ecbbc1d7605f9f8..48e4734cad9579f 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -36,14 +36,14 @@ the :mod:`glob` module.) Since different operating systems have different path name conventions, there are several versions of this module in the standard library. The - :mod:`os.path` module is always the path module suitable for the operating + :mod:`!os.path` module is always the path module suitable for the operating system Python is running on, and therefore usable for local paths. However, you can also import and use the individual modules if you want to manipulate a path that is *always* in one of the different formats. They all have the same interface: - * :mod:`posixpath` for UNIX-style paths - * :mod:`ntpath` for Windows paths + * :mod:`!posixpath` for UNIX-style paths + * :mod:`!ntpath` for Windows paths .. versionchanged:: 3.8 @@ -57,14 +57,15 @@ the :mod:`glob` module.) .. function:: abspath(path) Return a normalized absolutized version of the pathname *path*. On most - platforms, this is equivalent to calling the function :func:`normpath` as - follows: ``normpath(join(os.getcwd(), path))``. + platforms, this is equivalent to calling ``normpath(join(os.getcwd(), path))``. + + .. seealso:: :func:`os.path.join` and :func:`os.path.normpath`. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. -.. function:: basename(path) +.. function:: basename(path, /) Return the base name of pathname *path*. This is the second element of the pair returned by passing *path* to the function :func:`split`. Note that @@ -94,17 +95,19 @@ the :mod:`glob` module.) Any iterable can now be passed, rather than just sequences. -.. function:: commonprefix(list) +.. function:: commonprefix(list, /) - Return the longest path prefix (taken character-by-character) that is a - prefix of all paths in *list*. If *list* is empty, return the empty string + Return the longest string prefix (taken character-by-character) that is a + prefix of all strings in *list*. If *list* is empty, return the empty string (``''``). - .. note:: + .. warning:: This function may return invalid paths because it works a - character at a time. To obtain a valid path, see - :func:`commonpath`. + character at a time. + If you need a **common path prefix**, then the algorithm + implemented in this function is not secure. Use + :func:`commonpath` for finding a common path prefix. :: @@ -118,7 +121,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: dirname(path) +.. function:: dirname(path, /) Return the directory name of pathname *path*. This is the first element of the pair returned by passing *path* to the function :func:`split`. @@ -199,14 +202,14 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: getatime(path) +.. function:: getatime(path, /) Return the time of last access of *path*. The return value is a floating-point number giving the number of seconds since the epoch (see the :mod:`time` module). Raise :exc:`OSError` if the file does not exist or is inaccessible. -.. function:: getmtime(path) +.. function:: getmtime(path, /) Return the time of last modification of *path*. The return value is a floating-point number giving the number of seconds since the epoch (see the :mod:`time` module). @@ -216,7 +219,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: getctime(path) +.. function:: getctime(path, /) Return the system's ctime which, on some systems (like Unix) is the time of the last metadata change, and, on others (like Windows), is the creation time for *path*. @@ -228,7 +231,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: getsize(path) +.. function:: getsize(path, /) Return the size, in bytes, of *path*. Raise :exc:`OSError` if the file does not exist or is inaccessible. @@ -237,12 +240,14 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: isabs(path) +.. function:: isabs(path, /) Return ``True`` if *path* is an absolute pathname. On Unix, that means it begins with a slash, on Windows that it begins with two (back)slashes, or a drive letter, colon, and (back)slash together. + .. seealso:: :func:`abspath` + .. versionchanged:: 3.6 Accepts a :term:`path-like object`. @@ -261,7 +266,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: isdir(path) +.. function:: isdir(path, /) Return ``True`` if *path* is an :func:`existing ` directory. This follows symbolic links, so both :func:`islink` and :func:`isdir` can be true @@ -298,9 +303,10 @@ the :mod:`glob` module.) device than *path*, or whether :file:`{path}/..` and *path* point to the same i-node on the same device --- this should detect mount points for all Unix and POSIX variants. It is not able to reliably detect bind mounts on the - same filesystem. On Windows, a drive letter root and a share UNC are - always mount points, and for any other path ``GetVolumePathName`` is called - to see if it is different from the input path. + same filesystem. On Linux systems, it will always return ``True`` for btrfs + subvolumes, even if they aren't mount points. On Windows, a drive letter root + and a share UNC are always mount points, and for any other path + ``GetVolumePathName`` is called to see if it is different from the input path. .. versionchanged:: 3.4 Added support for detecting non-root mount points on Windows. @@ -350,20 +356,34 @@ the :mod:`glob` module.) .. versionadded:: 3.13 -.. function:: join(path, *paths) +.. function:: join(path, /, *paths) Join one or more path segments intelligently. The return value is the concatenation of *path* and all members of *\*paths*, with exactly one directory separator following each non-empty part, except the last. That is, the result will only end in a separator if the last part is either empty or - ends in a separator. If a segment is an absolute path (which on Windows - requires both a drive and a root), then all previous segments are ignored and - joining continues from the absolute path segment. + ends in a separator. + + If a segment is an absolute path (which on Windows requires both a drive and + a root), then all previous segments are ignored and joining continues from the + absolute path segment. On Linux, for example:: + + >>> os.path.join('/home/foo', 'bar') + '/home/foo/bar' + >>> os.path.join('/home/foo', '/home/bar') + '/home/bar' On Windows, the drive is not reset when a rooted path segment (e.g., ``r'\foo'``) is encountered. If a segment is on a different drive or is an - absolute path, all previous segments are ignored and the drive is reset. Note - that since there is a current directory for each drive, + absolute path, all previous segments are ignored and the drive is reset. For + example:: + + >>> os.path.join('c:\\', 'foo') + 'c:\\foo' + >>> os.path.join('c:\\foo', 'd:\\bar') + 'd:\\bar' + + Note that since there is a current directory for each drive, ``os.path.join("c:", "foo")`` represents a path relative to the current directory on drive :file:`C:` (:file:`c:foo`), not :file:`c:\\foo`. @@ -371,7 +391,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object` for *path* and *paths*. -.. function:: normcase(path) +.. function:: normcase(path, /) Normalize the case of a pathname. On Windows, convert all characters in the pathname to lowercase, and also convert forward slashes to backward slashes. @@ -401,16 +421,33 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: realpath(path, *, strict=False) +.. function:: realpath(path, /, *, strict=False) Return the canonical path of the specified filename, eliminating any symbolic links encountered in the path (if they are supported by the operating system). On Windows, this function will also resolve MS-DOS (also called 8.3) style names such as ``C:\\PROGRA~1`` to ``C:\\Program Files``. - If a path doesn't exist or a symlink loop is encountered, and *strict* is - ``True``, :exc:`OSError` is raised. If *strict* is ``False`` these errors - are ignored, and so the result might be missing or otherwise inaccessible. + By default, the path is evaluated up to the first component that does not + exist, is a symlink loop, or whose evaluation raises :exc:`OSError`. + All such components are appended unchanged to the existing part of the path. + + Some errors that are handled this way include "access denied", "not a + directory", or "bad argument to internal function". Thus, the + resulting path may be missing or inaccessible, may still contain + links or loops, and may traverse non-directories. + + This behavior can be modified by keyword arguments: + + If *strict* is ``True``, the first error encountered when evaluating the path is + re-raised. + In particular, :exc:`FileNotFoundError` is raised if *path* does not exist, + or another :exc:`OSError` if it is otherwise inaccessible. + + If *strict* is :py:data:`os.path.ALLOW_MISSING`, errors other than + :exc:`FileNotFoundError` are re-raised (as with ``strict=True``). + Thus, the returned path will not contain any symbolic links, but the named + file and some of its parent directories may be missing. .. note:: This function emulates the operating system's procedure for making a path @@ -429,6 +466,15 @@ the :mod:`glob` module.) .. versionchanged:: 3.10 The *strict* parameter was added. + .. versionchanged:: 3.14 + The :py:data:`~os.path.ALLOW_MISSING` value for the *strict* parameter + was added. + +.. data:: ALLOW_MISSING + + Special value used for the *strict* argument in :func:`realpath`. + + .. versionadded:: 3.14 .. function:: relpath(path, start=os.curdir) @@ -444,7 +490,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: samefile(path1, path2) +.. function:: samefile(path1, path2, /) Return ``True`` if both pathname arguments refer to the same file or directory. This is determined by the device number and i-node number and raises an @@ -471,7 +517,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: samestat(stat1, stat2) +.. function:: samestat(stat1, stat2, /) Return ``True`` if the stat tuples *stat1* and *stat2* refer to the same file. These structures may have been returned by :func:`os.fstat`, @@ -481,11 +527,8 @@ the :mod:`glob` module.) .. versionchanged:: 3.4 Added Windows support. - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - -.. function:: split(path) +.. function:: split(path, /) Split the pathname *path* into a pair, ``(head, tail)`` where *tail* is the last pathname component and *head* is everything leading up to that. The @@ -494,14 +537,14 @@ the :mod:`glob` module.) *path* is empty, both *head* and *tail* are empty. Trailing slashes are stripped from *head* unless it is the root (one or more slashes only). In all cases, ``join(head, tail)`` returns a path to the same location as *path* - (but the strings may differ). Also see the functions :func:`dirname` and - :func:`basename`. + (but the strings may differ). Also see the functions :func:`join`, + :func:`dirname` and :func:`basename`. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. -.. function:: splitdrive(path) +.. function:: splitdrive(path, /) Split the pathname *path* into a pair ``(drive, tail)`` where *drive* is either a mount point or the empty string. On systems which do not use drive @@ -526,7 +569,7 @@ the :mod:`glob` module.) Accepts a :term:`path-like object`. -.. function:: splitroot(path) +.. function:: splitroot(path, /) Split the pathname *path* into a 3-item tuple ``(drive, root, tail)`` where *drive* is a device name or mount point, *root* is a string of separators @@ -559,7 +602,7 @@ the :mod:`glob` module.) .. versionadded:: 3.12 -.. function:: splitext(path) +.. function:: splitext(path, /) Split the pathname *path* into a pair ``(root, ext)`` such that ``root + ext == path``, and the extension, *ext*, is empty or begins with a period and contains at diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 1e54cfec609bd22..fe5cba128576ff0 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -25,7 +25,7 @@ Notes on the availability of these functions: with the POSIX interface). * Extensions peculiar to a particular operating system are also available - through the :mod:`os` module, but using them is of course a threat to + through the :mod:`!os` module, but using them is of course a threat to portability. * All functions accepting path or file names accept both bytes and string @@ -34,7 +34,7 @@ Notes on the availability of these functions: * On VxWorks, os.popen, os.fork, os.execv and os.spawn*p* are not supported. -* On WebAssembly platforms, Android and iOS, large parts of the :mod:`os` module are +* On WebAssembly platforms, Android and iOS, large parts of the :mod:`!os` module are not available or behave differently. APIs related to processes (e.g. :func:`~os.fork`, :func:`~os.execve`) and resources (e.g. :func:`~os.nice`) are not available. Others like :func:`~os.getuid` and :func:`~os.getpid` are @@ -188,7 +188,7 @@ process and user. of your home directory (on some platforms), and is equivalent to ``getenv("HOME")`` in C. - This mapping is captured the first time the :mod:`os` module is imported, + This mapping is captured the first time the :mod:`!os` module is imported, typically during Python startup as part of processing :file:`site.py`. Changes to the environment made after this time are not reflected in :data:`os.environ`, except for changes made by modifying :data:`os.environ` directly. @@ -257,7 +257,7 @@ process and user. .. warning:: This function is not thread-safe. Calling it while the environment is - being modified in an other thread is an undefined behavior. Reading from + being modified in another thread is an undefined behavior. Reading from :data:`os.environ` or :data:`os.environb`, or calling :func:`os.getenv` while reloading, may return an empty result. @@ -532,7 +532,7 @@ process and user. Return a tuple (ruid, euid, suid) denoting the current process's real, effective, and saved user ids. - .. availability:: Unix, not WASI. + .. availability:: Unix, not WASI, not macOS, not iOS. .. versionadded:: 3.2 @@ -542,7 +542,7 @@ process and user. Return a tuple (rgid, egid, sgid) denoting the current process's real, effective, and saved group ids. - .. availability:: Unix, not WASI. + .. availability:: Unix, not WASI, not macOS, not iOS. .. versionadded:: 3.2 @@ -561,7 +561,7 @@ process and user. .. function:: initgroups(username, gid, /) - Call the system initgroups() to initialize the group access list with all of + Call the system ``initgroups()`` to initialize the group access list with all of the groups of which the specified username is a member, plus the specified group id. @@ -716,7 +716,7 @@ process and user. Set the current process's real, effective, and saved group ids. - .. availability:: Unix, not WASI, not Android. + .. availability:: Unix, not WASI, not Android, not macOS, not iOS. .. versionadded:: 3.2 @@ -725,7 +725,7 @@ process and user. Set the current process's real, effective, and saved user ids. - .. availability:: Unix, not WASI, not Android. + .. availability:: Unix, not WASI, not Android, not macOS, not iOS. .. versionadded:: 3.2 @@ -791,29 +791,19 @@ process and user. single: gethostbyaddr() (in module socket) Returns information identifying the current operating system. - The return value is an object with five attributes: - - * :attr:`sysname` - operating system name - * :attr:`nodename` - name of machine on network (implementation-defined) - * :attr:`release` - operating system release - * :attr:`version` - operating system version - * :attr:`machine` - hardware identifier - - For backwards compatibility, this object is also iterable, behaving - like a five-tuple containing :attr:`sysname`, :attr:`nodename`, - :attr:`release`, :attr:`version`, and :attr:`machine` - in that order. - - Some systems truncate :attr:`nodename` to 8 characters or to the - leading component; a better way to get the hostname is - :func:`socket.gethostname` or even - ``socket.gethostbyaddr(socket.gethostname())``. + The return value is a :class:`uname_result`. On macOS, iOS and Android, this returns the *kernel* name and version (i.e., ``'Darwin'`` on macOS and iOS; ``'Linux'`` on Android). :func:`platform.uname` can be used to get the user-facing operating system name and version on iOS and Android. + .. seealso:: + :data:`sys.platform` which has finer granularity. + + The :mod:`platform` module provides detailed checks for the + system's identity. + .. availability:: Unix. .. versionchanged:: 3.3 @@ -821,6 +811,41 @@ process and user. with named attributes. +.. class:: uname_result + + Name and information about the system returned by :func:`os.uname`. + These attributes correspond to the members described in :manpage:`uname(2)`. + + For backwards compatibility, this object is also iterable, behaving + like a five-tuple containing :attr:`~uname_result.sysname`, + :attr:`~uname_result.nodename`, :attr:`~uname_result.release`, + :attr:`~uname_result.version`, and :attr:`~uname_result.machine` + in that order. + + .. attribute:: sysname + + Operating system name. + + .. attribute:: nodename + + Name of machine on network. Some systems truncate + :attr:`~uname_result.nodename` to 8 characters or to the leading + component; a better way to get the hostname is :func:`socket.gethostname` + or even ``socket.gethostbyaddr(socket.gethostname())``. + + .. attribute:: release + + Operating system release. + + .. attribute:: version + + Operating system version. + + .. attribute:: machine + + Hardware identifier. + + .. function:: unsetenv(key, /) .. index:: single: environment variables; deleting @@ -1062,10 +1087,7 @@ as internal buffering of data. Force write of file with filedescriptor *fd* to disk. Does not force update of metadata. - .. availability:: Unix. - - .. note:: - This function is not available on MacOS. + .. availability:: Unix, not macOS, not iOS. .. function:: fpathconf(fd, name, /) @@ -1103,8 +1125,8 @@ as internal buffering of data. .. function:: fstatvfs(fd, /) Return information about the filesystem containing the file associated with - file descriptor *fd*, like :func:`statvfs`. As of Python 3.3, this is - equivalent to ``os.statvfs(fd)``. + file descriptor *fd* in a :class:`statvfs_result`, like :func:`statvfs`. + As of Python 3.3, this is equivalent to ``os.statvfs(fd)``. .. availability:: Unix. @@ -1282,7 +1304,7 @@ as internal buffering of data. For a description of the flag and mode values, see the C run-time documentation; flag constants (like :const:`O_RDONLY` and :const:`O_WRONLY`) are defined in - the :mod:`os` module. In particular, on Windows adding + the :mod:`!os` module. In particular, on Windows adding :const:`O_BINARY` is needed to open files in binary mode. This function can support :ref:`paths relative to directory descriptors @@ -1297,8 +1319,8 @@ as internal buffering of data. This function is intended for low-level I/O. For normal usage, use the built-in function :func:`open`, which returns a :term:`file object` with - :meth:`~file.read` and :meth:`~file.write` methods (and many more). To - wrap a file descriptor in a file object, use :func:`fdopen`. + :meth:`~io.BufferedIOBase.read` and :meth:`~io.BufferedIOBase.write` methods. + To wrap a file descriptor in a file object, use :func:`fdopen`. .. versionchanged:: 3.3 Added the *dir_fd* parameter. @@ -1417,7 +1439,7 @@ or `the MSDN `_ on Windo Return a pair of file descriptors ``(r, w)`` usable for reading and writing, respectively. - .. availability:: Unix, not WASI. + .. availability:: Unix, not WASI, not macOS, not iOS. .. versionadded:: 3.3 @@ -1427,7 +1449,7 @@ or `the MSDN `_ on Windo Ensures that enough disk space is allocated for the file specified by *fd* starting from *offset* and continuing for *len* bytes. - .. availability:: Unix. + .. availability:: Unix, not macOS, not iOS. .. versionadded:: 3.3 @@ -1442,7 +1464,7 @@ or `the MSDN `_ on Windo :data:`POSIX_FADV_RANDOM`, :data:`POSIX_FADV_NOREUSE`, :data:`POSIX_FADV_WILLNEED` or :data:`POSIX_FADV_DONTNEED`. - .. availability:: Unix. + .. availability:: Unix, not macOS, not iOS. .. versionadded:: 3.3 @@ -1652,7 +1674,7 @@ or `the MSDN `_ on Windo descriptor as returned by :func:`os.open` or :func:`pipe`. To read a "file object" returned by the built-in function :func:`open` or by :func:`popen` or :func:`fdopen`, or :data:`sys.stdin`, use its - :meth:`~file.read` or :meth:`~file.readline` methods. + :meth:`~io.TextIOBase.read` or :meth:`~io.IOBase.readline` methods. .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise an @@ -1887,7 +1909,7 @@ or `the MSDN `_ on Windo descriptor as returned by :func:`os.open` or :func:`pipe`. To write a "file object" returned by the built-in function :func:`open` or by :func:`popen` or :func:`fdopen`, or :data:`sys.stdout` or :data:`sys.stderr`, use its - :meth:`~file.write` method. + :meth:`~io.TextIOBase.write` method. .. versionchanged:: 3.5 If the system call is interrupted and the signal handler does not raise an @@ -1961,7 +1983,8 @@ can be inherited by child processes. Since Python 3.4, file descriptors created by Python are non-inheritable by default. On UNIX, non-inheritable file descriptors are closed in child processes at the -execution of a new program, other file descriptors are inherited. +execution of a new program, other file descriptors are inherited. Note that +non-inheritable file descriptors are still *inherited* by child processes on :func:`os.fork`. On Windows, non-inheritable handles and file descriptors are closed in child processes, except for standard streams (file descriptors 0, 1 and 2: stdin, stdout @@ -2005,12 +2028,12 @@ features: .. _path_fd: * **specifying a file descriptor:** - Normally the *path* argument provided to functions in the :mod:`os` module + Normally the *path* argument provided to functions in the :mod:`!os` module must be a string specifying a file path. However, some functions now alternatively accept an open file descriptor for their *path* argument. The function will then operate on the file referred to by the descriptor. - (For POSIX systems, Python will call the variant of the function prefixed - with ``f`` (e.g. call ``fchdir`` instead of ``chdir``).) + For POSIX systems, Python will call the variant of the function prefixed + with ``f`` (e.g. call ``fchdir`` instead of ``chdir``). You can check whether or not *path* can be specified as a file descriptor for a particular function on your platform using :data:`os.supports_fd`. @@ -2025,7 +2048,7 @@ features: * **paths relative to directory descriptors:** If *dir_fd* is not ``None``, it should be a file descriptor referring to a directory, and the path to operate on should be relative; path will then be relative to that directory. If the - path is absolute, *dir_fd* is ignored. (For POSIX systems, Python will call + path is absolute, *dir_fd* is ignored. For POSIX systems, Python will call the variant of the function with an ``at`` suffix and possibly prefixed with ``f`` (e.g. call ``faccessat`` instead of ``access``). @@ -2038,8 +2061,8 @@ features: * **not following symlinks:** If *follow_symlinks* is ``False``, and the last element of the path to operate on is a symbolic link, the function will operate on the symbolic link itself rather than the file - pointed to by the link. (For POSIX systems, Python will call the ``l...`` - variant of the function.) + pointed to by the link. For POSIX systems, Python will call the ``l...`` + variant of the function. You can check whether or not *follow_symlinks* is supported for a particular function on your platform using :data:`os.supports_follow_symlinks`. @@ -2127,6 +2150,11 @@ features: .. audit-event:: os.chdir path os.chdir + .. seealso:: + + The :func:`contextlib.chdir` context manager, which changes the current + working directory on entering and restores the previous one on exit. + .. versionchanged:: 3.3 Added support for specifying *path* as a file descriptor on some platforms. @@ -2936,6 +2964,9 @@ features: To be directly usable as a :term:`path-like object`, ``os.DirEntry`` implements the :class:`PathLike` interface. + :class:`!DirEntry` objects are :ref:`generic ` over the type of the + path (:class:`str` or :class:`bytes`). + Attributes and methods on a ``os.DirEntry`` instance are as follows: .. attribute:: name @@ -2949,10 +2980,16 @@ features: .. attribute:: path - The entry's full path name: equivalent to ``os.path.join(scandir_path, - entry.name)`` where *scandir_path* is the :func:`scandir` *path* - argument. The path is only absolute if the :func:`scandir` *path* - argument was absolute. If the :func:`scandir` *path* + The entry's path name: equivalent to ``os.path.join(scandir_path, + entry.name)`` where *scandir_path* is the original :func:`scandir` + *path* argument. Apart from the filename, the path preserves the + original :func:`scandir` argument. If the :func:`scandir` *path* + argument was relative, the :attr:`path` attribute is also relative. + Changing the current working directory after creating the + :func:`scandir` iterator may cause later uses of :attr:`path` to resolve + differently. On some platforms, the constructed path may not be valid + if the original :func:`scandir` argument was usable for enumeration but + not for joining with the entry name. If the :func:`scandir` *path* argument was a :ref:`file descriptor `, the :attr:`path` attribute is the same as the :attr:`name` attribute. @@ -3370,53 +3407,179 @@ features: .. function:: statvfs(path) - Perform a :c:func:`!statvfs` system call on the given path. The return value is - an object whose attributes describe the filesystem on the given path, and - correspond to the members of the :c:struct:`statvfs` structure, namely: - :attr:`f_bsize`, :attr:`f_frsize`, :attr:`f_blocks`, :attr:`f_bfree`, - :attr:`f_bavail`, :attr:`f_files`, :attr:`f_ffree`, :attr:`f_favail`, - :attr:`f_flag`, :attr:`f_namemax`, :attr:`f_fsid`. - - Two module-level constants are defined for the :attr:`f_flag` attribute's - bit-flags: if :const:`ST_RDONLY` is set, the filesystem is mounted - read-only, and if :const:`ST_NOSUID` is set, the semantics of - setuid/setgid bits are disabled or not supported. - - Additional module-level constants are defined for GNU/glibc based systems. - These are :const:`ST_NODEV` (disallow access to device special files), - :const:`ST_NOEXEC` (disallow program execution), :const:`ST_SYNCHRONOUS` - (writes are synced at once), :const:`ST_MANDLOCK` (allow mandatory locks on an FS), - :const:`ST_WRITE` (write on file/directory/symlink), :const:`ST_APPEND` - (append-only file), :const:`ST_IMMUTABLE` (immutable file), :const:`ST_NOATIME` - (do not update access times), :const:`ST_NODIRATIME` (do not update directory access - times), :const:`ST_RELATIME` (update atime relative to mtime/ctime). + Perform a :manpage:`statvfs(3)` system call on the given path. The return value + is a :class:`statvfs_result` whose attributes describe the filesystem + on the given path and correspond to the members of the :c:struct:`statvfs` + structure. This function can support :ref:`specifying a file descriptor `. .. availability:: Unix. - .. versionchanged:: 3.2 - The :const:`ST_RDONLY` and :const:`ST_NOSUID` constants were added. - .. versionchanged:: 3.3 Added support for specifying *path* as an open file descriptor. - .. versionchanged:: 3.4 - The :const:`ST_NODEV`, :const:`ST_NOEXEC`, :const:`ST_SYNCHRONOUS`, - :const:`ST_MANDLOCK`, :const:`ST_WRITE`, :const:`ST_APPEND`, - :const:`ST_IMMUTABLE`, :const:`ST_NOATIME`, :const:`ST_NODIRATIME`, - and :const:`ST_RELATIME` constants were added. - .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. versionchanged:: 3.7 - Added the :attr:`f_fsid` attribute. + +.. class:: statvfs_result + + Filesystem statistics returned by :func:`os.statvfs` and :func:`os.fstatvfs`. + See :manpage:`statvfs(3)` for more details. + + .. attribute:: f_bsize + + Block size. + + .. attribute:: f_frsize + + Fragment size. + + .. attribute:: f_blocks + + Number of :attr:`~statvfs_result.f_frsize` sized blocks the filesystem + can contain. + + .. attribute:: f_bfree + + Number of free blocks. + + .. attribute:: f_bavail + + Number of free blocks for unprivileged users. + + .. attribute:: f_files + + Number of file entries, inodes, the filesystem can contain. + + .. attribute:: f_ffree + + Number of free files entries. + + .. attribute:: f_favail + + Number of free file entries for unprivileged users. + + .. attribute:: f_flag + + Bit-mask of mount flags. The following flags are defined: + :data:`ST_RDONLY`, :data:`ST_NOSUID`, :data:`ST_NODEV`, + :data:`ST_NOEXEC`, :data:`ST_SYNCHRONOUS`, :data:`ST_MANDLOCK`, + :data:`ST_WRITE`, :data:`ST_APPEND`, :data:`ST_IMMUTABLE`, + :data:`ST_NOATIME`, :data:`ST_NODIRATIME`, and :data:`ST_RELATIME`. + + .. attribute:: f_namemax + + Filesystem max filename length. OS specific limitations such as + :ref:`Windows MAX_PATH ` and those described in Linux + :manpage:`pathname(7)` may exist. + + .. attribute:: f_fsid + + Filesystem ID. + + .. versionadded:: 3.7 + + +The following flags are used in :attr:`statvfs_result.f_flag`. + +.. data:: ST_RDONLY + + Read-only filesystem. + + .. versionadded:: 3.2 + +.. data:: ST_NOSUID + + Setuid/setgid bits are disabled or not supported. + + .. versionadded:: 3.2 + +.. data:: ST_NODEV + + Disallow access to device special files. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_NOEXEC + + Disallow program execution. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_SYNCHRONOUS + + Writes are synced at once. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_MANDLOCK + + Allow mandatory locks on an FS. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_WRITE + + Write on file/directory/symlink. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_APPEND + + Append-only file. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_IMMUTABLE + + Immutable file. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_NOATIME + + Do not update access times. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_NODIRATIME + + Do not update directory access times. + + .. availability:: Linux. + + .. versionadded:: 3.4 + +.. data:: ST_RELATIME + + Update atime relative to mtime/ctime. + + .. availability:: Linux. + + .. versionadded:: 3.4 .. data:: supports_dir_fd - A :class:`set` object indicating which functions in the :mod:`os` + A :class:`set` object indicating which functions in the :mod:`!os` module accept an open file descriptor for their *dir_fd* parameter. Different platforms provide different features, and the underlying functionality Python uses to implement the *dir_fd* parameter is not @@ -3461,7 +3624,7 @@ features: .. data:: supports_fd A :class:`set` object indicating which functions in the - :mod:`os` module permit specifying their *path* parameter as an open file + :mod:`!os` module permit specifying their *path* parameter as an open file descriptor on the local platform. Different platforms provide different features, and the underlying functionality Python uses to accept open file descriptors as *path* arguments is not available on all platforms Python @@ -3480,7 +3643,7 @@ features: .. data:: supports_follow_symlinks - A :class:`set` object indicating which functions in the :mod:`os` module + A :class:`set` object indicating which functions in the :mod:`!os` module accept ``False`` for their *follow_symlinks* parameter on the local platform. Different platforms provide different features, and the underlying functionality Python uses to implement *follow_symlinks* is not available @@ -3505,6 +3668,9 @@ features: Create a symbolic link pointing to *src* named *dst*. + The *src* parameter refers to the target of the link (the file or directory being linked to), + and *dst* is the name of the link being created. + On Windows, a symlink represents either a file or a directory, and does not morph to the target dynamically. If the target is present, the type of the symlink will be created to match. Otherwise, the symlink will be created @@ -3760,9 +3926,9 @@ features: import os for root, dirs, files, rootfd in os.fwalk('python/Lib/xml'): - print(root, "consumes", end="") + print(root, "consumes", end=" ") print(sum([os.stat(name, dir_fd=rootfd).st_size for name in files]), - end="") + end=" ") print("bytes in", len(files), "non-directory files") if '__pycache__' in dirs: dirs.remove('__pycache__') # don't visit __pycache__ directories @@ -3876,7 +4042,7 @@ features: import os # semaphore with start value '1' - fd = os.eventfd(1, os.EFD_SEMAPHORE | os.EFC_CLOEXEC) + fd = os.eventfd(1, os.EFD_SEMAPHORE | os.EFD_CLOEXEC) try: # acquire semaphore v = os.eventfd_read(fd) @@ -3985,7 +4151,7 @@ Naturally, they are all only available on Linux. except it includes any time that the system is suspended. The file descriptor's behaviour can be modified by specifying a *flags* value. - Any of the following variables may used, combined using bitwise OR + Any of the following variables may be used, combined using bitwise OR (the ``|`` operator): - :const:`TFD_NONBLOCK` @@ -4017,7 +4183,7 @@ Naturally, they are all only available on Linux. *fd* must be a valid timer file descriptor. The timer's behaviour can be modified by specifying a *flags* value. - Any of the following variables may used, combined using bitwise OR + Any of the following variables may be used, combined using bitwise OR (the ``|`` operator): - :const:`TFD_TIMER_ABSTIME` @@ -4086,7 +4252,7 @@ Naturally, they are all only available on Linux. Return a two-item tuple of floats (``next_expiration``, ``interval``). - ``next_expiration`` denotes the relative time until next the timer next fires, + ``next_expiration`` denotes the relative time until the timer next fires, regardless of if the :const:`TFD_TIMER_ABSTIME` flag is set. ``interval`` denotes the timer's interval. @@ -4322,7 +4488,7 @@ to be ignored. The current process is replaced immediately. Open file objects and descriptors are not flushed, so if there may be data buffered on these open files, you should flush them using - :func:`sys.stdout.flush` or :func:`os.fsync` before calling an + :func:`~io.IOBase.flush` or :func:`os.fsync` before calling an :func:`exec\* ` function. The "l" and "v" variants of the :func:`exec\* ` functions differ in how @@ -4665,7 +4831,7 @@ written in Python, such as a mail server's external command delivery program. Lock program segments into memory. The value of *op* (defined in ````) determines which segments are locked. - .. availability:: Unix, not WASI, not iOS. + .. availability:: Unix, not WASI, not macOS, not iOS. .. function:: popen(cmd, mode='r', buffering=-1) @@ -4707,9 +4873,8 @@ written in Python, such as a mail server's external command delivery program. Use :class:`subprocess.Popen` or :func:`subprocess.run` to control options like encodings. - .. deprecated:: 3.14 - The function is :term:`soft deprecated` and should no longer be used to - write new code. The :mod:`subprocess` module is recommended instead. + .. soft-deprecated:: 3.14 + The :mod:`subprocess` module is recommended instead. .. function:: posix_spawn(path, argv, env, *, file_actions=None, \ @@ -4937,9 +5102,8 @@ written in Python, such as a mail server's external command delivery program. .. versionchanged:: 3.6 Accepts a :term:`path-like object`. - .. deprecated:: 3.14 - These functions are :term:`soft deprecated` and should no longer be used - to write new code. The :mod:`subprocess` module is recommended instead. + .. soft-deprecated:: 3.14 + The :mod:`subprocess` module is recommended instead. .. data:: P_NOWAIT @@ -5613,7 +5777,7 @@ Miscellaneous System Information .. versionchanged:: 3.13 If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, - :func:`cpu_count` returns the overridden value *n*. + :func:`cpu_count` returns the override value *n*. .. function:: getloadavg() @@ -5635,7 +5799,7 @@ Miscellaneous System Information in the **system**. If :option:`-X cpu_count <-X>` is given or :envvar:`PYTHON_CPU_COUNT` is set, - :func:`process_cpu_count` returns the overridden value *n*. + :func:`process_cpu_count` returns the override value *n*. See also the :func:`sched_getaffinity` function. diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 7d7692dea5c38ce..792cd255753c5bf 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -311,7 +311,7 @@ Pure paths provide the following methods and properties: .. attribute:: PurePath.parser The implementation of the :mod:`os.path` module used for low-level path - parsing and joining: either :mod:`posixpath` or :mod:`ntpath`. + parsing and joining: either :mod:`!posixpath` or :mod:`!ntpath`. .. versionadded:: 3.13 @@ -486,6 +486,10 @@ Pure paths provide the following methods and properties: >>> PurePosixPath('my/library').stem 'library' + .. versionchanged:: 3.14 + + A single dot ("``.``") is considered a valid suffix. + .. method:: PurePath.as_posix() @@ -1345,6 +1349,10 @@ Reading directories PosixPath('setup.py'), PosixPath('test_pathlib.py')] + .. note:: + The paths are returned in no particular order. + If you need a specific order, sort the results. + .. seealso:: :ref:`pathlib-pattern-language` documentation. @@ -1357,6 +1365,11 @@ Reading directories ``False``, this method follows symlinks except when expanding "``**``" wildcards. Set *recurse_symlinks* to ``True`` to always follow symlinks. + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. audit-event:: pathlib.Path.glob self,pattern pathlib.Path.glob .. versionchanged:: 3.12 @@ -1379,6 +1392,15 @@ Reading directories Glob the given relative *pattern* recursively. This is like calling :func:`Path.glob` with "``**/``" added in front of the *pattern*. + .. note:: + The paths are returned in no particular order. + If you need a specific order, sort the results. + + .. note:: + Any :exc:`OSError` exceptions raised from scanning the filesystem are + suppressed. This includes :exc:`PermissionError` when accessing + directories without read permission. + .. seealso:: :ref:`pathlib-pattern-language` and :meth:`Path.glob` documentation. @@ -1781,9 +1803,12 @@ The following wildcards are supported in patterns for ``?`` Matches one non-separator character. ``[seq]`` - Matches one character in *seq*. + Matches one character in *seq*, where *seq* is a sequence of characters. + Range expressions are supported; for example, ``[a-z]`` matches any lowercase ASCII letter. + Multiple ranges can be combined: ``[a-zA-Z0-9_]`` matches any ASCII letter, digit, or underscore. + ``[!seq]`` - Matches one character not in *seq*. + Matches one character not in *seq*, where *seq* follows the same rules as above. For a literal match, wrap the meta-characters in brackets. For example, ``"[?]"`` matches the character ``"?"``. @@ -1886,7 +1911,7 @@ Below is a table mapping various :mod:`os` functions to their corresponding :class:`PurePath`/:class:`Path` equivalent. ===================================== ============================================== -:mod:`os` and :mod:`os.path` :mod:`pathlib` +:mod:`os` and :mod:`os.path` :mod:`!pathlib` ===================================== ============================================== :func:`os.path.dirname` :attr:`PurePath.parent` :func:`os.path.basename` :attr:`PurePath.name` @@ -1945,7 +1970,7 @@ Protocols :synopsis: pathlib types for static type checking -The :mod:`pathlib.types` module provides types for static type checking. +The :mod:`!pathlib.types` module provides types for static type checking. .. versionadded:: 3.14 @@ -1982,7 +2007,7 @@ The :mod:`pathlib.types` module provides types for static type checking. If *follow_symlinks* is ``False``, return ``True`` only if the path is a file (without following symlinks); return ``False`` if the path - is a directory or other other non-file, or if it doesn't exist. + is a directory or other non-file, or if it doesn't exist. .. method:: is_symlink() diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index a0304edddf64789..e7dd2e5524abca2 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -1,7 +1,7 @@ .. _debugger: -:mod:`pdb` --- The Python Debugger -================================== +:mod:`!pdb` --- The Python Debugger +=================================== .. module:: pdb :synopsis: The Python debugger for interactive interpreters. @@ -12,7 +12,7 @@ -------------- -The module :mod:`pdb` defines an interactive source code debugger for Python +The module :mod:`!pdb` defines an interactive source code debugger for Python programs. It supports setting (conditional) breakpoints and single stepping at the source line level, inspection of stack frames, source code listing, and evaluation of arbitrary Python code in the context of any stack frame. It also @@ -75,12 +75,17 @@ The debugger's prompt is ``(Pdb)``, which is the indicator that you are in debug arguments of the ``p`` command. +.. _pdb-cli: + +Command-line interface +---------------------- + .. program:: pdb -You can also invoke :mod:`pdb` from the command line to debug other scripts. For +You can also invoke :mod:`!pdb` from the command line to debug other scripts. For example:: - python -m pdb [-c command] (-m module | pyfile) [args ...] + python -m pdb [-c command] (-m module | -p pid | pyfile) [args ...] When invoked as a module, pdb will automatically enter post-mortem debugging if the program being debugged exits abnormally. After post-mortem debugging (or @@ -104,6 +109,24 @@ useful than quitting the debugger upon program's exit. .. versionchanged:: 3.7 Added the ``-m`` option. +.. option:: -p, --pid + + Attach to the process with the specified PID. + + .. versionadded:: 3.14 + + +To attach to a running Python process for remote debugging, use the ``-p`` or +``--pid`` option with the target process's PID:: + + python -m pdb -p 1234 + +.. note:: + + Attaching to a process that is blocked in a system call or waiting for I/O + will only work once the next bytecode instruction is executed or when the + process receives a signal. + Typical usage to execute a statement under control of the debugger is:: >>> import pdb @@ -315,7 +338,7 @@ access further features, you have to do this yourself: .. _debugger-commands: -Debugger Commands +Debugger commands ----------------- The commands recognized by the debugger are listed below. Most commands can be diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index 007c9fe1b950cf2..efae4910ca43585 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -19,7 +19,7 @@ -------------- -The :mod:`pickle` module implements binary protocols for serializing and +The :mod:`!pickle` module implements binary protocols for serializing and de-serializing a Python object structure. *"Pickling"* is the process whereby a Python object hierarchy is converted into a byte stream, and *"unpickling"* is the inverse operation, whereby a byte stream @@ -50,35 +50,22 @@ Comparison with ``marshal`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^ Python has a more primitive serialization module called :mod:`marshal`, but in -general :mod:`pickle` should always be the preferred way to serialize Python +general :mod:`!pickle` should always be the preferred way to serialize Python objects. :mod:`marshal` exists primarily to support Python's :file:`.pyc` files. -The :mod:`pickle` module differs from :mod:`marshal` in several significant ways: - -* The :mod:`pickle` module keeps track of the objects it has already serialized, - so that later references to the same object won't be serialized again. - :mod:`marshal` doesn't do this. - - This has implications both for recursive objects and object sharing. Recursive - objects are objects that contain references to themselves. These are not - handled by marshal, and in fact, attempting to marshal recursive objects will - crash your Python interpreter. Object sharing happens when there are multiple - references to the same object in different places in the object hierarchy being - serialized. :mod:`pickle` stores such objects only once, and ensures that all - other references point to the master copy. Shared objects remain shared, which - can be very important for mutable objects. +The :mod:`!pickle` module differs from :mod:`marshal` in several significant ways: * :mod:`marshal` cannot be used to serialize user-defined classes and their - instances. :mod:`pickle` can save and restore class instances transparently, + instances. :mod:`!pickle` can save and restore class instances transparently, however the class definition must be importable and live in the same module as - when the object was stored. + when the object was pickled. * The :mod:`marshal` serialization format is not guaranteed to be portable across Python versions. Because its primary job in life is to support :file:`.pyc` files, the Python implementers reserve the right to change the serialization format in non-backwards compatible ways should the need arise. - The :mod:`pickle` serialization format is guaranteed to be backwards compatible + The :mod:`!pickle` serialization format is guaranteed to be backwards compatible across Python releases provided a compatible pickle protocol is chosen and pickling and unpickling code deals with Python 2 to Python 3 type differences if your data is crossing that unique breaking change language boundary. @@ -123,17 +110,17 @@ Data stream format .. index:: single: External Data Representation -The data format used by :mod:`pickle` is Python-specific. This has the +The data format used by :mod:`!pickle` is Python-specific. This has the advantage that there are no restrictions imposed by external standards such as JSON (which can't represent pointer sharing); however it means that non-Python programs may not be able to reconstruct pickled Python objects. -By default, the :mod:`pickle` data format uses a relatively compact binary +By default, the :mod:`!pickle` data format uses a relatively compact binary representation. If you need optimal size characteristics, you can efficiently :doc:`compress ` pickled data. The module :mod:`pickletools` contains tools for analyzing data streams -generated by :mod:`pickle`. :mod:`pickletools` source code has extensive +generated by :mod:`!pickle`. :mod:`pickletools` source code has extensive comments about opcodes used by pickle protocols. There are currently 6 different protocols which can be used for pickling. @@ -167,9 +154,9 @@ to read the pickle produced. .. note:: Serialization is a more primitive notion than persistence; although - :mod:`pickle` reads and writes file objects, it does not handle the issue of + :mod:`!pickle` reads and writes file objects, it does not handle the issue of naming persistent objects, nor the (even more complicated) issue of concurrent - access to persistent objects. The :mod:`pickle` module can transform a complex + access to persistent objects. The :mod:`!pickle` module can transform a complex object into a byte stream and it can transform the byte stream into an object with the same internal structure. Perhaps the most obvious thing to do with these byte streams is to write them onto a file, but it is also conceivable to @@ -186,7 +173,7 @@ Similarly, to de-serialize a data stream, you call the :func:`loads` function. However, if you want more control over serialization and de-serialization, you can create a :class:`Pickler` or an :class:`Unpickler` object, respectively. -The :mod:`pickle` module provides the following constants: +The :mod:`!pickle` module provides the following constants: .. data:: HIGHEST_PROTOCOL @@ -217,7 +204,7 @@ The :mod:`pickle` module provides the following constants: The default protocol is 5. -The :mod:`pickle` module provides the following functions to make the pickling +The :mod:`!pickle` module provides the following functions to make the pickling process more convenient: .. function:: dump(obj, file, protocol=None, *, fix_imports=True, buffer_callback=None) @@ -275,7 +262,7 @@ process more convenient: The *buffers* argument was added. -The :mod:`pickle` module defines three exceptions: +The :mod:`!pickle` module defines three exceptions: .. exception:: PickleError @@ -300,7 +287,7 @@ The :mod:`pickle` module defines three exceptions: IndexError. -The :mod:`pickle` module exports three classes, :class:`Pickler`, +The :mod:`!pickle` module exports three classes, :class:`Pickler`, :class:`Unpickler` and :class:`PickleBuffer`: .. class:: Pickler(file, protocol=None, *, fix_imports=True, buffer_callback=None) @@ -532,7 +519,7 @@ The following types can be pickled: * classes accessible from the top level of a module; -* instances of such classes whose the result of calling :meth:`~object.__getstate__` +* instances of such classes for which the result of calling :meth:`~object.__getstate__` is picklable (see section :ref:`pickle-inst` for details). Attempts to pickle unpicklable objects will raise the :exc:`PicklingError` @@ -601,7 +588,7 @@ methods: .. method:: object.__getnewargs_ex__() - In protocols 2 and newer, classes that implements the + In protocols 2 and newer, classes that implement the :meth:`__getnewargs_ex__` method can dictate the values passed to the :meth:`__new__` method upon unpickling. The method must return a pair ``(args, kwargs)`` where *args* is a tuple of positional arguments @@ -709,7 +696,10 @@ or both. If a string is returned, the string should be interpreted as the name of a global variable. It should be the object's local name relative to its module; the pickle module searches the module namespace to determine the - object's module. This behaviour is typically useful for singletons. + object's module: for a given ``obj`` to be pickled, the ``__module__`` + attribute is looked up on ``obj`` directly, which falls back to a lookup + on the type of ``obj`` if no ``__module__`` instance attribute is set. + This behaviour is typically useful for singletons. When a tuple is returned, it must be between two and six items long. Optional items can either be omitted, or ``None`` can be provided as their @@ -732,8 +722,8 @@ or both. These items will be appended to the object either using ``obj.append(item)`` or, in batch, using ``obj.extend(list_of_items)``. This is primarily used for list subclasses, but may be used by other - classes as long as they have - :ref:`append and extend methods ` with + classes as long as they have :meth:`~sequence.append` + and :meth:`~sequence.extend` methods with the appropriate signature. (Whether :meth:`!append` or :meth:`!extend` is used depends on which pickle protocol version is used as well as the number of items to append, so both must be supported.) @@ -773,13 +763,13 @@ Persistence of External Objects single: persistent_id (pickle protocol) single: persistent_load (pickle protocol) -For the benefit of object persistence, the :mod:`pickle` module supports the +For the benefit of object persistence, the :mod:`!pickle` module supports the notion of a reference to an object outside the pickled data stream. Such objects are referenced by a persistent ID, which should be either a string of alphanumeric characters (for protocol 0) [#]_ or just an arbitrary object (for any newer protocol). -The resolution of such persistent IDs is not defined by the :mod:`pickle` +The resolution of such persistent IDs is not defined by the :mod:`!pickle` module; it will delegate this resolution to the user-defined methods on the pickler and unpickler, :meth:`~Pickler.persistent_id` and :meth:`~Unpickler.persistent_load` respectively. @@ -973,10 +963,10 @@ Out-of-band Buffers .. versionadded:: 3.8 -In some contexts, the :mod:`pickle` module is used to transfer massive amounts +In some contexts, the :mod:`!pickle` module is used to transfer massive amounts of data. Therefore, it can be important to minimize the number of memory copies, to preserve performance and resource consumption. However, normal -operation of the :mod:`pickle` module, as it transforms a graph-like structure +operation of the :mod:`!pickle` module, as it transforms a graph-like structure of objects into a sequential stream of bytes, intrinsically involves copying data to and from the pickle stream. @@ -995,8 +985,8 @@ for any large data. A :class:`PickleBuffer` object *signals* that the underlying buffer is eligible for out-of-band data transfer. Those objects remain compatible -with normal usage of the :mod:`pickle` module. However, consumers can also -opt-in to tell :mod:`pickle` that they will handle those buffers by +with normal usage of the :mod:`!pickle` module. However, consumers can also +opt-in to tell :mod:`!pickle` that they will handle those buffers by themselves. Consumer API @@ -1172,7 +1162,7 @@ Performance Recent versions of the pickle protocol (from protocol 2 and upwards) feature efficient binary encodings for several common features and built-in types. -Also, the :mod:`pickle` module has a transparent optimizer written in C. +Also, the :mod:`!pickle` module has a transparent optimizer written in C. .. _pickle-example: @@ -1215,7 +1205,7 @@ The following example reads the resulting pickled data. :: Command-line interface ---------------------- -The :mod:`pickle` module can be invoked as a script from the command line, +The :mod:`!pickle` module can be invoked as a script from the command line, it will display contents of the pickle files. However, when the pickle file that you want to examine comes from an untrusted source, ``-m pickletools`` is a safer option because it does not execute pickle bytecode, see @@ -1243,7 +1233,7 @@ The following option is accepted: Tools for working with and analyzing pickled data. Module :mod:`shelve` - Indexed databases of objects; uses :mod:`pickle`. + Indexed databases of objects; uses :mod:`!pickle`. Module :mod:`copy` Shallow and deep object copying. diff --git a/Doc/library/pickletools.rst b/Doc/library/pickletools.rst index 30fc2962e0bf785..7a771ea3ab93d41 100644 --- a/Doc/library/pickletools.rst +++ b/Doc/library/pickletools.rst @@ -15,7 +15,7 @@ This module contains various constants relating to the intimate details of the few useful functions for analyzing pickled data. The contents of this module are useful for Python core developers who are working on the :mod:`pickle`; ordinary users of the :mod:`pickle` module probably won't find the -:mod:`pickletools` module relevant. +:mod:`!pickletools` module relevant. .. _pickletools-cli: diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index 20b8f6bcf19117b..aa7dd71c1329dfd 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -69,8 +69,8 @@ support. Yield :term:`finder` objects for the given module name. - If fullname contains a ``'.'``, the finders will be for the package - containing fullname, otherwise they will be all registered top level + If *fullname* contains a ``'.'``, the finders will be for the package + containing *fullname*, otherwise they will be all registered top level finders (i.e. those on both :data:`sys.meta_path` and :data:`sys.path_hooks`). If the named module is in a package, that package is imported as a side @@ -151,24 +151,48 @@ support. :meth:`get_data ` API. The *package* argument should be the name of a package, in standard module format (``foo.bar``). The *resource* argument should be in the form of a relative - filename, using ``/`` as the path separator. The parent directory name - ``..`` is not allowed, and nor is a rooted name (starting with a ``/``). + filename, using ``/`` as the path separator. The function returns a binary string that is the contents of the specified resource. + This function uses the :term:`loader` method + :func:`~importlib.abc.FileLoader.get_data` + to support modules installed in the filesystem, but also in zip files, + databases, or elsewhere. + For packages located in the filesystem, which have already been imported, this is the rough equivalent of:: d = os.path.dirname(sys.modules[package].__file__) data = open(os.path.join(d, resource), 'rb').read() + Like the :func:`open` function, :func:`!get_data` can follow parent + directories (``../``) and absolute paths (starting with ``/`` or ``C:/``, + for example). + It can open compilation/installation artifacts like ``.py`` and ``.pyc`` + files or files with :func:`reserved filenames `. + To be compatible with non-filesystem loaders, avoid using these features. + + .. warning:: + + This function is intended for trusted input. + It does not verify that *resource* "belongs" to *package*. + + If you use a user-provided *resource* path, consider verifying it. + For example, require an alphanumeric filename with a known extension, or + install and check a list of known resources. + If the package cannot be located or loaded, or it uses a :term:`loader` which does not support :meth:`get_data `, then ``None`` is returned. In particular, the :term:`loader` for :term:`namespace packages ` does not support :meth:`get_data `. + .. seealso:: + + The :mod:`importlib.resources` module provides structured access to + module resources. .. function:: resolve_name(name) diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index 5c999054323be55..ff254caa5fb8ba0 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -56,6 +56,8 @@ Cross platform Returns the machine type, e.g. ``'AMD64'``. An empty string is returned if the value cannot be determined. + The output is platform-dependent and may differ in casing and naming conventions. + .. function:: node() @@ -176,8 +178,8 @@ Cross platform :attr:`processor` is resolved late, on demand. Note: the first two attribute names differ from the names presented by - :func:`os.uname`, where they are named :attr:`sysname` and - :attr:`nodename`. + :func:`os.uname`, where they are named :attr:`!sysname` and + :attr:`!nodename`. Entries which cannot be determined are set to ``''``. @@ -187,6 +189,14 @@ Cross platform .. versionchanged:: 3.9 :attr:`processor` is resolved late instead of immediately. +.. function:: invalidate_caches() + + Clear out the internal cache of information, such as the :func:`uname`. + This is typically useful when the platform's :func:`node` is changed + by an external process and one needs to retrieve the updated value. + + .. versionadded:: 3.14 + Java platform ------------- @@ -365,7 +375,7 @@ Android platform Command-line usage ------------------ -:mod:`platform` can also be invoked directly using the :option:`-m` +:mod:`!platform` can also be invoked directly using the :option:`-m` switch of the interpreter:: python -m platform [--terse] [--nonaliased] [{nonaliased,terse} ...] @@ -388,14 +398,3 @@ The following options are accepted: You can also pass one or more positional arguments (``terse``, ``nonaliased``) to explicitly control the output format. These behave similarly to their corresponding options. - -Miscellaneous -------------- - -.. function:: invalidate_caches() - - Clear out the internal cache of information, such as the :func:`uname`. - This is typically useful when the platform's :func:`node` is changed - by an external process and one needs to retrieve the updated value. - - .. versionadded:: 3.14 diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index 415c4b45c4f100e..41802ada7e1dbb0 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -184,7 +184,7 @@ Examples Generating a plist:: - import datetime + import datetime as dt import plistlib pl = dict( @@ -200,7 +200,7 @@ Generating a plist:: ), someData = b"", someMoreData = b"" * 10, - aDate = datetime.datetime.now() + aDate = dt.datetime.now() ) print(plistlib.dumps(pl).decode()) diff --git a/Doc/library/poplib.rst b/Doc/library/poplib.rst index 23f20b00e6dc6d8..51ae480338ddb7a 100644 --- a/Doc/library/poplib.rst +++ b/Doc/library/poplib.rst @@ -30,7 +30,7 @@ mailserver supports IMAP, you would be better off using the .. include:: ../includes/wasm-notavail.rst -The :mod:`poplib` module provides two classes: +The :mod:`!poplib` module provides two classes: .. class:: POP3(host, port=POP3_PORT[, timeout]) @@ -86,7 +86,7 @@ The :mod:`poplib` module provides two classes: .. versionchanged:: 3.12 The deprecated *keyfile* and *certfile* parameters have been removed. -One exception is defined as an attribute of the :mod:`poplib` module: +One exception is defined as an attribute of the :mod:`!poplib` module: .. exception:: error_proto diff --git a/Doc/library/posix.rst b/Doc/library/posix.rst index 14ab3e91e8a8e4c..1f1ca03dfd6222f 100644 --- a/Doc/library/posix.rst +++ b/Doc/library/posix.rst @@ -2,7 +2,6 @@ ==================================================== .. module:: posix - :platform: Unix :synopsis: The most common POSIX system calls (normally used via module os). -------------- @@ -17,10 +16,10 @@ interface). **Do not import this module directly.** Instead, import the module :mod:`os`, which provides a *portable* version of this interface. On Unix, the :mod:`os` -module provides a superset of the :mod:`posix` interface. On non-Unix operating -systems the :mod:`posix` module is not available, but a subset is always +module provides a superset of the :mod:`!posix` interface. On non-Unix operating +systems the :mod:`!posix` module is not available, but a subset is always available through the :mod:`os` interface. Once :mod:`os` is imported, there is -*no* performance penalty in using it instead of :mod:`posix`. In addition, +*no* performance penalty in using it instead of :mod:`!posix`. In addition, :mod:`os` provides some additional functionality, such as automatically calling :func:`~os.putenv` when an entry in ``os.environ`` is changed. @@ -67,7 +66,7 @@ Notable Module Contents ----------------------- In addition to many functions described in the :mod:`os` module documentation, -:mod:`posix` defines the following data item: +:mod:`!posix` defines the following data item: .. data:: environ @@ -91,4 +90,4 @@ In addition to many functions described in the :mod:`os` module documentation, which updates the environment on modification. Note also that updating :data:`os.environ` will render this dictionary obsolete. Use of the :mod:`os` module version of this is recommended over direct access to the - :mod:`posix` module. + :mod:`!posix` module. diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst index 2985f31bacb47a5..be942949d3ebfe3 100644 --- a/Doc/library/pprint.rst +++ b/Doc/library/pprint.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`pprint` module provides a capability to "pretty-print" arbitrary +The :mod:`!pprint` module provides a capability to "pretty-print" arbitrary Python data structures in a form which can be used as input to the interpreter. If the formatted structures include objects which are not fundamental Python types, the representation may not be loadable. This may be the case if objects @@ -22,8 +22,6 @@ The formatted representation keeps objects on a single line if it can, and breaks them onto multiple lines if they don't fit within the allowed width, adjustable by the *width* parameter defaulting to 80 characters. -Dictionaries are sorted by key before the display is computed. - .. versionchanged:: 3.9 Added support for pretty-printing :class:`types.SimpleNamespace`. diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index b6e51dffc401573..57f997792b85b12 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -217,14 +217,14 @@ Invoked as a script, the :mod:`pstats` module is a statistics browser for reading and examining profile dumps. It has a simple line-oriented interface (implemented using :mod:`cmd`) and interactive help. -:mod:`profile` and :mod:`cProfile` Module Reference -======================================================= +:mod:`profile` and :mod:`!cProfile` Module Reference +==================================================== .. module:: cProfile .. module:: profile :synopsis: Python source profiler. -Both the :mod:`profile` and :mod:`cProfile` modules provide the following +Both the :mod:`profile` and :mod:`!cProfile` modules provide the following functions: .. function:: run(command, filename=None, sort=-1) @@ -278,7 +278,7 @@ functions: print(s.getvalue()) The :class:`Profile` class can also be used as a context manager (supported - only in :mod:`cProfile` module. see :ref:`typecontextmanager`):: + only in :mod:`!cProfile` module. see :ref:`typecontextmanager`):: import cProfile @@ -292,11 +292,11 @@ functions: .. method:: enable() - Start collecting profiling data. Only in :mod:`cProfile`. + Start collecting profiling data. Only in :mod:`!cProfile`. .. method:: disable() - Stop collecting profiling data. Only in :mod:`cProfile`. + Stop collecting profiling data. Only in :mod:`!cProfile`. .. method:: create_stats() @@ -499,7 +499,7 @@ Analysis of the profiler data is done using the :class:`~pstats.Stats` class. significant entries. Initially, the list is taken to be the complete set of profiled functions. Each restriction is either an integer (to select a count of lines), or a decimal fraction between 0.0 and 1.0 inclusive (to - select a percentage of lines), or a string that will interpreted as a + select a percentage of lines), or a string that will be interpreted as a regular expression (to pattern match the standard name that is printed). If several restrictions are provided, then they are applied sequentially. For example:: diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst index 1a44bb13a841de6..9d6036d92c4cc80 100644 --- a/Doc/library/pty.rst +++ b/Doc/library/pty.rst @@ -2,7 +2,6 @@ ========================================= .. module:: pty - :platform: Unix :synopsis: Pseudo-Terminal Handling for Unix. .. moduleauthor:: Steen Lumholt @@ -12,7 +11,7 @@ -------------- -The :mod:`pty` module defines operations for handling the pseudo-terminal +The :mod:`!pty` module defines operations for handling the pseudo-terminal concept: starting another process and being able to write to and read from its controlling terminal programmatically. @@ -22,7 +21,7 @@ Pseudo-terminal handling is highly platform dependent. This code is mainly tested on Linux, FreeBSD, and macOS (it is supposed to work on other POSIX platforms but it's not been thoroughly tested). -The :mod:`pty` module defines the following functions: +The :mod:`!pty` module defines the following functions: .. function:: fork() diff --git a/Doc/library/pwd.rst b/Doc/library/pwd.rst index e1ff32912132f7d..7691fed2c7cb835 100644 --- a/Doc/library/pwd.rst +++ b/Doc/library/pwd.rst @@ -2,7 +2,6 @@ ===================================== .. module:: pwd - :platform: Unix :synopsis: The password database (getpwnam() and friends). -------------- diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index 75aa739d1003b8b..1cff16b6c1bf974 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -13,7 +13,7 @@ -------------- -The :mod:`py_compile` module provides a function to generate a byte-code file +The :mod:`!py_compile` module provides a function to generate a byte-code file from a source file, and another function used when the module source file is invoked as a script. diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst index 5efb11d89dd143d..40f93646af2ceb5 100644 --- a/Doc/library/pyclbr.rst +++ b/Doc/library/pyclbr.rst @@ -10,7 +10,7 @@ -------------- -The :mod:`pyclbr` module provides limited information about the +The :mod:`!pyclbr` module provides limited information about the functions, classes, and methods defined in a Python-coded module. The information is sufficient to implement a module browser. The information is extracted from the Python source code rather than by diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst index e8f153ee1b35ce0..a59793c25159df9 100644 --- a/Doc/library/pydoc.rst +++ b/Doc/library/pydoc.rst @@ -71,6 +71,11 @@ will start a HTTP server on port 1234, allowing you to browse the documentation at ``http://localhost:1234/`` in your preferred web browser. Specifying ``0`` as the port number will select an arbitrary unused port. +.. warning:: + + The :mod:`!pydoc` HTTP server is intended for local use during + development and is not suitable for production use. + :program:`python -m pydoc -n ` will start the server listening at the given hostname. By default the hostname is 'localhost' but if you want the server to be reached from other machines, you may want to change the host name that the diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index 2d57cff10a9278a..de9130bf1560d93 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -16,16 +16,15 @@ references to these attributes should be marked using the :member: role. -.. warning:: +.. note:: - The :mod:`pyexpat` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. + If you need to parse untrusted or unauthenticated data, see + :ref:`xml-security`. .. index:: single: Expat -The :mod:`xml.parsers.expat` module is a Python interface to the Expat +The :mod:`!xml.parsers.expat` module is a Python interface to the Expat non-validating XML parser. The module provides a single extension type, :class:`xmlparser`, that represents the current state of an XML parser. After an :class:`xmlparser` object has been created, various attributes of the object @@ -56,7 +55,7 @@ This module provides one exception and one type object: The type of the return values from the :func:`ParserCreate` function. -The :mod:`xml.parsers.expat` module contains two functions: +The :mod:`!xml.parsers.expat` module contains two functions: .. function:: ErrorString(errno) @@ -73,6 +72,13 @@ The :mod:`xml.parsers.expat` module contains two functions: *encoding* [1]_ is given it will override the implicit or explicit encoding of the document. + .. _xmlparser-non-root: + + Parsers created through :func:`!ParserCreate` are called "root" parsers, + in the sense that they do not have any parent parser attached. Non-root + parsers are created by :meth:`parser.ExternalEntityParserCreate + `. + Expat can optionally do XML namespace processing for you, enabled by providing a value for *namespace_separator*. The value must be a one-character string; a :exc:`ValueError` will be raised if the string has an illegal length (``None`` @@ -232,6 +238,111 @@ XMLParser Objects .. versionadded:: 3.13 +:class:`!xmlparser` objects have the following methods to tune protections +against some common XML vulnerabilities. + +.. method:: xmlparser.SetBillionLaughsAttackProtectionActivationThreshold(threshold, /) + + Sets the number of output bytes needed to activate protection against + `billion laughs`_ attacks. + + The number of output bytes includes amplification from entity expansion + and reading DTD files. + + Parser objects usually have a protection activation threshold of 8 MiB, + but the actual default value depends on the underlying Expat library. + + An :exc:`ExpatError` is raised if this method is called on a + |xml-non-root-parser| parser. + The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset` + should not be used as they may have no special meaning. + + .. note:: + + Activation thresholds below 4 MiB are known to break support for DITA 1.3 + payload and are hence not recommended. + + .. versionadded:: 3.14.6 + +.. method:: xmlparser.SetBillionLaughsAttackProtectionMaximumAmplification(max_factor, /) + + Sets the maximum tolerated amplification factor for protection against + `billion laughs`_ attacks. + + The amplification factor is calculated as ``(direct + indirect) / direct`` + while parsing, where ``direct`` is the number of bytes read from + the primary document in parsing and ``indirect`` is the number of + bytes added by expanding entities and reading of external DTD files. + + The *max_factor* value must be a non-NaN :class:`float` value greater than + or equal to 1.0. Peak amplifications of factor 15,000 for the entire payload + and of factor 30,000 in the middle of parsing have been observed with small + benign files in practice. In particular, the activation threshold should be + carefully chosen to avoid false positives. + + Parser objects usually have a maximum amplification factor of 100, + but the actual default value depends on the underlying Expat library. + + An :exc:`ExpatError` is raised if this method is called on a + |xml-non-root-parser| parser or if *max_factor* is outside the valid range. + The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset` + should not be used as they may have no special meaning. + + .. note:: + + The maximum amplification factor is only considered if the threshold + that can be adjusted by :meth:`.SetBillionLaughsAttackProtectionActivationThreshold` + is exceeded. + + .. versionadded:: 3.14.6 + +.. method:: xmlparser.SetAllocTrackerActivationThreshold(threshold, /) + + Sets the number of allocated bytes of dynamic memory needed to activate + protection against disproportionate use of RAM. + + Parser objects usually have an allocation activation threshold of 64 MiB, + but the actual default value depends on the underlying Expat library. + + An :exc:`ExpatError` is raised if this method is called on a + |xml-non-root-parser| parser. + The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset` + should not be used as they may have no special meaning. + + .. versionadded:: 3.14.1 + +.. method:: xmlparser.SetAllocTrackerMaximumAmplification(max_factor, /) + + Sets the maximum amplification factor between direct input and bytes + of dynamic memory allocated. + + The amplification factor is calculated as ``allocated / direct`` + while parsing, where ``direct`` is the number of bytes read from + the primary document in parsing and ``allocated`` is the number + of bytes of dynamic memory allocated in the parser hierarchy. + + The *max_factor* value must be a non-NaN :class:`float` value greater than + or equal to 1.0. Amplification factors greater than 100.0 can be observed + near the start of parsing even with benign files in practice. In particular, + the activation threshold should be carefully chosen to avoid false positives. + + Parser objects usually have a maximum amplification factor of 100, + but the actual default value depends on the underlying Expat library. + + An :exc:`ExpatError` is raised if this method is called on a + |xml-non-root-parser| parser or if *max_factor* is outside the valid range. + The corresponding :attr:`~ExpatError.lineno` and :attr:`~ExpatError.offset` + should not be used as they may have no special meaning. + + .. note:: + + The maximum amplification factor is only considered if the threshold + that can be adjusted by :meth:`.SetAllocTrackerActivationThreshold` + is exceeded. + + .. versionadded:: 3.14.1 + + :class:`xmlparser` objects have the following attributes: @@ -354,7 +465,7 @@ otherwise stated. ...``). The *doctypeName* is provided exactly as presented. The *systemId* and *publicId* parameters give the system and public identifiers if specified, or ``None`` if omitted. *has_internal_subset* will be true if the document - contains and internal document declaration subset. This requires Expat version + contains an internal document declaration subset. This requires Expat version 1.2 or newer. @@ -503,6 +614,15 @@ otherwise stated. .. method:: xmlparser.ExternalEntityRefHandler(context, base, systemId, publicId) + .. warning:: + + Implementing a handler that accesses local files and/or the network + may create a vulnerability to + `external entity attacks `_ + if :class:`xmlparser` is used with user-provided XML content. + Please reflect on your `threat model `_ + before implementing this handler. + Called for references to external entities. *base* is the current base, as set by a previous call to :meth:`SetBase`. The public and system identifiers, *systemId* and *publicId*, are strings if given; if the public identifier is not @@ -619,7 +739,7 @@ values: the type, the quantifier, the name, and a tuple of children. Children are simply additional content model descriptions. The values of the first two fields are constants defined in the -:mod:`xml.parsers.expat.model` module. These constants can be collected in two +:mod:`!xml.parsers.expat.model` module. These constants can be collected in two groups: the model type group and the quantifier group. The constants in the model type group are: @@ -693,7 +813,7 @@ Expat error constants .. module:: xml.parsers.expat.errors -The following constants are provided in the :mod:`xml.parsers.expat.errors` +The following constants are provided in the :mod:`!xml.parsers.expat.errors` module. These constants are useful in interpreting some of the attributes of the :exc:`ExpatError` exception objects raised when an error has occurred. Since for backwards compatibility reasons, the constants' value is the error @@ -840,7 +960,7 @@ The ``errors`` module has the following attributes: An operation was requested that requires DTD support to be compiled in, but Expat was configured without DTD support. This should never be reported by a - standard build of the :mod:`xml.parsers.expat` module. + standard build of the :mod:`!xml.parsers.expat` module. .. data:: XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING @@ -955,3 +1075,6 @@ The ``errors`` module has the following attributes: not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl and https://www.iana.org/assignments/character-sets/character-sets.xhtml. + +.. _billion laughs: https://en.wikipedia.org/wiki/Billion_laughs_attack +.. |xml-non-root-parser| replace:: :ref:`non-root ` diff --git a/Doc/library/python.rst b/Doc/library/python.rst index c2c231af7c30330..c5c762e11b99e51 100644 --- a/Doc/library/python.rst +++ b/Doc/library/python.rst @@ -27,3 +27,8 @@ overview: inspect.rst annotationlib.rst site.rst + +.. seealso:: + + * See the :mod:`concurrent.interpreters` module, which similarly + exposes core runtime functionality. diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst index fbbebcf4ed8f927..f5326aff7236bd6 100644 --- a/Doc/library/queue.rst +++ b/Doc/library/queue.rst @@ -8,7 +8,7 @@ -------------- -The :mod:`queue` module implements multi-producer, multi-consumer queues. +The :mod:`!queue` module implements multi-producer, multi-consumer queues. It is especially useful in threaded programming when information must be exchanged safely between multiple threads. The :class:`Queue` class in this module implements all the required locking semantics. @@ -30,7 +30,7 @@ In addition, the module implements a "simple" specific implementation provides additional guarantees in exchange for the smaller functionality. -The :mod:`queue` module defines the following classes and exceptions: +The :mod:`!queue` module defines the following classes and exceptions: .. class:: Queue(maxsize=0) @@ -76,6 +76,8 @@ The :mod:`queue` module defines the following classes and exceptions: Constructor for an unbounded :abbr:`FIFO (first-in, first-out)` queue. Simple queues lack advanced functionality such as task tracking. + Simple queues are :ref:`generic ` over the type of their items. + .. versionadded:: 3.7 @@ -187,9 +189,6 @@ fully processed by daemon consumer threads. processed (meaning that a :meth:`task_done` call was received for every item that had been :meth:`put` into the queue). - ``shutdown(immediate=True)`` calls :meth:`task_done` for each remaining item - in the queue. - Raises a :exc:`ValueError` if called more times than there were items placed in the queue. @@ -204,6 +203,9 @@ fully processed by daemon consumer threads. count of unfinished tasks drops to zero, :meth:`join` unblocks. +Waiting for task completion +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + Example of how to wait for enqueued tasks to be completed:: import threading @@ -233,22 +235,39 @@ Example of how to wait for enqueued tasks to be completed:: Terminating queues ^^^^^^^^^^^^^^^^^^ -:class:`Queue` objects can be made to prevent further interaction by shutting -them down. +When no longer needed, :class:`Queue` objects can be wound down +until empty or terminated immediately with a hard shutdown. .. method:: Queue.shutdown(immediate=False) - Shut down the queue, making :meth:`~Queue.get` and :meth:`~Queue.put` raise - :exc:`ShutDown`. + Put a :class:`Queue` instance into a shutdown mode. + + The queue can no longer grow. + Future calls to :meth:`~Queue.put` raise :exc:`ShutDown`. + Currently blocked callers of :meth:`~Queue.put` will be unblocked + and will raise :exc:`ShutDown` in the formerly blocked thread. + + If *immediate* is false (the default), the queue can be wound + down normally with :meth:`~Queue.get` calls to extract tasks + that have already been loaded. + + And if :meth:`~Queue.task_done` is called for each remaining task, a + pending :meth:`~Queue.join` will be unblocked normally. + + Once the queue is empty, future calls to :meth:`~Queue.get` will + raise :exc:`ShutDown`. - By default, :meth:`~Queue.get` on a shut down queue will only raise once the - queue is empty. Set *immediate* to true to make :meth:`~Queue.get` raise - immediately instead. + If *immediate* is true, the queue is terminated immediately. + The queue is drained to be completely empty and the count + of unfinished tasks is reduced by the number of tasks drained. + If unfinished tasks is zero, callers of :meth:`~Queue.join` + are unblocked. Also, blocked callers of :meth:`~Queue.get` + are unblocked and will raise :exc:`ShutDown` because the + queue is empty. - All blocked callers of :meth:`~Queue.put` and :meth:`~Queue.get` will be - unblocked. If *immediate* is true, a task will be marked as done for each - remaining item in the queue, which may unblock callers of - :meth:`~Queue.join`. + Use caution when using :meth:`~Queue.join` with *immediate* set + to true. This unblocks the join even when no work has been done + on the tasks, violating the usual invariant for joining a queue. .. versionadded:: 3.13 diff --git a/Doc/library/random.rst b/Doc/library/random.rst index ef0cfb0e76cef6e..9327e5a23d621a4 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -37,7 +37,7 @@ Class :class:`Random` can also be subclassed if you want to use a different basic generator of your own devising: see the documentation on that class for more details. -The :mod:`random` module also provides the :class:`SystemRandom` class which +The :mod:`!random` module also provides the :class:`SystemRandom` class which uses the system function :func:`os.urandom` to generate random numbers from sources provided by the operating system. @@ -78,7 +78,7 @@ Bookkeeping functions instead of the system time (see the :func:`os.urandom` function for details on availability). - If *a* is an int, it is used directly. + If *a* is an int, its absolute value is used directly. With version 2 (the default), a :class:`str`, :class:`bytes`, or :class:`bytearray` object gets converted to an :class:`int` and all of its bits are used. @@ -410,7 +410,7 @@ Alternative Generator .. class:: Random([seed]) Class that implements the default pseudo-random number generator used by the - :mod:`random` module. + :mod:`!random` module. .. versionchanged:: 3.11 Formerly the *seed* could be any hashable object. Now it is limited to: @@ -447,6 +447,11 @@ Alternative Generator Override this method in subclasses to customise the :meth:`~random.getrandbits` behaviour of :class:`!Random` instances. + .. method:: Random.randbytes(n) + + Override this method in subclasses to customise the + :meth:`~random.randbytes` behaviour of :class:`!Random` instances. + .. class:: SystemRandom([seed]) @@ -628,11 +633,12 @@ These recipes show how to efficiently make random selections from the combinatoric iterators in the :mod:`itertools` module: .. testcode:: + import random - def random_product(*args, repeat=1): - "Random selection from itertools.product(*args, **kwds)" - pools = [tuple(pool) for pool in args] * repeat + def random_product(*iterables, repeat=1): + "Random selection from itertools.product(*iterables, repeat=repeat)" + pools = tuple(map(tuple, iterables)) * repeat return tuple(map(random.choice, pools)) def random_permutation(iterable, r=None): @@ -656,6 +662,91 @@ from the combinatoric iterators in the :mod:`itertools` module: indices = sorted(random.choices(range(n), k=r)) return tuple(pool[i] for i in indices) + def random_derangement(iterable): + "Choose a permutation where no element stays in its original position." + seq = tuple(iterable) + if len(seq) < 2: + if not seq: + return () + raise IndexError('No derangments to choose from') + perm = list(range(len(seq))) + start = tuple(perm) + while True: + random.shuffle(perm) + if all(p != q for p, q in zip(start, perm)): + return tuple([seq[i] for i in perm]) + +.. doctest:: + :hide: + + >>> import random + + + >>> random.seed(8675309) + >>> random_product('ABCDEFG', repeat=5) + ('D', 'B', 'E', 'F', 'E') + + + >>> random.seed(8675309) + >>> random_permutation('ABCDEFG') + ('D', 'B', 'E', 'C', 'G', 'A', 'F') + >>> random_permutation('ABCDEFG', 5) + ('A', 'G', 'D', 'C', 'B') + + + >>> random.seed(8675309) + >>> random_combination('ABCDEFG', 7) + ('A', 'B', 'C', 'D', 'E', 'F', 'G') + >>> random_combination('ABCDEFG', 6) + ('A', 'B', 'C', 'D', 'F', 'G') + >>> random_combination('ABCDEFG', 5) + ('A', 'B', 'C', 'E', 'F') + >>> random_combination('ABCDEFG', 4) + ('B', 'C', 'D', 'G') + >>> random_combination('ABCDEFG', 3) + ('B', 'E', 'G') + >>> random_combination('ABCDEFG', 2) + ('E', 'G') + >>> random_combination('ABCDEFG', 1) + ('C',) + >>> random_combination('ABCDEFG', 0) + () + + + >>> random.seed(8675309) + >>> random_combination_with_replacement('ABCDEFG', 7) + ('B', 'C', 'D', 'E', 'E', 'E', 'G') + >>> random_combination_with_replacement('ABCDEFG', 3) + ('A', 'B', 'E') + >>> random_combination_with_replacement('ABCDEFG', 2) + ('A', 'G') + >>> random_combination_with_replacement('ABCDEFG', 1) + ('E',) + >>> random_combination_with_replacement('ABCDEFG', 0) + () + + + >>> random.seed(8675309) + >>> random_derangement('') + () + >>> random_derangement('A') + Traceback (most recent call last): + ... + IndexError: No derangments to choose from + >>> random_derangement('AB') + ('B', 'A') + >>> random_derangement('ABC') + ('C', 'A', 'B') + >>> random_derangement('ABCD') + ('B', 'A', 'D', 'C') + >>> random_derangement('ABCDE') + ('B', 'C', 'A', 'E', 'D') + >>> # Identical inputs treated as distinct + >>> identical = 20 + >>> random_derangement((10, identical, 30, identical)) + (20, 30, 10, 20) + + The default :func:`.random` returns multiples of 2⁻⁵³ in the range *0.0 ≤ x < 1.0*. All such numbers are evenly spaced and are exactly representable as Python floats. However, many other representable diff --git a/Doc/library/re.rst b/Doc/library/re.rst index eb3b1e5549cc982..456d74292ef4a7e 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -49,7 +49,7 @@ fine-tuning parameters. .. seealso:: The third-party :pypi:`regex` module, - which has an API compatible with the standard library :mod:`re` module, + which has an API compatible with the standard library :mod:`!re` module, but offers additional functionality and a more thorough Unicode support. @@ -512,8 +512,9 @@ The special characters are: *name* exists, and with ``no-pattern`` if it doesn't. ``no-pattern`` is optional and can be omitted. For example, ``(<)?(\w+@\w+(?:\.\w+)+)(?(1)>|$)`` is a poor email matching pattern, which - will match with ``''`` as well as ``'user@host.com'``, but - not with ``''``. + matches ``''`` as well as ``'user@host.com'``, but does not + match ``''`` in their entirety + (:func:`re.search` finds only ``'user@host.com'`` in the former). .. versionchanged:: 3.12 Group *id* can only contain ASCII digits. @@ -991,8 +992,8 @@ Functions That way, separator components are always found at the same relative indices within the result list. - Empty matches for the pattern split the string only when not adjacent - to a previous empty match. + Adjacent empty matches are not possible, but an empty match can occur + immediately after a non-empty match. .. code:: pycon @@ -1095,9 +1096,12 @@ Functions The optional argument *count* is the maximum number of pattern occurrences to be replaced; *count* must be a non-negative integer. If omitted or zero, all - occurrences will be replaced. Empty matches for the pattern are replaced only - when not adjacent to a previous empty match, so ``sub('x*', '-', 'abxd')`` returns - ``'-a-b--d-'``. + occurrences will be replaced. + + Adjacent empty matches are not possible, but an empty match can occur + immediately after a non-empty match. + As a result, ``sub('x*', '-', 'abxd')`` returns ``'-a-b--d-'`` + instead of ``'-a-b-d-'``. .. index:: single: \g; in regular expressions @@ -1128,8 +1132,7 @@ Functions .. versionchanged:: 3.7 Unknown escapes in *repl* consisting of ``'\'`` and an ASCII letter now are errors. - Empty matches for the pattern are replaced when adjacent to a previous - non-empty match. + An empty match can occur immediately after a non-empty match. .. versionchanged:: 3.12 Group *id* can only contain ASCII digits. @@ -1239,6 +1242,9 @@ Regular Expression Objects Compiled regular expression object returned by :func:`re.compile`. + Patterns are :ref:`generic ` over the type of string they handle + (:class:`str` or :class:`bytes`). + .. versionchanged:: 3.9 :py:class:`re.Pattern` supports ``[]`` to indicate a Unicode (str) or bytes pattern. See :ref:`types-genericalias`. @@ -1382,6 +1388,9 @@ when there is no match, you can test whether there was a match with a simple Match object returned by successful ``match``\ es and ``search``\ es. + Matches are :ref:`generic ` over the type of string which was + matched (:class:`str` or :class:`bytes`). + .. versionchanged:: 3.9 :py:class:`re.Match` supports ``[]`` to indicate a Unicode (str) or bytes match. See :ref:`types-genericalias`. @@ -1405,10 +1414,10 @@ when there is no match, you can test whether there was a match with a simple result is a single string; if there are multiple arguments, the result is a tuple with one item per argument. Without arguments, *group1* defaults to zero (the whole match is returned). If a *groupN* argument is zero, the corresponding - return value is the entire matching string; if it is in the inclusive range - [1..99], it is the string matching the corresponding parenthesized group. If a - group number is negative or larger than the number of groups defined in the - pattern, an :exc:`IndexError` exception is raised. If a group is contained in a + return value is the entire matching string; if it is a positive integer, it is + the string matching the corresponding parenthesized group. If a group number is + negative or larger than the number of groups defined in the pattern, an + :exc:`IndexError` exception is raised. If a group is contained in a part of the pattern that did not match, the corresponding result is ``None``. If a group is contained in a part of the pattern that matched multiple times, the last match is returned. :: @@ -1881,7 +1890,7 @@ successive matches:: class Token(NamedTuple): type: str - value: str + value: int | float | str line: int column: int diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst index f649fce5efc3779..be36f7b1b6a2ea3 100644 --- a/Doc/library/readline.rst +++ b/Doc/library/readline.rst @@ -2,14 +2,13 @@ =========================================== .. module:: readline - :platform: Unix :synopsis: GNU readline support for Python. .. sectionauthor:: Skip Montanaro -------------- -The :mod:`readline` module defines a number of functions to facilitate +The :mod:`!readline` module defines a number of functions to facilitate completion and reading/writing of history files from the Python interpreter. This module can be used directly, or via the :mod:`rlcompleter` module, which supports completion of Python identifiers at the interactive prompt. Settings @@ -26,11 +25,15 @@ Readline library in general. .. include:: ../includes/wasm-mobile-notavail.rst +.. include:: ../includes/optional-module.rst + +.. availability:: Unix. + .. note:: The underlying Readline library API may be implemented by the ``editline`` (``libedit``) library instead of GNU readline. - On macOS the :mod:`readline` module detects which library is being used + On macOS the :mod:`!readline` module detects which library is being used at run time. The configuration file for ``editline`` is different from that @@ -253,7 +256,7 @@ The following functions relate to implementing a custom word completion function. This is typically operated by the Tab key, and can suggest and automatically complete a word being typed. By default, Readline is set up to be used by :mod:`rlcompleter` to complete Python identifiers for -the interactive interpreter. If the :mod:`readline` module is to be used +the interactive interpreter. If the :mod:`!readline` module is to be used with a custom completer, a different set of word delimiters should be set. @@ -322,7 +325,7 @@ with a custom completer, a different set of word delimiters should be set. Example ------- -The following example demonstrates how to use the :mod:`readline` module's +The following example demonstrates how to use the :mod:`!readline` module's history reading and writing functions to automatically load and save a history file named :file:`.python_history` from the user's home directory. The code below would normally be executed automatically during interactive sessions @@ -392,3 +395,9 @@ support history save/restore. :: def save_history(self, histfile): readline.set_history_length(1000) readline.write_history_file(histfile) + +.. note:: + + The new :term:`REPL` introduced in version 3.13 doesn't support readline. + However, readline can still be used by setting the :envvar:`PYTHON_BASIC_REPL` + environment variable. diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index 0515d205bbca0bb..357686da00ca4e1 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -2,7 +2,6 @@ =============================================== .. module:: resource - :platform: Unix :synopsis: An interface to provide resource usage information on the current process. .. moduleauthor:: Jeremy Hylton @@ -63,12 +62,12 @@ this module for those platforms. Sets new limits of consumption of *resource*. The *limits* argument must be a tuple ``(soft, hard)`` of two integers describing the new limits. A value of - :data:`~resource.RLIM_INFINITY` can be used to request a limit that is + :const:`~resource.RLIM_INFINITY` can be used to request a limit that is unlimited. Raises :exc:`ValueError` if an invalid resource is specified, if the new soft limit exceeds the hard limit, or if a process tries to raise its hard limit. - Specifying a limit of :data:`~resource.RLIM_INFINITY` when the hard or + Specifying a limit of :const:`~resource.RLIM_INFINITY` when the hard or system limit for that resource is not unlimited will result in a :exc:`ValueError`. A process with the effective UID of super-user can request any valid limit value, including unlimited, but :exc:`ValueError` @@ -78,7 +77,7 @@ this module for those platforms. ``setrlimit`` may also raise :exc:`error` if the underlying system call fails. - VxWorks only supports setting :data:`RLIMIT_NOFILE`. + VxWorks only supports setting :const:`RLIMIT_NOFILE`. .. audit-event:: resource.setrlimit resource,limits resource.setrlimit @@ -127,7 +126,7 @@ platform. .. data:: RLIMIT_CPU The maximum amount of processor time (in seconds) that a process can use. If - this limit is exceeded, a :const:`SIGXCPU` signal is sent to the process. (See + this limit is exceeded, a :const:`~signal.SIGXCPU` signal is sent to the process. (See the :mod:`signal` module documentation for information about how to catch this signal and do something useful, e.g. flush open files to disk.) @@ -176,8 +175,9 @@ platform. .. data:: RLIMIT_VMEM The largest area of mapped memory which the process may occupy. + Usually an alias of :const:`RLIMIT_AS`. - .. availability:: FreeBSD >= 11. + .. availability:: Solaris, FreeBSD, NetBSD. .. data:: RLIMIT_AS @@ -230,16 +230,18 @@ platform. .. versionadded:: 3.4 + .. data:: RLIMIT_SBSIZE The maximum size (in bytes) of socket buffer usage for this user. This limits the amount of network memory, and hence the amount of mbufs, that this user may hold at any time. - .. availability:: FreeBSD. + .. availability:: FreeBSD, NetBSD. .. versionadded:: 3.4 + .. data:: RLIMIT_SWAP The maximum size (in bytes) of the swap space that may be reserved or @@ -249,18 +251,20 @@ platform. `tuning(7) `__ for a complete description of this sysctl. - .. availability:: FreeBSD. + .. availability:: FreeBSD >= 8. .. versionadded:: 3.4 + .. data:: RLIMIT_NPTS The maximum number of pseudo-terminals created by this user id. - .. availability:: FreeBSD. + .. availability:: FreeBSD >= 8. .. versionadded:: 3.4 + .. data:: RLIMIT_KQUEUES The maximum number of kqueues this user id is allowed to create. @@ -269,6 +273,7 @@ platform. .. versionadded:: 3.10 + Resource Usage -------------- @@ -297,54 +302,54 @@ These functions are used to retrieve resource usage information: print(getrusage(RUSAGE_SELF)) The fields of the return value each describe how a particular system resource - has been used, e.g. amount of time spent running is user mode or number of times + has been used, e.g. amount of time spent running in user mode or number of times the process was swapped out of main memory. Some values are dependent on the - clock tick internal, e.g. the amount of memory the process is using. + clock tick interval, e.g. the amount of memory the process is using. For backward compatibility, the return value is also accessible as a tuple of 16 elements. - The fields :attr:`ru_utime` and :attr:`ru_stime` of the return value are + The fields :attr:`!ru_utime` and :attr:`!ru_stime` of the return value are floating-point values representing the amount of time spent executing in user mode and the amount of time spent executing in system mode, respectively. The remaining values are integers. Consult the :manpage:`getrusage(2)` man page for detailed information about these values. A brief summary is presented here: - +--------+---------------------+---------------------------------------+ - | Index | Field | Resource | - +========+=====================+=======================================+ - | ``0`` | :attr:`ru_utime` | time in user mode (float seconds) | - +--------+---------------------+---------------------------------------+ - | ``1`` | :attr:`ru_stime` | time in system mode (float seconds) | - +--------+---------------------+---------------------------------------+ - | ``2`` | :attr:`ru_maxrss` | maximum resident set size | - +--------+---------------------+---------------------------------------+ - | ``3`` | :attr:`ru_ixrss` | shared memory size | - +--------+---------------------+---------------------------------------+ - | ``4`` | :attr:`ru_idrss` | unshared memory size | - +--------+---------------------+---------------------------------------+ - | ``5`` | :attr:`ru_isrss` | unshared stack size | - +--------+---------------------+---------------------------------------+ - | ``6`` | :attr:`ru_minflt` | page faults not requiring I/O | - +--------+---------------------+---------------------------------------+ - | ``7`` | :attr:`ru_majflt` | page faults requiring I/O | - +--------+---------------------+---------------------------------------+ - | ``8`` | :attr:`ru_nswap` | number of swap outs | - +--------+---------------------+---------------------------------------+ - | ``9`` | :attr:`ru_inblock` | block input operations | - +--------+---------------------+---------------------------------------+ - | ``10`` | :attr:`ru_oublock` | block output operations | - +--------+---------------------+---------------------------------------+ - | ``11`` | :attr:`ru_msgsnd` | messages sent | - +--------+---------------------+---------------------------------------+ - | ``12`` | :attr:`ru_msgrcv` | messages received | - +--------+---------------------+---------------------------------------+ - | ``13`` | :attr:`ru_nsignals` | signals received | - +--------+---------------------+---------------------------------------+ - | ``14`` | :attr:`ru_nvcsw` | voluntary context switches | - +--------+---------------------+---------------------------------------+ - | ``15`` | :attr:`ru_nivcsw` | involuntary context switches | - +--------+---------------------+---------------------------------------+ + +--------+----------------------+---------------------------------------+ + | Index | Field | Resource | + +========+======================+=======================================+ + | ``0`` | :attr:`!ru_utime` | time in user mode (float seconds) | + +--------+----------------------+---------------------------------------+ + | ``1`` | :attr:`!ru_stime` | time in system mode (float seconds) | + +--------+----------------------+---------------------------------------+ + | ``2`` | :attr:`!ru_maxrss` | maximum resident set size | + +--------+----------------------+---------------------------------------+ + | ``3`` | :attr:`!ru_ixrss` | shared memory size | + +--------+----------------------+---------------------------------------+ + | ``4`` | :attr:`!ru_idrss` | unshared memory size | + +--------+----------------------+---------------------------------------+ + | ``5`` | :attr:`!ru_isrss` | unshared stack size | + +--------+----------------------+---------------------------------------+ + | ``6`` | :attr:`!ru_minflt` | page faults not requiring I/O | + +--------+----------------------+---------------------------------------+ + | ``7`` | :attr:`!ru_majflt` | page faults requiring I/O | + +--------+----------------------+---------------------------------------+ + | ``8`` | :attr:`!ru_nswap` | number of swap outs | + +--------+----------------------+---------------------------------------+ + | ``9`` | :attr:`!ru_inblock` | block input operations | + +--------+----------------------+---------------------------------------+ + | ``10`` | :attr:`!ru_oublock` | block output operations | + +--------+----------------------+---------------------------------------+ + | ``11`` | :attr:`!ru_msgsnd` | messages sent | + +--------+----------------------+---------------------------------------+ + | ``12`` | :attr:`!ru_msgrcv` | messages received | + +--------+----------------------+---------------------------------------+ + | ``13`` | :attr:`!ru_nsignals` | signals received | + +--------+----------------------+---------------------------------------+ + | ``14`` | :attr:`!ru_nvcsw` | voluntary context switches | + +--------+----------------------+---------------------------------------+ + | ``15`` | :attr:`!ru_nivcsw` | involuntary context switches | + +--------+----------------------+---------------------------------------+ This function will raise a :exc:`ValueError` if an invalid *who* parameter is specified. It may also raise :exc:`error` exception in unusual circumstances. diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst index b07ec6e93f80aba..2e8081f3bf0d611 100644 --- a/Doc/library/runpy.rst +++ b/Doc/library/runpy.rst @@ -10,7 +10,7 @@ -------------- -The :mod:`runpy` module is used to locate and run Python modules without +The :mod:`!runpy` module is used to locate and run Python modules without importing them first. Its main use is to implement the :option:`-m` command line switch that allows scripts to be located using the Python module namespace rather than the filesystem. @@ -20,11 +20,11 @@ current process, and any side effects (such as cached imports of other modules) will remain in place after the functions have returned. Furthermore, any functions and classes defined by the executed code are not -guaranteed to work correctly after a :mod:`runpy` function has returned. +guaranteed to work correctly after a :mod:`!runpy` function has returned. If that limitation is not acceptable for a given use case, :mod:`importlib` is likely to be a more suitable choice than this module. -The :mod:`runpy` module provides two functions: +The :mod:`!runpy` module provides two functions: .. function:: run_module(mod_name, init_globals=None, run_name=None, alter_sys=False) diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst index 517dbe8c3218989..302231d95f89791 100644 --- a/Doc/library/sched.rst +++ b/Doc/library/sched.rst @@ -12,7 +12,7 @@ -------------- -The :mod:`sched` module defines a class which implements a general purpose event +The :mod:`!sched` module defines a class which implements a general purpose event scheduler: .. class:: scheduler(timefunc=time.monotonic, delayfunc=time.sleep) @@ -119,9 +119,11 @@ Scheduler Objects function passed to the constructor) for the next event, then execute it and so on until there are no more scheduled events. - If *blocking* is false executes the scheduled events due to expire soonest - (if any) and then return the deadline of the next scheduled call in the - scheduler (if any). + If *blocking* is false, immediately executes all events in the queue which have + a time value less than or equal to the current *timefunc* value (if any) and + returns the difference between the current *timefunc* value and the time value + of the next scheduled event in the scheduler's event queue. If the queue is + empty, returns ``None``. Either *action* or *delayfunc* can raise an exception. In either case, the scheduler will maintain a consistent state and propagate the exception. If an diff --git a/Doc/library/secrets.rst b/Doc/library/secrets.rst index 75dafc54d40ca58..6d708d1f713f9d6 100644 --- a/Doc/library/secrets.rst +++ b/Doc/library/secrets.rst @@ -17,11 +17,11 @@ ------------- -The :mod:`secrets` module is used for generating cryptographically strong +The :mod:`!secrets` module is used for generating cryptographically strong random numbers suitable for managing data such as passwords, account authentication, security tokens, and related secrets. -In particular, :mod:`secrets` should be used in preference to the +In particular, :mod:`!secrets` should be used in preference to the default pseudo-random number generator in the :mod:`random` module, which is designed for modelling and simulation, not security or cryptography. @@ -33,7 +33,7 @@ is designed for modelling and simulation, not security or cryptography. Random numbers -------------- -The :mod:`secrets` module provides access to the most secure source of +The :mod:`!secrets` module provides access to the most secure source of randomness that your operating system provides. .. class:: SystemRandom @@ -58,43 +58,48 @@ randomness that your operating system provides. Generating tokens ----------------- -The :mod:`secrets` module provides functions for generating secure +The :mod:`!secrets` module provides functions for generating secure tokens, suitable for applications such as password resets, hard-to-guess URLs, and similar. -.. function:: token_bytes([nbytes=None]) +.. function:: token_bytes(nbytes=None) Return a random byte string containing *nbytes* number of bytes. - If *nbytes* is ``None`` or not supplied, a reasonable default is - used. + + If *nbytes* is not specified or ``None``, :const:`DEFAULT_ENTROPY` + is used instead. .. doctest:: - >>> token_bytes(16) #doctest:+SKIP + >>> token_bytes(16) # doctest: +SKIP b'\xebr\x17D*t\xae\xd4\xe3S\xb6\xe2\xebP1\x8b' -.. function:: token_hex([nbytes=None]) +.. function:: token_hex(nbytes=None) Return a random text string, in hexadecimal. The string has *nbytes* - random bytes, each byte converted to two hex digits. If *nbytes* is - ``None`` or not supplied, a reasonable default is used. + random bytes, each byte converted to two hex digits. + + If *nbytes* is not specified or ``None``, :const:`DEFAULT_ENTROPY` + is used instead. .. doctest:: - >>> token_hex(16) #doctest:+SKIP + >>> token_hex(16) # doctest: +SKIP 'f9bf78b9a18ce6d46a0cd2b0b86df9da' -.. function:: token_urlsafe([nbytes=None]) +.. function:: token_urlsafe(nbytes=None) Return a random URL-safe text string, containing *nbytes* random bytes. The text is Base64 encoded, so on average each byte results - in approximately 1.3 characters. If *nbytes* is ``None`` or not - supplied, a reasonable default is used. + in approximately 1.3 characters. + + If *nbytes* is not specified or ``None``, :const:`DEFAULT_ENTROPY` + is used instead. .. doctest:: - >>> token_urlsafe(16) #doctest:+SKIP + >>> token_urlsafe(16) # doctest: +SKIP 'Drmhze6EPcv0fN_81Bj-nA' @@ -107,7 +112,7 @@ tokens need to have sufficient randomness. Unfortunately, what is considered sufficient will necessarily increase as computers get more powerful and able to make more guesses in a shorter period. As of 2015, it is believed that 32 bytes (256 bits) of randomness is sufficient for -the typical use-case expected for the :mod:`secrets` module. +the typical use-case expected for the :mod:`!secrets` module. For those who want to manage their own token length, you can explicitly specify how much randomness is used for tokens by giving an :class:`int` @@ -115,11 +120,13 @@ argument to the various ``token_*`` functions. That argument is taken as the number of bytes of randomness to use. Otherwise, if no argument is provided, or if the argument is ``None``, -the ``token_*`` functions will use a reasonable default instead. +the ``token_*`` functions use :const:`DEFAULT_ENTROPY` instead. -.. note:: +.. data:: DEFAULT_ENTROPY + + Default number of bytes of randomness used by the ``token_*`` functions. - That default is subject to change at any time, including during + The exact value is subject to change at any time, including during maintenance releases. @@ -132,14 +139,14 @@ Other functions :term:`bytes-like objects ` *a* and *b* are equal, otherwise ``False``, using a "constant-time compare" to reduce the risk of - `timing attacks `_. + `timing attacks `__. See :func:`hmac.compare_digest` for additional details. Recipes and best practices -------------------------- -This section shows recipes and best practices for using :mod:`secrets` +This section shows recipes and best practices for using :mod:`!secrets` to manage a basic level of security. Generate an eight-character alphanumeric password: diff --git a/Doc/library/security_warnings.rst b/Doc/library/security_warnings.rst index a573c98f73eb0a0..70c359cc1c0fc34 100644 --- a/Doc/library/security_warnings.rst +++ b/Doc/library/security_warnings.rst @@ -28,7 +28,7 @@ The following modules have specific security considerations: ` * :mod:`tempfile`: :ref:`mktemp is deprecated due to vulnerability to race conditions ` -* :mod:`xml`: :ref:`XML vulnerabilities ` +* :mod:`xml`: :ref:`XML security ` * :mod:`zipfile`: :ref:`maliciously prepared .zip files can cause disk volume exhaustion ` diff --git a/Doc/library/select.rst b/Doc/library/select.rst index d2094283d547361..3c72c2d59150c99 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -18,7 +18,7 @@ it was last read. .. note:: The :mod:`selectors` module allows high-level and efficient I/O - multiplexing, built upon the :mod:`select` module primitives. Users are + multiplexing, built upon the :mod:`!select` module primitives. Users are encouraged to use the :mod:`selectors` module instead, unless they want precise control over the OS-level primitives used. @@ -37,7 +37,7 @@ The module defines the following: .. function:: devpoll() - (Only supported on Solaris and derivatives.) Returns a ``/dev/poll`` + Returns a ``/dev/poll`` polling object; see section :ref:`devpoll-objects` below for the methods supported by devpoll objects. @@ -54,9 +54,11 @@ The module defines the following: .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. + .. availability:: Solaris. + .. function:: epoll(sizehint=-1, flags=0) - (Only supported on Linux 2.5.44 and newer.) Return an edge polling object, + Return an edge polling object, which can be used as Edge or Level Triggered interface for I/O events. @@ -89,18 +91,22 @@ The module defines the following: The *flags* parameter. ``select.EPOLL_CLOEXEC`` is used by default now. Use :func:`os.set_inheritable` to make the file descriptor inheritable. + .. availability:: Linux >= 2.5.44. + .. function:: poll() - (Not supported by all operating systems.) Returns a polling object, which + Returns a polling object, which supports registering and unregistering file descriptors, and then polling them for I/O events; see section :ref:`poll-objects` below for the methods supported by polling objects. + .. availability:: Unix. + .. function:: kqueue() - (Only supported on BSD.) Returns a kernel queue object; see section + Returns a kernel queue object; see section :ref:`kqueue-objects` below for the methods supported by kqueue objects. The new file descriptor is :ref:`non-inheritable `. @@ -108,14 +114,18 @@ The module defines the following: .. versionchanged:: 3.4 The new file descriptor is now non-inheritable. + .. availability:: BSD, macOS. + .. function:: kevent(ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=0) - (Only supported on BSD.) Returns a kernel event object; see section + Returns a kernel event object; see section :ref:`kevent-objects` below for the methods supported by kevent objects. + .. availability:: BSD, macOS. + -.. function:: select(rlist, wlist, xlist[, timeout]) +.. function:: select(rlist, wlist, xlist, timeout=None) This is a straightforward interface to the Unix :c:func:`!select` system call. The first three arguments are iterables of 'waitable objects': either @@ -130,7 +140,8 @@ The module defines the following: Empty iterables are allowed, but acceptance of three empty iterables is platform-dependent. (It is known to work on Unix but not on Windows.) The optional *timeout* argument specifies a time-out as a floating-point number - in seconds. When the *timeout* argument is omitted the function blocks until + in seconds. + When the *timeout* argument is omitted or ``None``, the function blocks until at least one file descriptor is ready. A time-out value of zero specifies a poll and never blocks. @@ -170,7 +181,7 @@ The module defines the following: The minimum number of bytes which can be written without blocking to a pipe when the pipe has been reported as ready for writing by :func:`~select.select`, :func:`!poll` or another interface in this module. This doesn't apply - to other kind of file-like objects such as sockets. + to other kinds of file-like objects such as sockets. This value is guaranteed by POSIX to be at least 512. @@ -181,7 +192,7 @@ The module defines the following: .. _devpoll-objects: -``/dev/poll`` Polling Objects +``/dev/poll`` polling objects ----------------------------- Solaris and derivatives have ``/dev/poll``. While :c:func:`!select` is @@ -222,7 +233,7 @@ object. implement :meth:`!fileno`, so they can also be used as the argument. *eventmask* is an optional bitmask describing the type of events you want to - check for. The constants are the same that with :c:func:`!poll` + check for. The constants are the same as with :c:func:`!poll` object. The default value is a combination of the constants :const:`POLLIN`, :const:`POLLPRI`, and :const:`POLLOUT`. @@ -237,7 +248,7 @@ object. .. method:: devpoll.modify(fd[, eventmask]) This method does an :meth:`unregister` followed by a - :meth:`register`. It is (a bit) more efficient that doing the same + :meth:`register`. It is (a bit) more efficient than doing the same explicitly. @@ -273,52 +284,52 @@ object. .. _epoll-objects: -Edge and Level Trigger Polling (epoll) Objects +Edge and level trigger polling (epoll) objects ---------------------------------------------- https://linux.die.net/man/4/epoll - *eventmask* - - +-------------------------+-----------------------------------------------+ - | Constant | Meaning | - +=========================+===============================================+ - | :const:`EPOLLIN` | Available for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLOUT` | Available for write | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLPRI` | Urgent data for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLERR` | Error condition happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLHUP` | Hang up happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLET` | Set Edge Trigger behavior, the default is | - | | Level Trigger behavior | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLONESHOT` | Set one-shot behavior. After one event is | - | | pulled out, the fd is internally disabled | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | - | | associated fd has an event. The default (if | - | | this flag is not set) is to wake all epoll | - | | objects polling on a fd. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | - | | down writing half of connection. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDNORM` | Equivalent to :const:`EPOLLIN` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDBAND` | Priority data band can be read. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRNORM` | Equivalent to :const:`EPOLLOUT` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRBAND` | Priority data may be written. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLMSG` | Ignored. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWAKEUP` | Prevents sleep during event waiting. | - +-------------------------+-----------------------------------------------+ + The *eventmask* is a bit mask using the following constants: + + +-------------------------+------------------------------------------------+ + | Constant | Meaning | + +=========================+================================================+ + | :const:`EPOLLIN` | Available for read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLOUT` | Available for write. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLPRI` | Urgent data for read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLERR` | Error condition happened on the associated fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLHUP` | Hang up happened on the associated fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLET` | Set Edge Trigger behavior, the default is | + | | Level Trigger behavior. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLONESHOT` | Set one-shot behavior. After one event is | + | | pulled out, the fd is internally disabled. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | + | | associated fd has an event. The default (if | + | | this flag is not set) is to wake all epoll | + | | objects polling on an fd. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | + | | down writing half of connection. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDNORM` | Equivalent to :const:`EPOLLIN` | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLRDBAND` | Priority data band can be read. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWRNORM` | Equivalent to :const:`EPOLLOUT`. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWRBAND` | Priority data may be written. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLMSG` | Ignored. | + +-------------------------+------------------------------------------------+ + | :const:`EPOLLWAKEUP` | Prevents sleep during event waiting. | + +-------------------------+------------------------------------------------+ .. versionadded:: 3.6 :const:`EPOLLEXCLUSIVE` was added. It's only supported by Linux Kernel 4.5 @@ -350,12 +361,12 @@ Edge and Level Trigger Polling (epoll) Objects .. method:: epoll.register(fd[, eventmask]) - Register a fd descriptor with the epoll object. + Register a file descriptor *fd* with the epoll object. .. method:: epoll.modify(fd, eventmask) - Modify a registered file descriptor. + Modify a registered file descriptor *fd*. .. method:: epoll.unregister(fd) @@ -379,7 +390,7 @@ Edge and Level Trigger Polling (epoll) Objects .. _poll-objects: -Polling Objects +Polling objects --------------- The :c:func:`!poll` system call, supported on most Unix systems, provides better @@ -404,24 +415,24 @@ linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), w :const:`POLLPRI`, and :const:`POLLOUT`, described in the table below. If not specified, the default value used will check for all 3 types of events. - +-------------------+------------------------------------------+ - | Constant | Meaning | - +===================+==========================================+ - | :const:`POLLIN` | There is data to read | - +-------------------+------------------------------------------+ - | :const:`POLLPRI` | There is urgent data to read | - +-------------------+------------------------------------------+ - | :const:`POLLOUT` | Ready for output: writing will not block | - +-------------------+------------------------------------------+ - | :const:`POLLERR` | Error condition of some sort | - +-------------------+------------------------------------------+ - | :const:`POLLHUP` | Hung up | - +-------------------+------------------------------------------+ - | :const:`POLLRDHUP`| Stream socket peer closed connection, or | - | | shut down writing half of connection | - +-------------------+------------------------------------------+ - | :const:`POLLNVAL` | Invalid request: descriptor not open | - +-------------------+------------------------------------------+ + +-------------------+-------------------------------------------+ + | Constant | Meaning | + +===================+===========================================+ + | :const:`POLLIN` | There is data to read. | + +-------------------+-------------------------------------------+ + | :const:`POLLPRI` | There is urgent data to read. | + +-------------------+-------------------------------------------+ + | :const:`POLLOUT` | Ready for output: writing will not block. | + +-------------------+-------------------------------------------+ + | :const:`POLLERR` | Error condition of some sort. | + +-------------------+-------------------------------------------+ + | :const:`POLLHUP` | Hung up. | + +-------------------+-------------------------------------------+ + | :const:`POLLRDHUP`| Stream socket peer closed connection, or | + | | shut down writing half of connection. | + +-------------------+-------------------------------------------+ + | :const:`POLLNVAL` | Invalid request: descriptor not open. | + +-------------------+-------------------------------------------+ Registering a file descriptor that's already registered is not an error, and has the same effect as registering the descriptor exactly once. @@ -467,7 +478,7 @@ linearly scanned again. :c:func:`!select` is *O*\ (*highest file descriptor*), w .. _kqueue-objects: -Kqueue Objects +Kqueue objects -------------- .. method:: kqueue.close() @@ -508,7 +519,7 @@ Kqueue Objects .. _kevent-objects: -Kevent Objects +Kevent objects -------------- https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 @@ -528,66 +539,66 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 | Constant | Meaning | +===========================+=============================================+ | :const:`KQ_FILTER_READ` | Takes a descriptor and returns whenever | - | | there is data available to read | + | | there is data available to read. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_WRITE` | Takes a descriptor and returns whenever | - | | there is data available to write | + | | there is data available to write. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_AIO` | AIO requests | + | :const:`KQ_FILTER_AIO` | AIO requests. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_VNODE` | Returns when one or more of the requested | - | | events watched in *fflag* occurs | + | | events watched in *fflag* occurs. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_PROC` | Watch for events on a process id | + | :const:`KQ_FILTER_PROC` | Watch for events on a process ID. | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_NETDEV` | Watch for events on a network device | - | | [not available on macOS] | + | | (not available on macOS). | +---------------------------+---------------------------------------------+ | :const:`KQ_FILTER_SIGNAL` | Returns whenever the watched signal is | - | | delivered to the process | + | | delivered to the process. | +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_TIMER` | Establishes an arbitrary timer | + | :const:`KQ_FILTER_TIMER` | Establishes an arbitrary timer. | +---------------------------+---------------------------------------------+ .. attribute:: kevent.flags Filter action. - +---------------------------+---------------------------------------------+ - | Constant | Meaning | - +===========================+=============================================+ - | :const:`KQ_EV_ADD` | Adds or modifies an event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DELETE` | Removes an event from the queue | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ENABLE` | Permitscontrol() to returns the event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DISABLE` | Disablesevent | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_CLEAR` | Reset the state after an event is retrieved | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_SYSFLAGS` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_FLAG1` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_EOF` | Filter specific EOF condition | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ERROR` | See return values | - +---------------------------+---------------------------------------------+ + +---------------------------+----------------------------------------------+ + | Constant | Meaning | + +===========================+==============================================+ + | :const:`KQ_EV_ADD` | Adds or modifies an event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_DELETE` | Removes an event from the queue. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ENABLE` | Permits control() to return the event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_DISABLE` | Disables event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_CLEAR` | Reset the state after an event is retrieved. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_SYSFLAGS` | Internal event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_FLAG1` | Internal event. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_EOF` | Filter-specific EOF condition. | + +---------------------------+----------------------------------------------+ + | :const:`KQ_EV_ERROR` | See return values. | + +---------------------------+----------------------------------------------+ .. attribute:: kevent.fflags - Filter specific flags. + Filter-specific flags. :const:`KQ_FILTER_READ` and :const:`KQ_FILTER_WRITE` filter flags: +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_LOWAT` | low water mark of a socket buffer | + | :const:`KQ_NOTE_LOWAT` | Low water mark of a socket buffer. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_VNODE` filter flags: @@ -595,19 +606,19 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_DELETE` | *unlink()* was called | + | :const:`KQ_NOTE_DELETE` | *unlink()* was called. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_WRITE` | a write occurred | + | :const:`KQ_NOTE_WRITE` | A write occurred. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXTEND` | the file was extended | + | :const:`KQ_NOTE_EXTEND` | The file was extended. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_ATTRIB` | an attribute was changed | + | :const:`KQ_NOTE_ATTRIB` | An attribute was changed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINK` | the link count has changed | + | :const:`KQ_NOTE_LINK` | The link count has changed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_RENAME` | the file was renamed | + | :const:`KQ_NOTE_RENAME` | The file was renamed. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_REVOKE` | access to the file was revoked | + | :const:`KQ_NOTE_REVOKE` | Access to the file was revoked. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_PROC` filter flags: @@ -615,22 +626,22 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_EXIT` | the process has exited | + | :const:`KQ_NOTE_EXIT` | The process has exited. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_FORK` | the process has called *fork()* | + | :const:`KQ_NOTE_FORK` | The process has called *fork()*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXEC` | the process has executed a new process | + | :const:`KQ_NOTE_EXEC` | The process has executed a new process. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PCTRLMASK` | internal filter flag | + | :const:`KQ_NOTE_PCTRLMASK` | Internal filter flag. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PDATAMASK` | internal filter flag | + | :const:`KQ_NOTE_PDATAMASK` | Internal filter flag. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACK` | follow a process across *fork()* | + | :const:`KQ_NOTE_TRACK` | Follow a process across *fork()*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_CHILD` | returned on the child process for | - | | *NOTE_TRACK* | + | :const:`KQ_NOTE_CHILD` | Returned on the child process for | + | | *NOTE_TRACK*. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACKERR` | unable to attach to a child | + | :const:`KQ_NOTE_TRACKERR` | Unable to attach to a child. | +----------------------------+--------------------------------------------+ :const:`KQ_FILTER_NETDEV` filter flags (not available on macOS): @@ -638,19 +649,19 @@ https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 +----------------------------+--------------------------------------------+ | Constant | Meaning | +============================+============================================+ - | :const:`KQ_NOTE_LINKUP` | link is up | + | :const:`KQ_NOTE_LINKUP` | Link is up. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKDOWN` | link is down | + | :const:`KQ_NOTE_LINKDOWN` | Link is down. | +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKINV` | link state is invalid | + | :const:`KQ_NOTE_LINKINV` | Link state is invalid. | +----------------------------+--------------------------------------------+ .. attribute:: kevent.data - Filter specific data. + Filter-specific data. .. attribute:: kevent.udata - User defined value. + User-defined value. diff --git a/Doc/library/selectors.rst b/Doc/library/selectors.rst index ee556f1f3cecae6..2d523a9d2ea440b 100644 --- a/Doc/library/selectors.rst +++ b/Doc/library/selectors.rst @@ -54,7 +54,7 @@ Classes hierarchy:: In the following, *events* is a bitwise mask indicating which I/O events should -be waited for on a given file object. It can be a combination of the modules +be waited for on a given file object. It can be a combination of the module's constants below: +-----------------------+-----------------------------------------------+ diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst index 6e74a59b82b8ec6..a2c9065ae788a33 100644 --- a/Doc/library/shelve.rst +++ b/Doc/library/shelve.rst @@ -61,11 +61,11 @@ lots of shared sub-objects. The keys are ordinary strings. .. warning:: - Because the :mod:`shelve` module is backed by :mod:`pickle`, it is insecure + Because the :mod:`!shelve` module is backed by :mod:`pickle`, it is insecure to load a shelf from an untrusted source. Like with pickle, loading a shelf can execute arbitrary code. -Shelf objects support most of methods and operations supported by dictionaries +Shelf objects support most of the methods and operations supported by dictionaries (except copying, constructors and operators ``|`` and ``|=``). This eases the transition from dictionary based scripts to those requiring persistent storage. @@ -106,7 +106,7 @@ Restrictions database should be fairly small, and in rare cases key collisions may cause the database to refuse updates. -* The :mod:`shelve` module does not support *concurrent* read/write access to +* The :mod:`!shelve` module does not support *concurrent* read/write access to shelved objects. (Multiple simultaneous read accesses are safe.) When a program has a shelf open for writing, no other program should have it open for reading or writing. Unix file locking can be used to solve this, but this @@ -219,5 +219,5 @@ object):: Generic interface to ``dbm``-style databases. Module :mod:`pickle` - Object serialization used by :mod:`shelve`. + Object serialization used by :mod:`!shelve`. diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst index a96f0864dc12604..0653bf2f4189c20 100644 --- a/Doc/library/shlex.rst +++ b/Doc/library/shlex.rst @@ -18,7 +18,7 @@ simple syntaxes resembling that of the Unix shell. This will often be useful for writing minilanguages, (for example, in run control files for Python applications) or for parsing quoted strings. -The :mod:`shlex` module defines the following functions: +The :mod:`!shlex` module defines the following functions: .. function:: split(s, comments=False, posix=True) @@ -98,7 +98,7 @@ The :mod:`shlex` module defines the following functions: .. versionadded:: 3.3 -The :mod:`shlex` module defines the following class: +The :mod:`!shlex` module defines the following class: .. class:: shlex(instream=None, infile=None, posix=False, punctuation_chars=False) @@ -214,7 +214,7 @@ A :class:`~shlex.shlex` instance has the following methods: with the name of the current source file and the ``%d`` with the current input line number (the optional arguments can be used to override these). - This convenience is provided to encourage :mod:`shlex` users to generate error + This convenience is provided to encourage :mod:`!shlex` users to generate error messages in the standard, parseable format understood by Emacs and other Unix tools. @@ -343,7 +343,7 @@ variables which either control lexical analysis or can be used for debugging: Parsing Rules ------------- -When operating in non-POSIX mode, :class:`~shlex.shlex` will try to obey to the +When operating in non-POSIX mode, :class:`~shlex.shlex` will try to obey the following rules. * Quote characters are not recognized within words (``Do"Not"Separate`` is @@ -366,7 +366,7 @@ following rules. * It's not possible to parse empty strings, even if quoted. -When operating in POSIX mode, :class:`~shlex.shlex` will try to obey to the +When operating in POSIX mode, :class:`~shlex.shlex` will try to obey the following parsing rules. * Quotes are stripped out, and do not separate words (``"Do"Not"Separate"`` is @@ -382,7 +382,7 @@ following parsing rules. * Enclosing characters in quotes which are part of :attr:`~shlex.escapedquotes` (e.g. ``'"'``) preserves the literal value of all characters within the quotes, with the exception of the characters - mentioned in :attr:`~shlex.escape`. The escape characters retain its + mentioned in :attr:`~shlex.escape`. The escape characters retain their special meaning only when followed by the quote in use, or the escape character itself. Otherwise the escape character will be considered a normal character. diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 2cbf95bcf535e4b..d802a2572871eaf 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -15,7 +15,7 @@ -------------- -The :mod:`shutil` module offers a number of high-level operations on files and +The :mod:`!shutil` module offers a number of high-level operations on files and collections of files. In particular, functions are provided which support file copying and removal. For operations on individual files, see also the :mod:`os` module. @@ -47,6 +47,13 @@ Directory and files operations 0, only the contents from the current file position to the end of the file will be copied. + :func:`copyfileobj` will *not* guarantee that the destination stream has + been flushed on completion of the copy. If you want to read from the + destination at the completion of the copy operation (for example, reading + the contents of a temporary file that has been copied from a HTTP stream), + you must ensure that you have called :func:`~io.IOBase.flush` or + :func:`~io.IOBase.close` on the file-like object before attempting to read + the destination file. .. function:: copyfile(src, dst, *, follow_symlinks=True) @@ -83,6 +90,13 @@ Directory and files operations copy the file more efficiently. See :ref:`shutil-platform-dependent-efficient-copy-operations` section. +.. exception:: SpecialFileError + + This exception is raised when :func:`copyfile` or :func:`copytree` attempt + to copy a named pipe. + + .. versionadded:: 2.7 + .. exception:: SameFileError This exception is raised if source and destination in :func:`copyfile` @@ -327,6 +341,10 @@ Directory and files operations The deprecated *onerror* is similar to *onexc*, except that the third parameter it receives is the tuple returned from :func:`sys.exc_info`. + .. seealso:: + :ref:`shutil-rmtree-example` for an example of handling the removal + of a directory tree that contains read-only files. + .. audit-event:: shutil.rmtree path,dir_fd shutil.rmtree .. versionchanged:: 3.3 @@ -370,10 +388,14 @@ Directory and files operations If *dst* already exists but is not a directory, it may be overwritten depending on :func:`os.rename` semantics. - If the destination is on the current filesystem, then :func:`os.rename` is - used. Otherwise, *src* is copied to the destination using *copy_function* - and then removed. In case of symlinks, a new symlink pointing to the target - of *src* will be created as the destination and *src* will be removed. + :func:`os.rename` is preferably used internally when *src* and the destination are on + the same filesystem. In case :func:`os.rename` fails due to :exc:`OSError` + (e.g. the user has write permission to the destination file but not to its parent + directory), this method falls back to using *copy_function*, in which case + *src* is copied to the destination using *copy_function* and then removed. + + In case of symlinks, a new symlink pointing to the target of *src* will be + created in or as the destination, and *src* will be removed. If *copy_function* is given, it must be a callable that takes two arguments, *src* and the destination, and will be used to copy *src* to the destination @@ -454,6 +476,10 @@ Directory and files operations :envvar:`PATH` environment variable is read from :data:`os.environ`, falling back to :data:`os.defpath` if it is not set. + If *cmd* contains a directory component, :func:`!which` only checks the + specified path directly and does not search the directories listed in + *path* or in the system's :envvar:`PATH` environment variable. + On Windows, the current directory is prepended to the *path* if *mode* does not include ``os.X_OK``. When the *mode* does include ``os.X_OK``, the Windows API ``NeedCurrentDirectoryForExePathW`` will be consulted to @@ -521,7 +547,7 @@ instead of 64 KiB) and a :func:`memoryview`-based variant of :func:`shutil.copyfileobj` is used. If the fast-copy operation fails and no data was written in the destination -file then shutil will silently fallback on using less efficient +file then shutil will silently fall back to less efficient :func:`copyfileobj` function internally. .. versionchanged:: 3.8 @@ -603,7 +629,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. *format* is the archive format: one of "zip" (if the :mod:`zlib` module is available), "tar", "gztar" (if the :mod:`zlib` module is available), "bztar" (if the :mod:`bz2` module is - available), or "xztar" (if the :mod:`lzma` module is available). + available), "xztar" (if the :mod:`lzma` module is available), or "zstdtar" + (if the :mod:`compression.zstd` module is available). *root_dir* is a directory that will be the root directory of the archive, all paths in the archive will be relative to it; for example, @@ -651,13 +678,15 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Return a list of supported formats for archiving. Each element of the returned sequence is a tuple ``(name, description)``. - By default :mod:`shutil` provides these formats: + By default :mod:`!shutil` provides these formats: - *zip*: ZIP file (if the :mod:`zlib` module is available). - *tar*: Uncompressed tar file. Uses POSIX.1-2001 pax format for new archives. - *gztar*: gzip'ed tar-file (if the :mod:`zlib` module is available). - *bztar*: bzip2'ed tar-file (if the :mod:`bz2` module is available). - *xztar*: xz'ed tar-file (if the :mod:`lzma` module is available). + - *zstdtar*: Zstandard compressed tar-file (if the :mod:`compression.zstd` + module is available). You can register new formats or provide your own archiver for any existing formats, by using :func:`register_archive_format`. @@ -667,7 +696,7 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Register an archiver for the format *name*. - *function* is the callable that will be used to unpack archives. The callable + *function* is the callable that will be used to create archives. The callable will receive the *base_name* of the file to create, followed by the *base_dir* (which defaults to :data:`os.curdir`) to start archiving from. Further arguments are passed as keyword arguments: *owner*, *group*, @@ -701,8 +730,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. *extract_dir* is the name of the target directory where the archive is unpacked. If not provided, the current working directory is used. - *format* is the archive format: one of "zip", "tar", "gztar", "bztar", or - "xztar". Or any other format registered with + *format* is the archive format: one of "zip", "tar", "gztar", "bztar", + "xztar", or "zstdtar". Or any other format registered with :func:`register_unpack_format`. If not provided, :func:`unpack_archive` will use the archive file name extension and see if an unpacker was registered for that extension. In case none is found, @@ -720,8 +749,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Never extract archives from untrusted sources without prior inspection. It is possible that files are created outside of the path specified in - the *extract_dir* argument, e.g. members that have absolute filenames - starting with "/" or filenames with two dots "..". + the *extract_dir* argument, for example, members that have absolute filenames + or filenames with ".." components. Since Python 3.14, the defaults for both built-in formats (zip and tar files) will prevent the most dangerous of such security issues, @@ -766,7 +795,7 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. Each element of the returned sequence is a tuple ``(name, extensions, description)``. - By default :mod:`shutil` provides these formats: + By default :mod:`!shutil` provides these formats: - *zip*: ZIP file (unpacking compressed files works only if the corresponding module is available). @@ -774,6 +803,8 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. - *gztar*: gzip'ed tar-file (if the :mod:`zlib` module is available). - *bztar*: bzip2'ed tar-file (if the :mod:`bz2` module is available). - *xztar*: xz'ed tar-file (if the :mod:`lzma` module is available). + - *zstdtar*: Zstandard compressed tar-file (if the :mod:`compression.zstd` + module is available). You can register new formats or provide your own unpacker for any existing formats, by using :func:`register_unpack_format`. @@ -840,7 +871,7 @@ In the final archive, :file:`please_add.txt` should be included, but ... root_dir='tmp/root', ... base_dir='structure/content', ... ) - '/Users/tarek/my_archive.tar' + '/Users/tarek/myarchive.tar' Listing the files in the resulting archive gives us: diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index c28841dbb8cfc89..8e543d9205ec6aa 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -36,7 +36,7 @@ Execution of Python signal handlers A Python signal handler does not get executed inside the low-level (C) signal handler. Instead, the low-level signal handler sets a flag which tells the :term:`virtual machine` to execute the corresponding Python signal handler -at a later point(for example at the next :term:`bytecode` instruction). +at a later point (for example, at the next :term:`bytecode` instruction). This has consequences: * It makes little sense to catch synchronous errors like :const:`SIGFPE` or @@ -68,6 +68,11 @@ the synchronization primitives from the :mod:`threading` module instead. Besides, only the main thread of the main interpreter is allowed to set a new signal handler. +.. warning:: + + Synchronization primitives such as :class:`threading.Lock` should not be used + within signal handlers. Doing so can lead to unexpected deadlocks. + Module contents --------------- @@ -92,13 +97,13 @@ The signal module defines three enums: .. class:: Handlers - :class:`enum.IntEnum` collection the constants :const:`SIG_DFL` and :const:`SIG_IGN`. + :class:`enum.IntEnum` collection of the constants :const:`SIG_DFL` and :const:`SIG_IGN`. .. versionadded:: 3.5 .. class:: Sigmasks - :class:`enum.IntEnum` collection the constants :const:`SIG_BLOCK`, :const:`SIG_UNBLOCK` and :const:`SIG_SETMASK`. + :class:`enum.IntEnum` collection of the constants :const:`SIG_BLOCK`, :const:`SIG_UNBLOCK` and :const:`SIG_SETMASK`. .. availability:: Unix. @@ -108,7 +113,7 @@ The signal module defines three enums: .. versionadded:: 3.5 -The variables defined in the :mod:`signal` module are: +The variables defined in the :mod:`!signal` module are: .. data:: SIG_DFL @@ -205,14 +210,32 @@ The variables defined in the :mod:`signal` module are: .. availability:: Unix. +.. data:: SIGPROF + + Profiling timer expired. + + .. availability:: Unix. + +.. data:: SIGQUIT + + Terminal quit signal. + + .. availability:: Unix. + .. data:: SIGSEGV Segmentation fault: invalid memory reference. +.. data:: SIGSTOP + + Stop executing (cannot be caught or ignored). + + .. availability:: Unix. + .. data:: SIGSTKFLT - Stack fault on coprocessor. The Linux kernel does not raise this signal: it - can only be raised in user space. + Stack fault on coprocessor. The Linux kernel does not raise this signal: it + can only be raised in user space. .. availability:: Linux. @@ -237,18 +260,30 @@ The variables defined in the :mod:`signal` module are: .. availability:: Unix. +.. data:: SIGVTALRM + + Virtual timer expired. + + .. availability:: Unix. + .. data:: SIGWINCH Window resize signal. .. availability:: Unix. +.. data:: SIGXCPU + + CPU time limit exceeded. + + .. availability:: Unix. + .. data:: SIG* All the signal numbers are defined symbolically. For example, the hangup signal is defined as :const:`signal.SIGHUP`; the variable names are identical to the names used in C programs, as found in ````. The Unix man page for - ':c:func:`signal`' lists the existing signals (on some systems this is + '``signal``' lists the existing signals (on some systems this is :manpage:`signal(2)`, on others the list is in :manpage:`signal(7)`). Note that not all systems define the same set of signal names; only those names defined by the system are defined by this module. @@ -322,7 +357,7 @@ The variables defined in the :mod:`signal` module are: .. versionadded:: 3.3 -The :mod:`signal` module defines one exception: +The :mod:`!signal` module defines one exception: .. exception:: ItimerError @@ -336,7 +371,7 @@ The :mod:`signal` module defines one exception: alias of :exc:`OSError`. -The :mod:`signal` module defines the following functions: +The :mod:`!signal` module defines the following functions: .. function:: alarm(time) @@ -639,9 +674,8 @@ The :mod:`signal` module defines the following functions: *sigset*. The return value is an object representing the data contained in the - :c:type:`siginfo_t` structure, namely: :attr:`si_signo`, :attr:`si_code`, - :attr:`si_errno`, :attr:`si_pid`, :attr:`si_uid`, :attr:`si_status`, - :attr:`si_band`. + ``siginfo_t`` structure, namely: ``si_signo``, ``si_code``, + ``si_errno``, ``si_pid``, ``si_uid``, ``si_status``, ``si_band``. .. availability:: Unix. diff --git a/Doc/library/site.rst b/Doc/library/site.rst index e98dd83b60eb600..4686c9fc92ced29 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -34,7 +34,7 @@ For the head part, it uses ``sys.prefix`` and ``sys.exec_prefix``; empty heads are skipped. For the tail part, it uses the empty string and then :file:`lib/site-packages` (on Windows) or :file:`lib/python{X.Y[t]}/site-packages` (on Unix and macOS). (The -optional suffix "t" indicates the :term:`free threading` build, and is +optional suffix "t" indicates the :term:`free-threaded build`, and is appended if ``"t"`` is present in the :data:`sys.abiflags` constant.) For each of the distinct head-tail combinations, it sees if it refers to an existing @@ -51,11 +51,11 @@ added path for configuration files. .. versionchanged:: 3.14 - :mod:`site` is no longer responsible for updating :data:`sys.prefix` and + :mod:`!site` is no longer responsible for updating :data:`sys.prefix` and :data:`sys.exec_prefix` on :ref:`sys-path-init-virtual-environments`. This is now done during the :ref:`path initialization `. As a result, under :ref:`sys-path-init-virtual-environments`, :data:`sys.prefix` and - :data:`sys.exec_prefix` no longer depend on the :mod:`site` initialization, + :data:`sys.exec_prefix` no longer depend on the :mod:`!site` initialization, and are therefore unaffected by :option:`-S`. .. _site-virtual-environments-configuration: @@ -130,38 +130,38 @@ directory precedes the :file:`foo` directory because :file:`bar.pth` comes alphabetically before :file:`foo.pth`; and :file:`spam` is omitted because it is not mentioned in either path configuration file. -:mod:`sitecustomize` --------------------- +:mod:`!sitecustomize` +--------------------- .. module:: sitecustomize After these path manipulations, an attempt is made to import a module named -:mod:`sitecustomize`, which can perform arbitrary site-specific customizations. +:mod:`!sitecustomize`, which can perform arbitrary site-specific customizations. It is typically created by a system administrator in the site-packages directory. If this import fails with an :exc:`ImportError` or its subclass exception, and the exception's :attr:`~ImportError.name` -attribute equals to ``'sitecustomize'``, +attribute equals ``'sitecustomize'``, it is silently ignored. If Python is started without output streams available, as with :file:`pythonw.exe` on Windows (which is used by default to start IDLE), -attempted output from :mod:`sitecustomize` is ignored. Any other exception +attempted output from :mod:`!sitecustomize` is ignored. Any other exception causes a silent and perhaps mysterious failure of the process. -:mod:`usercustomize` --------------------- +:mod:`!usercustomize` +--------------------- .. module:: usercustomize -After this, an attempt is made to import a module named :mod:`usercustomize`, +After this, an attempt is made to import a module named :mod:`!usercustomize`, which can perform arbitrary user-specific customizations, if :data:`~site.ENABLE_USER_SITE` is true. This file is intended to be created in the user site-packages directory (see below), which is part of ``sys.path`` unless disabled by :option:`-s`. If this import fails with an :exc:`ImportError` or its subclass exception, and the exception's :attr:`~ImportError.name` -attribute equals to ``'usercustomize'``, it is silently ignored. +attribute equals ``'usercustomize'``, it is silently ignored. Note that for some non-Unix systems, ``sys.prefix`` and ``sys.exec_prefix`` are empty, and the path manipulations are skipped; however the import of -:mod:`sitecustomize` and :mod:`usercustomize` is still attempted. +:mod:`sitecustomize` and :mod:`!usercustomize` is still attempted. .. currentmodule:: site @@ -173,7 +173,7 @@ Readline configuration On systems that support :mod:`readline`, this module will also import and configure the :mod:`rlcompleter` module, if Python is started in :ref:`interactive mode ` and without the :option:`-S` option. -The default behavior is enable tab-completion and to use +The default behavior is to enable tab completion and to use :file:`~/.python_history` as the history save file. To disable it, delete (or override) the :data:`sys.__interactivehook__` attribute in your :mod:`sitecustomize` or :mod:`usercustomize` module or your @@ -270,12 +270,12 @@ Module contents .. _site-commandline: -Command Line Interface +Command-line interface ---------------------- .. program:: site -The :mod:`site` module also provides a way to get the user directories from the +The :mod:`!site` module also provides a way to get the user directories from the command line: .. code-block:: shell-session diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index c5f8516f768a68b..583bc3afcad0407 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -14,7 +14,7 @@ -------------- -The :mod:`smtplib` module defines an SMTP client session object that can be used +The :mod:`!smtplib` module defines an SMTP client session object that can be used to send mail to any internet machine with an SMTP or ESMTP listener daemon. For details of SMTP and ESMTP operation, consult :rfc:`821` (Simple Mail Transfer Protocol) and :rfc:`1869` (SMTP Service Extensions). @@ -80,8 +80,8 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). An :class:`SMTP_SSL` instance behaves exactly the same as instances of :class:`SMTP`. :class:`SMTP_SSL` should be used for situations where SSL is - required from the beginning of the connection and using :meth:`starttls` is - not appropriate. If *host* is not specified, the local host is used. If + required from the beginning of the connection and using :meth:`~SMTP.starttls` + is not appropriate. If *host* is not specified, the local host is used. If *port* is zero, the standard SMTP-over-SSL port (465) is used. The optional arguments *local_hostname*, *timeout* and *source_address* have the same meaning as they do in the :class:`SMTP` class. *context*, also optional, @@ -112,7 +112,7 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). The LMTP protocol, which is very similar to ESMTP, is heavily based on the standard SMTP client. It's common to use Unix sockets for LMTP, so our - :meth:`connect` method must support that as well as a regular host:port + :meth:`~SMTP.connect` method must support that as well as a regular host:port server. The optional arguments *local_hostname* and *source_address* have the same meaning as they do in the :class:`SMTP` class. To specify a Unix socket, you must use an absolute path for *host*, starting with a '/'. @@ -147,9 +147,15 @@ A nice selection of exceptions is defined as well: .. exception:: SMTPResponseException Base class for all exceptions that include an SMTP error code. These exceptions - are generated in some instances when the SMTP server returns an error code. The - error code is stored in the :attr:`smtp_code` attribute of the error, and the - :attr:`smtp_error` attribute is set to the error message. + are generated in some instances when the SMTP server returns an error code. + + .. attribute:: smtp_code + + The error code. + + .. attribute:: smtp_error + + The error message. .. exception:: SMTPSenderRefused @@ -161,9 +167,13 @@ A nice selection of exceptions is defined as well: .. exception:: SMTPRecipientsRefused - All recipient addresses refused. The errors for each recipient are accessible - through the attribute :attr:`recipients`, which is a dictionary of exactly the - same sort as :meth:`SMTP.sendmail` returns. + All recipient addresses refused. + + .. attribute:: recipients + + A dictionary of exactly the same sort as returned + by :meth:`SMTP.sendmail` containing the errors for + each recipient. .. exception:: SMTPDataError @@ -213,7 +223,6 @@ SMTP Objects An :class:`SMTP` instance has the following methods: - .. method:: SMTP.set_debuglevel(level) Set the debug output level. A value of 1 or ``True`` for *level* results in @@ -327,7 +336,7 @@ An :class:`SMTP` instance has the following methods: :exc:`SMTPException` No suitable authentication method was found. - Each of the authentication methods supported by :mod:`smtplib` are tried in + Each of the authentication methods supported by :mod:`!smtplib` are tried in turn if they are advertised as supported by the server. See :meth:`auth` for a list of supported authentication methods. *initial_response_ok* is passed through to :meth:`auth`. @@ -379,7 +388,7 @@ An :class:`SMTP` instance has the following methods: call the :meth:`login` method, which will try each of the above mechanisms in turn, in the order listed. ``auth`` is exposed to facilitate the implementation of authentication methods not (or not yet) supported - directly by :mod:`smtplib`. + directly by :mod:`!smtplib`. .. versionadded:: 3.5 @@ -417,7 +426,7 @@ An :class:`SMTP` instance has the following methods: .. versionchanged:: 3.4 The method now supports hostname check with - :attr:`SSLContext.check_hostname` and *Server Name Indicator* (see + :attr:`ssl.SSLContext.check_hostname` and *Server Name Indicator* (see :const:`~ssl.HAS_SNI`). .. versionchanged:: 3.5 @@ -431,11 +440,13 @@ An :class:`SMTP` instance has the following methods: Send mail. The required arguments are an :rfc:`822` from-address string, a list of :rfc:`822` to-address strings (a bare string will be treated as a list with 1 address), and a message string. The caller may pass a list of ESMTP options - (such as ``8bitmime``) to be used in ``MAIL FROM`` commands as *mail_options*. + (such as ``"8bitmime"``) to be used in ``MAIL FROM`` commands as *mail_options*. ESMTP options (such as ``DSN`` commands) that should be used with all ``RCPT`` - commands can be passed as *rcpt_options*. (If you need to use different ESMTP + commands can be passed as *rcpt_options*. Each option should be passed as a string + containing the full text of the option, including any potential key + (for instance, ``"NOTIFY=SUCCESS,FAILURE"``). (If you need to use different ESMTP options to different recipients you have to use the low-level methods such as - :meth:`mail`, :meth:`rcpt` and :meth:`data` to send the message.) + :meth:`!mail`, :meth:`!rcpt` and :meth:`!data` to send the message.) .. note:: @@ -467,10 +478,7 @@ An :class:`SMTP` instance has the following methods: This method may raise the following exceptions: :exc:`SMTPRecipientsRefused` - All recipients were refused. Nobody got the mail. The :attr:`recipients` - attribute of the exception object is a dictionary with information about the - refused recipients (like the one returned when at least one recipient was - accepted). + All recipients were refused. Nobody got the mail. :exc:`SMTPHeloError` The server didn't reply properly to the ``HELO`` greeting. @@ -546,6 +554,30 @@ Low-level methods corresponding to the standard SMTP/ESMTP commands ``HELP``, Normally these do not need to be called directly, so they are not documented here. For details, consult the module code. +Additionally, an SMTP instance has the following attributes: + + +.. attribute:: SMTP.helo_resp + + The response to the ``HELO`` command, see :meth:`helo`. + + +.. attribute:: SMTP.ehlo_resp + + The response to the ``EHLO`` command, see :meth:`ehlo`. + + +.. attribute:: SMTP.does_esmtp + + A boolean value indicating whether the server supports ESMTP, see + :meth:`ehlo`. + + +.. attribute:: SMTP.esmtp_features + + A dictionary of the names of SMTP service extensions supported by the server, + see :meth:`ehlo`. + .. _smtp-example: diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 8fd5187e3a4a361..71f5fb6ab0cc750 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -39,6 +39,8 @@ is implicit on send operations. A TLS/SSL wrapper for socket objects. +.. _socket-addresses: + Socket families --------------- @@ -83,7 +85,7 @@ created. Socket addresses are represented as follows: - For :const:`AF_INET6` address family, a four-tuple ``(host, port, flowinfo, scope_id)`` is used, where *flowinfo* and *scope_id* represent the ``sin6_flowinfo`` and ``sin6_scope_id`` members in :const:`struct sockaddr_in6` in C. For - :mod:`socket` module methods, *flowinfo* and *scope_id* can be omitted just for + :mod:`!socket` module methods, *flowinfo* and *scope_id* can be omitted just for backward compatibility. Note, however, omission of *scope_id* can cause problems in manipulating scoped IPv6 addresses. @@ -118,10 +120,10 @@ created. Socket addresses are represented as follows: ``'can0'``. The network interface name ``''`` can be used to receive packets from all network interfaces of this family. - - :const:`CAN_ISOTP` protocol require a tuple ``(interface, rx_addr, tx_addr)`` + - :const:`CAN_ISOTP` protocol requires a tuple ``(interface, rx_addr, tx_addr)`` where both additional parameters are unsigned long integer that represent a CAN identifier (standard or extended). - - :const:`CAN_J1939` protocol require a tuple ``(interface, name, pgn, addr)`` + - :const:`CAN_J1939` protocol requires a tuple ``(interface, name, pgn, addr)`` where additional parameters are 64-bit unsigned integer representing the ECU name, a 32-bit unsigned integer representing the Parameter Group Number (PGN), and an 8-bit integer representing the address. @@ -302,7 +304,7 @@ generalization of this based on timeouts is supported through Module contents --------------- -The module :mod:`socket` exports the following elements. +The module :mod:`!socket` exports the following elements. Exceptions @@ -362,10 +364,10 @@ Exceptions Constants ^^^^^^^^^ - The AF_* and SOCK_* constants are now :class:`AddressFamily` and - :class:`SocketKind` :class:`.IntEnum` collections. +The AF_* and SOCK_* constants are now :class:`AddressFamily` and +:class:`SocketKind` :class:`.IntEnum` collections. - .. versionadded:: 3.4 +.. versionadded:: 3.4 .. data:: AF_UNIX AF_INET @@ -773,9 +775,9 @@ Constants Constant to optimize CPU locality, to be used in conjunction with :data:`SO_REUSEPORT`. - .. versionadded:: 3.11 + .. versionadded:: 3.11 - .. availability:: Linux >= 3.9 + .. availability:: Linux >= 3.9 .. data:: SO_REUSEPORT_LB @@ -836,71 +838,14 @@ Creating sockets The following functions all create :ref:`socket objects `. -.. class:: socket(family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None) - - Create a new socket using the given address family, socket type and protocol - number. The address family should be :const:`AF_INET` (the default), - :const:`AF_INET6`, :const:`AF_UNIX`, :const:`AF_CAN`, :const:`AF_PACKET`, - or :const:`AF_RDS`. The socket type should be :const:`SOCK_STREAM` (the - default), :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other - ``SOCK_`` constants. The protocol number is usually zero and may be omitted - or in the case where the address family is :const:`AF_CAN` the protocol - should be one of :const:`CAN_RAW`, :const:`CAN_BCM`, :const:`CAN_ISOTP` or - :const:`CAN_J1939`. - - If *fileno* is specified, the values for *family*, *type*, and *proto* are - auto-detected from the specified file descriptor. Auto-detection can be - overruled by calling the function with explicit *family*, *type*, or *proto* - arguments. This only affects how Python represents e.g. the return value - of :meth:`socket.getpeername` but not the actual OS resource. Unlike - :func:`socket.fromfd`, *fileno* will return the same socket and not a - duplicate. This may help close a detached socket using - :meth:`socket.close`. - - The newly created socket is :ref:`non-inheritable `. - - .. audit-event:: socket.__new__ self,family,type,protocol socket.socket - - .. versionchanged:: 3.3 - The AF_CAN family was added. - The AF_RDS family was added. - - .. versionchanged:: 3.4 - The CAN_BCM protocol was added. - - .. versionchanged:: 3.4 - The returned socket is now non-inheritable. - - .. versionchanged:: 3.7 - The CAN_ISOTP protocol was added. - - .. versionchanged:: 3.7 - When :const:`SOCK_NONBLOCK` or :const:`SOCK_CLOEXEC` - bit flags are applied to *type* they are cleared, and - :attr:`socket.type` will not reflect them. They are still passed - to the underlying system ``socket()`` call. Therefore, - - :: - - sock = socket.socket( - socket.AF_INET, - socket.SOCK_STREAM | socket.SOCK_NONBLOCK) - - will still create a non-blocking socket on OSes that support - ``SOCK_NONBLOCK``, but ``sock.type`` will be set to - ``socket.SOCK_STREAM``. - - .. versionchanged:: 3.9 - The CAN_J1939 protocol was added. - - .. versionchanged:: 3.10 - The IPPROTO_MPTCP protocol was added. +The :class:`socket ` class constructor creates a new socket +directly; see :ref:`socket-objects` for its parameters and full description. .. function:: socketpair([family[, type[, proto]]]) Build a pair of connected socket objects using the given address family, socket type, and protocol number. Address family, socket type, and protocol number are - as for the :func:`~socket.socket` function above. The default family is :const:`AF_UNIX` + as for the :func:`~socket.socket` function. The default family is :const:`AF_UNIX` if defined on the platform; otherwise, the default is :const:`AF_INET`. The newly created sockets are :ref:`non-inheritable `. @@ -996,8 +941,8 @@ The following functions all create :ref:`socket objects `. Duplicate the file descriptor *fd* (an integer as returned by a file object's :meth:`~io.IOBase.fileno` method) and build a socket object from the result. Address - family, socket type and protocol number are as for the :func:`~socket.socket` function - above. The file descriptor should refer to a socket, but this is not checked --- + family, socket type and protocol number are as for the :func:`~socket.socket` function. + The file descriptor should refer to a socket, but this is not checked --- subsequent operations on the object may fail if the file descriptor is invalid. This function is rarely needed, but can be used to get or set socket options on a socket passed to a program as standard input or output (such as a server @@ -1019,22 +964,16 @@ The following functions all create :ref:`socket objects `. .. versionadded:: 3.3 -.. data:: SocketType - - This is a Python type object that represents the socket object type. It is the - same as ``type(socket(...))``. - - Other functions ''''''''''''''' -The :mod:`socket` module also offers various network-related services: +The :mod:`!socket` module also offers various network-related services: .. function:: close(fd) Close a socket file descriptor. This is like :func:`os.close`, but for - sockets. On some platforms (most noticeable Windows) :func:`os.close` + sockets. On some platforms (most notably Windows) :func:`os.close` does not work for socket file descriptors. .. versionadded:: 3.7 @@ -1492,7 +1431,7 @@ The :mod:`socket` module also offers various network-related services: The *fds* parameter is a sequence of file descriptors. Consult :meth:`~socket.sendmsg` for the documentation of these parameters. - .. availability:: Unix, Windows, not WASI. + .. availability:: Unix, not WASI. Unix platforms supporting :meth:`~socket.sendmsg` and :const:`SCM_RIGHTS` mechanism. @@ -1506,9 +1445,9 @@ The :mod:`socket` module also offers various network-related services: Return ``(msg, list(fds), flags, addr)``. Consult :meth:`~socket.recvmsg` for the documentation of these parameters. - .. availability:: Unix, Windows, not WASI. + .. availability:: Unix, not WASI. - Unix platforms supporting :meth:`~socket.sendmsg` + Unix platforms supporting :meth:`~socket.recvmsg` and :const:`SCM_RIGHTS` mechanism. .. versionadded:: 3.9 @@ -1523,641 +1462,706 @@ The :mod:`socket` module also offers various network-related services: Socket Objects -------------- -Socket objects have the following methods. Except for -:meth:`~socket.makefile`, these correspond to Unix system calls applicable -to sockets. +.. class:: socket(family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None) + + Create a new socket using the given address family, socket type and protocol + number. The address family should be :const:`AF_INET` (the default), + :const:`AF_INET6`, :const:`AF_UNIX`, :const:`AF_CAN`, :const:`AF_PACKET`, + or :const:`AF_RDS`. The socket type should be :const:`SOCK_STREAM` (the + default), :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other + ``SOCK_`` constants. The protocol number is usually zero and may be omitted + or in the case where the address family is :const:`AF_CAN` the protocol + should be one of :const:`CAN_RAW`, :const:`CAN_BCM`, :const:`CAN_ISOTP` or + :const:`CAN_J1939`. -.. versionchanged:: 3.2 - Support for the :term:`context manager` protocol was added. Exiting the - context manager is equivalent to calling :meth:`~socket.close`. + If *fileno* is specified, the values for *family*, *type*, and *proto* are + auto-detected from the specified file descriptor. Auto-detection can be + overruled by calling the function with explicit *family*, *type*, or *proto* + arguments. This only affects how Python represents e.g. the return value + of :meth:`socket.getpeername` but not the actual OS resource. Unlike + :func:`socket.fromfd`, *fileno* will return the same socket and not a + duplicate. This may help close a detached socket using + :meth:`socket.close`. + The newly created socket is :ref:`non-inheritable `. -.. method:: socket.accept() + .. audit-event:: socket.__new__ self,family,type,protocol socket.socket - Accept a connection. The socket must be bound to an address and listening for - connections. The return value is a pair ``(conn, address)`` where *conn* is a - *new* socket object usable to send and receive data on the connection, and - *address* is the address bound to the socket on the other end of the connection. + .. versionchanged:: 3.3 + The AF_CAN family was added. + The AF_RDS family was added. - The newly created socket is :ref:`non-inheritable `. + .. versionchanged:: 3.4 + The CAN_BCM protocol was added. .. versionchanged:: 3.4 - The socket is now non-inheritable. + The returned socket is now non-inheritable. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + .. versionchanged:: 3.7 + The CAN_ISOTP protocol was added. + .. versionchanged:: 3.7 + When :const:`SOCK_NONBLOCK` or :const:`SOCK_CLOEXEC` + bit flags are applied to *type* they are cleared, and + :attr:`socket.type` will not reflect them. They are still passed + to the underlying system ``socket()`` call. Therefore, -.. method:: socket.bind(address) + :: - Bind the socket to *address*. The socket must not already be bound. (The format - of *address* depends on the address family --- see above.) + sock = socket.socket( + socket.AF_INET, + socket.SOCK_STREAM | socket.SOCK_NONBLOCK) - .. audit-event:: socket.bind self,address socket.socket.bind + will still create a non-blocking socket on OSes that support + ``SOCK_NONBLOCK``, but ``sock.type`` will be set to + ``socket.SOCK_STREAM``. - .. availability:: not WASI. + .. versionchanged:: 3.9 + The CAN_J1939 protocol was added. + .. versionchanged:: 3.10 + The IPPROTO_MPTCP protocol was added. -.. method:: socket.close() + Socket objects have the following methods. Except for + :meth:`~socket.makefile`, these correspond to Unix system calls applicable + to sockets. - Mark the socket closed. The underlying system resource (e.g. a file - descriptor) is also closed when all file objects from :meth:`makefile` - are closed. Once that happens, all future operations on the socket - object will fail. The remote end will receive no more data (after - queued data is flushed). + .. versionchanged:: 3.2 + Support for the :term:`context manager` protocol was added. Exiting the + context manager is equivalent to calling :meth:`~socket.close`. - Sockets are automatically closed when they are garbage-collected, but - it is recommended to :meth:`close` them explicitly, or to use a - :keyword:`with` statement around them. - .. versionchanged:: 3.6 - :exc:`OSError` is now raised if an error occurs when the underlying - :c:func:`close` call is made. + .. method:: accept() - .. note:: + Accept a connection. The socket must be bound to an address and listening for + connections. The return value is a pair ``(conn, address)`` where *conn* is a + *new* socket object usable to send and receive data on the connection, and + *address* is the address bound to the socket on the other end of the connection. - :meth:`close` releases the resource associated with a connection but - does not necessarily close the connection immediately. If you want - to close the connection in a timely fashion, call :meth:`shutdown` - before :meth:`close`. + The newly created socket is :ref:`non-inheritable `. + .. versionchanged:: 3.4 + The socket is now non-inheritable. -.. method:: socket.connect(address) + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - Connect to a remote socket at *address*. (The format of *address* depends on the - address family --- see above.) - If the connection is interrupted by a signal, the method waits until the - connection completes, or raise a :exc:`TimeoutError` on timeout, if the - signal handler doesn't raise an exception and the socket is blocking or has - a timeout. For non-blocking sockets, the method raises an - :exc:`InterruptedError` exception if the connection is interrupted by a - signal (or the exception raised by the signal handler). + .. method:: bind(address) - .. audit-event:: socket.connect self,address socket.socket.connect + Bind the socket to *address*. The socket must not already be bound. The format + of *address* depends on the address family --- see :ref:`socket-addresses`. - .. versionchanged:: 3.5 - The method now waits until the connection completes instead of raising an + .. audit-event:: socket.bind self,address socket.socket.bind + + .. availability:: not WASI. + + + .. method:: close() + + Mark the socket closed. The underlying system resource (e.g. a file + descriptor) is also closed when all file objects from :meth:`makefile` + are closed. Once that happens, all future operations on the socket + object will fail. The remote end will receive no more data (after + queued data is flushed). + + Sockets are automatically closed when they are garbage-collected, but + it is recommended to :meth:`close` them explicitly, or to use a + :keyword:`with` statement around them. + + .. versionchanged:: 3.6 + :exc:`OSError` is now raised if an error occurs when the underlying + :c:func:`!close` call is made. + + .. note:: + + :meth:`close` releases the resource associated with a connection but + does not necessarily close the connection immediately. If you want + to close the connection in a timely fashion, call :meth:`shutdown` + before :meth:`close`. + + + .. method:: connect(address) + + Connect to a remote socket at *address*. The format of *address* depends on the + address family --- see :ref:`socket-addresses`. + + If the connection is interrupted by a signal, the method waits until the + connection completes, or raises a :exc:`TimeoutError` on timeout, if the + signal handler doesn't raise an exception and the socket is blocking or has + a timeout. For non-blocking sockets, the method raises an :exc:`InterruptedError` exception if the connection is interrupted by a - signal, the signal handler doesn't raise an exception and the socket is - blocking or has a timeout (see the :pep:`475` for the rationale). + signal (or the exception raised by the signal handler). - .. availability:: not WASI. + .. audit-event:: socket.connect self,address socket.socket.connect + .. versionchanged:: 3.5 + The method now waits until the connection completes instead of raising an + :exc:`InterruptedError` exception if the connection is interrupted by a + signal, the signal handler doesn't raise an exception and the socket is + blocking or has a timeout (see the :pep:`475` for the rationale). -.. method:: socket.connect_ex(address) + .. availability:: not WASI. - Like ``connect(address)``, but return an error indicator instead of raising an - exception for errors returned by the C-level :c:func:`connect` call (other - problems, such as "host not found," can still raise exceptions). The error - indicator is ``0`` if the operation succeeded, otherwise the value of the - :c:data:`errno` variable. This is useful to support, for example, asynchronous - connects. - .. audit-event:: socket.connect self,address socket.socket.connect_ex + .. method:: connect_ex(address) - .. availability:: not WASI. + Like ``connect(address)``, but return an error indicator instead of raising an + exception for errors returned by the C-level :c:func:`!connect` call (other + problems, such as "host not found," can still raise exceptions). The error + indicator is ``0`` if the operation succeeded, otherwise the value of the + :c:data:`errno` variable. This is useful to support, for example, asynchronous + connects. -.. method:: socket.detach() + .. audit-event:: socket.connect self,address socket.socket.connect_ex - Put the socket object into closed state without actually closing the - underlying file descriptor. The file descriptor is returned, and can - be reused for other purposes. + .. availability:: not WASI. - .. versionadded:: 3.2 + .. method:: detach() + Put the socket object into closed state without actually closing the + underlying file descriptor. The file descriptor is returned, and can + be reused for other purposes. -.. method:: socket.dup() + .. versionadded:: 3.2 - Duplicate the socket. - The newly created socket is :ref:`non-inheritable `. + .. method:: dup() - .. versionchanged:: 3.4 - The socket is now non-inheritable. + Duplicate the socket. - .. availability:: not WASI. + The newly created socket is :ref:`non-inheritable `. + .. versionchanged:: 3.4 + The socket is now non-inheritable. -.. method:: socket.fileno() + .. availability:: not WASI. - Return the socket's file descriptor (a small integer), or -1 on failure. This - is useful with :func:`select.select`. - Under Windows the small integer returned by this method cannot be used where a - file descriptor can be used (such as :func:`os.fdopen`). Unix does not have - this limitation. + .. method:: fileno() -.. method:: socket.get_inheritable() + Return the socket's file descriptor (a small integer), or -1 on failure. This + is useful with :func:`select.select`. - Get the :ref:`inheritable flag ` of the socket's file - descriptor or socket's handle: ``True`` if the socket can be inherited in - child processes, ``False`` if it cannot. + Under Windows the small integer returned by this method cannot be used where a + file descriptor can be used (such as :func:`os.fdopen`). Unix does not have + this limitation. - .. versionadded:: 3.4 + .. method:: get_inheritable() + Get the :ref:`inheritable flag ` of the socket's file + descriptor or socket's handle: ``True`` if the socket can be inherited in + child processes, ``False`` if it cannot. -.. method:: socket.getpeername() + .. versionadded:: 3.4 - Return the remote address to which the socket is connected. This is useful to - find out the port number of a remote IPv4/v6 socket, for instance. (The format - of the address returned depends on the address family --- see above.) On some - systems this function is not supported. + .. method:: getpeername() -.. method:: socket.getsockname() + Return the remote address to which the socket is connected. This is useful to + find out the port number of a remote IPv4/v6 socket, for instance. The format + of the address returned depends on the address family --- see :ref:`socket-addresses`. + On some systems this function is not supported. - Return the socket's own address. This is useful to find out the port number of - an IPv4/v6 socket, for instance. (The format of the address returned depends on - the address family --- see above.) + .. method:: getsockname() -.. method:: socket.getsockopt(level, optname[, buflen]) + Return the socket's own address. This is useful to find out the port number of + an IPv4/v6 socket, for instance. The format of the address returned depends on + the address family --- see :ref:`socket-addresses`. - Return the value of the given socket option (see the Unix man page - :manpage:`getsockopt(2)`). The needed symbolic constants (:ref:`SO_\* etc. `) - are defined in this module. If *buflen* is absent, an integer option is assumed - and its integer value is returned by the function. If *buflen* is present, it - specifies the maximum length of the buffer used to receive the option in, and - this buffer is returned as a bytes object. It is up to the caller to decode the - contents of the buffer (see the optional built-in module :mod:`struct` for a way - to decode C structures encoded as byte strings). - .. availability:: not WASI. + .. method:: getsockopt(level, optname[, buflen]) + Return the value of the given socket option (see the Unix man page + :manpage:`getsockopt(2)`). The needed symbolic constants (:ref:`SO_\* etc. `) + are defined in this module. If *buflen* is absent, an integer option is assumed + and its integer value is returned by the function. If *buflen* is present, it + specifies the maximum length of the buffer used to receive the option in, and + this buffer is returned as a bytes object. It is up to the caller to decode the + contents of the buffer (see the optional built-in module :mod:`struct` for a way + to decode C structures encoded as byte strings). -.. method:: socket.getblocking() + .. availability:: not WASI. - Return ``True`` if socket is in blocking mode, ``False`` if in - non-blocking. - This is equivalent to checking ``socket.gettimeout() != 0``. + .. method:: getblocking() - .. versionadded:: 3.7 + Return ``True`` if socket is in blocking mode, ``False`` if in + non-blocking. + This is equivalent to checking ``socket.gettimeout() != 0``. -.. method:: socket.gettimeout() + .. versionadded:: 3.7 - Return the timeout in seconds (float) associated with socket operations, - or ``None`` if no timeout is set. This reflects the last call to - :meth:`setblocking` or :meth:`settimeout`. + .. method:: gettimeout() -.. method:: socket.ioctl(control, option) + Return the timeout in seconds (float) associated with socket operations, + or ``None`` if no timeout is set. This reflects the last call to + :meth:`setblocking` or :meth:`settimeout`. - The :meth:`ioctl` method is a limited interface to the WSAIoctl system - interface. Please refer to the `Win32 documentation - `_ for more - information. - On other platforms, the generic :func:`fcntl.fcntl` and :func:`fcntl.ioctl` - functions may be used; they accept a socket object as their first argument. + .. method:: ioctl(control, option) - Currently only the following control codes are supported: - ``SIO_RCVALL``, ``SIO_KEEPALIVE_VALS``, and ``SIO_LOOPBACK_FAST_PATH``. + The :meth:`ioctl` method is a limited interface to the WSAIoctl system + interface. Please refer to the `Win32 documentation + `_ for more + information. - .. availability:: Windows + On other platforms, the generic :func:`fcntl.fcntl` and :func:`fcntl.ioctl` + functions may be used; they accept a socket object as their first argument. - .. versionchanged:: 3.6 - ``SIO_LOOPBACK_FAST_PATH`` was added. + Currently only the following control codes are supported: + ``SIO_RCVALL``, ``SIO_KEEPALIVE_VALS``, and ``SIO_LOOPBACK_FAST_PATH``. + .. availability:: Windows -.. method:: socket.listen([backlog]) + .. versionchanged:: 3.6 + ``SIO_LOOPBACK_FAST_PATH`` was added. - Enable a server to accept connections. If *backlog* is specified, it must - be at least 0 (if it is lower, it is set to 0); it specifies the number of - unaccepted connections that the system will allow before refusing new - connections. If not specified, a default reasonable value is chosen. - .. availability:: not WASI. + .. method:: listen([backlog]) - .. versionchanged:: 3.5 - The *backlog* parameter is now optional. + Enable a server to accept connections. If *backlog* is specified, it must + be at least 0 (if it is lower, it is set to 0); it specifies the number of + unaccepted connections that the system will allow before refusing new + connections. If not specified, a default reasonable value is chosen. + .. availability:: not WASI. -.. method:: socket.makefile(mode='r', buffering=None, *, encoding=None, \ - errors=None, newline=None) + .. versionchanged:: 3.5 + The *backlog* parameter is now optional. - .. index:: single: I/O control; buffering - Return a :term:`file object` associated with the socket. The exact returned - type depends on the arguments given to :meth:`makefile`. These arguments are - interpreted the same way as by the built-in :func:`open` function, except - the only supported *mode* values are ``'r'`` (default), ``'w'``, ``'b'``, or - a combination of those. + .. method:: makefile(mode='r', buffering=None, *, encoding=None, \ + errors=None, newline=None) - The socket must be in blocking mode; it can have a timeout, but the file - object's internal buffer may end up in an inconsistent state if a timeout - occurs. + .. index:: single: I/O control; buffering - Closing the file object returned by :meth:`makefile` won't close the - original socket unless all other file objects have been closed and - :meth:`socket.close` has been called on the socket object. + Return a :term:`file object` associated with the socket. The exact returned + type depends on the arguments given to :meth:`makefile`. These arguments are + interpreted the same way as by the built-in :func:`open` function, except + the only supported *mode* values are ``'r'`` (default), ``'w'``, ``'b'``, or + a combination of those. - .. note:: + The socket must be in blocking mode; it can have a timeout, but the file + object's internal buffer may end up in an inconsistent state if a timeout + occurs. - On Windows, the file-like object created by :meth:`makefile` cannot be - used where a file object with a file descriptor is expected, such as the - stream arguments of :meth:`subprocess.Popen`. + Closing the file object returned by :meth:`makefile` won't close the + original socket unless all other file objects have been closed and + :meth:`socket.close` has been called on the socket object. + .. note:: -.. method:: socket.recv(bufsize[, flags]) + On Windows, the file-like object created by :meth:`makefile` cannot be + used where a file object with a file descriptor is expected, such as the + stream arguments of :meth:`subprocess.Popen`. - Receive data from the socket. The return value is a bytes object representing the - data received. The maximum amount of data to be received at once is specified - by *bufsize*. A returned empty bytes object indicates that the client has disconnected. - See the Unix manual page :manpage:`recv(2)` for the meaning of the optional argument - *flags*; it defaults to zero. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + .. method:: recv(bufsize[, flags]) + Receive data from the socket. The return value is a bytes object representing the + data received. The maximum amount of data to be received at once is specified + by *bufsize*. A returned empty bytes object indicates that the client has disconnected. + See the Unix manual page :manpage:`recv(2)` for the meaning of the optional argument + *flags*; it defaults to zero. -.. method:: socket.recvfrom(bufsize[, flags]) + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - Receive data from the socket. The return value is a pair ``(bytes, address)`` - where *bytes* is a bytes object representing the data received and *address* is the - address of the socket sending the data. See the Unix manual page - :manpage:`recv(2)` for the meaning of the optional argument *flags*; it defaults - to zero. (The format of *address* depends on the address family --- see above.) - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + .. method:: recvfrom(bufsize[, flags]) - .. versionchanged:: 3.7 - For multicast IPv6 address, first item of *address* does not contain - ``%scope_id`` part anymore. In order to get full IPv6 address use - :func:`getnameinfo`. - -.. method:: socket.recvmsg(bufsize[, ancbufsize[, flags]]) - - Receive normal data (up to *bufsize* bytes) and ancillary data from - the socket. The *ancbufsize* argument sets the size in bytes of - the internal buffer used to receive the ancillary data; it defaults - to 0, meaning that no ancillary data will be received. Appropriate - buffer sizes for ancillary data can be calculated using - :func:`CMSG_SPACE` or :func:`CMSG_LEN`, and items which do not fit - into the buffer might be truncated or discarded. The *flags* - argument defaults to 0 and has the same meaning as for - :meth:`recv`. - - The return value is a 4-tuple: ``(data, ancdata, msg_flags, - address)``. The *data* item is a :class:`bytes` object holding the - non-ancillary data received. The *ancdata* item is a list of zero - or more tuples ``(cmsg_level, cmsg_type, cmsg_data)`` representing - the ancillary data (control messages) received: *cmsg_level* and - *cmsg_type* are integers specifying the protocol level and - protocol-specific type respectively, and *cmsg_data* is a - :class:`bytes` object holding the associated data. The *msg_flags* - item is the bitwise OR of various flags indicating conditions on - the received message; see your system documentation for details. - If the receiving socket is unconnected, *address* is the address of - the sending socket, if available; otherwise, its value is - unspecified. - - On some systems, :meth:`sendmsg` and :meth:`recvmsg` can be used to - pass file descriptors between processes over an :const:`AF_UNIX` - socket. When this facility is used (it is often restricted to - :const:`SOCK_STREAM` sockets), :meth:`recvmsg` will return, in its - ancillary data, items of the form ``(socket.SOL_SOCKET, - socket.SCM_RIGHTS, fds)``, where *fds* is a :class:`bytes` object - representing the new file descriptors as a binary array of the - native C :c:expr:`int` type. If :meth:`recvmsg` raises an - exception after the system call returns, it will first attempt to - close any file descriptors received via this mechanism. - - Some systems do not indicate the truncated length of ancillary data - items which have been only partially received. If an item appears - to extend beyond the end of the buffer, :meth:`recvmsg` will issue - a :exc:`RuntimeWarning`, and will return the part of it which is - inside the buffer provided it has not been truncated before the - start of its associated data. - - On systems which support the :const:`SCM_RIGHTS` mechanism, the - following function will receive up to *maxfds* file descriptors, - returning the message data and a list containing the descriptors - (while ignoring unexpected conditions such as unrelated control - messages being received). See also :meth:`sendmsg`. :: - - import socket, array - - def recv_fds(sock, msglen, maxfds): - fds = array.array("i") # Array of ints - msg, ancdata, flags, addr = sock.recvmsg(msglen, socket.CMSG_LEN(maxfds * fds.itemsize)) - for cmsg_level, cmsg_type, cmsg_data in ancdata: - if cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS: - # Append data, ignoring any truncated integers at the end. - fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) - return msg, list(fds) - - .. availability:: Unix. + Receive data from the socket. The return value is a pair ``(bytes, address)`` + where *bytes* is a bytes object representing the data received and *address* is the + address of the socket sending the data. See the Unix manual page + :manpage:`recv(2)` for the meaning of the optional argument *flags*; it defaults + to zero. The format of *address* depends on the address family --- see + :ref:`socket-addresses`. - Most Unix platforms. + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - .. versionadded:: 3.3 + .. versionchanged:: 3.7 + For multicast IPv6 address, first item of *address* does not contain + ``%scope_id`` part anymore. In order to get full IPv6 address use + :func:`getnameinfo`. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.recvmsg_into(buffers[, ancbufsize[, flags]]) - - Receive normal data and ancillary data from the socket, behaving as - :meth:`recvmsg` would, but scatter the non-ancillary data into a - series of buffers instead of returning a new bytes object. The - *buffers* argument must be an iterable of objects that export - writable buffers (e.g. :class:`bytearray` objects); these will be - filled with successive chunks of the non-ancillary data until it - has all been written or there are no more buffers. The operating - system may set a limit (:func:`~os.sysconf` value ``SC_IOV_MAX``) - on the number of buffers that can be used. The *ancbufsize* and - *flags* arguments have the same meaning as for :meth:`recvmsg`. - - The return value is a 4-tuple: ``(nbytes, ancdata, msg_flags, - address)``, where *nbytes* is the total number of bytes of - non-ancillary data written into the buffers, and *ancdata*, - *msg_flags* and *address* are the same as for :meth:`recvmsg`. - - Example:: - - >>> import socket - >>> s1, s2 = socket.socketpair() - >>> b1 = bytearray(b'----') - >>> b2 = bytearray(b'0123456789') - >>> b3 = bytearray(b'--------------') - >>> s1.send(b'Mary had a little lamb') - 22 - >>> s2.recvmsg_into([b1, memoryview(b2)[2:9], b3]) - (22, [], 0, None) - >>> [b1, b2, b3] - [bytearray(b'Mary'), bytearray(b'01 had a 9'), bytearray(b'little lamb---')] - - .. availability:: Unix. + .. method:: recvmsg(bufsize[, ancbufsize[, flags]]) + + Receive normal data (up to *bufsize* bytes) and ancillary data from + the socket. The *ancbufsize* argument sets the size in bytes of + the internal buffer used to receive the ancillary data; it defaults + to 0, meaning that no ancillary data will be received. Appropriate + buffer sizes for ancillary data can be calculated using + :func:`CMSG_SPACE` or :func:`CMSG_LEN`, and items which do not fit + into the buffer might be truncated or discarded. The *flags* + argument defaults to 0 and has the same meaning as for + :meth:`recv`. + + The return value is a 4-tuple: ``(data, ancdata, msg_flags, + address)``. The *data* item is a :class:`bytes` object holding the + non-ancillary data received. The *ancdata* item is a list of zero + or more tuples ``(cmsg_level, cmsg_type, cmsg_data)`` representing + the ancillary data (control messages) received: *cmsg_level* and + *cmsg_type* are integers specifying the protocol level and + protocol-specific type respectively, and *cmsg_data* is a + :class:`bytes` object holding the associated data. The *msg_flags* + item is the bitwise OR of various flags indicating conditions on + the received message; see your system documentation for details. + If the receiving socket is unconnected, *address* is the address of + the sending socket, if available; otherwise, its value is + unspecified. + + On some systems, :meth:`sendmsg` and :meth:`recvmsg` can be used to + pass file descriptors between processes over an :const:`AF_UNIX` + socket. When this facility is used (it is often restricted to + :const:`SOCK_STREAM` sockets), :meth:`recvmsg` will return, in its + ancillary data, items of the form ``(socket.SOL_SOCKET, + socket.SCM_RIGHTS, fds)``, where *fds* is a :class:`bytes` object + representing the new file descriptors as a binary array of the + native C :c:expr:`int` type. If :meth:`recvmsg` raises an + exception after the system call returns, it will first attempt to + close any file descriptors received via this mechanism. + + Some systems do not indicate the truncated length of ancillary data + items which have been only partially received. If an item appears + to extend beyond the end of the buffer, :meth:`recvmsg` will issue + a :exc:`RuntimeWarning`, and will return the part of it which is + inside the buffer provided it has not been truncated before the + start of its associated data. + + On systems which support the :const:`!SCM_RIGHTS` mechanism, the + following function will receive up to *maxfds* file descriptors, + returning the message data and a list containing the descriptors + (while ignoring unexpected conditions such as unrelated control + messages being received). See also :meth:`sendmsg`. :: + + import socket, array + + def recv_fds(sock, msglen, maxfds): + fds = array.array("i") # Array of ints + msg, ancdata, flags, addr = sock.recvmsg(msglen, socket.CMSG_LEN(maxfds * fds.itemsize)) + for cmsg_level, cmsg_type, cmsg_data in ancdata: + if cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS: + # Append data, ignoring any truncated integers at the end. + fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + return msg, list(fds) + + .. availability:: Unix. + + Most Unix platforms. + + .. versionadded:: 3.3 + + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + + + .. method:: recvmsg_into(buffers[, ancbufsize[, flags]]) + + Receive normal data and ancillary data from the socket, behaving as + :meth:`recvmsg` would, but scatter the non-ancillary data into a + series of buffers instead of returning a new bytes object. The + *buffers* argument must be an iterable of objects that export + writable buffers (e.g. :class:`bytearray` objects); these will be + filled with successive chunks of the non-ancillary data until it + has all been written or there are no more buffers. The operating + system may set a limit (:func:`~os.sysconf` value ``SC_IOV_MAX``) + on the number of buffers that can be used. The *ancbufsize* and + *flags* arguments have the same meaning as for :meth:`recvmsg`. + + The return value is a 4-tuple: ``(nbytes, ancdata, msg_flags, + address)``, where *nbytes* is the total number of bytes of + non-ancillary data written into the buffers, and *ancdata*, + *msg_flags* and *address* are the same as for :meth:`recvmsg`. + + Example:: + + >>> import socket + >>> s1, s2 = socket.socketpair() + >>> b1 = bytearray(b'----') + >>> b2 = bytearray(b'0123456789') + >>> b3 = bytearray(b'--------------') + >>> s1.send(b'Mary had a little lamb') + 22 + >>> s2.recvmsg_into([b1, memoryview(b2)[2:9], b3]) + (22, [], 0, None) + >>> [b1, b2, b3] + [bytearray(b'Mary'), bytearray(b'01 had a 9'), bytearray(b'little lamb---')] + + .. availability:: Unix. + + Most Unix platforms. + + .. versionadded:: 3.3 + + + .. method:: recvfrom_into(buffer[, nbytes[, flags]]) + + Receive data from the socket, writing it into *buffer* instead of creating a + new bytestring. The return value is a pair ``(nbytes, address)`` where *nbytes* is + the number of bytes received and *address* is the address of the socket sending + the data. See the Unix manual page :manpage:`recv(2)` for the meaning of the + optional argument *flags*; it defaults to zero. The format of *address* + depends on the address family --- see :ref:`socket-addresses`. + + + .. method:: recv_into(buffer[, nbytes[, flags]]) + + Receive up to *nbytes* bytes from the socket, storing the data into a buffer + rather than creating a new bytestring. If *nbytes* is not specified (or 0), + receive up to the size available in the given buffer. Returns the number of + bytes received. See the Unix manual page :manpage:`recv(2)` for the meaning + of the optional argument *flags*; it defaults to zero. - Most Unix platforms. - .. versionadded:: 3.3 + .. method:: send(bytes[, flags]) + Send data to the socket. The socket must be connected to a remote socket. The + optional *flags* argument has the same meaning as for :meth:`recv`. + Returns the number of bytes sent. Applications are responsible for checking that + all data has been sent; if only some of the data was transmitted, the + application needs to attempt delivery of the remaining data. For further + information on this topic, consult the :ref:`socket-howto`. -.. method:: socket.recvfrom_into(buffer[, nbytes[, flags]]) + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - Receive data from the socket, writing it into *buffer* instead of creating a - new bytestring. The return value is a pair ``(nbytes, address)`` where *nbytes* is - the number of bytes received and *address* is the address of the socket sending - the data. See the Unix manual page :manpage:`recv(2)` for the meaning of the - optional argument *flags*; it defaults to zero. (The format of *address* - depends on the address family --- see above.) + .. method:: sendall(bytes[, flags]) -.. method:: socket.recv_into(buffer[, nbytes[, flags]]) + Send data to the socket. The socket must be connected to a remote socket. The + optional *flags* argument has the same meaning as for :meth:`recv`. + Unlike :meth:`send`, this method continues to send data from *bytes* until + either all data has been sent or an error occurs. ``None`` is returned on + success. On error, an exception is raised, and there is no way to determine how + much data, if any, was successfully sent. - Receive up to *nbytes* bytes from the socket, storing the data into a buffer - rather than creating a new bytestring. If *nbytes* is not specified (or 0), - receive up to the size available in the given buffer. Returns the number of - bytes received. See the Unix manual page :manpage:`recv(2)` for the meaning - of the optional argument *flags*; it defaults to zero. + .. versionchanged:: 3.5 + The socket timeout is no longer reset each time data is sent successfully. + The socket timeout is now the maximum total duration to send all data. + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). -.. method:: socket.send(bytes[, flags]) - Send data to the socket. The socket must be connected to a remote socket. The - optional *flags* argument has the same meaning as for :meth:`recv` above. - Returns the number of bytes sent. Applications are responsible for checking that - all data has been sent; if only some of the data was transmitted, the - application needs to attempt delivery of the remaining data. For further - information on this topic, consult the :ref:`socket-howto`. + .. method:: sendto(bytes, address) + sendto(bytes, flags, address) - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + Send data to the socket. The socket should not be connected to a remote socket, + since the destination socket is specified by *address*. The optional *flags* + argument has the same meaning as for :meth:`recv`. Return the number of + bytes sent. The format of *address* depends on the address family --- see + :ref:`socket-addresses`. + .. audit-event:: socket.sendto self,address socket.socket.sendto -.. method:: socket.sendall(bytes[, flags]) + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - Send data to the socket. The socket must be connected to a remote socket. The - optional *flags* argument has the same meaning as for :meth:`recv` above. - Unlike :meth:`send`, this method continues to send data from *bytes* until - either all data has been sent or an error occurs. ``None`` is returned on - success. On error, an exception is raised, and there is no way to determine how - much data, if any, was successfully sent. - .. versionchanged:: 3.5 - The socket timeout is no longer reset each time data is sent successfully. - The socket timeout is now the maximum total duration to send all data. + .. method:: sendmsg(buffers[, ancdata[, flags[, address]]]) - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + Send normal and ancillary data to the socket, gathering the + non-ancillary data from a series of buffers and concatenating it + into a single message. The *buffers* argument specifies the + non-ancillary data as an iterable of + :term:`bytes-like objects ` + (e.g. :class:`bytes` objects); the operating system may set a limit + (:func:`~os.sysconf` value ``SC_IOV_MAX``) on the number of buffers + that can be used. The *ancdata* argument specifies the ancillary + data (control messages) as an iterable of zero or more tuples + ``(cmsg_level, cmsg_type, cmsg_data)``, where *cmsg_level* and + *cmsg_type* are integers specifying the protocol level and + protocol-specific type respectively, and *cmsg_data* is a + bytes-like object holding the associated data. Note that + some systems (in particular, systems without :func:`CMSG_SPACE`) + might support sending only one control message per call. The + *flags* argument defaults to 0 and has the same meaning as for + :meth:`send`. If *address* is supplied and not ``None``, it sets a + destination address for the message. The return value is the + number of bytes of non-ancillary data sent. + The following function sends the list of file descriptors *fds* + over an :const:`AF_UNIX` socket, on systems which support the + :const:`!SCM_RIGHTS` mechanism. See also :meth:`recvmsg`. :: -.. method:: socket.sendto(bytes, address) - socket.sendto(bytes, flags, address) + import socket, array - Send data to the socket. The socket should not be connected to a remote socket, - since the destination socket is specified by *address*. The optional *flags* - argument has the same meaning as for :meth:`recv` above. Return the number of - bytes sent. (The format of *address* depends on the address family --- see - above.) + def send_fds(sock, msg, fds): + return sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array("i", fds))]) - .. audit-event:: socket.sendto self,address socket.socket.sendto + .. availability:: Unix, not WASI. - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.sendmsg(buffers[, ancdata[, flags[, address]]]) - - Send normal and ancillary data to the socket, gathering the - non-ancillary data from a series of buffers and concatenating it - into a single message. The *buffers* argument specifies the - non-ancillary data as an iterable of - :term:`bytes-like objects ` - (e.g. :class:`bytes` objects); the operating system may set a limit - (:func:`~os.sysconf` value ``SC_IOV_MAX``) on the number of buffers - that can be used. The *ancdata* argument specifies the ancillary - data (control messages) as an iterable of zero or more tuples - ``(cmsg_level, cmsg_type, cmsg_data)``, where *cmsg_level* and - *cmsg_type* are integers specifying the protocol level and - protocol-specific type respectively, and *cmsg_data* is a - bytes-like object holding the associated data. Note that - some systems (in particular, systems without :func:`CMSG_SPACE`) - might support sending only one control message per call. The - *flags* argument defaults to 0 and has the same meaning as for - :meth:`send`. If *address* is supplied and not ``None``, it sets a - destination address for the message. The return value is the - number of bytes of non-ancillary data sent. - - The following function sends the list of file descriptors *fds* - over an :const:`AF_UNIX` socket, on systems which support the - :const:`SCM_RIGHTS` mechanism. See also :meth:`recvmsg`. :: - - import socket, array - - def send_fds(sock, msg, fds): - return sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array("i", fds))]) + Most Unix platforms. - .. availability:: Unix, not WASI. + .. audit-event:: socket.sendmsg self,address socket.socket.sendmsg - Most Unix platforms. + .. versionadded:: 3.3 - .. audit-event:: socket.sendmsg self,address socket.socket.sendmsg + .. versionchanged:: 3.5 + If the system call is interrupted and the signal handler does not raise + an exception, the method now retries the system call instead of raising + an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - .. versionadded:: 3.3 + .. method:: sendmsg_afalg([msg], *, op[, iv[, assoclen[, flags]]]) - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + Specialized version of :meth:`~socket.sendmsg` for :const:`AF_ALG` socket. + Set mode, IV, AEAD associated data length and flags for :const:`AF_ALG` socket. -.. method:: socket.sendmsg_afalg([msg], *, op[, iv[, assoclen[, flags]]]) + .. availability:: Linux >= 2.6.38. - Specialized version of :meth:`~socket.sendmsg` for :const:`AF_ALG` socket. - Set mode, IV, AEAD associated data length and flags for :const:`AF_ALG` socket. + .. versionadded:: 3.6 - .. availability:: Linux >= 2.6.38. + .. method:: sendfile(file, offset=0, count=None) - .. versionadded:: 3.6 + Send a file until EOF is reached by using high-performance + :mod:`os.sendfile` and return the total number of bytes which were sent. + *file* must be a regular file object opened in binary mode. If + :mod:`os.sendfile` is not available (e.g. Windows) or *file* is not a + regular file :meth:`send` will be used instead. *offset* tells from where to + start reading the file. If specified, *count* is the total number of bytes + to transmit as opposed to sending the file until EOF is reached. File + position is updated on return or also in case of error in which case + :meth:`file.tell() ` can be used to figure out the number of + bytes which were sent. The socket must be of :const:`SOCK_STREAM` type. + Non-blocking sockets are not supported. -.. method:: socket.sendfile(file, offset=0, count=None) + .. versionadded:: 3.5 - Send a file until EOF is reached by using high-performance - :mod:`os.sendfile` and return the total number of bytes which were sent. - *file* must be a regular file object opened in binary mode. If - :mod:`os.sendfile` is not available (e.g. Windows) or *file* is not a - regular file :meth:`send` will be used instead. *offset* tells from where to - start reading the file. If specified, *count* is the total number of bytes - to transmit as opposed to sending the file until EOF is reached. File - position is updated on return or also in case of error in which case - :meth:`file.tell() ` can be used to figure out the number of - bytes which were sent. The socket must be of :const:`SOCK_STREAM` type. - Non-blocking sockets are not supported. + .. method:: set_inheritable(inheritable) - .. versionadded:: 3.5 + Set the :ref:`inheritable flag ` of the socket's file + descriptor or socket's handle. -.. method:: socket.set_inheritable(inheritable) + .. versionadded:: 3.4 - Set the :ref:`inheritable flag ` of the socket's file - descriptor or socket's handle. - .. versionadded:: 3.4 + .. method:: setblocking(flag) + Set blocking or non-blocking mode of the socket: if *flag* is false, the + socket is set to non-blocking, else to blocking mode. -.. method:: socket.setblocking(flag) + This method is a shorthand for certain :meth:`~socket.settimeout` calls: - Set blocking or non-blocking mode of the socket: if *flag* is false, the - socket is set to non-blocking, else to blocking mode. + * ``sock.setblocking(True)`` is equivalent to ``sock.settimeout(None)`` - This method is a shorthand for certain :meth:`~socket.settimeout` calls: + * ``sock.setblocking(False)`` is equivalent to ``sock.settimeout(0.0)`` - * ``sock.setblocking(True)`` is equivalent to ``sock.settimeout(None)`` + .. versionchanged:: 3.7 + The method no longer applies :const:`SOCK_NONBLOCK` flag on + :attr:`socket.type`. - * ``sock.setblocking(False)`` is equivalent to ``sock.settimeout(0.0)`` - .. versionchanged:: 3.7 - The method no longer applies :const:`SOCK_NONBLOCK` flag on - :attr:`socket.type`. + .. method:: settimeout(value) + Set a timeout on blocking socket operations. The *value* argument can be a + nonnegative real number expressing seconds, or ``None``. + If a non-zero value is given, subsequent socket operations will raise a + :exc:`timeout` exception if the timeout period *value* has elapsed before + the operation has completed. If zero is given, the socket is put in + non-blocking mode. If ``None`` is given, the socket is put in blocking mode. -.. method:: socket.settimeout(value) + For further information, please consult the :ref:`notes on socket timeouts `. - Set a timeout on blocking socket operations. The *value* argument can be a - nonnegative floating-point number expressing seconds, or ``None``. - If a non-zero value is given, subsequent socket operations will raise a - :exc:`timeout` exception if the timeout period *value* has elapsed before - the operation has completed. If zero is given, the socket is put in - non-blocking mode. If ``None`` is given, the socket is put in blocking mode. + .. versionchanged:: 3.7 + The method no longer toggles :const:`SOCK_NONBLOCK` flag on + :attr:`socket.type`. - For further information, please consult the :ref:`notes on socket timeouts `. - .. versionchanged:: 3.7 - The method no longer toggles :const:`SOCK_NONBLOCK` flag on - :attr:`socket.type`. + .. method:: setsockopt(level, optname, value: int | Buffer) + setsockopt(level, optname, None, optlen: int) + .. index:: pair: module; struct -.. method:: socket.setsockopt(level, optname, value: int) -.. method:: socket.setsockopt(level, optname, value: buffer) - :noindex: -.. method:: socket.setsockopt(level, optname, None, optlen: int) - :noindex: + Set the value of the given socket option (see the Unix manual page + :manpage:`setsockopt(2)`). The needed symbolic constants are defined in this + module (:ref:`!SO_\* etc. `). The value can be an integer, + ``None`` or a :term:`bytes-like object` representing a buffer. In the latter + case it is up to the caller to ensure that the bytestring contains the + proper bits (see the optional built-in module :mod:`struct` for a way to + encode C structures as bytestrings). When *value* is set to ``None``, + *optlen* argument is required. It's equivalent to calling :c:func:`!setsockopt` C + function with ``optval=NULL`` and ``optlen=optlen``. - .. index:: pair: module; struct + .. versionchanged:: 3.5 + Writable :term:`bytes-like object` is now accepted. - Set the value of the given socket option (see the Unix manual page - :manpage:`setsockopt(2)`). The needed symbolic constants are defined in this - module (:ref:`!SO_\* etc. `). The value can be an integer, - ``None`` or a :term:`bytes-like object` representing a buffer. In the later - case it is up to the caller to ensure that the bytestring contains the - proper bits (see the optional built-in module :mod:`struct` for a way to - encode C structures as bytestrings). When *value* is set to ``None``, - *optlen* argument is required. It's equivalent to call :c:func:`setsockopt` C - function with ``optval=NULL`` and ``optlen=optlen``. + .. versionchanged:: 3.6 + setsockopt(level, optname, None, optlen: int) form added. - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. + .. availability:: not WASI. - .. versionchanged:: 3.6 - setsockopt(level, optname, None, optlen: int) form added. - .. availability:: not WASI. + .. method:: shutdown(how) + Shut down one or both halves of the connection. If *how* is :const:`SHUT_RD`, + further receives are disallowed. If *how* is :const:`SHUT_WR`, further sends + are disallowed. If *how* is :const:`SHUT_RDWR`, further sends and receives are + disallowed. -.. method:: socket.shutdown(how) + .. availability:: not WASI. - Shut down one or both halves of the connection. If *how* is :const:`SHUT_RD`, - further receives are disallowed. If *how* is :const:`SHUT_WR`, further sends - are disallowed. If *how* is :const:`SHUT_RDWR`, further sends and receives are - disallowed. - .. availability:: not WASI. + .. method:: share(process_id) + Duplicate a socket and prepare it for sharing with a target process. The + target process must be provided with *process_id*. The resulting bytes object + can then be passed to the target process using some form of interprocess + communication and the socket can be recreated there using :func:`fromshare`. + Once this method has been called, it is safe to close the socket since + the operating system has already duplicated it for the target process. -.. method:: socket.share(process_id) + .. availability:: Windows. - Duplicate a socket and prepare it for sharing with a target process. The - target process must be provided with *process_id*. The resulting bytes object - can then be passed to the target process using some form of interprocess - communication and the socket can be recreated there using :func:`fromshare`. - Once this method has been called, it is safe to close the socket since - the operating system has already duplicated it for the target process. + .. versionadded:: 3.3 - .. availability:: Windows. - .. versionadded:: 3.3 + Note that there are no methods :meth:`!read` or :meth:`!write`; use + :meth:`~socket.recv` and :meth:`~socket.send` without *flags* argument instead. + Socket objects also have these (read-only) attributes that correspond to the + values given to the :class:`~socket.socket` constructor. -Note that there are no methods :meth:`read` or :meth:`write`; use -:meth:`~socket.recv` and :meth:`~socket.send` without *flags* argument instead. -Socket objects also have these (read-only) attributes that correspond to the -values given to the :class:`~socket.socket` constructor. + .. attribute:: family + The socket family. -.. attribute:: socket.family - The socket family. + .. attribute:: type + The socket type. -.. attribute:: socket.type - The socket type. + .. attribute:: proto + The socket protocol. -.. attribute:: socket.proto - The socket protocol. +.. class:: SocketType + The base class of the :class:`~socket.socket` type, re-exported from + :mod:`!_socket`. An instance check such as + ``isinstance(socket(...), SocketType)`` is true, but ``SocketType`` is not + the same as ``type(socket(...))``, which is :class:`~socket.socket` itself. .. _socket-timeouts: @@ -2415,7 +2419,7 @@ lead to this error:: This is because the previous execution has left the socket in a ``TIME_WAIT`` state, and can't be immediately reused. -There is a :mod:`socket` flag to set, in order to prevent this, +There is a :mod:`!socket` flag to set, in order to prevent this, :const:`socket.SO_REUSEADDR`:: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst index 59cfa136a3b7da9..e1d47b69b7578ac 100644 --- a/Doc/library/socketserver.rst +++ b/Doc/library/socketserver.rst @@ -8,7 +8,7 @@ -------------- -The :mod:`socketserver` module simplifies the task of writing network servers. +The :mod:`!socketserver` module simplifies the task of writing network servers. .. include:: ../includes/wasm-notavail.rst @@ -541,10 +541,10 @@ objects that simplify communication by providing the standard file interface):: The difference is that the ``readline()`` call in the second handler will call ``recv()`` multiple times until it encounters a newline character, while the -the first handler had to use a ``recv()`` loop to accumulate data until a +first handler had to use a ``recv()`` loop to accumulate data until a newline itself. If it had just used a single ``recv()`` without the loop it would just have returned what has been received so far from the client. -TCP is stream based: data arrives in the order it was sent, but there no +TCP is stream based: data arrives in the order it was sent, but there is no correlation between client ``send()`` or ``sendall()`` calls and the number of ``recv()`` calls on the server required to receive it. diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index c615650b622f7fd..8e022e1b31d0f3b 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -31,7 +31,9 @@ PostgreSQL or Oracle. The :mod:`!sqlite3` module was written by Gerhard Häring. It provides an SQL interface compliant with the DB-API 2.0 specification described by :pep:`249`, and -requires SQLite 3.15.2 or newer. +requires the third-party `SQLite `_ library. + +.. include:: ../includes/optional-module.rst This document includes four main sections: @@ -55,7 +57,7 @@ This document includes four main sections: PEP written by Marc-André Lemburg. -.. We use the following practises for SQL code: +.. We use the following practices for SQL code: - UPPERCASE for keywords - snake_case for schema - single quotes for string literals @@ -289,7 +291,7 @@ Module functions Set it to any combination (using ``|``, bitwise or) of :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES` to enable this. - Column names takes precedence over declared types if both flags are set. + Column names take precedence over declared types if both flags are set. By default (``0``), type detection is disabled. :param isolation_level: @@ -614,7 +616,7 @@ Connection objects supplied, this must be a :term:`callable` returning an instance of :class:`Cursor` or its subclasses. - .. method:: blobopen(table, column, row, /, *, readonly=False, name="main") + .. method:: blobopen(table, column, rowid, /, *, readonly=False, name="main") Open a :class:`Blob` handle to an existing :abbr:`BLOB (Binary Large OBject)`. @@ -625,8 +627,8 @@ Connection objects :param str column: The name of the column where the blob is located. - :param str row: - The name of the row where the blob is located. + :param int rowid: + The row id where the blob is located. :param bool readonly: Set to ``True`` if the blob should be opened without write @@ -1418,6 +1420,9 @@ Connection objects See :ref:`sqlite3-howto-row-factory` for more details. + .. versionchanged:: 3.14.6 + Deleting the ``row_factory`` attribute is no longer allowed. + .. attribute:: text_factory A :term:`callable` that accepts a :class:`bytes` parameter @@ -1427,6 +1432,9 @@ Connection objects See :ref:`sqlite3-howto-encoding` for more details. + .. versionchanged:: 3.14.6 + Deleting the ``text_factory`` attribute is no longer allowed. + .. attribute:: total_changes Return the total number of database rows that have been modified, inserted, or @@ -1492,7 +1500,9 @@ Cursor objects :type parameters: :class:`dict` | :term:`sequence` :raises ProgrammingError: - If *sql* contains more than one SQL statement. + When *sql* contains more than one SQL statement. + When :ref:`named placeholders ` are used + and *parameters* is a sequence instead of a :class:`dict`. If :attr:`~Connection.autocommit` is :data:`LEGACY_TRANSACTION_CONTROL`, @@ -1501,13 +1511,11 @@ Cursor objects and there is no open transaction, a transaction is implicitly opened before executing *sql*. - .. deprecated-removed:: 3.12 3.14 + .. versionchanged:: 3.14 - :exc:`DeprecationWarning` is emitted if + :exc:`ProgrammingError` is emitted if :ref:`named placeholders ` are used and *parameters* is a sequence instead of a :class:`dict`. - Starting with Python 3.14, :exc:`ProgrammingError` will - be raised instead. Use :meth:`executescript` to execute multiple SQL statements. @@ -1529,8 +1537,10 @@ Cursor objects :type parameters: :term:`iterable` :raises ProgrammingError: - If *sql* contains more than one SQL statement, - or is not a DML statement. + When *sql* contains more than one SQL statement + or is not a DML statement, + When :ref:`named placeholders ` are used + and the items in *parameters* are sequences instead of :class:`dict`\s. Example: @@ -1554,14 +1564,12 @@ Cursor objects .. _RETURNING clauses: https://www.sqlite.org/lang_returning.html - .. deprecated-removed:: 3.12 3.14 + .. versionchanged:: 3.14 - :exc:`DeprecationWarning` is emitted if + :exc:`ProgrammingError` is emitted if :ref:`named placeholders ` are used and the items in *parameters* are sequences instead of :class:`dict`\s. - Starting with Python 3.14, :exc:`ProgrammingError` will - be raised instead. .. method:: executescript(sql_script, /) @@ -1612,6 +1620,9 @@ Cursor objects If the *size* parameter is used, then it is best for it to retain the same value from one :meth:`fetchmany` call to the next. + .. versionchanged:: 3.14.1 + Negative *size* values are rejected by raising :exc:`ValueError`. + .. method:: fetchall() Return all (remaining) rows of a query result as a :class:`list`. @@ -1639,6 +1650,9 @@ Cursor objects Read/write attribute that controls the number of rows returned by :meth:`fetchmany`. The default value is 1 which means a single row would be fetched per call. + .. versionchanged:: 3.14.1 + Negative values are rejected by raising :exc:`ValueError`. + .. attribute:: connection Read-only attribute that provides the SQLite database :class:`Connection` @@ -1704,6 +1718,9 @@ Cursor objects See :ref:`sqlite3-howto-row-factory` for more details. + .. versionchanged:: 3.14.6 + Deleting the ``row_factory`` attribute is no longer allowed. + .. The sqlite3.Row example used to be a how-to. It has now been incorporated into the Row reference. We keep the anchor here in order not to break @@ -2280,7 +2297,7 @@ This section shows recipes for common adapters and converters. .. testcode:: - import datetime + import datetime as dt import sqlite3 def adapt_date_iso(val): @@ -2289,27 +2306,27 @@ This section shows recipes for common adapters and converters. def adapt_datetime_iso(val): """Adapt datetime.datetime to timezone-naive ISO 8601 date.""" - return val.isoformat() + return val.replace(tzinfo=None).isoformat() def adapt_datetime_epoch(val): """Adapt datetime.datetime to Unix timestamp.""" return int(val.timestamp()) - sqlite3.register_adapter(datetime.date, adapt_date_iso) - sqlite3.register_adapter(datetime.datetime, adapt_datetime_iso) - sqlite3.register_adapter(datetime.datetime, adapt_datetime_epoch) + sqlite3.register_adapter(dt.date, adapt_date_iso) + sqlite3.register_adapter(dt.datetime, adapt_datetime_iso) + sqlite3.register_adapter(dt.datetime, adapt_datetime_epoch) def convert_date(val): """Convert ISO 8601 date to datetime.date object.""" - return datetime.date.fromisoformat(val.decode()) + return dt.date.fromisoformat(val.decode()) def convert_datetime(val): """Convert ISO 8601 datetime to datetime.datetime object.""" - return datetime.datetime.fromisoformat(val.decode()) + return dt.datetime.fromisoformat(val.decode()) def convert_timestamp(val): """Convert Unix epoch timestamp to datetime.datetime object.""" - return datetime.datetime.fromtimestamp(int(val)) + return dt.datetime.fromtimestamp(int(val)) sqlite3.register_converter("date", convert_date) sqlite3.register_converter("datetime", convert_datetime) @@ -2318,17 +2335,17 @@ This section shows recipes for common adapters and converters. .. testcode:: :hide: - dt = datetime.datetime(2019, 5, 18, 15, 17, 8, 123456) + when = dt.datetime(2019, 5, 18, 15, 17, 8, 123456) - assert adapt_date_iso(dt.date()) == "2019-05-18" - assert convert_date(b"2019-05-18") == dt.date() + assert adapt_date_iso(when.date()) == "2019-05-18" + assert convert_date(b"2019-05-18") == when.date() - assert adapt_datetime_iso(dt) == "2019-05-18T15:17:08.123456" - assert convert_datetime(b"2019-05-18T15:17:08.123456") == dt + assert adapt_datetime_iso(when) == "2019-05-18T15:17:08.123456" + assert convert_datetime(b"2019-05-18T15:17:08.123456") == when # Using current time as fromtimestamp() returns local date/time. # Dropping microseconds as adapt_datetime_epoch truncates fractional second part. - now = datetime.datetime.now().replace(microsecond=0) + now = dt.datetime.now().replace(microsecond=0) current_timestamp = int(now.timestamp()) assert adapt_datetime_epoch(now) == current_timestamp diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index c0dcecf737ef765..92282bf624fe62f 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -18,8 +18,9 @@ This module provides access to Transport Layer Security (often known as "Secure Sockets Layer") encryption and peer authentication facilities for network sockets, both client-side and server-side. This module uses the OpenSSL -library. It is available on all modern Unix systems, Windows, macOS, and -probably additional platforms, as long as OpenSSL is installed on that platform. +library. + +.. include:: ../includes/optional-module.rst .. note:: @@ -69,7 +70,7 @@ by SSL sockets created through the :meth:`SSLContext.wrap_socket` method. Use of deprecated constants and functions result in deprecation warnings. -Functions, Constants, and Exceptions +Functions, constants, and exceptions ------------------------------------ @@ -125,10 +126,11 @@ Context creation A convenience function helps create :class:`SSLContext` objects for common purposes. -.. function:: create_default_context(purpose=Purpose.SERVER_AUTH, cafile=None, capath=None, cadata=None) +.. function:: create_default_context(purpose=Purpose.SERVER_AUTH, *,\ + cafile=None, capath=None, cadata=None) Return a new :class:`SSLContext` object with default settings for - the given *purpose*. The settings are chosen by the :mod:`ssl` module, + the given *purpose*. The settings are chosen by the :mod:`!ssl` module, and usually represent a higher security level than when calling the :class:`SSLContext` constructor directly. @@ -314,7 +316,7 @@ Exceptions Random generation ^^^^^^^^^^^^^^^^^ -.. function:: RAND_bytes(num) +.. function:: RAND_bytes(num, /) Return *num* cryptographically strong pseudo-random bytes. Raises an :class:`SSLError` if the PRNG has not been seeded with enough data or if the @@ -338,7 +340,7 @@ Random generation :func:`ssl.RAND_egd` and :func:`ssl.RAND_add` to increase the randomness of the pseudo-random number generator. -.. function:: RAND_add(bytes, entropy) +.. function:: RAND_add(bytes, entropy, /) Mix the given *bytes* into the SSL pseudo-random number generator. The parameter *entropy* (a float) is a lower bound on the entropy contained in @@ -357,7 +359,7 @@ Certificate handling .. function:: cert_time_to_seconds(cert_time) - Return the time in seconds since the Epoch, given the ``cert_time`` + Return the time in seconds since the epoch, given the ``cert_time`` string representing the "notBefore" or "notAfter" date from a certificate in ``"%b %d %H:%M:%S %Y %Z"`` strptime format (C locale). @@ -367,12 +369,12 @@ Certificate handling .. doctest:: newcontext >>> import ssl + >>> import datetime as dt >>> timestamp = ssl.cert_time_to_seconds("Jan 5 09:34:43 2018 GMT") >>> timestamp # doctest: +SKIP 1515144883 - >>> from datetime import datetime - >>> print(datetime.utcfromtimestamp(timestamp)) # doctest: +SKIP - 2018-01-05 09:34:43 + >>> print(dt.datetime.fromtimestamp(timestamp, dt.UTC)) # doctest: +SKIP + 2018-01-05 09:34:43+00:00 "notBefore" or "notAfter" dates must use GMT (:rfc:`5280`). @@ -406,12 +408,12 @@ Certificate handling .. versionchanged:: 3.10 The *timeout* parameter was added. -.. function:: DER_cert_to_PEM_cert(DER_cert_bytes) +.. function:: DER_cert_to_PEM_cert(der_cert_bytes) Given a certificate as a DER-encoded blob of bytes, returns a PEM-encoded string version of the same certificate. -.. function:: PEM_cert_to_DER_cert(PEM_cert_string) +.. function:: PEM_cert_to_DER_cert(pem_cert_string) Given a certificate as an ASCII PEM string, returns a DER-encoded sequence of bytes for that same certificate. @@ -1048,7 +1050,7 @@ Constants :attr:`TLSVersion.TLSv1_3` are deprecated. -SSL Sockets +SSL sockets ----------- .. class:: SSLSocket(socket.socket) @@ -1096,7 +1098,7 @@ SSL Sockets :meth:`SSLContext.wrap_socket` to wrap a socket. .. versionchanged:: 3.7 - :class:`SSLSocket` instances must to created with + :class:`SSLSocket` instances must be created with :meth:`~SSLContext.wrap_socket`. In earlier versions, it was possible to create instances directly. This was never documented or officially supported. @@ -1128,10 +1130,10 @@ SSL sockets also have the following additional methods and attributes: .. deprecated:: 3.6 Use :meth:`~SSLSocket.recv` instead of :meth:`~SSLSocket.read`. -.. method:: SSLSocket.write(buf) +.. method:: SSLSocket.write(data) - Write *buf* to the SSL socket and return the number of bytes written. The - *buf* argument must be an object supporting the buffer interface. + Write *data* to the SSL socket and return the number of bytes written. The + *data* argument must be an object supporting the buffer interface. Raise :exc:`SSLWantReadError` or :exc:`SSLWantWriteError` if the socket is :ref:`non-blocking ` and the write would block. @@ -1141,7 +1143,7 @@ SSL sockets also have the following additional methods and attributes: .. versionchanged:: 3.5 The socket timeout is no longer reset each time bytes are received or sent. - The socket timeout is now the maximum total duration to write *buf*. + The socket timeout is now the maximum total duration to write *data*. .. deprecated:: 3.6 Use :meth:`~SSLSocket.send` instead of :meth:`~SSLSocket.write`. @@ -1158,12 +1160,15 @@ SSL sockets also have the following additional methods and attributes: :meth:`~socket.socket.recv` and :meth:`~socket.socket.send` instead of these methods. -.. method:: SSLSocket.do_handshake() +.. method:: SSLSocket.do_handshake(block=False) Perform the SSL setup handshake. + If *block* is true and the timeout obtained by :meth:`~socket.socket.gettimeout` + is zero, the socket is set in blocking mode until the handshake is performed. + .. versionchanged:: 3.4 - The handshake method also performs :func:`match_hostname` when the + The handshake method also performs :func:`!match_hostname` when the :attr:`~SSLContext.check_hostname` attribute of the socket's :attr:`~SSLSocket.context` is true. @@ -1173,7 +1178,7 @@ SSL sockets also have the following additional methods and attributes: .. versionchanged:: 3.7 Hostname or IP address is matched by OpenSSL during handshake. The - function :func:`match_hostname` is no longer used. In case OpenSSL + function :func:`!match_hostname` is no longer used. In case OpenSSL refuses a hostname or IP address, the handshake is aborted early and a TLS alert message is sent to the peer. @@ -1406,7 +1411,7 @@ SSL sockets also have the following additional methods and attributes: .. versionadded:: 3.6 -SSL Contexts +SSL contexts ------------ .. versionadded:: 3.2 @@ -1451,7 +1456,7 @@ to speed up repeated connections from the same clients. TLS 1.3. .. seealso:: - :func:`create_default_context` lets the :mod:`ssl` module choose + :func:`create_default_context` lets the :mod:`!ssl` module choose security settings for a given purpose. .. versionchanged:: 3.6 @@ -1643,7 +1648,7 @@ to speed up repeated connections from the same clients. provided as part of the operating system, though, it is likely to be configured properly. -.. method:: SSLContext.set_ciphers(ciphers) +.. method:: SSLContext.set_ciphers(ciphers, /) Set the available ciphers for sockets created with this context. It should be a string in the `OpenSSL cipher list format @@ -1659,7 +1664,7 @@ to speed up repeated connections from the same clients. TLS 1.3 cipher suites cannot be disabled with :meth:`~SSLContext.set_ciphers`. -.. method:: SSLContext.set_alpn_protocols(protocols) +.. method:: SSLContext.set_alpn_protocols(alpn_protocols) Specify which protocols the socket should advertise during the SSL/TLS handshake. It should be a list of ASCII strings, like ``['http/1.1', @@ -1673,7 +1678,7 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.5 -.. method:: SSLContext.set_npn_protocols(protocols) +.. method:: SSLContext.set_npn_protocols(npn_protocols) Specify which protocols the socket should advertise during the SSL/TLS handshake. It should be a list of strings, like ``['http/1.1', 'spdy/2']``, @@ -1740,7 +1745,7 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.7 -.. attribute:: SSLContext.set_servername_callback(server_name_callback) +.. method:: SSLContext.set_servername_callback(server_name_callback) This is a legacy API retained for backwards compatibility. When possible, you should use :attr:`sni_callback` instead. The given *server_name_callback* @@ -1754,7 +1759,7 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.4 -.. method:: SSLContext.load_dh_params(dhfile) +.. method:: SSLContext.load_dh_params(dhfile, /) Load the key generation parameters for Diffie-Hellman (DH) key exchange. Using DH key exchange improves forward secrecy at the expense of @@ -1767,7 +1772,7 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.3 -.. method:: SSLContext.set_ecdh_curve(curve_name) +.. method:: SSLContext.set_ecdh_curve(curve_name, /) Set the curve name for Elliptic Curve-based Diffie-Hellman (ECDH) key exchange. ECDH is significantly faster than regular DH while arguably @@ -1846,8 +1851,9 @@ to speed up repeated connections from the same clients. .. attribute:: SSLContext.sslsocket_class The return type of :meth:`SSLContext.wrap_socket`, defaults to - :class:`SSLSocket`. The attribute can be overridden on instance of class - in order to return a custom subclass of :class:`SSLSocket`. + :class:`SSLSocket`. The attribute can be assigned to on instances of + :class:`SSLContext` in order to return a custom subclass of + :class:`SSLSocket`. .. versionadded:: 3.7 @@ -1944,7 +1950,7 @@ to speed up repeated connections from the same clients. :attr:`~SSLContext.minimum_version` and :attr:`SSLContext.options` all affect the supported SSL and TLS versions of the context. The implementation does not prevent - invalid combination. For example a context with + invalid combinations. For example a context with :attr:`OP_NO_TLSv1_2` in :attr:`~SSLContext.options` and :attr:`~SSLContext.maximum_version` set to :attr:`TLSVersion.TLSv1_2` will not be able to establish a TLS 1.2 connection. @@ -2521,7 +2527,7 @@ thus several things you need to be aware of: as well. -Memory BIO Support +Memory BIO support ------------------ .. versionadded:: 3.5 @@ -2640,12 +2646,12 @@ purpose. It wraps an OpenSSL memory BIO (Basic IO) object: A boolean indicating whether the memory BIO is current at the end-of-file position. - .. method:: MemoryBIO.read(n=-1) + .. method:: MemoryBIO.read(n=-1, /) Read up to *n* bytes from the memory buffer. If *n* is not specified or negative, all bytes are returned. - .. method:: MemoryBIO.write(buf) + .. method:: MemoryBIO.write(buf, /) Write the bytes from *buf* to the memory BIO. The *buf* argument must be an object supporting the buffer protocol. @@ -2728,7 +2734,7 @@ This common check is automatically performed when .. versionchanged:: 3.7 Hostname matchings is now performed by OpenSSL. Python no longer uses - :func:`match_hostname`. + :func:`!match_hostname`. In server mode, if you want to authenticate your clients using the SSL layer (rather than using a higher-level authentication mechanism), you'll also have @@ -2747,11 +2753,11 @@ disabled by default. :: >>> client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - >>> client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + >>> client_context.minimum_version = ssl.TLSVersion.TLSv1_2 >>> client_context.maximum_version = ssl.TLSVersion.TLSv1_3 -The SSL context created above will only allow TLSv1.3 and later (if +The SSL client context created above will only allow TLSv1.2 and TLSv1.3 (if supported by your system) connections to a server. :const:`PROTOCOL_TLS_CLIENT` implies certificate validation and hostname checks by default. You have to load certificates into the context. @@ -2819,16 +2825,16 @@ of TLS/SSL. Some new TLS 1.3 features are not yet available. Steve Kent :rfc:`RFC 4086: Randomness Requirements for Security <4086>` - Donald E., Jeffrey I. Schiller + Donald E. Eastlake, Jeffrey I. Schiller, Steve Crocker :rfc:`RFC 5280: Internet X.509 Public Key Infrastructure Certificate and Certificate Revocation List (CRL) Profile <5280>` - D. Cooper + David Cooper et al. :rfc:`RFC 5246: The Transport Layer Security (TLS) Protocol Version 1.2 <5246>` - T. Dierks et. al. + Tim Dierks and Eric Rescorla. :rfc:`RFC 6066: Transport Layer Security (TLS) Extensions <6066>` - D. Eastlake + Donald E. Eastlake `IANA TLS: Transport Layer Security (TLS) Parameters `_ IANA diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst index 8434b2e8c75cf42..bcb1dc269b61071 100644 --- a/Doc/library/stat.rst +++ b/Doc/library/stat.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`stat` module defines constants and functions for interpreting the +The :mod:`!stat` module defines constants and functions for interpreting the results of :func:`os.stat`, :func:`os.fstat` and :func:`os.lstat` (if they exist). For complete details about the :c:func:`stat`, :c:func:`!fstat` and :c:func:`!lstat` calls, consult the documentation for your system. @@ -19,7 +19,7 @@ exist). For complete details about the :c:func:`stat`, :c:func:`!fstat` and .. versionchanged:: 3.4 The stat module is backed by a C implementation. -The :mod:`stat` module defines the following functions to test for specific file +The :mod:`!stat` module defines the following functions to test for specific file types: diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 39aaa5da0786f82..7ad13845f44bad4 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -46,8 +46,10 @@ Any object can be tested for truth value, for use in an :keyword:`if` or By default, an object is considered true unless its class defines either a :meth:`~object.__bool__` method that returns ``False`` or a :meth:`~object.__len__` method that -returns zero, when called with the object. [1]_ Here are most of the built-in -objects considered false: +returns zero, when called with the object. [1]_ If one of the methods raises an +exception when called, the exception is propagated and the object does +not have a truth value (for example, :data:`NotImplemented`). +Here are most of the built-in objects considered false: .. index:: single: None (Built-in object) @@ -164,7 +166,7 @@ This table summarizes the comparison operations: pair: object; numeric pair: objects; comparing -Objects of different types, except different numeric types, never compare equal. +Unless stated otherwise, objects of different types never compare equal. The ``==`` operator is always defined but for some object types (for example, class objects) is equivalent to :keyword:`is`. The ``<``, ``<=``, ``>`` and ``>=`` operators are only defined where they make sense; for example, they raise a @@ -263,9 +265,17 @@ The constructors :func:`int`, :func:`float`, and pair: operator; % (percent) pair: operator; ** +.. _stdtypes-mixed-arithmetic: + Python fully supports mixed arithmetic: when a binary arithmetic operator has -operands of different numeric types, the operand with the "narrower" type is -widened to that of the other, where integer is narrower than floating point. +operands of different built-in numeric types, the operand with the "narrower" +type is widened to that of the other: + +* If both arguments are complex numbers, no conversion is performed; +* if either argument is a complex or a floating-point number, the other is + converted to a floating-point number; +* otherwise, both must be integers and no conversion is necessary. + Arithmetic with complex and real operands is defined by the usual mathematical formula, for example:: @@ -1000,8 +1010,6 @@ operations have the same priority as the corresponding numeric operations. [3]_ pair: slice; operation pair: operator; in pair: operator; not in - single: count() (sequence method) - single: index() (sequence method) +--------------------------+--------------------------------+----------+ | Operation | Result | Notes | @@ -1018,7 +1026,7 @@ operations have the same priority as the corresponding numeric operations. [3]_ | ``s * n`` or | equivalent to adding *s* to | (2)(7) | | ``n * s`` | itself *n* times | | +--------------------------+--------------------------------+----------+ -| ``s[i]`` | *i*\ th item of *s*, origin 0 | \(3) | +| ``s[i]`` | *i*\ th item of *s*, origin 0 | (3)(8) | +--------------------------+--------------------------------+----------+ | ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) | +--------------------------+--------------------------------+----------+ @@ -1031,13 +1039,6 @@ operations have the same priority as the corresponding numeric operations. [3]_ +--------------------------+--------------------------------+----------+ | ``max(s)`` | largest item of *s* | | +--------------------------+--------------------------------+----------+ -| ``s.index(x[, i[, j]])`` | index of the first occurrence | \(8) | -| | of *x* in *s* (at or after | | -| | index *i* and before index *j*)| | -+--------------------------+--------------------------------+----------+ -| ``s.count(x)`` | total number of occurrences of | | -| | *x* in *s* | | -+--------------------------+--------------------------------+----------+ Sequences of the same type also support comparisons. In particular, tuples and lists are compared lexicographically by comparing corresponding elements. @@ -1100,11 +1101,14 @@ Notes: still ``0``. (4) - The slice of *s* from *i* to *j* is defined as the sequence of items with index - *k* such that ``i <= k < j``. If *i* or *j* is greater than ``len(s)``, use - ``len(s)``. If *i* is omitted or ``None``, use ``0``. If *j* is omitted or - ``None``, use ``len(s)``. If *i* is greater than or equal to *j*, the slice is - empty. + The slice of *s* from *i* to *j* is defined as the sequence of items with + index *k* such that ``i <= k < j``. + + * If *i* is omitted or ``None``, use ``0``. + * If *j* is omitted or ``None``, use ``len(s)``. + * If *i* or *j* is less than ``-len(s)``, use ``0``. + * If *i* or *j* is greater than ``len(s)``, use ``len(s)``. + * If *i* is greater than or equal to *j*, the slice is empty. (5) The slice of *s* from *i* to *j* with step *k* is defined as the sequence of @@ -1143,12 +1147,41 @@ Notes: concatenation or repetition. (8) - ``index`` raises :exc:`ValueError` when *x* is not found in *s*. - Not all implementations support passing the additional arguments *i* and *j*. - These arguments allow efficient searching of subsections of the sequence. Passing - the extra arguments is roughly equivalent to using ``s[i:j].index(x)``, only - without copying any data and with the returned index being relative to - the start of the sequence rather than the start of the slice. + An :exc:`IndexError` is raised if *i* is outside the sequence range. + +.. rubric:: Sequence Methods + +Sequence types also support the following methods: + +.. method:: list.count(value, /) + range.count(value, /) + tuple.count(value, /) + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.count(value, /) + + Return the total number of occurrences of *value* in *sequence*. + +.. method:: list.index(value[, start[, stop]]) + range.index(value[, start[, stop]]) + tuple.index(value[, start[, stop]]) + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.index(value[, start[, stop]]) + + Return the index of the first occurrence of *value* in *sequence*. + + Raises :exc:`ValueError` if *value* is not found in *sequence*. + + The *start* or *stop* arguments allow for efficient searching + of subsections of the sequence, beginning at *start* and ending at *stop*. + This is roughly equivalent to ``start + sequence[start:stop].index(value)``, + only without copying any data. + + .. caution:: + Not all sequence types support passing the *start* and *stop* arguments. .. _typesseq-immutable: @@ -1199,14 +1232,6 @@ accepts integers that meet the value restriction ``0 <= x <= 255``). pair: subscript; assignment pair: slice; assignment pair: statement; del - single: append() (sequence method) - single: clear() (sequence method) - single: copy() (sequence method) - single: extend() (sequence method) - single: insert() (sequence method) - single: pop() (sequence method) - single: remove() (sequence method) - single: reverse() (sequence method) +------------------------------+--------------------------------+---------------------+ | Operation | Result | Notes | @@ -1214,11 +1239,15 @@ accepts integers that meet the value restriction ``0 <= x <= 255``). | ``s[i] = x`` | item *i* of *s* is replaced by | | | | *x* | | +------------------------------+--------------------------------+---------------------+ +| ``del s[i]`` | removes item *i* of *s* | | ++------------------------------+--------------------------------+---------------------+ | ``s[i:j] = t`` | slice of *s* from *i* to *j* | | | | is replaced by the contents of | | | | the iterable *t* | | +------------------------------+--------------------------------+---------------------+ -| ``del s[i:j]`` | same as ``s[i:j] = []`` | | +| ``del s[i:j]`` | removes the elements of | | +| | ``s[i:j]`` from the list | | +| | (same as ``s[i:j] = []``) | | +------------------------------+--------------------------------+---------------------+ | ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` | \(1) | | | are replaced by those of *t* | | @@ -1226,39 +1255,14 @@ accepts integers that meet the value restriction ``0 <= x <= 255``). | ``del s[i:j:k]`` | removes the elements of | | | | ``s[i:j:k]`` from the list | | +------------------------------+--------------------------------+---------------------+ -| ``s.append(x)`` | appends *x* to the end of the | | -| | sequence (same as | | -| | ``s[len(s):len(s)] = [x]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.clear()`` | removes all items from *s* | \(5) | -| | (same as ``del s[:]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.copy()`` | creates a shallow copy of *s* | \(5) | -| | (same as ``s[:]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.extend(t)`` or | extends *s* with the | | -| ``s += t`` | contents of *t* (for the | | +| ``s += t`` | extends *s* with the | | +| | contents of *t* (for the | | | | most part the same as | | | | ``s[len(s):len(s)] = t``) | | +------------------------------+--------------------------------+---------------------+ -| ``s *= n`` | updates *s* with its contents | \(6) | +| ``s *= n`` | updates *s* with its contents | \(2) | | | repeated *n* times | | +------------------------------+--------------------------------+---------------------+ -| ``s.insert(i, x)`` | inserts *x* into *s* at the | | -| | index given by *i* | | -| | (same as ``s[i:i] = [x]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.pop()`` or ``s.pop(i)`` | retrieves the item at *i* and | \(2) | -| | also removes it from *s* | | -+------------------------------+--------------------------------+---------------------+ -| ``s.remove(x)`` | removes the first item from | \(3) | -| | *s* where ``s[i]`` is equal to | | -| | *x* | | -+------------------------------+--------------------------------+---------------------+ -| ``s.reverse()`` | reverses the items of *s* in | \(4) | -| | place | | -+------------------------------+--------------------------------+---------------------+ - Notes: @@ -1266,32 +1270,105 @@ Notes: If *k* is not equal to ``1``, *t* must have the same length as the slice it is replacing. (2) - The optional argument *i* defaults to ``-1``, so that by default the last - item is removed and returned. + The value *n* is an integer, or an object implementing + :meth:`~object.__index__`. Zero and negative values of *n* clear + the sequence. Items in the sequence are not copied; they are referenced + multiple times, as explained for ``s * n`` under :ref:`typesseq-common`. -(3) - :meth:`remove` raises :exc:`ValueError` when *x* is not found in *s*. +.. rubric:: Mutable Sequence Methods -(4) - The :meth:`reverse` method modifies the sequence in place for economy of - space when reversing a large sequence. To remind users that it operates by - side effect, it does not return the reversed sequence. +Mutable sequence types also support the following methods: -(5) - :meth:`clear` and :meth:`!copy` are included for consistency with the - interfaces of mutable containers that don't support slicing operations - (such as :class:`dict` and :class:`set`). :meth:`!copy` is not part of the - :class:`collections.abc.MutableSequence` ABC, but most concrete - mutable sequence classes provide it. +.. method:: bytearray.append(value, /) + list.append(value, /) + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.append(value, /) + + Append *value* to the end of the sequence. + This is equivalent to writing ``seq[len(seq):len(seq)] = [value]``. + +.. method:: bytearray.clear() + list.clear() + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.clear() .. versionadded:: 3.3 - :meth:`clear` and :meth:`!copy` methods. -(6) - The value *n* is an integer, or an object implementing - :meth:`~object.__index__`. Zero and negative values of *n* clear - the sequence. Items in the sequence are not copied; they are referenced - multiple times, as explained for ``s * n`` under :ref:`typesseq-common`. + Remove all items from *sequence*. + This is equivalent to writing ``del sequence[:]``. + +.. method:: bytearray.copy() + list.copy() + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.copy() + + .. versionadded:: 3.3 + + Create a shallow copy of *sequence*. + This is equivalent to writing ``sequence[:]``. + + .. hint:: The :meth:`!copy` method is not part of the + :class:`~collections.abc.MutableSequence` :class:`~abc.ABC`, + but most concrete mutable sequence types provide it. + +.. method:: bytearray.extend(iterable, /) + list.extend(iterable, /) + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.extend(iterable, /) + + Extend *sequence* with the contents of *iterable*. + For the most part, this is the same as writing + ``seq[len(seq):len(seq)] = iterable``. + +.. method:: bytearray.insert(index, value, /) + list.insert(index, value, /) + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.insert(index, value, /) + + Insert *value* into *sequence* at the given *index*. + This is equivalent to writing ``sequence[index:index] = [value]``. + +.. method:: bytearray.pop(index=-1, /) + list.pop(index=-1, /) + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.pop(index=-1, /) + + Retrieve the item at *index* and also removes it from *sequence*. + By default, the last item in *sequence* is removed and returned. + +.. method:: bytearray.remove(value, /) + list.remove(value, /) + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.remove(value, /) + + Remove the first item from *sequence* where ``sequence[i] == value``. + + Raises :exc:`ValueError` if *value* is not found in *sequence*. + +.. method:: bytearray.reverse() + list.reverse() + :no-contents-entry: + :no-index-entry: + :no-typesetting: +.. method:: sequence.reverse() + + Reverse the items of *sequence* in place. + This method maintains economy of space when reversing a large sequence. + To remind users that it operates by side-effect, it returns ``None``. .. _typesseq-list: @@ -1305,7 +1382,7 @@ Lists are mutable sequences, typically used to store collections of homogeneous items (where the precise degree of similarity will vary by application). -.. class:: list([iterable]) +.. class:: list(iterable=(), /) Lists may be constructed in several ways: @@ -1326,6 +1403,8 @@ application). Many other operations also produce lists, including the :func:`sorted` built-in. + Lists are :ref:`generic ` over the types of their items. + Lists implement all of the :ref:`common ` and :ref:`mutable ` sequence operations. Lists also provide the following additional method: @@ -1372,6 +1451,11 @@ application). list appear empty for the duration, and raises :exc:`ValueError` if it can detect that the list has been mutated during a sort. +.. seealso:: + + For detailed information on thread-safety guarantees for :class:`list` + objects, see :ref:`thread-safety-list`. + .. _typesseq-tuple: @@ -1386,7 +1470,7 @@ built-in). Tuples are also used for cases where an immutable sequence of homogeneous data is needed (such as allowing storage in a :class:`set` or :class:`dict` instance). -.. class:: tuple([iterable]) +.. class:: tuple(iterable=(), /) Tuples may be constructed in a number of ways: @@ -1412,6 +1496,10 @@ homogeneous data is needed (such as allowing storage in a :class:`set` or Tuples implement all of the :ref:`common ` sequence operations. + Tuples are :ref:`generic ` over the types of their contents. + For more information, refer to + :ref:`the typing documentation on annotating tuples `. + For heterogeneous collections of data where access by name is clearer than access by index, :func:`collections.namedtuple` may be a more appropriate choice than a simple tuple object. @@ -1428,8 +1516,8 @@ The :class:`range` type represents an immutable sequence of numbers and is commonly used for looping a specific number of times in :keyword:`for` loops. -.. class:: range(stop) - range(start, stop[, step]) +.. class:: range(stop, /) + range(start, stop, step=1, /) The arguments to the range constructor must be integers (either built-in :class:`int` or any object that implements the :meth:`~object.__index__` special @@ -1632,7 +1720,11 @@ category. | | :meth:`str.strip` | :meth:`bytes.strip` | | +--------------------+----------------------+----------------------+----------------------------+ | | :meth:`str.lstrip` | :meth:`str.rstrip` | :meth:`bytes.lstrip` | :meth:`bytes.rstrip` | -+--------------------------+--------------------+----------------------+----------------------+----------------------------+ +| +--------------------+----------------------+----------------------+----------------------------+ +| | :meth:`str.removeprefix` | :meth:`bytes.removeprefix` | +| +-------------------------------------------+---------------------------------------------------+ +| | :meth:`str.removesuffix` | :meth:`bytes.removesuffix` | ++--------------------------+-------------------------------------------+---------------------------------------------------+ | Translation and Encoding | :meth:`str.translate` | :meth:`bytes.translate` | | +-------------------------------------------+---------------------------------------------------+ | | :meth:`str.maketrans` | :meth:`bytes.maketrans` | @@ -1689,8 +1781,10 @@ multiple fragments. .. index:: single: string; str (built-in class) -.. class:: str(object='') - str(object=b'', encoding='utf-8', errors='strict') +.. class:: str(*, encoding='utf-8', errors='strict') + str(object) + str(object, encoding, errors='strict') + str(object, *, errors) Return a :ref:`string ` version of *object*. If *object* is not provided, returns the empty string. Otherwise, the behavior of ``str()`` @@ -1776,6 +1870,14 @@ expression support in the :mod:`re` module). lowercase letter ``'ß'`` is equivalent to ``"ss"``. Since it is already lowercase, :meth:`lower` would do nothing to ``'ß'``; :meth:`casefold` converts it to ``"ss"``. + For example: + + .. doctest:: + + >>> 'straße'.lower() + 'straße' + >>> 'straße'.casefold() + 'strasse' The casefolding algorithm is `described in section 3.13 'Default Case Folding' of the Unicode Standard @@ -1784,12 +1886,18 @@ expression support in the :mod:`re` module). .. versionadded:: 3.3 -.. method:: str.center(width[, fillchar]) +.. method:: str.center(width, fillchar=' ', /) Return centered in a string of length *width*. Padding is done using the specified *fillchar* (default is an ASCII space). The original string is - returned if *width* is less than or equal to ``len(s)``. + returned if *width* is less than or equal to ``len(s)``. For example:: + >>> 'Python'.center(10) + ' Python ' + >>> 'Python'.center(10, '-') + '--Python--' + >>> 'Python'.center(4) + 'Python' .. method:: str.count(sub[, start[, end]]) @@ -1799,8 +1907,18 @@ expression support in the :mod:`re` module). interpreted as in slice notation. If *sub* is empty, returns the number of empty strings between characters - which is the length of the string plus one. - + which is the length of the string plus one. For example:: + + >>> 'spam, spam, spam'.count('spam') + 3 + >>> 'spam, spam, spam'.count('spam', 5) + 2 + >>> 'spam, spam, spam'.count('spam', 5, 10) + 1 + >>> 'spam, spam, spam'.count('eggs') + 0 + >>> 'spam, spam, spam'.count('') + 17 .. method:: str.encode(encoding="utf-8", errors="strict") @@ -1820,6 +1938,14 @@ expression support in the :mod:`re` module). unless an encoding error actually occurs, :ref:`devmode` is enabled or a :ref:`debug build ` is used. + For example:: + + >>> encoded_str_to_bytes = 'Python'.encode() + >>> type(encoded_str_to_bytes) + + >>> encoded_str_to_bytes + b'Python' + .. versionchanged:: 3.1 Added support for keyword arguments. @@ -1834,7 +1960,19 @@ expression support in the :mod:`re` module). Return ``True`` if the string ends with the specified *suffix*, otherwise return ``False``. *suffix* can also be a tuple of suffixes to look for. With optional *start*, test beginning at that position. With optional *end*, stop comparing - at that position. + at that position. Using *start* and *end* is equivalent to + ``str[start:end].endswith(suffix)``. For example:: + + >>> 'Python'.endswith('on') + True + >>> 'a tuple of suffixes'.endswith(('at', 'in')) + False + >>> 'a tuple of suffixes'.endswith(('at', 'es')) + True + >>> 'Python is amazing'.endswith('is', 0, 9) + True + + See also :meth:`startswith` and :meth:`removesuffix`. .. method:: str.expandtabs(tabsize=8) @@ -1850,12 +1988,15 @@ expression support in the :mod:`re` module). (``\n``) or return (``\r``), it is copied and the current column is reset to zero. Any other character is copied unchanged and the current column is incremented by one regardless of how the character is represented when - printed. + printed. For example:: >>> '01\t012\t0123\t01234'.expandtabs() '01 012 0123 01234' >>> '01\t012\t0123\t01234'.expandtabs(4) '01 012 0123 01234' + >>> print('01\t012\n0123\t01234'.expandtabs(4)) + 01 012 + 0123 01234 .. method:: str.find(sub[, start[, end]]) @@ -1863,6 +2004,14 @@ expression support in the :mod:`re` module). Return the lowest index in the string where substring *sub* is found within the slice ``s[start:end]``. Optional arguments *start* and *end* are interpreted as in slice notation. Return ``-1`` if *sub* is not found. + For example:: + + >>> 'spam, spam, spam'.find('sp') + 0 + >>> 'spam, spam, spam'.find('sp', 5) + 6 + + See also :meth:`rfind` and :meth:`index`. .. note:: @@ -1881,10 +2030,16 @@ expression support in the :mod:`re` module). ``{}``. Each replacement field contains either the numeric index of a positional argument, or the name of a keyword argument. Returns a copy of the string where each replacement field is replaced with the string value of - the corresponding argument. + the corresponding argument. For example: + + .. doctest:: >>> "The sum of 1 + 2 is {0}".format(1+2) 'The sum of 1 + 2 is 3' + >>> "The sum of {a} + {b} is {answer}".format(answer=1+2, a=1, b=2) + 'The sum of 1 + 2 is 3' + >>> "{1} expects the {0} Inquisition!".format("Spanish", "Nobody") + 'Nobody expects the Spanish Inquisition!' See :ref:`formatstrings` for a description of the various formatting options that can be specified in format strings. @@ -1924,7 +2079,20 @@ expression support in the :mod:`re` module). .. method:: str.index(sub[, start[, end]]) Like :meth:`~str.find`, but raise :exc:`ValueError` when the substring is - not found. + not found. For example: + + .. doctest:: + + >>> 'spam, spam, spam'.index('spam') + 0 + >>> 'spam, spam, spam'.index('eggs') + Traceback (most recent call last): + File "", line 1, in + 'spam, spam, spam'.index('eggs') + ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ + ValueError: substring not found + + See also :meth:`rindex`. .. method:: str.isalnum() @@ -1932,7 +2100,18 @@ expression support in the :mod:`re` module). Return ``True`` if all characters in the string are alphanumeric and there is at least one character, ``False`` otherwise. A character ``c`` is alphanumeric if one of the following returns ``True``: ``c.isalpha()``, ``c.isdecimal()``, - ``c.isdigit()``, or ``c.isnumeric()``. + ``c.isdigit()``, or ``c.isnumeric()``. For example: + + .. doctest:: + + >>> 'abc123'.isalnum() + True + >>> 'abc123!@#'.isalnum() + False + >>> ''.isalnum() + False + >>> ' '.isalnum() + False .. method:: str.isalpha() @@ -1944,13 +2123,32 @@ expression support in the :mod:`re` module). from the `Alphabetic property defined in the section 4.10 'Letters, Alphabetic, and Ideographic' of the Unicode Standard `_. + For example: + + .. doctest:: + + >>> 'Letters and spaces'.isalpha() + False + >>> 'LettersOnly'.isalpha() + True + >>> 'µ'.isalpha() # non-ASCII characters can be considered alphabetical too + True + + See :ref:`unicode-properties`. .. method:: str.isascii() Return ``True`` if the string is empty or all characters in the string are ASCII, ``False`` otherwise. - ASCII characters have code points in the range U+0000-U+007F. + ASCII characters have code points in the range U+0000-U+007F. For example: + + .. doctest:: + + >>> 'ASCII characters'.isascii() + True + >>> 'µ'.isascii() + False .. versionadded:: 3.7 @@ -1960,9 +2158,18 @@ expression support in the :mod:`re` module). Return ``True`` if all characters in the string are decimal characters and there is at least one character, ``False`` otherwise. Decimal characters are those that can be used to form - numbers in base 10, e.g. U+0660, ARABIC-INDIC DIGIT + numbers in base 10, such as U+0660, ARABIC-INDIC DIGIT ZERO. Formally a decimal character is a character in the Unicode - General Category "Nd". + General Category "Nd". For example: + + .. doctest:: + + >>> '0123456789'.isdecimal() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isdecimal() # Arabic-Indic digits zero to nine + True + >>> 'alphabetic'.isdecimal() + False .. method:: str.isdigit() @@ -1971,9 +2178,25 @@ expression support in the :mod:`re` module). character, ``False`` otherwise. Digits include decimal characters and digits that need special handling, such as the compatibility superscript digits. This covers digits which cannot be used to form numbers in base 10, - like the Kharosthi numbers. Formally, a digit is a character that has the + like the `Kharosthi numbers `__. + Formally, a digit is a character that has the property value Numeric_Type=Digit or Numeric_Type=Decimal. + For example: + + .. doctest:: + + >>> '0123456789'.isdigit() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isdigit() # Arabic-Indic digits zero to nine + True + >>> '⅕'.isdigit() # Vulgar fraction one fifth + False + >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() + (False, True, True) + + See also :meth:`isdecimal` and :meth:`isnumeric`. + .. method:: str.isidentifier() @@ -2008,11 +2231,25 @@ expression support in the :mod:`re` module). that have the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION ONE FIFTH. Formally, numeric characters are those with the property value Numeric_Type=Digit, Numeric_Type=Decimal or Numeric_Type=Numeric. + For example: + + .. doctest:: + + >>> '0123456789'.isnumeric() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isnumeric() # Arabic-Indic digits zero to nine + True + >>> '⅕'.isnumeric() # Vulgar fraction one fifth + True + >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() + (False, True, True) + + See also :meth:`isdecimal` and :meth:`isdigit`. .. method:: str.isprintable() - Return true if all characters in the string are printable, false if it + Return ``True`` if all characters in the string are printable, ``False`` if it contains at least one non-printable character. Here "printable" means the character is suitable for :func:`repr` to use in @@ -2026,17 +2263,43 @@ expression support in the :mod:`re` module). Nonprintable characters are those in group Separator or Other (Z or C), except the ASCII space. + For example: + + .. doctest:: + + >>> ''.isprintable(), ' '.isprintable() + (True, True) + >>> '\t'.isprintable(), '\n'.isprintable() + (False, False) + + See also :meth:`isspace`. + .. method:: str.isspace() Return ``True`` if there are only whitespace characters in the string and there is at least one character, ``False`` otherwise. + For example: + + .. doctest:: + + >>> ''.isspace() + False + >>> ' '.isspace() + True + >>> '\t\n'.isspace() # TAB and BREAK LINE + True + >>> '\u3000'.isspace() # IDEOGRAPHIC SPACE + True + A character is *whitespace* if in the Unicode character database (see :mod:`unicodedata`), either its general category is ``Zs`` ("Separator, space"), or its bidirectional class is one of ``WS``, ``B``, or ``S``. + See also :meth:`isprintable`. + .. method:: str.istitle() @@ -2044,6 +2307,19 @@ expression support in the :mod:`re` module). character, for example uppercase characters may only follow uncased characters and lowercase characters only cased ones. Return ``False`` otherwise. + For example: + + .. doctest:: + + >>> 'Spam, Spam, Spam'.istitle() + True + >>> 'spam, spam, spam'.istitle() + False + >>> 'SPAM, SPAM, SPAM'.istitle() + False + + See also :meth:`title`. + .. method:: str.isupper() @@ -2063,32 +2339,59 @@ expression support in the :mod:`re` module). .. _meth-str-join: -.. method:: str.join(iterable) +.. method:: str.join(iterable, /) Return a string which is the concatenation of the strings in *iterable*. A :exc:`TypeError` will be raised if there are any non-string values in *iterable*, including :class:`bytes` objects. The separator between - elements is the string providing this method. + elements is the string providing this method. For example: + + .. doctest:: + + >>> ', '.join(['spam', 'spam', 'spam']) + 'spam, spam, spam' + >>> '-'.join('Python') + 'P-y-t-h-o-n' + See also :meth:`split`. -.. method:: str.ljust(width[, fillchar]) + +.. method:: str.ljust(width, fillchar=' ', /) Return the string left justified in a string of length *width*. Padding is done using the specified *fillchar* (default is an ASCII space). The original string is returned if *width* is less than or equal to ``len(s)``. + For example: + + .. doctest:: + + >>> 'Python'.ljust(10) + 'Python ' + >>> 'Python'.ljust(10, '.') + 'Python....' + >>> 'Monty Python'.ljust(10, '.') + 'Monty Python' + + See also :meth:`rjust`. + .. method:: str.lower() Return a copy of the string with all the cased characters [4]_ converted to - lowercase. + lowercase. For example: + + .. doctest:: + + >>> 'Lower Method Example'.lower() + 'lower method example' The lowercasing algorithm used is `described in section 3.13 'Default Case Folding' of the Unicode Standard `__. -.. method:: str.lstrip([chars]) +.. method:: str.lstrip(chars=None, /) Return a copy of the string with leading characters removed. The *chars* argument is a string specifying the set of characters to be removed. If omitted @@ -2109,7 +2412,8 @@ expression support in the :mod:`re` module). 'three!' -.. staticmethod:: str.maketrans(x[, y[, z]]) +.. staticmethod:: str.maketrans(dict, /) + str.maketrans(from, to, remove='', /) This static method returns a translation table usable for :meth:`str.translate`. @@ -2119,24 +2423,39 @@ expression support in the :mod:`re` module). converted to ordinals. If there are two arguments, they must be strings of equal length, and in the - resulting dictionary, each character in x will be mapped to the character at - the same position in y. If there is a third argument, it must be a string, + resulting dictionary, each character in *from* will be mapped to the character at + the same position in *to*. If there is a third argument, it must be a string, whose characters will be mapped to ``None`` in the result. -.. method:: str.partition(sep) +.. method:: str.partition(sep, /) Split the string at the first occurrence of *sep*, and return a 3-tuple containing the part before the separator, the separator itself, and the part after the separator. If the separator is not found, return a 3-tuple containing the string itself, followed by two empty strings. + For example: + + .. doctest:: + + >>> 'Monty Python'.partition(' ') + ('Monty', ' ', 'Python') + >>> "Monty Python's Flying Circus".partition(' ') + ('Monty', ' ', "Python's Flying Circus") + >>> 'Monty Python'.partition('-') + ('Monty Python', '', '') + + See also :meth:`rpartition`. + .. method:: str.removeprefix(prefix, /) If the string starts with the *prefix* string, return ``string[len(prefix):]``. Otherwise, return a copy of the original - string:: + string: + + .. doctest:: >>> 'TestHook'.removeprefix('Test') 'Hook' @@ -2145,12 +2464,16 @@ expression support in the :mod:`re` module). .. versionadded:: 3.9 + See also :meth:`removesuffix` and :meth:`startswith`. + .. method:: str.removesuffix(suffix, /) If the string ends with the *suffix* string and that *suffix* is not empty, return ``string[:-len(suffix)]``. Otherwise, return a copy of the - original string:: + original string: + + .. doctest:: >>> 'MiscTests'.removesuffix('Tests') 'Misc' @@ -2159,12 +2482,22 @@ expression support in the :mod:`re` module). .. versionadded:: 3.9 + See also :meth:`removeprefix` and :meth:`endswith`. -.. method:: str.replace(old, new, count=-1) + +.. method:: str.replace(old, new, /, count=-1) Return a copy of the string with all occurrences of substring *old* replaced by *new*. If *count* is given, only the first *count* occurrences are replaced. If *count* is not specified or ``-1``, then all occurrences are replaced. + For example: + + .. doctest:: + + >>> 'spam, spam, spam'.replace('spam', 'eggs') + 'eggs, eggs, eggs' + >>> 'spam, spam, spam'.replace('spam', 'eggs', 1) + 'eggs, spam, spam' .. versionchanged:: 3.13 *count* is now supported as a keyword argument. @@ -2175,28 +2508,78 @@ expression support in the :mod:`re` module). Return the highest index in the string where substring *sub* is found, such that *sub* is contained within ``s[start:end]``. Optional arguments *start* and *end* are interpreted as in slice notation. Return ``-1`` on failure. + For example: + + .. doctest:: + + >>> 'spam, spam, spam'.rfind('sp') + 12 + >>> 'spam, spam, spam'.rfind('sp', 0, 10) + 6 + + See also :meth:`find` and :meth:`rindex`. .. method:: str.rindex(sub[, start[, end]]) Like :meth:`rfind` but raises :exc:`ValueError` when the substring *sub* is not found. + For example: + + .. doctest:: + + >>> 'spam, spam, spam'.rindex('spam') + 12 + >>> 'spam, spam, spam'.rindex('eggs') + Traceback (most recent call last): + File "", line 1, in + 'spam, spam, spam'.rindex('eggs') + ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ + ValueError: substring not found + + See also :meth:`index` and :meth:`find`. -.. method:: str.rjust(width[, fillchar]) +.. method:: str.rjust(width, fillchar=' ', /) Return the string right justified in a string of length *width*. Padding is done using the specified *fillchar* (default is an ASCII space). The original string is returned if *width* is less than or equal to ``len(s)``. + For example: -.. method:: str.rpartition(sep) + .. doctest:: + + >>> 'Python'.rjust(10) + ' Python' + >>> 'Python'.rjust(10, '.') + '....Python' + >>> 'Monty Python'.rjust(10, '.') + 'Monty Python' + + See also :meth:`ljust` and :meth:`zfill`. + + +.. method:: str.rpartition(sep, /) Split the string at the last occurrence of *sep*, and return a 3-tuple containing the part before the separator, the separator itself, and the part after the separator. If the separator is not found, return a 3-tuple containing two empty strings, followed by the string itself. + For example: + + .. doctest:: + + >>> 'Monty Python'.rpartition(' ') + ('Monty', ' ', 'Python') + >>> "Monty Python's Flying Circus".rpartition(' ') + ("Monty Python's Flying", ' ', 'Circus') + >>> 'Monty Python'.rpartition('-') + ('', '', 'Monty Python') + + See also :meth:`partition`. + .. method:: str.rsplit(sep=None, maxsplit=-1) @@ -2207,19 +2590,22 @@ expression support in the :mod:`re` module). :meth:`split` which is described in detail below. -.. method:: str.rstrip([chars]) +.. method:: str.rstrip(chars=None, /) Return a copy of the string with trailing characters removed. The *chars* argument is a string specifying the set of characters to be removed. If omitted or ``None``, the *chars* argument defaults to removing whitespace. The *chars* - argument is not a suffix; rather, all combinations of its values are stripped:: + argument is not a suffix; rather, all combinations of its values are stripped. + For example: + + .. doctest:: >>> ' spacious '.rstrip() ' spacious' >>> 'mississippi'.rstrip('ipz') 'mississ' - See :meth:`str.removesuffix` for a method that will remove a single suffix + See :meth:`removesuffix` for a method that will remove a single suffix string rather than all of a set of characters. For example:: >>> 'Monty Python'.rstrip(' Python') @@ -2227,6 +2613,9 @@ expression support in the :mod:`re` module). >>> 'Monty Python'.removesuffix(' Python') 'Monty' + See also :meth:`strip`. + + .. method:: str.split(sep=None, maxsplit=-1) Return a list of the words in the string, using *sep* as the delimiter @@ -2242,7 +2631,9 @@ expression support in the :mod:`re` module). :func:`re.split`). Splitting an empty string with a specified separator returns ``['']``. - For example:: + For example: + + .. doctest:: >>> '1,2,3'.split(',') ['1', '2', '3'] @@ -2260,7 +2651,9 @@ expression support in the :mod:`re` module). string or a string consisting of just whitespace with a ``None`` separator returns ``[]``. - For example:: + For example: + + .. doctest:: >>> '1 2 3'.split() ['1', '2', '3'] @@ -2269,6 +2662,22 @@ expression support in the :mod:`re` module). >>> ' 1 2 3 '.split() ['1', '2', '3'] + If *sep* is not specified or is ``None`` and *maxsplit* is ``0``, only + leading runs of consecutive whitespace are considered. + + For example: + + .. doctest:: + + >>> "".split(None, 0) + [] + >>> " ".split(None, 0) + [] + >>> " foo ".split(maxsplit=0) + ['foo '] + + See also :meth:`join` and :meth:`rsplit`. + .. index:: single: universal newlines; str.splitlines method @@ -2343,14 +2752,33 @@ expression support in the :mod:`re` module). test string beginning at that position. With optional *end*, stop comparing string at that position. + For example: -.. method:: str.strip([chars]) + .. doctest:: + + >>> 'Python'.startswith('Py') + True + >>> 'a tuple of prefixes'.startswith(('at', 'a')) + True + >>> 'Python is amazing'.startswith('is', 7) + True + + See also :meth:`endswith` and :meth:`removeprefix`. + + +.. method:: str.strip(chars=None, /) Return a copy of the string with the leading and trailing characters removed. The *chars* argument is a string specifying the set of characters to be removed. If omitted or ``None``, the *chars* argument defaults to removing whitespace. The *chars* argument is not a prefix or suffix; rather, all combinations of its - values are stripped:: + values are stripped. + + Whitespace characters are defined by :meth:`str.isspace`. + + For example: + + .. doctest:: >>> ' spacious '.strip() 'spacious' @@ -2361,18 +2789,37 @@ expression support in the :mod:`re` module). from the string. Characters are removed from the leading end until reaching a string character that is not contained in the set of characters in *chars*. A similar action takes place on the trailing end. - For example:: + + For example: + + .. doctest:: >>> comment_string = '#....... Section 3.2.1 Issue #32 .......' >>> comment_string.strip('.#! ') 'Section 3.2.1 Issue #32' + See also :meth:`rstrip`. + .. method:: str.swapcase() Return a copy of the string with uppercase characters converted to lowercase and - vice versa. Note that it is not necessarily true that - ``s.swapcase().swapcase() == s``. + vice versa. For example: + + .. doctest:: + + >>> 'Hello World'.swapcase() + 'hELLO wORLD' + + Note that it is not necessarily true that ``s.swapcase().swapcase() == s``. + For example: + + .. doctest:: + + >>> 'straße'.swapcase().swapcase() + 'strasse' + + See also :meth:`str.lower` and :meth:`str.upper`. .. method:: str.title() @@ -2408,8 +2855,10 @@ expression support in the :mod:`re` module). >>> titlecase("they're bill's friends.") "They're Bill's Friends." + See also :meth:`istitle`. -.. method:: str.translate(table) + +.. method:: str.translate(table, /) Return a copy of the string in which each character has been mapped through the given translation table. The table must be an object that implements @@ -2440,7 +2889,7 @@ expression support in the :mod:`re` module). `__. -.. method:: str.zfill(width) +.. method:: str.zfill(width, /) Return a copy of the string left filled with ASCII ``'0'`` digits to make a string of length *width*. A leading sign prefix (``'+'``/``'-'``) @@ -2448,13 +2897,17 @@ expression support in the :mod:`re` module). than before. The original string is returned if *width* is less than or equal to ``len(s)``. - For example:: + For example: + + .. doctest:: >>> "42".zfill(5) '00042' >>> "-42".zfill(5) '-0042' + See also :meth:`rjust`. + .. index:: single: ! formatted string literal @@ -2470,6 +2923,8 @@ expression support in the :mod:`re` module). single: : (colon); in formatted string literal single: = (equals); for help in debugging using string literals +.. _stdtypes-fstrings: + Formatted String Literals (f-strings) ------------------------------------- @@ -2478,123 +2933,147 @@ Formatted String Literals (f-strings) The :keyword:`await` and :keyword:`async for` can be used in expressions within f-strings. .. versionchanged:: 3.8 - Added the debugging operator (``=``) + Added the debug specifier (``=``) .. versionchanged:: 3.12 Many restrictions on expressions within f-strings have been removed. Notably, nested strings, comments, and backslashes are now permitted. An :dfn:`f-string` (formally a :dfn:`formatted string literal`) is a string literal that is prefixed with ``f`` or ``F``. -This type of string literal allows embedding arbitrary Python expressions -within *replacement fields*, which are delimited by curly brackets (``{}``). -These expressions are evaluated at runtime, similarly to :meth:`str.format`, -and are converted into regular :class:`str` objects. -For example: +This type of string literal allows embedding the results of arbitrary Python +expressions within *replacement fields*, which are delimited by curly +brackets (``{}``). +Each replacement field must contain an expression, optionally followed by: -.. doctest:: +* a *debug specifier* -- an equal sign (``=``); +* a *conversion specifier* -- ``!s``, ``!r`` or ``!a``; and/or +* a *format specifier* prefixed with a colon (``:``). - >>> who = 'nobody' - >>> nationality = 'Spanish' - >>> f'{who.title()} expects the {nationality} Inquisition!' - 'Nobody expects the Spanish Inquisition!' +See the :ref:`Lexical Analysis section on f-strings ` for details +on the syntax of these fields. -It is also possible to use a multi line f-string: +Debug specifier +^^^^^^^^^^^^^^^ -.. doctest:: +.. versionadded:: 3.8 - >>> f'''This is a string - ... on two lines''' - 'This is a string\non two lines' +If a debug specifier -- an equal sign (``=``) -- appears after the replacement +field expression, the resulting f-string will contain the expression's source, +the equal sign, and the value of the expression. +This is often useful for debugging:: -A single opening curly bracket, ``'{'``, marks a *replacement field* that -can contain any Python expression: + >>> number = 14.3 + >>> f'{number=}' + 'number=14.3' -.. doctest:: +Whitespace before, inside and after the expression, as well as whitespace +after the equal sign, is significant --- it is retained in the result:: - >>> nationality = 'Spanish' - >>> f'The {nationality} Inquisition!' - 'The Spanish Inquisition!' + >>> f'{ number - 4 = }' + ' number - 4 = 10.3' -To include a literal ``{`` or ``}``, use a double bracket: -.. doctest:: +Conversion specifier +^^^^^^^^^^^^^^^^^^^^ - >>> x = 42 - >>> f'{{x}} is {x}' - '{x} is 42' - -Functions can also be used, and :ref:`format specifiers `: - -.. doctest:: - - >>> from math import sqrt - >>> f'√2 \N{ALMOST EQUAL TO} {sqrt(2):.5f}' - '√2 ≈ 1.41421' - -Any non-string expression is converted using :func:`str`, by default: - -.. doctest:: +By default, the value of a replacement field expression is converted to +a string using :func:`str`:: >>> from fractions import Fraction - >>> f'{Fraction(1, 3)}' + >>> one_third = Fraction(1, 3) + >>> f'{one_third}' '1/3' -To use an explicit conversion, use the ``!`` (exclamation mark) operator, -followed by any of the valid formats, which are: +When a debug specifier but no format specifier is used, the default conversion +instead uses :func:`repr`:: -========== ============== -Conversion Meaning -========== ============== -``!a`` :func:`ascii` -``!r`` :func:`repr` -``!s`` :func:`str` -========== ============== + >>> f'{one_third = }' + 'one_third = Fraction(1, 3)' -For example: +The conversion can be specified explicitly using one of these specifiers: -.. doctest:: +* ``!s`` for :func:`str` +* ``!r`` for :func:`repr` +* ``!a`` for :func:`ascii` - >>> from fractions import Fraction - >>> f'{Fraction(1, 3)!s}' +For example:: + + >>> str(one_third) '1/3' - >>> f'{Fraction(1, 3)!r}' + >>> repr(one_third) 'Fraction(1, 3)' - >>> question = '¿Dónde está el Presidente?' - >>> print(f'{question!a}') - '\xbfD\xf3nde est\xe1 el Presidente?' - -While debugging it may be helpful to see both the expression and its value, -by using the equals sign (``=``) after the expression. -This preserves spaces within the brackets, and can be used with a converter. -By default, the debugging operator uses the :func:`repr` (``!r``) conversion. -For example: -.. doctest:: + >>> f'{one_third!s} is {one_third!r}' + '1/3 is Fraction(1, 3)' - >>> from fractions import Fraction - >>> calculation = Fraction(1, 3) - >>> f'{calculation=}' - 'calculation=Fraction(1, 3)' - >>> f'{calculation = }' - 'calculation = Fraction(1, 3)' - >>> f'{calculation = !s}' - 'calculation = 1/3' - -Once the output has been evaluated, it can be formatted using a -:ref:`format specifier ` following a colon (``':'``). -After the expression has been evaluated, and possibly converted to a string, -the :meth:`!__format__` method of the result is called with the format specifier, -or the empty string if no format specifier is given. -The formatted result is then used as the final value for the replacement field. -For example: + >>> string = "¡kočka 😸!" + >>> ascii(string) + "'\\xa1ko\\u010dka \\U0001f638!'" -.. doctest:: + >>> f'{string = !a}' + "string = '\\xa1ko\\u010dka \\U0001f638!'" + + +Format specifier +^^^^^^^^^^^^^^^^ + +After the expression has been evaluated, and possibly converted using an +explicit conversion specifier, it is formatted using the :func:`format` function. +If the replacement field includes a *format specifier* introduced by a colon +(``:``), the specifier is passed to :func:`!format` as the second argument. +The result of :func:`!format` is then used as the final value for the +replacement field. For example:: >>> from fractions import Fraction - >>> f'{Fraction(1, 7):.6f}' - '0.142857' - >>> f'{Fraction(1, 7):_^+10}' - '___+1/7___' + >>> one_third = Fraction(1, 3) + >>> f'{one_third:.6f}' + '0.333333' + >>> f'{one_third:_^+10}' + '___+1/3___' + >>> >>> f'{one_third!r:_^20}' + '___Fraction(1, 3)___' + >>> f'{one_third = :~>10}~' + 'one_third = ~~~~~~~1/3~' + +.. _stdtypes-tstrings: + +Template String Literals (t-strings) +------------------------------------ + +An :dfn:`t-string` (formally a :dfn:`template string literal`) is +a string literal that is prefixed with ``t`` or ``T``. + +These strings follow the same syntax and evaluation rules as +:ref:`formatted string literals `, +with for the following differences: + +* Rather than evaluating to a ``str`` object, template string literals evaluate + to a :class:`string.templatelib.Template` object. + +* The :func:`format` protocol is not used. + Instead, the format specifier and conversions (if any) are passed to + a new :class:`~string.templatelib.Interpolation` object that is created + for each evaluated expression. + It is up to code that processes the resulting :class:`~string.templatelib.Template` + object to decide how to handle format specifiers and conversions. + +* Format specifiers containing nested replacement fields are evaluated eagerly, + prior to being passed to the :class:`~string.templatelib.Interpolation` object. + For instance, an interpolation of the form ``{amount:.{precision}f}`` will + evaluate the inner expression ``{precision}`` to determine the value of the + ``format_spec`` attribute. + If ``precision`` were to be ``2``, the resulting format specifier + would be ``'.2f'``. + +* When the equals sign ``'='`` is provided in an interpolation expression, + the text of the expression is appended to the literal string that precedes + the relevant interpolation. + This includes the equals sign and any surrounding whitespace. + The :class:`!Interpolation` instance for the expression will be created as + normal, except that :attr:`~string.templatelib.Interpolation.conversion` will + be set to '``r``' (:func:`repr`) by default. + If an explicit conversion or format specifier are provided, + this will override the default behaviour. .. _old-string-formatting: @@ -2615,11 +3094,12 @@ For example: The formatting operations described here exhibit a variety of quirks that lead to a number of common errors (such as failing to display tuples and - dictionaries correctly). Using the newer :ref:`formatted string literals - `, the :meth:`str.format` interface, or :ref:`template strings - ` may help avoid these errors. Each of these - alternatives provides their own trade-offs and benefits of simplicity, - flexibility, and/or extensibility. + dictionaries correctly). + + Using :ref:`formatted string literals `, the :meth:`str.format` + interface, or :class:`string.Template` may help avoid these errors. + Each of these alternatives provides their own trade-offs and benefits of + simplicity, flexibility, and/or extensibility. String objects have one unique built-in operation: the ``%`` operator (modulo). This is also known as the string *formatting* or *interpolation* operator. @@ -2757,6 +3237,10 @@ The conversion types are: | | character in the result. | | +------------+-----------------------------------------------------+-------+ +For floating-point formats, the result should be correctly rounded to a given +precision ``p`` of digits after the decimal point. The rounding mode matches +that of the :func:`round` builtin. + Notes: (1) @@ -2831,7 +3315,8 @@ binary protocols are based on the ASCII text encoding, bytes objects offer several methods that are only valid when working with ASCII compatible data and are closely related to string objects in a variety of other ways. -.. class:: bytes([source[, encoding[, errors]]]) +.. class:: bytes(source=b'') + bytes(source, encoding, errors='strict') Firstly, the syntax for bytes literals is largely the same as that for string literals, except that a ``b`` prefix is added: @@ -2871,7 +3356,7 @@ data and are closely related to string objects in a variety of other ways. numbers are a commonly used format for describing binary data. Accordingly, the bytes type has an additional class method to read data in that format: - .. classmethod:: fromhex(string) + .. classmethod:: fromhex(string, /) This :class:`bytes` class method returns a bytes object, decoding the given string object. The string must contain two hexadecimal digits per @@ -2891,7 +3376,8 @@ data and are closely related to string objects in a variety of other ways. A reverse conversion function exists to transform a bytes object into its hexadecimal representation. - .. method:: hex([sep[, bytes_per_sep]]) + .. method:: hex(*, bytes_per_sep=1) + hex(sep, bytes_per_sep=1) Return a string object containing two hexadecimal digits for each byte in the instance. @@ -2940,7 +3426,8 @@ Bytearray Objects :class:`bytearray` objects are a mutable counterpart to :class:`bytes` objects. -.. class:: bytearray([source[, encoding[, errors]]]) +.. class:: bytearray(source=b'') + bytearray(source, encoding, errors='strict') There is no dedicated literal syntax for bytearray objects, instead they are always created by calling the constructor: @@ -2960,7 +3447,7 @@ objects. numbers are a commonly used format for describing binary data. Accordingly, the bytearray type has an additional class method to read data in that format: - .. classmethod:: fromhex(string) + .. classmethod:: fromhex(string, /) This :class:`bytearray` class method returns bytearray object, decoding the given string object. The string must contain two hexadecimal digits @@ -2980,7 +3467,8 @@ objects. A reverse conversion function exists to transform a bytearray object into its hexadecimal representation. - .. method:: hex([sep[, bytes_per_sep]]) + .. method:: hex(*, bytes_per_sep=1) + hex(sep, bytes_per_sep=1) Return a string object containing two hexadecimal digits for each byte in the instance. @@ -2995,7 +3483,7 @@ objects. optional *sep* and *bytes_per_sep* parameters to insert separators between bytes in the hex output. - .. method:: resize(size) + .. method:: resize(size, /) Resize the :class:`bytearray` to contain *size* bytes. *size* must be greater than or equal to 0. @@ -3037,6 +3525,11 @@ The representation of bytearray objects uses the bytes literal format ``bytearray([46, 46, 46])``. You can always convert a bytearray object into a list of integers using ``list(b)``. +.. seealso:: + + For detailed information on thread-safety guarantees for :class:`bytearray` + objects, see :ref:`thread-safety-bytearray`. + .. _bytes-methods: @@ -3218,8 +3711,8 @@ arbitrary binary data. Also accept an integer in the range 0 to 255 as the subsequence. -.. method:: bytes.join(iterable) - bytearray.join(iterable) +.. method:: bytes.join(iterable, /) + bytearray.join(iterable, /) Return a bytes or bytearray object which is the concatenation of the binary data sequences in *iterable*. A :exc:`TypeError` will be raised @@ -3229,8 +3722,8 @@ arbitrary binary data. bytearray object providing this method. -.. staticmethod:: bytes.maketrans(from, to) - bytearray.maketrans(from, to) +.. staticmethod:: bytes.maketrans(from, to, /) + bytearray.maketrans(from, to, /) This static method returns a translation table usable for :meth:`bytes.translate` that will map each character in *from* into the @@ -3240,8 +3733,8 @@ arbitrary binary data. .. versionadded:: 3.1 -.. method:: bytes.partition(sep) - bytearray.partition(sep) +.. method:: bytes.partition(sep, /) + bytearray.partition(sep, /) Split the sequence at the first occurrence of *sep*, and return a 3-tuple containing the part before the separator, the separator itself or its @@ -3253,8 +3746,8 @@ arbitrary binary data. The separator to search for may be any :term:`bytes-like object`. -.. method:: bytes.replace(old, new[, count]) - bytearray.replace(old, new[, count]) +.. method:: bytes.replace(old, new, count=-1, /) + bytearray.replace(old, new, count=-1, /) Return a copy of the sequence with all occurrences of subsequence *old* replaced by *new*. If the optional argument *count* is given, only the @@ -3297,8 +3790,8 @@ arbitrary binary data. Also accept an integer in the range 0 to 255 as the subsequence. -.. method:: bytes.rpartition(sep) - bytearray.rpartition(sep) +.. method:: bytes.rpartition(sep, /) + bytearray.rpartition(sep, /) Split the sequence at the last occurrence of *sep*, and return a 3-tuple containing the part before the separator, the separator itself or its @@ -3348,8 +3841,8 @@ with arbitrary binary data by passing appropriate arguments. Note that all of the bytearray methods in this section do *not* operate in place, and instead produce new objects. -.. method:: bytes.center(width[, fillbyte]) - bytearray.center(width[, fillbyte]) +.. method:: bytes.center(width, fillbyte=b' ', /) + bytearray.center(width, fillbyte=b' ', /) Return a copy of the object centered in a sequence of length *width*. Padding is done using the specified *fillbyte* (default is an ASCII @@ -3362,8 +3855,8 @@ produce new objects. it always produces a new object, even if no changes were made. -.. method:: bytes.ljust(width[, fillbyte]) - bytearray.ljust(width[, fillbyte]) +.. method:: bytes.ljust(width, fillbyte=b' ', /) + bytearray.ljust(width, fillbyte=b' ', /) Return a copy of the object left justified in a sequence of length *width*. Padding is done using the specified *fillbyte* (default is an ASCII @@ -3376,14 +3869,13 @@ produce new objects. it always produces a new object, even if no changes were made. -.. method:: bytes.lstrip([chars]) - bytearray.lstrip([chars]) +.. method:: bytes.lstrip(bytes=None, /) + bytearray.lstrip(bytes=None, /) Return a copy of the sequence with specified leading bytes removed. The - *chars* argument is a binary sequence specifying the set of byte values to - be removed - the name refers to the fact this method is usually used with - ASCII characters. If omitted or ``None``, the *chars* argument defaults - to removing ASCII whitespace. The *chars* argument is not a prefix; + *bytes* argument is a binary sequence specifying the set of byte values to + be removed. If omitted or ``None``, the *bytes* argument defaults + to removing ASCII whitespace. The *bytes* argument is not a prefix; rather, all combinations of its values are stripped:: >>> b' spacious '.lstrip() @@ -3407,8 +3899,8 @@ produce new objects. it always produces a new object, even if no changes were made. -.. method:: bytes.rjust(width[, fillbyte]) - bytearray.rjust(width[, fillbyte]) +.. method:: bytes.rjust(width, fillbyte=b' ', /) + bytearray.rjust(width, fillbyte=b' ', /) Return a copy of the object right justified in a sequence of length *width*. Padding is done using the specified *fillbyte* (default is an ASCII @@ -3432,14 +3924,13 @@ produce new objects. :meth:`split` which is described in detail below. -.. method:: bytes.rstrip([chars]) - bytearray.rstrip([chars]) +.. method:: bytes.rstrip(bytes=None, /) + bytearray.rstrip(bytes=None, /) Return a copy of the sequence with specified trailing bytes removed. The - *chars* argument is a binary sequence specifying the set of byte values to - be removed - the name refers to the fact this method is usually used with - ASCII characters. If omitted or ``None``, the *chars* argument defaults to - removing ASCII whitespace. The *chars* argument is not a suffix; rather, + *bytes* argument is a binary sequence specifying the set of byte values to + be removed. If omitted or ``None``, the *bytes* argument defaults to + removing ASCII whitespace. The *bytes* argument is not a suffix; rather, all combinations of its values are stripped:: >>> b' spacious '.rstrip() @@ -3509,14 +4000,13 @@ produce new objects. [b'1', b'2', b'3'] -.. method:: bytes.strip([chars]) - bytearray.strip([chars]) +.. method:: bytes.strip(bytes=None, /) + bytearray.strip(bytes=None, /) Return a copy of the sequence with specified leading and trailing bytes - removed. The *chars* argument is a binary sequence specifying the set of - byte values to be removed - the name refers to the fact this method is - usually used with ASCII characters. If omitted or ``None``, the *chars* - argument defaults to removing ASCII whitespace. The *chars* argument is + removed. The *bytes* argument is a binary sequence specifying the set of + byte values to be removed. If omitted or ``None``, the *bytes* + argument defaults to removing ASCII whitespace. The *bytes* argument is not a prefix or suffix; rather, all combinations of its values are stripped:: @@ -3838,8 +4328,8 @@ place, and instead produce new objects. always produces a new object, even if no changes were made. -.. method:: bytes.zfill(width) - bytearray.zfill(width) +.. method:: bytes.zfill(width, /) + bytearray.zfill(width, /) Return a copy of the sequence left filled with ASCII ``b'0'`` digits to make a sequence of length *width*. A leading sign prefix (``b'+'``/ @@ -4087,6 +4577,9 @@ copying. types such as :class:`bytes` and :class:`bytearray`, an element is a single byte, but other types such as :class:`array.array` may have bigger elements. + :class:`!memoryview`\s are :ref:`generic ` over the type of their + underlying data. + ``len(view)`` is equal to the length of :class:`~memoryview.tolist`, which is the nested list representation of the view. If ``view.ndim = 1``, this is equal to the number of elements in the view. @@ -4251,7 +4744,8 @@ copying. in-memory Fortran order is preserved. For non-contiguous views, the data is converted to C first. *order=None* is the same as *order='C'*. - .. method:: hex([sep[, bytes_per_sep]]) + .. method:: hex(*, bytes_per_sep=1) + hex(sep, bytes_per_sep=1) Return a string object containing two hexadecimal digits for each byte in the buffer. :: @@ -4336,7 +4830,8 @@ copying. .. versionadded:: 3.2 - .. method:: cast(format[, shape]) + .. method:: cast(format, /) + cast(format, shape, /) Cast a memoryview to a new format or shape. *shape* defaults to ``[byte_length//new_itemsize]``, which means that the result view @@ -4564,6 +5059,9 @@ copying. .. versionadded:: 3.3 +For information on the thread safety of :class:`memoryview` objects in +the :term:`free-threaded build`, see :ref:`thread-safety-memoryview`. + .. _types-set: @@ -4586,11 +5084,12 @@ other sequence-like behavior. There are currently two built-in set types, :class:`set` and :class:`frozenset`. The :class:`set` type is mutable --- the contents can be changed using methods -like :meth:`~set.add` and :meth:`~set.remove`. Since it is mutable, it has no -hash value and cannot be used as either a dictionary key or as an element of -another set. The :class:`frozenset` type is immutable and :term:`hashable` --- -its contents cannot be altered after it is created; it can therefore be used as -a dictionary key or as an element of another set. +like :meth:`~set.add` and :meth:`~set.remove`. +Since it is mutable, it has no hash value and cannot be used as +either a dictionary key or as an element of another set. +The :class:`frozenset` type is immutable and :term:`hashable` --- +its contents cannot be altered after it is created; +it can therefore be used as a dictionary key or as an element of another set. Non-empty sets (not frozensets) can be created by placing a comma-separated list of elements within braces, for example: ``{'jack', 'sjoerd'}``, in addition to the @@ -4598,8 +5097,8 @@ of elements within braces, for example: ``{'jack', 'sjoerd'}``, in addition to t The constructors for both classes work the same: -.. class:: set([iterable]) - frozenset([iterable]) +.. class:: set(iterable=(), /) + frozenset(iterable=(), /) Return a new set or frozenset object whose elements are taken from *iterable*. The elements of a set must be :term:`hashable`. To @@ -4607,164 +5106,179 @@ The constructors for both classes work the same: objects. If *iterable* is not specified, a new empty set is returned. - Sets can be created by several means: +Sets can be created by several means: - * Use a comma-separated list of elements within braces: ``{'jack', 'sjoerd'}`` - * Use a set comprehension: ``{c for c in 'abracadabra' if c not in 'abc'}`` - * Use the type constructor: ``set()``, ``set('foobar')``, ``set(['a', 'b', 'foo'])`` +* Use a comma-separated list of elements within braces: ``{'jack', 'sjoerd'}`` +* Use a set comprehension: ``{c for c in 'abracadabra' if c not in 'abc'}`` +* Use the type constructor: ``set()``, ``set('foobar')``, ``set(['a', 'b', 'foo'])`` - Instances of :class:`set` and :class:`frozenset` provide the following - operations: +Instances of :class:`set` and :class:`frozenset` provide the following +operations: - .. describe:: len(s) +.. describe:: len(s) - Return the number of elements in set *s* (cardinality of *s*). + Return the number of elements in set *s* (cardinality of *s*). - .. describe:: x in s +.. describe:: x in s - Test *x* for membership in *s*. + Test *x* for membership in *s*. - .. describe:: x not in s +.. describe:: x not in s - Test *x* for non-membership in *s*. + Test *x* for non-membership in *s*. - .. method:: isdisjoint(other) +.. method:: frozenset.isdisjoint(other, /) + set.isdisjoint(other, /) - Return ``True`` if the set has no elements in common with *other*. Sets are - disjoint if and only if their intersection is the empty set. + Return ``True`` if the set has no elements in common with *other*. Sets are + disjoint if and only if their intersection is the empty set. - .. method:: issubset(other) - set <= other +.. method:: frozenset.issubset(other, /) + set.issubset(other, /) +.. describe:: set <= other - Test whether every element in the set is in *other*. + Test whether every element in the set is in *other*. - .. method:: set < other +.. describe:: set < other - Test whether the set is a proper subset of *other*, that is, - ``set <= other and set != other``. + Test whether the set is a proper subset of *other*, that is, + ``set <= other and set != other``. - .. method:: issuperset(other) - set >= other +.. method:: frozenset.issuperset(other, /) + set.issuperset(other, /) +.. describe:: set >= other - Test whether every element in *other* is in the set. + Test whether every element in *other* is in the set. - .. method:: set > other +.. describe:: set > other - Test whether the set is a proper superset of *other*, that is, ``set >= - other and set != other``. + Test whether the set is a proper superset of *other*, that is, ``set >= + other and set != other``. - .. method:: union(*others) - set | other | ... +.. method:: frozenset.union(*others) + set.union(*others) +.. describe:: set | other | ... - Return a new set with elements from the set and all others. + Return a new set with elements from the set and all others. - .. method:: intersection(*others) - set & other & ... +.. method:: frozenset.intersection(*others) + set.intersection(*others) +.. describe:: set & other & ... - Return a new set with elements common to the set and all others. + Return a new set with elements common to the set and all others. - .. method:: difference(*others) - set - other - ... +.. method:: frozenset.difference(*others) + set.difference(*others) +.. describe:: set - other - ... - Return a new set with elements in the set that are not in the others. + Return a new set with elements in the set that are not in the others. - .. method:: symmetric_difference(other) - set ^ other +.. method:: frozenset.symmetric_difference(other, /) + set.symmetric_difference(other, /) +.. describe:: set ^ other - Return a new set with elements in either the set or *other* but not both. + Return a new set with elements in either the set or *other* but not both. - .. method:: copy() +.. method:: frozenset.copy() + set.copy() - Return a shallow copy of the set. + Return a shallow copy of the set. - Note, the non-operator versions of :meth:`union`, :meth:`intersection`, - :meth:`difference`, :meth:`symmetric_difference`, :meth:`issubset`, and - :meth:`issuperset` methods will accept any iterable as an argument. In - contrast, their operator based counterparts require their arguments to be - sets. This precludes error-prone constructions like ``set('abc') & 'cbs'`` - in favor of the more readable ``set('abc').intersection('cbs')``. +Note, the non-operator versions of :meth:`~frozenset.union`, +:meth:`~frozenset.intersection`, :meth:`~frozenset.difference`, :meth:`~frozenset.symmetric_difference`, :meth:`~frozenset.issubset`, and +:meth:`~frozenset.issuperset` methods will accept any iterable as an argument. In +contrast, their operator based counterparts require their arguments to be +sets. This precludes error-prone constructions like ``set('abc') & 'cbs'`` +in favor of the more readable ``set('abc').intersection('cbs')``. - Both :class:`set` and :class:`frozenset` support set to set comparisons. Two - sets are equal if and only if every element of each set is contained in the - other (each is a subset of the other). A set is less than another set if and - only if the first set is a proper subset of the second set (is a subset, but - is not equal). A set is greater than another set if and only if the first set - is a proper superset of the second set (is a superset, but is not equal). +Both :class:`set` and :class:`frozenset` support set to set comparisons. Two +sets are equal if and only if every element of each set is contained in the +other (each is a subset of the other). A set is less than another set if and +only if the first set is a proper subset of the second set (is a subset, but +is not equal). A set is greater than another set if and only if the first set +is a proper superset of the second set (is a superset, but is not equal). - Instances of :class:`set` are compared to instances of :class:`frozenset` - based on their members. For example, ``set('abc') == frozenset('abc')`` - returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``. +Instances of :class:`set` are compared to instances of :class:`frozenset` +based on their members. For example, ``set('abc') == frozenset('abc')`` +returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``. - The subset and equality comparisons do not generalize to a total ordering - function. For example, any two nonempty disjoint sets are not equal and are not - subsets of each other, so *all* of the following return ``False``: ``ab``. +The subset and equality comparisons do not generalize to a total ordering +function. For example, any two nonempty disjoint sets are not equal and are not +subsets of each other, so *all* of the following return ``False``: ``ab``. - Since sets only define partial ordering (subset relationships), the output of - the :meth:`list.sort` method is undefined for lists of sets. +Since sets only define partial ordering (subset relationships), the output of +the :meth:`list.sort` method is undefined for lists of sets. - Set elements, like dictionary keys, must be :term:`hashable`. +Set elements, like dictionary keys, must be :term:`hashable`. - Binary operations that mix :class:`set` instances with :class:`frozenset` - return the type of the first operand. For example: ``frozenset('ab') | - set('bc')`` returns an instance of :class:`frozenset`. +Binary operations that mix :class:`set` instances with :class:`frozenset` +return the type of the first operand. For example: ``frozenset('ab') | +set('bc')`` returns an instance of :class:`frozenset`. - The following table lists operations available for :class:`set` that do not - apply to immutable instances of :class:`frozenset`: +The following table lists operations available for :class:`set` that do not +apply to immutable instances of :class:`frozenset`: - .. method:: update(*others) - set |= other | ... +.. method:: set.update(*others) +.. describe:: set |= other | ... - Update the set, adding elements from all others. + Update the set, adding elements from all others. - .. method:: intersection_update(*others) - set &= other & ... +.. method:: set.intersection_update(*others) +.. describe:: set &= other & ... - Update the set, keeping only elements found in it and all others. + Update the set, keeping only elements found in it and all others. - .. method:: difference_update(*others) - set -= other | ... +.. method:: set.difference_update(*others) +.. describe:: set -= other | ... - Update the set, removing elements found in others. + Update the set, removing elements found in others. - .. method:: symmetric_difference_update(other) - set ^= other +.. method:: set.symmetric_difference_update(other, /) +.. describe:: set ^= other - Update the set, keeping only elements found in either set, but not in both. + Update the set, keeping only elements found in either set, but not in both. - .. method:: add(elem) +.. method:: set.add(elem, /) - Add element *elem* to the set. + Add element *elem* to the set. - .. method:: remove(elem) +.. method:: set.remove(elem, /) - Remove element *elem* from the set. Raises :exc:`KeyError` if *elem* is - not contained in the set. + Remove element *elem* from the set. Raises :exc:`KeyError` if *elem* is + not contained in the set. - .. method:: discard(elem) +.. method:: set.discard(elem, /) - Remove element *elem* from the set if it is present. + Remove element *elem* from the set if it is present. - .. method:: pop() +.. method:: set.pop() - Remove and return an arbitrary element from the set. Raises - :exc:`KeyError` if the set is empty. + Remove and return an arbitrary element from the set. Raises + :exc:`KeyError` if the set is empty. - .. method:: clear() +.. method:: set.clear() - Remove all elements from the set. + Remove all elements from the set. - Note, the non-operator versions of the :meth:`update`, - :meth:`intersection_update`, :meth:`difference_update`, and - :meth:`symmetric_difference_update` methods will accept any iterable as an - argument. +Note, the non-operator versions of the :meth:`~set.update`, +:meth:`~set.intersection_update`, :meth:`~set.difference_update`, and +:meth:`~set.symmetric_difference_update` methods will accept any iterable as an +argument. - Note, the *elem* argument to the :meth:`~object.__contains__`, - :meth:`remove`, and - :meth:`discard` methods may be a set. To support searching for an equivalent - frozenset, a temporary one is created from *elem*. +Note, the *elem* argument to the :meth:`~object.__contains__`, +:meth:`~set.remove`, and +:meth:`~set.discard` methods may be a set. To support searching for an equivalent +frozenset, a temporary one is created from *elem*. + +.. seealso:: + + For detailed information on thread-safety guarantees for :class:`set` + objects, see :ref:`thread-safety-set`. + + Sets and frozensets are :ref:`generic ` over the type of their elements. .. _typesmapping: @@ -4794,8 +5308,8 @@ Values that compare equal (such as ``1``, ``1.0``, and ``True``) can be used interchangeably to index the same dictionary entry. .. class:: dict(**kwargs) - dict(mapping, **kwargs) - dict(iterable, **kwargs) + dict(mapping, /, **kwargs) + dict(iterable, /, **kwargs) Return a new dictionary initialized from an optional positional argument and a possibly empty set of keyword arguments. @@ -4823,7 +5337,10 @@ can be used interchangeably to index the same dictionary entry. being added is already present, the value from the keyword argument replaces the value from the positional argument. - To illustrate, the following examples all return a dictionary equal to + Dictionaries compare equal if and only if they have the same ``(key, + value)`` pairs (regardless of ordering). Order comparisons ('<', '<=', '>=', '>') raise + :exc:`TypeError`. To illustrate dictionary creation and equality, + the following examples all return a dictionary equal to ``{"one": 1, "two": 2, "three": 3}``:: >>> a = dict(one=1, two=2, three=3) @@ -4838,6 +5355,30 @@ can be used interchangeably to index the same dictionary entry. Providing keyword arguments as in the first example only works for keys that are valid Python identifiers. Otherwise, any valid keys can be used. + Dictionaries preserve insertion order. Note that updating a key does not + affect the order. Keys added after deletion are inserted at the end. :: + + >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} + >>> d + {'one': 1, 'two': 2, 'three': 3, 'four': 4} + >>> list(d) + ['one', 'two', 'three', 'four'] + >>> list(d.values()) + [1, 2, 3, 4] + >>> d["one"] = 42 + >>> d + {'one': 42, 'two': 2, 'three': 3, 'four': 4} + >>> del d["two"] + >>> d["two"] = None + >>> d + {'one': 42, 'three': 3, 'four': 4, 'two': None} + + .. versionchanged:: 3.7 + Dictionary order is guaranteed to be insertion order. This behavior was + an implementation detail of CPython from 3.6. + + Dictionaries are :ref:`generic ` over two types, signifying + (respectively) the types of the dictionary's keys and values. These are the operations that dictionaries support (and therefore, custom mapping types should support too): @@ -4857,13 +5398,13 @@ can be used interchangeably to index the same dictionary entry. .. index:: __missing__() - If a subclass of dict defines a method :meth:`__missing__` and *key* + If a subclass of dict defines a method :meth:`~object.__missing__` and *key* is not present, the ``d[key]`` operation calls that method with the key *key* as argument. The ``d[key]`` operation then returns or raises whatever is returned or raised by the ``__missing__(key)`` call. - No other operations or methods invoke :meth:`__missing__`. If - :meth:`__missing__` is not defined, :exc:`KeyError` is raised. - :meth:`__missing__` must be a method; it cannot be an instance variable:: + No other operations or methods invoke :meth:`~object.__missing__`. If + :meth:`~object.__missing__` is not defined, :exc:`KeyError` is raised. + :meth:`~object.__missing__` must be a method; it cannot be an instance variable:: >>> class Counter(dict): ... def __missing__(self, key): @@ -4877,7 +5418,8 @@ can be used interchangeably to index the same dictionary entry. 1 The example above shows part of the implementation of - :class:`collections.Counter`. A different ``__missing__`` method is used + :class:`collections.Counter`. + A different :meth:`!__missing__` method is used by :class:`collections.defaultdict`. .. describe:: d[key] = value @@ -4936,7 +5478,8 @@ can be used interchangeably to index the same dictionary entry. Return a new view of the dictionary's keys. See the :ref:`documentation of view objects `. - .. method:: pop(key[, default]) + .. method:: pop(key, /) + pop(key, default, /) If *key* is in the dictionary, remove it and return its value, else return *default*. If *default* is not given and *key* is not in the dictionary, @@ -4968,9 +5511,11 @@ can be used interchangeably to index the same dictionary entry. with a value of *default* and return *default*. *default* defaults to ``None``. - .. method:: update([other]) + .. method:: update(**kwargs) + update(mapping, /, **kwargs) + update(iterable, /, **kwargs) - Update the dictionary with the key/value pairs from *other*, overwriting + Update the dictionary with the key/value pairs from *mapping* or *iterable* and *kwargs*, overwriting existing keys. Return ``None``. :meth:`update` accepts either another object with a ``keys()`` method (in @@ -5008,32 +5553,6 @@ can be used interchangeably to index the same dictionary entry. .. versionadded:: 3.9 - Dictionaries compare equal if and only if they have the same ``(key, - value)`` pairs (regardless of ordering). Order comparisons ('<', '<=', '>=', '>') raise - :exc:`TypeError`. - - Dictionaries preserve insertion order. Note that updating a key does not - affect the order. Keys added after deletion are inserted at the end. :: - - >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} - >>> d - {'one': 1, 'two': 2, 'three': 3, 'four': 4} - >>> list(d) - ['one', 'two', 'three', 'four'] - >>> list(d.values()) - [1, 2, 3, 4] - >>> d["one"] = 42 - >>> d - {'one': 42, 'two': 2, 'three': 3, 'four': 4} - >>> del d["two"] - >>> d["two"] = None - >>> d - {'one': 42, 'three': 3, 'four': 4, 'two': None} - - .. versionchanged:: 3.7 - Dictionary order is guaranteed to be insertion order. This behavior was - an implementation detail of CPython from 3.6. - Dictionaries and dictionary views are reversible. :: >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} @@ -5055,6 +5574,12 @@ can be used interchangeably to index the same dictionary entry. of a :class:`dict`. +.. seealso:: + + For detailed information on thread-safety guarantees for :class:`dict` + objects, see :ref:`thread-safety-dict`. + + .. _dict-views: Dictionary view objects @@ -5206,9 +5731,11 @@ before the statement body is executed and exited when the statement ends: Returning a true value from this method will cause the :keyword:`with` statement to suppress the exception and continue execution with the statement immediately following the :keyword:`!with` statement. Otherwise the exception continues - propagating after this method has finished executing. Exceptions that occur - during execution of this method will replace any exception that occurred in the - body of the :keyword:`!with` statement. + propagating after this method has finished executing. + + If this method raises an exception while handling an earlier exception from the + :keyword:`with` block, the new exception is raised, and the original exception + is stored in its :attr:`~BaseException.__context__` attribute. The exception passed in should never be reraised explicitly - instead, this method should return a false value to indicate that the method completed @@ -5297,7 +5824,8 @@ type and the :class:`bytes` data type: ``GenericAlias`` objects are instances of the class :class:`types.GenericAlias`, which can also be used to create ``GenericAlias`` -objects directly. +objects directly. Specializations of user-defined :ref:`generic classes ` +may not be instances of :class:`types.GenericAlias`, but they provide similar functionality. .. describe:: T[X, Y, ...] @@ -5346,6 +5874,15 @@ creation:: >>> type(l) + +Instances of ``GenericAlias`` are not classes at runtime, even though they behave like classes (they can be instantiated and subclassed):: + + >>> import inspect + >>> inspect.isclass(list[int]) + False + +This is true for :ref:`user-defined generics ` also. + Calling :func:`repr` or :func:`str` on a generic shows the parameterized type:: >>> repr(list[int]) @@ -5409,6 +5946,7 @@ list is non-exhaustive. * :class:`collections.abc.MutableMapping` * :class:`collections.abc.Sequence` * :class:`collections.abc.MutableSequence` +* :class:`collections.abc.ByteString` * :class:`collections.abc.MappingView` * :class:`collections.abc.KeysView` * :class:`collections.abc.ItemsView` @@ -5684,9 +6222,10 @@ Methods .. index:: pair: object; method -Methods are functions that are called using the attribute notation. There are -two flavors: :ref:`built-in methods ` (such as :meth:`append` -on lists) and :ref:`class instance method `. +Methods are functions that are called using the attribute notation. +There are two flavors: :ref:`built-in methods ` +(such as :meth:`~list.append` on lists) +and :ref:`class instance method `. Built-in methods are described with the types that support them. If you access a method (a function defined in a class namespace) through an @@ -5792,13 +6331,34 @@ It is written as ``None``. The Ellipsis Object ------------------- -This object is commonly used by slicing (see :ref:`slicings`). It supports no -special operations. There is exactly one ellipsis object, named +This object is commonly used to indicate that something is omitted. +It supports no special operations. There is exactly one ellipsis object, named :const:`Ellipsis` (a built-in name). ``type(Ellipsis)()`` produces the :const:`Ellipsis` singleton. It is written as ``Ellipsis`` or ``...``. +In typical use, ``...`` as the ``Ellipsis`` object appears in a few different +places, for instance: + +- In type annotations, such as :ref:`callable arguments ` + or :ref:`tuple elements `. + +- As the body of a function instead of a :ref:`pass statement `. + +- In third-party libraries, such as `Numpy's slicing and striding + `_. + +Python also uses three dots in ways that are not ``Ellipsis`` objects, for instance: + +- Doctest's :const:`ELLIPSIS `, as a pattern for missing content. + +- The default Python prompt of the :term:`interactive` shell when partial input is incomplete. + +Lastly, the Python documentation often uses three dots in conventional English +usage to mean omitted content, even in code examples that also use them as the +``Ellipsis``. + .. _bltin-notimplemented-object: diff --git a/Doc/library/string.rst b/Doc/library/string.rst index b44d98819b6998b..f1abed91d20ed51 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -4,7 +4,7 @@ .. module:: string :synopsis: Common string operations. -**Source code:** :source:`Lib/string.py` +**Source code:** :source:`Lib/string/__init__.py` -------------- @@ -82,12 +82,12 @@ The constants defined in this module are: .. _string-formatting: -Custom String Formatting +Custom string formatting ------------------------ The built-in string class provides the ability to do complex variable substitutions and value formatting via the :meth:`~str.format` method described in -:pep:`3101`. The :class:`Formatter` class in the :mod:`string` module allows +:pep:`3101`. The :class:`Formatter` class in the :mod:`!string` module allows you to create and customize your own string formatting behaviors using the same implementation as the built-in :meth:`~str.format` method. @@ -192,14 +192,15 @@ implementation as the built-in :meth:`~str.format` method. .. _formatstrings: -Format String Syntax +Format string syntax -------------------- The :meth:`str.format` method and the :class:`Formatter` class share the same syntax for format strings (although in the case of :class:`Formatter`, subclasses can define their own format string syntax). The syntax is -related to that of :ref:`formatted string literals `, but it is -less sophisticated and, in particular, does not support arbitrary expressions. +related to that of :ref:`formatted string literals ` and +:ref:`template string literals `, but it is less sophisticated +and, in particular, does not support arbitrary expressions in interpolations. .. index:: single: {} (curly brackets); in string formatting @@ -264,6 +265,8 @@ Some simple format string examples:: "Weight in tons {0.weight}" # 'weight' attribute of first positional arg "Units destroyed: {players[0]}" # First element of keyword argument 'players'. +.. _formatstrings-conversion: + The *conversion* field causes a type coercion before formatting. Normally, the job of formatting a value is done by the :meth:`~object.__format__` method of the value itself. However, in some cases it is desirable to force a type to be formatted @@ -301,12 +304,12 @@ See the :ref:`formatexamples` section for some examples. .. _formatspec: -Format Specification Mini-Language +Format specification mini-language ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ "Format specifications" are used within replacement fields contained within a format string to define how individual values are presented (see -:ref:`formatstrings` and :ref:`f-strings`). +:ref:`formatstrings`, :ref:`f-strings`, and :ref:`t-strings`). They can also be passed directly to the built-in :func:`format` function. Each formattable type may define how the format specification is to be interpreted. @@ -328,7 +331,7 @@ The general form of a *standard format specifier* is: sign: "+" | "-" | " " width_and_precision: [`width_with_grouping`][`precision_with_grouping`] width_with_grouping: [`width`][`grouping`] - precision_with_grouping: "." [`precision`][`grouping`] + precision_with_grouping: "." [`precision`][`grouping`] | "." `grouping` width: `~python-grammar:digit`+ precision: `~python-grammar:digit`+ grouping: "," | "_" @@ -469,7 +472,9 @@ of a number respectively. It can be one of the following: | | this option is not supported. | +---------+----------------------------------------------------------+ -For a locale aware separator, use the ``'n'`` presentation type instead. +For a locale-aware separator, use the ``'n'`` +:ref:`float presentation type ` or +:ref:`integer presentation type ` instead. .. versionchanged:: 3.1 Added the ``','`` option (see also :pep:`378`). @@ -515,9 +520,14 @@ The available integer presentation types are: | | In case ``'#'`` is specified, the prefix ``'0x'`` will | | | be upper-cased to ``'0X'`` as well. | +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'d'``, except that it uses | + | ``'n'`` | .. _n-format-integer: | + | | | + | | Number. This is the same as ``'d'``, except that it uses | | | the current locale setting to insert the appropriate | - | | digit group separators. | + | | digit group separators. Note that the default locale is | + | | not the system locale. Depending on your use case, you | + | | may wish to set :const:`~locale.LC_NUMERIC` with | + | | :func:`locale.setlocale` before using ``'n'``. | +---------+----------------------------------------------------------+ | None | The same as ``'d'``. | +---------+----------------------------------------------------------+ @@ -543,6 +553,9 @@ The available presentation types for :class:`float` and | | :class:`float`, and shows all coefficient digits | | | for :class:`~decimal.Decimal`. If ``p=0``, the decimal | | | point is omitted unless the ``#`` option is used. | + | | | + | | For :class:`float`, the exponent always contains at | + | | least two digits, and is zero if the value is zero. | +---------+----------------------------------------------------------+ | ``'E'`` | Scientific notation. Same as ``'e'`` except it uses | | | an upper case 'E' as the separator character. | @@ -597,10 +610,15 @@ The available presentation types for :class:`float` and | | ``'E'`` if the number gets too large. The | | | representations of infinity and NaN are uppercased, too. | +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'g'``, except that it uses | + | ``'n'`` | .. _n-format-float: | + | | | + | | Number. This is the same as ``'g'``, except that it uses | | | the current locale setting to insert the appropriate | - | | digit group separators | - | | for the integral part of a number. | + | | digit group separators for the integral part of a | + | | number. Note that the default locale is not the system | + | | locale. Depending on your use case, you may wish to set | + | | :const:`~locale.LC_NUMERIC` with | + | | :func:`locale.setlocale` before using ``'n'``. | +---------+----------------------------------------------------------+ | ``'%'`` | Percentage. Multiplies the number by 100 and displays | | | in fixed (``'f'``) format, followed by a percent sign. | @@ -753,8 +771,8 @@ Expressing a percentage:: Using type-specific formatting:: - >>> import datetime - >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58) + >>> import datetime as dt + >>> d = dt.datetime(2010, 7, 4, 12, 15, 58) >>> '{:%Y-%m-%d %H:%M:%S}'.format(d) '2010-07-04 12:15:58' @@ -789,10 +807,22 @@ Nesting arguments and more complex examples:: -.. _template-strings: +.. _template-strings-pep292: -Template strings ----------------- +Template strings ($-strings) +---------------------------- + +.. note:: + + The feature described here was introduced in Python 2.4; + a simple templating method based upon regular expressions. + It predates :meth:`str.format`, :ref:`formatted string literals `, + and :ref:`template string literals `. + + It is unrelated to template string literals (t-strings), + which were introduced in Python 3.14. + These evaluate to :class:`string.templatelib.Template` objects, + found in the :mod:`string.templatelib` module. Template strings provide simpler string substitutions as described in :pep:`292`. A primary use case for template strings is for @@ -822,7 +852,7 @@ Template strings support ``$``-based substitutions, using the following rules: Any other appearance of ``$`` in the string will result in a :exc:`ValueError` being raised. -The :mod:`string` module provides a :class:`Template` class that implements +The :mod:`!string` module provides a :class:`Template` class that implements these rules. The methods of :class:`Template` are: @@ -858,7 +888,7 @@ these rules. The methods of :class:`Template` are: .. method:: is_valid() - Returns false if the template has invalid placeholders that will cause + Returns ``False`` if the template has invalid placeholders that will cause :meth:`substitute` to raise :exc:`ValueError`. .. versionadded:: 3.11 @@ -941,7 +971,8 @@ attributes: Alternatively, you can provide the entire regular expression pattern by overriding the class attribute *pattern*. If you do this, the value must be a -regular expression object with four named capturing groups. The capturing +regular expression pattern string, or a compiled regular expression +object, with four named capturing groups. The capturing groups correspond to the rules given above, along with the invalid placeholder rule: diff --git a/Doc/library/string.templatelib.rst b/Doc/library/string.templatelib.rst new file mode 100644 index 000000000000000..6e91850fdf59ca4 --- /dev/null +++ b/Doc/library/string.templatelib.rst @@ -0,0 +1,351 @@ +:mod:`!string.templatelib` --- Support for template string literals +=================================================================== + +.. module:: string.templatelib + :synopsis: Support for template string literals. + +**Source code:** :source:`Lib/string/templatelib.py` + +-------------- + +.. seealso:: + + * :ref:`Format strings ` + * :ref:`Template string literal (t-string) syntax ` + * :pep:`750` + +.. _template-strings: + +Template strings +---------------- + +.. versionadded:: 3.14 + +Template strings are a mechanism for custom string processing. +They have the full flexibility of Python's :ref:`f-strings`, +but return a :class:`Template` instance that gives access +to the static and interpolated (in curly brackets) parts of a string +*before* they are combined. + +To write a t-string, use a ``'t'`` prefix instead of an ``'f'``, like so: + +.. code-block:: pycon + + >>> pi = 3.14 + >>> t't-strings are new in Python {pi!s}!' + Template( + strings=('t-strings are new in Python ', '!'), + interpolations=(Interpolation(3.14, 'pi', 's', ''),) + ) + +Types +----- + +.. class:: Template + + The :class:`!Template` class describes the contents of a template string. + It is immutable, meaning that attributes of a template cannot be reassigned. + + The most common way to create a :class:`!Template` instance is to use the + :ref:`template string literal syntax `. + This syntax is identical to that of :ref:`f-strings `, + except that it uses a ``t`` prefix in place of an ``f``: + + >>> cheese = 'Red Leicester' + >>> template = t"We're fresh out of {cheese}, sir." + >>> type(template) + + + Templates are stored as sequences of literal :attr:`~Template.strings` + and dynamic :attr:`~Template.interpolations`. + A :attr:`~Template.values` attribute holds the values of the interpolations: + + >>> cheese = 'Camembert' + >>> template = t'Ah! We do have {cheese}.' + >>> template.strings + ('Ah! We do have ', '.') + >>> template.interpolations + (Interpolation('Camembert', ...),) + >>> template.values + ('Camembert',) + + The :attr:`!strings` tuple has one more element than :attr:`!interpolations` + and :attr:`!values`; the interpolations “belong” between the strings. + This may be easier to understand when tuples are aligned + + .. code-block:: python + + template.strings: ('Ah! We do have ', '.') + template.values: ( 'Camembert', ) + + .. rubric:: Attributes + + .. attribute:: strings + :type: tuple[str, ...] + + A :class:`tuple` of the static strings in the template. + + >>> cheese = 'Camembert' + >>> template = t'Ah! We do have {cheese}.' + >>> template.strings + ('Ah! We do have ', '.') + + Empty strings *are* included in the tuple: + + >>> response = 'We do have ' + >>> cheese = 'Camembert' + >>> template = t'Ah! {response}{cheese}.' + >>> template.strings + ('Ah! ', '', '.') + + The ``strings`` tuple is never empty, and always contains one more + string than the ``interpolations`` and ``values`` tuples: + + >>> t''.strings + ('',) + >>> t''.values + () + >>> t'{'cheese'}'.strings + ('', '') + >>> t'{'cheese'}'.values + ('cheese',) + + .. attribute:: interpolations + :type: tuple[Interpolation, ...] + + A :class:`tuple` of the interpolations in the template. + + >>> cheese = 'Camembert' + >>> template = t'Ah! We do have {cheese}.' + >>> template.interpolations + (Interpolation('Camembert', 'cheese', None, ''),) + + The ``interpolations`` tuple may be empty and always contains one fewer + values than the ``strings`` tuple: + + >>> t'Red Leicester'.interpolations + () + + .. attribute:: values + :type: tuple[object, ...] + + A tuple of all interpolated values in the template. + + >>> cheese = 'Camembert' + >>> template = t'Ah! We do have {cheese}.' + >>> template.values + ('Camembert',) + + The ``values`` tuple always has the same length as the + ``interpolations`` tuple. It is always equivalent to + ``tuple(i.value for i in template.interpolations)``. + + .. rubric:: Methods + + .. method:: __new__(*args: str | Interpolation) + + While literal syntax is the most common way to create a :class:`!Template`, + it is also possible to create them directly using the constructor: + + >>> from string.templatelib import Interpolation, Template + >>> cheese = 'Camembert' + >>> template = Template( + ... 'Ah! We do have ', Interpolation(cheese, 'cheese'), '.' + ... ) + >>> list(template) + ['Ah! We do have ', Interpolation('Camembert', 'cheese', None, ''), '.'] + + If multiple strings are passed consecutively, they will be concatenated + into a single value in the :attr:`~Template.strings` attribute. For example, + the following code creates a :class:`Template` with a single final string: + + >>> from string.templatelib import Template + >>> template = Template('Ah! We do have ', 'Camembert', '.') + >>> template.strings + ('Ah! We do have Camembert.',) + + If multiple interpolations are passed consecutively, they will be treated + as separate interpolations and an empty string will be inserted between them. + For example, the following code creates a template with empty placeholders + in the :attr:`~Template.strings` attribute: + + >>> from string.templatelib import Interpolation, Template + >>> template = Template( + ... Interpolation('Camembert', 'cheese'), + ... Interpolation('.', 'punctuation'), + ... ) + >>> template.strings + ('', '', '') + + .. describe:: iter(template) + + Iterate over the template, yielding each non-empty string and + :class:`Interpolation` in the correct order: + + >>> cheese = 'Camembert' + >>> list(t'Ah! We do have {cheese}.') + ['Ah! We do have ', Interpolation('Camembert', 'cheese', None, ''), '.'] + + .. caution:: + + Empty strings are **not** included in the iteration: + + >>> response = 'We do have ' + >>> cheese = 'Camembert' + >>> list(t'Ah! {response}{cheese}.') # doctest: +NORMALIZE_WHITESPACE + ['Ah! ', + Interpolation('We do have ', 'response', None, ''), + Interpolation('Camembert', 'cheese', None, ''), + '.'] + + .. describe:: template + other + template += other + + Concatenate this template with another, returning a new + :class:`!Template` instance: + + >>> cheese = 'Camembert' + >>> list(t'Ah! ' + t'We do have {cheese}.') + ['Ah! We do have ', Interpolation('Camembert', 'cheese', None, ''), '.'] + + Concatenating a :class:`!Template` and a ``str`` is **not** supported. + This is because it is unclear whether the string should be treated as + a static string or an interpolation. + If you want to concatenate a :class:`!Template` with a string, + you should either wrap the string directly in a :class:`!Template` + (to treat it as a static string) + or use an :class:`!Interpolation` (to treat it as dynamic): + + >>> from string.templatelib import Interpolation, Template + >>> template = t'Ah! ' + >>> # Treat 'We do have ' as a static string + >>> template += Template('We do have ') + >>> # Treat cheese as an interpolation + >>> cheese = 'Camembert' + >>> template += Template(Interpolation(cheese, 'cheese')) + >>> list(template) + ['Ah! We do have ', Interpolation('Camembert', 'cheese', None, '')] + + +.. class:: Interpolation + + The :class:`!Interpolation` type represents an expression inside a template string. + It is immutable, meaning that attributes of an interpolation cannot be reassigned. + + Interpolations support pattern matching, allowing you to match against + their attributes with the :ref:`match statement `: + + >>> from string.templatelib import Interpolation + >>> interpolation = t'{1. + 2.:.2f}'.interpolations[0] + >>> interpolation + Interpolation(3.0, '1. + 2.', None, '.2f') + >>> match interpolation: + ... case Interpolation(value, expression, conversion, format_spec): + ... print(value, expression, conversion, format_spec, sep=' | ') + ... + 3.0 | 1. + 2. | None | .2f + + Interpolations are :ref:`generic ` over the types of their values. + + .. rubric:: Attributes + + .. attribute:: value + :type: object + + The evaluated value of the interpolation. + + >>> t'{1 + 2}'.interpolations[0].value + 3 + + .. attribute:: expression + :type: str + + For interpolations created by t-string literals, :attr:`!expression` + is the expression text found inside the curly brackets (``{`` & ``}``), + including any whitespace, excluding the curly brackets themselves, + and ending before the first ``!``, ``:``, or ``=`` if any is present. + For manually created interpolations, :attr:`!expression` is the arbitrary + string provided when constructing the interpolation instance. + + We recommend using valid Python expressions or the empty string for the + ``expression`` field of manually created :class:`!Interpolation` + instances, although this is not enforced at runtime. + + >>> t'{1 + 2}'.interpolations[0].expression + '1 + 2' + + .. attribute:: conversion + :type: typing.Literal['a', 'r', 's'] | None + + The conversion to apply to the value, or ``None``. + + The :attr:`!conversion` is the optional conversion to apply + to the value: + + >>> t'{1 + 2!a}'.interpolations[0].conversion + 'a' + + .. note:: + + Unlike f-strings, where conversions are applied automatically, + the expected behavior with t-strings is that code that *processes* the + :class:`!Template` will decide how to interpret and whether to apply + the :attr:`!conversion`. + For convenience, the :func:`convert` function can be used to mimic + f-string conversion semantics. + + .. attribute:: format_spec + :type: str + + The format specification to apply to the value. + + The :attr:`!format_spec` is an optional, arbitrary string + used as the format specification to present the value: + + >>> t'{1 + 2:.2f}'.interpolations[0].format_spec + '.2f' + + .. note:: + + Unlike f-strings, where format specifications are applied automatically + via the :func:`format` protocol, the expected behavior with + t-strings is that code that *processes* the interpolation will + decide how to interpret and whether to apply the format specification. + As a result, :attr:`!format_spec` values in interpolations + can be arbitrary strings, + including those that do not conform to the :func:`format` protocol. + + .. rubric:: Methods + + .. method:: __new__(value: object, \ + expression: str, \ + conversion: typing.Literal['a', 'r', 's'] | None = None, \ + format_spec: str = '') + + Create a new :class:`!Interpolation` object from component parts. + + :param value: The evaluated, in-scope result of the interpolation. + :param expression: The text of a valid Python expression, + or an empty string. + :param conversion: The :ref:`conversion ` to be used, + one of ``None``, ``'a'``, ``'r'``, or ``'s'``. + :param format_spec: An optional, arbitrary string used as the + :ref:`format specification ` to present the value. + + +Helper functions +---------------- + +.. function:: convert(obj, /, conversion) + + Applies formatted string literal :ref:`conversion ` + semantics to the given object *obj*. + This is frequently useful for custom template string processing logic. + + Three conversion flags are currently supported: + + * ``'s'`` which calls :func:`str` on the value (like ``!s``), + * ``'r'`` which calls :func:`repr` (like ``!r``), and + * ``'a'`` which calls :func:`ascii` (like ``!a``). + + If the conversion flag is ``None``, *obj* is returned unchanged. diff --git a/Doc/library/stringprep.rst b/Doc/library/stringprep.rst index 37d5adf0fa95411..b9caa2aa830e944 100644 --- a/Doc/library/stringprep.rst +++ b/Doc/library/stringprep.rst @@ -26,14 +26,14 @@ define which tables it uses, and what other optional parts of the ``stringprep`` procedure are part of the profile. One example of a ``stringprep`` profile is ``nameprep``, which is used for internationalized domain names. -The module :mod:`stringprep` only exposes the tables from :rfc:`3454`. As these +The module :mod:`!stringprep` only exposes the tables from :rfc:`3454`. As these tables would be very large to represent as dictionaries or lists, the module uses the Unicode character database internally. The module source code itself was generated using the ``mkstringprep.py`` utility. As a result, these tables are exposed as functions, not as data structures. There are two kinds of tables in the RFC: sets and mappings. For a set, -:mod:`stringprep` provides the "characteristic function", i.e. a function that +:mod:`!stringprep` provides the "characteristic function", i.e. a function that returns ``True`` if the parameter is part of the set. For mappings, it provides the mapping function: given the key, it returns the associated value. Below is a list of all functions available in the module. diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 17fc479fd0c8c93..f4fdff951661b6d 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -36,7 +36,7 @@ and the C layer. responsible for defining byte ordering and padding between elements. See :ref:`struct-alignment` for details. -Several :mod:`struct` functions (and methods of :class:`Struct`) take a *buffer* +Several :mod:`!struct` functions (and methods of :class:`Struct`) take a *buffer* argument. This refers to objects that implement the :ref:`bufferobjects` and provide either a readable or read-writable buffer. The most common types used for that purpose are :class:`bytes` and :class:`bytearray`, but many other types @@ -68,6 +68,7 @@ The module defines the following exception and functions: Pack the values *v1*, *v2*, ... according to the format string *format* and write the packed bytes into the writable buffer *buffer* starting at position *offset*. Note that *offset* is a required argument. + A negative *offset* counts from the end of *buffer*. .. function:: unpack(format, buffer) @@ -84,6 +85,7 @@ The module defines the following exception and functions: string *format*. The result is a tuple even if it contains exactly one item. The buffer's size in bytes, starting at position *offset*, must be at least the size required by the format, as reflected by :func:`calcsize`. + A negative *offset* counts from the end of *buffer*. .. function:: iter_unpack(format, buffer) @@ -227,34 +229,34 @@ platform-dependent. +--------+--------------------------+--------------------+----------------+------------+ | ``c`` | :c:expr:`char` | bytes of length 1 | 1 | | +--------+--------------------------+--------------------+----------------+------------+ -| ``b`` | :c:expr:`signed char` | integer | 1 | \(1), \(2) | +| ``b`` | :c:expr:`signed char` | int | 1 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``B`` | :c:expr:`unsigned char` | integer | 1 | \(2) | +| ``B`` | :c:expr:`unsigned char` | int | 1 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ | ``?`` | :c:expr:`_Bool` | bool | 1 | \(1) | +--------+--------------------------+--------------------+----------------+------------+ -| ``h`` | :c:expr:`short` | integer | 2 | \(2) | +| ``h`` | :c:expr:`short` | int | 2 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``H`` | :c:expr:`unsigned short` | integer | 2 | \(2) | +| ``H`` | :c:expr:`unsigned short` | int | 2 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``i`` | :c:expr:`int` | integer | 4 | \(2) | +| ``i`` | :c:expr:`int` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``I`` | :c:expr:`unsigned int` | integer | 4 | \(2) | +| ``I`` | :c:expr:`unsigned int` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``l`` | :c:expr:`long` | integer | 4 | \(2) | +| ``l`` | :c:expr:`long` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``L`` | :c:expr:`unsigned long` | integer | 4 | \(2) | +| ``L`` | :c:expr:`unsigned long` | int | 4 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``q`` | :c:expr:`long long` | integer | 8 | \(2) | +| ``q`` | :c:expr:`long long` | int | 8 | \(2) | +--------+--------------------------+--------------------+----------------+------------+ -| ``Q`` | :c:expr:`unsigned long | integer | 8 | \(2) | +| ``Q`` | :c:expr:`unsigned long | int | 8 | \(2) | | | long` | | | | +--------+--------------------------+--------------------+----------------+------------+ -| ``n`` | :c:type:`ssize_t` | integer | | \(3) | +| ``n`` | :c:type:`ssize_t` | int | | \(2), \(3) | +--------+--------------------------+--------------------+----------------+------------+ -| ``N`` | :c:type:`size_t` | integer | | \(3) | +| ``N`` | :c:type:`size_t` | int | | \(2), \(3) | +--------+--------------------------+--------------------+----------------+------------+ -| ``e`` | \(6) | float | 2 | \(4) | +| ``e`` | :c:expr:`_Float16` | float | 2 | \(4), \(6) | +--------+--------------------------+--------------------+----------------+------------+ | ``f`` | :c:expr:`float` | float | 4 | \(4) | +--------+--------------------------+--------------------+----------------+------------+ @@ -268,7 +270,7 @@ platform-dependent. +--------+--------------------------+--------------------+----------------+------------+ | ``p`` | :c:expr:`char[]` | bytes | | \(8) | +--------+--------------------------+--------------------+----------------+------------+ -| ``P`` | :c:expr:`void \*` | integer | | \(5) | +| ``P`` | :c:expr:`void \*` | int | | \(2), \(5) | +--------+--------------------------+--------------------+----------------+------------+ .. versionchanged:: 3.3 @@ -280,6 +282,12 @@ platform-dependent. .. versionchanged:: 3.14 Added support for the ``'F'`` and ``'D'`` formats. +.. seealso:: + + The :mod:`array` and :ref:`ctypes ` modules, + as well as third-party modules like `numpy `__, + use similar -- but slightly different -- type codes. + Notes: @@ -322,7 +330,9 @@ Notes: revision of the `IEEE 754 standard `_. It has a sign bit, a 5-bit exponent and 11-bit precision (with 10 bits explicitly stored), and can represent numbers between approximately ``6.1e-05`` and ``6.5e+04`` - at full precision. This type is not widely supported by C compilers: on a + at full precision. This type is not widely supported by C compilers: + it's available as :c:expr:`_Float16` type, if the compiler supports the Annex H + of the C23 standard. On a typical machine, an unsigned short can be used for storage, but not for math operations. See the Wikipedia page on the `half-precision floating-point format `_ for more information. @@ -334,27 +344,31 @@ Notes: The ``'p'`` format character encodes a "Pascal string", meaning a short variable-length string stored in a *fixed number of bytes*, given by the count. The first byte stored is the length of the string, or 255, whichever is - smaller. The bytes of the string follow. If the string passed in to + smaller. The bytes of the string follow. If the byte string passed in to :func:`pack` is too long (longer than the count minus 1), only the leading - ``count-1`` bytes of the string are stored. If the string is shorter than + ``count-1`` bytes of the string are stored. If the byte string is shorter than ``count-1``, it is padded with null bytes so that exactly count bytes in all are used. Note that for :func:`unpack`, the ``'p'`` format character consumes - ``count`` bytes, but that the string returned can never contain more than 255 + ``count`` bytes, but that the :class:`!bytes` object returned can never contain more than 255 bytes. + When packing, arguments of types :class:`bytes` and :class:`bytearray` + are accepted. (9) For the ``'s'`` format character, the count is interpreted as the length of the - bytes, not a repeat count like for the other format characters; for example, + byte string, not a repeat count like for the other format characters; for example, ``'10s'`` means a single 10-byte string mapping to or from a single Python byte string, while ``'10c'`` means 10 separate one byte character elements (e.g., ``cccccccccc``) mapping to or from ten different Python byte objects. (See :ref:`struct-examples` for a concrete demonstration of the difference.) - If a count is not given, it defaults to 1. For packing, the string is + If a count is not given, it defaults to 1. For packing, the byte string is truncated or padded with null bytes as appropriate to make it fit. For - unpacking, the resulting bytes object always has exactly the specified number - of bytes. As a special case, ``'0s'`` means a single, empty string (while + unpacking, the resulting :class:`!bytes` object always has exactly the specified number + of bytes. As a special case, ``'0s'`` means a single, empty byte string (while ``'0c'`` means 0 characters). + When packing, arguments of types :class:`bytes` and :class:`bytearray` + are accepted. (10) For the ``'F'`` and ``'D'`` format characters, the packed representation uses @@ -479,7 +493,7 @@ at the end, assuming the platform's longs are aligned on 4-byte boundaries:: Applications ------------ -Two main applications for the :mod:`struct` module exist, data +Two main applications for the :mod:`!struct` module exist, data interchange between Python and C code within an application or another application compiled using the same compiler (:ref:`native formats`), and data interchange between applications using agreed upon data layout @@ -571,7 +585,7 @@ below were executed on a 32-bit machine:: Classes ------- -The :mod:`struct` module also defines the following type: +The :mod:`!struct` module also defines the following type: .. class:: Struct(format) diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 028a7861f36798a..66a3d6a484a8a8f 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -11,14 +11,14 @@ -------------- -The :mod:`subprocess` module allows you to spawn new processes, connect to their +The :mod:`!subprocess` module allows you to spawn new processes, connect to their input/output/error pipes, and obtain their return codes. This module intends to replace several older modules and functions:: os.system os.spawn* -Information about how the :mod:`subprocess` module can be used to replace these +Information about how the :mod:`!subprocess` module can be used to replace these modules and functions can be found in the following sections. .. seealso:: @@ -27,8 +27,8 @@ modules and functions can be found in the following sections. .. include:: ../includes/wasm-mobile-notavail.rst -Using the :mod:`subprocess` Module ----------------------------------- +Using the :mod:`!subprocess` Module +----------------------------------- The recommended approach to invoking subprocesses is to use the :func:`run` function for all use cases it can handle. For more advanced use cases, the @@ -630,6 +630,12 @@ functions. the value in ``pw_uid`` will be used. If the value is an integer, it will be passed verbatim. (POSIX only) + .. note:: + + Specifying *user* will not drop existing supplementary group memberships! + The caller must also pass ``extra_groups=()`` to reduce the group membership + of the child process for security purposes. + .. availability:: POSIX .. versionadded:: 3.9 @@ -649,7 +655,7 @@ functions. If specified, *env* must provide any variables required for the program to execute. On Windows, in order to run a `side-by-side assembly`_ the - specified *env* **must** include a valid :envvar:`SystemRoot`. + specified *env* **must** include a valid ``%SystemRoot%``. .. _side-by-side assembly: https://en.wikipedia.org/wiki/Side-by-Side_Assembly @@ -831,7 +837,9 @@ Instances of the :class:`Popen` class have the following methods: If the process does not terminate after *timeout* seconds, a :exc:`TimeoutExpired` exception will be raised. Catching this exception and - retrying communication will not lose any output. + retrying communication will not lose any output. Supplying *input* to a + subsequent post-timeout :meth:`communicate` call is in undefined behavior + and may become an error in the future. The child process is not killed if the timeout expires, so in order to cleanup properly a well-behaved application should kill the child process and @@ -844,6 +852,11 @@ Instances of the :class:`Popen` class have the following methods: proc.kill() outs, errs = proc.communicate() + After a call to :meth:`~Popen.communicate` raises :exc:`TimeoutExpired`, do + not call :meth:`~Popen.wait`. Use an additional :meth:`~Popen.communicate` + call to finish handling pipes and populate the :attr:`~Popen.returncode` + attribute. + .. note:: The data read is buffered in memory, so do not use this method if the data @@ -945,6 +958,11 @@ Reassigning them to new values is unsupported: A negative value ``-N`` indicates that the child was terminated by signal ``N`` (POSIX only). + When ``shell=True``, the return code reflects the exit status of the shell + itself (e.g. ``/bin/sh``), which may map signals to codes such as + ``128+N``. See the documentation of the shell (for example, the Bash + manual's Exit Status) for details. + Windows Popen Helpers --------------------- @@ -1034,7 +1052,7 @@ on Windows. Windows Constants ^^^^^^^^^^^^^^^^^ -The :mod:`subprocess` module exposes the following constants. +The :mod:`!subprocess` module exposes the following constants. .. data:: STD_INPUT_HANDLE @@ -1323,8 +1341,8 @@ calls these functions. .. _subprocess-replacements: -Replacing Older Functions with the :mod:`subprocess` Module ------------------------------------------------------------ +Replacing Older Functions with the :mod:`!subprocess` Module +------------------------------------------------------------ In this section, "a becomes b" means that b can be used as a replacement for a. @@ -1340,7 +1358,7 @@ In this section, "a becomes b" means that b can be used as a replacement for a. :attr:`~CalledProcessError.output` attribute of the raised exception. In the following examples, we assume that the relevant functions have already -been imported from the :mod:`subprocess` module. +been imported from the :mod:`!subprocess` module. Replacing :program:`/bin/sh` shell command substitution @@ -1400,7 +1418,7 @@ Notes: * The :func:`os.system` function ignores SIGINT and SIGQUIT signals while the command is running, but the caller must do this separately when - using the :mod:`subprocess` module. + using the :mod:`!subprocess` module. A more realistic example would look like this:: @@ -1473,7 +1491,7 @@ handling consistency are valid for these functions. Return ``(exitcode, output)`` of executing *cmd* in a shell. - Execute the string *cmd* in a shell with :meth:`Popen.check_output` and + Execute the string *cmd* in a shell with :func:`check_output` and return a 2-tuple ``(exitcode, output)``. *encoding* and *errors* are used to decode output; see the notes on :ref:`frequently-used-arguments` for more details. @@ -1584,7 +1602,7 @@ runtime): Disable use of ``posix_spawn()`` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -On Linux, :mod:`subprocess` defaults to using the ``vfork()`` system call +On Linux, :mod:`!subprocess` defaults to using the ``vfork()`` system call internally when it is safe to do so rather than ``fork()``. This greatly improves performance. diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 54e19af4bd69a67..2f7bba2537dc5fd 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -14,7 +14,7 @@ Symbol tables are generated by the compiler from AST just before bytecode is generated. The symbol table is responsible for calculating the scope of every -identifier in the code. :mod:`symtable` provides an interface to examine these +identifier in the code. :mod:`!symtable` provides an interface to examine these tables. @@ -355,7 +355,7 @@ Command-Line Usage .. versionadded:: 3.13 -The :mod:`symtable` module can be executed as a script from the command line. +The :mod:`!symtable` module can be executed as a script from the command line. .. code-block:: sh diff --git a/Doc/library/sys.monitoring.rst b/Doc/library/sys.monitoring.rst index 0674074b8c0df68..16e6b1d6dc786b0 100644 --- a/Doc/library/sys.monitoring.rst +++ b/Doc/library/sys.monitoring.rst @@ -10,17 +10,17 @@ .. note:: - :mod:`sys.monitoring` is a namespace within the :mod:`sys` module, - not an independent module, so there is no need to - ``import sys.monitoring``, simply ``import sys`` and then use - ``sys.monitoring``. + :mod:`!sys.monitoring` is a namespace within the :mod:`sys` module, + not an independent module, and ``import sys.monitoring`` would fail + with a :exc:`ModuleNotFoundError`. Instead, simply ``import sys`` + and then use ``sys.monitoring``. This namespace provides access to the functions and constants necessary to activate and control event monitoring. As programs execute, events occur that might be of interest to tools that -monitor execution. The :mod:`sys.monitoring` namespace provides means to +monitor execution. The :mod:`!sys.monitoring` namespace provides means to receive callbacks when events of interest occur. The monitoring API consists of three components: @@ -137,7 +137,8 @@ The following events are supported: .. monitoring-event:: PY_UNWIND - Exit from a Python function during exception unwinding. + Exit from a Python function during exception unwinding. This includes exceptions raised directly within the + function and that are allowed to continue to propagate. .. monitoring-event:: PY_YIELD @@ -171,7 +172,7 @@ events, use the expression ``PY_RETURN | PY_START``. if get_events(DEBUGGER_ID) == NO_EVENTS: ... -Events are divided into three groups: + Setting this event deactivates all events. .. _monitoring-event-local: @@ -215,14 +216,17 @@ by another event: The :monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events are controlled by the :monitoring-event:`CALL` event. -:monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events will only be seen if the -corresponding :monitoring-event:`CALL` event is being monitored. +:monitoring-event:`C_RETURN` and :monitoring-event:`C_RAISE` events will only be +seen if the corresponding :monitoring-event:`CALL` event is being monitored. + + +.. _monitoring-event-global: Other events '''''''''''' Other events are not necessarily tied to a specific location in the -program and cannot be individually disabled. +program and cannot be individually disabled via :data:`DISABLE`. The other events that can be monitored are: @@ -243,20 +247,23 @@ raise an exception unless it would be visible to other code. To allow tools to monitor for real exceptions without slowing down generators and coroutines, the :monitoring-event:`STOP_ITERATION` event is provided. -:monitoring-event:`STOP_ITERATION` can be locally disabled, unlike :monitoring-event:`RAISE`. +:monitoring-event:`STOP_ITERATION` can be locally disabled, unlike +:monitoring-event:`RAISE`. -Note that the :monitoring-event:`STOP_ITERATION` event and the :monitoring-event:`RAISE` -event for a :exc:`StopIteration` exception are equivalent, and are treated as interchangeable -when generating events. Implementations will favor :monitoring-event:`STOP_ITERATION` for -performance reasons, but may generate a :monitoring-event:`RAISE` event with a :exc:`StopIteration`. +Note that the :monitoring-event:`STOP_ITERATION` event and the +:monitoring-event:`RAISE` event for a :exc:`StopIteration` exception are +equivalent, and are treated as interchangeable when generating events. +Implementations will favor :monitoring-event:`STOP_ITERATION` for performance +reasons, but may generate a :monitoring-event:`RAISE` event with a +:exc:`StopIteration`. Turning events on and off ------------------------- In order to monitor an event, it must be turned on and a corresponding callback -must be registered. -Events can be turned on or off by setting the events either globally or -for a particular code object. +must be registered. Events can be turned on or off by setting the events either +globally and/or for a particular code object. An event will trigger only once, +even if it is turned on both globally and locally. Setting events globally @@ -285,16 +292,13 @@ in Python (see :ref:`c-api-monitoring`). .. function:: get_local_events(tool_id: int, code: CodeType, /) -> int - Returns all the local events for *code* + Returns all the :ref:`local events ` for *code* .. function:: set_local_events(tool_id: int, code: CodeType, event_set: int, /) -> None - Activates all the local events for *code* which are set in *event_set*. - Raises a :exc:`ValueError` if *tool_id* is not in use. - -Local events add to global events, but do not mask them. -In other words, all global events will trigger for a code object, -regardless of the local events. + Activates all the :ref:`local events ` for *code* + which are set in *event_set*. Raises a :exc:`ValueError` if *tool_id* is not + in use. Disabling events @@ -305,15 +309,21 @@ Disabling events A special value that can be returned from a callback function to disable events for the current code location. -Local events can be disabled for a specific code location by returning -:data:`sys.monitoring.DISABLE` from a callback function. This does not change -which events are set, or any other code locations for the same event. +:ref:`Local events ` can be disabled for a specific code +location by returning :data:`sys.monitoring.DISABLE` from a callback function. +This does not change which events are set, or any other code locations for the +same event. Disabling events for specific locations is very important for high performance monitoring. For example, a program can be run under a debugger with no overhead if the debugger disables all monitoring except for a few breakpoints. +If :data:`DISABLE` is returned by a callback for a +:ref:`global event `, :exc:`ValueError` will be raised +by the interpreter in a non-specific location (that is, no traceback will be +provided). + .. function:: restart_events() -> None Enable all the events that were disabled by :data:`sys.monitoring.DISABLE` @@ -325,8 +335,6 @@ except for a few breakpoints. Registering callback functions ------------------------------ -To register a callable for events call - .. function:: register_callback(tool_id: int, event: int, func: Callable | None, /) -> Callable | None Registers the callable *func* for the *event* with the given *tool_id* @@ -335,13 +343,17 @@ To register a callable for events call it is unregistered and returned. Otherwise :func:`register_callback` returns ``None``. + .. audit-event:: sys.monitoring.register_callback func sys.monitoring.register_callback Functions can be unregistered by calling ``sys.monitoring.register_callback(tool_id, event, None)``. Callback functions can be registered and unregistered at any time. -Registering or unregistering a callback function will generate a :func:`sys.audit` event. +Callbacks are called only once regardless if the event is turned on both +globally and locally. As such, if an event could be turned on for both global +and local events by your code then the callback needs to be written to handle +either trigger. Callback function arguments @@ -353,37 +365,46 @@ Callback function arguments that there are no arguments to the call. When an active event occurs, the registered callback function is called. +Callback functions returning an object other than :data:`DISABLE` will have no effect. Different events will provide the callback function with different arguments, as follows: * :monitoring-event:`PY_START` and :monitoring-event:`PY_RESUME`:: - func(code: CodeType, instruction_offset: int) -> DISABLE | Any + func(code: CodeType, instruction_offset: int) -> object * :monitoring-event:`PY_RETURN` and :monitoring-event:`PY_YIELD`:: - func(code: CodeType, instruction_offset: int, retval: object) -> DISABLE | Any + func(code: CodeType, instruction_offset: int, retval: object) -> object -* :monitoring-event:`CALL`, :monitoring-event:`C_RAISE` and :monitoring-event:`C_RETURN`:: +* :monitoring-event:`CALL`, :monitoring-event:`C_RAISE` and :monitoring-event:`C_RETURN` + (*arg0* can be :data:`MISSING` specifically):: - func(code: CodeType, instruction_offset: int, callable: object, arg0: object | MISSING) -> DISABLE | Any + func(code: CodeType, instruction_offset: int, callable: object, arg0: object) -> object + *code* represents the code object where the call is being made, while + *callable* is the object that is about to be called (and thus + triggered the event). If there are no arguments, *arg0* is set to :data:`sys.monitoring.MISSING`. + For instance methods, *callable* will be the function object as found on the + class with *arg0* set to the instance (i.e. the ``self`` argument to the + method). + * :monitoring-event:`RAISE`, :monitoring-event:`RERAISE`, :monitoring-event:`EXCEPTION_HANDLED`, :monitoring-event:`PY_UNWIND`, :monitoring-event:`PY_THROW` and :monitoring-event:`STOP_ITERATION`:: - func(code: CodeType, instruction_offset: int, exception: BaseException) -> DISABLE | Any + func(code: CodeType, instruction_offset: int, exception: BaseException) -> object * :monitoring-event:`LINE`:: - func(code: CodeType, line_number: int) -> DISABLE | Any + func(code: CodeType, line_number: int) -> object * :monitoring-event:`BRANCH_LEFT`, :monitoring-event:`BRANCH_RIGHT` and :monitoring-event:`JUMP`:: - func(code: CodeType, instruction_offset: int, destination_offset: int) -> DISABLE | Any + func(code: CodeType, instruction_offset: int, destination_offset: int) -> object Note that the *destination_offset* is where the code will next execute. * :monitoring-event:`INSTRUCTION`:: - func(code: CodeType, instruction_offset: int) -> DISABLE | Any + func(code: CodeType, instruction_offset: int) -> object diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 55e442b20ff8775..9078af7cac78d61 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -515,7 +515,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only in the range 0--127, and produce undefined results otherwise. Some systems have a convention for assigning specific meanings to specific exit codes, but these are generally underdeveloped; Unix programs generally use 2 for command - line syntax errors and 1 for all other kind of errors. If another type of + line syntax errors and 1 for all other kinds of errors. If another type of object is passed, ``None`` is equivalent to passing zero, and any other object is printed to :data:`stderr` and results in an exit code of 1. In particular, ``sys.exit("some error message")`` is a quick way to exit a @@ -953,6 +953,8 @@ always available. Unless explicitly noted otherwise, all variables are read-only This function should be used for internal and specialized purposes only. It is not guaranteed to exist in all implementations of Python. + .. versionadded:: 3.12 + .. function:: getobjects(limit[, type]) @@ -1128,10 +1130,14 @@ always available. Unless explicitly noted otherwise, all variables are read-only The size of the seed key of the hash algorithm + .. attribute:: hash_info.cutoff + + Cutoff for small string DJBX33A optimization in range ``[1, cutoff)``. + .. versionadded:: 3.2 .. versionchanged:: 3.4 - Added *algorithm*, *hash_bits* and *seed_bits* + Added *algorithm*, *hash_bits*, *seed_bits*, and *cutoff*. .. data:: hexversion @@ -1185,6 +1191,15 @@ always available. Unless explicitly noted otherwise, all variables are read-only ``cache_tag`` is set to ``None``, it indicates that module caching should be disabled. + *supports_isolated_interpreters* is a boolean value, whether + this implementation supports multiple isolated interpreters. + It is ``True`` for CPython on most platforms. Platforms with + this support implement the low-level :mod:`!_interpreters` module. + + .. seealso:: + + :pep:`684`, :pep:`734`, and :mod:`concurrent.interpreters`. + :data:`sys.implementation` may contain additional attributes specific to the Python implementation. These non-standard attributes must start with an underscore, and are not described here. Regardless of its contents, @@ -1194,6 +1209,9 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. versionadded:: 3.3 + .. versionchanged:: 3.14 + Added ``supports_isolated_interpreters`` field. + .. note:: The addition of new required attributes must go through the normal PEP @@ -1750,7 +1768,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only :func:`settrace` for each thread being debugged or use :func:`threading.settrace`. Trace functions should have three arguments: *frame*, *event*, and - *arg*. *frame* is the current stack frame. *event* is a string: ``'call'``, + *arg*. *frame* is the :ref:`current stack frame `. *event* is a string: ``'call'``, ``'line'``, ``'return'``, ``'exception'`` or ``'opcode'``. *arg* depends on the event type. @@ -1933,8 +1951,28 @@ always available. Unless explicitly noted otherwise, all variables are read-only interpreter is pre-release (alpha, beta, or release candidate) then the local and remote interpreters must be the same exact version. + See :ref:`remote-debugging` for more information about the remote debugging + mechanism. + + .. audit-event:: sys.remote_exec pid script_path + + When the code is executed in the remote process, an + :ref:`auditing event ` ``sys.remote_exec`` is raised with + the *pid* and the path to the script file. + This event is raised in the process that called :func:`sys.remote_exec`. + + .. audit-event:: cpython.remote_debugger_script script_path + + When the script is executed in the remote process, an + :ref:`auditing event ` + ``cpython.remote_debugger_script`` is raised + with the path in the remote process. + This event is raised in the remote process, not the one + that called :func:`sys.remote_exec`. + .. availability:: Unix, Windows. .. versionadded:: 3.14 + See :pep:`768` for more details. .. function:: _enablelegacywindowsfsencoding() @@ -1979,7 +2017,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only returned by the :func:`open` function. Their parameters are chosen as follows: - * The encoding and error handling are is initialized from + * The encoding and error handling are initialized from :c:member:`PyConfig.stdio_encoding` and :c:member:`PyConfig.stdio_errors`. On Windows, UTF-8 is used for the console device. Non-character @@ -2161,8 +2199,11 @@ always available. Unless explicitly noted otherwise, all variables are read-only .. data:: api_version - The C API version for this interpreter. Programmers may find this useful when - debugging version conflicts between Python and extension modules. + The C API version, equivalent to the C macro :c:macro:`PYTHON_API_VERSION`. + Defined for backwards compatibility. + + Currently, this constant is not updated in new Python versions, and is not + useful for versioning. This may change in the future. .. data:: version_info @@ -2189,7 +2230,7 @@ always available. Unless explicitly noted otherwise, all variables are read-only The version number used to form registry keys on Windows platforms. This is stored as string resource 1000 in the Python DLL. The value is normally the - major and minor versions of the running Python interpreter. It is provided in the :mod:`sys` + major and minor versions of the running Python interpreter. It is provided in the :mod:`!sys` module for informational purposes; modifying this value has no effect on the registry keys used by Python. diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 684d14a74c48ab0..138af76a66f6b8b 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -16,7 +16,7 @@ -------------- -The :mod:`sysconfig` module provides access to Python's configuration +The :mod:`!sysconfig` module provides access to Python's configuration information like the list of installation paths and the configuration variables relevant for the current platform. @@ -28,7 +28,7 @@ A Python distribution contains a :file:`Makefile` and a :file:`pyconfig.h` header file that are necessary to build both the Python binary itself and third-party C extensions compiled using ``setuptools``. -:mod:`sysconfig` puts all variables found in these files in a dictionary that +:mod:`!sysconfig` puts all variables found in these files in a dictionary that can be accessed using :func:`get_config_vars` or :func:`get_config_var`. Notice that on Windows, it's a much smaller set. @@ -68,7 +68,7 @@ Installation paths ------------------ Python uses an installation scheme that differs depending on the platform and on -the installation options. These schemes are stored in :mod:`sysconfig` under +the installation options. These schemes are stored in :mod:`!sysconfig` under unique identifiers based on the value returned by :const:`os.name`. The schemes are used by package installers to determine where to copy files to. @@ -258,12 +258,12 @@ Path Installation directory Installation path functions --------------------------- -:mod:`sysconfig` provides some functions to determine these installation paths. +:mod:`!sysconfig` provides some functions to determine these installation paths. .. function:: get_scheme_names() Return a tuple containing all schemes currently supported in - :mod:`sysconfig`. + :mod:`!sysconfig`. .. function:: get_default_scheme() @@ -285,7 +285,7 @@ Installation path functions *key* must be either ``"prefix"``, ``"home"``, or ``"user"``. The return value is a scheme name listed in :func:`get_scheme_names`. It - can be passed to :mod:`sysconfig` functions that take a *scheme* argument, + can be passed to :mod:`!sysconfig` functions that take a *scheme* argument, such as :func:`get_paths`. .. versionadded:: 3.10 @@ -313,7 +313,7 @@ Installation path functions .. function:: get_path_names() Return a tuple containing all path names currently supported in - :mod:`sysconfig`. + :mod:`!sysconfig`. .. function:: get_path(name, [scheme, [vars, [expand]]]) @@ -323,7 +323,7 @@ Installation path functions *name* has to be a value from the list returned by :func:`get_path_names`. - :mod:`sysconfig` stores installation paths corresponding to each path name, + :mod:`!sysconfig` stores installation paths corresponding to each path name, for each platform, with variables to be expanded. For instance the *stdlib* path for the *nt* scheme is: ``{base}/Lib``. @@ -382,22 +382,19 @@ Other functions Examples of returned values: - - linux-i586 - - linux-alpha (?) - - solaris-2.6-sun4u - Windows will return one of: + Windows: - win-amd64 (64-bit Windows on AMD64, aka x86_64, Intel64, and EM64T) - win-arm64 (64-bit Windows on ARM64, aka AArch64) - win32 (all others - specifically, sys.platform is returned) - macOS can return: + POSIX based OS: - - macosx-10.6-ppc - - macosx-10.4-ppc64 - - macosx-10.3-i386 - - macosx-10.4-fat + - linux-x86_64 + - macosx-15.5-arm64 + - macosx-26.0-universal2 (macOS on Apple Silicon or Intel) + - android-24-arm64_v8a For other non-POSIX platforms, currently just returns :data:`sys.platform`. @@ -434,7 +431,7 @@ Other functions Command-line usage ------------------ -You can use :mod:`sysconfig` as a script with Python's *-m* option: +You can use :mod:`!sysconfig` as a script with Python's *-m* option: .. code-block:: shell-session diff --git a/Doc/library/syslog.rst b/Doc/library/syslog.rst index 548898a37bc6ea7..b6bf01240951ebd 100644 --- a/Doc/library/syslog.rst +++ b/Doc/library/syslog.rst @@ -2,7 +2,6 @@ =============================================== .. module:: syslog - :platform: Unix :synopsis: An interface to the Unix syslog library routines. -------------- diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index f9cb5495e60cd28..a87ec10b3082335 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -11,15 +11,23 @@ -------------- -The :mod:`tarfile` module makes it possible to read and write tar +The :mod:`!tarfile` module makes it possible to read and write tar archives, including those using gzip, bz2 and lzma compression. Use the :mod:`zipfile` module to read or write :file:`.zip` files, or the higher-level functions in :ref:`shutil `. Some facts and figures: -* reads and writes :mod:`gzip`, :mod:`bz2` and :mod:`lzma` compressed archives - if the respective modules are available. +* reads and writes :mod:`gzip`, :mod:`bz2`, :mod:`compression.zstd`, and + :mod:`lzma` compressed archives if the respective modules are available. + + .. + The following paragraph should be similar to ../includes/optional-module.rst + + If any of these :term:`optional modules ` are missing from + your copy of CPython, look for documentation from your distributor (that is, + whoever provided Python to you). + If you are the distributor, see :ref:`optional-module-requirements`. * read/write support for the POSIX.1-1988 (ustar) format. @@ -47,6 +55,10 @@ Some facts and figures: or paths outside of the destination. Previously, the filter strategy was equivalent to :func:`fully_trusted `. +.. versionchanged:: 3.14 + + Added support for Zstandard compression using :mod:`compression.zstd`. + .. function:: open(name=None, mode='r', fileobj=None, bufsize=10240, **kwargs) Return a :class:`TarFile` object for the pathname *name*. For detailed @@ -59,8 +71,8 @@ Some facts and figures: +------------------+---------------------------------------------+ | mode | action | +==================+=============================================+ - | ``'r' or 'r:*'`` | Open for reading with transparent | - | | compression (recommended). | + | ``'r'`` or | Open for reading with transparent | + | ``'r:*'`` | compression (recommended). | +------------------+---------------------------------------------+ | ``'r:'`` | Open for reading exclusively without | | | compression. | @@ -71,6 +83,8 @@ Some facts and figures: +------------------+---------------------------------------------+ | ``'r:xz'`` | Open for reading with lzma compression. | +------------------+---------------------------------------------+ + | ``'r:zst'`` | Open for reading with Zstandard compression.| + +------------------+---------------------------------------------+ | ``'x'`` or | Create a tarfile exclusively without | | ``'x:'`` | compression. | | | Raise a :exc:`FileExistsError` exception | @@ -88,10 +102,15 @@ Some facts and figures: | | Raise a :exc:`FileExistsError` exception | | | if it already exists. | +------------------+---------------------------------------------+ - | ``'a' or 'a:'`` | Open for appending with no compression. The | - | | file is created if it does not exist. | + | ``'x:zst'`` | Create a tarfile with Zstandard compression.| + | | Raise a :exc:`FileExistsError` exception | + | | if it already exists. | + +------------------+---------------------------------------------+ + | ``'a'`` or | Open for appending with no compression. The | + | ``'a:'`` | file is created if it does not exist. | +------------------+---------------------------------------------+ - | ``'w' or 'w:'`` | Open for uncompressed writing. | + | ``'w'`` or | Open for uncompressed writing. | + | ``'w:'`` | | +------------------+---------------------------------------------+ | ``'w:gz'`` | Open for gzip compressed writing. | +------------------+---------------------------------------------+ @@ -99,6 +118,8 @@ Some facts and figures: +------------------+---------------------------------------------+ | ``'w:xz'`` | Open for lzma compressed writing. | +------------------+---------------------------------------------+ + | ``'w:zst'`` | Open for Zstandard compressed writing. | + +------------------+---------------------------------------------+ Note that ``'a:gz'``, ``'a:bz2'`` or ``'a:xz'`` is not possible. If *mode* is not suitable to open a certain (compressed) file for reading, @@ -115,6 +136,15 @@ Some facts and figures: For modes ``'w:xz'``, ``'x:xz'`` and ``'w|xz'``, :func:`tarfile.open` accepts the keyword argument *preset* to specify the compression level of the file. + For modes ``'w:zst'``, ``'x:zst'`` and ``'w|zst'``, :func:`tarfile.open` + accepts the keyword argument *level* to specify the compression level of + the file. The keyword argument *options* may also be passed, providing + advanced Zstandard compression parameters described by + :class:`~compression.zstd.CompressionParameter`. The keyword argument + *zstd_dict* can be passed to provide a :class:`~compression.zstd.ZstdDict`, + a Zstandard dictionary used to improve compression of smaller amounts of + data. + For special purposes, there is a second format for *mode*: ``'filemode|[compression]'``. :func:`tarfile.open` will return a :class:`TarFile` object that processes its data as a stream of blocks. No random seeking will @@ -146,6 +176,9 @@ Some facts and figures: | ``'r|xz'`` | Open an lzma compressed *stream* for | | | reading. | +-------------+--------------------------------------------+ + | ``'r|zst'`` | Open a Zstandard compressed *stream* for | + | | reading. | + +-------------+--------------------------------------------+ | ``'w|'`` | Open an uncompressed *stream* for writing. | +-------------+--------------------------------------------+ | ``'w|gz'`` | Open a gzip compressed *stream* for | @@ -157,6 +190,9 @@ Some facts and figures: | ``'w|xz'`` | Open an lzma compressed *stream* for | | | writing. | +-------------+--------------------------------------------+ + | ``'w|zst'`` | Open a Zstandard compressed *stream* for | + | | writing. | + +-------------+--------------------------------------------+ .. versionchanged:: 3.5 The ``'x'`` (exclusive creation) mode was added. @@ -180,25 +216,25 @@ Some facts and figures: .. function:: is_tarfile(name) - Return :const:`True` if *name* is a tar archive file, that the :mod:`tarfile` + Return :const:`True` if *name* is a tar archive file, that the :mod:`!tarfile` module can read. *name* may be a :class:`str`, file, or file-like object. .. versionchanged:: 3.9 Support for file and file-like objects. -The :mod:`tarfile` module defines the following exceptions: +The :mod:`!tarfile` module defines the following exceptions: .. exception:: TarError - Base class for all :mod:`tarfile` exceptions. + Base class for all :mod:`!tarfile` exceptions. .. exception:: ReadError Is raised when a tar archive is opened, that either cannot be handled by the - :mod:`tarfile` module or is somehow invalid. + :mod:`!tarfile` module or is somehow invalid. .. exception:: CompressionError @@ -255,6 +291,15 @@ The :mod:`tarfile` module defines the following exceptions: Raised to refuse extracting a symbolic link pointing outside the destination directory. +.. exception:: LinkFallbackError + + Raised to refuse emulating a link (hard or symbolic) by extracting another + archive member, when that member would be rejected by the filter location. + The exception that was raised to reject the replacement member is available + as :attr:`!BaseException.__context__`. + + .. versionadded:: 3.14 + The following constants are available at the module level: @@ -310,7 +355,7 @@ The following constants are available at the module level: Each of the following constants defines a tar archive format that the -:mod:`tarfile` module is able to create. See section :ref:`tar-formats` for +:mod:`!tarfile` module is able to create. See section :ref:`tar-formats` for details. @@ -1068,6 +1113,12 @@ reused in custom filters: Implements the ``'data'`` filter. In addition to what ``tar_filter`` does: + - Normalize link targets (:attr:`TarInfo.linkname`) using + :func:`os.path.normpath`. + Note that this removes internal ``..`` components, which may change the + meaning of the link if the path in :attr:`!TarInfo.linkname` traverses + symbolic links. + - :ref:`Refuse ` to extract links (hard or soft) that link to absolute paths, or ones that link outside the destination. @@ -1099,6 +1150,10 @@ reused in custom filters: Note that this filter does not block *all* dangerous archive features. See :ref:`tarfile-further-verification` for details. + .. versionchanged:: 3.14 + + Link targets are now normalized. + .. _tarfile-extraction-refuse: @@ -1127,6 +1182,7 @@ Here is an incomplete list of things to consider: * Extract to a :func:`new temporary directory ` to prevent e.g. exploiting pre-existing links, and to make it easier to clean up after a failed extraction. +* Disallow symbolic links if you do not need the functionality. * When working with untrusted data, use external (e.g. OS-level) limits on disk, memory and CPU usage. * Check filenames against an allow-list of characters @@ -1229,7 +1285,7 @@ Command-Line Interface .. versionadded:: 3.4 -The :mod:`tarfile` module provides a simple command-line interface to interact +The :mod:`!tarfile` module provides a simple command-line interface to interact with tar archives. If you want to create a new tar archive, specify its name after the :option:`-c` @@ -1305,6 +1361,9 @@ Command-line options Examples -------- +Reading examples +~~~~~~~~~~~~~~~~~~~ + How to extract an entire tar archive to the current working directory:: import tarfile @@ -1327,6 +1386,23 @@ a generator function instead of a list:: tar.extractall(members=py_files(tar)) tar.close() +How to read a gzip compressed tar archive and display some member information:: + + import tarfile + tar = tarfile.open("sample.tar.gz", "r:gz") + for tarinfo in tar: + print(tarinfo.name, "is", tarinfo.size, "bytes in size and is ", end="") + if tarinfo.isreg(): + print("a regular file.") + elif tarinfo.isdir(): + print("a directory.") + else: + print("something else.") + tar.close() + +Writing examples +~~~~~~~~~~~~~~~~ + How to create an uncompressed tar archive from a list of filenames:: import tarfile @@ -1342,19 +1418,15 @@ The same example using the :keyword:`with` statement:: for name in ["foo", "bar", "quux"]: tar.add(name) -How to read a gzip compressed tar archive and display some member information:: +How to create and write an archive to stdout using +:data:`sys.stdout.buffer ` in the *fileobj* parameter +in :meth:`TarFile.add`:: - import tarfile - tar = tarfile.open("sample.tar.gz", "r:gz") - for tarinfo in tar: - print(tarinfo.name, "is", tarinfo.size, "bytes in size and is ", end="") - if tarinfo.isreg(): - print("a regular file.") - elif tarinfo.isdir(): - print("a directory.") - else: - print("something else.") - tar.close() + import sys + import tarfile + with tarfile.open("sample.tar.gz", "w|gz", fileobj=sys.stdout.buffer) as tar: + for name in ["foo", "bar", "quux"]: + tar.add(name) How to create an archive and reset the user information using the *filter* parameter in :meth:`TarFile.add`:: @@ -1374,7 +1446,7 @@ parameter in :meth:`TarFile.add`:: Supported tar formats --------------------- -There are three tar formats that can be created with the :mod:`tarfile` module: +There are three tar formats that can be created with the :mod:`!tarfile` module: * The POSIX.1-1988 ustar format (:const:`USTAR_FORMAT`). It supports filenames up to a length of at best 256 characters and linknames up to 100 characters. @@ -1383,7 +1455,7 @@ There are three tar formats that can be created with the :mod:`tarfile` module: * The GNU tar format (:const:`GNU_FORMAT`). It supports long filenames and linknames, files bigger than 8 GiB and sparse files. It is the de facto - standard on GNU/Linux systems. :mod:`tarfile` fully supports the GNU tar + standard on GNU/Linux systems. :mod:`!tarfile` fully supports the GNU tar extensions for long names, sparse file support is read-only. * The POSIX.1-2001 pax format (:const:`PAX_FORMAT`). It is the most flexible @@ -1428,7 +1500,7 @@ Unfortunately, there is no way to autodetect the encoding of an archive. The pax format was designed to solve this problem. It stores non-ASCII metadata using the universal character encoding *UTF-8*. -The details of character conversion in :mod:`tarfile` are controlled by the +The details of character conversion in :mod:`!tarfile` are controlled by the *encoding* and *errors* keyword arguments of the :class:`TarFile` class. *encoding* defines the character encoding to use for the metadata in the diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index f0a81a093b435b6..78d37d8135cc513 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -225,8 +225,9 @@ The module defines the following user-callable items: properly implements the :const:`os.O_EXCL` flag for :func:`os.open`. The file is readable and writable only by the creating user ID. If the platform uses permission bits to indicate whether a file is executable, - the file is executable by no one. The file descriptor is not inherited - by child processes. + the file is executable by no one. + + The file descriptor is :ref:`not inherited by child processes `. Unlike :func:`TemporaryFile`, the user of :func:`mkstemp` is responsible for deleting the temporary file when done with it. @@ -385,7 +386,7 @@ not surprise other unsuspecting code by changing global API behavior. Examples -------- -Here are some examples of typical usage of the :mod:`tempfile` module:: +Here are some examples of typical usage of the :mod:`!tempfile` module:: >>> import tempfile diff --git a/Doc/library/termios.rst b/Doc/library/termios.rst index 0c6f3059fe71d16..537dfcedd8cf5a3 100644 --- a/Doc/library/termios.rst +++ b/Doc/library/termios.rst @@ -2,7 +2,6 @@ =========================================== .. module:: termios - :platform: Unix :synopsis: POSIX style tty control. .. index:: @@ -38,7 +37,7 @@ The module defines the following functions: items with indices :const:`VMIN` and :const:`VTIME`, which are integers when these fields are defined). The interpretation of the flags and the speeds as well as the indexing in the *cc* array must be done using the symbolic - constants defined in the :mod:`termios` module. + constants defined in the :mod:`!termios` module. .. function:: tcsetattr(fd, when, attributes) diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 0aae14c15a61048..b5a3005f8544109 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -7,7 +7,7 @@ .. sectionauthor:: Brett Cannon .. note:: - The :mod:`test` package is meant for internal use by Python only. It is + The :mod:`!test` package is meant for internal use by Python only. It is documented for the benefit of the core developers of Python. Any use of this package outside of Python's standard library is discouraged as code mentioned here can change or be removed without notice between releases of @@ -15,12 +15,12 @@ -------------- -The :mod:`test` package contains all regression tests for Python as well as the +The :mod:`!test` package contains all regression tests for Python as well as the modules :mod:`test.support` and :mod:`test.regrtest`. :mod:`test.support` is used to enhance your tests while :mod:`test.regrtest` drives the testing suite. -Each module in the :mod:`test` package whose name starts with ``test_`` is a +Each module in the :mod:`!test` package whose name starts with ``test_`` is a testing suite for a specific module or feature. All new tests should be written using the :mod:`unittest` or :mod:`doctest` module. Some older tests are written using a "traditional" testing style that compares output printed to @@ -38,8 +38,8 @@ written using a "traditional" testing style that compares output printed to .. _writing-tests: -Writing Unit Tests for the :mod:`test` package ----------------------------------------------- +Writing Unit Tests for the :mod:`!test` package +----------------------------------------------- It is preferred that tests that use the :mod:`unittest` module follow a few guidelines. One is to name the test module by starting it with ``test_`` and end @@ -162,12 +162,12 @@ Running tests using the command-line interface .. module:: test.regrtest :synopsis: Drives the regression test suite. -The :mod:`test` package can be run as a script to drive Python's regression +The :mod:`!test` package can be run as a script to drive Python's regression test suite, thanks to the :option:`-m` option: :program:`python -m test`. Under -the hood, it uses :mod:`test.regrtest`; the call :program:`python -m +the hood, it uses :mod:`!test.regrtest`; the call :program:`python -m test.regrtest` used in previous Python versions still works. Running the script by itself automatically starts running all regression tests in the -:mod:`test` package. It does this by finding all modules in the package whose +:mod:`!test` package. It does this by finding all modules in the package whose name starts with ``test_``, importing them, and executing the function :func:`test_main` if present or loading the tests via unittest.TestLoader.loadTestsFromModule if ``test_main`` does not exist. The @@ -175,14 +175,14 @@ names of tests to execute may also be passed to the script. Specifying a single regression test (:program:`python -m test test_spam`) will minimize output and only print whether the test passed or failed. -Running :mod:`test` directly allows what resources are available for +Running :mod:`!test` directly allows what resources are available for tests to use to be set. You do this by using the ``-u`` command-line option. Specifying ``all`` as the value for the ``-u`` option enables all possible resources: :program:`python -m test -uall`. If all but one resource is desired (a more common case), a comma-separated list of resources that are not desired may be listed after ``all``. The command :program:`python -m test -uall,-audio,-largefile` -will run :mod:`test` with all resources except the ``audio`` and +will run :mod:`!test` with all resources except the ``audio`` and ``largefile`` resources. For a list of all resources and more command-line options, run :program:`python -m test -h`. @@ -197,19 +197,19 @@ regression tests. :ref:`controlled using environment variables `. -:mod:`test.support` --- Utilities for the Python test suite -=========================================================== +:mod:`!test.support` --- Utilities for the Python test suite +============================================================ .. module:: test.support :synopsis: Support for Python's regression test suite. -The :mod:`test.support` module provides support for Python's regression +The :mod:`!test.support` module provides support for Python's regression test suite. .. note:: - :mod:`test.support` is not a public module. It is documented here to help + :mod:`!test.support` is not a public module. It is documented here to help Python developers write tests. The API of this module is subject to change without backwards compatibility concerns between releases. @@ -230,7 +230,7 @@ This module defines the following exceptions: function. -The :mod:`test.support` module defines the following constants: +The :mod:`!test.support` module defines the following constants: .. data:: verbose @@ -363,7 +363,7 @@ The :mod:`test.support` module defines the following constants: .. data:: TEST_SUPPORT_DIR - Set to the top level directory that contains :mod:`test.support`. + Set to the top level directory that contains :mod:`!test.support`. .. data:: TEST_HOME_DIR @@ -438,7 +438,7 @@ The :mod:`test.support` module defines the following constants: Used to test mixed type comparison. -The :mod:`test.support` module defines the following functions: +The :mod:`!test.support` module defines the following functions: .. function:: busy_retry(timeout, err_msg=None, /, *, error=True) @@ -492,6 +492,12 @@ The :mod:`test.support` module defines the following functions: tests. +.. function:: get_resource_value(resource) + + Return the value specified for *resource* (as :samp:`-u {resource}={value}`). + Return ``None`` if *resource* is disabled or no value is specified. + + .. function:: python_is_optimized() Return ``True`` if Python was not built with ``-O0`` or ``-Og``. @@ -851,7 +857,7 @@ The :mod:`test.support` module defines the following functions: Decorator for tests that fill the address space. -.. function:: linked_with_musl() +.. function:: linked_to_musl() Return ``False`` if there is no evidence the interpreter was compiled with ``musl``, otherwise return a version triple, either ``(0, 0, 0)`` if the @@ -1037,7 +1043,7 @@ The :mod:`test.support` module defines the following functions: .. versionadded:: 3.11 -The :mod:`test.support` module defines the following classes: +The :mod:`!test.support` module defines the following classes: .. class:: SuppressCrashReport() @@ -1083,14 +1089,14 @@ The :mod:`test.support` module defines the following classes: Try to match a single stored value (*dv*) with a supplied value (*v*). -:mod:`test.support.socket_helper` --- Utilities for socket tests -================================================================ +:mod:`!test.support.socket_helper` --- Utilities for socket tests +================================================================= .. module:: test.support.socket_helper :synopsis: Support for socket tests. -The :mod:`test.support.socket_helper` module provides support for socket tests. +The :mod:`!test.support.socket_helper` module provides support for socket tests. .. versionadded:: 3.9 @@ -1161,14 +1167,14 @@ The :mod:`test.support.socket_helper` module provides support for socket tests. exceptions. -:mod:`test.support.script_helper` --- Utilities for the Python execution tests -============================================================================== +:mod:`!test.support.script_helper` --- Utilities for the Python execution tests +=============================================================================== .. module:: test.support.script_helper :synopsis: Support for Python's script execution tests. -The :mod:`test.support.script_helper` module provides support for Python's +The :mod:`!test.support.script_helper` module provides support for Python's script execution tests. .. function:: interpreter_requires_environment() @@ -1272,13 +1278,13 @@ script execution tests. path and the archive name for the zip file. -:mod:`test.support.bytecode_helper` --- Support tools for testing correct bytecode generation -============================================================================================= +:mod:`!test.support.bytecode_helper` --- Support tools for testing correct bytecode generation +============================================================================================== .. module:: test.support.bytecode_helper :synopsis: Support tools for testing correct bytecode generation. -The :mod:`test.support.bytecode_helper` module provides support for testing +The :mod:`!test.support.bytecode_helper` module provides support for testing and inspecting bytecode generation. .. versionadded:: 3.9 @@ -1304,13 +1310,13 @@ The module defines the following class: Throws :exc:`AssertionError` if *opname* is found. -:mod:`test.support.threading_helper` --- Utilities for threading tests -====================================================================== +:mod:`!test.support.threading_helper` --- Utilities for threading tests +======================================================================= .. module:: test.support.threading_helper :synopsis: Support for threading tests. -The :mod:`test.support.threading_helper` module provides support for threading tests. +The :mod:`!test.support.threading_helper` module provides support for threading tests. .. versionadded:: 3.10 @@ -1384,13 +1390,20 @@ The :mod:`test.support.threading_helper` module provides support for threading t .. versionadded:: 3.8 -:mod:`test.support.os_helper` --- Utilities for os tests -======================================================================== +.. function:: run_concurrently(worker_func, nthreads, args=(), kwargs={}) + + Run the worker function concurrently in multiple threads. + Re-raises an exception if any thread raises one, after all threads have + finished. + + +:mod:`!test.support.os_helper` --- Utilities for os tests +========================================================= .. module:: test.support.os_helper :synopsis: Support for os tests. -The :mod:`test.support.os_helper` module provides support for os tests. +The :mod:`!test.support.os_helper` module provides support for os tests. .. versionadded:: 3.10 @@ -1579,13 +1592,13 @@ The :mod:`test.support.os_helper` module provides support for os tests. wrapped with a wait loop that checks for the existence of the file. -:mod:`test.support.import_helper` --- Utilities for import tests -================================================================ +:mod:`!test.support.import_helper` --- Utilities for import tests +================================================================= .. module:: test.support.import_helper :synopsis: Support for import tests. -The :mod:`test.support.import_helper` module provides support for import tests. +The :mod:`!test.support.import_helper` module provides support for import tests. .. versionadded:: 3.10 @@ -1693,13 +1706,13 @@ The :mod:`test.support.import_helper` module provides support for import tests. will be reverted at the end of the block. -:mod:`test.support.warnings_helper` --- Utilities for warnings tests -==================================================================== +:mod:`!test.support.warnings_helper` --- Utilities for warnings tests +===================================================================== .. module:: test.support.warnings_helper :synopsis: Support for warnings tests. -The :mod:`test.support.warnings_helper` module provides support for warnings tests. +The :mod:`!test.support.warnings_helper` module provides support for warnings tests. .. versionadded:: 3.10 diff --git a/Doc/library/text.rst b/Doc/library/text.rst index 47b678434fc899c..92e7dd9a53b80d5 100644 --- a/Doc/library/text.rst +++ b/Doc/library/text.rst @@ -16,6 +16,7 @@ Python's built-in string type in :ref:`textseq`. .. toctree:: string.rst + string.templatelib.rst re.rst difflib.rst textwrap.rst diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst index a58b460fef409c6..c9230f7d82705fc 100644 --- a/Doc/library/textwrap.rst +++ b/Doc/library/textwrap.rst @@ -11,7 +11,7 @@ -------------- -The :mod:`textwrap` module provides some convenience functions, +The :mod:`!textwrap` module provides some convenience functions, as well as :class:`TextWrapper`, the class that does all the work. If you're just wrapping or filling one or two text strings, the convenience functions should be good enough; otherwise, you should use an instance of @@ -102,6 +102,10 @@ functions should be good enough; otherwise, you should use an instance of print(repr(s)) # prints ' hello\n world\n ' print(repr(dedent(s))) # prints 'hello\n world\n' + .. versionchanged:: 3.14 + The :func:`!dedent` function now correctly normalizes blank lines containing + only whitespace characters. Previously, the implementation only normalized + blank lines containing tabs and spaces. .. function:: indent(text, prefix, predicate=None) diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 249c0a5cb035c34..69539fa1fd4bb57 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -11,6 +11,52 @@ This module constructs higher-level threading interfaces on top of the lower level :mod:`_thread` module. +.. include:: ../includes/wasm-notavail.rst + +Introduction +------------ + +The :mod:`!threading` module provides a way to run multiple `threads +`_ (smaller +units of a process) concurrently within a single process. It allows for the +creation and management of threads, making it possible to execute tasks in +parallel, sharing memory space. Threads are particularly useful when tasks are +I/O bound, such as file operations or making network requests, +where much of the time is spent waiting for external resources. + +A typical use case for :mod:`!threading` includes managing a pool of worker +threads that can process multiple tasks concurrently. Here's a basic example of +creating and starting threads using :class:`~threading.Thread`:: + + import threading + import time + + def crawl(link, delay=3): + print(f"crawl started for {link}") + time.sleep(delay) # Blocking I/O (simulating a network request) + print(f"crawl ended for {link}") + + links = [ + "https://python.org", + "https://docs.python.org", + "https://peps.python.org", + ] + + # Start threads for each link + threads = [] + for link in links: + # Using `args` to pass positional arguments and `kwargs` for keyword arguments + t = threading.Thread(target=crawl, args=(link,), kwargs={"delay": 2}) + threads.append(t) + + # Start each thread + for t in threads: + t.start() + + # Wait for all threads to finish + for t in threads: + t.join() + .. versionchanged:: 3.7 This module used to be optional, it is now always available. @@ -45,7 +91,25 @@ level :mod:`_thread` module. However, threading is still an appropriate model if you want to run multiple I/O-bound tasks simultaneously. -.. include:: ../includes/wasm-notavail.rst +GIL and performance considerations +---------------------------------- + +Unlike the :mod:`multiprocessing` module, which uses separate processes to +bypass the :term:`global interpreter lock` (GIL), the threading module operates +within a single process, meaning that all threads share the same memory space. +However, the GIL limits the performance gains of threading when it comes to +CPU-bound tasks, as only one thread can execute Python bytecode at a time. +Despite this, threads remain a useful tool for achieving concurrency in many +scenarios. + +As of Python 3.13, :term:`free-threaded ` builds +can disable the GIL, enabling true parallel execution of threads, but this +feature is not available by default (see :pep:`703`). + +.. TODO: At some point this feature will become available by default. + +Reference +--------- This module defines the following functions: @@ -62,7 +126,7 @@ This module defines the following functions: Return the current :class:`Thread` object, corresponding to the caller's thread of control. If the caller's thread of control was not created through the - :mod:`threading` module, a dummy thread object with limited functionality is + :mod:`!threading` module, a dummy thread object with limited functionality is returned. The function ``currentThread`` is a deprecated alias for this function. @@ -157,13 +221,13 @@ This module defines the following functions: .. index:: single: trace function - Set a trace function for all threads started from the :mod:`threading` module. + Set a trace function for all threads started from the :mod:`!threading` module. The *func* will be passed to :func:`sys.settrace` for each thread, before its :meth:`~Thread.run` method is called. .. function:: settrace_all_threads(func) - Set a trace function for all threads started from the :mod:`threading` module + Set a trace function for all threads started from the :mod:`!threading` module and all Python threads that are currently executing. The *func* will be passed to :func:`sys.settrace` for each thread, before its @@ -186,13 +250,13 @@ This module defines the following functions: .. index:: single: profile function - Set a profile function for all threads started from the :mod:`threading` module. + Set a profile function for all threads started from the :mod:`!threading` module. The *func* will be passed to :func:`sys.setprofile` for each thread, before its :meth:`~Thread.run` method is called. .. function:: setprofile_all_threads(func) - Set a profile function for all threads started from the :mod:`threading` module + Set a profile function for all threads started from the :mod:`!threading` module and all Python threads that are currently executing. The *func* will be passed to :func:`sys.setprofile` for each thread, before its @@ -257,8 +321,8 @@ when implemented, are mapped to module-level functions. All of the methods described below are executed atomically. -Thread-Local Data ------------------ +Thread-local data +^^^^^^^^^^^^^^^^^ Thread-local data is data whose values are thread specific. If you have data that you want to be local to a thread, create a @@ -389,8 +453,8 @@ affects what we see:: .. _thread-objects: -Thread Objects --------------- +Thread objects +^^^^^^^^^^^^^^ The :class:`Thread` class represents an activity that is run in a separate thread of control. There are two ways to specify the activity: by passing a @@ -448,7 +512,7 @@ since it is impossible to detect the termination of alien threads. This constructor should always be called with keyword arguments. Arguments are: - *group* should be ``None``; reserved for future extension when a + *group* must be ``None`` as it is reserved for future extension when a :class:`!ThreadGroup` class is implemented. *target* is the callable object to be invoked by the :meth:`run` method. @@ -557,7 +621,7 @@ since it is impossible to detect the termination of alien threads. an error to :meth:`~Thread.join` a thread before it has been started and attempts to do so raise the same exception. - If an attempt is made to join a running daemonic thread in in late stages + If an attempt is made to join a running daemonic thread in late stages of :term:`Python finalization ` :meth:`!join` raises a :exc:`PythonFinalizationError`. @@ -645,8 +709,8 @@ since it is impossible to detect the termination of alien threads. .. _lock-objects: -Lock Objects ------------- +Lock objects +^^^^^^^^^^^^ A primitive lock is a synchronization primitive that is not owned by a particular thread when locked. In Python, it is currently the lowest level @@ -738,8 +802,8 @@ All methods are executed atomically. .. _rlock-objects: -RLock Objects -------------- +RLock objects +^^^^^^^^^^^^^ A reentrant lock is a synchronization primitive that may be acquired multiple times by the same thread. Internally, it uses the concepts of "owning thread" @@ -848,8 +912,8 @@ call release as many times the lock has been acquired can lead to deadlock. .. _condition-objects: -Condition Objects ------------------ +Condition objects +^^^^^^^^^^^^^^^^^ A condition variable is always associated with some kind of lock; this can be passed in or one will be created by default. Passing one in is useful when @@ -1026,8 +1090,8 @@ item to the buffer only needs to wake up one consumer thread. .. _semaphore-objects: -Semaphore Objects ------------------ +Semaphore objects +^^^^^^^^^^^^^^^^^ This is one of the oldest synchronization primitives in the history of computer science, invented by the early Dutch computer scientist Edsger W. Dijkstra (he @@ -1107,7 +1171,7 @@ Semaphores also support the :ref:`context management protocol `. .. _semaphore-examples: -:class:`Semaphore` Example +:class:`Semaphore` example ^^^^^^^^^^^^^^^^^^^^^^^^^^ Semaphores are often used to guard resources with limited capacity, for example, @@ -1135,8 +1199,8 @@ causes the semaphore to be released more than it's acquired will go undetected. .. _event-objects: -Event Objects -------------- +Event objects +^^^^^^^^^^^^^ This is one of the simplest mechanisms for communication between threads: one thread signals an event and other threads wait for it. @@ -1192,8 +1256,8 @@ method. The :meth:`~Event.wait` method blocks until the flag is true. .. _timer-objects: -Timer Objects -------------- +Timer objects +^^^^^^^^^^^^^ This class represents an action that should be run only after a certain amount of time has passed --- a timer. :class:`Timer` is a subclass of :class:`Thread` @@ -1230,8 +1294,8 @@ For example:: only work if the timer is still in its waiting stage. -Barrier Objects ---------------- +Barrier objects +^^^^^^^^^^^^^^^ .. versionadded:: 3.2 diff --git a/Doc/library/threadsafety.rst b/Doc/library/threadsafety.rst new file mode 100644 index 000000000000000..a529f7803affbc2 --- /dev/null +++ b/Doc/library/threadsafety.rst @@ -0,0 +1,606 @@ +.. _threadsafety: + +************************ +Thread Safety Guarantees +************************ + +This page documents thread-safety guarantees for built-in types in Python's +free-threaded build. The guarantees described here apply when using Python with +the :term:`GIL` disabled (free-threaded mode). When the GIL is enabled, most +operations are implicitly serialized. + +For general guidance on writing thread-safe code in free-threaded Python, see +:ref:`freethreading-python-howto`. + + +.. _threadsafety-levels: + +Thread safety levels +==================== + +The C API documentation uses the following levels to describe the thread +safety guarantees of each function. The levels are listed from least to +most safe. + +.. _threadsafety-level-incompatible: + +Incompatible +------------ + +A function or operation that cannot be made safe for concurrent use even +with external synchronization. Incompatible code typically accesses +global state in an unsynchronized way and must only be called from a single +thread throughout the program's lifetime. + +Example: a function that modifies process-wide state such as signal handlers +or environment variables, where concurrent calls from any threads, even with +external locking, can conflict with the runtime or other libraries. + +.. _threadsafety-level-compatible: + +Compatible +---------- + +A function or operation that is safe to call from multiple threads +*provided* the caller supplies appropriate external synchronization, for +example by holding a :term:`lock` for the duration of each call. Without +such synchronization, concurrent calls may produce :term:`race conditions +` or :term:`data races `. + +Example: a function that reads from or writes to an object whose internal +state is not protected by a lock. Callers must ensure that no two threads +access the same object at the same time. + +.. _threadsafety-level-distinct: + +Safe on distinct objects +------------------------ + +A function or operation that is safe to call from multiple threads without +external synchronization, as long as each thread operates on a **different** +object. Two threads may call the function at the same time, but they must +not pass the same object (or objects that share underlying state) as +arguments. + +Example: a function that modifies fields of a struct using non-atomic +writes. Two threads can each call the function on their own struct +instance safely, but concurrent calls on the *same* instance require +external synchronization. + +.. _threadsafety-level-shared: + +Safe on shared objects +---------------------- + +A function or operation that is safe for concurrent use on the **same** +object. The implementation uses internal synchronization (such as +:term:`per-object locks ` or +:ref:`critical sections `) to protect shared +mutable state, so callers do not need to supply their own locking. + +Example: :c:func:`PyList_GetItemRef` can be called from multiple threads on the +same :c:type:`PyListObject` - it uses internal synchronization to serialize +access. + +.. _threadsafety-level-atomic: + +Atomic +------ + +A function or operation that appears :term:`atomic ` with +respect to other threads - it executes instantaneously from the perspective +of other threads. This is the strongest form of thread safety. + +Example: :c:func:`PyMutex_IsLocked` performs an atomic read of the mutex +state and can be called from any thread at any time. + + +.. _thread-safety-list: + +Thread safety for list objects +============================== + +Reading a single element from a :class:`list` is +:term:`atomic `: + +.. code-block:: + :class: good + + lst[i] # list.__getitem__ + +The following methods traverse the list and use :term:`atomic ` +reads of each item to perform their function. That means that they may +return results affected by concurrent modifications: + +.. code-block:: + :class: maybe + + item in lst + lst.index(item) + lst.count(item) + +All of the above operations avoid acquiring :term:`per-object locks +`. They do not block concurrent modifications. Other +operations that hold a lock will not block these from observing intermediate +states. + +All other operations from here on block using the :term:`per-object lock`. + +Writing a single item via ``lst[i] = x`` is safe to call from multiple +threads and will not corrupt the list. + +The following operations return new objects and appear +:term:`atomic ` to other threads: + +.. code-block:: + :class: good + + lst1 + lst2 # concatenates two lists into a new list + x * lst # repeats lst x times into a new list + lst.copy() # returns a shallow copy of the list + +The following methods that only operate on a single element with no shifting +required are :term:`atomic `: + +.. code-block:: + :class: good + + lst.append(x) # append to the end of the list, no shifting required + lst.pop() # pop element from the end of the list, no shifting required + +The :meth:`~list.clear` method is also :term:`atomic `. +Other threads cannot observe elements being removed. + +The :meth:`~list.sort` method is not :term:`atomic `. +Other threads cannot observe intermediate states during sorting, but the +list appears empty for the duration of the sort. + +The following operations may allow :term:`lock-free` operations to observe +intermediate states since they modify multiple elements in place: + +.. code-block:: + :class: maybe + + lst.insert(idx, item) # shifts elements + lst.pop(idx) # idx not at the end of the list, shifts elements + lst *= x # copies elements in place + +The :meth:`~list.remove` method may allow concurrent modifications since +element comparison may execute arbitrary Python code (via +:meth:`~object.__eq__`). + +:meth:`~list.extend` is safe to call from multiple threads. However, its +guarantees depend on the iterable passed to it. If it is a :class:`list`, a +:class:`tuple`, a :class:`set`, a :class:`frozenset`, a :class:`dict` or a +:ref:`dictionary view object ` (but not their subclasses), the +``extend`` operation is safe from concurrent modifications to the iterable. +Otherwise, an iterator is created which can be concurrently modified by +another thread. The same applies to inplace concatenation of a list with +other iterables when using ``lst += iterable``. + +Similarly, assigning to a list slice with ``lst[i:j] = iterable`` is safe +to call from multiple threads, but ``iterable`` is only locked when it is +also a :class:`list` (but not its subclasses). + +Operations that involve multiple accesses, as well as iteration, are never +atomic. For example: + +.. code-block:: + :class: bad + + # NOT atomic: read-modify-write + lst[i] = lst[i] + 1 + + # NOT atomic: check-then-act + if lst: + item = lst.pop() + + # NOT thread-safe: iteration while modifying + for item in lst: + process(item) # another thread may modify lst + +Consider external synchronization when sharing :class:`list` instances +across threads. + + +.. _thread-safety-dict: + +Thread safety for dict objects +============================== + +Creating a dictionary with the :class:`dict` constructor is atomic when the +argument to it is a :class:`dict` or a :class:`tuple`. When using the +:meth:`dict.fromkeys` method, dictionary creation is atomic when the +argument is a :class:`dict`, :class:`tuple`, :class:`set` or +:class:`frozenset`. + +The following operations and functions are :term:`lock-free` and +:term:`atomic `. + +.. code-block:: + :class: good + + d[key] # dict.__getitem__ + d.get(key) # dict.get + key in d # dict.__contains__ + len(d) # dict.__len__ + +All other operations from here on hold the :term:`per-object lock`. + +Writing or removing a single item is safe to call from multiple threads +and will not corrupt the dictionary: + +.. code-block:: + :class: good + + d[key] = value # write + del d[key] # delete + d.pop(key) # remove and return + d.popitem() # remove and return last item + d.setdefault(key, v) # insert if missing + +These operations may compare keys using :meth:`~object.__eq__`, which can +execute arbitrary Python code. During such comparisons, the dictionary may +be modified by another thread. For built-in types like :class:`str`, +:class:`int`, and :class:`float`, that implement :meth:`~object.__eq__` in C, +the underlying lock is not released during comparisons and this is not a +concern. + +The following operations return new objects and hold the :term:`per-object lock` +for the duration of the operation: + +.. code-block:: + :class: good + + d.copy() # returns a shallow copy of the dictionary + d | other # merges two dicts into a new dict + d.keys() # returns a new dict_keys view object + d.values() # returns a new dict_values view object + d.items() # returns a new dict_items view object + +The :meth:`~dict.clear` method holds the lock for its duration. Other +threads cannot observe elements being removed. + +The following operations lock both dictionaries. For :meth:`~dict.update` +and ``|=``, this applies only when the other operand is a :class:`dict` +that uses the standard dict iterator (but not subclasses that override +iteration). For equality comparison, this applies to :class:`dict` and +its subclasses: + +.. code-block:: + :class: good + + d.update(other_dict) # both locked when other_dict is a dict + d |= other_dict # both locked when other_dict is a dict + d == other_dict # both locked for dict and subclasses + +All comparison operations also compare values using :meth:`~object.__eq__`, +so for non-built-in types the lock may be released during comparison. + +:meth:`~dict.fromkeys` locks both the new dictionary and the iterable +when the iterable is exactly a :class:`dict`, :class:`set`, or +:class:`frozenset` (not subclasses): + +.. code-block:: + :class: good + + dict.fromkeys(a_dict) # locks both + dict.fromkeys(a_set) # locks both + dict.fromkeys(a_frozenset) # locks both + +When updating from a non-dict iterable, only the target dictionary is +locked. The iterable may be concurrently modified by another thread: + +.. code-block:: + :class: maybe + + d.update(iterable) # iterable is not a dict: only d locked + d |= iterable # iterable is not a dict: only d locked + dict.fromkeys(iterable) # iterable is not a dict/set/frozenset: only result locked + +Operations that involve multiple accesses, as well as iteration, are never +atomic: + +.. code-block:: + :class: bad + + # NOT atomic: read-modify-write + d[key] = d[key] + 1 + + # NOT atomic: check-then-act (TOCTOU) + if key in d: + del d[key] + + # NOT thread-safe: iteration while modifying + for key, value in d.items(): + process(key) # another thread may modify d + +To avoid time-of-check to time-of-use (TOCTOU) issues, use atomic +operations or handle exceptions: + +.. code-block:: + :class: good + + # Use pop() with default instead of check-then-delete + d.pop(key, None) + + # Or handle the exception + try: + del d[key] + except KeyError: + pass + +To safely iterate over a dictionary that may be modified by another +thread, iterate over a copy: + +.. code-block:: + :class: good + + # Make a copy to iterate safely + for key, value in d.copy().items(): + process(key) + +Consider external synchronization when sharing :class:`dict` instances +across threads. + + +.. _thread-safety-set: + +Thread safety for set objects +============================== + +The :func:`len` function is lock-free and :term:`atomic `. + +The following read operation is lock-free. It does not block concurrent +modifications and may observe intermediate states from operations that +hold the per-object lock: + +.. code-block:: + :class: good + + elem in s # set.__contains__ + +This operation may compare elements using :meth:`~object.__eq__`, which can +execute arbitrary Python code. During such comparisons, the set may be +modified by another thread. For built-in types like :class:`str`, +:class:`int`, and :class:`float`, :meth:`!__eq__` does not release the +underlying lock during comparisons and this is not a concern. + +All other operations from here on hold the per-object lock. + +Adding or removing a single element is safe to call from multiple threads +and will not corrupt the set: + +.. code-block:: + :class: good + + s.add(elem) # add element + s.remove(elem) # remove element, raise if missing + s.discard(elem) # remove element if present + s.pop() # remove and return arbitrary element + +These operations also compare elements, so the same :meth:`~object.__eq__` +considerations as above apply. + +The :meth:`~set.copy` method returns a new object and holds the per-object lock +for the duration so that it is always atomic. + +The :meth:`~set.clear` method holds the lock for its duration. Other +threads cannot observe elements being removed. + +The following operations only accept :class:`set` or :class:`frozenset` +as operands and always lock both objects: + +.. code-block:: + :class: good + + s |= other # other must be set/frozenset + s &= other # other must be set/frozenset + s -= other # other must be set/frozenset + s ^= other # other must be set/frozenset + s & other # other must be set/frozenset + s | other # other must be set/frozenset + s - other # other must be set/frozenset + s ^ other # other must be set/frozenset + +:meth:`set.update`, :meth:`set.union`, :meth:`set.intersection` and +:meth:`set.difference` can take multiple iterables as arguments. They all +iterate through all the passed iterables and do the following: + + * :meth:`set.update` and :meth:`set.union` lock both objects only when + the other operand is a :class:`set`, :class:`frozenset`, or :class:`dict`. + * :meth:`set.intersection` and :meth:`set.difference` always try to lock + all objects. + +:meth:`set.symmetric_difference` tries to lock both objects. + +The update variants of the above methods also have some differences between +them: + + * :meth:`set.difference_update` and :meth:`set.intersection_update` try + to lock all objects one-by-one. + * :meth:`set.symmetric_difference_update` only locks the arguments if it is + of type :class:`set`, :class:`frozenset`, or :class:`dict`. + +The following methods always try to lock both objects: + +.. code-block:: + :class: good + + s.isdisjoint(other) # both locked + s.issubset(other) # both locked + s.issuperset(other) # both locked + +Operations that involve multiple accesses, as well as iteration, are never +atomic: + +.. code-block:: + :class: bad + + # NOT atomic: check-then-act + if elem in s: + s.remove(elem) + + # NOT thread-safe: iteration while modifying + for elem in s: + process(elem) # another thread may modify s + +Consider external synchronization when sharing :class:`set` instances +across threads. See :ref:`freethreading-python-howto` for more information. + + +.. _thread-safety-bytearray: + +Thread safety for bytearray objects +=================================== + + The :func:`len` function is lock-free and :term:`atomic `. + + Concatenation and comparisons use the buffer protocol, which prevents + resizing but does not hold the per-object lock. These operations may + observe intermediate states from concurrent modifications: + + .. code-block:: + :class: maybe + + ba + other # may observe concurrent writes + ba == other # may observe concurrent writes + ba < other # may observe concurrent writes + + All other operations from here on hold the per-object lock. + + Reading a single element or slice is safe to call from multiple threads: + + .. code-block:: + :class: good + + ba[i] # bytearray.__getitem__ + ba[i:j] # slice + + The following operations are safe to call from multiple threads and will + not corrupt the bytearray: + + .. code-block:: + :class: good + + ba[i] = x # write single byte + ba[i:j] = values # write slice + ba.append(x) # append single byte + ba.extend(other) # extend with iterable + ba.insert(i, x) # insert single byte + ba.pop() # remove and return last byte + ba.pop(i) # remove and return byte at index + ba.remove(x) # remove first occurrence + ba.reverse() # reverse in place + ba.clear() # remove all bytes + + Slice assignment locks both objects when *values* is a :class:`bytearray`: + + .. code-block:: + :class: good + + ba[i:j] = other_bytearray # both locked + + The following operations return new objects and hold the per-object lock + for the duration: + + .. code-block:: + :class: good + + ba.copy() # returns a shallow copy + ba * n # repeat into new bytearray + + The membership test holds the lock for its duration: + + .. code-block:: + :class: good + + x in ba # bytearray.__contains__ + + All other bytearray methods (such as :meth:`~bytearray.find`, + :meth:`~bytearray.replace`, :meth:`~bytearray.split`, + :meth:`~bytearray.decode`, etc.) hold the per-object lock for their + duration. + + Operations that involve multiple accesses, as well as iteration, are never + atomic: + + .. code-block:: + :class: bad + + # NOT atomic: check-then-act + if x in ba: + ba.remove(x) + + # NOT thread-safe: iteration while modifying + for byte in ba: + process(byte) # another thread may modify ba + + To safely iterate over a bytearray that may be modified by another + thread, iterate over a copy: + + .. code-block:: + :class: good + + # Make a copy to iterate safely + for byte in ba.copy(): + process(byte) + + Consider external synchronization when sharing :class:`bytearray` instances + across threads. See :ref:`freethreading-python-howto` for more information. + + +.. _thread-safety-memoryview: + +Thread safety for memoryview objects +==================================== + +:class:`memoryview` objects provide access to the internal data of an +underlying object without copying. Thread safety depends on both the +memoryview itself and the underlying buffer exporter. + +The memoryview implementation uses atomic operations to track its own +exports in the :term:`free-threaded build`. Creating and +releasing a memoryview are thread-safe. Attribute access (e.g., +:attr:`~memoryview.shape`, :attr:`~memoryview.format`) reads fields that +are immutable for the lifetime of the memoryview, so concurrent reads +are safe as long as the memoryview has not been released. + +However, the actual data accessed through the memoryview is owned by the +underlying object. Concurrent access to this data is only safe if the +underlying object supports it: + +* For immutable objects like :class:`bytes`, concurrent reads through + multiple memoryviews are safe. + +* For mutable objects like :class:`bytearray`, reading and writing the + same memory region from multiple threads without external + synchronization is not safe and may result in data corruption. + Note that even read-only memoryviews of mutable objects do not + prevent data races if the underlying object is modified from + another thread. + +.. code-block:: + :class: bad + + # NOT safe: concurrent writes to the same buffer + data = bytearray(1000) + view = memoryview(data) + # Thread 1: view[0:500] = b'x' * 500 + # Thread 2: view[0:500] = b'y' * 500 + +.. code-block:: + :class: good + + # Safe: use a lock for concurrent access + import threading + lock = threading.Lock() + data = bytearray(1000) + view = memoryview(data) + + with lock: + view[0:500] = b'x' * 500 + +Resizing or reallocating the underlying object (such as calling +:meth:`bytearray.resize`) while a memoryview is exported raises +:exc:`BufferError`. This is enforced regardless of threading. diff --git a/Doc/library/time.rst b/Doc/library/time.rst index 542493a82af94d4..a8b721b59df3488 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -238,8 +238,8 @@ Functions The result has the following attributes: - - *adjustable*: ``True`` if the clock can be changed automatically (e.g. by - a NTP daemon) or manually by the system administrator, ``False`` otherwise + - *adjustable*: ``True`` if the clock can be set to jump forward or backward + in time, ``False`` otherwise. Does not refer to gradual NTP rate adjustments. - *implementation*: The name of the underlying C function used to get the clock value. Refer to :ref:`time-clock-id-constants` for possible values. - *monotonic*: ``True`` if the clock cannot go backward, @@ -306,10 +306,11 @@ Functions .. versionadded:: 3.3 .. versionchanged:: 3.5 - The function is now always available and always system-wide. + The function is now always available and the clock is now the same for + all processes. .. versionchanged:: 3.10 - On macOS, the function is now system-wide. + On macOS, the clock is now the same for all processes. .. function:: monotonic_ns() -> int @@ -325,7 +326,8 @@ Functions Return the value (in fractional seconds) of a performance counter, i.e. a clock with the highest available resolution to measure a short duration. It - does include time elapsed during sleep and is system-wide. The reference + does include time elapsed during sleep. The clock is the same for all + processes. The reference point of the returned value is undefined, so that only the difference between the results of two calls is valid. @@ -340,7 +342,7 @@ Functions .. versionadded:: 3.3 .. versionchanged:: 3.10 - On Windows, the function is now system-wide. + On Windows, the clock is now the same for all processes. .. versionchanged:: 3.13 Use the same clock as :func:`time.monotonic`. @@ -394,9 +396,9 @@ Functions On Windows, if *secs* is zero, the thread relinquishes the remainder of its time slice to any other thread that is ready to run. If there are no other threads ready to run, the function returns immediately, and the thread - continues execution. On Windows 8.1 and newer the implementation uses + continues execution. On Windows 10 and newer the implementation uses a `high-resolution timer - `_ + `_ which provides resolution of 100 nanoseconds. If *secs* is zero, ``Sleep(0)`` is used. .. rubric:: Unix implementation @@ -568,7 +570,7 @@ Functions calculations when the day of the week and the year are specified. Here is an example, a format for dates compatible with that specified in the - :rfc:`2822` Internet email standard. [1]_ :: + :rfc:`5322` Internet email standard. [1]_ :: >>> from time import gmtime, strftime >>> strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()) @@ -712,13 +714,18 @@ Functions Clock: - * On Windows, call ``GetSystemTimeAsFileTime()``. + * On Windows, call ``GetSystemTimePreciseAsFileTime()``. * Call ``clock_gettime(CLOCK_REALTIME)`` if available. * Otherwise, call ``gettimeofday()``. Use :func:`time_ns` to avoid the precision loss caused by the :class:`float` type. +.. versionchanged:: 3.13 + + On Windows, calls ``GetSystemTimePreciseAsFileTime()`` instead of + ``GetSystemTimeAsFileTime()``. + .. function:: time_ns() -> int @@ -928,7 +935,7 @@ These constants are used as parameters for :func:`clock_getres` and .. data:: CLOCK_TAI - `International Atomic Time `_ + `International Atomic Time `_ The system must have a current leap second table in order for this to give the correct answer. PTP or NTP software can maintain a leap second table. @@ -982,8 +989,8 @@ The following constant is the only parameter that can be sent to .. data:: CLOCK_REALTIME - System-wide real-time clock. Setting this clock requires appropriate - privileges. + Real-time clock. Setting this clock requires appropriate privileges. + The clock is the same for all processes. .. availability:: Unix. @@ -1045,4 +1052,5 @@ Timezone Constants strict reading of the original 1982 :rfc:`822` standard calls for a two-digit year (``%y`` rather than ``%Y``), but practice moved to 4-digit years long before the year 2000. After that, :rfc:`822` became obsolete and the 4-digit year has - been first recommended by :rfc:`1123` and then mandated by :rfc:`2822`. + been first recommended by :rfc:`1123` and then mandated by :rfc:`2822`, + with :rfc:`5322` continuing this requirement. diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 548a3ee05405069..bc12061a2aeb2d5 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -355,7 +355,7 @@ to test for missing and present object attributes: 0.08588060699912603 -To give the :mod:`timeit` module access to functions you define, you can pass a +To give the :mod:`!timeit` module access to functions you define, you can pass a *setup* parameter which contains an import statement:: def test(): diff --git a/Doc/library/tk.rst b/Doc/library/tk.rst index 0593f8b73ea5456..fa3c7e910ce21f0 100644 --- a/Doc/library/tk.rst +++ b/Doc/library/tk.rst @@ -1,7 +1,7 @@ .. _tkinter: ********************************* -Graphical User Interfaces with Tk +Graphical user interfaces with Tk ********************************* .. index:: @@ -39,6 +39,7 @@ alternative `GUI frameworks and tools .dnd_leave(source, event)* -#. Call to *.dnd_enter(source, event)* -#. Call to *.dnd_commit(source, event)* to notify of drop -#. Call to *.dnd_end(target, event)* to signal end of drag-and-drop +#. Call to ``.dnd_leave(source, event)``. +#. Call to ``.dnd_enter(source, event)``. +#. Call to ``.dnd_commit(source, event)`` to notify of the drop. +#. Call to ``.dnd_end(target, event)`` to signal the end of drag-and-drop. .. class:: DndHandler(source, event) @@ -49,7 +49,8 @@ Selection of a target object occurs as follows: .. method:: on_motion(event) - Inspect area below mouse for target objects while drag is performed. + Inspect area below mouse for target objects while a drag + is performed. .. method:: on_release(event) @@ -57,7 +58,9 @@ Selection of a target object occurs as follows: .. function:: dnd_start(source, event) - Factory function for drag-and-drop process. + Factory function for the drag-and-drop process. + Return the :class:`DndHandler` instance managing the drag, or ``None`` if a + drag could not be started. .. seealso:: diff --git a/Doc/library/tkinter.font.rst b/Doc/library/tkinter.font.rst index ed01bd5f4839438..947207724ab0409 100644 --- a/Doc/library/tkinter.font.rst +++ b/Doc/library/tkinter.font.rst @@ -2,14 +2,13 @@ ============================================= .. module:: tkinter.font - :platform: Tk :synopsis: Tkinter font-wrapping class **Source code:** :source:`Lib/tkinter/font.py` -------------- -The :mod:`tkinter.font` module provides the :class:`Font` class for creating +The :mod:`!tkinter.font` module provides the :class:`Font` class for creating and using named fonts. The different font weights and slants are: @@ -27,6 +26,10 @@ The different font weights and slants are: fonts as a single object, rather than specifying a font by its attributes with each occurrence. + .. versionchanged:: 3.10 + Two fonts now compare equal (``==``) only when both are :class:`Font` + instances with the same name belonging to the same Tcl interpreter. + arguments: | *font* - font specifier tuple (family, size, options) @@ -35,7 +38,7 @@ The different font weights and slants are: additional keyword options (ignored if *font* is specified): - | *family* - font family i.e. Courier, Times + | *family* - font family, for example, Courier, Times | *size* - font size | If *size* is positive it is interpreted as size in points. | If *size* is a negative number its absolute value is treated @@ -47,29 +50,47 @@ The different font weights and slants are: .. method:: actual(option=None, displayof=None) - Return the attributes of the font. + Return the actual attributes of the font, which may differ from the + requested ones because of platform limitations. + With no *option*, return a dictionary of all the attributes; if *option* + is given, return the value of that single attribute. + The attributes are resolved on the display of the *displayof* widget, + or the main application window if it is not specified. .. method:: cget(option) Retrieve an attribute of the font. .. method:: config(**options) + :no-typesetting: + + .. method:: configure(**options) + + Modify one or more attributes of the font. + With no arguments, return a dictionary of the current attributes. - Modify attributes of the font. + :meth:`config` is an alias of :meth:`!configure`. .. method:: copy() - Return new instance of the current font. + Return a distinct copy of the current font: + a new named font with the same attributes but a different name, + which can be reconfigured independently of the original. + If the current font wraps a font description, + the copy is instead a named font with its resolved attributes. .. method:: measure(text, displayof=None) Return amount of space the text would occupy on the specified display - when formatted in the current font. If no display is specified then the - main application window is assumed. + when formatted in the current font, as an integer number of pixels. + If no display is specified then the main application window is assumed. .. method:: metrics(*options, **kw) Return font-specific data. + With no options, return a dictionary mapping each metric name to its + integer value; if one option name is given, return that metric's value as + an integer. Options include: *ascent* - distance between baseline and highest point that a @@ -85,15 +106,17 @@ The different font weights and slants are: .. function:: families(root=None, displayof=None) - Return the different font families. + Return a tuple of the names of the available font families. .. function:: names(root=None) - Return the names of defined fonts. + Return a tuple of the names of all the defined fonts. .. function:: nametofont(name, root=None) - Return a :class:`Font` representation of a tk named font. + Return a :class:`Font` representation of the existing named font *name*. + *root* is the widget whose Tcl interpreter owns the font; if omitted, the + default root window is used. .. versionchanged:: 3.10 The *root* parameter was added. diff --git a/Doc/library/tkinter.messagebox.rst b/Doc/library/tkinter.messagebox.rst index 0dc9632ca73304c..47fe680bac9107e 100644 --- a/Doc/library/tkinter.messagebox.rst +++ b/Doc/library/tkinter.messagebox.rst @@ -2,19 +2,19 @@ ====================================================== .. module:: tkinter.messagebox - :platform: Tk :synopsis: Various types of alert dialogs **Source code:** :source:`Lib/tkinter/messagebox.py` -------------- -The :mod:`tkinter.messagebox` module provides a template base class as well as -a variety of convenience methods for commonly used configurations. The message -boxes are modal and will return a subset of (``True``, ``False``, ``None``, -:data:`OK`, :data:`CANCEL`, :data:`YES`, :data:`NO`) based on -the user's selection. Common message box styles and layouts include but are not -limited to: +The :mod:`!tkinter.messagebox` module provides a template base class as well as +a variety of convenience methods for commonly used configurations. +The message boxes are modal: each blocks until the user responds, then returns +a value that depends on the function. +The ``show*`` functions and :meth:`Message.show` return the symbolic name of +the button the user pressed, as a string (such as :data:`OK` or :data:`YES`). +Common message box styles and layouts include but are not limited to: .. figure:: tk_msg.png @@ -67,6 +67,10 @@ limited to: Arranges for a :ref:`predefined set of buttons ` to be displayed. + .. note:: + + Tk 8.6 added the *command* option. + .. method:: show(**options) Display a message window and wait for the user to select one of the buttons. Then return the symbolic name of the selected button. @@ -107,7 +111,7 @@ limited to: .. function:: askretrycancel(title=None, message=None, **options) Ask if operation should be retried. Shows buttons :data:`RETRY` and :data:`CANCEL`. - Return ``True`` if the answer is yes and ``False`` otherwise. + Return ``True`` if the answer is retry and ``False`` otherwise. .. function:: askyesno(title=None, message=None, **options) diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index f284988daf2d4e3..542f0484ac9aed2 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -10,19 +10,24 @@ -------------- -The :mod:`tkinter` package ("Tk interface") is the standard Python interface to -the Tcl/Tk GUI toolkit. Both Tk and :mod:`tkinter` are available on most Unix +The :mod:`!tkinter` package ("Tk interface") is the standard Python interface to +the Tcl/Tk GUI toolkit. Both Tk and :mod:`!tkinter` are available on most Unix platforms, including macOS, as well as on Windows systems. Running ``python -m tkinter`` from the command line should open a window -demonstrating a simple Tk interface, letting you know that :mod:`tkinter` is +demonstrating a simple Tk interface, letting you know that :mod:`!tkinter` is properly installed on your system, and also showing what version of Tcl/Tk is installed, so you can read the Tcl/Tk documentation specific to that version. -Tkinter supports a range of Tcl/Tk versions, built either with or -without thread support. The official Python binary release bundles Tcl/Tk 8.6 -threaded. See the source code for the :mod:`_tkinter` module -for more information about supported versions. +Tkinter supports a range of Tcl/Tk versions, built either with or without +thread support. +Tcl/Tk 8.5.12 is the minimum supported version; the official Python binary +release bundles Tcl/Tk 8.6. +See the source code for the :mod:`_tkinter` module for more information about +supported versions. + +.. versionchanged:: 3.11 + Support for Tcl/Tk versions older than 8.5.12 was removed. Tkinter is not a thin wrapper, but adds a fair amount of its own logic to make the experience more pythonic. This documentation will concentrate on these @@ -32,10 +37,13 @@ details that are unchanged. .. note:: Tcl/Tk 8.5 (2007) introduced a modern set of themed user interface components - along with a new API to use them. Both old and new APIs are still available. + along with a new API to use them (see :mod:`tkinter.ttk`). + Both old and new APIs are still available. Most documentation you will find online still uses the old API and can be woefully outdated. +.. include:: ../includes/optional-module.rst + .. seealso:: * `TkDocs `_ @@ -47,7 +55,7 @@ details that are unchanged. Tcl/Tk Resources: - * `Tk commands `_ + * `Tk commands `_ Comprehensive reference to each of the underlying Tcl/Tk commands used by Tkinter. * `Tcl/Tk Home Page `_ @@ -103,20 +111,20 @@ Ttk bindings are provided in a separate module, :mod:`tkinter.ttk`. Internally, Tk and Ttk use facilities of the underlying operating system, -i.e., Xlib on Unix/X11, Cocoa on macOS, GDI on Windows. +that is, Xlib on Unix/X11, Cocoa on macOS, GDI on Windows. -When your Python application uses a class in Tkinter, e.g., to create a widget, -the :mod:`tkinter` module first assembles a Tcl/Tk command string. It passes that +When your Python application uses a class in Tkinter, for example, to create a widget, +the :mod:`!tkinter` module first assembles a Tcl/Tk command string. It passes that Tcl command string to an internal :mod:`_tkinter` binary module, which then calls the Tcl interpreter to evaluate it. The Tcl interpreter will then call into the Tk and/or Ttk packages, which will in turn make calls to Xlib, Cocoa, or GDI. -Tkinter Modules +Tkinter modules --------------- Support for Tkinter is spread across several modules. Most applications will need the -main :mod:`tkinter` module, as well as the :mod:`tkinter.ttk` module, which provides +main :mod:`!tkinter` module, as well as the :mod:`tkinter.ttk` module, which provides the modern themed widget set and API:: @@ -124,85 +132,9 @@ the modern themed widget set and API:: from tkinter import ttk -.. class:: Tk(screenName=None, baseName=None, className='Tk', useTk=True, sync=False, use=None) - - Construct a toplevel Tk widget, which is usually the main window of an - application, and initialize a Tcl interpreter for this widget. Each - instance has its own associated Tcl interpreter. - - The :class:`Tk` class is typically instantiated using all default values. - However, the following keyword arguments are currently recognized: - - *screenName* - When given (as a string), sets the :envvar:`DISPLAY` environment - variable. (X11 only) - *baseName* - Name of the profile file. By default, *baseName* is derived from the - program name (``sys.argv[0]``). - *className* - Name of the widget class. Used as a profile file and also as the name - with which Tcl is invoked (*argv0* in *interp*). - *useTk* - If ``True``, initialize the Tk subsystem. The :func:`tkinter.Tcl() ` - function sets this to ``False``. - *sync* - If ``True``, execute all X server commands synchronously, so that errors - are reported immediately. Can be used for debugging. (X11 only) - *use* - Specifies the *id* of the window in which to embed the application, - instead of it being created as an independent toplevel window. *id* must - be specified in the same way as the value for the -use option for - toplevel widgets (that is, it has a form like that returned by - :meth:`winfo_id`). - - Note that on some platforms this will only work correctly if *id* refers - to a Tk frame or toplevel that has its -container option enabled. - - :class:`Tk` reads and interprets profile files, named - :file:`.{className}.tcl` and :file:`.{baseName}.tcl`, into the Tcl - interpreter and calls :func:`exec` on the contents of - :file:`.{className}.py` and :file:`.{baseName}.py`. The path for the - profile files is the :envvar:`HOME` environment variable or, if that - isn't defined, then :data:`os.curdir`. - - .. attribute:: tk - - The Tk application object created by instantiating :class:`Tk`. This - provides access to the Tcl interpreter. Each widget that is attached - the same instance of :class:`Tk` has the same value for its :attr:`tk` - attribute. - - .. attribute:: master - - The widget object that contains this widget. For :class:`Tk`, the - *master* is :const:`None` because it is the main window. The terms - *master* and *parent* are similar and sometimes used interchangeably - as argument names; however, calling :meth:`winfo_parent` returns a - string of the widget name whereas :attr:`master` returns the object. - *parent*/*child* reflects the tree-like relationship while - *master*/*slave* reflects the container structure. - - .. attribute:: children - - The immediate descendants of this widget as a :class:`dict` with the - child widget names as the keys and the child instance objects as the - values. - - -.. function:: Tcl(screenName=None, baseName=None, className='Tk', useTk=False) - - The :func:`Tcl` function is a factory function which creates an object much like - that created by the :class:`Tk` class, except that it does not initialize the Tk - subsystem. This is most often useful when driving the Tcl interpreter in an - environment where one doesn't want to create extraneous toplevel windows, or - where one cannot (such as Unix/Linux systems without an X server). An object - created by the :func:`Tcl` object can have a Toplevel window created (and the Tk - subsystem initialized) by calling its :meth:`loadtk` method. - - The modules that provide Tk support include: -:mod:`tkinter` +:mod:`!tkinter` Main Tkinter module. :mod:`tkinter.colorchooser` @@ -228,7 +160,7 @@ The modules that provide Tk support include: :mod:`tkinter.ttk` Themed widget set introduced in Tk 8.5, providing modern alternatives - for many of the classic widgets in the main :mod:`tkinter` module. + for many of the classic widgets in the main :mod:`!tkinter` module. Additional modules: @@ -237,29 +169,31 @@ Additional modules: :mod:`_tkinter` A binary module that contains the low-level interface to Tcl/Tk. - It is automatically imported by the main :mod:`tkinter` module, + It is automatically imported by the main :mod:`!tkinter` module, and should never be used directly by application programmers. It is usually a shared library (or DLL), but might in some cases be statically linked with the Python interpreter. :mod:`idlelib` Python's Integrated Development and Learning Environment (IDLE). Based - on :mod:`tkinter`. + on :mod:`!tkinter`. -:mod:`tkinter.constants` +:mod:`!tkinter.constants` Symbolic constants that can be used in place of strings when passing various parameters to Tkinter calls. Automatically imported by the - main :mod:`tkinter` module. + main :mod:`!tkinter` module. :mod:`tkinter.dnd` - (experimental) Drag-and-drop support for :mod:`tkinter`. This will + (experimental) Drag-and-drop support for :mod:`!tkinter`. This will become deprecated when it is replaced with the Tk DND. :mod:`turtle` Turtle graphics in a Tk window. +.. currentmodule:: tkinter + -Tkinter Life Preserver +Tkinter life preserver ---------------------- This section is not designed to be an exhaustive tutorial on either Tk or @@ -274,7 +208,7 @@ find more detailed documentation on them, including in the official Tcl/Tk reference manual. -A Hello World Program +A Hello World program ^^^^^^^^^^^^^^^^^^^^^ We'll start by walking through a "Hello World" application in Tkinter. This @@ -302,19 +236,20 @@ The following line creates a frame widget, which in this case will contain a label and a button we'll create next. The frame is fit inside the root window. -The next line creates a label widget holding a static text string. The -:meth:`grid` method is used to specify the relative layout (position) of the -label within its containing frame widget, similar to how tables in HTML work. +The next line creates a label widget holding a static text string. +The :meth:`~Grid.grid` method is used to specify the relative layout (position) +of the label within its containing frame widget, similar to how tables in HTML +work. -A button widget is then created, and placed to the right of the label. When -pressed, it will call the :meth:`destroy` method of the root window. +A button widget is then created, and placed to the right of the label. +When pressed, it will call the :meth:`~Misc.destroy` method of the root window. Finally, the :meth:`mainloop` method puts everything on the display, and responds to user input until the program terminates. -Important Tk Concepts +Important Tk concepts ^^^^^^^^^^^^^^^^^^^^^ Even this simple program illustrates the following key Tk concepts: @@ -346,7 +281,7 @@ event loop isn't running the event loop, your user interface won't update. -Understanding How Tkinter Wraps Tcl/Tk +Understanding how Tkinter wraps Tcl/Tk ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ When your application uses Tkinter's classes and methods, internally Tkinter @@ -392,7 +327,7 @@ by spaces. Without getting into too many details, notice the following: * Operations which are implemented as separate *commands* in Tcl (like ``grid`` or ``destroy``) are represented as *methods* on Tkinter widget objects. As you'll see shortly, at other times Tcl uses what appear to be - method calls on widget objects, which more closely mirror what would is + method calls on widget objects, which more closely mirror what is used in Tkinter. @@ -414,9 +349,9 @@ interactive Python shell or with :func:`print`, can help you identify what you need. To find out what configuration options are available on any widget, call its -:meth:`configure` method, which returns a dictionary containing a variety of -information about each object, including its default and current values. Use -:meth:`keys` to get just the names of each option. +:meth:`~Misc.configure` method, which returns a dictionary containing a variety +of information about each object, including its default and current values. +Use :meth:`~Misc.keys` to get just the names of each option. :: @@ -443,18 +378,20 @@ is helpful. print(set(dir(btn)) - set(dir(frm))) -Navigating the Tcl/Tk Reference Manual +Navigating the Tcl/Tk reference manual ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -As noted, the official `Tk commands `_ -reference manual (man pages) is often the most accurate description of what -specific operations on widgets do. Even when you know the name of the option -or method that you need, you may still have a few places to look. +As noted, the official +`Tk commands `_ reference +manual (man pages) is often the most accurate description of what specific +operations on widgets do. +Even when you know the name of the option or method that you need, you may +still have a few places to look. While all operations in Tkinter are implemented as method calls on widget objects, you've seen that many Tcl/Tk operations appear as commands that take a widget pathname as its first parameter, followed by optional -parameters, e.g. +parameters, for example :: @@ -473,22 +410,23 @@ name of a method to call). In the official Tcl/Tk reference documentation, you'll find most operations -that look like method calls on the man page for a specific widget (e.g., -you'll find the :meth:`invoke` method on the -`ttk::button `_ +that look like method calls on the man page for a specific widget (for example, +you'll find the :meth:`~tkinter.ttk.Button.invoke` method on the +`ttk::button `_ man page), while functions that take a widget as a parameter often have -their own man page (e.g., -`grid `_). +their own man page (for example, +`grid `_). You'll find many common options and methods in the -`options `_ or -`ttk::widget `_ man +`options `_ or +`ttk::widget `_ man pages, while others are found in the man page for a specific widget class. -You'll also find that many Tkinter methods have compound names, e.g., -:func:`winfo_x`, :func:`winfo_height`, :func:`winfo_viewable`. You'd find -documentation for all of these in the -`winfo `_ man page. +You'll also find that many Tkinter methods have compound names, for example, +:meth:`~Misc.winfo_x`, :meth:`~Misc.winfo_height`, +:meth:`~Misc.winfo_viewable`. +You'd find documentation for all of these in the +`winfo `_ man page. .. note:: Somewhat confusingly, there are also methods on all Tkinter widgets @@ -502,7 +440,7 @@ documentation for all of these in the Threading model --------------- -Python and Tcl/Tk have very different threading models, which :mod:`tkinter` +Python and Tcl/Tk have very different threading models, which :mod:`!tkinter` tries to bridge. If you use threads, you may need to be aware of this. A Python interpreter may have many threads associated with it. In Tcl, multiple @@ -510,34 +448,38 @@ threads can be created, but each thread has a separate Tcl interpreter instance associated with it. Threads can also create more than one interpreter instance, though each interpreter instance can be used only by the one thread that created it. -Each :class:`Tk` object created by :mod:`tkinter` contains a Tcl interpreter. +Each :class:`Tk` object created by :mod:`!tkinter` contains a Tcl interpreter. It also keeps track of which thread created that interpreter. Calls to -:mod:`tkinter` can be made from any Python thread. Internally, if a call comes +:mod:`!tkinter` can be made from any Python thread. Internally, if a call comes from a thread other than the one that created the :class:`Tk` object, an event is posted to the interpreter's event queue, and when executed, the result is returned to the calling Python thread. -Tcl/Tk applications are normally event-driven, meaning that after initialization, -the interpreter runs an event loop (i.e. :func:`Tk.mainloop`) and responds to events. -Because it is single-threaded, event handlers must respond quickly, otherwise they -will block other events from being processed. To avoid this, any long-running -computations should not run in an event handler, but are either broken into smaller -pieces using timers, or run in another thread. This is different from many GUI -toolkits where the GUI runs in a completely separate thread from all application -code including event handlers. +Tcl/Tk applications are normally event-driven, meaning that after +initialization, the interpreter runs an event loop (that is, +:meth:`Tk.mainloop `) and responds to events. +Because it is single-threaded, event handlers must respond quickly, otherwise +they will block other events from being processed. +To avoid this, any long-running computations should not run in an event +handler, but are either broken into smaller pieces using timers, or run in +another thread. +This is different from many GUI toolkits where the GUI runs in a completely +separate thread from all application code including event handlers. If the Tcl interpreter is not running the event loop and processing events, any -:mod:`tkinter` calls made from threads other than the one running the Tcl +:mod:`!tkinter` calls made from threads other than the one running the Tcl interpreter will fail. A number of special cases exist: -* Tcl/Tk libraries can be built so they are not thread-aware. In this case, - :mod:`tkinter` calls the library from the originating Python thread, even +* Tcl/Tk libraries built without thread support are now rare: the bundled + Tcl/Tk 8.6 is built with thread support, so this case only arises with some + older non-threaded builds. When the library is not thread-aware, + :mod:`!tkinter` calls the library from the originating Python thread, even if this is different than the thread that created the Tcl interpreter. A global lock ensures only one call occurs at a time. -* While :mod:`tkinter` allows you to create more than one instance of a :class:`Tk` +* While :mod:`!tkinter` allows you to create more than one instance of a :class:`Tk` object (with its own interpreter), all interpreters that are part of the same thread share a common event queue, which gets ugly fast. In practice, don't create more than one instance of :class:`Tk` at a time. Otherwise, it's best to create @@ -548,17 +490,17 @@ A number of special cases exist: or abandon the event loop entirely. If you're doing anything tricky when it comes to events or threads, be aware of these possibilities. -* There are a few select :mod:`tkinter` functions that presently work only when +* There are a few select :mod:`!tkinter` functions that presently work only when called from the thread that created the Tcl interpreter. -Handy Reference +Handy reference --------------- .. _tkinter-setting-options: -Setting Options +Setting options ^^^^^^^^^^^^^^^ Options control things like the color and border width of a widget. Options can @@ -580,6 +522,17 @@ Use the config() method to update multiple attrs subsequent to object creation fred.config(fg="red", bg="blue") +.. note:: + + The ``fg`` and ``bg`` options used here, + and other options that control a widget's appearance, + belong to the classic :mod:`!tkinter` widgets. + The themed :mod:`tkinter.ttk` widgets recommended in the introduction + do not accept them; + style a themed widget through the :class:`ttk.Style ` + class instead. + The three ways of setting an option shown above apply to both widget sets. + For a complete explanation of a given option and its behavior, see the Tk man pages for the widget in question. @@ -594,17 +547,14 @@ document. Some options don't apply to some kinds of widgets. Whether a given widget responds to a particular option depends on the class of the widget; buttons have a ``command`` option, labels do not. -The options supported by a given widget are listed in that widget's man page, or -can be queried at runtime by calling the :meth:`config` method without -arguments, or by calling the :meth:`keys` method on that widget. The return -value of these calls is a dictionary whose key is the name of the option as a -string (for example, ``'relief'``) and whose values are 5-tuples. +The options supported by a given widget are listed in that widget's man page, +or can be queried at runtime by calling the :meth:`~Misc.config` method without +arguments, or by calling the :meth:`~Misc.keys` method on that widget. +The return value of these calls is a dictionary whose key is the name of the +option as a string (for example, ``'relief'``) and whose values are 5-tuples. -Some options, like ``bg`` are synonyms for common options with long names -(``bg`` is shorthand for "background"). Passing the ``config()`` method the name -of a shorthand option will return a 2-tuple, not 5-tuple. The 2-tuple passed -back will contain the name of the synonym and the "real" option (such as -``('bg', 'background')``). +Some options, like ``bg``, are synonyms for common options with long names +(``bg`` is shorthand for "background"). +-------+---------------------------------+--------------+ | Index | Meaning | Example | @@ -630,174 +580,206 @@ Of course, the dictionary printed will include all the options available and their values. This is meant only as an example. -The Packer -^^^^^^^^^^ +.. _pack-the-packer: +.. _tkinter-geometry-management: +.. _the-packer: +.. _packer-options: -.. index:: single: packing (widgets) +Geometry management +^^^^^^^^^^^^^^^^^^^ -The packer is one of Tk's geometry-management mechanisms. Geometry managers -are used to specify the relative positioning of widgets within their container - -their mutual *master*. In contrast to the more cumbersome *placer* (which is -used less commonly, and we do not cover here), the packer takes qualitative -relationship specification - *above*, *to the left of*, *filling*, etc - and -works everything out to determine the exact placement coordinates for you. +.. index:: + single: geometry management (widgets) + single: packing (widgets) + +Creating a widget does not display it. +A widget appears only after it has been handed to a *geometry manager*, +which works out its size and position inside its container +and keeps the layout up to date as the container is resized or its content changes. +Forgetting to call a geometry manager is a common early mistake: +the widget is created, but nothing shows up. + +Tk provides three geometry managers. +Each is inherited by every widget, so any widget can be managed by any of them +(but see the warning below about the incompatibility of grid and pack). +The choice depends on the kind of layout you want. + +:meth:`grid ` + Arranges widgets in a two-dimensional table of rows and columns. + It is the most flexible manager and the one to reach for by default: + layouts that would otherwise need several nested frames can often be + expressed as a single grid, + and rows and columns can be told how to absorb extra space. -The size of any *master* widget is determined by the size of the "slave widgets" -inside. The packer is used to control where slave widgets appear inside the -master into which they are packed. You can pack widgets into frames, and frames -into other frames, in order to achieve the kind of layout you desire. -Additionally, the arrangement is dynamically adjusted to accommodate incremental -changes to the configuration, once it is packed. + :: -Note that widgets do not appear until they have had their geometry specified -with a geometry manager. It's a common early mistake to leave out the geometry -specification, and then be surprised when the widget is created but nothing -appears. A widget will appear only after it has had, for example, the packer's -:meth:`pack` method applied to it. + ttk.Label(frm, text="Name:").grid(column=0, row=0, sticky="w") + ttk.Entry(frm).grid(column=1, row=0) + ttk.Button(frm, text="OK").grid(column=1, row=1, sticky="e") -The pack() method can be called with keyword-option/value pairs that control -where the widget is to appear within its container, and how it is to behave when -the main application window is resized. Here are some examples:: +:meth:`pack ` + Stacks widgets against one side of their container + -- ``"top"`` (the default), ``"bottom"``, ``"left"`` or ``"right"`` -- + and can make them fill or expand into the space that is left. + It is convenient for simple arrangements, + such as a single row or column of widgets + or a content area framed by a toolbar and a status bar. - fred.pack() # defaults to side = "top" - fred.pack(side="left") - fred.pack(expand=1) + :: + toolbar.pack(side="top", fill="x") + status.pack(side="bottom", fill="x") + body.pack(side="left", expand=True, fill="both") -Packer Options -^^^^^^^^^^^^^^ +:meth:`place ` + Positions each widget at an explicit spot, + given either as absolute screen distances or as a fraction of the container's size. + It offers the most control but the least automatic behavior, and is used the least; + it suits special cases such as overlapping widgets or precise custom layouts. -For more extensive information on the packer and the options that it can take, -see the man pages and page 183 of John Ousterhout's book. + :: -anchor - Anchor type. Denotes where the packer is to place each slave in its parcel. + background.place(x=0, y=0, relwidth=1.0, relheight=1.0) + badge.place(relx=1.0, rely=0.0, anchor="ne") -expand - Boolean, ``0`` or ``1``. +Layouts are built up by nesting: +grid or pack widgets, including frames, inside a frame or toplevel. +Toplevels are managed by the OS window manager. +Classic and themed :mod:`tkinter.ttk` widgets can be managed interchangeably. -fill - Legal values: ``'x'``, ``'y'``, ``'both'``, ``'none'``. +.. warning:: -ipadx and ipady - A distance - designating internal padding on each side of the slave widget. + Do not apply :meth:`!pack` and :meth:`!grid` to two widgets that share the same container. + The two managers negotiate sizes in incompatible ways, + and the application can hang as they repeatedly resize the container against each other. + To combine them, keep each manager's widgets in a separate frame. -padx and pady - A distance - designating external padding on each side of the slave widget. +The full set of options accepted by each manager, with their values and defaults, +is documented under :meth:`Grid.grid_configure`, :meth:`Pack.pack_configure` +and :meth:`Place.place_configure`; +see also the :manpage:`grid(3tk)`, :manpage:`pack(3tk)` and :manpage:`place(3tk)` +man pages. -side - Legal values are: ``'left'``, ``'right'``, ``'top'``, ``'bottom'``. +.. _coupling-widget-variables: -Coupling Widget Variables +Coupling widget variables ^^^^^^^^^^^^^^^^^^^^^^^^^ -The current-value setting of some widgets (like text entry widgets) can be -connected directly to application variables by using special options. These -options are ``variable``, ``textvariable``, ``onvalue``, ``offvalue``, and -``value``. This connection works both ways: if the variable changes for any -reason, the widget it's connected to will be updated to reflect the new value. - -Unfortunately, in the current implementation of :mod:`tkinter` it is not -possible to hand over an arbitrary Python variable to a widget through a -``variable`` or ``textvariable`` option. The only kinds of variables for which -this works are variables that are subclassed from a class called Variable, -defined in :mod:`tkinter`. - -There are many useful subclasses of Variable already defined: -:class:`StringVar`, :class:`IntVar`, :class:`DoubleVar`, and -:class:`BooleanVar`. To read the current value of such a variable, call the -:meth:`get` method on it, and to change its value you call the :meth:`!set` -method. If you follow this protocol, the widget will always track the value of -the variable, with no further intervention on your part. +Some widgets can tie their current value directly to a program variable, +so that the two stay in sync. +Options such as ``variable``, ``textvariable``, ``value``, ``onvalue`` and +``offvalue`` set up this connection: +when the user changes the widget the variable is updated, +and when the variable is set the widget redraws to match. + +A widget can be linked only to a :class:`Variable` object, +not to an ordinary Python variable. +This is not a limitation of :mod:`!tkinter` +but a consequence of how the two languages differ: +the link relies on Tcl being notified every time the value changes, +and Python offers no way to react when a plain variable is reassigned. +A :class:`Variable` sidesteps this by keeping its value inside the Tcl interpreter +and exposing it through explicit :meth:`~Variable.get` and :meth:`~Variable.set` methods. + +Ready-made subclasses cover the common types: +:class:`StringVar`, :class:`IntVar`, :class:`DoubleVar` and :class:`BooleanVar`. +Pass one as a widget's ``textvariable`` (or ``variable``) option, +then read and update it with :meth:`~Variable.get` and :meth:`~Variable.set`; +the widget tracks it with no further work on your part. + +Keep a reference to the variable for as long as the widget uses it +-- for example by storing it as an attribute. +A :class:`Variable` that is garbage collected removes its underlying Tcl variable, +breaking the connection to the widget (see :class:`Variable`). For example:: import tkinter as tk + from tkinter import ttk + + root = tk.Tk() - class App(tk.Frame): - def __init__(self, master): - super().__init__(master) - self.pack() + # Create the application variable and give it an initial value. + contents = tk.StringVar(value="this is a variable") - self.entrythingy = tk.Entry() - self.entrythingy.pack() + # Tell the entry widget to track the variable. + entry = ttk.Entry(root, textvariable=contents) + entry.pack() - # Create the application variable. - self.contents = tk.StringVar() - # Set it to some value. - self.contents.set("this is a variable") - # Tell the entry widget to watch this variable. - self.entrythingy["textvariable"] = self.contents + # Print the current value whenever the user presses Return. + def print_contents(event): + print("The current entry content is:", contents.get()) - # Define a callback for when the user hits return. - # It prints the current value of the variable. - self.entrythingy.bind('', - self.print_contents) + entry.bind("", print_contents) - def print_contents(self, event): - print("Hi. The current entry content is:", - self.contents.get()) + # Setting the variable from the program updates the entry through the + # same link. + def clear(): + contents.set("") - root = tk.Tk() - myapp = App(root) - myapp.mainloop() + ttk.Button(root, text="Clear", command=clear).pack() + + root.mainloop() + +.. _tkinter-window-manager: -The Window Manager +The window manager ^^^^^^^^^^^^^^^^^^ .. index:: single: window manager (widgets) -In Tk, there is a utility command, ``wm``, for interacting with the window -manager. Options to the ``wm`` command allow you to control things like titles, -placement, icon bitmaps, and the like. In :mod:`tkinter`, these commands have -been implemented as methods on the :class:`Wm` class. Toplevel widgets are -subclassed from the :class:`Wm` class, and so can call the :class:`Wm` methods -directly. +The *window manager* is the part of the desktop responsible for the title bar, +border and controls drawn around each top-level window, +and for such things as its title, position, size and icon. +Tk gives access to these through the :class:`Wm` mixin, +which is inherited by the :class:`Tk` root window and by every :class:`Toplevel`. +You therefore call the window-manager methods directly on a top-level window. +Each has a short name and an equivalent ``wm_``-prefixed name, +for example :meth:`~Wm.title` and :meth:`~Wm.wm_title`. -To get at the toplevel window that contains a given widget, you can often just -refer to the widget's master. Of course if the widget has been packed inside of -a frame, the master won't represent a toplevel window. To get at the toplevel -window that contains an arbitrary widget, you can call the :meth:`_root` method. -This method begins with an underscore to denote the fact that this function is -part of the implementation, and not an interface to Tk functionality. +These methods act on the top-level window +whether its content is built from the classic widgets or the themed :mod:`tkinter.ttk` widgets. +To reach the top-level window containing an arbitrary widget, +call its :meth:`~Misc.winfo_toplevel` method. -Here are some examples of typical usage:: +For example:: import tkinter as tk + from tkinter import ttk + + root = tk.Tk() + root.title("My Application") + root.geometry("640x480") + root.minsize(320, 240) - class App(tk.Frame): - def __init__(self, master=None): - super().__init__(master) - self.pack() + ttk.Label(root, text="Hello").pack(padx=20, pady=20) - # create the application - myapp = App() + root.mainloop() - # - # here are method calls to the window manager class - # - myapp.master.title("My Do-Nothing Application") - myapp.master.maxsize(1000, 400) +See :class:`Wm` for the full set of window-manager methods. - # start the program - myapp.mainloop() +.. _Tk-option-data-types: -Tk Option Data Types +Tk option data types ^^^^^^^^^^^^^^^^^^^^ .. index:: single: Tk Option Data Types +Many widget options documented in the reference +accept values of a small number of common types, described here. + anchor Legal values are points of the compass: ``"n"``, ``"ne"``, ``"e"``, ``"se"``, ``"s"``, ``"sw"``, ``"w"``, ``"nw"``, and also ``"center"``. bitmap - There are eight built-in, named bitmaps: ``'error'``, ``'gray25'``, - ``'gray50'``, ``'hourglass'``, ``'info'``, ``'questhead'``, ``'question'``, - ``'warning'``. To specify an X bitmap filename, give the full path to the file, - preceded with an ``@``, as in ``"@/usr/contrib/bitmap/gumby.bit"``. + There are ten built-in, named bitmaps: ``'error'``, ``'gray12'``, + ``'gray25'``, ``'gray50'``, ``'gray75'``, ``'hourglass'``, ``'info'``, + ``'questhead'``, ``'question'``, ``'warning'``. To specify an X bitmap + filename, give the full path to the file, preceded with an ``@``, as in + ``"@/usr/contrib/bitmap/gumby.bit"``. boolean You can pass integers 0 or 1 or the strings ``"yes"`` or ``"no"``. @@ -813,13 +795,20 @@ color Colors can be given as the names of X colors in the rgb.txt file, or as strings representing RGB values in 4 bit: ``"#RGB"``, 8 bit: ``"#RRGGBB"``, 12 bit: ``"#RRRGGGBBB"``, or 16 bit: ``"#RRRRGGGGBBBB"`` ranges, where R,G,B here - represent any legal hex digit. See page 160 of Ousterhout's book for details. + represent any legal hex digit. See the :manpage:`colors(3tk)` man page for + the list of named colors. cursor - The standard X cursor names from :file:`cursorfont.h` can be used, without the - ``XC_`` prefix. For example to get a hand cursor (:const:`XC_hand2`), use the - string ``"hand2"``. You can also specify a bitmap and mask file of your own. - See page 179 of Ousterhout's book. + The name of the mouse cursor to display while the pointer is over the widget. + Tk provides a portable set of cursor names available on all platforms + (for example ``"arrow"``, ``"watch"``, ``"cross"``, or ``"hand2"``); + the standard X cursor names from :file:`cursorfont.h` may also be used, + without the ``XC_`` prefix (so ``XC_hand2`` becomes ``"hand2"``). + The full list of names, including the platform-specific ones, + is given in the :manpage:`cursors(3tk)` manual page. + You can also specify a bitmap and mask file of your own. + On Windows a cursor file (:file:`.cur` or :file:`.ani`) may be used directly, + giving its path preceded with an ``@``, as in ``"@C:/cursors/bart.ani"``. distance Screen distances can be specified in either pixels or absolute distances. @@ -829,9 +818,11 @@ distance as ``"3.5i"``. font - Tk uses a list font name format, such as ``{courier 10 bold}``. Font sizes with - positive numbers are measured in points; sizes with negative numbers are - measured in pixels. + Tk uses a font description such as ``{courier 10 bold}``; in + :mod:`!tkinter` this is most naturally passed as a tuple of + ``(family, size, *styles)`` (or as the equivalent string + ``"Courier 10 bold"``). Font sizes with positive numbers are measured in + points; sizes with negative numbers are measured in pixels. geometry This is a string of the form ``widthxheight``, where width and height are @@ -839,8 +830,7 @@ geometry For example: ``fred["geometry"] = "200x100"``. justify - Legal values are the strings: ``"left"``, ``"center"``, ``"right"``, and - ``"fill"``. + Legal values are the strings: ``"left"``, ``"center"``, and ``"right"``. region This is a string with four space-delimited elements, each of which is a legal @@ -849,7 +839,8 @@ region relief Determines what the border style of a widget will be. Legal values are: - ``"raised"``, ``"sunken"``, ``"flat"``, ``"groove"``, and ``"ridge"``. + ``"raised"``, ``"sunken"``, ``"flat"``, ``"groove"``, ``"ridge"``, and + ``"solid"``. scrollcommand This is almost always the :meth:`!set` method of some scrollbar widget, but can @@ -860,7 +851,7 @@ wrap .. _Bindings-and-Events: -Bindings and Events +Bindings and events ^^^^^^^^^^^^^^^^^^^ .. index:: @@ -876,9 +867,11 @@ of the bind method is:: where: sequence - is a string that denotes the target kind of event. (See the - :manpage:`bind(3tk)` man page, and page 201 of John Ousterhout's book, - :title-reference:`Tcl and the Tk Toolkit (2nd edition)`, for details). + is a string that denotes the target kind of event. Physical events use the + ```` form (for example ``""`` or + ``""``); application-defined virtual events use double angle + brackets, as in ``"<>"``. (See the + :manpage:`bind(3tk)` man page for details.) func is a Python function, taking one argument, to be invoked when the event occurs. @@ -922,9 +915,27 @@ they are denoted in Tk, which can be useful when referring to the Tk man pages. +----+---------------------+----+---------------------+ | %y | y | %Y | y_root | +----+---------------------+----+---------------------+ +| %# | serial | %b | num | ++----+---------------------+----+---------------------+ +| %d | detail | %D | delta | ++----+---------------------+----+---------------------+ + +The ``add`` parameter above only affects the bindings you make yourself. +Every widget also inherits *class bindings* +that implement its standard behavior -- +for example a :class:`Text` widget binds :kbd:`Control-t` +to transpose two characters. +These are described in the bindings section of the widget's Tk man page +(such as :manpage:`text(3tk)` or :manpage:`entry(3tk)`). +Class bindings are processed separately from your own, +so binding an event yourself does not replace the default; both run. +To suppress an unwanted default binding, +bind the event on the widget +and return the string ``"break"`` from your callback. -The index Parameter + +The index parameter ^^^^^^^^^^^^^^^^^^^ A number of widgets require "index" parameters to be passed. These are used to @@ -932,9 +943,32 @@ point at a specific place in a Text widget, or to particular characters in an Entry widget, or to particular menu items in a Menu widget. Entry widget indexes (index, view index, etc.) - Entry widgets have options that refer to character positions in the text being - displayed. You can use these :mod:`tkinter` functions to access these special - points in text widgets: + Entry widgets have methods and options that refer to character positions + in the text being displayed. + Anytime an index is needed, you may pass in: + + * an integer which refers to the numeric position of a character, + counted from the beginning of the text, starting with 0; + + * the string ``"anchor"``, + which refers to the anchor point of the selection, + set with the widget's selection methods; + + * the string ``"end"``, + which refers to the position just after the last character; + + * the string ``"insert"``, + which refers to the character just after the insertion cursor; + + * the strings ``"sel.first"`` and ``"sel.last"``, + which refer to the first character in the selection + and the position just after the last + (it is an error to use these if there is no selection); + + * a string consisting of ``@`` followed by an integer, as in ``"@6"``, + where the integer is interpreted as an x pixel coordinate + in the entry's coordinate system, + selecting the character spanning that point. Text widget indexes The index notation for Text widgets is very rich and is best described in the Tk @@ -952,8 +986,9 @@ Menu indexes (menu.invoke(), menu.entryconfig(), etc.) * the string ``"last"`` which refers to the last menu item; - * An integer preceded by ``@``, as in ``@6``, where the integer is interpreted - as a y pixel coordinate in the menu's coordinate system; + * a string consisting of ``@`` followed by an integer, as in ``"@6"``, where + the integer is interpreted as a y pixel coordinate in the menu's coordinate + system; * the string ``"none"``, which indicates no menu entry at all, most often used with menu.activate() to deactivate all entries, and finally, @@ -989,7 +1024,7 @@ option (other options are available as well). :meth:`!copy`. The image object can then be used wherever an ``image`` option is supported by -some widget (e.g. labels, buttons, menus). In these cases, Tk will not keep a +some widget (for example, labels, buttons, menus). In these cases, Tk will not keep a reference to the image. When the last Python reference to the image object is deleted, the image data is deleted as well, and Tk will display an empty box wherever the image was used. @@ -999,51 +1034,5520 @@ wherever the image was used. The `Pillow `_ package adds support for formats such as BMP, JPEG, TIFF, and WebP, among others. -.. _tkinter-file-handlers: -File Handlers -------------- +Reference +--------- -Tk allows you to register and unregister a callback function which will be -called from the Tk mainloop when I/O is possible on a file descriptor. -Only one handler may be registered per file descriptor. Example code:: +.. currentmodule:: tkinter - import tkinter - widget = tkinter.Tk() - mask = tkinter.READABLE | tkinter.WRITABLE - widget.tk.createfilehandler(file, mask, callback) - ... - widget.tk.deletefilehandler(file) +This section documents the classes, methods, functions and constants of the +:mod:`!tkinter` module. +Most of them wrap Tcl/Tk commands; consult the official Tcl/Tk manual pages for +the full list of widget options and further details. -This feature is not available on Windows. +.. exception:: TclError -Since you don't know how many bytes are available for reading, you may not -want to use the :class:`~io.BufferedIOBase` or :class:`~io.TextIOBase` -:meth:`~io.BufferedIOBase.read` or :meth:`~io.IOBase.readline` methods, -since these will insist on reading a predefined number of bytes. -For sockets, the :meth:`~socket.socket.recv` or -:meth:`~socket.socket.recvfrom` methods will work fine; for other files, -use raw reads or ``os.read(file.fileno(), maxbytecount)``. + The exception raised when a call into the Tcl interpreter fails, for example + when a widget is given an unknown option or an invalid value. +Base and mixin classes +^^^^^^^^^^^^^^^^^^^^^^ -.. method:: Widget.tk.createfilehandler(file, mask, func) +.. class:: Misc() - Registers the file handler callback function *func*. The *file* argument - may either be an object with a :meth:`~io.IOBase.fileno` method (such as - a file or socket object), or an integer file descriptor. The *mask* - argument is an ORed combination of any of the three constants below. - The callback is called as follows:: + The :class:`!Misc` class is a mix-in inherited by :class:`Tk` and, through + :class:`BaseWidget`, by every widget. + It provides the large set of methods common to all Tk objects: querying + window information, managing event bindings and the event loop, controlling + the keyboard focus and pointer grabs, accessing the selection, clipboard and + option database, and assorted utility and introspection services. + Because they are inherited, these methods are available on every widget and + on the :class:`Tk` application object, and are documented here once rather + than repeated for each widget. - callback(file, mask) + .. method:: cget(key) + Return the current value of the configuration option named *key* for this + widget, as a string. + The expression ``widget[key]`` is equivalent and may be used instead. -.. method:: Widget.tk.deletefilehandler(file) + .. method:: config(cnf=None, **kw) + :no-typesetting: - Unregisters a file handler. + .. method:: configure(cnf=None, **kw) + Query or modify the configuration options of the widget. + With no arguments, return a dictionary mapping every available option + name to a tuple describing it (its name, X resource name, X resource + class, default value and current value). + If a single option name is given as a string, return the tuple for just + that option. + If one or more keyword arguments are given, or a dictionary is passed as + *cnf*, set each named option to the corresponding value; the expression + ``widget[key] = value`` sets a single option in the same way. -.. data:: READABLE - WRITABLE - EXCEPTION + :meth:`config` is an alias of :meth:`!configure`. - Constants used in the *mask* arguments. + .. method:: keys() + + Return a list of the names of all configuration options of this widget. + + .. method:: getboolean(s) + + Interpret the string *s* as a Tcl boolean and return the corresponding + :class:`bool`. + Tcl accepts values such as ``'1'``, ``'0'``, ``'yes'``, ``'no'``, + ``'true'`` and ``'false'``. + Raise :exc:`ValueError` if *s* is not a valid boolean. + + .. method:: getdouble(s) + + Interpret the string *s* as a Tcl floating-point number and return it as + a :class:`float`. + Raise :exc:`ValueError` if *s* is not a valid number. + + .. versionadded:: 3.5 + + + .. method:: getint(s) + + Interpret the string *s* as a Tcl integer and return it as an + :class:`int`. + Raise :exc:`ValueError` if *s* is not a valid integer. + + .. method:: getvar(name) + + Return the value of the Tcl global variable named *name*. + + .. method:: setvar(name, value) + + Set the Tcl global variable named *name* to *value*. + + The :meth:`!getvar` and :meth:`!setvar` methods give direct access to Tcl + variables. + In most code you will instead use a :class:`Variable` subclass such as + :class:`StringVar` or :class:`IntVar`, which wraps a Tcl variable and + converts its value to and from a Python type. + + .. method:: register(func, subst=None, needcleanup=1) + + Register the Python callable *func* as a Tcl command and return the name + of the new command as a string. + Whenever Tcl invokes that command, *func* is called; if *subst* is given, + it is applied to the command's arguments first. + This is the mechanism used internally to turn Python callbacks into the + command names passed to Tk options such as *command*. + Unless *needcleanup* is false, the command is deleted automatically when + the widget is destroyed. + + .. versionchanged:: 3.13 + The arguments passed to *func* are no longer converted to strings. + + .. method:: deletecommand(name) + + Delete the Tcl command named *name*, such as one previously returned by + :meth:`register`. + + .. method:: nametowidget(name) + + Return the widget instance corresponding to the Tk pathname *name*. + + .. method:: send(interp, cmd, *args) + + Send the Tcl command *cmd*, with the given *args*, to the Tcl interpreter + registered under the name *interp*, and return its result. + This is not available on all platforms. + + .. method:: destroy() + + Destroy this widget and all of its descendant widgets, and delete the Tcl + commands associated with them. + + .. method:: lift(aboveThis=None) + :no-typesetting: + + .. method:: tkraise(aboveThis=None) + + Raise this widget in the stacking order so that it is drawn on top of its + siblings. + If *aboveThis* is given, the widget is moved to be just above it in the + stacking order instead. + + :meth:`lift` is an alias of :meth:`!tkraise`. + + .. method:: lower(belowThis=None) + + Lower this widget in the stacking order so that it is drawn beneath its + siblings. + If *belowThis* is given, the widget is moved to be just below it in the + stacking order instead. + + :meth:`tkraise`/:meth:`lift` and :meth:`lower` are overridden by the + :class:`Canvas` widget, + where they restack canvas items instead. + + .. method:: image_names() + + Return the names of all images that currently exist in the Tcl + interpreter. + + This is overridden by the :class:`Text` widget, + where :meth:`!image_names` returns the names of its embedded images + instead. + + .. method:: image_types() + + Return the available image types, such as ``'photo'`` and ``'bitmap'``. + + .. method:: anchor(anchor=None) + :no-typesetting: + + .. method:: grid_anchor(anchor=None) + + Set the anchor that controls where the grid is placed inside this + container when the container is larger than the grid and no row or column + has a non-zero weight. + *anchor* is one of the usual anchor strings, such as ``'nw'`` (the + default) or ``'center'``. + Called with no argument, this method has no effect. + + :meth:`anchor` is an alias of :meth:`!grid_anchor`. + + .. versionadded:: 3.3 + + .. method:: bbox(column=None, row=None, col2=None, row2=None) + :no-typesetting: + + .. method:: grid_bbox(column=None, row=None, col2=None, row2=None) + + Return the bounding box, in pixels, of a region of the grid laid out in + this container, as a 4-tuple ``(xoffset, yoffset, width, height)``. + With no arguments the bounding box of the whole grid is returned. + If *column* and *row* are given, the box spans from the cell at row and + column 0 to that cell; if *col2* and *row2* are also given, it spans from + the cell (*column*, *row*) to the cell (*col2*, *row2*). + + :meth:`bbox` is an alias of :meth:`!grid_bbox`, + except on :class:`Canvas`, :class:`Listbox`, :class:`Spinbox`, + :class:`Text`, :class:`ttk.Entry ` and + :class:`ttk.Treeview `, + which provide their own :meth:`!bbox` method. + + .. method:: columnconfigure(index, cnf={}, **kw) + :no-typesetting: + + .. method:: grid_columnconfigure(index, cnf={}, **kw) + + Query or set the properties of the column (or columns) *index* of the + grid managed by this container. + *index* may be a column number; when setting options it may also be a + list of column numbers, the string ``'all'`` to affect every column, or + a child widget whose occupied columns are affected. + The supported options are: + + *minsize* + The column's minimum size, in pixels. + + *weight* + An integer setting how much of any extra space is apportioned to the + column. + A weight of ``0`` keeps the column at its requested size, and a column + of weight two grows twice as fast as a column of weight one. + + *uniform* + The name of a uniform group. + Columns sharing a non-empty group name are kept in sizes that are + strictly proportional to their weights. + + *pad* + Extra space, in pixels, added to the largest widget in the column when + computing the column's size. + + With a single option name, return that option's value; with no options, + return a dictionary of all of them. + + :meth:`columnconfigure` is an alias of :meth:`!grid_columnconfigure`. + + .. method:: rowconfigure(index, cnf={}, **kw) + :no-typesetting: + + .. method:: grid_rowconfigure(index, cnf={}, **kw) + + Query or set the properties of the row (or rows) *index* of the grid + managed by this container. + *index* is interpreted as for :meth:`grid_columnconfigure`, and the + supported options (*minsize*, *weight*, *uniform* and *pad*) are the + same, applied to a row instead of a column. + + :meth:`rowconfigure` is an alias of :meth:`!grid_rowconfigure`. + + .. method:: grid_location(x, y) + + Return the ``(column, row)`` of the grid cell that contains the pixel at + position (*x*, *y*), given in pixels relative to this container. + For locations above or to the left of the grid, ``-1`` is returned for + the corresponding coordinate. + + .. method:: grid_propagate() + grid_propagate(flag) + + Enable or disable geometry propagation for this container when it manages + its children with the grid geometry manager. + When *flag* is true, the container resizes itself to fit the requested + sizes of its children; when it is false, its size is left under your + control. + Called with no argument, return the current setting as a boolean. + + .. method:: size() + :no-typesetting: + + .. method:: grid_size() + + Return the size of the grid managed by this container as a + ``(columns, rows)`` tuple. + + :meth:`size` is an alias of :meth:`!grid_size`, + except on the :class:`Listbox` widget, + which provides its own :meth:`!size` method. + + .. method:: grid_slaves(row=None, column=None) + + Return a list of the child widgets managed in this container's grid, most + recently managed first. + If *row* or *column* is given, only the children in that row or column + are returned. + + .. method:: propagate() + propagate(flag) + :no-typesetting: + + .. method:: pack_propagate() + pack_propagate(flag) + + Enable or disable geometry propagation for this container when it manages + its children with the pack geometry manager. + When *flag* is true, the container resizes itself to fit the requested + sizes of its children; when it is false, its size is left under your + control. + Called with no argument, return the current setting as a boolean. + + :meth:`propagate` is an alias of :meth:`!pack_propagate`. + + .. method:: slaves() + :no-typesetting: + + .. method:: pack_slaves() + + Return a list of the child widgets managed by this container with the + pack geometry manager, in packing order. + + :meth:`slaves` is an alias of :meth:`!pack_slaves`. + + .. method:: place_slaves() + + Return a list of the child widgets managed by this container with the + place geometry manager. + + .. method:: bind(sequence=None, func=None, add=None) + + Bind the event pattern *sequence* on this widget to the callable *func*. + + *sequence* is an event pattern, such as ``''`` (a mouse click) + or ``''``, optionally a concatenation of several such + patterns that must occur shortly after one another. + When the event occurs, *func* is called with an :class:`Event` instance + describing it as its only argument; if *func* returns the string + ``'break'``, no further bindings for the event are invoked. + + If *add* is true, *func* is added to any functions already bound to + *sequence*; otherwise it replaces them. + The binding applies only to this widget. + + :meth:`!bind` returns a string identifier (a *funcid*) that can later be + passed to :meth:`unbind` to remove the binding without leaking the + associated Tcl command. + + If *func* is omitted, return the binding currently associated with + *sequence*; if *sequence* is also omitted, return a list of all the + sequences for which bindings exist on this widget. + + .. method:: bind_class(className, sequence=None, func=None, add=None) + + Like :meth:`bind`, but bind *func* to the binding tag *className* rather + than to a single widget, so that the binding applies to every widget + having that tag. + *className* is usually the name of a widget class, such as ``'Button'``, + in which case the binding affects all widgets of that class. + The set of binding tags for a widget can be inspected and changed with + :meth:`bindtags`. + + The remaining arguments and the return value are as for :meth:`bind`. + + .. method:: bind_all(sequence=None, func=None, add=None) + + Like :meth:`bind`, but bind *func* to the special binding tag ``'all'``, + so that the binding applies to every widget in the application. + + The remaining arguments and the return value are as for :meth:`bind`. + + .. method:: unbind(sequence, funcid=None) + + Remove bindings for the event pattern *sequence* on this widget. + + If *funcid* is given, only the function identified by it (a value + returned from a previous call to :meth:`bind`) is removed, and its + associated Tcl command is deleted. + Otherwise all bindings for *sequence* are destroyed, leaving it unbound. + + .. versionchanged:: 3.13 + If *funcid* is given, only that callback is unbound; other callbacks + bound to *sequence* are kept. + + + .. method:: unbind_class(className, sequence) + + Remove all bindings for the event pattern *sequence* from the binding tag + *className*. + See :meth:`bind_class`. + + .. method:: unbind_all(sequence) + + Remove all bindings for the event pattern *sequence* from the special + binding tag ``'all'``. + See :meth:`bind_all`. + + .. method:: bindtags(tagList=None) + + If *tagList* is omitted, return a tuple of the binding tags associated + with this widget. + When an event occurs in a widget, it is applied to each of the widget's + binding tags in order, and for each tag the most specific matching + binding is executed. + By default a widget has four binding tags: its own pathname, its widget + class, the pathname of its nearest toplevel ancestor, and ``'all'``, in + that order. + + If *tagList* is given, it must be a sequence of strings; the widget's + binding tags are set to its elements, which determines the order in which + bindings are evaluated. + + The methods with the ``event_`` prefix define virtual events and generate + events programmatically. + + .. method:: event_add(virtual, *sequences) + + Associate the virtual event *virtual*, whose name has the form + ``'<>'``, with each of the physical event patterns given by + *sequences*, so that the virtual event triggers whenever any of them + occurs. + If *virtual* is already defined, the new sequences are added to its + existing ones. + + .. method:: event_delete(virtual, *sequences) + + Remove each of *sequences* from those associated with the virtual event + *virtual*. + Sequences that are not currently associated with *virtual* are ignored. + If no *sequences* are given, all physical event sequences are removed, so + that *virtual* no longer triggers. + + .. method:: event_generate(sequence, **kw) + + Generate the event *sequence* on this widget and arrange for it to be + processed just as if it had come from the window system. + *sequence* must be a single event pattern, such as ``''`` or + ``'<>'``, not a concatenation of several. + Keyword arguments specify additional fields of the event, for example *x* + and *y* for the pointer position, or *when* to control when the event is + processed; refer to the Tk ``event`` manual page for the full list. + + .. method:: event_info(virtual=None) + + If *virtual* is omitted, return a tuple of all the virtual events that + are currently defined. + If *virtual* is given, return a tuple of the physical event sequences + currently associated with it, or an empty tuple if it is not defined. + + The methods with the ``after`` prefix schedule callbacks to run after a + delay or when the application is idle. + + .. method:: after(ms, func=None, *args, **kw) + + Schedule the callable *func* to be called after *ms* milliseconds, with + *args* and *kw* passed to it as positional and keyword arguments. + Return an identifier that can be passed to :meth:`after_cancel` to cancel + the call. + + If *func* is omitted, sleep for *ms* milliseconds instead, processing no + events during that time, and return ``None``. + + .. versionchanged:: 3.10 + *func* can now be any callable object, not only a function. + + .. versionchanged:: 3.14 + Keyword arguments are now passed to *func*. + + + .. method:: after_cancel(id) + + Cancel a callback previously scheduled with :meth:`after` or + :meth:`after_idle`. + *id* must be an identifier returned by one of those methods; passing a + value that is not such an identifier raises :exc:`ValueError`. + If the callback has already run or been cancelled, this has no effect. + + .. versionchanged:: 3.7 + Passing ``None`` (or any false value) as *id* now raises + :exc:`ValueError`. + + + .. method:: after_idle(func, *args, **kw) + + Schedule the callable *func* to be called, with *args* and *kw* passed to + it, when the Tk main loop next becomes idle, that is, when it has no + other events to process. + Return an identifier that can be passed to :meth:`after_cancel` to cancel + the call. + + .. versionchanged:: 3.14 + Keyword arguments are now passed to *func*. + + + .. method:: after_info(id=None) + + If *id* is omitted, return a tuple of the identifiers of all callbacks + currently scheduled with :meth:`after` and :meth:`after_idle` for this + interpreter. + + If *id* is given, it must identify a callback that has not yet run or + been cancelled, and the return value is a tuple ``(script, type)``, where + *script* refers to the function to be called and *type* is either + ``'idle'`` or ``'timer'``. + A :exc:`TclError` is raised if *id* does not exist. + + .. versionadded:: 3.13 + + + .. method:: mainloop(n=0) + + Enter the Tk event loop, which processes events until all windows are + destroyed. + This is normally called once, on the root window, to run the application. + + .. method:: quit() + + Quit the Tcl interpreter, causing :meth:`mainloop` to return. + + .. method:: update() + + Enter the event loop until all pending events, including idle callbacks, + have been processed. + This brings the display up to date and handles any events that are + already queued, then returns. + + .. method:: update_idletasks() + + Enter the event loop until all pending idle callbacks have been called. + This updates the display of windows, for example after geometry changes, + but does not process events caused by the user. + + .. method:: waitvar(name) + :no-typesetting: + + .. method:: wait_variable(name) + + Wait until the Tcl variable *name* is modified, continuing to process + events in the meantime so that the application stays responsive. + *name* is usually a :class:`Variable` instance, such as an + :class:`IntVar` or :class:`StringVar`. + + :meth:`waitvar` is an alias of :meth:`!wait_variable`. + + .. method:: wait_window(window=None) + + Wait until *window* is destroyed, continuing to process events in the + meantime. + If *window* is omitted, this widget is used. + This is typically used to wait for the user to finish interacting with a + dialog box. + + .. method:: wait_visibility(window=None) + + Wait until the visibility state of *window* changes, for example when it + first appears on the screen, continuing to process events in the + meantime. + If *window* is omitted, this widget is used. + This is typically used to wait for a newly created window to become + visible before acting on it. + + The methods with the ``focus_`` prefix manage the keyboard focus. + + .. method:: focus_set() + :no-typesetting: + + .. method:: focus() + + Direct the keyboard input focus for this widget's display to this widget. + If the application does not currently have the input focus on this + widget's display, the widget is remembered as the focus window for its + top level, and the focus will be redirected to it the next time the + window manager gives the focus to the top level. + :meth:`focus` is an alias of :meth:`!focus_set`, + except on the :class:`Canvas` and + :class:`ttk.Treeview ` widgets, + which provide their own :meth:`!focus` method. + + .. method:: focus_force() + + Direct the keyboard input focus to this widget even if the application + does not currently have the input focus for the widget's display. + This method should be used sparingly, if at all; normally an application + should wait for the window manager to give it the focus rather than + claiming it. + + .. method:: focus_get() + + Return the widget that currently has the keyboard focus in the + application, or ``None`` if no widget in the application has the focus. + Use :meth:`focus_displayof` to work correctly with several displays. + + .. method:: focus_displayof() + + Return the widget that currently has the keyboard focus on the display + where this widget is located, or ``None`` if no widget in the application + has the focus on that display. + + .. method:: focus_lastfor() + + Return the most recent widget to have had the keyboard focus among all + the widgets in the same top level as this widget; this is the widget that + will receive the focus the next time the window manager gives the focus + to the top level. + If no widget in that top level has ever had the focus, or if the most + recent focus widget has been deleted, the top level itself is returned. + + .. method:: tk_focusFollowsMouse() + + Reconfigure Tk to use an implicit focus model in which the focus is set + to a widget whenever the mouse pointer enters it. + This cannot easily be disabled once enabled. + + .. method:: tk_focusNext() + + Return the next widget after this one in the keyboard traversal order, or + ``None`` if there is none. + The traversal order goes first to the next child, then recursively to the + children of that child, and then to the next sibling higher in the + stacking order. + A widget is skipped if its ``takefocus`` option is set to ``0``. + This method is used in the default bindings for the :kbd:`Tab` key. + + .. method:: tk_focusPrev() + + Return the previous widget before this one in the keyboard traversal + order, or ``None`` if there is none. + See :meth:`tk_focusNext` for how the order is defined. + This method is used in the default bindings for the :kbd:`Shift-Tab` key. + + The methods with the ``grab_`` prefix set and query the input grab, which + directs all input events to a single widget. + + .. method:: grab_set() + + Set a local grab on this widget. + A grab confines pointer events to this widget and its descendants: while + the pointer is outside the widget's subtree, button presses and releases + and pointer motion are reported to the grab widget, and windows outside + the subtree become insensitive until the grab is released. + A local grab affects only the grabbing application. + Any grab previously set by this application on the widget's display is + automatically released. + Setting a grab is the usual way to make a dialog modal: while the grab is + in effect the user cannot interact with the other windows of the + application. + + .. method:: grab_set_global() + + Set a global grab on this widget. + A global grab is like the local grab set by :meth:`grab_set`, but it + locks out all other applications on the screen, so that only this + widget's subtree is sensitive to pointer events, and it also grabs the + keyboard. + Use with caution: it is easy to render a display unusable with a global + grab, since other applications stop receiving events until it is + released. + + .. method:: grab_release() + + Release the grab on this widget if there is one; otherwise do nothing. + + .. method:: grab_current() + + Return the widget that currently holds the grab in this application for + this widget's display, or ``None`` if there is no such widget. + + .. method:: grab_status() + + Return ``None`` if no grab is currently set on this widget, ``"local"`` + if a local grab is set, or ``"global"`` if a global grab is set. + + The methods with the ``selection_`` prefix retrieve and manage the X + selection. + + .. method:: selection_clear(**kw) + + Clear the X selection, so that no window owns it anymore. + The selection to clear is given by the keyword argument *selection*, an + atom name such as ``'PRIMARY'`` or ``'CLIPBOARD'``; it defaults to + ``PRIMARY``. + The *displayof* keyword argument names a widget that determines the + display on which to operate, and defaults to this widget. + + This is overridden by the :class:`Entry`, :class:`Listbox` and + :class:`Spinbox` widgets, + where :meth:`!selection_clear` clears the widget's own selection instead. + + .. method:: selection_get(**kw) + + Return the contents of the current X selection. + The keyword argument *selection* names the selection and defaults to + ``PRIMARY``. + The keyword argument *type* specifies the form in which the data is to be + returned (the desired conversion target), an atom name such as + ``'STRING'`` or ``'FILE_NAME'``; it defaults to ``STRING``, except on + X11, where ``UTF8_STRING`` is tried first and ``STRING`` is used as a + fallback. + The *displayof* keyword argument names a widget that determines the + display from which to retrieve the selection, and defaults to this + widget. + + .. method:: selection_handle(command, **kw) + + Register *command* as a handler to supply the X selection owned by this + widget when another application requests it. + When the selection is retrieved, *command* is called with two arguments, + the starting character offset and the maximum number of characters to + return, and must return at most that many characters of the selection + starting at that offset; for very long selections it is called repeatedly + with increasing offsets. + The keyword argument *selection* names the selection (default + ``PRIMARY``) and the keyword argument *type* gives the form of the + selection that the handler supplies (such as ``'STRING'`` or + ``'FILE_NAME'``, default ``STRING``). + + .. method:: selection_own(**kw) + + Make this widget the owner of the X selection on its display. + The previous owner, if any, is notified that it has lost the selection. + The keyword argument *selection* names the selection and defaults to + ``PRIMARY``. + + .. method:: selection_own_get(**kw) + + Return the widget in this application that owns the X selection on the + display containing this widget, or ``None`` if no widget in this + application owns the selection. + The keyword argument *selection* names the selection and defaults to + ``PRIMARY``. + The *displayof* keyword argument names a widget that determines the + display to query, and defaults to this widget. + + The methods with the ``clipboard_`` prefix manage the clipboard. + + .. method:: clipboard_append(string, **kw) + + Append *string* to the Tk clipboard and claim ownership of the clipboard + on this widget's display. + Before appending, the clipboard should be emptied with + :meth:`clipboard_clear`; all appends should be completed before returning + to the event loop so that the clipboard is updated atomically. + The keyword argument *type* specifies the form of the data, an atom name + such as ``'STRING'`` or ``'FILE_NAME'`` (default ``STRING``), and the + keyword argument *format* specifies the representation used to transmit + it (default ``STRING``). + The *displayof* keyword argument names a widget that determines the + target display, and defaults to this widget. + The contents can be retrieved with :meth:`clipboard_get` or + :meth:`selection_get`. + + .. method:: clipboard_clear(**kw) + + Claim ownership of the clipboard on this widget's display and remove any + previous contents. + The *displayof* keyword argument names a widget that determines the + target display, and defaults to this widget. + + .. method:: clipboard_get(**kw) + + Retrieve data from the clipboard on this widget's display. + The keyword argument *type* specifies the form in which the data is to be + returned, an atom name such as ``'STRING'`` or ``'FILE_NAME'``; it + defaults to ``STRING``, except on X11, where ``UTF8_STRING`` is tried + first and ``STRING`` is used as a fallback. + The *displayof* keyword argument names a widget that determines the + display, and defaults to the root window of the application. + This is equivalent to ``selection_get(selection='CLIPBOARD')``. + + The methods with the ``option_`` prefix query and modify the Tk option + database. + + .. method:: option_add(pattern, value, priority=None) + + Add an option to the Tk option database that associates *value* with + *pattern*. + *pattern* consists of names and/or classes separated by asterisks or + dots, in the usual X format. + *priority* is an integer between 0 and 100, or one of the symbolic names + ``'widgetDefault'`` (20), ``'startupFile'`` (40), ``'userDefault'`` (60), + or ``'interactive'`` (80); it defaults to ``interactive``. + + .. method:: option_clear() + + Clear the Tk option database. + Default options from the :envvar:`!RESOURCE_MANAGER` property or the + :file:`.Xdefaults` file are reloaded automatically the next time an + option is added to or removed from the database. + + .. method:: option_get(name, className) + + Return the value of the option matching this widget under *name* and + *className* from the Tk option database, or an empty string if there is + no matching entry. + When several entries match, the one with the highest priority is + returned, and among entries of equal priority the most recently added + one. + + .. method:: option_readfile(fileName, priority=None) + + Read the file named *fileName*, which should have the standard format for + an X resource database such as :file:`.Xdefaults`, and add all the + options it specifies to the Tk option database. + *priority* is interpreted as for :meth:`option_add` and defaults to + ``interactive``. + + .. method:: bell(displayof=0) + + Ring the bell on the display for this widget, using the display's current + bell-related settings, and reset the screen saver for the screen. + If *displayof* is given as a widget, the bell is rung on that widget's + display instead. + + .. method:: tk_setPalette(background, /) + tk_setPalette(*args, **kw) + + Set a new color scheme for all Tk widget elements. + Existing widgets are updated and the option database is changed so that + future widgets use the new colors. + A single color argument is taken as the normal background color, from + which a complete palette is computed. + Alternatively, the arguments may be given as keyword *name*/*value* pairs + naming individual options in the option database. + The recognized option names are ``activeBackground``, + ``activeForeground``, ``background``, ``disabledForeground``, + ``foreground``, ``highlightBackground``, ``highlightColor``, + ``insertBackground``, ``selectColor``, ``selectBackground``, + ``selectForeground``, and ``troughColor``; reasonable defaults are + computed for any that are not specified. + + .. method:: tk_bisque() + + Restore the application's colors to the light brown (bisque) color scheme + used in Tk 3.6 and earlier versions. + Provided for backward compatibility. + + .. method:: tk_strictMotif(boolean=None) + + Query or set whether Tk's look and feel should strictly adhere to Motif. + A true *boolean* value enables strict Motif compliance (for example, no + color change when the mouse passes over a slider). + Return the resulting setting. + + The methods with the ``busy_`` prefix manage the busy state of a window, + which shows a busy cursor and ignores user input. + + .. method:: busy(**kw) + :no-typesetting: + + .. method:: busy_hold(**kw) + :no-typesetting: + + .. method:: tk_busy(**kw) + :no-typesetting: + + .. method:: tk_busy_hold(**kw) + + Make this widget appear busy. + A transparent window is placed in front of the widget, so that it and all + of its descendants in the widget hierarchy are blocked from pointer + events and display a busy cursor. + Normally :meth:`update` should be called immediately afterwards to ensure + that the hold operation is in effect before the application starts its + processing. + + The only supported configuration option is *cursor*, the cursor to be + displayed while the widget is busy; it may have any of the values + accepted by :meth:`!configure`. + + :meth:`busy_hold`, :meth:`busy` and :meth:`tk_busy` are aliases of + :meth:`!tk_busy_hold`. + + .. versionadded:: 3.13 + + + .. method:: busy_configure(cnf=None, **kw) + :no-typesetting: + + .. method:: busy_config(cnf=None, **kw) + :no-typesetting: + + .. method:: tk_busy_config(cnf=None, **kw) + :no-typesetting: + + .. method:: tk_busy_configure(cnf=None, **kw) + + Query or modify the configuration options of the busy window. + The widget must have been previously made busy by :meth:`tk_busy_hold`. + With no arguments, return a dictionary describing all of the available + options; if *cnf* is the name of an option, return a tuple describing + that one option. + Otherwise set the given options to the given values. + Options may have any of the values accepted by :meth:`tk_busy_hold`. + + The option database is referenced through the widget name or class. + For example, if a :class:`Frame` widget named ``frame`` is to be made + busy, the busy cursor can be specified for it by either of the calls:: + + w.option_add('*frame.busyCursor', 'gumby') + w.option_add('*Frame.BusyCursor', 'gumby') + + :meth:`busy_configure`, :meth:`busy_config` and :meth:`tk_busy_config` + are aliases of :meth:`!tk_busy_configure`. + + .. versionadded:: 3.13 + + + .. method:: busy_cget(option) + :no-typesetting: + + .. method:: tk_busy_cget(option) + + Return the current value of the busy configuration *option*. + The widget must have been previously made busy by :meth:`tk_busy_hold`, + and *option* may have any of the values accepted by that method. + + :meth:`busy_cget` is an alias of :meth:`!tk_busy_cget`. + + .. versionadded:: 3.13 + + + .. method:: busy_forget() + :no-typesetting: + + .. method:: tk_busy_forget() + + Make this widget no longer busy, releasing the resources (including the + transparent window) allocated when it was made busy. + User events will again be received by the widget. + These resources are also released when the widget is destroyed. + + :meth:`busy_forget` is an alias of :meth:`!tk_busy_forget`. + + .. versionadded:: 3.13 + + + .. method:: busy_status() + :no-typesetting: + + .. method:: tk_busy_status() + + Return ``True`` if the widget is currently busy, ``False`` otherwise. + + :meth:`busy_status` is an alias of :meth:`!tk_busy_status`. + + .. versionadded:: 3.13 + + + .. method:: busy_current(pattern=None) + :no-typesetting: + + .. method:: tk_busy_current(pattern=None) + + Return a list of widgets that are currently busy. + If *pattern* is given, only busy widgets whose path names match the + pattern are returned. + + :meth:`busy_current` is an alias of :meth:`!tk_busy_current`. + + .. versionadded:: 3.13 + + The methods with the ``winfo_`` prefix retrieve information about windows + managed by Tk. + + .. method:: winfo_atom(name, displayof=0) + + Return the integer identifier for the atom whose name is *name*, creating + a new atom if none exists. + If *displayof* is given, the atom is looked up on the display of that + window; otherwise it is looked up on the display of the application's + main window. + + .. method:: winfo_atomname(id, displayof=0) + + Return the textual name for the atom whose integer identifier is *id*. + This is the inverse of :meth:`winfo_atom`. + If *displayof* is given, the identifier is looked up on the display of + that window; otherwise it is looked up on the display of the + application's main window. + + .. method:: winfo_cells() + + Return the number of cells in the colormap for the widget. + + .. method:: winfo_children() + + Return a list containing the widgets that are children of the widget, in + stacking order from lowest to highest. + Toplevel windows are returned as children of their logical parents. + + .. method:: winfo_class() + + Return the class name of the widget. + + .. method:: winfo_colormapfull() + + Return ``True`` if the colormap for the widget is known to be full, + ``False`` otherwise. + + .. method:: winfo_containing(rootX, rootY, displayof=0) + + Return the widget containing the point given by *rootX* and *rootY*, or + ``None`` if no window in this application contains the point. + The coordinates are in screen units in the coordinate system of the root + window. + If *displayof* is given, the coordinates refer to the screen containing + that window; otherwise they refer to the screen of the application's main + window. + + .. method:: winfo_depth() + + Return the depth of the widget, that is, the number of bits per pixel. + + .. method:: winfo_exists() + + Return true if the widget exists, false otherwise. + + .. method:: winfo_fpixels(number) + + Return a floating-point value giving the number of pixels in the widget + corresponding to the screen distance *number* (for example, + ``"2.0c"`` or ``"1i"``). + The result may be fractional; for a rounded integer value use + :meth:`winfo_pixels`. + + .. method:: winfo_geometry() + + Return the geometry of the widget, in the form ``widthxheight+x+y``. + All dimensions are in pixels. + An offset can be negative; see :meth:`~Wm.geometry`. + + .. method:: winfo_height() + + Return the height of the widget in pixels. + When a window is first created its height is 1 pixel; it is eventually + changed by a geometry manager. + See also :meth:`winfo_reqheight`. + + .. method:: winfo_id() + + Return a low-level platform-specific identifier for the widget. + On Unix this is the X window identifier, and on Windows it is the window + handle. + + .. method:: winfo_interps(displayof=0) + + Return a tuple of the names of all Tcl interpreters currently registered + for a particular display. + If *displayof* is given, the return value refers to the display of that + window; otherwise it refers to the display of the application's main + window. + + .. method:: winfo_ismapped() + + Return true if the widget is currently mapped, false otherwise. + + .. method:: winfo_manager() + + Return the name of the geometry manager currently responsible for the + widget, or an empty string if it is not managed by any geometry manager. + + .. method:: winfo_name() + + Return the widget's name within its parent, as opposed to its full path + name. + + .. method:: winfo_parent() + + Return the path name of the widget's parent, or an empty string if the + widget is the main window of the application. + + .. method:: winfo_pathname(id, displayof=0) + + Return the path name of the window whose identifier is *id*. + If *displayof* is given, the identifier is looked up on the display of + that window; otherwise it is looked up on the display of the + application's main window. + + .. method:: winfo_pixels(number) + + Return the number of pixels in the widget corresponding to the screen + distance *number* (for example, ``"2.0c"`` or ``"1i"``). + The result is rounded to the nearest integer; for a fractional result use + :meth:`winfo_fpixels`. + + .. method:: winfo_pointerx() + + Return the pointer's *x* coordinate, in pixels, relative to the screen's + root window (or virtual root, if one is in use). + Return ``-1`` if the pointer is not on the same screen as the widget. + + .. method:: winfo_pointerxy() + + Return the pointer's coordinates as an ``(x, y)`` tuple, in pixels, + relative to the screen's root window (or virtual root, if one is in use). + Both coordinates are ``-1`` if the pointer is not on the same screen as + the widget. + + .. method:: winfo_pointery() + + Return the pointer's *y* coordinate, in pixels, relative to the screen's + root window (or virtual root, if one is in use). + Return ``-1`` if the pointer is not on the same screen as the widget. + + .. method:: winfo_reqheight() + + Return the widget's requested height in pixels. + This is the value used by the widget's geometry manager to compute its + geometry. + + .. method:: winfo_reqwidth() + + Return the widget's requested width in pixels. + This is the value used by the widget's geometry manager to compute its + geometry. + + .. method:: winfo_rgb(color) + + Return an ``(r, g, b)`` tuple of the red, green, and blue intensities, in + the range 0 to 65535, that correspond to *color* in the widget. + *color* may be specified in any of the forms acceptable for a color + option. + + .. method:: winfo_rootx() + + Return the *x* coordinate, in the root window of the screen, of the + upper-left corner of the widget's border (or of the widget itself if it + has no border). + + .. method:: winfo_rooty() + + Return the *y* coordinate, in the root window of the screen, of the + upper-left corner of the widget's border (or of the widget itself if it + has no border). + + .. method:: winfo_screen() + + Return the name of the screen associated with the widget, in the form + ``displayName.screenIndex``. + + .. method:: winfo_screencells() + + Return the number of cells in the default colormap for the widget's + screen. + + .. method:: winfo_screendepth() + + Return the depth of the root window of the widget's screen, that is, the + number of bits per pixel. + + .. method:: winfo_screenheight() + + Return the height of the widget's screen in pixels. + + .. method:: winfo_screenmmheight() + + Return the height of the widget's screen in millimeters. + + .. method:: winfo_screenmmwidth() + + Return the width of the widget's screen in millimeters. + + .. method:: winfo_screenvisual() + + Return the default visual class for the widget's screen, one of + ``"directcolor"``, ``"grayscale"``, ``"pseudocolor"``, ``"staticcolor"``, + ``"staticgray"``, or ``"truecolor"``. + + .. method:: winfo_screenwidth() + + Return the width of the widget's screen in pixels. + + .. method:: winfo_server() + + Return a string containing information about the server for the widget's + display. + The exact format of this string may vary from platform to platform. + + .. method:: winfo_toplevel() + + Return the top-of-hierarchy window containing the widget. + In standard Tk this is always a :class:`Toplevel` widget. + + .. method:: winfo_viewable() + + Return true if the widget and all of its ancestors up through the + nearest toplevel window are mapped, false otherwise. + + .. method:: winfo_visual() + + Return the visual class for the widget, one of ``"directcolor"``, + ``"grayscale"``, ``"pseudocolor"``, ``"staticcolor"``, ``"staticgray"``, + or ``"truecolor"``. + + .. method:: winfo_visualid() + + Return the X identifier for the visual for the widget. + + .. method:: winfo_visualsavailable(includeids=False) + + Return a list describing the visuals available for the widget's screen. + Each item consists of a visual class (see :meth:`winfo_visual`) followed + by an integer depth. + If *includeids* is true, the X identifier for the visual is also + included. + + .. method:: winfo_vrootheight() + + Return the height of the virtual root window associated with the widget + if there is one; otherwise return the height of the widget's screen. + + .. method:: winfo_vrootwidth() + + Return the width of the virtual root window associated with the widget if + there is one; otherwise return the width of the widget's screen. + + .. method:: winfo_vrootx() + + Return the *x* offset of the virtual root window associated with the + widget, relative to the root window of its screen. + This is normally zero or negative, and is ``0`` if there is no virtual + root window. + + .. method:: winfo_vrooty() + + Return the *y* offset of the virtual root window associated with the + widget, relative to the root window of its screen. + This is normally zero or negative, and is ``0`` if there is no virtual + root window. + + .. method:: winfo_width() + + Return the width of the widget in pixels. + When a window is first created its width is 1 pixel; it is eventually + changed by a geometry manager. + See also :meth:`winfo_reqwidth`. + + .. method:: winfo_x() + + Return the *x* coordinate, in the widget's parent, of the upper-left + corner of the widget's border (or of the widget itself if it has no + border). + + .. method:: winfo_y() + + Return the *y* coordinate, in the widget's parent, of the upper-left + corner of the widget's border (or of the widget itself if it has no + border). + + .. method:: info_patchlevel() + + Return the Tcl/Tk patch level as a named tuple with the same five fields + as :data:`sys.version_info`: *major*, *minor*, *micro*, *releaselevel* + and *serial*. + *releaselevel* is ``'alpha'``, ``'beta'`` or ``'final'``. + Converting it to a string gives the version in the usual Tcl/Tk notation, + for example ``'9.0.3'`` for a final release or ``'9.1b2'`` for a + pre-release. + + .. versionadded:: 3.11 + + + +.. class:: Wm() + + The :class:`!Wm` mixin provides access to the window manager, allowing an + application to control such things as the title, geometry and icon of a + top-level window, the way it is resized, and how it responds to window + manager protocols. + It is mixed into :class:`Tk` and :class:`Toplevel`, so its methods are + available on every top-level window. + Each method has two equivalent spellings: a short name and a + ``wm_``-prefixed name (for example, :meth:`title` and :meth:`wm_title`). + See also :ref:`tkinter-window-manager`. + + .. method:: wm_aspect(minNumer=None, minDenom=None, maxNumer=None, maxDenom=None) + :no-typesetting: + + .. method:: aspect(minNumer=None, minDenom=None, maxNumer=None, maxDenom=None) + + Constrain the aspect ratio (the ratio of width to height) of the window. + If all four arguments are given, the window manager keeps the ratio + between ``minNumer/minDenom`` and ``maxNumer/maxDenom``; passing empty + strings removes any existing restriction. + With no arguments, return a tuple of the four current values, or ``None`` + if no aspect restriction is in effect. + :meth:`wm_aspect` is an alias of :meth:`!aspect`. + + .. method:: wm_attributes(*args, return_python_dict=False, **kwargs) + :no-typesetting: + + .. method:: attributes(*args, return_python_dict=False, **kwargs) + + Query or set platform-specific attributes of the window. + With no arguments, return the platform-specific flags and their values; + pass *return_python_dict* as true to get them as a dictionary. + A single option name such as ``'alpha'`` returns the value of that + option, and options are set using keyword arguments (``alpha=0.5``). + + The available attributes differ by platform. + All platforms support: + + *alpha* + The window's opacity, from ``0.0`` (fully transparent) to ``1.0`` + (opaque). + Where transparency is unsupported the value stays at ``1.0``. + + *appearance* + Whether the window is rendered in dark mode on Windows and macOS: + ``'auto'``, ``'light'`` or ``'dark'`` (this has no effect on X11). + + *fullscreen* + Whether the window takes up the entire screen and has no borders. + + *topmost* + Whether the window is displayed above all other windows. + + Windows additionally supports: + + *disabled* + Whether the window is in a disabled state. + + *toolwindow* + Whether the window uses the tool window style. + + *transparentcolor* + The color that is made fully transparent, or an empty string for none. + + macOS additionally supports: + + *class* + Whether the underlying Aqua window is an ``nswindow`` or an + ``nspanel``; this can only be set before the window is created. + + *modified* + The modification state shown by the window's close button and proxy + icon. + + *notify* + Whether the application's dock icon bounces to request attention. + + *stylemask* + The style mask of the underlying Aqua window, given as a list of bit + names such as ``titled`` or ``resizable``. + + *tabbingid* + The identifier of the tab group that the window belongs to. + + *tabbingmode* + Whether the window may be opened as a tab: ``'auto'``, ``'preferred'`` + or ``'disallowed'``. + + *titlepath* + The path of the file represented by the window's proxy icon. + + *transparent* + Whether the content area is transparent and the window shadow is + turned off. + + X11 additionally supports: + + *type* + The window type, or a list of types in order of preference, that the + window manager should use to interpret the window, such as + ``'dialog'`` or ``'splash'``. + + *zoomed* + Whether the window is maximized. + + .. note:: + + Tk 8.6 added the *type* attribute, and Tk 9.0 added the *appearance*, + *class*, *stylemask*, *tabbingid* and *tabbingmode* attributes. + + On X11 changes are applied asynchronously, so a queried value may not yet + reflect the most recent request. + :meth:`wm_attributes` is an alias of :meth:`!attributes`. + + .. versionchanged:: 3.13 + A single attribute may now be queried by name without the leading + ``-``, and attributes may be set using keyword arguments. + The *return_python_dict* parameter was added. + + .. deprecated:: 3.13 + Setting an attribute by passing the option name (with a leading + ``-``) and its value as two positional arguments, as in + ``w.attributes('-alpha', 0.5)``, is deprecated; use keyword arguments + instead. + + + .. method:: wm_client(name=None) + :no-typesetting: + + .. method:: client(name=None) + + Store *name*, which should be the name of the host on which the + application is running, in the window's ``WM_CLIENT_MACHINE`` property + for use by the window or session manager. + An empty string deletes the property. + With no argument, return the last name set, or an empty string. + :meth:`wm_client` is an alias of :meth:`!client`. + + .. method:: wm_colormapwindows(*wlist) + :no-typesetting: + + .. method:: colormapwindows(*wlist) + + Manipulate the ``WM_COLORMAP_WINDOWS`` property, which tells the window + manager about windows that have private colormaps. + If *wlist* is given, overwrite the property with those windows (their + order is a priority order for installing colormaps). + With no arguments, return the list of windows currently named in the + property. + :meth:`wm_colormapwindows` is an alias of :meth:`!colormapwindows`. + + .. method:: wm_command(value=None) + :no-typesetting: + + .. method:: command(value=None) + + Store *value* in the window's ``WM_COMMAND`` property for use by the + window or session manager; it should be a list giving the words of the + command used to invoke the application. + An empty string deletes the property. + With no argument, return the last value set, or an empty string. + :meth:`wm_command` is an alias of :meth:`!command`. + + .. method:: wm_deiconify() + :no-typesetting: + + .. method:: deiconify() + + Display the window in normal (non-iconified) form by mapping it. + If the window has never been mapped, this ensures it appears de-iconified + when it is first mapped. + On Windows the window is also raised and given the focus. + :meth:`wm_deiconify` is an alias of :meth:`!deiconify`. + + .. method:: wm_focusmodel(model=None) + :no-typesetting: + + .. method:: focusmodel(model=None) + + Set or query the focus model for the window. + *model* is either ``'active'`` (the window claims the input focus for + itself or its descendants, even when the focus is in another application) + or ``'passive'`` (the window relies on the window manager to give it the + focus). + With no argument, return the current model. + The default is ``'passive'``, which is what the :meth:`!focus` command + assumes. + :meth:`wm_focusmodel` is an alias of :meth:`!focusmodel`. + + .. method:: wm_forget(window) + :no-typesetting: + + .. method:: forget(window) + + Unmap *window* from the screen so that it is no longer managed by the + window manager. + A :class:`Toplevel` is then treated like a :class:`Frame`, although its + ``-menu`` configuration is remembered and the menu reappears if the + widget is managed again. + :meth:`wm_forget` is an alias of :meth:`!forget`. + + Not to be confused with :meth:`Pack.forget`. + + .. versionadded:: 3.3 + + .. method:: wm_frame() + :no-typesetting: + + .. method:: frame() + + Return the platform-specific window identifier for the outermost + decorative frame containing the window, if the window manager has + reparented it into such a frame; otherwise return the identifier of the + window itself. + :meth:`wm_frame` is an alias of :meth:`!frame`. + + .. method:: wm_geometry(newGeometry=None) + :no-typesetting: + + .. method:: geometry(newGeometry=None) + + Set or query the geometry of the window. + *newGeometry* has the form ``=widthxheight+x+y``, where any of ``=``, + ``widthxheight`` and the ``+x+y`` position may be omitted. + *width* and *height* are in pixels (or grid units for a gridded window); + a position preceded by ``+`` is measured from the left or top edge of the + screen and one preceded by ``-`` from the right or bottom edge. + An offset can be negative, as in ``'200x100+-9+-8'``, when the window + edge is positioned beyond the corresponding screen edge. + An empty string cancels any user-specified geometry, letting the window + revert to its natural size. + With no argument, return the current geometry as a string of the form + ``'200x200+10+10'``. + :meth:`wm_geometry` is an alias of :meth:`!geometry`. + + .. method:: wm_grid(baseWidth=None, baseHeight=None, widthInc=None, heightInc=None) + :no-typesetting: + + .. method:: grid(baseWidth=None, baseHeight=None, widthInc=None, heightInc=None) + + Manage the window as a gridded window and define the relationship between + grid units and pixels. + *baseWidth* and *baseHeight* are the numbers of grid units for the + window's internally requested size, and *widthInc* and *heightInc* are + the pixel sizes of a horizontal and vertical grid unit. + Empty strings turn off gridded management. + With no arguments, return a tuple of the four current values, or ``None`` + if the window is not gridded. + :meth:`wm_grid` is an alias of :meth:`!grid`. + + Not to be confused with the grid geometry manager :meth:`Grid.grid`. + + .. method:: wm_group(pathName=None) + :no-typesetting: + + .. method:: group(pathName=None) + + Set or query the leader of a group of related windows. + *pathName* gives the path name of the group leader; the window manager + may, for example, unmap all windows in the group when the leader is + iconified. + An empty string removes the window from any group. + With no argument, return the path name of the current group leader, or an + empty string. + :meth:`wm_group` is an alias of :meth:`!group`. + + .. method:: wm_iconbitmap(bitmap=None, default=None) + :no-typesetting: + + .. method:: iconbitmap(bitmap=None, default=None) + + Set or query the bitmap used by the window manager for the window's icon. + *bitmap* names a bitmap in one of the standard forms accepted by Tk; an + empty string cancels the current icon bitmap. + With no argument, return the name of the current icon bitmap, or an empty + string. + On Windows the *default* argument names an icon (for example an ``.ico`` + file) applied to all top-level windows that have no icon of their own. + :meth:`wm_iconbitmap` is an alias of :meth:`!iconbitmap`. + + .. method:: wm_iconify() + :no-typesetting: + + .. method:: iconify() + + Iconify the window. + If the window has not yet been mapped for the first time, arrange for it + to appear in the iconified state when it is eventually mapped. + :meth:`wm_iconify` is an alias of :meth:`!iconify`. + + .. method:: wm_iconmask(bitmap=None) + :no-typesetting: + + .. method:: iconmask(bitmap=None) + + Set or query the bitmap used as a mask for the icon (see + :meth:`iconbitmap`). + Where the mask is zero no icon is displayed; where it is one, the + corresponding bits of the icon bitmap are shown. + An empty string cancels the current mask. + With no argument, return the name of the current icon mask, or an empty + string. + :meth:`wm_iconmask` is an alias of :meth:`!iconmask`. + + .. method:: wm_iconname(newName=None) + :no-typesetting: + + .. method:: iconname(newName=None) + + Set or query the name displayed by the window manager inside the window's + icon. + With no argument, return the current icon name, or an empty string if + none has been set (in which case the window manager normally displays the + window's title). + :meth:`wm_iconname` is an alias of :meth:`!iconname`. + + .. method:: wm_iconphoto(default=False, *images) + :no-typesetting: + + .. method:: iconphoto(default=False, *images) + + Set the titlebar icon for the window from one or more :class:`PhotoImage` + objects given in *images*. + Several images of different sizes (for example 16x16 and 32x32) may be + supplied so that the window manager can choose an appropriate one. + The image data is taken as a snapshot at the time of the call; later + changes to the images are not reflected. + If *default* is true, the icon is also applied to all top-level windows + created in the future. + On macOS only the first image is used. + :meth:`wm_iconphoto` is an alias of :meth:`!iconphoto`. + + .. versionadded:: 3.3 + + .. method:: wm_iconposition(x=None, y=None) + :no-typesetting: + + .. method:: iconposition(x=None, y=None) + + Set or query a hint to the window manager about where the window's icon + should be positioned. + Empty strings cancel an existing hint. + With no arguments, return a tuple of the two current values, or ``None`` + if no hint is in effect. + :meth:`wm_iconposition` is an alias of :meth:`!iconposition`. + + .. method:: wm_iconwindow(pathName=None) + :no-typesetting: + + .. method:: iconwindow(pathName=None) + + Set or query the window used as the icon for the window. + When the window is iconified, *pathName* is mapped to serve as its icon + and unmapped again when it is de-iconified. + An empty string cancels the association. + With no argument, return the path name of the current icon window, or an + empty string. + Not all window managers support icon windows, and the concept is + meaningless on non-X11 platforms. + :meth:`wm_iconwindow` is an alias of :meth:`!iconwindow`. + + .. method:: wm_manage(widget) + :no-typesetting: + + .. method:: manage(widget) + + Make *widget* a stand-alone top-level window, decorated by the window + manager with a title bar and so on. + Only :class:`Frame`, :class:`LabelFrame` and :class:`Toplevel` widgets + may be used (the :mod:`tkinter.ttk` versions are **not** accepted); + passing any other widget type raises an error. + :meth:`wm_manage` is an alias of :meth:`!manage`. + + .. versionadded:: 3.3 + + .. method:: wm_maxsize(width=None, height=None) + :no-typesetting: + + .. method:: maxsize(width=None, height=None) + + Set or query the maximum permissible dimensions of the window, in pixels + (or grid units for a gridded window). + The window manager restricts the window to be no larger than *width* and + *height*. + With no arguments, return a tuple of the current maximum width and + height. + The maximum size defaults to the size of the screen. + :meth:`wm_maxsize` is an alias of :meth:`!maxsize`. + + .. method:: wm_minsize(width=None, height=None) + :no-typesetting: + + .. method:: minsize(width=None, height=None) + + Set or query the minimum permissible dimensions of the window, in pixels + (or grid units for a gridded window). + The window manager restricts the window to be no smaller than *width* and + *height*. + With no arguments, return a tuple of the current minimum width and + height. + The minimum size defaults to one pixel in each dimension. + :meth:`wm_minsize` is an alias of :meth:`!minsize`. + + .. method:: wm_overrideredirect(boolean=None) + :no-typesetting: + + .. method:: overrideredirect(boolean=None) + + Set or query the override-redirect flag for the window. + When this flag is set, the window is ignored by the window manager: it is + not reparented into a decorative frame and the user cannot manipulate it + through the usual window manager controls. + With no argument, return a boolean indicating whether the flag is set, + or ``None`` if it has not been set. + The flag is reliably honored only when the window is first mapped or + remapped from the withdrawn state. + :meth:`wm_overrideredirect` is an alias of :meth:`!overrideredirect`. + + .. method:: wm_positionfrom(who=None) + :no-typesetting: + + .. method:: positionfrom(who=None) + + Set or query the source of the window's current position. + *who* is either ``'program'`` or ``'user'`` and indicates whether the + position was requested by the program or by the user; an empty string + cancels the current source. + With no argument, return the current source, or an empty string if none + has been set. + Tk automatically sets the source to ``'user'`` when :meth:`geometry` is + called, unless it has been set explicitly to ``'program'``. + :meth:`wm_positionfrom` is an alias of :meth:`!positionfrom`. + + .. method:: wm_protocol(name=None, func=None) + :no-typesetting: + + .. method:: protocol(name=None, func=None) + + Register *func* as the handler for the window manager protocol *name*, an + atom such as ``'WM_DELETE_WINDOW'``, ``'WM_SAVE_YOURSELF'`` or + ``'WM_TAKE_FOCUS'``; *func* is then called whenever the window manager + sends a message of that protocol. + Tk installs a default ``WM_DELETE_WINDOW`` handler that destroys the + window, which this method can replace. + If *func* is an empty string, the handler is removed. + With only *name*, return the name of its registered handler command, or + an empty string if none is set (the default ``WM_DELETE_WINDOW`` handler + is not reported); with no arguments, return a tuple of the protocols that + currently have handlers. + :meth:`wm_protocol` is an alias of :meth:`!protocol`. + + .. method:: wm_resizable(width=None, height=None) + :no-typesetting: + + .. method:: resizable(width=None, height=None) + + Control whether the user may interactively resize the window. + *width* and *height* are boolean values that determine whether the + window's width and height may be changed. + With no arguments, return a tuple of two ``0``/``1`` values indicating + whether each dimension is currently resizable. + By default a window is resizable in both dimensions. + :meth:`wm_resizable` is an alias of :meth:`!resizable`. + + .. method:: wm_sizefrom(who=None) + :no-typesetting: + + .. method:: sizefrom(who=None) + + Set or query the source of the window's current size. + *who* is either ``'program'`` or ``'user'`` and indicates whether the + size was requested by the program or by the user; an empty string cancels + the current source. + With no argument, return the current source, or an empty string if none + has been set. + :meth:`wm_sizefrom` is an alias of :meth:`!sizefrom`. + + .. method:: wm_state(newstate=None) + :no-typesetting: + + .. method:: state(newstate=None) + + Set or query the state of the window. + With no argument, return the current state: one of ``'normal'``, + ``'iconic'``, ``'withdrawn'``, ``'icon'`` or, on Windows and macOS only, + ``'zoomed'``. + ``'iconic'`` refers to a window that has been iconified, while ``'icon'`` + refers to a window serving as the icon for another window (see + :meth:`iconwindow`); the ``'icon'`` state cannot be set. + :meth:`wm_state` is an alias of :meth:`!state`. + + Not to be confused with :meth:`ttk.Widget.state + `. + + .. method:: wm_title(string=None) + :no-typesetting: + + .. method:: title(string=None) + + Set or query the title for the window, which the window manager should + display in the window's title bar. + With no argument, return the current title. + The title defaults to the window's name. + :meth:`wm_title` is an alias of :meth:`!title`. + + .. method:: wm_transient(master=None) + :no-typesetting: + + .. method:: transient(master=None) + + Mark the window as a transient window (such as a pull-down menu or + dialog) working on behalf of *master*, the path name of another top-level + window. + An empty string clears the transient status. + With no argument, return the path name of the current master, or an empty + string. + A transient window mirrors state changes in its master and may be + decorated differently by the window manager; it is an error to make a + window a transient of itself. + :meth:`wm_transient` is an alias of :meth:`!transient`. + + .. method:: wm_withdraw() + :no-typesetting: + + .. method:: withdraw() + + Withdraw the window from the screen, unmapping it and causing the window + manager to forget about it. + If the window has never been mapped, it is instead mapped in the + withdrawn state. + It is sometimes necessary to withdraw a window and then re-map it (for + example with :meth:`deiconify`) to make some window managers notice + changes to window attributes. + :meth:`wm_withdraw` is an alias of :meth:`!withdraw`. + + +.. class:: Pack() + + Geometry manager that arranges widgets by packing them against the sides of + their container. + The :class:`!Pack` mix-in is inherited by all widgets (through + :class:`Widget`) and provides the methods for managing a widget with the + *pack* geometry manager. + See also :ref:`tkinter-geometry-management`. + + .. note:: + + :class:`Pack`, :class:`Place` and :class:`Grid` all define the short + method names :meth:`!forget`, :meth:`!info`, :meth:`!slaves`, + :meth:`!content` and :meth:`!propagate`. + On a widget the bare names resolve to the *pack* manager's versions, + since :class:`Pack` and :class:`Misc` precede :class:`Place` and + :class:`Grid` in the method resolution order, + whatever manager actually manages the widget; + and :meth:`!configure`/:meth:`!config` configure the widget's options, + not its geometry. + Use the explicit ``pack_*``, ``grid_*`` and ``place_*`` methods + (and ``pack``, ``grid``, ``place`` for geometry configuration) + to act on a specific geometry manager. + + .. method:: configure(cnf={}, **kw) + :no-typesetting: + + .. method:: config(cnf={}, **kw) + :no-typesetting: + + .. method:: pack_configure(cnf={}, **kw) + pack(cnf={}, **kw) + + Pack the widget inside its container, positioning it relative to the + siblings already packed there. + The supported options are: + + *side* + Which side of the container to pack the widget against: ``'top'`` (the + default), ``'bottom'``, ``'left'`` or ``'right'``. + + *fill* + Whether to stretch the widget to fill its parcel: ``'none'`` (the + default), ``'x'``, ``'y'`` or ``'both'``. + + *expand* + Whether the widget should expand to consume any extra space in its + container (a boolean, default false). + + *anchor* + Where to position the widget in its parcel when the parcel is larger + than the widget: an anchor such as ``'n'`` or ``'sw'`` (default + ``'center'``). + + *ipadx*, *ipady* + Internal padding added on the left and right (*ipadx*) or top and + bottom (*ipady*) of the widget, as a screen distance (default ``0``). + + *padx*, *pady* + External padding left on the left and right (*padx*) or top and bottom + (*pady*) of the widget, as a screen distance or a pair of two + distances for the two sides (default ``0``). + + *after* + Pack the widget after the given widget in the packing order, using the + same container. + + *before* + Pack the widget before the given widget in the packing order, using + the same container. + + *in_* + The container in which to pack the widget; it defaults to the parent + widget. + + :meth:`pack`, :meth:`configure` and :meth:`config` are aliases of + :meth:`!pack_configure`. + + .. method:: forget() + :no-typesetting: + + .. method:: pack_forget() + + Unmap the widget and remove it from the packing order, forgetting its + packing options. + It can be packed again later with :meth:`pack_configure`. + :meth:`forget` is an alias of :meth:`!pack_forget`, + except on :class:`PanedWindow`, + :class:`ttk.Notebook ` and + :class:`ttk.PanedWindow `, + which provide their own :meth:`!forget` method. + + Not to be confused with :meth:`Wm.forget`. + + .. method:: info() + :no-typesetting: + + .. method:: pack_info() + + Return a dictionary of the widget's current packing options. + :meth:`info` is an alias of :meth:`!pack_info`. + + .. method:: propagate() + propagate(flag) + :no-typesetting: + + .. method:: pack_propagate() + pack_propagate(flag) + + Same as :meth:`Misc.pack_propagate`, treating this widget as a container: + enable or disable geometry propagation. + :meth:`propagate` is an alias of :meth:`!pack_propagate`. + + .. method:: slaves() + :no-typesetting: + + .. method:: pack_slaves() + + Same as :meth:`Misc.pack_slaves`: return the list of widgets packed in + this widget. + :meth:`slaves` is an alias of :meth:`!pack_slaves`. + + +.. class:: Place() + + Geometry manager that places widgets at explicit positions and sizes within + their container. + The :class:`!Place` mix-in is inherited by all widgets (through + :class:`Widget`). + See also :ref:`tkinter-geometry-management`. + + .. method:: configure(cnf={}, **kw) + :no-typesetting: + + .. method:: config(cnf={}, **kw) + :no-typesetting: + + .. method:: place_configure(cnf={}, **kw) + place(cnf={}, **kw) + + Place the widget inside its container at an absolute or relative + position. + The supported options are: + + *x*, *y* + The absolute horizontal and vertical position of the widget's anchor + point, as a screen distance (default ``0``). + + *relx*, *rely* + The horizontal and vertical position of the widget's anchor point as a + fraction of the container's width and height, where ``0.0`` is the + left or top edge and ``1.0`` is the right or bottom edge. + If both the absolute and the relative option are given, their values + are summed. + + *anchor* + Which point of the widget is placed at the given position: an anchor + such as ``'n'`` or ``'se'`` (default ``'nw'``). + + *width*, *height* + The absolute width and height of the widget, as a screen distance. + By default the widget's requested size is used. + + *relwidth*, *relheight* + The width and height of the widget as a fraction of the container's + width and height. + If both the absolute and the relative option are given, their values + are summed. + + *bordermode* + How the container's border affects placement: ``'inside'`` (the + default) measures the area inside the border, ``'outside'`` measures + the area including the border, and ``'ignore'`` uses the official X + area. + + *in_* + The container relative to which the widget is placed; it must be the + widget's parent or a descendant of the parent, and defaults to the + parent. + + :meth:`place`, :meth:`configure` and :meth:`config` are aliases of + :meth:`!place_configure`. + + .. method:: forget() + :no-typesetting: + + .. method:: place_forget() + + Unmap the widget and remove it from the placement, forgetting its place + options. + + .. method:: info() + :no-typesetting: + + .. method:: place_info() + + Return a dictionary of the widget's current place options. + + .. method:: slaves() + :no-typesetting: + + .. method:: place_slaves() + + Same as :meth:`Misc.place_slaves`: return the list of widgets placed in + this widget. + + +.. class:: Grid() + + Geometry manager that arranges widgets in a two-dimensional grid of rows and + columns within their container. + The :class:`!Grid` mix-in is inherited by all widgets (through + :class:`Widget`). + See also :ref:`tkinter-geometry-management`. + + .. method:: configure(cnf={}, **kw) + :no-typesetting: + + .. method:: config(cnf={}, **kw) + :no-typesetting: + + .. method:: grid_configure(cnf={}, **kw) + grid(cnf={}, **kw) + + Position the widget in a cell of its container's grid. + + Not to be confused with :meth:`Wm.grid`. + + The supported options are: + + *row*, *column* + The row and column of the cell to place the widget in, counting from + ``0``. + *column* defaults to the column after the previous widget placed in + the same :meth:`!grid_configure` call (or ``0``), and *row* defaults + to the next empty row. + + *rowspan*, *columnspan* + The number of rows and columns the widget should span (default ``1``). + + *sticky* + How to position or stretch the widget when its cell is larger than the + widget: a string containing zero or more of the characters ``'n'``, + ``'s'``, ``'e'`` and ``'w'``, naming the cell sides the widget sticks + to. + Specifying both ``'n'`` and ``'s'`` (or ``'e'`` and ``'w'``) stretches + the widget to fill the height (or width) of the cell. + The default is ``''``, which centers the widget at its requested size. + + *ipadx*, *ipady* + Internal padding added on the left and right (*ipadx*) or top and + bottom (*ipady*) of the widget, as a screen distance (default ``0``). + + *padx*, *pady* + External padding left on the left and right (*padx*) or top and bottom + (*pady*) of the widget, as a screen distance or a pair of two + distances for the two sides (default ``0``). + + *in_* + The container in whose grid to place the widget; it defaults to the + parent widget. + + :meth:`grid`, :meth:`configure` and :meth:`config` are aliases of + :meth:`!grid_configure`. + + .. method:: forget() + :no-typesetting: + + .. method:: grid_forget() + + Unmap the widget and remove it from the grid, forgetting its grid + options. + + .. method:: grid_remove() + + Unmap the widget and remove it from the grid, but remember its grid + options so that it is restored to the same cell if it is gridded again. + + .. method:: info() + :no-typesetting: + + .. method:: grid_info() + + Return a dictionary of the widget's current grid options. + + .. method:: bbox(column=None, row=None, col2=None, row2=None) + :no-typesetting: + + .. method:: grid_bbox(column=None, row=None, col2=None, row2=None) + + Same as :meth:`Misc.grid_bbox`. + :meth:`bbox` is an alias of :meth:`!grid_bbox`, + except on :class:`Canvas`, :class:`Listbox`, :class:`Spinbox`, + :class:`Text`, :class:`ttk.Entry ` and + :class:`ttk.Treeview `, + which provide their own :meth:`!bbox` method. + + .. method:: columnconfigure(index, cnf={}, **kw) + :no-typesetting: + + .. method:: grid_columnconfigure(index, cnf={}, **kw) + + Same as :meth:`Misc.grid_columnconfigure`: query or set the options (such + as *weight*, *minsize*, *pad* and *uniform*) of a grid column. + :meth:`columnconfigure` is an alias of :meth:`!grid_columnconfigure`. + + .. method:: rowconfigure(index, cnf={}, **kw) + :no-typesetting: + + .. method:: grid_rowconfigure(index, cnf={}, **kw) + + Same as :meth:`Misc.grid_rowconfigure`: query or set the options of a + grid row. + :meth:`rowconfigure` is an alias of :meth:`!grid_rowconfigure`. + + .. method:: location(x, y) + :no-typesetting: + + .. method:: grid_location(x, y) + + Same as :meth:`Misc.grid_location`: return the ``(column, row)`` of the + cell that covers the pixel at *x*, *y*. + :meth:`location` is an alias of :meth:`!grid_location`. + + .. method:: size() + :no-typesetting: + + .. method:: grid_size() + + Same as :meth:`Misc.grid_size`: return a ``(columns, rows)`` tuple giving + the size of the grid. + :meth:`size` is an alias of :meth:`!grid_size`, + except on the :class:`Listbox` widget, + which provides its own :meth:`!size` method. + + .. method:: propagate() + propagate(flag) + :no-typesetting: + + .. method:: grid_propagate() + grid_propagate(flag) + + Same as :meth:`Misc.grid_propagate`. + + .. method:: slaves(row=None, column=None) + :no-typesetting: + + .. method:: grid_slaves(row=None, column=None) + + Same as :meth:`Misc.grid_slaves`: return the widgets managed in the grid, + optionally restricted to a *row* and/or *column*. + + +.. class:: XView() + + Mix-in providing the horizontal-scrolling interface shared by widgets such + as :class:`Entry`, :class:`Canvas`, :class:`Listbox`, :class:`Text` and + :class:`Spinbox`. + A widget's :meth:`xview` method is registered as the *command* of a + horizontal :class:`Scrollbar`. + + .. method:: xview(*args) + + Query or change the horizontal position of the view. + With no arguments, return a tuple ``(first, last)`` of two fractions + between 0 and 1 giving the portion of the document that is currently + visible. + Otherwise the arguments are passed to the Tk ``xview`` widget command and + are usually generated by a scrollbar; :meth:`xview_moveto` and + :meth:`xview_scroll` provide a more convenient interface. + + .. method:: xview_moveto(fraction) + + Adjust the view so that *fraction* of the total width of the document is + off-screen to the left. + *fraction* is a number between 0 and 1. + + .. method:: xview_scroll(number, what) + + Shift the view left or right by *number* units. + *what* is either ``'units'`` or ``'pages'``; a negative *number* scrolls + left and a positive one scrolls right. + + +.. class:: YView() + + Mix-in providing the vertical-scrolling interface shared by widgets such as + :class:`Canvas`, :class:`Listbox` and :class:`Text`. + A widget's :meth:`yview` method is registered as the *command* of a vertical + :class:`Scrollbar`. + + .. method:: yview(*args) + + Query or change the vertical position of the view. + With no arguments, return a tuple ``(first, last)`` of two fractions + between 0 and 1 giving the portion of the document that is currently + visible. + Otherwise the arguments are passed to the Tk ``yview`` widget command, + usually generated by a scrollbar; :meth:`yview_moveto` and + :meth:`yview_scroll` provide a more convenient interface. + + .. method:: yview_moveto(fraction) + + Adjust the view so that *fraction* of the total height of the document is + off-screen above the top. + *fraction* is a number between 0 and 1. + + .. method:: yview_scroll(number, what) + + Shift the view up or down by *number* units. + *what* is either ``'units'`` or ``'pages'``; a negative *number* scrolls + up and a positive one scrolls down. + + +.. class:: BaseWidget(master, widgetName, cnf={}, kw={}, extra=()) + + Internal base class for all widgets. + It inherits from :class:`Misc` and adds the machinery that creates the + underlying Tk widget; application code normally uses :class:`Widget` or a + concrete widget class rather than instantiating :class:`!BaseWidget` + directly. + + .. method:: destroy() + + Destroy this widget and all of its children, removing the corresponding + Tk widgets and deleting the associated Tcl commands. + + +.. class:: Widget(master, widgetName, cnf={}, kw={}, extra=()) + + Internal base class for the standard widgets. + It combines :class:`BaseWidget` with the geometry-manager mix-ins + :class:`Pack`, :class:`Place` and :class:`Grid`, so that every widget can be + managed by any of the three geometry managers. + The concrete widget classes (:class:`Button`, :class:`Label`, and so on) + derive from :class:`!Widget`. + + +Toplevel widgets +^^^^^^^^^^^^^^^^ + +.. class:: Tk(screenName=None, baseName=None, className='Tk', useTk=True, sync=False, use=None) + + Construct a toplevel Tk widget, which is usually the main window of an + application, and initialize a Tcl interpreter for this widget. Each + instance has its own associated Tcl interpreter. + Inherits from :class:`Misc` and :class:`Wm`. + + To create a Tcl interpreter without initializing the Tk subsystem, use the + :func:`Tcl` factory function instead. + + The :class:`Tk` class is typically instantiated using all default values. + However, the following keyword arguments are currently recognized: + + *screenName* + When given (as a string), sets the :envvar:`DISPLAY` environment + variable. (X11 only) + *baseName* + Name of the profile file. By default, *baseName* is derived from the + program name (``sys.argv[0]``). + *className* + Name of the widget class. Used as a profile file and also as the name + with which Tcl is invoked (*argv0* in *interp*). + *useTk* + If ``True``, initialize the Tk subsystem. The :func:`tkinter.Tcl() ` + function sets this to ``False``. + *sync* + If ``True``, execute all X server commands synchronously, so that errors + are reported immediately. Can be used for debugging. (X11 only) + *use* + Specifies the *id* of the window in which to embed the application, + instead of it being created as an independent toplevel window. *id* must + be specified in the same way as the value for the -use option for + toplevel widgets (that is, it has a form like that returned by + :meth:`~Misc.winfo_id`). + + Note that on some platforms this will only work correctly if *id* refers + to a Tk frame or toplevel that has its -container option enabled. + + :class:`Tk` reads and interprets profile files, named + :file:`.{className}.tcl` and :file:`.{baseName}.tcl`, into the Tcl + interpreter and calls :func:`exec` on the contents of + :file:`.{className}.py` and :file:`.{baseName}.py`. The path for the + profile files is the :envvar:`HOME` environment variable or, if that + isn't defined, then :data:`os.curdir`. + + .. note:: + + On Windows, creating a Tcl interpreter (by instantiating :class:`Tk` or + calling :func:`Tcl`) sets the :envvar:`HOME` environment variable for + the process, if it is not already set, to ``%HOMEDRIVE%%HOMEPATH%`` (or + :envvar:`USERPROFILE`, or ``c:\``). This is done by Tcl and can affect + other code that reads :envvar:`HOME`. + + .. attribute:: tk + + The Tk application object created by instantiating :class:`Tk`. This + provides access to the Tcl interpreter. Each widget that is attached + the same instance of :class:`Tk` has the same value for its :attr:`tk` + attribute. + + .. attribute:: master + + The widget object that contains this widget. + For :class:`Tk`, the :attr:`!master` is :const:`None` because it is the + main window. + The terms *master* and *parent* are similar and sometimes used + interchangeably as argument names; however, calling + :meth:`~Misc.winfo_parent` returns a string of the widget name whereas + :attr:`!master` returns the object. + *parent*/*child* reflects the tree-like relationship while *master* (or + *container*)/*content* reflects the container structure. + + .. attribute:: children + + The immediate descendants of this widget as a :class:`dict` with the + child widget names as the keys and the child instance objects as the + values. + + .. method:: destroy() + + Destroy this and all descendant widgets and, for the main window, end the + connection to the underlying Tcl interpreter. + + .. method:: loadtk() + + Finish loading and initializing the Tk subsystem. + This is needed only when the interpreter was created without Tk (for + example through :func:`Tcl`); it is called automatically when *useTk* is + true. + + .. method:: readprofile(baseName, className) + + Read and source the user's profile files :file:`.{className}.tcl` and + :file:`.{baseName}.tcl` into the Tcl interpreter, and execute the + corresponding :file:`.{className}.py` and :file:`.{baseName}.py` files. + This is called during initialization; see the description of the + constructor above. + + .. method:: report_callback_exception(exc, val, tb) + + Report a callback exception. + This is called when an exception propagates out of a Tkinter callback; + *exc*, *val* and *tb* are the exception type, value and traceback as + returned by :func:`sys.exc_info`. + The default implementation prints a traceback to :data:`sys.stderr`. + It can be overridden to customize error handling, for example to display + the traceback in a dialog. + + +.. class:: Toplevel(master=None, cnf={}, **kw) + + A :class:`!Toplevel` widget is a top-level window, similar to a + :class:`Frame` except that its X parent is the root window of a screen + rather than its logical parent. + Its primary purpose is to serve as a container for dialog boxes and other + collections of widgets; its only visible features are its background and an + optional 3-D border. + Notable options include *menu*, which installs a :class:`Menu` as the + window's menubar. + Inherits from :class:`BaseWidget` and :class:`Wm`, so a toplevel is managed + by the window manager. + Refer to the Tk ``toplevel`` manual page for the full list of options. + + +Widget classes +^^^^^^^^^^^^^^ + +.. class:: Button(master=None, cnf={}, **kw) + + A :class:`!Button` widget displays a textual string, bitmap or image and + invokes a command when the user presses it (by clicking mouse button 1 over + the button or, when the button has focus, by pressing the space key). + Inherits from :class:`Widget`. + In addition to the standard widget options, a button accepts the options + documented in the Tk ``button`` manual page, such as *command* (the callback + invoked when the button is pressed), *textvariable*, *state* and *default*. + + .. method:: invoke() + + Invoke the command associated with the button, if there is one, and + return its result, or an empty string if no command is associated with + the button. + This is ignored if the button's state is ``disabled``. + + .. method:: flash() + + Flash the button by redisplaying it several times, alternating between + the active and normal colors. + At the end of the flash the button is left in the same normal or active + state as when the method was called. + This is ignored if the button's state is ``disabled``. + + +.. class:: Canvas(master=None, cnf={}, **kw) + + A :class:`!Canvas` widget implements structured graphics. + It displays any number of *items*, such as arcs, lines, ovals, polygons, + rectangles, text, bitmaps, images and embedded windows, which may be drawn, + moved, re-colored and bound to events. + Inherits from :class:`Widget`, :class:`XView` and :class:`YView`, so the + view can be scrolled horizontally and vertically with :meth:`~XView.xview` + and :meth:`~YView.yview`. + Refer to the Tk ``canvas`` manual page for the full list of widget and item + options. + + Each item has a unique integer *id*, assigned when it is created, and zero + or more string *tags*. + A tag is an arbitrary string that does not have the form of an integer; the + same tag may be shared by many items, which makes tags convenient for + grouping items. + The special tag ``'all'`` matches every item in the canvas, and + ``'current'`` matches the topmost item under the mouse pointer. + Most methods take a *tagOrId* argument that may be an integer id naming a + single item, or a tag naming zero or more items; as described in the Tk + ``canvas`` manual page, a tag may also be a logical expression of tags + combined with the operators ``&&``, ``||``, ``^``, ``!`` and parentheses. + When a method that operates on a single item is given a *tagOrId* matching + several items, it normally uses the lowest matching item in the display + list. + + The items are kept in a *display list* that determines drawing order: items + later in the list are drawn on top of earlier ones. + A newly created item is placed at the top of the list; the order can be + changed with :meth:`tag_raise` and :meth:`tag_lower`. + + .. method:: create_arc(*args, **kw) + create_bitmap(*args, **kw) + create_image(*args, **kw) + create_line(*args, **kw) + create_oval(*args, **kw) + create_polygon(*args, **kw) + create_rectangle(*args, **kw) + create_text(*args, **kw) + create_window(*args, **kw) + + Create a new item of the corresponding type and return its integer id. + Each method is called as ``create_TYPE(coord..., **options)``: the + leading positional arguments give the coordinates that define the item + (as separate numbers, as a single sequence of numbers, or as coordinate + pairs), and the keyword arguments set item-specific options. + Coordinates and screen distances may be given as numbers (interpreted as + pixels) or as strings with a unit suffix (``'m'``, ``'c'``, ``'i'`` or + ``'p'`` for millimetres, centimetres, inches or printer's points), but + are always stored and returned in pixels. + + The item types are: ``arc`` (an arc-shaped region that is a section of an + oval, defined by two diagonally opposite corners ``x1, y1, x2, y2`` of + the enclosing rectangle); ``bitmap`` (a two-color bitmap positioned at a + point ``x, y``); ``image`` (a Tk image positioned at a point ``x, y``); + ``line`` (a line or curve through the points ``x1, y1, ..., xn, yn``); + ``oval`` (a circle or ellipse inscribed in the rectangle + ``x1, y1, x2, y2``); ``polygon`` (a closed polygon through the points + ``x1, y1, ..., xn, yn``); ``rectangle`` (a rectangle with corners + ``x1, y1, x2, y2``); ``text`` (a string of text positioned at a point + ``x, y``); and ``window`` (a child widget embedded in the canvas at a + point ``x, y``, specified with the *window* option). + + Most item types accept a common set of *standard item options*, plus a + few options specific to each type. + Option names are passed as keyword arguments, without the leading + hyphen. + + The standard item options are: + + *fill* + The color used to fill the item's interior, or to draw a *line* + item or the characters of a *text* item. + An empty string (the default for all types except *line* and *text*) + leaves the item unfilled. + + *outline* + The color used to draw the item's outline. + An empty string draws no outline. + + *width* + The width of the outline, defaulting to ``1.0``. + Has no effect if *outline* is empty. + + *dash* + A dash pattern for the outline, given either as a sequence of + segment lengths in pixels or as a string of the characters + ``'.'``, ``','``, ``'-'``, ``'_'`` and space. + An empty pattern (the default) draws a solid outline. + + *dashoffset* + The starting offset in pixels into the *dash* pattern. + Ignored if there is no *dash* pattern. + + *stipple* + A bitmap used as a stipple pattern when filling the item. + Only well supported on X11. + + *outlinestipple* + A bitmap used as a stipple pattern when drawing the outline. + Has no effect if *outline* is empty. + + *offset*, *outlineoffset* + The offset of the fill and outline stipple patterns, given as + ``'x,y'`` or as a side such as ``'n'``, ``'se'`` or ``'center'``. + Stipple offsets are only supported on X11. + + *state* + Overrides the canvas state for this item; one of ``'normal'``, + ``'disabled'`` or ``'hidden'``. + + *tags* + A single tag or a sequence of tags to associate with the item, + replacing any existing tags. + + Many of these options have *active...* and *disabled...* variants + (such as *activefill*, *disabledfill*, *activewidth*, *disableddash*, + *activeoutline*, *disabledstipple*) that override the base option when + the item is the active item (under the mouse pointer) or is in the + disabled state. + + The following item types support additional options. + + For ``arc`` items: + + *start* + The start of the arc's angular range, in degrees measured + counter-clockwise from the 3-o'clock position. + + *extent* + The size of the angular range, in degrees counter-clockwise from + *start*. + + *style* + How the arc is drawn: ``'pieslice'`` (the default), ``'chord'`` or + ``'arc'``. + + For ``line`` items: + + *arrow* + Where to draw arrowheads: ``'none'`` (the default), ``'first'``, + ``'last'`` or ``'both'``. + + *arrowshape* + A sequence of three distances describing the shape of the + arrowheads. + + *capstyle* + How line ends are drawn: ``'butt'`` (the default), ``'projecting'`` + or ``'round'``. + + *joinstyle* + How line vertices are drawn: ``'round'`` (the default), ``'bevel'`` + or ``'miter'``. + + *smooth* + The smoothing method: a false value (the default) for no smoothing, + or ``'true'``/``'bezier'`` or ``'raw'`` to draw the line as a curve. + + *splinesteps* + The number of line segments approximating each spline when + *smooth* is enabled. + + For ``polygon`` items: + + *joinstyle*, *smooth*, *splinesteps* + As for ``line`` items, applied to the polygon's outline. + + For ``text`` items: + + *text* + The string to display; newline characters start new lines. + + *font* + The font used for the text. + + *justify* + How lines are justified: ``'left'`` (the default), ``'right'`` or + ``'center'``. + + *anchor* + How the text is positioned relative to its point, defaulting to + ``'center'``. + + *width* + The maximum line length; if non-zero, lines are wrapped at spaces. + + *angle* + How many degrees to rotate the text counter-clockwise about its + positioning point, from ``0.0`` to ``360.0`` (default ``0.0``). + + *underline* + The index of a character to underline, or ``-1`` for none. + + For ``bitmap`` items: + + *bitmap* + The bitmap to display. + + *anchor* + How the bitmap is positioned relative to its point. + + *background*, *foreground* + The colors used for the bitmap's ``0`` and ``1`` pixels; an empty + *background* makes the ``0`` pixels transparent. + Both have *active...* and *disabled...* variants, and *bitmap* has + *activebitmap* and *disabledbitmap* variants. + + For ``image`` items: + + *image* + The Tk image to display, previously created with the image + protocols. + + *anchor* + How the image is positioned relative to its point. + + Both options have *active...* and *disabled...* variants + (*activeimage*, *disabledimage*) used in the active and disabled + states. + + For ``window`` items: + + *window* + The widget to embed; it must be a child of the canvas or of one of + its ancestors, and may not be a top-level window. + + *anchor* + How the window is positioned relative to its point. + + *width*, *height* + The size to assign to the window; if zero (the default), the window + is given its requested size. + + ``oval`` and ``rectangle`` items have no type-specific options; they + use only the standard item options. + + .. note:: + + Tk 8.6 added the *angle* option and Tk 9.0 added the *underline* + option for ``text`` items. + + .. method:: coords(tagOrId) + coords(tagOrId, coordList, /) + coords(tagOrId, /, *coordList) + + Query or modify the coordinates of an item. + With only *tagOrId*, return a list of the floating-point coordinates of + the item given by *tagOrId* (the first matching item if it matches + several). + Given new coordinates, replace the coordinates of that item with them; + like the ``create_*`` methods, the coordinates may be given as separate + numbers, as a single sequence, or as coordinate pairs. + The returned coordinates are always in pixels, regardless of the units + used to specify them; for rectangles, ovals and arcs they are ordered + left, top, right, bottom. + + .. versionchanged:: 3.12 + The arguments are now flattened: the coordinates may be given as + separate arguments, as a single sequence, or grouped in pairs, like + the ``create_*`` methods. + + + .. method:: move(tagOrId, xAmount, yAmount, /) + + Move each of the items given by *tagOrId* in the canvas coordinate space + by adding *xAmount* to every x-coordinate and *yAmount* to every + y-coordinate of the item. + + .. method:: moveto(tagOrId, x='', y='') + + Move the items given by *tagOrId* so that the first coordinate pair (the + upper-left corner of the bounding box) of the lowest matching item is at + position (*x*, *y*). + *x* or *y* may be an empty string, in which case the corresponding + coordinate is unchanged. + All matching items keep their positions relative to each other. + + .. versionadded:: 3.8 + + + .. method:: scale(tagOrId, xOrigin, yOrigin, xScale, yScale, /) + + Rescale the coordinates of all items given by *tagOrId* in canvas + coordinate space. + Each x-coordinate is adjusted so that its distance from *xOrigin* changes + by a factor of *xScale*, and each y-coordinate so that its distance from + *yOrigin* changes by a factor of *yScale* (a factor of ``1.0`` leaves the + coordinate unchanged). + + .. method:: delete(*tagOrIds) + + Delete each of the items given by the *tagOrIds* arguments. + + .. method:: dchars(tagOrId, first, /) + dchars(tagOrId, first, last, /) + + Delete from each of the items given by *tagOrId* the characters (for text + items) or coordinates (for line and polygon items) in the range from + *first* to *last* inclusive; *last* defaults to *first*. + Items that do not support indexing ignore this operation. + + .. method:: insert(tagOrId, beforeThis, string, /) + + Insert *string* into each of the items given by *tagOrId* just before the + character or coordinate whose index is *beforeThis*. + For line and polygon items *string* must be a valid sequence of + coordinates. + + .. method:: itemcget(tagOrId, option) + + Return the current value of the configuration option *option* for the + item given by *tagOrId* (the lowest matching item if it matches several). + This is like :meth:`~Misc.cget` but applies to an individual item. + + .. method:: itemconfig(tagOrId, cnf=None, **kw) + :no-typesetting: + + .. method:: itemconfigure(tagOrId, cnf=None, **kw) + + Query or modify the configuration options of the items given by + *tagOrId*. + This mirrors :meth:`~Misc.configure`, except that it applies to + individual items rather than to the canvas as a whole. + With no options, it returns a dictionary describing the current options + of the first matching item; otherwise it sets the given options on every + matching item. + The legal options are those accepted by the corresponding ``create_*`` + method. + :meth:`itemconfig` is an alias of :meth:`!itemconfigure`. + + .. method:: type(tagOrId) + + Return the type of the item given by *tagOrId* (the first matching item + if it matches several), such as ``'rectangle'`` or ``'text'``, or + ``None`` if *tagOrId* does not match any item. + + .. method:: gettags(tagOrId, /) + + Return a tuple of the tags associated with the item given by *tagOrId* + (the first matching item in display-list order if it matches several). + Return an empty tuple if no item matches or the item has no tags. + + .. method:: dtag(tagOrId, /) + dtag(tagOrId, tagToDelete, /) + + Remove the tag *tagToDelete* (which defaults to *tagOrId*) from each of + the items given by *tagOrId*. + Items that do not have that tag are unaffected. + + .. method:: addtag(newtag, searchSpec, /, *args) + + Add the tag *newtag* to each item selected by the search specification + *searchSpec* (and any further *args*). + *searchSpec* is one of ``'above'``, ``'all'``, ``'below'``, + ``'closest'``, ``'enclosed'``, ``'overlapping'`` or ``'withtag'``; the + ``addtag_*`` methods below are convenient wrappers that supply each of + these forms. + + .. method:: addtag_above(newtag, tagOrId) + + Add the tag *newtag* to the item just above (after) *tagOrId* in the + display list. + + .. method:: addtag_all(newtag) + + Add the tag *newtag* to all items in the canvas. + + .. method:: addtag_below(newtag, tagOrId) + + Add the tag *newtag* to the item just below (before) *tagOrId* in the + display list. + + .. method:: addtag_closest(newtag, x, y, halo=None, start=None) + + Add the tag *newtag* to the item closest to the point (*x*, *y*). + If *halo* is given, any item within that distance of the point is treated + as overlapping it. + If *start* is given (a tag or id), select the topmost closest item that + lies below *start* in the display list, which can be used to step through + all the closest items. + + .. method:: addtag_enclosed(newtag, x1, y1, x2, y2) + + Add the tag *newtag* to every item completely enclosed within the + rectangle (*x1*, *y1*, *x2*, *y2*), where *x1* <= *x2* and *y1* <= *y2*. + + .. method:: addtag_overlapping(newtag, x1, y1, x2, y2) + + Add the tag *newtag* to every item that overlaps or is enclosed within + the rectangle (*x1*, *y1*, *x2*, *y2*), where *x1* <= *x2* and *y1* <= + *y2*. + + .. method:: addtag_withtag(newtag, tagOrId) + + Add the tag *newtag* to every item given by *tagOrId*. + + .. method:: find(searchSpec, /, *args) + + Return a tuple of the ids of all items selected by the search + specification *searchSpec* (and any further *args*), in stacking order + with the lowest item first. + The search specification has any of the forms accepted by :meth:`addtag`. + The ``find_*`` methods below are more convenient wrappers around it. + + .. method:: find_above(tagOrId) + + Return a tuple containing the id of the item just above *tagOrId* in the + display list. + + .. method:: find_all() + + Return a tuple of the ids of all items in the canvas, in stacking order. + + .. method:: find_below(tagOrId) + + Return a tuple containing the id of the item just below *tagOrId* in the + display list. + + .. method:: find_closest(x, y, halo=None, start=None) + + Return a tuple containing the id of the item closest to the point (*x*, + *y*). + *halo* and *start* are interpreted as for :meth:`addtag_closest`. + + .. method:: find_enclosed(x1, y1, x2, y2) + + Return a tuple of the ids of all items completely enclosed within the + rectangle (*x1*, *y1*, *x2*, *y2*). + + .. method:: find_overlapping(x1, y1, x2, y2) + + Return a tuple of the ids of all items that overlap or are enclosed + within the rectangle (*x1*, *y1*, *x2*, *y2*). + + .. method:: find_withtag(tagOrId) + + Return a tuple of the ids of all items given by *tagOrId*. + + .. method:: lift(tagOrId, aboveThis=None, /) + :no-typesetting: + + .. method:: tkraise(tagOrId, aboveThis=None, /) + :no-typesetting: + + .. method:: tag_raise(tagOrId, aboveThis=None, /) + + Move all items given by *tagOrId* to a new position in the display list + just above the item given by *aboveThis*, or to the top of the display + list if *aboveThis* is omitted. + When several items are moved their relative order is preserved. + This has no effect on embedded window items, whose stacking order is + controlled by :meth:`Misc.tkraise` and :meth:`Misc.lower` instead. + :meth:`lift` and :meth:`tkraise` are aliases of :meth:`!tag_raise`. + + .. method:: lower(tagOrId, belowThis=None, /) + :no-typesetting: + + .. method:: tag_lower(tagOrId, belowThis=None, /) + + Move all items given by *tagOrId* to a new position in the display list + just below the item given by *belowThis*, or to the bottom of the display + list if *belowThis* is omitted. + When several items are moved their relative order is preserved. + This has no effect on embedded window items. + :meth:`lower` is an alias of :meth:`!tag_lower`. + + .. note:: + + On a :class:`Canvas`, :meth:`tkraise`/:meth:`lift` and :meth:`lower` + restack canvas items, + shadowing the inherited :meth:`Misc.tkraise`/:meth:`Misc.lift` and + :meth:`Misc.lower` methods that restack the widget itself, + which are therefore not available. + + .. method:: tag_bind(tagOrId, sequence=None, func=None, add=None) + + Bind the callback *func* to the event *sequence* for all items given by + *tagOrId*, so that *func* is invoked whenever that event occurs for one + of the items. + This is like :meth:`Widget.bind ` but operates on canvas items + rather than on whole widgets; only mouse, keyboard and virtual events may + be bound. + Mouse events are directed to the current item and keyboard events to the + focus item (see :meth:`focus`). + If *add* is true the new binding is added to any existing bindings for + the same sequence, rather than replacing them. + Return the identifier of the bound function, which can be passed to + :meth:`tag_unbind`. + + .. method:: tag_unbind(tagOrId, sequence, funcid=None) + + Remove for all items given by *tagOrId* the binding for the event + *sequence*. + If *funcid* is given, only that callback (as returned by + :meth:`tag_bind`) is unbound and deregistered. + + .. versionchanged:: 3.13 + If *funcid* is given, only that callback is unbound. + + + .. method:: bbox(tagOrId, /, *tagOrIds) + + Return a 4-tuple ``(x1, y1, x2, y2)`` giving an approximate bounding box, + in pixels, that encloses all the items given by *tagOrId* and any further + *tagOrIds*. + The result may overestimate the true bounding box by a few pixels. + Return ``None`` if no item matches or the matching items have nothing to + display. + + This shadows the inherited :meth:`!Misc.bbox`; + use :meth:`~Misc.grid_bbox` for the grid bounding box. + + .. method:: canvasx(screenx, gridspacing=None) + + Given a window x-coordinate *screenx*, return the canvas x-coordinate + displayed at that location. + If *gridspacing* is given, the result is rounded to the nearest multiple + of *gridspacing* units. + + .. method:: canvasy(screeny, gridspacing=None) + + Given a window y-coordinate *screeny*, return the canvas y-coordinate + displayed at that location. + If *gridspacing* is given, the result is rounded to the nearest multiple + of *gridspacing* units. + + .. method:: focus() + focus(tagOrId, /) + + With *tagOrId*, set the keyboard focus for the canvas to the first item + given by *tagOrId* that supports the insertion cursor; the focus is left + unchanged if no such item exists. + If *tagOrId* is an empty string, reset the focus so that no item has it. + With no argument, return the id of the item that currently has the focus, + or an empty string if none does. + An item only displays the insertion cursor when both it is the focus item + and its canvas has the input focus. + + This shadows the inherited :meth:`!Misc.focus`; + use :meth:`~Misc.focus_set` to focus the widget itself. + + .. method:: icursor(tagOrId, index, /) + + Set the insertion cursor of the items given by *tagOrId* to just before + the character given by *index*. + Items that do not support an insertion cursor are unaffected. + The cursor is only displayed when the item has the focus, but its + position may be set at any time. + + .. method:: index(tagOrId, index, /) + + Return as an integer the numerical index within *tagOrId* corresponding + to *index*, which is a textual description of a position (for text items + an index into the characters, for line and polygon items an index into + the coordinates). + If *tagOrId* matches several items, the first one that supports indexing + is used. + + .. method:: select_adjust(tagOrId, index) + + Adjust the end of the selection in *tagOrId* nearest to *index* so that + it is at *index*, and make the other end the anchor point for future + :meth:`select_to` calls. + If the selection is not currently in *tagOrId*, this behaves like + :meth:`select_to`. + + .. method:: select_clear() + + Clear the selection if it is in this canvas; otherwise do nothing. + + .. method:: select_from(tagOrId, index) + + Set the selection anchor point to just before the character given by + *index* in the item given by *tagOrId*. + This does not change the selection itself; it sets the fixed end for + future :meth:`select_to` calls. + + .. method:: select_item() + + Return the id of the item that holds the selection, or ``None`` if the + selection is not in this canvas. + Unlike :meth:`find` and the ``find_*`` methods, this returns the id as a + string rather than an integer. + + .. method:: select_to(tagOrId, index) + + Set the selection to the characters of *tagOrId* between the selection + anchor point and *index*, inclusive of *index*. + The anchor point is the one set by the most recent :meth:`select_adjust` + or :meth:`select_from` call. + + .. method:: scan_mark(x, y) + + Record *x*, *y* and the current view, for use with later + :meth:`scan_dragto` calls. + This is typically bound to a mouse button press in the widget. + + .. method:: scan_dragto(x, y, gain=10) + + Scroll the canvas by *gain* times the difference between *x*, *y* and the + coordinates passed to the last :meth:`scan_mark` call. + This is typically bound to mouse motion events in the widget, producing + the effect of dragging the canvas at high speed through its window. + + .. method:: postscript(cnf={}, **kw) + + Generate a PostScript (Encapsulated PostScript, version 3.0) + representation of part or all of the canvas. + If the *file* or *channel* option is given, the PostScript is written + there and an empty string is returned; otherwise it is returned as a + string. + By default only the area currently visible in the window is generated, so + it is usually necessary either to call :meth:`~Misc.update` first or to + use the *width* and *height* options. + Supported options include *colormap*, *colormode*, *file*, *fontmap*, + *height*, *pageanchor*, *pageheight*, *pagewidth*, *pagex*, *pagey*, + *rotate*, *width*, *x* and *y*. + + +.. class:: Checkbutton(master=None, cnf={}, **kw) + + A :class:`!Checkbutton` widget displays a textual string, bitmap or image + together with a square indicator, and toggles a boolean selection when + pressed. + It has all the behavior of a simple button and, in addition, can be + selected: when selected the indicator is drawn with a check mark and the + associated variable is set to the ``onvalue``, and when deselected the + indicator is drawn empty and the variable is set to the ``offvalue``. + Inherits from :class:`Widget`. + In addition to the standard widget options, a checkbutton accepts the + options documented in the Tk ``checkbutton`` manual page, such as + *variable*, *onvalue*, *offvalue* and *command*. + + .. method:: invoke() + + Do just what would happen if the user pressed the checkbutton with the + mouse: toggle the selection state of the button and invoke the associated + command, if there is one. + Return the result of the command, or an empty string if no command is + associated with the checkbutton. + This is ignored if the checkbutton's state is ``disabled``. + + .. method:: select() + + Select the checkbutton and set the associated variable to its + ``onvalue``. + + .. method:: deselect() + + Deselect the checkbutton and set the associated variable to its + ``offvalue``. + + .. method:: toggle() + + Toggle the selection state of the button, redisplaying it and modifying + its associated variable to reflect the new state. + + .. method:: flash() + + Flash the checkbutton by redisplaying it several times, alternating + between the active and normal colors. + At the end of the flash the checkbutton is left in the same normal or + active state as when the method was called. + This is ignored if the checkbutton's state is ``disabled``. + + +.. class:: Entry(master=None, cnf={}, **kw) + + An :class:`!Entry` widget displays a single line of text and lets the user + edit it. + Inherits from :class:`Widget` and :class:`XView`; since entries can hold + strings too long to fit in the window, they support horizontal scrolling + through :meth:`~XView.xview`. + + In addition to the standard widget options, an entry accepts the options + documented in the Tk ``entry`` manual page. + Notable ones are *textvariable* (the name of a variable kept in sync with + the entry's contents), *show* (if set, each character is displayed as the + given character rather than its true value, useful for password entry), + *validate* and *validatecommand* (which together let a callback accept or + reject edits), and *state* (one of ``'normal'``, ``'disabled'`` or + ``'readonly'``). + + Many of the methods below take an *index* argument that selects a character + in the entry's string. + As described in the Tk ``entry`` manual page, *index* may be a number + (counting from 0), ``'insert'`` (the character just after the insertion + cursor), ``'end'`` (just after the last character), ``'anchor'`` (the + selection anchor point), ``'sel.first'`` and ``'sel.last'`` (the ends of the + selection), or ``@x`` (the character covering pixel x-coordinate *x* in the + window). + Out-of-range indices are rounded to the nearest legal value. + + .. method:: delete(first, last=None) + + Delete the characters from index *first* up to but not including index + *last*. + If *last* is omitted, only the single character at *first* is deleted. + + .. method:: get() + + Return the entry's current string. + + .. method:: insert(index, string) + + Insert *string* just before the character given by *index*. + + .. method:: icursor(index) + + Arrange for the insertion cursor to be displayed just before the + character given by *index*. + + .. method:: index(index) + + Return the numerical index corresponding to *index*. + + .. method:: select_adjust(index) + :no-typesetting: + + .. method:: selection_adjust(index) + + Locate the end of the selection nearest to the character given by + *index*, and adjust that end to be at *index* (including but not going + beyond it); the other end becomes the anchor point for future + :meth:`selection_to` calls. + If there is no selection in the entry, a new one is created between + *index* and the most recent anchor point, inclusive. + :meth:`select_adjust` is an alias of :meth:`!selection_adjust`. + + .. method:: select_clear() + :no-typesetting: + + .. method:: selection_clear() + + Clear the selection if it is currently in this widget. + If the selection is not in this widget the method has no effect. + :meth:`select_clear` is an alias of :meth:`!selection_clear`. + + .. note:: + + This shadows the inherited :meth:`Misc.selection_clear`, + which clears the X selection; + that method is not available on an :class:`Entry`. + + .. method:: select_from(index) + :no-typesetting: + + .. method:: selection_from(index) + + Set the selection anchor point to just before the character given by + *index*, without changing the selection. + :meth:`select_from` is an alias of :meth:`!selection_from`. + + .. method:: select_present() + :no-typesetting: + + .. method:: selection_present() + + Return ``True`` if there are characters selected in the entry, ``False`` + otherwise. + :meth:`select_present` is an alias of :meth:`!selection_present`. + + .. method:: select_range(start, end) + :no-typesetting: + + .. method:: selection_range(start, end) + + Set the selection to include the characters starting with the one indexed + by *start* and ending with the one just before *end*. + If *end* refers to the same character as *start* or an earlier one, the + selection is cleared. + :meth:`select_range` is an alias of :meth:`!selection_range`. + + .. method:: select_to(index) + :no-typesetting: + + .. method:: selection_to(index) + + Set the selection between the anchor point and *index*: if *index* is + before the anchor point, the selection runs from *index* up to but not + including the anchor; if *index* is after it, from the anchor up to but + not including *index*; if they coincide, nothing happens. + The anchor point is the one set by the most recent :meth:`selection_from` + or :meth:`selection_adjust` call. + If there is no selection in the entry, a new one is created using the + most recent anchor point. + :meth:`select_to` is an alias of :meth:`!selection_to`. + + .. method:: scan_mark(x) + + Record *x* and the current view in the entry window, for use with later + :meth:`scan_dragto` calls. + Typically associated with a mouse button press in the widget. + + .. method:: scan_dragto(x) + + Compute the difference between *x* and the *x* given to the last + :meth:`scan_mark` call, and adjust the view left or right by 10 times + that difference. + Typically associated with mouse motion events, to produce the effect of + dragging the entry at high speed through the window. + + +.. class:: Frame(master=None, cnf={}, **kw) + + A :class:`!Frame` widget is a simple container. + Its primary purpose is to act as a spacer or container for complex window + layouts; its only features are its background and an optional 3-D border to + make the frame appear raised or sunken. + Inherits from :class:`Widget`. + Refer to the Tk ``frame`` manual page for the full list of options. + + +.. class:: Label(master=None, cnf={}, **kw) + + A :class:`!Label` widget displays a non-interactive textual string, bitmap + or image. + The displayed text is set with the *text* option or linked to a variable + through *textvariable*, and an image can be shown using the *image* option. + Text must all be in a single font but may occupy multiple lines, and one + character may be underlined with the *underline* option. + Inherits from :class:`Widget`. + Refer to the Tk ``label`` manual page for the full list of options. + + +.. class:: LabelFrame(master=None, cnf={}, **kw) + + A :class:`!LabelFrame` widget is a container that has the features of a + :class:`Frame` plus the ability to display a label. + The label text is set with the *text* option and positioned with + *labelanchor*, or an arbitrary widget may be used as the label by giving it + as the *labelwidget* option. + Inherits from :class:`Widget`. + Refer to the Tk ``labelframe`` manual page for the full list of options. + + +.. class:: Listbox(master=None, cnf={}, **kw) + + A :class:`!Listbox` widget displays a list of single-line text items, one + per line, of which the user can select one or more. + The way the selection behaves is governed by the *selectmode* option, which + is one of ``browse`` (the default; at most one item, which may be dragged + with the mouse), ``single`` (at most one item), ``multiple`` (any number of + items, toggled individually), or ``extended`` (any number of items, + including discontiguous ranges, selected by clicking and dragging). + Inherits from :class:`Widget`, :class:`XView` and :class:`YView`, so the + view can be scrolled horizontally and vertically with :meth:`~XView.xview` + and :meth:`~YView.yview`. + Refer to the Tk ``listbox`` manual page for the full list of options. + + Many of the methods take an *index* argument identifying a particular item. + As described in the Tk ``listbox`` manual page, *index* may be a numeric + index (counting from 0 at the top), ``'active'`` (the item with the location + cursor, set with :meth:`activate`), ``'anchor'`` (the selection anchor, set + with :meth:`selection_anchor`), ``'end'`` (the last item, or for + :meth:`index` and :meth:`insert` the position just after it), or ``@x,y`` + (the item covering pixel coordinates *x*, *y* in the listbox window). + Arguments named *first* and *last* are indices of the same forms. + + .. method:: insert(index, *elements) + + Insert the given *elements* as new items just before the item given by + *index*. + If *index* is ``'end'``, the new items are appended to the end of the + list. + + .. method:: delete(first, last=None) + + Delete the items in the range from *first* to *last* inclusive. + If *last* is omitted, it defaults to *first*, so that a single item is + deleted. + + .. method:: get(first, last=None) + + If *last* is omitted, return the contents of the item given by *first*, + or an empty string if *first* refers to a non-existent item. + If *last* is given, return a tuple of all the items in the range from + *first* to *last* inclusive. + + .. method:: size() + + Return the total number of items in the listbox. + + This shadows the inherited :meth:`!Misc.size`; + use :meth:`~Misc.grid_size` for the grid size. + + .. method:: index(index) + + Return the integer index value corresponding to *index*, or ``None`` if + *index* is out of range. + If *index* is ``'end'``, the result is a count of the number of items in + the listbox (not the index of the last item). + + .. method:: bbox(index) + + Return a tuple ``(x, y, width, height)`` describing the bounding box, in + pixels relative to the widget, of the text of the item given by *index*. + Return ``None`` if no part of that item is visible on the screen, or if + *index* refers to a non-existent item; if the item is only partly + visible, the result still gives the full area of the item, including the + parts that are not visible. + + This shadows the inherited :meth:`!Misc.bbox`; + use :meth:`~Misc.grid_bbox` for the grid bounding box. + + .. method:: nearest(y) + + Given a y-coordinate within the listbox window, return the index of the + visible item nearest to that y-coordinate. + + .. method:: see(index) + + Adjust the view so that the item given by *index* is visible. + If the item is already visible the method has no effect; if it is near an + edge of the window the listbox scrolls just enough to bring it into view + at that edge, otherwise the listbox scrolls to center the item. + + .. method:: activate(index) + + Set the active item to the one given by *index*. + If *index* is outside the range of items, the closest item is activated + instead. + The active item is drawn as specified by the *activestyle* option when + the widget has the input focus, and its index may be retrieved with the + ``'active'`` index. + + .. method:: curselection() + + Return a tuple containing the numerical indices of all of the items that + are currently selected, or an empty tuple if no items are selected. + + .. method:: select_anchor(index) + :no-typesetting: + + .. method:: selection_anchor(index) + + Set the selection anchor to the item given by *index*. + If *index* refers to a non-existent item, the closest item is used. + The selection anchor is the end of the selection that is fixed while + dragging out a selection with the mouse, and may afterwards be referred + to with the ``'anchor'`` index. + :meth:`select_anchor` is an alias of :meth:`!selection_anchor`. + + .. method:: select_clear(first, last=None) + :no-typesetting: + + .. method:: selection_clear(first, last=None) + + Deselect any of the items in the range from *first* to *last* inclusive + that are selected. + The selection state of items outside this range is not changed. + :meth:`select_clear` is an alias of :meth:`!selection_clear`. + + .. note:: + + This shadows the inherited :meth:`Misc.selection_clear`, + which clears the X selection; + that method is not available on a :class:`Listbox`. + + .. method:: select_includes(index) + :no-typesetting: + + .. method:: selection_includes(index) + + Return ``True`` if the item given by *index* is currently selected, + ``False`` otherwise. + :meth:`select_includes` is an alias of :meth:`!selection_includes`. + + .. method:: select_set(first, last=None) + :no-typesetting: + + .. method:: selection_set(first, last=None) + + Select all of the items in the range from *first* to *last* inclusive, + without affecting the selection state of items outside that range. + :meth:`select_set` is an alias of :meth:`!selection_set`. + + .. method:: itemcget(index, option) + + Return the current value of the configuration option *option* for the + item given by *index*. + + .. method:: itemconfig(index, cnf=None, **kw) + :no-typesetting: + + .. method:: itemconfigure(index, cnf=None, **kw) + + Query or modify the configuration options of the item given by *index*. + This mirrors :meth:`~Misc.configure`, except that it applies to an + individual item rather than to the listbox as a whole. + With no options, it returns a dictionary describing the current options + of the item; otherwise it sets the given options. + The supported item options are *background*, *foreground*, + *selectbackground* and *selectforeground*. + :meth:`itemconfig` is an alias of :meth:`!itemconfigure`. + + .. method:: scan_mark(x, y) + + Record *x*, *y* and the current view, for use with later + :meth:`scan_dragto` calls. + This is typically bound to a mouse button press in the widget. + + .. method:: scan_dragto(x, y) + + Scroll the listbox by 10 times the difference between *x*, *y* and the + coordinates passed to the last :meth:`scan_mark` call. + This is typically bound to mouse motion events in the widget, producing + the effect of dragging the list at high speed through the window. + + +.. class:: Menu(master=None, cnf={}, **kw) + + A :class:`!Menu` widget displays a column of entries, each of which may be a + command, a checkbutton, a radiobutton, a cascade (which posts an associated + submenu) or a separator. + Menus are used as the menubar of a toplevel window, as pulldown menus posted + from a cascade entry or menubutton, and as popup menus. + Inherits from :class:`Widget`. + + Many of the entry methods take an *index* argument that selects which entry + to operate on. + As described in the Tk ``menu`` manual page, *index* may be a numeric index + (counting from 0 at the top), ``'active'`` (the currently active entry), + ``'end'`` or ``'last'`` (the bottommost entry), ``'none'`` (no entry at all, + written ``{}`` in Tcl), ``@y`` (the entry covering pixel y-coordinate *y* in + the menu window), or a pattern matched against the labels of the entries + from the top down. + + .. method:: add(itemType, cnf={}, **kw) + + Add a new entry to the bottom of the menu. + *itemType* is one of ``'command'``, ``'cascade'``, ``'checkbutton'``, + ``'radiobutton'`` or ``'separator'`` and determines the type of the new + entry; the remaining options configure it. + The :meth:`!add_command`, :meth:`!add_cascade`, :meth:`!add_checkbutton`, + :meth:`!add_radiobutton` and :meth:`!add_separator` convenience methods + call this method with the corresponding *itemType*. + + The entry is configured by the following options, although not every + option applies to every entry type (a separator accepts none of them): + + *label* + The text to display in the entry. + + *command* + The function to call when the entry is invoked (command, checkbutton + and radiobutton entries). + + *accelerator* + A string displayed at the right of the entry to advertise an + accelerator keystroke; it does not itself create the binding. + + *underline* + The index of a character in the label to underline for keyboard + traversal. + + *state* + One of ``'normal'``, ``'active'`` or ``'disabled'``. + + *image* + An image to display instead of, or together with, the text label. + + *compound* + Where to show the image relative to the text: ``'none'`` (the + default), ``'text'``, ``'image'``, ``'top'``, ``'bottom'``, ``'left'`` + or ``'right'``. + + *bitmap* + A bitmap to display instead of the text label. + + *font* + The font to use for the text. + + *background*, *foreground* + The entry's background and foreground colors in its normal state + (ignored on macOS). + + *activebackground*, *activeforeground* + The background and foreground colors used when the entry is active + (ignored on macOS). + + *columnbreak* + If true, the entry starts a new column instead of being placed below + the previous entry. + + *hidemargin* + If true, the standard margin around the entry is omitted, which is + useful when a menu is used as a palette. + + *menu* + The submenu posted by a cascade entry; it must be a child of this + menu. + + *variable* + The variable associated with a checkbutton or radiobutton entry. + + *onvalue*, *offvalue* + The values stored in *variable* when a checkbutton entry is selected + or cleared. + + *value* + The value stored in *variable* when a radiobutton entry is selected. + + *indicatoron* + Whether to display the indicator of a checkbutton or radiobutton + entry. + + *selectcolor* + The color of the indicator of a checkbutton or radiobutton entry when + it is selected. + + *selectimage* + The image displayed when a checkbutton or radiobutton entry is + selected and *image* is also given. + + .. method:: add_cascade(cnf={}, **kw) + + Add a new cascade entry to the bottom of the menu. + A cascade entry has an associated submenu, given by its *menu* option, + which must be a child of this menu; posting the entry posts the submenu + next to it. + + .. method:: add_checkbutton(cnf={}, **kw) + + Add a new checkbutton entry to the bottom of the menu. + When invoked, a checkbutton entry toggles between its *onvalue* and + *offvalue*, storing the result in its associated *variable*, and displays + an indicator showing whether it is selected. + + .. method:: add_command(cnf={}, **kw) + + Add a new command entry to the bottom of the menu. + A command entry behaves much like a button: when it is invoked, the + callback given by its *command* option is called. + + .. method:: add_radiobutton(cnf={}, **kw) + + Add a new radiobutton entry to the bottom of the menu. + Radiobutton entries sharing the same *variable* form a group of which + only one may be selected at a time; selecting an entry stores its *value* + in the variable. + + .. method:: add_separator(cnf={}, **kw) + + Add a separator to the bottom of the menu. + A separator is displayed as a horizontal dividing line and cannot be + activated or invoked. + + .. method:: insert(index, itemType, cnf={}, **kw) + + Same as :meth:`add`, except that the new entry is inserted just before + the entry given by *index* instead of being appended to the end of the + menu. + *itemType* is one of ``'command'``, ``'cascade'``, ``'checkbutton'``, + ``'radiobutton'`` or ``'separator'``. + The :meth:`!insert_command`, :meth:`!insert_cascade`, + :meth:`!insert_checkbutton`, :meth:`!insert_radiobutton` and + :meth:`!insert_separator` convenience methods call this method with the + corresponding *itemType*. + + .. method:: insert_cascade(index, cnf={}, **kw) + + Insert a new cascade entry before the entry given by *index* (see + :meth:`add_cascade`). + + .. method:: insert_checkbutton(index, cnf={}, **kw) + + Insert a new checkbutton entry before the entry given by *index* (see + :meth:`add_checkbutton`). + + .. method:: insert_command(index, cnf={}, **kw) + + Insert a new command entry before the entry given by *index* (see + :meth:`add_command`). + + .. method:: insert_radiobutton(index, cnf={}, **kw) + + Insert a new radiobutton entry before the entry given by *index* (see + :meth:`add_radiobutton`). + + .. method:: insert_separator(index, cnf={}, **kw) + + Insert a separator before the entry given by *index* (see + :meth:`add_separator`). + + .. method:: delete(index1, index2=None) + + Delete all of the menu entries between *index1* and *index2* inclusive. + If *index2* is omitted, it defaults to *index1*, so that a single entry + is deleted. + Attempts to delete a tear-off entry are ignored; remove it by changing + the *tearoff* option instead. + + .. method:: entrycget(index, option) + + Return the current value of the configuration option *option* for the + entry given by *index*. + + .. method:: entryconfig(index, cnf=None, **kw) + :no-typesetting: + + .. method:: entryconfigure(index, cnf=None, **kw) + + Query or modify the configuration options of the entry given by *index*. + This mirrors :meth:`~Misc.configure`, except that it applies to an + individual entry rather than to the menu as a whole. + With no options, it returns a dictionary describing the current options + of the entry; otherwise it sets the given options. + The supported options are those accepted by :meth:`add` for the entry's + type. + :meth:`entryconfig` is an alias of :meth:`!entryconfigure`. + + .. method:: index(index) + + Return the numerical index corresponding to *index*, or ``None`` if + *index* selects no entry. + + .. method:: type(index) + + Return the type of the entry given by *index*: one of ``'command'``, + ``'cascade'``, ``'checkbutton'``, ``'radiobutton'``, ``'separator'`` or + ``'tearoff'`` (for the tear-off entry). + + .. method:: activate(index) + + Make the entry given by *index* the active entry, redisplaying it with + its active colors, and deactivate any previously active entry. + If *index* selects no entry, or the selected entry is disabled, the menu + ends up with no active entry. + + .. method:: invoke(index) + + Invoke the action of the entry given by *index*, as if it had been + clicked. + Nothing happens if the entry is disabled. + If the entry has a *command* associated with it, the result of that + command is returned; otherwise the result is an empty string. + + .. method:: post(x, y) + + Display the menu on the screen at the root-window coordinates *x* and + *y*, adjusting them if necessary so that the whole menu is visible. + If the *postcommand* option has been specified, it is evaluated before + the menu is posted. + + .. method:: tk_popup(x, y, entry='') + + Post the menu as a popup at the root-window coordinates *x* and *y*. + If *entry* is given, the menu is positioned so that this entry is + displayed under the pointer. + + .. method:: unpost() + + Unmap the menu so that it is no longer displayed, also unposting any + posted lower-level cascaded submenu. + This has no effect on Windows and macOS, which manage the unposting of + menus themselves. + + .. method:: xposition(index) + + Return the x-coordinate, within the menu window, of the leftmost pixel of + the entry given by *index*. + + .. versionadded:: 3.3 + + .. method:: yposition(index) + + Return the y-coordinate, within the menu window, of the topmost pixel of + the entry given by *index*. + + +.. class:: Menubutton(master=None, cnf={}, **kw) + + A :class:`!Menubutton` widget displays a textual string, bitmap or image and + posts an associated :class:`Menu`, given by its *menu* option, when the user + presses it. + Like a :class:`Label` it can show *text*, a *textvariable*, or an *image*, + and the *direction* option controls where the menu appears relative to the + button. + Inherits from :class:`Widget`. + Refer to the Tk ``menubutton`` manual page for the full list of options. + + +.. class:: Message(master=None, cnf={}, **kw) + + A :class:`!Message` widget displays a non-interactive textual string, given + by the *text* option or linked to a variable through *textvariable*. + Unlike a :class:`Label`, it breaks the string into multiple lines in order + to produce a given aspect ratio, choosing line breaks at word boundaries, + and it can justify the text left, centered or right. + Inherits from :class:`Widget`. + Refer to the Tk ``message`` manual page for the full list of options. + + +.. class:: OptionMenu(master, variable, value, *values, **kwargs) + + A helper subclass of :class:`Menubutton` that displays a pop-up menu of + mutually exclusive choices. + *variable* is a :class:`Variable` kept in sync with the selection, *value* + is the initial choice, and *values* are the remaining menu entries. + The keyword argument *command* may be given a callback that is invoked with + the selected value, and the keyword argument *name* sets the Tk widget name. + + .. method:: destroy() + + Destroy the widget, also cleaning up the associated pop-up menu. + + .. versionchanged:: 3.14 + Added support for the *name* keyword argument. + + + +.. class:: PanedWindow(master=None, cnf={}, **kw) + + A :class:`!PanedWindow` is a geometry-manager widget that arranges any + number of child *panes* in a row (when *orient* is ``'horizontal'``) or a + column (when *orient* is ``'vertical'``). + Each pane holds one widget, and each pair of adjacent panes is separated by + a movable *sash* that the user can drag with the mouse to resize the widgets + on either side of it. + Inherits from :class:`Widget`. + + The *orient* option selects the layout direction, *sashwidth* sets the width + of each sash and *sashrelief* its relief. + When *showhandle* is true a small handle is drawn on each sash that the user + can grab to drag it. + Refer to the Tk ``panedwindow`` manual page for the full list of options. + + .. method:: add(child, **kw) + + Add *child* to the panedwindow as a new pane, placed after any existing + panes. + The keyword arguments specify per-pane management options for *child*; + they may be any of the options accepted by :meth:`paneconfigure`. + + .. method:: forget(child) + :no-typesetting: + + .. method:: remove(child) + + Remove the pane containing *child* from the panedwindow. + All geometry management options for *child* are forgotten. + :meth:`forget` is an alias of :meth:`!remove`. + This shadows the inherited geometry-manager :meth:`!forget`; + use :meth:`~Pack.pack_forget`, :meth:`~Grid.grid_forget` or + :meth:`~Place.place_forget` to remove the widget itself from its manager. + + .. method:: panes() + + Return a tuple of the widgets managed by the panedwindow, one per pane, + in order. + + .. method:: panecget(child, option) + + Return the current value of the management option *option* for the pane + containing *child*. + *option* may be any value allowed by :meth:`paneconfigure`. + + .. method:: paneconfig(tagOrId, cnf=None, **kw) + :no-typesetting: + + .. method:: paneconfigure(tagOrId, cnf=None, **kw) + + Query or modify the management options of the pane containing the widget + *tagOrId*. + With no options, it returns a dictionary describing all of the available + options for the pane; given a single option name as a string, it returns + a description of that one option; otherwise it sets the given options. + The supported options include *after* and *before* (insert the pane after + or before another managed window), *height* and *width* (the outer + dimensions of the window, including any border), *minsize* (the minimum + size in the paned dimension), *padx* and *pady* (extra space to leave on + each side of the window), *sticky* (position or stretch the window within + an oversized pane, using a string of the characters ``n``, ``s``, ``e`` + and ``w``), *hide* (hide the pane while keeping it in the list of panes) + and *stretch* (how extra space is allocated to the pane: one of + ``'always'``, ``'first'``, ``'last'``, ``'middle'`` or ``'never'``). + :meth:`paneconfig` is an alias of :meth:`!paneconfigure`. + + .. method:: identify(x, y) + + Identify the panedwindow component underneath the point given by *x* and + *y*, in window coordinates. + If the point is over a sash or a sash handle, the result is a two-element + tuple containing the index of the sash or handle and a word indicating + whether it is over a sash or a handle, such as ``(0, 'sash')`` or + ``(2, 'handle')``. + If the point is over any other part of the panedwindow, the result is an + empty string. + + .. method:: sash(*args) + + Query or change the position of the sashes in the panedwindow. + This is a thin wrapper around the Tk ``sash`` subcommand; the convenience + methods :meth:`sash_coord`, :meth:`sash_mark` and :meth:`sash_place` + should normally be used instead. + + .. method:: sash_coord(index) + + Return the current x and y coordinate pair for the sash given by *index*, + which must be an integer between 0 and one less than the number of panes + in the panedwindow. + The coordinates returned are those of the top left corner of the region + containing the sash. + + .. method:: sash_mark(index) + + Record the current mouse position for the sash given by *index*, for use + together with later sash-drag operations to move the sash. + + .. method:: sash_place(index, x, y) + + Place the sash given by *index* at the coordinates *x* and *y*. + + .. method:: proxy(*args) + + Query or change the position of the sash proxy, the "ghost" sash shown + while a sash is being dragged with non-opaque resizing. + This is a thin wrapper around the Tk ``proxy`` subcommand; the + convenience methods :meth:`proxy_coord`, :meth:`proxy_forget` and + :meth:`proxy_place` should normally be used instead. + + .. method:: proxy_coord() + + Return a tuple containing the x and y coordinates of the most recent + proxy location. + + .. method:: proxy_forget() + + Remove the proxy from the display. + + .. method:: proxy_place(x, y) + + Place the proxy at the coordinates *x* and *y*. + + +.. class:: Radiobutton(master=None, cnf={}, **kw) + + A :class:`!Radiobutton` widget displays a textual string, bitmap or image + together with a diamond or circular indicator, and selects one choice out of + several. + It has all the behavior of a simple button and, in addition, can be + selected: typically several radiobuttons share a single *variable*, and + selecting one sets that variable to the radiobutton's *value*; each + radiobutton also monitors the variable and automatically selects or + deselects itself when the variable changes. + Inherits from :class:`Widget`. + In addition to the standard widget options, a radiobutton accepts the + options documented in the Tk ``radiobutton`` manual page, such as + *variable*, *value* and *command*. + + .. method:: invoke() + + Do just what would happen if the user pressed the radiobutton with the + mouse: select the button and invoke the associated command, if there is + one. + Return the result of the command, or an empty string if no command is + associated with the radiobutton. + This is ignored if the radiobutton's state is ``disabled``. + + .. method:: select() + + Select the radiobutton and set the associated variable to the value + corresponding to this widget. + + .. method:: deselect() + + Deselect the radiobutton and set the associated variable to an empty + string. + If this radiobutton was not currently selected, this has no effect. + + .. method:: flash() + + Flash the radiobutton by redisplaying it several times, alternating + between the active and normal colors. + At the end of the flash the radiobutton is left in the same normal or + active state as when the method was called. + This is ignored if the radiobutton's state is ``disabled``. + + +.. class:: Scale(master=None, cnf={}, **kw) + + A :class:`!Scale` widget lets the user select a numerical value by moving a + slider along a trough. + It can be oriented vertically or horizontally and can optionally display a + label and the current value. + Inherits from :class:`Widget`. + + In addition to the standard widget options, a scale accepts the options + documented in the Tk ``scale`` manual page, such as *from_*, *to*, + *resolution*, *orient*, *tickinterval*, *variable* and *command*. + As elsewhere in :mod:`!tkinter`, the leading ``-`` of the Tk option name is + dropped; *from* is spelled ``from_`` because :keyword:`from` is a Python + keyword. + + With a non-integer *resolution*, see :ref:`numeric values and the locale + `. + + .. method:: get() + + Return the current value of the scale. + The result is an integer if the scale's *resolution* yields whole + numbers, and a float otherwise. + + .. method:: set(value) + + Set the scale to *value*, moving the slider accordingly. + This has no effect if the scale is disabled. + + .. method:: coords(value=None) + + Return a tuple ``(x, y)`` giving the pixel coordinates, relative to the + widget, of the point on the centerline of the trough that corresponds to + *value*. + If *value* is omitted, the scale's current value is used. + + .. method:: identify(x, y) + + Return a string describing the part of the scale at the pixel coordinates + *x*, *y*: ``'slider'``, ``'trough1'`` (the part of the trough above or to + the left of the slider), ``'trough2'`` (below or to the right of the + slider), or an empty string if the point is not over any of these + elements. + + +.. class:: Scrollbar(master=None, cnf={}, **kw) + + A :class:`!Scrollbar` widget displays a slider and two arrows that let the + user scroll an associated widget, such as a :class:`Listbox`, :class:`Text`, + :class:`Canvas` or :class:`Entry`. + It is connected to the scrolled widget by setting that widget's + *xscrollcommand* or *yscrollcommand* option to the scrollbar's :meth:`set` + method, and the scrollbar's *command* option to the scrolled widget's + :meth:`~XView.xview` or :meth:`~YView.yview` method. + Inherits from :class:`Widget`. + + .. method:: get() + + Return the current scrollbar settings as a tuple ``(first, last)`` of two + fractions between 0 and 1, describing the portion of the document that is + currently visible, as last passed to :meth:`set`. + + .. method:: set(first, last) + + Set the scrollbar. + *first* and *last* are fractions between 0 and 1 giving the positions of + the start and end of the visible portion of the associated document. + This method is normally registered as the scrolled widget's + *xscrollcommand* or *yscrollcommand* and called by that widget. + + .. method:: activate(index=None) + + Mark the element *index* (one of ``'arrow1'``, ``'slider'`` or + ``'arrow2'``) as active, displaying it according to the + *activebackground* and *activerelief* options. + If *index* is omitted, return the name of the currently active element, + or ``None`` if no element is active. + + .. versionchanged:: 3.5 + The *index* argument is now optional. + + .. method:: delta(deltax, deltay) + + Return a float indicating the fractional change in the scrollbar setting + that corresponds to moving the slider by *deltax* pixels horizontally + (for horizontal scrollbars) or *deltay* pixels vertically (for vertical + scrollbars). + + .. method:: fraction(x, y) + + Return a float between 0 and 1 indicating where the point at pixel + coordinates *x*, *y* lies in the trough: 0 corresponds to the top or left + of the trough and 1 to the bottom or right. + + .. method:: identify(x, y) + + Return the name of the element under the pixel coordinates *x*, *y* (such + as ``'arrow1'``), or an empty string if the point does not lie in any + element of the scrollbar. + + +.. class:: Spinbox(master=None, cnf={}, **kw) + + A :class:`!Spinbox` widget is an :class:`Entry`-like widget with a pair of + up/down arrow buttons that let the user step through a range of values in + addition to editing the value directly. + The set of values may be a numeric range given by the *from_*, *to* and + *increment* options, or an explicit list of strings given by the *values* + option (which takes precedence over the range). + Each time an arrow is invoked the *command* callback, if any, is called; the + *wrap* option controls whether stepping past either end of the range wraps + around to the other end; the *format* option specifies how numeric values + are formatted; and the *validate* option enables validation of the entered + text. + Inherits from :class:`Widget` and :class:`XView`. + + With a non-integer *increment*, see :ref:`numeric values and the locale + `. + + Many of the methods take an *index* argument identifying a character in the + spinbox's string. + As described in the Tk ``spinbox`` manual page, *index* may be a numeric + index (counting from 0), ``'anchor'`` (the selection anchor point), + ``'end'`` (just after the last character), ``'insert'`` (the character just + after the insertion cursor), ``'sel.first'`` or ``'sel.last'`` (the ends of + the selection), or ``@x`` (the character covering pixel x-coordinate *x* in + the window). + + .. method:: get() + + Return the spinbox's string. + + .. method:: insert(index, s) + + Insert the characters of the string *s* just before the character given + by *index*. + + .. method:: delete(first, last=None) + + Delete one or more characters of the spinbox. + *first* is the index of the first character to delete, and *last* is the + index of the character just after the last one to delete. + If *last* is omitted, a single character at *first* is deleted. + + .. method:: icursor(index) + + Arrange for the insertion cursor to be displayed just before the + character given by *index*. + + .. method:: index(index) + + Return the numerical index corresponding to *index*, as a string. + + .. method:: bbox(index) + + Return a tuple of four integers ``(x, y, width, height)`` describing the + bounding box of the character given by *index*. + *x* and *y* are the pixel coordinates of the upper-left corner of the + character relative to the widget, and *width* and *height* are its size + in pixels. + The bounding box may refer to a region outside the visible area of the + window. + + This shadows the inherited :meth:`!Misc.bbox`; + use :meth:`~Misc.grid_bbox` for the grid bounding box. + + .. method:: identify(x, y) + + Return the name of the window element at the pixel coordinates *x*, *y*: + one of ``'buttondown'``, ``'buttonup'``, ``'entry'`` or ``'none'``. + + .. method:: invoke(element) + + Invoke the spin button given by *element*, either ``'buttonup'`` or + ``'buttondown'``, triggering the action associated with it. + + .. method:: scan(*args) + + A thin wrapper around the Tk ``scan`` widget subcommand, used to + implement fast dragging of the view: ``scan('mark', x)`` records *x* and + the current view, and ``scan('dragto', x)`` adjusts the view relative to + that mark. + The :meth:`scan_mark` and :meth:`scan_dragto` methods wrap the two forms. + + .. method:: scan_mark(x) + + Record *x* and the current view in the spinbox window, for use with a + later :meth:`scan_dragto` call. + This is typically associated with a mouse button press in the widget. + + .. method:: scan_dragto(x) + + Adjust the view by 10 times the difference between *x* and the *x* passed + to the last :meth:`scan_mark` call. + This is typically associated with mouse motion events, producing the + effect of dragging the spinbox at high speed through the window. + + .. method:: selection(*args) + + A thin wrapper around the Tk ``selection`` widget subcommand, used to + adjust the selection within the spinbox. + It has several forms depending on the first argument, such as + ``selection('adjust', index)``, ``selection('clear')``, + ``selection('element', ?elem?)``, ``selection('from', index)``, + ``selection('present')``, ``selection('range', start, end)`` and + ``selection('to', index)``. + The :meth:`selection_adjust`, :meth:`selection_clear`, + :meth:`selection_element`, :meth:`selection_from`, + :meth:`selection_present`, :meth:`selection_range` and + :meth:`selection_to` methods wrap these forms. + + .. method:: selection_adjust(index) + + Locate the end of the selection nearest to the character given by *index* + and adjust that end of the selection to be at *index* (including but not + going beyond *index*). + The other end becomes the anchor point for future :meth:`selection_to` + calls. + If the selection is not currently in the spinbox, a new selection is + created to include the characters between *index* and the most recent + anchor point, inclusive. + + .. method:: selection_clear() + + Clear the selection if it is currently in this widget. + If the selection is not in this widget, the method has no effect. + + .. note:: + + This shadows the inherited :meth:`Misc.selection_clear`, + which clears the X selection; + that method is not available on a :class:`Spinbox`. + + .. method:: selection_element(element=None) + + Set or get the currently selected element. + If *element* (one of ``'buttonup'``, ``'buttondown'`` or ``'none'``) is + given, that spin button is selected and displayed depressed; otherwise + the name of the currently selected element is returned. + + .. method:: selection_from(index) + + Set the selection anchor point to just before the character given by + *index*, without changing the selection itself. + + .. versionadded:: 3.8 + + + .. method:: selection_present() + + Return ``True`` if there are characters selected in the spinbox, + ``False`` otherwise. + + .. versionadded:: 3.8 + + + .. method:: selection_range(start, end) + + Set the selection to include the characters starting with the one indexed + by *start* and ending with the one just before *end*. + If *end* refers to the same character as *start* or an earlier one, the + selection is cleared. + + .. versionadded:: 3.8 + + + .. method:: selection_to(index) + + Set the selection between *index* and the anchor point. + If *index* is before the anchor point, the selection runs from *index* up + to but not including the anchor point; if it is after, the selection runs + from the anchor point up to but not including *index*; if it is the same, + nothing happens. + The anchor point is the one set by the most recent :meth:`selection_from` + or :meth:`selection_adjust` call. + If the selection is not in this widget, a new selection is created using + the most recent anchor point. + + .. versionadded:: 3.8 + + + +.. class:: Text(master=None, cnf={}, **kw) + + A :class:`!Text` widget displays and edits multi-line text. + Portions of the text may be styled with **tags**, particular positions may + be annotated with floating **marks**, and arbitrary images and other widgets + may be embedded in the text. + The widget also provides an unlimited undo/redo mechanism and supports peer + widgets that share the same underlying data. + Inherits from :class:`Widget`, :class:`XView` and :class:`YView`, so the + view can be scrolled horizontally and vertically with :meth:`~XView.xview` + and :meth:`~YView.yview`. + Refer to the Tk ``text`` manual page for the full list of options. + + Most of the methods take one or more *index* arguments that identify a + position within the text. + As described in the Tk ``text`` manual page, an index is a string consisting + of a base, optionally followed by one or more modifiers. + The base may be ``'line.char'`` (line *line*, character *char*, where lines + are counted from 1 and characters within a line from 0; ``'line.end'`` + refers to the newline ending the line), ``'end'`` (the position just after + the last newline), the name of a mark, ``'tag.first'`` or ``'tag.last'`` + (the first character tagged with *tag*, or the position just after the last + such character), the name of an embedded image or window, or ``@x,y`` (the + character covering pixel coordinates *x*, *y* in the widget). + A modifier such as ``'+5 chars'``, ``'-3 lines'``, ``'linestart'``, + ``'lineend'``, ``'wordstart'`` or ``'wordend'`` adjusts the index relative + to its base; several modifiers may be combined and are applied from left to + right, for example ``'insert wordstart - 1 c'``. + + .. method:: insert(index, chars, *args) + + Insert the string *chars* just before the character at *index* (if + *index* is ``'end'``, just before the final newline). + By default the new text inherits any tags present on both sides of the + insertion point. + If *args* is given, it consists of alternating *tagList*, *chars* values: + the preceding *chars* receives exactly the tags listed (a tag list may be + a single tag name or a sequence of names), overriding the surrounding + tags. + + .. method:: delete(index1, index2=None) + + Delete the range of characters from *index1* up to but not including + *index2*. + If *index2* is omitted, the single character at *index1* is deleted. + The widget always keeps a newline as its last character, so a deletion + that would remove it is adjusted accordingly. + + .. method:: replace(index1, index2, chars, *args) + + Replace the range of characters from *index1* up to but not including + *index2* with *chars*. + This is equivalent to a :meth:`delete` followed by an :meth:`insert` at + *index1*; *args* is interpreted as for :meth:`insert`. + + .. versionadded:: 3.3 + + .. method:: get(index1, index2=None) + + Return the text from *index1* up to but not including *index2* as a + string. + If *index2* is omitted, return the single character at *index1*. + Embedded images and windows are omitted from the result. + + .. method:: index(index) + + Return the position corresponding to *index* in the canonical + ``'line.char'`` form. + + .. method:: compare(index1, op, index2) + + Compare the positions of *index1* and *index2* using the relational + operator *op*, which must be one of ``'<'``, ``'<='``, ``'=='``, + ``'>='``, ``'>'`` or ``'!='``, and return the boolean result. + + .. method:: count(index1, index2, *options, return_ints=False) + + Count the number of items of the requested kinds between *index1* and + *index2*; the count is negative if *index1* is after *index2*. + Each of *options* names a kind of item to count: ``'chars'``, + ``'displaychars'``, ``'displayindices'``, ``'displaylines'``, + ``'indices'``, ``'lines'``, ``'xpixels'`` or ``'ypixels'`` (the default, + used when no option is given, is ``'indices'``). + The pseudo-option ``'update'`` forces any out-of-date layout information + to be recalculated before the following options are evaluated. + When *return_ints* is true and a single counting option is given, return + a plain integer; otherwise return a tuple with one integer per counting + option (or ``None`` if the result is empty). + + .. versionadded:: 3.3 + + .. versionchanged:: 3.13 + Added the *return_ints* parameter. + + + .. method:: see(index) + + Adjust the view so that the character given by *index* is visible. + If it is already visible the method has no effect; if it is a short + distance out of view the widget scrolls just enough to bring it to the + nearest edge, otherwise it scrolls to center *index* in the window. + + .. method:: bbox(index) + + Return a tuple ``(x, y, width, height)`` giving the bounding box, in + pixels, of the visible part of the character at *index*, or ``None`` if + that character is not visible on the screen. + + This shadows the inherited :meth:`!Misc.bbox`; + use :meth:`~Misc.grid_bbox` for the grid bounding box. + + .. method:: dlineinfo(index) + + Return a tuple ``(x, y, width, height, baseline)`` describing the display + line that contains *index*: the first four values give the bounding box + of the line in pixels and *baseline* gives the offset of the baseline + measured down from the top of the area. + Return ``None`` if that display line is not visible on the screen. + + .. method:: mark_set(markName, index) + + Set the mark named *markName* to the position just before the character + at *index*, creating the mark if it does not already exist. + A mark created this way has right gravity by default. + + .. method:: mark_unset(*markNames) + + Remove each of the marks named in *markNames*. + The special ``insert`` and ``current`` marks may not be removed. + + .. method:: mark_names() + + Return a tuple of the names of all marks currently set in the widget. + + .. method:: mark_gravity(markName, direction=None) + + If *direction* is omitted, return the gravity of mark *markName*, either + ``'left'`` or ``'right'``. + Otherwise set its gravity to *direction*. + The gravity determines on which side of the mark text inserted at the + mark's position appears: a mark with right gravity (the default) stays to + the right of such text. + + .. method:: mark_next(index) + + Return the name of the first mark at or after *index*, or ``None`` if + there is none. + When *index* is the name of a mark, the search starts just after that + mark. + + .. method:: mark_previous(index) + + Return the name of the last mark at or before *index*, or ``None`` if + there is none. + When *index* is the name of a mark, the search starts just before that + mark. + + .. method:: tag_add(tagName, index1, *args) + + Add the tag *tagName* to the range of characters from *index1* up to but + not including the next index in *args*. + Further pairs of indices may follow in *args* to tag additional ranges; a + trailing single index tags just the character at that index. + + .. method:: tag_remove(tagName, index1, index2=None) + + Remove the tag *tagName* from the characters from *index1* up to but not + including *index2* (or from the single character at *index1* if *index2* + is omitted). + The tag itself continues to exist even if no characters carry it. + + .. method:: tag_delete(*tagNames) + + Delete each of the tags named in *tagNames*, removing them from all + characters and discarding their options and bindings. + + .. method:: tag_config(tagName, cnf=None, **kw) + :no-typesetting: + + .. method:: tag_configure(tagName, cnf=None, **kw) + + Query or modify the configuration options of the tag *tagName*. + This mirrors :meth:`~Misc.configure`, except that it applies to a tag + rather than to the widget as a whole: with no options it returns a + dictionary describing the current options, otherwise it sets the given + options. + Defining a tag this way also gives it a priority higher than any existing + tag. + + The supported tag options, all controlling the appearance of the tagged + text, are: + + *font* + The font to use for the text. + + *foreground* + The color to use for the text. + + *background* + The color to use for the area behind the text. + + *fgstipple*, *bgstipple* + Bitmaps used to stipple the foreground (text) and the background; + only well supported on X11. + + *borderwidth* + The width of the border drawn around the text according to *relief* + (default ``0``). + + *relief* + The 3-D appearance of the text's border: ``'flat'`` (the default), + ``'raised'``, ``'sunken'``, ``'ridge'``, ``'groove'`` or ``'solid'``. + + *offset* + How far the text is raised above (or, if negative, lowered below) the + baseline, for superscripts and subscripts. + + *underline* + Whether to underline the text. + + *underlinefg* + The color of the underline; it defaults to the text color. + + *overstrike* + Whether to draw a line through the middle of the text. + + *overstrikefg* + The color of the overstrike line; it defaults to the text color. + + *elide* + Whether the text is elided (hidden). + + *justify* + How to justify the first character of a display line: ``'left'`` (the + default), ``'right'`` or ``'center'``. + + *wrap* + How to wrap lines that are too long: ``'char'``, ``'word'`` or + ``'none'``. + + *lmargin1*, *lmargin2* + The indentation, in pixels, of the first display line of a logical + line and of the remaining display lines. + + *lmargincolor* + The color of the left margin area. + + *rmargin* + The right-hand margin, in pixels. + + *rmargincolor* + The color of the right margin area. + + *spacing1*, *spacing2*, *spacing3* + Extra space, in pixels, above the first display line of a logical + line, between its display lines, and below its last display line. + + *tabs* + The set of tab stops, in the same form as the widget's *tabs* option. + + *tabstyle* + How tab stops are interpreted: ``'tabular'`` or ``'wordprocessor'``. + + *selectbackground*, *selectforeground* + The background and foreground colors used for the text while it is + selected. + + .. note:: + + Tk 8.6 added the *lmargincolor*, *overstrikefg*, *rmargincolor*, + *selectbackground*, *selectforeground* and *underlinefg* options. + + :meth:`tag_config` is an alias of :meth:`!tag_configure`. + + .. method:: tag_cget(tagName, option) + + Return the current value of the configuration option *option* for the tag + *tagName*. + + .. method:: tag_names(index=None) + + If *index* is omitted, return a tuple of the names of all tags defined in + the widget; otherwise return only the names of the tags applied to the + character at *index*. + The names are ordered from lowest to highest priority. + + .. method:: tag_ranges(tagName) + + Return a tuple of indices describing all ranges of text tagged with + *tagName*. + The result alternates start and end indices, so that elements ``2*i`` and + ``2*i+1`` bound the *i*-th range. + + .. method:: tag_nextrange(tagName, index1, index2=None) + + Search forward from *index1* (up to *index2* if given) for the first + range of characters tagged with *tagName*, and return a two-element tuple + of its start and end indices, or an empty tuple if there is no such + range. + + .. method:: tag_prevrange(tagName, index1, index2=None) + + Search backward from *index1* (down to *index2* if given) for the nearest + preceding range of characters tagged with *tagName*, and return a + two-element tuple of its start and end indices, or an empty tuple if + there is no such range. + + .. method:: tag_raise(tagName, aboveThis=None) + + Raise the priority of tag *tagName* so that it is just above the priority + of *aboveThis*, or to the highest priority of all tags if *aboveThis* is + omitted. + When the display options of overlapping tags conflict, the + higher-priority tag wins. + + .. method:: tag_lower(tagName, belowThis=None) + + Lower the priority of tag *tagName* so that it is just below the priority + of *belowThis*, or to the lowest priority of all tags if *belowThis* is + omitted. + + .. method:: tag_bind(tagName, sequence, func, add=None) + + Bind the event *sequence* on characters tagged with *tagName* to the + callback *func*, so that *func* is invoked when that event occurs over + such a character. + If *add* is true the binding is added alongside any existing bindings for + *sequence*, otherwise it replaces them. + Works like :meth:`~Misc.bind` and returns the identifier of the new + binding. + + .. method:: tag_unbind(tagName, sequence, funcid=None) + + Remove the bindings of the event *sequence* on characters tagged with + *tagName*. + If *funcid* is given, only that binding (as returned by :meth:`tag_bind`) + is removed and its callback is unregistered. + + .. versionchanged:: 3.13 + If *funcid* is given, only that callback is unbound. + + + .. method:: image_create(index, cnf={}, **kw) + + Embed an image at *index* and return the name assigned to this image + instance, which may then be used as an index or passed to the other + ``image_*`` methods. + The options, given in *cnf* and *kw*, include *image* (the Tk image to + display), *name* (a base name for the instance), *align*, *padx* and + *pady*. + + .. method:: image_cget(index, option) + + Return the current value of the configuration option *option* for the + embedded image at *index*. + + .. method:: image_configure(index, cnf=None, **kw) + + Query or modify the configuration options of the embedded image at + *index*, like :meth:`~Misc.configure` but applied to that image. + + .. method:: image_names() + + Return a tuple of the names of all images embedded in the widget. + + .. note:: + + This shadows the inherited :meth:`Misc.image_names`, + which returns the names of all images in the Tcl interpreter; + that method is not available on a :class:`Text`. + + .. method:: window_create(index, cnf={}, **kw) + + Embed a window (any widget) at *index*. + The options, given in *cnf* and *kw*, include *window* (the widget to + embed), *create* (a callback that creates the widget on demand), *align*, + *stretch*, *padx* and *pady*. + The embedded widget must be a descendant of the text widget's parent. + + .. method:: window_cget(index, option) + + Return the current value of the configuration option *option* for the + embedded window at *index*. + + .. method:: window_config(index, cnf=None, **kw) + :no-typesetting: + + .. method:: window_configure(index, cnf=None, **kw) + + Query or modify the configuration options of the embedded window at + *index*, like :meth:`~Misc.configure` but applied to that window. + + :meth:`window_config` is an alias of :meth:`!window_configure`. + + .. method:: window_names() + + Return a tuple of the names of all windows embedded in the widget. + + .. method:: edit(*args) + + Low-level wrapper around the Tk ``edit`` widget command that controls the + undo/redo mechanism and the modified flag; *args* is the ``edit`` + subcommand and its arguments. + The :meth:`!edit_\*` methods below are thin wrappers around it and are + usually more convenient. + + .. method:: edit_modified(arg=None) + + If *arg* is omitted, return the current state of the modified flag as + true or false; the flag is set automatically whenever the text is + inserted or deleted. + Otherwise set the flag to the boolean *arg*. + + .. method:: edit_undo() + + Undo the most recent edit action, that is, all the inserts and deletes + recorded on the undo stack since the previous separator, and move it to + the redo stack. + Raises :exc:`TclError` if the undo stack is empty. + Has no effect unless the *undo* option is true. + Since Tk 9.0, returns a tuple of indices delimiting the ranges of text + that were changed. + + .. method:: edit_redo() + + Reapply the most recently undone edit action, provided no further edits + have been made since, and move it back to the undo stack. + Raises :exc:`TclError` if the redo stack is empty. + Has no effect unless the *undo* option is true. + Since Tk 9.0, returns a tuple of indices delimiting the ranges of text + that were changed. + + .. method:: edit_reset() + + Clear the undo and redo stacks. + + .. method:: edit_separator() + + Push a separator onto the undo stack, marking a boundary between edit + actions for undo and redo. + Has no effect unless the *undo* option is true. + Separators are inserted automatically when the *autoseparators* option is + true. + + .. method:: search(pattern, index, stopindex=None, forwards=None, backwards=None, exact=None, regexp=None, nocase=None, count=None, elide=None) + + Search for *pattern* starting at *index* and return the index of the + first character of the first match, or an empty string if there is no + match. + Searching stops at *stopindex* if given; otherwise it wraps around the + ends of the text until the starting position is reached again. + The following boolean keyword flags control the search: *forwards* or + *backwards* select the direction (forward is the default); *exact* (the + default) or *regexp* select literal or regular-expression matching; + *nocase* makes the match case-insensitive; and *elide* causes hidden text + to be searched as well. + If *count* is a :class:`Variable`, the number of index positions in the + match is stored in it. + + + .. method:: scan_mark(x, y) + + Record *x*, *y* and the current view, for use with later + :meth:`scan_dragto` calls. + This is typically bound to a mouse button press in the widget. + + .. method:: scan_dragto(x, y) + + Scroll the widget by 10 times the difference between *x*, *y* and the + coordinates passed to the last :meth:`scan_mark` call. + This is typically bound to mouse motion events, producing the effect of + dragging the text at high speed through the window. + + .. method:: debug(boolean=None) + + If *boolean* is omitted, return whether internal consistency checks of + the B-tree data structure are enabled. + Otherwise enable or disable them. + The setting is shared by all text widgets and may noticeably slow down + widgets holding large amounts of text. + + .. method:: dump(index1, index2=None, command=None, **kw) + + Return the contents of the widget from *index1* up to but not including + *index2* (or just the segment at *index1* if *index2* is omitted), + including text and information about marks, tags, images and windows. + The result is a list of ``(key, value, index)`` triples, where *key* is + one of ``'text'``, ``'mark'``, ``'tagon'``, ``'tagoff'``, ``'image'`` or + ``'window'``. + By default all kinds are reported; passing any of the keyword arguments + *all*, *text*, *mark*, *tag*, *image* or *window* as true restricts the + dump to the selected kinds. + If *command* is given, it is called once per triple with the three values + as arguments and nothing is returned. + + .. method:: peer_create(newPathName, cnf={}, **kw) + + Create a peer text widget with the path name *newPathName* that shares + this widget's underlying data (text, marks, tags, images and the undo + stack). + Changes made through any peer are reflected in all of them. + By default the peer covers the same lines as this widget; standard text + options, including *startline* and *endline*, may be given to override + this. + + .. versionadded:: 3.3 + + .. method:: peer_names() + + Return a tuple of the path names of this widget's peers, not including + the widget itself. + + .. versionadded:: 3.3 + + .. method:: yview_pickplace(*what) + + Adjust the view so that the location given by *what* is visible. + This is an obsolete equivalent of :meth:`see`, which should be used + instead. + + +Variable classes +^^^^^^^^^^^^^^^^ + +.. class:: Variable(master=None, value=None, name=None) + + The base class for the Tk variable wrappers. + A Tk variable is a value stored in the Tcl interpreter that can be linked to + widgets through their *variable* or *textvariable* options (see + :ref:`coupling-widget-variables`), so that changes propagate both ways: + updating the variable updates every widget bound to it, and a user editing + such a widget updates the variable. + + *master* is the widget whose Tcl interpreter owns the variable; if omitted, + the default root window is used. + *value* is the initial value; if omitted, a type-specific default is used. + *name* is the name of the variable in the Tcl interpreter; if omitted, a + unique name of the form ``'PY_VARnum'`` is generated. + If *name* matches an existing variable and *value* is omitted, the existing + value is retained. + + In most cases you should use one of the typed subclasses below -- + :class:`StringVar`, :class:`IntVar`, :class:`DoubleVar` or + :class:`BooleanVar` -- rather than :class:`!Variable` directly. + + .. note:: + + When a :class:`!Variable` is garbage collected, its Tcl variable is unset. + Keep a reference to it for as long as a widget is linked to it, for example + by storing it as an attribute rather than in a local variable. + Otherwise Tk recreates the Tcl variable to keep the widget working, but it + is never unset again, leaking one Tcl variable per dropped wrapper. + + .. versionchanged:: 3.10 + Two variables now compare equal (``==``) only when they have the same + name, are of the same class, and belong to the same Tcl interpreter. + + .. method:: get() + + Return the current value of the variable. + For the base class the value is returned as a string; the typed + subclasses convert it to the appropriate Python type. + + .. method:: initialize(value) + :no-typesetting: + + .. method:: set(value) + + Set the variable to *value*. + :meth:`initialize` is an alias of :meth:`!set`. + + .. versionadded:: 3.3 + The *initialize* spelling. + + .. method:: trace_add(mode, callback) + + Register *callback* to be called when the variable is accessed according + to *mode*. + *mode* is one of the strings ``'array'``, ``'read'``, ``'write'`` or + ``'unset'``, or a list or tuple of such strings. + + When triggered, *callback* is called with three arguments: the name of + the Tcl variable, an index (or an empty string if the variable is not an + element of an array), and the *mode* that triggered the call. + + Return the internal name of the registered callback, which can be passed + to :meth:`trace_remove`. + + .. versionadded:: 3.6 + + .. method:: trace_remove(mode, cbname) + + Remove a trace callback from the variable. + *mode* must match the *mode* that was passed to :meth:`trace_add`, and + *cbname* is the callback name returned by :meth:`trace_add`. + + .. versionadded:: 3.6 + + .. method:: trace_info() + + Return a list of ``(modes, cbname)`` pairs describing all traces + currently set on the variable, where *modes* is a tuple of mode strings + and *cbname* is the internal callback name. + + .. versionadded:: 3.6 + + .. method:: trace(mode, callback) + :no-typesetting: + + .. method:: trace_variable(mode, callback) + + Register *callback* to be called when the variable is accessed according + to *mode*. + *mode* is one of the strings ``'r'``, ``'w'`` or ``'u'``, for read, write + or unset. + Return the internal name of the registered callback. + :meth:`trace` is an alias of :meth:`!trace_variable`. + + .. deprecated:: 3.6 + Use :meth:`trace_add` instead. This method wraps a Tcl feature that + was removed in Tcl 9.0. + + .. method:: trace_vdelete(mode, cbname) + + Remove the trace callback named *cbname* registered for *mode* with + :meth:`trace_variable`. + + .. deprecated:: 3.6 + Use :meth:`trace_remove` instead. This method wraps a Tcl feature + that was removed in Tcl 9.0. + + .. method:: trace_vinfo() + + Return a list of ``(mode, cbname)`` pairs for all traces set on the + variable with :meth:`trace_variable`. + + .. deprecated:: 3.6 + Use :meth:`trace_info` instead. This method wraps a Tcl feature that + was removed in Tcl 9.0. + + +.. class:: StringVar(master=None, value=None, name=None) + + A :class:`Variable` subclass that holds a string. + The default value is ``''``. + + .. method:: get() + + Return the value of the variable as a :class:`str`. + + +.. class:: IntVar(master=None, value=None, name=None) + + A :class:`Variable` subclass that holds an integer. + The default value is ``0``. + + .. method:: get() + + Return the value of the variable as an :class:`int`. + + +.. class:: DoubleVar(master=None, value=None, name=None) + + A :class:`Variable` subclass that holds a float. + The default value is ``0.0``. + + .. method:: get() + + Return the value of the variable as a :class:`float`. + + .. _tkinter-numeric-locale: + + .. note:: + + A floating-point value is always parsed with a period (``.``) as the + decimal separator, but :class:`Spinbox`, :class:`Scale` and + :class:`ttk.Spinbox ` format it according to the + ``LC_NUMERIC`` locale. Under a locale that uses a comma they produce a + value that :meth:`get` cannot read, raising :exc:`TclError`. Set + ``LC_NUMERIC`` to a locale that uses a period (such as ``'C'``) to avoid + this. + + +.. class:: BooleanVar(master=None, value=None, name=None) + + A :class:`Variable` subclass that holds a boolean. + The default value is ``False``. + + .. method:: get() + + Return the value of the variable as a :class:`bool`. + Raise a :exc:`ValueError` if the value cannot be interpreted as a + boolean. + + .. method:: initialize(value) + :no-typesetting: + + .. method:: set(value) + + Set the variable to *value*, converting it to a boolean. + :meth:`initialize` is an alias of :meth:`!set`. + + .. versionadded:: 3.3 + The *initialize* spelling. + + +Image classes +^^^^^^^^^^^^^ + +.. class:: Image(imgtype, name=None, cnf={}, master=None, **kw) + + Base class for Tk images. + *imgtype* is the Tk image type, one of ``'photo'`` or ``'bitmap'``. + An image is a named object that can be displayed by widgets through their + *image* option; deleting all references to the :class:`!Image` object + deletes the underlying Tk image. + Usually you create a :class:`PhotoImage` or :class:`BitmapImage` rather than + an :class:`!Image` directly. + + The image's configuration options are given by *cnf* and *kw* and may be + queried and changed later with the mapping protocol (using ``image[key]``) + or with the :meth:`configure` method. + + .. method:: config(**kw) + :no-typesetting: + + .. method:: configure(**kw) + + Modify one or more configuration options of the image. + The valid options depend on the image type; see :class:`PhotoImage` and + :class:`BitmapImage`. + :meth:`config` is an alias of :meth:`!configure`. + + .. method:: height() + + Return the height of the image, in pixels. + + .. method:: width() + + Return the width of the image, in pixels. + + .. method:: type() + + Return the type of the image, that is the value of *imgtype* with which + it was created (for example ``'photo'`` or ``'bitmap'``). + + +.. class:: PhotoImage(name=None, cnf={}, master=None, **kw) + + A full-color image (the Tk ``photo`` image type), stored internally with a + varying degree of transparency per pixel. + It can read and write GIF, PPM/PGM and (in Tk 8.6 and later) PNG files, read + SVG files (in Tk 9.0 and later), and be drawn in widgets. + Inherits from :class:`Image`. + + The configuration options include *data* (the image contents as a string), + *file* (the name of a file to read the contents from), *format* (the name of + the file format handler), *width* and *height* (the size of the image, used + when building it up piece by piece), *gamma* and *palette*. + + .. method:: blank() + + Blank the image; that is, set the entire image to have no data, so that + it is displayed as transparent and the background of whatever window it + is displayed in shows through. + + .. method:: cget(option) + + Return the current value of the configuration option *option*. + + .. method:: copy(*, from_coords=None, zoom=None, subsample=None) + + Return a new :class:`PhotoImage` with a copy of this image. + + *from_coords* specifies a rectangular sub-region of the source image to + be copied. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + ``(x1, y1)`` and ``(x2, y2)`` specify diagonally opposite corners of the + rectangle. + If *x2* and *y2* are not specified, they default to the bottom-right + corner of the source image. + The pixels copied include the left and top edges of the rectangle but not + the bottom or right edges. + If *from_coords* is not given, the whole source image is copied. + + If *zoom* or *subsample* are specified, the image is transformed as in + the :meth:`zoom` or :meth:`subsample` methods. + The value must be a single integer or a pair of integers. + + .. versionchanged:: 3.13 + Added the *from_coords*, *zoom* and *subsample* parameters. + + + .. method:: copy_replace(sourceImage, *, from_coords=None, to=None, \ + shrink=False, zoom=None, subsample=None, \ + compositingrule=None) + + Copy a region from *sourceImage* (which must be a :class:`PhotoImage`) + into this image, possibly with pixel zooming and/or subsampling. + If no options are specified, the whole of *sourceImage* is copied into + this image, starting at coordinates ``(0, 0)``. + + *from_coords* specifies a rectangular sub-region of the source image to + be copied, as in the :meth:`copy` method. + + *to* specifies a rectangular sub-region of the destination image to be + affected. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + If *x2* and *y2* are not specified, they default to ``(x1, y1)`` plus the + size of the source region (after subsampling and zooming, if specified). + If *x2* and *y2* are specified, the source region is replicated if + necessary to fill the destination region in a tiled fashion. + + If *shrink* is true, the size of the destination image is reduced, if + necessary, so that the region being copied into is at the bottom-right + corner of the image. + + If *zoom* or *subsample* are specified, the image is transformed as in + the :meth:`zoom` or :meth:`subsample` methods. + The value must be a single integer or a pair of integers. + + *compositingrule* specifies how transparent pixels in the source image + are combined with the destination image. + With ``'overlay'`` (the default), the old contents of the destination + image remain visible, as if the source image were printed on a piece of + transparent film and placed over the top of the destination. + With ``'set'``, the old contents of the destination image are discarded + and the source image is used as-is. + + .. versionadded:: 3.13 + + + .. method:: data(format=None, *, from_coords=None, background=None, \ + grayscale=False) + + Return the image data. + + *format* specifies the name of the image file format handler to use. + If it is not given, the data is returned as a tuple (one element per row) + of strings containing space-separated (one element per pixel/column) + colors in ``#RRGGBB`` format. + + *from_coords* specifies a rectangular region of the image to be returned. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + If only *x1* and *y1* are specified, the region extends from ``(x1, y1)`` + to the bottom-right corner of the image. + If all four coordinates are given, they specify diagonally opposite + corners of the region, including ``(x1, y1)`` and excluding ``(x2, y2)``. + If *from_coords* is not given, the whole image is returned. + + If *background* is specified, the data does not contain any transparency + information; in all transparent pixels the color is replaced by the + specified color. + + If *grayscale* is true, the data does not contain color information; all + pixel data is transformed into grayscale. + + .. versionadded:: 3.13 + + + .. method:: get(x, y) + + Return the color of the pixel at coordinates (*x*, *y*) as an + ``(r, g, b)`` tuple of three integers between 0 and 255, representing the + red, green and blue components respectively. + + .. method:: put(data, to=None) + + Set pixels of the image to the colors given in *data*, which must be a + string or a nested sequence of horizontal rows of pixel colors (for + example ``"{red green} {blue yellow}"``). + + *to* specifies the coordinates of the region of the image into which the + data are copied. + It must be a tuple or a list of 2 or 4 integers ``(x1, y1)`` or + ``(x1, y1, x2, y2)`` giving the top-left corner, and optionally the + bottom-right corner, of the region. + The default position is ``(0, 0)``. + + .. method:: read(filename, format=None, *, from_coords=None, to=None, \ + shrink=False) + + Read image data from the file named *filename* into the image. + + *format* specifies the format of the image data in the file. + + *from_coords* specifies a rectangular sub-region of the image file data + to be copied to the destination image. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + If only *x1* and *y1* are specified, the region extends from ``(x1, y1)`` + to the bottom-right corner of the image in the file. + If all four coordinates are given, they specify diagonally opposite + corners of the region. + If *from_coords* is not given, the whole of the image in the file is + read. + + *to* specifies the coordinates of the top-left corner of the region of + the image into which the data are read. + The default is ``(0, 0)``. + + If *shrink* is true, the size of the image is reduced, if necessary, so + that the region into which the file data are read is at the bottom-right + corner of the image. + + .. versionadded:: 3.13 + + + .. method:: subsample(x, y='', *, from_coords=None) + + Return a new :class:`PhotoImage` based on this image but using only every + *x*-th pixel in the X direction and every *y*-th pixel in the Y + direction. + If *y* is not given, it defaults to the same value as *x*. + + *from_coords* specifies a rectangular sub-region of the source image to + be copied, as in the :meth:`copy` method. + + .. versionchanged:: 3.13 + Added the *from_coords* parameter. + + + .. method:: transparency_get(x, y) + + Return ``True`` if the pixel at coordinates (*x*, *y*) is fully + transparent, ``False`` otherwise. + + .. versionadded:: 3.8 + + + .. method:: transparency_set(x, y, boolean) + + Make the pixel at coordinates (*x*, *y*) fully transparent if *boolean* + is true, fully opaque otherwise. + + .. versionadded:: 3.8 + + + .. method:: write(filename, format=None, from_coords=None, *, \ + background=None, grayscale=False) + + Write image data from the image to the file named *filename*. + + *format* specifies the name of the image file format handler to use. + If it is not given, the format is guessed from the file extension. + + *from_coords* specifies a rectangular region of the image to be written. + It must be a tuple or a list of 1 to 4 integers ``(x1, y1, x2, y2)``. + If only *x1* and *y1* are specified, the region extends from ``(x1, y1)`` + to the bottom-right corner of the image. + If all four coordinates are given, they specify diagonally opposite + corners of the region. + If *from_coords* is not given, the whole image is written. + + If *background* is specified, the data does not contain any transparency + information; in all transparent pixels the color is replaced by the + specified color. + + If *grayscale* is true, the data does not contain color information; all + pixel data is transformed into grayscale. + + .. versionchanged:: 3.13 + Added the *background* and *grayscale* parameters. + + + .. method:: zoom(x, y='', *, from_coords=None) + + Return a new :class:`PhotoImage` with this image magnified by a factor of + *x* in the X direction and *y* in the Y direction. + If *y* is not given, it defaults to the same value as *x*. + + *from_coords* specifies a rectangular sub-region of the source image to + be copied, as in the :meth:`copy` method. + + .. versionchanged:: 3.13 + Added the *from_coords* parameter. + + + +.. class:: BitmapImage(name=None, cnf={}, master=None, **kw) + + A two-color image (the Tk ``bitmap`` image type) created from an X11 bitmap. + Each pixel displays a foreground color, a background color, or nothing + (producing a transparent effect). + Inherits from :class:`Image`. + + The configuration options are *data* or *file* (the source bitmap, given as + a string in X11 bitmap format or as the name of a file in that format), + *maskdata* or *maskfile* (the mask bitmap, in the same forms), and + *foreground* and *background* (the two colors). + For pixels where the mask is zero the image displays nothing; for other + pixels it displays the foreground color where the source is one and the + background color where the source is zero. + If *background* is set to an empty string, the background pixels are + transparent. + + :class:`!BitmapImage` has no methods of its own beyond those inherited from + :class:`Image`. + + +Other classes +^^^^^^^^^^^^^ + +.. class:: Event() + + A container for the attributes of an event passed to a callback bound with + :meth:`Misc.bind`. + An :class:`!Event` instance has the following attributes, each corresponding + to a field of the underlying Tk event; depending on the event type, some + attributes may be set to the string ``'??'`` to indicate that they are not + meaningful. + See :ref:`bindings-and-events`. + + .. attribute:: serial + + The serial number of the event. + + .. attribute:: num + + The mouse button that was pressed or released (for button events). + + .. attribute:: focus + + Whether the window has the focus (for ``Enter`` and ``Leave`` events). + + .. attribute:: height + width + + The new height and width of the window (for ``Configure`` and ``Expose`` + events). + + .. attribute:: keycode + + The keycode of the key that was pressed or released. + + .. attribute:: state + + The state of the event, as a number (for most events) or a string (for + ``Visibility`` events). + + .. attribute:: time + + The timestamp of the event, in milliseconds. + + .. attribute:: x + y + + The pointer position relative to the widget, in pixels. + + .. attribute:: x_root + y_root + + The pointer position relative to the top-left corner of the screen, in + pixels. + + .. attribute:: char + + The character typed, as a string (for key events). + + .. attribute:: send_event + + ``True`` if the event was sent by another application. + + .. attribute:: keysym + + The symbolic name of the key that was pressed or released. + + .. attribute:: keysym_num + + The numeric value of :attr:`keysym`. + + .. attribute:: type + + The :class:`EventType` of the event. + + .. attribute:: widget + + The widget on which the event occurred. + + .. attribute:: delta + + The amount the mouse wheel was rotated (for ``MouseWheel`` events). + + +.. class:: EventType(*values) + + An :class:`enum.StrEnum` enumerating the Tk event types, used as the value + of :attr:`Event.type`. + Its members include, among others, ``KeyPress``, ``KeyRelease``, + ``ButtonPress``, ``ButtonRelease``, ``Motion``, ``Enter``, ``Leave``, + ``FocusIn``, ``FocusOut``, ``Configure``, ``Map``, ``Unmap``, ``Expose``, + ``Destroy`` and ``MouseWheel``. + + .. versionadded:: 3.6 + + + +.. class:: CallWrapper(func, subst, widget) + + Internal helper that wraps a Python callback so that it can be invoked from + Tcl. + *func* is the Python function, *subst* is an optional function that + pre-processes the Tcl arguments, and *widget* is the widget used for error + reporting. + Instances are created automatically by :meth:`Misc.register`; this class is + not normally used directly. + + +Module-level functions +^^^^^^^^^^^^^^^^^^^^^^ + +.. function:: Tcl(screenName=None, baseName=None, className='Tk', useTk=False) + + The :func:`Tcl` function is a factory function which creates an object much + like that created by the :class:`Tk` class, except that it does not + initialize the Tk subsystem. + This is most often useful when driving the Tcl interpreter in an environment + where one doesn't want to create extraneous toplevel windows, or where one + cannot (such as Unix/Linux systems without an X server). + An object created by the :func:`Tcl` object can have a Toplevel window + created (and the Tk subsystem initialized) by calling its :meth:`~Tk.loadtk` + method. + +.. function:: NoDefaultRoot() + + Inhibit the creation of an implicit default root window. + Afterwards :mod:`!tkinter` no longer creates a shared default root + automatically, and operations that rely on one --- such as constructing a + widget without an explicit *master* --- raise a :exc:`RuntimeError`. + Call this early in larger applications to make the root window explicit. + +.. function:: mainloop(n=0) + + Run the Tk main event loop on the default root window until all windows are + destroyed. + Equivalent to calling :meth:`Misc.mainloop` on the default root. + +.. function:: getboolean(s) + + Convert the Tcl boolean string *s* (one of ``'1'``, ``'true'``, ``'yes'``, + ``'on'`` and similar, or their false counterparts) to a Python + :class:`bool`. + Raise :exc:`TclError` for an invalid value. + +.. function:: getdouble(s) + + Convert *s* to a floating-point number. + This is the built-in :class:`float`. + +.. function:: getint(s) + + Convert *s* to an integer. + This is the built-in :class:`int`. + +.. function:: image_names() + + Return the names of all existing images in the default root's interpreter. + +.. function:: image_types() + + Return the available image types (such as ``'photo'`` and ``'bitmap'``) in + the default root's interpreter. + + +.. _tkinter-file-handlers: + +File handlers +^^^^^^^^^^^^^ + +Tk allows you to register and unregister a callback function which will be +called from the Tk mainloop when I/O is possible on a file descriptor. +Only one handler may be registered per file descriptor. Example code:: + + import tkinter + widget = tkinter.Tk() + mask = tkinter.READABLE | tkinter.WRITABLE + widget.tk.createfilehandler(file, mask, callback) + ... + widget.tk.deletefilehandler(file) + +This feature is not available on Windows. + +Since you don't know how many bytes are available for reading, you may not +want to use the :class:`~io.BufferedIOBase` or :class:`~io.TextIOBase` +:meth:`~io.BufferedIOBase.read` or :meth:`~io.IOBase.readline` methods, +since these will insist on reading a predefined number of bytes. +For sockets, the :meth:`~socket.socket.recv` or +:meth:`~socket.socket.recvfrom` methods will work fine; for other files, +use raw reads or ``os.read(file.fileno(), maxbytecount)``. + + +.. method:: Widget.tk.createfilehandler(file, mask, func) + + Registers the file handler callback function *func*. The *file* argument + may either be an object with a :meth:`~io.IOBase.fileno` method (such as + a file or socket object), or an integer file descriptor. The *mask* + argument is an ORed combination of any of the three constants below. + The callback is called as follows:: + + callback(file, mask) + + +.. method:: Widget.tk.deletefilehandler(file) + + Unregisters a file handler. + + +.. data:: READABLE + WRITABLE + EXCEPTION + + Constants used in the *mask* arguments. + + +Constants +^^^^^^^^^ + +The following symbolic constants are available in both the :mod:`!tkinter` +and :mod:`!tkinter.constants` namespaces. + +.. data:: TRUE + YES + ON + + Truthy values, all equal to the integer ``1``. + +.. data:: FALSE + NO + OFF + + Falsy values, all equal to the integer ``0``. + +.. data:: N + S + E + W + NE + NW + SE + SW + NS + EW + NSEW + CENTER + + Compass directions (``'n'``, ``'s'``, ``'e'``, ``'w'`` and the diagonals and + edges) plus ``CENTER`` (``'center'``), used as values for the *anchor* and + *sticky* options and by methods such as :meth:`Misc.grid_anchor`. + +.. data:: LEFT + RIGHT + TOP + BOTTOM + + Sides for the *side* option of the packer (see :meth:`Pack.pack_configure`). + +.. data:: X + Y + BOTH + NONE + + Values for the *fill* option of the packer: ``'x'``, ``'y'``, ``'both'`` or + ``'none'``. + +.. data:: RAISED + SUNKEN + FLAT + RIDGE + GROOVE + SOLID + + Values for the *relief* option, which controls a widget's 3-D border. + +.. data:: HORIZONTAL + VERTICAL + + Values for the *orient* option of widgets such as :class:`Scale`, + :class:`Scrollbar` and :class:`PanedWindow`. + +.. data:: CHAR + WORD + + Values for the *wrap* option of the :class:`Text` widget, selecting line + wrapping on character or word boundaries. + +.. data:: BASELINE + + The text-alignment value ``'baseline'``. + +.. data:: INSIDE + OUTSIDE + + Values for the *bordermode* option of the placer (see + :meth:`Place.place_configure`). + +.. data:: INSERT + CURRENT + END + ANCHOR + SEL + SEL_FIRST + SEL_LAST + + Symbolic indices used by the :class:`Text`, :class:`Entry`, :class:`Listbox` + and :class:`Canvas` widgets, such as ``'insert'`` (the insertion cursor), + ``'current'``, ``'end'``, ``'anchor'`` and the bounds of the selection + (``'sel.first'`` and ``'sel.last'``). + +.. data:: ALL + + The special tag ``'all'``, which matches every item of a :class:`Canvas` or + every character of a :class:`Text` (for example ``canvas.delete(ALL)``). + +.. data:: NORMAL + DISABLED + ACTIVE + HIDDEN + + Values for the *state* option of various widgets and items. + +.. data:: CASCADE + CHECKBUTTON + COMMAND + RADIOBUTTON + SEPARATOR + + Menu entry types, used as the *itemType* argument of :meth:`Menu.add` and + :meth:`Menu.insert`. + +.. data:: SINGLE + BROWSE + MULTIPLE + EXTENDED + + Values for the *selectmode* option of the :class:`Listbox` widget. + +.. data:: PIESLICE + CHORD + ARC + + Values for the *style* option of :class:`Canvas` arc items. + +.. data:: BUTT + PROJECTING + ROUND + BEVEL + MITER + + Values for the *capstyle* (``'butt'``, ``'projecting'``, ``'round'``) and + *joinstyle* (``'round'``, ``'bevel'``, ``'miter'``) options of + :class:`Canvas` line items. + +.. data:: FIRST + LAST + + Values for the *arrow* option of :class:`Canvas` line items, indicating + which ends have arrowheads. + +.. data:: MOVETO + SCROLL + + The first argument passed by a :class:`Scrollbar` to the :meth:`XView.xview` + or :meth:`YView.yview` method of the scrolled widget. + +.. data:: UNITS + PAGES + + Values for the *what* argument of :meth:`XView.xview_scroll` and + :meth:`YView.yview_scroll`. + +.. data:: UNDERLINE + NUMERIC + DOTBOX + + Other option values: ``'underline'``, ``'numeric'`` and ``'dotbox'``. diff --git a/Doc/library/tkinter.scrolledtext.rst b/Doc/library/tkinter.scrolledtext.rst index 763e24929d74b51..37e26a486d10c0c 100644 --- a/Doc/library/tkinter.scrolledtext.rst +++ b/Doc/library/tkinter.scrolledtext.rst @@ -1,8 +1,7 @@ -:mod:`!tkinter.scrolledtext` --- Scrolled Text Widget +:mod:`!tkinter.scrolledtext` --- Scrolled text widget ===================================================== .. module:: tkinter.scrolledtext - :platform: Tk :synopsis: Text widget with a vertical scroll bar. .. sectionauthor:: Fred L. Drake, Jr. @@ -11,15 +10,17 @@ -------------- -The :mod:`tkinter.scrolledtext` module provides a class of the same name which +The :mod:`!tkinter.scrolledtext` module provides a class of the same name which implements a basic text widget which has a vertical scroll bar configured to do the "right thing." Using the :class:`ScrolledText` class is a lot easier than setting up a text widget and scroll bar directly. -The text widget and scrollbar are packed together in a :class:`Frame`, and the -methods of the :class:`Grid` and :class:`Pack` geometry managers are acquired -from the :class:`Frame` object. This allows the :class:`ScrolledText` widget to -be used directly to achieve most normal geometry management behavior. +The text widget and scrollbar are packed together in a :class:`~tkinter.Frame`, +and the methods of the :class:`~tkinter.Pack`, :class:`~tkinter.Grid` and +:class:`~tkinter.Place` geometry managers are acquired from the +:class:`~tkinter.Frame` object. +This allows the :class:`ScrolledText` widget to be used directly to achieve +most normal geometry management behavior. Should more specific control be necessary, the following attributes are available: diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index 628e9f945ac3658..68ea834747ebfe7 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -12,20 +12,29 @@ -------------- -The :mod:`tkinter.ttk` module provides access to the Tk themed widget set, -introduced in Tk 8.5. It provides additional benefits including anti-aliased font -rendering under X11 and window transparency (requiring a composition -window manager on X11). +The :mod:`!tkinter.ttk` module provides access to the Tk themed widget set, +introduced in Tk 8.5. +Its widgets adapt their appearance to the platform's native theme, +giving an application a better and more consistent look and feel +than the classic :mod:`tkinter` widgets, whose appearance is fixed. -The basic idea for :mod:`tkinter.ttk` is to separate, to the extent possible, +The basic idea for :mod:`!tkinter.ttk` is to separate, to the extent possible, the code implementing a widget's behavior from the code implementing its appearance. +Ttk widgets are used just like the classic :mod:`tkinter` widgets +and share the same machinery: +the widget hierarchy, the geometry managers, variable coupling and event binding. +Those foundational concepts are covered in the :mod:`tkinter` documentation +and are not repeated here. + +.. versionadded:: 3.1 + .. seealso:: - `Tk Widget Styling Support `_ - A document introducing theming support for Tk + `Tk Widget Styling Support (TIP #48) `_ + The Tcl Improvement Proposal that introduced the themed widget styling engine. Using Ttk @@ -40,28 +49,30 @@ To override the basic Tk widgets, the import should follow the Tk import:: from tkinter import * from tkinter.ttk import * -That code causes several :mod:`tkinter.ttk` widgets (:class:`Button`, +That code causes several :mod:`!tkinter.ttk` widgets (:class:`Button`, :class:`Checkbutton`, :class:`Entry`, :class:`Frame`, :class:`Label`, -:class:`LabelFrame`, :class:`Menubutton`, :class:`PanedWindow`, -:class:`Radiobutton`, :class:`Scale` and :class:`Scrollbar`) to +:class:`LabelFrame`, :class:`Menubutton`, :class:`OptionMenu`, +:class:`PanedWindow`, :class:`Radiobutton`, :class:`Scale`, +:class:`Scrollbar` and :class:`Spinbox`) to automatically replace the Tk widgets. -This has the direct benefit of using the new widgets which gives a better -look and feel across platforms; however, the replacement widgets are not -completely compatible. The main difference is that widget options such as -"fg", "bg" and others related to widget styling are no -longer present in Ttk widgets. Instead, use the :class:`ttk.Style` class -for improved styling effects. +.. note:: + Overriding the classic widgets with ``from tkinter.ttk import *`` + is convenient for adapting existing code, + but new code is usually clearer if it imports the module + as ``from tkinter import ttk`` and refers to the themed widgets explicitly, + such as ``ttk.Button``. -.. seealso:: - - `Converting existing applications to use Tile widgets `_ - A monograph (using Tcl terminology) about differences typically - encountered when moving applications to use the new widgets. +This has the direct benefit of using the new widgets which gives a better look +and feel across platforms; however, the replacement widgets are not completely +compatible. +The main difference is that widget options such as ``fg``, ``bg`` and others +related to widget styling are no longer present in Ttk widgets. +Instead, use the :class:`ttk.Style {%- endblock -%} {% block body %}

{{ docstitle|e }}

@@ -21,63 +50,76 @@

{{ docstitle|e }}

{% trans %}Welcome! This is the official documentation for Python {{ release }}.{% endtrans %}

{% trans %}Documentation sections:{% endtrans %}

- - -
- - - - - - - - - - - - - -
+
+ + +
+ +

{% trans %}Other resources:{% endtrans %}

+
+ + +

{% trans %}Indices, glossary, and search:{% endtrans %}

- - -
- - - - - - -
+
+ + +

{% trans %}Project information:{% endtrans %}

- - -
- - - - - - - -
+ {% endblock %} diff --git a/Doc/tools/templates/indexsidebar.html b/Doc/tools/templates/indexsidebar.html index eea29e2449a9cf3..086f15662cf87be 100644 --- a/Doc/tools/templates/indexsidebar.html +++ b/Doc/tools/templates/indexsidebar.html @@ -9,9 +9,9 @@

{% trans %}Docs by version{% endtrans %}

{% trans %}Other resources{% endtrans %}

diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 56023ebf9627036..54c6eb9b5ef7e47 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -26,11 +26,11 @@ {% endblock %} {% block extrahead %} - {% if builder == "html" and enable_analytics %} - - {% endif %} - - {% if builder != "htmlhelp" %} + {% if builder == "html" %} + {% if enable_analytics %} + + {% endif %} + {% if pagename == 'whatsnew/changelog' and not embedded %} {% endif %} {% endif %} diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index 6a1611afadb57c1..2539edb51ba3a09 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -14,8 +14,7 @@ There are two variants of the interactive :term:`REPL`. The classic basic interpreter is supported on all platforms with minimal line control capabilities. -On Windows, or Unix-like systems with :mod:`curses` support, -a new interactive shell is used by default. +Since Python 3.13, a new interactive shell is used by default. This one supports color, multiline editing, history browsing, and paste mode. To disable color, see :ref:`using-on-controlling-color` for details. Function keys provide some additional functionality. diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 9d0fab8861d2a96..3c6903430226186 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -359,7 +359,7 @@ Usually, a method is called right after it is bound:: x.f() -In the :class:`!MyClass` example, this will return the string ``'hello world'``. +If ``x = MyClass()``, as above, this will return the string ``'hello world'``. However, it is not necessary to call a method right away: ``x.f`` is a method object, and can be stored away and called at a later time. For example:: @@ -420,7 +420,7 @@ of the class:: 'Buddy' As discussed in :ref:`tut-object`, shared data can have possibly surprising -effects with involving :term:`mutable` objects such as lists and dictionaries. +effects involving :term:`mutable` objects such as lists and dictionaries. For example, the *tricks* list in the following code should not be used as a class variable because just a single list would be shared by all *Dog* instances:: diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 95939242fb7d44c..bee6cc39fafcdb9 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -155,8 +155,8 @@ that takes an iterable is :func:`sum`:: 6 Later we will see more functions that return iterables and take iterables as -arguments. In chapter :ref:`tut-structures`, we will discuss in more detail about -:func:`list`. +arguments. In chapter :ref:`tut-structures`, we will discuss :func:`list` in more +detail. .. _tut-break: @@ -251,6 +251,7 @@ statements: a ``try`` statement's ``else`` clause runs when no exception occurs, and a loop's ``else`` clause runs when no ``break`` occurs. For more on the ``try`` statement and exceptions, see :ref:`tut-handling`. +.. index:: single: ...; ellipsis literal .. _tut-pass: :keyword:`!pass` Statements @@ -277,6 +278,12 @@ at a more abstract level. The :keyword:`!pass` is silently ignored:: ... pass # Remember to implement this! ... +For this last case, many people use the ellipsis literal :code:`...` instead of +:code:`pass`. This use has no special meaning to Python, and is not part of +the language definition (you could use any constant expression here), but +:code:`...` is used conventionally as a placeholder body as well. +See :ref:`bltin-ellipsis-object`. + .. _tut-match: @@ -434,7 +441,7 @@ Several other key features of this statement: ``False`` and ``None`` are compared by identity. - Patterns may use named constants. These must be dotted names - to prevent them from being interpreted as capture variable:: + to prevent them from being interpreted as capture variables:: from enum import Enum class Color(Enum): @@ -561,7 +568,7 @@ This example, as usual, demonstrates some new Python features: Different types define different methods. Methods of different types may have the same name without causing ambiguity. (It is possible to define your own object types and methods, using *classes*, see :ref:`tut-classes`) - The method :meth:`!append` shown in the example is defined for list objects; it + The method :meth:`~list.append` shown in the example is defined for list objects; it adds a new element at the end of the list. In this example it is equivalent to ``result = result + [a]``, but more efficient. @@ -999,7 +1006,8 @@ scope:: 43 The above example uses a lambda expression to return a function. Another use -is to pass a small function as an argument:: +is to pass a small function as an argument. For instance, :meth:`list.sort` +takes a sorting key function *key* which can be a lambda function:: >>> pairs = [(1, 'one'), (2, 'two'), (3, 'three'), (4, 'four')] >>> pairs.sort(key=lambda pair: pair[1]) @@ -1031,31 +1039,28 @@ blank, visually separating the summary from the rest of the description. The following lines should be one or more paragraphs describing the object's calling conventions, its side effects, etc. -The Python parser does not strip indentation from multi-line string literals in -Python, so tools that process documentation have to strip indentation if -desired. This is done using the following convention. The first non-blank line -*after* the first line of the string determines the amount of indentation for -the entire documentation string. (We can't use the first line since it is -generally adjacent to the string's opening quotes so its indentation is not -apparent in the string literal.) Whitespace "equivalent" to this indentation is -then stripped from the start of all lines of the string. Lines that are -indented less should not occur, but if they occur all their leading whitespace -should be stripped. Equivalence of whitespace should be tested after expansion -of tabs (to 8 spaces, normally). +The Python parser strips indentation from multi-line string literals when they +serve as module, class, or function docstrings. Here is an example of a multi-line docstring:: >>> def my_function(): ... """Do nothing, but document it. ... - ... No, really, it doesn't do anything. + ... No, really, it doesn't do anything: + ... + ... >>> my_function() + ... >>> ... """ ... pass ... >>> print(my_function.__doc__) Do nothing, but document it. - No, really, it doesn't do anything. + No, really, it doesn't do anything: + + >>> my_function() + >>> .. _tut-annotations: @@ -1073,7 +1078,7 @@ Function Annotations information about the types used by user-defined functions (see :pep:`3107` and :pep:`484` for more information). -:term:`Annotations ` are stored in the :attr:`!__annotations__` +:term:`Annotations ` are stored in the :attr:`~object.__annotations__` attribute of the function as a dictionary and have no effect on any other part of the function. Parameter annotations are defined by a colon after the parameter name, followed by an expression evaluating to the value of the annotation. Return annotations are @@ -1102,7 +1107,7 @@ Intermezzo: Coding Style Now that you are about to write longer, more complex pieces of Python, it is a good time to talk about *coding style*. Most languages can be written (or more -concise, *formatted*) in different styles; some are more readable than others. +concisely, *formatted*) in different styles; some are more readable than others. Making it easy for others to read your code is always a good idea, and adopting a nice coding style helps tremendously for that. diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index cbe780e075baf59..db9b4cae1d489da 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -12,24 +12,23 @@ and adds some new things as well. More on Lists ============= -The list data type has some more methods. Here are all of the methods of list -objects: +The :ref:`list ` data type has some more methods. Here are all +of the methods of list objects: - -.. method:: list.append(x) +.. method:: list.append(value, /) :noindex: Add an item to the end of the list. Similar to ``a[len(a):] = [x]``. -.. method:: list.extend(iterable) +.. method:: list.extend(iterable, /) :noindex: Extend the list by appending all the items from the iterable. Similar to ``a[len(a):] = iterable``. -.. method:: list.insert(i, x) +.. method:: list.insert(index, value, /) :noindex: Insert an item at a given position. The first argument is the index of the @@ -37,14 +36,14 @@ objects: the list, and ``a.insert(len(a), x)`` is equivalent to ``a.append(x)``. -.. method:: list.remove(x) +.. method:: list.remove(value, /) :noindex: - Remove the first item from the list whose value is equal to *x*. It raises a + Remove the first item from the list whose value is equal to *value*. It raises a :exc:`ValueError` if there is no such item. -.. method:: list.pop([i]) +.. method:: list.pop(index=-1, /) :noindex: Remove the item at the given position in the list, and return it. If no index @@ -59,10 +58,10 @@ objects: Remove all items from the list. Similar to ``del a[:]``. -.. method:: list.index(x[, start[, end]]) +.. method:: list.index(value[, start[, stop]]) :noindex: - Return zero-based index in the list of the first item whose value is equal to *x*. + Return zero-based index of the first occurrence of *value* in the list. Raises a :exc:`ValueError` if there is no such item. The optional arguments *start* and *end* are interpreted as in the slice @@ -71,10 +70,10 @@ objects: sequence rather than the *start* argument. -.. method:: list.count(x) +.. method:: list.count(value, /) :noindex: - Return the number of times *x* appears in the list. + Return the number of times *value* appears in the list. .. method:: list.sort(*, key=None, reverse=False) @@ -142,8 +141,8 @@ Using Lists as Stacks The list methods make it very easy to use a list as a stack, where the last element added is the first element retrieved ("last-in, first-out"). To add an -item to the top of the stack, use :meth:`!append`. To retrieve an item from the -top of the stack, use :meth:`!pop` without an explicit index. For example:: +item to the top of the stack, use :meth:`~list.append`. To retrieve an item from the +top of the stack, use :meth:`~list.pop` without an explicit index. For example:: >>> stack = [3, 4, 5] >>> stack.append(6) @@ -340,7 +339,7 @@ The :keyword:`!del` statement ============================= There is a way to remove an item from a list given its index instead of its -value: the :keyword:`del` statement. This differs from the :meth:`!pop` method +value: the :keyword:`del` statement. This differs from the :meth:`~list.pop` method which returns a value. The :keyword:`!del` statement can also be used to remove slices from a list or clear the entire list (which we did earlier by assignment of an empty list to the slice). For example:: @@ -445,15 +444,19 @@ packing and sequence unpacking. Sets ==== -Python also includes a data type for *sets*. A set is an unordered collection -with no duplicate elements. Basic uses include membership testing and -eliminating duplicate entries. Set objects also support mathematical operations -like union, intersection, difference, and symmetric difference. +Python also includes a data type for :ref:`sets `. A set is +an unordered collection with no duplicate elements. Basic uses include +membership testing and eliminating duplicate entries. Set objects also +support mathematical operations like union, intersection, difference, and +symmetric difference. Curly braces or the :func:`set` function can be used to create sets. Note: to create an empty set you have to use ``set()``, not ``{}``; the latter creates an empty dictionary, a data structure that we discuss in the next section. +Because sets are unordered, iterating over them or printing them can +produce the elements in a different order than you expect. + Here is a brief demonstration:: >>> basket = {'apple', 'orange', 'apple', 'pear', 'orange', 'banana'} @@ -500,8 +503,8 @@ any immutable type; strings and numbers can always be keys. Tuples can be used as keys if they contain only strings, numbers, or tuples; if a tuple contains any mutable object either directly or indirectly, it cannot be used as a key. You can't use lists as keys, since lists can be modified in place using index -assignments, slice assignments, or methods like :meth:`!append` and -:meth:`!extend`. +assignments, slice assignments, or methods like :meth:`~list.append` and +:meth:`~list.extend`. It is best to think of a dictionary as a set of *key: value* pairs, with the requirement that the keys are unique (within one dictionary). A pair of @@ -512,8 +515,12 @@ dictionary; this is also the way dictionaries are written on output. The main operations on a dictionary are storing a value with some key and extracting the value given the key. It is also possible to delete a key:value pair with ``del``. If you store using a key that is already in use, the old -value associated with that key is forgotten. It is an error to extract a value -using a non-existent key. +value associated with that key is forgotten. + +Extracting a value for a non-existent key by subscripting (``d[key]``) raises a +:exc:`KeyError`. To avoid getting this error when trying to access a possibly +non-existent key, use the :meth:`~dict.get` method instead, which returns +``None`` (or a specified default value) if the key is not in the dictionary. Performing ``list(d)`` on a dictionary returns a list of all the keys used in the dictionary, in insertion order (if you want it sorted, just use @@ -528,6 +535,12 @@ Here is a small example using a dictionary:: {'jack': 4098, 'sape': 4139, 'guido': 4127} >>> tel['jack'] 4098 + >>> tel['irv'] + Traceback (most recent call last): + File "", line 1, in + KeyError: 'irv' + >>> print(tel.get('irv')) + None >>> del tel['sape'] >>> tel['irv'] = 4127 >>> tel diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index 1c20fa2f0b6ae58..3c6edf2c4793abd 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -121,9 +121,9 @@ A :keyword:`try` statement may have more than one *except clause*, to specify handlers for different exceptions. At most one handler will be executed. Handlers only handle exceptions that occur in the corresponding *try clause*, not in other handlers of the same :keyword:`!try` statement. An *except clause* -may name multiple exceptions as a parenthesized tuple, for example:: +may name multiple exceptions, for example:: - ... except (RuntimeError, TypeError, NameError): + ... except RuntimeError, TypeError, NameError: ... pass A class in an :keyword:`except` clause matches exceptions which are instances of the @@ -549,9 +549,9 @@ caught like any other exception. :: >>> try: ... f() ... except Exception as e: - ... print(f'caught {type(e)}: e') + ... print(f'caught {type(e)}: {e}') ... - caught : e + caught : there were problems (2 sub-exceptions) >>> By using ``except*`` instead of ``except``, we can selectively diff --git a/Doc/tutorial/index.rst b/Doc/tutorial/index.rst index 96791f88c867ab0..20fe161be4acc26 100644 --- a/Doc/tutorial/index.rst +++ b/Doc/tutorial/index.rst @@ -4,6 +4,10 @@ The Python Tutorial ###################### +.. Tip:: This tutorial is designed for + *programmers* that are new to the Python language, + **not** *beginners* who are new to programming. + Python is an easy to learn, powerful programming language. It has efficient high-level data structures and a simple but effective approach to object-oriented programming. Python's elegant syntax and dynamic typing, @@ -11,7 +15,7 @@ together with its interpreted nature, make it an ideal language for scripting and rapid application development in many areas on most platforms. The Python interpreter and the extensive standard library are freely available -in source or binary form for all major platforms from the Python web site, +in source or binary form for all major platforms from the Python website, https://www.python.org/, and may be freely distributed. The same site also contains distributions of and pointers to many free third party Python modules, programs and tools, and additional documentation. @@ -21,7 +25,8 @@ implemented in C or C++ (or other languages callable from C). Python is also suitable as an extension language for customizable applications. This tutorial introduces the reader informally to the basic concepts and -features of the Python language and system. It helps to have a Python +features of the Python language and system. Be aware that it expects you to +have a basic understanding of programming in general. It helps to have a Python interpreter handy for hands-on experience, but all examples are self-contained, so the tutorial can be read off-line as well. diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index 35b8c7cd8eb0498..a00f06cf46c41a3 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -95,10 +95,11 @@ Some examples:: >>> repr((x, y, ('spam', 'eggs'))) "(32.5, 40000, ('spam', 'eggs'))" -The :mod:`string` module contains a :class:`~string.Template` class that offers -yet another way to substitute values into strings, using placeholders like -``$x`` and replacing them with values from a dictionary, but offers much less -control of the formatting. +The :mod:`string` module contains support for a simple templating approach +based upon regular expressions, via :class:`string.Template`. +This offers yet another way to substitute values into strings, +using placeholders like ``$x`` and replacing them with values from a dictionary. +This syntax is easy to use, although it offers much less control for formatting. .. index:: single: formatted string literal diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst index 02e7de77322e99a..af01a4a96be2445 100644 --- a/Doc/tutorial/interpreter.rst +++ b/Doc/tutorial/interpreter.rst @@ -34,13 +34,13 @@ status. If that doesn't work, you can exit the interpreter by typing the following command: ``quit()``. The interpreter's line-editing features include interactive editing, history -substitution and code completion on systems that support the `GNU Readline -`_ library. +substitution and code completion on most systems. Perhaps the quickest check to see whether command line editing is supported is -typing :kbd:`Control-P` to the first Python prompt you get. If it beeps, you -have command line editing; see Appendix :ref:`tut-interacting` for an -introduction to the keys. If nothing appears to happen, or if ``^P`` is -echoed, command line editing isn't available; you'll only be able to use +typing a word in on the Python prompt, then pressing Left arrow (or :kbd:`Control-b`). +If the cursor moves, you have command line editing; see Appendix +:ref:`tut-interacting` for an introduction to the keys. +If nothing appears to happen, or if a sequence like ``^[[D`` or ``^B`` appears, +command line editing isn't available; you'll only be able to use backspace to remove characters from the current line. The interpreter operates somewhat like the Unix shell: when called with standard diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index bec5da8fd759acd..7778e37a9adaa95 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -13,10 +13,9 @@ end a multi-line command. .. only:: html - You can toggle the display of prompts and output by clicking on ``>>>`` - in the upper-right corner of an example box. If you hide the prompts - and output for an example, then you can easily copy and paste the input - lines into your interpreter. + You can use the "Copy" button (it appears in the upper-right corner + when hovering over or tapping a code example), which strips prompts + and omits output, to copy and paste the input lines into your interpreter. .. index:: single: # (hash); comment @@ -50,7 +49,7 @@ primary prompt, ``>>>``. (It shouldn't take long.) Numbers ------- -The interpreter acts as a simple calculator: you can type an expression at it +The interpreter acts as a simple calculator: you can type an expression into it and it will write the value. Expression syntax is straightforward: the operators ``+``, ``-``, ``*`` and ``/`` can be used to perform arithmetic; parentheses (``()``) can be used for grouping. @@ -147,6 +146,8 @@ Python can manipulate text (represented by type :class:`str`, so-called "``Yay! :)``". They can be enclosed in single quotes (``'...'``) or double quotes (``"..."``) with the same result [#]_. +.. code-block:: pycon + >>> 'spam eggs' # single quotes 'spam eggs' >>> "Paris rabbit got your back :)! Yay!" # double quotes @@ -183,11 +184,11 @@ If you don't want characters prefaced by ``\`` to be interpreted as special characters, you can use *raw strings* by adding an ``r`` before the first quote:: - >>> print('C:\some\name') # here \n means newline! - C:\some + >>> print('C:\this\name') # here \t means tab, \n means newline + C: his ame - >>> print(r'C:\some\name') # note the r before the quote - C:\some\name + >>> print(r'C:\this\name') # note the r before the quote + C:\this\name There is one subtle aspect to raw strings: a raw string may not end in an odd number of ``\`` characters; see @@ -419,7 +420,7 @@ type, i.e. it is possible to change their content:: [1, 8, 27, 64, 125] You can also add new items at the end of the list, by using -the :meth:`!list.append` *method* (we will see more about methods later):: +the :meth:`list.append` *method* (we will see more about methods later):: >>> cubes.append(216) # add the cube of 6 >>> cubes.append(7 ** 3) # and the cube of 7 diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index de7aa0e23429468..f8105cd5441fec2 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -27,14 +27,16 @@ called :file:`fibo.py` in the current directory with the following contents:: # Fibonacci numbers module - def fib(n): # write Fibonacci series up to n + def fib(n): + """Write Fibonacci series up to n.""" a, b = 0, 1 while a < n: print(a, end=' ') a, b = b, a+b print() - def fib2(n): # return Fibonacci series up to n + def fib2(n): + """Return Fibonacci series up to n.""" result = [] a, b = 0, 1 while a < n: @@ -577,8 +579,8 @@ module for example, you might use:: from .. import formats from ..filters import equalizer -Note that relative imports are based on the name of the current module. Since -the name of the main module is always ``"__main__"``, modules intended for use +Note that relative imports are based on the name of the current module's package. +Since the main module does not have a package, modules intended for use as the main module of a Python application must always use absolute imports. diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst index 4b3eef313e76d72..320d365313225ce 100644 --- a/Doc/tutorial/stdlib.rst +++ b/Doc/tutorial/stdlib.rst @@ -1,13 +1,13 @@ .. _tut-brieftour: ********************************** -Brief Tour of the Standard Library +Brief tour of the standard library ********************************** .. _tut-os-interface: -Operating System Interface +Operating system interface ========================== The :mod:`os` module provides dozens of functions for interacting with the @@ -47,7 +47,7 @@ a higher level interface that is easier to use:: .. _tut-file-wildcards: -File Wildcards +File wildcards ============== The :mod:`glob` module provides a function for making file lists from directory @@ -60,7 +60,7 @@ wildcard searches:: .. _tut-command-line-arguments: -Command Line Arguments +Command-line arguments ====================== Common utility scripts often need to process command line arguments. These @@ -97,7 +97,7 @@ to ``['alpha.txt', 'beta.txt']``. .. _tut-stderr: -Error Output Redirection and Program Termination +Error output redirection and program termination ================================================ The :mod:`sys` module also has attributes for *stdin*, *stdout*, and *stderr*. @@ -112,7 +112,7 @@ The most direct way to terminate a script is to use ``sys.exit()``. .. _tut-string-pattern-matching: -String Pattern Matching +String pattern matching ======================= The :mod:`re` module provides regular expression tools for advanced string @@ -175,7 +175,7 @@ computations. .. _tut-internet-access: -Internet Access +Internet access =============== There are a number of modules for accessing the internet and processing internet @@ -183,13 +183,13 @@ protocols. Two of the simplest are :mod:`urllib.request` for retrieving data from URLs and :mod:`smtplib` for sending mail:: >>> from urllib.request import urlopen - >>> with urlopen('http://worldtimeapi.org/api/timezone/etc/UTC.txt') as response: + >>> with urlopen('https://docs.python.org/3/') as response: ... for line in response: ... line = line.decode() # Convert bytes to a str - ... if line.startswith('datetime'): + ... if 'updated' in line: ... print(line.rstrip()) # Remove trailing newline ... - datetime: 2022-01-01T01:36:47.689215+00:00 + Last updated on Nov 11, 2025 (20:11 UTC). >>> import smtplib >>> server = smtplib.SMTP('localhost') @@ -206,7 +206,7 @@ from URLs and :mod:`smtplib` for sending mail:: .. _tut-dates-and-times: -Dates and Times +Dates and times =============== The :mod:`datetime` module supplies classes for manipulating dates and times in @@ -216,15 +216,15 @@ formatting and manipulation. The module also supports objects that are timezone aware. :: >>> # dates are easily constructed and formatted - >>> from datetime import date - >>> now = date.today() + >>> import datetime as dt + >>> now = dt.date.today() >>> now datetime.date(2003, 12, 2) >>> now.strftime("%m-%d-%y. %d %b %Y is a %A on the %d day of %B.") '12-02-03. 02 Dec 2003 is a Tuesday on the 02 day of December.' >>> # dates support calendar arithmetic - >>> birthday = date(1964, 7, 31) + >>> birthday = dt.date(1964, 7, 31) >>> age = now - birthday >>> age.days 14368 @@ -232,7 +232,7 @@ aware. :: .. _tut-data-compression: -Data Compression +Data compression ================ Common data archiving and compression formats are directly supported by modules @@ -254,7 +254,7 @@ including: :mod:`zlib`, :mod:`gzip`, :mod:`bz2`, :mod:`lzma`, :mod:`zipfile` and .. _tut-performance-measurement: -Performance Measurement +Performance measurement ======================= Some Python users develop a deep interest in knowing the relative performance of @@ -278,7 +278,7 @@ larger blocks of code. .. _tut-quality-control: -Quality Control +Quality control =============== One approach for developing high quality software is to write tests for each @@ -324,7 +324,7 @@ file:: .. _tut-batteries-included: -Batteries Included +Batteries included ================== Python has a "batteries included" philosophy. This is best seen through the @@ -335,7 +335,7 @@ sophisticated and robust capabilities of its larger packages. For example: names, no direct knowledge or handling of XML is needed. * The :mod:`email` package is a library for managing email messages, including - MIME and other :rfc:`2822`-based message documents. Unlike :mod:`smtplib` and + MIME and other :rfc:`5322`-based message documents. Unlike :mod:`smtplib` and :mod:`poplib` which actually send and receive messages, the email package has a complete toolset for building or decoding complex message structures (including attachments) and for implementing internet encoding and header diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst index a2f96b34b2dead3..cc28f6ef6c5fdc8 100644 --- a/Doc/tutorial/stdlib2.rst +++ b/Doc/tutorial/stdlib2.rst @@ -1,7 +1,7 @@ .. _tut-brieftourtwo: ********************************************** -Brief Tour of the Standard Library --- Part II +Brief tour of the standard library --- part II ********************************************** This second tour covers more advanced modules that support professional @@ -10,7 +10,7 @@ programming needs. These modules rarely occur in small scripts. .. _tut-output-formatting: -Output Formatting +Output formatting ================= The :mod:`reprlib` module provides a version of :func:`repr` customized for @@ -130,7 +130,7 @@ templates for XML files, plain text reports, and HTML web reports. .. _tut-binary-formats: -Working with Binary Data Record Layouts +Working with binary data record layouts ======================================= The :mod:`struct` module provides :func:`~struct.pack` and @@ -178,14 +178,13 @@ tasks in background while the main program continues to run:: class AsyncZip(threading.Thread): def __init__(self, infile, outfile): - threading.Thread.__init__(self) + super().__init__() self.infile = infile self.outfile = outfile def run(self): - f = zipfile.ZipFile(self.outfile, 'w', zipfile.ZIP_DEFLATED) - f.write(self.infile) - f.close() + with zipfile.ZipFile(self.outfile, 'w', zipfile.ZIP_DEFLATED) as f: + f.write(self.infile) print('Finished background zip of:', self.infile) background = AsyncZip('mydata.txt', 'myarchive.zip') @@ -245,7 +244,7 @@ application. .. _tut-weak-references: -Weak References +Weak references =============== Python does automatic memory management (reference counting for most objects and @@ -286,7 +285,7 @@ applications include caching objects that are expensive to create:: .. _tut-list-tools: -Tools for Working with Lists +Tools for working with lists ============================ Many data structure needs can be met with the built-in list type. However, @@ -352,7 +351,7 @@ not want to run a full list sort:: .. _tut-decimal-fp: -Decimal Floating-Point Arithmetic +Decimal floating-point arithmetic ================================= The :mod:`decimal` module offers a :class:`~decimal.Decimal` datatype for diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index f362e1943b666f7..6b6b8a768e3f58d 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -88,6 +88,11 @@ For example: '~/envs/tutorial-env/lib/python3.5/site-packages'] >>> +Note that the activated virtual environment does not alter the ``PYTHONPATH`` variable in any way. +This may lead to unexpected results if the path includes references to code which is incompatible with +the Python version the virtual environment is using. The best practice is to ``unset PYTHONPATH`` +in bash or the equivalent for the shell you are using. + To deactivate a virtual environment, type:: deactivate diff --git a/Doc/tutorial/whatnow.rst b/Doc/tutorial/whatnow.rst index dbe2d7fc09927e2..aae8f29b0077627 100644 --- a/Doc/tutorial/whatnow.rst +++ b/Doc/tutorial/whatnow.rst @@ -30,7 +30,7 @@ the set are: More Python resources: -* https://www.python.org: The major Python web site. It contains code, +* https://www.python.org: The major Python website. It contains code, documentation, and pointers to Python-related pages around the web. * https://docs.python.org: Fast access to Python's documentation. @@ -68,6 +68,6 @@ already contain the solution for your problem. .. rubric:: Footnotes -.. [#] "Cheese Shop" is a Monty Python's sketch: a customer enters a cheese shop, +.. [#] "Cheese Shop" is a Monty Python sketch: a customer enters a cheese shop, but whatever cheese he asks for, the clerk says it's missing. diff --git a/Doc/using/android.rst b/Doc/using/android.rst index 65bf23dc9948560..60a135693181412 100644 --- a/Doc/using/android.rst +++ b/Doc/using/android.rst @@ -30,7 +30,7 @@ Adding Python to an Android app Most app developers should use one of the following tools, which will provide a much easier experience: -* `Briefcase `__, from the BeeWare project +* `Briefcase `__, from the BeeWare project * `Buildozer `__, from the Kivy project * `Chaquopy `__ * `pyqtdeploy `__ @@ -40,8 +40,15 @@ If you're sure you want to do all of this manually, read on. You can use the :source:`testbed app ` as a guide; each step below contains a link to the relevant file. -* Build Python by following the instructions in :source:`Android/README.md`. - This will create the directory ``cross-build/HOST/prefix``. +* First, acquire a build of Python for Android: + + * The easiest way is to download an Android release from `python.org + `__. The ``prefix`` directory + mentioned below is at the top level of the package. + + * Or if you want to build it yourself, follow the instructions in + :source:`Android/README.md`. The ``prefix`` directory will be created under + :samp:`cross-build/{HOST}`. * Add code to your :source:`build.gradle ` file to copy the following items into your project. All except your own Python @@ -63,3 +70,12 @@ link to the relevant file. * Add code to your app to :source:`start Python in embedded mode `. This will need to be C code called via JNI. + +Building a Python package for Android +------------------------------------- + +Python packages can be built for Android as wheels and released on PyPI. The +recommended tool for doing this is `cibuildwheel +`__, which automates +all the details of setting up a cross-compilation environment, building the +wheel, and testing it on an emulator. diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 40a46a62031ef7b..0c947de005b58ea 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -50,8 +50,8 @@ additional methods of invocation: * When called with ``-c command``, it executes the Python statement(s) given as *command*. Here *command* may contain multiple statements separated by newlines. Leading whitespace is significant in Python statements! -* When called with ``-m module-name``, the given module is located on the - Python module path and executed as a script. +* When called with ``-m module-name``, the given module is located using the standard + import mechanism and executed as a script. In non-interactive mode, the entire input is parsed before it is executed. @@ -78,8 +78,8 @@ source. .. option:: -m - Search :data:`sys.path` for the named module and execute its contents as - the :mod:`__main__` module. + Locate the module using the standard import mechanism and execute its contents + as the :mod:`__main__` module. Since the argument is a *module* name, you must not give a file extension (``.py``). The module name should be a valid absolute Python module name, but @@ -369,8 +369,8 @@ Miscellaneous options .. option:: -R Turn on hash randomization. This option only has an effect if the - :envvar:`PYTHONHASHSEED` environment variable is set to ``0``, since hash - randomization is enabled by default. + :envvar:`PYTHONHASHSEED` environment variable is set to anything other + than ``random``, since hash randomization is enabled by default. On previous versions of Python, this option turns on hash randomization, so that the :meth:`~object.__hash__` values of str and bytes objects @@ -381,7 +381,7 @@ Miscellaneous options Hash randomization is intended to provide protection against a denial-of-service caused by carefully chosen inputs that exploit the worst case performance of a dict construction, *O*\ (*n*\ :sup:`2`) complexity. See - http://ocert.org/advisories/ocert-2011-003.html for details. + https://ocert.org/advisories/ocert-2011-003.html for details. :envvar:`PYTHONHASHSEED` allows you to set a fixed value for the hash seed secret. @@ -653,7 +653,7 @@ Miscellaneous options .. versionadded:: 3.13 * :samp:`-X thread_inherit_context={0,1}` causes :class:`~threading.Thread` - to, by default, use a copy of context of of the caller of + to, by default, use a copy of context of the caller of ``Thread.start()`` when starting. Otherwise, threads will start with an empty context. If unset, the value of this option defaults to ``1`` on free-threaded builds and to ``0`` otherwise. See also @@ -669,6 +669,13 @@ Miscellaneous options .. versionadded:: 3.14 + * :samp:`-X tlbc={0,1}` enables (1, the default) or disables (0) thread-local + bytecode in builds configured with :option:`--disable-gil`. When disabled, + this also disables the specializing interpreter. See also + :envvar:`PYTHON_TLBC`. + + .. versionadded:: 3.14 + It also allows passing arbitrary values and retrieving them through the :data:`sys._xoptions` dictionary. @@ -1038,6 +1045,13 @@ conflict. * ``pymalloc_debug``: same as ``pymalloc`` but also install debug hooks. * ``mimalloc_debug``: same as ``mimalloc`` but also install debug hooks. + .. note:: + + In the :term:`free-threaded ` build, the ``malloc``, + ``malloc_debug``, ``pymalloc``, and ``pymalloc_debug`` values are not + supported. Only ``default``, ``debug``, ``mimalloc``, and + ``mimalloc_debug`` are accepted. + .. versionadded:: 3.6 .. versionchanged:: 3.7 @@ -1047,12 +1061,13 @@ conflict. .. envvar:: PYTHONMALLOCSTATS If set to a non-empty string, Python will print statistics of the - :ref:`pymalloc memory allocator ` every time a new pymalloc object - arena is created, and on shutdown. + :ref:`pymalloc memory allocator ` or the + :ref:`mimalloc memory allocator ` (whichever is in use) + every time a new object arena is created, and on shutdown. This variable is ignored if the :envvar:`PYTHONMALLOC` environment variable is used to force the :c:func:`malloc` allocator of the C library, or if - Python is configured without ``pymalloc`` support. + Python is configured without both ``pymalloc`` and ``mimalloc`` support. .. versionchanged:: 3.6 This variable can now also be used on Python compiled in release mode. @@ -1249,9 +1264,8 @@ conflict. .. envvar:: PYTHON_BASIC_REPL If this variable is set to any value, the interpreter will not attempt to - load the Python-based :term:`REPL` that requires :mod:`curses` and - :mod:`readline`, and will instead use the traditional parser-based - :term:`REPL`. + load the Python-based :term:`REPL` that requires :mod:`readline`, and will + instead use the traditional parser-based :term:`REPL`. .. versionadded:: 3.13 @@ -1277,7 +1291,7 @@ conflict. .. envvar:: PYTHON_THREAD_INHERIT_CONTEXT If this variable is set to ``1`` then :class:`~threading.Thread` will, - by default, use a copy of context of of the caller of ``Thread.start()`` + by default, use a copy of context of the caller of ``Thread.start()`` when starting. Otherwise, new threads will start with an empty context. If unset, this variable defaults to ``1`` on free-threaded builds and to ``0`` otherwise. See also :option:`-X thread_inherit_context<-X>`. @@ -1302,6 +1316,16 @@ conflict. .. versionadded:: 3.13 +.. envvar:: PYTHON_TLBC + + If set to ``1`` enables thread-local bytecode. If set to ``0`` thread-local + bytecode and the specializing interpreter are disabled. Only applies to + builds configured with :option:`--disable-gil`. + + See also the :option:`-X tlbc <-X>` command-line option. + + .. versionadded:: 3.14 + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index b914d3397b6dda4..2c0d81d48007013 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -4,10 +4,13 @@ Configure Python .. highlight:: sh + +.. _build-requirements: + Build Requirements ================== -Features and minimum versions required to build CPython: +To build CPython, you will need: * A `C11 `_ compiler. `Optional C11 features @@ -22,47 +25,136 @@ Features and minimum versions required to build CPython: * Support for threads. -* OpenSSL 1.1.1 is the minimum version and OpenSSL 3.0.16 is the recommended - minimum version for the :mod:`ssl` and :mod:`hashlib` extension modules. +.. versionchanged:: 3.5 + On Windows, Visual Studio 2015 or later is now required. + +.. versionchanged:: 3.6 + Selected C99 features, like ```` and ``static inline`` functions, + are now required. + +.. versionchanged:: 3.7 + Thread support is now required. + +.. versionchanged:: 3.11 + C11 compiler, IEEE 754 and NaN support are now required. + On Windows, Visual Studio 2017 or later is required. -* SQLite 3.15.2 for the :mod:`sqlite3` extension module. +See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform +support". -* Tcl/Tk 8.5.12 for the :mod:`tkinter` module. -* Autoconf 2.72 and aclocal 1.16.5 are required to regenerate the - :file:`configure` script. +.. _optional-module-requirements: + +Requirements for optional modules +--------------------------------- + +Some :term:`optional modules ` of the standard library +require third-party libraries installed for development +(for example, header files must be available). + +Missing requirements are reported in the ``configure`` output. +Modules that are missing due to missing dependencies are listed near the end +of the ``make`` output, +sometimes using an internal name, for example, ``_ctypes`` for :mod:`ctypes` +module. + +If you distribute a CPython interpreter without optional modules, +it's best practice to advise users, who generally expect that +standard library modules are available. + +Dependencies to build optional modules are: + +.. list-table:: + :header-rows: 1 + :align: left + + * - Dependency + - Minimum version + - Python module + * - `libbz2 `_ + - + - :mod:`bz2` + * - `libffi `_ + - 3.3.0 recommended + - :mod:`ctypes` + * - `liblzma `_ + - + - :mod:`lzma` + * - `libmpdec `_ + - 2.5.0 + - :mod:`decimal` [1]_ + * - `libreadline `_ or + `libedit `_ [2]_ + - + - :mod:`readline` + * - `libuuid `_ + - + - ``_uuid`` [3]_ + * - `ncurses `_ [4]_ + - + - :mod:`curses` + * - `OpenSSL `_ + - | 3.0.18 recommended + | (1.1.1 minimum) + - :mod:`ssl`, :mod:`hashlib` [5]_ + * - `SQLite `_ + - 3.15.2 + - :mod:`sqlite3` + * - `Tcl/Tk `_ + - 8.5.12 + - :mod:`tkinter`, :ref:`IDLE `, :mod:`turtle` + * - `zlib `_ + - 1.2.2.1 + - :mod:`zlib`, :mod:`gzip`, :mod:`ensurepip` + * - `zstd `_ + - 1.4.5 + - :mod:`compression.zstd` + +.. [1] If *libmpdec* is not available, the :mod:`decimal` module will use + a pure-Python implementation. + See :option:`--with-system-libmpdec` for details. +.. [2] See :option:`--with-readline` for choosing the backend for the + :mod:`readline` module. +.. [3] The :mod:`uuid` module uses ``_uuid`` to generate "safe" UUIDs. + See the module documentation for details. +.. [4] The :mod:`curses` module requires the ``libncurses`` or ``libncursesw`` + library. + The :mod:`curses.panel` module additionally requires the ``libpanel`` or + ``libpanelw`` library. +.. [5] If OpenSSL is not available, the :mod:`hashlib` module will use + bundled implementations of several hash functions. + See :option:`--with-builtin-hashlib-hashes` for *forcing* usage of OpenSSL. + +Note that the table does not include all optional modules; in particular, +platform-specific modules like :mod:`winreg` are not listed here. + +.. seealso:: + + * The `devguide `_ + includes a full list of dependencies required to build all modules and + instructions on how to install them on common platforms. + * :option:`--with-system-expat` allows building with an external + `libexpat `_ library. + * :ref:`configure-options-for-dependencies` .. versionchanged:: 3.1 - Tcl/Tk version 8.3.1 is now required. + Tcl/Tk version 8.3.1 is now required for :mod:`tkinter`. .. versionchanged:: 3.5 - On Windows, Visual Studio 2015 or later is now required. - Tcl/Tk version 8.4 is now required. - -.. versionchanged:: 3.6 - Selected C99 features are now required, like ```` and ``static - inline`` functions. + Tcl/Tk version 8.4 is now required for :mod:`tkinter`. .. versionchanged:: 3.7 - Thread support and OpenSSL 1.0.2 are now required. + OpenSSL 1.0.2 is now required for :mod:`hashlib` and :mod:`ssl`. .. versionchanged:: 3.10 - OpenSSL 1.1.1 is now required. - Require SQLite 3.7.15. + OpenSSL 1.1.1 is now required for :mod:`hashlib` and :mod:`ssl`. + SQLite 3.7.15 is now required for :mod:`sqlite3`. .. versionchanged:: 3.11 - C11 compiler, IEEE 754 and NaN support are now required. - On Windows, Visual Studio 2017 or later is required. - Tcl/Tk version 8.5.12 is now required for the :mod:`tkinter` module. + Tcl/Tk version 8.5.12 is now required for :mod:`tkinter`. .. versionchanged:: 3.13 - Autoconf 2.71, aclocal 1.16.5 and SQLite 3.15.2 are now required. - -.. versionchanged:: 3.14 - Autoconf 2.72 is now required. - -See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform -support". + SQLite 3.15.2 is now required for :mod:`sqlite3`. Generated files @@ -91,8 +183,19 @@ The container is optional, the following command can be run locally:: autoreconf -ivf -Werror -The generated files can change depending on the exact ``autoconf-archive``, -``aclocal`` and ``pkg-config`` versions. +The generated files can change depending on the exact versions of the +tools used. +The container that CPython uses has +`Autoconf `_ 2.72, +``aclocal`` from `Automake `_ 1.16.5, +and `pkg-config `_ 1.8.1. + +.. versionchanged:: 3.13 + Autoconf 2.71 and aclocal 1.16.5 and are now used to regenerate + :file:`configure`. + +.. versionchanged:: 3.14 + Autoconf 2.72 is now used to regenerate :file:`configure`. .. _configure-options: @@ -290,8 +393,8 @@ General Options .. option:: --disable-gil - Enables **experimental** support for running Python without the - :term:`global interpreter lock` (GIL): free threading build. + Enables support for running Python without the :term:`global interpreter + lock` (GIL): :term:`free-threaded build`. Defines the ``Py_GIL_DISABLED`` macro and adds ``"t"`` to :data:`sys.abiflags`. @@ -370,6 +473,8 @@ Linker options Name for machine-dependent library files. +.. _configure-options-for-dependencies: + Options for third-party dependencies ------------------------------------ @@ -392,12 +497,6 @@ Options for third-party dependencies C compiler and linker flags for ``gdbm``. -.. option:: LIBB2_CFLAGS -.. option:: LIBB2_LIBS - - C compiler and linker flags for ``libb2`` (:ref:`BLAKE2 `), - used by :mod:`hashlib` module, overriding ``pkg-config``. - .. option:: LIBEDIT_CFLAGS .. option:: LIBEDIT_LIBS @@ -445,6 +544,14 @@ Options for third-party dependencies C compiler and linker flags for ``libuuid``, used by :mod:`uuid` module, overriding ``pkg-config``. +.. option:: LIBZSTD_CFLAGS +.. option:: LIBZSTD_LIBS + + C compiler and linker flags for ``libzstd``, used by :mod:`compression.zstd` module, + overriding ``pkg-config``. + + .. versionadded:: 3.14 + .. option:: PANEL_CFLAGS .. option:: PANEL_LIBS @@ -640,6 +747,9 @@ also be used to improve performance. Disable the fast :ref:`mimalloc ` allocator (enabled by default). + This option cannot be used together with :option:`--disable-gil` + because the :term:`free-threaded ` build requires mimalloc. + See also :envvar:`PYTHONMALLOC` environment variable. .. option:: --without-pymalloc @@ -675,6 +785,13 @@ also be used to improve performance. not compiled. This includes both the functionality to schedule code to be executed and the functionality to receive code to be executed. + .. c:macro:: Py_REMOTE_DEBUG + + This macro is defined by default, unless Python is configured with + :option:`--without-remote-debug`. + + Note that even if the macro is defined, remote debugging may not be + available (for example, on an incompatible platform). .. versionadded:: 3.14 @@ -721,9 +838,11 @@ See also the :ref:`Python Development Mode ` and the :option:`--with-trace-refs` configure option. .. versionchanged:: 3.8 - Release builds and debug builds are now ABI compatible: defining the + Release builds are now ABI compatible with debug builds: defining the ``Py_DEBUG`` macro no longer implies the ``Py_TRACE_REFS`` macro (see the - :option:`--with-trace-refs` option). + :option:`--with-trace-refs` option). However, debug builds still expose + more symbols than release builds and code built against a debug build is not + necessarily compatible with a release build. Debug options @@ -784,6 +903,9 @@ Debug options .. option:: --with-address-sanitizer Enable AddressSanitizer memory error detector, ``asan`` (default is no). + To improve ASan detection capabilities you may also want to combine this + with :option:`--without-pymalloc` to disable the specialized small-object + allocator whose allocations are not tracked by ASan. .. versionadded:: 3.6 @@ -845,9 +967,16 @@ Libraries options .. versionchanged:: 3.13 Default to using the installed ``mpdecimal`` library. - .. deprecated-removed:: 3.13 3.15 + .. versionchanged:: 3.15 + + A bundled copy of the library will no longer be selected + implicitly if an installed ``mpdecimal`` library is not found. + In Python 3.15 only, it can still be selected explicitly using + ``--with-system-libmpdec=no`` or ``--without-system-libmpdec``. + + .. deprecated-removed:: 3.13 3.16 A copy of the ``mpdecimal`` library sources will no longer be distributed - with Python 3.15. + with Python 3.16. .. seealso:: :option:`LIBMPDEC_CFLAGS` and :option:`LIBMPDEC_LIBS`. @@ -1377,6 +1506,12 @@ Compiler flags .. versionadded:: 3.7 +.. envvar:: CFLAGS_CEVAL + + Flags used to compile ``Python/ceval.c``. + + .. versionadded:: 3.14.5 + .. envvar:: CCSHARED Compiler flags used to build a shared library. diff --git a/Doc/using/ios.rst b/Doc/using/ios.rst index 7d5c6331bef5ce0..5e4033fb6cec7a0 100644 --- a/Doc/using/ios.rst +++ b/Doc/using/ios.rst @@ -170,7 +170,7 @@ helpful. To add Python to an iOS Xcode project: 1. Build or obtain a Python ``XCFramework``. See the instructions in - :source:`iOS/README.rst` (in the CPython source distribution) for details on + :source:`Apple/iOS/README.md` (in the CPython source distribution) for details on how to build a Python ``XCFramework``. At a minimum, you will need a build that supports ``arm64-apple-ios``, plus one of either ``arm64-apple-ios-simulator`` or ``x86_64-apple-ios-simulator``. @@ -180,22 +180,19 @@ To add Python to an iOS Xcode project: of your project; however, you can use any other location that you want by adjusting paths as needed. -3. Drag the ``iOS/Resources/dylib-Info-template.plist`` file into your project, - and ensure it is associated with the app target. - -4. Add your application code as a folder in your Xcode project. In the +3. Add your application code as a folder in your Xcode project. In the following instructions, we'll assume that your user code is in a folder named ``app`` in the root of your project; you can use any other location by adjusting paths as needed. Ensure that this folder is associated with your app target. -5. Select the app target by selecting the root node of your Xcode project, then +4. Select the app target by selecting the root node of your Xcode project, then the target name in the sidebar that appears. -6. In the "General" settings, under "Frameworks, Libraries and Embedded +5. In the "General" settings, under "Frameworks, Libraries and Embedded Content", add ``Python.xcframework``, with "Embed & Sign" selected. -7. In the "Build Settings" tab, modify the following: +6. In the "Build Settings" tab, modify the following: - Build Options @@ -211,86 +208,24 @@ To add Python to an iOS Xcode project: * Quoted Include In Framework Header: No -8. Add a build step that copies the Python standard library into your app. In - the "Build Phases" tab, add a new "Run Script" build step *before* the - "Embed Frameworks" step, but *after* the "Copy Bundle Resources" step. Name - the step "Install Target Specific Python Standard Library", disable the - "Based on dependency analysis" checkbox, and set the script content to: +7. Add a build step that processes the Python standard library, and your own + Python binary dependencies. In the "Build Phases" tab, add a new "Run + Script" build step *before* the "Embed Frameworks" step, but *after* the + "Copy Bundle Resources" step. Name the step "Process Python libraries", + disable the "Based on dependency analysis" checkbox, and set the script + content to: .. code-block:: bash - set -e - - mkdir -p "$CODESIGNING_FOLDER_PATH/python/lib" - if [ "$EFFECTIVE_PLATFORM_NAME" = "-iphonesimulator" ]; then - echo "Installing Python modules for iOS Simulator" - rsync -au --delete "$PROJECT_DIR/Python.xcframework/ios-arm64_x86_64-simulator/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" - else - echo "Installing Python modules for iOS Device" - rsync -au --delete "$PROJECT_DIR/Python.xcframework/ios-arm64/lib/" "$CODESIGNING_FOLDER_PATH/python/lib/" - fi - - Note that the name of the simulator "slice" in the XCframework may be - different, depending the CPU architectures your ``XCFramework`` supports. - -9. Add a second build step that processes the binary extension modules in the - standard library into "Framework" format. Add a "Run Script" build step - *directly after* the one you added in step 8, named "Prepare Python Binary - Modules". It should also have "Based on dependency analysis" unchecked, with - the following script content: + set -e + source $PROJECT_DIR/Python.xcframework/build/build_utils.sh + install_python Python.xcframework app - .. code-block:: bash + If you have placed your XCframework somewhere other than the root of your + project, modify the path to the first argument. - set -e - - install_dylib () { - INSTALL_BASE=$1 - FULL_EXT=$2 - - # The name of the extension file - EXT=$(basename "$FULL_EXT") - # The location of the extension file, relative to the bundle - RELATIVE_EXT=${FULL_EXT#$CODESIGNING_FOLDER_PATH/} - # The path to the extension file, relative to the install base - PYTHON_EXT=${RELATIVE_EXT/$INSTALL_BASE/} - # The full dotted name of the extension module, constructed from the file path. - FULL_MODULE_NAME=$(echo $PYTHON_EXT | cut -d "." -f 1 | tr "/" "."); - # A bundle identifier; not actually used, but required by Xcode framework packaging - FRAMEWORK_BUNDLE_ID=$(echo $PRODUCT_BUNDLE_IDENTIFIER.$FULL_MODULE_NAME | tr "_" "-") - # The name of the framework folder. - FRAMEWORK_FOLDER="Frameworks/$FULL_MODULE_NAME.framework" - - # If the framework folder doesn't exist, create it. - if [ ! -d "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" ]; then - echo "Creating framework for $RELATIVE_EXT" - mkdir -p "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER" - cp "$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" - plutil -replace CFBundleExecutable -string "$FULL_MODULE_NAME" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" - plutil -replace CFBundleIdentifier -string "$FRAMEWORK_BUNDLE_ID" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/Info.plist" - fi - - echo "Installing binary for $FRAMEWORK_FOLDER/$FULL_MODULE_NAME" - mv "$FULL_EXT" "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" - # Create a placeholder .fwork file where the .so was - echo "$FRAMEWORK_FOLDER/$FULL_MODULE_NAME" > ${FULL_EXT%.so}.fwork - # Create a back reference to the .so file location in the framework - echo "${RELATIVE_EXT%.so}.fwork" > "$CODESIGNING_FOLDER_PATH/$FRAMEWORK_FOLDER/$FULL_MODULE_NAME.origin" - } - - PYTHON_VER=$(ls -1 "$CODESIGNING_FOLDER_PATH/python/lib") - echo "Install Python $PYTHON_VER standard library extension modules..." - find "$CODESIGNING_FOLDER_PATH/python/lib/$PYTHON_VER/lib-dynload" -name "*.so" | while read FULL_EXT; do - install_dylib python/lib/$PYTHON_VER/lib-dynload/ "$FULL_EXT" - done - - # Clean up dylib template - rm -f "$CODESIGNING_FOLDER_PATH/dylib-Info-template.plist" - - echo "Signing frameworks as $EXPANDED_CODE_SIGN_IDENTITY_NAME ($EXPANDED_CODE_SIGN_IDENTITY)..." - find "$CODESIGNING_FOLDER_PATH/Frameworks" -name "*.framework" -exec /usr/bin/codesign --force --sign "$EXPANDED_CODE_SIGN_IDENTITY" ${OTHER_CODE_SIGN_FLAGS:-} -o runtime --timestamp=none --preserve-metadata=identifier,entitlements,flags --generate-entitlement-der "{}" \; - -10. Add Objective C code to initialize and use a Python interpreter in embedded - mode. You should ensure that: +8. Add Objective C code to initialize and use a Python interpreter in embedded + mode. You should ensure that: * UTF-8 mode (:c:member:`PyPreConfig.utf8_mode`) is *enabled*; * Buffered stdio (:c:member:`PyConfig.buffered_stdio`) is *disabled*; @@ -298,9 +233,9 @@ To add Python to an iOS Xcode project: * Signal handlers (:c:member:`PyConfig.install_signal_handlers`) are *enabled*; * System logging (:c:member:`PyConfig.use_system_logger`) is *enabled* (optional, but strongly recommended; this is enabled by default); - * ``PYTHONHOME`` for the interpreter is configured to point at the + * :envvar:`PYTHONHOME` for the interpreter is configured to point at the ``python`` subfolder of your app's bundle; and - * The ``PYTHONPATH`` for the interpreter includes: + * The :envvar:`PYTHONPATH` for the interpreter includes: - the ``python/lib/python3.X`` subfolder of your app's bundle, - the ``python/lib/python3.X/lib-dynload`` subfolder of your app's bundle, and @@ -309,45 +244,52 @@ To add Python to an iOS Xcode project: Your app's bundle location can be determined using ``[[NSBundle mainBundle] resourcePath]``. -Steps 8, 9 and 10 of these instructions assume that you have a single folder of +Steps 7 and 8 of these instructions assume that you have a single folder of pure Python application code, named ``app``. If you have third-party binary modules in your app, some additional steps will be required: * You need to ensure that any folders containing third-party binaries are - either associated with the app target, or copied in as part of step 8. Step 8 - should also purge any binaries that are not appropriate for the platform a - specific build is targeting (i.e., delete any device binaries if you're - building an app targeting the simulator). + either associated with the app target, or are explicitly copied as part of + step 7. Step 7 should also purge any binaries that are not appropriate for + the platform a specific build is targeting (i.e., delete any device binaries + if you're building an app targeting the simulator). -* Any folders that contain third-party binaries must be processed into - framework form by step 9. The invocation of ``install_dylib`` that processes - the ``lib-dynload`` folder can be copied and adapted for this purpose. +* If you're using a separate folder for third-party packages, ensure that + folder is added to the end of the call to ``install_python`` in step 7, and + as part of the :envvar:`PYTHONPATH` configuration in step 8. -* If you're using a separate folder for third-party packages, ensure that folder - is included as part of the ``PYTHONPATH`` configuration in step 10. +* If any of the folders that contain third-party packages will contain ``.pth`` + files, you should add that folder as a *site directory* (using + :meth:`site.addsitedir`), rather than adding to :envvar:`PYTHONPATH` or + :attr:`sys.path` directly. Testing a Python package ------------------------ -The CPython source tree contains :source:`a testbed project ` that +The CPython source tree contains :source:`a testbed project ` that is used to run the CPython test suite on the iOS simulator. This testbed can also be used as a testbed project for running your Python library's test suite on iOS. -After building or obtaining an iOS XCFramework (See :source:`iOS/README.rst` -for details), create a clone of the Python iOS testbed project by running: +After building or obtaining an iOS XCFramework (see :source:`Apple/iOS/README.md` +for details), create a clone of the Python iOS testbed project. If you used the +``Apple`` build script to build the XCframework, you can run: .. code-block:: bash - $ python iOS/testbed clone --framework --app --app app-testbed + $ python cross-build/iOS/testbed clone --app --app app-testbed + +Or, if you've sourced your own XCframework, by running: + +.. code-block:: bash -You will need to modify the ``iOS/testbed`` reference to point to that -directory in the CPython source tree; any folders specified with the ``--app`` -flag will be copied into the cloned testbed project. The resulting testbed will -be created in the ``app-testbed`` folder. In this example, the ``module1`` and -``module2`` would be importable modules at runtime. If your project has -additional dependencies, they can be installed into the -``app-testbed/iOSTestbed/app_packages`` folder (using ``pip install --target -app-testbed/iOSTestbed/app_packages`` or similar). + $ python Apple/testbed clone --platform iOS --framework --app --app app-testbed + +Any folders specified with the ``--app`` flag will be copied into the cloned +testbed project. The resulting testbed will be created in the ``app-testbed`` +folder. In this example, the ``module1`` and ``module2`` would be importable +modules at runtime. If your project has additional dependencies, they can be +installed into the ``app-testbed/Testbed/app_packages`` folder (using ``pip +install --target app-testbed/Testbed/app_packages`` or similar). You can then use the ``app-testbed`` folder to run the test suite for your app, For example, if ``module1.tests`` was the entry point to your test suite, you @@ -369,13 +311,29 @@ You can also open the testbed project in Xcode by running: This will allow you to use the full Xcode suite of tools for debugging. +The arguments used to run the test suite are defined as part of the test plan. +To modify the test plan, select the test plan node of the project tree (it +should be the first child of the root node), and select the "Configurations" +tab. Modify the "Arguments Passed On Launch" value to change the testing +arguments. + +The test plan also disables parallel testing, and specifies the use of the +``Testbed.lldbinit`` file for providing configuration of the debugger. The +default debugger configuration disables automatic breakpoints on the +``SIGINT``, ``SIGUSR1``, ``SIGUSR2``, and ``SIGXFSZ`` signals. + App Store Compliance ==================== The only mechanism for distributing apps to third-party iOS devices is to submit the app to the iOS App Store; apps submitted for distribution must pass Apple's app review process. This process includes a set of automated validation -rules that inspect the submitted application bundle for problematic code. +rules that inspect the submitted application bundle for problematic code. There +are some steps that must be taken to ensure that your app will be able to pass +these validation steps. + +Incompatible code in the standard library +----------------------------------------- The Python standard library contains some code that is known to violate these automated rules. While these violations appear to be false positives, Apple's @@ -386,3 +344,18 @@ The Python source tree contains :source:`a patch file ` that will remove all code that is known to cause issues with the App Store review process. This patch is applied automatically when building for iOS. + +Privacy manifests +----------------- + +In April 2025, Apple introduced a requirement for `certain third-party +libraries to provide a Privacy Manifest +`__. +As a result, if you have a binary module that uses one of the affected +libraries, you must provide an ``.xcprivacy`` file for that library. +OpenSSL is one library affected by this requirement, but there are others. + +If you produce a binary module named ``mymodule.so``, and use you the Xcode +build script described in step 7 above, you can place a ``mymodule.xcprivacy`` +file next to ``mymodule.so``, and the privacy manifest will be installed into +the required location when the binary module is converted into a framework. diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index 4b6c884f3d4f253..2fd6eace2b5dda2 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -20,13 +20,6 @@ the Pythons provided by the CPython release team for download from the `python.org website `_. See :ref:`alternative_bundles` for some other options. -.. |usemac_x_dot_y| replace:: 3.13 -.. |usemac_python_x_dot_y_literal| replace:: ``python3.13`` -.. |usemac_python_x_dot_y_t_literal| replace:: ``python3.13t`` -.. |usemac_python_x_dot_y_t_literal_config| replace:: ``python3.13t-config`` -.. |usemac_applications_folder_name| replace:: ``Python 3.13`` -.. |usemac_applications_folder_version| replace:: ``/Applications/Python 3.13/`` - .. _getting-osx: .. _getting-and-installing-macpython: @@ -46,7 +39,7 @@ Current installers provide a `universal2 binary `_ build of Python which runs natively on all Macs (Apple Silicon and Intel) that are supported by a wide range of macOS versions, -currently typically from at least **macOS 10.13 High Sierra** on. +currently typically from at least **macOS 10.15 Catalina** on. The downloaded file is a standard macOS installer package file (``.pkg``). File integrity information (checksum, size, sigstore signature, etc) for each file is included @@ -64,7 +57,7 @@ Clicking on the **Continue** button brings up the **Read Me** for this installer Besides other important information, the **Read Me** documents which Python version is going to be installed and on what versions of macOS it is supported. You may need to scroll through to read the whole file. By default, this **Read Me** will also be -installed in |usemac_applications_folder_version| and available to read anytime. +installed in |applications_python_version_literal| and available to read anytime. .. image:: mac_installer_02_readme.png @@ -83,7 +76,7 @@ display. For most uses, the standard set of installation operations is appropria By pressing the **Customize** button, you can choose to omit or select certain package components of the installer. Click on each package name to see a description of what it installs. -To also install support for the optional experimental free-threaded feature, +To also install support for the optional free-threaded feature, see :ref:`install-freethreaded-macos`. .. image:: mac_installer_05_custom_install.png @@ -97,7 +90,7 @@ When the installation is complete, the **Summary** window will appear. .. image:: mac_installer_06_summary.png Double-click on the :command:`Install Certificates.command` -icon or file in the |usemac_applications_folder_version| window to complete the +icon or file in the |applications_python_version_literal| window to complete the installation. .. image:: mac_installer_07_applications.png @@ -114,7 +107,7 @@ Close this terminal window and the installer window. A default install will include: -* A |usemac_applications_folder_name| folder in your :file:`Applications` folder. In here +* A |python_version_literal| folder in your :file:`Applications` folder. In here you find :program:`IDLE`, the development environment that is a standard part of official Python distributions; and :program:`Python Launcher`, which handles double-clicking Python scripts from the macOS `Finder `_. @@ -141,7 +134,7 @@ How to run a Python script There are two ways to invoke the Python interpreter. If you are familiar with using a Unix shell in a terminal -window, you can invoke |usemac_python_x_dot_y_literal| or ``python3`` optionally +window, you can invoke |python_x_dot_y_literal| or ``python3`` optionally followed by one or more command line options (described in :ref:`using-on-general`). The Python tutorial also has a useful section on :ref:`using Python interactively from a shell `. @@ -160,7 +153,7 @@ for more information. To run a Python script file from the terminal window, you can invoke the interpreter with the name of the script file: - |usemac_python_x_dot_y_literal| ``myscript.py`` + |python_x_dot_y_literal| ``myscript.py`` To run your script from the Finder, you can either: @@ -259,20 +252,20 @@ Advanced Topics Installing Free-threaded Binaries --------------------------------- -.. versionadded:: 3.13 (Experimental) - -.. note:: - - Everything described in this section is considered experimental, - and should be expected to change in future releases. +.. versionadded:: 3.13 The ``python.org`` :ref:`Python for macOS ` installer package can optionally install an additional build of -Python |usemac_x_dot_y| that supports :pep:`703`, the experimental free-threading feature +Python |version| that supports :pep:`703`, the free-threading feature (running with the :term:`global interpreter lock` disabled). Check the release page on ``python.org`` for possible updated information. -Because this feature is still considered experimental, the support for it +The free-threaded mode is working and continues to be improved, but +there is some additional overhead in single-threaded workloads compared +to the regular build. Additionally, third-party packages, in particular ones +with an :term:`extension module`, may not be ready for use in a +free-threaded build, and will re-enable the :term:`GIL`. +Therefore, the support for free-threading is not installed by default. It is packaged as a separate install option, available by clicking the **Customize** button on the **Installation Type** step of the installer as described above. @@ -282,46 +275,54 @@ step of the installer as described above. If the box next to the **Free-threaded Python** package name is checked, a separate :file:`PythonT.framework` will also be installed alongside the normal :file:`Python.framework` in :file:`/Library/Frameworks`. -This configuration allows a free-threaded Python |usemac_x_dot_y| build to co-exist -on your system with a traditional (GIL only) Python |usemac_x_dot_y| build with -minimal risk while installing or testing. This installation layout is itself -experimental and is subject to change in future releases. +This configuration allows a free-threaded Python |version| build to co-exist +on your system with a traditional (GIL only) Python |version| build with +minimal risk while installing or testing. This installation layout may +change in future releases. Known cautions and limitations: - The **UNIX command-line tools** package, which is selected by default, - will install links in :file:`/usr/local/bin` for |usemac_python_x_dot_y_t_literal|, - the free-threaded interpreter, and |usemac_python_x_dot_y_t_literal_config|, + will install links in :file:`/usr/local/bin` for |python_x_dot_y_t_literal|, + the free-threaded interpreter, and |python_x_dot_y_t_literal_config|, a configuration utility which may be useful for package builders. Since :file:`/usr/local/bin` is typically included in your shell ``PATH``, in most cases no changes to your ``PATH`` environment variables should - be needed to use |usemac_python_x_dot_y_t_literal|. + be needed to use |python_x_dot_y_t_literal|. - For this release, the **Shell profile updater** package and the - :file:`Update Shell Profile.command` in |usemac_applications_folder_version| + :file:`Update Shell Profile.command` in |applications_python_version_literal| do not support the free-threaded package. - The free-threaded build and the traditional build have separate search paths and separate :file:`site-packages` directories so, by default, if you need a package available in both builds, it may need to be installed in both. The free-threaded package will install a separate instance of :program:`pip` for use - with |usemac_python_x_dot_y_t_literal|. + with |python_x_dot_y_t_literal|. - To install a package using :command:`pip` without a :command:`venv`: - |usemac_python_x_dot_y_t_literal| ``-m pip install `` + .. parsed-literal:: + + python\ |version|\ t -m pip install - When working with multiple Python environments, it is usually safest and easiest to :ref:`create and use virtual environments `. This can avoid possible command name conflicts and confusion about which Python is in use: - |usemac_python_x_dot_y_t_literal| ``-m venv `` + .. parsed-literal:: + + python\ |version|\ t -m venv + then :command:`activate`. - To run a free-threaded version of IDLE: - |usemac_python_x_dot_y_t_literal| ``-m idlelib`` + .. parsed-literal:: + + python\ |version|\ t -m idlelib + - The interpreters in both builds respond to the same :ref:`PYTHON environment variables ` @@ -337,28 +338,28 @@ Known cautions and limitations: thus it only needs to be run once. - If you cannot depend on the link in ``/usr/local/bin`` pointing to the - ``python.org`` free-threaded |usemac_python_x_dot_y_t_literal| (for example, if you want + ``python.org`` free-threaded |python_x_dot_y_t_literal| (for example, if you want to install your own version there or some other distribution does), you can explicitly set your shell ``PATH`` environment variable to include the ``PythonT`` framework ``bin`` directory: - .. code-block:: sh + .. parsed-literal:: - export PATH="/Library/Frameworks/PythonT.framework/Versions/3.13/bin":"$PATH" + export PATH="/Library/Frameworks/PythonT.framework/Versions/\ |version|\ /bin":"$PATH" The traditional framework installation by default does something similar, except for :file:`Python.framework`. Be aware that having both framework ``bin`` directories in ``PATH`` can lead to confusion if there are duplicate names - like ``python3.13`` in both; which one is actually used depends on the order + like |python_x_dot_y_literal| in both; which one is actually used depends on the order they appear in ``PATH``. The ``which python3.x`` or ``which python3.xt`` commands can show which path is being used. Using virtual environments can help avoid such ambiguities. Another option might be to create a shell :command:`alias` to the desired interpreter, like: - .. code-block:: sh + .. parsed-literal:: - alias py3.13="/Library/Frameworks/Python.framework/Versions/3.13/bin/python3.13" - alias py3.13t="/Library/Frameworks/PythonT.framework/Versions/3.13/bin/python3.13t" + alias py\ |version|\ ="/Library/Frameworks/Python.framework/Versions/\ |version|\ /bin/python\ |version|\ " + alias py\ |version|\ t="/Library/Frameworks/PythonT.framework/Versions/\ |version|\ /bin/python\ |version|\ t" Installing using the command line --------------------------------- @@ -369,22 +370,22 @@ the macOS command line :command:`installer` utility lets you select non-default options, too. If you are not familiar with :command:`installer`, it can be somewhat cryptic (see :command:`man installer` for more information). As an example, the following shell snippet shows one way to do it, -using the ``3.13.0b2`` release and selecting the free-threaded interpreter +using the |x_dot_y_b2_literal| release and selecting the free-threaded interpreter option: -.. code-block:: sh +.. parsed-literal:: - RELEASE="python-3.13.0b2-macos11.pkg" + RELEASE="python-\ |version|\ 0b2-macos11.pkg" # download installer pkg - curl -O https://www.python.org/ftp/python/3.13.0/${RELEASE} + curl -O \https://www.python.org/ftp/python/\ |version|\ .0/${RELEASE} # create installer choicechanges to customize the install: - # enable the PythonTFramework-3.13 package + # enable the PythonTFramework-\ |version|\ package # while accepting the other defaults (install all other packages) cat > ./choicechanges.plist < - + @@ -393,7 +394,7 @@ option: choiceAttribute selected choiceIdentifier - org.python.Python.PythonTFramework-3.13 + org.python.Python.PythonTFramework-\ |version|\ @@ -404,19 +405,19 @@ option: You can then test that both installer builds are now available with something like: -.. code-block:: console +.. parsed-literal:: $ # test that the free-threaded interpreter was installed if the Unix Command Tools package was enabled - $ /usr/local/bin/python3.13t -VV - Python 3.13.0b2 experimental free-threading build (v3.13.0b2:3a83b172af, Jun 5 2024, 12:57:31) [Clang 15.0.0 (clang-1500.3.9.4)] + $ /usr/local/bin/python\ |version|\ t -VV + Python \ |version|\ .0b2 free-threading build (v\ |version|\ .0b2:3a83b172af, Jun 5 2024, 12:57:31) [Clang 15.0.0 (clang-1500.3.9.4)] $ # and the traditional interpreter - $ /usr/local/bin/python3.13 -VV - Python 3.13.0b2 (v3.13.0b2:3a83b172af, Jun 5 2024, 12:50:24) [Clang 15.0.0 (clang-1500.3.9.4)] + $ /usr/local/bin/python\ |version|\ -VV + Python \ |version|\ .0b2 (v\ |version|\ .0b2:3a83b172af, Jun 5 2024, 12:50:24) [Clang 15.0.0 (clang-1500.3.9.4)] $ # test that they are also available without the prefix if /usr/local/bin is on $PATH - $ python3.13t -VV - Python 3.13.0b2 experimental free-threading build (v3.13.0b2:3a83b172af, Jun 5 2024, 12:57:31) [Clang 15.0.0 (clang-1500.3.9.4)] - $ python3.13 -VV - Python 3.13.0b2 (v3.13.0b2:3a83b172af, Jun 5 2024, 12:50:24) [Clang 15.0.0 (clang-1500.3.9.4)] + $ python\ |version|\ t -VV + Python \ |version|\ .0b2 free-threading build (v\ |version|\ .0b2:3a83b172af, Jun 5 2024, 12:57:31) [Clang 15.0.0 (clang-1500.3.9.4)] + $ python\ |version|\ -VV + Python \ |version|\ .0b2 (v\ |version|\ .0b2:3a83b172af, Jun 5 2024, 12:50:24) [Clang 15.0.0 (clang-1500.3.9.4)] .. note:: diff --git a/Doc/using/mac_installer_07_applications.png b/Doc/using/mac_installer_07_applications.png index 940219cad6f61c0..c8b3aa1099adfc9 100644 Binary files a/Doc/using/mac_installer_07_applications.png and b/Doc/using/mac_installer_07_applications.png differ diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index 9ec4e3419321a4e..a9950ef75254976 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -84,11 +84,17 @@ On FreeBSD and OpenBSD Building Python =============== +.. seealso:: + + If you want to contribute to CPython, refer to the + `devguide `_, + which includes build instructions and other tips on setting up environment. + If you want to compile CPython yourself, first thing you should do is get the `source `_. You can download either the -latest release's source or just grab a fresh `clone -`_. (If you want -to contribute patches, you will need a clone.) +latest release's source or grab a fresh `clone +`_. +You will also need to install the :ref:`build requirements `. The build process consists of the usual commands:: diff --git a/Doc/using/win_install_freethreaded.png b/Doc/using/win_install_freethreaded.png index 0aa01c1df6e051b..12b89c0165d994d 100644 Binary files a/Doc/using/win_install_freethreaded.png and b/Doc/using/win_install_freethreaded.png differ diff --git a/Doc/using/win_installer.png b/Doc/using/win_installer.png index 03bf2d7b16c599b..fc9605a79cfcc51 100644 Binary files a/Doc/using/win_installer.png and b/Doc/using/win_installer.png differ diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 74d6db5d7d1a98b..46eebaebd5d7f3d 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -4,6 +4,8 @@ .. _Microsoft Store app: https://apps.microsoft.com/detail/9NQ7512CXL7T +.. _legacy launcher: https://www.python.org/ftp/python/3.14.0/win32/launcher.msi + .. _using-on-windows: ************************* @@ -60,7 +62,7 @@ packages. .. _windows-path-mod: .. _launcher: -Python Install Manager +Python install manager ====================== Installation @@ -77,34 +79,33 @@ To install the file downloaded from python.org, either double-click and select "Install", or run ``Add-AppxPackage `` in Windows Powershell. After installation, the ``python``, ``py``, and ``pymanager`` commands should be -available. If they are not, click Start and search for "Manage app execution -aliases". This settings page will let you enable the relevant commands. They -will be labelled "Python (default)", "Python (default windowed)", and "Python -install manager". - -If you have existing installations of Python, or you have modified your -:envvar:`PATH` variable, you may need to remove them or undo the modifications -in order for the commands to work. Old versions of Python can be reinstalled -using the Python install manager. +available. If you have existing installations of Python, or you have modified +your :envvar:`PATH` variable, you may need to remove them or undo the +modifications. See :ref:`pymanager-troubleshoot` for more help with fixing +non-working commands. When you first install a runtime, you will likely be prompted to add a directory to your :envvar:`PATH`. This is optional, if you prefer to use the ``py`` command, but is offered for those who prefer the full range of aliases (such as ``python3.14.exe``) to be available. The directory will be -:file:`%LocalAppData%\Python\bin` by default, but may be customized by an +:file:`%LocalAppData%\\Python\\bin` by default, but may be customized by an administrator. Click Start and search for "Edit environment variables for your account" for the system settings page to add the path. +Each Python runtime you install will have its own directory for scripts. These +also need to be added to :envvar:`PATH` if you want to use them. + The Python install manager will be automatically updated to new releases. This does not affect any installs of Python runtimes. Uninstalling the Python install manager does not uninstall any Python runtimes. If you are not able to install an MSIX in your context, for example, you are -using automated deployment software that does not support it, please see -:ref:`pymanager-advancedinstall` below for more information. +using automated deployment software that does not support it, or are targeting +Windows Server 2019, please see :ref:`pymanager-advancedinstall` below for more +information. -Basic Use +Basic use --------- The recommended command for launching Python is ``python``, which will either @@ -125,12 +126,11 @@ is also an unambiguous ``pymanager`` command. Scripted installs that are intending to use Python install manager should consider using ``pymanager``, due to the lower chance of encountering a conflict with existing installs. The only difference between the two commands is when running without any arguments: -``py`` will install and launch your default interpreter, while ``pymanager`` -will display help (``pymanager exec ...`` provides equivalent behaviour to -``py ...``). +``py`` will launch your default interpreter, while ``pymanager`` will display +help (``pymanager exec ...`` provides equivalent behaviour to ``py ...``). Each of these commands also has a windowed version that avoids creating a -console window. These are ``pyw``, ``pythonw`` and ``pymanagerw``. A ``python3`` +console window. These are ``pyw``, ``pythonw`` and ``pywmanager``. A ``python3`` command is also included that mimics the ``python`` command. It is intended to catch accidental uses of the typical POSIX command on Windows, but is not meant to be widely used or recommended. @@ -147,6 +147,10 @@ want to be passed to the runtime (such as script files or the module to launch): $> py -m this ... +The default runtime can be overridden with the :envvar:`PYTHON_MANAGER_DEFAULT` +environment variable, or a configuration file. See :ref:`pymanager-config` for +information about configuration settings. + To launch a specific runtime, the ``py`` command accepts a ``-V:`` option. This option must be specified before any others. The tag is part or all of the identifier for the runtime; for those from the CPython team, it looks like the @@ -160,7 +164,9 @@ omitted in cases where the tag refers to an official release and starts with $> py -V:3-arm64 ... Runtimes from other distributors may require the *company* to be included as -well. This should be separated from the tag by a slash, and may be a prefix. +well. +It should be separated from the tag by a slash (either ``/`` or ``\``), +and may be shortened to any prefix of its full value. Specifying the company is optional when it is ``PythonCore``, and specifying the tag is optional (but not the slash) when you want the latest release from a specific company. @@ -182,15 +188,14 @@ that virtual environment. In this scenario, the ``python`` command was likely already overridden and none of these checks occurred. However, this behaviour ensures that the ``py`` command can be used interchangeably. -When you launch either ``python`` or ``py`` but do not have any runtimes -installed, and the requested version is the default, it will be installed -automatically and then launched. Otherwise, the requested version will be -installed if automatic installation is configured (most likely by setting -``PYTHON_MANAGER_AUTOMATIC_INSTALL`` to ``true``), or if the ``py exec`` or -``pymanager exec`` forms of the command were used. +When no runtimes are installed, any launch command will try to install the +requested version and launch it. However, after any version is installed, only +the ``py exec ...`` and ``pymanager exec ...`` commands will install if the +requested version is absent. Other forms of commands will display an error and +direct you to use ``py install`` first. -Command Help +Command help ------------ The ``py help`` command will display the full list of supported commands, along @@ -215,7 +220,7 @@ override multiple settings at once. See :ref:`pymanager-config` below for more information about these files. -Listing Runtimes +Listing runtimes ---------------- .. code:: @@ -256,7 +261,7 @@ For compatibility with the old launcher, the ``--list``, ``--list-paths``, additional options, and will produce legacy formatted output. -Installing Runtimes +Installing runtimes ------------------- .. code:: @@ -284,18 +289,36 @@ work. Passing ``--dry-run`` will generate output and logs, but will not modify any installs. +Passing ``--refresh`` will update all registrations for installed runtimes. This +will recreate Start menu shortcuts, registry keys, and global aliases (such as +``python3.14.exe`` or for any installed scripts). These are automatically +refreshed on installation of any runtime, but may need to be manually refreshed +after installing packages. + In addition to the above options, the ``--target`` option will extract the -runtime to the specified directory instead of doing a normal install. This is -useful for embedding runtimes into larger applications. +runtime to the specified directory instead of doing a normal install. +This is useful for embedding runtimes into larger applications. +Unlike a normal install, ``py`` will not be aware of the extracted runtime, +and no Start menu or other shortcuts will be created. +To launch the runtime, directly execute the main executable (typically +``python.exe``) in the target directory. .. code:: $> py install ... [-t=|--target=] +The ``py exec`` command will install the requested runtime if it is not already +present. This is controlled by the ``automatic_install`` configuration +(:envvar:`PYTHON_MANAGER_AUTOMATIC_INSTALL`), and is enabled by default. +If no runtimes are available at all, all launch commands will do an automatic +install if the configuration setting allows. This is to ensure a good experience +for new users, but should not generally be relied on rather than using the +``py exec`` command or explicit install commands. + .. _pymanager-offline: -Offline Installs +Offline installs ---------------- To perform offline installs of Python, you will need to first create an offline @@ -327,7 +350,7 @@ In this way, Python runtimes can be installed and managed on a machine without access to the internet. -Uninstalling Runtimes +Uninstalling runtimes --------------------- .. code:: @@ -373,59 +396,125 @@ overridden installs may resolve settings differently. A global configuration file may be configured by an administrator, and would be read first. The user configuration file is stored at -:file:`%AppData%\\Python\\pymanager.json` (by default) and is read next, +:file:`%AppData%\\Python\\pymanager.json` +(note that this location is under ``Roaming``, not ``Local``) and is read next, overwriting any settings from earlier files. An additional configuration file may be specified as the ``PYTHON_MANAGER_CONFIG`` environment variable or the ``--config`` command line option (but not both). +These locations may be modified by administrative customization options listed +later. The following settings are those that are considered likely to be modified in normal use. Later sections list those that are intended for administrative customization. -.. csv-table:: Standard configuration options - :header: "Config Key", "Environment Variable", "Description" - :widths: 2, 2, 4 - - ``default_tag``,``PYTHON_MANAGER_DEFAULT``,"The preferred default - version to launch or install. By default, this is interpreted as the most - recent non-prerelease version from the CPython team. - " - ``default_platform``,``PYTHON_MANAGER_DEFAULT_PLATFORM``,"The preferred - default platform to launch or install. This is treated as a suffix to the - specified tag, such that ``py -V:3.14`` would prefer an install for - ``3.14-64`` if it exists (and ``default_platform`` is ``-64``), but will use - ``3.14`` if no tagged install exists. - " - ``logs_dir``,``PYTHON_MANAGER_LOGS``,"The location where log files are - written. By default, :file:`%TEMP%`. - " - ``automatic_install``,``PYTHON_MANAGER_AUTOMATIC_INSTALL``,"True to - allow automatic installs when specifying a particular runtime to launch. - By default, true. - " - ``include_unmanaged``,``PYTHON_MANAGER_INCLUDE_UNMANAGED``,"True to - allow listing and launching runtimes that were not installed by the Python - install manager, or false to exclude them. By default, true. - " - ``shebang_can_run_anything``,"``PYTHON_MANAGER_SHEBANG_CAN_RUN_ANYTHING`` - ","True to allow shebangs in ``.py`` files to launch applications other than - Python runtimes, or false to prevent it. By default, true. - " - ``log_level``,"``PYMANAGER_VERBOSE``, ``PYMANAGER_DEBUG``","Set - the default level of output (0-50) By default, 20. Lower values produce more - output. The environment variables are boolean, and may produce additional - output during startup that is later suppressed by other configuration. - " - ``confirm``,``PYTHON_MANAGER_CONFIRM``,"True to confirm certain actions - before taking them (such as uninstall), or false to skip the confirmation. By - default, true. - " - ``install.source``,``PYTHON_MANAGER_SOURCE_URL``,"Override the index - feed to obtain new installs from. - " - ``list.format``,``PYTHON_MANAGER_LIST_FORMAT``,"Specify the default - format used by the ``py list`` command. By default, ``table``. - " +.. Sphinx bug with text writer; remove widths & caption temporarily +.. :widths: 2, 2, 4 + +.. rubric:: Standard configuration options + +.. list-table:: + :header-rows: 1 + + * - Config Key + - Environment Variable + - Description + + * - ``default_tag`` + - .. envvar:: PYTHON_MANAGER_DEFAULT + - The preferred default version to launch or install. + By default, this is interpreted as the most recent non-prerelease version + from the CPython team. + + * - ``default_platform`` + - ``PYTHON_MANAGER_DEFAULT_PLATFORM`` + - The preferred default platform to launch or install. + This is treated as a suffix to the specified tag, such that ``py -V:3.14`` + would prefer an install for ``3.14-64`` if it exists + (and ``default_platform`` is ``-64``), + but will use ``3.14`` if no tagged install exists. + + * - ``logs_dir`` + - ``PYTHON_MANAGER_LOGS`` + - The location where log files are written. + By default, :file:`%TEMP%`. + + * - ``automatic_install`` + - .. envvar:: PYTHON_MANAGER_AUTOMATIC_INSTALL + - True to allow automatic installs when using ``py exec`` to launch (or + ``py`` when no runtimes are installed yet). + Other commands will not automatically install, regardless of this + setting. + By default, true. + + * - ``include_unmanaged`` + - ``PYTHON_MANAGER_INCLUDE_UNMANAGED`` + - True to allow listing and launching runtimes that were not installed + by the Python install manager, or false to exclude them. + By default, true. + + * - ``shebang_can_run_anything`` + - ``PYTHON_MANAGER_SHEBANG_CAN_RUN_ANYTHING`` + - True to allow shebangs in ``.py`` files to launch applications other than + Python runtimes, or false to prevent it. + By default, true. + + * - ``shebang_templates`` + - (none) + - Mapping from shebang line template to alternative command, such as + ``py -V:`` or a substitute string. + See :ref:`pymanager-shebang` for more details. + + * - ``log_level`` + - ``PYMANAGER_VERBOSE``, ``PYMANAGER_DEBUG`` + - Set the default level of output (0-50). + By default, 20. + Lower values produce more output. + The environment variables are boolean, and may produce additional + output during startup that is later suppressed by other configuration. + + * - ``confirm`` + - ``PYTHON_MANAGER_CONFIRM`` + - True to confirm certain actions before taking them (such as uninstall), + or false to skip the confirmation. + By default, true. + + * - ``install.source`` + - ``PYTHON_MANAGER_SOURCE_URL`` + - Override the index feed to obtain new installs from. + + * - ``install.enable_entrypoints`` + - (none) + - True to generate global commands for installed packages (such as + ``pip.exe``). These are defined by the packages themselves. + If set to false, only the Python interpreter has global commands created. + By default, true. You should run ``py install --refresh`` after changing + this setting. + + * - ``list.format`` + - ``PYTHON_MANAGER_LIST_FORMAT`` + - Specify the default format used by the ``py list`` command. + By default, ``table``. + + * - ``install_dir`` + - (none) + - Specify the root directory that runtimes will be installed into. + If you change this setting, previously installed runtimes will not be + usable unless you move them to the new location. + + * - ``global_dir`` + - (none) + - Specify the directory where global commands (such as ``python3.14.exe`` + and ``pip.exe``) are stored. + This directory should be added to your :envvar:`PATH` to make the + commands available from your terminal. + + * - ``download_dir`` + - (none) + - Specify the directory where downloaded files are stored. + This directory is a temporary cache, and can be cleaned up from time to + time. + Dotted names should be nested inside JSON objects, for example, ``list.format`` would be specified as ``{"list": {"format": "table"}}``. @@ -469,6 +558,10 @@ directory (which you may have added to your :envvar:`PATH` environment variable) can be used in a shebang, even if it is not on your :envvar:`PATH`. This allows the use of shebangs like ``/usr/bin/python3.12`` to select a particular runtime. +If no runtimes are installed, or if automatic installation is enabled, the +requested runtime will be installed if necessary. See :ref:`pymanager-config` +for information about configuration settings. + The ``/usr/bin/env`` form of shebang line will also search the :envvar:`PATH` environment variable for unrecognized commands. This corresponds to the behaviour of the Unix ``env`` program, which performs the same search, but @@ -485,21 +578,50 @@ which the path to the script and any additional arguments will be appended. This functionality may be disabled by the ``shebang_can_run_anything`` configuration option. -.. note: +Since version 26.3 of the Python install manager, custom shebang templates may +be added to your configuration file. Add the ``shebang_templates`` object with +one member for each template (the string to match) and the command to use when +the template is matched. Most commands should be ``py -V:`` (or ``pyw``) to +launch one of your installed runtimes. The ``py -3.`` form is also +allowed, as is a plain ``py`` to launch the default. No other arguments are +supported. + +.. code:: json5 + + { + "shebang_templates": { + "/usr/bin/python": "py", + "/usr/bin/my_custom_python": "py -V:MyCustomPython/3" + } + } + +If the substitute command is not ``py`` or ``pyw``, it will be written back into +the shebang and regular handling continues. If launching arbitrary executables +is permitted, then providing a full path will allow you to redirect from Python +to any executable. The template should match either the entire line (ignoring +leading and trailing whitespace), or up to the first space in the shebang line. + + +.. note:: The behaviour of shebangs in the Python install manager is subtly different from the previous ``py.exe`` launcher, and the old configuration options no longer apply. If you are specifically reliant on the old behaviour or - configuration, we recommend keeping the legacy launcher. It may be - `downloaded independently `_ - and installed on its own. The legacy launcher's ``py`` command will override - PyManager's one, and you will need to use ``pymanager`` commands for - installing and uninstalling. + configuration, we recommend installing the `legacy launcher`_. The legacy + launcher's ``py`` command will override PyManager's one by default, and you + will need to use ``pymanager`` commands for installing and uninstalling. +.. _Add-AppxPackage: https://learn.microsoft.com/powershell/module/appx/add-appxpackage + +.. _Remove-AppxPackage: https://learn.microsoft.com/powershell/module/appx/remove-appxpackage + +.. _Add-AppxProvisionedPackage: https://learn.microsoft.com/powershell/module/dism/add-appxprovisionedpackage + +.. _PackageManager: https://learn.microsoft.com/uwp/api/windows.management.deployment.packagemanager .. _pymanager-advancedinstall: -Advanced Installation +Advanced installation --------------------- For situations where an MSIX cannot be installed, such as some older @@ -509,6 +631,11 @@ per-machine installs to its default location in Program Files. It will attempt to modify the system :envvar:`PATH` environment variable to include this install location, but be sure to validate this on your configuration. +.. note:: + + Windows Server 2019 is the only version of Windows that CPython supports that + does not support MSIX. For Windows Server 2019, you should use the MSI. + Be aware that the MSI package does not bundle any runtimes, and so is not suitable for installs into offline environments without also creating an offline install index. See :ref:`pymanager-offline` and :ref:`pymanager-admin-config` @@ -529,29 +656,66 @@ depending on whether it was installed from python.org or through the Windows Store. Attempting to run the executable directly from Program Files is not recommended. -To programmatically install or uninstall the MSIX without using your -distribution platform's native support, the `Add-AppxPackage -`_ and -`Remove-AppxPackage `_ -PowerShell cmdlets are simplest to use: +To programmatically install the Python install manager, it is easiest to use +WinGet, which is included with all supported versions of Windows: -.. code:: +.. code-block:: powershell + + $> winget install 9NQ7512CXL7T -e --accept-package-agreements --disable-interactivity + + # Optionally run the configuration checker and accept all changes + $> py install --configure -y + +To download the Python install manager and install on another machine, the +following WinGet command will download the required files from the Store to your +Downloads directory (add ``-d `` to customize the output location). +This also generates a YAML file that appears to be unnecessary, as the +downloaded MSIX can be installed by launching or using the commands below. + +.. code-block:: powershell + + $> winget download 9NQ7512CXL7T -e --skip-license --accept-package-agreements --accept-source-agreements + +To programmatically install or uninstall an MSIX using only PowerShell, the +`Add-AppxPackage`_ and `Remove-AppxPackage`_ PowerShell cmdlets are recommended: + +.. code-block:: powershell $> Add-AppxPackage C:\Downloads\python-manager-25.0.msix ... $> Get-AppxPackage PythonSoftwareFoundation.PythonManager | Remove-AppxPackage -The native APIs for package management may be found on the Windows -`PackageManager `_ -class. The :func:`!AddPackageAsync` method installs for the current user, or use -:func:`!StagePackageAsync` followed by :func:`!ProvisionPackageForAllUsersAsync` -to install the Python install manager for all users from the MSIX package. Users -will still need to install their own copies of Python itself, as there is no way -to trigger those installs without being a logged in user. +The latest release can be downloaded and installed by Windows by passing the +AppInstaller file to the Add-AppxPackage command. This installs using the MSIX +on python.org, and is only recommended for cases where installing via the Store +(interactively or using WinGet) is not possible. + +.. code-block:: powershell + + $> Add-AppxPackage -AppInstallerFile https://www.python.org/ftp/python/pymanager/pymanager.appinstaller + +Other tools and APIs may also be used to provision an MSIX package for all users +on a machine, but Python does not consider this a supported scenario. We suggest +looking into the PowerShell `Add-AppxProvisionedPackage`_ cmdlet, the native +Windows `PackageManager`_ class, or the documentation and support for your +deployment tool. + +Regardless of the install method, users will still need to install their own +copies of Python itself, as there is no way to trigger those installs without +being a logged in user. When using the MSIX, the latest version of Python will +be available for all users to install without network access. + +Note that the MSIX downloadable from the Store and from the Python website are +subtly different and cannot be installed at the same time. Wherever possible, +we suggest using the above WinGet commands to download the package from the +Store to reduce the risk of setting up conflicting installs. There are no +licensing restrictions on the Python install manager that would prevent using +the Store package in this way. + .. _pymanager-admin-config: -Administrative Configuration +Administrative configuration ---------------------------- There are a number of options that may be useful for administrators to override @@ -581,60 +745,89 @@ variable will be used instead. Configuration settings that are paths are interpreted as relative to the directory containing the configuration file that specified them. -.. csv-table:: Administrative configuration options - :header: "Config Key", "Description" - :widths: 1, 4 - - ``base_config``,"The highest priority configuration file to read. Note that - only the built-in configuration file and the registry can modify this - setting. - " - ``user_config``,"The second configuration file to read. - " - ``additional_config``,"The third configuration file to read. - " - ``registry_override_key``,"Registry location to check for overrides. Note - that only the built-in configuration file can modify this setting. - " - ``bundled_dir``,"Read-only directory containing locally cached files. - " - ``install.fallback_source``,"Path or URL to an index to consult when the - main index cannot be accessed. - " - ``install.enable_shortcut_kinds``,"Comma-separated list of shortcut kinds - to allow (e.g. ``""pep514,start""``). Enabled shortcuts may still be disabled - by ``disable_shortcut_kinds``. - " - ``install.disable_shortcut_kinds``,"Comma-separated list of shortcut kinds - to exclude (e.g. ``""pep514,start""``). Disabled shortcuts are not - reactivated by ``enable_shortcut_kinds``. - " - ``pep514_root``,"Registry location to read and write PEP 514 entries into. - By default, :file:`HKEY_CURRENT_USER\\Software\\Python`. - " - ``start_folder``,"Start menu folder to write shortcuts into. By default, - ``Python``. This path is relative to the user's Programs folder. - " - ``virtual_env``,"Path to the active virtual environment. By default, this - is ``%VIRTUAL_ENV%``, but may be set empty to disable venv detection. - " - ``shebang_can_run_anything_silently``,"True to suppress visible warnings - when a shebang launches an application other than a Python runtime. - " +.. Sphinx bug with text writer; remove widths & caption temporarily +.. :widths: 1, 4 -.. _install-freethreaded-windows: +.. rubric:: Administrative configuration options -Installing Free-threaded Binaries ---------------------------------- +.. list-table:: + :header-rows: 1 -.. versionadded:: 3.13 (Experimental) + * - Config Key + - Description -.. note:: + * - ``base_config`` + - The highest priority configuration file to read. + Note that only the built-in configuration file and the registry can + modify this setting. + + * - ``user_config`` + - The second configuration file to read. + + * - ``additional_config`` + - The third configuration file to read. + + * - ``registry_override_key`` + - Registry location to check for overrides. + Note that only the built-in configuration file can modify this setting. + + * - ``bundled_dir`` + - Read-only directory containing locally cached files. + + * - ``install.fallback_source`` + - Path or URL to an index to consult when the main index cannot be accessed. + + * - ``install.enable_shortcut_kinds`` + - Comma-separated list of shortcut kinds to allow (e.g. ``"pep514,start"``). + Enabled shortcuts may still be disabled by ``disable_shortcut_kinds``. + + * - ``install.disable_shortcut_kinds`` + - Comma-separated list of shortcut kinds to exclude + (e.g. ``"pep514,start"``). + Disabled shortcuts are not reactivated by ``enable_shortcut_kinds``. + + * - ``install.hard_link_entrypoints`` + - True to use hard links for global shortcuts to save disk space. If false, + each shortcut executable is copied instead. After changing this setting, + you must run ``py install --refresh --force`` to update existing + commands. + By default, true. Disabling this may be necessary for troubleshooting or + systems that have issues with file links. + + * - ``pep514_root`` + - Registry location to read and write PEP 514 entries into. + By default, :file:`HKEY_CURRENT_USER\\Software\\Python`. + + * - ``start_folder`` + - Start menu folder to write shortcuts into. + By default, ``Python``. + This path is relative to the user's Programs folder. + + * - ``virtual_env`` + - Path to the active virtual environment. + By default, this is ``%VIRTUAL_ENV%``, but may be set empty + to disable venv detection. + + * - ``shebang_can_run_anything_silently`` + - True to suppress visible warnings when a shebang launches an application + other than a Python runtime. + + * - ``source_settings`` + - A mapping from source URL to settings specific to that index. + When multiple configuration files include this section, URL settings are + added or overwritten, but individual settings are not merged. + These settings are currently only for :ref:`index signatures + `. + + +.. _install-freethreaded-windows: + +Installing free-threaded binaries +--------------------------------- - Everything described in this section is considered experimental, - and should be expected to change in future releases. +.. versionadded:: 3.13 -Pre-built distributions of the experimental free-threaded build are available +Pre-built distributions of the free-threaded build are available by installing tags with the ``t`` suffix. .. code:: @@ -648,6 +841,101 @@ installed, then ``python`` will launch this one. Otherwise, you will need to use ``py -V:3.14t ...`` or, if you have added the global aliases directory to your :envvar:`PATH` environment variable, the ``python3.14t.exe`` commands. + +.. _pymanager-index-signatures: + +Index signatures +---------------- + +.. versionadded:: 26.2 + +Index files may be signed to detect tampering. A signature is a catalog file +at the same URL as the index with ``.cat`` added to the filename. The catalog +file should contain the hash of its matching index file, and should be signed +with a valid Authenticode signature. This allows standard tooling (on Windows) +to generate a signature, and any certificate may be used as long as the client +operating system already trusts its certification authority (root CA). + +Index signatures are only downloaded and checked when the local configuration's +``source_settings`` section includes the index URL and ``requires_signature`` is +true, or the index JSON contains ``requires_signature`` set to true. When the +setting exists in local configuration, even when false, settings in the index +are ignored. + +As well as requiring a valid signature, the ``required_root_subject`` and +``required_publisher_subject`` settings can further restrict acceptable +signatures based on the certificate Subject fields. Any attribute specified in +the configuration must match the attribute in the certificate (additional +attributes in the certificate are ignored). Typical attributes are ``CN=`` for +the common name, ``O=`` for the organizational unit, and ``C=`` for the +publisher's country. + +Finally, the ``required_publisher_eku`` setting allows requiring that a specific +Enhanced Key Usage (EKU) has been assigned to the publisher certificate. For +example, the EKU ``1.3.6.1.5.5.7.3.3`` indicates that the certificate was +intended for code signing (as opposed to server or client authentication). +In combination with a specific root CA, this provides another mechanism to +verify a legitimate signature. + +This is an example ``source_settings`` section from a configuration file. In +this case, the publisher of the feed is uniquely identified by the combination +of the Microsoft Identity Verification root and the EKU assigned by that root. +The signature for this case would be found at +``https://www.python.org/ftp/python/index-windows.json.cat``. + +.. code:: json5 + + { + "source_settings": { + "https://www.python.org/ftp/python/index-windows.json": { + "requires_signature": true, + "required_root_subject": "CN=Microsoft Identity Verification Root Certificate Authority 2020", + "required_publisher_subject": "CN=Python Software Foundation", + "required_publisher_eku": "1.3.6.1.4.1.311.97.608394634.79987812.305991749.578777327" + } + } + } + +The same settings could be specified in the ``index.json`` file instead. In this +case, the root and EKU are omitted, meaning that the signature must be valid and +have a specific common name in the publisher's certificate, but no other checks +are used. + +.. code:: json5 + + { + "requires_signature": true, + "required_publisher_subject": "CN=Python Software Foundation", + "versions": [ + // ... + ] + } + +When settings from inside a feed are used, the user is notified and the settings +are shown in the log file or verbose output. It is recommended to copy these +settings into a local configuration file for feeds that will be used frequently, +so that unauthorised modifications to the feed cannot disable verification. + +It is not possible to override the location of the signature file in the feed or +through a configuration file. Administrators can provide their own +``source_settings`` in a mandatory configuration file (see +:ref:`pymanager-admin-config`). + +If signature validation fails, you will be notified and prompted to continue. +When interactive confirmation is not allowed (for example, because ``--yes`` was +specified), it will always abort. To use a feed with invalid configuration in +this scenario, you must provide a configuration file that disables signature +checking for that feed. + +.. code:: json5 + + "source_settings": { + "https://www.example.com/feed-with-invalid-signature.json": { + "requires_signature": false + } + } + + .. _pymanager-troubleshoot: Troubleshooting @@ -655,65 +943,141 @@ Troubleshooting If your Python install manager does not seem to be working correctly, please work through these tests and fixes to see if it helps. If not, please report an -issue at `our bug tracker `_, +issue at `our bug tracker `_, including any relevant log files (written to your :file:`%TEMP%` directory by default). -.. csv-table:: Troubleshooting - :header: "Symptom", "Things to try" - :widths: 1, 1 - - "``python`` gives me a ""command not found"" error or opens the Store app - when I type it in my terminal.", "Did you :ref:`install the Python install - manager `? - " - "", "Click Start, open ""Manage app execution aliases"", and check that the - aliases for ""Python (default)"" are enabled. If they already are, try - disabling and re-enabling to refresh the command. The ""Python (default - windowed)"" and ""Python install manager"" commands may also need refreshing. - " - "", "Check that the ``py`` and ``pymanager`` commands work. - " - "``py`` gives me a ""command not found"" error when I type it in my - terminal.","Did you :ref:`install the Python install manager `? - " - "", "Click Start, open ""Manage app execution aliases"", and check that the - aliases for ""Python install manager"" are enabled. If they already are, try - disabling and re-enabling to refresh the command. The ""Python (default - windowed)"" and ""Python install manager"" commands may also need refreshing. - " - "``py`` gives me a ""can't open file"" error when I type commands in my - terminal.", "This usually means you have the legacy launcher installed and it - has priority over the Python install manager. To remove, click Start, open - ""Installed apps"", search for ""Python launcher"" and uninstall it. - " - "``python`` doesn't launch the same runtime as ``py``", "Click Start, open - ""Installed apps"", look for any existing Python runtimes, and either remove - them or Modify and disable the :envvar:`PATH` options. - " - "", "Click Start, open ""Manage app execution aliases"", and check that your - ``python.exe`` alias is set to ""Python (default)"" - " - "``python`` and ``py`` don't launch the runtime I expect", "Check your - ``PYTHON_MANAGER_DEFAULT`` environment variable or ``default_tag`` - configuration. The ``py list`` command will show your default based on these - settings. - " - "", "Installs that are managed by the Python install manager will be chosen - ahead of unmanaged installs. Use ``py install`` to install the runtime you - expect, or configure your default tag. - " - "", "Prerelease and experimental installs that are not managed by the Python - install manager may be chosen ahead of stable releases. Configure your - default tag or uninstall the prerelease runtime and reinstall using ``py - install``. - " - "``pythonw`` or ``pyw`` don't launch the same runtime as ``python`` or - ``py``","Click Start, open ""Manage app execution aliases"", and check that - your ``pythonw.exe`` and ``pyw.exe`` aliases are consistent with your - others. - " - +.. Sphinx bug with text writer; remove widths & caption temporarily +.. :widths: 1, 1 + +.. rubric:: Troubleshooting + +.. list-table:: + :header-rows: 1 + + * - Symptom + - Things to try + + * - ``python`` gives me a "command not found" error or opens the Store app + when I type it in my terminal. + - Did you :ref:`install the Python install manager `? + + * - + - Click Start, open "Manage app execution aliases", and check that the + aliases for "Python (default)" are enabled. + If they already are, try disabling and re-enabling to refresh the command. + The "Python (default windowed)" and "Python install manager" commands + may also need refreshing. + + * - + - Check that the ``py`` and ``pymanager`` commands work. + + * - + - Ensure your :envvar:`PATH` variable contains the entry for + ``%UserProfile%\AppData\Local\Microsoft\WindowsApps``. + The operating system includes this entry once by default, after other + user paths. If removed, shortcuts will not be found. + + * - ``py`` gives me a "command not found" error when I type it in my terminal. + - Did you :ref:`install the Python install manager `? + + * - + - Click Start, open "Manage app execution aliases", and check that the + aliases for "Python (default)" are enabled. + If they already are, try disabling and re-enabling to refresh the command. + The "Python (default windowed)" and "Python install manager" commands + may also need refreshing. + + * - + - Ensure your :envvar:`PATH` variable contains the entry for + ``%UserProfile%\AppData\Local\Microsoft\WindowsApps``. + The operating system includes this entry once by default, after other + user paths. If removed, shortcuts will not be found. + + * - ``py`` gives me a "can't open file" error when I type commands in my + terminal. + - This usually means you have the legacy launcher installed and + it has priority over the Python install manager. + To remove, click Start, open "Installed apps", + search for "Python launcher" and uninstall it. + + * - ``python`` doesn't launch the same runtime as ``py`` + - Click Start, open "Installed apps", look for any existing Python runtimes, + and either remove them or Modify and disable the :envvar:`PATH` options. + + * - + - Click Start, open "Manage app execution aliases", and check that your + ``python.exe`` alias is set to "Python (default)" + + * - ``python`` and ``py`` don't launch the runtime I expect + - Check your :envvar:`PYTHON_MANAGER_DEFAULT` environment variable + or ``default_tag`` configuration. + The ``py list`` command will show your default based on these settings. + + * - + - Installs that are managed by the Python install manager will be chosen + ahead of unmanaged installs. + Use ``py install`` to install the runtime you expect, + or configure your default tag. + + * - + - Prerelease and experimental installs that are not managed by the Python + install manager may be chosen ahead of stable releases. + Configure your default tag or uninstall the prerelease runtime + and reinstall it using ``py install``. + + * - ``pythonw`` or ``pyw`` don't launch the same runtime as ``python`` or ``py`` + - Click Start, open "Manage app execution aliases", and check that your + ``pythonw.exe`` and ``pyw.exe`` aliases are consistent with your others. + + * - ``pip`` gives me a "command not found" error when I type it in my terminal. + - Have you activated a virtual environment? + Run the ``.venv\Scripts\activate`` script in your terminal to activate. + + * - + - The package may be available but missing the generated executable. + We recommend using the ``python -m pip`` command instead. + Running ``py install --refresh`` and ensuring that the global shortcuts + directory is on :envvar:`PATH` (it will be shown in the command output if + it is not) should make commands such as ``pip`` (and other installed + packages) available. + + * - I installed a package with ``pip`` but its command is not found. + - Have you activated a virtual environment? + Run the ``.venv\Scripts\activate`` script in your terminal to activate. + + * - + - New packages do not automatically have global shortcuts created by the + Python install manager. Similarly, uninstalled packages do not have their + shortcuts removed. + Run ``py install --refresh`` to update the global shortcuts for newly + installed packages. + + * - Typing ``script-name.py`` in the terminal opens in a new window. + - This is a known limitation of the operating system. Either specify ``py`` + before the script name, create a batch file containing ``@py "%~dpn0.py" %*`` + with the same name as the script, or install the `legacy launcher`_ + and select it as the association for scripts. + + * - Drag-dropping files onto a script doesn't work + - This is a known limitation of the operating system. It is supported with + the `legacy launcher`_, or with the Python install manager when installed + from the MSI. + + * - I have installed the Python install manager multiple times. + - It is possible to install from the Store or WinGet, from the MSIX on + the Python website, and from the MSI, all at once. + They are all compatible and will share configuration and runtimes. + + * - + - See the earlier :ref:`pymanager-advancedinstall` section for ways to + uninstall the install manager other than the typical Installed Apps + (Add and Remove Programs) settings page. + + * - My old ``py.ini`` settings no longer work. + - The new Python install manager no longer supports this configuration file + or its settings, and so it will be ignored. + See :ref:`pymanager-config` for information about configuration settings. .. _windows-embeddable: @@ -731,7 +1095,7 @@ To install an embedded distribution, we recommend using ``py install`` with the .. code:: - $> py install 3.14-embed --target=runtime + $> py install 3.14-embed --target= When extracted, the embedded distribution is (almost) fully isolated from the user's system, including environment variables, system registry settings, and @@ -754,7 +1118,7 @@ versions before providing updates to users. The two recommended use cases for this distribution are described below. -Python Application +Python application ------------------ An application written in Python does not necessarily require users to be aware @@ -814,9 +1178,9 @@ on using nuget. What follows is a summary that is sufficient for Python developers. The ``nuget.exe`` command line tool may be downloaded directly from -``https://aka.ms/nugetclidl``, for example, using curl or PowerShell. With the -tool, the latest version of Python for 64-bit or 32-bit machines is installed -using:: +``https://dist.nuget.org/win-x86-commandline/latest/nuget.exe``, for example, +using curl or PowerShell. With the tool, the latest version of Python for +64-bit or 32-bit machines is installed using:: nuget.exe install python -ExcludeVersion -OutputDirectory . nuget.exe install pythonx86 -ExcludeVersion -OutputDirectory . @@ -858,12 +1222,7 @@ for the 64-bit version, `www.nuget.org/packages/pythonx86 Free-threaded packages ---------------------- -.. versionadded:: 3.13 (Experimental) - -.. note:: - - Everything described in this section is considered experimental, - and should be expected to change in future releases. +.. versionadded:: 3.13 Packages containing free-threaded binaries are named `python-freethreaded `_ @@ -915,7 +1274,7 @@ please install Python 3.12. .. _max-path: -Removing the MAX_PATH Limitation +Removing the MAX_PATH limitation ================================ Windows historically has limited path lengths to 260 characters. This meant that @@ -1196,7 +1555,7 @@ installation". In this case: * Shortcuts are available for all users -Removing the MAX_PATH Limitation +Removing the MAX_PATH limitation -------------------------------- Windows historically has limited path lengths to 260 characters. This meant that @@ -1219,7 +1578,7 @@ After changing the above option, no further configuration is required. .. _install-quiet-option: -Installing Without UI +Installing without UI --------------------- All of the options available in the installer UI can also be specified from the @@ -1368,7 +1727,7 @@ example file sets the same options as the previous example: .. _install-layout-option: -Installing Without Downloading +Installing without downloading ------------------------------ As some features of Python are not included in the initial installer download, @@ -1409,15 +1768,10 @@ settings and replace any that have been removed or modified. :ref:`launcher`, which has its own entry in Programs and Features. -Installing Free-threaded Binaries +Installing free-threaded binaries --------------------------------- -.. versionadded:: 3.13 (Experimental) - -.. note:: - - Everything described in this section is considered experimental, - and should be expected to change in future releases. +.. versionadded:: 3.13 To install pre-built binaries with free-threading enabled (see :pep:`703`), you should select "Customize installation". The second page of options includes the @@ -1449,7 +1803,7 @@ builds. Free-threaded binaries are also available :ref:`on nuget.org `. -Python Launcher for Windows (Deprecated) +Python launcher for Windows (deprecated) ======================================== .. deprecated:: 3.14 @@ -1601,7 +1955,7 @@ have the script specify the version which should be used. The key benefit of this is that a single launcher can support multiple Python versions at the same time depending on the contents of the first line. -Shebang Lines +Shebang lines ------------- If the first line of a script file starts with ``#!``, it is known as a @@ -1666,7 +2020,7 @@ program, which performs a :envvar:`PATH` search. If an executable matching the first argument after the ``env`` command cannot be found, but the argument starts with ``python``, it will be handled as described for the other virtual commands. -The environment variable :envvar:`PYLAUNCHER_NO_SEARCH_PATH` may be set +The environment variable :envvar:`!PYLAUNCHER_NO_SEARCH_PATH` may be set (to any value) to skip this search of :envvar:`PATH`. Shebang lines that do not match any of these patterns are looked up in the @@ -1733,7 +2087,7 @@ For example, a shebang line of ``#!python`` has no version qualifier, while ``#!python3`` has a version qualifier which specifies only a major version. If no version qualifiers are found in a command, the environment -variable :envvar:`PY_PYTHON` can be set to specify the default version +variable :envvar:`!PY_PYTHON` can be set to specify the default version qualifier. If it is not set, the default is "3". The variable can specify any value that may be passed on the command line, such as "3", "3.7", "3.7-32" or "3.7-64". (Note that the "-64" option is only @@ -1806,17 +2160,17 @@ For example: Diagnostics ----------- -If an environment variable :envvar:`PYLAUNCHER_DEBUG` is set (to any value), the +If an environment variable :envvar:`!PYLAUNCHER_DEBUG` is set (to any value), the launcher will print diagnostic information to stderr (i.e. to the console). While this information manages to be simultaneously verbose *and* terse, it should allow you to see what versions of Python were located, why a particular version was chosen and the exact command-line used to execute the target Python. It is primarily intended for testing and debugging. -Dry Run +Dry run ------- -If an environment variable :envvar:`PYLAUNCHER_DRYRUN` is set (to any value), +If an environment variable :envvar:`!PYLAUNCHER_DRYRUN` is set (to any value), the launcher will output the command it would have run, but will not actually launch Python. This may be useful for tools that want to use the launcher to detect and then launch Python directly. Note that the command written to @@ -1826,14 +2180,14 @@ the console. Install on demand ----------------- -If an environment variable :envvar:`PYLAUNCHER_ALLOW_INSTALL` is set (to any +If an environment variable :envvar:`!PYLAUNCHER_ALLOW_INSTALL` is set (to any value), and the requested Python version is not installed but is available on the Microsoft Store, the launcher will attempt to install it. This may require user interaction to complete, and you may need to run the command again. -An additional :envvar:`PYLAUNCHER_ALWAYS_INSTALL` variable causes the launcher +An additional :envvar:`!PYLAUNCHER_ALWAYS_INSTALL` variable causes the launcher to always try to install Python, even if it is detected. This is mainly intended -for testing (and should be used with :envvar:`PYLAUNCHER_DRYRUN`). +for testing (and should be used with :envvar:`!PYLAUNCHER_DRYRUN`). Return codes ------------ diff --git a/Doc/whatsnew/2.0.rst b/Doc/whatsnew/2.0.rst index 1a949ec40358072..c157c0337a13427 100644 --- a/Doc/whatsnew/2.0.rst +++ b/Doc/whatsnew/2.0.rst @@ -656,7 +656,8 @@ break. The change which will probably break the most code is tightening up the arguments accepted by some methods. Some methods would take multiple arguments and treat them as a tuple, particularly various list methods such as -:meth:`!append` and :meth:`!insert`. In earlier versions of Python, if ``L`` is +:meth:`~list.append` and :meth:`~list.insert`. +In earlier versions of Python, if ``L`` is a list, ``L.append( 1,2 )`` appends the tuple ``(1,2)`` to the list. In Python 2.0 this causes a :exc:`TypeError` exception to be raised, with the message: 'append requires exactly 1 argument; 2 given'. The fix is to simply add an diff --git a/Doc/whatsnew/2.3.rst b/Doc/whatsnew/2.3.rst index b7e4e73f4ce4aa0..43ab19037d26273 100644 --- a/Doc/whatsnew/2.3.rst +++ b/Doc/whatsnew/2.3.rst @@ -66,7 +66,7 @@ Here's a simple example:: The union and intersection of sets can be computed with the :meth:`~frozenset.union` and :meth:`~frozenset.intersection` methods; an alternative notation uses the bitwise operators ``&`` and ``|``. Mutable sets also have in-place versions of these methods, -:meth:`!union_update` and :meth:`~frozenset.intersection_update`. :: +:meth:`!union_update` and :meth:`~set.intersection_update`. :: >>> S1 = sets.Set([1,2,3]) >>> S2 = sets.Set([4,5,6]) @@ -87,7 +87,7 @@ It's also possible to take the symmetric difference of two sets. This is the set of all elements in the union that aren't in the intersection. Another way of putting it is that the symmetric difference contains all elements that are in exactly one set. Again, there's an alternative notation (``^``), and an -in-place version with the ungainly name :meth:`~frozenset.symmetric_difference_update`. :: +in-place version with the ungainly name :meth:`~set.symmetric_difference_update`. :: >>> S1 = sets.Set([1,2,3,4]) >>> S2 = sets.Set([3,4,5,6]) @@ -1698,8 +1698,8 @@ current local date. Once created, instances of the date/time classes are all immutable. There are a number of methods for producing formatted strings from objects:: - >>> import datetime - >>> now = datetime.datetime.now() + >>> import datetime as dt + >>> now = dt.datetime.now() >>> now.isoformat() '2002-12-30T21:27:03.994956' >>> now.ctime() # Only available on date, datetime @@ -1710,10 +1710,10 @@ number of methods for producing formatted strings from objects:: The :meth:`~datetime.datetime.replace` method allows modifying one or more fields of a :class:`~datetime.date` or :class:`~datetime.datetime` instance, returning a new instance:: - >>> d = datetime.datetime.now() + >>> d = dt.datetime.now() >>> d datetime.datetime(2002, 12, 30, 22, 15, 38, 827738) - >>> d.replace(year=2001, hour = 12) + >>> d.replace(year=2001, hour=12) datetime.datetime(2001, 12, 30, 12, 15, 38, 827738) >>> diff --git a/Doc/whatsnew/2.4.rst b/Doc/whatsnew/2.4.rst index 7628cfefe0ec96e..87c7b6b17477b50 100644 --- a/Doc/whatsnew/2.4.rst +++ b/Doc/whatsnew/2.4.rst @@ -35,7 +35,7 @@ implementation and design rationale. PEP 218: Built-In Set Objects ============================= -Python 2.3 introduced the :mod:`sets` module. C implementations of set data +Python 2.3 introduced the :mod:`!sets` module. C implementations of set data types have now been added to the Python core as two new built-in types, ``set(iterable)`` and ``frozenset(iterable)``. They provide high speed operations for membership testing, for eliminating duplicates from sequences, @@ -72,8 +72,8 @@ The :func:`frozenset` type is an immutable version of :func:`set`. Since it is immutable and hashable, it may be used as a dictionary key or as a member of another set. -The :mod:`sets` module remains in the standard library, and may be useful if you -wish to subclass the :class:`Set` or :class:`ImmutableSet` classes. There are +The :mod:`!sets` module remains in the standard library, and may be useful if you +wish to subclass the :class:`!Set` or :class:`!ImmutableSet` classes. There are currently no plans to deprecate the module. @@ -189,7 +189,7 @@ aren't aware of the Python language. The format string's syntax is complicated to explain to such users, and if they make a mistake, it's difficult to provide helpful feedback to them. -PEP 292 adds a :class:`Template` class to the :mod:`string` module that uses +PEP 292 adds a :class:`~string.Template` class to the :mod:`string` module that uses ``$`` to indicate a substitution:: >>> import string @@ -197,9 +197,9 @@ PEP 292 adds a :class:`Template` class to the :mod:`string` module that uses >>> t.substitute({'page':2, 'title': 'The Best of Times'}) '2: The Best of Times' -If a key is missing from the dictionary, the :meth:`substitute` method will -raise a :exc:`KeyError`. There's also a :meth:`safe_substitute` method that -ignores missing keys:: +If a key is missing from the dictionary, the :meth:`~string.Template.substitute` +method will raise a :exc:`KeyError`. There's also a +:meth:`~string.Template.safe_substitute` method that ignores missing keys:: >>> t = string.Template('$page: $title') >>> t.safe_substitute({'page':3}) @@ -430,14 +430,14 @@ The constructor has a number of handy options: * *universal_newlines* opens the child's input and output using Python's :term:`universal newlines` feature. -Once you've created the :class:`Popen` instance, you can call its :meth:`wait` -method to pause until the subprocess has exited, :meth:`poll` to check if it's +Once you've created the :class:`~subprocess.Popen` instance, you can call its :meth:`~subprocess.Popen.wait` +method to pause until the subprocess has exited, :meth:`~subprocess.Popen.poll` to check if it's exited without pausing, or ``communicate(data)`` to send the string *data* to the subprocess's standard input. ``communicate(data)`` then reads any data that the subprocess has sent to its standard output or standard error, returning a tuple ``(stdout_data, stderr_data)``. -:func:`call` is a shortcut that passes its arguments along to the :class:`Popen` +:func:`~subprocess.call` is a shortcut that passes its arguments along to the :class:`~subprocess.Popen` constructor, waits for the command to complete, and returns the status code of the subprocess. It can serve as a safer analog to :func:`os.system`:: @@ -476,7 +476,7 @@ Python has always supported floating-point (FP) numbers, based on the underlying C :c:expr:`double` type, as a data type. However, while most programming languages provide a floating-point type, many people (even programmers) are unaware that floating-point numbers don't represent certain decimal fractions -accurately. The new :class:`Decimal` type can represent these fractions +accurately. The new :class:`~decimal.Decimal` type can represent these fractions accurately, up to a user-specified precision limit. @@ -531,20 +531,20 @@ decimal places, the error is never apparent. However, for applications where it does matter, it's a lot of work to implement your own custom arithmetic routines. -Hence, the :class:`Decimal` type was created. +Hence, the :class:`~decimal.Decimal` type was created. -The :class:`Decimal` type -------------------------- +The :class:`~decimal.Decimal` type +---------------------------------- A new module, :mod:`decimal`, was added to Python's standard library. It -contains two classes, :class:`Decimal` and :class:`Context`. :class:`Decimal` -instances represent numbers, and :class:`Context` instances are used to wrap up +contains two classes, :class:`~decimal.Decimal` and :class:`~decimal.Context`. :class:`~decimal.Decimal` +instances represent numbers, and :class:`~decimal.Context` instances are used to wrap up various settings such as the precision and default rounding mode. -:class:`Decimal` instances are immutable, like regular Python integers and FP +:class:`~decimal.Decimal` instances are immutable, like regular Python integers and FP numbers; once it's been created, you can't change the value an instance -represents. :class:`Decimal` instances can be created from integers or +represents. :class:`~decimal.Decimal` instances can be created from integers or strings:: >>> import decimal @@ -567,7 +567,7 @@ number representing 1.1 turn into the decimal number for exactly 1.1, or for 1.1 plus whatever inaccuracies are introduced? The decision was to dodge the issue and leave such a conversion out of the API. Instead, you should convert the floating-point number into a string using the desired precision and pass the -string to the :class:`Decimal` constructor:: +string to the :class:`~decimal.Decimal` constructor:: >>> f = 1.1 >>> decimal.Decimal(str(f)) @@ -575,7 +575,7 @@ string to the :class:`Decimal` constructor:: >>> decimal.Decimal('%.12f' % f) Decimal("1.100000000000") -Once you have :class:`Decimal` instances, you can perform the usual mathematical +Once you have :class:`~decimal.Decimal` instances, you can perform the usual mathematical operations on them. One limitation: exponentiation requires an integer exponent:: @@ -596,7 +596,7 @@ exponent:: ... decimal.InvalidOperation: x ** (non-integer) -You can combine :class:`Decimal` instances with integers, but not with +You can combine :class:`~decimal.Decimal` instances with integers, but not with floating-point numbers:: >>> a + 4 @@ -607,11 +607,11 @@ floating-point numbers:: TypeError: You can interact Decimal only with int, long or Decimal data types. >>> -:class:`Decimal` numbers can be used with the :mod:`math` and :mod:`cmath` +:class:`~decimal.Decimal` numbers can be used with the :mod:`math` and :mod:`cmath` modules, but note that they'll be immediately converted to floating-point numbers before the operation is performed, resulting in a possible loss of precision and accuracy. You'll also get back a regular floating-point number -and not a :class:`Decimal`. :: +and not a :class:`~decimal.Decimal`. :: >>> import math, cmath >>> d = decimal.Decimal('123456789012.345') @@ -620,32 +620,33 @@ and not a :class:`Decimal`. :: >>> cmath.sqrt(-d) 351364.18288201344j -:class:`Decimal` instances have a :meth:`sqrt` method that returns a -:class:`Decimal`, but if you need other things such as trigonometric functions +:class:`~decimal.Decimal` instances have a :meth:`~decimal.Decimal.sqrt` method that returns a +:class:`~decimal.Decimal`, but if you need other things such as trigonometric functions you'll have to implement them. :: >>> d.sqrt() Decimal("351364.1828820134592177245001") -The :class:`Context` type -------------------------- +The :class:`~decimal.Context` type +---------------------------------- -Instances of the :class:`Context` class encapsulate several settings for +Instances of the :class:`~decimal.Context` class encapsulate several settings for decimal operations: -* :attr:`prec` is the precision, the number of decimal places. +* :attr:`~decimal.Context.prec` is the precision, the number of decimal places. -* :attr:`rounding` specifies the rounding mode. The :mod:`decimal` module has - constants for the various possibilities: :const:`ROUND_DOWN`, - :const:`ROUND_CEILING`, :const:`ROUND_HALF_EVEN`, and various others. +* :attr:`~decimal.Context.rounding` specifies the rounding mode. The + :mod:`decimal` module has constants for the various possibilities: + :const:`~decimal.ROUND_DOWN`, :const:`~decimal.ROUND_CEILING`, + :const:`~decimal.ROUND_HALF_EVEN`, and various others. -* :attr:`traps` is a dictionary specifying what happens on encountering certain +* :attr:`~decimal.Context.traps` is a dictionary specifying what happens on encountering certain error conditions: either an exception is raised or a value is returned. Some examples of error conditions are division by zero, loss of precision, and overflow. -There's a thread-local default context available by calling :func:`getcontext`; +There's a thread-local default context available by calling :func:`~decimal.getcontext`; you can change the properties of this context to alter the default precision, rounding, or trap handling. The following example shows the effect of changing the precision of the default context:: @@ -671,8 +672,8 @@ raised:: Decimal("Infinity") >>> -The :class:`Context` instance also has various methods for formatting numbers -such as :meth:`to_eng_string` and :meth:`to_sci_string`. +The :class:`~decimal.Context` instance also has various methods for formatting numbers +such as :meth:`~decimal.Context.to_eng_string` and :meth:`~decimal.Context.to_sci_string`. For more information, see the documentation for the :mod:`decimal` module, which includes a quick-start tutorial and a reference. @@ -740,7 +741,7 @@ display conventions that are localized to a particular country or language. However, the module was careful to not change the numeric locale because various functions in Python's implementation required that the numeric locale remain set to the ``'C'`` locale. Often this was because the code was using the C -library's :c:func:`atof` function. +library's :c:func:`!atof` function. Not setting the numeric locale caused trouble for extensions that used third-party C libraries, however, because they wouldn't have the correct locale set. @@ -793,11 +794,11 @@ Here are all of the changes that Python 2.4 makes to the core Python language. :class:`dict` constructor. This includes any mapping, any iterable of key/value pairs, and keyword arguments. (Contributed by Raymond Hettinger.) -* The string methods :meth:`ljust`, :meth:`rjust`, and :meth:`center` now take +* The string methods :meth:`~str.ljust`, :meth:`~str.rjust`, and :meth:`~str.center` now take an optional argument for specifying a fill character other than a space. (Contributed by Raymond Hettinger.) -* Strings also gained an :meth:`rsplit` method that works like the :meth:`split` +* Strings also gained an :meth:`~str.rsplit` method that works like the :meth:`~str.split` method but splits from the end of the string. (Contributed by Sean Reifschneider.) :: @@ -807,13 +808,13 @@ Here are all of the changes that Python 2.4 makes to the core Python language. ['www.python', 'org'] * Three keyword parameters, *cmp*, *key*, and *reverse*, were added to the - :meth:`sort` method of lists. These parameters make some common usages of - :meth:`sort` simpler. All of these parameters are optional. + :meth:`~list.sort` method of lists. These parameters make some common usages of + :meth:`~list.sort` simpler. All of these parameters are optional. For the *cmp* parameter, the value should be a comparison function that takes two parameters and returns -1, 0, or +1 depending on how the parameters compare. This function will then be used to sort the list. Previously this was the only - parameter that could be provided to :meth:`sort`. + parameter that could be provided to :meth:`~list.sort`. *key* should be a single-parameter function that takes a list element and returns a comparison key for the element. The list is then sorted using the @@ -834,9 +835,9 @@ Here are all of the changes that Python 2.4 makes to the core Python language. The last example, which uses the *cmp* parameter, is the old way to perform a case-insensitive sort. It works but is slower than using a *key* parameter. - Using *key* calls :meth:`lower` method once for each element in the list while + Using *key* calls :meth:`~str.lower` method once for each element in the list while using *cmp* will call it twice for each comparison, so using *key* saves on - invocations of the :meth:`lower` method. + invocations of the :meth:`~str.lower` method. For simple key functions and comparison functions, it is often possible to avoid a :keyword:`lambda` expression by using an unbound method instead. For example, @@ -856,7 +857,7 @@ Here are all of the changes that Python 2.4 makes to the core Python language. age, resulting in a list sorted by age where people with the same age are in name-sorted order. - (All changes to :meth:`sort` contributed by Raymond Hettinger.) + (All changes to :meth:`~list.sort` contributed by Raymond Hettinger.) * There is a new built-in function ``sorted(iterable)`` that works like the in-place :meth:`list.sort` method but can be used in expressions. The @@ -891,8 +892,8 @@ Here are all of the changes that Python 2.4 makes to the core Python language. (Contributed by Raymond Hettinger.) -* Integer operations will no longer trigger an :exc:`OverflowWarning`. The - :exc:`OverflowWarning` warning will disappear in Python 2.5. +* Integer operations will no longer trigger an :exc:`!OverflowWarning`. The + :exc:`!OverflowWarning` warning will disappear in Python 2.5. * The interpreter gained a new switch, :option:`-m`, that takes a name, searches for the corresponding module on ``sys.path``, and runs the module as a script. @@ -904,7 +905,7 @@ Here are all of the changes that Python 2.4 makes to the core Python language. for the *locals* parameter. Previously this had to be a regular Python dictionary. (Contributed by Raymond Hettinger.) -* The :func:`zip` built-in function and :func:`itertools.izip` now return an +* The :func:`zip` built-in function and :func:`!itertools.izip` now return an empty list if called with no arguments. Previously they raised a :exc:`TypeError` exception. This makes them more suitable for use with variable length argument lists:: @@ -935,8 +936,8 @@ Optimizations * The inner loops for list and tuple slicing were optimized and now run about one-third faster. The inner loops for dictionaries were also optimized, - resulting in performance boosts for :meth:`keys`, :meth:`values`, :meth:`items`, - :meth:`iterkeys`, :meth:`itervalues`, and :meth:`iteritems`. (Contributed by + resulting in performance boosts for :meth:`~dict.keys`, :meth:`~dict.values`, :meth:`~dict.items`, + :meth:`!iterkeys`, :meth:`!itervalues`, and :meth:`!iteritems`. (Contributed by Raymond Hettinger.) * The machinery for growing and shrinking lists was optimized for speed and for @@ -948,11 +949,11 @@ Optimizations * :func:`list`, :func:`tuple`, :func:`map`, :func:`filter`, and :func:`zip` now run several times faster with non-sequence arguments that supply a - :meth:`__len__` method. (Contributed by Raymond Hettinger.) + :meth:`~object.__len__` method. (Contributed by Raymond Hettinger.) -* The methods :meth:`list.__getitem__`, :meth:`dict.__getitem__`, and - :meth:`dict.__contains__` are now implemented as :class:`method_descriptor` - objects rather than :class:`wrapper_descriptor` objects. This form of access +* The methods :meth:`!list.__getitem__`, :meth:`!dict.__getitem__`, and + :meth:`!dict.__contains__` are now implemented as :class:`!method_descriptor` + objects rather than :class:`!wrapper_descriptor` objects. This form of access doubles their performance and makes them more suitable for use as arguments to functionals: ``map(mydict.__getitem__, keylist)``. (Contributed by Raymond Hettinger.) @@ -968,7 +969,7 @@ Optimizations * String concatenations in statements of the form ``s = s + "abc"`` and ``s += "abc"`` are now performed more efficiently in certain circumstances. This optimization won't be present in other Python implementations such as Jython, so - you shouldn't rely on it; using the :meth:`join` method of strings is still + you shouldn't rely on it; using the :meth:`~str.join` method of strings is still recommended when you want to efficiently glue a large number of strings together. (Contributed by Armin Rigo.) @@ -1023,13 +1024,13 @@ complete list of changes, or look through the CVS logs for all the details. PCTP-154, and TIS-620. * The UTF-8 and UTF-16 codecs now cope better with receiving partial input. - Previously the :class:`StreamReader` class would try to read more data, making - it impossible to resume decoding from the stream. The :meth:`read` method will + Previously the :class:`~codecs.StreamReader` class would try to read more data, making + it impossible to resume decoding from the stream. The :meth:`~codecs.StreamReader.read` method will now return as much data as it can and future calls will resume decoding where previous ones left off. (Implemented by Walter Dörwald.) * There is a new :mod:`collections` module for various specialized collection - datatypes. Currently it contains just one type, :class:`deque`, a double-ended + datatypes. Currently it contains just one type, :class:`~collections.deque`, a double-ended queue that supports efficiently adding and removing elements from either end:: @@ -1048,7 +1049,7 @@ complete list of changes, or look through the CVS logs for all the details. >>> 'h' in d # search the deque True - Several modules, such as the :mod:`Queue` and :mod:`threading` modules, now take + Several modules, such as the :mod:`queue` and :mod:`threading` modules, now take advantage of :class:`collections.deque` for improved performance. (Contributed by Raymond Hettinger.) @@ -1058,49 +1059,49 @@ complete list of changes, or look through the CVS logs for all the details. isn't a string. (Contributed by John Belmonte and David Goodger.) * The :mod:`curses` module now supports the ncurses extension - :func:`use_default_colors`. On platforms where the terminal supports + :func:`~curses.use_default_colors`. On platforms where the terminal supports transparency, this makes it possible to use a transparent background. (Contributed by Jörg Lehmann.) -* The :mod:`difflib` module now includes an :class:`HtmlDiff` class that creates +* The :mod:`difflib` module now includes an :class:`~difflib.HtmlDiff` class that creates an HTML table showing a side by side comparison of two versions of a text. (Contributed by Dan Gass.) * The :mod:`email` package was updated to version 3.0, which dropped various deprecated APIs and removes support for Python versions earlier than 2.3. The 3.0 version of the package uses a new incremental parser for MIME messages, - available in the :mod:`email.FeedParser` module. The new parser doesn't require + available in the :mod:`!email.FeedParser` module. The new parser doesn't require reading the entire message into memory, and doesn't raise exceptions if a - message is malformed; instead it records any problems in the :attr:`defect` + message is malformed; instead it records any problems in the :attr:`!defect` attribute of the message. (Developed by Anthony Baxter, Barry Warsaw, Thomas Wouters, and others.) * The :mod:`heapq` module has been converted to C. The resulting tenfold improvement in speed makes the module suitable for handling high volumes of - data. In addition, the module has two new functions :func:`nlargest` and - :func:`nsmallest` that use heaps to find the N largest or smallest values in a + data. In addition, the module has two new functions :func:`~heapq.nlargest` and + :func:`~heapq.nsmallest` that use heaps to find the N largest or smallest values in a dataset without the expense of a full sort. (Contributed by Raymond Hettinger.) * The :mod:`httplib ` module now contains constants for HTTP status codes defined in various HTTP-related RFC documents. Constants have names such as - :const:`OK`, :const:`CREATED`, :const:`CONTINUE`, and - :const:`MOVED_PERMANENTLY`; use pydoc to get a full list. (Contributed by + :const:`!OK`, :const:`!CREATED`, :const:`!CONTINUE`, and + :const:`!MOVED_PERMANENTLY`; use pydoc to get a full list. (Contributed by Andrew Eland.) * The :mod:`imaplib` module now supports IMAP's THREAD command (contributed by - Yves Dionne) and new :meth:`deleteacl` and :meth:`myrights` methods (contributed - by Arnaud Mazin). + Yves Dionne) and new :meth:`~imaplib.IMAP4.deleteacl` and + :meth:`~imaplib.IMAP4.myrights` methods (contributed by Arnaud Mazin). * The :mod:`itertools` module gained a ``groupby(iterable[, *func*])`` function. *iterable* is something that can be iterated over to return a stream of elements, and the optional *func* parameter is a function that takes an element and returns a key value; if omitted, the key is simply the element - itself. :func:`groupby` then groups the elements into subsequences which have + itself. :func:`~itertools.groupby` then groups the elements into subsequences which have matching values of the key, and returns a series of 2-tuples containing the key value and an iterator over the subsequence. Here's an example to make this clearer. The *key* function simply returns - whether a number is even or odd, so the result of :func:`groupby` is to return + whether a number is even or odd, so the result of :func:`~itertools.groupby` is to return consecutive runs of odd or even numbers. :: >>> import itertools @@ -1115,8 +1116,8 @@ complete list of changes, or look through the CVS logs for all the details. 0 [12, 14] >>> - :func:`groupby` is typically used with sorted input. The logic for - :func:`groupby` is similar to the Unix ``uniq`` filter which makes it handy for + :func:`~itertools.groupby` is typically used with sorted input. The logic for + :func:`~itertools.groupby` is similar to the Unix ``uniq`` filter which makes it handy for eliminating, counting, or identifying duplicate elements:: >>> word = 'abracadabra' @@ -1153,22 +1154,22 @@ complete list of changes, or look through the CVS logs for all the details. >>> list(i2) # Run the second iterator to exhaustion [1, 2, 3] - Note that :func:`tee` has to keep copies of the values returned by the + Note that :func:`~itertools.tee` has to keep copies of the values returned by the iterator; in the worst case, it may need to keep all of them. This should therefore be used carefully if the leading iterator can run far ahead of the trailing iterator in a long stream of inputs. If the separation is large, then you might as well use :func:`list` instead. When the iterators track closely - with one another, :func:`tee` is ideal. Possible applications include + with one another, :func:`~itertools.tee` is ideal. Possible applications include bookmarking, windowing, or lookahead iterators. (Contributed by Raymond Hettinger.) * A number of functions were added to the :mod:`locale` module, such as - :func:`bind_textdomain_codeset` to specify a particular encoding and a family of + :func:`~locale.bind_textdomain_codeset` to specify a particular encoding and a family of :func:`!l\*gettext` functions that return messages in the chosen encoding. (Contributed by Gustavo Niemeyer.) * Some keyword arguments were added to the :mod:`logging` package's - :func:`basicConfig` function to simplify log configuration. The default + :func:`~logging.basicConfig` function to simplify log configuration. The default behavior is to log messages to standard error, but various keyword arguments can be specified to log to a particular file, change the logging format, or set the logging level. For example:: @@ -1179,10 +1180,10 @@ complete list of changes, or look through the CVS logs for all the details. format='%(levelname):%(process):%(thread):%(message)') Other additions to the :mod:`logging` package include a ``log(level, msg)`` - convenience method, as well as a :class:`TimedRotatingFileHandler` class that + convenience method, as well as a :class:`~logging.handlers.TimedRotatingFileHandler` class that rotates its log files at a timed interval. The module already had - :class:`RotatingFileHandler`, which rotated logs once the file exceeded a - certain size. Both classes derive from a new :class:`BaseRotatingHandler` class + :class:`~logging.handlers.RotatingFileHandler`, which rotated logs once the file exceeded a + certain size. Both classes derive from a new :class:`~logging.handlers.BaseRotatingHandler` class that can be used to implement other rotating handlers. (Changes implemented by Vinay Sajip.) @@ -1192,8 +1193,8 @@ complete list of changes, or look through the CVS logs for all the details. effect is to make :file:`.pyc` files significantly smaller. (Contributed by Martin von Löwis.) -* The :mod:`!nntplib` module's :class:`NNTP` class gained :meth:`description` and - :meth:`descriptions` methods to retrieve newsgroup descriptions for a single +* The :mod:`!nntplib` module's :class:`!NNTP` class gained :meth:`!description` and + :meth:`!descriptions` methods to retrieve newsgroup descriptions for a single group or for a range of groups. (Contributed by Jürgen A. Erhard.) * Two new functions were added to the :mod:`operator` module, @@ -1235,7 +1236,7 @@ complete list of changes, or look through the CVS logs for all the details. *path* is a symlink that points to a destination that doesn't exist. (Contributed by Beni Cherniavsky.) -* A new :func:`getsid` function was added to the :mod:`posix` module that +* A new :func:`~os.getsid` function was added to the :mod:`posix` module that underlies the :mod:`os` module. (Contributed by J. Raynor.) * The :mod:`poplib` module now supports POP over SSL. (Contributed by Hector @@ -1245,8 +1246,8 @@ complete list of changes, or look through the CVS logs for all the details. by Nick Bastin.) * The :mod:`random` module has a new method called ``getrandbits(N)`` that - returns a long integer *N* bits in length. The existing :meth:`randrange` - method now uses :meth:`getrandbits` where appropriate, making generation of + returns a long integer *N* bits in length. The existing :meth:`~random.randrange` + method now uses :meth:`~random.getrandbits` where appropriate, making generation of arbitrarily large random numbers more efficient. (Contributed by Raymond Hettinger.) @@ -1269,24 +1270,24 @@ complete list of changes, or look through the CVS logs for all the details. * The :mod:`signal` module now performs tighter error-checking on the parameters to the :func:`signal.signal` function. For example, you can't set a handler on - the :const:`SIGKILL` signal; previous versions of Python would quietly accept + the :const:`~signal.SIGKILL` signal; previous versions of Python would quietly accept this, but 2.4 will raise a :exc:`RuntimeError` exception. -* Two new functions were added to the :mod:`socket` module. :func:`socketpair` +* Two new functions were added to the :mod:`socket` module. :func:`~socket.socketpair` returns a pair of connected sockets and ``getservbyport(port)`` looks up the service name for a given port number. (Contributed by Dave Cole and Barry Warsaw.) -* The :func:`sys.exitfunc` function has been deprecated. Code should be using +* The :func:`!sys.exitfunc` function has been deprecated. Code should be using the existing :mod:`atexit` module, which correctly handles calling multiple exit - functions. Eventually :func:`sys.exitfunc` will become a purely internal + functions. Eventually :func:`!sys.exitfunc` will become a purely internal interface, accessed only by :mod:`atexit`. * The :mod:`tarfile` module now generates GNU-format tar files by default. (Contributed by Lars Gustäbel.) * The :mod:`threading` module now has an elegantly simple way to support - thread-local data. The module contains a :class:`local` class whose attribute + thread-local data. The module contains a :class:`~threading.local` class whose attribute values are local to different threads. :: import threading @@ -1295,8 +1296,8 @@ complete list of changes, or look through the CVS logs for all the details. data.number = 42 data.url = ('www.python.org', 80) - Other threads can assign and retrieve their own values for the :attr:`number` - and :attr:`url` attributes. You can subclass :class:`local` to initialize + Other threads can assign and retrieve their own values for the :attr:`!number` + and :attr:`!url` attributes. You can subclass :class:`~threading.local` to initialize attributes or to add methods. (Contributed by Jim Fulton.) * The :mod:`timeit` module now automatically disables periodic garbage @@ -1312,7 +1313,7 @@ complete list of changes, or look through the CVS logs for all the details. transmitting multiple XML-RPC calls in a single HTTP operation. (Contributed by Brian Quinlan.) -* The :mod:`mpz`, :mod:`rotor`, and :mod:`xreadlines` modules have been +* The :mod:`!mpz`, :mod:`!rotor`, and :mod:`!xreadlines` modules have been removed. .. ====================================================================== @@ -1336,7 +1337,7 @@ use the Mozilla or Lynx cookie files, and one that stores cookies in the same format as the Perl libwww library. :mod:`urllib2 ` has been changed to interact with :mod:`cookielib `: -:class:`HTTPCookieProcessor` manages a cookie jar that is used when accessing +:class:`~urllib.request.HTTPCookieProcessor` manages a cookie jar that is used when accessing URLs. This module was contributed by John J. Lee. @@ -1352,7 +1353,7 @@ Loper and Tim Peters. Testing can still be as simple as running :func:`doctest.testmod`, but the refactorings allow customizing the module's operation in various ways -The new :class:`DocTestFinder` class extracts the tests from a given object's +The new :class:`~doctest.DocTestFinder` class extracts the tests from a given object's docstrings:: def f (x, y): @@ -1368,7 +1369,7 @@ docstrings:: # Get list of DocTest instances tests = finder.find(f) -The new :class:`DocTestRunner` class then runs individual tests and can produce +The new :class:`~doctest.DocTestRunner` class then runs individual tests and can produce a summary of the results:: runner = doctest.DocTestRunner() @@ -1385,10 +1386,10 @@ The above example produces the following output:: 2 passed and 0 failed. Test passed. -:class:`DocTestRunner` uses an instance of the :class:`OutputChecker` class to +:class:`~doctest.DocTestRunner` uses an instance of the :class:`~doctest.OutputChecker` class to compare the expected output with the actual output. This class takes a number of different flags that customize its behaviour; ambitious users can also write -a completely new subclass of :class:`OutputChecker`. +a completely new subclass of :class:`~doctest.OutputChecker`. The default output checker provides a number of handy features. For example, with the :const:`doctest.ELLIPSIS` option flag, an ellipsis (``...``) in the @@ -1465,7 +1466,7 @@ Some of the changes to Python's build process and to the C API are: lookups without masking exceptions raised during the look-up process. (Contributed by Raymond Hettinger.) -* The :c:expr:`Py_IS_NAN(X)` macro returns 1 if its float or double argument +* The :c:macro:`Py_IS_NAN` macro returns 1 if its float or double argument *X* is a NaN. (Contributed by Tim Peters.) * C code can avoid unnecessary locking by using the new @@ -1479,7 +1480,7 @@ Some of the changes to Python's build process and to the C API are: * A new method flag, :c:macro:`METH_COEXIST`, allows a function defined in slots to co-exist with a :c:type:`PyCFunction` having the same name. This can halve - the access time for a method such as :meth:`set.__contains__`. (Contributed by + the access time for a method such as :meth:`!set.__contains__`. (Contributed by Raymond Hettinger.) * Python can now be built with additional profiling for the interpreter itself, @@ -1493,7 +1494,7 @@ Some of the changes to Python's build process and to the C API are: register". (Contributed by Jeremy Hylton.) * The :c:type:`!tracebackobject` type has been renamed to - :c:type:`PyTracebackObject`. + :c:type:`!PyTracebackObject`. .. ====================================================================== @@ -1517,14 +1518,14 @@ code: trigger a :exc:`FutureWarning` and return a value limited to 32 or 64 bits; instead they return a long integer. -* Integer operations will no longer trigger an :exc:`OverflowWarning`. The - :exc:`OverflowWarning` warning will disappear in Python 2.5. +* Integer operations will no longer trigger an :exc:`!OverflowWarning`. The + :exc:`!OverflowWarning` warning will disappear in Python 2.5. -* The :func:`zip` built-in function and :func:`itertools.izip` now return an +* The :func:`zip` built-in function and :func:`!itertools.izip` now return an empty list instead of raising a :exc:`TypeError` exception if called with no arguments. -* You can no longer compare the :class:`date` and :class:`~datetime.datetime` instances +* You can no longer compare the :class:`~datetime.date` and :class:`~datetime.datetime` instances provided by the :mod:`datetime` module. Two instances of different classes will now always be unequal, and relative comparisons (``<``, ``>``) will raise a :exc:`TypeError`. @@ -1532,7 +1533,8 @@ code: * :func:`!dircache.listdir` now passes exceptions to the caller instead of returning empty lists. -* :func:`LexicalHandler.startDTD` used to receive the public and system IDs in +* :meth:`LexicalHandler.startDTD ` + used to receive the public and system IDs in the wrong order. This has been corrected; applications relying on the wrong order need to be fixed. @@ -1547,9 +1549,9 @@ code: * :const:`None` is now a constant; code that binds a new value to the name ``None`` is now a syntax error. -* The :func:`signals.signal` function now raises a :exc:`RuntimeError` exception +* The :func:`signal.signal` function now raises a :exc:`RuntimeError` exception for certain illegal values; previously these errors would pass silently. For - example, you can no longer set a handler on the :const:`SIGKILL` signal. + example, you can no longer set a handler on the :const:`~signal.SIGKILL` signal. .. ====================================================================== diff --git a/Doc/whatsnew/2.5.rst b/Doc/whatsnew/2.5.rst index 3430ac8668e280c..0449644833b7e9c 100644 --- a/Doc/whatsnew/2.5.rst +++ b/Doc/whatsnew/2.5.rst @@ -16,8 +16,8 @@ release schedule. Python 2.5 was released on September 19, 2006. The changes in Python 2.5 are an interesting mix of language and library improvements. The library enhancements will be more important to Python's user community, I think, because several widely useful packages were added. New -modules include ElementTree for XML processing (:mod:`xml.etree`), -the SQLite database module (:mod:`sqlite`), and the :mod:`ctypes` +modules include ElementTree for XML processing (:mod:`xml.etree.ElementTree`), +the SQLite database module (:mod:`sqlite3`), and the :mod:`ctypes` module for calling C functions. The language changes are of middling significance. Some pleasant new features @@ -134,14 +134,14 @@ PEP 309: Partial Function Application The :mod:`functools` module is intended to contain tools for functional-style programming. -One useful tool in this module is the :func:`partial` function. For programs +One useful tool in this module is the :func:`~functools.partial` function. For programs written in a functional style, you'll sometimes want to construct variants of existing functions that have some of the parameters filled in. Consider a Python function ``f(a, b, c)``; you could create a new function ``g(b, c)`` that was equivalent to ``f(1, b, c)``. This is called "partial function application". -:func:`partial` takes the arguments ``(function, arg1, arg2, ... kwarg1=value1, +:func:`~functools.partial` takes the arguments ``(function, arg1, arg2, ... kwarg1=value1, kwarg2=value2)``. The resulting object is callable, so you can just call it to invoke *function* with the filled-in arguments. @@ -159,7 +159,7 @@ Here's a small but realistic example:: Here's another example, from a program that uses PyGTK. Here a context-sensitive pop-up menu is being constructed dynamically. The callback provided -for the menu option is a partially applied version of the :meth:`open_item` +for the menu option is a partially applied version of the ``open_item()`` method, where the first argument has been provided. :: ... @@ -172,7 +172,7 @@ method, where the first argument has been provided. :: Another function in the :mod:`functools` module is the ``update_wrapper(wrapper, wrapped)`` function that helps you write -well-behaved decorators. :func:`update_wrapper` copies the name, module, and +well-behaved decorators. :func:`~functools.update_wrapper` copies the name, module, and docstring attribute to a wrapper function so that tracebacks inside the wrapped function are easier to understand. For example, you might write:: @@ -183,7 +183,7 @@ function are easier to understand. For example, you might write:: functools.update_wrapper(wrapper, f) return wrapper -:func:`wraps` is a decorator that can be used inside your own decorators to copy +:func:`~functools.wraps` is a decorator that can be used inside your own decorators to copy the wrapped function's information. An alternate version of the previous example would be:: @@ -209,7 +209,7 @@ example would be:: PEP 314: Metadata for Python Software Packages v1.1 =================================================== -Some simple dependency support was added to Distutils. The :func:`setup` +Some simple dependency support was added to Distutils. The :func:`!setup` function now has ``requires``, ``provides``, and ``obsoletes`` keyword parameters. When you build a source distribution using the ``sdist`` command, the dependency information will be recorded in the :file:`PKG-INFO` file. @@ -271,26 +271,26 @@ Let's say you have a package directory like this:: pkg/main.py pkg/string.py -This defines a package named :mod:`pkg` containing the :mod:`pkg.main` and -:mod:`pkg.string` submodules. +This defines a package named ``pkg`` containing the ``pkg.main`` and +``pkg.string`` submodules. Consider the code in the :file:`main.py` module. What happens if it executes the statement ``import string``? In Python 2.4 and earlier, it will first look in the package's directory to perform a relative import, finds :file:`pkg/string.py`, imports the contents of that file as the -:mod:`pkg.string` module, and that module is bound to the name ``string`` in the -:mod:`pkg.main` module's namespace. +``pkg.string`` module, and that module is bound to the name ``string`` in the +``pkg.main`` module's namespace. -That's fine if :mod:`pkg.string` was what you wanted. But what if you wanted +That's fine if ``pkg.string`` was what you wanted. But what if you wanted Python's standard :mod:`string` module? There's no clean way to ignore -:mod:`pkg.string` and look for the standard module; generally you had to look at +``pkg.string`` and look for the standard module; generally you had to look at the contents of ``sys.modules``, which is slightly unclean. Holger Krekel's -:mod:`py.std` package provides a tidier way to perform imports from the standard +:mod:`!py.std` package provides a tidier way to perform imports from the standard library, ``import py; py.std.string.join()``, but that package isn't available on all Python installations. Reading code which relies on relative imports is also less clear, because a -reader may be confused about which module, :mod:`string` or :mod:`pkg.string`, +reader may be confused about which module, :mod:`string` or ``pkg.string``, is intended to be used. Python users soon learned not to duplicate the names of standard library modules in the names of their packages' submodules, but you can't protect against having your submodule's name being used for a new module @@ -313,9 +313,9 @@ name when using the ``from ... import`` form:: from . import string This imports the :mod:`string` module relative to the current package, so in -:mod:`pkg.main` this will import *name1* and *name2* from :mod:`pkg.string`. +``pkg.main`` this will import *name1* and *name2* from ``pkg.string``. Additional leading periods perform the relative import starting from the parent -of the current package. For example, code in the :mod:`A.B.C` module can do:: +of the current package. For example, code in the ``A.B.C`` module can do:: from . import D # Imports A.B.D from .. import E # Imports A.E @@ -331,7 +331,7 @@ statement, only the ``from ... import`` form. PEP written by Aahz; implemented by Thomas Wouters. https://pylib.readthedocs.io/ - The py library by Holger Krekel, which contains the :mod:`py.std` package. + The py library by Holger Krekel, which contains the :mod:`!py.std` package. .. ====================================================================== @@ -347,7 +347,7 @@ Python interpreter, the switch now uses an implementation in a new module, :mod:`runpy`. The :mod:`runpy` module implements a more sophisticated import mechanism so that -it's now possible to run modules in a package such as :mod:`pychecker.checker`. +it's now possible to run modules in a package such as :mod:`!pychecker.checker`. The module also supports alternative import mechanisms such as the :mod:`zipimport` module. This means you can add a .zip archive's path to ``sys.path`` and then use the :option:`-m` switch to execute code from the @@ -392,8 +392,8 @@ write:: The code in *block-1* is executed. If the code raises an exception, the various :keyword:`except` blocks are tested: if the exception is of class -:class:`Exception1`, *handler-1* is executed; otherwise if it's of class -:class:`Exception2`, *handler-2* is executed, and so forth. If no exception is +``Exception1``, *handler-1* is executed; otherwise if it's of class +``Exception2``, *handler-2* is executed, and so forth. If no exception is raised, the *else-block* is executed. No matter what happened previously, the *final-block* is executed once the code @@ -492,21 +492,21 @@ And here's an example of changing the counter:: :keyword:`yield` will usually return :const:`None`, so you should always check for this case. Don't just use its value in expressions unless you're sure that -the :meth:`send` method will be the only method used to resume your generator +the :meth:`~generator.send` method will be the only method used to resume your generator function. -In addition to :meth:`send`, there are two other new methods on generators: +In addition to :meth:`~generator.send`, there are two other new methods on generators: * ``throw(type, value=None, traceback=None)`` is used to raise an exception inside the generator; the exception is raised by the :keyword:`yield` expression where the generator's execution is paused. -* :meth:`close` raises a new :exc:`GeneratorExit` exception inside the generator +* :meth:`~generator.close` raises a new :exc:`GeneratorExit` exception inside the generator to terminate the iteration. On receiving this exception, the generator's code must either raise :exc:`GeneratorExit` or :exc:`StopIteration`. Catching the :exc:`GeneratorExit` exception and returning a value is illegal and will trigger a :exc:`RuntimeError`; if the function raises some other exception, that - exception is propagated to the caller. :meth:`close` will also be called by + exception is propagated to the caller. :meth:`~generator.close` will also be called by Python's garbage collector when the generator is garbage-collected. If you need to run cleanup code when a :exc:`GeneratorExit` occurs, I suggest @@ -521,8 +521,8 @@ function, and a :keyword:`return` statement), but coroutines can be entered, exited, and resumed at many different points (the :keyword:`yield` statements). We'll have to figure out patterns for using coroutines effectively in Python. -The addition of the :meth:`close` method has one side effect that isn't obvious. -:meth:`close` is called when a generator is garbage-collected, so this means the +The addition of the :meth:`~generator.close` method has one side effect that isn't obvious. +:meth:`~generator.close` is called when a generator is garbage-collected, so this means the generator's code gets one last chance to run before the generator is destroyed. This last chance means that ``try...finally`` statements in generators can now be guaranteed to work; the :keyword:`finally` clause will now always get a @@ -534,8 +534,8 @@ seems like a minor bit of language trivia, but using generators and in the following section. Another even more esoteric effect of this change: previously, the -:attr:`gi_frame` attribute of a generator was always a frame object. It's now -possible for :attr:`gi_frame` to be ``None`` once the generator has been +:attr:`!gi_frame` attribute of a generator was always a frame object. It's now +possible for :attr:`!gi_frame` to be ``None`` once the generator has been exhausted. @@ -622,7 +622,7 @@ The :mod:`threading` module's locks and condition variables also support the The lock is acquired before the block is executed and always released once the block is complete. -The new :func:`localcontext` function in the :mod:`decimal` module makes it easy +The new :func:`~decimal.localcontext` function in the :mod:`decimal` module makes it easy to save and restore the current decimal context, which encapsulates the desired precision and rounding characteristics for computations:: @@ -692,7 +692,7 @@ be to let the user write code like this:: The transaction should be committed if the code in the block runs flawlessly or rolled back if there's an exception. Here's the basic interface for -:class:`DatabaseConnection` that I'll assume:: +``DatabaseConnection`` that I'll assume:: class DatabaseConnection: # Database interface @@ -745,7 +745,7 @@ The contextlib module The new :mod:`contextlib` module provides some functions and a decorator that are useful for writing objects for use with the ':keyword:`with`' statement. -The decorator is called :func:`contextmanager`, and lets you write a single +The decorator is called :deco:`~contextlib.contextmanager`, and lets you write a single generator function instead of defining a new class. The generator should yield exactly one value. The code up to the :keyword:`yield` will be executed as the :meth:`~object.__enter__` method, and the value yielded will be the method's return @@ -910,7 +910,7 @@ may therefore need to have some variables changed to :c:type:`Py_ssize_t`. The :c:func:`PyArg_ParseTuple` and :c:func:`Py_BuildValue` functions have a new conversion code, ``n``, for :c:type:`Py_ssize_t`. :c:func:`PyArg_ParseTuple`'s ``s#`` and ``t#`` still output :c:expr:`int` by default, but you can define the -macro :c:macro:`PY_SSIZE_T_CLEAN` before including :file:`Python.h` to make +macro :c:macro:`!PY_SSIZE_T_CLEAN` before including :file:`Python.h` to make them return :c:type:`Py_ssize_t`. :pep:`353` has a section on conversion guidelines that extension authors should @@ -931,19 +931,19 @@ PEP 357: The '__index__' method =============================== The NumPy developers had a problem that could only be solved by adding a new -special method, :meth:`__index__`. When using slice notation, as in +special method, :meth:`~object.__index__`. When using slice notation, as in ``[start:stop:step]``, the values of the *start*, *stop*, and *step* indexes must all be either integers or long integers. NumPy defines a variety of specialized integer types corresponding to unsigned and signed integers of 8, 16, 32, and 64 bits, but there was no way to signal that these types could be used as slice indexes. -Slicing can't just use the existing :meth:`__int__` method because that method +Slicing can't just use the existing :meth:`~object.__int__` method because that method is also used to implement coercion to integers. If slicing used -:meth:`__int__`, floating-point numbers would also become legal slice indexes +:meth:`~object.__int__`, floating-point numbers would also become legal slice indexes and that's clearly an undesirable behaviour. -Instead, a new special method called :meth:`__index__` was added. It takes no +Instead, a new special method called :meth:`~object.__index__` was added. It takes no arguments and returns an integer giving the slice index to use. For example:: class C: @@ -957,7 +957,7 @@ interpreter will check that the type returned is correct, and raises a A corresponding :c:member:`~PyNumberMethods.nb_index` slot was added to the C-level :c:type:`PyNumberMethods` structure to let C extensions implement this protocol. ``PyNumber_Index(obj)`` can be used in extension code to call the -:meth:`__index__` function and retrieve its result. +:meth:`~object.__index__` function and retrieve its result. .. seealso:: @@ -978,7 +978,7 @@ Here are all of the changes that Python 2.5 makes to the core Python language. * The :class:`dict` type has a new hook for letting subclasses provide a default value when a key isn't contained in the dictionary. When a key isn't found, the dictionary's ``__missing__(key)`` method will be called. This hook is used - to implement the new :class:`defaultdict` class in the :mod:`collections` + to implement the new :class:`~collections.defaultdict` class in the :mod:`collections` module. The following example defines a dictionary that returns zero for any missing key:: @@ -1017,7 +1017,7 @@ Here are all of the changes that Python 2.5 makes to the core Python language. (Implemented by Fredrik Lundh following a suggestion by Raymond Hettinger.) -* The :meth:`startswith` and :meth:`endswith` methods of string types now accept +* The :meth:`~str.startswith` and :meth:`~str.endswith` methods of string types now accept tuples of strings to check for. :: def is_image_file (filename): @@ -1028,7 +1028,7 @@ Here are all of the changes that Python 2.5 makes to the core Python language. .. RFE #1491485 * The :func:`min` and :func:`max` built-in functions gained a ``key`` keyword - parameter analogous to the ``key`` argument for :meth:`sort`. This parameter + parameter analogous to the ``key`` argument for :meth:`~list.sort`. This parameter supplies a function that takes a single argument and is called for every value in the list; :func:`min`/:func:`max` will return the element with the smallest/largest return value from this function. For example, to find the @@ -1049,12 +1049,12 @@ Here are all of the changes that Python 2.5 makes to the core Python language. returned by the iterator evaluate as true. (Suggested by Guido van Rossum, and implemented by Raymond Hettinger.) -* The result of a class's :meth:`__hash__` method can now be either a long +* The result of a class's :meth:`~object.__hash__` method can now be either a long integer or a regular integer. If a long integer is returned, the hash of that value is taken. In earlier versions the hash value was required to be a regular integer, but in 2.5 the :func:`id` built-in was changed to always return non-negative numbers, and users often seem to use ``id(self)`` in - :meth:`__hash__` methods (though this is discouraged). + :meth:`~object.__hash__` methods (though this is discouraged). .. Bug #1536021 @@ -1166,7 +1166,7 @@ marked in the following list. .. Patch 1442927 * It's now illegal to mix iterating over a file with ``for line in file`` and - calling the file object's :meth:`read`/:meth:`readline`/:meth:`readlines` + calling the file object's :meth:`!read`/:meth:`!readline`/:meth:`!readlines` methods. Iteration uses an internal buffer and the :meth:`!read\*` methods don't use that buffer. Instead they would return the data following the buffer, causing the data to appear out of order. Mixing iteration and these @@ -1204,7 +1204,7 @@ marked in the following list. Sean Reifschneider at the NeedForSpeed sprint.) * Importing now caches the paths tried, recording whether they exist or not so - that the interpreter makes fewer :c:func:`open` and :c:func:`stat` calls on + that the interpreter makes fewer :c:func:`!open` and :c:func:`!stat` calls on startup. (Contributed by Martin von Löwis and Georg Brandl.) .. Patch 921466 @@ -1226,23 +1226,24 @@ complete list of changes, or look through the SVN logs for all the details. u-LAW encoding has been improved. (Contributed by Lars Immisch.) * The :mod:`codecs` module gained support for incremental codecs. The - :func:`codec.lookup` function now returns a :class:`CodecInfo` instance instead - of a tuple. :class:`CodecInfo` instances behave like a 4-tuple to preserve - backward compatibility but also have the attributes :attr:`encode`, - :attr:`decode`, :attr:`incrementalencoder`, :attr:`incrementaldecoder`, - :attr:`streamwriter`, and :attr:`streamreader`. Incremental codecs can receive + :func:`codecs.lookup` function now returns a :class:`~codecs.CodecInfo` instance instead + of a tuple. :class:`~codecs.CodecInfo` instances behave like a 4-tuple to preserve + backward compatibility but also have the attributes :attr:`~codecs.CodecInfo.encode`, + :attr:`~codecs.CodecInfo.decode`, :attr:`~codecs.CodecInfo.incrementalencoder`, + :attr:`~codecs.CodecInfo.incrementaldecoder`, :attr:`~codecs.CodecInfo.streamwriter`, + and :attr:`~codecs.CodecInfo.streamreader`. Incremental codecs can receive input and produce output in multiple chunks; the output is the same as if the entire input was fed to the non-incremental codec. See the :mod:`codecs` module documentation for details. (Designed and implemented by Walter Dörwald.) .. Patch 1436130 -* The :mod:`collections` module gained a new type, :class:`defaultdict`, that +* The :mod:`collections` module gained a new type, :class:`~collections.defaultdict`, that subclasses the standard :class:`dict` type. The new type mostly behaves like a dictionary but constructs a default value when a key isn't present, automatically adding it to the dictionary for the requested key value. - The first argument to :class:`defaultdict`'s constructor is a factory function + The first argument to :class:`~collections.defaultdict`'s constructor is a factory function that gets called whenever a key is requested but not found. This factory function receives no arguments, so you can use built-in type constructors such as :func:`list` or :func:`int`. For example, you can make an index of words @@ -1268,7 +1269,7 @@ complete list of changes, or look through the SVN logs for all the details. (Contributed by Guido van Rossum.) -* The :class:`deque` double-ended queue type supplied by the :mod:`collections` +* The :class:`~collections.deque` double-ended queue type supplied by the :mod:`collections` module now has a ``remove(value)`` method that removes the first occurrence of *value* in the queue, raising :exc:`ValueError` if the value isn't found. (Contributed by Raymond Hettinger.) @@ -1287,15 +1288,15 @@ complete list of changes, or look through the SVN logs for all the details. Also, the :mod:`pstats` module for analyzing the data measured by the profiler now supports directing the output to any file object by supplying a *stream* - argument to the :class:`Stats` constructor. (Contributed by Skip Montanaro.) + argument to the :class:`~pstats.Stats` constructor. (Contributed by Skip Montanaro.) * The :mod:`csv` module, which parses files in comma-separated value format, received several enhancements and a number of bugfixes. You can now set the maximum size in bytes of a field by calling the ``csv.field_size_limit(new_limit)`` function; omitting the *new_limit* - argument will return the currently set limit. The :class:`reader` class now has - a :attr:`line_num` attribute that counts the number of physical lines read from - the source; records can span multiple physical lines, so :attr:`line_num` is not + argument will return the currently set limit. The :class:`~csv.reader` class now has + a :attr:`~csv.csvreader.line_num` attribute that counts the number of physical lines read from + the source; records can span multiple physical lines, so :attr:`~csv.csvreader.line_num` is not the same as the number of records read. The CSV parser is now stricter about multi-line quoted fields. Previously, if a @@ -1313,12 +1314,12 @@ complete list of changes, or look through the SVN logs for all the details. by Josh Spoerri. It uses the same format characters as :func:`time.strptime` and :func:`time.strftime`:: - from datetime import datetime + import datetime as dt - ts = datetime.strptime('10:13:15 2006-03-07', - '%H:%M:%S %Y-%m-%d') + ts = dt.datetime.strptime('10:13:15 2006-03-07', + '%H:%M:%S %Y-%m-%d') -* The :meth:`SequenceMatcher.get_matching_blocks` method in the :mod:`difflib` +* The :meth:`difflib.SequenceMatcher.get_matching_blocks` method in the :mod:`difflib` module now guarantees to return a minimal list of blocks describing matching subsequences. Previously, the algorithm would occasionally break a block of matching elements into two list entries. (Enhancement by Tim Peters.) @@ -1327,8 +1328,8 @@ complete list of changes, or look through the SVN logs for all the details. being executed at all. This is intended for code snippets that are usage examples intended for the reader and aren't actually test cases. - An *encoding* parameter was added to the :func:`testfile` function and the - :class:`DocFileSuite` class to specify the file's encoding. This makes it + An *encoding* parameter was added to the :func:`~doctest.testfile` function and the + :class:`~doctest.DocFileSuite` class to specify the file's encoding. This makes it easier to use non-ASCII characters in tests contained within a docstring. (Contributed by Bjorn Tillenius.) @@ -1347,20 +1348,20 @@ complete list of changes, or look through the SVN logs for all the details. :func:`input` function to allow opening files in binary or :term:`universal newlines` mode. Another new parameter, *openhook*, lets you use a function other than :func:`open` to open the input files. Once you're iterating over - the set of files, the :class:`FileInput` object's new :meth:`~fileinput.fileno` returns + the set of files, the :class:`~fileinput.FileInput` object's new :meth:`~fileinput.fileno` returns the file descriptor for the currently opened file. (Contributed by Georg Brandl.) -* In the :mod:`gc` module, the new :func:`get_count` function returns a 3-tuple +* In the :mod:`gc` module, the new :func:`~gc.get_count` function returns a 3-tuple containing the current collection counts for the three GC generations. This is accounting information for the garbage collector; when these counts reach a specified threshold, a garbage collection sweep will be made. The existing :func:`gc.collect` function now takes an optional *generation* argument of 0, 1, or 2 to specify which generation to collect. (Contributed by Barry Warsaw.) -* The :func:`nsmallest` and :func:`nlargest` functions in the :mod:`heapq` +* The :func:`~heapq.nsmallest` and :func:`~heapq.nlargest` functions in the :mod:`heapq` module now support a ``key`` keyword parameter similar to the one provided by - the :func:`min`/:func:`max` functions and the :meth:`sort` methods. For + the :func:`min`/:func:`max` functions and the :meth:`~list.sort` methods. For example:: >>> import heapq @@ -1382,7 +1383,7 @@ complete list of changes, or look through the SVN logs for all the details. (Contributed by Raymond Hettinger.) * The :func:`format` function in the :mod:`locale` module has been modified and - two new functions were added, :func:`format_string` and :func:`currency`. + two new functions were added, :func:`~locale.format_string` and :func:`~locale.currency`. The :func:`format` function's *val* parameter could previously be a string as long as no more than one %char specifier appeared; now the parameter must be @@ -1391,10 +1392,10 @@ complete list of changes, or look through the SVN logs for all the details. formatting currency in placing a separator between groups of three digits. To format strings with multiple %char specifiers, use the new - :func:`format_string` function that works like :func:`format` but also supports + :func:`~locale.format_string` function that works like :func:`format` but also supports mixing %char specifiers with arbitrary text. - A new :func:`currency` function was also added that formats a number according + A new :func:`~locale.currency` function was also added that formats a number according to the current locale's settings. (Contributed by Georg Brandl.) @@ -1403,9 +1404,11 @@ complete list of changes, or look through the SVN logs for all the details. * The :mod:`mailbox` module underwent a massive rewrite to add the capability to modify mailboxes in addition to reading them. A new set of classes that include - :class:`mbox`, :class:`MH`, and :class:`Maildir` are used to read mailboxes, and + :class:`~mailbox.mbox`, :class:`~mailbox.MH`, and :class:`~mailbox.Maildir` + are used to read mailboxes, and have an ``add(message)`` method to add messages, ``remove(key)`` to - remove messages, and :meth:`lock`/:meth:`unlock` to lock/unlock the mailbox. + remove messages, and :meth:`~mailbox.Mailbox.lock`/:meth:`~mailbox.Mailbox.unlock` + to lock/unlock the mailbox. The following example converts a maildir-format mailbox into an mbox-format one:: @@ -1430,19 +1433,19 @@ complete list of changes, or look through the SVN logs for all the details. default domain by supplying a *domain* argument to the :func:`!nis.match` and :func:`!nis.maps` functions. (Contributed by Ben Bell.) -* The :mod:`operator` module's :func:`itemgetter` and :func:`attrgetter` +* The :mod:`operator` module's :func:`~operator.itemgetter` and :func:`~operator.attrgetter` functions now support multiple fields. A call such as ``operator.attrgetter('a', 'b')`` will return a function that retrieves the - :attr:`a` and :attr:`b` attributes. Combining this new feature with the - :meth:`sort` method's ``key`` parameter lets you easily sort lists using + ``a`` and ``b`` attributes. Combining this new feature with the + :meth:`~list.sort` method's ``key`` parameter lets you easily sort lists using multiple fields. (Contributed by Raymond Hettinger.) * The :mod:`optparse` module was updated to version 1.5.1 of the Optik library. - The :class:`OptionParser` class gained an :attr:`epilog` attribute, a string - that will be printed after the help message, and a :meth:`destroy` method to + The :class:`~optparse.OptionParser` class gained an :attr:`!epilog` attribute, a string + that will be printed after the help message, and a :meth:`!destroy` method to break reference cycles created by the object. (Contributed by Greg Ward.) -* The :mod:`os` module underwent several changes. The :attr:`stat_float_times` +* The :mod:`os` module underwent several changes. The :attr:`!stat_float_times` variable now defaults to true, meaning that :func:`os.stat` will now return time values as floats. (This doesn't necessarily mean that :func:`os.stat` will return times that are precise to fractions of a second; not all systems support @@ -1453,19 +1456,20 @@ complete list of changes, or look through the SVN logs for all the details. :func:`os.lseek` function. Two new constants for locking are :const:`os.O_SHLOCK` and :const:`os.O_EXLOCK`. - Two new functions, :func:`wait3` and :func:`wait4`, were added. They're similar - the :func:`waitpid` function which waits for a child process to exit and returns - a tuple of the process ID and its exit status, but :func:`wait3` and - :func:`wait4` return additional information. :func:`wait3` doesn't take a + Two new functions, :func:`~os.wait3` and :func:`~os.wait4`, were added. They're similar + the :func:`~os.waitpid` function which waits for a child process to exit and returns + a tuple of the process ID and its exit status, but :func:`~os.wait3` and + :func:`~os.wait4` return additional information. :func:`~os.wait3` doesn't take a process ID as input, so it waits for any child process to exit and returns a 3-tuple of *process-id*, *exit-status*, *resource-usage* as returned from the :func:`resource.getrusage` function. ``wait4(pid)`` does take a process ID. (Contributed by Chad J. Schroeder.) On FreeBSD, the :func:`os.stat` function now returns times with nanosecond - resolution, and the returned object now has :attr:`st_gen` and - :attr:`st_birthtime`. The :attr:`st_flags` attribute is also available, if the - platform supports it. (Contributed by Antti Louko and Diego Pettenò.) + resolution, and the returned object now has :attr:`~os.stat_result.st_gen` and + :attr:`~os.stat_result.st_birthtime`. The :attr:`~os.stat_result.st_flags` + attribute is also available, if the platform supports it. + (Contributed by Antti Louko and Diego Pettenò.) .. (Patch 1180695, 1212117) @@ -1495,21 +1499,21 @@ complete list of changes, or look through the SVN logs for all the details. instead of performing many different operations and reducing the result to a single number as :file:`pystone.py` does. -* The :mod:`pyexpat` module now uses version 2.0 of the Expat parser. +* The :mod:`!pyexpat` module now uses version 2.0 of the Expat parser. (Contributed by Trent Mick.) -* The :class:`~queue.Queue` class provided by the :mod:`Queue` module gained two new - methods. :meth:`join` blocks until all items in the queue have been retrieved +* The :class:`~queue.Queue` class provided by the :mod:`queue` module gained two new + methods. :meth:`~queue.Queue.join` blocks until all items in the queue have been retrieved and all processing work on the items have been completed. Worker threads call - the other new method, :meth:`task_done`, to signal that processing for an item + the other new method, :meth:`~queue.Queue.task_done`, to signal that processing for an item has been completed. (Contributed by Raymond Hettinger.) -* The old :mod:`regex` and :mod:`regsub` modules, which have been deprecated +* The old :mod:`!regex` and :mod:`!regsub` modules, which have been deprecated ever since Python 2.0, have finally been deleted. Other deleted modules: - :mod:`statcache`, :mod:`tzparse`, :mod:`whrandom`. + :mod:`!statcache`, :mod:`!tzparse`, :mod:`!whrandom`. * Also deleted: the :file:`lib-old` directory, which includes ancient modules - such as :mod:`dircmp` and :mod:`ni`, was removed. :file:`lib-old` wasn't on the + such as :mod:`!dircmp` and :mod:`!ni`, was removed. :file:`lib-old` wasn't on the default ``sys.path``, so unless your programs explicitly added the directory to ``sys.path``, this removal shouldn't affect your code. @@ -1519,14 +1523,16 @@ complete list of changes, or look through the SVN logs for all the details. .. Patch #1472854 -* The :mod:`SimpleXMLRPCServer ` and :mod:`DocXMLRPCServer ` classes now have a - :attr:`rpc_paths` attribute that constrains XML-RPC operations to a limited set +* The :mod:`SimpleXMLRPCServer ` and :mod:`DocXMLRPCServer + ` classes now have a :attr:`~xmlrpc.server.SimpleXMLRPCRequestHandler.rpc_paths` + attribute that constrains XML-RPC operations to a limited set of URL paths; the default is to allow only ``'/'`` and ``'/RPC2'``. Setting - :attr:`rpc_paths` to ``None`` or an empty tuple disables this path checking. + :attr:`~xmlrpc.server.SimpleXMLRPCRequestHandler.rpc_paths` to ``None`` or an + empty tuple disables this path checking. .. Bug #1473048 -* The :mod:`socket` module now supports :const:`AF_NETLINK` sockets on Linux, +* The :mod:`socket` module now supports :const:`!AF_NETLINK` sockets on Linux, thanks to a patch from Philippe Biondi. Netlink sockets are a Linux-specific mechanism for communications between a user-space process and kernel code; an introductory article about them is at https://www.linuxjournal.com/article/7356. @@ -1538,19 +1544,19 @@ complete list of changes, or look through the SVN logs for all the details. supports the buffer protocol instead of returning the data as a string. This means you can put the data directly into an array or a memory-mapped file. - Socket objects also gained :meth:`getfamily`, :meth:`gettype`, and - :meth:`getproto` accessor methods to retrieve the family, type, and protocol + Socket objects also gained :meth:`!getfamily`, :meth:`!gettype`, and + :meth:`!getproto` accessor methods to retrieve the family, type, and protocol values for the socket. * New module: the :mod:`!spwd` module provides functions for accessing the shadow password database on systems that support shadow passwords. * The :mod:`struct` is now faster because it compiles format strings into - :class:`Struct` objects with :meth:`pack` and :meth:`unpack` methods. This is + :class:`~struct.Struct` objects with :meth:`~struct.Struct.pack` and :meth:`~struct.Struct.unpack` methods. This is similar to how the :mod:`re` module lets you create compiled regular expression - objects. You can still use the module-level :func:`pack` and :func:`unpack` - functions; they'll create :class:`Struct` objects and cache them. Or you can - use :class:`Struct` instances directly:: + objects. You can still use the module-level :func:`~struct.pack` and :func:`~struct.unpack` + functions; they'll create :class:`~struct.Struct` objects and cache them. Or you can + use :class:`~struct.Struct` instances directly:: s = struct.Struct('ih3s') @@ -1562,7 +1568,7 @@ complete list of changes, or look through the SVN logs for all the details. offset)`` methods. This lets you store data directly into an array or a memory-mapped file. - (:class:`Struct` objects were implemented by Bob Ippolito at the NeedForSpeed + (:class:`~struct.Struct` objects were implemented by Bob Ippolito at the NeedForSpeed sprint. Support for buffer objects was added by Martin Blais, also at the NeedForSpeed sprint.) @@ -1582,8 +1588,8 @@ complete list of changes, or look through the SVN logs for all the details. topmost stack frame currently active in that thread at the time the function is called. (Contributed by Tim Peters.) -* The :class:`TarFile` class in the :mod:`tarfile` module now has an - :meth:`extractall` method that extracts all members from the archive into the +* The :class:`~tarfile.TarFile` class in the :mod:`tarfile` module now has an + :meth:`~tarfile.TarFile.extractall` method that extracts all members from the archive into the current working directory. It's also possible to set a different directory as the extraction target, and to unpack only a subset of the archive's members. @@ -1607,8 +1613,8 @@ complete list of changes, or look through the SVN logs for all the details. * New module: the :mod:`uuid` module generates universally unique identifiers (UUIDs) according to :rfc:`4122`. The RFC defines several different UUID versions that are generated from a starting string, from system properties, or - purely randomly. This module contains a :class:`UUID` class and functions - named :func:`uuid1`, :func:`uuid3`, :func:`uuid4`, and :func:`uuid5` to + purely randomly. This module contains a :class:`~uuid.UUID` class and functions + named :func:`~uuid.uuid1`, :func:`~uuid.uuid3`, :func:`~uuid.uuid4`, and :func:`~uuid.uuid5` to generate different versions of UUID. (Version 2 UUIDs are not specified in :rfc:`4122` and are not supported by this module.) :: @@ -1631,18 +1637,18 @@ complete list of changes, or look through the SVN logs for all the details. (Contributed by Ka-Ping Yee.) -* The :mod:`weakref` module's :class:`WeakKeyDictionary` and - :class:`WeakValueDictionary` types gained new methods for iterating over the - weak references contained in the dictionary. :meth:`iterkeyrefs` and - :meth:`keyrefs` methods were added to :class:`WeakKeyDictionary`, and - :meth:`itervaluerefs` and :meth:`valuerefs` were added to - :class:`WeakValueDictionary`. (Contributed by Fred L. Drake, Jr.) +* The :mod:`weakref` module's :class:`~weakref.WeakKeyDictionary` and + :class:`~weakref.WeakValueDictionary` types gained new methods for iterating over the + weak references contained in the dictionary. :meth:`!iterkeyrefs` and + :meth:`~weakref.WeakKeyDictionary.keyrefs` methods were added to :class:`~weakref.WeakKeyDictionary`, and + :meth:`!itervaluerefs` and :meth:`~weakref.WeakValueDictionary.valuerefs` were added to + :class:`~weakref.WeakValueDictionary`. (Contributed by Fred L. Drake, Jr.) * The :mod:`webbrowser` module received a number of enhancements. It's now usable as a script with ``python -m webbrowser``, taking a URL as the argument; there are a number of switches to control the behaviour (:option:`!-n` for a new browser window, :option:`!-t` for a new tab). New module-level functions, - :func:`open_new` and :func:`open_new_tab`, were added to support this. The + :func:`~webbrowser.open_new` and :func:`~webbrowser.open_new_tab`, were added to support this. The module's :func:`open` function supports an additional feature, an *autoraise* parameter that signals whether to raise the open window when possible. A number of additional browsers were added to the supported list such as Firefox, Opera, @@ -1663,9 +1669,9 @@ complete list of changes, or look through the SVN logs for all the details. .. Patch 1446489 -* The :mod:`zlib` module's :class:`Compress` and :class:`Decompress` objects now +* The :mod:`zlib` module's :class:`!Compress` and :class:`!Decompress` objects now support a :meth:`copy` method that makes a copy of the object's internal state - and returns a new :class:`Compress` or :class:`Decompress` object. + and returns a new :class:`!Compress` or :class:`!Decompress` object. (Contributed by Chris AtLee.) .. Patch 1435422 @@ -1685,34 +1691,34 @@ provides functions for loading shared libraries and calling functions in them. The :mod:`ctypes` package is much fancier. To load a shared library or DLL, you must create an instance of the -:class:`CDLL` class and provide the name or path of the shared library or DLL. +:class:`~ctypes.CDLL` class and provide the name or path of the shared library or DLL. Once that's done, you can call arbitrary functions by accessing them as -attributes of the :class:`CDLL` object. :: +attributes of the :class:`~ctypes.CDLL` object. :: import ctypes libc = ctypes.CDLL('libc.so.6') result = libc.printf("Line of output\n") -Type constructors for the various C types are provided: :func:`c_int`, -:func:`c_float`, :func:`c_double`, :func:`c_char_p` (equivalent to :c:expr:`char +Type constructors for the various C types are provided: :func:`~ctypes.c_int`, +:func:`~ctypes.c_float`, :func:`~ctypes.c_double`, :func:`~ctypes.c_char_p` (equivalent to :c:expr:`char \*`), and so forth. Unlike Python's types, the C versions are all mutable; you -can assign to their :attr:`value` attribute to change the wrapped value. Python +can assign to their :attr:`~ctypes._SimpleCData.value` attribute to change the wrapped value. Python integers and strings will be automatically converted to the corresponding C types, but for other types you must call the correct type constructor. (And I mean *must*; getting it wrong will often result in the interpreter crashing with a segmentation fault.) -You shouldn't use :func:`c_char_p` with a Python string when the C function will +You shouldn't use :func:`~ctypes.c_char_p` with a Python string when the C function will be modifying the memory area, because Python strings are supposed to be immutable; breaking this rule will cause puzzling bugs. When you need a -modifiable memory area, use :func:`create_string_buffer`:: +modifiable memory area, use :func:`~ctypes.create_string_buffer`:: s = "this is a string" buf = ctypes.create_string_buffer(s) libc.strfry(buf) -C functions are assumed to return integers, but you can set the :attr:`restype` +C functions are assumed to return integers, but you can set the :attr:`~ctypes._CFuncPtr.restype` attribute of the function object to change this:: >>> libc.atof('2.71828') @@ -1760,9 +1766,9 @@ The ElementTree package ----------------------- A subset of Fredrik Lundh's ElementTree library for processing XML has been -added to the standard library as :mod:`xml.etree`. The available modules are -:mod:`ElementTree`, :mod:`ElementPath`, and :mod:`ElementInclude` from -ElementTree 1.2.6. The :mod:`cElementTree` accelerator module is also +added to the standard library as :mod:`!xml.etree`. The available modules are +:mod:`!ElementTree`, :mod:`!ElementPath`, and :mod:`!ElementInclude` from +ElementTree 1.2.6. The :mod:`!cElementTree` accelerator module is also included. The rest of this section will provide a brief overview of using ElementTree. @@ -1770,14 +1776,15 @@ Full documentation for ElementTree is available at https://web.archive.org/web/20201124024954/http://effbot.org/zone/element-index.htm. ElementTree represents an XML document as a tree of element nodes. The text -content of the document is stored as the :attr:`text` and :attr:`tail` +content of the document is stored as the :attr:`~xml.etree.ElementTree.Element.text` +and :attr:`~xml.etree.ElementTree.Element.tail` attributes of (This is one of the major differences between ElementTree and the Document Object Model; in the DOM there are many different types of node, -including :class:`TextNode`.) +including :class:`!TextNode`.) -The most commonly used parsing function is :func:`parse`, that takes either a +The most commonly used parsing function is :func:`~xml.etree.ElementTree.parse`, that takes either a string (assumed to contain a filename) or a file-like object and returns an -:class:`ElementTree` instance:: +:class:`~xml.etree.ElementTree.ElementTree` instance:: from xml.etree import ElementTree as ET @@ -1787,11 +1794,13 @@ string (assumed to contain a filename) or a file-like object and returns an 'http://planet.python.org/rss10.xml') tree = ET.parse(feed) -Once you have an :class:`ElementTree` instance, you can call its :meth:`getroot` -method to get the root :class:`Element` node. +Once you have an :class:`~xml.etree.ElementTree.ElementTree` instance, you can +call its :meth:`~xml.etree.ElementTree.ElementTree.getroot` +method to get the root :class:`~xml.etree.ElementTree.Element` node. -There's also an :func:`XML` function that takes a string literal and returns an -:class:`Element` node (not an :class:`ElementTree`). This function provides a +There's also an :func:`~xml.etree.ElementTree.XML` function that takes a string +literal and returns an :class:`~xml.etree.ElementTree.Element` node (not an +:class:`~xml.etree.ElementTree.ElementTree`). This function provides a tidy way to incorporate XML fragments, approaching the convenience of an XML literal:: @@ -1834,7 +1843,7 @@ list-like operations are used to access child nodes. | ``del elem.attrib[name]`` | Deletes attribute *name*. | +-------------------------------+--------------------------------------------+ -Comments and processing instructions are also represented as :class:`Element` +Comments and processing instructions are also represented as :class:`~xml.etree.ElementTree.Element` nodes. To check if a node is a comment or processing instructions:: if elem.tag is ET.Comment: @@ -1842,8 +1851,8 @@ nodes. To check if a node is a comment or processing instructions:: elif elem.tag is ET.ProcessingInstruction: ... -To generate XML output, you should call the :meth:`ElementTree.write` method. -Like :func:`parse`, it can take either a string or a file-like object:: +To generate XML output, you should call the :meth:`xml.etree.ElementTree.ElementTree.write` method. +Like :func:`~xml.etree.ElementTree.parse`, it can take either a string or a file-like object:: # Encoding is US-ASCII tree.write('output.xml') @@ -1913,7 +1922,7 @@ differently. :: Once a hash object has been created, its methods are the same as before: ``update(string)`` hashes the specified string into the current digest -state, :meth:`digest` and :meth:`hexdigest` return the digest value as a binary +state, :meth:`~hashlib.hash.digest` and :meth:`~hashlib.hash.hexdigest` return the digest value as a binary string or a string of hex digits, and :meth:`copy` returns a new hashing object with the same digest state. @@ -1949,7 +1958,7 @@ doesn't include the SQLite code, only the wrapper module. You'll need to have the SQLite libraries and headers installed before compiling Python, and the build process will compile the module when the necessary headers are available. -To use the module, you must first create a :class:`Connection` object that +To use the module, you must first create a :class:`~sqlite3.Connection` object that represents the database. Here the data will be stored in the :file:`/tmp/example` file:: @@ -1957,8 +1966,8 @@ represents the database. Here the data will be stored in the You can also supply the special name ``:memory:`` to create a database in RAM. -Once you have a :class:`Connection`, you can create a :class:`Cursor` object -and call its :meth:`execute` method to perform SQL commands:: +Once you have a :class:`~sqlite3.Connection`, you can create a :class:`~sqlite3.Cursor` object +and call its :meth:`~sqlite3.Cursor.execute` method to perform SQL commands:: c = conn.cursor() @@ -1977,7 +1986,7 @@ is insecure; it makes your program vulnerable to an SQL injection attack. Instead, use the DB-API's parameter substitution. Put ``?`` as a placeholder wherever you want to use a value, and then provide a tuple of values as the -second argument to the cursor's :meth:`execute` method. (Other database modules +second argument to the cursor's :meth:`~sqlite3.Cursor.execute` method. (Other database modules may use a different placeholder, such as ``%s`` or ``:1``.) For example:: # Never do this -- insecure! @@ -1996,8 +2005,8 @@ may use a different placeholder, such as ``%s`` or ``:1``.) For example:: c.execute('insert into stocks values (?,?,?,?,?)', t) To retrieve data after executing a SELECT statement, you can either treat the -cursor as an iterator, call the cursor's :meth:`fetchone` method to retrieve a -single matching row, or call :meth:`fetchall` to get a list of the matching +cursor as an iterator, call the cursor's :meth:`~sqlite3.Cursor.fetchone` method to retrieve a +single matching row, or call :meth:`~sqlite3.Cursor.fetchall` to get a list of the matching rows. This example uses the iterator form:: @@ -2118,7 +2127,7 @@ Changes to Python's build process and to the C API include: discusses the design. To start learning about the code, read the definition of the various AST nodes in :file:`Parser/Python.asdl`. A Python script reads this file and generates a set of C structure definitions in - :file:`Include/Python-ast.h`. The :c:func:`PyParser_ASTFromString` and + :file:`Include/Python-ast.h`. The :c:func:`!PyParser_ASTFromString` and :c:func:`!PyParser_ASTFromFile`, defined in :file:`Include/pythonrun.h`, take Python source as input and return the root of an AST representing the contents. This AST can then be turned into a code object by :c:func:`!PyAST_Compile`. For @@ -2169,9 +2178,9 @@ Changes to Python's build process and to the C API include: * Two new macros can be used to indicate C functions that are local to the current file so that a faster calling convention can be used. - ``Py_LOCAL(type)`` declares the function as returning a value of the + :c:macro:`Py_LOCAL` declares the function as returning a value of the specified *type* and uses a fast-calling qualifier. - ``Py_LOCAL_INLINE(type)`` does the same thing and also requests the + :c:macro:`Py_LOCAL_INLINE` does the same thing and also requests the function be inlined. If macro :c:macro:`!PY_LOCAL_AGGRESSIVE` is defined before :file:`python.h` is included, a set of more aggressive optimizations are enabled for the module; you should benchmark the results to find out if these @@ -2235,9 +2244,9 @@ code: encoding declaration. In Python 2.4 this triggered a warning, not a syntax error. -* Previously, the :attr:`gi_frame` attribute of a generator was always a frame +* Previously, the :attr:`!gi_frame` attribute of a generator was always a frame object. Because of the :pep:`342` changes described in section :ref:`pep-342`, - it's now possible for :attr:`gi_frame` to be ``None``. + it's now possible for :attr:`!gi_frame` to be ``None``. * A new warning, :class:`UnicodeWarning`, is triggered when you attempt to compare a Unicode string and an 8-bit string that can't be converted to Unicode @@ -2258,11 +2267,12 @@ code: return a tuple of arguments instead. The modules also no longer accept the deprecated *bin* keyword parameter. -* Library: The :mod:`SimpleXMLRPCServer ` and :mod:`DocXMLRPCServer ` classes now - have a :attr:`rpc_paths` attribute that constrains XML-RPC operations to a +* Library: The :mod:`SimpleXMLRPCServer ` and :mod:`DocXMLRPCServer + ` classes now have a :attr:`~xmlrpc.server.SimpleXMLRPCRequestHandler.rpc_paths` + attribute that constrains XML-RPC operations to a limited set of URL paths; the default is to allow only ``'/'`` and ``'/RPC2'``. - Setting :attr:`rpc_paths` to ``None`` or an empty tuple disables this path - checking. + Setting :attr:`~xmlrpc.server.SimpleXMLRPCRequestHandler.rpc_paths` to ``None`` + or an empty tuple disables this path checking. * C API: Many functions now use :c:type:`Py_ssize_t` instead of :c:expr:`int` to allow processing more data on 64-bit machines. Extension code may need to make diff --git a/Doc/whatsnew/2.6.rst b/Doc/whatsnew/2.6.rst index 9dbc07a34e20a1d..7100279d8411789 100644 --- a/Doc/whatsnew/2.6.rst +++ b/Doc/whatsnew/2.6.rst @@ -56,7 +56,7 @@ Python 2.6 incorporates new features and syntax from 3.0 while remaining compatible with existing code by not removing older features or syntax. When it's not possible to do that, Python 2.6 tries to do what it can, adding compatibility functions in a -:mod:`future_builtins` module and a :option:`!-3` switch to warn about +:mod:`!future_builtins` module and a :option:`!-3` switch to warn about usages that will become unsupported in 3.0. Some significant new packages have been added to the standard library, @@ -109,7 +109,7 @@ are: Python 3.0 adds several new built-in functions and changes the semantics of some existing builtins. Functions that are new in 3.0 such as :func:`bin` have simply been added to Python 2.6, but existing -builtins haven't been changed; instead, the :mod:`future_builtins` +builtins haven't been changed; instead, the :mod:`!future_builtins` module has versions with the new 3.0 semantics. Code written to be compatible with 3.0 can do ``from future_builtins import hex, map`` as necessary. @@ -118,7 +118,7 @@ A new command-line switch, :option:`!-3`, enables warnings about features that will be removed in Python 3.0. You can run code with this switch to see how much work will be necessary to port code to 3.0. The value of this switch is available -to Python code as the boolean variable :data:`sys.py3kwarning`, +to Python code as the boolean variable :data:`!sys.py3kwarning`, and to C extension code as :c:data:`!Py_Py3kWarningFlag`. .. seealso:: @@ -307,9 +307,9 @@ The :mod:`threading` module's locks and condition variables also support the The lock is acquired before the block is executed and always released once the block is complete. -The :func:`localcontext` function in the :mod:`decimal` module makes it easy -to save and restore the current decimal context, which encapsulates the desired -precision and rounding characteristics for computations:: +The :func:`~decimal.localcontext` function in the :mod:`decimal` module makes +it easy to save and restore the current decimal context, which encapsulates +the desired precision and rounding characteristics for computations:: from decimal import Decimal, Context, localcontext @@ -337,12 +337,12 @@ underlying implementation and should keep reading. A high-level explanation of the context management protocol is: * The expression is evaluated and should result in an object called a "context - manager". The context manager must have :meth:`~object.__enter__` and :meth:`~object.__exit__` - methods. + manager". The context manager must have :meth:`~object.__enter__` and + :meth:`~object.__exit__` methods. -* The context manager's :meth:`~object.__enter__` method is called. The value returned - is assigned to *VAR*. If no ``as VAR`` clause is present, the value is simply - discarded. +* The context manager's :meth:`~object.__enter__` method is called. The value + returned is assigned to *VAR*. If no ``as VAR`` clause is present, the + value is simply discarded. * The code in *BLOCK* is executed. @@ -378,7 +378,7 @@ be to let the user write code like this:: The transaction should be committed if the code in the block runs flawlessly or rolled back if there's an exception. Here's the basic interface for -:class:`DatabaseConnection` that I'll assume:: +:class:`!DatabaseConnection` that I'll assume:: class DatabaseConnection: # Database interface @@ -431,14 +431,15 @@ The contextlib module The :mod:`contextlib` module provides some functions and a decorator that are useful when writing objects for use with the ':keyword:`with`' statement. -The decorator is called :func:`contextmanager`, and lets you write a single -generator function instead of defining a new class. The generator should yield -exactly one value. The code up to the :keyword:`yield` will be executed as the -:meth:`~object.__enter__` method, and the value yielded will be the method's return -value that will get bound to the variable in the ':keyword:`with`' statement's -:keyword:`!as` clause, if any. The code after the :keyword:`!yield` will be -executed in the :meth:`~object.__exit__` method. Any exception raised in the block will -be raised by the :keyword:`!yield` statement. +The decorator is called :func:`~contextlib.contextmanager`, and lets you write +a single generator function instead of defining a new class. The generator +should yield exactly one value. The code up to the :keyword:`yield` will be +executed as the :meth:`~object.__enter__` method, and the value yielded will +be the method's return value that will get bound to the variable in the +':keyword:`with`' statement's :keyword:`!as` clause, if any. The code after +the :keyword:`!yield` will be executed in the :meth:`~object.__exit__` method. +Any exception raised in the block will be raised by the :keyword:`!yield` +statement. Using this decorator, our database example from the previous section could be written as:: @@ -469,7 +470,7 @@ statement both starts a database transaction and acquires a thread lock:: with nested (db_transaction(db), lock) as (cursor, locked): ... -Finally, the :func:`closing` function returns its argument so that it can be +Finally, the :func:`~contextlib.closing` function returns its argument so that it can be bound to a variable, and calls the argument's ``.close()`` method at the end of the block. :: @@ -538,7 +539,7 @@ If you don't like the default directory, it can be overridden by an environment variable. :envvar:`PYTHONUSERBASE` sets the root directory used for all Python versions supporting this feature. On Windows, the directory for application-specific data can be changed by -setting the :envvar:`APPDATA` environment variable. You can also +setting the :envvar:`!APPDATA` environment variable. You can also modify the :file:`site.py` file for your Python installation. The feature can be disabled entirely by running Python with the @@ -568,11 +569,12 @@ The :mod:`multiprocessing` module started out as an exact emulation of the :mod:`threading` module using processes instead of threads. That goal was discarded along the path to Python 2.6, but the general approach of the module is still similar. The fundamental class -is the :class:`Process`, which is passed a callable object and -a collection of arguments. The :meth:`start` method +is the :class:`~multiprocessing.Process`, which is passed a callable object and +a collection of arguments. The :meth:`~multiprocessing.Process.start` method sets the callable running in a subprocess, after which you can call -the :meth:`is_alive` method to check whether the subprocess is still running -and the :meth:`join` method to wait for the process to exit. +the :meth:`~multiprocessing.Process.is_alive` method to check whether the +subprocess is still running and the :meth:`~multiprocessing.Process.join` +method to wait for the process to exit. Here's a simple example where the subprocess will calculate a factorial. The function doing the calculation is written strangely so @@ -619,13 +621,16 @@ the object to communicate. (If the parent were to change the value of the global variable, the child's value would be unaffected, and vice versa.) -Two other classes, :class:`Pool` and :class:`Manager`, provide -higher-level interfaces. :class:`Pool` will create a fixed number of -worker processes, and requests can then be distributed to the workers -by calling :meth:`apply` or :meth:`apply_async` to add a single request, -and :meth:`map` or :meth:`map_async` to add a number of -requests. The following code uses a :class:`Pool` to spread requests -across 5 worker processes and retrieve a list of results:: +Two other classes, :class:`~multiprocessing.pool.Pool` and +:class:`~multiprocessing.Manager`, provide higher-level interfaces. +:class:`~multiprocessing.pool.Pool` will create a fixed number of worker +processes, and requests can then be distributed to the workers by calling +:meth:`~multiprocessing.pool.Pool.apply` or +:meth:`~multiprocessing.pool.Pool.apply_async` to add a single request, and +:meth:`~multiprocessing.pool.Pool.map` or +:meth:`~multiprocessing.pool.Pool.map_async` to add a number of +requests. The following code uses a :class:`~multiprocessing.pool.Pool` to +spread requests across 5 worker processes and retrieve a list of results:: from multiprocessing import Pool @@ -646,15 +651,18 @@ This produces the following output:: 33452526613163807108170062053440751665152000000000 ... -The other high-level interface, the :class:`Manager` class, creates a -separate server process that can hold master copies of Python data +The other high-level interface, the :class:`~multiprocessing.Manager` class, +creates a separate server process that can hold master copies of Python data structures. Other processes can then access and modify these data structures using proxy objects. The following example creates a shared dictionary by calling the :meth:`dict` method; the worker processes then insert values into the dictionary. (Locking is not done for you automatically, which doesn't matter in this example. -:class:`Manager`'s methods also include :meth:`Lock`, :meth:`RLock`, -and :meth:`Semaphore` to create shared locks.) +:class:`~multiprocessing.Manager`'s methods also include +:meth:`~multiprocessing.managers.SyncManager.Lock`, +:meth:`~multiprocessing.managers.SyncManager.RLock`, +and :meth:`~multiprocessing.managers.SyncManager.Semaphore` to create +shared locks.) :: @@ -824,7 +832,7 @@ documentation for a :ref:`complete list `; here's a sample: format, followed by a percent sign. ===== ======================================================================== -Classes and types can define a :meth:`__format__` method to control how they're +Classes and types can define a :meth:`~object.__format__` method to control how they're formatted. It receives a single argument, the format specifier:: def __format__(self, format_spec): @@ -834,7 +842,7 @@ formatted. It receives a single argument, the format specifier:: return str(self) There's also a :func:`format` builtin that will format a single -value. It calls the type's :meth:`__format__` method with the +value. It calls the type's :meth:`~object.__format__` method with the provided specifier:: >>> format(75.6564, '.2f') @@ -997,9 +1005,10 @@ sequence of bytes:: u'\u31ef \u3244' Byte arrays support most of the methods of string types, such as -:meth:`startswith`/:meth:`endswith`, :meth:`find`/:meth:`rfind`, -and some of the methods of lists, such as :meth:`append`, -:meth:`pop`, and :meth:`reverse`. +:meth:`~bytearray.startswith`/:meth:`~bytearray.endswith`, +:meth:`~bytearray.find`/:meth:`~bytearray.rfind`, +and some of the methods of lists, such as :meth:`~bytearray.append`, +:meth:`~bytearray.pop`, and :meth:`~bytearray.reverse`. :: @@ -1028,56 +1037,58 @@ PEP 3116: New I/O Library Python's built-in file objects support a number of methods, but file-like objects don't necessarily support all of them. Objects that -imitate files usually support :meth:`read` and :meth:`write`, but they -may not support :meth:`readline`, for example. Python 3.0 introduces -a layered I/O library in the :mod:`io` module that separates buffering -and text-handling features from the fundamental read and write -operations. +imitate files usually support :meth:`!read` and +:meth:`!write`, but they may not support :meth:`!readline`, +for example. Python 3.0 introduces a layered I/O library in the :mod:`io` +module that separates buffering and text-handling features from the +fundamental read and write operations. There are three levels of abstract base classes provided by the :mod:`io` module: -* :class:`RawIOBase` defines raw I/O operations: :meth:`read`, - :meth:`readinto`, - :meth:`write`, :meth:`seek`, :meth:`tell`, :meth:`truncate`, - and :meth:`close`. +* :class:`~io.RawIOBase` defines raw I/O operations: :meth:`~io.RawIOBase.read`, + :meth:`~io.RawIOBase.readinto`, :meth:`~io.RawIOBase.write`, + :meth:`~io.IOBase.seek`, :meth:`~io.IOBase.tell`, :meth:`~io.IOBase.truncate`, + and :meth:`~io.IOBase.close`. Most of the methods of this class will often map to a single system call. - There are also :meth:`readable`, :meth:`writable`, and :meth:`seekable` - methods for determining what operations a given object will allow. + There are also :meth:`~io.IOBase.readable`, :meth:`~io.IOBase.writable`, + and :meth:`~io.IOBase.seekable` methods for determining what operations a + given object will allow. Python 3.0 has concrete implementations of this class for files and sockets, but Python 2.6 hasn't restructured its file and socket objects in this way. -* :class:`BufferedIOBase` is an abstract base class that +* :class:`~io.BufferedIOBase` is an abstract base class that buffers data in memory to reduce the number of system calls used, making I/O processing more efficient. - It supports all of the methods of :class:`RawIOBase`, - and adds a :attr:`raw` attribute holding the underlying raw object. + It supports all of the methods of :class:`~io.RawIOBase`, + and adds a :attr:`~io.BufferedIOBase.raw` attribute holding the underlying + raw object. There are five concrete classes implementing this ABC. - :class:`BufferedWriter` and :class:`BufferedReader` are for objects - that support write-only or read-only usage that have a :meth:`seek` - method for random access. :class:`BufferedRandom` objects support + :class:`~io.BufferedWriter` and :class:`~io.BufferedReader` are for objects + that support write-only or read-only usage that have a :meth:`~io.IOBase.seek` + method for random access. :class:`~io.BufferedRandom` objects support read and write access upon the same underlying stream, and - :class:`BufferedRWPair` is for objects such as TTYs that have both + :class:`~io.BufferedRWPair` is for objects such as TTYs that have both read and write operations acting upon unconnected streams of data. - The :class:`BytesIO` class supports reading, writing, and seeking + The :class:`~io.BytesIO` class supports reading, writing, and seeking over an in-memory buffer. .. index:: single: universal newlines; What's new -* :class:`TextIOBase`: Provides functions for reading and writing +* :class:`~io.TextIOBase`: Provides functions for reading and writing strings (remember, strings will be Unicode in Python 3.0), - and supporting :term:`universal newlines`. :class:`TextIOBase` defines + and supporting :term:`universal newlines`. :class:`~io.TextIOBase` defines the :meth:`readline` method and supports iteration upon objects. - There are two concrete implementations. :class:`TextIOWrapper` + There are two concrete implementations. :class:`~io.TextIOWrapper` wraps a buffered I/O object, supporting all of the methods for - text I/O and adding a :attr:`buffer` attribute for access - to the underlying object. :class:`StringIO` simply buffers + text I/O and adding a :attr:`~io.TextIOBase.buffer` attribute for access + to the underlying object. :class:`~io.StringIO` simply buffers everything in memory without ever writing anything to disk. (In Python 2.6, :class:`io.StringIO` is implemented in @@ -1161,7 +1172,7 @@ Some object-oriented languages such as Java support interfaces, declaring that a class has a given set of methods or supports a given access protocol. Abstract Base Classes (or ABCs) are an equivalent feature for Python. The ABC support consists of an :mod:`abc` module -containing a metaclass called :class:`ABCMeta`, special handling of +containing a metaclass called :class:`~abc.ABCMeta`, special handling of this metaclass by the :func:`isinstance` and :func:`issubclass` builtins, and a collection of basic ABCs that the Python developers think will be widely useful. Future versions of Python will probably @@ -1171,17 +1182,17 @@ Let's say you have a particular class and wish to know whether it supports dictionary-style access. The phrase "dictionary-style" is vague, however. It probably means that accessing items with ``obj[1]`` works. Does it imply that setting items with ``obj[2] = value`` works? -Or that the object will have :meth:`keys`, :meth:`values`, and :meth:`items` -methods? What about the iterative variants such as :meth:`iterkeys`? :meth:`copy` -and :meth:`update`? Iterating over the object with :func:`iter`? +Or that the object will have :meth:`!keys`, :meth:`!values`, and :meth:`!items` +methods? What about the iterative variants such as :meth:`!iterkeys`? +:meth:`!copy`and :meth:`!update`? Iterating over the object with :func:`!iter`? The Python 2.6 :mod:`collections` module includes a number of different ABCs that represent these distinctions. :class:`Iterable` -indicates that a class defines :meth:`__iter__`, and -:class:`Container` means the class defines a :meth:`__contains__` +indicates that a class defines :meth:`~object.__iter__`, and +:class:`Container` means the class defines a :meth:`~object.__contains__` method and therefore supports ``x in y`` expressions. The basic dictionary interface of getting items, setting items, and -:meth:`keys`, :meth:`values`, and :meth:`items`, is defined by the +:meth:`!keys`, :meth:`!values`, and :meth:`!items`, is defined by the :class:`MutableMapping` ABC. You can derive your own classes from a particular ABC @@ -1195,7 +1206,7 @@ to indicate they support that ABC's interface:: Alternatively, you could write the class without deriving from the desired ABC and instead register the class by -calling the ABC's :meth:`register` method:: +calling the ABC's :meth:`~abc.ABCMeta.register` method:: import collections @@ -1205,10 +1216,10 @@ calling the ABC's :meth:`register` method:: collections.MutableMapping.register(Storage) For classes that you write, deriving from the ABC is probably clearer. -The :meth:`register` method is useful when you've written a new +The :meth:`~abc.ABCMeta.register` method is useful when you've written a new ABC that can describe an existing type or class, or if you want to declare that some third-party class implements an ABC. -For example, if you defined a :class:`PrintableType` ABC, +For example, if you defined a :class:`!PrintableType` ABC, it's legal to do:: # Register Python's types @@ -1255,16 +1266,16 @@ metaclass in a class definition:: ... -In the :class:`Drawable` ABC above, the :meth:`draw_doubled` method +In the :class:`!Drawable` ABC above, the :meth:`!draw_doubled` method renders the object at twice its size and can be implemented in terms -of other methods described in :class:`Drawable`. Classes implementing +of other methods described in :class:`!Drawable`. Classes implementing this ABC therefore don't need to provide their own implementation -of :meth:`draw_doubled`, though they can do so. An implementation -of :meth:`draw` is necessary, though; the ABC can't provide +of :meth:`!draw_doubled`, though they can do so. An implementation +of :meth:`!draw` is necessary, though; the ABC can't provide a useful generic implementation. -You can apply the ``@abstractmethod`` decorator to methods such as -:meth:`draw` that must be implemented; Python will then raise an +You can apply the :deco:`~abc.abstractmethod` decorator to methods such as +:meth:`!draw` that must be implemented; Python will then raise an exception for classes that don't define the method. Note that the exception is only raised when you actually try to create an instance of a subclass lacking the method:: @@ -1288,7 +1299,7 @@ Abstract data attributes can be declared using the def readonly(self): return self._x -Subclasses must then define a :meth:`readonly` property. +Subclasses must then define a ``readonly`` property. .. seealso:: @@ -1528,8 +1539,8 @@ Some smaller changes made to the core Python language are: the :exc:`StopIteration` exception will be raised. (Backported in :issue:`2719`.) -* Tuples now have :meth:`index` and :meth:`count` methods matching the - list type's :meth:`index` and :meth:`count` methods:: +* Tuples now have :meth:`~tuple.index` and :meth:`~tuple.count` methods + matching the list type's :meth:`~list.index` and :meth:`~list.count` methods:: >>> t = (0,1,2,3,4,0,1,2) >>> t.index(3) @@ -2738,13 +2749,13 @@ numbers. .. ====================================================================== -The :mod:`future_builtins` module +The :mod:`!future_builtins` module -------------------------------------- Python 3.0 makes many changes to the repertoire of built-in functions, and most of the changes can't be introduced in the Python 2.x series because they would break compatibility. -The :mod:`future_builtins` module provides versions +The :mod:`!future_builtins` module provides versions of these built-in functions that can be imported when writing 3.0-compatible code. @@ -2811,10 +2822,10 @@ Using the module is simple:: import sys import plistlib - import datetime + import datetime as dt # Create data structure - data_struct = dict(lastAccessed=datetime.datetime.now(), + data_struct = dict(lastAccessed=dt.datetime.now(), version=1, categories=('Personal','Shared','Private')) diff --git a/Doc/whatsnew/2.7.rst b/Doc/whatsnew/2.7.rst index caed3192be871de..7296296d144803d 100644 --- a/Doc/whatsnew/2.7.rst +++ b/Doc/whatsnew/2.7.rst @@ -858,8 +858,8 @@ Some smaller changes made to the core Python language are: .. XXX bytearray doesn't seem to be documented -* When using :class:`@classmethod ` and - :class:`@staticmethod ` to wrap +* When using :deco:`classmethod` and + :deco:`staticmethod` to wrap methods as class or static methods, the wrapper object now exposes the wrapped function as their :attr:`~method.__func__` attribute. @@ -1541,7 +1541,7 @@ changes, or look through the Subversion logs for all the details. buffer API, which fixed a test suite failure (fix by Antoine Pitrou; :issue:`7133`) and automatically set OpenSSL's :c:macro:`!SSL_MODE_AUTO_RETRY`, which will prevent an error - code being returned from :meth:`recv` operations that trigger an SSL + code being returned from :meth:`!recv` operations that trigger an SSL renegotiation (fix by Antoine Pitrou; :issue:`8222`). The :func:`~ssl.SSLContext.wrap_socket` constructor function now takes a @@ -2031,7 +2031,7 @@ version 1.3. Some of the new features are: * ElementTree's code for converting trees to a string has been significantly reworked, making it roughly twice as fast in many cases. The :meth:`ElementTree.write() ` - and :meth:`Element.write` methods now have a *method* parameter that can be + and :meth:`!Element.write` methods now have a *method* parameter that can be "xml" (the default), "html", or "text". HTML mode will output empty elements as ```` instead of ````, and text mode will skip over elements and only output the text chunks. If @@ -2044,7 +2044,7 @@ version 1.3. Some of the new features are: Namespace handling has also been improved. All ``xmlns:`` declarations are now output on the root element, not scattered throughout the resulting XML. You can set the default namespace for a tree - by setting the :attr:`default_namespace` attribute and can + by setting the :attr:`!default_namespace` attribute and can register new prefixes with :meth:`~xml.etree.ElementTree.register_namespace`. In XML mode, you can use the true/false *xml_declaration* parameter to suppress the XML declaration. @@ -2196,8 +2196,6 @@ Changes to Python's build process and to the C API include: locale-independent way. (Added by Eric Smith; :issue:`5793`.) - .. XXX these macros don't seem to be described in the c-api docs. - * Removed function: :c:func:`!PyEval_CallObject` is now only available as a macro. A function version was being kept around to preserve ABI linking compatibility, but that was in 1997; it can certainly be @@ -2234,7 +2232,7 @@ Changes to Python's build process and to the C API include: * When using the :c:type:`PyMemberDef` structure to define attributes of a type, Python will no longer let you try to delete or set a - :c:macro:`T_STRING_INPLACE` attribute. + :c:macro:`!T_STRING_INPLACE` attribute. .. rev 79644 @@ -2259,12 +2257,12 @@ Changes to Python's build process and to the C API include: :issue:`6491`.) * The :program:`configure` script now checks for floating-point rounding bugs - on certain 32-bit Intel chips and defines a :c:macro:`X87_DOUBLE_ROUNDING` + on certain 32-bit Intel chips and defines a :c:macro:`!X87_DOUBLE_ROUNDING` preprocessor definition. No code currently uses this definition, but it's available if anyone wishes to use it. (Added by Mark Dickinson; :issue:`2937`.) - :program:`configure` also now sets a :envvar:`LDCXXSHARED` Makefile + :program:`configure` also now sets a :envvar:`!LDCXXSHARED` Makefile variable for supporting C++ linking. (Contributed by Arfrever Frehtes Taifersar Arahesis; :issue:`1222585`.) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 3c815721a92f8c8..d0449444f07e1d5 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -551,11 +551,12 @@ Patterns and classes If you are using classes to structure your data, you can use as a pattern the class name followed by an argument list resembling a constructor. This -pattern has the ability to capture class attributes into variables:: +pattern has the ability to capture instance attributes into variables:: class Point: - x: int - y: int + def __init__(self, x, y): + self.x = x + self.y = y def location(point): match point: @@ -846,8 +847,8 @@ Other Language Changes respectively. (Contributed by Joshua Bronson, Daniel Pope, and Justin Wang in :issue:`31861`.) -* Static methods (:func:`@staticmethod `) and class methods - (:func:`@classmethod `) now inherit the method attributes +* Static methods (:deco:`staticmethod`) and class methods + (:deco:`classmethod`) now inherit the method attributes (``__module__``, ``__name__``, ``__qualname__``, ``__doc__``, ``__annotations__``) and have a new ``__wrapped__`` attribute. Moreover, static methods are now callable as regular functions. @@ -900,7 +901,7 @@ Improved Modules asyncio ------- -Add missing :meth:`~asyncio.events.AbstractEventLoop.connect_accepted_socket` +Add missing :meth:`~asyncio.loop.connect_accepted_socket` method. (Contributed by Alex Grönholm in :issue:`41332`.) @@ -932,7 +933,7 @@ Base32 Encoding with Extended Hex Alphabet. bdb --- -Add :meth:`~bdb.Breakpoint.clearBreakpoints` to reset all set breakpoints. +Add :meth:`!clearBreakpoints` to reset all set breakpoints. (Contributed by Irit Katriel in :issue:`24160`.) bisect @@ -1397,7 +1398,7 @@ A new verify flag :const:`~ssl.VERIFY_X509_PARTIAL_CHAIN` has been added. sqlite3 ------- -Add audit events for :func:`~sqlite3.connect/handle`, +Add audit events for :func:`~sqlite3.connect`, :meth:`~sqlite3.Connection.enable_load_extension`, and :meth:`~sqlite3.Connection.load_extension`. (Contributed by Erlend E. Aasland in :issue:`43762`.) @@ -1445,8 +1446,8 @@ as a positional-only argument. types ----- -Reintroduce the :data:`types.EllipsisType`, :data:`types.NoneType` -and :data:`types.NotImplementedType` classes, providing a new set +Reintroduce the :class:`types.EllipsisType`, :class:`types.NoneType` +and :class:`types.NotImplementedType` classes, providing a new set of types readily interpretable by type checkers. (Contributed by Bas van Beek in :issue:`41810`.) @@ -1927,7 +1928,7 @@ Changes in the Python API (Contributed by Yurii Karabas, Andrew Svetlov, Yury Selivanov and Kyle Stanley in :issue:`42392`.) -* The :data:`types.FunctionType` constructor now inherits the current builtins +* The :class:`types.FunctionType` constructor now inherits the current builtins if the *globals* dictionary has no ``"__builtins__"`` key, rather than using ``{"None": None}`` as builtins: same behavior as :func:`eval` and :func:`exec` functions. Defining a function with ``def function(...): ...`` diff --git a/Doc/whatsnew/3.12.rst b/Doc/whatsnew/3.12.rst index a65f59c0a72315d..2b6939cd323dcf7 100644 --- a/Doc/whatsnew/3.12.rst +++ b/Doc/whatsnew/3.12.rst @@ -1191,9 +1191,23 @@ Deprecated replaced by :data:`calendar.JANUARY` and :data:`calendar.FEBRUARY`. (Contributed by Prince Roshan in :gh:`103636`.) -* :mod:`collections.abc`: Deprecated :class:`!collections.abc.ByteString`. - Prefer :class:`Sequence` or :class:`collections.abc.Buffer`. - For use in typing, prefer a union, like ``bytes | bytearray``, or :class:`collections.abc.Buffer`. +* :mod:`collections.abc`: Deprecated :class:`collections.abc.ByteString`. + + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` implements + the :ref:`buffer protocol ` at runtime. For use in type + annotations, either use :class:`~collections.abc.Buffer` or a union + that explicitly specifies the types your code supports (e.g., + ``bytes | bytearray | memoryview``). + + :class:`!ByteString` was originally intended to be an abstract class that + would serve as a supertype of both :class:`bytes` and :class:`bytearray`. + However, since the ABC never had any methods, knowing that an object was an + instance of :class:`!ByteString` never actually told you anything useful + about the object. Other common buffer types such as :class:`memoryview` were + also never understood as subtypes of :class:`!ByteString` (either at + runtime or by static type checkers). + + See :pep:`PEP 688 <688#current-options>` for more details. (Contributed by Shantanu Jain in :gh:`91896`.) * :mod:`datetime`: :class:`datetime.datetime`'s :meth:`~datetime.datetime.utcnow` and @@ -1301,7 +1315,7 @@ Deprecated :class:`collections.abc.Hashable` and :class:`collections.abc.Sized` respectively, are deprecated. (:gh:`94309`.) - * :class:`!typing.ByteString`, deprecated since Python 3.9, now causes a + * :class:`typing.ByteString`, deprecated since Python 3.9, now causes a :exc:`DeprecationWarning` to be emitted when it is used. (Contributed by Alex Waygood in :gh:`91896`.) @@ -2233,6 +2247,8 @@ Deprecated .. include:: ../deprecations/c-api-pending-removal-in-3.15.rst +.. include:: ../deprecations/c-api-pending-removal-in-3.16.rst + .. include:: ../deprecations/c-api-pending-removal-in-future.rst Removed diff --git a/Doc/whatsnew/3.13.rst b/Doc/whatsnew/3.13.rst index e20e49325c01d59..bf64f4221d5de35 100644 --- a/Doc/whatsnew/3.13.rst +++ b/Doc/whatsnew/3.13.rst @@ -60,7 +60,7 @@ Summary -- Release Highlights .. This section singles out the most important changes in Python 3.13. Brevity is key. -Python 3.13 is the latest stable release of the Python programming +Python 3.13 is a stable release of the Python programming language, with a mix of changes to the language, the implementation and the standard library. The biggest changes include a new `interactive interpreter @@ -730,6 +730,22 @@ asyncio never awaited). (Contributed by Arthur Tacca and Jason Zhang in :gh:`115957`.) +* The function and methods named ``create_task`` have received a new + ``**kwargs`` argument that is passed through to the task constructor. + This change was accidentally added in 3.13.3, + and broke the API contract for custom task factories. + Several third-party task factories implemented workarounds for this. + In 3.13.4 and later releases the old factory contract is honored + once again (until 3.14). + To keep the workarounds working, the extra ``**kwargs`` argument still + allows passing additional keyword arguments to :class:`~asyncio.Task` + and to custom task factories. + + This affects the following function and methods: + :meth:`asyncio.create_task`, + :meth:`asyncio.loop.create_task`, + :meth:`asyncio.TaskGroup.create_task`. + (Contributed by Thomas Grainger in :gh:`128307`.) base64 ------ @@ -818,7 +834,7 @@ dbm (Contributed by Raymond Hettinger and Erlend E. Aasland in :gh:`100414`.) * Allow removing all items from the database through - the new :meth:`.gdbm.clear` and :meth:`.ndbm.clear` methods. + the new :meth:`!clear` methods of the GDBM and NDBM database objects. (Contributed by Donghee Na in :gh:`107122`.) @@ -1553,8 +1569,6 @@ and are now removed: * :pypi:`bcrypt`: Modern password hashing for your software and your servers. - * :pypi:`passlib`: - Comprehensive password hashing framework supporting over 30 schemes. * :pypi:`argon2-cffi`: The secure Argon2 password hashing algorithm. * :pypi:`legacycrypt`: @@ -1847,6 +1861,7 @@ New Deprecations * Deprecate the non-standard and undocumented :class:`~decimal.Decimal` format specifier ``'N'``, which is only supported in the :mod:`!decimal` module's C implementation. + Scheduled to be removed in Python 3.18. (Contributed by Serhiy Storchaka in :gh:`89902`.) * :mod:`dis`: @@ -1980,7 +1995,7 @@ New Deprecations (Contributed by Alex Waygood in :gh:`105566` and :gh:`105570`.) * Deprecate the :func:`typing.no_type_check_decorator` decorator function, - to be removed in in Python 3.15. + to be removed in Python 3.15. After eight years in the :mod:`typing` module, it has yet to be supported by any major type checker. (Contributed by Alex Waygood in :gh:`106309`.) @@ -2011,6 +2026,10 @@ New Deprecations .. include:: ../deprecations/pending-removal-in-3.17.rst +.. include:: ../deprecations/pending-removal-in-3.18.rst + +.. include:: ../deprecations/pending-removal-in-3.19.rst + .. include:: ../deprecations/pending-removal-in-future.rst CPython Bytecode Changes @@ -2161,7 +2180,7 @@ New Features (Contributed by Sam Gross in :gh:`114329`.) * Add the :c:func:`PyList_Extend` and :c:func:`PyList_Clear` functions, - mirroring the Python :meth:`!list.extend` and :meth:`!list.clear` methods. + mirroring the Python :meth:`list.extend` and :meth:`list.clear` methods. (Contributed by Victor Stinner in :gh:`111138`.) * Add the :c:func:`PyLong_AsInt` function. @@ -2320,7 +2339,7 @@ Limited C API Changes * :c:func:`PySys_AuditTuple` * :c:func:`PyType_GetModuleByDef` - (Contributed by Victor Stinner in :gh:`85283`, :gh:`85283`, and :gh:`116936`.) + (Contributed by Victor Stinner in :gh:`85283` and :gh:`116936`.) * Python built with :option:`--with-trace-refs` (tracing references) now supports the :ref:`Limited API `. @@ -2531,6 +2550,8 @@ Deprecated C APIs .. include:: ../deprecations/c-api-pending-removal-in-3.15.rst +.. include:: ../deprecations/c-api-pending-removal-in-3.16.rst + .. include:: ../deprecations/c-api-pending-removal-in-3.18.rst .. include:: ../deprecations/c-api-pending-removal-in-future.rst @@ -2577,7 +2598,7 @@ Build Changes * The :file:`configure` option :option:`--with-system-libmpdec` now defaults to ``yes``. - The bundled copy of ``libmpdecimal`` will be removed in Python 3.15. + The bundled copy of ``libmpdec`` will be removed in Python 3.16. * Python built with :file:`configure` :option:`--with-trace-refs` (tracing references) is now ABI compatible with the Python release build diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 894f011ec86a308..72ef7e6079281e7 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1,8 +1,9 @@ + **************************** What's new in Python 3.14 **************************** -:Editor: Hugo van Kemenade +:Editors: Adam Turner and Hugo van Kemenade .. Rules for maintenance: @@ -45,404 +46,480 @@ when researching a change. This article explains the new features in Python 3.14, compared to 3.13. - +Python 3.14 was released on 7 October 2025. For full details, see the :ref:`changelog `. .. seealso:: :pep:`745` -- Python 3.14 release schedule -.. note:: - - Prerelease users should be aware that this document is currently in draft - form. It will be updated substantially as Python 3.14 moves towards release, - so it's worth checking back even after reading earlier versions. - -Summary -- release highlights +Summary -- Release highlights ============================= .. This section singles out the most important changes in Python 3.14. Brevity is key. -Python 3.14 beta is the pre-release of the next version of the Python -programming language, with a mix of changes to the language, the -implementation and the standard library. - -The biggest changes to the implementation include template strings (:pep:`750`), -deferred evaluation of annotations (:pep:`649`), -and a new type of interpreter that uses tail calls. - -The library changes include the addition of a new :mod:`!annotationlib` module -for introspecting and wrapping annotations (:pep:`649`), -a new :mod:`!compression.zstd` module for Zstandard support (:pep:`784`), -plus syntax highlighting in the REPL, +Python 3.14 is the latest stable release of the Python programming +language, with a mix of changes to the language, the implementation, +and the standard library. +The biggest changes include :ref:`template string literals +`, +:ref:`deferred evaluation of annotations `, +and support for :ref:`subinterpreters ` in +the standard library. + +The library changes include significantly improved capabilities for +:ref:`introspection in asyncio `, +:ref:`support for Zstandard ` via a new +:mod:`compression.zstd` module, syntax highlighting in the REPL, as well as the usual deprecations and removals, and improvements in user-friendliness and correctness. -.. PEP-sized items next. - -* :ref:`PEP 649: deferred evaluation of annotations ` -* :ref:`PEP 741: Python Configuration C API ` -* :ref:`PEP 750: Template strings ` -* :ref:`PEP 758: Allow except and except* expressions without parentheses ` -* :ref:`PEP 761: Discontinuation of PGP signatures ` -* :ref:`PEP 765: Disallow return/break/continue that exit a finally block ` -* :ref:`PEP 768: Safe external debugger interface for CPython ` -* :ref:`PEP 784: Adding Zstandard to the standard library ` -* :ref:`A new type of interpreter ` -* :ref:`Syntax highlighting in PyREPL `, - and color output in :ref:`unittest `, - :ref:`argparse `, - :ref:`json ` and - :ref:`calendar ` CLIs -* :ref:`Binary releases for the experimental just-in-time compiler ` - - -Incompatible changes -==================== - -On platforms other than macOS and Windows, the default :ref:`start -method ` for :mod:`multiprocessing` -and :class:`~concurrent.futures.ProcessPoolExecutor` switches from -*fork* to *forkserver*. - -See :ref:`(1) ` and -:ref:`(2) ` for details. - -If you encounter :exc:`NameError`\s or pickling errors coming out of -:mod:`multiprocessing` or :mod:`concurrent.futures`, see the -:ref:`forkserver restrictions `. - -The interpreter avoids some reference count modifications internally when -it's safe to do so. This can lead to different values returned from -:func:`sys.getrefcount` and :c:func:`Py_REFCNT` compared to previous versions -of Python. See :ref:`below ` for details. - -New features -============ - -.. _whatsnew314-pep750: - -PEP 750: Template strings -------------------------- - -Template string literals (t-strings) are a generalization of f-strings, -using a ``t`` in place of the ``f`` prefix. Instead of evaluating -to :class:`str`, t-strings evaluate to a new :class:`!string.templatelib.Template` type: +This article doesn't attempt to provide a complete specification +of all new features, but instead gives a convenient overview. +For full details refer to the documentation, +such as the :ref:`Library Reference ` +and :ref:`Language Reference `. +To understand the complete implementation and design rationale for a change, +refer to the PEP for a particular new feature; +but note that PEPs usually are not kept up-to-date +once a feature has been fully implemented. +See `Porting to Python 3.14`_ for guidance on upgrading from +earlier versions of Python. -.. code-block:: python - - from string.templatelib import Template - - name = "World" - template: Template = t"Hello {name}" - -The template can then be combined with functions that operate on the template's -structure to produce a :class:`str` or a string-like result. -For example, sanitizing input: - -.. code-block:: python - - evil = "" - template = t"

{evil}

" - assert html(template) == "

<script>alert('evil')</script>

" - -As another example, generating HTML attributes from data: - -.. code-block:: python - - attributes = {"src": "shrubbery.jpg", "alt": "looks nice"} - template = t"" - assert html(template) == 'looks nice' - -Unlike f-strings, the ``html`` function has access to template attributes -containing the original information: static strings, interpolations, and values -from the original scope. Unlike existing templating approaches, t-strings build -from the well-known f-string syntax and rules. Template systems thus benefit -from Python tooling as they are much closer to the Python language, syntax, -scoping, and more. - -Writing template handlers is straightforward: +-------------- -.. code-block:: python +.. PEP-sized items next. - from string.templatelib import Template, Interpolation +Interpreter improvements: - def lower_upper(template: Template) -> str: - """Render static parts lowercased and interpolations uppercased.""" - parts: list[str] = [] - for item in template: - if isinstance(item, Interpolation): - parts.append(str(item.value).upper()) - else: - parts.append(item.lower()) - return "".join(parts) +* :pep:`649` and :pep:`749`: :ref:`Deferred evaluation of annotations + ` +* :pep:`734`: :ref:`Multiple interpreters in the standard library + ` +* :pep:`750`: :ref:`Template strings ` +* :pep:`758`: :ref:`Allow except and except* expressions without brackets + ` +* :pep:`765`: :ref:`Control flow in finally blocks + ` +* :pep:`768`: :ref:`Safe external debugger interface for CPython + ` +* :ref:`A new type of interpreter ` +* :ref:`Free-threaded mode improvements ` +* :ref:`Improved error messages ` +* :ref:`Incremental garbage collection ` - name = "world" - assert lower_upper(t"HELLO {name}") == "hello WORLD" +Significant improvements in the standard library: -With this in place, developers can write template systems to sanitize SQL, make -safe shell operations, improve logging, tackle modern ideas in web development -(HTML, CSS, and so on), and implement lightweight, custom business DSLs. +* :pep:`784`: :ref:`Zstandard support in the standard library + ` +* :ref:`whatsnew314-asyncio-introspection` +* :ref:`whatsnew314-concurrent-warnings-control` +* :ref:`Syntax highlighting in the default interactive shell + `, and color output in several + standard library CLIs -(Contributed by Jim Baker, Guido van Rossum, Paul Everitt, Koudai Aono, -Lysandros Nikolaou, Dave Peck, Adam Turner, Jelle Zijlstra, Bénédikt Tran, -and Pablo Galindo Salgado in :gh:`132661`.) +C API improvements: -.. seealso:: - :pep:`750`. +* :pep:`741`: :ref:`Python configuration C API ` +Platform support: -.. _whatsnew314-pep768: +* :pep:`776`: Emscripten is now an :ref:`officially supported platform + `, at :pep:`tier 3 <11#tier-3>`. -PEP 768: Safe external debugger interface for CPython ------------------------------------------------------ +Release changes: -:pep:`768` introduces a zero-overhead debugging interface that allows debuggers and profilers -to safely attach to running Python processes. This is a significant enhancement to Python's -debugging capabilities allowing debuggers to forego unsafe alternatives. See -:ref:`below ` for how this feature is leveraged to -implement the new :mod:`pdb` module's remote attaching capabilities. +* :pep:`779`: :ref:`Free-threaded Python is officially supported + ` +* :pep:`761`: :ref:`PGP signatures have been discontinued for official releases + ` +* :ref:`Windows and macOS binary releases now support the experimental + just-in-time compiler ` +* :ref:`Binary releases for Android are now provided ` -The new interface provides safe execution points for attaching debugger code without modifying -the interpreter's normal execution path or adding runtime overhead. This enables tools to -inspect and interact with Python applications in real-time without stopping or restarting -them — a crucial capability for high-availability systems and production environments. -For convenience, CPython implements this interface through the :mod:`sys` module with a -:func:`sys.remote_exec` function:: +New features +============ - sys.remote_exec(pid, script_path) +.. _whatsnew314-deferred-annotations: -This function allows sending Python code to be executed in a target process at the next safe -execution point. However, tool authors can also implement the protocol directly as described -in the PEP, which details the underlying mechanisms used to safely attach to running processes. +:pep:`649` & :pep:`749`: Deferred evaluation of annotations +------------------------------------------------------------ -Here's a simple example that inspects object types in a running Python process: +The :term:`annotations ` on functions, classes, and modules are no +longer evaluated eagerly. Instead, annotations are stored in special-purpose +:term:`annotate functions ` and evaluated only when +necessary (except if ``from __future__ import annotations`` is used). - .. code-block:: python +This change is designed to improve performance and usability of annotations +in Python in most circumstances. The runtime cost for defining annotations is +minimized, but it remains possible to introspect annotations at runtime. +It is no longer necessary to enclose annotations in strings if they +contain forward references. - import os - import sys - import tempfile +The new :mod:`annotationlib` module provides tools for inspecting deferred +annotations. Annotations may be evaluated in the :attr:`~annotationlib.Format.VALUE` +format (which evaluates annotations to runtime values, similar to the behavior in +earlier Python versions), the :attr:`~annotationlib.Format.FORWARDREF` format +(which replaces undefined names with special markers), and the +:attr:`~annotationlib.Format.STRING` format (which returns annotations as strings). - # Create a temporary script - with tempfile.NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: - script_path = f.name - f.write(f"import my_debugger; my_debugger.connect({os.getpid()})") - try: - # Execute in process with PID 1234 - print("Behold! An offering:") - sys.remote_exec(1234, script_path) - finally: - os.unlink(script_path) +This example shows how these formats behave: -The debugging interface has been carefully designed with security in mind and includes several -mechanisms to control access: +.. doctest:: -* A :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` environment variable. -* A :option:`-X disable-remote-debug` command-line option. -* A :option:`--without-remote-debug` configure flag to completely disable the feature at build time. + >>> from annotationlib import get_annotations, Format + >>> def func(arg: Undefined): + ... pass + >>> get_annotations(func, format=Format.VALUE) + Traceback (most recent call last): + ... + NameError: name 'Undefined' is not defined + >>> get_annotations(func, format=Format.FORWARDREF) + {'arg': ForwardRef('Undefined', owner=)} + >>> get_annotations(func, format=Format.STRING) + {'arg': 'Undefined'} -A key implementation detail is that the interface piggybacks on the interpreter's existing evaluation -loop and safe points, ensuring zero overhead during normal execution while providing a reliable way -for external processes to coordinate debugging operations. +The :ref:`porting ` section contains guidance +on changes that may be needed due to these changes, though in the majority of +cases, code will continue working as-is. -(Contributed by Pablo Galindo Salgado, Matt Wozniski, and Ivona Stojanovic in :gh:`131591`.) +(Contributed by Jelle Zijlstra in :pep:`749` and :gh:`119180`; +:pep:`649` was written by Larry Hastings.) .. seealso:: - :pep:`768`. + :pep:`649` + Deferred Evaluation Of Annotations Using Descriptors + :pep:`749` + Implementing PEP 649 + + +.. _whatsnew314-multiple-interpreters: + +:pep:`734`: Multiple interpreters in the standard library +--------------------------------------------------------- + +The CPython runtime supports running multiple copies of Python in the +same process simultaneously and has done so for over 20 years. +Each of these separate copies is called an 'interpreter'. +However, the feature had been available only through +the :ref:`C-API `. + +That limitation is removed in Python 3.14, +with the new :mod:`concurrent.interpreters` module. + +There are at least two notable reasons why using multiple interpreters +has significant benefits: + +* they support a new (to Python), human-friendly concurrency model +* true multi-core parallelism + +For some use cases, concurrency in software improves efficiency and +can simplify design, at a high level. +At the same time, implementing and maintaining all but the simplest concurrency +is often a struggle for the human brain. +That especially applies to plain threads (for example, :mod:`threading`), +where all memory is shared between all threads. + +With multiple isolated interpreters, you can take advantage of a class +of concurrency models, like Communicating Sequential Processes (CSP) +or the actor model, that have found +success in other programming languages, like Smalltalk, Erlang, +Haskell, and Go. Think of multiple interpreters as threads +but with opt-in sharing. + +Regarding multi-core parallelism: as of Python 3.12, interpreters +are now sufficiently isolated from one another to be used in parallel +(see :pep:`684`). This unlocks a variety of CPU-intensive use cases +for Python that were limited by the :term:`GIL`. + +Using multiple interpreters is similar in many ways to +:mod:`multiprocessing`, in that they both provide isolated logical +"processes" that can run in parallel, with no sharing by default. +However, when using multiple interpreters, an application will use +fewer system resources and will operate more efficiently (since it +stays within the same process). Think of multiple interpreters as +having the isolation of processes with the efficiency of threads. + +.. XXX Add an example or two. +.. XXX Link to the not-yet-added HOWTO doc. + +While the feature has been around for decades, multiple interpreters +have not been used widely, due to low awareness and the lack of a +standard library module. Consequently, they currently have several +notable limitations, which are expected to improve significantly now +that the feature is going mainstream. + +Current limitations: + +* starting each interpreter has not been optimized yet +* each interpreter uses more memory than necessary + (work continues on extensive internal sharing between interpreters) +* there aren't many options *yet* for truly sharing objects or other + data between interpreters (other than :type:`memoryview`) +* many third-party extension modules on PyPI are not yet compatible + with multiple interpreters + (all standard library extension modules *are* compatible) +* the approach to writing applications that use multiple isolated + interpreters is mostly unfamiliar to Python users, for now + +The impact of these limitations will depend on future CPython +improvements, how interpreters are used, and what the community solves +through PyPI packages. Depending on the use case, the limitations may +not have much impact, so try it out! + +Furthermore, future CPython releases will reduce or eliminate overhead +and provide utilities that are less appropriate on PyPI. In the +meantime, most of the limitations can also be addressed through +extension modules, meaning PyPI packages can fill any gap for 3.14, and +even back to 3.12 where interpreters were finally properly isolated and +stopped sharing the :term:`GIL`. Likewise, libraries on PyPI are expected +to emerge for high-level abstractions on top of interpreters. + +Regarding extension modules, work is in progress to update some PyPI +projects, as well as tools like Cython, pybind11, nanobind, and PyO3. +The steps for isolating an extension module are found at +:ref:`isolating-extensions-howto`. +Isolating a module has a lot of overlap with what is required to support +:ref:`free-threading `, so the ongoing +work in the community in that area will help accelerate support +for multiple interpreters. + +Also added in 3.14: :ref:`concurrent.futures.InterpreterPoolExecutor +`. + +(Contributed by Eric Snow in :gh:`134939`.) + +.. seealso:: :pep:`734` + + +.. _whatsnew314-template-string-literals: + +:pep:`750`: Template string literals +------------------------------------ + +Template strings are a new mechanism for custom string processing. +They share the familiar syntax of f-strings but, unlike f-strings, +return an object representing the static and interpolated parts of +the string, instead of a simple :class:`str`. + +To write a t-string, use a ``'t'`` prefix instead of an ``'f'``: -.. _whatsnew314-pep784: - -PEP 784: Adding Zstandard to the standard library -------------------------------------------------- - -The new ``compression`` package contains modules :mod:`!compression.lzma`, -:mod:`!compression.bz2`, :mod:`!compression.gzip` and :mod:`!compression.zlib` -which re-export the :mod:`lzma`, :mod:`bz2`, :mod:`gzip` and :mod:`zlib` -modules respectively. The new import names under ``compression`` are the -canonical names for importing these compression modules going forward. However, -the existing modules names have not been deprecated. Any deprecation or removal -of the existing compression modules will occur no sooner than five years after -the release of 3.14. - -The new :mod:`!compression.zstd` module provides compression and decompression -APIs for the Zstandard format via bindings to `Meta's zstd library -`__. Zstandard is a widely adopted, highly -efficient, and fast compression format. In addition to the APIs introduced in -:mod:`!compression.zstd`, support for reading and writing Zstandard compressed -archives has been added to the :mod:`tarfile`, :mod:`zipfile`, and -:mod:`shutil` modules. +.. doctest:: -Here's an example of using the new module to compress some data: + >>> variety = 'Stilton' + >>> template = t'Try some {variety} cheese!' + >>> type(template) + -.. code-block:: python +:class:`~string.templatelib.Template` objects provide access to the static +and interpolated (in curly braces) parts of a string *before* they are combined. +Iterate over :class:`!Template` instances to access their parts in order: - from compression import zstd - import math +.. testsetup:: - data = str(math.pi).encode() * 20 + variety = 'Stilton' + template = t'Try some {variety} cheese!' - compressed = zstd.compress(data) +.. doctest:: - ratio = len(compressed) / len(data) - print(f"Achieved compression ratio of {ratio}") + >>> list(template) + ['Try some ', Interpolation('Stilton', 'variety', None, ''), ' cheese!'] -As can be seen, the API is similar to the APIs of the :mod:`!lzma` and -:mod:`!bz2` modules. +It's easy to write (or call) code to process :class:`!Template` instances. +For example, here's a function that renders static parts lowercase and +:class:`~string.templatelib.Interpolation` instances uppercase: -(Contributed by Emma Harper Smith, Adam Turner, Gregory P. Smith, Tomas Roun, -Victor Stinner, and Rogdham in :gh:`132983`) +.. code-block:: python -.. seealso:: - :pep:`784`. + from string.templatelib import Interpolation + def lower_upper(template): + """Render static parts lowercase and interpolations uppercase.""" + parts = [] + for part in template: + if isinstance(part, Interpolation): + parts.append(str(part.value).upper()) + else: + parts.append(part.lower()) + return ''.join(parts) -.. _whatsnew314-remote-pdb: + name = 'Wenslydale' + template = t'Mister {name}' + assert lower_upper(template) == 'mister WENSLYDALE' -Remote attaching to a running Python process with PDB ------------------------------------------------------ +Because :class:`!Template` instances distinguish between static strings and +interpolations at runtime, they can be useful for sanitising user input. +Writing a :func:`!html` function that escapes user input in HTML is an exercise +left to the reader! +Template processing code can provide improved flexibility. +For instance, a more advanced :func:`!html` function could accept +a :class:`!dict` of HTML attributes directly in the template: -The :mod:`pdb` module now supports remote attaching to a running Python process -using a new ``-p PID`` command-line option: +.. code-block:: python -.. code-block:: sh + attributes = {'src': 'limburger.jpg', 'alt': 'lovely cheese'} + template = t'' + assert html(template) == 'lovely cheese' - python -m pdb -p 1234 +Of course, template processing code does not need to return a string-like result. +An even *more* advanced :func:`!html` could return a custom type representing +a DOM-like structure. -This will connect to the Python process with the given PID and allow you to -debug it interactively. Notice that due to how the Python interpreter works -attaching to a remote process that is blocked in a system call or waiting for -I/O will only work once the next bytecode instruction is executed or when the -process receives a signal. +With t-strings in place, developers can write systems that sanitise SQL, +make safe shell operations, improve logging, tackle modern ideas in web +development (HTML, CSS, and so on), and implement lightweight custom business DSLs. -This feature uses :pep:`768` and the :func:`sys.remote_exec` function -to attach to the remote process and send the PDB commands to it. +(Contributed by Jim Baker, Guido van Rossum, Paul Everitt, Koudai Aono, +Lysandros Nikolaou, Dave Peck, Adam Turner, Jelle Zijlstra, Bénédikt Tran, +and Pablo Galindo Salgado in :gh:`132661`.) +.. seealso:: :pep:`750`. -(Contributed by Matt Wozniski and Pablo Galindo in :gh:`131591`.) -.. seealso:: - :pep:`768`. +.. _whatsnew314-remote-debugging: +:pep:`768`: Safe external debugger interface +-------------------------------------------- -.. _whatsnew314-pep758: +Python 3.14 introduces a zero-overhead debugging interface that allows +debuggers and profilers to safely attach to running Python processes +without stopping or restarting them. +This is a significant enhancement to Python's debugging capabilities, +meaning that unsafe alternatives are no longer required. -PEP 758 – Allow except and except* expressions without parentheses ------------------------------------------------------------------- +The new interface provides safe execution points for attaching debugger code +without modifying the interpreter's normal execution path +or adding any overhead at runtime. +Due to this, tools can now inspect and interact with Python applications +in real-time, which is a crucial capability for high-availability systems +and production environments. -The :keyword:`except` and :keyword:`except* ` expressions now allow -parentheses to be omitted when there are multiple exception types and the ``as`` clause is not used. -For example the following expressions are now valid: +For convenience, this interface is implemented in the :func:`sys.remote_exec` +function. For example: .. code-block:: python - try: - release_new_sleep_token_album() - except AlbumNotFound, SongsTooGoodToBeReleased: - print("Sorry, no new album this year.") - - # The same applies to except* (for exception groups): - try: - release_new_sleep_token_album() - except* AlbumNotFound, SongsTooGoodToBeReleased: - print("Sorry, no new album this year.") - -Check :pep:`758` for more details. + import sys + from tempfile import NamedTemporaryFile -(Contributed by Pablo Galindo and Brett Cannon in :gh:`131831`.) + with NamedTemporaryFile(mode='w', suffix='.py', delete=False) as f: + script_path = f.name + f.write(f'import my_debugger; my_debugger.connect({os.getpid()})') -.. seealso:: - :pep:`758`. + # Execute in process with PID 1234 + print('Behold! An offering:') + sys.remote_exec(1234, script_path) -.. _whatsnew314-pep649: +This function allows sending Python code to be executed in a target process +at the next safe execution point. +However, tool authors can also implement the protocol directly as described +in the PEP, which details the underlying mechanisms used to safely attach to +running processes. -PEP 649: deferred evaluation of annotations -------------------------------------------- +The debugging interface has been carefully designed with security in mind +and includes several mechanisms to control access: -The :term:`annotations ` on functions, classes, and modules are no -longer evaluated eagerly. Instead, annotations are stored in special-purpose -:term:`annotate functions ` and evaluated only when -necessary. This is specified in :pep:`649` and :pep:`749`. +* A :envvar:`PYTHON_DISABLE_REMOTE_DEBUG` environment variable. +* A :option:`-X disable-remote-debug` command-line option. +* A :option:`--without-remote-debug` configure flag to completely disable + the feature at build time. -This change is designed to make annotations in Python more performant and more -usable in most circumstances. The runtime cost for defining annotations is -minimized, but it remains possible to introspect annotations at runtime. -It is usually no longer necessary to enclose annotations in strings if they -contain forward references. +(Contributed by Pablo Galindo Salgado, Matt Wozniski, and Ivona Stojanovic +in :gh:`131591`.) -The new :mod:`annotationlib` module provides tools for inspecting deferred -annotations. Annotations may be evaluated in the :attr:`~annotationlib.Format.VALUE` -format (which evaluates annotations to runtime values, similar to the behavior in -earlier Python versions), the :attr:`~annotationlib.Format.FORWARDREF` format -(which replaces undefined names with special markers), and the -:attr:`~annotationlib.Format.STRING` format (which returns annotations as strings). +.. seealso:: :pep:`768`. -This example shows how these formats behave: -.. doctest:: +.. _whatsnew314-tail-call-interpreter: - >>> from annotationlib import get_annotations, Format - >>> def func(arg: Undefined): - ... pass - >>> get_annotations(func, format=Format.VALUE) - Traceback (most recent call last): - ... - NameError: name 'Undefined' is not defined - >>> get_annotations(func, format=Format.FORWARDREF) - {'arg': ForwardRef('Undefined', owner=)} - >>> get_annotations(func, format=Format.STRING) - {'arg': 'Undefined'} +A new type of interpreter +------------------------- -Implications for annotated code -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +A new type of interpreter has been added to CPython. +It uses tail calls between small C functions that implement individual +Python opcodes, rather than one large C ``case`` statement. +For certain newer compilers, this interpreter provides +significantly better performance. Preliminary benchmarks suggest a geometric +mean of 3-5% faster on the standard ``pyperformance`` benchmark suite, +depending on platform and architecture. +The baseline is Python 3.14 built with Clang 19, without this new interpreter. -If you define annotations in your code (for example, for use with a static type -checker), then this change probably does not affect you: you can keep -writing annotations the same way you did with previous versions of Python. +This interpreter currently only works with Clang 19 and newer +on x86-64 and AArch64 architectures. +However, a future release of GCC is expected to support this as well. -You will likely be able to remove quoted strings in annotations, which are frequently -used for forward references. Similarly, if you use ``from __future__ import annotations`` -to avoid having to write strings in annotations, you may well be able to -remove that import. However, if you rely on third-party libraries that read annotations, -those libraries may need changes to support unquoted annotations before they -work as expected. +This feature is opt-in for now. Enabling profile-guided optimization is highly +recommended when using the new interpreter as it is the only configuration +that has been tested and validated for improved performance. +For further information, see :option:`--with-tail-call-interp`. -Implications for readers of ``__annotations__`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +.. note:: -If your code reads the ``__annotations__`` attribute on objects, you may want -to make changes in order to support code that relies on deferred evaluation of -annotations. For example, you may want to use :func:`annotationlib.get_annotations` -with the :attr:`~annotationlib.Format.FORWARDREF` format, as the :mod:`dataclasses` -module now does. + This is not to be confused with `tail call optimization`__ of Python + functions, which is currently not implemented in CPython. -Related changes -^^^^^^^^^^^^^^^ + This new interpreter type is an internal implementation detail of the CPython + interpreter. It doesn't change the visible behavior of Python programs at + all. It can improve their performance, but doesn't change anything else. -The changes in Python 3.14 are designed to rework how ``__annotations__`` -works at runtime while minimizing breakage to code that contains -annotations in source code and to code that reads ``__annotations__``. However, -if you rely on undocumented details of the annotation behavior or on private -functions in the standard library, there are many ways in which your code may -not work in Python 3.14. To safeguard your code against future changes, -use only the documented functionality of the :mod:`annotationlib` module. + __ https://en.wikipedia.org/wiki/Tail_call -``from __future__ import annotations`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +(Contributed by Ken Jin in :gh:`128563`, with ideas on how to implement this +in CPython by Mark Shannon, Garrett Gu, Haoran Xu, and Josh Haberman.) -In Python 3.7, :pep:`563` introduced the ``from __future__ import annotations`` -directive, which turns all annotations into strings. This directive is now -considered deprecated and it is expected to be removed in a future version of Python. -However, this removal will not happen until after Python 3.13, the last version of -Python without deferred evaluation of annotations, reaches its end of life in 2029. -In Python 3.14, the behavior of code using ``from __future__ import annotations`` -is unchanged. +.. _whatsnew314-free-threaded-cpython: + +Free-threaded mode improvements +------------------------------- + +CPython's free-threaded mode (:pep:`703`), initially added in 3.13, +has been significantly improved in Python 3.14. +The implementation described in PEP 703 has been finished, including C API +changes, and temporary workarounds in the interpreter were replaced with +more permanent solutions. +The specializing adaptive interpreter (:pep:`659`) is now enabled +in free-threaded mode, which along with many other optimizations +greatly improves its performance. +The performance penalty on single-threaded code in free-threaded mode +is now roughly 5-10%, depending on the platform and C compiler used. + +From Python 3.14, when compiling extension modules for the free-threaded build of +CPython on Windows, the preprocessor variable ``Py_GIL_DISABLED`` now needs to +be specified by the build backend, as it will no longer be determined +automatically by the C compiler. For a running interpreter, the setting that +was used at compile time can be found using :func:`sysconfig.get_config_var`. + +The new :option:`-X context_aware_warnings <-X>` flag controls if +:ref:`concurrent safe warnings control ` +is enabled. The flag defaults to true for the free-threaded build +and false for the GIL-enabled build. + +A new :data:`~sys.flags.thread_inherit_context` flag has been added, +which if enabled means that threads created with :class:`threading.Thread` +start with a copy of the :class:`~contextvars.Context()` of the caller of +:meth:`~threading.Thread.start`. Most significantly, this makes the warning +filtering context established by :class:`~warnings.catch_warnings` be +"inherited" by threads (or asyncio tasks) started within that context. It also +affects other modules that use context variables, such as the :mod:`decimal` +context manager. +This flag defaults to true for the free-threaded build and false for +the GIL-enabled build. + +(Contributed by Sam Gross, Matt Page, Neil Schemenauer, Thomas Wouters, +Donghee Na, Kirill Podoprigora, Ken Jin, Itamar Oren, Brett Simmers, +Dino Viehland, Nathan Goldbaum, Ralf Gommers, Lysandros Nikolaou, Kumar Aditya, +Edgar Margffoy, and many others. +Some of these contributors are employed by Meta, which has continued to provide +significant engineering resources to support this project.) + + +.. _whatsnew314-improved-error-messages: Improved error messages ----------------------- @@ -463,47 +540,12 @@ Improved error messages ^^^^^^ SyntaxError: invalid syntax. Did you mean 'while'? - >>> asynch def fetch_data(): - ... pass - Traceback (most recent call last): - File "", line 1 - asynch def fetch_data(): - ^^^^^^ - SyntaxError: invalid syntax. Did you mean 'async'? - - >>> async def foo(): - ... awaid fetch_data() - Traceback (most recent call last): - File "", line 2 - awaid fetch_data() - ^^^^^ - SyntaxError: invalid syntax. Did you mean 'await'? - - >>> raisee ValueError("Error") - Traceback (most recent call last): - File "", line 1 - raisee ValueError("Error") - ^^^^^^ - SyntaxError: invalid syntax. Did you mean 'raise'? - While the feature focuses on the most common cases, some variations of misspellings may still result in regular syntax errors. (Contributed by Pablo Galindo in :gh:`132449`.) -* When unpacking assignment fails due to incorrect number of variables, the - error message prints the received number of values in more cases than before. - (Contributed by Tushar Sadhwani in :gh:`122239`.) - - .. code-block:: pycon - - >>> x, y, z = 1, 2, 3, 4 - Traceback (most recent call last): - File "", line 1, in - x, y, z = 1, 2, 3, 4 - ^^^^^^^ - ValueError: too many values to unpack (expected 3, got 4) - -* :keyword:`elif` statements that follow an :keyword:`else` block now have a specific error message. +* :keyword:`elif` statements that follow an :keyword:`else` block now have + a specific error message. (Contributed by Steele Farnsworth in :gh:`129902`.) .. code-block:: pycon @@ -519,11 +561,9 @@ Improved error messages ^^^^ SyntaxError: 'elif' block follows an 'else' block -* If a statement (:keyword:`pass`, :keyword:`del`, :keyword:`return`, - :keyword:`yield`, :keyword:`raise`, :keyword:`break`, :keyword:`continue`, - :keyword:`assert`, :keyword:`import`, :keyword:`from`) is passed to the - :ref:`if_expr` after :keyword:`else`, or one of :keyword:`pass`, - :keyword:`break`, or :keyword:`continue` is passed before :keyword:`if`, then the +* If a statement is passed to the :ref:`if_expr` after :keyword:`else`, + or one of :keyword:`pass`, :keyword:`break`, or :keyword:`continue` + is passed before :keyword:`if`, then the error message highlights where the :token:`~python-grammar:expression` is required. (Contributed by Sergey Miryanov in :gh:`129515`.) @@ -543,10 +583,9 @@ Improved error messages ^^^^^^^^ SyntaxError: expected expression before 'if', but statement is given - * When incorrectly closed strings are detected, the error message suggests - that the string may be intended to be part of the string. (Contributed by - Pablo Galindo in :gh:`88535`.) + that the string may be intended to be part of the string. + (Contributed by Pablo Galindo in :gh:`88535`.) .. code-block:: pycon @@ -555,8 +594,8 @@ Improved error messages SyntaxError: invalid syntax. Is this intended to be part of the string? * When strings have incompatible prefixes, the error now shows - which prefixes are incompatible. (Contributed by - Nikita Sobolev in :gh:`133197`.) + which prefixes are incompatible. + (Contributed by Nikita Sobolev in :gh:`133197`.) .. code-block:: pycon @@ -573,80 +612,101 @@ Improved error messages - Except handlers: ``except ... as ...`` - Pattern-match cases: ``case ... as ...`` - (Contributed by Nikita Sobolev in :gh:`123539`, - :gh:`123562`, and :gh:`123440`.) + (Contributed by Nikita Sobolev in :gh:`123539`, :gh:`123562`, and :gh:`123440`.) + +* Improved error message when trying to add an instance of an unhashable type to + a :class:`dict` or :class:`set`. + (Contributed by CF Bolz-Tereick and Victor Stinner in :gh:`132828`.) .. code-block:: pycon - >>> import ast as arr[0] - File "", line 1 - import ast as arr[0] - ^^^^^^ - SyntaxError: cannot use subscript as import target + >>> s = set() + >>> s.add({'pages': 12, 'grade': 'A'}) + Traceback (most recent call last): + File "", line 1, in + s.add({'pages': 12, 'grade': 'A'}) + ~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + TypeError: cannot use 'dict' as a set element (unhashable type: 'dict') + >>> d = {} + >>> l = [1, 2, 3] + >>> d[l] = 12 + Traceback (most recent call last): + File "", line 1, in + d[l] = 12 + ~^^^ + TypeError: cannot use 'list' as a dict key (unhashable type: 'list') + +* Improved error message when an object supporting the synchronous + context manager protocol is entered using :keyword:`async with` + instead of :keyword:`with`, + and vice versa for the asynchronous context manager protocol. + (Contributed by Bénédikt Tran in :gh:`128398`.) + -.. seealso:: - :pep:`649`. +.. _whatsnew314-zstandard: +:pep:`784`: Zstandard support in the standard library +----------------------------------------------------- -.. _whatsnew314-pep741: +The new :mod:`!compression` package contains modules :mod:`!compression.lzma`, +:mod:`!compression.bz2`, :mod:`!compression.gzip` and :mod:`!compression.zlib` +which re-export the :mod:`lzma`, :mod:`bz2`, :mod:`gzip` and :mod:`zlib` +modules respectively. The new import names under :mod:`!compression` are the +preferred names for importing these compression modules from Python 3.14. However, +the existing modules names have not been deprecated. Any deprecation or removal +of the existing compression modules will occur no sooner than five years after +the release of 3.14. -PEP 741: Python Configuration C API ------------------------------------ +The new :mod:`!compression.zstd` module provides compression and decompression +APIs for the Zstandard format via bindings to `Meta's zstd library +`__. Zstandard is a widely adopted, highly +efficient, and fast compression format. In addition to the APIs introduced in +:mod:`!compression.zstd`, support for reading and writing Zstandard compressed +archives has been added to the :mod:`tarfile`, :mod:`zipfile`, and +:mod:`shutil` modules. -Add a :ref:`PyInitConfig C API ` to configure the Python -initialization without relying on C structures and the ability to make -ABI-compatible changes in the future. +Here's an example of using the new module to compress some data: -Complete the :pep:`587` :ref:`PyConfig C API ` by adding -:c:func:`PyInitConfig_AddModule` which can be used to add a built-in extension -module; feature previously referred to as the “inittab”. +.. code-block:: python -Add :c:func:`PyConfig_Get` and :c:func:`PyConfig_Set` functions to get and set -the current runtime configuration. + from compression import zstd + import math -PEP 587 “Python Initialization Configuration” unified all the ways to configure -the Python initialization. This PEP unifies also the configuration of the -Python preinitialization and the Python initialization in a single API. -Moreover, this PEP only provides a single choice to embed Python, instead of -having two “Python” and “Isolated” choices (PEP 587), to simplify the API -further. + data = str(math.pi).encode() * 20 + compressed = zstd.compress(data) + ratio = len(compressed) / len(data) + print(f"Achieved compression ratio of {ratio}") -The lower level PEP 587 PyConfig API remains available for use cases with an -intentionally higher level of coupling to CPython implementation details (such -as emulating the full functionality of CPython’s CLI, including its -configuration mechanisms). +As can be seen, the API is similar to the APIs of the :mod:`!lzma` and +:mod:`!bz2` modules. -(Contributed by Victor Stinner in :gh:`107954`.) +(Contributed by Emma Harper Smith, Adam Turner, Gregory P. Smith, Tomas Roun, +Victor Stinner, and Rogdham in :gh:`132983`.) + +.. seealso:: :pep:`784`. -.. seealso:: - :pep:`741`. .. _whatsnew314-asyncio-introspection: Asyncio introspection capabilities ---------------------------------- -Added a new command-line interface to inspect running Python processes using -asynchronous tasks, available via: - -.. code-block:: bash - - python -m asyncio ps PID +Added a new command-line interface to inspect running Python processes +using asynchronous tasks, available via ``python -m asyncio ps PID`` +or ``python -m asyncio pstree PID``. -This tool inspects the given process ID (PID) and displays information about -currently running asyncio tasks. It outputs a task table: a flat -listing of all tasks, their names, their coroutine stacks, and which tasks are -awaiting them. +The ``ps`` subcommand inspects the given process ID (PID) and displays +information about currently running asyncio tasks. +It outputs a task table: a flat listing of all tasks, their names, +their coroutine stacks, and which tasks are awaiting them. -.. code-block:: bash - - python -m asyncio pstree PID -This tool fetches the same information, but renders a visual async call tree, -showing coroutine relationships in a hierarchical format. This command is -particularly useful for debugging long-running or stuck asynchronous programs. -It can help developers quickly identify where a program is blocked, what tasks -are pending, and how coroutines are chained together. +The ``pstree`` subcommand fetches the same information, but instead renders a +visual async call tree, showing coroutine relationships in a hierarchical format. +This command is particularly useful for debugging long-running or stuck +asynchronous programs. +It can help developers quickly identify where a program is blocked, +what tasks are pending, and how coroutines are chained together. For example given this code: @@ -654,23 +714,25 @@ For example given this code: import asyncio - async def play(track): + async def play_track(track): await asyncio.sleep(5) - print(f"🎵 Finished: {track}") + print(f'🎵 Finished: {track}') - async def album(name, tracks): + async def play_album(name, tracks): async with asyncio.TaskGroup() as tg: for track in tracks: - tg.create_task(play(track), name=track) + tg.create_task(play_track(track), name=track) async def main(): async with asyncio.TaskGroup() as tg: tg.create_task( - album("Sundowning", ["TNDNBTG", "Levitate"]), name="Sundowning") + play_album('Sundowning', ['TNDNBTG', 'Levitate']), + name='Sundowning') tg.create_task( - album("TMBTE", ["DYWTYLM", "Aqua Regia"]), name="TMBTE") + play_album('TMBTE', ['DYWTYLM', 'Aqua Regia']), + name='TMBTE') - if __name__ == "__main__": + if __name__ == '__main__': asyncio.run(main()) Executing the new tool on the running process will yield a table like this: @@ -679,277 +741,312 @@ Executing the new tool on the running process will yield a table like this: python -m asyncio ps 12345 - tid task id task name coroutine chain awaiter name awaiter id - --------------------------------------------------------------------------------------------------------------------------------------- - 8138752 0x564bd3d0210 Task-1 0x0 - 8138752 0x564bd3d0410 Sundowning _aexit -> __aexit__ -> main Task-1 0x564bd3d0210 - 8138752 0x564bd3d0610 TMBTE _aexit -> __aexit__ -> main Task-1 0x564bd3d0210 - 8138752 0x564bd3d0810 TNDNBTG _aexit -> __aexit__ -> album Sundowning 0x564bd3d0410 - 8138752 0x564bd3d0a10 Levitate _aexit -> __aexit__ -> album Sundowning 0x564bd3d0410 - 8138752 0x564bd3e0550 DYWTYLM _aexit -> __aexit__ -> album TMBTE 0x564bd3d0610 - 8138752 0x564bd3e0710 Aqua Regia _aexit -> __aexit__ -> album TMBTE 0x564bd3d0610 + tid task id task name coroutine stack awaiter chain awaiter name awaiter id + ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + 1935500 0x7fc930c18050 Task-1 TaskGroup._aexit -> TaskGroup.__aexit__ -> main 0x0 + 1935500 0x7fc930c18230 Sundowning TaskGroup._aexit -> TaskGroup.__aexit__ -> album TaskGroup._aexit -> TaskGroup.__aexit__ -> main Task-1 0x7fc930c18050 + 1935500 0x7fc93173fa50 TMBTE TaskGroup._aexit -> TaskGroup.__aexit__ -> album TaskGroup._aexit -> TaskGroup.__aexit__ -> main Task-1 0x7fc930c18050 + 1935500 0x7fc93173fdf0 TNDNBTG sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album Sundowning 0x7fc930c18230 + 1935500 0x7fc930d32510 Levitate sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album Sundowning 0x7fc930c18230 + 1935500 0x7fc930d32890 DYWTYLM sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album TMBTE 0x7fc93173fa50 + 1935500 0x7fc93161ec30 Aqua Regia sleep -> play TaskGroup._aexit -> TaskGroup.__aexit__ -> album TMBTE 0x7fc93173fa50 - -or: +or a tree like this: .. code-block:: bash python -m asyncio pstree 12345 └── (T) Task-1 - └── main - └── __aexit__ - └── _aexit + └── main example.py:13 + └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:72 + └── TaskGroup._aexit Lib/asyncio/taskgroups.py:121 ├── (T) Sundowning - │ └── album - │ └── __aexit__ - │ └── _aexit + │ └── album example.py:8 + │ └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:72 + │ └── TaskGroup._aexit Lib/asyncio/taskgroups.py:121 │ ├── (T) TNDNBTG + │ │ └── play example.py:4 + │ │ └── sleep Lib/asyncio/tasks.py:702 │ └── (T) Levitate + │ └── play example.py:4 + │ └── sleep Lib/asyncio/tasks.py:702 └── (T) TMBTE - └── album - └── __aexit__ - └── _aexit + └── album example.py:8 + └── TaskGroup.__aexit__ Lib/asyncio/taskgroups.py:72 + └── TaskGroup._aexit Lib/asyncio/taskgroups.py:121 ├── (T) DYWTYLM + │ └── play example.py:4 + │ └── sleep Lib/asyncio/tasks.py:702 └── (T) Aqua Regia + └── play example.py:4 + └── sleep Lib/asyncio/tasks.py:702 If a cycle is detected in the async await graph (which could indicate a programming issue), the tool raises an error and lists the cycle paths that -prevent tree construction. +prevent tree construction: -(Contributed by Pablo Galindo, Łukasz Langa, Yury Selivanov, and Marta -Gomez Macias in :gh:`91048`.) +.. code-block:: bash -.. _whatsnew314-tail-call: + python -m asyncio pstree 12345 -A new type of interpreter -------------------------- + ERROR: await-graph contains cycles - cannot print a tree! -A new type of interpreter has been added to CPython. -It uses tail calls between small C functions that implement individual -Python opcodes, rather than one large C case statement. -For certain newer compilers, this interpreter provides -significantly better performance. Preliminary numbers on our machines suggest -anywhere up to 30% faster Python code, and a geometric mean of 3-5% -faster on ``pyperformance`` depending on platform and architecture. The -baseline is Python 3.14 built with Clang 19 without this new interpreter. + cycle: Task-2 → Task-3 → Task-2 -This interpreter currently only works with Clang 19 and newer -on x86-64 and AArch64 architectures. However, we expect -that a future release of GCC will support this as well. +(Contributed by Pablo Galindo, Łukasz Langa, Yury Selivanov, and Marta +Gomez Macias in :gh:`91048`.) -This feature is opt-in for now. We highly recommend enabling profile-guided -optimization with the new interpreter as it is the only configuration we have -tested and can validate its improved performance. -For further information on how to build Python, see -:option:`--with-tail-call-interp`. -.. note:: +.. _whatsnew314-concurrent-warnings-control: - This is not to be confused with `tail call optimization`__ of Python - functions, which is currently not implemented in CPython. +Concurrent safe warnings control +-------------------------------- - This new interpreter type is an internal implementation detail of the CPython - interpreter. It doesn't change the visible behavior of Python programs at - all. It can improve their performance, but doesn't change anything else. +The :class:`warnings.catch_warnings` context manager will now optionally +use a context variable for warning filters. This is enabled by setting +the :data:`~sys.flags.context_aware_warnings` flag, either with the ``-X`` +command-line option or an environment variable. This gives predictable +warnings control when using :class:`~warnings.catch_warnings` combined with +multiple threads or asynchronous tasks. The flag defaults to true for the +free-threaded build and false for the GIL-enabled build. - __ https://en.wikipedia.org/wiki/Tail_call +(Contributed by Neil Schemenauer and Kumar Aditya in :gh:`130010`.) -.. attention:: - This section previously reported a 9-15% geometric mean speedup. This number has since been - cautiously revised down to 3-5%. While we expect performance results to be better - than what we report, our estimates are more conservative due to a - `compiler bug `_ found in - Clang/LLVM 19, which causes the normal interpreter to be slower. We were unaware of this bug, - resulting in inaccurate results. We sincerely apologize for - communicating results that were only accurate for LLVM v19.1.x and v20.1.0. In the meantime, - the bug has been fixed in LLVM v20.1.1 and for the upcoming v21.1, but it will remain - unfixed for LLVM v19.1.x and v20.1.0. Thus - any benchmarks with those versions of LLVM may produce inaccurate numbers. - (Thanks to Nelson Elhage for bringing this to light.) +Other language changes +====================== -(Contributed by Ken Jin in :gh:`128563`, with ideas on how to implement this -in CPython by Mark Shannon, Garrett Gu, Haoran Xu, and Josh Haberman.) - - -.. _whatsnew314-pyrepl-highlighting: - -Syntax highlighting in PyREPL ------------------------------ - -The default :term:`interactive` shell now highlights Python syntax as you -type. The feature is enabled by default unless the -:envvar:`PYTHON_BASIC_REPL` environment is set or any color-disabling -environment variables are used. See :ref:`using-on-controlling-color` for -details. - -The default color theme for syntax highlighting strives for good contrast -and uses exclusively the 4-bit VGA standard ANSI color codes for maximum -compatibility. The theme can be customized using an experimental API -``_colorize.set_theme()``. This can be called interactively, as well as -in the :envvar:`PYTHONSTARTUP` script. - -(Contributed by Łukasz Langa in :gh:`131507`.) - - -.. _whatsnew314-jit-compiler: - -Binary releases for the experimental just-in-time compiler ----------------------------------------------------------- - -The official macOS and Windows release binaries now include an *experimental* -just-in-time (JIT) compiler. Although it is **not** recommended for production -use, it can be tested by setting :envvar:`PYTHON_JIT=1 ` as an -environment variable. Downstream source builds and redistributors can use the -:option:`--enable-experimental-jit=yes-off` configuration option for similar -behavior. - -The JIT is at an early stage and still in active development. As such, the -typical performance impact of enabling it can range from 10% slower to 20% -faster, depending on workload. To aid in testing and evaluation, a set of -introspection functions has been provided in the :data:`sys._jit` namespace. -:func:`sys._jit.is_available` can be used to determine if the current executable -supports JIT compilation, while :func:`sys._jit.is_enabled` can be used to tell -if JIT compilation has been enabled for the current process. - -Currently, the most significant missing functionality is that native debuggers -and profilers like ``gdb`` and ``perf`` are unable to unwind through JIT frames -(Python debuggers and profilers, like :mod:`pdb` or :mod:`profile`, continue to -work without modification). Free-threaded builds do not support JIT compilation. - -Please report any bugs or major performance regressions that you encounter! - -.. seealso:: :pep:`744` +* All Windows code pages are now supported as 'cpXXX' codecs on Windows. + (Contributed by Serhiy Storchaka in :gh:`123803`.) +* Implement mixed-mode arithmetic rules combining real and complex numbers + as specified by the C standard since C99. + (Contributed by Sergey B Kirpichev in :gh:`69639`.) -Other language changes -====================== +* More syntax errors are now detected regardless of optimisation and + the :option:`-O` command-line option. + This includes writes to ``__debug__``, incorrect use of :keyword:`await`, + and asynchronous comprehensions outside asynchronous functions. + For example, ``python -O -c 'assert (__debug__ := 1)'`` + or ``python -O -c 'assert await 1'`` now produce :exc:`SyntaxError`\ s. + (Contributed by Irit Katriel and Jelle Zijlstra in :gh:`122245` & :gh:`121637`.) -* The default :term:`interactive` shell now supports import autocompletion. - This means that typing ``import foo`` and pressing ```` will suggest - modules starting with ``foo``. Similarly, typing ``from foo import b`` will - suggest submodules of ``foo`` starting with ``b``. Note that autocompletion - of module attributes is not currently supported. - (Contributed by Tomas Roun in :gh:`69605`.) +* When subclassing a pure C type, the C slots for the new type + are no longer replaced with a wrapped version on class creation + if they are not explicitly overridden in the subclass. + (Contributed by Tomasz Pytel in :gh:`132284`.) -* The :func:`map` built-in now has an optional keyword-only *strict* flag - like :func:`zip` to check that all the iterables are of equal length. - (Contributed by Wannes Boeykens in :gh:`119793`.) -* Incorrect usage of :keyword:`await` and asynchronous comprehensions - is now detected even if the code is optimized away by the :option:`-O` - command-line option. For example, ``python -O -c 'assert await 1'`` - now produces a :exc:`SyntaxError`. (Contributed by Jelle Zijlstra in :gh:`121637`.) +Built-ins +--------- -* Writes to ``__debug__`` are now detected even if the code is optimized - away by the :option:`-O` command-line option. For example, - ``python -O -c 'assert (__debug__ := 1)'`` now produces a - :exc:`SyntaxError`. (Contributed by Irit Katriel in :gh:`122245`.) +* The :meth:`bytes.fromhex` and :meth:`bytearray.fromhex` methods now accept + ASCII :class:`bytes` and :term:`bytes-like objects `. + (Contributed by Daniel Pope in :gh:`129349`.) * Add class methods :meth:`float.from_number` and :meth:`complex.from_number` to convert a number to :class:`float` or :class:`complex` type correspondingly. - They raise an error if the argument is a string. + They raise a :exc:`TypeError` if the argument is not a real number. (Contributed by Serhiy Storchaka in :gh:`84978`.) -* Implement mixed-mode arithmetic rules combining real and complex numbers as - specified by C standards since C99. - (Contributed by Sergey B Kirpichev in :gh:`69639`.) - -* All Windows code pages are now supported as "cpXXX" codecs on Windows. - (Contributed by Serhiy Storchaka in :gh:`123803`.) - -* :class:`super` objects are now :mod:`pickleable ` and - :mod:`copyable `. - (Contributed by Serhiy Storchaka in :gh:`125767`.) - -* The :class:`memoryview` type now supports subscription, - making it a :term:`generic type`. - (Contributed by Brian Schubert in :gh:`126012`.) - * Support underscore and comma as thousands separators in the fractional part for floating-point presentation types of the new-style string formatting (with :func:`format` or :ref:`f-strings`). (Contributed by Sergey B Kirpichev in :gh:`87790`.) -* The :func:`bytes.fromhex` and :func:`bytearray.fromhex` methods now accept - ASCII :class:`bytes` and :term:`bytes-like objects `. - (Contributed by Daniel Pope in :gh:`129349`.) - -* Support ``\z`` as a synonym for ``\Z`` in :mod:`regular expressions `. - It is interpreted unambiguously in many other regular expression engines, - unlike ``\Z``, which has subtly different behavior. - (Contributed by Serhiy Storchaka in :gh:`133306`.) +* The :func:`int` function no longer delegates to :meth:`~object.__trunc__`. + Classes that want to support conversion to :func:`!int` must implement + either :meth:`~object.__int__` or :meth:`~object.__index__`. + (Contributed by Mark Dickinson in :gh:`119743`.) -* ``\B`` in :mod:`regular expression ` now matches empty input string. - Now it is always the opposite of ``\b``. - (Contributed by Serhiy Storchaka in :gh:`124130`.) +* The :func:`map` function now has an optional keyword-only *strict* flag + like :func:`zip` to check that all the iterables are of equal length. + (Contributed by Wannes Boeykens in :gh:`119793`.) -* iOS and macOS apps can now be configured to redirect ``stdout`` and - ``stderr`` content to the system log. (Contributed by Russell Keith-Magee in - :gh:`127592`.) +* The :class:`memoryview` type now supports subscription, + making it a :term:`generic type`. + (Contributed by Brian Schubert in :gh:`126012`.) -* The iOS testbed is now able to stream test output while the test is running. - The testbed can also be used to run the test suite of projects other than - CPython itself. (Contributed by Russell Keith-Magee in :gh:`127592`.) +* Using :data:`NotImplemented` in a boolean context + will now raise a :exc:`TypeError`. + This has raised a :exc:`DeprecationWarning` since Python 3.9. + (Contributed by Jelle Zijlstra in :gh:`118767`.) -* Three-argument :func:`pow` now tries calling :meth:`~object.__rpow__` if - necessary. Previously it was only called in two-argument :func:`!pow` and the - binary power operator. +* Three-argument :func:`pow` now tries calling :meth:`~object.__rpow__` + if necessary. + Previously it was only called in two-argument :func:`!pow` + and the binary power operator. (Contributed by Serhiy Storchaka in :gh:`130104`.) -* Add a built-in implementation for HMAC (:rfc:`2104`) using formally verified - code from the `HACL* `__ project. - This implementation is used as a fallback when the OpenSSL implementation - of HMAC is not available. - (Contributed by Bénédikt Tran in :gh:`99108`.) +* :class:`super` objects are now :mod:`copyable ` and :mod:`pickleable + `. + (Contributed by Serhiy Storchaka in :gh:`125767`.) + + +Command line and environment +---------------------------- * The import time flag can now track modules that are already loaded ('cached'), via the new :option:`-X importtime=2 <-X>`. When such a module is imported, the ``self`` and ``cumulative`` times are replaced by the string ``cached``. + Values above ``2`` for ``-X importtime`` are now reserved for future use. - (Contributed by Noah Kim and Adam Turner in :gh:`118655`.) -* When subclassing from a pure C type, the C slots for the new type are no - longer replaced with a wrapped version on class creation if they are not - explicitly overridden in the subclass. - (Contributed by Tomasz Pytel in :gh:`132329`.) + (Contributed by Noah Kim and Adam Turner in :gh:`118655`.) * The command-line option :option:`-c` now automatically dedents its code argument before execution. The auto-dedentation behavior mirrors :func:`textwrap.dedent`. (Contributed by Jon Crall and Steven Sun in :gh:`103998`.) -* Improve error message when an object supporting the synchronous - context manager protocol is entered using :keyword:`async - with` instead of :keyword:`with`. - And vice versa with the asynchronous context manager protocol. - (Contributed by Bénédikt Tran in :gh:`128398`.) - * :option:`!-J` is no longer a reserved flag for Jython_, and now has no special meaning. (Contributed by Adam Turner in :gh:`133336`.) .. _Jython: https://www.jython.org/ -.. _whatsnew314-pep765: -PEP 765: Disallow ``return``/``break``/``continue`` that exit a ``finally`` block ---------------------------------------------------------------------------------- +.. _whatsnew314-bracketless-except: + +PEP 758: Allow ``except`` and ``except*`` expressions without brackets +---------------------------------------------------------------------- + +The :keyword:`except` and :keyword:`except* ` expressions +now allow brackets to be omitted when there are multiple exception types +and the ``as`` clause is not used. +For example: + +.. code-block:: python + + try: + connect_to_server() + except TimeoutError, ConnectionRefusedError: + print('The network has ceased to be!') + +(Contributed by Pablo Galindo and Brett Cannon in :pep:`758` and :gh:`131831`.) + -The compiler emits a :exc:`SyntaxWarning` when a :keyword:`return`, :keyword:`break` or -:keyword:`continue` statements appears where it exits a :keyword:`finally` block. +.. _whatsnew314-finally-syntaxwarning: + +PEP 765: Control flow in :keyword:`finally` blocks +-------------------------------------------------- + +The compiler now emits a :exc:`SyntaxWarning` when a :keyword:`return`, +:keyword:`break`, or :keyword:`continue` statement have the effect of +leaving a :keyword:`finally` block. This change is specified in :pep:`765`. +In situations where this change is inconvenient (such as those where the +warnings are redundant due to code linting), the :ref:`warning filter +` can be used to turn off all syntax warnings by adding +``ignore::SyntaxWarning`` as a filter. This can be specified in combination +with a filter that converts other warnings to errors (for example, passing +``-Werror -Wignore::SyntaxWarning`` as CLI options, or setting +``PYTHONWARNINGS=error,ignore::SyntaxWarning``). + +Note that applying such a filter at runtime using the :mod:`warnings` module +will only suppress the warning in code that is compiled *after* the filter is +adjusted. Code that is compiled prior to the filter adjustment (for example, +when a module is imported) will still emit the syntax warning. + +(Contributed by Irit Katriel in :gh:`130080`.) + + +.. _incremental-garbage-collection: +.. _whatsnew314-incremental-gc: + +Garbage collection +------------------ + +**From Python 3.14.5 onwards:** + +The garbage collector (GC) has changed in Python 3.14.5. + +Python 3.14.0-3.14.4 shipped with a new incremental GC. +However, due to a number of `reports +`__ +of significant memory pressure in production environments, +it has been reverted back to the generational GC from 3.13. +This is the GC now used in Python 3.14.5 and later. + +**Previously in Python 3.14.0-3.14.4:** + +The cycle garbage collector is now incremental. +This means that maximum pause times are reduced +by an order of magnitude or more for larger heaps. + +There are now only two generations: young and old. +When :func:`gc.collect` is not called directly, the +GC is invoked a little less frequently. When invoked, it +collects the young generation and an increment of the +old generation, instead of collecting one or more generations. + +The behavior of :func:`!gc.collect` changes slightly: + +* ``gc.collect(1)``: Performs an increment of garbage collection, + rather than collecting generation 1. +* Other calls to :func:`!gc.collect` are unchanged. + +(Contributed by Mark Shannon in :gh:`108362`.) + + +Default interactive shell +------------------------- + +.. _whatsnew314-pyrepl-highlighting: + +* The default :term:`interactive` shell now highlights Python syntax. + The feature is enabled by default, save if :envvar:`PYTHON_BASIC_REPL` + or any other environment variable that disables colour is set. + See :ref:`using-on-controlling-color` for details. + + The default color theme for syntax highlighting strives for good contrast + and exclusively uses the 4-bit VGA standard ANSI color codes for maximum + compatibility. The theme can be customized using an experimental API + :func:`!_colorize.set_theme`. + This can be called interactively or in the :envvar:`PYTHONSTARTUP` script. + Note that this function has no stability guarantees, + and may change or be removed. + + (Contributed by Łukasz Langa in :gh:`131507`.) + +* The default :term:`interactive` shell now supports import auto-completion. + This means that typing ``import co`` and pressing :kbd:`` will suggest + modules starting with ``co``. Similarly, typing ``from concurrent import i`` + will suggest submodules of ``concurrent`` starting with ``i``. + Note that autocompletion of module attributes is not currently supported. + (Contributed by Tomas Roun in :gh:`69605`.) + New modules =========== -* :mod:`annotationlib`: For introspecting :term:`annotations `. - See :pep:`749` for more details. +* :mod:`annotationlib`: + For introspecting :term:`annotations `. + See :ref:`PEP 749 ` for more details. (Contributed by Jelle Zijlstra in :gh:`119180`.) +* :mod:`compression` (including :mod:`compression.zstd`): + A package for compression-related modules, + including a new module to support the Zstandard compression format. + See :ref:`PEP 784 ` for more details. + (Contributed by Emma Harper Smith, Adam Turner, Gregory P. Smith, Tomas Roun, + Victor Stinner, and Rogdham in :gh:`132983`.) + +* :mod:`concurrent.interpreters`: + Support for multiple interpreters in the standard library. + See :ref:`PEP 734 ` for more details. + (Contributed by Eric Snow in :gh:`134939`.) + +* :mod:`string.templatelib`: + Support for template string literals (t-strings). + See :ref:`PEP 750 ` for more details. + (Contributed by Jim Baker, Guido van Rossum, Paul Everitt, Koudai Aono, + Lysandros Nikolaou, Dave Peck, Adam Turner, Jelle Zijlstra, Bénédikt Tran, + and Pablo Galindo Salgado in :gh:`132661`.) + Improved modules ================ @@ -967,20 +1064,17 @@ argparse and subparser names if mistyped by the user. (Contributed by Savannah Ostrowski in :gh:`124456`.) - .. _whatsnew314-color-argparse: - -* Introduced the optional *color* parameter to - :class:`argparse.ArgumentParser`, enabling color for help text. - This can be controlled by :ref:`environment variables - `. Color has also been enabled for help in the - :ref:`stdlib CLIs ` which use :mod:`!argparse`. +* Enable color for help text, which can be disabled with the optional *color* + parameter to :class:`argparse.ArgumentParser`. + This can also be controlled by :ref:`environment variables + `. (Contributed by Hugo van Kemenade in :gh:`130645`.) ast --- -* Add :func:`ast.compare` for comparing two ASTs. +* Add :func:`~ast.compare`, a function for comparing two ASTs. (Contributed by Batuhan Taskaya and Jeremy Hylton in :gh:`60191`.) * Add support for :func:`copy.replace` for AST nodes. @@ -989,30 +1083,52 @@ ast * Docstrings are now removed from an optimized AST in optimization level 2. (Contributed by Irit Katriel in :gh:`123958`.) -* The ``repr()`` output for AST nodes now includes more information. +* The :func:`repr` output for AST nodes now includes more information. (Contributed by Tomas Roun in :gh:`116022`.) -* :func:`ast.parse`, when called with an AST as input, now always verifies - that the root node type is appropriate. +* When called with an AST as input, the :func:`~ast.parse` function + now always verifies that the root node type is appropriate. (Contributed by Irit Katriel in :gh:`130139`.) -* Add new ``--feature-version``, ``--optimize``, ``--show-empty`` options to - command-line interface. +* Add new options to the command-line interface: + :option:`--feature-version `, + :option:`--optimize `, and + :option:`--show-empty `. (Contributed by Semyon Moroz in :gh:`133367`.) -bdb ---- - -* The :mod:`bdb` module now supports the :mod:`sys.monitoring` backend. - (Contributed by Tian Gao in :gh:`124533`.) +asyncio +------- +* The function and methods named :func:`!create_task` now take an arbitrary + list of keyword arguments. All keyword arguments are passed to the + :class:`~asyncio.Task` constructor or the custom task factory. + (See :meth:`~asyncio.loop.set_task_factory` for details.) + The ``name`` and ``context`` keyword arguments are no longer special; + the name should now be set using the ``name`` keyword argument of the factory, + and ``context`` may be ``None``. + + This affects the following function and methods: + :meth:`asyncio.create_task`, + :meth:`asyncio.loop.create_task`, + :meth:`asyncio.TaskGroup.create_task`. + + (Contributed by Thomas Grainger in :gh:`128307`.) + +* There are two new utility functions for + introspecting and printing a program's call graph: + :func:`~asyncio.capture_call_graph` and :func:`~asyncio.print_call_graph`. + See :ref:`Asyncio introspection capabilities + ` for more details. + (Contributed by Yury Selivanov, Pablo Galindo Salgado, and Łukasz Langa + in :gh:`91048`.) - .. _whatsnew314-color-calendar: calendar -------- +.. _whatsnew314-color-calendar: + * By default, today's date is highlighted in color in :mod:`calendar`'s :ref:`command-line ` text output. This can be controlled by :ref:`environment variables @@ -1023,19 +1139,27 @@ calendar concurrent.futures ------------------ -* Add :class:`~concurrent.futures.InterpreterPoolExecutor`, - which exposes "subinterpreters" (multiple Python interpreters in the - same process) to Python code. This is separate from the proposed API - in :pep:`734`. +.. _whatsnew314-concurrent-futures-interp-pool: + +* Add a new executor class, :class:`~concurrent.futures.InterpreterPoolExecutor`, + which exposes multiple Python interpreters in the same process + ('subinterpreters') to Python code. + This uses a pool of independent Python interpreters to execute calls + asynchronously. + + This is separate from the new :mod:`~concurrent.interpreters` module + introduced by :ref:`PEP 734 `. (Contributed by Eric Snow in :gh:`124548`.) .. _whatsnew314-concurrent-futures-start-method: -* The default :class:`~concurrent.futures.ProcessPoolExecutor` - :ref:`start method ` changed - from :ref:`fork ` to :ref:`forkserver - ` on platforms other than macOS and - Windows where it was already :ref:`spawn `. +* On Unix platforms other than macOS, :ref:`'forkserver' + ` is now the default :ref:`start + method ` for + :class:`~concurrent.futures.ProcessPoolExecutor` + (replacing :ref:`'fork' `). + This change does not affect Windows or macOS, where :ref:`'spawn' + ` remains the default start method. If the threading incompatible *fork* method is required, you must explicitly request it by supplying a multiprocessing context *mp_context* to @@ -1048,23 +1172,36 @@ concurrent.futures (Contributed by Gregory P. Smith in :gh:`84559`.) -* Add :meth:`concurrent.futures.ProcessPoolExecutor.terminate_workers` and - :meth:`concurrent.futures.ProcessPoolExecutor.kill_workers` as - ways to terminate or kill all living worker processes in the given pool. +* Add two new methods to :class:`~concurrent.futures.ProcessPoolExecutor`, + :meth:`~concurrent.futures.ProcessPoolExecutor.terminate_workers` + and :meth:`~concurrent.futures.ProcessPoolExecutor.kill_workers`, + as ways to terminate or kill all living worker processes in the given pool. (Contributed by Charles Machalow in :gh:`130849`.) -* Add the optional ``buffersize`` parameter to - :meth:`concurrent.futures.Executor.map` to limit the number of submitted +* Add the optional *buffersize* parameter to :meth:`Executor.map + ` to limit the number of submitted tasks whose results have not yet been yielded. If the buffer is full, iteration over the *iterables* pauses until a result is yielded from the buffer. (Contributed by Enzo Bonnal and Josh Rosenberg in :gh:`74028`.) +configparser +------------ + +* :mod:`!configparser` will no longer write config files it cannot read, + to improve security. + Attempting to :meth:`~configparser.ConfigParser.write` keys containing + delimiters or beginning with the section header pattern will raise an + :class:`~configparser.InvalidWriteError`. + (Contributed by Jacob Lincoln in :gh:`129270`.) + + contextvars ----------- -* Support context manager protocol by :class:`contextvars.Token`. +* Support the :term:`context manager` protocol + for :class:`~contextvars.Token` objects. (Contributed by Andrew Svetlov in :gh:`129889`.) @@ -1072,8 +1209,8 @@ ctypes ------ * The layout of :ref:`bit fields ` - in :class:`~ctypes.Structure` and :class:`~ctypes.Union` - now matches platform defaults (GCC/Clang or MSVC) more closely. + in :class:`~ctypes.Structure` and :class:`~ctypes.Union` objects + is now a closer match to platform defaults (GCC/Clang or MSVC). In particular, fields no longer overlap. (Contributed by Matthias Görgens in :gh:`97702`.) @@ -1092,7 +1229,7 @@ ctypes * On Windows, the :func:`~ctypes.CopyComPointer` function is now public. (Contributed by Jun Komoda in :gh:`127275`.) -* :func:`ctypes.memoryview_at` now exists to create a +* Add :func:`~ctypes.memoryview_at`, a function to create a :class:`memoryview` object that refers to the supplied pointer and length. This works like :func:`ctypes.string_at` except it avoids a buffer copy, and is typically useful when implementing pure Python @@ -1100,7 +1237,7 @@ ctypes (Contributed by Rian Hunter in :gh:`112018`.) * Complex types, :class:`~ctypes.c_float_complex`, - :class:`~ctypes.c_double_complex` and :class:`~ctypes.c_longdouble_complex`, + :class:`~ctypes.c_double_complex`, and :class:`~ctypes.c_longdouble_complex`, are now available if both the compiler and the ``libffi`` library support complex C types. (Contributed by Sergey B Kirpichev in :gh:`61103`.) @@ -1110,47 +1247,57 @@ ctypes (Contributed by Brian Ward in :gh:`119349`.) * Move :func:`ctypes.POINTER` types cache from a global internal cache - (``_pointer_type_cache``) to the :attr:`ctypes._CData.__pointer_type__` - attribute of the corresponding :mod:`ctypes` types. + (``_pointer_type_cache``) to the :attr:`_CData.__pointer_type__ + ` attribute of the corresponding + :mod:`!ctypes` types. This will stop the cache from growing without limits in some situations. (Contributed by Sergey Miryanov in :gh:`100926`.) -* The :class:`ctypes.py_object` type now supports subscription, +* The :class:`~ctypes.py_object` type now supports subscription, making it a :term:`generic type`. (Contributed by Brian Schubert in :gh:`132168`.) +* :mod:`!ctypes` now supports :term:`free-threading builds `. + (Contributed by Kumar Aditya and Peter Bierma in :gh:`127945`.) + + curses ------ * Add the :func:`~curses.assume_default_colors` function, a refinement of the :func:`~curses.use_default_colors` function which - allows to change the color pair ``0``. + allows changing the color pair ``0``. (Contributed by Serhiy Storchaka in :gh:`133139`.) + datetime -------- -* Add :meth:`datetime.time.strptime` and :meth:`datetime.date.strptime`. +* Add the :meth:`~datetime.date.strptime` method to the + :class:`datetime.date` and :class:`datetime.time` classes. (Contributed by Wannes Boeykens in :gh:`41431`.) + decimal ------- -* Add alternative :class:`~decimal.Decimal` constructor - :meth:`Decimal.from_number() `. +* Add :meth:`.Decimal.from_number` as an alternative constructor for + :class:`~decimal.Decimal`. (Contributed by Serhiy Storchaka in :gh:`121798`.) -* Expose :func:`decimal.IEEEContext` to support creation of contexts +* Expose :func:`~decimal.IEEEContext` to support creation of contexts corresponding to the IEEE 754 (2008) decimal interchange formats. (Contributed by Sergey B Kirpichev in :gh:`53032`.) + difflib ------- * Comparison pages with highlighted changes generated by the - :class:`difflib.HtmlDiff` class now support dark mode. + :class:`~difflib.HtmlDiff` class now support 'dark mode'. (Contributed by Jiahao Li in :gh:`129939`.) + dis --- @@ -1175,7 +1322,7 @@ dis errno ----- -* Add :data:`errno.EHWPOISON` error code. +* Add the :data:`~errno.EHWPOISON` error code constant. (Contributed by James Roy in :gh:`126585`.) @@ -1183,39 +1330,43 @@ faulthandler ------------ * Add support for printing the C stack trace on systems that - :ref:`support it ` via :func:`faulthandler.dump_c_stack` - or via the *c_stack* argument in :func:`faulthandler.enable`. + :ref:`support it ` via the new + :func:`~faulthandler.dump_c_stack` function or via the *c_stack* argument + in :func:`faulthandler.enable`. (Contributed by Peter Bierma in :gh:`127604`.) fnmatch ------- -* Added :func:`fnmatch.filterfalse` for excluding names matching a pattern. +* Add :func:`~fnmatch.filterfalse`, a function to reject names + matching a given pattern. (Contributed by Bénédikt Tran in :gh:`74598`.) fractions --------- -* Add support for converting any objects that have the - :meth:`!as_integer_ratio` method to a :class:`~fractions.Fraction`. +* A :class:`~fractions.Fraction` object may now be constructed from any + object with the :meth:`!as_integer_ratio` method. (Contributed by Serhiy Storchaka in :gh:`82017`.) -* Add alternative :class:`~fractions.Fraction` constructor - :meth:`Fraction.from_number() `. +* Add :meth:`.Fraction.from_number` as an alternative constructor for + :class:`~fractions.Fraction`. (Contributed by Serhiy Storchaka in :gh:`121797`.) functools --------- -* Add support to :func:`functools.partial` and - :func:`functools.partialmethod` for :data:`functools.Placeholder` sentinels - to reserve a place for positional arguments. +* Add the :data:`~functools.Placeholder` sentinel. + This may be used with the :func:`~functools.partial` + or :func:`~functools.partialmethod` functions to reserve a place + for positional arguments in the returned :ref:`partial object + `. (Contributed by Dominykas Grigonis in :gh:`119127`.) -* Allow the *initial* parameter of :func:`functools.reduce` to be passed +* Allow the *initial* parameter of :func:`~functools.reduce` to be passed as a keyword argument. (Contributed by Sayandip Dutta in :gh:`125916`.) @@ -1233,16 +1384,17 @@ getopt getpass ------- -* Support keyboard feedback by :func:`getpass.getpass` via the keyword-only - optional argument ``echo_char``. Placeholder characters are rendered whenever - a character is entered, and removed when a character is deleted. +* Support keyboard feedback in the :func:`~getpass.getpass` function via + the keyword-only optional argument *echo_char*. + Placeholder characters are rendered whenever a character is entered, + and removed when a character is deleted. (Contributed by Semyon Moroz in :gh:`77065`.) graphlib -------- -* Allow :meth:`graphlib.TopologicalSorter.prepare` to be called more than once +* Allow :meth:`.TopologicalSorter.prepare` to be called more than once as long as sorting has not started. (Contributed by Daniel Pope in :gh:`130914`.) @@ -1250,13 +1402,14 @@ graphlib heapq ----- -* Add functions for working with max-heaps: +* The :mod:`!heapq` module has improved support for working with max-heaps, + via the following new functions: - * :func:`heapq.heapify_max`, - * :func:`heapq.heappush_max`, - * :func:`heapq.heappop_max`, - * :func:`heapq.heapreplace_max` - * :func:`heapq.heappushpop_max` + * :func:`~heapq.heapify_max` + * :func:`~heapq.heappush_max` + * :func:`~heapq.heappop_max` + * :func:`~heapq.heapreplace_max` + * :func:`~heapq.heappushpop_max` hmac @@ -1264,6 +1417,8 @@ hmac * Add a built-in implementation for HMAC (:rfc:`2104`) using formally verified code from the `HACL* `__ project. + This implementation is used as a fallback when the OpenSSL implementation + of HMAC is not available. (Contributed by Bénédikt Tran in :gh:`99108`.) @@ -1279,9 +1434,12 @@ http the command-line interface (``python -m http.server``) through the following options: - * ``--tls-cert ``: Path to the TLS certificate file. - * ``--tls-key ``: Optional path to the private key file. - * ``--tls-password-file ``: Optional path to the password file for the private key. + * :option:`--tls-cert \ `: + Path to the TLS certificate file. + * :option:`--tls-key \ `: + Optional path to the private key file. + * :option:`--tls-password-file \ `: + Optional path to the password file for the private key. (Contributed by Semyon Moroz in :gh:`85162`.) @@ -1289,7 +1447,7 @@ http imaplib ------- -* Add :meth:`IMAP4.idle() `, implementing the IMAP4 +* Add :meth:`.IMAP4.idle`, implementing the IMAP4 ``IDLE`` command as defined in :rfc:`2177`. (Contributed by Forest in :gh:`55454`.) @@ -1297,15 +1455,16 @@ imaplib inspect ------- -* :func:`inspect.signature` takes a new argument *annotation_format* to control +* :func:`~inspect.signature` takes a new argument *annotation_format* to control the :class:`annotationlib.Format` used for representing annotations. (Contributed by Jelle Zijlstra in :gh:`101552`.) -* :meth:`inspect.Signature.format` takes a new argument *unquote_annotations*. - If true, string :term:`annotations ` are displayed without surrounding quotes. +* :meth:`.Signature.format` takes a new argument *unquote_annotations*. + If true, string :term:`annotations ` are displayed without + surrounding quotes. (Contributed by Jelle Zijlstra in :gh:`101552`.) -* Add function :func:`inspect.ispackage` to determine whether an object is a +* Add function :func:`~inspect.ispackage` to determine whether an object is a :term:`package` or not. (Contributed by Zhikang Yan in :gh:`125634`.) @@ -1317,7 +1476,7 @@ io :exc:`BlockingIOError` if the operation cannot immediately return bytes. (Contributed by Giovanni Siragusa in :gh:`109523`.) -* Add protocols :class:`io.Reader` and :class:`io.Writer` as a simpler +* Add the :class:`~io.Reader` and :class:`~io.Writer` protocols as simpler alternatives to the pseudo-protocols :class:`typing.IO`, :class:`typing.TextIO`, and :class:`typing.BinaryIO`. (Contributed by Sebastian Rittau in :gh:`127648`.) @@ -1326,35 +1485,36 @@ io json ---- -* Add notes for JSON serialization errors that allow to identify the source - of the error. +* Add exception notes for JSON serialization errors that allow + identifying the source of the error. (Contributed by Serhiy Storchaka in :gh:`122163`.) -* Enable the :mod:`json` module to work as a script using the :option:`-m` - switch: :program:`python -m json`. +* Allow using the :mod:`json` module as a script using the :option:`-m` switch: + :program:`python -m json`. + This is now preferred to :program:`python -m json.tool`, + which is :term:`soft deprecated`. See the :ref:`JSON command-line interface ` documentation. (Contributed by Trey Hunner in :gh:`122873`.) - .. _whatsnew314-color-json: - * By default, the output of the :ref:`JSON command-line interface ` is highlighted in color. This can be controlled by :ref:`environment variables `. (Contributed by Tomas Roun in :gh:`131952`.) + linecache --------- -* :func:`linecache.getline` can retrieve source code for frozen modules. +* :func:`~linecache.getline` can now retrieve source code for frozen modules. (Contributed by Tian Gao in :gh:`131638`.) logging.handlers ---------------- -* :class:`logging.handlers.QueueListener` now implements the context - manager protocol, allowing it to be used in a :keyword:`with` statement. +* :class:`~logging.handlers.QueueListener` objects now support the + :term:`context manager` protocol. (Contributed by Charles Machalow in :gh:`132106`.) * :meth:`QueueListener.start ` now @@ -1366,20 +1526,19 @@ math ---- * Added more detailed error messages for domain errors in the module. - (Contributed by by Charlie Zhao and Sergey B Kirpichev in :gh:`101410`.) + (Contributed by Charlie Zhao and Sergey B Kirpichev in :gh:`101410`.) mimetypes --------- -* Document the command-line for :mod:`mimetypes`. - It now exits with ``1`` on failure instead of ``0`` - and ``2`` on incorrect command-line parameters instead of ``1``. - Also, errors are printed to stderr instead of stdout and their text is made - tighter. +* Add a public :ref:`command-line ` for the module, + invoked via :program:`python -m mimetypes`. (Contributed by Oleg Iarygin and Hugo van Kemenade in :gh:`93096`.) -* Add MS and :rfc:`8081` MIME types for fonts: +* Add several new MIME types based on RFCs and common usage: + + .. rubric:: Microsoft and :rfc:`8081` MIME types for fonts * Embedded OpenType: ``application/vnd.ms-fontobject`` * OpenType Layout (OTF) ``font/otf`` @@ -1387,18 +1546,14 @@ mimetypes * WOFF 1.0 ``font/woff`` * WOFF 2.0 ``font/woff2`` - (Contributed by Sahil Prajapati and Hugo van Kemenade in :gh:`84852`.) - -* Add :rfc:`9559` MIME types for Matroska audiovisual data container - structures, containing: + .. rubric:: :rfc:`9559` MIME types for Matroska audiovisual + data container structures * audio with no video: ``audio/matroska`` (``.mka``) * video: ``video/matroska`` (``.mkv``) * stereoscopic video: ``video/matroska-3d`` (``.mk3d``) - (Contributed by Hugo van Kemenade in :gh:`89416`.) - -* Add MIME types for images with RFCs: + .. rubric:: Images with RFCs * :rfc:`1494`: CCITT Group 3 (``.g3``) * :rfc:`3362`: Real-time Facsimile, T.38 (``.t38``) @@ -1407,9 +1562,7 @@ mimetypes * :rfc:`4047`: Flexible Image Transport System (``.fits``) * :rfc:`7903`: Enhanced Metafile (``.emf``) and Windows Metafile (``.wmf``) - (Contributed by Hugo van Kemenade in :gh:`85957`.) - -* More MIME type changes: + .. rubric:: Other MIME type additions and changes * :rfc:`2361`: Change type for ``.avi`` to ``video/vnd.avi`` and for ``.wav`` to ``audio/vnd.wave`` @@ -1417,6 +1570,8 @@ mimetypes * :rfc:`5334`: Add Ogg media (``.oga``, ``.ogg`` and ``.ogx``) * :rfc:`6713`: Add gzip ``application/gzip`` (``.gz``) * :rfc:`9639`: Add FLAC ``audio/flac`` (``.flac``) + * :rfc:`9512` ``application/yaml`` MIME type for YAML files (``.yaml`` + and ``.yml``) * Add 7z ``application/x-7z-compressed`` (``.7z``) * Add Android Package ``application/vnd.android.package-archive`` (``.apk``) when not strict @@ -1439,11 +1594,9 @@ mimetypes * `W3C `__: Add EPUB ``application/epub+zip`` (``.epub``) - (Contributed by Hugo van Kemenade in :gh:`129965`.) - -* Add :rfc:`9512` ``application/yaml`` MIME type for YAML files (``.yaml`` - and ``.yml``). (Contributed by Sasha "Nelie" Chernykh and Hugo van Kemenade - in :gh:`132056`.) + (Contributed by Sahil Prajapati and Hugo van Kemenade in :gh:`84852`, + by Sasha "Nelie" Chernykh and Hugo van Kemenade in :gh:`132056`, + and by Hugo van Kemenade in :gh:`89416`, :gh:`85957`, and :gh:`129965`.) multiprocessing @@ -1451,14 +1604,16 @@ multiprocessing .. _whatsnew314-multiprocessing-start-method: -* The default :ref:`start method ` changed - from :ref:`fork ` to :ref:`forkserver - ` on platforms other than macOS and - Windows where it was already :ref:`spawn `. +* On Unix platforms other than macOS, :ref:`'forkserver' + ` is now the default :ref:`start + method ` + (replacing :ref:`'fork' `). + This change does not affect Windows or macOS, where :ref:`'spawn' + ` remains the default start method. If the threading incompatible *fork* method is required, you must explicitly - request it via a context from :func:`multiprocessing.get_context` (preferred) - or change the default via :func:`multiprocessing.set_start_method`. + request it via a context from :func:`~multiprocessing.get_context` (preferred) + or change the default via :func:`~multiprocessing.set_start_method`. See :ref:`forkserver restrictions ` for information and differences with the *fork* method and how this change @@ -1467,7 +1622,7 @@ multiprocessing (Contributed by Gregory P. Smith in :gh:`84559`.) -* :mod:`multiprocessing`'s ``"forkserver"`` start method now authenticates +* :mod:`multiprocessing`'s ``'forkserver'`` start method now authenticates its control socket to avoid solely relying on filesystem permissions to restrict what other processes could cause the forkserver to spawn workers and run code. @@ -1483,20 +1638,22 @@ multiprocessing (Contributed by Roy Hyunjin Han for :gh:`103134`.) * Add support for shared :class:`set` objects via - :meth:`SyncManager.set() `. - The :func:`set` in :func:`multiprocessing.Manager` method is now available. + :meth:`.SyncManager.set`. + The :func:`set` in :func:`~multiprocessing.Manager` method is now available. (Contributed by Mingyu Park in :gh:`129949`.) -* Add :func:`multiprocessing.Process.interrupt` which terminates the child +* Add the :meth:`~multiprocessing.Process.interrupt` + to :class:`multiprocessing.Process` objects, which terminates the child process by sending :py:const:`~signal.SIGINT`. This enables :keyword:`finally` clauses to print a stack trace for the terminated process. (Contributed by Artem Pulkin in :gh:`131913`.) + operator -------- -* Two new functions :func:`operator.is_none` and :func:`operator.is_not_none` - have been added, such that ``operator.is_none(obj)`` is equivalent +* Add :func:`~operator.is_none` and :func:`~operator.is_not_none` as a pair + of functions, such that ``operator.is_none(obj)`` is equivalent to ``obj is None`` and ``operator.is_not_none(obj)`` is equivalent to ``obj is not None``. (Contributed by Raymond Hettinger and Nico Mexis in :gh:`115808`.) @@ -1505,21 +1662,31 @@ operator os -- -* Add the :func:`os.reload_environ` function to update :data:`os.environ` and +* Add the :func:`~os.reload_environ` function to update :data:`os.environ` and :data:`os.environb` with changes to the environment made by :func:`os.putenv`, by :func:`os.unsetenv`, or made outside Python in the same process. (Contributed by Victor Stinner in :gh:`120057`.) * Add the :data:`~os.SCHED_DEADLINE` and :data:`~os.SCHED_NORMAL` constants - to the :mod:`os` module. + to the :mod:`!os` module. (Contributed by James Roy in :gh:`127688`.) -* Add the :func:`os.readinto` function to read into a +* Add the :func:`~os.readinto` function to read into a :ref:`buffer object ` from a file descriptor. (Contributed by Cody Maloney in :gh:`129205`.) +os.path +------- + +* The *strict* parameter to :func:`~os.path.realpath` accepts a new value, + :data:`~os.path.ALLOW_MISSING`. + If used, errors other than :exc:`FileNotFoundError` will be re-raised; + the resulting path can be missing but it will be free of symlinks. + (Contributed by Petr Viktorin for :cve:`2025-4517`.) + + pathlib ------- @@ -1533,8 +1700,8 @@ pathlib (Contributed by Barney Gale in :gh:`73991`.) -* Add :attr:`pathlib.Path.info` attribute, which stores an object - implementing the :class:`pathlib.types.PathInfo` protocol (also new). The +* Add the :attr:`~pathlib.Path.info` attribute, which stores an object + implementing the new :class:`pathlib.types.PathInfo` protocol. The object supports querying the file type and internally caching :func:`~os.stat` results. Path objects generated by :meth:`~pathlib.Path.iterdir` are initialized with file type information @@ -1545,7 +1712,26 @@ pathlib pdb --- -* Hardcoded breakpoints (:func:`breakpoint` and :func:`pdb.set_trace`) now +* The :mod:`pdb` module now supports remote attaching to a running Python process + using a new :option:`-p PID ` command-line option: + + .. code-block:: sh + + python -m pdb -p 1234 + + This will connect to the Python process with the given PID and allow you to + debug it interactively. Notice that due to how the Python interpreter works + attaching to a remote process that is blocked in a system call or waiting for + I/O will only work once the next bytecode instruction is executed or when the + process receives a signal. + + This feature uses :ref:`PEP 768 ` + and the new :func:`sys.remote_exec` function to attach to the remote process + and send the PDB commands to it. + + (Contributed by Matt Wozniski and Pablo Galindo in :gh:`131591`.) + +* Hardcoded breakpoints (:func:`breakpoint` and :func:`~pdb.set_trace`) now reuse the most recent :class:`~pdb.Pdb` instance that calls :meth:`~pdb.Pdb.set_trace`, instead of creating a new one each time. As a result, all the instance specific data like :pdbcmd:`display` and @@ -1578,21 +1764,14 @@ pdb * ``$_asynctask`` is added to access the current asyncio task if applicable. (Contributed by Tian Gao in :gh:`124367`.) -* :mod:`pdb` now supports two backends: :func:`sys.settrace` and - :mod:`sys.monitoring`. Using :mod:`pdb` CLI or :func:`breakpoint` will - always use the :mod:`sys.monitoring` backend. Explicitly instantiating - :class:`pdb.Pdb` and its derived classes will use the :func:`sys.settrace` - backend by default, which is configurable. - (Contributed by Tian Gao in :gh:`124533`.) - * :func:`pdb.set_trace_async` is added to support debugging asyncio coroutines. :keyword:`await` statements are supported with this function. (Contributed by Tian Gao in :gh:`132576`.) * Source code displayed in :mod:`pdb` will be syntax-highlighted. This feature - can be controlled using the same methods as PyREPL, in addition to the newly - added ``colorize`` argument of :class:`pdb.Pdb`. + can be controlled using the same methods as the default :term:`interactive` + shell, in addition to the newly added ``colorize`` argument of :class:`pdb.Pdb`. (Contributed by Tian Gao and Łukasz Langa in :gh:`133355`.) @@ -1602,15 +1781,16 @@ pickle * Set the default protocol version on the :mod:`pickle` module to 5. For more details, see :ref:`pickle protocols `. -* Add notes for pickle serialization errors that allow to identify the source - of the error. +* Add exception notes for pickle serialization errors that allow + identifying the source of the error. (Contributed by Serhiy Storchaka in :gh:`122213`.) platform -------- -* Add :func:`platform.invalidate_caches` to invalidate the cached results. +* Add :func:`~platform.invalidate_caches`, a function to invalidate + cached results in the :mod:`!platform` module. (Contributed by Bénédikt Tran in :gh:`122549`.) @@ -1622,6 +1802,19 @@ pydoc (Contributed by Jelle Zijlstra in :gh:`101552`.) +re +-- + +* Support ``\z`` as a synonym for ``\Z`` in :mod:`regular expressions `. + It is interpreted unambiguously in many other regular expression engines, + unlike ``\Z``, which has subtly different behavior. + (Contributed by Serhiy Storchaka in :gh:`133306`.) + +* ``\B`` in :mod:`regular expression ` now matches the empty input string, + meaning that it is now always the opposite of ``\b``. + (Contributed by Serhiy Storchaka in :gh:`124130`.) + + socket ------ @@ -1648,11 +1841,12 @@ socket * Add many new constants. (Contributed by Serhiy Storchaka in :gh:`132734`.) + ssl --- -* Indicate through :data:`ssl.HAS_PHA` whether the :mod:`ssl` module supports - TLSv1.3 post-handshake client authentication (PHA). +* Indicate through the :data:`~ssl.HAS_PHA` Boolean whether the :mod:`!ssl` + module supports TLSv1.3 post-handshake client authentication (PHA). (Contributed by Will Childs-Klein in :gh:`128036`.) @@ -1668,7 +1862,7 @@ struct symtable -------- -* Expose the following :class:`symtable.Symbol` methods: +* Expose the following :class:`~symtable.Symbol` methods: * :meth:`~symtable.Symbol.is_comp_cell` * :meth:`~symtable.Symbol.is_comp_iter` @@ -1683,31 +1877,69 @@ sys * The previously undocumented special function :func:`sys.getobjects`, which only exists in specialized builds of Python, may now return objects from other interpreters than the one it's called in. + (Contributed by Eric Snow in :gh:`125286`.) * Add :func:`sys._is_immortal` for determining if an object is :term:`immortal`. (Contributed by Peter Bierma in :gh:`128509`.) -* On FreeBSD, :data:`sys.platform` doesn't contain the major version anymore. +* On FreeBSD, :data:`sys.platform` no longer contains the major version number. It is always ``'freebsd'``, instead of ``'freebsd13'`` or ``'freebsd14'``. + (Contributed by Michael Osipov in :gh:`129393`.) * Raise :exc:`DeprecationWarning` for :func:`sys._clear_type_cache`. This function was deprecated in Python 3.13 but it didn't raise a runtime warning. +* Add :func:`sys.remote_exec` to implement the new external debugger interface. + See :ref:`PEP 768 ` for details. + (Contributed by Pablo Galindo Salgado, Matt Wozniski, and Ivona Stojanovic + in :gh:`131591`.) + +* Add the :data:`sys._jit` namespace, containing utilities for introspecting + just-in-time compilation. + (Contributed by Brandt Bucher in :gh:`133231`.) + sys.monitoring -------------- -* Two new events are added: :monitoring-event:`BRANCH_LEFT` and - :monitoring-event:`BRANCH_RIGHT`. The ``BRANCH`` event is deprecated. +* Add two new monitoring events, :monitoring-event:`BRANCH_LEFT` and + :monitoring-event:`BRANCH_RIGHT`. + These replace and deprecate the :monitoring-event:`!BRANCH` event. + (Contributed by Mark Shannon in :gh:`122548`.) sysconfig --------- -* Add ``ABIFLAGS`` key to :func:`sysconfig.get_config_vars` on Windows. +* Add ``ABIFLAGS`` key to :func:`~sysconfig.get_config_vars` on Windows. (Contributed by Xuehai Pan in :gh:`131799`.) +tarfile +------- + +* :func:`~tarfile.data_filter` now normalizes symbolic link targets in order to + avoid path traversal attacks. + (Contributed by Petr Viktorin in :gh:`127987` and :cve:`2025-4138`.) + +* :func:`~tarfile.TarFile.extractall` now skips fixing up directory attributes + when a directory was removed or replaced by another kind of file. + (Contributed by Petr Viktorin in :gh:`127987` and :cve:`2024-12718`.) + +* :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` + now (re-)apply the extraction filter when substituting a link (hard or + symbolic) with a copy of another archive member, and when fixing up + directory attributes. + The former raises a new exception, :exc:`~tarfile.LinkFallbackError`. + (Contributed by Petr Viktorin for :cve:`2025-4330` and :cve:`2024-12718`.) + +* :func:`~tarfile.TarFile.extract` and :func:`~tarfile.TarFile.extractall` + no longer extract rejected members when + :func:`~tarfile.TarFile.errorlevel` is zero. + (Contributed by Matt Prodani and Petr Viktorin in :gh:`112887` + and :cve:`2025-4435`.) + + threading --------- @@ -1720,17 +1952,18 @@ tkinter ------- * Make :mod:`tkinter` widget methods :meth:`!after` and :meth:`!after_idle` - accept arguments passed by keyword. + accept keyword arguments. (Contributed by Zhikang Yan in :gh:`126899`.) -* Add ability to specify name for :class:`!tkinter.OptionMenu` and +* Add ability to specify a name for :class:`!tkinter.OptionMenu` and :class:`!tkinter.ttk.OptionMenu`. (Contributed by Zhikang Yan in :gh:`130482`.) + turtle ------ -* Add context managers for :func:`turtle.fill`, :func:`turtle.poly` +* Add context managers for :func:`turtle.fill`, :func:`turtle.poly`, and :func:`turtle.no_animation`. (Contributed by Marie Roald and Yngve Mardal Moe in :gh:`126350`.) @@ -1748,43 +1981,60 @@ typing .. _whatsnew314-typing-union: -* :class:`types.UnionType` and :class:`typing.Union` are now aliases for each other, - meaning that both old-style unions (created with ``Union[int, str]``) and new-style - unions (``int | str``) now create instances of the same runtime type. This unifies - the behavior between the two syntaxes, but leads to some differences in behavior that +* The :class:`types.UnionType` and :class:`typing.Union` types are now + aliases for each other, meaning that both old-style unions + (created with ``Union[int, str]``) and new-style unions (``int | str``) + now create instances of the same runtime type. This unifies the behavior + between the two syntaxes, but leads to some differences in behavior that may affect users who introspect types at runtime: - - Both syntaxes for creating a union now produce the same string representation in - ``repr()``. For example, ``repr(Union[int, str])`` - is now ``"int | str"`` instead of ``"typing.Union[int, str]"``. - - Unions created using the old syntax are no longer cached. Previously, running - ``Union[int, str]`` multiple times would return the same object - (``Union[int, str] is Union[int, str]`` would be ``True``), but now it will - return two different objects. Users should use ``==`` to compare unions for equality, not - ``is``. New-style unions have never been cached this way. - This change could increase memory usage for some programs that use a large number of - unions created by subscripting ``typing.Union``. However, several factors offset this cost: - unions used in annotations are no longer evaluated by default in Python 3.14 - because of :pep:`649`; an instance of :class:`types.UnionType` is - itself much smaller than the object returned by ``Union[]`` was on prior Python - versions; and removing the cache also saves some space. It is therefore - unlikely that this change will cause a significant increase in memory usage for most - users. + - Both syntaxes for creating a union now produce the same string + representation in :func:`repr`. + For example, ``repr(Union[int, str])`` is now ``"int | str"`` instead of + ``"typing.Union[int, str]"``. + + - Unions created using the old syntax are no longer cached. + Previously, running ``Union[int, str]`` multiple times would return + the same object (``Union[int, str] is Union[int, str]`` would be ``True``), + but now it will return two different objects. + Use ``==`` to compare unions for equality, not ``is``. + New-style unions have never been cached this way. + This change could increase memory usage for some programs that use + a large number of unions created by subscripting ``typing.Union``. + However, several factors offset this cost: + unions used in annotations are no longer evaluated by default in Python + 3.14 because of :pep:`649`; an instance of :class:`types.UnionType` is + itself much smaller than the object returned by ``Union[]`` was on prior + Python versions; and removing the cache also saves some space. + It is therefore unlikely that this change will cause a significant increase + in memory usage for most users. + - Previously, old-style unions were implemented using the private class - ``typing._UnionGenericAlias``. This class is no longer needed for the implementation, - but it has been retained for backward compatibility, with removal scheduled for Python - 3.17. Users should use documented introspection helpers like :func:`typing.get_origin` - and :func:`typing.get_args` instead of relying on private implementation details. - - It is now possible to use :class:`typing.Union` itself in :func:`isinstance` checks. - For example, ``isinstance(int | str, typing.Union)`` will return ``True``; previously - this raised :exc:`TypeError`. - - The ``__args__`` attribute of :class:`typing.Union` objects is no longer writable. - - It is no longer possible to set any attributes on :class:`typing.Union` objects. + ``typing._UnionGenericAlias``. + This class is no longer needed for the implementation, + but it has been retained for backward compatibility, + with removal scheduled for Python 3.17. + Users should use documented introspection helpers like + :func:`~typing.get_origin` and :func:`typing.get_args` instead of + relying on private implementation details. + + - It is now possible to use :class:`typing.Union` itself in + :func:`isinstance` checks. + For example, ``isinstance(int | str, typing.Union)`` will return ``True``; + previously this raised :exc:`TypeError`. + + - The :attr:`!__args__` attribute of :class:`typing.Union` objects is + no longer writable. + + - It is no longer possible to set any attributes on :class:`~typing.Union` + objects. This only ever worked for dunder attributes on previous versions, was never documented to work, and was subtly broken in many cases. (Contributed by Jelle Zijlstra in :gh:`105499`.) +* :class:`~typing.TypeAliasType` now supports star unpacking. + unicodedata ----------- @@ -1792,11 +2042,11 @@ unicodedata * The Unicode database has been updated to Unicode 16.0.0. -.. _whatsnew314-color-unittest: - unittest -------- +.. _whatsnew314-color-unittest: + * :mod:`unittest` output is now colored by default. This can be controlled by :ref:`environment variables `. @@ -1819,7 +2069,7 @@ unittest :meth:`~unittest.TestCase.assertNotStartsWith`, :meth:`~unittest.TestCase.assertEndsWith` and :meth:`~unittest.TestCase.assertNotEndsWith` check whether the Unicode - or byte string starts or ends with particular string(s). + or byte string starts or ends with particular strings. (Contributed by Serhiy Storchaka in :gh:`71339`.) @@ -1834,17 +2084,18 @@ urllib * Improve ergonomics and standards compliance when parsing and emitting ``file:`` URLs. - In :func:`urllib.request.url2pathname`: + In :func:`~urllib.request.url2pathname`: - Accept a complete URL when the new *require_scheme* argument is set to true. - Discard URL authority if it matches the local hostname. - Discard URL authority if it resolves to a local IP address when the new *resolve_host* argument is set to true. + - Discard URL query and fragment components. - Raise :exc:`~urllib.error.URLError` if a URL authority isn't local, except on Windows where we return a UNC path as before. - In :func:`urllib.request.pathname2url`: + In :func:`~urllib.request.pathname2url`: - Return a complete URL when the new *add_scheme* argument is set to true. - Include an empty URL authority when a path begins with a slash. For @@ -1860,16 +2111,17 @@ urllib uuid ---- -* Add support for UUID versions 6, 7, and 8 via :func:`uuid.uuid6`, - :func:`uuid.uuid7`, and :func:`uuid.uuid8` respectively, as specified +* Add support for UUID versions 6, 7, and 8 via :func:`~uuid.uuid6`, + :func:`~uuid.uuid7`, and :func:`~uuid.uuid8` respectively, as specified in :rfc:`9562`. (Contributed by Bénédikt Tran in :gh:`89083`.) -* :const:`uuid.NIL` and :const:`uuid.MAX` are now available to represent the +* :const:`~uuid.NIL` and :const:`~uuid.MAX` are now available to represent the Nil and Max UUID formats as defined by :rfc:`9562`. (Contributed by Nick Pope in :gh:`128427`.) -* Allow to generate multiple UUIDs at once via :option:`python -m uuid --count `. +* Allow generating multiple UUIDs simultaneously on the command-line via + :option:`python -m uuid --count `. (Contributed by Simon Legner in :gh:`131236`.) @@ -1884,233 +2136,197 @@ webbrowser supported browsers on macOS. -zipinfo +zipfile ------- -* Added :func:`ZipInfo._for_archive ` +* Added :meth:`ZipInfo._for_archive `, a method to resolve suitable defaults for a :class:`~zipfile.ZipInfo` object as used by :func:`ZipFile.writestr `. (Contributed by Bénédikt Tran in :gh:`123424`.) -* :meth:`zipfile.ZipFile.writestr` now respect ``SOURCE_DATE_EPOCH`` that - distributions can set centrally and have build tools consume this in order - to produce reproducible output. +* :meth:`.ZipFile.writestr` now respects the :envvar:`SOURCE_DATE_EPOCH` + environment variable in order to better support reproducible builds. (Contributed by Jiahao Li in :gh:`91279`.) .. Add improved modules above alphabetically, not here at the end. + Optimizations ============= * The import time for several standard library modules has been improved, - including :mod:`ast`, :mod:`asyncio`, :mod:`base64`, :mod:`cmd`, :mod:`csv`, - :mod:`gettext`, :mod:`importlib.util`, :mod:`locale`, :mod:`mimetypes`, - :mod:`optparse`, :mod:`pickle`, :mod:`pprint`, :mod:`pstats`, :mod:`socket`, - :mod:`subprocess`, :mod:`threading`, :mod:`tomllib`, and :mod:`zipfile`. + including :mod:`annotationlib`, :mod:`ast`, :mod:`asyncio`, :mod:`base64`, + :mod:`cmd`, :mod:`csv`, :mod:`gettext`, :mod:`importlib.util`, :mod:`locale`, + :mod:`mimetypes`, :mod:`optparse`, :mod:`pickle`, :mod:`pprint`, + :mod:`pstats`, :mod:`shlex`, :mod:`socket`, :mod:`string`, :mod:`subprocess`, + :mod:`threading`, :mod:`tomllib`, :mod:`types`, and :mod:`zipfile`. (Contributed by Adam Turner, Bénédikt Tran, Chris Markiewicz, Eli Schwartz, Hugo van Kemenade, Jelle Zijlstra, and others in :gh:`118761`.) +* The interpreter now avoids some reference count modifications internally + when it's safe to do so. + This can lead to different values being returned from :func:`sys.getrefcount` + and :c:func:`Py_REFCNT` compared to previous versions of Python. + See :ref:`below ` for details. + asyncio ------- -* :mod:`asyncio` now uses double linked list implementation for native tasks - which speeds up execution by 10% on standard pyperformance benchmarks and - reduces memory usage. +* Standard benchmark results have improved by 10-20% following the + implementation of a new per-thread doubly linked list + for :class:`native tasks `, + also reducing memory usage. + This enables external introspection tools such as + :ref:`python -m asyncio pstree ` + to introspect the call graph of asyncio tasks running in all threads. (Contributed by Kumar Aditya in :gh:`107803`.) -* :mod:`asyncio` has new utility functions for introspecting and printing - the program's call graph: :func:`asyncio.capture_call_graph` and - :func:`asyncio.print_call_graph`. - (Contributed by Yury Selivanov, Pablo Galindo Salgado, and Łukasz Langa - in :gh:`91048`.) +* The module now has first class support for + :term:`free-threading builds `. + This enables parallel execution of multiple event loops across + different threads, scaling linearly with the number of threads. + (Contributed by Kumar Aditya in :gh:`128002`.) + base64 ------ -* Improve the performance of :func:`base64.b16decode` by up to ten times, - and reduce the import time of :mod:`base64` by up to six times. - (Contributed by Bénédikt Tran, Chris Markiewicz, and Adam Turner in :gh:`118761`.) +* :func:`~base64.b16decode` is now up to six times faster. + (Contributed by Bénédikt Tran, Chris Markiewicz, and Adam Turner + in :gh:`118761`.) -io +bdb --- -* :mod:`io` which provides the built-in :func:`open` makes less system calls - when opening regular files as well as reading whole files. Reading a small - operating system cached file in full is up to 15% faster. - :func:`pathlib.Path.read_bytes` has the most optimizations for reading a - file's bytes in full. (Contributed by Cody Maloney and Victor Stinner in - :gh:`120754` and :gh:`90102`.) +* The basic debugger now has a :mod:`sys.monitoring`-based backend, + which can be selected via the passing ``'monitoring'`` + to the :class:`~bdb.Bdb` class's new *backend* parameter. + (Contributed by Tian Gao in :gh:`124533`.) -uuid ----- -* Improve generation of :class:`~uuid.UUID` objects via their dedicated - functions: +difflib +------- - * :func:`~uuid.uuid3` and :func:`~uuid.uuid5` are both roughly 40% faster - for 16-byte names and 20% faster for 1024-byte names. Performance for - longer names remains unchanged. - * :func:`~uuid.uuid4` and :func:`~uuid.uuid8` are 30% and 40% faster - respectively. +* The :func:`~difflib.IS_LINE_JUNK` function is now up to twice as fast. + (Contributed by Adam Turner and Semyon Moroz in :gh:`130167`.) - (Contributed by Bénédikt Tran in :gh:`128150`.) +gc +-- -zlib ----- +* **From Python 3.14.5 onwards:** -* On Windows, ``zlib-ng`` is now used as the implementation of the - :mod:`zlib` module. This should produce compatible and comparable - results with better performance, though it is worth noting that - ``zlib.Z_BEST_SPEED`` (1) may result in significantly less - compression than the previous implementation (while also significantly - reducing the time taken to compress). - (Contributed by Steve Dower in :gh:`91349`.) + Python 3.14.0-3.14.4 shipped with a new incremental garbage collector. + However, due to a number of `reports + `__ + of significant memory pressure in production environments, + it has been reverted back to the generational GC from 3.13. + This is the GC now used in Python 3.14.5 and later. +* **Previously in Python 3.14.0-3.14.4:** -Deprecated -========== + The new :ref:`incremental garbage collector ` + means that maximum pause times are reduced + by an order of magnitude or more for larger heaps. -* :mod:`argparse`: + Because of this optimization, the meaning of the results of + :meth:`~gc.get_threshold` and :meth:`~gc.set_threshold` have changed, + along with :meth:`~gc.get_count` and :meth:`~gc.get_stats`. - * Passing the undocumented keyword argument *prefix_chars* to - :meth:`~argparse.ArgumentParser.add_argument_group` is now - deprecated. - (Contributed by Savannah Ostrowski in :gh:`125563`.) - * Deprecated the :class:`argparse.FileType` type converter. - Anything with resource management should be done downstream after the - arguments are parsed. - (Contributed by Serhiy Storchaka in :gh:`58032`.) + - For backwards compatibility, :meth:`~gc.get_threshold` continues to return + a three-item tuple. + The first value is the threshold for young collections, as before; + the second value determines the rate at which the old collection is scanned + (the default is 10, and higher values mean that the old collection + is scanned more slowly). + The third value is now meaningless and is always zero. -* :mod:`asyncio`: + - :meth:`~gc.set_threshold` now ignores any items after the second. - * :func:`!asyncio.iscoroutinefunction` is deprecated - and will be removed in Python 3.16; - use :func:`inspect.iscoroutinefunction` instead. - (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) + - :meth:`~gc.get_count` and :meth:`~gc.get_stats` continue to return + the same format of results. + The only difference is that instead of the results referring to + the young, aging and old generations, + the results refer to the young generation + and the aging and collecting spaces of the old generation. - * :mod:`asyncio` policy system is deprecated and will be removed in Python 3.16. - In particular, the following classes and functions are deprecated: + In summary, code that attempted to manipulate the behavior of the cycle GC + may not work exactly as intended, but it is very unlikely to be harmful. + All other code will work just fine. - * :class:`asyncio.AbstractEventLoopPolicy` - * :class:`asyncio.DefaultEventLoopPolicy` - * :class:`asyncio.WindowsSelectorEventLoopPolicy` - * :class:`asyncio.WindowsProactorEventLoopPolicy` - * :func:`asyncio.get_event_loop_policy` - * :func:`asyncio.set_event_loop_policy` - * :func:`asyncio.set_event_loop` + (Contributed by Mark Shannon in :gh:`108362`.) - Users should use :func:`asyncio.run` or :class:`asyncio.Runner` with - *loop_factory* to use the desired event loop implementation. - For example, to use :class:`asyncio.SelectorEventLoop` on Windows:: +io +--- - import asyncio +* Opening and reading files now executes fewer system calls. + Reading a small operating system cached file in full is up to 15% faster. + (Contributed by Cody Maloney and Victor Stinner + in :gh:`120754` and :gh:`90102`.) - async def main(): - ... - asyncio.run(main(), loop_factory=asyncio.SelectorEventLoop) +pathlib +------- - (Contributed by Kumar Aditya in :gh:`127949`.) +* :func:`Path.read_bytes ` now uses unbuffered mode + to open files, which is between 9% and 17% faster to read in full. + (Contributed by Cody Maloney in :gh:`120754`.) -* :mod:`builtins`: - Passing a complex number as the *real* or *imag* argument in the - :func:`complex` constructor is now deprecated; it should only be passed - as a single positional argument. - (Contributed by Serhiy Storchaka in :gh:`109218`.) -* :mod:`codecs`: - :func:`codecs.open` is now deprecated. Use :func:`open` instead. - (Contributed by Inada Naoki in :gh:`133036`.) +pdb +--- -* :mod:`ctypes`: +* :mod:`pdb` now supports two backends, based on either + :func:`sys.settrace` or :mod:`sys.monitoring`. + Using the :ref:`pdb CLI ` or :func:`breakpoint` + will always use the :mod:`sys.monitoring` backend. + Explicitly instantiating :class:`pdb.Pdb` and its derived classes + will use the :func:`sys.settrace` backend by default, which is configurable. + (Contributed by Tian Gao in :gh:`124533`.) - * On non-Windows platforms, setting :attr:`.Structure._pack_` to use a - MSVC-compatible default memory layout is deprecated in favor of setting - :attr:`.Structure._layout_` to ``'ms'``. - (Contributed by Petr Viktorin in :gh:`131747`.) - * Calling :func:`ctypes.POINTER` on a string is deprecated. - Use :ref:`ctypes-incomplete-types` for self-referential structures. - Also, the internal ``ctypes._pointer_type_cache`` is deprecated. - See :func:`ctypes.POINTER` for updated implementation details. - (Contributed by Sergey Myrianov in :gh:`100926`.) +textwrap +-------- -* :mod:`functools`: - Calling the Python implementation of :func:`functools.reduce` with *function* - or *sequence* as keyword arguments is now deprecated. - (Contributed by Kirill Podoprigora in :gh:`121676`.) +* Optimize the :func:`~textwrap.dedent` function, improving performance by + an average of 2.4x, with larger improvements for bigger inputs, + and fix a bug with incomplete normalization of blank lines with whitespace + characters other than space and tab. -* :mod:`logging`: - Support for custom logging handlers with the *strm* argument is deprecated - and scheduled for removal in Python 3.16. Define handlers with the *stream* - argument instead. (Contributed by Mariusz Felisiak in :gh:`115032`.) -* :mod:`mimetypes`: - Valid extensions start with a '.' or are empty for - :meth:`mimetypes.MimeTypes.add_type`. - Undotted extensions are deprecated and will - raise a :exc:`ValueError` in Python 3.16. - (Contributed by Hugo van Kemenade in :gh:`75223`.) - -* :mod:`!nturl2path`: This module is now deprecated. Call - :func:`urllib.request.url2pathname` and :func:`~urllib.request.pathname2url` - instead. - (Contributed by Barney Gale in :gh:`125866`.) - -* :mod:`os`: - :term:`Soft deprecate ` :func:`os.popen` and - :func:`os.spawn* ` functions. They should no longer be used to - write new code. The :mod:`subprocess` module is recommended instead. - (Contributed by Victor Stinner in :gh:`120743`.) - -* :mod:`pathlib`: - :meth:`!pathlib.PurePath.as_uri` is deprecated and will be removed in Python - 3.19. Use :meth:`pathlib.Path.as_uri` instead. - (Contributed by Barney Gale in :gh:`123599`.) - -* :mod:`pdb`: - The undocumented ``pdb.Pdb.curframe_locals`` attribute is now a deprecated - read-only property. The low overhead dynamic frame locals access added in - Python 3.13 by PEP 667 means the frame locals cache reference previously - stored in this attribute is no longer needed. Derived debuggers should access - ``pdb.Pdb.curframe.f_locals`` directly in Python 3.13 and later versions. - (Contributed by Tian Gao in :gh:`124369` and :gh:`125951`.) - -* :mod:`symtable`: - Deprecate :meth:`symtable.Class.get_methods` due to the lack of interest. - (Contributed by Bénédikt Tran in :gh:`119698`.) +uuid +---- -* :mod:`tkinter`: - The :class:`!tkinter.Variable` methods :meth:`!trace_variable`, - :meth:`!trace_vdelete` and :meth:`!trace_vinfo` are now deprecated. - Use :meth:`!trace_add`, :meth:`!trace_remove` and :meth:`!trace_info` - instead. - (Contributed by Serhiy Storchaka in :gh:`120220`.) +* :func:`~uuid.uuid3` and :func:`~uuid.uuid5` are now both roughly 40% faster + for 16-byte names and 20% faster for 1024-byte names. + Performance for longer names remains unchanged. + (Contributed by Bénédikt Tran in :gh:`128150`.) -* :mod:`urllib.parse`: - Accepting objects with false values (like ``0`` and ``[]``) except empty - strings, byte-like objects and ``None`` in :mod:`urllib.parse` functions - :func:`~urllib.parse.parse_qsl` and :func:`~urllib.parse.parse_qs` is now - deprecated. - (Contributed by Serhiy Storchaka in :gh:`116897`.) +* :func:`~uuid.uuid4` is now c. 30% faster. + (Contributed by Bénédikt Tran in :gh:`128150`.) -.. Add deprecations above alphabetically, not here at the end. -.. include:: ../deprecations/pending-removal-in-3.15.rst +zlib +---- -.. include:: ../deprecations/pending-removal-in-3.16.rst +* On Windows, `zlib-ng `__ + is now used as the implementation of the :mod:`zlib` module + in the default binaries. + There are no known incompatibilities between ``zlib-ng`` + and the previously-used ``zlib`` implementation. + This should result in better performance at all compression levels. -.. include:: ../deprecations/pending-removal-in-3.17.rst + It is worth noting that ``zlib.Z_BEST_SPEED`` (``1``) may result in + significantly less compression than the previous implementation, + whilst also significantly reducing the time taken to compress. -.. include:: ../deprecations/pending-removal-in-3.19.rst + (Contributed by Steve Dower in :gh:`91349`.) -.. include:: ../deprecations/pending-removal-in-future.rst Removed ======= @@ -2119,86 +2335,102 @@ argparse -------- * Remove the *type*, *choices*, and *metavar* parameters - of :class:`!argparse.BooleanOptionalAction`. - They were deprecated since 3.12. - -* Calling :meth:`~argparse.ArgumentParser.add_argument_group` on an argument - group, and calling :meth:`~argparse.ArgumentParser.add_argument_group` or - :meth:`~argparse.ArgumentParser.add_mutually_exclusive_group` on a mutually - exclusive group now raise exceptions. This nesting was never supported, - often failed to work correctly, and was unintentionally exposed through - inheritance. This functionality has been deprecated since Python 3.11. + of :class:`!BooleanOptionalAction`. + These have been deprecated since Python 3.12. + (Contributed by Nikita Sobolev in :gh:`118805`.) + +* Calling :meth:`~argparse.ArgumentParser.add_argument_group` + on an argument group now raises a :exc:`ValueError`. + Similarly, :meth:`~argparse.ArgumentParser.add_argument_group` + or :meth:`~argparse.ArgumentParser.add_mutually_exclusive_group` + on a mutually exclusive group now both raise :exc:`ValueError`\ s. + This 'nesting' was never supported, often failed to work correctly, + and was unintentionally exposed through inheritance. + This functionality has been deprecated since Python 3.11. (Contributed by Savannah Ostrowski in :gh:`127186`.) + ast --- -* Remove the following classes. They were all deprecated since Python 3.8, - and have emitted deprecation warnings since Python 3.12: +* Remove the following classes, which have been deprecated aliases of + :class:`~ast.Constant` since Python 3.8 and have emitted + deprecation warnings since Python 3.12: + + * :class:`!Bytes` + * :class:`!Ellipsis` + * :class:`!NameConstant` + * :class:`!Num` + * :class:`!Str` - * :class:`!ast.Bytes` - * :class:`!ast.Ellipsis` - * :class:`!ast.NameConstant` - * :class:`!ast.Num` - * :class:`!ast.Str` + As a consequence of these removals, user-defined ``visit_Num``, ``visit_Str``, + ``visit_Bytes``, ``visit_NameConstant`` and ``visit_Ellipsis`` methods + on custom :class:`~ast.NodeVisitor` subclasses will no longer be called + when the :class:`!NodeVisitor` subclass is visiting an AST. + Define a ``visit_Constant`` method instead. - Use :class:`ast.Constant` instead. As a consequence of these removals, - user-defined ``visit_Num``, ``visit_Str``, ``visit_Bytes``, - ``visit_NameConstant`` and ``visit_Ellipsis`` methods on custom - :class:`ast.NodeVisitor` subclasses will no longer be called when the - :class:`!NodeVisitor` subclass is visiting an AST. Define a ``visit_Constant`` - method instead. + (Contributed by Alex Waygood in :gh:`119562`.) - Also, remove the following deprecated properties on :class:`ast.Constant`, +* Remove the following deprecated properties on :class:`ast.Constant`, which were present for compatibility with the now-removed AST classes: - * :attr:`!ast.Constant.n` - * :attr:`!ast.Constant.s` + * :attr:`!Constant.n` + * :attr:`!Constant.s` - Use :attr:`!ast.Constant.value` instead. + Use :attr:`!Constant.value` instead. (Contributed by Alex Waygood in :gh:`119562`.) + asyncio ------- -* Remove the following classes and functions. They were all deprecated and - emitted deprecation warnings since Python 3.12: +* Remove the following classes, methods, and functions, + which have been deprecated since Python 3.12: - * :func:`!asyncio.get_child_watcher` - * :func:`!asyncio.set_child_watcher` - * :meth:`!asyncio.AbstractEventLoopPolicy.get_child_watcher` - * :meth:`!asyncio.AbstractEventLoopPolicy.set_child_watcher` - * :class:`!asyncio.AbstractChildWatcher` - * :class:`!asyncio.FastChildWatcher` - * :class:`!asyncio.MultiLoopChildWatcher` - * :class:`!asyncio.PidfdChildWatcher` - * :class:`!asyncio.SafeChildWatcher` - * :class:`!asyncio.ThreadedChildWatcher` + * :class:`!AbstractChildWatcher` + * :class:`!FastChildWatcher` + * :class:`!MultiLoopChildWatcher` + * :class:`!PidfdChildWatcher` + * :class:`!SafeChildWatcher` + * :class:`!ThreadedChildWatcher` + * :meth:`!AbstractEventLoopPolicy.get_child_watcher` + * :meth:`!AbstractEventLoopPolicy.set_child_watcher` + * :func:`!get_child_watcher` + * :func:`!set_child_watcher` (Contributed by Kumar Aditya in :gh:`120804`.) -* Removed implicit creation of event loop by :func:`asyncio.get_event_loop`. - It now raises a :exc:`RuntimeError` if there is no current event loop. +* :func:`asyncio.get_event_loop` now raises a :exc:`RuntimeError` + if there is no current event loop, + and no longer implicitly creates an event loop. + (Contributed by Kumar Aditya in :gh:`126353`.) + .. TODO: move these patterns to the asyncio docs? + quite long for What's New + There's a few patterns that use :func:`asyncio.get_event_loop`, most of them can be replaced with :func:`asyncio.run`. If you're running an async function, simply use :func:`asyncio.run`. - Before:: + Before: - async def main(): - ... + .. code:: python + async def main(): + ... - loop = asyncio.get_event_loop() - try: - loop.run_until_complete(main()) - finally: - loop.close() - After:: + loop = asyncio.get_event_loop() + try: + loop.run_until_complete(main()) + finally: + loop.close() + + After: + + .. code:: python async def main(): ... @@ -2209,410 +2441,996 @@ asyncio and then run forever, use :func:`asyncio.run` and an :class:`asyncio.Event`. - Before:: - - def start_server(loop): - ... - - loop = asyncio.get_event_loop() - try: - start_server(loop) - loop.run_forever() - finally: - loop.close() - - After:: + Before: - def start_server(loop): - ... + .. code:: python - async def main(): - start_server(asyncio.get_running_loop()) - await asyncio.Event().wait() + def start_server(loop): ... - asyncio.run(main()) + loop = asyncio.get_event_loop() + try: + start_server(loop) + loop.run_forever() + finally: + loop.close() - If you need to run something in an event loop, then run some blocking - code around it, use :class:`asyncio.Runner`. + After: - Before:: + .. code:: python - async def operation_one(): - ... + def start_server(loop): ... - def blocking_code(): - ... + async def main(): + start_server(asyncio.get_running_loop()) + await asyncio.Event().wait() - async def operation_two(): - ... + asyncio.run(main()) - loop = asyncio.get_event_loop() - try: - loop.run_until_complete(operation_one()) - blocking_code() - loop.run_until_complete(operation_two()) - finally: - loop.close() + If you need to run something in an event loop, then run some blocking + code around it, use :class:`asyncio.Runner`. - After:: + Before: - async def operation_one(): - ... + .. code:: python - def blocking_code(): - ... + async def operation_one(): ... + def blocking_code(): ... + async def operation_two(): ... - async def operation_two(): - ... + loop = asyncio.get_event_loop() + try: + loop.run_until_complete(operation_one()) + blocking_code() + loop.run_until_complete(operation_two()) + finally: + loop.close() - with asyncio.Runner() as runner: - runner.run(operation_one()) - blocking_code() - runner.run(operation_two()) + After: + .. code:: python + async def operation_one(): ... + def blocking_code(): ... + async def operation_two(): ... -collections.abc ---------------- + with asyncio.Runner() as runner: + runner.run(operation_one()) + blocking_code() + runner.run(operation_two()) -* Remove :class:`!collections.abc.ByteString`. It had previously raised a - :exc:`DeprecationWarning` since Python 3.12. email ----- -* Remove the *isdst* parameter from :func:`email.utils.localtime`. +* Remove :func:`email.utils.localtime`'s *isdst* parameter, + which was deprecated in and has been ignored since Python 3.12. (Contributed by Hugo van Kemenade in :gh:`118798`.) -importlib ---------- - -* Remove deprecated :mod:`importlib.abc` classes: - * :class:`!importlib.abc.ResourceReader` - * :class:`!importlib.abc.Traversable` - * :class:`!importlib.abc.TraversableResources` +importlib.abc +------------- - Use :mod:`importlib.resources.abc` classes instead: +* Remove deprecated :mod:`importlib.abc` classes: - * :class:`importlib.resources.abc.Traversable` - * :class:`importlib.resources.abc.TraversableResources` + * :class:`!ResourceReader` + (use :class:`~importlib.resources.abc.TraversableResources`) + * :class:`!Traversable` + (use :class:`~importlib.resources.abc.Traversable`) + * :class:`!TraversableResources` + (use :class:`~importlib.resources.abc.TraversableResources`) (Contributed by Jason R. Coombs and Hugo van Kemenade in :gh:`93963`.) + itertools --------- -* Remove :mod:`itertools` support for copy, deepcopy, and pickle operations. - These had previously raised a :exc:`DeprecationWarning` since Python 3.12. +* Remove support for copy, deepcopy, and pickle operations + from :mod:`itertools` iterators. + These have emitted a :exc:`DeprecationWarning` since Python 3.12. (Contributed by Raymond Hettinger in :gh:`101588`.) + pathlib ------- -* Remove support for passing additional keyword arguments to - :class:`pathlib.Path`. In previous versions, any such arguments are ignored. +* Remove support for passing additional keyword arguments + to :class:`~pathlib.Path`. + In previous versions, any such arguments are ignored. + (Contributed by Barney Gale in :gh:`74033`.) + * Remove support for passing additional positional arguments to - :meth:`pathlib.PurePath.relative_to` and - :meth:`~pathlib.PurePath.is_relative_to`. In previous versions, any such - arguments are joined onto *other*. + :meth:`.PurePath.relative_to` and :meth:`~pathlib.PurePath.is_relative_to`. + In previous versions, any such arguments are joined onto *other*. + (Contributed by Barney Gale in :gh:`78707`.) + pkgutil ------- -* Remove deprecated :func:`!pkgutil.get_loader` and :func:`!pkgutil.find_loader`. - These had previously raised a :exc:`DeprecationWarning` since Python 3.12. +* Remove the :func:`!get_loader` and :func:`!find_loader` functions, + which have been deprecated since Python 3.12. (Contributed by Bénédikt Tran in :gh:`97850`.) + pty --- -* Remove deprecated :func:`!pty.master_open` and :func:`!pty.slave_open`. - They had previously raised a :exc:`DeprecationWarning` since Python 3.12. +* Remove the :func:`!master_open` and :func:`!slave_open` functions, + which have been deprecated since Python 3.12. Use :func:`pty.openpty` instead. (Contributed by Nikita Sobolev in :gh:`118824`.) + sqlite3 ------- -* Remove :data:`!version` and :data:`!version_info` from :mod:`sqlite3`. +* Remove :data:`!version` and :data:`!version_info` from + the :mod:`sqlite3` module; + use :data:`~sqlite3.sqlite_version` and :data:`~sqlite3.sqlite_version_info` + for the actual version number of the runtime SQLite library. (Contributed by Hugo van Kemenade in :gh:`118924`.) -* Disallow using a sequence of parameters with named placeholders. - This had previously raised a :exc:`DeprecationWarning` since Python 3.12; - it will now raise a :exc:`sqlite3.ProgrammingError`. +* Using a sequence of parameters with named placeholders now + raises a :exc:`~sqlite3.ProgrammingError`, + having been deprecated since Python 3.12. (Contributed by Erlend E. Aasland in :gh:`118928` and :gh:`101693`.) -typing ------- - -* Remove :class:`!typing.ByteString`. It had previously raised a - :exc:`DeprecationWarning` since Python 3.12. - -* :class:`typing.TypeAliasType` now supports star unpacking. urllib ------ -* Remove deprecated :class:`!Quoter` class from :mod:`urllib.parse`. - It had previously raised a :exc:`DeprecationWarning` since Python 3.11. +* Remove the :class:`!Quoter` class from :mod:`urllib.parse`, + which has been deprecated since Python 3.11. (Contributed by Nikita Sobolev in :gh:`118827`.) -* Remove deprecated :class:`!URLopener` and :class:`!FancyURLopener` classes - from :mod:`urllib.request`. They had previously raised a - :exc:`DeprecationWarning` since Python 3.3. - - ``myopener.open()`` can be replaced with :func:`~urllib.request.urlopen`, - and ``myopener.retrieve()`` can be replaced with - :func:`~urllib.request.urlretrieve`. Customizations to the opener - classes can be replaced by passing customized handlers to - :func:`~urllib.request.build_opener`. + +* Remove the :class:`!URLopener` and :class:`!FancyURLopener` classes + from :mod:`urllib.request`, + which have been deprecated since Python 3.3. + + ``myopener.open()`` can be replaced with :func:`~urllib.request.urlopen`. + ``myopener.retrieve()`` can be replaced with + :func:`~urllib.request.urlretrieve`. + Customisations to the opener classes can be replaced by passing + customized handlers to :func:`~urllib.request.build_opener`. (Contributed by Barney Gale in :gh:`84850`.) -Others ------- -* Using :data:`NotImplemented` in a boolean context will now raise a :exc:`TypeError`. - It had previously raised a :exc:`DeprecationWarning` since Python 3.9. (Contributed - by Jelle Zijlstra in :gh:`118767`.) +Deprecated +========== -* The :func:`int` built-in no longer delegates to - :meth:`~object.__trunc__`. Classes that want to support conversion to - integer must implement either :meth:`~object.__int__` or - :meth:`~object.__index__`. (Contributed by Mark Dickinson in :gh:`119743`.) +New deprecations +---------------- +* Passing a complex number as the *real* or *imag* argument in the + :func:`complex` constructor is now deprecated; + complex numbers should only be passed as a single positional argument. + (Contributed by Serhiy Storchaka in :gh:`109218`.) -CPython bytecode changes -======================== +* :mod:`argparse`: -* Replaced the opcode ``BINARY_SUBSCR`` by :opcode:`BINARY_OP` with oparg ``NB_SUBSCR``. - (Contributed by Irit Katriel in :gh:`100239`.) + * Passing the undocumented keyword argument *prefix_chars* to the + :meth:`~argparse.ArgumentParser.add_argument_group` method is now deprecated. + (Contributed by Savannah Ostrowski in :gh:`125563`.) -Porting to Python 3.14 -====================== + * Deprecated the :class:`argparse.FileType` type converter. + Anything relating to resource management should be handled + downstream, after the arguments have been parsed. + (Contributed by Serhiy Storchaka in :gh:`58032`.) -This section lists previously described changes and other bugfixes -that may require changes to your code. +* :mod:`asyncio`: -Changes in the Python API -------------------------- + * The :func:`!asyncio.iscoroutinefunction` is now deprecated + and will be removed in Python 3.16; + use :func:`inspect.iscoroutinefunction` instead. + (Contributed by Jiahao Li and Kumar Aditya in :gh:`122875`.) -* :class:`functools.partial` is now a method descriptor. - Wrap it in :func:`staticmethod` if you want to preserve the old behavior. - (Contributed by Serhiy Storchaka and Dominykas Grigonis in :gh:`121027`.) + * The :mod:`asyncio` policy system is deprecated + and will be removed in Python 3.16. + In particular, the following classes and functions are deprecated: -* The :func:`locale.nl_langinfo` function now sets temporarily the ``LC_CTYPE`` - locale in some cases. - This temporary change affects other threads. - (Contributed by Serhiy Storchaka in :gh:`69998`.) + * :class:`asyncio.AbstractEventLoopPolicy` + * :class:`asyncio.DefaultEventLoopPolicy` + * :class:`asyncio.WindowsSelectorEventLoopPolicy` + * :class:`asyncio.WindowsProactorEventLoopPolicy` + * :func:`asyncio.get_event_loop_policy` + * :func:`asyncio.set_event_loop_policy` -* :class:`types.UnionType` is now an alias for :class:`typing.Union`, - causing changes in some behaviors. - See :ref:`above ` for more details. - (Contributed by Jelle Zijlstra in :gh:`105499`.) + Users should use :func:`asyncio.run` or :class:`asyncio.Runner` with + the *loop_factory* argument to use the desired event loop implementation. + For example, to use :class:`asyncio.SelectorEventLoop` on Windows: -Build changes -============= + .. code-block:: python -* GNU Autoconf 2.72 is now required to generate :file:`configure`. - (Contributed by Erlend Aasland in :gh:`115765`.) + import asyncio -* ``#pragma``-based linking with ``python3*.lib`` can now be switched off - with :c:expr:`Py_NO_LINK_LIB`. (Contributed by Jean-Christophe - Fillion-Robin in :gh:`82909`.) + async def main(): + ... -.. _whatsnew314-pep761: + asyncio.run(main(), loop_factory=asyncio.SelectorEventLoop) -PEP 761: Discontinuation of PGP signatures ------------------------------------------- + (Contributed by Kumar Aditya in :gh:`127949`.) -PGP signatures will not be available for CPython 3.14 and onwards. -Users verifying artifacts must use `Sigstore verification materials`_ for -verifying CPython artifacts. This change in release process is specified -in :pep:`761`. +* :mod:`codecs`: + The :func:`codecs.open` function is now deprecated, + and will be removed in a future version of Python. + Use :func:`open` instead. + (Contributed by Inada Naoki in :gh:`133036`.) -.. _Sigstore verification materials: https://www.python.org/downloads/metadata/sigstore/ +* :mod:`ctypes`: + * On non-Windows platforms, setting :attr:`.Structure._pack_` to use a + MSVC-compatible default memory layout is now deprecated in favor of setting + :attr:`.Structure._layout_` to ``'ms'``, and will be removed in Python 3.19. + (Contributed by Petr Viktorin in :gh:`131747`.) -C API changes -============= + * Calling :func:`ctypes.POINTER` on a string is now deprecated. + Use :ref:`incomplete types ` + for self-referential structures. + Also, the internal ``ctypes._pointer_type_cache`` is deprecated. + See :func:`ctypes.POINTER` for updated implementation details. + (Contributed by Sergey Myrianov in :gh:`100926`.) -New features ------------- +* :mod:`functools`: + Calling the Python implementation of :func:`functools.reduce` with *function* + or *sequence* as keyword arguments is now deprecated; + the parameters will be made positional-only in Python 3.16. + (Contributed by Kirill Podoprigora in :gh:`121676`.) -* Add :c:func:`PyLong_GetSign` function to get the sign of :class:`int` objects. - (Contributed by Sergey B Kirpichev in :gh:`116560`.) +* :mod:`logging`: + Support for custom logging handlers with the *strm* argument + is now deprecated and scheduled for removal in Python 3.16. + Define handlers with the *stream* argument instead. + (Contributed by Mariusz Felisiak in :gh:`115032`.) -* Add a new :c:type:`PyUnicodeWriter` API to create a Python :class:`str` - object: +* :mod:`mimetypes`: + Valid extensions are either empty or must start with '.' for + :meth:`mimetypes.MimeTypes.add_type`. + Undotted extensions are deprecated and will + raise a :exc:`ValueError` in Python 3.16. + (Contributed by Hugo van Kemenade in :gh:`75223`.) - * :c:func:`PyUnicodeWriter_Create` - * :c:func:`PyUnicodeWriter_DecodeUTF8Stateful` - * :c:func:`PyUnicodeWriter_Discard` - * :c:func:`PyUnicodeWriter_Finish` - * :c:func:`PyUnicodeWriter_Format` - * :c:func:`PyUnicodeWriter_WriteChar` - * :c:func:`PyUnicodeWriter_WriteRepr` - * :c:func:`PyUnicodeWriter_WriteStr` - * :c:func:`PyUnicodeWriter_WriteSubstring` - * :c:func:`PyUnicodeWriter_WriteUCS4` - * :c:func:`PyUnicodeWriter_WriteUTF8` - * :c:func:`PyUnicodeWriter_WriteWideChar` +* :mod:`!nturl2path`: + This module is now deprecated. Call :func:`urllib.request.url2pathname` + and :func:`~urllib.request.pathname2url` instead. + (Contributed by Barney Gale in :gh:`125866`.) - (Contributed by Victor Stinner in :gh:`119182`.) +* :mod:`os`: + The :func:`os.popen` and :func:`os.spawn* ` functions + are now :term:`soft deprecated`. + They should no longer be used to write new code. + The :mod:`subprocess` module is recommended instead. + (Contributed by Victor Stinner in :gh:`120743`.) -* Add :c:func:`PyIter_NextItem` to replace :c:func:`PyIter_Next`, - which has an ambiguous return value. - (Contributed by Irit Katriel and Erlend Aasland in :gh:`105201`.) +* :mod:`pathlib`: + :meth:`!pathlib.PurePath.as_uri` is now deprecated + and scheduled for removal in Python 3.19. + Use :meth:`pathlib.Path.as_uri` instead. + (Contributed by Barney Gale in :gh:`123599`.) -* Add :c:func:`PyLong_IsPositive`, :c:func:`PyLong_IsNegative` - and :c:func:`PyLong_IsZero` for checking if :c:type:`PyLongObject` - is positive, negative, or zero, respectively. - (Contributed by James Roy and Sergey B Kirpichev in :gh:`126061`.) +* :mod:`pdb`: + The undocumented ``pdb.Pdb.curframe_locals`` attribute is now a deprecated + read-only property, which will be removed in a future version of Python. + The low overhead dynamic frame locals access added in Python 3.13 by :pep:`667` + means the frame locals cache reference previously stored in this attribute + is no longer needed. Derived debuggers should access + ``pdb.Pdb.curframe.f_locals`` directly in Python 3.13 and later versions. + (Contributed by Tian Gao in :gh:`124369` and :gh:`125951`.) -* Add new functions to convert C ```` numbers from/to Python - :class:`int`: +* :mod:`symtable`: + Deprecate :meth:`symtable.Class.get_methods` due to the lack of interest, + scheduled for removal in Python 3.16. + (Contributed by Bénédikt Tran in :gh:`119698`.) - * :c:func:`PyLong_AsInt32` - * :c:func:`PyLong_AsInt64` - * :c:func:`PyLong_AsUInt32` - * :c:func:`PyLong_AsUInt64` - * :c:func:`PyLong_FromInt32` - * :c:func:`PyLong_FromInt64` - * :c:func:`PyLong_FromUInt32` - * :c:func:`PyLong_FromUInt64` +* :mod:`tkinter`: + The :class:`!tkinter.Variable` methods :meth:`!trace_variable`, + :meth:`!trace_vdelete` and :meth:`!trace_vinfo` are now deprecated. + Use :meth:`!trace_add`, :meth:`!trace_remove` and :meth:`!trace_info` instead. + (Contributed by Serhiy Storchaka in :gh:`120220`.) - (Contributed by Victor Stinner in :gh:`120389`.) +* :mod:`urllib.parse`: + Accepting objects with false values (like ``0`` and ``[]``) except empty + strings, bytes-like objects and ``None`` in :func:`~urllib.parse.parse_qsl` + and :func:`~urllib.parse.parse_qs` is now deprecated. + (Contributed by Serhiy Storchaka in :gh:`116897`.) -* Add :c:func:`PyBytes_Join(sep, iterable) ` function, - similar to ``sep.join(iterable)`` in Python. - (Contributed by Victor Stinner in :gh:`121645`.) +.. Add deprecations above alphabetically, not here at the end. -* Add :c:func:`Py_HashBuffer` to compute and return the hash value of a buffer. - (Contributed by Antoine Pitrou and Victor Stinner in :gh:`122854`.) +.. include:: ../deprecations/pending-removal-in-3.15.rst -* Add functions to get and set the current runtime Python configuration - (:pep:`741`): +.. include:: ../deprecations/pending-removal-in-3.16.rst - * :c:func:`PyConfig_Get` - * :c:func:`PyConfig_GetInt` - * :c:func:`PyConfig_Set` - * :c:func:`PyConfig_Names` +.. include:: ../deprecations/pending-removal-in-3.17.rst - (Contributed by Victor Stinner in :gh:`107954`.) +.. include:: ../deprecations/pending-removal-in-3.18.rst -* Add functions to configure the Python initialization (:pep:`741`): +.. include:: ../deprecations/pending-removal-in-3.19.rst - * :c:func:`Py_InitializeFromInitConfig` - * :c:func:`PyInitConfig_AddModule` - * :c:func:`PyInitConfig_Create` - * :c:func:`PyInitConfig_Free` - * :c:func:`PyInitConfig_FreeStrList` - * :c:func:`PyInitConfig_GetError` - * :c:func:`PyInitConfig_GetExitCode` - * :c:func:`PyInitConfig_GetInt` - * :c:func:`PyInitConfig_GetStr` - * :c:func:`PyInitConfig_GetStrList` - * :c:func:`PyInitConfig_HasOption` - * :c:func:`PyInitConfig_SetInt` - * :c:func:`PyInitConfig_SetStr` - * :c:func:`PyInitConfig_SetStrList` +.. include:: ../deprecations/pending-removal-in-future.rst - (Contributed by Victor Stinner in :gh:`107954`.) -* Add a new import and export API for Python :class:`int` objects (:pep:`757`): +CPython bytecode changes +======================== - * :c:func:`PyLong_GetNativeLayout` - * :c:func:`PyLong_Export` - * :c:func:`PyLong_FreeExport` - * :c:func:`PyLongWriter_Create` - * :c:func:`PyLongWriter_Finish` - * :c:func:`PyLongWriter_Discard` +* Replaced the opcode :opcode:`!BINARY_SUBSCR` by the :opcode:`BINARY_OP` + opcode with the ``NB_SUBSCR`` oparg. + (Contributed by Irit Katriel in :gh:`100239`.) - (Contributed by Sergey B Kirpichev and Victor Stinner in :gh:`102471`.) +* Add the :opcode:`BUILD_INTERPOLATION` and :opcode:`BUILD_TEMPLATE` + opcodes to construct new :class:`~string.templatelib.Interpolation` + and :class:`~string.templatelib.Template` instances, respectively. + (Contributed by Lysandros Nikolaou and others in :gh:`132661`; + see also :ref:`PEP 750: Template strings `). -* Add :c:func:`PyType_GetBaseByToken` and :c:data:`Py_tp_token` slot for easier - superclass identification, which attempts to resolve the `type checking issue - `__ mentioned in :pep:`630` - (:gh:`124153`). +* Remove the :opcode:`!BUILD_CONST_KEY_MAP` opcode. + Use :opcode:`BUILD_MAP` instead. + (Contributed by Mark Shannon in :gh:`122160`.) -* Add :c:func:`PyUnicode_Equal` function to the limited C API: - test if two strings are equal. - (Contributed by Victor Stinner in :gh:`124502`.) +* Replace the :opcode:`!LOAD_ASSERTION_ERROR` opcode with + :opcode:`LOAD_COMMON_CONSTANT` and add support for loading + :exc:`NotImplementedError`. -* Add :c:func:`PyType_Freeze` function to make a type immutable. - (Contributed by Victor Stinner in :gh:`121654`.) +* Add the :opcode:`LOAD_FAST_BORROW` and :opcode:`LOAD_FAST_BORROW_LOAD_FAST_BORROW` + opcodes to reduce reference counting overhead when the interpreter can prove + that the reference in the frame outlives the reference loaded onto the stack. + (Contributed by Matt Page in :gh:`130704`.) -* Add :c:func:`PyUnstable_Object_EnableDeferredRefcount` for enabling - deferred reference counting, as outlined in :pep:`703`. +* Add the :opcode:`LOAD_SMALL_INT` opcode, which pushes a small integer + equal to the ``oparg`` to the stack. + The :opcode:`!RETURN_CONST` opcode is removed as it is no longer used. + (Contributed by Mark Shannon in :gh:`125837`.) -* Add :c:func:`PyMonitoring_FireBranchLeftEvent` and - :c:func:`PyMonitoring_FireBranchRightEvent` for generating - :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT` - events, respectively. +* Add the new :opcode:`LOAD_SPECIAL` instruction. + Generate code for :keyword:`with` and :keyword:`async with` statements + using the new instruction. + Removed the :opcode:`!BEFORE_WITH` and :opcode:`!BEFORE_ASYNC_WITH` instructions. + (Contributed by Mark Shannon in :gh:`120507`.) -* Add :c:func:`Py_fopen` function to open a file. Similar to the - :c:func:`!fopen` function, but the *path* parameter is a Python object and an - exception is set on error. Add also :c:func:`Py_fclose` function to close a - file. - (Contributed by Victor Stinner in :gh:`127350`.) +* Add the :opcode:`POP_ITER` opcode to support 'virtual' iterators. + (Contributed by Mark Shannon in :gh:`132554`.) -* Add support of nullable arguments in :c:func:`PyArg_ParseTuple` and - similar functions. - Adding ``?`` after any format unit makes ``None`` be accepted as a value. - (Contributed by Serhiy Storchaka in :gh:`112068`.) -* The ``k`` and ``K`` formats in :c:func:`PyArg_ParseTuple` and - similar functions now use :meth:`~object.__index__` if available, - like all other integer formats. - (Contributed by Serhiy Storchaka in :gh:`112068`.) +Pseudo-instructions +------------------- + +* Add the :opcode:`!ANNOTATIONS_PLACEHOLDER` pseudo instruction + to support partially executed module-level annotations with + :ref:`deferred evaluation of annotations `. + (Contributed by Jelle Zijlstra in :gh:`130907`.) + +* Add the :opcode:`!BINARY_OP_EXTEND` pseudo instruction, + which executes a pair of functions (guard and specialization functions) + accessed from the inline cache. + (Contributed by Irit Katriel in :gh:`100239`.) + +* Add three specializations for :opcode:`CALL_KW`; + :opcode:`!CALL_KW_PY` for calls to Python functions, + :opcode:`!CALL_KW_BOUND_METHOD` for calls to bound methods, and + :opcode:`!CALL_KW_NON_PY` for all other calls. + (Contributed by Mark Shannon in :gh:`118093`.) + +* Add the :opcode:`JUMP_IF_TRUE` and :opcode:`JUMP_IF_FALSE` pseudo instructions, + conditional jumps which do not impact the stack. + Replaced by the sequence ``COPY 1``, ``TO_BOOL``, ``POP_JUMP_IF_TRUE/FALSE``. + (Contributed by Irit Katriel in :gh:`124285`.) + +* Add the :opcode:`!LOAD_CONST_MORTAL` pseudo instruction. + (Contributed by Mark Shannon in :gh:`128685`.) + +* Add the :opcode:`LOAD_CONST_IMMORTAL` pseudo instruction, + which does the same as :opcode:`!LOAD_CONST`, but is more efficient + for immortal objects. + (Contributed by Mark Shannon in :gh:`125837`.) + +* Add the :opcode:`NOT_TAKEN` pseudo instruction, used by :mod:`sys.monitoring` + to record branch events (such as :monitoring-event:`BRANCH_LEFT`). + (Contributed by Mark Shannon in :gh:`122548`.) + + +C API changes +============= + +.. _whatsnew314-capi-config: + +Python configuration C API +-------------------------- + +Add a :ref:`PyInitConfig C API ` to configure the Python +initialization without relying on C structures and the ability to make +ABI-compatible changes in the future. + +Complete the :pep:`587` :ref:`PyConfig C API ` by adding +:c:func:`PyInitConfig_AddModule` which can be used to add a built-in extension +module; a feature previously referred to as the "inittab". + +Add :c:func:`PyConfig_Get` and :c:func:`PyConfig_Set` functions to get and set +the current runtime configuration. + +:pep:`587` 'Python Initialization Configuration' unified all the ways +to configure Python's initialization. This PEP also unifies the configuration +of Python's preinitialization and initialization in a single API. +Moreover, this PEP only provides a single choice to embed Python, +instead of having two 'Python' and 'Isolated' choices (PEP 587), +to further simplify the API. -* Add macros :c:func:`Py_PACK_VERSION` and :c:func:`Py_PACK_FULL_VERSION` for - bit-packing Python version numbers. +The lower level PEP 587 PyConfig API remains available for use cases +with an intentionally higher level of coupling to CPython implementation details +(such as emulating the full functionality of CPython's CLI, including its +configuration mechanisms). + +(Contributed by Victor Stinner in :gh:`107954`.) + +.. seealso:: :pep:`741` and :pep:`587` + + +New features in the C API +------------------------- + +* Add :c:func:`Py_PACK_VERSION` and :c:func:`Py_PACK_FULL_VERSION`, + two new macros for bit-packing Python version numbers. + This is useful for comparisons with :c:var:`Py_Version` + or :c:macro:`PY_VERSION_HEX`. (Contributed by Petr Viktorin in :gh:`128629`.) -* Add :c:func:`PyUnstable_IsImmortal` for determining whether an object is :term:`immortal`, - for debugging purposes. +* Add :c:func:`PyBytes_Join(sep, iterable) ` function, + similar to ``sep.join(iterable)`` in Python. + (Contributed by Victor Stinner in :gh:`121645`.) + +* Add functions to manipulate the configuration of the current + runtime Python interpreter + (:ref:`PEP 741: Python configuration C API `): + + * :c:func:`PyConfig_Get` + * :c:func:`PyConfig_GetInt` + * :c:func:`PyConfig_Set` + * :c:func:`PyConfig_Names` + + (Contributed by Victor Stinner in :gh:`107954`.) + +* Add functions to configure Python initialization + (:ref:`PEP 741: Python configuration C API `): + + * :c:func:`Py_InitializeFromInitConfig` + * :c:func:`PyInitConfig_AddModule` + * :c:func:`PyInitConfig_Create` + * :c:func:`PyInitConfig_Free` + * :c:func:`PyInitConfig_FreeStrList` + * :c:func:`PyInitConfig_GetError` + * :c:func:`PyInitConfig_GetExitCode` + * :c:func:`PyInitConfig_GetInt` + * :c:func:`PyInitConfig_GetStr` + * :c:func:`PyInitConfig_GetStrList` + * :c:func:`PyInitConfig_HasOption` + * :c:func:`PyInitConfig_SetInt` + * :c:func:`PyInitConfig_SetStr` + * :c:func:`PyInitConfig_SetStrList` + + (Contributed by Victor Stinner in :gh:`107954`.) + +* Add :c:func:`Py_fopen` function to open a file. + This works similarly to the standard C :c:func:`!fopen` function, + instead accepting a Python object for the *path* parameter + and setting an exception on error. + The corresponding new :c:func:`Py_fclose` function should be used + to close a file. + (Contributed by Victor Stinner in :gh:`127350`.) + +* Add :c:func:`Py_HashBuffer` to compute and return the hash value of a buffer. + (Contributed by Antoine Pitrou and Victor Stinner in :gh:`122854`.) * Add :c:func:`PyImport_ImportModuleAttr` and :c:func:`PyImport_ImportModuleAttrString` helper functions to import a module and get an attribute of the module. (Contributed by Victor Stinner in :gh:`128911`.) -* Add support for a new ``p`` format unit in :c:func:`Py_BuildValue` that allows to - take a C integer and produce a Python :class:`bool` object. (Contributed by - Pablo Galindo in :issue:`45325`.) +* Add :c:func:`PyIter_NextItem` to replace :c:func:`PyIter_Next`, + which has an ambiguous return value. + (Contributed by Irit Katriel and Erlend Aasland in :gh:`105201`.) + +* Add :c:func:`PyLong_GetSign` function to get the sign of :class:`int` objects. + (Contributed by Sergey B Kirpichev in :gh:`116560`.) + +* Add :c:func:`PyLong_IsPositive`, :c:func:`PyLong_IsNegative` + and :c:func:`PyLong_IsZero` for checking if :c:type:`PyLongObject` + is positive, negative, or zero, respectively. + (Contributed by James Roy and Sergey B Kirpichev in :gh:`126061`.) + +* Add new functions to convert C ```` numbers to/from + Python :class:`int` objects: + + * :c:func:`PyLong_AsInt32` + * :c:func:`PyLong_AsInt64` + * :c:func:`PyLong_AsUInt32` + * :c:func:`PyLong_AsUInt64` + * :c:func:`PyLong_FromInt32` + * :c:func:`PyLong_FromInt64` + * :c:func:`PyLong_FromUInt32` + * :c:func:`PyLong_FromUInt64` + + (Contributed by Victor Stinner in :gh:`120389`.) + +* Add a new import and export API for Python :class:`int` objects + (:pep:`757`): + + * :c:func:`PyLong_GetNativeLayout` + * :c:func:`PyLong_Export` + * :c:func:`PyLong_FreeExport` + * :c:func:`PyLongWriter_Create` + * :c:func:`PyLongWriter_Finish` + * :c:func:`PyLongWriter_Discard` + + (Contributed by Sergey B Kirpichev and Victor Stinner in :gh:`102471`.) + +* Add :c:func:`PyMonitoring_FireBranchLeftEvent` and + :c:func:`PyMonitoring_FireBranchRightEvent` for generating + :monitoring-event:`BRANCH_LEFT` and :monitoring-event:`BRANCH_RIGHT` + events, respectively. + (Contributed by Mark Shannon in :gh:`122548`.) + +* Add :c:func:`PyType_Freeze` function to make a type immutable. + (Contributed by Victor Stinner in :gh:`121654`.) + +* Add :c:func:`PyType_GetBaseByToken` and :c:data:`Py_tp_token` slot + for easier superclass identification, which attempts to resolve the + type checking issue mentioned in :pep:`PEP 630 <630#type-checking>`. + (Contributed in :gh:`124153`.) + +* Add a new :c:func:`PyUnicode_Equal` function to test if two + strings are equal. + The function is also added to the Limited C API. + (Contributed by Victor Stinner in :gh:`124502`.) -* Add :c:func:`PyUnstable_Object_IsUniqueReferencedTemporary` to determine if an object - is a unique temporary object on the interpreter's operand stack. This can - be used in some cases as a replacement for checking if :c:func:`Py_REFCNT` - is ``1`` for Python objects passed as arguments to C API functions. +* Add a new :c:type:`PyUnicodeWriter` API to create a Python :class:`str` + object, with the following functions: -* Add :c:func:`PyUnstable_Object_IsUniquelyReferenced` as a replacement for - ``Py_REFCNT(op) == 1`` on :term:`free threaded ` builds. + * :c:func:`PyUnicodeWriter_Create` + * :c:func:`PyUnicodeWriter_DecodeUTF8Stateful` + * :c:func:`PyUnicodeWriter_Discard` + * :c:func:`PyUnicodeWriter_Finish` + * :c:func:`PyUnicodeWriter_Format` + * :c:func:`PyUnicodeWriter_WriteASCII` + * :c:func:`PyUnicodeWriter_WriteChar` + * :c:func:`PyUnicodeWriter_WriteRepr` + * :c:func:`PyUnicodeWriter_WriteStr` + * :c:func:`PyUnicodeWriter_WriteSubstring` + * :c:func:`PyUnicodeWriter_WriteUCS4` + * :c:func:`PyUnicodeWriter_WriteUTF8` + * :c:func:`PyUnicodeWriter_WriteWideChar` + + (Contributed by Victor Stinner in :gh:`119182`.) + +* The ``k`` and ``K`` formats in :c:func:`PyArg_ParseTuple` and + similar functions now use :meth:`~object.__index__` if available, + like all other integer formats. + (Contributed by Serhiy Storchaka in :gh:`112068`.) + +* Add support for a new ``p`` format unit in :c:func:`Py_BuildValue` + that produces a Python :class:`bool` object from a C integer. + (Contributed by Pablo Galindo in :issue:`45325`.) + +* Add :c:func:`PyUnstable_IsImmortal` for determining if + an object is :term:`immortal`, for debugging purposes. + (Contributed by Peter Bierma in :gh:`128509`.) + +* Add :c:func:`PyUnstable_Object_EnableDeferredRefcount` for enabling + deferred reference counting, as outlined in :pep:`703`. + +* Add :c:func:`PyUnstable_Object_IsUniquelyReferenced` as + a replacement for ``Py_REFCNT(op) == 1`` on :term:`free threaded + ` builds. (Contributed by Peter Bierma in :gh:`133140`.) +* Add :c:func:`PyUnstable_Object_IsUniqueReferencedTemporary` to + determine if an object is a unique temporary object on the + interpreter's operand stack. + This can be used in some cases as a replacement for checking + if :c:func:`Py_REFCNT` is ``1`` for Python objects passed + as arguments to C API functions. + (Contributed by Sam Gross in :gh:`133164`.) + Limited C API changes --------------------- -* In the limited C API 3.14 and newer, :c:func:`Py_TYPE` and - :c:func:`Py_REFCNT` are now implemented as an opaque function call to hide - implementation details. +* In the limited C API version 3.14 and newer, :c:func:`Py_TYPE` and + :c:func:`Py_REFCNT` are now implemented as an opaque function call + to hide implementation details. (Contributed by Victor Stinner in :gh:`120600` and :gh:`124127`.) * Remove the :c:macro:`PySequence_Fast_GET_SIZE`, - :c:macro:`PySequence_Fast_GET_ITEM` and :c:macro:`PySequence_Fast_ITEMS` - macros from the limited C API, since these macros never worked in the limited - C API. Keep :c:func:`PySequence_Fast` in the limited C API. + :c:macro:`PySequence_Fast_GET_ITEM`, + and :c:macro:`PySequence_Fast_ITEMS` + macros from the limited C API, since they have always been broken + in the limited C API. (Contributed by Victor Stinner in :gh:`91417`.) +.. _whatsnew314-c-api-removed: + +Removed C APIs +-------------- + +* Creating :c:data:`immutable types ` with + mutable bases was deprecated in Python 3.12, + and now raises a :exc:`TypeError`. + (Contributed by Nikita Sobolev in :gh:`119775`.) + +* Remove ``PyDictObject.ma_version_tag`` member, which was deprecated + in Python 3.12. + Use the :c:func:`PyDict_AddWatcher` API instead. + (Contributed by Sam Gross in :gh:`124296`.) + +* Remove the private ``_Py_InitializeMain()`` function. + It was a :term:`provisional API` added to Python 3.8 by :pep:`587`. + (Contributed by Victor Stinner in :gh:`129033`.) + +* Remove the undocumented APIs :c:macro:`!Py_C_RECURSION_LIMIT` + and :c:member:`!PyThreadState.c_recursion_remaining`. + These were added in 3.13 and have been removed without deprecation. + Use :c:func:`Py_EnterRecursiveCall` to guard against runaway + recursion in C code. + (Removed by Petr Viktorin in :gh:`133079`, see also :gh:`130396`.) + + +.. _whatsnew314-c-api-deprecated: + +Deprecated C APIs +----------------- + +* The :c:macro:`!Py_HUGE_VAL` macro is now :term:`soft deprecated`. + Use :c:macro:`!Py_INFINITY` instead. + (Contributed by Sergey B Kirpichev in :gh:`120026`.) + +* The :c:macro:`!Py_IS_NAN`, :c:macro:`!Py_IS_INFINITY`, + and :c:macro:`!Py_IS_FINITE` macros are now :term:`soft deprecated`. + Use :c:macro:`!isnan`, :c:macro:`!isinf` and :c:macro:`!isfinite` + instead, available from :file:`math.h` since C99. + (Contributed by Sergey B Kirpichev in :gh:`119613`.) + +* Non-tuple sequences are now deprecated as argument for the ``(items)`` + format unit in :c:func:`PyArg_ParseTuple` and other :ref:`argument + parsing ` functions if *items* contains format units + which store a :ref:`borrowed buffer ` or a + :term:`borrowed reference`. + (Contributed by Serhiy Storchaka in :gh:`50333`.) + +* The ``_PyMonitoring_FireBranchEvent`` function is now deprecated + and should be replaced with calls to + :c:func:`PyMonitoring_FireBranchLeftEvent` and + :c:func:`PyMonitoring_FireBranchRightEvent`. + +* The previously undocumented function :c:func:`PySequence_In` is + now :term:`soft deprecated`. + Use :c:func:`PySequence_Contains` instead. + (Contributed by Yuki Kobayashi in :gh:`127896`.) + +.. Add C API deprecations above alphabetically, not here at the end. + +.. include:: ../deprecations/c-api-pending-removal-in-3.15.rst + +.. include:: ../deprecations/c-api-pending-removal-in-3.16.rst + +.. include:: ../deprecations/c-api-pending-removal-in-3.18.rst + +.. include:: ../deprecations/c-api-pending-removal-in-future.rst + + +.. _whatsnew314-build-changes: + +Build changes +============= + +* :pep:`776`: Emscripten is now an officially supported platform at + :pep:`tier 3 <11#tier-3>`. As a part of this effort, more than 25 bugs in + `Emscripten libc`__ were fixed. Emscripten now includes support + for :mod:`ctypes`, :mod:`termios`, and :mod:`fcntl`, as well as + experimental support for the new :ref:`default interactive shell + `. + (Contributed by R. Hood Chatham in :gh:`127146`, :gh:`127683`, and :gh:`136931`.) + + __ https://emscripten.org/docs/porting/emscripten-runtime-environment.html + +* Official Android binary releases are now provided on python.org__. + + __ https://www.python.org/downloads/android/ + +* GNU Autoconf 2.72 is now required to generate :file:`configure`. + (Contributed by Erlend Aasland in :gh:`115765`.) + +* ``wasm32-unknown-emscripten`` is now a :pep:`11` tier 3 platform. + (Contributed by R. Hood Chatham in :gh:`127146`, :gh:`127683`, and :gh:`136931`.) + +* ``#pragma``-based linking with ``python3*.lib`` can now be switched off + with :c:expr:`Py_NO_LINK_LIB`. + (Contributed by Jean-Christophe Fillion-Robin in :gh:`82909`.) + +* CPython now enables a set of recommended compiler options by default + for improved security. + Use the :option:`--disable-safety` :file:`configure` option to disable them, + or the :option:`--enable-slower-safety` option for a larger set + of compiler options, albeit with a performance cost. + +* The ``WITH_FREELISTS`` macro and ``--without-freelists`` :file:`configure` + option have been removed. + +* The new :file:`configure` option :option:`--with-tail-call-interp` + may be used to enable the experimental tail call interpreter. + See :ref:`whatsnew314-tail-call-interpreter` for further details. + +* To disable the new remote debugging support, use the + :option:`--without-remote-debug` :file:`configure` option. + This may be useful for security reasons. + +* iOS and macOS apps can now be configured to redirect ``stdout`` and + ``stderr`` content to the system log. + (Contributed by Russell Keith-Magee in :gh:`127592`.) + +* The iOS testbed is now able to stream test output while the test is running. + The testbed can also be used to run the test suite of projects other than + CPython itself. + (Contributed by Russell Keith-Magee in :gh:`127592`.) + + +.. _whatsnew314-build_details: + +:file:`build-details.json` +-------------------------- + +Installations of Python now contain a new file, :file:`build-details.json`. +This is a static JSON document containing build details for CPython, +to allow for introspection without needing to run code. +This is helpful for use-cases such as Python launchers, cross-compilation, +and so on. + +:file:`build-details.json` must be installed in the platform-independent +standard library directory. This corresponds to the :ref:`'stdlib' +` :mod:`sysconfig` installation path, +which can be found by running ``sysconfig.get_path('stdlib')``. + +.. seealso:: + :pep:`739` -- ``build-details.json`` 1.0 -- a static description file + for Python build details + + +.. _whatsnew314-no-more-pgp: + +Discontinuation of PGP signatures +--------------------------------- + +PGP (Pretty Good Privacy) signatures will not be provided +for releases of Python 3.14 or future versions. +To verify CPython artifacts, users must use `Sigstore verification materials +`__. +Releases have been signed using Sigstore_ since Python 3.11. + +This change in release process was specified in :pep:`761`. + +.. _Sigstore: https://www.sigstore.dev/ + + +.. _whatsnew314-free-threaded-now-supported: + +Free-threaded Python is officially supported +-------------------------------------------- + +The free-threaded build of Python is now supported and no longer experimental. +This is the start of `phase II `__ where +free-threaded Python is officially supported but still optional. + +The free-threading team are confident that the project is on the right path, +and appreciate the continued dedication from everyone working to make +free-threading ready for broader adoption across the Python community. + +With these recommendations and the acceptance of this PEP, the Python developer +community should broadly advertise that free-threading is a supported +Python build option now and into the future, and that it will not be removed +without a proper deprecation schedule. + +Any decision to transition to `phase III `__, +with free-threading as the default or sole build of Python is still undecided, +and dependent on many factors both within CPython itself and the community. +This decision is for the future. + +.. seealso:: + + :pep:`779` + + `PEP 779's acceptance `__ + + +.. _whatsnew314-jit-compiler: + +Binary releases for the experimental just-in-time compiler +---------------------------------------------------------- + +The official macOS and Windows release binaries now include an *experimental* +just-in-time (JIT) compiler. Although it is **not** recommended for production +use, it can be tested by setting :envvar:`PYTHON_JIT=1 ` as an +environment variable. Downstream source builds and redistributors can use the +:option:`--enable-experimental-jit=yes-off` configuration option for similar +behavior. + +The JIT is at an early stage and still in active development. As such, the +typical performance impact of enabling it can range from 10% slower to 20% +faster, depending on workload. To aid in testing and evaluation, a set of +introspection functions has been provided in the :data:`sys._jit` namespace. +:func:`sys._jit.is_available` can be used to determine if the current executable +supports JIT compilation, while :func:`sys._jit.is_enabled` can be used to tell +if JIT compilation has been enabled for the current process. + +Currently, the most significant missing functionality is that native debuggers +and profilers like ``gdb`` and ``perf`` are unable to unwind through JIT frames +(Python debuggers and profilers, like :mod:`pdb` or :mod:`profile`, continue to +work without modification). Free-threaded builds do not support JIT compilation. + +Please report any bugs or major performance regressions that you encounter! + +.. seealso:: :pep:`744` + + Porting to Python 3.14 ----------------------- +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + + +Changes in the Python API +------------------------- + +* On Unix platforms other than macOS, *forkserver* is now the default + :ref:`start method ` for :mod:`multiprocessing` + and :class:`~concurrent.futures.ProcessPoolExecutor`, instead of *fork*. + + See :ref:`(1) ` and + :ref:`(2) ` for details. + + If you encounter :exc:`NameError`\s or pickling errors coming out of + :mod:`multiprocessing` or :mod:`concurrent.futures`, see the + :ref:`forkserver restrictions `. + + This change does not affect Windows or macOS, where :ref:`'spawn' + ` remains the default start method. + +* :class:`functools.partial` is now a method descriptor. + Wrap it in :func:`staticmethod` if you want to preserve the old behavior. + (Contributed by Serhiy Storchaka and Dominykas Grigonis in :gh:`121027`.) + +* The :ref:`garbage collector is now incremental `, + which means that the behavior of :func:`gc.collect` changes slightly: + + * ``gc.collect(1)``: Performs an increment of garbage collection, + rather than collecting generation 1. + * Other calls to :func:`!gc.collect` are unchanged. + +* The :func:`locale.nl_langinfo` function now temporarily sets the ``LC_CTYPE`` + locale in some cases. + This temporary change affects other threads. + (Contributed by Serhiy Storchaka in :gh:`69998`.) + +* :func:`pickle.dump` and :func:`pickle.dumps` now raise + :exc:`~pickle.PicklingError` for some failures that previously raised + :exc:`AttributeError`, :exc:`ImportError`, :exc:`ValueError`, + :exc:`UnicodeEncodeError` or :exc:`!PicklingError`, + depending on the implementation + (for example, pickling a local object, or an object whose module cannot be imported). + The original exception is chained to the :exc:`!PicklingError`. + Code that caught these exceptions should also catch :exc:`!PicklingError`. + (Contributed by Serhiy Storchaka in :gh:`122311`.) + +* :class:`types.UnionType` is now an alias for :class:`typing.Union`, + causing changes in some behaviors. + See :ref:`above ` for more details. + (Contributed by Jelle Zijlstra in :gh:`105499`.) + +* The runtime behavior of annotations has changed in various ways; see + :ref:`above ` for details. While most code that interacts + with annotations should continue to work, some undocumented details may behave + differently. + +* As part of making the :mod:`mimetypes` CLI public, + it now exits with ``1`` on failure instead of ``0`` + and ``2`` on incorrect command-line parameters instead of ``1``. + Error messages are now printed to stderr. + +* The ``\B`` pattern in regular expression now matches the empty string + when given as the entire pattern, which may cause behavioural changes. + +* On FreeBSD, :data:`sys.platform` no longer contains the major version number. + + +.. _whatsnew314-porting-annotations: + +Changes in annotations (:pep:`649` and :pep:`749`) +-------------------------------------------------- + +This section contains guidance on changes that may be needed to annotations +or Python code that interacts with or introspects annotations, +due to the changes related to :ref:`deferred evaluation of annotations +`. + +In the majority of cases, working code from older versions of Python +will not require any changes. + + +Implications for annotated code +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you define annotations in your code (for example, for use with a static type +checker), then this change probably does not affect you: you can keep +writing annotations the same way you did with previous versions of Python. + +You will likely be able to remove quoted strings in annotations, which are frequently +used for forward references. Similarly, if you use ``from __future__ import annotations`` +to avoid having to write strings in annotations, you may well be able to +remove that import once you support only Python 3.14 and newer. +However, if you rely on third-party libraries that read annotations, +those libraries may need changes to support unquoted annotations before they +work as expected. + + +Implications for readers of ``__annotations__`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If your code reads the :attr:`~object.__annotations__` attribute on objects, +you may want to make changes in order to support code that relies on +deferred evaluation of annotations. +For example, you may want to use :func:`annotationlib.get_annotations` with +the :attr:`~annotationlib.Format.FORWARDREF` format, +as the :mod:`dataclasses` module now does. + +The external :pypi:`typing_extensions` package provides partial backports +of some of the functionality of the :mod:`annotationlib` module, +such as the :class:`~annotationlib.Format` enum and +the :func:`~annotationlib.get_annotations` function. +These can be used to write cross-version code that takes advantage of +the new behavior in Python 3.14. + + +Related changes +^^^^^^^^^^^^^^^ + +The changes in Python 3.14 are designed to rework how :attr:`!__annotations__` +works at runtime while minimizing breakage to code that contains +annotations in source code and to code that reads :attr:`!__annotations__`. +However, if you rely on undocumented details of the annotation behavior +or on private functions in the standard library, there are many ways in which +your code may not work in Python 3.14. +To safeguard your code against future changes, only use the documented +functionality of the :mod:`annotationlib` module. + +In particular, do not read annotations directly from the namespace dictionary +attribute of type objects. +Use :func:`annotationlib.get_annotate_from_class_namespace` during class +construction and :func:`annotationlib.get_annotations` afterwards. + +In previous releases, it was sometimes possible to access class annotations +from an instance of an annotated class. This behavior was undocumented +and accidental, and will no longer work in Python 3.14. + + +``from __future__ import annotations`` +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In Python 3.7, :pep:`563` introduced the ``from __future__ import annotations`` +:ref:`future statement `, which turns all annotations into strings. + +However, this statement is now deprecated and it is expected to be removed +in a future version of Python. +This removal will not happen until after Python 3.13 reaches its end of life +in 2029, being the last version of Python without support for deferred +evaluation of annotations. + +In Python 3.14, the behavior of code using ``from __future__ import annotations`` +is unchanged. + + +Changes in the C API +-------------------- * :c:func:`Py_Finalize` now deletes all interned strings. This - is backwards incompatible to any C-Extension that holds onto an interned + is backwards incompatible to any C extension that holds onto an interned string after a call to :c:func:`Py_Finalize` and is then reused after a call to :c:func:`Py_Initialize`. Any issues arising from this behavior will normally result in crashes during the execution of the subsequent call to @@ -2657,6 +3475,7 @@ Porting to Python 3.14 * ``_Py_GetConfig()``: :c:func:`PyConfig_Get` and :c:func:`PyConfig_GetInt` * ``_Py_HashBytes()``: :c:func:`Py_HashBuffer` * ``_Py_fopen_obj()``: :c:func:`Py_fopen` + * ``PyMutex_IsLocked()`` : :c:func:`PyMutex_IsLocked` The `pythoncapi-compat project`_ can be used to get most of these new functions on Python 3.13 and older. @@ -2664,106 +3483,25 @@ Porting to Python 3.14 .. _pythoncapi-compat project: https://github.com/python/pythoncapi-compat/ -.. _whatsnew314-c-api-deprecated: +Notable changes in 3.14.1 +========================= -Deprecated ----------- +* Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and + :c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set + the stack protection base address and stack protection size of a Python + thread state. + (Contributed by Victor Stinner in :gh:`139653`.) -* The :c:macro:`!Py_HUGE_VAL` macro is :term:`soft deprecated`, - use :c:macro:`!Py_INFINITY` instead. - (Contributed by Sergey B Kirpichev in :gh:`120026`.) -* Macros :c:macro:`!Py_IS_NAN`, :c:macro:`!Py_IS_INFINITY` - and :c:macro:`!Py_IS_FINITE` are :term:`soft deprecated`, - use instead :c:macro:`!isnan`, :c:macro:`!isinf` and - :c:macro:`!isfinite` available from :file:`math.h` - since C99. (Contributed by Sergey B Kirpichev in :gh:`119613`.) - -* Non-tuple sequences are deprecated as argument for the ``(items)`` - format unit in :c:func:`PyArg_ParseTuple` and other - :ref:`argument parsing ` functions if *items* contains - format units which store a :ref:`borrowed buffer ` - or a :term:`borrowed reference`. - (Contributed by Serhiy Storchaka in :gh:`50333`.) +Notable changes in 3.14.5 +========================= -* The previously undocumented function :c:func:`PySequence_In` is :term:`soft deprecated`. - Use :c:func:`PySequence_Contains` instead. - (Contributed by Yuki Kobayashi in :gh:`127896`.) - -.. Add C API deprecations above alphabetically, not here at the end. - -* The ``PyMonitoring_FireBranchEvent`` function is deprecated and should - be replaced with calls to :c:func:`PyMonitoring_FireBranchLeftEvent` - and :c:func:`PyMonitoring_FireBranchRightEvent`. - -* The following private functions are deprecated and planned for removal in - Python 3.18: - - * :c:func:`!_PyBytes_Join`: use :c:func:`PyBytes_Join`. - * :c:func:`!_PyDict_GetItemStringWithError`: use :c:func:`PyDict_GetItemStringRef`. - * :c:func:`!_PyDict_Pop()`: use :c:func:`PyDict_Pop`. - * :c:func:`!_PyLong_Sign()`: use :c:func:`PyLong_GetSign`. - * :c:func:`!_PyLong_FromDigits` and :c:func:`!_PyLong_New`: - use :c:func:`PyLongWriter_Create`. - * :c:func:`!_PyThreadState_UncheckedGet`: use :c:func:`PyThreadState_GetUnchecked`. - * :c:func:`!_PyUnicode_AsString`: use :c:func:`PyUnicode_AsUTF8`. - * :c:func:`!_PyUnicodeWriter_Init`: - replace ``_PyUnicodeWriter_Init(&writer)`` with - :c:func:`writer = PyUnicodeWriter_Create(0) `. - * :c:func:`!_PyUnicodeWriter_Finish`: - replace ``_PyUnicodeWriter_Finish(&writer)`` with - :c:func:`PyUnicodeWriter_Finish(writer) `. - * :c:func:`!_PyUnicodeWriter_Dealloc`: - replace ``_PyUnicodeWriter_Dealloc(&writer)`` with - :c:func:`PyUnicodeWriter_Discard(writer) `. - * :c:func:`!_PyUnicodeWriter_WriteChar`: - replace ``_PyUnicodeWriter_WriteChar(&writer, ch)`` with - :c:func:`PyUnicodeWriter_WriteChar(writer, ch) `. - * :c:func:`!_PyUnicodeWriter_WriteStr`: - replace ``_PyUnicodeWriter_WriteStr(&writer, str)`` with - :c:func:`PyUnicodeWriter_WriteStr(writer, str) `. - * :c:func:`!_PyUnicodeWriter_WriteSubstring`: - replace ``_PyUnicodeWriter_WriteSubstring(&writer, str, start, end)`` with - :c:func:`PyUnicodeWriter_WriteSubstring(writer, str, start, end) `. - * :c:func:`!_PyUnicodeWriter_WriteASCIIString`: - replace ``_PyUnicodeWriter_WriteASCIIString(&writer, str)`` with - :c:func:`PyUnicodeWriter_WriteUTF8(writer, str) `. - * :c:func:`!_PyUnicodeWriter_WriteLatin1String`: - replace ``_PyUnicodeWriter_WriteLatin1String(&writer, str)`` with - :c:func:`PyUnicodeWriter_WriteUTF8(writer, str) `. - * :c:func:`!_Py_HashPointer`: use :c:func:`Py_HashPointer`. - * :c:func:`!_Py_fopen_obj`: use :c:func:`Py_fopen`. - - The `pythoncapi-compat project`_ can be used to get these new public - functions on Python 3.13 and older. - (Contributed by Victor Stinner in :gh:`128863`.) - -.. include:: ../deprecations/c-api-pending-removal-in-3.15.rst - -.. include:: ../deprecations/c-api-pending-removal-in-3.18.rst - -.. include:: ../deprecations/c-api-pending-removal-in-future.rst - - -.. _whatsnew314-c-api-removed: - -Removed -------- - -* Creating :c:data:`immutable types ` with mutable - bases was deprecated since 3.12 and now raises a :exc:`TypeError`. - -* Remove ``PyDictObject.ma_version_tag`` member which was deprecated since - Python 3.12. Use the :c:func:`PyDict_AddWatcher` API instead. - (Contributed by Sam Gross in :gh:`124296`.) - -* Remove the private ``_Py_InitializeMain()`` function. It was a - :term:`provisional API` added to Python 3.8 by :pep:`587`. - (Contributed by Victor Stinner in :gh:`129033`.) +gc +-- -* The undocumented APIs :c:macro:`!Py_C_RECURSION_LIMIT` and - :c:member:`!PyThreadState.c_recursion_remaining`, added in 3.13, are removed - without a deprecation period. - Please use :c:func:`Py_EnterRecursiveCall` to guard against runaway recursion - in C code. - (Removed in :gh:`133079`, see also :gh:`130396`.) +* The incremental garbage collector shipped in Python 3.14.0-3.14.4 has been + reverted back to the generational garbage collector from 3.13, + due to a number of `reports + `__ + of significant memory pressure in production environments. + See :ref:`whatsnew314-incremental-gc` for details. diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index 7104904c956a7a9..2191b2310142f24 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -458,7 +458,7 @@ Some smaller changes made to the core Python language are: :class:`~collections.defaultdict`, :class:`~shelve.Shelf`, :class:`~configparser.ConfigParser`, or :mod:`dbm`. It is also useful with custom :class:`dict` subclasses that normalize keys before look-up or that - supply a :meth:`__missing__` method for unknown keys:: + supply a :meth:`~object.__missing__` method for unknown keys:: >>> import shelve >>> d = shelve.open('tmp.shl') @@ -992,12 +992,12 @@ datetime and time offset and timezone name. This makes it easier to create timezone-aware datetime objects:: - >>> from datetime import datetime, timezone + >>> import datetime as dt - >>> datetime.now(timezone.utc) + >>> dt.datetime.now(dt.timezone.utc) datetime.datetime(2010, 12, 8, 21, 4, 2, 923754, tzinfo=datetime.timezone.utc) - >>> datetime.strptime("01/01/2000 12:00 +0000", "%m/%d/%Y %H:%M %z") + >>> dt.datetime.strptime("01/01/2000 12:00 +0000", "%m/%d/%Y %H:%M %z") datetime.datetime(2000, 1, 1, 12, 0, tzinfo=datetime.timezone.utc) * Also, :class:`~datetime.timedelta` objects can now be multiplied by diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index 7a8eb47cbdb354e..87fb3f54e687b52 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -331,7 +331,7 @@ simplified and finer-grained. You don't have to worry anymore about choosing the appropriate exception type between :exc:`OSError`, :exc:`IOError`, :exc:`EnvironmentError`, -:exc:`WindowsError`, :exc:`mmap.error`, :exc:`socket.error` or +:exc:`WindowsError`, :exc:`!mmap.error`, :exc:`socket.error` or :exc:`select.error`. All these exception types are now only one: :exc:`OSError`. The other names are kept as aliases for compatibility reasons. @@ -805,7 +805,7 @@ Some smaller changes made to the core Python language are: * New methods have been added to :class:`list` and :class:`bytearray`: ``copy()`` and ``clear()`` (:issue:`10516`). Consequently, :class:`~collections.abc.MutableSequence` now also defines a - :meth:`~collections.abc.MutableSequence.clear` method (:issue:`11388`). + :meth:`!clear` method (:issue:`11388`). * Raw bytes literals can now be written ``rb"..."`` as well as ``br"..."``. @@ -869,10 +869,10 @@ faulthandler This new debug module :mod:`faulthandler` contains functions to dump Python tracebacks explicitly, on a fault (a crash like a segmentation fault), after a timeout, or on a user signal. Call :func:`faulthandler.enable` to install fault handlers for the -:const:`SIGSEGV`, :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS`, and -:const:`SIGILL` signals. You can also enable them at startup by setting the -:envvar:`PYTHONFAULTHANDLER` environment variable or by using :option:`-X` -``faulthandler`` command line option. +:const:`~signal.SIGSEGV`, :const:`~signal.SIGFPE`, :const:`~signal.SIGABRT`, +:const:`~signal.SIGBUS`, and :const:`~signal.SIGILL` signals. +You can also enable them at startup by setting the :envvar:`PYTHONFAULTHANDLER` +environment variable or by using :option:`-X` ``faulthandler`` command line option. Example of a segmentation fault on Linux: @@ -916,7 +916,7 @@ abc Improved support for abstract base classes containing descriptors composed with abstract methods. The recommended approach to declaring abstract descriptors is -now to provide :attr:`__isabstractmethod__` as a dynamically updated +now to provide :attr:`!__isabstractmethod__` as a dynamically updated property. The built-in descriptors have been updated accordingly. * :class:`abc.abstractproperty` has been deprecated, use :class:`property` @@ -979,7 +979,7 @@ new features have been added: (Contributed by Nir Aides in :issue:`1625`.) * :class:`bz2.BZ2File` now implements all of the :class:`io.BufferedIOBase` API, - except for the :meth:`detach` and :meth:`truncate` methods. + except for the :meth:`!detach` and :meth:`!truncate` methods. codecs @@ -1064,7 +1064,7 @@ curses * If the :mod:`curses` module is linked to the ncursesw library, use Unicode functions when Unicode strings or characters are passed (e.g. - :c:func:`waddwstr`), and bytes functions otherwise (e.g. :c:func:`waddstr`). + :c:func:`!waddwstr`), and bytes functions otherwise (e.g. :c:func:`!waddstr`). * Use the locale encoding instead of ``utf-8`` to encode Unicode strings. * :class:`curses.window` has a new :attr:`curses.window.encoding` attribute. * The :class:`curses.window` class has a new :meth:`~curses.window.get_wch` @@ -1137,15 +1137,15 @@ API changes * The C module has the following context limits, depending on the machine architecture: - +-------------------+----------------+-------------------------+ - | | 32-bit | 64-bit | - +===================+================+=========================+ - | :const:`MAX_PREC` | ``425000000`` | ``999999999999999999`` | - +-------------------+----------------+-------------------------+ - | :const:`MAX_EMAX` | ``425000000`` | ``999999999999999999`` | - +-------------------+----------------+-------------------------+ - | :const:`MIN_EMIN` | ``-425000000`` | ``-999999999999999999`` | - +-------------------+----------------+-------------------------+ + +----------------------------+----------------+-------------------------+ + | | 32-bit | 64-bit | + +============================+================+=========================+ + | :const:`~decimal.MAX_PREC` | ``425000000`` | ``999999999999999999`` | + +----------------------------+----------------+-------------------------+ + | :const:`~decimal.MAX_EMAX` | ``425000000`` | ``999999999999999999`` | + +----------------------------+----------------+-------------------------+ + | :const:`~decimal.MIN_EMIN` | ``-425000000`` | ``-999999999999999999`` | + +----------------------------+----------------+-------------------------+ * In the context templates (:const:`~decimal.DefaultContext`, :const:`~decimal.BasicContext` and :const:`~decimal.ExtendedContext`) @@ -1434,7 +1434,7 @@ html :class:`html.parser.HTMLParser` is now able to parse broken markup without raising errors, therefore the *strict* argument of the constructor and the -:exc:`~html.parser.HTMLParseError` exception are now deprecated. +:exc:`!HTMLParseError` exception are now deprecated. The ability to parse broken markup is the result of a number of bug fixes that are also available on the latest bug fix releases of Python 2.7/3.2. (Contributed by Ezio Melotti in :issue:`15114`, and :issue:`14538`, @@ -1486,7 +1486,7 @@ already exists. It is based on the C11 'x' mode to fopen(). The constructor of the :class:`~io.TextIOWrapper` class has a new *write_through* optional argument. If *write_through* is ``True``, calls to -:meth:`~io.TextIOWrapper.write` are guaranteed not to be buffered: any data +:meth:`!write` are guaranteed not to be buffered: any data written on the :class:`~io.TextIOWrapper` object is immediately handled to its underlying binary buffer. @@ -1504,7 +1504,7 @@ logging The :func:`~logging.basicConfig` function now supports an optional ``handlers`` argument taking an iterable of handlers to be added to the root logger. -A class level attribute :attr:`~logging.handlers.SysLogHandler.append_nul` has +A class level attribute :attr:`!append_nul` has been added to :class:`~logging.handlers.SysLogHandler` to allow control of the appending of the ``NUL`` (``\000``) byte to syslog records, since for some daemons it is required while for others it is passed through to the log. @@ -1536,8 +1536,8 @@ The new :func:`multiprocessing.connection.wait` function allows polling multiple objects (such as connections, sockets and pipes) with a timeout. (Contributed by Richard Oudkerk in :issue:`12328`.) -:class:`multiprocessing.Connection` objects can now be transferred over -multiprocessing connections. +:class:`multiprocessing.connection.Connection` objects can now be transferred +over multiprocessing connections. (Contributed by Richard Oudkerk in :issue:`4892`.) :class:`multiprocessing.Process` now accepts a ``daemon`` keyword argument @@ -1611,7 +1611,7 @@ os :func:`~os.rename`, :func:`~os.replace`, :func:`~os.rmdir`, :func:`~os.stat`, :func:`~os.symlink`, :func:`~os.unlink`, :func:`~os.utime`. Platform support for using these parameters can be checked via the sets - :data:`os.supports_dir_fd` and :data:`os.supports_follows_symlinks`. + :data:`os.supports_dir_fd` and :data:`os.supports_follow_symlinks`. - The following functions now support a file descriptor for their path argument: :func:`~os.chdir`, :func:`~os.chmod`, :func:`~os.chown`, @@ -1698,7 +1698,7 @@ os :const:`~os.RTLD_NOLOAD`, and :const:`~os.RTLD_DEEPBIND` are available on platforms that support them. These are for use with the :func:`sys.setdlopenflags` function, and supersede the similar constants - defined in :mod:`ctypes` and :mod:`DLFCN`. (Contributed by Victor Stinner + defined in :mod:`ctypes` and :mod:`!DLFCN`. (Contributed by Victor Stinner in :issue:`13226`.) * :func:`os.symlink` now accepts (and ignores) the ``target_is_directory`` @@ -1728,8 +1728,8 @@ reduction functions to be set. pydoc ----- -The Tk GUI and the :func:`~pydoc.serve` function have been removed from the -:mod:`pydoc` module: ``pydoc -g`` and :func:`~pydoc.serve` have been deprecated +The Tk GUI and the :func:`!serve` function have been removed from the +:mod:`pydoc` module: ``pydoc -g`` and :func:`!serve` have been deprecated in Python 3.2. @@ -1931,7 +1931,7 @@ ssl * :func:`~ssl.RAND_bytes`: generate cryptographically strong pseudo-random bytes. - * :func:`~ssl.RAND_pseudo_bytes`: generate pseudo-random bytes. + * :func:`!RAND_pseudo_bytes`: generate pseudo-random bytes. (Contributed by Victor Stinner in :issue:`12049`.) @@ -2020,8 +2020,7 @@ tarfile tempfile -------- -:class:`tempfile.SpooledTemporaryFile`\'s -:meth:`~tempfile.SpooledTemporaryFile.truncate` method now accepts +:class:`tempfile.SpooledTemporaryFile`\'s :meth:`!truncate` method now accepts a ``size`` parameter. (Contributed by Ryan Kelly in :issue:`9957`.) @@ -2129,7 +2128,7 @@ xml.etree.ElementTree The :mod:`xml.etree.ElementTree` module now imports its C accelerator by default; there is no longer a need to explicitly import -:mod:`xml.etree.cElementTree` (this module stays for backwards compatibility, +:mod:`!xml.etree.cElementTree` (this module stays for backwards compatibility, but is now deprecated). In addition, the ``iter`` family of methods of :class:`~xml.etree.ElementTree.Element` has been optimized (rewritten in C). The module's documentation has also been greatly improved with added examples @@ -2197,7 +2196,7 @@ Changes to Python's build process and to the C API include: * :c:func:`PyUnicode_AsUCS4`, :c:func:`PyUnicode_AsUCS4Copy` * :c:macro:`PyUnicode_DATA`, :c:macro:`PyUnicode_1BYTE_DATA`, :c:macro:`PyUnicode_2BYTE_DATA`, :c:macro:`PyUnicode_4BYTE_DATA` - * :c:macro:`PyUnicode_KIND` with :c:enum:`PyUnicode_Kind` enum: + * :c:macro:`PyUnicode_KIND` with :c:enum:`!PyUnicode_Kind` enum: :c:data:`!PyUnicode_WCHAR_KIND`, :c:data:`PyUnicode_1BYTE_KIND`, :c:data:`PyUnicode_2BYTE_KIND`, :c:data:`PyUnicode_4BYTE_KIND` * :c:macro:`PyUnicode_READ`, :c:macro:`PyUnicode_READ_CHAR`, :c:macro:`PyUnicode_WRITE` @@ -2232,17 +2231,17 @@ Deprecated Python modules, functions and methods (``utf-32-le`` or ``utf-32-be``) * :meth:`ftplib.FTP.nlst` and :meth:`ftplib.FTP.dir`: use :meth:`ftplib.FTP.mlsd` -* :func:`platform.popen`: use the :mod:`subprocess` module. Check especially +* :func:`!platform.popen`: use the :mod:`subprocess` module. Check especially the :ref:`subprocess-replacements` section (:issue:`11377`). * :issue:`13374`: The Windows bytes API has been deprecated in the :mod:`os` module. Use Unicode filenames, instead of bytes filenames, to not depend on the ANSI code page anymore and to support any filename. -* :issue:`13988`: The :mod:`xml.etree.cElementTree` module is deprecated. The +* :issue:`13988`: The :mod:`!xml.etree.cElementTree` module is deprecated. The accelerator is used automatically whenever available. -* The behaviour of :func:`time.clock` depends on the platform: use the new +* The behaviour of :func:`!time.clock` depends on the platform: use the new :func:`time.perf_counter` or :func:`time.process_time` function instead, depending on your requirements, to have a well defined behaviour. -* The :func:`os.stat_float_times` function is deprecated. +* The :func:`!os.stat_float_times` function is deprecated. * :mod:`abc` module: * :class:`abc.abstractproperty` has been deprecated, use :class:`property` @@ -2466,7 +2465,7 @@ Porting C code * In the course of changes to the buffer API the undocumented :c:member:`!smalltable` member of the :c:type:`Py_buffer` structure has been removed and the - layout of the :c:type:`PyMemoryViewObject` has changed. + layout of the :c:type:`!PyMemoryViewObject` has changed. All extensions relying on the relevant parts in ``memoryobject.h`` or ``object.h`` must be rebuilt. diff --git a/Doc/whatsnew/3.4.rst b/Doc/whatsnew/3.4.rst index e4f602a17ee9687..9f4068116e3726f 100644 --- a/Doc/whatsnew/3.4.rst +++ b/Doc/whatsnew/3.4.rst @@ -2,7 +2,7 @@ What's New In Python 3.4 **************************** -:Author: R. David Murray (Editor) +:Author: \R. David Murray (Editor) .. Rules for maintenance: @@ -122,7 +122,7 @@ Significantly improved library modules: on Unix ` (:issue:`8713`). * :mod:`email` has a new submodule, :mod:`~email.contentmanager`, and a new :mod:`~email.message.Message` subclass - (:class:`~email.contentmanager.EmailMessage`) that :ref:`simplify MIME + (:class:`~email.message.EmailMessage`) that :ref:`simplify MIME handling ` (:issue:`18891`). * The :mod:`inspect` and :mod:`pydoc` modules are now capable of correct introspection of a much wider variety of callable objects, @@ -154,7 +154,7 @@ Security improvements: `. * All modules in the standard library that support SSL now support server certificate verification, including hostname matching - (:func:`ssl.match_hostname`) and CRLs (Certificate Revocation lists, see + (:func:`!ssl.match_hostname`) and CRLs (Certificate Revocation lists, see :func:`ssl.SSLContext.load_verify_locations`). CPython implementation improvements: @@ -739,7 +739,7 @@ these new components. In addition, a new application-friendly class :class:`~dis.Bytecode` provides an object-oriented API for inspecting bytecode in both in human-readable form and for iterating over instructions. The :class:`~dis.Bytecode` constructor -takes the same arguments that :func:`~dis.get_instruction` does (plus an +takes the same arguments that :func:`~dis.get_instructions` does (plus an optional *current_offset*), and the resulting object can be iterated to produce :class:`~dis.Instruction` objects. But it also has a :mod:`~dis.Bytecode.dis` method, equivalent to calling :mod:`~dis.dis` on the constructor argument, but @@ -958,8 +958,9 @@ http optional additional *explain* parameter which can be used to provide an extended error description, overriding the hardcoded default if there is one. This extended error description will be formatted using the -:attr:`~http.server.HTTP.error_message_format` attribute and sent as the body -of the error response. (Contributed by Karl Cow in :issue:`12921`.) +:attr:`~http.server.BaseHTTPRequestHandler.error_message_format` attribute +and sent as the body of the error response. +(Contributed by Karl Cow in :issue:`12921`.) The :mod:`http.server` :ref:`command line interface ` now has a ``-b/--bind`` option that causes the server to listen on a specific address. @@ -1038,7 +1039,7 @@ As part of the implementation of the new :mod:`enum` module, the metaclasses. (Contributed by Ethan Furman in :issue:`18929` and :issue:`19030`.) -:func:`~inspect.getfullargspec` and :func:`~inspect.getargspec` +:func:`~inspect.getfullargspec` and :func:`!getargspec` now use the :func:`~inspect.signature` API. This allows them to support a much broader range of callables, including those with ``__signature__`` attributes, those with metadata provided by argument @@ -1221,7 +1222,7 @@ pickle :mod:`pickle` now supports (but does not use by default) a new pickle protocol, protocol 4. This new protocol addresses a number of issues that were present in previous protocols, such as the serialization of nested classes, very large -strings and containers, and classes whose :meth:`__new__` method takes +strings and containers, and classes whose :meth:`~object.__new__` method takes keyword-only arguments. It also provides some efficiency improvements. .. seealso:: @@ -1299,7 +1300,7 @@ affect the behaviour of :func:`help`. re -- -New :func:`~re.fullmatch` function and :meth:`.regex.fullmatch` method anchor +New :func:`~re.fullmatch` function and :meth:`.Pattern.fullmatch` method anchor the pattern at both ends of the string to match. This provides a way to be explicit about the goal of the match, which avoids a class of subtle bugs where ``$`` characters get lost during code changes or the addition of alternatives @@ -1519,7 +1520,7 @@ subprocess be used to provide the contents of ``stdin`` for the command that is run. (Contributed by Zack Weinberg in :issue:`16624`.) -:func:`~subprocess.getstatus` and :func:`~subprocess.getstatusoutput` now +:func:`~subprocess.getoutput` and :func:`~subprocess.getstatusoutput` now work on Windows. This change was actually inadvertently made in 3.3.4. (Contributed by Tim Golden in :issue:`10197`.) @@ -1535,7 +1536,7 @@ plain tuple. (Contributed by Claudiu Popa in :issue:`18901`.) called automatically at the end of the block. (Contributed by Serhiy Storchaka in :issue:`18878`.) -:meth:`.AU_write.setsampwidth` now supports 24 bit samples, thus adding +:meth:`!AU_write.setsampwidth` now supports 24 bit samples, thus adding support for writing 24 sample using the module. (Contributed by Serhiy Storchaka in :issue:`19261`.) @@ -1615,7 +1616,7 @@ A new :func:`~types.DynamicClassAttribute` descriptor provides a way to define an attribute that acts normally when looked up through an instance object, but which is routed to the *class* ``__getattr__`` when looked up through the class. This allows one to have properties active on a class, and have virtual -attributes on the class with the same name (see :mod:`Enum` for an example). +attributes on the class with the same name (see :mod:`enum` for an example). (Contributed by Ethan Furman in :issue:`19030`.) @@ -1709,7 +1710,7 @@ matching calls, which means an argument can now be matched by either position or name, instead of only by position. (Contributed by Antoine Pitrou in :issue:`17015`.) -:func:`~mock.mock_open` objects now have ``readline`` and ``readlines`` +:func:`~unittest.mock.mock_open` objects now have ``readline`` and ``readlines`` methods. (Contributed by Toshio Kuratomi in :issue:`17467`.) @@ -1729,8 +1730,8 @@ installed in the virtual environment. (Contributed by Nick Coghlan in wave ---- -The :meth:`~wave.getparams` method now returns a namedtuple rather than a -plain tuple. (Contributed by Claudiu Popa in :issue:`17487`.) +The :meth:`~wave.Wave_read.getparams` method now returns a namedtuple rather +than a plain tuple. (Contributed by Claudiu Popa in :issue:`17487`.) :meth:`wave.open` now supports the context management protocol. (Contributed by Claudiu Popa in :issue:`17616`.) @@ -1788,7 +1789,7 @@ example, this could be used to exclude test files from the archive. (Contributed by Christian Tismer in :issue:`19274`.) The *allowZip64* parameter to :class:`~zipfile.ZipFile` and -:class:`~zipfile.PyZipfile` is now ``True`` by default. (Contributed by +:class:`~zipfile.PyZipFile` is now ``True`` by default. (Contributed by William Mallard in :issue:`17201`.) @@ -1817,7 +1818,7 @@ PEP 442: Safe Object Finalization --------------------------------- :pep:`442` removes the current limitations and quirks of object finalization -in CPython. With it, objects with :meth:`__del__` methods, as well as +in CPython. With it, objects with :meth:`~object.__del__` methods, as well as generators with :keyword:`finally` clauses, can be finalized when they are part of a reference cycle. @@ -2043,7 +2044,7 @@ Significant Optimizations strings is now significantly faster. (Contributed by Victor Stinner and Antoine Pitrou in :issue:`15596`.) -* A performance issue in :meth:`io.FileIO.readall` has been solved. This +* A performance issue in :meth:`!io.FileIO.readall` has been solved. This particularly affects Windows, and significantly speeds up the case of piping significant amounts of data through :mod:`subprocess`. (Contributed by Richard Oudkerk in :issue:`15758`.) @@ -2103,7 +2104,7 @@ Deprecations in the Python API * The :mod:`!imp` module is pending deprecation. To keep compatibility with Python 2/3 code bases, the module's removal is currently not scheduled. -* The :mod:`formatter` module is pending deprecation and is slated for removal +* The :mod:`!formatter` module is pending deprecation and is slated for removal in Python 3.6. * ``MD5`` as the default *digestmod* for the :func:`hmac.new` function is @@ -2120,11 +2121,11 @@ Deprecations in the Python API * The *strict* argument of :class:`~html.parser.HTMLParser` is deprecated. -* The :mod:`plistlib` :func:`~plistlib.readPlist`, - :func:`~plistlib.writePlist`, :func:`~plistlib.readPlistFromBytes`, and - :func:`~plistlib.writePlistToBytes` functions are deprecated in favor of the +* The :mod:`plistlib` :func:`!readPlist`, + :func:`!writePlist`, :func:`!readPlistFromBytes`, and + :func:`!writePlistToBytes` functions are deprecated in favor of the corresponding new functions :func:`~plistlib.load`, :func:`~plistlib.dump`, - :func:`~plistlib.loads`, and :func:`~plistlib.dumps`. :func:`~plistlib.Data` + :func:`~plistlib.loads`, and :func:`~plistlib.dumps`. :func:`!Data` is deprecated in favor of just using the :class:`bytes` constructor. * The :mod:`sysconfig` key ``SO`` is deprecated, it has been replaced by @@ -2212,8 +2213,8 @@ removed: that do not have a __format__ method that handles it. See :issue:`7994` for background. -* :meth:`difflib.SequenceMatcher.isbjunk` and - :meth:`difflib.SequenceMatcher.isbpopular` were deprecated in 3.2, and have +* :meth:`!difflib.SequenceMatcher.isbjunk` and + :meth:`!difflib.SequenceMatcher.isbpopular` were deprecated in 3.2, and have now been removed: use ``x in sm.bjunk`` and ``x in sm.bpopular``, where *sm* is a :class:`~difflib.SequenceMatcher` object (:issue:`13248`). @@ -2280,7 +2281,7 @@ Changes in the Python API * :meth:`!importlib.util.module_for_loader` now sets ``__loader__`` and ``__package__`` unconditionally to properly support reloading. If this is not desired then you will need to set these attributes manually. You can use - :func:`importlib.util.module_to_load` for module management. + :func:`!importlib.util.module_to_load` for module management. * Import now resets relevant attributes (e.g. ``__name__``, ``__loader__``, ``__package__``, ``__file__``, ``__cached__``) unconditionally when reloading. @@ -2428,7 +2429,7 @@ Changes in the Python API disallowed command forms didn't make any sense and are unlikely to be in use. * The :func:`re.split`, :func:`re.findall`, and :func:`re.sub` functions, and - the :meth:`~re.match.group` and :meth:`~re.match.groups` methods of + the :meth:`~re.Match.group` and :meth:`~re.Match.groups` methods of ``match`` objects now always return a *bytes* object when the string to be matched is a :term:`bytes-like object`. Previously the return type matched the input type, so if your code was depending on the return value diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index db3f1db3bd74ad5..6009dd8a71eea52 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -181,7 +181,7 @@ Coroutine functions are declared using the new :keyword:`async def` syntax:: Inside a coroutine function, the new :keyword:`await` expression can be used to suspend coroutine execution until the result is available. Any object can be *awaited*, as long as it implements the :term:`awaitable` protocol by -defining the :meth:`__await__` method. +defining the :meth:`~object.__await__` method. PEP 492 also adds :keyword:`async for` statement for convenient iteration over asynchronous iterables. @@ -273,9 +273,10 @@ PEP 465 - A dedicated infix operator for matrix multiplication :pep:`465` adds the ``@`` infix operator for matrix multiplication. Currently, no builtin Python types implement the new operator, however, it -can be implemented by defining :meth:`__matmul__`, :meth:`__rmatmul__`, -and :meth:`__imatmul__` for regular, reflected, and in-place matrix -multiplication. The semantics of these methods is similar to that of +can be implemented by defining :meth:`~object.__matmul__`, +:meth:`~object.__rmatmul__`, and :meth:`~object.__imatmul__` for regular, +reflected, and in-place matrix multiplication. +The semantics of these methods is similar to that of methods defining other infix arithmetic operators. Matrix multiplication is a notably common operation in many fields of @@ -800,7 +801,7 @@ Notable changes in the :mod:`asyncio` module since Python 3.4.0: control. (Contributed by Victor Stinner.) -* The :func:`~asyncio.async` function is deprecated in favor of +* The :func:`!async` function is deprecated in favor of :func:`~asyncio.ensure_future`. (Contributed by Yury Selivanov.) @@ -905,10 +906,8 @@ collections The :class:`~collections.OrderedDict` class is now implemented in C, which makes it 4 to 100 times faster. (Contributed by Eric Snow in :issue:`16991`.) -:meth:`OrderedDict.items() `, -:meth:`OrderedDict.keys() `, -:meth:`OrderedDict.values() ` views now support -:func:`reversed` iteration. +:meth:`!OrderedDict.items`, :meth:`!OrderedDict.keys`, +and :meth:`!OrderedDict.values` views now support :func:`reversed` iteration. (Contributed by Serhiy Storchaka in :issue:`19505`.) The :class:`~collections.deque` class now defines @@ -928,7 +927,7 @@ Docstrings produced by :func:`~collections.namedtuple` can now be updated:: (Contributed by Berker Peksag in :issue:`24064`.) The :class:`~collections.UserString` class now implements the -:meth:`__getnewargs__`, :meth:`__rmod__`, :meth:`~str.casefold`, +:meth:`~object.__getnewargs__`, :meth:`~object.__rmod__`, :meth:`~str.casefold`, :meth:`~str.format_map`, :meth:`~str.isprintable`, and :meth:`~str.maketrans` methods to match the corresponding methods of :class:`str`. (Contributed by Joe Jevnik in :issue:`22189`.) @@ -937,7 +936,7 @@ methods to match the corresponding methods of :class:`str`. collections.abc --------------- -The :meth:`Sequence.index() ` method now +The :meth:`!Sequence.index` method now accepts *start* and *stop* arguments to match the corresponding methods of :class:`tuple`, :class:`list`, etc. (Contributed by Devin Jeanpierre in :issue:`23086`.) @@ -1045,8 +1044,8 @@ not just sequences. (Contributed by Serhiy Storchaka in :issue:`23171`.) curses ------ -The new :func:`~curses.update_lines_cols` function updates the :data:`LINES` -and :data:`COLS` module variables. This is useful for detecting +The new :func:`~curses.update_lines_cols` function updates the :data:`~curses.LINES` +and :data:`~curses.COLS` module variables. This is useful for detecting manual screen resizing. (Contributed by Arnon Yaari in :issue:`4254`.) @@ -1347,8 +1346,8 @@ network objects from existing addresses:: (Contributed by Peter Moody and Antoine Pitrou in :issue:`16531`.) -A new :attr:`~ipaddress.IPv4Network.reverse_pointer` attribute for the -:class:`~ipaddress.IPv4Network` and :class:`~ipaddress.IPv6Network` classes +A new :attr:`~ipaddress.IPv4Address.reverse_pointer` attribute for the +:class:`~ipaddress.IPv4Address` and :class:`~ipaddress.IPv6Address` classes returns the name of the reverse DNS PTR record:: >>> import ipaddress @@ -1451,7 +1450,7 @@ and :data:`~math.nan`. (Contributed by Mark Dickinson in :issue:`23185`.) A new function :func:`~math.isclose` provides a way to test for approximate equality. (Contributed by Chris Barker and Tal Einat in :issue:`24270`.) -A new :func:`~math.gcd` function has been added. The :func:`fractions.gcd` +A new :func:`~math.gcd` function has been added. The :func:`!fractions.gcd` function is now deprecated. (Contributed by Mark Dickinson and Serhiy Storchaka in :issue:`22486`.) @@ -1602,10 +1601,10 @@ The :func:`~re.sub` and :func:`~re.subn` functions now replace unmatched groups with empty strings instead of raising an exception. (Contributed by Serhiy Storchaka in :issue:`1519638`.) -The :class:`re.error` exceptions have new attributes, -:attr:`~re.error.msg`, :attr:`~re.error.pattern`, -:attr:`~re.error.pos`, :attr:`~re.error.lineno`, -and :attr:`~re.error.colno`, that provide better context +The :class:`re.error ` exceptions have new attributes, +:attr:`~re.PatternError.msg`, :attr:`~re.PatternError.pattern`, +:attr:`~re.PatternError.pos`, :attr:`~re.PatternError.lineno`, +and :attr:`~re.PatternError.colno`, that provide better context information about the error:: >>> re.compile(""" @@ -1794,10 +1793,10 @@ query the actual protocol version in use. (Contributed by Antoine Pitrou in :issue:`20421`.) The :class:`~ssl.SSLSocket` class now implements -a :meth:`SSLSocket.sendfile() ` method. +a :meth:`!SSLSocket.sendfile` method. (Contributed by Giampaolo Rodola' in :issue:`17552`.) -The :meth:`SSLSocket.send() ` method now raises either +The :meth:`!SSLSocket.send` method now raises either the :exc:`ssl.SSLWantReadError` or :exc:`ssl.SSLWantWriteError` exception on a non-blocking socket if the operation would block. Previously, it would return ``0``. (Contributed by Nikolaus Rath in :issue:`20951`.) @@ -1806,20 +1805,20 @@ The :func:`~ssl.cert_time_to_seconds` function now interprets the input time as UTC and not as local time, per :rfc:`5280`. Additionally, the return value is always an :class:`int`. (Contributed by Akira Li in :issue:`19940`.) -New :meth:`SSLObject.shared_ciphers() ` and +New :meth:`!SSLObject.shared_ciphers` and :meth:`SSLSocket.shared_ciphers() ` methods return the list of ciphers sent by the client during the handshake. (Contributed by Benjamin Peterson in :issue:`23186`.) The :meth:`SSLSocket.do_handshake() `, :meth:`SSLSocket.read() `, -:meth:`SSLSocket.shutdown() `, and +:meth:`!SSLSocket.shutdown`, and :meth:`SSLSocket.write() ` methods of the :class:`~ssl.SSLSocket` class no longer reset the socket timeout every time bytes are received or sent. The socket timeout is now the maximum total duration of the method. (Contributed by Victor Stinner in :issue:`23853`.) -The :func:`~ssl.match_hostname` function now supports matching of IP addresses. +The :func:`!match_hostname` function now supports matching of IP addresses. (Contributed by Antoine Pitrou in :issue:`23239`.) @@ -1863,10 +1862,10 @@ Examples:: sys --- -A new :func:`~sys.set_coroutine_wrapper` function allows setting a global +A new :func:`!set_coroutine_wrapper` function allows setting a global hook that will be called whenever a :term:`coroutine object ` is created by an :keyword:`async def` function. A corresponding -:func:`~sys.get_coroutine_wrapper` can be used to obtain a currently set +:func:`!get_coroutine_wrapper` can be used to obtain a currently set wrapper. Both functions are :term:`provisional `, and are intended for debugging purposes only. (Contributed by Yury Selivanov in :issue:`24017`.) @@ -2014,8 +2013,9 @@ The :class:`~unittest.mock.Mock` class has the following improvements: method to check if the mock object was called. (Contributed by Kushal Das in :issue:`21262`.) -The :class:`~unittest.mock.MagicMock` class now supports :meth:`__truediv__`, -:meth:`__divmod__` and :meth:`__matmul__` operators. +The :class:`~unittest.mock.MagicMock` class now supports +:meth:`~object.__truediv__`, :meth:`~object.__divmod__` +and :meth:`~object.__matmul__` operators. (Contributed by Johannes Baiter in :issue:`20968`, and Håkan Lövdahl in :issue:`23581` and :issue:`23568`.) @@ -2290,10 +2290,10 @@ Windows XP is no longer supported by Microsoft, thus, per :PEP:`11`, CPython Deprecated Python modules, functions and methods ------------------------------------------------ -The :mod:`formatter` module has now graduated to full deprecation and is still +The :mod:`!formatter` module has now graduated to full deprecation and is still slated for removal in Python 3.6. -The :func:`asyncio.async` function is deprecated in favor of +The :func:`!asyncio.async` function is deprecated in favor of :func:`~asyncio.ensure_future`. The :mod:`!smtpd` module has in the past always decoded the DATA portion of @@ -2314,7 +2314,7 @@ Passing a format string as keyword argument *format_string* to the class has been deprecated. (Contributed by Serhiy Storchaka in :issue:`23671`.) -The :func:`platform.dist` and :func:`platform.linux_distribution` functions +The :func:`!platform.dist` and :func:`!platform.linux_distribution` functions are now deprecated. Linux distributions use too many different ways of describing themselves, so the functionality is left to a package. (Contributed by Vajrasky Kok and Berker Peksag in :issue:`1322`.) @@ -2324,11 +2324,11 @@ The previously undocumented ``from_function`` and ``from_builtin`` methods of :meth:`Signature.from_callable() ` method instead. (Contributed by Yury Selivanov in :issue:`24248`.) -The :func:`inspect.getargspec` function is deprecated and scheduled to be +The :func:`!inspect.getargspec` function is deprecated and scheduled to be removed in Python 3.6. (See :issue:`20438` for details.) The :mod:`inspect` :func:`~inspect.getfullargspec`, -:func:`~inspect.getcallargs`, and :func:`~inspect.formatargspec` functions are +:func:`~inspect.getcallargs`, and :func:`!formatargspec` functions are deprecated in favor of the :func:`inspect.signature` API. (Contributed by Yury Selivanov in :issue:`20438`.) @@ -2405,7 +2405,7 @@ Changes in the Python API error-prone and has been removed in Python 3.5. See :issue:`13936` for full details. -* The :meth:`ssl.SSLSocket.send` method now raises either +* The :meth:`!ssl.SSLSocket.send` method now raises either :exc:`ssl.SSLWantReadError` or :exc:`ssl.SSLWantWriteError` on a non-blocking socket if the operation would block. Previously, it would return ``0``. (Contributed by Nikolaus Rath in :issue:`20951`.) @@ -2440,12 +2440,12 @@ Changes in the Python API :mod:`http.client` and :mod:`http.server` remain available for backwards compatibility. (Contributed by Demian Brecht in :issue:`21793`.) -* When an import loader defines :meth:`importlib.machinery.Loader.exec_module` +* When an import loader defines :meth:`~importlib.abc.Loader.exec_module` it is now expected to also define - :meth:`~importlib.machinery.Loader.create_module` (raises a + :meth:`~importlib.abc.Loader.create_module` (raises a :exc:`DeprecationWarning` now, will be an error in Python 3.6). If the loader inherits from :class:`importlib.abc.Loader` then there is nothing to do, else - simply define :meth:`~importlib.machinery.Loader.create_module` to return + simply define :meth:`~importlib.abc.Loader.create_module` to return ``None``. (Contributed by Brett Cannon in :issue:`23014`.) * The :func:`re.split` function always ignored empty pattern matches, so the @@ -2488,7 +2488,7 @@ Changes in the Python API the POT-Creation-Date header. * The :mod:`smtplib` module now uses :data:`sys.stderr` instead of the previous - module-level :data:`stderr` variable for debug output. If your (test) + module-level :data:`!stderr` variable for debug output. If your (test) program depends on patching the module-level variable to capture the debug output, you will need to update it to capture sys.stderr instead. @@ -2514,11 +2514,11 @@ Changes in the C API -------------------- * The undocumented :c:member:`!format` member of the - (non-public) :c:type:`PyMemoryViewObject` structure has been removed. + (non-public) :c:type:`!PyMemoryViewObject` structure has been removed. All extensions relying on the relevant parts in ``memoryobject.h`` must be rebuilt. -* The :c:type:`PyMemAllocator` structure was renamed to +* The :c:type:`!PyMemAllocator` structure was renamed to :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added. * Removed non-documented macro :c:macro:`!PyObject_REPR()` which leaked references. diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 050c9103b00c98d..b1e3269239d6297 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -745,7 +745,7 @@ Some smaller changes made to the core Python language are: * It is now possible to set a :ref:`special method ` to ``None`` to indicate that the corresponding operation is not available. - For example, if a class sets :meth:`__iter__` to ``None``, the class + For example, if a class sets :meth:`~object.__iter__` to ``None``, the class is not iterable. (Contributed by Andrew Barnert and Ivan Levkivskyi in :issue:`25958`.) @@ -871,7 +871,7 @@ Notable changes in the :mod:`asyncio` module since Python 3.5.0 of the last iteration will be discarded. (Contributed by Guido van Rossum in :issue:`25593`.) -* :meth:`Future.set_exception ` +* :meth:`Future.set_exception ` will now raise :exc:`TypeError` when passed an instance of the :exc:`StopIteration` exception. (Contributed by Chris Angelico in :issue:`26221`.) @@ -925,7 +925,7 @@ added to represent sized iterable container classes. (Contributed by Ivan Levkivskyi, docs by Neil Girdhar in :issue:`27598`.) The new :class:`~collections.abc.Reversible` abstract base class represents -iterable classes that also provide the :meth:`__reversed__` method. +iterable classes that also provide the :meth:`~object.__reversed__` method. (Contributed by Ivan Levkivskyi in :issue:`25987`.) The new :class:`~collections.abc.AsyncGenerator` abstract base class represents @@ -971,7 +971,7 @@ datetime -------- The :class:`~datetime.datetime` and :class:`~datetime.time` classes have -the new :attr:`~time.fold` attribute used to disambiguate local time +the new :attr:`~datetime.time.fold` attribute used to disambiguate local time when necessary. Many functions in the :mod:`datetime` have been updated to support local time disambiguation. See :ref:`Local Time Disambiguation ` section for more @@ -1052,12 +1052,12 @@ enum ---- Two new enumeration base classes have been added to the :mod:`enum` module: -:class:`~enum.Flag` and :class:`~enum.IntFlags`. Both are used to define +:class:`~enum.Flag` and :class:`~enum.IntFlag`. Both are used to define constants that can be combined using the bitwise operators. (Contributed by Ethan Furman in :issue:`23591`.) Many standard library modules have been updated to use the -:class:`~enum.IntFlags` class for their constants. +:class:`~enum.IntFlag` class for their constants. The new :class:`enum.auto` value can be used to assign values to enum members automatically:: @@ -1224,7 +1224,7 @@ generator expression scopes as if they were positional-only parameters called ``implicit0``. (Contributed by Jelle Zijlstra in :issue:`19611`.) To reduce code churn when upgrading from Python 2.7 and the legacy -:func:`inspect.getargspec` API, the previously documented deprecation of +:func:`!inspect.getargspec` API, the previously documented deprecation of :func:`inspect.getfullargspec` has been reversed. While this function is convenient for single/source Python 2/3 code bases, the richer :func:`inspect.signature` interface remains the recommended approach for new @@ -1275,7 +1275,7 @@ See the summary of :ref:`PEP 519 ` for details on how the A new :meth:`~os.scandir.close` method allows explicitly closing a :func:`~os.scandir` iterator. The :func:`~os.scandir` iterator now -supports the :term:`context manager` protocol. If a :func:`scandir` +supports the :term:`context manager` protocol. If a :func:`!scandir` iterator is neither exhausted nor explicitly closed a :exc:`ResourceWarning` will be emitted in its destructor. (Contributed by Serhiy Storchaka in :issue:`25994`.) @@ -1434,7 +1434,7 @@ defined in :mod:`http.server`, :mod:`xmlrpc.server` and protocol. (Contributed by Aviv Palivoda in :issue:`26404`.) -The :attr:`~socketserver.StreamRequestHandler.wfile` attribute of +The :attr:`wfile ` attribute of :class:`~socketserver.StreamRequestHandler` classes now implements the :class:`io.BufferedIOBase` writable interface. In particular, calling :meth:`~io.BufferedIOBase.write` is now guaranteed to send the @@ -1465,7 +1465,7 @@ The new :meth:`~ssl.SSLContext.get_ciphers` method can be used to get a list of enabled ciphers in order of cipher priority. All constants and flags have been converted to :class:`~enum.IntEnum` and -:class:`~enum.IntFlags`. +:class:`~enum.IntFlag`. (Contributed by Christian Heimes in :issue:`28025`.) Server and client-side specific TLS protocols for :class:`~ssl.SSLContext` @@ -1531,8 +1531,8 @@ Stéphane Wirtel in :issue:`25485`). time ---- -The :class:`~time.struct_time` attributes :attr:`tm_gmtoff` and -:attr:`tm_zone` are now available on all platforms. +The :class:`~time.struct_time` attributes :attr:`!tm_gmtoff` and +:attr:`!tm_zone` are now available on all platforms. timeit @@ -1551,12 +1551,12 @@ between best and worst times. tkinter ------- -Added methods :meth:`~tkinter.Variable.trace_add`, -:meth:`~tkinter.Variable.trace_remove` and :meth:`~tkinter.Variable.trace_info` -in the :class:`tkinter.Variable` class. They replace old methods -:meth:`~tkinter.Variable.trace_variable`, :meth:`~tkinter.Variable.trace`, -:meth:`~tkinter.Variable.trace_vdelete` and -:meth:`~tkinter.Variable.trace_vinfo` that use obsolete Tcl commands and might +Added methods :meth:`!Variable.trace_add`, +:meth:`!Variable.trace_remove` and :meth:`!trace_info` +in the :class:`!tkinter.Variable` class. They replace old methods +:meth:`!trace_variable`, :meth:`!trace`, +:meth:`!trace_vdelete` and +:meth:`!trace_vinfo` that use obsolete Tcl commands and might not work in future versions of Tcl. (Contributed by Serhiy Storchaka in :issue:`22115`). @@ -1674,8 +1674,8 @@ urllib.request If a HTTP request has a file or iterable body (other than a bytes object) but no ``Content-Length`` header, rather than -throwing an error, :class:`~urllib.request.AbstractHTTPHandler` now -falls back to use chunked transfer encoding. +throwing an error, :class:`AbstractHTTPHandler ` +now falls back to use chunked transfer encoding. (Contributed by Demian Brecht and Rolf Krahl in :issue:`12319`.) @@ -1701,7 +1701,7 @@ warnings A new optional *source* parameter has been added to the :func:`warnings.warn_explicit` function: the destroyed object which emitted a :exc:`ResourceWarning`. A *source* attribute has also been added to -:class:`warnings.WarningMessage` (contributed by Victor Stinner in +:class:`!warnings.WarningMessage` (contributed by Victor Stinner in :issue:`26568` and :issue:`26567`). When a :exc:`ResourceWarning` warning is logged, the :mod:`tracemalloc` module is now @@ -1942,7 +1942,7 @@ Raising the :exc:`StopIteration` exception inside a generator will now generate a :exc:`DeprecationWarning`, and will trigger a :exc:`RuntimeError` in Python 3.7. See :ref:`whatsnew-pep-479` for details. -The :meth:`__aiter__` method is now expected to return an asynchronous +The :meth:`~object.__aiter__` method is now expected to return an asynchronous iterator directly instead of returning an awaitable as previously. Doing the former will trigger a :exc:`DeprecationWarning`. Backward compatibility will be removed in Python 3.7. @@ -2189,7 +2189,7 @@ Changes in the Python API booleans being a subclass of integers, this should only be an issue if you were doing identity checks for ``1`` or ``0``. See :issue:`25768`. -* Reading the :attr:`~urllib.parse.SplitResult.port` attribute of +* Reading the :attr:`!port` attribute of :func:`urllib.parse.urlsplit` and :func:`~urllib.parse.urlparse` results now raises :exc:`ValueError` for out-of-range values, rather than returning :const:`None`. See :issue:`20059`. @@ -2197,8 +2197,8 @@ Changes in the Python API * The :mod:`!imp` module now raises a :exc:`DeprecationWarning` instead of :exc:`PendingDeprecationWarning`. -* The following modules have had missing APIs added to their :attr:`__all__` - attributes to match the documented APIs: +* The following modules have had missing APIs added to their + :attr:`~module.__all__` attributes to match the documented APIs: :mod:`calendar`, :mod:`!cgi`, :mod:`csv`, :mod:`~xml.etree.ElementTree`, :mod:`enum`, :mod:`fileinput`, :mod:`ftplib`, :mod:`logging`, :mod:`mailbox`, @@ -2253,11 +2253,13 @@ Changes in the Python API * As part of :pep:`487`, the handling of keyword arguments passed to :class:`type` (other than the metaclass hint, ``metaclass``) is now consistently delegated to :meth:`object.__init_subclass__`. This means that - :meth:`type.__new__` and :meth:`type.__init__` both now accept arbitrary - keyword arguments, but :meth:`object.__init_subclass__` (which is called from - :meth:`type.__new__`) will reject them by default. Custom metaclasses - accepting additional keyword arguments will need to adjust their calls to - :meth:`type.__new__` (whether direct or via :class:`super`) accordingly. + :meth:`type.__new__ ` and :meth:`type.__init__ + ` both now accept arbitrary keyword arguments, + but :meth:`object.__init_subclass__` (which is called from + :meth:`type.__new__ `) will reject them by default. + Custom metaclasses accepting additional keyword arguments will need to adjust + their calls to :meth:`type.__new__ ` + (whether direct or via :class:`super`) accordingly. * In ``distutils.command.sdist.sdist``, the ``default_format`` attribute has been removed and is no longer honored. Instead, the @@ -2305,7 +2307,7 @@ Changes in the Python API real-world compatibility. (Contributed by Lita Cho in :issue:`21815`.) -* The :func:`mmap.write() ` function now returns the number +* The :func:`mmap.mmap.write` function now returns the number of bytes written like other write methods. (Contributed by Jakub Stasiak in :issue:`26335`.) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index f420fa5c04479b0..dafc351ea8bbbb5 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -320,9 +320,9 @@ effort will be made to add such support. PEP 562: Customization of Access to Module Attributes ----------------------------------------------------- -Python 3.7 allows defining :meth:`__getattr__` on modules and will call +Python 3.7 allows defining :meth:`~module.__getattr__` on modules and will call it whenever a module attribute is otherwise not found. Defining -:meth:`__dir__` on modules is now also allowed. +:meth:`~module.__dir__` on modules is now also allowed. A typical example of where this may be useful is module attribute deprecation and lazy loading. @@ -409,8 +409,8 @@ PEP 560: Core Support for ``typing`` module and Generic Types Initially :pep:`484` was designed in such way that it would not introduce *any* changes to the core CPython interpreter. Now type hints and the :mod:`typing` module are extensively used by the community, so this restriction is removed. -The PEP introduces two special methods :meth:`__class_getitem__` and -``__mro_entries__``, these methods are now used by most classes and special +The PEP introduces two special methods :meth:`~object.__class_getitem__` and +:meth:`~object.__mro_entries__`, these methods are now used by most classes and special constructs in :mod:`typing`. As a result, the speed of various operations with types increased up to 7 times, the generic types can be used without metaclass conflicts, and several long standing bugs in :mod:`typing` module are @@ -603,8 +603,8 @@ The new :mod:`importlib.resources` module provides several new APIs and one new ABC for access to, opening, and reading *resources* inside packages. Resources are roughly similar to files inside packages, but they needn't be actual files on the physical file system. Module loaders can provide a -:meth:`get_resource_reader` function which returns -a :class:`importlib.abc.ResourceReader` instance to support this +:meth:`!get_resource_reader` function which returns +a :class:`!importlib.abc.ResourceReader` instance to support this new API. Built-in file path loaders and zip file loaders both support this. Contributed by Barry Warsaw and Brett Cannon in :issue:`32248`. @@ -910,9 +910,9 @@ which allows listing the names of properties which should not become enum members. (Contributed by Ethan Furman in :issue:`31801`.) -In Python 3.8, attempting to check for non-Enum objects in :class:`Enum` +In Python 3.8, attempting to check for non-Enum objects in :class:`~enum.Enum` classes will raise a :exc:`TypeError` (e.g. ``1 in Color``); similarly, -attempting to check for non-Flag objects in a :class:`Flag` member will +attempting to check for non-Flag objects in a :class:`~enum.Flag` member will raise :exc:`TypeError` (e.g. ``1 in Perm.RW``); currently, both operations return :const:`False` instead and are deprecated. (Contributed by Ethan Furman in :issue:`33217`.) @@ -969,7 +969,7 @@ uses the current working directory. (Contributed by Stéphane Wirtel and Julien Palard in :issue:`28707`.) The new :class:`ThreadingHTTPServer ` class -uses threads to handle requests using :class:`~socketserver.ThreadingMixin`. +uses threads to handle requests using :class:`~socketserver.ThreadingMixIn`. It is used when ``http.server`` is run with ``-m``. (Contributed by Julien Palard in :issue:`31639`.) @@ -1043,7 +1043,7 @@ window are shown and hidden in the Options menu. importlib --------- -The :class:`importlib.abc.ResourceReader` ABC was introduced to +The :class:`!importlib.abc.ResourceReader` ABC was introduced to support the loading of resources from packages. See also :ref:`whatsnew37_importlib_resources`. (Contributed by Barry Warsaw, Brett Cannon in :issue:`32248`.) @@ -1052,12 +1052,12 @@ support the loading of resources from packages. See also lacks a spec. (Contributed by Garvit Khatri in :issue:`29851`.) -:func:`importlib.find_spec` now raises :exc:`ModuleNotFoundError` instead of +:func:`importlib.util.find_spec` now raises :exc:`ModuleNotFoundError` instead of :exc:`AttributeError` if the specified parent module is not a package (i.e. lacks a ``__path__`` attribute). (Contributed by Milan Oberkirch in :issue:`30436`.) -The new :func:`importlib.source_hash` can be used to compute the hash of +The new :func:`importlib.util.source_hash` can be used to compute the hash of the passed source. A :ref:`hash-based .pyc file ` embeds the value returned by this function. @@ -1148,7 +1148,7 @@ running. (Contributed by Antoine Pitrou in :issue:`30596`.) The new :meth:`Process.kill() ` method can -be used to terminate the process using the :data:`SIGKILL` signal on Unix. +be used to terminate the process using the :data:`~signal.SIGKILL` signal on Unix. (Contributed by Vitor Pereira in :issue:`30794`.) Non-daemonic threads created by :class:`~multiprocessing.Process` are now @@ -1280,9 +1280,10 @@ This function should be used instead of :func:`os.close` for better compatibility across platforms. (Contributed by Christian Heimes in :issue:`32454`.) -The :mod:`socket` module now exposes the :const:`socket.TCP_CONGESTION` -(Linux 2.6.13), :const:`socket.TCP_USER_TIMEOUT` (Linux 2.6.37), and -:const:`socket.TCP_NOTSENT_LOWAT` (Linux 3.12) constants. +The :mod:`socket` module now exposes the :ref:`socket.TCP_CONGESTION +` (Linux 2.6.13), :ref:`socket.TCP_USER_TIMEOUT +` (Linux 2.6.37), and :ref:`socket.TCP_NOTSENT_LOWAT +` (Linux 3.12) constants. (Contributed by Omar Sandoval in :issue:`26273` and Nathaniel J. Smith in :issue:`29728`.) @@ -1298,11 +1299,14 @@ by default. socketserver ------------ -:meth:`socketserver.ThreadingMixIn.server_close` now waits until all non-daemon -threads complete. :meth:`socketserver.ForkingMixIn.server_close` now waits +:meth:`socketserver.ThreadingMixIn.server_close +` now waits until all non-daemon +threads complete. :meth:`socketserver.ForkingMixIn.server_close +` now waits until all child processes complete. -Add a new :attr:`socketserver.ForkingMixIn.block_on_close` class attribute to +Add a new :attr:`socketserver.ForkingMixIn.block_on_close +` class attribute to :class:`socketserver.ForkingMixIn` and :class:`socketserver.ThreadingMixIn` classes. Set the class attribute to ``False`` to get the pre-3.7 behaviour. @@ -1323,7 +1327,7 @@ ssl --- The :mod:`ssl` module now uses OpenSSL's builtin API instead of -:func:`~ssl.match_hostname` to check a host name or an IP address. Values +:func:`!match_hostname` to check a host name or an IP address. Values are validated during TLS handshake. Any certificate validation error including failing the host name check now raises :exc:`~ssl.SSLCertVerificationError` and aborts the handshake with a proper @@ -1341,7 +1345,7 @@ Host name validation can be customized with The ``ssl`` module no longer sends IP addresses in SNI TLS extension. (Contributed by Christian Heimes in :issue:`32185`.) -:func:`~ssl.match_hostname` no longer supports partial wildcards like +:func:`!match_hostname` no longer supports partial wildcards like ``www*.example.org``. (Contributed by Mandeep Singh in :issue:`23033` and Christian Heimes in :issue:`31399`.) @@ -1438,7 +1442,7 @@ The new :func:`sys.get_coroutine_origin_tracking_depth` function returns the current coroutine origin tracking depth, as set by the new :func:`sys.set_coroutine_origin_tracking_depth`. :mod:`asyncio` has been converted to use this new API instead of -the deprecated :func:`sys.set_coroutine_wrapper`. +the deprecated :func:`!sys.set_coroutine_wrapper`. (Contributed by Nathaniel J. Smith in :issue:`32591`.) @@ -1615,7 +1619,7 @@ external entities by default. xml.etree --------- -:ref:`ElementPath ` predicates in the :meth:`find` +:ref:`ElementPath ` predicates in the :meth:`!find` methods can now compare text of the current node with ``[. = "text"]``, not only text in children. Predicates also allow adding spaces for better readability. (Contributed by Stefan Behnel in :issue:`31648`.) @@ -1624,7 +1628,7 @@ better readability. (Contributed by Stefan Behnel in :issue:`31648`.) xmlrpc.server ------------- -:meth:`SimpleXMLRPCDispatcher.register_function ` +:meth:`!SimpleXMLRPCDispatcher.register_function` can now be used as a decorator. (Contributed by Xiang Zhang in :issue:`7769`.) @@ -1682,15 +1686,15 @@ The :mod:`tracemalloc` now exposes a C API through the new functions. (Contributed by Victor Stinner in :issue:`30054`.) -The new :c:func:`import__find__load__start` and -:c:func:`import__find__load__done` static markers can be used to trace -module imports. +The new :ref:`import__find__load__start ` and +:ref:`import__find__load__done ` static markers can be used +to trace module imports. (Contributed by Christian Heimes in :issue:`31574`.) The fields :c:member:`!name` and :c:member:`!doc` of structures :c:type:`PyMemberDef`, :c:type:`PyGetSetDef`, :c:type:`PyStructSequence_Field`, :c:type:`PyStructSequence_Desc`, -and :c:struct:`wrapperbase` are now of type ``const char *`` rather of +and :c:struct:`!wrapperbase` are now of type ``const char *`` rather of ``char *``. (Contributed by Serhiy Storchaka in :issue:`28761`.) The result of :c:func:`PyUnicode_AsUTF8AndSize` and :c:func:`PyUnicode_AsUTF8` @@ -1719,8 +1723,8 @@ Added C API support for timezones with timezone constructors and access to the UTC singleton with :c:data:`PyDateTime_TimeZone_UTC`. Contributed by Paul Ganssle in :issue:`10381`. -The type of results of :c:func:`PyThread_start_new_thread` and -:c:func:`PyThread_get_thread_ident`, and the *id* parameter of +The type of results of :c:func:`!PyThread_start_new_thread` and +:c:func:`!PyThread_get_thread_ident`, and the *id* parameter of :c:func:`PyThreadState_SetAsyncExc` changed from :c:expr:`long` to :c:expr:`unsigned long`. (Contributed by Serhiy Storchaka in :issue:`6532`.) @@ -1847,8 +1851,8 @@ make the creation of named tuples 4 to 6 times faster. (Contributed by Jelle Zijlstra with further improvements by INADA Naoki, Serhiy Storchaka, and Raymond Hettinger in :issue:`28638`.) -:meth:`date.fromordinal` and :meth:`date.fromtimestamp` are now up to -30% faster in the common case. +:meth:`datetime.date.fromordinal` and :meth:`datetime.date.fromtimestamp` +are now up to 30% faster in the common case. (Contributed by Paul Ganssle in :issue:`32403`.) The :func:`os.fwalk` function is now up to 2 times faster thanks to @@ -1997,9 +2001,9 @@ modes (this will be an error in future Python releases). enum ---- -In Python 3.8, attempting to check for non-Enum objects in :class:`Enum` +In Python 3.8, attempting to check for non-Enum objects in :class:`~enum.Enum` classes will raise a :exc:`TypeError` (e.g. ``1 in Color``); similarly, -attempting to check for non-Flag objects in a :class:`Flag` member will +attempting to check for non-Flag objects in a :class:`~enum.Flag` member will raise :exc:`TypeError` (e.g. ``1 in Perm.RW``); currently, both operations return :const:`False` instead. (Contributed by Ethan Furman in :issue:`33217`.) @@ -2028,20 +2032,20 @@ both deprecated in Python 3.4 now emit :exc:`DeprecationWarning`. (Contributed by Matthias Bussonnier in :issue:`29576`.) The :class:`importlib.abc.ResourceLoader` ABC has been deprecated in -favour of :class:`importlib.abc.ResourceReader`. +favour of :class:`!importlib.abc.ResourceReader`. locale ------ -:func:`locale.format` has been deprecated, use :meth:`locale.format_string` +:func:`!locale.format` has been deprecated, use :meth:`locale.format_string` instead. (Contributed by Garvit in :issue:`10379`.) macpath ------- -The :mod:`macpath` is now deprecated and will be removed in Python 3.8. +The :mod:`!macpath` is now deprecated and will be removed in Python 3.8. (Contributed by Chi Hsuan Yen in :issue:`9850`.) @@ -2066,7 +2070,7 @@ if the passed argument is larger than 16 bits, an exception will be raised. ssl --- -:func:`ssl.wrap_socket` is deprecated. Use +:func:`!ssl.wrap_socket` is deprecated. Use :meth:`ssl.SSLContext.wrap_socket` instead. (Contributed by Christian Heimes in :issue:`28124`.) @@ -2082,8 +2086,8 @@ Use :func:`!sunau.open` instead. sys --- -Deprecated :func:`sys.set_coroutine_wrapper` and -:func:`sys.get_coroutine_wrapper`. +Deprecated :func:`!sys.set_coroutine_wrapper` and +:func:`!sys.get_coroutine_wrapper`. The undocumented ``sys.callstats()`` function has been deprecated and will be removed in a future Python version. @@ -2093,7 +2097,7 @@ will be removed in a future Python version. wave ---- -:func:`wave.openfp` has been deprecated and will be removed in Python 3.9. +:func:`!wave.openfp` has been deprecated and will be removed in Python 3.9. Use :func:`wave.open` instead. (Contributed by Brian Curtin in :issue:`31985`.) @@ -2173,8 +2177,8 @@ The following features and APIs have been removed from Python 3.7: * Removed previously deprecated in Python 2.4 classes ``Plist``, ``Dict`` and ``_InternalDict`` in the :mod:`plistlib` module. Dict values in the result - of functions :func:`~plistlib.readPlist` and - :func:`~plistlib.readPlistFromBytes` are now normal dicts. You no longer + of functions :func:`!readPlist` and + :func:`!readPlistFromBytes` are now normal dicts. You no longer can use attribute access to access items of these dictionaries. * The ``asyncio.windows_utils.socketpair()`` function has been @@ -2191,7 +2195,7 @@ The following features and APIs have been removed from Python 3.7: * Direct instantiation of :class:`ssl.SSLSocket` and :class:`ssl.SSLObject` objects is now prohibited. The constructors were never documented, tested, or designed as public constructors. Users were supposed to use - :func:`ssl.wrap_socket` or :class:`ssl.SSLContext`. + :func:`!ssl.wrap_socket` or :class:`ssl.SSLContext`. (Contributed by Christian Heimes in :issue:`32951`.) * The unused ``distutils`` ``install_misc`` command has been removed. @@ -2275,15 +2279,18 @@ Changes in Python Behavior Changes in the Python API ------------------------- -* :meth:`socketserver.ThreadingMixIn.server_close` now waits until all +* :meth:`socketserver.ThreadingMixIn.server_close + ` now waits until all non-daemon threads complete. Set the new :attr:`socketserver.ThreadingMixIn.block_on_close` class attribute to ``False`` to get the pre-3.7 behaviour. (Contributed by Victor Stinner in :issue:`31233` and :issue:`33540`.) -* :meth:`socketserver.ForkingMixIn.server_close` now waits until all +* :meth:`socketserver.ForkingMixIn.server_close + ` now waits until all child processes complete. Set the new - :attr:`socketserver.ForkingMixIn.block_on_close` class attribute to ``False`` + :attr:`socketserver.ForkingMixIn.block_on_close + ` class attribute to ``False`` to get the pre-3.7 behaviour. (Contributed by Victor Stinner in :issue:`31151` and :issue:`33540`.) @@ -2476,7 +2483,7 @@ avoiding possible problems use new functions :c:func:`PySlice_Unpack` and CPython bytecode changes ------------------------ -There are two new opcodes: :opcode:`LOAD_METHOD` and :opcode:`!CALL_METHOD`. +There are two new opcodes: :opcode:`!LOAD_METHOD` and :opcode:`!CALL_METHOD`. (Contributed by Yury Selivanov and INADA Naoki in :issue:`26110`.) The :opcode:`!STORE_ANNOTATION` opcode has been removed. diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index 7aca35b2959cd2a..d1062c0a402e816 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -50,7 +50,6 @@ For full details, see the :ref:`changelog `. .. testsetup:: - from datetime import date from math import cos, radians from unicodedata import normalize import re @@ -208,14 +207,15 @@ subdirectories). Debug build uses the same ABI as release build ----------------------------------------------- -Python now uses the same ABI whether it's built in release or debug mode. On -Unix, when Python is built in debug mode, it is now possible to load C -extensions built in release mode and C extensions built using the stable ABI. +The ABI of Python :ref:`debug builds ` is now compatible with +Python release builds. On Unix, when Python is built in debug mode, it is now +possible to load C extensions built in release mode and C extensions built +using the stable ABI. The inverse is not true, as debug builds expose +additional symbols not available in release builds. -Release builds and :ref:`debug builds ` are now ABI compatible: defining the -``Py_DEBUG`` macro no longer implies the ``Py_TRACE_REFS`` macro, which -introduces the only ABI incompatibility. The ``Py_TRACE_REFS`` macro, which -adds the :func:`sys.getobjects` function and the :envvar:`PYTHONDUMPREFS` +Defining the ``Py_DEBUG`` macro no longer implies the ``Py_TRACE_REFS`` macro, +which introduces the only ABI incompatibility. The ``Py_TRACE_REFS`` macro, +which adds the :func:`sys.getobjects` function and the :envvar:`PYTHONDUMPREFS` environment variable, can be set using the new :option:`./configure --with-trace-refs <--with-trace-refs>` build option. (Contributed by Victor Stinner in :issue:`36465`.) @@ -259,15 +259,16 @@ Added an ``=`` specifier to :term:`f-string`\s. An f-string such as ``f'{expr=}'`` will expand to the text of the expression, an equal sign, then the representation of the evaluated expression. For example: + >>> import datetime as dt >>> user = 'eric_idle' - >>> member_since = date(1975, 7, 31) + >>> member_since = dt.date(1975, 7, 31) >>> f'{user=} {member_since=}' "user='eric_idle' member_since=datetime.date(1975, 7, 31)" The usual :ref:`f-string format specifiers ` allow more control over how the result of the expression is displayed:: - >>> delta = date.today() - member_since + >>> delta = dt.date.today() - member_since >>> f'{user=!s} {delta.days=:,d}' 'user=eric_idle delta.days=16,075' @@ -536,7 +537,7 @@ Other Language Changes element is a callable with a ``(obj, state)`` signature. This allows the direct control over the state-updating behavior of a specific object. If not *None*, this callable will have priority over the object's - :meth:`~__setstate__` method. + :meth:`~object.__setstate__` method. (Contributed by Pierre Glaser and Olivier Grisel in :issue:`35900`.) New Modules @@ -1035,9 +1036,9 @@ symlinks and directory junctions) has been delegated to the operating system. Specifically, :func:`os.stat` will now traverse anything supported by the operating system, while :func:`os.lstat` will only open reparse points that identify as "name surrogates" while others are opened as for :func:`os.stat`. -In all cases, :attr:`stat_result.st_mode` will only have ``S_IFLNK`` set for +In all cases, :attr:`os.stat_result.st_mode` will only have ``S_IFLNK`` set for symbolic links and not other kinds of reparse points. To identify other kinds -of reparse point, check the new :attr:`stat_result.st_reparse_tag` attribute. +of reparse point, check the new :attr:`os.stat_result.st_reparse_tag` attribute. On Windows, :func:`os.readlink` is now able to read directory junctions. Note that :func:`~os.path.islink` will return ``False`` for directory junctions, @@ -1285,20 +1286,20 @@ now matches what the C tokenizer does internally. tkinter ------- -Added methods :meth:`~tkinter.Spinbox.selection_from`, -:meth:`~tkinter.Spinbox.selection_present`, -:meth:`~tkinter.Spinbox.selection_range` and -:meth:`~tkinter.Spinbox.selection_to` -in the :class:`tkinter.Spinbox` class. +Added methods :meth:`!selection_from`, +:meth:`!selection_present`, +:meth:`!selection_range` and +:meth:`!selection_to` +in the :class:`!tkinter.Spinbox` class. (Contributed by Juliette Monsel in :issue:`34829`.) -Added method :meth:`~tkinter.Canvas.moveto` -in the :class:`tkinter.Canvas` class. +Added method :meth:`!moveto` +in the :class:`!tkinter.Canvas` class. (Contributed by Juliette Monsel in :issue:`23831`.) -The :class:`tkinter.PhotoImage` class now has -:meth:`~tkinter.PhotoImage.transparency_get` and -:meth:`~tkinter.PhotoImage.transparency_set` methods. (Contributed by +The :class:`!tkinter.PhotoImage` class now has +:meth:`!transparency_get` and +:meth:`!transparency_set` methods. (Contributed by Zackery Spytz in :issue:`25451`.) @@ -1432,7 +1433,7 @@ and ``{namespace}*`` which returns all tags in the given namespace. (Contributed by Stefan Behnel in :issue:`28238`.) The :mod:`xml.etree.ElementTree` module provides a new function -:func:`–xml.etree.ElementTree.canonicalize` that implements C14N 2.0. +:func:`~xml.etree.ElementTree.canonicalize` that implements C14N 2.0. (Contributed by Stefan Behnel in :issue:`13611`.) The target object of :class:`xml.etree.ElementTree.XMLParser` can @@ -1573,7 +1574,7 @@ Build and C API Changes * :c:func:`Py_INCREF`, :c:func:`Py_DECREF` * :c:func:`Py_XINCREF`, :c:func:`Py_XDECREF` - * :c:func:`PyObject_INIT`, :c:func:`PyObject_INIT_VAR` + * :c:macro:`!PyObject_INIT`, :c:macro:`!PyObject_INIT_VAR` * Private functions: :c:func:`!_PyObject_GC_TRACK`, :c:func:`!_PyObject_GC_UNTRACK`, :c:func:`!_Py_Dealloc` @@ -1677,7 +1678,7 @@ Deprecated constant nodes. (Contributed by Serhiy Storchaka in :issue:`36917`.) -* The :func:`asyncio.coroutine` :term:`decorator` is deprecated and will be +* The :deco:`!asyncio.coroutine` :term:`decorator` is deprecated and will be removed in version 3.10. Instead of ``@asyncio.coroutine``, use :keyword:`async def` instead. (Contributed by Andrew Svetlov in :issue:`36921`.) @@ -1697,22 +1698,22 @@ Deprecated (Contributed by Yury Selivanov in :issue:`34790`.) * The following functions and methods are deprecated in the :mod:`gettext` - module: :func:`~gettext.lgettext`, :func:`~gettext.ldgettext`, - :func:`~gettext.lngettext` and :func:`~gettext.ldngettext`. + module: :func:`!lgettext`, :func:`!ldgettext`, + :func:`!lngettext` and :func:`!ldngettext`. They return encoded bytes, and it's possible that you will get unexpected Unicode-related exceptions if there are encoding problems with the translated strings. It's much better to use alternatives which return Unicode strings in Python 3. These functions have been broken for a long time. - Function :func:`~gettext.bind_textdomain_codeset`, methods - :meth:`~gettext.NullTranslations.output_charset` and - :meth:`~gettext.NullTranslations.set_output_charset`, and the *codeset* + Function :func:`!bind_textdomain_codeset`, methods + :meth:`!NullTranslations.output_charset` and + :meth:`!NullTranslations.set_output_charset`, and the *codeset* parameter of functions :func:`~gettext.translation` and :func:`~gettext.install` are also deprecated, since they are only used for the ``l*gettext()`` functions. (Contributed by Serhiy Storchaka in :issue:`33710`.) -* The :meth:`~threading.Thread.isAlive` method of :class:`threading.Thread` +* The :meth:`!isAlive` method of :class:`threading.Thread` has been deprecated. (Contributed by Donghee Na in :issue:`35283`.) @@ -1727,7 +1728,7 @@ Deprecated * Deprecated passing the following arguments as keyword arguments: - *func* in :func:`functools.partialmethod`, :func:`weakref.finalize`, - :meth:`profile.Profile.runcall`, :meth:`cProfile.Profile.runcall`, + :meth:`profile.Profile.runcall`, :meth:`!cProfile.Profile.runcall`, :meth:`bdb.Bdb.runcall`, :meth:`trace.Trace.runfunc` and :func:`curses.wrapper`. - *function* in :meth:`unittest.TestCase.addCleanup`. @@ -1735,11 +1736,11 @@ Deprecated :class:`concurrent.futures.ThreadPoolExecutor` and :class:`concurrent.futures.ProcessPoolExecutor`. - *callback* in :meth:`contextlib.ExitStack.callback`, - :meth:`contextlib.AsyncExitStack.callback` and + :meth:`!contextlib.AsyncExitStack.callback` and :meth:`contextlib.AsyncExitStack.push_async_callback`. - - *c* and *typeid* in the :meth:`~multiprocessing.managers.Server.create` - method of :class:`multiprocessing.managers.Server` and - :class:`multiprocessing.managers.SharedMemoryServer`. + - *c* and *typeid* in the :meth:`!create` + method of :class:`!multiprocessing.managers.Server` and + :class:`!multiprocessing.managers.SharedMemoryServer`. - *obj* in :func:`weakref.finalize`. In future releases of Python, they will be :ref:`positional-only @@ -1757,14 +1758,14 @@ The following features and APIs have been removed from Python 3.8: able to import from collections was marked for removal in 3.8, but has been delayed to 3.9. (See :gh:`81134`.) -* The :mod:`macpath` module, deprecated in Python 3.7, has been removed. +* The :mod:`!macpath` module, deprecated in Python 3.7, has been removed. (Contributed by Victor Stinner in :issue:`35471`.) -* The function :func:`platform.popen` has been removed, after having been +* The function :func:`!platform.popen` has been removed, after having been deprecated since Python 3.3: use :func:`os.popen` instead. (Contributed by Victor Stinner in :issue:`35345`.) -* The function :func:`time.clock` has been removed, after having been +* The function :func:`!time.clock` has been removed, after having been deprecated since Python 3.3: use :func:`time.perf_counter` or :func:`time.process_time` instead, depending on your requirements, to have well-defined behavior. @@ -1800,8 +1801,8 @@ The following features and APIs have been removed from Python 3.8: :func:`fileinput.FileInput` which was ignored and deprecated since Python 3.6 has been removed. :issue:`36952` (Contributed by Matthias Bussonnier.) -* The functions :func:`sys.set_coroutine_wrapper` and - :func:`sys.get_coroutine_wrapper` deprecated in Python 3.7 have been removed; +* The functions :func:`!sys.set_coroutine_wrapper` and + :func:`!sys.get_coroutine_wrapper` deprecated in Python 3.7 have been removed; :issue:`36933` (Contributed by Matthias Bussonnier.) @@ -1864,9 +1865,10 @@ Changes in the Python API * :class:`subprocess.Popen` can now use :func:`os.posix_spawn` in some cases for better performance. On Windows Subsystem for Linux and QEMU User - Emulation, the :class:`Popen` constructor using :func:`os.posix_spawn` no longer raises an - exception on errors like "missing program". Instead the child process fails with a - non-zero :attr:`~Popen.returncode`. + Emulation, the :class:`~subprocess.Popen` constructor using + :func:`os.posix_spawn` no longer raises an exception on errors like + "missing program". Instead the child process fails with a + non-zero :attr:`~subprocess.Popen.returncode`. (Contributed by Joannah Nanjekye and Victor Stinner in :issue:`35537`.) * The *preexec_fn* argument of * :class:`subprocess.Popen` is no longer @@ -1875,11 +1877,11 @@ Changes in the Python API (Contributed by Eric Snow in :issue:`34651`, modified by Christian Heimes in :issue:`37951`.) -* The :meth:`imap.IMAP4.logout` method no longer silently ignores arbitrary +* The :meth:`imaplib.IMAP4.logout` method no longer silently ignores arbitrary exceptions. (Contributed by Victor Stinner in :issue:`36348`.) -* The function :func:`platform.popen` has been removed, after having been deprecated since +* The function :func:`!platform.popen` has been removed, after having been deprecated since Python 3.3: use :func:`os.popen` instead. (Contributed by Victor Stinner in :issue:`35345`.) @@ -1894,9 +1896,11 @@ Changes in the Python API specialized methods like :meth:`~tkinter.ttk.Treeview.selection_set` for changing the selection. (Contributed by Serhiy Storchaka in :issue:`31508`.) -* The :meth:`writexml`, :meth:`toxml` and :meth:`toprettyxml` methods of - :mod:`xml.dom.minidom`, and the :meth:`write` method of :mod:`xml.etree`, - now preserve the attribute order specified by the user. +* The :meth:`~xml.dom.minidom.Node.writexml`, :meth:`~xml.dom.minidom.Node.toxml` + and :meth:`~xml.dom.minidom.Node.toprettyxml` methods of + :mod:`xml.dom.minidom` and the :meth:`~xml.etree.ElementTree.ElementTree.write` + method of :mod:`xml.etree.ElementTree` now preserve the attribute order + specified by the user. (Contributed by Diego Rojas and Raymond Hettinger in :issue:`34160`.) * A :mod:`dbm.dumb` database opened with flags ``'r'`` is now read-only. @@ -1916,8 +1920,8 @@ Changes in the Python API ``type.__new__``. A :exc:`DeprecationWarning` was emitted in Python 3.6--3.7. (Contributed by Serhiy Storchaka in :issue:`23722`.) -* The :class:`cProfile.Profile` class can now be used as a context - manager. (Contributed by Scott Sanderson in :issue:`29235`.) +* The :class:`cProfile.Profile ` class can now be used as + a context manager. (Contributed by Scott Sanderson in :issue:`29235`.) * :func:`shutil.copyfile`, :func:`shutil.copy`, :func:`shutil.copy2`, :func:`shutil.copytree` and :func:`shutil.move` use platform-specific @@ -1952,7 +1956,7 @@ Changes in the Python API (Contributed by Christian Heimes in :issue:`17239`.) * Deleting a key from a read-only :mod:`dbm` database (:mod:`dbm.dumb`, - :mod:`dbm.gnu` or :mod:`dbm.ndbm`) raises :attr:`error` (:exc:`dbm.dumb.error`, + :mod:`dbm.gnu` or :mod:`dbm.ndbm`) raises :attr:`!error` (:exc:`dbm.dumb.error`, :exc:`dbm.gnu.error` or :exc:`dbm.ndbm.error`) instead of :exc:`KeyError`. (Contributed by Xiang Zhang in :issue:`33106`.) @@ -2044,7 +2048,7 @@ Changes in the C API :c:func:`PyType_FromSpec`) hold a reference to their type object. Increasing the reference count of these type objects has been moved from :c:func:`PyType_GenericAlloc` to the more low-level functions, - :c:func:`PyObject_Init` and :c:func:`PyObject_INIT`. + :c:func:`PyObject_Init` and :c:macro:`!PyObject_INIT`. This makes types created through :c:func:`PyType_FromSpec` behave like other classes in managed code. @@ -2064,7 +2068,7 @@ Changes in the C API This may happen after calling :c:macro:`PyObject_New`, :c:macro:`PyObject_NewVar`, :c:func:`PyObject_GC_New`, :c:func:`PyObject_GC_NewVar`, or any other custom allocator that uses - :c:func:`PyObject_Init` or :c:func:`PyObject_INIT`. + :c:func:`PyObject_Init` or :c:macro:`!PyObject_INIT`. Example: diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 896e8f4a489649c..49a52b7504bc951 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -282,20 +282,20 @@ the standard library. It adds :class:`zoneinfo.ZoneInfo`, a concrete Example:: >>> from zoneinfo import ZoneInfo - >>> from datetime import datetime, timedelta + >>> import datetime as dt >>> # Daylight saving time - >>> dt = datetime(2020, 10, 31, 12, tzinfo=ZoneInfo("America/Los_Angeles")) - >>> print(dt) + >>> when = dt.datetime(2020, 10, 31, 12, tzinfo=ZoneInfo("America/Los_Angeles")) + >>> print(when) 2020-10-31 12:00:00-07:00 - >>> dt.tzname() + >>> when.tzname() 'PDT' >>> # Standard time - >>> dt += timedelta(days=7) - >>> print(dt) + >>> when += dt.timedelta(days=7) + >>> print(when) 2020-11-07 12:00:00-08:00 - >>> print(dt.tzname()) + >>> print(when.tzname()) PST @@ -423,8 +423,8 @@ digests. It skips MD5 on platforms that block MD5 digest. fcntl ----- -Added constants :const:`~fcntl.F_OFD_GETLK`, :const:`~fcntl.F_OFD_SETLK` -and :const:`~fcntl.F_OFD_SETLKW`. +Added constants :const:`!fcntl.F_OFD_GETLK`, :const:`!fcntl.F_OFD_SETLK` +and :const:`!fcntl.F_OFD_SETLKW`. (Contributed by Donghee Na in :issue:`38602`.) ftplib @@ -644,7 +644,7 @@ attribute. random ------ -Added a new :attr:`random.Random.randbytes` method: generate random bytes. +Added a new :meth:`random.Random.randbytes` method: generate random bytes. (Contributed by Victor Stinner in :issue:`40286`.) signal @@ -776,7 +776,7 @@ Optimizations :pep:`590` vectorcall protocol. (Contributed by Donghee Na, Mark Shannon, Jeroen Demeyer and Petr Viktorin in :issue:`37207`.) -* Optimized :func:`~set.difference_update` for the case when the other set +* Optimized :meth:`!set.difference_update` for the case when the other set is much larger than the base set. (Suggested by Evgeny Kapun with code contributed by Michele Orrù in :issue:`8425`.) diff --git a/Grammar/Tokens b/Grammar/Tokens index e40a4437afb009d..0547e6ed08f79ae 100644 --- a/Grammar/Tokens +++ b/Grammar/Tokens @@ -1,3 +1,8 @@ +# When adding new tokens, remember to update the PEG generator in +# Tools/peg_generator/pegen/parser_generator.py +# This will ensure that older versions of Python can generate a Python parser +# using "python -m pegen python ". + ENDMARKER NAME NUMBER diff --git a/Grammar/python.gram b/Grammar/python.gram index f3ef990923eec3a..7b27b04bf7989ef 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -94,14 +94,16 @@ func_type[mod_ty]: '(' a=[type_expressions] ')' '->' b=expression NEWLINE* ENDMA # GENERAL STATEMENTS # ================== -statements[asdl_stmt_seq*]: a=statement+ { _PyPegen_register_stmts(p, (asdl_stmt_seq*)_PyPegen_seq_flatten(p, a)) } +statements[asdl_stmt_seq*]: a=statement+ { (asdl_stmt_seq*)_PyPegen_seq_flatten(p, a) } -statement[asdl_stmt_seq*]: - | a=compound_stmt { (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) } +statement[asdl_stmt_seq*]: + | a=compound_stmt { _PyPegen_register_stmts(p , + (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a) + ) } | a[asdl_stmt_seq*]=simple_stmts { a } single_compound_stmt[asdl_stmt_seq*]: - | a=compound_stmt { + | a=compound_stmt { _PyPegen_register_stmts(p, (asdl_stmt_seq*)_PyPegen_singleton_seq(p, a)) } statement_newline[asdl_stmt_seq*]: @@ -449,9 +451,9 @@ except_block[excepthandler_ty]: _PyAST_ExceptHandler(e, ((expr_ty) t)->v.Name.id, b, EXTRA) } | 'except' e=expressions ':' b=block { CHECK_VERSION( - excepthandler_ty, - 14, - "except expressions without parentheses are", + excepthandler_ty, + 14, + "except expressions without parentheses are", _PyAST_ExceptHandler(e, NULL, b, EXTRA)) } | 'except' ':' b=block { _PyAST_ExceptHandler(NULL, NULL, b, EXTRA) } | invalid_except_stmt @@ -463,9 +465,9 @@ except_star_block[excepthandler_ty]: _PyAST_ExceptHandler(e, ((expr_ty) t)->v.Name.id, b, EXTRA) } | 'except' '*' e=expressions ':' b=block { CHECK_VERSION( - excepthandler_ty, - 14, - "except expressions without parentheses are", + excepthandler_ty, + 14, + "except expressions without parentheses are", _PyAST_ExceptHandler(e, NULL, b, EXTRA)) } | invalid_except_star_stmt finally_block[asdl_stmt_seq*]: @@ -977,15 +979,18 @@ tstring_middle[expr_ty]: | tstring_replacement_field | t=TSTRING_MIDDLE { _PyPegen_constant_from_token(p, t) } tstring[expr_ty] (memo): - | a=TSTRING_START b=tstring_middle* c=TSTRING_END { + | a=TSTRING_START b=tstring_middle* c=TSTRING_END { CHECK_VERSION( - expr_ty, - 14, - "t-strings are", + expr_ty, + 14, + "t-strings are", _PyPegen_template_str(p, a, (asdl_expr_seq*)b, c)) } string[expr_ty]: s[Token*]=STRING { _PyPegen_constant_from_string(p, s) } -strings[expr_ty] (memo): a[asdl_expr_seq*]=(fstring|string|tstring)+ { _PyPegen_concatenate_strings(p, a, EXTRA) } +strings[expr_ty] (memo): + | invalid_string_tstring_concat + | a[asdl_expr_seq*]=(fstring|string)+ { _PyPegen_concatenate_strings(p, a, EXTRA) } + | a[asdl_expr_seq*]=tstring+ { _PyPegen_concatenate_tstrings(p, a, EXTRA) } list[expr_ty]: | '[' a=[star_named_expressions] ']' { _PyAST_List(a, Load, EXTRA) } @@ -1382,12 +1387,12 @@ invalid_import: | 'import' token=NEWLINE { RAISE_SYNTAX_ERROR_STARTING_FROM(token, "Expected one or more names after 'import'") } invalid_dotted_as_name: - | dotted_name 'as' !(NAME (',' | ')' | NEWLINE)) a=expression { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, + | dotted_name 'as' !(NAME (',' | ')' | ';' | NEWLINE)) a=expression { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use %s as import target", _PyPegen_get_expr_name(a)) } invalid_import_from_as_name: - | NAME 'as' !(NAME (',' | ')' | NEWLINE)) a=expression { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, + | NAME 'as' !(NAME (',' | ')' | ';' | NEWLINE)) a=expression { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot use %s as import target", _PyPegen_get_expr_name(a)) } invalid_import_from_targets: @@ -1418,7 +1423,7 @@ invalid_except_stmt: RAISE_SYNTAX_ERROR_STARTING_FROM(a, "multiple exception types must be parenthesized when using 'as'") } | a='except' expression ['as' NAME ] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } | a='except' NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } - | 'except' expression 'as' a=expression { + | 'except' expression 'as' a=expression ':' block { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( a, "cannot use except statement with %s", _PyPegen_get_expr_name(a)) } invalid_except_star_stmt: @@ -1426,7 +1431,7 @@ invalid_except_star_stmt: RAISE_SYNTAX_ERROR_STARTING_FROM(a, "multiple exception types must be parenthesized when using 'as'") } | a='except' '*' expression ['as' NAME ] NEWLINE { RAISE_SYNTAX_ERROR("expected ':'") } | a='except' '*' (NEWLINE | ':') { RAISE_SYNTAX_ERROR("expected one or more exception types") } - | 'except' '*' expression 'as' a=expression { + | 'except' '*' expression 'as' a=expression ':' block { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( a, "cannot use except* statement with %s", _PyPegen_get_expr_name(a)) } invalid_finally_stmt: @@ -1546,6 +1551,12 @@ invalid_tstring_conversion_character: | '!' &(':' | '}') { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: missing conversion character") } | '!' !NAME { RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN("t-string: invalid conversion character") } +invalid_string_tstring_concat: + | a=(fstring|string)+ b[expr_ty]=tstring { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") } + | a=tstring+ b[expr_ty]=(fstring|string) { + RAISE_SYNTAX_ERROR_KNOWN_RANGE(PyPegen_last_item(a, expr_ty), b, "cannot mix t-string literals with string or bytes literals") } + invalid_arithmetic: | sum ('+'|'-'|'*'|'/'|'%'|'//'|'@') a='not' b=inversion { RAISE_SYNTAX_ERROR_KNOWN_RANGE(a, b, "'not' after an operator must be parenthesized") } invalid_factor: diff --git a/Include/Python.h b/Include/Python.h index 64be80145890a36..db8165e4b2e8583 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -16,6 +16,7 @@ // Include standard header files +// When changing these files, remember to update Doc/extending/extending.rst. #include // assert() #include // uintptr_t #include // INT_MAX @@ -59,6 +60,14 @@ # include // __readgsqword() #endif +// Suppress known warnings in Python header files. +#if defined(_MSC_VER) +// Warning that alignas behaviour has changed. Doesn't affect us, because we +// never relied on the old behaviour. +#pragma warning(push) +#pragma warning(disable: 5274) +#endif + // Include Python header files #include "pyport.h" #include "pymacro.h" @@ -138,4 +147,9 @@ #include "cpython/pyfpe.h" #include "cpython/tracemalloc.h" +// Restore warning filter +#ifdef _MSC_VER +#pragma warning(pop) +#endif + #endif /* !Py_PYTHON_H */ diff --git a/Include/abstract.h b/Include/abstract.h index b9199fc03a399a8..80f3298701d249e 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -138,7 +138,12 @@ extern "C" { Delete attribute named attr_name, for object o. Returns -1 on failure. - This is the equivalent of the Python statement: del o.attr_name. */ + This is the equivalent of the Python statement: del o.attr_name. + + Implemented as a macro in the limited C API 3.12 and older. */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030d0000 +# define PyObject_DelAttrString(O, A) PyObject_SetAttrString((O), (A), NULL) +#endif /* Implemented elsewhere: @@ -147,7 +152,12 @@ extern "C" { Delete attribute named attr_name, for object o. Returns -1 on failure. This is the equivalent of the Python - statement: del o.attr_name. */ + statement: del o.attr_name. + + Implemented as a macro in the limited C API 3.12 and older. */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030d0000 +# define PyObject_DelAttr(O, A) PyObject_SetAttr((O), (A), NULL) +#endif /* Implemented elsewhere: diff --git a/Include/audit.h b/Include/audit.h index 793b7077e1027ba..9be54ad44110963 100644 --- a/Include/audit.h +++ b/Include/audit.h @@ -1,5 +1,5 @@ -#ifndef Py_AUDIT_H -#define Py_AUDIT_H +#ifndef _Py_AUDIT_H +#define _Py_AUDIT_H #ifdef __cplusplus extern "C" { #endif @@ -18,13 +18,13 @@ PyAPI_FUNC(int) PySys_AuditTuple( #ifndef Py_LIMITED_API -# define Py_CPYTHON_AUDIT_H +# define _Py_CPYTHON_AUDIT_H # include "cpython/audit.h" -# undef Py_CPYTHON_AUDIT_H +# undef _Py_CPYTHON_AUDIT_H #endif #ifdef __cplusplus } #endif -#endif /* !Py_AUDIT_H */ +#endif /* !_Py_AUDIT_H */ diff --git a/Include/boolobject.h b/Include/boolobject.h index 3037e61bbf6d0c7..b56e2baecaa36c5 100644 --- a/Include/boolobject.h +++ b/Include/boolobject.h @@ -34,9 +34,16 @@ PyAPI_FUNC(int) Py_IsTrue(PyObject *x); PyAPI_FUNC(int) Py_IsFalse(PyObject *x); #define Py_IsFalse(x) Py_Is((x), Py_False) -/* Macros for returning Py_True or Py_False, respectively */ -#define Py_RETURN_TRUE return Py_True -#define Py_RETURN_FALSE return Py_False +/* Macros for returning Py_True or Py_False, respectively. + * Only treat Py_True and Py_False as immortal in the limited C API 3.12 + * and newer. */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030c0000 +# define Py_RETURN_TRUE return Py_NewRef(Py_True) +# define Py_RETURN_FALSE return Py_NewRef(Py_False) +#else +# define Py_RETURN_TRUE return Py_True +# define Py_RETURN_FALSE return Py_False +#endif /* Function to return a bool from a C long */ PyAPI_FUNC(PyObject *) PyBool_FromLong(long); diff --git a/Include/ceval.h b/Include/ceval.h index 32ab38972e548f3..e9df8684996e23f 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -133,13 +133,6 @@ PyAPI_FUNC(void) PyEval_ReleaseThread(PyThreadState *tstate); #define FVS_MASK 0x4 #define FVS_HAVE_SPEC 0x4 -/* Special methods used by LOAD_SPECIAL */ -#define SPECIAL___ENTER__ 0 -#define SPECIAL___EXIT__ 1 -#define SPECIAL___AENTER__ 2 -#define SPECIAL___AEXIT__ 3 -#define SPECIAL_MAX 3 - #ifndef Py_LIMITED_API # define Py_CPYTHON_CEVAL_H # include "cpython/ceval.h" diff --git a/Include/cpython/audit.h b/Include/cpython/audit.h index 3c5c7a8c06091d0..536f92486320974 100644 --- a/Include/cpython/audit.h +++ b/Include/cpython/audit.h @@ -1,4 +1,4 @@ -#ifndef Py_CPYTHON_AUDIT_H +#ifndef _Py_CPYTHON_AUDIT_H # error "this header file must not be included directly" #endif diff --git a/Include/cpython/critical_section.h b/Include/cpython/critical_section.h index 35db3fb6a59ce66..4fc46fefb93a24a 100644 --- a/Include/cpython/critical_section.h +++ b/Include/cpython/critical_section.h @@ -73,22 +73,32 @@ typedef struct PyCriticalSection2 PyCriticalSection2; PyAPI_FUNC(void) PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op); +PyAPI_FUNC(void) +PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m); + PyAPI_FUNC(void) PyCriticalSection_End(PyCriticalSection *c); PyAPI_FUNC(void) PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b); +PyAPI_FUNC(void) +PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2); + PyAPI_FUNC(void) PyCriticalSection2_End(PyCriticalSection2 *c); #ifndef Py_GIL_DISABLED # define Py_BEGIN_CRITICAL_SECTION(op) \ { +# define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex) \ + { # define Py_END_CRITICAL_SECTION() \ } # define Py_BEGIN_CRITICAL_SECTION2(a, b) \ { +# define Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) \ + { # define Py_END_CRITICAL_SECTION2() \ } #else /* !Py_GIL_DISABLED */ @@ -118,6 +128,11 @@ struct PyCriticalSection2 { PyCriticalSection _py_cs; \ PyCriticalSection_Begin(&_py_cs, _PyObject_CAST(op)) +# define Py_BEGIN_CRITICAL_SECTION_MUTEX(mutex) \ + { \ + PyCriticalSection _py_cs; \ + PyCriticalSection_BeginMutex(&_py_cs, mutex) + # define Py_END_CRITICAL_SECTION() \ PyCriticalSection_End(&_py_cs); \ } @@ -127,6 +142,11 @@ struct PyCriticalSection2 { PyCriticalSection2 _py_cs2; \ PyCriticalSection2_Begin(&_py_cs2, _PyObject_CAST(a), _PyObject_CAST(b)) +# define Py_BEGIN_CRITICAL_SECTION2_MUTEX(m1, m2) \ + { \ + PyCriticalSection2 _py_cs2; \ + PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) + # define Py_END_CRITICAL_SECTION2() \ PyCriticalSection2_End(&_py_cs2); \ } diff --git a/Include/cpython/funcobject.h b/Include/cpython/funcobject.h index 18249b95befe65c..598cd330bc9ca95 100644 --- a/Include/cpython/funcobject.h +++ b/Include/cpython/funcobject.h @@ -97,11 +97,6 @@ static inline PyObject* PyFunction_GET_GLOBALS(PyObject *func) { } #define PyFunction_GET_GLOBALS(func) PyFunction_GET_GLOBALS(_PyObject_CAST(func)) -static inline PyObject* PyFunction_GET_BUILTINS(PyObject *func) { - return _PyFunction_CAST(func)->func_builtins; -} -#define PyFunction_GET_BUILTINS(func) PyFunction_GET_BUILTINS(_PyObject_CAST(func)) - static inline PyObject* PyFunction_GET_MODULE(PyObject *func) { return _PyFunction_CAST(func)->func_module; } diff --git a/Include/cpython/lock.h b/Include/cpython/lock.h index 8ee03e82f74dfdc..63886fca28eae24 100644 --- a/Include/cpython/lock.h +++ b/Include/cpython/lock.h @@ -36,6 +36,9 @@ PyAPI_FUNC(void) PyMutex_Lock(PyMutex *m); // exported function for unlocking the mutex PyAPI_FUNC(void) PyMutex_Unlock(PyMutex *m); +// exported function for checking if the mutex is locked +PyAPI_FUNC(int) PyMutex_IsLocked(PyMutex *m); + // Locks the mutex. // // If the mutex is currently locked, the calling thread will be parked until @@ -61,3 +64,11 @@ _PyMutex_Unlock(PyMutex *m) } } #define PyMutex_Unlock _PyMutex_Unlock + +// Checks if the mutex is currently locked. +static inline int +_PyMutex_IsLocked(PyMutex *m) +{ + return (_Py_atomic_load_uint8(&m->_bits) & _Py_LOCKED) != 0; +} +#define PyMutex_IsLocked _PyMutex_IsLocked diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h index 2a0c11e7b3ad665..3a71ed689d88d8f 100644 --- a/Include/cpython/pyatomic.h +++ b/Include/cpython/pyatomic.h @@ -72,8 +72,8 @@ // def _Py_atomic_load_ptr_acquire(obj): // return obj # acquire // -// def _Py_atomic_store_ptr_release(obj): -// return obj # release +// def _Py_atomic_store_ptr_release(obj, value): +// obj = value # release // // def _Py_atomic_fence_seq_cst(): // # sequential consistency @@ -529,6 +529,9 @@ _Py_atomic_store_int_release(int *obj, int value); static inline int _Py_atomic_load_int_acquire(const int *obj); +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value); + static inline void _Py_atomic_store_uint32_release(uint32_t *obj, uint32_t value); @@ -591,6 +594,17 @@ static inline void _Py_atomic_fence_release(void); // --- aliases --------------------------------------------------------------- +// Compilers don't really support "consume" semantics, so we fake it. Use +// "acquire" with TSan to support false positives. Use "relaxed" otherwise, +// because CPUs on all platforms we support respect address dependencies without +// extra barriers. +// See 2.6.7 in https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2020/p2055r0.pdf +#if defined(_Py_THREAD_SANITIZER) +# define _Py_atomic_load_ptr_consume _Py_atomic_load_ptr_acquire +#else +# define _Py_atomic_load_ptr_consume _Py_atomic_load_ptr_relaxed +#endif + #if SIZEOF_LONG == 8 # define _Py_atomic_load_ulong(p) \ _Py_atomic_load_uint64((uint64_t *)p) diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h index 1566b83b9f6a1b6..465226a76ab2556 100644 --- a/Include/cpython/pyatomic_gcc.h +++ b/Include/cpython/pyatomic_gcc.h @@ -576,6 +576,10 @@ static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { __atomic_store_n(obj, value, __ATOMIC_RELEASE); } +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); } + static inline int _Py_atomic_load_int_acquire(const int *obj) { return __atomic_load_n(obj, __ATOMIC_ACQUIRE); } diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h index d155955df0cddf6..eb6df819fa7ee88 100644 --- a/Include/cpython/pyatomic_msc.h +++ b/Include/cpython/pyatomic_msc.h @@ -971,12 +971,6 @@ _Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) *(volatile unsigned short *)obj = value; } -static inline void -_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) -{ - *(volatile unsigned int *)obj = value; -} - static inline void _Py_atomic_store_long_relaxed(long *obj, long value) { @@ -1066,6 +1060,19 @@ _Py_atomic_store_int_release(int *obj, int value) #endif } +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ +#if defined(_M_X64) || defined(_M_IX86) + *(volatile unsigned int *)obj = value; +#elif defined(_M_ARM64) + _Py_atomic_ASSERT_ARG_TYPE(unsigned __int32); + __stlr32((unsigned __int32 volatile *)obj, (unsigned __int32)value); +#else +# error "no implementation of _Py_atomic_store_uint_release" +#endif +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h index 69a8b9e615ea5ff..1a2ccd0cc12b488 100644 --- a/Include/cpython/pyatomic_std.h +++ b/Include/cpython/pyatomic_std.h @@ -459,7 +459,7 @@ static inline uint16_t _Py_atomic_load_uint16(const uint16_t *obj) { _Py_USING_STD; - return atomic_load((const _Atomic(uint32_t)*)obj); + return atomic_load((const _Atomic(uint16_t)*)obj); } static inline uint32_t @@ -948,14 +948,6 @@ _Py_atomic_store_ushort_relaxed(unsigned short *obj, unsigned short value) memory_order_relaxed); } -static inline void -_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) -{ - _Py_USING_STD; - atomic_store_explicit((_Atomic(unsigned int)*)obj, value, - memory_order_relaxed); -} - static inline void _Py_atomic_store_long_relaxed(long *obj, long value) { @@ -1031,6 +1023,14 @@ _Py_atomic_store_int_release(int *obj, int value) memory_order_release); } +static inline void +_Py_atomic_store_uint_release(unsigned int *obj, unsigned int value) +{ + _Py_USING_STD; + atomic_store_explicit((_Atomic(unsigned int)*)obj, value, + memory_order_release); +} + static inline void _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value) { diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index 6b63d304b0d9297..be2e3b641c25cb6 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -18,6 +18,7 @@ typedef struct { PyException_HEAD PyObject *msg; PyObject *excs; + PyObject *excs_str; } PyBaseExceptionGroupObject; typedef struct { diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 7f1bc363861ddf0..c1b97d428e2e9b6 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -28,10 +28,10 @@ typedef int (*Py_tracefunc)(PyObject *, PyFrameObject *, int, PyObject *); #define PyTrace_OPCODE 7 /* Remote debugger support */ -#define MAX_SCRIPT_PATH_SIZE 512 -typedef struct _remote_debugger_support { +#define _Py_MAX_SCRIPT_PATH_SIZE 512 +typedef struct { int32_t debugger_pending_call; - char debugger_script_path[MAX_SCRIPT_PATH_SIZE]; + char debugger_script_path[_Py_MAX_SCRIPT_PATH_SIZE]; } _PyRemoteDebuggerSupport; typedef struct _err_stackitem { @@ -61,6 +61,8 @@ typedef struct _stack_chunk { PyObject * data[1]; /* Variable sized */ } _PyStackChunk; +/* Minimum size of data stack chunk */ +#define _PY_DATA_STACK_CHUNK_SIZE (16*1024) struct _ts { /* See Python/ceval.c for comments explaining most fields */ @@ -206,6 +208,8 @@ struct _ts { */ PyObject *threading_local_sentinel; _PyRemoteDebuggerSupport remote_debugger_support; + + _PyStackChunk *datastack_cached_chunk; }; /* other API */ @@ -241,6 +245,18 @@ PyAPI_FUNC(int) PyGILState_Check(void); */ PyAPI_FUNC(PyObject*) _PyThread_CurrentFrames(void); +// Set the stack protection start address and stack protection size +// of a Python thread state +PyAPI_FUNC(int) PyUnstable_ThreadState_SetStackProtection( + PyThreadState *tstate, + void *stack_start_addr, // Stack start address + size_t stack_size); // Stack size (in bytes) + +// Reset the stack protection start address and stack protection size +// of a Python thread state +PyAPI_FUNC(void) PyUnstable_ThreadState_ResetStackProtection( + PyThreadState *tstate); + /* Routines for advanced debuggers, requested by David Beazley. Don't use unless you know what you are doing! */ PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Main(void); diff --git a/Include/cpython/pythonrun.h b/Include/cpython/pythonrun.h index edc40952254029f..b0bf527f4f0ec19 100644 --- a/Include/cpython/pythonrun.h +++ b/Include/cpython/pythonrun.h @@ -40,6 +40,11 @@ PyAPI_FUNC(PyObject *) PyRun_FileExFlags( PyCompilerFlags *flags); +PyAPI_FUNC(PyObject *) Py_CompileStringFlags( + const char *str, + const char *filename, + int start, + PyCompilerFlags *flags); PyAPI_FUNC(PyObject *) Py_CompileStringExFlags( const char *str, const char *filename, /* decoded from the filesystem encoding */ diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 136f5d5c5f84252..3d0414f5291fe45 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -478,6 +478,10 @@ PyAPI_FUNC(int) PyUnicodeWriter_WriteUTF8( PyUnicodeWriter *writer, const char *str, Py_ssize_t size); +PyAPI_FUNC(int) PyUnicodeWriter_WriteASCII( + PyUnicodeWriter *writer, + const char *str, + Py_ssize_t size); PyAPI_FUNC(int) PyUnicodeWriter_WriteWideChar( PyUnicodeWriter *writer, const wchar_t *str, diff --git a/Include/cpython/warnings.h b/Include/cpython/warnings.h index 8731fd2e96b716f..4e3eb88e8ff4472 100644 --- a/Include/cpython/warnings.h +++ b/Include/cpython/warnings.h @@ -18,9 +18,3 @@ PyAPI_FUNC(int) PyErr_WarnExplicitFormat( // DEPRECATED: Use PyErr_WarnEx() instead. #define PyErr_Warn(category, msg) PyErr_WarnEx((category), (msg), 1) - -int _PyErr_WarnExplicitObjectWithContext( - PyObject *category, - PyObject *message, - PyObject *filename, - int lineno); diff --git a/Include/dynamic_annotations.h b/Include/dynamic_annotations.h index 4d4def9bf8983e2..5290319dd762c55 100644 --- a/Include/dynamic_annotations.h +++ b/Include/dynamic_annotations.h @@ -461,6 +461,7 @@ int RunningOnValgrind(void); #if DYNAMIC_ANNOTATIONS_ENABLED != 0 && defined(__cplusplus) +extern "C++" { /* _Py_ANNOTATE_UNPROTECTED_READ is the preferred way to annotate racey reads. Instead of doing @@ -476,6 +477,8 @@ int RunningOnValgrind(void); _Py_ANNOTATE_IGNORE_READS_END(); return res; } +} + /* Apply _Py_ANNOTATE_BENIGN_RACE_SIZED to a static variable. */ #define _Py_ANNOTATE_BENIGN_RACE_STATIC(static_var, description) \ namespace { \ diff --git a/Include/internal/mimalloc/mimalloc/internal.h b/Include/internal/mimalloc/mimalloc/internal.h index d97f51b8eefbe55..a7daa3a40a4c0bc 100644 --- a/Include/internal/mimalloc/mimalloc/internal.h +++ b/Include/internal/mimalloc/mimalloc/internal.h @@ -634,10 +634,10 @@ static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* bl mi_track_mem_defined(block,sizeof(mi_block_t)); mi_block_t* next; #ifdef MI_ENCODE_FREELIST - next = (mi_block_t*)mi_ptr_decode(null, block->next, keys); + next = (mi_block_t*)mi_ptr_decode(null, mi_atomic_load_relaxed((_Atomic(mi_encoded_t)*)&block->next), keys); #else MI_UNUSED(keys); MI_UNUSED(null); - next = (mi_block_t*)block->next; + next = (mi_block_t*)mi_atomic_load_relaxed((_Atomic(mi_encoded_t)*)&block->next); #endif mi_track_mem_noaccess(block,sizeof(mi_block_t)); return next; @@ -646,10 +646,10 @@ static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* bl static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_block_t* next, const uintptr_t* keys) { mi_track_mem_undefined(block,sizeof(mi_block_t)); #ifdef MI_ENCODE_FREELIST - block->next = mi_ptr_encode(null, next, keys); + mi_atomic_store_relaxed(&block->next, mi_ptr_encode(null, next, keys)); #else MI_UNUSED(keys); MI_UNUSED(null); - block->next = (mi_encoded_t)next; + mi_atomic_store_relaxed(&block->next, (mi_encoded_t)next); #endif mi_track_mem_noaccess(block,sizeof(mi_block_t)); } diff --git a/Include/internal/mimalloc/mimalloc/types.h b/Include/internal/mimalloc/mimalloc/types.h index 354839ba955b364..70c600e920bad19 100644 --- a/Include/internal/mimalloc/mimalloc/types.h +++ b/Include/internal/mimalloc/mimalloc/types.h @@ -235,7 +235,7 @@ typedef size_t mi_threadid_t; // free lists contain blocks typedef struct mi_block_s { - mi_encoded_t next; + _Atomic(mi_encoded_t) next; } mi_block_t; @@ -678,7 +678,7 @@ void _mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount); // Thread Local data // ------------------------------------------------------ -// A "span" is is an available range of slices. The span queues keep +// A "span" is an available range of slices. The span queues keep // track of slice spans of at most the given `slice_count` (but more than the previous size class). typedef struct mi_span_queue_s { mi_slice_t* first; diff --git a/Include/internal/pycore_bytesobject.h b/Include/internal/pycore_bytesobject.h index 300e7f4896a39ef..8ea9b3ebb884541 100644 --- a/Include/internal/pycore_bytesobject.h +++ b/Include/internal/pycore_bytesobject.h @@ -20,8 +20,9 @@ extern PyObject* _PyBytes_FromHex( // Helper for PyBytes_DecodeEscape that detects invalid escape chars. // Export for test_peg_generator. -PyAPI_FUNC(PyObject*) _PyBytes_DecodeEscape(const char *, Py_ssize_t, - const char *, const char **); +PyAPI_FUNC(PyObject*) _PyBytes_DecodeEscape2(const char *, Py_ssize_t, + const char *, + int *, const char **); // Substring Search. diff --git a/Include/internal/pycore_call.h b/Include/internal/pycore_call.h index 32ac3d17f220778..05b58b2ba54a104 100644 --- a/Include/internal/pycore_call.h +++ b/Include/internal/pycore_call.h @@ -98,6 +98,14 @@ _PyObject_CallMethodIdOneArg(PyObject *self, _Py_Identifier *name, PyObject *arg } +extern PyObject *_PyObject_VectorcallPrepend( + PyThreadState *tstate, + PyObject *callable, + PyObject *arg, + PyObject *const *args, + size_t nargsf, + PyObject *kwnames); + /* === Vectorcall protocol (PEP 590) ============================= */ // Call callable using tp_call. Arguments are like PyObject_Vectorcall(), diff --git a/Include/internal/pycore_cell.h b/Include/internal/pycore_cell.h index cef01e80514f4b1..d0d45a2343654fa 100644 --- a/Include/internal/pycore_cell.h +++ b/Include/internal/pycore_cell.h @@ -53,7 +53,7 @@ _PyCell_GetStackRef(PyCellObject *cell) { PyObject *value; #ifdef Py_GIL_DISABLED - value = _Py_atomic_load_ptr(&cell->ob_ref); + value = _PyObject_CAST(_Py_atomic_load_ptr(&cell->ob_ref)); if (value == NULL) { return PyStackRef_NULL; } diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 3d8247df31ce32b..e3fa8e10eb447ee 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -22,8 +22,10 @@ struct _ceval_runtime_state; // Export for '_lsprof' shared extension PyAPI_FUNC(int) _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); +extern int _PyEval_SetProfileAllThreads(PyInterpreterState *interp, Py_tracefunc func, PyObject *arg); extern int _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg); +extern int _PyEval_SetTraceAllThreads(PyInterpreterState *interp, Py_tracefunc func, PyObject *arg); extern int _PyEval_SetOpcodeTrace(PyFrameObject *f, bool enable); @@ -103,7 +105,6 @@ extern int _PyPerfTrampoline_SetCallbacks(_PyPerf_Callbacks *); extern void _PyPerfTrampoline_GetCallbacks(_PyPerf_Callbacks *); extern int _PyPerfTrampoline_Init(int activate); extern int _PyPerfTrampoline_Fini(void); -extern void _PyPerfTrampoline_FreeArenas(void); extern int _PyIsPerfTrampolineActive(void); extern PyStatus _PyPerfTrampoline_AfterFork_Child(void); #ifdef PY_HAVE_PERF_TRAMPOLINE @@ -199,7 +200,14 @@ extern void _PyEval_DeactivateOpCache(void); static inline int _Py_MakeRecCheck(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - return here_addr < _tstate->c_stack_soft_limit; + // Overflow if stack pointer is between soft limit and the base of the hardware stack. + // If it is below the hardware stack base, assume that we have the wrong stack limits, and do nothing. + // We could have the wrong stack limits because of limited platform support, or user-space threads. +#if _Py_STACK_GROWS_DOWN + return here_addr < _tstate->c_stack_soft_limit && here_addr >= _tstate->c_stack_soft_limit - 2 * _PyOS_STACK_MARGIN_BYTES; +#else + return here_addr > _tstate->c_stack_soft_limit && here_addr <= _tstate->c_stack_soft_limit + 2 * _PyOS_STACK_MARGIN_BYTES; +#endif } // Export for '_json' shared extension, used via _Py_EnterRecursiveCall() @@ -231,7 +239,11 @@ static inline int _Py_ReachedRecursionLimit(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; assert(_tstate->c_stack_hard_limit != 0); +#if _Py_STACK_GROWS_DOWN return here_addr <= _tstate->c_stack_soft_limit; +#else + return here_addr >= _tstate->c_stack_soft_limit; +#endif } static inline void _Py_LeaveRecursiveCall(void) { @@ -239,6 +251,16 @@ static inline void _Py_LeaveRecursiveCall(void) { extern _PyInterpreterFrame* _PyEval_GetFrame(void); +extern PyObject * _PyEval_GetGlobalsFromRunningMain(PyThreadState *); +extern int _PyEval_EnsureBuiltins( + PyThreadState *, + PyObject *, + PyObject **p_builtins); +extern int _PyEval_EnsureBuiltinsWithModule( + PyThreadState *, + PyObject *, + PyObject **p_builtins); + PyAPI_FUNC(PyObject *)_Py_MakeCoro(PyFunctionObject *func); /* Handle signals, pending calls, GIL drop request @@ -273,6 +295,7 @@ PyAPI_FUNC(PyObject *) _PyEval_ImportName(PyThreadState *, _PyInterpreterFrame * PyAPI_FUNC(PyObject *)_PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, Py_ssize_t nargs, PyObject *kwargs); PyAPI_FUNC(PyObject *)_PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys); PyAPI_FUNC(void) _PyEval_MonitorRaise(PyThreadState *tstate, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr); +PyAPI_FUNC(bool) _PyEval_NoToolsForUnwind(PyThreadState *tstate); PyAPI_FUNC(int) _PyEval_UnpackIterableStackRef(PyThreadState *tstate, PyObject *v, int argcnt, int argcntafter, _PyStackRef *sp); PyAPI_FUNC(void) _PyEval_FrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame *frame); PyAPI_FUNC(PyObject **) _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject **scratch); @@ -353,6 +376,18 @@ PyAPI_FUNC(_PyStackRef) _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyS extern int _PyRunRemoteDebugger(PyThreadState *tstate); #endif +/* Special methods used by LOAD_SPECIAL */ +#define SPECIAL___ENTER__ 0 +#define SPECIAL___EXIT__ 1 +#define SPECIAL___AENTER__ 2 +#define SPECIAL___AEXIT__ 3 +#define SPECIAL_MAX 3 + +PyAPI_FUNC(_PyStackRef) +_Py_LoadAttr_StackRefSteal( + PyThreadState *tstate, _PyStackRef owner, + PyObject *name, _PyStackRef *self_or_null); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_code.h b/Include/internal/pycore_code.h index b135e30b7ad62ae..dc0ea9c669b7312 100644 --- a/Include/internal/pycore_code.h +++ b/Include/internal/pycore_code.h @@ -275,6 +275,13 @@ extern void _PyLineTable_InitAddressRange( extern int _PyLineTable_NextAddressRange(PyCodeAddressRange *range); extern int _PyLineTable_PreviousAddressRange(PyCodeAddressRange *range); +// Similar to PyCode_Addr2Line(), but return -1 if the code object is invalid +// and can be called without an attached tstate. Used by dump_frame() in +// Python/traceback.c. The function uses heuristics to detect freed memory, +// it's not 100% reliable. +extern int _PyCode_SafeAddr2Line(PyCodeObject *co, int addr); + + /** API for executors */ extern void _PyCode_Clear_Executors(PyCodeObject *code); @@ -614,6 +621,47 @@ PyAPI_FUNC(int) _PyCode_SetUnboundVarCounts( PyObject *globalsns, PyObject *builtinsns); + +/* "Stateless" code is a function or code object which does not rely on + * external state or internal state. It may rely on arguments and + * builtins, but not globals or a closure. Thus it does not rely + * on __globals__ or __closure__, and a stateless function + * is equivalent to its code object. + * + * Stateless code also does not keep any persistent state + * of its own, so it can't have any executors, monitoring, + * instrumentation, or "extras" (i.e. co_extra). + * + * Stateless code may create nested functions, including closures. + * However, nested functions must themselves be stateless, except they + * *can* close on the enclosing locals. + * + * Stateless code may return any value, including nested functions and closures. + * + * Stateless code that takes no arguments and doesn't return anything + * may be treated like a script. + * + * We consider stateless code to be "portable" if it does not return + * any object that holds a reference to any of the code's locals. Thus + * generators and coroutines are not portable. Likewise a function + * that returns a closure is not portable. The concept of + * portability is useful in cases where the code is run + * in a different execution context than where + * the return value will be used. */ + +PyAPI_FUNC(int) _PyCode_CheckNoInternalState(PyCodeObject *, const char **); +PyAPI_FUNC(int) _PyCode_CheckNoExternalState( + PyCodeObject *, + _PyCode_var_counts_t *, + const char **); +PyAPI_FUNC(int) _PyCode_VerifyStateless( + PyThreadState *, + PyCodeObject *, + PyObject *globalnames, + PyObject *globalsns, + PyObject *builtinsns); + +PyAPI_FUNC(int) _PyCode_CheckPureFunction(PyCodeObject *, const char **); PyAPI_FUNC(int) _PyCode_ReturnsOnlyNone(PyCodeObject *); diff --git a/Include/internal/pycore_compile.h b/Include/internal/pycore_compile.h index aecc50be1e6c34a..ed776bbb73a0ee9 100644 --- a/Include/internal/pycore_compile.h +++ b/Include/internal/pycore_compile.h @@ -49,7 +49,8 @@ extern int _PyAST_Preprocess( PyObject *filename, int optimize, int ff_features, - int syntax_check_only); + int syntax_check_only, + int enable_warnings); typedef struct { diff --git a/Include/internal/pycore_critical_section.h b/Include/internal/pycore_critical_section.h index 42f06b935bd0a0f..942bcff7e01df77 100644 --- a/Include/internal/pycore_critical_section.h +++ b/Include/internal/pycore_critical_section.h @@ -21,16 +21,6 @@ extern "C" { #define _Py_CRITICAL_SECTION_MASK 0x3 #ifdef Py_GIL_DISABLED -# define Py_BEGIN_CRITICAL_SECTION_MUT(mutex) \ - { \ - PyCriticalSection _py_cs; \ - _PyCriticalSection_BeginMutex(&_py_cs, mutex) - -# define Py_BEGIN_CRITICAL_SECTION2_MUT(m1, m2) \ - { \ - PyCriticalSection2 _py_cs2; \ - _PyCriticalSection2_BeginMutex(&_py_cs2, m1, m2) - // Specialized version of critical section locking to safely use // PySequence_Fast APIs without the GIL. For performance, the argument *to* // PySequence_Fast() is provided to the macro, not the *result* of @@ -75,8 +65,6 @@ extern "C" { #else /* !Py_GIL_DISABLED */ // The critical section APIs are no-ops with the GIL. -# define Py_BEGIN_CRITICAL_SECTION_MUT(mut) { -# define Py_BEGIN_CRITICAL_SECTION2_MUT(m1, m2) { # define Py_BEGIN_CRITICAL_SECTION_SEQUENCE_FAST(original) { # define Py_END_CRITICAL_SECTION_SEQUENCE_FAST() } # define _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(mutex) @@ -119,6 +107,7 @@ _PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m) _PyCriticalSection_BeginSlow(c, m); } } +#define PyCriticalSection_BeginMutex _PyCriticalSection_BeginMutex static inline void _PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op) @@ -194,6 +183,7 @@ _PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) _PyCriticalSection2_BeginSlow(c, m1, m2, 0); } } +#define PyCriticalSection2_BeginMutex _PyCriticalSection2_BeginMutex static inline void _PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) @@ -205,10 +195,9 @@ _PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) static inline void _PyCriticalSection2_End(PyCriticalSection2 *c) { - // if mutex1 is NULL, we used the fast path in - // _PyCriticalSection_BeginSlow for mutexes that are already held, - // which should only happen when mutex1 and mutex2 were the same mutex, - // and mutex2 should also be NULL. + // if mutex1 is NULL, we used the fast path in either + // _PyCriticalSection_BeginSlow or _PyCriticalSection2_BeginSlow for mutexes + // that are already held, and mutex2 should also be NULL. if (c->_cs_base._cs_mutex == NULL) { assert(c->_cs_mutex2 == NULL); return; diff --git a/Include/internal/pycore_crossinterp.h b/Include/internal/pycore_crossinterp.h index 9de61ef54125d53..81faffac1941719 100644 --- a/Include/internal/pycore_crossinterp.h +++ b/Include/internal/pycore_crossinterp.h @@ -131,7 +131,23 @@ PyAPI_FUNC(void) _PyXIData_Clear(PyInterpreterState *, _PyXIData_t *); /* getting cross-interpreter data */ -typedef int (*xidatafunc)(PyThreadState *tstate, PyObject *, _PyXIData_t *); +typedef int xidata_fallback_t; +#define _PyXIDATA_XIDATA_ONLY (0) +#define _PyXIDATA_FULL_FALLBACK (1) + +// Technically, we don't need two different function types; +// we could go with just the fallback one. However, only container +// types like tuple need it, so always having the extra arg would be +// a bit unfortunate. It's also nice to be able to clearly distinguish +// between types that might call _PyObject_GetXIData() and those that won't. +// +typedef int (*xidatafunc)(PyThreadState *, PyObject *, _PyXIData_t *); +typedef int (*xidatafbfunc)( + PyThreadState *, PyObject *, xidata_fallback_t, _PyXIData_t *); +typedef struct { + xidatafunc basic; + xidatafbfunc fallback; +} _PyXIData_getdata_t; PyAPI_FUNC(PyObject *) _PyXIData_GetNotShareableErrorType(PyThreadState *); PyAPI_FUNC(void) _PyXIData_SetNotShareableError(PyThreadState *, const char *); @@ -140,16 +156,21 @@ PyAPI_FUNC(void) _PyXIData_FormatNotShareableError( const char *, ...); -PyAPI_FUNC(xidatafunc) _PyXIData_Lookup( +PyAPI_FUNC(_PyXIData_getdata_t) _PyXIData_Lookup( PyThreadState *, PyObject *); PyAPI_FUNC(int) _PyObject_CheckXIData( PyThreadState *, PyObject *); +PyAPI_FUNC(int) _PyObject_GetXIDataNoFallback( + PyThreadState *, + PyObject *, + _PyXIData_t *); PyAPI_FUNC(int) _PyObject_GetXIData( PyThreadState *, PyObject *, + xidata_fallback_t, _PyXIData_t *); // _PyObject_GetXIData() for bytes @@ -191,6 +212,21 @@ PyAPI_FUNC(int) _PyCode_GetXIData( PyThreadState *, PyObject *, _PyXIData_t *); +PyAPI_FUNC(int) _PyCode_GetScriptXIData( + PyThreadState *, + PyObject *, + _PyXIData_t *); +PyAPI_FUNC(int) _PyCode_GetPureScriptXIData( + PyThreadState *, + PyObject *, + _PyXIData_t *); + +// _PyObject_GetXIData() for functions +PyAPI_FUNC(PyObject *) _PyFunction_FromXIData(_PyXIData_t *); +PyAPI_FUNC(int) _PyFunction_GetXIData( + PyThreadState *, + PyObject *, + _PyXIData_t *); /* using cross-interpreter data */ @@ -267,10 +303,10 @@ typedef struct _excinfo { const char *errdisplay; } _PyXI_excinfo; -PyAPI_FUNC(int) _PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc); +PyAPI_FUNC(_PyXI_excinfo *) _PyXI_NewExcInfo(PyObject *exc); +PyAPI_FUNC(void) _PyXI_FreeExcInfo(_PyXI_excinfo *info); PyAPI_FUNC(PyObject *) _PyXI_FormatExcInfo(_PyXI_excinfo *info); PyAPI_FUNC(PyObject *) _PyXI_ExcInfoAsObject(_PyXI_excinfo *info); -PyAPI_FUNC(void) _PyXI_ClearExcInfo(_PyXI_excinfo *info); typedef enum error_code { @@ -281,42 +317,30 @@ typedef enum error_code { _PyXI_ERR_ALREADY_RUNNING = -4, _PyXI_ERR_MAIN_NS_FAILURE = -5, _PyXI_ERR_APPLY_NS_FAILURE = -6, - _PyXI_ERR_NOT_SHAREABLE = -7, + _PyXI_ERR_PRESERVE_FAILURE = -7, + _PyXI_ERR_EXC_PROPAGATION_FAILURE = -8, + _PyXI_ERR_NOT_SHAREABLE = -9, } _PyXI_errcode; +typedef struct xi_failure _PyXI_failure; -typedef struct _sharedexception { - // The originating interpreter. - PyInterpreterState *interp; - // The kind of error to propagate. - _PyXI_errcode code; - // The exception information to propagate, if applicable. - // This is populated only for some error codes, - // but always for _PyXI_ERR_UNCAUGHT_EXCEPTION. - _PyXI_excinfo uncaught; -} _PyXI_error; - -PyAPI_FUNC(PyObject *) _PyXI_ApplyError(_PyXI_error *err); - - -typedef struct xi_session _PyXI_session; -typedef struct _sharedns _PyXI_namespace; +PyAPI_FUNC(_PyXI_failure *) _PyXI_NewFailure(void); +PyAPI_FUNC(void) _PyXI_FreeFailure(_PyXI_failure *); +PyAPI_FUNC(_PyXI_errcode) _PyXI_GetFailureCode(_PyXI_failure *); +PyAPI_FUNC(int) _PyXI_InitFailure(_PyXI_failure *, _PyXI_errcode, PyObject *); +PyAPI_FUNC(void) _PyXI_InitFailureUTF8( + _PyXI_failure *, + _PyXI_errcode, + const char *); -PyAPI_FUNC(void) _PyXI_FreeNamespace(_PyXI_namespace *ns); -PyAPI_FUNC(_PyXI_namespace *) _PyXI_NamespaceFromNames(PyObject *names); -PyAPI_FUNC(int) _PyXI_FillNamespaceFromDict( - _PyXI_namespace *ns, - PyObject *nsobj, - _PyXI_session *session); -PyAPI_FUNC(int) _PyXI_ApplyNamespace( - _PyXI_namespace *ns, - PyObject *nsobj, - PyObject *dflt); +PyAPI_FUNC(int) _PyXI_UnwrapNotShareableError( + PyThreadState *, + _PyXI_failure *); // A cross-interpreter session involves entering an interpreter -// (_PyXI_Enter()), doing some work with it, and finally exiting -// that interpreter (_PyXI_Exit()). +// with _PyXI_Enter(), doing some work with it, and finally exiting +// that interpreter with _PyXI_Exit(). // // At the boundaries of the session, both entering and exiting, // data may be exchanged between the previous interpreter and the @@ -324,48 +348,40 @@ PyAPI_FUNC(int) _PyXI_ApplyNamespace( // isolation between interpreters. This includes setting objects // in the target's __main__ module on the way in, and capturing // uncaught exceptions on the way out. -struct xi_session { - // Once a session has been entered, this is the tstate that was - // current before the session. If it is different from cur_tstate - // then we must have switched interpreters. Either way, this will - // be the current tstate once we exit the session. - PyThreadState *prev_tstate; - // Once a session has been entered, this is the current tstate. - // It must be current when the session exits. - PyThreadState *init_tstate; - // This is true if init_tstate needs cleanup during exit. - int own_init_tstate; - - // This is true if, while entering the session, init_thread took - // "ownership" of the interpreter's __main__ module. This means - // it is the only thread that is allowed to run code there. - // (Caveat: for now, users may still run exec() against the - // __main__ module's dict, though that isn't advisable.) - int running; - // This is a cached reference to the __dict__ of the entered - // interpreter's __main__ module. It is looked up when at the - // beginning of the session as a convenience. - PyObject *main_ns; - - // This is set if the interpreter is entered and raised an exception - // that needs to be handled in some special way during exit. - _PyXI_errcode *error_override; - // This is set if exit captured an exception to propagate. - _PyXI_error *error; - - // -- pre-allocated memory -- - _PyXI_error _error; - _PyXI_errcode _error_override; -}; +typedef struct xi_session _PyXI_session; + +PyAPI_FUNC(_PyXI_session *) _PyXI_NewSession(void); +PyAPI_FUNC(void) _PyXI_FreeSession(_PyXI_session *); + +typedef struct { + PyObject *preserved; + PyObject *excinfo; + _PyXI_errcode errcode; +} _PyXI_session_result; +PyAPI_FUNC(void) _PyXI_ClearResult(_PyXI_session_result *); PyAPI_FUNC(int) _PyXI_Enter( _PyXI_session *session, PyInterpreterState *interp, - PyObject *nsupdates); -PyAPI_FUNC(void) _PyXI_Exit(_PyXI_session *session); - -PyAPI_FUNC(PyObject *) _PyXI_ApplyCapturedException(_PyXI_session *session); -PyAPI_FUNC(int) _PyXI_HasCapturedException(_PyXI_session *session); + PyObject *nsupdates, + _PyXI_session_result *); +PyAPI_FUNC(int) _PyXI_Exit( + _PyXI_session *, + _PyXI_failure *, + _PyXI_session_result *); + +PyAPI_FUNC(PyObject *) _PyXI_GetMainNamespace( + _PyXI_session *, + _PyXI_failure *); + +PyAPI_FUNC(int) _PyXI_Preserve( + _PyXI_session *, + const char *, + PyObject *, + _PyXI_failure *); +PyAPI_FUNC(PyObject *) _PyXI_GetPreserved( + _PyXI_session_result *, + const char *); /*************/ diff --git a/Include/internal/pycore_crossinterp_data_registry.h b/Include/internal/pycore_crossinterp_data_registry.h index 8f4bcb948e5a45e..fbb4cad5cac32e0 100644 --- a/Include/internal/pycore_crossinterp_data_registry.h +++ b/Include/internal/pycore_crossinterp_data_registry.h @@ -17,7 +17,7 @@ typedef struct _xid_regitem { /* This is NULL for builtin types. */ PyObject *weakref; size_t refcount; - xidatafunc getdata; + _PyXIData_getdata_t getdata; } _PyXIData_regitem_t; typedef struct { @@ -30,7 +30,7 @@ typedef struct { PyAPI_FUNC(int) _PyXIData_RegisterClass( PyThreadState *, PyTypeObject *, - xidatafunc); + _PyXIData_getdata_t); PyAPI_FUNC(int) _PyXIData_UnregisterClass( PyThreadState *, PyTypeObject *); diff --git a/Include/internal/pycore_debug_offsets.h b/Include/internal/pycore_debug_offsets.h index 59d2c9d53779538..8e7cd16acffa48a 100644 --- a/Include/internal/pycore_debug_offsets.h +++ b/Include/internal/pycore_debug_offsets.h @@ -52,9 +52,15 @@ extern "C" { #ifdef Py_GIL_DISABLED # define _Py_Debug_gilruntimestate_enabled offsetof(struct _gil_runtime_state, enabled) # define _Py_Debug_Free_Threaded 1 +# define _Py_Debug_code_object_co_tlbc offsetof(PyCodeObject, co_tlbc) +# define _Py_Debug_interpreter_frame_tlbc_index offsetof(_PyInterpreterFrame, tlbc_index) +# define _Py_Debug_interpreter_state_tlbc_generation offsetof(PyInterpreterState, tlbc_indices.tlbc_generation) #else # define _Py_Debug_gilruntimestate_enabled 0 # define _Py_Debug_Free_Threaded 0 +# define _Py_Debug_code_object_co_tlbc 0 +# define _Py_Debug_interpreter_frame_tlbc_index 0 +# define _Py_Debug_interpreter_state_tlbc_generation 0 #endif @@ -85,6 +91,8 @@ typedef struct _Py_DebugOffsets { uint64_t gil_runtime_state_enabled; uint64_t gil_runtime_state_locked; uint64_t gil_runtime_state_holder; + uint64_t code_object_generation; + uint64_t tlbc_generation; } interpreter_state; // Thread state offset; @@ -109,6 +117,7 @@ typedef struct _Py_DebugOffsets { uint64_t localsplus; uint64_t owner; uint64_t stackpointer; + uint64_t tlbc_index; } interpreter_frame; // Code object offset; @@ -123,6 +132,7 @@ typedef struct _Py_DebugOffsets { uint64_t localsplusnames; uint64_t localspluskinds; uint64_t co_code_adaptive; + uint64_t co_tlbc; } code_object; // PyObject offset; @@ -210,6 +220,11 @@ typedef struct _Py_DebugOffsets { uint64_t gi_frame_state; } gen_object; + struct _llist_node { + uint64_t next; + uint64_t prev; + } llist_node; + struct _debugger_support { uint64_t eval_breaker; uint64_t remote_debugger_support; @@ -245,6 +260,8 @@ typedef struct _Py_DebugOffsets { .gil_runtime_state_enabled = _Py_Debug_gilruntimestate_enabled, \ .gil_runtime_state_locked = offsetof(PyInterpreterState, _gil.locked), \ .gil_runtime_state_holder = offsetof(PyInterpreterState, _gil.last_holder), \ + .code_object_generation = offsetof(PyInterpreterState, _code_object_generation), \ + .tlbc_generation = _Py_Debug_interpreter_state_tlbc_generation, \ }, \ .thread_state = { \ .size = sizeof(PyThreadState), \ @@ -265,6 +282,7 @@ typedef struct _Py_DebugOffsets { .localsplus = offsetof(_PyInterpreterFrame, localsplus), \ .owner = offsetof(_PyInterpreterFrame, owner), \ .stackpointer = offsetof(_PyInterpreterFrame, stackpointer), \ + .tlbc_index = _Py_Debug_interpreter_frame_tlbc_index, \ }, \ .code_object = { \ .size = sizeof(PyCodeObject), \ @@ -277,6 +295,7 @@ typedef struct _Py_DebugOffsets { .localsplusnames = offsetof(PyCodeObject, co_localsplusnames), \ .localspluskinds = offsetof(PyCodeObject, co_localspluskinds), \ .co_code_adaptive = offsetof(PyCodeObject, co_code_adaptive), \ + .co_tlbc = _Py_Debug_code_object_co_tlbc, \ }, \ .pyobject = { \ .size = sizeof(PyObject), \ @@ -339,13 +358,17 @@ typedef struct _Py_DebugOffsets { .gi_iframe = offsetof(PyGenObject, gi_iframe), \ .gi_frame_state = offsetof(PyGenObject, gi_frame_state), \ }, \ + .llist_node = { \ + .next = offsetof(struct llist_node, next), \ + .prev = offsetof(struct llist_node, prev), \ + }, \ .debugger_support = { \ .eval_breaker = offsetof(PyThreadState, eval_breaker), \ .remote_debugger_support = offsetof(PyThreadState, remote_debugger_support), \ .remote_debugging_enabled = offsetof(PyInterpreterState, config.remote_debug), \ .debugger_pending_call = offsetof(_PyRemoteDebuggerSupport, debugger_pending_call), \ .debugger_script_path = offsetof(_PyRemoteDebuggerSupport, debugger_script_path), \ - .debugger_script_path_size = MAX_SCRIPT_PATH_SIZE, \ + .debugger_script_path_size = _Py_MAX_SCRIPT_PATH_SIZE, \ }, \ } diff --git a/Include/internal/pycore_dict.h b/Include/internal/pycore_dict.h index 754eb88a85c33c2..77d99891b306819 100644 --- a/Include/internal/pycore_dict.h +++ b/Include/internal/pycore_dict.h @@ -30,6 +30,10 @@ PyAPI_FUNC(int) _PyDict_SetItem_KnownHash(PyObject *mp, PyObject *key, // Export for '_asyncio' shared extension PyAPI_FUNC(int) _PyDict_DelItem_KnownHash(PyObject *mp, PyObject *key, Py_hash_t hash); + +extern int _PyDict_DelItem_KnownHash_LockHeld(PyObject *mp, PyObject *key, + Py_hash_t hash); + extern int _PyDict_Contains_KnownHash(PyObject *, PyObject *, Py_hash_t); // "Id" variants @@ -47,6 +51,8 @@ extern int _PyDict_HasOnlyStringKeys(PyObject *mp); // Export for '_ctypes' shared extension PyAPI_FUNC(Py_ssize_t) _PyDict_SizeOf(PyDictObject *); +extern Py_ssize_t _PyDict_SizeOf_LockHeld(PyDictObject *); + #define _PyDict_HasSplitTable(d) ((d)->ma_values != NULL) /* Like PyDict_Merge, but override can be 0, 1 or 2. If override is 0, @@ -113,6 +119,8 @@ extern Py_ssize_t _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t has extern Py_ssize_t _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **value_addr); extern Py_ssize_t _Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject *key, Py_hash_t hash, _PyStackRef *value_addr); +extern int _PyDict_GetMethodStackRef(PyDictObject *dict, PyObject *name, _PyStackRef *method); + extern Py_ssize_t _PyDict_LookupIndex(PyDictObject *, PyObject *); extern Py_ssize_t _PyDictKeys_StringLookup(PyDictKeysObject* dictkeys, PyObject *key); @@ -150,6 +158,8 @@ extern int _PyDict_Pop_KnownHash( Py_hash_t hash, PyObject **result); +extern void _PyDict_Clear_LockHeld(PyObject *op); + #ifdef Py_GIL_DISABLED PyAPI_FUNC(void) _PyDict_EnsureSharedOnRead(PyDictObject *mp); #endif @@ -276,7 +286,7 @@ _PyDict_NotifyEvent(PyInterpreterState *interp, PyObject *value) { assert(Py_REFCNT((PyObject*)mp) > 0); - int watcher_bits = mp->_ma_watcher_tag & DICT_WATCHER_MASK; + int watcher_bits = FT_ATOMIC_LOAD_UINT64_ACQUIRE(mp->_ma_watcher_tag) & DICT_WATCHER_MASK; if (watcher_bits) { RARE_EVENT_STAT_INC(watched_dict_modification); _PyDict_SendEvent(watcher_bits, event, mp, key, value); @@ -352,7 +362,7 @@ PyDictObject *_PyObject_MaterializeManagedDict_LockHeld(PyObject *); static inline Py_ssize_t _PyDict_UniqueId(PyDictObject *mp) { - return (Py_ssize_t)(mp->_ma_watcher_tag >> DICT_UNIQUE_ID_SHIFT); + return (Py_ssize_t)(FT_ATOMIC_LOAD_UINT64_RELAXED(mp->_ma_watcher_tag) >> DICT_UNIQUE_ID_SHIFT); } static inline void diff --git a/Include/internal/pycore_dict_state.h b/Include/internal/pycore_dict_state.h index 11932b8d1e1ab60..bb6fe2625975596 100644 --- a/Include/internal/pycore_dict_state.h +++ b/Include/internal/pycore_dict_state.h @@ -13,6 +13,8 @@ extern "C" { struct _Py_dict_state { uint32_t next_keys_version; + PyMutex watcher_mutex; // Protects the watchers array (free-threaded builds) + _PyOnceFlag watcher_setup_once; // One-time optimizer watcher setup PyDict_WatchCallback watchers[DICT_MAX_WATCHERS]; }; diff --git a/Include/internal/pycore_emscripten_trampoline.h b/Include/internal/pycore_emscripten_trampoline.h index 16916f1a8eb16cc..e37c53a64f4a723 100644 --- a/Include/internal/pycore_emscripten_trampoline.h +++ b/Include/internal/pycore_emscripten_trampoline.h @@ -27,9 +27,6 @@ #if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) -void -_Py_EmscriptenTrampoline_Init(_PyRuntimeState *runtime); - PyObject* _PyEM_TrampolineCall(PyCFunctionWithKeywords func, PyObject* self, diff --git a/Include/internal/pycore_function.h b/Include/internal/pycore_function.h index 209252b2ddc0de4..69b9ad548295c7d 100644 --- a/Include/internal/pycore_function.h +++ b/Include/internal/pycore_function.h @@ -35,6 +35,29 @@ PyFunctionObject *_PyFunction_LookupByVersion(uint32_t version, PyObject **p_cod extern PyObject *_Py_set_function_type_params( PyThreadState* unused, PyObject *func, PyObject *type_params); + +/* See pycore_code.h for explanation about what "stateless" means. */ + +PyAPI_FUNC(int) +_PyFunction_VerifyStateless(PyThreadState *, PyObject *); + +static inline PyObject* _PyFunction_GET_BUILTINS(PyObject *func) { + return _PyFunction_CAST(func)->func_builtins; +} +#define _PyFunction_GET_BUILTINS(func) _PyFunction_GET_BUILTINS(_PyObject_CAST(func)) + + +/* Get the callable wrapped by a classmethod. + Returns a borrowed reference. + The caller must ensure 'cm' is a classmethod object. */ +extern PyObject *_PyClassMethod_GetFunc(PyObject *cm); + +/* Get the callable wrapped by a staticmethod. + Returns a borrowed reference. + The caller must ensure 'sm' is a staticmethod object. */ +extern PyObject *_PyStaticMethod_GetFunc(PyObject *sm); + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h index a6519aa086309de..bfe52f42f1141cc 100644 --- a/Include/internal/pycore_gc.h +++ b/Include/internal/pycore_gc.h @@ -118,21 +118,6 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) { /* Bit 1 is set when the object is in generation which is GCed currently. */ #define _PyGC_PREV_MASK_COLLECTING ((uintptr_t)2) -/* Bit 0 in _gc_next is the old space bit. - * It is set as follows: - * Young: gcstate->visited_space - * old[0]: 0 - * old[1]: 1 - * permanent: 0 - * - * During a collection all objects handled should have the bit set to - * gcstate->visited_space, as objects are moved from the young gen - * and the increment into old[gcstate->visited_space]. - * When object are moved from the pending space, old[gcstate->visited_space^1] - * into the increment, the old space bit is flipped. -*/ -#define _PyGC_NEXT_MASK_OLD_SPACE_1 1 - #define _PyGC_PREV_SHIFT 2 #define _PyGC_PREV_MASK (((uintptr_t) -1) << _PyGC_PREV_SHIFT) @@ -159,13 +144,11 @@ typedef enum { // Lowest bit of _gc_next is used for flags only in GC. // But it is always 0 for normal code. static inline PyGC_Head* _PyGCHead_NEXT(PyGC_Head *gc) { - uintptr_t next = gc->_gc_next & _PyGC_PREV_MASK; + uintptr_t next = gc->_gc_next; return (PyGC_Head*)next; } static inline void _PyGCHead_SET_NEXT(PyGC_Head *gc, PyGC_Head *next) { - uintptr_t unext = (uintptr_t)next; - assert((unext & ~_PyGC_PREV_MASK) == 0); - gc->_gc_next = (gc->_gc_next & ~_PyGC_PREV_MASK) | unext; + gc->_gc_next = (uintptr_t)next; } // Lowest two bits of _gc_prev is used for _PyGC_PREV_MASK_* flags. @@ -205,6 +188,8 @@ static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) { #endif } +extern void _Py_ScheduleGC(PyThreadState *tstate); + /* Tell the GC to track this object. * @@ -214,7 +199,7 @@ static inline void _PyGC_CLEAR_FINALIZED(PyObject *op) { * ob_traverse method. * * Internal note: interp->gc.generation0->_gc_prev doesn't have any bit flags - * because it's not object header. So we don't use _PyGCHead_PREV() and + * because it's not an object header. So we don't use _PyGCHead_PREV() and * _PyGCHead_SET_PREV() for it to avoid unnecessary bitwise operations. * * See also the public PyObject_GC_Track() function. @@ -238,14 +223,14 @@ static inline void _PyObject_GC_TRACK( "object is in generation which is garbage collected", filename, lineno, __func__); - PyInterpreterState *interp = _PyInterpreterState_GET(); - PyGC_Head *generation0 = &interp->gc.young.head; + struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc; + PyGC_Head *generation0 = gcstate->generation0; PyGC_Head *last = (PyGC_Head*)(generation0->_gc_prev); _PyGCHead_SET_NEXT(last, gc); _PyGCHead_SET_PREV(gc, last); - uintptr_t not_visited = 1 ^ interp->gc.visited_space; - gc->_gc_next = ((uintptr_t)generation0) | not_visited; + _PyGCHead_SET_NEXT(gc, generation0); generation0->_gc_prev = (uintptr_t)gc; + gcstate->heap_size++; #endif } @@ -280,6 +265,8 @@ static inline void _PyObject_GC_UNTRACK( _PyGCHead_SET_PREV(next, prev); gc->_gc_next = 0; gc->_gc_prev &= _PyGC_PREV_MASK_FINALIZED; + struct _gc_runtime_state *gcstate = &_PyInterpreterState_GET()->gc; + gcstate->heap_size--; #endif } @@ -343,7 +330,6 @@ extern PyObject *_PyGC_GetReferrers(PyInterpreterState *interp, PyObject *objs); // Functions to clear types free lists extern void _PyGC_ClearAllFreeLists(PyInterpreterState *interp); -extern void _Py_ScheduleGC(PyThreadState *tstate); extern void _Py_RunGC(PyThreadState *tstate); union _PyStackRef; diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 9bde4faaf5a0400..5997858378cfe0a 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -796,6 +796,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(alias)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(align)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(all_threads)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(allow_code)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(any)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(append)); @@ -893,6 +894,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(data)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(database)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(day)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(debug)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decode)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(decoder)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(default)); @@ -904,7 +906,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(deterministic)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(device)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dict)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dict_content)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dictcomp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(difference_update)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(digest)); @@ -1139,6 +1140,7 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(offset_src)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(on_type_read)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(onceregistry)); + _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(only_active_thread)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(only_keys)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(oparg)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(opcode)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index 3a83fd6b6042e21..bab5da70f3400c1 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -287,6 +287,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(alias) STRUCT_FOR_ID(align) STRUCT_FOR_ID(all) + STRUCT_FOR_ID(all_threads) STRUCT_FOR_ID(allow_code) STRUCT_FOR_ID(any) STRUCT_FOR_ID(append) @@ -384,6 +385,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(data) STRUCT_FOR_ID(database) STRUCT_FOR_ID(day) + STRUCT_FOR_ID(debug) STRUCT_FOR_ID(decode) STRUCT_FOR_ID(decoder) STRUCT_FOR_ID(default) @@ -395,7 +397,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(deterministic) STRUCT_FOR_ID(device) STRUCT_FOR_ID(dict) - STRUCT_FOR_ID(dict_content) STRUCT_FOR_ID(dictcomp) STRUCT_FOR_ID(difference_update) STRUCT_FOR_ID(digest) @@ -630,6 +631,7 @@ struct _Py_global_strings { STRUCT_FOR_ID(offset_src) STRUCT_FOR_ID(on_type_read) STRUCT_FOR_ID(onceregistry) + STRUCT_FOR_ID(only_active_thread) STRUCT_FOR_ID(only_keys) STRUCT_FOR_ID(oparg) STRUCT_FOR_ID(opcode) diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 368dafb90635d7b..183b2d45c5ede1c 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -153,10 +153,8 @@ typedef enum { } _PyConfigInitEnum; typedef enum { - /* For now, this means the GIL is enabled. - - gh-116329: This will eventually change to "the GIL is disabled but can - be re-enabled by loading an incompatible extension module." */ + /* In free threaded builds, this means that the GIL is disabled at startup, + but may be enabled by loading an incompatible extension module. */ _PyConfig_GIL_DEFAULT = -1, /* The GIL has been forced off or on, and will not be affected by module loading. */ diff --git a/Include/internal/pycore_instruments.h b/Include/internal/pycore_instruments.h index 7658adca719e86d..98684ff1024f6e4 100644 --- a/Include/internal/pycore_instruments.h +++ b/Include/internal/pycore_instruments.h @@ -62,9 +62,6 @@ extern void _Py_call_instrumentation_exc2(PyThreadState *tstate, int event, _PyInterpreterFrame *frame, _Py_CODEUNIT *instr, PyObject *arg0, PyObject *arg1); -extern int -_Py_Instrumentation_GetLine(PyCodeObject *code, int index); - extern PyObject _PyInstrumentation_MISSING; extern PyObject _PyInstrumentation_DISABLE; @@ -120,6 +117,8 @@ typedef struct _PyCoMonitoringData { uint8_t *per_instruction_tools; } _PyCoMonitoringData; +extern int +_Py_Instrumentation_GetLine(PyCodeObject *code, _PyCoLineInstrumentationData *line_data, int index); #ifdef __cplusplus } diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index c3e6c77405bfe7a..b93d6d3c8eda8b3 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -87,6 +87,9 @@ struct _ceval_runtime_state { struct trampoline_api_st trampoline_api; FILE *map_file; Py_ssize_t persist_after_fork; + _PyFrameEvalFunction prev_eval_frame; + Py_ssize_t trampoline_refcount; + int code_watcher_id; #else int _not_used; #endif @@ -97,7 +100,7 @@ struct _ceval_runtime_state { // For example, we use a preallocated array // for the list of pending calls. struct _pending_calls pending_mainthread; - PyMutex sys_trace_profile_mutex; + PyMutex unused_sys_trace_profile_mutex; // kept for ABI compatibility }; @@ -192,11 +195,6 @@ struct gc_generation_stats { Py_ssize_t uncollectable; }; -enum _GCPhase { - GC_PHASE_MARK = 0, - GC_PHASE_COLLECT = 1 -}; - /* If we change this, we need to change the default value in the signature of gc.collect. */ #define NUM_GENERATIONS 3 @@ -212,8 +210,13 @@ struct _gc_runtime_state { int enabled; int debug; /* linked lists of container objects */ +#ifndef Py_GIL_DISABLED + struct gc_generation generations[NUM_GENERATIONS]; +#else struct gc_generation young; struct gc_generation old[2]; +#endif + /* a permanent generation which won't be collected */ struct gc_generation permanent_generation; struct gc_generation_stats generation_stats[NUM_GENERATIONS]; @@ -224,13 +227,14 @@ struct _gc_runtime_state { /* a list of callbacks to be invoked when collection is performed */ PyObject *callbacks; + /* The number of live objects. */ Py_ssize_t heap_size; - Py_ssize_t work_to_do; - /* Which of the old spaces is the visited space */ - int visited_space; - int phase; -#ifdef Py_GIL_DISABLED + /* dummy members to preserve other offsets */ + Py_ssize_t dummy1; /* was work_to_do */ + int dummy2; /* was visited_space */ + int dummy3; /* was phase */ + /* This is the number of objects that survived the last full collection. It approximates the number of long lived objects tracked by the GC. @@ -243,6 +247,7 @@ struct _gc_runtime_state { the first time. */ Py_ssize_t long_lived_pending; +#ifdef Py_GIL_DISABLED /* True if gc.freeze() has been used. */ int freeze_active; @@ -255,9 +260,28 @@ struct _gc_runtime_state { /* Mutex held for gc_should_collect_mem_usage(). */ PyMutex mutex; +#else + PyGC_Head *generation0; #endif }; +#ifndef Py_GIL_DISABLED +#define GC_GENERATION_INIT \ + .generations = { \ + { .threshold = 2000, }, \ + { .threshold = 10, }, \ + { .threshold = 10, }, \ + }, \ + .heap_size = 0, +#else +#define GC_GENERATION_INIT \ + .young = { .threshold = 2000, }, \ + .old = { \ + { .threshold = 10, }, \ + { .threshold = 10, }, \ + }, +#endif + #include "pycore_gil.h" // struct _gil_runtime_state /**** Import ********/ @@ -726,6 +750,10 @@ typedef struct _PyIndexPool { // Next index to allocate if no free indices are available int32_t next_index; + + // Generation counter incremented on thread creation/destruction + // Used for TLBC cache invalidation in remote debugging + uint32_t tlbc_generation; } _PyIndexPool; typedef union _Py_unique_id_entry { @@ -764,10 +792,7 @@ struct _is { * and should be placed at the beginning. */ struct _ceval_state ceval; - /* This structure is carefully allocated so that it's correctly aligned - * to avoid undefined behaviors during LOAD and STORE. The '_malloced' - * field stores the allocated pointer address that will later be freed. - */ + // unused, kept for ABI compatibility void *_malloced; PyInterpreterState *next; @@ -843,6 +868,8 @@ struct _is { /* The per-interpreter GIL, which might not be used. */ struct _gil_runtime_state _gil; + uint64_t _code_object_generation; + /* ---------- IMPORTANT --------------------------- The fields above this line are declared as early as possible to facilitate out-of-process observability @@ -940,8 +967,8 @@ struct _is { PyDict_WatchCallback builtins_dict_watcher; _Py_GlobalMonitors monitors; - bool sys_profile_initialized; - bool sys_trace_initialized; + _PyOnceFlag sys_profile_once_flag; + _PyOnceFlag sys_trace_once_flag; Py_ssize_t sys_profiling_threads; /* Count of threads with c_profilefunc set */ Py_ssize_t sys_tracing_threads; /* Count of threads with c_tracefunc set */ PyObject *monitoring_callables[PY_MONITORING_TOOL_IDS][_PY_MONITORING_EVENTS]; diff --git a/Include/internal/pycore_interpframe.h b/Include/internal/pycore_interpframe.h index d3fd218b27eed7f..b5b729f141bab5d 100644 --- a/Include/internal/pycore_interpframe.h +++ b/Include/internal/pycore_interpframe.h @@ -24,6 +24,36 @@ static inline PyCodeObject *_PyFrame_GetCode(_PyInterpreterFrame *f) { return (PyCodeObject *)executable; } +// Similar to _PyFrame_GetCode(), but return NULL if the frame is invalid or +// freed. Used by dump_frame() in Python/traceback.c. The function uses +// heuristics to detect freed memory, it's not 100% reliable. +static inline PyCodeObject* +_PyFrame_SafeGetCode(_PyInterpreterFrame *f) +{ + // globals and builtins may be NULL on a legit frame, but it's unlikely. + // It's more likely that it's a sign of an invalid frame. + if (f->f_globals == NULL || f->f_builtins == NULL) { + return NULL; + } + + if (PyStackRef_IsNull(f->f_executable)) { + return NULL; + } + void *ptr; + memcpy(&ptr, &f->f_executable, sizeof(f->f_executable)); + if (_PyMem_IsPtrFreed(ptr)) { + return NULL; + } + PyObject *executable = PyStackRef_AsPyObjectBorrow(f->f_executable); + if (_PyObject_IsFreed(executable)) { + return NULL; + } + if (!PyCode_Check(executable)) { + return NULL; + } + return (PyCodeObject *)executable; +} + static inline _Py_CODEUNIT * _PyFrame_GetBytecode(_PyInterpreterFrame *f) { @@ -37,6 +67,31 @@ _PyFrame_GetBytecode(_PyInterpreterFrame *f) #endif } +// Similar to PyUnstable_InterpreterFrame_GetLasti(), but return NULL if the +// frame is invalid or freed. Used by dump_frame() in Python/traceback.c. The +// function uses heuristics to detect freed memory, it's not 100% reliable. +static inline int +_PyFrame_SafeGetLasti(struct _PyInterpreterFrame *f) +{ + // Code based on _PyFrame_GetBytecode() but replace _PyFrame_GetCode() + // with _PyFrame_SafeGetCode(). + PyCodeObject *co = _PyFrame_SafeGetCode(f); + if (co == NULL) { + return -1; + } + + _Py_CODEUNIT *bytecode; +#ifdef Py_GIL_DISABLED + _PyCodeArray *tlbc = _PyCode_GetTLBCArray(co); + assert(f->tlbc_index >= 0 && f->tlbc_index < tlbc->size); + bytecode = (_Py_CODEUNIT *)tlbc->entries[f->tlbc_index]; +#else + bytecode = _PyCode_CODE(co); +#endif + + return (int)(f->instr_ptr - bytecode) * sizeof(_Py_CODEUNIT); +} + static inline PyFunctionObject *_PyFrame_GetFunction(_PyInterpreterFrame *f) { PyObject *func = PyStackRef_AsPyObjectBorrow(f->f_funcobj); assert(PyFunction_Check(func)); @@ -94,6 +149,11 @@ static inline void _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame * int stacktop = (int)(src->stackpointer - src->localsplus); assert(stacktop >= 0); dest->stackpointer = dest->localsplus + stacktop; + // visited is GC bookkeeping for the current stack walk, not frame state. + dest->visited = 0; +#ifdef Py_DEBUG + dest->lltrace = src->lltrace; +#endif for (int i = 0; i < stacktop; i++) { dest->localsplus[i] = PyStackRef_MakeHeapSafe(src->localsplus[i]); } diff --git a/Include/internal/pycore_lock.h b/Include/internal/pycore_lock.h index 7484b05d7f2446a..9b071573ad3c749 100644 --- a/Include/internal/pycore_lock.h +++ b/Include/internal/pycore_lock.h @@ -25,13 +25,6 @@ PyMutex_LockFast(PyMutex *m) return _Py_atomic_compare_exchange_uint8(lock_bits, &expected, _Py_LOCKED); } -// Checks if the mutex is currently locked. -static inline int -PyMutex_IsLocked(PyMutex *m) -{ - return (_Py_atomic_load_uint8(&m->_bits) & _Py_LOCKED) != 0; -} - // Re-initializes the mutex after a fork to the unlocked state. static inline void _PyMutex_at_fork_reinit(PyMutex *m) diff --git a/Include/internal/pycore_long.h b/Include/internal/pycore_long.h index ed6c435316708ed..36d6d7d702b4d0d 100644 --- a/Include/internal/pycore_long.h +++ b/Include/internal/pycore_long.h @@ -64,7 +64,8 @@ PyAPI_FUNC(void) _PyLong_ExactDealloc(PyObject *self); # error "_PY_NSMALLPOSINTS must be greater than or equal to 257" #endif -#define _PY_IS_SMALL_INT(val) ((val) >= 0 && (val) < 256 && (val) < _PY_NSMALLPOSINTS) +#define _PY_IS_SMALL_INT(val) \ + (-_PY_NSMALLNEGINTS <= (val) && (val) < _PY_NSMALLPOSINTS) // Return a reference to the immortal zero singleton. // The function cannot return NULL. @@ -158,6 +159,11 @@ PyAPI_FUNC(int) _PyLong_UnsignedLongLong_Converter(PyObject *, void *); // Export for '_testclinic' shared extension (Argument Clinic code) PyAPI_FUNC(int) _PyLong_Size_t_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_UInt8_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_UInt16_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_UInt32_Converter(PyObject *, void *); +PyAPI_FUNC(int) _PyLong_UInt64_Converter(PyObject *, void *); + /* Long value tag bits: * 0-1: Sign bits value = (1-sign), ie. negative=2, positive=0, zero=1. * 2: Set to 1 for the small ints @@ -227,6 +233,20 @@ _PyLong_IsPositive(const PyLongObject *op) return (op->long_value.lv_tag & SIGN_MASK) == 0; } +/* Return true if the argument is a small int */ +static inline bool +_PyLong_IsSmallInt(const PyLongObject *op) +{ + assert(PyLong_Check(op)); + bool is_small_int = (op->long_value.lv_tag & IMMORTALITY_BIT_MASK) != 0; + assert(PyLong_CheckExact(op) || (!is_small_int)); + assert(_Py_IsImmortal(op) || (!is_small_int)); + assert((_PyLong_IsCompact(op) + && _PY_IS_SMALL_INT(_PyLong_CompactValue(op))) + || (!is_small_int)); + return is_small_int; +} + static inline Py_ssize_t _PyLong_DigitCount(const PyLongObject *op) { @@ -288,7 +308,9 @@ _PyLong_SetDigitCount(PyLongObject *op, Py_ssize_t size) #define NON_SIZE_MASK ~(uintptr_t)((1 << NON_SIZE_BITS) - 1) static inline void -_PyLong_FlipSign(PyLongObject *op) { +_PyLong_FlipSign(PyLongObject *op) +{ + assert(!_PyLong_IsSmallInt(op)); unsigned int flipped_sign = 2 - (op->long_value.lv_tag & SIGN_MASK); op->long_value.lv_tag &= NON_SIZE_MASK; op->long_value.lv_tag |= flipped_sign; diff --git a/Include/internal/pycore_magic_number.h b/Include/internal/pycore_magic_number.h index 1cc897c5a694696..f31951057121d92 100644 --- a/Include/internal/pycore_magic_number.h +++ b/Include/internal/pycore_magic_number.h @@ -277,6 +277,9 @@ Known values: Python 3.14a7 3622 (Store annotations in different class dict keys) Python 3.14a7 3623 (Add BUILD_INTERPOLATION & BUILD_TEMPLATE opcodes) Python 3.14b1 3624 (Don't optimize LOAD_FAST when local is killed by DELETE_FAST) + Python 3.14b3 3625 (Fix handling of opcodes that may leave operands on the stack when optimizing LOAD_FAST) + Python 3.14rc2 3626 (Fix missing exception handlers in logical expression) + Python 3.14rc3 3627 (Fix miscompilation of some module-level annotations) Python 3.15 will start with 3650 @@ -289,7 +292,7 @@ PC/launcher.c must also be updated. */ -#define PYC_MAGIC_NUMBER 3624 +#define PYC_MAGIC_NUMBER 3627 /* This is equivalent to converting PYC_MAGIC_NUMBER to 2 bytes (little-endian) and then appending b'\r\n'. */ #define PYC_MAGIC_NUMBER_TOKEN \ diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index b7e162c8abcabf7..a25998af96bb933 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -614,7 +614,7 @@ static inline PyObject * _Py_XGetRef(PyObject **ptr) { for (;;) { - PyObject *value = _Py_atomic_load_ptr(ptr); + PyObject *value = _PyObject_CAST(_Py_atomic_load_ptr(ptr)); if (value == NULL) { return value; } @@ -629,7 +629,7 @@ _Py_XGetRef(PyObject **ptr) static inline PyObject * _Py_TryXGetRef(PyObject **ptr) { - PyObject *value = _Py_atomic_load_ptr(ptr); + PyObject *value = _PyObject_CAST(_Py_atomic_load_ptr(ptr)); if (value == NULL) { return value; } @@ -875,7 +875,8 @@ static inline int _PyType_SUPPORTS_WEAKREFS(PyTypeObject *type) { return (type->tp_weaklistoffset != 0); } -extern PyObject* _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); +// Export for 'array' shared extension. +PyAPI_FUNC(PyObject*) _PyType_AllocNoTrack(PyTypeObject *type, Py_ssize_t nitems); PyAPI_FUNC(PyObject *) _PyType_NewManagedObject(PyTypeObject *type); extern PyTypeObject* _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); @@ -897,6 +898,9 @@ extern PyObject *_PyType_LookupRefAndVersion(PyTypeObject *, PyObject *, extern unsigned int _PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef *out); +extern int _PyObject_GetMethodStackRef(PyThreadState *ts, _PyStackRef *self, + PyObject *name, _PyStackRef *method); + // Cache the provided init method in the specialization cache of type if the // provided type version matches the current version of the type. // @@ -1007,6 +1011,22 @@ enum _PyAnnotateFormat { _Py_ANNOTATE_FORMAT_STRING = 4, }; +int _PyObject_SetDict(PyObject *obj, PyObject *value); + +#ifndef Py_GIL_DISABLED +static inline Py_ALWAYS_INLINE void _Py_INCREF_MORTAL(PyObject *op) +{ + assert(!_Py_IsStaticImmortal(op)); + op->ob_refcnt++; + _Py_INCREF_STAT_INC(); +#if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API) + if (!_Py_IsImmortal(op)) { + _Py_INCREF_IncRefTotal(); + } +#endif +} +#endif + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 0e34074f1600a71..273e1d8381be74f 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1082,7 +1082,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [BINARY_OP_ADD_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_ADD_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, - [BINARY_OP_EXTEND] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG }, + [BINARY_OP_EXTEND] = { true, INSTR_FMT_IXC0000, HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC0000, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [BINARY_OP_MULTIPLY_FLOAT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_MULTIPLY_INT] = { true, INSTR_FMT_IXC0000, HAS_EXIT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1271,7 +1271,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [SETUP_ANNOTATIONS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [SET_FUNCTION_ATTRIBUTE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [SET_UPDATE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [STORE_ATTR_INSTANCE_VALUE] = { true, INSTR_FMT_IXC000, HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, @@ -1304,7 +1304,7 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[267] = { [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG }, [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_ESCAPES_FLAG }, [WITH_EXCEPT_START] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, - [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ESCAPES_FLAG }, [ANNOTATIONS_PLACEHOLDER] = { true, -1, HAS_PURE_FLAG }, [JUMP] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, [JUMP_IF_FALSE] = { true, -1, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG }, @@ -1366,7 +1366,7 @@ _PyOpcode_macro_expansion[256] = { [CALL_ISINSTANCE] = { .nuops = 1, .uops = { { _CALL_ISINSTANCE, OPARG_SIMPLE, 3 } } }, [CALL_KW_BOUND_METHOD] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_METHOD_VERSION_KW, 2, 1 }, { _EXPAND_METHOD_KW, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_KW_NON_PY] = { .nuops = 3, .uops = { { _CHECK_IS_NOT_PY_CALLABLE_KW, OPARG_SIMPLE, 3 }, { _CALL_KW_NON_PY, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, - [CALL_KW_PY] = { .nuops = 5, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, + [CALL_KW_PY] = { .nuops = 6, .uops = { { _CHECK_PEP_523, OPARG_SIMPLE, 1 }, { _CHECK_FUNCTION_VERSION_KW, 2, 1 }, { _CHECK_RECURSION_REMAINING, OPARG_SIMPLE, 3 }, { _PY_FRAME_KW, OPARG_SIMPLE, 3 }, { _SAVE_RETURN_OFFSET, OPARG_SAVE_RETURN_OFFSET, 3 }, { _PUSH_FRAME, OPARG_SIMPLE, 3 } } }, [CALL_LEN] = { .nuops = 3, .uops = { { _GUARD_NOS_NULL, OPARG_SIMPLE, 3 }, { _GUARD_CALLABLE_LEN, OPARG_SIMPLE, 3 }, { _CALL_LEN, OPARG_SIMPLE, 3 } } }, [CALL_LIST_APPEND] = { .nuops = 1, .uops = { { _CALL_LIST_APPEND, OPARG_SIMPLE, 3 } } }, [CALL_METHOD_DESCRIPTOR_FAST] = { .nuops = 2, .uops = { { _CALL_METHOD_DESCRIPTOR_FAST, OPARG_SIMPLE, 3 }, { _CHECK_PERIODIC, OPARG_SIMPLE, 3 } } }, @@ -1787,6 +1787,35 @@ const uint8_t _PyOpcode_Caches[256] = { extern const uint8_t _PyOpcode_Deopt[256]; #ifdef NEED_OPCODE_METADATA const uint8_t _PyOpcode_Deopt[256] = { + [121] = 121, + [122] = 122, + [123] = 123, + [124] = 124, + [125] = 125, + [126] = 126, + [127] = 127, + [212] = 212, + [213] = 213, + [214] = 214, + [215] = 215, + [216] = 216, + [217] = 217, + [218] = 218, + [219] = 219, + [220] = 220, + [221] = 221, + [222] = 222, + [223] = 223, + [224] = 224, + [225] = 225, + [226] = 226, + [227] = 227, + [228] = 228, + [229] = 229, + [230] = 230, + [231] = 231, + [232] = 232, + [233] = 233, [BINARY_OP] = BINARY_OP, [BINARY_OP_ADD_FLOAT] = BINARY_OP, [BINARY_OP_ADD_INT] = BINARY_OP, diff --git a/Include/internal/pycore_opcode_utils.h b/Include/internal/pycore_opcode_utils.h index 62af06dc01c125d..79a1a242556a52b 100644 --- a/Include/internal/pycore_opcode_utils.h +++ b/Include/internal/pycore_opcode_utils.h @@ -56,6 +56,8 @@ extern "C" { #define IS_RETURN_OPCODE(opcode) \ (opcode == RETURN_VALUE) +#define IS_RAISE_OPCODE(opcode) \ + (opcode == RAISE_VARARGS || opcode == RERAISE) /* Flags used in the oparg for MAKE_FUNCTION */ diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h index 3e41e2fd1569caf..bcb9e01a5c8e258 100644 --- a/Include/internal/pycore_pyatomic_ft_wrappers.h +++ b/Include/internal/pycore_pyatomic_ft_wrappers.h @@ -31,6 +31,8 @@ extern "C" { _Py_atomic_store_ptr(&value, new_value) #define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) \ _Py_atomic_load_ptr_acquire(&value) +#define FT_ATOMIC_LOAD_PTR_CONSUME(value) \ + _Py_atomic_load_ptr_consume(&value) #define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) \ _Py_atomic_load_uintptr_acquire(&value) #define FT_ATOMIC_LOAD_PTR_RELAXED(value) \ @@ -45,6 +47,10 @@ extern "C" { _Py_atomic_load_uint16_relaxed(&value) #define FT_ATOMIC_LOAD_UINT32_RELAXED(value) \ _Py_atomic_load_uint32_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT64_ACQUIRE(value) \ + _Py_atomic_load_uint64_acquire(&value) +#define FT_ATOMIC_LOAD_UINT64_RELAXED(value) \ + _Py_atomic_load_uint64_relaxed(&value) #define FT_ATOMIC_LOAD_ULONG_RELAXED(value) \ _Py_atomic_load_ulong_relaxed(&value) #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \ @@ -61,6 +67,12 @@ extern "C" { _Py_atomic_store_uint16_relaxed(&value, new_value) #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) \ _Py_atomic_store_uint32_relaxed(&value, new_value) +#define FT_ATOMIC_AND_UINT64(value, new_value) \ + (void)_Py_atomic_and_uint64(&value, new_value) +#define FT_ATOMIC_OR_UINT64(value, new_value) \ + (void)_Py_atomic_or_uint64(&value, new_value) +#define FT_ATOMIC_ADD_UINT64(value, new_value) \ + (void)_Py_atomic_add_uint64(&value, new_value) #define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) \ _Py_atomic_store_char_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) \ @@ -77,10 +89,16 @@ extern "C" { _Py_atomic_store_ushort_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_USHORT_RELAXED(value) \ _Py_atomic_load_ushort_relaxed(&value) +#define FT_ATOMIC_LOAD_INT(value) \ + _Py_atomic_load_int(&value) +#define FT_ATOMIC_STORE_INT(value, new_value) \ + _Py_atomic_store_int(&value, new_value) #define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) \ _Py_atomic_store_int_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_INT_RELAXED(value) \ _Py_atomic_load_int_relaxed(&value) +#define FT_ATOMIC_LOAD_UINT(value) \ + _Py_atomic_load_uint(&value) #define FT_ATOMIC_STORE_UINT_RELAXED(value, new_value) \ _Py_atomic_store_uint_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_UINT_RELAXED(value) \ @@ -93,6 +111,8 @@ extern "C" { _Py_atomic_store_ulong_relaxed(&value, new_value) #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) \ _Py_atomic_store_ssize_relaxed(&value, new_value) +#define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) \ + _Py_atomic_store_ssize_release(&value, new_value) #define FT_ATOMIC_STORE_FLOAT_RELAXED(value, new_value) \ _Py_atomic_store_float_relaxed(&value, new_value) #define FT_ATOMIC_LOAD_FLOAT_RELAXED(value) \ @@ -111,6 +131,9 @@ extern "C" { _Py_atomic_load_ullong_relaxed(&value) #define FT_ATOMIC_ADD_SSIZE(value, new_value) \ (void)_Py_atomic_add_ssize(&value, new_value) +#define FT_MUTEX_LOCK(lock) PyMutex_Lock(lock) +#define FT_MUTEX_LOCK_FLAGS(lock, flags) PyMutex_LockFlags(lock, flags) +#define FT_MUTEX_UNLOCK(lock) PyMutex_Unlock(lock) #else #define FT_ATOMIC_LOAD_PTR(value) value @@ -119,6 +142,7 @@ extern "C" { #define FT_ATOMIC_LOAD_SSIZE_ACQUIRE(value) value #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value #define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) value +#define FT_ATOMIC_LOAD_PTR_CONSUME(value) value #define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) value #define FT_ATOMIC_LOAD_PTR_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT8(value) value @@ -126,14 +150,20 @@ extern "C" { #define FT_ATOMIC_LOAD_UINT8_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT16_RELAXED(value) value #define FT_ATOMIC_LOAD_UINT32_RELAXED(value) value +#define FT_ATOMIC_LOAD_UINT64_ACQUIRE(value) value +#define FT_ATOMIC_LOAD_UINT64_RELAXED(value) value #define FT_ATOMIC_LOAD_ULONG_RELAXED(value) value #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_STORE_SSIZE_RELEASE(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT16_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_STORE_UINT32_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_AND_UINT64(value, new_value) (void)(value &= new_value) +#define FT_ATOMIC_OR_UINT64(value, new_value) (void)(value |= new_value) +#define FT_ATOMIC_ADD_UINT64(value, new_value) (void)(value += new_value) #define FT_ATOMIC_LOAD_CHAR_RELAXED(value) value #define FT_ATOMIC_STORE_CHAR_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_UCHAR_RELAXED(value) value @@ -142,8 +172,11 @@ extern "C" { #define FT_ATOMIC_STORE_SHORT_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_USHORT_RELAXED(value) value #define FT_ATOMIC_STORE_USHORT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_INT(value) value +#define FT_ATOMIC_STORE_INT(value, new_value) value = new_value #define FT_ATOMIC_LOAD_INT_RELAXED(value) value #define FT_ATOMIC_STORE_INT_RELAXED(value, new_value) value = new_value +#define FT_ATOMIC_LOAD_UINT(value) value #define FT_ATOMIC_LOAD_UINT_RELAXED(value) value #define FT_ATOMIC_STORE_UINT_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_LOAD_LONG_RELAXED(value) value @@ -159,6 +192,9 @@ extern "C" { #define FT_ATOMIC_LOAD_ULLONG_RELAXED(value) value #define FT_ATOMIC_STORE_ULLONG_RELAXED(value, new_value) value = new_value #define FT_ATOMIC_ADD_SSIZE(value, new_value) (void)(value += new_value) +#define FT_MUTEX_LOCK(lock) do {} while (0) +#define FT_MUTEX_LOCK_FLAGS(lock, flags) do {} while (0) +#define FT_MUTEX_UNLOCK(lock) do {} while (0) #endif diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index f357b88e220e6e0..2c2048f7e1272a3 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -94,13 +94,13 @@ extern void _PyErr_Fetch( PyObject **value, PyObject **traceback); -extern PyObject* _PyErr_GetRaisedException(PyThreadState *tstate); +PyAPI_FUNC(PyObject*) _PyErr_GetRaisedException(PyThreadState *tstate); PyAPI_FUNC(int) _PyErr_ExceptionMatches( PyThreadState *tstate, PyObject *exc); -extern void _PyErr_SetRaisedException(PyThreadState *tstate, PyObject *exc); +PyAPI_FUNC(void) _PyErr_SetRaisedException(PyThreadState *tstate, PyObject *exc); extern void _PyErr_Restore( PyThreadState *tstate, diff --git a/Include/internal/pycore_pyhash.h b/Include/internal/pycore_pyhash.h index 84cb72fa6fd1b26..3056dc44cc0f1b1 100644 --- a/Include/internal/pycore_pyhash.h +++ b/Include/internal/pycore_pyhash.h @@ -27,14 +27,14 @@ _Py_HashPointerRaw(const void *ptr) * pppppppp ssssssss ........ fnv -- two Py_hash_t * k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t * ........ ........ ssssssss djbx33a -- 16 bytes padding + one Py_hash_t - * ........ ........ eeeeeeee pyexpat XML hash salt + * eeeeeeee eeeeeeee eeeeeeee pyexpat XML hash salt * * memory layout on 32 bit systems * cccccccc cccccccc cccccccc uc * ppppssss ........ ........ fnv -- two Py_hash_t * k0k0k0k0 k1k1k1k1 ........ siphash -- two uint64_t (*) * ........ ........ ssss.... djbx33a -- 16 bytes padding + one Py_hash_t - * ........ ........ eeee.... pyexpat XML hash salt + * eeeeeeee eeeeeeee eeee.... pyexpat XML hash salt * * (*) The siphash member may not be available on 32 bit platforms without * an unsigned int64 data type. @@ -58,7 +58,9 @@ typedef union { Py_hash_t suffix; } djbx33a; struct { - unsigned char padding[16]; + /* 16 bytes for XML_SetHashSalt16Bytes */ + uint8_t hashsalt16[16]; + /* 4/8 bytes for legacy XML_SetHashSalt */ Py_hash_t hashsalt; } expat; } _Py_HashSecret_t; diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 6e89ca33e4208c6..386ce2e145fb756 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -41,6 +41,7 @@ extern PyStatus _Py_HashRandomization_Init(const PyConfig *); extern PyStatus _PyGC_Init(PyInterpreterState *interp); extern PyStatus _PyAtExit_Init(PyInterpreterState *interp); +extern PyStatus _PyDateTime_InitTypes(PyInterpreterState *interp); /* Various internal finalizers */ @@ -110,9 +111,6 @@ extern int _Py_LegacyLocaleDetected(int warn); // Export for 'readline' shared extension PyAPI_FUNC(char*) _Py_SetLocaleFromEnv(int category); -// Export for special main.c string compiling with source tracebacks -int _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompilerFlags *flags); - /* interpreter config */ diff --git a/Include/internal/pycore_pymath.h b/Include/internal/pycore_pymath.h index eea8996ba68ca08..5bbbdaab8f42dcd 100644 --- a/Include/internal/pycore_pymath.h +++ b/Include/internal/pycore_pymath.h @@ -146,17 +146,17 @@ extern void _Py_set_387controlword(unsigned short); unsigned int old_fpcr, new_fpcr #define _Py_SET_53BIT_PRECISION_START \ do { \ - __asm__ ("fmove.l %%fpcr,%0" : "=g" (old_fpcr)); \ + __asm__ ("fmove.l %%fpcr,%0" : "=dm" (old_fpcr)); \ /* Set double precision / round to nearest. */ \ new_fpcr = (old_fpcr & ~0xf0) | 0x80; \ if (new_fpcr != old_fpcr) { \ - __asm__ volatile ("fmove.l %0,%%fpcr" : : "g" (new_fpcr));\ + __asm__ volatile ("fmove.l %0,%%fpcr" : : "dm" (new_fpcr)); \ } \ } while (0) #define _Py_SET_53BIT_PRECISION_END \ do { \ if (new_fpcr != old_fpcr) { \ - __asm__ volatile ("fmove.l %0,%%fpcr" : : "g" (old_fpcr)); \ + __asm__ volatile ("fmove.l %0,%%fpcr" : : "dm" (old_fpcr)); \ } \ } while (0) #endif diff --git a/Include/internal/pycore_pymem.h b/Include/internal/pycore_pymem.h index 02537bdfef85989..f0b90a25597d0d8 100644 --- a/Include/internal/pycore_pymem.h +++ b/Include/internal/pycore_pymem.h @@ -54,20 +54,43 @@ static inline int _PyMem_IsPtrFreed(const void *ptr) { uintptr_t value = (uintptr_t)ptr; #if SIZEOF_VOID_P == 8 - return (value == 0 + return (value <= 0xff // NULL, 0x1, 0x2, ..., 0xff || value == (uintptr_t)0xCDCDCDCDCDCDCDCD || value == (uintptr_t)0xDDDDDDDDDDDDDDDD - || value == (uintptr_t)0xFDFDFDFDFDFDFDFD); + || value == (uintptr_t)0xFDFDFDFDFDFDFDFD + || value >= (uintptr_t)0xFFFFFFFFFFFFFF00); // -0xff, ..., -2, -1 #elif SIZEOF_VOID_P == 4 - return (value == 0 + return (value <= 0xff || value == (uintptr_t)0xCDCDCDCD || value == (uintptr_t)0xDDDDDDDD - || value == (uintptr_t)0xFDFDFDFD); + || value == (uintptr_t)0xFDFDFDFD + || value >= (uintptr_t)0xFFFFFF00); #else # error "unknown pointer size" #endif } +// Similar to _PyMem_IsPtrFreed() but expects an 'unsigned long' instead of a +// pointer. +static inline int _PyMem_IsULongFreed(unsigned long value) +{ +#if SIZEOF_LONG == 8 + return (value == 0 + || value == (unsigned long)0xCDCDCDCDCDCDCDCD + || value == (unsigned long)0xDDDDDDDDDDDDDDDD + || value == (unsigned long)0xFDFDFDFDFDFDFDFD + || value == (unsigned long)0xFFFFFFFFFFFFFFFF); +#elif SIZEOF_LONG == 4 + return (value == 0 + || value == (unsigned long)0xCDCDCDCD + || value == (unsigned long)0xDDDDDDDD + || value == (unsigned long)0xFDFDFDFD + || value == (unsigned long)0xFFFFFFFF); +#else +# error "unknown long size" +#endif +} + extern int _PyMem_GetAllocatorName( const char *name, PyMemAllocatorName *allocator); @@ -88,7 +111,7 @@ extern wchar_t *_PyMem_DefaultRawWcsdup(const wchar_t *str); extern int _PyMem_DebugEnabled(void); // Enqueue a pointer to be freed possibly after some delay. -extern void _PyMem_FreeDelayed(void *ptr); +extern void _PyMem_FreeDelayed(void *ptr, size_t size); // Enqueue an object to be freed possibly after some delay #ifdef Py_GIL_DISABLED diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 633e5cf77db918c..7f1b0b7523b20d9 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -8,6 +8,7 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif +#include "pycore_pythonrun.h" // _PyOS_STACK_MARGIN_SHIFT #include "pycore_typedefs.h" // _PyRuntimeState #include "pycore_tstate.h" @@ -325,7 +326,11 @@ _Py_RecursionLimit_GetMargin(PyThreadState *tstate) _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; assert(_tstate->c_stack_hard_limit != 0); intptr_t here_addr = _Py_get_machine_stack_pointer(); - return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, here_addr - (intptr_t)_tstate->c_stack_soft_limit, PYOS_STACK_MARGIN_SHIFT); +#if _Py_STACK_GROWS_DOWN + return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, here_addr - (intptr_t)_tstate->c_stack_soft_limit, _PyOS_STACK_MARGIN_SHIFT); +#else + return Py_ARITHMETIC_RIGHT_SHIFT(intptr_t, (intptr_t)_tstate->c_stack_soft_limit - here_addr, _PyOS_STACK_MARGIN_SHIFT); +#endif } #ifdef __cplusplus diff --git a/Include/internal/pycore_pythonrun.h b/Include/internal/pycore_pythonrun.h index 0bfc5704dc4c594..5dd93bd61b6e6b2 100644 --- a/Include/internal/pycore_pythonrun.h +++ b/Include/internal/pycore_pythonrun.h @@ -8,23 +8,19 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -extern int _PyRun_SimpleFileObject( +extern PyObject* _PyRun_SimpleFile( FILE *fp, PyObject *filename, int closeit, PyCompilerFlags *flags); -extern int _PyRun_AnyFileObject( +extern PyObject* _PyRun_AnyFile( FILE *fp, PyObject *filename, int closeit, PyCompilerFlags *flags); -extern int _PyRun_InteractiveLoopObject( - FILE *fp, - PyObject *filename, - PyCompilerFlags *flags); - +extern int _PyObject_SupportedAsScript(PyObject *); extern const char* _Py_SourceAsString( PyObject *cmd, const char *funcname, @@ -32,6 +28,40 @@ extern const char* _Py_SourceAsString( PyCompilerFlags *cf, PyObject **cmd_copy); +// Export for special main.c string compiling with source tracebacks +extern PyObject* _PyRun_SimpleString( + const char *command, + PyObject* name, + PyCompilerFlags *flags); + + +/* Stack size, in "pointers". This must be large enough, so + * no two calls to check recursion depth are more than this far + * apart. In practice, that means it must be larger than the C + * stack consumption of PyEval_EvalDefault */ +#if (defined(Py_DEBUG) \ + || defined(_Py_ADDRESS_SANITIZER) \ + || defined(_Py_THREAD_SANITIZER)) +# define _PyOS_LOG2_STACK_MARGIN 12 +#else +# define _PyOS_LOG2_STACK_MARGIN 11 +#endif +#define _PyOS_STACK_MARGIN (1 << _PyOS_LOG2_STACK_MARGIN) +#define _PyOS_STACK_MARGIN_BYTES (_PyOS_STACK_MARGIN * sizeof(void *)) + +#if SIZEOF_VOID_P == 8 +# define _PyOS_STACK_MARGIN_SHIFT (_PyOS_LOG2_STACK_MARGIN + 3) +#else +# define _PyOS_STACK_MARGIN_SHIFT (_PyOS_LOG2_STACK_MARGIN + 2) +#endif + +#ifdef _Py_THREAD_SANITIZER +# define _PyOS_MIN_STACK_SIZE (_PyOS_STACK_MARGIN_BYTES * 6) +#else +# define _PyOS_MIN_STACK_SIZE (_PyOS_STACK_MARGIN_BYTES * 3) +#endif + + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_qsbr.h b/Include/internal/pycore_qsbr.h index b835c3abaf5d0b2..eeca6fc472be37b 100644 --- a/Include/internal/pycore_qsbr.h +++ b/Include/internal/pycore_qsbr.h @@ -48,8 +48,21 @@ struct _qsbr_thread_state { // Thread state (or NULL) PyThreadState *tstate; - // Used to defer advancing write sequence a fixed number of times - int deferrals; + // Number of held items added by this thread since the last write sequence + // advance + int deferred_count; + + // Estimate for the amount of memory that is held by this thread since + // the last write sequence advance + size_t deferred_memory; + + // Amount of memory in mimalloc pages deferred from collection. When + // deferred, they are prevented from being used for a different size class + // and in a different thread. + size_t deferred_page_memory; + + // True if the deferred memory frees should be processed. + bool should_process; // Is this thread state allocated? bool allocated; @@ -70,8 +83,9 @@ struct _qsbr_shared { // Minimum observed read sequence of all QSBR thread states uint64_t rd_seq; - // Array of QSBR thread states. + // Array of QSBR thread states (aligned to 64 bytes). struct _qsbr_pad *array; + void *array_raw; // raw allocation pointer (for free) Py_ssize_t size; // Freelist of unused _qsbr_thread_states (protected by mutex) @@ -109,11 +123,17 @@ _Py_qbsr_goal_reached(struct _qsbr_thread_state *qsbr, uint64_t goal) extern uint64_t _Py_qsbr_advance(struct _qsbr_shared *shared); -// Batches requests to advance the write sequence. This advances the write -// sequence every N calls, which reduces overhead but increases time to -// reclamation. Returns the new goal. +// Return the next value for the write sequence (current plus the increment). extern uint64_t -_Py_qsbr_deferred_advance(struct _qsbr_thread_state *qsbr); +_Py_qsbr_shared_next(struct _qsbr_shared *shared); + +// Return true if deferred memory frees held by QSBR should be processed to +// determine if they can be safely freed. +static inline bool +_Py_qsbr_should_process(struct _qsbr_thread_state *qsbr) +{ + return qsbr->should_process; +} // Have the read sequences advanced to the given goal? If this returns true, // it safe to reclaim any memory tagged with the goal (or earlier goal). diff --git a/Include/internal/pycore_runtime_init.h b/Include/internal/pycore_runtime_init.h index 4200d91a2fcd9d2..e250d84d265ee37 100644 --- a/Include/internal/pycore_runtime_init.h +++ b/Include/internal/pycore_runtime_init.h @@ -137,13 +137,7 @@ extern PyTypeObject _PyExc_MemoryError; }, \ .gc = { \ .enabled = 1, \ - .young = { .threshold = 2000, }, \ - .old = { \ - { .threshold = 10, }, \ - { .threshold = 0, }, \ - }, \ - .work_to_do = -5000, \ - .phase = GC_PHASE_MARK, \ + GC_GENERATION_INIT \ }, \ .qsbr = { \ .wr_seq = QSBR_INITIAL, \ diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 4a34ffa559e124f..e642a42bbfc3e41 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -794,6 +794,7 @@ extern "C" { INIT_ID(alias), \ INIT_ID(align), \ INIT_ID(all), \ + INIT_ID(all_threads), \ INIT_ID(allow_code), \ INIT_ID(any), \ INIT_ID(append), \ @@ -891,6 +892,7 @@ extern "C" { INIT_ID(data), \ INIT_ID(database), \ INIT_ID(day), \ + INIT_ID(debug), \ INIT_ID(decode), \ INIT_ID(decoder), \ INIT_ID(default), \ @@ -902,7 +904,6 @@ extern "C" { INIT_ID(deterministic), \ INIT_ID(device), \ INIT_ID(dict), \ - INIT_ID(dict_content), \ INIT_ID(dictcomp), \ INIT_ID(difference_update), \ INIT_ID(digest), \ @@ -1137,6 +1138,7 @@ extern "C" { INIT_ID(offset_src), \ INIT_ID(on_type_read), \ INIT_ID(onceregistry), \ + INIT_ID(only_active_thread), \ INIT_ID(only_keys), \ INIT_ID(oparg), \ INIT_ID(opcode), \ diff --git a/Include/internal/pycore_runtime_structs.h b/Include/internal/pycore_runtime_structs.h index 6bf3aae7175a978..78f236a51a1ff7c 100644 --- a/Include/internal/pycore_runtime_structs.h +++ b/Include/internal/pycore_runtime_structs.h @@ -280,9 +280,13 @@ struct pyruntimestate { struct _Py_time_runtime_state time; #if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) - // Used in "Python/emscripten_trampoline.c" to choose between type - // reflection trampoline and EM_JS trampoline. - int (*emscripten_count_args_function)(PyCFunctionWithKeywords func); + // Used in "Python/emscripten_trampoline.c" to choose between wasm-gc + // trampoline and JavaScript trampoline. + PyObject* (*emscripten_trampoline)(int* success, + PyCFunctionWithKeywords func, + PyObject* self, + PyObject* args, + PyObject* kw); #endif /* All the objects that are shared by the runtime's interpreters. */ diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index a2acf311ff4c654..42fdbeb21f652a8 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -127,6 +127,13 @@ _PyStackRef_FromPyObjectSteal(PyObject *obj, const char *filename, int linenumbe } #define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj), __FILE__, __LINE__) +static inline _PyStackRef +_PyStackRef_FromPyObjectBorrow(PyObject *obj, const char *filename, int linenumber) +{ + return _Py_stackref_create(obj, filename, linenumber); +} +#define PyStackRef_FromPyObjectBorrow(obj) _PyStackRef_FromPyObjectBorrow(_PyObject_CAST(obj), __FILE__, __LINE__) + static inline _PyStackRef _PyStackRef_FromPyObjectImmortal(PyObject *obj, const char *filename, int linenumber) { @@ -239,6 +246,7 @@ PyStackRef_IsNullOrInt(_PyStackRef ref); #else #define Py_INT_TAG 3 +#define Py_TAG_REFCNT 1 static inline bool PyStackRef_IsTaggedInt(_PyStackRef i) @@ -264,7 +272,7 @@ PyStackRef_UntagInt(_PyStackRef i) #ifdef Py_GIL_DISABLED -#define Py_TAG_DEFERRED (1) +#define Py_TAG_DEFERRED Py_TAG_REFCNT #define Py_TAG_PTR ((uintptr_t)0) #define Py_TAG_BITS ((uintptr_t)1) @@ -319,10 +327,18 @@ _PyStackRef_FromPyObjectSteal(PyObject *obj) } # define PyStackRef_FromPyObjectSteal(obj) _PyStackRef_FromPyObjectSteal(_PyObject_CAST(obj)) +static inline _PyStackRef +PyStackRef_FromPyObjectBorrow(PyObject *obj) +{ + assert(obj != NULL); + assert(((uintptr_t)obj & Py_TAG_BITS) == 0); + return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_DEFERRED }; +} + static inline bool PyStackRef_IsHeapSafe(_PyStackRef stackref) { - if (PyStackRef_IsDeferred(stackref)) { + if (PyStackRef_IsDeferred(stackref) && !PyStackRef_IsTaggedInt(stackref)) { PyObject *obj = PyStackRef_AsPyObjectBorrow(stackref); return obj == NULL || _Py_IsImmortal(obj) || _PyObject_HasDeferredRefcount(obj); } @@ -442,14 +458,13 @@ PyStackRef_AsStrongReference(_PyStackRef stackref) /* References to immortal objects always have their tag bit set to Py_TAG_REFCNT * as they can (must) have their reclamation deferred */ -#define Py_TAG_BITS 1 -#define Py_TAG_REFCNT 1 +#define Py_TAG_BITS 3 #if _Py_IMMORTAL_FLAGS != Py_TAG_REFCNT # error "_Py_IMMORTAL_FLAGS != Py_TAG_REFCNT" #endif #define BITS_TO_PTR(REF) ((PyObject *)((REF).bits)) -#define BITS_TO_PTR_MASKED(REF) ((PyObject *)(((REF).bits) & (~Py_TAG_BITS))) +#define BITS_TO_PTR_MASKED(REF) ((PyObject *)(((REF).bits) & (~Py_TAG_REFCNT))) #define PyStackRef_NULL_BITS Py_TAG_REFCNT static const _PyStackRef PyStackRef_NULL = { .bits = PyStackRef_NULL_BITS }; @@ -529,7 +544,7 @@ PyStackRef_FromPyObjectSteal(PyObject *obj) { assert(obj != NULL); #if SIZEOF_VOID_P > 4 - unsigned int tag = obj->ob_flags & Py_TAG_BITS; + unsigned int tag = obj->ob_flags & Py_TAG_REFCNT; #else unsigned int tag = _Py_IsImmortal(obj) ? Py_TAG_REFCNT : 0; #endif @@ -538,6 +553,13 @@ PyStackRef_FromPyObjectSteal(PyObject *obj) return ref; } +static inline _PyStackRef +PyStackRef_FromPyObjectBorrow(PyObject *obj) +{ + assert(obj != NULL); + return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT }; +} + static inline _PyStackRef PyStackRef_FromPyObjectStealMortal(PyObject *obj) { @@ -548,12 +570,6 @@ PyStackRef_FromPyObjectStealMortal(PyObject *obj) return ref; } -// Check if a stackref is exactly the same as another stackref, including the -// the deferred bit. This can only be used safely if you know that the deferred -// bits of `a` and `b` match. -#define PyStackRef_IsExactly(a, b) \ - (assert(((a).bits & Py_TAG_BITS) == ((b).bits & Py_TAG_BITS)), (a).bits == (b).bits) - static inline _PyStackRef _PyStackRef_FromPyObjectNew(PyObject *obj) { @@ -561,7 +577,7 @@ _PyStackRef_FromPyObjectNew(PyObject *obj) if (_Py_IsImmortal(obj)) { return (_PyStackRef){ .bits = ((uintptr_t)obj) | Py_TAG_REFCNT}; } - Py_INCREF_MORTAL(obj); + _Py_INCREF_MORTAL(obj); _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj }; PyStackRef_CheckValid(ref); return ref; @@ -572,7 +588,7 @@ static inline _PyStackRef _PyStackRef_FromPyObjectNewMortal(PyObject *obj) { assert(obj != NULL); - Py_INCREF_MORTAL(obj); + _Py_INCREF_MORTAL(obj); _PyStackRef ref = (_PyStackRef){ .bits = (uintptr_t)obj }; PyStackRef_CheckValid(ref); return ref; @@ -590,14 +606,14 @@ PyStackRef_FromPyObjectImmortal(PyObject *obj) /* WARNING: This macro evaluates its argument more than once */ #ifdef _WIN32 #define PyStackRef_DUP(REF) \ - (PyStackRef_RefcountOnObject(REF) ? (Py_INCREF_MORTAL(BITS_TO_PTR(REF)), (REF)) : (REF)) + (PyStackRef_RefcountOnObject(REF) ? (_Py_INCREF_MORTAL(BITS_TO_PTR(REF)), (REF)) : (REF)) #else static inline _PyStackRef PyStackRef_DUP(_PyStackRef ref) { assert(!PyStackRef_IsNull(ref)); if (PyStackRef_RefcountOnObject(ref)) { - Py_INCREF_MORTAL(BITS_TO_PTR(ref)); + _Py_INCREF_MORTAL(BITS_TO_PTR(ref)); } return ref; } @@ -606,7 +622,7 @@ PyStackRef_DUP(_PyStackRef ref) static inline bool PyStackRef_IsHeapSafe(_PyStackRef ref) { - return (ref.bits & Py_TAG_BITS) == 0 || ref.bits == PyStackRef_NULL_BITS || _Py_IsImmortal(BITS_TO_PTR_MASKED(ref)); + return (ref.bits & Py_TAG_BITS) != Py_TAG_REFCNT || ref.bits == PyStackRef_NULL_BITS || _Py_IsImmortal(BITS_TO_PTR_MASKED(ref)); } static inline _PyStackRef @@ -681,7 +697,7 @@ PyStackRef_XCLOSE(_PyStackRef ref) // Note: this is a macro because MSVC (Windows) has trouble inlining it. -#define PyStackRef_Is(a, b) (((a).bits & (~Py_TAG_BITS)) == ((b).bits & (~Py_TAG_BITS))) +#define PyStackRef_Is(a, b) (((a).bits & (~Py_TAG_REFCNT)) == ((b).bits & (~Py_TAG_REFCNT))) #endif // !defined(Py_GIL_DISABLED) && defined(Py_STACKREF_DEBUG) @@ -741,6 +757,13 @@ _PyThreadState_PushCStackRef(PyThreadState *tstate, _PyCStackRef *ref) ref->ref = PyStackRef_NULL; } +static inline void +_PyThreadState_PushCStackRefNew(PyThreadState *tstate, _PyCStackRef *ref, PyObject *obj) +{ + _PyThreadState_PushCStackRef(tstate, ref); + ref->ref = PyStackRef_FromPyObjectNew(obj); +} + static inline void _PyThreadState_PopCStackRef(PyThreadState *tstate, _PyCStackRef *ref) { @@ -752,6 +775,17 @@ _PyThreadState_PopCStackRef(PyThreadState *tstate, _PyCStackRef *ref) PyStackRef_XCLOSE(ref->ref); } +static inline _PyStackRef +_PyThreadState_PopCStackRefSteal(PyThreadState *tstate, _PyCStackRef *ref) +{ +#ifdef Py_GIL_DISABLED + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + assert(tstate_impl->c_stack_refs == ref); + tstate_impl->c_stack_refs = ref->next; +#endif + return ref->ref; +} + #ifdef Py_GIL_DISABLED static inline int @@ -771,7 +805,7 @@ _Py_TryIncrefCompareStackRef(PyObject **src, PyObject *op, _PyStackRef *out) static inline int _Py_TryXGetStackRef(PyObject **src, _PyStackRef *out) { - PyObject *op = _Py_atomic_load_ptr_relaxed(src); + PyObject *op = _PyObject_CAST(_Py_atomic_load_ptr_relaxed(src)); if (op == NULL) { *out = PyStackRef_NULL; return 1; @@ -781,6 +815,13 @@ _Py_TryXGetStackRef(PyObject **src, _PyStackRef *out) #endif +#define PyStackRef_XSETREF(dst, src) \ + do { \ + _PyStackRef _tmp_dst_ref = (dst); \ + (dst) = (src); \ + PyStackRef_XCLOSE(_tmp_dst_ref); \ + } while(0) + // Like Py_VISIT but for _PyStackRef fields #define _Py_VISIT_STACKREF(ref) \ do { \ diff --git a/Include/internal/pycore_stats.h b/Include/internal/pycore_stats.h index ab649574f33dbf1..24f239a2135b93d 100644 --- a/Include/internal/pycore_stats.h +++ b/Include/internal/pycore_stats.h @@ -77,7 +77,7 @@ PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void); #define RARE_EVENT_INTERP_INC(interp, name) \ do { \ /* saturating add */ \ - int val = FT_ATOMIC_LOAD_UINT8_RELAXED(interp->rare_events.name); \ + uint8_t val = FT_ATOMIC_LOAD_UINT8_RELAXED(interp->rare_events.name); \ if (val < UINT8_MAX) { \ FT_ATOMIC_STORE_UINT8(interp->rare_events.name, val + 1); \ } \ diff --git a/Include/internal/pycore_traceback.h b/Include/internal/pycore_traceback.h index d71dd2886999a6c..8357cce9d899fbb 100644 --- a/Include/internal/pycore_traceback.h +++ b/Include/internal/pycore_traceback.h @@ -100,8 +100,11 @@ extern int _Py_WriteIndentedMargin(int, const char*, PyObject *); extern int _Py_WriteIndent(int, PyObject *); // Export for the faulthandler module +PyAPI_FUNC(void) _Py_InitDumpStack(void); PyAPI_FUNC(void) _Py_DumpStack(int fd); +extern void _Py_DumpTraceback_Init(void); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_tracemalloc.h b/Include/internal/pycore_tracemalloc.h index 572e80258763197..41e1c91bee4fde7 100644 --- a/Include/internal/pycore_tracemalloc.h +++ b/Include/internal/pycore_tracemalloc.h @@ -21,7 +21,10 @@ struct _PyTraceMalloc_Config { } initialized; /* Is tracemalloc tracing memory allocations? - Variable protected by the TABLES_LOCK(). */ + Variable protected by the TABLES_LOCK() and stored atomically. + Atomic store is used so that it can read without locking for the + general case of checking if tracemalloc is enabled. + */ int tracing; /* limit of the number of frames in a traceback, 1 by default. diff --git a/Include/internal/pycore_tstate.h b/Include/internal/pycore_tstate.h index bad968428c73a1a..87f59c274ac7470 100644 --- a/Include/internal/pycore_tstate.h +++ b/Include/internal/pycore_tstate.h @@ -76,6 +76,15 @@ typedef struct _PyThreadStateImpl { Py_ssize_t reftotal; // this thread's total refcount operations #endif + // PyUnstable_ThreadState_ResetStackProtection() values + uintptr_t c_stack_init_base; + uintptr_t c_stack_init_top; + +#ifdef Py_GIL_DISABLED + // gh-144438: Add padding to ensure that the fields above don't share a + // cache line with other allocations. + char __padding[64]; +#endif } _PyThreadStateImpl; #ifdef __cplusplus diff --git a/Include/internal/pycore_typeobject.h b/Include/internal/pycore_typeobject.h index 1a4f89fd2449a05..ba66364d3e94b11 100644 --- a/Include/internal/pycore_typeobject.h +++ b/Include/internal/pycore_typeobject.h @@ -149,6 +149,11 @@ typedef int (*_py_validate_type)(PyTypeObject *); extern int _PyType_Validate(PyTypeObject *ty, _py_validate_type validate, unsigned int *tp_version); extern int _PyType_CacheGetItemForSpecialization(PyHeapTypeObject *ht, PyObject *descriptor, uint32_t tp_version); +// Like PyType_GetBaseByToken, but does not modify refcounts. +// Cannot fail; arguments must be valid. +PyAPI_FUNC(int) +_PyType_GetBaseByToken_Borrow(PyTypeObject *type, void *token, PyTypeObject **result); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index c85d53b89accdb6..3791b913c175467 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -139,14 +139,18 @@ extern PyObject* _PyUnicode_DecodeUnicodeEscapeStateful( // Helper for PyUnicode_DecodeUnicodeEscape that detects invalid escape // chars. // Export for test_peg_generator. -PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeInternal( +PyAPI_FUNC(PyObject*) _PyUnicode_DecodeUnicodeEscapeInternal2( const char *string, /* Unicode-Escape encoded string */ Py_ssize_t length, /* size of string */ const char *errors, /* error handling */ Py_ssize_t *consumed, /* bytes consumed */ - const char **first_invalid_escape); /* on return, points to first - invalid escaped char in - string. */ + int *first_invalid_escape_char, /* on return, if not -1, contain the first + invalid escaped char (<= 0xff) or invalid + octal escape (> 0xff) in string. */ + const char **first_invalid_escape_ptr); /* on return, if not NULL, may + point to the first invalid escaped + char in string. + May be NULL if errors is not NULL. */ /* --- Raw-Unicode-Escape Codecs ---------------------------------------------- */ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index fefacef77c89ee6..c3ecdf64c6ee006 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -936,6 +936,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(all_threads); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(allow_code); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1324,6 +1328,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(debug); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(decode); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -1368,10 +1376,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(dict_content); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dictcomp); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); @@ -2308,6 +2312,10 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); + string = &_Py_ID(only_active_thread); + _PyUnicode_InternStatic(interp, &string); + assert(_PyUnicode_CheckConsistency(string, 1)); + assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(only_keys); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 912b1e56692e1c5..efdd9395d663843 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -95,7 +95,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_BINARY_OP_ADD_UNICODE] = HAS_ERROR_FLAG | HAS_PURE_FLAG, [_BINARY_OP_INPLACE_ADD_UNICODE] = HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_BINARY_OP_EXTEND] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, - [_BINARY_OP_EXTEND] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_BINARY_OP_EXTEND] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_BINARY_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_STORE_SLICE] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BINARY_OP_SUBSCR_LIST_INT] = HAS_DEOPT_FLAG | HAS_ESCAPES_FLAG, @@ -122,7 +122,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_GET_ANEXT] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_GET_AWAITABLE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_SEND_GEN_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_YIELD_VALUE] = HAS_ARG_FLAG, + [_YIELD_VALUE] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG, [_POP_EXCEPT] = HAS_ESCAPES_FLAG, [_LOAD_COMMON_CONSTANT] = HAS_ARG_FLAG, [_LOAD_BUILD_CLASS] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -277,7 +277,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_CALL_KW_NON_PY] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_MAKE_CALLARGS_A_TUPLE] = HAS_ERROR_FLAG | HAS_ERROR_NO_POP_FLAG | HAS_ESCAPES_FLAG, [_MAKE_FUNCTION] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, - [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG, + [_SET_FUNCTION_ATTRIBUTE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_RETURN_GENERATOR] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_BUILD_SLICE] = HAS_ARG_FLAG | HAS_ERROR_FLAG, [_CONVERT_VALUE] = HAS_ARG_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, diff --git a/Include/internal/pycore_weakref.h b/Include/internal/pycore_weakref.h index 950aa0af290056b..4ed8928c0b92a8c 100644 --- a/Include/internal/pycore_weakref.h +++ b/Include/internal/pycore_weakref.h @@ -29,6 +29,12 @@ extern "C" { PyMutex_LockFlags(wr->weakrefs_lock, _Py_LOCK_DONT_DETACH) #define UNLOCK_WEAKREFS_FOR_WR(wr) PyMutex_Unlock(wr->weakrefs_lock) +#define FT_CLEAR_WEAKREFS(obj, weakref_list) \ + do { \ + assert(Py_REFCNT(obj) == 0); \ + PyObject_ClearWeakRefs(obj); \ + } while (0) + #else #define LOCK_WEAKREFS(obj) @@ -37,6 +43,14 @@ extern "C" { #define LOCK_WEAKREFS_FOR_WR(wr) #define UNLOCK_WEAKREFS_FOR_WR(wr) +#define FT_CLEAR_WEAKREFS(obj, weakref_list) \ + do { \ + assert(Py_REFCNT(obj) == 0); \ + if (weakref_list != NULL) { \ + PyObject_ClearWeakRefs(obj); \ + } \ + } while (0) + #endif static inline int _is_dead(PyObject *obj) diff --git a/Include/object.h b/Include/object.h index 8cc83abb8574e37..50a41e355eda0a3 100644 --- a/Include/object.h +++ b/Include/object.h @@ -199,11 +199,11 @@ _Py_ThreadId(void) #elif defined(__MINGW32__) && defined(_M_ARM64) tid = __getReg(18); #elif defined(__i386__) - __asm__("movl %%gs:0, %0" : "=r" (tid)); // 32-bit always uses GS + __asm__("{movl %%gs:0, %0|mov %0, dword ptr gs:[0]}" : "=r" (tid)); // 32-bit always uses GS #elif defined(__MACH__) && defined(__x86_64__) - __asm__("movq %%gs:0, %0" : "=r" (tid)); // x86_64 macOSX uses GS + __asm__("{movq %%gs:0, %0|mov %0, qword ptr gs:[0]}" : "=r" (tid)); // x86_64 macOSX uses GS #elif defined(__x86_64__) - __asm__("movq %%fs:0, %0" : "=r" (tid)); // x86_64 Linux, BSD uses FS + __asm__("{movq %%fs:0, %0|mov %0, qword ptr fs:[0]}" : "=r" (tid)); // x86_64 Linux, BSD uses FS #elif defined(__arm__) && __ARM_ARCH >= 7 __asm__ ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tid)); #elif defined(__aarch64__) && defined(__APPLE__) @@ -654,8 +654,13 @@ PyAPI_DATA(PyObject) _Py_NoneStruct; /* Don't use this directly */ PyAPI_FUNC(int) Py_IsNone(PyObject *x); #define Py_IsNone(x) Py_Is((x), Py_None) -/* Macro for returning Py_None from a function */ -#define Py_RETURN_NONE return Py_None +/* Macro for returning Py_None from a function. + * Only treat Py_None as immortal in the limited C API 3.12 and newer. */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030c0000 +# define Py_RETURN_NONE return Py_NewRef(Py_None) +#else +# define Py_RETURN_NONE return Py_None +#endif /* Py_NotImplemented is a singleton used to signal that an operation is @@ -669,8 +674,13 @@ PyAPI_DATA(PyObject) _Py_NotImplementedStruct; /* Don't use this directly */ # define Py_NotImplemented (&_Py_NotImplementedStruct) #endif -/* Macro for returning Py_NotImplemented from a function */ -#define Py_RETURN_NOTIMPLEMENTED return Py_NotImplemented +/* Macro for returning Py_NotImplemented from a function. Only treat + * Py_NotImplemented as immortal in the limited C API 3.12 and newer. */ +#if defined(Py_LIMITED_API) && Py_LIMITED_API+0 < 0x030c0000 +# define Py_RETURN_NOTIMPLEMENTED return Py_NewRef(Py_NotImplemented) +#else +# define Py_RETURN_NOTIMPLEMENTED return Py_NotImplemented +#endif /* Rich comparison opcodes */ #define Py_LT 0 diff --git a/Include/patchlevel.h b/Include/patchlevel.h index c9dd659046a3671..c6fab81f8124ac6 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -19,12 +19,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 14 -#define PY_MICRO_VERSION 0 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_BETA -#define PY_RELEASE_SERIAL 1 +#define PY_MICRO_VERSION 6 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL +#define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.14.0b1" +#define PY_VERSION "3.14.6+" /*--end constants--*/ diff --git a/Include/py_curses.h b/Include/py_curses.h index 49fc3c9d127aa6f..3443ff7d54fc8f0 100644 --- a/Include/py_curses.h +++ b/Include/py_curses.h @@ -36,6 +36,13 @@ #define NCURSES_OPAQUE 0 #endif +/* PDCurses exposes its ncurses-compatible mouse API, the one this module uses, + only when this is defined before the curses header is included below. + Ignored by other curses implementations. */ +#ifndef PDC_NCMOUSE +# define PDC_NCMOUSE +#endif + #if defined(HAVE_NCURSESW_NCURSES_H) # include #elif defined(HAVE_NCURSESW_CURSES_H) diff --git a/Include/pyexpat.h b/Include/pyexpat.h index 9824d099c3df7d0..a676e16a7a457ea 100644 --- a/Include/pyexpat.h +++ b/Include/pyexpat.h @@ -52,6 +52,19 @@ struct PyExpat_CAPI int (*SetHashSalt)(XML_Parser parser, unsigned long hash_salt); /* might be NULL for expat < 2.6.0 */ XML_Bool (*SetReparseDeferralEnabled)(XML_Parser parser, XML_Bool enabled); + /* might be NULL for expat < 2.7.2 */ + XML_Bool (*SetAllocTrackerActivationThreshold)( + XML_Parser parser, unsigned long long activationThresholdBytes); + XML_Bool (*SetAllocTrackerMaximumAmplification)( + XML_Parser parser, float maxAmplificationFactor); + /* might be NULL for expat < 2.4.0 */ + XML_Bool (*SetBillionLaughsAttackProtectionActivationThreshold)( + XML_Parser parser, unsigned long long activationThresholdBytes); + XML_Bool (*SetBillionLaughsAttackProtectionMaximumAmplification)( + XML_Parser parser, float maxAmplificationFactor); + /* might be NULL for expat < 2.8.0 */ + XML_Bool (*SetHashSalt16Bytes)( + XML_Parser parser, const uint8_t entropy[16]); /* always add new stuff to the end! */ }; diff --git a/Include/pymacro.h b/Include/pymacro.h index 218987a80b0d913..1af057ad78266af 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -72,6 +72,12 @@ /* Absolute value of the number x */ #define Py_ABS(x) ((x) < 0 ? -(x) : (x)) +/* Safer implementation that avoids an undefined behavior for the minimal + value of the signed integer type if its absolute value is larger than + the maximal value of the signed integer type (in the two's complement + representations, which is common). + */ +#define _Py_ABS_CAST(T, x) ((x) >= 0 ? ((T) (x)) : ((T) (((T) -((x) + 1)) + 1u))) #define _Py_XSTRINGIFY(x) #x diff --git a/Include/pyport.h b/Include/pyport.h index 3eac119bf8e8d89..2c8567f2554d9b9 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -49,8 +49,9 @@ // Static inline functions should use _Py_NULL rather than using directly NULL // to prevent C++ compiler warnings. On C23 and newer and on C++11 and newer, // _Py_NULL is defined as nullptr. -#if (defined (__STDC_VERSION__) && __STDC_VERSION__ > 201710L) \ - || (defined(__cplusplus) && __cplusplus >= 201103) +#if !defined(_MSC_VER) && \ + ((defined (__STDC_VERSION__) && __STDC_VERSION__ >= 202311L) \ + || (defined(__cplusplus) && __cplusplus >= 201103)) # define _Py_NULL nullptr #else # define _Py_NULL NULL @@ -700,4 +701,10 @@ extern "C" { #endif +// Assume the stack grows down unless specified otherwise +#ifndef _Py_STACK_GROWS_DOWN +# define _Py_STACK_GROWS_DOWN 1 +#endif + + #endif /* Py_PYPORT_H */ diff --git a/Include/pythonrun.h b/Include/pythonrun.h index fad2b3c77476e41..92b50aa807bf704 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -21,39 +21,15 @@ PyAPI_FUNC(void) PyErr_DisplayException(PyObject *); /* Stuff with no proper home (yet) */ PyAPI_DATA(int) (*PyOS_InputHook)(void); -/* Stack size, in "pointers". This must be large enough, so - * no two calls to check recursion depth are more than this far - * apart. In practice, that means it must be larger than the C - * stack consumption of PyEval_EvalDefault */ -#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER) -# define PYOS_LOG2_STACK_MARGIN 12 -#elif defined(Py_DEBUG) && defined(WIN32) -# define PYOS_LOG2_STACK_MARGIN 12 -#elif defined(__wasi__) - /* Web assembly has two stacks, so this isn't really a size */ -# define PYOS_LOG2_STACK_MARGIN 9 -#else -# define PYOS_LOG2_STACK_MARGIN 11 -#endif -#define PYOS_STACK_MARGIN (1 << PYOS_LOG2_STACK_MARGIN) -#define PYOS_STACK_MARGIN_BYTES (PYOS_STACK_MARGIN * sizeof(void *)) - -#if SIZEOF_VOID_P == 8 -#define PYOS_STACK_MARGIN_SHIFT (PYOS_LOG2_STACK_MARGIN + 3) -#else -#define PYOS_STACK_MARGIN_SHIFT (PYOS_LOG2_STACK_MARGIN + 2) -#endif - - #if defined(WIN32) -#define USE_STACKCHECK +# define USE_STACKCHECK #endif - #ifdef USE_STACKCHECK /* Check that we aren't overflowing our stack */ PyAPI_FUNC(int) PyOS_CheckStack(void); #endif + #ifndef Py_LIMITED_API # define Py_CPYTHON_PYTHONRUN_H # include "cpython/pythonrun.h" diff --git a/Include/refcount.h b/Include/refcount.h index 177bbdaf0c59772..1ef6b2186bba039 100644 --- a/Include/refcount.h +++ b/Include/refcount.h @@ -1,5 +1,5 @@ -#ifndef Py_REFCOUNT_H -#define Py_REFCOUNT_H +#ifndef _Py_REFCOUNT_H +#define _Py_REFCOUNT_H #ifdef __cplusplus extern "C" { #endif @@ -33,7 +33,7 @@ increase and decrease the objects reference count. In order to offer sufficient resilience to C extensions using the stable ABI compiled against 3.11 or earlier, we set the initial value near the -middle of the range (2**31, 2**32). That way the the refcount can be +middle of the range (2**31, 2**32). That way the refcount can be off by ~1 billion without affecting immortality. Reference count increases will use saturated arithmetic, taking advantage of @@ -117,6 +117,8 @@ PyAPI_FUNC(Py_ssize_t) Py_REFCNT(PyObject *ob); } #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 < 0x030b0000 # define Py_REFCNT(ob) _Py_REFCNT(_PyObject_CAST(ob)) + #else + # define Py_REFCNT(ob) _Py_REFCNT(ob) #endif #endif @@ -247,20 +249,6 @@ PyAPI_FUNC(void) Py_DecRef(PyObject *); PyAPI_FUNC(void) _Py_IncRef(PyObject *); PyAPI_FUNC(void) _Py_DecRef(PyObject *); -#ifndef Py_GIL_DISABLED -static inline Py_ALWAYS_INLINE void Py_INCREF_MORTAL(PyObject *op) -{ - assert(!_Py_IsStaticImmortal(op)); - op->ob_refcnt++; - _Py_INCREF_STAT_INC(); -#if defined(Py_REF_DEBUG) && !defined(Py_LIMITED_API) - if (!_Py_IsImmortal(op)) { - _Py_INCREF_IncRefTotal(); - } -#endif -} -#endif - static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op) { #if defined(Py_LIMITED_API) && (Py_LIMITED_API+0 >= 0x030c0000 || defined(Py_REF_DEBUG)) @@ -564,4 +552,4 @@ static inline PyObject* _Py_XNewRef(PyObject *obj) #ifdef __cplusplus } #endif -#endif // !Py_REFCOUNT_H +#endif // !_Py_REFCOUNT_H diff --git a/InternalDocs/compiler.md b/InternalDocs/compiler.md index 02bbdf6071fa12e..742af5efcf5ce50 100644 --- a/InternalDocs/compiler.md +++ b/InternalDocs/compiler.md @@ -604,4 +604,4 @@ References pp. 213--227, 1997. [^2]: The Zephyr Abstract Syntax Description Language.: - https://www.cs.princeton.edu/research/techreps/TR-554-97 + https://www.cs.princeton.edu/research/techreps/254 diff --git a/InternalDocs/exception_handling.md b/InternalDocs/exception_handling.md index 28589787e1fad79..9e38da4c862f164 100644 --- a/InternalDocs/exception_handling.md +++ b/InternalDocs/exception_handling.md @@ -8,7 +8,7 @@ The cost of raising an exception is increased, but not by much. The following code: -``` +```python try: g(0) except: @@ -18,7 +18,7 @@ except: compiles into intermediate code like the following: -``` +```python RESUME 0 1 SETUP_FINALLY 8 (to L1) @@ -118,13 +118,13 @@ All offsets and lengths are in code units, not bytes. We want the format to be compact, but quickly searchable. For it to be compact, it needs to have variable sized entries so that we can store common (small) offsets compactly, but handle large offsets if needed. -For it to be searchable quickly, we need to support binary search giving us log(n) performance in all cases. +For it to be searchable quickly, we need to support binary search giving us `log(n)` performance in all cases. Binary search typically assumes fixed size entries, but that is not necessary, as long as we can identify the start of an entry. It is worth noting that the size (end-start) is always smaller than the end, so we encode the entries as: `start, size, target, depth, push-lasti`. -Also, sizes are limited to 2**30 as the code length cannot exceed 2**31 and each code unit takes 2 bytes. +Also, sizes are limited to `2**30` as the code length cannot exceed `2**31` and each code unit takes 2 bytes. It also happens that depth is generally quite small. So, we need to encode: @@ -140,7 +140,7 @@ lasti (1 bit) We need a marker for the start of the entry, so the first byte of entry will have the most significant bit set. Since the most significant bit is reserved for marking the start of an entry, we have 7 bits per byte to encode offsets. Encoding uses a standard varint encoding, but with only 7 bits instead of the usual 8. -The 8 bits of a byte are (msb left) SXdddddd where S is the start bit. X is the extend bit meaning that the next byte is required to extend the offset. +The 8 bits of a byte are (msb left) `SXdddddd` where `S` is the start bit. `X` is the extend bit meaning that the next byte is required to extend the offset. In addition, we combine `depth` and `lasti` into a single value, `((depth<<1)+lasti)`, before encoding. diff --git a/InternalDocs/garbage_collector.md b/InternalDocs/garbage_collector.md index 4da6cd47dc859e0..9ecdd75a5b2ca6f 100644 --- a/InternalDocs/garbage_collector.md +++ b/InternalDocs/garbage_collector.md @@ -107,7 +107,7 @@ As is explained later in the [Optimization: reusing fields to save memory](#optimization-reusing-fields-to-save-memory) section, these two extra fields are normally used to keep doubly linked lists of all the objects tracked by the garbage collector (these lists are the GC generations, more on -that in the [Optimization: incremental collection](#Optimization-incremental-collection) section), but +that in the [Optimization: generations](#Optimization-generations) section), but they are also reused to fulfill other purposes when the full doubly linked list structure is not needed as a memory optimization. @@ -199,22 +199,22 @@ unreachable: ```pycon >>> import gc ->>> +>>> >>> class Link: ... def __init__(self, next_link=None): ... self.next_link = next_link -... +... >>> link_3 = Link() >>> link_2 = Link(link_3) >>> link_1 = Link(link_2) >>> link_3.next_link = link_1 >>> A = link_1 >>> del link_1, link_2, link_3 ->>> +>>> >>> link_4 = Link() >>> link_4.next_link = link_4 >>> del link_4 ->>> +>>> >>> # Collect the unreachable Link object (and its .__dict__ dict). >>> gc.collect() 2 @@ -286,7 +286,7 @@ object, the GC does not process it twice. Notice that an object that was marked as "tentatively unreachable" and was later moved back to the reachable list will be visited again by the garbage collector -as now all the references that that object has need to be processed as well. This +as now all the references that the object has need to be processed as well. This process is really a breadth first search over the object graph. Once all the objects are scanned, the GC knows that all container objects in the tentatively unreachable list are really unreachable and can thus be garbage collected. @@ -350,11 +350,12 @@ follows these steps in order: the reference counts fall to 0, triggering the destruction of all unreachable objects. -Optimization: incremental collection -==================================== +Optimization: generations +========================= -In order to bound the length of each garbage collection pause, the GC implementation -for the default build uses incremental collection with two generations. +In order to limit the time each garbage collection takes, the GC +implementation for the default build uses a popular optimization: +generations. Generational garbage collection takes advantage of what is known as the weak generational hypothesis: Most objects die young. @@ -362,76 +363,29 @@ This has proven to be very close to the reality of many Python programs as many temporary objects are created and destroyed very quickly. To take advantage of this fact, all container objects are segregated into -two generations: young and old. Every new object starts in the young generation. -Each garbage collection scans the entire young generation and part of the old generation. - -The time taken to scan the young generation can be controlled by controlling its -size, but the size of the old generation cannot be controlled. -In order to keep pause times down, scanning of the old generation of the heap -occurs in increments. - -To keep track of what has been scanned, the old generation contains two lists: - -* Those objects that have not yet been scanned, referred to as the `pending` list. -* Those objects that have been scanned, referred to as the `visited` list. - -To detect and collect all unreachable objects in the heap, the garbage collector -must scan the whole heap. This whole heap scan is called a full scavenge. - -Increments ----------- - -Each full scavenge is performed in a series of increments. -For each full scavenge, the combined increments will cover the whole heap. - -Each increment is made up of: - -* The young generation -* The old generation's least recently scanned objects -* All objects reachable from those objects that have not yet been scanned this full scavenge - -The surviving objects (those that are not collected) are moved to the back of the -`visited` list in the old generation. - -When a full scavenge starts, no objects in the heap are considered to have been scanned, -so all objects in the old generation must be in the `pending` space. -When all objects in the heap have been scanned a cycle ends, and all objects are moved -to the `pending` list again. To avoid having to traverse the entire list, which list is -`pending` and which is `visited` is determined by a field in the `GCState` struct. -The `visited` and `pending` lists can be swapped by toggling this bit. - -Correctness ------------ - -The [algorithm for identifying cycles](#Identifying-reference-cycles) will find all -unreachable cycles in a list of objects, but will not find any cycles that are -even partly outside of that list. -Therefore, to be guaranteed that a full scavenge will find all unreachable cycles, -each cycle must be fully contained within a single increment. - -To make sure that no partial cycles are included in the increment we perform a -[transitive closure](https://en.wikipedia.org/wiki/Transitive_closure) -over reachable, unscanned objects from the initial increment. -Since the transitive closure of objects reachable from an object must be a (non-strict) -superset of any unreachable cycle including that object, we are guaranteed that a -transitive closure cannot contain any partial cycles. -We can exclude scanned objects, as they must have been reachable when scanned. -If a scanned object becomes part of an unreachable cycle after being scanned, it will -not be collected at this time, but it will be collected in the next full scavenge. +three spaces/generations. Every new +object starts in the first generation (generation 0). The previous algorithm is +executed only over the objects of a particular generation and if an object +survives a collection of its generation it will be moved to the next one +(generation 1), where it will be surveyed for collection less often. If +the same object survives another GC round in this new generation (generation 1) +it will be moved to the last generation (generation 2) where it will be +surveyed the least often. > [!NOTE] -> The GC implementation for the free-threaded build does not use incremental collection. -> Every collection operates on the entire heap. +> The GC implementation for the free-threaded build does not use generational +> collection. Every collection operates on the entire heap. + In order to decide when to run, the collector keeps track of the number of object allocations and deallocations since the last collection. When the number of allocations minus the number of deallocations exceeds `threshold0`, -collection starts. `threshold1` determines the fraction of the old -collection that is included in the increment. -The fraction is inversely proportional to `threshold1`, -as historically a larger `threshold1` meant that old generation -collections were performed less frequently. -`threshold2` is ignored. +collection starts. Initially only generation 0 is examined. If generation 0 has +been examined more than `threshold_1` times since generation 1 has been +examined, then generation 1 is examined as well. With generation 2, +things are a bit more complicated; see +[Collecting the oldest generation](#Collecting-the-oldest-generation) for +more information. These thresholds can be examined using the [`gc.get_threshold()`](https://docs.python.org/3/library/gc.html#gc.get_threshold) @@ -440,7 +394,7 @@ function: ```pycon >>> import gc >>> gc.get_threshold() -(700, 10, 10) +(2000, 10, 10) ``` The content of these generations can be examined using the @@ -453,84 +407,61 @@ specifically in a generation by calling `gc.collect(generation=NUM)`. ... pass ... >>> # Move everything to the old generation so it's easier to inspect ->>> # the young generation. +>>> # the younger generation. >>> gc.collect() 0 >>> # Create a reference cycle. >>> x = MyObj() >>> x.self = x ->>> ->>> # Initially the object is in the young generation. +>>> +>>> # Initially the object is in the youngest generation. >>> gc.get_objects(generation=0) [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] ->>> +>>> >>> # After a collection of the youngest generation the object ->>> # moves to the old generation. +>>> # moves to the next generation. >>> gc.collect(generation=0) 0 >>> gc.get_objects(generation=0) [] >>> gc.get_objects(generation=1) -[] ->>> gc.get_objects(generation=2) [..., <__main__.MyObj object at 0x7fbcc12a3400>, ...] ``` +Collecting the oldest generation +-------------------------------- + +In addition to the various configurable thresholds, the GC only triggers a full +collection of the oldest generation if the ratio `long_lived_pending / long_lived_total` +is above a given value (hardwired to 25%). The reason is that, while "non-full" +collections (that is, collections of the young and middle generations) will always +examine roughly the same number of objects (determined by the aforementioned +thresholds) the cost of a full collection is proportional to the total +number of long-lived objects, which is virtually unbounded. Indeed, it has +been remarked that doing a full collection every of object +creations entails a dramatic performance degradation in workloads which consist +of creating and storing lots of long-lived objects (for example, building a large list +of GC-tracked objects would show quadratic performance, instead of linear as +expected). Using the above ratio, instead, yields amortized linear performance +in the total number of objects (the effect of which can be summarized thusly: +"each full garbage collection is more and more costly as the number of objects +grows, but we do fewer and fewer of them"). + Optimization: excluding reachable objects ========================================= An object cannot be garbage if it can be reached. To avoid having to identify -reference cycles across the whole heap, we can reduce the amount of work done -considerably by first identifying objects reachable from objects known to be -alive. These objects are excluded from the normal cyclic detection process. - -The default and free-threaded build both implement this optimization but in -slightly different ways. - -Finding reachable objects for the default build GC --------------------------------------------------- - -This works by first moving most reachable objects to the `visited` space. -Empirically, most reachable objects can be reached from a small set of global -objects and local variables. This step does much less work per object, so -reduces the time spent performing garbage collection by at least half. - -> [!NOTE] -> Objects that are not determined to be reachable by this pass are not necessarily -> unreachable. We still need to perform the main algorithm to determine which objects -> are actually unreachable. -We use the same technique of forming a transitive closure as the incremental -collector does to find reachable objects, seeding the list with some global -objects and the currently executing frames. - -This phase moves objects to the `visited` space, as follows: - -1. All objects directly referred to by any builtin class, the `sys` module, the `builtins` -module and all objects directly referred to from stack frames are added to a working -set of reachable objects. -2. Until this working set is empty: - 1. Pop an object from the set and move it to the `visited` space - 2. For each object directly reachable from that object: - * If it is not already in `visited` space and it is a GC object, - add it to the working set - - -Before each increment of collection is performed, the stacks are scanned -to check for any new stack frames that have been created since the last -increment. All objects directly referred to from those stack frames are -added to the working set. -Then the above algorithm is repeated, starting from step 2. - +reference cycles across the whole heap, the free-threaded build first identifies +objects reachable from objects known to be alive. These objects are excluded +from the normal cyclic detection process. Finding reachable objects for the free-threaded GC -------------------------------------------------- Within the `gc_free_threading.c` implementation, this is known as the "mark -alive" pass or phase. It is similar in concept to what is done for the default -build GC. Rather than moving objects between double-linked lists, the -free-threaded GC uses a flag in `ob_gc_bits` to track if an object is -found to be definitely alive (not garbage). +alive" pass or phase. The free-threaded GC uses a flag in `ob_gc_bits` to track +if an object is found to be definitely alive (not garbage). To find objects reachable from known alive objects, known as the "roots", the `gc_mark_alive_from_roots()` function is used. Root objects include @@ -761,6 +692,14 @@ tuples can be untracked. A tuple can be untracked if all of its contents are already not tracked. Tuples are examined for untracking in all garbage collection cycles. +Dictionaries are always tracked from creation and are not untracked by the +garbage collector. Earlier versions (up to 3.13) used lazy tracking: empty or +atomic-only dicts were untracked on creation and re-tracked when a trackable +value was inserted (via `MAINTAIN_TRACKING`), and full collections called +`_PyDict_MaybeUntrack` to prune dicts whose values had become atomic. That +machinery was removed in 3.14 (GH-127010) because the per-set-item cost of +checking the tracking invariant outweighed the savings on full collections. + The garbage collector module provides the Python function `is_tracked(obj)`, which returns the current tracking status of the object. Subsequent garbage collections may change the tracking status of the object. diff --git a/InternalDocs/generators.md b/InternalDocs/generators.md index 87fbb91236844bf..a7a40fcd0ed009d 100644 --- a/InternalDocs/generators.md +++ b/InternalDocs/generators.md @@ -64,7 +64,7 @@ Iteration The [`FOR_ITER`](https://docs.python.org/dev/library/dis.html#opcode-FOR_ITER) instruction calls `__next__` on the iterator which is on the top of the stack, -and pushes the result to the stack. It has [`specializations`](adaptive.md) +and pushes the result to the stack. It has [`specializations`](interpreter.md) for a few common iterator types, including `FOR_ITER_GEN`, for iterating over a generator. `FOR_ITER_GEN` bypasses the call to `__next__`, and instead directly pushes the generator stack and resumes its execution from the diff --git a/InternalDocs/interpreter.md b/InternalDocs/interpreter.md index 38e9f6fced60880..75acdf596a7f302 100644 --- a/InternalDocs/interpreter.md +++ b/InternalDocs/interpreter.md @@ -226,10 +226,11 @@ Up through 3.10, the call stack was implemented as a singly-linked list of heap allocation for the stack frame. Since 3.11, frames are no longer fully-fledged objects. Instead, a leaner internal -`_PyInterpreterFrame` structure is used, which is allocated using a custom allocator -function (`_PyThreadState_BumpFramePointer()`), which allocates and initializes a -frame structure. Usually a frame allocation is just a pointer bump, which improves -memory locality. +`_PyInterpreterFrame` structure is used. Most frames are allocated contiguously in a +per-thread stack (see `_PyThreadState_PushFrame` in [Python/pystate.c](../Python/pystate.c)), +which improves memory locality and reduces overhead. +If the current `datastack_chunk` has enough space (`_PyThreadState_HasStackSpace`) +then the lightweight `_PyFrame_PushUnchecked` can be used instead of `_PyThreadState_PushFrame`. Sometimes an actual `PyFrameObject` is needed, such as when Python code calls `sys._getframe()` or an extension module calls diff --git a/InternalDocs/jit.md b/InternalDocs/jit.md index c98ccb3ace8d9ec..095853807377a45 100644 --- a/InternalDocs/jit.md +++ b/InternalDocs/jit.md @@ -89,7 +89,7 @@ When the full jit is enabled (python was configured with [`--enable-experimental-jit`](https://docs.python.org/dev/using/configure.html#cmdoption-enable-experimental-jit), the uop executor's `jit_code` field is populated with a pointer to a compiled C function that implements the executor logic. This function's signature is -defined by `jit_func` in [`pycore_jit.h`](Include/internal/pycore_jit.h). +defined by `jit_func` in [`pycore_jit.h`](../Include/internal/pycore_jit.h). When the executor is invoked by `ENTER_EXECUTOR`, instead of jumping to the uop interpreter at `tier2_dispatch`, the executor runs the function that `jit_code` points to. This function returns the instruction pointer diff --git a/InternalDocs/stack_protection.md b/InternalDocs/stack_protection.md new file mode 100644 index 000000000000000..14802e57d095f4d --- /dev/null +++ b/InternalDocs/stack_protection.md @@ -0,0 +1,68 @@ +# Stack Protection + +CPython protects against stack overflow in the form of runaway, or just very deep, recursion by raising a `RecursionError` instead of just crashing. +Protection against pure Python stack recursion has existed since very early, but in 3.12 we added protection against stack overflow +in C code. This was initially implemented using a counter and later improved in 3.14 to use the actual stack depth. +For those platforms that support it (Windows, Mac, and most Linuxes) we query the operating system to find the stack bounds. +For other platforms we use conservative estimates. + + +The C stack looks like this: + +``` + +-------+ <--- Top of machine stack + | | + | | + + ~~ + + | | + | | + +-------+ <--- Soft limit + | | + | | _PyOS_STACK_MARGIN_BYTES + | | + +-------+ <--- Hard limit + | | + | | _PyOS_STACK_MARGIN_BYTES + | | + +-------+ <--- Bottom of machine stack +``` + + +We get the current stack pointer using compiler intrinsics where available, or by taking the address of a C local variable. See `_Py_get_machine_stack_pointer()`. + +The soft and hard limits pointers are set by calling `_Py_InitializeRecursionLimits()` during thread initialization. + +Recursion checks are performed by `_Py_EnterRecursiveCall()` or `_Py_EnterRecursiveCallTstate()` which compare the stack pointer to the soft limit. If the stack pointer is lower than the soft limit, then `_Py_CheckRecursiveCall()` is called which checks against both the hard and soft limits: + +```python +kb_used = (stack_top - stack_pointer)>>10 +if stack_pointer < bottom_of_machine_stack: + pass # Our stack limits could be wrong so it is safest to do nothing. +elif stack_pointer < hard_limit: + FatalError(f"Unrecoverable stack overflow (used {kb_used} kB)") +elif stack_pointer < soft_limit: + raise RecursionError(f"Stack overflow (used {kb_used} kB)") +``` + +### User space threads and other oddities + +Some libraries provide user-space threads. These will change the C stack at runtime. +To guard against this we only raise if the stack pointer is in the window between the expected stack base and the soft limit. + +### Diagnosing and fixing stack overflows + +For stack protection to work correctly the amount of stack consumed between calls to `_Py_EnterRecursiveCall()` must be less than `_PyOS_STACK_MARGIN_BYTES`. + +If you see a traceback ending in: `RecursionError: Stack overflow (used ... kB)` then the stack protection is working as intended. If you don't expect to see the error, then check the amount of stack used. If it seems low then CPython may not be configured properly. + +However, if you see a fatal error or crash, then something is not right. +Either a recursive call is not checking `_Py_EnterRecursiveCall()`, or the amount of C stack consumed by a single call exceeds `_PyOS_STACK_MARGIN_BYTES`. If a hard crash occurs, it probably means that the amount of C stack consumed is more than double `_PyOS_STACK_MARGIN_BYTES`. + +Likely causes: +* Recursive code is not calling `_Py_EnterRecursiveCall()` +* `-O0` compilation flags, especially for Clang. With no optimization, C calls can consume a lot of stack space +* Giant, complex functions in third-party C extensions. This is unlikely as the function in question would need to be more complicated than the bytecode interpreter. +* `_PyOS_STACK_MARGIN_BYTES` is just too low. +* `_Py_InitializeRecursionLimits()` is not setting the soft and hard limits correctly for that platform. diff --git a/InternalDocs/string_interning.md b/InternalDocs/string_interning.md index 26a5197c6e70f38..829a27654a37d34 100644 --- a/InternalDocs/string_interning.md +++ b/InternalDocs/string_interning.md @@ -16,8 +16,8 @@ dynamic interning. The 256 possible one-character latin-1 strings, which can be retrieved with `_Py_LATIN1_CHR(c)`, are stored in statically allocated arrays, -`_PyRuntime.static_objects.strings.ascii` and -`_PyRuntime.static_objects.strings.latin1`. +`_PyRuntime.static_objects.singletons.strings.ascii` and +`_PyRuntime.static_objects.singletons.strings.latin1`. Longer singleton strings are marked in C source with `_Py_ID` (if the string is a valid C identifier fragment) or `_Py_STR` (if it needs a separate @@ -52,15 +52,9 @@ The key and value of each entry in this dict reference the same object. ## Immortality and reference counting -Invariant: Every immortal string is interned. +In the GIL-enabled build interned strings may be mortal or immortal. In the +free-threaded build, interned strings are always immortal. -In practice, this means that you must not use `_Py_SetImmortal` on -a string. (If you know it's already immortal, don't immortalize it; -if you know it's not interned you might be immortalizing a redundant copy; -if it's interned and mortal it needs extra processing in -`_PyUnicode_InternImmortal`.) - -The converse is not true: interned strings can be mortal. For mortal interned strings: - the 2 references from the interned dict (key & value) are excluded from diff --git a/Lib/_android_support.py b/Lib/_android_support.py index ae506f6a4b57b89..320dab52acdc0bd 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -29,15 +29,19 @@ def init_streams(android_log_write, stdout_prio, stderr_prio): global logcat logcat = Logcat(android_log_write) - - sys.stdout = TextLogStream( - stdout_prio, "python.stdout", sys.stdout.fileno()) - sys.stderr = TextLogStream( - stderr_prio, "python.stderr", sys.stderr.fileno()) + sys.stdout = TextLogStream(stdout_prio, "python.stdout", sys.stdout) + sys.stderr = TextLogStream(stderr_prio, "python.stderr", sys.stderr) class TextLogStream(io.TextIOWrapper): - def __init__(self, prio, tag, fileno=None, **kwargs): + def __init__(self, prio, tag, original=None, **kwargs): + # Respect the -u option. + if original: + kwargs.setdefault("write_through", original.write_through) + fileno = original.fileno() + else: + fileno = None + # The default is surrogateescape for stdout and backslashreplace for # stderr, but in the context of an Android log, readability is more # important than reversibility. @@ -164,6 +168,13 @@ def write(self, prio, tag, message): # message. message = message.replace(b"\x00", b"\xc0\x80") + # On API level 30 and higher, Logcat will strip any number of leading + # newlines. This is visible in all `logcat` modes, even --binary. Work + # around this by adding a leading space, which shouldn't make any + # difference to the log's usability. + if message.startswith(b"\n"): + message = b" " + message + with self._lock: now = time() self._bucket_level += ( diff --git a/Lib/_ast_unparse.py b/Lib/_ast_unparse.py index 0b669edb2ffec69..1c8741b5a554833 100644 --- a/Lib/_ast_unparse.py +++ b/Lib/_ast_unparse.py @@ -626,32 +626,11 @@ def _write_ftstring(self, values, prefix): ) self._ftstring_helper(fstring_parts) - def _tstring_helper(self, node): - last_idx = 0 - for i, value in enumerate(node.values): - # This can happen if we have an implicit concat of a t-string - # with an f-string - if isinstance(value, FormattedValue): - if i > last_idx: - # Write t-string until here - self._write_ftstring(node.values[last_idx:i], "t") - self.write(" ") - # Write f-string with the current formatted value - self._write_ftstring([node.values[i]], "f") - if i + 1 < len(node.values): - # Only add a space if there are more values after this - self.write(" ") - last_idx = i + 1 - - if last_idx < len(node.values): - # Write t-string from last_idx to end - self._write_ftstring(node.values[last_idx:], "t") - def visit_JoinedStr(self, node): self._write_ftstring(node.values, "f") def visit_TemplateStr(self, node): - self._tstring_helper(node) + self._write_ftstring(node.values, "t") def _write_ftstring_inner(self, node, is_format_spec=False): if isinstance(node, JoinedStr): @@ -679,9 +658,12 @@ def _unparse_interpolation_value(self, inner): unparser.set_precedence(_Precedence.TEST.next(), inner) return unparser.visit(inner) - def _write_interpolation(self, node): + def _write_interpolation(self, node, use_str_attr=False): with self.delimit("{", "}"): - expr = self._unparse_interpolation_value(node.value) + if use_str_attr: + expr = node.str + else: + expr = self._unparse_interpolation_value(node.value) if expr.startswith("{"): # Separate pair of opening brackets as "{ {" self.write(" ") @@ -696,7 +678,8 @@ def visit_FormattedValue(self, node): self._write_interpolation(node) def visit_Interpolation(self, node): - self._write_interpolation(node) + # If `str` is set to `None`, use the `value` to generate the source code. + self._write_interpolation(node, use_str_attr=node.str is not None) def visit_Name(self, node): self.write(node.id) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 51263d696a17773..996b094377a6cd6 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -49,7 +49,7 @@ def _f(): pass "Mapping", "MutableMapping", "MappingView", "KeysView", "ItemsView", "ValuesView", "Sequence", "MutableSequence", - "Buffer", + "ByteString", "Buffer", ] # This module has been renamed from collections.abc to _collections_abc to @@ -461,8 +461,8 @@ def __subclasshook__(cls, C): class _CallableGenericAlias(GenericAlias): """ Represent `Callable[argtypes, resulttype]`. - This sets ``__args__`` to a tuple containing the flattened ``argtypes`` - followed by ``resulttype``. + This sets ``__args__`` to a tuple containing the flattened + ``argtypes`` followed by ``resulttype``. Example: ``Callable[[int, str], float]`` sets ``__args__`` to ``(int, str, float)``. @@ -927,8 +927,9 @@ def __delitem__(self, key): __marker = object() def pop(self, key, default=__marker): - '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. + '''D.pop(k[,d]) -> v, remove specified key and return the corresponding + value. If key is not found, d is returned if given, otherwise + KeyError is raised. ''' try: value = self[key] @@ -962,9 +963,12 @@ def clear(self): def update(self, other=(), /, **kwds): ''' D.update([E, ]**F) -> None. Update D from mapping/iterable E and F. - If E present and has a .keys() method, does: for k in E.keys(): D[k] = E[k] - If E present and lacks .keys() method, does: for (k, v) in E: D[k] = v - In either case, this is followed by: for k, v in F.items(): D[k] = v + If E present and has a .keys() method, does: + for k in E.keys(): D[k] = E[k] + If E present and lacks .keys() method, does: + for (k, v) in E: D[k] = v + In either case, this is followed by: + for k, v in F.items(): D[k] = v ''' if isinstance(other, Mapping): for key in other: @@ -1029,8 +1033,8 @@ def __reversed__(self): yield self[i] def index(self, value, start=0, stop=None): - '''S.index(value, [start, [stop]]) -> integer -- return first index of value. - Raises ValueError if the value is not present. + '''S.index(value, [start, [stop]]) -> integer -- return first index of + value. Raises ValueError if the value is not present. Supporting start and stop arguments is optional, but recommended. @@ -1061,6 +1065,41 @@ def count(self, value): Sequence.register(range) Sequence.register(memoryview) +class _DeprecateByteStringMeta(ABCMeta): + def __new__(cls, name, bases, namespace, **kwargs): + if name != "ByteString": + import warnings + + warnings._deprecated( + "collections.abc.ByteString", + remove=(3, 17), + ) + return super().__new__(cls, name, bases, namespace, **kwargs) + + def __instancecheck__(cls, instance): + import warnings + + warnings._deprecated( + "collections.abc.ByteString", + remove=(3, 17), + ) + return super().__instancecheck__(instance) + +class ByteString(Sequence, metaclass=_DeprecateByteStringMeta): + """Deprecated ABC serving as a common supertype of ``bytes`` and ``bytearray``. + + This ABC is scheduled for removal in Python 3.17. + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the buffer protocol at runtime. For use in type annotations, + either use ``Buffer`` or a union that explicitly specifies the types your + code supports (e.g., ``bytes | bytearray | memoryview``). + """ + + __slots__ = () + +ByteString.register(bytes) +ByteString.register(bytearray) + class MutableSequence(Sequence): """All the operations on a read-write sequence. @@ -1103,15 +1142,16 @@ def reverse(self): self[i], self[n-i-1] = self[n-i-1], self[i] def extend(self, values): - 'S.extend(iterable) -- extend sequence by appending elements from the iterable' + """S.extend(iterable) -- extend sequence by appending elements from the + iterable""" if values is self: values = list(values) for v in values: self.append(v) def pop(self, index=-1): - '''S.pop([index]) -> item -- remove and return item at index (default last). - Raise IndexError if list is empty or index is out of range. + '''S.pop([index]) -> item -- remove and return item at index (default + last). Raise IndexError if list is empty or index is out of range. ''' v = self[index] del self[index] diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 4a310a402358b6c..d6673f6692f7615 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -1,4 +1,3 @@ -import io import os import sys @@ -155,7 +154,7 @@ def __iter__(self) -> Iterator[str]: return iter(self.__dataclass_fields__) -@dataclass(frozen=True) +@dataclass(frozen=True, kw_only=True) class Argparse(ThemeSection): usage: str = ANSIColors.BOLD_BLUE prog: str = ANSIColors.BOLD_MAGENTA @@ -176,6 +175,7 @@ class Argparse(ThemeSection): class Syntax(ThemeSection): prompt: str = ANSIColors.BOLD_MAGENTA keyword: str = ANSIColors.BOLD_BLUE + keyword_constant: str = ANSIColors.BOLD_BLUE builtin: str = ANSIColors.CYAN comment: str = ANSIColors.RED string: str = ANSIColors.GREEN @@ -272,21 +272,29 @@ def decolor(text: str) -> str: def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool: + + def _safe_getenv(k: str, fallback: str | None = None) -> str | None: + """Exception-safe environment retrieval. See gh-128636.""" + try: + return os.environ.get(k, fallback) + except Exception: + return fallback + if file is None: file = sys.stdout if not sys.flags.ignore_environment: - if os.environ.get("PYTHON_COLORS") == "0": + if _safe_getenv("PYTHON_COLORS") == "0": return False - if os.environ.get("PYTHON_COLORS") == "1": + if _safe_getenv("PYTHON_COLORS") == "1": return True - if os.environ.get("NO_COLOR"): + if _safe_getenv("NO_COLOR"): return False if not COLORIZE: return False - if os.environ.get("FORCE_COLOR"): + if _safe_getenv("FORCE_COLOR"): return True - if os.environ.get("TERM") == "dumb": + if _safe_getenv("TERM") == "dumb": return False if not hasattr(file, "fileno"): @@ -303,7 +311,7 @@ def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool: try: return os.isatty(file.fileno()) - except io.UnsupportedOperation: + except OSError: return hasattr(file, "isatty") and file.isatty() @@ -329,7 +337,8 @@ def get_theme( environment (including environment variable state and console configuration on Windows) can also change in the course of the application life cycle. """ - if force_color or (not force_no_color and can_colorize(file=tty_file)): + if force_color or (not force_no_color and + can_colorize(file=tty_file)): return _theme return theme_no_color diff --git a/Lib/_py_warnings.py b/Lib/_py_warnings.py index cbaa94458629ac6..36513ba2e05ef4d 100644 --- a/Lib/_py_warnings.py +++ b/Lib/_py_warnings.py @@ -589,6 +589,9 @@ def __str__(self): "line : %r}" % (self.message, self._category_name, self.filename, self.lineno, self.line)) + def __repr__(self): + return f'<{type(self).__qualname__} {self}>' + class catch_warnings(object): @@ -644,8 +647,8 @@ def __enter__(self): context = None self._filters = self._module.filters self._module.filters = self._filters[:] - self._showwarning = self._module.showwarning self._showwarnmsg_impl = self._module._showwarnmsg_impl + self._showwarning = self._module.showwarning self._module._filters_mutated_lock_held() if self._record: if _use_context: @@ -653,9 +656,9 @@ def __enter__(self): else: log = [] self._module._showwarnmsg_impl = log.append - # Reset showwarning() to the default implementation to make sure - # that _showwarnmsg() calls _showwarnmsg_impl() - self._module.showwarning = self._module._showwarning_orig + # Reset showwarning() to the default implementation to make sure + # that _showwarnmsg() calls _showwarnmsg_impl() + self._module.showwarning = self._module._showwarning_orig else: log = None if self._filter is not None: @@ -670,8 +673,8 @@ def __exit__(self, *exc_info): self._module._warnings_context.set(self._saved_context) else: self._module.filters = self._filters - self._module.showwarning = self._showwarning self._module._showwarnmsg_impl = self._showwarnmsg_impl + self._module.showwarning = self._showwarning self._module._filters_mutated_lock_held() @@ -762,27 +765,27 @@ def __new__(cls, /, *args, **kwargs): arg.__new__ = staticmethod(__new__) - original_init_subclass = arg.__init_subclass__ - # We need slightly different behavior if __init_subclass__ - # is a bound method (likely if it was implemented in Python) - if isinstance(original_init_subclass, MethodType): - original_init_subclass = original_init_subclass.__func__ + if "__init_subclass__" in arg.__dict__: + # __init_subclass__ is directly present on the decorated class. + # Synthesize a wrapper that calls this method directly. + original_init_subclass = arg.__init_subclass__ + # We need slightly different behavior if __init_subclass__ + # is a bound method (likely if it was implemented in Python). + # Otherwise, it likely means it's a builtin such as + # object's implementation of __init_subclass__. + if isinstance(original_init_subclass, MethodType): + original_init_subclass = original_init_subclass.__func__ @functools.wraps(original_init_subclass) def __init_subclass__(*args, **kwargs): _wm.warn(msg, category=category, stacklevel=stacklevel + 1) return original_init_subclass(*args, **kwargs) - - arg.__init_subclass__ = classmethod(__init_subclass__) - # Or otherwise, which likely means it's a builtin such as - # object's implementation of __init_subclass__. else: - @functools.wraps(original_init_subclass) - def __init_subclass__(*args, **kwargs): + def __init_subclass__(cls, *args, **kwargs): _wm.warn(msg, category=category, stacklevel=stacklevel + 1) - return original_init_subclass(*args, **kwargs) + return super(arg, cls).__init_subclass__(*args, **kwargs) - arg.__init_subclass__ = __init_subclass__ + arg.__init_subclass__ = classmethod(__init_subclass__) arg.__deprecated__ = __new__.__deprecated__ = msg __init_subclass__.__deprecated__ = msg diff --git a/Lib/_pydatetime.py b/Lib/_pydatetime.py index e3db1b52b1159bf..0f95a5750848237 100644 --- a/Lib/_pydatetime.py +++ b/Lib/_pydatetime.py @@ -55,7 +55,7 @@ def _days_before_year(year): def _days_in_month(year, month): "year, month -> number of days in that month in that year." - assert 1 <= month <= 12, month + assert 1 <= month <= 12, f"month must be in 1..12, not {month}" if month == 2 and _is_leap(year): return 29 return _DAYS_IN_MONTH[month] @@ -213,17 +213,6 @@ def _need_normalize_century(): _normalize_century = True return _normalize_century -_supports_c99 = None -def _can_support_c99(): - global _supports_c99 - if _supports_c99 is None: - try: - _supports_c99 = ( - _time.strftime("%F", (1900, 1, 1, 0, 0, 0, 0, 1, 0)) == "1900-01-01") - except ValueError: - _supports_c99 = False - return _supports_c99 - # Correctly substitute for %z and %Z escapes in strftime formats. def _wrap_strftime(object, format, timetuple): # Don't call utcoffset() or tzname() unless actually needed. @@ -283,12 +272,12 @@ def _wrap_strftime(object, format, timetuple): newformat.append(Zreplace) # Note that datetime(1000, 1, 1).strftime('%G') == '1000' so # year 1000 for %G can go on the fast path. - elif ((ch in 'YG' or ch in 'FC' and _can_support_c99()) and - object.year < 1000 and _need_normalize_century()): + elif (ch in 'YGFC' and timetuple[0] < 1000 and + _need_normalize_century()): if ch == 'G': year = int(_time.strftime("%G", timetuple)) else: - year = object.year + year = timetuple[0] if ch == 'C': push('{:02}'.format(year // 100)) else: @@ -369,7 +358,8 @@ def _find_isoformat_datetime_separator(dtstr): def _parse_isoformat_date(dtstr): # It is assumed that this is an ASCII-only string of lengths 7, 8 or 10, # see the comment on Modules/_datetimemodule.c:_find_isoformat_datetime_separator - assert len(dtstr) in (7, 8, 10) + if len(dtstr) not in (7, 8, 10): + raise ValueError("Invalid isoformat string") year = int(dtstr[0:4]) has_sep = dtstr[4] == '-' @@ -1127,8 +1117,8 @@ def isoformat(self): This is 'YYYY-MM-DD'. References: - - http://www.w3.org/TR/NOTE-datetime - - http://www.cl.cam.ac.uk/~mgk25/iso-time.html + - https://www.w3.org/TR/NOTE-datetime + - https://www.cl.cam.ac.uk/~mgk25/iso-time.html """ return "%04d-%02d-%02d" % (self._year, self._month, self._day) @@ -1262,7 +1252,7 @@ def isocalendar(self): The first week is 1; Monday is 1 ... Sunday is 7. ISO calendar algorithm taken from - http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm + https://www.phys.uu.nl/~vgent/calendar/isocalendar.htm (used with permission) """ year = self._year @@ -1958,7 +1948,7 @@ def fromisoformat(cls, date_string): if became_next_day: year, month, day = date_components # Only wrap day/month when it was previously valid - if month <= 12 and day <= (days_in_month := _days_in_month(year, month)): + if 1 <= month <= 12 and day <= (days_in_month := _days_in_month(year, month)): # Calculate midnight of the next day day += 1 if day > days_in_month: @@ -2139,7 +2129,7 @@ def isoformat(self, sep='T', timespec='auto'): By default, the fractional part is omitted if self.microsecond == 0. If self.tzinfo is not None, the UTC offset is also attached, giving - giving a full format of 'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM'. + a full format of 'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM'. Optional argument sep specifies the separator between date and time, default 'T'. diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index 46fa9ffcb1e0565..2277578df5412b7 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -104,8 +104,8 @@ class DecimalException(ArithmeticError): anything, though. handle -- Called when context._raise_error is called and the - trap_enabler is not set. First argument is self, second is the - context. More arguments can be given, those being after + trap_enabler is not set. First argument is self, second is + the context. More arguments can be given, those being after the explanation in _raise_error (For example, context._raise_error(NewError, '(-x)!', self._sign) would call NewError().handle(context, self._sign).) @@ -222,11 +222,12 @@ class InvalidContext(InvalidOperation): """Invalid context. Unknown rounding, for example. This occurs and signals invalid-operation if an invalid context was - detected during an operation. This can occur if contexts are not checked - on creation and either the precision exceeds the capability of the - underlying concrete representation or an unknown or unsupported rounding - was specified. These aspects of the context need only be checked when - the values are required to be used. The result is [0,qNaN]. + detected during an operation. This can occur if contexts are not + checked on creation and either the precision exceeds the capability of + the underlying concrete representation or an unknown or unsupported + rounding was specified. These aspects of the context need only be + checked when the values are required to be used. The result is + [0,qNaN]. """ def handle(self, context, *args): @@ -319,8 +320,9 @@ class FloatOperation(DecimalException, TypeError): Decimal.from_float() or context.create_decimal_from_float() do not set the flag. - Otherwise (the signal is trapped), only equality comparisons and explicit - conversions are silent. All other mixed operations raise FloatOperation. + Otherwise (the signal is trapped), only equality comparisons and + explicit conversions are silent. All other mixed operations raise + FloatOperation. """ # List of public traps and flags @@ -2898,8 +2900,8 @@ def compare_total(self, other, context=None): """Compares self to other using the abstract representations. This is not like the standard compare, which use their numerical - value. Note that a total ordering is defined for all possible abstract - representations. + value. Note that a total ordering is defined for all possible + abstract representations. """ other = _convert_other(other, raiseit=True) @@ -2970,7 +2972,8 @@ def compare_total(self, other, context=None): def compare_total_mag(self, other, context=None): """Compares self to other using abstract repr., ignoring sign. - Like compare_total, but with operand's sign ignored and assumed to be 0. + Like compare_total, but with operand's sign ignored and assumed to + be 0. """ other = _convert_other(other, raiseit=True) @@ -3340,7 +3343,10 @@ def _fill_logical(self, context, opa, opb): return opa, opb def logical_and(self, other, context=None): - """Applies an 'and' operation between self and other's digits.""" + """Applies an 'and' operation between self and other's digits. + + Both self and other must be logical numbers. + """ if context is None: context = getcontext() @@ -3357,14 +3363,20 @@ def logical_and(self, other, context=None): return _dec_from_triple(0, result.lstrip('0') or '0', 0) def logical_invert(self, context=None): - """Invert all its digits.""" + """Invert all its digits. + + The self must be logical number. + """ if context is None: context = getcontext() return self.logical_xor(_dec_from_triple(0,'1'*context.prec,0), context) def logical_or(self, other, context=None): - """Applies an 'or' operation between self and other's digits.""" + """Applies an 'or' operation between self and other's digits. + + Both self and other must be logical numbers. + """ if context is None: context = getcontext() @@ -3381,7 +3393,10 @@ def logical_or(self, other, context=None): return _dec_from_triple(0, result.lstrip('0') or '0', 0) def logical_xor(self, other, context=None): - """Applies an 'xor' operation between self and other's digits.""" + """Applies an 'xor' operation between self and other's digits. + + Both self and other must be logical numbers. + """ if context is None: context = getcontext() @@ -4095,9 +4110,9 @@ def create_decimal_from_float(self, f): def abs(self, a): """Returns the absolute value of the operand. - If the operand is negative, the result is the same as using the minus - operation on the operand. Otherwise, the result is the same as using - the plus operation on the operand. + If the operand is negative, the result is the same as using the + minus operation on the operand. Otherwise, the result is the same + as using the plus operation on the operand. >>> ExtendedContext.abs(Decimal('2.1')) Decimal('2.1') @@ -4153,16 +4168,17 @@ def canonical(self, a): def compare(self, a, b): """Compares values numerically. - If the signs of the operands differ, a value representing each operand - ('-1' if the operand is less than zero, '0' if the operand is zero or - negative zero, or '1' if the operand is greater than zero) is used in - place of that operand for the comparison instead of the actual - operand. + If the signs of the operands differ, a value representing each + operand ('-1' if the operand is less than zero, '0' if the operand + is zero or negative zero, or '1' if the operand is greater than + zero) is used in place of that operand for the comparison instead of + the actual operand. - The comparison is then effected by subtracting the second operand from - the first and then returning a value according to the result of the - subtraction: '-1' if the result is less than zero, '0' if the result is - zero or negative zero, or '1' if the result is greater than zero. + The comparison is then effected by subtracting the second operand + from the first and then returning a value according to the result of + the subtraction: '-1' if the result is less than zero, '0' if the + result is zero or negative zero, or '1' if the result is greater + than zero. >>> ExtendedContext.compare(Decimal('2.1'), Decimal('3')) Decimal('-1') @@ -4225,8 +4241,8 @@ def compare_total(self, a, b): """Compares two operands using their abstract representation. This is not like the standard compare, which use their numerical - value. Note that a total ordering is defined for all possible abstract - representations. + value. Note that a total ordering is defined for all possible + abstract representations. >>> ExtendedContext.compare_total(Decimal('12.73'), Decimal('127.9')) Decimal('-1') @@ -4253,7 +4269,8 @@ def compare_total(self, a, b): def compare_total_mag(self, a, b): """Compares two operands using their abstract representation ignoring sign. - Like compare_total, but with operand's sign ignored and assumed to be 0. + Like compare_total, but with operand's sign ignored and assumed to + be 0. """ a = _convert_other(a, raiseit=True) return a.compare_total_mag(b) @@ -4911,8 +4928,8 @@ def multiply(self, a, b): If either operand is a special value then the general rules apply. Otherwise, the operands are multiplied together - ('long multiplication'), resulting in a number which may be as long as - the sum of the lengths of the two operands. + ('long multiplication'), resulting in a number which may be as long + as the sum of the lengths of the two operands. >>> ExtendedContext.multiply(Decimal('1.20'), Decimal('3')) Decimal('3.60') @@ -5188,19 +5205,19 @@ def quantize(self, a, b): """Returns a value equal to 'a' (rounded), having the exponent of 'b'. The coefficient of the result is derived from that of the left-hand - operand. It may be rounded using the current rounding setting (if the - exponent is being increased), multiplied by a positive power of ten (if - the exponent is being decreased), or is unchanged (if the exponent is - already equal to that of the right-hand operand). + operand. It may be rounded using the current rounding setting (if + the exponent is being increased), multiplied by a positive power of + ten (if the exponent is being decreased), or is unchanged (if the + exponent is already equal to that of the right-hand operand). Unlike other operations, if the length of the coefficient after the quantize operation would be greater than precision then an Invalid - operation condition is raised. This guarantees that, unless there is - an error condition, the exponent of the result of a quantize is always - equal to that of the right-hand operand. + operation condition is raised. This guarantees that, unless there + is an error condition, the exponent of the result of a quantize is + always equal to that of the right-hand operand. - Also unlike other operations, quantize will never raise Underflow, even - if the result is subnormal and inexact. + Also unlike other operations, quantize will never raise Underflow, + even if the result is subnormal and inexact. >>> ExtendedContext.quantize(Decimal('2.17'), Decimal('0.001')) Decimal('2.170') @@ -5254,13 +5271,13 @@ def remainder(self, a, b): """Returns the remainder from integer division. The result is the residue of the dividend after the operation of - calculating integer division as described for divide-integer, rounded - to precision digits if necessary. The sign of the result, if - non-zero, is the same as that of the original dividend. + calculating integer division as described for divide-integer, + rounded to precision digits if necessary. The sign of the result, + if non-zero, is the same as that of the original dividend. - This operation will fail under the same conditions as integer division - (that is, if integer division on the same two operands would fail, the - remainder cannot be calculated). + This operation will fail under the same conditions as integer + division (that is, if integer division on the same two operands + would fail, the remainder cannot be calculated). >>> ExtendedContext.remainder(Decimal('2.1'), Decimal('3')) Decimal('2.1') @@ -5294,9 +5311,9 @@ def remainder_near(self, a, b): is chosen). If the result is equal to 0 then its sign will be the sign of a. - This operation will fail under the same conditions as integer division - (that is, if integer division on the same two operands would fail, the - remainder cannot be calculated). + This operation will fail under the same conditions as integer + division (that is, if integer division on the same two operands + would fail, the remainder cannot be calculated). >>> ExtendedContext.remainder_near(Decimal('2.1'), Decimal('3')) Decimal('-0.9') @@ -5354,8 +5371,8 @@ def rotate(self, a, b): def same_quantum(self, a, b): """Returns True if the two operands have the same exponent. - The result is never affected by either the sign or the coefficient of - either operand. + The result is never affected by either the sign or the coefficient + of either operand. >>> ExtendedContext.same_quantum(Decimal('2.17'), Decimal('0.001')) False @@ -5427,8 +5444,8 @@ def shift(self, a, b): def sqrt(self, a): """Square root of a non-negative number to context precision. - If the result must be inexact, it is rounded using the round-half-even - algorithm. + If the result must be inexact, it is rounded using the + round-half-even algorithm. >>> ExtendedContext.sqrt(Decimal('0')) Decimal('0') @@ -6120,9 +6137,13 @@ def _convert_for_comparison(self, other, equality_op=False): (?Pz)? (?P\#)? (?P0)? -(?P(?!0)\d+)? +(?P\d+)? (?P[,_])? -(?:\.(?P0|(?!0)\d+))? +(?:\. + (?=[\d,_]) # lookahead for digit or separator + (?P\d+)? + (?P[,_])? +)? (?P[eEfFgGn%])? \z """, re.VERBOSE|re.DOTALL) @@ -6215,6 +6236,9 @@ def _parse_format_specifier(format_spec, _localeconv=None): format_dict['grouping'] = [3, 0] format_dict['decimal_point'] = '.' + if format_dict['frac_separators'] is None: + format_dict['frac_separators'] = '' + return format_dict def _format_align(sign, body, spec): @@ -6334,6 +6358,11 @@ def _format_number(is_negative, intpart, fracpart, exp, spec): sign = _format_sign(is_negative, spec) + frac_sep = spec['frac_separators'] + if fracpart and frac_sep: + fracpart = frac_sep.join(fracpart[pos:pos + 3] + for pos in range(0, len(fracpart), 3)) + if fracpart or spec['alt']: fracpart = spec['decimal_point'] + fracpart diff --git a/Lib/_pyio.py b/Lib/_pyio.py index a870de5b5325429..cc336d2939443cd 100644 --- a/Lib/_pyio.py +++ b/Lib/_pyio.py @@ -83,27 +83,28 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, wrapped. (If a file descriptor is given, it is closed when the returned I/O object is closed, unless closefd is set to False.) - mode is an optional string that specifies the mode in which the file is - opened. It defaults to 'r' which means open for reading in text mode. Other - common values are 'w' for writing (truncating the file if it already - exists), 'x' for exclusive creation of a new file, and 'a' for appending - (which on some Unix systems, means that all writes append to the end of the - file regardless of the current seek position). In text mode, if encoding is - not specified the encoding used is platform dependent. (For reading and - writing raw bytes use binary mode and leave encoding unspecified.) The - available modes are: - - ========= =============================================================== + mode is an optional string that specifies the mode in which the file + is opened. It defaults to 'r' which means open for reading in text + mode. Other common values are 'w' for writing (truncating the file if + it already exists), 'x' for exclusive creation of a new file, and + 'a' for appending (which on some Unix systems, means that all writes + append to the end of the file regardless of the current seek position). + In text mode, if encoding is not specified the encoding used is platform + dependent. (For reading and writing raw bytes use binary mode and leave + encoding unspecified.) The available modes are: + + ========= ========================================================== Character Meaning - --------- --------------------------------------------------------------- + --------- ---------------------------------------------------------- 'r' open for reading (default) 'w' open for writing, truncating the file first 'x' create a new file and open it for writing - 'a' open for writing, appending to the end of the file if it exists + 'a' open for writing, appending to the end of the file if it + exists 'b' binary mode 't' text mode (default) '+' open a disk file for updating (reading and writing) - ========= =============================================================== + ========= ========================================================== The default mode is 'rt' (open for reading text). For binary random access, the mode 'w+b' opens and truncates the file to 0 bytes, while @@ -111,23 +112,23 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, raises an `FileExistsError` if the file already exists. Python distinguishes between files opened in binary and text modes, - even when the underlying operating system doesn't. Files opened in + even when the underlying operating system doesn't. Files opened in binary mode (appending 'b' to the mode argument) return contents as - bytes objects without any decoding. In text mode (the default, or when + bytes objects without any decoding. In text mode (the default, or when 't' is appended to the mode argument), the contents of the file are returned as strings, the bytes having been first decoded using a platform-dependent encoding or using the specified encoding if given. buffering is an optional integer used to set the buffering policy. - Pass 0 to switch buffering off (only allowed in binary mode), 1 to select - line buffering (only usable in text mode), and an integer > 1 to indicate - the size of a fixed-size chunk buffer. When no buffering argument is - given, the default buffering policy works as follows: + Pass 0 to switch buffering off (only allowed in binary mode), 1 to + select line buffering (only usable in text mode), and an integer > 1 to + indicate the size of a fixed-size chunk buffer. When no buffering + argument is given, the default buffering policy works as follows: - * Binary files are buffered in fixed-size chunks; the size of the buffer - is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) - when the device block size is available. - On most systems, the buffer will typically be 128 kilobytes long. + * Binary files are buffered in fixed-size chunks; the size of the buffer + is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) when the device + block size is available. + On most systems, the buffer will typically be 128 kilobytes long. * "Interactive" text files (files for which isatty() returns True) use line buffering. Other text files use the policy described above @@ -147,8 +148,8 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, encoding error strings. newline is a string controlling how universal newlines works (it only - applies to text mode). It can be None, '', '\n', '\r', and '\r\n'. It works - as follows: + applies to text mode). It can be None, '', '\n', '\r', and '\r\n'. It + works as follows: * On input, if newline is None, universal newlines mode is enabled. Lines in the input can end in '\n', '\r', or '\r\n', and @@ -164,17 +165,17 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None, other legal values, any '\n' characters written are translated to the given string. - closedfd is a bool. If closefd is False, the underlying file descriptor will - be kept open when the file is closed. This does not work when a file name is - given and must be True in that case. + closedfd is a bool. If closefd is False, the underlying file descriptor + will be kept open when the file is closed. This does not work when + a file name is given and must be True in that case. The newly created file is non-inheritable. A custom opener can be used by passing a callable as *opener*. The - underlying file descriptor for the file object is then obtained by calling - *opener* with (*file*, *flags*). *opener* must return an open file - descriptor (passing os.open as *opener* results in functionality similar to - passing None). + underlying file descriptor for the file object is then obtained by + calling *opener* with (*file*, *flags*). *opener* must return an open + file descriptor (passing os.open as *opener* results in functionality + similar to passing None). open() returns a file object whose type depends on the mode, and through which the standard file operations such as reading and writing @@ -351,10 +352,12 @@ def seek(self, pos, whence=0): interpreted relative to the position indicated by whence. Values for whence are ints: - * 0 -- start of stream (the default); offset should be zero or positive + * 0 -- start of stream (the default); offset should be zero or + positive * 1 -- current stream position; offset may be negative * 2 -- end of stream; offset is usually negative - Some operating systems / file systems could provide additional values. + Some operating systems / file systems could provide additional + values. Return an int indicating the new absolute position. """ @@ -367,8 +370,8 @@ def tell(self): def truncate(self, pos=None): """Truncate file to size bytes. - Size defaults to the current IO position as reported by tell(). Return - the new size. + Size defaults to the current IO position as reported by tell(). + Return the new size. """ self._unsupported("truncate") @@ -407,6 +410,9 @@ def __del__(self): if closed: return + if dealloc_warn := getattr(self, "_dealloc_warn", None): + dealloc_warn(self) + # If close() fails, the caller logs the exception with # sys.unraisablehook. close() must be called at the end at __del__(). self.close() @@ -489,7 +495,8 @@ def __exit__(self, *args): def fileno(self): """Returns underlying file descriptor (an int) if one exists. - An OSError is raised if the IO object does not use a file descriptor. + An OSError is raised if the IO object does not use a file + descriptor. """ self._unsupported("fileno") @@ -614,6 +621,8 @@ def read(self, size=-1): n = self.readinto(b) if n is None: return None + if n < 0 or n > len(b): + raise ValueError(f"readinto returned {n} outside buffer size {len(b)}") del b[n:] return bytes(b) @@ -645,8 +654,6 @@ def write(self, b): self._unsupported("write") io.RawIOBase.register(RawIOBase) -from _io import FileIO -RawIOBase.register(FileIO) class BufferedIOBase(IOBase): @@ -853,6 +860,10 @@ def __repr__(self): else: return "<{}.{} name={!r}>".format(modname, clsname, name) + def _dealloc_warn(self, source): + if dealloc_warn := getattr(self.raw, "_dealloc_warn", None): + dealloc_warn(source) + ### Lower-level APIs ### def fileno(self): @@ -928,20 +939,22 @@ def read1(self, size=-1): return self.read(size) def write(self, b): - if self.closed: - raise ValueError("write to closed file") if isinstance(b, str): raise TypeError("can't write str to binary stream") with memoryview(b) as view: + if self.closed: + raise ValueError("write to closed file") + n = view.nbytes # Size of any bytes-like object - if n == 0: - return 0 - pos = self._pos - if pos > len(self._buffer): - # Pad buffer to pos with null bytes. - self._buffer.resize(pos) - self._buffer[pos:pos + n] = b - self._pos += n + if n == 0: + return 0 + + pos = self._pos + if pos > len(self._buffer): + # Pad buffer to pos with null bytes. + self._buffer.resize(pos) + self._buffer[pos:pos + n] = view + self._pos += n return n def seek(self, pos, whence=0): @@ -1476,17 +1489,22 @@ class FileIO(RawIOBase): _closefd = True def __init__(self, file, mode='r', closefd=True, opener=None): - """Open a file. The mode can be 'r' (default), 'w', 'x' or 'a' for reading, - writing, exclusive creation or appending. The file will be created if it - doesn't exist when opened for writing or appending; it will be truncated - when opened for writing. A FileExistsError will be raised if it already - exists when opened for creating. Opening a file for creating implies - writing so this mode behaves in a similar way to 'w'. Add a '+' to the mode - to allow simultaneous reading and writing. A custom opener can be used by - passing a callable as *opener*. The underlying file descriptor for the file - object is then obtained by calling opener with (*name*, *flags*). - *opener* must return an open file descriptor (passing os.open as *opener* - results in functionality similar to passing None). + """Open a file. + + The mode can be 'r' (default), 'w', 'x' or 'a' for reading, + writing, exclusive creation or appending. The file will be created + if it doesn't exist when opened for writing or appending; it will be + truncated when opened for writing. A FileExistsError will be raised + if it already exists when opened for creating. Opening a file for + creating implies writing so this mode behaves in a similar way to + 'w'. Add a '+' to the mode to allow simultaneous reading and + writing. + + A custom opener can be used by passing a callable as *opener*. + The underlying file descriptor for the file object is then obtained + by calling opener with (*name*, *flags*). *opener* must return + an open file descriptor (passing os.open as *opener* results in + functionality similar to passing None). """ if self._fd >= 0: # Have to close the existing file first. @@ -1563,7 +1581,8 @@ def __init__(self, file, mode='r', closefd=True, opener=None): if not isinstance(fd, int): raise TypeError('expected integer from opener') if fd < 0: - raise OSError('Negative file descriptor') + # bpo-27066: Raise a ValueError for bad value. + raise ValueError(f'opener returned {fd}') owned_fd = fd if not noinherit_flag: os.set_inheritable(fd, False) @@ -1600,12 +1619,11 @@ def __init__(self, file, mode='r', closefd=True, opener=None): raise self._fd = fd - def __del__(self): + def _dealloc_warn(self, source): if self._fd >= 0 and self._closefd and not self.closed: import warnings - warnings.warn('unclosed file %r' % (self,), ResourceWarning, + warnings.warn(f'unclosed file {source!r}', ResourceWarning, stacklevel=2, source=self) - self.close() def __getstate__(self): raise TypeError(f"cannot pickle {self.__class__.__name__!r} object") @@ -1724,8 +1742,8 @@ def write(self, b): """Write bytes b to file, return number written. Only makes one system call, so not all of the data may be written. - The number of bytes actually written is returned. In non-blocking mode, - returns None if the write would block. + The number of bytes actually written is returned. In non-blocking + mode, returns None if the write would block. """ self._checkClosed() self._checkWritable() @@ -1737,11 +1755,12 @@ def write(self, b): def seek(self, pos, whence=SEEK_SET): """Move to new file position. - Argument offset is a byte count. Optional argument whence defaults to - SEEK_SET or 0 (offset from start of file, offset should be >= 0); other values - are SEEK_CUR or 1 (move relative to current position, positive or negative), - and SEEK_END or 2 (move relative to end of file, usually negative, although - many platforms allow seeking beyond the end of a file). + Argument offset is a byte count. Optional argument whence defaults + to SEEK_SET or 0 (offset from start of file, offset should be >= 0); + other values are SEEK_CUR or 1 (move relative to current position, + positive or negative), and SEEK_END or 2 (move relative to end of + file, usually negative, although many platforms allow seeking beyond + the end of a file). Note that not all file objects are seekable. """ @@ -1774,13 +1793,13 @@ def truncate(self, size=None): def close(self): """Close the file. - A closed file cannot be used for further I/O operations. close() may be - called more than once without error. + A closed file cannot be used for further I/O operations. + close() may be called more than once without error. """ if not self.closed: self._stat_atopen = None try: - if self._closefd: + if self._closefd and self._fd >= 0: os.close(self._fd) finally: super().close() @@ -1870,8 +1889,8 @@ class TextIOBase(IOBase): def read(self, size=-1): """Read at most size characters from stream, where size is an int. - Read from underlying buffer until we have size characters or we hit EOF. - If size is negative or omitted, read until EOF. + Read from underlying buffer until we have size characters or we hit + EOF. If size is negative or omitted, read until EOF. Returns a string. """ @@ -1885,7 +1904,7 @@ def truncate(self, pos=None): """Truncate size to pos, where pos is an int.""" self._unsupported("truncate") - def readline(self): + def readline(self, size=-1, /): """Read until newline or EOF. Returns an empty string if EOF is hit immediately. @@ -2689,6 +2708,10 @@ def readline(self, size=None): def newlines(self): return self._decoder.newlines if self._decoder else None + def _dealloc_warn(self, source): + if dealloc_warn := getattr(self.buffer, "_dealloc_warn", None): + dealloc_warn(source) + class StringIO(TextIOWrapper): """Text I/O implementation using an in-memory buffer. diff --git a/Lib/_pyrepl/_minimal_curses.py b/Lib/_pyrepl/_minimal_curses.py deleted file mode 100644 index d884f880f50ac75..000000000000000 --- a/Lib/_pyrepl/_minimal_curses.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Minimal '_curses' module, the low-level interface for curses module -which is not meant to be used directly. - -Based on ctypes. It's too incomplete to be really called '_curses', so -to use it, you have to import it and stick it in sys.modules['_curses'] -manually. - -Note that there is also a built-in module _minimal_curses which will -hide this one if compiled in. -""" - -import ctypes -import ctypes.util - - -class error(Exception): - pass - - -def _find_clib() -> str: - trylibs = ["ncursesw", "ncurses", "curses"] - - for lib in trylibs: - path = ctypes.util.find_library(lib) - if path: - return path - raise ModuleNotFoundError("curses library not found", name="_pyrepl._minimal_curses") - - -_clibpath = _find_clib() -clib = ctypes.cdll.LoadLibrary(_clibpath) - -clib.setupterm.argtypes = [ctypes.c_char_p, ctypes.c_int, ctypes.POINTER(ctypes.c_int)] -clib.setupterm.restype = ctypes.c_int - -clib.tigetstr.argtypes = [ctypes.c_char_p] -clib.tigetstr.restype = ctypes.c_ssize_t - -clib.tparm.argtypes = [ctypes.c_char_p] + 9 * [ctypes.c_int] # type: ignore[operator] -clib.tparm.restype = ctypes.c_char_p - -OK = 0 -ERR = -1 - -# ____________________________________________________________ - - -def setupterm(termstr, fd): - err = ctypes.c_int(0) - result = clib.setupterm(termstr, fd, ctypes.byref(err)) - if result == ERR: - raise error("setupterm() failed (err=%d)" % err.value) - - -def tigetstr(cap): - if not isinstance(cap, bytes): - cap = cap.encode("ascii") - result = clib.tigetstr(cap) - if result == ERR: - return None - return ctypes.cast(result, ctypes.c_char_p).value - - -def tparm(str, i1=0, i2=0, i3=0, i4=0, i5=0, i6=0, i7=0, i8=0, i9=0): - result = clib.tparm(str, i1, i2, i3, i4, i5, i6, i7, i8, i9) - if result is None: - raise error("tparm() returned NULL") - return result diff --git a/Lib/_pyrepl/_module_completer.py b/Lib/_pyrepl/_module_completer.py index 347f05607c75c51..2098d0a54aba316 100644 --- a/Lib/_pyrepl/_module_completer.py +++ b/Lib/_pyrepl/_module_completer.py @@ -1,9 +1,12 @@ from __future__ import annotations +import importlib +import os import pkgutil import sys import token import tokenize +from importlib.machinery import FileFinder from io import StringIO from contextlib import contextmanager from dataclasses import dataclass @@ -16,9 +19,18 @@ from typing import Any, Iterable, Iterator, Mapping +HARDCODED_SUBMODULES = { + # Standard library submodules that are not detected by pkgutil.iter_modules + # but can be imported, so should be proposed in completion + "collections": ["abc"], + "os": ["path"], + "xml.parsers.expat": ["errors", "model"], +} + + def make_default_module_completer() -> ModuleCompleter: - # Inside pyrepl, __package__ is set to '_pyrepl' - return ModuleCompleter(namespace={'__package__': '_pyrepl'}) + # Inside pyrepl, __package__ is set to None by default + return ModuleCompleter(namespace={'__package__': None}) class ModuleCompleter: @@ -41,12 +53,13 @@ def __init__(self, namespace: Mapping[str, Any] | None = None) -> None: self.namespace = namespace or {} self._global_cache: list[pkgutil.ModuleInfo] = [] self._curr_sys_path: list[str] = sys.path[:] + self._stdlib_path = os.path.dirname(importlib.__path__[0]) - def get_completions(self, line: str) -> list[str]: + def get_completions(self, line: str) -> list[str] | None: """Return the next possible import completions for 'line'.""" result = ImportParser(line).parse() if not result: - return [] + return None try: return self.complete(*result) except Exception: @@ -81,8 +94,11 @@ def find_modules(self, path: str, prefix: str) -> list[str]: def _find_modules(self, path: str, prefix: str) -> list[str]: if not path: # Top-level import (e.g. `import foo`` or `from foo`)` - return [name for _, name, _ in self.global_cache - if name.startswith(prefix)] + builtin_modules = [name for name in sys.builtin_module_names + if self.is_suggestion_match(name, prefix)] + third_party_modules = [module.name for module in self.global_cache + if self.is_suggestion_match(module.name, prefix)] + return sorted(builtin_modules + third_party_modules) if path.startswith('.'): # Convert relative path to absolute path @@ -92,12 +108,45 @@ def _find_modules(self, path: str, prefix: str) -> list[str]: return [] modules: Iterable[pkgutil.ModuleInfo] = self.global_cache + imported_module = sys.modules.get(path.split('.')[0]) + if imported_module: + # Filter modules to those who name and specs match the + # imported module to avoid invalid suggestions + spec = imported_module.__spec__ + if spec: + modules = [mod for mod in modules + if mod.name == spec.name + and mod.module_finder.find_spec(mod.name, None) == spec] + else: + modules = [] + + is_stdlib_import: bool | None = None for segment in path.split('.'): modules = [mod_info for mod_info in modules if mod_info.ispkg and mod_info.name == segment] + if is_stdlib_import is None: + # Top-level import decide if we import from stdlib or not + is_stdlib_import = all( + self._is_stdlib_module(mod_info) for mod_info in modules + ) modules = self.iter_submodules(modules) - return [module.name for module in modules - if module.name.startswith(prefix)] + + module_names = [module.name for module in modules] + if is_stdlib_import: + module_names.extend(HARDCODED_SUBMODULES.get(path, ())) + return [module_name for module_name in module_names + if self.is_suggestion_match(module_name, prefix)] + + def _is_stdlib_module(self, module_info: pkgutil.ModuleInfo) -> bool: + return (isinstance(module_info.module_finder, FileFinder) + and module_info.module_finder.path == self._stdlib_path) + + def is_suggestion_match(self, module_name: str, prefix: str) -> bool: + if prefix: + return module_name.startswith(prefix) + # For consistency with attribute completion, which + # does not suggest private attributes unless requested. + return not module_name.startswith("_") def iter_submodules(self, parent_modules: list[pkgutil.ModuleInfo]) -> Iterator[pkgutil.ModuleInfo]: """Iterate over all submodules of the given parent modules.""" @@ -159,7 +208,6 @@ def global_cache(self) -> list[pkgutil.ModuleInfo]: """Global module cache""" if not self._global_cache or self._curr_sys_path != sys.path: self._curr_sys_path = sys.path[:] - # print('getting packages') self._global_cache = list(pkgutil.iter_modules()) return self._global_cache diff --git a/Lib/_pyrepl/base_eventqueue.py b/Lib/_pyrepl/base_eventqueue.py index 842599bd1877fb4..0589a0f437ec7c5 100644 --- a/Lib/_pyrepl/base_eventqueue.py +++ b/Lib/_pyrepl/base_eventqueue.py @@ -87,7 +87,7 @@ def push(self, char: int | bytes) -> None: if isinstance(k, dict): self.keymap = k else: - self.insert(Event('key', k, self.flush_buf())) + self.insert(Event('key', k, bytes(self.flush_buf()))) self.keymap = self.compiled_keymap elif self.buf and self.buf[0] == 27: # escape @@ -96,7 +96,7 @@ def push(self, char: int | bytes) -> None: # the docstring in keymap.py trace('unrecognized escape sequence, propagating...') self.keymap = self.compiled_keymap - self.insert(Event('key', '\033', bytearray(b'\033'))) + self.insert(Event('key', '\033', b'\033')) for _c in self.flush_buf()[1:]: self.push(_c) @@ -106,5 +106,5 @@ def push(self, char: int | bytes) -> None: except UnicodeError: return else: - self.insert(Event('key', decoded, self.flush_buf())) + self.insert(Event('key', decoded, bytes(self.flush_buf()))) self.keymap = self.compiled_keymap diff --git a/Lib/_pyrepl/commands.py b/Lib/_pyrepl/commands.py index 2354fbb2ec2c1ea..10127e58897a580 100644 --- a/Lib/_pyrepl/commands.py +++ b/Lib/_pyrepl/commands.py @@ -370,6 +370,13 @@ def do(self) -> None: r = self.reader text = self.event * r.get_arg() r.insert(text) + if r.paste_mode: + data = "" + ev = r.console.getpending() + data += ev.data + if data: + r.insert(data) + r.last_refresh_cache.invalidated = True class insert_nl(EditCommand): @@ -413,14 +420,17 @@ class delete(EditCommand): def do(self) -> None: r = self.reader b = r.buffer - if ( - r.pos == 0 - and len(b) == 0 # this is something of a hack - and self.event[-1] == "\004" - ): - r.update_screen() - r.console.finish() - raise EOFError + if self.event[-1] == "\004": + if b and b[-1].endswith("\n"): + self.finish = True + elif ( + r.pos == 0 + and len(b) == 0 # this is something of a hack + ): + r.update_screen() + r.console.finish() + raise EOFError + for i in range(r.get_arg()): if r.pos != len(b): del b[r.pos] @@ -484,7 +494,6 @@ def do(self) -> None: data = "" start = time.time() while done not in data: - self.reader.console.wait(100) ev = self.reader.console.getpending() data += ev.data trace( diff --git a/Lib/_pyrepl/curses.py b/Lib/_pyrepl/curses.py deleted file mode 100644 index 3a624d9f6835d13..000000000000000 --- a/Lib/_pyrepl/curses.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2000-2010 Michael Hudson-Doyle -# Armin Rigo -# -# All Rights Reserved -# -# -# Permission to use, copy, modify, and distribute this software and -# its documentation for any purpose is hereby granted without fee, -# provided that the above copyright notice appear in all copies and -# that both that copyright notice and this permission notice appear in -# supporting documentation. -# -# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO -# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, -# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER -# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF -# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - - -try: - import _curses -except ImportError: - try: - import curses as _curses # type: ignore[no-redef] - except ImportError: - from . import _minimal_curses as _curses # type: ignore[no-redef] - -setupterm = _curses.setupterm -tigetstr = _curses.tigetstr -tparm = _curses.tparm -error = _curses.error diff --git a/Lib/_pyrepl/fancy_termios.py b/Lib/_pyrepl/fancy_termios.py index 0468b9a26702673..8d5bd183f213398 100644 --- a/Lib/_pyrepl/fancy_termios.py +++ b/Lib/_pyrepl/fancy_termios.py @@ -20,19 +20,25 @@ import termios +TYPE_CHECKING = False + +if TYPE_CHECKING: + from typing import cast +else: + cast = lambda typ, val: val + + class TermState: - def __init__(self, tuples): - ( - self.iflag, - self.oflag, - self.cflag, - self.lflag, - self.ispeed, - self.ospeed, - self.cc, - ) = tuples + def __init__(self, attrs: list[int | list[bytes]]) -> None: + self.iflag = cast(int, attrs[0]) + self.oflag = cast(int, attrs[1]) + self.cflag = cast(int, attrs[2]) + self.lflag = cast(int, attrs[3]) + self.ispeed = cast(int, attrs[4]) + self.ospeed = cast(int, attrs[5]) + self.cc = cast(list[bytes], attrs[6]) - def as_list(self): + def as_list(self) -> list[int | list[bytes]]: return [ self.iflag, self.oflag, @@ -45,32 +51,32 @@ def as_list(self): self.cc[:], ] - def copy(self): + def copy(self) -> "TermState": return self.__class__(self.as_list()) -def tcgetattr(fd): +def tcgetattr(fd: int) -> TermState: return TermState(termios.tcgetattr(fd)) -def tcsetattr(fd, when, attrs): +def tcsetattr(fd: int, when: int, attrs: TermState) -> None: termios.tcsetattr(fd, when, attrs.as_list()) class Term(TermState): TS__init__ = TermState.__init__ - def __init__(self, fd=0): + def __init__(self, fd: int = 0) -> None: self.TS__init__(termios.tcgetattr(fd)) self.fd = fd - self.stack = [] + self.stack: list[list[int | list[bytes]]] = [] - def save(self): + def save(self) -> None: self.stack.append(self.as_list()) - def set(self, when=termios.TCSANOW): + def set(self, when: int = termios.TCSANOW) -> None: termios.tcsetattr(self.fd, when, self.as_list()) - def restore(self): + def restore(self) -> None: self.TS__init__(self.stack.pop()) self.set() diff --git a/Lib/_pyrepl/main.py b/Lib/_pyrepl/main.py index a6f824dcc4ad140..447eb1e551e7743 100644 --- a/Lib/_pyrepl/main.py +++ b/Lib/_pyrepl/main.py @@ -1,6 +1,7 @@ import errno import os import sys +import types CAN_USE_PYREPL: bool @@ -29,12 +30,10 @@ def interactive_console(mainmodule=None, quiet=False, pythonstartup=False): print(FAIL_REASON, file=sys.stderr) return sys._baserepl() - if mainmodule: - namespace = mainmodule.__dict__ - else: - import __main__ - namespace = __main__.__dict__ - namespace.pop("__pyrepl_interactive_console", None) + if not mainmodule: + mainmodule = types.ModuleType("__main__") + + namespace = mainmodule.__dict__ # sys._baserepl() above does this internally, we do it here startup_path = os.getenv("PYTHONSTARTUP") diff --git a/Lib/_pyrepl/reader.py b/Lib/_pyrepl/reader.py index 0ebd9162eca4bba..f0116e742d2398e 100644 --- a/Lib/_pyrepl/reader.py +++ b/Lib/_pyrepl/reader.py @@ -619,6 +619,16 @@ def suspend(self) -> SimpleContextManager: setattr(self, arg, prev_state[arg]) self.prepare() + @contextmanager + def suspend_colorization(self) -> SimpleContextManager: + try: + old_can_colorize = self.can_colorize + self.can_colorize = False + yield + finally: + self.can_colorize = old_can_colorize + + def finish(self) -> None: """Called when a command signals that we're finished.""" pass @@ -634,6 +644,7 @@ def update_screen(self) -> None: def refresh(self) -> None: """Recalculate and refresh the screen.""" + self.console.height, self.console.width = self.console.getheightwidth() # this call sets up self.cxy, so call it first. self.screen = self.calc_screen() self.console.refresh(self.screen, self.cxy) diff --git a/Lib/_pyrepl/readline.py b/Lib/_pyrepl/readline.py index 560a9db192169e1..23b8fa6b9c7625e 100644 --- a/Lib/_pyrepl/readline.py +++ b/Lib/_pyrepl/readline.py @@ -43,10 +43,11 @@ Console: type[ConsoleType] _error: tuple[type[Exception], ...] | type[Exception] -try: - from .unix_console import UnixConsole as Console, _error -except ImportError: + +if os.name == "nt": from .windows_console import WindowsConsole as Console, _error +else: + from .unix_console import UnixConsole as Console, _error ENCODING = sys.getdefaultencoding() or "latin1" @@ -134,7 +135,8 @@ def get_stem(self) -> str: return "".join(b[p + 1 : self.pos]) def get_completions(self, stem: str) -> list[str]: - if module_completions := self.get_module_completions(): + module_completions = self.get_module_completions() + if module_completions is not None: return module_completions if len(stem) == 0 and self.more_lines is not None: b = self.buffer @@ -165,7 +167,7 @@ def get_completions(self, stem: str) -> list[str]: result.sort() return result - def get_module_completions(self) -> list[str]: + def get_module_completions(self) -> list[str] | None: line = self.get_line() return self.config.module_completer.get_completions(line) @@ -606,6 +608,7 @@ def _setup(namespace: Mapping[str, Any]) -> None: # set up namespace in rlcompleter, which requires it to be a bona fide dict if not isinstance(namespace, dict): namespace = dict(namespace) + _wrapper.config.module_completer = ModuleCompleter(namespace) _wrapper.config.readline_completer = RLCompleter(namespace).complete # this is not really what readline.c does. Better than nothing I guess diff --git a/Lib/_pyrepl/simple_interact.py b/Lib/_pyrepl/simple_interact.py index b3848833e142089..6508f0233b963ad 100644 --- a/Lib/_pyrepl/simple_interact.py +++ b/Lib/_pyrepl/simple_interact.py @@ -31,6 +31,7 @@ import sys import code import warnings +import errno from .readline import _get_reader, multiline_input, append_history_file @@ -110,6 +111,10 @@ def run_multiline_interactive_console( more_lines = functools.partial(_more_lines, console) input_n = 0 + _is_x_showrefcount_set = sys._xoptions.get("showrefcount") + _is_pydebug_build = hasattr(sys, "gettotalrefcount") + show_ref_count = _is_x_showrefcount_set and _is_pydebug_build + def maybe_run_command(statement: str) -> bool: statement = statement.strip() if statement in console.locals or statement not in REPL_COMMANDS: @@ -120,7 +125,7 @@ def maybe_run_command(statement: str) -> bool: command = REPL_COMMANDS[statement] if callable(command): # Make sure that history does not change because of commands - with reader.suspend_history(): + with reader.suspend_history(), reader.suspend_colorization(): command() return True return False @@ -149,9 +154,11 @@ def maybe_run_command(statement: str) -> bool: append_history_file() except (FileNotFoundError, PermissionError, OSError) as e: warnings.warn(f"failed to open the history file for writing: {e}") + input_n += 1 except KeyboardInterrupt: r = _get_reader() + r.cmpltn_reset() if r.input_trans is r.isearch_trans: r.do_cmd(("isearch-end", [""])) r.pos = len(r.get_unicode()) @@ -167,3 +174,8 @@ def maybe_run_command(statement: str) -> bool: except: console.showtraceback() console.resetbuffer() + if show_ref_count: + console.write( + f"[{sys.gettotalrefcount()} refs," + f" {sys.getallocatedblocks()} blocks]\n" + ) diff --git a/Lib/_pyrepl/terminfo.py b/Lib/_pyrepl/terminfo.py new file mode 100644 index 000000000000000..d02ef69cce0bd84 --- /dev/null +++ b/Lib/_pyrepl/terminfo.py @@ -0,0 +1,488 @@ +"""Pure Python curses-like terminal capability queries.""" + +from dataclasses import dataclass, field +import errno +import os +from pathlib import Path +import re +import struct + + +# Terminfo constants +MAGIC16 = 0o432 # Magic number for 16-bit terminfo format +MAGIC32 = 0o1036 # Magic number for 32-bit terminfo format + +# Special values for absent/cancelled capabilities +ABSENT_BOOLEAN = -1 +ABSENT_NUMERIC = -1 +CANCELLED_NUMERIC = -2 +ABSENT_STRING = None +CANCELLED_STRING = None + + +# Standard string capability names from ncurses Caps file +# This matches the order used by ncurses when compiling terminfo +# fmt: off +_STRING_NAMES: tuple[str, ...] = ( + "cbt", "bel", "cr", "csr", "tbc", "clear", "el", "ed", "hpa", "cmdch", + "cup", "cud1", "home", "civis", "cub1", "mrcup", "cnorm", "cuf1", "ll", + "cuu1", "cvvis", "dch1", "dl1", "dsl", "hd", "smacs", "blink", "bold", + "smcup", "smdc", "dim", "smir", "invis", "prot", "rev", "smso", "smul", + "ech", "rmacs", "sgr0", "rmcup", "rmdc", "rmir", "rmso", "rmul", "flash", + "ff", "fsl", "is1", "is2", "is3", "if", "ich1", "il1", "ip", "kbs", "ktbc", + "kclr", "kctab", "kdch1", "kdl1", "kcud1", "krmir", "kel", "ked", "kf0", + "kf1", "kf10", "kf2", "kf3", "kf4", "kf5", "kf6", "kf7", "kf8", "kf9", + "khome", "kich1", "kil1", "kcub1", "kll", "knp", "kpp", "kcuf1", "kind", + "kri", "khts", "kcuu1", "rmkx", "smkx", "lf0", "lf1", "lf10", "lf2", "lf3", + "lf4", "lf5", "lf6", "lf7", "lf8", "lf9", "rmm", "smm", "nel", "pad", "dch", + "dl", "cud", "ich", "indn", "il", "cub", "cuf", "rin", "cuu", "pfkey", + "pfloc", "pfx", "mc0", "mc4", "mc5", "rep", "rs1", "rs2", "rs3", "rf", "rc", + "vpa", "sc", "ind", "ri", "sgr", "hts", "wind", "ht", "tsl", "uc", "hu", + "iprog", "ka1", "ka3", "kb2", "kc1", "kc3", "mc5p", "rmp", "acsc", "pln", + "kcbt", "smxon", "rmxon", "smam", "rmam", "xonc", "xoffc", "enacs", "smln", + "rmln", "kbeg", "kcan", "kclo", "kcmd", "kcpy", "kcrt", "kend", "kent", + "kext", "kfnd", "khlp", "kmrk", "kmsg", "kmov", "knxt", "kopn", "kopt", + "kprv", "kprt", "krdo", "kref", "krfr", "krpl", "krst", "kres", "ksav", + "kspd", "kund", "kBEG", "kCAN", "kCMD", "kCPY", "kCRT", "kDC", "kDL", + "kslt", "kEND", "kEOL", "kEXT", "kFND", "kHLP", "kHOM", "kIC", "kLFT", + "kMSG", "kMOV", "kNXT", "kOPT", "kPRV", "kPRT", "kRDO", "kRPL", "kRIT", + "kRES", "kSAV", "kSPD", "kUND", "rfi", "kf11", "kf12", "kf13", "kf14", + "kf15", "kf16", "kf17", "kf18", "kf19", "kf20", "kf21", "kf22", "kf23", + "kf24", "kf25", "kf26", "kf27", "kf28", "kf29", "kf30", "kf31", "kf32", + "kf33", "kf34", "kf35", "kf36", "kf37", "kf38", "kf39", "kf40", "kf41", + "kf42", "kf43", "kf44", "kf45", "kf46", "kf47", "kf48", "kf49", "kf50", + "kf51", "kf52", "kf53", "kf54", "kf55", "kf56", "kf57", "kf58", "kf59", + "kf60", "kf61", "kf62", "kf63", "el1", "mgc", "smgl", "smgr", "fln", "sclk", + "dclk", "rmclk", "cwin", "wingo", "hup","dial", "qdial", "tone", "pulse", + "hook", "pause", "wait", "u0", "u1", "u2", "u3", "u4", "u5", "u6", "u7", + "u8", "u9", "op", "oc", "initc", "initp", "scp", "setf", "setb", "cpi", + "lpi", "chr", "cvr", "defc", "swidm", "sdrfq", "sitm", "slm", "smicm", + "snlq", "snrmq", "sshm", "ssubm", "ssupm", "sum", "rwidm", "ritm", "rlm", + "rmicm", "rshm", "rsubm", "rsupm", "rum", "mhpa", "mcud1", "mcub1", "mcuf1", + "mvpa", "mcuu1", "porder", "mcud", "mcub", "mcuf", "mcuu", "scs", "smgb", + "smgbp", "smglp", "smgrp", "smgt", "smgtp", "sbim", "scsd", "rbim", "rcsd", + "subcs", "supcs", "docr", "zerom", "csnm", "kmous", "minfo", "reqmp", + "getm", "setaf", "setab", "pfxl", "devt", "csin", "s0ds", "s1ds", "s2ds", + "s3ds", "smglr", "smgtb", "birep", "binel", "bicr", "colornm", "defbi", + "endbi", "setcolor", "slines", "dispc", "smpch", "rmpch", "smsc", "rmsc", + "pctrm", "scesc", "scesa", "ehhlm", "elhlm", "elohlm", "erhlm", "ethlm", + "evhlm", "sgr1", "slength", "OTi2", "OTrs", "OTnl", "OTbc", "OTko", "OTma", + "OTG2", "OTG3", "OTG1", "OTG4", "OTGR", "OTGL", "OTGU", "OTGD", "OTGH", + "OTGV", "OTGC","meml", "memu", "box1" +) +# fmt: on + + +def _get_terminfo_dirs() -> list[Path]: + """Get list of directories to search for terminfo files. + + Based on ncurses behavior in: + - ncurses/tinfo/db_iterator.c:_nc_next_db() + - ncurses/tinfo/read_entry.c:_nc_read_entry() + """ + dirs = [] + + terminfo = os.environ.get("TERMINFO") + if terminfo: + dirs.append(terminfo) + + try: + home = Path.home() + dirs.append(str(home / ".terminfo")) + except RuntimeError: + pass + + # Check TERMINFO_DIRS + terminfo_dirs = os.environ.get("TERMINFO_DIRS", "") + if terminfo_dirs: + for d in terminfo_dirs.split(":"): + if d: + dirs.append(d) + + dirs.extend( + [ + "/etc/terminfo", + "/lib/terminfo", + "/usr/lib/terminfo", + "/usr/share/terminfo", + "/usr/share/lib/terminfo", + "/usr/share/misc/terminfo", + "/usr/local/lib/terminfo", + "/usr/local/share/terminfo", + ] + ) + + return [Path(d) for d in dirs if Path(d).is_dir()] + + +def _validate_terminal_name_or_raise(terminal_name: str) -> None: + if not isinstance(terminal_name, str): + raise TypeError("`terminal_name` must be a string") + + if not terminal_name: + raise ValueError("`terminal_name` cannot be empty") + + if "\x00" in terminal_name: + raise ValueError("NUL character found in `terminal_name`") + + t = Path(terminal_name) + if len(t.parts) > 1: + raise ValueError("`terminal_name` cannot contain path separators") + + +def _read_terminfo_file(terminal_name: str) -> bytes: + """Find and read terminfo file for given terminal name. + + Terminfo files are stored in directories using the first character + of the terminal name as a subdirectory. + """ + _validate_terminal_name_or_raise(terminal_name) + first_char = terminal_name[0].lower() + filename = terminal_name + + for directory in _get_terminfo_dirs(): + path = directory / first_char / filename + if path.is_file(): + return path.read_bytes() + + # Try with hex encoding of first char (for special chars) + hex_dir = "%02x" % ord(first_char) + path = directory / hex_dir / filename + if path.is_file(): + return path.read_bytes() + + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), filename) + + +# Hard-coded terminal capabilities for common terminals +# This is a minimal subset needed by PyREPL +_TERMINAL_CAPABILITIES = { + # ANSI/xterm-compatible terminals + "ansi": { + # Bell + "bel": b"\x07", + # Cursor movement + "cub": b"\x1b[%p1%dD", # Move cursor left N columns + "cud": b"\x1b[%p1%dB", # Move cursor down N rows + "cuf": b"\x1b[%p1%dC", # Move cursor right N columns + "cuu": b"\x1b[%p1%dA", # Move cursor up N rows + "cub1": b"\x08", # Move cursor left 1 column + "cud1": b"\n", # Move cursor down 1 row + "cuf1": b"\x1b[C", # Move cursor right 1 column + "cuu1": b"\x1b[A", # Move cursor up 1 row + "cup": b"\x1b[%i%p1%d;%p2%dH", # Move cursor to row, column + "hpa": b"\x1b[%i%p1%dG", # Move cursor to column + # Clear operations + "clear": b"\x1b[H\x1b[2J", # Clear screen and home cursor + "el": b"\x1b[K", # Clear to end of line + # Insert/delete + "dch": b"\x1b[%p1%dP", # Delete N characters + "dch1": b"\x1b[P", # Delete 1 character + "ich": b"\x1b[%p1%d@", # Insert N characters + "ich1": b"", # Insert 1 character + # Cursor visibility + "civis": b"\x1b[?25l", # Make cursor invisible + "cnorm": b"\x1b[?12l\x1b[?25h", # Make cursor normal (visible) + # Scrolling + "ind": b"\n", # Scroll up one line + "ri": b"\x1bM", # Scroll down one line + # Keypad mode + "smkx": b"\x1b[?1h\x1b=", # Enable keypad mode + "rmkx": b"\x1b[?1l\x1b>", # Disable keypad mode + # Padding (not used in modern terminals) + "pad": b"", + # Function keys and special keys + "kdch1": b"\x1b[3~", # Delete key + "kcud1": b"\x1bOB", # Down arrow + "kend": b"\x1bOF", # End key + "kent": b"\x1bOM", # Enter key + "khome": b"\x1bOH", # Home key + "kich1": b"\x1b[2~", # Insert key + "kcub1": b"\x1bOD", # Left arrow + "knp": b"\x1b[6~", # Page down + "kpp": b"\x1b[5~", # Page up + "kcuf1": b"\x1bOC", # Right arrow + "kcuu1": b"\x1bOA", # Up arrow + # Function keys F1-F20 + "kf1": b"\x1bOP", + "kf2": b"\x1bOQ", + "kf3": b"\x1bOR", + "kf4": b"\x1bOS", + "kf5": b"\x1b[15~", + "kf6": b"\x1b[17~", + "kf7": b"\x1b[18~", + "kf8": b"\x1b[19~", + "kf9": b"\x1b[20~", + "kf10": b"\x1b[21~", + "kf11": b"\x1b[23~", + "kf12": b"\x1b[24~", + "kf13": b"\x1b[1;2P", + "kf14": b"\x1b[1;2Q", + "kf15": b"\x1b[1;2R", + "kf16": b"\x1b[1;2S", + "kf17": b"\x1b[15;2~", + "kf18": b"\x1b[17;2~", + "kf19": b"\x1b[18;2~", + "kf20": b"\x1b[19;2~", + }, + # Dumb terminal - minimal capabilities + "dumb": { + "bel": b"\x07", # Bell + "cud1": b"\n", # Move down 1 row (newline) + "ind": b"\n", # Scroll up one line (newline) + }, + # Linux console + "linux": { + # Bell + "bel": b"\x07", + # Cursor movement + "cub": b"\x1b[%p1%dD", # Move cursor left N columns + "cud": b"\x1b[%p1%dB", # Move cursor down N rows + "cuf": b"\x1b[%p1%dC", # Move cursor right N columns + "cuu": b"\x1b[%p1%dA", # Move cursor up N rows + "cub1": b"\x08", # Move cursor left 1 column (backspace) + "cud1": b"\n", # Move cursor down 1 row (newline) + "cuf1": b"\x1b[C", # Move cursor right 1 column + "cuu1": b"\x1b[A", # Move cursor up 1 row + "cup": b"\x1b[%i%p1%d;%p2%dH", # Move cursor to row, column + "hpa": b"\x1b[%i%p1%dG", # Move cursor to column + # Clear operations + "clear": b"\x1b[H\x1b[J", # Clear screen and home cursor (different from ansi!) + "el": b"\x1b[K", # Clear to end of line + # Insert/delete + "dch": b"\x1b[%p1%dP", # Delete N characters + "dch1": b"\x1b[P", # Delete 1 character + "ich": b"\x1b[%p1%d@", # Insert N characters + "ich1": b"\x1b[@", # Insert 1 character + # Cursor visibility + "civis": b"\x1b[?25l\x1b[?1c", # Make cursor invisible + "cnorm": b"\x1b[?25h\x1b[?0c", # Make cursor normal + # Scrolling + "ind": b"\n", # Scroll up one line + "ri": b"\x1bM", # Scroll down one line + # Keypad mode + "smkx": b"\x1b[?1h\x1b=", # Enable keypad mode + "rmkx": b"\x1b[?1l\x1b>", # Disable keypad mode + # Function keys and special keys + "kdch1": b"\x1b[3~", # Delete key + "kcud1": b"\x1b[B", # Down arrow + "kend": b"\x1b[4~", # End key (different from ansi!) + "khome": b"\x1b[1~", # Home key (different from ansi!) + "kich1": b"\x1b[2~", # Insert key + "kcub1": b"\x1b[D", # Left arrow + "knp": b"\x1b[6~", # Page down + "kpp": b"\x1b[5~", # Page up + "kcuf1": b"\x1b[C", # Right arrow + "kcuu1": b"\x1b[A", # Up arrow + # Function keys + "kf1": b"\x1b[[A", + "kf2": b"\x1b[[B", + "kf3": b"\x1b[[C", + "kf4": b"\x1b[[D", + "kf5": b"\x1b[[E", + "kf6": b"\x1b[17~", + "kf7": b"\x1b[18~", + "kf8": b"\x1b[19~", + "kf9": b"\x1b[20~", + "kf10": b"\x1b[21~", + "kf11": b"\x1b[23~", + "kf12": b"\x1b[24~", + "kf13": b"\x1b[25~", + "kf14": b"\x1b[26~", + "kf15": b"\x1b[28~", + "kf16": b"\x1b[29~", + "kf17": b"\x1b[31~", + "kf18": b"\x1b[32~", + "kf19": b"\x1b[33~", + "kf20": b"\x1b[34~", + }, +} + +# Map common TERM values to capability sets +_TERM_ALIASES = { + "xterm": "ansi", + "xterm-color": "ansi", + "xterm-256color": "ansi", + "screen": "ansi", + "screen-256color": "ansi", + "tmux": "ansi", + "tmux-256color": "ansi", + "vt100": "ansi", + "vt220": "ansi", + "rxvt": "ansi", + "rxvt-unicode": "ansi", + "rxvt-unicode-256color": "ansi", + "unknown": "dumb", +} + + +@dataclass +class TermInfo: + terminal_name: str | bytes | None + fallback: bool = True + + _capabilities: dict[str, bytes] = field(default_factory=dict) + + def __post_init__(self) -> None: + """Initialize terminal capabilities for the given terminal type. + + Based on ncurses implementation in: + - ncurses/tinfo/lib_setup.c:setupterm() and _nc_setupterm() + - ncurses/tinfo/lib_setup.c:TINFO_SETUP_TERM() + + This version first attempts to read terminfo database files like ncurses, + then, if `fallback` is True, falls back to hardcoded capabilities for + common terminal types. + """ + # If termstr is None or empty, try to get from environment + if not self.terminal_name: + self.terminal_name = os.environ.get("TERM") or "ANSI" + + if isinstance(self.terminal_name, bytes): + self.terminal_name = self.terminal_name.decode("ascii") + + try: + self._parse_terminfo_file(self.terminal_name) + except (OSError, ValueError): + if not self.fallback: + raise + + term_type = _TERM_ALIASES.get( + self.terminal_name, self.terminal_name + ) + if term_type not in _TERMINAL_CAPABILITIES: + term_type = "dumb" + self._capabilities = _TERMINAL_CAPABILITIES[term_type].copy() + + def _parse_terminfo_file(self, terminal_name: str) -> None: + """Parse a terminfo file. + + Populate the _capabilities dict for easy retrieval + + Based on ncurses implementation in: + - ncurses/tinfo/read_entry.c:_nc_read_termtype() + - ncurses/tinfo/read_entry.c:_nc_read_file_entry() + - ncurses/tinfo/lib_ti.c:tigetstr() + """ + data = _read_terminfo_file(terminal_name) + too_short = f"TermInfo file for {terminal_name!r} too short" + offset = 12 + if len(data) < offset: + raise ValueError(too_short) + + magic, name_size, bool_count, num_count, str_count, str_size = ( + struct.unpack(" len(data): + raise ValueError(too_short) + + # Read string offsets + end_offset = offset + 2 * str_count + if offset > len(data): + raise ValueError(too_short) + string_offset_data = data[offset:end_offset] + string_offsets = [ + off for [off] in struct.iter_unpack(" len(data): + raise ValueError(too_short) + string_table = data[offset : offset + str_size] + + # Extract strings from string table + capabilities = {} + for cap, off in zip(_STRING_NAMES, string_offsets): + if off < 0: + # CANCELLED_STRING; we do not store those + continue + elif off < len(string_table): + # Find null terminator + end = string_table.find(0, off) + if end >= 0: + capabilities[cap] = string_table[off:end] + # in other cases this is ABSENT_STRING; we don't store those. + + # Note: we don't support extended capabilities since PyREPL doesn't + # need them. + + self._capabilities = capabilities + + def get(self, cap: str) -> bytes | None: + """Get terminal capability string by name. + """ + if not isinstance(cap, str): + raise TypeError(f"`cap` must be a string, not {type(cap)}") + + return self._capabilities.get(cap) + + +def tparm(cap_bytes: bytes, *params: int) -> bytes: + """Parameterize a terminal capability string. + + Based on ncurses implementation in: + - ncurses/tinfo/lib_tparm.c:tparm() + - ncurses/tinfo/lib_tparm.c:tparam_internal() + + The ncurses version implements a full stack-based interpreter for + terminfo parameter strings. This pure Python version implements only + the subset of parameter substitution operations needed by PyREPL: + - %i (increment parameters for 1-based indexing) + - %p[1-9]%d (parameter substitution) + - %p[1-9]%{n}%+%d (parameter plus constant) + """ + if not isinstance(cap_bytes, bytes): + raise TypeError(f"`cap` must be bytes, not {type(cap_bytes)}") + + result = cap_bytes + + # %i - increment parameters (1-based instead of 0-based) + increment = b"%i" in result + if increment: + result = result.replace(b"%i", b"") + + # Replace %p1%d, %p2%d, etc. with actual parameter values + for i in range(len(params)): + pattern = b"%%p%d%%d" % (i + 1) + if pattern in result: + value = params[i] + if increment: + value += 1 + result = result.replace(pattern, str(value).encode("ascii")) + + # Handle %p1%{1}%+%d (parameter plus constant) + # Used in some cursor positioning sequences + pattern_re = re.compile(rb"%p(\d)%\{(\d+)\}%\+%d") + matches = list(pattern_re.finditer(result)) + for match in reversed(matches): # reversed to maintain positions + param_idx = int(match.group(1)) + constant = int(match.group(2)) + value = params[param_idx] + constant + result = ( + result[: match.start()] + + str(value).encode("ascii") + + result[match.end() :] + ) + + return result diff --git a/Lib/_pyrepl/trace.py b/Lib/_pyrepl/trace.py index a8eb2433cd3ccee..943ee12f964b296 100644 --- a/Lib/_pyrepl/trace.py +++ b/Lib/_pyrepl/trace.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import sys # types if False: @@ -12,10 +13,22 @@ trace_file = open(trace_filename, "a") -def trace(line: str, *k: object, **kw: object) -> None: - if trace_file is None: - return - if k or kw: - line = line.format(*k, **kw) - trace_file.write(line + "\n") - trace_file.flush() + +if sys.platform == "emscripten": + from posix import _emscripten_log + + def trace(line: str, *k: object, **kw: object) -> None: + if "PYREPL_TRACE" not in os.environ: + return + if k or kw: + line = line.format(*k, **kw) + _emscripten_log(line) + +else: + def trace(line: str, *k: object, **kw: object) -> None: + if trace_file is None: + return + if k or kw: + line = line.format(*k, **kw) + trace_file.write(line + "\n") + trace_file.flush() diff --git a/Lib/_pyrepl/unix_console.py b/Lib/_pyrepl/unix_console.py index d21cdd9b076d869..639d16db3f88d41 100644 --- a/Lib/_pyrepl/unix_console.py +++ b/Lib/_pyrepl/unix_console.py @@ -33,9 +33,9 @@ import platform from fcntl import ioctl -from . import curses +from . import terminfo from .console import Console, Event -from .fancy_termios import tcgetattr, tcsetattr +from .fancy_termios import tcgetattr, tcsetattr, TermState from .trace import trace from .unix_eventqueue import EventQueue from .utils import wlen @@ -51,16 +51,19 @@ # types if TYPE_CHECKING: - from typing import IO, Literal, overload + from typing import AbstractSet, IO, Literal, overload, cast else: overload = lambda func: None + cast = lambda typ, val: val class InvalidTerminal(RuntimeError): - pass + def __init__(self, message: str) -> None: + super().__init__(errno.EIO, message) -_error = (termios.error, curses.error, InvalidTerminal) +_error = (termios.error, InvalidTerminal) +_error_codes_to_ignore = frozenset([errno.EIO, errno.ENXIO, errno.EPERM]) SIGWINCH_EVENT = "repaint" @@ -125,12 +128,13 @@ def __init__(self): def register(self, fd, flag): self.fd = fd + # note: The 'timeout' argument is received as *milliseconds* def poll(self, timeout: float | None = None) -> list[int]: if timeout is None: r, w, e = select.select([self.fd], [], []) else: - r, w, e = select.select([self.fd], [], [], timeout/1000) + r, w, e = select.select([self.fd], [], [], timeout / 1000) return r poll = MinimalPoll # type: ignore[assignment] @@ -157,17 +161,28 @@ def __init__( self.pollob = poll() self.pollob.register(self.input_fd, select.POLLIN) - curses.setupterm(term or None, self.output_fd) + self.terminfo = terminfo.TermInfo(term or None) self.term = term + self.is_apple_terminal = ( + platform.system() == "Darwin" + and os.getenv("TERM_PROGRAM") == "Apple_Terminal" + ) + + try: + self.__input_fd_set(tcgetattr(self.input_fd), ignore=frozenset()) + except _error as e: + raise RuntimeError(f"termios failure ({e.args[1]})") @overload - def _my_getstr(cap: str, optional: Literal[False] = False) -> bytes: ... + def _my_getstr( + cap: str, optional: Literal[False] = False + ) -> bytes: ... @overload def _my_getstr(cap: str, optional: bool) -> bytes | None: ... def _my_getstr(cap: str, optional: bool = False) -> bytes | None: - r = curses.tigetstr(cap) + r = self.terminfo.get(cap) if not optional and r is None: raise InvalidTerminal( f"terminal doesn't have the required {cap} capability" @@ -201,7 +216,9 @@ def _my_getstr(cap: str, optional: bool = False) -> bytes | None: self.__setup_movement() - self.event_queue = EventQueue(self.input_fd, self.encoding) + self.event_queue = EventQueue( + self.input_fd, self.encoding, self.terminfo + ) self.cursor_visible = 1 signal.signal(signal.SIGCONT, self._sigcont_handler) @@ -213,7 +230,6 @@ def _sigcont_handler(self, signum, frame): def __read(self, n: int) -> bytes: return os.read(self.input_fd, n) - def change_encoding(self, encoding: str) -> None: """ Change the encoding used for I/O operations. @@ -235,8 +251,9 @@ def refresh(self, screen, c_xy): if not self.__gone_tall: while len(self.screen) < min(len(screen), self.height): self.__hide_cursor() - self.__move(0, len(self.screen) - 1) - self.__write("\n") + if self.screen: + self.__move(0, len(self.screen) - 1) + self.__write("\n") self.posxy = 0, len(self.screen) self.screen.append("") else: @@ -325,6 +342,8 @@ def prepare(self): """ Prepare the console for input/output operations. """ + self.__buffer = [] + self.__svtermstate = tcgetattr(self.input_fd) raw = self.__svtermstate.copy() raw.iflag &= ~(termios.INPCK | termios.ISTRIP | termios.IXON) @@ -336,17 +355,15 @@ def prepare(self): raw.lflag |= termios.ISIG raw.cc[termios.VMIN] = 1 raw.cc[termios.VTIME] = 0 - tcsetattr(self.input_fd, termios.TCSADRAIN, raw) + self.__input_fd_set(raw) # In macOS terminal we need to deactivate line wrap via ANSI escape code - if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + if self.is_apple_terminal: os.write(self.output_fd, b"\033[?7l") self.screen = [] self.height, self.width = self.getheightwidth() - self.__buffer = [] - self.posxy = 0, 0 self.__gone_tall = 0 self.__move = self.__move_short @@ -368,13 +385,18 @@ def restore(self): self.__disable_bracketed_paste() self.__maybe_write_code(self._rmkx) self.flushoutput() - tcsetattr(self.input_fd, termios.TCSADRAIN, self.__svtermstate) + self.__input_fd_set(self.__svtermstate) - if platform.system() == "Darwin" and os.getenv("TERM_PROGRAM") == "Apple_Terminal": + if self.is_apple_terminal: os.write(self.output_fd, b"\033[?7h") if hasattr(self, "old_sigwinch"): - signal.signal(signal.SIGWINCH, self.old_sigwinch) + try: + signal.signal(signal.SIGWINCH, self.old_sigwinch) + except ValueError as e: + import threading + if threading.current_thread() is threading.main_thread(): + raise e del self.old_sigwinch def push_char(self, char: int | bytes) -> None: @@ -407,6 +429,8 @@ def get_event(self, block: bool = True) -> Event | None: return self.event_queue.get() else: continue + elif err.errno == errno.EIO: + raise SystemExit(errno.EIO) else: raise else: @@ -597,14 +621,14 @@ def __setup_movement(self): if self._dch1: self.dch1 = self._dch1 elif self._dch: - self.dch1 = curses.tparm(self._dch, 1) + self.dch1 = terminfo.tparm(self._dch, 1) else: self.dch1 = None if self._ich1: self.ich1 = self._ich1 elif self._ich: - self.ich1 = curses.tparm(self._ich, 1) + self.ich1 = terminfo.tparm(self._ich, 1) else: self.ich1 = None @@ -701,7 +725,7 @@ def __write(self, text): self.__buffer.append((text, 0)) def __write_code(self, fmt, *args): - self.__buffer.append((curses.tparm(fmt, *args), 1)) + self.__buffer.append((terminfo.tparm(fmt, *args), 1)) def __maybe_write_code(self, fmt, *args): if fmt: @@ -752,7 +776,6 @@ def __move_tall(self, x, y): self.__write_code(self._cup, y - self.__offset, x) def __sigwinch(self, signum, frame): - self.height, self.width = self.getheightwidth() self.event_queue.insert(Event("resize", None)) def __hide_cursor(self): @@ -785,7 +808,7 @@ def __tputs(self, fmt, prog=delayprog): will never do anyone any good.""" # using .get() means that things will blow up # only if the bps is actually needed (which I'm - # betting is pretty unlkely) + # betting is pretty unlikely) bps = ratedict.get(self.__svtermstate.ospeed) while True: m = prog.search(fmt) @@ -803,3 +826,17 @@ def __tputs(self, fmt, prog=delayprog): os.write(self.output_fd, self._pad * nchars) else: time.sleep(float(delay) / 1000.0) + + def __input_fd_set( + self, + state: TermState, + ignore: AbstractSet[int] = _error_codes_to_ignore, + ) -> bool: + try: + tcsetattr(self.input_fd, termios.TCSADRAIN, state) + except termios.error as te: + if te.args[0] not in ignore: + raise + return False + else: + return True diff --git a/Lib/_pyrepl/unix_eventqueue.py b/Lib/_pyrepl/unix_eventqueue.py index 29b3e9dd5efd07f..2a9cca59e7477f7 100644 --- a/Lib/_pyrepl/unix_eventqueue.py +++ b/Lib/_pyrepl/unix_eventqueue.py @@ -18,7 +18,7 @@ # CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from . import curses +from .terminfo import TermInfo from .trace import trace from .base_eventqueue import BaseEventQueue from termios import tcgetattr, VERASE @@ -54,22 +54,23 @@ b'\033Oc': 'ctrl right', } -def get_terminal_keycodes() -> dict[bytes, str]: +def get_terminal_keycodes(ti: TermInfo) -> dict[bytes, str]: """ Generates a dictionary mapping terminal keycodes to human-readable names. """ keycodes = {} for key, terminal_code in TERMINAL_KEYNAMES.items(): - keycode = curses.tigetstr(terminal_code) + keycode = ti.get(terminal_code) trace('key {key} tiname {terminal_code} keycode {keycode!r}', **locals()) if keycode: keycodes[keycode] = key keycodes.update(CTRL_ARROW_KEYCODES) return keycodes + class EventQueue(BaseEventQueue): - def __init__(self, fd: int, encoding: str) -> None: - keycodes = get_terminal_keycodes() + def __init__(self, fd: int, encoding: str, ti: TermInfo) -> None: + keycodes = get_terminal_keycodes(ti) if os.isatty(fd): backspace = tcgetattr(fd)[6][VERASE] keycodes[backspace] = "backspace" diff --git a/Lib/_pyrepl/utils.py b/Lib/_pyrepl/utils.py index 38cf6b5a08e8926..1a697ad61c8e896 100644 --- a/Lib/_pyrepl/utils.py +++ b/Lib/_pyrepl/utils.py @@ -20,6 +20,7 @@ ZERO_WIDTH_BRACKET = re.compile(r"\x01.*?\x02") ZERO_WIDTH_TRANS = str.maketrans({"\x01": "", "\x02": ""}) IDENTIFIERS_AFTER = {"def", "class"} +KEYWORD_CONSTANTS = {"True", "False", "None"} BUILTINS = {str(name) for name in dir(builtins) if not name.startswith('_')} @@ -41,9 +42,15 @@ def from_re(cls, m: Match[str], group: int | str) -> Self: @classmethod def from_token(cls, token: TI, line_len: list[int]) -> Self: + end_offset = -1 + if (token.type in {T.FSTRING_MIDDLE, T.TSTRING_MIDDLE} + and token.string.endswith(("{", "}"))): + # gh-134158: a visible trailing brace comes from a double brace in input + end_offset += 1 + return cls( line_len[token.start[0] - 1] + token.start[1], - line_len[token.end[0] - 1] + token.end[1] - 1, + line_len[token.end[0] - 1] + token.end[1] + end_offset, ) @@ -56,6 +63,12 @@ class ColorSpan(NamedTuple): def str_width(c: str) -> int: if ord(c) < 128: return 1 + # gh-139246 for zero-width joiner and combining characters + if unicodedata.combining(c): + return 0 + category = unicodedata.category(c) + if category == "Cf" and c != "\u00ad": + return 0 w = unicodedata.east_asian_width(c) if w in ("N", "Na", "H", "A"): return 1 @@ -102,6 +115,8 @@ def gen_colors(buffer: str) -> Iterator[ColorSpan]: for color in gen_colors_from_token_stream(gen, line_lengths): yield color last_emitted = color + except SyntaxError: + return except tokenize.TokenError as te: yield from recover_unterminated_string( te, line_lengths, last_emitted, buffer @@ -189,8 +204,11 @@ def gen_colors_from_token_stream( span = Span.from_token(token, line_lengths) yield ColorSpan(span, "definition") elif keyword.iskeyword(token.string): + span_cls = "keyword" + if token.string in KEYWORD_CONSTANTS: + span_cls = "keyword_constant" span = Span.from_token(token, line_lengths) - yield ColorSpan(span, "keyword") + yield ColorSpan(span, span_cls) if token.string in IDENTIFIERS_AFTER: is_def_name = True elif ( @@ -200,7 +218,10 @@ def gen_colors_from_token_stream( ): span = Span.from_token(token, line_lengths) yield ColorSpan(span, "soft_keyword") - elif token.string in BUILTINS: + elif ( + token.string in BUILTINS + and not (prev_token and prev_token.exact_type == T.DOT) + ): span = Span.from_token(token, line_lengths) yield ColorSpan(span, "builtin") @@ -221,7 +242,7 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool: None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"), TI(string="match"), TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START) - | TI(T.OP, string="(" | "*" | "[" | "{" | "~" | "...") + | TI(T.OP, string="(" | "*" | "-" | "+" | "[" | "{" | "~" | "...") ): return True case ( @@ -233,14 +254,14 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool: return s in keyword_first_sets_match return True case ( - None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"), + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), TI(string="case"), TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START) | TI(T.OP, string="(" | "*" | "-" | "[" | "{") ): return True case ( - None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"), + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), TI(string="case"), TI(T.NAME, string=s) ): @@ -249,6 +270,12 @@ def is_soft_keyword_used(*tokens: TI | None) -> bool: return True case (TI(string="case"), TI(string="_"), TI(string=":")): return True + case ( + None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"), + TI(string="type"), + TI(T.NAME, string=s) + ): + return not keyword.iskeyword(s) case _: return False diff --git a/Lib/_pyrepl/windows_console.py b/Lib/_pyrepl/windows_console.py index 95749198b3b2f96..46c6030748b8a6b 100644 --- a/Lib/_pyrepl/windows_console.py +++ b/Lib/_pyrepl/windows_console.py @@ -183,8 +183,9 @@ def refresh(self, screen: list[str], c_xy: tuple[int, int]) -> None: while len(self.screen) < min(len(screen), self.height): self._hide_cursor() - self._move_relative(0, len(self.screen) - 1) - self.__write("\n") + if self.screen: + self._move_relative(0, len(self.screen) - 1) + self.__write("\n") self.posxy = 0, len(self.screen) self.screen.append("") @@ -282,18 +283,13 @@ def __write_changed_line( self._erase_to_end() self.__write(newline[x_pos:]) - if wlen(newline) == self.width: - # If we wrapped we want to start at the next line - self._move_relative(0, y + 1) - self.posxy = 0, y + 1 - else: - self.posxy = wlen(newline), y + self.posxy = min(wlen(newline), self.width - 1), y - if "\x1b" in newline or y != self.posxy[1] or '\x1a' in newline: - # ANSI escape characters are present, so we can't assume - # anything about the position of the cursor. Moving the cursor - # to the left margin should work to get to a known position. - self.move_cursor(0, y) + if "\x1b" in newline or y != self.posxy[1] or '\x1a' in newline: + # ANSI escape characters are present, so we can't assume + # anything about the position of the cursor. Moving the cursor + # to the left margin should work to get to a known position. + self.move_cursor(0, y) def _scroll( self, top: int, bottom: int, left: int | None = None, right: int | None = None @@ -419,10 +415,7 @@ def _getscrollbacksize(self) -> int: return info.srWindow.Bottom # type: ignore[no-any-return] - def _read_input(self, block: bool = True) -> INPUT_RECORD | None: - if not block and not self.wait(timeout=0): - return None - + def _read_input(self) -> INPUT_RECORD | None: rec = INPUT_RECORD() read = DWORD() if not ReadConsoleInput(InHandle, rec, 1, read): @@ -431,14 +424,10 @@ def _read_input(self, block: bool = True) -> INPUT_RECORD | None: return rec def _read_input_bulk( - self, block: bool, n: int + self, n: int ) -> tuple[ctypes.Array[INPUT_RECORD], int]: rec = (n * INPUT_RECORD)() read = DWORD() - - if not block and not self.wait(timeout=0): - return rec, 0 - if not ReadConsoleInput(InHandle, rec, n, read): raise WinError(GetLastError()) @@ -449,8 +438,11 @@ def get_event(self, block: bool = True) -> Event | None: and there is no event pending, otherwise waits for the completion of an event.""" + if not block and not self.wait(timeout=0): + return None + while self.event_queue.empty(): - rec = self._read_input(block) + rec = self._read_input() if rec is None: return None @@ -518,7 +510,7 @@ def clear(self) -> None: """Wipe the screen""" self.__write(CLEAR) self.posxy = 0, 0 - self.screen = [""] + self.screen = [] def finish(self) -> None: """Move the cursor to the end of the display and otherwise get @@ -551,18 +543,26 @@ def getpending(self) -> Event: if e2: e.data += e2.data - recs, rec_count = self._read_input_bulk(False, 1024) + recs, rec_count = self._read_input_bulk(1024) for i in range(rec_count): rec = recs[i] + # In case of a legacy console, we do not only receive a keydown + # event, but also a keyup event - and for uppercase letters + # an additional SHIFT_PRESSED event. if rec and rec.EventType == KEY_EVENT: key_event = rec.Event.KeyEvent + if not key_event.bKeyDown: + continue ch = key_event.uChar.UnicodeChar + if ch == "\x00": + # ignore SHIFT_PRESSED and special keys + continue if ch == "\r": ch += "\n" e.data += ch return e - def wait(self, timeout: float | None) -> bool: + def wait_for_event(self, timeout: float | None) -> bool: """Wait for an event.""" if timeout is None: timeout = INFINITE @@ -575,6 +575,15 @@ def wait(self, timeout: float | None) -> bool: return False return True + def wait(self, timeout: float | None) -> bool: + """ + Wait for events on the console. + """ + return ( + not self.event_queue.empty() + or self.wait_for_event(timeout) + ) + def repaint(self) -> None: raise NotImplementedError("No repaint support") diff --git a/Lib/_sitebuiltins.py b/Lib/_sitebuiltins.py index c66269a571967f8..81b36efc6c285f0 100644 --- a/Lib/_sitebuiltins.py +++ b/Lib/_sitebuiltins.py @@ -36,7 +36,7 @@ def __init__(self, name, data, files=(), dirs=()): import os self.__name = name self.__data = data - self.__lines = None + self.__lines = [] self.__filenames = [os.path.join(dir, filename) for dir in dirs for filename in files] @@ -65,24 +65,12 @@ def __repr__(self): return "Type %s() to see the full %s text" % ((self.__name,)*2) def __call__(self): + from _pyrepl.pager import get_pager self.__setup() - prompt = 'Hit Return for more, or q (and Return) to quit: ' - lineno = 0 - while 1: - try: - for i in range(lineno, lineno + self.MAXLINES): - print(self.__lines[i]) - except IndexError: - break - else: - lineno += self.MAXLINES - key = None - while key is None: - key = input(prompt) - if key not in ('', 'q'): - key = None - if key == 'q': - break + + pager = get_pager() + text = "\n".join(self.__lines) + pager(text, title=self.__name) class _Helper(object): diff --git a/Lib/_strptime.py b/Lib/_strptime.py index aa63933a49d16df..fc7e369c3d1e26f 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -14,6 +14,7 @@ import time import locale import calendar +import re from re import compile as re_compile from re import sub as re_sub from re import IGNORECASE @@ -41,6 +42,29 @@ def _findall(haystack, needle): yield i i += len(needle) +def _fixmonths(months): + yield from months + # The lower case of 'İ' ('\u0130') is 'i\u0307'. + # The re module only supports 1-to-1 character matching in + # case-insensitive mode. + for s in months: + if 'i\u0307' in s: + yield s.replace('i\u0307', '\u0130') + +lzh_TW_alt_digits = ( + # 〇:一:二:三:四:五:六:七:八:九 + '\u3007', '\u4e00', '\u4e8c', '\u4e09', '\u56db', + '\u4e94', '\u516d', '\u4e03', '\u516b', '\u4e5d', + # 十:十一:十二:十三:十四:十五:十六:十七:十八:十九 + '\u5341', '\u5341\u4e00', '\u5341\u4e8c', '\u5341\u4e09', '\u5341\u56db', + '\u5341\u4e94', '\u5341\u516d', '\u5341\u4e03', '\u5341\u516b', '\u5341\u4e5d', + # 廿:廿一:廿二:廿三:廿四:廿五:廿六:廿七:廿八:廿九 + '\u5eff', '\u5eff\u4e00', '\u5eff\u4e8c', '\u5eff\u4e09', '\u5eff\u56db', + '\u5eff\u4e94', '\u5eff\u516d', '\u5eff\u4e03', '\u5eff\u516b', '\u5eff\u4e5d', + # 卅:卅一 + '\u5345', '\u5345\u4e00') + + class LocaleTime(object): """Stores and handles locale-specific information related to time. @@ -84,6 +108,7 @@ def __init__(self): self.__calc_weekday() self.__calc_month() self.__calc_am_pm() + self.__calc_alt_digits() self.__calc_timezone() self.__calc_date_time() if _getlang() != self.lang: @@ -119,9 +144,43 @@ def __calc_am_pm(self): am_pm.append(time.strftime("%p", time_tuple).lower().strip()) self.am_pm = am_pm + def __calc_alt_digits(self): + # Set self.LC_alt_digits by using time.strftime(). + + # The magic data should contain all decimal digits. + time_tuple = time.struct_time((1998, 1, 27, 10, 43, 56, 1, 27, 0)) + s = time.strftime("%x%X", time_tuple) + if s.isascii(): + # Fast path -- all digits are ASCII. + self.LC_alt_digits = () + return + + digits = ''.join(sorted(set(re.findall(r'\d', s)))) + if len(digits) == 10 and ord(digits[-1]) == ord(digits[0]) + 9: + # All 10 decimal digits from the same set. + if digits.isascii(): + # All digits are ASCII. + self.LC_alt_digits = () + return + + self.LC_alt_digits = [a + b for a in digits for b in digits] + # Test whether the numbers contain leading zero. + time_tuple2 = time.struct_time((2000, 1, 1, 1, 1, 1, 5, 1, 0)) + if self.LC_alt_digits[1] not in time.strftime("%x %X", time_tuple2): + self.LC_alt_digits[:10] = digits + return + + # Either non-Gregorian calendar or non-decimal numbers. + if {'\u4e00', '\u4e03', '\u4e5d', '\u5341', '\u5eff'}.issubset(s): + # lzh_TW + self.LC_alt_digits = lzh_TW_alt_digits + return + + self.LC_alt_digits = None + def __calc_date_time(self): - # Set self.date_time, self.date, & self.time by using - # time.strftime(). + # Set self.LC_date_time, self.LC_date, self.LC_time and + # self.LC_time_ampm by using time.strftime(). # Use (1999,3,17,22,44,55,2,76,0) for magic date because the amount of # overloaded numbers is minimized. The order in which searches for @@ -129,26 +188,32 @@ def __calc_date_time(self): # possible ambiguity for what something represents. time_tuple = time.struct_time((1999,3,17,22,44,55,2,76,0)) time_tuple2 = time.struct_time((1999,1,3,1,1,1,6,3,0)) - replacement_pairs = [ + replacement_pairs = [] + + # Non-ASCII digits + if self.LC_alt_digits or self.LC_alt_digits is None: + for n, d in [(19, '%OC'), (99, '%Oy'), (22, '%OH'), + (44, '%OM'), (55, '%OS'), (17, '%Od'), + (3, '%Om'), (2, '%Ow'), (10, '%OI')]: + if self.LC_alt_digits is None: + s = chr(0x660 + n // 10) + chr(0x660 + n % 10) + replacement_pairs.append((s, d)) + if n < 10: + replacement_pairs.append((s[1], d)) + elif len(self.LC_alt_digits) > n: + replacement_pairs.append((self.LC_alt_digits[n], d)) + else: + replacement_pairs.append((time.strftime(d, time_tuple), d)) + replacement_pairs += [ ('1999', '%Y'), ('99', '%y'), ('22', '%H'), ('44', '%M'), ('55', '%S'), ('76', '%j'), ('17', '%d'), ('03', '%m'), ('3', '%m'), # '3' needed for when no leading zero. ('2', '%w'), ('10', '%I'), - # Non-ASCII digits - ('\u0661\u0669\u0669\u0669', '%Y'), - ('\u0669\u0669', '%Oy'), - ('\u0662\u0662', '%OH'), - ('\u0664\u0664', '%OM'), - ('\u0665\u0665', '%OS'), - ('\u0661\u0667', '%Od'), - ('\u0660\u0663', '%Om'), - ('\u0663', '%Om'), - ('\u0662', '%Ow'), - ('\u0661\u0660', '%OI'), ] + date_time = [] - for directive in ('%c', '%x', '%X'): + for directive in ('%c', '%x', '%X', '%r'): current_format = time.strftime(directive, time_tuple).lower() current_format = current_format.replace('%', '%%') # The month and the day of the week formats are treated specially @@ -172,9 +237,10 @@ def __calc_date_time(self): if tz: current_format = current_format.replace(tz, "%Z") # Transform all non-ASCII digits to digits in range U+0660 to U+0669. - current_format = re_sub(r'\d(?3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])", 'f': r"(?P[0-9]{1,6})", - 'H': r"(?P2[0-3]|[0-1]\d|\d)", + 'H': r"(?P2[0-3]|[0-1]\d|\d| \d)", + 'k': r"(?P2[0-3]|[0-1]\d|\d| \d)", 'I': r"(?P1[0-2]|0[1-9]|[1-9]| [1-9])", + 'l': r"(?P1[0-2]|0[1-9]|[1-9]| [1-9])", 'G': r"(?P\d\d\d\d)", 'j': r"(?P36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])", 'm': r"(?P1[0-2]|0[1-9]|[1-9])", @@ -305,23 +374,56 @@ def __init__(self, locale_time=None): 'z': r"(?P[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|(?-i:Z))", 'A': self.__seqToRE(self.locale_time.f_weekday, 'A'), 'a': self.__seqToRE(self.locale_time.a_weekday, 'a'), - 'B': self.__seqToRE(self.locale_time.f_month[1:], 'B'), - 'b': self.__seqToRE(self.locale_time.a_month[1:], 'b'), + 'B': self.__seqToRE(_fixmonths(self.locale_time.f_month[1:]), 'B'), + 'b': self.__seqToRE(_fixmonths(self.locale_time.a_month[1:]), 'b'), 'p': self.__seqToRE(self.locale_time.am_pm, 'p'), 'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone for tz in tz_names), 'Z'), '%': '%'} - for d in 'dmyHIMS': - mapping['O' + d] = r'(?P<%s>\d\d|\d| \d)' % d - mapping['Ow'] = r'(?P\d)' + if self.locale_time.LC_alt_digits is None: + for d in 'dmyCHIMS': + mapping['O' + d] = r'(?P<%s>\d\d|\d| \d)' % d + mapping['Ow'] = r'(?P\d)' + else: + mapping.update({ + 'Od': self.__seqToRE(self.locale_time.LC_alt_digits[1:32], 'd', + '3[0-1]|[1-2][0-9]|0[1-9]|[1-9]'), + 'Om': self.__seqToRE(self.locale_time.LC_alt_digits[1:13], 'm', + '1[0-2]|0[1-9]|[1-9]'), + 'Ow': self.__seqToRE(self.locale_time.LC_alt_digits[:7], 'w', + '[0-6]'), + 'Oy': self.__seqToRE(self.locale_time.LC_alt_digits, 'y', + '[0-9][0-9]'), + 'OC': self.__seqToRE(self.locale_time.LC_alt_digits, 'C', + '[0-9][0-9]'), + 'OH': self.__seqToRE(self.locale_time.LC_alt_digits[:24], 'H', + '2[0-3]|[0-1][0-9]|[0-9]'), + 'OI': self.__seqToRE(self.locale_time.LC_alt_digits[1:13], 'I', + '1[0-2]|0[1-9]|[1-9]'), + 'OM': self.__seqToRE(self.locale_time.LC_alt_digits[:60], 'M', + '[0-5][0-9]|[0-9]'), + 'OS': self.__seqToRE(self.locale_time.LC_alt_digits[:62], 'S', + '6[0-1]|[0-5][0-9]|[0-9]'), + }) + mapping.update({ + 'e': mapping['d'], + 'Oe': mapping['Od'], + 'P': mapping['p'], + 'Op': mapping['p'], + 'W': mapping['U'].replace('U', 'W'), + }) mapping['W'] = mapping['U'].replace('U', 'W') + base.__init__(mapping) + base.__setitem__('T', self.pattern('%H:%M:%S')) + base.__setitem__('R', self.pattern('%H:%M')) + base.__setitem__('r', self.pattern(self.locale_time.LC_time_ampm)) base.__setitem__('X', self.pattern(self.locale_time.LC_time)) base.__setitem__('x', self.pattern(self.locale_time.LC_date)) base.__setitem__('c', self.pattern(self.locale_time.LC_date_time)) - def __seqToRE(self, to_convert, directive): + def __seqToRE(self, to_convert, directive, altregex=None): """Convert a list to a regex string for matching a directive. Want possible matching values to be from longest to shortest. This @@ -337,8 +439,9 @@ def __seqToRE(self, to_convert, directive): else: return '' regex = '|'.join(re_escape(stuff) for stuff in to_convert) - regex = '(?P<%s>%s' % (directive, regex) - return '%s)' % regex + if altregex is not None: + regex += '|' + altregex + return '(?P<%s>%s)' % (directive, regex) def pattern(self, format): """Return regex pattern for the format string. @@ -365,7 +468,7 @@ def repl(m): nonlocal day_of_month_in_format day_of_month_in_format = True return self[format_char] - format = re_sub(r'%([OE]?\\?.?)', repl, format) + format = re_sub(r'%[-_0^#]*[0-9]*([OE]?\\?.?)', repl, format) if day_of_month_in_format and not year_in_format: import warnings warnings.warn("""\ @@ -467,6 +570,15 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): # values weekday = julian = None found_dict = found.groupdict() + if locale_time.LC_alt_digits: + def parse_int(s): + try: + return locale_time.LC_alt_digits.index(s) + except ValueError: + return int(s) + else: + parse_int = int + for group_key in found_dict.keys(): # Directives not explicitly handled below: # c, x, X @@ -474,30 +586,34 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): # U, W # worthless without day of the week if group_key == 'y': - year = int(found_dict['y']) - # Open Group specification for strptime() states that a %y - #value in the range of [00, 68] is in the century 2000, while - #[69,99] is in the century 1900 - if year <= 68: - year += 2000 + year = parse_int(found_dict['y']) + if 'C' in found_dict: + century = parse_int(found_dict['C']) + year += century * 100 else: - year += 1900 + # Open Group specification for strptime() states that a %y + #value in the range of [00, 68] is in the century 2000, while + #[69,99] is in the century 1900 + if year <= 68: + year += 2000 + else: + year += 1900 elif group_key == 'Y': year = int(found_dict['Y']) elif group_key == 'G': iso_year = int(found_dict['G']) elif group_key == 'm': - month = int(found_dict['m']) + month = parse_int(found_dict['m']) elif group_key == 'B': month = locale_time.f_month.index(found_dict['B'].lower()) elif group_key == 'b': month = locale_time.a_month.index(found_dict['b'].lower()) elif group_key == 'd': - day = int(found_dict['d']) + day = parse_int(found_dict['d']) elif group_key == 'H': - hour = int(found_dict['H']) + hour = parse_int(found_dict['H']) elif group_key == 'I': - hour = int(found_dict['I']) + hour = parse_int(found_dict['I']) ampm = found_dict.get('p', '').lower() # If there was no AM/PM indicator, we'll treat this like AM if ampm in ('', locale_time.am_pm[0]): @@ -513,9 +629,9 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): if hour != 12: hour += 12 elif group_key == 'M': - minute = int(found_dict['M']) + minute = parse_int(found_dict['M']) elif group_key == 'S': - second = int(found_dict['S']) + second = parse_int(found_dict['S']) elif group_key == 'f': s = found_dict['f'] # Pad to always return microseconds. diff --git a/Lib/annotationlib.py b/Lib/annotationlib.py index c0b1d4395d14ed1..5c9a0812646f814 100644 --- a/Lib/annotationlib.py +++ b/Lib/annotationlib.py @@ -27,6 +27,9 @@ class Format(enum.IntEnum): _sentinel = object() +# Following `NAME_ERROR_MSG` in `ceval_macros.h`: +_NAME_ERROR_MSG = "name '{name:.200}' is not defined" + # Slots shared by ForwardRef and _Stringifier. The __forward__ names must be # preserved for compatibility with the old typing.ForwardRef class. The remaining @@ -44,6 +47,7 @@ class Format(enum.IntEnum): "__cell__", "__owner__", "__stringifier_dict__", + "__resolved_str_cache__", ) @@ -82,12 +86,16 @@ def __init__( # These are always set to None here but may be non-None if a ForwardRef # is created through __class__ assignment on a _Stringifier object. self.__globals__ = None + # This may be either a cell object (for a ForwardRef referring to a single name) + # or a dict mapping cell names to cell objects (for a ForwardRef containing references + # to multiple names). self.__cell__ = None self.__extra_names__ = None # These are initially None but serve as a cache and may be set to a non-None # value later. self.__code__ = None self.__ast_node__ = None + self.__resolved_str_cache__ = None def __init_subclass__(cls, /, *args, **kwds): raise TypeError("Cannot subclass ForwardRef") @@ -107,14 +115,14 @@ def evaluate( """ match format: case Format.STRING: - return self.__forward_arg__ + return self.__resolved_str__ case Format.VALUE: is_forwardref_format = False case Format.FORWARDREF: is_forwardref_format = True case _: raise NotImplementedError(format) - if self.__cell__ is not None: + if isinstance(self.__cell__, types.CellType): try: return self.__cell__.cell_contents except ValueError: @@ -144,34 +152,42 @@ def evaluate( if globals is None: globals = {} + if type_params is None and owner is not None: + type_params = getattr(owner, "__type_params__", None) + if locals is None: locals = {} if isinstance(owner, type): locals.update(vars(owner)) + elif ( + type_params is not None + or isinstance(self.__cell__, dict) + or self.__extra_names__ + ): + # Create a new locals dict if necessary, + # to avoid mutating the argument. + locals = dict(locals) - if type_params is None and owner is not None: - # "Inject" type parameters into the local namespace - # (unless they are shadowed by assignments *in* the local namespace), - # as a way of emulating annotation scopes when calling `eval()` - type_params = getattr(owner, "__type_params__", None) - - # type parameters require some special handling, - # as they exist in their own scope - # but `eval()` does not have a dedicated parameter for that scope. - # For classes, names in type parameter scopes should override - # names in the global scope (which here are called `localns`!), - # but should in turn be overridden by names in the class scope - # (which here are called `globalns`!) + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` if type_params is not None: - globals = dict(globals) - locals = dict(locals) for param in type_params: - param_name = param.__name__ - if not self.__forward_is_class__ or param_name not in globals: - globals[param_name] = param - locals.pop(param_name, None) + locals.setdefault(param.__name__, param) + + # Similar logic can be used for nonlocals, which should not + # override locals. + if isinstance(self.__cell__, dict): + for cell_name, cell in self.__cell__.items(): + try: + cell_value = cell.cell_contents + except ValueError: + pass + else: + locals.setdefault(cell_name, cell_value) + if self.__extra_names__: - locals = {**locals, **self.__extra_names__} + locals.update(self.__extra_names__) arg = self.__forward_arg__ if arg.isidentifier() and not keyword.iskeyword(arg): @@ -184,7 +200,7 @@ def evaluate( elif is_forwardref_format: return self else: - raise NameError(arg) + raise NameError(_NAME_ERROR_MSG.format(name=arg), name=arg) else: code = self.__forward_code__ try: @@ -192,8 +208,11 @@ def evaluate( except Exception: if not is_forwardref_format: raise + + # All variables, in scoping order, should be checked before + # triggering __missing__ to create a _Stringifier. new_locals = _StringifierDict( - {**builtins.__dict__, **locals}, + {**builtins.__dict__, **globals, **locals}, globals=globals, owner=owner, is_class=self.__forward_is_class__, @@ -204,7 +223,7 @@ def evaluate( except Exception: return self else: - new_locals.transmogrify() + new_locals.transmogrify(self.__cell__) return result def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard): @@ -241,20 +260,31 @@ def __forward_arg__(self): "Attempted to access '__forward_arg__' on an uninitialized ForwardRef" ) + @property + def __resolved_str__(self): + # __forward_arg__ with any names from __extra_names__ replaced + # with the type_repr of the value they represent + if self.__resolved_str_cache__ is None: + resolved_str = self.__forward_arg__ + names = self.__extra_names__ + + if names: + visitor = _ExtraNameFixer(names) + ast_expr = ast.parse(resolved_str, mode="eval").body + node = visitor.visit(ast_expr) + resolved_str = ast.unparse(node) + + self.__resolved_str_cache__ = resolved_str + + return self.__resolved_str_cache__ + @property def __forward_code__(self): if self.__code__ is not None: return self.__code__ arg = self.__forward_arg__ - # If we do `def f(*args: *Ts)`, then we'll have `arg = '*Ts'`. - # Unfortunately, this isn't a valid expression on its own, so we - # do the unpacking manually. - if arg.startswith("*"): - arg_to_compile = f"({arg},)[0]" # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] - else: - arg_to_compile = arg try: - self.__code__ = compile(arg_to_compile, "", "eval") + self.__code__ = compile(_rewrite_star_unpack(arg), "", "eval") except SyntaxError: raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}") return self.__code__ @@ -269,7 +299,13 @@ def __eq__(self, other): # because dictionaries are not hashable. and self.__globals__ is other.__globals__ and self.__forward_is_class__ == other.__forward_is_class__ - and self.__cell__ == other.__cell__ + # Two separate cells are always considered unequal in forward refs. + and ( + {name: id(cell) for name, cell in self.__cell__.items()} + == {name: id(cell) for name, cell in other.__cell__.items()} + if isinstance(self.__cell__, dict) and isinstance(other.__cell__, dict) + else self.__cell__ is other.__cell__ + ) and self.__owner__ == other.__owner__ and ( (tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None) == @@ -283,7 +319,10 @@ def __hash__(self): self.__forward_module__, id(self.__globals__), # dictionaries are not hashable, so hash by identity self.__forward_is_class__, - self.__cell__, + ( # cells are not hashable as well + tuple(sorted([(name, id(cell)) for name, cell in self.__cell__.items()])) + if isinstance(self.__cell__, dict) else id(self.__cell__), + ), self.__owner__, tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None, )) @@ -302,7 +341,10 @@ def __repr__(self): extra.append(", is_class=True") if self.__owner__ is not None: extra.append(f", owner={self.__owner__!r}") - return f"ForwardRef({self.__forward_arg__!r}{''.join(extra)})" + return f"ForwardRef({self.__resolved_str__!r}{''.join(extra)})" + + +_Template = type(t"") class _Stringifier: @@ -335,12 +377,15 @@ def __init__( self.__cell__ = cell self.__owner__ = owner self.__stringifier_dict__ = stringifier_dict + self.__resolved_str_cache__ = None # Needed for ForwardRef def __convert_to_ast(self, other): if isinstance(other, _Stringifier): if isinstance(other.__ast_node__, str): return ast.Name(id=other.__ast_node__), other.__extra_names__ return other.__ast_node__, other.__extra_names__ + elif type(other) is _Template: + return _template_to_ast(other), None elif ( # In STRING format we don't bother with the create_unique_name() dance; # it's better to emit the repr() of the object instead of an opaque name. @@ -560,6 +605,70 @@ def unary_op(self): del _make_unary_op +def _template_to_ast_constructor(template): + """Convert a `template` instance to a non-literal AST.""" + args = [] + for part in template: + match part: + case str(): + args.append(ast.Constant(value=part)) + case _: + interp = ast.Call( + func=ast.Name(id="Interpolation"), + args=[ + ast.Constant(value=part.value), + ast.Constant(value=part.expression), + ast.Constant(value=part.conversion), + ast.Constant(value=part.format_spec), + ] + ) + args.append(interp) + return ast.Call(func=ast.Name(id="Template"), args=args, keywords=[]) + + +def _template_to_ast_literal(template, parsed): + """Convert a `template` instance to a t-string literal AST.""" + values = [] + interp_count = 0 + for part in template: + match part: + case str(): + values.append(ast.Constant(value=part)) + case _: + interp = ast.Interpolation( + str=part.expression, + value=parsed[interp_count], + conversion=ord(part.conversion) if part.conversion else -1, + format_spec=ast.Constant(value=part.format_spec) + if part.format_spec + else None, + ) + values.append(interp) + interp_count += 1 + return ast.TemplateStr(values=values) + + +def _template_to_ast(template): + """Make a best-effort conversion of a `template` instance to an AST.""" + # gh-138558: Not all Template instances can be represented as t-string + # literals. Return the most accurate AST we can. See issue for details. + + # If any expr is empty or whitespace only, we cannot convert to a literal. + if any(part.expression.strip() == "" for part in template.interpolations): + return _template_to_ast_constructor(template) + + try: + # Wrap in parens to allow whitespace inside interpolation curly braces + parsed = tuple( + ast.parse(f"({part.expression})", mode="eval").body + for part in template.interpolations + ) + except SyntaxError: + return _template_to_ast_constructor(template) + + return _template_to_ast_literal(template, parsed) + + class _StringifierDict(dict): def __init__(self, namespace, *, globals=None, owner=None, is_class=False, format): super().__init__(namespace) @@ -582,13 +691,15 @@ def __missing__(self, key): self.stringifiers.append(fwdref) return fwdref - def transmogrify(self): + def transmogrify(self, cell_dict): for obj in self.stringifiers: obj.__class__ = ForwardRef obj.__stringifier_dict__ = None # not needed for ForwardRef if isinstance(obj.__ast_node__, str): obj.__arg__ = obj.__ast_node__ obj.__ast_node__ = None + if cell_dict is not None and obj.__cell__ is None: + obj.__cell__ = cell_dict def create_unique_name(self): name = f"__annotationlib_name_{self.next_id}__" @@ -638,9 +749,21 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): # possibly constants if the annotate function uses them directly). We then # convert each of those into a string to get an approximation of the # original source. + + # Attempt to call with VALUE_WITH_FAKE_GLOBALS to check if it is implemented + # See: https://github.com/python/cpython/issues/138764 + # Only fail on NotImplementedError + try: + annotate(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # Both STRING and VALUE_WITH_FAKE_GLOBALS are not implemented: fallback to VALUE + return annotations_to_string(annotate(Format.VALUE)) + except Exception: + pass + globals = _StringifierDict({}, format=format) is_class = isinstance(owner, type) - closure = _build_closure( + closure, _ = _build_closure( annotate, owner, is_class, globals, allow_evaluation=False ) func = types.FunctionType( @@ -684,7 +807,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): is_class=is_class, format=format, ) - closure = _build_closure( + closure, cell_dict = _build_closure( annotate, owner, is_class, globals, allow_evaluation=True ) func = types.FunctionType( @@ -696,10 +819,13 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): ) try: result = func(Format.VALUE_WITH_FAKE_GLOBALS) + except NotImplementedError: + # FORWARDREF and VALUE_WITH_FAKE_GLOBALS not supported, fall back to VALUE + return annotate(Format.VALUE) except Exception: pass else: - globals.transmogrify() + globals.transmogrify(cell_dict) return result # Try again, but do not provide any globals. This allows us to return @@ -711,7 +837,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): is_class=is_class, format=format, ) - closure = _build_closure( + closure, cell_dict = _build_closure( annotate, owner, is_class, globals, allow_evaluation=False ) func = types.FunctionType( @@ -722,7 +848,7 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): kwdefaults=annotate.__kwdefaults__, ) result = func(Format.VALUE_WITH_FAKE_GLOBALS) - globals.transmogrify() + globals.transmogrify(cell_dict) if _is_evaluate: if isinstance(result, ForwardRef): return result.evaluate(format=Format.FORWARDREF) @@ -747,14 +873,11 @@ def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False): def _build_closure(annotate, owner, is_class, stringifier_dict, *, allow_evaluation): if not annotate.__closure__: - return None - freevars = annotate.__code__.co_freevars + return None, None new_closure = [] - for i, cell in enumerate(annotate.__closure__): - if i < len(freevars): - name = freevars[i] - else: - name = "__cell__" + cell_dict = {} + for name, cell in zip(annotate.__code__.co_freevars, annotate.__closure__, strict=True): + cell_dict[name] = cell new_cell = None if allow_evaluation: try: @@ -775,7 +898,7 @@ def _build_closure(annotate, owner, is_class, stringifier_dict, *, allow_evaluat stringifier_dict.stringifiers.append(fwdref) new_cell = types.CellType(fwdref) new_closure.append(new_cell) - return tuple(new_closure) + return tuple(new_closure), cell_dict def _stringify_single(anno): @@ -784,6 +907,8 @@ def _stringify_single(anno): # We have to handle str specially to support PEP 563 stringified annotations. elif isinstance(anno, str): return anno + elif isinstance(anno, _Template): + return ast.unparse(_template_to_ast(anno)) else: return repr(anno) @@ -815,7 +940,7 @@ def get_annotations( does not exist, the __annotate__ function is called. The FORWARDREF format uses __annotations__ if it exists and can be evaluated, and otherwise falls back to calling the __annotate__ function. - The SOURCE format tries __annotate__ first, and falls back to + The STRING format tries __annotate__ first, and falls back to using __annotations__, stringified using annotations_to_string(). This function handles several details for you: @@ -906,48 +1031,62 @@ def get_annotations( if not eval_str: return dict(ann) - if isinstance(obj, type): - # class - obj_globals = None - module_name = getattr(obj, "__module__", None) - if module_name: - module = sys.modules.get(module_name, None) - if module: - obj_globals = getattr(module, "__dict__", None) - obj_locals = dict(vars(obj)) - unwrap = obj - elif isinstance(obj, types.ModuleType): - # module - obj_globals = getattr(obj, "__dict__") - obj_locals = None - unwrap = None - elif callable(obj): - # this includes types.Function, types.BuiltinFunctionType, - # types.BuiltinMethodType, functools.partial, functools.singledispatch, - # "class funclike" from Lib/test/test_inspect... on and on it goes. - obj_globals = getattr(obj, "__globals__", None) - obj_locals = None - unwrap = obj - else: - obj_globals = obj_locals = unwrap = None - - if unwrap is not None: - while True: - if hasattr(unwrap, "__wrapped__"): - unwrap = unwrap.__wrapped__ - continue - if functools := sys.modules.get("functools"): - if isinstance(unwrap, functools.partial): - unwrap = unwrap.func + if globals is None or locals is None: + if isinstance(obj, type): + # class + obj_globals = None + module_name = getattr(obj, "__module__", None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + obj_globals = getattr(module, "__dict__", None) + obj_locals = dict(vars(obj)) + unwrap = obj + elif isinstance(obj, types.ModuleType): + # module + obj_globals = getattr(obj, "__dict__") + obj_locals = None + unwrap = None + elif callable(obj): + # this includes types.Function, types.BuiltinFunctionType, + # types.BuiltinMethodType, functools.partial, functools.singledispatch, + # "class funclike" from Lib/test/test_inspect... on and on it goes. + obj_globals = getattr(obj, "__globals__", None) + obj_locals = None + unwrap = obj + else: + obj_globals = obj_locals = unwrap = None + + if unwrap is not None: + # Use an id-based visited set to detect cycles in the __wrapped__ + # and functools.partial.func chain (e.g. f.__wrapped__ = f). + # On cycle detection we stop and use whatever __globals__ we have + # found so far, mirroring the approach of inspect.unwrap(). + _seen_ids = {id(unwrap)} + while True: + if hasattr(unwrap, "__wrapped__"): + candidate = unwrap.__wrapped__ + if id(candidate) in _seen_ids: + break + _seen_ids.add(id(candidate)) + unwrap = candidate continue - break - if hasattr(unwrap, "__globals__"): - obj_globals = unwrap.__globals__ + if functools := sys.modules.get("functools"): + if isinstance(unwrap, functools.partial): + candidate = unwrap.func + if id(candidate) in _seen_ids: + break + _seen_ids.add(id(candidate)) + unwrap = candidate + continue + break + if hasattr(unwrap, "__globals__"): + obj_globals = unwrap.__globals__ - if globals is None: - globals = obj_globals - if locals is None: - locals = obj_locals + if globals is None: + globals = obj_globals + if locals is None: + locals = obj_locals # "Inject" type parameters into the local namespace # (unless they are shadowed by assignments *in* the local namespace), @@ -958,7 +1097,8 @@ def get_annotations( locals = {param.__name__: param for param in type_params} | locals return_value = { - key: value if not isinstance(value, str) else eval(value, globals, locals) + key: value if not isinstance(value, str) + else eval(_rewrite_star_unpack(value), globals, locals) for key, value in ann.items() } return return_value @@ -976,6 +1116,9 @@ def type_repr(value): if value.__module__ == "builtins": return value.__qualname__ return f"{value.__module__}.{value.__qualname__}" + elif isinstance(value, _Template): + tree = _template_to_ast(value) + return ast.unparse(tree) if value is ...: return "..." return repr(value) @@ -992,6 +1135,16 @@ def annotations_to_string(annotations): } +def _rewrite_star_unpack(arg): + """If the given argument annotation expression is a star unpack e.g. `'*Ts'` + rewrite it to a valid expression. + """ + if arg.lstrip().startswith("*"): + return f"({arg},)[0]" # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] + else: + return arg + + def _get_and_call_annotate(obj, format): """Get the __annotate__ function and call it. @@ -1006,15 +1159,39 @@ def _get_and_call_annotate(obj, format): return None +_BASE_GET_ANNOTATIONS = type.__dict__["__annotations__"].__get__ + + def _get_dunder_annotations(obj): """Return the annotations for an object, checking that it is a dictionary. Does not return a fresh dictionary. """ - ann = getattr(obj, "__annotations__", None) - if ann is None: - return None + # This special case is needed to support types defined under + # from __future__ import annotations, where accessing the __annotations__ + # attribute directly might return annotations for the wrong class. + if isinstance(obj, type): + try: + ann = _BASE_GET_ANNOTATIONS(obj) + except AttributeError: + # For static types, the descriptor raises AttributeError. + return None + else: + ann = getattr(obj, "__annotations__", None) + if ann is None: + return None if not isinstance(ann, dict): raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") return ann + + +class _ExtraNameFixer(ast.NodeTransformer): + """Fixer for __extra_names__ items in ForwardRef __repr__ and string evaluation""" + def __init__(self, extra_names): + self.extra_names = extra_names + + def visit_Name(self, node: ast.Name): + if (new_name := self.extra_names.get(node.id, _sentinel)) is not _sentinel: + node = ast.Name(id=type_repr(new_name)) + return node diff --git a/Lib/argparse.py b/Lib/argparse.py index f13ac82dbc50b34..24f4740562d0159 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -149,6 +149,10 @@ def _copy_items(items): return copy.copy(items) +def _identity(value): + return value + + # =============== # Formatting Help # =============== @@ -167,8 +171,7 @@ def __init__( indent_increment=2, max_help_position=24, width=None, - prefix_chars='-', - color=False, + color=True, ): # default setting for width if width is None: @@ -176,16 +179,7 @@ def __init__( width = shutil.get_terminal_size().columns width -= 2 - from _colorize import can_colorize, decolor, get_theme - - if color and can_colorize(): - self._theme = get_theme(force_color=True).argparse - self._decolor = decolor - else: - self._theme = get_theme(force_no_color=True).argparse - self._decolor = lambda text: text - - self._prefix_chars = prefix_chars + self._set_color(color) self._prog = prog self._indent_increment = indent_increment self._max_help_position = min(max_help_position, @@ -202,9 +196,20 @@ def __init__( self._whitespace_matcher = _re.compile(r'\s+', _re.ASCII) self._long_break_matcher = _re.compile(r'\n\n\n+') + def _set_color(self, color): + from _colorize import can_colorize, decolor, get_theme + + if color and can_colorize(): + self._theme = get_theme(force_color=True).argparse + self._decolor = decolor + else: + self._theme = get_theme(force_no_color=True).argparse + self._decolor = _identity + # =============================== # Section and indentation methods # =============================== + def _indent(self): self._current_indent += self._indent_increment self._level += 1 @@ -256,6 +261,7 @@ def _add_item(self, func, args): # ======================== # Message building methods # ======================== + def start_section(self, heading): self._indent() section = self._Section(self, self._current_section, heading) @@ -279,7 +285,7 @@ def add_argument(self, action): if action.help is not SUPPRESS: # find all invocations - get_invocation = self._format_action_invocation + get_invocation = lambda x: self._decolor(self._format_action_invocation(x)) invocation_lengths = [len(get_invocation(action)) + self._current_indent] for subaction in self._iter_indented_subactions(action): invocation_lengths.append(len(get_invocation(subaction)) + self._current_indent) @@ -299,6 +305,7 @@ def add_arguments(self, actions): # ======================= # Help-formatting methods # ======================= + def format_help(self): help = self._root_section.format_help() if help: @@ -334,27 +341,17 @@ def _format_usage(self, usage, actions, groups, prefix): elif usage is None: prog = '%(prog)s' % dict(prog=self._prog) - # split optionals from positionals - optionals = [] - positionals = [] - for action in actions: - if action.option_strings: - optionals.append(action) - else: - positionals.append(action) - + parts, pos_start = self._get_actions_usage_parts(actions, groups) # build full usage string - format = self._format_actions_usage - action_usage = format(optionals + positionals, groups) - usage = ' '.join([s for s in [prog, action_usage] if s]) + usage = ' '.join(filter(None, [prog, *parts])) # wrap the usage parts if it's too long text_width = self._width - self._current_indent if len(prefix) + len(self._decolor(usage)) > text_width: # break usage into wrappable parts - opt_parts = self._get_actions_usage_parts(optionals, groups) - pos_parts = self._get_actions_usage_parts(positionals, groups) + opt_parts = parts[:pos_start] + pos_parts = parts[pos_start:] # helper for wrapping lines def get_lines(parts, indent, prefix=None): @@ -411,120 +408,114 @@ def get_lines(parts, indent, prefix=None): # prefix with 'usage:' return f'{t.usage}{prefix}{t.reset}{usage}\n\n' - def _format_actions_usage(self, actions, groups): - return ' '.join(self._get_actions_usage_parts(actions, groups)) - def _is_long_option(self, string): - return len(string) >= 2 and string[1] in self._prefix_chars - - def _is_short_option(self, string): - return ( - not self._is_long_option(string) - and len(string) >= 1 - and string[0] in self._prefix_chars - ) + return len(string) > 2 def _get_actions_usage_parts(self, actions, groups): - # find group indices and identify actions in groups - group_actions = set() - inserts = {} - for group in groups: - if not group._group_actions: - raise ValueError(f'empty group {group}') - - if all(action.help is SUPPRESS for action in group._group_actions): - continue + """Get usage parts with split index for optionals/positionals. - try: - start = min(actions.index(item) for item in group._group_actions) - except ValueError: - continue - else: - end = start + len(group._group_actions) - if set(actions[start:end]) == set(group._group_actions): - group_actions.update(group._group_actions) - inserts[start, end] = group + Returns (parts, pos_start) where pos_start is the index in parts + where positionals begin. + This preserves mutually exclusive group formatting across the + optionals/positionals boundary (gh-75949). + """ + actions = [action for action in actions if action.help is not SUPPRESS] + # group actions by mutually exclusive groups + action_groups = dict.fromkeys(actions) + for group in groups: + for action in group._group_actions: + if action in action_groups: + action_groups[action] = group + # positional arguments keep their position + positionals = [] + for action in actions: + if not action.option_strings: + group = action_groups.pop(action) + if group: + group_actions = [ + action2 for action2 in group._group_actions + if action2.option_strings and + action_groups.pop(action2, None) + ] + [action] + positionals.append((group.required, group_actions)) + else: + positionals.append((None, [action])) + # the remaining optional arguments are sorted by the position of + # the first option in the group + optionals = [] + for action in actions: + if action.option_strings and action in action_groups: + group = action_groups.pop(action) + if group: + group_actions = [action] + [ + action2 for action2 in group._group_actions + if action2.option_strings and + action_groups.pop(action2, None) + ] + optionals.append((group.required, group_actions)) + else: + optionals.append((None, [action])) # collect all actions format strings parts = [] t = self._theme - for action in actions: - - # suppressed arguments are marked with None - if action.help is SUPPRESS: - part = None - - # produce all arg strings - elif not action.option_strings: - default = self._get_default_metavar_for_positional(action) - part = ( - t.summary_action - + self._format_args(action, default) - + t.reset - ) - - # if it's in a group, strip the outer [] - if action in group_actions: - if part[0] == '[' and part[-1] == ']': - part = part[1:-1] - - # produce the first way to invoke the option in brackets - else: - option_string = action.option_strings[0] - - # if the Optional doesn't take a value, format is: - # -s or --long - if action.nargs == 0: - part = action.format_usage() - if self._is_long_option(part): - part = f"{t.summary_long_option}{part}{t.reset}" - elif self._is_short_option(part): - part = f"{t.summary_short_option}{part}{t.reset}" - - # if the Optional takes a value, format is: - # -s ARGS or --long ARGS + pos_start = None + for i, (required, group) in enumerate(optionals + positionals): + start = len(parts) + if i == len(optionals): + pos_start = start + in_group = len(group) > 1 + for action in group: + # produce all arg strings + if not action.option_strings: + default = self._get_default_metavar_for_positional(action) + part = self._format_args(action, default) + # if it's in a group, strip the outer [] + if in_group: + if part[0] == '[' and part[-1] == ']': + part = part[1:-1] + part = t.summary_action + part + t.reset + + # produce the first way to invoke the option in brackets else: - default = self._get_default_metavar_for_optional(action) - args_string = self._format_args(action, default) + option_string = action.option_strings[0] if self._is_long_option(option_string): option_color = t.summary_long_option - elif self._is_short_option(option_string): + else: option_color = t.summary_short_option - part = ( - f"{option_color}{option_string} " - f"{t.summary_label}{args_string}{t.reset}" - ) - - # make it look optional if it's not required or in a group - if not action.required and action not in group_actions: - part = '[%s]' % part - - # add the action string to the list - parts.append(part) - - # group mutually exclusive actions - inserted_separators_indices = set() - for start, end in sorted(inserts, reverse=True): - group = inserts[start, end] - group_parts = [item for item in parts[start:end] if item is not None] - group_size = len(group_parts) - if group.required: - open, close = "()" if group_size > 1 else ("", "") - else: - open, close = "[]" - group_parts[0] = open + group_parts[0] - group_parts[-1] = group_parts[-1] + close - for i, part in enumerate(group_parts[:-1], start=start): - # insert a separator if not already done in a nested group - if i not in inserted_separators_indices: - parts[i] = part + ' |' - inserted_separators_indices.add(i) - parts[start + group_size - 1] = group_parts[-1] - for i in range(start + group_size, end): - parts[i] = None - - # return the usage parts - return [item for item in parts if item is not None] + + # if the Optional doesn't take a value, format is: + # -s or --long + if action.nargs == 0: + part = action.format_usage() + part = f"{option_color}{part}{t.reset}" + + # if the Optional takes a value, format is: + # -s ARGS or --long ARGS + else: + default = self._get_default_metavar_for_optional(action) + args_string = self._format_args(action, default) + part = ( + f"{option_color}{option_string} " + f"{t.summary_label}{args_string}{t.reset}" + ) + + # make it look optional if it's not required or in a group + if not (action.required or required or in_group): + part = '[%s]' % part + + # add the action string to the list + parts.append(part) + + if in_group: + parts[start] = ('(' if required else '[') + parts[start] + for i in range(start, len(parts) - 1): + parts[i] += ' |' + parts[-1] += ')' if required else ']' + + if pos_start is None: + pos_start = len(parts) + return parts, pos_start def _format_text(self, text): if '%(prog)' in text: @@ -606,10 +597,8 @@ def color_option_strings(strings): for s in strings: if self._is_long_option(s): parts.append(f"{t.long_option}{s}{t.reset}") - elif self._is_short_option(s): - parts.append(f"{t.short_option}{s}{t.reset}") else: - parts.append(s) + parts.append(f"{t.short_option}{s}{t.reset}") return parts # if the Optional doesn't take a value, format is: @@ -754,11 +743,14 @@ def _get_help_string(self, action): if help is None: help = '' - if '%(default)' not in help: - if action.default is not SUPPRESS: - defaulting_nargs = [OPTIONAL, ZERO_OR_MORE] - if action.option_strings or action.nargs in defaulting_nargs: - help += _(' (default: %(default)s)') + if ( + '%(default)' not in help + and action.default is not SUPPRESS + and not action.required + ): + defaulting_nargs = (OPTIONAL, ZERO_OR_MORE) + if action.option_strings or action.nargs in defaulting_nargs: + help += _(' (default: %(default)s)') return help @@ -1240,7 +1232,7 @@ def __init__(self, self._name_parser_map = {} self._choices_actions = [] self._deprecated = set() - self._color = False + self._color = True super(_SubParsersAction, self).__init__( option_strings=option_strings, @@ -1479,6 +1471,7 @@ def __init__(self, # ==================== # Registration methods # ==================== + def register(self, registry_name, value, object): registry = self._registries.setdefault(registry_name, {}) registry[value] = object @@ -1489,6 +1482,7 @@ def _registry_get(self, registry_name, value, default=None): # ================================== # Namespace default accessor methods # ================================== + def set_defaults(self, **kwargs): self._defaults.update(kwargs) @@ -1508,6 +1502,7 @@ def get_default(self, dest): # ======================= # Adding argument actions # ======================= + def add_argument(self, *args, **kwargs): """ add_argument(dest, ..., name=value, ...) @@ -1540,7 +1535,7 @@ def add_argument(self, *args, **kwargs): action_name = kwargs.get('action') action_class = self._pop_action_class(kwargs) if not callable(action_class): - raise ValueError('unknown action {action_class!r}') + raise ValueError(f'unknown action {action_class!r}') action = action_class(**kwargs) # raise an error if action for positional argument does not @@ -1558,8 +1553,8 @@ def add_argument(self, *args, **kwargs): f'instance of it must be passed') # raise an error if the metavar does not match the type - if hasattr(self, "_get_formatter"): - formatter = self._get_formatter() + if hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() try: formatter._format_args(action, None) except TypeError: @@ -1747,8 +1742,8 @@ def _handle_conflict_resolve(self, action, conflicting_actions): action.container._remove_action(action) def _check_help(self, action): - if action.help and hasattr(self, "_get_formatter"): - formatter = self._get_formatter() + if action.help and hasattr(self, "_get_validation_formatter"): + formatter = self._get_validation_formatter() try: formatter._expand_help(action) except (ValueError, TypeError, KeyError) as exc: @@ -1884,7 +1879,7 @@ def __init__(self, exit_on_error=True, *, suggest_on_error=False, - color=False, + color=True, ): superinit = super(ArgumentParser, self).__init__ superinit(description=description, @@ -1903,15 +1898,16 @@ def __init__(self, self.suggest_on_error = suggest_on_error self.color = color + # Cached formatter for validation (avoids repeated _set_color calls) + self._cached_formatter = None + add_group = self.add_argument_group self._positionals = add_group(_('positional arguments')) self._optionals = add_group(_('options')) self._subparsers = None # register types - def identity(string): - return string - self.register('type', None, identity) + self.register('type', None, _identity) # add help argument if necessary # (using explicit default to override global argument_default) @@ -1933,6 +1929,7 @@ def identity(string): # ======================= # Pretty __repr__ methods # ======================= + def _get_kwargs(self): names = [ 'prog', @@ -1947,6 +1944,7 @@ def _get_kwargs(self): # ================================== # Optional/Positional adding methods # ================================== + def add_subparsers(self, **kwargs): if self._subparsers is not None: raise ValueError('cannot have multiple subparser arguments') @@ -1964,7 +1962,9 @@ def add_subparsers(self, **kwargs): # prog defaults to the usage message of this parser, skipping # optional arguments and with no "usage:" prefix if kwargs.get('prog') is None: - formatter = self._get_formatter() + # Create formatter without color to avoid storing ANSI codes in prog + formatter = self.formatter_class(prog=self.prog) + formatter._set_color(False) positionals = self._get_positional_actions() groups = self._mutually_exclusive_groups formatter.add_usage(None, positionals, groups, '') @@ -2000,6 +2000,7 @@ def _get_positional_actions(self): # ===================================== # Command line argument parsing methods # ===================================== + def parse_args(self, args=None, namespace=None): args, argv = self.parse_known_args(args, namespace) if argv: @@ -2594,6 +2595,7 @@ def parse_known_intermixed_args(self, args=None, namespace=None): # ======================== # Value conversion methods # ======================== + def _get_values(self, action, arg_strings): # optional argument produces a default when not present if not arg_strings and action.nargs == OPTIONAL: @@ -2676,7 +2678,7 @@ def _check_value(self, action, value): if value not in choices: args = {'value': str(value), - 'choices': ', '.join(map(str, action.choices))} + 'choices': ', '.join(repr(str(choice)) for choice in action.choices)} msg = _('invalid choice: %(value)r (choose from %(choices)s)') if self.suggest_on_error and isinstance(value, str): @@ -2693,6 +2695,7 @@ def _check_value(self, action, value): # ======================= # Help-formatting methods # ======================= + def format_usage(self): formatter = self._get_formatter() formatter.add_usage(self.usage, self._actions, @@ -2723,20 +2726,21 @@ def format_help(self): return formatter.format_help() def _get_formatter(self): - if isinstance(self.formatter_class, type) and issubclass( - self.formatter_class, HelpFormatter - ): - return self.formatter_class( - prog=self.prog, - prefix_chars=self.prefix_chars, - color=self.color, - ) - else: - return self.formatter_class(prog=self.prog) + formatter = self.formatter_class(prog=self.prog) + formatter._set_color(self.color) + return formatter + + def _get_validation_formatter(self): + # Return cached formatter for read-only validation operations + # (_expand_help and _format_args). Avoids repeated slow _set_color calls. + if self._cached_formatter is None: + self._cached_formatter = self._get_formatter() + return self._cached_formatter # ===================== # Help-printing methods # ===================== + def print_usage(self, file=None): if file is None: file = _sys.stdout @@ -2758,6 +2762,7 @@ def _print_message(self, message, file=None): # =============== # Exiting methods # =============== + def exit(self, status=0, message=None): if message: self._print_message(message, _sys.stderr) diff --git a/Lib/ast.py b/Lib/ast.py index b9791bf52d3e08a..2f11683ecf7c681 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -147,18 +147,16 @@ def _format(node, level=0): if value is None and getattr(cls, name, ...) is None: keywords = True continue - if ( - not show_empty - and (value is None or value == []) - # Special cases: - # `Constant(value=None)` and `MatchSingleton(value=None)` - and not isinstance(node, (Constant, MatchSingleton)) - ): - args_buffer.append(repr(value)) - continue - elif not keywords: - args.extend(args_buffer) - args_buffer = [] + if not show_empty: + if value == []: + field_type = cls._field_types.get(name, object) + if getattr(field_type, '__origin__', ...) is list: + if not keywords: + args_buffer.append(repr(value)) + continue + if not keywords: + args.extend(args_buffer) + args_buffer = [] value, simple = _format(value, level) allsimple = allsimple and simple if keywords: diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index 4be7112fa017d41..32a5dbae03af216 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -51,15 +51,24 @@ def __getattr__(name: str): import warnings - deprecated = { - "AbstractEventLoopPolicy", - "DefaultEventLoopPolicy", - "WindowsSelectorEventLoopPolicy", - "WindowsProactorEventLoopPolicy", - } - if name in deprecated: - warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) - # deprecated things have underscores in front of them - return globals()["_" + name] + match name: + case "AbstractEventLoopPolicy": + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return events._AbstractEventLoopPolicy + case "DefaultEventLoopPolicy": + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + if sys.platform == 'win32': + return windows_events._DefaultEventLoopPolicy + return unix_events._DefaultEventLoopPolicy + case "WindowsSelectorEventLoopPolicy": + if sys.platform == 'win32': + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return windows_events._WindowsSelectorEventLoopPolicy + # Else fall through to the AttributeError below. + case "WindowsProactorEventLoopPolicy": + if sys.platform == 'win32': + warnings._deprecated(f"asyncio.{name}", remove=(3, 16)) + return windows_events._WindowsProactorEventLoopPolicy + # Else fall through to the AttributeError below. raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/asyncio/__main__.py b/Lib/asyncio/__main__.py index 21ca5c5f62ae838..7a8fa6ad5726f8a 100644 --- a/Lib/asyncio/__main__.py +++ b/Lib/asyncio/__main__.py @@ -64,7 +64,7 @@ def callback(): except BaseException as exc: future.set_exception(exc) - loop.call_soon_threadsafe(callback, context=self.context) + self.loop.call_soon_threadsafe(callback, context=self.context) try: return future.result() @@ -74,7 +74,8 @@ def callback(): return except BaseException: if keyboard_interrupted: - self.write("\nKeyboardInterrupt\n") + if not CAN_USE_PYREPL: + self.write("\nKeyboardInterrupt\n") else: self.showtraceback() return self.STATEMENT_FAILED @@ -85,22 +86,27 @@ def run(self): global return_code try: - banner = ( - f'asyncio REPL {sys.version} on {sys.platform}\n' - f'Use "await" directly instead of "asyncio.run()".\n' - f'Type "help", "copyright", "credits" or "license" ' - f'for more information.\n' - ) + if not sys.flags.quiet: + banner = ( + f'asyncio REPL {sys.version} on {sys.platform}\n' + f'Use "await" directly instead of "asyncio.run()".\n' + f'Type "help", "copyright", "credits" or "license" ' + f'for more information.\n' + ) - console.write(banner) + console.write(banner) - if startup_path := os.getenv("PYTHONSTARTUP"): + if not sys.flags.ignore_environment and (startup_path := os.getenv("PYTHONSTARTUP")): sys.audit("cpython.run_startup", startup_path) - - import tokenize - with tokenize.open(startup_path) as f: - startup_code = compile(f.read(), startup_path, "exec") + try: + import tokenize + with tokenize.open(startup_path) as f: + startup_code = compile(f.read(), startup_path, "exec") exec(startup_code, console.locals) + except SystemExit: + raise + except BaseException: + console.showtraceback() ps1 = getattr(sys, "ps1", ">>> ") if CAN_USE_PYREPL: @@ -176,7 +182,7 @@ def interrupt(self) -> None: sys.audit("cpython.run_stdin") - if os.getenv('PYTHON_BASIC_REPL'): + if not sys.flags.ignore_environment and os.getenv('PYTHON_BASIC_REPL'): CAN_USE_PYREPL = False else: from _pyrepl.main import CAN_USE_PYREPL @@ -235,4 +241,5 @@ def interrupt(self) -> None: break console.write('exiting asyncio REPL...\n') + loop.close() sys.exit(return_code) diff --git a/Lib/asyncio/base_events.py b/Lib/asyncio/base_events.py index 04fb961e9985e4d..6d1f1f843235c1b 100644 --- a/Lib/asyncio/base_events.py +++ b/Lib/asyncio/base_events.py @@ -483,10 +483,10 @@ def set_task_factory(self, factory): If factory is None the default task factory will be set. If factory is a callable, it should have a signature matching - '(loop, coro, **kwargs)', where 'loop' will be a reference to the active - event loop, 'coro' will be a coroutine object, and **kwargs will be - arbitrary keyword arguments that should be passed on to Task. - The callable must return a Task. + '(loop, coro, **kwargs)', where 'loop' will be a reference to the + active event loop, 'coro' will be a coroutine object, and **kwargs + will be arbitrary keyword arguments that should be passed on to + Task. The callable must return a Task. """ if factory is not None and not callable(factory): raise TypeError('task factory must be a callable or None') @@ -636,7 +636,7 @@ def _check_running(self): def _run_forever_setup(self): """Prepare the run loop to process events. - This method exists so that custom custom event loop subclasses (e.g., event loops + This method exists so that custom event loop subclasses (e.g., event loops that integrate a GUI event loop with Python's event loop) have access to all the loop setup logic. """ @@ -656,7 +656,7 @@ def _run_forever_setup(self): def _run_forever_cleanup(self): """Clean up after an event loop finishes the looping over events. - This method exists so that custom custom event loop subclasses (e.g., event loops + This method exists so that custom event loop subclasses (e.g., event loops that integrate a GUI event loop with Python's event loop) have access to all the loop cleanup logic. """ @@ -721,8 +721,8 @@ def run_until_complete(self, future): def stop(self): """Stop running the event loop. - Every callback already scheduled will still run. This simply informs - run_forever to stop looping after a complete iteration. + Every callback already scheduled will still run. This simply + informs run_forever to stop looping after a complete iteration. """ self._stopping = True @@ -963,7 +963,7 @@ async def _sock_sendfile_native(self, sock, file, offset, count): f"and file {file!r} combination") async def _sock_sendfile_fallback(self, sock, file, offset, count): - if offset: + if hasattr(file, 'seek'): file.seek(offset) blocksize = ( min(count, constants.SENDFILE_FALLBACK_READBUFFER_SIZE) @@ -1016,38 +1016,43 @@ async def _connect_sock(self, exceptions, addr_info, local_addr_infos=None): family, type_, proto, _, address = addr_info sock = None try: - sock = socket.socket(family=family, type=type_, proto=proto) - sock.setblocking(False) - if local_addr_infos is not None: - for lfamily, _, _, _, laddr in local_addr_infos: - # skip local addresses of different family - if lfamily != family: - continue - try: - sock.bind(laddr) - break - except OSError as exc: - msg = ( - f'error while attempting to bind on ' - f'address {laddr!r}: {str(exc).lower()}' - ) - exc = OSError(exc.errno, msg) - my_exceptions.append(exc) - else: # all bind attempts failed - if my_exceptions: - raise my_exceptions.pop() - else: - raise OSError(f"no matching local address with {family=} found") - await self.sock_connect(sock, address) - return sock - except OSError as exc: - my_exceptions.append(exc) - if sock is not None: - sock.close() - raise + try: + sock = socket.socket(family=family, type=type_, proto=proto) + sock.setblocking(False) + if local_addr_infos is not None: + for lfamily, _, _, _, laddr in local_addr_infos: + # skip local addresses of different family + if lfamily != family: + continue + try: + sock.bind(laddr) + break + except OSError as exc: + msg = ( + f'error while attempting to bind on ' + f'address {laddr!r}: {str(exc).lower()}' + ) + exc = OSError(exc.errno, msg) + my_exceptions.append(exc) + else: # all bind attempts failed + if my_exceptions: + raise my_exceptions.pop() + else: + raise OSError(f"no matching local address with {family=} found") + await self.sock_connect(sock, address) + return sock + except OSError as exc: + my_exceptions.append(exc) + raise except: if sock is not None: - sock.close() + try: + sock.close() + except OSError: + # An error when closing a newly created socket is + # not important, but it can overwrite more important + # non-OSError error. So ignore it. + pass raise finally: exceptions = my_exceptions = None @@ -1065,12 +1070,12 @@ async def create_connection( Create a streaming transport connection to a given internet host and port: socket family AF_INET or socket.AF_INET6 depending on host (or - family if specified), socket type SOCK_STREAM. protocol_factory must be - a callable returning a protocol instance. + family if specified), socket type SOCK_STREAM. protocol_factory must + be a callable returning a protocol instance. - This method is a coroutine which will try to establish the connection - in the background. When successful, the coroutine returns a - (transport, protocol) pair. + This method is a coroutine which will try to establish the + connection in the background. When successful, the coroutine + returns a (transport, protocol) pair. """ if server_hostname is not None and not ssl: raise ValueError('server_hostname is only meaningful with ssl') @@ -1161,7 +1166,7 @@ async def create_connection( raise ExceptionGroup("create_connection failed", exceptions) if len(exceptions) == 1: raise exceptions[0] - else: + elif exceptions: # If they all have the same str(), raise one. model = str(exceptions[0]) if all(str(exc) == model for exc in exceptions): @@ -1170,6 +1175,9 @@ async def create_connection( # the various error messages. raise OSError('Multiple exceptions: {}'.format( ', '.join(str(exc) for exc in exceptions))) + else: + # No exceptions were collected, raise a timeout error + raise TimeoutError('create_connection failed') finally: exceptions = None @@ -1270,7 +1278,6 @@ async def sendfile(self, transport, file, offset=0, count=None, raise RuntimeError( f"fallback is disabled and native sendfile is not " f"supported for transport {transport!r}") - return await self._sendfile_fallback(transport, file, offset, count) @@ -1279,7 +1286,7 @@ async def _sendfile_native(self, transp, file, offset, count): "sendfile syscall is not supported") async def _sendfile_fallback(self, transp, file, offset, count): - if offset: + if hasattr(file, 'seek'): file.seek(offset) blocksize = min(count, 16384) if count else 16384 buf = bytearray(blocksize) @@ -1337,6 +1344,17 @@ async def start_tls(self, transport, protocol, sslcontext, *, # have a chance to get called before "ssl_protocol.connection_made()". transport.pause_reading() + # gh-142352: move buffered StreamReader data to SSLProtocol + if server_side: + from .streams import StreamReaderProtocol + if isinstance(protocol, StreamReaderProtocol): + stream_reader = getattr(protocol, '_stream_reader', None) + if stream_reader is not None: + buffer = stream_reader._buffer + if buffer: + ssl_protocol._incoming.write(buffer) + buffer.clear() + transport.set_protocol(ssl_protocol) conmade_cb = self.call_soon(ssl_protocol.connection_made, transport) resume_cb = self.call_soon(transport.resume_reading) @@ -1524,11 +1542,11 @@ async def create_server( The host parameter can be a string, in that case the TCP server is bound to host and port. - The host parameter can also be a sequence of strings and in that case - the TCP server is bound to all hosts of the sequence. If a host - appears multiple times (possibly indirectly e.g. when hostnames - resolve to the same IP address), the server is only bound once to that - host. + The host parameter can also be a sequence of strings and in that + case the TCP server is bound to all hosts of the sequence. If + a host appears multiple times (possibly indirectly e.g. when + hostnames resolve to the same IP address), the server is only bound + once to that host. Return a Server object which can be used to stop the service. diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py index 9c2ba679ce2bf1d..0423bc100b2d932 100644 --- a/Lib/asyncio/base_subprocess.py +++ b/Lib/asyncio/base_subprocess.py @@ -22,7 +22,7 @@ def __init__(self, loop, protocol, args, shell, self._proc = None self._pid = None self._returncode = None - self._exit_waiters = [] + self._exit_waiters = set() self._pending_calls = collections.deque() self._pipes = {} self._finished = False @@ -104,7 +104,12 @@ def close(self): for proto in self._pipes.values(): if proto is None: continue - proto.pipe.close() + # See gh-114177 + # skip closing the pipe if loop is already closed + # this can happen e.g. when loop is closed immediately after + # process is killed + if self._loop and not self._loop.is_closed(): + proto.pipe.close() if (self._proc is not None and # has the child process finished? @@ -160,6 +165,9 @@ def kill(self): else: def send_signal(self, signal): self._check_proc() + if self._returncode is not None: + # The process already exited + return try: os.kill(self._proc.pid, signal) except ProcessLookupError: @@ -203,6 +211,14 @@ async def _connect_pipes(self, waiter): except (SystemExit, KeyboardInterrupt): raise except BaseException as exc: + # Close any pipes that were already connected before the + # error/cancellation to avoid leaking file descriptors. + for proto in self._pipes.values(): + if proto is not None: + proto.pipe.close() + for raw_pipe in (proc.stdin, proc.stdout, proc.stderr): + if raw_pipe is not None: + raw_pipe.close() if waiter is not None and not waiter.cancelled(): waiter.set_exception(exc) else: @@ -228,6 +244,7 @@ def _process_exited(self, returncode): if self._loop.get_debug(): logger.info('%r exited with return code %r', self, returncode) self._returncode = returncode + if self._proc.returncode is None: # asyncio uses a child watcher: copy the status into the Popen # object. On Python 3.6, it is required to avoid a ResourceWarning. @@ -236,6 +253,13 @@ def _process_exited(self, returncode): self._try_finish() + # gh-119710: Wake up futures waiting for wait() as soon as the process + # exits. + for waiter in self._exit_waiters: + if not waiter.done(): + waiter.set_result(returncode) + self._exit_waiters = None + async def _wait(self): """Wait until the process exit and return the process return code. @@ -244,13 +268,18 @@ async def _wait(self): return self._returncode waiter = self._loop.create_future() - self._exit_waiters.append(waiter) - return await waiter + self._exit_waiters.add(waiter) + try: + return await waiter + finally: + if self._exit_waiters is not None: + self._exit_waiters.discard(waiter) def _try_finish(self): assert not self._finished if self._returncode is None: return + if all(p is not None and p.disconnected for p in self._pipes.values()): self._finished = True @@ -260,11 +289,6 @@ def _call_connection_lost(self, exc): try: self._protocol.connection_lost(exc) finally: - # wake up futures waiting for wait() - for waiter in self._exit_waiters: - if not waiter.cancelled(): - waiter.set_result(self._returncode) - self._exit_waiters = None self._loop = None self._proc = None self._protocol = None diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index a51319cb72a6a9b..6727065bbe323ff 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -18,8 +18,8 @@ def _is_debug_mode(): def iscoroutinefunction(func): - import warnings """Return True if func is a decorated coroutine function.""" + import warnings warnings._deprecated("asyncio.iscoroutinefunction", f"{warnings._DEPRECATED_MSG}; " "use inspect.iscoroutinefunction() instead", diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 2913f901dca65fa..42d8408a38dfd1f 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -5,14 +5,11 @@ # SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io __all__ = ( - "_AbstractEventLoopPolicy", "AbstractEventLoop", "AbstractServer", "Handle", "TimerHandle", - "_get_event_loop_policy", "get_event_loop_policy", - "_set_event_loop_policy", "set_event_loop_policy", "get_event_loop", "set_event_loop", @@ -377,8 +374,8 @@ async def create_server( If host is an empty string or None all interfaces are assumed and a list of multiple sockets will be returned (most likely - one for IPv4 and another one for IPv6). The host parameter can also be - a sequence (e.g. list) of hosts to bind to. + one for IPv4 and another one for IPv6). The host parameter can also + be a sequence (e.g. list) of hosts to bind to. family can be set to either AF_INET or AF_INET6 to force the socket to use IPv4 or IPv6. If not set it will be determined @@ -418,8 +415,9 @@ async def create_server( start_serving set to True (default) causes the created server to start accepting connections immediately. When set to False, - the user should await Server.start_serving() or Server.serve_forever() - to make the server to start accepting connections. + the user should await Server.start_serving() or + Server.serve_forever() to make the server to start accepting + connections. """ raise NotImplementedError @@ -482,8 +480,9 @@ async def create_unix_server( start_serving set to True (default) causes the created server to start accepting connections immediately. When set to False, - the user should await Server.start_serving() or Server.serve_forever() - to make the server to start accepting connections. + the user should await Server.start_serving() or + Server.serve_forever() to make the server to start accepting + connections. """ raise NotImplementedError @@ -514,8 +513,8 @@ async def create_datagram_endpoint(self, protocol_factory, protocol_factory must be a callable returning a protocol instance. - socket family AF_INET, socket.AF_INET6 or socket.AF_UNIX depending on - host (or family if specified), socket type SOCK_DGRAM. + socket family AF_INET, socket.AF_INET6 or socket.AF_UNIX depending + on host (or family if specified), socket type SOCK_DGRAM. reuse_address tells the kernel to reuse a local socket in TIME_WAIT state, without waiting for its natural timeout to @@ -555,7 +554,8 @@ async def connect_read_pipe(self, protocol_factory, pipe): async def connect_write_pipe(self, protocol_factory, pipe): """Register write pipe in event loop. - protocol_factory should instantiate object with BaseProtocol interface. + protocol_factory should instantiate object with BaseProtocol + interface. Pipe is file-like object already switched to nonblocking. Return pair (transport, protocol), where transport support WriteTransport interface.""" @@ -791,7 +791,10 @@ def _init_event_loop_policy(): global _event_loop_policy with _lock: if _event_loop_policy is None: # pragma: no branch - from . import _DefaultEventLoopPolicy + if sys.platform == 'win32': + from .windows_events import _DefaultEventLoopPolicy + else: + from .unix_events import _DefaultEventLoopPolicy _event_loop_policy = _DefaultEventLoopPolicy() diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index d1df6707302277e..e8a99556b1885ec 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -392,7 +392,7 @@ def _set_state(future, other): def _call_check_cancel(destination): if destination.cancelled(): - if source_loop is None or source_loop is dest_loop: + if source_loop is None or source_loop is events._get_running_loop(): source.cancel() else: source_loop.call_soon_threadsafe(source.cancel) @@ -401,7 +401,7 @@ def _call_set_state(source): if (destination.cancelled() and dest_loop is not None and dest_loop.is_closed()): return - if dest_loop is None or dest_loop is source_loop: + if dest_loop is None or dest_loop is events._get_running_loop(): _set_state(destination, source) else: if dest_loop.is_closed(): diff --git a/Lib/asyncio/graph.py b/Lib/asyncio/graph.py index d8df7c9919abbf7..35f7fa62140bd49 100644 --- a/Lib/asyncio/graph.py +++ b/Lib/asyncio/graph.py @@ -1,6 +1,7 @@ """Introspection utils for tasks call graphs.""" import dataclasses +import io import sys import types @@ -16,9 +17,6 @@ 'FutureCallGraph', ) -if False: # for type checkers - from typing import TextIO - # Sadly, we can't re-use the traceback module's datastructures as those # are tailored for error reporting, whereas we need to represent an # async call graph. @@ -114,13 +112,13 @@ def capture_call_graph( optional keyword-only 'depth' argument can be used to skip the specified number of frames from top of the stack. - If the optional keyword-only 'limit' argument is provided, each call stack - in the resulting graph is truncated to include at most ``abs(limit)`` - entries. If 'limit' is positive, the entries left are the closest to - the invocation point. If 'limit' is negative, the topmost entries are - left. If 'limit' is omitted or None, all entries are present. - If 'limit' is 0, the call stack is not captured at all, only - "awaited by" information is present. + If the optional keyword-only 'limit' argument is provided, each call + stack in the resulting graph is truncated to include at most + ``abs(limit)`` entries. If 'limit' is positive, the entries left are + the closest to the invocation point. If 'limit' is negative, the + topmost entries are left. If 'limit' is omitted or None, all entries + are present. If 'limit' is 0, the call stack is not captured at all, + only "awaited by" information is present. """ loop = events._get_running_loop() @@ -270,7 +268,7 @@ def print_call_graph( future: futures.Future | None = None, /, *, - file: TextIO | None = None, + file: io.Writer[str] | None = None, depth: int = 1, limit: int | None = None, ) -> None: diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index fa3a94764b507ae..1592d30ed45832c 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -158,10 +158,10 @@ def _wake_up_first(self): class Event(mixins._LoopBoundMixin): """Asynchronous equivalent to threading.Event. - Class implementing event objects. An event manages a flag that can be set - to true with the set() method and reset to false with the clear() method. - The wait() method blocks until the flag is true. The flag is initially - false. + Class implementing event objects. An event manages a flag that can be + set to true with the set() method and reset to false with the clear() + method. The wait() method blocks until the flag is true. The flag is + initially false. """ def __init__(self): @@ -353,9 +353,9 @@ class Semaphore(_ContextManagerMixin, mixins._LoopBoundMixin): """A Semaphore implementation. A semaphore manages an internal counter which is decremented by each - acquire() call and incremented by each release() call. The counter - can never go below zero; when acquire() finds that it is zero, it blocks, - waiting until some other thread calls release(). + acquire() call and incremented by each release() call. The counter + can never go below zero; when acquire() finds that it is zero, it + blocks, waiting until some other thread calls release(). Semaphores also support the context management protocol. @@ -511,8 +511,8 @@ async def __aexit__(self, *args): async def wait(self): """Wait for the barrier. - When the specified number of tasks have started waiting, they are all - simultaneously awoken. + When the specified number of tasks have started waiting, they are + all simultaneously awoken. Returns an unique and individual index number from 0 to 'parties-1'. """ async with self._cond: diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index 7eb55bd63ddb73e..3cd8ceadaf272f5 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -62,6 +62,7 @@ def __init__(self, loop, sock, protocol, waiter=None, self._closing = False # Set when close() called. self._called_connection_lost = False self._eof_written = False + self._empty_waiter = None if self._server is not None: self._server._attach(self) self._loop.call_soon(self._protocol.connection_made, self) @@ -331,10 +332,6 @@ class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport, _start_tls_compatible = True - def __init__(self, *args, **kw): - super().__init__(*args, **kw) - self._empty_waiter = None - def write(self, data): if not isinstance(data, (bytes, bytearray, memoryview)): raise TypeError( @@ -460,10 +457,11 @@ def _pipe_closed(self, fut): class _ProactorDatagramTransport(_ProactorBasePipeTransport, transports.DatagramTransport): max_size = 256 * 1024 + _header_size = 8 + def __init__(self, loop, sock, protocol, address=None, waiter=None, extra=None): self._address = address - self._empty_waiter = None self._buffer_size = 0 # We don't need to call _protocol.connection_made() since our base # constructor does it for us. @@ -499,7 +497,7 @@ def sendto(self, data, addr=None): # Ensure that what we buffer is immutable. self._buffer.append((bytes(data), addr)) - self._buffer_size += len(data) + 8 # include header bytes + self._buffer_size += len(data) + self._header_size if self._write_fut is None: # No current write operations are active, kick one off @@ -526,7 +524,7 @@ def _loop_writing(self, fut=None): return data, addr = self._buffer.popleft() - self._buffer_size -= len(data) + self._buffer_size -= len(data) + self._header_size if self._address is not None: self._write_fut = self._loop._proactor.send(self._sock, data) @@ -754,8 +752,7 @@ async def _sock_sendfile_native(self, sock, file, offset, count): offset += blocksize total_sent += blocksize finally: - if total_sent > 0: - file.seek(offset) + file.seek(offset) async def _sendfile_native(self, transp, file, offset, count): resume_reading = transp.is_reading() diff --git a/Lib/asyncio/queues.py b/Lib/asyncio/queues.py index 2f3865114a84f95..30004f2bc9bacd2 100644 --- a/Lib/asyncio/queues.py +++ b/Lib/asyncio/queues.py @@ -33,11 +33,11 @@ class QueueShutDown(Exception): class Queue(mixins._LoopBoundMixin): """A queue, useful for coordinating producer and consumer coroutines. - If maxsize is less than or equal to zero, the queue size is infinite. If it - is an integer greater than 0, then "await put()" will block when the - queue reaches maxsize, until an item is removed by get(). + If maxsize is less than or equal to zero, the queue size is infinite. + If it is an integer greater than 0, then "await put()" will block when + the queue reaches maxsize, until an item is removed by get(). - Unlike the standard library Queue, you can reliably know this Queue's size + Unlike queue.Queue, you can reliably know this Queue's size with qsize(), since your single-threaded asyncio application won't be interrupted between calling qsize() and doing an operation on the Queue. """ @@ -174,8 +174,8 @@ async def get(self): If queue is empty, wait until an item is available. - Raises QueueShutDown if the queue has been shut down and is empty, or - if the queue has been shut down immediately. + Raises QueueShutDown if the queue has been shut down and is empty, + or if the queue has been shut down immediately. """ while self.empty(): if self._is_shutdown and self.empty(): @@ -203,10 +203,11 @@ async def get(self): def get_nowait(self): """Remove and return an item from the queue. - Return an item if one is immediately available, else raise QueueEmpty. + Return an item if one is immediately available, else raise + QueueEmpty. - Raises QueueShutDown if the queue has been shut down and is empty, or - if the queue has been shut down immediately. + Raises QueueShutDown if the queue has been shut down and is empty, + or if the queue has been shut down immediately. """ if self.empty(): if self._is_shutdown: @@ -223,15 +224,12 @@ def task_done(self): a subsequent call to task_done() tells the queue that the processing on the task is complete. - If a join() is currently blocking, it will resume when all items have - been processed (meaning that a task_done() call was received for every - item that had been put() into the queue). + If a join() is currently blocking, it will resume when all items + have been processed (meaning that a task_done() call was received + for every item that had been put() into the queue). - shutdown(immediate=True) calls task_done() for each remaining item in - the queue. - - Raises ValueError if called more times than there were items placed in - the queue. + Raises ValueError if called more times than there were items placed + in the queue. """ if self._unfinished_tasks <= 0: raise ValueError('task_done() called too many times') @@ -242,10 +240,11 @@ def task_done(self): async def join(self): """Block until all items in the queue have been gotten and processed. - The count of unfinished tasks goes up whenever an item is added to the - queue. The count goes down whenever a consumer calls task_done() to - indicate that the item was retrieved and all work on it is complete. - When the count of unfinished tasks drops to zero, join() unblocks. + The count of unfinished tasks goes up whenever an item is added to + the queue. The count goes down whenever a consumer calls + task_done() to indicate that the item was retrieved and all work on + it is complete. When the count of unfinished tasks drops to zero, + join() unblocks. """ if self._unfinished_tasks > 0: await self._finished.wait() @@ -256,9 +255,11 @@ def shutdown(self, immediate=False): By default, gets will only raise once the queue is empty. Set 'immediate' to True to make gets raise immediately instead. - All blocked callers of put() and get() will be unblocked. If - 'immediate', a task is marked as done for each item remaining in - the queue, which may unblock callers of join(). + All blocked callers of put() and get() will be unblocked. + + If 'immediate', the queue is drained and unfinished tasks + is reduced by the number of drained tasks. If unfinished tasks + is reduced to zero, callers of Queue.join are unblocked. """ self._is_shutdown = True if immediate: diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index ba37e003a655c0a..774a317564df22c 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -35,7 +35,8 @@ class Runner: with asyncio.Runner(debug=True) as runner: runner.run(main()) - The run() method can be called multiple times within the runner's context. + The run() method can be called multiple times within the runner's + context. This can be useful for interactive console (e.g. IPython), unittest runners, console tools, -- everywhere when async code diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 22147451fa7ebd2..5c07f5d31c5a6ae 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -173,7 +173,7 @@ def _accept_connection( # listening socket has triggered an EVENT_READ. There may be multiple # connections waiting for an .accept() so it is called in a loop. # See https://bugs.python.org/issue27906 for more details. - for _ in range(backlog): + for _ in range(backlog + 1): try: conn, addr = sock.accept() if self._debug: @@ -530,11 +530,12 @@ def _sock_recvfrom_into(self, fut, sock, buf, bufsize): async def sock_sendall(self, sock, data): """Send data to the socket. - The socket must be connected to a remote socket. This method continues - to send data from data until either all data has been sent or an - error occurs. None is returned on success. On error, an exception is - raised, and there is no way to determine how much data, if any, was - successfully processed by the receiving end of the connection. + The socket must be connected to a remote socket. This method + continues to send data from data until either all data has been + sent or an error occurs. None is returned on success. On error, + an exception is raised, and there is no way to determine how much + data, if any, was successfully processed by the receiving end of + the connection. """ base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: @@ -583,11 +584,12 @@ def _sock_sendall(self, fut, sock, view, pos): async def sock_sendto(self, sock, data, address): """Send data to the socket. - The socket must be connected to a remote socket. This method continues - to send data from data until either all data has been sent or an - error occurs. None is returned on success. On error, an exception is - raised, and there is no way to determine how much data, if any, was - successfully processed by the receiving end of the connection. + The socket must be connected to a remote socket. This method + continues to send data from data until either all data has been + sent or an error occurs. None is returned on success. On error, + an exception is raised, and there is no way to determine how much + data, if any, was successfully processed by the receiving end of + the connection. """ base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: @@ -698,10 +700,11 @@ def _sock_connect_cb(self, fut, sock, address): async def sock_accept(self, sock): """Accept a connection. - The socket must be bound to an address and listening for connections. - The return value is a pair (conn, address) where conn is a new socket - object usable to send and receive data on the connection, and address - is the address bound to the socket on the other end of the connection. + The socket must be bound to an address and listening for + connections. The return value is a pair (conn, address) where + conn is a new socket object usable to send and receive data on the + connection, and address is the address bound to the socket on the + other end of the connection. """ base_events._check_ssl_socket(sock) if self._debug and sock.gettimeout() != 0: @@ -711,6 +714,9 @@ async def sock_accept(self, sock): return await fut def _sock_accept(self, fut, sock): + # gh-153761: _sock_accept must not scheduled with already cancelled future + if fut.done(): + return fd = sock.fileno() try: conn, address = sock.accept() @@ -1050,8 +1056,8 @@ def _read_ready__on_eof(self): def write(self, data): if not isinstance(data, (bytes, bytearray, memoryview)): - raise TypeError(f'data argument must be a bytes-like object, ' - f'not {type(data).__name__!r}') + raise TypeError(f'data argument must be a bytes, bytearray, or memoryview ' + f'object, not {type(data).__name__!r}') if self._eof: raise RuntimeError('Cannot call write() after write_eof()') if self._empty_waiter is not None: @@ -1174,6 +1180,13 @@ def writelines(self, list_of_data): raise RuntimeError('unable to writelines; sendfile is in progress') if not list_of_data: return + + if self._conn_lost: + if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES: + logger.warning('socket.send() raised exception.') + self._conn_lost += 1 + return + self._buffer.extend([memoryview(data) for data in list_of_data]) self._write_ready() # If the entire buffer couldn't be written, register a write handler @@ -1212,6 +1225,7 @@ def close(self): class _SelectorDatagramTransport(_SelectorTransport, transports.DatagramTransport): _buffer_factory = collections.deque + _header_size = 8 def __init__(self, loop, sock, protocol, address=None, waiter=None, extra=None): @@ -1285,13 +1299,13 @@ def sendto(self, data, addr=None): # Ensure that what we buffer is immutable. self._buffer.append((bytes(data), addr)) - self._buffer_size += len(data) + 8 # include header bytes + self._buffer_size += len(data) + self._header_size self._maybe_pause_protocol() def _sendto_ready(self): while self._buffer: data, addr = self._buffer.popleft() - self._buffer_size -= len(data) + self._buffer_size -= len(data) + self._header_size try: if self._extra['peername']: self._sock.send(data) @@ -1299,7 +1313,7 @@ def _sendto_ready(self): self._sock.sendto(data, addr) except (BlockingIOError, InterruptedError): self._buffer.appendleft((data, addr)) # Try again later. - self._buffer_size += len(data) + self._buffer_size += len(data) + self._header_size break except OSError as exc: self._protocol.error_received(exc) diff --git a/Lib/asyncio/streams.py b/Lib/asyncio/streams.py index 64aac4cc50d15a5..dd8f661862383fa 100644 --- a/Lib/asyncio/streams.py +++ b/Lib/asyncio/streams.py @@ -541,17 +541,17 @@ async def _wait_for_data(self, func_name): self._waiter = None async def readline(self): - """Read chunk of data from the stream until newline (b'\n') is found. + r"""Read chunk of data from the stream until newline (b'\n') is found. - On success, return chunk that ends with newline. If only partial + On success, return chunk that ends with newline. If only partial line can be read due to EOF, return incomplete line without - terminating newline. When EOF was reached while no bytes read, empty - bytes object is returned. + terminating newline. When EOF was reached while no bytes read, + empty bytes object is returned. - If limit is reached, ValueError will be raised. In that case, if + If limit is reached, ValueError will be raised. In that case, if newline was found, complete line including newline will be removed - from internal buffer. Else, internal buffer will be cleared. Limit is - compared against part of the line without newline. + from internal buffer. Else, internal buffer will be cleared. + Limit is compared against part of the line without newline. If stream was paused, this function will automatically resume it if needed. diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 888615f8e5e1b34..9d20930dc300c67 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -524,6 +524,7 @@ def _on_completion(f): timeout_handle.cancel() for f in fs: f.remove_done_callback(_on_completion) + futures.future_discard_from_awaited_by(f, cur_task) done, pending = set(), set() for f in fs: @@ -624,10 +625,11 @@ def as_completed(fs, *, timeout=None): Run the supplied awaitables concurrently. The returned object can be iterated to obtain the results of the awaitables as they finish. - The object returned can be iterated as an asynchronous iterator or a plain - iterator. When asynchronous iteration is used, the originally-supplied - awaitables are yielded if they are tasks or futures. This makes it easy to - correlate previously-scheduled tasks with their results: + The object returned can be iterated as an asynchronous iterator or + a plain iterator. When asynchronous iteration is used, the + originally-supplied awaitables are yielded if they are tasks or + futures. This makes it easy to correlate previously-scheduled tasks + with their results: ipv4_connect = create_task(open_connection("127.0.0.1", 80)) ipv6_connect = create_task(open_connection("::1", 80)) @@ -643,26 +645,27 @@ def as_completed(fs, *, timeout=None): else: print("IPv4 connection established.") - During asynchronous iteration, implicitly-created tasks will be yielded for - supplied awaitables that aren't tasks or futures. + During asynchronous iteration, implicitly-created tasks will be + yielded for supplied awaitables that aren't tasks or futures. - When used as a plain iterator, each iteration yields a new coroutine that - returns the result or raises the exception of the next completed awaitable. - This pattern is compatible with Python versions older than 3.13: + When used as a plain iterator, each iteration yields a new coroutine + that returns the result or raises the exception of the next completed + awaitable. This pattern is compatible with Python versions older than + 3.13: ipv4_connect = create_task(open_connection("127.0.0.1", 80)) ipv6_connect = create_task(open_connection("::1", 80)) tasks = [ipv4_connect, ipv6_connect] for next_connect in as_completed(tasks): - # next_connect is not one of the original task objects. It must be - # awaited to obtain the result value or raise the exception of the - # awaitable that finishes next. + # next_connect is not one of the original task objects. It must + # be awaited to obtain the result value or raise the exception + # of the awaitable that finishes next. reader, writer = await next_connect - A TimeoutError is raised if the timeout occurs before all awaitables are - done. This is raised by the async for loop during asynchronous iteration or - by the coroutines yielded during plain iteration. + A TimeoutError is raised if the timeout occurs before all awaitables + are done. This is raised by the async for loop during asynchronous + iteration or by the coroutines yielded during plain iteration. """ if inspect.isawaitable(fs): raise TypeError( @@ -908,6 +911,25 @@ def _done_callback(fut, cur_task=cur_task): return outer +def _log_on_exception(fut): + if fut.cancelled(): + return + + exc = fut.exception() + if exc is None: + return + + context = { + 'message': + f'{exc.__class__.__name__} exception in shielded future', + 'exception': exc, + 'future': fut, + } + if fut._source_traceback: + context['source_traceback'] = fut._source_traceback + fut._loop.call_exception_handler(context) + + def shield(arg): """Wait for a future, shielding it from cancellation. @@ -953,14 +975,11 @@ def shield(arg): else: cur_task = None - def _inner_done_callback(inner, cur_task=cur_task): - if cur_task is not None: - futures.future_discard_from_awaited_by(inner, cur_task) + def _clear_awaited_by_callback(inner): + futures.future_discard_from_awaited_by(inner, cur_task) + def _inner_done_callback(inner): if outer.cancelled(): - if not inner.cancelled(): - # Mark inner's result as retrieved. - inner.exception() return if inner.cancelled(): @@ -972,10 +991,16 @@ def _inner_done_callback(inner, cur_task=cur_task): else: outer.set_result(inner.result()) - def _outer_done_callback(outer): if not inner.done(): inner.remove_done_callback(_inner_done_callback) + # Keep only one callback to log on cancel + inner.remove_done_callback(_log_on_exception) + inner.add_done_callback(_log_on_exception) + + if cur_task is not None: + inner.add_done_callback(_clear_awaited_by_callback) + inner.add_done_callback(_inner_done_callback) outer.add_done_callback(_outer_done_callback) @@ -1008,21 +1033,22 @@ def callback(): def create_eager_task_factory(custom_task_constructor): """Create a function suitable for use as a task factory on an event-loop. - Example usage: + Example usage: - loop.set_task_factory( - asyncio.create_eager_task_factory(my_task_constructor)) + loop.set_task_factory( + asyncio.create_eager_task_factory(my_task_constructor)) - Now, tasks created will be started immediately (rather than being first - scheduled to an event loop). The constructor argument can be any callable - that returns a Task-compatible object and has a signature compatible - with `Task.__init__`; it must have the `eager_start` keyword argument. + Now, tasks created will be started immediately (rather than being first + scheduled to an event loop). The constructor argument can be any + callable that returns a Task-compatible object and has a signature + compatible with `Task.__init__`; it must have the `eager_start` + keyword argument. - Most applications will use `Task` for `custom_task_constructor` and in - this case there's no need to call `create_eager_task_factory()` - directly. Instead the global `eager_task_factory` instance can be - used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. - """ + Most applications will use `Task` for `custom_task_constructor` and in + this case there's no need to call `create_eager_task_factory()` + directly. Instead the global `eager_task_factory` instance can be + used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`. + """ def factory(loop, coro, *, eager_start=True, **kwargs): return custom_task_constructor( diff --git a/Lib/asyncio/threads.py b/Lib/asyncio/threads.py index db048a8231de166..5001351b0976ecd 100644 --- a/Lib/asyncio/threads.py +++ b/Lib/asyncio/threads.py @@ -17,7 +17,8 @@ async def to_thread(func, /, *args, **kwargs): allowing context variables from the main thread to be accessed in the separate thread. - Return a coroutine that can be awaited to get the eventual result of *func*. + Return a coroutine that can be awaited to get the eventual result of + *func*. """ loop = events.get_running_loop() ctx = contextvars.copy_context() diff --git a/Lib/asyncio/timeouts.py b/Lib/asyncio/timeouts.py index 09342dc7c1310b0..65ddc285abd971d 100644 --- a/Lib/asyncio/timeouts.py +++ b/Lib/asyncio/timeouts.py @@ -25,7 +25,8 @@ class _State(enum.Enum): class Timeout: """Asynchronous context manager for cancelling overdue coroutines. - Use `timeout()` or `timeout_at()` rather than instantiating this class directly. + Use `timeout()` or `timeout_at()` rather than instantiating this class + directly. """ def __init__(self, when: float | None) -> None: diff --git a/Lib/asyncio/tools.py b/Lib/asyncio/tools.py index bf1cb5e64cbfbe3..f39e11fdd513b4d 100644 --- a/Lib/asyncio/tools.py +++ b/Lib/asyncio/tools.py @@ -1,71 +1,99 @@ """Tools to analyze tasks running in asyncio programs.""" -from dataclasses import dataclass -from collections import defaultdict +from collections import defaultdict, namedtuple from itertools import count from enum import Enum import sys -from _remote_debugging import get_all_awaited_by - +from _remote_debugging import RemoteUnwinder, FrameInfo class NodeType(Enum): COROUTINE = 1 TASK = 2 -@dataclass(frozen=True) class CycleFoundException(Exception): """Raised when there is a cycle when drawing the call tree.""" - cycles: list[list[int]] - id2name: dict[int, str] + def __init__( + self, + cycles: list[list[int]], + id2name: dict[int, str], + ) -> None: + super().__init__(cycles, id2name) + self.cycles = cycles + self.id2name = id2name -# ─── indexing helpers ─────────────────────────────────────────── -def _format_stack_entry(elem: tuple[str, str, int] | str) -> str: - if isinstance(elem, tuple): - fqname, path, line_no = elem - return f"{fqname} {path}:{line_no}" +# ─── indexing helpers ─────────────────────────────────────────── +def _format_stack_entry(elem: str|FrameInfo) -> str: + if not isinstance(elem, str): + if elem.lineno == 0 and elem.filename == "": + return f"{elem.funcname}" + else: + return f"{elem.funcname} {elem.filename}:{elem.lineno}" return elem def _index(result): - id2name, awaits = {}, [] - for _thr_id, tasks in result: - for tid, tname, awaited in tasks: - id2name[tid] = tname - for stack, parent_id in awaited: - stack = [_format_stack_entry(elem) for elem in stack] - awaits.append((parent_id, stack, tid)) - return id2name, awaits - - -def _build_tree(id2name, awaits): + id2name, awaits, task_stacks = {}, [], {} + for awaited_info in result: + for task_info in awaited_info.awaited_by: + task_id = task_info.task_id + task_name = task_info.task_name + id2name[task_id] = task_name + + # Store the internal coroutine stack for this task + if task_info.coroutine_stack: + for coro_info in task_info.coroutine_stack: + call_stack = coro_info.call_stack + internal_stack = [_format_stack_entry(frame) for frame in call_stack] + task_stacks[task_id] = internal_stack + + # Add the awaited_by relationships (external dependencies) + if task_info.awaited_by: + for coro_info in task_info.awaited_by: + call_stack = coro_info.call_stack + parent_task_id = coro_info.task_name + stack = [_format_stack_entry(frame) for frame in call_stack] + awaits.append((parent_task_id, stack, task_id)) + return id2name, awaits, task_stacks + + +def _build_tree(id2name, awaits, task_stacks): id2label = {(NodeType.TASK, tid): name for tid, name in id2name.items()} children = defaultdict(list) - cor_names = defaultdict(dict) # (parent) -> {frame: node} - cor_id_seq = count(1) - - def _cor_node(parent_key, frame_name): - """Return an existing or new (NodeType.COROUTINE, …) node under *parent_key*.""" - bucket = cor_names[parent_key] - if frame_name in bucket: - return bucket[frame_name] - node_key = (NodeType.COROUTINE, f"c{next(cor_id_seq)}") - id2label[node_key] = frame_name - children[parent_key].append(node_key) - bucket[frame_name] = node_key + cor_nodes = defaultdict(dict) # Maps parent -> {frame_name: node_key} + next_cor_id = count(1) + + def get_or_create_cor_node(parent, frame): + """Get existing coroutine node or create new one under parent""" + if frame in cor_nodes[parent]: + return cor_nodes[parent][frame] + + node_key = (NodeType.COROUTINE, f"c{next(next_cor_id)}") + id2label[node_key] = frame + children[parent].append(node_key) + cor_nodes[parent][frame] = node_key return node_key - # lay down parent ➜ …frames… ➜ child paths + # Build task dependency tree with coroutine frames for parent_id, stack, child_id in awaits: cur = (NodeType.TASK, parent_id) - for frame in reversed(stack): # outer-most → inner-most - cur = _cor_node(cur, frame) + for frame in reversed(stack): + cur = get_or_create_cor_node(cur, frame) + child_key = (NodeType.TASK, child_id) if child_key not in children[cur]: children[cur].append(child_key) + # Add coroutine stacks for leaf tasks + awaiting_tasks = {parent_id for parent_id, _, _ in awaits} + for task_id in id2name: + if task_id not in awaiting_tasks and task_id in task_stacks: + cur = (NodeType.TASK, task_id) + for frame in reversed(task_stacks[task_id]): + cur = get_or_create_cor_node(cur, frame) + return id2label, children @@ -112,6 +140,11 @@ def dfs(v): # ─── PRINT TREE FUNCTION ─────────────────────────────────────── +def get_all_awaited_by(pid): + unwinder = RemoteUnwinder(pid) + return unwinder.get_all_awaited_by() + + def build_async_tree(result, task_emoji="(T)", cor_emoji=""): """ Build a list of strings for pretty-print an async call tree. @@ -119,12 +152,12 @@ def build_async_tree(result, task_emoji="(T)", cor_emoji=""): The call tree is produced by `get_all_async_stacks()`, prefixing tasks with `task_emoji` and coroutine frames with `cor_emoji`. """ - id2name, awaits = _index(result) + id2name, awaits, task_stacks = _index(result) g = _task_graph(awaits) cycles = _find_cycles(g) if cycles: raise CycleFoundException(cycles, id2name) - labels, children = _build_tree(id2name, awaits) + labels, children = _build_tree(id2name, awaits, task_stacks) def pretty(node): flag = task_emoji if node[0] == NodeType.TASK else cor_emoji @@ -144,35 +177,40 @@ def render(node, prefix="", last=True, buf=None): def build_task_table(result): - id2name, awaits = _index(result) + id2name, _, _ = _index(result) table = [] - for tid, tasks in result: - for task_id, task_name, awaited in tasks: - if not awaited: - table.append( - [ - tid, - hex(task_id), - task_name, - "", - "", - "0x0" - ] - ) - for stack, awaiter_id in awaited: - stack = [elem[0] if isinstance(elem, tuple) else elem for elem in stack] - coroutine_chain = " -> ".join(stack) - awaiter_name = id2name.get(awaiter_id, "Unknown") - table.append( - [ - tid, - hex(task_id), - task_name, - coroutine_chain, - awaiter_name, - hex(awaiter_id), - ] - ) + + for awaited_info in result: + thread_id = awaited_info.thread_id + for task_info in awaited_info.awaited_by: + # Get task info + task_id = task_info.task_id + task_name = task_info.task_name + + # Build coroutine stack string + frames = [frame for coro in task_info.coroutine_stack + for frame in coro.call_stack] + coro_stack = " -> ".join(_format_stack_entry(x).split(" ")[0] + for x in frames) + + # Handle tasks with no awaiters + if not task_info.awaited_by: + table.append([thread_id, hex(task_id), task_name, coro_stack, + "", "", "0x0"]) + continue + + # Handle tasks with awaiters + for coro_info in task_info.awaited_by: + parent_id = coro_info.task_name + awaiter_frames = [_format_stack_entry(x).split(" ")[0] + for x in coro_info.call_stack] + awaiter_chain = " -> ".join(awaiter_frames) + awaiter_name = id2name.get(parent_id, "Unknown") + parent_id_str = (hex(parent_id) if isinstance(parent_id, int) + else str(parent_id)) + + table.append([thread_id, hex(task_id), task_name, coro_stack, + awaiter_chain, awaiter_name, parent_id_str]) return table @@ -184,6 +222,20 @@ def _print_cycle_exception(exception: CycleFoundException): print(f"cycle: {inames}", file=sys.stderr) +def exit_with_permission_help_text(): + """ + Prints a message pointing to platform-specific permission help text and exits the program. + This function is called when a PermissionError is encountered while trying + to attach to a process. + """ + print( + "Error: The specified process cannot be attached to due to insufficient permissions.\n" + "See the Python documentation for details on required privileges and troubleshooting:\n" + "https://docs.python.org/3.14/howto/remote_debugging.html#permission-requirements\n" + ) + sys.exit(1) + + def _get_awaited_by_tasks(pid: int) -> list: try: return get_all_awaited_by(pid) @@ -192,6 +244,8 @@ def _get_awaited_by_tasks(pid: int) -> list: e = e.__context__ print(f"Error retrieving tasks: {e}") sys.exit(1) + except PermissionError as e: + exit_with_permission_help_text() def display_awaited_by_tasks_table(pid: int) -> None: @@ -201,11 +255,11 @@ def display_awaited_by_tasks_table(pid: int) -> None: table = build_task_table(tasks) # Print the table in a simple tabular format print( - f"{'tid':<10} {'task id':<20} {'task name':<20} {'coroutine chain':<50} {'awaiter name':<20} {'awaiter id':<15}" + f"{'tid':<10} {'task id':<20} {'task name':<20} {'coroutine stack':<50} {'awaiter chain':<50} {'awaiter name':<15} {'awaiter id':<15}" ) - print("-" * 135) + print("-" * 180) for row in table: - print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<20} {row[5]:<15}") + print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<50} {row[5]:<15} {row[6]:<15}") def display_awaited_by_tasks_tree(pid: int) -> None: diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index f69c6a64c39ae6e..36de51970c6bd5b 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -28,7 +28,6 @@ __all__ = ( 'SelectorEventLoop', - '_DefaultEventLoopPolicy', 'EventLoop', ) @@ -55,7 +54,8 @@ def waitstatus_to_exitcode(status): class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop): """Unix event loop. - Adds signal handling and UNIX Domain Socket support to SelectorEventLoop. + Adds signal handling and UNIX Domain Socket support to + SelectorEventLoop. """ def __init__(self, selector=None): @@ -212,13 +212,13 @@ async def _make_subprocess_transport(self, protocol, args, shell, raise except BaseException: transp.close() - await transp._wait() + await tasks.shield(transp._wait()) raise return transp def _child_watcher_callback(self, pid, returncode, transp): - self.call_soon_threadsafe(transp._process_exited, returncode) + transp._process_exited(returncode) async def create_unix_connection( self, protocol_factory, path=None, *, @@ -385,12 +385,12 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, # order to simplify the common case. self.remove_writer(registered_fd) if fut.cancelled(): - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) return if count: blocksize = count - total_sent if blocksize <= 0: - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_result(total_sent) return @@ -424,20 +424,20 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, # plain send(). err = exceptions.SendfileNotAvailableError( "os.sendfile call failed") - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_exception(err) else: - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_exception(exc) except (SystemExit, KeyboardInterrupt): raise except BaseException as exc: - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_exception(exc) else: if sent == 0: # EOF - self._sock_sendfile_update_filepos(fileno, offset, total_sent) + self._sock_sendfile_update_filepos(fileno, offset) fut.set_result(total_sent) else: offset += sent @@ -448,9 +448,9 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno, fd, sock, fileno, offset, count, blocksize, total_sent) - def _sock_sendfile_update_filepos(self, fileno, offset, total_sent): - if total_sent > 0: - os.lseek(fileno, offset, os.SEEK_SET) + def _sock_sendfile_update_filepos(self, fileno, offset): + # After this helper runs, the source fd's lseek pointer is at offset." + os.lseek(fileno, offset, os.SEEK_SET) def _sock_add_cancellation_callback(self, fut, sock): def cb(fut): @@ -928,6 +928,49 @@ def add_child_handler(self, pid, callback, *args): def _do_waitpid(self, loop, expected_pid, callback, args): assert expected_pid > 0 + if hasattr(os, 'waitid'): + # Wait for the child process using waitid() on platforms which support it. + # WNOWAIT is used to avoid reaping the child process, allowing the event loop to + # reap the child process with waitpid() later in event loop thread. + # This makes the reaping of the child and notification of the return code + # atomic with respect to the event loop thread. + try: + os.waitid(os.P_PID, expected_pid, os.WEXITED | os.WNOWAIT) + except ChildProcessError: + # The child process is already reaped + pass + if loop.is_closed(): + # loop is already closed, reap the zombie here so that it is not leaked. + pid, _ = self._reap(loop, expected_pid) + logger.warning("Loop %r that handles pid %r is closed", + loop, pid) + else: + try: + loop.call_soon_threadsafe( + self._reap_and_notify, loop, expected_pid, + callback, args) + except RuntimeError: + # The event loop was closed concurrently. + pid, _ = self._reap(loop, expected_pid) + logger.warning("Loop %r that handles pid %r is closed", + loop, pid) + else: + # Fallback for platforms that don't support waitid(): we have to + # reap the child here, which is racy with respect to send_signal() + pid, returncode = self._reap(loop, expected_pid) + if loop.is_closed(): + logger.warning("Loop %r that handles pid %r is closed", + loop, pid) + else: + loop.call_soon_threadsafe(callback, pid, returncode, *args) + + self._threads.pop(expected_pid) + + def _reap_and_notify(self, loop, expected_pid, callback, args): + pid, returncode = self._reap(loop, expected_pid) + callback(pid, returncode, *args) + + def _reap(self, loop, expected_pid): try: pid, status = os.waitpid(expected_pid, 0) except ChildProcessError: @@ -943,13 +986,7 @@ def _do_waitpid(self, loop, expected_pid, callback, args): if loop.get_debug(): logger.debug('process %s exited with returncode %s', expected_pid, returncode) - - if loop.is_closed(): - logger.warning("Loop %r that handles pid %r is closed", loop, pid) - else: - loop.call_soon_threadsafe(callback, pid, returncode, *args) - - self._threads.pop(expected_pid) + return pid, returncode def can_use_pidfd(): if not hasattr(os, 'pidfd_open'): diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index 5f75b17d8ca649b..c905db1016807f9 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -408,7 +408,7 @@ async def _make_subprocess_transport(self, protocol, args, shell, raise except BaseException: transp.close() - await transp._wait() + await tasks.shield(transp._wait()) raise return transp @@ -610,6 +610,9 @@ def sendfile(self, sock, file, offset, count): ov = _overlapped.Overlapped(NULL) offset_low = offset & 0xffff_ffff offset_high = (offset >> 32) & 0xffff_ffff + # TransmitFile ignores OVERLAPPED.Offset for handles not opened with + # FILE_FLAG_OVERLAPPED, so seek the CRT file pointer to match. + file.seek(offset) ov.TransmitFile(sock.fileno(), msvcrt.get_osfhandle(file.fileno()), offset_low, offset_high, diff --git a/Lib/asyncio/windows_utils.py b/Lib/asyncio/windows_utils.py index ef277fac3e291c3..d6393f0b1ffee5d 100644 --- a/Lib/asyncio/windows_utils.py +++ b/Lib/asyncio/windows_utils.py @@ -10,7 +10,6 @@ import msvcrt import os import subprocess -import tempfile import warnings @@ -24,6 +23,7 @@ PIPE = subprocess.PIPE STDOUT = subprocess.STDOUT _mmap_counter = itertools.count() +_MAX_PIPE_ATTEMPTS = 20 # Replacement for os.pipe() using handles instead of fds @@ -31,10 +31,6 @@ def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE): """Like os.pipe() but with overlapped support and using handles not fds.""" - address = tempfile.mktemp( - prefix=r'\\.\pipe\python-pipe-{:d}-{:d}-'.format( - os.getpid(), next(_mmap_counter))) - if duplex: openmode = _winapi.PIPE_ACCESS_DUPLEX access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE @@ -56,9 +52,20 @@ def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE): h1 = h2 = None try: - h1 = _winapi.CreateNamedPipe( - address, openmode, _winapi.PIPE_WAIT, - 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL) + for attempts in itertools.count(): + address = r'\\.\pipe\python-pipe-{:d}-{:d}-{}'.format( + os.getpid(), next(_mmap_counter), os.urandom(8).hex()) + try: + h1 = _winapi.CreateNamedPipe( + address, openmode, _winapi.PIPE_WAIT, + 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL) + break + except OSError as e: + if attempts >= _MAX_PIPE_ATTEMPTS: + raise + if e.winerror not in (_winapi.ERROR_PIPE_BUSY, + _winapi.ERROR_ACCESS_DENIED): + raise h2 = _winapi.CreateFile( address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING, @@ -104,8 +111,9 @@ def fileno(self): def close(self, *, CloseHandle=_winapi.CloseHandle): if self._handle is not None: - CloseHandle(self._handle) + handle = self._handle self._handle = None + CloseHandle(handle) def __del__(self, _warn=warnings.warn): if self._handle is not None: diff --git a/Lib/base64.py b/Lib/base64.py index 5d78cc09f40cd3e..0f84420d7a20629 100644 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -1,4 +1,4 @@ -"""Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings""" +"""Base16, Base32, Base64 (RFC 4648), Base85 and Ascii85 data encodings""" # Modified 04-Oct-1995 by Jack Jansen to use binascii module # Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support @@ -147,7 +147,7 @@ def urlsafe_b64decode(s): characters present in the input. ''' _B32_DECODE_MAP01_DOCSTRING = ''' -RFC 3548 allows for optional mapping of the digit 0 (zero) to the +RFC 4648 allows for optional mapping of the digit 0 (zero) to the letter O (oh), and for optional mapping of the digit 1 (one) to either the letter I (eye) or letter L (el). The optional argument map01 when not None, specifies which letter the digit 1 should be @@ -261,7 +261,7 @@ def b32hexdecode(s, casefold=False): extra_args='') -# RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns +# RFC 4648, Base 16 Alphabet specifies uppercase, but hexlify() returns # lowercase. The RFC also recommends against accepting input case # insensitively. def b16encode(s): @@ -325,17 +325,20 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): foldspaces is an optional flag that uses the special short sequence 'y' instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This - feature is not supported by the "standard" Adobe encoding. + feature is not supported by the standard encoding used in PDF. - wrapcol controls whether the output should have newline (b'\\n') characters - added to it. If this is non-zero, each output line will be at most this - many characters long, excluding the trailing newline. + If wrapcol is non-zero, insert a newline (b'\\n') character after at most + every wrapcol characters. - pad controls whether the input is padded to a multiple of 4 before - encoding. Note that the btoa implementation always pads. + pad controls whether zero-padding applied to the end of the input + is fully retained in the output encoding, as done by btoa, + producing an exact multiple of 5 bytes of output. + + adobe controls whether the encoded byte sequence is framed with <~ + and ~>, as in a PostScript base-85 string literal. Note that + while ASCII85Decode streams in PDF documents must be terminated + with ~>, they must not use a leading <~. - adobe controls whether the encoded byte sequence is framed with <~ and ~>, - which is used by the Adobe implementation. """ global _a85chars, _a85chars2 # Delay the initialization of tables to not waste memory @@ -364,12 +367,14 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'): """Decode the Ascii85 encoded bytes-like object or ASCII string b. - foldspaces is a flag that specifies whether the 'y' short sequence should be - accepted as shorthand for 4 consecutive spaces (ASCII 0x20). This feature is - not supported by the "standard" Adobe encoding. + foldspaces is a flag that specifies whether the 'y' short sequence + should be accepted as shorthand for 4 consecutive spaces (ASCII + 0x20). This feature is not supported by the standard Ascii85 + encoding used in PDF and PostScript. - adobe controls whether the input sequence is in Adobe Ascii85 format (i.e. - is framed with <~ and ~>). + adobe controls whether the <~ and ~> markers are present. While + the leading <~ is not required, the input must end with ~>, or a + ValueError is raised. ignorechars should be a byte string containing characters to ignore from the input. This should only contain whitespace characters, and by default @@ -442,8 +447,10 @@ def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'): def b85encode(b, pad=False): """Encode bytes-like object b in base85 format and return a bytes object. - If pad is true, the input is padded with b'\\0' so its length is a multiple of - 4 bytes before encoding. + The input is padded with b'\0' so its length is a multiple of 4 + bytes before encoding. If pad is true, all the resulting + characters are retained in the output, which will always be a + multiple of 5 bytes. """ global _b85chars, _b85chars2 # Delay the initialization of tables to not waste memory @@ -462,9 +469,12 @@ def b85decode(b): # Delay the initialization of tables to not waste memory # if the function is never called if _b85dec is None: - _b85dec = [None] * 256 + # we don't assign to _b85dec directly to avoid issues when + # multiple threads call this function simultaneously + b85dec_tmp = [None] * 256 for i, c in enumerate(_b85alphabet): - _b85dec[c] = i + b85dec_tmp[c] = i + _b85dec = b85dec_tmp b = _bytes_from_decode_data(b) padding = (-len(b)) % 5 @@ -601,7 +611,14 @@ def main(): with open(args[0], 'rb') as f: func(f, sys.stdout.buffer) else: - func(sys.stdin.buffer, sys.stdout.buffer) + if sys.stdin.isatty(): + # gh-138775: read terminal input data all at once to detect EOF + import io + data = sys.stdin.buffer.read() + buffer = io.BytesIO(data) + else: + buffer = sys.stdin.buffer + func(buffer, sys.stdout.buffer) if __name__ == '__main__': diff --git a/Lib/bdb.py b/Lib/bdb.py index 4290ef22302a426..79da4bab9c90347 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -199,6 +199,8 @@ def __init__(self, skip=None, backend='settrace'): self.frame_returning = None self.trace_opcodes = False self.enterframe = None + self.cmdframe = None + self.cmdlineno = None self.code_linenos = weakref.WeakKeyDictionary() self.backend = backend if backend == 'monitoring': @@ -306,7 +308,12 @@ def dispatch_line(self, frame): self.user_line(). Raise BdbQuit if self.quitting is set. Return self.trace_dispatch to continue tracing in this scope. """ - if self.stop_here(frame) or self.break_here(frame): + # GH-136057 + # For line events, we don't want to stop at the same line where + # the latest next/step command was issued. + if (self.stop_here(frame) or self.break_here(frame)) and not ( + self.cmdframe == frame and self.cmdlineno == frame.f_lineno + ): self.user_line(frame) self.restart_events() if self.quitting: raise BdbQuit @@ -535,7 +542,8 @@ def _set_trace_opcodes(self, trace_opcodes): if self.monitoring_tracer: self.monitoring_tracer.update_local_events() - def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): + def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False, + cmdframe=None, cmdlineno=None): """Set the attributes for stopping. If stoplineno is greater than or equal to 0, then stop at line @@ -548,6 +556,10 @@ def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False): # stoplineno >= 0 means: stop at line >= the stoplineno # stoplineno -1 means: don't stop at all self.stoplineno = stoplineno + # cmdframe/cmdlineno is the frame/line number when the user issued + # step/next commands. + self.cmdframe = cmdframe + self.cmdlineno = cmdlineno self._set_trace_opcodes(opcode) def _set_caller_tracefunc(self, current_frame): @@ -573,7 +585,9 @@ def set_until(self, frame, lineno=None): def set_step(self): """Stop after one line of code.""" - self._set_stopinfo(None, None) + # set_step() could be called from signal handler so enterframe might be None + self._set_stopinfo(None, None, cmdframe=self.enterframe, + cmdlineno=getattr(self.enterframe, 'f_lineno', None)) def set_stepinstr(self): """Stop before the next instruction.""" @@ -581,7 +595,7 @@ def set_stepinstr(self): def set_next(self, frame): """Stop on the next line in or below the given frame.""" - self._set_stopinfo(frame, None) + self._set_stopinfo(frame, None, cmdframe=frame, cmdlineno=frame.f_lineno) def set_return(self, frame): """Stop when returning from the given frame.""" diff --git a/Lib/codecs.py b/Lib/codecs.py index fc38e922257644f..e4a8010aba90a50 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -618,7 +618,7 @@ def readlines(self, sizehint=None, keepends=True): method and are included in the list entries. sizehint, if given, is ignored since there is no efficient - way to finding the true end-of-line. + way of finding the true end-of-line. """ data = self.read() @@ -709,13 +709,13 @@ def read(self, size=-1): return self.reader.read(size) - def readline(self, size=None): + def readline(self, size=None, keepends=True): - return self.reader.readline(size) + return self.reader.readline(size, keepends) - def readlines(self, sizehint=None): + def readlines(self, sizehint=None, keepends=True): - return self.reader.readlines(sizehint) + return self.reader.readlines(sizehint, keepends) def __next__(self): diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index d2ddc1cd9ec2ea2..803de0c6792d643 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -1495,6 +1495,8 @@ def format_map(self, mapping): return self.data.format_map(mapping) def index(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data return self.data.index(sub, start, end) def isalpha(self): @@ -1563,6 +1565,8 @@ def rfind(self, sub, start=0, end=_sys.maxsize): return self.data.rfind(sub, start, end) def rindex(self, sub, start=0, end=_sys.maxsize): + if isinstance(sub, UserString): + sub = sub.data return self.data.rindex(sub, start, end) def rjust(self, width, *args): diff --git a/Lib/compression/bz2/__init__.py b/Lib/compression/bz2.py similarity index 100% rename from Lib/compression/bz2/__init__.py rename to Lib/compression/bz2.py diff --git a/Lib/compression/gzip/__init__.py b/Lib/compression/gzip.py similarity index 100% rename from Lib/compression/gzip/__init__.py rename to Lib/compression/gzip.py diff --git a/Lib/compression/lzma/__init__.py b/Lib/compression/lzma.py similarity index 100% rename from Lib/compression/lzma/__init__.py rename to Lib/compression/lzma.py diff --git a/Lib/compression/zlib/__init__.py b/Lib/compression/zlib.py similarity index 100% rename from Lib/compression/zlib/__init__.py rename to Lib/compression/zlib.py diff --git a/Lib/compression/zstd/__init__.py b/Lib/compression/zstd/__init__.py index 4f734eb07b00e3e..5326cf9b19cf883 100644 --- a/Lib/compression/zstd/__init__.py +++ b/Lib/compression/zstd/__init__.py @@ -2,41 +2,48 @@ __all__ = ( # compression.zstd - "COMPRESSION_LEVEL_DEFAULT", - "compress", - "CompressionParameter", - "decompress", - "DecompressionParameter", - "finalize_dict", - "get_frame_info", - "Strategy", - "train_dict", + 'COMPRESSION_LEVEL_DEFAULT', + 'compress', + 'CompressionParameter', + 'decompress', + 'DecompressionParameter', + 'finalize_dict', + 'get_frame_info', + 'Strategy', + 'train_dict', # compression.zstd._zstdfile - "open", - "ZstdFile", + 'open', + 'ZstdFile', # _zstd - "get_frame_size", - "zstd_version", - "zstd_version_info", - "ZstdCompressor", - "ZstdDecompressor", - "ZstdDict", - "ZstdError", + 'get_frame_size', + 'zstd_version', + 'zstd_version_info', + 'ZstdCompressor', + 'ZstdDecompressor', + 'ZstdDict', + 'ZstdError', ) import _zstd import enum -from _zstd import * +from _zstd import (ZstdCompressor, ZstdDecompressor, ZstdDict, ZstdError, + get_frame_size, zstd_version) from compression.zstd._zstdfile import ZstdFile, open, _nbytes -COMPRESSION_LEVEL_DEFAULT = _zstd._compressionLevel_values[0] +# zstd_version_number is (MAJOR * 100 * 100 + MINOR * 100 + RELEASE) +zstd_version_info = (*divmod(_zstd.zstd_version_number // 100, 100), + _zstd.zstd_version_number % 100) +"""Version number of the runtime zstd library as a tuple of integers.""" + +COMPRESSION_LEVEL_DEFAULT = _zstd.ZSTD_CLEVEL_DEFAULT """The default compression level for Zstandard, currently '3'.""" class FrameInfo: """Information about a Zstandard frame.""" + __slots__ = 'decompressed_size', 'dictionary_id' def __init__(self, decompressed_size, dictionary_id): @@ -54,8 +61,9 @@ def __setattr__(self, name, _): def get_frame_info(frame_buffer): """Get Zstandard frame information from a frame header. - *frame_buffer* is a bytes-like object. It should start from the beginning - of a frame, and needs to include at least the frame header (6 to 18 bytes). + *frame_buffer* is a bytes-like object. It should start from the + beginning of a frame, and needs to include at least the frame header + (6 to 18 bytes). The returned FrameInfo object has two attributes. 'decompressed_size' is the size in bytes of the data in the frame when @@ -65,7 +73,7 @@ def get_frame_info(frame_buffer): the frame may or may not need a dictionary to be decoded, and the ID of such a dictionary is not specified. """ - return FrameInfo(*_zstd._get_frame_info(frame_buffer)) + return FrameInfo(*_zstd.get_frame_info(frame_buffer)) def train_dict(samples, dict_size): @@ -85,7 +93,7 @@ def train_dict(samples, dict_size): chunk_sizes = tuple(_nbytes(sample) for sample in samples) if not chunks: raise ValueError("samples contained no data; can't train dictionary.") - dict_content = _zstd._train_dict(chunks, chunk_sizes, dict_size) + dict_content = _zstd.train_dict(chunks, chunk_sizes, dict_size) return ZstdDict(dict_content) @@ -96,16 +104,17 @@ def finalize_dict(zstd_dict, /, samples, dict_size, level): finalize *zstd_dict* by adding headers and statistics according to the Zstandard dictionary format. - You may compose an effective dictionary content by hand, which is used as - basis dictionary, and use some samples to finalize a dictionary. The basis - dictionary may be a "raw content" dictionary. See *is_raw* in ZstdDict. + You may compose an effective dictionary content by hand, which is used + as basis dictionary, and use some samples to finalize a dictionary. The + basis dictionary may be a "raw content" dictionary. See *is_raw* in + ZstdDict. - *samples* is an iterable of samples, where a sample is a bytes-like object - representing a file. + *samples* is an iterable of samples, where a sample is a bytes-like + object representing a file. *dict_size* is the dictionary's maximum size, in bytes. *level* is the expected compression level. The statistics for each - compression level differ, so tuning the dictionary to the compression level - can provide improvements. + compression level differ, so tuning the dictionary to the compression + level can provide improvements. """ if not isinstance(zstd_dict, ZstdDict): @@ -119,13 +128,13 @@ def finalize_dict(zstd_dict, /, samples, dict_size, level): chunks = b''.join(samples) chunk_sizes = tuple(_nbytes(sample) for sample in samples) if not chunks: - raise ValueError("The samples are empty content, can't finalize the" + raise ValueError("The samples are empty content, can't finalize the " "dictionary.") - dict_content = _zstd._finalize_dict(zstd_dict.dict_content, - chunks, chunk_sizes, - dict_size, level) + dict_content = _zstd.finalize_dict(zstd_dict.dict_content, chunks, + chunk_sizes, dict_size, level) return ZstdDict(dict_content) + def compress(data, level=None, options=None, zstd_dict=None): """Return Zstandard compressed *data* as bytes. @@ -133,19 +142,20 @@ def compress(data, level=None, options=None, zstd_dict=None): COMPRESSION_LEVEL_DEFAULT ('3'). *options* is a dict object that contains advanced compression parameters. See CompressionParameter for more on options. - *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See - the function train_dict for how to train a ZstdDict on sample data. + *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. + See the function train_dict for how to train a ZstdDict on sample data. For incremental compression, use a ZstdCompressor instead. """ comp = ZstdCompressor(level=level, options=options, zstd_dict=zstd_dict) return comp.compress(data, mode=ZstdCompressor.FLUSH_FRAME) + def decompress(data, zstd_dict=None, options=None): """Decompress one or more frames of Zstandard compressed *data*. - *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See - the function train_dict for how to train a ZstdDict on sample data. + *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. + See the function train_dict for how to train a ZstdDict on sample data. *options* is a dict object that contains advanced compression parameters. See DecompressionParameter for more on options. @@ -156,59 +166,59 @@ def decompress(data, zstd_dict=None, options=None): decomp = ZstdDecompressor(options=options, zstd_dict=zstd_dict) results.append(decomp.decompress(data)) if not decomp.eof: - raise ZstdError("Compressed data ended before the " - "end-of-stream marker was reached") + raise ZstdError('Compressed data ended before the ' + 'end-of-stream marker was reached') data = decomp.unused_data if not data: break - return b"".join(results) + return b''.join(results) class CompressionParameter(enum.IntEnum): """Compression parameters.""" - compression_level = _zstd._ZSTD_c_compressionLevel - window_log = _zstd._ZSTD_c_windowLog - hash_log = _zstd._ZSTD_c_hashLog - chain_log = _zstd._ZSTD_c_chainLog - search_log = _zstd._ZSTD_c_searchLog - min_match = _zstd._ZSTD_c_minMatch - target_length = _zstd._ZSTD_c_targetLength - strategy = _zstd._ZSTD_c_strategy - - enable_long_distance_matching = _zstd._ZSTD_c_enableLongDistanceMatching - ldm_hash_log = _zstd._ZSTD_c_ldmHashLog - ldm_min_match = _zstd._ZSTD_c_ldmMinMatch - ldm_bucket_size_log = _zstd._ZSTD_c_ldmBucketSizeLog - ldm_hash_rate_log = _zstd._ZSTD_c_ldmHashRateLog - - content_size_flag = _zstd._ZSTD_c_contentSizeFlag - checksum_flag = _zstd._ZSTD_c_checksumFlag - dict_id_flag = _zstd._ZSTD_c_dictIDFlag - - nb_workers = _zstd._ZSTD_c_nbWorkers - job_size = _zstd._ZSTD_c_jobSize - overlap_log = _zstd._ZSTD_c_overlapLog + compression_level = _zstd.ZSTD_c_compressionLevel + window_log = _zstd.ZSTD_c_windowLog + hash_log = _zstd.ZSTD_c_hashLog + chain_log = _zstd.ZSTD_c_chainLog + search_log = _zstd.ZSTD_c_searchLog + min_match = _zstd.ZSTD_c_minMatch + target_length = _zstd.ZSTD_c_targetLength + strategy = _zstd.ZSTD_c_strategy + + enable_long_distance_matching = _zstd.ZSTD_c_enableLongDistanceMatching + ldm_hash_log = _zstd.ZSTD_c_ldmHashLog + ldm_min_match = _zstd.ZSTD_c_ldmMinMatch + ldm_bucket_size_log = _zstd.ZSTD_c_ldmBucketSizeLog + ldm_hash_rate_log = _zstd.ZSTD_c_ldmHashRateLog + + content_size_flag = _zstd.ZSTD_c_contentSizeFlag + checksum_flag = _zstd.ZSTD_c_checksumFlag + dict_id_flag = _zstd.ZSTD_c_dictIDFlag + + nb_workers = _zstd.ZSTD_c_nbWorkers + job_size = _zstd.ZSTD_c_jobSize + overlap_log = _zstd.ZSTD_c_overlapLog def bounds(self): """Return the (lower, upper) int bounds of a compression parameter. Both the lower and upper bounds are inclusive. """ - return _zstd._get_param_bounds(self.value, is_compress=True) + return _zstd.get_param_bounds(self.value, is_compress=True) class DecompressionParameter(enum.IntEnum): """Decompression parameters.""" - window_log_max = _zstd._ZSTD_d_windowLogMax + window_log_max = _zstd.ZSTD_d_windowLogMax def bounds(self): """Return the (lower, upper) int bounds of a decompression parameter. Both the lower and upper bounds are inclusive. """ - return _zstd._get_param_bounds(self.value, is_compress=False) + return _zstd.get_param_bounds(self.value, is_compress=False) class Strategy(enum.IntEnum): @@ -219,16 +229,16 @@ class Strategy(enum.IntEnum): the numeric value might change. """ - fast = _zstd._ZSTD_fast - dfast = _zstd._ZSTD_dfast - greedy = _zstd._ZSTD_greedy - lazy = _zstd._ZSTD_lazy - lazy2 = _zstd._ZSTD_lazy2 - btlazy2 = _zstd._ZSTD_btlazy2 - btopt = _zstd._ZSTD_btopt - btultra = _zstd._ZSTD_btultra - btultra2 = _zstd._ZSTD_btultra2 + fast = _zstd.ZSTD_fast + dfast = _zstd.ZSTD_dfast + greedy = _zstd.ZSTD_greedy + lazy = _zstd.ZSTD_lazy + lazy2 = _zstd.ZSTD_lazy2 + btlazy2 = _zstd.ZSTD_btlazy2 + btopt = _zstd.ZSTD_btopt + btultra = _zstd.ZSTD_btultra + btultra2 = _zstd.ZSTD_btultra2 # Check validity of the CompressionParameter & DecompressionParameter types -_zstd._set_parameter_types(CompressionParameter, DecompressionParameter) +_zstd.set_parameter_types(CompressionParameter, DecompressionParameter) diff --git a/Lib/compression/zstd/_zstdfile.py b/Lib/compression/zstd/_zstdfile.py index fbc9e02a733626e..26d7c9006375fb6 100644 --- a/Lib/compression/zstd/_zstdfile.py +++ b/Lib/compression/zstd/_zstdfile.py @@ -1,12 +1,9 @@ import io from os import PathLike -from _zstd import (ZstdCompressor, ZstdDecompressor, _ZSTD_DStreamSizes, - ZstdError) +from _zstd import ZstdCompressor, ZstdDecompressor, ZSTD_DStreamOutSize from compression._common import _streams -__all__ = ("ZstdFile", "open") - -_ZSTD_DStreamOutSize = _ZSTD_DStreamSizes[1] +__all__ = ('ZstdFile', 'open') _MODE_CLOSED = 0 _MODE_READ = 1 @@ -33,15 +30,15 @@ class ZstdFile(_streams.BaseStream): FLUSH_BLOCK = ZstdCompressor.FLUSH_BLOCK FLUSH_FRAME = ZstdCompressor.FLUSH_FRAME - def __init__(self, file, /, mode="r", *, + def __init__(self, file, /, mode='r', *, level=None, options=None, zstd_dict=None): """Open a Zstandard compressed file in binary mode. - *file* can be either an file-like object, or a file name to open. + *file* can be either a file-like object, or a file name to open. - *mode* can be "r" for reading (default), "w" for (over)writing, "x" for - creating exclusively, or "a" for appending. These can equivalently be - given as "rb", "wb", "xb" and "ab" respectively. + *mode* can be 'r' for reading (default), 'w' for (over)writing, 'x' + for creating exclusively, or 'a' for appending. These can + equivalently be given as 'rb', 'wb', 'xb' and 'ab' respectively. *level* is an optional int specifying the compression level to use, or COMPRESSION_LEVEL_DEFAULT if not given. @@ -59,39 +56,38 @@ def __init__(self, file, /, mode="r", *, self._buffer = None if not isinstance(mode, str): - raise ValueError("mode must be a str") + raise ValueError('mode must be a str') if options is not None and not isinstance(options, dict): - raise TypeError("options must be a dict or None") - mode = mode.removesuffix("b") # handle rb, wb, xb, ab - if mode == "r": + raise TypeError('options must be a dict or None') + mode = mode.removesuffix('b') # handle rb, wb, xb, ab + if mode == 'r': if level is not None: - raise TypeError("level is illegal in read mode") + raise TypeError('level is illegal in read mode') self._mode = _MODE_READ - elif mode in {"w", "a", "x"}: + elif mode in {'w', 'a', 'x'}: if level is not None and not isinstance(level, int): - raise TypeError("level must be int or None") + raise TypeError('level must be int or None') self._mode = _MODE_WRITE self._compressor = ZstdCompressor(level=level, options=options, zstd_dict=zstd_dict) self._pos = 0 else: - raise ValueError(f"Invalid mode: {mode!r}") + raise ValueError(f'Invalid mode: {mode!r}') if isinstance(file, (str, bytes, PathLike)): self._fp = io.open(file, f'{mode}b') self._close_fp = True - elif ((mode == 'r' and hasattr(file, "read")) - or (mode != 'r' and hasattr(file, "write"))): + elif ((mode == 'r' and hasattr(file, 'read')) + or (mode != 'r' and hasattr(file, 'write'))): self._fp = file else: - raise TypeError("file must be a file-like object " - "or a str, bytes, or PathLike object") + raise TypeError('file must be a file-like object ' + 'or a str, bytes, or PathLike object') if self._mode == _MODE_READ: raw = _streams.DecompressReader( self._fp, ZstdDecompressor, - trailing_error=ZstdError, zstd_dict=zstd_dict, options=options, ) @@ -154,22 +150,22 @@ def flush(self, mode=FLUSH_BLOCK): return self._check_not_closed() if mode not in {self.FLUSH_BLOCK, self.FLUSH_FRAME}: - raise ValueError("Invalid mode argument, expected either " - "ZstdFile.FLUSH_FRAME or " - "ZstdFile.FLUSH_BLOCK") + raise ValueError('Invalid mode argument, expected either ' + 'ZstdFile.FLUSH_FRAME or ' + 'ZstdFile.FLUSH_BLOCK') if self._compressor.last_mode == mode: return # Flush zstd block/frame, and write. data = self._compressor.flush(mode) self._fp.write(data) - if hasattr(self._fp, "flush"): + if hasattr(self._fp, 'flush'): self._fp.flush() def read(self, size=-1): """Read up to size uncompressed bytes from the file. If size is negative or omitted, read until EOF is reached. - Returns b"" if the file is already at EOF. + Returns b'' if the file is already at EOF. """ if size is None: size = -1 @@ -181,14 +177,14 @@ def read1(self, size=-1): making multiple reads from the underlying stream. Reads up to a buffer's worth of data if size is negative. - Returns b"" if the file is at EOF. + Returns b'' if the file is at EOF. """ self._check_can_read() if size < 0: # Note this should *not* be io.DEFAULT_BUFFER_SIZE. # ZSTD_DStreamOutSize is the minimum amount to read guaranteeing # a full block is read. - size = _ZSTD_DStreamOutSize + size = ZSTD_DStreamOutSize return self._buffer.read1(size) def readinto(self, b): @@ -250,6 +246,7 @@ def peek(self, size=-1): return self._buffer.peek(size) def __next__(self): + self._check_can_read() if ret := self._buffer.readline(): return ret raise StopIteration @@ -296,49 +293,50 @@ def writable(self): return self._mode == _MODE_WRITE -def open(file, /, mode="rb", *, level=None, options=None, zstd_dict=None, +def open(file, /, mode='rb', *, level=None, options=None, zstd_dict=None, encoding=None, errors=None, newline=None): """Open a Zstandard compressed file in binary or text mode. - file can be either a file name (given as a str, bytes, or PathLike object), - in which case the named file is opened, or it can be an existing file object - to read from or write to. + file can be either a file name (given as a str, bytes, or PathLike + object), in which case the named file is opened, or it can be + an existing file object to read from or write to. - The mode parameter can be "r", "rb" (default), "w", "wb", "x", "xb", "a", - "ab" for binary mode, or "rt", "wt", "xt", "at" for text mode. + The mode parameter can be 'r', 'rb' (default), 'w', 'wb', 'x', 'xb', + 'a', 'ab' for binary mode, or 'rt', 'wt', 'xt', 'at' for text mode. - The level, options, and zstd_dict parameters specify the settings the same - as ZstdFile. + The level, options, and zstd_dict parameters specify the settings the + same as ZstdFile. When using read mode (decompression), the options parameter is a dict - representing advanced decompression options. The level parameter is not - supported in this case. When using write mode (compression), only one of - level, an int representing the compression level, or options, a dict - representing advanced compression options, may be passed. In both modes, - zstd_dict is a ZstdDict instance containing a trained Zstandard dictionary. + representing advanced decompression options. The level parameter is not + supported in this case. When using write mode (compression), only one + of level, an int representing the compression level, or options, a dict + representing advanced compression options, may be passed. In both + modes, zstd_dict is a ZstdDict instance containing a trained Zstandard + dictionary. - For binary mode, this function is equivalent to the ZstdFile constructor: - ZstdFile(filename, mode, ...). In this case, the encoding, errors and - newline parameters must not be provided. + For binary mode, this function is equivalent to the ZstdFile + constructor: ZstdFile(filename, mode, ...). In this case, the encoding, + errors and newline parameters must not be provided. For text mode, an ZstdFile object is created, and wrapped in an io.TextIOWrapper instance with the specified encoding, error handling behavior, and line ending(s). """ - text_mode = "t" in mode - mode = mode.replace("t", "") + text_mode = 't' in mode + mode = mode.replace('t', '') if text_mode: - if "b" in mode: - raise ValueError(f"Invalid mode: {mode!r}") + if 'b' in mode: + raise ValueError(f'Invalid mode: {mode!r}') else: if encoding is not None: - raise ValueError("Argument 'encoding' not supported in binary mode") + raise ValueError('Argument "encoding" not supported in binary mode') if errors is not None: - raise ValueError("Argument 'errors' not supported in binary mode") + raise ValueError('Argument "errors" not supported in binary mode') if newline is not None: - raise ValueError("Argument 'newline' not supported in binary mode") + raise ValueError('Argument "newline" not supported in binary mode') binary_file = ZstdFile(file, mode, level=level, options=options, zstd_dict=zstd_dict) diff --git a/Lib/concurrent/futures/__init__.py b/Lib/concurrent/futures/__init__.py index 7ada7431c1ab8c2..d6ac4b3e0b675f6 100644 --- a/Lib/concurrent/futures/__init__.py +++ b/Lib/concurrent/futures/__init__.py @@ -17,7 +17,7 @@ wait, as_completed) -__all__ = ( +__all__ = [ 'FIRST_COMPLETED', 'FIRST_EXCEPTION', 'ALL_COMPLETED', @@ -29,36 +29,37 @@ 'Executor', 'wait', 'as_completed', - 'InterpreterPoolExecutor', 'ProcessPoolExecutor', 'ThreadPoolExecutor', -) +] + + +try: + import _interpreters +except ImportError: + _interpreters = None + +if _interpreters: + __all__.append('InterpreterPoolExecutor') def __dir__(): - return __all__ + ('__author__', '__doc__') + return __all__ + ['__author__', '__doc__'] def __getattr__(name): global ProcessPoolExecutor, ThreadPoolExecutor, InterpreterPoolExecutor if name == 'ProcessPoolExecutor': - from .process import ProcessPoolExecutor as pe - ProcessPoolExecutor = pe - return pe + from .process import ProcessPoolExecutor + return ProcessPoolExecutor if name == 'ThreadPoolExecutor': - from .thread import ThreadPoolExecutor as te - ThreadPoolExecutor = te - return te + from .thread import ThreadPoolExecutor + return ThreadPoolExecutor - if name == 'InterpreterPoolExecutor': - try: - from .interpreter import InterpreterPoolExecutor as ie - except ModuleNotFoundError: - ie = InterpreterPoolExecutor = None - else: - InterpreterPoolExecutor = ie - return ie + if _interpreters and name == 'InterpreterPoolExecutor': + from .interpreter import InterpreterPoolExecutor + return InterpreterPoolExecutor raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py index d98b1ebdd584b5a..43ae9c8b9014788 100644 --- a/Lib/concurrent/futures/_base.py +++ b/Lib/concurrent/futures/_base.py @@ -194,15 +194,15 @@ def as_completed(fs, timeout=None): """An iterator over the given futures that yields each as it completes. Args: - fs: The sequence of Futures (possibly created by different Executors) to - iterate over. - timeout: The maximum number of seconds to wait. If None, then there - is no limit on the wait time. + fs: The sequence of Futures (possibly created by different + Executors) to iterate over. + timeout: The maximum number of seconds to wait. If None, then + there is no limit on the wait time. Returns: - An iterator that yields the given Futures as they complete (finished or - cancelled). If any given Futures are duplicated, they will be returned - once. + An iterator that yields the given Futures as they complete + (finished or cancelled). If any given Futures are duplicated, + they will be returned once. Raises: TimeoutError: If the entire result iterator could not be generated @@ -258,19 +258,20 @@ def wait(fs, timeout=None, return_when=ALL_COMPLETED): """Wait for the futures in the given sequence to complete. Args: - fs: The sequence of Futures (possibly created by different Executors) to - wait upon. - timeout: The maximum number of seconds to wait. If None, then there - is no limit on the wait time. - return_when: Indicates when this function should return. The options - are: + fs: The sequence of Futures (possibly created by different + Executors) to wait upon. + timeout: The maximum number of seconds to wait. If None, then + there is no limit on the wait time. + return_when: Indicates when this function should return. + The options are: FIRST_COMPLETED - Return when any future finishes or is cancelled. FIRST_EXCEPTION - Return when any future finishes by raising an - exception. If no future raises an exception + exception. If no future raises an exception then it is equivalent to ALL_COMPLETED. - ALL_COMPLETED - Return when all futures finish or are cancelled. + ALL_COMPLETED - Return when all futures finish or are + cancelled. Returns: A named 2-tuple of sets. The first set, named 'done', contains the @@ -404,11 +405,12 @@ def add_done_callback(self, fn): Args: fn: A callable that will be called with this future as its only - argument when the future completes or is cancelled. The callable - will always be called by a thread in the same process in which - it was added. If the future has already completed or been - cancelled then the callable will be called immediately. These - callables are called in the order that they were added. + argument when the future completes or is cancelled. The + callable will always be called by a thread in the same + process in which it was added. If the future has already + completed or been cancelled then the callable will be + called immediately. These callables are called in the + order that they were added. """ with self._condition: if self._state not in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]: @@ -423,17 +425,19 @@ def result(self, timeout=None): """Return the result of the call that the future represents. Args: - timeout: The number of seconds to wait for the result if the future - isn't done. If None, then there is no limit on the wait time. + timeout: The number of seconds to wait for the result if the + future isn't done. If None, then there is no limit on the + wait time. Returns: The result of the call that the future represents. Raises: CancelledError: If the future was cancelled. - TimeoutError: If the future didn't finish executing before the given - timeout. - Exception: If the call raised then that exception will be raised. + TimeoutError: If the future didn't finish executing before the + given timeout. + Exception: If the call raised then that exception will be + raised. """ try: with self._condition: @@ -459,17 +463,17 @@ def exception(self, timeout=None): Args: timeout: The number of seconds to wait for the exception if the - future isn't done. If None, then there is no limit on the wait - time. + future isn't done. If None, then there is no limit on the + wait time. Returns: - The exception raised by the call that the future represents or None - if the call completed without raising. + The exception raised by the call that the future represents or + None if the call completed without raising. Raises: CancelledError: If the future was cancelled. - TimeoutError: If the future didn't finish executing before the given - timeout. + TimeoutError: If the future didn't finish executing before the + given timeout. """ with self._condition: @@ -494,22 +498,23 @@ def set_running_or_notify_cancel(self): Should only be used by Executor implementations and unit tests. If the future has been cancelled (cancel() was called and returned - True) then any threads waiting on the future completing (though calls - to as_completed() or wait()) are notified and False is returned. + True) then any threads waiting on the future completing (though + calls to as_completed() or wait()) are notified and False is + returned. If the future was not cancelled then it is put in the running state (future calls to running() will return True) and True is returned. This method should be called by Executor implementations before - executing the work associated with this future. If this method returns - False then the work should not be executed. + executing the work associated with this future. If this method + returns False then the work should not be executed. Returns: False if the Future was cancelled, True otherwise. Raises: - RuntimeError: if this method was already called or if set_result() - or set_exception() was called. + RuntimeError: if this method was already called or if + set_result() or set_exception() was called. """ with self._condition: if self._state == CANCELLED: @@ -566,8 +571,9 @@ class Executor(object): def submit(self, fn, /, *args, **kwargs): """Submits a callable to be executed with the given arguments. - Schedules the callable to be executed as fn(*args, **kwargs) and returns - a Future instance representing the execution of the callable. + Schedules the callable to be executed as fn(*args, **kwargs) and + returns a Future instance representing the execution of the + callable. Returns: A Future representing the given call. @@ -580,25 +586,25 @@ def map(self, fn, *iterables, timeout=None, chunksize=1, buffersize=None): Args: fn: A callable that will take as many arguments as there are passed iterables. - timeout: The maximum number of seconds to wait. If None, then there - is no limit on the wait time. - chunksize: The size of the chunks the iterable will be broken into - before being passed to a child process. This argument is only - used by ProcessPoolExecutor; it is ignored by + timeout: The maximum number of seconds to wait. If None, then + there is no limit on the wait time. + chunksize: The size of the chunks the iterable will be broken + into before being passed to a child process. This argument + is only used by ProcessPoolExecutor; it is ignored by ThreadPoolExecutor. buffersize: The number of submitted tasks whose results have not - yet been yielded. If the buffer is full, iteration over the + yet been yielded. If the buffer is full, iteration over the iterables pauses until a result is yielded from the buffer. - If None, all input elements are eagerly collected, and a task is - submitted for each. + If None, all input elements are eagerly collected, and + a task is submitted for each. Returns: - An iterator equivalent to: map(func, *iterables) but the calls may - be evaluated out-of-order. + An iterator equivalent to: map(func, *iterables) but the calls + may be evaluated out-of-order. Raises: - TimeoutError: If the entire result iterator could not be generated - before the given timeout. + TimeoutError: If the entire result iterator could not be + generated before the given timeout. Exception: If fn(*args) raises for any values. """ if buffersize is not None and not isinstance(buffersize, int): @@ -652,8 +658,8 @@ def shutdown(self, wait=True, *, cancel_futures=False): Args: wait: If True then shutdown will not return until all running - futures have finished executing and the resources used by the - executor have been reclaimed. + futures have finished executing and the resources used by + the executor have been reclaimed. cancel_futures: If True then shutdown will cancel all pending futures. Futures that are completed or running will not be cancelled. @@ -670,5 +676,5 @@ def __exit__(self, exc_type, exc_val, exc_tb): class BrokenExecutor(RuntimeError): """ - Raised when a executor has become non-functional after a severe failure. + Raised when an executor has become non-functional after a severe failure. """ diff --git a/Lib/concurrent/futures/interpreter.py b/Lib/concurrent/futures/interpreter.py index d17688dc9d73462..fd3d45144b49a71 100644 --- a/Lib/concurrent/futures/interpreter.py +++ b/Lib/concurrent/futures/interpreter.py @@ -1,69 +1,38 @@ """Implements InterpreterPoolExecutor.""" -import contextlib -import pickle -import textwrap +from concurrent import interpreters +import sys from . import thread as _thread -import _interpreters -import _interpqueues +import traceback -class ExecutionFailed(_interpreters.InterpreterError): - """An unhandled exception happened during execution.""" - - def __init__(self, excinfo): - msg = excinfo.formatted - if not msg: - if excinfo.type and excinfo.msg: - msg = f'{excinfo.type.__name__}: {excinfo.msg}' - else: - msg = excinfo.type.__name__ or excinfo.msg - super().__init__(msg) - self.excinfo = excinfo - - def __str__(self): +def do_call(results, func, args, kwargs): + try: + return func(*args, **kwargs) + except BaseException as exc: + # Send the captured exception out on the results queue, + # but still leave it unhandled for the interpreter to handle. try: - formatted = self.excinfo.errdisplay - except Exception: - return super().__str__() - else: - return textwrap.dedent(f""" -{super().__str__()} - -Uncaught in the interpreter: - -{formatted} - """.strip()) - - -UNBOUND = 2 # error; this should not happen. + results.put(exc) + except interpreters.NotShareableError: + # The exception is not shareable. + print('exception is not shareable:', file=sys.stderr) + traceback.print_exception(exc) + results.put(None) + raise # re-raise class WorkerContext(_thread.WorkerContext): @classmethod - def prepare(cls, initializer, initargs, shared): + def prepare(cls, initializer, initargs): def resolve_task(fn, args, kwargs): if isinstance(fn, str): # XXX Circle back to this later. raise TypeError('scripts not supported') - if args or kwargs: - raise ValueError(f'a script does not take args or kwargs, got {args!r} and {kwargs!r}') - data = textwrap.dedent(fn) - kind = 'script' - # Make sure the script compiles. - # Ideally we wouldn't throw away the resulting code - # object. However, there isn't much to be done until - # code objects are shareable and/or we do a better job - # of supporting code objects in _interpreters.exec(). - compile(data, '', 'exec') else: - # Functions defined in the __main__ module can't be pickled, - # so they can't be used here. In the future, we could possibly - # borrow from multiprocessing to work around this. - data = pickle.dumps((fn, args, kwargs)) - kind = 'function' - return (data, kind) + task = (fn, args, kwargs) + return task if initializer is not None: try: @@ -75,73 +44,24 @@ def resolve_task(fn, args, kwargs): else: initdata = None def create_context(): - return cls(initdata, shared) + return cls(initdata) return create_context, resolve_task - @classmethod - @contextlib.contextmanager - def _capture_exc(cls, resultsid): - try: - yield - except BaseException as exc: - # Send the captured exception out on the results queue, - # but still leave it unhandled for the interpreter to handle. - err = pickle.dumps(exc) - _interpqueues.put(resultsid, (None, err), 1, UNBOUND) - raise # re-raise - - @classmethod - def _send_script_result(cls, resultsid): - _interpqueues.put(resultsid, (None, None), 0, UNBOUND) - - @classmethod - def _call(cls, func, args, kwargs, resultsid): - with cls._capture_exc(resultsid): - res = func(*args or (), **kwargs or {}) - # Send the result back. - try: - _interpqueues.put(resultsid, (res, None), 0, UNBOUND) - except _interpreters.NotShareableError: - res = pickle.dumps(res) - _interpqueues.put(resultsid, (res, None), 1, UNBOUND) - - @classmethod - def _call_pickled(cls, pickled, resultsid): - with cls._capture_exc(resultsid): - fn, args, kwargs = pickle.loads(pickled) - cls._call(fn, args, kwargs, resultsid) - - def __init__(self, initdata, shared=None): + def __init__(self, initdata): self.initdata = initdata - self.shared = dict(shared) if shared else None - self.interpid = None - self.resultsid = None + self.interp = None + self.results = None def __del__(self): - if self.interpid is not None: + if self.interp is not None: self.finalize() - def _exec(self, script): - assert self.interpid is not None - excinfo = _interpreters.exec(self.interpid, script, restrict=True) - if excinfo is not None: - raise ExecutionFailed(excinfo) - def initialize(self): - assert self.interpid is None, self.interpid - self.interpid = _interpreters.create(reqrefs=True) + assert self.interp is None, self.interp + self.interp = interpreters.create() try: - _interpreters.incref(self.interpid) - maxsize = 0 - fmt = 0 - self.resultsid = _interpqueues.create(maxsize, fmt, UNBOUND) - - self._exec(f'from {__name__} import WorkerContext') - - if self.shared: - _interpreters.set___main___attrs( - self.interpid, self.shared, restrict=True) + self.results = interpreters.create_queue(maxsize) if self.initdata: self.run(self.initdata) @@ -150,64 +70,25 @@ def initialize(self): raise # re-raise def finalize(self): - interpid = self.interpid - resultsid = self.resultsid - self.resultsid = None - self.interpid = None - if resultsid is not None: - try: - _interpqueues.destroy(resultsid) - except _interpqueues.QueueNotFoundError: - pass - if interpid is not None: - try: - _interpreters.decref(interpid) - except _interpreters.InterpreterNotFoundError: - pass + interp = self.interp + results = self.results + self.results = None + self.interp = None + if results is not None: + del results + if interp is not None: + interp.close() def run(self, task): - data, kind = task - if kind == 'script': - raise NotImplementedError('script kind disabled') - script = f""" -with WorkerContext._capture_exc({self.resultsid}): -{textwrap.indent(data, ' ')} -WorkerContext._send_script_result({self.resultsid})""" - elif kind == 'function': - script = f'WorkerContext._call_pickled({data!r}, {self.resultsid})' - else: - raise NotImplementedError(kind) - try: - self._exec(script) - except ExecutionFailed as exc: - exc_wrapper = exc - else: - exc_wrapper = None - - # Return the result, or raise the exception. - while True: - try: - obj = _interpqueues.get(self.resultsid) - except _interpqueues.QueueNotFoundError: + return self.interp.call(do_call, self.results, *task) + except interpreters.ExecutionFailed as wrapper: + # Wait for the exception data to show up. + exc = self.results.get() + if exc is None: + # The exception must have been not shareable. raise # re-raise - except _interpqueues.QueueError: - continue - except ModuleNotFoundError: - # interpreters.queues doesn't exist, which means - # QueueEmpty doesn't. Act as though it does. - continue - else: - break - (res, excdata), pickled, unboundop = obj - assert unboundop is None, unboundop - if excdata is not None: - assert res is None, res - assert pickled - assert exc_wrapper is not None - exc = pickle.loads(excdata) - raise exc from exc_wrapper - return pickle.loads(res) if pickled else res + raise exc from wrapper class BrokenInterpreterPool(_thread.BrokenThreadPool): @@ -221,22 +102,22 @@ class InterpreterPoolExecutor(_thread.ThreadPoolExecutor): BROKEN = BrokenInterpreterPool @classmethod - def prepare_context(cls, initializer, initargs, shared): - return WorkerContext.prepare(initializer, initargs, shared) + def prepare_context(cls, initializer, initargs): + return WorkerContext.prepare(initializer, initargs) def __init__(self, max_workers=None, thread_name_prefix='', - initializer=None, initargs=(), shared=None): + initializer=None, initargs=()): """Initializes a new InterpreterPoolExecutor instance. Args: - max_workers: The maximum number of interpreters that can be used to - execute the given calls. + max_workers: The maximum number of interpreters that can be used + to execute the given calls. thread_name_prefix: An optional name prefix to give our threads. initializer: A callable or script used to initialize each worker interpreter. initargs: A tuple of arguments to pass to the initializer. - shared: A mapping of shareabled objects to be inserted into - each worker interpreter. """ + thread_name_prefix = (thread_name_prefix or + (f"InterpreterPoolExecutor-{self._counter()}")) super().__init__(max_workers, thread_name_prefix, - initializer, initargs, shared=shared) + initializer, initargs) diff --git a/Lib/concurrent/futures/process.py b/Lib/concurrent/futures/process.py index 76b7b2abe836d82..ba4928dc4ae47f1 100644 --- a/Lib/concurrent/futures/process.py +++ b/Lib/concurrent/futures/process.py @@ -360,7 +360,7 @@ def run(self): if executor := self.executor_reference(): if process_exited: with self.shutdown_lock: - executor._adjust_process_count() + executor._replace_dead_worker() else: executor._idle_worker_semaphore.release() del executor @@ -469,17 +469,20 @@ def _terminate_broken(self, cause): executor._shutdown_thread = True executor = None - # All pending tasks are to be marked failed with the following - # BrokenProcessPool error - bpe = BrokenProcessPool("A process in the process pool was " - "terminated abruptly while the future was " - "running or pending.") + # All pending tasks are to be marked failed with a + # BrokenProcessPool error, as separate instances to avoid sharing + # a traceback (gh-101267). + cause_tb = None if cause is not None: - bpe.__cause__ = _RemoteTraceback( - f"\n'''\n{''.join(cause)}'''") + cause_tb = f"\n'''\n{''.join(cause)}'''" # Mark pending tasks as failed. for work_id, work_item in self.pending_work_items.items(): + bpe = BrokenProcessPool("A process in the process pool was " + "terminated abruptly while the future was " + "running or pending.") + if cause_tb is not None: + bpe.__cause__ = _RemoteTraceback(cause_tb) try: work_item.future.set_exception(bpe) except _base.InvalidStateError: @@ -642,19 +645,21 @@ def __init__(self, max_workers=None, mp_context=None, Args: max_workers: The maximum number of processes that can be used to - execute the given calls. If None or not given then as many - worker processes will be created as the machine has processors. - mp_context: A multiprocessing context to launch the workers created - using the multiprocessing.get_context('start method') API. This - object should provide SimpleQueue, Queue and Process. + execute the given calls. If None or not given then as many + worker processes will be created as the machine has + processors. + mp_context: A multiprocessing context to launch the workers + created using the multiprocessing.get_context('start method') + API. This object should provide SimpleQueue, Queue and + Process. initializer: A callable used to initialize worker processes. initargs: A tuple of arguments to pass to the initializer. - max_tasks_per_child: The maximum number of tasks a worker process - can complete before it will exit and be replaced with a fresh - worker process. The default of None means worker process will - live as long as the executor. Requires a non-'fork' mp_context - start method. When given, we default to using 'spawn' if no - mp_context is supplied. + max_tasks_per_child: The maximum number of tasks a worker + process can complete before it will exit and be replaced + with a fresh worker process. The default of None means + worker process will live as long as the executor. Requires + a non-'fork' mp_context start method. When given, we + default to using 'spawn' if no mp_context is supplied. """ _check_system_limits() @@ -754,19 +759,48 @@ def _start_executor_manager_thread(self): _threads_wakeups[self._executor_manager_thread] = \ self._executor_manager_thread_wakeup + def _replace_dead_worker(self): + # gh-132969: avoid error when state is reset and executor is still running, + # which will happen when shutdown(wait=False) is called. + if self._processes is None: + return + + # A replacement is pointless when shutting down with nothing left + # to run. Both attributes are read under _shutdown_lock, which + # shutdown() holds while setting _shutdown_thread. + assert self._shutdown_lock.locked() + if self._shutdown_thread and not self._pending_work_items: + return + + # gh-115634: A worker exited after reaching max_tasks_per_child and + # has been removed from self._processes. Do not consult + # _idle_worker_semaphore here: it counts task completions, not idle + # workers, so it can hold a stale token released by the now-dead + # worker. Trusting such a token would leave the pool a worker short, + # deadlocking once all workers reach their task limit. Spawning is + # safe from this (manager) thread despite gh-90622 because + # max_tasks_per_child is rejected for the "fork" start method. + if len(self._processes) < self._max_workers: + self._spawn_process() + def _adjust_process_count(self): + # gh-132969: avoid error when state is reset and executor is still running, + # which will happen when shutdown(wait=False) is called. + if self._processes is None: + return + # if there's an idle process, we don't need to spawn a new one. if self._idle_worker_semaphore.acquire(blocking=False): return process_count = len(self._processes) if process_count < self._max_workers: - # Assertion disabled as this codepath is also used to replace a - # worker that unexpectedly dies, even when using the 'fork' start - # method. That means there is still a potential deadlock bug. If a - # 'fork' mp_context worker dies, we'll be forking a new one when - # we know a thread is running (self._executor_manager_thread). - #assert self._safe_to_dynamically_spawn_children or not self._executor_manager_thread, 'https://github.com/python/cpython/issues/90622' + # gh-90622: spawning a child via fork while another thread is + # running can deadlock in the child. submit() only calls this + # method when using a non-fork start method. + assert (self._safe_to_dynamically_spawn_children + or not self._executor_manager_thread), ( + 'https://github.com/python/cpython/issues/90622') self._spawn_process() def _launch_processes(self): @@ -819,24 +853,25 @@ def map(self, fn, *iterables, timeout=None, chunksize=1, buffersize=None): Args: fn: A callable that will take as many arguments as there are passed iterables. - timeout: The maximum number of seconds to wait. If None, then there - is no limit on the wait time. - chunksize: If greater than one, the iterables will be chopped into - chunks of size chunksize and submitted to the process pool. - If set to one, the items in the list will be sent one at a time. + timeout: The maximum number of seconds to wait. If None, then + there is no limit on the wait time. + chunksize: If greater than one, the iterables will be chopped + into chunks of size chunksize and submitted to the process + pool. If set to one, the items in the list will be sent + one at a time. buffersize: The number of submitted tasks whose results have not - yet been yielded. If the buffer is full, iteration over the + yet been yielded. If the buffer is full, iteration over the iterables pauses until a result is yielded from the buffer. - If None, all input elements are eagerly collected, and a task is - submitted for each. + If None, all input elements are eagerly collected, and + a task is submitted for each. Returns: - An iterator equivalent to: map(func, *iterables) but the calls may - be evaluated out-of-order. + An iterator equivalent to: map(func, *iterables) but the calls + may be evaluated out-of-order. Raises: - TimeoutError: If the entire result iterator could not be generated - before the given timeout. + TimeoutError: If the entire result iterator could not be + generated before the given timeout. Exception: If fn(*args) raises for any values. """ if chunksize < 1: diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/concurrent/interpreters/__init__.py similarity index 84% rename from Lib/test/support/interpreters/__init__.py rename to Lib/concurrent/interpreters/__init__.py index e067f259364d2aa..ea4147ee9a25da5 100644 --- a/Lib/test/support/interpreters/__init__.py +++ b/Lib/concurrent/interpreters/__init__.py @@ -9,6 +9,10 @@ InterpreterError, InterpreterNotFoundError, NotShareableError, is_shareable, ) +from ._queues import ( + create as create_queue, + Queue, QueueEmpty, QueueFull, +) __all__ = [ @@ -20,21 +24,6 @@ ] -_queuemod = None - -def __getattr__(name): - if name in ('Queue', 'QueueEmpty', 'QueueFull', 'create_queue'): - global create_queue, Queue, QueueEmpty, QueueFull - ns = globals() - from .queues import ( - create as create_queue, - Queue, QueueEmpty, QueueFull, - ) - return ns[name] - else: - raise AttributeError(name) - - _EXEC_FAILURE_STR = """ {superstr} @@ -157,19 +146,20 @@ def __del__(self): self._decref() # for pickling: - def __getnewargs__(self): - return (self._id,) - - # for pickling: - def __getstate__(self): - return None - - def _decref(self): + def __reduce__(self): + return (type(self), (self._id,)) + + # gh-135729: Globals might be destroyed by the time this is called, so we + # need to keep references ourself + def _decref(self, *, + InterpreterNotFoundError=InterpreterNotFoundError, + _interp_decref=_interpreters.decref, + ): if not self._ownsref: return self._ownsref = False try: - _interpreters.decref(self._id) + _interp_decref(self._id) except InterpreterNotFoundError: pass @@ -226,33 +216,32 @@ def exec(self, code, /): if excinfo is not None: raise ExecutionFailed(excinfo) - def call(self, callable, /): - """Call the object in the interpreter with given args/kwargs. + def _call(self, callable, args, kwargs): + res, excinfo = _interpreters.call(self._id, callable, args, kwargs, restrict=True) + if excinfo is not None: + raise ExecutionFailed(excinfo) + return res - Only functions that take no arguments and have no closure - are supported. + def call(self, callable, /, *args, **kwargs): + """Call the object in the interpreter with given args/kwargs. - The return value is discarded. + Nearly all callables, args, kwargs, and return values are + supported. All "shareable" objects are supported, as are + "stateless" functions (meaning non-closures that do not use + any globals). This method will fall back to pickle. If the callable raises an exception then the error display - (including full traceback) is send back between the interpreters + (including full traceback) is sent back between the interpreters and an ExecutionFailed exception is raised, much like what happens with Interpreter.exec(). """ - # XXX Support args and kwargs. - # XXX Support arbitrary callables. - # XXX Support returning the return value (e.g. via pickle). - excinfo = _interpreters.call(self._id, callable, restrict=True) - if excinfo is not None: - raise ExecutionFailed(excinfo) + return self._call(callable, args, kwargs) - def call_in_thread(self, callable, /): + def call_in_thread(self, callable, /, *args, **kwargs): """Return a new thread that calls the object in the interpreter. The return value and any raised exception are discarded. """ - def task(): - self.call(callable) - t = threading.Thread(target=task) + t = threading.Thread(target=self._call, args=(callable, args, kwargs)) t.start() return t diff --git a/Lib/test/support/interpreters/_crossinterp.py b/Lib/concurrent/interpreters/_crossinterp.py similarity index 86% rename from Lib/test/support/interpreters/_crossinterp.py rename to Lib/concurrent/interpreters/_crossinterp.py index 544e197ba4c0288..a5f46b20fbb4c56 100644 --- a/Lib/test/support/interpreters/_crossinterp.py +++ b/Lib/concurrent/interpreters/_crossinterp.py @@ -40,16 +40,21 @@ class UnboundItem: @classonly def singleton(cls, kind, module, name='UNBOUND'): - doc = cls.__doc__.replace('cross-interpreter container', kind) - doc = doc.replace('cross-interpreter', kind) + doc = cls.__doc__ + if doc: + doc = doc.replace( + 'cross-interpreter container', kind, + ).replace( + 'cross-interpreter', kind, + ) subclass = type( f'Unbound{kind.capitalize()}Item', (cls,), - dict( - _MODULE=module, - _NAME=name, - __doc__=doc, - ), + { + "_MODULE": module, + "_NAME": name, + "__doc__": doc, + }, ) return object.__new__(subclass) @@ -61,7 +66,7 @@ def __new__(cls): def __repr__(self): return f'{self._MODULE}.{self._NAME}' -# return f'interpreters.queues.UNBOUND' +# return f'interpreters._queues.UNBOUND' UNBOUND = object.__new__(UnboundItem) diff --git a/Lib/test/support/interpreters/queues.py b/Lib/concurrent/interpreters/_queues.py similarity index 59% rename from Lib/test/support/interpreters/queues.py rename to Lib/concurrent/interpreters/_queues.py index deb8e8613af731b..56c947f7d2aa12d 100644 --- a/Lib/test/support/interpreters/queues.py +++ b/Lib/concurrent/interpreters/_queues.py @@ -63,29 +63,34 @@ def _resolve_unbound(flag): return resolved -def create(maxsize=0, *, syncobj=False, unbounditems=UNBOUND): +def create(maxsize=0, *, unbounditems=UNBOUND): """Return a new cross-interpreter queue. The queue may be used to pass data safely between interpreters. - "syncobj" sets the default for Queue.put() - and Queue.put_nowait(). - - "unbounditems" likewise sets the default. See Queue.put() for + "unbounditems" sets the default for Queue.put(); see that method for supported values. The default value is UNBOUND, which replaces the unbound item. """ - fmt = _SHARED_ONLY if syncobj else _PICKLED unbound = _serialize_unbound(unbounditems) unboundop, = unbound - qid = _queues.create(maxsize, fmt, unboundop) - return Queue(qid, _fmt=fmt, _unbound=unbound) + qid = _queues.create(maxsize, unboundop, -1) + self = Queue(qid) + self._set_unbound(unboundop, unbounditems) + return self def list_all(): """Return a list of all open queues.""" - return [Queue(qid, _fmt=fmt, _unbound=(unboundop,)) - for qid, fmt, unboundop in _queues.list_all()] + queues = [] + for qid, unboundop, _ in _queues.list_all(): + self = Queue(qid) + if not hasattr(self, '_unbound'): + self._set_unbound(unboundop) + else: + assert self._unbound[0] == unboundop + queues.append(self) + return queues _known_queues = weakref.WeakValueDictionary() @@ -93,28 +98,17 @@ def list_all(): class Queue: """A cross-interpreter queue.""" - def __new__(cls, id, /, *, _fmt=None, _unbound=None): + def __new__(cls, id, /): # There is only one instance for any given ID. if isinstance(id, int): id = int(id) else: raise TypeError(f'id must be an int, got {id!r}') - if _fmt is None: - if _unbound is None: - _fmt, op = _queues.get_queue_defaults(id) - _unbound = (op,) - else: - _fmt, _ = _queues.get_queue_defaults(id) - elif _unbound is None: - _, op = _queues.get_queue_defaults(id) - _unbound = (op,) try: self = _known_queues[id] except KeyError: self = super().__new__(cls) self._id = id - self._fmt = _fmt - self._unbound = _unbound _known_queues[id] = self _queues.bind(id) return self @@ -136,17 +130,30 @@ def __hash__(self): return hash(self._id) # for pickling: - def __getnewargs__(self): - return (self._id,) + def __reduce__(self): + return (type(self), (self._id,)) - # for pickling: - def __getstate__(self): - return None + def _set_unbound(self, op, items=None): + assert not hasattr(self, '_unbound') + if items is None: + items = _resolve_unbound(op) + unbound = (op, items) + self._unbound = unbound + return unbound @property def id(self): return self._id + @property + def unbounditems(self): + try: + _, items = self._unbound + except AttributeError: + op, _ = _queues.get_queue_defaults(self._id) + _, items = self._set_unbound(op) + return items + @property def maxsize(self): try: @@ -164,78 +171,60 @@ def full(self): def qsize(self): return _queues.get_count(self._id) - def put(self, obj, timeout=None, *, - syncobj=None, - unbound=None, + def put(self, obj, block=True, timeout=None, *, + unbounditems=None, _delay=10 / 1000, # 10 milliseconds ): """Add the object to the queue. - This blocks while the queue is full. + If "block" is true, this blocks while the queue is full. - If "syncobj" is None (the default) then it uses the - queue's default, set with create_queue(). + For most objects, the object received through Queue.get() will + be a new one, equivalent to the original and not sharing any + actual underlying data. The notable exceptions include + cross-interpreter types (like Queue) and memoryview, where the + underlying data is actually shared. Furthermore, some types + can be sent through a queue more efficiently than others. This + group includes various immutable types like int, str, bytes, and + tuple (if the items are likewise efficiently shareable). + See interpreters.is_shareable(). - If "syncobj" is false then all objects are supported, - at the expense of worse performance. - - If "syncobj" is true then the object must be "shareable". - Examples of "shareable" objects include the builtin singletons, - str, and memoryview. One benefit is that such objects are - passed through the queue efficiently. - - The key difference, though, is conceptual: the corresponding - object returned from Queue.get() will be strictly equivalent - to the given obj. In other words, the two objects will be - effectively indistinguishable from each other, even if the - object is mutable. The received object may actually be the - same object, or a copy (immutable values only), or a proxy. - Regardless, the received object should be treated as though - the original has been shared directly, whether or not it - actually is. That's a slightly different and stronger promise - than just (initial) equality, which is all "syncobj=False" - can promise. - - "unbound" controls the behavior of Queue.get() for the given + "unbounditems" controls the behavior of Queue.get() for the given object if the current interpreter (calling put()) is later destroyed. - If "unbound" is None (the default) then it uses the + If "unbounditems" is None (the default) then it uses the queue's default, set with create_queue(), which is usually UNBOUND. - If "unbound" is UNBOUND_ERROR then get() will raise an + If "unbounditems" is UNBOUND_ERROR then get() will raise an ItemInterpreterDestroyed exception if the original interpreter has been destroyed. This does not otherwise affect the queue; the next call to put() will work like normal, returning the next item in the queue. - If "unbound" is UNBOUND_REMOVE then the item will be removed + If "unbounditems" is UNBOUND_REMOVE then the item will be removed from the queue as soon as the original interpreter is destroyed. Be aware that this will introduce an imbalance between put() and get() calls. - If "unbound" is UNBOUND then it is returned by get() in place + If "unbounditems" is UNBOUND then it is returned by get() in place of the unbound item. """ - if syncobj is None: - fmt = self._fmt - else: - fmt = _SHARED_ONLY if syncobj else _PICKLED - if unbound is None: - unboundop, = self._unbound + if not block: + return self.put_nowait(obj, unbounditems=unbounditems) + if unbounditems is None: + unboundop = -1 else: - unboundop, = _serialize_unbound(unbound) + unboundop, = _serialize_unbound(unbounditems) if timeout is not None: timeout = int(timeout) if timeout < 0: raise ValueError(f'timeout value must be non-negative') end = time.time() + timeout - if fmt is _PICKLED: - obj = pickle.dumps(obj) while True: try: - _queues.put(self._id, obj, fmt, unboundop) + _queues.put(self._id, obj, unboundop) except QueueFull as exc: if timeout is not None and time.time() >= end: raise # re-raise @@ -243,30 +232,26 @@ def put(self, obj, timeout=None, *, else: break - def put_nowait(self, obj, *, syncobj=None, unbound=None): - if syncobj is None: - fmt = self._fmt + def put_nowait(self, obj, *, unbounditems=None): + if unbounditems is None: + unboundop = -1 else: - fmt = _SHARED_ONLY if syncobj else _PICKLED - if unbound is None: - unboundop, = self._unbound - else: - unboundop, = _serialize_unbound(unbound) - if fmt is _PICKLED: - obj = pickle.dumps(obj) - _queues.put(self._id, obj, fmt, unboundop) + unboundop, = _serialize_unbound(unbounditems) + _queues.put(self._id, obj, unboundop) - def get(self, timeout=None, *, + def get(self, block=True, timeout=None, *, _delay=10 / 1000, # 10 milliseconds ): """Return the next object from the queue. - This blocks while the queue is empty. + If "block" is true, this blocks while the queue is empty. If the next item's original interpreter has been destroyed then the "next object" is determined by the value of the - "unbound" argument to put(). + "unbounditems" argument to put(). """ + if not block: + return self.get_nowait() if timeout is not None: timeout = int(timeout) if timeout < 0: @@ -274,7 +259,7 @@ def get(self, timeout=None, *, end = time.time() + timeout while True: try: - obj, fmt, unboundop = _queues.get(self._id) + obj, unboundop = _queues.get(self._id) except QueueEmpty as exc: if timeout is not None and time.time() >= end: raise # re-raise @@ -284,10 +269,6 @@ def get(self, timeout=None, *, if unboundop is not None: assert obj is None, repr(obj) return _resolve_unbound(unboundop) - if fmt == _PICKLED: - obj = pickle.loads(obj) - else: - assert fmt == _SHARED_ONLY return obj def get_nowait(self): @@ -297,16 +278,12 @@ def get_nowait(self): is the same as get(). """ try: - obj, fmt, unboundop = _queues.get(self._id) + obj, unboundop = _queues.get(self._id) except QueueEmpty as exc: raise # re-raise if unboundop is not None: assert obj is None, repr(obj) return _resolve_unbound(unboundop) - if fmt == _PICKLED: - obj = pickle.loads(obj) - else: - assert fmt == _SHARED_ONLY return obj diff --git a/Lib/configparser.py b/Lib/configparser.py index 239fda60a02ca03..3c452afe8ade485 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -315,12 +315,15 @@ def __init__(self, source, *args): def append(self, lineno, line): self.errors.append((lineno, line)) - self.message += '\n\t[line %2d]: %s' % (lineno, repr(line)) + self.message += f'\n\t[line {lineno:2d}]: {line!r}' def combine(self, others): + messages = [self.message] for other in others: - for error in other.errors: - self.append(*error) + for lineno, line in other.errors: + self.errors.append((lineno, line)) + messages.append(f'\n\t[line {lineno:2d}]: {line!r}') + self.message = "".join(messages) return self @staticmethod @@ -613,7 +616,9 @@ class RawConfigParser(MutableMapping): \] # ] """ _OPT_TMPL = r""" - (?P
, but looking for > after the name should cover - # most of the cases and is much simpler - gtpos = rawdata.find('>', namematch.end()) - self.handle_endtag(tagname) - return gtpos+1 + if not endtagopen.match(rawdata, i): # ': # is ignored + # "missing-end-tag-name" parser error + return i+3 + else: + return self.parse_bogus_comment(i) - elem = match.group(1).lower() # script or style - if self.cdata_elem is not None: - if elem != self.cdata_elem: - self.handle_data(rawdata[i:gtpos]) - return gtpos + match = locatetagend.match(rawdata, i+2) + assert match + j = match.end() + if rawdata[j-1] != ">": + return -1 - self.handle_endtag(elem) + # find the name: "13.2.5.8 Tag name state" + # https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state + match = tagfind_tolerant.match(rawdata, i+2) + assert match + tag = match.group(1).lower() + self.handle_endtag(tag) self.clear_cdata_mode() - return gtpos + return j # Overridable -- finish processing of start+end tag: def handle_startendtag(self, tag, attrs): diff --git a/Lib/http/client.py b/Lib/http/client.py index 33a858d34ae1ba8..50f7862d3d2ab86 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -111,6 +111,18 @@ _MAXLINE = 65536 _MAXHEADERS = 100 +# maximal number of interim (1xx) responses tolerated before the final +# response. Real servers send at most a few; without a bound, a server +# streaming "100 Continue" responses would hang getresponse() forever. +# A socket timeout cannot detect that as data keeps arriving within every +# timeout window. +_MAXINTERIMRESPONSES = 100 + +# Data larger than this will be read in chunks, to prevent extreme +# overallocation. +_MIN_READ_BUF_SIZE = 1 << 20 + + # Header name/value ABNF (http://tools.ietf.org/html/rfc7230#section-3.2) # # VCHAR = %x21-7E @@ -230,7 +242,7 @@ def _read_headers(fp): def _parse_header_lines(header_lines, _class=HTTPMessage): """ - Parses only RFC2822 headers from header lines. + Parses only RFC 5322 headers from header lines. email Parser wants to see strings rather than bytes. But a TextIOWrapper around self.rfile would buffer too many bytes @@ -243,7 +255,7 @@ def _parse_header_lines(header_lines, _class=HTTPMessage): return email.parser.Parser(_class=_class).parsestr(hstring) def parse_headers(fp, _class=HTTPMessage): - """Parses only RFC2822 headers from a file pointer.""" + """Parses only RFC 5322 headers from a file pointer.""" headers = _read_headers(fp) return _parse_header_lines(headers, _class) @@ -327,7 +339,7 @@ def begin(self): return # read until we get a non-100 response - while True: + for _ in range(_MAXINTERIMRESPONSES): version, status, reason = self._read_status() if status != CONTINUE: break @@ -336,6 +348,9 @@ def begin(self): if self.debuglevel > 0: print("headers:", skipped_headers) del skipped_headers + else: + raise HTTPException( + f"got more than {_MAXINTERIMRESPONSES} interim responses") self.code = self.status = status self.reason = reason.strip() @@ -553,6 +568,7 @@ def _read_next_chunk_size(self): def _read_and_discard_trailer(self): # read and discard trailer up to the CRLF terminator ### note: we shouldn't have any trailers! + trailers_read = 0 while True: line = self.fp.readline(_MAXLINE + 1) if len(line) > _MAXLINE: @@ -563,6 +579,14 @@ def _read_and_discard_trailer(self): break if line in (b'\r\n', b'\n', b''): break + # Bound the trailer count just as response headers are bounded. + # A server streaming trailer lines forever would otherwise hang + # the client; a socket timeout cannot detect that as data keeps + # arriving within every timeout window. + trailers_read += 1 + if trailers_read > _MAXHEADERS: + raise HTTPException( + f"got more than {_MAXHEADERS} trailers") def _get_chunk_left(self): # return self.chunk_left, reading a new chunk if necessary. @@ -639,10 +663,25 @@ def _safe_read(self, amt): reading. If the bytes are truly not available (due to EOF), then the IncompleteRead exception can be used to detect the problem. """ - data = self.fp.read(amt) - if len(data) < amt: - raise IncompleteRead(data, amt-len(data)) - return data + cursize = min(amt, _MIN_READ_BUF_SIZE) + data = self.fp.read(cursize) + if len(data) >= amt: + return data + if len(data) < cursize: + raise IncompleteRead(data, amt - len(data)) + + data = io.BytesIO(data) + data.seek(0, 2) + while True: + # This is a geometric increase in read size (never more than + # doubling out the current length of data per loop iteration). + delta = min(cursize, amt - cursize) + data.write(self.fp.read(delta)) + if data.tell() >= amt: + return data.getvalue() + cursize += delta + if data.tell() < cursize: + raise IncompleteRead(data.getvalue(), amt - data.tell()) def _safe_readinto(self, b): """Same as _safe_read, but for reading into a buffer.""" @@ -952,13 +991,22 @@ def _wrap_ipv6(self, ip): return ip def _tunnel(self): + if _contains_disallowed_url_pchar_re.search(self._tunnel_host): + raise ValueError('Tunnel host can\'t contain control characters %r' + % (self._tunnel_host,)) connect = b"CONNECT %s:%d %s\r\n" % ( self._wrap_ipv6(self._tunnel_host.encode("idna")), self._tunnel_port, self._http_vsn_str.encode("ascii")) headers = [connect] for header, value in self._tunnel_headers.items(): - headers.append(f"{header}: {value}\r\n".encode("latin-1")) + header_bytes = header.encode("latin-1") + value_bytes = value.encode("latin-1") + if not _is_legal_header_name(header_bytes): + raise ValueError('Invalid header name %r' % (header_bytes,)) + if _is_illegal_header_value(value_bytes): + raise ValueError('Invalid header value %r' % (value_bytes,)) + headers.append(b"%s: %s\r\n" % (header_bytes, value_bytes)) headers.append(b"\r\n") # Making a single send() call instead of one per line encourages # the host OS to use a more optimal packet size instead of diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index 694b1b09a0567c7..abebb4b69fd05f4 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -87,9 +87,9 @@ such trickeries do not confuse it. >>> C = cookies.SimpleCookie() - >>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=\\012;";') + >>> C.load('keebler="E=everybody; L=\\"Loves\\"; fudge=;";') >>> print(C) - Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=\012;" + Set-Cookie: keebler="E=everybody; L=\"Loves\"; fudge=;" Each element of the Cookie also supports all of the RFC 2109 Cookie attributes. Here's an example which sets the Path @@ -170,6 +170,15 @@ class CookieError(Exception): }) _is_legal_key = re.compile('[%s]+' % re.escape(_LegalChars)).fullmatch +_control_character_re = re.compile(r'[\x00-\x1F\x7F]') + + +def _has_control_character(*val): + """Detects control characters within a value. + Supports any type, as header values can be any type. + """ + return any(_control_character_re.search(str(v)) for v in val) + def _quote(str): r"""Quote a string for use in a cookie header. @@ -294,12 +303,16 @@ def __setitem__(self, K, V): K = K.lower() if not K in self._reserved: raise CookieError("Invalid attribute %r" % (K,)) + if _has_control_character(K, V): + raise CookieError(f"Control characters are not allowed in cookies {K!r} {V!r}") dict.__setitem__(self, K, V) def setdefault(self, key, val=None): key = key.lower() if key not in self._reserved: raise CookieError("Invalid attribute %r" % (key,)) + if _has_control_character(key, val): + raise CookieError("Control characters are not allowed in cookies %r %r" % (key, val,)) return dict.setdefault(self, key, val) def __eq__(self, morsel): @@ -324,9 +337,16 @@ def update(self, values): key = key.lower() if key not in self._reserved: raise CookieError("Invalid attribute %r" % (key,)) + if _has_control_character(key, val): + raise CookieError("Control characters are not allowed in " + f"cookies {key!r} {val!r}") data[key] = val dict.update(self, data) + def __ior__(self, values): + self.update(values) + return self + def isReservedKey(self, K): return K.lower() in self._reserved @@ -335,6 +355,9 @@ def set(self, key, val, coded_val): raise CookieError('Attempt to set a reserved key %r' % (key,)) if not _is_legal_key(key): raise CookieError('Illegal key %r' % (key,)) + if _has_control_character(key, val, coded_val): + raise CookieError( + "Control characters are not allowed in cookies %r %r %r" % (key, val, coded_val,)) # It's a good key, so save it. self._key = key @@ -349,9 +372,15 @@ def __getstate__(self): } def __setstate__(self, state): - self._key = state['key'] - self._value = state['value'] - self._coded_value = state['coded_value'] + key = state['key'] + value = state['value'] + coded_value = state['coded_value'] + if _has_control_character(key, value, coded_value): + raise CookieError("Control characters are not allowed in cookies " + f"{key!r} {value!r} {coded_value!r}") + self._key = key + self._value = value + self._coded_value = coded_value def output(self, attrs=None, header="Set-Cookie:"): return "%s %s" % (header, self.OutputString(attrs)) @@ -362,14 +391,21 @@ def __repr__(self): return '<%s: %s>' % (self.__class__.__name__, self.OutputString()) def js_output(self, attrs=None): + import urllib.parse # Print javascript + output_string = self.OutputString(attrs) + if _has_control_character(output_string): + raise CookieError("Control characters are not allowed in cookies") + # Base64-encode value to avoid template + # injection in cookie values. + output_encoded = urllib.parse.quote(output_string, safe='', encoding='utf-8') return """ - """ % (self.OutputString(attrs).replace('"', r'\"')) + """ % (output_encoded,) def OutputString(self, attrs=None): # Build up our result @@ -488,7 +524,10 @@ def output(self, attrs=None, header="Set-Cookie:", sep="\015\012"): result = [] items = sorted(self.items()) for key, value in items: - result.append(value.output(attrs, header)) + value_output = value.output(attrs, header) + if _has_control_character(value_output): + raise CookieError("Control characters are not allowed in cookies") + result.append(value_output) return sep.join(result) __str__ = output diff --git a/Lib/http/server.py b/Lib/http/server.py index 64f766f9bc2c1b4..118e039aef3be39 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -134,10 +134,14 @@ DEFAULT_ERROR_CONTENT_TYPE = "text/html;charset=utf-8" +# Data larger than this will be read in chunks, to prevent extreme +# overallocation. +_MIN_READ_BUF_SIZE = 1 << 20 + class HTTPServer(socketserver.TCPServer): allow_reuse_address = True # Seems to make sense in testing environment - allow_reuse_port = True + allow_reuse_port = False def server_bind(self): """Override server_bind to store the server name.""" @@ -324,6 +328,7 @@ def parse_request(self): error response has already been sent back. """ + is_http_0_9 = False self.command = None # set in case of error on the first line self.request_version = version = self.default_request_version self.close_connection = True @@ -355,10 +360,13 @@ def parse_request(self): raise ValueError("unreasonable length http version") version_number = int(version_number[0]), int(version_number[1]) except (ValueError, IndexError): + # Send the error response with a status line and headers. + self.request_version = '' self.send_error( HTTPStatus.BAD_REQUEST, "Bad request version (%r)" % version) return False + self.request_version = version if version_number >= (1, 1) and self.protocol_version >= "HTTP/1.1": self.close_connection = False if version_number >= (2, 0): @@ -366,9 +374,9 @@ def parse_request(self): HTTPStatus.HTTP_VERSION_NOT_SUPPORTED, "Invalid HTTP version (%s)" % base_version_number) return False - self.request_version = version if not 2 <= len(words) <= 3: + self.request_version = '' self.send_error( HTTPStatus.BAD_REQUEST, "Bad request syntax (%r)" % requestline) @@ -377,10 +385,12 @@ def parse_request(self): if len(words) == 2: self.close_connection = True if command != 'GET': + self.request_version = '' self.send_error( HTTPStatus.BAD_REQUEST, "Bad HTTP/0.9 request type (%r)" % command) return False + is_http_0_9 = True self.command, self.path = command, path # gh-87389: The purpose of replacing '//' with '/' is to protect @@ -390,6 +400,11 @@ def parse_request(self): if self.path.startswith('//'): self.path = '/' + self.path.lstrip('/') # Reduce to a single / + # For HTTP/0.9, headers are not expected at all. + if is_http_0_9: + self.headers = {} + return True + # Examine the headers and look for a Connection directive. try: self.headers = http.client.parse_headers(self.rfile, @@ -750,7 +765,7 @@ def send_head(self): f = None if os.path.isdir(path): parts = urllib.parse.urlsplit(self.path) - if not parts.path.endswith('/'): + if not parts.path.endswith(('/', '%2f', '%2F')): # redirect browser - doing basically what apache does self.send_response(HTTPStatus.MOVED_PERMANENTLY) new_parts = (parts[0], parts[1], parts[2] + '/', @@ -840,11 +855,14 @@ def list_directory(self, path): return None list.sort(key=lambda a: a.lower()) r = [] + displaypath = self.path + displaypath = displaypath.split('#', 1)[0] + displaypath = displaypath.split('?', 1)[0] try: - displaypath = urllib.parse.unquote(self.path, + displaypath = urllib.parse.unquote(displaypath, errors='surrogatepass') except UnicodeDecodeError: - displaypath = urllib.parse.unquote(self.path) + displaypath = urllib.parse.unquote(displaypath) displaypath = html.escape(displaypath, quote=False) enc = sys.getfilesystemencoding() title = f'Directory listing for {displaypath}' @@ -890,14 +908,14 @@ def translate_path(self, path): """ # abandon query parameters - path = path.split('?',1)[0] - path = path.split('#',1)[0] + path = path.split('#', 1)[0] + path = path.split('?', 1)[0] # Don't forget explicit trailing slash when normalizing. Issue17324 - trailing_slash = path.rstrip().endswith('/') try: path = urllib.parse.unquote(path, errors='surrogatepass') except UnicodeDecodeError: path = urllib.parse.unquote(path) + trailing_slash = path.endswith('/') path = posixpath.normpath(path) words = path.split('/') words = filter(None, words) @@ -1274,7 +1292,18 @@ def run_cgi(self): env = env ) if self.command.lower() == "post" and nbytes > 0: - data = self.rfile.read(nbytes) + cursize = 0 + data = self.rfile.read(min(nbytes, _MIN_READ_BUF_SIZE)) + while len(data) < nbytes and len(data) != cursize: + cursize = len(data) + # This is a geometric increase in read size (never more + # than doubling out the current length of data per loop + # iteration). + delta = min(cursize, nbytes - cursize) + try: + data += self.rfile.read(delta) + except TimeoutError: + break else: data = None # throw away additional data [see bug #427345] @@ -1317,8 +1346,8 @@ def test(HandlerClass=BaseHTTPRequestHandler, HandlerClass.protocol_version = protocol if tls_cert: - server = ThreadingHTTPSServer(addr, HandlerClass, certfile=tls_cert, - keyfile=tls_key, password=tls_password) + server = ServerClass(addr, HandlerClass, certfile=tls_cert, + keyfile=tls_key, password=tls_password) else: server = ServerClass(addr, HandlerClass) @@ -1384,7 +1413,7 @@ def test(HandlerClass=BaseHTTPRequestHandler, handler_class = SimpleHTTPRequestHandler # ensure dual-stack is not disabled; ref #38907 - class DualStackServer(ThreadingHTTPServer): + class DualStackServerMixin: def server_bind(self): # suppress exception when protocol is IPv4 @@ -1397,9 +1426,16 @@ def finish_request(self, request, client_address): self.RequestHandlerClass(request, client_address, self, directory=args.directory) + class HTTPDualStackServer(DualStackServerMixin, ThreadingHTTPServer): + pass + class HTTPSDualStackServer(DualStackServerMixin, ThreadingHTTPSServer): + pass + + ServerClass = HTTPSDualStackServer if args.tls_cert else HTTPDualStackServer + test( HandlerClass=handler_class, - ServerClass=DualStackServer, + ServerClass=ServerClass, port=args.port, bind=args.bind, protocol=args.protocol, diff --git a/Lib/idlelib/CREDITS.txt b/Lib/idlelib/CREDITS.txt index bea3ba7c20de22e..1b853e8cc1c4621 100644 --- a/Lib/idlelib/CREDITS.txt +++ b/Lib/idlelib/CREDITS.txt @@ -37,6 +37,7 @@ Major contributors since 2005: - 2014: Saimadhav Heblikar - 2015: Mark Roseman - 2017: Louie Lu, Cheryl Sabella, and Serhiy Storchaka +- 2025: Stan Ulbrych For additional details refer to NEWS.txt and Changelog. diff --git a/Lib/idlelib/HISTORY.txt b/Lib/idlelib/HISTORY.txt index a601b25b5f838f9..8bd4e890260de46 100644 --- a/Lib/idlelib/HISTORY.txt +++ b/Lib/idlelib/HISTORY.txt @@ -1,5 +1,5 @@ -IDLE History -============ +IDLE History from Python 1.5.2 to 2.1 +===================================== This file contains the release messages for previous IDLE releases. As you read on you go back to the dark ages of IDLE's history. @@ -59,8 +59,8 @@ IDLEfork 0.7.1 - 29 May 2000 get to keep both pieces. - If you have problems or suggestions, you should either contact me or post to - the list at http://www.python.org/mailman/listinfo/idle-dev (making it clear - that you are using this modified version of IDLE). + the idle-dev mailing list (making it clear that you are using this modified + version of IDLE). - Changes: @@ -164,7 +164,7 @@ with Python 1.5.2; you can drop this version on top of it.) COPYRIGHT IDLE is covered by the standard Python copyright notice -(http://www.python.org/doc/Copyright.html). +(https://www.python.org/doc/copyright). New in IDLE 0.5 (2/15/2000) diff --git a/Lib/idlelib/Icons/idle_256.png b/Lib/idlelib/Icons/idle_256.png index 64f276b40d23b79..84ffdfddc6f9181 100644 Binary files a/Lib/idlelib/Icons/idle_256.png and b/Lib/idlelib/Icons/idle_256.png differ diff --git a/Lib/idlelib/NEWS2x.txt b/Lib/idlelib/NEWS2x.txt index 6751ca5f111b187..3721193007e59b7 100644 --- a/Lib/idlelib/NEWS2x.txt +++ b/Lib/idlelib/NEWS2x.txt @@ -1,6 +1,6 @@ What's New in IDLE 2.7? (Merged into 3.1 before 2.7 release.) ======================= -*Release date: XX-XXX-2010* +*Release date: 03-Jul-2010* - idle.py modified and simplified to better support developing experimental versions of IDLE which are not installed in the standard location. diff --git a/Lib/idlelib/News3.txt b/Lib/idlelib/News3.txt index 74d84b3893125a2..2f9bb6d1bbb3b4e 100644 --- a/Lib/idlelib/News3.txt +++ b/Lib/idlelib/News3.txt @@ -1,8 +1,74 @@ +What's New in IDLE 3.14.z +(since 3.14.0) +Released after 2025-10-07 +========================= + + +gh-152745: When "Run... Customized" with "Restart shell" unchecked +while Shell is running code, including waiting for an input('prompt:') +response, just report that the shell is executing instead of +reporting twice and restarting anyway. Patch by Serhiy Storchaka. + +gh-152743: When an integer entry in IDLE's Settings dialog, such as +"Auto squeeze min lines", is deleted, do not save '', which is an +invalid configuration value. Patch by Serhiy Storchaka. + +gh-152742: MakeChange the default extension when saving a Shell or +Output window '.txt' and list text files before Python files. Their +content is not Python source. Patch by Serhiy Storchaka and Claude Code. + +gh-152740: Fill the "In files:" field of IDLE's Find in Files dialog +with a full directory path, even for an unsaved editor or the Shell. +In the grep output whow which directory was searched. Patch by Serhiy +Storchaka and Claude Code. + +gh-152739: Omit the idlelib directory from the path of the IDLE user +process. User code imports such as `import help' can no longer import +an idlelib submodule (here idlelib.help) instead of the intended +top-level module. Patch by Serhiy Storchaka and Claude Code. + +gh-152738: Fix a rare crash in the IDLE editor when the completion +window is closed: deleting a key binding for a sequence that is not +bound to the virtual event is now ignored instead of raising a +ValueError. Patch by Serhiy Storchaka and Claude Code. + +gh-152737: Fix Replace All in the IDLE editor's Replace dialog when the +search direction is "Up" and "Wrap around" is off: it now replaces all +matches above the current position instead of only the first one. +Patch by Serhiy Storchaka and Claude Code. + +gh-152733: For X11 window managers, set toplevel window classes to +"Idle". Patch by Serhiy Storchaka and Claude Code. + +gh-152728: Move functions run.fix_scaling, editor.fixwordbreaks +(as 'fix_word_breaks') and pyshell.fix_x11_paste to module util. +Patch by Terry J. Reedy. + +gh-85320: IDLE now reads and writes its configuration files and the +breakpoints file using UTF-8 instead of the locale encoding. +Files with non-ASCII characters and non-UTF-8 encoding may need +to be opened in an editor and resaved with UTF-8 encoding. + +gh-143774: Better explain the operation of Format / Format Paragraph. +Patch by Terry J. Reedy. + +gh-139742: Colorize t-string prefixes for template strings in IDLE, +as done for f-string prefixes. Patch by Anuradha Agrawal. + What's New in IDLE 3.14.0 (since 3.13.0) Released on 2025-10-07 ========================= +gh-129873: Simplify displaying the IDLE doc by only copying the text +section of idle.html to idlelib/help.html. Patch by Stan Ulbrych. + +gh-112936: IDLE - Include Shell menu in single-process mode, +though with Restart Shell and View Last Restart disabled. +Patch by Zhikang Yan. + +gh-112938: IDLE - Fix uninteruptable hang when Shell gets +rapid continuous output. gh-127060: Set TERM environment variable to 'dumb' to not add ANSI escape sequences for text color in tracebacks. IDLE does not understand them. @@ -19,9 +85,6 @@ Released on 2024-10-07 gh-120104: Fix padding in config and search dialog windows in IDLE. -gh-129873: Simplify displaying the IDLE doc by only copying the text -section of idle.html to idlelib/help.html. Patch by Stan Ulbrych. - gh-120083: Add explicit black IDLE Hovertip foreground color needed for recent macOS. Fixes Sonoma showing unreadable white on pale yellow. Patch by John Riggles. diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index b4df353012b7880..bffa2ddd3cd9cd2 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -47,7 +47,7 @@ def make_pat(): name not in keyword.kwlist] builtin = r"([^.'\"\\#]\b|^)" + any("BUILTIN", builtinlist) + r"\b" comment = any("COMMENT", [r"#[^\n]*"]) - stringprefix = r"(?i:r|u|f|fr|rf|b|br|rb)?" + stringprefix = r"(?i:r|u|f|fr|rf|b|br|rb|t|rt|tr)?" sqstring = stringprefix + r"'[^'\\\n]*(\\.[^'\\\n]*)*'?" dqstring = stringprefix + r'"[^"\\\n]*(\\.[^"\\\n]*)*"?' sq3string = stringprefix + r"'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?" diff --git a/Lib/idlelib/config-main.def b/Lib/idlelib/config-main.def index 28ae94161d5c035..54bdce34af31e08 100644 --- a/Lib/idlelib/config-main.def +++ b/Lib/idlelib/config-main.def @@ -34,9 +34,8 @@ # relevant settings from the default file. # # Additional help sources are listed in the [HelpFiles] section below -# and should be viewable by a web browser (or the Windows Help viewer in -# the case of .chm files). These sources will be listed on the Help -# menu. The pattern, and two examples, are: +# and should be viewable by a web browser. These sources will be listed +# on the Help menu. The pattern, and two examples, are: # # # 1 = IDLE;C:/Programs/Python36/Lib/idlelib/help.html diff --git a/Lib/idlelib/config.py b/Lib/idlelib/config.py index d10c88a43f9231d..82afd6c49269d2d 100644 --- a/Lib/idlelib/config.py +++ b/Lib/idlelib/config.py @@ -73,8 +73,9 @@ def GetOptionList(self, section): def Load(self): "Load the configuration file from disk." - if self.file: - self.read(self.file) + if self.file and os.path.exists(self.file): + with open(self.file, encoding='utf-8', errors='replace') as f: + self.read_file(f) class IdleUserConfParser(IdleConfParser): """ @@ -133,10 +134,10 @@ def Save(self): if fname and fname[0] != '#': if not self.IsEmpty(): try: - cfgFile = open(fname, 'w') + cfgFile = open(fname, 'w', encoding='utf-8') except OSError: os.unlink(fname) - cfgFile = open(fname, 'w') + cfgFile = open(fname, 'w', encoding='utf-8') with cfgFile: self.write(cfgFile) elif os.path.exists(self.file): @@ -476,34 +477,58 @@ def GetExtensionKeys(self, extensionName): Keybindings come from GetCurrentKeySet() active key dict, where previously used bindings are disabled. """ - keysName = extensionName + '_cfgBindings' - activeKeys = self.GetCurrentKeySet() - extKeys = {} - if self.defaultCfg['extensions'].has_section(keysName): - eventNames = self.defaultCfg['extensions'].GetOptionList(keysName) - for eventName in eventNames: - event = '<<' + eventName + '>>' - binding = activeKeys[event] - extKeys[event] = binding - return extKeys - - def __GetRawExtensionKeys(self,extensionName): + bindings_section = f'{extensionName}_cfgBindings' + current_keyset = self.GetCurrentKeySet() + extension_keys = {} + + event_names = set() + if self.userCfg['extensions'].has_section(bindings_section): + event_names |= set( + self.userCfg['extensions'].GetOptionList(bindings_section) + ) + if self.defaultCfg['extensions'].has_section(bindings_section): + event_names |= set( + self.defaultCfg['extensions'].GetOptionList(bindings_section) + ) + + for event_name in event_names: + event = f'<<{event_name}>>' + binding = current_keyset.get(event, None) + if binding is None: + continue + extension_keys[event] = binding + return extension_keys + + def __GetRawExtensionKeys(self, extension_name): """Return dict {configurable extensionName event : keybinding list}. Events come from default config extension_cfgBindings section. Keybindings list come from the splitting of GetOption, which tries user config before default config. """ - keysName = extensionName+'_cfgBindings' - extKeys = {} - if self.defaultCfg['extensions'].has_section(keysName): - eventNames = self.defaultCfg['extensions'].GetOptionList(keysName) - for eventName in eventNames: - binding = self.GetOption( - 'extensions', keysName, eventName, default='').split() - event = '<<' + eventName + '>>' - extKeys[event] = binding - return extKeys + bindings_section = f'{extension_name}_cfgBindings' + extension_keys = {} + + event_names = set() + if self.userCfg['extensions'].has_section(bindings_section): + event_names |= set( + self.userCfg['extensions'].GetOptionList(bindings_section) + ) + if self.defaultCfg['extensions'].has_section(bindings_section): + event_names |= set( + self.defaultCfg['extensions'].GetOptionList(bindings_section) + ) + + for event_name in event_names: + binding = self.GetOption( + 'extensions', + bindings_section, + event_name, + default='', + ).split() + event = f'<<{event_name}>>' + extension_keys[event] = binding + return extension_keys def GetExtensionBindings(self, extensionName): """Return dict {extensionName event : active or defined keybinding}. @@ -512,18 +537,30 @@ def GetExtensionBindings(self, extensionName): configurable events (from default config) to GetOption splits, as in self.__GetRawExtensionKeys. """ - bindsName = extensionName + '_bindings' - extBinds = self.GetExtensionKeys(extensionName) - #add the non-configurable bindings - if self.defaultCfg['extensions'].has_section(bindsName): - eventNames = self.defaultCfg['extensions'].GetOptionList(bindsName) - for eventName in eventNames: - binding = self.GetOption( - 'extensions', bindsName, eventName, default='').split() - event = '<<' + eventName + '>>' - extBinds[event] = binding - - return extBinds + bindings_section = f'{extensionName}_bindings' + extension_keys = self.GetExtensionKeys(extensionName) + + # add the non-configurable bindings + event_names = set() + if self.userCfg['extensions'].has_section(bindings_section): + event_names |= set( + self.userCfg['extensions'].GetOptionList(bindings_section) + ) + if self.defaultCfg['extensions'].has_section(bindings_section): + event_names |= set( + self.defaultCfg['extensions'].GetOptionList(bindings_section) + ) + + for event_name in event_names: + binding = self.GetOption( + 'extensions', + bindings_section, + event_name, + default='' + ).split() + event = f'<<{event_name}>>' + extension_keys[event] = binding + return extension_keys def GetKeyBinding(self, keySetName, eventStr): """Return the keybinding list for keySetName eventStr. diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 4d2adb48570d495..4c94d9be69e95e1 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -435,7 +435,7 @@ def on_fontlist_select(self, event): self.font_name.set(font.lower()) def set_samples(self, event=None): - """Update update both screen samples with the font settings. + """Update both screen samples with the font settings. Called on font initialization and change events. Accesses font_name, font_size, and font_bold Variables. @@ -1960,12 +1960,15 @@ def create_page_extensions(self): def load_extensions(self): "Fill self.extensions with data from the default and user configs." self.extensions = {} + for ext_name in idleConf.GetExtensions(active_only=False): # Former built-in extensions are already filtered out. self.extensions[ext_name] = [] for ext_name in self.extensions: - opt_list = sorted(self.ext_defaultCfg.GetOptionList(ext_name)) + default = set(self.ext_defaultCfg.GetOptionList(ext_name)) + user = set(self.ext_userCfg.GetOptionList(ext_name)) + opt_list = sorted(default | user) # Bring 'enable' options to the beginning of the list. enables = [opt_name for opt_name in opt_list @@ -1975,8 +1978,12 @@ def load_extensions(self): opt_list = enables + opt_list for opt_name in opt_list: - def_str = self.ext_defaultCfg.Get( - ext_name, opt_name, raw=True) + if opt_name in default: + def_str = self.ext_defaultCfg.Get( + ext_name, opt_name, raw=True) + else: + def_str = self.ext_userCfg.Get( + ext_name, opt_name, raw=True) try: def_obj = {'True':True, 'False':False}[def_str] opt_type = 'bool' @@ -2054,10 +2061,11 @@ def set_extension_value(self, section, opt): default = opt['default'] value = opt['var'].get().strip() or default opt['var'].set(value) - # if self.defaultCfg.has_section(section): - # Currently, always true; if not, indent to return. - if (value == default): + + # Only save option in user config if it differs from the default + if self.ext_defaultCfg.has_section(section) and value == default: return self.ext_userCfg.RemoveOption(section, name) + # Set the option. return self.ext_userCfg.SetOption(section, name, value) @@ -2257,7 +2265,11 @@ def make_callback(var, config): "Return default callback function to add values to changes instance." def default_callback(*params): "Add config values to changes instance." - changes.add_option(*config, var.get()) + value = var.get() + # A blanked int entry is an empty string; do not save it as an + # invalid config value (gh-83653). + if value != '': + changes.add_option(*config, value) return default_callback def attach(self): diff --git a/Lib/idlelib/debugger.py b/Lib/idlelib/debugger.py index d90dbcd11f9f61b..1fae1d4b0adbd75 100644 --- a/Lib/idlelib/debugger.py +++ b/Lib/idlelib/debugger.py @@ -1,6 +1,6 @@ """Debug user code with a GUI interface to a subclass of bdb.Bdb. -The Idb idb and Debugger gui instances each need a reference to each +The Idb instance 'idb' and Debugger instance 'gui' need references to each other or to an rpc proxy for each other. If IDLE is started with '-n', so that user code and idb both run in the diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index c76db20c58792da..a040d791bdeb528 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -29,25 +29,12 @@ from idlelib.tree import wheel_event from idlelib.util import py_extensions from idlelib import window +from idlelib.help import _get_dochome # The default tab setting for a Text widget, in average-width characters. TK_TABWIDTH_DEFAULT = 8 -_py_version = ' (%s)' % platform.python_version() darwin = sys.platform == 'darwin' -def _sphinx_version(): - "Format sys.version_info to produce the Sphinx version string used to install the chm docs" - major, minor, micro, level, serial = sys.version_info - # TODO remove unneeded function since .chm no longer installed - release = f'{major}{minor}' - release += f'{micro}' - if level == 'candidate': - release += f'rc{serial}' - elif level != 'final': - release += f'{level[0]}{serial}' - return release - - class EditorWindow: from idlelib.percolator import Percolator from idlelib.colorizer import ColorDelegator, color_config @@ -76,44 +63,7 @@ def __init__(self, flist=None, filename=None, key=None, root=None): from idlelib.runscript import ScriptBinding if EditorWindow.help_url is None: - dochome = os.path.join(sys.base_prefix, 'Doc', 'index.html') - if sys.platform.count('linux'): - # look for html docs in a couple of standard places - pyver = 'python-docs-' + '%s.%s.%s' % sys.version_info[:3] - if os.path.isdir('/var/www/html/python/'): # "python2" rpm - dochome = '/var/www/html/python/index.html' - else: - basepath = '/usr/share/doc/' # standard location - dochome = os.path.join(basepath, pyver, - 'Doc', 'index.html') - elif sys.platform[:3] == 'win': - import winreg # Windows only, block only executed once. - docfile = '' - KEY = (rf"Software\Python\PythonCore\{sys.winver}" - r"\Help\Main Python Documentation") - try: - docfile = winreg.QueryValue(winreg.HKEY_CURRENT_USER, KEY) - except FileNotFoundError: - try: - docfile = winreg.QueryValue(winreg.HKEY_LOCAL_MACHINE, - KEY) - except FileNotFoundError: - pass - if os.path.isfile(docfile): - dochome = docfile - elif sys.platform == 'darwin': - # documentation may be stored inside a python framework - dochome = os.path.join(sys.base_prefix, - 'Resources/English.lproj/Documentation/index.html') - dochome = os.path.normpath(dochome) - if os.path.isfile(dochome): - EditorWindow.help_url = dochome - if sys.platform == 'darwin': - # Safari requires real file:-URLs - EditorWindow.help_url = 'file://' + EditorWindow.help_url - else: - EditorWindow.help_url = ("https://docs.python.org/%d.%d/" - % sys.version_info[:2]) + EditorWindow.help_url = _get_dochome() self.flist = flist root = root or flist.root self.root = root @@ -362,6 +312,9 @@ def __init__(self, flist=None, filename=None, key=None, root=None): else: self.update_menu_state('options', '*ine*umbers', 'disabled') + self.mtime = self.last_mtime() + text_frame.bind('', self.focus_in_event) + def handle_winconfig(self, event=None): self.set_width() @@ -907,9 +860,8 @@ def RemoveKeybindings(self): self.text.event_delete(event, *keylist) for extensionName in self.get_standard_extension_names(): xkeydefs = idleConf.GetExtensionBindings(extensionName) - if xkeydefs: - for event, keylist in xkeydefs.items(): - self.text.event_delete(event, *keylist) + for event, keylist in xkeydefs.items(): + self.text.event_delete(event, *keylist) def ApplyKeybindings(self): """Apply the virtual, configurable keybindings. @@ -1044,12 +996,16 @@ def open_recent_file(fn_closure=file_name): def saved_change_hook(self): short = self.short_title() long = self.long_title() + _py_version = ' (%s)' % platform.python_version() if short and long and not macosx.isCocoaTk(): # Don't use both values on macOS because # that doesn't match platform conventions. title = short + " - " + long + _py_version elif short: - title = short + if short == "IDLE Shell": + title = short + " " + platform.python_version() + else: + title = short + _py_version elif long: title = long else: @@ -1073,6 +1029,8 @@ def get_saved(self): def set_saved(self, flag): self.undo.set_saved(flag) + if flag: + self.mtime = self.last_mtime() def reset_undo(self): self.undo.reset_undo() @@ -1158,6 +1116,21 @@ def _close(self): # unless override: unregister from flist, terminate if last window self.close_hook() + def last_mtime(self): + file = self.io.filename + return os.path.getmtime(file) if file else 0 + + def focus_in_event(self, event): + mtime = self.last_mtime() + if self.mtime != mtime: + self.mtime = mtime + if self. askyesno( + 'Reload', '"%s"\n\nThis script has been modified by another program.' + '\nDo you want to reload it?' % self.io.filename, parent=self.text): + self.io.loadfile(self.io.filename) + else: + self.set_saved(False) + def load_extensions(self): self.extensions = {} self.load_standard_extensions() @@ -1649,7 +1622,7 @@ def tokeneater(self, type, token, start, end, line, self.finished = 1 def run(self): - """Return 2 lines containing block opener and and indent. + """Return 2 lines containing block opener and indent. Either the indent line or both may be None. """ @@ -1731,19 +1704,10 @@ def get_accelerator(keydefs, eventname): return s -def fixwordbreaks(root): - # On Windows, tcl/tk breaks 'words' only on spaces, as in Command Prompt. - # We want Motif style everywhere. See #21474, msg218992 and followup. - tk = root.tk - tk.call('tcl_wordBreakAfter', 'a b', 0) # make sure word.tcl is loaded - tk.call('set', 'tcl_wordchars', r'\w') - tk.call('set', 'tcl_nonwordchars', r'\W') - - -def _editor_window(parent): # htest # - # error if close master window first - timer event, after script - root = parent - fixwordbreaks(root) +def _editor_window(root): # htest # + # Error if close master window first - timer event, after script + from util import fix_word_breaks + fix_word_breaks(root) if sys.argv[1:]: filename = sys.argv[1] else: diff --git a/Lib/idlelib/filelist.py b/Lib/idlelib/filelist.py index e27e5d32a0ff633..384908f911f5153 100644 --- a/Lib/idlelib/filelist.py +++ b/Lib/idlelib/filelist.py @@ -112,12 +112,12 @@ def canonize(self, filename): def _test(): # TODO check and convert to htest + # Maybe redundant with test_filelist.FileListTest.test_new_empty. from tkinter import Tk - from idlelib.editor import fixwordbreaks - from idlelib.run import fix_scaling + from idlelib.util import fix_scaling, fix_word_breaks root = Tk() fix_scaling(root) - fixwordbreaks(root) + fix_word_breaks(root) root.withdraw() flist = FileList(root) flist.new() diff --git a/Lib/idlelib/grep.py b/Lib/idlelib/grep.py index 42048ff2395fe19..5d604faec23f32d 100644 --- a/Lib/idlelib/grep.py +++ b/Lib/idlelib/grep.py @@ -40,6 +40,20 @@ def grep(text, io=None, flist=None): dialog.open(text, searchphrase, io) +def default_glob(path): + """Return the initial "In files:" pattern for a file path (gh-80504). + + Always include a full directory so that grep output shows which + directory was searched. + """ + dir, base = os.path.split(path) + dir = os.path.abspath(dir) # An empty dir becomes the current directory. + head, tail = os.path.splitext(base) + if not tail: + tail = ".py" + return os.path.join(dir, "*" + tail) + + def walk_error(msg): "Handle os.walk error." print(msg) @@ -103,11 +117,7 @@ def open(self, text, searchphrase, io=None): path = io.filename or "" else: path = "" - dir, base = os.path.split(path) - head, tail = os.path.splitext(base) - if not tail: - tail = ".py" - self.globvar.set(os.path.join(dir, "*" + tail)) + self.globvar.set(default_glob(path)) def create_entries(self): "Create base entry widgets and add widget for search path." diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index ebff9a309d90816..eda16ac5bed118b 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -16,6 +16,12 @@ of global and local namespaces

  • configuration, browsers, and other dialogs

  • +

    The IDLE application is implemented in the idlelib package.

    +

    This is an optional module. +If it is missing from your copy of CPython, +look for documentation from your distributor (that is, +whoever provided Python to you). +If you are the distributor, see Requirements for optional modules.

    @@ -390,7 +396,7 @@

    Search and ReplaceC-space. If one types a prefix for the desired name +key is C-space. If one types a prefix for the desired name before opening the box, the first match or near miss is made visible. The result is the same as if one enters a prefix after the box is displayed. Show Completions after a quote completes @@ -437,8 +443,19 @@

    Search and Replace +

    Format block

    +

    Reformat Paragraph rewraps a block (‘paragraph’) of contiguous equally +indented non-blank comments, a similar block of text within a multiline +string, or a selected subset of either. +If needed, add a blank line to separate string from code. +Partial lines in a selection expand to complete lines. +The resulting lines have the same indent as before +but have maximum total length of N columns (characters). +Change the default N of 72 on the Window tab of IDLE Settings.

    +

    -

    Code Context

    +

    Code Context

    Within an editor window containing Python code, code context can be toggled in order to show or hide a pane at the top of the window. When shown, this pane freezes the opening lines for block code, such as those beginning with @@ -473,9 +490,9 @@

    Shell window -
  • C-c attempts to interrupt statement execution (but may fail).

  • -
  • C-d closes Shell if typed at a >>> prompt.

  • -
  • Alt-p and Alt-n (C-p and C-n on macOS) +

  • C-c attempts to interrupt statement execution (but may fail).

  • +
  • C-d closes Shell if typed at a >>> prompt.

  • +
  • Alt-p and Alt-n (C-p and C-n on macOS) retrieve to the current prompt the previous or next previously entered statement that matches anything already typed.

  • Return while the cursor is on any previous statement @@ -516,28 +533,74 @@

    Startup and Code Execution -

    Command line usage

    -
    idle.py [-c command] [-d] [-e] [-h] [-i] [-r file] [-s] [-t title] [-] [arg] ...
    -
    --c command  run command in the shell window
    --d          enable debugger and open shell window
    --e          open editor window
    --h          print help message with legal combinations and exit
    --i          open shell window
    --r file     run file in shell window
    --s          run $IDLESTARTUP or $PYTHONSTARTUP first, in shell window
    --t title    set title of shell window
    --           run stdin in shell (- must be last option before args)
    +

    Command-line usage

    +

    IDLE can be invoked from the command line with various options. The general syntax is:

    +
    python -m idlelib [options] [file ...]
     
    -

    If there are arguments:

    +

    The following options are available:

    +
    +
    +-c <command>
    +

    Run the specified Python command in the shell window. +For example, pass -c "print('Hello, World!')". +On Windows, the outer quotes must be double quotes as shown.

    +
    + +
    +
    +-d
    +

    Enable the debugger and open the shell window.

    +
    + +
    +
    +-e
    +

    Open an editor window.

    +
    + +
    +
    +-h
    +

    Print a help message with legal combinations of options and exit.

    +
    + +
    +
    +-i
    +

    Open a shell window.

    +
    + +
    +
    +-r <file>
    +

    Run the specified file in the shell window.

    +
    + +
    +
    +-s
    +

    Run the startup file (as defined by the environment variables IDLESTARTUP or PYTHONSTARTUP) before opening the shell window.

    +
    + +
    +
    +-t <title>
    +

    Set the title of the shell window.

    +
    + +
    +
    +-
    +

    Read and execute standard input in the shell window. This option must be the last one before any arguments.

    +
    + +

    If arguments are provided:

      -
    • If -, -c, or r is used, all arguments are placed in -sys.argv[1:...] and sys.argv[0] is set to '', '-c', -or '-r'. No editor window is opened, even if that is the default -set in the Options dialog.

    • -
    • Otherwise, arguments are files opened for editing and -sys.argv reflects the arguments passed to IDLE itself.

    • +
    • If -, -c, or -r is used, all arguments are placed in sys.argv[1:], +and sys.argv[0] is set to '', '-c', or '-r' respectively. +No editor window is opened, even if that is the default set in the Options dialog.

    • +
    • Otherwise, arguments are treated as files to be opened for editing, and sys.argv reflects the arguments passed to IDLE itself.

  • @@ -739,7 +802,7 @@

    Running without a subprocess

    Help and Preferences

    -

    Help sources

    +

    Help sources

    Help menu entry “IDLE Help” displays a formatted html version of the IDLE chapter of the Library Reference. The result, in a read-only tkinter text window, is close to what one sees in a web browser. @@ -798,7 +861,7 @@

    ExtensionsPEP 434).

    +sense that feature changes can be backported (see PEP 434).

    diff --git a/Lib/idlelib/help.py b/Lib/idlelib/help.py index 063a749df54bc0c..48e7eca280ebf81 100644 --- a/Lib/idlelib/help.py +++ b/Lib/idlelib/help.py @@ -23,7 +23,12 @@ copy_strip - Copy the text part of idle.html to help.html while rstripping each line. show_idlehelp - Create HelpWindow. Called in EditorWindow.help_dialog. + +_get_dochome() - Return path to docs on user's system if present, +otherwise return link to docs.python.org. """ +import os +import sys from html.parser import HTMLParser from os.path import abspath, dirname, isfile, join from platform import python_version @@ -289,6 +294,47 @@ def show_idlehelp(parent): return HelpWindow(parent, filename, 'IDLE Doc (%s)' % python_version()) +def _get_dochome(): + "Return path to local docs if present, otherwise link to docs.python.org." + + dochome = os.path.join(sys.base_prefix, 'Doc', 'index.html') + if sys.platform.count('linux'): + # look for html docs in a couple of standard places + pyver = 'python-docs-' + '%s.%s.%s' % sys.version_info[:3] + if os.path.isdir('/var/www/html/python/'): # rpm package manager + dochome = '/var/www/html/python/index.html' + else: + basepath = '/usr/share/doc/' # dnf/apt package managers + dochome = os.path.join(basepath, pyver, 'Doc', 'index.html') + + elif sys.platform[:3] == 'win': + import winreg # Windows only, block only executed once. + docfile = '' + KEY = (rf"Software\Python\PythonCore\{sys.winver}" + r"\Help\Main Python Documentation") + try: + docfile = winreg.QueryValue(winreg.HKEY_CURRENT_USER, KEY) + except FileNotFoundError: + try: + docfile = winreg.QueryValue(winreg.HKEY_LOCAL_MACHINE, KEY) + except FileNotFoundError: + pass + if os.path.isfile(docfile): + dochome = docfile + elif sys.platform == 'darwin': + # documentation may be stored inside a python framework + dochome = os.path.join(sys.base_prefix, + 'Resources/English.lproj/Documentation/index.html') + dochome = os.path.normpath(dochome) + if os.path.isfile(dochome): + if sys.platform == 'darwin': + # Safari requires real file:-URLs + return 'file://' + dochome + return dochome + else: + return "https://docs.python.org/%d.%d/" % sys.version_info[:2] + + if __name__ == '__main__': from unittest import main main('idlelib.idle_test.test_help', verbosity=2, exit=False) diff --git a/Lib/idlelib/idle_test/__init__.py b/Lib/idlelib/idle_test/__init__.py index 79b5d102dd7da5e..8eed2699c412111 100644 --- a/Lib/idlelib/idle_test/__init__.py +++ b/Lib/idlelib/idle_test/__init__.py @@ -20,8 +20,4 @@ def load_tests(loader, standard_tests, pattern): pattern='test_*.py', # Insert here. top_level_dir=top_dir) standard_tests.addTests(module_tests) -## module_tests = loader.discover(start_dir=this_dir, -## pattern='test_*.py', # Insert here. -## top_level_dir=top_dir) -## standard_tests.addTests(module_tests) return standard_tests diff --git a/Lib/idlelib/idle_test/htest.py b/Lib/idlelib/idle_test/htest.py index a7293774eecaeb9..778e5c3d84e4963 100644 --- a/Lib/idlelib/idle_test/htest.py +++ b/Lib/idlelib/idle_test/htest.py @@ -216,9 +216,9 @@ 'file': 'browser', 'kwds': {}, 'msg': textwrap.dedent(""" - "Inspect names of module, class(with superclass if applicable), - "methods and functions. Toggle nested items. Double clicking - "on items prints a traceback for an exception that is ignored.""") + Inspect names of module, class(with superclass if applicable), + methods and functions. Toggle nested items. Double clicking + on items prints a traceback for an exception that is ignored.""") } _multistatus_bar_spec = { @@ -337,7 +337,7 @@ 'file': 'tree', 'kwds': {}, 'msg': "The canvas is scrollable.\n" - "Click on folders up to to the lowest level." + "Click on folders up to the lowest level." } _undo_delegator_spec = { diff --git a/Lib/idlelib/idle_test/template.py b/Lib/idlelib/idle_test/template.py index 725a55b9c47230c..0a4bd8b8e981fc7 100644 --- a/Lib/idlelib/idle_test/template.py +++ b/Lib/idlelib/idle_test/template.py @@ -1,6 +1,5 @@ "Test , coverage %." -from idlelib import zzdummy import unittest from test.support import requires from tkinter import Tk @@ -17,7 +16,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): cls.root.update_idletasks() -## for id in cls.root.tk.call('after', 'info'): +## for id in cls.root.after_info(): ## cls.root.after_cancel(id) # Need for EditorWindow. cls.root.destroy() del cls.root diff --git a/Lib/idlelib/idle_test/test_codecontext.py b/Lib/idlelib/idle_test/test_codecontext.py index 6969ad73b01a81a..3f070da022384ec 100644 --- a/Lib/idlelib/idle_test/test_codecontext.py +++ b/Lib/idlelib/idle_test/test_codecontext.py @@ -127,7 +127,7 @@ def test_del_with_timer(self): timer = self.cc.t1 = self.text.after(10000, lambda: None) self.cc.__del__() with self.assertRaises(TclError) as cm: - self.root.tk.call('after', 'info', timer) + self.root.after_info(timer) self.assertIn("doesn't exist", str(cm.exception)) def test_reload(self): @@ -151,7 +151,7 @@ def test_toggle_code_context_event(self): eq(cc.context['bg'], self.highlight_cfg['background']) eq(cc.context.get('1.0', 'end-1c'), '') eq(cc.editwin.label, 'Hide Code Context') - eq(self.root.tk.call('after', 'info', self.cc.t1)[1], 'timer') + eq(self.root.after_info(self.cc.t1)[1], 'timer') # Toggle off. toggle() diff --git a/Lib/idlelib/idle_test/test_colorizer.py b/Lib/idlelib/idle_test/test_colorizer.py index 308bc389384d33d..b9e746a10e455ff 100644 --- a/Lib/idlelib/idle_test/test_colorizer.py +++ b/Lib/idlelib/idle_test/test_colorizer.py @@ -36,6 +36,7 @@ async def f(): await g() # All valid prefixes for unicode and byte strings should be colored. r'x', u'x', R'x', U'x', f'x', F'x' fr'x', Fr'x', fR'x', FR'x', rf'x', rF'x', Rf'x', RF'x' + tr'x', Tr'x', tR'x', TR'x', rt'x', rT'x', Rt'x', RT'x' b'x',B'x', br'x',Br'x',bR'x',BR'x', rb'x', rB'x',Rb'x',RB'x' # Invalid combinations of legal characters should be half colored. ur'x', ru'x', uf'x', fu'x', UR'x', ufr'x', rfu'x', xf'x', fx'x' @@ -259,7 +260,7 @@ def test_notify_range(self): # Colorizing already scheduled. save_id = color.after_id - eq(self.root.tk.call('after', 'info', save_id)[1], 'timer') + eq(self.root.after_info(save_id)[1], 'timer') self.assertFalse(color.colorizing) self.assertFalse(color.stop_colorizing) self.assertTrue(color.allow_colorizing) @@ -276,7 +277,7 @@ def test_notify_range(self): color.notify_range('1.0', '1.0+3c') self.assertTrue(color.stop_colorizing) self.assertIsNotNone(color.after_id) - eq(self.root.tk.call('after', 'info', color.after_id)[1], 'timer') + eq(self.root.after_info(color.after_id)[1], 'timer') # New event scheduled. self.assertNotEqual(color.after_id, save_id) @@ -296,7 +297,7 @@ def test_toggle_colorize_event(self): self.assertFalse(color.colorizing) self.assertFalse(color.stop_colorizing) self.assertTrue(color.allow_colorizing) - eq(self.root.tk.call('after', 'info', color.after_id)[1], 'timer') + eq(self.root.after_info(color.after_id)[1], 'timer') # Toggle colorizing off. color.toggle_colorize_event() @@ -323,7 +324,7 @@ def test_toggle_colorize_event(self): # Toggle on while colorizing not in progress. color.colorizing = False color.toggle_colorize_event() - eq(self.root.tk.call('after', 'info', color.after_id)[1], 'timer') + eq(self.root.after_info(color.after_id)[1], 'timer') self.assertFalse(color.colorizing) self.assertTrue(color.stop_colorizing) self.assertTrue(color.allow_colorizing) @@ -362,7 +363,7 @@ def test_recolorize(self, mock_recmain): mock_recmain.assert_called() eq(mock_recmain.call_count, 1) # Rescheduled when TODO tag still exists. - eq(self.root.tk.call('after', 'info', color.after_id)[1], 'timer') + eq(self.root.after_info(color.after_id)[1], 'timer') # No changes to text, so no scheduling added. text.tag_remove('TODO', '1.0', 'end') @@ -390,19 +391,19 @@ def test_recolorize_main(self, mock_notify): ('6.0', ('KEYWORD',)), ('6.10', ('DEFINITION',)), ('6.11', ()), ('8.0', ('STRING',)), ('8.4', ()), ('8.5', ('STRING',)), ('8.12', ()), ('8.14', ('STRING',)), - ('19.0', ('KEYWORD',)), - ('20.4', ('KEYWORD',)), ('20.16', ('KEYWORD',)),# ('20.19', ('KEYWORD',)), - #('22.4', ('KEYWORD',)), ('22.10', ('KEYWORD',)), ('22.14', ('KEYWORD',)), ('22.19', ('STRING',)), - #('23.12', ('KEYWORD',)), - ('24.8', ('KEYWORD',)), - ('25.4', ('KEYWORD',)), ('25.9', ('KEYWORD',)), - ('25.11', ('KEYWORD',)), ('25.15', ('STRING',)), - ('25.19', ('KEYWORD',)), ('25.22', ()), - ('25.24', ('KEYWORD',)), ('25.29', ('BUILTIN',)), ('25.37', ('KEYWORD',)), - ('26.4', ('KEYWORD',)), ('26.9', ('KEYWORD',)),# ('26.11', ('KEYWORD',)), ('26.14', (),), - ('27.25', ('STRING',)), ('27.38', ('STRING',)), - ('29.0', ('STRING',)), - ('30.1', ('STRING',)), + ('20.0', ('KEYWORD',)), + ('21.4', ('KEYWORD',)), ('21.16', ('KEYWORD',)),# ('21.19', ('KEYWORD',)), + #('23.4', ('KEYWORD',)), ('23.10', ('KEYWORD',)), ('23.14', ('KEYWORD',)), ('23.19', ('STRING',)), + #('24.12', ('KEYWORD',)), + ('25.8', ('KEYWORD',)), + ('26.4', ('KEYWORD',)), ('26.9', ('KEYWORD',)), + ('26.11', ('KEYWORD',)), ('26.15', ('STRING',)), + ('26.19', ('KEYWORD',)), ('26.22', ()), + ('26.24', ('KEYWORD',)), ('26.29', ('BUILTIN',)), ('26.37', ('KEYWORD',)), + ('27.4', ('KEYWORD',)), ('27.9', ('KEYWORD',)),# ('27.11', ('KEYWORD',)), ('27.14', (),), + ('28.25', ('STRING',)), ('28.38', ('STRING',)), + ('30.0', ('STRING',)), + ('31.1', ('STRING',)), # SYNC at the end of every line. ('1.55', ('SYNC',)), ('2.50', ('SYNC',)), ('3.34', ('SYNC',)), ) @@ -433,7 +434,7 @@ def test_recolorize_main(self, mock_notify): eq(text.tag_nextrange('STRING', '8.12'), ('8.14', '8.17')) eq(text.tag_nextrange('STRING', '8.17'), ('8.19', '8.26')) eq(text.tag_nextrange('SYNC', '8.0'), ('8.26', '9.0')) - eq(text.tag_nextrange('SYNC', '30.0'), ('30.10', '32.0')) + eq(text.tag_nextrange('SYNC', '31.0'), ('31.10', '33.0')) def _assert_highlighting(self, source, tag_ranges): """Check highlighting of a given piece of code. diff --git a/Lib/idlelib/idle_test/test_configdialog.py b/Lib/idlelib/idle_test/test_configdialog.py index 2773ed7ce614b55..696a3b2f8f1bc23 100644 --- a/Lib/idlelib/idle_test/test_configdialog.py +++ b/Lib/idlelib/idle_test/test_configdialog.py @@ -244,6 +244,7 @@ def test_font_set(self): def test_set_samples(self): d = self.page + self.addCleanup(setattr, d, 'set_samples', Func()) # Re-mask for other tests. del d.set_samples # Unmask method for test orig_samples = d.font_sample, d.highlight_sample d.font_sample, d.highlight_sample = {}, {} @@ -257,7 +258,6 @@ def test_set_samples(self): self.assertTrue(d.font_sample == d.highlight_sample == expected) d.font_sample, d.highlight_sample = orig_samples - d.set_samples = Func() # Re-mask for other tests. class HighPageTest(unittest.TestCase): @@ -307,7 +307,7 @@ def test_load_theme_cfg(self): # builtinlist sets variable builtin_name to the CurrentTheme default. eq(d.builtin_name.get(), 'IDLE Classic') eq(d.custom_name.get(), '- no custom themes -') - eq(d.custom_theme_on.state(), ('disabled',)) + self.assertIn('disabled', d.custom_theme_on.state()) eq(d.set_theme_type.called, 1) eq(d.paint_theme_sample.called, 1) eq(d.set_highlight_target.called, 1) @@ -495,6 +495,7 @@ def test_highlight_sample_b1_motion(self): def test_set_theme_type(self): eq = self.assertEqual d = self.page + self.addCleanup(setattr, d, 'set_theme_type', Func()) # Re-mask method. del d.set_theme_type # Builtin theme selected. @@ -502,16 +503,15 @@ def test_set_theme_type(self): d.set_theme_type() eq(d.builtinlist['state'], NORMAL) eq(d.customlist['state'], DISABLED) - eq(d.button_delete_custom.state(), ('disabled',)) + self.assertIn('disabled', d.button_delete_custom.state()) # Custom theme selected. d.theme_source.set(False) d.set_theme_type() eq(d.builtinlist['state'], DISABLED) - eq(d.custom_theme_on.state(), ('selected',)) + self.assertNotIn('disabled', d.custom_theme_on.state()) eq(d.customlist['state'], NORMAL) - eq(d.button_delete_custom.state(), ()) - d.set_theme_type = Func() + self.assertNotIn('disabled', d.button_delete_custom.state()) def test_get_color(self): eq = self.assertEqual @@ -637,24 +637,23 @@ def test_create_new_and_save_new(self): def test_set_highlight_target(self): eq = self.assertEqual d = self.page + self.addCleanup(setattr, d, 'set_highlight_target', Func()) # Re-mask method. del d.set_highlight_target # Target is cursor. d.highlight_target.set('Cursor') - eq(d.fg_on.state(), ('disabled', 'selected')) - eq(d.bg_on.state(), ('disabled',)) + self.assertIn('disabled', d.fg_on.state()) + self.assertIn('disabled', d.bg_on.state()) self.assertTrue(d.fg_bg_toggle) eq(d.set_color_sample.called, 1) # Target is not cursor. d.highlight_target.set('Comment') - eq(d.fg_on.state(), ('selected',)) - eq(d.bg_on.state(), ()) + self.assertNotIn('disabled', d.fg_on.state()) + self.assertNotIn('disabled', d.bg_on.state()) self.assertTrue(d.fg_bg_toggle) eq(d.set_color_sample.called, 2) - d.set_highlight_target = Func() - def test_set_color_sample_binding(self): d = self.page scs = d.set_color_sample @@ -667,6 +666,7 @@ def test_set_color_sample_binding(self): def test_set_color_sample(self): d = self.page + self.addCleanup(setattr, d, 'set_color_sample', Func()) # Re-mask method. del d.set_color_sample d.highlight_target.set('Selected Text') d.fg_bg_toggle.set(True) @@ -674,7 +674,6 @@ def test_set_color_sample(self): self.assertEqual( d.style.lookup(d.frame_color_set['style'], 'background'), d.highlight_sample.tag_cget('hilite', 'foreground')) - d.set_color_sample = Func() def test_paint_theme_sample(self): eq = self.assertEqual @@ -745,7 +744,7 @@ def test_delete_custom(self): eq(yesno.called, 2) self.assertNotIn(theme_name, highpage) eq(idleConf.GetSectionList('user', 'highlight'), [theme_name2]) - eq(d.custom_theme_on.state(), ()) + self.assertNotIn('disabled', d.custom_theme_on.state()) eq(d.custom_name.get(), theme_name2) eq(dialog.deactivate_current_config.called, 1) eq(dialog.activate_config_changes.called, 1) @@ -758,7 +757,7 @@ def test_delete_custom(self): eq(yesno.called, 3) self.assertNotIn(theme_name, highpage) eq(idleConf.GetSectionList('user', 'highlight'), []) - eq(d.custom_theme_on.state(), ('disabled',)) + self.assertIn('disabled', d.custom_theme_on.state()) eq(d.custom_name.get(), '- no custom themes -') eq(dialog.deactivate_current_config.called, 2) eq(dialog.activate_config_changes.called, 2) @@ -809,7 +808,7 @@ def test_load_key_cfg(self): # builtinlist sets variable builtin_name to the CurrentKeys default. eq(d.builtin_name.get(), 'IDLE Classic OSX') eq(d.custom_name.get(), '- no custom keys -') - eq(d.custom_keyset_on.state(), ('disabled',)) + self.assertIn('disabled', d.custom_keyset_on.state()) eq(d.set_keys_type.called, 1) eq(d.load_keys_list.called, 1) eq(d.load_keys_list.args, ('IDLE Classic OSX', )) @@ -930,6 +929,7 @@ def test_keybinding(self): def test_set_keys_type(self): eq = self.assertEqual d = self.page + self.addCleanup(setattr, d, 'set_keys_type', Func()) # Re-mask method. del d.set_keys_type # Builtin keyset selected. @@ -937,16 +937,15 @@ def test_set_keys_type(self): d.set_keys_type() eq(d.builtinlist['state'], NORMAL) eq(d.customlist['state'], DISABLED) - eq(d.button_delete_custom_keys.state(), ('disabled',)) + self.assertIn('disabled', d.button_delete_custom_keys.state()) # Custom keyset selected. d.keyset_source.set(False) d.set_keys_type() eq(d.builtinlist['state'], DISABLED) - eq(d.custom_keyset_on.state(), ('selected',)) + self.assertNotIn('disabled', d.custom_keyset_on.state()) eq(d.customlist['state'], NORMAL) - eq(d.button_delete_custom_keys.state(), ()) - d.set_keys_type = Func() + self.assertNotIn('disabled', d.button_delete_custom_keys.state()) def test_get_new_keys(self): eq = self.assertEqual @@ -1043,7 +1042,7 @@ def test_on_bindingslist_select(self): b.event_generate('', x=x, y=y) b.event_generate('', x=x, y=y) self.assertEqual(b.get('anchor'), 'find') - self.assertEqual(d.button_new_keys.state(), ()) + self.assertNotIn('disabled', d.button_new_keys.state()) def test_create_new_key_set_and_save_new_key_set(self): eq = self.assertEqual @@ -1080,6 +1079,7 @@ def test_load_keys_list(self): eq = self.assertEqual d = self.page gks = idleConf.GetKeySet = Func() + self.addCleanup(setattr, d, 'load_keys_list', Func()) # Re-mask method. del d.load_keys_list b = d.bindingslist @@ -1115,7 +1115,6 @@ def test_load_keys_list(self): eq(b.get(0, 'end'), expected) eq(b.get('anchor'), 'spam - ') eq(b.curselection(), (2, )) - d.load_keys_list = Func() del idleConf.GetKeySet @@ -1156,7 +1155,7 @@ def test_delete_custom_keys(self): eq(yesno.called, 2) self.assertNotIn(keyset_name, keyspage) eq(idleConf.GetSectionList('user', 'keys'), [keyset_name2]) - eq(d.custom_keyset_on.state(), ()) + self.assertNotIn('disabled', d.custom_keyset_on.state()) eq(d.custom_name.get(), keyset_name2) eq(dialog.deactivate_current_config.called, 1) eq(dialog.activate_config_changes.called, 1) @@ -1169,7 +1168,7 @@ def test_delete_custom_keys(self): eq(yesno.called, 3) self.assertNotIn(keyset_name, keyspage) eq(idleConf.GetSectionList('user', 'keys'), []) - eq(d.custom_keyset_on.state(), ('disabled',)) + self.assertIn('disabled', d.custom_keyset_on.state()) eq(d.custom_name.get(), '- no custom keys -') eq(dialog.deactivate_current_config.called, 2) eq(dialog.activate_config_changes.called, 2) @@ -1365,25 +1364,25 @@ def test_set_add_delete_state(self): # Call with 0 items, 1 unselected item, 1 selected item. eq = self.assertEqual fr = self.frame + self.addCleanup(setattr, fr, 'set_add_delete_state', Func()) # Re-mask method. del fr.set_add_delete_state # Unmask method. sad = fr.set_add_delete_state h = fr.helplist h.delete(0, 'end') sad() - eq(fr.button_helplist_edit.state(), ('disabled',)) - eq(fr.button_helplist_remove.state(), ('disabled',)) + self.assertIn('disabled', fr.button_helplist_edit.state()) + self.assertIn('disabled', fr.button_helplist_remove.state()) h.insert(0, 'source') sad() - eq(fr.button_helplist_edit.state(), ('disabled',)) - eq(fr.button_helplist_remove.state(), ('disabled',)) + self.assertIn('disabled', fr.button_helplist_edit.state()) + self.assertIn('disabled', fr.button_helplist_remove.state()) h.selection_set(0) sad() - eq(fr.button_helplist_edit.state(), ()) - eq(fr.button_helplist_remove.state(), ()) - fr.set_add_delete_state = Func() # Mask method. + self.assertNotIn('disabled', fr.button_helplist_edit.state()) + self.assertNotIn('disabled', fr.button_helplist_remove.state()) def test_helplist_item_add(self): # Call without and twice with HelpSource result. @@ -1461,6 +1460,7 @@ def test_helplist_item_remove(self): def test_update_help_changes(self): fr = self.frame + self.addCleanup(setattr, fr, 'update_help_changes', Func()) # Re-mask method. del fr.update_help_changes fr.user_helplist.clear() fr.user_helplist.append(('name1', 'file1')) @@ -1469,7 +1469,6 @@ def test_update_help_changes(self): fr.update_help_changes() self.assertEqual(mainpage['HelpFiles'], {'1': 'name1;file1', '2': 'name2;file2'}) - fr.update_help_changes = Func() class VarTraceTest(unittest.TestCase): @@ -1544,6 +1543,17 @@ def test_make_callback(self): self.assertEqual(changes['main']['section']['option'], '42') changes.clear() + # gh-83653: a blank int entry is not saved as bad config data. + sv = StringVar(root) + cb = self.tracers.make_callback(sv, ('main', 'section', 'option')) + sv.set('') + cb() + self.assertNotIn('section', changes['main']) + sv.set('5') + cb() + self.assertEqual(changes['main']['section']['option'], '5') + changes.clear() + def test_attach_detach(self): tr = self.tracers iv = tr.add(self.iv, self.var_changed_increment) diff --git a/Lib/idlelib/idle_test/test_editmenu.py b/Lib/idlelib/idle_test/test_editmenu.py index 17478473a3d1b27..0d1eb9054918da7 100644 --- a/Lib/idlelib/idle_test/test_editmenu.py +++ b/Lib/idlelib/idle_test/test_editmenu.py @@ -7,7 +7,7 @@ import tkinter as tk from tkinter import ttk import unittest -from idlelib import pyshell +from idlelib.util import fix_x11_paste class PasteTest(unittest.TestCase): '''Test pasting into widgets that allow pasting. @@ -18,7 +18,7 @@ class PasteTest(unittest.TestCase): def setUpClass(cls): cls.root = root = tk.Tk() cls.root.withdraw() - pyshell.fix_x11_paste(root) + fix_x11_paste(root) cls.text = tk.Text(root) cls.entry = tk.Entry(root) cls.tentry = ttk.Entry(root) diff --git a/Lib/idlelib/idle_test/test_editor.py b/Lib/idlelib/idle_test/test_editor.py index 0dfe2f3c58befad..e28ee549f180aa0 100644 --- a/Lib/idlelib/idle_test/test_editor.py +++ b/Lib/idlelib/idle_test/test_editor.py @@ -20,7 +20,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): cls.root.update_idletasks() - for id in cls.root.tk.call('after', 'info'): + for id in cls.root.after_info(): cls.root.after_cancel(id) cls.root.destroy() del cls.root @@ -114,7 +114,7 @@ def tearDownClass(cls): cls.window._close() del cls.window cls.root.update_idletasks() - for id in cls.root.tk.call('after', 'info'): + for id in cls.root.after_info(): cls.root.after_cancel(id) cls.root.destroy() del cls.root @@ -225,7 +225,7 @@ def tearDownClass(cls): cls.window._close() del cls.window cls.root.update_idletasks() - for id in cls.root.tk.call('after', 'info'): + for id in cls.root.after_info(): cls.root.after_cancel(id) cls.root.destroy() del cls.root diff --git a/Lib/idlelib/idle_test/test_filelist.py b/Lib/idlelib/idle_test/test_filelist.py index 731f1975e50e239..e22cc3eced30822 100644 --- a/Lib/idlelib/idle_test/test_filelist.py +++ b/Lib/idlelib/idle_test/test_filelist.py @@ -16,7 +16,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): cls.root.update_idletasks() - for id in cls.root.tk.call('after', 'info'): + for id in cls.root.after_info(): cls.root.after_cancel(id) cls.root.destroy() del cls.root diff --git a/Lib/idlelib/idle_test/test_grep.py b/Lib/idlelib/idle_test/test_grep.py index d67dba76911fcfa..94f217effb656e1 100644 --- a/Lib/idlelib/idle_test/test_grep.py +++ b/Lib/idlelib/idle_test/test_grep.py @@ -37,6 +37,25 @@ def close(self): # gui method _grep = Dummy_grep() +class DefaultGlobTest(unittest.TestCase): + + def test_no_path(self): + # gh-80504: an unsaved editor or the Shell has no path, so the + # pattern uses the current directory, not just '*.py'. + self.assertEqual(grep.default_glob(''), + os.path.join(os.getcwd(), '*.py')) + + def test_full_path(self): + path = os.path.join(os.path.abspath(os.sep), 'ab', 'foo.py') + self.assertEqual(grep.default_glob(path), + os.path.join(os.path.dirname(path), '*.py')) + + def test_other_extension(self): + path = os.path.join(os.path.abspath(os.sep), 'ab', 'foo.txt') + self.assertEqual(grep.default_glob(path), + os.path.join(os.path.dirname(path), '*.txt')) + + class FindfilesTest(unittest.TestCase): @classmethod diff --git a/Lib/idlelib/idle_test/test_help.py b/Lib/idlelib/idle_test/test_help.py index ebb02b5c0d8356a..ebbaceb2eb7bbfd 100644 --- a/Lib/idlelib/idle_test/test_help.py +++ b/Lib/idlelib/idle_test/test_help.py @@ -4,7 +4,6 @@ import unittest from test.support import requires requires('gui') -from os.path import abspath, dirname, join from tkinter import Tk diff --git a/Lib/idlelib/idle_test/test_iomenu.py b/Lib/idlelib/idle_test/test_iomenu.py index e0642cf0cabef04..90a23620354f313 100644 --- a/Lib/idlelib/idle_test/test_iomenu.py +++ b/Lib/idlelib/idle_test/test_iomenu.py @@ -23,15 +23,14 @@ def setUpClass(cls): cls.root = Tk() cls.root.withdraw() cls.editwin = EditorWindow(root=cls.root) - cls.io = iomenu.IOBinding(cls.editwin) + cls.io = cls.editwin.io @classmethod def tearDownClass(cls): - cls.io.close() cls.editwin._close() del cls.editwin cls.root.update_idletasks() - for id in cls.root.tk.call('after', 'info'): + for id in cls.root.after_info(): cls.root.after_cancel(id) # Need for EditorWindow. cls.root.destroy() del cls.root diff --git a/Lib/idlelib/idle_test/test_multicall.py b/Lib/idlelib/idle_test/test_multicall.py index 67f28db6b0875cf..fcce2030579f4be 100644 --- a/Lib/idlelib/idle_test/test_multicall.py +++ b/Lib/idlelib/idle_test/test_multicall.py @@ -19,7 +19,7 @@ def setUpClass(cls): def tearDownClass(cls): del cls.mc cls.root.update_idletasks() -## for id in cls.root.tk.call('after', 'info'): +## for id in cls.root.after_info(): ## cls.root.after_cancel(id) # Need for EditorWindow. cls.root.destroy() del cls.root @@ -43,6 +43,22 @@ def test_yview(self): mctext = self.mc(self.root) self.assertIs(mctext.yview.__func__, Text.yview) + def test_event_delete_unbound_sequence(self): + # gh-89360: deleting a sequence that was not added to a virtual + # event is ignored instead of raising ValueError. + mctext = self.mc(self.root) + mctext.event_add('<>', '') + info = mctext.event_info('<>') + self.assertEqual(len(info), 1) + + # A different sequence, never added: a no-op, not an error. + mctext.event_delete('<>', '') + self.assertEqual(mctext.event_info('<>'), info) + + # The added sequence can still be deleted normally. + mctext.event_delete('<>', '') + self.assertNotIn(info[0], mctext.event_info('<>')) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/Lib/idlelib/idle_test/test_outwin.py b/Lib/idlelib/idle_test/test_outwin.py index 81f4aad7e95e955..00a7432b9088732 100644 --- a/Lib/idlelib/idle_test/test_outwin.py +++ b/Lib/idlelib/idle_test/test_outwin.py @@ -1,6 +1,7 @@ "Test outwin, coverage 76%." from idlelib import outwin +import platform import sys import unittest from test.support import requires @@ -40,8 +41,15 @@ def test_ispythonsource(self): self.assertFalse(w.ispythonsource('test.txt')) self.assertFalse(w.ispythonsource(__file__)) + def test_save_defaults_to_text(self): + # gh-65339: output is saved as text, not Python source. + io = self.window.io + self.assertEqual(io.defaultextension, '.txt') + # Text files are offered before Python files. + self.assertEqual(io.filetypes[0][0], 'Text files') + def test_window_title(self): - self.assertEqual(self.window.top.title(), 'Output') + self.assertEqual(self.window.top.title(), 'Output' + ' (%s)' % platform.python_version()) def test_maybesave(self): w = self.window diff --git a/Lib/idlelib/idle_test/test_pyshell.py b/Lib/idlelib/idle_test/test_pyshell.py index 706703965bffd6f..dec81bdbbccd67e 100644 --- a/Lib/idlelib/idle_test/test_pyshell.py +++ b/Lib/idlelib/idle_test/test_pyshell.py @@ -2,6 +2,7 @@ # Plus coverage of test_warning. Was 20% with test_openshell. from idlelib import pyshell +import os import unittest from test.support import requires from tkinter import Tk @@ -28,6 +29,14 @@ def test_restart_line_narrow(self): self.assertEqual(pyshell.restart_line(width, ''), expect) self.assertEqual(pyshell.restart_line(taglen+2, ''), expect+' =') + def test_fix_user_path(self): + # gh-134300: the idlelib directory is removed, other entries kept. + eq = self.assertEqual + idlelib_dir = os.path.dirname(os.path.abspath(pyshell.__file__)) + eq(pyshell.fix_user_path(['', '/a', idlelib_dir, '/b']), ['', '/a', '/b']) + eq(pyshell.fix_user_path(['/a', '/b']), ['/a', '/b']) + eq(pyshell.fix_user_path([idlelib_dir]), []) + class PyShellFileListTest(unittest.TestCase): @@ -40,7 +49,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): #cls.root.update_idletasks() -## for id in cls.root.tk.call('after', 'info'): +## for id in cls.root.after_info(): ## cls.root.after_cancel(id) # Need for EditorWindow. cls.root.destroy() del cls.root diff --git a/Lib/idlelib/idle_test/test_replace.py b/Lib/idlelib/idle_test/test_replace.py index 6c07389b29ad455..5342a3856ef65d9 100644 --- a/Lib/idlelib/idle_test/test_replace.py +++ b/Lib/idlelib/idle_test/test_replace.py @@ -246,32 +246,108 @@ def test_replace_backwards(self): equal(text.get('1.2', '1.5'), 'was') def test_replace_all(self): + # The default mode, forward with wrap around, replaces every + # match, both below and above the current position. + equal = self.assertEqual text = self.text pv = self.engine.patvar rv = self.dialog.replvar replace_all = self.dialog.replace_all - text.insert('insert', '\n') - text.insert('insert', text.get('1.0', 'end')*100) - pv.set('is') - rv.set('was') + text.delete('1.0', 'end') + text.insert('1.0', 'a\na\na\n') + text.mark_set('insert', '2.1') + pv.set('a') + rv.set('b') replace_all() - self.assertNotIn('is', text.get('1.0', 'end')) + equal(text.get('1.0', '3.end'), 'b\nb\nb') # Wrapped around. + # An empty regular expression is reported as an error. self.engine.revar.set(True) pv.set('') replace_all() self.assertIn('error', showerror.title) self.assertIn('Empty', showerror.message) + # An invalid replacement expression is reported as an error, + # and nothing is replaced. + text.delete('1.0', 'end') + text.insert('1.0', 'asT') pv.set('[s][T]') rv.set('\\') replace_all() + self.assertIn('error', showerror.title) + self.assertIn('Invalid Replace Expression', showerror.message) + equal(text.get('1.0', '1.end'), 'asT') + # A pattern that is not present replaces nothing. self.engine.revar.set(False) + text.delete('1.0', 'end') + text.insert('1.0', 'unchanged') pv.set('text which is not present') rv.set('foobar') replace_all() + equal(text.get('1.0', '1.end'), 'unchanged') + + def test_replace_all_backwards_no_wrap(self): + # gh-71956: 'up' without wrap replaces all matches from the start + # of the text down to the current position, not just one up. + equal = self.assertEqual + text = self.text + pv = self.engine.patvar + rv = self.dialog.replvar + replace_all = self.dialog.replace_all + self.engine.backvar.set(True) + self.engine.wrapvar.set(False) + + text.delete('1.0', 'end') + text.insert('1.0', 'a\na\na\n') + text.mark_set('insert', '2.1') + pv.set('a') + rv.set('b') + replace_all() + equal(text.get('1.0', '1.end'), 'b') # Above the cursor. + equal(text.get('2.0', '2.end'), 'b') # At the cursor. + equal(text.get('3.0', '3.end'), 'a') # Below the cursor, untouched. + + def test_replace_all_forwards_no_wrap(self): + # 'down' without wrap replaces all matches from the current + # position to the end of the text, and none before it. + equal = self.assertEqual + text = self.text + pv = self.engine.patvar + rv = self.dialog.replvar + replace_all = self.dialog.replace_all + self.engine.backvar.set(False) + self.engine.wrapvar.set(False) + + text.delete('1.0', 'end') + text.insert('1.0', 'a\na\na\n') + text.mark_set('insert', '2.1') + pv.set('a') + rv.set('b') + replace_all() + equal(text.get('1.0', '1.end'), 'a') # Before the cursor, untouched. + equal(text.get('2.0', '2.end'), 'a') # Before the cursor, untouched. + equal(text.get('3.0', '3.end'), 'b') # After the cursor. + + def test_replace_all_backwards_wrap(self): + # With wrap around, an 'up' search also replaces every match. + equal = self.assertEqual + text = self.text + pv = self.engine.patvar + rv = self.dialog.replvar + replace_all = self.dialog.replace_all + self.engine.backvar.set(True) + self.engine.wrapvar.set(True) + + text.delete('1.0', 'end') + text.insert('1.0', 'a\na\na\n') + text.mark_set('insert', '2.1') + pv.set('a') + rv.set('b') + replace_all() + equal(text.get('1.0', '3.end'), 'b\nb\nb') def test_default_command(self): text = self.text diff --git a/Lib/idlelib/idle_test/test_run.py b/Lib/idlelib/idle_test/test_run.py index 83ecbffa2a197e7..2ec43d59c60682f 100644 --- a/Lib/idlelib/idle_test/test_run.py +++ b/Lib/idlelib/idle_test/test_run.py @@ -82,6 +82,99 @@ def test_get_multiple_message(self, mock): subtests += 1 self.assertEqual(subtests, len(data2)) # All subtests ran? + def _capture_exception(self): + """Call run.print_exception() and return its stderr output.""" + with captured_stderr() as output: + with mock.patch.object(run, 'cleanup_traceback') as ct: + ct.side_effect = lambda t, e: t + run.print_exception() + return output.getvalue() + + @force_not_colorized + def test_print_exception_group_nested(self): + try: + try: + raise ExceptionGroup('inner', [ValueError('v1')]) + except ExceptionGroup as inner: + raise ExceptionGroup('outer', [inner, TypeError('t1')]) + except ExceptionGroup: + tb = self._capture_exception() + + self.assertIn('ExceptionGroup: outer (2 sub-exceptions)', tb) + self.assertIn('ExceptionGroup: inner', tb) + self.assertIn('ValueError: v1', tb) + self.assertIn('TypeError: t1', tb) + # Verify tree structure characters. + self.assertIn('+-+---------------- 1 ----------------', tb) + self.assertIn('+---------------- 2 ----------------', tb) + self.assertIn('+------------------------------------', tb) + + @force_not_colorized + def test_print_exception_group_chaining(self): + # __cause__ on a sub-exception exercises the prefixed + # chaining-message path (margin chars on separator lines). + sub = TypeError('t1') + sub.__cause__ = ValueError('original') + try: + raise ExceptionGroup('eg1', [sub]) + except ExceptionGroup: + tb = self._capture_exception() + self.assertIn('ValueError: original', tb) + self.assertIn('| The above exception was the direct cause', tb) + self.assertIn('ExceptionGroup: eg1', tb) + + # __context__ (implicit chaining) on a sub-exception. + sub = TypeError('t2') + sub.__context__ = ValueError('first') + try: + raise ExceptionGroup('eg2', [sub]) + except ExceptionGroup: + tb = self._capture_exception() + self.assertIn('ValueError: first', tb) + self.assertIn('| During handling of the above exception', tb) + self.assertIn('ExceptionGroup: eg2', tb) + + @force_not_colorized + def test_print_exception_group_seen(self): + shared = ValueError('shared') + try: + raise ExceptionGroup('eg', [shared, shared]) + except ExceptionGroup: + tb = self._capture_exception() + + self.assertIn('ValueError: shared', tb) + self.assertIn('', tb) + + @force_not_colorized + def test_print_exception_group_max_width(self): + excs = [ValueError(f'v{i}') for i in range(20)] + try: + raise ExceptionGroup('eg', excs) + except ExceptionGroup: + tb = self._capture_exception() + + self.assertIn('+---------------- 15 ----------------', tb) + self.assertIn('+---------------- ... ----------------', tb) + self.assertIn('and 5 more exceptions', tb) + self.assertNotIn('+---------------- 16 ----------------', tb) + + @force_not_colorized + def test_print_exception_group_max_depth(self): + def make_nested(depth): + if depth == 0: + return ValueError('leaf') + return ExceptionGroup(f'level{depth}', + [make_nested(depth - 1)]) + + try: + raise make_nested(15) + except ExceptionGroup: + tb = self._capture_exception() + + self.assertIn('... (max_group_depth is 10)', tb) + self.assertIn('ExceptionGroup: level15', tb) + self.assertNotIn('ValueError: leaf', tb) + # StdioFile tests. class S(str): diff --git a/Lib/idlelib/idle_test/test_runscript.py b/Lib/idlelib/idle_test/test_runscript.py index 5fc60185a663e8f..1e47f402d504f9d 100644 --- a/Lib/idlelib/idle_test/test_runscript.py +++ b/Lib/idlelib/idle_test/test_runscript.py @@ -2,6 +2,7 @@ from idlelib import runscript import unittest +from unittest import mock from test.support import requires from tkinter import Tk from idlelib.editor import EditorWindow @@ -18,7 +19,7 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): cls.root.update_idletasks() - for id in cls.root.tk.call('after', 'info'): + for id in cls.root.after_info(): cls.root.after_cancel(id) # Need for EditorWindow. cls.root.destroy() del cls.root @@ -28,6 +29,26 @@ def test_init(self): sb = runscript.ScriptBinding(ew) ew._close() + def test_run_module_event_shell_busy_no_restart(self): + # gh-82183: running without restarting the busy shell aborts. + ew = EditorWindow(root=self.root) + sb = runscript.ScriptBinding(ew) + sb.getfilename = mock.Mock(return_value='test.py') + sb.checksyntax = mock.Mock(return_value='code') + sb.tabnanny = mock.Mock(return_value=True) + sb.shell = shell = mock.Mock() + shell.executing = True + interp = shell.interp + with mock.patch.object(runscript, 'CustomRun') as customrun: + # Restart shell unchecked. + customrun.return_value.result = (['arg'], False) + result = sb.run_module_event(None, customize=True) + self.assertEqual(result, 'break') + interp.display_executing_dialog.assert_called_once() + interp.restart_subprocess.assert_not_called() + interp.runcode.assert_not_called() + ew._close() + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/Lib/idlelib/idle_test/test_sidebar.py b/Lib/idlelib/idle_test/test_sidebar.py index 4157a4b4dcdd2aa..df84554c4f1a515 100644 --- a/Lib/idlelib/idle_test/test_sidebar.py +++ b/Lib/idlelib/idle_test/test_sidebar.py @@ -11,11 +11,10 @@ from idlelib.idle_test.tkinter_testing_utils import run_in_tk_mainloop from idlelib.delegator import Delegator -from idlelib.editor import fixwordbreaks from idlelib.percolator import Percolator import idlelib.pyshell -from idlelib.pyshell import fix_x11_paste, PyShell, PyShellFileList -from idlelib.run import fix_scaling +from idlelib.pyshell import PyShell, PyShellFileList +from idlelib.util import fix_scaling, fix_word_breaks, fix_x11_paste import idlelib.sidebar from idlelib.sidebar import get_end_linenumber, get_lineno @@ -403,7 +402,7 @@ def setUpClass(cls): root.withdraw() fix_scaling(root) - fixwordbreaks(root) + fix_word_breaks(root) fix_x11_paste(root) cls.flist = flist = PyShellFileList(root) @@ -690,11 +689,15 @@ def test_mousewheel(self): last_lineno = get_end_linenumber(text) self.assertIsNotNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) - # Delta for , whose meaning is platform-dependent. + # Simulate a mouse wheel notch. Tk 8.7 replaced the X11 + # / wheel events with (whose delta is + # platform-dependent); older Tk on X11 still uses the button events. + x11_buttons = (sidebar.canvas._windowingsystem == 'x11' + and tk.TkVersion < 8.7) delta = 1 if sidebar.canvas._windowingsystem == 'aqua' else 120 # Scroll up. - if sidebar.canvas._windowingsystem == 'x11': + if x11_buttons: sidebar.canvas.event_generate('', x=0, y=0) else: sidebar.canvas.event_generate('', x=0, y=0, delta=delta) @@ -702,7 +705,7 @@ def test_mousewheel(self): self.assertIsNone(text.dlineinfo(text.index(f'{last_lineno}.0'))) # Scroll back down. - if sidebar.canvas._windowingsystem == 'x11': + if x11_buttons: sidebar.canvas.event_generate('', x=0, y=0) else: sidebar.canvas.event_generate('', x=0, y=0, delta=-delta) diff --git a/Lib/idlelib/idle_test/test_squeezer.py b/Lib/idlelib/idle_test/test_squeezer.py index 86c5d41b6297192..86c21f00bb8d00b 100644 --- a/Lib/idlelib/idle_test/test_squeezer.py +++ b/Lib/idlelib/idle_test/test_squeezer.py @@ -170,6 +170,7 @@ def test_write_not_stdout(self): def test_write_stdout(self): """Test Squeezer's overriding of the EditorWindow's write() method.""" + requires('gui') editwin = self.make_mock_editor_window() for text in ['', 'TEXT']: diff --git a/Lib/idlelib/idle_test/test_stackviewer.py b/Lib/idlelib/idle_test/test_stackviewer.py index 55f510382bf4c36..341f158031d5f42 100644 --- a/Lib/idlelib/idle_test/test_stackviewer.py +++ b/Lib/idlelib/idle_test/test_stackviewer.py @@ -21,7 +21,7 @@ def setUpClass(cls): def tearDownClass(cls): cls.root.update_idletasks() -## for id in cls.root.tk.call('after', 'info'): +## for id in cls.root.after_info(): ## cls.root.after_cancel(id) # Need for EditorWindow. cls.root.destroy() del cls.root @@ -35,6 +35,8 @@ def test_init(self): isi(stackviewer.sc, ScrolledCanvas) isi(stackviewer.item, stackviewer.StackTreeItem) isi(stackviewer.node, TreeNode) + top = stackviewer.sc.frame.winfo_toplevel() + self.assertEqual(top.winfo_class(), 'Idle') if __name__ == '__main__': diff --git a/Lib/idlelib/idle_test/test_window.py b/Lib/idlelib/idle_test/test_window.py index 5a2645b9cc27dce..d8a77817205dc39 100644 --- a/Lib/idlelib/idle_test/test_window.py +++ b/Lib/idlelib/idle_test/test_window.py @@ -29,7 +29,7 @@ def setUpClass(cls): def tearDownClass(cls): window.registry = window.WindowList() cls.root.update_idletasks() -## for id in cls.root.tk.call('after', 'info'): +## for id in cls.root.after_info(): ## cls.root.after_cancel(id) # Need for EditorWindow. cls.root.destroy() del cls.root @@ -39,6 +39,11 @@ def test_init(self): win = window.ListedToplevel(self.root) self.assertIn(win, window.registry) self.assertEqual(win.focused_widget, win) + self.assertEqual(win.winfo_class(), 'Idle') + + def test_init_class_override(self): + win = window.ListedToplevel(self.root, class_='Other') + self.assertEqual(win.winfo_class(), 'Other') if __name__ == '__main__': diff --git a/Lib/idlelib/idle_test/test_zoomheight.py b/Lib/idlelib/idle_test/test_zoomheight.py index aa5bdfb4fbd4c62..3b97c34d4ab29d6 100644 --- a/Lib/idlelib/idle_test/test_zoomheight.py +++ b/Lib/idlelib/idle_test/test_zoomheight.py @@ -21,7 +21,7 @@ def setUpClass(cls): def tearDownClass(cls): cls.editwin._close() cls.root.update_idletasks() - for id in cls.root.tk.call('after', 'info'): + for id in cls.root.after_info(): cls.root.after_cancel(id) # Need for EditorWindow. cls.root.destroy() del cls.root diff --git a/Lib/idlelib/idle_test/test_zzdummy.py b/Lib/idlelib/idle_test/test_zzdummy.py index 209d8564da06641..14c343cf9b30872 100644 --- a/Lib/idlelib/idle_test/test_zzdummy.py +++ b/Lib/idlelib/idle_test/test_zzdummy.py @@ -38,38 +38,8 @@ def __init__(self, root, text): self.text.undo_block_stop = mock.Mock() -class ZZDummyTest(unittest.TestCase): - - @classmethod - def setUpClass(cls): - requires('gui') - root = cls.root = Tk() - root.withdraw() - text = cls.text = Text(cls.root) - cls.editor = DummyEditwin(root, text) - zzdummy.idleConf.userCfg = testcfg - - @classmethod - def tearDownClass(cls): - zzdummy.idleConf.userCfg = usercfg - del cls.editor, cls.text - cls.root.update_idletasks() - for id in cls.root.tk.call('after', 'info'): - cls.root.after_cancel(id) # Need for EditorWindow. - cls.root.destroy() - del cls.root - - def setUp(self): - text = self.text - text.insert('1.0', code_sample) - text.undo_block_start.reset_mock() - text.undo_block_stop.reset_mock() - zz = self.zz = zzdummy.ZzDummy(self.editor) - zzdummy.ZzDummy.ztext = '# ignore #' - - def tearDown(self): - self.text.delete('1.0', 'end') - del self.zz +class ZZDummyMixin: + """Shared tests for ZzDummy with default and user configs.""" def checklines(self, text, value): # Verify that there are lines being checked. @@ -89,7 +59,8 @@ def test_init(self): def test_reload(self): self.assertEqual(self.zz.ztext, '# ignore #') - testcfg['extensions'].SetOption('ZzDummy', 'z-text', 'spam') + zzdummy.idleConf.userCfg['extensions'].SetOption( + 'ZzDummy', 'z-text', 'spam') zzdummy.ZzDummy.reload() self.assertEqual(self.zz.ztext, 'spam') @@ -148,5 +119,75 @@ def test_roundtrip(self): self.assertEqual(text.get('1.0', 'end-1c'), code_sample) +class ZZDummyTest(ZZDummyMixin, unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + root = cls.root = Tk() + root.withdraw() + text = cls.text = Text(cls.root) + cls.editor = DummyEditwin(root, text) + zzdummy.idleConf.userCfg = testcfg + + @classmethod + def tearDownClass(cls): + zzdummy.idleConf.userCfg = usercfg + del cls.editor, cls.text + cls.root.update_idletasks() + for id in cls.root.tk.call('after', 'info'): + cls.root.after_cancel(id) # Need for EditorWindow. + cls.root.destroy() + del cls.root + + def setUp(self): + text = self.text + text.insert('1.0', code_sample) + text.undo_block_start.reset_mock() + text.undo_block_stop.reset_mock() + zz = self.zz = zzdummy.ZzDummy(self.editor) + zzdummy.ZzDummy.ztext = '# ignore #' + + def tearDown(self): + self.text.delete('1.0', 'end') + del self.zz + + def test_exists(self): + conf = zzdummy.idleConf + self.assertEqual( + conf.GetSectionList('user', 'extensions'), []) + self.assertEqual( + conf.GetSectionList('default', 'extensions'), + ['AutoComplete', 'CodeContext', 'FormatParagraph', + 'ParenMatch', 'ZzDummy', 'ZzDummy_cfgBindings', + 'ZzDummy_bindings']) + self.assertIn("ZzDummy", conf.GetExtensions(False)) + self.assertNotIn("ZzDummy", conf.GetExtensions()) + self.assertEqual( + conf.GetExtensionKeys("ZzDummy"), {}) + self.assertEqual( + conf.GetExtensionBindings("ZzDummy"), + {'<>': ['']}) + + def test_exists_user(self): + conf = zzdummy.idleConf + conf.userCfg["extensions"].read_dict({ + "ZzDummy": {'enable': 'True'} + }) + self.assertEqual( + conf.GetSectionList('user', 'extensions'), + ["ZzDummy"]) + self.assertIn("ZzDummy", conf.GetExtensions()) + self.assertEqual( + conf.GetExtensionKeys("ZzDummy"), + {'<>': ['']}) + self.assertEqual( + conf.GetExtensionBindings("ZzDummy"), + {'<>': [''], + '<>': ['']}) + # Restore + conf.userCfg["extensions"].remove_section("ZzDummy") + + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/Lib/idlelib/idle_test/test_zzdummy_user.py b/Lib/idlelib/idle_test/test_zzdummy_user.py new file mode 100644 index 000000000000000..1d3f2ac3096fb0e --- /dev/null +++ b/Lib/idlelib/idle_test/test_zzdummy_user.py @@ -0,0 +1,108 @@ +"Test zzdummy with user config, coverage 100%." + +from idlelib import zzdummy +import unittest +from test.support import requires +from tkinter import Tk, Text +from idlelib import config + +from idlelib.idle_test.test_zzdummy import ( + ZZDummyMixin, DummyEditwin, code_sample, +) + + +real_usercfg = zzdummy.idleConf.userCfg +test_usercfg = { + 'main': config.IdleUserConfParser(''), + 'highlight': config.IdleUserConfParser(''), + 'keys': config.IdleUserConfParser(''), + 'extensions': config.IdleUserConfParser(''), +} +test_usercfg["extensions"].read_dict({ + "ZzDummy": {'enable': 'True', 'enable_shell': 'False', + 'enable_editor': 'True', 'z-text': 'Z'}, + "ZzDummy_cfgBindings": { + 'z-in': ''}, + "ZzDummy_bindings": { + 'z-out': ''}, +}) +real_defaultcfg = zzdummy.idleConf.defaultCfg +test_defaultcfg = { + 'main': config.IdleUserConfParser(''), + 'highlight': config.IdleUserConfParser(''), + 'keys': config.IdleUserConfParser(''), + 'extensions': config.IdleUserConfParser(''), +} +test_defaultcfg["extensions"].read_dict({ + "AutoComplete": {'popupwait': '2000'}, + "CodeContext": {'maxlines': '15'}, + "FormatParagraph": {'max-width': '72'}, + "ParenMatch": {'style': 'expression', + 'flash-delay': '500', 'bell': 'True'}, +}) +test_defaultcfg["main"].read_dict({ + "Theme": {"default": 1, "name": "IDLE Classic", "name2": ""}, + "Keys": {"default": 1, "name": "IDLE Classic", "name2": ""}, +}) +for key in ("keys",): + real_default = real_defaultcfg[key] + value = {name: dict(real_default[name]) for name in real_default} + test_defaultcfg[key].read_dict(value) + + +class ZZDummyTest(ZZDummyMixin, unittest.TestCase): + + @classmethod + def setUpClass(cls): + requires('gui') + root = cls.root = Tk() + root.withdraw() + text = cls.text = Text(cls.root) + cls.editor = DummyEditwin(root, text) + zzdummy.idleConf.userCfg = test_usercfg + zzdummy.idleConf.defaultCfg = test_defaultcfg + + @classmethod + def tearDownClass(cls): + zzdummy.idleConf.defaultCfg = real_defaultcfg + zzdummy.idleConf.userCfg = real_usercfg + del cls.editor, cls.text + cls.root.update_idletasks() + for id in cls.root.tk.call('after', 'info'): + cls.root.after_cancel(id) # Need for EditorWindow. + cls.root.destroy() + del cls.root + + def setUp(self): + text = self.text + text.insert('1.0', code_sample) + text.undo_block_start.reset_mock() + text.undo_block_stop.reset_mock() + zz = self.zz = zzdummy.ZzDummy(self.editor) + zzdummy.ZzDummy.ztext = '# ignore #' + + def tearDown(self): + self.text.delete('1.0', 'end') + del self.zz + + def test_exists(self): + self.assertEqual( + zzdummy.idleConf.GetSectionList('user', 'extensions'), + ['ZzDummy', 'ZzDummy_cfgBindings', 'ZzDummy_bindings']) + self.assertEqual( + zzdummy.idleConf.GetSectionList('default', 'extensions'), + ['AutoComplete', 'CodeContext', 'FormatParagraph', + 'ParenMatch']) + self.assertIn("ZzDummy", + zzdummy.idleConf.GetExtensions()) + self.assertEqual( + zzdummy.idleConf.GetExtensionKeys("ZzDummy"), + {'<>': ['']}) + self.assertEqual( + zzdummy.idleConf.GetExtensionBindings("ZzDummy"), + {'<>': [''], + '<>': ['']}) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/Lib/idlelib/iomenu.py b/Lib/idlelib/iomenu.py index 464126e2df06682..9560aaa2da0c424 100644 --- a/Lib/idlelib/iomenu.py +++ b/Lib/idlelib/iomenu.py @@ -61,6 +61,7 @@ def set_filename_change_hook(self, hook): self.filename_change_hook = hook filename = None + file_timestamp = None dirname = None def set_filename(self, filename): @@ -127,6 +128,7 @@ def loadfile(self, filename): chars = f.read() fileencoding = f.encoding eol_convention = f.newlines + file_timestamp = self.getmtime(filename) converted = False except (UnicodeDecodeError, SyntaxError): # Wait for the editor window to appear @@ -142,6 +144,7 @@ def loadfile(self, filename): chars = f.read() fileencoding = f.encoding eol_convention = f.newlines + file_timestamp = self.getmtime(filename) converted = True except OSError as err: messagebox.showerror("I/O Error", str(err), parent=self.text) @@ -170,6 +173,7 @@ def loadfile(self, filename): self.text.insert("1.0", chars) self.reset_undo() self.set_filename(filename) + self.file_timestamp = file_timestamp if converted: # We need to save the conversion results first # before being able to execute the code @@ -206,7 +210,26 @@ def save(self, event): if not self.filename: self.save_as(event) else: + # Check the time of most recent content modification so the + # user doesn't accidentally overwrite a newer version of the file. + try: + file_timestamp = self.getmtime(self.filename) + except OSError: + pass + else: + if self.file_timestamp != file_timestamp: + confirm = messagebox.askokcancel( + title="File has changed", + message=( + "The file has changed on disk since reading it!\n\n" + "Do you really want to overwrite it?"), + default=messagebox.CANCEL, + parent=self.text) + if not confirm: + return "break" + if self.writefile(self.filename): + self.file_timestamp = self.getmtime(self.filename) self.set_saved(True) try: self.editwin.store_file_breaks() @@ -219,6 +242,7 @@ def save_as(self, event): filename = self.asksavefile() if filename: if self.writefile(filename): + self.file_timestamp = self.getmtime(filename) self.set_filename(filename) self.set_saved(1) try: @@ -251,6 +275,9 @@ def writefile(self, filename): parent=self.text) return False + def getmtime(self, filename): + return os.stat(filename).st_mtime + def fixnewlines(self): """Return text with os eols. @@ -357,7 +384,16 @@ def print_window(self, event): ("All files", "*"), ) + # Output windows (Shell, Output) are not Python source, so they list + # text files first and default to ".txt" (gh-65339). + text_filetypes = ( + ("Text files", "*.txt", "TEXT"), + ("Python files", py_extensions, "TEXT"), + ("All files", "*"), + ) + defaultextension = '.py' if sys.platform == 'darwin' else '' + text_defaultextension = '.txt' def askopenfile(self): dir, base = self.defaultfilename("open") diff --git a/Lib/idlelib/macosx.py b/Lib/idlelib/macosx.py index 332952f4572cbda..428e49f3eb7d8ec 100644 --- a/Lib/idlelib/macosx.py +++ b/Lib/idlelib/macosx.py @@ -39,7 +39,7 @@ def _init_tk_type(): _tk_type = "xquartz" elif 'aqua' not in ws: _tk_type = "other" - elif 'AppKit' in root.tk.call('winfo', 'server', '.'): + elif 'AppKit' in root.winfo_server(): _tk_type = "cocoa" else: _tk_type = "carbon" diff --git a/Lib/idlelib/multicall.py b/Lib/idlelib/multicall.py index 41f818131130621..95f4fee7fbe740a 100644 --- a/Lib/idlelib/multicall.py +++ b/Lib/idlelib/multicall.py @@ -386,10 +386,11 @@ def event_delete(self, virtual, *sequences): if triplet is None: #print("Tkinter event_delete: %s" % seq, file=sys.__stderr__) widget.event_delete(self, virtual, seq) - else: + elif triplet in triplets: if func is not None: self.__binders[triplet[1]].unbind(triplet, func) triplets.remove(triplet) + # Else the sequence is not bound; ignore it (gh-89360). def event_info(self, virtual=None): if virtual is None or virtual not in self.__eventinfo: diff --git a/Lib/idlelib/outwin.py b/Lib/idlelib/outwin.py index 8baa657550de948..f7e92bb8ac0c776 100644 --- a/Lib/idlelib/outwin.py +++ b/Lib/idlelib/outwin.py @@ -78,6 +78,10 @@ class OutputWindow(EditorWindow): def __init__(self, *args): EditorWindow.__init__(self, *args) self.text.bind("<>", self.goto_file_line) + # Output is not Python source, so save it as text by default + # (gh-65339). + self.io.filetypes = self.io.text_filetypes + self.io.defaultextension = self.io.text_defaultextension # Customize EditorWindow def ispythonsource(self, filename): diff --git a/Lib/idlelib/pyshell.py b/Lib/idlelib/pyshell.py index 74a0e03994f69ab..ef3d014d936ce85 100755 --- a/Lib/idlelib/pyshell.py +++ b/Lib/idlelib/pyshell.py @@ -22,7 +22,6 @@ import linecache import os import os.path -from platform import python_version import re import socket import subprocess @@ -37,13 +36,14 @@ from idlelib.delegator import Delegator from idlelib import debugger from idlelib import debugger_r -from idlelib.editor import EditorWindow, fixwordbreaks +from idlelib.editor import EditorWindow from idlelib.filelist import FileList from idlelib.outwin import OutputWindow from idlelib import replace from idlelib import rpc from idlelib.run import idle_formatwarning, StdInputFile, StdOutputFile from idlelib.undo import UndoDelegator +from idlelib.util import fix_word_breaks # Default for testing; defaults to True in main() for running. use_subprocess = False @@ -243,12 +243,13 @@ def store_file_breaks(self): breaks = self.breakpoints filename = self.io.filename try: - with open(self.breakpointPath) as fp: + with open(self.breakpointPath, + encoding='utf-8', errors='replace') as fp: lines = fp.readlines() except OSError: lines = [] try: - with open(self.breakpointPath, "w") as new_file: + with open(self.breakpointPath, "w", encoding='utf-8') as new_file: for line in lines: if not line.startswith(filename + '='): new_file.write(line) @@ -273,7 +274,8 @@ def restore_file_breaks(self): if filename is None: return if os.path.isfile(self.breakpointPath): - with open(self.breakpointPath) as fp: + with open(self.breakpointPath, + encoding='utf-8', errors='replace') as fp: lines = fp.readlines() for line in lines: if line.startswith(filename + '='): @@ -406,6 +408,17 @@ def restart_line(width, filename): # See bpo-38141. return tag[:-2] # Remove ' ='. +def fix_user_path(path): + """Return path without the idlelib directory (gh-134300). + + That directory is on sys.path when idle.py is run as a script. + Otherwise user code could import idlelib submodules as top-level + modules, such as "import help". + """ + idlelib_dir = os.path.dirname(os.path.abspath(__file__)) + return [p for p in path if p != idlelib_dir] + + class ModifiedInterpreter(InteractiveInterpreter): def __init__(self, tkconsole): @@ -499,7 +512,6 @@ def restart_subprocess(self, with_cwd=False, filename=''): self.rpcclt.close() self.terminate_subprocess() console = self.tkconsole - was_executing = console.executing console.executing = False self.spawn_subprocess() try: @@ -567,6 +579,7 @@ def transfer_path(self, with_cwd=False): path.extend(sys.path) else: path = sys.path + path = fix_user_path(path) # gh-134300 self.runcommand("""if 1: import sys as _sys @@ -643,7 +656,7 @@ def remote_stack_viewer(self): return item = debugobj_r.StubObjectTreeItem(self.rpcclt, oid) from idlelib.tree import ScrolledCanvas, TreeNode - top = Toplevel(self.tkconsole.root) + top = Toplevel(self.tkconsole.root, class_='Idle') theme = idleConf.CurrentTheme() background = idleConf.GetHighlight(theme, 'normal')['background'] sc = ScrolledCanvas(top, bg=background, highlightthickness=0) @@ -841,7 +854,7 @@ def display_executing_dialog(self): class PyShell(OutputWindow): from idlelib.squeezer import Squeezer - shell_title = "IDLE Shell " + python_version() + shell_title = "IDLE Shell" # Override classes ColorDelegator = ModifiedColorDelegator @@ -881,9 +894,9 @@ def __init__(self, flist=None): if ms[2][0] != "shell": ms.insert(2, ("shell", "She_ll")) self.interp = ModifiedInterpreter(self) - if flist is None: + if flist is None: # TODO possible? root and flist in main. root = Tk() - fixwordbreaks(root) + fix_word_breaks(root) root.withdraw() flist = PyShellFileList(root) @@ -1452,17 +1465,6 @@ def on_squeezed_expand(self, index, text, tags): self.shell_sidebar.update_sidebar() -def fix_x11_paste(root): - "Make paste replace selection on x11. See issue #5124." - if root._windowingsystem == 'x11': - for cls in 'Text', 'Entry', 'Spinbox': - root.bind_class( - cls, - '<>', - 'catch {%W delete sel.first sel.last}\n' + - root.bind_class(cls, '<>')) - - usage_msg = """\ USAGE: idle [-deins] [-t title] [file]* @@ -1522,6 +1524,7 @@ def main(): from platform import system from idlelib import testing # bool value from idlelib import macosx + from idlelib.util import fix_scaling, fix_x11_paste global flist, root, use_subprocess @@ -1607,7 +1610,6 @@ def main(): NoDefaultRoot() root = Tk(className="Idle") root.withdraw() - from idlelib.run import fix_scaling fix_scaling(root) # set application icon @@ -1629,7 +1631,7 @@ def main(): root.wm_iconphoto(True, *icons) # start editor and/or shell windows: - fixwordbreaks(root) + fix_word_breaks(root) fix_x11_paste(root) flist = PyShellFileList(root) macosx.setupApp(root, flist) diff --git a/Lib/idlelib/query.py b/Lib/idlelib/query.py index 57230e2aaca66da..5f9bdc031e544b7 100644 --- a/Lib/idlelib/query.py +++ b/Lib/idlelib/query.py @@ -289,8 +289,6 @@ def askfilename(self, filetypes, initdir, initfile): # htest # def browse_file(self): filetypes = [ ("HTML Files", "*.htm *.html", "TEXT"), - ("PDF Files", "*.pdf", "TEXT"), - ("Windows Help Files", "*.chm"), ("Text Files", "*.txt", "TEXT"), ("All Files", "*")] path = self.pathvar.get() diff --git a/Lib/idlelib/replace.py b/Lib/idlelib/replace.py index 3716d841568d309..93d4f1c0c6c573e 100644 --- a/Lib/idlelib/replace.py +++ b/Lib/idlelib/replace.py @@ -122,12 +122,13 @@ def _replace_expand(self, m, repl): def replace_all(self, event=None): """Handle the Replace All button. - Search text for occurrences of the Find value and replace - each of them. The 'wrap around' value controls the start - point for searching. If wrap isn't set, then the searching - starts at the first occurrence after the current selection; - if wrap is set, the replacement starts at the first line. - The replacement is always done top-to-bottom in the text. + Search text for occurrences of the Find value and replace each + of them. The 'wrap around' and direction values control which + occurrences are replaced. With wrap around, every occurrence is + replaced. Without it, a forward search replaces occurrences from + the current position to the end of the text, and a backward search + replaces occurrences from the beginning of the text to the current + position. The replacement is always done top-to-bottom. """ prog = self.engine.getprog() if not prog: @@ -142,9 +143,18 @@ def replace_all(self, event=None): text.tag_remove("hit", "1.0", "end") line = res[0] col = res[1].start() + # For a backward search without wrap, replace top-to-bottom from + # the start of the text down to the first match at or above the + # current position (gh-71956). A mark tracks that stop point. + stop = None if self.engine.iswrap(): line = 1 col = 0 + elif self.engine.isback(): + stop = "replace_all_stop" + text.mark_set(stop, "%d.%d" % (line, res[1].end())) + line = 1 + col = 0 ok = True first = last = None # XXX ought to replace circular instead of top-to-bottom when wrapping @@ -152,12 +162,13 @@ def replace_all(self, event=None): while res := self.engine.search_forward( text, prog, line, col, wrap=False, ok=ok): line, m = res - chars = text.get("%d.0" % line, "%d.0" % (line+1)) + i, j = m.span() + if stop is not None and text.compare("%d.%d" % (line, i), ">=", stop): + break orig = m.group() new = self._replace_expand(m, repl) if new is None: break - i, j = m.span() first = "%d.%d" % (line, i) last = "%d.%d" % (line, j) if new == orig: @@ -170,6 +181,8 @@ def replace_all(self, event=None): text.insert(first, new, self.insert_tags) col = i + len(new) ok = False + if stop is not None: + text.mark_unset(stop) text.undo_block_stop() if first and last: self.show_hit(first, last) diff --git a/Lib/idlelib/run.py b/Lib/idlelib/run.py index a30db99a619a93b..f5564ccf90f7f46 100644 --- a/Lib/idlelib/run.py +++ b/Lib/idlelib/run.py @@ -25,6 +25,7 @@ from idlelib import iomenu # encoding from idlelib import rpc # multiple objects from idlelib import stackviewer # StackTreeItem +from idlelib import util # fix_scaling import __main__ import tkinter # Use tcl and, if startup fails, messagebox. @@ -216,7 +217,7 @@ def show_socket_error(err, address): import tkinter from tkinter.messagebox import showerror root = tkinter.Tk() - fix_scaling(root) + util.fix_scaling(root) root.withdraw() showerror( "Subprocess Connection Error", @@ -249,31 +250,94 @@ def print_exception(): sys.last_type, sys.last_value, sys.last_traceback = excinfo sys.last_exc = val seen = set() + exclude = ("run.py", "rpc.py", "threading.py", "queue.py", + "debugger_r.py", "bdb.py") + max_group_width = 15 + max_group_depth = 10 + group_depth = 0 + + def print_exc_group(typ, exc, tb, prefix=""): + nonlocal group_depth + group_depth += 1 + prefix2 = prefix or " " + if group_depth > max_group_depth: + print(f"{prefix2}| ... (max_group_depth is {max_group_depth})", + file=efile) + group_depth -= 1 + return + if tb: + if not prefix: + print(" + Exception Group Traceback (most recent call last):", file=efile) + else: + print(f"{prefix}| Exception Group Traceback (most recent call last):", file=efile) + tbe = traceback.extract_tb(tb) + cleanup_traceback(tbe, exclude) + for line in traceback.format_list(tbe): + for subline in line.rstrip().splitlines(): + print(f"{prefix2}| {subline}", file=efile) + lines = get_message_lines(typ, exc, tb) + for line in lines: + print(f"{prefix2}| {line}", end="", file=efile) + num_excs = len(exc.exceptions) + if num_excs <= max_group_width: + n = num_excs + else: + n = max_group_width + 1 + for i, sub in enumerate(exc.exceptions[:n], 1): + truncated = (i > max_group_width) + first_line_pre = "+-" if i == 1 else " " + title = str(i) if not truncated else '...' + print(f"{prefix2}{first_line_pre}+---------------- {title} ----------------", file=efile) + if truncated: + remaining = num_excs - max_group_width + plural = 's' if remaining > 1 else '' + print(f"{prefix2} | and {remaining} more exception{plural}", + file=efile) + need_print_underline = True + elif id(sub) not in seen: + if not prefix: + print_exc(type(sub), sub, sub.__traceback__, " ") + else: + print_exc(type(sub), sub, sub.__traceback__, prefix + " ") + need_print_underline = not isinstance(sub, BaseExceptionGroup) + else: + print(f"{prefix2} | ", file=efile) + need_print_underline = True + if need_print_underline and i == n: + print(f"{prefix2} +------------------------------------", file=efile) + group_depth -= 1 - def print_exc(typ, exc, tb): + def print_exc(typ, exc, tb, prefix=""): seen.add(id(exc)) context = exc.__context__ cause = exc.__cause__ + prefix2 = f"{prefix}| " if prefix else "" if cause is not None and id(cause) not in seen: - print_exc(type(cause), cause, cause.__traceback__) - print("\nThe above exception was the direct cause " - "of the following exception:\n", file=efile) + print_exc(type(cause), cause, cause.__traceback__, prefix) + print(f"{prefix2}\n{prefix2}The above exception was the direct cause " + f"of the following exception:\n{prefix2}", file=efile) elif (context is not None and not exc.__suppress_context__ and id(context) not in seen): - print_exc(type(context), context, context.__traceback__) - print("\nDuring handling of the above exception, " - "another exception occurred:\n", file=efile) - if tb: - tbe = traceback.extract_tb(tb) - print('Traceback (most recent call last):', file=efile) - exclude = ("run.py", "rpc.py", "threading.py", "queue.py", - "debugger_r.py", "bdb.py") - cleanup_traceback(tbe, exclude) - traceback.print_list(tbe, file=efile) - lines = get_message_lines(typ, exc, tb) - for line in lines: - print(line, end='', file=efile) + print_exc(type(context), context, context.__traceback__, prefix) + print(f"{prefix2}\n{prefix2}During handling of the above exception, " + f"another exception occurred:\n{prefix2}", file=efile) + if isinstance(exc, BaseExceptionGroup): + print_exc_group(typ, exc, tb, prefix=prefix) + else: + if tb: + print(f"{prefix2}Traceback (most recent call last):", file=efile) + tbe = traceback.extract_tb(tb) + cleanup_traceback(tbe, exclude) + if prefix: + for line in traceback.format_list(tbe): + for subline in line.rstrip().splitlines(): + print(f"{prefix}| {subline}", file=efile) + else: + traceback.print_list(tbe, file=efile) + lines = get_message_lines(typ, exc, tb) + for line in lines: + print(f"{prefix2}{line}", end="", file=efile) print_exc(typ, val, tb) @@ -326,16 +390,6 @@ def exit(): sys.exit(0) -def fix_scaling(root): - """Scale fonts on HiDPI displays.""" - import tkinter.font - scaling = float(root.tk.call('tk', 'scaling')) - if scaling > 1.4: - for name in tkinter.font.names(root): - font = tkinter.font.Font(root=root, name=name, exists=True) - size = int(font['size']) - if size < 0: - font['size'] = round(-0.75*size) def fixdoc(fun, text): diff --git a/Lib/idlelib/runscript.py b/Lib/idlelib/runscript.py index 55712e904603f86..cd52d206c9ca0f3 100644 --- a/Lib/idlelib/runscript.py +++ b/Lib/idlelib/runscript.py @@ -139,6 +139,10 @@ def run_module_event(self, event, *, customize=False): return 'break' self.cli_args, restart = run_args if customize else ([], True) interp = self.shell.interp + if self.shell.executing and not restart: + # Cannot run without restarting the busy shell (gh-82183). + interp.display_executing_dialog() + return 'break' if pyshell.use_subprocess and restart: interp.restart_subprocess( with_cwd=False, filename=filename) diff --git a/Lib/idlelib/stackviewer.py b/Lib/idlelib/stackviewer.py index 95042d4debdc03b..268e2f1daeb98cc 100644 --- a/Lib/idlelib/stackviewer.py +++ b/Lib/idlelib/stackviewer.py @@ -11,7 +11,7 @@ def StackBrowser(root, exc, flist=None, top=None): global sc, item, node # For testing. if top is None: - top = tk.Toplevel(root) + top = tk.Toplevel(root, class_='Idle') sc = ScrolledCanvas(top, bg="white", highlightthickness=0) sc.frame.pack(expand=1, fill="both") item = StackTreeItem(exc, flist) diff --git a/Lib/idlelib/textview.py b/Lib/idlelib/textview.py index 23f0f4cb5027ec3..b326331efa7f9a1 100644 --- a/Lib/idlelib/textview.py +++ b/Lib/idlelib/textview.py @@ -92,10 +92,10 @@ def __init__(self, parent, contents, wrap='word'): color_config(text) text.focus_set() - self.button_ok = button_ok = Button( + self.button_ok = Button( self, text='Close', command=self.ok, takefocus=False) self.textframe.pack(side='top', expand=True, fill='both') - button_ok.pack(side='bottom') + self.button_ok.pack(side='bottom') def ok(self, event=None): """Dismiss text viewer dialog.""" @@ -127,10 +127,8 @@ def __init__(self, parent, title, contents, modal=True, wrap=WORD, self.geometry(f'=750x500+{x}+{y}') self.title(title) - self.viewframe = ViewFrame(self, contents, wrap=wrap) self.protocol("WM_DELETE_WINDOW", self.ok) - self.button_ok = button_ok = Button(self, text='Close', - command=self.ok, takefocus=False) + self.viewframe = ViewFrame(self, contents, wrap=wrap) self.viewframe.pack(side='top', expand=True, fill='both') self.is_modal = modal diff --git a/Lib/idlelib/util.py b/Lib/idlelib/util.py index e05604ab4853f6d..bf88c905e1d177d 100644 --- a/Lib/idlelib/util.py +++ b/Lib/idlelib/util.py @@ -19,6 +19,20 @@ py_extensions = ('.py', '.pyw', '.pyi') +# fix_x functions seem only needed once per process. + +def fix_scaling(root): # Called in filelist _test, pyshell, and run. + """Scale fonts on HiDPI displays, once per process.""" + import tkinter.font + scaling = float(root.tk.call('tk', 'scaling')) + if scaling > 1.4: + for name in tkinter.font.names(root): + font = tkinter.font.Font(root=root, name=name, exists=True) + size = int(font['size']) + if size < 0: + font['size'] = round(-0.75*size) + + # Fix for HiDPI screens on Windows. CALL BEFORE ANY TK OPERATIONS! # URL for arguments for the ...Awareness call below. # https://msdn.microsoft.com/en-us/library/windows/desktop/dn280512(v=vs.85).aspx @@ -31,6 +45,25 @@ def fix_win_hidpi(): # Called in pyshell and turtledemo. except (ImportError, AttributeError, OSError): pass +def fix_word_breaks(root): # Called in editor htest, filelist _test, pyshell. + # On Windows, tcl/tk breaks 'words' only on spaces, as in Command Prompt. + # We want Motif style everywhere. See #21474, msg218992 and followup. + tk = root.tk + tk.call('tcl_wordBreakAfter', 'a b', 0) # make sure word.tcl is loaded + tk.call('set', 'tcl_wordchars', r'\w') + tk.call('set', 'tcl_nonwordchars', r'\W') + + +def fix_x11_paste(root): + "Make paste replace selection on x11. See issue #5124." + if root._windowingsystem == 'x11': + for cls in 'Text', 'Entry', 'Spinbox': + root.bind_class( + cls, + '<>', + 'catch {%W delete sel.first sel.last}\n' + + root.bind_class(cls, '<>')) + if __name__ == '__main__': from unittest import main diff --git a/Lib/idlelib/window.py b/Lib/idlelib/window.py index 460d5b67948dde2..a41d69927dcdd57 100644 --- a/Lib/idlelib/window.py +++ b/Lib/idlelib/window.py @@ -61,6 +61,10 @@ def call_callbacks(self): class ListedToplevel(Toplevel): def __init__(self, master, **kw): + # Set the WM_CLASS property so that X11 window managers group and + # label IDLE's windows under 'Idle' instead of the default + # 'Toplevel' (gh-66331). It matches the class name passed to Tk(). + kw.setdefault('class_', 'Idle') Toplevel.__init__(self, master, kw) registry.add(self) self.focused_widget = self diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 2c3925958d011b4..4c0aefd082f8dce 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -1,6 +1,6 @@ """IMAP4 client. -Based on RFC 2060. +Based on RFC 3501. Public class: IMAP4 Public variable: Debug @@ -21,7 +21,7 @@ # GET/SETANNOTATION contributed by Tomas Lindroos June 2005. # IDLE contributed by Forest August 2024. -__version__ = "2.59" +__version__ = "2.60" import binascii, errno, random, re, socket, subprocess, sys, time, calendar from datetime import datetime, timezone, timedelta @@ -45,8 +45,8 @@ AllowedVersions = ('IMAP4REV1', 'IMAP4') # Most recent first # Maximal line length when calling readline(). This is to prevent -# reading arbitrary length lines. RFC 3501 and 2060 (IMAP 4rev1) -# don't specify a line length. RFC 2683 suggests limiting client +# reading arbitrary length lines. RFC 3501 (IMAP 4rev1) +# doesn't specify a line length. RFC 2683 suggests limiting client # command lines to 1000 octets and that servers should be prepared # to accept command lines up to 8000 octets, so we used to use 10K here. # In the modern world (eg: gmail) the response to, for example, a @@ -131,8 +131,19 @@ # We compile these in _mode_xxx. _Literal = br'.*{(?P\d+)}$' _Untagged_status = br'\* (?P\d+) (?P[A-Z-]+)( (?P.*))?' +# Only NUL, CR and LF are unsafe (they cannot be represented even in +# a quoted string); other control characters are sent quoted. +_control_chars = re.compile(b'[\x00\r\n]') +_non_astring_char = re.compile(br'[(){ \x00-\x1f\x7f-\xff%*\\"]') +_non_list_char = re.compile(br'[(){ \x00-\x1f\x7f-\xff\\"]') +_quoted = re.compile(br'"(?:[^"\\]|\\.)*+"') +def _paren_depth(data, depth=0): + # Net parenthesis nesting of data, ignoring parentheses in quoted strings. + data = _quoted.sub(b'', data) + return depth + data.count(b'(') - data.count(b')') + class IMAP4: @@ -256,9 +267,13 @@ def _connect(self): elif 'OK' in self.untagged_responses: self.state = 'NONAUTH' else: - raise self.error(self.welcome) + # A continuation ('+') greeting is returned as None; report its + # raw line, still held by the last match (gh-108280). + greeting = (self.welcome or self.mo.string).decode( + self._encoding, 'replace') + raise self.error('invalid greeting: ' + greeting) - self._get_capabilities() + self._refresh_capabilities() if __debug__: if self.debug >= 3: self._mesg('CAPABILITIES: %r' % (self.capabilities,)) @@ -488,8 +503,7 @@ def append(self, mailbox, flags, date_time, message): if not mailbox: mailbox = 'INBOX' if flags: - if (flags[0],flags[-1]) != ('(',')'): - flags = '(%s)' % flags + flags = self._set_quote(flags) else: flags = None if date_time: @@ -497,10 +511,8 @@ def append(self, mailbox, flags, date_time, message): else: date_time = None literal = MapCRLF.sub(CRLF, message) - if self.utf8_enabled: - literal = b'UTF8 (' + literal + b')' self.literal = literal - return self._simple_command(name, mailbox, flags, date_time) + return self._simple_command(name, self._astring(mailbox), flags, date_time) def authenticate(self, mechanism, authobject): @@ -525,10 +537,11 @@ def authenticate(self, mechanism, authobject): #if not cap in self.capabilities: # Let the server decide! # raise self.error("Server doesn't allow %s authentication." % mech) self.literal = _Authenticator(authobject).process - typ, dat = self._simple_command('AUTHENTICATE', mech) + typ, dat = self._simple_command('AUTHENTICATE', self._atom(mech)) if typ != 'OK': raise self.error(dat[-1].decode('utf-8', 'replace')) self.state = 'AUTH' + self._refresh_capabilities() return typ, dat @@ -569,7 +582,8 @@ def copy(self, message_set, new_mailbox): (typ, [data]) = .copy(message_set, new_mailbox) """ - return self._simple_command('COPY', message_set, new_mailbox) + return self._simple_command('COPY', self._sequence_set(message_set), + self._astring(new_mailbox)) def create(self, mailbox): @@ -577,7 +591,7 @@ def create(self, mailbox): (typ, [data]) = .create(mailbox) """ - return self._simple_command('CREATE', mailbox) + return self._simple_command('CREATE', self._astring(mailbox)) def delete(self, mailbox): @@ -585,14 +599,15 @@ def delete(self, mailbox): (typ, [data]) = .delete(mailbox) """ - return self._simple_command('DELETE', mailbox) + return self._simple_command('DELETE', self._astring(mailbox)) def deleteacl(self, mailbox, who): """Delete the ACLs (remove any rights) set for who on mailbox. (typ, [data]) = .deleteacl(mailbox, who) """ - return self._simple_command('DELETEACL', mailbox, who) + return self._simple_command('DELETEACL', self._astring(mailbox), + self._astring(who)) def enable(self, capability): """Send an RFC5161 enable string to the server. @@ -631,7 +646,8 @@ def fetch(self, message_set, message_parts): 'data' are tuples of message part envelope and data. """ name = 'FETCH' - typ, dat = self._simple_command(name, message_set, message_parts) + typ, dat = self._simple_command(name, self._sequence_set(message_set), + self._fetch_parts(message_parts)) return self._untagged_response(typ, dat, name) @@ -640,7 +656,7 @@ def getacl(self, mailbox): (typ, [data]) = .getacl(mailbox) """ - typ, dat = self._simple_command('GETACL', mailbox) + typ, dat = self._simple_command('GETACL', self._astring(mailbox)) return self._untagged_response(typ, dat, 'ACL') @@ -648,7 +664,8 @@ def getannotation(self, mailbox, entry, attribute): """(typ, [data]) = .getannotation(mailbox, entry, attribute) Retrieve ANNOTATIONs.""" - typ, dat = self._simple_command('GETANNOTATION', mailbox, entry, attribute) + typ, dat = self._simple_command('GETANNOTATION', self._astring(mailbox), + entry, attribute) return self._untagged_response(typ, dat, 'ANNOTATION') @@ -659,7 +676,7 @@ def getquota(self, root): (typ, [data]) = .getquota(root) """ - typ, dat = self._simple_command('GETQUOTA', root) + typ, dat = self._simple_command('GETQUOTA', self._astring(root)) return self._untagged_response(typ, dat, 'QUOTA') @@ -668,7 +685,7 @@ def getquotaroot(self, mailbox): (typ, [[QUOTAROOT responses...], [QUOTA responses]]) = .getquotaroot(mailbox) """ - typ, dat = self._simple_command('GETQUOTAROOT', mailbox) + typ, dat = self._simple_command('GETQUOTAROOT', self._astring(mailbox)) typ, quota = self._untagged_response(typ, dat, 'QUOTA') typ, quotaroot = self._untagged_response(typ, dat, 'QUOTAROOT') return typ, [quotaroot, quota] @@ -687,15 +704,16 @@ def idle(self, duration=None): return Idler(self, duration) - def list(self, directory='""', pattern='*'): + def list(self, directory='', pattern='*'): """List mailbox names in directory matching pattern. - (typ, [data]) = .list(directory='""', pattern='*') + (typ, [data]) = .list(directory='', pattern='*') 'data' is list of LIST responses. """ name = 'LIST' - typ, dat = self._simple_command(name, directory, pattern) + typ, dat = self._simple_command(name, self._astring(directory), + self._list_mailbox(pattern)) return self._untagged_response(typ, dat, name) @@ -706,10 +724,12 @@ def login(self, user, password): NB: 'password' will be quoted. """ - typ, dat = self._simple_command('LOGIN', user, self._quote(password)) + typ, dat = self._simple_command('LOGIN', self._astring(user), + self._quote(password)) if typ != 'OK': - raise self.error(dat[-1]) + raise self.error(dat[-1].decode('UTF-8', 'replace')) self.state = 'AUTH' + self._refresh_capabilities() return typ, dat @@ -725,9 +745,17 @@ def login_cram_md5(self, user, password): def _CRAM_MD5_AUTH(self, challenge): """ Authobject to use with CRAM-MD5 authentication. """ import hmac - pwd = (self.password.encode('utf-8') if isinstance(self.password, str) - else self.password) - return self.user + " " + hmac.HMAC(pwd, challenge, 'md5').hexdigest() + + if isinstance(self.password, str): + password = self.password.encode('utf-8') + else: + password = self.password + + try: + authcode = hmac.HMAC(password, challenge, 'md5') + except ValueError: # HMAC-MD5 is not available + raise self.error("CRAM-MD5 authentication is not supported") + return f"{self.user} {authcode.hexdigest()}" def logout(self): @@ -743,15 +771,16 @@ def logout(self): return typ, dat - def lsub(self, directory='""', pattern='*'): + def lsub(self, directory='', pattern='*'): """List 'subscribed' mailbox names in directory matching pattern. - (typ, [data, ...]) = .lsub(directory='""', pattern='*') + (typ, [data, ...]) = .lsub(directory='', pattern='*') 'data' are tuples of message part envelope and data. """ name = 'LSUB' - typ, dat = self._simple_command(name, directory, pattern) + typ, dat = self._simple_command(name, self._astring(directory), + self._list_mailbox(pattern)) return self._untagged_response(typ, dat, name) def myrights(self, mailbox): @@ -759,7 +788,7 @@ def myrights(self, mailbox): (typ, [data]) = .myrights(mailbox) """ - typ,dat = self._simple_command('MYRIGHTS', mailbox) + typ,dat = self._simple_command('MYRIGHTS', self._astring(mailbox)) return self._untagged_response(typ, dat, 'MYRIGHTS') def namespace(self): @@ -805,7 +834,7 @@ def proxyauth(self, user): """ name = 'PROXYAUTH' - return self._simple_command('PROXYAUTH', user) + return self._simple_command(name, self._astring(user)) def rename(self, oldmailbox, newmailbox): @@ -813,7 +842,8 @@ def rename(self, oldmailbox, newmailbox): (typ, [data]) = .rename(oldmailbox, newmailbox) """ - return self._simple_command('RENAME', oldmailbox, newmailbox) + return self._simple_command('RENAME', self._astring(oldmailbox), + self._astring(newmailbox)) def search(self, charset, *criteria): @@ -825,10 +855,11 @@ def search(self, charset, *criteria): If UTF8 is enabled, charset MUST be None. """ name = 'SEARCH' - if charset: + if charset is not None: if self.utf8_enabled: raise IMAP4.error("Non-None charset not valid in UTF8 mode") - typ, dat = self._simple_command(name, 'CHARSET', charset, *criteria) + typ, dat = self._simple_command(name, + 'CHARSET', self._astring(charset), *criteria) else: typ, dat = self._simple_command(name, *criteria) return self._untagged_response(typ, dat, name) @@ -852,7 +883,7 @@ def select(self, mailbox='INBOX', readonly=False): name = 'EXAMINE' else: name = 'SELECT' - typ, dat = self._simple_command(name, mailbox) + typ, dat = self._simple_command(name, self._astring(mailbox)) if typ != 'OK': self.state = 'AUTH' # Might have been 'SELECTED' return typ, dat @@ -862,7 +893,7 @@ def select(self, mailbox='INBOX', readonly=False): if __debug__: if self.debug >= 1: self._dump_ur(self.untagged_responses) - raise self.readonly('%s is not writable' % mailbox) + raise self.readonly('%r is not writable' % (mailbox,)) return typ, self.untagged_responses.get('EXISTS', [None]) @@ -871,14 +902,15 @@ def setacl(self, mailbox, who, what): (typ, [data]) = .setacl(mailbox, who, what) """ - return self._simple_command('SETACL', mailbox, who, what) + return self._simple_command('SETACL', self._astring(mailbox), + self._astring(who), self._astring(what)) - def setannotation(self, *args): + def setannotation(self, mailbox, *args): """(typ, [data]) = .setannotation(mailbox[, entry, attribute]+) Set ANNOTATIONs.""" - typ, dat = self._simple_command('SETANNOTATION', *args) + typ, dat = self._simple_command('SETANNOTATION', self._astring(mailbox), *args) return self._untagged_response(typ, dat, 'ANNOTATION') @@ -887,7 +919,8 @@ def setquota(self, root, limits): (typ, [data]) = .setquota(root, limits) """ - typ, dat = self._simple_command('SETQUOTA', root, limits) + typ, dat = self._simple_command('SETQUOTA', self._astring(root), + self._set_quote(limits)) return self._untagged_response(typ, dat, 'QUOTA') @@ -899,8 +932,9 @@ def sort(self, sort_criteria, charset, *search_criteria): name = 'SORT' #if not name in self.capabilities: # Let the server decide! # raise self.error('unimplemented extension command: %s' % name) - if (sort_criteria[0],sort_criteria[-1]) != ('(',')'): - sort_criteria = '(%s)' % sort_criteria + sort_criteria = self._set_quote(sort_criteria) + if charset is not None: + charset = self._astring(charset) typ, dat = self._simple_command(name, sort_criteria, charset, *search_criteria) return self._untagged_response(typ, dat, name) @@ -936,7 +970,8 @@ def status(self, mailbox, names): name = 'STATUS' #if self.PROTOCOL_VERSION == 'IMAP4': # Let the server decide! # raise self.error('%s unimplemented in IMAP4 (obtain IMAP4rev1 server, or re-code)' % name) - typ, dat = self._simple_command(name, mailbox, names) + typ, dat = self._simple_command(name, self._astring(mailbox), + self._set_quote(names)) return self._untagged_response(typ, dat, name) @@ -945,9 +980,9 @@ def store(self, message_set, command, flags): (typ, [data]) = .store(message_set, command, flags) """ - if (flags[0],flags[-1]) != ('(',')'): - flags = '(%s)' % flags # Avoid quoting the flags - typ, dat = self._simple_command('STORE', message_set, command, flags) + flags = self._set_quote(flags) + typ, dat = self._simple_command('STORE', self._sequence_set(message_set), + command, flags) return self._untagged_response(typ, dat, 'FETCH') @@ -956,7 +991,7 @@ def subscribe(self, mailbox): (typ, [data]) = .subscribe(mailbox) """ - return self._simple_command('SUBSCRIBE', mailbox) + return self._simple_command('SUBSCRIBE', self._astring(mailbox)) def thread(self, threading_algorithm, charset, *search_criteria): @@ -965,7 +1000,10 @@ def thread(self, threading_algorithm, charset, *search_criteria): (type, [data]) = .thread(threading_algorithm, charset, search_criteria, ...) """ name = 'THREAD' - typ, dat = self._simple_command(name, threading_algorithm, charset, *search_criteria) + if charset is not None: + charset = self._astring(charset) + typ, dat = self._simple_command(name, self._atom(threading_algorithm), + charset, *search_criteria) return self._untagged_response(typ, dat, name) @@ -979,14 +1017,38 @@ def uid(self, command, *args): """ command = command.upper() if not command in Commands: - raise self.error("Unknown IMAP4 UID command: %s" % command) + raise self.error("Unknown IMAP4 UID command: %r" % (command,)) if self.state not in Commands[command]: raise self.error("command %s illegal in state %s, " "only allowed in states %s" % (command, self.state, ', '.join(Commands[command]))) name = 'UID' - typ, dat = self._simple_command(name, command, *args) + if command == 'COPY': + message_set, new_mailbox = args + args = (self._sequence_set(message_set), + self._astring(new_mailbox)) + elif command == 'FETCH': + message_set, message_parts = args + args = (self._sequence_set(message_set), + self._fetch_parts(message_parts)) + elif command == 'STORE': + message_set, op, flags = args + args = (self._sequence_set(message_set), op, + self._set_quote(flags)) + elif command == 'SORT': + sort_criteria, charset, *search_criteria = args + if charset is not None: + charset = self._astring(charset) + args = (self._set_quote(sort_criteria), charset, + *search_criteria) + elif command == 'THREAD': + threading_algorithm, charset, *search_criteria = args + if charset is not None: + charset = self._astring(charset) + args = (self._atom(threading_algorithm), charset, + *search_criteria) + typ, dat = self._simple_command(name, self._atom(command), *args) if command in ('SEARCH', 'SORT', 'THREAD'): name = command else: @@ -999,7 +1061,7 @@ def unsubscribe(self, mailbox): (typ, [data]) = .unsubscribe(mailbox) """ - return self._simple_command('UNSUBSCRIBE', mailbox) + return self._simple_command('UNSUBSCRIBE', self._astring(mailbox)) def unselect(self): @@ -1102,6 +1164,8 @@ def _command(self, name, *args): if arg is None: continue if isinstance(arg, str): arg = bytes(arg, self._encoding) + if _control_chars.search(arg): + raise ValueError("NUL, CR and LF not allowed in commands") data = data + b' ' + arg literal = self.literal @@ -1111,7 +1175,11 @@ def _command(self, name, *args): literator = literal else: literator = None - data = data + bytes(' {%s}' % len(literal), self._encoding) + if self.utf8_enabled: + data = data + bytes(' UTF8 (~{%s}' % len(literal), self._encoding) + literal = literal + b')' + else: + data = data + bytes(' {%s}' % len(literal), self._encoding) if __debug__: if self.debug >= 4: @@ -1182,6 +1250,15 @@ def _get_capabilities(self): self.capabilities = tuple(dat.split()) + def _refresh_capabilities(self): + # Use a CAPABILITY response sent by the server, or ask for it. + if 'CAPABILITY' in self.untagged_responses: + dat = self.untagged_responses.pop('CAPABILITY')[-1] + self.capabilities = tuple(str(dat, self._encoding).upper().split()) + else: + self._get_capabilities() + + def _get_response(self, start_timeout=False): # Read response and store. @@ -1206,6 +1283,11 @@ def _get_response(self, start_timeout=False): else: resp = self._get_line() + # Skip spurious blank lines between responses (some servers send one + # after a literal that ends a response). + while resp == b'': + resp = self._get_line() + # Command completion response? if self._match(self.tagre, resp): @@ -1243,6 +1325,7 @@ def _get_response(self, start_timeout=False): # Is there a literal to come? + depth = 0 # open parenthesis nesting so far while self._match(self.Literal, dat): # Read literal direct from connection. @@ -1256,11 +1339,17 @@ def _get_response(self, start_timeout=False): # Store response with literal as tuple self._append_untagged(typ, (dat, data)) + depth = _paren_depth(dat, depth) # Read trailer - possibly containing another literal dat = self._get_line() + # Skip spurious blank lines after a literal, but only inside an + # unclosed parenthesis (at top level they end the response). + while dat == b'' and depth > 0: + dat = self._get_line() + self._append_untagged(typ, dat) # Bracketed response information? @@ -1349,13 +1438,53 @@ def _new_tag(self): return tag - def _quote(self, arg): + def _atom(self, arg): + return arg + + def _sequence_set(self, arg): + return arg - arg = arg.replace('\\', '\\\\') - arg = arg.replace('"', '\\"') + def _set_quote(self, arg): + if arg and arg[0] == '(' and arg[-1] == ')': + return arg + return '(' + arg + ')' - return '"' + arg + '"' + def _fetch_parts(self, arg): + # "ALL", "FULL" and "FAST" are macros, not data item names; + # they cannot be enclosed in parentheses. + if arg.upper() in ('ALL', 'FULL', 'FAST'): + return arg + return self._set_quote(arg) + def _quote(self, arg): + if isinstance(arg, str): + arg = bytes(arg, self._encoding) + arg = arg.replace(b'\\', br'\\') + arg = arg.replace(b'"', br'\"') + return b'"' + arg + b'"' + + # For backward compatibility, an argument already enclosed in double + # quotes is left unquoted, so that code which quotes arguments itself + # keeps working. New code should pass arguments unquoted and let the + # module quote them as needed. + + def _astring(self, arg): + if isinstance(arg, str): + arg = bytes(arg, self._encoding) + if _quoted.fullmatch(arg): + return arg + if arg and _non_astring_char.search(arg) is None: + return arg + return self._quote(arg) + + def _list_mailbox(self, arg): + if isinstance(arg, str): + arg = bytes(arg, self._encoding) + if _quoted.fullmatch(arg): + return arg + if arg and _non_list_char.search(arg) is None: + return arg + return self._quote(arg) def _simple_command(self, name, *args): @@ -1825,9 +1954,8 @@ def Time2Internaldate(date_time): dt = datetime.fromtimestamp(date_time, timezone.utc).astimezone() elif isinstance(date_time, tuple): - try: - gmtoff = date_time.tm_gmtoff - except AttributeError: + gmtoff = getattr(date_time, "tm_gmtoff", None) + if gmtoff is None: if time.daylight: dst = date_time[8] if dst == -1: diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 499da1e04efea8d..9d911e1dcaba61d 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -1375,6 +1375,14 @@ def _find_and_load(name, import_): # NOTE: because of this, initializing must be set *before* # putting the new module in sys.modules. _lock_unlock_module(name) + else: + # Verify the module is still in sys.modules. Another thread may have + # removed it (due to import failure) between our sys.modules.get() + # above and the _initializing check. If removed, we retry the import + # to preserve normal semantics: the caller gets the exception from + # the actual import failure rather than a synthetic error. + if sys.modules.get(name) is not module: + return _find_and_load(name, import_) if module is None: message = f'import of {name} halted; None in sys.modules' diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 8bcd741c446bd27..6a828ae75ed34c1 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -208,12 +208,8 @@ def _write_atomic(path, data, mode=0o666): try: # We first write data to a temporary file, and then use os.replace() to # perform an atomic rename. - with _io.FileIO(fd, 'wb') as file: - bytes_written = file.write(data) - if bytes_written != len(data): - # Raise an OSError so the 'except' below cleans up the partially - # written file. - raise OSError("os.write() didn't write the full pyc file") + with _io.open(fd, 'wb') as file: + file.write(data) _os.replace(path_tmp, path) except OSError: try: @@ -297,7 +293,8 @@ def cache_from_source(path, debug_override=None, *, optimization=None): # Strip initial drive from a Windows path. We know we have an absolute # path here, so the second part of the check rules out a POSIX path that # happens to contain a colon at the second character. - if head[1] == ':' and head[0] not in path_separators: + # Slicing avoids issues with an empty (or short) `head`. + if head[1:2] == ':' and head[0:1] not in path_separators: head = head[2:] # Strip initial path separator from `head` to complete the conversion @@ -949,7 +946,7 @@ def get_filename(self, fullname): def get_data(self, path): """Return the data from path as raw bytes.""" - if isinstance(self, (SourceLoader, ExtensionFileLoader)): + if isinstance(self, (SourceLoader, SourcelessFileLoader, ExtensionFileLoader)): with _io.open_code(str(path)) as file: return file.read() else: @@ -1497,7 +1494,13 @@ def create_module(self, spec): ) # Ensure that the __file__ points at the .fwork location - module.__file__ = path + try: + module.__file__ = path + except AttributeError: + # Not important enough to report. + # (The error is also ignored in _bootstrap._init_module_attrs or + # import_run_extension in import.c) + pass return module diff --git a/Lib/importlib/abc.py b/Lib/importlib/abc.py index 29f01f77eff4a0d..1e47495f65fa02f 100644 --- a/Lib/importlib/abc.py +++ b/Lib/importlib/abc.py @@ -64,20 +64,14 @@ def invalidate_caches(self): class ResourceLoader(Loader): """Abstract base class for loaders which can return data from their - back-end storage. + back-end storage to facilitate reading data to perform an import. This ABC represents one of the optional protocols specified by PEP 302. - """ - - def __init__(self): - import warnings - warnings.warn('importlib.abc.ResourceLoader is deprecated in ' - 'favour of supporting resource loading through ' - 'importlib.resources.abc.TraversableResources.', - DeprecationWarning, stacklevel=2) - super().__init__() + For directly loading resources, use TraversableResources instead. This class + primarily exists for backwards compatibility with other ABCs in this module. + """ @abc.abstractmethod def get_data(self, path): diff --git a/Lib/inspect.py b/Lib/inspect.py index 52c9bb05b31f372..cf61ae943b1624f 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -1,7 +1,7 @@ """Get useful information from live Python objects. This module encapsulates the interface provided by the internal special -attributes (co_*, im_*, tb_*, etc.) in a friendlier fashion. +attributes (co_*, tb_*, etc.) in a friendlier fashion. It also provides some help for examining source code and class layout. Here are some of the useful functions provided by this module: @@ -196,18 +196,18 @@ def ispackage(object): def ismethoddescriptor(object): """Return true if the object is a method descriptor. - But not if ismethod() or isclass() or isfunction() are true. + But not if ismethod(), isclass() or isfunction() is true. - This is new in Python 2.2, and, for example, is true of int.__add__. - An object passing this test has a __get__ attribute, but not a - __set__ attribute or a __delete__ attribute. Beyond that, the set - of attributes varies; __name__ is usually sensible, and __doc__ - often is. + An object passing this test (for example, int.__add__) has a __get__ + attribute, but not a __set__ attribute or a __delete__ attribute. + Beyond that, the set of attributes varies; __name__ is usually + sensible, and __doc__ often is. Methods implemented via descriptors that also pass one of the other - tests return false from the ismethoddescriptor() test, simply because - the other tests promise more -- you can, e.g., count on having the - __func__ attribute (etc) when an object passes ismethod().""" + tests (ismethod(), isclass(), isfunction()) make this function return + false, simply because those other tests promise more -- you can, for + example, count on having the __func__ attribute when an object passes + ismethod().""" if isclass(object) or ismethod(object) or isfunction(object): # mutual exclusion return False @@ -219,8 +219,13 @@ def ismethoddescriptor(object): def isdatadescriptor(object): """Return true if the object is a data descriptor. + But not if ismethod(), isclass() or isfunction() is true. + Data descriptors have a __set__ or a __delete__ attribute. Examples are - properties (defined in Python) and getsets and members (defined in C). + properties, getsets, and members. For the latter two (defined only in C + extension modules) more specific tests are available as well: + isgetsetdescriptor() and ismemberdescriptor(), respectively. + Typically, data descriptors will also have __name__ and __doc__ attributes (properties, getsets, and members have both of these attributes), but this is not guaranteed.""" @@ -348,6 +353,7 @@ def isgenerator(object): gi_frame frame object or possibly None once the generator has been exhausted gi_running set to 1 when generator is executing, 0 otherwise + gi_suspended set to 1 when the generator is suspended at a yield point, 0 otherwise gi_yieldfrom object being iterated by yield from or None __iter__() defined to support iteration over container @@ -1056,7 +1062,7 @@ class BlockFinder: """Provide a tokeneater() method to detect the end of a code block.""" def __init__(self): self.indent = 0 - self.islambda = False + self.singleline = False self.started = False self.passline = False self.indecorator = False @@ -1065,19 +1071,24 @@ def __init__(self): def tokeneater(self, type, token, srowcol, erowcol, line): if not self.started and not self.indecorator: + if type in (tokenize.INDENT, tokenize.COMMENT, tokenize.NL): + pass + elif token == "async": + pass # skip any decorators - if token == "@": + elif token == "@": self.indecorator = True - # look for the first "def", "class" or "lambda" - elif token in ("def", "class", "lambda"): - if token == "lambda": - self.islambda = True + else: + # For "def" and "class" scan to the end of the block. + # For "lambda" and generator expression scan to + # the end of the logical line. + self.singleline = token not in ("def", "class") self.started = True self.passline = True # skip to the end of the line elif type == tokenize.NEWLINE: self.passline = False # stop skipping when a NEWLINE is seen self.last = srowcol[0] - if self.islambda: # lambdas always end at the first NEWLINE + if self.singleline: raise EndOfBlock # hitting a NEWLINE when in a decorator without args # ends the decorator @@ -1913,17 +1924,21 @@ def _signature_get_user_defined_method(cls, method_name, *, follow_wrapper_chain if meth is None: return None + # NOTE: The meth may wraps a non-user-defined callable. + # In this case, we treat the meth as non-user-defined callable too. + # (e.g. cls.__new__ generated by @warnings.deprecated) + unwrapped_meth = None if follow_wrapper_chains: - meth = unwrap(meth, stop=(lambda m: hasattr(m, "__signature__") + unwrapped_meth = unwrap(meth, stop=(lambda m: hasattr(m, "__signature__") or _signature_is_builtin(m))) - if isinstance(meth, _NonUserDefinedCallables): + + if (isinstance(meth, _NonUserDefinedCallables) + or isinstance(unwrapped_meth, _NonUserDefinedCallables)): # Once '__signature__' will be added to 'C'-level # callables, this check won't be necessary return None if method_name != '__new__': meth = _descriptor_get(meth, cls) - if follow_wrapper_chains: - meth = unwrap(meth, stop=lambda m: hasattr(m, "__signature__")) return meth @@ -2074,13 +2089,11 @@ def _signature_is_functionlike(obj): code = getattr(obj, '__code__', None) defaults = getattr(obj, '__defaults__', _void) # Important to use _void ... kwdefaults = getattr(obj, '__kwdefaults__', _void) # ... and not None here - annotations = getattr(obj, '__annotations__', None) return (isinstance(code, types.CodeType) and isinstance(name, str) and (defaults is None or isinstance(defaults, tuple)) and - (kwdefaults is None or isinstance(kwdefaults, dict)) and - (isinstance(annotations, (dict)) or annotations is None) ) + (kwdefaults is None or isinstance(kwdefaults, dict))) def _signature_strip_non_python_syntax(signature): @@ -2652,11 +2665,12 @@ class Parameter: The annotation for the parameter if specified. If the parameter has no annotation, this attribute is set to `Parameter.empty`. - * kind : str + * kind Describes how argument values are bound to the parameter. Possible values: `Parameter.POSITIONAL_ONLY`, `Parameter.POSITIONAL_OR_KEYWORD`, `Parameter.VAR_POSITIONAL`, `Parameter.KEYWORD_ONLY`, `Parameter.VAR_KEYWORD`. + Every value has a `description` attribute describing meaning. """ __slots__ = ('_name', '_kind', '_default', '_annotation') @@ -2703,6 +2717,10 @@ def __init__(self, name, kind, *, default=_empty, annotation=_empty): raise ValueError(msg) self._kind = _POSITIONAL_ONLY name = 'implicit{}'.format(name[1:]) + elif name == '.format': + # gh-151665: Hidden parameter of compiler-generated annotation and type + # alias/typevar evaluators. Show it as "format". + name = 'format' # It's possible for C functions to have a positional-only parameter # where the name is a keyword, so for compatibility we'll allow it. diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py index 703fa289dda1fb5..ca732e4f2e85a81 100644 --- a/Lib/ipaddress.py +++ b/Lib/ipaddress.py @@ -729,7 +729,7 @@ def __eq__(self, other): return NotImplemented def __hash__(self): - return hash(int(self.network_address) ^ int(self.netmask)) + return hash((int(self.network_address), int(self.netmask))) def __contains__(self, other): # always false if one is v4 and the other is v6. @@ -1545,7 +1545,7 @@ def __init__(self, address, strict=True): if self._prefixlen == (self.max_prefixlen - 1): self.hosts = self.__iter__ elif self._prefixlen == (self.max_prefixlen): - self.hosts = lambda: [IPv4Address(addr)] + self.hosts = lambda: iter((IPv4Address(addr),)) @property @functools.lru_cache() @@ -1660,8 +1660,18 @@ def _ip_int_from_string(cls, ip_str): """ if not ip_str: raise AddressValueError('Address cannot be empty') - - parts = ip_str.split(':') + if len(ip_str) > 45: + shorten = ip_str + if len(shorten) > 100: + shorten = f'{ip_str[:45]}({len(ip_str)-90} chars elided){ip_str[-45:]}' + raise AddressValueError(f"At most 45 characters expected in " + f"{shorten!r}") + + # We want to allow more parts than the max to be 'split' + # to preserve the correct error message when there are + # too many parts combined with '::' + _max_parts = cls._HEXTET_COUNT + 1 + parts = ip_str.split(':', maxsplit=_max_parts) # An IPv6 address needs at least 2 colons (3 parts). _min_parts = 3 @@ -1681,7 +1691,6 @@ def _ip_int_from_string(cls, ip_str): # An IPv6 address can't have more than 8 colons (9 parts). # The extra colon comes from using the "::" notation for a single # leading or trailing zero part. - _max_parts = cls._HEXTET_COUNT + 1 if len(parts) > _max_parts: msg = "At most %d colons permitted in %r" % (_max_parts-1, ip_str) raise AddressValueError(msg) @@ -2327,7 +2336,7 @@ def __init__(self, address, strict=True): if self._prefixlen == (self.max_prefixlen - 1): self.hosts = self.__iter__ elif self._prefixlen == self.max_prefixlen: - self.hosts = lambda: [IPv6Address(addr)] + self.hosts = lambda: iter((IPv6Address(addr),)) def hosts(self): """Generate Iterator over usable hosts in a network. diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index 1d972d22ded072a..800cc37f6afe31d 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -128,8 +128,9 @@ def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True, instead of raising a ``TypeError``. If ``ensure_ascii`` is false, then the strings written to ``fp`` can - contain non-ASCII characters if they appear in strings contained in - ``obj``. Otherwise, all such characters are escaped in JSON strings. + contain non-ASCII and non-printable characters if they appear in strings + contained in ``obj``. Otherwise, all such characters are escaped in JSON + strings. If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will @@ -142,13 +143,14 @@ def dump(obj, fp, *, skipkeys=False, ensure_ascii=True, check_circular=True, If ``indent`` is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent - level of 0 will only insert newlines. ``None`` is the most compact - representation. + level of 0 will only insert newlines. ``None`` is the default and gives + a representation with no newlines inserted. - If specified, ``separators`` should be an ``(item_separator, key_separator)`` - tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and - ``(',', ': ')`` otherwise. To get the most compact JSON representation, - you should specify ``(',', ':')`` to eliminate whitespace. + If specified, ``separators`` should be an ``(item_separator, + key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is + ``None`` and ``(',', ': ')`` otherwise. To get the most compact JSON + representation, you should specify ``(',', ':')`` to eliminate + whitespace. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. @@ -189,9 +191,10 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, (``str``, ``int``, ``float``, ``bool``, ``None``) will be skipped instead of raising a ``TypeError``. - If ``ensure_ascii`` is false, then the return value can contain non-ASCII - characters if they appear in strings contained in ``obj``. Otherwise, all - such characters are escaped in JSON strings. + If ``ensure_ascii`` is false, then the return value can contain + non-ASCII and non-printable characters if they appear in strings + contained in ``obj``. Otherwise, all such characters are escaped in + JSON strings. If ``check_circular`` is false, then the circular reference check for container types will be skipped and a circular reference will @@ -204,13 +207,14 @@ def dumps(obj, *, skipkeys=False, ensure_ascii=True, check_circular=True, If ``indent`` is a non-negative integer, then JSON array elements and object members will be pretty-printed with that indent level. An indent - level of 0 will only insert newlines. ``None`` is the most compact - representation. + level of 0 will only insert newlines. ``None`` is the default and gives + a representation with no newlines inserted. - If specified, ``separators`` should be an ``(item_separator, key_separator)`` - tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and - ``(',', ': ')`` otherwise. To get the most compact JSON representation, - you should specify ``(',', ':')`` to eliminate whitespace. + If specified, ``separators`` should be an ``(item_separator, + key_separator)`` tuple. The default is ``(', ', ': ')`` if *indent* is + ``None`` and ``(',', ': ')`` otherwise. To get the most compact JSON + representation, you should specify ``(',', ':')`` to eliminate + whitespace. ``default(obj)`` is a function that should return a serializable version of obj or raise TypeError. The default simply raises TypeError. @@ -281,11 +285,12 @@ def load(fp, *, cls=None, object_hook=None, parse_float=None, ``object_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders (e.g. JSON-RPC class hinting). - ``object_pairs_hook`` is an optional function that will be called with the - result of any object literal decoded with an ordered list of pairs. The - return value of ``object_pairs_hook`` will be used instead of the ``dict``. - This feature can be used to implement custom decoders. If ``object_hook`` - is also defined, the ``object_pairs_hook`` takes priority. + ``object_pairs_hook`` is an optional function that will be called with + the result of any object literal decoded with an ordered list of pairs. + The return value of ``object_pairs_hook`` will be used instead of the + ``dict``. This feature can be used to implement custom decoders. If + ``object_hook`` is also defined, the ``object_pairs_hook`` takes + priority. To use a custom ``JSONDecoder`` subclass, specify it with the ``cls`` kwarg; otherwise ``JSONDecoder`` is used. @@ -306,11 +311,12 @@ def loads(s, *, cls=None, object_hook=None, parse_float=None, ``object_hook`` will be used instead of the ``dict``. This feature can be used to implement custom decoders (e.g. JSON-RPC class hinting). - ``object_pairs_hook`` is an optional function that will be called with the - result of any object literal decoded with an ordered list of pairs. The - return value of ``object_pairs_hook`` will be used instead of the ``dict``. - This feature can be used to implement custom decoders. If ``object_hook`` - is also defined, the ``object_pairs_hook`` takes priority. + ``object_pairs_hook`` is an optional function that will be called with + the result of any object literal decoded with an ordered list of pairs. + The return value of ``object_pairs_hook`` will be used instead of the + ``dict``. This feature can be used to implement custom decoders. If + ``object_hook`` is also defined, the ``object_pairs_hook`` takes + priority. ``parse_float``, if specified, will be called with the string of every JSON float to be decoded. By default this is equivalent to diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py index ff4bfcdcc407b92..92ad63525576409 100644 --- a/Lib/json/decoder.py +++ b/Lib/json/decoder.py @@ -297,10 +297,10 @@ def __init__(self, *, object_hook=None, parse_float=None, place of the given ``dict``. This can be used to provide custom deserializations (e.g. to support JSON-RPC class hinting). - ``object_pairs_hook``, if specified will be called with the result of - every JSON object decoded with an ordered list of pairs. The return - value of ``object_pairs_hook`` will be used instead of the ``dict``. - This feature can be used to implement custom decoders. + ``object_pairs_hook``, if specified will be called with the result + of every JSON object decoded with an ordered list of pairs. The + return value of ``object_pairs_hook`` will be used instead of the + ``dict``. This feature can be used to implement custom decoders. If ``object_hook`` is also defined, the ``object_pairs_hook`` takes priority. diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 016638549aa59b6..5cf6d64f3eade60 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -111,9 +111,10 @@ def __init__(self, *, skipkeys=False, ensure_ascii=True, encoding of keys that are not str, int, float, bool or None. If skipkeys is True, such items are simply skipped. - If ensure_ascii is true, the output is guaranteed to be str - objects with all incoming non-ASCII characters escaped. If - ensure_ascii is false, the output can contain non-ASCII characters. + If ensure_ascii is true, the output is guaranteed to be str objects + with all incoming non-ASCII and non-printable characters escaped. + If ensure_ascii is false, the output can contain non-ASCII and + non-printable characters. If check_circular is true, then lists, dicts, and custom encoded objects will be checked for circular references during encoding to @@ -134,14 +135,15 @@ def __init__(self, *, skipkeys=False, ensure_ascii=True, indent level. An indent level of 0 will only insert newlines. None is the most compact representation. - If specified, separators should be an (item_separator, key_separator) - tuple. The default is (', ', ': ') if *indent* is ``None`` and - (',', ': ') otherwise. To get the most compact JSON representation, - you should specify (',', ':') to eliminate whitespace. + If specified, separators should be an (item_separator, + key_separator) tuple. The default is (', ', ': ') if *indent* is + ``None`` and (',', ': ') otherwise. To get the most compact JSON + representation, you should specify (',', ':') to eliminate + whitespace. If specified, default is a function that gets called for objects - that can't otherwise be serialized. It should return a JSON encodable - version of the object or raise a ``TypeError``. + that can't otherwise be serialized. It should return a JSON + encodable version of the object or raise a ``TypeError``. """ @@ -348,7 +350,6 @@ def _iterencode_dict(dct, _current_indent_level): _current_indent_level += 1 newline_indent = '\n' + _indent * _current_indent_level item_separator = _item_separator + newline_indent - yield newline_indent else: newline_indent = None item_separator = _item_separator @@ -381,6 +382,8 @@ def _iterencode_dict(dct, _current_indent_level): f'not {key.__class__.__name__}') if first: first = False + if newline_indent is not None: + yield newline_indent else: yield item_separator yield _encoder(key) @@ -413,7 +416,7 @@ def _iterencode_dict(dct, _current_indent_level): except BaseException as exc: exc.add_note(f'when serializing {type(dct).__name__} item {key!r}') raise - if newline_indent is not None: + if not first and newline_indent is not None: _current_indent_level -= 1 yield '\n' + _indent * _current_indent_level yield '}' diff --git a/Lib/json/tool.py b/Lib/json/tool.py index 1967817add8abce..0cabbdba85a1551 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py @@ -88,7 +88,8 @@ def main(): infile = open(options.infile, encoding='utf-8') try: if options.json_lines: - objs = (json.loads(line) for line in infile) + lines = infile.readlines() + objs = (json.loads(line) for line in lines) else: objs = (json.load(infile),) finally: diff --git a/Lib/linecache.py b/Lib/linecache.py index 87d7d6fda657e47..ef3b2d9136b4d20 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -33,10 +33,9 @@ def getlines(filename, module_globals=None): """Get the lines for a Python source file from the cache. Update the cache if it doesn't contain an entry for this file already.""" - if filename in cache: - entry = cache[filename] - if len(entry) != 1: - return cache[filename][2] + entry = cache.get(filename, None) + if entry is not None and len(entry) != 1: + return entry[2] try: return updatecache(filename, module_globals) @@ -56,10 +55,9 @@ def _make_key(code): def _getlines_from_code(code): code_id = _make_key(code) - if code_id in _interactive_cache: - entry = _interactive_cache[code_id] - if len(entry) != 1: - return _interactive_cache[code_id][2] + entry = _interactive_cache.get(code_id, None) + if entry is not None and len(entry) != 1: + return entry[2] return [] @@ -84,12 +82,8 @@ def checkcache(filename=None): filenames = [filename] for filename in filenames: - try: - entry = cache[filename] - except KeyError: - continue - - if len(entry) == 1: + entry = cache.get(filename, None) + if entry is None or len(entry) == 1: # lazy cache entry, leave it lazy. continue size, mtime, lines, fullname = entry @@ -125,15 +119,16 @@ def updatecache(filename, module_globals=None): # These import can fail if the interpreter is shutting down return [] - if filename in cache: - if len(cache[filename]) != 1: - cache.pop(filename, None) + entry = cache.pop(filename, None) if _source_unavailable(filename): return [] - if filename.startswith('')): - return False + return None # Try for a __loader__, if available if module_globals and '__name__' in module_globals: spec = module_globals.get('__spec__') @@ -230,9 +236,10 @@ def lazycache(filename, module_globals): if name and get_source: def get_lines(name=name, *args, **kwargs): return get_source(name, *args, **kwargs) - cache[filename] = (get_lines,) - return True - return False + return (get_lines,) + return None + + def _register_code(code, string, name): entry = (len(string), @@ -245,4 +252,5 @@ def _register_code(code, string, name): for const in code.co_consts: if isinstance(const, type(code)): stack.append(const) - _interactive_cache[_make_key(code)] = entry + key = _make_key(code) + _interactive_cache[key] = entry diff --git a/Lib/locale.py b/Lib/locale.py index 2feb10e59c96a38..6984a6a5e2bc187 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -545,12 +545,6 @@ def getdefaultlocale(envvars=('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE')): """ - import warnings - warnings._deprecated( - "locale.getdefaultlocale", - "{name!r} is deprecated and slated for removal in Python {remove}. " - "Use setlocale(), getencoding() and getlocale() instead.", - remove=(3, 15)) return _getdefaultlocale(envvars) @@ -883,6 +877,10 @@ def getpreferredencoding(do_setlocale=True): # updated 'sr@latn' -> 'sr_CS.UTF-8@latin' to 'sr_RS.UTF-8@latin' # removed 'univ' # removed 'universal' +# +# SS 2025-06-10: +# Remove 'c.utf8' -> 'en_US.UTF-8' because 'en_US.UTF-8' does not exist +# on all platforms. locale_alias = { 'a3': 'az_AZ.KOI8-C', @@ -962,7 +960,6 @@ def getpreferredencoding(do_setlocale=True): 'c.ascii': 'C', 'c.en': 'C', 'c.iso88591': 'en_US.ISO8859-1', - 'c.utf8': 'en_US.UTF-8', 'c_c': 'C', 'c_c.c': 'C', 'ca': 'ca_ES.ISO8859-1', @@ -1493,8 +1490,8 @@ def getpreferredencoding(do_setlocale=True): # This maps Windows language identifiers to locale strings. # # This list has been updated from -# http://msdn.microsoft.com/library/default.asp?url=/library/en-us/intl/nls_238z.asp -# to include every locale up to Windows Vista. +# https://learn.microsoft.com/en-us/openspecs/windows_protocols/ms-lcid/70feba9f-294e-491e-b6eb-56532684c37f +# to include every locale up to protocol revision 16.0 (2024-04-23). # # NOTE: this mapping is incomplete. If your language is missing, please # submit a bug report as detailed in the Python devguide at: @@ -1504,10 +1501,15 @@ def getpreferredencoding(do_setlocale=True): # windows_locale = { - 0x0436: "af_ZA", # Afrikaans - 0x041c: "sq_AL", # Albanian - 0x0484: "gsw_FR",# Alsatian - France + 0x0036: "af", # Afrikaans + 0x0436: "af_ZA", # Afrikaans - South Africa + 0x001c: "sq", # Albanian + 0x041c: "sq_AL", # Albanian - Albania + 0x0084: "gsw", # Alsatian + 0x0484: "gsw_FR", # Alsatian - France + 0x005e: "am", # Amharic 0x045e: "am_ET", # Amharic - Ethiopia + 0x0001: "ar", # Arabic 0x0401: "ar_SA", # Arabic - Saudi Arabia 0x0801: "ar_IQ", # Arabic - Iraq 0x0c01: "ar_EG", # Arabic - Egypt @@ -1521,39 +1523,72 @@ def getpreferredencoding(do_setlocale=True): 0x2c01: "ar_JO", # Arabic - Jordan 0x3001: "ar_LB", # Arabic - Lebanon 0x3401: "ar_KW", # Arabic - Kuwait - 0x3801: "ar_AE", # Arabic - United Arab Emirates + 0x3801: "ar_AE", # Arabic - U.A.E. 0x3c01: "ar_BH", # Arabic - Bahrain 0x4001: "ar_QA", # Arabic - Qatar - 0x042b: "hy_AM", # Armenian + 0x002b: "hy", # Armenian + 0x042b: "hy_AM", # Armenian - Armenia + 0x004d: "as", # Assamese 0x044d: "as_IN", # Assamese - India - 0x042c: "az_AZ", # Azeri - Latin - 0x082c: "az_AZ", # Azeri - Cyrillic - 0x046d: "ba_RU", # Bashkir - 0x042d: "eu_ES", # Basque - Russia - 0x0423: "be_BY", # Belarusian - 0x0445: "bn_IN", # Begali - 0x201a: "bs_BA", # Bosnian - Cyrillic - 0x141a: "bs_BA", # Bosnian - Latin + 0x002c: "az", # Azerbaijani (Latin) + 0x742c: "az", # Azerbaijani (Cyrillic) + 0x782c: "az", # Azerbaijani (Latin) + 0x042c: "az_AZ", # Azerbaijani (Latin) - Azerbaijan + 0x0045: "bn", # Bangla + 0x0445: "bn_IN", # Bangla - India + 0x0845: "bn_BD", # Bangla - Bangladesh + 0x006d: "ba", # Bashkir + 0x046d: "ba_RU", # Bashkir - Russia + 0x002d: "eu", # Basque + 0x042d: "eu_ES", # Basque - Spain + 0x0023: "be", # Belarusian + 0x0423: "be_BY", # Belarusian - Belarus + 0x641a: "bs", # Bosnian (Cyrillic) + 0x681a: "bs", # Bosnian (Latin) + 0x141a: "bs_BA", # Bosnian (Latin) - Bosnia and Herzegovina + 0x201a: "bs_BA", # Bosnian (Cyrillic) - Bosnia and Herzegovina + 0x781a: "bs", # Bosnian (Latin) + 0x007e: "br", # Breton 0x047e: "br_FR", # Breton - France - 0x0402: "bg_BG", # Bulgarian -# 0x0455: "my_MM", # Burmese - Not supported - 0x0403: "ca_ES", # Catalan - 0x0004: "zh_CHS",# Chinese - Simplified - 0x0404: "zh_TW", # Chinese - Taiwan - 0x0804: "zh_CN", # Chinese - PRC - 0x0c04: "zh_HK", # Chinese - Hong Kong S.A.R. - 0x1004: "zh_SG", # Chinese - Singapore - 0x1404: "zh_MO", # Chinese - Macao S.A.R. - 0x7c04: "zh_CHT",# Chinese - Traditional + 0x0002: "bg", # Bulgarian + 0x0402: "bg_BG", # Bulgarian - Bulgaria + 0x0055: "my", # Burmese + 0x0455: "my_MM", # Burmese - Myanmar + 0x0003: "ca", # Catalan + 0x0403: "ca_ES", # Catalan - Spain + 0x0803: "ca_ES", # Valencian - Spain + 0x0092: "ku", # Central Kurdish + 0x7c92: "ku", # Central Kurdish + 0x0492: "ku_IQ", # Central Kurdish - Iraq + 0x005c: "chr", # Cherokee + 0x7c5c: "chr", # Cherokee + 0x045c: "chr_US", # Cherokee - United States + 0x0004: "zh", # Chinese (Simplified) + 0x7804: "zh", # Chinese (Simplified) + 0x7c04: "zh", # Chinese (Traditional) + 0x0404: "zh_TW", # Chinese (Traditional) - Taiwan + 0x0804: "zh_CN", # Chinese (Simplified) - People's Republic of China + 0x0c04: "zh_HK", # Chinese (Traditional) - Hong Kong S.A.R. + 0x1004: "zh_SG", # Chinese (Simplified) - Singapore + 0x1404: "zh_MO", # Chinese (Traditional) - Macao S.A.R. + 0x0083: "co", # Corsican 0x0483: "co_FR", # Corsican - France - 0x041a: "hr_HR", # Croatian - 0x101a: "hr_BA", # Croatian - Bosnia - 0x0405: "cs_CZ", # Czech - 0x0406: "da_DK", # Danish - 0x048c: "gbz_AF",# Dari - Afghanistan - 0x0465: "div_MV",# Divehi - Maldives - 0x0413: "nl_NL", # Dutch - The Netherlands + 0x001a: "hr", # Croatian + 0x041a: "hr_HR", # Croatian - Croatia + 0x101a: "hr_BA", # Croatian (Latin) - Bosnia and Herzegovina + 0x0005: "cs", # Czech + 0x0405: "cs_CZ", # Czech - Czech Republic + 0x0006: "da", # Danish + 0x0406: "da_DK", # Danish - Denmark + 0x008c: "prs", # Dari + 0x048c: "prs_AF", # Dari - Afghanistan + 0x0065: "dv", # Divehi + 0x0465: "dv_MV", # Divehi - Maldives + 0x0013: "nl", # Dutch + 0x0413: "nl_NL", # Dutch - Netherlands 0x0813: "nl_BE", # Dutch - Belgium + 0x0c51: "dz_BT", # Dzongkha - Bhutan + 0x0009: "en", # English 0x0409: "en_US", # English - United States 0x0809: "en_GB", # English - United Kingdom 0x0c09: "en_AU", # English - Australia @@ -1561,122 +1596,248 @@ def getpreferredencoding(do_setlocale=True): 0x1409: "en_NZ", # English - New Zealand 0x1809: "en_IE", # English - Ireland 0x1c09: "en_ZA", # English - South Africa - 0x2009: "en_JA", # English - Jamaica - 0x2409: "en_CB", # English - Caribbean + 0x2009: "en_JM", # English - Jamaica 0x2809: "en_BZ", # English - Belize - 0x2c09: "en_TT", # English - Trinidad + 0x2c09: "en_TT", # English - Trinidad and Tobago 0x3009: "en_ZW", # English - Zimbabwe - 0x3409: "en_PH", # English - Philippines + 0x3409: "en_PH", # English - Republic of the Philippines + 0x3c09: "en_HK", # English - Hong Kong 0x4009: "en_IN", # English - India 0x4409: "en_MY", # English - Malaysia - 0x4809: "en_IN", # English - Singapore - 0x0425: "et_EE", # Estonian - 0x0438: "fo_FO", # Faroese - 0x0464: "fil_PH",# Filipino - 0x040b: "fi_FI", # Finnish + 0x4809: "en_SG", # English - Singapore + 0x4c09: "en_AE", # English - United Arab Emirates + 0x0025: "et", # Estonian + 0x0425: "et_EE", # Estonian - Estonia + 0x0038: "fo", # Faroese + 0x0438: "fo_FO", # Faroese - Faroe Islands + 0x0064: "fil", # Filipino + 0x0464: "fil_PH", # Filipino - Philippines + 0x000b: "fi", # Finnish + 0x040b: "fi_FI", # Finnish - Finland + 0x000c: "fr", # French 0x040c: "fr_FR", # French - France 0x080c: "fr_BE", # French - Belgium 0x0c0c: "fr_CA", # French - Canada 0x100c: "fr_CH", # French - Switzerland 0x140c: "fr_LU", # French - Luxembourg - 0x180c: "fr_MC", # French - Monaco + 0x180c: "fr_MC", # French - Principality of Monaco + 0x1c0c: "fr_029", # French - Caribbean + 0x200c: "fr_RE", # French - Reunion + 0x240c: "fr_CD", # French - Congo, DRC + 0x280c: "fr_SN", # French - Senegal + 0x2c0c: "fr_CM", # French - Cameroon + 0x300c: "fr_CI", # French - Côte d'Ivoire + 0x340c: "fr_ML", # French - Mali + 0x380c: "fr_MA", # French - Morocco + 0x3c0c: "fr_HT", # French - Haiti + 0x0062: "fy", # Frisian 0x0462: "fy_NL", # Frisian - Netherlands - 0x0456: "gl_ES", # Galician - 0x0437: "ka_GE", # Georgian + 0x0067: "ff", # Fulah + 0x7c67: "ff", # Fulah (Latin) + 0x0467: "ff_NG", + 0x0867: "ff_SN", # Fulah - Senegal + 0x0056: "gl", # Galician + 0x0456: "gl_ES", # Galician - Spain + 0x0037: "ka", # Georgian + 0x0437: "ka_GE", # Georgian - Georgia + 0x0007: "de", # German 0x0407: "de_DE", # German - Germany 0x0807: "de_CH", # German - Switzerland 0x0c07: "de_AT", # German - Austria 0x1007: "de_LU", # German - Luxembourg 0x1407: "de_LI", # German - Liechtenstein - 0x0408: "el_GR", # Greek + 0x0008: "el", # Greek + 0x0408: "el_GR", # Greek - Greece + 0x006f: "kl", # Greenlandic 0x046f: "kl_GL", # Greenlandic - Greenland - 0x0447: "gu_IN", # Gujarati - 0x0468: "ha_NG", # Hausa - Latin - 0x040d: "he_IL", # Hebrew - 0x0439: "hi_IN", # Hindi - 0x040e: "hu_HU", # Hungarian - 0x040f: "is_IS", # Icelandic - 0x0421: "id_ID", # Indonesian - 0x045d: "iu_CA", # Inuktitut - Syllabics - 0x085d: "iu_CA", # Inuktitut - Latin + 0x0074: "gn", # Guarani + 0x0474: "gn_PY", # Guarani - Paraguay + 0x0047: "gu", # Gujarati + 0x0447: "gu_IN", # Gujarati - India + 0x0068: "ha", # Hausa (Latin) + 0x7c68: "ha", # Hausa (Latin) + 0x0468: "ha_NG", # Hausa (Latin) - Nigeria + 0x0075: "haw", # Hawaiian + 0x0475: "haw_US", # Hawaiian - United States + 0x000d: "he", # Hebrew + 0x040d: "he_IL", # Hebrew - Israel + 0x0039: "hi", # Hindi + 0x0439: "hi_IN", # Hindi - India + 0x000e: "hu", # Hungarian + 0x040e: "hu_HU", # Hungarian - Hungary + 0x000f: "is", # Icelandic + 0x040f: "is_IS", # Icelandic - Iceland + 0x0070: "ig", # Igbo + 0x0470: "ig_NG", # Igbo - Nigeria + 0x0021: "id", # Indonesian + 0x0421: "id_ID", # Indonesian - Indonesia + 0x005d: "iu", # Inuktitut (Latin) + 0x785d: "iu", # Inuktitut (Syllabics) + 0x7c5d: "iu", # Inuktitut (Latin) + 0x045d: "iu_CA", # Inuktitut (Syllabics) - Canada + 0x085d: "iu_CA", # Inuktitut (Latin) - Canada + 0x003c: "ga", # Irish 0x083c: "ga_IE", # Irish - Ireland + 0x0010: "it", # Italian 0x0410: "it_IT", # Italian - Italy 0x0810: "it_CH", # Italian - Switzerland - 0x0411: "ja_JP", # Japanese + 0x0011: "ja", # Japanese + 0x0411: "ja_JP", # Japanese - Japan + 0x004b: "kn", # Kannada 0x044b: "kn_IN", # Kannada - India - 0x043f: "kk_KZ", # Kazakh - 0x0453: "kh_KH", # Khmer - Cambodia - 0x0486: "qut_GT",# K'iche - Guatemala + 0x0471: "kr_NG", # Kanuri (Latin) - Nigeria + 0x0060: "ks", # Kashmiri + 0x0460: "ks", # Kashmiri - Perso_Arabic + 0x0860: "ks_IN", # Kashmiri (Devanagari) - India + 0x003f: "kk", # Kazakh + 0x043f: "kk_KZ", # Kazakh - Kazakhstan + 0x0053: "km", # Khmer + 0x0453: "km_KH", # Khmer - Cambodia + 0x0087: "rw", # Kinyarwanda 0x0487: "rw_RW", # Kinyarwanda - Rwanda - 0x0457: "kok_IN",# Konkani - 0x0412: "ko_KR", # Korean - 0x0440: "ky_KG", # Kyrgyz - 0x0454: "lo_LA", # Lao - Lao PDR - 0x0426: "lv_LV", # Latvian - 0x0427: "lt_LT", # Lithuanian - 0x082e: "dsb_DE",# Lower Sorbian - Germany - 0x046e: "lb_LU", # Luxembourgish - 0x042f: "mk_MK", # FYROM Macedonian + 0x0041: "sw", # Kiswahili + 0x0441: "sw_KE", # Kiswahili - Kenya + 0x0057: "kok", # Konkani + 0x0457: "kok_IN", # Konkani - India + 0x0012: "ko", # Korean + 0x0412: "ko_KR", # Korean - Korea + 0x0040: "ky", # Kyrgyz + 0x0440: "ky_KG", # Kyrgyz - Kyrgyzstan + 0x0054: "lo", # Lao + 0x0454: "lo_LA", # Lao - Lao P.D.R. + 0x0476: "la_VA", # Latin - Vatican City + 0x0026: "lv", # Latvian + 0x0426: "lv_LV", # Latvian - Latvia + 0x0027: "lt", # Lithuanian + 0x0427: "lt_LT", # Lithuanian - Lithuania + 0x7c2e: "dsb", # Lower Sorbian + 0x082e: "dsb_DE", # Lower Sorbian - Germany + 0x006e: "lb", # Luxembourgish + 0x046e: "lb_LU", # Luxembourgish - Luxembourg + 0x002f: "mk", # Macedonian + 0x042f: "mk_MK", # Macedonian - North Macedonia + 0x003e: "ms", # Malay 0x043e: "ms_MY", # Malay - Malaysia 0x083e: "ms_BN", # Malay - Brunei Darussalam + 0x004c: "ml", # Malayalam 0x044c: "ml_IN", # Malayalam - India - 0x043a: "mt_MT", # Maltese - 0x0481: "mi_NZ", # Maori - 0x047a: "arn_CL",# Mapudungun - 0x044e: "mr_IN", # Marathi - 0x047c: "moh_CA",# Mohawk - Canada - 0x0450: "mn_MN", # Mongolian - Cyrillic - 0x0850: "mn_CN", # Mongolian - PRC - 0x0461: "ne_NP", # Nepali - 0x0414: "nb_NO", # Norwegian - Bokmal - 0x0814: "nn_NO", # Norwegian - Nynorsk + 0x003a: "mt", # Maltese + 0x043a: "mt_MT", # Maltese - Malta + 0x0081: "mi", # Maori + 0x0481: "mi_NZ", # Maori - New Zealand + 0x007a: "arn", # Mapudungun + 0x047a: "arn_CL", # Mapudungun - Chile + 0x004e: "mr", # Marathi + 0x044e: "mr_IN", # Marathi - India + 0x007c: "moh", # Mohawk + 0x047c: "moh_CA", # Mohawk - Canada + 0x0050: "mn", # Mongolian (Cyrillic) + 0x7850: "mn", # Mongolian (Cyrillic) + 0x7c50: "mn", # Mongolian (Traditional Mongolian) + 0x0450: "mn_MN", # Mongolian (Cyrillic) - Mongolia + 0x0c50: "mn_MN", # Mongolian (Traditional Mongolian) - Mongolia + 0x0061: "ne", # Nepali + 0x0461: "ne_NP", # Nepali - Nepal + 0x0861: "ne_IN", # Nepali - India + 0x0014: "no", # Norwegian (Bokmal) + 0x0414: "nb_NO", # Norwegian (Bokmal) - Norway + 0x0814: "nn_NO", # Norwegian (Nynorsk) - Norway + 0x7814: "nn", # Norwegian (Nynorsk) + 0x7c14: "nb", # Norwegian (Bokmal) + 0x0082: "oc", # Occitan 0x0482: "oc_FR", # Occitan - France - 0x0448: "or_IN", # Oriya - India + 0x0048: "or", # Odia + 0x0448: "or_IN", # Odia - India + 0x0072: "om", # Oromo + 0x0472: "om_ET", # Oromo - Ethiopia + 0x0063: "ps", # Pashto 0x0463: "ps_AF", # Pashto - Afghanistan - 0x0429: "fa_IR", # Persian - 0x0415: "pl_PL", # Polish + 0x0029: "fa", # Persian + 0x0429: "fa_IR", # Persian - Iran + 0x0015: "pl", # Polish + 0x0415: "pl_PL", # Polish - Poland + 0x0016: "pt", # Portuguese 0x0416: "pt_BR", # Portuguese - Brazil 0x0816: "pt_PT", # Portuguese - Portugal - 0x0446: "pa_IN", # Punjabi - 0x046b: "quz_BO",# Quechua (Bolivia) - 0x086b: "quz_EC",# Quechua (Ecuador) - 0x0c6b: "quz_PE",# Quechua (Peru) + 0x0046: "pa", # Punjabi + 0x7c46: "pa", # Punjabi + 0x0446: "pa_IN", # Punjabi - India + 0x0846: "pa_PK", # Punjabi - Islamic Republic of Pakistan + 0x006b: "quz", # Quechua + 0x046b: "quz_BO", # Quechua - Bolivia + 0x086b: "quz_EC", # Quechua - Ecuador + 0x0c6b: "quz_PE", # Quechua - Peru + 0x0018: "ro", # Romanian 0x0418: "ro_RO", # Romanian - Romania - 0x0417: "rm_CH", # Romansh - 0x0419: "ru_RU", # Russian - 0x243b: "smn_FI",# Sami Finland - 0x103b: "smj_NO",# Sami Norway - 0x143b: "smj_SE",# Sami Sweden - 0x043b: "se_NO", # Sami Northern Norway - 0x083b: "se_SE", # Sami Northern Sweden - 0x0c3b: "se_FI", # Sami Northern Finland - 0x203b: "sms_FI",# Sami Skolt - 0x183b: "sma_NO",# Sami Southern Norway - 0x1c3b: "sma_SE",# Sami Southern Sweden - 0x044f: "sa_IN", # Sanskrit - 0x0c1a: "sr_SP", # Serbian - Cyrillic - 0x1c1a: "sr_BA", # Serbian - Bosnia Cyrillic - 0x081a: "sr_SP", # Serbian - Latin - 0x181a: "sr_BA", # Serbian - Bosnia Latin + 0x0818: "ro_MD", # Romanian - Moldova + 0x0017: "rm", # Romansh + 0x0417: "rm_CH", # Romansh - Switzerland + 0x0019: "ru", # Russian + 0x0419: "ru_RU", # Russian - Russia + 0x0819: "ru_MD", # Russian - Moldova + 0x0085: "sah", # Sakha + 0x0485: "sah_RU", # Sakha - Russia + 0x003b: "se", # Sami (Northern) + 0x043b: "se_NO", # Sami (Northern) - Norway + 0x083b: "se_SE", # Sami (Northern) - Sweden + 0x0c3b: "se_FI", # Sami (Northern) - Finland + 0x7c3b: "smj", # Sami (Lule) + 0x103b: "smj_NO", # Sami (Lule) - Norway + 0x143b: "smj_SE", # Sami (Lule) - Sweden + 0x783b: "sma", # Sami (Southern) + 0x183b: "sma_NO", # Sami (Southern) - Norway + 0x1c3b: "sma_SE", # Sami (Southern) - Sweden + 0x743b: "sms", # Sami (Skolt) + 0x203b: "sms_FI", # Sami (Skolt) - Finland + 0x703b: "smn", # Sami (Inari) + 0x243b: "smn_FI", # Sami (Inari) - Finland + 0x004f: "sa", # Sanskrit + 0x044f: "sa_IN", # Sanskrit - India + 0x0091: "gd", # Scottish Gaelic + 0x0491: "gd_GB", # Scottish Gaelic - United Kingdom + 0x6c1a: "sr", # Serbian (Cyrillic) + 0x701a: "sr", # Serbian (Latin) + 0x7c1a: "sr", # Serbian (Latin) + 0x081a: "sr_CS", # Serbian (Latin) - Serbia and Montenegro (Former) + 0x0c1a: "sr_CS", # Serbian (Cyrillic) - Serbia and Montenegro (Former) + 0x181a: "sr_BA", # Serbian (Latin) - Bosnia and Herzegovina + 0x1c1a: "sr_BA", # Serbian (Cyrillic) - Bosnia and Herzegovina + 0x241a: "sr_RS", # Serbian (Latin) - Serbia + 0x281a: "sr_RS", # Serbian (Cyrillic) - Serbia + 0x2c1a: "sr_ME", # Serbian (Latin) - Montenegro + 0x301a: "sr_ME", # Serbian (Cyrillic) - Montenegro + 0x006c: "nso", # Sesotho sa Leboa + 0x046c: "nso_ZA", # Sesotho sa Leboa - South Africa + 0x0032: "tn", # Setswana + 0x0432: "tn_ZA", # Setswana - South Africa + 0x0832: "tn_BW", # Setswana - Botswana + 0x0059: "sd", # Sindhi + 0x7c59: "sd", # Sindhi + 0x0859: "sd_PK", # Sindhi - Islamic Republic of Pakistan + 0x005b: "si", # Sinhala 0x045b: "si_LK", # Sinhala - Sri Lanka - 0x046c: "ns_ZA", # Northern Sotho - 0x0432: "tn_ZA", # Setswana - Southern Africa - 0x041b: "sk_SK", # Slovak - 0x0424: "sl_SI", # Slovenian + 0x001b: "sk", # Slovak + 0x041b: "sk_SK", # Slovak - Slovakia + 0x0024: "sl", # Slovenian + 0x0424: "sl_SI", # Slovenian - Slovenia + 0x0477: "so_SO", # Somali - Somalia + 0x0030: "st", # Sotho + 0x0430: "st_ZA", # Sotho - South Africa + 0x000a: "es", # Spanish 0x040a: "es_ES", # Spanish - Spain 0x080a: "es_MX", # Spanish - Mexico - 0x0c0a: "es_ES", # Spanish - Spain (Modern) + 0x0c0a: "es_ES", # Spanish - Spain 0x100a: "es_GT", # Spanish - Guatemala 0x140a: "es_CR", # Spanish - Costa Rica 0x180a: "es_PA", # Spanish - Panama 0x1c0a: "es_DO", # Spanish - Dominican Republic - 0x200a: "es_VE", # Spanish - Venezuela + 0x200a: "es_VE", # Spanish - Bolivarian Republic of Venezuela 0x240a: "es_CO", # Spanish - Colombia 0x280a: "es_PE", # Spanish - Peru 0x2c0a: "es_AR", # Spanish - Argentina 0x300a: "es_EC", # Spanish - Ecuador 0x340a: "es_CL", # Spanish - Chile - 0x380a: "es_UR", # Spanish - Uruguay + 0x380a: "es_UY", # Spanish - Uruguay 0x3c0a: "es_PY", # Spanish - Paraguay 0x400a: "es_BO", # Spanish - Bolivia 0x440a: "es_SV", # Spanish - El Salvador @@ -1684,36 +1845,87 @@ def getpreferredencoding(do_setlocale=True): 0x4c0a: "es_NI", # Spanish - Nicaragua 0x500a: "es_PR", # Spanish - Puerto Rico 0x540a: "es_US", # Spanish - United States -# 0x0430: "", # Sutu - Not supported - 0x0441: "sw_KE", # Swahili + 0x5c0a: "es_CU", # Spanish - Cuba + 0x001d: "sv", # Swedish 0x041d: "sv_SE", # Swedish - Sweden 0x081d: "sv_FI", # Swedish - Finland - 0x045a: "syr_SY",# Syriac - 0x0428: "tg_TJ", # Tajik - Cyrillic - 0x085f: "tmz_DZ",# Tamazight - Latin - 0x0449: "ta_IN", # Tamil - 0x0444: "tt_RU", # Tatar - 0x044a: "te_IN", # Telugu - 0x041e: "th_TH", # Thai - 0x0851: "bo_BT", # Tibetan - Bhutan - 0x0451: "bo_CN", # Tibetan - PRC - 0x041f: "tr_TR", # Turkish - 0x0442: "tk_TM", # Turkmen - Cyrillic - 0x0480: "ug_CN", # Uighur - Arabic - 0x0422: "uk_UA", # Ukrainian - 0x042e: "wen_DE",# Upper Sorbian - Germany - 0x0420: "ur_PK", # Urdu + 0x005a: "syr", # Syriac + 0x045a: "syr_SY", # Syriac - Syria + 0x0028: "tg", # Tajik (Cyrillic) + 0x7c28: "tg", # Tajik (Cyrillic) + 0x0428: "tg_TJ", # Tajik (Cyrillic) - Tajikistan + 0x005f: "tzm", # Tamazight (Latin) + 0x785f: "tzm", + 0x7c5f: "tzm", # Tamazight (Latin) + 0x085f: "tzm_DZ", # Tamazight (Latin) - Algeria + 0x045f: "tzm_MA", # Central Atlas Tamazight (Arabic) - Morocco + 0x105f: "tzm_MA", + 0x0049: "ta", # Tamil + 0x0449: "ta_IN", # Tamil - India + 0x0849: "ta_LK", # Tamil - Sri Lanka + 0x0044: "tt", # Tatar + 0x0444: "tt_RU", # Tatar - Russia + 0x004a: "te", # Telugu + 0x044a: "te_IN", # Telugu - India + 0x001e: "th", # Thai + 0x041e: "th_TH", # Thai - Thailand + 0x0051: "bo", # Tibetan + 0x0451: "bo_CN", # Tibetan - People's Republic of China + 0x0073: "ti", # Tigrinya + 0x0473: "ti_ET", # Tigrinya - Ethiopia + 0x0873: "ti_ER", # Tigrinya - Eritrea + 0x0031: "ts", # Tsonga + 0x0431: "ts_ZA", # Tsonga - South Africa + 0x001f: "tr", # Turkish + 0x041f: "tr_TR", # Turkish - Turkey + 0x0042: "tk", # Turkmen + 0x0442: "tk_TM", # Turkmen - Turkmenistan + 0x0022: "uk", # Ukrainian + 0x0422: "uk_UA", # Ukrainian - Ukraine + 0x002e: "hsb", # Upper Sorbian + 0x042e: "hsb_DE", # Upper Sorbian - Germany + 0x0020: "ur", # Urdu + 0x0420: "ur_PK", # Urdu - Islamic Republic of Pakistan 0x0820: "ur_IN", # Urdu - India - 0x0443: "uz_UZ", # Uzbek - Latin - 0x0843: "uz_UZ", # Uzbek - Cyrillic - 0x042a: "vi_VN", # Vietnamese - 0x0452: "cy_GB", # Welsh + 0x0080: "ug", # Uyghur + 0x0480: "ug_CN", # Uyghur - People's Republic of China + 0x0043: "uz", # Uzbek (Latin) + 0x7843: "uz", # Uzbek (Cyrillic) + 0x7c43: "uz", # Uzbek (Latin) + 0x0443: "uz_UZ", # Uzbek (Latin) - Uzbekistan + 0x0033: "ve", # Venda + 0x0433: "ve_ZA", # Venda - South Africa + 0x002a: "vi", # Vietnamese + 0x042a: "vi_VN", # Vietnamese - Vietnam + 0x0052: "cy", # Welsh + 0x0452: "cy_GB", # Welsh - United Kingdom + 0x0088: "wo", # Wolof 0x0488: "wo_SN", # Wolof - Senegal + 0x0034: "xh", # Xhosa 0x0434: "xh_ZA", # Xhosa - South Africa - 0x0485: "sah_RU",# Yakut - Cyrillic - 0x0478: "ii_CN", # Yi - PRC + 0x0078: "ii", # Yi + 0x0478: "ii_CN", # Yi - People's Republic of China + 0x043d: "yi_001", # Yiddish - World + 0x006a: "yo", # Yoruba 0x046a: "yo_NG", # Yoruba - Nigeria - 0x0435: "zu_ZA", # Zulu + 0x0035: "zu", # Zulu + 0x0435: "zu_ZA", # Zulu - South Africa + 0x0086: "qut", + +# 0x0001007f: "x-IV-mathan", # math alphanumeric sorting + 0x00010407: "de_DE", + 0x0001040e: "hu_HU", + 0x00010437: "ka_GE", + 0x00020804: "zh_CN", + 0x00021004: "zh_SG", + 0x00021404: "zh_MO", + 0x00030404: "zh_TW", + 0x00040404: "zh_TW", + 0x00040411: "ja_JP", + 0x00040c04: "zh_HK", + 0x00041404: "zh_MO", + 0x00050804: "zh_CN", + 0x00051004: "zh_SG", } def _print_locale(): diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index aa9b79d8cab4bb8..9005f1ef865c909 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -591,6 +591,7 @@ class Formatter(object): %(threadName)s Thread name (if available) %(taskName)s Task name (if available) %(process)d Process ID (if available) + %(processName)s Process name (if available) %(message)s The result of record.getMessage(), computed just as the record is emitted """ @@ -1851,9 +1852,9 @@ class LoggerAdapter(object): def __init__(self, logger, extra=None, merge_extra=False): """ - Initialize the adapter with a logger and a dict-like object which - provides contextual information. This constructor signature allows - easy stacking of LoggerAdapters, if so desired. + Initialize the adapter with a logger and an optional dict-like object + which provides contextual information. This constructor signature + allows easy stacking of LoggerAdapters, if so desired. You can effectively pass keyword arguments as shown in the following example: @@ -1884,8 +1885,9 @@ def process(self, msg, kwargs): Normally, you'll only need to override this one method in a LoggerAdapter subclass for your specific needs. """ - if self.merge_extra and "extra" in kwargs: - kwargs["extra"] = {**self.extra, **kwargs["extra"]} + if self.merge_extra and kwargs.get("extra") is not None: + if self.extra is not None: + kwargs["extra"] = {**self.extra, **kwargs["extra"]} else: kwargs["extra"] = self.extra return msg, kwargs diff --git a/Lib/logging/config.py b/Lib/logging/config.py index c994349fd6eee54..e0524eb5e3d89c2 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -32,6 +32,7 @@ import os import queue import re +import socket import struct import threading import traceback @@ -1018,10 +1019,19 @@ class ConfigSocketReceiver(ThreadingTCPServer): """ allow_reuse_address = True - allow_reuse_port = True + allow_reuse_port = False def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT, handler=None, ready=None, verify=None): + # The host can have no IPv4 address, for example if "localhost" + # is only aliased to ::1. Leave resolution errors to the server. + try: + infos = socket.getaddrinfo(host, port, type=socket.SOCK_STREAM) + except OSError: + pass + else: + if not any(info[0] == socket.AF_INET for info in infos): + self.address_family = infos[0][0] ThreadingTCPServer.__init__(self, (host, port), handler) with logging._lock: self.abort = 0 @@ -1053,9 +1063,14 @@ def __init__(self, rcvr, hdlr, port, verify): self.ready = threading.Event() def run(self): - server = self.rcvr(port=self.port, handler=self.hdlr, - ready=self.ready, - verify=self.verify) + try: + server = self.rcvr(port=self.port, handler=self.hdlr, + ready=self.ready, + verify=self.verify) + except BaseException: + # Do not leave the caller waiting for ready forever. + self.ready.set() + raise if self.port == 0: self.port = server.server_address[1] self.ready.set() diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index 2748b5941eade2e..4a07258f8d6d07e 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -196,7 +196,11 @@ def shouldRollover(self, record): if self.stream is None: # delay was set... self.stream = self._open() if self.maxBytes > 0: # are we rolling over? - pos = self.stream.tell() + try: + pos = self.stream.tell() + except io.UnsupportedOperation: + # gh-143237: Never rollover a named pipe. + return False if not pos: # gh-116263: Never rollover an empty file return False diff --git a/Lib/mailbox.py b/Lib/mailbox.py index b00d9e8634c7853..364af6bb0109597 100644 --- a/Lib/mailbox.py +++ b/Lib/mailbox.py @@ -2183,11 +2183,7 @@ def _unlock_file(f): def _create_carefully(path): """Create a file if it doesn't exist and open for reading and writing.""" - fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR, 0o666) - try: - return open(path, 'rb+') - finally: - os.close(fd) + return open(path, 'xb+') def _create_temporary(path): """Create a temp file based on path and open for reading and writing.""" diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 33e86d51a0fe508..7d0f4c1fd400d53 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -718,24 +718,30 @@ def _parse_args(args): def _main(args=None): """Run the mimetypes command-line interface and return a text to print.""" - import sys - args, help_text = _parse_args(args) + results = [] if args.extension: for gtype in args.type: guess = guess_extension(gtype, not args.lenient) if guess: - return str(guess) - sys.exit(f"error: unknown type {gtype}") + results.append(str(guess)) + else: + results.append(f"error: unknown type {gtype}") + return results else: for gtype in args.type: guess, encoding = guess_type(gtype, not args.lenient) if guess: - return f"type: {guess} encoding: {encoding}" - sys.exit(f"error: media type unknown for {gtype}") - return help_text + results.append(f"type: {guess} encoding: {encoding}") + else: + results.append(f"error: media type unknown for {gtype}") + return results if __name__ == '__main__': - print(_main()) + import sys + + results = _main() + print("\n".join(results)) + sys.exit(any(result.startswith("error: ") for result in results)) diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index 5f288a8d3932402..f577186e91b0d4b 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -16,7 +16,6 @@ import sys import socket import struct -import tempfile import time @@ -46,6 +45,7 @@ CONNECTION_TIMEOUT = 20. _mmap_counter = itertools.count() +_MAX_PIPE_ATTEMPTS = 100 default_family = 'AF_INET' families = ['AF_INET'] @@ -76,10 +76,14 @@ def arbitrary_address(family): if family == 'AF_INET': return ('localhost', 0) elif family == 'AF_UNIX': - return tempfile.mktemp(prefix='listener-', dir=util.get_temp_dir()) + # NOTE: util.get_temp_dir() is a 0o700 per-process directory. A + # mktemp-style ToC vs ToU concern is not important; bind() surfaces + # the extremely unlikely collision as EADDRINUSE. + return os.path.join(util.get_temp_dir(), + f'sock-{os.urandom(6).hex()}') elif family == 'AF_PIPE': - return tempfile.mktemp(prefix=r'\\.\pipe\pyc-%d-%d-' % - (os.getpid(), next(_mmap_counter)), dir="") + return (r'\\.\pipe\pyc-%d-%d-%s' % + (os.getpid(), next(_mmap_counter), os.urandom(8).hex())) else: raise ValueError('unrecognized family') @@ -472,17 +476,29 @@ class Listener(object): def __init__(self, address=None, family=None, backlog=1, authkey=None): family = family or (address and address_type(address)) \ or default_family - address = address or arbitrary_address(family) - _validate_family(family) + if authkey is not None and not isinstance(authkey, bytes): + raise TypeError('authkey should be a byte string') + if family == 'AF_PIPE': - self._listener = PipeListener(address, backlog) + if address: + self._listener = PipeListener(address, backlog) + else: + for attempts in itertools.count(): + address = arbitrary_address(family) + try: + self._listener = PipeListener(address, backlog) + break + except OSError as e: + if attempts >= _MAX_PIPE_ATTEMPTS: + raise + if e.winerror not in (_winapi.ERROR_PIPE_BUSY, + _winapi.ERROR_ACCESS_DENIED): + raise else: + address = address or arbitrary_address(family) self._listener = SocketListener(address, family, backlog) - if authkey is not None and not isinstance(authkey, bytes): - raise TypeError('authkey should be a byte string') - self._authkey = authkey def accept(self): @@ -570,7 +586,6 @@ def Pipe(duplex=True): ''' Returns pair of connection objects at either end of a pipe ''' - address = arbitrary_address('AF_PIPE') if duplex: openmode = _winapi.PIPE_ACCESS_DUPLEX access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE @@ -580,15 +595,25 @@ def Pipe(duplex=True): access = _winapi.GENERIC_WRITE obsize, ibsize = 0, BUFSIZE - h1 = _winapi.CreateNamedPipe( - address, openmode | _winapi.FILE_FLAG_OVERLAPPED | - _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE, - _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE | - _winapi.PIPE_WAIT, - 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, - # default security descriptor: the handle cannot be inherited - _winapi.NULL - ) + for attempts in itertools.count(): + address = arbitrary_address('AF_PIPE') + try: + h1 = _winapi.CreateNamedPipe( + address, openmode | _winapi.FILE_FLAG_OVERLAPPED | + _winapi.FILE_FLAG_FIRST_PIPE_INSTANCE, + _winapi.PIPE_TYPE_MESSAGE | _winapi.PIPE_READMODE_MESSAGE | + _winapi.PIPE_WAIT, + 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, + # default security descriptor: the handle cannot be inherited + _winapi.NULL + ) + break + except OSError as e: + if attempts >= _MAX_PIPE_ATTEMPTS: + raise + if e.winerror not in (_winapi.ERROR_PIPE_BUSY, + _winapi.ERROR_ACCESS_DENIED): + raise h2 = _winapi.CreateFile( address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING, _winapi.FILE_FLAG_OVERLAPPED, _winapi.NULL diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index d0a3ad00e53ad80..5fa6d7e48611f03 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -145,7 +145,13 @@ def freeze_support(self): '''Check whether this is a fake forked process in a frozen executable. If so then run code specified by commandline and exit. ''' - if sys.platform == 'win32' and getattr(sys, 'frozen', False): + # gh-140814: allow_none=True avoids locking in the default start + # method, which would cause a later set_start_method() to fail. + # None is safe to pass through: spawn.freeze_support() + # independently detects whether this process is a spawned + # child, so the start method check here is only an optimization. + if (getattr(sys, 'frozen', False) + and self.get_start_method(allow_none=True) in ('spawn', None)): from .spawn import freeze_support freeze_support() diff --git a/Lib/multiprocessing/dummy/__init__.py b/Lib/multiprocessing/dummy/__init__.py index 6a1468609e347b3..7dc5d1c8dde8484 100644 --- a/Lib/multiprocessing/dummy/__init__.py +++ b/Lib/multiprocessing/dummy/__init__.py @@ -33,7 +33,7 @@ class DummyProcess(threading.Thread): - def __init__(self, group=None, target=None, name=None, args=(), kwargs={}): + def __init__(self, group=None, target=None, name=None, args=(), kwargs=None): threading.Thread.__init__(self, group, target, name, args, kwargs) self._pid = None self._children = weakref.WeakKeyDictionary() diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 681af2610e9b37d..e431b3f12988067 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -142,15 +142,25 @@ def ensure_running(self): self._forkserver_alive_fd = None self._forkserver_pid = None - cmd = ('from multiprocessing.forkserver import main; ' + - 'main(%d, %d, %r, **%r)') - + # gh-144503: sys_argv is passed as real argv elements after the + # ``-c cmd`` rather than repr'd into main_kws so that a large + # parent sys.argv cannot push the single ``-c`` command string + # over the OS per-argument length limit (MAX_ARG_STRLEN on Linux). + # The child sees them as sys.argv[1:]. + cmd = ('import sys; ' + 'from multiprocessing.forkserver import main; ' + 'main(%d, %d, %r, sys_argv=sys.argv[1:], **%r)') + + main_kws = {} + sys_argv = None if self._preload_modules: - desired_keys = {'main_path', 'sys_path'} data = spawn.get_preparation_data('ignore') - main_kws = {x: y for x, y in data.items() if x in desired_keys} - else: - main_kws = {} + if 'sys_path' in data: + main_kws['sys_path'] = data['sys_path'] + if 'init_main_from_path' in data: + main_kws['main_path'] = data['init_main_from_path'] + if 'sys_argv' in data: + sys_argv = data['sys_argv'] with socket.socket(socket.AF_UNIX) as listener: address = connection.arbitrary_address('AF_UNIX') @@ -172,6 +182,8 @@ def ensure_running(self): exe = spawn.get_executable() args = [exe] + util._args_from_interpreter_flags() args += ['-c', cmd] + if sys_argv is not None: + args += sys_argv pid = util.spawnv_passfds(exe, args, fds_to_pass) except: os.close(alive_w) @@ -196,7 +208,7 @@ def ensure_running(self): # def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, - *, authkey_r=None): + *, sys_argv=None, authkey_r=None): """Run forkserver.""" if authkey_r is not None: try: @@ -208,6 +220,8 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, authkey = b'' if preload: + if sys_argv is not None: + sys.argv[:] = sys_argv if sys_path is not None: sys.path[:] = sys_path if '__main__' in preload and main_path is not None: @@ -222,6 +236,10 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None, except ImportError: pass + # gh-135335: flush stdout/stderr in case any of the preloaded modules + # wrote to them, otherwise children might inherit buffered data + util._flush_std_streams() + util._close_stdin() sig_r, sig_w = os.pipe() diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 91bcf243e78e5b8..11c6e21de90e693 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -1294,9 +1294,9 @@ class SyncManager(BaseManager): class _SharedMemoryTracker: "Manages one or more shared memory segments." - def __init__(self, name, segment_names=[]): + def __init__(self, name, segment_names=None): self.shared_memory_context_name = name - self.segment_names = segment_names + self.segment_names = [] if segment_names is None else segment_names def register_segment(self, segment_name): "Adds the supplied shared memory block name to tracker." diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index 7affa1b985f0916..a02a53b6a176da9 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -67,7 +67,17 @@ def _launch(self, process_obj): code = 1 parent_r, child_w = os.pipe() child_r, parent_w = os.pipe() - self.pid = os.fork() + # gh-146313: Tell the resource tracker's at-fork handler to keep + # the inherited pipe fd so this child reuses the parent's tracker + # (gh-80849) rather than closing it and launching its own. + from .resource_tracker import _fork_intent + _fork_intent.preserve_fd = True + try: + self.pid = os.fork() + finally: + # Reset in both parent and child so the flag does not leak + # into a subsequent raw os.fork() or nested Process launch. + _fork_intent.preserve_fd = False if self.pid == 0: try: atexit._clear() diff --git a/Lib/multiprocessing/popen_spawn_posix.py b/Lib/multiprocessing/popen_spawn_posix.py index 24b8634523e5f2c..cccd659ae776377 100644 --- a/Lib/multiprocessing/popen_spawn_posix.py +++ b/Lib/multiprocessing/popen_spawn_posix.py @@ -57,6 +57,10 @@ def _launch(self, process_obj): self._fds.extend([child_r, child_w]) self.pid = util.spawnv_passfds(spawn.get_executable(), cmd, self._fds) + os.close(child_r) + child_r = None + os.close(child_w) + child_w = None self.sentinel = parent_r with open(parent_w, 'wb', closefd=False) as f: f.write(fp.getbuffer()) diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 9db322be1aa6d63..262513f295fde56 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -77,7 +77,7 @@ class BaseProcess(object): def _Popen(self): raise NotImplementedError - def __init__(self, group=None, target=None, name=None, args=(), kwargs={}, + def __init__(self, group=None, target=None, name=None, args=(), kwargs=None, *, daemon=None): assert group is None, 'group argument must be None for now' count = next(_process_counter) @@ -89,7 +89,7 @@ def __init__(self, group=None, target=None, name=None, args=(), kwargs={}, self._closed = False self._target = target self._args = tuple(args) - self._kwargs = dict(kwargs) + self._kwargs = dict(kwargs) if kwargs else {} self._name = name or type(self).__name__ + '-' + \ ':'.join(str(i) for i in self._identity) if daemon is not None: diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index 925f043900004e7..981599acf5ef26e 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -121,7 +121,7 @@ def get(self, block=True, timeout=None): def qsize(self): # Raises NotImplementedError on Mac OSX because of broken sem_getvalue() - return self._maxsize - self._sem._semlock._get_value() + return self._maxsize - self._sem.get_value() def empty(self): return not self._poll() diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 05633ac21a259cf..dfb5a3dfe8b7b6e 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -15,11 +15,16 @@ # this resource tracker process, "killall python" would probably leave unlinked # resources. +import base64 import os import signal import sys import threading +import time import warnings +from collections import deque + +import json from . import spawn from . import util @@ -47,12 +52,8 @@ def cleanup_noop(name): # absence of POSIX named semaphores. In that case, no named semaphores were # ever opened, so no cleanup would be necessary. if hasattr(_multiprocessing, 'sem_unlink'): - _CLEANUP_FUNCS.update({ - 'semaphore': _multiprocessing.sem_unlink, - }) - _CLEANUP_FUNCS.update({ - 'shared_memory': _posixshmem.shm_unlink, - }) + _CLEANUP_FUNCS['semaphore'] = _multiprocessing.sem_unlink + _CLEANUP_FUNCS['shared_memory'] = _posixshmem.shm_unlink class ReentrantCallError(RuntimeError): @@ -66,6 +67,18 @@ def __init__(self): self._fd = None self._pid = None self._exitcode = None + self._reentrant_messages = deque() + + # True to use colon-separated lines, rather than JSON lines, + # for internal communication. (Mainly for testing). + # Filenames not supported by the simple format will always be sent + # using JSON. + # The reader should understand all formats. + self._use_simple_format = True + + # Set to True by _stop_locked() if the waitpid polling loop ran to + # its timeout without reaping the tracker. Exposed for tests. + self._waitpid_timed_out = False def _reentrant_call_error(self): # gh-109629: this happens if an explicit call to the ResourceTracker @@ -79,16 +92,51 @@ def __del__(self): # making sure child processess are cleaned before ResourceTracker # gets destructed. # see https://github.com/python/cpython/issues/88887 - self._stop(use_blocking_lock=False) + # gh-146313: use a timeout to avoid deadlocking if a forked child + # still holds the pipe's write end open. + self._stop(use_blocking_lock=False, wait_timeout=1.0) + + def _after_fork_in_child(self): + # gh-146313: Called in the child right after os.fork(). + # + # The tracker process is a child of the *parent*, not of us, so we + # could never waitpid() it anyway. Clearing _pid means our __del__ + # becomes a no-op (the early return for _pid is None). + # + # Whether we keep the inherited _fd depends on who forked us: + # + # - multiprocessing.Process with the 'fork' start method sets + # _fork_intent.preserve_fd before forking. The child keeps the + # fd and reuses the parent's tracker (gh-80849). This is safe + # because multiprocessing's atexit handler joins all children + # before the parent's __del__ runs, so by then the fd copies + # are gone and the parent can reap the tracker promptly. + # + # - A raw os.fork() leaves the flag unset. We close the fd in the child after forking so + # the parent's __del__ can reap the tracker without waiting + # for the child to exit. If we later need a tracker, ensure_running() + # will launch a fresh one. + self._lock._at_fork_reinit() + self._reentrant_messages.clear() + self._pid = None + self._exitcode = None + if (self._fd is not None and + not getattr(_fork_intent, 'preserve_fd', False)): + fd = self._fd + self._fd = None + try: + os.close(fd) + except OSError: + pass - def _stop(self, use_blocking_lock=True): + def _stop(self, use_blocking_lock=True, wait_timeout=None): if use_blocking_lock: with self._lock: - self._stop_locked() + self._stop_locked(wait_timeout=wait_timeout) else: acquired = self._lock.acquire(blocking=False) try: - self._stop_locked() + self._stop_locked(wait_timeout=wait_timeout) finally: if acquired: self._lock.release() @@ -98,11 +146,15 @@ def _stop_locked( close=os.close, waitpid=os.waitpid, waitstatus_to_exitcode=os.waitstatus_to_exitcode, + monotonic=time.monotonic, + sleep=time.sleep, + WNOHANG=getattr(os, 'WNOHANG', None), + wait_timeout=None, ): # This shouldn't happen (it might when called by a finalizer) # so we check for it anyway. if self._lock._recursion_count() > 1: - return self._reentrant_call_error() + raise self._reentrant_call_error() if self._fd is None: # not running return @@ -113,7 +165,35 @@ def _stop_locked( close(self._fd) self._fd = None - _, status = waitpid(self._pid, 0) + try: + if wait_timeout is None: + _, status = waitpid(self._pid, 0) + else: + # gh-146313: A forked child may still hold the pipe's write + # end open, preventing the tracker from seeing EOF and + # exiting. Poll with WNOHANG to avoid blocking forever. + deadline = monotonic() + wait_timeout + delay = 0.001 + while True: + result_pid, status = waitpid(self._pid, WNOHANG) + if result_pid != 0: + break + remaining = deadline - monotonic() + if remaining <= 0: + # The tracker is still running; it will be + # reparented to PID 1 (or the nearest subreaper) + # when we exit, and reaped there once all pipe + # holders release their fd. + self._pid = None + self._exitcode = None + self._waitpid_timed_out = True + return + delay = min(delay * 2, remaining, 0.1) + sleep(delay) + except ChildProcessError: + self._pid = None + self._exitcode = None + return self._pid = None @@ -132,76 +212,119 @@ def ensure_running(self): This can be run from any process. Usually a child process will use the resource created by its parent.''' + return self._ensure_running_and_write() + + def _teardown_dead_process(self): + os.close(self._fd) + + # Clean-up to avoid dangling processes. + try: + # _pid can be None if this process is a child from another + # python process, which has started the resource_tracker. + if self._pid is not None: + os.waitpid(self._pid, 0) + except ChildProcessError: + # The resource_tracker has already been terminated. + pass + self._fd = None + self._pid = None + self._exitcode = None + + warnings.warn('resource_tracker: process died unexpectedly, ' + 'relaunching. Some resources might leak.') + + def _launch(self): + fds_to_pass = [] + try: + fds_to_pass.append(sys.stderr.fileno()) + except Exception: + pass + r, w = os.pipe() + try: + fds_to_pass.append(r) + # process will out live us, so no need to wait on pid + exe = spawn.get_executable() + args = [ + exe, + *util._args_from_interpreter_flags(), + '-c', + f'from multiprocessing.resource_tracker import main;main({r})', + ] + # bpo-33613: Register a signal mask that will block the signals. + # This signal mask will be inherited by the child that is going + # to be spawned and will protect the child from a race condition + # that can make the child die before it registers signal handlers + # for SIGINT and SIGTERM. The mask is unregistered after spawning + # the child. + prev_sigmask = None + try: + if _HAVE_SIGMASK: + prev_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) + pid = util.spawnv_passfds(exe, args, fds_to_pass) + finally: + if prev_sigmask is not None: + signal.pthread_sigmask(signal.SIG_SETMASK, prev_sigmask) + except: + os.close(w) + raise + else: + self._fd = w + self._pid = pid + finally: + os.close(r) + + def _make_probe_message(self): + """Return a probe message.""" + if self._use_simple_format: + return b'PROBE:0:noop\n' + return ( + json.dumps( + {"cmd": "PROBE", "rtype": "noop"}, + ensure_ascii=True, + separators=(",", ":"), + ) + + "\n" + ).encode("ascii") + + def _ensure_running_and_write(self, msg=None): with self._lock: if self._lock._recursion_count() > 1: # The code below is certainly not reentrant-safe, so bail out - return self._reentrant_call_error() + if msg is None: + raise self._reentrant_call_error() + return self._reentrant_messages.append(msg) + if self._fd is not None: # resource tracker was launched before, is it still running? - if self._check_alive(): - # => still alive - return - # => dead, launch it again - os.close(self._fd) - - # Clean-up to avoid dangling processes. + if msg is None: + to_send = self._make_probe_message() + else: + to_send = msg try: - # _pid can be None if this process is a child from another - # python process, which has started the resource_tracker. - if self._pid is not None: - os.waitpid(self._pid, 0) - except ChildProcessError: - # The resource_tracker has already been terminated. - pass - self._fd = None - self._pid = None - self._exitcode = None + self._write(to_send) + except OSError: + self._teardown_dead_process() + self._launch() - warnings.warn('resource_tracker: process died unexpectedly, ' - 'relaunching. Some resources might leak.') + msg = None # message was sent in probe + else: + self._launch() - fds_to_pass = [] + while True: try: - fds_to_pass.append(sys.stderr.fileno()) - except Exception: - pass - cmd = 'from multiprocessing.resource_tracker import main;main(%d)' - r, w = os.pipe() - try: - fds_to_pass.append(r) - # process will out live us, so no need to wait on pid - exe = spawn.get_executable() - args = [exe] + util._args_from_interpreter_flags() - args += ['-c', cmd % r] - # bpo-33613: Register a signal mask that will block the signals. - # This signal mask will be inherited by the child that is going - # to be spawned and will protect the child from a race condition - # that can make the child die before it registers signal handlers - # for SIGINT and SIGTERM. The mask is unregistered after spawning - # the child. - prev_sigmask = None - try: - if _HAVE_SIGMASK: - prev_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) - pid = util.spawnv_passfds(exe, args, fds_to_pass) - finally: - if prev_sigmask is not None: - signal.pthread_sigmask(signal.SIG_SETMASK, prev_sigmask) - except: - os.close(w) - raise - else: - self._fd = w - self._pid = pid - finally: - os.close(r) + reentrant_msg = self._reentrant_messages.popleft() + except IndexError: + break + self._write(reentrant_msg) + if msg is not None: + self._write(msg) def _check_alive(self): '''Check that the pipe has not been closed by sending a probe.''' try: # We cannot use send here as it calls ensure_running, creating # a cycle. - os.write(self._fd, b'PROBE:0:noop\n') + os.write(self._fd, self._make_probe_message()) except OSError: return False else: @@ -215,27 +338,50 @@ def unregister(self, name, rtype): '''Unregister name of resource with resource tracker.''' self._send('UNREGISTER', name, rtype) - def _send(self, cmd, name, rtype): - try: - self.ensure_running() - except ReentrantCallError: - # The code below might or might not work, depending on whether - # the resource tracker was already running and still alive. - # Better warn the user. - # (XXX is warnings.warn itself reentrant-safe? :-) - warnings.warn( - f"ResourceTracker called reentrantly for resource cleanup, " - f"which is unsupported. " - f"The {rtype} object {name!r} might leak.") - msg = '{0}:{1}:{2}\n'.format(cmd, name, rtype).encode('ascii') - if len(msg) > 512: - # posix guarantees that writes to a pipe of less than PIPE_BUF - # bytes are atomic, and that PIPE_BUF >= 512 - raise ValueError('msg too long') + def _write(self, msg): nbytes = os.write(self._fd, msg) - assert nbytes == len(msg), "nbytes {0:n} but len(msg) {1:n}".format( - nbytes, len(msg)) + assert nbytes == len(msg), f"{nbytes=} != {len(msg)=}" + + def _send(self, cmd, name, rtype): + if self._use_simple_format and '\n' not in name: + msg = f"{cmd}:{name}:{rtype}\n".encode("ascii") + if len(msg) > 512: + # posix guarantees that writes to a pipe of less than PIPE_BUF + # bytes are atomic, and that PIPE_BUF >= 512 + raise ValueError('msg too long') + self._ensure_running_and_write(msg) + return + # POSIX guarantees that writes to a pipe of less than PIPE_BUF (512 on Linux) + # bytes are atomic. Therefore, we want the message to be shorter than 512 bytes. + # POSIX shm_open() and sem_open() require the name, including its leading slash, + # to be at most NAME_MAX bytes (255 on Linux) + # With json.dump(..., ensure_ascii=True) every non-ASCII byte becomes a 6-char + # escape like \uDC80. + # As we want the overall message to be kept atomic and therefore smaller than 512, + # we encode encode the raw name bytes with URL-safe Base64 - so a 255 long name + # will not exceed 340 bytes. + b = name.encode('utf-8', 'surrogateescape') + if len(b) > 255: + raise ValueError('shared memory name too long (max 255 bytes)') + b64 = base64.urlsafe_b64encode(b).decode('ascii') + + payload = {"cmd": cmd, "rtype": rtype, "base64_name": b64} + msg = (json.dumps(payload, ensure_ascii=True, separators=(",", ":")) + "\n").encode("ascii") + + # The entire JSON message is guaranteed < PIPE_BUF (512 bytes) by construction. + assert len(msg) <= 512, f"internal error: message too long ({len(msg)} bytes)" + assert msg.startswith(b'{') + + self._ensure_running_and_write(msg) + +# gh-146313: Per-thread flag set by .popen_fork.Popen._launch() just before +# os.fork(), telling _after_fork_in_child() to keep the inherited pipe fd so +# the child can reuse this tracker (gh-80849). Unset for raw os.fork() calls, +# where the child instead closes the fd so the parent's __del__ can reap the +# tracker. Using threading.local() keeps multiple threads calling +# popen_fork.Popen._launch() at once from clobbering eachothers intent. +_fork_intent = threading.local() _resource_tracker = ResourceTracker() ensure_running = _resource_tracker.ensure_running @@ -243,6 +389,34 @@ def _send(self, cmd, name, rtype): unregister = _resource_tracker.unregister getfd = _resource_tracker.getfd +# gh-146313: See _after_fork_in_child docstring. +if hasattr(os, 'register_at_fork'): + os.register_at_fork(after_in_child=_resource_tracker._after_fork_in_child) + + +def _decode_message(line): + if line.startswith(b'{'): + try: + obj = json.loads(line.decode('ascii')) + except Exception as e: + raise ValueError("malformed resource_tracker message: %r" % (line,)) from e + + cmd = obj["cmd"] + rtype = obj["rtype"] + b64 = obj.get("base64_name", "") + + if not isinstance(cmd, str) or not isinstance(rtype, str) or not isinstance(b64, str): + raise ValueError("malformed resource_tracker fields: %r" % (obj,)) + + try: + name = base64.urlsafe_b64decode(b64).decode('utf-8', 'surrogateescape') + except ValueError as e: + raise ValueError("malformed resource_tracker base64_name: %r" % (b64,)) from e + else: + cmd, rest = line.strip().decode('ascii').split(':', maxsplit=1) + name, rtype = rest.rsplit(':', maxsplit=1) + return cmd, rtype, name + def main(fd): '''Run resource tracker.''' @@ -266,7 +440,7 @@ def main(fd): with open(fd, 'rb') as f: for line in f: try: - cmd, name, rtype = line.strip().decode('ascii').split(':') + cmd, rtype, name = _decode_message(line) cleanup_func = _CLEANUP_FUNCS.get(rtype, None) if cleanup_func is None: raise ValueError( diff --git a/Lib/multiprocessing/spawn.py b/Lib/multiprocessing/spawn.py index daac1ecc34b55ec..d43864c939cb63f 100644 --- a/Lib/multiprocessing/spawn.py +++ b/Lib/multiprocessing/spawn.py @@ -184,7 +184,7 @@ def get_preparation_data(name): sys_argv=sys.argv, orig_dir=process.ORIGINAL_DIR, dir=os.getcwd(), - start_method=get_start_method(), + start_method=get_start_method(allow_none=True), ) # Figure out whether to initialise main in the subprocess as a module diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index 30425047e9801a5..9188114ae284c74 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -135,11 +135,16 @@ def __init__(self, value=1, *, ctx): SemLock.__init__(self, SEMAPHORE, value, SEM_VALUE_MAX, ctx=ctx) def get_value(self): + '''Returns current value of Semaphore. + + Raises NotImplementedError on Mac OSX + because of broken sem_getvalue(). + ''' return self._semlock._get_value() def __repr__(self): try: - value = self._semlock._get_value() + value = self.get_value() except Exception: value = 'unknown' return '<%s(value=%s)>' % (self.__class__.__name__, value) @@ -155,7 +160,7 @@ def __init__(self, value=1, *, ctx): def __repr__(self): try: - value = self._semlock._get_value() + value = self.get_value() except Exception: value = 'unknown' return '<%s(value=%s, maxvalue=%s)>' % \ @@ -247,8 +252,8 @@ def _make_methods(self): def __repr__(self): try: - num_waiters = (self._sleeping_count._semlock._get_value() - - self._woken_count._semlock._get_value()) + num_waiters = (self._sleeping_count.get_value() - + self._woken_count.get_value()) except Exception: num_waiters = 'unknown' return '<%s(%s, %s)>' % (self.__class__.__name__, self._lock, num_waiters) diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index b7192042b9cf474..549fb07c27549e0 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -19,7 +19,7 @@ from . import process __all__ = [ - 'sub_debug', 'debug', 'info', 'sub_warning', 'get_logger', + 'sub_debug', 'debug', 'info', 'sub_warning', 'warn', 'get_logger', 'log_to_stderr', 'get_temp_dir', 'register_after_fork', 'is_exiting', 'Finalize', 'ForkAwareThreadLock', 'ForkAwareLocal', 'close_all_fds_except', 'SUBDEBUG', 'SUBWARNING', @@ -34,6 +34,7 @@ DEBUG = 10 INFO = 20 SUBWARNING = 25 +WARNING = 30 LOGGER_NAME = 'multiprocessing' DEFAULT_LOGGING_FORMAT = '[%(levelname)s/%(processName)s] %(message)s' @@ -53,6 +54,10 @@ def info(msg, *args): if _logger: _logger.log(INFO, msg, *args, stacklevel=2) +def warn(msg, *args): + if _logger: + _logger.log(WARNING, msg, *args, stacklevel=2) + def sub_warning(msg, *args): if _logger: _logger.log(SUBWARNING, msg, *args, stacklevel=2) @@ -121,6 +126,23 @@ def is_abstract_socket_namespace(address): # Function returning a temp directory which will be removed on exit # +# Maximum length of a NULL-terminated [1] socket file path is usually +# between 92 and 108 [2], but Linux is known to use a size of 108 [3]. +# BSD-based systems usually use a size of 104 or 108 and Windows does +# not create AF_UNIX sockets. +# +# [1]: https://github.com/python/cpython/issues/140734 +# [2]: https://pubs.opengroup.org/onlinepubs/9799919799/basedefs/sys_un.h.html +# [3]: https://man7.org/linux/man-pages/man7/unix.7.html + +if sys.platform == 'linux': + _SUN_PATH_MAX = 108 +elif sys.platform.startswith(('openbsd', 'freebsd')): + _SUN_PATH_MAX = 104 +else: + # On Windows platforms, we do not create AF_UNIX sockets. + _SUN_PATH_MAX = None if os.name == 'nt' else 92 + def _remove_temp_dir(rmtree, tempdir): rmtree(tempdir) @@ -130,12 +152,69 @@ def _remove_temp_dir(rmtree, tempdir): if current_process is not None: current_process._config['tempdir'] = None +def _get_base_temp_dir(tempfile): + """Get a temporary directory where socket files will be created. + + To prevent additional imports, pass a pre-imported 'tempfile' module. + """ + if os.name == 'nt': + return None + # Most of the time, the default temporary directory is /tmp. Thus, + # listener sockets files "$TMPDIR/pymp-XXXXXXXX/sock-XXXXXXXX" do + # not have a path length exceeding SUN_PATH_MAX. + # + # If users specify their own temporary directory, we may be unable + # to create those files. Therefore, we fall back to the system-wide + # temporary directory /tmp, assumed to exist on POSIX systems. + # + # See https://github.com/python/cpython/issues/132124. + base_tempdir = tempfile.gettempdir() + # Files created in a temporary directory are suffixed by a string + # generated by tempfile._RandomNameSequence, which, by design, + # is 8 characters long. + # + # Thus, the socket file path length (without NULL terminator) will be: + # + # len(base_tempdir + '/pymp-XXXXXXXX' + '/sock-XXXXXXXX') + sun_path_len = len(base_tempdir) + 14 + 14 + # Strict inequality to account for the NULL terminator. + # See https://github.com/python/cpython/issues/140734. + if sun_path_len < _SUN_PATH_MAX: + return base_tempdir + # Fallback to the default system-wide temporary directory. + # This ignores user-defined environment variables. + # + # On POSIX systems, /tmp MUST be writable by any application [1]. + # We however emit a warning if this is not the case to prevent + # obscure errors later in the execution. + # + # On some legacy systems, /var/tmp and /usr/tmp can be present + # and will be used instead. + # + # [1]: https://refspecs.linuxfoundation.org/FHS_3.0/fhs/ch03s18.html + dirlist = ['/tmp', '/var/tmp', '/usr/tmp'] + try: + base_system_tempdir = tempfile._get_default_tempdir(dirlist) + except FileNotFoundError: + warn("Process-wide temporary directory %s will not be usable for " + "creating socket files and no usable system-wide temporary " + "directory was found in %s", base_tempdir, dirlist) + # At this point, the system-wide temporary directory is not usable + # but we may assume that the user-defined one is, even if we will + # not be able to write socket files out there. + return base_tempdir + warn("Ignoring user-defined temporary directory: %s", base_tempdir) + # at most max(map(len, dirlist)) + 14 + 14 = 36 characters + assert len(base_system_tempdir) + 14 + 14 < _SUN_PATH_MAX + return base_system_tempdir + def get_temp_dir(): # get name of a temp directory which will be automatically cleaned up tempdir = process.current_process()._config.get('tempdir') if tempdir is None: import shutil, tempfile - tempdir = tempfile.mkdtemp(prefix='pymp-') + base_tempdir = _get_base_temp_dir(tempfile) + tempdir = tempfile.mkdtemp(prefix='pymp-', dir=base_tempdir) info('created temp directory %s', tempdir) # keep a strong reference to shutil.rmtree(), since the finalizer # can be called late during Python shutdown diff --git a/Lib/netrc.py b/Lib/netrc.py index b285fd8e357ddbc..bd003e80a48081e 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -7,6 +7,19 @@ __all__ = ["netrc", "NetrcParseError"] +def _can_security_check(): + # On WASI, getuid() is indicated as a stub but it may also be missing. + return os.name == 'posix' and hasattr(os, 'getuid') + + +def _getpwuid(uid): + try: + import pwd + return pwd.getpwuid(uid)[0] + except (ImportError, LookupError): + return f'uid {uid}' + + class NetrcParseError(Exception): """Exception raised on syntax errors in the .netrc file.""" def __init__(self, msg, filename=None, lineno=None): @@ -142,18 +155,12 @@ def _parse(self, file, fp, default_netrc): self._security_check(fp, default_netrc, self.hosts[entryname][0]) def _security_check(self, fp, default_netrc, login): - if os.name == 'posix' and default_netrc and login != "anonymous": + if _can_security_check() and default_netrc and login != "anonymous": prop = os.fstat(fp.fileno()) - if prop.st_uid != os.getuid(): - import pwd - try: - fowner = pwd.getpwuid(prop.st_uid)[0] - except KeyError: - fowner = 'uid %s' % prop.st_uid - try: - user = pwd.getpwuid(os.getuid())[0] - except KeyError: - user = 'uid %s' % os.getuid() + current_user_id = os.getuid() + if prop.st_uid != current_user_id: + fowner = _getpwuid(prop.st_uid) + user = _getpwuid(current_user_id) raise NetrcParseError( (f"~/.netrc file owner ({fowner}, {user}) does not match" " current user")) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 5481bb8888ef591..36da5b7cb0fb2a6 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -29,7 +29,7 @@ "abspath","curdir","pardir","sep","pathsep","defpath","altsep", "extsep","devnull","realpath","supports_unicode_filenames","relpath", "samefile", "sameopenfile", "samestat", "commonpath", "isjunction", - "isdevdrive"] + "isdevdrive", "ALLOW_MISSING"] def _get_bothseps(path): if isinstance(path, bytes): @@ -152,12 +152,14 @@ def splitdrive(p): It is always true that: result[0] + result[1] == p - If the path contained a drive letter, drive_or_unc will contain everything - up to and including the colon. e.g. splitdrive("c:/dir") returns ("c:", "/dir") + If the path contained a drive letter, drive_or_unc will contain + everything up to and including the colon. e.g. splitdrive("c:/dir") + returns ("c:", "/dir") - If the path contained a UNC path, the drive_or_unc will contain the host name - and share up to but not including the fourth directory separator character. - e.g. splitdrive("//host/computer/dir") returns ("//host/computer", "/dir") + If the path contained a UNC path, the drive_or_unc will contain the + host name and share up to but not including the fourth directory + separator character. e.g. splitdrive("//host/computer/dir") returns + ("//host/computer", "/dir") Paths cannot contain both a drive letter and a UNC path. @@ -222,8 +224,8 @@ def splitroot(p): def split(p): """Split a pathname. - Return tuple (head, tail) where tail is everything after the final slash. - Either part may be empty.""" + Return tuple (head, tail) where tail is everything after the final + slash. Either part may be empty.""" p = os.fspath(p) seps = _get_bothseps(p) d, r, p = splitroot(p) @@ -343,7 +345,7 @@ def _isreservedname(name): def expanduser(path): """Expand ~ and ~user constructs. - If user or $HOME is unknown, do nothing.""" + If user or home directory is unknown, do nothing.""" path = os.fspath(path) if isinstance(path, bytes): seps = b'\\/' @@ -400,17 +402,23 @@ def expanduser(path): # XXX With COMMAND.COM you can use any characters in a variable name, # XXX except '^|<>='. +_varpattern = r"'[^']*'?|%(%|[^%]*%?)|\$(\$|[-\w]+|\{[^}]*\}?)" +_varsub = None +_varsubb = None + def expandvars(path): """Expand shell variables of the forms $var, ${var} and %var%. Unknown variables are left unchanged.""" path = os.fspath(path) + global _varsub, _varsubb if isinstance(path, bytes): if b'$' not in path and b'%' not in path: return path - import string - varchars = bytes(string.ascii_letters + string.digits + '_-', 'ascii') - quote = b'\'' + if not _varsubb: + import re + _varsubb = re.compile(_varpattern.encode(), re.ASCII).sub + sub = _varsubb percent = b'%' brace = b'{' rbrace = b'}' @@ -419,94 +427,44 @@ def expandvars(path): else: if '$' not in path and '%' not in path: return path - import string - varchars = string.ascii_letters + string.digits + '_-' - quote = '\'' + if not _varsub: + import re + _varsub = re.compile(_varpattern, re.ASCII).sub + sub = _varsub percent = '%' brace = '{' rbrace = '}' dollar = '$' environ = os.environ - res = path[:0] - index = 0 - pathlen = len(path) - while index < pathlen: - c = path[index:index+1] - if c == quote: # no expansion within single quotes - path = path[index + 1:] - pathlen = len(path) - try: - index = path.index(c) - res += c + path[:index + 1] - except ValueError: - res += c + path - index = pathlen - 1 - elif c == percent: # variable or '%' - if path[index + 1:index + 2] == percent: - res += c - index += 1 - else: - path = path[index+1:] - pathlen = len(path) - try: - index = path.index(percent) - except ValueError: - res += percent + path - index = pathlen - 1 - else: - var = path[:index] - try: - if environ is None: - value = os.fsencode(os.environ[os.fsdecode(var)]) - else: - value = environ[var] - except KeyError: - value = percent + var + percent - res += value - elif c == dollar: # variable or '$$' - if path[index + 1:index + 2] == dollar: - res += c - index += 1 - elif path[index + 1:index + 2] == brace: - path = path[index+2:] - pathlen = len(path) - try: - index = path.index(rbrace) - except ValueError: - res += dollar + brace + path - index = pathlen - 1 - else: - var = path[:index] - try: - if environ is None: - value = os.fsencode(os.environ[os.fsdecode(var)]) - else: - value = environ[var] - except KeyError: - value = dollar + brace + var + rbrace - res += value - else: - var = path[:0] - index += 1 - c = path[index:index + 1] - while c and c in varchars: - var += c - index += 1 - c = path[index:index + 1] - try: - if environ is None: - value = os.fsencode(os.environ[os.fsdecode(var)]) - else: - value = environ[var] - except KeyError: - value = dollar + var - res += value - if c: - index -= 1 + + def repl(m): + lastindex = m.lastindex + if lastindex is None: + return m[0] + name = m[lastindex] + if lastindex == 1: + if name == percent: + return name + if not name.endswith(percent): + return m[0] + name = name[:-1] else: - res += c - index += 1 - return res + if name == dollar: + return name + if name.startswith(brace): + if not name.endswith(rbrace): + return m[0] + name = name[1:-1] + + try: + if environ is None: + return os.fsencode(os.environ[os.fsdecode(name)]) + else: + return environ[name] + except KeyError: + return m[0] + + return sub(repl, path) # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A\B. @@ -601,9 +559,10 @@ def abspath(path): from nt import _findfirstfile, _getfinalpathname, readlink as _nt_readlink except ImportError: # realpath is a no-op on systems without _getfinalpathname support. - realpath = abspath + def realpath(path, *, strict=False): + return abspath(path) else: - def _readlink_deep(path): + def _readlink_deep(path, ignored_error=OSError): # These error codes indicate that we should stop reading links and # return the path we currently have. # 1: ERROR_INVALID_FUNCTION @@ -636,7 +595,7 @@ def _readlink_deep(path): path = old_path break path = normpath(join(dirname(old_path), path)) - except OSError as ex: + except ignored_error as ex: if ex.winerror in allowed_winerror: break raise @@ -645,7 +604,7 @@ def _readlink_deep(path): break return path - def _getfinalpathname_nonstrict(path): + def _getfinalpathname_nonstrict(path, ignored_error=OSError): # These error codes indicate that we should stop resolving the path # and return the value we currently have. # 1: ERROR_INVALID_FUNCTION @@ -661,9 +620,10 @@ def _getfinalpathname_nonstrict(path): # 87: ERROR_INVALID_PARAMETER # 123: ERROR_INVALID_NAME # 161: ERROR_BAD_PATHNAME + # 1005: ERROR_UNRECOGNIZED_VOLUME # 1920: ERROR_CANT_ACCESS_FILE # 1921: ERROR_CANT_RESOLVE_FILENAME (implies unfollowable symlink) - allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 53, 65, 67, 87, 123, 161, 1920, 1921 + allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 53, 65, 67, 87, 123, 161, 1005, 1920, 1921 # Non-strict algorithm is to find as much of the target directory # as we can and join the rest. @@ -672,17 +632,18 @@ def _getfinalpathname_nonstrict(path): try: path = _getfinalpathname(path) return join(path, tail) if tail else path - except OSError as ex: + except ignored_error as ex: if ex.winerror not in allowed_winerror: raise try: # The OS could not resolve this path fully, so we attempt # to follow the link ourselves. If we succeed, join the tail # and return. - new_path = _readlink_deep(path) + new_path = _readlink_deep(path, + ignored_error=ignored_error) if new_path != path: return join(new_path, tail) if tail else new_path - except OSError: + except ignored_error: # If we fail to readlink(), let's keep traversing pass # If we get these errors, try to get the real name of the file without accessing it. @@ -690,7 +651,7 @@ def _getfinalpathname_nonstrict(path): try: name = _findfirstfile(path) path, _ = split(path) - except OSError: + except ignored_error: path, name = split(path) else: path, name = split(path) @@ -720,6 +681,15 @@ def realpath(path, *, strict=False): if normcase(path) == devnull: return '\\\\.\\NUL' had_prefix = path.startswith(prefix) + + if strict is ALLOW_MISSING: + ignored_error = FileNotFoundError + strict = True + elif strict: + ignored_error = () + else: + ignored_error = OSError + if not had_prefix and not isabs(path): path = join(cwd, path) try: @@ -727,17 +697,16 @@ def realpath(path, *, strict=False): initial_winerror = 0 except ValueError as ex: # gh-106242: Raised for embedded null characters - # In strict mode, we convert into an OSError. + # In strict modes, we convert into an OSError. # Non-strict mode returns the path as-is, since we've already # made it absolute. if strict: raise OSError(str(ex)) from None path = normpath(path) - except OSError as ex: - if strict: - raise + except ignored_error as ex: initial_winerror = ex.winerror - path = _getfinalpathname_nonstrict(path) + path = _getfinalpathname_nonstrict(path, + ignored_error=ignored_error) # The path returned by _getfinalpathname will always start with \\?\ - # strip off that prefix unless it was already provided on the original # path. diff --git a/Lib/numbers.py b/Lib/numbers.py index a2913e32cfada76..37fddb8917727b6 100644 --- a/Lib/numbers.py +++ b/Lib/numbers.py @@ -290,18 +290,27 @@ def conjugate(self): class Rational(Real): - """.numerator and .denominator should be in lowest terms.""" + """To Real, Rational adds numerator and denominator properties. + + The numerator and denominator values should be in lowest terms, + with a positive denominator. + """ __slots__ = () @property @abstractmethod def numerator(self): + """The numerator of a rational number in lowest terms.""" raise NotImplementedError @property @abstractmethod def denominator(self): + """The denominator of a rational number in lowest terms. + + This denominator should be positive. + """ raise NotImplementedError # Concrete implementation of Real's conversion to float. diff --git a/Lib/os.py b/Lib/os.py index 266e40b56f6c81a..9bb00f45f0121fd 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -10,7 +10,7 @@ - os.extsep is the extension separator (always '.') - os.altsep is the alternate pathname separator (None or '/') - os.pathsep is the component separator used in $PATH etc - - os.linesep is the line separator in text files ('\r' or '\n' or '\r\n') + - os.linesep is the line separator in text files ('\n' or '\r\n') - os.defpath is the default search path for executables - os.devnull is the file path of the null device ('/dev/null', etc.) @@ -118,6 +118,7 @@ def _add(str, fn): _add("HAVE_FCHMODAT", "chmod") _add("HAVE_FCHOWNAT", "chown") _add("HAVE_FSTATAT", "stat") + _add("HAVE_LSTAT", "lstat") _add("HAVE_FUTIMESAT", "utime") _add("HAVE_LINKAT", "link") _add("HAVE_MKDIRAT", "mkdir") @@ -210,11 +211,11 @@ def _add(str, fn): def makedirs(name, mode=0o777, exist_ok=False): """makedirs(name [, mode=0o777][, exist_ok=False]) - Super-mkdir; create a leaf directory and all intermediate ones. Works like - mkdir, except that any intermediate path segment (not just the rightmost) - will be created if it does not exist. If the target directory already - exists, raise an OSError if exist_ok is False. Otherwise no exception is - raised. This is recursive. + Super-mkdir; create a leaf directory and all intermediate ones. Works + like mkdir, except that any intermediate path segment (not just the + rightmost) will be created if it does not exist. If the target + directory already exists, raise an OSError if exist_ok is False. + Otherwise no exception is raised. This is recursive. """ head, tail = path.split(name) @@ -302,12 +303,12 @@ def walk(top, topdown=True, onerror=None, followlinks=False): dirpath, dirnames, filenames dirpath is a string, the path to the directory. dirnames is a list of - the names of the subdirectories in dirpath (including symlinks to directories, - and excluding '.' and '..'). + the names of the subdirectories in dirpath (including symlinks to + directories, and excluding '.' and '..'). filenames is a list of the names of the non-directory files in dirpath. - Note that the names in the lists are just names, with no path components. - To get a full path (which begins with top) to a file or directory in - dirpath, do os.path.join(dirpath, name). + Note that the names in the lists are just names, with no path + components. To get a full path (which begins with top) to a file or + directory in dirpath, do os.path.join(dirpath, name). If optional arg 'topdown' is true or not specified, the triple for a directory is generated before the triples for any of its subdirectories @@ -317,13 +318,13 @@ def walk(top, topdown=True, onerror=None, followlinks=False): When topdown is true, the caller can modify the dirnames list in-place (e.g., via del or slice assignment), and walk will only recurse into the - subdirectories whose names remain in dirnames; this can be used to prune the - search, or to impose a specific order of visiting. Modifying dirnames when - topdown is false has no effect on the behavior of os.walk(), since the - directories in dirnames have already been generated by the time dirnames - itself is generated. No matter the value of topdown, the list of - subdirectories is retrieved before the tuples for the directory and its - subdirectories are generated. + subdirectories whose names remain in dirnames; this can be used to prune + the search, or to impose a specific order of visiting. Modifying + dirnames when topdown is false has no effect on the behavior of + os.walk(), since the directories in dirnames have already been generated + by the time dirnames itself is generated. No matter the value of + topdown, the list of subdirectories is retrieved before the tuples for + the directory and its subdirectories are generated. By default errors from the os.scandir() call are ignored. If optional arg 'onerror' is specified, it should be a function; it @@ -448,9 +449,9 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= The advantage of fwalk() over walk() is that it's safe against symlink races (when follow_symlinks is False). - If dir_fd is not None, it should be a file descriptor open to a directory, - and top should be relative; top will then be relative to that directory. - (dir_fd is always supported for fwalk.) + If dir_fd is not None, it should be a file descriptor open to + a directory, and top should be relative; top will then be relative to + that directory. (dir_fd is always supported for fwalk.) Caution: Since fwalk() yields file descriptors, those are only valid until the @@ -812,6 +813,7 @@ def reload_environ(): env_data.clear() env_data.update(data) + __all__.append("reload_environ") def getenv(key, default=None): """Get an environment variable, return None if it doesn't exist. diff --git a/Lib/pathlib/__init__.py b/Lib/pathlib/__init__.py index 12cf9f579cb32d3..0d763d1f0dcaa55 100644 --- a/Lib/pathlib/__init__.py +++ b/Lib/pathlib/__init__.py @@ -334,13 +334,8 @@ def _raw_path(self): return paths[0] elif paths: # Join path segments from the initializer. - path = self.parser.join(*paths) - # Cache the joined path. - paths.clear() - paths.append(path) - return path + return self.parser.join(*paths) else: - paths.append('') return '' @property diff --git a/Lib/pathlib/_local.py b/Lib/pathlib/_local.py new file mode 100644 index 000000000000000..58e137f2a92d3b9 --- /dev/null +++ b/Lib/pathlib/_local.py @@ -0,0 +1,12 @@ +""" +This module exists so that pathlib objects pickled under Python 3.13 can be +unpickled in 3.14+. +""" + +from pathlib import * + +__all__ = [ + "UnsupportedOperation", + "PurePath", "PurePosixPath", "PureWindowsPath", + "Path", "PosixPath", "WindowsPath", +] diff --git a/Lib/pdb.py b/Lib/pdb.py index f89d104fcddb9ad..7dcd0cb1b37e71d 100644 --- a/Lib/pdb.py +++ b/Lib/pdb.py @@ -75,6 +75,7 @@ import code import glob import json +import stat import token import types import atexit @@ -99,7 +100,6 @@ import _pyrepl.utils from contextlib import ExitStack, closing, contextmanager -from rlcompleter import Completer from types import CodeType from warnings import deprecated @@ -183,19 +183,37 @@ class _ExecutableTarget: class _ScriptTarget(_ExecutableTarget): def __init__(self, target): - self._target = os.path.realpath(target) + self._check(target) + self._target = self._safe_realpath(target) + + # If PYTHONSAFEPATH (-P) is not set, sys.path[0] is the directory + # of pdb, and we should replace it with the directory of the script + if not sys.flags.safe_path: + sys.path[0] = os.path.dirname(self._target) - if not os.path.exists(self._target): + @staticmethod + def _check(target): + """ + Check that target is plausibly a script. + """ + if not os.path.exists(target): print(f'Error: {target} does not exist') sys.exit(1) - if os.path.isdir(self._target): + if os.path.isdir(target): print(f'Error: {target} is a directory') sys.exit(1) - # If safe_path(-P) is not set, sys.path[0] is the directory - # of pdb, and we should replace it with the directory of the script - if not sys.flags.safe_path: - sys.path[0] = os.path.dirname(self._target) + @staticmethod + def _safe_realpath(path): + """ + Return the canonical path (realpath) if it is accessible from the userspace. + Otherwise (for example, if the path is a symlink to an anonymous pipe), + return the original path. + + See GH-142315. + """ + realpath = os.path.realpath(path) + return realpath if os.path.exists(realpath) else path def __repr__(self): return self._target @@ -363,6 +381,7 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, readline.set_completer_delims(' \t\n`@#%^&*()=+[{]}\\|;:\'",<>?') except ImportError: pass + self.allow_kbdint = False self.nosigint = nosigint # Consider these characters as part of the command so when the users type @@ -858,7 +877,7 @@ def _exec_in_closure(self, source, globals, locals): locals.update(pdb_eval["write_back"]) eval_result = pdb_eval["result"] if eval_result is not None: - print(repr(eval_result)) + self.message(repr(eval_result)) return True @@ -1091,6 +1110,31 @@ def set_convenience_variable(self, frame, name, value): # Generic completion functions. Individual complete_foo methods can be # assigned below to one of these functions. + @property + def rlcompleter(self): + """Return the `Completer` class from `rlcompleter`, while avoiding the + side effects of changing the completer from `import rlcompleter`. + + This is a compromise between GH-138860 and GH-139289. If GH-139289 is + fixed, then we don't need this and we can just `import rlcompleter` in + `Pdb.__init__`. + """ + if not hasattr(self, "_rlcompleter"): + try: + import readline + except ImportError: + # readline is not available, just get the Completer + from rlcompleter import Completer + self._rlcompleter = Completer + else: + # importing rlcompleter could have side effect of changing + # the current completer, we need to restore it + prev_completer = readline.get_completer() + from rlcompleter import Completer + self._rlcompleter = Completer + readline.set_completer(prev_completer) + return self._rlcompleter + def completenames(self, text, line, begidx, endidx): # Overwrite completenames() of cmd so for the command completion, # if no current command matches, check for expressions as well @@ -1185,10 +1229,9 @@ def completedefault(self, text, line, begidx, endidx): conv_vars = self.curframe.f_globals.get('__pdb_convenience_variables', {}) return [f"${name}" for name in conv_vars if name.startswith(text[1:])] - # Use rlcompleter to do the completion state = 0 matches = [] - completer = Completer(self.curframe.f_globals | self.curframe.f_locals) + completer = self.rlcompleter(self.curframe.f_globals | self.curframe.f_locals) while (match := completer.complete(text, state)) is not None: matches.append(match) state += 1 @@ -1203,8 +1246,8 @@ def _enable_rlcompleter(self, ns): return try: + completer = self.rlcompleter(ns) old_completer = readline.get_completer() - completer = Completer(ns) readline.set_completer(completer.complete) yield finally: @@ -1456,7 +1499,9 @@ def lineinfo(self, identifier): f = self.lookupmodule(parts[0]) if f: fname = f - item = parts[1] + item = parts[1] + else: + return failed answer = find_function(item, self.canonic(fname)) return answer or failed @@ -3419,6 +3464,8 @@ def attach(pid, commands=()): ) ) connect_script.close() + orig_mode = os.stat(connect_script.name).st_mode + os.chmod(connect_script.name, orig_mode | stat.S_IROTH | stat.S_IRGRP) sys.remote_exec(pid, connect_script.name) # TODO Add a timeout? Or don't bother since the user can ^C? @@ -3490,7 +3537,8 @@ def help(): _usage = """\ Debug the Python program given by pyfile. Alternatively, an executable module or package to debug can be specified using -the -m switch. +the -m switch. You can also attach to a running Python process +using the -p option with its PID. Initial commands are read from .pdbrc files in your home directory and in the current directory, if they exist. Commands supplied with @@ -3501,7 +3549,29 @@ def help(): "-c 'until X'".""" -def main(): +def exit_with_permission_help_text(): + """ + Prints a message pointing to platform-specific permission help text and exits the program. + This function is called when a PermissionError is encountered while trying + to attach to a process. + """ + print( + "Error: The specified process cannot be attached to due to insufficient permissions.\n" + "See the Python documentation for details on required privileges and troubleshooting:\n" + "https://docs.python.org/3.14/howto/remote_debugging.html#permission-requirements\n" + ) + sys.exit(1) + + +def parse_args(): + # We want pdb to be as intuitive as possible to users, so we need to do some + # heuristic parsing to deal with ambiguity. + # For example: + # "python -m pdb -m foo -p 1" should pass "-p 1" to "foo". + # "python -m pdb foo.py -m bar" should pass "-m bar" to "foo.py". + # "python -m pdb -m foo -m bar" should pass "-m bar" to "foo". + # This require some customized parsing logic to find the actual debug target. + import argparse parser = argparse.ArgumentParser( @@ -3512,55 +3582,61 @@ def main(): color=True, ) - # We need to maunally get the script from args, because the first positional - # arguments could be either the script we need to debug, or the argument - # to the -m module + # Get all the commands out first. For backwards compatibility, we allow + # -c commands to be after the target. parser.add_argument('-c', '--command', action='append', default=[], metavar='command', dest='commands', help='pdb commands to execute as if given in a .pdbrc file') - parser.add_argument('-m', metavar='module', dest='module') - parser.add_argument('-p', '--pid', type=int, help="attach to the specified PID", default=None) - if len(sys.argv) == 1: + opts, args = parser.parse_known_args() + + if not args: # If no arguments were given (python -m pdb), print the whole help message. # Without this check, argparse would only complain about missing required arguments. + # We need to add the arguments definitions here to get a proper help message. + parser.add_argument('-m', metavar='module', dest='module') + parser.add_argument('-p', '--pid', type=int, help="attach to the specified PID", default=None) parser.print_help() sys.exit(2) + elif args[0] == '-p' or args[0] == '--pid': + # Attach to a pid + parser.add_argument('-p', '--pid', type=int, help="attach to the specified PID", default=None) + opts, args = parser.parse_known_args() + if args: + # For --pid, any extra arguments are invalid. + parser.error(f"unrecognized arguments: {' '.join(args)}") + elif args[0] == '-m': + # Debug a module, we only need the first -m module argument. + # The rest is passed to the module itself. + parser.add_argument('-m', metavar='module', dest='module') + opt_module = parser.parse_args(args[:2]) + opts.module = opt_module.module + args = args[2:] + elif args[0] == '--': + args.pop(0) + if not args: + parser.error("missing script or module to run") + elif args[0].startswith('-'): + # Invalid argument before the script name. + invalid_args = list(itertools.takewhile(lambda a: a.startswith('-'), args)) + parser.error(f"unrecognized arguments: {' '.join(invalid_args)}") - opts, args = parser.parse_known_args() + # Otherwise it's debugging a script and we already parsed all -c commands. - if opts.pid: - # If attaching to a remote pid, unrecognized arguments are not allowed. - # This will raise an error if there are extra unrecognized arguments. - opts = parser.parse_args() - if opts.module: - parser.error("argument -m: not allowed with argument --pid") - attach(opts.pid, opts.commands) - return - elif opts.module: - # If a module is being debugged, we consider the arguments after "-m module" to - # be potential arguments to the module itself. We need to parse the arguments - # before "-m" to check if there is any invalid argument. - # e.g. "python -m pdb -m foo --spam" means passing "--spam" to "foo" - # "python -m pdb --spam -m foo" means passing "--spam" to "pdb" and is invalid - idx = sys.argv.index('-m') - args_to_pdb = sys.argv[1:idx] - # This will raise an error if there are invalid arguments - parser.parse_args(args_to_pdb) - else: - # If a script is being debugged, then pdb expects the script name as the first argument. - # Anything before the script is considered an argument to pdb itself, which would - # be invalid because it's not parsed by argparse. - invalid_args = list(itertools.takewhile(lambda a: a.startswith('-'), args)) - if invalid_args: - parser.error(f"unrecognized arguments: {' '.join(invalid_args)}") - sys.exit(2) + return opts, args - if opts.module: +def main(): + opts, args = parse_args() + + if getattr(opts, 'pid', None) is not None: + try: + attach(opts.pid, opts.commands) + except PermissionError as e: + exit_with_permission_help_text() + return + elif getattr(opts, 'module', None) is not None: file = opts.module target = _ModuleTarget(file) else: - if not args: - parser.error("no module or script to run") file = args.pop(0) if file.endswith('.pyz'): target = _ZipTarget(file) diff --git a/Lib/pickle.py b/Lib/pickle.py index beaefae0479d3ca..7b951858604bb93 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -904,17 +904,11 @@ def save_picklebuffer(self, obj): # Write data in-band # XXX The C implementation avoids a copy here buf = m.tobytes() - in_memo = id(buf) in self.memo if m.readonly: - if in_memo: - self._save_bytes_no_memo(buf) - else: - self.save_bytes(buf) + self._save_bytes_no_memo(buf) else: - if in_memo: - self._save_bytearray_no_memo(buf) - else: - self.save_bytearray(buf) + self._save_bytearray_no_memo(buf) + self.memoize(obj) else: # Write data out-of-band self.write(NEXT_BUFFER) diff --git a/Lib/pickletools.py b/Lib/pickletools.py index bcddfb722bd26de..254b6c7fcc9dd23 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -348,7 +348,7 @@ def read_stringnl(f, decode=True, stripquotes=True, *, encoding='latin-1'): for q in (b'"', b"'"): if data.startswith(q): if not data.endswith(q): - raise ValueError("strinq quote %r not found at both " + raise ValueError("string quote %r not found at both " "ends of %r" % (q, data)) data = data[1:-1] break diff --git a/Lib/platform.py b/Lib/platform.py index 55e211212d42097..b017b841311be3e 100644 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -173,6 +173,11 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384): """ if not executable: + if sys.platform == "emscripten": + # Emscripten's os.confstr reports that it is glibc, so special case + # it. + ver = ".".join(str(x) for x in sys._emscripten_info.emscripten_version) + return ("emscripten", ver) try: ver = os.confstr('CS_GNU_LIBC_VERSION') # parse 'glibc 2.28' as ('glibc', '2.28') @@ -194,6 +199,7 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384): | (GLIBC_([0-9.]+)) | (libc(_\w+)?\.so(?:\.(\d[0-9.]*))?) | (musl-([0-9.]+)) + | ((?:libc\.|ld-)musl(?:-\w+)?.so(?:\.(\d[0-9.]*))?) """, re.ASCII | re.VERBOSE) @@ -219,9 +225,10 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384): continue if not m: break - libcinit, glibc, glibcversion, so, threads, soversion, musl, muslversion = [ - s.decode('latin1') if s is not None else s - for s in m.groups()] + decoded_groups = [s.decode('latin1') if s is not None else s + for s in m.groups()] + (libcinit, glibc, glibcversion, so, threads, soversion, + musl, muslversion, musl_so, musl_sover) = decoded_groups if libcinit and not lib: lib = 'libc' elif glibc: @@ -231,7 +238,7 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384): elif V(glibcversion) > V(ver): ver = glibcversion elif so: - if lib != 'glibc': + if lib not in ('glibc', 'musl'): lib = 'libc' if soversion and (not ver or V(soversion) > V(ver)): ver = soversion @@ -241,6 +248,10 @@ def libc_ver(executable=None, lib='', version='', chunksize=16384): lib = 'musl' if not ver or V(muslversion) > V(ver): ver = muslversion + elif musl_so: + lib = 'musl' + if musl_sover and (not ver or V(musl_sover) > V(ver)): + ver = musl_sover pos = m.end() return lib, version if ver is None else ver @@ -1198,7 +1209,7 @@ def _sys_version(sys_version=None): # CPython cpython_sys_version_parser = re.compile( r'([\w.+]+)\s*' # "version" - r'(?:experimental free-threading build\s+)?' # "free-threading-build" + r'(?:free-threading build\s+)?' # "free-threading-build" r'\(#?([^,]+)' # "(#buildno" r'(?:,\s*([\w ]*)' # ", builddate" r'(?:,\s*([\w :]*))?)?\)\s*' # ", buildtime)" diff --git a/Lib/plistlib.py b/Lib/plistlib.py index 67e832db217319b..c3aee1e15c767a6 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -21,7 +21,7 @@ Generate Plist example: - import datetime + import datetime as dt import plistlib pl = dict( @@ -37,7 +37,7 @@ ), someData = b"", someMoreData = b"" * 10, - aDate = datetime.datetime.now() + aDate = dt.datetime.now() ) print(plistlib.dumps(pl).decode()) @@ -73,6 +73,9 @@ PlistFormat = enum.Enum('PlistFormat', 'FMT_XML FMT_BINARY', module=__name__) globals().update(PlistFormat.__members__) +# Data larger than this will be read in chunks, to prevent extreme +# overallocation. +_MIN_READ_BUF_SIZE = 1 << 20 class UID: def __init__(self, data): @@ -381,7 +384,7 @@ def write_bytes(self, data): self._indent_level -= 1 maxlinelength = max( 16, - 76 - len(self.indent.replace(b"\t", b" " * 8) * self._indent_level)) + 76 - len((self.indent * self._indent_level).expandtabs())) for line in _encode_base64(data, maxlinelength).split(b"\n"): if line: @@ -508,12 +511,24 @@ def _get_size(self, tokenL): return tokenL + def _read(self, size): + cursize = min(size, _MIN_READ_BUF_SIZE) + data = self._fp.read(cursize) + while True: + if len(data) != cursize: + raise InvalidFileException + if cursize == size: + return data + delta = min(cursize, size - cursize) + data += self._fp.read(delta) + cursize += delta + def _read_ints(self, n, size): - data = self._fp.read(size * n) + data = self._read(size * n) if size in _BINARY_FORMAT: return struct.unpack(f'>{n}{_BINARY_FORMAT[size]}', data) else: - if not size or len(data) != size * n: + if not size: raise InvalidFileException() return tuple(int.from_bytes(data[i: i + size], 'big') for i in range(0, size * n, size)) @@ -573,22 +588,16 @@ def _read_object(self, ref): elif tokenH == 0x40: # data s = self._get_size(tokenL) - result = self._fp.read(s) - if len(result) != s: - raise InvalidFileException() + result = self._read(s) elif tokenH == 0x50: # ascii string s = self._get_size(tokenL) - data = self._fp.read(s) - if len(data) != s: - raise InvalidFileException() + data = self._read(s) result = data.decode('ascii') elif tokenH == 0x60: # unicode string s = self._get_size(tokenL) * 2 - data = self._fp.read(s) - if len(data) != s: - raise InvalidFileException() + data = self._read(s) result = data.decode('utf-16be') elif tokenH == 0x80: # UID diff --git a/Lib/posixpath.py b/Lib/posixpath.py index db72ded88260565..ad86cc06c017a05 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -36,7 +36,7 @@ "samefile","sameopenfile","samestat", "curdir","pardir","sep","pathsep","defpath","altsep","extsep", "devnull","realpath","supports_unicode_filenames","relpath", - "commonpath", "isjunction","isdevdrive"] + "commonpath", "isjunction","isdevdrive","ALLOW_MISSING"] def _get_sep(path): @@ -284,42 +284,41 @@ def expanduser(path): # This expands the forms $variable and ${variable} only. # Non-existent variables are left unchanged. -_varprog = None -_varprogb = None +_varpattern = r'\$(\w+|\{[^}]*\}?)' +_varsub = None +_varsubb = None def expandvars(path): """Expand shell variables of form $var and ${var}. Unknown variables are left unchanged.""" path = os.fspath(path) - global _varprog, _varprogb + global _varsub, _varsubb if isinstance(path, bytes): if b'$' not in path: return path - if not _varprogb: + if not _varsubb: import re - _varprogb = re.compile(br'\$(\w+|\{[^}]*\})', re.ASCII) - search = _varprogb.search + _varsubb = re.compile(_varpattern.encode(), re.ASCII).sub + sub = _varsubb start = b'{' end = b'}' environ = getattr(os, 'environb', None) else: if '$' not in path: return path - if not _varprog: + if not _varsub: import re - _varprog = re.compile(r'\$(\w+|\{[^}]*\})', re.ASCII) - search = _varprog.search + _varsub = re.compile(_varpattern, re.ASCII).sub + sub = _varsub start = '{' end = '}' environ = os.environ - i = 0 - while True: - m = search(path, i) - if not m: - break - i, j = m.span(0) - name = m.group(1) - if name.startswith(start) and name.endswith(end): + + def repl(m): + name = m[1] + if name.startswith(start): + if not name.endswith(end): + return m[0] name = name[1:-1] try: if environ is None: @@ -327,13 +326,11 @@ def expandvars(path): else: value = environ[name] except KeyError: - i = j + return m[0] else: - tail = path[j:] - path = path[:i] + value - i = len(path) - path += tail - return path + return value + + return sub(repl, path) # Normalize a path, e.g. A//B, A/./B and A/foo/../B all become A/B. @@ -402,10 +399,18 @@ def realpath(filename, *, strict=False): curdir = '.' pardir = '..' getcwd = os.getcwd - return _realpath(filename, strict, sep, curdir, pardir, getcwd) + if strict is ALLOW_MISSING: + ignored_error = FileNotFoundError + strict = True + elif strict: + ignored_error = () + else: + ignored_error = OSError + + lstat = os.lstat + readlink = os.readlink + maxlinks = None -def _realpath(filename, strict=False, sep=sep, curdir=curdir, pardir=pardir, - getcwd=os.getcwd, lstat=os.lstat, readlink=os.readlink, maxlinks=None): # The stack of unresolved path parts. When popped, a special value of None # indicates that a symlink target has been resolved, and that the original # symlink path can be retrieved by popping again. The [::-1] slice is a @@ -477,27 +482,28 @@ def _realpath(filename, strict=False, sep=sep, curdir=curdir, pardir=pardir, path = newpath continue target = readlink(newpath) - except OSError: - if strict: - raise - path = newpath + except ignored_error: + pass + else: + # Resolve the symbolic link + if target.startswith(sep): + # Symlink target is absolute; reset resolved path. + path = sep + if maxlinks is None: + # Mark this symlink as seen but not fully resolved. + seen[newpath] = None + # Push the symlink path onto the stack, and signal its specialness + # by also pushing None. When these entries are popped, we'll + # record the fully-resolved symlink target in the 'seen' mapping. + rest.append(newpath) + rest.append(None) + # Push the unresolved symlink target parts onto the stack. + target_parts = target.split(sep)[::-1] + rest.extend(target_parts) + part_count += len(target_parts) continue - # Resolve the symbolic link - if target.startswith(sep): - # Symlink target is absolute; reset resolved path. - path = sep - if maxlinks is None: - # Mark this symlink as seen but not fully resolved. - seen[newpath] = None - # Push the symlink path onto the stack, and signal its specialness - # by also pushing None. When these entries are popped, we'll - # record the fully-resolved symlink target in the 'seen' mapping. - rest.append(newpath) - rest.append(None) - # Push the unresolved symlink target parts onto the stack. - target_parts = target.split(sep)[::-1] - rest.extend(target_parts) - part_count += len(target_parts) + # An error occurred and was ignored. + path = newpath return path diff --git a/Lib/pydoc.py b/Lib/pydoc.py index def76d076a29892..091f8af01b2b422 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -579,10 +579,20 @@ def getdocloc(self, object, basedir=sysconfig.get_path('stdlib')): (file.startswith(basedir) and not file.startswith(os.path.join(basedir, 'site-packages')))) and object.__name__ not in ('xml.etree', 'test.test_pydoc.pydoc_mod')): - if docloc.startswith(("http://", "https://")): - docloc = "{}/{}.html".format(docloc.rstrip("/"), object.__name__.lower()) + + try: + from pydoc_data import module_docs + except ImportError: + module_docs = None + + if module_docs and object.__name__ in module_docs.module_docs: + doc_name = module_docs.module_docs[object.__name__] + if docloc.startswith(("http://", "https://")): + docloc = "{}/{}".format(docloc.rstrip("/"), doc_name) + else: + docloc = os.path.join(docloc, doc_name) else: - docloc = os.path.join(docloc, object.__name__.lower() + ".html") + docloc = None else: docloc = None return docloc @@ -884,6 +894,7 @@ def docmodule(self, object, name=None, mod=None, *ignored): for key, value in inspect.getmembers(object, inspect.isroutine): # if __all__ exists, believe it. Otherwise use a heuristic. if (all is not None + or inspect.isbuiltin(value) or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): funcs.append((key, value)) @@ -1328,6 +1339,7 @@ def docmodule(self, object, name=None, mod=None, *ignored): for key, value in inspect.getmembers(object, inspect.isroutine): # if __all__ exists, believe it. Otherwise use a heuristic. if (all is not None + or inspect.isbuiltin(value) or (inspect.getmodule(value) or object) is object): if visiblename(key, all, object): funcs.append((key, value)) @@ -1864,6 +1876,7 @@ class Helper: 'async': ('async', ''), 'await': ('await', ''), 'break': ('break', 'while for'), + 'case': 'match', 'class': ('class', 'CLASSES SPECIALMETHODS'), 'continue': ('continue', 'while for'), 'def': ('function', ''), @@ -1875,11 +1888,12 @@ class Helper: 'for': ('for', 'break continue while'), 'from': 'import', 'global': ('global', 'nonlocal NAMESPACES'), - 'if': ('if', 'TRUTHVALUE'), + 'if': ('if', 'TRUTHVALUE match'), 'import': ('import', 'MODULES'), 'in': ('in', 'SEQUENCEMETHODS'), 'is': 'COMPARISON', 'lambda': ('lambda', 'FUNCTIONS'), + 'match': ('match', 'if'), 'nonlocal': ('nonlocal', 'global NAMESPACES'), 'not': 'BOOLEAN', 'or': 'BOOLEAN', @@ -2060,10 +2074,11 @@ def interact(self): while True: try: request = self.getline('help> ') - if not request: break except (KeyboardInterrupt, EOFError): break request = request.strip() + if not request: + continue # back to the prompt # Make sure significant trailing quoting marks of literals don't # get deleted while cleaning input @@ -2110,7 +2125,7 @@ def intro(self): self.output.write(_introdoc()) def list(self, items, columns=4, width=80): - items = list(sorted(items)) + items = sorted(items) colw = width // columns rows = (len(items) + columns - 1) // columns for row in range(rows): @@ -2142,7 +2157,7 @@ def listtopics(self): Here is a list of available topics. Enter any topic name to get more help. ''') - self.list(self.topics.keys()) + self.list(self.topics.keys(), columns=3) def showtopic(self, topic, more_xrefs=''): try: diff --git a/Lib/pydoc_data/module_docs.py b/Lib/pydoc_data/module_docs.py new file mode 100644 index 000000000000000..69b9046f49f25ab --- /dev/null +++ b/Lib/pydoc_data/module_docs.py @@ -0,0 +1,320 @@ +# Autogenerated by Sphinx on Wed Jun 10 13:03:46 2026 +# as part of the release process. + +module_docs = { + '__future__': '__future__#module-__future__', + '__main__': '__main__#module-__main__', + '_thread': '_thread#module-_thread', + '_tkinter': 'tkinter#module-_tkinter', + 'abc': 'abc#module-abc', + 'aifc': 'aifc#module-aifc', + 'annotationlib': 'annotationlib#module-annotationlib', + 'argparse': 'argparse#module-argparse', + 'array': 'array#module-array', + 'ast': 'ast#module-ast', + 'asynchat': 'asynchat#module-asynchat', + 'asyncio': 'asyncio#module-asyncio', + 'asyncore': 'asyncore#module-asyncore', + 'atexit': 'atexit#module-atexit', + 'audioop': 'audioop#module-audioop', + 'base64': 'base64#module-base64', + 'bdb': 'bdb#module-bdb', + 'binascii': 'binascii#module-binascii', + 'bisect': 'bisect#module-bisect', + 'builtins': 'builtins#module-builtins', + 'bz2': 'bz2#module-bz2', + 'cProfile': 'profile#module-cProfile', + 'calendar': 'calendar#module-calendar', + 'cgi': 'cgi#module-cgi', + 'cgitb': 'cgitb#module-cgitb', + 'chunk': 'chunk#module-chunk', + 'cmath': 'cmath#module-cmath', + 'cmd': 'cmd#module-cmd', + 'code': 'code#module-code', + 'codecs': 'codecs#module-codecs', + 'codeop': 'codeop#module-codeop', + 'collections': 'collections#module-collections', + 'collections.abc': 'collections.abc#module-collections.abc', + 'colorsys': 'colorsys#module-colorsys', + 'compileall': 'compileall#module-compileall', + 'compression': 'compression#module-compression', + 'compression.zstd': 'compression.zstd#module-compression.zstd', + 'concurrent.futures': 'concurrent.futures#module-concurrent.futures', + 'concurrent.interpreters': 'concurrent.interpreters#module-concurrent.interpreters', + 'configparser': 'configparser#module-configparser', + 'contextlib': 'contextlib#module-contextlib', + 'contextvars': 'contextvars#module-contextvars', + 'copy': 'copy#module-copy', + 'copyreg': 'copyreg#module-copyreg', + 'crypt': 'crypt#module-crypt', + 'csv': 'csv#module-csv', + 'ctypes': 'ctypes#module-ctypes', + 'curses': 'curses#module-curses', + 'curses.ascii': 'curses.ascii#module-curses.ascii', + 'curses.panel': 'curses.panel#module-curses.panel', + 'curses.textpad': 'curses#module-curses.textpad', + 'dataclasses': 'dataclasses#module-dataclasses', + 'datetime': 'datetime#module-datetime', + 'dbm': 'dbm#module-dbm', + 'dbm.dumb': 'dbm#module-dbm.dumb', + 'dbm.gnu': 'dbm#module-dbm.gnu', + 'dbm.ndbm': 'dbm#module-dbm.ndbm', + 'dbm.sqlite3': 'dbm#module-dbm.sqlite3', + 'decimal': 'decimal#module-decimal', + 'difflib': 'difflib#module-difflib', + 'dis': 'dis#module-dis', + 'distutils': 'distutils#module-distutils', + 'doctest': 'doctest#module-doctest', + 'email': 'email#module-email', + 'email.charset': 'email.charset#module-email.charset', + 'email.contentmanager': 'email.contentmanager#module-email.contentmanager', + 'email.encoders': 'email.encoders#module-email.encoders', + 'email.errors': 'email.errors#module-email.errors', + 'email.generator': 'email.generator#module-email.generator', + 'email.header': 'email.header#module-email.header', + 'email.headerregistry': 'email.headerregistry#module-email.headerregistry', + 'email.iterators': 'email.iterators#module-email.iterators', + 'email.message': 'email.message#module-email.message', + 'email.mime': 'email.mime#module-email.mime', + 'email.mime.application': 'email.mime#module-email.mime.application', + 'email.mime.audio': 'email.mime#module-email.mime.audio', + 'email.mime.base': 'email.mime#module-email.mime.base', + 'email.mime.image': 'email.mime#module-email.mime.image', + 'email.mime.message': 'email.mime#module-email.mime.message', + 'email.mime.multipart': 'email.mime#module-email.mime.multipart', + 'email.mime.nonmultipart': 'email.mime#module-email.mime.nonmultipart', + 'email.mime.text': 'email.mime#module-email.mime.text', + 'email.parser': 'email.parser#module-email.parser', + 'email.policy': 'email.policy#module-email.policy', + 'email.utils': 'email.utils#module-email.utils', + 'encodings': 'codecs#module-encodings', + 'encodings.idna': 'codecs#module-encodings.idna', + 'encodings.mbcs': 'codecs#module-encodings.mbcs', + 'encodings.utf_8_sig': 'codecs#module-encodings.utf_8_sig', + 'ensurepip': 'ensurepip#module-ensurepip', + 'enum': 'enum#module-enum', + 'errno': 'errno#module-errno', + 'faulthandler': 'faulthandler#module-faulthandler', + 'fcntl': 'fcntl#module-fcntl', + 'filecmp': 'filecmp#module-filecmp', + 'fileinput': 'fileinput#module-fileinput', + 'fnmatch': 'fnmatch#module-fnmatch', + 'fractions': 'fractions#module-fractions', + 'ftplib': 'ftplib#module-ftplib', + 'functools': 'functools#module-functools', + 'gc': 'gc#module-gc', + 'getopt': 'getopt#module-getopt', + 'getpass': 'getpass#module-getpass', + 'gettext': 'gettext#module-gettext', + 'glob': 'glob#module-glob', + 'graphlib': 'graphlib#module-graphlib', + 'grp': 'grp#module-grp', + 'gzip': 'gzip#module-gzip', + 'hashlib': 'hashlib#module-hashlib', + 'heapq': 'heapq#module-heapq', + 'hmac': 'hmac#module-hmac', + 'html': 'html#module-html', + 'html.entities': 'html.entities#module-html.entities', + 'html.parser': 'html.parser#module-html.parser', + 'http': 'http#module-http', + 'http.client': 'http.client#module-http.client', + 'http.cookiejar': 'http.cookiejar#module-http.cookiejar', + 'http.cookies': 'http.cookies#module-http.cookies', + 'http.server': 'http.server#module-http.server', + 'idlelib': 'idle#module-idlelib', + 'imaplib': 'imaplib#module-imaplib', + 'imghdr': 'imghdr#module-imghdr', + 'imp': 'imp#module-imp', + 'importlib': 'importlib#module-importlib', + 'importlib.abc': 'importlib#module-importlib.abc', + 'importlib.machinery': 'importlib#module-importlib.machinery', + 'importlib.metadata': 'importlib.metadata#module-importlib.metadata', + 'importlib.resources': 'importlib.resources#module-importlib.resources', + 'importlib.resources.abc': 'importlib.resources.abc#module-importlib.resources.abc', + 'importlib.util': 'importlib#module-importlib.util', + 'inspect': 'inspect#module-inspect', + 'io': 'io#module-io', + 'ipaddress': 'ipaddress#module-ipaddress', + 'itertools': 'itertools#module-itertools', + 'json': 'json#module-json', + 'json.tool': 'json#module-json.tool', + 'keyword': 'keyword#module-keyword', + 'linecache': 'linecache#module-linecache', + 'locale': 'locale#module-locale', + 'logging': 'logging#module-logging', + 'logging.config': 'logging.config#module-logging.config', + 'logging.handlers': 'logging.handlers#module-logging.handlers', + 'lzma': 'lzma#module-lzma', + 'mailbox': 'mailbox#module-mailbox', + 'mailcap': 'mailcap#module-mailcap', + 'marshal': 'marshal#module-marshal', + 'math': 'math#module-math', + 'mimetypes': 'mimetypes#module-mimetypes', + 'mmap': 'mmap#module-mmap', + 'modulefinder': 'modulefinder#module-modulefinder', + 'msilib': 'msilib#module-msilib', + 'msvcrt': 'msvcrt#module-msvcrt', + 'multiprocessing': 'multiprocessing#module-multiprocessing', + 'multiprocessing.connection': 'multiprocessing#module-multiprocessing.connection', + 'multiprocessing.dummy': 'multiprocessing#module-multiprocessing.dummy', + 'multiprocessing.managers': 'multiprocessing#module-multiprocessing.managers', + 'multiprocessing.pool': 'multiprocessing#module-multiprocessing.pool', + 'multiprocessing.shared_memory': 'multiprocessing.shared_memory#module-multiprocessing.shared_memory', + 'multiprocessing.sharedctypes': 'multiprocessing#module-multiprocessing.sharedctypes', + 'netrc': 'netrc#module-netrc', + 'nis': 'nis#module-nis', + 'nntplib': 'nntplib#module-nntplib', + 'numbers': 'numbers#module-numbers', + 'operator': 'operator#module-operator', + 'optparse': 'optparse#module-optparse', + 'os': 'os#module-os', + 'os.path': 'os.path#module-os.path', + 'ossaudiodev': 'ossaudiodev#module-ossaudiodev', + 'pathlib': 'pathlib#module-pathlib', + 'pathlib.types': 'pathlib#module-pathlib.types', + 'pdb': 'pdb#module-pdb', + 'pickle': 'pickle#module-pickle', + 'pickletools': 'pickletools#module-pickletools', + 'pipes': 'pipes#module-pipes', + 'pkgutil': 'pkgutil#module-pkgutil', + 'platform': 'platform#module-platform', + 'plistlib': 'plistlib#module-plistlib', + 'poplib': 'poplib#module-poplib', + 'posix': 'posix#module-posix', + 'pprint': 'pprint#module-pprint', + 'profile': 'profile#module-profile', + 'pstats': 'profile#module-pstats', + 'pty': 'pty#module-pty', + 'pwd': 'pwd#module-pwd', + 'py_compile': 'py_compile#module-py_compile', + 'pyclbr': 'pyclbr#module-pyclbr', + 'pydoc': 'pydoc#module-pydoc', + 'queue': 'queue#module-queue', + 'quopri': 'quopri#module-quopri', + 'random': 'random#module-random', + 're': 're#module-re', + 'readline': 'readline#module-readline', + 'reprlib': 'reprlib#module-reprlib', + 'resource': 'resource#module-resource', + 'rlcompleter': 'rlcompleter#module-rlcompleter', + 'runpy': 'runpy#module-runpy', + 'sched': 'sched#module-sched', + 'secrets': 'secrets#module-secrets', + 'select': 'select#module-select', + 'selectors': 'selectors#module-selectors', + 'shelve': 'shelve#module-shelve', + 'shlex': 'shlex#module-shlex', + 'shutil': 'shutil#module-shutil', + 'signal': 'signal#module-signal', + 'site': 'site#module-site', + 'sitecustomize': 'site#module-sitecustomize', + 'smtpd': 'smtpd#module-smtpd', + 'smtplib': 'smtplib#module-smtplib', + 'sndhdr': 'sndhdr#module-sndhdr', + 'socket': 'socket#module-socket', + 'socketserver': 'socketserver#module-socketserver', + 'spwd': 'spwd#module-spwd', + 'sqlite3': 'sqlite3#module-sqlite3', + 'ssl': 'ssl#module-ssl', + 'stat': 'stat#module-stat', + 'statistics': 'statistics#module-statistics', + 'string': 'string#module-string', + 'string.templatelib': 'string.templatelib#module-string.templatelib', + 'stringprep': 'stringprep#module-stringprep', + 'struct': 'struct#module-struct', + 'subprocess': 'subprocess#module-subprocess', + 'sunau': 'sunau#module-sunau', + 'symtable': 'symtable#module-symtable', + 'sys': 'sys#module-sys', + 'sys.monitoring': 'sys.monitoring#module-sys.monitoring', + 'sysconfig': 'sysconfig#module-sysconfig', + 'syslog': 'syslog#module-syslog', + 'tabnanny': 'tabnanny#module-tabnanny', + 'tarfile': 'tarfile#module-tarfile', + 'telnetlib': 'telnetlib#module-telnetlib', + 'tempfile': 'tempfile#module-tempfile', + 'termios': 'termios#module-termios', + 'test': 'test#module-test', + 'test.regrtest': 'test#module-test.regrtest', + 'test.support': 'test#module-test.support', + 'test.support.bytecode_helper': 'test#module-test.support.bytecode_helper', + 'test.support.import_helper': 'test#module-test.support.import_helper', + 'test.support.os_helper': 'test#module-test.support.os_helper', + 'test.support.script_helper': 'test#module-test.support.script_helper', + 'test.support.socket_helper': 'test#module-test.support.socket_helper', + 'test.support.threading_helper': 'test#module-test.support.threading_helper', + 'test.support.warnings_helper': 'test#module-test.support.warnings_helper', + 'textwrap': 'textwrap#module-textwrap', + 'threading': 'threading#module-threading', + 'time': 'time#module-time', + 'timeit': 'timeit#module-timeit', + 'tkinter': 'tkinter#module-tkinter', + 'tkinter.colorchooser': 'tkinter.colorchooser#module-tkinter.colorchooser', + 'tkinter.commondialog': 'dialog#module-tkinter.commondialog', + 'tkinter.dnd': 'tkinter.dnd#module-tkinter.dnd', + 'tkinter.filedialog': 'dialog#module-tkinter.filedialog', + 'tkinter.font': 'tkinter.font#module-tkinter.font', + 'tkinter.messagebox': 'tkinter.messagebox#module-tkinter.messagebox', + 'tkinter.scrolledtext': 'tkinter.scrolledtext#module-tkinter.scrolledtext', + 'tkinter.simpledialog': 'dialog#module-tkinter.simpledialog', + 'tkinter.ttk': 'tkinter.ttk#module-tkinter.ttk', + 'token': 'token#module-token', + 'tokenize': 'tokenize#module-tokenize', + 'tomllib': 'tomllib#module-tomllib', + 'trace': 'trace#module-trace', + 'traceback': 'traceback#module-traceback', + 'tracemalloc': 'tracemalloc#module-tracemalloc', + 'tty': 'tty#module-tty', + 'turtle': 'turtle#module-turtle', + 'turtledemo': 'turtle#module-turtledemo', + 'types': 'types#module-types', + 'typing': 'typing#module-typing', + 'unicodedata': 'unicodedata#module-unicodedata', + 'unittest': 'unittest#module-unittest', + 'unittest.mock': 'unittest.mock#module-unittest.mock', + 'urllib': 'urllib#module-urllib', + 'urllib.error': 'urllib.error#module-urllib.error', + 'urllib.parse': 'urllib.parse#module-urllib.parse', + 'urllib.request': 'urllib.request#module-urllib.request', + 'urllib.response': 'urllib.request#module-urllib.response', + 'urllib.robotparser': 'urllib.robotparser#module-urllib.robotparser', + 'usercustomize': 'site#module-usercustomize', + 'uu': 'uu#module-uu', + 'uuid': 'uuid#module-uuid', + 'venv': 'venv#module-venv', + 'warnings': 'warnings#module-warnings', + 'wave': 'wave#module-wave', + 'weakref': 'weakref#module-weakref', + 'webbrowser': 'webbrowser#module-webbrowser', + 'winreg': 'winreg#module-winreg', + 'winsound': 'winsound#module-winsound', + 'wsgiref': 'wsgiref#module-wsgiref', + 'wsgiref.handlers': 'wsgiref#module-wsgiref.handlers', + 'wsgiref.headers': 'wsgiref#module-wsgiref.headers', + 'wsgiref.simple_server': 'wsgiref#module-wsgiref.simple_server', + 'wsgiref.types': 'wsgiref#module-wsgiref.types', + 'wsgiref.util': 'wsgiref#module-wsgiref.util', + 'wsgiref.validate': 'wsgiref#module-wsgiref.validate', + 'xdrlib': 'xdrlib#module-xdrlib', + 'xml': 'xml#module-xml', + 'xml.dom': 'xml.dom#module-xml.dom', + 'xml.dom.minidom': 'xml.dom.minidom#module-xml.dom.minidom', + 'xml.dom.pulldom': 'xml.dom.pulldom#module-xml.dom.pulldom', + 'xml.etree.ElementInclude': 'xml.etree.elementtree#module-xml.etree.ElementInclude', + 'xml.etree.ElementTree': 'xml.etree.elementtree#module-xml.etree.ElementTree', + 'xml.parsers.expat': 'pyexpat#module-xml.parsers.expat', + 'xml.parsers.expat.errors': 'pyexpat#module-xml.parsers.expat.errors', + 'xml.parsers.expat.model': 'pyexpat#module-xml.parsers.expat.model', + 'xml.sax': 'xml.sax#module-xml.sax', + 'xml.sax.handler': 'xml.sax.handler#module-xml.sax.handler', + 'xml.sax.saxutils': 'xml.sax.utils#module-xml.sax.saxutils', + 'xml.sax.xmlreader': 'xml.sax.reader#module-xml.sax.xmlreader', + 'xmlrpc': 'xmlrpc#module-xmlrpc', + 'xmlrpc.client': 'xmlrpc.client#module-xmlrpc.client', + 'xmlrpc.server': 'xmlrpc.server#module-xmlrpc.server', + 'zipapp': 'zipapp#module-zipapp', + 'zipfile': 'zipfile#module-zipfile', + 'zipimport': 'zipimport#module-zipimport', + 'zlib': 'zlib#module-zlib', + 'zoneinfo': 'zoneinfo#module-zoneinfo', +} diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 5f7e14a79d3356d..05d283c9540f30c 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Tue May 6 18:33:44 2025 +# Autogenerated by Sphinx on Wed Jun 10 13:03:46 2026 # as part of the release process. topics = { @@ -46,11 +46,10 @@ | "[" [target_list] "]" | attributeref | subscription - | slicing | "*" target -(See section Primaries for the syntax definitions for *attributeref*, -*subscription*, and *slicing*.) +(See section Primaries for the syntax definitions for *attributeref* +and *subscription*.) An assignment statement evaluates the expression list (remember that this can be a single expression or a comma-separated list, the latter @@ -59,12 +58,11 @@ Assignment is defined recursively depending on the form of the target (list). When a target is part of a mutable object (an attribute -reference, subscription or slicing), the mutable object must -ultimately perform the assignment and decide about its validity, and -may raise an exception if the assignment is unacceptable. The rules -observed by various types and the exceptions raised are given with the -definition of the object types (see section The standard type -hierarchy). +reference or subscription), the mutable object must ultimately perform +the assignment and decide about its validity, and may raise an +exception if the assignment is unacceptable. The rules observed by +various types and the exceptions raised are given with the definition +of the object types (see section The standard type hierarchy). Assignment of an object to a target list, optionally enclosed in parentheses or square brackets, is recursively defined as follows. @@ -130,9 +128,13 @@ class Cls: attributes, such as properties created with "property()". * If the target is a subscription: The primary expression in the - reference is evaluated. It should yield either a mutable sequence - object (such as a list) or a mapping object (such as a dictionary). - Next, the subscript expression is evaluated. + reference is evaluated. Next, the subscript expression is evaluated. + Then, the primary’s "__setitem__()" method is called with two + arguments: the subscript and the assigned object. + + Typically, "__setitem__()" is defined on mutable sequence objects + (such as lists) and mapping objects (such as dictionaries), and + behaves as follows. If the primary is a mutable sequence object (such as a list), the subscript must yield an integer. If it is negative, the sequence’s @@ -149,27 +151,17 @@ class Cls: existing key/value pair with the same key value, or insert a new key/value pair (if no key with the same value existed). - For user-defined objects, the "__setitem__()" method is called with - appropriate arguments. - -* If the target is a slicing: The primary expression in the reference - is evaluated. It should yield a mutable sequence object (such as a - list). The assigned object should be a sequence object of the same - type. Next, the lower and upper bound expressions are evaluated, - insofar they are present; defaults are zero and the sequence’s - length. The bounds should evaluate to integers. If either bound is - negative, the sequence’s length is added to it. The resulting - bounds are clipped to lie between zero and the sequence’s length, - inclusive. Finally, the sequence object is asked to replace the - slice with the items of the assigned sequence. The length of the - slice may be different from the length of the assigned sequence, - thus changing the length of the target sequence, if the target - sequence allows it. - -**CPython implementation detail:** In the current implementation, the -syntax for targets is taken to be the same as for expressions, and -invalid syntax is rejected during the code generation phase, causing -less detailed error messages. + If the target is a slicing: The primary expression should evaluate + to a mutable sequence object (such as a list). The assigned object + should be *iterable*. The slicing’s lower and upper bounds should be + integers; if they are "None" (or not present), the defaults are zero + and the sequence’s length. If either bound is negative, the + sequence’s length is added to it. The resulting bounds are clipped + to lie between zero and the sequence’s length, inclusive. Finally, + the sequence object is asked to replace the slice with the items of + the assigned sequence. The length of the slice may be different + from the length of the assigned sequence, thus changing the length + of the target sequence, if the target sequence allows it. Although the definition of assignment implies that overlaps between the left-hand side and the right-hand side are ‘simultaneous’ (for @@ -196,7 +188,7 @@ class Cls: binary operation and an assignment statement: augmented_assignment_stmt: augtarget augop (expression_list | yield_expression) - augtarget: identifier | attributeref | subscription | slicing + augtarget: identifier | attributeref | subscription augop: "+=" | "-=" | "*=" | "@=" | "/=" | "//=" | "%=" | "**=" | ">>=" | "<<=" | "&=" | "^=" | "|=" @@ -369,13 +361,12 @@ async def func(param1, param2): Is semantically equivalent to: - iter = (ITER) - iter = type(iter).__aiter__(iter) + iter = (ITER).__aiter__() running = True while running: try: - TARGET = await type(iter).__anext__(iter) + TARGET = await iter.__anext__() except StopAsyncIteration: running = False else: @@ -383,7 +374,8 @@ async def func(param1, param2): else: SUITE2 -See also "__aiter__()" and "__anext__()" for details. +except that implicit special method lookup is used for "__aiter__()" +and "__anext__()". It is a "SyntaxError" to use an "async for" statement outside the body of a coroutine function. @@ -405,9 +397,9 @@ async def func(param1, param2): is semantically equivalent to: manager = (EXPRESSION) - aenter = type(manager).__aenter__ - aexit = type(manager).__aexit__ - value = await aenter(manager) + aenter = manager.__aenter__ + aexit = manager.__aexit__ + value = await aenter() hit_except = False try: @@ -415,13 +407,14 @@ async def func(param1, param2): SUITE except: hit_except = True - if not await aexit(manager, *sys.exc_info()): + if not await aexit(*sys.exc_info()): raise finally: if not hit_except: - await aexit(manager, None, None, None) + await aexit(None, None, None) -See also "__aenter__()" and "__aexit__()" for details. +except that implicit special method lookup is used for "__aenter__()" +and "__aexit__()". It is a "SyntaxError" to use an "async with" statement outside the body of a coroutine function. @@ -435,9 +428,9 @@ async def func(param1, param2): 'atom-identifiers': r'''Identifiers (Names) ******************* -An identifier occurring as an atom is a name. See section Identifiers -and keywords for lexical definition and section Naming and binding for -documentation of naming and binding. +An identifier occurring as an atom is a name. See section Names +(identifiers and keywords) for lexical definition and section Naming +and binding for documentation of naming and binding. When the name is bound to an object, evaluation of the atom yields that object. When a name is not bound, an attempt to evaluate it @@ -489,22 +482,122 @@ async def func(param1, param2): 'atom-literals': r'''Literals ******** -Python supports string and bytes literals and various numeric -literals: +A *literal* is a textual representation of a value. Python supports +numeric, string and bytes literals. Format strings and template +strings are treated as string literals. + +Numeric literals consist of a single "NUMBER" token, which names an +integer, floating-point number, or an imaginary number. See the +Numeric literals section in Lexical analysis documentation for +details. + +String and bytes literals may consist of several tokens. See section +String literal concatenation for details. + +Note that negative and complex numbers, like "-3" or "3+4.2j", are +syntactically not literals, but unary or binary arithmetic operations +involving the "-" or "+" operator. - literal: stringliteral | bytesliteral - | integer | floatnumber | imagnumber +Evaluation of a literal yields an object of the given type ("int", +"float", "complex", "str", "bytes", or "Template") with the given +value. The value may be approximated in the case of floating-point and +imaginary literals. -Evaluation of a literal yields an object of the given type (string, -bytes, integer, floating-point number, complex number) with the given -value. The value may be approximated in the case of floating-point -and imaginary (complex) literals. See section Literals for details. +The formal grammar for literals is: + + literal: strings | NUMBER + + +Literals and object identity +============================ All literals correspond to immutable data types, and hence the object’s identity is less important than its value. Multiple evaluations of literals with the same value (either the same occurrence in the program text or a different occurrence) may obtain the same object or a different object with the same value. + +CPython implementation detail: For example, in CPython, *small* +integers with the same value evaluate to the same object: + + >>> x = 7 + >>> y = 7 + >>> x is y + True + +However, large integers evaluate to different objects: + + >>> x = 123456789 + >>> y = 123456789 + >>> x is y + False + +This behavior may change in future versions of CPython. In particular, +the boundary between “small” and “large” integers has already changed +in the past.CPython will emit a "SyntaxWarning" when you compare +literals using "is": + + >>> x = 7 + >>> x is 7 + :1: SyntaxWarning: "is" with 'int' literal. Did you mean "=="? + True + +See When can I rely on identity tests with the is operator? for more +information. + +Template strings are immutable but may reference mutable objects as +"Interpolation" values. For the purposes of this section, two +t-strings have the “same value” if both their structure and the +*identity* of the values match. + +**CPython implementation detail:** Currently, each evaluation of a +template string results in a different object. + + +String literal concatenation +============================ + +Multiple adjacent string or bytes literals, possibly using different +quoting conventions, are allowed, and their meaning is the same as +their concatenation: + + >>> "hello" 'world' + "helloworld" + +This feature is defined at the syntactical level, so it only works +with literals. To concatenate string expressions at run time, the ‘+’ +operator may be used: + + >>> greeting = "Hello" + >>> space = " " + >>> name = "Blaise" + >>> print(greeting + space + name) # not: print(greeting space name) + Hello Blaise + +Literal concatenation can freely mix raw strings, triple-quoted +strings, and formatted string literals. For example: + + >>> "Hello" r', ' f"{name}!" + "Hello, Blaise!" + +This feature can be used to reduce the number of backslashes needed, +to split long strings conveniently across long lines, or even to add +comments to parts of strings. For example: + + re.compile("[A-Za-z_]" # letter or underscore + "[A-Za-z0-9_]*" # letter, digit or underscore + ) + +However, bytes literals may only be combined with other byte literals; +not with string literals of any kind. Also, template string literals +may only be combined with other template string literals: + + >>> t"Hello" t"{name}!" + Template(strings=('Hello', '!'), interpolations=(...)) + +Formally: + + strings: (STRING | fstring)+ | tstring+ ''', 'attribute-access': r'''Customizing attribute access **************************** @@ -588,6 +681,9 @@ class instances. Customizing module attribute access =================================== +module.__getattr__() +module.__dir__() + Special names "__getattr__" and "__dir__" can be also used to customize access to module attributes. The "__getattr__" function at the module level should accept one argument which is the name of an @@ -603,6 +699,8 @@ class instances. present, this function overrides the standard "dir()" search on a module. +module.__class__ + For a more fine grained customization of the module behavior (setting attributes, properties, etc.), one can set the "__class__" attribute of a module object to a subclass of "types.ModuleType". For example: @@ -865,7 +963,7 @@ class derived from a ""variable-length" built-in type" such as binary operation and an assignment statement: augmented_assignment_stmt: augtarget augop (expression_list | yield_expression) - augtarget: identifier | attributeref | subscription | slicing + augtarget: identifier | attributeref | subscription augop: "+=" | "-=" | "*=" | "@=" | "/=" | "//=" | "%=" | "**=" | ">>=" | "<<=" | "&=" | "^=" | "|=" @@ -959,7 +1057,7 @@ class and instance attributes applies as for regular assignments. The "%" (modulo) operator yields the remainder from the division of the first argument by the second. The numeric arguments are first -converted to a common type. A zero right argument raises the +converted to a common type. A zero right argument raises the "ZeroDivisionError" exception. The arguments may be floating-point numbers, e.g., "3.14%0.7" equals "0.34" (since "3.14" equals "4*0.7 + 0.34".) The modulo operator always yields a result with the same sign @@ -1047,12 +1145,33 @@ class and instance attributes applies as for regular assignments. 'bltin-ellipsis-object': r'''The Ellipsis Object ******************* -This object is commonly used by slicing (see Slicings). It supports -no special operations. There is exactly one ellipsis object, named -"Ellipsis" (a built-in name). "type(Ellipsis)()" produces the +This object is commonly used to indicate that something is omitted. It +supports no special operations. There is exactly one ellipsis object, +named "Ellipsis" (a built-in name). "type(Ellipsis)()" produces the "Ellipsis" singleton. It is written as "Ellipsis" or "...". + +In typical use, "..." as the "Ellipsis" object appears in a few +different places, for instance: + +* In type annotations, such as callable arguments or tuple elements. + +* As the body of a function instead of a pass statement. + +* In third-party libraries, such as Numpy’s slicing and striding. + +Python also uses three dots in ways that are not "Ellipsis" objects, +for instance: + +* Doctest’s "ELLIPSIS", as a pattern for missing content. + +* The default Python prompt of the *interactive* shell when partial + input is incomplete. + +Lastly, the Python documentation often uses three dots in conventional +English usage to mean omitted content, even in code examples that also +use them as the "Ellipsis". ''', 'bltin-null-object': r'''The Null Object *************** @@ -1724,16 +1843,16 @@ class attributes; they are shared by instances. Instance attributes The "for" statement is used to iterate over the elements of a sequence (such as a string, tuple or list) or other iterable object: - for_stmt: "for" target_list "in" starred_list ":" suite + for_stmt: "for" target_list "in" starred_expression_list ":" suite ["else" ":" suite] -The "starred_list" expression is evaluated once; it should yield an -*iterable* object. An *iterator* is created for that iterable. The -first item provided by the iterator is then assigned to the target -list using the standard rules for assignments (see Assignment -statements), and the suite is executed. This repeats for each item -provided by the iterator. When the iterator is exhausted, the suite -in the "else" clause, if present, is executed, and the loop +The "starred_expression_list" expression is evaluated once; it should +yield an *iterable* object. An *iterator* is created for that +iterable. The first item provided by the iterator is then assigned to +the target list using the standard rules for assignments (see +Assignment statements), and the suite is executed. This repeats for +each item provided by the iterator. When the iterator is exhausted, +the suite in the "else" clause, if present, is executed, and the loop terminates. A "break" statement executed in the first suite terminates the loop @@ -1874,15 +1993,29 @@ class attributes; they are shared by instances. Instance attributes "except*" clause ---------------- -The "except*" clause(s) are used for handling "ExceptionGroup"s. The -exception type for matching is interpreted as in the case of "except", -but in the case of exception groups we can have partial matches when -the type matches some of the exceptions in the group. This means that -multiple "except*" clauses can execute, each handling part of the -exception group. Each clause executes at most once and handles an -exception group of all matching exceptions. Each exception in the -group is handled by at most one "except*" clause, the first that -matches it. +The "except*" clause(s) specify one or more handlers for groups of +exceptions ("BaseExceptionGroup" instances). A "try" statement can +have either "except" or "except*" clauses, but not both. The exception +type for matching is mandatory in the case of "except*", so "except*:" +is a syntax error. The type is interpreted as in the case of "except", +but matching is performed on the exceptions contained in the group +that is being handled. An "TypeError" is raised if a matching type is +a subclass of "BaseExceptionGroup", because that would have ambiguous +semantics. + +When an exception group is raised in the try block, each "except*" +clause splits (see "split()") it into the subgroups of matching and +non-matching exceptions. If the matching subgroup is not empty, it +becomes the handled exception (the value returned from +"sys.exception()") and assigned to the target of the "except*" clause +(if there is one). Then, the body of the "except*" clause executes. If +the non-matching subgroup is not empty, it is processed by the next +"except*" in the same manner. This continues until all exceptions in +the group have been matched, or the last "except*" clause has run. + +After all "except*" clauses execute, the group of unhandled exceptions +is merged with any exceptions that were raised or re-raised from +within "except*" clauses. This merged exception group propagates on.: >>> try: ... raise ExceptionGroup("eg", @@ -1895,33 +2028,27 @@ class attributes; they are shared by instances. Instance attributes caught with nested (TypeError(2),) caught with nested (OSError(3), OSError(4)) + Exception Group Traceback (most recent call last): - | File "", line 2, in - | ExceptionGroup: eg + | File "", line 2, in + | raise ExceptionGroup("eg", + | [ValueError(1), TypeError(2), OSError(3), OSError(4)]) + | ExceptionGroup: eg (1 sub-exception) +-+---------------- 1 ---------------- | ValueError: 1 +------------------------------------ -Any remaining exceptions that were not handled by any "except*" clause -are re-raised at the end, along with all exceptions that were raised -from within the "except*" clauses. If this list contains more than one -exception to reraise, they are combined into an exception group. - -If the raised exception is not an exception group and its type matches -one of the "except*" clauses, it is caught and wrapped by an exception -group with an empty message string. +If the exception raised from the "try" block is not an exception group +and its type matches one of the "except*" clauses, it is caught and +wrapped by an exception group with an empty message string. This +ensures that the type of the target "e" is consistently +"BaseExceptionGroup": >>> try: ... raise BlockingIOError ... except* BlockingIOError as e: ... print(repr(e)) ... - ExceptionGroup('', (BlockingIOError())) + ExceptionGroup('', (BlockingIOError(),)) -An "except*" clause must have a matching expression; it cannot be -"except*:". Furthermore, this expression cannot contain exception -group types, because that would have ambiguous semantics. - -It is not possible to mix "except" and "except*" in the same "try". "break", "continue" and "return" cannot appear in an "except*" clause. @@ -1938,11 +2065,11 @@ class attributes; they are shared by instances. Instance attributes ---------------- If "finally" is present, it specifies a ‘cleanup’ handler. The "try" -clause is executed, including any "except" and "else" clauses. If an +clause is executed, including any "except" and "else" clauses. If an exception occurs in any of the clauses and is not handled, the exception is temporarily saved. The "finally" clause is executed. If there is a saved exception it is re-raised at the end of the "finally" -clause. If the "finally" clause raises another exception, the saved +clause. If the "finally" clause raises another exception, the saved exception is set as the context of the new exception. If the "finally" clause executes a "return", "break" or "continue" statement, the saved exception is discarded. For example, this function returns 42. @@ -2040,9 +2167,9 @@ def foo(): is semantically equivalent to: manager = (EXPRESSION) - enter = type(manager).__enter__ - exit = type(manager).__exit__ - value = enter(manager) + enter = manager.__enter__ + exit = manager.__exit__ + value = enter() hit_except = False try: @@ -2050,11 +2177,14 @@ def foo(): SUITE except: hit_except = True - if not exit(manager, *sys.exc_info()): + if not exit(*sys.exc_info()): raise finally: if not hit_except: - exit(manager, None, None, None) + exit(None, None, None) + +except that implicit special method lookup is used for "__enter__()" +and "__exit__()". With more than one item, the context managers are processed as if multiple "with" statements were nested: @@ -2097,9 +2227,9 @@ def foo(): The match statement is used for pattern matching. Syntax: match_stmt: 'match' subject_expr ":" NEWLINE INDENT case_block+ DEDENT - subject_expr: star_named_expression "," star_named_expressions? - | named_expression - case_block: 'case' patterns [guard] ":" block + subject_expr: flexible_expression "," [flexible_expression_list [',']] + | assignment_expression + case_block: 'case' patterns [guard] ":" suite Note: @@ -2190,7 +2320,7 @@ def foo(): Guards ------ - guard: "if" named_expression + guard: "if" assignment_expression A "guard" (which is part of the "case") must succeed for code inside the "case" block to execute. It takes the form: "if" followed by an @@ -2329,7 +2459,8 @@ def foo(): The rule "strings" and the token "NUMBER" are defined in the standard Python grammar. Triple-quoted strings are supported. Raw strings and -byte strings are supported. f-strings are not supported. +byte strings are supported. f-strings and t-strings are not +supported. The forms "signed_number '+' NUMBER" and "signed_number '-' NUMBER" are for expressing complex numbers; they require a real number on the @@ -2483,7 +2614,7 @@ def foo(): Note: The length of the subject sequence is obtained via "len()" (i.e. - via the "__len__()" protocol). This length may be cached by the + via the "__len__()" protocol). This length may be cached by the interpreter in a similar manner as value patterns. In simple terms "[P1, P2, P3," … ", P]" matches only if all the @@ -2591,7 +2722,7 @@ def foo(): If only keyword patterns are present, they are processed as follows, one by one: - I. The keyword is looked up as an attribute on the subject. + 1. The keyword is looked up as an attribute on the subject. * If this raises an exception other than "AttributeError", the exception bubbles up. @@ -2603,14 +2734,14 @@ def foo(): the class pattern fails; if this succeeds, the match proceeds to the next keyword. - II. If all keyword patterns succeed, the class pattern succeeds. + 2. If all keyword patterns succeed, the class pattern succeeds. If any positional patterns are present, they are converted to keyword patterns using the "__match_args__" attribute on the class "name_or_attr" before matching: - I. The equivalent of "getattr(cls, "__match_args__", ())" is - called. + 1. The equivalent of "getattr(cls, "__match_args__", ())" is + called. * If this raises an exception, the exception bubbles up. @@ -2631,9 +2762,9 @@ def foo(): Customizing positional arguments in class pattern matching - II. Once all positional patterns have been converted to keyword - patterns, - the match proceeds as if there were only keyword patterns. + 2. Once all positional patterns have been converted to keyword + patterns, the match proceeds as if there were only keyword + patterns. For the following built-in types the handling of positional subpatterns is different: @@ -2985,13 +3116,12 @@ async def func(param1, param2): Is semantically equivalent to: - iter = (ITER) - iter = type(iter).__aiter__(iter) + iter = (ITER).__aiter__() running = True while running: try: - TARGET = await type(iter).__anext__(iter) + TARGET = await iter.__anext__() except StopAsyncIteration: running = False else: @@ -2999,7 +3129,8 @@ async def func(param1, param2): else: SUITE2 -See also "__aiter__()" and "__anext__()" for details. +except that implicit special method lookup is used for "__aiter__()" +and "__anext__()". It is a "SyntaxError" to use an "async for" statement outside the body of a coroutine function. @@ -3021,9 +3152,9 @@ async def func(param1, param2): is semantically equivalent to: manager = (EXPRESSION) - aenter = type(manager).__aenter__ - aexit = type(manager).__aexit__ - value = await aenter(manager) + aenter = manager.__aenter__ + aexit = manager.__aexit__ + value = await aenter() hit_except = False try: @@ -3031,13 +3162,14 @@ async def func(param1, param2): SUITE except: hit_except = True - if not await aexit(manager, *sys.exc_info()): + if not await aexit(*sys.exc_info()): raise finally: if not hit_except: - await aexit(manager, None, None, None) + await aexit(None, None, None) -See also "__aenter__()" and "__aexit__()" for details. +except that implicit special method lookup is used for "__aenter__()" +and "__aexit__()". It is a "SyntaxError" to use an "async with" statement outside the body of a coroutine function. @@ -3304,7 +3436,7 @@ def f() -> annotation: ... introspects and uses the annotations (such as "dataclasses" or "functools.singledispatch()"). -By default, annotations are lazily evaluated in a annotation scope. +By default, annotations are lazily evaluated in an annotation scope. This means that they are not evaluated when the code containing the annotation is evaluated. Instead, the interpreter saves information that can be used to evaluate the annotation later if requested. The @@ -3318,6 +3450,12 @@ def f() -> annotation: ... >>> f.__annotations__ {'param': 'annotation'} +This future statement will be deprecated and removed in a future +version of Python, but not before Python 3.13 reaches its end of life +(see **PEP 749**). When it is used, introspection tools like +"annotationlib.get_annotations()" and "typing.get_type_hints()" are +less likely to be able to resolve annotations at runtime. + -[ Footnotes ]- [1] The exception is propagated to the invocation stack unless there @@ -3442,19 +3580,13 @@ def f() -> annotation: ... When a description of an arithmetic operator below uses the phrase “the numeric arguments are converted to a common real type”, this -means that the operator implementation for built-in types works as -follows: - -* If both arguments are complex numbers, no conversion is performed; - -* if either argument is a complex or a floating-point number, the - other is converted to a floating-point number; +means that the operator implementation for built-in numeric types +works as described in the Numeric Types section of the standard +library documentation. -* otherwise, both must be integers and no conversion is necessary. - -Some additional rules apply for certain operators (e.g., a string as a -left argument to the ‘%’ operator). Extensions must define their own -conversion behavior. +Some additional rules apply for certain operators and non-numeric +operands (for example, a string as a left argument to the "%" +operator). Extensions must define their own conversion behavior. ''', 'customization': r'''Basic customization ******************* @@ -3611,7 +3743,7 @@ def f() -> annotation: ... formatting to one of the built-in types, or use a similar formatting option syntax. - See Format Specification Mini-Language for a description of the + See Format specification mini-language for a description of the standard formatting syntax. The return value must be a string object. @@ -3748,7 +3880,7 @@ def __hash__(self): intended to provide protection against a denial-of-service caused by carefully chosen inputs that exploit the worst case performance of a dict insertion, *O*(*n*^2) complexity. See - http://ocert.org/advisories/ocert-2011-003.html for + https://ocert.org/advisories/ocert-2011-003.html for details.Changing hash values affects the iteration order of sets. Python has never made guarantees about this ordering (and it typically varies between 32-bit and 64-bit builds).See also @@ -3829,10 +3961,14 @@ def double(x): available for commands and command arguments, e.g. the current global and local names are offered as arguments of the "p" command. + +Command-line interface +====================== + You can also invoke "pdb" from the command line to debug other scripts. For example: - python -m pdb [-c command] (-m module | pyfile) [args ...] + python -m pdb [-c command] (-m module | -p pid | pyfile) [args ...] When invoked as a module, pdb will automatically enter post-mortem debugging if the program being debugged exits abnormally. After post- @@ -3844,7 +3980,7 @@ def double(x): -c, --command To execute commands as if given in a ".pdbrc" file; see Debugger - Commands. + commands. Changed in version 3.2: Added the "-c" option. @@ -3856,6 +3992,23 @@ def double(x): Changed in version 3.7: Added the "-m" option. +-p, --pid + + Attach to the process with the specified PID. + + Added in version 3.14. + +To attach to a running Python process for remote debugging, use the +"-p" or "--pid" option with the target process’s PID: + + python -m pdb -p 1234 + +Note: + + Attaching to a process that is blocked in a system call or waiting + for I/O will only work once the next bytecode instruction is + executed or when the process receives a signal. + Typical usage to execute a statement under control of the debugger is: >>> import pdb @@ -4048,7 +4201,7 @@ class pdb.Pdb(completekey='tab', stdin=None, stdout=None, skip=None, nosigint=Fa See the documentation for the functions explained above. -Debugger Commands +Debugger commands ================= The commands recognized by the debugger are listed below. Most @@ -4574,11 +4727,11 @@ def inner(x): Deletion of a name removes the binding of that name from the local or global namespace, depending on whether the name occurs in a "global" -statement in the same code block. If the name is unbound, a -"NameError" exception will be raised. +statement in the same code block. Trying to delete an unbound name +raises a "NameError" exception. -Deletion of attribute references, subscriptions and slicings is passed -to the primary object involved; deletion of a slicing is in general +Deletion of attribute references and subscriptions is passed to the +primary object involved; deletion of a slicing is in general equivalent to assignment of an empty slice of the right type (but even this is determined by the sliced object). @@ -4711,11 +4864,6 @@ class of the instance or a *non-virtual base class* thereof. The See also the description of the "try" statement in section The try statement and "raise" statement in section The raise statement. - --[ Footnotes ]- - -[1] This limitation occurs because the code that is executed by these - operations is not available at the time the module is compiled. ''', 'execmodel': r'''Execution model *************** @@ -5069,6 +5217,181 @@ class of the instance or a *non-virtual base class* thereof. The See also the description of the "try" statement in section The try statement and "raise" statement in section The raise statement. + +Runtime Components +================== + + +General Computing Model +----------------------- + +Python’s execution model does not operate in a vacuum. It runs on a +host machine and through that host’s runtime environment, including +its operating system (OS), if there is one. When a program runs, the +conceptual layers of how it runs on the host look something like this: + + **host machine** + **process** (global resources) + **thread** (runs machine code) + +Each process represents a program running on the host. Think of each +process itself as the data part of its program. Think of the process’ +threads as the execution part of the program. This distinction will +be important to understand the conceptual Python runtime. + +The process, as the data part, is the execution context in which the +program runs. It mostly consists of the set of resources assigned to +the program by the host, including memory, signals, file handles, +sockets, and environment variables. + +Processes are isolated and independent from one another. (The same is +true for hosts.) The host manages the process’ access to its assigned +resources, in addition to coordinating between processes. + +Each thread represents the actual execution of the program’s machine +code, running relative to the resources assigned to the program’s +process. It’s strictly up to the host how and when that execution +takes place. + +From the point of view of Python, a program always starts with exactly +one thread. However, the program may grow to run in multiple +simultaneous threads. Not all hosts support multiple threads per +process, but most do. Unlike processes, threads in a process are not +isolated and independent from one another. Specifically, all threads +in a process share all of the process’ resources. + +The fundamental point of threads is that each one does *run* +independently, at the same time as the others. That may be only +conceptually at the same time (“concurrently”) or physically (“in +parallel”). Either way, the threads effectively run at a non- +synchronized rate. + +Note: + + That non-synchronized rate means none of the process’ memory is + guaranteed to stay consistent for the code running in any given + thread. Thus multi-threaded programs must take care to coordinate + access to intentionally shared resources. Likewise, they must take + care to be absolutely diligent about not accessing any *other* + resources in multiple threads; otherwise two threads running at the + same time might accidentally interfere with each other’s use of some + shared data. All this is true for both Python programs and the + Python runtime.The cost of this broad, unstructured requirement is + the tradeoff for the kind of raw concurrency that threads provide. + The alternative to the required discipline generally means dealing + with non-deterministic bugs and data corruption. + + +Python Runtime Model +-------------------- + +The same conceptual layers apply to each Python program, with some +extra data layers specific to Python: + + **host machine** + **process** (global resources) + Python global runtime (*state*) + Python interpreter (*state*) + **thread** (runs Python bytecode and “C-API”) + Python thread *state* + +At the conceptual level: when a Python program starts, it looks +exactly like that diagram, with one of each. The runtime may grow to +include multiple interpreters, and each interpreter may grow to +include multiple thread states. + +Note: + + A Python implementation won’t necessarily implement the runtime + layers distinctly or even concretely. The only exception is places + where distinct layers are directly specified or exposed to users, + like through the "threading" module. + +Note: + + The initial interpreter is typically called the “main” interpreter. + Some Python implementations, like CPython, assign special roles to + the main interpreter.Likewise, the host thread where the runtime was + initialized is known as the “main” thread. It may be different from + the process’ initial thread, though they are often the same. In + some cases “main thread” may be even more specific and refer to the + initial thread state. A Python runtime might assign specific + responsibilities to the main thread, such as handling signals. + +As a whole, the Python runtime consists of the global runtime state, +interpreters, and thread states. The runtime ensures all that state +stays consistent over its lifetime, particularly when used with +multiple host threads. + +The global runtime, at the conceptual level, is just a set of +interpreters. While those interpreters are otherwise isolated and +independent from one another, they may share some data or other +resources. The runtime is responsible for managing these global +resources safely. The actual nature and management of these resources +is implementation-specific. Ultimately, the external utility of the +global runtime is limited to managing interpreters. + +In contrast, an “interpreter” is conceptually what we would normally +think of as the (full-featured) “Python runtime”. When machine code +executing in a host thread interacts with the Python runtime, it calls +into Python in the context of a specific interpreter. + +Note: + + The term “interpreter” here is not the same as the “bytecode + interpreter”, which is what regularly runs in threads, executing + compiled Python code.In an ideal world, “Python runtime” would refer + to what we currently call “interpreter”. However, it’s been called + “interpreter” at least since introduced in 1997 (CPython:a027efa5b). + +Each interpreter completely encapsulates all of the non-process- +global, non-thread-specific state needed for the Python runtime to +work. Notably, the interpreter’s state persists between uses. It +includes fundamental data like "sys.modules". The runtime ensures +multiple threads using the same interpreter will safely share it +between them. + +A Python implementation may support using multiple interpreters at the +same time in the same process. They are independent and isolated from +one another. For example, each interpreter has its own "sys.modules". + +For thread-specific runtime state, each interpreter has a set of +thread states, which it manages, in the same way the global runtime +contains a set of interpreters. It can have thread states for as many +host threads as it needs. It may even have multiple thread states for +the same host thread, though that isn’t as common. + +Each thread state, conceptually, has all the thread-specific runtime +data an interpreter needs to operate in one host thread. The thread +state includes the current raised exception and the thread’s Python +call stack. It may include other thread-specific resources. + +Note: + + The term “Python thread” can sometimes refer to a thread state, but + normally it means a thread created using the "threading" module. + +Each thread state, over its lifetime, is always tied to exactly one +interpreter and exactly one host thread. It will only ever be used in +that thread and with that interpreter. + +Multiple thread states may be tied to the same host thread, whether +for different interpreters or even the same interpreter. However, for +any given host thread, only one of the thread states tied to it can be +used by the thread at a time. + +Thread states are isolated and independent from one another and don’t +share any data, except for possibly sharing an interpreter and objects +or other resources belonging to that interpreter. + +Once a program is running, new Python threads can be created using the +"threading" module (on platforms and Python implementations that +support threads). Additional processes can be created using the "os", +"subprocess", and "multiprocessing" modules. Interpreters can be +created and used with the "interpreters" module. Coroutines (async) +can be run using "asyncio" in each interpreter, typically only in a +single thread (often the main thread). + -[ Footnotes ]- [1] This limitation occurs because the code that is executed by these @@ -5077,7 +5400,7 @@ class of the instance or a *non-virtual base class* thereof. The 'exprlists': r'''Expression lists **************** - starred_expression: ["*"] or_expr + starred_expression: "*" or_expr | expression flexible_expression: assignment_expression | starred_expression flexible_expression_list: flexible_expression ("," flexible_expression)* [","] starred_expression_list: starred_expression ("," starred_expression)* [","] @@ -5109,25 +5432,53 @@ class of the instance or a *non-virtual base class* thereof. The 'floating': r'''Floating-point literals *********************** -Floating-point literals are described by the following lexical -definitions: +Floating-point (float) literals, such as "3.14" or "1.5", denote +approximations of real numbers. + +They consist of *integer* and *fraction* parts, each composed of +decimal digits. The parts are separated by a decimal point, ".": + + 2.71828 + 4.0 + +Unlike in integer literals, leading zeros are allowed. For example, +"077.010" is legal, and denotes the same number as "77.01". + +As in integer literals, single underscores may occur between digits to +help readability: + + 96_485.332_123 + 3.14_15_93 + +Either of these parts, but not both, can be empty. For example: + + 10. # (equivalent to 10.0) + .001 # (equivalent to 0.001) + +Optionally, the integer and fraction may be followed by an *exponent*: +the letter "e" or "E", followed by an optional sign, "+" or "-", and a +number in the same format as the integer and fraction parts. The "e" +or "E" represents “times ten raised to the power of”: + + 1.0e3 # (represents 1.0×10³, or 1000.0) + 1.166e-5 # (represents 1.166×10⁻⁵, or 0.00001166) + 6.02214076e+23 # (represents 6.02214076×10²³, or 602214076000000000000000.) - floatnumber: pointfloat | exponentfloat - pointfloat: [digitpart] fraction | digitpart "." - exponentfloat: (digitpart | pointfloat) exponent - digitpart: digit (["_"] digit)* - fraction: "." digitpart - exponent: ("e" | "E") ["+" | "-"] digitpart +In floats with only integer and exponent parts, the decimal point may +be omitted: -Note that the integer and exponent parts are always interpreted using -radix 10. For example, "077e010" is legal, and denotes the same number -as "77e10". The allowed range of floating-point literals is -implementation-dependent. As in integer literals, underscores are -supported for digit grouping. + 1e3 # (equivalent to 1.e3 and 1.0e3) + 0e0 # (equivalent to 0.) -Some examples of floating-point literals: +Formally, floating-point literals are described by the following +lexical definitions: - 3.14 10. .001 1e100 3.14e-10 0e0 3.14_15_93 + floatnumber: + | digitpart "." [digitpart] [exponent] + | "." digitpart [exponent] + | digitpart exponent + digitpart: digit (["_"] digit)* + exponent: ("e" | "E") ["+" | "-"] digitpart Changed in version 3.6: Underscores are now allowed for grouping purposes in literals. @@ -5138,16 +5489,16 @@ class of the instance or a *non-virtual base class* thereof. The The "for" statement is used to iterate over the elements of a sequence (such as a string, tuple or list) or other iterable object: - for_stmt: "for" target_list "in" starred_list ":" suite + for_stmt: "for" target_list "in" starred_expression_list ":" suite ["else" ":" suite] -The "starred_list" expression is evaluated once; it should yield an -*iterable* object. An *iterator* is created for that iterable. The -first item provided by the iterator is then assigned to the target -list using the standard rules for assignments (see Assignment -statements), and the suite is executed. This repeats for each item -provided by the iterator. When the iterator is exhausted, the suite -in the "else" clause, if present, is executed, and the loop +The "starred_expression_list" expression is evaluated once; it should +yield an *iterable* object. An *iterator* is created for that +iterable. The first item provided by the iterator is then assigned to +the target list using the standard rules for assignments (see +Assignment statements), and the suite is executed. This repeats for +each item provided by the iterator. When the iterator is exhausted, +the suite in the "else" clause, if present, is executed, and the loop terminates. A "break" statement executed in the first suite terminates the loop @@ -5175,15 +5526,15 @@ class of the instance or a *non-virtual base class* thereof. The Changed in version 3.11: Starred elements are now allowed in the expression list. ''', - 'formatstrings': r'''Format String Syntax + 'formatstrings': r'''Format string syntax ******************** The "str.format()" method and the "Formatter" class share the same syntax for format strings (although in the case of "Formatter", subclasses can define their own format string syntax). The syntax is -related to that of formatted string literals, but it is less -sophisticated and, in particular, does not support arbitrary -expressions. +related to that of formatted string literals and template string +literals, but it is less sophisticated and, in particular, does not +support arbitrary expressions in interpolations. Format strings contain “replacement fields” surrounded by curly braces "{}". Anything that is not contained in braces is considered literal @@ -5210,7 +5561,7 @@ class of the instance or a *non-virtual base class* thereof. The preceded by a colon "':'". These specify a non-default format for the replacement value. -See also the Format Specification Mini-Language section. +See also the Format specification mini-language section. The *field_name* itself begins with an *arg_name* that is either a number or a keyword. If it’s a number, it refers to a positional @@ -5278,14 +5629,14 @@ class of the instance or a *non-virtual base class* thereof. The See the Format examples section for some examples. -Format Specification Mini-Language +Format specification mini-language ================================== “Format specifications” are used within replacement fields contained within a format string to define how individual values are presented -(see Format String Syntax and f-strings). They can also be passed -directly to the built-in "format()" function. Each formattable type -may define how the format specification is to be interpreted. +(see Format string syntax, f-strings, and t-strings). They can also be +passed directly to the built-in "format()" function. Each formattable +type may define how the format specification is to be interpreted. Most built-in types implement the following options for format specifications, although some of the formatting options are only @@ -5304,7 +5655,7 @@ class of the instance or a *non-virtual base class* thereof. The sign: "+" | "-" | " " width_and_precision: [width_with_grouping][precision_with_grouping] width_with_grouping: [width][grouping] - precision_with_grouping: "." [precision][grouping] + precision_with_grouping: "." [precision][grouping] | "." grouping width: digit+ precision: digit+ grouping: "," | "_" @@ -5421,7 +5772,8 @@ class of the instance or a *non-virtual base class* thereof. The | | is not supported. | +-----------+------------------------------------------------------------+ -For a locale aware separator, use the "'n'" presentation type instead. +For a locale-aware separator, use the "'n'" float presentation type or +integer presentation type instead. Changed in version 3.1: Added the "','" option (see also **PEP 378**). @@ -5467,7 +5819,10 @@ class of the instance or a *non-virtual base class* thereof. The +-----------+------------------------------------------------------------+ | "'n'" | Number. This is the same as "'d'", except that it uses the | | | current locale setting to insert the appropriate digit | - | | group separators. | + | | group separators. Note that the default locale is not the | + | | system locale. Depending on your use case, you may wish to | + | | set "LC_NUMERIC" with "locale.setlocale()" before using | + | | "'n'". | +-----------+------------------------------------------------------------+ | None | The same as "'d'". | +-----------+------------------------------------------------------------+ @@ -5490,7 +5845,9 @@ class of the instance or a *non-virtual base class* thereof. The | | With no precision given, uses a precision of "6" digits | | | after the decimal point for "float", and shows all | | | coefficient digits for "Decimal". If "p=0", the decimal | - | | point is omitted unless the "#" option is used. | + | | point is omitted unless the "#" option is used. For | + | | "float", the exponent always contains at least two digits, | + | | and is zero if the value is zero. | +-----------+------------------------------------------------------------+ | "'E'" | Scientific notation. Same as "'e'" except it uses an upper | | | case ‘E’ as the separator character. | @@ -5539,7 +5896,10 @@ class of the instance or a *non-virtual base class* thereof. The +-----------+------------------------------------------------------------+ | "'n'" | Number. This is the same as "'g'", except that it uses the | | | current locale setting to insert the appropriate digit | - | | group separators for the integral part of a number. | + | | group separators for the integral part of a number. Note | + | | that the default locale is not the system locale. | + | | Depending on your use case, you may wish to set | + | | "LC_NUMERIC" with "locale.setlocale()" before using "'n'". | +-----------+------------------------------------------------------------+ | "'%'" | Percentage. Multiplies the number by 100 and displays in | | | fixed ("'f'") format, followed by a percent sign. | @@ -5688,8 +6048,8 @@ class of the instance or a *non-virtual base class* thereof. The Using type-specific formatting: - >>> import datetime - >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58) + >>> import datetime as dt + >>> d = dt.datetime(2010, 7, 4, 12, 15, 58) >>> '{:%Y-%m-%d %H:%M:%S}'.format(d) '2010-07-04 12:15:58' @@ -5886,9 +6246,15 @@ def whats_on_the_telly(penguin=None): without "global", although free variables may refer to globals without being declared global. -The "global" statement applies to the entire scope of a function or -class body. A "SyntaxError" is raised if a variable is used or -assigned to prior to its global declaration in the scope. +The "global" statement applies to the entire current scope (module, +function body or class definition). A "SyntaxError" is raised if a +variable is used or assigned to prior to its global declaration in the +scope. + +At the module level, all variables are global, so a "global" statement +has no effect. However, variables must still not be used or assigned +to prior to their "global" declaration. This requirement is relaxed in +the interactive prompt (*REPL*). **Programmer’s note:** "global" is a directive to the parser. It applies only to code parsed at the same time as the "global" @@ -5942,73 +6308,45 @@ class body. A "SyntaxError" is raised if a variable is used or to help avoid name clashes between “private” attributes of base and derived classes. See section Identifiers (Names). ''', - 'identifiers': r'''Identifiers and keywords -************************ - -Identifiers (also referred to as *names*) are described by the -following lexical definitions. - -The syntax of identifiers in Python is based on the Unicode standard -annex UAX-31, with elaboration and changes as defined below; see also -**PEP 3131** for further details. - -Within the ASCII range (U+0001..U+007F), the valid characters for -identifiers include the uppercase and lowercase letters "A" through -"Z", the underscore "_" and, except for the first character, the -digits "0" through "9". Python 3.0 introduced additional characters -from outside the ASCII range (see **PEP 3131**). For these -characters, the classification uses the version of the Unicode -Character Database as included in the "unicodedata" module. - -Identifiers are unlimited in length. Case is significant. + 'identifiers': r'''Names (identifiers and keywords) +******************************** - identifier: xid_start xid_continue* - id_start: - id_continue: - xid_start: - xid_continue: +"NAME" tokens represent *identifiers*, *keywords*, and *soft +keywords*. -The Unicode category codes mentioned above stand for: +Names are composed of the following characters: -* *Lu* - uppercase letters +* uppercase and lowercase letters ("A-Z" and "a-z"), -* *Ll* - lowercase letters +* the underscore ("_"), -* *Lt* - titlecase letters +* digits ("0" through "9"), which cannot appear as the first + character, and -* *Lm* - modifier letters +* non-ASCII characters. Valid names may only contain “letter-like” and + “digit-like” characters; see Non-ASCII characters in names for + details. -* *Lo* - other letters +Names must contain at least one character, but have no upper length +limit. Case is significant. -* *Nl* - letter numbers +Formally, names are described by the following lexical definitions: -* *Mn* - nonspacing marks + NAME: name_start name_continue* + name_start: "a"..."z" | "A"..."Z" | "_" | + name_continue: name_start | "0"..."9" + identifier: -* *Mc* - spacing combining marks - -* *Nd* - decimal numbers - -* *Pc* - connector punctuations - -* *Other_ID_Start* - explicit list of characters in PropList.txt to - support backwards compatibility - -* *Other_ID_Continue* - likewise - -All identifiers are converted into the normal form NFKC while parsing; -comparison of identifiers is based on NFKC. - -A non-normative HTML file listing all valid identifier characters for -Unicode 16.0.0 can be found at -https://www.unicode.org/Public/16.0.0/ucd/DerivedCoreProperties.txt +Note that not all names matched by this grammar are valid; see Non- +ASCII characters in names for details. Keywords ======== -The following identifiers are used as reserved words, or *keywords* of -the language, and cannot be used as ordinary identifiers. They must -be spelled exactly as written here: +The following names are used as reserved words, or *keywords* of the +language, and cannot be used as ordinary identifiers. They must be +spelled exactly as written here: False await else import pass None break except in raise @@ -6024,18 +6362,20 @@ class body. A "SyntaxError" is raised if a variable is used or Added in version 3.10. -Some identifiers are only reserved under specific contexts. These are -known as *soft keywords*. The identifiers "match", "case", "type" and -"_" can syntactically act as keywords in certain contexts, but this -distinction is done at the parser level, not when tokenizing. +Some names are only reserved under specific contexts. These are known +as *soft keywords*: + +* "match", "case", and "_", when used in the "match" statement. + +* "type", when used in the "type" statement. + +These syntactically act as keywords in their specific contexts, but +this distinction is done at the parser level, not when tokenizing. As soft keywords, their use in the grammar is possible while still preserving compatibility with existing code that uses these names as identifier names. -"match", "case", and "_" are used in the "match" statement. "type" is -used in the "type" statement. - Changed in version 3.12: "type" is now a soft keyword. @@ -6081,6 +6421,101 @@ class body. A "SyntaxError" is raised if a variable is used or context of a class definition, are re-written to use a mangled form to help avoid name clashes between “private” attributes of base and derived classes. See section Identifiers (Names). + + +Non-ASCII characters in names +============================= + +Names that contain non-ASCII characters need additional normalization +and validation beyond the rules and grammar explained above. For +example, "ř_1", "蛇", or "साँप" are valid names, but "r〰2", "€", or +"🐍" are not. + +This section explains the exact rules. + +All names are converted into the normalization form NFKC while +parsing. This means that, for example, some typographic variants of +characters are converted to their “basic” form. For example, +"fiⁿₐˡᵢᶻₐᵗᵢᵒₙ" normalizes to "finalization", so Python treats them as +the same name: + + >>> fiⁿₐˡᵢᶻₐᵗᵢᵒₙ = 3 + >>> finalization + 3 + +Note: + + Normalization is done at the lexical level only. Run-time functions + that take names as *strings* generally do not normalize their + arguments. For example, the variable defined above is accessible at + run time in the "globals()" dictionary as + "globals()["finalization"]" but not "globals()["fiⁿₐˡᵢᶻₐᵗᵢᵒₙ"]". + +Similarly to how ASCII-only names must contain only letters, digits +and the underscore, and cannot start with a digit, a valid name must +start with a character in the “letter-like” set "xid_start", and the +remaining characters must be in the “letter- and digit-like” set +"xid_continue". + +These sets are based on the *XID_Start* and *XID_Continue* sets as +defined by the Unicode standard annex UAX-31. Python’s "xid_start" +additionally includes the underscore ("_"). Note that Python does not +necessarily conform to UAX-31. + +A non-normative listing of characters in the *XID_Start* and +*XID_Continue* sets as defined by Unicode is available in the +DerivedCoreProperties.txt file in the Unicode Character Database. For +reference, the construction rules for the "xid_*" sets are given +below. + +The set "id_start" is defined as the union of: + +* Unicode category "" - uppercase letters (includes "A" to "Z") + +* Unicode category "" - lowercase letters (includes "a" to "z") + +* Unicode category "" - titlecase letters + +* Unicode category "" - modifier letters + +* Unicode category "" - other letters + +* Unicode category "" - letter numbers + +* {""_""} - the underscore + +* "" - an explicit set of characters in PropList.txt + to support backwards compatibility + +The set "xid_start" then closes this set under NFKC normalization, by +removing all characters whose normalization is not of the form +"id_start id_continue*". + +The set "id_continue" is defined as the union of: + +* "id_start" (see above) + +* Unicode category "" - decimal numbers (includes "0" to "9") + +* Unicode category "" - connector punctuations + +* Unicode category "" - nonspacing marks + +* Unicode category "" - spacing combining marks + +* "" - another explicit set of characters in + PropList.txt to support backwards compatibility + +Again, "xid_continue" closes this set under NFKC normalization. + +Unicode categories use the version of the Unicode Character Database +as included in the "unicodedata" module. + +See also: + + * **PEP 3131** – Supporting Non-ASCII Identifiers + + * **PEP 672** – Unicode-related Security Considerations for Python ''', 'if': r'''The "if" statement ****************** @@ -6101,17 +6536,53 @@ class body. A "SyntaxError" is raised if a variable is used or 'imaginary': r'''Imaginary literals ****************** -Imaginary literals are described by the following lexical definitions: +Python has complex number objects, but no complex literals. Instead, +*imaginary literals* denote complex numbers with a zero real part. - imagnumber: (floatnumber | digitpart) ("j" | "J") +For example, in math, the complex number 3+4.2*i* is written as the +real number 3 added to the imaginary number 4.2*i*. Python uses a +similar syntax, except the imaginary unit is written as "j" rather +than *i*: + + 3+4.2j + +This is an expression composed of the integer literal "3", the +operator ‘"+"’, and the imaginary literal "4.2j". Since these are +three separate tokens, whitespace is allowed between them: + + 3 + 4.2j + +No whitespace is allowed *within* each token. In particular, the "j" +suffix, may not be separated from the number before it. + +The number before the "j" has the same syntax as a floating-point +literal. Thus, the following are valid imaginary literals: + + 4.2j + 3.14j + 10.j + .001j + 1e100j + 3.14e-10j + 3.14_15_93j + +Unlike in a floating-point literal the decimal point can be omitted if +the imaginary number only has an integer part. The number is still +evaluated as a floating-point number, not an integer: + + 10j + 0j + 1000000000000000000000000j # equivalent to 1e+24j -An imaginary literal yields a complex number with a real part of 0.0. -Complex numbers are represented as a pair of floating-point numbers -and have the same restrictions on their range. To create a complex -number with a nonzero real part, add a floating-point number to it, -e.g., "(3+4j)". Some examples of imaginary literals: +The "j" suffix is case-insensitive. That means you can use "J" +instead: - 3.14j 10.j 10j .001j 1e100j 3.14e-10j 3.14_15_93j + 3.14J # equivalent to 3.14j + +Formally, imaginary literals are described by the following lexical +definition: + + imagnumber: (floatnumber | digitpart) ("j" | "J") ''', 'import': r'''The "import" statement ********************** @@ -6130,8 +6601,9 @@ class body. A "SyntaxError" is raised if a variable is used or 1. find a module, loading and initializing it if necessary -2. define a name or names in the local namespace for the scope where - the "import" statement occurs. +2. define a name or names in the current namespace for the scope where + the "import" statement occurs, just as an assignment statement + would (including "global" and "nonlocal" semantics). When the statement contains multiple clauses (separated by commas) the two steps are carried out separately for each clause, just as though @@ -6176,7 +6648,7 @@ class body. A "SyntaxError" is raised if a variable is used or 3. if the attribute is not found, "ImportError" is raised. - 4. otherwise, a reference to that value is stored in the local + 4. otherwise, a reference to that value is stored in the current namespace, using the name in the "as" clause if it is present, otherwise using the attribute name @@ -6195,7 +6667,9 @@ class body. A "SyntaxError" is raised if a variable is used or The *public names* defined by a module are determined by checking the module’s namespace for a variable named "__all__"; if defined, it must be a sequence of strings which are names defined or imported by that -module. The names given in "__all__" are all considered public and +module. Names containing non-ASCII characters must be in the +normalization form NFKC; see Non-ASCII characters in names for +details. The names given in "__all__" are all considered public and are required to exist. If "__all__" is not defined, the set of public names includes all names found in the module’s namespace which do not begin with an underscore character ("'_'"). "__all__" should contain @@ -6353,37 +6827,62 @@ class body. A "SyntaxError" is raised if a variable is used or 'integers': r'''Integer literals **************** -Integer literals are described by the following lexical definitions: +Integer literals denote whole numbers. For example: + + 7 + 3 + 2147483647 + +There is no limit for the length of integer literals apart from what +can be stored in available memory: + + 7922816251426433759354395033679228162514264337593543950336 + +Underscores can be used to group digits for enhanced readability, and +are ignored for determining the numeric value of the literal. For +example, the following literals are equivalent: + + 100_000_000_000 + 100000000000 + 1_00_00_00_00_000 + +Underscores can only occur between digits. For example, "_123", +"321_", and "123__321" are *not* valid literals. + +Integers can be specified in binary (base 2), octal (base 8), or +hexadecimal (base 16) using the prefixes "0b", "0o" and "0x", +respectively. Hexadecimal digits 10 through 15 are represented by +letters "A"-"F", case-insensitive. For example: + + 0b100110111 + 0b_1110_0101 + 0o177 + 0o377 + 0xdeadbeef + 0xDead_Beef + +An underscore can follow the base specifier. For example, "0x_1f" is a +valid literal, but "0_x1f" and "0x__1f" are not. + +Leading zeros in a non-zero decimal number are not allowed. For +example, "0123" is not a valid literal. This is for disambiguation +with C-style octal literals, which Python used before version 3.0. + +Formally, integer literals are described by the following lexical +definitions: - integer: decinteger | bininteger | octinteger | hexinteger - decinteger: nonzerodigit (["_"] digit)* | "0"+ (["_"] "0")* + integer: decinteger | bininteger | octinteger | hexinteger | zerointeger + decinteger: nonzerodigit (["_"] digit)* bininteger: "0" ("b" | "B") (["_"] bindigit)+ octinteger: "0" ("o" | "O") (["_"] octdigit)+ hexinteger: "0" ("x" | "X") (["_"] hexdigit)+ + zerointeger: "0"+ (["_"] "0")* nonzerodigit: "1"..."9" digit: "0"..."9" bindigit: "0" | "1" octdigit: "0"..."7" hexdigit: digit | "a"..."f" | "A"..."F" -There is no limit for the length of integer literals apart from what -can be stored in available memory. - -Underscores are ignored for determining the numeric value of the -literal. They can be used to group digits for enhanced readability. -One underscore can occur between digits, and after base specifiers -like "0x". - -Note that leading zeros in a non-zero decimal number are not allowed. -This is for disambiguation with C-style octal literals, which Python -used before version 3.0. - -Some examples of integer literals: - - 7 2147483647 0o177 0b100110111 - 3 79228162514264337593543950336 0o377 0xdeadbeef - 100_000_000_000 0b_1110_0101 - Changed in version 3.6: Underscores are now allowed for grouping purposes in literals. ''', @@ -6730,76 +7229,251 @@ class body. A "SyntaxError" is raised if a variable is used or 'numbers': r'''Numeric literals **************** -There are three types of numeric literals: integers, floating-point -numbers, and imaginary numbers. There are no complex literals -(complex numbers can be formed by adding a real number and an -imaginary number). +"NUMBER" tokens represent numeric literals, of which there are three +types: integers, floating-point numbers, and imaginary numbers. -Note that numeric literals do not include a sign; a phrase like "-1" -is actually an expression composed of the unary operator ‘"-"’ and the -literal "1". -''', - 'numeric-types': r'''Emulating numeric types -*********************** + NUMBER: integer | floatnumber | imagnumber -The following methods can be defined to emulate numeric objects. -Methods corresponding to operations that are not supported by the -particular kind of number implemented (e.g., bitwise operations for -non-integral numbers) should be left undefined. +The numeric value of a numeric literal is the same as if it were +passed as a string to the "int", "float" or "complex" class +constructor, respectively. Note that not all valid inputs for those +constructors are also valid literals. -object.__add__(self, other) -object.__sub__(self, other) -object.__mul__(self, other) -object.__matmul__(self, other) -object.__truediv__(self, other) -object.__floordiv__(self, other) -object.__mod__(self, other) -object.__divmod__(self, other) -object.__pow__(self, other[, modulo]) -object.__lshift__(self, other) -object.__rshift__(self, other) -object.__and__(self, other) -object.__xor__(self, other) -object.__or__(self, other) +Numeric literals do not include a sign; a phrase like "-1" is actually +an expression composed of the unary operator ‘"-"’ and the literal +"1". - These methods are called to implement the binary arithmetic - operations ("+", "-", "*", "@", "/", "//", "%", "divmod()", - "pow()", "**", "<<", ">>", "&", "^", "|"). For instance, to - evaluate the expression "x + y", where *x* is an instance of a - class that has an "__add__()" method, "type(x).__add__(x, y)" is - called. The "__divmod__()" method should be the equivalent to - using "__floordiv__()" and "__mod__()"; it should not be related to - "__truediv__()". Note that "__pow__()" should be defined to accept - an optional third argument if the three-argument version of the - built-in "pow()" function is to be supported. - If one of those methods does not support the operation with the - supplied arguments, it should return "NotImplemented". +Integer literals +================ -object.__radd__(self, other) -object.__rsub__(self, other) -object.__rmul__(self, other) -object.__rmatmul__(self, other) -object.__rtruediv__(self, other) -object.__rfloordiv__(self, other) -object.__rmod__(self, other) -object.__rdivmod__(self, other) -object.__rpow__(self, other[, modulo]) -object.__rlshift__(self, other) -object.__rrshift__(self, other) -object.__rand__(self, other) -object.__rxor__(self, other) -object.__ror__(self, other) +Integer literals denote whole numbers. For example: - These methods are called to implement the binary arithmetic - operations ("+", "-", "*", "@", "/", "//", "%", "divmod()", - "pow()", "**", "<<", ">>", "&", "^", "|") with reflected (swapped) - operands. These functions are only called if the operands are of - different types, when the left operand does not support the - corresponding operation [3], or the right operand’s class is - derived from the left operand’s class. [4] For instance, to - evaluate the expression "x - y", where *y* is an instance of a - class that has an "__rsub__()" method, "type(y).__rsub__(y, x)" is + 7 + 3 + 2147483647 + +There is no limit for the length of integer literals apart from what +can be stored in available memory: + + 7922816251426433759354395033679228162514264337593543950336 + +Underscores can be used to group digits for enhanced readability, and +are ignored for determining the numeric value of the literal. For +example, the following literals are equivalent: + + 100_000_000_000 + 100000000000 + 1_00_00_00_00_000 + +Underscores can only occur between digits. For example, "_123", +"321_", and "123__321" are *not* valid literals. + +Integers can be specified in binary (base 2), octal (base 8), or +hexadecimal (base 16) using the prefixes "0b", "0o" and "0x", +respectively. Hexadecimal digits 10 through 15 are represented by +letters "A"-"F", case-insensitive. For example: + + 0b100110111 + 0b_1110_0101 + 0o177 + 0o377 + 0xdeadbeef + 0xDead_Beef + +An underscore can follow the base specifier. For example, "0x_1f" is a +valid literal, but "0_x1f" and "0x__1f" are not. + +Leading zeros in a non-zero decimal number are not allowed. For +example, "0123" is not a valid literal. This is for disambiguation +with C-style octal literals, which Python used before version 3.0. + +Formally, integer literals are described by the following lexical +definitions: + + integer: decinteger | bininteger | octinteger | hexinteger | zerointeger + decinteger: nonzerodigit (["_"] digit)* + bininteger: "0" ("b" | "B") (["_"] bindigit)+ + octinteger: "0" ("o" | "O") (["_"] octdigit)+ + hexinteger: "0" ("x" | "X") (["_"] hexdigit)+ + zerointeger: "0"+ (["_"] "0")* + nonzerodigit: "1"..."9" + digit: "0"..."9" + bindigit: "0" | "1" + octdigit: "0"..."7" + hexdigit: digit | "a"..."f" | "A"..."F" + +Changed in version 3.6: Underscores are now allowed for grouping +purposes in literals. + + +Floating-point literals +======================= + +Floating-point (float) literals, such as "3.14" or "1.5", denote +approximations of real numbers. + +They consist of *integer* and *fraction* parts, each composed of +decimal digits. The parts are separated by a decimal point, ".": + + 2.71828 + 4.0 + +Unlike in integer literals, leading zeros are allowed. For example, +"077.010" is legal, and denotes the same number as "77.01". + +As in integer literals, single underscores may occur between digits to +help readability: + + 96_485.332_123 + 3.14_15_93 + +Either of these parts, but not both, can be empty. For example: + + 10. # (equivalent to 10.0) + .001 # (equivalent to 0.001) + +Optionally, the integer and fraction may be followed by an *exponent*: +the letter "e" or "E", followed by an optional sign, "+" or "-", and a +number in the same format as the integer and fraction parts. The "e" +or "E" represents “times ten raised to the power of”: + + 1.0e3 # (represents 1.0×10³, or 1000.0) + 1.166e-5 # (represents 1.166×10⁻⁵, or 0.00001166) + 6.02214076e+23 # (represents 6.02214076×10²³, or 602214076000000000000000.) + +In floats with only integer and exponent parts, the decimal point may +be omitted: + + 1e3 # (equivalent to 1.e3 and 1.0e3) + 0e0 # (equivalent to 0.) + +Formally, floating-point literals are described by the following +lexical definitions: + + floatnumber: + | digitpart "." [digitpart] [exponent] + | "." digitpart [exponent] + | digitpart exponent + digitpart: digit (["_"] digit)* + exponent: ("e" | "E") ["+" | "-"] digitpart + +Changed in version 3.6: Underscores are now allowed for grouping +purposes in literals. + + +Imaginary literals +================== + +Python has complex number objects, but no complex literals. Instead, +*imaginary literals* denote complex numbers with a zero real part. + +For example, in math, the complex number 3+4.2*i* is written as the +real number 3 added to the imaginary number 4.2*i*. Python uses a +similar syntax, except the imaginary unit is written as "j" rather +than *i*: + + 3+4.2j + +This is an expression composed of the integer literal "3", the +operator ‘"+"’, and the imaginary literal "4.2j". Since these are +three separate tokens, whitespace is allowed between them: + + 3 + 4.2j + +No whitespace is allowed *within* each token. In particular, the "j" +suffix, may not be separated from the number before it. + +The number before the "j" has the same syntax as a floating-point +literal. Thus, the following are valid imaginary literals: + + 4.2j + 3.14j + 10.j + .001j + 1e100j + 3.14e-10j + 3.14_15_93j + +Unlike in a floating-point literal the decimal point can be omitted if +the imaginary number only has an integer part. The number is still +evaluated as a floating-point number, not an integer: + + 10j + 0j + 1000000000000000000000000j # equivalent to 1e+24j + +The "j" suffix is case-insensitive. That means you can use "J" +instead: + + 3.14J # equivalent to 3.14j + +Formally, imaginary literals are described by the following lexical +definition: + + imagnumber: (floatnumber | digitpart) ("j" | "J") +''', + 'numeric-types': r'''Emulating numeric types +*********************** + +The following methods can be defined to emulate numeric objects. +Methods corresponding to operations that are not supported by the +particular kind of number implemented (e.g., bitwise operations for +non-integral numbers) should be left undefined. + +object.__add__(self, other) +object.__sub__(self, other) +object.__mul__(self, other) +object.__matmul__(self, other) +object.__truediv__(self, other) +object.__floordiv__(self, other) +object.__mod__(self, other) +object.__divmod__(self, other) +object.__pow__(self, other[, modulo]) +object.__lshift__(self, other) +object.__rshift__(self, other) +object.__and__(self, other) +object.__xor__(self, other) +object.__or__(self, other) + + These methods are called to implement the binary arithmetic + operations ("+", "-", "*", "@", "/", "//", "%", "divmod()", + "pow()", "**", "<<", ">>", "&", "^", "|"). For instance, to + evaluate the expression "x + y", where *x* is an instance of a + class that has an "__add__()" method, "type(x).__add__(x, y)" is + called. The "__divmod__()" method should be the equivalent to + using "__floordiv__()" and "__mod__()"; it should not be related to + "__truediv__()". Note that "__pow__()" should be defined to accept + an optional third argument if the three-argument version of the + built-in "pow()" function is to be supported. + + If one of those methods does not support the operation with the + supplied arguments, it should return "NotImplemented". + +object.__radd__(self, other) +object.__rsub__(self, other) +object.__rmul__(self, other) +object.__rmatmul__(self, other) +object.__rtruediv__(self, other) +object.__rfloordiv__(self, other) +object.__rmod__(self, other) +object.__rdivmod__(self, other) +object.__rpow__(self, other[, modulo]) +object.__rlshift__(self, other) +object.__rrshift__(self, other) +object.__rand__(self, other) +object.__rxor__(self, other) +object.__ror__(self, other) + + These methods are called to implement the binary arithmetic + operations ("+", "-", "*", "@", "/", "//", "%", "divmod()", + "pow()", "**", "<<", ">>", "&", "^", "|") with reflected (swapped) + operands. These functions are only called if the operands are of + different types, when the left operand does not support the + corresponding operation [3], or the right operand’s class is + derived from the left operand’s class. [4] For instance, to + evaluate the expression "x - y", where *y* is an instance of a + class that has an "__rsub__()" method, "type(y).__rsub__(y, x)" is called if "type(x).__sub__(x, y)" returns "NotImplemented" or "type(y)" is a subclass of "type(x)". [5] @@ -6807,9 +7481,9 @@ class that has an "__rsub__()" method, "type(y).__rsub__(y, x)" is third argument if the three-argument version of the built-in "pow()" function is to be supported. - Changed in version 3.14.0a7 (unreleased): Three-argument "pow()" - now try calling "__rpow__()" if necessary. Previously it was only - called in two-argument "pow()" and the binary power operator. + Changed in version 3.14: Three-argument "pow()" now try calling + "__rpow__()" if necessary. Previously it was only called in two- + argument "pow()" and the binary power operator. Note: @@ -6894,9 +7568,8 @@ class that has an "__rsub__()" method, "type(y).__rsub__(y, x)" is ************************* *Objects* are Python’s abstraction for data. All data in a Python -program is represented by objects or by relations between objects. (In -a sense, and in conformance to Von Neumann’s model of a “stored -program computer”, code is also represented by objects.) +program is represented by objects or by relations between objects. +Even code is represented by objects. Every object has an identity, a type and a value. An object’s *identity* never changes once it has been created; you may think of it @@ -7002,8 +7675,8 @@ class that has an "__rsub__()" method, "type(y).__rsub__(y, x)" is | value...}", "{expressions...}" | list display, dictionary display, set | | | display | +-------------------------------------------------+---------------------------------------+ -| "x[index]", "x[index:index]", | Subscription, slicing, call, | -| "x(arguments...)", "x.attribute" | attribute reference | +| "x[index]", "x[index:index]" "x(arguments...)", | Subscription (including slicing), | +| "x.attribute" | call, attribute reference | +-------------------------------------------------+---------------------------------------+ | "await x" | Await expression | +-------------------------------------------------+---------------------------------------+ @@ -7120,8 +7793,8 @@ class C: pass # a class with no methods (yet) The power operator has the same semantics as the built-in "pow()" function, when called with two arguments: it yields its left argument -raised to the power of its right argument. The numeric arguments are -first converted to a common type, and the result is of that type. +raised to the power of its right argument. Numeric arguments are first +converted to a common type, and the result is of that type. For int operands, the result has the same type as the operands unless the second argument is negative; in that case, all arguments are @@ -7283,21 +7956,24 @@ class C: pass # a class with no methods (yet) The "collections.abc" module provides a "MutableMapping" *abstract base class* to help create those methods from a base set of "__getitem__()", "__setitem__()", "__delitem__()", and "keys()". -Mutable sequences should provide methods "append()", "count()", -"index()", "extend()", "insert()", "pop()", "remove()", "reverse()" -and "sort()", like Python standard "list" objects. Finally, sequence + +Mutable sequences should provide methods "append()", "clear()", +"count()", "extend()", "index()", "insert()", "pop()", "remove()", and +"reverse()", like Python standard "list" objects. Finally, sequence types should implement addition (meaning concatenation) and multiplication (meaning repetition) by defining the methods "__add__()", "__radd__()", "__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" described below; they should not define other numerical -operators. It is recommended that both mappings and sequences -implement the "__contains__()" method to allow efficient use of the -"in" operator; for mappings, "in" should search the mapping’s keys; -for sequences, it should search through the values. It is further -recommended that both mappings and sequences implement the -"__iter__()" method to allow efficient iteration through the -container; for mappings, "__iter__()" should iterate through the -object’s keys; for sequences, it should iterate through the values. +operators. + +It is recommended that both mappings and sequences implement the +"__contains__()" method to allow efficient use of the "in" operator; +for mappings, "in" should search the mapping’s keys; for sequences, it +should search through the values. It is further recommended that both +mappings and sequences implement the "__iter__()" method to allow +efficient iteration through the container; for mappings, "__iter__()" +should iterate through the object’s keys; for sequences, it should +iterate through the values. object.__len__(self) @@ -7324,35 +8000,46 @@ class C: pass # a class with no methods (yet) Added in version 3.4. -Note: +object.__getitem__(self, subscript) - Slicing is done exclusively with the following three methods. A - call like + Called to implement *subscription*, that is, "self[subscript]". See + Subscriptions and slicings for details on the syntax. - a[1:2] = b + There are two types of built-in objects that support subscription + via "__getitem__()": - is translated to + * **sequences**, where *subscript* (also called *index*) should be + an integer or a "slice" object. See the sequence documentation + for the expected behavior, including handling "slice" objects and + negative indices. - a[slice(1, 2, None)] = b + * **mappings**, where *subscript* is also called the *key*. See + mapping documentation for the expected behavior. - and so forth. Missing slice items are always filled in with "None". + If *subscript* is of an inappropriate type, "__getitem__()" should + raise "TypeError". If *subscript* has an inappropriate value, + "__getitem__()" should raise an "LookupError" or one of its + subclasses ("IndexError" for sequences; "KeyError" for mappings). -object.__getitem__(self, key) + Note: + + Slicing is handled by "__getitem__()", "__setitem__()", and + "__delitem__()". A call like + + a[1:2] = b + + is translated to - Called to implement evaluation of "self[key]". For *sequence* - types, the accepted keys should be integers. Optionally, they may - support "slice" objects as well. Negative index support is also - optional. If *key* is of an inappropriate type, "TypeError" may be - raised; if *key* is a value outside the set of indexes for the - sequence (after any special interpretation of negative values), - "IndexError" should be raised. For *mapping* types, if *key* is - missing (not in the container), "KeyError" should be raised. + a[slice(1, 2, None)] = b + + and so forth. Missing slice items are always filled in with + "None". Note: - "for" loops expect that an "IndexError" will be raised for - illegal indexes to allow proper detection of the end of the - sequence. + The sequence iteration protocol (used, for example, in "for" + loops), expects that an "IndexError" will be raised for illegal + indexes to allow proper detection of the end of a sequence. Note: @@ -7442,37 +8129,40 @@ class C: pass # a class with no methods (yet) 'slicings': r'''Slicings ******** -A slicing selects a range of items in a sequence object (e.g., a -string, tuple or list). Slicings may be used as expressions or as -targets in assignment or "del" statements. The syntax for a slicing: - - slicing: primary "[" slice_list "]" - slice_list: slice_item ("," slice_item)* [","] - slice_item: expression | proper_slice - proper_slice: [lower_bound] ":" [upper_bound] [ ":" [stride] ] - lower_bound: expression - upper_bound: expression - stride: expression - -There is ambiguity in the formal syntax here: anything that looks like -an expression list also looks like a slice list, so any subscription -can be interpreted as a slicing. Rather than further complicating the -syntax, this is disambiguated by defining that in this case the -interpretation as a subscription takes priority over the -interpretation as a slicing (this is the case if the slice list -contains no proper slice). - -The semantics for a slicing are as follows. The primary is indexed -(using the same "__getitem__()" method as normal subscription) with a -key that is constructed from the slice list, as follows. If the slice -list contains at least one comma, the key is a tuple containing the -conversion of the slice items; otherwise, the conversion of the lone -slice item is the key. The conversion of a slice item that is an -expression is that expression. The conversion of a proper slice is a -slice object (see section The standard type hierarchy) whose "start", -"stop" and "step" attributes are the values of the expressions given -as lower bound, upper bound and stride, respectively, substituting -"None" for missing expressions. +A more advanced form of subscription, *slicing*, is commonly used to +extract a portion of a sequence. In this form, the subscript is a +*slice*: up to three expressions separated by colons. Any of the +expressions may be omitted, but a slice must contain at least one +colon: + + >>> number_names = ['zero', 'one', 'two', 'three', 'four', 'five'] + >>> number_names[1:3] + ['one', 'two'] + >>> number_names[1:] + ['one', 'two', 'three', 'four', 'five'] + >>> number_names[:3] + ['zero', 'one', 'two'] + >>> number_names[:] + ['zero', 'one', 'two', 'three', 'four', 'five'] + >>> number_names[::2] + ['zero', 'two', 'four'] + >>> number_names[:-3] + ['zero', 'one', 'two'] + >>> del number_names[4:] + >>> number_names + ['zero', 'one', 'two', 'three'] + +When a slice is evaluated, the interpreter constructs a "slice" object +whose "start", "stop" and "step" attributes, respectively, are the +results of the expressions between the colons. Any missing expression +evaluates to "None". This "slice" object is then passed to the +"__getitem__()" or "__class_getitem__()" *special method*, as above. + + # continuing with the SubscriptionDemo instance defined above: + >>> demo[2:3] + subscripted with: slice(2, 3, None) + >>> demo[::'spam'] + subscripted with: slice(None, None, 'spam') ''', 'specialattrs': r'''Special Attributes ****************** @@ -7534,8 +8224,8 @@ class C: pass # a class with no methods (yet) important that the emulation only be implemented to the degree that it makes sense for the object being modelled. For example, some sequences may work well with retrieval of individual elements, but -extracting a slice may not make sense. (One example of this is the -"NodeList" interface in the W3C’s Document Object Model.) +extracting a slice may not make sense. (One example of this is the +NodeList interface in the W3C’s Document Object Model.) Basic customization @@ -7693,7 +8383,7 @@ class C: pass # a class with no methods (yet) formatting to one of the built-in types, or use a similar formatting option syntax. - See Format Specification Mini-Language for a description of the + See Format specification mini-language for a description of the standard formatting syntax. The return value must be a string object. @@ -7830,7 +8520,7 @@ def __hash__(self): intended to provide protection against a denial-of-service caused by carefully chosen inputs that exploit the worst case performance of a dict insertion, *O*(*n*^2) complexity. See - http://ocert.org/advisories/ocert-2011-003.html for + https://ocert.org/advisories/ocert-2011-003.html for details.Changing hash values affects the iteration order of sets. Python has never made guarantees about this ordering (and it typically varies between 32-bit and 64-bit builds).See also @@ -7930,6 +8620,9 @@ class instances. Customizing module attribute access ----------------------------------- +module.__getattr__() +module.__dir__() + Special names "__getattr__" and "__dir__" can be also used to customize access to module attributes. The "__getattr__" function at the module level should accept one argument which is the name of an @@ -7945,6 +8638,8 @@ class instances. present, this function overrides the standard "dir()" search on a module. +module.__class__ + For a more fine grained customization of the module behavior (setting attributes, properties, etc.), one can set the "__class__" attribute of a module object to a subclass of "types.ModuleType". For example: @@ -8640,21 +9335,24 @@ class of a class is known as that class’s *metaclass*, and most The "collections.abc" module provides a "MutableMapping" *abstract base class* to help create those methods from a base set of "__getitem__()", "__setitem__()", "__delitem__()", and "keys()". -Mutable sequences should provide methods "append()", "count()", -"index()", "extend()", "insert()", "pop()", "remove()", "reverse()" -and "sort()", like Python standard "list" objects. Finally, sequence + +Mutable sequences should provide methods "append()", "clear()", +"count()", "extend()", "index()", "insert()", "pop()", "remove()", and +"reverse()", like Python standard "list" objects. Finally, sequence types should implement addition (meaning concatenation) and multiplication (meaning repetition) by defining the methods "__add__()", "__radd__()", "__iadd__()", "__mul__()", "__rmul__()" and "__imul__()" described below; they should not define other numerical -operators. It is recommended that both mappings and sequences -implement the "__contains__()" method to allow efficient use of the -"in" operator; for mappings, "in" should search the mapping’s keys; -for sequences, it should search through the values. It is further -recommended that both mappings and sequences implement the -"__iter__()" method to allow efficient iteration through the -container; for mappings, "__iter__()" should iterate through the -object’s keys; for sequences, it should iterate through the values. +operators. + +It is recommended that both mappings and sequences implement the +"__contains__()" method to allow efficient use of the "in" operator; +for mappings, "in" should search the mapping’s keys; for sequences, it +should search through the values. It is further recommended that both +mappings and sequences implement the "__iter__()" method to allow +efficient iteration through the container; for mappings, "__iter__()" +should iterate through the object’s keys; for sequences, it should +iterate through the values. object.__len__(self) @@ -8681,35 +9379,46 @@ class of a class is known as that class’s *metaclass*, and most Added in version 3.4. -Note: +object.__getitem__(self, subscript) + + Called to implement *subscription*, that is, "self[subscript]". See + Subscriptions and slicings for details on the syntax. + + There are two types of built-in objects that support subscription + via "__getitem__()": - Slicing is done exclusively with the following three methods. A - call like + * **sequences**, where *subscript* (also called *index*) should be + an integer or a "slice" object. See the sequence documentation + for the expected behavior, including handling "slice" objects and + negative indices. - a[1:2] = b + * **mappings**, where *subscript* is also called the *key*. See + mapping documentation for the expected behavior. - is translated to + If *subscript* is of an inappropriate type, "__getitem__()" should + raise "TypeError". If *subscript* has an inappropriate value, + "__getitem__()" should raise an "LookupError" or one of its + subclasses ("IndexError" for sequences; "KeyError" for mappings). - a[slice(1, 2, None)] = b + Note: + + Slicing is handled by "__getitem__()", "__setitem__()", and + "__delitem__()". A call like - and so forth. Missing slice items are always filled in with "None". + a[1:2] = b -object.__getitem__(self, key) + is translated to - Called to implement evaluation of "self[key]". For *sequence* - types, the accepted keys should be integers. Optionally, they may - support "slice" objects as well. Negative index support is also - optional. If *key* is of an inappropriate type, "TypeError" may be - raised; if *key* is a value outside the set of indexes for the - sequence (after any special interpretation of negative values), - "IndexError" should be raised. For *mapping* types, if *key* is - missing (not in the container), "KeyError" should be raised. + a[slice(1, 2, None)] = b + + and so forth. Missing slice items are always filled in with + "None". Note: - "for" loops expect that an "IndexError" will be raised for - illegal indexes to allow proper detection of the end of the - sequence. + The sequence iteration protocol (used, for example, in "for" + loops), expects that an "IndexError" will be raised for illegal + indexes to allow proper detection of the end of a sequence. Note: @@ -8845,9 +9554,9 @@ class that has an "__rsub__()" method, "type(y).__rsub__(y, x)" is third argument if the three-argument version of the built-in "pow()" function is to be supported. - Changed in version 3.14.0a7 (unreleased): Three-argument "pow()" - now try calling "__rpow__()" if necessary. Previously it was only - called in two-argument "pow()" and the binary power operator. + Changed in version 3.14: Three-argument "pow()" now try calling + "__rpow__()" if necessary. Previously it was only called in two- + argument "pow()" and the binary power operator. Note: @@ -9027,14 +9736,27 @@ class is used in a class pattern with positional arguments, each "inspect.BufferFlags" provides a convenient way to interpret the flags. The method must return a "memoryview" object. + **Thread safety:** In *free-threaded* Python, implementations must + manage any internal export counter using atomic operations. The + method must be safe to call concurrently from multiple threads, and + the returned buffer’s underlying data must remain valid until the + corresponding "__release_buffer__()" call completes. See Thread + safety for memoryview objects for details. + object.__release_buffer__(self, buffer) Called when a buffer is no longer needed. The *buffer* argument is a "memoryview" object that was previously returned by "__buffer__()". The method must release any resources associated - with the buffer. This method should return "None". Buffer objects - that do not need to perform any cleanup are not required to - implement this method. + with the buffer. This method should return "None". + + **Thread safety:** In *free-threaded* Python, any export counter + decrement must use atomic operations. Resource cleanup must be + thread-safe, as the final release may race with concurrent releases + from other threads. + + Buffer objects that do not need to perform any cleanup are not + required to implement this method. Added in version 3.12. @@ -9175,7 +9897,7 @@ class is used in a class pattern with positional arguments, each Strings also support two styles of string formatting, one providing a large degree of flexibility and customization (see "str.format()", -Format String Syntax and Custom String Formatting) and the other based +Format string syntax and Custom string formatting) and the other based on C "printf" style formatting that handles a narrower range of types and is slightly harder to use correctly, but is often faster for the cases it can handle (printf-style String Formatting). @@ -9203,19 +9925,31 @@ class is used in a class pattern with positional arguments, each it is intended to remove all case distinctions in a string. For example, the German lowercase letter "'ß'" is equivalent to ""ss"". Since it is already lowercase, "lower()" would do nothing to "'ß'"; - "casefold()" converts it to ""ss"". + "casefold()" converts it to ""ss"". For example: + + >>> 'straße'.lower() + 'straße' + >>> 'straße'.casefold() + 'strasse' The casefolding algorithm is described in section 3.13 ‘Default Case Folding’ of the Unicode Standard. Added in version 3.3. -str.center(width[, fillchar]) +str.center(width, fillchar=' ', /) Return centered in a string of length *width*. Padding is done using the specified *fillchar* (default is an ASCII space). The original string is returned if *width* is less than or equal to - "len(s)". + "len(s)". For example: + + >>> 'Python'.center(10) + ' Python ' + >>> 'Python'.center(10, '-') + '--Python--' + >>> 'Python'.center(4) + 'Python' str.count(sub[, start[, end]]) @@ -9224,7 +9958,18 @@ class is used in a class pattern with positional arguments, each *end* are interpreted as in slice notation. If *sub* is empty, returns the number of empty strings between - characters which is the length of the string plus one. + characters which is the length of the string plus one. For example: + + >>> 'spam, spam, spam'.count('spam') + 3 + >>> 'spam, spam, spam'.count('spam', 5) + 2 + >>> 'spam, spam, spam'.count('spam', 5, 10) + 1 + >>> 'spam, spam, spam'.count('eggs') + 0 + >>> 'spam, spam, spam'.count('') + 17 str.encode(encoding='utf-8', errors='strict') @@ -9241,7 +9986,13 @@ class is used in a class pattern with positional arguments, each For performance reasons, the value of *errors* is not checked for validity unless an encoding error actually occurs, Python - Development Mode is enabled or a debug build is used. + Development Mode is enabled or a debug build is used. For example: + + >>> encoded_str_to_bytes = 'Python'.encode() + >>> type(encoded_str_to_bytes) + + >>> encoded_str_to_bytes + b'Python' Changed in version 3.1: Added support for keyword arguments. @@ -9254,6 +10005,19 @@ class is used in a class pattern with positional arguments, each otherwise return "False". *suffix* can also be a tuple of suffixes to look for. With optional *start*, test beginning at that position. With optional *end*, stop comparing at that position. + Using *start* and *end* is equivalent to + "str[start:end].endswith(suffix)". For example: + + >>> 'Python'.endswith('on') + True + >>> 'a tuple of suffixes'.endswith(('at', 'in')) + False + >>> 'a tuple of suffixes'.endswith(('at', 'es')) + True + >>> 'Python is amazing'.endswith('is', 0, 9) + True + + See also "startswith()" and "removesuffix()". str.expandtabs(tabsize=8) @@ -9269,19 +10033,29 @@ class is used in a class pattern with positional arguments, each ("\n") or return ("\r"), it is copied and the current column is reset to zero. Any other character is copied unchanged and the current column is incremented by one regardless of how the - character is represented when printed. + character is represented when printed. For example: - >>> '01\t012\t0123\t01234'.expandtabs() - '01 012 0123 01234' - >>> '01\t012\t0123\t01234'.expandtabs(4) - '01 012 0123 01234' + >>> '01\t012\t0123\t01234'.expandtabs() + '01 012 0123 01234' + >>> '01\t012\t0123\t01234'.expandtabs(4) + '01 012 0123 01234' + >>> print('01\t012\n0123\t01234'.expandtabs(4)) + 01 012 + 0123 01234 str.find(sub[, start[, end]]) Return the lowest index in the string where substring *sub* is found within the slice "s[start:end]". Optional arguments *start* and *end* are interpreted as in slice notation. Return "-1" if - *sub* is not found. + *sub* is not found. For example: + + >>> 'spam, spam, spam'.find('sp') + 0 + >>> 'spam, spam, spam'.find('sp', 5) + 6 + + See also "rfind()" and "index()". Note: @@ -9300,12 +10074,16 @@ class is used in a class pattern with positional arguments, each the numeric index of a positional argument, or the name of a keyword argument. Returns a copy of the string where each replacement field is replaced with the string value of the - corresponding argument. + corresponding argument. For example: - >>> "The sum of 1 + 2 is {0}".format(1+2) - 'The sum of 1 + 2 is 3' + >>> "The sum of 1 + 2 is {0}".format(1+2) + 'The sum of 1 + 2 is 3' + >>> "The sum of {a} + {b} is {answer}".format(answer=1+2, a=1, b=2) + 'The sum of 1 + 2 is 3' + >>> "{1} expects the {0} Inquisition!".format("Spanish", "Nobody") + 'Nobody expects the Spanish Inquisition!' - See Format String Syntax for a description of the various + See Format string syntax for a description of the various formatting options that can be specified in format strings. Note: @@ -9341,7 +10119,18 @@ class is used in a class pattern with positional arguments, each str.index(sub[, start[, end]]) Like "find()", but raise "ValueError" when the substring is not - found. + found. For example: + + >>> 'spam, spam, spam'.index('spam') + 0 + >>> 'spam, spam, spam'.index('eggs') + Traceback (most recent call last): + File "", line 1, in + 'spam, spam, spam'.index('eggs') + ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ + ValueError: substring not found + + See also "rindex()". str.isalnum() @@ -9349,6 +10138,16 @@ class is used in a class pattern with positional arguments, each there is at least one character, "False" otherwise. A character "c" is alphanumeric if one of the following returns "True": "c.isalpha()", "c.isdecimal()", "c.isdigit()", or "c.isnumeric()". + For example: + + >>> 'abc123'.isalnum() + True + >>> 'abc123!@#'.isalnum() + False + >>> ''.isalnum() + False + >>> ' '.isalnum() + False str.isalpha() @@ -9358,13 +10157,28 @@ class is used in a class pattern with positional arguments, each database as “Letter”, i.e., those with general category property being one of “Lm”, “Lt”, “Lu”, “Ll”, or “Lo”. Note that this is different from the Alphabetic property defined in the section 4.10 - ‘Letters, Alphabetic, and Ideographic’ of the Unicode Standard. + ‘Letters, Alphabetic, and Ideographic’ of the Unicode Standard. For + example: + + >>> 'Letters and spaces'.isalpha() + False + >>> 'LettersOnly'.isalpha() + True + >>> 'µ'.isalpha() # non-ASCII characters can be considered alphabetical too + True + + See Unicode Properties. str.isascii() Return "True" if the string is empty or all characters in the string are ASCII, "False" otherwise. ASCII characters have code - points in the range U+0000-U+007F. + points in the range U+0000-U+007F. For example: + + >>> 'ASCII characters'.isascii() + True + >>> 'µ'.isascii() + False Added in version 3.7. @@ -9373,8 +10187,16 @@ class is used in a class pattern with positional arguments, each Return "True" if all characters in the string are decimal characters and there is at least one character, "False" otherwise. Decimal characters are those that can be used to form numbers in - base 10, e.g. U+0660, ARABIC-INDIC DIGIT ZERO. Formally a decimal - character is a character in the Unicode General Category “Nd”. + base 10, such as U+0660, ARABIC-INDIC DIGIT ZERO. Formally a + decimal character is a character in the Unicode General Category + “Nd”. For example: + + >>> '0123456789'.isdecimal() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isdecimal() # Arabic-Indic digits zero to nine + True + >>> 'alphabetic'.isdecimal() + False str.isdigit() @@ -9383,13 +10205,26 @@ class is used in a class pattern with positional arguments, each decimal characters and digits that need special handling, such as the compatibility superscript digits. This covers digits which cannot be used to form numbers in base 10, like the Kharosthi - numbers. Formally, a digit is a character that has the property + numbers. Formally, a digit is a character that has the property value Numeric_Type=Digit or Numeric_Type=Decimal. + For example: + + >>> '0123456789'.isdigit() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isdigit() # Arabic-Indic digits zero to nine + True + >>> '⅕'.isdigit() # Vulgar fraction one fifth + False + >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() + (False, True, True) + + See also "isdecimal()" and "isnumeric()". + str.isidentifier() Return "True" if the string is a valid identifier according to the - language definition, section Identifiers and keywords. + language definition, section Names (identifiers and keywords). "keyword.iskeyword()" can be used to test whether string "s" is a reserved identifier, such as "def" and "class". @@ -9417,12 +10252,23 @@ class is used in a class pattern with positional arguments, each that have the Unicode numeric value property, e.g. U+2155, VULGAR FRACTION ONE FIFTH. Formally, numeric characters are those with the property value Numeric_Type=Digit, Numeric_Type=Decimal or - Numeric_Type=Numeric. + Numeric_Type=Numeric. For example: + + >>> '0123456789'.isnumeric() + True + >>> '٠١٢٣٤٥٦٧٨٩'.isnumeric() # Arabic-Indic digits zero to nine + True + >>> '⅕'.isnumeric() # Vulgar fraction one fifth + True + >>> '²'.isdecimal(), '²'.isdigit(), '²'.isnumeric() + (False, True, True) + + See also "isdecimal()" and "isdigit()". str.isprintable() - Return true if all characters in the string are printable, false if - it contains at least one non-printable character. + Return "True" if all characters in the string are printable, + "False" if it contains at least one non-printable character. Here “printable” means the character is suitable for "repr()" to use in its output; “non-printable” means that "repr()" on built-in @@ -9435,16 +10281,38 @@ class is used in a class pattern with positional arguments, each plus the ASCII space 0x20. Nonprintable characters are those in group Separator or Other (Z or C), except the ASCII space. + For example: + + >>> ''.isprintable(), ' '.isprintable() + (True, True) + >>> '\t'.isprintable(), '\n'.isprintable() + (False, False) + + See also "isspace()". + str.isspace() Return "True" if there are only whitespace characters in the string and there is at least one character, "False" otherwise. + For example: + + >>> ''.isspace() + False + >>> ' '.isspace() + True + >>> '\t\n'.isspace() # TAB and BREAK LINE + True + >>> '\u3000'.isspace() # IDEOGRAPHIC SPACE + True + A character is *whitespace* if in the Unicode character database (see "unicodedata"), either its general category is "Zs" (“Separator, space”), or its bidirectional class is one of "WS", "B", or "S". + See also "isprintable()". + str.istitle() Return "True" if the string is a titlecased string and there is at @@ -9452,6 +10320,17 @@ class is used in a class pattern with positional arguments, each follow uncased characters and lowercase characters only cased ones. Return "False" otherwise. + For example: + + >>> 'Spam, Spam, Spam'.istitle() + True + >>> 'spam, spam, spam'.istitle() + False + >>> 'SPAM, SPAM, SPAM'.istitle() + False + + See also "title()". + str.isupper() Return "True" if all cased characters [4] in the string are @@ -9467,29 +10346,51 @@ class is used in a class pattern with positional arguments, each >>> ' '.isupper() False -str.join(iterable) +str.join(iterable, /) Return a string which is the concatenation of the strings in *iterable*. A "TypeError" will be raised if there are any non- string values in *iterable*, including "bytes" objects. The - separator between elements is the string providing this method. + separator between elements is the string providing this method. For + example: + + >>> ', '.join(['spam', 'spam', 'spam']) + 'spam, spam, spam' + >>> '-'.join('Python') + 'P-y-t-h-o-n' + + See also "split()". -str.ljust(width[, fillchar]) +str.ljust(width, fillchar=' ', /) Return the string left justified in a string of length *width*. Padding is done using the specified *fillchar* (default is an ASCII space). The original string is returned if *width* is less than or equal to "len(s)". + For example: + + >>> 'Python'.ljust(10) + 'Python ' + >>> 'Python'.ljust(10, '.') + 'Python....' + >>> 'Monty Python'.ljust(10, '.') + 'Monty Python' + + See also "rjust()". + str.lower() Return a copy of the string with all the cased characters [4] - converted to lowercase. + converted to lowercase. For example: + + >>> 'Lower Method Example'.lower() + 'lower method example' The lowercasing algorithm used is described in section 3.13 ‘Default Case Folding’ of the Unicode Standard. -str.lstrip([chars]) +str.lstrip(chars=None, /) Return a copy of the string with leading characters removed. The *chars* argument is a string specifying the set of characters to be @@ -9510,7 +10411,8 @@ class is used in a class pattern with positional arguments, each >>> 'Arthur: three!'.removeprefix('Arthur: ') 'three!' -static str.maketrans(x[, y[, z]]) +static str.maketrans(dict, /) +static str.maketrans(from, to, remove='', /) This static method returns a translation table usable for "str.translate()". @@ -9521,12 +10423,12 @@ class is used in a class pattern with positional arguments, each Character keys will then be converted to ordinals. If there are two arguments, they must be strings of equal length, - and in the resulting dictionary, each character in x will be mapped - to the character at the same position in y. If there is a third - argument, it must be a string, whose characters will be mapped to - "None" in the result. + and in the resulting dictionary, each character in *from* will be + mapped to the character at the same position in *to*. If there is + a third argument, it must be a string, whose characters will be + mapped to "None" in the result. -str.partition(sep) +str.partition(sep, /) Split the string at the first occurrence of *sep*, and return a 3-tuple containing the part before the separator, the separator @@ -9534,6 +10436,17 @@ class is used in a class pattern with positional arguments, each found, return a 3-tuple containing the string itself, followed by two empty strings. + For example: + + >>> 'Monty Python'.partition(' ') + ('Monty', ' ', 'Python') + >>> "Monty Python's Flying Circus".partition(' ') + ('Monty', ' ', "Python's Flying Circus") + >>> 'Monty Python'.partition('-') + ('Monty Python', '', '') + + See also "rpartition()". + str.removeprefix(prefix, /) If the string starts with the *prefix* string, return @@ -9547,6 +10460,8 @@ class is used in a class pattern with positional arguments, each Added in version 3.9. + See also "removesuffix()" and "startswith()". + str.removesuffix(suffix, /) If the string ends with the *suffix* string and that *suffix* is @@ -9560,12 +10475,19 @@ class is used in a class pattern with positional arguments, each Added in version 3.9. -str.replace(old, new, count=-1) + See also "removeprefix()" and "endswith()". + +str.replace(old, new, /, count=-1) Return a copy of the string with all occurrences of substring *old* replaced by *new*. If *count* is given, only the first *count* occurrences are replaced. If *count* is not specified or "-1", then - all occurrences are replaced. + all occurrences are replaced. For example: + + >>> 'spam, spam, spam'.replace('spam', 'eggs') + 'eggs, eggs, eggs' + >>> 'spam, spam, spam'.replace('spam', 'eggs', 1) + 'eggs, spam, spam' Changed in version 3.13: *count* is now supported as a keyword argument. @@ -9575,21 +10497,50 @@ class is used in a class pattern with positional arguments, each Return the highest index in the string where substring *sub* is found, such that *sub* is contained within "s[start:end]". Optional arguments *start* and *end* are interpreted as in slice - notation. Return "-1" on failure. + notation. Return "-1" on failure. For example: + + >>> 'spam, spam, spam'.rfind('sp') + 12 + >>> 'spam, spam, spam'.rfind('sp', 0, 10) + 6 + + See also "find()" and "rindex()". str.rindex(sub[, start[, end]]) Like "rfind()" but raises "ValueError" when the substring *sub* is - not found. + not found. For example: + + >>> 'spam, spam, spam'.rindex('spam') + 12 + >>> 'spam, spam, spam'.rindex('eggs') + Traceback (most recent call last): + File "", line 1, in + 'spam, spam, spam'.rindex('eggs') + ~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^ + ValueError: substring not found + + See also "index()" and "find()". -str.rjust(width[, fillchar]) +str.rjust(width, fillchar=' ', /) Return the string right justified in a string of length *width*. Padding is done using the specified *fillchar* (default is an ASCII space). The original string is returned if *width* is less than or equal to "len(s)". -str.rpartition(sep) + For example: + + >>> 'Python'.rjust(10) + ' Python' + >>> 'Python'.rjust(10, '.') + '....Python' + >>> 'Monty Python'.rjust(10, '.') + 'Monty Python' + + See also "ljust()" and "zfill()". + +str.rpartition(sep, /) Split the string at the last occurrence of *sep*, and return a 3-tuple containing the part before the separator, the separator @@ -9597,6 +10548,17 @@ class is used in a class pattern with positional arguments, each found, return a 3-tuple containing two empty strings, followed by the string itself. + For example: + + >>> 'Monty Python'.rpartition(' ') + ('Monty', ' ', 'Python') + >>> "Monty Python's Flying Circus".rpartition(' ') + ("Monty Python's Flying", ' ', 'Circus') + >>> 'Monty Python'.rpartition('-') + ('', '', 'Monty Python') + + See also "partition()". + str.rsplit(sep=None, maxsplit=-1) Return a list of the words in the string, using *sep* as the @@ -9606,27 +10568,29 @@ class is used in a class pattern with positional arguments, each from the right, "rsplit()" behaves like "split()" which is described in detail below. -str.rstrip([chars]) +str.rstrip(chars=None, /) Return a copy of the string with trailing characters removed. The *chars* argument is a string specifying the set of characters to be removed. If omitted or "None", the *chars* argument defaults to removing whitespace. The *chars* argument is not a suffix; rather, - all combinations of its values are stripped: + all combinations of its values are stripped. For example: >>> ' spacious '.rstrip() ' spacious' >>> 'mississippi'.rstrip('ipz') 'mississ' - See "str.removesuffix()" for a method that will remove a single - suffix string rather than all of a set of characters. For example: + See "removesuffix()" for a method that will remove a single suffix + string rather than all of a set of characters. For example: >>> 'Monty Python'.rstrip(' Python') 'M' >>> 'Monty Python'.removesuffix(' Python') 'Monty' + See also "strip()". + str.split(sep=None, maxsplit=-1) Return a list of the words in the string, using *sep* as the @@ -9669,6 +10633,20 @@ class is used in a class pattern with positional arguments, each >>> ' 1 2 3 '.split() ['1', '2', '3'] + If *sep* is not specified or is "None" and *maxsplit* is "0", only + leading runs of consecutive whitespace are considered. + + For example: + + >>> "".split(None, 0) + [] + >>> " ".split(None, 0) + [] + >>> " foo ".split(maxsplit=0) + ['foo '] + + See also "join()" and "rsplit()". + str.splitlines(keepends=False) Return a list of the lines in the string, breaking at line @@ -9737,14 +10715,29 @@ class is used in a class pattern with positional arguments, each With optional *start*, test string beginning at that position. With optional *end*, stop comparing string at that position. -str.strip([chars]) + For example: + + >>> 'Python'.startswith('Py') + True + >>> 'a tuple of prefixes'.startswith(('at', 'a')) + True + >>> 'Python is amazing'.startswith('is', 7) + True + + See also "endswith()" and "removeprefix()". + +str.strip(chars=None, /) Return a copy of the string with the leading and trailing characters removed. The *chars* argument is a string specifying the set of characters to be removed. If omitted or "None", the *chars* argument defaults to removing whitespace. The *chars* argument is not a prefix or suffix; rather, all combinations of its values are - stripped: + stripped. + + Whitespace characters are defined by "str.isspace()". + + For example: >>> ' spacious '.strip() 'spacious' @@ -9755,17 +10748,31 @@ class is used in a class pattern with positional arguments, each stripped from the string. Characters are removed from the leading end until reaching a string character that is not contained in the set of characters in *chars*. A similar action takes place on the - trailing end. For example: + trailing end. + + For example: >>> comment_string = '#....... Section 3.2.1 Issue #32 .......' >>> comment_string.strip('.#! ') 'Section 3.2.1 Issue #32' + See also "rstrip()". + str.swapcase() Return a copy of the string with uppercase characters converted to - lowercase and vice versa. Note that it is not necessarily true that - "s.swapcase().swapcase() == s". + lowercase and vice versa. For example: + + >>> 'Hello World'.swapcase() + 'hELLO wORLD' + + Note that it is not necessarily true that "s.swapcase().swapcase() + == s". For example: + + >>> 'straße'.swapcase().swapcase() + 'strasse' + + See also "str.lower()" and "str.upper()". str.title() @@ -9801,7 +10808,9 @@ class is used in a class pattern with positional arguments, each >>> titlecase("they're bill's friends.") "They're Bill's Friends." -str.translate(table) + See also "istitle()". + +str.translate(table, /) Return a copy of the string in which each character has been mapped through the given translation table. The table must be an object @@ -9829,7 +10838,7 @@ class is used in a class pattern with positional arguments, each The uppercasing algorithm used is described in section 3.13 ‘Default Case Folding’ of the Unicode Standard. -str.zfill(width) +str.zfill(width, /) Return a copy of the string left filled with ASCII "'0'" digits to make a string of length *width*. A leading sign prefix @@ -9843,178 +10852,319 @@ class is used in a class pattern with positional arguments, each '00042' >>> "-42".zfill(5) '-0042' + + See also "rjust()". ''', 'strings': '''String and Bytes literals ************************* -String literals are described by the following lexical definitions: - - stringliteral: [stringprefix](shortstring | longstring) - stringprefix: "r" | "u" | "R" | "U" | "f" | "F" - | "fr" | "Fr" | "fR" | "FR" | "rf" | "rF" | "Rf" | "RF" - shortstring: "'" shortstringitem* "'" | '"' shortstringitem* '"' - longstring: "\'\'\'" longstringitem* "\'\'\'" | '"""' longstringitem* '"""' - shortstringitem: shortstringchar | stringescapeseq - longstringitem: longstringchar | stringescapeseq - shortstringchar: - longstringchar: - stringescapeseq: "\\" - - bytesliteral: bytesprefix(shortbytes | longbytes) - bytesprefix: "b" | "B" | "br" | "Br" | "bR" | "BR" | "rb" | "rB" | "Rb" | "RB" - shortbytes: "'" shortbytesitem* "'" | '"' shortbytesitem* '"' - longbytes: "\'\'\'" longbytesitem* "\'\'\'" | '"""' longbytesitem* '"""' - shortbytesitem: shortbyteschar | bytesescapeseq - longbytesitem: longbyteschar | bytesescapeseq - shortbyteschar: - longbyteschar: - bytesescapeseq: "\\" - -One syntactic restriction not indicated by these productions is that -whitespace is not allowed between the "stringprefix" or "bytesprefix" -and the rest of the literal. The source character set is defined by -the encoding declaration; it is UTF-8 if no encoding declaration is -given in the source file; see section Encoding declarations. - -In plain English: Both types of literals can be enclosed in matching -single quotes ("'") or double quotes ("""). They can also be enclosed -in matching groups of three single or double quotes (these are -generally referred to as *triple-quoted strings*). The backslash ("\\") -character is used to give special meaning to otherwise ordinary -characters like "n", which means ‘newline’ when escaped ("\\n"). It can -also be used to escape characters that otherwise have a special -meaning, such as newline, backslash itself, or the quote character. -See escape sequences below for examples. - -Bytes literals are always prefixed with "'b'" or "'B'"; they produce -an instance of the "bytes" type instead of the "str" type. They may -only contain ASCII characters; bytes with a numeric value of 128 or -greater must be expressed with escapes. +String literals are text enclosed in single quotes ("'") or double +quotes ("""). For example: -Both string and bytes literals may optionally be prefixed with a -letter "'r'" or "'R'"; such constructs are called *raw string -literals* and *raw bytes literals* respectively and treat backslashes -as literal characters. As a result, in raw string literals, "'\\U'" -and "'\\u'" escapes are not treated specially. + "spam" + 'eggs' -Added in version 3.3: The "'rb'" prefix of raw bytes literals has been +The quote used to start the literal also terminates it, so a string +literal can only contain the other quote (except with escape +sequences, see below). For example: + + 'Say "Hello", please.' + "Don't do that!" + +Except for this limitation, the choice of quote character ("'" or """) +does not affect how the literal is parsed. + +Inside a string literal, the backslash ("\\") character introduces an +*escape sequence*, which has special meaning depending on the +character after the backslash. For example, "\\"" denotes the double +quote character, and does *not* end the string: + + >>> print("Say \\"Hello\\" to everyone!") + Say "Hello" to everyone! + +See escape sequences below for a full list of such sequences, and more +details. + + +Triple-quoted strings +===================== + +Strings can also be enclosed in matching groups of three single or +double quotes. These are generally referred to as *triple-quoted +strings*: + + """This is a triple-quoted string.""" + +In triple-quoted literals, unescaped quotes are allowed (and are +retained), except that three unescaped quotes in a row terminate the +literal, if they are of the same kind ("'" or """) used at the start: + + """This string has "quotes" inside.""" + +Unescaped newlines are also allowed and retained: + + \'\'\'This triple-quoted string + continues on the next line.\'\'\' + + +String prefixes +=============== + +String literals can have an optional *prefix* that influences how the +content of the literal is parsed, for example: + + b"data" + f'{result=}' + +The allowed prefixes are: + +* "b": Bytes literal + +* "r": Raw string + +* "f": Formatted string literal (“f-string”) + +* "t": Template string literal (“t-string”) + +* "u": No effect (allowed for backwards compatibility) + +See the linked sections for details on each type. + +Prefixes are case-insensitive (for example, ‘"B"’ works the same as +‘"b"’). The ‘"r"’ prefix can be combined with ‘"f"’, ‘"t"’ or ‘"b"’, +so ‘"fr"’, ‘"rf"’, ‘"tr"’, ‘"rt"’, ‘"br"’, and ‘"rb"’ are also valid +prefixes. + +Added in version 3.3: The "'rb'" prefix of raw bytes literals has been added as a synonym of "'br'".Support for the unicode legacy literal ("u'value'") was reintroduced to simplify the maintenance of dual Python 2.x and 3.x codebases. See **PEP 414** for more information. -A string literal with "'f'" or "'F'" in its prefix is a *formatted -string literal*; see f-strings. The "'f'" may be combined with "'r'", -but not with "'b'" or "'u'", therefore raw formatted strings are -possible, but formatted bytes literals are not. -In triple-quoted literals, unescaped newlines and quotes are allowed -(and are retained), except that three unescaped quotes in a row -terminate the literal. (A “quote” is the character used to open the -literal, i.e. either "'" or """.) +Formal grammar +============== + +String literals, except “f-strings” and “t-strings”, are described by +the following lexical definitions. + +These definitions use negative lookaheads ("!") to indicate that an +ending quote ends the literal. + + STRING: [stringprefix] (stringcontent) + stringprefix: <("r" | "u" | "b" | "br" | "rb"), case-insensitive> + stringcontent: + | "\'\'\'" ( !"\'\'\'" longstringitem)* "\'\'\'" + | '"""' ( !'"""' longstringitem)* '"""' + | "'" ( !"'" stringitem)* "'" + | '"' ( !'"' stringitem)* '"' + stringitem: stringchar | stringescapeseq + stringchar: + longstringitem: stringitem | newline + stringescapeseq: "\\" + +Note that as in all lexical definitions, whitespace is significant. In +particular, the prefix (if any) must be immediately followed by the +starting quote. Escape sequences ================ -Unless an "'r'" or "'R'" prefix is present, escape sequences in string +Unless an ‘"r"’ or ‘"R"’ prefix is present, escape sequences in string and bytes literals are interpreted according to rules similar to those used by Standard C. The recognized escape sequences are: -+---------------------------+-----------------------------------+---------+ -| Escape Sequence | Meaning | Notes | -|===========================|===================================|=========| -| "\\" | Backslash and newline ignored | (1) | -+---------------------------+-----------------------------------+---------+ -| "\\\\" | Backslash ("\\") | | -+---------------------------+-----------------------------------+---------+ -| "\\'" | Single quote ("'") | | -+---------------------------+-----------------------------------+---------+ -| "\\"" | Double quote (""") | | -+---------------------------+-----------------------------------+---------+ -| "\\a" | ASCII Bell (BEL) | | -+---------------------------+-----------------------------------+---------+ -| "\\b" | ASCII Backspace (BS) | | -+---------------------------+-----------------------------------+---------+ -| "\\f" | ASCII Formfeed (FF) | | -+---------------------------+-----------------------------------+---------+ -| "\\n" | ASCII Linefeed (LF) | | -+---------------------------+-----------------------------------+---------+ -| "\\r" | ASCII Carriage Return (CR) | | -+---------------------------+-----------------------------------+---------+ -| "\\t" | ASCII Horizontal Tab (TAB) | | -+---------------------------+-----------------------------------+---------+ -| "\\v" | ASCII Vertical Tab (VT) | | -+---------------------------+-----------------------------------+---------+ -| "\\*ooo*" | Character with octal value *ooo* | (2,4) | -+---------------------------+-----------------------------------+---------+ -| "\\x*hh*" | Character with hex value *hh* | (3,4) | -+---------------------------+-----------------------------------+---------+ - -Escape sequences only recognized in string literals are: - -+---------------------------+-----------------------------------+---------+ -| Escape Sequence | Meaning | Notes | -|===========================|===================================|=========| -| "\\N{*name*}" | Character named *name* in the | (5) | -| | Unicode database | | -+---------------------------+-----------------------------------+---------+ -| "\\u*xxxx*" | Character with 16-bit hex value | (6) | -| | *xxxx* | | -+---------------------------+-----------------------------------+---------+ -| "\\U*xxxxxxxx*" | Character with 32-bit hex value | (7) | -| | *xxxxxxxx* | | -+---------------------------+-----------------------------------+---------+ ++----------------------------------------------------+----------------------------------------------------+ +| Escape Sequence | Meaning | +|====================================================|====================================================| +| "\\" | Ignored end of line | ++----------------------------------------------------+----------------------------------------------------+ +| "\\\\" | Backslash | ++----------------------------------------------------+----------------------------------------------------+ +| "\\'" | Single quote | ++----------------------------------------------------+----------------------------------------------------+ +| "\\"" | Double quote | ++----------------------------------------------------+----------------------------------------------------+ +| "\\a" | ASCII Bell (BEL) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\b" | ASCII Backspace (BS) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\f" | ASCII Formfeed (FF) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\n" | ASCII Linefeed (LF) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\r" | ASCII Carriage Return (CR) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\t" | ASCII Horizontal Tab (TAB) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\v" | ASCII Vertical Tab (VT) | ++----------------------------------------------------+----------------------------------------------------+ +| "\\*ooo*" | Octal character | ++----------------------------------------------------+----------------------------------------------------+ +| "\\x*hh*" | Hexadecimal character | ++----------------------------------------------------+----------------------------------------------------+ +| "\\N{*name*}" | Named Unicode character | ++----------------------------------------------------+----------------------------------------------------+ +| "\\u*xxxx*" | Hexadecimal Unicode character | ++----------------------------------------------------+----------------------------------------------------+ +| "\\U*xxxxxxxx*" | Hexadecimal Unicode character | ++----------------------------------------------------+----------------------------------------------------+ -Notes: -1. A backslash can be added at the end of a line to ignore the - newline: +Ignored end of line +------------------- + +A backslash can be added at the end of a line to ignore the newline: + + >>> 'This string will not include \\ + ... backslashes or newline characters.' + 'This string will not include backslashes or newline characters.' + +The same result can be achieved using triple-quoted strings, or +parentheses and string literal concatenation. + + +Escaped characters +------------------ + +To include a backslash in a non-raw Python string literal, it must be +doubled. The "\\\\" escape sequence denotes a single backslash +character: + + >>> print('C:\\\\Program Files') + C:\\Program Files + +Similarly, the "\\'" and "\\"" sequences denote the single and double +quote character, respectively: + + >>> print('\\' and \\"') + ' and " + + +Octal character +--------------- + +The sequence "\\*ooo*" denotes a *character* with the octal (base 8) +value *ooo*: + + >>> '\\120' + 'P' + +Up to three octal digits (0 through 7) are accepted. + +In a bytes literal, *character* means a *byte* with the given value. +In a string literal, it means a Unicode character with the given +value. + +Changed in version 3.11: Octal escapes with value larger than "0o377" +(255) produce a "DeprecationWarning". + +Changed in version 3.12: Octal escapes with value larger than "0o377" +(255) produce a "SyntaxWarning". In a future Python version they will +raise a "SyntaxError". + + +Hexadecimal character +--------------------- + +The sequence "\\x*hh*" denotes a *character* with the hex (base 16) +value *hh*: + + >>> '\\x50' + 'P' - >>> 'This string will not include \\ - ... backslashes or newline characters.' - 'This string will not include backslashes or newline characters.' +Unlike in Standard C, exactly two hex digits are required. - The same result can be achieved using triple-quoted strings, or - parentheses and string literal concatenation. +In a bytes literal, *character* means a *byte* with the given value. +In a string literal, it means a Unicode character with the given +value. -2. As in Standard C, up to three octal digits are accepted. - Changed in version 3.11: Octal escapes with value larger than - "0o377" produce a "DeprecationWarning". +Named Unicode character +----------------------- + +The sequence "\\N{*name*}" denotes a Unicode character with the given +*name*: + + >>> '\\N{LATIN CAPITAL LETTER P}' + 'P' + >>> '\\N{SNAKE}' + '🐍' - Changed in version 3.12: Octal escapes with value larger than - "0o377" produce a "SyntaxWarning". In a future Python version they - will be eventually a "SyntaxError". +This sequence cannot appear in bytes literals. -3. Unlike in Standard C, exactly two hex digits are required. +Changed in version 3.3: Support for name aliases has been added. -4. In a bytes literal, hexadecimal and octal escapes denote the byte - with the given value. In a string literal, these escapes denote a - Unicode character with the given value. -5. Changed in version 3.3: Support for name aliases [1] has been - added. +Hexadecimal Unicode characters +------------------------------ -6. Exactly four hex digits are required. +These sequences "\\u*xxxx*" and "\\U*xxxxxxxx*" denote the Unicode +character with the given hex (base 16) value. Exactly four digits are +required for "\\u"; exactly eight digits are required for "\\U". The +latter can encode any Unicode character. -7. Any Unicode character can be encoded this way. Exactly eight hex - digits are required. + >>> '\\u1234' + 'ሴ' + >>> '\\U0001f40d' + '🐍' -Unlike Standard C, all unrecognized escape sequences are left in the -string unchanged, i.e., *the backslash is left in the result*. (This -behavior is useful when debugging: if an escape sequence is mistyped, -the resulting output is more easily recognized as broken.) It is also -important to note that the escape sequences only recognized in string -literals fall into the category of unrecognized escapes for bytes -literals. +These sequences cannot appear in bytes literals. + + +Unrecognized escape sequences +----------------------------- + +Unlike in Standard C, all unrecognized escape sequences are left in +the string unchanged, that is, *the backslash is left in the result*: + + >>> print('\\q') + \\q + >>> list('\\q') + ['\\\\', 'q'] + +Note that for bytes literals, the escape sequences only recognized in +string literals ("\\N...", "\\u...", "\\U...") fall into the category of +unrecognized escapes. Changed in version 3.6: Unrecognized escape sequences produce a "DeprecationWarning". Changed in version 3.12: Unrecognized escape sequences produce a -"SyntaxWarning". In a future Python version they will be eventually a +"SyntaxWarning". In a future Python version they will raise a "SyntaxError". + +Bytes literals +============== + +*Bytes literals* are always prefixed with ‘"b"’ or ‘"B"’; they produce +an instance of the "bytes" type instead of the "str" type. They may +only contain ASCII characters; bytes with a numeric value of 128 or +greater must be expressed with escape sequences (typically Hexadecimal +character or Octal character): + + >>> b'\\x89PNG\\r\\n\\x1a\\n' + b'\\x89PNG\\r\\n\\x1a\\n' + >>> list(b'\\x89PNG\\r\\n\\x1a\\n') + [137, 80, 78, 71, 13, 10, 26, 10] + +Similarly, a zero byte must be expressed using an escape sequence +(typically "\\0" or "\\x00"). + + +Raw string literals +=================== + +Both string and bytes literals may optionally be prefixed with a +letter ‘"r"’ or ‘"R"’; such constructs are called *raw string +literals* and *raw bytes literals* respectively and treat backslashes +as literal characters. As a result, in raw string literals, escape +sequences are not treated specially: + + >>> r'\\d{4}-\\d{2}-\\d{2}' + '\\\\d{4}-\\\\d{2}-\\\\d{2}' + Even in a raw literal, quotes can be escaped with a backslash, but the backslash remains in the result; for example, "r"\\""" is a valid string literal consisting of two characters: a backslash and a double @@ -10024,63 +11174,414 @@ class is used in a class pattern with positional arguments, each the following quote character). Note also that a single backslash followed by a newline is interpreted as those two characters as part of the literal, *not* as a line continuation. + + +f-strings +========= + +Added in version 3.6. + +Changed in version 3.7: The "await" and "async for" can be used in +expressions within f-strings. + +Changed in version 3.8: Added the debug specifier ("=") + +Changed in version 3.12: Many restrictions on expressions within +f-strings have been removed. Notably, nested strings, comments, and +backslashes are now permitted. + +A *formatted string literal* or *f-string* is a string literal that is +prefixed with ‘"f"’ or ‘"F"’. Unlike other string literals, f-strings +do not have a constant value. They may contain *replacement fields* +delimited by curly braces "{}". Replacement fields contain expressions +which are evaluated at run time. For example: + + >>> who = 'nobody' + >>> nationality = 'Spanish' + >>> f'{who.title()} expects the {nationality} Inquisition!' + 'Nobody expects the Spanish Inquisition!' + +Any doubled curly braces ("{{" or "}}") outside replacement fields are +replaced with the corresponding single curly brace: + + >>> print(f'{{...}}') + {...} + +Other characters outside replacement fields are treated like in +ordinary string literals. This means that escape sequences are decoded +(except when a literal is also marked as a raw string), and newlines +are possible in triple-quoted f-strings: + + >>> name = 'Galahad' + >>> favorite_color = 'blue' + >>> print(f'{name}:\\t{favorite_color}') + Galahad: blue + >>> print(rf"C:\\Users\\{name}") + C:\\Users\\Galahad + >>> print(f\'\'\'Three shall be the number of the counting + ... and the number of the counting shall be three.\'\'\') + Three shall be the number of the counting + and the number of the counting shall be three. + +Expressions in formatted string literals are treated like regular +Python expressions. Each expression is evaluated in the context where +the formatted string literal appears, in order from left to right. An +empty expression is not allowed, and both "lambda" and assignment +expressions ":=" must be surrounded by explicit parentheses: + + >>> f'{(half := 1/2)}, {half * 42}' + '0.5, 21.0' + +Reusing the outer f-string quoting type inside a replacement field is +permitted: + + >>> a = dict(x=2) + >>> f"abc {a["x"]} def" + 'abc 2 def' + +Backslashes are also allowed in replacement fields and are evaluated +the same way as in any other context: + + >>> a = ["a", "b", "c"] + >>> print(f"List a contains:\\n{"\\n".join(a)}") + List a contains: + a + b + c + +It is possible to nest f-strings: + + >>> name = 'world' + >>> f'Repeated:{f' hello {name}' * 3}' + 'Repeated: hello world hello world hello world' + +Portable Python programs should not use more than 5 levels of nesting. + +**CPython implementation detail:** CPython does not limit nesting of +f-strings. + +Replacement expressions can contain newlines in both single-quoted and +triple-quoted f-strings and they can contain comments. Everything that +comes after a "#" inside a replacement field is a comment (even +closing braces and quotes). This means that replacement fields with +comments must be closed in a different line: + + >>> a = 2 + >>> f"abc{a # This comment }" continues until the end of the line + ... + 3}" + 'abc5' + +After the expression, replacement fields may optionally contain: + +* a *debug specifier* – an equal sign ("="), optionally surrounded by + whitespace on one or both sides; + +* a *conversion specifier* – "!s", "!r" or "!a"; and/or + +* a *format specifier* prefixed with a colon (":"). + +See the Standard Library section on f-strings for details on how these +fields are evaluated. + +As that section explains, *format specifiers* are passed as the second +argument to the "format()" function to format a replacement field +value. For example, they can be used to specify a field width and +padding characters using the Format Specification Mini-Language: + + >>> number = 14.3 + >>> f'{number:20.7f}' + ' 14.3000000' + +Top-level format specifiers may include nested replacement fields: + + >>> field_size = 20 + >>> precision = 7 + >>> f'{number:{field_size}.{precision}f}' + ' 14.3000000' + +These nested fields may include their own conversion fields and format +specifiers: + + >>> number = 3 + >>> f'{number:{field_size}}' + ' 3' + >>> f'{number:{field_size:05}}' + '00000000000000000003' + +However, these nested fields may not include more deeply nested +replacement fields. + +Formatted string literals cannot be used as *docstrings*, even if they +do not include expressions: + + >>> def foo(): + ... f"Not a docstring" + ... + >>> print(foo.__doc__) + None + +See also: + + * **PEP 498** – Literal String Interpolation + + * **PEP 701** – Syntactic formalization of f-strings + + * "str.format()", which uses a related format string mechanism. + + +t-strings +========= + +Added in version 3.14. + +A *template string literal* or *t-string* is a string literal that is +prefixed with ‘"t"’ or ‘"T"’. These strings follow the same syntax +rules as formatted string literals. For differences in evaluation +rules, see the Standard Library section on t-strings + + +Formal grammar for f-strings +============================ + +F-strings are handled partly by the *lexical analyzer*, which produces +the tokens "FSTRING_START", "FSTRING_MIDDLE" and "FSTRING_END", and +partly by the parser, which handles expressions in the replacement +field. The exact way the work is split is a CPython implementation +detail. + +Correspondingly, the f-string grammar is a mix of lexical and +syntactic definitions. + +Whitespace is significant in these situations: + +* There may be no whitespace in "FSTRING_START" (between the prefix + and quote). + +* Whitespace in "FSTRING_MIDDLE" is part of the literal string + contents. + +* In "fstring_replacement_field", if "f_debug_specifier" is present, + all whitespace after the opening brace until the + "f_debug_specifier", as well as whitespace immediately following + "f_debug_specifier", is retained as part of the expression. + + **CPython implementation detail:** The expression is not handled in + the tokenization phase; it is retrieved from the source code using + locations of the "{" token and the token after "=". + +The "FSTRING_MIDDLE" definition uses negative lookaheads ("!") to +indicate special characters (backslash, newline, "{", "}") and +sequences ("f_quote"). + + fstring: FSTRING_START fstring_middle* FSTRING_END + + FSTRING_START: fstringprefix ("'" | '"' | "\'\'\'" | '"""') + FSTRING_END: f_quote + fstringprefix: <("f" | "fr" | "rf"), case-insensitive> + f_debug_specifier: '=' + f_quote: + + fstring_middle: + | fstring_replacement_field + | FSTRING_MIDDLE + FSTRING_MIDDLE: + | (!"\\" !newline !'{' !'}' !f_quote) source_character + | stringescapeseq + | "{{" + | "}}" + | + fstring_replacement_field: + | '{' f_expression [f_debug_specifier] [fstring_conversion] + [fstring_full_format_spec] '}' + fstring_conversion: + | "!" ("s" | "r" | "a") + fstring_full_format_spec: + | ':' fstring_format_spec* + fstring_format_spec: + | FSTRING_MIDDLE + | fstring_replacement_field + f_expression: + | ','.(conditional_expression | "*" or_expr)+ [","] + | yield_expression + +Note: + + In the above grammar snippet, the "f_quote" and "FSTRING_MIDDLE" + rules are context-sensitive – they depend on the contents of + "FSTRING_START" of the nearest enclosing "fstring".Constructing a + more traditional formal grammar from this template is left as an + exercise for the reader. + +The grammar for t-strings is identical to the one for f-strings, with +*t* instead of *f* at the beginning of rule and token names and in the +prefix. + + tstring: TSTRING_START tstring_middle* TSTRING_END + + ''', - 'subscriptions': r'''Subscriptions -************* + 'subscriptions': r'''Subscriptions and slicings +************************** + +The *subscription* syntax is usually used for selecting an element +from a container – for example, to get a value from a "dict": + + >>> digits_by_name = {'one': 1, 'two': 2} + >>> digits_by_name['two'] # Subscripting a dictionary using the key 'two' + 2 + +In the subscription syntax, the object being subscribed – a primary – +is followed by a *subscript* in square brackets. In the simplest case, +the subscript is a single expression. -The subscription of an instance of a container class will generally -select an element from the container. The subscription of a *generic -class* will generally return a GenericAlias object. +Depending on the type of the object being subscribed, the subscript is +sometimes called a *key* (for mappings), *index* (for sequences), or +*type argument* (for *generic types*). Syntactically, these are all +equivalent: - subscription: primary "[" flexible_expression_list "]" + >>> colors = ['red', 'blue', 'green', 'black'] + >>> colors[3] # Subscripting a list using the index 3 + 'black' -When an object is subscripted, the interpreter will evaluate the -primary and the expression list. + >>> list[str] # Parameterizing the list type using the type argument str + list[str] -The primary must evaluate to an object that supports subscription. An -object may support subscription through defining one or both of -"__getitem__()" and "__class_getitem__()". When the primary is -subscripted, the evaluated result of the expression list will be -passed to one of these methods. For more details on when -"__class_getitem__" is called instead of "__getitem__", see +At runtime, the interpreter will evaluate the primary and the +subscript, and call the primary’s "__getitem__()" or +"__class_getitem__()" *special method* with the subscript as argument. +For more details on which of these methods is called, see __class_getitem__ versus __getitem__. -If the expression list contains at least one comma, or if any of the -expressions are starred, the expression list will evaluate to a -"tuple" containing the items of the expression list. Otherwise, the -expression list will evaluate to the value of the list’s sole member. - -Changed in version 3.11: Expressions in an expression list may be -starred. See **PEP 646**. - -For built-in objects, there are two types of objects that support -subscription via "__getitem__()": - -1. Mappings. If the primary is a *mapping*, the expression list must - evaluate to an object whose value is one of the keys of the - mapping, and the subscription selects the value in the mapping that - corresponds to that key. An example of a builtin mapping class is - the "dict" class. - -2. Sequences. If the primary is a *sequence*, the expression list must - evaluate to an "int" or a "slice" (as discussed in the following - section). Examples of builtin sequence classes include the "str", - "list" and "tuple" classes. - -The formal syntax makes no special provision for negative indices in -*sequences*. However, built-in sequences all provide a "__getitem__()" -method that interprets negative indices by adding the length of the -sequence to the index so that, for example, "x[-1]" selects the last -item of "x". The resulting value must be a nonnegative integer less -than the number of items in the sequence, and the subscription selects -the item whose index is that value (counting from zero). Since the -support for negative indices and slicing occurs in the object’s -"__getitem__()" method, subclasses overriding this method will need to -explicitly add that support. - -A "string" is a special kind of sequence whose items are *characters*. -A character is not a separate data type but a string of exactly one -character. +To show how subscription works, we can define a custom object that +implements "__getitem__()" and prints out the value of the subscript: + + >>> class SubscriptionDemo: + ... def __getitem__(self, key): + ... print(f'subscripted with: {key!r}') + ... + >>> demo = SubscriptionDemo() + >>> demo[1] + subscripted with: 1 + >>> demo['a' * 3] + subscripted with: 'aaa' + +See "__getitem__()" documentation for how built-in types handle +subscription. + +Subscriptions may also be used as targets in assignment or deletion +statements. In these cases, the interpreter will call the subscripted +object’s "__setitem__()" or "__delitem__()" *special method*, +respectively, instead of "__getitem__()". + + >>> colors = ['red', 'blue', 'green', 'black'] + >>> colors[3] = 'white' # Setting item at index + >>> colors + ['red', 'blue', 'green', 'white'] + >>> del colors[3] # Deleting item at index 3 + >>> colors + ['red', 'blue', 'green'] + +All advanced forms of *subscript* documented in the following sections +are also usable for assignment and deletion. + + +Slicings +======== + +A more advanced form of subscription, *slicing*, is commonly used to +extract a portion of a sequence. In this form, the subscript is a +*slice*: up to three expressions separated by colons. Any of the +expressions may be omitted, but a slice must contain at least one +colon: + + >>> number_names = ['zero', 'one', 'two', 'three', 'four', 'five'] + >>> number_names[1:3] + ['one', 'two'] + >>> number_names[1:] + ['one', 'two', 'three', 'four', 'five'] + >>> number_names[:3] + ['zero', 'one', 'two'] + >>> number_names[:] + ['zero', 'one', 'two', 'three', 'four', 'five'] + >>> number_names[::2] + ['zero', 'two', 'four'] + >>> number_names[:-3] + ['zero', 'one', 'two'] + >>> del number_names[4:] + >>> number_names + ['zero', 'one', 'two', 'three'] + +When a slice is evaluated, the interpreter constructs a "slice" object +whose "start", "stop" and "step" attributes, respectively, are the +results of the expressions between the colons. Any missing expression +evaluates to "None". This "slice" object is then passed to the +"__getitem__()" or "__class_getitem__()" *special method*, as above. + + # continuing with the SubscriptionDemo instance defined above: + >>> demo[2:3] + subscripted with: slice(2, 3, None) + >>> demo[::'spam'] + subscripted with: slice(None, None, 'spam') + + +Comma-separated subscripts +========================== + +The subscript can also be given as two or more comma-separated +expressions or slices: + + # continuing with the SubscriptionDemo instance defined above: + >>> demo[1, 2, 3] + subscripted with: (1, 2, 3) + >>> demo[1:2, 3] + subscripted with: (slice(1, 2, None), 3) + +This form is commonly used with numerical libraries for slicing multi- +dimensional data. In this case, the interpreter constructs a "tuple" +of the results of the expressions or slices, and passes this tuple to +the "__getitem__()" or "__class_getitem__()" *special method*, as +above. + +The subscript may also be given as a single expression or slice +followed by a comma, to specify a one-element tuple: + + >>> demo['spam',] + subscripted with: ('spam',) + + +“Starred” subscriptions +======================= + +Added in version 3.11: Expressions in *tuple_slices* may be starred. +See **PEP 646**. + +The subscript can also contain a starred expression. In this case, the +interpreter unpacks the result into a tuple, and passes this tuple to +"__getitem__()" or "__class_getitem__()": + + # continuing with the SubscriptionDemo instance defined above: + >>> demo[*range(10)] + subscripted with: (0, 1, 2, 3, 4, 5, 6, 7, 8, 9) + +Starred expressions may be combined with comma-separated expressions +and slices: + + >>> demo['a', 'b', *range(3), 'c'] + subscripted with: ('a', 'b', 0, 1, 2, 'c') + + +Formal subscription grammar +=========================== + + subscription: primary '[' subscript ']' + subscript: single_subscript | tuple_subscript + single_subscript: proper_slice | assignment_expression + proper_slice: [expression] ":" [expression] [ ":" [expression] ] + tuple_subscript: ','.(single_subscript | starred_expression)+ [','] + +Recall that the "|" operator denotes ordered choice. Specifically, in +"subscript", if both alternatives would match, the first +("single_subscript") has priority. ''', 'truth': r'''Truth Value Testing ******************* @@ -10090,8 +11591,11 @@ class is used in a class pattern with positional arguments, each By default, an object is considered true unless its class defines either a "__bool__()" method that returns "False" or a "__len__()" -method that returns zero, when called with the object. [1] Here are -most of the built-in objects considered false: +method that returns zero, when called with the object. [1] If one of +the methods raises an exception when called, the exception is +propagated and the object does not have a truth value (for example, +"NotImplemented"). Here are most of the built-in objects considered +false: * constants defined to be false: "None" and "False" @@ -10218,15 +11722,29 @@ class is used in a class pattern with positional arguments, each "except*" clause ================ -The "except*" clause(s) are used for handling "ExceptionGroup"s. The -exception type for matching is interpreted as in the case of "except", -but in the case of exception groups we can have partial matches when -the type matches some of the exceptions in the group. This means that -multiple "except*" clauses can execute, each handling part of the -exception group. Each clause executes at most once and handles an -exception group of all matching exceptions. Each exception in the -group is handled by at most one "except*" clause, the first that -matches it. +The "except*" clause(s) specify one or more handlers for groups of +exceptions ("BaseExceptionGroup" instances). A "try" statement can +have either "except" or "except*" clauses, but not both. The exception +type for matching is mandatory in the case of "except*", so "except*:" +is a syntax error. The type is interpreted as in the case of "except", +but matching is performed on the exceptions contained in the group +that is being handled. An "TypeError" is raised if a matching type is +a subclass of "BaseExceptionGroup", because that would have ambiguous +semantics. + +When an exception group is raised in the try block, each "except*" +clause splits (see "split()") it into the subgroups of matching and +non-matching exceptions. If the matching subgroup is not empty, it +becomes the handled exception (the value returned from +"sys.exception()") and assigned to the target of the "except*" clause +(if there is one). Then, the body of the "except*" clause executes. If +the non-matching subgroup is not empty, it is processed by the next +"except*" in the same manner. This continues until all exceptions in +the group have been matched, or the last "except*" clause has run. + +After all "except*" clauses execute, the group of unhandled exceptions +is merged with any exceptions that were raised or re-raised from +within "except*" clauses. This merged exception group propagates on.: >>> try: ... raise ExceptionGroup("eg", @@ -10239,33 +11757,27 @@ class is used in a class pattern with positional arguments, each caught with nested (TypeError(2),) caught with nested (OSError(3), OSError(4)) + Exception Group Traceback (most recent call last): - | File "", line 2, in - | ExceptionGroup: eg + | File "", line 2, in + | raise ExceptionGroup("eg", + | [ValueError(1), TypeError(2), OSError(3), OSError(4)]) + | ExceptionGroup: eg (1 sub-exception) +-+---------------- 1 ---------------- | ValueError: 1 +------------------------------------ -Any remaining exceptions that were not handled by any "except*" clause -are re-raised at the end, along with all exceptions that were raised -from within the "except*" clauses. If this list contains more than one -exception to reraise, they are combined into an exception group. - -If the raised exception is not an exception group and its type matches -one of the "except*" clauses, it is caught and wrapped by an exception -group with an empty message string. +If the exception raised from the "try" block is not an exception group +and its type matches one of the "except*" clauses, it is caught and +wrapped by an exception group with an empty message string. This +ensures that the type of the target "e" is consistently +"BaseExceptionGroup": >>> try: ... raise BlockingIOError ... except* BlockingIOError as e: ... print(repr(e)) ... - ExceptionGroup('', (BlockingIOError())) - -An "except*" clause must have a matching expression; it cannot be -"except*:". Furthermore, this expression cannot contain exception -group types, because that would have ambiguous semantics. + ExceptionGroup('', (BlockingIOError(),)) -It is not possible to mix "except" and "except*" in the same "try". "break", "continue" and "return" cannot appear in an "except*" clause. @@ -10282,11 +11794,11 @@ class is used in a class pattern with positional arguments, each ================ If "finally" is present, it specifies a ‘cleanup’ handler. The "try" -clause is executed, including any "except" and "else" clauses. If an +clause is executed, including any "except" and "else" clauses. If an exception occurs in any of the clauses and is not handled, the exception is temporarily saved. The "finally" clause is executed. If there is a saved exception it is re-raised at the end of the "finally" -clause. If the "finally" clause raises another exception, the saved +clause. If the "finally" clause raises another exception, the saved exception is set as the context of the new exception. If the "finally" clause executes a "return", "break" or "continue" statement, the saved exception is discarded. For example, this function returns 42. @@ -10475,10 +11987,19 @@ def foo(): "a[-2]" equals "a[n-2]", the second to last item of sequence a with length "n". -Sequences also support slicing: "a[i:j]" selects all items with index -*k* such that *i* "<=" *k* "<" *j*. When used as an expression, a -slice is a sequence of the same type. The comment above about negative -indexes also applies to negative slice positions. +The resulting value must be a nonnegative integer less than the number +of items in the sequence. If it is not, an "IndexError" is raised. + +Sequences also support slicing: "a[start:stop]" selects all items with +index *k* such that *start* "<=" *k* "<" *stop*. When used as an +expression, a slice is a sequence of the same type. The comment above +about negative subscripts also applies to negative slice positions. +Note that no error is raised if a slice position is less than zero or +larger than the length of the sequence. + +If *start* is missing or "None", slicing behaves as if *start* was +zero. If *stop* is missing or "None", slicing behaves as if *stop* was +equal to the length of the sequence. Some sequences also support “extended slicing” with a third “step” parameter: "a[i:j:k]" selects all items of *a* with index *x* where "x @@ -10499,27 +12020,33 @@ def foo(): The following types are immutable sequences: Strings - A string is a sequence of values that represent Unicode code - points. All the code points in the range "U+0000 - U+10FFFF" can be - represented in a string. Python doesn’t have a char type; instead, - every code point in the string is represented as a string object - with length "1". The built-in function "ord()" converts a code - point from its string form to an integer in the range "0 - 10FFFF"; - "chr()" converts an integer in the range "0 - 10FFFF" to the - corresponding length "1" string object. "str.encode()" can be used - to convert a "str" to "bytes" using the given text encoding, and + A string ("str") is a sequence of values that represent + *characters*, or more formally, *Unicode code points*. All the code + points in the range "0" to "0x10FFFF" can be represented in a + string. + + Python doesn’t have a dedicated *character* type. Instead, every + code point in the string is represented as a string object with + length "1". + + The built-in function "ord()" converts a code point from its string + form to an integer in the range "0" to "0x10FFFF"; "chr()" converts + an integer in the range "0" to "0x10FFFF" to the corresponding + length "1" string object. "str.encode()" can be used to convert a + "str" to "bytes" using the given text encoding, and "bytes.decode()" can be used to achieve the opposite. Tuples - The items of a tuple are arbitrary Python objects. Tuples of two or - more items are formed by comma-separated lists of expressions. A - tuple of one item (a ‘singleton’) can be formed by affixing a comma - to an expression (an expression by itself does not create a tuple, - since parentheses must be usable for grouping of expressions). An - empty tuple can be formed by an empty pair of parentheses. + The items of a "tuple" are arbitrary Python objects. Tuples of two + or more items are formed by comma-separated lists of expressions. + A tuple of one item (a ‘singleton’) can be formed by affixing a + comma to an expression (an expression by itself does not create a + tuple, since parentheses must be usable for grouping of + expressions). An empty tuple can be formed by an empty pair of + parentheses. Bytes - A bytes object is an immutable array. The items are 8-bit bytes, + A "bytes" object is an immutable array. The items are 8-bit bytes, represented by integers in the range 0 <= x < 256. Bytes literals (like "b'abc'") and the built-in "bytes()" constructor can be used to create bytes objects. Also, bytes objects can be decoded to @@ -10648,6 +12175,10 @@ def foo(): +----------------------------------------------------+----------------------------------------------------+ | Attribute | Meaning | |====================================================|====================================================| +| function.__builtins__ | A reference to the "dictionary" that holds the | +| | function’s builtins namespace. Added in version | +| | 3.10. | ++----------------------------------------------------+----------------------------------------------------+ | function.__globals__ | A reference to the "dictionary" that holds the | | | function’s global variables – the global namespace | | | of the module in which the function was defined. | @@ -11161,6 +12692,11 @@ class method object, it is transformed into an instance method object | | "X.__bases__" will be exactly equal to "(A, B, | | | C)". | +----------------------------------------------------+----------------------------------------------------+ +| type.__base__ | **CPython implementation detail:** The single base | +| | class in the inheritance chain that is responsible | +| | for the memory layout of instances. This attribute | +| | corresponds to "tp_base" at the C level. | ++----------------------------------------------------+----------------------------------------------------+ | type.__doc__ | The class’s documentation string, or "None" if | | | undefined. Not inherited by subclasses. | +----------------------------------------------------+----------------------------------------------------+ @@ -11168,11 +12704,20 @@ class method object, it is transformed into an instance method object | | collected during class body execution. See also: | | | "__annotations__ attributes". For best practices | | | on working with "__annotations__", please see | -| | "annotationlib". Where possible, use | +| | "annotationlib". Use | | | "annotationlib.get_annotations()" instead of | -| | accessing this attribute directly. Changed in | -| | version 3.14: Annotations are now lazily | -| | evaluated. See **PEP 649**. | +| | accessing this attribute directly. Warning: | +| | Accessing the "__annotations__" attribute directly | +| | on a class object may return annotations for the | +| | wrong class, specifically in certain cases where | +| | the class, its base class, or a metaclass is | +| | defined under "from __future__ import | +| | annotations". See **749** for details.This | +| | attribute does not exist on certain builtin | +| | classes. On user-defined classes without | +| | "__annotations__", it is an empty dictionary. | +| | Changed in version 3.14: Annotations are now | +| | lazily evaluated. See **PEP 649**. | +----------------------------------------------------+----------------------------------------------------+ | type.__annotate__() | The *annotate function* for this class, or "None" | | | if the class has no annotations. See also: | @@ -11271,11 +12816,28 @@ class instance has a namespace implemented as a dictionary which is socket objects (and perhaps by other functions or methods provided by extension modules). +File objects implement common methods, listed below, to simplify usage +in generic code. They are expected to be With Statement Context +Managers. + The objects "sys.stdin", "sys.stdout" and "sys.stderr" are initialized to file objects corresponding to the interpreter’s standard input, output and error streams; they are all open in text mode and therefore follow the interface defined by the "io.TextIOBase" abstract class. +file.read(size=-1, /) + + Retrieve up to *size* data from the file. As a convenience if + *size* is unspecified or -1 retrieve all data available. + +file.write(data, /) + + Store *data* to the file. + +file.close() + + Flush any buffers and close the underlying file. + Internal types ============== @@ -11510,6 +13072,10 @@ class instance has a namespace implemented as a dictionary which is | | (this is an index into the *bytecode* string of | | | the code object) | +----------------------------------------------------+----------------------------------------------------+ +| frame.f_generator | The *generator* or *coroutine* object that owns | +| | this frame, or "None" if the frame is a normal | +| | function. Added in version 3.14. | ++----------------------------------------------------+----------------------------------------------------+ Special writable attributes @@ -11690,8 +13256,8 @@ class instance has a namespace implemented as a dictionary which is dictionary entry. class dict(**kwargs) -class dict(mapping, **kwargs) -class dict(iterable, **kwargs) +class dict(mapping, /, **kwargs) +class dict(iterable, /, **kwargs) Return a new dictionary initialized from an optional positional argument and a possibly empty set of keyword arguments. @@ -11724,8 +13290,11 @@ class dict(iterable, **kwargs) the keyword argument replaces the value from the positional argument. - To illustrate, the following examples all return a dictionary equal - to "{"one": 1, "two": 2, "three": 3}": + Dictionaries compare equal if and only if they have the same "(key, + value)" pairs (regardless of ordering). Order comparisons (‘<’, + ‘<=’, ‘>=’, ‘>’) raise "TypeError". To illustrate dictionary + creation and equality, the following examples all return a + dictionary equal to "{"one": 1, "two": 2, "three": 3}": >>> a = dict(one=1, two=2, three=3) >>> b = {'one': 1, 'two': 2, 'three': 3} @@ -11740,6 +13309,32 @@ class dict(iterable, **kwargs) keys that are valid Python identifiers. Otherwise, any valid keys can be used. + Dictionaries preserve insertion order. Note that updating a key + does not affect the order. Keys added after deletion are inserted + at the end. + + >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} + >>> d + {'one': 1, 'two': 2, 'three': 3, 'four': 4} + >>> list(d) + ['one', 'two', 'three', 'four'] + >>> list(d.values()) + [1, 2, 3, 4] + >>> d["one"] = 42 + >>> d + {'one': 42, 'two': 2, 'three': 3, 'four': 4} + >>> del d["two"] + >>> d["two"] = None + >>> d + {'one': 42, 'three': 3, 'four': 4, 'two': None} + + Changed in version 3.7: Dictionary order is guaranteed to be + insertion order. This behavior was an implementation detail of + CPython from 3.6. + + Dictionaries are generic over two types, signifying (respectively) + the types of the dictionary’s keys and values. + These are the operations that dictionaries support (and therefore, custom mapping types should support too): @@ -11777,8 +13372,8 @@ class dict(iterable, **kwargs) 1 The example above shows part of the implementation of - "collections.Counter". A different "__missing__" method is used - by "collections.defaultdict". + "collections.Counter". A different "__missing__()" method is + used by "collections.defaultdict". d[key] = value @@ -11837,7 +13432,8 @@ class dict(iterable, **kwargs) Return a new view of the dictionary’s keys. See the documentation of view objects. - pop(key[, default]) + pop(key, /) + pop(key, default, /) If *key* is in the dictionary, remove it and return its value, else return *default*. If *default* is not given and *key* is @@ -11868,10 +13464,13 @@ class dict(iterable, **kwargs) *key* with a value of *default* and return *default*. *default* defaults to "None". - update([other]) + update(**kwargs) + update(mapping, /, **kwargs) + update(iterable, /, **kwargs) - Update the dictionary with the key/value pairs from *other*, - overwriting existing keys. Return "None". + Update the dictionary with the key/value pairs from *mapping* or + *iterable* and *kwargs*, overwriting existing keys. Return + "None". "update()" accepts either another object with a "keys()" method (in which case "__getitem__()" is called with every key returned @@ -11910,33 +13509,6 @@ class dict(iterable, **kwargs) Added in version 3.9. - Dictionaries compare equal if and only if they have the same "(key, - value)" pairs (regardless of ordering). Order comparisons (‘<’, - ‘<=’, ‘>=’, ‘>’) raise "TypeError". - - Dictionaries preserve insertion order. Note that updating a key - does not affect the order. Keys added after deletion are inserted - at the end. - - >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} - >>> d - {'one': 1, 'two': 2, 'three': 3, 'four': 4} - >>> list(d) - ['one', 'two', 'three', 'four'] - >>> list(d.values()) - [1, 2, 3, 4] - >>> d["one"] = 42 - >>> d - {'one': 42, 'two': 2, 'three': 3, 'four': 4} - >>> del d["two"] - >>> d["two"] = None - >>> d - {'one': 42, 'three': 3, 'four': 4, 'two': None} - - Changed in version 3.7: Dictionary order is guaranteed to be - insertion order. This behavior was an implementation detail of - CPython from 3.6. - Dictionaries and dictionary views are reversible. >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} @@ -11956,6 +13528,11 @@ class dict(iterable, **kwargs) "types.MappingProxyType" can be used to create a read-only view of a "dict". +See also: + + For detailed information on thread-safety guarantees for "dict" + objects, see Thread safety for dict objects. + Dictionary view objects ======================= @@ -12163,7 +13740,7 @@ class dict(iterable, **kwargs) | "s * n" or "n * s" | equivalent to adding *s* to | (2)(7) | | | itself *n* times | | +----------------------------+----------------------------------+------------+ -| "s[i]" | *i*th item of *s*, origin 0 | (3) | +| "s[i]" | *i*th item of *s*, origin 0 | (3)(8) | +----------------------------+----------------------------------+------------+ | "s[i:j]" | slice of *s* from *i* to *j* | (3)(4) | +----------------------------+----------------------------------+------------+ @@ -12176,13 +13753,6 @@ class dict(iterable, **kwargs) +----------------------------+----------------------------------+------------+ | "max(s)" | largest item of *s* | | +----------------------------+----------------------------------+------------+ -| "s.index(x[, i[, j]])" | index of the first occurrence of | (8) | -| | *x* in *s* (at or after index | | -| | *i* and before index *j*) | | -+----------------------------+----------------------------------+------------+ -| "s.count(x)" | total number of occurrences of | | -| | *x* in *s* | | -+----------------------------+----------------------------------+------------+ Sequences of the same type also support comparisons. In particular, tuples and lists are compared lexicographically by comparing @@ -12240,10 +13810,17 @@ class dict(iterable, **kwargs) note that "-0" is still "0". 4. The slice of *s* from *i* to *j* is defined as the sequence of - items with index *k* such that "i <= k < j". If *i* or *j* is - greater than "len(s)", use "len(s)". If *i* is omitted or "None", - use "0". If *j* is omitted or "None", use "len(s)". If *i* is - greater than or equal to *j*, the slice is empty. + items with index *k* such that "i <= k < j". + + * If *i* is omitted or "None", use "0". + + * If *j* is omitted or "None", use "len(s)". + + * If *i* or *j* is less than "-len(s)", use "0". + + * If *i* or *j* is greater than "len(s)", use "len(s)". + + * If *i* is greater than or equal to *j*, the slice is empty. 5. The slice of *s* from *i* to *j* with step *k* is defined as the sequence of items with index "x = i + n*k" such that "0 <= n < @@ -12279,13 +13856,31 @@ class dict(iterable, **kwargs) that follow specific patterns, and hence don’t support sequence concatenation or repetition. -8. "index" raises "ValueError" when *x* is not found in *s*. Not all - implementations support passing the additional arguments *i* and - *j*. These arguments allow efficient searching of subsections of - the sequence. Passing the extra arguments is roughly equivalent to - using "s[i:j].index(x)", only without copying any data and with the - returned index being relative to the start of the sequence rather - than the start of the slice. +8. An "IndexError" is raised if *i* is outside the sequence range. + +-[ Sequence Methods ]- + +Sequence types also support the following methods: + +sequence.count(value, /) + + Return the total number of occurrences of *value* in *sequence*. + +sequence.index(value[, start[, stop]]) + + Return the index of the first occurrence of *value* in *sequence*. + + Raises "ValueError" if *value* is not found in *sequence*. + + The *start* or *stop* arguments allow for efficient searching of + subsections of the sequence, beginning at *start* and ending at + *stop*. This is roughly equivalent to "start + + sequence[start:stop].index(value)", only without copying any data. + + Caution: + + Not all sequence types support passing the *start* and *stop* + arguments. Immutable Sequence Types @@ -12321,11 +13916,15 @@ class dict(iterable, **kwargs) | "s[i] = x" | item *i* of *s* is replaced by | | | | *x* | | +--------------------------------+----------------------------------+-----------------------+ +| "del s[i]" | removes item *i* of *s* | | ++--------------------------------+----------------------------------+-----------------------+ | "s[i:j] = t" | slice of *s* from *i* to *j* is | | | | replaced by the contents of the | | | | iterable *t* | | +--------------------------------+----------------------------------+-----------------------+ -| "del s[i:j]" | same as "s[i:j] = []" | | +| "del s[i:j]" | removes the elements of "s[i:j]" | | +| | from the list (same as "s[i:j] = | | +| | []") | | +--------------------------------+----------------------------------+-----------------------+ | "s[i:j:k] = t" | the elements of "s[i:j:k]" are | (1) | | | replaced by those of *t* | | @@ -12333,63 +13932,79 @@ class dict(iterable, **kwargs) | "del s[i:j:k]" | removes the elements of | | | | "s[i:j:k]" from the list | | +--------------------------------+----------------------------------+-----------------------+ -| "s.append(x)" | appends *x* to the end of the | | -| | sequence (same as | | -| | "s[len(s):len(s)] = [x]") | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.clear()" | removes all items from *s* (same | (5) | -| | as "del s[:]") | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.copy()" | creates a shallow copy of *s* | (5) | -| | (same as "s[:]") | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.extend(t)" or "s += t" | extends *s* with the contents of | | +| "s += t" | extends *s* with the contents of | | | | *t* (for the most part the same | | | | as "s[len(s):len(s)] = t") | | +--------------------------------+----------------------------------+-----------------------+ -| "s *= n" | updates *s* with its contents | (6) | +| "s *= n" | updates *s* with its contents | (2) | | | repeated *n* times | | +--------------------------------+----------------------------------+-----------------------+ -| "s.insert(i, x)" | inserts *x* into *s* at the | | -| | index given by *i* (same as | | -| | "s[i:i] = [x]") | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.pop()" or "s.pop(i)" | retrieves the item at *i* and | (2) | -| | also removes it from *s* | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.remove(x)" | removes the first item from *s* | (3) | -| | where "s[i]" is equal to *x* | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.reverse()" | reverses the items of *s* in | (4) | -| | place | | -+--------------------------------+----------------------------------+-----------------------+ Notes: 1. If *k* is not equal to "1", *t* must have the same length as the slice it is replacing. -2. The optional argument *i* defaults to "-1", so that by default the - last item is removed and returned. +2. The value *n* is an integer, or an object implementing + "__index__()". Zero and negative values of *n* clear the sequence. + Items in the sequence are not copied; they are referenced multiple + times, as explained for "s * n" under Common Sequence Operations. -3. "remove()" raises "ValueError" when *x* is not found in *s*. +-[ Mutable Sequence Methods ]- -4. The "reverse()" method modifies the sequence in place for economy - of space when reversing a large sequence. To remind users that it - operates by side effect, it does not return the reversed sequence. +Mutable sequence types also support the following methods: -5. "clear()" and "copy()" are included for consistency with the - interfaces of mutable containers that don’t support slicing - operations (such as "dict" and "set"). "copy()" is not part of the - "collections.abc.MutableSequence" ABC, but most concrete mutable - sequence classes provide it. +sequence.append(value, /) - Added in version 3.3: "clear()" and "copy()" methods. + Append *value* to the end of the sequence. This is equivalent to + writing "seq[len(seq):len(seq)] = [value]". -6. The value *n* is an integer, or an object implementing - "__index__()". Zero and negative values of *n* clear the sequence. - Items in the sequence are not copied; they are referenced multiple - times, as explained for "s * n" under Common Sequence Operations. +sequence.clear() + + Added in version 3.3. + + Remove all items from *sequence*. This is equivalent to writing + "del sequence[:]". + +sequence.copy() + + Added in version 3.3. + + Create a shallow copy of *sequence*. This is equivalent to writing + "sequence[:]". + + Hint: + + The "copy()" method is not part of the "MutableSequence" "ABC", + but most concrete mutable sequence types provide it. + +sequence.extend(iterable, /) + + Extend *sequence* with the contents of *iterable*. For the most + part, this is the same as writing "seq[len(seq):len(seq)] = + iterable". + +sequence.insert(index, value, /) + + Insert *value* into *sequence* at the given *index*. This is + equivalent to writing "sequence[index:index] = [value]". + +sequence.pop(index=-1, /) + + Retrieve the item at *index* and also removes it from *sequence*. + By default, the last item in *sequence* is removed and returned. + +sequence.remove(value, /) + + Remove the first item from *sequence* where "sequence[i] == value". + + Raises "ValueError" if *value* is not found in *sequence*. + +sequence.reverse() + + Reverse the items of *sequence* in place. This method maintains + economy of space when reversing a large sequence. To remind users + that it operates by side-effect, it returns "None". Lists @@ -12399,7 +14014,7 @@ class dict(iterable, **kwargs) homogeneous items (where the precise degree of similarity will vary by application). -class list([iterable]) +class list(iterable=(), /) Lists may be constructed in several ways: @@ -12424,6 +14039,8 @@ class list([iterable]) Many other operations also produce lists, including the "sorted()" built-in. + Lists are generic over the types of their items. + Lists implement all of the common and mutable sequence operations. Lists also provide the following additional method: @@ -12470,6 +14087,11 @@ class list([iterable]) empty for the duration, and raises "ValueError" if it can detect that the list has been mutated during a sort. +See also: + + For detailed information on thread-safety guarantees for "list" + objects, see Thread safety for list objects. + Tuples ====== @@ -12480,7 +14102,7 @@ class list([iterable]) of homogeneous data is needed (such as allowing storage in a "set" or "dict" instance). -class tuple([iterable]) +class tuple(iterable=(), /) Tuples may be constructed in a number of ways: @@ -12509,6 +14131,10 @@ class tuple([iterable]) Tuples implement all of the common sequence operations. + Tuples are generic over the types of their contents. For more + information, refer to the typing documentation on annotating + tuples. + For heterogeneous collections of data where access by name is clearer than access by index, "collections.namedtuple()" may be a more appropriate choice than a simple tuple object. @@ -12520,8 +14146,8 @@ class tuple([iterable]) The "range" type represents an immutable sequence of numbers and is commonly used for looping a specific number of times in "for" loops. -class range(stop) -class range(start, stop[, step]) +class range(stop, /) +class range(start, stop, step=1, /) The arguments to the range constructor must be integers (either built-in "int" or any object that implements the "__index__()" @@ -12649,11 +14275,15 @@ class range(start, stop[, step]) | "s[i] = x" | item *i* of *s* is replaced by | | | | *x* | | +--------------------------------+----------------------------------+-----------------------+ +| "del s[i]" | removes item *i* of *s* | | ++--------------------------------+----------------------------------+-----------------------+ | "s[i:j] = t" | slice of *s* from *i* to *j* is | | | | replaced by the contents of the | | | | iterable *t* | | +--------------------------------+----------------------------------+-----------------------+ -| "del s[i:j]" | same as "s[i:j] = []" | | +| "del s[i:j]" | removes the elements of "s[i:j]" | | +| | from the list (same as "s[i:j] = | | +| | []") | | +--------------------------------+----------------------------------+-----------------------+ | "s[i:j:k] = t" | the elements of "s[i:j:k]" are | (1) | | | replaced by those of *t* | | @@ -12661,63 +14291,79 @@ class range(start, stop[, step]) | "del s[i:j:k]" | removes the elements of | | | | "s[i:j:k]" from the list | | +--------------------------------+----------------------------------+-----------------------+ -| "s.append(x)" | appends *x* to the end of the | | -| | sequence (same as | | -| | "s[len(s):len(s)] = [x]") | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.clear()" | removes all items from *s* (same | (5) | -| | as "del s[:]") | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.copy()" | creates a shallow copy of *s* | (5) | -| | (same as "s[:]") | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.extend(t)" or "s += t" | extends *s* with the contents of | | +| "s += t" | extends *s* with the contents of | | | | *t* (for the most part the same | | | | as "s[len(s):len(s)] = t") | | +--------------------------------+----------------------------------+-----------------------+ -| "s *= n" | updates *s* with its contents | (6) | +| "s *= n" | updates *s* with its contents | (2) | | | repeated *n* times | | +--------------------------------+----------------------------------+-----------------------+ -| "s.insert(i, x)" | inserts *x* into *s* at the | | -| | index given by *i* (same as | | -| | "s[i:i] = [x]") | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.pop()" or "s.pop(i)" | retrieves the item at *i* and | (2) | -| | also removes it from *s* | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.remove(x)" | removes the first item from *s* | (3) | -| | where "s[i]" is equal to *x* | | -+--------------------------------+----------------------------------+-----------------------+ -| "s.reverse()" | reverses the items of *s* in | (4) | -| | place | | -+--------------------------------+----------------------------------+-----------------------+ Notes: 1. If *k* is not equal to "1", *t* must have the same length as the slice it is replacing. -2. The optional argument *i* defaults to "-1", so that by default the - last item is removed and returned. +2. The value *n* is an integer, or an object implementing + "__index__()". Zero and negative values of *n* clear the sequence. + Items in the sequence are not copied; they are referenced multiple + times, as explained for "s * n" under Common Sequence Operations. -3. "remove()" raises "ValueError" when *x* is not found in *s*. +-[ Mutable Sequence Methods ]- -4. The "reverse()" method modifies the sequence in place for economy - of space when reversing a large sequence. To remind users that it - operates by side effect, it does not return the reversed sequence. +Mutable sequence types also support the following methods: -5. "clear()" and "copy()" are included for consistency with the - interfaces of mutable containers that don’t support slicing - operations (such as "dict" and "set"). "copy()" is not part of the - "collections.abc.MutableSequence" ABC, but most concrete mutable - sequence classes provide it. +sequence.append(value, /) - Added in version 3.3: "clear()" and "copy()" methods. + Append *value* to the end of the sequence. This is equivalent to + writing "seq[len(seq):len(seq)] = [value]". -6. The value *n* is an integer, or an object implementing - "__index__()". Zero and negative values of *n* clear the sequence. - Items in the sequence are not copied; they are referenced multiple - times, as explained for "s * n" under Common Sequence Operations. +sequence.clear() + + Added in version 3.3. + + Remove all items from *sequence*. This is equivalent to writing + "del sequence[:]". + +sequence.copy() + + Added in version 3.3. + + Create a shallow copy of *sequence*. This is equivalent to writing + "sequence[:]". + + Hint: + + The "copy()" method is not part of the "MutableSequence" "ABC", + but most concrete mutable sequence types provide it. + +sequence.extend(iterable, /) + + Extend *sequence* with the contents of *iterable*. For the most + part, this is the same as writing "seq[len(seq):len(seq)] = + iterable". + +sequence.insert(index, value, /) + + Insert *value* into *sequence* at the given *index*. This is + equivalent to writing "sequence[index:index] = [value]". + +sequence.pop(index=-1, /) + + Retrieve the item at *index* and also removes it from *sequence*. + By default, the last item in *sequence* is removed and returned. + +sequence.remove(value, /) + + Remove the first item from *sequence* where "sequence[i] == value". + + Raises "ValueError" if *value* is not found in *sequence*. + +sequence.reverse() + + Reverse the items of *sequence* in place. This method maintains + economy of space when reversing a large sequence. To remind users + that it operates by side-effect, it returns "None". ''', 'unary': r'''Unary arithmetic and bitwise operations *************************************** @@ -12820,9 +14466,9 @@ class range(start, stop[, step]) is semantically equivalent to: manager = (EXPRESSION) - enter = type(manager).__enter__ - exit = type(manager).__exit__ - value = enter(manager) + enter = manager.__enter__ + exit = manager.__exit__ + value = enter() hit_except = False try: @@ -12830,11 +14476,14 @@ class range(start, stop[, step]) SUITE except: hit_except = True - if not exit(manager, *sys.exc_info()): + if not exit(*sys.exc_info()): raise finally: if not hit_except: - exit(manager, None, None, None) + exit(None, None, None) + +except that implicit special method lookup is used for "__enter__()" +and "__exit__()". With more than one item, the context managers are processed as if multiple "with" statements were nested: diff --git a/Lib/queue.py b/Lib/queue.py index 25beb46e30d6bd9..c0b359876543f7b 100644 --- a/Lib/queue.py +++ b/Lib/queue.py @@ -80,9 +80,6 @@ def task_done(self): have been processed (meaning that a task_done() call was received for every item that had been put() into the queue). - shutdown(immediate=True) calls task_done() for each remaining item in - the queue. - Raises a ValueError if called more times than there were items placed in the queue. ''' @@ -239,9 +236,11 @@ def shutdown(self, immediate=False): By default, gets will only raise once the queue is empty. Set 'immediate' to True to make gets raise immediately instead. - All blocked callers of put() and get() will be unblocked. If - 'immediate', a task is marked as done for each item remaining in - the queue, which may unblock callers of join(). + All blocked callers of put() and get() will be unblocked. + + If 'immediate', the queue is drained and unfinished tasks + is reduced by the number of drained tasks. If unfinished tasks + is reduced to zero, callers of Queue.join are unblocked. ''' with self.mutex: self.is_shutdown = True diff --git a/Lib/random.py b/Lib/random.py index 86d562f0b8aaf60..0e6a183d179f488 100644 --- a/Lib/random.py +++ b/Lib/random.py @@ -836,7 +836,11 @@ def binomialvariate(self, n=1, p=0.5): if not c: return x while True: - y += _floor(_log2(random()) / c) + 1 + try: + y += _floor(_log2(random()) / c) + 1 + except ValueError: + # Reject case where random() returned 0.0 + continue if y > n: return x x += 1 @@ -844,8 +848,8 @@ def binomialvariate(self, n=1, p=0.5): # BTRS: Transformed rejection with squeeze method by Wolfgang Hörmann # https://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.47.8407&rep=rep1&type=pdf assert n*p >= 10.0 and p <= 0.5 - setup_complete = False + setup_complete = False spq = _sqrt(n * p * (1.0 - p)) # Standard deviation of the distribution b = 1.15 + 2.53 * spq a = -0.0873 + 0.0248 * b + 0.01 * p @@ -857,25 +861,30 @@ def binomialvariate(self, n=1, p=0.5): u = random() u -= 0.5 us = 0.5 - _fabs(u) - k = _floor((2.0 * a / us + b) * u + c) + try: + k = _floor((2.0 * a / us + b) * u + c) + except ZeroDivisionError: + # Reject case where random() returned 0.0 + continue if k < 0 or k > n: continue + v = random() # The early-out "squeeze" test substantially reduces # the number of acceptance condition evaluations. - v = random() if us >= 0.07 and v <= vr: return k - # Acceptance-rejection test. - # Note, the original paper erroneously omits the call to log(v) - # when comparing to the log of the rescaled binomial distribution. if not setup_complete: alpha = (2.83 + 5.1 / b) * spq lpq = _log(p / (1.0 - p)) m = _floor((n + 1) * p) # Mode of the distribution h = _lgamma(m + 1) + _lgamma(n - m + 1) setup_complete = True # Only needs to be done once + + # Acceptance-rejection test. + # Note, the original paper erroneously omits the call to log(v) + # when comparing to the log of the rescaled binomial distribution. v *= alpha / (a / (us * us) + b) if _log(v) <= h - _lgamma(k + 1) - _lgamma(n - k + 1) + (k - m) * lpq: return k diff --git a/Lib/reprlib.py b/Lib/reprlib.py index 441d1be4bdede2c..ab18247682b69aa 100644 --- a/Lib/reprlib.py +++ b/Lib/reprlib.py @@ -181,7 +181,22 @@ def repr_str(self, x, level): return s def repr_int(self, x, level): - s = builtins.repr(x) # XXX Hope this isn't too slow... + try: + s = builtins.repr(x) + except ValueError as exc: + assert 'sys.set_int_max_str_digits()' in str(exc) + # Those imports must be deferred due to Python's build system + # where the reprlib module is imported before the math module. + import math, sys + # Integers with more than sys.get_int_max_str_digits() digits + # are rendered differently as their repr() raises a ValueError. + # See https://github.com/python/cpython/issues/135487. + k = 1 + int(math.log10(abs(x))) + # Note: math.log10(abs(x)) may be overestimated or underestimated, + # but for simplicity, we do not compute the exact number of digits. + max_digits = sys.get_int_max_str_digits() + return (f'<{x.__class__.__name__} instance with roughly {k} ' + f'digits (limit at {max_digits}) at 0x{id(x):x}>') if len(s) > self.maxlong: i = max(0, (self.maxlong-3)//2) j = max(0, self.maxlong-3-i) diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index 23eb0020f42e8a0..e75dd0a9e3dc197 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -34,6 +34,7 @@ import inspect import keyword import re +import types import __main__ import warnings @@ -178,14 +179,14 @@ def attr_matches(self, text): if (word[:n] == attr and not (noprefix and word[:n+1] == noprefix)): match = "%s.%s" % (expr, word) - if isinstance(getattr(type(thisobject), word, None), - property): - # bpo-44752: thisobject.word is a method decorated by - # `@property`. What follows applies a postfix if - # thisobject.word is callable, but know we know that - # this is not callable (because it is a property). - # Also, getattr(thisobject, word) will evaluate the - # property method, which is not desirable. + + class_attr = getattr(type(thisobject), word, None) + if isinstance( + class_attr, + (property, types.GetSetDescriptorType, types.MemberDescriptorType) + ) or (hasattr(class_attr, '__get__') and not callable(class_attr)): + # Avoid evaluating descriptors, which could run + # arbitrary code or raise exceptions. matches.append(match) continue if (value := getattr(thisobject, word, None)) is not None: diff --git a/Lib/runpy.py b/Lib/runpy.py index ef54d3282eee06d..1d5ecf0cf15bc0e 100644 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -103,8 +103,10 @@ def _run_module_code(code, init_globals=None, # Helper to get the full name, spec and code for a module def _get_module_details(mod_name, error=ImportError): + # name= is only accepted by ImportError and its subclasses. + kwargs = {"name": mod_name} if issubclass(error, ImportError) else {} if mod_name.startswith("."): - raise error("Relative module names not supported") + raise error("Relative module names not supported", **kwargs) pkg_name, _, _ = mod_name.rpartition(".") if pkg_name: # Try importing the parent to avoid catching initialization errors @@ -137,12 +139,13 @@ def _get_module_details(mod_name, error=ImportError): if mod_name.endswith(".py"): msg += (f". Try using '{mod_name[:-3]}' instead of " f"'{mod_name}' as the module name.") - raise error(msg.format(mod_name, type(ex).__name__, ex)) from ex + raise error(msg.format(mod_name, type(ex).__name__, ex), + **kwargs) from ex if spec is None: - raise error("No module named %s" % mod_name) + raise error("No module named %s" % mod_name, **kwargs) if spec.submodule_search_locations is not None: if mod_name == "__main__" or mod_name.endswith(".__main__"): - raise error("Cannot use package as __main__ module") + raise error("Cannot use package as __main__ module", **kwargs) try: pkg_main_name = mod_name + ".__main__" return _get_module_details(pkg_main_name, error) @@ -150,17 +153,19 @@ def _get_module_details(mod_name, error=ImportError): if mod_name not in sys.modules: raise # No module loaded; being a package is irrelevant raise error(("%s; %r is a package and cannot " + - "be directly executed") %(e, mod_name)) + "be directly executed") %(e, mod_name), + **kwargs) loader = spec.loader if loader is None: raise error("%r is a namespace package and cannot be executed" - % mod_name) + % mod_name, + **kwargs) try: code = loader.get_code(mod_name) except ImportError as e: - raise error(format(e)) from e + raise error(format(e), **kwargs) from e if code is None: - raise error("No code object available for %s" % mod_name) + raise error("No code object available for %s" % mod_name, **kwargs) return mod_name, spec, code class _Error(Exception): @@ -234,6 +239,7 @@ def _get_main_module_details(error=ImportError): # Also moves the standard __main__ out of the way so that the # preexisting __loader__ entry doesn't cause issues main_name = "__main__" + kwargs = {"name": main_name} if issubclass(error, ImportError) else {} saved_main = sys.modules[main_name] del sys.modules[main_name] try: @@ -241,7 +247,8 @@ def _get_main_module_details(error=ImportError): except ImportError as exc: if main_name in str(exc): raise error("can't find %r module in %r" % - (main_name, sys.path[0])) from exc + (main_name, sys.path[0]), + **kwargs) from exc raise finally: sys.modules[main_name] = saved_main diff --git a/Lib/shlex.py b/Lib/shlex.py index 5bf6e0d70e00121..5959f52dd12639d 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -322,6 +322,9 @@ def quote(s): if not s: return "''" + if not isinstance(s, str): + raise TypeError(f"expected string object, got {type(s).__name__!r}") + # Use bytes.translate() for performance safe_chars = (b'%+,-./0123456789:=@' b'ABCDEFGHIJKLMNOPQRSTUVWXYZ_' diff --git a/Lib/shutil.py b/Lib/shutil.py index ca0a2ea2f7fa8a0..a3a112f69541ef3 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -885,10 +885,14 @@ def move(src, dst, copy_function=copy2): If dst already exists but is not a directory, it may be overwritten depending on os.rename() semantics. - If the destination is on our current filesystem, then rename() is used. - Otherwise, src is copied to the destination and then removed. Symlinks are - recreated under the new name if os.rename() fails because of cross - filesystem renames. + os.rename() is preferably used if the source and destination are on the + same filesystem. In case os.rename() fails due to OSError (e.g. the user + has write permission to *dst* file but not to its parent directory), + this method falls back to using *copy_function* silently. + Symlinks are also recreated under the new name if os.rename() fails + because of cross filesystem renames. + + It's recommended to use os.rename() if atomic move is strictly required. The optional `copy_function` argument is a callable that will be used to copy the source or it will be delegated to `copytree`. @@ -940,8 +944,8 @@ def move(src, dst, copy_function=copy2): return real_dst def _destinsrc(src, dst): - src = os.path.abspath(src) - dst = os.path.abspath(dst) + src = os.path.realpath(src) + dst = os.path.realpath(dst) if not src.endswith(os.path.sep): src += os.path.sep if not dst.endswith(os.path.sep): @@ -994,14 +998,14 @@ def _make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0, """Create a (possibly compressed) tar file from all the files under 'base_dir'. - 'compress' must be "gzip" (the default), "bzip2", "xz", or None. + 'compress' must be "gzip" (the default), "bzip2", "xz", "zst", or None. 'owner' and 'group' can be used to define an owner and a group for the archive that is being built. If not provided, the current owner and group will be used. The output tar file will be named 'base_name' + ".tar", possibly plus - the appropriate compression extension (".gz", ".bz2", or ".xz"). + the appropriate compression extension (".gz", ".bz2", ".xz", or ".zst"). Returns the output filename. """ @@ -1187,7 +1191,7 @@ def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0, 'base_name' is the name of the file to create, minus any format-specific extension; 'format' is the archive format: one of "zip", "tar", "gztar", - "bztar", "zstdtar", or "xztar". Or any other registered format. + "bztar", "xztar", or "zstdtar". Or any other registered format. 'root_dir' is a directory that will be the root directory of the archive; ie. we typically chdir into 'root_dir' before creating the @@ -1314,30 +1318,12 @@ def _unpack_zipfile(filename, extract_dir): if not zipfile.is_zipfile(filename): raise ReadError("%s is not a zip file" % filename) - zip = zipfile.ZipFile(filename) - try: - for info in zip.infolist(): - name = info.filename - - # don't extract absolute paths or ones with .. in them - if name.startswith('/') or '..' in name: - continue - - targetpath = os.path.join(extract_dir, *name.split('/')) - if not targetpath: - continue - - _ensure_directory(targetpath) - if not name.endswith('/'): - # file - with zip.open(name, 'r') as source, \ - open(targetpath, 'wb') as target: - copyfileobj(source, target) - finally: - zip.close() + with zipfile.ZipFile(filename) as zip: + zip._ignore_invalid_names = True + zip.extractall(extract_dir) def _unpack_tarfile(filename, extract_dir, *, filter=None): - """Unpack tar/tar.gz/tar.bz2/tar.xz `filename` to `extract_dir` + """Unpack tar/tar.gz/tar.bz2/tar.xz/tar.zst `filename` to `extract_dir` """ import tarfile # late import for breaking circular dependency try: @@ -1392,7 +1378,7 @@ def unpack_archive(filename, extract_dir=None, format=None, *, filter=None): is unpacked. If not provided, the current working directory is used. `format` is the archive format: one of "zip", "tar", "gztar", "bztar", - or "xztar". Or any other registered format. If not provided, + "xztar", or "zstdtar". Or any other registered format. If not provided, unpack_archive will use the filename extension and see if an unpacker was registered for that extension. diff --git a/Lib/site.py b/Lib/site.py index 5c38b1b17d5abda..aeb7c6cfc71de99 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -75,6 +75,7 @@ import _sitebuiltins import _io as io import stat +import errno # Prefixes for site-packages; add additional prefixes like /usr/local here PREFIXES = [sys.prefix, sys.exec_prefix] @@ -448,9 +449,9 @@ def setcopyright(): """Set 'copyright' and 'credits' in builtins""" builtins.copyright = _sitebuiltins._Printer("copyright", sys.copyright) builtins.credits = _sitebuiltins._Printer("credits", """\ - Thanks to CWI, CNRI, BeOpen, Zope Corporation, the Python Software - Foundation, and a cast of thousands for supporting Python - development. See www.python.org for more information.""") +Thanks to CWI, CNRI, BeOpen, Zope Corporation, the Python Software +Foundation, and a cast of thousands for supporting Python +development. See www.python.org for more information.""") files, dirs = [], [] # Not all modules are required to have a __file__ attribute. See # PEP 420 for more details. @@ -578,10 +579,15 @@ def register_readline(): def write_history(): try: readline_module.write_history_file(history) - except (FileNotFoundError, PermissionError): + except FileNotFoundError, PermissionError: # home directory does not exist or is not writable # https://bugs.python.org/issue19891 pass + except OSError: + if errno.EROFS: + pass # gh-128066: read-only file system + else: + raise atexit.register(write_history) diff --git a/Lib/smtplib.py b/Lib/smtplib.py index 84d6d858e7dec14..4cfc2338d99c67e 100644 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -177,6 +177,15 @@ def _quote_periods(bindata): def _fix_eols(data): return re.sub(r'(?:\r\n|\n|\r(?!\n))', CRLF, data) + +try: + hmac.digest(b'', b'', 'md5') +except ValueError: + _have_cram_md5_support = False +else: + _have_cram_md5_support = True + + try: import ssl except ImportError: @@ -242,7 +251,6 @@ def __init__(self, host='', port=0, local_hostname=None, will be used. """ - self._host = host self.timeout = timeout self.esmtp_features = {} self.command_encoding = 'ascii' @@ -333,6 +341,7 @@ def connect(self, host='localhost', port=0, source_address=None): port = int(port) except ValueError: raise OSError("nonnumeric port") + self._host = host if not port: port = self.default_port sys.audit("smtplib.connect", self, host, port) @@ -665,8 +674,11 @@ def auth_cram_md5(self, challenge=None): # CRAM-MD5 does not support initial-response. if challenge is None: return None - return self.user + " " + hmac.HMAC( - self.password.encode('ascii'), challenge, 'md5').hexdigest() + if not _have_cram_md5_support: + raise SMTPException("CRAM-MD5 is not supported") + password = self.password.encode('ascii') + authcode = hmac.HMAC(password, challenge, 'md5') + return f"{self.user} {authcode.hexdigest()}" def auth_plain(self, challenge=None): """ Authobject to use with PLAIN authentication. Requires self.user and @@ -718,8 +730,10 @@ def login(self, user, password, *, initial_response_ok=True): advertised_authlist = self.esmtp_features["auth"].split() # Authentication methods we can handle in our preferred order: - preferred_auths = ['CRAM-MD5', 'PLAIN', 'LOGIN'] - + if _have_cram_md5_support: + preferred_auths = ['CRAM-MD5', 'PLAIN', 'LOGIN'] + else: + preferred_auths = ['PLAIN', 'LOGIN'] # We try the supported authentications in our preferred order, if # the server supports them. authlist = [auth for auth in preferred_auths @@ -903,7 +917,7 @@ def send_message(self, msg, from_addr=None, to_addrs=None, The arguments are as for sendmail, except that msg is an email.message.Message object. If from_addr is None or to_addrs is None, these arguments are taken from the headers of the Message as - described in RFC 2822 (a ValueError is raised if there is more than + described in RFC 5322 (a ValueError is raised if there is more than one set of 'Resent-' headers). Regardless of the values of from_addr and to_addr, any Bcc field (or Resent-Bcc field, when the Message is a resent) of the Message object won't be transmitted. The Message @@ -917,7 +931,7 @@ def send_message(self, msg, from_addr=None, to_addrs=None, policy. """ - # 'Resent-Date' is a mandatory field if the Message is resent (RFC 2822 + # 'Resent-Date' is a mandatory field if the Message is resent (RFC 5322 # Section 3.6.6). In such a case, we use the 'Resent-*' fields. However, # if there is more than one 'Resent-' block there's no way to # unambiguously determine which one is the most recent in all cases, @@ -936,7 +950,7 @@ def send_message(self, msg, from_addr=None, to_addrs=None, else: raise ValueError("message has more than one 'Resent-' header block") if from_addr is None: - # Prefer the sender field per RFC 2822:3.6.2. + # Prefer the sender field per RFC 5322 section 3.6.2. from_addr = (msg[header_prefix + 'Sender'] if (header_prefix + 'Sender') in msg else msg[header_prefix + 'From']) diff --git a/Lib/socket.py b/Lib/socket.py index 727b0e75f035955..9ac0badc6606c19 100644 --- a/Lib/socket.py +++ b/Lib/socket.py @@ -640,18 +640,22 @@ def _fallback_socketpair(family=AF_INET, type=SOCK_STREAM, proto=0): # Authenticating avoids using a connection from something else # able to connect to {host}:{port} instead of us. # We expect only AF_INET and AF_INET6 families. - try: - if ( - ssock.getsockname() != csock.getpeername() - or csock.getsockname() != ssock.getpeername() - ): - raise ConnectionError("Unexpected peer connection") - except: - # getsockname() and getpeername() can fail - # if either socket isn't connected. - ssock.close() - csock.close() - raise + # + # Note that we skip this on WASI because on that platorm the client socket + # may not have finished connecting by the time we've reached this point (gh-146139). + if sys.platform != "wasi": + try: + if ( + ssock.getsockname() != csock.getpeername() + or csock.getsockname() != ssock.getpeername() + ): + raise ConnectionError("Unexpected peer connection") + except: + # getsockname() and getpeername() can fail + # if either socket isn't connected. + ssock.close() + csock.close() + raise return (ssock, csock) @@ -887,7 +891,9 @@ def has_dualstack_ipv6(): try: with socket(AF_INET6, SOCK_STREAM) as sock: sock.setsockopt(IPPROTO_IPV6, IPV6_V6ONLY, 0) - return True + # On some platforms (e.g. DragonFly BSD) setting IPV6_V6ONLY to 0 + # silently has no effect, so check that it was actually cleared. + return sock.getsockopt(IPPROTO_IPV6, IPV6_V6ONLY) == 0 except error: return False diff --git a/Lib/sqlite3/__main__.py b/Lib/sqlite3/__main__.py index 002f1986cddb89b..4ccf292ddf211c2 100644 --- a/Lib/sqlite3/__main__.py +++ b/Lib/sqlite3/__main__.py @@ -48,17 +48,25 @@ def runsource(self, source, filename="", symbol="single"): Return True if more input is needed; buffering is done automatically. Return False if input is a complete statement ready for execution. """ - match source: - case ".version": - print(f"{sqlite3.sqlite_version}") - case ".help": - print("Enter SQL code and press enter.") - case ".quit": - sys.exit(0) - case _: - if not sqlite3.complete_statement(source): - return True - execute(self._cur, source) + if not source or source.isspace(): + return False + if source[0] == ".": + match source[1:].strip(): + case "version": + print(f"{sqlite3.sqlite_version}") + case "help": + print("Enter SQL code and press enter.") + case "quit": + sys.exit(0) + case "": + pass + case _ as unknown: + self.write("Error: unknown command or invalid arguments:" + f' "{unknown}".\n') + else: + if not sqlite3.complete_statement(source): + return True + execute(self._cur, source) return False diff --git a/Lib/sqlite3/dump.py b/Lib/sqlite3/dump.py index 57e6a3b4f1e6eba..95e14a6655ef172 100644 --- a/Lib/sqlite3/dump.py +++ b/Lib/sqlite3/dump.py @@ -11,8 +11,12 @@ def _quote_name(name): return '"{0}"'.format(name.replace('"', '""')) +def _escape_single_quotes(value): + return value.replace("'", "''") + + def _quote_value(value): - return "'{0}'".format(value.replace("'", "''")) + return "'{0}'".format(_escape_single_quotes(value)) def _iterdump(connection, *, filter=None): @@ -80,11 +84,12 @@ def _iterdump(connection, *, filter=None): table_name_ident = _quote_name(table_name) res = cu.execute(f'PRAGMA table_info({table_name_ident})') column_names = [str(table_info[1]) for table_info in res.fetchall()] - q = "SELECT 'INSERT INTO {0} VALUES('{1}')' FROM {0};".format( - table_name_ident, + q = "SELECT 'INSERT INTO {0} VALUES('{1}')' FROM {2};".format( + _escape_single_quotes(table_name_ident), "','".join( "||quote({0})||".format(_quote_name(col)) for col in column_names - ) + ), + table_name_ident, ) query_res = cu.execute(q) for row in query_res: diff --git a/Lib/ssl.py b/Lib/ssl.py index 05df4ad7f0f05cf..8889aff92fa6c0c 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -186,7 +186,7 @@ class _TLSContentType: class _TLSAlertType: """Alert types for TLSContentType.ALERT messages - See RFC 8466, section B.2 + See RFC 8446, section B.2 """ CLOSE_NOTIFY = 0 UNEXPECTED_MESSAGE = 10 diff --git a/Lib/stat.py b/Lib/stat.py index 1b4ed1ebc940ef2..81f694329bf4ff0 100644 --- a/Lib/stat.py +++ b/Lib/stat.py @@ -166,9 +166,14 @@ def filemode(mode): perm = [] for index, table in enumerate(_filemode_table): for bit, char in table: - if mode & bit == bit: - perm.append(char) - break + if index == 0: + if S_IFMT(mode) == bit: + perm.append(char) + break + else: + if mode & bit == bit: + perm.append(char) + break else: if index == 0: # Unknown filetype diff --git a/Lib/statistics.py b/Lib/statistics.py index 3d805cb073987d8..26cf925529ea60b 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -619,9 +619,14 @@ def stdev(data, xbar=None): if n < 2: raise StatisticsError('stdev requires at least two data points') mss = ss / (n - 1) + try: + mss_numerator = mss.numerator + mss_denominator = mss.denominator + except AttributeError: + raise ValueError('inf or nan encountered in data') if issubclass(T, Decimal): - return _decimal_sqrt_of_frac(mss.numerator, mss.denominator) - return _float_sqrt_of_frac(mss.numerator, mss.denominator) + return _decimal_sqrt_of_frac(mss_numerator, mss_denominator) + return _float_sqrt_of_frac(mss_numerator, mss_denominator) def pstdev(data, mu=None): @@ -637,9 +642,14 @@ def pstdev(data, mu=None): if n < 1: raise StatisticsError('pstdev requires at least one data point') mss = ss / n + try: + mss_numerator = mss.numerator + mss_denominator = mss.denominator + except AttributeError: + raise ValueError('inf or nan encountered in data') if issubclass(T, Decimal): - return _decimal_sqrt_of_frac(mss.numerator, mss.denominator) - return _float_sqrt_of_frac(mss.numerator, mss.denominator) + return _decimal_sqrt_of_frac(mss_numerator, mss_denominator) + return _float_sqrt_of_frac(mss_numerator, mss_denominator) ## Statistics for relations between two inputs ############################# diff --git a/Lib/string/__init__.py b/Lib/string/__init__.py index eab5067c9b133ea..b2c369ef9d1f76c 100644 --- a/Lib/string/__init__.py +++ b/Lib/string/__init__.py @@ -83,7 +83,14 @@ def __init_subclass__(cls): def _compile_pattern(cls): import re # deferred import, for performance + # `pattern` may be the `_TemplatePattern` sentinel (not yet compiled), an + # already-compiled regular expression object (as documented), or a string + # regular expression. An already-compiled object is returned as-is; the + # other two are compiled and cached back on the class. pattern = cls.__dict__.get('pattern', _TemplatePattern) + if isinstance(pattern, re.Pattern): + # re.compile() rejects flags on an already-compiled pattern. + return pattern if pattern is _TemplatePattern: delim = re.escape(cls.delimiter) id = cls.idpattern diff --git a/Lib/string/templatelib.py b/Lib/string/templatelib.py index 14b40e1e36e30b3..8164872432ad09e 100644 --- a/Lib/string/templatelib.py +++ b/Lib/string/templatelib.py @@ -1,15 +1,22 @@ """Support for template string literals (t-strings).""" -__all__ = [ - "Interpolation", - "Template", -] - t = t"{0}" Template = type(t) Interpolation = type(t.interpolations[0]) del t +def convert(obj, /, conversion): + """Convert *obj* using formatted string literal semantics.""" + if conversion is None: + return obj + if conversion == 'r': + return repr(obj) + if conversion == 's': + return str(obj) + if conversion == 'a': + return ascii(obj) + raise ValueError(f'invalid conversion specifier: {conversion}') + def _template_unpickle(*args): import itertools diff --git a/Lib/subprocess.py b/Lib/subprocess.py index 54c2eb515b60daf..52b7b7117703d71 100644 --- a/Lib/subprocess.py +++ b/Lib/subprocess.py @@ -351,15 +351,16 @@ def _args_from_interpreter_flags(): # -X options if dev_mode: args.extend(('-X', 'dev')) - for opt in ('faulthandler', 'tracemalloc', 'importtime', - 'frozen_modules', 'showrefcount', 'utf8', 'gil'): - if opt in xoptions: - value = xoptions[opt] - if value is True: - arg = opt - else: - arg = '%s=%s' % (opt, value) - args.extend(('-X', arg)) + for opt in sorted(xoptions): + if opt == 'dev': + # handled above via sys.flags.dev_mode + continue + value = xoptions[opt] + if value is True: + arg = opt + else: + arg = '%s=%s' % (opt, value) + args.extend(('-X', arg)) return args @@ -1614,6 +1615,10 @@ def _readerthread(self, fh, buffer): fh.close() + def _writerthread(self, input): + self._stdin_write(input) + + def _communicate(self, input, endtime, orig_timeout): # Start reader threads feeding into a list hanging off of this # object, unless they've already been started. @@ -1632,8 +1637,23 @@ def _communicate(self, input, endtime, orig_timeout): self.stderr_thread.daemon = True self.stderr_thread.start() - if self.stdin: - self._stdin_write(input) + # Start writer thread to send input to stdin, unless already + # started. The thread writes input and closes stdin when done, + # or continues in the background on timeout. + if self.stdin and not hasattr(self, "_stdin_thread"): + self._stdin_thread = \ + threading.Thread(target=self._writerthread, + args=(input,)) + self._stdin_thread.daemon = True + self._stdin_thread.start() + + # Wait for the writer thread, or time out. If we time out, the + # thread remains writing and the fd left open in case the user + # calls communicate again. + if hasattr(self, "_stdin_thread"): + self._stdin_thread.join(self._remaining_time(endtime)) + if self._stdin_thread.is_alive(): + raise TimeoutExpired(self.args, orig_timeout) # Wait for the reader threads, or time out. If we time out, the # threads remain reading and the fds left open in case the user @@ -2078,6 +2098,10 @@ def _communicate(self, input, endtime, orig_timeout): self.stdin.flush() except BrokenPipeError: pass # communicate() must ignore BrokenPipeError. + except ValueError: + # ignore ValueError: I/O operation on closed file. + if not self.stdin.closed: + raise if not input: try: self.stdin.close() @@ -2103,10 +2127,13 @@ def _communicate(self, input, endtime, orig_timeout): self._save_input(input) if self._input: - input_view = memoryview(self._input) + if not isinstance(self._input, memoryview): + input_view = memoryview(self._input) + else: + input_view = self._input.cast("b") # byte input required with _PopenSelector() as selector: - if self.stdin and input: + if self.stdin and not self.stdin.closed and self._input: selector.register(self.stdin, selectors.EVENT_WRITE) if self.stdout and not self.stdout.closed: selector.register(self.stdout, selectors.EVENT_READ) @@ -2115,7 +2142,7 @@ def _communicate(self, input, endtime, orig_timeout): while selector.get_map(): timeout = self._remaining_time(endtime) - if timeout is not None and timeout < 0: + if timeout is not None and timeout <= 0: self._check_timeout(endtime, orig_timeout, stdout, stderr, skip_check_and_raise=True) @@ -2139,7 +2166,7 @@ def _communicate(self, input, endtime, orig_timeout): selector.unregister(key.fileobj) key.fileobj.close() else: - if self._input_offset >= len(self._input): + if self._input_offset >= len(input_view): selector.unregister(key.fileobj) key.fileobj.close() elif key.fileobj in (self.stdout, self.stderr): diff --git a/Lib/sysconfig/__init__.py b/Lib/sysconfig/__init__.py index dad715eb0873872..faf8273bd0b31f3 100644 --- a/Lib/sysconfig/__init__.py +++ b/Lib/sysconfig/__init__.py @@ -468,7 +468,7 @@ def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: if os.name == "nt": - inc_dir = os.path.dirname(sys._base_executable) + inc_dir = os.path.join(_PROJECT_BASE, 'PC') else: inc_dir = _PROJECT_BASE else: @@ -650,18 +650,22 @@ def get_platform(): isn't particularly important. Examples of returned values: - linux-i586 - linux-alpha (?) - solaris-2.6-sun4u - Windows will return one of: - win-amd64 (64-bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) - win-arm64 (64-bit Windows on ARM64 (aka AArch64) - win32 (all others - specifically, sys.platform is returned) - For other non-POSIX platforms, currently just returns 'sys.platform'. + Windows: - """ + - win-amd64 (64-bit Windows on AMD64, aka x86_64, Intel64, and EM64T) + - win-arm64 (64-bit Windows on ARM64, aka AArch64) + - win32 (all others - specifically, sys.platform is returned) + + POSIX based OS: + + - linux-x86_64 + - macosx-15.5-arm64 + - macosx-26.0-universal2 (macOS on Apple Silicon or Intel) + - android-24-arm64_v8a + + For other non-POSIX platforms, currently just returns :data:`sys.platform`.""" if os.name == 'nt': if 'amd64' in sys.version.lower(): return 'win-amd64' @@ -694,11 +698,19 @@ def get_platform(): release = get_config_var("ANDROID_API_LEVEL") # Wheel tags use the ABI names from Android's own tools. + # When Python is running on 32-bit ARM Android on a 64-bit ARM kernel, + # 'os.uname().machine' is 'armv8l'. Such devices run the same userspace + # code as 'armv7l' devices. + # During the build process of the Android testbed when targeting 32-bit ARM, + # '_PYTHON_HOST_PLATFORM' is 'arm-linux-androideabi', so 'machine' becomes + # 'arm'. machine = { - "x86_64": "x86_64", - "i686": "x86", "aarch64": "arm64_v8a", + "arm": "armeabi_v7a", "armv7l": "armeabi_v7a", + "armv8l": "armeabi_v7a", + "i686": "x86", + "x86_64": "x86_64", }[machine] elif osname == "linux": # At least on Linux/Intel, 'machine' is the processor -- diff --git a/Lib/tarfile.py b/Lib/tarfile.py index c0f5a609b9f42f3..d3c489997006e7a 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -67,7 +67,7 @@ "DEFAULT_FORMAT", "open","fully_trusted_filter", "data_filter", "tar_filter", "FilterError", "AbsoluteLinkError", "OutsideDestinationError", "SpecialFileError", "AbsolutePathError", - "LinkOutsideDestinationError"] + "LinkOutsideDestinationError", "LinkFallbackError"] #--------------------------------------------------------- @@ -258,6 +258,32 @@ def copyfileobj(src, dst, length=None, exception=OSError, bufsize=None): dst.write(buf) return +# Maximum number of bytes read in a single call when reading a member's +# extended header (a GNU long name/link or a pax header). The size of such +# a header is taken from the archive and is not trustworthy, so it is read in +# bounded chunks to avoid a huge up-front allocation when a crafted or +# truncated archive claims far more data than the file actually contains +# (gh-151497). +_EXTHEADER_READ_CHUNK = 1024 * 1024 # 1 MiB + +def _safe_read(fileobj, size): + """Read up to *size* bytes from *fileobj* in bounded chunks. + + Returns the same bytes as ``fileobj.read(size)`` would (including a short + result at end of file), but limits pre-allocation, so an + oversized size field in a crafted header cannot force a huge allocation. + """ + if size <= _EXTHEADER_READ_CHUNK: + return fileobj.read(size) + chunks = [] + while size > 0: + chunk = fileobj.read(min(size, _EXTHEADER_READ_CHUNK)) + if not chunk: + break + chunks.append(chunk) + size -= len(chunk) + return b"".join(chunks) + def _safe_print(s): encoding = getattr(sys.stdout, 'encoding', None) if encoding is not None: @@ -353,7 +379,7 @@ def __init__(self, name, mode, comptype, fileobj, bufsize, fileobj = _StreamProxy(fileobj) comptype = fileobj.getcomptype() - self.name = name or "" + self.name = os.fspath(name) if name is not None else "" self.mode = mode self.comptype = comptype self.fileobj = fileobj @@ -498,7 +524,7 @@ def _init_read_gz(self): if flag & 4: xlen = ord(self.__read(1)) + 256 * ord(self.__read(1)) - self.read(xlen) + self.__read(xlen) if flag & 8: while True: s = self.__read(1) @@ -524,7 +550,9 @@ def seek(self, pos=0): if pos - self.pos >= 0: blocks, remainder = divmod(pos - self.pos, self.bufsize) for i in range(blocks): - self.read(self.bufsize) + data = self.read(self.bufsize) + if not data: + break self.read(remainder) else: raise StreamError("seeking backwards is not allowed") @@ -766,10 +794,22 @@ def __init__(self, tarinfo, path): super().__init__(f'{tarinfo.name!r} would link to {path!r}, ' + 'which is outside the destination') +class LinkFallbackError(FilterError): + def __init__(self, tarinfo, path): + self.tarinfo = tarinfo + self._path = path + super().__init__(f'link {tarinfo.name!r} would be extracted as a ' + + f'copy of {path!r}, which was rejected') + +# Errors caused by filters -- both "fatal" and "non-fatal" -- that +# we consider to be issues with the argument, rather than a bug in the +# filter function +_FILTER_ERRORS = (FilterError, OSError, ExtractError) + def _get_filtered_attrs(member, dest_path, for_data=True): new_attrs = {} name = member.name - dest_path = os.path.realpath(dest_path) + dest_path = os.path.realpath(dest_path, strict=os.path.ALLOW_MISSING) # Strip leading / (tar's directory separator) from filenames. # Include os.sep (target OS directory separator) as well. if name.startswith(('/', os.sep)): @@ -779,7 +819,8 @@ def _get_filtered_attrs(member, dest_path, for_data=True): # For example, 'C:/foo' on Windows. raise AbsolutePathError(member) # Ensure we stay in the destination - target_path = os.path.realpath(os.path.join(dest_path, name)) + target_path = os.path.realpath(os.path.join(dest_path, name), + strict=os.path.ALLOW_MISSING) if os.path.commonpath([target_path, dest_path]) != dest_path: raise OutsideDestinationError(member, target_path) # Limit permissions (no high bits, and go-w) @@ -817,14 +858,24 @@ def _get_filtered_attrs(member, dest_path, for_data=True): if member.islnk() or member.issym(): if os.path.isabs(member.linkname): raise AbsoluteLinkError(member) + # A link member that resolves to the destination directory itself + # would replace it with a (sym)link, redirecting the destination + # for all subsequent members. + if target_path == dest_path: + raise OutsideDestinationError(member, target_path) + normalized = os.path.normpath(member.linkname) + if normalized != member.linkname: + new_attrs['linkname'] = normalized if member.issym(): - target_path = os.path.join(dest_path, - os.path.dirname(name), - member.linkname) + # The symlink is created at `name` with trailing separators + # stripped, so its target is relative to the directory + # containing that path. + link_dir = os.path.dirname(name.rstrip('/' + os.sep)) + target_path = os.path.join(dest_path, link_dir, normalized) else: - target_path = os.path.join(dest_path, - member.linkname) - target_path = os.path.realpath(target_path) + target_path = os.path.join(dest_path, normalized) + target_path = os.path.realpath(target_path, + strict=os.path.ALLOW_MISSING) if os.path.commonpath([target_path, dest_path]) != dest_path: raise LinkOutsideDestinationError(member, target_path) return new_attrs @@ -876,11 +927,14 @@ class TarInfo(object): size = 'Size in bytes.', mtime = 'Time of last modification.', chksum = 'Header checksum.', - type = ('File type. type is usually one of these constants: ' - 'REGTYPE, AREGTYPE, LNKTYPE, SYMTYPE, DIRTYPE, FIFOTYPE, ' - 'CONTTYPE, CHRTYPE, BLKTYPE, GNUTYPE_SPARSE.'), + type = ('File type. type is usually one of these constants: ' + 'REGTYPE,\n' + 'AREGTYPE, LNKTYPE, SYMTYPE, DIRTYPE, FIFOTYPE, ' + 'CONTTYPE, CHRTYPE,\n' + 'BLKTYPE, GNUTYPE_SPARSE.'), linkname = ('Name of the target file name, which is only present ' - 'in TarInfo objects of type LNKTYPE and SYMTYPE.'), + 'in TarInfo\n' + 'objects of type LNKTYPE and SYMTYPE.'), uname = 'User name.', gname = 'Group name.', devmajor = 'Device major number.', @@ -888,7 +942,8 @@ class TarInfo(object): offset = 'The tar header starts here.', offset_data = "The file's data starts here.", pax_headers = ('A dictionary containing key-value pairs of an ' - 'associated pax extended header.'), + 'associated pax\n' + 'extended header.'), sparse = 'Sparse member information.', _tarfile = None, _sparse_structs = None, @@ -1261,6 +1316,20 @@ def _create_pax_generic_header(cls, pax_headers, type, encoding): @classmethod def frombuf(cls, buf, encoding, errors): """Construct a TarInfo object from a 512 byte bytes object. + + To support the old v7 tar format AREGTYPE headers are + transformed to DIRTYPE headers if their name ends in '/'. + """ + return cls._frombuf(buf, encoding, errors) + + @classmethod + def _frombuf(cls, buf, encoding, errors, *, dircheck=True): + """Construct a TarInfo object from a 512 byte bytes object. + + If ``dircheck`` is set to ``True`` then ``AREGTYPE`` headers will + be normalized to ``DIRTYPE`` if the name ends in a trailing slash. + ``dircheck`` must be set to ``False`` if this function is called + on a follow-up header such as ``GNUTYPE_LONGNAME``. """ if len(buf) == 0: raise EmptyHeaderError("empty header") @@ -1291,7 +1360,7 @@ def frombuf(cls, buf, encoding, errors): # Old V7 tar format represents a directory as a regular # file with a trailing slash. - if obj.type == AREGTYPE and obj.name.endswith("/"): + if dircheck and obj.type == AREGTYPE and obj.name.endswith("/"): obj.type = DIRTYPE # The old GNU sparse format occupies some of the unused @@ -1326,8 +1395,15 @@ def fromtarfile(cls, tarfile): """Return the next TarInfo object from TarFile object tarfile. """ + return cls._fromtarfile(tarfile) + + @classmethod + def _fromtarfile(cls, tarfile, *, dircheck=True): + """ + See dircheck documentation in _frombuf(). + """ buf = tarfile.fileobj.read(BLOCKSIZE) - obj = cls.frombuf(buf, tarfile.encoding, tarfile.errors) + obj = cls._frombuf(buf, tarfile.encoding, tarfile.errors, dircheck=dircheck) obj.offset = tarfile.fileobj.tell() - BLOCKSIZE return obj._proc_member(tarfile) @@ -1381,11 +1457,11 @@ def _proc_gnulong(self, tarfile): """Process the blocks that hold a GNU longname or longlink member. """ - buf = tarfile.fileobj.read(self._block(self.size)) + buf = _safe_read(tarfile.fileobj, self._block(self.size)) # Fetch the next header and process it. try: - next = self.fromtarfile(tarfile) + next = self._fromtarfile(tarfile, dircheck=False) except HeaderError as e: raise SubsequentHeaderError(str(e)) from None @@ -1437,7 +1513,7 @@ def _proc_pax(self, tarfile): POSIX.1-2008. """ # Read the header information. - buf = tarfile.fileobj.read(self._block(self.size)) + buf = _safe_read(tarfile.fileobj, self._block(self.size)) # A pax header stores supplemental information for either # the following file (extended) or all following files @@ -1520,7 +1596,7 @@ def _proc_pax(self, tarfile): # Fetch the next header. try: - next = self.fromtarfile(tarfile) + next = self._fromtarfile(tarfile, dircheck=False) except HeaderError as e: raise SubsequentHeaderError(str(e)) from None @@ -1630,6 +1706,9 @@ def _block(self, count): """Round up a byte count by BLOCKSIZE and return it, e.g. _block(834) => 1024. """ + # Only non-negative offsets are allowed + if count < 0: + raise InvalidHeaderError("invalid offset") blocks, remainder = divmod(count, BLOCKSIZE) if remainder: blocks += 1 @@ -2065,7 +2144,7 @@ def zstopen(cls, name, mode="r", fileobj=None, level=None, options=None, "gz": "gzopen", # gzip compressed tar "bz2": "bz2open", # bzip2 compressed tar "xz": "xzopen", # lzma compressed tar - "zst": "zstopen" # zstd compressed tar + "zst": "zstopen", # zstd compressed tar } #-------------------------------------------------------------------------- @@ -2226,10 +2305,11 @@ def gettarinfo(self, name=None, arcname=None, fileobj=None): return tarinfo def list(self, verbose=True, *, members=None): - """Print a table of contents to sys.stdout. If 'verbose' is False, only - the names of the members are printed. If it is True, an 'ls -l'-like - output is produced. 'members' is optional and must be a subset of the - list returned by getmembers(). + """Print a table of contents to sys.stdout. + + If 'verbose' is False, only the names of the members are printed. + If it is True, an 'ls -l'-like output is produced. 'members' is + optional and must be a subset of the list returned by getmembers(). """ # Convert tarinfo type to stat type. type2mode = {REGTYPE: stat.S_IFREG, SYMTYPE: stat.S_IFLNK, @@ -2320,10 +2400,12 @@ def add(self, name, arcname=None, recursive=True, *, filter=None): self.addfile(tarinfo) def addfile(self, tarinfo, fileobj=None): - """Add the TarInfo object 'tarinfo' to the archive. If 'tarinfo' represents - a non zero-size regular file, the 'fileobj' argument should be a binary file, - and tarinfo.size bytes are read from it and added to the archive. - You can create TarInfo objects directly, or by using gettarinfo(). + """Add the TarInfo object 'tarinfo' to the archive. + + If 'tarinfo' represents a non zero-size regular file, the 'fileobj' + argument should be a binary file, and tarinfo.size bytes are read + from it and added to the archive. You can create TarInfo objects + directly, or by using gettarinfo(). """ self._check("awx") @@ -2386,30 +2468,58 @@ def extractall(self, path=".", members=None, *, numeric_owner=False, members = self for member in members: - tarinfo = self._get_extract_tarinfo(member, filter_function, path) + tarinfo, unfiltered = self._get_extract_tarinfo( + member, filter_function, path) if tarinfo is None: continue if tarinfo.isdir(): # For directories, delay setting attributes until later, # since permissions can interfere with extraction and # extracting contents can reset mtime. - directories.append(tarinfo) + directories.append(unfiltered) self._extract_one(tarinfo, path, set_attrs=not tarinfo.isdir(), - numeric_owner=numeric_owner) + numeric_owner=numeric_owner, + filter_function=filter_function) # Reverse sort directories. directories.sort(key=lambda a: a.name, reverse=True) + # Set correct owner, mtime and filemode on directories. - for tarinfo in directories: - dirpath = os.path.join(path, tarinfo.name) + for unfiltered in directories: try: + # Need to re-apply any filter, to take the *current* filesystem + # state into account. + try: + tarinfo = filter_function(unfiltered, path) + except _FILTER_ERRORS as exc: + self._log_no_directory_fixup(unfiltered, repr(exc)) + continue + if tarinfo is None: + self._log_no_directory_fixup(unfiltered, + 'excluded by filter') + continue + dirpath = os.path.join(path, tarinfo.name) + try: + lstat = os.lstat(dirpath) + except FileNotFoundError: + self._log_no_directory_fixup(tarinfo, 'missing') + continue + if not stat.S_ISDIR(lstat.st_mode): + # This is no longer a directory; presumably a later + # member overwrote the entry. + self._log_no_directory_fixup(tarinfo, 'not a directory') + continue self.chown(tarinfo, dirpath, numeric_owner=numeric_owner) self.utime(tarinfo, dirpath) self.chmod(tarinfo, dirpath) except ExtractError as e: self._handle_nonfatal_error(e) + def _log_no_directory_fixup(self, member, reason): + self._dbg(2, "tarfile: Not fixing up directory %r (%s)" % + (member.name, reason)) + def extract(self, member, path="", set_attrs=True, *, numeric_owner=False, filter=None): """Extract a member from the archive to the current working directory, @@ -2425,42 +2535,58 @@ def extract(self, member, path="", set_attrs=True, *, numeric_owner=False, String names of common filters are accepted. """ filter_function = self._get_filter_function(filter) - tarinfo = self._get_extract_tarinfo(member, filter_function, path) + tarinfo, unfiltered = self._get_extract_tarinfo( + member, filter_function, path) if tarinfo is not None: - self._extract_one(tarinfo, path, set_attrs, numeric_owner) + self._extract_one(tarinfo, path, set_attrs, numeric_owner, + filter_function=filter_function) def _get_extract_tarinfo(self, member, filter_function, path): - """Get filtered TarInfo (or None) from member, which might be a str""" + """Get (filtered, unfiltered) TarInfos from *member* + + *member* might be a string. + + Return (None, None) if not found. + """ + if isinstance(member, str): - tarinfo = self.getmember(member) + unfiltered = self.getmember(member) else: - tarinfo = member + unfiltered = member - unfiltered = tarinfo + filtered = None try: - tarinfo = filter_function(tarinfo, path) - except (OSError, FilterError) as e: + filtered = filter_function(unfiltered, path) + except (OSError, UnicodeEncodeError, FilterError) as e: self._handle_fatal_error(e) except ExtractError as e: self._handle_nonfatal_error(e) - if tarinfo is None: + if filtered is None: self._dbg(2, "tarfile: Excluded %r" % unfiltered.name) - return None + return None, None + # Prepare the link target for makelink(). - if tarinfo.islnk(): - tarinfo = copy.copy(tarinfo) - tarinfo._link_target = os.path.join(path, tarinfo.linkname) - return tarinfo + if filtered.islnk(): + filtered = copy.copy(filtered) + filtered._link_target = os.path.join(path, filtered.linkname) + return filtered, unfiltered + + def _extract_one(self, tarinfo, path, set_attrs, numeric_owner, + filter_function=None): + """Extract from filtered tarinfo to disk. - def _extract_one(self, tarinfo, path, set_attrs, numeric_owner): - """Extract from filtered tarinfo to disk""" + filter_function is only used when extracting a *different* + member (e.g. as fallback to creating a symlink) + """ self._check("r") try: self._extract_member(tarinfo, os.path.join(path, tarinfo.name), set_attrs=set_attrs, - numeric_owner=numeric_owner) - except OSError as e: + numeric_owner=numeric_owner, + filter_function=filter_function, + extraction_root=path) + except (OSError, UnicodeEncodeError) as e: self._handle_fatal_error(e) except ExtractError as e: self._handle_nonfatal_error(e) @@ -2517,9 +2643,13 @@ def extractfile(self, member): return None def _extract_member(self, tarinfo, targetpath, set_attrs=True, - numeric_owner=False): - """Extract the TarInfo object tarinfo to a physical + numeric_owner=False, *, filter_function=None, + extraction_root=None): + """Extract the filtered TarInfo object tarinfo to a physical file called targetpath. + + filter_function is only used when extracting a *different* + member (e.g. as fallback to creating a symlink) """ # Fetch the TarInfo object for the given name # and build the destination pathname, replacing @@ -2548,7 +2678,10 @@ def _extract_member(self, tarinfo, targetpath, set_attrs=True, elif tarinfo.ischr() or tarinfo.isblk(): self.makedev(tarinfo, targetpath) elif tarinfo.islnk() or tarinfo.issym(): - self.makelink(tarinfo, targetpath) + self.makelink_with_filter( + tarinfo, targetpath, + filter_function=filter_function, + extraction_root=extraction_root) elif tarinfo.type not in SUPPORTED_TYPES: self.makeunknown(tarinfo, targetpath) else: @@ -2631,10 +2764,18 @@ def makedev(self, tarinfo, targetpath): os.makedev(tarinfo.devmajor, tarinfo.devminor)) def makelink(self, tarinfo, targetpath): + return self.makelink_with_filter(tarinfo, targetpath, None, None) + + def makelink_with_filter(self, tarinfo, targetpath, + filter_function, extraction_root): """Make a (symbolic) link called targetpath. If it cannot be created (platform limitation), we try to make a copy of the referenced file instead of a link. + + filter_function is only used when extracting a *different* + member (e.g. as fallback to creating a link). """ + keyerror_to_extracterror = False try: # For systems that support symbolic and hard links. if tarinfo.issym(): @@ -2642,18 +2783,44 @@ def makelink(self, tarinfo, targetpath): # Avoid FileExistsError on following os.symlink. os.unlink(targetpath) os.symlink(tarinfo.linkname, targetpath) + return else: if os.path.exists(tarinfo._link_target): + if os.path.lexists(targetpath): + # Avoid FileExistsError on following os.link. + os.unlink(targetpath) os.link(tarinfo._link_target, targetpath) - else: - self._extract_member(self._find_link_target(tarinfo), - targetpath) + return except symlink_exception: + keyerror_to_extracterror = True + + try: + unfiltered = self._find_link_target(tarinfo) + except KeyError: + if keyerror_to_extracterror: + raise ExtractError( + "unable to resolve link inside archive") from None + else: + raise + + if filter_function is None: + filtered = unfiltered + else: + if extraction_root is None: + raise ExtractError( + "makelink_with_filter: if filter_function is not None, " + + "extraction_root must also not be None") try: - self._extract_member(self._find_link_target(tarinfo), - targetpath) - except KeyError: - raise ExtractError("unable to resolve link inside archive") from None + filter_function( + unfiltered.replace(name=tarinfo.name, deep=False), + extraction_root) + filtered = filter_function(unfiltered, extraction_root) + except _FILTER_ERRORS as cause: + raise LinkFallbackError(tarinfo, unfiltered.name) from cause + if filtered is not None: + self._extract_member(filtered, targetpath, + filter_function=filter_function, + extraction_root=extraction_root) def chown(self, tarinfo, targetpath, numeric_owner): """Set owner of targetpath according to tarinfo. If numeric_owner diff --git a/Lib/tempfile.py b/Lib/tempfile.py index cadb0bed3cce3b0..6c036d2fd51cb0f 100644 --- a/Lib/tempfile.py +++ b/Lib/tempfile.py @@ -57,10 +57,11 @@ if hasattr(_os, 'O_BINARY'): _bin_openflags |= _os.O_BINARY -if hasattr(_os, 'TMP_MAX'): - TMP_MAX = _os.TMP_MAX -else: - TMP_MAX = 10000 +# This is more than enough. +# Each name contains over 40 random bits. Even with a million temporary +# files, the chance of a conflict is less than 1 in a million, and with +# 20 attempts, it is less than 1e-120. +TMP_MAX = 20 # This variable _was_ unused for legacy reasons, see issue 10354. # But as of 3.5 we actually use it at runtime so changing it would @@ -180,7 +181,7 @@ def _candidate_tempdir_list(): return dirlist -def _get_default_tempdir(): +def _get_default_tempdir(dirlist=None): """Calculate the default directory to use for temporary files. This routine should be called exactly once. @@ -190,13 +191,13 @@ def _get_default_tempdir(): service, the name of the test file must be randomized.""" namer = _RandomNameSequence() - dirlist = _candidate_tempdir_list() + if dirlist is None: + dirlist = _candidate_tempdir_list() for dir in dirlist: if dir != _os.curdir: dir = _os.path.abspath(dir) - # Try only a few names per directory. - for seq in range(100): + for seq in range(TMP_MAX): name = next(namer) filename = _os.path.join(dir, name) try: @@ -212,10 +213,8 @@ def _get_default_tempdir(): except FileExistsError: pass except PermissionError: - # This exception is thrown when a directory with the chosen name - # already exists on windows. - if (_os.name == 'nt' and _os.path.isdir(dir) and - _os.access(dir, _os.W_OK)): + # See the comment in mkdtemp(). + if _os.name == 'nt' and _os.path.isdir(dir): continue break # no point trying more names in this directory except OSError: @@ -257,10 +256,8 @@ def _mkstemp_inner(dir, pre, suf, flags, output_type): except FileExistsError: continue # try again except PermissionError: - # This exception is thrown when a directory with the chosen name - # already exists on windows. - if (_os.name == 'nt' and _os.path.isdir(dir) and - _os.access(dir, _os.W_OK)): + # See the comment in mkdtemp(). + if _os.name == 'nt' and _os.path.isdir(dir) and seq < TMP_MAX - 1: continue else: raise @@ -385,10 +382,14 @@ def mkdtemp(suffix=None, prefix=None, dir=None): except FileExistsError: continue # try again except PermissionError: - # This exception is thrown when a directory with the chosen name - # already exists on windows. - if (_os.name == 'nt' and _os.path.isdir(dir) and - _os.access(dir, _os.W_OK)): + # On Posix, this exception is raised when the user has no + # write access to the parent directory. + # On Windows, it is also raised when a directory with + # the chosen name already exists, or if the parent directory + # is not a directory. + # We cannot distinguish between "directory-exists-error" and + # "access-denied-error". + if _os.name == 'nt' and _os.path.isdir(dir) and seq < TMP_MAX - 1: continue else: raise @@ -914,7 +915,8 @@ def __init__(self, suffix=None, prefix=None, dir=None, @classmethod def _rmtree(cls, name, ignore_errors=False, repeated=False): def onexc(func, path, exc): - if isinstance(exc, PermissionError): + # On DragonFly BSD, UF_NOUNLINK removal fails with EISDIR, not EPERM. + if isinstance(exc, (PermissionError, IsADirectoryError)): if repeated and path == name: if ignore_errors: return diff --git a/Lib/test/.ruff.toml b/Lib/test/.ruff.toml index a1eac32a83aae37..f6a4dc631c76b66 100644 --- a/Lib/test/.ruff.toml +++ b/Lib/test/.ruff.toml @@ -1,5 +1,8 @@ extend = "../../.ruff.toml" # Inherit the project-wide settings +# Unlike Tools/, tests can use newer syntax than PYTHON_FOR_REGEN +target-version = "py314" + extend-exclude = [ # Excluded (run with the other AC files in its own separate ruff job in pre-commit) "test_clinic.py", @@ -8,9 +11,6 @@ extend-exclude = [ # Non UTF-8 files "encoded_modules/module_iso_8859_1.py", "encoded_modules/module_koi8_r.py", - # SyntaxError because of t-strings - "test_tstring.py", - "test_string/test_templatelib.py", # New grammar constructions may not yet be recognized by Ruff, # and tests re-use the same names as only the grammar is being checked. "test_grammar.py", diff --git a/Lib/test/NormalizationTest-3.2.0.txt b/Lib/test/NormalizationTest-3.2.0.txt new file mode 100644 index 000000000000000..d1c7b5165e3618c --- /dev/null +++ b/Lib/test/NormalizationTest-3.2.0.txt @@ -0,0 +1,17035 @@ +# NormalizationTest-3.2.0.txt +# Date: 2002-03-19,23:31:18 GMT [MD] +# +# Normalization Test Suite +# Format: +# +# Columns (c1, c2,...) are separated by semicolons +# Comments are indicated with hash marks +# +# CONFORMANCE: +# 1. The following invariants must be true for all conformant implementations +# +# NFC +# c2 == NFC(c1) == NFC(c2) == NFC(c3) +# c4 == NFC(c4) == NFC(c5) +# +# NFD +# c3 == NFD(c1) == NFD(c2) == NFD(c3) +# c5 == NFD(c4) == NFD(c5) +# +# NFKC +# c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5) +# +# NFKD +# c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5) +# +# 2. For every assigned Unicode 3.1.0 code point X that is not specifically +# listed in Part 1, the following invariants must be true for all conformant +# implementations: +# +# X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X) +# +@Part0 # Specific cases +# +1E0A;1E0A;0044 0307;1E0A;0044 0307; # (Ḋ; Ḋ; D◌̇; Ḋ; D◌̇; ) LATIN CAPITAL LETTER D WITH DOT ABOVE +1E0C;1E0C;0044 0323;1E0C;0044 0323; # (Ḍ; Ḍ; D◌̣; Ḍ; D◌̣; ) LATIN CAPITAL LETTER D WITH DOT BELOW +1E0A 0323;1E0C 0307;0044 0323 0307;1E0C 0307;0044 0323 0307; # (Ḋ◌̣; Ḍ◌̇; D◌̣◌̇; Ḍ◌̇; D◌̣◌̇; ) LATIN CAPITAL LETTER D WITH DOT ABOVE, COMBINING DOT BELOW +1E0C 0307;1E0C 0307;0044 0323 0307;1E0C 0307;0044 0323 0307; # (Ḍ◌̇; Ḍ◌̇; D◌̣◌̇; Ḍ◌̇; D◌̣◌̇; ) LATIN CAPITAL LETTER D WITH DOT BELOW, COMBINING DOT ABOVE +0044 0307 0323;1E0C 0307;0044 0323 0307;1E0C 0307;0044 0323 0307; # (D◌̇◌̣; Ḍ◌̇; D◌̣◌̇; Ḍ◌̇; D◌̣◌̇; ) LATIN CAPITAL LETTER D, COMBINING DOT ABOVE, COMBINING DOT BELOW +0044 0323 0307;1E0C 0307;0044 0323 0307;1E0C 0307;0044 0323 0307; # (D◌̣◌̇; Ḍ◌̇; D◌̣◌̇; Ḍ◌̇; D◌̣◌̇; ) LATIN CAPITAL LETTER D, COMBINING DOT BELOW, COMBINING DOT ABOVE +1E0A 031B;1E0A 031B;0044 031B 0307;1E0A 031B;0044 031B 0307; # (Ḋ◌̛; Ḋ◌̛; D◌̛◌̇; Ḋ◌̛; D◌̛◌̇; ) LATIN CAPITAL LETTER D WITH DOT ABOVE, COMBINING HORN +1E0C 031B;1E0C 031B;0044 031B 0323;1E0C 031B;0044 031B 0323; # (Ḍ◌̛; Ḍ◌̛; D◌̛◌̣; Ḍ◌̛; D◌̛◌̣; ) LATIN CAPITAL LETTER D WITH DOT BELOW, COMBINING HORN +1E0A 031B 0323;1E0C 031B 0307;0044 031B 0323 0307;1E0C 031B 0307;0044 031B 0323 0307; # (Ḋ◌̛◌̣; Ḍ◌̛◌̇; D◌̛◌̣◌̇; Ḍ◌̛◌̇; D◌̛◌̣◌̇; ) LATIN CAPITAL LETTER D WITH DOT ABOVE, COMBINING HORN, COMBINING DOT BELOW +1E0C 031B 0307;1E0C 031B 0307;0044 031B 0323 0307;1E0C 031B 0307;0044 031B 0323 0307; # (Ḍ◌̛◌̇; Ḍ◌̛◌̇; D◌̛◌̣◌̇; Ḍ◌̛◌̇; D◌̛◌̣◌̇; ) LATIN CAPITAL LETTER D WITH DOT BELOW, COMBINING HORN, COMBINING DOT ABOVE +0044 031B 0307 0323;1E0C 031B 0307;0044 031B 0323 0307;1E0C 031B 0307;0044 031B 0323 0307; # (D◌̛◌̇◌̣; Ḍ◌̛◌̇; D◌̛◌̣◌̇; Ḍ◌̛◌̇; D◌̛◌̣◌̇; ) LATIN CAPITAL LETTER D, COMBINING HORN, COMBINING DOT ABOVE, COMBINING DOT BELOW +0044 031B 0323 0307;1E0C 031B 0307;0044 031B 0323 0307;1E0C 031B 0307;0044 031B 0323 0307; # (D◌̛◌̣◌̇; Ḍ◌̛◌̇; D◌̛◌̣◌̇; Ḍ◌̛◌̇; D◌̛◌̣◌̇; ) LATIN CAPITAL LETTER D, COMBINING HORN, COMBINING DOT BELOW, COMBINING DOT ABOVE +00C8;00C8;0045 0300;00C8;0045 0300; # (È; È; E◌̀; È; E◌̀; ) LATIN CAPITAL LETTER E WITH GRAVE +0112;0112;0045 0304;0112;0045 0304; # (Ē; Ē; E◌̄; Ē; E◌̄; ) LATIN CAPITAL LETTER E WITH MACRON +0045 0300;00C8;0045 0300;00C8;0045 0300; # (E◌̀; È; E◌̀; È; E◌̀; ) LATIN CAPITAL LETTER E, COMBINING GRAVE ACCENT +0045 0304;0112;0045 0304;0112;0045 0304; # (E◌̄; Ē; E◌̄; Ē; E◌̄; ) LATIN CAPITAL LETTER E, COMBINING MACRON +1E14;1E14;0045 0304 0300;1E14;0045 0304 0300; # (Ḕ; Ḕ; E◌̄◌̀; Ḕ; E◌̄◌̀; ) LATIN CAPITAL LETTER E WITH MACRON AND GRAVE +0112 0300;1E14;0045 0304 0300;1E14;0045 0304 0300; # (Ē◌̀; Ḕ; E◌̄◌̀; Ḕ; E◌̄◌̀; ) LATIN CAPITAL LETTER E WITH MACRON, COMBINING GRAVE ACCENT +1E14 0304;1E14 0304;0045 0304 0300 0304;1E14 0304;0045 0304 0300 0304; # (Ḕ◌̄; Ḕ◌̄; E◌̄◌̀◌̄; Ḕ◌̄; E◌̄◌̀◌̄; ) LATIN CAPITAL LETTER E WITH MACRON AND GRAVE, COMBINING MACRON +0045 0304 0300;1E14;0045 0304 0300;1E14;0045 0304 0300; # (E◌̄◌̀; Ḕ; E◌̄◌̀; Ḕ; E◌̄◌̀; ) LATIN CAPITAL LETTER E, COMBINING MACRON, COMBINING GRAVE ACCENT +0045 0300 0304;00C8 0304;0045 0300 0304;00C8 0304;0045 0300 0304; # (E◌̀◌̄; È◌̄; E◌̀◌̄; È◌̄; E◌̀◌̄; ) LATIN CAPITAL LETTER E, COMBINING GRAVE ACCENT, COMBINING MACRON +05B8 05B9 05B1 0591 05C3 05B0 05AC 059F;05B1 05B8 05B9 0591 05C3 05B0 05AC 059F;05B1 05B8 05B9 0591 05C3 05B0 05AC 059F;05B1 05B8 05B9 0591 05C3 05B0 05AC 059F;05B1 05B8 05B9 0591 05C3 05B0 05AC 059F; # (◌ָ◌ֹ◌ֱ◌֑׃◌ְ◌֬◌֟; ◌ֱ◌ָ◌ֹ◌֑׃◌ְ◌֬◌֟; ◌ֱ◌ָ◌ֹ◌֑׃◌ְ◌֬◌֟; ◌ֱ◌ָ◌ֹ◌֑׃◌ְ◌֬◌֟; ◌ֱ◌ָ◌ֹ◌֑׃◌ְ◌֬◌֟; ) HEBREW POINT QAMATS, HEBREW POINT HOLAM, HEBREW POINT HATAF SEGOL, HEBREW ACCENT ETNAHTA, HEBREW PUNCTUATION SOF PASUQ, HEBREW POINT SHEVA, HEBREW ACCENT ILUY, HEBREW ACCENT QARNEY PARA +0592 05B7 05BC 05A5 05B0 05C0 05C4 05AD;05B0 05B7 05BC 05A5 0592 05C0 05AD 05C4;05B0 05B7 05BC 05A5 0592 05C0 05AD 05C4;05B0 05B7 05BC 05A5 0592 05C0 05AD 05C4;05B0 05B7 05BC 05A5 0592 05C0 05AD 05C4; # (◌֒◌ַ◌ּ◌֥◌ְ׀◌ׄ◌֭; ◌ְ◌ַ◌ּ◌֥◌֒׀◌֭◌ׄ; ◌ְ◌ַ◌ּ◌֥◌֒׀◌֭◌ׄ; ◌ְ◌ַ◌ּ◌֥◌֒׀◌֭◌ׄ; ◌ְ◌ַ◌ּ◌֥◌֒׀◌֭◌ׄ; ) HEBREW ACCENT SEGOL, HEBREW POINT PATAH, HEBREW POINT DAGESH OR MAPIQ, HEBREW ACCENT MERKHA, HEBREW POINT SHEVA, HEBREW PUNCTUATION PASEQ, HEBREW MARK UPPER DOT, HEBREW ACCENT DEHI +# +@Part1 # Character by character test +# All characters not explicitly occurring in c1 of Part 1 have identical NFC, D, KC, KD forms. +# +00A0;00A0;00A0;0020;0020; # ( ;  ;  ; ; ; ) NO-BREAK SPACE +00A8;00A8;00A8;0020 0308;0020 0308; # (¨; ¨; ¨; ◌̈; ◌̈; ) DIAERESIS +00AA;00AA;00AA;0061;0061; # (ª; ª; ª; a; a; ) FEMININE ORDINAL INDICATOR +00AF;00AF;00AF;0020 0304;0020 0304; # (¯; ¯; ¯; ◌̄; ◌̄; ) MACRON +00B2;00B2;00B2;0032;0032; # (²; ²; ²; 2; 2; ) SUPERSCRIPT TWO +00B3;00B3;00B3;0033;0033; # (³; ³; ³; 3; 3; ) SUPERSCRIPT THREE +00B4;00B4;00B4;0020 0301;0020 0301; # (´; ´; ´; ◌́; ◌́; ) ACUTE ACCENT +00B5;00B5;00B5;03BC;03BC; # (µ; µ; µ; μ; μ; ) MICRO SIGN +00B8;00B8;00B8;0020 0327;0020 0327; # (¸; ¸; ¸; ◌̧; ◌̧; ) CEDILLA +00B9;00B9;00B9;0031;0031; # (¹; ¹; ¹; 1; 1; ) SUPERSCRIPT ONE +00BA;00BA;00BA;006F;006F; # (º; º; º; o; o; ) MASCULINE ORDINAL INDICATOR +00BC;00BC;00BC;0031 2044 0034;0031 2044 0034; # (¼; ¼; ¼; 1⁄4; 1⁄4; ) VULGAR FRACTION ONE QUARTER +00BD;00BD;00BD;0031 2044 0032;0031 2044 0032; # (½; ½; ½; 1⁄2; 1⁄2; ) VULGAR FRACTION ONE HALF +00BE;00BE;00BE;0033 2044 0034;0033 2044 0034; # (¾; ¾; ¾; 3⁄4; 3⁄4; ) VULGAR FRACTION THREE QUARTERS +00C0;00C0;0041 0300;00C0;0041 0300; # (À; À; A◌̀; À; A◌̀; ) LATIN CAPITAL LETTER A WITH GRAVE +00C1;00C1;0041 0301;00C1;0041 0301; # (Á; Á; A◌́; Á; A◌́; ) LATIN CAPITAL LETTER A WITH ACUTE +00C2;00C2;0041 0302;00C2;0041 0302; # (Â; Â; A◌̂; Â; A◌̂; ) LATIN CAPITAL LETTER A WITH CIRCUMFLEX +00C3;00C3;0041 0303;00C3;0041 0303; # (Ã; Ã; A◌̃; Ã; A◌̃; ) LATIN CAPITAL LETTER A WITH TILDE +00C4;00C4;0041 0308;00C4;0041 0308; # (Ä; Ä; A◌̈; Ä; A◌̈; ) LATIN CAPITAL LETTER A WITH DIAERESIS +00C5;00C5;0041 030A;00C5;0041 030A; # (Å; Å; A◌̊; Å; A◌̊; ) LATIN CAPITAL LETTER A WITH RING ABOVE +00C7;00C7;0043 0327;00C7;0043 0327; # (Ç; Ç; C◌̧; Ç; C◌̧; ) LATIN CAPITAL LETTER C WITH CEDILLA +00C8;00C8;0045 0300;00C8;0045 0300; # (È; È; E◌̀; È; E◌̀; ) LATIN CAPITAL LETTER E WITH GRAVE +00C9;00C9;0045 0301;00C9;0045 0301; # (É; É; E◌́; É; E◌́; ) LATIN CAPITAL LETTER E WITH ACUTE +00CA;00CA;0045 0302;00CA;0045 0302; # (Ê; Ê; E◌̂; Ê; E◌̂; ) LATIN CAPITAL LETTER E WITH CIRCUMFLEX +00CB;00CB;0045 0308;00CB;0045 0308; # (Ë; Ë; E◌̈; Ë; E◌̈; ) LATIN CAPITAL LETTER E WITH DIAERESIS +00CC;00CC;0049 0300;00CC;0049 0300; # (Ì; Ì; I◌̀; Ì; I◌̀; ) LATIN CAPITAL LETTER I WITH GRAVE +00CD;00CD;0049 0301;00CD;0049 0301; # (Í; Í; I◌́; Í; I◌́; ) LATIN CAPITAL LETTER I WITH ACUTE +00CE;00CE;0049 0302;00CE;0049 0302; # (Î; Î; I◌̂; Î; I◌̂; ) LATIN CAPITAL LETTER I WITH CIRCUMFLEX +00CF;00CF;0049 0308;00CF;0049 0308; # (Ï; Ï; I◌̈; Ï; I◌̈; ) LATIN CAPITAL LETTER I WITH DIAERESIS +00D1;00D1;004E 0303;00D1;004E 0303; # (Ñ; Ñ; N◌̃; Ñ; N◌̃; ) LATIN CAPITAL LETTER N WITH TILDE +00D2;00D2;004F 0300;00D2;004F 0300; # (Ò; Ò; O◌̀; Ò; O◌̀; ) LATIN CAPITAL LETTER O WITH GRAVE +00D3;00D3;004F 0301;00D3;004F 0301; # (Ó; Ó; O◌́; Ó; O◌́; ) LATIN CAPITAL LETTER O WITH ACUTE +00D4;00D4;004F 0302;00D4;004F 0302; # (Ô; Ô; O◌̂; Ô; O◌̂; ) LATIN CAPITAL LETTER O WITH CIRCUMFLEX +00D5;00D5;004F 0303;00D5;004F 0303; # (Õ; Õ; O◌̃; Õ; O◌̃; ) LATIN CAPITAL LETTER O WITH TILDE +00D6;00D6;004F 0308;00D6;004F 0308; # (Ö; Ö; O◌̈; Ö; O◌̈; ) LATIN CAPITAL LETTER O WITH DIAERESIS +00D9;00D9;0055 0300;00D9;0055 0300; # (Ù; Ù; U◌̀; Ù; U◌̀; ) LATIN CAPITAL LETTER U WITH GRAVE +00DA;00DA;0055 0301;00DA;0055 0301; # (Ú; Ú; U◌́; Ú; U◌́; ) LATIN CAPITAL LETTER U WITH ACUTE +00DB;00DB;0055 0302;00DB;0055 0302; # (Û; Û; U◌̂; Û; U◌̂; ) LATIN CAPITAL LETTER U WITH CIRCUMFLEX +00DC;00DC;0055 0308;00DC;0055 0308; # (Ü; Ü; U◌̈; Ü; U◌̈; ) LATIN CAPITAL LETTER U WITH DIAERESIS +00DD;00DD;0059 0301;00DD;0059 0301; # (Ý; Ý; Y◌́; Ý; Y◌́; ) LATIN CAPITAL LETTER Y WITH ACUTE +00E0;00E0;0061 0300;00E0;0061 0300; # (à; à; a◌̀; à; a◌̀; ) LATIN SMALL LETTER A WITH GRAVE +00E1;00E1;0061 0301;00E1;0061 0301; # (á; á; a◌́; á; a◌́; ) LATIN SMALL LETTER A WITH ACUTE +00E2;00E2;0061 0302;00E2;0061 0302; # (â; â; a◌̂; â; a◌̂; ) LATIN SMALL LETTER A WITH CIRCUMFLEX +00E3;00E3;0061 0303;00E3;0061 0303; # (ã; ã; a◌̃; ã; a◌̃; ) LATIN SMALL LETTER A WITH TILDE +00E4;00E4;0061 0308;00E4;0061 0308; # (ä; ä; a◌̈; ä; a◌̈; ) LATIN SMALL LETTER A WITH DIAERESIS +00E5;00E5;0061 030A;00E5;0061 030A; # (å; å; a◌̊; å; a◌̊; ) LATIN SMALL LETTER A WITH RING ABOVE +00E7;00E7;0063 0327;00E7;0063 0327; # (ç; ç; c◌̧; ç; c◌̧; ) LATIN SMALL LETTER C WITH CEDILLA +00E8;00E8;0065 0300;00E8;0065 0300; # (è; è; e◌̀; è; e◌̀; ) LATIN SMALL LETTER E WITH GRAVE +00E9;00E9;0065 0301;00E9;0065 0301; # (é; é; e◌́; é; e◌́; ) LATIN SMALL LETTER E WITH ACUTE +00EA;00EA;0065 0302;00EA;0065 0302; # (ê; ê; e◌̂; ê; e◌̂; ) LATIN SMALL LETTER E WITH CIRCUMFLEX +00EB;00EB;0065 0308;00EB;0065 0308; # (ë; ë; e◌̈; ë; e◌̈; ) LATIN SMALL LETTER E WITH DIAERESIS +00EC;00EC;0069 0300;00EC;0069 0300; # (ì; ì; i◌̀; ì; i◌̀; ) LATIN SMALL LETTER I WITH GRAVE +00ED;00ED;0069 0301;00ED;0069 0301; # (í; í; i◌́; í; i◌́; ) LATIN SMALL LETTER I WITH ACUTE +00EE;00EE;0069 0302;00EE;0069 0302; # (î; î; i◌̂; î; i◌̂; ) LATIN SMALL LETTER I WITH CIRCUMFLEX +00EF;00EF;0069 0308;00EF;0069 0308; # (ï; ï; i◌̈; ï; i◌̈; ) LATIN SMALL LETTER I WITH DIAERESIS +00F1;00F1;006E 0303;00F1;006E 0303; # (ñ; ñ; n◌̃; ñ; n◌̃; ) LATIN SMALL LETTER N WITH TILDE +00F2;00F2;006F 0300;00F2;006F 0300; # (ò; ò; o◌̀; ò; o◌̀; ) LATIN SMALL LETTER O WITH GRAVE +00F3;00F3;006F 0301;00F3;006F 0301; # (ó; ó; o◌́; ó; o◌́; ) LATIN SMALL LETTER O WITH ACUTE +00F4;00F4;006F 0302;00F4;006F 0302; # (ô; ô; o◌̂; ô; o◌̂; ) LATIN SMALL LETTER O WITH CIRCUMFLEX +00F5;00F5;006F 0303;00F5;006F 0303; # (õ; õ; o◌̃; õ; o◌̃; ) LATIN SMALL LETTER O WITH TILDE +00F6;00F6;006F 0308;00F6;006F 0308; # (ö; ö; o◌̈; ö; o◌̈; ) LATIN SMALL LETTER O WITH DIAERESIS +00F9;00F9;0075 0300;00F9;0075 0300; # (ù; ù; u◌̀; ù; u◌̀; ) LATIN SMALL LETTER U WITH GRAVE +00FA;00FA;0075 0301;00FA;0075 0301; # (ú; ú; u◌́; ú; u◌́; ) LATIN SMALL LETTER U WITH ACUTE +00FB;00FB;0075 0302;00FB;0075 0302; # (û; û; u◌̂; û; u◌̂; ) LATIN SMALL LETTER U WITH CIRCUMFLEX +00FC;00FC;0075 0308;00FC;0075 0308; # (ü; ü; u◌̈; ü; u◌̈; ) LATIN SMALL LETTER U WITH DIAERESIS +00FD;00FD;0079 0301;00FD;0079 0301; # (ý; ý; y◌́; ý; y◌́; ) LATIN SMALL LETTER Y WITH ACUTE +00FF;00FF;0079 0308;00FF;0079 0308; # (ÿ; ÿ; y◌̈; ÿ; y◌̈; ) LATIN SMALL LETTER Y WITH DIAERESIS +0100;0100;0041 0304;0100;0041 0304; # (Ā; Ā; A◌̄; Ā; A◌̄; ) LATIN CAPITAL LETTER A WITH MACRON +0101;0101;0061 0304;0101;0061 0304; # (ā; ā; a◌̄; ā; a◌̄; ) LATIN SMALL LETTER A WITH MACRON +0102;0102;0041 0306;0102;0041 0306; # (Ă; Ă; A◌̆; Ă; A◌̆; ) LATIN CAPITAL LETTER A WITH BREVE +0103;0103;0061 0306;0103;0061 0306; # (ă; ă; a◌̆; ă; a◌̆; ) LATIN SMALL LETTER A WITH BREVE +0104;0104;0041 0328;0104;0041 0328; # (Ą; Ą; A◌̨; Ą; A◌̨; ) LATIN CAPITAL LETTER A WITH OGONEK +0105;0105;0061 0328;0105;0061 0328; # (ą; ą; a◌̨; ą; a◌̨; ) LATIN SMALL LETTER A WITH OGONEK +0106;0106;0043 0301;0106;0043 0301; # (Ć; Ć; C◌́; Ć; C◌́; ) LATIN CAPITAL LETTER C WITH ACUTE +0107;0107;0063 0301;0107;0063 0301; # (ć; ć; c◌́; ć; c◌́; ) LATIN SMALL LETTER C WITH ACUTE +0108;0108;0043 0302;0108;0043 0302; # (Ĉ; Ĉ; C◌̂; Ĉ; C◌̂; ) LATIN CAPITAL LETTER C WITH CIRCUMFLEX +0109;0109;0063 0302;0109;0063 0302; # (ĉ; ĉ; c◌̂; ĉ; c◌̂; ) LATIN SMALL LETTER C WITH CIRCUMFLEX +010A;010A;0043 0307;010A;0043 0307; # (Ċ; Ċ; C◌̇; Ċ; C◌̇; ) LATIN CAPITAL LETTER C WITH DOT ABOVE +010B;010B;0063 0307;010B;0063 0307; # (ċ; ċ; c◌̇; ċ; c◌̇; ) LATIN SMALL LETTER C WITH DOT ABOVE +010C;010C;0043 030C;010C;0043 030C; # (Č; Č; C◌̌; Č; C◌̌; ) LATIN CAPITAL LETTER C WITH CARON +010D;010D;0063 030C;010D;0063 030C; # (č; č; c◌̌; č; c◌̌; ) LATIN SMALL LETTER C WITH CARON +010E;010E;0044 030C;010E;0044 030C; # (Ď; Ď; D◌̌; Ď; D◌̌; ) LATIN CAPITAL LETTER D WITH CARON +010F;010F;0064 030C;010F;0064 030C; # (ď; ď; d◌̌; ď; d◌̌; ) LATIN SMALL LETTER D WITH CARON +0112;0112;0045 0304;0112;0045 0304; # (Ē; Ē; E◌̄; Ē; E◌̄; ) LATIN CAPITAL LETTER E WITH MACRON +0113;0113;0065 0304;0113;0065 0304; # (ē; ē; e◌̄; ē; e◌̄; ) LATIN SMALL LETTER E WITH MACRON +0114;0114;0045 0306;0114;0045 0306; # (Ĕ; Ĕ; E◌̆; Ĕ; E◌̆; ) LATIN CAPITAL LETTER E WITH BREVE +0115;0115;0065 0306;0115;0065 0306; # (ĕ; ĕ; e◌̆; ĕ; e◌̆; ) LATIN SMALL LETTER E WITH BREVE +0116;0116;0045 0307;0116;0045 0307; # (Ė; Ė; E◌̇; Ė; E◌̇; ) LATIN CAPITAL LETTER E WITH DOT ABOVE +0117;0117;0065 0307;0117;0065 0307; # (ė; ė; e◌̇; ė; e◌̇; ) LATIN SMALL LETTER E WITH DOT ABOVE +0118;0118;0045 0328;0118;0045 0328; # (Ę; Ę; E◌̨; Ę; E◌̨; ) LATIN CAPITAL LETTER E WITH OGONEK +0119;0119;0065 0328;0119;0065 0328; # (ę; ę; e◌̨; ę; e◌̨; ) LATIN SMALL LETTER E WITH OGONEK +011A;011A;0045 030C;011A;0045 030C; # (Ě; Ě; E◌̌; Ě; E◌̌; ) LATIN CAPITAL LETTER E WITH CARON +011B;011B;0065 030C;011B;0065 030C; # (ě; ě; e◌̌; ě; e◌̌; ) LATIN SMALL LETTER E WITH CARON +011C;011C;0047 0302;011C;0047 0302; # (Ĝ; Ĝ; G◌̂; Ĝ; G◌̂; ) LATIN CAPITAL LETTER G WITH CIRCUMFLEX +011D;011D;0067 0302;011D;0067 0302; # (ĝ; ĝ; g◌̂; ĝ; g◌̂; ) LATIN SMALL LETTER G WITH CIRCUMFLEX +011E;011E;0047 0306;011E;0047 0306; # (Ğ; Ğ; G◌̆; Ğ; G◌̆; ) LATIN CAPITAL LETTER G WITH BREVE +011F;011F;0067 0306;011F;0067 0306; # (ğ; ğ; g◌̆; ğ; g◌̆; ) LATIN SMALL LETTER G WITH BREVE +0120;0120;0047 0307;0120;0047 0307; # (Ġ; Ġ; G◌̇; Ġ; G◌̇; ) LATIN CAPITAL LETTER G WITH DOT ABOVE +0121;0121;0067 0307;0121;0067 0307; # (ġ; ġ; g◌̇; ġ; g◌̇; ) LATIN SMALL LETTER G WITH DOT ABOVE +0122;0122;0047 0327;0122;0047 0327; # (Ģ; Ģ; G◌̧; Ģ; G◌̧; ) LATIN CAPITAL LETTER G WITH CEDILLA +0123;0123;0067 0327;0123;0067 0327; # (ģ; ģ; g◌̧; ģ; g◌̧; ) LATIN SMALL LETTER G WITH CEDILLA +0124;0124;0048 0302;0124;0048 0302; # (Ĥ; Ĥ; H◌̂; Ĥ; H◌̂; ) LATIN CAPITAL LETTER H WITH CIRCUMFLEX +0125;0125;0068 0302;0125;0068 0302; # (ĥ; ĥ; h◌̂; ĥ; h◌̂; ) LATIN SMALL LETTER H WITH CIRCUMFLEX +0128;0128;0049 0303;0128;0049 0303; # (Ĩ; Ĩ; I◌̃; Ĩ; I◌̃; ) LATIN CAPITAL LETTER I WITH TILDE +0129;0129;0069 0303;0129;0069 0303; # (ĩ; ĩ; i◌̃; ĩ; i◌̃; ) LATIN SMALL LETTER I WITH TILDE +012A;012A;0049 0304;012A;0049 0304; # (Ī; Ī; I◌̄; Ī; I◌̄; ) LATIN CAPITAL LETTER I WITH MACRON +012B;012B;0069 0304;012B;0069 0304; # (ī; ī; i◌̄; ī; i◌̄; ) LATIN SMALL LETTER I WITH MACRON +012C;012C;0049 0306;012C;0049 0306; # (Ĭ; Ĭ; I◌̆; Ĭ; I◌̆; ) LATIN CAPITAL LETTER I WITH BREVE +012D;012D;0069 0306;012D;0069 0306; # (ĭ; ĭ; i◌̆; ĭ; i◌̆; ) LATIN SMALL LETTER I WITH BREVE +012E;012E;0049 0328;012E;0049 0328; # (Į; Į; I◌̨; Į; I◌̨; ) LATIN CAPITAL LETTER I WITH OGONEK +012F;012F;0069 0328;012F;0069 0328; # (į; į; i◌̨; į; i◌̨; ) LATIN SMALL LETTER I WITH OGONEK +0130;0130;0049 0307;0130;0049 0307; # (İ; İ; I◌̇; İ; I◌̇; ) LATIN CAPITAL LETTER I WITH DOT ABOVE +0132;0132;0132;0049 004A;0049 004A; # (IJ; IJ; IJ; IJ; IJ; ) LATIN CAPITAL LIGATURE IJ +0133;0133;0133;0069 006A;0069 006A; # (ij; ij; ij; ij; ij; ) LATIN SMALL LIGATURE IJ +0134;0134;004A 0302;0134;004A 0302; # (Ĵ; Ĵ; J◌̂; Ĵ; J◌̂; ) LATIN CAPITAL LETTER J WITH CIRCUMFLEX +0135;0135;006A 0302;0135;006A 0302; # (ĵ; ĵ; j◌̂; ĵ; j◌̂; ) LATIN SMALL LETTER J WITH CIRCUMFLEX +0136;0136;004B 0327;0136;004B 0327; # (Ķ; Ķ; K◌̧; Ķ; K◌̧; ) LATIN CAPITAL LETTER K WITH CEDILLA +0137;0137;006B 0327;0137;006B 0327; # (ķ; ķ; k◌̧; ķ; k◌̧; ) LATIN SMALL LETTER K WITH CEDILLA +0139;0139;004C 0301;0139;004C 0301; # (Ĺ; Ĺ; L◌́; Ĺ; L◌́; ) LATIN CAPITAL LETTER L WITH ACUTE +013A;013A;006C 0301;013A;006C 0301; # (ĺ; ĺ; l◌́; ĺ; l◌́; ) LATIN SMALL LETTER L WITH ACUTE +013B;013B;004C 0327;013B;004C 0327; # (Ļ; Ļ; L◌̧; Ļ; L◌̧; ) LATIN CAPITAL LETTER L WITH CEDILLA +013C;013C;006C 0327;013C;006C 0327; # (ļ; ļ; l◌̧; ļ; l◌̧; ) LATIN SMALL LETTER L WITH CEDILLA +013D;013D;004C 030C;013D;004C 030C; # (Ľ; Ľ; L◌̌; Ľ; L◌̌; ) LATIN CAPITAL LETTER L WITH CARON +013E;013E;006C 030C;013E;006C 030C; # (ľ; ľ; l◌̌; ľ; l◌̌; ) LATIN SMALL LETTER L WITH CARON +013F;013F;013F;004C 00B7;004C 00B7; # (Ŀ; Ŀ; Ŀ; L·; L·; ) LATIN CAPITAL LETTER L WITH MIDDLE DOT +0140;0140;0140;006C 00B7;006C 00B7; # (ŀ; ŀ; ŀ; l·; l·; ) LATIN SMALL LETTER L WITH MIDDLE DOT +0143;0143;004E 0301;0143;004E 0301; # (Ń; Ń; N◌́; Ń; N◌́; ) LATIN CAPITAL LETTER N WITH ACUTE +0144;0144;006E 0301;0144;006E 0301; # (ń; ń; n◌́; ń; n◌́; ) LATIN SMALL LETTER N WITH ACUTE +0145;0145;004E 0327;0145;004E 0327; # (Ņ; Ņ; N◌̧; Ņ; N◌̧; ) LATIN CAPITAL LETTER N WITH CEDILLA +0146;0146;006E 0327;0146;006E 0327; # (ņ; ņ; n◌̧; ņ; n◌̧; ) LATIN SMALL LETTER N WITH CEDILLA +0147;0147;004E 030C;0147;004E 030C; # (Ň; Ň; N◌̌; Ň; N◌̌; ) LATIN CAPITAL LETTER N WITH CARON +0148;0148;006E 030C;0148;006E 030C; # (ň; ň; n◌̌; ň; n◌̌; ) LATIN SMALL LETTER N WITH CARON +0149;0149;0149;02BC 006E;02BC 006E; # (ʼn; ʼn; ʼn; ʼn; ʼn; ) LATIN SMALL LETTER N PRECEDED BY APOSTROPHE +014C;014C;004F 0304;014C;004F 0304; # (Ō; Ō; O◌̄; Ō; O◌̄; ) LATIN CAPITAL LETTER O WITH MACRON +014D;014D;006F 0304;014D;006F 0304; # (ō; ō; o◌̄; ō; o◌̄; ) LATIN SMALL LETTER O WITH MACRON +014E;014E;004F 0306;014E;004F 0306; # (Ŏ; Ŏ; O◌̆; Ŏ; O◌̆; ) LATIN CAPITAL LETTER O WITH BREVE +014F;014F;006F 0306;014F;006F 0306; # (ŏ; ŏ; o◌̆; ŏ; o◌̆; ) LATIN SMALL LETTER O WITH BREVE +0150;0150;004F 030B;0150;004F 030B; # (Ő; Ő; O◌̋; Ő; O◌̋; ) LATIN CAPITAL LETTER O WITH DOUBLE ACUTE +0151;0151;006F 030B;0151;006F 030B; # (ő; ő; o◌̋; ő; o◌̋; ) LATIN SMALL LETTER O WITH DOUBLE ACUTE +0154;0154;0052 0301;0154;0052 0301; # (Ŕ; Ŕ; R◌́; Ŕ; R◌́; ) LATIN CAPITAL LETTER R WITH ACUTE +0155;0155;0072 0301;0155;0072 0301; # (ŕ; ŕ; r◌́; ŕ; r◌́; ) LATIN SMALL LETTER R WITH ACUTE +0156;0156;0052 0327;0156;0052 0327; # (Ŗ; Ŗ; R◌̧; Ŗ; R◌̧; ) LATIN CAPITAL LETTER R WITH CEDILLA +0157;0157;0072 0327;0157;0072 0327; # (ŗ; ŗ; r◌̧; ŗ; r◌̧; ) LATIN SMALL LETTER R WITH CEDILLA +0158;0158;0052 030C;0158;0052 030C; # (Ř; Ř; R◌̌; Ř; R◌̌; ) LATIN CAPITAL LETTER R WITH CARON +0159;0159;0072 030C;0159;0072 030C; # (ř; ř; r◌̌; ř; r◌̌; ) LATIN SMALL LETTER R WITH CARON +015A;015A;0053 0301;015A;0053 0301; # (Ś; Ś; S◌́; Ś; S◌́; ) LATIN CAPITAL LETTER S WITH ACUTE +015B;015B;0073 0301;015B;0073 0301; # (ś; ś; s◌́; ś; s◌́; ) LATIN SMALL LETTER S WITH ACUTE +015C;015C;0053 0302;015C;0053 0302; # (Ŝ; Ŝ; S◌̂; Ŝ; S◌̂; ) LATIN CAPITAL LETTER S WITH CIRCUMFLEX +015D;015D;0073 0302;015D;0073 0302; # (ŝ; ŝ; s◌̂; ŝ; s◌̂; ) LATIN SMALL LETTER S WITH CIRCUMFLEX +015E;015E;0053 0327;015E;0053 0327; # (Ş; Ş; S◌̧; Ş; S◌̧; ) LATIN CAPITAL LETTER S WITH CEDILLA +015F;015F;0073 0327;015F;0073 0327; # (ş; ş; s◌̧; ş; s◌̧; ) LATIN SMALL LETTER S WITH CEDILLA +0160;0160;0053 030C;0160;0053 030C; # (Š; Š; S◌̌; Š; S◌̌; ) LATIN CAPITAL LETTER S WITH CARON +0161;0161;0073 030C;0161;0073 030C; # (š; š; s◌̌; š; s◌̌; ) LATIN SMALL LETTER S WITH CARON +0162;0162;0054 0327;0162;0054 0327; # (Ţ; Ţ; T◌̧; Ţ; T◌̧; ) LATIN CAPITAL LETTER T WITH CEDILLA +0163;0163;0074 0327;0163;0074 0327; # (ţ; ţ; t◌̧; ţ; t◌̧; ) LATIN SMALL LETTER T WITH CEDILLA +0164;0164;0054 030C;0164;0054 030C; # (Ť; Ť; T◌̌; Ť; T◌̌; ) LATIN CAPITAL LETTER T WITH CARON +0165;0165;0074 030C;0165;0074 030C; # (ť; ť; t◌̌; ť; t◌̌; ) LATIN SMALL LETTER T WITH CARON +0168;0168;0055 0303;0168;0055 0303; # (Ũ; Ũ; U◌̃; Ũ; U◌̃; ) LATIN CAPITAL LETTER U WITH TILDE +0169;0169;0075 0303;0169;0075 0303; # (ũ; ũ; u◌̃; ũ; u◌̃; ) LATIN SMALL LETTER U WITH TILDE +016A;016A;0055 0304;016A;0055 0304; # (Ū; Ū; U◌̄; Ū; U◌̄; ) LATIN CAPITAL LETTER U WITH MACRON +016B;016B;0075 0304;016B;0075 0304; # (ū; ū; u◌̄; ū; u◌̄; ) LATIN SMALL LETTER U WITH MACRON +016C;016C;0055 0306;016C;0055 0306; # (Ŭ; Ŭ; U◌̆; Ŭ; U◌̆; ) LATIN CAPITAL LETTER U WITH BREVE +016D;016D;0075 0306;016D;0075 0306; # (ŭ; ŭ; u◌̆; ŭ; u◌̆; ) LATIN SMALL LETTER U WITH BREVE +016E;016E;0055 030A;016E;0055 030A; # (Ů; Ů; U◌̊; Ů; U◌̊; ) LATIN CAPITAL LETTER U WITH RING ABOVE +016F;016F;0075 030A;016F;0075 030A; # (ů; ů; u◌̊; ů; u◌̊; ) LATIN SMALL LETTER U WITH RING ABOVE +0170;0170;0055 030B;0170;0055 030B; # (Ű; Ű; U◌̋; Ű; U◌̋; ) LATIN CAPITAL LETTER U WITH DOUBLE ACUTE +0171;0171;0075 030B;0171;0075 030B; # (ű; ű; u◌̋; ű; u◌̋; ) LATIN SMALL LETTER U WITH DOUBLE ACUTE +0172;0172;0055 0328;0172;0055 0328; # (Ų; Ų; U◌̨; Ų; U◌̨; ) LATIN CAPITAL LETTER U WITH OGONEK +0173;0173;0075 0328;0173;0075 0328; # (ų; ų; u◌̨; ų; u◌̨; ) LATIN SMALL LETTER U WITH OGONEK +0174;0174;0057 0302;0174;0057 0302; # (Ŵ; Ŵ; W◌̂; Ŵ; W◌̂; ) LATIN CAPITAL LETTER W WITH CIRCUMFLEX +0175;0175;0077 0302;0175;0077 0302; # (ŵ; ŵ; w◌̂; ŵ; w◌̂; ) LATIN SMALL LETTER W WITH CIRCUMFLEX +0176;0176;0059 0302;0176;0059 0302; # (Ŷ; Ŷ; Y◌̂; Ŷ; Y◌̂; ) LATIN CAPITAL LETTER Y WITH CIRCUMFLEX +0177;0177;0079 0302;0177;0079 0302; # (ŷ; ŷ; y◌̂; ŷ; y◌̂; ) LATIN SMALL LETTER Y WITH CIRCUMFLEX +0178;0178;0059 0308;0178;0059 0308; # (Ÿ; Ÿ; Y◌̈; Ÿ; Y◌̈; ) LATIN CAPITAL LETTER Y WITH DIAERESIS +0179;0179;005A 0301;0179;005A 0301; # (Ź; Ź; Z◌́; Ź; Z◌́; ) LATIN CAPITAL LETTER Z WITH ACUTE +017A;017A;007A 0301;017A;007A 0301; # (ź; ź; z◌́; ź; z◌́; ) LATIN SMALL LETTER Z WITH ACUTE +017B;017B;005A 0307;017B;005A 0307; # (Ż; Ż; Z◌̇; Ż; Z◌̇; ) LATIN CAPITAL LETTER Z WITH DOT ABOVE +017C;017C;007A 0307;017C;007A 0307; # (ż; ż; z◌̇; ż; z◌̇; ) LATIN SMALL LETTER Z WITH DOT ABOVE +017D;017D;005A 030C;017D;005A 030C; # (Ž; Ž; Z◌̌; Ž; Z◌̌; ) LATIN CAPITAL LETTER Z WITH CARON +017E;017E;007A 030C;017E;007A 030C; # (ž; ž; z◌̌; ž; z◌̌; ) LATIN SMALL LETTER Z WITH CARON +017F;017F;017F;0073;0073; # (ſ; ſ; ſ; s; s; ) LATIN SMALL LETTER LONG S +01A0;01A0;004F 031B;01A0;004F 031B; # (Ơ; Ơ; O◌̛; Ơ; O◌̛; ) LATIN CAPITAL LETTER O WITH HORN +01A1;01A1;006F 031B;01A1;006F 031B; # (ơ; ơ; o◌̛; ơ; o◌̛; ) LATIN SMALL LETTER O WITH HORN +01AF;01AF;0055 031B;01AF;0055 031B; # (Ư; Ư; U◌̛; Ư; U◌̛; ) LATIN CAPITAL LETTER U WITH HORN +01B0;01B0;0075 031B;01B0;0075 031B; # (ư; ư; u◌̛; ư; u◌̛; ) LATIN SMALL LETTER U WITH HORN +01C4;01C4;01C4;0044 017D;0044 005A 030C; # (DŽ; DŽ; DŽ; DŽ; DZ◌̌; ) LATIN CAPITAL LETTER DZ WITH CARON +01C5;01C5;01C5;0044 017E;0044 007A 030C; # (Dž; Dž; Dž; Dž; Dz◌̌; ) LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON +01C6;01C6;01C6;0064 017E;0064 007A 030C; # (dž; dž; dž; dž; dz◌̌; ) LATIN SMALL LETTER DZ WITH CARON +01C7;01C7;01C7;004C 004A;004C 004A; # (LJ; LJ; LJ; LJ; LJ; ) LATIN CAPITAL LETTER LJ +01C8;01C8;01C8;004C 006A;004C 006A; # (Lj; Lj; Lj; Lj; Lj; ) LATIN CAPITAL LETTER L WITH SMALL LETTER J +01C9;01C9;01C9;006C 006A;006C 006A; # (lj; lj; lj; lj; lj; ) LATIN SMALL LETTER LJ +01CA;01CA;01CA;004E 004A;004E 004A; # (NJ; NJ; NJ; NJ; NJ; ) LATIN CAPITAL LETTER NJ +01CB;01CB;01CB;004E 006A;004E 006A; # (Nj; Nj; Nj; Nj; Nj; ) LATIN CAPITAL LETTER N WITH SMALL LETTER J +01CC;01CC;01CC;006E 006A;006E 006A; # (nj; nj; nj; nj; nj; ) LATIN SMALL LETTER NJ +01CD;01CD;0041 030C;01CD;0041 030C; # (Ǎ; Ǎ; A◌̌; Ǎ; A◌̌; ) LATIN CAPITAL LETTER A WITH CARON +01CE;01CE;0061 030C;01CE;0061 030C; # (ǎ; ǎ; a◌̌; ǎ; a◌̌; ) LATIN SMALL LETTER A WITH CARON +01CF;01CF;0049 030C;01CF;0049 030C; # (Ǐ; Ǐ; I◌̌; Ǐ; I◌̌; ) LATIN CAPITAL LETTER I WITH CARON +01D0;01D0;0069 030C;01D0;0069 030C; # (ǐ; ǐ; i◌̌; ǐ; i◌̌; ) LATIN SMALL LETTER I WITH CARON +01D1;01D1;004F 030C;01D1;004F 030C; # (Ǒ; Ǒ; O◌̌; Ǒ; O◌̌; ) LATIN CAPITAL LETTER O WITH CARON +01D2;01D2;006F 030C;01D2;006F 030C; # (ǒ; ǒ; o◌̌; ǒ; o◌̌; ) LATIN SMALL LETTER O WITH CARON +01D3;01D3;0055 030C;01D3;0055 030C; # (Ǔ; Ǔ; U◌̌; Ǔ; U◌̌; ) LATIN CAPITAL LETTER U WITH CARON +01D4;01D4;0075 030C;01D4;0075 030C; # (ǔ; ǔ; u◌̌; ǔ; u◌̌; ) LATIN SMALL LETTER U WITH CARON +01D5;01D5;0055 0308 0304;01D5;0055 0308 0304; # (Ǖ; Ǖ; U◌̈◌̄; Ǖ; U◌̈◌̄; ) LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON +01D6;01D6;0075 0308 0304;01D6;0075 0308 0304; # (ǖ; ǖ; u◌̈◌̄; ǖ; u◌̈◌̄; ) LATIN SMALL LETTER U WITH DIAERESIS AND MACRON +01D7;01D7;0055 0308 0301;01D7;0055 0308 0301; # (Ǘ; Ǘ; U◌̈◌́; Ǘ; U◌̈◌́; ) LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE +01D8;01D8;0075 0308 0301;01D8;0075 0308 0301; # (ǘ; ǘ; u◌̈◌́; ǘ; u◌̈◌́; ) LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE +01D9;01D9;0055 0308 030C;01D9;0055 0308 030C; # (Ǚ; Ǚ; U◌̈◌̌; Ǚ; U◌̈◌̌; ) LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON +01DA;01DA;0075 0308 030C;01DA;0075 0308 030C; # (ǚ; ǚ; u◌̈◌̌; ǚ; u◌̈◌̌; ) LATIN SMALL LETTER U WITH DIAERESIS AND CARON +01DB;01DB;0055 0308 0300;01DB;0055 0308 0300; # (Ǜ; Ǜ; U◌̈◌̀; Ǜ; U◌̈◌̀; ) LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE +01DC;01DC;0075 0308 0300;01DC;0075 0308 0300; # (ǜ; ǜ; u◌̈◌̀; ǜ; u◌̈◌̀; ) LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE +01DE;01DE;0041 0308 0304;01DE;0041 0308 0304; # (Ǟ; Ǟ; A◌̈◌̄; Ǟ; A◌̈◌̄; ) LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON +01DF;01DF;0061 0308 0304;01DF;0061 0308 0304; # (ǟ; ǟ; a◌̈◌̄; ǟ; a◌̈◌̄; ) LATIN SMALL LETTER A WITH DIAERESIS AND MACRON +01E0;01E0;0041 0307 0304;01E0;0041 0307 0304; # (Ǡ; Ǡ; A◌̇◌̄; Ǡ; A◌̇◌̄; ) LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON +01E1;01E1;0061 0307 0304;01E1;0061 0307 0304; # (ǡ; ǡ; a◌̇◌̄; ǡ; a◌̇◌̄; ) LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON +01E2;01E2;00C6 0304;01E2;00C6 0304; # (Ǣ; Ǣ; Æ◌̄; Ǣ; Æ◌̄; ) LATIN CAPITAL LETTER AE WITH MACRON +01E3;01E3;00E6 0304;01E3;00E6 0304; # (ǣ; ǣ; æ◌̄; ǣ; æ◌̄; ) LATIN SMALL LETTER AE WITH MACRON +01E6;01E6;0047 030C;01E6;0047 030C; # (Ǧ; Ǧ; G◌̌; Ǧ; G◌̌; ) LATIN CAPITAL LETTER G WITH CARON +01E7;01E7;0067 030C;01E7;0067 030C; # (ǧ; ǧ; g◌̌; ǧ; g◌̌; ) LATIN SMALL LETTER G WITH CARON +01E8;01E8;004B 030C;01E8;004B 030C; # (Ǩ; Ǩ; K◌̌; Ǩ; K◌̌; ) LATIN CAPITAL LETTER K WITH CARON +01E9;01E9;006B 030C;01E9;006B 030C; # (ǩ; ǩ; k◌̌; ǩ; k◌̌; ) LATIN SMALL LETTER K WITH CARON +01EA;01EA;004F 0328;01EA;004F 0328; # (Ǫ; Ǫ; O◌̨; Ǫ; O◌̨; ) LATIN CAPITAL LETTER O WITH OGONEK +01EB;01EB;006F 0328;01EB;006F 0328; # (ǫ; ǫ; o◌̨; ǫ; o◌̨; ) LATIN SMALL LETTER O WITH OGONEK +01EC;01EC;004F 0328 0304;01EC;004F 0328 0304; # (Ǭ; Ǭ; O◌̨◌̄; Ǭ; O◌̨◌̄; ) LATIN CAPITAL LETTER O WITH OGONEK AND MACRON +01ED;01ED;006F 0328 0304;01ED;006F 0328 0304; # (ǭ; ǭ; o◌̨◌̄; ǭ; o◌̨◌̄; ) LATIN SMALL LETTER O WITH OGONEK AND MACRON +01EE;01EE;01B7 030C;01EE;01B7 030C; # (Ǯ; Ǯ; Ʒ◌̌; Ǯ; Ʒ◌̌; ) LATIN CAPITAL LETTER EZH WITH CARON +01EF;01EF;0292 030C;01EF;0292 030C; # (ǯ; ǯ; ʒ◌̌; ǯ; ʒ◌̌; ) LATIN SMALL LETTER EZH WITH CARON +01F0;01F0;006A 030C;01F0;006A 030C; # (ǰ; ǰ; j◌̌; ǰ; j◌̌; ) LATIN SMALL LETTER J WITH CARON +01F1;01F1;01F1;0044 005A;0044 005A; # (DZ; DZ; DZ; DZ; DZ; ) LATIN CAPITAL LETTER DZ +01F2;01F2;01F2;0044 007A;0044 007A; # (Dz; Dz; Dz; Dz; Dz; ) LATIN CAPITAL LETTER D WITH SMALL LETTER Z +01F3;01F3;01F3;0064 007A;0064 007A; # (dz; dz; dz; dz; dz; ) LATIN SMALL LETTER DZ +01F4;01F4;0047 0301;01F4;0047 0301; # (Ǵ; Ǵ; G◌́; Ǵ; G◌́; ) LATIN CAPITAL LETTER G WITH ACUTE +01F5;01F5;0067 0301;01F5;0067 0301; # (ǵ; ǵ; g◌́; ǵ; g◌́; ) LATIN SMALL LETTER G WITH ACUTE +01F8;01F8;004E 0300;01F8;004E 0300; # (Ǹ; Ǹ; N◌̀; Ǹ; N◌̀; ) LATIN CAPITAL LETTER N WITH GRAVE +01F9;01F9;006E 0300;01F9;006E 0300; # (ǹ; ǹ; n◌̀; ǹ; n◌̀; ) LATIN SMALL LETTER N WITH GRAVE +01FA;01FA;0041 030A 0301;01FA;0041 030A 0301; # (Ǻ; Ǻ; A◌̊◌́; Ǻ; A◌̊◌́; ) LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE +01FB;01FB;0061 030A 0301;01FB;0061 030A 0301; # (ǻ; ǻ; a◌̊◌́; ǻ; a◌̊◌́; ) LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE +01FC;01FC;00C6 0301;01FC;00C6 0301; # (Ǽ; Ǽ; Æ◌́; Ǽ; Æ◌́; ) LATIN CAPITAL LETTER AE WITH ACUTE +01FD;01FD;00E6 0301;01FD;00E6 0301; # (ǽ; ǽ; æ◌́; ǽ; æ◌́; ) LATIN SMALL LETTER AE WITH ACUTE +01FE;01FE;00D8 0301;01FE;00D8 0301; # (Ǿ; Ǿ; Ø◌́; Ǿ; Ø◌́; ) LATIN CAPITAL LETTER O WITH STROKE AND ACUTE +01FF;01FF;00F8 0301;01FF;00F8 0301; # (ǿ; ǿ; ø◌́; ǿ; ø◌́; ) LATIN SMALL LETTER O WITH STROKE AND ACUTE +0200;0200;0041 030F;0200;0041 030F; # (Ȁ; Ȁ; A◌̏; Ȁ; A◌̏; ) LATIN CAPITAL LETTER A WITH DOUBLE GRAVE +0201;0201;0061 030F;0201;0061 030F; # (ȁ; ȁ; a◌̏; ȁ; a◌̏; ) LATIN SMALL LETTER A WITH DOUBLE GRAVE +0202;0202;0041 0311;0202;0041 0311; # (Ȃ; Ȃ; A◌̑; Ȃ; A◌̑; ) LATIN CAPITAL LETTER A WITH INVERTED BREVE +0203;0203;0061 0311;0203;0061 0311; # (ȃ; ȃ; a◌̑; ȃ; a◌̑; ) LATIN SMALL LETTER A WITH INVERTED BREVE +0204;0204;0045 030F;0204;0045 030F; # (Ȅ; Ȅ; E◌̏; Ȅ; E◌̏; ) LATIN CAPITAL LETTER E WITH DOUBLE GRAVE +0205;0205;0065 030F;0205;0065 030F; # (ȅ; ȅ; e◌̏; ȅ; e◌̏; ) LATIN SMALL LETTER E WITH DOUBLE GRAVE +0206;0206;0045 0311;0206;0045 0311; # (Ȇ; Ȇ; E◌̑; Ȇ; E◌̑; ) LATIN CAPITAL LETTER E WITH INVERTED BREVE +0207;0207;0065 0311;0207;0065 0311; # (ȇ; ȇ; e◌̑; ȇ; e◌̑; ) LATIN SMALL LETTER E WITH INVERTED BREVE +0208;0208;0049 030F;0208;0049 030F; # (Ȉ; Ȉ; I◌̏; Ȉ; I◌̏; ) LATIN CAPITAL LETTER I WITH DOUBLE GRAVE +0209;0209;0069 030F;0209;0069 030F; # (ȉ; ȉ; i◌̏; ȉ; i◌̏; ) LATIN SMALL LETTER I WITH DOUBLE GRAVE +020A;020A;0049 0311;020A;0049 0311; # (Ȋ; Ȋ; I◌̑; Ȋ; I◌̑; ) LATIN CAPITAL LETTER I WITH INVERTED BREVE +020B;020B;0069 0311;020B;0069 0311; # (ȋ; ȋ; i◌̑; ȋ; i◌̑; ) LATIN SMALL LETTER I WITH INVERTED BREVE +020C;020C;004F 030F;020C;004F 030F; # (Ȍ; Ȍ; O◌̏; Ȍ; O◌̏; ) LATIN CAPITAL LETTER O WITH DOUBLE GRAVE +020D;020D;006F 030F;020D;006F 030F; # (ȍ; ȍ; o◌̏; ȍ; o◌̏; ) LATIN SMALL LETTER O WITH DOUBLE GRAVE +020E;020E;004F 0311;020E;004F 0311; # (Ȏ; Ȏ; O◌̑; Ȏ; O◌̑; ) LATIN CAPITAL LETTER O WITH INVERTED BREVE +020F;020F;006F 0311;020F;006F 0311; # (ȏ; ȏ; o◌̑; ȏ; o◌̑; ) LATIN SMALL LETTER O WITH INVERTED BREVE +0210;0210;0052 030F;0210;0052 030F; # (Ȑ; Ȑ; R◌̏; Ȑ; R◌̏; ) LATIN CAPITAL LETTER R WITH DOUBLE GRAVE +0211;0211;0072 030F;0211;0072 030F; # (ȑ; ȑ; r◌̏; ȑ; r◌̏; ) LATIN SMALL LETTER R WITH DOUBLE GRAVE +0212;0212;0052 0311;0212;0052 0311; # (Ȓ; Ȓ; R◌̑; Ȓ; R◌̑; ) LATIN CAPITAL LETTER R WITH INVERTED BREVE +0213;0213;0072 0311;0213;0072 0311; # (ȓ; ȓ; r◌̑; ȓ; r◌̑; ) LATIN SMALL LETTER R WITH INVERTED BREVE +0214;0214;0055 030F;0214;0055 030F; # (Ȕ; Ȕ; U◌̏; Ȕ; U◌̏; ) LATIN CAPITAL LETTER U WITH DOUBLE GRAVE +0215;0215;0075 030F;0215;0075 030F; # (ȕ; ȕ; u◌̏; ȕ; u◌̏; ) LATIN SMALL LETTER U WITH DOUBLE GRAVE +0216;0216;0055 0311;0216;0055 0311; # (Ȗ; Ȗ; U◌̑; Ȗ; U◌̑; ) LATIN CAPITAL LETTER U WITH INVERTED BREVE +0217;0217;0075 0311;0217;0075 0311; # (ȗ; ȗ; u◌̑; ȗ; u◌̑; ) LATIN SMALL LETTER U WITH INVERTED BREVE +0218;0218;0053 0326;0218;0053 0326; # (Ș; Ș; S◌̦; Ș; S◌̦; ) LATIN CAPITAL LETTER S WITH COMMA BELOW +0219;0219;0073 0326;0219;0073 0326; # (ș; ș; s◌̦; ș; s◌̦; ) LATIN SMALL LETTER S WITH COMMA BELOW +021A;021A;0054 0326;021A;0054 0326; # (Ț; Ț; T◌̦; Ț; T◌̦; ) LATIN CAPITAL LETTER T WITH COMMA BELOW +021B;021B;0074 0326;021B;0074 0326; # (ț; ț; t◌̦; ț; t◌̦; ) LATIN SMALL LETTER T WITH COMMA BELOW +021E;021E;0048 030C;021E;0048 030C; # (Ȟ; Ȟ; H◌̌; Ȟ; H◌̌; ) LATIN CAPITAL LETTER H WITH CARON +021F;021F;0068 030C;021F;0068 030C; # (ȟ; ȟ; h◌̌; ȟ; h◌̌; ) LATIN SMALL LETTER H WITH CARON +0226;0226;0041 0307;0226;0041 0307; # (Ȧ; Ȧ; A◌̇; Ȧ; A◌̇; ) LATIN CAPITAL LETTER A WITH DOT ABOVE +0227;0227;0061 0307;0227;0061 0307; # (ȧ; ȧ; a◌̇; ȧ; a◌̇; ) LATIN SMALL LETTER A WITH DOT ABOVE +0228;0228;0045 0327;0228;0045 0327; # (Ȩ; Ȩ; E◌̧; Ȩ; E◌̧; ) LATIN CAPITAL LETTER E WITH CEDILLA +0229;0229;0065 0327;0229;0065 0327; # (ȩ; ȩ; e◌̧; ȩ; e◌̧; ) LATIN SMALL LETTER E WITH CEDILLA +022A;022A;004F 0308 0304;022A;004F 0308 0304; # (Ȫ; Ȫ; O◌̈◌̄; Ȫ; O◌̈◌̄; ) LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON +022B;022B;006F 0308 0304;022B;006F 0308 0304; # (ȫ; ȫ; o◌̈◌̄; ȫ; o◌̈◌̄; ) LATIN SMALL LETTER O WITH DIAERESIS AND MACRON +022C;022C;004F 0303 0304;022C;004F 0303 0304; # (Ȭ; Ȭ; O◌̃◌̄; Ȭ; O◌̃◌̄; ) LATIN CAPITAL LETTER O WITH TILDE AND MACRON +022D;022D;006F 0303 0304;022D;006F 0303 0304; # (ȭ; ȭ; o◌̃◌̄; ȭ; o◌̃◌̄; ) LATIN SMALL LETTER O WITH TILDE AND MACRON +022E;022E;004F 0307;022E;004F 0307; # (Ȯ; Ȯ; O◌̇; Ȯ; O◌̇; ) LATIN CAPITAL LETTER O WITH DOT ABOVE +022F;022F;006F 0307;022F;006F 0307; # (ȯ; ȯ; o◌̇; ȯ; o◌̇; ) LATIN SMALL LETTER O WITH DOT ABOVE +0230;0230;004F 0307 0304;0230;004F 0307 0304; # (Ȱ; Ȱ; O◌̇◌̄; Ȱ; O◌̇◌̄; ) LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON +0231;0231;006F 0307 0304;0231;006F 0307 0304; # (ȱ; ȱ; o◌̇◌̄; ȱ; o◌̇◌̄; ) LATIN SMALL LETTER O WITH DOT ABOVE AND MACRON +0232;0232;0059 0304;0232;0059 0304; # (Ȳ; Ȳ; Y◌̄; Ȳ; Y◌̄; ) LATIN CAPITAL LETTER Y WITH MACRON +0233;0233;0079 0304;0233;0079 0304; # (ȳ; ȳ; y◌̄; ȳ; y◌̄; ) LATIN SMALL LETTER Y WITH MACRON +02B0;02B0;02B0;0068;0068; # (ʰ; ʰ; ʰ; h; h; ) MODIFIER LETTER SMALL H +02B1;02B1;02B1;0266;0266; # (ʱ; ʱ; ʱ; ɦ; ɦ; ) MODIFIER LETTER SMALL H WITH HOOK +02B2;02B2;02B2;006A;006A; # (ʲ; ʲ; ʲ; j; j; ) MODIFIER LETTER SMALL J +02B3;02B3;02B3;0072;0072; # (ʳ; ʳ; ʳ; r; r; ) MODIFIER LETTER SMALL R +02B4;02B4;02B4;0279;0279; # (ʴ; ʴ; ʴ; ɹ; ɹ; ) MODIFIER LETTER SMALL TURNED R +02B5;02B5;02B5;027B;027B; # (ʵ; ʵ; ʵ; ɻ; ɻ; ) MODIFIER LETTER SMALL TURNED R WITH HOOK +02B6;02B6;02B6;0281;0281; # (ʶ; ʶ; ʶ; ʁ; ʁ; ) MODIFIER LETTER SMALL CAPITAL INVERTED R +02B7;02B7;02B7;0077;0077; # (ʷ; ʷ; ʷ; w; w; ) MODIFIER LETTER SMALL W +02B8;02B8;02B8;0079;0079; # (ʸ; ʸ; ʸ; y; y; ) MODIFIER LETTER SMALL Y +02D8;02D8;02D8;0020 0306;0020 0306; # (˘; ˘; ˘; ◌̆; ◌̆; ) BREVE +02D9;02D9;02D9;0020 0307;0020 0307; # (˙; ˙; ˙; ◌̇; ◌̇; ) DOT ABOVE +02DA;02DA;02DA;0020 030A;0020 030A; # (˚; ˚; ˚; ◌̊; ◌̊; ) RING ABOVE +02DB;02DB;02DB;0020 0328;0020 0328; # (˛; ˛; ˛; ◌̨; ◌̨; ) OGONEK +02DC;02DC;02DC;0020 0303;0020 0303; # (˜; ˜; ˜; ◌̃; ◌̃; ) SMALL TILDE +02DD;02DD;02DD;0020 030B;0020 030B; # (˝; ˝; ˝; ◌̋; ◌̋; ) DOUBLE ACUTE ACCENT +02E0;02E0;02E0;0263;0263; # (ˠ; ˠ; ˠ; ɣ; ɣ; ) MODIFIER LETTER SMALL GAMMA +02E1;02E1;02E1;006C;006C; # (ˡ; ˡ; ˡ; l; l; ) MODIFIER LETTER SMALL L +02E2;02E2;02E2;0073;0073; # (ˢ; ˢ; ˢ; s; s; ) MODIFIER LETTER SMALL S +02E3;02E3;02E3;0078;0078; # (ˣ; ˣ; ˣ; x; x; ) MODIFIER LETTER SMALL X +02E4;02E4;02E4;0295;0295; # (ˤ; ˤ; ˤ; ʕ; ʕ; ) MODIFIER LETTER SMALL REVERSED GLOTTAL STOP +0340;0300;0300;0300;0300; # (◌̀; ◌̀; ◌̀; ◌̀; ◌̀; ) COMBINING GRAVE TONE MARK +0341;0301;0301;0301;0301; # (◌́; ◌́; ◌́; ◌́; ◌́; ) COMBINING ACUTE TONE MARK +0343;0313;0313;0313;0313; # (◌̓; ◌̓; ◌̓; ◌̓; ◌̓; ) COMBINING GREEK KORONIS +0344;0308 0301;0308 0301;0308 0301;0308 0301; # (◌̈́; ◌̈◌́; ◌̈◌́; ◌̈◌́; ◌̈◌́; ) COMBINING GREEK DIALYTIKA TONOS +0374;02B9;02B9;02B9;02B9; # (ʹ; ʹ; ʹ; ʹ; ʹ; ) GREEK NUMERAL SIGN +037A;037A;037A;0020 0345;0020 0345; # (ͺ; ͺ; ͺ; ◌ͅ; ◌ͅ; ) GREEK YPOGEGRAMMENI +037E;003B;003B;003B;003B; # (;; ;; ;; ;; ;; ) GREEK QUESTION MARK +0384;0384;0384;0020 0301;0020 0301; # (΄; ΄; ΄; ◌́; ◌́; ) GREEK TONOS +0385;0385;00A8 0301;0020 0308 0301;0020 0308 0301; # (΅; ΅; ¨◌́; ◌̈◌́; ◌̈◌́; ) GREEK DIALYTIKA TONOS +0386;0386;0391 0301;0386;0391 0301; # (Ά; Ά; Α◌́; Ά; Α◌́; ) GREEK CAPITAL LETTER ALPHA WITH TONOS +0387;00B7;00B7;00B7;00B7; # (·; ·; ·; ·; ·; ) GREEK ANO TELEIA +0388;0388;0395 0301;0388;0395 0301; # (Έ; Έ; Ε◌́; Έ; Ε◌́; ) GREEK CAPITAL LETTER EPSILON WITH TONOS +0389;0389;0397 0301;0389;0397 0301; # (Ή; Ή; Η◌́; Ή; Η◌́; ) GREEK CAPITAL LETTER ETA WITH TONOS +038A;038A;0399 0301;038A;0399 0301; # (Ί; Ί; Ι◌́; Ί; Ι◌́; ) GREEK CAPITAL LETTER IOTA WITH TONOS +038C;038C;039F 0301;038C;039F 0301; # (Ό; Ό; Ο◌́; Ό; Ο◌́; ) GREEK CAPITAL LETTER OMICRON WITH TONOS +038E;038E;03A5 0301;038E;03A5 0301; # (Ύ; Ύ; Υ◌́; Ύ; Υ◌́; ) GREEK CAPITAL LETTER UPSILON WITH TONOS +038F;038F;03A9 0301;038F;03A9 0301; # (Ώ; Ώ; Ω◌́; Ώ; Ω◌́; ) GREEK CAPITAL LETTER OMEGA WITH TONOS +0390;0390;03B9 0308 0301;0390;03B9 0308 0301; # (ΐ; ΐ; ι◌̈◌́; ΐ; ι◌̈◌́; ) GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS +03AA;03AA;0399 0308;03AA;0399 0308; # (Ϊ; Ϊ; Ι◌̈; Ϊ; Ι◌̈; ) GREEK CAPITAL LETTER IOTA WITH DIALYTIKA +03AB;03AB;03A5 0308;03AB;03A5 0308; # (Ϋ; Ϋ; Υ◌̈; Ϋ; Υ◌̈; ) GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA +03AC;03AC;03B1 0301;03AC;03B1 0301; # (ά; ά; α◌́; ά; α◌́; ) GREEK SMALL LETTER ALPHA WITH TONOS +03AD;03AD;03B5 0301;03AD;03B5 0301; # (έ; έ; ε◌́; έ; ε◌́; ) GREEK SMALL LETTER EPSILON WITH TONOS +03AE;03AE;03B7 0301;03AE;03B7 0301; # (ή; ή; η◌́; ή; η◌́; ) GREEK SMALL LETTER ETA WITH TONOS +03AF;03AF;03B9 0301;03AF;03B9 0301; # (ί; ί; ι◌́; ί; ι◌́; ) GREEK SMALL LETTER IOTA WITH TONOS +03B0;03B0;03C5 0308 0301;03B0;03C5 0308 0301; # (ΰ; ΰ; υ◌̈◌́; ΰ; υ◌̈◌́; ) GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS +03CA;03CA;03B9 0308;03CA;03B9 0308; # (ϊ; ϊ; ι◌̈; ϊ; ι◌̈; ) GREEK SMALL LETTER IOTA WITH DIALYTIKA +03CB;03CB;03C5 0308;03CB;03C5 0308; # (ϋ; ϋ; υ◌̈; ϋ; υ◌̈; ) GREEK SMALL LETTER UPSILON WITH DIALYTIKA +03CC;03CC;03BF 0301;03CC;03BF 0301; # (ό; ό; ο◌́; ό; ο◌́; ) GREEK SMALL LETTER OMICRON WITH TONOS +03CD;03CD;03C5 0301;03CD;03C5 0301; # (ύ; ύ; υ◌́; ύ; υ◌́; ) GREEK SMALL LETTER UPSILON WITH TONOS +03CE;03CE;03C9 0301;03CE;03C9 0301; # (ώ; ώ; ω◌́; ώ; ω◌́; ) GREEK SMALL LETTER OMEGA WITH TONOS +03D0;03D0;03D0;03B2;03B2; # (ϐ; ϐ; ϐ; β; β; ) GREEK BETA SYMBOL +03D1;03D1;03D1;03B8;03B8; # (ϑ; ϑ; ϑ; θ; θ; ) GREEK THETA SYMBOL +03D2;03D2;03D2;03A5;03A5; # (ϒ; ϒ; ϒ; Υ; Υ; ) GREEK UPSILON WITH HOOK SYMBOL +03D3;03D3;03D2 0301;038E;03A5 0301; # (ϓ; ϓ; ϒ◌́; Ύ; Υ◌́; ) GREEK UPSILON WITH ACUTE AND HOOK SYMBOL +03D4;03D4;03D2 0308;03AB;03A5 0308; # (ϔ; ϔ; ϒ◌̈; Ϋ; Υ◌̈; ) GREEK UPSILON WITH DIAERESIS AND HOOK SYMBOL +03D5;03D5;03D5;03C6;03C6; # (ϕ; ϕ; ϕ; φ; φ; ) GREEK PHI SYMBOL +03D6;03D6;03D6;03C0;03C0; # (ϖ; ϖ; ϖ; π; π; ) GREEK PI SYMBOL +03F0;03F0;03F0;03BA;03BA; # (ϰ; ϰ; ϰ; κ; κ; ) GREEK KAPPA SYMBOL +03F1;03F1;03F1;03C1;03C1; # (ϱ; ϱ; ϱ; ρ; ρ; ) GREEK RHO SYMBOL +03F2;03F2;03F2;03C2;03C2; # (ϲ; ϲ; ϲ; ς; ς; ) GREEK LUNATE SIGMA SYMBOL +03F4;03F4;03F4;0398;0398; # (ϴ; ϴ; ϴ; Θ; Θ; ) GREEK CAPITAL THETA SYMBOL +03F5;03F5;03F5;03B5;03B5; # (ϵ; ϵ; ϵ; ε; ε; ) GREEK LUNATE EPSILON SYMBOL +0400;0400;0415 0300;0400;0415 0300; # (Ѐ; Ѐ; Е◌̀; Ѐ; Е◌̀; ) CYRILLIC CAPITAL LETTER IE WITH GRAVE +0401;0401;0415 0308;0401;0415 0308; # (Ё; Ё; Е◌̈; Ё; Е◌̈; ) CYRILLIC CAPITAL LETTER IO +0403;0403;0413 0301;0403;0413 0301; # (Ѓ; Ѓ; Г◌́; Ѓ; Г◌́; ) CYRILLIC CAPITAL LETTER GJE +0407;0407;0406 0308;0407;0406 0308; # (Ї; Ї; І◌̈; Ї; І◌̈; ) CYRILLIC CAPITAL LETTER YI +040C;040C;041A 0301;040C;041A 0301; # (Ќ; Ќ; К◌́; Ќ; К◌́; ) CYRILLIC CAPITAL LETTER KJE +040D;040D;0418 0300;040D;0418 0300; # (Ѝ; Ѝ; И◌̀; Ѝ; И◌̀; ) CYRILLIC CAPITAL LETTER I WITH GRAVE +040E;040E;0423 0306;040E;0423 0306; # (Ў; Ў; У◌̆; Ў; У◌̆; ) CYRILLIC CAPITAL LETTER SHORT U +0419;0419;0418 0306;0419;0418 0306; # (Й; Й; И◌̆; Й; И◌̆; ) CYRILLIC CAPITAL LETTER SHORT I +0439;0439;0438 0306;0439;0438 0306; # (й; й; и◌̆; й; и◌̆; ) CYRILLIC SMALL LETTER SHORT I +0450;0450;0435 0300;0450;0435 0300; # (ѐ; ѐ; е◌̀; ѐ; е◌̀; ) CYRILLIC SMALL LETTER IE WITH GRAVE +0451;0451;0435 0308;0451;0435 0308; # (ё; ё; е◌̈; ё; е◌̈; ) CYRILLIC SMALL LETTER IO +0453;0453;0433 0301;0453;0433 0301; # (ѓ; ѓ; г◌́; ѓ; г◌́; ) CYRILLIC SMALL LETTER GJE +0457;0457;0456 0308;0457;0456 0308; # (ї; ї; і◌̈; ї; і◌̈; ) CYRILLIC SMALL LETTER YI +045C;045C;043A 0301;045C;043A 0301; # (ќ; ќ; к◌́; ќ; к◌́; ) CYRILLIC SMALL LETTER KJE +045D;045D;0438 0300;045D;0438 0300; # (ѝ; ѝ; и◌̀; ѝ; и◌̀; ) CYRILLIC SMALL LETTER I WITH GRAVE +045E;045E;0443 0306;045E;0443 0306; # (ў; ў; у◌̆; ў; у◌̆; ) CYRILLIC SMALL LETTER SHORT U +0476;0476;0474 030F;0476;0474 030F; # (Ѷ; Ѷ; Ѵ◌̏; Ѷ; Ѵ◌̏; ) CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT +0477;0477;0475 030F;0477;0475 030F; # (ѷ; ѷ; ѵ◌̏; ѷ; ѵ◌̏; ) CYRILLIC SMALL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT +04C1;04C1;0416 0306;04C1;0416 0306; # (Ӂ; Ӂ; Ж◌̆; Ӂ; Ж◌̆; ) CYRILLIC CAPITAL LETTER ZHE WITH BREVE +04C2;04C2;0436 0306;04C2;0436 0306; # (ӂ; ӂ; ж◌̆; ӂ; ж◌̆; ) CYRILLIC SMALL LETTER ZHE WITH BREVE +04D0;04D0;0410 0306;04D0;0410 0306; # (Ӑ; Ӑ; А◌̆; Ӑ; А◌̆; ) CYRILLIC CAPITAL LETTER A WITH BREVE +04D1;04D1;0430 0306;04D1;0430 0306; # (ӑ; ӑ; а◌̆; ӑ; а◌̆; ) CYRILLIC SMALL LETTER A WITH BREVE +04D2;04D2;0410 0308;04D2;0410 0308; # (Ӓ; Ӓ; А◌̈; Ӓ; А◌̈; ) CYRILLIC CAPITAL LETTER A WITH DIAERESIS +04D3;04D3;0430 0308;04D3;0430 0308; # (ӓ; ӓ; а◌̈; ӓ; а◌̈; ) CYRILLIC SMALL LETTER A WITH DIAERESIS +04D6;04D6;0415 0306;04D6;0415 0306; # (Ӗ; Ӗ; Е◌̆; Ӗ; Е◌̆; ) CYRILLIC CAPITAL LETTER IE WITH BREVE +04D7;04D7;0435 0306;04D7;0435 0306; # (ӗ; ӗ; е◌̆; ӗ; е◌̆; ) CYRILLIC SMALL LETTER IE WITH BREVE +04DA;04DA;04D8 0308;04DA;04D8 0308; # (Ӛ; Ӛ; Ә◌̈; Ӛ; Ә◌̈; ) CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS +04DB;04DB;04D9 0308;04DB;04D9 0308; # (ӛ; ӛ; ә◌̈; ӛ; ә◌̈; ) CYRILLIC SMALL LETTER SCHWA WITH DIAERESIS +04DC;04DC;0416 0308;04DC;0416 0308; # (Ӝ; Ӝ; Ж◌̈; Ӝ; Ж◌̈; ) CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS +04DD;04DD;0436 0308;04DD;0436 0308; # (ӝ; ӝ; ж◌̈; ӝ; ж◌̈; ) CYRILLIC SMALL LETTER ZHE WITH DIAERESIS +04DE;04DE;0417 0308;04DE;0417 0308; # (Ӟ; Ӟ; З◌̈; Ӟ; З◌̈; ) CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS +04DF;04DF;0437 0308;04DF;0437 0308; # (ӟ; ӟ; з◌̈; ӟ; з◌̈; ) CYRILLIC SMALL LETTER ZE WITH DIAERESIS +04E2;04E2;0418 0304;04E2;0418 0304; # (Ӣ; Ӣ; И◌̄; Ӣ; И◌̄; ) CYRILLIC CAPITAL LETTER I WITH MACRON +04E3;04E3;0438 0304;04E3;0438 0304; # (ӣ; ӣ; и◌̄; ӣ; и◌̄; ) CYRILLIC SMALL LETTER I WITH MACRON +04E4;04E4;0418 0308;04E4;0418 0308; # (Ӥ; Ӥ; И◌̈; Ӥ; И◌̈; ) CYRILLIC CAPITAL LETTER I WITH DIAERESIS +04E5;04E5;0438 0308;04E5;0438 0308; # (ӥ; ӥ; и◌̈; ӥ; и◌̈; ) CYRILLIC SMALL LETTER I WITH DIAERESIS +04E6;04E6;041E 0308;04E6;041E 0308; # (Ӧ; Ӧ; О◌̈; Ӧ; О◌̈; ) CYRILLIC CAPITAL LETTER O WITH DIAERESIS +04E7;04E7;043E 0308;04E7;043E 0308; # (ӧ; ӧ; о◌̈; ӧ; о◌̈; ) CYRILLIC SMALL LETTER O WITH DIAERESIS +04EA;04EA;04E8 0308;04EA;04E8 0308; # (Ӫ; Ӫ; Ө◌̈; Ӫ; Ө◌̈; ) CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS +04EB;04EB;04E9 0308;04EB;04E9 0308; # (ӫ; ӫ; ө◌̈; ӫ; ө◌̈; ) CYRILLIC SMALL LETTER BARRED O WITH DIAERESIS +04EC;04EC;042D 0308;04EC;042D 0308; # (Ӭ; Ӭ; Э◌̈; Ӭ; Э◌̈; ) CYRILLIC CAPITAL LETTER E WITH DIAERESIS +04ED;04ED;044D 0308;04ED;044D 0308; # (ӭ; ӭ; э◌̈; ӭ; э◌̈; ) CYRILLIC SMALL LETTER E WITH DIAERESIS +04EE;04EE;0423 0304;04EE;0423 0304; # (Ӯ; Ӯ; У◌̄; Ӯ; У◌̄; ) CYRILLIC CAPITAL LETTER U WITH MACRON +04EF;04EF;0443 0304;04EF;0443 0304; # (ӯ; ӯ; у◌̄; ӯ; у◌̄; ) CYRILLIC SMALL LETTER U WITH MACRON +04F0;04F0;0423 0308;04F0;0423 0308; # (Ӱ; Ӱ; У◌̈; Ӱ; У◌̈; ) CYRILLIC CAPITAL LETTER U WITH DIAERESIS +04F1;04F1;0443 0308;04F1;0443 0308; # (ӱ; ӱ; у◌̈; ӱ; у◌̈; ) CYRILLIC SMALL LETTER U WITH DIAERESIS +04F2;04F2;0423 030B;04F2;0423 030B; # (Ӳ; Ӳ; У◌̋; Ӳ; У◌̋; ) CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE +04F3;04F3;0443 030B;04F3;0443 030B; # (ӳ; ӳ; у◌̋; ӳ; у◌̋; ) CYRILLIC SMALL LETTER U WITH DOUBLE ACUTE +04F4;04F4;0427 0308;04F4;0427 0308; # (Ӵ; Ӵ; Ч◌̈; Ӵ; Ч◌̈; ) CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS +04F5;04F5;0447 0308;04F5;0447 0308; # (ӵ; ӵ; ч◌̈; ӵ; ч◌̈; ) CYRILLIC SMALL LETTER CHE WITH DIAERESIS +04F8;04F8;042B 0308;04F8;042B 0308; # (Ӹ; Ӹ; Ы◌̈; Ӹ; Ы◌̈; ) CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS +04F9;04F9;044B 0308;04F9;044B 0308; # (ӹ; ӹ; ы◌̈; ӹ; ы◌̈; ) CYRILLIC SMALL LETTER YERU WITH DIAERESIS +0587;0587;0587;0565 0582;0565 0582; # (և; և; և; եւ; եւ; ) ARMENIAN SMALL LIGATURE ECH YIWN +0622;0622;0627 0653;0622;0627 0653; # (آ; آ; ا◌ٓ; آ; ا◌ٓ; ) ARABIC LETTER ALEF WITH MADDA ABOVE +0623;0623;0627 0654;0623;0627 0654; # (أ; أ; ا◌ٔ; أ; ا◌ٔ; ) ARABIC LETTER ALEF WITH HAMZA ABOVE +0624;0624;0648 0654;0624;0648 0654; # (ؤ; ؤ; و◌ٔ; ؤ; و◌ٔ; ) ARABIC LETTER WAW WITH HAMZA ABOVE +0625;0625;0627 0655;0625;0627 0655; # (إ; إ; ا◌ٕ; إ; ا◌ٕ; ) ARABIC LETTER ALEF WITH HAMZA BELOW +0626;0626;064A 0654;0626;064A 0654; # (ئ; ئ; ي◌ٔ; ئ; ي◌ٔ; ) ARABIC LETTER YEH WITH HAMZA ABOVE +0675;0675;0675;0627 0674;0627 0674; # (ٵ; ٵ; ٵ; اٴ; اٴ; ) ARABIC LETTER HIGH HAMZA ALEF +0676;0676;0676;0648 0674;0648 0674; # (ٶ; ٶ; ٶ; وٴ; وٴ; ) ARABIC LETTER HIGH HAMZA WAW +0677;0677;0677;06C7 0674;06C7 0674; # (ٷ; ٷ; ٷ; ۇٴ; ۇٴ; ) ARABIC LETTER U WITH HAMZA ABOVE +0678;0678;0678;064A 0674;064A 0674; # (ٸ; ٸ; ٸ; يٴ; يٴ; ) ARABIC LETTER HIGH HAMZA YEH +06C0;06C0;06D5 0654;06C0;06D5 0654; # (ۀ; ۀ; ە◌ٔ; ۀ; ە◌ٔ; ) ARABIC LETTER HEH WITH YEH ABOVE +06C2;06C2;06C1 0654;06C2;06C1 0654; # (ۂ; ۂ; ہ◌ٔ; ۂ; ہ◌ٔ; ) ARABIC LETTER HEH GOAL WITH HAMZA ABOVE +06D3;06D3;06D2 0654;06D3;06D2 0654; # (ۓ; ۓ; ے◌ٔ; ۓ; ے◌ٔ; ) ARABIC LETTER YEH BARREE WITH HAMZA ABOVE +0929;0929;0928 093C;0929;0928 093C; # (ऩ; ऩ; न◌़; ऩ; न◌़; ) DEVANAGARI LETTER NNNA +0931;0931;0930 093C;0931;0930 093C; # (ऱ; ऱ; र◌़; ऱ; र◌़; ) DEVANAGARI LETTER RRA +0934;0934;0933 093C;0934;0933 093C; # (ऴ; ऴ; ळ◌़; ऴ; ळ◌़; ) DEVANAGARI LETTER LLLA +0958;0915 093C;0915 093C;0915 093C;0915 093C; # (क़; क◌़; क◌़; क◌़; क◌़; ) DEVANAGARI LETTER QA +0959;0916 093C;0916 093C;0916 093C;0916 093C; # (ख़; ख◌़; ख◌़; ख◌़; ख◌़; ) DEVANAGARI LETTER KHHA +095A;0917 093C;0917 093C;0917 093C;0917 093C; # (ग़; ग◌़; ग◌़; ग◌़; ग◌़; ) DEVANAGARI LETTER GHHA +095B;091C 093C;091C 093C;091C 093C;091C 093C; # (ज़; ज◌़; ज◌़; ज◌़; ज◌़; ) DEVANAGARI LETTER ZA +095C;0921 093C;0921 093C;0921 093C;0921 093C; # (ड़; ड◌़; ड◌़; ड◌़; ड◌़; ) DEVANAGARI LETTER DDDHA +095D;0922 093C;0922 093C;0922 093C;0922 093C; # (ढ़; ढ◌़; ढ◌़; ढ◌़; ढ◌़; ) DEVANAGARI LETTER RHA +095E;092B 093C;092B 093C;092B 093C;092B 093C; # (फ़; फ◌़; फ◌़; फ◌़; फ◌़; ) DEVANAGARI LETTER FA +095F;092F 093C;092F 093C;092F 093C;092F 093C; # (य़; य◌़; य◌़; य◌़; य◌़; ) DEVANAGARI LETTER YYA +09CB;09CB;09C7 09BE;09CB;09C7 09BE; # (ো; ো; ো; ো; ো; ) BENGALI VOWEL SIGN O +09CC;09CC;09C7 09D7;09CC;09C7 09D7; # (ৌ; ৌ; ৌ; ৌ; ৌ; ) BENGALI VOWEL SIGN AU +09DC;09A1 09BC;09A1 09BC;09A1 09BC;09A1 09BC; # (ড়; ড◌়; ড◌়; ড◌়; ড◌়; ) BENGALI LETTER RRA +09DD;09A2 09BC;09A2 09BC;09A2 09BC;09A2 09BC; # (ঢ়; ঢ◌়; ঢ◌়; ঢ◌়; ঢ◌়; ) BENGALI LETTER RHA +09DF;09AF 09BC;09AF 09BC;09AF 09BC;09AF 09BC; # (য়; য◌়; য◌়; য◌়; য◌়; ) BENGALI LETTER YYA +0A33;0A32 0A3C;0A32 0A3C;0A32 0A3C;0A32 0A3C; # (ਲ਼; ਲ◌਼; ਲ◌਼; ਲ◌਼; ਲ◌਼; ) GURMUKHI LETTER LLA +0A36;0A38 0A3C;0A38 0A3C;0A38 0A3C;0A38 0A3C; # (ਸ਼; ਸ◌਼; ਸ◌਼; ਸ◌਼; ਸ◌਼; ) GURMUKHI LETTER SHA +0A59;0A16 0A3C;0A16 0A3C;0A16 0A3C;0A16 0A3C; # (ਖ਼; ਖ◌਼; ਖ◌਼; ਖ◌਼; ਖ◌਼; ) GURMUKHI LETTER KHHA +0A5A;0A17 0A3C;0A17 0A3C;0A17 0A3C;0A17 0A3C; # (ਗ਼; ਗ◌਼; ਗ◌਼; ਗ◌਼; ਗ◌਼; ) GURMUKHI LETTER GHHA +0A5B;0A1C 0A3C;0A1C 0A3C;0A1C 0A3C;0A1C 0A3C; # (ਜ਼; ਜ◌਼; ਜ◌਼; ਜ◌਼; ਜ◌਼; ) GURMUKHI LETTER ZA +0A5E;0A2B 0A3C;0A2B 0A3C;0A2B 0A3C;0A2B 0A3C; # (ਫ਼; ਫ◌਼; ਫ◌਼; ਫ◌਼; ਫ◌਼; ) GURMUKHI LETTER FA +0B48;0B48;0B47 0B56;0B48;0B47 0B56; # (ୈ; ୈ; େ◌ୖ; ୈ; େ◌ୖ; ) ORIYA VOWEL SIGN AI +0B4B;0B4B;0B47 0B3E;0B4B;0B47 0B3E; # (ୋ; ୋ; ୋ; ୋ; ୋ; ) ORIYA VOWEL SIGN O +0B4C;0B4C;0B47 0B57;0B4C;0B47 0B57; # (ୌ; ୌ; ୌ; ୌ; ୌ; ) ORIYA VOWEL SIGN AU +0B5C;0B21 0B3C;0B21 0B3C;0B21 0B3C;0B21 0B3C; # (ଡ଼; ଡ◌଼; ଡ◌଼; ଡ◌଼; ଡ◌଼; ) ORIYA LETTER RRA +0B5D;0B22 0B3C;0B22 0B3C;0B22 0B3C;0B22 0B3C; # (ଢ଼; ଢ◌଼; ଢ◌଼; ଢ◌଼; ଢ◌଼; ) ORIYA LETTER RHA +0B94;0B94;0B92 0BD7;0B94;0B92 0BD7; # (ஔ; ஔ; ஔ; ஔ; ஔ; ) TAMIL LETTER AU +0BCA;0BCA;0BC6 0BBE;0BCA;0BC6 0BBE; # (ொ; ொ; ொ; ொ; ொ; ) TAMIL VOWEL SIGN O +0BCB;0BCB;0BC7 0BBE;0BCB;0BC7 0BBE; # (ோ; ோ; ோ; ோ; ோ; ) TAMIL VOWEL SIGN OO +0BCC;0BCC;0BC6 0BD7;0BCC;0BC6 0BD7; # (ௌ; ௌ; ௌ; ௌ; ௌ; ) TAMIL VOWEL SIGN AU +0C48;0C48;0C46 0C56;0C48;0C46 0C56; # (◌ై; ◌ై; ◌ె◌ౖ; ◌ై; ◌ె◌ౖ; ) TELUGU VOWEL SIGN AI +0CC0;0CC0;0CBF 0CD5;0CC0;0CBF 0CD5; # (ೀ; ೀ; ◌ೀ; ೀ; ◌ೀ; ) KANNADA VOWEL SIGN II +0CC7;0CC7;0CC6 0CD5;0CC7;0CC6 0CD5; # (ೇ; ೇ; ◌ೇ; ೇ; ◌ೇ; ) KANNADA VOWEL SIGN EE +0CC8;0CC8;0CC6 0CD6;0CC8;0CC6 0CD6; # (ೈ; ೈ; ◌ೈ; ೈ; ◌ೈ; ) KANNADA VOWEL SIGN AI +0CCA;0CCA;0CC6 0CC2;0CCA;0CC6 0CC2; # (ೊ; ೊ; ◌ೊ; ೊ; ◌ೊ; ) KANNADA VOWEL SIGN O +0CCB;0CCB;0CC6 0CC2 0CD5;0CCB;0CC6 0CC2 0CD5; # (ೋ; ೋ; ◌ೋ; ೋ; ◌ೋ; ) KANNADA VOWEL SIGN OO +0D4A;0D4A;0D46 0D3E;0D4A;0D46 0D3E; # (ൊ; ൊ; ൊ; ൊ; ൊ; ) MALAYALAM VOWEL SIGN O +0D4B;0D4B;0D47 0D3E;0D4B;0D47 0D3E; # (ോ; ോ; ോ; ോ; ോ; ) MALAYALAM VOWEL SIGN OO +0D4C;0D4C;0D46 0D57;0D4C;0D46 0D57; # (ൌ; ൌ; ൌ; ൌ; ൌ; ) MALAYALAM VOWEL SIGN AU +0DDA;0DDA;0DD9 0DCA;0DDA;0DD9 0DCA; # (ේ; ේ; ෙ◌්; ේ; ෙ◌්; ) SINHALA VOWEL SIGN DIGA KOMBUVA +0DDC;0DDC;0DD9 0DCF;0DDC;0DD9 0DCF; # (ො; ො; ො; ො; ො; ) SINHALA VOWEL SIGN KOMBUVA HAA AELA-PILLA +0DDD;0DDD;0DD9 0DCF 0DCA;0DDD;0DD9 0DCF 0DCA; # (ෝ; ෝ; ො◌්; ෝ; ො◌්; ) SINHALA VOWEL SIGN KOMBUVA HAA DIGA AELA-PILLA +0DDE;0DDE;0DD9 0DDF;0DDE;0DD9 0DDF; # (ෞ; ෞ; ෞ; ෞ; ෞ; ) SINHALA VOWEL SIGN KOMBUVA HAA GAYANUKITTA +0E33;0E33;0E33;0E4D 0E32;0E4D 0E32; # (ำ; ำ; ำ; ◌ํา; ◌ํา; ) THAI CHARACTER SARA AM +0EB3;0EB3;0EB3;0ECD 0EB2;0ECD 0EB2; # (ຳ; ຳ; ຳ; ◌ໍາ; ◌ໍາ; ) LAO VOWEL SIGN AM +0EDC;0EDC;0EDC;0EAB 0E99;0EAB 0E99; # (ໜ; ໜ; ໜ; ຫນ; ຫນ; ) LAO HO NO +0EDD;0EDD;0EDD;0EAB 0EA1;0EAB 0EA1; # (ໝ; ໝ; ໝ; ຫມ; ຫມ; ) LAO HO MO +0F0C;0F0C;0F0C;0F0B;0F0B; # (༌; ༌; ༌; ་; ་; ) TIBETAN MARK DELIMITER TSHEG BSTAR +0F43;0F42 0FB7;0F42 0FB7;0F42 0FB7;0F42 0FB7; # (གྷ; ག◌ྷ; ག◌ྷ; ག◌ྷ; ག◌ྷ; ) TIBETAN LETTER GHA +0F4D;0F4C 0FB7;0F4C 0FB7;0F4C 0FB7;0F4C 0FB7; # (ཌྷ; ཌ◌ྷ; ཌ◌ྷ; ཌ◌ྷ; ཌ◌ྷ; ) TIBETAN LETTER DDHA +0F52;0F51 0FB7;0F51 0FB7;0F51 0FB7;0F51 0FB7; # (དྷ; ད◌ྷ; ད◌ྷ; ད◌ྷ; ད◌ྷ; ) TIBETAN LETTER DHA +0F57;0F56 0FB7;0F56 0FB7;0F56 0FB7;0F56 0FB7; # (བྷ; བ◌ྷ; བ◌ྷ; བ◌ྷ; བ◌ྷ; ) TIBETAN LETTER BHA +0F5C;0F5B 0FB7;0F5B 0FB7;0F5B 0FB7;0F5B 0FB7; # (ཛྷ; ཛ◌ྷ; ཛ◌ྷ; ཛ◌ྷ; ཛ◌ྷ; ) TIBETAN LETTER DZHA +0F69;0F40 0FB5;0F40 0FB5;0F40 0FB5;0F40 0FB5; # (ཀྵ; ཀ◌ྵ; ཀ◌ྵ; ཀ◌ྵ; ཀ◌ྵ; ) TIBETAN LETTER KSSA +0F73;0F71 0F72;0F71 0F72;0F71 0F72;0F71 0F72; # (◌ཱི; ◌ཱ◌ི; ◌ཱ◌ི; ◌ཱ◌ི; ◌ཱ◌ི; ) TIBETAN VOWEL SIGN II +0F75;0F71 0F74;0F71 0F74;0F71 0F74;0F71 0F74; # (◌ཱུ; ◌ཱ◌ུ; ◌ཱ◌ུ; ◌ཱ◌ུ; ◌ཱ◌ུ; ) TIBETAN VOWEL SIGN UU +0F76;0FB2 0F80;0FB2 0F80;0FB2 0F80;0FB2 0F80; # (◌ྲྀ; ◌ྲ◌ྀ; ◌ྲ◌ྀ; ◌ྲ◌ྀ; ◌ྲ◌ྀ; ) TIBETAN VOWEL SIGN VOCALIC R +0F77;0F77;0F77;0FB2 0F71 0F80;0FB2 0F71 0F80; # (◌ཷ; ◌ཷ; ◌ཷ; ◌ྲ◌ཱ◌ྀ; ◌ྲ◌ཱ◌ྀ; ) TIBETAN VOWEL SIGN VOCALIC RR +0F78;0FB3 0F80;0FB3 0F80;0FB3 0F80;0FB3 0F80; # (◌ླྀ; ◌ླ◌ྀ; ◌ླ◌ྀ; ◌ླ◌ྀ; ◌ླ◌ྀ; ) TIBETAN VOWEL SIGN VOCALIC L +0F79;0F79;0F79;0FB3 0F71 0F80;0FB3 0F71 0F80; # (◌ཹ; ◌ཹ; ◌ཹ; ◌ླ◌ཱ◌ྀ; ◌ླ◌ཱ◌ྀ; ) TIBETAN VOWEL SIGN VOCALIC LL +0F81;0F71 0F80;0F71 0F80;0F71 0F80;0F71 0F80; # (◌ཱྀ; ◌ཱ◌ྀ; ◌ཱ◌ྀ; ◌ཱ◌ྀ; ◌ཱ◌ྀ; ) TIBETAN VOWEL SIGN REVERSED II +0F93;0F92 0FB7;0F92 0FB7;0F92 0FB7;0F92 0FB7; # (◌ྒྷ; ◌ྒ◌ྷ; ◌ྒ◌ྷ; ◌ྒ◌ྷ; ◌ྒ◌ྷ; ) TIBETAN SUBJOINED LETTER GHA +0F9D;0F9C 0FB7;0F9C 0FB7;0F9C 0FB7;0F9C 0FB7; # (◌ྜྷ; ◌ྜ◌ྷ; ◌ྜ◌ྷ; ◌ྜ◌ྷ; ◌ྜ◌ྷ; ) TIBETAN SUBJOINED LETTER DDHA +0FA2;0FA1 0FB7;0FA1 0FB7;0FA1 0FB7;0FA1 0FB7; # (◌ྡྷ; ◌ྡ◌ྷ; ◌ྡ◌ྷ; ◌ྡ◌ྷ; ◌ྡ◌ྷ; ) TIBETAN SUBJOINED LETTER DHA +0FA7;0FA6 0FB7;0FA6 0FB7;0FA6 0FB7;0FA6 0FB7; # (◌ྦྷ; ◌ྦ◌ྷ; ◌ྦ◌ྷ; ◌ྦ◌ྷ; ◌ྦ◌ྷ; ) TIBETAN SUBJOINED LETTER BHA +0FAC;0FAB 0FB7;0FAB 0FB7;0FAB 0FB7;0FAB 0FB7; # (◌ྫྷ; ◌ྫ◌ྷ; ◌ྫ◌ྷ; ◌ྫ◌ྷ; ◌ྫ◌ྷ; ) TIBETAN SUBJOINED LETTER DZHA +0FB9;0F90 0FB5;0F90 0FB5;0F90 0FB5;0F90 0FB5; # (◌ྐྵ; ◌ྐ◌ྵ; ◌ྐ◌ྵ; ◌ྐ◌ྵ; ◌ྐ◌ྵ; ) TIBETAN SUBJOINED LETTER KSSA +1026;1026;1025 102E;1026;1025 102E; # (ဦ; ဦ; ဥ◌ီ; ဦ; ဥ◌ီ; ) MYANMAR LETTER UU +1E00;1E00;0041 0325;1E00;0041 0325; # (Ḁ; Ḁ; A◌̥; Ḁ; A◌̥; ) LATIN CAPITAL LETTER A WITH RING BELOW +1E01;1E01;0061 0325;1E01;0061 0325; # (ḁ; ḁ; a◌̥; ḁ; a◌̥; ) LATIN SMALL LETTER A WITH RING BELOW +1E02;1E02;0042 0307;1E02;0042 0307; # (Ḃ; Ḃ; B◌̇; Ḃ; B◌̇; ) LATIN CAPITAL LETTER B WITH DOT ABOVE +1E03;1E03;0062 0307;1E03;0062 0307; # (ḃ; ḃ; b◌̇; ḃ; b◌̇; ) LATIN SMALL LETTER B WITH DOT ABOVE +1E04;1E04;0042 0323;1E04;0042 0323; # (Ḅ; Ḅ; B◌̣; Ḅ; B◌̣; ) LATIN CAPITAL LETTER B WITH DOT BELOW +1E05;1E05;0062 0323;1E05;0062 0323; # (ḅ; ḅ; b◌̣; ḅ; b◌̣; ) LATIN SMALL LETTER B WITH DOT BELOW +1E06;1E06;0042 0331;1E06;0042 0331; # (Ḇ; Ḇ; B◌̱; Ḇ; B◌̱; ) LATIN CAPITAL LETTER B WITH LINE BELOW +1E07;1E07;0062 0331;1E07;0062 0331; # (ḇ; ḇ; b◌̱; ḇ; b◌̱; ) LATIN SMALL LETTER B WITH LINE BELOW +1E08;1E08;0043 0327 0301;1E08;0043 0327 0301; # (Ḉ; Ḉ; C◌̧◌́; Ḉ; C◌̧◌́; ) LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE +1E09;1E09;0063 0327 0301;1E09;0063 0327 0301; # (ḉ; ḉ; c◌̧◌́; ḉ; c◌̧◌́; ) LATIN SMALL LETTER C WITH CEDILLA AND ACUTE +1E0A;1E0A;0044 0307;1E0A;0044 0307; # (Ḋ; Ḋ; D◌̇; Ḋ; D◌̇; ) LATIN CAPITAL LETTER D WITH DOT ABOVE +1E0B;1E0B;0064 0307;1E0B;0064 0307; # (ḋ; ḋ; d◌̇; ḋ; d◌̇; ) LATIN SMALL LETTER D WITH DOT ABOVE +1E0C;1E0C;0044 0323;1E0C;0044 0323; # (Ḍ; Ḍ; D◌̣; Ḍ; D◌̣; ) LATIN CAPITAL LETTER D WITH DOT BELOW +1E0D;1E0D;0064 0323;1E0D;0064 0323; # (ḍ; ḍ; d◌̣; ḍ; d◌̣; ) LATIN SMALL LETTER D WITH DOT BELOW +1E0E;1E0E;0044 0331;1E0E;0044 0331; # (Ḏ; Ḏ; D◌̱; Ḏ; D◌̱; ) LATIN CAPITAL LETTER D WITH LINE BELOW +1E0F;1E0F;0064 0331;1E0F;0064 0331; # (ḏ; ḏ; d◌̱; ḏ; d◌̱; ) LATIN SMALL LETTER D WITH LINE BELOW +1E10;1E10;0044 0327;1E10;0044 0327; # (Ḑ; Ḑ; D◌̧; Ḑ; D◌̧; ) LATIN CAPITAL LETTER D WITH CEDILLA +1E11;1E11;0064 0327;1E11;0064 0327; # (ḑ; ḑ; d◌̧; ḑ; d◌̧; ) LATIN SMALL LETTER D WITH CEDILLA +1E12;1E12;0044 032D;1E12;0044 032D; # (Ḓ; Ḓ; D◌̭; Ḓ; D◌̭; ) LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW +1E13;1E13;0064 032D;1E13;0064 032D; # (ḓ; ḓ; d◌̭; ḓ; d◌̭; ) LATIN SMALL LETTER D WITH CIRCUMFLEX BELOW +1E14;1E14;0045 0304 0300;1E14;0045 0304 0300; # (Ḕ; Ḕ; E◌̄◌̀; Ḕ; E◌̄◌̀; ) LATIN CAPITAL LETTER E WITH MACRON AND GRAVE +1E15;1E15;0065 0304 0300;1E15;0065 0304 0300; # (ḕ; ḕ; e◌̄◌̀; ḕ; e◌̄◌̀; ) LATIN SMALL LETTER E WITH MACRON AND GRAVE +1E16;1E16;0045 0304 0301;1E16;0045 0304 0301; # (Ḗ; Ḗ; E◌̄◌́; Ḗ; E◌̄◌́; ) LATIN CAPITAL LETTER E WITH MACRON AND ACUTE +1E17;1E17;0065 0304 0301;1E17;0065 0304 0301; # (ḗ; ḗ; e◌̄◌́; ḗ; e◌̄◌́; ) LATIN SMALL LETTER E WITH MACRON AND ACUTE +1E18;1E18;0045 032D;1E18;0045 032D; # (Ḙ; Ḙ; E◌̭; Ḙ; E◌̭; ) LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW +1E19;1E19;0065 032D;1E19;0065 032D; # (ḙ; ḙ; e◌̭; ḙ; e◌̭; ) LATIN SMALL LETTER E WITH CIRCUMFLEX BELOW +1E1A;1E1A;0045 0330;1E1A;0045 0330; # (Ḛ; Ḛ; E◌̰; Ḛ; E◌̰; ) LATIN CAPITAL LETTER E WITH TILDE BELOW +1E1B;1E1B;0065 0330;1E1B;0065 0330; # (ḛ; ḛ; e◌̰; ḛ; e◌̰; ) LATIN SMALL LETTER E WITH TILDE BELOW +1E1C;1E1C;0045 0327 0306;1E1C;0045 0327 0306; # (Ḝ; Ḝ; E◌̧◌̆; Ḝ; E◌̧◌̆; ) LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE +1E1D;1E1D;0065 0327 0306;1E1D;0065 0327 0306; # (ḝ; ḝ; e◌̧◌̆; ḝ; e◌̧◌̆; ) LATIN SMALL LETTER E WITH CEDILLA AND BREVE +1E1E;1E1E;0046 0307;1E1E;0046 0307; # (Ḟ; Ḟ; F◌̇; Ḟ; F◌̇; ) LATIN CAPITAL LETTER F WITH DOT ABOVE +1E1F;1E1F;0066 0307;1E1F;0066 0307; # (ḟ; ḟ; f◌̇; ḟ; f◌̇; ) LATIN SMALL LETTER F WITH DOT ABOVE +1E20;1E20;0047 0304;1E20;0047 0304; # (Ḡ; Ḡ; G◌̄; Ḡ; G◌̄; ) LATIN CAPITAL LETTER G WITH MACRON +1E21;1E21;0067 0304;1E21;0067 0304; # (ḡ; ḡ; g◌̄; ḡ; g◌̄; ) LATIN SMALL LETTER G WITH MACRON +1E22;1E22;0048 0307;1E22;0048 0307; # (Ḣ; Ḣ; H◌̇; Ḣ; H◌̇; ) LATIN CAPITAL LETTER H WITH DOT ABOVE +1E23;1E23;0068 0307;1E23;0068 0307; # (ḣ; ḣ; h◌̇; ḣ; h◌̇; ) LATIN SMALL LETTER H WITH DOT ABOVE +1E24;1E24;0048 0323;1E24;0048 0323; # (Ḥ; Ḥ; H◌̣; Ḥ; H◌̣; ) LATIN CAPITAL LETTER H WITH DOT BELOW +1E25;1E25;0068 0323;1E25;0068 0323; # (ḥ; ḥ; h◌̣; ḥ; h◌̣; ) LATIN SMALL LETTER H WITH DOT BELOW +1E26;1E26;0048 0308;1E26;0048 0308; # (Ḧ; Ḧ; H◌̈; Ḧ; H◌̈; ) LATIN CAPITAL LETTER H WITH DIAERESIS +1E27;1E27;0068 0308;1E27;0068 0308; # (ḧ; ḧ; h◌̈; ḧ; h◌̈; ) LATIN SMALL LETTER H WITH DIAERESIS +1E28;1E28;0048 0327;1E28;0048 0327; # (Ḩ; Ḩ; H◌̧; Ḩ; H◌̧; ) LATIN CAPITAL LETTER H WITH CEDILLA +1E29;1E29;0068 0327;1E29;0068 0327; # (ḩ; ḩ; h◌̧; ḩ; h◌̧; ) LATIN SMALL LETTER H WITH CEDILLA +1E2A;1E2A;0048 032E;1E2A;0048 032E; # (Ḫ; Ḫ; H◌̮; Ḫ; H◌̮; ) LATIN CAPITAL LETTER H WITH BREVE BELOW +1E2B;1E2B;0068 032E;1E2B;0068 032E; # (ḫ; ḫ; h◌̮; ḫ; h◌̮; ) LATIN SMALL LETTER H WITH BREVE BELOW +1E2C;1E2C;0049 0330;1E2C;0049 0330; # (Ḭ; Ḭ; I◌̰; Ḭ; I◌̰; ) LATIN CAPITAL LETTER I WITH TILDE BELOW +1E2D;1E2D;0069 0330;1E2D;0069 0330; # (ḭ; ḭ; i◌̰; ḭ; i◌̰; ) LATIN SMALL LETTER I WITH TILDE BELOW +1E2E;1E2E;0049 0308 0301;1E2E;0049 0308 0301; # (Ḯ; Ḯ; I◌̈◌́; Ḯ; I◌̈◌́; ) LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE +1E2F;1E2F;0069 0308 0301;1E2F;0069 0308 0301; # (ḯ; ḯ; i◌̈◌́; ḯ; i◌̈◌́; ) LATIN SMALL LETTER I WITH DIAERESIS AND ACUTE +1E30;1E30;004B 0301;1E30;004B 0301; # (Ḱ; Ḱ; K◌́; Ḱ; K◌́; ) LATIN CAPITAL LETTER K WITH ACUTE +1E31;1E31;006B 0301;1E31;006B 0301; # (ḱ; ḱ; k◌́; ḱ; k◌́; ) LATIN SMALL LETTER K WITH ACUTE +1E32;1E32;004B 0323;1E32;004B 0323; # (Ḳ; Ḳ; K◌̣; Ḳ; K◌̣; ) LATIN CAPITAL LETTER K WITH DOT BELOW +1E33;1E33;006B 0323;1E33;006B 0323; # (ḳ; ḳ; k◌̣; ḳ; k◌̣; ) LATIN SMALL LETTER K WITH DOT BELOW +1E34;1E34;004B 0331;1E34;004B 0331; # (Ḵ; Ḵ; K◌̱; Ḵ; K◌̱; ) LATIN CAPITAL LETTER K WITH LINE BELOW +1E35;1E35;006B 0331;1E35;006B 0331; # (ḵ; ḵ; k◌̱; ḵ; k◌̱; ) LATIN SMALL LETTER K WITH LINE BELOW +1E36;1E36;004C 0323;1E36;004C 0323; # (Ḷ; Ḷ; L◌̣; Ḷ; L◌̣; ) LATIN CAPITAL LETTER L WITH DOT BELOW +1E37;1E37;006C 0323;1E37;006C 0323; # (ḷ; ḷ; l◌̣; ḷ; l◌̣; ) LATIN SMALL LETTER L WITH DOT BELOW +1E38;1E38;004C 0323 0304;1E38;004C 0323 0304; # (Ḹ; Ḹ; L◌̣◌̄; Ḹ; L◌̣◌̄; ) LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON +1E39;1E39;006C 0323 0304;1E39;006C 0323 0304; # (ḹ; ḹ; l◌̣◌̄; ḹ; l◌̣◌̄; ) LATIN SMALL LETTER L WITH DOT BELOW AND MACRON +1E3A;1E3A;004C 0331;1E3A;004C 0331; # (Ḻ; Ḻ; L◌̱; Ḻ; L◌̱; ) LATIN CAPITAL LETTER L WITH LINE BELOW +1E3B;1E3B;006C 0331;1E3B;006C 0331; # (ḻ; ḻ; l◌̱; ḻ; l◌̱; ) LATIN SMALL LETTER L WITH LINE BELOW +1E3C;1E3C;004C 032D;1E3C;004C 032D; # (Ḽ; Ḽ; L◌̭; Ḽ; L◌̭; ) LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW +1E3D;1E3D;006C 032D;1E3D;006C 032D; # (ḽ; ḽ; l◌̭; ḽ; l◌̭; ) LATIN SMALL LETTER L WITH CIRCUMFLEX BELOW +1E3E;1E3E;004D 0301;1E3E;004D 0301; # (Ḿ; Ḿ; M◌́; Ḿ; M◌́; ) LATIN CAPITAL LETTER M WITH ACUTE +1E3F;1E3F;006D 0301;1E3F;006D 0301; # (ḿ; ḿ; m◌́; ḿ; m◌́; ) LATIN SMALL LETTER M WITH ACUTE +1E40;1E40;004D 0307;1E40;004D 0307; # (Ṁ; Ṁ; M◌̇; Ṁ; M◌̇; ) LATIN CAPITAL LETTER M WITH DOT ABOVE +1E41;1E41;006D 0307;1E41;006D 0307; # (ṁ; ṁ; m◌̇; ṁ; m◌̇; ) LATIN SMALL LETTER M WITH DOT ABOVE +1E42;1E42;004D 0323;1E42;004D 0323; # (Ṃ; Ṃ; M◌̣; Ṃ; M◌̣; ) LATIN CAPITAL LETTER M WITH DOT BELOW +1E43;1E43;006D 0323;1E43;006D 0323; # (ṃ; ṃ; m◌̣; ṃ; m◌̣; ) LATIN SMALL LETTER M WITH DOT BELOW +1E44;1E44;004E 0307;1E44;004E 0307; # (Ṅ; Ṅ; N◌̇; Ṅ; N◌̇; ) LATIN CAPITAL LETTER N WITH DOT ABOVE +1E45;1E45;006E 0307;1E45;006E 0307; # (ṅ; ṅ; n◌̇; ṅ; n◌̇; ) LATIN SMALL LETTER N WITH DOT ABOVE +1E46;1E46;004E 0323;1E46;004E 0323; # (Ṇ; Ṇ; N◌̣; Ṇ; N◌̣; ) LATIN CAPITAL LETTER N WITH DOT BELOW +1E47;1E47;006E 0323;1E47;006E 0323; # (ṇ; ṇ; n◌̣; ṇ; n◌̣; ) LATIN SMALL LETTER N WITH DOT BELOW +1E48;1E48;004E 0331;1E48;004E 0331; # (Ṉ; Ṉ; N◌̱; Ṉ; N◌̱; ) LATIN CAPITAL LETTER N WITH LINE BELOW +1E49;1E49;006E 0331;1E49;006E 0331; # (ṉ; ṉ; n◌̱; ṉ; n◌̱; ) LATIN SMALL LETTER N WITH LINE BELOW +1E4A;1E4A;004E 032D;1E4A;004E 032D; # (Ṋ; Ṋ; N◌̭; Ṋ; N◌̭; ) LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW +1E4B;1E4B;006E 032D;1E4B;006E 032D; # (ṋ; ṋ; n◌̭; ṋ; n◌̭; ) LATIN SMALL LETTER N WITH CIRCUMFLEX BELOW +1E4C;1E4C;004F 0303 0301;1E4C;004F 0303 0301; # (Ṍ; Ṍ; O◌̃◌́; Ṍ; O◌̃◌́; ) LATIN CAPITAL LETTER O WITH TILDE AND ACUTE +1E4D;1E4D;006F 0303 0301;1E4D;006F 0303 0301; # (ṍ; ṍ; o◌̃◌́; ṍ; o◌̃◌́; ) LATIN SMALL LETTER O WITH TILDE AND ACUTE +1E4E;1E4E;004F 0303 0308;1E4E;004F 0303 0308; # (Ṏ; Ṏ; O◌̃◌̈; Ṏ; O◌̃◌̈; ) LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS +1E4F;1E4F;006F 0303 0308;1E4F;006F 0303 0308; # (ṏ; ṏ; o◌̃◌̈; ṏ; o◌̃◌̈; ) LATIN SMALL LETTER O WITH TILDE AND DIAERESIS +1E50;1E50;004F 0304 0300;1E50;004F 0304 0300; # (Ṑ; Ṑ; O◌̄◌̀; Ṑ; O◌̄◌̀; ) LATIN CAPITAL LETTER O WITH MACRON AND GRAVE +1E51;1E51;006F 0304 0300;1E51;006F 0304 0300; # (ṑ; ṑ; o◌̄◌̀; ṑ; o◌̄◌̀; ) LATIN SMALL LETTER O WITH MACRON AND GRAVE +1E52;1E52;004F 0304 0301;1E52;004F 0304 0301; # (Ṓ; Ṓ; O◌̄◌́; Ṓ; O◌̄◌́; ) LATIN CAPITAL LETTER O WITH MACRON AND ACUTE +1E53;1E53;006F 0304 0301;1E53;006F 0304 0301; # (ṓ; ṓ; o◌̄◌́; ṓ; o◌̄◌́; ) LATIN SMALL LETTER O WITH MACRON AND ACUTE +1E54;1E54;0050 0301;1E54;0050 0301; # (Ṕ; Ṕ; P◌́; Ṕ; P◌́; ) LATIN CAPITAL LETTER P WITH ACUTE +1E55;1E55;0070 0301;1E55;0070 0301; # (ṕ; ṕ; p◌́; ṕ; p◌́; ) LATIN SMALL LETTER P WITH ACUTE +1E56;1E56;0050 0307;1E56;0050 0307; # (Ṗ; Ṗ; P◌̇; Ṗ; P◌̇; ) LATIN CAPITAL LETTER P WITH DOT ABOVE +1E57;1E57;0070 0307;1E57;0070 0307; # (ṗ; ṗ; p◌̇; ṗ; p◌̇; ) LATIN SMALL LETTER P WITH DOT ABOVE +1E58;1E58;0052 0307;1E58;0052 0307; # (Ṙ; Ṙ; R◌̇; Ṙ; R◌̇; ) LATIN CAPITAL LETTER R WITH DOT ABOVE +1E59;1E59;0072 0307;1E59;0072 0307; # (ṙ; ṙ; r◌̇; ṙ; r◌̇; ) LATIN SMALL LETTER R WITH DOT ABOVE +1E5A;1E5A;0052 0323;1E5A;0052 0323; # (Ṛ; Ṛ; R◌̣; Ṛ; R◌̣; ) LATIN CAPITAL LETTER R WITH DOT BELOW +1E5B;1E5B;0072 0323;1E5B;0072 0323; # (ṛ; ṛ; r◌̣; ṛ; r◌̣; ) LATIN SMALL LETTER R WITH DOT BELOW +1E5C;1E5C;0052 0323 0304;1E5C;0052 0323 0304; # (Ṝ; Ṝ; R◌̣◌̄; Ṝ; R◌̣◌̄; ) LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON +1E5D;1E5D;0072 0323 0304;1E5D;0072 0323 0304; # (ṝ; ṝ; r◌̣◌̄; ṝ; r◌̣◌̄; ) LATIN SMALL LETTER R WITH DOT BELOW AND MACRON +1E5E;1E5E;0052 0331;1E5E;0052 0331; # (Ṟ; Ṟ; R◌̱; Ṟ; R◌̱; ) LATIN CAPITAL LETTER R WITH LINE BELOW +1E5F;1E5F;0072 0331;1E5F;0072 0331; # (ṟ; ṟ; r◌̱; ṟ; r◌̱; ) LATIN SMALL LETTER R WITH LINE BELOW +1E60;1E60;0053 0307;1E60;0053 0307; # (Ṡ; Ṡ; S◌̇; Ṡ; S◌̇; ) LATIN CAPITAL LETTER S WITH DOT ABOVE +1E61;1E61;0073 0307;1E61;0073 0307; # (ṡ; ṡ; s◌̇; ṡ; s◌̇; ) LATIN SMALL LETTER S WITH DOT ABOVE +1E62;1E62;0053 0323;1E62;0053 0323; # (Ṣ; Ṣ; S◌̣; Ṣ; S◌̣; ) LATIN CAPITAL LETTER S WITH DOT BELOW +1E63;1E63;0073 0323;1E63;0073 0323; # (ṣ; ṣ; s◌̣; ṣ; s◌̣; ) LATIN SMALL LETTER S WITH DOT BELOW +1E64;1E64;0053 0301 0307;1E64;0053 0301 0307; # (Ṥ; Ṥ; S◌́◌̇; Ṥ; S◌́◌̇; ) LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE +1E65;1E65;0073 0301 0307;1E65;0073 0301 0307; # (ṥ; ṥ; s◌́◌̇; ṥ; s◌́◌̇; ) LATIN SMALL LETTER S WITH ACUTE AND DOT ABOVE +1E66;1E66;0053 030C 0307;1E66;0053 030C 0307; # (Ṧ; Ṧ; S◌̌◌̇; Ṧ; S◌̌◌̇; ) LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE +1E67;1E67;0073 030C 0307;1E67;0073 030C 0307; # (ṧ; ṧ; s◌̌◌̇; ṧ; s◌̌◌̇; ) LATIN SMALL LETTER S WITH CARON AND DOT ABOVE +1E68;1E68;0053 0323 0307;1E68;0053 0323 0307; # (Ṩ; Ṩ; S◌̣◌̇; Ṩ; S◌̣◌̇; ) LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE +1E69;1E69;0073 0323 0307;1E69;0073 0323 0307; # (ṩ; ṩ; s◌̣◌̇; ṩ; s◌̣◌̇; ) LATIN SMALL LETTER S WITH DOT BELOW AND DOT ABOVE +1E6A;1E6A;0054 0307;1E6A;0054 0307; # (Ṫ; Ṫ; T◌̇; Ṫ; T◌̇; ) LATIN CAPITAL LETTER T WITH DOT ABOVE +1E6B;1E6B;0074 0307;1E6B;0074 0307; # (ṫ; ṫ; t◌̇; ṫ; t◌̇; ) LATIN SMALL LETTER T WITH DOT ABOVE +1E6C;1E6C;0054 0323;1E6C;0054 0323; # (Ṭ; Ṭ; T◌̣; Ṭ; T◌̣; ) LATIN CAPITAL LETTER T WITH DOT BELOW +1E6D;1E6D;0074 0323;1E6D;0074 0323; # (ṭ; ṭ; t◌̣; ṭ; t◌̣; ) LATIN SMALL LETTER T WITH DOT BELOW +1E6E;1E6E;0054 0331;1E6E;0054 0331; # (Ṯ; Ṯ; T◌̱; Ṯ; T◌̱; ) LATIN CAPITAL LETTER T WITH LINE BELOW +1E6F;1E6F;0074 0331;1E6F;0074 0331; # (ṯ; ṯ; t◌̱; ṯ; t◌̱; ) LATIN SMALL LETTER T WITH LINE BELOW +1E70;1E70;0054 032D;1E70;0054 032D; # (Ṱ; Ṱ; T◌̭; Ṱ; T◌̭; ) LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW +1E71;1E71;0074 032D;1E71;0074 032D; # (ṱ; ṱ; t◌̭; ṱ; t◌̭; ) LATIN SMALL LETTER T WITH CIRCUMFLEX BELOW +1E72;1E72;0055 0324;1E72;0055 0324; # (Ṳ; Ṳ; U◌̤; Ṳ; U◌̤; ) LATIN CAPITAL LETTER U WITH DIAERESIS BELOW +1E73;1E73;0075 0324;1E73;0075 0324; # (ṳ; ṳ; u◌̤; ṳ; u◌̤; ) LATIN SMALL LETTER U WITH DIAERESIS BELOW +1E74;1E74;0055 0330;1E74;0055 0330; # (Ṵ; Ṵ; U◌̰; Ṵ; U◌̰; ) LATIN CAPITAL LETTER U WITH TILDE BELOW +1E75;1E75;0075 0330;1E75;0075 0330; # (ṵ; ṵ; u◌̰; ṵ; u◌̰; ) LATIN SMALL LETTER U WITH TILDE BELOW +1E76;1E76;0055 032D;1E76;0055 032D; # (Ṷ; Ṷ; U◌̭; Ṷ; U◌̭; ) LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW +1E77;1E77;0075 032D;1E77;0075 032D; # (ṷ; ṷ; u◌̭; ṷ; u◌̭; ) LATIN SMALL LETTER U WITH CIRCUMFLEX BELOW +1E78;1E78;0055 0303 0301;1E78;0055 0303 0301; # (Ṹ; Ṹ; U◌̃◌́; Ṹ; U◌̃◌́; ) LATIN CAPITAL LETTER U WITH TILDE AND ACUTE +1E79;1E79;0075 0303 0301;1E79;0075 0303 0301; # (ṹ; ṹ; u◌̃◌́; ṹ; u◌̃◌́; ) LATIN SMALL LETTER U WITH TILDE AND ACUTE +1E7A;1E7A;0055 0304 0308;1E7A;0055 0304 0308; # (Ṻ; Ṻ; U◌̄◌̈; Ṻ; U◌̄◌̈; ) LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS +1E7B;1E7B;0075 0304 0308;1E7B;0075 0304 0308; # (ṻ; ṻ; u◌̄◌̈; ṻ; u◌̄◌̈; ) LATIN SMALL LETTER U WITH MACRON AND DIAERESIS +1E7C;1E7C;0056 0303;1E7C;0056 0303; # (Ṽ; Ṽ; V◌̃; Ṽ; V◌̃; ) LATIN CAPITAL LETTER V WITH TILDE +1E7D;1E7D;0076 0303;1E7D;0076 0303; # (ṽ; ṽ; v◌̃; ṽ; v◌̃; ) LATIN SMALL LETTER V WITH TILDE +1E7E;1E7E;0056 0323;1E7E;0056 0323; # (Ṿ; Ṿ; V◌̣; Ṿ; V◌̣; ) LATIN CAPITAL LETTER V WITH DOT BELOW +1E7F;1E7F;0076 0323;1E7F;0076 0323; # (ṿ; ṿ; v◌̣; ṿ; v◌̣; ) LATIN SMALL LETTER V WITH DOT BELOW +1E80;1E80;0057 0300;1E80;0057 0300; # (Ẁ; Ẁ; W◌̀; Ẁ; W◌̀; ) LATIN CAPITAL LETTER W WITH GRAVE +1E81;1E81;0077 0300;1E81;0077 0300; # (ẁ; ẁ; w◌̀; ẁ; w◌̀; ) LATIN SMALL LETTER W WITH GRAVE +1E82;1E82;0057 0301;1E82;0057 0301; # (Ẃ; Ẃ; W◌́; Ẃ; W◌́; ) LATIN CAPITAL LETTER W WITH ACUTE +1E83;1E83;0077 0301;1E83;0077 0301; # (ẃ; ẃ; w◌́; ẃ; w◌́; ) LATIN SMALL LETTER W WITH ACUTE +1E84;1E84;0057 0308;1E84;0057 0308; # (Ẅ; Ẅ; W◌̈; Ẅ; W◌̈; ) LATIN CAPITAL LETTER W WITH DIAERESIS +1E85;1E85;0077 0308;1E85;0077 0308; # (ẅ; ẅ; w◌̈; ẅ; w◌̈; ) LATIN SMALL LETTER W WITH DIAERESIS +1E86;1E86;0057 0307;1E86;0057 0307; # (Ẇ; Ẇ; W◌̇; Ẇ; W◌̇; ) LATIN CAPITAL LETTER W WITH DOT ABOVE +1E87;1E87;0077 0307;1E87;0077 0307; # (ẇ; ẇ; w◌̇; ẇ; w◌̇; ) LATIN SMALL LETTER W WITH DOT ABOVE +1E88;1E88;0057 0323;1E88;0057 0323; # (Ẉ; Ẉ; W◌̣; Ẉ; W◌̣; ) LATIN CAPITAL LETTER W WITH DOT BELOW +1E89;1E89;0077 0323;1E89;0077 0323; # (ẉ; ẉ; w◌̣; ẉ; w◌̣; ) LATIN SMALL LETTER W WITH DOT BELOW +1E8A;1E8A;0058 0307;1E8A;0058 0307; # (Ẋ; Ẋ; X◌̇; Ẋ; X◌̇; ) LATIN CAPITAL LETTER X WITH DOT ABOVE +1E8B;1E8B;0078 0307;1E8B;0078 0307; # (ẋ; ẋ; x◌̇; ẋ; x◌̇; ) LATIN SMALL LETTER X WITH DOT ABOVE +1E8C;1E8C;0058 0308;1E8C;0058 0308; # (Ẍ; Ẍ; X◌̈; Ẍ; X◌̈; ) LATIN CAPITAL LETTER X WITH DIAERESIS +1E8D;1E8D;0078 0308;1E8D;0078 0308; # (ẍ; ẍ; x◌̈; ẍ; x◌̈; ) LATIN SMALL LETTER X WITH DIAERESIS +1E8E;1E8E;0059 0307;1E8E;0059 0307; # (Ẏ; Ẏ; Y◌̇; Ẏ; Y◌̇; ) LATIN CAPITAL LETTER Y WITH DOT ABOVE +1E8F;1E8F;0079 0307;1E8F;0079 0307; # (ẏ; ẏ; y◌̇; ẏ; y◌̇; ) LATIN SMALL LETTER Y WITH DOT ABOVE +1E90;1E90;005A 0302;1E90;005A 0302; # (Ẑ; Ẑ; Z◌̂; Ẑ; Z◌̂; ) LATIN CAPITAL LETTER Z WITH CIRCUMFLEX +1E91;1E91;007A 0302;1E91;007A 0302; # (ẑ; ẑ; z◌̂; ẑ; z◌̂; ) LATIN SMALL LETTER Z WITH CIRCUMFLEX +1E92;1E92;005A 0323;1E92;005A 0323; # (Ẓ; Ẓ; Z◌̣; Ẓ; Z◌̣; ) LATIN CAPITAL LETTER Z WITH DOT BELOW +1E93;1E93;007A 0323;1E93;007A 0323; # (ẓ; ẓ; z◌̣; ẓ; z◌̣; ) LATIN SMALL LETTER Z WITH DOT BELOW +1E94;1E94;005A 0331;1E94;005A 0331; # (Ẕ; Ẕ; Z◌̱; Ẕ; Z◌̱; ) LATIN CAPITAL LETTER Z WITH LINE BELOW +1E95;1E95;007A 0331;1E95;007A 0331; # (ẕ; ẕ; z◌̱; ẕ; z◌̱; ) LATIN SMALL LETTER Z WITH LINE BELOW +1E96;1E96;0068 0331;1E96;0068 0331; # (ẖ; ẖ; h◌̱; ẖ; h◌̱; ) LATIN SMALL LETTER H WITH LINE BELOW +1E97;1E97;0074 0308;1E97;0074 0308; # (ẗ; ẗ; t◌̈; ẗ; t◌̈; ) LATIN SMALL LETTER T WITH DIAERESIS +1E98;1E98;0077 030A;1E98;0077 030A; # (ẘ; ẘ; w◌̊; ẘ; w◌̊; ) LATIN SMALL LETTER W WITH RING ABOVE +1E99;1E99;0079 030A;1E99;0079 030A; # (ẙ; ẙ; y◌̊; ẙ; y◌̊; ) LATIN SMALL LETTER Y WITH RING ABOVE +1E9A;1E9A;1E9A;0061 02BE;0061 02BE; # (ẚ; ẚ; ẚ; aʾ; aʾ; ) LATIN SMALL LETTER A WITH RIGHT HALF RING +1E9B;1E9B;017F 0307;1E61;0073 0307; # (ẛ; ẛ; ſ◌̇; ṡ; s◌̇; ) LATIN SMALL LETTER LONG S WITH DOT ABOVE +1EA0;1EA0;0041 0323;1EA0;0041 0323; # (Ạ; Ạ; A◌̣; Ạ; A◌̣; ) LATIN CAPITAL LETTER A WITH DOT BELOW +1EA1;1EA1;0061 0323;1EA1;0061 0323; # (ạ; ạ; a◌̣; ạ; a◌̣; ) LATIN SMALL LETTER A WITH DOT BELOW +1EA2;1EA2;0041 0309;1EA2;0041 0309; # (Ả; Ả; A◌̉; Ả; A◌̉; ) LATIN CAPITAL LETTER A WITH HOOK ABOVE +1EA3;1EA3;0061 0309;1EA3;0061 0309; # (ả; ả; a◌̉; ả; a◌̉; ) LATIN SMALL LETTER A WITH HOOK ABOVE +1EA4;1EA4;0041 0302 0301;1EA4;0041 0302 0301; # (Ấ; Ấ; A◌̂◌́; Ấ; A◌̂◌́; ) LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE +1EA5;1EA5;0061 0302 0301;1EA5;0061 0302 0301; # (ấ; ấ; a◌̂◌́; ấ; a◌̂◌́; ) LATIN SMALL LETTER A WITH CIRCUMFLEX AND ACUTE +1EA6;1EA6;0041 0302 0300;1EA6;0041 0302 0300; # (Ầ; Ầ; A◌̂◌̀; Ầ; A◌̂◌̀; ) LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE +1EA7;1EA7;0061 0302 0300;1EA7;0061 0302 0300; # (ầ; ầ; a◌̂◌̀; ầ; a◌̂◌̀; ) LATIN SMALL LETTER A WITH CIRCUMFLEX AND GRAVE +1EA8;1EA8;0041 0302 0309;1EA8;0041 0302 0309; # (Ẩ; Ẩ; A◌̂◌̉; Ẩ; A◌̂◌̉; ) LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE +1EA9;1EA9;0061 0302 0309;1EA9;0061 0302 0309; # (ẩ; ẩ; a◌̂◌̉; ẩ; a◌̂◌̉; ) LATIN SMALL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE +1EAA;1EAA;0041 0302 0303;1EAA;0041 0302 0303; # (Ẫ; Ẫ; A◌̂◌̃; Ẫ; A◌̂◌̃; ) LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE +1EAB;1EAB;0061 0302 0303;1EAB;0061 0302 0303; # (ẫ; ẫ; a◌̂◌̃; ẫ; a◌̂◌̃; ) LATIN SMALL LETTER A WITH CIRCUMFLEX AND TILDE +1EAC;1EAC;0041 0323 0302;1EAC;0041 0323 0302; # (Ậ; Ậ; A◌̣◌̂; Ậ; A◌̣◌̂; ) LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW +1EAD;1EAD;0061 0323 0302;1EAD;0061 0323 0302; # (ậ; ậ; a◌̣◌̂; ậ; a◌̣◌̂; ) LATIN SMALL LETTER A WITH CIRCUMFLEX AND DOT BELOW +1EAE;1EAE;0041 0306 0301;1EAE;0041 0306 0301; # (Ắ; Ắ; A◌̆◌́; Ắ; A◌̆◌́; ) LATIN CAPITAL LETTER A WITH BREVE AND ACUTE +1EAF;1EAF;0061 0306 0301;1EAF;0061 0306 0301; # (ắ; ắ; a◌̆◌́; ắ; a◌̆◌́; ) LATIN SMALL LETTER A WITH BREVE AND ACUTE +1EB0;1EB0;0041 0306 0300;1EB0;0041 0306 0300; # (Ằ; Ằ; A◌̆◌̀; Ằ; A◌̆◌̀; ) LATIN CAPITAL LETTER A WITH BREVE AND GRAVE +1EB1;1EB1;0061 0306 0300;1EB1;0061 0306 0300; # (ằ; ằ; a◌̆◌̀; ằ; a◌̆◌̀; ) LATIN SMALL LETTER A WITH BREVE AND GRAVE +1EB2;1EB2;0041 0306 0309;1EB2;0041 0306 0309; # (Ẳ; Ẳ; A◌̆◌̉; Ẳ; A◌̆◌̉; ) LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE +1EB3;1EB3;0061 0306 0309;1EB3;0061 0306 0309; # (ẳ; ẳ; a◌̆◌̉; ẳ; a◌̆◌̉; ) LATIN SMALL LETTER A WITH BREVE AND HOOK ABOVE +1EB4;1EB4;0041 0306 0303;1EB4;0041 0306 0303; # (Ẵ; Ẵ; A◌̆◌̃; Ẵ; A◌̆◌̃; ) LATIN CAPITAL LETTER A WITH BREVE AND TILDE +1EB5;1EB5;0061 0306 0303;1EB5;0061 0306 0303; # (ẵ; ẵ; a◌̆◌̃; ẵ; a◌̆◌̃; ) LATIN SMALL LETTER A WITH BREVE AND TILDE +1EB6;1EB6;0041 0323 0306;1EB6;0041 0323 0306; # (Ặ; Ặ; A◌̣◌̆; Ặ; A◌̣◌̆; ) LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW +1EB7;1EB7;0061 0323 0306;1EB7;0061 0323 0306; # (ặ; ặ; a◌̣◌̆; ặ; a◌̣◌̆; ) LATIN SMALL LETTER A WITH BREVE AND DOT BELOW +1EB8;1EB8;0045 0323;1EB8;0045 0323; # (Ẹ; Ẹ; E◌̣; Ẹ; E◌̣; ) LATIN CAPITAL LETTER E WITH DOT BELOW +1EB9;1EB9;0065 0323;1EB9;0065 0323; # (ẹ; ẹ; e◌̣; ẹ; e◌̣; ) LATIN SMALL LETTER E WITH DOT BELOW +1EBA;1EBA;0045 0309;1EBA;0045 0309; # (Ẻ; Ẻ; E◌̉; Ẻ; E◌̉; ) LATIN CAPITAL LETTER E WITH HOOK ABOVE +1EBB;1EBB;0065 0309;1EBB;0065 0309; # (ẻ; ẻ; e◌̉; ẻ; e◌̉; ) LATIN SMALL LETTER E WITH HOOK ABOVE +1EBC;1EBC;0045 0303;1EBC;0045 0303; # (Ẽ; Ẽ; E◌̃; Ẽ; E◌̃; ) LATIN CAPITAL LETTER E WITH TILDE +1EBD;1EBD;0065 0303;1EBD;0065 0303; # (ẽ; ẽ; e◌̃; ẽ; e◌̃; ) LATIN SMALL LETTER E WITH TILDE +1EBE;1EBE;0045 0302 0301;1EBE;0045 0302 0301; # (Ế; Ế; E◌̂◌́; Ế; E◌̂◌́; ) LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE +1EBF;1EBF;0065 0302 0301;1EBF;0065 0302 0301; # (ế; ế; e◌̂◌́; ế; e◌̂◌́; ) LATIN SMALL LETTER E WITH CIRCUMFLEX AND ACUTE +1EC0;1EC0;0045 0302 0300;1EC0;0045 0302 0300; # (Ề; Ề; E◌̂◌̀; Ề; E◌̂◌̀; ) LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE +1EC1;1EC1;0065 0302 0300;1EC1;0065 0302 0300; # (ề; ề; e◌̂◌̀; ề; e◌̂◌̀; ) LATIN SMALL LETTER E WITH CIRCUMFLEX AND GRAVE +1EC2;1EC2;0045 0302 0309;1EC2;0045 0302 0309; # (Ể; Ể; E◌̂◌̉; Ể; E◌̂◌̉; ) LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE +1EC3;1EC3;0065 0302 0309;1EC3;0065 0302 0309; # (ể; ể; e◌̂◌̉; ể; e◌̂◌̉; ) LATIN SMALL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE +1EC4;1EC4;0045 0302 0303;1EC4;0045 0302 0303; # (Ễ; Ễ; E◌̂◌̃; Ễ; E◌̂◌̃; ) LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE +1EC5;1EC5;0065 0302 0303;1EC5;0065 0302 0303; # (ễ; ễ; e◌̂◌̃; ễ; e◌̂◌̃; ) LATIN SMALL LETTER E WITH CIRCUMFLEX AND TILDE +1EC6;1EC6;0045 0323 0302;1EC6;0045 0323 0302; # (Ệ; Ệ; E◌̣◌̂; Ệ; E◌̣◌̂; ) LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW +1EC7;1EC7;0065 0323 0302;1EC7;0065 0323 0302; # (ệ; ệ; e◌̣◌̂; ệ; e◌̣◌̂; ) LATIN SMALL LETTER E WITH CIRCUMFLEX AND DOT BELOW +1EC8;1EC8;0049 0309;1EC8;0049 0309; # (Ỉ; Ỉ; I◌̉; Ỉ; I◌̉; ) LATIN CAPITAL LETTER I WITH HOOK ABOVE +1EC9;1EC9;0069 0309;1EC9;0069 0309; # (ỉ; ỉ; i◌̉; ỉ; i◌̉; ) LATIN SMALL LETTER I WITH HOOK ABOVE +1ECA;1ECA;0049 0323;1ECA;0049 0323; # (Ị; Ị; I◌̣; Ị; I◌̣; ) LATIN CAPITAL LETTER I WITH DOT BELOW +1ECB;1ECB;0069 0323;1ECB;0069 0323; # (ị; ị; i◌̣; ị; i◌̣; ) LATIN SMALL LETTER I WITH DOT BELOW +1ECC;1ECC;004F 0323;1ECC;004F 0323; # (Ọ; Ọ; O◌̣; Ọ; O◌̣; ) LATIN CAPITAL LETTER O WITH DOT BELOW +1ECD;1ECD;006F 0323;1ECD;006F 0323; # (ọ; ọ; o◌̣; ọ; o◌̣; ) LATIN SMALL LETTER O WITH DOT BELOW +1ECE;1ECE;004F 0309;1ECE;004F 0309; # (Ỏ; Ỏ; O◌̉; Ỏ; O◌̉; ) LATIN CAPITAL LETTER O WITH HOOK ABOVE +1ECF;1ECF;006F 0309;1ECF;006F 0309; # (ỏ; ỏ; o◌̉; ỏ; o◌̉; ) LATIN SMALL LETTER O WITH HOOK ABOVE +1ED0;1ED0;004F 0302 0301;1ED0;004F 0302 0301; # (Ố; Ố; O◌̂◌́; Ố; O◌̂◌́; ) LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE +1ED1;1ED1;006F 0302 0301;1ED1;006F 0302 0301; # (ố; ố; o◌̂◌́; ố; o◌̂◌́; ) LATIN SMALL LETTER O WITH CIRCUMFLEX AND ACUTE +1ED2;1ED2;004F 0302 0300;1ED2;004F 0302 0300; # (Ồ; Ồ; O◌̂◌̀; Ồ; O◌̂◌̀; ) LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE +1ED3;1ED3;006F 0302 0300;1ED3;006F 0302 0300; # (ồ; ồ; o◌̂◌̀; ồ; o◌̂◌̀; ) LATIN SMALL LETTER O WITH CIRCUMFLEX AND GRAVE +1ED4;1ED4;004F 0302 0309;1ED4;004F 0302 0309; # (Ổ; Ổ; O◌̂◌̉; Ổ; O◌̂◌̉; ) LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE +1ED5;1ED5;006F 0302 0309;1ED5;006F 0302 0309; # (ổ; ổ; o◌̂◌̉; ổ; o◌̂◌̉; ) LATIN SMALL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE +1ED6;1ED6;004F 0302 0303;1ED6;004F 0302 0303; # (Ỗ; Ỗ; O◌̂◌̃; Ỗ; O◌̂◌̃; ) LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE +1ED7;1ED7;006F 0302 0303;1ED7;006F 0302 0303; # (ỗ; ỗ; o◌̂◌̃; ỗ; o◌̂◌̃; ) LATIN SMALL LETTER O WITH CIRCUMFLEX AND TILDE +1ED8;1ED8;004F 0323 0302;1ED8;004F 0323 0302; # (Ộ; Ộ; O◌̣◌̂; Ộ; O◌̣◌̂; ) LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW +1ED9;1ED9;006F 0323 0302;1ED9;006F 0323 0302; # (ộ; ộ; o◌̣◌̂; ộ; o◌̣◌̂; ) LATIN SMALL LETTER O WITH CIRCUMFLEX AND DOT BELOW +1EDA;1EDA;004F 031B 0301;1EDA;004F 031B 0301; # (Ớ; Ớ; O◌̛◌́; Ớ; O◌̛◌́; ) LATIN CAPITAL LETTER O WITH HORN AND ACUTE +1EDB;1EDB;006F 031B 0301;1EDB;006F 031B 0301; # (ớ; ớ; o◌̛◌́; ớ; o◌̛◌́; ) LATIN SMALL LETTER O WITH HORN AND ACUTE +1EDC;1EDC;004F 031B 0300;1EDC;004F 031B 0300; # (Ờ; Ờ; O◌̛◌̀; Ờ; O◌̛◌̀; ) LATIN CAPITAL LETTER O WITH HORN AND GRAVE +1EDD;1EDD;006F 031B 0300;1EDD;006F 031B 0300; # (ờ; ờ; o◌̛◌̀; ờ; o◌̛◌̀; ) LATIN SMALL LETTER O WITH HORN AND GRAVE +1EDE;1EDE;004F 031B 0309;1EDE;004F 031B 0309; # (Ở; Ở; O◌̛◌̉; Ở; O◌̛◌̉; ) LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE +1EDF;1EDF;006F 031B 0309;1EDF;006F 031B 0309; # (ở; ở; o◌̛◌̉; ở; o◌̛◌̉; ) LATIN SMALL LETTER O WITH HORN AND HOOK ABOVE +1EE0;1EE0;004F 031B 0303;1EE0;004F 031B 0303; # (Ỡ; Ỡ; O◌̛◌̃; Ỡ; O◌̛◌̃; ) LATIN CAPITAL LETTER O WITH HORN AND TILDE +1EE1;1EE1;006F 031B 0303;1EE1;006F 031B 0303; # (ỡ; ỡ; o◌̛◌̃; ỡ; o◌̛◌̃; ) LATIN SMALL LETTER O WITH HORN AND TILDE +1EE2;1EE2;004F 031B 0323;1EE2;004F 031B 0323; # (Ợ; Ợ; O◌̛◌̣; Ợ; O◌̛◌̣; ) LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW +1EE3;1EE3;006F 031B 0323;1EE3;006F 031B 0323; # (ợ; ợ; o◌̛◌̣; ợ; o◌̛◌̣; ) LATIN SMALL LETTER O WITH HORN AND DOT BELOW +1EE4;1EE4;0055 0323;1EE4;0055 0323; # (Ụ; Ụ; U◌̣; Ụ; U◌̣; ) LATIN CAPITAL LETTER U WITH DOT BELOW +1EE5;1EE5;0075 0323;1EE5;0075 0323; # (ụ; ụ; u◌̣; ụ; u◌̣; ) LATIN SMALL LETTER U WITH DOT BELOW +1EE6;1EE6;0055 0309;1EE6;0055 0309; # (Ủ; Ủ; U◌̉; Ủ; U◌̉; ) LATIN CAPITAL LETTER U WITH HOOK ABOVE +1EE7;1EE7;0075 0309;1EE7;0075 0309; # (ủ; ủ; u◌̉; ủ; u◌̉; ) LATIN SMALL LETTER U WITH HOOK ABOVE +1EE8;1EE8;0055 031B 0301;1EE8;0055 031B 0301; # (Ứ; Ứ; U◌̛◌́; Ứ; U◌̛◌́; ) LATIN CAPITAL LETTER U WITH HORN AND ACUTE +1EE9;1EE9;0075 031B 0301;1EE9;0075 031B 0301; # (ứ; ứ; u◌̛◌́; ứ; u◌̛◌́; ) LATIN SMALL LETTER U WITH HORN AND ACUTE +1EEA;1EEA;0055 031B 0300;1EEA;0055 031B 0300; # (Ừ; Ừ; U◌̛◌̀; Ừ; U◌̛◌̀; ) LATIN CAPITAL LETTER U WITH HORN AND GRAVE +1EEB;1EEB;0075 031B 0300;1EEB;0075 031B 0300; # (ừ; ừ; u◌̛◌̀; ừ; u◌̛◌̀; ) LATIN SMALL LETTER U WITH HORN AND GRAVE +1EEC;1EEC;0055 031B 0309;1EEC;0055 031B 0309; # (Ử; Ử; U◌̛◌̉; Ử; U◌̛◌̉; ) LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE +1EED;1EED;0075 031B 0309;1EED;0075 031B 0309; # (ử; ử; u◌̛◌̉; ử; u◌̛◌̉; ) LATIN SMALL LETTER U WITH HORN AND HOOK ABOVE +1EEE;1EEE;0055 031B 0303;1EEE;0055 031B 0303; # (Ữ; Ữ; U◌̛◌̃; Ữ; U◌̛◌̃; ) LATIN CAPITAL LETTER U WITH HORN AND TILDE +1EEF;1EEF;0075 031B 0303;1EEF;0075 031B 0303; # (ữ; ữ; u◌̛◌̃; ữ; u◌̛◌̃; ) LATIN SMALL LETTER U WITH HORN AND TILDE +1EF0;1EF0;0055 031B 0323;1EF0;0055 031B 0323; # (Ự; Ự; U◌̛◌̣; Ự; U◌̛◌̣; ) LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW +1EF1;1EF1;0075 031B 0323;1EF1;0075 031B 0323; # (ự; ự; u◌̛◌̣; ự; u◌̛◌̣; ) LATIN SMALL LETTER U WITH HORN AND DOT BELOW +1EF2;1EF2;0059 0300;1EF2;0059 0300; # (Ỳ; Ỳ; Y◌̀; Ỳ; Y◌̀; ) LATIN CAPITAL LETTER Y WITH GRAVE +1EF3;1EF3;0079 0300;1EF3;0079 0300; # (ỳ; ỳ; y◌̀; ỳ; y◌̀; ) LATIN SMALL LETTER Y WITH GRAVE +1EF4;1EF4;0059 0323;1EF4;0059 0323; # (Ỵ; Ỵ; Y◌̣; Ỵ; Y◌̣; ) LATIN CAPITAL LETTER Y WITH DOT BELOW +1EF5;1EF5;0079 0323;1EF5;0079 0323; # (ỵ; ỵ; y◌̣; ỵ; y◌̣; ) LATIN SMALL LETTER Y WITH DOT BELOW +1EF6;1EF6;0059 0309;1EF6;0059 0309; # (Ỷ; Ỷ; Y◌̉; Ỷ; Y◌̉; ) LATIN CAPITAL LETTER Y WITH HOOK ABOVE +1EF7;1EF7;0079 0309;1EF7;0079 0309; # (ỷ; ỷ; y◌̉; ỷ; y◌̉; ) LATIN SMALL LETTER Y WITH HOOK ABOVE +1EF8;1EF8;0059 0303;1EF8;0059 0303; # (Ỹ; Ỹ; Y◌̃; Ỹ; Y◌̃; ) LATIN CAPITAL LETTER Y WITH TILDE +1EF9;1EF9;0079 0303;1EF9;0079 0303; # (ỹ; ỹ; y◌̃; ỹ; y◌̃; ) LATIN SMALL LETTER Y WITH TILDE +1F00;1F00;03B1 0313;1F00;03B1 0313; # (ἀ; ἀ; α◌̓; ἀ; α◌̓; ) GREEK SMALL LETTER ALPHA WITH PSILI +1F01;1F01;03B1 0314;1F01;03B1 0314; # (ἁ; ἁ; α◌̔; ἁ; α◌̔; ) GREEK SMALL LETTER ALPHA WITH DASIA +1F02;1F02;03B1 0313 0300;1F02;03B1 0313 0300; # (ἂ; ἂ; α◌̓◌̀; ἂ; α◌̓◌̀; ) GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA +1F03;1F03;03B1 0314 0300;1F03;03B1 0314 0300; # (ἃ; ἃ; α◌̔◌̀; ἃ; α◌̔◌̀; ) GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA +1F04;1F04;03B1 0313 0301;1F04;03B1 0313 0301; # (ἄ; ἄ; α◌̓◌́; ἄ; α◌̓◌́; ) GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA +1F05;1F05;03B1 0314 0301;1F05;03B1 0314 0301; # (ἅ; ἅ; α◌̔◌́; ἅ; α◌̔◌́; ) GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA +1F06;1F06;03B1 0313 0342;1F06;03B1 0313 0342; # (ἆ; ἆ; α◌̓◌͂; ἆ; α◌̓◌͂; ) GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI +1F07;1F07;03B1 0314 0342;1F07;03B1 0314 0342; # (ἇ; ἇ; α◌̔◌͂; ἇ; α◌̔◌͂; ) GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI +1F08;1F08;0391 0313;1F08;0391 0313; # (Ἀ; Ἀ; Α◌̓; Ἀ; Α◌̓; ) GREEK CAPITAL LETTER ALPHA WITH PSILI +1F09;1F09;0391 0314;1F09;0391 0314; # (Ἁ; Ἁ; Α◌̔; Ἁ; Α◌̔; ) GREEK CAPITAL LETTER ALPHA WITH DASIA +1F0A;1F0A;0391 0313 0300;1F0A;0391 0313 0300; # (Ἂ; Ἂ; Α◌̓◌̀; Ἂ; Α◌̓◌̀; ) GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA +1F0B;1F0B;0391 0314 0300;1F0B;0391 0314 0300; # (Ἃ; Ἃ; Α◌̔◌̀; Ἃ; Α◌̔◌̀; ) GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA +1F0C;1F0C;0391 0313 0301;1F0C;0391 0313 0301; # (Ἄ; Ἄ; Α◌̓◌́; Ἄ; Α◌̓◌́; ) GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA +1F0D;1F0D;0391 0314 0301;1F0D;0391 0314 0301; # (Ἅ; Ἅ; Α◌̔◌́; Ἅ; Α◌̔◌́; ) GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA +1F0E;1F0E;0391 0313 0342;1F0E;0391 0313 0342; # (Ἆ; Ἆ; Α◌̓◌͂; Ἆ; Α◌̓◌͂; ) GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI +1F0F;1F0F;0391 0314 0342;1F0F;0391 0314 0342; # (Ἇ; Ἇ; Α◌̔◌͂; Ἇ; Α◌̔◌͂; ) GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI +1F10;1F10;03B5 0313;1F10;03B5 0313; # (ἐ; ἐ; ε◌̓; ἐ; ε◌̓; ) GREEK SMALL LETTER EPSILON WITH PSILI +1F11;1F11;03B5 0314;1F11;03B5 0314; # (ἑ; ἑ; ε◌̔; ἑ; ε◌̔; ) GREEK SMALL LETTER EPSILON WITH DASIA +1F12;1F12;03B5 0313 0300;1F12;03B5 0313 0300; # (ἒ; ἒ; ε◌̓◌̀; ἒ; ε◌̓◌̀; ) GREEK SMALL LETTER EPSILON WITH PSILI AND VARIA +1F13;1F13;03B5 0314 0300;1F13;03B5 0314 0300; # (ἓ; ἓ; ε◌̔◌̀; ἓ; ε◌̔◌̀; ) GREEK SMALL LETTER EPSILON WITH DASIA AND VARIA +1F14;1F14;03B5 0313 0301;1F14;03B5 0313 0301; # (ἔ; ἔ; ε◌̓◌́; ἔ; ε◌̓◌́; ) GREEK SMALL LETTER EPSILON WITH PSILI AND OXIA +1F15;1F15;03B5 0314 0301;1F15;03B5 0314 0301; # (ἕ; ἕ; ε◌̔◌́; ἕ; ε◌̔◌́; ) GREEK SMALL LETTER EPSILON WITH DASIA AND OXIA +1F18;1F18;0395 0313;1F18;0395 0313; # (Ἐ; Ἐ; Ε◌̓; Ἐ; Ε◌̓; ) GREEK CAPITAL LETTER EPSILON WITH PSILI +1F19;1F19;0395 0314;1F19;0395 0314; # (Ἑ; Ἑ; Ε◌̔; Ἑ; Ε◌̔; ) GREEK CAPITAL LETTER EPSILON WITH DASIA +1F1A;1F1A;0395 0313 0300;1F1A;0395 0313 0300; # (Ἒ; Ἒ; Ε◌̓◌̀; Ἒ; Ε◌̓◌̀; ) GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA +1F1B;1F1B;0395 0314 0300;1F1B;0395 0314 0300; # (Ἓ; Ἓ; Ε◌̔◌̀; Ἓ; Ε◌̔◌̀; ) GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA +1F1C;1F1C;0395 0313 0301;1F1C;0395 0313 0301; # (Ἔ; Ἔ; Ε◌̓◌́; Ἔ; Ε◌̓◌́; ) GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA +1F1D;1F1D;0395 0314 0301;1F1D;0395 0314 0301; # (Ἕ; Ἕ; Ε◌̔◌́; Ἕ; Ε◌̔◌́; ) GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA +1F20;1F20;03B7 0313;1F20;03B7 0313; # (ἠ; ἠ; η◌̓; ἠ; η◌̓; ) GREEK SMALL LETTER ETA WITH PSILI +1F21;1F21;03B7 0314;1F21;03B7 0314; # (ἡ; ἡ; η◌̔; ἡ; η◌̔; ) GREEK SMALL LETTER ETA WITH DASIA +1F22;1F22;03B7 0313 0300;1F22;03B7 0313 0300; # (ἢ; ἢ; η◌̓◌̀; ἢ; η◌̓◌̀; ) GREEK SMALL LETTER ETA WITH PSILI AND VARIA +1F23;1F23;03B7 0314 0300;1F23;03B7 0314 0300; # (ἣ; ἣ; η◌̔◌̀; ἣ; η◌̔◌̀; ) GREEK SMALL LETTER ETA WITH DASIA AND VARIA +1F24;1F24;03B7 0313 0301;1F24;03B7 0313 0301; # (ἤ; ἤ; η◌̓◌́; ἤ; η◌̓◌́; ) GREEK SMALL LETTER ETA WITH PSILI AND OXIA +1F25;1F25;03B7 0314 0301;1F25;03B7 0314 0301; # (ἥ; ἥ; η◌̔◌́; ἥ; η◌̔◌́; ) GREEK SMALL LETTER ETA WITH DASIA AND OXIA +1F26;1F26;03B7 0313 0342;1F26;03B7 0313 0342; # (ἦ; ἦ; η◌̓◌͂; ἦ; η◌̓◌͂; ) GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI +1F27;1F27;03B7 0314 0342;1F27;03B7 0314 0342; # (ἧ; ἧ; η◌̔◌͂; ἧ; η◌̔◌͂; ) GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI +1F28;1F28;0397 0313;1F28;0397 0313; # (Ἠ; Ἠ; Η◌̓; Ἠ; Η◌̓; ) GREEK CAPITAL LETTER ETA WITH PSILI +1F29;1F29;0397 0314;1F29;0397 0314; # (Ἡ; Ἡ; Η◌̔; Ἡ; Η◌̔; ) GREEK CAPITAL LETTER ETA WITH DASIA +1F2A;1F2A;0397 0313 0300;1F2A;0397 0313 0300; # (Ἢ; Ἢ; Η◌̓◌̀; Ἢ; Η◌̓◌̀; ) GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA +1F2B;1F2B;0397 0314 0300;1F2B;0397 0314 0300; # (Ἣ; Ἣ; Η◌̔◌̀; Ἣ; Η◌̔◌̀; ) GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA +1F2C;1F2C;0397 0313 0301;1F2C;0397 0313 0301; # (Ἤ; Ἤ; Η◌̓◌́; Ἤ; Η◌̓◌́; ) GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA +1F2D;1F2D;0397 0314 0301;1F2D;0397 0314 0301; # (Ἥ; Ἥ; Η◌̔◌́; Ἥ; Η◌̔◌́; ) GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA +1F2E;1F2E;0397 0313 0342;1F2E;0397 0313 0342; # (Ἦ; Ἦ; Η◌̓◌͂; Ἦ; Η◌̓◌͂; ) GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI +1F2F;1F2F;0397 0314 0342;1F2F;0397 0314 0342; # (Ἧ; Ἧ; Η◌̔◌͂; Ἧ; Η◌̔◌͂; ) GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI +1F30;1F30;03B9 0313;1F30;03B9 0313; # (ἰ; ἰ; ι◌̓; ἰ; ι◌̓; ) GREEK SMALL LETTER IOTA WITH PSILI +1F31;1F31;03B9 0314;1F31;03B9 0314; # (ἱ; ἱ; ι◌̔; ἱ; ι◌̔; ) GREEK SMALL LETTER IOTA WITH DASIA +1F32;1F32;03B9 0313 0300;1F32;03B9 0313 0300; # (ἲ; ἲ; ι◌̓◌̀; ἲ; ι◌̓◌̀; ) GREEK SMALL LETTER IOTA WITH PSILI AND VARIA +1F33;1F33;03B9 0314 0300;1F33;03B9 0314 0300; # (ἳ; ἳ; ι◌̔◌̀; ἳ; ι◌̔◌̀; ) GREEK SMALL LETTER IOTA WITH DASIA AND VARIA +1F34;1F34;03B9 0313 0301;1F34;03B9 0313 0301; # (ἴ; ἴ; ι◌̓◌́; ἴ; ι◌̓◌́; ) GREEK SMALL LETTER IOTA WITH PSILI AND OXIA +1F35;1F35;03B9 0314 0301;1F35;03B9 0314 0301; # (ἵ; ἵ; ι◌̔◌́; ἵ; ι◌̔◌́; ) GREEK SMALL LETTER IOTA WITH DASIA AND OXIA +1F36;1F36;03B9 0313 0342;1F36;03B9 0313 0342; # (ἶ; ἶ; ι◌̓◌͂; ἶ; ι◌̓◌͂; ) GREEK SMALL LETTER IOTA WITH PSILI AND PERISPOMENI +1F37;1F37;03B9 0314 0342;1F37;03B9 0314 0342; # (ἷ; ἷ; ι◌̔◌͂; ἷ; ι◌̔◌͂; ) GREEK SMALL LETTER IOTA WITH DASIA AND PERISPOMENI +1F38;1F38;0399 0313;1F38;0399 0313; # (Ἰ; Ἰ; Ι◌̓; Ἰ; Ι◌̓; ) GREEK CAPITAL LETTER IOTA WITH PSILI +1F39;1F39;0399 0314;1F39;0399 0314; # (Ἱ; Ἱ; Ι◌̔; Ἱ; Ι◌̔; ) GREEK CAPITAL LETTER IOTA WITH DASIA +1F3A;1F3A;0399 0313 0300;1F3A;0399 0313 0300; # (Ἲ; Ἲ; Ι◌̓◌̀; Ἲ; Ι◌̓◌̀; ) GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA +1F3B;1F3B;0399 0314 0300;1F3B;0399 0314 0300; # (Ἳ; Ἳ; Ι◌̔◌̀; Ἳ; Ι◌̔◌̀; ) GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA +1F3C;1F3C;0399 0313 0301;1F3C;0399 0313 0301; # (Ἴ; Ἴ; Ι◌̓◌́; Ἴ; Ι◌̓◌́; ) GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA +1F3D;1F3D;0399 0314 0301;1F3D;0399 0314 0301; # (Ἵ; Ἵ; Ι◌̔◌́; Ἵ; Ι◌̔◌́; ) GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA +1F3E;1F3E;0399 0313 0342;1F3E;0399 0313 0342; # (Ἶ; Ἶ; Ι◌̓◌͂; Ἶ; Ι◌̓◌͂; ) GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI +1F3F;1F3F;0399 0314 0342;1F3F;0399 0314 0342; # (Ἷ; Ἷ; Ι◌̔◌͂; Ἷ; Ι◌̔◌͂; ) GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI +1F40;1F40;03BF 0313;1F40;03BF 0313; # (ὀ; ὀ; ο◌̓; ὀ; ο◌̓; ) GREEK SMALL LETTER OMICRON WITH PSILI +1F41;1F41;03BF 0314;1F41;03BF 0314; # (ὁ; ὁ; ο◌̔; ὁ; ο◌̔; ) GREEK SMALL LETTER OMICRON WITH DASIA +1F42;1F42;03BF 0313 0300;1F42;03BF 0313 0300; # (ὂ; ὂ; ο◌̓◌̀; ὂ; ο◌̓◌̀; ) GREEK SMALL LETTER OMICRON WITH PSILI AND VARIA +1F43;1F43;03BF 0314 0300;1F43;03BF 0314 0300; # (ὃ; ὃ; ο◌̔◌̀; ὃ; ο◌̔◌̀; ) GREEK SMALL LETTER OMICRON WITH DASIA AND VARIA +1F44;1F44;03BF 0313 0301;1F44;03BF 0313 0301; # (ὄ; ὄ; ο◌̓◌́; ὄ; ο◌̓◌́; ) GREEK SMALL LETTER OMICRON WITH PSILI AND OXIA +1F45;1F45;03BF 0314 0301;1F45;03BF 0314 0301; # (ὅ; ὅ; ο◌̔◌́; ὅ; ο◌̔◌́; ) GREEK SMALL LETTER OMICRON WITH DASIA AND OXIA +1F48;1F48;039F 0313;1F48;039F 0313; # (Ὀ; Ὀ; Ο◌̓; Ὀ; Ο◌̓; ) GREEK CAPITAL LETTER OMICRON WITH PSILI +1F49;1F49;039F 0314;1F49;039F 0314; # (Ὁ; Ὁ; Ο◌̔; Ὁ; Ο◌̔; ) GREEK CAPITAL LETTER OMICRON WITH DASIA +1F4A;1F4A;039F 0313 0300;1F4A;039F 0313 0300; # (Ὂ; Ὂ; Ο◌̓◌̀; Ὂ; Ο◌̓◌̀; ) GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA +1F4B;1F4B;039F 0314 0300;1F4B;039F 0314 0300; # (Ὃ; Ὃ; Ο◌̔◌̀; Ὃ; Ο◌̔◌̀; ) GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA +1F4C;1F4C;039F 0313 0301;1F4C;039F 0313 0301; # (Ὄ; Ὄ; Ο◌̓◌́; Ὄ; Ο◌̓◌́; ) GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA +1F4D;1F4D;039F 0314 0301;1F4D;039F 0314 0301; # (Ὅ; Ὅ; Ο◌̔◌́; Ὅ; Ο◌̔◌́; ) GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA +1F50;1F50;03C5 0313;1F50;03C5 0313; # (ὐ; ὐ; υ◌̓; ὐ; υ◌̓; ) GREEK SMALL LETTER UPSILON WITH PSILI +1F51;1F51;03C5 0314;1F51;03C5 0314; # (ὑ; ὑ; υ◌̔; ὑ; υ◌̔; ) GREEK SMALL LETTER UPSILON WITH DASIA +1F52;1F52;03C5 0313 0300;1F52;03C5 0313 0300; # (ὒ; ὒ; υ◌̓◌̀; ὒ; υ◌̓◌̀; ) GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA +1F53;1F53;03C5 0314 0300;1F53;03C5 0314 0300; # (ὓ; ὓ; υ◌̔◌̀; ὓ; υ◌̔◌̀; ) GREEK SMALL LETTER UPSILON WITH DASIA AND VARIA +1F54;1F54;03C5 0313 0301;1F54;03C5 0313 0301; # (ὔ; ὔ; υ◌̓◌́; ὔ; υ◌̓◌́; ) GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA +1F55;1F55;03C5 0314 0301;1F55;03C5 0314 0301; # (ὕ; ὕ; υ◌̔◌́; ὕ; υ◌̔◌́; ) GREEK SMALL LETTER UPSILON WITH DASIA AND OXIA +1F56;1F56;03C5 0313 0342;1F56;03C5 0313 0342; # (ὖ; ὖ; υ◌̓◌͂; ὖ; υ◌̓◌͂; ) GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI +1F57;1F57;03C5 0314 0342;1F57;03C5 0314 0342; # (ὗ; ὗ; υ◌̔◌͂; ὗ; υ◌̔◌͂; ) GREEK SMALL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F59;1F59;03A5 0314;1F59;03A5 0314; # (Ὑ; Ὑ; Υ◌̔; Ὑ; Υ◌̔; ) GREEK CAPITAL LETTER UPSILON WITH DASIA +1F5B;1F5B;03A5 0314 0300;1F5B;03A5 0314 0300; # (Ὓ; Ὓ; Υ◌̔◌̀; Ὓ; Υ◌̔◌̀; ) GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA +1F5D;1F5D;03A5 0314 0301;1F5D;03A5 0314 0301; # (Ὕ; Ὕ; Υ◌̔◌́; Ὕ; Υ◌̔◌́; ) GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA +1F5F;1F5F;03A5 0314 0342;1F5F;03A5 0314 0342; # (Ὗ; Ὗ; Υ◌̔◌͂; Ὗ; Υ◌̔◌͂; ) GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI +1F60;1F60;03C9 0313;1F60;03C9 0313; # (ὠ; ὠ; ω◌̓; ὠ; ω◌̓; ) GREEK SMALL LETTER OMEGA WITH PSILI +1F61;1F61;03C9 0314;1F61;03C9 0314; # (ὡ; ὡ; ω◌̔; ὡ; ω◌̔; ) GREEK SMALL LETTER OMEGA WITH DASIA +1F62;1F62;03C9 0313 0300;1F62;03C9 0313 0300; # (ὢ; ὢ; ω◌̓◌̀; ὢ; ω◌̓◌̀; ) GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA +1F63;1F63;03C9 0314 0300;1F63;03C9 0314 0300; # (ὣ; ὣ; ω◌̔◌̀; ὣ; ω◌̔◌̀; ) GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA +1F64;1F64;03C9 0313 0301;1F64;03C9 0313 0301; # (ὤ; ὤ; ω◌̓◌́; ὤ; ω◌̓◌́; ) GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA +1F65;1F65;03C9 0314 0301;1F65;03C9 0314 0301; # (ὥ; ὥ; ω◌̔◌́; ὥ; ω◌̔◌́; ) GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA +1F66;1F66;03C9 0313 0342;1F66;03C9 0313 0342; # (ὦ; ὦ; ω◌̓◌͂; ὦ; ω◌̓◌͂; ) GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI +1F67;1F67;03C9 0314 0342;1F67;03C9 0314 0342; # (ὧ; ὧ; ω◌̔◌͂; ὧ; ω◌̔◌͂; ) GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI +1F68;1F68;03A9 0313;1F68;03A9 0313; # (Ὠ; Ὠ; Ω◌̓; Ὠ; Ω◌̓; ) GREEK CAPITAL LETTER OMEGA WITH PSILI +1F69;1F69;03A9 0314;1F69;03A9 0314; # (Ὡ; Ὡ; Ω◌̔; Ὡ; Ω◌̔; ) GREEK CAPITAL LETTER OMEGA WITH DASIA +1F6A;1F6A;03A9 0313 0300;1F6A;03A9 0313 0300; # (Ὢ; Ὢ; Ω◌̓◌̀; Ὢ; Ω◌̓◌̀; ) GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA +1F6B;1F6B;03A9 0314 0300;1F6B;03A9 0314 0300; # (Ὣ; Ὣ; Ω◌̔◌̀; Ὣ; Ω◌̔◌̀; ) GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA +1F6C;1F6C;03A9 0313 0301;1F6C;03A9 0313 0301; # (Ὤ; Ὤ; Ω◌̓◌́; Ὤ; Ω◌̓◌́; ) GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA +1F6D;1F6D;03A9 0314 0301;1F6D;03A9 0314 0301; # (Ὥ; Ὥ; Ω◌̔◌́; Ὥ; Ω◌̔◌́; ) GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA +1F6E;1F6E;03A9 0313 0342;1F6E;03A9 0313 0342; # (Ὦ; Ὦ; Ω◌̓◌͂; Ὦ; Ω◌̓◌͂; ) GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI +1F6F;1F6F;03A9 0314 0342;1F6F;03A9 0314 0342; # (Ὧ; Ὧ; Ω◌̔◌͂; Ὧ; Ω◌̔◌͂; ) GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI +1F70;1F70;03B1 0300;1F70;03B1 0300; # (ὰ; ὰ; α◌̀; ὰ; α◌̀; ) GREEK SMALL LETTER ALPHA WITH VARIA +1F71;03AC;03B1 0301;03AC;03B1 0301; # (ά; ά; α◌́; ά; α◌́; ) GREEK SMALL LETTER ALPHA WITH OXIA +1F72;1F72;03B5 0300;1F72;03B5 0300; # (ὲ; ὲ; ε◌̀; ὲ; ε◌̀; ) GREEK SMALL LETTER EPSILON WITH VARIA +1F73;03AD;03B5 0301;03AD;03B5 0301; # (έ; έ; ε◌́; έ; ε◌́; ) GREEK SMALL LETTER EPSILON WITH OXIA +1F74;1F74;03B7 0300;1F74;03B7 0300; # (ὴ; ὴ; η◌̀; ὴ; η◌̀; ) GREEK SMALL LETTER ETA WITH VARIA +1F75;03AE;03B7 0301;03AE;03B7 0301; # (ή; ή; η◌́; ή; η◌́; ) GREEK SMALL LETTER ETA WITH OXIA +1F76;1F76;03B9 0300;1F76;03B9 0300; # (ὶ; ὶ; ι◌̀; ὶ; ι◌̀; ) GREEK SMALL LETTER IOTA WITH VARIA +1F77;03AF;03B9 0301;03AF;03B9 0301; # (ί; ί; ι◌́; ί; ι◌́; ) GREEK SMALL LETTER IOTA WITH OXIA +1F78;1F78;03BF 0300;1F78;03BF 0300; # (ὸ; ὸ; ο◌̀; ὸ; ο◌̀; ) GREEK SMALL LETTER OMICRON WITH VARIA +1F79;03CC;03BF 0301;03CC;03BF 0301; # (ό; ό; ο◌́; ό; ο◌́; ) GREEK SMALL LETTER OMICRON WITH OXIA +1F7A;1F7A;03C5 0300;1F7A;03C5 0300; # (ὺ; ὺ; υ◌̀; ὺ; υ◌̀; ) GREEK SMALL LETTER UPSILON WITH VARIA +1F7B;03CD;03C5 0301;03CD;03C5 0301; # (ύ; ύ; υ◌́; ύ; υ◌́; ) GREEK SMALL LETTER UPSILON WITH OXIA +1F7C;1F7C;03C9 0300;1F7C;03C9 0300; # (ὼ; ὼ; ω◌̀; ὼ; ω◌̀; ) GREEK SMALL LETTER OMEGA WITH VARIA +1F7D;03CE;03C9 0301;03CE;03C9 0301; # (ώ; ώ; ω◌́; ώ; ω◌́; ) GREEK SMALL LETTER OMEGA WITH OXIA +1F80;1F80;03B1 0313 0345;1F80;03B1 0313 0345; # (ᾀ; ᾀ; α◌̓◌ͅ; ᾀ; α◌̓◌ͅ; ) GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI +1F81;1F81;03B1 0314 0345;1F81;03B1 0314 0345; # (ᾁ; ᾁ; α◌̔◌ͅ; ᾁ; α◌̔◌ͅ; ) GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI +1F82;1F82;03B1 0313 0300 0345;1F82;03B1 0313 0300 0345; # (ᾂ; ᾂ; α◌̓◌̀◌ͅ; ᾂ; α◌̓◌̀◌ͅ; ) GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1F83;1F83;03B1 0314 0300 0345;1F83;03B1 0314 0300 0345; # (ᾃ; ᾃ; α◌̔◌̀◌ͅ; ᾃ; α◌̔◌̀◌ͅ; ) GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1F84;1F84;03B1 0313 0301 0345;1F84;03B1 0313 0301 0345; # (ᾄ; ᾄ; α◌̓◌́◌ͅ; ᾄ; α◌̓◌́◌ͅ; ) GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1F85;1F85;03B1 0314 0301 0345;1F85;03B1 0314 0301 0345; # (ᾅ; ᾅ; α◌̔◌́◌ͅ; ᾅ; α◌̔◌́◌ͅ; ) GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1F86;1F86;03B1 0313 0342 0345;1F86;03B1 0313 0342 0345; # (ᾆ; ᾆ; α◌̓◌͂◌ͅ; ᾆ; α◌̓◌͂◌ͅ; ) GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1F87;1F87;03B1 0314 0342 0345;1F87;03B1 0314 0342 0345; # (ᾇ; ᾇ; α◌̔◌͂◌ͅ; ᾇ; α◌̔◌͂◌ͅ; ) GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1F88;1F88;0391 0313 0345;1F88;0391 0313 0345; # (ᾈ; ᾈ; Α◌̓◌ͅ; ᾈ; Α◌̓◌ͅ; ) GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI +1F89;1F89;0391 0314 0345;1F89;0391 0314 0345; # (ᾉ; ᾉ; Α◌̔◌ͅ; ᾉ; Α◌̔◌ͅ; ) GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI +1F8A;1F8A;0391 0313 0300 0345;1F8A;0391 0313 0300 0345; # (ᾊ; ᾊ; Α◌̓◌̀◌ͅ; ᾊ; Α◌̓◌̀◌ͅ; ) GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F8B;1F8B;0391 0314 0300 0345;1F8B;0391 0314 0300 0345; # (ᾋ; ᾋ; Α◌̔◌̀◌ͅ; ᾋ; Α◌̔◌̀◌ͅ; ) GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F8C;1F8C;0391 0313 0301 0345;1F8C;0391 0313 0301 0345; # (ᾌ; ᾌ; Α◌̓◌́◌ͅ; ᾌ; Α◌̓◌́◌ͅ; ) GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F8D;1F8D;0391 0314 0301 0345;1F8D;0391 0314 0301 0345; # (ᾍ; ᾍ; Α◌̔◌́◌ͅ; ᾍ; Α◌̔◌́◌ͅ; ) GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F8E;1F8E;0391 0313 0342 0345;1F8E;0391 0313 0342 0345; # (ᾎ; ᾎ; Α◌̓◌͂◌ͅ; ᾎ; Α◌̓◌͂◌ͅ; ) GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F8F;1F8F;0391 0314 0342 0345;1F8F;0391 0314 0342 0345; # (ᾏ; ᾏ; Α◌̔◌͂◌ͅ; ᾏ; Α◌̔◌͂◌ͅ; ) GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1F90;1F90;03B7 0313 0345;1F90;03B7 0313 0345; # (ᾐ; ᾐ; η◌̓◌ͅ; ᾐ; η◌̓◌ͅ; ) GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI +1F91;1F91;03B7 0314 0345;1F91;03B7 0314 0345; # (ᾑ; ᾑ; η◌̔◌ͅ; ᾑ; η◌̔◌ͅ; ) GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI +1F92;1F92;03B7 0313 0300 0345;1F92;03B7 0313 0300 0345; # (ᾒ; ᾒ; η◌̓◌̀◌ͅ; ᾒ; η◌̓◌̀◌ͅ; ) GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1F93;1F93;03B7 0314 0300 0345;1F93;03B7 0314 0300 0345; # (ᾓ; ᾓ; η◌̔◌̀◌ͅ; ᾓ; η◌̔◌̀◌ͅ; ) GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1F94;1F94;03B7 0313 0301 0345;1F94;03B7 0313 0301 0345; # (ᾔ; ᾔ; η◌̓◌́◌ͅ; ᾔ; η◌̓◌́◌ͅ; ) GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1F95;1F95;03B7 0314 0301 0345;1F95;03B7 0314 0301 0345; # (ᾕ; ᾕ; η◌̔◌́◌ͅ; ᾕ; η◌̔◌́◌ͅ; ) GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1F96;1F96;03B7 0313 0342 0345;1F96;03B7 0313 0342 0345; # (ᾖ; ᾖ; η◌̓◌͂◌ͅ; ᾖ; η◌̓◌͂◌ͅ; ) GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1F97;1F97;03B7 0314 0342 0345;1F97;03B7 0314 0342 0345; # (ᾗ; ᾗ; η◌̔◌͂◌ͅ; ᾗ; η◌̔◌͂◌ͅ; ) GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1F98;1F98;0397 0313 0345;1F98;0397 0313 0345; # (ᾘ; ᾘ; Η◌̓◌ͅ; ᾘ; Η◌̓◌ͅ; ) GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI +1F99;1F99;0397 0314 0345;1F99;0397 0314 0345; # (ᾙ; ᾙ; Η◌̔◌ͅ; ᾙ; Η◌̔◌ͅ; ) GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI +1F9A;1F9A;0397 0313 0300 0345;1F9A;0397 0313 0300 0345; # (ᾚ; ᾚ; Η◌̓◌̀◌ͅ; ᾚ; Η◌̓◌̀◌ͅ; ) GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1F9B;1F9B;0397 0314 0300 0345;1F9B;0397 0314 0300 0345; # (ᾛ; ᾛ; Η◌̔◌̀◌ͅ; ᾛ; Η◌̔◌̀◌ͅ; ) GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1F9C;1F9C;0397 0313 0301 0345;1F9C;0397 0313 0301 0345; # (ᾜ; ᾜ; Η◌̓◌́◌ͅ; ᾜ; Η◌̓◌́◌ͅ; ) GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1F9D;1F9D;0397 0314 0301 0345;1F9D;0397 0314 0301 0345; # (ᾝ; ᾝ; Η◌̔◌́◌ͅ; ᾝ; Η◌̔◌́◌ͅ; ) GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1F9E;1F9E;0397 0313 0342 0345;1F9E;0397 0313 0342 0345; # (ᾞ; ᾞ; Η◌̓◌͂◌ͅ; ᾞ; Η◌̓◌͂◌ͅ; ) GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1F9F;1F9F;0397 0314 0342 0345;1F9F;0397 0314 0342 0345; # (ᾟ; ᾟ; Η◌̔◌͂◌ͅ; ᾟ; Η◌̔◌͂◌ͅ; ) GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FA0;1FA0;03C9 0313 0345;1FA0;03C9 0313 0345; # (ᾠ; ᾠ; ω◌̓◌ͅ; ᾠ; ω◌̓◌ͅ; ) GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI +1FA1;1FA1;03C9 0314 0345;1FA1;03C9 0314 0345; # (ᾡ; ᾡ; ω◌̔◌ͅ; ᾡ; ω◌̔◌ͅ; ) GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI +1FA2;1FA2;03C9 0313 0300 0345;1FA2;03C9 0313 0300 0345; # (ᾢ; ᾢ; ω◌̓◌̀◌ͅ; ᾢ; ω◌̓◌̀◌ͅ; ) GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI +1FA3;1FA3;03C9 0314 0300 0345;1FA3;03C9 0314 0300 0345; # (ᾣ; ᾣ; ω◌̔◌̀◌ͅ; ᾣ; ω◌̔◌̀◌ͅ; ) GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI +1FA4;1FA4;03C9 0313 0301 0345;1FA4;03C9 0313 0301 0345; # (ᾤ; ᾤ; ω◌̓◌́◌ͅ; ᾤ; ω◌̓◌́◌ͅ; ) GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI +1FA5;1FA5;03C9 0314 0301 0345;1FA5;03C9 0314 0301 0345; # (ᾥ; ᾥ; ω◌̔◌́◌ͅ; ᾥ; ω◌̔◌́◌ͅ; ) GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI +1FA6;1FA6;03C9 0313 0342 0345;1FA6;03C9 0313 0342 0345; # (ᾦ; ᾦ; ω◌̓◌͂◌ͅ; ᾦ; ω◌̓◌͂◌ͅ; ) GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI +1FA7;1FA7;03C9 0314 0342 0345;1FA7;03C9 0314 0342 0345; # (ᾧ; ᾧ; ω◌̔◌͂◌ͅ; ᾧ; ω◌̔◌͂◌ͅ; ) GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI +1FA8;1FA8;03A9 0313 0345;1FA8;03A9 0313 0345; # (ᾨ; ᾨ; Ω◌̓◌ͅ; ᾨ; Ω◌̓◌ͅ; ) GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI +1FA9;1FA9;03A9 0314 0345;1FA9;03A9 0314 0345; # (ᾩ; ᾩ; Ω◌̔◌ͅ; ᾩ; Ω◌̔◌ͅ; ) GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI +1FAA;1FAA;03A9 0313 0300 0345;1FAA;03A9 0313 0300 0345; # (ᾪ; ᾪ; Ω◌̓◌̀◌ͅ; ᾪ; Ω◌̓◌̀◌ͅ; ) GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI +1FAB;1FAB;03A9 0314 0300 0345;1FAB;03A9 0314 0300 0345; # (ᾫ; ᾫ; Ω◌̔◌̀◌ͅ; ᾫ; Ω◌̔◌̀◌ͅ; ) GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI +1FAC;1FAC;03A9 0313 0301 0345;1FAC;03A9 0313 0301 0345; # (ᾬ; ᾬ; Ω◌̓◌́◌ͅ; ᾬ; Ω◌̓◌́◌ͅ; ) GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI +1FAD;1FAD;03A9 0314 0301 0345;1FAD;03A9 0314 0301 0345; # (ᾭ; ᾭ; Ω◌̔◌́◌ͅ; ᾭ; Ω◌̔◌́◌ͅ; ) GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI +1FAE;1FAE;03A9 0313 0342 0345;1FAE;03A9 0313 0342 0345; # (ᾮ; ᾮ; Ω◌̓◌͂◌ͅ; ᾮ; Ω◌̓◌͂◌ͅ; ) GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI +1FAF;1FAF;03A9 0314 0342 0345;1FAF;03A9 0314 0342 0345; # (ᾯ; ᾯ; Ω◌̔◌͂◌ͅ; ᾯ; Ω◌̔◌͂◌ͅ; ) GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI +1FB0;1FB0;03B1 0306;1FB0;03B1 0306; # (ᾰ; ᾰ; α◌̆; ᾰ; α◌̆; ) GREEK SMALL LETTER ALPHA WITH VRACHY +1FB1;1FB1;03B1 0304;1FB1;03B1 0304; # (ᾱ; ᾱ; α◌̄; ᾱ; α◌̄; ) GREEK SMALL LETTER ALPHA WITH MACRON +1FB2;1FB2;03B1 0300 0345;1FB2;03B1 0300 0345; # (ᾲ; ᾲ; α◌̀◌ͅ; ᾲ; α◌̀◌ͅ; ) GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI +1FB3;1FB3;03B1 0345;1FB3;03B1 0345; # (ᾳ; ᾳ; α◌ͅ; ᾳ; α◌ͅ; ) GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI +1FB4;1FB4;03B1 0301 0345;1FB4;03B1 0301 0345; # (ᾴ; ᾴ; α◌́◌ͅ; ᾴ; α◌́◌ͅ; ) GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI +1FB6;1FB6;03B1 0342;1FB6;03B1 0342; # (ᾶ; ᾶ; α◌͂; ᾶ; α◌͂; ) GREEK SMALL LETTER ALPHA WITH PERISPOMENI +1FB7;1FB7;03B1 0342 0345;1FB7;03B1 0342 0345; # (ᾷ; ᾷ; α◌͂◌ͅ; ᾷ; α◌͂◌ͅ; ) GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI +1FB8;1FB8;0391 0306;1FB8;0391 0306; # (Ᾰ; Ᾰ; Α◌̆; Ᾰ; Α◌̆; ) GREEK CAPITAL LETTER ALPHA WITH VRACHY +1FB9;1FB9;0391 0304;1FB9;0391 0304; # (Ᾱ; Ᾱ; Α◌̄; Ᾱ; Α◌̄; ) GREEK CAPITAL LETTER ALPHA WITH MACRON +1FBA;1FBA;0391 0300;1FBA;0391 0300; # (Ὰ; Ὰ; Α◌̀; Ὰ; Α◌̀; ) GREEK CAPITAL LETTER ALPHA WITH VARIA +1FBB;0386;0391 0301;0386;0391 0301; # (Ά; Ά; Α◌́; Ά; Α◌́; ) GREEK CAPITAL LETTER ALPHA WITH OXIA +1FBC;1FBC;0391 0345;1FBC;0391 0345; # (ᾼ; ᾼ; Α◌ͅ; ᾼ; Α◌ͅ; ) GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI +1FBD;1FBD;1FBD;0020 0313;0020 0313; # (᾽; ᾽; ᾽; ◌̓; ◌̓; ) GREEK KORONIS +1FBE;03B9;03B9;03B9;03B9; # (ι; ι; ι; ι; ι; ) GREEK PROSGEGRAMMENI +1FBF;1FBF;1FBF;0020 0313;0020 0313; # (᾿; ᾿; ᾿; ◌̓; ◌̓; ) GREEK PSILI +1FC0;1FC0;1FC0;0020 0342;0020 0342; # (῀; ῀; ῀; ◌͂; ◌͂; ) GREEK PERISPOMENI +1FC1;1FC1;00A8 0342;0020 0308 0342;0020 0308 0342; # (῁; ῁; ¨◌͂; ◌̈◌͂; ◌̈◌͂; ) GREEK DIALYTIKA AND PERISPOMENI +1FC2;1FC2;03B7 0300 0345;1FC2;03B7 0300 0345; # (ῂ; ῂ; η◌̀◌ͅ; ῂ; η◌̀◌ͅ; ) GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI +1FC3;1FC3;03B7 0345;1FC3;03B7 0345; # (ῃ; ῃ; η◌ͅ; ῃ; η◌ͅ; ) GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI +1FC4;1FC4;03B7 0301 0345;1FC4;03B7 0301 0345; # (ῄ; ῄ; η◌́◌ͅ; ῄ; η◌́◌ͅ; ) GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI +1FC6;1FC6;03B7 0342;1FC6;03B7 0342; # (ῆ; ῆ; η◌͂; ῆ; η◌͂; ) GREEK SMALL LETTER ETA WITH PERISPOMENI +1FC7;1FC7;03B7 0342 0345;1FC7;03B7 0342 0345; # (ῇ; ῇ; η◌͂◌ͅ; ῇ; η◌͂◌ͅ; ) GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI +1FC8;1FC8;0395 0300;1FC8;0395 0300; # (Ὲ; Ὲ; Ε◌̀; Ὲ; Ε◌̀; ) GREEK CAPITAL LETTER EPSILON WITH VARIA +1FC9;0388;0395 0301;0388;0395 0301; # (Έ; Έ; Ε◌́; Έ; Ε◌́; ) GREEK CAPITAL LETTER EPSILON WITH OXIA +1FCA;1FCA;0397 0300;1FCA;0397 0300; # (Ὴ; Ὴ; Η◌̀; Ὴ; Η◌̀; ) GREEK CAPITAL LETTER ETA WITH VARIA +1FCB;0389;0397 0301;0389;0397 0301; # (Ή; Ή; Η◌́; Ή; Η◌́; ) GREEK CAPITAL LETTER ETA WITH OXIA +1FCC;1FCC;0397 0345;1FCC;0397 0345; # (ῌ; ῌ; Η◌ͅ; ῌ; Η◌ͅ; ) GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI +1FCD;1FCD;1FBF 0300;0020 0313 0300;0020 0313 0300; # (῍; ῍; ᾿◌̀; ◌̓◌̀; ◌̓◌̀; ) GREEK PSILI AND VARIA +1FCE;1FCE;1FBF 0301;0020 0313 0301;0020 0313 0301; # (῎; ῎; ᾿◌́; ◌̓◌́; ◌̓◌́; ) GREEK PSILI AND OXIA +1FCF;1FCF;1FBF 0342;0020 0313 0342;0020 0313 0342; # (῏; ῏; ᾿◌͂; ◌̓◌͂; ◌̓◌͂; ) GREEK PSILI AND PERISPOMENI +1FD0;1FD0;03B9 0306;1FD0;03B9 0306; # (ῐ; ῐ; ι◌̆; ῐ; ι◌̆; ) GREEK SMALL LETTER IOTA WITH VRACHY +1FD1;1FD1;03B9 0304;1FD1;03B9 0304; # (ῑ; ῑ; ι◌̄; ῑ; ι◌̄; ) GREEK SMALL LETTER IOTA WITH MACRON +1FD2;1FD2;03B9 0308 0300;1FD2;03B9 0308 0300; # (ῒ; ῒ; ι◌̈◌̀; ῒ; ι◌̈◌̀; ) GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA +1FD3;0390;03B9 0308 0301;0390;03B9 0308 0301; # (ΐ; ΐ; ι◌̈◌́; ΐ; ι◌̈◌́; ) GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA +1FD6;1FD6;03B9 0342;1FD6;03B9 0342; # (ῖ; ῖ; ι◌͂; ῖ; ι◌͂; ) GREEK SMALL LETTER IOTA WITH PERISPOMENI +1FD7;1FD7;03B9 0308 0342;1FD7;03B9 0308 0342; # (ῗ; ῗ; ι◌̈◌͂; ῗ; ι◌̈◌͂; ) GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI +1FD8;1FD8;0399 0306;1FD8;0399 0306; # (Ῐ; Ῐ; Ι◌̆; Ῐ; Ι◌̆; ) GREEK CAPITAL LETTER IOTA WITH VRACHY +1FD9;1FD9;0399 0304;1FD9;0399 0304; # (Ῑ; Ῑ; Ι◌̄; Ῑ; Ι◌̄; ) GREEK CAPITAL LETTER IOTA WITH MACRON +1FDA;1FDA;0399 0300;1FDA;0399 0300; # (Ὶ; Ὶ; Ι◌̀; Ὶ; Ι◌̀; ) GREEK CAPITAL LETTER IOTA WITH VARIA +1FDB;038A;0399 0301;038A;0399 0301; # (Ί; Ί; Ι◌́; Ί; Ι◌́; ) GREEK CAPITAL LETTER IOTA WITH OXIA +1FDD;1FDD;1FFE 0300;0020 0314 0300;0020 0314 0300; # (῝; ῝; ῾◌̀; ◌̔◌̀; ◌̔◌̀; ) GREEK DASIA AND VARIA +1FDE;1FDE;1FFE 0301;0020 0314 0301;0020 0314 0301; # (῞; ῞; ῾◌́; ◌̔◌́; ◌̔◌́; ) GREEK DASIA AND OXIA +1FDF;1FDF;1FFE 0342;0020 0314 0342;0020 0314 0342; # (῟; ῟; ῾◌͂; ◌̔◌͂; ◌̔◌͂; ) GREEK DASIA AND PERISPOMENI +1FE0;1FE0;03C5 0306;1FE0;03C5 0306; # (ῠ; ῠ; υ◌̆; ῠ; υ◌̆; ) GREEK SMALL LETTER UPSILON WITH VRACHY +1FE1;1FE1;03C5 0304;1FE1;03C5 0304; # (ῡ; ῡ; υ◌̄; ῡ; υ◌̄; ) GREEK SMALL LETTER UPSILON WITH MACRON +1FE2;1FE2;03C5 0308 0300;1FE2;03C5 0308 0300; # (ῢ; ῢ; υ◌̈◌̀; ῢ; υ◌̈◌̀; ) GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA +1FE3;03B0;03C5 0308 0301;03B0;03C5 0308 0301; # (ΰ; ΰ; υ◌̈◌́; ΰ; υ◌̈◌́; ) GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA +1FE4;1FE4;03C1 0313;1FE4;03C1 0313; # (ῤ; ῤ; ρ◌̓; ῤ; ρ◌̓; ) GREEK SMALL LETTER RHO WITH PSILI +1FE5;1FE5;03C1 0314;1FE5;03C1 0314; # (ῥ; ῥ; ρ◌̔; ῥ; ρ◌̔; ) GREEK SMALL LETTER RHO WITH DASIA +1FE6;1FE6;03C5 0342;1FE6;03C5 0342; # (ῦ; ῦ; υ◌͂; ῦ; υ◌͂; ) GREEK SMALL LETTER UPSILON WITH PERISPOMENI +1FE7;1FE7;03C5 0308 0342;1FE7;03C5 0308 0342; # (ῧ; ῧ; υ◌̈◌͂; ῧ; υ◌̈◌͂; ) GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI +1FE8;1FE8;03A5 0306;1FE8;03A5 0306; # (Ῠ; Ῠ; Υ◌̆; Ῠ; Υ◌̆; ) GREEK CAPITAL LETTER UPSILON WITH VRACHY +1FE9;1FE9;03A5 0304;1FE9;03A5 0304; # (Ῡ; Ῡ; Υ◌̄; Ῡ; Υ◌̄; ) GREEK CAPITAL LETTER UPSILON WITH MACRON +1FEA;1FEA;03A5 0300;1FEA;03A5 0300; # (Ὺ; Ὺ; Υ◌̀; Ὺ; Υ◌̀; ) GREEK CAPITAL LETTER UPSILON WITH VARIA +1FEB;038E;03A5 0301;038E;03A5 0301; # (Ύ; Ύ; Υ◌́; Ύ; Υ◌́; ) GREEK CAPITAL LETTER UPSILON WITH OXIA +1FEC;1FEC;03A1 0314;1FEC;03A1 0314; # (Ῥ; Ῥ; Ρ◌̔; Ῥ; Ρ◌̔; ) GREEK CAPITAL LETTER RHO WITH DASIA +1FED;1FED;00A8 0300;0020 0308 0300;0020 0308 0300; # (῭; ῭; ¨◌̀; ◌̈◌̀; ◌̈◌̀; ) GREEK DIALYTIKA AND VARIA +1FEE;0385;00A8 0301;0020 0308 0301;0020 0308 0301; # (΅; ΅; ¨◌́; ◌̈◌́; ◌̈◌́; ) GREEK DIALYTIKA AND OXIA +1FEF;0060;0060;0060;0060; # (`; `; `; `; `; ) GREEK VARIA +1FF2;1FF2;03C9 0300 0345;1FF2;03C9 0300 0345; # (ῲ; ῲ; ω◌̀◌ͅ; ῲ; ω◌̀◌ͅ; ) GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI +1FF3;1FF3;03C9 0345;1FF3;03C9 0345; # (ῳ; ῳ; ω◌ͅ; ῳ; ω◌ͅ; ) GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI +1FF4;1FF4;03C9 0301 0345;1FF4;03C9 0301 0345; # (ῴ; ῴ; ω◌́◌ͅ; ῴ; ω◌́◌ͅ; ) GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI +1FF6;1FF6;03C9 0342;1FF6;03C9 0342; # (ῶ; ῶ; ω◌͂; ῶ; ω◌͂; ) GREEK SMALL LETTER OMEGA WITH PERISPOMENI +1FF7;1FF7;03C9 0342 0345;1FF7;03C9 0342 0345; # (ῷ; ῷ; ω◌͂◌ͅ; ῷ; ω◌͂◌ͅ; ) GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI +1FF8;1FF8;039F 0300;1FF8;039F 0300; # (Ὸ; Ὸ; Ο◌̀; Ὸ; Ο◌̀; ) GREEK CAPITAL LETTER OMICRON WITH VARIA +1FF9;038C;039F 0301;038C;039F 0301; # (Ό; Ό; Ο◌́; Ό; Ο◌́; ) GREEK CAPITAL LETTER OMICRON WITH OXIA +1FFA;1FFA;03A9 0300;1FFA;03A9 0300; # (Ὼ; Ὼ; Ω◌̀; Ὼ; Ω◌̀; ) GREEK CAPITAL LETTER OMEGA WITH VARIA +1FFB;038F;03A9 0301;038F;03A9 0301; # (Ώ; Ώ; Ω◌́; Ώ; Ω◌́; ) GREEK CAPITAL LETTER OMEGA WITH OXIA +1FFC;1FFC;03A9 0345;1FFC;03A9 0345; # (ῼ; ῼ; Ω◌ͅ; ῼ; Ω◌ͅ; ) GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI +1FFD;00B4;00B4;0020 0301;0020 0301; # (´; ´; ´; ◌́; ◌́; ) GREEK OXIA +1FFE;1FFE;1FFE;0020 0314;0020 0314; # (῾; ῾; ῾; ◌̔; ◌̔; ) GREEK DASIA +2000;2002;2002;0020;0020; # ( ;  ;  ; ; ; ) EN QUAD +2001;2003;2003;0020;0020; # ( ;  ;  ; ; ; ) EM QUAD +2002;2002;2002;0020;0020; # ( ;  ;  ; ; ; ) EN SPACE +2003;2003;2003;0020;0020; # ( ;  ;  ; ; ; ) EM SPACE +2004;2004;2004;0020;0020; # ( ;  ;  ; ; ; ) THREE-PER-EM SPACE +2005;2005;2005;0020;0020; # ( ;  ;  ; ; ; ) FOUR-PER-EM SPACE +2006;2006;2006;0020;0020; # ( ;  ;  ; ; ; ) SIX-PER-EM SPACE +2007;2007;2007;0020;0020; # ( ;  ;  ; ; ; ) FIGURE SPACE +2008;2008;2008;0020;0020; # ( ;  ;  ; ; ; ) PUNCTUATION SPACE +2009;2009;2009;0020;0020; # ( ;  ;  ; ; ; ) THIN SPACE +200A;200A;200A;0020;0020; # ( ;  ;  ; ; ; ) HAIR SPACE +2011;2011;2011;2010;2010; # (‑; ‑; ‑; ‐; ‐; ) NON-BREAKING HYPHEN +2017;2017;2017;0020 0333;0020 0333; # (‗; ‗; ‗; ◌̳; ◌̳; ) DOUBLE LOW LINE +2024;2024;2024;002E;002E; # (․; ․; ․; .; .; ) ONE DOT LEADER +2025;2025;2025;002E 002E;002E 002E; # (‥; ‥; ‥; ..; ..; ) TWO DOT LEADER +2026;2026;2026;002E 002E 002E;002E 002E 002E; # (…; …; …; ...; ...; ) HORIZONTAL ELLIPSIS +202F;202F;202F;0020;0020; # ( ;  ;  ; ; ; ) NARROW NO-BREAK SPACE +2033;2033;2033;2032 2032;2032 2032; # (″; ″; ″; ′′; ′′; ) DOUBLE PRIME +2034;2034;2034;2032 2032 2032;2032 2032 2032; # (‴; ‴; ‴; ′′′; ′′′; ) TRIPLE PRIME +2036;2036;2036;2035 2035;2035 2035; # (‶; ‶; ‶; ‵‵; ‵‵; ) REVERSED DOUBLE PRIME +2037;2037;2037;2035 2035 2035;2035 2035 2035; # (‷; ‷; ‷; ‵‵‵; ‵‵‵; ) REVERSED TRIPLE PRIME +203C;203C;203C;0021 0021;0021 0021; # (‼; ‼; ‼; !!; !!; ) DOUBLE EXCLAMATION MARK +203E;203E;203E;0020 0305;0020 0305; # (‾; ‾; ‾; ◌̅; ◌̅; ) OVERLINE +2047;2047;2047;003F 003F;003F 003F; # (⁇; ⁇; ⁇; ??; ??; ) DOUBLE QUESTION MARK +2048;2048;2048;003F 0021;003F 0021; # (⁈; ⁈; ⁈; ?!; ?!; ) QUESTION EXCLAMATION MARK +2049;2049;2049;0021 003F;0021 003F; # (⁉; ⁉; ⁉; !?; !?; ) EXCLAMATION QUESTION MARK +2057;2057;2057;2032 2032 2032 2032;2032 2032 2032 2032; # (⁗; ⁗; ⁗; ′′′′; ′′′′; ) QUADRUPLE PRIME +205F;205F;205F;0020;0020; # ( ;  ;  ; ; ; ) MEDIUM MATHEMATICAL SPACE +2070;2070;2070;0030;0030; # (⁰; ⁰; ⁰; 0; 0; ) SUPERSCRIPT ZERO +2071;2071;2071;0069;0069; # (ⁱ; ⁱ; ⁱ; i; i; ) SUPERSCRIPT LATIN SMALL LETTER I +2074;2074;2074;0034;0034; # (⁴; ⁴; ⁴; 4; 4; ) SUPERSCRIPT FOUR +2075;2075;2075;0035;0035; # (⁵; ⁵; ⁵; 5; 5; ) SUPERSCRIPT FIVE +2076;2076;2076;0036;0036; # (⁶; ⁶; ⁶; 6; 6; ) SUPERSCRIPT SIX +2077;2077;2077;0037;0037; # (⁷; ⁷; ⁷; 7; 7; ) SUPERSCRIPT SEVEN +2078;2078;2078;0038;0038; # (⁸; ⁸; ⁸; 8; 8; ) SUPERSCRIPT EIGHT +2079;2079;2079;0039;0039; # (⁹; ⁹; ⁹; 9; 9; ) SUPERSCRIPT NINE +207A;207A;207A;002B;002B; # (⁺; ⁺; ⁺; +; +; ) SUPERSCRIPT PLUS SIGN +207B;207B;207B;2212;2212; # (⁻; ⁻; ⁻; −; −; ) SUPERSCRIPT MINUS +207C;207C;207C;003D;003D; # (⁼; ⁼; ⁼; =; =; ) SUPERSCRIPT EQUALS SIGN +207D;207D;207D;0028;0028; # (⁽; ⁽; ⁽; (; (; ) SUPERSCRIPT LEFT PARENTHESIS +207E;207E;207E;0029;0029; # (⁾; ⁾; ⁾; ); ); ) SUPERSCRIPT RIGHT PARENTHESIS +207F;207F;207F;006E;006E; # (ⁿ; ⁿ; ⁿ; n; n; ) SUPERSCRIPT LATIN SMALL LETTER N +2080;2080;2080;0030;0030; # (₀; ₀; ₀; 0; 0; ) SUBSCRIPT ZERO +2081;2081;2081;0031;0031; # (₁; ₁; ₁; 1; 1; ) SUBSCRIPT ONE +2082;2082;2082;0032;0032; # (₂; ₂; ₂; 2; 2; ) SUBSCRIPT TWO +2083;2083;2083;0033;0033; # (₃; ₃; ₃; 3; 3; ) SUBSCRIPT THREE +2084;2084;2084;0034;0034; # (₄; ₄; ₄; 4; 4; ) SUBSCRIPT FOUR +2085;2085;2085;0035;0035; # (₅; ₅; ₅; 5; 5; ) SUBSCRIPT FIVE +2086;2086;2086;0036;0036; # (₆; ₆; ₆; 6; 6; ) SUBSCRIPT SIX +2087;2087;2087;0037;0037; # (₇; ₇; ₇; 7; 7; ) SUBSCRIPT SEVEN +2088;2088;2088;0038;0038; # (₈; ₈; ₈; 8; 8; ) SUBSCRIPT EIGHT +2089;2089;2089;0039;0039; # (₉; ₉; ₉; 9; 9; ) SUBSCRIPT NINE +208A;208A;208A;002B;002B; # (₊; ₊; ₊; +; +; ) SUBSCRIPT PLUS SIGN +208B;208B;208B;2212;2212; # (₋; ₋; ₋; −; −; ) SUBSCRIPT MINUS +208C;208C;208C;003D;003D; # (₌; ₌; ₌; =; =; ) SUBSCRIPT EQUALS SIGN +208D;208D;208D;0028;0028; # (₍; ₍; ₍; (; (; ) SUBSCRIPT LEFT PARENTHESIS +208E;208E;208E;0029;0029; # (₎; ₎; ₎; ); ); ) SUBSCRIPT RIGHT PARENTHESIS +20A8;20A8;20A8;0052 0073;0052 0073; # (₨; ₨; ₨; Rs; Rs; ) RUPEE SIGN +2100;2100;2100;0061 002F 0063;0061 002F 0063; # (℀; ℀; ℀; a/c; a/c; ) ACCOUNT OF +2101;2101;2101;0061 002F 0073;0061 002F 0073; # (℁; ℁; ℁; a/s; a/s; ) ADDRESSED TO THE SUBJECT +2102;2102;2102;0043;0043; # (ℂ; ℂ; ℂ; C; C; ) DOUBLE-STRUCK CAPITAL C +2103;2103;2103;00B0 0043;00B0 0043; # (℃; ℃; ℃; °C; °C; ) DEGREE CELSIUS +2105;2105;2105;0063 002F 006F;0063 002F 006F; # (℅; ℅; ℅; c/o; c/o; ) CARE OF +2106;2106;2106;0063 002F 0075;0063 002F 0075; # (℆; ℆; ℆; c/u; c/u; ) CADA UNA +2107;2107;2107;0190;0190; # (ℇ; ℇ; ℇ; Ɛ; Ɛ; ) EULER CONSTANT +2109;2109;2109;00B0 0046;00B0 0046; # (℉; ℉; ℉; °F; °F; ) DEGREE FAHRENHEIT +210A;210A;210A;0067;0067; # (ℊ; ℊ; ℊ; g; g; ) SCRIPT SMALL G +210B;210B;210B;0048;0048; # (ℋ; ℋ; ℋ; H; H; ) SCRIPT CAPITAL H +210C;210C;210C;0048;0048; # (ℌ; ℌ; ℌ; H; H; ) BLACK-LETTER CAPITAL H +210D;210D;210D;0048;0048; # (ℍ; ℍ; ℍ; H; H; ) DOUBLE-STRUCK CAPITAL H +210E;210E;210E;0068;0068; # (ℎ; ℎ; ℎ; h; h; ) PLANCK CONSTANT +210F;210F;210F;0127;0127; # (ℏ; ℏ; ℏ; ħ; ħ; ) PLANCK CONSTANT OVER TWO PI +2110;2110;2110;0049;0049; # (ℐ; ℐ; ℐ; I; I; ) SCRIPT CAPITAL I +2111;2111;2111;0049;0049; # (ℑ; ℑ; ℑ; I; I; ) BLACK-LETTER CAPITAL I +2112;2112;2112;004C;004C; # (ℒ; ℒ; ℒ; L; L; ) SCRIPT CAPITAL L +2113;2113;2113;006C;006C; # (ℓ; ℓ; ℓ; l; l; ) SCRIPT SMALL L +2115;2115;2115;004E;004E; # (ℕ; ℕ; ℕ; N; N; ) DOUBLE-STRUCK CAPITAL N +2116;2116;2116;004E 006F;004E 006F; # (№; №; №; No; No; ) NUMERO SIGN +2119;2119;2119;0050;0050; # (ℙ; ℙ; ℙ; P; P; ) DOUBLE-STRUCK CAPITAL P +211A;211A;211A;0051;0051; # (ℚ; ℚ; ℚ; Q; Q; ) DOUBLE-STRUCK CAPITAL Q +211B;211B;211B;0052;0052; # (ℛ; ℛ; ℛ; R; R; ) SCRIPT CAPITAL R +211C;211C;211C;0052;0052; # (ℜ; ℜ; ℜ; R; R; ) BLACK-LETTER CAPITAL R +211D;211D;211D;0052;0052; # (ℝ; ℝ; ℝ; R; R; ) DOUBLE-STRUCK CAPITAL R +2120;2120;2120;0053 004D;0053 004D; # (℠; ℠; ℠; SM; SM; ) SERVICE MARK +2121;2121;2121;0054 0045 004C;0054 0045 004C; # (℡; ℡; ℡; TEL; TEL; ) TELEPHONE SIGN +2122;2122;2122;0054 004D;0054 004D; # (™; ™; ™; TM; TM; ) TRADE MARK SIGN +2124;2124;2124;005A;005A; # (ℤ; ℤ; ℤ; Z; Z; ) DOUBLE-STRUCK CAPITAL Z +2126;03A9;03A9;03A9;03A9; # (Ω; Ω; Ω; Ω; Ω; ) OHM SIGN +2128;2128;2128;005A;005A; # (ℨ; ℨ; ℨ; Z; Z; ) BLACK-LETTER CAPITAL Z +212A;004B;004B;004B;004B; # (K; K; K; K; K; ) KELVIN SIGN +212B;00C5;0041 030A;00C5;0041 030A; # (Å; Å; A◌̊; Å; A◌̊; ) ANGSTROM SIGN +212C;212C;212C;0042;0042; # (ℬ; ℬ; ℬ; B; B; ) SCRIPT CAPITAL B +212D;212D;212D;0043;0043; # (ℭ; ℭ; ℭ; C; C; ) BLACK-LETTER CAPITAL C +212F;212F;212F;0065;0065; # (ℯ; ℯ; ℯ; e; e; ) SCRIPT SMALL E +2130;2130;2130;0045;0045; # (ℰ; ℰ; ℰ; E; E; ) SCRIPT CAPITAL E +2131;2131;2131;0046;0046; # (ℱ; ℱ; ℱ; F; F; ) SCRIPT CAPITAL F +2133;2133;2133;004D;004D; # (ℳ; ℳ; ℳ; M; M; ) SCRIPT CAPITAL M +2134;2134;2134;006F;006F; # (ℴ; ℴ; ℴ; o; o; ) SCRIPT SMALL O +2135;2135;2135;05D0;05D0; # (ℵ; ℵ; ℵ; א; א; ) ALEF SYMBOL +2136;2136;2136;05D1;05D1; # (ℶ; ℶ; ℶ; ב; ב; ) BET SYMBOL +2137;2137;2137;05D2;05D2; # (ℷ; ℷ; ℷ; ג; ג; ) GIMEL SYMBOL +2138;2138;2138;05D3;05D3; # (ℸ; ℸ; ℸ; ד; ד; ) DALET SYMBOL +2139;2139;2139;0069;0069; # (ℹ; ℹ; ℹ; i; i; ) INFORMATION SOURCE +213D;213D;213D;03B3;03B3; # (ℽ; ℽ; ℽ; γ; γ; ) DOUBLE-STRUCK SMALL GAMMA +213E;213E;213E;0393;0393; # (ℾ; ℾ; ℾ; Γ; Γ; ) DOUBLE-STRUCK CAPITAL GAMMA +213F;213F;213F;03A0;03A0; # (ℿ; ℿ; ℿ; Π; Π; ) DOUBLE-STRUCK CAPITAL PI +2140;2140;2140;2211;2211; # (⅀; ⅀; ⅀; ∑; ∑; ) DOUBLE-STRUCK N-ARY SUMMATION +2145;2145;2145;0044;0044; # (ⅅ; ⅅ; ⅅ; D; D; ) DOUBLE-STRUCK ITALIC CAPITAL D +2146;2146;2146;0064;0064; # (ⅆ; ⅆ; ⅆ; d; d; ) DOUBLE-STRUCK ITALIC SMALL D +2147;2147;2147;0065;0065; # (ⅇ; ⅇ; ⅇ; e; e; ) DOUBLE-STRUCK ITALIC SMALL E +2148;2148;2148;0069;0069; # (ⅈ; ⅈ; ⅈ; i; i; ) DOUBLE-STRUCK ITALIC SMALL I +2149;2149;2149;006A;006A; # (ⅉ; ⅉ; ⅉ; j; j; ) DOUBLE-STRUCK ITALIC SMALL J +2153;2153;2153;0031 2044 0033;0031 2044 0033; # (⅓; ⅓; ⅓; 1⁄3; 1⁄3; ) VULGAR FRACTION ONE THIRD +2154;2154;2154;0032 2044 0033;0032 2044 0033; # (⅔; ⅔; ⅔; 2⁄3; 2⁄3; ) VULGAR FRACTION TWO THIRDS +2155;2155;2155;0031 2044 0035;0031 2044 0035; # (⅕; ⅕; ⅕; 1⁄5; 1⁄5; ) VULGAR FRACTION ONE FIFTH +2156;2156;2156;0032 2044 0035;0032 2044 0035; # (⅖; ⅖; ⅖; 2⁄5; 2⁄5; ) VULGAR FRACTION TWO FIFTHS +2157;2157;2157;0033 2044 0035;0033 2044 0035; # (⅗; ⅗; ⅗; 3⁄5; 3⁄5; ) VULGAR FRACTION THREE FIFTHS +2158;2158;2158;0034 2044 0035;0034 2044 0035; # (⅘; ⅘; ⅘; 4⁄5; 4⁄5; ) VULGAR FRACTION FOUR FIFTHS +2159;2159;2159;0031 2044 0036;0031 2044 0036; # (⅙; ⅙; ⅙; 1⁄6; 1⁄6; ) VULGAR FRACTION ONE SIXTH +215A;215A;215A;0035 2044 0036;0035 2044 0036; # (⅚; ⅚; ⅚; 5⁄6; 5⁄6; ) VULGAR FRACTION FIVE SIXTHS +215B;215B;215B;0031 2044 0038;0031 2044 0038; # (⅛; ⅛; ⅛; 1⁄8; 1⁄8; ) VULGAR FRACTION ONE EIGHTH +215C;215C;215C;0033 2044 0038;0033 2044 0038; # (⅜; ⅜; ⅜; 3⁄8; 3⁄8; ) VULGAR FRACTION THREE EIGHTHS +215D;215D;215D;0035 2044 0038;0035 2044 0038; # (⅝; ⅝; ⅝; 5⁄8; 5⁄8; ) VULGAR FRACTION FIVE EIGHTHS +215E;215E;215E;0037 2044 0038;0037 2044 0038; # (⅞; ⅞; ⅞; 7⁄8; 7⁄8; ) VULGAR FRACTION SEVEN EIGHTHS +215F;215F;215F;0031 2044;0031 2044; # (⅟; ⅟; ⅟; 1⁄; 1⁄; ) FRACTION NUMERATOR ONE +2160;2160;2160;0049;0049; # (Ⅰ; Ⅰ; Ⅰ; I; I; ) ROMAN NUMERAL ONE +2161;2161;2161;0049 0049;0049 0049; # (Ⅱ; Ⅱ; Ⅱ; II; II; ) ROMAN NUMERAL TWO +2162;2162;2162;0049 0049 0049;0049 0049 0049; # (Ⅲ; Ⅲ; Ⅲ; III; III; ) ROMAN NUMERAL THREE +2163;2163;2163;0049 0056;0049 0056; # (Ⅳ; Ⅳ; Ⅳ; IV; IV; ) ROMAN NUMERAL FOUR +2164;2164;2164;0056;0056; # (Ⅴ; Ⅴ; Ⅴ; V; V; ) ROMAN NUMERAL FIVE +2165;2165;2165;0056 0049;0056 0049; # (Ⅵ; Ⅵ; Ⅵ; VI; VI; ) ROMAN NUMERAL SIX +2166;2166;2166;0056 0049 0049;0056 0049 0049; # (Ⅶ; Ⅶ; Ⅶ; VII; VII; ) ROMAN NUMERAL SEVEN +2167;2167;2167;0056 0049 0049 0049;0056 0049 0049 0049; # (Ⅷ; Ⅷ; Ⅷ; VIII; VIII; ) ROMAN NUMERAL EIGHT +2168;2168;2168;0049 0058;0049 0058; # (Ⅸ; Ⅸ; Ⅸ; IX; IX; ) ROMAN NUMERAL NINE +2169;2169;2169;0058;0058; # (Ⅹ; Ⅹ; Ⅹ; X; X; ) ROMAN NUMERAL TEN +216A;216A;216A;0058 0049;0058 0049; # (Ⅺ; Ⅺ; Ⅺ; XI; XI; ) ROMAN NUMERAL ELEVEN +216B;216B;216B;0058 0049 0049;0058 0049 0049; # (Ⅻ; Ⅻ; Ⅻ; XII; XII; ) ROMAN NUMERAL TWELVE +216C;216C;216C;004C;004C; # (Ⅼ; Ⅼ; Ⅼ; L; L; ) ROMAN NUMERAL FIFTY +216D;216D;216D;0043;0043; # (Ⅽ; Ⅽ; Ⅽ; C; C; ) ROMAN NUMERAL ONE HUNDRED +216E;216E;216E;0044;0044; # (Ⅾ; Ⅾ; Ⅾ; D; D; ) ROMAN NUMERAL FIVE HUNDRED +216F;216F;216F;004D;004D; # (Ⅿ; Ⅿ; Ⅿ; M; M; ) ROMAN NUMERAL ONE THOUSAND +2170;2170;2170;0069;0069; # (ⅰ; ⅰ; ⅰ; i; i; ) SMALL ROMAN NUMERAL ONE +2171;2171;2171;0069 0069;0069 0069; # (ⅱ; ⅱ; ⅱ; ii; ii; ) SMALL ROMAN NUMERAL TWO +2172;2172;2172;0069 0069 0069;0069 0069 0069; # (ⅲ; ⅲ; ⅲ; iii; iii; ) SMALL ROMAN NUMERAL THREE +2173;2173;2173;0069 0076;0069 0076; # (ⅳ; ⅳ; ⅳ; iv; iv; ) SMALL ROMAN NUMERAL FOUR +2174;2174;2174;0076;0076; # (ⅴ; ⅴ; ⅴ; v; v; ) SMALL ROMAN NUMERAL FIVE +2175;2175;2175;0076 0069;0076 0069; # (ⅵ; ⅵ; ⅵ; vi; vi; ) SMALL ROMAN NUMERAL SIX +2176;2176;2176;0076 0069 0069;0076 0069 0069; # (ⅶ; ⅶ; ⅶ; vii; vii; ) SMALL ROMAN NUMERAL SEVEN +2177;2177;2177;0076 0069 0069 0069;0076 0069 0069 0069; # (ⅷ; ⅷ; ⅷ; viii; viii; ) SMALL ROMAN NUMERAL EIGHT +2178;2178;2178;0069 0078;0069 0078; # (ⅸ; ⅸ; ⅸ; ix; ix; ) SMALL ROMAN NUMERAL NINE +2179;2179;2179;0078;0078; # (ⅹ; ⅹ; ⅹ; x; x; ) SMALL ROMAN NUMERAL TEN +217A;217A;217A;0078 0069;0078 0069; # (ⅺ; ⅺ; ⅺ; xi; xi; ) SMALL ROMAN NUMERAL ELEVEN +217B;217B;217B;0078 0069 0069;0078 0069 0069; # (ⅻ; ⅻ; ⅻ; xii; xii; ) SMALL ROMAN NUMERAL TWELVE +217C;217C;217C;006C;006C; # (ⅼ; ⅼ; ⅼ; l; l; ) SMALL ROMAN NUMERAL FIFTY +217D;217D;217D;0063;0063; # (ⅽ; ⅽ; ⅽ; c; c; ) SMALL ROMAN NUMERAL ONE HUNDRED +217E;217E;217E;0064;0064; # (ⅾ; ⅾ; ⅾ; d; d; ) SMALL ROMAN NUMERAL FIVE HUNDRED +217F;217F;217F;006D;006D; # (ⅿ; ⅿ; ⅿ; m; m; ) SMALL ROMAN NUMERAL ONE THOUSAND +219A;219A;2190 0338;219A;2190 0338; # (↚; ↚; ←◌̸; ↚; ←◌̸; ) LEFTWARDS ARROW WITH STROKE +219B;219B;2192 0338;219B;2192 0338; # (↛; ↛; →◌̸; ↛; →◌̸; ) RIGHTWARDS ARROW WITH STROKE +21AE;21AE;2194 0338;21AE;2194 0338; # (↮; ↮; ↔◌̸; ↮; ↔◌̸; ) LEFT RIGHT ARROW WITH STROKE +21CD;21CD;21D0 0338;21CD;21D0 0338; # (⇍; ⇍; ⇐◌̸; ⇍; ⇐◌̸; ) LEFTWARDS DOUBLE ARROW WITH STROKE +21CE;21CE;21D4 0338;21CE;21D4 0338; # (⇎; ⇎; ⇔◌̸; ⇎; ⇔◌̸; ) LEFT RIGHT DOUBLE ARROW WITH STROKE +21CF;21CF;21D2 0338;21CF;21D2 0338; # (⇏; ⇏; ⇒◌̸; ⇏; ⇒◌̸; ) RIGHTWARDS DOUBLE ARROW WITH STROKE +2204;2204;2203 0338;2204;2203 0338; # (∄; ∄; ∃◌̸; ∄; ∃◌̸; ) THERE DOES NOT EXIST +2209;2209;2208 0338;2209;2208 0338; # (∉; ∉; ∈◌̸; ∉; ∈◌̸; ) NOT AN ELEMENT OF +220C;220C;220B 0338;220C;220B 0338; # (∌; ∌; ∋◌̸; ∌; ∋◌̸; ) DOES NOT CONTAIN AS MEMBER +2224;2224;2223 0338;2224;2223 0338; # (∤; ∤; ∣◌̸; ∤; ∣◌̸; ) DOES NOT DIVIDE +2226;2226;2225 0338;2226;2225 0338; # (∦; ∦; ∥◌̸; ∦; ∥◌̸; ) NOT PARALLEL TO +222C;222C;222C;222B 222B;222B 222B; # (∬; ∬; ∬; ∫∫; ∫∫; ) DOUBLE INTEGRAL +222D;222D;222D;222B 222B 222B;222B 222B 222B; # (∭; ∭; ∭; ∫∫∫; ∫∫∫; ) TRIPLE INTEGRAL +222F;222F;222F;222E 222E;222E 222E; # (∯; ∯; ∯; ∮∮; ∮∮; ) SURFACE INTEGRAL +2230;2230;2230;222E 222E 222E;222E 222E 222E; # (∰; ∰; ∰; ∮∮∮; ∮∮∮; ) VOLUME INTEGRAL +2241;2241;223C 0338;2241;223C 0338; # (≁; ≁; ∼◌̸; ≁; ∼◌̸; ) NOT TILDE +2244;2244;2243 0338;2244;2243 0338; # (≄; ≄; ≃◌̸; ≄; ≃◌̸; ) NOT ASYMPTOTICALLY EQUAL TO +2247;2247;2245 0338;2247;2245 0338; # (≇; ≇; ≅◌̸; ≇; ≅◌̸; ) NEITHER APPROXIMATELY NOR ACTUALLY EQUAL TO +2249;2249;2248 0338;2249;2248 0338; # (≉; ≉; ≈◌̸; ≉; ≈◌̸; ) NOT ALMOST EQUAL TO +2260;2260;003D 0338;2260;003D 0338; # (≠; ≠; =◌̸; ≠; =◌̸; ) NOT EQUAL TO +2262;2262;2261 0338;2262;2261 0338; # (≢; ≢; ≡◌̸; ≢; ≡◌̸; ) NOT IDENTICAL TO +226D;226D;224D 0338;226D;224D 0338; # (≭; ≭; ≍◌̸; ≭; ≍◌̸; ) NOT EQUIVALENT TO +226E;226E;003C 0338;226E;003C 0338; # (≮; ≮; <◌̸; ≮; <◌̸; ) NOT LESS-THAN +226F;226F;003E 0338;226F;003E 0338; # (≯; ≯; >◌̸; ≯; >◌̸; ) NOT GREATER-THAN +2270;2270;2264 0338;2270;2264 0338; # (≰; ≰; ≤◌̸; ≰; ≤◌̸; ) NEITHER LESS-THAN NOR EQUAL TO +2271;2271;2265 0338;2271;2265 0338; # (≱; ≱; ≥◌̸; ≱; ≥◌̸; ) NEITHER GREATER-THAN NOR EQUAL TO +2274;2274;2272 0338;2274;2272 0338; # (≴; ≴; ≲◌̸; ≴; ≲◌̸; ) NEITHER LESS-THAN NOR EQUIVALENT TO +2275;2275;2273 0338;2275;2273 0338; # (≵; ≵; ≳◌̸; ≵; ≳◌̸; ) NEITHER GREATER-THAN NOR EQUIVALENT TO +2278;2278;2276 0338;2278;2276 0338; # (≸; ≸; ≶◌̸; ≸; ≶◌̸; ) NEITHER LESS-THAN NOR GREATER-THAN +2279;2279;2277 0338;2279;2277 0338; # (≹; ≹; ≷◌̸; ≹; ≷◌̸; ) NEITHER GREATER-THAN NOR LESS-THAN +2280;2280;227A 0338;2280;227A 0338; # (⊀; ⊀; ≺◌̸; ⊀; ≺◌̸; ) DOES NOT PRECEDE +2281;2281;227B 0338;2281;227B 0338; # (⊁; ⊁; ≻◌̸; ⊁; ≻◌̸; ) DOES NOT SUCCEED +2284;2284;2282 0338;2284;2282 0338; # (⊄; ⊄; ⊂◌̸; ⊄; ⊂◌̸; ) NOT A SUBSET OF +2285;2285;2283 0338;2285;2283 0338; # (⊅; ⊅; ⊃◌̸; ⊅; ⊃◌̸; ) NOT A SUPERSET OF +2288;2288;2286 0338;2288;2286 0338; # (⊈; ⊈; ⊆◌̸; ⊈; ⊆◌̸; ) NEITHER A SUBSET OF NOR EQUAL TO +2289;2289;2287 0338;2289;2287 0338; # (⊉; ⊉; ⊇◌̸; ⊉; ⊇◌̸; ) NEITHER A SUPERSET OF NOR EQUAL TO +22AC;22AC;22A2 0338;22AC;22A2 0338; # (⊬; ⊬; ⊢◌̸; ⊬; ⊢◌̸; ) DOES NOT PROVE +22AD;22AD;22A8 0338;22AD;22A8 0338; # (⊭; ⊭; ⊨◌̸; ⊭; ⊨◌̸; ) NOT TRUE +22AE;22AE;22A9 0338;22AE;22A9 0338; # (⊮; ⊮; ⊩◌̸; ⊮; ⊩◌̸; ) DOES NOT FORCE +22AF;22AF;22AB 0338;22AF;22AB 0338; # (⊯; ⊯; ⊫◌̸; ⊯; ⊫◌̸; ) NEGATED DOUBLE VERTICAL BAR DOUBLE RIGHT TURNSTILE +22E0;22E0;227C 0338;22E0;227C 0338; # (⋠; ⋠; ≼◌̸; ⋠; ≼◌̸; ) DOES NOT PRECEDE OR EQUAL +22E1;22E1;227D 0338;22E1;227D 0338; # (⋡; ⋡; ≽◌̸; ⋡; ≽◌̸; ) DOES NOT SUCCEED OR EQUAL +22E2;22E2;2291 0338;22E2;2291 0338; # (⋢; ⋢; ⊑◌̸; ⋢; ⊑◌̸; ) NOT SQUARE IMAGE OF OR EQUAL TO +22E3;22E3;2292 0338;22E3;2292 0338; # (⋣; ⋣; ⊒◌̸; ⋣; ⊒◌̸; ) NOT SQUARE ORIGINAL OF OR EQUAL TO +22EA;22EA;22B2 0338;22EA;22B2 0338; # (⋪; ⋪; ⊲◌̸; ⋪; ⊲◌̸; ) NOT NORMAL SUBGROUP OF +22EB;22EB;22B3 0338;22EB;22B3 0338; # (⋫; ⋫; ⊳◌̸; ⋫; ⊳◌̸; ) DOES NOT CONTAIN AS NORMAL SUBGROUP +22EC;22EC;22B4 0338;22EC;22B4 0338; # (⋬; ⋬; ⊴◌̸; ⋬; ⊴◌̸; ) NOT NORMAL SUBGROUP OF OR EQUAL TO +22ED;22ED;22B5 0338;22ED;22B5 0338; # (⋭; ⋭; ⊵◌̸; ⋭; ⊵◌̸; ) DOES NOT CONTAIN AS NORMAL SUBGROUP OR EQUAL +2329;3008;3008;3008;3008; # (〈; 〈; 〈; 〈; 〈; ) LEFT-POINTING ANGLE BRACKET +232A;3009;3009;3009;3009; # (〉; 〉; 〉; 〉; 〉; ) RIGHT-POINTING ANGLE BRACKET +2460;2460;2460;0031;0031; # (①; ①; ①; 1; 1; ) CIRCLED DIGIT ONE +2461;2461;2461;0032;0032; # (②; ②; ②; 2; 2; ) CIRCLED DIGIT TWO +2462;2462;2462;0033;0033; # (③; ③; ③; 3; 3; ) CIRCLED DIGIT THREE +2463;2463;2463;0034;0034; # (④; ④; ④; 4; 4; ) CIRCLED DIGIT FOUR +2464;2464;2464;0035;0035; # (⑤; ⑤; ⑤; 5; 5; ) CIRCLED DIGIT FIVE +2465;2465;2465;0036;0036; # (⑥; ⑥; ⑥; 6; 6; ) CIRCLED DIGIT SIX +2466;2466;2466;0037;0037; # (⑦; ⑦; ⑦; 7; 7; ) CIRCLED DIGIT SEVEN +2467;2467;2467;0038;0038; # (⑧; ⑧; ⑧; 8; 8; ) CIRCLED DIGIT EIGHT +2468;2468;2468;0039;0039; # (⑨; ⑨; ⑨; 9; 9; ) CIRCLED DIGIT NINE +2469;2469;2469;0031 0030;0031 0030; # (⑩; ⑩; ⑩; 10; 10; ) CIRCLED NUMBER TEN +246A;246A;246A;0031 0031;0031 0031; # (⑪; ⑪; ⑪; 11; 11; ) CIRCLED NUMBER ELEVEN +246B;246B;246B;0031 0032;0031 0032; # (⑫; ⑫; ⑫; 12; 12; ) CIRCLED NUMBER TWELVE +246C;246C;246C;0031 0033;0031 0033; # (⑬; ⑬; ⑬; 13; 13; ) CIRCLED NUMBER THIRTEEN +246D;246D;246D;0031 0034;0031 0034; # (⑭; ⑭; ⑭; 14; 14; ) CIRCLED NUMBER FOURTEEN +246E;246E;246E;0031 0035;0031 0035; # (⑮; ⑮; ⑮; 15; 15; ) CIRCLED NUMBER FIFTEEN +246F;246F;246F;0031 0036;0031 0036; # (⑯; ⑯; ⑯; 16; 16; ) CIRCLED NUMBER SIXTEEN +2470;2470;2470;0031 0037;0031 0037; # (⑰; ⑰; ⑰; 17; 17; ) CIRCLED NUMBER SEVENTEEN +2471;2471;2471;0031 0038;0031 0038; # (⑱; ⑱; ⑱; 18; 18; ) CIRCLED NUMBER EIGHTEEN +2472;2472;2472;0031 0039;0031 0039; # (⑲; ⑲; ⑲; 19; 19; ) CIRCLED NUMBER NINETEEN +2473;2473;2473;0032 0030;0032 0030; # (⑳; ⑳; ⑳; 20; 20; ) CIRCLED NUMBER TWENTY +2474;2474;2474;0028 0031 0029;0028 0031 0029; # (⑴; ⑴; ⑴; (1); (1); ) PARENTHESIZED DIGIT ONE +2475;2475;2475;0028 0032 0029;0028 0032 0029; # (⑵; ⑵; ⑵; (2); (2); ) PARENTHESIZED DIGIT TWO +2476;2476;2476;0028 0033 0029;0028 0033 0029; # (⑶; ⑶; ⑶; (3); (3); ) PARENTHESIZED DIGIT THREE +2477;2477;2477;0028 0034 0029;0028 0034 0029; # (⑷; ⑷; ⑷; (4); (4); ) PARENTHESIZED DIGIT FOUR +2478;2478;2478;0028 0035 0029;0028 0035 0029; # (⑸; ⑸; ⑸; (5); (5); ) PARENTHESIZED DIGIT FIVE +2479;2479;2479;0028 0036 0029;0028 0036 0029; # (⑹; ⑹; ⑹; (6); (6); ) PARENTHESIZED DIGIT SIX +247A;247A;247A;0028 0037 0029;0028 0037 0029; # (⑺; ⑺; ⑺; (7); (7); ) PARENTHESIZED DIGIT SEVEN +247B;247B;247B;0028 0038 0029;0028 0038 0029; # (⑻; ⑻; ⑻; (8); (8); ) PARENTHESIZED DIGIT EIGHT +247C;247C;247C;0028 0039 0029;0028 0039 0029; # (⑼; ⑼; ⑼; (9); (9); ) PARENTHESIZED DIGIT NINE +247D;247D;247D;0028 0031 0030 0029;0028 0031 0030 0029; # (⑽; ⑽; ⑽; (10); (10); ) PARENTHESIZED NUMBER TEN +247E;247E;247E;0028 0031 0031 0029;0028 0031 0031 0029; # (⑾; ⑾; ⑾; (11); (11); ) PARENTHESIZED NUMBER ELEVEN +247F;247F;247F;0028 0031 0032 0029;0028 0031 0032 0029; # (⑿; ⑿; ⑿; (12); (12); ) PARENTHESIZED NUMBER TWELVE +2480;2480;2480;0028 0031 0033 0029;0028 0031 0033 0029; # (⒀; ⒀; ⒀; (13); (13); ) PARENTHESIZED NUMBER THIRTEEN +2481;2481;2481;0028 0031 0034 0029;0028 0031 0034 0029; # (⒁; ⒁; ⒁; (14); (14); ) PARENTHESIZED NUMBER FOURTEEN +2482;2482;2482;0028 0031 0035 0029;0028 0031 0035 0029; # (⒂; ⒂; ⒂; (15); (15); ) PARENTHESIZED NUMBER FIFTEEN +2483;2483;2483;0028 0031 0036 0029;0028 0031 0036 0029; # (⒃; ⒃; ⒃; (16); (16); ) PARENTHESIZED NUMBER SIXTEEN +2484;2484;2484;0028 0031 0037 0029;0028 0031 0037 0029; # (⒄; ⒄; ⒄; (17); (17); ) PARENTHESIZED NUMBER SEVENTEEN +2485;2485;2485;0028 0031 0038 0029;0028 0031 0038 0029; # (⒅; ⒅; ⒅; (18); (18); ) PARENTHESIZED NUMBER EIGHTEEN +2486;2486;2486;0028 0031 0039 0029;0028 0031 0039 0029; # (⒆; ⒆; ⒆; (19); (19); ) PARENTHESIZED NUMBER NINETEEN +2487;2487;2487;0028 0032 0030 0029;0028 0032 0030 0029; # (⒇; ⒇; ⒇; (20); (20); ) PARENTHESIZED NUMBER TWENTY +2488;2488;2488;0031 002E;0031 002E; # (⒈; ⒈; ⒈; 1.; 1.; ) DIGIT ONE FULL STOP +2489;2489;2489;0032 002E;0032 002E; # (⒉; ⒉; ⒉; 2.; 2.; ) DIGIT TWO FULL STOP +248A;248A;248A;0033 002E;0033 002E; # (⒊; ⒊; ⒊; 3.; 3.; ) DIGIT THREE FULL STOP +248B;248B;248B;0034 002E;0034 002E; # (⒋; ⒋; ⒋; 4.; 4.; ) DIGIT FOUR FULL STOP +248C;248C;248C;0035 002E;0035 002E; # (⒌; ⒌; ⒌; 5.; 5.; ) DIGIT FIVE FULL STOP +248D;248D;248D;0036 002E;0036 002E; # (⒍; ⒍; ⒍; 6.; 6.; ) DIGIT SIX FULL STOP +248E;248E;248E;0037 002E;0037 002E; # (⒎; ⒎; ⒎; 7.; 7.; ) DIGIT SEVEN FULL STOP +248F;248F;248F;0038 002E;0038 002E; # (⒏; ⒏; ⒏; 8.; 8.; ) DIGIT EIGHT FULL STOP +2490;2490;2490;0039 002E;0039 002E; # (⒐; ⒐; ⒐; 9.; 9.; ) DIGIT NINE FULL STOP +2491;2491;2491;0031 0030 002E;0031 0030 002E; # (⒑; ⒑; ⒑; 10.; 10.; ) NUMBER TEN FULL STOP +2492;2492;2492;0031 0031 002E;0031 0031 002E; # (⒒; ⒒; ⒒; 11.; 11.; ) NUMBER ELEVEN FULL STOP +2493;2493;2493;0031 0032 002E;0031 0032 002E; # (⒓; ⒓; ⒓; 12.; 12.; ) NUMBER TWELVE FULL STOP +2494;2494;2494;0031 0033 002E;0031 0033 002E; # (⒔; ⒔; ⒔; 13.; 13.; ) NUMBER THIRTEEN FULL STOP +2495;2495;2495;0031 0034 002E;0031 0034 002E; # (⒕; ⒕; ⒕; 14.; 14.; ) NUMBER FOURTEEN FULL STOP +2496;2496;2496;0031 0035 002E;0031 0035 002E; # (⒖; ⒖; ⒖; 15.; 15.; ) NUMBER FIFTEEN FULL STOP +2497;2497;2497;0031 0036 002E;0031 0036 002E; # (⒗; ⒗; ⒗; 16.; 16.; ) NUMBER SIXTEEN FULL STOP +2498;2498;2498;0031 0037 002E;0031 0037 002E; # (⒘; ⒘; ⒘; 17.; 17.; ) NUMBER SEVENTEEN FULL STOP +2499;2499;2499;0031 0038 002E;0031 0038 002E; # (⒙; ⒙; ⒙; 18.; 18.; ) NUMBER EIGHTEEN FULL STOP +249A;249A;249A;0031 0039 002E;0031 0039 002E; # (⒚; ⒚; ⒚; 19.; 19.; ) NUMBER NINETEEN FULL STOP +249B;249B;249B;0032 0030 002E;0032 0030 002E; # (⒛; ⒛; ⒛; 20.; 20.; ) NUMBER TWENTY FULL STOP +249C;249C;249C;0028 0061 0029;0028 0061 0029; # (⒜; ⒜; ⒜; (a); (a); ) PARENTHESIZED LATIN SMALL LETTER A +249D;249D;249D;0028 0062 0029;0028 0062 0029; # (⒝; ⒝; ⒝; (b); (b); ) PARENTHESIZED LATIN SMALL LETTER B +249E;249E;249E;0028 0063 0029;0028 0063 0029; # (⒞; ⒞; ⒞; (c); (c); ) PARENTHESIZED LATIN SMALL LETTER C +249F;249F;249F;0028 0064 0029;0028 0064 0029; # (⒟; ⒟; ⒟; (d); (d); ) PARENTHESIZED LATIN SMALL LETTER D +24A0;24A0;24A0;0028 0065 0029;0028 0065 0029; # (⒠; ⒠; ⒠; (e); (e); ) PARENTHESIZED LATIN SMALL LETTER E +24A1;24A1;24A1;0028 0066 0029;0028 0066 0029; # (⒡; ⒡; ⒡; (f); (f); ) PARENTHESIZED LATIN SMALL LETTER F +24A2;24A2;24A2;0028 0067 0029;0028 0067 0029; # (⒢; ⒢; ⒢; (g); (g); ) PARENTHESIZED LATIN SMALL LETTER G +24A3;24A3;24A3;0028 0068 0029;0028 0068 0029; # (⒣; ⒣; ⒣; (h); (h); ) PARENTHESIZED LATIN SMALL LETTER H +24A4;24A4;24A4;0028 0069 0029;0028 0069 0029; # (⒤; ⒤; ⒤; (i); (i); ) PARENTHESIZED LATIN SMALL LETTER I +24A5;24A5;24A5;0028 006A 0029;0028 006A 0029; # (⒥; ⒥; ⒥; (j); (j); ) PARENTHESIZED LATIN SMALL LETTER J +24A6;24A6;24A6;0028 006B 0029;0028 006B 0029; # (⒦; ⒦; ⒦; (k); (k); ) PARENTHESIZED LATIN SMALL LETTER K +24A7;24A7;24A7;0028 006C 0029;0028 006C 0029; # (⒧; ⒧; ⒧; (l); (l); ) PARENTHESIZED LATIN SMALL LETTER L +24A8;24A8;24A8;0028 006D 0029;0028 006D 0029; # (⒨; ⒨; ⒨; (m); (m); ) PARENTHESIZED LATIN SMALL LETTER M +24A9;24A9;24A9;0028 006E 0029;0028 006E 0029; # (⒩; ⒩; ⒩; (n); (n); ) PARENTHESIZED LATIN SMALL LETTER N +24AA;24AA;24AA;0028 006F 0029;0028 006F 0029; # (⒪; ⒪; ⒪; (o); (o); ) PARENTHESIZED LATIN SMALL LETTER O +24AB;24AB;24AB;0028 0070 0029;0028 0070 0029; # (⒫; ⒫; ⒫; (p); (p); ) PARENTHESIZED LATIN SMALL LETTER P +24AC;24AC;24AC;0028 0071 0029;0028 0071 0029; # (⒬; ⒬; ⒬; (q); (q); ) PARENTHESIZED LATIN SMALL LETTER Q +24AD;24AD;24AD;0028 0072 0029;0028 0072 0029; # (⒭; ⒭; ⒭; (r); (r); ) PARENTHESIZED LATIN SMALL LETTER R +24AE;24AE;24AE;0028 0073 0029;0028 0073 0029; # (⒮; ⒮; ⒮; (s); (s); ) PARENTHESIZED LATIN SMALL LETTER S +24AF;24AF;24AF;0028 0074 0029;0028 0074 0029; # (⒯; ⒯; ⒯; (t); (t); ) PARENTHESIZED LATIN SMALL LETTER T +24B0;24B0;24B0;0028 0075 0029;0028 0075 0029; # (⒰; ⒰; ⒰; (u); (u); ) PARENTHESIZED LATIN SMALL LETTER U +24B1;24B1;24B1;0028 0076 0029;0028 0076 0029; # (⒱; ⒱; ⒱; (v); (v); ) PARENTHESIZED LATIN SMALL LETTER V +24B2;24B2;24B2;0028 0077 0029;0028 0077 0029; # (⒲; ⒲; ⒲; (w); (w); ) PARENTHESIZED LATIN SMALL LETTER W +24B3;24B3;24B3;0028 0078 0029;0028 0078 0029; # (⒳; ⒳; ⒳; (x); (x); ) PARENTHESIZED LATIN SMALL LETTER X +24B4;24B4;24B4;0028 0079 0029;0028 0079 0029; # (⒴; ⒴; ⒴; (y); (y); ) PARENTHESIZED LATIN SMALL LETTER Y +24B5;24B5;24B5;0028 007A 0029;0028 007A 0029; # (⒵; ⒵; ⒵; (z); (z); ) PARENTHESIZED LATIN SMALL LETTER Z +24B6;24B6;24B6;0041;0041; # (Ⓐ; Ⓐ; Ⓐ; A; A; ) CIRCLED LATIN CAPITAL LETTER A +24B7;24B7;24B7;0042;0042; # (Ⓑ; Ⓑ; Ⓑ; B; B; ) CIRCLED LATIN CAPITAL LETTER B +24B8;24B8;24B8;0043;0043; # (Ⓒ; Ⓒ; Ⓒ; C; C; ) CIRCLED LATIN CAPITAL LETTER C +24B9;24B9;24B9;0044;0044; # (Ⓓ; Ⓓ; Ⓓ; D; D; ) CIRCLED LATIN CAPITAL LETTER D +24BA;24BA;24BA;0045;0045; # (Ⓔ; Ⓔ; Ⓔ; E; E; ) CIRCLED LATIN CAPITAL LETTER E +24BB;24BB;24BB;0046;0046; # (Ⓕ; Ⓕ; Ⓕ; F; F; ) CIRCLED LATIN CAPITAL LETTER F +24BC;24BC;24BC;0047;0047; # (Ⓖ; Ⓖ; Ⓖ; G; G; ) CIRCLED LATIN CAPITAL LETTER G +24BD;24BD;24BD;0048;0048; # (Ⓗ; Ⓗ; Ⓗ; H; H; ) CIRCLED LATIN CAPITAL LETTER H +24BE;24BE;24BE;0049;0049; # (Ⓘ; Ⓘ; Ⓘ; I; I; ) CIRCLED LATIN CAPITAL LETTER I +24BF;24BF;24BF;004A;004A; # (Ⓙ; Ⓙ; Ⓙ; J; J; ) CIRCLED LATIN CAPITAL LETTER J +24C0;24C0;24C0;004B;004B; # (Ⓚ; Ⓚ; Ⓚ; K; K; ) CIRCLED LATIN CAPITAL LETTER K +24C1;24C1;24C1;004C;004C; # (Ⓛ; Ⓛ; Ⓛ; L; L; ) CIRCLED LATIN CAPITAL LETTER L +24C2;24C2;24C2;004D;004D; # (Ⓜ; Ⓜ; Ⓜ; M; M; ) CIRCLED LATIN CAPITAL LETTER M +24C3;24C3;24C3;004E;004E; # (Ⓝ; Ⓝ; Ⓝ; N; N; ) CIRCLED LATIN CAPITAL LETTER N +24C4;24C4;24C4;004F;004F; # (Ⓞ; Ⓞ; Ⓞ; O; O; ) CIRCLED LATIN CAPITAL LETTER O +24C5;24C5;24C5;0050;0050; # (Ⓟ; Ⓟ; Ⓟ; P; P; ) CIRCLED LATIN CAPITAL LETTER P +24C6;24C6;24C6;0051;0051; # (Ⓠ; Ⓠ; Ⓠ; Q; Q; ) CIRCLED LATIN CAPITAL LETTER Q +24C7;24C7;24C7;0052;0052; # (Ⓡ; Ⓡ; Ⓡ; R; R; ) CIRCLED LATIN CAPITAL LETTER R +24C8;24C8;24C8;0053;0053; # (Ⓢ; Ⓢ; Ⓢ; S; S; ) CIRCLED LATIN CAPITAL LETTER S +24C9;24C9;24C9;0054;0054; # (Ⓣ; Ⓣ; Ⓣ; T; T; ) CIRCLED LATIN CAPITAL LETTER T +24CA;24CA;24CA;0055;0055; # (Ⓤ; Ⓤ; Ⓤ; U; U; ) CIRCLED LATIN CAPITAL LETTER U +24CB;24CB;24CB;0056;0056; # (Ⓥ; Ⓥ; Ⓥ; V; V; ) CIRCLED LATIN CAPITAL LETTER V +24CC;24CC;24CC;0057;0057; # (Ⓦ; Ⓦ; Ⓦ; W; W; ) CIRCLED LATIN CAPITAL LETTER W +24CD;24CD;24CD;0058;0058; # (Ⓧ; Ⓧ; Ⓧ; X; X; ) CIRCLED LATIN CAPITAL LETTER X +24CE;24CE;24CE;0059;0059; # (Ⓨ; Ⓨ; Ⓨ; Y; Y; ) CIRCLED LATIN CAPITAL LETTER Y +24CF;24CF;24CF;005A;005A; # (Ⓩ; Ⓩ; Ⓩ; Z; Z; ) CIRCLED LATIN CAPITAL LETTER Z +24D0;24D0;24D0;0061;0061; # (ⓐ; ⓐ; ⓐ; a; a; ) CIRCLED LATIN SMALL LETTER A +24D1;24D1;24D1;0062;0062; # (ⓑ; ⓑ; ⓑ; b; b; ) CIRCLED LATIN SMALL LETTER B +24D2;24D2;24D2;0063;0063; # (ⓒ; ⓒ; ⓒ; c; c; ) CIRCLED LATIN SMALL LETTER C +24D3;24D3;24D3;0064;0064; # (ⓓ; ⓓ; ⓓ; d; d; ) CIRCLED LATIN SMALL LETTER D +24D4;24D4;24D4;0065;0065; # (ⓔ; ⓔ; ⓔ; e; e; ) CIRCLED LATIN SMALL LETTER E +24D5;24D5;24D5;0066;0066; # (ⓕ; ⓕ; ⓕ; f; f; ) CIRCLED LATIN SMALL LETTER F +24D6;24D6;24D6;0067;0067; # (ⓖ; ⓖ; ⓖ; g; g; ) CIRCLED LATIN SMALL LETTER G +24D7;24D7;24D7;0068;0068; # (ⓗ; ⓗ; ⓗ; h; h; ) CIRCLED LATIN SMALL LETTER H +24D8;24D8;24D8;0069;0069; # (ⓘ; ⓘ; ⓘ; i; i; ) CIRCLED LATIN SMALL LETTER I +24D9;24D9;24D9;006A;006A; # (ⓙ; ⓙ; ⓙ; j; j; ) CIRCLED LATIN SMALL LETTER J +24DA;24DA;24DA;006B;006B; # (ⓚ; ⓚ; ⓚ; k; k; ) CIRCLED LATIN SMALL LETTER K +24DB;24DB;24DB;006C;006C; # (ⓛ; ⓛ; ⓛ; l; l; ) CIRCLED LATIN SMALL LETTER L +24DC;24DC;24DC;006D;006D; # (ⓜ; ⓜ; ⓜ; m; m; ) CIRCLED LATIN SMALL LETTER M +24DD;24DD;24DD;006E;006E; # (ⓝ; ⓝ; ⓝ; n; n; ) CIRCLED LATIN SMALL LETTER N +24DE;24DE;24DE;006F;006F; # (ⓞ; ⓞ; ⓞ; o; o; ) CIRCLED LATIN SMALL LETTER O +24DF;24DF;24DF;0070;0070; # (ⓟ; ⓟ; ⓟ; p; p; ) CIRCLED LATIN SMALL LETTER P +24E0;24E0;24E0;0071;0071; # (ⓠ; ⓠ; ⓠ; q; q; ) CIRCLED LATIN SMALL LETTER Q +24E1;24E1;24E1;0072;0072; # (ⓡ; ⓡ; ⓡ; r; r; ) CIRCLED LATIN SMALL LETTER R +24E2;24E2;24E2;0073;0073; # (ⓢ; ⓢ; ⓢ; s; s; ) CIRCLED LATIN SMALL LETTER S +24E3;24E3;24E3;0074;0074; # (ⓣ; ⓣ; ⓣ; t; t; ) CIRCLED LATIN SMALL LETTER T +24E4;24E4;24E4;0075;0075; # (ⓤ; ⓤ; ⓤ; u; u; ) CIRCLED LATIN SMALL LETTER U +24E5;24E5;24E5;0076;0076; # (ⓥ; ⓥ; ⓥ; v; v; ) CIRCLED LATIN SMALL LETTER V +24E6;24E6;24E6;0077;0077; # (ⓦ; ⓦ; ⓦ; w; w; ) CIRCLED LATIN SMALL LETTER W +24E7;24E7;24E7;0078;0078; # (ⓧ; ⓧ; ⓧ; x; x; ) CIRCLED LATIN SMALL LETTER X +24E8;24E8;24E8;0079;0079; # (ⓨ; ⓨ; ⓨ; y; y; ) CIRCLED LATIN SMALL LETTER Y +24E9;24E9;24E9;007A;007A; # (ⓩ; ⓩ; ⓩ; z; z; ) CIRCLED LATIN SMALL LETTER Z +24EA;24EA;24EA;0030;0030; # (⓪; ⓪; ⓪; 0; 0; ) CIRCLED DIGIT ZERO +2A0C;2A0C;2A0C;222B 222B 222B 222B;222B 222B 222B 222B; # (⨌; ⨌; ⨌; ∫∫∫∫; ∫∫∫∫; ) QUADRUPLE INTEGRAL OPERATOR +2A74;2A74;2A74;003A 003A 003D;003A 003A 003D; # (⩴; ⩴; ⩴; ::=; ::=; ) DOUBLE COLON EQUAL +2A75;2A75;2A75;003D 003D;003D 003D; # (⩵; ⩵; ⩵; ==; ==; ) TWO CONSECUTIVE EQUALS SIGNS +2A76;2A76;2A76;003D 003D 003D;003D 003D 003D; # (⩶; ⩶; ⩶; ===; ===; ) THREE CONSECUTIVE EQUALS SIGNS +2ADC;2ADD 0338;2ADD 0338;2ADD 0338;2ADD 0338; # (⫝̸; ⫝◌̸; ⫝◌̸; ⫝◌̸; ⫝◌̸; ) FORKING +2E9F;2E9F;2E9F;6BCD;6BCD; # (⺟; ⺟; ⺟; 母; 母; ) CJK RADICAL MOTHER +2EF3;2EF3;2EF3;9F9F;9F9F; # (⻳; ⻳; ⻳; 龟; 龟; ) CJK RADICAL C-SIMPLIFIED TURTLE +2F00;2F00;2F00;4E00;4E00; # (⼀; ⼀; ⼀; 一; 一; ) KANGXI RADICAL ONE +2F01;2F01;2F01;4E28;4E28; # (⼁; ⼁; ⼁; 丨; 丨; ) KANGXI RADICAL LINE +2F02;2F02;2F02;4E36;4E36; # (⼂; ⼂; ⼂; 丶; 丶; ) KANGXI RADICAL DOT +2F03;2F03;2F03;4E3F;4E3F; # (⼃; ⼃; ⼃; 丿; 丿; ) KANGXI RADICAL SLASH +2F04;2F04;2F04;4E59;4E59; # (⼄; ⼄; ⼄; 乙; 乙; ) KANGXI RADICAL SECOND +2F05;2F05;2F05;4E85;4E85; # (⼅; ⼅; ⼅; 亅; 亅; ) KANGXI RADICAL HOOK +2F06;2F06;2F06;4E8C;4E8C; # (⼆; ⼆; ⼆; 二; 二; ) KANGXI RADICAL TWO +2F07;2F07;2F07;4EA0;4EA0; # (⼇; ⼇; ⼇; 亠; 亠; ) KANGXI RADICAL LID +2F08;2F08;2F08;4EBA;4EBA; # (⼈; ⼈; ⼈; 人; 人; ) KANGXI RADICAL MAN +2F09;2F09;2F09;513F;513F; # (⼉; ⼉; ⼉; 儿; 儿; ) KANGXI RADICAL LEGS +2F0A;2F0A;2F0A;5165;5165; # (⼊; ⼊; ⼊; 入; 入; ) KANGXI RADICAL ENTER +2F0B;2F0B;2F0B;516B;516B; # (⼋; ⼋; ⼋; 八; 八; ) KANGXI RADICAL EIGHT +2F0C;2F0C;2F0C;5182;5182; # (⼌; ⼌; ⼌; 冂; 冂; ) KANGXI RADICAL DOWN BOX +2F0D;2F0D;2F0D;5196;5196; # (⼍; ⼍; ⼍; 冖; 冖; ) KANGXI RADICAL COVER +2F0E;2F0E;2F0E;51AB;51AB; # (⼎; ⼎; ⼎; 冫; 冫; ) KANGXI RADICAL ICE +2F0F;2F0F;2F0F;51E0;51E0; # (⼏; ⼏; ⼏; 几; 几; ) KANGXI RADICAL TABLE +2F10;2F10;2F10;51F5;51F5; # (⼐; ⼐; ⼐; 凵; 凵; ) KANGXI RADICAL OPEN BOX +2F11;2F11;2F11;5200;5200; # (⼑; ⼑; ⼑; 刀; 刀; ) KANGXI RADICAL KNIFE +2F12;2F12;2F12;529B;529B; # (⼒; ⼒; ⼒; 力; 力; ) KANGXI RADICAL POWER +2F13;2F13;2F13;52F9;52F9; # (⼓; ⼓; ⼓; 勹; 勹; ) KANGXI RADICAL WRAP +2F14;2F14;2F14;5315;5315; # (⼔; ⼔; ⼔; 匕; 匕; ) KANGXI RADICAL SPOON +2F15;2F15;2F15;531A;531A; # (⼕; ⼕; ⼕; 匚; 匚; ) KANGXI RADICAL RIGHT OPEN BOX +2F16;2F16;2F16;5338;5338; # (⼖; ⼖; ⼖; 匸; 匸; ) KANGXI RADICAL HIDING ENCLOSURE +2F17;2F17;2F17;5341;5341; # (⼗; ⼗; ⼗; 十; 十; ) KANGXI RADICAL TEN +2F18;2F18;2F18;535C;535C; # (⼘; ⼘; ⼘; 卜; 卜; ) KANGXI RADICAL DIVINATION +2F19;2F19;2F19;5369;5369; # (⼙; ⼙; ⼙; 卩; 卩; ) KANGXI RADICAL SEAL +2F1A;2F1A;2F1A;5382;5382; # (⼚; ⼚; ⼚; 厂; 厂; ) KANGXI RADICAL CLIFF +2F1B;2F1B;2F1B;53B6;53B6; # (⼛; ⼛; ⼛; 厶; 厶; ) KANGXI RADICAL PRIVATE +2F1C;2F1C;2F1C;53C8;53C8; # (⼜; ⼜; ⼜; 又; 又; ) KANGXI RADICAL AGAIN +2F1D;2F1D;2F1D;53E3;53E3; # (⼝; ⼝; ⼝; 口; 口; ) KANGXI RADICAL MOUTH +2F1E;2F1E;2F1E;56D7;56D7; # (⼞; ⼞; ⼞; 囗; 囗; ) KANGXI RADICAL ENCLOSURE +2F1F;2F1F;2F1F;571F;571F; # (⼟; ⼟; ⼟; 土; 土; ) KANGXI RADICAL EARTH +2F20;2F20;2F20;58EB;58EB; # (⼠; ⼠; ⼠; 士; 士; ) KANGXI RADICAL SCHOLAR +2F21;2F21;2F21;5902;5902; # (⼡; ⼡; ⼡; 夂; 夂; ) KANGXI RADICAL GO +2F22;2F22;2F22;590A;590A; # (⼢; ⼢; ⼢; 夊; 夊; ) KANGXI RADICAL GO SLOWLY +2F23;2F23;2F23;5915;5915; # (⼣; ⼣; ⼣; 夕; 夕; ) KANGXI RADICAL EVENING +2F24;2F24;2F24;5927;5927; # (⼤; ⼤; ⼤; 大; 大; ) KANGXI RADICAL BIG +2F25;2F25;2F25;5973;5973; # (⼥; ⼥; ⼥; 女; 女; ) KANGXI RADICAL WOMAN +2F26;2F26;2F26;5B50;5B50; # (⼦; ⼦; ⼦; 子; 子; ) KANGXI RADICAL CHILD +2F27;2F27;2F27;5B80;5B80; # (⼧; ⼧; ⼧; 宀; 宀; ) KANGXI RADICAL ROOF +2F28;2F28;2F28;5BF8;5BF8; # (⼨; ⼨; ⼨; 寸; 寸; ) KANGXI RADICAL INCH +2F29;2F29;2F29;5C0F;5C0F; # (⼩; ⼩; ⼩; 小; 小; ) KANGXI RADICAL SMALL +2F2A;2F2A;2F2A;5C22;5C22; # (⼪; ⼪; ⼪; 尢; 尢; ) KANGXI RADICAL LAME +2F2B;2F2B;2F2B;5C38;5C38; # (⼫; ⼫; ⼫; 尸; 尸; ) KANGXI RADICAL CORPSE +2F2C;2F2C;2F2C;5C6E;5C6E; # (⼬; ⼬; ⼬; 屮; 屮; ) KANGXI RADICAL SPROUT +2F2D;2F2D;2F2D;5C71;5C71; # (⼭; ⼭; ⼭; 山; 山; ) KANGXI RADICAL MOUNTAIN +2F2E;2F2E;2F2E;5DDB;5DDB; # (⼮; ⼮; ⼮; 巛; 巛; ) KANGXI RADICAL RIVER +2F2F;2F2F;2F2F;5DE5;5DE5; # (⼯; ⼯; ⼯; 工; 工; ) KANGXI RADICAL WORK +2F30;2F30;2F30;5DF1;5DF1; # (⼰; ⼰; ⼰; 己; 己; ) KANGXI RADICAL ONESELF +2F31;2F31;2F31;5DFE;5DFE; # (⼱; ⼱; ⼱; 巾; 巾; ) KANGXI RADICAL TURBAN +2F32;2F32;2F32;5E72;5E72; # (⼲; ⼲; ⼲; 干; 干; ) KANGXI RADICAL DRY +2F33;2F33;2F33;5E7A;5E7A; # (⼳; ⼳; ⼳; 幺; 幺; ) KANGXI RADICAL SHORT THREAD +2F34;2F34;2F34;5E7F;5E7F; # (⼴; ⼴; ⼴; 广; 广; ) KANGXI RADICAL DOTTED CLIFF +2F35;2F35;2F35;5EF4;5EF4; # (⼵; ⼵; ⼵; 廴; 廴; ) KANGXI RADICAL LONG STRIDE +2F36;2F36;2F36;5EFE;5EFE; # (⼶; ⼶; ⼶; 廾; 廾; ) KANGXI RADICAL TWO HANDS +2F37;2F37;2F37;5F0B;5F0B; # (⼷; ⼷; ⼷; 弋; 弋; ) KANGXI RADICAL SHOOT +2F38;2F38;2F38;5F13;5F13; # (⼸; ⼸; ⼸; 弓; 弓; ) KANGXI RADICAL BOW +2F39;2F39;2F39;5F50;5F50; # (⼹; ⼹; ⼹; 彐; 彐; ) KANGXI RADICAL SNOUT +2F3A;2F3A;2F3A;5F61;5F61; # (⼺; ⼺; ⼺; 彡; 彡; ) KANGXI RADICAL BRISTLE +2F3B;2F3B;2F3B;5F73;5F73; # (⼻; ⼻; ⼻; 彳; 彳; ) KANGXI RADICAL STEP +2F3C;2F3C;2F3C;5FC3;5FC3; # (⼼; ⼼; ⼼; 心; 心; ) KANGXI RADICAL HEART +2F3D;2F3D;2F3D;6208;6208; # (⼽; ⼽; ⼽; 戈; 戈; ) KANGXI RADICAL HALBERD +2F3E;2F3E;2F3E;6236;6236; # (⼾; ⼾; ⼾; 戶; 戶; ) KANGXI RADICAL DOOR +2F3F;2F3F;2F3F;624B;624B; # (⼿; ⼿; ⼿; 手; 手; ) KANGXI RADICAL HAND +2F40;2F40;2F40;652F;652F; # (⽀; ⽀; ⽀; 支; 支; ) KANGXI RADICAL BRANCH +2F41;2F41;2F41;6534;6534; # (⽁; ⽁; ⽁; 攴; 攴; ) KANGXI RADICAL RAP +2F42;2F42;2F42;6587;6587; # (⽂; ⽂; ⽂; 文; 文; ) KANGXI RADICAL SCRIPT +2F43;2F43;2F43;6597;6597; # (⽃; ⽃; ⽃; 斗; 斗; ) KANGXI RADICAL DIPPER +2F44;2F44;2F44;65A4;65A4; # (⽄; ⽄; ⽄; 斤; 斤; ) KANGXI RADICAL AXE +2F45;2F45;2F45;65B9;65B9; # (⽅; ⽅; ⽅; 方; 方; ) KANGXI RADICAL SQUARE +2F46;2F46;2F46;65E0;65E0; # (⽆; ⽆; ⽆; 无; 无; ) KANGXI RADICAL NOT +2F47;2F47;2F47;65E5;65E5; # (⽇; ⽇; ⽇; 日; 日; ) KANGXI RADICAL SUN +2F48;2F48;2F48;66F0;66F0; # (⽈; ⽈; ⽈; 曰; 曰; ) KANGXI RADICAL SAY +2F49;2F49;2F49;6708;6708; # (⽉; ⽉; ⽉; 月; 月; ) KANGXI RADICAL MOON +2F4A;2F4A;2F4A;6728;6728; # (⽊; ⽊; ⽊; 木; 木; ) KANGXI RADICAL TREE +2F4B;2F4B;2F4B;6B20;6B20; # (⽋; ⽋; ⽋; 欠; 欠; ) KANGXI RADICAL LACK +2F4C;2F4C;2F4C;6B62;6B62; # (⽌; ⽌; ⽌; 止; 止; ) KANGXI RADICAL STOP +2F4D;2F4D;2F4D;6B79;6B79; # (⽍; ⽍; ⽍; 歹; 歹; ) KANGXI RADICAL DEATH +2F4E;2F4E;2F4E;6BB3;6BB3; # (⽎; ⽎; ⽎; 殳; 殳; ) KANGXI RADICAL WEAPON +2F4F;2F4F;2F4F;6BCB;6BCB; # (⽏; ⽏; ⽏; 毋; 毋; ) KANGXI RADICAL DO NOT +2F50;2F50;2F50;6BD4;6BD4; # (⽐; ⽐; ⽐; 比; 比; ) KANGXI RADICAL COMPARE +2F51;2F51;2F51;6BDB;6BDB; # (⽑; ⽑; ⽑; 毛; 毛; ) KANGXI RADICAL FUR +2F52;2F52;2F52;6C0F;6C0F; # (⽒; ⽒; ⽒; 氏; 氏; ) KANGXI RADICAL CLAN +2F53;2F53;2F53;6C14;6C14; # (⽓; ⽓; ⽓; 气; 气; ) KANGXI RADICAL STEAM +2F54;2F54;2F54;6C34;6C34; # (⽔; ⽔; ⽔; 水; 水; ) KANGXI RADICAL WATER +2F55;2F55;2F55;706B;706B; # (⽕; ⽕; ⽕; 火; 火; ) KANGXI RADICAL FIRE +2F56;2F56;2F56;722A;722A; # (⽖; ⽖; ⽖; 爪; 爪; ) KANGXI RADICAL CLAW +2F57;2F57;2F57;7236;7236; # (⽗; ⽗; ⽗; 父; 父; ) KANGXI RADICAL FATHER +2F58;2F58;2F58;723B;723B; # (⽘; ⽘; ⽘; 爻; 爻; ) KANGXI RADICAL DOUBLE X +2F59;2F59;2F59;723F;723F; # (⽙; ⽙; ⽙; 爿; 爿; ) KANGXI RADICAL HALF TREE TRUNK +2F5A;2F5A;2F5A;7247;7247; # (⽚; ⽚; ⽚; 片; 片; ) KANGXI RADICAL SLICE +2F5B;2F5B;2F5B;7259;7259; # (⽛; ⽛; ⽛; 牙; 牙; ) KANGXI RADICAL FANG +2F5C;2F5C;2F5C;725B;725B; # (⽜; ⽜; ⽜; 牛; 牛; ) KANGXI RADICAL COW +2F5D;2F5D;2F5D;72AC;72AC; # (⽝; ⽝; ⽝; 犬; 犬; ) KANGXI RADICAL DOG +2F5E;2F5E;2F5E;7384;7384; # (⽞; ⽞; ⽞; 玄; 玄; ) KANGXI RADICAL PROFOUND +2F5F;2F5F;2F5F;7389;7389; # (⽟; ⽟; ⽟; 玉; 玉; ) KANGXI RADICAL JADE +2F60;2F60;2F60;74DC;74DC; # (⽠; ⽠; ⽠; 瓜; 瓜; ) KANGXI RADICAL MELON +2F61;2F61;2F61;74E6;74E6; # (⽡; ⽡; ⽡; 瓦; 瓦; ) KANGXI RADICAL TILE +2F62;2F62;2F62;7518;7518; # (⽢; ⽢; ⽢; 甘; 甘; ) KANGXI RADICAL SWEET +2F63;2F63;2F63;751F;751F; # (⽣; ⽣; ⽣; 生; 生; ) KANGXI RADICAL LIFE +2F64;2F64;2F64;7528;7528; # (⽤; ⽤; ⽤; 用; 用; ) KANGXI RADICAL USE +2F65;2F65;2F65;7530;7530; # (⽥; ⽥; ⽥; 田; 田; ) KANGXI RADICAL FIELD +2F66;2F66;2F66;758B;758B; # (⽦; ⽦; ⽦; 疋; 疋; ) KANGXI RADICAL BOLT OF CLOTH +2F67;2F67;2F67;7592;7592; # (⽧; ⽧; ⽧; 疒; 疒; ) KANGXI RADICAL SICKNESS +2F68;2F68;2F68;7676;7676; # (⽨; ⽨; ⽨; 癶; 癶; ) KANGXI RADICAL DOTTED TENT +2F69;2F69;2F69;767D;767D; # (⽩; ⽩; ⽩; 白; 白; ) KANGXI RADICAL WHITE +2F6A;2F6A;2F6A;76AE;76AE; # (⽪; ⽪; ⽪; 皮; 皮; ) KANGXI RADICAL SKIN +2F6B;2F6B;2F6B;76BF;76BF; # (⽫; ⽫; ⽫; 皿; 皿; ) KANGXI RADICAL DISH +2F6C;2F6C;2F6C;76EE;76EE; # (⽬; ⽬; ⽬; 目; 目; ) KANGXI RADICAL EYE +2F6D;2F6D;2F6D;77DB;77DB; # (⽭; ⽭; ⽭; 矛; 矛; ) KANGXI RADICAL SPEAR +2F6E;2F6E;2F6E;77E2;77E2; # (⽮; ⽮; ⽮; 矢; 矢; ) KANGXI RADICAL ARROW +2F6F;2F6F;2F6F;77F3;77F3; # (⽯; ⽯; ⽯; 石; 石; ) KANGXI RADICAL STONE +2F70;2F70;2F70;793A;793A; # (⽰; ⽰; ⽰; 示; 示; ) KANGXI RADICAL SPIRIT +2F71;2F71;2F71;79B8;79B8; # (⽱; ⽱; ⽱; 禸; 禸; ) KANGXI RADICAL TRACK +2F72;2F72;2F72;79BE;79BE; # (⽲; ⽲; ⽲; 禾; 禾; ) KANGXI RADICAL GRAIN +2F73;2F73;2F73;7A74;7A74; # (⽳; ⽳; ⽳; 穴; 穴; ) KANGXI RADICAL CAVE +2F74;2F74;2F74;7ACB;7ACB; # (⽴; ⽴; ⽴; 立; 立; ) KANGXI RADICAL STAND +2F75;2F75;2F75;7AF9;7AF9; # (⽵; ⽵; ⽵; 竹; 竹; ) KANGXI RADICAL BAMBOO +2F76;2F76;2F76;7C73;7C73; # (⽶; ⽶; ⽶; 米; 米; ) KANGXI RADICAL RICE +2F77;2F77;2F77;7CF8;7CF8; # (⽷; ⽷; ⽷; 糸; 糸; ) KANGXI RADICAL SILK +2F78;2F78;2F78;7F36;7F36; # (⽸; ⽸; ⽸; 缶; 缶; ) KANGXI RADICAL JAR +2F79;2F79;2F79;7F51;7F51; # (⽹; ⽹; ⽹; 网; 网; ) KANGXI RADICAL NET +2F7A;2F7A;2F7A;7F8A;7F8A; # (⽺; ⽺; ⽺; 羊; 羊; ) KANGXI RADICAL SHEEP +2F7B;2F7B;2F7B;7FBD;7FBD; # (⽻; ⽻; ⽻; 羽; 羽; ) KANGXI RADICAL FEATHER +2F7C;2F7C;2F7C;8001;8001; # (⽼; ⽼; ⽼; 老; 老; ) KANGXI RADICAL OLD +2F7D;2F7D;2F7D;800C;800C; # (⽽; ⽽; ⽽; 而; 而; ) KANGXI RADICAL AND +2F7E;2F7E;2F7E;8012;8012; # (⽾; ⽾; ⽾; 耒; 耒; ) KANGXI RADICAL PLOW +2F7F;2F7F;2F7F;8033;8033; # (⽿; ⽿; ⽿; 耳; 耳; ) KANGXI RADICAL EAR +2F80;2F80;2F80;807F;807F; # (⾀; ⾀; ⾀; 聿; 聿; ) KANGXI RADICAL BRUSH +2F81;2F81;2F81;8089;8089; # (⾁; ⾁; ⾁; 肉; 肉; ) KANGXI RADICAL MEAT +2F82;2F82;2F82;81E3;81E3; # (⾂; ⾂; ⾂; 臣; 臣; ) KANGXI RADICAL MINISTER +2F83;2F83;2F83;81EA;81EA; # (⾃; ⾃; ⾃; 自; 自; ) KANGXI RADICAL SELF +2F84;2F84;2F84;81F3;81F3; # (⾄; ⾄; ⾄; 至; 至; ) KANGXI RADICAL ARRIVE +2F85;2F85;2F85;81FC;81FC; # (⾅; ⾅; ⾅; 臼; 臼; ) KANGXI RADICAL MORTAR +2F86;2F86;2F86;820C;820C; # (⾆; ⾆; ⾆; 舌; 舌; ) KANGXI RADICAL TONGUE +2F87;2F87;2F87;821B;821B; # (⾇; ⾇; ⾇; 舛; 舛; ) KANGXI RADICAL OPPOSE +2F88;2F88;2F88;821F;821F; # (⾈; ⾈; ⾈; 舟; 舟; ) KANGXI RADICAL BOAT +2F89;2F89;2F89;826E;826E; # (⾉; ⾉; ⾉; 艮; 艮; ) KANGXI RADICAL STOPPING +2F8A;2F8A;2F8A;8272;8272; # (⾊; ⾊; ⾊; 色; 色; ) KANGXI RADICAL COLOR +2F8B;2F8B;2F8B;8278;8278; # (⾋; ⾋; ⾋; 艸; 艸; ) KANGXI RADICAL GRASS +2F8C;2F8C;2F8C;864D;864D; # (⾌; ⾌; ⾌; 虍; 虍; ) KANGXI RADICAL TIGER +2F8D;2F8D;2F8D;866B;866B; # (⾍; ⾍; ⾍; 虫; 虫; ) KANGXI RADICAL INSECT +2F8E;2F8E;2F8E;8840;8840; # (⾎; ⾎; ⾎; 血; 血; ) KANGXI RADICAL BLOOD +2F8F;2F8F;2F8F;884C;884C; # (⾏; ⾏; ⾏; 行; 行; ) KANGXI RADICAL WALK ENCLOSURE +2F90;2F90;2F90;8863;8863; # (⾐; ⾐; ⾐; 衣; 衣; ) KANGXI RADICAL CLOTHES +2F91;2F91;2F91;897E;897E; # (⾑; ⾑; ⾑; 襾; 襾; ) KANGXI RADICAL WEST +2F92;2F92;2F92;898B;898B; # (⾒; ⾒; ⾒; 見; 見; ) KANGXI RADICAL SEE +2F93;2F93;2F93;89D2;89D2; # (⾓; ⾓; ⾓; 角; 角; ) KANGXI RADICAL HORN +2F94;2F94;2F94;8A00;8A00; # (⾔; ⾔; ⾔; 言; 言; ) KANGXI RADICAL SPEECH +2F95;2F95;2F95;8C37;8C37; # (⾕; ⾕; ⾕; 谷; 谷; ) KANGXI RADICAL VALLEY +2F96;2F96;2F96;8C46;8C46; # (⾖; ⾖; ⾖; 豆; 豆; ) KANGXI RADICAL BEAN +2F97;2F97;2F97;8C55;8C55; # (⾗; ⾗; ⾗; 豕; 豕; ) KANGXI RADICAL PIG +2F98;2F98;2F98;8C78;8C78; # (⾘; ⾘; ⾘; 豸; 豸; ) KANGXI RADICAL BADGER +2F99;2F99;2F99;8C9D;8C9D; # (⾙; ⾙; ⾙; 貝; 貝; ) KANGXI RADICAL SHELL +2F9A;2F9A;2F9A;8D64;8D64; # (⾚; ⾚; ⾚; 赤; 赤; ) KANGXI RADICAL RED +2F9B;2F9B;2F9B;8D70;8D70; # (⾛; ⾛; ⾛; 走; 走; ) KANGXI RADICAL RUN +2F9C;2F9C;2F9C;8DB3;8DB3; # (⾜; ⾜; ⾜; 足; 足; ) KANGXI RADICAL FOOT +2F9D;2F9D;2F9D;8EAB;8EAB; # (⾝; ⾝; ⾝; 身; 身; ) KANGXI RADICAL BODY +2F9E;2F9E;2F9E;8ECA;8ECA; # (⾞; ⾞; ⾞; 車; 車; ) KANGXI RADICAL CART +2F9F;2F9F;2F9F;8F9B;8F9B; # (⾟; ⾟; ⾟; 辛; 辛; ) KANGXI RADICAL BITTER +2FA0;2FA0;2FA0;8FB0;8FB0; # (⾠; ⾠; ⾠; 辰; 辰; ) KANGXI RADICAL MORNING +2FA1;2FA1;2FA1;8FB5;8FB5; # (⾡; ⾡; ⾡; 辵; 辵; ) KANGXI RADICAL WALK +2FA2;2FA2;2FA2;9091;9091; # (⾢; ⾢; ⾢; 邑; 邑; ) KANGXI RADICAL CITY +2FA3;2FA3;2FA3;9149;9149; # (⾣; ⾣; ⾣; 酉; 酉; ) KANGXI RADICAL WINE +2FA4;2FA4;2FA4;91C6;91C6; # (⾤; ⾤; ⾤; 釆; 釆; ) KANGXI RADICAL DISTINGUISH +2FA5;2FA5;2FA5;91CC;91CC; # (⾥; ⾥; ⾥; 里; 里; ) KANGXI RADICAL VILLAGE +2FA6;2FA6;2FA6;91D1;91D1; # (⾦; ⾦; ⾦; 金; 金; ) KANGXI RADICAL GOLD +2FA7;2FA7;2FA7;9577;9577; # (⾧; ⾧; ⾧; 長; 長; ) KANGXI RADICAL LONG +2FA8;2FA8;2FA8;9580;9580; # (⾨; ⾨; ⾨; 門; 門; ) KANGXI RADICAL GATE +2FA9;2FA9;2FA9;961C;961C; # (⾩; ⾩; ⾩; 阜; 阜; ) KANGXI RADICAL MOUND +2FAA;2FAA;2FAA;96B6;96B6; # (⾪; ⾪; ⾪; 隶; 隶; ) KANGXI RADICAL SLAVE +2FAB;2FAB;2FAB;96B9;96B9; # (⾫; ⾫; ⾫; 隹; 隹; ) KANGXI RADICAL SHORT TAILED BIRD +2FAC;2FAC;2FAC;96E8;96E8; # (⾬; ⾬; ⾬; 雨; 雨; ) KANGXI RADICAL RAIN +2FAD;2FAD;2FAD;9751;9751; # (⾭; ⾭; ⾭; 靑; 靑; ) KANGXI RADICAL BLUE +2FAE;2FAE;2FAE;975E;975E; # (⾮; ⾮; ⾮; 非; 非; ) KANGXI RADICAL WRONG +2FAF;2FAF;2FAF;9762;9762; # (⾯; ⾯; ⾯; 面; 面; ) KANGXI RADICAL FACE +2FB0;2FB0;2FB0;9769;9769; # (⾰; ⾰; ⾰; 革; 革; ) KANGXI RADICAL LEATHER +2FB1;2FB1;2FB1;97CB;97CB; # (⾱; ⾱; ⾱; 韋; 韋; ) KANGXI RADICAL TANNED LEATHER +2FB2;2FB2;2FB2;97ED;97ED; # (⾲; ⾲; ⾲; 韭; 韭; ) KANGXI RADICAL LEEK +2FB3;2FB3;2FB3;97F3;97F3; # (⾳; ⾳; ⾳; 音; 音; ) KANGXI RADICAL SOUND +2FB4;2FB4;2FB4;9801;9801; # (⾴; ⾴; ⾴; 頁; 頁; ) KANGXI RADICAL LEAF +2FB5;2FB5;2FB5;98A8;98A8; # (⾵; ⾵; ⾵; 風; 風; ) KANGXI RADICAL WIND +2FB6;2FB6;2FB6;98DB;98DB; # (⾶; ⾶; ⾶; 飛; 飛; ) KANGXI RADICAL FLY +2FB7;2FB7;2FB7;98DF;98DF; # (⾷; ⾷; ⾷; 食; 食; ) KANGXI RADICAL EAT +2FB8;2FB8;2FB8;9996;9996; # (⾸; ⾸; ⾸; 首; 首; ) KANGXI RADICAL HEAD +2FB9;2FB9;2FB9;9999;9999; # (⾹; ⾹; ⾹; 香; 香; ) KANGXI RADICAL FRAGRANT +2FBA;2FBA;2FBA;99AC;99AC; # (⾺; ⾺; ⾺; 馬; 馬; ) KANGXI RADICAL HORSE +2FBB;2FBB;2FBB;9AA8;9AA8; # (⾻; ⾻; ⾻; 骨; 骨; ) KANGXI RADICAL BONE +2FBC;2FBC;2FBC;9AD8;9AD8; # (⾼; ⾼; ⾼; 高; 高; ) KANGXI RADICAL TALL +2FBD;2FBD;2FBD;9ADF;9ADF; # (⾽; ⾽; ⾽; 髟; 髟; ) KANGXI RADICAL HAIR +2FBE;2FBE;2FBE;9B25;9B25; # (⾾; ⾾; ⾾; 鬥; 鬥; ) KANGXI RADICAL FIGHT +2FBF;2FBF;2FBF;9B2F;9B2F; # (⾿; ⾿; ⾿; 鬯; 鬯; ) KANGXI RADICAL SACRIFICIAL WINE +2FC0;2FC0;2FC0;9B32;9B32; # (⿀; ⿀; ⿀; 鬲; 鬲; ) KANGXI RADICAL CAULDRON +2FC1;2FC1;2FC1;9B3C;9B3C; # (⿁; ⿁; ⿁; 鬼; 鬼; ) KANGXI RADICAL GHOST +2FC2;2FC2;2FC2;9B5A;9B5A; # (⿂; ⿂; ⿂; 魚; 魚; ) KANGXI RADICAL FISH +2FC3;2FC3;2FC3;9CE5;9CE5; # (⿃; ⿃; ⿃; 鳥; 鳥; ) KANGXI RADICAL BIRD +2FC4;2FC4;2FC4;9E75;9E75; # (⿄; ⿄; ⿄; 鹵; 鹵; ) KANGXI RADICAL SALT +2FC5;2FC5;2FC5;9E7F;9E7F; # (⿅; ⿅; ⿅; 鹿; 鹿; ) KANGXI RADICAL DEER +2FC6;2FC6;2FC6;9EA5;9EA5; # (⿆; ⿆; ⿆; 麥; 麥; ) KANGXI RADICAL WHEAT +2FC7;2FC7;2FC7;9EBB;9EBB; # (⿇; ⿇; ⿇; 麻; 麻; ) KANGXI RADICAL HEMP +2FC8;2FC8;2FC8;9EC3;9EC3; # (⿈; ⿈; ⿈; 黃; 黃; ) KANGXI RADICAL YELLOW +2FC9;2FC9;2FC9;9ECD;9ECD; # (⿉; ⿉; ⿉; 黍; 黍; ) KANGXI RADICAL MILLET +2FCA;2FCA;2FCA;9ED1;9ED1; # (⿊; ⿊; ⿊; 黑; 黑; ) KANGXI RADICAL BLACK +2FCB;2FCB;2FCB;9EF9;9EF9; # (⿋; ⿋; ⿋; 黹; 黹; ) KANGXI RADICAL EMBROIDERY +2FCC;2FCC;2FCC;9EFD;9EFD; # (⿌; ⿌; ⿌; 黽; 黽; ) KANGXI RADICAL FROG +2FCD;2FCD;2FCD;9F0E;9F0E; # (⿍; ⿍; ⿍; 鼎; 鼎; ) KANGXI RADICAL TRIPOD +2FCE;2FCE;2FCE;9F13;9F13; # (⿎; ⿎; ⿎; 鼓; 鼓; ) KANGXI RADICAL DRUM +2FCF;2FCF;2FCF;9F20;9F20; # (⿏; ⿏; ⿏; 鼠; 鼠; ) KANGXI RADICAL RAT +2FD0;2FD0;2FD0;9F3B;9F3B; # (⿐; ⿐; ⿐; 鼻; 鼻; ) KANGXI RADICAL NOSE +2FD1;2FD1;2FD1;9F4A;9F4A; # (⿑; ⿑; ⿑; 齊; 齊; ) KANGXI RADICAL EVEN +2FD2;2FD2;2FD2;9F52;9F52; # (⿒; ⿒; ⿒; 齒; 齒; ) KANGXI RADICAL TOOTH +2FD3;2FD3;2FD3;9F8D;9F8D; # (⿓; ⿓; ⿓; 龍; 龍; ) KANGXI RADICAL DRAGON +2FD4;2FD4;2FD4;9F9C;9F9C; # (⿔; ⿔; ⿔; 龜; 龜; ) KANGXI RADICAL TURTLE +2FD5;2FD5;2FD5;9FA0;9FA0; # (⿕; ⿕; ⿕; 龠; 龠; ) KANGXI RADICAL FLUTE +3000;3000;3000;0020;0020; # ( ;  ;  ; ; ; ) IDEOGRAPHIC SPACE +3036;3036;3036;3012;3012; # (〶; 〶; 〶; 〒; 〒; ) CIRCLED POSTAL MARK +3038;3038;3038;5341;5341; # (〸; 〸; 〸; 十; 十; ) HANGZHOU NUMERAL TEN +3039;3039;3039;5344;5344; # (〹; 〹; 〹; 卄; 卄; ) HANGZHOU NUMERAL TWENTY +303A;303A;303A;5345;5345; # (〺; 〺; 〺; 卅; 卅; ) HANGZHOU NUMERAL THIRTY +304C;304C;304B 3099;304C;304B 3099; # (が; が; か◌゙; が; か◌゙; ) HIRAGANA LETTER GA +304E;304E;304D 3099;304E;304D 3099; # (ぎ; ぎ; き◌゙; ぎ; き◌゙; ) HIRAGANA LETTER GI +3050;3050;304F 3099;3050;304F 3099; # (ぐ; ぐ; く◌゙; ぐ; く◌゙; ) HIRAGANA LETTER GU +3052;3052;3051 3099;3052;3051 3099; # (げ; げ; け◌゙; げ; け◌゙; ) HIRAGANA LETTER GE +3054;3054;3053 3099;3054;3053 3099; # (ご; ご; こ◌゙; ご; こ◌゙; ) HIRAGANA LETTER GO +3056;3056;3055 3099;3056;3055 3099; # (ざ; ざ; さ◌゙; ざ; さ◌゙; ) HIRAGANA LETTER ZA +3058;3058;3057 3099;3058;3057 3099; # (じ; じ; し◌゙; じ; し◌゙; ) HIRAGANA LETTER ZI +305A;305A;3059 3099;305A;3059 3099; # (ず; ず; す◌゙; ず; す◌゙; ) HIRAGANA LETTER ZU +305C;305C;305B 3099;305C;305B 3099; # (ぜ; ぜ; せ◌゙; ぜ; せ◌゙; ) HIRAGANA LETTER ZE +305E;305E;305D 3099;305E;305D 3099; # (ぞ; ぞ; そ◌゙; ぞ; そ◌゙; ) HIRAGANA LETTER ZO +3060;3060;305F 3099;3060;305F 3099; # (だ; だ; た◌゙; だ; た◌゙; ) HIRAGANA LETTER DA +3062;3062;3061 3099;3062;3061 3099; # (ぢ; ぢ; ち◌゙; ぢ; ち◌゙; ) HIRAGANA LETTER DI +3065;3065;3064 3099;3065;3064 3099; # (づ; づ; つ◌゙; づ; つ◌゙; ) HIRAGANA LETTER DU +3067;3067;3066 3099;3067;3066 3099; # (で; で; て◌゙; で; て◌゙; ) HIRAGANA LETTER DE +3069;3069;3068 3099;3069;3068 3099; # (ど; ど; と◌゙; ど; と◌゙; ) HIRAGANA LETTER DO +3070;3070;306F 3099;3070;306F 3099; # (ば; ば; は◌゙; ば; は◌゙; ) HIRAGANA LETTER BA +3071;3071;306F 309A;3071;306F 309A; # (ぱ; ぱ; は◌゚; ぱ; は◌゚; ) HIRAGANA LETTER PA +3073;3073;3072 3099;3073;3072 3099; # (び; び; ひ◌゙; び; ひ◌゙; ) HIRAGANA LETTER BI +3074;3074;3072 309A;3074;3072 309A; # (ぴ; ぴ; ひ◌゚; ぴ; ひ◌゚; ) HIRAGANA LETTER PI +3076;3076;3075 3099;3076;3075 3099; # (ぶ; ぶ; ふ◌゙; ぶ; ふ◌゙; ) HIRAGANA LETTER BU +3077;3077;3075 309A;3077;3075 309A; # (ぷ; ぷ; ふ◌゚; ぷ; ふ◌゚; ) HIRAGANA LETTER PU +3079;3079;3078 3099;3079;3078 3099; # (べ; べ; へ◌゙; べ; へ◌゙; ) HIRAGANA LETTER BE +307A;307A;3078 309A;307A;3078 309A; # (ぺ; ぺ; へ◌゚; ぺ; へ◌゚; ) HIRAGANA LETTER PE +307C;307C;307B 3099;307C;307B 3099; # (ぼ; ぼ; ほ◌゙; ぼ; ほ◌゙; ) HIRAGANA LETTER BO +307D;307D;307B 309A;307D;307B 309A; # (ぽ; ぽ; ほ◌゚; ぽ; ほ◌゚; ) HIRAGANA LETTER PO +3094;3094;3046 3099;3094;3046 3099; # (ゔ; ゔ; う◌゙; ゔ; う◌゙; ) HIRAGANA LETTER VU +309B;309B;309B;0020 3099;0020 3099; # (゛; ゛; ゛; ◌゙; ◌゙; ) KATAKANA-HIRAGANA VOICED SOUND MARK +309C;309C;309C;0020 309A;0020 309A; # (゜; ゜; ゜; ◌゚; ◌゚; ) KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK +309E;309E;309D 3099;309E;309D 3099; # (ゞ; ゞ; ゝ◌゙; ゞ; ゝ◌゙; ) HIRAGANA VOICED ITERATION MARK +309F;309F;309F;3088 308A;3088 308A; # (ゟ; ゟ; ゟ; より; より; ) HIRAGANA DIGRAPH YORI +30AC;30AC;30AB 3099;30AC;30AB 3099; # (ガ; ガ; カ◌゙; ガ; カ◌゙; ) KATAKANA LETTER GA +30AE;30AE;30AD 3099;30AE;30AD 3099; # (ギ; ギ; キ◌゙; ギ; キ◌゙; ) KATAKANA LETTER GI +30B0;30B0;30AF 3099;30B0;30AF 3099; # (グ; グ; ク◌゙; グ; ク◌゙; ) KATAKANA LETTER GU +30B2;30B2;30B1 3099;30B2;30B1 3099; # (ゲ; ゲ; ケ◌゙; ゲ; ケ◌゙; ) KATAKANA LETTER GE +30B4;30B4;30B3 3099;30B4;30B3 3099; # (ゴ; ゴ; コ◌゙; ゴ; コ◌゙; ) KATAKANA LETTER GO +30B6;30B6;30B5 3099;30B6;30B5 3099; # (ザ; ザ; サ◌゙; ザ; サ◌゙; ) KATAKANA LETTER ZA +30B8;30B8;30B7 3099;30B8;30B7 3099; # (ジ; ジ; シ◌゙; ジ; シ◌゙; ) KATAKANA LETTER ZI +30BA;30BA;30B9 3099;30BA;30B9 3099; # (ズ; ズ; ス◌゙; ズ; ス◌゙; ) KATAKANA LETTER ZU +30BC;30BC;30BB 3099;30BC;30BB 3099; # (ゼ; ゼ; セ◌゙; ゼ; セ◌゙; ) KATAKANA LETTER ZE +30BE;30BE;30BD 3099;30BE;30BD 3099; # (ゾ; ゾ; ソ◌゙; ゾ; ソ◌゙; ) KATAKANA LETTER ZO +30C0;30C0;30BF 3099;30C0;30BF 3099; # (ダ; ダ; タ◌゙; ダ; タ◌゙; ) KATAKANA LETTER DA +30C2;30C2;30C1 3099;30C2;30C1 3099; # (ヂ; ヂ; チ◌゙; ヂ; チ◌゙; ) KATAKANA LETTER DI +30C5;30C5;30C4 3099;30C5;30C4 3099; # (ヅ; ヅ; ツ◌゙; ヅ; ツ◌゙; ) KATAKANA LETTER DU +30C7;30C7;30C6 3099;30C7;30C6 3099; # (デ; デ; テ◌゙; デ; テ◌゙; ) KATAKANA LETTER DE +30C9;30C9;30C8 3099;30C9;30C8 3099; # (ド; ド; ト◌゙; ド; ト◌゙; ) KATAKANA LETTER DO +30D0;30D0;30CF 3099;30D0;30CF 3099; # (バ; バ; ハ◌゙; バ; ハ◌゙; ) KATAKANA LETTER BA +30D1;30D1;30CF 309A;30D1;30CF 309A; # (パ; パ; ハ◌゚; パ; ハ◌゚; ) KATAKANA LETTER PA +30D3;30D3;30D2 3099;30D3;30D2 3099; # (ビ; ビ; ヒ◌゙; ビ; ヒ◌゙; ) KATAKANA LETTER BI +30D4;30D4;30D2 309A;30D4;30D2 309A; # (ピ; ピ; ヒ◌゚; ピ; ヒ◌゚; ) KATAKANA LETTER PI +30D6;30D6;30D5 3099;30D6;30D5 3099; # (ブ; ブ; フ◌゙; ブ; フ◌゙; ) KATAKANA LETTER BU +30D7;30D7;30D5 309A;30D7;30D5 309A; # (プ; プ; フ◌゚; プ; フ◌゚; ) KATAKANA LETTER PU +30D9;30D9;30D8 3099;30D9;30D8 3099; # (ベ; ベ; ヘ◌゙; ベ; ヘ◌゙; ) KATAKANA LETTER BE +30DA;30DA;30D8 309A;30DA;30D8 309A; # (ペ; ペ; ヘ◌゚; ペ; ヘ◌゚; ) KATAKANA LETTER PE +30DC;30DC;30DB 3099;30DC;30DB 3099; # (ボ; ボ; ホ◌゙; ボ; ホ◌゙; ) KATAKANA LETTER BO +30DD;30DD;30DB 309A;30DD;30DB 309A; # (ポ; ポ; ホ◌゚; ポ; ホ◌゚; ) KATAKANA LETTER PO +30F4;30F4;30A6 3099;30F4;30A6 3099; # (ヴ; ヴ; ウ◌゙; ヴ; ウ◌゙; ) KATAKANA LETTER VU +30F7;30F7;30EF 3099;30F7;30EF 3099; # (ヷ; ヷ; ワ◌゙; ヷ; ワ◌゙; ) KATAKANA LETTER VA +30F8;30F8;30F0 3099;30F8;30F0 3099; # (ヸ; ヸ; ヰ◌゙; ヸ; ヰ◌゙; ) KATAKANA LETTER VI +30F9;30F9;30F1 3099;30F9;30F1 3099; # (ヹ; ヹ; ヱ◌゙; ヹ; ヱ◌゙; ) KATAKANA LETTER VE +30FA;30FA;30F2 3099;30FA;30F2 3099; # (ヺ; ヺ; ヲ◌゙; ヺ; ヲ◌゙; ) KATAKANA LETTER VO +30FE;30FE;30FD 3099;30FE;30FD 3099; # (ヾ; ヾ; ヽ◌゙; ヾ; ヽ◌゙; ) KATAKANA VOICED ITERATION MARK +30FF;30FF;30FF;30B3 30C8;30B3 30C8; # (ヿ; ヿ; ヿ; コト; コト; ) KATAKANA DIGRAPH KOTO +3131;3131;3131;1100;1100; # (ㄱ; ㄱ; ㄱ; ᄀ; ᄀ; ) HANGUL LETTER KIYEOK +3132;3132;3132;1101;1101; # (ㄲ; ㄲ; ㄲ; ᄁ; ᄁ; ) HANGUL LETTER SSANGKIYEOK +3133;3133;3133;11AA;11AA; # (ㄳ; ㄳ; ㄳ; ᆪ; ᆪ; ) HANGUL LETTER KIYEOK-SIOS +3134;3134;3134;1102;1102; # (ㄴ; ㄴ; ㄴ; ᄂ; ᄂ; ) HANGUL LETTER NIEUN +3135;3135;3135;11AC;11AC; # (ㄵ; ㄵ; ㄵ; ᆬ; ᆬ; ) HANGUL LETTER NIEUN-CIEUC +3136;3136;3136;11AD;11AD; # (ㄶ; ㄶ; ㄶ; ᆭ; ᆭ; ) HANGUL LETTER NIEUN-HIEUH +3137;3137;3137;1103;1103; # (ㄷ; ㄷ; ㄷ; ᄃ; ᄃ; ) HANGUL LETTER TIKEUT +3138;3138;3138;1104;1104; # (ㄸ; ㄸ; ㄸ; ᄄ; ᄄ; ) HANGUL LETTER SSANGTIKEUT +3139;3139;3139;1105;1105; # (ㄹ; ㄹ; ㄹ; ᄅ; ᄅ; ) HANGUL LETTER RIEUL +313A;313A;313A;11B0;11B0; # (ㄺ; ㄺ; ㄺ; ᆰ; ᆰ; ) HANGUL LETTER RIEUL-KIYEOK +313B;313B;313B;11B1;11B1; # (ㄻ; ㄻ; ㄻ; ᆱ; ᆱ; ) HANGUL LETTER RIEUL-MIEUM +313C;313C;313C;11B2;11B2; # (ㄼ; ㄼ; ㄼ; ᆲ; ᆲ; ) HANGUL LETTER RIEUL-PIEUP +313D;313D;313D;11B3;11B3; # (ㄽ; ㄽ; ㄽ; ᆳ; ᆳ; ) HANGUL LETTER RIEUL-SIOS +313E;313E;313E;11B4;11B4; # (ㄾ; ㄾ; ㄾ; ᆴ; ᆴ; ) HANGUL LETTER RIEUL-THIEUTH +313F;313F;313F;11B5;11B5; # (ㄿ; ㄿ; ㄿ; ᆵ; ᆵ; ) HANGUL LETTER RIEUL-PHIEUPH +3140;3140;3140;111A;111A; # (ㅀ; ㅀ; ㅀ; ᄚ; ᄚ; ) HANGUL LETTER RIEUL-HIEUH +3141;3141;3141;1106;1106; # (ㅁ; ㅁ; ㅁ; ᄆ; ᄆ; ) HANGUL LETTER MIEUM +3142;3142;3142;1107;1107; # (ㅂ; ㅂ; ㅂ; ᄇ; ᄇ; ) HANGUL LETTER PIEUP +3143;3143;3143;1108;1108; # (ㅃ; ㅃ; ㅃ; ᄈ; ᄈ; ) HANGUL LETTER SSANGPIEUP +3144;3144;3144;1121;1121; # (ㅄ; ㅄ; ㅄ; ᄡ; ᄡ; ) HANGUL LETTER PIEUP-SIOS +3145;3145;3145;1109;1109; # (ㅅ; ㅅ; ㅅ; ᄉ; ᄉ; ) HANGUL LETTER SIOS +3146;3146;3146;110A;110A; # (ㅆ; ㅆ; ㅆ; ᄊ; ᄊ; ) HANGUL LETTER SSANGSIOS +3147;3147;3147;110B;110B; # (ㅇ; ㅇ; ㅇ; ᄋ; ᄋ; ) HANGUL LETTER IEUNG +3148;3148;3148;110C;110C; # (ㅈ; ㅈ; ㅈ; ᄌ; ᄌ; ) HANGUL LETTER CIEUC +3149;3149;3149;110D;110D; # (ㅉ; ㅉ; ㅉ; ᄍ; ᄍ; ) HANGUL LETTER SSANGCIEUC +314A;314A;314A;110E;110E; # (ㅊ; ㅊ; ㅊ; ᄎ; ᄎ; ) HANGUL LETTER CHIEUCH +314B;314B;314B;110F;110F; # (ㅋ; ㅋ; ㅋ; ᄏ; ᄏ; ) HANGUL LETTER KHIEUKH +314C;314C;314C;1110;1110; # (ㅌ; ㅌ; ㅌ; ᄐ; ᄐ; ) HANGUL LETTER THIEUTH +314D;314D;314D;1111;1111; # (ㅍ; ㅍ; ㅍ; ᄑ; ᄑ; ) HANGUL LETTER PHIEUPH +314E;314E;314E;1112;1112; # (ㅎ; ㅎ; ㅎ; ᄒ; ᄒ; ) HANGUL LETTER HIEUH +314F;314F;314F;1161;1161; # (ㅏ; ㅏ; ㅏ; ᅡ; ᅡ; ) HANGUL LETTER A +3150;3150;3150;1162;1162; # (ㅐ; ㅐ; ㅐ; ᅢ; ᅢ; ) HANGUL LETTER AE +3151;3151;3151;1163;1163; # (ㅑ; ㅑ; ㅑ; ᅣ; ᅣ; ) HANGUL LETTER YA +3152;3152;3152;1164;1164; # (ㅒ; ㅒ; ㅒ; ᅤ; ᅤ; ) HANGUL LETTER YAE +3153;3153;3153;1165;1165; # (ㅓ; ㅓ; ㅓ; ᅥ; ᅥ; ) HANGUL LETTER EO +3154;3154;3154;1166;1166; # (ㅔ; ㅔ; ㅔ; ᅦ; ᅦ; ) HANGUL LETTER E +3155;3155;3155;1167;1167; # (ㅕ; ㅕ; ㅕ; ᅧ; ᅧ; ) HANGUL LETTER YEO +3156;3156;3156;1168;1168; # (ㅖ; ㅖ; ㅖ; ᅨ; ᅨ; ) HANGUL LETTER YE +3157;3157;3157;1169;1169; # (ㅗ; ㅗ; ㅗ; ᅩ; ᅩ; ) HANGUL LETTER O +3158;3158;3158;116A;116A; # (ㅘ; ㅘ; ㅘ; ᅪ; ᅪ; ) HANGUL LETTER WA +3159;3159;3159;116B;116B; # (ㅙ; ㅙ; ㅙ; ᅫ; ᅫ; ) HANGUL LETTER WAE +315A;315A;315A;116C;116C; # (ㅚ; ㅚ; ㅚ; ᅬ; ᅬ; ) HANGUL LETTER OE +315B;315B;315B;116D;116D; # (ㅛ; ㅛ; ㅛ; ᅭ; ᅭ; ) HANGUL LETTER YO +315C;315C;315C;116E;116E; # (ㅜ; ㅜ; ㅜ; ᅮ; ᅮ; ) HANGUL LETTER U +315D;315D;315D;116F;116F; # (ㅝ; ㅝ; ㅝ; ᅯ; ᅯ; ) HANGUL LETTER WEO +315E;315E;315E;1170;1170; # (ㅞ; ㅞ; ㅞ; ᅰ; ᅰ; ) HANGUL LETTER WE +315F;315F;315F;1171;1171; # (ㅟ; ㅟ; ㅟ; ᅱ; ᅱ; ) HANGUL LETTER WI +3160;3160;3160;1172;1172; # (ㅠ; ㅠ; ㅠ; ᅲ; ᅲ; ) HANGUL LETTER YU +3161;3161;3161;1173;1173; # (ㅡ; ㅡ; ㅡ; ᅳ; ᅳ; ) HANGUL LETTER EU +3162;3162;3162;1174;1174; # (ㅢ; ㅢ; ㅢ; ᅴ; ᅴ; ) HANGUL LETTER YI +3163;3163;3163;1175;1175; # (ㅣ; ㅣ; ㅣ; ᅵ; ᅵ; ) HANGUL LETTER I +3164;3164;3164;1160;1160; # (ㅤ; ㅤ; ㅤ; ᅠ; ᅠ; ) HANGUL FILLER +3165;3165;3165;1114;1114; # (ㅥ; ㅥ; ㅥ; ᄔ; ᄔ; ) HANGUL LETTER SSANGNIEUN +3166;3166;3166;1115;1115; # (ㅦ; ㅦ; ㅦ; ᄕ; ᄕ; ) HANGUL LETTER NIEUN-TIKEUT +3167;3167;3167;11C7;11C7; # (ㅧ; ㅧ; ㅧ; ᇇ; ᇇ; ) HANGUL LETTER NIEUN-SIOS +3168;3168;3168;11C8;11C8; # (ㅨ; ㅨ; ㅨ; ᇈ; ᇈ; ) HANGUL LETTER NIEUN-PANSIOS +3169;3169;3169;11CC;11CC; # (ㅩ; ㅩ; ㅩ; ᇌ; ᇌ; ) HANGUL LETTER RIEUL-KIYEOK-SIOS +316A;316A;316A;11CE;11CE; # (ㅪ; ㅪ; ㅪ; ᇎ; ᇎ; ) HANGUL LETTER RIEUL-TIKEUT +316B;316B;316B;11D3;11D3; # (ㅫ; ㅫ; ㅫ; ᇓ; ᇓ; ) HANGUL LETTER RIEUL-PIEUP-SIOS +316C;316C;316C;11D7;11D7; # (ㅬ; ㅬ; ㅬ; ᇗ; ᇗ; ) HANGUL LETTER RIEUL-PANSIOS +316D;316D;316D;11D9;11D9; # (ㅭ; ㅭ; ㅭ; ᇙ; ᇙ; ) HANGUL LETTER RIEUL-YEORINHIEUH +316E;316E;316E;111C;111C; # (ㅮ; ㅮ; ㅮ; ᄜ; ᄜ; ) HANGUL LETTER MIEUM-PIEUP +316F;316F;316F;11DD;11DD; # (ㅯ; ㅯ; ㅯ; ᇝ; ᇝ; ) HANGUL LETTER MIEUM-SIOS +3170;3170;3170;11DF;11DF; # (ㅰ; ㅰ; ㅰ; ᇟ; ᇟ; ) HANGUL LETTER MIEUM-PANSIOS +3171;3171;3171;111D;111D; # (ㅱ; ㅱ; ㅱ; ᄝ; ᄝ; ) HANGUL LETTER KAPYEOUNMIEUM +3172;3172;3172;111E;111E; # (ㅲ; ㅲ; ㅲ; ᄞ; ᄞ; ) HANGUL LETTER PIEUP-KIYEOK +3173;3173;3173;1120;1120; # (ㅳ; ㅳ; ㅳ; ᄠ; ᄠ; ) HANGUL LETTER PIEUP-TIKEUT +3174;3174;3174;1122;1122; # (ㅴ; ㅴ; ㅴ; ᄢ; ᄢ; ) HANGUL LETTER PIEUP-SIOS-KIYEOK +3175;3175;3175;1123;1123; # (ㅵ; ㅵ; ㅵ; ᄣ; ᄣ; ) HANGUL LETTER PIEUP-SIOS-TIKEUT +3176;3176;3176;1127;1127; # (ㅶ; ㅶ; ㅶ; ᄧ; ᄧ; ) HANGUL LETTER PIEUP-CIEUC +3177;3177;3177;1129;1129; # (ㅷ; ㅷ; ㅷ; ᄩ; ᄩ; ) HANGUL LETTER PIEUP-THIEUTH +3178;3178;3178;112B;112B; # (ㅸ; ㅸ; ㅸ; ᄫ; ᄫ; ) HANGUL LETTER KAPYEOUNPIEUP +3179;3179;3179;112C;112C; # (ㅹ; ㅹ; ㅹ; ᄬ; ᄬ; ) HANGUL LETTER KAPYEOUNSSANGPIEUP +317A;317A;317A;112D;112D; # (ㅺ; ㅺ; ㅺ; ᄭ; ᄭ; ) HANGUL LETTER SIOS-KIYEOK +317B;317B;317B;112E;112E; # (ㅻ; ㅻ; ㅻ; ᄮ; ᄮ; ) HANGUL LETTER SIOS-NIEUN +317C;317C;317C;112F;112F; # (ㅼ; ㅼ; ㅼ; ᄯ; ᄯ; ) HANGUL LETTER SIOS-TIKEUT +317D;317D;317D;1132;1132; # (ㅽ; ㅽ; ㅽ; ᄲ; ᄲ; ) HANGUL LETTER SIOS-PIEUP +317E;317E;317E;1136;1136; # (ㅾ; ㅾ; ㅾ; ᄶ; ᄶ; ) HANGUL LETTER SIOS-CIEUC +317F;317F;317F;1140;1140; # (ㅿ; ㅿ; ㅿ; ᅀ; ᅀ; ) HANGUL LETTER PANSIOS +3180;3180;3180;1147;1147; # (ㆀ; ㆀ; ㆀ; ᅇ; ᅇ; ) HANGUL LETTER SSANGIEUNG +3181;3181;3181;114C;114C; # (ㆁ; ㆁ; ㆁ; ᅌ; ᅌ; ) HANGUL LETTER YESIEUNG +3182;3182;3182;11F1;11F1; # (ㆂ; ㆂ; ㆂ; ᇱ; ᇱ; ) HANGUL LETTER YESIEUNG-SIOS +3183;3183;3183;11F2;11F2; # (ㆃ; ㆃ; ㆃ; ᇲ; ᇲ; ) HANGUL LETTER YESIEUNG-PANSIOS +3184;3184;3184;1157;1157; # (ㆄ; ㆄ; ㆄ; ᅗ; ᅗ; ) HANGUL LETTER KAPYEOUNPHIEUPH +3185;3185;3185;1158;1158; # (ㆅ; ㆅ; ㆅ; ᅘ; ᅘ; ) HANGUL LETTER SSANGHIEUH +3186;3186;3186;1159;1159; # (ㆆ; ㆆ; ㆆ; ᅙ; ᅙ; ) HANGUL LETTER YEORINHIEUH +3187;3187;3187;1184;1184; # (ㆇ; ㆇ; ㆇ; ᆄ; ᆄ; ) HANGUL LETTER YO-YA +3188;3188;3188;1185;1185; # (ㆈ; ㆈ; ㆈ; ᆅ; ᆅ; ) HANGUL LETTER YO-YAE +3189;3189;3189;1188;1188; # (ㆉ; ㆉ; ㆉ; ᆈ; ᆈ; ) HANGUL LETTER YO-I +318A;318A;318A;1191;1191; # (ㆊ; ㆊ; ㆊ; ᆑ; ᆑ; ) HANGUL LETTER YU-YEO +318B;318B;318B;1192;1192; # (ㆋ; ㆋ; ㆋ; ᆒ; ᆒ; ) HANGUL LETTER YU-YE +318C;318C;318C;1194;1194; # (ㆌ; ㆌ; ㆌ; ᆔ; ᆔ; ) HANGUL LETTER YU-I +318D;318D;318D;119E;119E; # (ㆍ; ㆍ; ㆍ; ᆞ; ᆞ; ) HANGUL LETTER ARAEA +318E;318E;318E;11A1;11A1; # (ㆎ; ㆎ; ㆎ; ᆡ; ᆡ; ) HANGUL LETTER ARAEAE +3192;3192;3192;4E00;4E00; # (㆒; ㆒; ㆒; 一; 一; ) IDEOGRAPHIC ANNOTATION ONE MARK +3193;3193;3193;4E8C;4E8C; # (㆓; ㆓; ㆓; 二; 二; ) IDEOGRAPHIC ANNOTATION TWO MARK +3194;3194;3194;4E09;4E09; # (㆔; ㆔; ㆔; 三; 三; ) IDEOGRAPHIC ANNOTATION THREE MARK +3195;3195;3195;56DB;56DB; # (㆕; ㆕; ㆕; 四; 四; ) IDEOGRAPHIC ANNOTATION FOUR MARK +3196;3196;3196;4E0A;4E0A; # (㆖; ㆖; ㆖; 上; 上; ) IDEOGRAPHIC ANNOTATION TOP MARK +3197;3197;3197;4E2D;4E2D; # (㆗; ㆗; ㆗; 中; 中; ) IDEOGRAPHIC ANNOTATION MIDDLE MARK +3198;3198;3198;4E0B;4E0B; # (㆘; ㆘; ㆘; 下; 下; ) IDEOGRAPHIC ANNOTATION BOTTOM MARK +3199;3199;3199;7532;7532; # (㆙; ㆙; ㆙; 甲; 甲; ) IDEOGRAPHIC ANNOTATION FIRST MARK +319A;319A;319A;4E59;4E59; # (㆚; ㆚; ㆚; 乙; 乙; ) IDEOGRAPHIC ANNOTATION SECOND MARK +319B;319B;319B;4E19;4E19; # (㆛; ㆛; ㆛; 丙; 丙; ) IDEOGRAPHIC ANNOTATION THIRD MARK +319C;319C;319C;4E01;4E01; # (㆜; ㆜; ㆜; 丁; 丁; ) IDEOGRAPHIC ANNOTATION FOURTH MARK +319D;319D;319D;5929;5929; # (㆝; ㆝; ㆝; 天; 天; ) IDEOGRAPHIC ANNOTATION HEAVEN MARK +319E;319E;319E;5730;5730; # (㆞; ㆞; ㆞; 地; 地; ) IDEOGRAPHIC ANNOTATION EARTH MARK +319F;319F;319F;4EBA;4EBA; # (㆟; ㆟; ㆟; 人; 人; ) IDEOGRAPHIC ANNOTATION MAN MARK +3200;3200;3200;0028 1100 0029;0028 1100 0029; # (㈀; ㈀; ㈀; (ᄀ); (ᄀ); ) PARENTHESIZED HANGUL KIYEOK +3201;3201;3201;0028 1102 0029;0028 1102 0029; # (㈁; ㈁; ㈁; (ᄂ); (ᄂ); ) PARENTHESIZED HANGUL NIEUN +3202;3202;3202;0028 1103 0029;0028 1103 0029; # (㈂; ㈂; ㈂; (ᄃ); (ᄃ); ) PARENTHESIZED HANGUL TIKEUT +3203;3203;3203;0028 1105 0029;0028 1105 0029; # (㈃; ㈃; ㈃; (ᄅ); (ᄅ); ) PARENTHESIZED HANGUL RIEUL +3204;3204;3204;0028 1106 0029;0028 1106 0029; # (㈄; ㈄; ㈄; (ᄆ); (ᄆ); ) PARENTHESIZED HANGUL MIEUM +3205;3205;3205;0028 1107 0029;0028 1107 0029; # (㈅; ㈅; ㈅; (ᄇ); (ᄇ); ) PARENTHESIZED HANGUL PIEUP +3206;3206;3206;0028 1109 0029;0028 1109 0029; # (㈆; ㈆; ㈆; (ᄉ); (ᄉ); ) PARENTHESIZED HANGUL SIOS +3207;3207;3207;0028 110B 0029;0028 110B 0029; # (㈇; ㈇; ㈇; (ᄋ); (ᄋ); ) PARENTHESIZED HANGUL IEUNG +3208;3208;3208;0028 110C 0029;0028 110C 0029; # (㈈; ㈈; ㈈; (ᄌ); (ᄌ); ) PARENTHESIZED HANGUL CIEUC +3209;3209;3209;0028 110E 0029;0028 110E 0029; # (㈉; ㈉; ㈉; (ᄎ); (ᄎ); ) PARENTHESIZED HANGUL CHIEUCH +320A;320A;320A;0028 110F 0029;0028 110F 0029; # (㈊; ㈊; ㈊; (ᄏ); (ᄏ); ) PARENTHESIZED HANGUL KHIEUKH +320B;320B;320B;0028 1110 0029;0028 1110 0029; # (㈋; ㈋; ㈋; (ᄐ); (ᄐ); ) PARENTHESIZED HANGUL THIEUTH +320C;320C;320C;0028 1111 0029;0028 1111 0029; # (㈌; ㈌; ㈌; (ᄑ); (ᄑ); ) PARENTHESIZED HANGUL PHIEUPH +320D;320D;320D;0028 1112 0029;0028 1112 0029; # (㈍; ㈍; ㈍; (ᄒ); (ᄒ); ) PARENTHESIZED HANGUL HIEUH +320E;320E;320E;0028 AC00 0029;0028 1100 1161 0029; # (㈎; ㈎; ㈎; (가); (가); ) PARENTHESIZED HANGUL KIYEOK A +320F;320F;320F;0028 B098 0029;0028 1102 1161 0029; # (㈏; ㈏; ㈏; (나); (나); ) PARENTHESIZED HANGUL NIEUN A +3210;3210;3210;0028 B2E4 0029;0028 1103 1161 0029; # (㈐; ㈐; ㈐; (다); (다); ) PARENTHESIZED HANGUL TIKEUT A +3211;3211;3211;0028 B77C 0029;0028 1105 1161 0029; # (㈑; ㈑; ㈑; (라); (라); ) PARENTHESIZED HANGUL RIEUL A +3212;3212;3212;0028 B9C8 0029;0028 1106 1161 0029; # (㈒; ㈒; ㈒; (마); (마); ) PARENTHESIZED HANGUL MIEUM A +3213;3213;3213;0028 BC14 0029;0028 1107 1161 0029; # (㈓; ㈓; ㈓; (바); (바); ) PARENTHESIZED HANGUL PIEUP A +3214;3214;3214;0028 C0AC 0029;0028 1109 1161 0029; # (㈔; ㈔; ㈔; (사); (사); ) PARENTHESIZED HANGUL SIOS A +3215;3215;3215;0028 C544 0029;0028 110B 1161 0029; # (㈕; ㈕; ㈕; (아); (아); ) PARENTHESIZED HANGUL IEUNG A +3216;3216;3216;0028 C790 0029;0028 110C 1161 0029; # (㈖; ㈖; ㈖; (자); (자); ) PARENTHESIZED HANGUL CIEUC A +3217;3217;3217;0028 CC28 0029;0028 110E 1161 0029; # (㈗; ㈗; ㈗; (차); (차); ) PARENTHESIZED HANGUL CHIEUCH A +3218;3218;3218;0028 CE74 0029;0028 110F 1161 0029; # (㈘; ㈘; ㈘; (카); (카); ) PARENTHESIZED HANGUL KHIEUKH A +3219;3219;3219;0028 D0C0 0029;0028 1110 1161 0029; # (㈙; ㈙; ㈙; (타); (타); ) PARENTHESIZED HANGUL THIEUTH A +321A;321A;321A;0028 D30C 0029;0028 1111 1161 0029; # (㈚; ㈚; ㈚; (파); (파); ) PARENTHESIZED HANGUL PHIEUPH A +321B;321B;321B;0028 D558 0029;0028 1112 1161 0029; # (㈛; ㈛; ㈛; (하); (하); ) PARENTHESIZED HANGUL HIEUH A +321C;321C;321C;0028 C8FC 0029;0028 110C 116E 0029; # (㈜; ㈜; ㈜; (주); (주); ) PARENTHESIZED HANGUL CIEUC U +3220;3220;3220;0028 4E00 0029;0028 4E00 0029; # (㈠; ㈠; ㈠; (一); (一); ) PARENTHESIZED IDEOGRAPH ONE +3221;3221;3221;0028 4E8C 0029;0028 4E8C 0029; # (㈡; ㈡; ㈡; (二); (二); ) PARENTHESIZED IDEOGRAPH TWO +3222;3222;3222;0028 4E09 0029;0028 4E09 0029; # (㈢; ㈢; ㈢; (三); (三); ) PARENTHESIZED IDEOGRAPH THREE +3223;3223;3223;0028 56DB 0029;0028 56DB 0029; # (㈣; ㈣; ㈣; (四); (四); ) PARENTHESIZED IDEOGRAPH FOUR +3224;3224;3224;0028 4E94 0029;0028 4E94 0029; # (㈤; ㈤; ㈤; (五); (五); ) PARENTHESIZED IDEOGRAPH FIVE +3225;3225;3225;0028 516D 0029;0028 516D 0029; # (㈥; ㈥; ㈥; (六); (六); ) PARENTHESIZED IDEOGRAPH SIX +3226;3226;3226;0028 4E03 0029;0028 4E03 0029; # (㈦; ㈦; ㈦; (七); (七); ) PARENTHESIZED IDEOGRAPH SEVEN +3227;3227;3227;0028 516B 0029;0028 516B 0029; # (㈧; ㈧; ㈧; (八); (八); ) PARENTHESIZED IDEOGRAPH EIGHT +3228;3228;3228;0028 4E5D 0029;0028 4E5D 0029; # (㈨; ㈨; ㈨; (九); (九); ) PARENTHESIZED IDEOGRAPH NINE +3229;3229;3229;0028 5341 0029;0028 5341 0029; # (㈩; ㈩; ㈩; (十); (十); ) PARENTHESIZED IDEOGRAPH TEN +322A;322A;322A;0028 6708 0029;0028 6708 0029; # (㈪; ㈪; ㈪; (月); (月); ) PARENTHESIZED IDEOGRAPH MOON +322B;322B;322B;0028 706B 0029;0028 706B 0029; # (㈫; ㈫; ㈫; (火); (火); ) PARENTHESIZED IDEOGRAPH FIRE +322C;322C;322C;0028 6C34 0029;0028 6C34 0029; # (㈬; ㈬; ㈬; (水); (水); ) PARENTHESIZED IDEOGRAPH WATER +322D;322D;322D;0028 6728 0029;0028 6728 0029; # (㈭; ㈭; ㈭; (木); (木); ) PARENTHESIZED IDEOGRAPH WOOD +322E;322E;322E;0028 91D1 0029;0028 91D1 0029; # (㈮; ㈮; ㈮; (金); (金); ) PARENTHESIZED IDEOGRAPH METAL +322F;322F;322F;0028 571F 0029;0028 571F 0029; # (㈯; ㈯; ㈯; (土); (土); ) PARENTHESIZED IDEOGRAPH EARTH +3230;3230;3230;0028 65E5 0029;0028 65E5 0029; # (㈰; ㈰; ㈰; (日); (日); ) PARENTHESIZED IDEOGRAPH SUN +3231;3231;3231;0028 682A 0029;0028 682A 0029; # (㈱; ㈱; ㈱; (株); (株); ) PARENTHESIZED IDEOGRAPH STOCK +3232;3232;3232;0028 6709 0029;0028 6709 0029; # (㈲; ㈲; ㈲; (有); (有); ) PARENTHESIZED IDEOGRAPH HAVE +3233;3233;3233;0028 793E 0029;0028 793E 0029; # (㈳; ㈳; ㈳; (社); (社); ) PARENTHESIZED IDEOGRAPH SOCIETY +3234;3234;3234;0028 540D 0029;0028 540D 0029; # (㈴; ㈴; ㈴; (名); (名); ) PARENTHESIZED IDEOGRAPH NAME +3235;3235;3235;0028 7279 0029;0028 7279 0029; # (㈵; ㈵; ㈵; (特); (特); ) PARENTHESIZED IDEOGRAPH SPECIAL +3236;3236;3236;0028 8CA1 0029;0028 8CA1 0029; # (㈶; ㈶; ㈶; (財); (財); ) PARENTHESIZED IDEOGRAPH FINANCIAL +3237;3237;3237;0028 795D 0029;0028 795D 0029; # (㈷; ㈷; ㈷; (祝); (祝); ) PARENTHESIZED IDEOGRAPH CONGRATULATION +3238;3238;3238;0028 52B4 0029;0028 52B4 0029; # (㈸; ㈸; ㈸; (労); (労); ) PARENTHESIZED IDEOGRAPH LABOR +3239;3239;3239;0028 4EE3 0029;0028 4EE3 0029; # (㈹; ㈹; ㈹; (代); (代); ) PARENTHESIZED IDEOGRAPH REPRESENT +323A;323A;323A;0028 547C 0029;0028 547C 0029; # (㈺; ㈺; ㈺; (呼); (呼); ) PARENTHESIZED IDEOGRAPH CALL +323B;323B;323B;0028 5B66 0029;0028 5B66 0029; # (㈻; ㈻; ㈻; (学); (学); ) PARENTHESIZED IDEOGRAPH STUDY +323C;323C;323C;0028 76E3 0029;0028 76E3 0029; # (㈼; ㈼; ㈼; (監); (監); ) PARENTHESIZED IDEOGRAPH SUPERVISE +323D;323D;323D;0028 4F01 0029;0028 4F01 0029; # (㈽; ㈽; ㈽; (企); (企); ) PARENTHESIZED IDEOGRAPH ENTERPRISE +323E;323E;323E;0028 8CC7 0029;0028 8CC7 0029; # (㈾; ㈾; ㈾; (資); (資); ) PARENTHESIZED IDEOGRAPH RESOURCE +323F;323F;323F;0028 5354 0029;0028 5354 0029; # (㈿; ㈿; ㈿; (協); (協); ) PARENTHESIZED IDEOGRAPH ALLIANCE +3240;3240;3240;0028 796D 0029;0028 796D 0029; # (㉀; ㉀; ㉀; (祭); (祭); ) PARENTHESIZED IDEOGRAPH FESTIVAL +3241;3241;3241;0028 4F11 0029;0028 4F11 0029; # (㉁; ㉁; ㉁; (休); (休); ) PARENTHESIZED IDEOGRAPH REST +3242;3242;3242;0028 81EA 0029;0028 81EA 0029; # (㉂; ㉂; ㉂; (自); (自); ) PARENTHESIZED IDEOGRAPH SELF +3243;3243;3243;0028 81F3 0029;0028 81F3 0029; # (㉃; ㉃; ㉃; (至); (至); ) PARENTHESIZED IDEOGRAPH REACH +3251;3251;3251;0032 0031;0032 0031; # (㉑; ㉑; ㉑; 21; 21; ) CIRCLED NUMBER TWENTY ONE +3252;3252;3252;0032 0032;0032 0032; # (㉒; ㉒; ㉒; 22; 22; ) CIRCLED NUMBER TWENTY TWO +3253;3253;3253;0032 0033;0032 0033; # (㉓; ㉓; ㉓; 23; 23; ) CIRCLED NUMBER TWENTY THREE +3254;3254;3254;0032 0034;0032 0034; # (㉔; ㉔; ㉔; 24; 24; ) CIRCLED NUMBER TWENTY FOUR +3255;3255;3255;0032 0035;0032 0035; # (㉕; ㉕; ㉕; 25; 25; ) CIRCLED NUMBER TWENTY FIVE +3256;3256;3256;0032 0036;0032 0036; # (㉖; ㉖; ㉖; 26; 26; ) CIRCLED NUMBER TWENTY SIX +3257;3257;3257;0032 0037;0032 0037; # (㉗; ㉗; ㉗; 27; 27; ) CIRCLED NUMBER TWENTY SEVEN +3258;3258;3258;0032 0038;0032 0038; # (㉘; ㉘; ㉘; 28; 28; ) CIRCLED NUMBER TWENTY EIGHT +3259;3259;3259;0032 0039;0032 0039; # (㉙; ㉙; ㉙; 29; 29; ) CIRCLED NUMBER TWENTY NINE +325A;325A;325A;0033 0030;0033 0030; # (㉚; ㉚; ㉚; 30; 30; ) CIRCLED NUMBER THIRTY +325B;325B;325B;0033 0031;0033 0031; # (㉛; ㉛; ㉛; 31; 31; ) CIRCLED NUMBER THIRTY ONE +325C;325C;325C;0033 0032;0033 0032; # (㉜; ㉜; ㉜; 32; 32; ) CIRCLED NUMBER THIRTY TWO +325D;325D;325D;0033 0033;0033 0033; # (㉝; ㉝; ㉝; 33; 33; ) CIRCLED NUMBER THIRTY THREE +325E;325E;325E;0033 0034;0033 0034; # (㉞; ㉞; ㉞; 34; 34; ) CIRCLED NUMBER THIRTY FOUR +325F;325F;325F;0033 0035;0033 0035; # (㉟; ㉟; ㉟; 35; 35; ) CIRCLED NUMBER THIRTY FIVE +3260;3260;3260;1100;1100; # (㉠; ㉠; ㉠; ᄀ; ᄀ; ) CIRCLED HANGUL KIYEOK +3261;3261;3261;1102;1102; # (㉡; ㉡; ㉡; ᄂ; ᄂ; ) CIRCLED HANGUL NIEUN +3262;3262;3262;1103;1103; # (㉢; ㉢; ㉢; ᄃ; ᄃ; ) CIRCLED HANGUL TIKEUT +3263;3263;3263;1105;1105; # (㉣; ㉣; ㉣; ᄅ; ᄅ; ) CIRCLED HANGUL RIEUL +3264;3264;3264;1106;1106; # (㉤; ㉤; ㉤; ᄆ; ᄆ; ) CIRCLED HANGUL MIEUM +3265;3265;3265;1107;1107; # (㉥; ㉥; ㉥; ᄇ; ᄇ; ) CIRCLED HANGUL PIEUP +3266;3266;3266;1109;1109; # (㉦; ㉦; ㉦; ᄉ; ᄉ; ) CIRCLED HANGUL SIOS +3267;3267;3267;110B;110B; # (㉧; ㉧; ㉧; ᄋ; ᄋ; ) CIRCLED HANGUL IEUNG +3268;3268;3268;110C;110C; # (㉨; ㉨; ㉨; ᄌ; ᄌ; ) CIRCLED HANGUL CIEUC +3269;3269;3269;110E;110E; # (㉩; ㉩; ㉩; ᄎ; ᄎ; ) CIRCLED HANGUL CHIEUCH +326A;326A;326A;110F;110F; # (㉪; ㉪; ㉪; ᄏ; ᄏ; ) CIRCLED HANGUL KHIEUKH +326B;326B;326B;1110;1110; # (㉫; ㉫; ㉫; ᄐ; ᄐ; ) CIRCLED HANGUL THIEUTH +326C;326C;326C;1111;1111; # (㉬; ㉬; ㉬; ᄑ; ᄑ; ) CIRCLED HANGUL PHIEUPH +326D;326D;326D;1112;1112; # (㉭; ㉭; ㉭; ᄒ; ᄒ; ) CIRCLED HANGUL HIEUH +326E;326E;326E;AC00;1100 1161; # (㉮; ㉮; ㉮; 가; 가; ) CIRCLED HANGUL KIYEOK A +326F;326F;326F;B098;1102 1161; # (㉯; ㉯; ㉯; 나; 나; ) CIRCLED HANGUL NIEUN A +3270;3270;3270;B2E4;1103 1161; # (㉰; ㉰; ㉰; 다; 다; ) CIRCLED HANGUL TIKEUT A +3271;3271;3271;B77C;1105 1161; # (㉱; ㉱; ㉱; 라; 라; ) CIRCLED HANGUL RIEUL A +3272;3272;3272;B9C8;1106 1161; # (㉲; ㉲; ㉲; 마; 마; ) CIRCLED HANGUL MIEUM A +3273;3273;3273;BC14;1107 1161; # (㉳; ㉳; ㉳; 바; 바; ) CIRCLED HANGUL PIEUP A +3274;3274;3274;C0AC;1109 1161; # (㉴; ㉴; ㉴; 사; 사; ) CIRCLED HANGUL SIOS A +3275;3275;3275;C544;110B 1161; # (㉵; ㉵; ㉵; 아; 아; ) CIRCLED HANGUL IEUNG A +3276;3276;3276;C790;110C 1161; # (㉶; ㉶; ㉶; 자; 자; ) CIRCLED HANGUL CIEUC A +3277;3277;3277;CC28;110E 1161; # (㉷; ㉷; ㉷; 차; 차; ) CIRCLED HANGUL CHIEUCH A +3278;3278;3278;CE74;110F 1161; # (㉸; ㉸; ㉸; 카; 카; ) CIRCLED HANGUL KHIEUKH A +3279;3279;3279;D0C0;1110 1161; # (㉹; ㉹; ㉹; 타; 타; ) CIRCLED HANGUL THIEUTH A +327A;327A;327A;D30C;1111 1161; # (㉺; ㉺; ㉺; 파; 파; ) CIRCLED HANGUL PHIEUPH A +327B;327B;327B;D558;1112 1161; # (㉻; ㉻; ㉻; 하; 하; ) CIRCLED HANGUL HIEUH A +3280;3280;3280;4E00;4E00; # (㊀; ㊀; ㊀; 一; 一; ) CIRCLED IDEOGRAPH ONE +3281;3281;3281;4E8C;4E8C; # (㊁; ㊁; ㊁; 二; 二; ) CIRCLED IDEOGRAPH TWO +3282;3282;3282;4E09;4E09; # (㊂; ㊂; ㊂; 三; 三; ) CIRCLED IDEOGRAPH THREE +3283;3283;3283;56DB;56DB; # (㊃; ㊃; ㊃; 四; 四; ) CIRCLED IDEOGRAPH FOUR +3284;3284;3284;4E94;4E94; # (㊄; ㊄; ㊄; 五; 五; ) CIRCLED IDEOGRAPH FIVE +3285;3285;3285;516D;516D; # (㊅; ㊅; ㊅; 六; 六; ) CIRCLED IDEOGRAPH SIX +3286;3286;3286;4E03;4E03; # (㊆; ㊆; ㊆; 七; 七; ) CIRCLED IDEOGRAPH SEVEN +3287;3287;3287;516B;516B; # (㊇; ㊇; ㊇; 八; 八; ) CIRCLED IDEOGRAPH EIGHT +3288;3288;3288;4E5D;4E5D; # (㊈; ㊈; ㊈; 九; 九; ) CIRCLED IDEOGRAPH NINE +3289;3289;3289;5341;5341; # (㊉; ㊉; ㊉; 十; 十; ) CIRCLED IDEOGRAPH TEN +328A;328A;328A;6708;6708; # (㊊; ㊊; ㊊; 月; 月; ) CIRCLED IDEOGRAPH MOON +328B;328B;328B;706B;706B; # (㊋; ㊋; ㊋; 火; 火; ) CIRCLED IDEOGRAPH FIRE +328C;328C;328C;6C34;6C34; # (㊌; ㊌; ㊌; 水; 水; ) CIRCLED IDEOGRAPH WATER +328D;328D;328D;6728;6728; # (㊍; ㊍; ㊍; 木; 木; ) CIRCLED IDEOGRAPH WOOD +328E;328E;328E;91D1;91D1; # (㊎; ㊎; ㊎; 金; 金; ) CIRCLED IDEOGRAPH METAL +328F;328F;328F;571F;571F; # (㊏; ㊏; ㊏; 土; 土; ) CIRCLED IDEOGRAPH EARTH +3290;3290;3290;65E5;65E5; # (㊐; ㊐; ㊐; 日; 日; ) CIRCLED IDEOGRAPH SUN +3291;3291;3291;682A;682A; # (㊑; ㊑; ㊑; 株; 株; ) CIRCLED IDEOGRAPH STOCK +3292;3292;3292;6709;6709; # (㊒; ㊒; ㊒; 有; 有; ) CIRCLED IDEOGRAPH HAVE +3293;3293;3293;793E;793E; # (㊓; ㊓; ㊓; 社; 社; ) CIRCLED IDEOGRAPH SOCIETY +3294;3294;3294;540D;540D; # (㊔; ㊔; ㊔; 名; 名; ) CIRCLED IDEOGRAPH NAME +3295;3295;3295;7279;7279; # (㊕; ㊕; ㊕; 特; 特; ) CIRCLED IDEOGRAPH SPECIAL +3296;3296;3296;8CA1;8CA1; # (㊖; ㊖; ㊖; 財; 財; ) CIRCLED IDEOGRAPH FINANCIAL +3297;3297;3297;795D;795D; # (㊗; ㊗; ㊗; 祝; 祝; ) CIRCLED IDEOGRAPH CONGRATULATION +3298;3298;3298;52B4;52B4; # (㊘; ㊘; ㊘; 労; 労; ) CIRCLED IDEOGRAPH LABOR +3299;3299;3299;79D8;79D8; # (㊙; ㊙; ㊙; 秘; 秘; ) CIRCLED IDEOGRAPH SECRET +329A;329A;329A;7537;7537; # (㊚; ㊚; ㊚; 男; 男; ) CIRCLED IDEOGRAPH MALE +329B;329B;329B;5973;5973; # (㊛; ㊛; ㊛; 女; 女; ) CIRCLED IDEOGRAPH FEMALE +329C;329C;329C;9069;9069; # (㊜; ㊜; ㊜; 適; 適; ) CIRCLED IDEOGRAPH SUITABLE +329D;329D;329D;512A;512A; # (㊝; ㊝; ㊝; 優; 優; ) CIRCLED IDEOGRAPH EXCELLENT +329E;329E;329E;5370;5370; # (㊞; ㊞; ㊞; 印; 印; ) CIRCLED IDEOGRAPH PRINT +329F;329F;329F;6CE8;6CE8; # (㊟; ㊟; ㊟; 注; 注; ) CIRCLED IDEOGRAPH ATTENTION +32A0;32A0;32A0;9805;9805; # (㊠; ㊠; ㊠; 項; 項; ) CIRCLED IDEOGRAPH ITEM +32A1;32A1;32A1;4F11;4F11; # (㊡; ㊡; ㊡; 休; 休; ) CIRCLED IDEOGRAPH REST +32A2;32A2;32A2;5199;5199; # (㊢; ㊢; ㊢; 写; 写; ) CIRCLED IDEOGRAPH COPY +32A3;32A3;32A3;6B63;6B63; # (㊣; ㊣; ㊣; 正; 正; ) CIRCLED IDEOGRAPH CORRECT +32A4;32A4;32A4;4E0A;4E0A; # (㊤; ㊤; ㊤; 上; 上; ) CIRCLED IDEOGRAPH HIGH +32A5;32A5;32A5;4E2D;4E2D; # (㊥; ㊥; ㊥; 中; 中; ) CIRCLED IDEOGRAPH CENTRE +32A6;32A6;32A6;4E0B;4E0B; # (㊦; ㊦; ㊦; 下; 下; ) CIRCLED IDEOGRAPH LOW +32A7;32A7;32A7;5DE6;5DE6; # (㊧; ㊧; ㊧; 左; 左; ) CIRCLED IDEOGRAPH LEFT +32A8;32A8;32A8;53F3;53F3; # (㊨; ㊨; ㊨; 右; 右; ) CIRCLED IDEOGRAPH RIGHT +32A9;32A9;32A9;533B;533B; # (㊩; ㊩; ㊩; 医; 医; ) CIRCLED IDEOGRAPH MEDICINE +32AA;32AA;32AA;5B97;5B97; # (㊪; ㊪; ㊪; 宗; 宗; ) CIRCLED IDEOGRAPH RELIGION +32AB;32AB;32AB;5B66;5B66; # (㊫; ㊫; ㊫; 学; 学; ) CIRCLED IDEOGRAPH STUDY +32AC;32AC;32AC;76E3;76E3; # (㊬; ㊬; ㊬; 監; 監; ) CIRCLED IDEOGRAPH SUPERVISE +32AD;32AD;32AD;4F01;4F01; # (㊭; ㊭; ㊭; 企; 企; ) CIRCLED IDEOGRAPH ENTERPRISE +32AE;32AE;32AE;8CC7;8CC7; # (㊮; ㊮; ㊮; 資; 資; ) CIRCLED IDEOGRAPH RESOURCE +32AF;32AF;32AF;5354;5354; # (㊯; ㊯; ㊯; 協; 協; ) CIRCLED IDEOGRAPH ALLIANCE +32B0;32B0;32B0;591C;591C; # (㊰; ㊰; ㊰; 夜; 夜; ) CIRCLED IDEOGRAPH NIGHT +32B1;32B1;32B1;0033 0036;0033 0036; # (㊱; ㊱; ㊱; 36; 36; ) CIRCLED NUMBER THIRTY SIX +32B2;32B2;32B2;0033 0037;0033 0037; # (㊲; ㊲; ㊲; 37; 37; ) CIRCLED NUMBER THIRTY SEVEN +32B3;32B3;32B3;0033 0038;0033 0038; # (㊳; ㊳; ㊳; 38; 38; ) CIRCLED NUMBER THIRTY EIGHT +32B4;32B4;32B4;0033 0039;0033 0039; # (㊴; ㊴; ㊴; 39; 39; ) CIRCLED NUMBER THIRTY NINE +32B5;32B5;32B5;0034 0030;0034 0030; # (㊵; ㊵; ㊵; 40; 40; ) CIRCLED NUMBER FORTY +32B6;32B6;32B6;0034 0031;0034 0031; # (㊶; ㊶; ㊶; 41; 41; ) CIRCLED NUMBER FORTY ONE +32B7;32B7;32B7;0034 0032;0034 0032; # (㊷; ㊷; ㊷; 42; 42; ) CIRCLED NUMBER FORTY TWO +32B8;32B8;32B8;0034 0033;0034 0033; # (㊸; ㊸; ㊸; 43; 43; ) CIRCLED NUMBER FORTY THREE +32B9;32B9;32B9;0034 0034;0034 0034; # (㊹; ㊹; ㊹; 44; 44; ) CIRCLED NUMBER FORTY FOUR +32BA;32BA;32BA;0034 0035;0034 0035; # (㊺; ㊺; ㊺; 45; 45; ) CIRCLED NUMBER FORTY FIVE +32BB;32BB;32BB;0034 0036;0034 0036; # (㊻; ㊻; ㊻; 46; 46; ) CIRCLED NUMBER FORTY SIX +32BC;32BC;32BC;0034 0037;0034 0037; # (㊼; ㊼; ㊼; 47; 47; ) CIRCLED NUMBER FORTY SEVEN +32BD;32BD;32BD;0034 0038;0034 0038; # (㊽; ㊽; ㊽; 48; 48; ) CIRCLED NUMBER FORTY EIGHT +32BE;32BE;32BE;0034 0039;0034 0039; # (㊾; ㊾; ㊾; 49; 49; ) CIRCLED NUMBER FORTY NINE +32BF;32BF;32BF;0035 0030;0035 0030; # (㊿; ㊿; ㊿; 50; 50; ) CIRCLED NUMBER FIFTY +32C0;32C0;32C0;0031 6708;0031 6708; # (㋀; ㋀; ㋀; 1月; 1月; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR JANUARY +32C1;32C1;32C1;0032 6708;0032 6708; # (㋁; ㋁; ㋁; 2月; 2月; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR FEBRUARY +32C2;32C2;32C2;0033 6708;0033 6708; # (㋂; ㋂; ㋂; 3月; 3月; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR MARCH +32C3;32C3;32C3;0034 6708;0034 6708; # (㋃; ㋃; ㋃; 4月; 4月; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR APRIL +32C4;32C4;32C4;0035 6708;0035 6708; # (㋄; ㋄; ㋄; 5月; 5月; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR MAY +32C5;32C5;32C5;0036 6708;0036 6708; # (㋅; ㋅; ㋅; 6月; 6月; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR JUNE +32C6;32C6;32C6;0037 6708;0037 6708; # (㋆; ㋆; ㋆; 7月; 7月; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR JULY +32C7;32C7;32C7;0038 6708;0038 6708; # (㋇; ㋇; ㋇; 8月; 8月; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR AUGUST +32C8;32C8;32C8;0039 6708;0039 6708; # (㋈; ㋈; ㋈; 9月; 9月; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR SEPTEMBER +32C9;32C9;32C9;0031 0030 6708;0031 0030 6708; # (㋉; ㋉; ㋉; 10月; 10月; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR OCTOBER +32CA;32CA;32CA;0031 0031 6708;0031 0031 6708; # (㋊; ㋊; ㋊; 11月; 11月; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR NOVEMBER +32CB;32CB;32CB;0031 0032 6708;0031 0032 6708; # (㋋; ㋋; ㋋; 12月; 12月; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DECEMBER +32D0;32D0;32D0;30A2;30A2; # (㋐; ㋐; ㋐; ア; ア; ) CIRCLED KATAKANA A +32D1;32D1;32D1;30A4;30A4; # (㋑; ㋑; ㋑; イ; イ; ) CIRCLED KATAKANA I +32D2;32D2;32D2;30A6;30A6; # (㋒; ㋒; ㋒; ウ; ウ; ) CIRCLED KATAKANA U +32D3;32D3;32D3;30A8;30A8; # (㋓; ㋓; ㋓; エ; エ; ) CIRCLED KATAKANA E +32D4;32D4;32D4;30AA;30AA; # (㋔; ㋔; ㋔; オ; オ; ) CIRCLED KATAKANA O +32D5;32D5;32D5;30AB;30AB; # (㋕; ㋕; ㋕; カ; カ; ) CIRCLED KATAKANA KA +32D6;32D6;32D6;30AD;30AD; # (㋖; ㋖; ㋖; キ; キ; ) CIRCLED KATAKANA KI +32D7;32D7;32D7;30AF;30AF; # (㋗; ㋗; ㋗; ク; ク; ) CIRCLED KATAKANA KU +32D8;32D8;32D8;30B1;30B1; # (㋘; ㋘; ㋘; ケ; ケ; ) CIRCLED KATAKANA KE +32D9;32D9;32D9;30B3;30B3; # (㋙; ㋙; ㋙; コ; コ; ) CIRCLED KATAKANA KO +32DA;32DA;32DA;30B5;30B5; # (㋚; ㋚; ㋚; サ; サ; ) CIRCLED KATAKANA SA +32DB;32DB;32DB;30B7;30B7; # (㋛; ㋛; ㋛; シ; シ; ) CIRCLED KATAKANA SI +32DC;32DC;32DC;30B9;30B9; # (㋜; ㋜; ㋜; ス; ス; ) CIRCLED KATAKANA SU +32DD;32DD;32DD;30BB;30BB; # (㋝; ㋝; ㋝; セ; セ; ) CIRCLED KATAKANA SE +32DE;32DE;32DE;30BD;30BD; # (㋞; ㋞; ㋞; ソ; ソ; ) CIRCLED KATAKANA SO +32DF;32DF;32DF;30BF;30BF; # (㋟; ㋟; ㋟; タ; タ; ) CIRCLED KATAKANA TA +32E0;32E0;32E0;30C1;30C1; # (㋠; ㋠; ㋠; チ; チ; ) CIRCLED KATAKANA TI +32E1;32E1;32E1;30C4;30C4; # (㋡; ㋡; ㋡; ツ; ツ; ) CIRCLED KATAKANA TU +32E2;32E2;32E2;30C6;30C6; # (㋢; ㋢; ㋢; テ; テ; ) CIRCLED KATAKANA TE +32E3;32E3;32E3;30C8;30C8; # (㋣; ㋣; ㋣; ト; ト; ) CIRCLED KATAKANA TO +32E4;32E4;32E4;30CA;30CA; # (㋤; ㋤; ㋤; ナ; ナ; ) CIRCLED KATAKANA NA +32E5;32E5;32E5;30CB;30CB; # (㋥; ㋥; ㋥; ニ; ニ; ) CIRCLED KATAKANA NI +32E6;32E6;32E6;30CC;30CC; # (㋦; ㋦; ㋦; ヌ; ヌ; ) CIRCLED KATAKANA NU +32E7;32E7;32E7;30CD;30CD; # (㋧; ㋧; ㋧; ネ; ネ; ) CIRCLED KATAKANA NE +32E8;32E8;32E8;30CE;30CE; # (㋨; ㋨; ㋨; ノ; ノ; ) CIRCLED KATAKANA NO +32E9;32E9;32E9;30CF;30CF; # (㋩; ㋩; ㋩; ハ; ハ; ) CIRCLED KATAKANA HA +32EA;32EA;32EA;30D2;30D2; # (㋪; ㋪; ㋪; ヒ; ヒ; ) CIRCLED KATAKANA HI +32EB;32EB;32EB;30D5;30D5; # (㋫; ㋫; ㋫; フ; フ; ) CIRCLED KATAKANA HU +32EC;32EC;32EC;30D8;30D8; # (㋬; ㋬; ㋬; ヘ; ヘ; ) CIRCLED KATAKANA HE +32ED;32ED;32ED;30DB;30DB; # (㋭; ㋭; ㋭; ホ; ホ; ) CIRCLED KATAKANA HO +32EE;32EE;32EE;30DE;30DE; # (㋮; ㋮; ㋮; マ; マ; ) CIRCLED KATAKANA MA +32EF;32EF;32EF;30DF;30DF; # (㋯; ㋯; ㋯; ミ; ミ; ) CIRCLED KATAKANA MI +32F0;32F0;32F0;30E0;30E0; # (㋰; ㋰; ㋰; ム; ム; ) CIRCLED KATAKANA MU +32F1;32F1;32F1;30E1;30E1; # (㋱; ㋱; ㋱; メ; メ; ) CIRCLED KATAKANA ME +32F2;32F2;32F2;30E2;30E2; # (㋲; ㋲; ㋲; モ; モ; ) CIRCLED KATAKANA MO +32F3;32F3;32F3;30E4;30E4; # (㋳; ㋳; ㋳; ヤ; ヤ; ) CIRCLED KATAKANA YA +32F4;32F4;32F4;30E6;30E6; # (㋴; ㋴; ㋴; ユ; ユ; ) CIRCLED KATAKANA YU +32F5;32F5;32F5;30E8;30E8; # (㋵; ㋵; ㋵; ヨ; ヨ; ) CIRCLED KATAKANA YO +32F6;32F6;32F6;30E9;30E9; # (㋶; ㋶; ㋶; ラ; ラ; ) CIRCLED KATAKANA RA +32F7;32F7;32F7;30EA;30EA; # (㋷; ㋷; ㋷; リ; リ; ) CIRCLED KATAKANA RI +32F8;32F8;32F8;30EB;30EB; # (㋸; ㋸; ㋸; ル; ル; ) CIRCLED KATAKANA RU +32F9;32F9;32F9;30EC;30EC; # (㋹; ㋹; ㋹; レ; レ; ) CIRCLED KATAKANA RE +32FA;32FA;32FA;30ED;30ED; # (㋺; ㋺; ㋺; ロ; ロ; ) CIRCLED KATAKANA RO +32FB;32FB;32FB;30EF;30EF; # (㋻; ㋻; ㋻; ワ; ワ; ) CIRCLED KATAKANA WA +32FC;32FC;32FC;30F0;30F0; # (㋼; ㋼; ㋼; ヰ; ヰ; ) CIRCLED KATAKANA WI +32FD;32FD;32FD;30F1;30F1; # (㋽; ㋽; ㋽; ヱ; ヱ; ) CIRCLED KATAKANA WE +32FE;32FE;32FE;30F2;30F2; # (㋾; ㋾; ㋾; ヲ; ヲ; ) CIRCLED KATAKANA WO +3300;3300;3300;30A2 30D1 30FC 30C8;30A2 30CF 309A 30FC 30C8; # (㌀; ㌀; ㌀; アパート; アハ◌゚ート; ) SQUARE APAATO +3301;3301;3301;30A2 30EB 30D5 30A1;30A2 30EB 30D5 30A1; # (㌁; ㌁; ㌁; アルファ; アルファ; ) SQUARE ARUHUA +3302;3302;3302;30A2 30F3 30DA 30A2;30A2 30F3 30D8 309A 30A2; # (㌂; ㌂; ㌂; アンペア; アンヘ◌゚ア; ) SQUARE ANPEA +3303;3303;3303;30A2 30FC 30EB;30A2 30FC 30EB; # (㌃; ㌃; ㌃; アール; アール; ) SQUARE AARU +3304;3304;3304;30A4 30CB 30F3 30B0;30A4 30CB 30F3 30AF 3099; # (㌄; ㌄; ㌄; イニング; イニンク◌゙; ) SQUARE ININGU +3305;3305;3305;30A4 30F3 30C1;30A4 30F3 30C1; # (㌅; ㌅; ㌅; インチ; インチ; ) SQUARE INTI +3306;3306;3306;30A6 30A9 30F3;30A6 30A9 30F3; # (㌆; ㌆; ㌆; ウォン; ウォン; ) SQUARE UON +3307;3307;3307;30A8 30B9 30AF 30FC 30C9;30A8 30B9 30AF 30FC 30C8 3099; # (㌇; ㌇; ㌇; エスクード; エスクート◌゙; ) SQUARE ESUKUUDO +3308;3308;3308;30A8 30FC 30AB 30FC;30A8 30FC 30AB 30FC; # (㌈; ㌈; ㌈; エーカー; エーカー; ) SQUARE EEKAA +3309;3309;3309;30AA 30F3 30B9;30AA 30F3 30B9; # (㌉; ㌉; ㌉; オンス; オンス; ) SQUARE ONSU +330A;330A;330A;30AA 30FC 30E0;30AA 30FC 30E0; # (㌊; ㌊; ㌊; オーム; オーム; ) SQUARE OOMU +330B;330B;330B;30AB 30A4 30EA;30AB 30A4 30EA; # (㌋; ㌋; ㌋; カイリ; カイリ; ) SQUARE KAIRI +330C;330C;330C;30AB 30E9 30C3 30C8;30AB 30E9 30C3 30C8; # (㌌; ㌌; ㌌; カラット; カラット; ) SQUARE KARATTO +330D;330D;330D;30AB 30ED 30EA 30FC;30AB 30ED 30EA 30FC; # (㌍; ㌍; ㌍; カロリー; カロリー; ) SQUARE KARORII +330E;330E;330E;30AC 30ED 30F3;30AB 3099 30ED 30F3; # (㌎; ㌎; ㌎; ガロン; カ◌゙ロン; ) SQUARE GARON +330F;330F;330F;30AC 30F3 30DE;30AB 3099 30F3 30DE; # (㌏; ㌏; ㌏; ガンマ; カ◌゙ンマ; ) SQUARE GANMA +3310;3310;3310;30AE 30AC;30AD 3099 30AB 3099; # (㌐; ㌐; ㌐; ギガ; キ◌゙カ◌゙; ) SQUARE GIGA +3311;3311;3311;30AE 30CB 30FC;30AD 3099 30CB 30FC; # (㌑; ㌑; ㌑; ギニー; キ◌゙ニー; ) SQUARE GINII +3312;3312;3312;30AD 30E5 30EA 30FC;30AD 30E5 30EA 30FC; # (㌒; ㌒; ㌒; キュリー; キュリー; ) SQUARE KYURII +3313;3313;3313;30AE 30EB 30C0 30FC;30AD 3099 30EB 30BF 3099 30FC; # (㌓; ㌓; ㌓; ギルダー; キ◌゙ルタ◌゙ー; ) SQUARE GIRUDAA +3314;3314;3314;30AD 30ED;30AD 30ED; # (㌔; ㌔; ㌔; キロ; キロ; ) SQUARE KIRO +3315;3315;3315;30AD 30ED 30B0 30E9 30E0;30AD 30ED 30AF 3099 30E9 30E0; # (㌕; ㌕; ㌕; キログラム; キロク◌゙ラム; ) SQUARE KIROGURAMU +3316;3316;3316;30AD 30ED 30E1 30FC 30C8 30EB;30AD 30ED 30E1 30FC 30C8 30EB; # (㌖; ㌖; ㌖; キロメートル; キロメートル; ) SQUARE KIROMEETORU +3317;3317;3317;30AD 30ED 30EF 30C3 30C8;30AD 30ED 30EF 30C3 30C8; # (㌗; ㌗; ㌗; キロワット; キロワット; ) SQUARE KIROWATTO +3318;3318;3318;30B0 30E9 30E0;30AF 3099 30E9 30E0; # (㌘; ㌘; ㌘; グラム; ク◌゙ラム; ) SQUARE GURAMU +3319;3319;3319;30B0 30E9 30E0 30C8 30F3;30AF 3099 30E9 30E0 30C8 30F3; # (㌙; ㌙; ㌙; グラムトン; ク◌゙ラムトン; ) SQUARE GURAMUTON +331A;331A;331A;30AF 30EB 30BC 30A4 30ED;30AF 30EB 30BB 3099 30A4 30ED; # (㌚; ㌚; ㌚; クルゼイロ; クルセ◌゙イロ; ) SQUARE KURUZEIRO +331B;331B;331B;30AF 30ED 30FC 30CD;30AF 30ED 30FC 30CD; # (㌛; ㌛; ㌛; クローネ; クローネ; ) SQUARE KUROONE +331C;331C;331C;30B1 30FC 30B9;30B1 30FC 30B9; # (㌜; ㌜; ㌜; ケース; ケース; ) SQUARE KEESU +331D;331D;331D;30B3 30EB 30CA;30B3 30EB 30CA; # (㌝; ㌝; ㌝; コルナ; コルナ; ) SQUARE KORUNA +331E;331E;331E;30B3 30FC 30DD;30B3 30FC 30DB 309A; # (㌞; ㌞; ㌞; コーポ; コーホ◌゚; ) SQUARE KOOPO +331F;331F;331F;30B5 30A4 30AF 30EB;30B5 30A4 30AF 30EB; # (㌟; ㌟; ㌟; サイクル; サイクル; ) SQUARE SAIKURU +3320;3320;3320;30B5 30F3 30C1 30FC 30E0;30B5 30F3 30C1 30FC 30E0; # (㌠; ㌠; ㌠; サンチーム; サンチーム; ) SQUARE SANTIIMU +3321;3321;3321;30B7 30EA 30F3 30B0;30B7 30EA 30F3 30AF 3099; # (㌡; ㌡; ㌡; シリング; シリンク◌゙; ) SQUARE SIRINGU +3322;3322;3322;30BB 30F3 30C1;30BB 30F3 30C1; # (㌢; ㌢; ㌢; センチ; センチ; ) SQUARE SENTI +3323;3323;3323;30BB 30F3 30C8;30BB 30F3 30C8; # (㌣; ㌣; ㌣; セント; セント; ) SQUARE SENTO +3324;3324;3324;30C0 30FC 30B9;30BF 3099 30FC 30B9; # (㌤; ㌤; ㌤; ダース; タ◌゙ース; ) SQUARE DAASU +3325;3325;3325;30C7 30B7;30C6 3099 30B7; # (㌥; ㌥; ㌥; デシ; テ◌゙シ; ) SQUARE DESI +3326;3326;3326;30C9 30EB;30C8 3099 30EB; # (㌦; ㌦; ㌦; ドル; ト◌゙ル; ) SQUARE DORU +3327;3327;3327;30C8 30F3;30C8 30F3; # (㌧; ㌧; ㌧; トン; トン; ) SQUARE TON +3328;3328;3328;30CA 30CE;30CA 30CE; # (㌨; ㌨; ㌨; ナノ; ナノ; ) SQUARE NANO +3329;3329;3329;30CE 30C3 30C8;30CE 30C3 30C8; # (㌩; ㌩; ㌩; ノット; ノット; ) SQUARE NOTTO +332A;332A;332A;30CF 30A4 30C4;30CF 30A4 30C4; # (㌪; ㌪; ㌪; ハイツ; ハイツ; ) SQUARE HAITU +332B;332B;332B;30D1 30FC 30BB 30F3 30C8;30CF 309A 30FC 30BB 30F3 30C8; # (㌫; ㌫; ㌫; パーセント; ハ◌゚ーセント; ) SQUARE PAASENTO +332C;332C;332C;30D1 30FC 30C4;30CF 309A 30FC 30C4; # (㌬; ㌬; ㌬; パーツ; ハ◌゚ーツ; ) SQUARE PAATU +332D;332D;332D;30D0 30FC 30EC 30EB;30CF 3099 30FC 30EC 30EB; # (㌭; ㌭; ㌭; バーレル; ハ◌゙ーレル; ) SQUARE BAARERU +332E;332E;332E;30D4 30A2 30B9 30C8 30EB;30D2 309A 30A2 30B9 30C8 30EB; # (㌮; ㌮; ㌮; ピアストル; ヒ◌゚アストル; ) SQUARE PIASUTORU +332F;332F;332F;30D4 30AF 30EB;30D2 309A 30AF 30EB; # (㌯; ㌯; ㌯; ピクル; ヒ◌゚クル; ) SQUARE PIKURU +3330;3330;3330;30D4 30B3;30D2 309A 30B3; # (㌰; ㌰; ㌰; ピコ; ヒ◌゚コ; ) SQUARE PIKO +3331;3331;3331;30D3 30EB;30D2 3099 30EB; # (㌱; ㌱; ㌱; ビル; ヒ◌゙ル; ) SQUARE BIRU +3332;3332;3332;30D5 30A1 30E9 30C3 30C9;30D5 30A1 30E9 30C3 30C8 3099; # (㌲; ㌲; ㌲; ファラッド; ファラット◌゙; ) SQUARE HUARADDO +3333;3333;3333;30D5 30A3 30FC 30C8;30D5 30A3 30FC 30C8; # (㌳; ㌳; ㌳; フィート; フィート; ) SQUARE HUIITO +3334;3334;3334;30D6 30C3 30B7 30A7 30EB;30D5 3099 30C3 30B7 30A7 30EB; # (㌴; ㌴; ㌴; ブッシェル; フ◌゙ッシェル; ) SQUARE BUSSYERU +3335;3335;3335;30D5 30E9 30F3;30D5 30E9 30F3; # (㌵; ㌵; ㌵; フラン; フラン; ) SQUARE HURAN +3336;3336;3336;30D8 30AF 30BF 30FC 30EB;30D8 30AF 30BF 30FC 30EB; # (㌶; ㌶; ㌶; ヘクタール; ヘクタール; ) SQUARE HEKUTAARU +3337;3337;3337;30DA 30BD;30D8 309A 30BD; # (㌷; ㌷; ㌷; ペソ; ヘ◌゚ソ; ) SQUARE PESO +3338;3338;3338;30DA 30CB 30D2;30D8 309A 30CB 30D2; # (㌸; ㌸; ㌸; ペニヒ; ヘ◌゚ニヒ; ) SQUARE PENIHI +3339;3339;3339;30D8 30EB 30C4;30D8 30EB 30C4; # (㌹; ㌹; ㌹; ヘルツ; ヘルツ; ) SQUARE HERUTU +333A;333A;333A;30DA 30F3 30B9;30D8 309A 30F3 30B9; # (㌺; ㌺; ㌺; ペンス; ヘ◌゚ンス; ) SQUARE PENSU +333B;333B;333B;30DA 30FC 30B8;30D8 309A 30FC 30B7 3099; # (㌻; ㌻; ㌻; ページ; ヘ◌゚ーシ◌゙; ) SQUARE PEEZI +333C;333C;333C;30D9 30FC 30BF;30D8 3099 30FC 30BF; # (㌼; ㌼; ㌼; ベータ; ヘ◌゙ータ; ) SQUARE BEETA +333D;333D;333D;30DD 30A4 30F3 30C8;30DB 309A 30A4 30F3 30C8; # (㌽; ㌽; ㌽; ポイント; ホ◌゚イント; ) SQUARE POINTO +333E;333E;333E;30DC 30EB 30C8;30DB 3099 30EB 30C8; # (㌾; ㌾; ㌾; ボルト; ホ◌゙ルト; ) SQUARE BORUTO +333F;333F;333F;30DB 30F3;30DB 30F3; # (㌿; ㌿; ㌿; ホン; ホン; ) SQUARE HON +3340;3340;3340;30DD 30F3 30C9;30DB 309A 30F3 30C8 3099; # (㍀; ㍀; ㍀; ポンド; ホ◌゚ント◌゙; ) SQUARE PONDO +3341;3341;3341;30DB 30FC 30EB;30DB 30FC 30EB; # (㍁; ㍁; ㍁; ホール; ホール; ) SQUARE HOORU +3342;3342;3342;30DB 30FC 30F3;30DB 30FC 30F3; # (㍂; ㍂; ㍂; ホーン; ホーン; ) SQUARE HOON +3343;3343;3343;30DE 30A4 30AF 30ED;30DE 30A4 30AF 30ED; # (㍃; ㍃; ㍃; マイクロ; マイクロ; ) SQUARE MAIKURO +3344;3344;3344;30DE 30A4 30EB;30DE 30A4 30EB; # (㍄; ㍄; ㍄; マイル; マイル; ) SQUARE MAIRU +3345;3345;3345;30DE 30C3 30CF;30DE 30C3 30CF; # (㍅; ㍅; ㍅; マッハ; マッハ; ) SQUARE MAHHA +3346;3346;3346;30DE 30EB 30AF;30DE 30EB 30AF; # (㍆; ㍆; ㍆; マルク; マルク; ) SQUARE MARUKU +3347;3347;3347;30DE 30F3 30B7 30E7 30F3;30DE 30F3 30B7 30E7 30F3; # (㍇; ㍇; ㍇; マンション; マンション; ) SQUARE MANSYON +3348;3348;3348;30DF 30AF 30ED 30F3;30DF 30AF 30ED 30F3; # (㍈; ㍈; ㍈; ミクロン; ミクロン; ) SQUARE MIKURON +3349;3349;3349;30DF 30EA;30DF 30EA; # (㍉; ㍉; ㍉; ミリ; ミリ; ) SQUARE MIRI +334A;334A;334A;30DF 30EA 30D0 30FC 30EB;30DF 30EA 30CF 3099 30FC 30EB; # (㍊; ㍊; ㍊; ミリバール; ミリハ◌゙ール; ) SQUARE MIRIBAARU +334B;334B;334B;30E1 30AC;30E1 30AB 3099; # (㍋; ㍋; ㍋; メガ; メカ◌゙; ) SQUARE MEGA +334C;334C;334C;30E1 30AC 30C8 30F3;30E1 30AB 3099 30C8 30F3; # (㍌; ㍌; ㍌; メガトン; メカ◌゙トン; ) SQUARE MEGATON +334D;334D;334D;30E1 30FC 30C8 30EB;30E1 30FC 30C8 30EB; # (㍍; ㍍; ㍍; メートル; メートル; ) SQUARE MEETORU +334E;334E;334E;30E4 30FC 30C9;30E4 30FC 30C8 3099; # (㍎; ㍎; ㍎; ヤード; ヤート◌゙; ) SQUARE YAADO +334F;334F;334F;30E4 30FC 30EB;30E4 30FC 30EB; # (㍏; ㍏; ㍏; ヤール; ヤール; ) SQUARE YAARU +3350;3350;3350;30E6 30A2 30F3;30E6 30A2 30F3; # (㍐; ㍐; ㍐; ユアン; ユアン; ) SQUARE YUAN +3351;3351;3351;30EA 30C3 30C8 30EB;30EA 30C3 30C8 30EB; # (㍑; ㍑; ㍑; リットル; リットル; ) SQUARE RITTORU +3352;3352;3352;30EA 30E9;30EA 30E9; # (㍒; ㍒; ㍒; リラ; リラ; ) SQUARE RIRA +3353;3353;3353;30EB 30D4 30FC;30EB 30D2 309A 30FC; # (㍓; ㍓; ㍓; ルピー; ルヒ◌゚ー; ) SQUARE RUPII +3354;3354;3354;30EB 30FC 30D6 30EB;30EB 30FC 30D5 3099 30EB; # (㍔; ㍔; ㍔; ルーブル; ルーフ◌゙ル; ) SQUARE RUUBURU +3355;3355;3355;30EC 30E0;30EC 30E0; # (㍕; ㍕; ㍕; レム; レム; ) SQUARE REMU +3356;3356;3356;30EC 30F3 30C8 30B2 30F3;30EC 30F3 30C8 30B1 3099 30F3; # (㍖; ㍖; ㍖; レントゲン; レントケ◌゙ン; ) SQUARE RENTOGEN +3357;3357;3357;30EF 30C3 30C8;30EF 30C3 30C8; # (㍗; ㍗; ㍗; ワット; ワット; ) SQUARE WATTO +3358;3358;3358;0030 70B9;0030 70B9; # (㍘; ㍘; ㍘; 0点; 0点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR ZERO +3359;3359;3359;0031 70B9;0031 70B9; # (㍙; ㍙; ㍙; 1点; 1点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR ONE +335A;335A;335A;0032 70B9;0032 70B9; # (㍚; ㍚; ㍚; 2点; 2点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR TWO +335B;335B;335B;0033 70B9;0033 70B9; # (㍛; ㍛; ㍛; 3点; 3点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR THREE +335C;335C;335C;0034 70B9;0034 70B9; # (㍜; ㍜; ㍜; 4点; 4点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR FOUR +335D;335D;335D;0035 70B9;0035 70B9; # (㍝; ㍝; ㍝; 5点; 5点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR FIVE +335E;335E;335E;0036 70B9;0036 70B9; # (㍞; ㍞; ㍞; 6点; 6点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR SIX +335F;335F;335F;0037 70B9;0037 70B9; # (㍟; ㍟; ㍟; 7点; 7点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR SEVEN +3360;3360;3360;0038 70B9;0038 70B9; # (㍠; ㍠; ㍠; 8点; 8点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR EIGHT +3361;3361;3361;0039 70B9;0039 70B9; # (㍡; ㍡; ㍡; 9点; 9点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR NINE +3362;3362;3362;0031 0030 70B9;0031 0030 70B9; # (㍢; ㍢; ㍢; 10点; 10点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR TEN +3363;3363;3363;0031 0031 70B9;0031 0031 70B9; # (㍣; ㍣; ㍣; 11点; 11点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR ELEVEN +3364;3364;3364;0031 0032 70B9;0031 0032 70B9; # (㍤; ㍤; ㍤; 12点; 12点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR TWELVE +3365;3365;3365;0031 0033 70B9;0031 0033 70B9; # (㍥; ㍥; ㍥; 13点; 13点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR THIRTEEN +3366;3366;3366;0031 0034 70B9;0031 0034 70B9; # (㍦; ㍦; ㍦; 14点; 14点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR FOURTEEN +3367;3367;3367;0031 0035 70B9;0031 0035 70B9; # (㍧; ㍧; ㍧; 15点; 15点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR FIFTEEN +3368;3368;3368;0031 0036 70B9;0031 0036 70B9; # (㍨; ㍨; ㍨; 16点; 16点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR SIXTEEN +3369;3369;3369;0031 0037 70B9;0031 0037 70B9; # (㍩; ㍩; ㍩; 17点; 17点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR SEVENTEEN +336A;336A;336A;0031 0038 70B9;0031 0038 70B9; # (㍪; ㍪; ㍪; 18点; 18点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR EIGHTEEN +336B;336B;336B;0031 0039 70B9;0031 0039 70B9; # (㍫; ㍫; ㍫; 19点; 19点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR NINETEEN +336C;336C;336C;0032 0030 70B9;0032 0030 70B9; # (㍬; ㍬; ㍬; 20点; 20点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR TWENTY +336D;336D;336D;0032 0031 70B9;0032 0031 70B9; # (㍭; ㍭; ㍭; 21点; 21点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR TWENTY-ONE +336E;336E;336E;0032 0032 70B9;0032 0032 70B9; # (㍮; ㍮; ㍮; 22点; 22点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR TWENTY-TWO +336F;336F;336F;0032 0033 70B9;0032 0033 70B9; # (㍯; ㍯; ㍯; 23点; 23点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR TWENTY-THREE +3370;3370;3370;0032 0034 70B9;0032 0034 70B9; # (㍰; ㍰; ㍰; 24点; 24点; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR HOUR TWENTY-FOUR +3371;3371;3371;0068 0050 0061;0068 0050 0061; # (㍱; ㍱; ㍱; hPa; hPa; ) SQUARE HPA +3372;3372;3372;0064 0061;0064 0061; # (㍲; ㍲; ㍲; da; da; ) SQUARE DA +3373;3373;3373;0041 0055;0041 0055; # (㍳; ㍳; ㍳; AU; AU; ) SQUARE AU +3374;3374;3374;0062 0061 0072;0062 0061 0072; # (㍴; ㍴; ㍴; bar; bar; ) SQUARE BAR +3375;3375;3375;006F 0056;006F 0056; # (㍵; ㍵; ㍵; oV; oV; ) SQUARE OV +3376;3376;3376;0070 0063;0070 0063; # (㍶; ㍶; ㍶; pc; pc; ) SQUARE PC +337B;337B;337B;5E73 6210;5E73 6210; # (㍻; ㍻; ㍻; 平成; 平成; ) SQUARE ERA NAME HEISEI +337C;337C;337C;662D 548C;662D 548C; # (㍼; ㍼; ㍼; 昭和; 昭和; ) SQUARE ERA NAME SYOUWA +337D;337D;337D;5927 6B63;5927 6B63; # (㍽; ㍽; ㍽; 大正; 大正; ) SQUARE ERA NAME TAISYOU +337E;337E;337E;660E 6CBB;660E 6CBB; # (㍾; ㍾; ㍾; 明治; 明治; ) SQUARE ERA NAME MEIZI +337F;337F;337F;682A 5F0F 4F1A 793E;682A 5F0F 4F1A 793E; # (㍿; ㍿; ㍿; 株式会社; 株式会社; ) SQUARE CORPORATION +3380;3380;3380;0070 0041;0070 0041; # (㎀; ㎀; ㎀; pA; pA; ) SQUARE PA AMPS +3381;3381;3381;006E 0041;006E 0041; # (㎁; ㎁; ㎁; nA; nA; ) SQUARE NA +3382;3382;3382;03BC 0041;03BC 0041; # (㎂; ㎂; ㎂; μA; μA; ) SQUARE MU A +3383;3383;3383;006D 0041;006D 0041; # (㎃; ㎃; ㎃; mA; mA; ) SQUARE MA +3384;3384;3384;006B 0041;006B 0041; # (㎄; ㎄; ㎄; kA; kA; ) SQUARE KA +3385;3385;3385;004B 0042;004B 0042; # (㎅; ㎅; ㎅; KB; KB; ) SQUARE KB +3386;3386;3386;004D 0042;004D 0042; # (㎆; ㎆; ㎆; MB; MB; ) SQUARE MB +3387;3387;3387;0047 0042;0047 0042; # (㎇; ㎇; ㎇; GB; GB; ) SQUARE GB +3388;3388;3388;0063 0061 006C;0063 0061 006C; # (㎈; ㎈; ㎈; cal; cal; ) SQUARE CAL +3389;3389;3389;006B 0063 0061 006C;006B 0063 0061 006C; # (㎉; ㎉; ㎉; kcal; kcal; ) SQUARE KCAL +338A;338A;338A;0070 0046;0070 0046; # (㎊; ㎊; ㎊; pF; pF; ) SQUARE PF +338B;338B;338B;006E 0046;006E 0046; # (㎋; ㎋; ㎋; nF; nF; ) SQUARE NF +338C;338C;338C;03BC 0046;03BC 0046; # (㎌; ㎌; ㎌; μF; μF; ) SQUARE MU F +338D;338D;338D;03BC 0067;03BC 0067; # (㎍; ㎍; ㎍; μg; μg; ) SQUARE MU G +338E;338E;338E;006D 0067;006D 0067; # (㎎; ㎎; ㎎; mg; mg; ) SQUARE MG +338F;338F;338F;006B 0067;006B 0067; # (㎏; ㎏; ㎏; kg; kg; ) SQUARE KG +3390;3390;3390;0048 007A;0048 007A; # (㎐; ㎐; ㎐; Hz; Hz; ) SQUARE HZ +3391;3391;3391;006B 0048 007A;006B 0048 007A; # (㎑; ㎑; ㎑; kHz; kHz; ) SQUARE KHZ +3392;3392;3392;004D 0048 007A;004D 0048 007A; # (㎒; ㎒; ㎒; MHz; MHz; ) SQUARE MHZ +3393;3393;3393;0047 0048 007A;0047 0048 007A; # (㎓; ㎓; ㎓; GHz; GHz; ) SQUARE GHZ +3394;3394;3394;0054 0048 007A;0054 0048 007A; # (㎔; ㎔; ㎔; THz; THz; ) SQUARE THZ +3395;3395;3395;03BC 006C;03BC 006C; # (㎕; ㎕; ㎕; μl; μl; ) SQUARE MU L +3396;3396;3396;006D 006C;006D 006C; # (㎖; ㎖; ㎖; ml; ml; ) SQUARE ML +3397;3397;3397;0064 006C;0064 006C; # (㎗; ㎗; ㎗; dl; dl; ) SQUARE DL +3398;3398;3398;006B 006C;006B 006C; # (㎘; ㎘; ㎘; kl; kl; ) SQUARE KL +3399;3399;3399;0066 006D;0066 006D; # (㎙; ㎙; ㎙; fm; fm; ) SQUARE FM +339A;339A;339A;006E 006D;006E 006D; # (㎚; ㎚; ㎚; nm; nm; ) SQUARE NM +339B;339B;339B;03BC 006D;03BC 006D; # (㎛; ㎛; ㎛; μm; μm; ) SQUARE MU M +339C;339C;339C;006D 006D;006D 006D; # (㎜; ㎜; ㎜; mm; mm; ) SQUARE MM +339D;339D;339D;0063 006D;0063 006D; # (㎝; ㎝; ㎝; cm; cm; ) SQUARE CM +339E;339E;339E;006B 006D;006B 006D; # (㎞; ㎞; ㎞; km; km; ) SQUARE KM +339F;339F;339F;006D 006D 0032;006D 006D 0032; # (㎟; ㎟; ㎟; mm2; mm2; ) SQUARE MM SQUARED +33A0;33A0;33A0;0063 006D 0032;0063 006D 0032; # (㎠; ㎠; ㎠; cm2; cm2; ) SQUARE CM SQUARED +33A1;33A1;33A1;006D 0032;006D 0032; # (㎡; ㎡; ㎡; m2; m2; ) SQUARE M SQUARED +33A2;33A2;33A2;006B 006D 0032;006B 006D 0032; # (㎢; ㎢; ㎢; km2; km2; ) SQUARE KM SQUARED +33A3;33A3;33A3;006D 006D 0033;006D 006D 0033; # (㎣; ㎣; ㎣; mm3; mm3; ) SQUARE MM CUBED +33A4;33A4;33A4;0063 006D 0033;0063 006D 0033; # (㎤; ㎤; ㎤; cm3; cm3; ) SQUARE CM CUBED +33A5;33A5;33A5;006D 0033;006D 0033; # (㎥; ㎥; ㎥; m3; m3; ) SQUARE M CUBED +33A6;33A6;33A6;006B 006D 0033;006B 006D 0033; # (㎦; ㎦; ㎦; km3; km3; ) SQUARE KM CUBED +33A7;33A7;33A7;006D 2215 0073;006D 2215 0073; # (㎧; ㎧; ㎧; m∕s; m∕s; ) SQUARE M OVER S +33A8;33A8;33A8;006D 2215 0073 0032;006D 2215 0073 0032; # (㎨; ㎨; ㎨; m∕s2; m∕s2; ) SQUARE M OVER S SQUARED +33A9;33A9;33A9;0050 0061;0050 0061; # (㎩; ㎩; ㎩; Pa; Pa; ) SQUARE PA +33AA;33AA;33AA;006B 0050 0061;006B 0050 0061; # (㎪; ㎪; ㎪; kPa; kPa; ) SQUARE KPA +33AB;33AB;33AB;004D 0050 0061;004D 0050 0061; # (㎫; ㎫; ㎫; MPa; MPa; ) SQUARE MPA +33AC;33AC;33AC;0047 0050 0061;0047 0050 0061; # (㎬; ㎬; ㎬; GPa; GPa; ) SQUARE GPA +33AD;33AD;33AD;0072 0061 0064;0072 0061 0064; # (㎭; ㎭; ㎭; rad; rad; ) SQUARE RAD +33AE;33AE;33AE;0072 0061 0064 2215 0073;0072 0061 0064 2215 0073; # (㎮; ㎮; ㎮; rad∕s; rad∕s; ) SQUARE RAD OVER S +33AF;33AF;33AF;0072 0061 0064 2215 0073 0032;0072 0061 0064 2215 0073 0032; # (㎯; ㎯; ㎯; rad∕s2; rad∕s2; ) SQUARE RAD OVER S SQUARED +33B0;33B0;33B0;0070 0073;0070 0073; # (㎰; ㎰; ㎰; ps; ps; ) SQUARE PS +33B1;33B1;33B1;006E 0073;006E 0073; # (㎱; ㎱; ㎱; ns; ns; ) SQUARE NS +33B2;33B2;33B2;03BC 0073;03BC 0073; # (㎲; ㎲; ㎲; μs; μs; ) SQUARE MU S +33B3;33B3;33B3;006D 0073;006D 0073; # (㎳; ㎳; ㎳; ms; ms; ) SQUARE MS +33B4;33B4;33B4;0070 0056;0070 0056; # (㎴; ㎴; ㎴; pV; pV; ) SQUARE PV +33B5;33B5;33B5;006E 0056;006E 0056; # (㎵; ㎵; ㎵; nV; nV; ) SQUARE NV +33B6;33B6;33B6;03BC 0056;03BC 0056; # (㎶; ㎶; ㎶; μV; μV; ) SQUARE MU V +33B7;33B7;33B7;006D 0056;006D 0056; # (㎷; ㎷; ㎷; mV; mV; ) SQUARE MV +33B8;33B8;33B8;006B 0056;006B 0056; # (㎸; ㎸; ㎸; kV; kV; ) SQUARE KV +33B9;33B9;33B9;004D 0056;004D 0056; # (㎹; ㎹; ㎹; MV; MV; ) SQUARE MV MEGA +33BA;33BA;33BA;0070 0057;0070 0057; # (㎺; ㎺; ㎺; pW; pW; ) SQUARE PW +33BB;33BB;33BB;006E 0057;006E 0057; # (㎻; ㎻; ㎻; nW; nW; ) SQUARE NW +33BC;33BC;33BC;03BC 0057;03BC 0057; # (㎼; ㎼; ㎼; μW; μW; ) SQUARE MU W +33BD;33BD;33BD;006D 0057;006D 0057; # (㎽; ㎽; ㎽; mW; mW; ) SQUARE MW +33BE;33BE;33BE;006B 0057;006B 0057; # (㎾; ㎾; ㎾; kW; kW; ) SQUARE KW +33BF;33BF;33BF;004D 0057;004D 0057; # (㎿; ㎿; ㎿; MW; MW; ) SQUARE MW MEGA +33C0;33C0;33C0;006B 03A9;006B 03A9; # (㏀; ㏀; ㏀; kΩ; kΩ; ) SQUARE K OHM +33C1;33C1;33C1;004D 03A9;004D 03A9; # (㏁; ㏁; ㏁; MΩ; MΩ; ) SQUARE M OHM +33C2;33C2;33C2;0061 002E 006D 002E;0061 002E 006D 002E; # (㏂; ㏂; ㏂; a.m.; a.m.; ) SQUARE AM +33C3;33C3;33C3;0042 0071;0042 0071; # (㏃; ㏃; ㏃; Bq; Bq; ) SQUARE BQ +33C4;33C4;33C4;0063 0063;0063 0063; # (㏄; ㏄; ㏄; cc; cc; ) SQUARE CC +33C5;33C5;33C5;0063 0064;0063 0064; # (㏅; ㏅; ㏅; cd; cd; ) SQUARE CD +33C6;33C6;33C6;0043 2215 006B 0067;0043 2215 006B 0067; # (㏆; ㏆; ㏆; C∕kg; C∕kg; ) SQUARE C OVER KG +33C7;33C7;33C7;0043 006F 002E;0043 006F 002E; # (㏇; ㏇; ㏇; Co.; Co.; ) SQUARE CO +33C8;33C8;33C8;0064 0042;0064 0042; # (㏈; ㏈; ㏈; dB; dB; ) SQUARE DB +33C9;33C9;33C9;0047 0079;0047 0079; # (㏉; ㏉; ㏉; Gy; Gy; ) SQUARE GY +33CA;33CA;33CA;0068 0061;0068 0061; # (㏊; ㏊; ㏊; ha; ha; ) SQUARE HA +33CB;33CB;33CB;0048 0050;0048 0050; # (㏋; ㏋; ㏋; HP; HP; ) SQUARE HP +33CC;33CC;33CC;0069 006E;0069 006E; # (㏌; ㏌; ㏌; in; in; ) SQUARE IN +33CD;33CD;33CD;004B 004B;004B 004B; # (㏍; ㏍; ㏍; KK; KK; ) SQUARE KK +33CE;33CE;33CE;004B 004D;004B 004D; # (㏎; ㏎; ㏎; KM; KM; ) SQUARE KM CAPITAL +33CF;33CF;33CF;006B 0074;006B 0074; # (㏏; ㏏; ㏏; kt; kt; ) SQUARE KT +33D0;33D0;33D0;006C 006D;006C 006D; # (㏐; ㏐; ㏐; lm; lm; ) SQUARE LM +33D1;33D1;33D1;006C 006E;006C 006E; # (㏑; ㏑; ㏑; ln; ln; ) SQUARE LN +33D2;33D2;33D2;006C 006F 0067;006C 006F 0067; # (㏒; ㏒; ㏒; log; log; ) SQUARE LOG +33D3;33D3;33D3;006C 0078;006C 0078; # (㏓; ㏓; ㏓; lx; lx; ) SQUARE LX +33D4;33D4;33D4;006D 0062;006D 0062; # (㏔; ㏔; ㏔; mb; mb; ) SQUARE MB SMALL +33D5;33D5;33D5;006D 0069 006C;006D 0069 006C; # (㏕; ㏕; ㏕; mil; mil; ) SQUARE MIL +33D6;33D6;33D6;006D 006F 006C;006D 006F 006C; # (㏖; ㏖; ㏖; mol; mol; ) SQUARE MOL +33D7;33D7;33D7;0050 0048;0050 0048; # (㏗; ㏗; ㏗; PH; PH; ) SQUARE PH +33D8;33D8;33D8;0070 002E 006D 002E;0070 002E 006D 002E; # (㏘; ㏘; ㏘; p.m.; p.m.; ) SQUARE PM +33D9;33D9;33D9;0050 0050 004D;0050 0050 004D; # (㏙; ㏙; ㏙; PPM; PPM; ) SQUARE PPM +33DA;33DA;33DA;0050 0052;0050 0052; # (㏚; ㏚; ㏚; PR; PR; ) SQUARE PR +33DB;33DB;33DB;0073 0072;0073 0072; # (㏛; ㏛; ㏛; sr; sr; ) SQUARE SR +33DC;33DC;33DC;0053 0076;0053 0076; # (㏜; ㏜; ㏜; Sv; Sv; ) SQUARE SV +33DD;33DD;33DD;0057 0062;0057 0062; # (㏝; ㏝; ㏝; Wb; Wb; ) SQUARE WB +33E0;33E0;33E0;0031 65E5;0031 65E5; # (㏠; ㏠; ㏠; 1日; 1日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY ONE +33E1;33E1;33E1;0032 65E5;0032 65E5; # (㏡; ㏡; ㏡; 2日; 2日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWO +33E2;33E2;33E2;0033 65E5;0033 65E5; # (㏢; ㏢; ㏢; 3日; 3日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY THREE +33E3;33E3;33E3;0034 65E5;0034 65E5; # (㏣; ㏣; ㏣; 4日; 4日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY FOUR +33E4;33E4;33E4;0035 65E5;0035 65E5; # (㏤; ㏤; ㏤; 5日; 5日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY FIVE +33E5;33E5;33E5;0036 65E5;0036 65E5; # (㏥; ㏥; ㏥; 6日; 6日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY SIX +33E6;33E6;33E6;0037 65E5;0037 65E5; # (㏦; ㏦; ㏦; 7日; 7日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY SEVEN +33E7;33E7;33E7;0038 65E5;0038 65E5; # (㏧; ㏧; ㏧; 8日; 8日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY EIGHT +33E8;33E8;33E8;0039 65E5;0039 65E5; # (㏨; ㏨; ㏨; 9日; 9日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY NINE +33E9;33E9;33E9;0031 0030 65E5;0031 0030 65E5; # (㏩; ㏩; ㏩; 10日; 10日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TEN +33EA;33EA;33EA;0031 0031 65E5;0031 0031 65E5; # (㏪; ㏪; ㏪; 11日; 11日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY ELEVEN +33EB;33EB;33EB;0031 0032 65E5;0031 0032 65E5; # (㏫; ㏫; ㏫; 12日; 12日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWELVE +33EC;33EC;33EC;0031 0033 65E5;0031 0033 65E5; # (㏬; ㏬; ㏬; 13日; 13日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY THIRTEEN +33ED;33ED;33ED;0031 0034 65E5;0031 0034 65E5; # (㏭; ㏭; ㏭; 14日; 14日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY FOURTEEN +33EE;33EE;33EE;0031 0035 65E5;0031 0035 65E5; # (㏮; ㏮; ㏮; 15日; 15日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY FIFTEEN +33EF;33EF;33EF;0031 0036 65E5;0031 0036 65E5; # (㏯; ㏯; ㏯; 16日; 16日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY SIXTEEN +33F0;33F0;33F0;0031 0037 65E5;0031 0037 65E5; # (㏰; ㏰; ㏰; 17日; 17日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY SEVENTEEN +33F1;33F1;33F1;0031 0038 65E5;0031 0038 65E5; # (㏱; ㏱; ㏱; 18日; 18日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY EIGHTEEN +33F2;33F2;33F2;0031 0039 65E5;0031 0039 65E5; # (㏲; ㏲; ㏲; 19日; 19日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY NINETEEN +33F3;33F3;33F3;0032 0030 65E5;0032 0030 65E5; # (㏳; ㏳; ㏳; 20日; 20日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWENTY +33F4;33F4;33F4;0032 0031 65E5;0032 0031 65E5; # (㏴; ㏴; ㏴; 21日; 21日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWENTY-ONE +33F5;33F5;33F5;0032 0032 65E5;0032 0032 65E5; # (㏵; ㏵; ㏵; 22日; 22日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWENTY-TWO +33F6;33F6;33F6;0032 0033 65E5;0032 0033 65E5; # (㏶; ㏶; ㏶; 23日; 23日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWENTY-THREE +33F7;33F7;33F7;0032 0034 65E5;0032 0034 65E5; # (㏷; ㏷; ㏷; 24日; 24日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWENTY-FOUR +33F8;33F8;33F8;0032 0035 65E5;0032 0035 65E5; # (㏸; ㏸; ㏸; 25日; 25日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWENTY-FIVE +33F9;33F9;33F9;0032 0036 65E5;0032 0036 65E5; # (㏹; ㏹; ㏹; 26日; 26日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWENTY-SIX +33FA;33FA;33FA;0032 0037 65E5;0032 0037 65E5; # (㏺; ㏺; ㏺; 27日; 27日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWENTY-SEVEN +33FB;33FB;33FB;0032 0038 65E5;0032 0038 65E5; # (㏻; ㏻; ㏻; 28日; 28日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWENTY-EIGHT +33FC;33FC;33FC;0032 0039 65E5;0032 0039 65E5; # (㏼; ㏼; ㏼; 29日; 29日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY TWENTY-NINE +33FD;33FD;33FD;0033 0030 65E5;0033 0030 65E5; # (㏽; ㏽; ㏽; 30日; 30日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY THIRTY +33FE;33FE;33FE;0033 0031 65E5;0033 0031 65E5; # (㏾; ㏾; ㏾; 31日; 31日; ) IDEOGRAPHIC TELEGRAPH SYMBOL FOR DAY THIRTY-ONE +AC00;AC00;1100 1161;AC00;1100 1161; # (가; 가; 가; 가; 가; ) HANGUL SYLLABLE GA +AC01;AC01;1100 1161 11A8;AC01;1100 1161 11A8; # (각; 각; 각; 각; 각; ) HANGUL SYLLABLE GAG +AC02;AC02;1100 1161 11A9;AC02;1100 1161 11A9; # (갂; 갂; 갂; 갂; 갂; ) HANGUL SYLLABLE GAGG +AC03;AC03;1100 1161 11AA;AC03;1100 1161 11AA; # (갃; 갃; 갃; 갃; 갃; ) HANGUL SYLLABLE GAGS +AC04;AC04;1100 1161 11AB;AC04;1100 1161 11AB; # (간; 간; 간; 간; 간; ) HANGUL SYLLABLE GAN +AC05;AC05;1100 1161 11AC;AC05;1100 1161 11AC; # (갅; 갅; 갅; 갅; 갅; ) HANGUL SYLLABLE GANJ +AC06;AC06;1100 1161 11AD;AC06;1100 1161 11AD; # (갆; 갆; 갆; 갆; 갆; ) HANGUL SYLLABLE GANH +AC07;AC07;1100 1161 11AE;AC07;1100 1161 11AE; # (갇; 갇; 갇; 갇; 갇; ) HANGUL SYLLABLE GAD +AC08;AC08;1100 1161 11AF;AC08;1100 1161 11AF; # (갈; 갈; 갈; 갈; 갈; ) HANGUL SYLLABLE GAL +AC09;AC09;1100 1161 11B0;AC09;1100 1161 11B0; # (갉; 갉; 갉; 갉; 갉; ) HANGUL SYLLABLE GALG +AC0A;AC0A;1100 1161 11B1;AC0A;1100 1161 11B1; # (갊; 갊; 갊; 갊; 갊; ) HANGUL SYLLABLE GALM +AC0B;AC0B;1100 1161 11B2;AC0B;1100 1161 11B2; # (갋; 갋; 갋; 갋; 갋; ) HANGUL SYLLABLE GALB +AC0C;AC0C;1100 1161 11B3;AC0C;1100 1161 11B3; # (갌; 갌; 갌; 갌; 갌; ) HANGUL SYLLABLE GALS +AC0D;AC0D;1100 1161 11B4;AC0D;1100 1161 11B4; # (갍; 갍; 갍; 갍; 갍; ) HANGUL SYLLABLE GALT +AC0E;AC0E;1100 1161 11B5;AC0E;1100 1161 11B5; # (갎; 갎; 갎; 갎; 갎; ) HANGUL SYLLABLE GALP +AC0F;AC0F;1100 1161 11B6;AC0F;1100 1161 11B6; # (갏; 갏; 갏; 갏; 갏; ) HANGUL SYLLABLE GALH +AC10;AC10;1100 1161 11B7;AC10;1100 1161 11B7; # (감; 감; 감; 감; 감; ) HANGUL SYLLABLE GAM +AC11;AC11;1100 1161 11B8;AC11;1100 1161 11B8; # (갑; 갑; 갑; 갑; 갑; ) HANGUL SYLLABLE GAB +AC12;AC12;1100 1161 11B9;AC12;1100 1161 11B9; # (값; 값; 값; 값; 값; ) HANGUL SYLLABLE GABS +AC13;AC13;1100 1161 11BA;AC13;1100 1161 11BA; # (갓; 갓; 갓; 갓; 갓; ) HANGUL SYLLABLE GAS +AC14;AC14;1100 1161 11BB;AC14;1100 1161 11BB; # (갔; 갔; 갔; 갔; 갔; ) HANGUL SYLLABLE GASS +AC15;AC15;1100 1161 11BC;AC15;1100 1161 11BC; # (강; 강; 강; 강; 강; ) HANGUL SYLLABLE GANG +AC16;AC16;1100 1161 11BD;AC16;1100 1161 11BD; # (갖; 갖; 갖; 갖; 갖; ) HANGUL SYLLABLE GAJ +AC17;AC17;1100 1161 11BE;AC17;1100 1161 11BE; # (갗; 갗; 갗; 갗; 갗; ) HANGUL SYLLABLE GAC +AC18;AC18;1100 1161 11BF;AC18;1100 1161 11BF; # (갘; 갘; 갘; 갘; 갘; ) HANGUL SYLLABLE GAK +AC19;AC19;1100 1161 11C0;AC19;1100 1161 11C0; # (같; 같; 같; 같; 같; ) HANGUL SYLLABLE GAT +AC1A;AC1A;1100 1161 11C1;AC1A;1100 1161 11C1; # (갚; 갚; 갚; 갚; 갚; ) HANGUL SYLLABLE GAP +AC1B;AC1B;1100 1161 11C2;AC1B;1100 1161 11C2; # (갛; 갛; 갛; 갛; 갛; ) HANGUL SYLLABLE GAH +AC1C;AC1C;1100 1162;AC1C;1100 1162; # (개; 개; 개; 개; 개; ) HANGUL SYLLABLE GAE +AC1D;AC1D;1100 1162 11A8;AC1D;1100 1162 11A8; # (객; 객; 객; 객; 객; ) HANGUL SYLLABLE GAEG +AC1E;AC1E;1100 1162 11A9;AC1E;1100 1162 11A9; # (갞; 갞; 갞; 갞; 갞; ) HANGUL SYLLABLE GAEGG +AC1F;AC1F;1100 1162 11AA;AC1F;1100 1162 11AA; # (갟; 갟; 갟; 갟; 갟; ) HANGUL SYLLABLE GAEGS +AC20;AC20;1100 1162 11AB;AC20;1100 1162 11AB; # (갠; 갠; 갠; 갠; 갠; ) HANGUL SYLLABLE GAEN +AC21;AC21;1100 1162 11AC;AC21;1100 1162 11AC; # (갡; 갡; 갡; 갡; 갡; ) HANGUL SYLLABLE GAENJ +AC22;AC22;1100 1162 11AD;AC22;1100 1162 11AD; # (갢; 갢; 갢; 갢; 갢; ) HANGUL SYLLABLE GAENH +AC23;AC23;1100 1162 11AE;AC23;1100 1162 11AE; # (갣; 갣; 갣; 갣; 갣; ) HANGUL SYLLABLE GAED +AC24;AC24;1100 1162 11AF;AC24;1100 1162 11AF; # (갤; 갤; 갤; 갤; 갤; ) HANGUL SYLLABLE GAEL +AC25;AC25;1100 1162 11B0;AC25;1100 1162 11B0; # (갥; 갥; 갥; 갥; 갥; ) HANGUL SYLLABLE GAELG +AC26;AC26;1100 1162 11B1;AC26;1100 1162 11B1; # (갦; 갦; 갦; 갦; 갦; ) HANGUL SYLLABLE GAELM +AC27;AC27;1100 1162 11B2;AC27;1100 1162 11B2; # (갧; 갧; 갧; 갧; 갧; ) HANGUL SYLLABLE GAELB +AC28;AC28;1100 1162 11B3;AC28;1100 1162 11B3; # (갨; 갨; 갨; 갨; 갨; ) HANGUL SYLLABLE GAELS +AC29;AC29;1100 1162 11B4;AC29;1100 1162 11B4; # (갩; 갩; 갩; 갩; 갩; ) HANGUL SYLLABLE GAELT +AC2A;AC2A;1100 1162 11B5;AC2A;1100 1162 11B5; # (갪; 갪; 갪; 갪; 갪; ) HANGUL SYLLABLE GAELP +AC2B;AC2B;1100 1162 11B6;AC2B;1100 1162 11B6; # (갫; 갫; 갫; 갫; 갫; ) HANGUL SYLLABLE GAELH +AC2C;AC2C;1100 1162 11B7;AC2C;1100 1162 11B7; # (갬; 갬; 갬; 갬; 갬; ) HANGUL SYLLABLE GAEM +AC2D;AC2D;1100 1162 11B8;AC2D;1100 1162 11B8; # (갭; 갭; 갭; 갭; 갭; ) HANGUL SYLLABLE GAEB +AC2E;AC2E;1100 1162 11B9;AC2E;1100 1162 11B9; # (갮; 갮; 갮; 갮; 갮; ) HANGUL SYLLABLE GAEBS +AC2F;AC2F;1100 1162 11BA;AC2F;1100 1162 11BA; # (갯; 갯; 갯; 갯; 갯; ) HANGUL SYLLABLE GAES +AC30;AC30;1100 1162 11BB;AC30;1100 1162 11BB; # (갰; 갰; 갰; 갰; 갰; ) HANGUL SYLLABLE GAESS +AC31;AC31;1100 1162 11BC;AC31;1100 1162 11BC; # (갱; 갱; 갱; 갱; 갱; ) HANGUL SYLLABLE GAENG +AC32;AC32;1100 1162 11BD;AC32;1100 1162 11BD; # (갲; 갲; 갲; 갲; 갲; ) HANGUL SYLLABLE GAEJ +AC33;AC33;1100 1162 11BE;AC33;1100 1162 11BE; # (갳; 갳; 갳; 갳; 갳; ) HANGUL SYLLABLE GAEC +AC34;AC34;1100 1162 11BF;AC34;1100 1162 11BF; # (갴; 갴; 갴; 갴; 갴; ) HANGUL SYLLABLE GAEK +AC35;AC35;1100 1162 11C0;AC35;1100 1162 11C0; # (갵; 갵; 갵; 갵; 갵; ) HANGUL SYLLABLE GAET +AC36;AC36;1100 1162 11C1;AC36;1100 1162 11C1; # (갶; 갶; 갶; 갶; 갶; ) HANGUL SYLLABLE GAEP +AC37;AC37;1100 1162 11C2;AC37;1100 1162 11C2; # (갷; 갷; 갷; 갷; 갷; ) HANGUL SYLLABLE GAEH +AC38;AC38;1100 1163;AC38;1100 1163; # (갸; 갸; 갸; 갸; 갸; ) HANGUL SYLLABLE GYA +AC39;AC39;1100 1163 11A8;AC39;1100 1163 11A8; # (갹; 갹; 갹; 갹; 갹; ) HANGUL SYLLABLE GYAG +AC3A;AC3A;1100 1163 11A9;AC3A;1100 1163 11A9; # (갺; 갺; 갺; 갺; 갺; ) HANGUL SYLLABLE GYAGG +AC3B;AC3B;1100 1163 11AA;AC3B;1100 1163 11AA; # (갻; 갻; 갻; 갻; 갻; ) HANGUL SYLLABLE GYAGS +AC3C;AC3C;1100 1163 11AB;AC3C;1100 1163 11AB; # (갼; 갼; 갼; 갼; 갼; ) HANGUL SYLLABLE GYAN +AC3D;AC3D;1100 1163 11AC;AC3D;1100 1163 11AC; # (갽; 갽; 갽; 갽; 갽; ) HANGUL SYLLABLE GYANJ +AC3E;AC3E;1100 1163 11AD;AC3E;1100 1163 11AD; # (갾; 갾; 갾; 갾; 갾; ) HANGUL SYLLABLE GYANH +AC3F;AC3F;1100 1163 11AE;AC3F;1100 1163 11AE; # (갿; 갿; 갿; 갿; 갿; ) HANGUL SYLLABLE GYAD +AC40;AC40;1100 1163 11AF;AC40;1100 1163 11AF; # (걀; 걀; 걀; 걀; 걀; ) HANGUL SYLLABLE GYAL +AC41;AC41;1100 1163 11B0;AC41;1100 1163 11B0; # (걁; 걁; 걁; 걁; 걁; ) HANGUL SYLLABLE GYALG +AC42;AC42;1100 1163 11B1;AC42;1100 1163 11B1; # (걂; 걂; 걂; 걂; 걂; ) HANGUL SYLLABLE GYALM +AC43;AC43;1100 1163 11B2;AC43;1100 1163 11B2; # (걃; 걃; 걃; 걃; 걃; ) HANGUL SYLLABLE GYALB +AC44;AC44;1100 1163 11B3;AC44;1100 1163 11B3; # (걄; 걄; 걄; 걄; 걄; ) HANGUL SYLLABLE GYALS +AC45;AC45;1100 1163 11B4;AC45;1100 1163 11B4; # (걅; 걅; 걅; 걅; 걅; ) HANGUL SYLLABLE GYALT +AC46;AC46;1100 1163 11B5;AC46;1100 1163 11B5; # (걆; 걆; 걆; 걆; 걆; ) HANGUL SYLLABLE GYALP +AC47;AC47;1100 1163 11B6;AC47;1100 1163 11B6; # (걇; 걇; 걇; 걇; 걇; ) HANGUL SYLLABLE GYALH +AC48;AC48;1100 1163 11B7;AC48;1100 1163 11B7; # (걈; 걈; 걈; 걈; 걈; ) HANGUL SYLLABLE GYAM +AC49;AC49;1100 1163 11B8;AC49;1100 1163 11B8; # (걉; 걉; 걉; 걉; 걉; ) HANGUL SYLLABLE GYAB +AC4A;AC4A;1100 1163 11B9;AC4A;1100 1163 11B9; # (걊; 걊; 걊; 걊; 걊; ) HANGUL SYLLABLE GYABS +AC4B;AC4B;1100 1163 11BA;AC4B;1100 1163 11BA; # (걋; 걋; 걋; 걋; 걋; ) HANGUL SYLLABLE GYAS +AC4C;AC4C;1100 1163 11BB;AC4C;1100 1163 11BB; # (걌; 걌; 걌; 걌; 걌; ) HANGUL SYLLABLE GYASS +AC4D;AC4D;1100 1163 11BC;AC4D;1100 1163 11BC; # (걍; 걍; 걍; 걍; 걍; ) HANGUL SYLLABLE GYANG +AC4E;AC4E;1100 1163 11BD;AC4E;1100 1163 11BD; # (걎; 걎; 걎; 걎; 걎; ) HANGUL SYLLABLE GYAJ +AC4F;AC4F;1100 1163 11BE;AC4F;1100 1163 11BE; # (걏; 걏; 걏; 걏; 걏; ) HANGUL SYLLABLE GYAC +AC50;AC50;1100 1163 11BF;AC50;1100 1163 11BF; # (걐; 걐; 걐; 걐; 걐; ) HANGUL SYLLABLE GYAK +AC51;AC51;1100 1163 11C0;AC51;1100 1163 11C0; # (걑; 걑; 걑; 걑; 걑; ) HANGUL SYLLABLE GYAT +AC52;AC52;1100 1163 11C1;AC52;1100 1163 11C1; # (걒; 걒; 걒; 걒; 걒; ) HANGUL SYLLABLE GYAP +AC53;AC53;1100 1163 11C2;AC53;1100 1163 11C2; # (걓; 걓; 걓; 걓; 걓; ) HANGUL SYLLABLE GYAH +AC54;AC54;1100 1164;AC54;1100 1164; # (걔; 걔; 걔; 걔; 걔; ) HANGUL SYLLABLE GYAE +AC55;AC55;1100 1164 11A8;AC55;1100 1164 11A8; # (걕; 걕; 걕; 걕; 걕; ) HANGUL SYLLABLE GYAEG +AC56;AC56;1100 1164 11A9;AC56;1100 1164 11A9; # (걖; 걖; 걖; 걖; 걖; ) HANGUL SYLLABLE GYAEGG +AC57;AC57;1100 1164 11AA;AC57;1100 1164 11AA; # (걗; 걗; 걗; 걗; 걗; ) HANGUL SYLLABLE GYAEGS +AC58;AC58;1100 1164 11AB;AC58;1100 1164 11AB; # (걘; 걘; 걘; 걘; 걘; ) HANGUL SYLLABLE GYAEN +AC59;AC59;1100 1164 11AC;AC59;1100 1164 11AC; # (걙; 걙; 걙; 걙; 걙; ) HANGUL SYLLABLE GYAENJ +AC5A;AC5A;1100 1164 11AD;AC5A;1100 1164 11AD; # (걚; 걚; 걚; 걚; 걚; ) HANGUL SYLLABLE GYAENH +AC5B;AC5B;1100 1164 11AE;AC5B;1100 1164 11AE; # (걛; 걛; 걛; 걛; 걛; ) HANGUL SYLLABLE GYAED +AC5C;AC5C;1100 1164 11AF;AC5C;1100 1164 11AF; # (걜; 걜; 걜; 걜; 걜; ) HANGUL SYLLABLE GYAEL +AC5D;AC5D;1100 1164 11B0;AC5D;1100 1164 11B0; # (걝; 걝; 걝; 걝; 걝; ) HANGUL SYLLABLE GYAELG +AC5E;AC5E;1100 1164 11B1;AC5E;1100 1164 11B1; # (걞; 걞; 걞; 걞; 걞; ) HANGUL SYLLABLE GYAELM +AC5F;AC5F;1100 1164 11B2;AC5F;1100 1164 11B2; # (걟; 걟; 걟; 걟; 걟; ) HANGUL SYLLABLE GYAELB +AC60;AC60;1100 1164 11B3;AC60;1100 1164 11B3; # (걠; 걠; 걠; 걠; 걠; ) HANGUL SYLLABLE GYAELS +AC61;AC61;1100 1164 11B4;AC61;1100 1164 11B4; # (걡; 걡; 걡; 걡; 걡; ) HANGUL SYLLABLE GYAELT +AC62;AC62;1100 1164 11B5;AC62;1100 1164 11B5; # (걢; 걢; 걢; 걢; 걢; ) HANGUL SYLLABLE GYAELP +AC63;AC63;1100 1164 11B6;AC63;1100 1164 11B6; # (걣; 걣; 걣; 걣; 걣; ) HANGUL SYLLABLE GYAELH +AC64;AC64;1100 1164 11B7;AC64;1100 1164 11B7; # (걤; 걤; 걤; 걤; 걤; ) HANGUL SYLLABLE GYAEM +AC65;AC65;1100 1164 11B8;AC65;1100 1164 11B8; # (걥; 걥; 걥; 걥; 걥; ) HANGUL SYLLABLE GYAEB +AC66;AC66;1100 1164 11B9;AC66;1100 1164 11B9; # (걦; 걦; 걦; 걦; 걦; ) HANGUL SYLLABLE GYAEBS +AC67;AC67;1100 1164 11BA;AC67;1100 1164 11BA; # (걧; 걧; 걧; 걧; 걧; ) HANGUL SYLLABLE GYAES +AC68;AC68;1100 1164 11BB;AC68;1100 1164 11BB; # (걨; 걨; 걨; 걨; 걨; ) HANGUL SYLLABLE GYAESS +AC69;AC69;1100 1164 11BC;AC69;1100 1164 11BC; # (걩; 걩; 걩; 걩; 걩; ) HANGUL SYLLABLE GYAENG +AC6A;AC6A;1100 1164 11BD;AC6A;1100 1164 11BD; # (걪; 걪; 걪; 걪; 걪; ) HANGUL SYLLABLE GYAEJ +AC6B;AC6B;1100 1164 11BE;AC6B;1100 1164 11BE; # (걫; 걫; 걫; 걫; 걫; ) HANGUL SYLLABLE GYAEC +AC6C;AC6C;1100 1164 11BF;AC6C;1100 1164 11BF; # (걬; 걬; 걬; 걬; 걬; ) HANGUL SYLLABLE GYAEK +AC6D;AC6D;1100 1164 11C0;AC6D;1100 1164 11C0; # (걭; 걭; 걭; 걭; 걭; ) HANGUL SYLLABLE GYAET +AC6E;AC6E;1100 1164 11C1;AC6E;1100 1164 11C1; # (걮; 걮; 걮; 걮; 걮; ) HANGUL SYLLABLE GYAEP +AC6F;AC6F;1100 1164 11C2;AC6F;1100 1164 11C2; # (걯; 걯; 걯; 걯; 걯; ) HANGUL SYLLABLE GYAEH +AC70;AC70;1100 1165;AC70;1100 1165; # (거; 거; 거; 거; 거; ) HANGUL SYLLABLE GEO +AC71;AC71;1100 1165 11A8;AC71;1100 1165 11A8; # (걱; 걱; 걱; 걱; 걱; ) HANGUL SYLLABLE GEOG +AC72;AC72;1100 1165 11A9;AC72;1100 1165 11A9; # (걲; 걲; 걲; 걲; 걲; ) HANGUL SYLLABLE GEOGG +AC73;AC73;1100 1165 11AA;AC73;1100 1165 11AA; # (걳; 걳; 걳; 걳; 걳; ) HANGUL SYLLABLE GEOGS +AC74;AC74;1100 1165 11AB;AC74;1100 1165 11AB; # (건; 건; 건; 건; 건; ) HANGUL SYLLABLE GEON +AC75;AC75;1100 1165 11AC;AC75;1100 1165 11AC; # (걵; 걵; 걵; 걵; 걵; ) HANGUL SYLLABLE GEONJ +AC76;AC76;1100 1165 11AD;AC76;1100 1165 11AD; # (걶; 걶; 걶; 걶; 걶; ) HANGUL SYLLABLE GEONH +AC77;AC77;1100 1165 11AE;AC77;1100 1165 11AE; # (걷; 걷; 걷; 걷; 걷; ) HANGUL SYLLABLE GEOD +AC78;AC78;1100 1165 11AF;AC78;1100 1165 11AF; # (걸; 걸; 걸; 걸; 걸; ) HANGUL SYLLABLE GEOL +AC79;AC79;1100 1165 11B0;AC79;1100 1165 11B0; # (걹; 걹; 걹; 걹; 걹; ) HANGUL SYLLABLE GEOLG +AC7A;AC7A;1100 1165 11B1;AC7A;1100 1165 11B1; # (걺; 걺; 걺; 걺; 걺; ) HANGUL SYLLABLE GEOLM +AC7B;AC7B;1100 1165 11B2;AC7B;1100 1165 11B2; # (걻; 걻; 걻; 걻; 걻; ) HANGUL SYLLABLE GEOLB +AC7C;AC7C;1100 1165 11B3;AC7C;1100 1165 11B3; # (걼; 걼; 걼; 걼; 걼; ) HANGUL SYLLABLE GEOLS +AC7D;AC7D;1100 1165 11B4;AC7D;1100 1165 11B4; # (걽; 걽; 걽; 걽; 걽; ) HANGUL SYLLABLE GEOLT +AC7E;AC7E;1100 1165 11B5;AC7E;1100 1165 11B5; # (걾; 걾; 걾; 걾; 걾; ) HANGUL SYLLABLE GEOLP +AC7F;AC7F;1100 1165 11B6;AC7F;1100 1165 11B6; # (걿; 걿; 걿; 걿; 걿; ) HANGUL SYLLABLE GEOLH +AC80;AC80;1100 1165 11B7;AC80;1100 1165 11B7; # (검; 검; 검; 검; 검; ) HANGUL SYLLABLE GEOM +AC81;AC81;1100 1165 11B8;AC81;1100 1165 11B8; # (겁; 겁; 겁; 겁; 겁; ) HANGUL SYLLABLE GEOB +AC82;AC82;1100 1165 11B9;AC82;1100 1165 11B9; # (겂; 겂; 겂; 겂; 겂; ) HANGUL SYLLABLE GEOBS +AC83;AC83;1100 1165 11BA;AC83;1100 1165 11BA; # (것; 것; 것; 것; 것; ) HANGUL SYLLABLE GEOS +AC84;AC84;1100 1165 11BB;AC84;1100 1165 11BB; # (겄; 겄; 겄; 겄; 겄; ) HANGUL SYLLABLE GEOSS +AC85;AC85;1100 1165 11BC;AC85;1100 1165 11BC; # (겅; 겅; 겅; 겅; 겅; ) HANGUL SYLLABLE GEONG +AC86;AC86;1100 1165 11BD;AC86;1100 1165 11BD; # (겆; 겆; 겆; 겆; 겆; ) HANGUL SYLLABLE GEOJ +AC87;AC87;1100 1165 11BE;AC87;1100 1165 11BE; # (겇; 겇; 겇; 겇; 겇; ) HANGUL SYLLABLE GEOC +AC88;AC88;1100 1165 11BF;AC88;1100 1165 11BF; # (겈; 겈; 겈; 겈; 겈; ) HANGUL SYLLABLE GEOK +AC89;AC89;1100 1165 11C0;AC89;1100 1165 11C0; # (겉; 겉; 겉; 겉; 겉; ) HANGUL SYLLABLE GEOT +AC8A;AC8A;1100 1165 11C1;AC8A;1100 1165 11C1; # (겊; 겊; 겊; 겊; 겊; ) HANGUL SYLLABLE GEOP +AC8B;AC8B;1100 1165 11C2;AC8B;1100 1165 11C2; # (겋; 겋; 겋; 겋; 겋; ) HANGUL SYLLABLE GEOH +AC8C;AC8C;1100 1166;AC8C;1100 1166; # (게; 게; 게; 게; 게; ) HANGUL SYLLABLE GE +AC8D;AC8D;1100 1166 11A8;AC8D;1100 1166 11A8; # (겍; 겍; 겍; 겍; 겍; ) HANGUL SYLLABLE GEG +AC8E;AC8E;1100 1166 11A9;AC8E;1100 1166 11A9; # (겎; 겎; 겎; 겎; 겎; ) HANGUL SYLLABLE GEGG +AC8F;AC8F;1100 1166 11AA;AC8F;1100 1166 11AA; # (겏; 겏; 겏; 겏; 겏; ) HANGUL SYLLABLE GEGS +AC90;AC90;1100 1166 11AB;AC90;1100 1166 11AB; # (겐; 겐; 겐; 겐; 겐; ) HANGUL SYLLABLE GEN +AC91;AC91;1100 1166 11AC;AC91;1100 1166 11AC; # (겑; 겑; 겑; 겑; 겑; ) HANGUL SYLLABLE GENJ +AC92;AC92;1100 1166 11AD;AC92;1100 1166 11AD; # (겒; 겒; 겒; 겒; 겒; ) HANGUL SYLLABLE GENH +AC93;AC93;1100 1166 11AE;AC93;1100 1166 11AE; # (겓; 겓; 겓; 겓; 겓; ) HANGUL SYLLABLE GED +AC94;AC94;1100 1166 11AF;AC94;1100 1166 11AF; # (겔; 겔; 겔; 겔; 겔; ) HANGUL SYLLABLE GEL +AC95;AC95;1100 1166 11B0;AC95;1100 1166 11B0; # (겕; 겕; 겕; 겕; 겕; ) HANGUL SYLLABLE GELG +AC96;AC96;1100 1166 11B1;AC96;1100 1166 11B1; # (겖; 겖; 겖; 겖; 겖; ) HANGUL SYLLABLE GELM +AC97;AC97;1100 1166 11B2;AC97;1100 1166 11B2; # (겗; 겗; 겗; 겗; 겗; ) HANGUL SYLLABLE GELB +AC98;AC98;1100 1166 11B3;AC98;1100 1166 11B3; # (겘; 겘; 겘; 겘; 겘; ) HANGUL SYLLABLE GELS +AC99;AC99;1100 1166 11B4;AC99;1100 1166 11B4; # (겙; 겙; 겙; 겙; 겙; ) HANGUL SYLLABLE GELT +AC9A;AC9A;1100 1166 11B5;AC9A;1100 1166 11B5; # (겚; 겚; 겚; 겚; 겚; ) HANGUL SYLLABLE GELP +AC9B;AC9B;1100 1166 11B6;AC9B;1100 1166 11B6; # (겛; 겛; 겛; 겛; 겛; ) HANGUL SYLLABLE GELH +AC9C;AC9C;1100 1166 11B7;AC9C;1100 1166 11B7; # (겜; 겜; 겜; 겜; 겜; ) HANGUL SYLLABLE GEM +AC9D;AC9D;1100 1166 11B8;AC9D;1100 1166 11B8; # (겝; 겝; 겝; 겝; 겝; ) HANGUL SYLLABLE GEB +AC9E;AC9E;1100 1166 11B9;AC9E;1100 1166 11B9; # (겞; 겞; 겞; 겞; 겞; ) HANGUL SYLLABLE GEBS +AC9F;AC9F;1100 1166 11BA;AC9F;1100 1166 11BA; # (겟; 겟; 겟; 겟; 겟; ) HANGUL SYLLABLE GES +ACA0;ACA0;1100 1166 11BB;ACA0;1100 1166 11BB; # (겠; 겠; 겠; 겠; 겠; ) HANGUL SYLLABLE GESS +ACA1;ACA1;1100 1166 11BC;ACA1;1100 1166 11BC; # (겡; 겡; 겡; 겡; 겡; ) HANGUL SYLLABLE GENG +ACA2;ACA2;1100 1166 11BD;ACA2;1100 1166 11BD; # (겢; 겢; 겢; 겢; 겢; ) HANGUL SYLLABLE GEJ +ACA3;ACA3;1100 1166 11BE;ACA3;1100 1166 11BE; # (겣; 겣; 겣; 겣; 겣; ) HANGUL SYLLABLE GEC +ACA4;ACA4;1100 1166 11BF;ACA4;1100 1166 11BF; # (겤; 겤; 겤; 겤; 겤; ) HANGUL SYLLABLE GEK +ACA5;ACA5;1100 1166 11C0;ACA5;1100 1166 11C0; # (겥; 겥; 겥; 겥; 겥; ) HANGUL SYLLABLE GET +ACA6;ACA6;1100 1166 11C1;ACA6;1100 1166 11C1; # (겦; 겦; 겦; 겦; 겦; ) HANGUL SYLLABLE GEP +ACA7;ACA7;1100 1166 11C2;ACA7;1100 1166 11C2; # (겧; 겧; 겧; 겧; 겧; ) HANGUL SYLLABLE GEH +ACA8;ACA8;1100 1167;ACA8;1100 1167; # (겨; 겨; 겨; 겨; 겨; ) HANGUL SYLLABLE GYEO +ACA9;ACA9;1100 1167 11A8;ACA9;1100 1167 11A8; # (격; 격; 격; 격; 격; ) HANGUL SYLLABLE GYEOG +ACAA;ACAA;1100 1167 11A9;ACAA;1100 1167 11A9; # (겪; 겪; 겪; 겪; 겪; ) HANGUL SYLLABLE GYEOGG +ACAB;ACAB;1100 1167 11AA;ACAB;1100 1167 11AA; # (겫; 겫; 겫; 겫; 겫; ) HANGUL SYLLABLE GYEOGS +ACAC;ACAC;1100 1167 11AB;ACAC;1100 1167 11AB; # (견; 견; 견; 견; 견; ) HANGUL SYLLABLE GYEON +ACAD;ACAD;1100 1167 11AC;ACAD;1100 1167 11AC; # (겭; 겭; 겭; 겭; 겭; ) HANGUL SYLLABLE GYEONJ +ACAE;ACAE;1100 1167 11AD;ACAE;1100 1167 11AD; # (겮; 겮; 겮; 겮; 겮; ) HANGUL SYLLABLE GYEONH +ACAF;ACAF;1100 1167 11AE;ACAF;1100 1167 11AE; # (겯; 겯; 겯; 겯; 겯; ) HANGUL SYLLABLE GYEOD +ACB0;ACB0;1100 1167 11AF;ACB0;1100 1167 11AF; # (결; 결; 결; 결; 결; ) HANGUL SYLLABLE GYEOL +ACB1;ACB1;1100 1167 11B0;ACB1;1100 1167 11B0; # (겱; 겱; 겱; 겱; 겱; ) HANGUL SYLLABLE GYEOLG +ACB2;ACB2;1100 1167 11B1;ACB2;1100 1167 11B1; # (겲; 겲; 겲; 겲; 겲; ) HANGUL SYLLABLE GYEOLM +ACB3;ACB3;1100 1167 11B2;ACB3;1100 1167 11B2; # (겳; 겳; 겳; 겳; 겳; ) HANGUL SYLLABLE GYEOLB +ACB4;ACB4;1100 1167 11B3;ACB4;1100 1167 11B3; # (겴; 겴; 겴; 겴; 겴; ) HANGUL SYLLABLE GYEOLS +ACB5;ACB5;1100 1167 11B4;ACB5;1100 1167 11B4; # (겵; 겵; 겵; 겵; 겵; ) HANGUL SYLLABLE GYEOLT +ACB6;ACB6;1100 1167 11B5;ACB6;1100 1167 11B5; # (겶; 겶; 겶; 겶; 겶; ) HANGUL SYLLABLE GYEOLP +ACB7;ACB7;1100 1167 11B6;ACB7;1100 1167 11B6; # (겷; 겷; 겷; 겷; 겷; ) HANGUL SYLLABLE GYEOLH +ACB8;ACB8;1100 1167 11B7;ACB8;1100 1167 11B7; # (겸; 겸; 겸; 겸; 겸; ) HANGUL SYLLABLE GYEOM +ACB9;ACB9;1100 1167 11B8;ACB9;1100 1167 11B8; # (겹; 겹; 겹; 겹; 겹; ) HANGUL SYLLABLE GYEOB +ACBA;ACBA;1100 1167 11B9;ACBA;1100 1167 11B9; # (겺; 겺; 겺; 겺; 겺; ) HANGUL SYLLABLE GYEOBS +ACBB;ACBB;1100 1167 11BA;ACBB;1100 1167 11BA; # (겻; 겻; 겻; 겻; 겻; ) HANGUL SYLLABLE GYEOS +ACBC;ACBC;1100 1167 11BB;ACBC;1100 1167 11BB; # (겼; 겼; 겼; 겼; 겼; ) HANGUL SYLLABLE GYEOSS +ACBD;ACBD;1100 1167 11BC;ACBD;1100 1167 11BC; # (경; 경; 경; 경; 경; ) HANGUL SYLLABLE GYEONG +ACBE;ACBE;1100 1167 11BD;ACBE;1100 1167 11BD; # (겾; 겾; 겾; 겾; 겾; ) HANGUL SYLLABLE GYEOJ +ACBF;ACBF;1100 1167 11BE;ACBF;1100 1167 11BE; # (겿; 겿; 겿; 겿; 겿; ) HANGUL SYLLABLE GYEOC +ACC0;ACC0;1100 1167 11BF;ACC0;1100 1167 11BF; # (곀; 곀; 곀; 곀; 곀; ) HANGUL SYLLABLE GYEOK +ACC1;ACC1;1100 1167 11C0;ACC1;1100 1167 11C0; # (곁; 곁; 곁; 곁; 곁; ) HANGUL SYLLABLE GYEOT +ACC2;ACC2;1100 1167 11C1;ACC2;1100 1167 11C1; # (곂; 곂; 곂; 곂; 곂; ) HANGUL SYLLABLE GYEOP +ACC3;ACC3;1100 1167 11C2;ACC3;1100 1167 11C2; # (곃; 곃; 곃; 곃; 곃; ) HANGUL SYLLABLE GYEOH +ACC4;ACC4;1100 1168;ACC4;1100 1168; # (계; 계; 계; 계; 계; ) HANGUL SYLLABLE GYE +ACC5;ACC5;1100 1168 11A8;ACC5;1100 1168 11A8; # (곅; 곅; 곅; 곅; 곅; ) HANGUL SYLLABLE GYEG +ACC6;ACC6;1100 1168 11A9;ACC6;1100 1168 11A9; # (곆; 곆; 곆; 곆; 곆; ) HANGUL SYLLABLE GYEGG +ACC7;ACC7;1100 1168 11AA;ACC7;1100 1168 11AA; # (곇; 곇; 곇; 곇; 곇; ) HANGUL SYLLABLE GYEGS +ACC8;ACC8;1100 1168 11AB;ACC8;1100 1168 11AB; # (곈; 곈; 곈; 곈; 곈; ) HANGUL SYLLABLE GYEN +ACC9;ACC9;1100 1168 11AC;ACC9;1100 1168 11AC; # (곉; 곉; 곉; 곉; 곉; ) HANGUL SYLLABLE GYENJ +ACCA;ACCA;1100 1168 11AD;ACCA;1100 1168 11AD; # (곊; 곊; 곊; 곊; 곊; ) HANGUL SYLLABLE GYENH +ACCB;ACCB;1100 1168 11AE;ACCB;1100 1168 11AE; # (곋; 곋; 곋; 곋; 곋; ) HANGUL SYLLABLE GYED +ACCC;ACCC;1100 1168 11AF;ACCC;1100 1168 11AF; # (곌; 곌; 곌; 곌; 곌; ) HANGUL SYLLABLE GYEL +ACCD;ACCD;1100 1168 11B0;ACCD;1100 1168 11B0; # (곍; 곍; 곍; 곍; 곍; ) HANGUL SYLLABLE GYELG +ACCE;ACCE;1100 1168 11B1;ACCE;1100 1168 11B1; # (곎; 곎; 곎; 곎; 곎; ) HANGUL SYLLABLE GYELM +ACCF;ACCF;1100 1168 11B2;ACCF;1100 1168 11B2; # (곏; 곏; 곏; 곏; 곏; ) HANGUL SYLLABLE GYELB +ACD0;ACD0;1100 1168 11B3;ACD0;1100 1168 11B3; # (곐; 곐; 곐; 곐; 곐; ) HANGUL SYLLABLE GYELS +ACD1;ACD1;1100 1168 11B4;ACD1;1100 1168 11B4; # (곑; 곑; 곑; 곑; 곑; ) HANGUL SYLLABLE GYELT +ACD2;ACD2;1100 1168 11B5;ACD2;1100 1168 11B5; # (곒; 곒; 곒; 곒; 곒; ) HANGUL SYLLABLE GYELP +ACD3;ACD3;1100 1168 11B6;ACD3;1100 1168 11B6; # (곓; 곓; 곓; 곓; 곓; ) HANGUL SYLLABLE GYELH +ACD4;ACD4;1100 1168 11B7;ACD4;1100 1168 11B7; # (곔; 곔; 곔; 곔; 곔; ) HANGUL SYLLABLE GYEM +ACD5;ACD5;1100 1168 11B8;ACD5;1100 1168 11B8; # (곕; 곕; 곕; 곕; 곕; ) HANGUL SYLLABLE GYEB +ACD6;ACD6;1100 1168 11B9;ACD6;1100 1168 11B9; # (곖; 곖; 곖; 곖; 곖; ) HANGUL SYLLABLE GYEBS +ACD7;ACD7;1100 1168 11BA;ACD7;1100 1168 11BA; # (곗; 곗; 곗; 곗; 곗; ) HANGUL SYLLABLE GYES +ACD8;ACD8;1100 1168 11BB;ACD8;1100 1168 11BB; # (곘; 곘; 곘; 곘; 곘; ) HANGUL SYLLABLE GYESS +ACD9;ACD9;1100 1168 11BC;ACD9;1100 1168 11BC; # (곙; 곙; 곙; 곙; 곙; ) HANGUL SYLLABLE GYENG +ACDA;ACDA;1100 1168 11BD;ACDA;1100 1168 11BD; # (곚; 곚; 곚; 곚; 곚; ) HANGUL SYLLABLE GYEJ +ACDB;ACDB;1100 1168 11BE;ACDB;1100 1168 11BE; # (곛; 곛; 곛; 곛; 곛; ) HANGUL SYLLABLE GYEC +ACDC;ACDC;1100 1168 11BF;ACDC;1100 1168 11BF; # (곜; 곜; 곜; 곜; 곜; ) HANGUL SYLLABLE GYEK +ACDD;ACDD;1100 1168 11C0;ACDD;1100 1168 11C0; # (곝; 곝; 곝; 곝; 곝; ) HANGUL SYLLABLE GYET +ACDE;ACDE;1100 1168 11C1;ACDE;1100 1168 11C1; # (곞; 곞; 곞; 곞; 곞; ) HANGUL SYLLABLE GYEP +ACDF;ACDF;1100 1168 11C2;ACDF;1100 1168 11C2; # (곟; 곟; 곟; 곟; 곟; ) HANGUL SYLLABLE GYEH +ACE0;ACE0;1100 1169;ACE0;1100 1169; # (고; 고; 고; 고; 고; ) HANGUL SYLLABLE GO +ACE1;ACE1;1100 1169 11A8;ACE1;1100 1169 11A8; # (곡; 곡; 곡; 곡; 곡; ) HANGUL SYLLABLE GOG +ACE2;ACE2;1100 1169 11A9;ACE2;1100 1169 11A9; # (곢; 곢; 곢; 곢; 곢; ) HANGUL SYLLABLE GOGG +ACE3;ACE3;1100 1169 11AA;ACE3;1100 1169 11AA; # (곣; 곣; 곣; 곣; 곣; ) HANGUL SYLLABLE GOGS +ACE4;ACE4;1100 1169 11AB;ACE4;1100 1169 11AB; # (곤; 곤; 곤; 곤; 곤; ) HANGUL SYLLABLE GON +ACE5;ACE5;1100 1169 11AC;ACE5;1100 1169 11AC; # (곥; 곥; 곥; 곥; 곥; ) HANGUL SYLLABLE GONJ +ACE6;ACE6;1100 1169 11AD;ACE6;1100 1169 11AD; # (곦; 곦; 곦; 곦; 곦; ) HANGUL SYLLABLE GONH +ACE7;ACE7;1100 1169 11AE;ACE7;1100 1169 11AE; # (곧; 곧; 곧; 곧; 곧; ) HANGUL SYLLABLE GOD +ACE8;ACE8;1100 1169 11AF;ACE8;1100 1169 11AF; # (골; 골; 골; 골; 골; ) HANGUL SYLLABLE GOL +ACE9;ACE9;1100 1169 11B0;ACE9;1100 1169 11B0; # (곩; 곩; 곩; 곩; 곩; ) HANGUL SYLLABLE GOLG +ACEA;ACEA;1100 1169 11B1;ACEA;1100 1169 11B1; # (곪; 곪; 곪; 곪; 곪; ) HANGUL SYLLABLE GOLM +ACEB;ACEB;1100 1169 11B2;ACEB;1100 1169 11B2; # (곫; 곫; 곫; 곫; 곫; ) HANGUL SYLLABLE GOLB +ACEC;ACEC;1100 1169 11B3;ACEC;1100 1169 11B3; # (곬; 곬; 곬; 곬; 곬; ) HANGUL SYLLABLE GOLS +ACED;ACED;1100 1169 11B4;ACED;1100 1169 11B4; # (곭; 곭; 곭; 곭; 곭; ) HANGUL SYLLABLE GOLT +ACEE;ACEE;1100 1169 11B5;ACEE;1100 1169 11B5; # (곮; 곮; 곮; 곮; 곮; ) HANGUL SYLLABLE GOLP +ACEF;ACEF;1100 1169 11B6;ACEF;1100 1169 11B6; # (곯; 곯; 곯; 곯; 곯; ) HANGUL SYLLABLE GOLH +ACF0;ACF0;1100 1169 11B7;ACF0;1100 1169 11B7; # (곰; 곰; 곰; 곰; 곰; ) HANGUL SYLLABLE GOM +ACF1;ACF1;1100 1169 11B8;ACF1;1100 1169 11B8; # (곱; 곱; 곱; 곱; 곱; ) HANGUL SYLLABLE GOB +ACF2;ACF2;1100 1169 11B9;ACF2;1100 1169 11B9; # (곲; 곲; 곲; 곲; 곲; ) HANGUL SYLLABLE GOBS +ACF3;ACF3;1100 1169 11BA;ACF3;1100 1169 11BA; # (곳; 곳; 곳; 곳; 곳; ) HANGUL SYLLABLE GOS +ACF4;ACF4;1100 1169 11BB;ACF4;1100 1169 11BB; # (곴; 곴; 곴; 곴; 곴; ) HANGUL SYLLABLE GOSS +ACF5;ACF5;1100 1169 11BC;ACF5;1100 1169 11BC; # (공; 공; 공; 공; 공; ) HANGUL SYLLABLE GONG +ACF6;ACF6;1100 1169 11BD;ACF6;1100 1169 11BD; # (곶; 곶; 곶; 곶; 곶; ) HANGUL SYLLABLE GOJ +ACF7;ACF7;1100 1169 11BE;ACF7;1100 1169 11BE; # (곷; 곷; 곷; 곷; 곷; ) HANGUL SYLLABLE GOC +ACF8;ACF8;1100 1169 11BF;ACF8;1100 1169 11BF; # (곸; 곸; 곸; 곸; 곸; ) HANGUL SYLLABLE GOK +ACF9;ACF9;1100 1169 11C0;ACF9;1100 1169 11C0; # (곹; 곹; 곹; 곹; 곹; ) HANGUL SYLLABLE GOT +ACFA;ACFA;1100 1169 11C1;ACFA;1100 1169 11C1; # (곺; 곺; 곺; 곺; 곺; ) HANGUL SYLLABLE GOP +ACFB;ACFB;1100 1169 11C2;ACFB;1100 1169 11C2; # (곻; 곻; 곻; 곻; 곻; ) HANGUL SYLLABLE GOH +ACFC;ACFC;1100 116A;ACFC;1100 116A; # (과; 과; 과; 과; 과; ) HANGUL SYLLABLE GWA +ACFD;ACFD;1100 116A 11A8;ACFD;1100 116A 11A8; # (곽; 곽; 곽; 곽; 곽; ) HANGUL SYLLABLE GWAG +ACFE;ACFE;1100 116A 11A9;ACFE;1100 116A 11A9; # (곾; 곾; 곾; 곾; 곾; ) HANGUL SYLLABLE GWAGG +ACFF;ACFF;1100 116A 11AA;ACFF;1100 116A 11AA; # (곿; 곿; 곿; 곿; 곿; ) HANGUL SYLLABLE GWAGS +AD00;AD00;1100 116A 11AB;AD00;1100 116A 11AB; # (관; 관; 관; 관; 관; ) HANGUL SYLLABLE GWAN +AD01;AD01;1100 116A 11AC;AD01;1100 116A 11AC; # (괁; 괁; 괁; 괁; 괁; ) HANGUL SYLLABLE GWANJ +AD02;AD02;1100 116A 11AD;AD02;1100 116A 11AD; # (괂; 괂; 괂; 괂; 괂; ) HANGUL SYLLABLE GWANH +AD03;AD03;1100 116A 11AE;AD03;1100 116A 11AE; # (괃; 괃; 괃; 괃; 괃; ) HANGUL SYLLABLE GWAD +AD04;AD04;1100 116A 11AF;AD04;1100 116A 11AF; # (괄; 괄; 괄; 괄; 괄; ) HANGUL SYLLABLE GWAL +AD05;AD05;1100 116A 11B0;AD05;1100 116A 11B0; # (괅; 괅; 괅; 괅; 괅; ) HANGUL SYLLABLE GWALG +AD06;AD06;1100 116A 11B1;AD06;1100 116A 11B1; # (괆; 괆; 괆; 괆; 괆; ) HANGUL SYLLABLE GWALM +AD07;AD07;1100 116A 11B2;AD07;1100 116A 11B2; # (괇; 괇; 괇; 괇; 괇; ) HANGUL SYLLABLE GWALB +AD08;AD08;1100 116A 11B3;AD08;1100 116A 11B3; # (괈; 괈; 괈; 괈; 괈; ) HANGUL SYLLABLE GWALS +AD09;AD09;1100 116A 11B4;AD09;1100 116A 11B4; # (괉; 괉; 괉; 괉; 괉; ) HANGUL SYLLABLE GWALT +AD0A;AD0A;1100 116A 11B5;AD0A;1100 116A 11B5; # (괊; 괊; 괊; 괊; 괊; ) HANGUL SYLLABLE GWALP +AD0B;AD0B;1100 116A 11B6;AD0B;1100 116A 11B6; # (괋; 괋; 괋; 괋; 괋; ) HANGUL SYLLABLE GWALH +AD0C;AD0C;1100 116A 11B7;AD0C;1100 116A 11B7; # (괌; 괌; 괌; 괌; 괌; ) HANGUL SYLLABLE GWAM +AD0D;AD0D;1100 116A 11B8;AD0D;1100 116A 11B8; # (괍; 괍; 괍; 괍; 괍; ) HANGUL SYLLABLE GWAB +AD0E;AD0E;1100 116A 11B9;AD0E;1100 116A 11B9; # (괎; 괎; 괎; 괎; 괎; ) HANGUL SYLLABLE GWABS +AD0F;AD0F;1100 116A 11BA;AD0F;1100 116A 11BA; # (괏; 괏; 괏; 괏; 괏; ) HANGUL SYLLABLE GWAS +AD10;AD10;1100 116A 11BB;AD10;1100 116A 11BB; # (괐; 괐; 괐; 괐; 괐; ) HANGUL SYLLABLE GWASS +AD11;AD11;1100 116A 11BC;AD11;1100 116A 11BC; # (광; 광; 광; 광; 광; ) HANGUL SYLLABLE GWANG +AD12;AD12;1100 116A 11BD;AD12;1100 116A 11BD; # (괒; 괒; 괒; 괒; 괒; ) HANGUL SYLLABLE GWAJ +AD13;AD13;1100 116A 11BE;AD13;1100 116A 11BE; # (괓; 괓; 괓; 괓; 괓; ) HANGUL SYLLABLE GWAC +AD14;AD14;1100 116A 11BF;AD14;1100 116A 11BF; # (괔; 괔; 괔; 괔; 괔; ) HANGUL SYLLABLE GWAK +AD15;AD15;1100 116A 11C0;AD15;1100 116A 11C0; # (괕; 괕; 괕; 괕; 괕; ) HANGUL SYLLABLE GWAT +AD16;AD16;1100 116A 11C1;AD16;1100 116A 11C1; # (괖; 괖; 괖; 괖; 괖; ) HANGUL SYLLABLE GWAP +AD17;AD17;1100 116A 11C2;AD17;1100 116A 11C2; # (괗; 괗; 괗; 괗; 괗; ) HANGUL SYLLABLE GWAH +AD18;AD18;1100 116B;AD18;1100 116B; # (괘; 괘; 괘; 괘; 괘; ) HANGUL SYLLABLE GWAE +AD19;AD19;1100 116B 11A8;AD19;1100 116B 11A8; # (괙; 괙; 괙; 괙; 괙; ) HANGUL SYLLABLE GWAEG +AD1A;AD1A;1100 116B 11A9;AD1A;1100 116B 11A9; # (괚; 괚; 괚; 괚; 괚; ) HANGUL SYLLABLE GWAEGG +AD1B;AD1B;1100 116B 11AA;AD1B;1100 116B 11AA; # (괛; 괛; 괛; 괛; 괛; ) HANGUL SYLLABLE GWAEGS +AD1C;AD1C;1100 116B 11AB;AD1C;1100 116B 11AB; # (괜; 괜; 괜; 괜; 괜; ) HANGUL SYLLABLE GWAEN +AD1D;AD1D;1100 116B 11AC;AD1D;1100 116B 11AC; # (괝; 괝; 괝; 괝; 괝; ) HANGUL SYLLABLE GWAENJ +AD1E;AD1E;1100 116B 11AD;AD1E;1100 116B 11AD; # (괞; 괞; 괞; 괞; 괞; ) HANGUL SYLLABLE GWAENH +AD1F;AD1F;1100 116B 11AE;AD1F;1100 116B 11AE; # (괟; 괟; 괟; 괟; 괟; ) HANGUL SYLLABLE GWAED +AD20;AD20;1100 116B 11AF;AD20;1100 116B 11AF; # (괠; 괠; 괠; 괠; 괠; ) HANGUL SYLLABLE GWAEL +AD21;AD21;1100 116B 11B0;AD21;1100 116B 11B0; # (괡; 괡; 괡; 괡; 괡; ) HANGUL SYLLABLE GWAELG +AD22;AD22;1100 116B 11B1;AD22;1100 116B 11B1; # (괢; 괢; 괢; 괢; 괢; ) HANGUL SYLLABLE GWAELM +AD23;AD23;1100 116B 11B2;AD23;1100 116B 11B2; # (괣; 괣; 괣; 괣; 괣; ) HANGUL SYLLABLE GWAELB +AD24;AD24;1100 116B 11B3;AD24;1100 116B 11B3; # (괤; 괤; 괤; 괤; 괤; ) HANGUL SYLLABLE GWAELS +AD25;AD25;1100 116B 11B4;AD25;1100 116B 11B4; # (괥; 괥; 괥; 괥; 괥; ) HANGUL SYLLABLE GWAELT +AD26;AD26;1100 116B 11B5;AD26;1100 116B 11B5; # (괦; 괦; 괦; 괦; 괦; ) HANGUL SYLLABLE GWAELP +AD27;AD27;1100 116B 11B6;AD27;1100 116B 11B6; # (괧; 괧; 괧; 괧; 괧; ) HANGUL SYLLABLE GWAELH +AD28;AD28;1100 116B 11B7;AD28;1100 116B 11B7; # (괨; 괨; 괨; 괨; 괨; ) HANGUL SYLLABLE GWAEM +AD29;AD29;1100 116B 11B8;AD29;1100 116B 11B8; # (괩; 괩; 괩; 괩; 괩; ) HANGUL SYLLABLE GWAEB +AD2A;AD2A;1100 116B 11B9;AD2A;1100 116B 11B9; # (괪; 괪; 괪; 괪; 괪; ) HANGUL SYLLABLE GWAEBS +AD2B;AD2B;1100 116B 11BA;AD2B;1100 116B 11BA; # (괫; 괫; 괫; 괫; 괫; ) HANGUL SYLLABLE GWAES +AD2C;AD2C;1100 116B 11BB;AD2C;1100 116B 11BB; # (괬; 괬; 괬; 괬; 괬; ) HANGUL SYLLABLE GWAESS +AD2D;AD2D;1100 116B 11BC;AD2D;1100 116B 11BC; # (괭; 괭; 괭; 괭; 괭; ) HANGUL SYLLABLE GWAENG +AD2E;AD2E;1100 116B 11BD;AD2E;1100 116B 11BD; # (괮; 괮; 괮; 괮; 괮; ) HANGUL SYLLABLE GWAEJ +AD2F;AD2F;1100 116B 11BE;AD2F;1100 116B 11BE; # (괯; 괯; 괯; 괯; 괯; ) HANGUL SYLLABLE GWAEC +AD30;AD30;1100 116B 11BF;AD30;1100 116B 11BF; # (괰; 괰; 괰; 괰; 괰; ) HANGUL SYLLABLE GWAEK +AD31;AD31;1100 116B 11C0;AD31;1100 116B 11C0; # (괱; 괱; 괱; 괱; 괱; ) HANGUL SYLLABLE GWAET +AD32;AD32;1100 116B 11C1;AD32;1100 116B 11C1; # (괲; 괲; 괲; 괲; 괲; ) HANGUL SYLLABLE GWAEP +AD33;AD33;1100 116B 11C2;AD33;1100 116B 11C2; # (괳; 괳; 괳; 괳; 괳; ) HANGUL SYLLABLE GWAEH +AD34;AD34;1100 116C;AD34;1100 116C; # (괴; 괴; 괴; 괴; 괴; ) HANGUL SYLLABLE GOE +AD35;AD35;1100 116C 11A8;AD35;1100 116C 11A8; # (괵; 괵; 괵; 괵; 괵; ) HANGUL SYLLABLE GOEG +AD36;AD36;1100 116C 11A9;AD36;1100 116C 11A9; # (괶; 괶; 괶; 괶; 괶; ) HANGUL SYLLABLE GOEGG +AD37;AD37;1100 116C 11AA;AD37;1100 116C 11AA; # (괷; 괷; 괷; 괷; 괷; ) HANGUL SYLLABLE GOEGS +AD38;AD38;1100 116C 11AB;AD38;1100 116C 11AB; # (괸; 괸; 괸; 괸; 괸; ) HANGUL SYLLABLE GOEN +AD39;AD39;1100 116C 11AC;AD39;1100 116C 11AC; # (괹; 괹; 괹; 괹; 괹; ) HANGUL SYLLABLE GOENJ +AD3A;AD3A;1100 116C 11AD;AD3A;1100 116C 11AD; # (괺; 괺; 괺; 괺; 괺; ) HANGUL SYLLABLE GOENH +AD3B;AD3B;1100 116C 11AE;AD3B;1100 116C 11AE; # (괻; 괻; 괻; 괻; 괻; ) HANGUL SYLLABLE GOED +AD3C;AD3C;1100 116C 11AF;AD3C;1100 116C 11AF; # (괼; 괼; 괼; 괼; 괼; ) HANGUL SYLLABLE GOEL +AD3D;AD3D;1100 116C 11B0;AD3D;1100 116C 11B0; # (괽; 괽; 괽; 괽; 괽; ) HANGUL SYLLABLE GOELG +AD3E;AD3E;1100 116C 11B1;AD3E;1100 116C 11B1; # (괾; 괾; 괾; 괾; 괾; ) HANGUL SYLLABLE GOELM +AD3F;AD3F;1100 116C 11B2;AD3F;1100 116C 11B2; # (괿; 괿; 괿; 괿; 괿; ) HANGUL SYLLABLE GOELB +AD40;AD40;1100 116C 11B3;AD40;1100 116C 11B3; # (굀; 굀; 굀; 굀; 굀; ) HANGUL SYLLABLE GOELS +AD41;AD41;1100 116C 11B4;AD41;1100 116C 11B4; # (굁; 굁; 굁; 굁; 굁; ) HANGUL SYLLABLE GOELT +AD42;AD42;1100 116C 11B5;AD42;1100 116C 11B5; # (굂; 굂; 굂; 굂; 굂; ) HANGUL SYLLABLE GOELP +AD43;AD43;1100 116C 11B6;AD43;1100 116C 11B6; # (굃; 굃; 굃; 굃; 굃; ) HANGUL SYLLABLE GOELH +AD44;AD44;1100 116C 11B7;AD44;1100 116C 11B7; # (굄; 굄; 굄; 굄; 굄; ) HANGUL SYLLABLE GOEM +AD45;AD45;1100 116C 11B8;AD45;1100 116C 11B8; # (굅; 굅; 굅; 굅; 굅; ) HANGUL SYLLABLE GOEB +AD46;AD46;1100 116C 11B9;AD46;1100 116C 11B9; # (굆; 굆; 굆; 굆; 굆; ) HANGUL SYLLABLE GOEBS +AD47;AD47;1100 116C 11BA;AD47;1100 116C 11BA; # (굇; 굇; 굇; 굇; 굇; ) HANGUL SYLLABLE GOES +AD48;AD48;1100 116C 11BB;AD48;1100 116C 11BB; # (굈; 굈; 굈; 굈; 굈; ) HANGUL SYLLABLE GOESS +AD49;AD49;1100 116C 11BC;AD49;1100 116C 11BC; # (굉; 굉; 굉; 굉; 굉; ) HANGUL SYLLABLE GOENG +AD4A;AD4A;1100 116C 11BD;AD4A;1100 116C 11BD; # (굊; 굊; 굊; 굊; 굊; ) HANGUL SYLLABLE GOEJ +AD4B;AD4B;1100 116C 11BE;AD4B;1100 116C 11BE; # (굋; 굋; 굋; 굋; 굋; ) HANGUL SYLLABLE GOEC +AD4C;AD4C;1100 116C 11BF;AD4C;1100 116C 11BF; # (굌; 굌; 굌; 굌; 굌; ) HANGUL SYLLABLE GOEK +AD4D;AD4D;1100 116C 11C0;AD4D;1100 116C 11C0; # (굍; 굍; 굍; 굍; 굍; ) HANGUL SYLLABLE GOET +AD4E;AD4E;1100 116C 11C1;AD4E;1100 116C 11C1; # (굎; 굎; 굎; 굎; 굎; ) HANGUL SYLLABLE GOEP +AD4F;AD4F;1100 116C 11C2;AD4F;1100 116C 11C2; # (굏; 굏; 굏; 굏; 굏; ) HANGUL SYLLABLE GOEH +AD50;AD50;1100 116D;AD50;1100 116D; # (교; 교; 교; 교; 교; ) HANGUL SYLLABLE GYO +AD51;AD51;1100 116D 11A8;AD51;1100 116D 11A8; # (굑; 굑; 굑; 굑; 굑; ) HANGUL SYLLABLE GYOG +AD52;AD52;1100 116D 11A9;AD52;1100 116D 11A9; # (굒; 굒; 굒; 굒; 굒; ) HANGUL SYLLABLE GYOGG +AD53;AD53;1100 116D 11AA;AD53;1100 116D 11AA; # (굓; 굓; 굓; 굓; 굓; ) HANGUL SYLLABLE GYOGS +AD54;AD54;1100 116D 11AB;AD54;1100 116D 11AB; # (굔; 굔; 굔; 굔; 굔; ) HANGUL SYLLABLE GYON +AD55;AD55;1100 116D 11AC;AD55;1100 116D 11AC; # (굕; 굕; 굕; 굕; 굕; ) HANGUL SYLLABLE GYONJ +AD56;AD56;1100 116D 11AD;AD56;1100 116D 11AD; # (굖; 굖; 굖; 굖; 굖; ) HANGUL SYLLABLE GYONH +AD57;AD57;1100 116D 11AE;AD57;1100 116D 11AE; # (굗; 굗; 굗; 굗; 굗; ) HANGUL SYLLABLE GYOD +AD58;AD58;1100 116D 11AF;AD58;1100 116D 11AF; # (굘; 굘; 굘; 굘; 굘; ) HANGUL SYLLABLE GYOL +AD59;AD59;1100 116D 11B0;AD59;1100 116D 11B0; # (굙; 굙; 굙; 굙; 굙; ) HANGUL SYLLABLE GYOLG +AD5A;AD5A;1100 116D 11B1;AD5A;1100 116D 11B1; # (굚; 굚; 굚; 굚; 굚; ) HANGUL SYLLABLE GYOLM +AD5B;AD5B;1100 116D 11B2;AD5B;1100 116D 11B2; # (굛; 굛; 굛; 굛; 굛; ) HANGUL SYLLABLE GYOLB +AD5C;AD5C;1100 116D 11B3;AD5C;1100 116D 11B3; # (굜; 굜; 굜; 굜; 굜; ) HANGUL SYLLABLE GYOLS +AD5D;AD5D;1100 116D 11B4;AD5D;1100 116D 11B4; # (굝; 굝; 굝; 굝; 굝; ) HANGUL SYLLABLE GYOLT +AD5E;AD5E;1100 116D 11B5;AD5E;1100 116D 11B5; # (굞; 굞; 굞; 굞; 굞; ) HANGUL SYLLABLE GYOLP +AD5F;AD5F;1100 116D 11B6;AD5F;1100 116D 11B6; # (굟; 굟; 굟; 굟; 굟; ) HANGUL SYLLABLE GYOLH +AD60;AD60;1100 116D 11B7;AD60;1100 116D 11B7; # (굠; 굠; 굠; 굠; 굠; ) HANGUL SYLLABLE GYOM +AD61;AD61;1100 116D 11B8;AD61;1100 116D 11B8; # (굡; 굡; 굡; 굡; 굡; ) HANGUL SYLLABLE GYOB +AD62;AD62;1100 116D 11B9;AD62;1100 116D 11B9; # (굢; 굢; 굢; 굢; 굢; ) HANGUL SYLLABLE GYOBS +AD63;AD63;1100 116D 11BA;AD63;1100 116D 11BA; # (굣; 굣; 굣; 굣; 굣; ) HANGUL SYLLABLE GYOS +AD64;AD64;1100 116D 11BB;AD64;1100 116D 11BB; # (굤; 굤; 굤; 굤; 굤; ) HANGUL SYLLABLE GYOSS +AD65;AD65;1100 116D 11BC;AD65;1100 116D 11BC; # (굥; 굥; 굥; 굥; 굥; ) HANGUL SYLLABLE GYONG +AD66;AD66;1100 116D 11BD;AD66;1100 116D 11BD; # (굦; 굦; 굦; 굦; 굦; ) HANGUL SYLLABLE GYOJ +AD67;AD67;1100 116D 11BE;AD67;1100 116D 11BE; # (굧; 굧; 굧; 굧; 굧; ) HANGUL SYLLABLE GYOC +AD68;AD68;1100 116D 11BF;AD68;1100 116D 11BF; # (굨; 굨; 굨; 굨; 굨; ) HANGUL SYLLABLE GYOK +AD69;AD69;1100 116D 11C0;AD69;1100 116D 11C0; # (굩; 굩; 굩; 굩; 굩; ) HANGUL SYLLABLE GYOT +AD6A;AD6A;1100 116D 11C1;AD6A;1100 116D 11C1; # (굪; 굪; 굪; 굪; 굪; ) HANGUL SYLLABLE GYOP +AD6B;AD6B;1100 116D 11C2;AD6B;1100 116D 11C2; # (굫; 굫; 굫; 굫; 굫; ) HANGUL SYLLABLE GYOH +AD6C;AD6C;1100 116E;AD6C;1100 116E; # (구; 구; 구; 구; 구; ) HANGUL SYLLABLE GU +AD6D;AD6D;1100 116E 11A8;AD6D;1100 116E 11A8; # (국; 국; 국; 국; 국; ) HANGUL SYLLABLE GUG +AD6E;AD6E;1100 116E 11A9;AD6E;1100 116E 11A9; # (굮; 굮; 굮; 굮; 굮; ) HANGUL SYLLABLE GUGG +AD6F;AD6F;1100 116E 11AA;AD6F;1100 116E 11AA; # (굯; 굯; 굯; 굯; 굯; ) HANGUL SYLLABLE GUGS +AD70;AD70;1100 116E 11AB;AD70;1100 116E 11AB; # (군; 군; 군; 군; 군; ) HANGUL SYLLABLE GUN +AD71;AD71;1100 116E 11AC;AD71;1100 116E 11AC; # (굱; 굱; 굱; 굱; 굱; ) HANGUL SYLLABLE GUNJ +AD72;AD72;1100 116E 11AD;AD72;1100 116E 11AD; # (굲; 굲; 굲; 굲; 굲; ) HANGUL SYLLABLE GUNH +AD73;AD73;1100 116E 11AE;AD73;1100 116E 11AE; # (굳; 굳; 굳; 굳; 굳; ) HANGUL SYLLABLE GUD +AD74;AD74;1100 116E 11AF;AD74;1100 116E 11AF; # (굴; 굴; 굴; 굴; 굴; ) HANGUL SYLLABLE GUL +AD75;AD75;1100 116E 11B0;AD75;1100 116E 11B0; # (굵; 굵; 굵; 굵; 굵; ) HANGUL SYLLABLE GULG +AD76;AD76;1100 116E 11B1;AD76;1100 116E 11B1; # (굶; 굶; 굶; 굶; 굶; ) HANGUL SYLLABLE GULM +AD77;AD77;1100 116E 11B2;AD77;1100 116E 11B2; # (굷; 굷; 굷; 굷; 굷; ) HANGUL SYLLABLE GULB +AD78;AD78;1100 116E 11B3;AD78;1100 116E 11B3; # (굸; 굸; 굸; 굸; 굸; ) HANGUL SYLLABLE GULS +AD79;AD79;1100 116E 11B4;AD79;1100 116E 11B4; # (굹; 굹; 굹; 굹; 굹; ) HANGUL SYLLABLE GULT +AD7A;AD7A;1100 116E 11B5;AD7A;1100 116E 11B5; # (굺; 굺; 굺; 굺; 굺; ) HANGUL SYLLABLE GULP +AD7B;AD7B;1100 116E 11B6;AD7B;1100 116E 11B6; # (굻; 굻; 굻; 굻; 굻; ) HANGUL SYLLABLE GULH +AD7C;AD7C;1100 116E 11B7;AD7C;1100 116E 11B7; # (굼; 굼; 굼; 굼; 굼; ) HANGUL SYLLABLE GUM +AD7D;AD7D;1100 116E 11B8;AD7D;1100 116E 11B8; # (굽; 굽; 굽; 굽; 굽; ) HANGUL SYLLABLE GUB +AD7E;AD7E;1100 116E 11B9;AD7E;1100 116E 11B9; # (굾; 굾; 굾; 굾; 굾; ) HANGUL SYLLABLE GUBS +AD7F;AD7F;1100 116E 11BA;AD7F;1100 116E 11BA; # (굿; 굿; 굿; 굿; 굿; ) HANGUL SYLLABLE GUS +AD80;AD80;1100 116E 11BB;AD80;1100 116E 11BB; # (궀; 궀; 궀; 궀; 궀; ) HANGUL SYLLABLE GUSS +AD81;AD81;1100 116E 11BC;AD81;1100 116E 11BC; # (궁; 궁; 궁; 궁; 궁; ) HANGUL SYLLABLE GUNG +AD82;AD82;1100 116E 11BD;AD82;1100 116E 11BD; # (궂; 궂; 궂; 궂; 궂; ) HANGUL SYLLABLE GUJ +AD83;AD83;1100 116E 11BE;AD83;1100 116E 11BE; # (궃; 궃; 궃; 궃; 궃; ) HANGUL SYLLABLE GUC +AD84;AD84;1100 116E 11BF;AD84;1100 116E 11BF; # (궄; 궄; 궄; 궄; 궄; ) HANGUL SYLLABLE GUK +AD85;AD85;1100 116E 11C0;AD85;1100 116E 11C0; # (궅; 궅; 궅; 궅; 궅; ) HANGUL SYLLABLE GUT +AD86;AD86;1100 116E 11C1;AD86;1100 116E 11C1; # (궆; 궆; 궆; 궆; 궆; ) HANGUL SYLLABLE GUP +AD87;AD87;1100 116E 11C2;AD87;1100 116E 11C2; # (궇; 궇; 궇; 궇; 궇; ) HANGUL SYLLABLE GUH +AD88;AD88;1100 116F;AD88;1100 116F; # (궈; 궈; 궈; 궈; 궈; ) HANGUL SYLLABLE GWEO +AD89;AD89;1100 116F 11A8;AD89;1100 116F 11A8; # (궉; 궉; 궉; 궉; 궉; ) HANGUL SYLLABLE GWEOG +AD8A;AD8A;1100 116F 11A9;AD8A;1100 116F 11A9; # (궊; 궊; 궊; 궊; 궊; ) HANGUL SYLLABLE GWEOGG +AD8B;AD8B;1100 116F 11AA;AD8B;1100 116F 11AA; # (궋; 궋; 궋; 궋; 궋; ) HANGUL SYLLABLE GWEOGS +AD8C;AD8C;1100 116F 11AB;AD8C;1100 116F 11AB; # (권; 권; 권; 권; 권; ) HANGUL SYLLABLE GWEON +AD8D;AD8D;1100 116F 11AC;AD8D;1100 116F 11AC; # (궍; 궍; 궍; 궍; 궍; ) HANGUL SYLLABLE GWEONJ +AD8E;AD8E;1100 116F 11AD;AD8E;1100 116F 11AD; # (궎; 궎; 궎; 궎; 궎; ) HANGUL SYLLABLE GWEONH +AD8F;AD8F;1100 116F 11AE;AD8F;1100 116F 11AE; # (궏; 궏; 궏; 궏; 궏; ) HANGUL SYLLABLE GWEOD +AD90;AD90;1100 116F 11AF;AD90;1100 116F 11AF; # (궐; 궐; 궐; 궐; 궐; ) HANGUL SYLLABLE GWEOL +AD91;AD91;1100 116F 11B0;AD91;1100 116F 11B0; # (궑; 궑; 궑; 궑; 궑; ) HANGUL SYLLABLE GWEOLG +AD92;AD92;1100 116F 11B1;AD92;1100 116F 11B1; # (궒; 궒; 궒; 궒; 궒; ) HANGUL SYLLABLE GWEOLM +AD93;AD93;1100 116F 11B2;AD93;1100 116F 11B2; # (궓; 궓; 궓; 궓; 궓; ) HANGUL SYLLABLE GWEOLB +AD94;AD94;1100 116F 11B3;AD94;1100 116F 11B3; # (궔; 궔; 궔; 궔; 궔; ) HANGUL SYLLABLE GWEOLS +AD95;AD95;1100 116F 11B4;AD95;1100 116F 11B4; # (궕; 궕; 궕; 궕; 궕; ) HANGUL SYLLABLE GWEOLT +AD96;AD96;1100 116F 11B5;AD96;1100 116F 11B5; # (궖; 궖; 궖; 궖; 궖; ) HANGUL SYLLABLE GWEOLP +AD97;AD97;1100 116F 11B6;AD97;1100 116F 11B6; # (궗; 궗; 궗; 궗; 궗; ) HANGUL SYLLABLE GWEOLH +AD98;AD98;1100 116F 11B7;AD98;1100 116F 11B7; # (궘; 궘; 궘; 궘; 궘; ) HANGUL SYLLABLE GWEOM +AD99;AD99;1100 116F 11B8;AD99;1100 116F 11B8; # (궙; 궙; 궙; 궙; 궙; ) HANGUL SYLLABLE GWEOB +AD9A;AD9A;1100 116F 11B9;AD9A;1100 116F 11B9; # (궚; 궚; 궚; 궚; 궚; ) HANGUL SYLLABLE GWEOBS +AD9B;AD9B;1100 116F 11BA;AD9B;1100 116F 11BA; # (궛; 궛; 궛; 궛; 궛; ) HANGUL SYLLABLE GWEOS +AD9C;AD9C;1100 116F 11BB;AD9C;1100 116F 11BB; # (궜; 궜; 궜; 궜; 궜; ) HANGUL SYLLABLE GWEOSS +AD9D;AD9D;1100 116F 11BC;AD9D;1100 116F 11BC; # (궝; 궝; 궝; 궝; 궝; ) HANGUL SYLLABLE GWEONG +AD9E;AD9E;1100 116F 11BD;AD9E;1100 116F 11BD; # (궞; 궞; 궞; 궞; 궞; ) HANGUL SYLLABLE GWEOJ +AD9F;AD9F;1100 116F 11BE;AD9F;1100 116F 11BE; # (궟; 궟; 궟; 궟; 궟; ) HANGUL SYLLABLE GWEOC +ADA0;ADA0;1100 116F 11BF;ADA0;1100 116F 11BF; # (궠; 궠; 궠; 궠; 궠; ) HANGUL SYLLABLE GWEOK +ADA1;ADA1;1100 116F 11C0;ADA1;1100 116F 11C0; # (궡; 궡; 궡; 궡; 궡; ) HANGUL SYLLABLE GWEOT +ADA2;ADA2;1100 116F 11C1;ADA2;1100 116F 11C1; # (궢; 궢; 궢; 궢; 궢; ) HANGUL SYLLABLE GWEOP +ADA3;ADA3;1100 116F 11C2;ADA3;1100 116F 11C2; # (궣; 궣; 궣; 궣; 궣; ) HANGUL SYLLABLE GWEOH +ADA4;ADA4;1100 1170;ADA4;1100 1170; # (궤; 궤; 궤; 궤; 궤; ) HANGUL SYLLABLE GWE +ADA5;ADA5;1100 1170 11A8;ADA5;1100 1170 11A8; # (궥; 궥; 궥; 궥; 궥; ) HANGUL SYLLABLE GWEG +ADA6;ADA6;1100 1170 11A9;ADA6;1100 1170 11A9; # (궦; 궦; 궦; 궦; 궦; ) HANGUL SYLLABLE GWEGG +ADA7;ADA7;1100 1170 11AA;ADA7;1100 1170 11AA; # (궧; 궧; 궧; 궧; 궧; ) HANGUL SYLLABLE GWEGS +ADA8;ADA8;1100 1170 11AB;ADA8;1100 1170 11AB; # (궨; 궨; 궨; 궨; 궨; ) HANGUL SYLLABLE GWEN +ADA9;ADA9;1100 1170 11AC;ADA9;1100 1170 11AC; # (궩; 궩; 궩; 궩; 궩; ) HANGUL SYLLABLE GWENJ +ADAA;ADAA;1100 1170 11AD;ADAA;1100 1170 11AD; # (궪; 궪; 궪; 궪; 궪; ) HANGUL SYLLABLE GWENH +ADAB;ADAB;1100 1170 11AE;ADAB;1100 1170 11AE; # (궫; 궫; 궫; 궫; 궫; ) HANGUL SYLLABLE GWED +ADAC;ADAC;1100 1170 11AF;ADAC;1100 1170 11AF; # (궬; 궬; 궬; 궬; 궬; ) HANGUL SYLLABLE GWEL +ADAD;ADAD;1100 1170 11B0;ADAD;1100 1170 11B0; # (궭; 궭; 궭; 궭; 궭; ) HANGUL SYLLABLE GWELG +ADAE;ADAE;1100 1170 11B1;ADAE;1100 1170 11B1; # (궮; 궮; 궮; 궮; 궮; ) HANGUL SYLLABLE GWELM +ADAF;ADAF;1100 1170 11B2;ADAF;1100 1170 11B2; # (궯; 궯; 궯; 궯; 궯; ) HANGUL SYLLABLE GWELB +ADB0;ADB0;1100 1170 11B3;ADB0;1100 1170 11B3; # (궰; 궰; 궰; 궰; 궰; ) HANGUL SYLLABLE GWELS +ADB1;ADB1;1100 1170 11B4;ADB1;1100 1170 11B4; # (궱; 궱; 궱; 궱; 궱; ) HANGUL SYLLABLE GWELT +ADB2;ADB2;1100 1170 11B5;ADB2;1100 1170 11B5; # (궲; 궲; 궲; 궲; 궲; ) HANGUL SYLLABLE GWELP +ADB3;ADB3;1100 1170 11B6;ADB3;1100 1170 11B6; # (궳; 궳; 궳; 궳; 궳; ) HANGUL SYLLABLE GWELH +ADB4;ADB4;1100 1170 11B7;ADB4;1100 1170 11B7; # (궴; 궴; 궴; 궴; 궴; ) HANGUL SYLLABLE GWEM +ADB5;ADB5;1100 1170 11B8;ADB5;1100 1170 11B8; # (궵; 궵; 궵; 궵; 궵; ) HANGUL SYLLABLE GWEB +ADB6;ADB6;1100 1170 11B9;ADB6;1100 1170 11B9; # (궶; 궶; 궶; 궶; 궶; ) HANGUL SYLLABLE GWEBS +ADB7;ADB7;1100 1170 11BA;ADB7;1100 1170 11BA; # (궷; 궷; 궷; 궷; 궷; ) HANGUL SYLLABLE GWES +ADB8;ADB8;1100 1170 11BB;ADB8;1100 1170 11BB; # (궸; 궸; 궸; 궸; 궸; ) HANGUL SYLLABLE GWESS +ADB9;ADB9;1100 1170 11BC;ADB9;1100 1170 11BC; # (궹; 궹; 궹; 궹; 궹; ) HANGUL SYLLABLE GWENG +ADBA;ADBA;1100 1170 11BD;ADBA;1100 1170 11BD; # (궺; 궺; 궺; 궺; 궺; ) HANGUL SYLLABLE GWEJ +ADBB;ADBB;1100 1170 11BE;ADBB;1100 1170 11BE; # (궻; 궻; 궻; 궻; 궻; ) HANGUL SYLLABLE GWEC +ADBC;ADBC;1100 1170 11BF;ADBC;1100 1170 11BF; # (궼; 궼; 궼; 궼; 궼; ) HANGUL SYLLABLE GWEK +ADBD;ADBD;1100 1170 11C0;ADBD;1100 1170 11C0; # (궽; 궽; 궽; 궽; 궽; ) HANGUL SYLLABLE GWET +ADBE;ADBE;1100 1170 11C1;ADBE;1100 1170 11C1; # (궾; 궾; 궾; 궾; 궾; ) HANGUL SYLLABLE GWEP +ADBF;ADBF;1100 1170 11C2;ADBF;1100 1170 11C2; # (궿; 궿; 궿; 궿; 궿; ) HANGUL SYLLABLE GWEH +ADC0;ADC0;1100 1171;ADC0;1100 1171; # (귀; 귀; 귀; 귀; 귀; ) HANGUL SYLLABLE GWI +ADC1;ADC1;1100 1171 11A8;ADC1;1100 1171 11A8; # (귁; 귁; 귁; 귁; 귁; ) HANGUL SYLLABLE GWIG +ADC2;ADC2;1100 1171 11A9;ADC2;1100 1171 11A9; # (귂; 귂; 귂; 귂; 귂; ) HANGUL SYLLABLE GWIGG +ADC3;ADC3;1100 1171 11AA;ADC3;1100 1171 11AA; # (귃; 귃; 귃; 귃; 귃; ) HANGUL SYLLABLE GWIGS +ADC4;ADC4;1100 1171 11AB;ADC4;1100 1171 11AB; # (귄; 귄; 귄; 귄; 귄; ) HANGUL SYLLABLE GWIN +ADC5;ADC5;1100 1171 11AC;ADC5;1100 1171 11AC; # (귅; 귅; 귅; 귅; 귅; ) HANGUL SYLLABLE GWINJ +ADC6;ADC6;1100 1171 11AD;ADC6;1100 1171 11AD; # (귆; 귆; 귆; 귆; 귆; ) HANGUL SYLLABLE GWINH +ADC7;ADC7;1100 1171 11AE;ADC7;1100 1171 11AE; # (귇; 귇; 귇; 귇; 귇; ) HANGUL SYLLABLE GWID +ADC8;ADC8;1100 1171 11AF;ADC8;1100 1171 11AF; # (귈; 귈; 귈; 귈; 귈; ) HANGUL SYLLABLE GWIL +ADC9;ADC9;1100 1171 11B0;ADC9;1100 1171 11B0; # (귉; 귉; 귉; 귉; 귉; ) HANGUL SYLLABLE GWILG +ADCA;ADCA;1100 1171 11B1;ADCA;1100 1171 11B1; # (귊; 귊; 귊; 귊; 귊; ) HANGUL SYLLABLE GWILM +ADCB;ADCB;1100 1171 11B2;ADCB;1100 1171 11B2; # (귋; 귋; 귋; 귋; 귋; ) HANGUL SYLLABLE GWILB +ADCC;ADCC;1100 1171 11B3;ADCC;1100 1171 11B3; # (귌; 귌; 귌; 귌; 귌; ) HANGUL SYLLABLE GWILS +ADCD;ADCD;1100 1171 11B4;ADCD;1100 1171 11B4; # (귍; 귍; 귍; 귍; 귍; ) HANGUL SYLLABLE GWILT +ADCE;ADCE;1100 1171 11B5;ADCE;1100 1171 11B5; # (귎; 귎; 귎; 귎; 귎; ) HANGUL SYLLABLE GWILP +ADCF;ADCF;1100 1171 11B6;ADCF;1100 1171 11B6; # (귏; 귏; 귏; 귏; 귏; ) HANGUL SYLLABLE GWILH +ADD0;ADD0;1100 1171 11B7;ADD0;1100 1171 11B7; # (귐; 귐; 귐; 귐; 귐; ) HANGUL SYLLABLE GWIM +ADD1;ADD1;1100 1171 11B8;ADD1;1100 1171 11B8; # (귑; 귑; 귑; 귑; 귑; ) HANGUL SYLLABLE GWIB +ADD2;ADD2;1100 1171 11B9;ADD2;1100 1171 11B9; # (귒; 귒; 귒; 귒; 귒; ) HANGUL SYLLABLE GWIBS +ADD3;ADD3;1100 1171 11BA;ADD3;1100 1171 11BA; # (귓; 귓; 귓; 귓; 귓; ) HANGUL SYLLABLE GWIS +ADD4;ADD4;1100 1171 11BB;ADD4;1100 1171 11BB; # (귔; 귔; 귔; 귔; 귔; ) HANGUL SYLLABLE GWISS +ADD5;ADD5;1100 1171 11BC;ADD5;1100 1171 11BC; # (귕; 귕; 귕; 귕; 귕; ) HANGUL SYLLABLE GWING +ADD6;ADD6;1100 1171 11BD;ADD6;1100 1171 11BD; # (귖; 귖; 귖; 귖; 귖; ) HANGUL SYLLABLE GWIJ +ADD7;ADD7;1100 1171 11BE;ADD7;1100 1171 11BE; # (귗; 귗; 귗; 귗; 귗; ) HANGUL SYLLABLE GWIC +ADD8;ADD8;1100 1171 11BF;ADD8;1100 1171 11BF; # (귘; 귘; 귘; 귘; 귘; ) HANGUL SYLLABLE GWIK +ADD9;ADD9;1100 1171 11C0;ADD9;1100 1171 11C0; # (귙; 귙; 귙; 귙; 귙; ) HANGUL SYLLABLE GWIT +ADDA;ADDA;1100 1171 11C1;ADDA;1100 1171 11C1; # (귚; 귚; 귚; 귚; 귚; ) HANGUL SYLLABLE GWIP +ADDB;ADDB;1100 1171 11C2;ADDB;1100 1171 11C2; # (귛; 귛; 귛; 귛; 귛; ) HANGUL SYLLABLE GWIH +ADDC;ADDC;1100 1172;ADDC;1100 1172; # (규; 규; 규; 규; 규; ) HANGUL SYLLABLE GYU +ADDD;ADDD;1100 1172 11A8;ADDD;1100 1172 11A8; # (귝; 귝; 귝; 귝; 귝; ) HANGUL SYLLABLE GYUG +ADDE;ADDE;1100 1172 11A9;ADDE;1100 1172 11A9; # (귞; 귞; 귞; 귞; 귞; ) HANGUL SYLLABLE GYUGG +ADDF;ADDF;1100 1172 11AA;ADDF;1100 1172 11AA; # (귟; 귟; 귟; 귟; 귟; ) HANGUL SYLLABLE GYUGS +ADE0;ADE0;1100 1172 11AB;ADE0;1100 1172 11AB; # (균; 균; 균; 균; 균; ) HANGUL SYLLABLE GYUN +ADE1;ADE1;1100 1172 11AC;ADE1;1100 1172 11AC; # (귡; 귡; 귡; 귡; 귡; ) HANGUL SYLLABLE GYUNJ +ADE2;ADE2;1100 1172 11AD;ADE2;1100 1172 11AD; # (귢; 귢; 귢; 귢; 귢; ) HANGUL SYLLABLE GYUNH +ADE3;ADE3;1100 1172 11AE;ADE3;1100 1172 11AE; # (귣; 귣; 귣; 귣; 귣; ) HANGUL SYLLABLE GYUD +ADE4;ADE4;1100 1172 11AF;ADE4;1100 1172 11AF; # (귤; 귤; 귤; 귤; 귤; ) HANGUL SYLLABLE GYUL +ADE5;ADE5;1100 1172 11B0;ADE5;1100 1172 11B0; # (귥; 귥; 귥; 귥; 귥; ) HANGUL SYLLABLE GYULG +ADE6;ADE6;1100 1172 11B1;ADE6;1100 1172 11B1; # (귦; 귦; 귦; 귦; 귦; ) HANGUL SYLLABLE GYULM +ADE7;ADE7;1100 1172 11B2;ADE7;1100 1172 11B2; # (귧; 귧; 귧; 귧; 귧; ) HANGUL SYLLABLE GYULB +ADE8;ADE8;1100 1172 11B3;ADE8;1100 1172 11B3; # (귨; 귨; 귨; 귨; 귨; ) HANGUL SYLLABLE GYULS +ADE9;ADE9;1100 1172 11B4;ADE9;1100 1172 11B4; # (귩; 귩; 귩; 귩; 귩; ) HANGUL SYLLABLE GYULT +ADEA;ADEA;1100 1172 11B5;ADEA;1100 1172 11B5; # (귪; 귪; 귪; 귪; 귪; ) HANGUL SYLLABLE GYULP +ADEB;ADEB;1100 1172 11B6;ADEB;1100 1172 11B6; # (귫; 귫; 귫; 귫; 귫; ) HANGUL SYLLABLE GYULH +ADEC;ADEC;1100 1172 11B7;ADEC;1100 1172 11B7; # (귬; 귬; 귬; 귬; 귬; ) HANGUL SYLLABLE GYUM +ADED;ADED;1100 1172 11B8;ADED;1100 1172 11B8; # (귭; 귭; 귭; 귭; 귭; ) HANGUL SYLLABLE GYUB +ADEE;ADEE;1100 1172 11B9;ADEE;1100 1172 11B9; # (귮; 귮; 귮; 귮; 귮; ) HANGUL SYLLABLE GYUBS +ADEF;ADEF;1100 1172 11BA;ADEF;1100 1172 11BA; # (귯; 귯; 귯; 귯; 귯; ) HANGUL SYLLABLE GYUS +ADF0;ADF0;1100 1172 11BB;ADF0;1100 1172 11BB; # (귰; 귰; 귰; 귰; 귰; ) HANGUL SYLLABLE GYUSS +ADF1;ADF1;1100 1172 11BC;ADF1;1100 1172 11BC; # (귱; 귱; 귱; 귱; 귱; ) HANGUL SYLLABLE GYUNG +ADF2;ADF2;1100 1172 11BD;ADF2;1100 1172 11BD; # (귲; 귲; 귲; 귲; 귲; ) HANGUL SYLLABLE GYUJ +ADF3;ADF3;1100 1172 11BE;ADF3;1100 1172 11BE; # (귳; 귳; 귳; 귳; 귳; ) HANGUL SYLLABLE GYUC +ADF4;ADF4;1100 1172 11BF;ADF4;1100 1172 11BF; # (귴; 귴; 귴; 귴; 귴; ) HANGUL SYLLABLE GYUK +ADF5;ADF5;1100 1172 11C0;ADF5;1100 1172 11C0; # (귵; 귵; 귵; 귵; 귵; ) HANGUL SYLLABLE GYUT +ADF6;ADF6;1100 1172 11C1;ADF6;1100 1172 11C1; # (귶; 귶; 귶; 귶; 귶; ) HANGUL SYLLABLE GYUP +ADF7;ADF7;1100 1172 11C2;ADF7;1100 1172 11C2; # (귷; 귷; 귷; 귷; 귷; ) HANGUL SYLLABLE GYUH +ADF8;ADF8;1100 1173;ADF8;1100 1173; # (그; 그; 그; 그; 그; ) HANGUL SYLLABLE GEU +ADF9;ADF9;1100 1173 11A8;ADF9;1100 1173 11A8; # (극; 극; 극; 극; 극; ) HANGUL SYLLABLE GEUG +ADFA;ADFA;1100 1173 11A9;ADFA;1100 1173 11A9; # (귺; 귺; 귺; 귺; 귺; ) HANGUL SYLLABLE GEUGG +ADFB;ADFB;1100 1173 11AA;ADFB;1100 1173 11AA; # (귻; 귻; 귻; 귻; 귻; ) HANGUL SYLLABLE GEUGS +ADFC;ADFC;1100 1173 11AB;ADFC;1100 1173 11AB; # (근; 근; 근; 근; 근; ) HANGUL SYLLABLE GEUN +ADFD;ADFD;1100 1173 11AC;ADFD;1100 1173 11AC; # (귽; 귽; 귽; 귽; 귽; ) HANGUL SYLLABLE GEUNJ +ADFE;ADFE;1100 1173 11AD;ADFE;1100 1173 11AD; # (귾; 귾; 귾; 귾; 귾; ) HANGUL SYLLABLE GEUNH +ADFF;ADFF;1100 1173 11AE;ADFF;1100 1173 11AE; # (귿; 귿; 귿; 귿; 귿; ) HANGUL SYLLABLE GEUD +AE00;AE00;1100 1173 11AF;AE00;1100 1173 11AF; # (글; 글; 글; 글; 글; ) HANGUL SYLLABLE GEUL +AE01;AE01;1100 1173 11B0;AE01;1100 1173 11B0; # (긁; 긁; 긁; 긁; 긁; ) HANGUL SYLLABLE GEULG +AE02;AE02;1100 1173 11B1;AE02;1100 1173 11B1; # (긂; 긂; 긂; 긂; 긂; ) HANGUL SYLLABLE GEULM +AE03;AE03;1100 1173 11B2;AE03;1100 1173 11B2; # (긃; 긃; 긃; 긃; 긃; ) HANGUL SYLLABLE GEULB +AE04;AE04;1100 1173 11B3;AE04;1100 1173 11B3; # (긄; 긄; 긄; 긄; 긄; ) HANGUL SYLLABLE GEULS +AE05;AE05;1100 1173 11B4;AE05;1100 1173 11B4; # (긅; 긅; 긅; 긅; 긅; ) HANGUL SYLLABLE GEULT +AE06;AE06;1100 1173 11B5;AE06;1100 1173 11B5; # (긆; 긆; 긆; 긆; 긆; ) HANGUL SYLLABLE GEULP +AE07;AE07;1100 1173 11B6;AE07;1100 1173 11B6; # (긇; 긇; 긇; 긇; 긇; ) HANGUL SYLLABLE GEULH +AE08;AE08;1100 1173 11B7;AE08;1100 1173 11B7; # (금; 금; 금; 금; 금; ) HANGUL SYLLABLE GEUM +AE09;AE09;1100 1173 11B8;AE09;1100 1173 11B8; # (급; 급; 급; 급; 급; ) HANGUL SYLLABLE GEUB +AE0A;AE0A;1100 1173 11B9;AE0A;1100 1173 11B9; # (긊; 긊; 긊; 긊; 긊; ) HANGUL SYLLABLE GEUBS +AE0B;AE0B;1100 1173 11BA;AE0B;1100 1173 11BA; # (긋; 긋; 긋; 긋; 긋; ) HANGUL SYLLABLE GEUS +AE0C;AE0C;1100 1173 11BB;AE0C;1100 1173 11BB; # (긌; 긌; 긌; 긌; 긌; ) HANGUL SYLLABLE GEUSS +AE0D;AE0D;1100 1173 11BC;AE0D;1100 1173 11BC; # (긍; 긍; 긍; 긍; 긍; ) HANGUL SYLLABLE GEUNG +AE0E;AE0E;1100 1173 11BD;AE0E;1100 1173 11BD; # (긎; 긎; 긎; 긎; 긎; ) HANGUL SYLLABLE GEUJ +AE0F;AE0F;1100 1173 11BE;AE0F;1100 1173 11BE; # (긏; 긏; 긏; 긏; 긏; ) HANGUL SYLLABLE GEUC +AE10;AE10;1100 1173 11BF;AE10;1100 1173 11BF; # (긐; 긐; 긐; 긐; 긐; ) HANGUL SYLLABLE GEUK +AE11;AE11;1100 1173 11C0;AE11;1100 1173 11C0; # (긑; 긑; 긑; 긑; 긑; ) HANGUL SYLLABLE GEUT +AE12;AE12;1100 1173 11C1;AE12;1100 1173 11C1; # (긒; 긒; 긒; 긒; 긒; ) HANGUL SYLLABLE GEUP +AE13;AE13;1100 1173 11C2;AE13;1100 1173 11C2; # (긓; 긓; 긓; 긓; 긓; ) HANGUL SYLLABLE GEUH +AE14;AE14;1100 1174;AE14;1100 1174; # (긔; 긔; 긔; 긔; 긔; ) HANGUL SYLLABLE GYI +AE15;AE15;1100 1174 11A8;AE15;1100 1174 11A8; # (긕; 긕; 긕; 긕; 긕; ) HANGUL SYLLABLE GYIG +AE16;AE16;1100 1174 11A9;AE16;1100 1174 11A9; # (긖; 긖; 긖; 긖; 긖; ) HANGUL SYLLABLE GYIGG +AE17;AE17;1100 1174 11AA;AE17;1100 1174 11AA; # (긗; 긗; 긗; 긗; 긗; ) HANGUL SYLLABLE GYIGS +AE18;AE18;1100 1174 11AB;AE18;1100 1174 11AB; # (긘; 긘; 긘; 긘; 긘; ) HANGUL SYLLABLE GYIN +AE19;AE19;1100 1174 11AC;AE19;1100 1174 11AC; # (긙; 긙; 긙; 긙; 긙; ) HANGUL SYLLABLE GYINJ +AE1A;AE1A;1100 1174 11AD;AE1A;1100 1174 11AD; # (긚; 긚; 긚; 긚; 긚; ) HANGUL SYLLABLE GYINH +AE1B;AE1B;1100 1174 11AE;AE1B;1100 1174 11AE; # (긛; 긛; 긛; 긛; 긛; ) HANGUL SYLLABLE GYID +AE1C;AE1C;1100 1174 11AF;AE1C;1100 1174 11AF; # (긜; 긜; 긜; 긜; 긜; ) HANGUL SYLLABLE GYIL +AE1D;AE1D;1100 1174 11B0;AE1D;1100 1174 11B0; # (긝; 긝; 긝; 긝; 긝; ) HANGUL SYLLABLE GYILG +AE1E;AE1E;1100 1174 11B1;AE1E;1100 1174 11B1; # (긞; 긞; 긞; 긞; 긞; ) HANGUL SYLLABLE GYILM +AE1F;AE1F;1100 1174 11B2;AE1F;1100 1174 11B2; # (긟; 긟; 긟; 긟; 긟; ) HANGUL SYLLABLE GYILB +AE20;AE20;1100 1174 11B3;AE20;1100 1174 11B3; # (긠; 긠; 긠; 긠; 긠; ) HANGUL SYLLABLE GYILS +AE21;AE21;1100 1174 11B4;AE21;1100 1174 11B4; # (긡; 긡; 긡; 긡; 긡; ) HANGUL SYLLABLE GYILT +AE22;AE22;1100 1174 11B5;AE22;1100 1174 11B5; # (긢; 긢; 긢; 긢; 긢; ) HANGUL SYLLABLE GYILP +AE23;AE23;1100 1174 11B6;AE23;1100 1174 11B6; # (긣; 긣; 긣; 긣; 긣; ) HANGUL SYLLABLE GYILH +AE24;AE24;1100 1174 11B7;AE24;1100 1174 11B7; # (긤; 긤; 긤; 긤; 긤; ) HANGUL SYLLABLE GYIM +AE25;AE25;1100 1174 11B8;AE25;1100 1174 11B8; # (긥; 긥; 긥; 긥; 긥; ) HANGUL SYLLABLE GYIB +AE26;AE26;1100 1174 11B9;AE26;1100 1174 11B9; # (긦; 긦; 긦; 긦; 긦; ) HANGUL SYLLABLE GYIBS +AE27;AE27;1100 1174 11BA;AE27;1100 1174 11BA; # (긧; 긧; 긧; 긧; 긧; ) HANGUL SYLLABLE GYIS +AE28;AE28;1100 1174 11BB;AE28;1100 1174 11BB; # (긨; 긨; 긨; 긨; 긨; ) HANGUL SYLLABLE GYISS +AE29;AE29;1100 1174 11BC;AE29;1100 1174 11BC; # (긩; 긩; 긩; 긩; 긩; ) HANGUL SYLLABLE GYING +AE2A;AE2A;1100 1174 11BD;AE2A;1100 1174 11BD; # (긪; 긪; 긪; 긪; 긪; ) HANGUL SYLLABLE GYIJ +AE2B;AE2B;1100 1174 11BE;AE2B;1100 1174 11BE; # (긫; 긫; 긫; 긫; 긫; ) HANGUL SYLLABLE GYIC +AE2C;AE2C;1100 1174 11BF;AE2C;1100 1174 11BF; # (긬; 긬; 긬; 긬; 긬; ) HANGUL SYLLABLE GYIK +AE2D;AE2D;1100 1174 11C0;AE2D;1100 1174 11C0; # (긭; 긭; 긭; 긭; 긭; ) HANGUL SYLLABLE GYIT +AE2E;AE2E;1100 1174 11C1;AE2E;1100 1174 11C1; # (긮; 긮; 긮; 긮; 긮; ) HANGUL SYLLABLE GYIP +AE2F;AE2F;1100 1174 11C2;AE2F;1100 1174 11C2; # (긯; 긯; 긯; 긯; 긯; ) HANGUL SYLLABLE GYIH +AE30;AE30;1100 1175;AE30;1100 1175; # (기; 기; 기; 기; 기; ) HANGUL SYLLABLE GI +AE31;AE31;1100 1175 11A8;AE31;1100 1175 11A8; # (긱; 긱; 긱; 긱; 긱; ) HANGUL SYLLABLE GIG +AE32;AE32;1100 1175 11A9;AE32;1100 1175 11A9; # (긲; 긲; 긲; 긲; 긲; ) HANGUL SYLLABLE GIGG +AE33;AE33;1100 1175 11AA;AE33;1100 1175 11AA; # (긳; 긳; 긳; 긳; 긳; ) HANGUL SYLLABLE GIGS +AE34;AE34;1100 1175 11AB;AE34;1100 1175 11AB; # (긴; 긴; 긴; 긴; 긴; ) HANGUL SYLLABLE GIN +AE35;AE35;1100 1175 11AC;AE35;1100 1175 11AC; # (긵; 긵; 긵; 긵; 긵; ) HANGUL SYLLABLE GINJ +AE36;AE36;1100 1175 11AD;AE36;1100 1175 11AD; # (긶; 긶; 긶; 긶; 긶; ) HANGUL SYLLABLE GINH +AE37;AE37;1100 1175 11AE;AE37;1100 1175 11AE; # (긷; 긷; 긷; 긷; 긷; ) HANGUL SYLLABLE GID +AE38;AE38;1100 1175 11AF;AE38;1100 1175 11AF; # (길; 길; 길; 길; 길; ) HANGUL SYLLABLE GIL +AE39;AE39;1100 1175 11B0;AE39;1100 1175 11B0; # (긹; 긹; 긹; 긹; 긹; ) HANGUL SYLLABLE GILG +AE3A;AE3A;1100 1175 11B1;AE3A;1100 1175 11B1; # (긺; 긺; 긺; 긺; 긺; ) HANGUL SYLLABLE GILM +AE3B;AE3B;1100 1175 11B2;AE3B;1100 1175 11B2; # (긻; 긻; 긻; 긻; 긻; ) HANGUL SYLLABLE GILB +AE3C;AE3C;1100 1175 11B3;AE3C;1100 1175 11B3; # (긼; 긼; 긼; 긼; 긼; ) HANGUL SYLLABLE GILS +AE3D;AE3D;1100 1175 11B4;AE3D;1100 1175 11B4; # (긽; 긽; 긽; 긽; 긽; ) HANGUL SYLLABLE GILT +AE3E;AE3E;1100 1175 11B5;AE3E;1100 1175 11B5; # (긾; 긾; 긾; 긾; 긾; ) HANGUL SYLLABLE GILP +AE3F;AE3F;1100 1175 11B6;AE3F;1100 1175 11B6; # (긿; 긿; 긿; 긿; 긿; ) HANGUL SYLLABLE GILH +AE40;AE40;1100 1175 11B7;AE40;1100 1175 11B7; # (김; 김; 김; 김; 김; ) HANGUL SYLLABLE GIM +AE41;AE41;1100 1175 11B8;AE41;1100 1175 11B8; # (깁; 깁; 깁; 깁; 깁; ) HANGUL SYLLABLE GIB +AE42;AE42;1100 1175 11B9;AE42;1100 1175 11B9; # (깂; 깂; 깂; 깂; 깂; ) HANGUL SYLLABLE GIBS +AE43;AE43;1100 1175 11BA;AE43;1100 1175 11BA; # (깃; 깃; 깃; 깃; 깃; ) HANGUL SYLLABLE GIS +AE44;AE44;1100 1175 11BB;AE44;1100 1175 11BB; # (깄; 깄; 깄; 깄; 깄; ) HANGUL SYLLABLE GISS +AE45;AE45;1100 1175 11BC;AE45;1100 1175 11BC; # (깅; 깅; 깅; 깅; 깅; ) HANGUL SYLLABLE GING +AE46;AE46;1100 1175 11BD;AE46;1100 1175 11BD; # (깆; 깆; 깆; 깆; 깆; ) HANGUL SYLLABLE GIJ +AE47;AE47;1100 1175 11BE;AE47;1100 1175 11BE; # (깇; 깇; 깇; 깇; 깇; ) HANGUL SYLLABLE GIC +AE48;AE48;1100 1175 11BF;AE48;1100 1175 11BF; # (깈; 깈; 깈; 깈; 깈; ) HANGUL SYLLABLE GIK +AE49;AE49;1100 1175 11C0;AE49;1100 1175 11C0; # (깉; 깉; 깉; 깉; 깉; ) HANGUL SYLLABLE GIT +AE4A;AE4A;1100 1175 11C1;AE4A;1100 1175 11C1; # (깊; 깊; 깊; 깊; 깊; ) HANGUL SYLLABLE GIP +AE4B;AE4B;1100 1175 11C2;AE4B;1100 1175 11C2; # (깋; 깋; 깋; 깋; 깋; ) HANGUL SYLLABLE GIH +AE4C;AE4C;1101 1161;AE4C;1101 1161; # (까; 까; 까; 까; 까; ) HANGUL SYLLABLE GGA +AE4D;AE4D;1101 1161 11A8;AE4D;1101 1161 11A8; # (깍; 깍; 깍; 깍; 깍; ) HANGUL SYLLABLE GGAG +AE4E;AE4E;1101 1161 11A9;AE4E;1101 1161 11A9; # (깎; 깎; 깎; 깎; 깎; ) HANGUL SYLLABLE GGAGG +AE4F;AE4F;1101 1161 11AA;AE4F;1101 1161 11AA; # (깏; 깏; 깏; 깏; 깏; ) HANGUL SYLLABLE GGAGS +AE50;AE50;1101 1161 11AB;AE50;1101 1161 11AB; # (깐; 깐; 깐; 깐; 깐; ) HANGUL SYLLABLE GGAN +AE51;AE51;1101 1161 11AC;AE51;1101 1161 11AC; # (깑; 깑; 깑; 깑; 깑; ) HANGUL SYLLABLE GGANJ +AE52;AE52;1101 1161 11AD;AE52;1101 1161 11AD; # (깒; 깒; 깒; 깒; 깒; ) HANGUL SYLLABLE GGANH +AE53;AE53;1101 1161 11AE;AE53;1101 1161 11AE; # (깓; 깓; 깓; 깓; 깓; ) HANGUL SYLLABLE GGAD +AE54;AE54;1101 1161 11AF;AE54;1101 1161 11AF; # (깔; 깔; 깔; 깔; 깔; ) HANGUL SYLLABLE GGAL +AE55;AE55;1101 1161 11B0;AE55;1101 1161 11B0; # (깕; 깕; 깕; 깕; 깕; ) HANGUL SYLLABLE GGALG +AE56;AE56;1101 1161 11B1;AE56;1101 1161 11B1; # (깖; 깖; 깖; 깖; 깖; ) HANGUL SYLLABLE GGALM +AE57;AE57;1101 1161 11B2;AE57;1101 1161 11B2; # (깗; 깗; 깗; 깗; 깗; ) HANGUL SYLLABLE GGALB +AE58;AE58;1101 1161 11B3;AE58;1101 1161 11B3; # (깘; 깘; 깘; 깘; 깘; ) HANGUL SYLLABLE GGALS +AE59;AE59;1101 1161 11B4;AE59;1101 1161 11B4; # (깙; 깙; 깙; 깙; 깙; ) HANGUL SYLLABLE GGALT +AE5A;AE5A;1101 1161 11B5;AE5A;1101 1161 11B5; # (깚; 깚; 깚; 깚; 깚; ) HANGUL SYLLABLE GGALP +AE5B;AE5B;1101 1161 11B6;AE5B;1101 1161 11B6; # (깛; 깛; 깛; 깛; 깛; ) HANGUL SYLLABLE GGALH +AE5C;AE5C;1101 1161 11B7;AE5C;1101 1161 11B7; # (깜; 깜; 깜; 깜; 깜; ) HANGUL SYLLABLE GGAM +AE5D;AE5D;1101 1161 11B8;AE5D;1101 1161 11B8; # (깝; 깝; 깝; 깝; 깝; ) HANGUL SYLLABLE GGAB +AE5E;AE5E;1101 1161 11B9;AE5E;1101 1161 11B9; # (깞; 깞; 깞; 깞; 깞; ) HANGUL SYLLABLE GGABS +AE5F;AE5F;1101 1161 11BA;AE5F;1101 1161 11BA; # (깟; 깟; 깟; 깟; 깟; ) HANGUL SYLLABLE GGAS +AE60;AE60;1101 1161 11BB;AE60;1101 1161 11BB; # (깠; 깠; 깠; 깠; 깠; ) HANGUL SYLLABLE GGASS +AE61;AE61;1101 1161 11BC;AE61;1101 1161 11BC; # (깡; 깡; 깡; 깡; 깡; ) HANGUL SYLLABLE GGANG +AE62;AE62;1101 1161 11BD;AE62;1101 1161 11BD; # (깢; 깢; 깢; 깢; 깢; ) HANGUL SYLLABLE GGAJ +AE63;AE63;1101 1161 11BE;AE63;1101 1161 11BE; # (깣; 깣; 깣; 깣; 깣; ) HANGUL SYLLABLE GGAC +AE64;AE64;1101 1161 11BF;AE64;1101 1161 11BF; # (깤; 깤; 깤; 깤; 깤; ) HANGUL SYLLABLE GGAK +AE65;AE65;1101 1161 11C0;AE65;1101 1161 11C0; # (깥; 깥; 깥; 깥; 깥; ) HANGUL SYLLABLE GGAT +AE66;AE66;1101 1161 11C1;AE66;1101 1161 11C1; # (깦; 깦; 깦; 깦; 깦; ) HANGUL SYLLABLE GGAP +AE67;AE67;1101 1161 11C2;AE67;1101 1161 11C2; # (깧; 깧; 깧; 깧; 깧; ) HANGUL SYLLABLE GGAH +AE68;AE68;1101 1162;AE68;1101 1162; # (깨; 깨; 깨; 깨; 깨; ) HANGUL SYLLABLE GGAE +AE69;AE69;1101 1162 11A8;AE69;1101 1162 11A8; # (깩; 깩; 깩; 깩; 깩; ) HANGUL SYLLABLE GGAEG +AE6A;AE6A;1101 1162 11A9;AE6A;1101 1162 11A9; # (깪; 깪; 깪; 깪; 깪; ) HANGUL SYLLABLE GGAEGG +AE6B;AE6B;1101 1162 11AA;AE6B;1101 1162 11AA; # (깫; 깫; 깫; 깫; 깫; ) HANGUL SYLLABLE GGAEGS +AE6C;AE6C;1101 1162 11AB;AE6C;1101 1162 11AB; # (깬; 깬; 깬; 깬; 깬; ) HANGUL SYLLABLE GGAEN +AE6D;AE6D;1101 1162 11AC;AE6D;1101 1162 11AC; # (깭; 깭; 깭; 깭; 깭; ) HANGUL SYLLABLE GGAENJ +AE6E;AE6E;1101 1162 11AD;AE6E;1101 1162 11AD; # (깮; 깮; 깮; 깮; 깮; ) HANGUL SYLLABLE GGAENH +AE6F;AE6F;1101 1162 11AE;AE6F;1101 1162 11AE; # (깯; 깯; 깯; 깯; 깯; ) HANGUL SYLLABLE GGAED +AE70;AE70;1101 1162 11AF;AE70;1101 1162 11AF; # (깰; 깰; 깰; 깰; 깰; ) HANGUL SYLLABLE GGAEL +AE71;AE71;1101 1162 11B0;AE71;1101 1162 11B0; # (깱; 깱; 깱; 깱; 깱; ) HANGUL SYLLABLE GGAELG +AE72;AE72;1101 1162 11B1;AE72;1101 1162 11B1; # (깲; 깲; 깲; 깲; 깲; ) HANGUL SYLLABLE GGAELM +AE73;AE73;1101 1162 11B2;AE73;1101 1162 11B2; # (깳; 깳; 깳; 깳; 깳; ) HANGUL SYLLABLE GGAELB +AE74;AE74;1101 1162 11B3;AE74;1101 1162 11B3; # (깴; 깴; 깴; 깴; 깴; ) HANGUL SYLLABLE GGAELS +AE75;AE75;1101 1162 11B4;AE75;1101 1162 11B4; # (깵; 깵; 깵; 깵; 깵; ) HANGUL SYLLABLE GGAELT +AE76;AE76;1101 1162 11B5;AE76;1101 1162 11B5; # (깶; 깶; 깶; 깶; 깶; ) HANGUL SYLLABLE GGAELP +AE77;AE77;1101 1162 11B6;AE77;1101 1162 11B6; # (깷; 깷; 깷; 깷; 깷; ) HANGUL SYLLABLE GGAELH +AE78;AE78;1101 1162 11B7;AE78;1101 1162 11B7; # (깸; 깸; 깸; 깸; 깸; ) HANGUL SYLLABLE GGAEM +AE79;AE79;1101 1162 11B8;AE79;1101 1162 11B8; # (깹; 깹; 깹; 깹; 깹; ) HANGUL SYLLABLE GGAEB +AE7A;AE7A;1101 1162 11B9;AE7A;1101 1162 11B9; # (깺; 깺; 깺; 깺; 깺; ) HANGUL SYLLABLE GGAEBS +AE7B;AE7B;1101 1162 11BA;AE7B;1101 1162 11BA; # (깻; 깻; 깻; 깻; 깻; ) HANGUL SYLLABLE GGAES +AE7C;AE7C;1101 1162 11BB;AE7C;1101 1162 11BB; # (깼; 깼; 깼; 깼; 깼; ) HANGUL SYLLABLE GGAESS +AE7D;AE7D;1101 1162 11BC;AE7D;1101 1162 11BC; # (깽; 깽; 깽; 깽; 깽; ) HANGUL SYLLABLE GGAENG +AE7E;AE7E;1101 1162 11BD;AE7E;1101 1162 11BD; # (깾; 깾; 깾; 깾; 깾; ) HANGUL SYLLABLE GGAEJ +AE7F;AE7F;1101 1162 11BE;AE7F;1101 1162 11BE; # (깿; 깿; 깿; 깿; 깿; ) HANGUL SYLLABLE GGAEC +AE80;AE80;1101 1162 11BF;AE80;1101 1162 11BF; # (꺀; 꺀; 꺀; 꺀; 꺀; ) HANGUL SYLLABLE GGAEK +AE81;AE81;1101 1162 11C0;AE81;1101 1162 11C0; # (꺁; 꺁; 꺁; 꺁; 꺁; ) HANGUL SYLLABLE GGAET +AE82;AE82;1101 1162 11C1;AE82;1101 1162 11C1; # (꺂; 꺂; 꺂; 꺂; 꺂; ) HANGUL SYLLABLE GGAEP +AE83;AE83;1101 1162 11C2;AE83;1101 1162 11C2; # (꺃; 꺃; 꺃; 꺃; 꺃; ) HANGUL SYLLABLE GGAEH +AE84;AE84;1101 1163;AE84;1101 1163; # (꺄; 꺄; 꺄; 꺄; 꺄; ) HANGUL SYLLABLE GGYA +AE85;AE85;1101 1163 11A8;AE85;1101 1163 11A8; # (꺅; 꺅; 꺅; 꺅; 꺅; ) HANGUL SYLLABLE GGYAG +AE86;AE86;1101 1163 11A9;AE86;1101 1163 11A9; # (꺆; 꺆; 꺆; 꺆; 꺆; ) HANGUL SYLLABLE GGYAGG +AE87;AE87;1101 1163 11AA;AE87;1101 1163 11AA; # (꺇; 꺇; 꺇; 꺇; 꺇; ) HANGUL SYLLABLE GGYAGS +AE88;AE88;1101 1163 11AB;AE88;1101 1163 11AB; # (꺈; 꺈; 꺈; 꺈; 꺈; ) HANGUL SYLLABLE GGYAN +AE89;AE89;1101 1163 11AC;AE89;1101 1163 11AC; # (꺉; 꺉; 꺉; 꺉; 꺉; ) HANGUL SYLLABLE GGYANJ +AE8A;AE8A;1101 1163 11AD;AE8A;1101 1163 11AD; # (꺊; 꺊; 꺊; 꺊; 꺊; ) HANGUL SYLLABLE GGYANH +AE8B;AE8B;1101 1163 11AE;AE8B;1101 1163 11AE; # (꺋; 꺋; 꺋; 꺋; 꺋; ) HANGUL SYLLABLE GGYAD +AE8C;AE8C;1101 1163 11AF;AE8C;1101 1163 11AF; # (꺌; 꺌; 꺌; 꺌; 꺌; ) HANGUL SYLLABLE GGYAL +AE8D;AE8D;1101 1163 11B0;AE8D;1101 1163 11B0; # (꺍; 꺍; 꺍; 꺍; 꺍; ) HANGUL SYLLABLE GGYALG +AE8E;AE8E;1101 1163 11B1;AE8E;1101 1163 11B1; # (꺎; 꺎; 꺎; 꺎; 꺎; ) HANGUL SYLLABLE GGYALM +AE8F;AE8F;1101 1163 11B2;AE8F;1101 1163 11B2; # (꺏; 꺏; 꺏; 꺏; 꺏; ) HANGUL SYLLABLE GGYALB +AE90;AE90;1101 1163 11B3;AE90;1101 1163 11B3; # (꺐; 꺐; 꺐; 꺐; 꺐; ) HANGUL SYLLABLE GGYALS +AE91;AE91;1101 1163 11B4;AE91;1101 1163 11B4; # (꺑; 꺑; 꺑; 꺑; 꺑; ) HANGUL SYLLABLE GGYALT +AE92;AE92;1101 1163 11B5;AE92;1101 1163 11B5; # (꺒; 꺒; 꺒; 꺒; 꺒; ) HANGUL SYLLABLE GGYALP +AE93;AE93;1101 1163 11B6;AE93;1101 1163 11B6; # (꺓; 꺓; 꺓; 꺓; 꺓; ) HANGUL SYLLABLE GGYALH +AE94;AE94;1101 1163 11B7;AE94;1101 1163 11B7; # (꺔; 꺔; 꺔; 꺔; 꺔; ) HANGUL SYLLABLE GGYAM +AE95;AE95;1101 1163 11B8;AE95;1101 1163 11B8; # (꺕; 꺕; 꺕; 꺕; 꺕; ) HANGUL SYLLABLE GGYAB +AE96;AE96;1101 1163 11B9;AE96;1101 1163 11B9; # (꺖; 꺖; 꺖; 꺖; 꺖; ) HANGUL SYLLABLE GGYABS +AE97;AE97;1101 1163 11BA;AE97;1101 1163 11BA; # (꺗; 꺗; 꺗; 꺗; 꺗; ) HANGUL SYLLABLE GGYAS +AE98;AE98;1101 1163 11BB;AE98;1101 1163 11BB; # (꺘; 꺘; 꺘; 꺘; 꺘; ) HANGUL SYLLABLE GGYASS +AE99;AE99;1101 1163 11BC;AE99;1101 1163 11BC; # (꺙; 꺙; 꺙; 꺙; 꺙; ) HANGUL SYLLABLE GGYANG +AE9A;AE9A;1101 1163 11BD;AE9A;1101 1163 11BD; # (꺚; 꺚; 꺚; 꺚; 꺚; ) HANGUL SYLLABLE GGYAJ +AE9B;AE9B;1101 1163 11BE;AE9B;1101 1163 11BE; # (꺛; 꺛; 꺛; 꺛; 꺛; ) HANGUL SYLLABLE GGYAC +AE9C;AE9C;1101 1163 11BF;AE9C;1101 1163 11BF; # (꺜; 꺜; 꺜; 꺜; 꺜; ) HANGUL SYLLABLE GGYAK +AE9D;AE9D;1101 1163 11C0;AE9D;1101 1163 11C0; # (꺝; 꺝; 꺝; 꺝; 꺝; ) HANGUL SYLLABLE GGYAT +AE9E;AE9E;1101 1163 11C1;AE9E;1101 1163 11C1; # (꺞; 꺞; 꺞; 꺞; 꺞; ) HANGUL SYLLABLE GGYAP +AE9F;AE9F;1101 1163 11C2;AE9F;1101 1163 11C2; # (꺟; 꺟; 꺟; 꺟; 꺟; ) HANGUL SYLLABLE GGYAH +AEA0;AEA0;1101 1164;AEA0;1101 1164; # (꺠; 꺠; 꺠; 꺠; 꺠; ) HANGUL SYLLABLE GGYAE +AEA1;AEA1;1101 1164 11A8;AEA1;1101 1164 11A8; # (꺡; 꺡; 꺡; 꺡; 꺡; ) HANGUL SYLLABLE GGYAEG +AEA2;AEA2;1101 1164 11A9;AEA2;1101 1164 11A9; # (꺢; 꺢; 꺢; 꺢; 꺢; ) HANGUL SYLLABLE GGYAEGG +AEA3;AEA3;1101 1164 11AA;AEA3;1101 1164 11AA; # (꺣; 꺣; 꺣; 꺣; 꺣; ) HANGUL SYLLABLE GGYAEGS +AEA4;AEA4;1101 1164 11AB;AEA4;1101 1164 11AB; # (꺤; 꺤; 꺤; 꺤; 꺤; ) HANGUL SYLLABLE GGYAEN +AEA5;AEA5;1101 1164 11AC;AEA5;1101 1164 11AC; # (꺥; 꺥; 꺥; 꺥; 꺥; ) HANGUL SYLLABLE GGYAENJ +AEA6;AEA6;1101 1164 11AD;AEA6;1101 1164 11AD; # (꺦; 꺦; 꺦; 꺦; 꺦; ) HANGUL SYLLABLE GGYAENH +AEA7;AEA7;1101 1164 11AE;AEA7;1101 1164 11AE; # (꺧; 꺧; 꺧; 꺧; 꺧; ) HANGUL SYLLABLE GGYAED +AEA8;AEA8;1101 1164 11AF;AEA8;1101 1164 11AF; # (꺨; 꺨; 꺨; 꺨; 꺨; ) HANGUL SYLLABLE GGYAEL +AEA9;AEA9;1101 1164 11B0;AEA9;1101 1164 11B0; # (꺩; 꺩; 꺩; 꺩; 꺩; ) HANGUL SYLLABLE GGYAELG +AEAA;AEAA;1101 1164 11B1;AEAA;1101 1164 11B1; # (꺪; 꺪; 꺪; 꺪; 꺪; ) HANGUL SYLLABLE GGYAELM +AEAB;AEAB;1101 1164 11B2;AEAB;1101 1164 11B2; # (꺫; 꺫; 꺫; 꺫; 꺫; ) HANGUL SYLLABLE GGYAELB +AEAC;AEAC;1101 1164 11B3;AEAC;1101 1164 11B3; # (꺬; 꺬; 꺬; 꺬; 꺬; ) HANGUL SYLLABLE GGYAELS +AEAD;AEAD;1101 1164 11B4;AEAD;1101 1164 11B4; # (꺭; 꺭; 꺭; 꺭; 꺭; ) HANGUL SYLLABLE GGYAELT +AEAE;AEAE;1101 1164 11B5;AEAE;1101 1164 11B5; # (꺮; 꺮; 꺮; 꺮; 꺮; ) HANGUL SYLLABLE GGYAELP +AEAF;AEAF;1101 1164 11B6;AEAF;1101 1164 11B6; # (꺯; 꺯; 꺯; 꺯; 꺯; ) HANGUL SYLLABLE GGYAELH +AEB0;AEB0;1101 1164 11B7;AEB0;1101 1164 11B7; # (꺰; 꺰; 꺰; 꺰; 꺰; ) HANGUL SYLLABLE GGYAEM +AEB1;AEB1;1101 1164 11B8;AEB1;1101 1164 11B8; # (꺱; 꺱; 꺱; 꺱; 꺱; ) HANGUL SYLLABLE GGYAEB +AEB2;AEB2;1101 1164 11B9;AEB2;1101 1164 11B9; # (꺲; 꺲; 꺲; 꺲; 꺲; ) HANGUL SYLLABLE GGYAEBS +AEB3;AEB3;1101 1164 11BA;AEB3;1101 1164 11BA; # (꺳; 꺳; 꺳; 꺳; 꺳; ) HANGUL SYLLABLE GGYAES +AEB4;AEB4;1101 1164 11BB;AEB4;1101 1164 11BB; # (꺴; 꺴; 꺴; 꺴; 꺴; ) HANGUL SYLLABLE GGYAESS +AEB5;AEB5;1101 1164 11BC;AEB5;1101 1164 11BC; # (꺵; 꺵; 꺵; 꺵; 꺵; ) HANGUL SYLLABLE GGYAENG +AEB6;AEB6;1101 1164 11BD;AEB6;1101 1164 11BD; # (꺶; 꺶; 꺶; 꺶; 꺶; ) HANGUL SYLLABLE GGYAEJ +AEB7;AEB7;1101 1164 11BE;AEB7;1101 1164 11BE; # (꺷; 꺷; 꺷; 꺷; 꺷; ) HANGUL SYLLABLE GGYAEC +AEB8;AEB8;1101 1164 11BF;AEB8;1101 1164 11BF; # (꺸; 꺸; 꺸; 꺸; 꺸; ) HANGUL SYLLABLE GGYAEK +AEB9;AEB9;1101 1164 11C0;AEB9;1101 1164 11C0; # (꺹; 꺹; 꺹; 꺹; 꺹; ) HANGUL SYLLABLE GGYAET +AEBA;AEBA;1101 1164 11C1;AEBA;1101 1164 11C1; # (꺺; 꺺; 꺺; 꺺; 꺺; ) HANGUL SYLLABLE GGYAEP +AEBB;AEBB;1101 1164 11C2;AEBB;1101 1164 11C2; # (꺻; 꺻; 꺻; 꺻; 꺻; ) HANGUL SYLLABLE GGYAEH +AEBC;AEBC;1101 1165;AEBC;1101 1165; # (꺼; 꺼; 꺼; 꺼; 꺼; ) HANGUL SYLLABLE GGEO +AEBD;AEBD;1101 1165 11A8;AEBD;1101 1165 11A8; # (꺽; 꺽; 꺽; 꺽; 꺽; ) HANGUL SYLLABLE GGEOG +AEBE;AEBE;1101 1165 11A9;AEBE;1101 1165 11A9; # (꺾; 꺾; 꺾; 꺾; 꺾; ) HANGUL SYLLABLE GGEOGG +AEBF;AEBF;1101 1165 11AA;AEBF;1101 1165 11AA; # (꺿; 꺿; 꺿; 꺿; 꺿; ) HANGUL SYLLABLE GGEOGS +AEC0;AEC0;1101 1165 11AB;AEC0;1101 1165 11AB; # (껀; 껀; 껀; 껀; 껀; ) HANGUL SYLLABLE GGEON +AEC1;AEC1;1101 1165 11AC;AEC1;1101 1165 11AC; # (껁; 껁; 껁; 껁; 껁; ) HANGUL SYLLABLE GGEONJ +AEC2;AEC2;1101 1165 11AD;AEC2;1101 1165 11AD; # (껂; 껂; 껂; 껂; 껂; ) HANGUL SYLLABLE GGEONH +AEC3;AEC3;1101 1165 11AE;AEC3;1101 1165 11AE; # (껃; 껃; 껃; 껃; 껃; ) HANGUL SYLLABLE GGEOD +AEC4;AEC4;1101 1165 11AF;AEC4;1101 1165 11AF; # (껄; 껄; 껄; 껄; 껄; ) HANGUL SYLLABLE GGEOL +AEC5;AEC5;1101 1165 11B0;AEC5;1101 1165 11B0; # (껅; 껅; 껅; 껅; 껅; ) HANGUL SYLLABLE GGEOLG +AEC6;AEC6;1101 1165 11B1;AEC6;1101 1165 11B1; # (껆; 껆; 껆; 껆; 껆; ) HANGUL SYLLABLE GGEOLM +AEC7;AEC7;1101 1165 11B2;AEC7;1101 1165 11B2; # (껇; 껇; 껇; 껇; 껇; ) HANGUL SYLLABLE GGEOLB +AEC8;AEC8;1101 1165 11B3;AEC8;1101 1165 11B3; # (껈; 껈; 껈; 껈; 껈; ) HANGUL SYLLABLE GGEOLS +AEC9;AEC9;1101 1165 11B4;AEC9;1101 1165 11B4; # (껉; 껉; 껉; 껉; 껉; ) HANGUL SYLLABLE GGEOLT +AECA;AECA;1101 1165 11B5;AECA;1101 1165 11B5; # (껊; 껊; 껊; 껊; 껊; ) HANGUL SYLLABLE GGEOLP +AECB;AECB;1101 1165 11B6;AECB;1101 1165 11B6; # (껋; 껋; 껋; 껋; 껋; ) HANGUL SYLLABLE GGEOLH +AECC;AECC;1101 1165 11B7;AECC;1101 1165 11B7; # (껌; 껌; 껌; 껌; 껌; ) HANGUL SYLLABLE GGEOM +AECD;AECD;1101 1165 11B8;AECD;1101 1165 11B8; # (껍; 껍; 껍; 껍; 껍; ) HANGUL SYLLABLE GGEOB +AECE;AECE;1101 1165 11B9;AECE;1101 1165 11B9; # (껎; 껎; 껎; 껎; 껎; ) HANGUL SYLLABLE GGEOBS +AECF;AECF;1101 1165 11BA;AECF;1101 1165 11BA; # (껏; 껏; 껏; 껏; 껏; ) HANGUL SYLLABLE GGEOS +AED0;AED0;1101 1165 11BB;AED0;1101 1165 11BB; # (껐; 껐; 껐; 껐; 껐; ) HANGUL SYLLABLE GGEOSS +AED1;AED1;1101 1165 11BC;AED1;1101 1165 11BC; # (껑; 껑; 껑; 껑; 껑; ) HANGUL SYLLABLE GGEONG +AED2;AED2;1101 1165 11BD;AED2;1101 1165 11BD; # (껒; 껒; 껒; 껒; 껒; ) HANGUL SYLLABLE GGEOJ +AED3;AED3;1101 1165 11BE;AED3;1101 1165 11BE; # (껓; 껓; 껓; 껓; 껓; ) HANGUL SYLLABLE GGEOC +AED4;AED4;1101 1165 11BF;AED4;1101 1165 11BF; # (껔; 껔; 껔; 껔; 껔; ) HANGUL SYLLABLE GGEOK +AED5;AED5;1101 1165 11C0;AED5;1101 1165 11C0; # (껕; 껕; 껕; 껕; 껕; ) HANGUL SYLLABLE GGEOT +AED6;AED6;1101 1165 11C1;AED6;1101 1165 11C1; # (껖; 껖; 껖; 껖; 껖; ) HANGUL SYLLABLE GGEOP +AED7;AED7;1101 1165 11C2;AED7;1101 1165 11C2; # (껗; 껗; 껗; 껗; 껗; ) HANGUL SYLLABLE GGEOH +AED8;AED8;1101 1166;AED8;1101 1166; # (께; 께; 께; 께; 께; ) HANGUL SYLLABLE GGE +AED9;AED9;1101 1166 11A8;AED9;1101 1166 11A8; # (껙; 껙; 껙; 껙; 껙; ) HANGUL SYLLABLE GGEG +AEDA;AEDA;1101 1166 11A9;AEDA;1101 1166 11A9; # (껚; 껚; 껚; 껚; 껚; ) HANGUL SYLLABLE GGEGG +AEDB;AEDB;1101 1166 11AA;AEDB;1101 1166 11AA; # (껛; 껛; 껛; 껛; 껛; ) HANGUL SYLLABLE GGEGS +AEDC;AEDC;1101 1166 11AB;AEDC;1101 1166 11AB; # (껜; 껜; 껜; 껜; 껜; ) HANGUL SYLLABLE GGEN +AEDD;AEDD;1101 1166 11AC;AEDD;1101 1166 11AC; # (껝; 껝; 껝; 껝; 껝; ) HANGUL SYLLABLE GGENJ +AEDE;AEDE;1101 1166 11AD;AEDE;1101 1166 11AD; # (껞; 껞; 껞; 껞; 껞; ) HANGUL SYLLABLE GGENH +AEDF;AEDF;1101 1166 11AE;AEDF;1101 1166 11AE; # (껟; 껟; 껟; 껟; 껟; ) HANGUL SYLLABLE GGED +AEE0;AEE0;1101 1166 11AF;AEE0;1101 1166 11AF; # (껠; 껠; 껠; 껠; 껠; ) HANGUL SYLLABLE GGEL +AEE1;AEE1;1101 1166 11B0;AEE1;1101 1166 11B0; # (껡; 껡; 껡; 껡; 껡; ) HANGUL SYLLABLE GGELG +AEE2;AEE2;1101 1166 11B1;AEE2;1101 1166 11B1; # (껢; 껢; 껢; 껢; 껢; ) HANGUL SYLLABLE GGELM +AEE3;AEE3;1101 1166 11B2;AEE3;1101 1166 11B2; # (껣; 껣; 껣; 껣; 껣; ) HANGUL SYLLABLE GGELB +AEE4;AEE4;1101 1166 11B3;AEE4;1101 1166 11B3; # (껤; 껤; 껤; 껤; 껤; ) HANGUL SYLLABLE GGELS +AEE5;AEE5;1101 1166 11B4;AEE5;1101 1166 11B4; # (껥; 껥; 껥; 껥; 껥; ) HANGUL SYLLABLE GGELT +AEE6;AEE6;1101 1166 11B5;AEE6;1101 1166 11B5; # (껦; 껦; 껦; 껦; 껦; ) HANGUL SYLLABLE GGELP +AEE7;AEE7;1101 1166 11B6;AEE7;1101 1166 11B6; # (껧; 껧; 껧; 껧; 껧; ) HANGUL SYLLABLE GGELH +AEE8;AEE8;1101 1166 11B7;AEE8;1101 1166 11B7; # (껨; 껨; 껨; 껨; 껨; ) HANGUL SYLLABLE GGEM +AEE9;AEE9;1101 1166 11B8;AEE9;1101 1166 11B8; # (껩; 껩; 껩; 껩; 껩; ) HANGUL SYLLABLE GGEB +AEEA;AEEA;1101 1166 11B9;AEEA;1101 1166 11B9; # (껪; 껪; 껪; 껪; 껪; ) HANGUL SYLLABLE GGEBS +AEEB;AEEB;1101 1166 11BA;AEEB;1101 1166 11BA; # (껫; 껫; 껫; 껫; 껫; ) HANGUL SYLLABLE GGES +AEEC;AEEC;1101 1166 11BB;AEEC;1101 1166 11BB; # (껬; 껬; 껬; 껬; 껬; ) HANGUL SYLLABLE GGESS +AEED;AEED;1101 1166 11BC;AEED;1101 1166 11BC; # (껭; 껭; 껭; 껭; 껭; ) HANGUL SYLLABLE GGENG +AEEE;AEEE;1101 1166 11BD;AEEE;1101 1166 11BD; # (껮; 껮; 껮; 껮; 껮; ) HANGUL SYLLABLE GGEJ +AEEF;AEEF;1101 1166 11BE;AEEF;1101 1166 11BE; # (껯; 껯; 껯; 껯; 껯; ) HANGUL SYLLABLE GGEC +AEF0;AEF0;1101 1166 11BF;AEF0;1101 1166 11BF; # (껰; 껰; 껰; 껰; 껰; ) HANGUL SYLLABLE GGEK +AEF1;AEF1;1101 1166 11C0;AEF1;1101 1166 11C0; # (껱; 껱; 껱; 껱; 껱; ) HANGUL SYLLABLE GGET +AEF2;AEF2;1101 1166 11C1;AEF2;1101 1166 11C1; # (껲; 껲; 껲; 껲; 껲; ) HANGUL SYLLABLE GGEP +AEF3;AEF3;1101 1166 11C2;AEF3;1101 1166 11C2; # (껳; 껳; 껳; 껳; 껳; ) HANGUL SYLLABLE GGEH +AEF4;AEF4;1101 1167;AEF4;1101 1167; # (껴; 껴; 껴; 껴; 껴; ) HANGUL SYLLABLE GGYEO +AEF5;AEF5;1101 1167 11A8;AEF5;1101 1167 11A8; # (껵; 껵; 껵; 껵; 껵; ) HANGUL SYLLABLE GGYEOG +AEF6;AEF6;1101 1167 11A9;AEF6;1101 1167 11A9; # (껶; 껶; 껶; 껶; 껶; ) HANGUL SYLLABLE GGYEOGG +AEF7;AEF7;1101 1167 11AA;AEF7;1101 1167 11AA; # (껷; 껷; 껷; 껷; 껷; ) HANGUL SYLLABLE GGYEOGS +AEF8;AEF8;1101 1167 11AB;AEF8;1101 1167 11AB; # (껸; 껸; 껸; 껸; 껸; ) HANGUL SYLLABLE GGYEON +AEF9;AEF9;1101 1167 11AC;AEF9;1101 1167 11AC; # (껹; 껹; 껹; 껹; 껹; ) HANGUL SYLLABLE GGYEONJ +AEFA;AEFA;1101 1167 11AD;AEFA;1101 1167 11AD; # (껺; 껺; 껺; 껺; 껺; ) HANGUL SYLLABLE GGYEONH +AEFB;AEFB;1101 1167 11AE;AEFB;1101 1167 11AE; # (껻; 껻; 껻; 껻; 껻; ) HANGUL SYLLABLE GGYEOD +AEFC;AEFC;1101 1167 11AF;AEFC;1101 1167 11AF; # (껼; 껼; 껼; 껼; 껼; ) HANGUL SYLLABLE GGYEOL +AEFD;AEFD;1101 1167 11B0;AEFD;1101 1167 11B0; # (껽; 껽; 껽; 껽; 껽; ) HANGUL SYLLABLE GGYEOLG +AEFE;AEFE;1101 1167 11B1;AEFE;1101 1167 11B1; # (껾; 껾; 껾; 껾; 껾; ) HANGUL SYLLABLE GGYEOLM +AEFF;AEFF;1101 1167 11B2;AEFF;1101 1167 11B2; # (껿; 껿; 껿; 껿; 껿; ) HANGUL SYLLABLE GGYEOLB +AF00;AF00;1101 1167 11B3;AF00;1101 1167 11B3; # (꼀; 꼀; 꼀; 꼀; 꼀; ) HANGUL SYLLABLE GGYEOLS +AF01;AF01;1101 1167 11B4;AF01;1101 1167 11B4; # (꼁; 꼁; 꼁; 꼁; 꼁; ) HANGUL SYLLABLE GGYEOLT +AF02;AF02;1101 1167 11B5;AF02;1101 1167 11B5; # (꼂; 꼂; 꼂; 꼂; 꼂; ) HANGUL SYLLABLE GGYEOLP +AF03;AF03;1101 1167 11B6;AF03;1101 1167 11B6; # (꼃; 꼃; 꼃; 꼃; 꼃; ) HANGUL SYLLABLE GGYEOLH +AF04;AF04;1101 1167 11B7;AF04;1101 1167 11B7; # (꼄; 꼄; 꼄; 꼄; 꼄; ) HANGUL SYLLABLE GGYEOM +AF05;AF05;1101 1167 11B8;AF05;1101 1167 11B8; # (꼅; 꼅; 꼅; 꼅; 꼅; ) HANGUL SYLLABLE GGYEOB +AF06;AF06;1101 1167 11B9;AF06;1101 1167 11B9; # (꼆; 꼆; 꼆; 꼆; 꼆; ) HANGUL SYLLABLE GGYEOBS +AF07;AF07;1101 1167 11BA;AF07;1101 1167 11BA; # (꼇; 꼇; 꼇; 꼇; 꼇; ) HANGUL SYLLABLE GGYEOS +AF08;AF08;1101 1167 11BB;AF08;1101 1167 11BB; # (꼈; 꼈; 꼈; 꼈; 꼈; ) HANGUL SYLLABLE GGYEOSS +AF09;AF09;1101 1167 11BC;AF09;1101 1167 11BC; # (꼉; 꼉; 꼉; 꼉; 꼉; ) HANGUL SYLLABLE GGYEONG +AF0A;AF0A;1101 1167 11BD;AF0A;1101 1167 11BD; # (꼊; 꼊; 꼊; 꼊; 꼊; ) HANGUL SYLLABLE GGYEOJ +AF0B;AF0B;1101 1167 11BE;AF0B;1101 1167 11BE; # (꼋; 꼋; 꼋; 꼋; 꼋; ) HANGUL SYLLABLE GGYEOC +AF0C;AF0C;1101 1167 11BF;AF0C;1101 1167 11BF; # (꼌; 꼌; 꼌; 꼌; 꼌; ) HANGUL SYLLABLE GGYEOK +AF0D;AF0D;1101 1167 11C0;AF0D;1101 1167 11C0; # (꼍; 꼍; 꼍; 꼍; 꼍; ) HANGUL SYLLABLE GGYEOT +AF0E;AF0E;1101 1167 11C1;AF0E;1101 1167 11C1; # (꼎; 꼎; 꼎; 꼎; 꼎; ) HANGUL SYLLABLE GGYEOP +AF0F;AF0F;1101 1167 11C2;AF0F;1101 1167 11C2; # (꼏; 꼏; 꼏; 꼏; 꼏; ) HANGUL SYLLABLE GGYEOH +AF10;AF10;1101 1168;AF10;1101 1168; # (꼐; 꼐; 꼐; 꼐; 꼐; ) HANGUL SYLLABLE GGYE +AF11;AF11;1101 1168 11A8;AF11;1101 1168 11A8; # (꼑; 꼑; 꼑; 꼑; 꼑; ) HANGUL SYLLABLE GGYEG +AF12;AF12;1101 1168 11A9;AF12;1101 1168 11A9; # (꼒; 꼒; 꼒; 꼒; 꼒; ) HANGUL SYLLABLE GGYEGG +AF13;AF13;1101 1168 11AA;AF13;1101 1168 11AA; # (꼓; 꼓; 꼓; 꼓; 꼓; ) HANGUL SYLLABLE GGYEGS +AF14;AF14;1101 1168 11AB;AF14;1101 1168 11AB; # (꼔; 꼔; 꼔; 꼔; 꼔; ) HANGUL SYLLABLE GGYEN +AF15;AF15;1101 1168 11AC;AF15;1101 1168 11AC; # (꼕; 꼕; 꼕; 꼕; 꼕; ) HANGUL SYLLABLE GGYENJ +AF16;AF16;1101 1168 11AD;AF16;1101 1168 11AD; # (꼖; 꼖; 꼖; 꼖; 꼖; ) HANGUL SYLLABLE GGYENH +AF17;AF17;1101 1168 11AE;AF17;1101 1168 11AE; # (꼗; 꼗; 꼗; 꼗; 꼗; ) HANGUL SYLLABLE GGYED +AF18;AF18;1101 1168 11AF;AF18;1101 1168 11AF; # (꼘; 꼘; 꼘; 꼘; 꼘; ) HANGUL SYLLABLE GGYEL +AF19;AF19;1101 1168 11B0;AF19;1101 1168 11B0; # (꼙; 꼙; 꼙; 꼙; 꼙; ) HANGUL SYLLABLE GGYELG +AF1A;AF1A;1101 1168 11B1;AF1A;1101 1168 11B1; # (꼚; 꼚; 꼚; 꼚; 꼚; ) HANGUL SYLLABLE GGYELM +AF1B;AF1B;1101 1168 11B2;AF1B;1101 1168 11B2; # (꼛; 꼛; 꼛; 꼛; 꼛; ) HANGUL SYLLABLE GGYELB +AF1C;AF1C;1101 1168 11B3;AF1C;1101 1168 11B3; # (꼜; 꼜; 꼜; 꼜; 꼜; ) HANGUL SYLLABLE GGYELS +AF1D;AF1D;1101 1168 11B4;AF1D;1101 1168 11B4; # (꼝; 꼝; 꼝; 꼝; 꼝; ) HANGUL SYLLABLE GGYELT +AF1E;AF1E;1101 1168 11B5;AF1E;1101 1168 11B5; # (꼞; 꼞; 꼞; 꼞; 꼞; ) HANGUL SYLLABLE GGYELP +AF1F;AF1F;1101 1168 11B6;AF1F;1101 1168 11B6; # (꼟; 꼟; 꼟; 꼟; 꼟; ) HANGUL SYLLABLE GGYELH +AF20;AF20;1101 1168 11B7;AF20;1101 1168 11B7; # (꼠; 꼠; 꼠; 꼠; 꼠; ) HANGUL SYLLABLE GGYEM +AF21;AF21;1101 1168 11B8;AF21;1101 1168 11B8; # (꼡; 꼡; 꼡; 꼡; 꼡; ) HANGUL SYLLABLE GGYEB +AF22;AF22;1101 1168 11B9;AF22;1101 1168 11B9; # (꼢; 꼢; 꼢; 꼢; 꼢; ) HANGUL SYLLABLE GGYEBS +AF23;AF23;1101 1168 11BA;AF23;1101 1168 11BA; # (꼣; 꼣; 꼣; 꼣; 꼣; ) HANGUL SYLLABLE GGYES +AF24;AF24;1101 1168 11BB;AF24;1101 1168 11BB; # (꼤; 꼤; 꼤; 꼤; 꼤; ) HANGUL SYLLABLE GGYESS +AF25;AF25;1101 1168 11BC;AF25;1101 1168 11BC; # (꼥; 꼥; 꼥; 꼥; 꼥; ) HANGUL SYLLABLE GGYENG +AF26;AF26;1101 1168 11BD;AF26;1101 1168 11BD; # (꼦; 꼦; 꼦; 꼦; 꼦; ) HANGUL SYLLABLE GGYEJ +AF27;AF27;1101 1168 11BE;AF27;1101 1168 11BE; # (꼧; 꼧; 꼧; 꼧; 꼧; ) HANGUL SYLLABLE GGYEC +AF28;AF28;1101 1168 11BF;AF28;1101 1168 11BF; # (꼨; 꼨; 꼨; 꼨; 꼨; ) HANGUL SYLLABLE GGYEK +AF29;AF29;1101 1168 11C0;AF29;1101 1168 11C0; # (꼩; 꼩; 꼩; 꼩; 꼩; ) HANGUL SYLLABLE GGYET +AF2A;AF2A;1101 1168 11C1;AF2A;1101 1168 11C1; # (꼪; 꼪; 꼪; 꼪; 꼪; ) HANGUL SYLLABLE GGYEP +AF2B;AF2B;1101 1168 11C2;AF2B;1101 1168 11C2; # (꼫; 꼫; 꼫; 꼫; 꼫; ) HANGUL SYLLABLE GGYEH +AF2C;AF2C;1101 1169;AF2C;1101 1169; # (꼬; 꼬; 꼬; 꼬; 꼬; ) HANGUL SYLLABLE GGO +AF2D;AF2D;1101 1169 11A8;AF2D;1101 1169 11A8; # (꼭; 꼭; 꼭; 꼭; 꼭; ) HANGUL SYLLABLE GGOG +AF2E;AF2E;1101 1169 11A9;AF2E;1101 1169 11A9; # (꼮; 꼮; 꼮; 꼮; 꼮; ) HANGUL SYLLABLE GGOGG +AF2F;AF2F;1101 1169 11AA;AF2F;1101 1169 11AA; # (꼯; 꼯; 꼯; 꼯; 꼯; ) HANGUL SYLLABLE GGOGS +AF30;AF30;1101 1169 11AB;AF30;1101 1169 11AB; # (꼰; 꼰; 꼰; 꼰; 꼰; ) HANGUL SYLLABLE GGON +AF31;AF31;1101 1169 11AC;AF31;1101 1169 11AC; # (꼱; 꼱; 꼱; 꼱; 꼱; ) HANGUL SYLLABLE GGONJ +AF32;AF32;1101 1169 11AD;AF32;1101 1169 11AD; # (꼲; 꼲; 꼲; 꼲; 꼲; ) HANGUL SYLLABLE GGONH +AF33;AF33;1101 1169 11AE;AF33;1101 1169 11AE; # (꼳; 꼳; 꼳; 꼳; 꼳; ) HANGUL SYLLABLE GGOD +AF34;AF34;1101 1169 11AF;AF34;1101 1169 11AF; # (꼴; 꼴; 꼴; 꼴; 꼴; ) HANGUL SYLLABLE GGOL +AF35;AF35;1101 1169 11B0;AF35;1101 1169 11B0; # (꼵; 꼵; 꼵; 꼵; 꼵; ) HANGUL SYLLABLE GGOLG +AF36;AF36;1101 1169 11B1;AF36;1101 1169 11B1; # (꼶; 꼶; 꼶; 꼶; 꼶; ) HANGUL SYLLABLE GGOLM +AF37;AF37;1101 1169 11B2;AF37;1101 1169 11B2; # (꼷; 꼷; 꼷; 꼷; 꼷; ) HANGUL SYLLABLE GGOLB +AF38;AF38;1101 1169 11B3;AF38;1101 1169 11B3; # (꼸; 꼸; 꼸; 꼸; 꼸; ) HANGUL SYLLABLE GGOLS +AF39;AF39;1101 1169 11B4;AF39;1101 1169 11B4; # (꼹; 꼹; 꼹; 꼹; 꼹; ) HANGUL SYLLABLE GGOLT +AF3A;AF3A;1101 1169 11B5;AF3A;1101 1169 11B5; # (꼺; 꼺; 꼺; 꼺; 꼺; ) HANGUL SYLLABLE GGOLP +AF3B;AF3B;1101 1169 11B6;AF3B;1101 1169 11B6; # (꼻; 꼻; 꼻; 꼻; 꼻; ) HANGUL SYLLABLE GGOLH +AF3C;AF3C;1101 1169 11B7;AF3C;1101 1169 11B7; # (꼼; 꼼; 꼼; 꼼; 꼼; ) HANGUL SYLLABLE GGOM +AF3D;AF3D;1101 1169 11B8;AF3D;1101 1169 11B8; # (꼽; 꼽; 꼽; 꼽; 꼽; ) HANGUL SYLLABLE GGOB +AF3E;AF3E;1101 1169 11B9;AF3E;1101 1169 11B9; # (꼾; 꼾; 꼾; 꼾; 꼾; ) HANGUL SYLLABLE GGOBS +AF3F;AF3F;1101 1169 11BA;AF3F;1101 1169 11BA; # (꼿; 꼿; 꼿; 꼿; 꼿; ) HANGUL SYLLABLE GGOS +AF40;AF40;1101 1169 11BB;AF40;1101 1169 11BB; # (꽀; 꽀; 꽀; 꽀; 꽀; ) HANGUL SYLLABLE GGOSS +AF41;AF41;1101 1169 11BC;AF41;1101 1169 11BC; # (꽁; 꽁; 꽁; 꽁; 꽁; ) HANGUL SYLLABLE GGONG +AF42;AF42;1101 1169 11BD;AF42;1101 1169 11BD; # (꽂; 꽂; 꽂; 꽂; 꽂; ) HANGUL SYLLABLE GGOJ +AF43;AF43;1101 1169 11BE;AF43;1101 1169 11BE; # (꽃; 꽃; 꽃; 꽃; 꽃; ) HANGUL SYLLABLE GGOC +AF44;AF44;1101 1169 11BF;AF44;1101 1169 11BF; # (꽄; 꽄; 꽄; 꽄; 꽄; ) HANGUL SYLLABLE GGOK +AF45;AF45;1101 1169 11C0;AF45;1101 1169 11C0; # (꽅; 꽅; 꽅; 꽅; 꽅; ) HANGUL SYLLABLE GGOT +AF46;AF46;1101 1169 11C1;AF46;1101 1169 11C1; # (꽆; 꽆; 꽆; 꽆; 꽆; ) HANGUL SYLLABLE GGOP +AF47;AF47;1101 1169 11C2;AF47;1101 1169 11C2; # (꽇; 꽇; 꽇; 꽇; 꽇; ) HANGUL SYLLABLE GGOH +AF48;AF48;1101 116A;AF48;1101 116A; # (꽈; 꽈; 꽈; 꽈; 꽈; ) HANGUL SYLLABLE GGWA +AF49;AF49;1101 116A 11A8;AF49;1101 116A 11A8; # (꽉; 꽉; 꽉; 꽉; 꽉; ) HANGUL SYLLABLE GGWAG +AF4A;AF4A;1101 116A 11A9;AF4A;1101 116A 11A9; # (꽊; 꽊; 꽊; 꽊; 꽊; ) HANGUL SYLLABLE GGWAGG +AF4B;AF4B;1101 116A 11AA;AF4B;1101 116A 11AA; # (꽋; 꽋; 꽋; 꽋; 꽋; ) HANGUL SYLLABLE GGWAGS +AF4C;AF4C;1101 116A 11AB;AF4C;1101 116A 11AB; # (꽌; 꽌; 꽌; 꽌; 꽌; ) HANGUL SYLLABLE GGWAN +AF4D;AF4D;1101 116A 11AC;AF4D;1101 116A 11AC; # (꽍; 꽍; 꽍; 꽍; 꽍; ) HANGUL SYLLABLE GGWANJ +AF4E;AF4E;1101 116A 11AD;AF4E;1101 116A 11AD; # (꽎; 꽎; 꽎; 꽎; 꽎; ) HANGUL SYLLABLE GGWANH +AF4F;AF4F;1101 116A 11AE;AF4F;1101 116A 11AE; # (꽏; 꽏; 꽏; 꽏; 꽏; ) HANGUL SYLLABLE GGWAD +AF50;AF50;1101 116A 11AF;AF50;1101 116A 11AF; # (꽐; 꽐; 꽐; 꽐; 꽐; ) HANGUL SYLLABLE GGWAL +AF51;AF51;1101 116A 11B0;AF51;1101 116A 11B0; # (꽑; 꽑; 꽑; 꽑; 꽑; ) HANGUL SYLLABLE GGWALG +AF52;AF52;1101 116A 11B1;AF52;1101 116A 11B1; # (꽒; 꽒; 꽒; 꽒; 꽒; ) HANGUL SYLLABLE GGWALM +AF53;AF53;1101 116A 11B2;AF53;1101 116A 11B2; # (꽓; 꽓; 꽓; 꽓; 꽓; ) HANGUL SYLLABLE GGWALB +AF54;AF54;1101 116A 11B3;AF54;1101 116A 11B3; # (꽔; 꽔; 꽔; 꽔; 꽔; ) HANGUL SYLLABLE GGWALS +AF55;AF55;1101 116A 11B4;AF55;1101 116A 11B4; # (꽕; 꽕; 꽕; 꽕; 꽕; ) HANGUL SYLLABLE GGWALT +AF56;AF56;1101 116A 11B5;AF56;1101 116A 11B5; # (꽖; 꽖; 꽖; 꽖; 꽖; ) HANGUL SYLLABLE GGWALP +AF57;AF57;1101 116A 11B6;AF57;1101 116A 11B6; # (꽗; 꽗; 꽗; 꽗; 꽗; ) HANGUL SYLLABLE GGWALH +AF58;AF58;1101 116A 11B7;AF58;1101 116A 11B7; # (꽘; 꽘; 꽘; 꽘; 꽘; ) HANGUL SYLLABLE GGWAM +AF59;AF59;1101 116A 11B8;AF59;1101 116A 11B8; # (꽙; 꽙; 꽙; 꽙; 꽙; ) HANGUL SYLLABLE GGWAB +AF5A;AF5A;1101 116A 11B9;AF5A;1101 116A 11B9; # (꽚; 꽚; 꽚; 꽚; 꽚; ) HANGUL SYLLABLE GGWABS +AF5B;AF5B;1101 116A 11BA;AF5B;1101 116A 11BA; # (꽛; 꽛; 꽛; 꽛; 꽛; ) HANGUL SYLLABLE GGWAS +AF5C;AF5C;1101 116A 11BB;AF5C;1101 116A 11BB; # (꽜; 꽜; 꽜; 꽜; 꽜; ) HANGUL SYLLABLE GGWASS +AF5D;AF5D;1101 116A 11BC;AF5D;1101 116A 11BC; # (꽝; 꽝; 꽝; 꽝; 꽝; ) HANGUL SYLLABLE GGWANG +AF5E;AF5E;1101 116A 11BD;AF5E;1101 116A 11BD; # (꽞; 꽞; 꽞; 꽞; 꽞; ) HANGUL SYLLABLE GGWAJ +AF5F;AF5F;1101 116A 11BE;AF5F;1101 116A 11BE; # (꽟; 꽟; 꽟; 꽟; 꽟; ) HANGUL SYLLABLE GGWAC +AF60;AF60;1101 116A 11BF;AF60;1101 116A 11BF; # (꽠; 꽠; 꽠; 꽠; 꽠; ) HANGUL SYLLABLE GGWAK +AF61;AF61;1101 116A 11C0;AF61;1101 116A 11C0; # (꽡; 꽡; 꽡; 꽡; 꽡; ) HANGUL SYLLABLE GGWAT +AF62;AF62;1101 116A 11C1;AF62;1101 116A 11C1; # (꽢; 꽢; 꽢; 꽢; 꽢; ) HANGUL SYLLABLE GGWAP +AF63;AF63;1101 116A 11C2;AF63;1101 116A 11C2; # (꽣; 꽣; 꽣; 꽣; 꽣; ) HANGUL SYLLABLE GGWAH +AF64;AF64;1101 116B;AF64;1101 116B; # (꽤; 꽤; 꽤; 꽤; 꽤; ) HANGUL SYLLABLE GGWAE +AF65;AF65;1101 116B 11A8;AF65;1101 116B 11A8; # (꽥; 꽥; 꽥; 꽥; 꽥; ) HANGUL SYLLABLE GGWAEG +AF66;AF66;1101 116B 11A9;AF66;1101 116B 11A9; # (꽦; 꽦; 꽦; 꽦; 꽦; ) HANGUL SYLLABLE GGWAEGG +AF67;AF67;1101 116B 11AA;AF67;1101 116B 11AA; # (꽧; 꽧; 꽧; 꽧; 꽧; ) HANGUL SYLLABLE GGWAEGS +AF68;AF68;1101 116B 11AB;AF68;1101 116B 11AB; # (꽨; 꽨; 꽨; 꽨; 꽨; ) HANGUL SYLLABLE GGWAEN +AF69;AF69;1101 116B 11AC;AF69;1101 116B 11AC; # (꽩; 꽩; 꽩; 꽩; 꽩; ) HANGUL SYLLABLE GGWAENJ +AF6A;AF6A;1101 116B 11AD;AF6A;1101 116B 11AD; # (꽪; 꽪; 꽪; 꽪; 꽪; ) HANGUL SYLLABLE GGWAENH +AF6B;AF6B;1101 116B 11AE;AF6B;1101 116B 11AE; # (꽫; 꽫; 꽫; 꽫; 꽫; ) HANGUL SYLLABLE GGWAED +AF6C;AF6C;1101 116B 11AF;AF6C;1101 116B 11AF; # (꽬; 꽬; 꽬; 꽬; 꽬; ) HANGUL SYLLABLE GGWAEL +AF6D;AF6D;1101 116B 11B0;AF6D;1101 116B 11B0; # (꽭; 꽭; 꽭; 꽭; 꽭; ) HANGUL SYLLABLE GGWAELG +AF6E;AF6E;1101 116B 11B1;AF6E;1101 116B 11B1; # (꽮; 꽮; 꽮; 꽮; 꽮; ) HANGUL SYLLABLE GGWAELM +AF6F;AF6F;1101 116B 11B2;AF6F;1101 116B 11B2; # (꽯; 꽯; 꽯; 꽯; 꽯; ) HANGUL SYLLABLE GGWAELB +AF70;AF70;1101 116B 11B3;AF70;1101 116B 11B3; # (꽰; 꽰; 꽰; 꽰; 꽰; ) HANGUL SYLLABLE GGWAELS +AF71;AF71;1101 116B 11B4;AF71;1101 116B 11B4; # (꽱; 꽱; 꽱; 꽱; 꽱; ) HANGUL SYLLABLE GGWAELT +AF72;AF72;1101 116B 11B5;AF72;1101 116B 11B5; # (꽲; 꽲; 꽲; 꽲; 꽲; ) HANGUL SYLLABLE GGWAELP +AF73;AF73;1101 116B 11B6;AF73;1101 116B 11B6; # (꽳; 꽳; 꽳; 꽳; 꽳; ) HANGUL SYLLABLE GGWAELH +AF74;AF74;1101 116B 11B7;AF74;1101 116B 11B7; # (꽴; 꽴; 꽴; 꽴; 꽴; ) HANGUL SYLLABLE GGWAEM +AF75;AF75;1101 116B 11B8;AF75;1101 116B 11B8; # (꽵; 꽵; 꽵; 꽵; 꽵; ) HANGUL SYLLABLE GGWAEB +AF76;AF76;1101 116B 11B9;AF76;1101 116B 11B9; # (꽶; 꽶; 꽶; 꽶; 꽶; ) HANGUL SYLLABLE GGWAEBS +AF77;AF77;1101 116B 11BA;AF77;1101 116B 11BA; # (꽷; 꽷; 꽷; 꽷; 꽷; ) HANGUL SYLLABLE GGWAES +AF78;AF78;1101 116B 11BB;AF78;1101 116B 11BB; # (꽸; 꽸; 꽸; 꽸; 꽸; ) HANGUL SYLLABLE GGWAESS +AF79;AF79;1101 116B 11BC;AF79;1101 116B 11BC; # (꽹; 꽹; 꽹; 꽹; 꽹; ) HANGUL SYLLABLE GGWAENG +AF7A;AF7A;1101 116B 11BD;AF7A;1101 116B 11BD; # (꽺; 꽺; 꽺; 꽺; 꽺; ) HANGUL SYLLABLE GGWAEJ +AF7B;AF7B;1101 116B 11BE;AF7B;1101 116B 11BE; # (꽻; 꽻; 꽻; 꽻; 꽻; ) HANGUL SYLLABLE GGWAEC +AF7C;AF7C;1101 116B 11BF;AF7C;1101 116B 11BF; # (꽼; 꽼; 꽼; 꽼; 꽼; ) HANGUL SYLLABLE GGWAEK +AF7D;AF7D;1101 116B 11C0;AF7D;1101 116B 11C0; # (꽽; 꽽; 꽽; 꽽; 꽽; ) HANGUL SYLLABLE GGWAET +AF7E;AF7E;1101 116B 11C1;AF7E;1101 116B 11C1; # (꽾; 꽾; 꽾; 꽾; 꽾; ) HANGUL SYLLABLE GGWAEP +AF7F;AF7F;1101 116B 11C2;AF7F;1101 116B 11C2; # (꽿; 꽿; 꽿; 꽿; 꽿; ) HANGUL SYLLABLE GGWAEH +AF80;AF80;1101 116C;AF80;1101 116C; # (꾀; 꾀; 꾀; 꾀; 꾀; ) HANGUL SYLLABLE GGOE +AF81;AF81;1101 116C 11A8;AF81;1101 116C 11A8; # (꾁; 꾁; 꾁; 꾁; 꾁; ) HANGUL SYLLABLE GGOEG +AF82;AF82;1101 116C 11A9;AF82;1101 116C 11A9; # (꾂; 꾂; 꾂; 꾂; 꾂; ) HANGUL SYLLABLE GGOEGG +AF83;AF83;1101 116C 11AA;AF83;1101 116C 11AA; # (꾃; 꾃; 꾃; 꾃; 꾃; ) HANGUL SYLLABLE GGOEGS +AF84;AF84;1101 116C 11AB;AF84;1101 116C 11AB; # (꾄; 꾄; 꾄; 꾄; 꾄; ) HANGUL SYLLABLE GGOEN +AF85;AF85;1101 116C 11AC;AF85;1101 116C 11AC; # (꾅; 꾅; 꾅; 꾅; 꾅; ) HANGUL SYLLABLE GGOENJ +AF86;AF86;1101 116C 11AD;AF86;1101 116C 11AD; # (꾆; 꾆; 꾆; 꾆; 꾆; ) HANGUL SYLLABLE GGOENH +AF87;AF87;1101 116C 11AE;AF87;1101 116C 11AE; # (꾇; 꾇; 꾇; 꾇; 꾇; ) HANGUL SYLLABLE GGOED +AF88;AF88;1101 116C 11AF;AF88;1101 116C 11AF; # (꾈; 꾈; 꾈; 꾈; 꾈; ) HANGUL SYLLABLE GGOEL +AF89;AF89;1101 116C 11B0;AF89;1101 116C 11B0; # (꾉; 꾉; 꾉; 꾉; 꾉; ) HANGUL SYLLABLE GGOELG +AF8A;AF8A;1101 116C 11B1;AF8A;1101 116C 11B1; # (꾊; 꾊; 꾊; 꾊; 꾊; ) HANGUL SYLLABLE GGOELM +AF8B;AF8B;1101 116C 11B2;AF8B;1101 116C 11B2; # (꾋; 꾋; 꾋; 꾋; 꾋; ) HANGUL SYLLABLE GGOELB +AF8C;AF8C;1101 116C 11B3;AF8C;1101 116C 11B3; # (꾌; 꾌; 꾌; 꾌; 꾌; ) HANGUL SYLLABLE GGOELS +AF8D;AF8D;1101 116C 11B4;AF8D;1101 116C 11B4; # (꾍; 꾍; 꾍; 꾍; 꾍; ) HANGUL SYLLABLE GGOELT +AF8E;AF8E;1101 116C 11B5;AF8E;1101 116C 11B5; # (꾎; 꾎; 꾎; 꾎; 꾎; ) HANGUL SYLLABLE GGOELP +AF8F;AF8F;1101 116C 11B6;AF8F;1101 116C 11B6; # (꾏; 꾏; 꾏; 꾏; 꾏; ) HANGUL SYLLABLE GGOELH +AF90;AF90;1101 116C 11B7;AF90;1101 116C 11B7; # (꾐; 꾐; 꾐; 꾐; 꾐; ) HANGUL SYLLABLE GGOEM +AF91;AF91;1101 116C 11B8;AF91;1101 116C 11B8; # (꾑; 꾑; 꾑; 꾑; 꾑; ) HANGUL SYLLABLE GGOEB +AF92;AF92;1101 116C 11B9;AF92;1101 116C 11B9; # (꾒; 꾒; 꾒; 꾒; 꾒; ) HANGUL SYLLABLE GGOEBS +AF93;AF93;1101 116C 11BA;AF93;1101 116C 11BA; # (꾓; 꾓; 꾓; 꾓; 꾓; ) HANGUL SYLLABLE GGOES +AF94;AF94;1101 116C 11BB;AF94;1101 116C 11BB; # (꾔; 꾔; 꾔; 꾔; 꾔; ) HANGUL SYLLABLE GGOESS +AF95;AF95;1101 116C 11BC;AF95;1101 116C 11BC; # (꾕; 꾕; 꾕; 꾕; 꾕; ) HANGUL SYLLABLE GGOENG +AF96;AF96;1101 116C 11BD;AF96;1101 116C 11BD; # (꾖; 꾖; 꾖; 꾖; 꾖; ) HANGUL SYLLABLE GGOEJ +AF97;AF97;1101 116C 11BE;AF97;1101 116C 11BE; # (꾗; 꾗; 꾗; 꾗; 꾗; ) HANGUL SYLLABLE GGOEC +AF98;AF98;1101 116C 11BF;AF98;1101 116C 11BF; # (꾘; 꾘; 꾘; 꾘; 꾘; ) HANGUL SYLLABLE GGOEK +AF99;AF99;1101 116C 11C0;AF99;1101 116C 11C0; # (꾙; 꾙; 꾙; 꾙; 꾙; ) HANGUL SYLLABLE GGOET +AF9A;AF9A;1101 116C 11C1;AF9A;1101 116C 11C1; # (꾚; 꾚; 꾚; 꾚; 꾚; ) HANGUL SYLLABLE GGOEP +AF9B;AF9B;1101 116C 11C2;AF9B;1101 116C 11C2; # (꾛; 꾛; 꾛; 꾛; 꾛; ) HANGUL SYLLABLE GGOEH +AF9C;AF9C;1101 116D;AF9C;1101 116D; # (꾜; 꾜; 꾜; 꾜; 꾜; ) HANGUL SYLLABLE GGYO +AF9D;AF9D;1101 116D 11A8;AF9D;1101 116D 11A8; # (꾝; 꾝; 꾝; 꾝; 꾝; ) HANGUL SYLLABLE GGYOG +AF9E;AF9E;1101 116D 11A9;AF9E;1101 116D 11A9; # (꾞; 꾞; 꾞; 꾞; 꾞; ) HANGUL SYLLABLE GGYOGG +AF9F;AF9F;1101 116D 11AA;AF9F;1101 116D 11AA; # (꾟; 꾟; 꾟; 꾟; 꾟; ) HANGUL SYLLABLE GGYOGS +AFA0;AFA0;1101 116D 11AB;AFA0;1101 116D 11AB; # (꾠; 꾠; 꾠; 꾠; 꾠; ) HANGUL SYLLABLE GGYON +AFA1;AFA1;1101 116D 11AC;AFA1;1101 116D 11AC; # (꾡; 꾡; 꾡; 꾡; 꾡; ) HANGUL SYLLABLE GGYONJ +AFA2;AFA2;1101 116D 11AD;AFA2;1101 116D 11AD; # (꾢; 꾢; 꾢; 꾢; 꾢; ) HANGUL SYLLABLE GGYONH +AFA3;AFA3;1101 116D 11AE;AFA3;1101 116D 11AE; # (꾣; 꾣; 꾣; 꾣; 꾣; ) HANGUL SYLLABLE GGYOD +AFA4;AFA4;1101 116D 11AF;AFA4;1101 116D 11AF; # (꾤; 꾤; 꾤; 꾤; 꾤; ) HANGUL SYLLABLE GGYOL +AFA5;AFA5;1101 116D 11B0;AFA5;1101 116D 11B0; # (꾥; 꾥; 꾥; 꾥; 꾥; ) HANGUL SYLLABLE GGYOLG +AFA6;AFA6;1101 116D 11B1;AFA6;1101 116D 11B1; # (꾦; 꾦; 꾦; 꾦; 꾦; ) HANGUL SYLLABLE GGYOLM +AFA7;AFA7;1101 116D 11B2;AFA7;1101 116D 11B2; # (꾧; 꾧; 꾧; 꾧; 꾧; ) HANGUL SYLLABLE GGYOLB +AFA8;AFA8;1101 116D 11B3;AFA8;1101 116D 11B3; # (꾨; 꾨; 꾨; 꾨; 꾨; ) HANGUL SYLLABLE GGYOLS +AFA9;AFA9;1101 116D 11B4;AFA9;1101 116D 11B4; # (꾩; 꾩; 꾩; 꾩; 꾩; ) HANGUL SYLLABLE GGYOLT +AFAA;AFAA;1101 116D 11B5;AFAA;1101 116D 11B5; # (꾪; 꾪; 꾪; 꾪; 꾪; ) HANGUL SYLLABLE GGYOLP +AFAB;AFAB;1101 116D 11B6;AFAB;1101 116D 11B6; # (꾫; 꾫; 꾫; 꾫; 꾫; ) HANGUL SYLLABLE GGYOLH +AFAC;AFAC;1101 116D 11B7;AFAC;1101 116D 11B7; # (꾬; 꾬; 꾬; 꾬; 꾬; ) HANGUL SYLLABLE GGYOM +AFAD;AFAD;1101 116D 11B8;AFAD;1101 116D 11B8; # (꾭; 꾭; 꾭; 꾭; 꾭; ) HANGUL SYLLABLE GGYOB +AFAE;AFAE;1101 116D 11B9;AFAE;1101 116D 11B9; # (꾮; 꾮; 꾮; 꾮; 꾮; ) HANGUL SYLLABLE GGYOBS +AFAF;AFAF;1101 116D 11BA;AFAF;1101 116D 11BA; # (꾯; 꾯; 꾯; 꾯; 꾯; ) HANGUL SYLLABLE GGYOS +AFB0;AFB0;1101 116D 11BB;AFB0;1101 116D 11BB; # (꾰; 꾰; 꾰; 꾰; 꾰; ) HANGUL SYLLABLE GGYOSS +AFB1;AFB1;1101 116D 11BC;AFB1;1101 116D 11BC; # (꾱; 꾱; 꾱; 꾱; 꾱; ) HANGUL SYLLABLE GGYONG +AFB2;AFB2;1101 116D 11BD;AFB2;1101 116D 11BD; # (꾲; 꾲; 꾲; 꾲; 꾲; ) HANGUL SYLLABLE GGYOJ +AFB3;AFB3;1101 116D 11BE;AFB3;1101 116D 11BE; # (꾳; 꾳; 꾳; 꾳; 꾳; ) HANGUL SYLLABLE GGYOC +AFB4;AFB4;1101 116D 11BF;AFB4;1101 116D 11BF; # (꾴; 꾴; 꾴; 꾴; 꾴; ) HANGUL SYLLABLE GGYOK +AFB5;AFB5;1101 116D 11C0;AFB5;1101 116D 11C0; # (꾵; 꾵; 꾵; 꾵; 꾵; ) HANGUL SYLLABLE GGYOT +AFB6;AFB6;1101 116D 11C1;AFB6;1101 116D 11C1; # (꾶; 꾶; 꾶; 꾶; 꾶; ) HANGUL SYLLABLE GGYOP +AFB7;AFB7;1101 116D 11C2;AFB7;1101 116D 11C2; # (꾷; 꾷; 꾷; 꾷; 꾷; ) HANGUL SYLLABLE GGYOH +AFB8;AFB8;1101 116E;AFB8;1101 116E; # (꾸; 꾸; 꾸; 꾸; 꾸; ) HANGUL SYLLABLE GGU +AFB9;AFB9;1101 116E 11A8;AFB9;1101 116E 11A8; # (꾹; 꾹; 꾹; 꾹; 꾹; ) HANGUL SYLLABLE GGUG +AFBA;AFBA;1101 116E 11A9;AFBA;1101 116E 11A9; # (꾺; 꾺; 꾺; 꾺; 꾺; ) HANGUL SYLLABLE GGUGG +AFBB;AFBB;1101 116E 11AA;AFBB;1101 116E 11AA; # (꾻; 꾻; 꾻; 꾻; 꾻; ) HANGUL SYLLABLE GGUGS +AFBC;AFBC;1101 116E 11AB;AFBC;1101 116E 11AB; # (꾼; 꾼; 꾼; 꾼; 꾼; ) HANGUL SYLLABLE GGUN +AFBD;AFBD;1101 116E 11AC;AFBD;1101 116E 11AC; # (꾽; 꾽; 꾽; 꾽; 꾽; ) HANGUL SYLLABLE GGUNJ +AFBE;AFBE;1101 116E 11AD;AFBE;1101 116E 11AD; # (꾾; 꾾; 꾾; 꾾; 꾾; ) HANGUL SYLLABLE GGUNH +AFBF;AFBF;1101 116E 11AE;AFBF;1101 116E 11AE; # (꾿; 꾿; 꾿; 꾿; 꾿; ) HANGUL SYLLABLE GGUD +AFC0;AFC0;1101 116E 11AF;AFC0;1101 116E 11AF; # (꿀; 꿀; 꿀; 꿀; 꿀; ) HANGUL SYLLABLE GGUL +AFC1;AFC1;1101 116E 11B0;AFC1;1101 116E 11B0; # (꿁; 꿁; 꿁; 꿁; 꿁; ) HANGUL SYLLABLE GGULG +AFC2;AFC2;1101 116E 11B1;AFC2;1101 116E 11B1; # (꿂; 꿂; 꿂; 꿂; 꿂; ) HANGUL SYLLABLE GGULM +AFC3;AFC3;1101 116E 11B2;AFC3;1101 116E 11B2; # (꿃; 꿃; 꿃; 꿃; 꿃; ) HANGUL SYLLABLE GGULB +AFC4;AFC4;1101 116E 11B3;AFC4;1101 116E 11B3; # (꿄; 꿄; 꿄; 꿄; 꿄; ) HANGUL SYLLABLE GGULS +AFC5;AFC5;1101 116E 11B4;AFC5;1101 116E 11B4; # (꿅; 꿅; 꿅; 꿅; 꿅; ) HANGUL SYLLABLE GGULT +AFC6;AFC6;1101 116E 11B5;AFC6;1101 116E 11B5; # (꿆; 꿆; 꿆; 꿆; 꿆; ) HANGUL SYLLABLE GGULP +AFC7;AFC7;1101 116E 11B6;AFC7;1101 116E 11B6; # (꿇; 꿇; 꿇; 꿇; 꿇; ) HANGUL SYLLABLE GGULH +AFC8;AFC8;1101 116E 11B7;AFC8;1101 116E 11B7; # (꿈; 꿈; 꿈; 꿈; 꿈; ) HANGUL SYLLABLE GGUM +AFC9;AFC9;1101 116E 11B8;AFC9;1101 116E 11B8; # (꿉; 꿉; 꿉; 꿉; 꿉; ) HANGUL SYLLABLE GGUB +AFCA;AFCA;1101 116E 11B9;AFCA;1101 116E 11B9; # (꿊; 꿊; 꿊; 꿊; 꿊; ) HANGUL SYLLABLE GGUBS +AFCB;AFCB;1101 116E 11BA;AFCB;1101 116E 11BA; # (꿋; 꿋; 꿋; 꿋; 꿋; ) HANGUL SYLLABLE GGUS +AFCC;AFCC;1101 116E 11BB;AFCC;1101 116E 11BB; # (꿌; 꿌; 꿌; 꿌; 꿌; ) HANGUL SYLLABLE GGUSS +AFCD;AFCD;1101 116E 11BC;AFCD;1101 116E 11BC; # (꿍; 꿍; 꿍; 꿍; 꿍; ) HANGUL SYLLABLE GGUNG +AFCE;AFCE;1101 116E 11BD;AFCE;1101 116E 11BD; # (꿎; 꿎; 꿎; 꿎; 꿎; ) HANGUL SYLLABLE GGUJ +AFCF;AFCF;1101 116E 11BE;AFCF;1101 116E 11BE; # (꿏; 꿏; 꿏; 꿏; 꿏; ) HANGUL SYLLABLE GGUC +AFD0;AFD0;1101 116E 11BF;AFD0;1101 116E 11BF; # (꿐; 꿐; 꿐; 꿐; 꿐; ) HANGUL SYLLABLE GGUK +AFD1;AFD1;1101 116E 11C0;AFD1;1101 116E 11C0; # (꿑; 꿑; 꿑; 꿑; 꿑; ) HANGUL SYLLABLE GGUT +AFD2;AFD2;1101 116E 11C1;AFD2;1101 116E 11C1; # (꿒; 꿒; 꿒; 꿒; 꿒; ) HANGUL SYLLABLE GGUP +AFD3;AFD3;1101 116E 11C2;AFD3;1101 116E 11C2; # (꿓; 꿓; 꿓; 꿓; 꿓; ) HANGUL SYLLABLE GGUH +AFD4;AFD4;1101 116F;AFD4;1101 116F; # (꿔; 꿔; 꿔; 꿔; 꿔; ) HANGUL SYLLABLE GGWEO +AFD5;AFD5;1101 116F 11A8;AFD5;1101 116F 11A8; # (꿕; 꿕; 꿕; 꿕; 꿕; ) HANGUL SYLLABLE GGWEOG +AFD6;AFD6;1101 116F 11A9;AFD6;1101 116F 11A9; # (꿖; 꿖; 꿖; 꿖; 꿖; ) HANGUL SYLLABLE GGWEOGG +AFD7;AFD7;1101 116F 11AA;AFD7;1101 116F 11AA; # (꿗; 꿗; 꿗; 꿗; 꿗; ) HANGUL SYLLABLE GGWEOGS +AFD8;AFD8;1101 116F 11AB;AFD8;1101 116F 11AB; # (꿘; 꿘; 꿘; 꿘; 꿘; ) HANGUL SYLLABLE GGWEON +AFD9;AFD9;1101 116F 11AC;AFD9;1101 116F 11AC; # (꿙; 꿙; 꿙; 꿙; 꿙; ) HANGUL SYLLABLE GGWEONJ +AFDA;AFDA;1101 116F 11AD;AFDA;1101 116F 11AD; # (꿚; 꿚; 꿚; 꿚; 꿚; ) HANGUL SYLLABLE GGWEONH +AFDB;AFDB;1101 116F 11AE;AFDB;1101 116F 11AE; # (꿛; 꿛; 꿛; 꿛; 꿛; ) HANGUL SYLLABLE GGWEOD +AFDC;AFDC;1101 116F 11AF;AFDC;1101 116F 11AF; # (꿜; 꿜; 꿜; 꿜; 꿜; ) HANGUL SYLLABLE GGWEOL +AFDD;AFDD;1101 116F 11B0;AFDD;1101 116F 11B0; # (꿝; 꿝; 꿝; 꿝; 꿝; ) HANGUL SYLLABLE GGWEOLG +AFDE;AFDE;1101 116F 11B1;AFDE;1101 116F 11B1; # (꿞; 꿞; 꿞; 꿞; 꿞; ) HANGUL SYLLABLE GGWEOLM +AFDF;AFDF;1101 116F 11B2;AFDF;1101 116F 11B2; # (꿟; 꿟; 꿟; 꿟; 꿟; ) HANGUL SYLLABLE GGWEOLB +AFE0;AFE0;1101 116F 11B3;AFE0;1101 116F 11B3; # (꿠; 꿠; 꿠; 꿠; 꿠; ) HANGUL SYLLABLE GGWEOLS +AFE1;AFE1;1101 116F 11B4;AFE1;1101 116F 11B4; # (꿡; 꿡; 꿡; 꿡; 꿡; ) HANGUL SYLLABLE GGWEOLT +AFE2;AFE2;1101 116F 11B5;AFE2;1101 116F 11B5; # (꿢; 꿢; 꿢; 꿢; 꿢; ) HANGUL SYLLABLE GGWEOLP +AFE3;AFE3;1101 116F 11B6;AFE3;1101 116F 11B6; # (꿣; 꿣; 꿣; 꿣; 꿣; ) HANGUL SYLLABLE GGWEOLH +AFE4;AFE4;1101 116F 11B7;AFE4;1101 116F 11B7; # (꿤; 꿤; 꿤; 꿤; 꿤; ) HANGUL SYLLABLE GGWEOM +AFE5;AFE5;1101 116F 11B8;AFE5;1101 116F 11B8; # (꿥; 꿥; 꿥; 꿥; 꿥; ) HANGUL SYLLABLE GGWEOB +AFE6;AFE6;1101 116F 11B9;AFE6;1101 116F 11B9; # (꿦; 꿦; 꿦; 꿦; 꿦; ) HANGUL SYLLABLE GGWEOBS +AFE7;AFE7;1101 116F 11BA;AFE7;1101 116F 11BA; # (꿧; 꿧; 꿧; 꿧; 꿧; ) HANGUL SYLLABLE GGWEOS +AFE8;AFE8;1101 116F 11BB;AFE8;1101 116F 11BB; # (꿨; 꿨; 꿨; 꿨; 꿨; ) HANGUL SYLLABLE GGWEOSS +AFE9;AFE9;1101 116F 11BC;AFE9;1101 116F 11BC; # (꿩; 꿩; 꿩; 꿩; 꿩; ) HANGUL SYLLABLE GGWEONG +AFEA;AFEA;1101 116F 11BD;AFEA;1101 116F 11BD; # (꿪; 꿪; 꿪; 꿪; 꿪; ) HANGUL SYLLABLE GGWEOJ +AFEB;AFEB;1101 116F 11BE;AFEB;1101 116F 11BE; # (꿫; 꿫; 꿫; 꿫; 꿫; ) HANGUL SYLLABLE GGWEOC +AFEC;AFEC;1101 116F 11BF;AFEC;1101 116F 11BF; # (꿬; 꿬; 꿬; 꿬; 꿬; ) HANGUL SYLLABLE GGWEOK +AFED;AFED;1101 116F 11C0;AFED;1101 116F 11C0; # (꿭; 꿭; 꿭; 꿭; 꿭; ) HANGUL SYLLABLE GGWEOT +AFEE;AFEE;1101 116F 11C1;AFEE;1101 116F 11C1; # (꿮; 꿮; 꿮; 꿮; 꿮; ) HANGUL SYLLABLE GGWEOP +AFEF;AFEF;1101 116F 11C2;AFEF;1101 116F 11C2; # (꿯; 꿯; 꿯; 꿯; 꿯; ) HANGUL SYLLABLE GGWEOH +AFF0;AFF0;1101 1170;AFF0;1101 1170; # (꿰; 꿰; 꿰; 꿰; 꿰; ) HANGUL SYLLABLE GGWE +AFF1;AFF1;1101 1170 11A8;AFF1;1101 1170 11A8; # (꿱; 꿱; 꿱; 꿱; 꿱; ) HANGUL SYLLABLE GGWEG +AFF2;AFF2;1101 1170 11A9;AFF2;1101 1170 11A9; # (꿲; 꿲; 꿲; 꿲; 꿲; ) HANGUL SYLLABLE GGWEGG +AFF3;AFF3;1101 1170 11AA;AFF3;1101 1170 11AA; # (꿳; 꿳; 꿳; 꿳; 꿳; ) HANGUL SYLLABLE GGWEGS +AFF4;AFF4;1101 1170 11AB;AFF4;1101 1170 11AB; # (꿴; 꿴; 꿴; 꿴; 꿴; ) HANGUL SYLLABLE GGWEN +AFF5;AFF5;1101 1170 11AC;AFF5;1101 1170 11AC; # (꿵; 꿵; 꿵; 꿵; 꿵; ) HANGUL SYLLABLE GGWENJ +AFF6;AFF6;1101 1170 11AD;AFF6;1101 1170 11AD; # (꿶; 꿶; 꿶; 꿶; 꿶; ) HANGUL SYLLABLE GGWENH +AFF7;AFF7;1101 1170 11AE;AFF7;1101 1170 11AE; # (꿷; 꿷; 꿷; 꿷; 꿷; ) HANGUL SYLLABLE GGWED +AFF8;AFF8;1101 1170 11AF;AFF8;1101 1170 11AF; # (꿸; 꿸; 꿸; 꿸; 꿸; ) HANGUL SYLLABLE GGWEL +AFF9;AFF9;1101 1170 11B0;AFF9;1101 1170 11B0; # (꿹; 꿹; 꿹; 꿹; 꿹; ) HANGUL SYLLABLE GGWELG +AFFA;AFFA;1101 1170 11B1;AFFA;1101 1170 11B1; # (꿺; 꿺; 꿺; 꿺; 꿺; ) HANGUL SYLLABLE GGWELM +AFFB;AFFB;1101 1170 11B2;AFFB;1101 1170 11B2; # (꿻; 꿻; 꿻; 꿻; 꿻; ) HANGUL SYLLABLE GGWELB +AFFC;AFFC;1101 1170 11B3;AFFC;1101 1170 11B3; # (꿼; 꿼; 꿼; 꿼; 꿼; ) HANGUL SYLLABLE GGWELS +AFFD;AFFD;1101 1170 11B4;AFFD;1101 1170 11B4; # (꿽; 꿽; 꿽; 꿽; 꿽; ) HANGUL SYLLABLE GGWELT +AFFE;AFFE;1101 1170 11B5;AFFE;1101 1170 11B5; # (꿾; 꿾; 꿾; 꿾; 꿾; ) HANGUL SYLLABLE GGWELP +AFFF;AFFF;1101 1170 11B6;AFFF;1101 1170 11B6; # (꿿; 꿿; 꿿; 꿿; 꿿; ) HANGUL SYLLABLE GGWELH +B000;B000;1101 1170 11B7;B000;1101 1170 11B7; # (뀀; 뀀; 뀀; 뀀; 뀀; ) HANGUL SYLLABLE GGWEM +B001;B001;1101 1170 11B8;B001;1101 1170 11B8; # (뀁; 뀁; 뀁; 뀁; 뀁; ) HANGUL SYLLABLE GGWEB +B002;B002;1101 1170 11B9;B002;1101 1170 11B9; # (뀂; 뀂; 뀂; 뀂; 뀂; ) HANGUL SYLLABLE GGWEBS +B003;B003;1101 1170 11BA;B003;1101 1170 11BA; # (뀃; 뀃; 뀃; 뀃; 뀃; ) HANGUL SYLLABLE GGWES +B004;B004;1101 1170 11BB;B004;1101 1170 11BB; # (뀄; 뀄; 뀄; 뀄; 뀄; ) HANGUL SYLLABLE GGWESS +B005;B005;1101 1170 11BC;B005;1101 1170 11BC; # (뀅; 뀅; 뀅; 뀅; 뀅; ) HANGUL SYLLABLE GGWENG +B006;B006;1101 1170 11BD;B006;1101 1170 11BD; # (뀆; 뀆; 뀆; 뀆; 뀆; ) HANGUL SYLLABLE GGWEJ +B007;B007;1101 1170 11BE;B007;1101 1170 11BE; # (뀇; 뀇; 뀇; 뀇; 뀇; ) HANGUL SYLLABLE GGWEC +B008;B008;1101 1170 11BF;B008;1101 1170 11BF; # (뀈; 뀈; 뀈; 뀈; 뀈; ) HANGUL SYLLABLE GGWEK +B009;B009;1101 1170 11C0;B009;1101 1170 11C0; # (뀉; 뀉; 뀉; 뀉; 뀉; ) HANGUL SYLLABLE GGWET +B00A;B00A;1101 1170 11C1;B00A;1101 1170 11C1; # (뀊; 뀊; 뀊; 뀊; 뀊; ) HANGUL SYLLABLE GGWEP +B00B;B00B;1101 1170 11C2;B00B;1101 1170 11C2; # (뀋; 뀋; 뀋; 뀋; 뀋; ) HANGUL SYLLABLE GGWEH +B00C;B00C;1101 1171;B00C;1101 1171; # (뀌; 뀌; 뀌; 뀌; 뀌; ) HANGUL SYLLABLE GGWI +B00D;B00D;1101 1171 11A8;B00D;1101 1171 11A8; # (뀍; 뀍; 뀍; 뀍; 뀍; ) HANGUL SYLLABLE GGWIG +B00E;B00E;1101 1171 11A9;B00E;1101 1171 11A9; # (뀎; 뀎; 뀎; 뀎; 뀎; ) HANGUL SYLLABLE GGWIGG +B00F;B00F;1101 1171 11AA;B00F;1101 1171 11AA; # (뀏; 뀏; 뀏; 뀏; 뀏; ) HANGUL SYLLABLE GGWIGS +B010;B010;1101 1171 11AB;B010;1101 1171 11AB; # (뀐; 뀐; 뀐; 뀐; 뀐; ) HANGUL SYLLABLE GGWIN +B011;B011;1101 1171 11AC;B011;1101 1171 11AC; # (뀑; 뀑; 뀑; 뀑; 뀑; ) HANGUL SYLLABLE GGWINJ +B012;B012;1101 1171 11AD;B012;1101 1171 11AD; # (뀒; 뀒; 뀒; 뀒; 뀒; ) HANGUL SYLLABLE GGWINH +B013;B013;1101 1171 11AE;B013;1101 1171 11AE; # (뀓; 뀓; 뀓; 뀓; 뀓; ) HANGUL SYLLABLE GGWID +B014;B014;1101 1171 11AF;B014;1101 1171 11AF; # (뀔; 뀔; 뀔; 뀔; 뀔; ) HANGUL SYLLABLE GGWIL +B015;B015;1101 1171 11B0;B015;1101 1171 11B0; # (뀕; 뀕; 뀕; 뀕; 뀕; ) HANGUL SYLLABLE GGWILG +B016;B016;1101 1171 11B1;B016;1101 1171 11B1; # (뀖; 뀖; 뀖; 뀖; 뀖; ) HANGUL SYLLABLE GGWILM +B017;B017;1101 1171 11B2;B017;1101 1171 11B2; # (뀗; 뀗; 뀗; 뀗; 뀗; ) HANGUL SYLLABLE GGWILB +B018;B018;1101 1171 11B3;B018;1101 1171 11B3; # (뀘; 뀘; 뀘; 뀘; 뀘; ) HANGUL SYLLABLE GGWILS +B019;B019;1101 1171 11B4;B019;1101 1171 11B4; # (뀙; 뀙; 뀙; 뀙; 뀙; ) HANGUL SYLLABLE GGWILT +B01A;B01A;1101 1171 11B5;B01A;1101 1171 11B5; # (뀚; 뀚; 뀚; 뀚; 뀚; ) HANGUL SYLLABLE GGWILP +B01B;B01B;1101 1171 11B6;B01B;1101 1171 11B6; # (뀛; 뀛; 뀛; 뀛; 뀛; ) HANGUL SYLLABLE GGWILH +B01C;B01C;1101 1171 11B7;B01C;1101 1171 11B7; # (뀜; 뀜; 뀜; 뀜; 뀜; ) HANGUL SYLLABLE GGWIM +B01D;B01D;1101 1171 11B8;B01D;1101 1171 11B8; # (뀝; 뀝; 뀝; 뀝; 뀝; ) HANGUL SYLLABLE GGWIB +B01E;B01E;1101 1171 11B9;B01E;1101 1171 11B9; # (뀞; 뀞; 뀞; 뀞; 뀞; ) HANGUL SYLLABLE GGWIBS +B01F;B01F;1101 1171 11BA;B01F;1101 1171 11BA; # (뀟; 뀟; 뀟; 뀟; 뀟; ) HANGUL SYLLABLE GGWIS +B020;B020;1101 1171 11BB;B020;1101 1171 11BB; # (뀠; 뀠; 뀠; 뀠; 뀠; ) HANGUL SYLLABLE GGWISS +B021;B021;1101 1171 11BC;B021;1101 1171 11BC; # (뀡; 뀡; 뀡; 뀡; 뀡; ) HANGUL SYLLABLE GGWING +B022;B022;1101 1171 11BD;B022;1101 1171 11BD; # (뀢; 뀢; 뀢; 뀢; 뀢; ) HANGUL SYLLABLE GGWIJ +B023;B023;1101 1171 11BE;B023;1101 1171 11BE; # (뀣; 뀣; 뀣; 뀣; 뀣; ) HANGUL SYLLABLE GGWIC +B024;B024;1101 1171 11BF;B024;1101 1171 11BF; # (뀤; 뀤; 뀤; 뀤; 뀤; ) HANGUL SYLLABLE GGWIK +B025;B025;1101 1171 11C0;B025;1101 1171 11C0; # (뀥; 뀥; 뀥; 뀥; 뀥; ) HANGUL SYLLABLE GGWIT +B026;B026;1101 1171 11C1;B026;1101 1171 11C1; # (뀦; 뀦; 뀦; 뀦; 뀦; ) HANGUL SYLLABLE GGWIP +B027;B027;1101 1171 11C2;B027;1101 1171 11C2; # (뀧; 뀧; 뀧; 뀧; 뀧; ) HANGUL SYLLABLE GGWIH +B028;B028;1101 1172;B028;1101 1172; # (뀨; 뀨; 뀨; 뀨; 뀨; ) HANGUL SYLLABLE GGYU +B029;B029;1101 1172 11A8;B029;1101 1172 11A8; # (뀩; 뀩; 뀩; 뀩; 뀩; ) HANGUL SYLLABLE GGYUG +B02A;B02A;1101 1172 11A9;B02A;1101 1172 11A9; # (뀪; 뀪; 뀪; 뀪; 뀪; ) HANGUL SYLLABLE GGYUGG +B02B;B02B;1101 1172 11AA;B02B;1101 1172 11AA; # (뀫; 뀫; 뀫; 뀫; 뀫; ) HANGUL SYLLABLE GGYUGS +B02C;B02C;1101 1172 11AB;B02C;1101 1172 11AB; # (뀬; 뀬; 뀬; 뀬; 뀬; ) HANGUL SYLLABLE GGYUN +B02D;B02D;1101 1172 11AC;B02D;1101 1172 11AC; # (뀭; 뀭; 뀭; 뀭; 뀭; ) HANGUL SYLLABLE GGYUNJ +B02E;B02E;1101 1172 11AD;B02E;1101 1172 11AD; # (뀮; 뀮; 뀮; 뀮; 뀮; ) HANGUL SYLLABLE GGYUNH +B02F;B02F;1101 1172 11AE;B02F;1101 1172 11AE; # (뀯; 뀯; 뀯; 뀯; 뀯; ) HANGUL SYLLABLE GGYUD +B030;B030;1101 1172 11AF;B030;1101 1172 11AF; # (뀰; 뀰; 뀰; 뀰; 뀰; ) HANGUL SYLLABLE GGYUL +B031;B031;1101 1172 11B0;B031;1101 1172 11B0; # (뀱; 뀱; 뀱; 뀱; 뀱; ) HANGUL SYLLABLE GGYULG +B032;B032;1101 1172 11B1;B032;1101 1172 11B1; # (뀲; 뀲; 뀲; 뀲; 뀲; ) HANGUL SYLLABLE GGYULM +B033;B033;1101 1172 11B2;B033;1101 1172 11B2; # (뀳; 뀳; 뀳; 뀳; 뀳; ) HANGUL SYLLABLE GGYULB +B034;B034;1101 1172 11B3;B034;1101 1172 11B3; # (뀴; 뀴; 뀴; 뀴; 뀴; ) HANGUL SYLLABLE GGYULS +B035;B035;1101 1172 11B4;B035;1101 1172 11B4; # (뀵; 뀵; 뀵; 뀵; 뀵; ) HANGUL SYLLABLE GGYULT +B036;B036;1101 1172 11B5;B036;1101 1172 11B5; # (뀶; 뀶; 뀶; 뀶; 뀶; ) HANGUL SYLLABLE GGYULP +B037;B037;1101 1172 11B6;B037;1101 1172 11B6; # (뀷; 뀷; 뀷; 뀷; 뀷; ) HANGUL SYLLABLE GGYULH +B038;B038;1101 1172 11B7;B038;1101 1172 11B7; # (뀸; 뀸; 뀸; 뀸; 뀸; ) HANGUL SYLLABLE GGYUM +B039;B039;1101 1172 11B8;B039;1101 1172 11B8; # (뀹; 뀹; 뀹; 뀹; 뀹; ) HANGUL SYLLABLE GGYUB +B03A;B03A;1101 1172 11B9;B03A;1101 1172 11B9; # (뀺; 뀺; 뀺; 뀺; 뀺; ) HANGUL SYLLABLE GGYUBS +B03B;B03B;1101 1172 11BA;B03B;1101 1172 11BA; # (뀻; 뀻; 뀻; 뀻; 뀻; ) HANGUL SYLLABLE GGYUS +B03C;B03C;1101 1172 11BB;B03C;1101 1172 11BB; # (뀼; 뀼; 뀼; 뀼; 뀼; ) HANGUL SYLLABLE GGYUSS +B03D;B03D;1101 1172 11BC;B03D;1101 1172 11BC; # (뀽; 뀽; 뀽; 뀽; 뀽; ) HANGUL SYLLABLE GGYUNG +B03E;B03E;1101 1172 11BD;B03E;1101 1172 11BD; # (뀾; 뀾; 뀾; 뀾; 뀾; ) HANGUL SYLLABLE GGYUJ +B03F;B03F;1101 1172 11BE;B03F;1101 1172 11BE; # (뀿; 뀿; 뀿; 뀿; 뀿; ) HANGUL SYLLABLE GGYUC +B040;B040;1101 1172 11BF;B040;1101 1172 11BF; # (끀; 끀; 끀; 끀; 끀; ) HANGUL SYLLABLE GGYUK +B041;B041;1101 1172 11C0;B041;1101 1172 11C0; # (끁; 끁; 끁; 끁; 끁; ) HANGUL SYLLABLE GGYUT +B042;B042;1101 1172 11C1;B042;1101 1172 11C1; # (끂; 끂; 끂; 끂; 끂; ) HANGUL SYLLABLE GGYUP +B043;B043;1101 1172 11C2;B043;1101 1172 11C2; # (끃; 끃; 끃; 끃; 끃; ) HANGUL SYLLABLE GGYUH +B044;B044;1101 1173;B044;1101 1173; # (끄; 끄; 끄; 끄; 끄; ) HANGUL SYLLABLE GGEU +B045;B045;1101 1173 11A8;B045;1101 1173 11A8; # (끅; 끅; 끅; 끅; 끅; ) HANGUL SYLLABLE GGEUG +B046;B046;1101 1173 11A9;B046;1101 1173 11A9; # (끆; 끆; 끆; 끆; 끆; ) HANGUL SYLLABLE GGEUGG +B047;B047;1101 1173 11AA;B047;1101 1173 11AA; # (끇; 끇; 끇; 끇; 끇; ) HANGUL SYLLABLE GGEUGS +B048;B048;1101 1173 11AB;B048;1101 1173 11AB; # (끈; 끈; 끈; 끈; 끈; ) HANGUL SYLLABLE GGEUN +B049;B049;1101 1173 11AC;B049;1101 1173 11AC; # (끉; 끉; 끉; 끉; 끉; ) HANGUL SYLLABLE GGEUNJ +B04A;B04A;1101 1173 11AD;B04A;1101 1173 11AD; # (끊; 끊; 끊; 끊; 끊; ) HANGUL SYLLABLE GGEUNH +B04B;B04B;1101 1173 11AE;B04B;1101 1173 11AE; # (끋; 끋; 끋; 끋; 끋; ) HANGUL SYLLABLE GGEUD +B04C;B04C;1101 1173 11AF;B04C;1101 1173 11AF; # (끌; 끌; 끌; 끌; 끌; ) HANGUL SYLLABLE GGEUL +B04D;B04D;1101 1173 11B0;B04D;1101 1173 11B0; # (끍; 끍; 끍; 끍; 끍; ) HANGUL SYLLABLE GGEULG +B04E;B04E;1101 1173 11B1;B04E;1101 1173 11B1; # (끎; 끎; 끎; 끎; 끎; ) HANGUL SYLLABLE GGEULM +B04F;B04F;1101 1173 11B2;B04F;1101 1173 11B2; # (끏; 끏; 끏; 끏; 끏; ) HANGUL SYLLABLE GGEULB +B050;B050;1101 1173 11B3;B050;1101 1173 11B3; # (끐; 끐; 끐; 끐; 끐; ) HANGUL SYLLABLE GGEULS +B051;B051;1101 1173 11B4;B051;1101 1173 11B4; # (끑; 끑; 끑; 끑; 끑; ) HANGUL SYLLABLE GGEULT +B052;B052;1101 1173 11B5;B052;1101 1173 11B5; # (끒; 끒; 끒; 끒; 끒; ) HANGUL SYLLABLE GGEULP +B053;B053;1101 1173 11B6;B053;1101 1173 11B6; # (끓; 끓; 끓; 끓; 끓; ) HANGUL SYLLABLE GGEULH +B054;B054;1101 1173 11B7;B054;1101 1173 11B7; # (끔; 끔; 끔; 끔; 끔; ) HANGUL SYLLABLE GGEUM +B055;B055;1101 1173 11B8;B055;1101 1173 11B8; # (끕; 끕; 끕; 끕; 끕; ) HANGUL SYLLABLE GGEUB +B056;B056;1101 1173 11B9;B056;1101 1173 11B9; # (끖; 끖; 끖; 끖; 끖; ) HANGUL SYLLABLE GGEUBS +B057;B057;1101 1173 11BA;B057;1101 1173 11BA; # (끗; 끗; 끗; 끗; 끗; ) HANGUL SYLLABLE GGEUS +B058;B058;1101 1173 11BB;B058;1101 1173 11BB; # (끘; 끘; 끘; 끘; 끘; ) HANGUL SYLLABLE GGEUSS +B059;B059;1101 1173 11BC;B059;1101 1173 11BC; # (끙; 끙; 끙; 끙; 끙; ) HANGUL SYLLABLE GGEUNG +B05A;B05A;1101 1173 11BD;B05A;1101 1173 11BD; # (끚; 끚; 끚; 끚; 끚; ) HANGUL SYLLABLE GGEUJ +B05B;B05B;1101 1173 11BE;B05B;1101 1173 11BE; # (끛; 끛; 끛; 끛; 끛; ) HANGUL SYLLABLE GGEUC +B05C;B05C;1101 1173 11BF;B05C;1101 1173 11BF; # (끜; 끜; 끜; 끜; 끜; ) HANGUL SYLLABLE GGEUK +B05D;B05D;1101 1173 11C0;B05D;1101 1173 11C0; # (끝; 끝; 끝; 끝; 끝; ) HANGUL SYLLABLE GGEUT +B05E;B05E;1101 1173 11C1;B05E;1101 1173 11C1; # (끞; 끞; 끞; 끞; 끞; ) HANGUL SYLLABLE GGEUP +B05F;B05F;1101 1173 11C2;B05F;1101 1173 11C2; # (끟; 끟; 끟; 끟; 끟; ) HANGUL SYLLABLE GGEUH +B060;B060;1101 1174;B060;1101 1174; # (끠; 끠; 끠; 끠; 끠; ) HANGUL SYLLABLE GGYI +B061;B061;1101 1174 11A8;B061;1101 1174 11A8; # (끡; 끡; 끡; 끡; 끡; ) HANGUL SYLLABLE GGYIG +B062;B062;1101 1174 11A9;B062;1101 1174 11A9; # (끢; 끢; 끢; 끢; 끢; ) HANGUL SYLLABLE GGYIGG +B063;B063;1101 1174 11AA;B063;1101 1174 11AA; # (끣; 끣; 끣; 끣; 끣; ) HANGUL SYLLABLE GGYIGS +B064;B064;1101 1174 11AB;B064;1101 1174 11AB; # (끤; 끤; 끤; 끤; 끤; ) HANGUL SYLLABLE GGYIN +B065;B065;1101 1174 11AC;B065;1101 1174 11AC; # (끥; 끥; 끥; 끥; 끥; ) HANGUL SYLLABLE GGYINJ +B066;B066;1101 1174 11AD;B066;1101 1174 11AD; # (끦; 끦; 끦; 끦; 끦; ) HANGUL SYLLABLE GGYINH +B067;B067;1101 1174 11AE;B067;1101 1174 11AE; # (끧; 끧; 끧; 끧; 끧; ) HANGUL SYLLABLE GGYID +B068;B068;1101 1174 11AF;B068;1101 1174 11AF; # (끨; 끨; 끨; 끨; 끨; ) HANGUL SYLLABLE GGYIL +B069;B069;1101 1174 11B0;B069;1101 1174 11B0; # (끩; 끩; 끩; 끩; 끩; ) HANGUL SYLLABLE GGYILG +B06A;B06A;1101 1174 11B1;B06A;1101 1174 11B1; # (끪; 끪; 끪; 끪; 끪; ) HANGUL SYLLABLE GGYILM +B06B;B06B;1101 1174 11B2;B06B;1101 1174 11B2; # (끫; 끫; 끫; 끫; 끫; ) HANGUL SYLLABLE GGYILB +B06C;B06C;1101 1174 11B3;B06C;1101 1174 11B3; # (끬; 끬; 끬; 끬; 끬; ) HANGUL SYLLABLE GGYILS +B06D;B06D;1101 1174 11B4;B06D;1101 1174 11B4; # (끭; 끭; 끭; 끭; 끭; ) HANGUL SYLLABLE GGYILT +B06E;B06E;1101 1174 11B5;B06E;1101 1174 11B5; # (끮; 끮; 끮; 끮; 끮; ) HANGUL SYLLABLE GGYILP +B06F;B06F;1101 1174 11B6;B06F;1101 1174 11B6; # (끯; 끯; 끯; 끯; 끯; ) HANGUL SYLLABLE GGYILH +B070;B070;1101 1174 11B7;B070;1101 1174 11B7; # (끰; 끰; 끰; 끰; 끰; ) HANGUL SYLLABLE GGYIM +B071;B071;1101 1174 11B8;B071;1101 1174 11B8; # (끱; 끱; 끱; 끱; 끱; ) HANGUL SYLLABLE GGYIB +B072;B072;1101 1174 11B9;B072;1101 1174 11B9; # (끲; 끲; 끲; 끲; 끲; ) HANGUL SYLLABLE GGYIBS +B073;B073;1101 1174 11BA;B073;1101 1174 11BA; # (끳; 끳; 끳; 끳; 끳; ) HANGUL SYLLABLE GGYIS +B074;B074;1101 1174 11BB;B074;1101 1174 11BB; # (끴; 끴; 끴; 끴; 끴; ) HANGUL SYLLABLE GGYISS +B075;B075;1101 1174 11BC;B075;1101 1174 11BC; # (끵; 끵; 끵; 끵; 끵; ) HANGUL SYLLABLE GGYING +B076;B076;1101 1174 11BD;B076;1101 1174 11BD; # (끶; 끶; 끶; 끶; 끶; ) HANGUL SYLLABLE GGYIJ +B077;B077;1101 1174 11BE;B077;1101 1174 11BE; # (끷; 끷; 끷; 끷; 끷; ) HANGUL SYLLABLE GGYIC +B078;B078;1101 1174 11BF;B078;1101 1174 11BF; # (끸; 끸; 끸; 끸; 끸; ) HANGUL SYLLABLE GGYIK +B079;B079;1101 1174 11C0;B079;1101 1174 11C0; # (끹; 끹; 끹; 끹; 끹; ) HANGUL SYLLABLE GGYIT +B07A;B07A;1101 1174 11C1;B07A;1101 1174 11C1; # (끺; 끺; 끺; 끺; 끺; ) HANGUL SYLLABLE GGYIP +B07B;B07B;1101 1174 11C2;B07B;1101 1174 11C2; # (끻; 끻; 끻; 끻; 끻; ) HANGUL SYLLABLE GGYIH +B07C;B07C;1101 1175;B07C;1101 1175; # (끼; 끼; 끼; 끼; 끼; ) HANGUL SYLLABLE GGI +B07D;B07D;1101 1175 11A8;B07D;1101 1175 11A8; # (끽; 끽; 끽; 끽; 끽; ) HANGUL SYLLABLE GGIG +B07E;B07E;1101 1175 11A9;B07E;1101 1175 11A9; # (끾; 끾; 끾; 끾; 끾; ) HANGUL SYLLABLE GGIGG +B07F;B07F;1101 1175 11AA;B07F;1101 1175 11AA; # (끿; 끿; 끿; 끿; 끿; ) HANGUL SYLLABLE GGIGS +B080;B080;1101 1175 11AB;B080;1101 1175 11AB; # (낀; 낀; 낀; 낀; 낀; ) HANGUL SYLLABLE GGIN +B081;B081;1101 1175 11AC;B081;1101 1175 11AC; # (낁; 낁; 낁; 낁; 낁; ) HANGUL SYLLABLE GGINJ +B082;B082;1101 1175 11AD;B082;1101 1175 11AD; # (낂; 낂; 낂; 낂; 낂; ) HANGUL SYLLABLE GGINH +B083;B083;1101 1175 11AE;B083;1101 1175 11AE; # (낃; 낃; 낃; 낃; 낃; ) HANGUL SYLLABLE GGID +B084;B084;1101 1175 11AF;B084;1101 1175 11AF; # (낄; 낄; 낄; 낄; 낄; ) HANGUL SYLLABLE GGIL +B085;B085;1101 1175 11B0;B085;1101 1175 11B0; # (낅; 낅; 낅; 낅; 낅; ) HANGUL SYLLABLE GGILG +B086;B086;1101 1175 11B1;B086;1101 1175 11B1; # (낆; 낆; 낆; 낆; 낆; ) HANGUL SYLLABLE GGILM +B087;B087;1101 1175 11B2;B087;1101 1175 11B2; # (낇; 낇; 낇; 낇; 낇; ) HANGUL SYLLABLE GGILB +B088;B088;1101 1175 11B3;B088;1101 1175 11B3; # (낈; 낈; 낈; 낈; 낈; ) HANGUL SYLLABLE GGILS +B089;B089;1101 1175 11B4;B089;1101 1175 11B4; # (낉; 낉; 낉; 낉; 낉; ) HANGUL SYLLABLE GGILT +B08A;B08A;1101 1175 11B5;B08A;1101 1175 11B5; # (낊; 낊; 낊; 낊; 낊; ) HANGUL SYLLABLE GGILP +B08B;B08B;1101 1175 11B6;B08B;1101 1175 11B6; # (낋; 낋; 낋; 낋; 낋; ) HANGUL SYLLABLE GGILH +B08C;B08C;1101 1175 11B7;B08C;1101 1175 11B7; # (낌; 낌; 낌; 낌; 낌; ) HANGUL SYLLABLE GGIM +B08D;B08D;1101 1175 11B8;B08D;1101 1175 11B8; # (낍; 낍; 낍; 낍; 낍; ) HANGUL SYLLABLE GGIB +B08E;B08E;1101 1175 11B9;B08E;1101 1175 11B9; # (낎; 낎; 낎; 낎; 낎; ) HANGUL SYLLABLE GGIBS +B08F;B08F;1101 1175 11BA;B08F;1101 1175 11BA; # (낏; 낏; 낏; 낏; 낏; ) HANGUL SYLLABLE GGIS +B090;B090;1101 1175 11BB;B090;1101 1175 11BB; # (낐; 낐; 낐; 낐; 낐; ) HANGUL SYLLABLE GGISS +B091;B091;1101 1175 11BC;B091;1101 1175 11BC; # (낑; 낑; 낑; 낑; 낑; ) HANGUL SYLLABLE GGING +B092;B092;1101 1175 11BD;B092;1101 1175 11BD; # (낒; 낒; 낒; 낒; 낒; ) HANGUL SYLLABLE GGIJ +B093;B093;1101 1175 11BE;B093;1101 1175 11BE; # (낓; 낓; 낓; 낓; 낓; ) HANGUL SYLLABLE GGIC +B094;B094;1101 1175 11BF;B094;1101 1175 11BF; # (낔; 낔; 낔; 낔; 낔; ) HANGUL SYLLABLE GGIK +B095;B095;1101 1175 11C0;B095;1101 1175 11C0; # (낕; 낕; 낕; 낕; 낕; ) HANGUL SYLLABLE GGIT +B096;B096;1101 1175 11C1;B096;1101 1175 11C1; # (낖; 낖; 낖; 낖; 낖; ) HANGUL SYLLABLE GGIP +B097;B097;1101 1175 11C2;B097;1101 1175 11C2; # (낗; 낗; 낗; 낗; 낗; ) HANGUL SYLLABLE GGIH +B098;B098;1102 1161;B098;1102 1161; # (나; 나; 나; 나; 나; ) HANGUL SYLLABLE NA +B099;B099;1102 1161 11A8;B099;1102 1161 11A8; # (낙; 낙; 낙; 낙; 낙; ) HANGUL SYLLABLE NAG +B09A;B09A;1102 1161 11A9;B09A;1102 1161 11A9; # (낚; 낚; 낚; 낚; 낚; ) HANGUL SYLLABLE NAGG +B09B;B09B;1102 1161 11AA;B09B;1102 1161 11AA; # (낛; 낛; 낛; 낛; 낛; ) HANGUL SYLLABLE NAGS +B09C;B09C;1102 1161 11AB;B09C;1102 1161 11AB; # (난; 난; 난; 난; 난; ) HANGUL SYLLABLE NAN +B09D;B09D;1102 1161 11AC;B09D;1102 1161 11AC; # (낝; 낝; 낝; 낝; 낝; ) HANGUL SYLLABLE NANJ +B09E;B09E;1102 1161 11AD;B09E;1102 1161 11AD; # (낞; 낞; 낞; 낞; 낞; ) HANGUL SYLLABLE NANH +B09F;B09F;1102 1161 11AE;B09F;1102 1161 11AE; # (낟; 낟; 낟; 낟; 낟; ) HANGUL SYLLABLE NAD +B0A0;B0A0;1102 1161 11AF;B0A0;1102 1161 11AF; # (날; 날; 날; 날; 날; ) HANGUL SYLLABLE NAL +B0A1;B0A1;1102 1161 11B0;B0A1;1102 1161 11B0; # (낡; 낡; 낡; 낡; 낡; ) HANGUL SYLLABLE NALG +B0A2;B0A2;1102 1161 11B1;B0A2;1102 1161 11B1; # (낢; 낢; 낢; 낢; 낢; ) HANGUL SYLLABLE NALM +B0A3;B0A3;1102 1161 11B2;B0A3;1102 1161 11B2; # (낣; 낣; 낣; 낣; 낣; ) HANGUL SYLLABLE NALB +B0A4;B0A4;1102 1161 11B3;B0A4;1102 1161 11B3; # (낤; 낤; 낤; 낤; 낤; ) HANGUL SYLLABLE NALS +B0A5;B0A5;1102 1161 11B4;B0A5;1102 1161 11B4; # (낥; 낥; 낥; 낥; 낥; ) HANGUL SYLLABLE NALT +B0A6;B0A6;1102 1161 11B5;B0A6;1102 1161 11B5; # (낦; 낦; 낦; 낦; 낦; ) HANGUL SYLLABLE NALP +B0A7;B0A7;1102 1161 11B6;B0A7;1102 1161 11B6; # (낧; 낧; 낧; 낧; 낧; ) HANGUL SYLLABLE NALH +B0A8;B0A8;1102 1161 11B7;B0A8;1102 1161 11B7; # (남; 남; 남; 남; 남; ) HANGUL SYLLABLE NAM +B0A9;B0A9;1102 1161 11B8;B0A9;1102 1161 11B8; # (납; 납; 납; 납; 납; ) HANGUL SYLLABLE NAB +B0AA;B0AA;1102 1161 11B9;B0AA;1102 1161 11B9; # (낪; 낪; 낪; 낪; 낪; ) HANGUL SYLLABLE NABS +B0AB;B0AB;1102 1161 11BA;B0AB;1102 1161 11BA; # (낫; 낫; 낫; 낫; 낫; ) HANGUL SYLLABLE NAS +B0AC;B0AC;1102 1161 11BB;B0AC;1102 1161 11BB; # (났; 났; 났; 났; 났; ) HANGUL SYLLABLE NASS +B0AD;B0AD;1102 1161 11BC;B0AD;1102 1161 11BC; # (낭; 낭; 낭; 낭; 낭; ) HANGUL SYLLABLE NANG +B0AE;B0AE;1102 1161 11BD;B0AE;1102 1161 11BD; # (낮; 낮; 낮; 낮; 낮; ) HANGUL SYLLABLE NAJ +B0AF;B0AF;1102 1161 11BE;B0AF;1102 1161 11BE; # (낯; 낯; 낯; 낯; 낯; ) HANGUL SYLLABLE NAC +B0B0;B0B0;1102 1161 11BF;B0B0;1102 1161 11BF; # (낰; 낰; 낰; 낰; 낰; ) HANGUL SYLLABLE NAK +B0B1;B0B1;1102 1161 11C0;B0B1;1102 1161 11C0; # (낱; 낱; 낱; 낱; 낱; ) HANGUL SYLLABLE NAT +B0B2;B0B2;1102 1161 11C1;B0B2;1102 1161 11C1; # (낲; 낲; 낲; 낲; 낲; ) HANGUL SYLLABLE NAP +B0B3;B0B3;1102 1161 11C2;B0B3;1102 1161 11C2; # (낳; 낳; 낳; 낳; 낳; ) HANGUL SYLLABLE NAH +B0B4;B0B4;1102 1162;B0B4;1102 1162; # (내; 내; 내; 내; 내; ) HANGUL SYLLABLE NAE +B0B5;B0B5;1102 1162 11A8;B0B5;1102 1162 11A8; # (낵; 낵; 낵; 낵; 낵; ) HANGUL SYLLABLE NAEG +B0B6;B0B6;1102 1162 11A9;B0B6;1102 1162 11A9; # (낶; 낶; 낶; 낶; 낶; ) HANGUL SYLLABLE NAEGG +B0B7;B0B7;1102 1162 11AA;B0B7;1102 1162 11AA; # (낷; 낷; 낷; 낷; 낷; ) HANGUL SYLLABLE NAEGS +B0B8;B0B8;1102 1162 11AB;B0B8;1102 1162 11AB; # (낸; 낸; 낸; 낸; 낸; ) HANGUL SYLLABLE NAEN +B0B9;B0B9;1102 1162 11AC;B0B9;1102 1162 11AC; # (낹; 낹; 낹; 낹; 낹; ) HANGUL SYLLABLE NAENJ +B0BA;B0BA;1102 1162 11AD;B0BA;1102 1162 11AD; # (낺; 낺; 낺; 낺; 낺; ) HANGUL SYLLABLE NAENH +B0BB;B0BB;1102 1162 11AE;B0BB;1102 1162 11AE; # (낻; 낻; 낻; 낻; 낻; ) HANGUL SYLLABLE NAED +B0BC;B0BC;1102 1162 11AF;B0BC;1102 1162 11AF; # (낼; 낼; 낼; 낼; 낼; ) HANGUL SYLLABLE NAEL +B0BD;B0BD;1102 1162 11B0;B0BD;1102 1162 11B0; # (낽; 낽; 낽; 낽; 낽; ) HANGUL SYLLABLE NAELG +B0BE;B0BE;1102 1162 11B1;B0BE;1102 1162 11B1; # (낾; 낾; 낾; 낾; 낾; ) HANGUL SYLLABLE NAELM +B0BF;B0BF;1102 1162 11B2;B0BF;1102 1162 11B2; # (낿; 낿; 낿; 낿; 낿; ) HANGUL SYLLABLE NAELB +B0C0;B0C0;1102 1162 11B3;B0C0;1102 1162 11B3; # (냀; 냀; 냀; 냀; 냀; ) HANGUL SYLLABLE NAELS +B0C1;B0C1;1102 1162 11B4;B0C1;1102 1162 11B4; # (냁; 냁; 냁; 냁; 냁; ) HANGUL SYLLABLE NAELT +B0C2;B0C2;1102 1162 11B5;B0C2;1102 1162 11B5; # (냂; 냂; 냂; 냂; 냂; ) HANGUL SYLLABLE NAELP +B0C3;B0C3;1102 1162 11B6;B0C3;1102 1162 11B6; # (냃; 냃; 냃; 냃; 냃; ) HANGUL SYLLABLE NAELH +B0C4;B0C4;1102 1162 11B7;B0C4;1102 1162 11B7; # (냄; 냄; 냄; 냄; 냄; ) HANGUL SYLLABLE NAEM +B0C5;B0C5;1102 1162 11B8;B0C5;1102 1162 11B8; # (냅; 냅; 냅; 냅; 냅; ) HANGUL SYLLABLE NAEB +B0C6;B0C6;1102 1162 11B9;B0C6;1102 1162 11B9; # (냆; 냆; 냆; 냆; 냆; ) HANGUL SYLLABLE NAEBS +B0C7;B0C7;1102 1162 11BA;B0C7;1102 1162 11BA; # (냇; 냇; 냇; 냇; 냇; ) HANGUL SYLLABLE NAES +B0C8;B0C8;1102 1162 11BB;B0C8;1102 1162 11BB; # (냈; 냈; 냈; 냈; 냈; ) HANGUL SYLLABLE NAESS +B0C9;B0C9;1102 1162 11BC;B0C9;1102 1162 11BC; # (냉; 냉; 냉; 냉; 냉; ) HANGUL SYLLABLE NAENG +B0CA;B0CA;1102 1162 11BD;B0CA;1102 1162 11BD; # (냊; 냊; 냊; 냊; 냊; ) HANGUL SYLLABLE NAEJ +B0CB;B0CB;1102 1162 11BE;B0CB;1102 1162 11BE; # (냋; 냋; 냋; 냋; 냋; ) HANGUL SYLLABLE NAEC +B0CC;B0CC;1102 1162 11BF;B0CC;1102 1162 11BF; # (냌; 냌; 냌; 냌; 냌; ) HANGUL SYLLABLE NAEK +B0CD;B0CD;1102 1162 11C0;B0CD;1102 1162 11C0; # (냍; 냍; 냍; 냍; 냍; ) HANGUL SYLLABLE NAET +B0CE;B0CE;1102 1162 11C1;B0CE;1102 1162 11C1; # (냎; 냎; 냎; 냎; 냎; ) HANGUL SYLLABLE NAEP +B0CF;B0CF;1102 1162 11C2;B0CF;1102 1162 11C2; # (냏; 냏; 냏; 냏; 냏; ) HANGUL SYLLABLE NAEH +B0D0;B0D0;1102 1163;B0D0;1102 1163; # (냐; 냐; 냐; 냐; 냐; ) HANGUL SYLLABLE NYA +B0D1;B0D1;1102 1163 11A8;B0D1;1102 1163 11A8; # (냑; 냑; 냑; 냑; 냑; ) HANGUL SYLLABLE NYAG +B0D2;B0D2;1102 1163 11A9;B0D2;1102 1163 11A9; # (냒; 냒; 냒; 냒; 냒; ) HANGUL SYLLABLE NYAGG +B0D3;B0D3;1102 1163 11AA;B0D3;1102 1163 11AA; # (냓; 냓; 냓; 냓; 냓; ) HANGUL SYLLABLE NYAGS +B0D4;B0D4;1102 1163 11AB;B0D4;1102 1163 11AB; # (냔; 냔; 냔; 냔; 냔; ) HANGUL SYLLABLE NYAN +B0D5;B0D5;1102 1163 11AC;B0D5;1102 1163 11AC; # (냕; 냕; 냕; 냕; 냕; ) HANGUL SYLLABLE NYANJ +B0D6;B0D6;1102 1163 11AD;B0D6;1102 1163 11AD; # (냖; 냖; 냖; 냖; 냖; ) HANGUL SYLLABLE NYANH +B0D7;B0D7;1102 1163 11AE;B0D7;1102 1163 11AE; # (냗; 냗; 냗; 냗; 냗; ) HANGUL SYLLABLE NYAD +B0D8;B0D8;1102 1163 11AF;B0D8;1102 1163 11AF; # (냘; 냘; 냘; 냘; 냘; ) HANGUL SYLLABLE NYAL +B0D9;B0D9;1102 1163 11B0;B0D9;1102 1163 11B0; # (냙; 냙; 냙; 냙; 냙; ) HANGUL SYLLABLE NYALG +B0DA;B0DA;1102 1163 11B1;B0DA;1102 1163 11B1; # (냚; 냚; 냚; 냚; 냚; ) HANGUL SYLLABLE NYALM +B0DB;B0DB;1102 1163 11B2;B0DB;1102 1163 11B2; # (냛; 냛; 냛; 냛; 냛; ) HANGUL SYLLABLE NYALB +B0DC;B0DC;1102 1163 11B3;B0DC;1102 1163 11B3; # (냜; 냜; 냜; 냜; 냜; ) HANGUL SYLLABLE NYALS +B0DD;B0DD;1102 1163 11B4;B0DD;1102 1163 11B4; # (냝; 냝; 냝; 냝; 냝; ) HANGUL SYLLABLE NYALT +B0DE;B0DE;1102 1163 11B5;B0DE;1102 1163 11B5; # (냞; 냞; 냞; 냞; 냞; ) HANGUL SYLLABLE NYALP +B0DF;B0DF;1102 1163 11B6;B0DF;1102 1163 11B6; # (냟; 냟; 냟; 냟; 냟; ) HANGUL SYLLABLE NYALH +B0E0;B0E0;1102 1163 11B7;B0E0;1102 1163 11B7; # (냠; 냠; 냠; 냠; 냠; ) HANGUL SYLLABLE NYAM +B0E1;B0E1;1102 1163 11B8;B0E1;1102 1163 11B8; # (냡; 냡; 냡; 냡; 냡; ) HANGUL SYLLABLE NYAB +B0E2;B0E2;1102 1163 11B9;B0E2;1102 1163 11B9; # (냢; 냢; 냢; 냢; 냢; ) HANGUL SYLLABLE NYABS +B0E3;B0E3;1102 1163 11BA;B0E3;1102 1163 11BA; # (냣; 냣; 냣; 냣; 냣; ) HANGUL SYLLABLE NYAS +B0E4;B0E4;1102 1163 11BB;B0E4;1102 1163 11BB; # (냤; 냤; 냤; 냤; 냤; ) HANGUL SYLLABLE NYASS +B0E5;B0E5;1102 1163 11BC;B0E5;1102 1163 11BC; # (냥; 냥; 냥; 냥; 냥; ) HANGUL SYLLABLE NYANG +B0E6;B0E6;1102 1163 11BD;B0E6;1102 1163 11BD; # (냦; 냦; 냦; 냦; 냦; ) HANGUL SYLLABLE NYAJ +B0E7;B0E7;1102 1163 11BE;B0E7;1102 1163 11BE; # (냧; 냧; 냧; 냧; 냧; ) HANGUL SYLLABLE NYAC +B0E8;B0E8;1102 1163 11BF;B0E8;1102 1163 11BF; # (냨; 냨; 냨; 냨; 냨; ) HANGUL SYLLABLE NYAK +B0E9;B0E9;1102 1163 11C0;B0E9;1102 1163 11C0; # (냩; 냩; 냩; 냩; 냩; ) HANGUL SYLLABLE NYAT +B0EA;B0EA;1102 1163 11C1;B0EA;1102 1163 11C1; # (냪; 냪; 냪; 냪; 냪; ) HANGUL SYLLABLE NYAP +B0EB;B0EB;1102 1163 11C2;B0EB;1102 1163 11C2; # (냫; 냫; 냫; 냫; 냫; ) HANGUL SYLLABLE NYAH +B0EC;B0EC;1102 1164;B0EC;1102 1164; # (냬; 냬; 냬; 냬; 냬; ) HANGUL SYLLABLE NYAE +B0ED;B0ED;1102 1164 11A8;B0ED;1102 1164 11A8; # (냭; 냭; 냭; 냭; 냭; ) HANGUL SYLLABLE NYAEG +B0EE;B0EE;1102 1164 11A9;B0EE;1102 1164 11A9; # (냮; 냮; 냮; 냮; 냮; ) HANGUL SYLLABLE NYAEGG +B0EF;B0EF;1102 1164 11AA;B0EF;1102 1164 11AA; # (냯; 냯; 냯; 냯; 냯; ) HANGUL SYLLABLE NYAEGS +B0F0;B0F0;1102 1164 11AB;B0F0;1102 1164 11AB; # (냰; 냰; 냰; 냰; 냰; ) HANGUL SYLLABLE NYAEN +B0F1;B0F1;1102 1164 11AC;B0F1;1102 1164 11AC; # (냱; 냱; 냱; 냱; 냱; ) HANGUL SYLLABLE NYAENJ +B0F2;B0F2;1102 1164 11AD;B0F2;1102 1164 11AD; # (냲; 냲; 냲; 냲; 냲; ) HANGUL SYLLABLE NYAENH +B0F3;B0F3;1102 1164 11AE;B0F3;1102 1164 11AE; # (냳; 냳; 냳; 냳; 냳; ) HANGUL SYLLABLE NYAED +B0F4;B0F4;1102 1164 11AF;B0F4;1102 1164 11AF; # (냴; 냴; 냴; 냴; 냴; ) HANGUL SYLLABLE NYAEL +B0F5;B0F5;1102 1164 11B0;B0F5;1102 1164 11B0; # (냵; 냵; 냵; 냵; 냵; ) HANGUL SYLLABLE NYAELG +B0F6;B0F6;1102 1164 11B1;B0F6;1102 1164 11B1; # (냶; 냶; 냶; 냶; 냶; ) HANGUL SYLLABLE NYAELM +B0F7;B0F7;1102 1164 11B2;B0F7;1102 1164 11B2; # (냷; 냷; 냷; 냷; 냷; ) HANGUL SYLLABLE NYAELB +B0F8;B0F8;1102 1164 11B3;B0F8;1102 1164 11B3; # (냸; 냸; 냸; 냸; 냸; ) HANGUL SYLLABLE NYAELS +B0F9;B0F9;1102 1164 11B4;B0F9;1102 1164 11B4; # (냹; 냹; 냹; 냹; 냹; ) HANGUL SYLLABLE NYAELT +B0FA;B0FA;1102 1164 11B5;B0FA;1102 1164 11B5; # (냺; 냺; 냺; 냺; 냺; ) HANGUL SYLLABLE NYAELP +B0FB;B0FB;1102 1164 11B6;B0FB;1102 1164 11B6; # (냻; 냻; 냻; 냻; 냻; ) HANGUL SYLLABLE NYAELH +B0FC;B0FC;1102 1164 11B7;B0FC;1102 1164 11B7; # (냼; 냼; 냼; 냼; 냼; ) HANGUL SYLLABLE NYAEM +B0FD;B0FD;1102 1164 11B8;B0FD;1102 1164 11B8; # (냽; 냽; 냽; 냽; 냽; ) HANGUL SYLLABLE NYAEB +B0FE;B0FE;1102 1164 11B9;B0FE;1102 1164 11B9; # (냾; 냾; 냾; 냾; 냾; ) HANGUL SYLLABLE NYAEBS +B0FF;B0FF;1102 1164 11BA;B0FF;1102 1164 11BA; # (냿; 냿; 냿; 냿; 냿; ) HANGUL SYLLABLE NYAES +B100;B100;1102 1164 11BB;B100;1102 1164 11BB; # (넀; 넀; 넀; 넀; 넀; ) HANGUL SYLLABLE NYAESS +B101;B101;1102 1164 11BC;B101;1102 1164 11BC; # (넁; 넁; 넁; 넁; 넁; ) HANGUL SYLLABLE NYAENG +B102;B102;1102 1164 11BD;B102;1102 1164 11BD; # (넂; 넂; 넂; 넂; 넂; ) HANGUL SYLLABLE NYAEJ +B103;B103;1102 1164 11BE;B103;1102 1164 11BE; # (넃; 넃; 넃; 넃; 넃; ) HANGUL SYLLABLE NYAEC +B104;B104;1102 1164 11BF;B104;1102 1164 11BF; # (넄; 넄; 넄; 넄; 넄; ) HANGUL SYLLABLE NYAEK +B105;B105;1102 1164 11C0;B105;1102 1164 11C0; # (넅; 넅; 넅; 넅; 넅; ) HANGUL SYLLABLE NYAET +B106;B106;1102 1164 11C1;B106;1102 1164 11C1; # (넆; 넆; 넆; 넆; 넆; ) HANGUL SYLLABLE NYAEP +B107;B107;1102 1164 11C2;B107;1102 1164 11C2; # (넇; 넇; 넇; 넇; 넇; ) HANGUL SYLLABLE NYAEH +B108;B108;1102 1165;B108;1102 1165; # (너; 너; 너; 너; 너; ) HANGUL SYLLABLE NEO +B109;B109;1102 1165 11A8;B109;1102 1165 11A8; # (넉; 넉; 넉; 넉; 넉; ) HANGUL SYLLABLE NEOG +B10A;B10A;1102 1165 11A9;B10A;1102 1165 11A9; # (넊; 넊; 넊; 넊; 넊; ) HANGUL SYLLABLE NEOGG +B10B;B10B;1102 1165 11AA;B10B;1102 1165 11AA; # (넋; 넋; 넋; 넋; 넋; ) HANGUL SYLLABLE NEOGS +B10C;B10C;1102 1165 11AB;B10C;1102 1165 11AB; # (넌; 넌; 넌; 넌; 넌; ) HANGUL SYLLABLE NEON +B10D;B10D;1102 1165 11AC;B10D;1102 1165 11AC; # (넍; 넍; 넍; 넍; 넍; ) HANGUL SYLLABLE NEONJ +B10E;B10E;1102 1165 11AD;B10E;1102 1165 11AD; # (넎; 넎; 넎; 넎; 넎; ) HANGUL SYLLABLE NEONH +B10F;B10F;1102 1165 11AE;B10F;1102 1165 11AE; # (넏; 넏; 넏; 넏; 넏; ) HANGUL SYLLABLE NEOD +B110;B110;1102 1165 11AF;B110;1102 1165 11AF; # (널; 널; 널; 널; 널; ) HANGUL SYLLABLE NEOL +B111;B111;1102 1165 11B0;B111;1102 1165 11B0; # (넑; 넑; 넑; 넑; 넑; ) HANGUL SYLLABLE NEOLG +B112;B112;1102 1165 11B1;B112;1102 1165 11B1; # (넒; 넒; 넒; 넒; 넒; ) HANGUL SYLLABLE NEOLM +B113;B113;1102 1165 11B2;B113;1102 1165 11B2; # (넓; 넓; 넓; 넓; 넓; ) HANGUL SYLLABLE NEOLB +B114;B114;1102 1165 11B3;B114;1102 1165 11B3; # (넔; 넔; 넔; 넔; 넔; ) HANGUL SYLLABLE NEOLS +B115;B115;1102 1165 11B4;B115;1102 1165 11B4; # (넕; 넕; 넕; 넕; 넕; ) HANGUL SYLLABLE NEOLT +B116;B116;1102 1165 11B5;B116;1102 1165 11B5; # (넖; 넖; 넖; 넖; 넖; ) HANGUL SYLLABLE NEOLP +B117;B117;1102 1165 11B6;B117;1102 1165 11B6; # (넗; 넗; 넗; 넗; 넗; ) HANGUL SYLLABLE NEOLH +B118;B118;1102 1165 11B7;B118;1102 1165 11B7; # (넘; 넘; 넘; 넘; 넘; ) HANGUL SYLLABLE NEOM +B119;B119;1102 1165 11B8;B119;1102 1165 11B8; # (넙; 넙; 넙; 넙; 넙; ) HANGUL SYLLABLE NEOB +B11A;B11A;1102 1165 11B9;B11A;1102 1165 11B9; # (넚; 넚; 넚; 넚; 넚; ) HANGUL SYLLABLE NEOBS +B11B;B11B;1102 1165 11BA;B11B;1102 1165 11BA; # (넛; 넛; 넛; 넛; 넛; ) HANGUL SYLLABLE NEOS +B11C;B11C;1102 1165 11BB;B11C;1102 1165 11BB; # (넜; 넜; 넜; 넜; 넜; ) HANGUL SYLLABLE NEOSS +B11D;B11D;1102 1165 11BC;B11D;1102 1165 11BC; # (넝; 넝; 넝; 넝; 넝; ) HANGUL SYLLABLE NEONG +B11E;B11E;1102 1165 11BD;B11E;1102 1165 11BD; # (넞; 넞; 넞; 넞; 넞; ) HANGUL SYLLABLE NEOJ +B11F;B11F;1102 1165 11BE;B11F;1102 1165 11BE; # (넟; 넟; 넟; 넟; 넟; ) HANGUL SYLLABLE NEOC +B120;B120;1102 1165 11BF;B120;1102 1165 11BF; # (넠; 넠; 넠; 넠; 넠; ) HANGUL SYLLABLE NEOK +B121;B121;1102 1165 11C0;B121;1102 1165 11C0; # (넡; 넡; 넡; 넡; 넡; ) HANGUL SYLLABLE NEOT +B122;B122;1102 1165 11C1;B122;1102 1165 11C1; # (넢; 넢; 넢; 넢; 넢; ) HANGUL SYLLABLE NEOP +B123;B123;1102 1165 11C2;B123;1102 1165 11C2; # (넣; 넣; 넣; 넣; 넣; ) HANGUL SYLLABLE NEOH +B124;B124;1102 1166;B124;1102 1166; # (네; 네; 네; 네; 네; ) HANGUL SYLLABLE NE +B125;B125;1102 1166 11A8;B125;1102 1166 11A8; # (넥; 넥; 넥; 넥; 넥; ) HANGUL SYLLABLE NEG +B126;B126;1102 1166 11A9;B126;1102 1166 11A9; # (넦; 넦; 넦; 넦; 넦; ) HANGUL SYLLABLE NEGG +B127;B127;1102 1166 11AA;B127;1102 1166 11AA; # (넧; 넧; 넧; 넧; 넧; ) HANGUL SYLLABLE NEGS +B128;B128;1102 1166 11AB;B128;1102 1166 11AB; # (넨; 넨; 넨; 넨; 넨; ) HANGUL SYLLABLE NEN +B129;B129;1102 1166 11AC;B129;1102 1166 11AC; # (넩; 넩; 넩; 넩; 넩; ) HANGUL SYLLABLE NENJ +B12A;B12A;1102 1166 11AD;B12A;1102 1166 11AD; # (넪; 넪; 넪; 넪; 넪; ) HANGUL SYLLABLE NENH +B12B;B12B;1102 1166 11AE;B12B;1102 1166 11AE; # (넫; 넫; 넫; 넫; 넫; ) HANGUL SYLLABLE NED +B12C;B12C;1102 1166 11AF;B12C;1102 1166 11AF; # (넬; 넬; 넬; 넬; 넬; ) HANGUL SYLLABLE NEL +B12D;B12D;1102 1166 11B0;B12D;1102 1166 11B0; # (넭; 넭; 넭; 넭; 넭; ) HANGUL SYLLABLE NELG +B12E;B12E;1102 1166 11B1;B12E;1102 1166 11B1; # (넮; 넮; 넮; 넮; 넮; ) HANGUL SYLLABLE NELM +B12F;B12F;1102 1166 11B2;B12F;1102 1166 11B2; # (넯; 넯; 넯; 넯; 넯; ) HANGUL SYLLABLE NELB +B130;B130;1102 1166 11B3;B130;1102 1166 11B3; # (넰; 넰; 넰; 넰; 넰; ) HANGUL SYLLABLE NELS +B131;B131;1102 1166 11B4;B131;1102 1166 11B4; # (넱; 넱; 넱; 넱; 넱; ) HANGUL SYLLABLE NELT +B132;B132;1102 1166 11B5;B132;1102 1166 11B5; # (넲; 넲; 넲; 넲; 넲; ) HANGUL SYLLABLE NELP +B133;B133;1102 1166 11B6;B133;1102 1166 11B6; # (넳; 넳; 넳; 넳; 넳; ) HANGUL SYLLABLE NELH +B134;B134;1102 1166 11B7;B134;1102 1166 11B7; # (넴; 넴; 넴; 넴; 넴; ) HANGUL SYLLABLE NEM +B135;B135;1102 1166 11B8;B135;1102 1166 11B8; # (넵; 넵; 넵; 넵; 넵; ) HANGUL SYLLABLE NEB +B136;B136;1102 1166 11B9;B136;1102 1166 11B9; # (넶; 넶; 넶; 넶; 넶; ) HANGUL SYLLABLE NEBS +B137;B137;1102 1166 11BA;B137;1102 1166 11BA; # (넷; 넷; 넷; 넷; 넷; ) HANGUL SYLLABLE NES +B138;B138;1102 1166 11BB;B138;1102 1166 11BB; # (넸; 넸; 넸; 넸; 넸; ) HANGUL SYLLABLE NESS +B139;B139;1102 1166 11BC;B139;1102 1166 11BC; # (넹; 넹; 넹; 넹; 넹; ) HANGUL SYLLABLE NENG +B13A;B13A;1102 1166 11BD;B13A;1102 1166 11BD; # (넺; 넺; 넺; 넺; 넺; ) HANGUL SYLLABLE NEJ +B13B;B13B;1102 1166 11BE;B13B;1102 1166 11BE; # (넻; 넻; 넻; 넻; 넻; ) HANGUL SYLLABLE NEC +B13C;B13C;1102 1166 11BF;B13C;1102 1166 11BF; # (넼; 넼; 넼; 넼; 넼; ) HANGUL SYLLABLE NEK +B13D;B13D;1102 1166 11C0;B13D;1102 1166 11C0; # (넽; 넽; 넽; 넽; 넽; ) HANGUL SYLLABLE NET +B13E;B13E;1102 1166 11C1;B13E;1102 1166 11C1; # (넾; 넾; 넾; 넾; 넾; ) HANGUL SYLLABLE NEP +B13F;B13F;1102 1166 11C2;B13F;1102 1166 11C2; # (넿; 넿; 넿; 넿; 넿; ) HANGUL SYLLABLE NEH +B140;B140;1102 1167;B140;1102 1167; # (녀; 녀; 녀; 녀; 녀; ) HANGUL SYLLABLE NYEO +B141;B141;1102 1167 11A8;B141;1102 1167 11A8; # (녁; 녁; 녁; 녁; 녁; ) HANGUL SYLLABLE NYEOG +B142;B142;1102 1167 11A9;B142;1102 1167 11A9; # (녂; 녂; 녂; 녂; 녂; ) HANGUL SYLLABLE NYEOGG +B143;B143;1102 1167 11AA;B143;1102 1167 11AA; # (녃; 녃; 녃; 녃; 녃; ) HANGUL SYLLABLE NYEOGS +B144;B144;1102 1167 11AB;B144;1102 1167 11AB; # (년; 년; 년; 년; 년; ) HANGUL SYLLABLE NYEON +B145;B145;1102 1167 11AC;B145;1102 1167 11AC; # (녅; 녅; 녅; 녅; 녅; ) HANGUL SYLLABLE NYEONJ +B146;B146;1102 1167 11AD;B146;1102 1167 11AD; # (녆; 녆; 녆; 녆; 녆; ) HANGUL SYLLABLE NYEONH +B147;B147;1102 1167 11AE;B147;1102 1167 11AE; # (녇; 녇; 녇; 녇; 녇; ) HANGUL SYLLABLE NYEOD +B148;B148;1102 1167 11AF;B148;1102 1167 11AF; # (녈; 녈; 녈; 녈; 녈; ) HANGUL SYLLABLE NYEOL +B149;B149;1102 1167 11B0;B149;1102 1167 11B0; # (녉; 녉; 녉; 녉; 녉; ) HANGUL SYLLABLE NYEOLG +B14A;B14A;1102 1167 11B1;B14A;1102 1167 11B1; # (녊; 녊; 녊; 녊; 녊; ) HANGUL SYLLABLE NYEOLM +B14B;B14B;1102 1167 11B2;B14B;1102 1167 11B2; # (녋; 녋; 녋; 녋; 녋; ) HANGUL SYLLABLE NYEOLB +B14C;B14C;1102 1167 11B3;B14C;1102 1167 11B3; # (녌; 녌; 녌; 녌; 녌; ) HANGUL SYLLABLE NYEOLS +B14D;B14D;1102 1167 11B4;B14D;1102 1167 11B4; # (녍; 녍; 녍; 녍; 녍; ) HANGUL SYLLABLE NYEOLT +B14E;B14E;1102 1167 11B5;B14E;1102 1167 11B5; # (녎; 녎; 녎; 녎; 녎; ) HANGUL SYLLABLE NYEOLP +B14F;B14F;1102 1167 11B6;B14F;1102 1167 11B6; # (녏; 녏; 녏; 녏; 녏; ) HANGUL SYLLABLE NYEOLH +B150;B150;1102 1167 11B7;B150;1102 1167 11B7; # (념; 념; 념; 념; 념; ) HANGUL SYLLABLE NYEOM +B151;B151;1102 1167 11B8;B151;1102 1167 11B8; # (녑; 녑; 녑; 녑; 녑; ) HANGUL SYLLABLE NYEOB +B152;B152;1102 1167 11B9;B152;1102 1167 11B9; # (녒; 녒; 녒; 녒; 녒; ) HANGUL SYLLABLE NYEOBS +B153;B153;1102 1167 11BA;B153;1102 1167 11BA; # (녓; 녓; 녓; 녓; 녓; ) HANGUL SYLLABLE NYEOS +B154;B154;1102 1167 11BB;B154;1102 1167 11BB; # (녔; 녔; 녔; 녔; 녔; ) HANGUL SYLLABLE NYEOSS +B155;B155;1102 1167 11BC;B155;1102 1167 11BC; # (녕; 녕; 녕; 녕; 녕; ) HANGUL SYLLABLE NYEONG +B156;B156;1102 1167 11BD;B156;1102 1167 11BD; # (녖; 녖; 녖; 녖; 녖; ) HANGUL SYLLABLE NYEOJ +B157;B157;1102 1167 11BE;B157;1102 1167 11BE; # (녗; 녗; 녗; 녗; 녗; ) HANGUL SYLLABLE NYEOC +B158;B158;1102 1167 11BF;B158;1102 1167 11BF; # (녘; 녘; 녘; 녘; 녘; ) HANGUL SYLLABLE NYEOK +B159;B159;1102 1167 11C0;B159;1102 1167 11C0; # (녙; 녙; 녙; 녙; 녙; ) HANGUL SYLLABLE NYEOT +B15A;B15A;1102 1167 11C1;B15A;1102 1167 11C1; # (녚; 녚; 녚; 녚; 녚; ) HANGUL SYLLABLE NYEOP +B15B;B15B;1102 1167 11C2;B15B;1102 1167 11C2; # (녛; 녛; 녛; 녛; 녛; ) HANGUL SYLLABLE NYEOH +B15C;B15C;1102 1168;B15C;1102 1168; # (녜; 녜; 녜; 녜; 녜; ) HANGUL SYLLABLE NYE +B15D;B15D;1102 1168 11A8;B15D;1102 1168 11A8; # (녝; 녝; 녝; 녝; 녝; ) HANGUL SYLLABLE NYEG +B15E;B15E;1102 1168 11A9;B15E;1102 1168 11A9; # (녞; 녞; 녞; 녞; 녞; ) HANGUL SYLLABLE NYEGG +B15F;B15F;1102 1168 11AA;B15F;1102 1168 11AA; # (녟; 녟; 녟; 녟; 녟; ) HANGUL SYLLABLE NYEGS +B160;B160;1102 1168 11AB;B160;1102 1168 11AB; # (녠; 녠; 녠; 녠; 녠; ) HANGUL SYLLABLE NYEN +B161;B161;1102 1168 11AC;B161;1102 1168 11AC; # (녡; 녡; 녡; 녡; 녡; ) HANGUL SYLLABLE NYENJ +B162;B162;1102 1168 11AD;B162;1102 1168 11AD; # (녢; 녢; 녢; 녢; 녢; ) HANGUL SYLLABLE NYENH +B163;B163;1102 1168 11AE;B163;1102 1168 11AE; # (녣; 녣; 녣; 녣; 녣; ) HANGUL SYLLABLE NYED +B164;B164;1102 1168 11AF;B164;1102 1168 11AF; # (녤; 녤; 녤; 녤; 녤; ) HANGUL SYLLABLE NYEL +B165;B165;1102 1168 11B0;B165;1102 1168 11B0; # (녥; 녥; 녥; 녥; 녥; ) HANGUL SYLLABLE NYELG +B166;B166;1102 1168 11B1;B166;1102 1168 11B1; # (녦; 녦; 녦; 녦; 녦; ) HANGUL SYLLABLE NYELM +B167;B167;1102 1168 11B2;B167;1102 1168 11B2; # (녧; 녧; 녧; 녧; 녧; ) HANGUL SYLLABLE NYELB +B168;B168;1102 1168 11B3;B168;1102 1168 11B3; # (녨; 녨; 녨; 녨; 녨; ) HANGUL SYLLABLE NYELS +B169;B169;1102 1168 11B4;B169;1102 1168 11B4; # (녩; 녩; 녩; 녩; 녩; ) HANGUL SYLLABLE NYELT +B16A;B16A;1102 1168 11B5;B16A;1102 1168 11B5; # (녪; 녪; 녪; 녪; 녪; ) HANGUL SYLLABLE NYELP +B16B;B16B;1102 1168 11B6;B16B;1102 1168 11B6; # (녫; 녫; 녫; 녫; 녫; ) HANGUL SYLLABLE NYELH +B16C;B16C;1102 1168 11B7;B16C;1102 1168 11B7; # (녬; 녬; 녬; 녬; 녬; ) HANGUL SYLLABLE NYEM +B16D;B16D;1102 1168 11B8;B16D;1102 1168 11B8; # (녭; 녭; 녭; 녭; 녭; ) HANGUL SYLLABLE NYEB +B16E;B16E;1102 1168 11B9;B16E;1102 1168 11B9; # (녮; 녮; 녮; 녮; 녮; ) HANGUL SYLLABLE NYEBS +B16F;B16F;1102 1168 11BA;B16F;1102 1168 11BA; # (녯; 녯; 녯; 녯; 녯; ) HANGUL SYLLABLE NYES +B170;B170;1102 1168 11BB;B170;1102 1168 11BB; # (녰; 녰; 녰; 녰; 녰; ) HANGUL SYLLABLE NYESS +B171;B171;1102 1168 11BC;B171;1102 1168 11BC; # (녱; 녱; 녱; 녱; 녱; ) HANGUL SYLLABLE NYENG +B172;B172;1102 1168 11BD;B172;1102 1168 11BD; # (녲; 녲; 녲; 녲; 녲; ) HANGUL SYLLABLE NYEJ +B173;B173;1102 1168 11BE;B173;1102 1168 11BE; # (녳; 녳; 녳; 녳; 녳; ) HANGUL SYLLABLE NYEC +B174;B174;1102 1168 11BF;B174;1102 1168 11BF; # (녴; 녴; 녴; 녴; 녴; ) HANGUL SYLLABLE NYEK +B175;B175;1102 1168 11C0;B175;1102 1168 11C0; # (녵; 녵; 녵; 녵; 녵; ) HANGUL SYLLABLE NYET +B176;B176;1102 1168 11C1;B176;1102 1168 11C1; # (녶; 녶; 녶; 녶; 녶; ) HANGUL SYLLABLE NYEP +B177;B177;1102 1168 11C2;B177;1102 1168 11C2; # (녷; 녷; 녷; 녷; 녷; ) HANGUL SYLLABLE NYEH +B178;B178;1102 1169;B178;1102 1169; # (노; 노; 노; 노; 노; ) HANGUL SYLLABLE NO +B179;B179;1102 1169 11A8;B179;1102 1169 11A8; # (녹; 녹; 녹; 녹; 녹; ) HANGUL SYLLABLE NOG +B17A;B17A;1102 1169 11A9;B17A;1102 1169 11A9; # (녺; 녺; 녺; 녺; 녺; ) HANGUL SYLLABLE NOGG +B17B;B17B;1102 1169 11AA;B17B;1102 1169 11AA; # (녻; 녻; 녻; 녻; 녻; ) HANGUL SYLLABLE NOGS +B17C;B17C;1102 1169 11AB;B17C;1102 1169 11AB; # (논; 논; 논; 논; 논; ) HANGUL SYLLABLE NON +B17D;B17D;1102 1169 11AC;B17D;1102 1169 11AC; # (녽; 녽; 녽; 녽; 녽; ) HANGUL SYLLABLE NONJ +B17E;B17E;1102 1169 11AD;B17E;1102 1169 11AD; # (녾; 녾; 녾; 녾; 녾; ) HANGUL SYLLABLE NONH +B17F;B17F;1102 1169 11AE;B17F;1102 1169 11AE; # (녿; 녿; 녿; 녿; 녿; ) HANGUL SYLLABLE NOD +B180;B180;1102 1169 11AF;B180;1102 1169 11AF; # (놀; 놀; 놀; 놀; 놀; ) HANGUL SYLLABLE NOL +B181;B181;1102 1169 11B0;B181;1102 1169 11B0; # (놁; 놁; 놁; 놁; 놁; ) HANGUL SYLLABLE NOLG +B182;B182;1102 1169 11B1;B182;1102 1169 11B1; # (놂; 놂; 놂; 놂; 놂; ) HANGUL SYLLABLE NOLM +B183;B183;1102 1169 11B2;B183;1102 1169 11B2; # (놃; 놃; 놃; 놃; 놃; ) HANGUL SYLLABLE NOLB +B184;B184;1102 1169 11B3;B184;1102 1169 11B3; # (놄; 놄; 놄; 놄; 놄; ) HANGUL SYLLABLE NOLS +B185;B185;1102 1169 11B4;B185;1102 1169 11B4; # (놅; 놅; 놅; 놅; 놅; ) HANGUL SYLLABLE NOLT +B186;B186;1102 1169 11B5;B186;1102 1169 11B5; # (놆; 놆; 놆; 놆; 놆; ) HANGUL SYLLABLE NOLP +B187;B187;1102 1169 11B6;B187;1102 1169 11B6; # (놇; 놇; 놇; 놇; 놇; ) HANGUL SYLLABLE NOLH +B188;B188;1102 1169 11B7;B188;1102 1169 11B7; # (놈; 놈; 놈; 놈; 놈; ) HANGUL SYLLABLE NOM +B189;B189;1102 1169 11B8;B189;1102 1169 11B8; # (놉; 놉; 놉; 놉; 놉; ) HANGUL SYLLABLE NOB +B18A;B18A;1102 1169 11B9;B18A;1102 1169 11B9; # (놊; 놊; 놊; 놊; 놊; ) HANGUL SYLLABLE NOBS +B18B;B18B;1102 1169 11BA;B18B;1102 1169 11BA; # (놋; 놋; 놋; 놋; 놋; ) HANGUL SYLLABLE NOS +B18C;B18C;1102 1169 11BB;B18C;1102 1169 11BB; # (놌; 놌; 놌; 놌; 놌; ) HANGUL SYLLABLE NOSS +B18D;B18D;1102 1169 11BC;B18D;1102 1169 11BC; # (농; 농; 농; 농; 농; ) HANGUL SYLLABLE NONG +B18E;B18E;1102 1169 11BD;B18E;1102 1169 11BD; # (놎; 놎; 놎; 놎; 놎; ) HANGUL SYLLABLE NOJ +B18F;B18F;1102 1169 11BE;B18F;1102 1169 11BE; # (놏; 놏; 놏; 놏; 놏; ) HANGUL SYLLABLE NOC +B190;B190;1102 1169 11BF;B190;1102 1169 11BF; # (놐; 놐; 놐; 놐; 놐; ) HANGUL SYLLABLE NOK +B191;B191;1102 1169 11C0;B191;1102 1169 11C0; # (놑; 놑; 놑; 놑; 놑; ) HANGUL SYLLABLE NOT +B192;B192;1102 1169 11C1;B192;1102 1169 11C1; # (높; 높; 높; 높; 높; ) HANGUL SYLLABLE NOP +B193;B193;1102 1169 11C2;B193;1102 1169 11C2; # (놓; 놓; 놓; 놓; 놓; ) HANGUL SYLLABLE NOH +B194;B194;1102 116A;B194;1102 116A; # (놔; 놔; 놔; 놔; 놔; ) HANGUL SYLLABLE NWA +B195;B195;1102 116A 11A8;B195;1102 116A 11A8; # (놕; 놕; 놕; 놕; 놕; ) HANGUL SYLLABLE NWAG +B196;B196;1102 116A 11A9;B196;1102 116A 11A9; # (놖; 놖; 놖; 놖; 놖; ) HANGUL SYLLABLE NWAGG +B197;B197;1102 116A 11AA;B197;1102 116A 11AA; # (놗; 놗; 놗; 놗; 놗; ) HANGUL SYLLABLE NWAGS +B198;B198;1102 116A 11AB;B198;1102 116A 11AB; # (놘; 놘; 놘; 놘; 놘; ) HANGUL SYLLABLE NWAN +B199;B199;1102 116A 11AC;B199;1102 116A 11AC; # (놙; 놙; 놙; 놙; 놙; ) HANGUL SYLLABLE NWANJ +B19A;B19A;1102 116A 11AD;B19A;1102 116A 11AD; # (놚; 놚; 놚; 놚; 놚; ) HANGUL SYLLABLE NWANH +B19B;B19B;1102 116A 11AE;B19B;1102 116A 11AE; # (놛; 놛; 놛; 놛; 놛; ) HANGUL SYLLABLE NWAD +B19C;B19C;1102 116A 11AF;B19C;1102 116A 11AF; # (놜; 놜; 놜; 놜; 놜; ) HANGUL SYLLABLE NWAL +B19D;B19D;1102 116A 11B0;B19D;1102 116A 11B0; # (놝; 놝; 놝; 놝; 놝; ) HANGUL SYLLABLE NWALG +B19E;B19E;1102 116A 11B1;B19E;1102 116A 11B1; # (놞; 놞; 놞; 놞; 놞; ) HANGUL SYLLABLE NWALM +B19F;B19F;1102 116A 11B2;B19F;1102 116A 11B2; # (놟; 놟; 놟; 놟; 놟; ) HANGUL SYLLABLE NWALB +B1A0;B1A0;1102 116A 11B3;B1A0;1102 116A 11B3; # (놠; 놠; 놠; 놠; 놠; ) HANGUL SYLLABLE NWALS +B1A1;B1A1;1102 116A 11B4;B1A1;1102 116A 11B4; # (놡; 놡; 놡; 놡; 놡; ) HANGUL SYLLABLE NWALT +B1A2;B1A2;1102 116A 11B5;B1A2;1102 116A 11B5; # (놢; 놢; 놢; 놢; 놢; ) HANGUL SYLLABLE NWALP +B1A3;B1A3;1102 116A 11B6;B1A3;1102 116A 11B6; # (놣; 놣; 놣; 놣; 놣; ) HANGUL SYLLABLE NWALH +B1A4;B1A4;1102 116A 11B7;B1A4;1102 116A 11B7; # (놤; 놤; 놤; 놤; 놤; ) HANGUL SYLLABLE NWAM +B1A5;B1A5;1102 116A 11B8;B1A5;1102 116A 11B8; # (놥; 놥; 놥; 놥; 놥; ) HANGUL SYLLABLE NWAB +B1A6;B1A6;1102 116A 11B9;B1A6;1102 116A 11B9; # (놦; 놦; 놦; 놦; 놦; ) HANGUL SYLLABLE NWABS +B1A7;B1A7;1102 116A 11BA;B1A7;1102 116A 11BA; # (놧; 놧; 놧; 놧; 놧; ) HANGUL SYLLABLE NWAS +B1A8;B1A8;1102 116A 11BB;B1A8;1102 116A 11BB; # (놨; 놨; 놨; 놨; 놨; ) HANGUL SYLLABLE NWASS +B1A9;B1A9;1102 116A 11BC;B1A9;1102 116A 11BC; # (놩; 놩; 놩; 놩; 놩; ) HANGUL SYLLABLE NWANG +B1AA;B1AA;1102 116A 11BD;B1AA;1102 116A 11BD; # (놪; 놪; 놪; 놪; 놪; ) HANGUL SYLLABLE NWAJ +B1AB;B1AB;1102 116A 11BE;B1AB;1102 116A 11BE; # (놫; 놫; 놫; 놫; 놫; ) HANGUL SYLLABLE NWAC +B1AC;B1AC;1102 116A 11BF;B1AC;1102 116A 11BF; # (놬; 놬; 놬; 놬; 놬; ) HANGUL SYLLABLE NWAK +B1AD;B1AD;1102 116A 11C0;B1AD;1102 116A 11C0; # (놭; 놭; 놭; 놭; 놭; ) HANGUL SYLLABLE NWAT +B1AE;B1AE;1102 116A 11C1;B1AE;1102 116A 11C1; # (놮; 놮; 놮; 놮; 놮; ) HANGUL SYLLABLE NWAP +B1AF;B1AF;1102 116A 11C2;B1AF;1102 116A 11C2; # (놯; 놯; 놯; 놯; 놯; ) HANGUL SYLLABLE NWAH +B1B0;B1B0;1102 116B;B1B0;1102 116B; # (놰; 놰; 놰; 놰; 놰; ) HANGUL SYLLABLE NWAE +B1B1;B1B1;1102 116B 11A8;B1B1;1102 116B 11A8; # (놱; 놱; 놱; 놱; 놱; ) HANGUL SYLLABLE NWAEG +B1B2;B1B2;1102 116B 11A9;B1B2;1102 116B 11A9; # (놲; 놲; 놲; 놲; 놲; ) HANGUL SYLLABLE NWAEGG +B1B3;B1B3;1102 116B 11AA;B1B3;1102 116B 11AA; # (놳; 놳; 놳; 놳; 놳; ) HANGUL SYLLABLE NWAEGS +B1B4;B1B4;1102 116B 11AB;B1B4;1102 116B 11AB; # (놴; 놴; 놴; 놴; 놴; ) HANGUL SYLLABLE NWAEN +B1B5;B1B5;1102 116B 11AC;B1B5;1102 116B 11AC; # (놵; 놵; 놵; 놵; 놵; ) HANGUL SYLLABLE NWAENJ +B1B6;B1B6;1102 116B 11AD;B1B6;1102 116B 11AD; # (놶; 놶; 놶; 놶; 놶; ) HANGUL SYLLABLE NWAENH +B1B7;B1B7;1102 116B 11AE;B1B7;1102 116B 11AE; # (놷; 놷; 놷; 놷; 놷; ) HANGUL SYLLABLE NWAED +B1B8;B1B8;1102 116B 11AF;B1B8;1102 116B 11AF; # (놸; 놸; 놸; 놸; 놸; ) HANGUL SYLLABLE NWAEL +B1B9;B1B9;1102 116B 11B0;B1B9;1102 116B 11B0; # (놹; 놹; 놹; 놹; 놹; ) HANGUL SYLLABLE NWAELG +B1BA;B1BA;1102 116B 11B1;B1BA;1102 116B 11B1; # (놺; 놺; 놺; 놺; 놺; ) HANGUL SYLLABLE NWAELM +B1BB;B1BB;1102 116B 11B2;B1BB;1102 116B 11B2; # (놻; 놻; 놻; 놻; 놻; ) HANGUL SYLLABLE NWAELB +B1BC;B1BC;1102 116B 11B3;B1BC;1102 116B 11B3; # (놼; 놼; 놼; 놼; 놼; ) HANGUL SYLLABLE NWAELS +B1BD;B1BD;1102 116B 11B4;B1BD;1102 116B 11B4; # (놽; 놽; 놽; 놽; 놽; ) HANGUL SYLLABLE NWAELT +B1BE;B1BE;1102 116B 11B5;B1BE;1102 116B 11B5; # (놾; 놾; 놾; 놾; 놾; ) HANGUL SYLLABLE NWAELP +B1BF;B1BF;1102 116B 11B6;B1BF;1102 116B 11B6; # (놿; 놿; 놿; 놿; 놿; ) HANGUL SYLLABLE NWAELH +B1C0;B1C0;1102 116B 11B7;B1C0;1102 116B 11B7; # (뇀; 뇀; 뇀; 뇀; 뇀; ) HANGUL SYLLABLE NWAEM +B1C1;B1C1;1102 116B 11B8;B1C1;1102 116B 11B8; # (뇁; 뇁; 뇁; 뇁; 뇁; ) HANGUL SYLLABLE NWAEB +B1C2;B1C2;1102 116B 11B9;B1C2;1102 116B 11B9; # (뇂; 뇂; 뇂; 뇂; 뇂; ) HANGUL SYLLABLE NWAEBS +B1C3;B1C3;1102 116B 11BA;B1C3;1102 116B 11BA; # (뇃; 뇃; 뇃; 뇃; 뇃; ) HANGUL SYLLABLE NWAES +B1C4;B1C4;1102 116B 11BB;B1C4;1102 116B 11BB; # (뇄; 뇄; 뇄; 뇄; 뇄; ) HANGUL SYLLABLE NWAESS +B1C5;B1C5;1102 116B 11BC;B1C5;1102 116B 11BC; # (뇅; 뇅; 뇅; 뇅; 뇅; ) HANGUL SYLLABLE NWAENG +B1C6;B1C6;1102 116B 11BD;B1C6;1102 116B 11BD; # (뇆; 뇆; 뇆; 뇆; 뇆; ) HANGUL SYLLABLE NWAEJ +B1C7;B1C7;1102 116B 11BE;B1C7;1102 116B 11BE; # (뇇; 뇇; 뇇; 뇇; 뇇; ) HANGUL SYLLABLE NWAEC +B1C8;B1C8;1102 116B 11BF;B1C8;1102 116B 11BF; # (뇈; 뇈; 뇈; 뇈; 뇈; ) HANGUL SYLLABLE NWAEK +B1C9;B1C9;1102 116B 11C0;B1C9;1102 116B 11C0; # (뇉; 뇉; 뇉; 뇉; 뇉; ) HANGUL SYLLABLE NWAET +B1CA;B1CA;1102 116B 11C1;B1CA;1102 116B 11C1; # (뇊; 뇊; 뇊; 뇊; 뇊; ) HANGUL SYLLABLE NWAEP +B1CB;B1CB;1102 116B 11C2;B1CB;1102 116B 11C2; # (뇋; 뇋; 뇋; 뇋; 뇋; ) HANGUL SYLLABLE NWAEH +B1CC;B1CC;1102 116C;B1CC;1102 116C; # (뇌; 뇌; 뇌; 뇌; 뇌; ) HANGUL SYLLABLE NOE +B1CD;B1CD;1102 116C 11A8;B1CD;1102 116C 11A8; # (뇍; 뇍; 뇍; 뇍; 뇍; ) HANGUL SYLLABLE NOEG +B1CE;B1CE;1102 116C 11A9;B1CE;1102 116C 11A9; # (뇎; 뇎; 뇎; 뇎; 뇎; ) HANGUL SYLLABLE NOEGG +B1CF;B1CF;1102 116C 11AA;B1CF;1102 116C 11AA; # (뇏; 뇏; 뇏; 뇏; 뇏; ) HANGUL SYLLABLE NOEGS +B1D0;B1D0;1102 116C 11AB;B1D0;1102 116C 11AB; # (뇐; 뇐; 뇐; 뇐; 뇐; ) HANGUL SYLLABLE NOEN +B1D1;B1D1;1102 116C 11AC;B1D1;1102 116C 11AC; # (뇑; 뇑; 뇑; 뇑; 뇑; ) HANGUL SYLLABLE NOENJ +B1D2;B1D2;1102 116C 11AD;B1D2;1102 116C 11AD; # (뇒; 뇒; 뇒; 뇒; 뇒; ) HANGUL SYLLABLE NOENH +B1D3;B1D3;1102 116C 11AE;B1D3;1102 116C 11AE; # (뇓; 뇓; 뇓; 뇓; 뇓; ) HANGUL SYLLABLE NOED +B1D4;B1D4;1102 116C 11AF;B1D4;1102 116C 11AF; # (뇔; 뇔; 뇔; 뇔; 뇔; ) HANGUL SYLLABLE NOEL +B1D5;B1D5;1102 116C 11B0;B1D5;1102 116C 11B0; # (뇕; 뇕; 뇕; 뇕; 뇕; ) HANGUL SYLLABLE NOELG +B1D6;B1D6;1102 116C 11B1;B1D6;1102 116C 11B1; # (뇖; 뇖; 뇖; 뇖; 뇖; ) HANGUL SYLLABLE NOELM +B1D7;B1D7;1102 116C 11B2;B1D7;1102 116C 11B2; # (뇗; 뇗; 뇗; 뇗; 뇗; ) HANGUL SYLLABLE NOELB +B1D8;B1D8;1102 116C 11B3;B1D8;1102 116C 11B3; # (뇘; 뇘; 뇘; 뇘; 뇘; ) HANGUL SYLLABLE NOELS +B1D9;B1D9;1102 116C 11B4;B1D9;1102 116C 11B4; # (뇙; 뇙; 뇙; 뇙; 뇙; ) HANGUL SYLLABLE NOELT +B1DA;B1DA;1102 116C 11B5;B1DA;1102 116C 11B5; # (뇚; 뇚; 뇚; 뇚; 뇚; ) HANGUL SYLLABLE NOELP +B1DB;B1DB;1102 116C 11B6;B1DB;1102 116C 11B6; # (뇛; 뇛; 뇛; 뇛; 뇛; ) HANGUL SYLLABLE NOELH +B1DC;B1DC;1102 116C 11B7;B1DC;1102 116C 11B7; # (뇜; 뇜; 뇜; 뇜; 뇜; ) HANGUL SYLLABLE NOEM +B1DD;B1DD;1102 116C 11B8;B1DD;1102 116C 11B8; # (뇝; 뇝; 뇝; 뇝; 뇝; ) HANGUL SYLLABLE NOEB +B1DE;B1DE;1102 116C 11B9;B1DE;1102 116C 11B9; # (뇞; 뇞; 뇞; 뇞; 뇞; ) HANGUL SYLLABLE NOEBS +B1DF;B1DF;1102 116C 11BA;B1DF;1102 116C 11BA; # (뇟; 뇟; 뇟; 뇟; 뇟; ) HANGUL SYLLABLE NOES +B1E0;B1E0;1102 116C 11BB;B1E0;1102 116C 11BB; # (뇠; 뇠; 뇠; 뇠; 뇠; ) HANGUL SYLLABLE NOESS +B1E1;B1E1;1102 116C 11BC;B1E1;1102 116C 11BC; # (뇡; 뇡; 뇡; 뇡; 뇡; ) HANGUL SYLLABLE NOENG +B1E2;B1E2;1102 116C 11BD;B1E2;1102 116C 11BD; # (뇢; 뇢; 뇢; 뇢; 뇢; ) HANGUL SYLLABLE NOEJ +B1E3;B1E3;1102 116C 11BE;B1E3;1102 116C 11BE; # (뇣; 뇣; 뇣; 뇣; 뇣; ) HANGUL SYLLABLE NOEC +B1E4;B1E4;1102 116C 11BF;B1E4;1102 116C 11BF; # (뇤; 뇤; 뇤; 뇤; 뇤; ) HANGUL SYLLABLE NOEK +B1E5;B1E5;1102 116C 11C0;B1E5;1102 116C 11C0; # (뇥; 뇥; 뇥; 뇥; 뇥; ) HANGUL SYLLABLE NOET +B1E6;B1E6;1102 116C 11C1;B1E6;1102 116C 11C1; # (뇦; 뇦; 뇦; 뇦; 뇦; ) HANGUL SYLLABLE NOEP +B1E7;B1E7;1102 116C 11C2;B1E7;1102 116C 11C2; # (뇧; 뇧; 뇧; 뇧; 뇧; ) HANGUL SYLLABLE NOEH +B1E8;B1E8;1102 116D;B1E8;1102 116D; # (뇨; 뇨; 뇨; 뇨; 뇨; ) HANGUL SYLLABLE NYO +B1E9;B1E9;1102 116D 11A8;B1E9;1102 116D 11A8; # (뇩; 뇩; 뇩; 뇩; 뇩; ) HANGUL SYLLABLE NYOG +B1EA;B1EA;1102 116D 11A9;B1EA;1102 116D 11A9; # (뇪; 뇪; 뇪; 뇪; 뇪; ) HANGUL SYLLABLE NYOGG +B1EB;B1EB;1102 116D 11AA;B1EB;1102 116D 11AA; # (뇫; 뇫; 뇫; 뇫; 뇫; ) HANGUL SYLLABLE NYOGS +B1EC;B1EC;1102 116D 11AB;B1EC;1102 116D 11AB; # (뇬; 뇬; 뇬; 뇬; 뇬; ) HANGUL SYLLABLE NYON +B1ED;B1ED;1102 116D 11AC;B1ED;1102 116D 11AC; # (뇭; 뇭; 뇭; 뇭; 뇭; ) HANGUL SYLLABLE NYONJ +B1EE;B1EE;1102 116D 11AD;B1EE;1102 116D 11AD; # (뇮; 뇮; 뇮; 뇮; 뇮; ) HANGUL SYLLABLE NYONH +B1EF;B1EF;1102 116D 11AE;B1EF;1102 116D 11AE; # (뇯; 뇯; 뇯; 뇯; 뇯; ) HANGUL SYLLABLE NYOD +B1F0;B1F0;1102 116D 11AF;B1F0;1102 116D 11AF; # (뇰; 뇰; 뇰; 뇰; 뇰; ) HANGUL SYLLABLE NYOL +B1F1;B1F1;1102 116D 11B0;B1F1;1102 116D 11B0; # (뇱; 뇱; 뇱; 뇱; 뇱; ) HANGUL SYLLABLE NYOLG +B1F2;B1F2;1102 116D 11B1;B1F2;1102 116D 11B1; # (뇲; 뇲; 뇲; 뇲; 뇲; ) HANGUL SYLLABLE NYOLM +B1F3;B1F3;1102 116D 11B2;B1F3;1102 116D 11B2; # (뇳; 뇳; 뇳; 뇳; 뇳; ) HANGUL SYLLABLE NYOLB +B1F4;B1F4;1102 116D 11B3;B1F4;1102 116D 11B3; # (뇴; 뇴; 뇴; 뇴; 뇴; ) HANGUL SYLLABLE NYOLS +B1F5;B1F5;1102 116D 11B4;B1F5;1102 116D 11B4; # (뇵; 뇵; 뇵; 뇵; 뇵; ) HANGUL SYLLABLE NYOLT +B1F6;B1F6;1102 116D 11B5;B1F6;1102 116D 11B5; # (뇶; 뇶; 뇶; 뇶; 뇶; ) HANGUL SYLLABLE NYOLP +B1F7;B1F7;1102 116D 11B6;B1F7;1102 116D 11B6; # (뇷; 뇷; 뇷; 뇷; 뇷; ) HANGUL SYLLABLE NYOLH +B1F8;B1F8;1102 116D 11B7;B1F8;1102 116D 11B7; # (뇸; 뇸; 뇸; 뇸; 뇸; ) HANGUL SYLLABLE NYOM +B1F9;B1F9;1102 116D 11B8;B1F9;1102 116D 11B8; # (뇹; 뇹; 뇹; 뇹; 뇹; ) HANGUL SYLLABLE NYOB +B1FA;B1FA;1102 116D 11B9;B1FA;1102 116D 11B9; # (뇺; 뇺; 뇺; 뇺; 뇺; ) HANGUL SYLLABLE NYOBS +B1FB;B1FB;1102 116D 11BA;B1FB;1102 116D 11BA; # (뇻; 뇻; 뇻; 뇻; 뇻; ) HANGUL SYLLABLE NYOS +B1FC;B1FC;1102 116D 11BB;B1FC;1102 116D 11BB; # (뇼; 뇼; 뇼; 뇼; 뇼; ) HANGUL SYLLABLE NYOSS +B1FD;B1FD;1102 116D 11BC;B1FD;1102 116D 11BC; # (뇽; 뇽; 뇽; 뇽; 뇽; ) HANGUL SYLLABLE NYONG +B1FE;B1FE;1102 116D 11BD;B1FE;1102 116D 11BD; # (뇾; 뇾; 뇾; 뇾; 뇾; ) HANGUL SYLLABLE NYOJ +B1FF;B1FF;1102 116D 11BE;B1FF;1102 116D 11BE; # (뇿; 뇿; 뇿; 뇿; 뇿; ) HANGUL SYLLABLE NYOC +B200;B200;1102 116D 11BF;B200;1102 116D 11BF; # (눀; 눀; 눀; 눀; 눀; ) HANGUL SYLLABLE NYOK +B201;B201;1102 116D 11C0;B201;1102 116D 11C0; # (눁; 눁; 눁; 눁; 눁; ) HANGUL SYLLABLE NYOT +B202;B202;1102 116D 11C1;B202;1102 116D 11C1; # (눂; 눂; 눂; 눂; 눂; ) HANGUL SYLLABLE NYOP +B203;B203;1102 116D 11C2;B203;1102 116D 11C2; # (눃; 눃; 눃; 눃; 눃; ) HANGUL SYLLABLE NYOH +B204;B204;1102 116E;B204;1102 116E; # (누; 누; 누; 누; 누; ) HANGUL SYLLABLE NU +B205;B205;1102 116E 11A8;B205;1102 116E 11A8; # (눅; 눅; 눅; 눅; 눅; ) HANGUL SYLLABLE NUG +B206;B206;1102 116E 11A9;B206;1102 116E 11A9; # (눆; 눆; 눆; 눆; 눆; ) HANGUL SYLLABLE NUGG +B207;B207;1102 116E 11AA;B207;1102 116E 11AA; # (눇; 눇; 눇; 눇; 눇; ) HANGUL SYLLABLE NUGS +B208;B208;1102 116E 11AB;B208;1102 116E 11AB; # (눈; 눈; 눈; 눈; 눈; ) HANGUL SYLLABLE NUN +B209;B209;1102 116E 11AC;B209;1102 116E 11AC; # (눉; 눉; 눉; 눉; 눉; ) HANGUL SYLLABLE NUNJ +B20A;B20A;1102 116E 11AD;B20A;1102 116E 11AD; # (눊; 눊; 눊; 눊; 눊; ) HANGUL SYLLABLE NUNH +B20B;B20B;1102 116E 11AE;B20B;1102 116E 11AE; # (눋; 눋; 눋; 눋; 눋; ) HANGUL SYLLABLE NUD +B20C;B20C;1102 116E 11AF;B20C;1102 116E 11AF; # (눌; 눌; 눌; 눌; 눌; ) HANGUL SYLLABLE NUL +B20D;B20D;1102 116E 11B0;B20D;1102 116E 11B0; # (눍; 눍; 눍; 눍; 눍; ) HANGUL SYLLABLE NULG +B20E;B20E;1102 116E 11B1;B20E;1102 116E 11B1; # (눎; 눎; 눎; 눎; 눎; ) HANGUL SYLLABLE NULM +B20F;B20F;1102 116E 11B2;B20F;1102 116E 11B2; # (눏; 눏; 눏; 눏; 눏; ) HANGUL SYLLABLE NULB +B210;B210;1102 116E 11B3;B210;1102 116E 11B3; # (눐; 눐; 눐; 눐; 눐; ) HANGUL SYLLABLE NULS +B211;B211;1102 116E 11B4;B211;1102 116E 11B4; # (눑; 눑; 눑; 눑; 눑; ) HANGUL SYLLABLE NULT +B212;B212;1102 116E 11B5;B212;1102 116E 11B5; # (눒; 눒; 눒; 눒; 눒; ) HANGUL SYLLABLE NULP +B213;B213;1102 116E 11B6;B213;1102 116E 11B6; # (눓; 눓; 눓; 눓; 눓; ) HANGUL SYLLABLE NULH +B214;B214;1102 116E 11B7;B214;1102 116E 11B7; # (눔; 눔; 눔; 눔; 눔; ) HANGUL SYLLABLE NUM +B215;B215;1102 116E 11B8;B215;1102 116E 11B8; # (눕; 눕; 눕; 눕; 눕; ) HANGUL SYLLABLE NUB +B216;B216;1102 116E 11B9;B216;1102 116E 11B9; # (눖; 눖; 눖; 눖; 눖; ) HANGUL SYLLABLE NUBS +B217;B217;1102 116E 11BA;B217;1102 116E 11BA; # (눗; 눗; 눗; 눗; 눗; ) HANGUL SYLLABLE NUS +B218;B218;1102 116E 11BB;B218;1102 116E 11BB; # (눘; 눘; 눘; 눘; 눘; ) HANGUL SYLLABLE NUSS +B219;B219;1102 116E 11BC;B219;1102 116E 11BC; # (눙; 눙; 눙; 눙; 눙; ) HANGUL SYLLABLE NUNG +B21A;B21A;1102 116E 11BD;B21A;1102 116E 11BD; # (눚; 눚; 눚; 눚; 눚; ) HANGUL SYLLABLE NUJ +B21B;B21B;1102 116E 11BE;B21B;1102 116E 11BE; # (눛; 눛; 눛; 눛; 눛; ) HANGUL SYLLABLE NUC +B21C;B21C;1102 116E 11BF;B21C;1102 116E 11BF; # (눜; 눜; 눜; 눜; 눜; ) HANGUL SYLLABLE NUK +B21D;B21D;1102 116E 11C0;B21D;1102 116E 11C0; # (눝; 눝; 눝; 눝; 눝; ) HANGUL SYLLABLE NUT +B21E;B21E;1102 116E 11C1;B21E;1102 116E 11C1; # (눞; 눞; 눞; 눞; 눞; ) HANGUL SYLLABLE NUP +B21F;B21F;1102 116E 11C2;B21F;1102 116E 11C2; # (눟; 눟; 눟; 눟; 눟; ) HANGUL SYLLABLE NUH +B220;B220;1102 116F;B220;1102 116F; # (눠; 눠; 눠; 눠; 눠; ) HANGUL SYLLABLE NWEO +B221;B221;1102 116F 11A8;B221;1102 116F 11A8; # (눡; 눡; 눡; 눡; 눡; ) HANGUL SYLLABLE NWEOG +B222;B222;1102 116F 11A9;B222;1102 116F 11A9; # (눢; 눢; 눢; 눢; 눢; ) HANGUL SYLLABLE NWEOGG +B223;B223;1102 116F 11AA;B223;1102 116F 11AA; # (눣; 눣; 눣; 눣; 눣; ) HANGUL SYLLABLE NWEOGS +B224;B224;1102 116F 11AB;B224;1102 116F 11AB; # (눤; 눤; 눤; 눤; 눤; ) HANGUL SYLLABLE NWEON +B225;B225;1102 116F 11AC;B225;1102 116F 11AC; # (눥; 눥; 눥; 눥; 눥; ) HANGUL SYLLABLE NWEONJ +B226;B226;1102 116F 11AD;B226;1102 116F 11AD; # (눦; 눦; 눦; 눦; 눦; ) HANGUL SYLLABLE NWEONH +B227;B227;1102 116F 11AE;B227;1102 116F 11AE; # (눧; 눧; 눧; 눧; 눧; ) HANGUL SYLLABLE NWEOD +B228;B228;1102 116F 11AF;B228;1102 116F 11AF; # (눨; 눨; 눨; 눨; 눨; ) HANGUL SYLLABLE NWEOL +B229;B229;1102 116F 11B0;B229;1102 116F 11B0; # (눩; 눩; 눩; 눩; 눩; ) HANGUL SYLLABLE NWEOLG +B22A;B22A;1102 116F 11B1;B22A;1102 116F 11B1; # (눪; 눪; 눪; 눪; 눪; ) HANGUL SYLLABLE NWEOLM +B22B;B22B;1102 116F 11B2;B22B;1102 116F 11B2; # (눫; 눫; 눫; 눫; 눫; ) HANGUL SYLLABLE NWEOLB +B22C;B22C;1102 116F 11B3;B22C;1102 116F 11B3; # (눬; 눬; 눬; 눬; 눬; ) HANGUL SYLLABLE NWEOLS +B22D;B22D;1102 116F 11B4;B22D;1102 116F 11B4; # (눭; 눭; 눭; 눭; 눭; ) HANGUL SYLLABLE NWEOLT +B22E;B22E;1102 116F 11B5;B22E;1102 116F 11B5; # (눮; 눮; 눮; 눮; 눮; ) HANGUL SYLLABLE NWEOLP +B22F;B22F;1102 116F 11B6;B22F;1102 116F 11B6; # (눯; 눯; 눯; 눯; 눯; ) HANGUL SYLLABLE NWEOLH +B230;B230;1102 116F 11B7;B230;1102 116F 11B7; # (눰; 눰; 눰; 눰; 눰; ) HANGUL SYLLABLE NWEOM +B231;B231;1102 116F 11B8;B231;1102 116F 11B8; # (눱; 눱; 눱; 눱; 눱; ) HANGUL SYLLABLE NWEOB +B232;B232;1102 116F 11B9;B232;1102 116F 11B9; # (눲; 눲; 눲; 눲; 눲; ) HANGUL SYLLABLE NWEOBS +B233;B233;1102 116F 11BA;B233;1102 116F 11BA; # (눳; 눳; 눳; 눳; 눳; ) HANGUL SYLLABLE NWEOS +B234;B234;1102 116F 11BB;B234;1102 116F 11BB; # (눴; 눴; 눴; 눴; 눴; ) HANGUL SYLLABLE NWEOSS +B235;B235;1102 116F 11BC;B235;1102 116F 11BC; # (눵; 눵; 눵; 눵; 눵; ) HANGUL SYLLABLE NWEONG +B236;B236;1102 116F 11BD;B236;1102 116F 11BD; # (눶; 눶; 눶; 눶; 눶; ) HANGUL SYLLABLE NWEOJ +B237;B237;1102 116F 11BE;B237;1102 116F 11BE; # (눷; 눷; 눷; 눷; 눷; ) HANGUL SYLLABLE NWEOC +B238;B238;1102 116F 11BF;B238;1102 116F 11BF; # (눸; 눸; 눸; 눸; 눸; ) HANGUL SYLLABLE NWEOK +B239;B239;1102 116F 11C0;B239;1102 116F 11C0; # (눹; 눹; 눹; 눹; 눹; ) HANGUL SYLLABLE NWEOT +B23A;B23A;1102 116F 11C1;B23A;1102 116F 11C1; # (눺; 눺; 눺; 눺; 눺; ) HANGUL SYLLABLE NWEOP +B23B;B23B;1102 116F 11C2;B23B;1102 116F 11C2; # (눻; 눻; 눻; 눻; 눻; ) HANGUL SYLLABLE NWEOH +B23C;B23C;1102 1170;B23C;1102 1170; # (눼; 눼; 눼; 눼; 눼; ) HANGUL SYLLABLE NWE +B23D;B23D;1102 1170 11A8;B23D;1102 1170 11A8; # (눽; 눽; 눽; 눽; 눽; ) HANGUL SYLLABLE NWEG +B23E;B23E;1102 1170 11A9;B23E;1102 1170 11A9; # (눾; 눾; 눾; 눾; 눾; ) HANGUL SYLLABLE NWEGG +B23F;B23F;1102 1170 11AA;B23F;1102 1170 11AA; # (눿; 눿; 눿; 눿; 눿; ) HANGUL SYLLABLE NWEGS +B240;B240;1102 1170 11AB;B240;1102 1170 11AB; # (뉀; 뉀; 뉀; 뉀; 뉀; ) HANGUL SYLLABLE NWEN +B241;B241;1102 1170 11AC;B241;1102 1170 11AC; # (뉁; 뉁; 뉁; 뉁; 뉁; ) HANGUL SYLLABLE NWENJ +B242;B242;1102 1170 11AD;B242;1102 1170 11AD; # (뉂; 뉂; 뉂; 뉂; 뉂; ) HANGUL SYLLABLE NWENH +B243;B243;1102 1170 11AE;B243;1102 1170 11AE; # (뉃; 뉃; 뉃; 뉃; 뉃; ) HANGUL SYLLABLE NWED +B244;B244;1102 1170 11AF;B244;1102 1170 11AF; # (뉄; 뉄; 뉄; 뉄; 뉄; ) HANGUL SYLLABLE NWEL +B245;B245;1102 1170 11B0;B245;1102 1170 11B0; # (뉅; 뉅; 뉅; 뉅; 뉅; ) HANGUL SYLLABLE NWELG +B246;B246;1102 1170 11B1;B246;1102 1170 11B1; # (뉆; 뉆; 뉆; 뉆; 뉆; ) HANGUL SYLLABLE NWELM +B247;B247;1102 1170 11B2;B247;1102 1170 11B2; # (뉇; 뉇; 뉇; 뉇; 뉇; ) HANGUL SYLLABLE NWELB +B248;B248;1102 1170 11B3;B248;1102 1170 11B3; # (뉈; 뉈; 뉈; 뉈; 뉈; ) HANGUL SYLLABLE NWELS +B249;B249;1102 1170 11B4;B249;1102 1170 11B4; # (뉉; 뉉; 뉉; 뉉; 뉉; ) HANGUL SYLLABLE NWELT +B24A;B24A;1102 1170 11B5;B24A;1102 1170 11B5; # (뉊; 뉊; 뉊; 뉊; 뉊; ) HANGUL SYLLABLE NWELP +B24B;B24B;1102 1170 11B6;B24B;1102 1170 11B6; # (뉋; 뉋; 뉋; 뉋; 뉋; ) HANGUL SYLLABLE NWELH +B24C;B24C;1102 1170 11B7;B24C;1102 1170 11B7; # (뉌; 뉌; 뉌; 뉌; 뉌; ) HANGUL SYLLABLE NWEM +B24D;B24D;1102 1170 11B8;B24D;1102 1170 11B8; # (뉍; 뉍; 뉍; 뉍; 뉍; ) HANGUL SYLLABLE NWEB +B24E;B24E;1102 1170 11B9;B24E;1102 1170 11B9; # (뉎; 뉎; 뉎; 뉎; 뉎; ) HANGUL SYLLABLE NWEBS +B24F;B24F;1102 1170 11BA;B24F;1102 1170 11BA; # (뉏; 뉏; 뉏; 뉏; 뉏; ) HANGUL SYLLABLE NWES +B250;B250;1102 1170 11BB;B250;1102 1170 11BB; # (뉐; 뉐; 뉐; 뉐; 뉐; ) HANGUL SYLLABLE NWESS +B251;B251;1102 1170 11BC;B251;1102 1170 11BC; # (뉑; 뉑; 뉑; 뉑; 뉑; ) HANGUL SYLLABLE NWENG +B252;B252;1102 1170 11BD;B252;1102 1170 11BD; # (뉒; 뉒; 뉒; 뉒; 뉒; ) HANGUL SYLLABLE NWEJ +B253;B253;1102 1170 11BE;B253;1102 1170 11BE; # (뉓; 뉓; 뉓; 뉓; 뉓; ) HANGUL SYLLABLE NWEC +B254;B254;1102 1170 11BF;B254;1102 1170 11BF; # (뉔; 뉔; 뉔; 뉔; 뉔; ) HANGUL SYLLABLE NWEK +B255;B255;1102 1170 11C0;B255;1102 1170 11C0; # (뉕; 뉕; 뉕; 뉕; 뉕; ) HANGUL SYLLABLE NWET +B256;B256;1102 1170 11C1;B256;1102 1170 11C1; # (뉖; 뉖; 뉖; 뉖; 뉖; ) HANGUL SYLLABLE NWEP +B257;B257;1102 1170 11C2;B257;1102 1170 11C2; # (뉗; 뉗; 뉗; 뉗; 뉗; ) HANGUL SYLLABLE NWEH +B258;B258;1102 1171;B258;1102 1171; # (뉘; 뉘; 뉘; 뉘; 뉘; ) HANGUL SYLLABLE NWI +B259;B259;1102 1171 11A8;B259;1102 1171 11A8; # (뉙; 뉙; 뉙; 뉙; 뉙; ) HANGUL SYLLABLE NWIG +B25A;B25A;1102 1171 11A9;B25A;1102 1171 11A9; # (뉚; 뉚; 뉚; 뉚; 뉚; ) HANGUL SYLLABLE NWIGG +B25B;B25B;1102 1171 11AA;B25B;1102 1171 11AA; # (뉛; 뉛; 뉛; 뉛; 뉛; ) HANGUL SYLLABLE NWIGS +B25C;B25C;1102 1171 11AB;B25C;1102 1171 11AB; # (뉜; 뉜; 뉜; 뉜; 뉜; ) HANGUL SYLLABLE NWIN +B25D;B25D;1102 1171 11AC;B25D;1102 1171 11AC; # (뉝; 뉝; 뉝; 뉝; 뉝; ) HANGUL SYLLABLE NWINJ +B25E;B25E;1102 1171 11AD;B25E;1102 1171 11AD; # (뉞; 뉞; 뉞; 뉞; 뉞; ) HANGUL SYLLABLE NWINH +B25F;B25F;1102 1171 11AE;B25F;1102 1171 11AE; # (뉟; 뉟; 뉟; 뉟; 뉟; ) HANGUL SYLLABLE NWID +B260;B260;1102 1171 11AF;B260;1102 1171 11AF; # (뉠; 뉠; 뉠; 뉠; 뉠; ) HANGUL SYLLABLE NWIL +B261;B261;1102 1171 11B0;B261;1102 1171 11B0; # (뉡; 뉡; 뉡; 뉡; 뉡; ) HANGUL SYLLABLE NWILG +B262;B262;1102 1171 11B1;B262;1102 1171 11B1; # (뉢; 뉢; 뉢; 뉢; 뉢; ) HANGUL SYLLABLE NWILM +B263;B263;1102 1171 11B2;B263;1102 1171 11B2; # (뉣; 뉣; 뉣; 뉣; 뉣; ) HANGUL SYLLABLE NWILB +B264;B264;1102 1171 11B3;B264;1102 1171 11B3; # (뉤; 뉤; 뉤; 뉤; 뉤; ) HANGUL SYLLABLE NWILS +B265;B265;1102 1171 11B4;B265;1102 1171 11B4; # (뉥; 뉥; 뉥; 뉥; 뉥; ) HANGUL SYLLABLE NWILT +B266;B266;1102 1171 11B5;B266;1102 1171 11B5; # (뉦; 뉦; 뉦; 뉦; 뉦; ) HANGUL SYLLABLE NWILP +B267;B267;1102 1171 11B6;B267;1102 1171 11B6; # (뉧; 뉧; 뉧; 뉧; 뉧; ) HANGUL SYLLABLE NWILH +B268;B268;1102 1171 11B7;B268;1102 1171 11B7; # (뉨; 뉨; 뉨; 뉨; 뉨; ) HANGUL SYLLABLE NWIM +B269;B269;1102 1171 11B8;B269;1102 1171 11B8; # (뉩; 뉩; 뉩; 뉩; 뉩; ) HANGUL SYLLABLE NWIB +B26A;B26A;1102 1171 11B9;B26A;1102 1171 11B9; # (뉪; 뉪; 뉪; 뉪; 뉪; ) HANGUL SYLLABLE NWIBS +B26B;B26B;1102 1171 11BA;B26B;1102 1171 11BA; # (뉫; 뉫; 뉫; 뉫; 뉫; ) HANGUL SYLLABLE NWIS +B26C;B26C;1102 1171 11BB;B26C;1102 1171 11BB; # (뉬; 뉬; 뉬; 뉬; 뉬; ) HANGUL SYLLABLE NWISS +B26D;B26D;1102 1171 11BC;B26D;1102 1171 11BC; # (뉭; 뉭; 뉭; 뉭; 뉭; ) HANGUL SYLLABLE NWING +B26E;B26E;1102 1171 11BD;B26E;1102 1171 11BD; # (뉮; 뉮; 뉮; 뉮; 뉮; ) HANGUL SYLLABLE NWIJ +B26F;B26F;1102 1171 11BE;B26F;1102 1171 11BE; # (뉯; 뉯; 뉯; 뉯; 뉯; ) HANGUL SYLLABLE NWIC +B270;B270;1102 1171 11BF;B270;1102 1171 11BF; # (뉰; 뉰; 뉰; 뉰; 뉰; ) HANGUL SYLLABLE NWIK +B271;B271;1102 1171 11C0;B271;1102 1171 11C0; # (뉱; 뉱; 뉱; 뉱; 뉱; ) HANGUL SYLLABLE NWIT +B272;B272;1102 1171 11C1;B272;1102 1171 11C1; # (뉲; 뉲; 뉲; 뉲; 뉲; ) HANGUL SYLLABLE NWIP +B273;B273;1102 1171 11C2;B273;1102 1171 11C2; # (뉳; 뉳; 뉳; 뉳; 뉳; ) HANGUL SYLLABLE NWIH +B274;B274;1102 1172;B274;1102 1172; # (뉴; 뉴; 뉴; 뉴; 뉴; ) HANGUL SYLLABLE NYU +B275;B275;1102 1172 11A8;B275;1102 1172 11A8; # (뉵; 뉵; 뉵; 뉵; 뉵; ) HANGUL SYLLABLE NYUG +B276;B276;1102 1172 11A9;B276;1102 1172 11A9; # (뉶; 뉶; 뉶; 뉶; 뉶; ) HANGUL SYLLABLE NYUGG +B277;B277;1102 1172 11AA;B277;1102 1172 11AA; # (뉷; 뉷; 뉷; 뉷; 뉷; ) HANGUL SYLLABLE NYUGS +B278;B278;1102 1172 11AB;B278;1102 1172 11AB; # (뉸; 뉸; 뉸; 뉸; 뉸; ) HANGUL SYLLABLE NYUN +B279;B279;1102 1172 11AC;B279;1102 1172 11AC; # (뉹; 뉹; 뉹; 뉹; 뉹; ) HANGUL SYLLABLE NYUNJ +B27A;B27A;1102 1172 11AD;B27A;1102 1172 11AD; # (뉺; 뉺; 뉺; 뉺; 뉺; ) HANGUL SYLLABLE NYUNH +B27B;B27B;1102 1172 11AE;B27B;1102 1172 11AE; # (뉻; 뉻; 뉻; 뉻; 뉻; ) HANGUL SYLLABLE NYUD +B27C;B27C;1102 1172 11AF;B27C;1102 1172 11AF; # (뉼; 뉼; 뉼; 뉼; 뉼; ) HANGUL SYLLABLE NYUL +B27D;B27D;1102 1172 11B0;B27D;1102 1172 11B0; # (뉽; 뉽; 뉽; 뉽; 뉽; ) HANGUL SYLLABLE NYULG +B27E;B27E;1102 1172 11B1;B27E;1102 1172 11B1; # (뉾; 뉾; 뉾; 뉾; 뉾; ) HANGUL SYLLABLE NYULM +B27F;B27F;1102 1172 11B2;B27F;1102 1172 11B2; # (뉿; 뉿; 뉿; 뉿; 뉿; ) HANGUL SYLLABLE NYULB +B280;B280;1102 1172 11B3;B280;1102 1172 11B3; # (늀; 늀; 늀; 늀; 늀; ) HANGUL SYLLABLE NYULS +B281;B281;1102 1172 11B4;B281;1102 1172 11B4; # (늁; 늁; 늁; 늁; 늁; ) HANGUL SYLLABLE NYULT +B282;B282;1102 1172 11B5;B282;1102 1172 11B5; # (늂; 늂; 늂; 늂; 늂; ) HANGUL SYLLABLE NYULP +B283;B283;1102 1172 11B6;B283;1102 1172 11B6; # (늃; 늃; 늃; 늃; 늃; ) HANGUL SYLLABLE NYULH +B284;B284;1102 1172 11B7;B284;1102 1172 11B7; # (늄; 늄; 늄; 늄; 늄; ) HANGUL SYLLABLE NYUM +B285;B285;1102 1172 11B8;B285;1102 1172 11B8; # (늅; 늅; 늅; 늅; 늅; ) HANGUL SYLLABLE NYUB +B286;B286;1102 1172 11B9;B286;1102 1172 11B9; # (늆; 늆; 늆; 늆; 늆; ) HANGUL SYLLABLE NYUBS +B287;B287;1102 1172 11BA;B287;1102 1172 11BA; # (늇; 늇; 늇; 늇; 늇; ) HANGUL SYLLABLE NYUS +B288;B288;1102 1172 11BB;B288;1102 1172 11BB; # (늈; 늈; 늈; 늈; 늈; ) HANGUL SYLLABLE NYUSS +B289;B289;1102 1172 11BC;B289;1102 1172 11BC; # (늉; 늉; 늉; 늉; 늉; ) HANGUL SYLLABLE NYUNG +B28A;B28A;1102 1172 11BD;B28A;1102 1172 11BD; # (늊; 늊; 늊; 늊; 늊; ) HANGUL SYLLABLE NYUJ +B28B;B28B;1102 1172 11BE;B28B;1102 1172 11BE; # (늋; 늋; 늋; 늋; 늋; ) HANGUL SYLLABLE NYUC +B28C;B28C;1102 1172 11BF;B28C;1102 1172 11BF; # (늌; 늌; 늌; 늌; 늌; ) HANGUL SYLLABLE NYUK +B28D;B28D;1102 1172 11C0;B28D;1102 1172 11C0; # (늍; 늍; 늍; 늍; 늍; ) HANGUL SYLLABLE NYUT +B28E;B28E;1102 1172 11C1;B28E;1102 1172 11C1; # (늎; 늎; 늎; 늎; 늎; ) HANGUL SYLLABLE NYUP +B28F;B28F;1102 1172 11C2;B28F;1102 1172 11C2; # (늏; 늏; 늏; 늏; 늏; ) HANGUL SYLLABLE NYUH +B290;B290;1102 1173;B290;1102 1173; # (느; 느; 느; 느; 느; ) HANGUL SYLLABLE NEU +B291;B291;1102 1173 11A8;B291;1102 1173 11A8; # (늑; 늑; 늑; 늑; 늑; ) HANGUL SYLLABLE NEUG +B292;B292;1102 1173 11A9;B292;1102 1173 11A9; # (늒; 늒; 늒; 늒; 늒; ) HANGUL SYLLABLE NEUGG +B293;B293;1102 1173 11AA;B293;1102 1173 11AA; # (늓; 늓; 늓; 늓; 늓; ) HANGUL SYLLABLE NEUGS +B294;B294;1102 1173 11AB;B294;1102 1173 11AB; # (는; 는; 는; 는; 는; ) HANGUL SYLLABLE NEUN +B295;B295;1102 1173 11AC;B295;1102 1173 11AC; # (늕; 늕; 늕; 늕; 늕; ) HANGUL SYLLABLE NEUNJ +B296;B296;1102 1173 11AD;B296;1102 1173 11AD; # (늖; 늖; 늖; 늖; 늖; ) HANGUL SYLLABLE NEUNH +B297;B297;1102 1173 11AE;B297;1102 1173 11AE; # (늗; 늗; 늗; 늗; 늗; ) HANGUL SYLLABLE NEUD +B298;B298;1102 1173 11AF;B298;1102 1173 11AF; # (늘; 늘; 늘; 늘; 늘; ) HANGUL SYLLABLE NEUL +B299;B299;1102 1173 11B0;B299;1102 1173 11B0; # (늙; 늙; 늙; 늙; 늙; ) HANGUL SYLLABLE NEULG +B29A;B29A;1102 1173 11B1;B29A;1102 1173 11B1; # (늚; 늚; 늚; 늚; 늚; ) HANGUL SYLLABLE NEULM +B29B;B29B;1102 1173 11B2;B29B;1102 1173 11B2; # (늛; 늛; 늛; 늛; 늛; ) HANGUL SYLLABLE NEULB +B29C;B29C;1102 1173 11B3;B29C;1102 1173 11B3; # (늜; 늜; 늜; 늜; 늜; ) HANGUL SYLLABLE NEULS +B29D;B29D;1102 1173 11B4;B29D;1102 1173 11B4; # (늝; 늝; 늝; 늝; 늝; ) HANGUL SYLLABLE NEULT +B29E;B29E;1102 1173 11B5;B29E;1102 1173 11B5; # (늞; 늞; 늞; 늞; 늞; ) HANGUL SYLLABLE NEULP +B29F;B29F;1102 1173 11B6;B29F;1102 1173 11B6; # (늟; 늟; 늟; 늟; 늟; ) HANGUL SYLLABLE NEULH +B2A0;B2A0;1102 1173 11B7;B2A0;1102 1173 11B7; # (늠; 늠; 늠; 늠; 늠; ) HANGUL SYLLABLE NEUM +B2A1;B2A1;1102 1173 11B8;B2A1;1102 1173 11B8; # (늡; 늡; 늡; 늡; 늡; ) HANGUL SYLLABLE NEUB +B2A2;B2A2;1102 1173 11B9;B2A2;1102 1173 11B9; # (늢; 늢; 늢; 늢; 늢; ) HANGUL SYLLABLE NEUBS +B2A3;B2A3;1102 1173 11BA;B2A3;1102 1173 11BA; # (늣; 늣; 늣; 늣; 늣; ) HANGUL SYLLABLE NEUS +B2A4;B2A4;1102 1173 11BB;B2A4;1102 1173 11BB; # (늤; 늤; 늤; 늤; 늤; ) HANGUL SYLLABLE NEUSS +B2A5;B2A5;1102 1173 11BC;B2A5;1102 1173 11BC; # (능; 능; 능; 능; 능; ) HANGUL SYLLABLE NEUNG +B2A6;B2A6;1102 1173 11BD;B2A6;1102 1173 11BD; # (늦; 늦; 늦; 늦; 늦; ) HANGUL SYLLABLE NEUJ +B2A7;B2A7;1102 1173 11BE;B2A7;1102 1173 11BE; # (늧; 늧; 늧; 늧; 늧; ) HANGUL SYLLABLE NEUC +B2A8;B2A8;1102 1173 11BF;B2A8;1102 1173 11BF; # (늨; 늨; 늨; 늨; 늨; ) HANGUL SYLLABLE NEUK +B2A9;B2A9;1102 1173 11C0;B2A9;1102 1173 11C0; # (늩; 늩; 늩; 늩; 늩; ) HANGUL SYLLABLE NEUT +B2AA;B2AA;1102 1173 11C1;B2AA;1102 1173 11C1; # (늪; 늪; 늪; 늪; 늪; ) HANGUL SYLLABLE NEUP +B2AB;B2AB;1102 1173 11C2;B2AB;1102 1173 11C2; # (늫; 늫; 늫; 늫; 늫; ) HANGUL SYLLABLE NEUH +B2AC;B2AC;1102 1174;B2AC;1102 1174; # (늬; 늬; 늬; 늬; 늬; ) HANGUL SYLLABLE NYI +B2AD;B2AD;1102 1174 11A8;B2AD;1102 1174 11A8; # (늭; 늭; 늭; 늭; 늭; ) HANGUL SYLLABLE NYIG +B2AE;B2AE;1102 1174 11A9;B2AE;1102 1174 11A9; # (늮; 늮; 늮; 늮; 늮; ) HANGUL SYLLABLE NYIGG +B2AF;B2AF;1102 1174 11AA;B2AF;1102 1174 11AA; # (늯; 늯; 늯; 늯; 늯; ) HANGUL SYLLABLE NYIGS +B2B0;B2B0;1102 1174 11AB;B2B0;1102 1174 11AB; # (늰; 늰; 늰; 늰; 늰; ) HANGUL SYLLABLE NYIN +B2B1;B2B1;1102 1174 11AC;B2B1;1102 1174 11AC; # (늱; 늱; 늱; 늱; 늱; ) HANGUL SYLLABLE NYINJ +B2B2;B2B2;1102 1174 11AD;B2B2;1102 1174 11AD; # (늲; 늲; 늲; 늲; 늲; ) HANGUL SYLLABLE NYINH +B2B3;B2B3;1102 1174 11AE;B2B3;1102 1174 11AE; # (늳; 늳; 늳; 늳; 늳; ) HANGUL SYLLABLE NYID +B2B4;B2B4;1102 1174 11AF;B2B4;1102 1174 11AF; # (늴; 늴; 늴; 늴; 늴; ) HANGUL SYLLABLE NYIL +B2B5;B2B5;1102 1174 11B0;B2B5;1102 1174 11B0; # (늵; 늵; 늵; 늵; 늵; ) HANGUL SYLLABLE NYILG +B2B6;B2B6;1102 1174 11B1;B2B6;1102 1174 11B1; # (늶; 늶; 늶; 늶; 늶; ) HANGUL SYLLABLE NYILM +B2B7;B2B7;1102 1174 11B2;B2B7;1102 1174 11B2; # (늷; 늷; 늷; 늷; 늷; ) HANGUL SYLLABLE NYILB +B2B8;B2B8;1102 1174 11B3;B2B8;1102 1174 11B3; # (늸; 늸; 늸; 늸; 늸; ) HANGUL SYLLABLE NYILS +B2B9;B2B9;1102 1174 11B4;B2B9;1102 1174 11B4; # (늹; 늹; 늹; 늹; 늹; ) HANGUL SYLLABLE NYILT +B2BA;B2BA;1102 1174 11B5;B2BA;1102 1174 11B5; # (늺; 늺; 늺; 늺; 늺; ) HANGUL SYLLABLE NYILP +B2BB;B2BB;1102 1174 11B6;B2BB;1102 1174 11B6; # (늻; 늻; 늻; 늻; 늻; ) HANGUL SYLLABLE NYILH +B2BC;B2BC;1102 1174 11B7;B2BC;1102 1174 11B7; # (늼; 늼; 늼; 늼; 늼; ) HANGUL SYLLABLE NYIM +B2BD;B2BD;1102 1174 11B8;B2BD;1102 1174 11B8; # (늽; 늽; 늽; 늽; 늽; ) HANGUL SYLLABLE NYIB +B2BE;B2BE;1102 1174 11B9;B2BE;1102 1174 11B9; # (늾; 늾; 늾; 늾; 늾; ) HANGUL SYLLABLE NYIBS +B2BF;B2BF;1102 1174 11BA;B2BF;1102 1174 11BA; # (늿; 늿; 늿; 늿; 늿; ) HANGUL SYLLABLE NYIS +B2C0;B2C0;1102 1174 11BB;B2C0;1102 1174 11BB; # (닀; 닀; 닀; 닀; 닀; ) HANGUL SYLLABLE NYISS +B2C1;B2C1;1102 1174 11BC;B2C1;1102 1174 11BC; # (닁; 닁; 닁; 닁; 닁; ) HANGUL SYLLABLE NYING +B2C2;B2C2;1102 1174 11BD;B2C2;1102 1174 11BD; # (닂; 닂; 닂; 닂; 닂; ) HANGUL SYLLABLE NYIJ +B2C3;B2C3;1102 1174 11BE;B2C3;1102 1174 11BE; # (닃; 닃; 닃; 닃; 닃; ) HANGUL SYLLABLE NYIC +B2C4;B2C4;1102 1174 11BF;B2C4;1102 1174 11BF; # (닄; 닄; 닄; 닄; 닄; ) HANGUL SYLLABLE NYIK +B2C5;B2C5;1102 1174 11C0;B2C5;1102 1174 11C0; # (닅; 닅; 닅; 닅; 닅; ) HANGUL SYLLABLE NYIT +B2C6;B2C6;1102 1174 11C1;B2C6;1102 1174 11C1; # (닆; 닆; 닆; 닆; 닆; ) HANGUL SYLLABLE NYIP +B2C7;B2C7;1102 1174 11C2;B2C7;1102 1174 11C2; # (닇; 닇; 닇; 닇; 닇; ) HANGUL SYLLABLE NYIH +B2C8;B2C8;1102 1175;B2C8;1102 1175; # (니; 니; 니; 니; 니; ) HANGUL SYLLABLE NI +B2C9;B2C9;1102 1175 11A8;B2C9;1102 1175 11A8; # (닉; 닉; 닉; 닉; 닉; ) HANGUL SYLLABLE NIG +B2CA;B2CA;1102 1175 11A9;B2CA;1102 1175 11A9; # (닊; 닊; 닊; 닊; 닊; ) HANGUL SYLLABLE NIGG +B2CB;B2CB;1102 1175 11AA;B2CB;1102 1175 11AA; # (닋; 닋; 닋; 닋; 닋; ) HANGUL SYLLABLE NIGS +B2CC;B2CC;1102 1175 11AB;B2CC;1102 1175 11AB; # (닌; 닌; 닌; 닌; 닌; ) HANGUL SYLLABLE NIN +B2CD;B2CD;1102 1175 11AC;B2CD;1102 1175 11AC; # (닍; 닍; 닍; 닍; 닍; ) HANGUL SYLLABLE NINJ +B2CE;B2CE;1102 1175 11AD;B2CE;1102 1175 11AD; # (닎; 닎; 닎; 닎; 닎; ) HANGUL SYLLABLE NINH +B2CF;B2CF;1102 1175 11AE;B2CF;1102 1175 11AE; # (닏; 닏; 닏; 닏; 닏; ) HANGUL SYLLABLE NID +B2D0;B2D0;1102 1175 11AF;B2D0;1102 1175 11AF; # (닐; 닐; 닐; 닐; 닐; ) HANGUL SYLLABLE NIL +B2D1;B2D1;1102 1175 11B0;B2D1;1102 1175 11B0; # (닑; 닑; 닑; 닑; 닑; ) HANGUL SYLLABLE NILG +B2D2;B2D2;1102 1175 11B1;B2D2;1102 1175 11B1; # (닒; 닒; 닒; 닒; 닒; ) HANGUL SYLLABLE NILM +B2D3;B2D3;1102 1175 11B2;B2D3;1102 1175 11B2; # (닓; 닓; 닓; 닓; 닓; ) HANGUL SYLLABLE NILB +B2D4;B2D4;1102 1175 11B3;B2D4;1102 1175 11B3; # (닔; 닔; 닔; 닔; 닔; ) HANGUL SYLLABLE NILS +B2D5;B2D5;1102 1175 11B4;B2D5;1102 1175 11B4; # (닕; 닕; 닕; 닕; 닕; ) HANGUL SYLLABLE NILT +B2D6;B2D6;1102 1175 11B5;B2D6;1102 1175 11B5; # (닖; 닖; 닖; 닖; 닖; ) HANGUL SYLLABLE NILP +B2D7;B2D7;1102 1175 11B6;B2D7;1102 1175 11B6; # (닗; 닗; 닗; 닗; 닗; ) HANGUL SYLLABLE NILH +B2D8;B2D8;1102 1175 11B7;B2D8;1102 1175 11B7; # (님; 님; 님; 님; 님; ) HANGUL SYLLABLE NIM +B2D9;B2D9;1102 1175 11B8;B2D9;1102 1175 11B8; # (닙; 닙; 닙; 닙; 닙; ) HANGUL SYLLABLE NIB +B2DA;B2DA;1102 1175 11B9;B2DA;1102 1175 11B9; # (닚; 닚; 닚; 닚; 닚; ) HANGUL SYLLABLE NIBS +B2DB;B2DB;1102 1175 11BA;B2DB;1102 1175 11BA; # (닛; 닛; 닛; 닛; 닛; ) HANGUL SYLLABLE NIS +B2DC;B2DC;1102 1175 11BB;B2DC;1102 1175 11BB; # (닜; 닜; 닜; 닜; 닜; ) HANGUL SYLLABLE NISS +B2DD;B2DD;1102 1175 11BC;B2DD;1102 1175 11BC; # (닝; 닝; 닝; 닝; 닝; ) HANGUL SYLLABLE NING +B2DE;B2DE;1102 1175 11BD;B2DE;1102 1175 11BD; # (닞; 닞; 닞; 닞; 닞; ) HANGUL SYLLABLE NIJ +B2DF;B2DF;1102 1175 11BE;B2DF;1102 1175 11BE; # (닟; 닟; 닟; 닟; 닟; ) HANGUL SYLLABLE NIC +B2E0;B2E0;1102 1175 11BF;B2E0;1102 1175 11BF; # (닠; 닠; 닠; 닠; 닠; ) HANGUL SYLLABLE NIK +B2E1;B2E1;1102 1175 11C0;B2E1;1102 1175 11C0; # (닡; 닡; 닡; 닡; 닡; ) HANGUL SYLLABLE NIT +B2E2;B2E2;1102 1175 11C1;B2E2;1102 1175 11C1; # (닢; 닢; 닢; 닢; 닢; ) HANGUL SYLLABLE NIP +B2E3;B2E3;1102 1175 11C2;B2E3;1102 1175 11C2; # (닣; 닣; 닣; 닣; 닣; ) HANGUL SYLLABLE NIH +B2E4;B2E4;1103 1161;B2E4;1103 1161; # (다; 다; 다; 다; 다; ) HANGUL SYLLABLE DA +B2E5;B2E5;1103 1161 11A8;B2E5;1103 1161 11A8; # (닥; 닥; 닥; 닥; 닥; ) HANGUL SYLLABLE DAG +B2E6;B2E6;1103 1161 11A9;B2E6;1103 1161 11A9; # (닦; 닦; 닦; 닦; 닦; ) HANGUL SYLLABLE DAGG +B2E7;B2E7;1103 1161 11AA;B2E7;1103 1161 11AA; # (닧; 닧; 닧; 닧; 닧; ) HANGUL SYLLABLE DAGS +B2E8;B2E8;1103 1161 11AB;B2E8;1103 1161 11AB; # (단; 단; 단; 단; 단; ) HANGUL SYLLABLE DAN +B2E9;B2E9;1103 1161 11AC;B2E9;1103 1161 11AC; # (닩; 닩; 닩; 닩; 닩; ) HANGUL SYLLABLE DANJ +B2EA;B2EA;1103 1161 11AD;B2EA;1103 1161 11AD; # (닪; 닪; 닪; 닪; 닪; ) HANGUL SYLLABLE DANH +B2EB;B2EB;1103 1161 11AE;B2EB;1103 1161 11AE; # (닫; 닫; 닫; 닫; 닫; ) HANGUL SYLLABLE DAD +B2EC;B2EC;1103 1161 11AF;B2EC;1103 1161 11AF; # (달; 달; 달; 달; 달; ) HANGUL SYLLABLE DAL +B2ED;B2ED;1103 1161 11B0;B2ED;1103 1161 11B0; # (닭; 닭; 닭; 닭; 닭; ) HANGUL SYLLABLE DALG +B2EE;B2EE;1103 1161 11B1;B2EE;1103 1161 11B1; # (닮; 닮; 닮; 닮; 닮; ) HANGUL SYLLABLE DALM +B2EF;B2EF;1103 1161 11B2;B2EF;1103 1161 11B2; # (닯; 닯; 닯; 닯; 닯; ) HANGUL SYLLABLE DALB +B2F0;B2F0;1103 1161 11B3;B2F0;1103 1161 11B3; # (닰; 닰; 닰; 닰; 닰; ) HANGUL SYLLABLE DALS +B2F1;B2F1;1103 1161 11B4;B2F1;1103 1161 11B4; # (닱; 닱; 닱; 닱; 닱; ) HANGUL SYLLABLE DALT +B2F2;B2F2;1103 1161 11B5;B2F2;1103 1161 11B5; # (닲; 닲; 닲; 닲; 닲; ) HANGUL SYLLABLE DALP +B2F3;B2F3;1103 1161 11B6;B2F3;1103 1161 11B6; # (닳; 닳; 닳; 닳; 닳; ) HANGUL SYLLABLE DALH +B2F4;B2F4;1103 1161 11B7;B2F4;1103 1161 11B7; # (담; 담; 담; 담; 담; ) HANGUL SYLLABLE DAM +B2F5;B2F5;1103 1161 11B8;B2F5;1103 1161 11B8; # (답; 답; 답; 답; 답; ) HANGUL SYLLABLE DAB +B2F6;B2F6;1103 1161 11B9;B2F6;1103 1161 11B9; # (닶; 닶; 닶; 닶; 닶; ) HANGUL SYLLABLE DABS +B2F7;B2F7;1103 1161 11BA;B2F7;1103 1161 11BA; # (닷; 닷; 닷; 닷; 닷; ) HANGUL SYLLABLE DAS +B2F8;B2F8;1103 1161 11BB;B2F8;1103 1161 11BB; # (닸; 닸; 닸; 닸; 닸; ) HANGUL SYLLABLE DASS +B2F9;B2F9;1103 1161 11BC;B2F9;1103 1161 11BC; # (당; 당; 당; 당; 당; ) HANGUL SYLLABLE DANG +B2FA;B2FA;1103 1161 11BD;B2FA;1103 1161 11BD; # (닺; 닺; 닺; 닺; 닺; ) HANGUL SYLLABLE DAJ +B2FB;B2FB;1103 1161 11BE;B2FB;1103 1161 11BE; # (닻; 닻; 닻; 닻; 닻; ) HANGUL SYLLABLE DAC +B2FC;B2FC;1103 1161 11BF;B2FC;1103 1161 11BF; # (닼; 닼; 닼; 닼; 닼; ) HANGUL SYLLABLE DAK +B2FD;B2FD;1103 1161 11C0;B2FD;1103 1161 11C0; # (닽; 닽; 닽; 닽; 닽; ) HANGUL SYLLABLE DAT +B2FE;B2FE;1103 1161 11C1;B2FE;1103 1161 11C1; # (닾; 닾; 닾; 닾; 닾; ) HANGUL SYLLABLE DAP +B2FF;B2FF;1103 1161 11C2;B2FF;1103 1161 11C2; # (닿; 닿; 닿; 닿; 닿; ) HANGUL SYLLABLE DAH +B300;B300;1103 1162;B300;1103 1162; # (대; 대; 대; 대; 대; ) HANGUL SYLLABLE DAE +B301;B301;1103 1162 11A8;B301;1103 1162 11A8; # (댁; 댁; 댁; 댁; 댁; ) HANGUL SYLLABLE DAEG +B302;B302;1103 1162 11A9;B302;1103 1162 11A9; # (댂; 댂; 댂; 댂; 댂; ) HANGUL SYLLABLE DAEGG +B303;B303;1103 1162 11AA;B303;1103 1162 11AA; # (댃; 댃; 댃; 댃; 댃; ) HANGUL SYLLABLE DAEGS +B304;B304;1103 1162 11AB;B304;1103 1162 11AB; # (댄; 댄; 댄; 댄; 댄; ) HANGUL SYLLABLE DAEN +B305;B305;1103 1162 11AC;B305;1103 1162 11AC; # (댅; 댅; 댅; 댅; 댅; ) HANGUL SYLLABLE DAENJ +B306;B306;1103 1162 11AD;B306;1103 1162 11AD; # (댆; 댆; 댆; 댆; 댆; ) HANGUL SYLLABLE DAENH +B307;B307;1103 1162 11AE;B307;1103 1162 11AE; # (댇; 댇; 댇; 댇; 댇; ) HANGUL SYLLABLE DAED +B308;B308;1103 1162 11AF;B308;1103 1162 11AF; # (댈; 댈; 댈; 댈; 댈; ) HANGUL SYLLABLE DAEL +B309;B309;1103 1162 11B0;B309;1103 1162 11B0; # (댉; 댉; 댉; 댉; 댉; ) HANGUL SYLLABLE DAELG +B30A;B30A;1103 1162 11B1;B30A;1103 1162 11B1; # (댊; 댊; 댊; 댊; 댊; ) HANGUL SYLLABLE DAELM +B30B;B30B;1103 1162 11B2;B30B;1103 1162 11B2; # (댋; 댋; 댋; 댋; 댋; ) HANGUL SYLLABLE DAELB +B30C;B30C;1103 1162 11B3;B30C;1103 1162 11B3; # (댌; 댌; 댌; 댌; 댌; ) HANGUL SYLLABLE DAELS +B30D;B30D;1103 1162 11B4;B30D;1103 1162 11B4; # (댍; 댍; 댍; 댍; 댍; ) HANGUL SYLLABLE DAELT +B30E;B30E;1103 1162 11B5;B30E;1103 1162 11B5; # (댎; 댎; 댎; 댎; 댎; ) HANGUL SYLLABLE DAELP +B30F;B30F;1103 1162 11B6;B30F;1103 1162 11B6; # (댏; 댏; 댏; 댏; 댏; ) HANGUL SYLLABLE DAELH +B310;B310;1103 1162 11B7;B310;1103 1162 11B7; # (댐; 댐; 댐; 댐; 댐; ) HANGUL SYLLABLE DAEM +B311;B311;1103 1162 11B8;B311;1103 1162 11B8; # (댑; 댑; 댑; 댑; 댑; ) HANGUL SYLLABLE DAEB +B312;B312;1103 1162 11B9;B312;1103 1162 11B9; # (댒; 댒; 댒; 댒; 댒; ) HANGUL SYLLABLE DAEBS +B313;B313;1103 1162 11BA;B313;1103 1162 11BA; # (댓; 댓; 댓; 댓; 댓; ) HANGUL SYLLABLE DAES +B314;B314;1103 1162 11BB;B314;1103 1162 11BB; # (댔; 댔; 댔; 댔; 댔; ) HANGUL SYLLABLE DAESS +B315;B315;1103 1162 11BC;B315;1103 1162 11BC; # (댕; 댕; 댕; 댕; 댕; ) HANGUL SYLLABLE DAENG +B316;B316;1103 1162 11BD;B316;1103 1162 11BD; # (댖; 댖; 댖; 댖; 댖; ) HANGUL SYLLABLE DAEJ +B317;B317;1103 1162 11BE;B317;1103 1162 11BE; # (댗; 댗; 댗; 댗; 댗; ) HANGUL SYLLABLE DAEC +B318;B318;1103 1162 11BF;B318;1103 1162 11BF; # (댘; 댘; 댘; 댘; 댘; ) HANGUL SYLLABLE DAEK +B319;B319;1103 1162 11C0;B319;1103 1162 11C0; # (댙; 댙; 댙; 댙; 댙; ) HANGUL SYLLABLE DAET +B31A;B31A;1103 1162 11C1;B31A;1103 1162 11C1; # (댚; 댚; 댚; 댚; 댚; ) HANGUL SYLLABLE DAEP +B31B;B31B;1103 1162 11C2;B31B;1103 1162 11C2; # (댛; 댛; 댛; 댛; 댛; ) HANGUL SYLLABLE DAEH +B31C;B31C;1103 1163;B31C;1103 1163; # (댜; 댜; 댜; 댜; 댜; ) HANGUL SYLLABLE DYA +B31D;B31D;1103 1163 11A8;B31D;1103 1163 11A8; # (댝; 댝; 댝; 댝; 댝; ) HANGUL SYLLABLE DYAG +B31E;B31E;1103 1163 11A9;B31E;1103 1163 11A9; # (댞; 댞; 댞; 댞; 댞; ) HANGUL SYLLABLE DYAGG +B31F;B31F;1103 1163 11AA;B31F;1103 1163 11AA; # (댟; 댟; 댟; 댟; 댟; ) HANGUL SYLLABLE DYAGS +B320;B320;1103 1163 11AB;B320;1103 1163 11AB; # (댠; 댠; 댠; 댠; 댠; ) HANGUL SYLLABLE DYAN +B321;B321;1103 1163 11AC;B321;1103 1163 11AC; # (댡; 댡; 댡; 댡; 댡; ) HANGUL SYLLABLE DYANJ +B322;B322;1103 1163 11AD;B322;1103 1163 11AD; # (댢; 댢; 댢; 댢; 댢; ) HANGUL SYLLABLE DYANH +B323;B323;1103 1163 11AE;B323;1103 1163 11AE; # (댣; 댣; 댣; 댣; 댣; ) HANGUL SYLLABLE DYAD +B324;B324;1103 1163 11AF;B324;1103 1163 11AF; # (댤; 댤; 댤; 댤; 댤; ) HANGUL SYLLABLE DYAL +B325;B325;1103 1163 11B0;B325;1103 1163 11B0; # (댥; 댥; 댥; 댥; 댥; ) HANGUL SYLLABLE DYALG +B326;B326;1103 1163 11B1;B326;1103 1163 11B1; # (댦; 댦; 댦; 댦; 댦; ) HANGUL SYLLABLE DYALM +B327;B327;1103 1163 11B2;B327;1103 1163 11B2; # (댧; 댧; 댧; 댧; 댧; ) HANGUL SYLLABLE DYALB +B328;B328;1103 1163 11B3;B328;1103 1163 11B3; # (댨; 댨; 댨; 댨; 댨; ) HANGUL SYLLABLE DYALS +B329;B329;1103 1163 11B4;B329;1103 1163 11B4; # (댩; 댩; 댩; 댩; 댩; ) HANGUL SYLLABLE DYALT +B32A;B32A;1103 1163 11B5;B32A;1103 1163 11B5; # (댪; 댪; 댪; 댪; 댪; ) HANGUL SYLLABLE DYALP +B32B;B32B;1103 1163 11B6;B32B;1103 1163 11B6; # (댫; 댫; 댫; 댫; 댫; ) HANGUL SYLLABLE DYALH +B32C;B32C;1103 1163 11B7;B32C;1103 1163 11B7; # (댬; 댬; 댬; 댬; 댬; ) HANGUL SYLLABLE DYAM +B32D;B32D;1103 1163 11B8;B32D;1103 1163 11B8; # (댭; 댭; 댭; 댭; 댭; ) HANGUL SYLLABLE DYAB +B32E;B32E;1103 1163 11B9;B32E;1103 1163 11B9; # (댮; 댮; 댮; 댮; 댮; ) HANGUL SYLLABLE DYABS +B32F;B32F;1103 1163 11BA;B32F;1103 1163 11BA; # (댯; 댯; 댯; 댯; 댯; ) HANGUL SYLLABLE DYAS +B330;B330;1103 1163 11BB;B330;1103 1163 11BB; # (댰; 댰; 댰; 댰; 댰; ) HANGUL SYLLABLE DYASS +B331;B331;1103 1163 11BC;B331;1103 1163 11BC; # (댱; 댱; 댱; 댱; 댱; ) HANGUL SYLLABLE DYANG +B332;B332;1103 1163 11BD;B332;1103 1163 11BD; # (댲; 댲; 댲; 댲; 댲; ) HANGUL SYLLABLE DYAJ +B333;B333;1103 1163 11BE;B333;1103 1163 11BE; # (댳; 댳; 댳; 댳; 댳; ) HANGUL SYLLABLE DYAC +B334;B334;1103 1163 11BF;B334;1103 1163 11BF; # (댴; 댴; 댴; 댴; 댴; ) HANGUL SYLLABLE DYAK +B335;B335;1103 1163 11C0;B335;1103 1163 11C0; # (댵; 댵; 댵; 댵; 댵; ) HANGUL SYLLABLE DYAT +B336;B336;1103 1163 11C1;B336;1103 1163 11C1; # (댶; 댶; 댶; 댶; 댶; ) HANGUL SYLLABLE DYAP +B337;B337;1103 1163 11C2;B337;1103 1163 11C2; # (댷; 댷; 댷; 댷; 댷; ) HANGUL SYLLABLE DYAH +B338;B338;1103 1164;B338;1103 1164; # (댸; 댸; 댸; 댸; 댸; ) HANGUL SYLLABLE DYAE +B339;B339;1103 1164 11A8;B339;1103 1164 11A8; # (댹; 댹; 댹; 댹; 댹; ) HANGUL SYLLABLE DYAEG +B33A;B33A;1103 1164 11A9;B33A;1103 1164 11A9; # (댺; 댺; 댺; 댺; 댺; ) HANGUL SYLLABLE DYAEGG +B33B;B33B;1103 1164 11AA;B33B;1103 1164 11AA; # (댻; 댻; 댻; 댻; 댻; ) HANGUL SYLLABLE DYAEGS +B33C;B33C;1103 1164 11AB;B33C;1103 1164 11AB; # (댼; 댼; 댼; 댼; 댼; ) HANGUL SYLLABLE DYAEN +B33D;B33D;1103 1164 11AC;B33D;1103 1164 11AC; # (댽; 댽; 댽; 댽; 댽; ) HANGUL SYLLABLE DYAENJ +B33E;B33E;1103 1164 11AD;B33E;1103 1164 11AD; # (댾; 댾; 댾; 댾; 댾; ) HANGUL SYLLABLE DYAENH +B33F;B33F;1103 1164 11AE;B33F;1103 1164 11AE; # (댿; 댿; 댿; 댿; 댿; ) HANGUL SYLLABLE DYAED +B340;B340;1103 1164 11AF;B340;1103 1164 11AF; # (덀; 덀; 덀; 덀; 덀; ) HANGUL SYLLABLE DYAEL +B341;B341;1103 1164 11B0;B341;1103 1164 11B0; # (덁; 덁; 덁; 덁; 덁; ) HANGUL SYLLABLE DYAELG +B342;B342;1103 1164 11B1;B342;1103 1164 11B1; # (덂; 덂; 덂; 덂; 덂; ) HANGUL SYLLABLE DYAELM +B343;B343;1103 1164 11B2;B343;1103 1164 11B2; # (덃; 덃; 덃; 덃; 덃; ) HANGUL SYLLABLE DYAELB +B344;B344;1103 1164 11B3;B344;1103 1164 11B3; # (덄; 덄; 덄; 덄; 덄; ) HANGUL SYLLABLE DYAELS +B345;B345;1103 1164 11B4;B345;1103 1164 11B4; # (덅; 덅; 덅; 덅; 덅; ) HANGUL SYLLABLE DYAELT +B346;B346;1103 1164 11B5;B346;1103 1164 11B5; # (덆; 덆; 덆; 덆; 덆; ) HANGUL SYLLABLE DYAELP +B347;B347;1103 1164 11B6;B347;1103 1164 11B6; # (덇; 덇; 덇; 덇; 덇; ) HANGUL SYLLABLE DYAELH +B348;B348;1103 1164 11B7;B348;1103 1164 11B7; # (덈; 덈; 덈; 덈; 덈; ) HANGUL SYLLABLE DYAEM +B349;B349;1103 1164 11B8;B349;1103 1164 11B8; # (덉; 덉; 덉; 덉; 덉; ) HANGUL SYLLABLE DYAEB +B34A;B34A;1103 1164 11B9;B34A;1103 1164 11B9; # (덊; 덊; 덊; 덊; 덊; ) HANGUL SYLLABLE DYAEBS +B34B;B34B;1103 1164 11BA;B34B;1103 1164 11BA; # (덋; 덋; 덋; 덋; 덋; ) HANGUL SYLLABLE DYAES +B34C;B34C;1103 1164 11BB;B34C;1103 1164 11BB; # (덌; 덌; 덌; 덌; 덌; ) HANGUL SYLLABLE DYAESS +B34D;B34D;1103 1164 11BC;B34D;1103 1164 11BC; # (덍; 덍; 덍; 덍; 덍; ) HANGUL SYLLABLE DYAENG +B34E;B34E;1103 1164 11BD;B34E;1103 1164 11BD; # (덎; 덎; 덎; 덎; 덎; ) HANGUL SYLLABLE DYAEJ +B34F;B34F;1103 1164 11BE;B34F;1103 1164 11BE; # (덏; 덏; 덏; 덏; 덏; ) HANGUL SYLLABLE DYAEC +B350;B350;1103 1164 11BF;B350;1103 1164 11BF; # (덐; 덐; 덐; 덐; 덐; ) HANGUL SYLLABLE DYAEK +B351;B351;1103 1164 11C0;B351;1103 1164 11C0; # (덑; 덑; 덑; 덑; 덑; ) HANGUL SYLLABLE DYAET +B352;B352;1103 1164 11C1;B352;1103 1164 11C1; # (덒; 덒; 덒; 덒; 덒; ) HANGUL SYLLABLE DYAEP +B353;B353;1103 1164 11C2;B353;1103 1164 11C2; # (덓; 덓; 덓; 덓; 덓; ) HANGUL SYLLABLE DYAEH +B354;B354;1103 1165;B354;1103 1165; # (더; 더; 더; 더; 더; ) HANGUL SYLLABLE DEO +B355;B355;1103 1165 11A8;B355;1103 1165 11A8; # (덕; 덕; 덕; 덕; 덕; ) HANGUL SYLLABLE DEOG +B356;B356;1103 1165 11A9;B356;1103 1165 11A9; # (덖; 덖; 덖; 덖; 덖; ) HANGUL SYLLABLE DEOGG +B357;B357;1103 1165 11AA;B357;1103 1165 11AA; # (덗; 덗; 덗; 덗; 덗; ) HANGUL SYLLABLE DEOGS +B358;B358;1103 1165 11AB;B358;1103 1165 11AB; # (던; 던; 던; 던; 던; ) HANGUL SYLLABLE DEON +B359;B359;1103 1165 11AC;B359;1103 1165 11AC; # (덙; 덙; 덙; 덙; 덙; ) HANGUL SYLLABLE DEONJ +B35A;B35A;1103 1165 11AD;B35A;1103 1165 11AD; # (덚; 덚; 덚; 덚; 덚; ) HANGUL SYLLABLE DEONH +B35B;B35B;1103 1165 11AE;B35B;1103 1165 11AE; # (덛; 덛; 덛; 덛; 덛; ) HANGUL SYLLABLE DEOD +B35C;B35C;1103 1165 11AF;B35C;1103 1165 11AF; # (덜; 덜; 덜; 덜; 덜; ) HANGUL SYLLABLE DEOL +B35D;B35D;1103 1165 11B0;B35D;1103 1165 11B0; # (덝; 덝; 덝; 덝; 덝; ) HANGUL SYLLABLE DEOLG +B35E;B35E;1103 1165 11B1;B35E;1103 1165 11B1; # (덞; 덞; 덞; 덞; 덞; ) HANGUL SYLLABLE DEOLM +B35F;B35F;1103 1165 11B2;B35F;1103 1165 11B2; # (덟; 덟; 덟; 덟; 덟; ) HANGUL SYLLABLE DEOLB +B360;B360;1103 1165 11B3;B360;1103 1165 11B3; # (덠; 덠; 덠; 덠; 덠; ) HANGUL SYLLABLE DEOLS +B361;B361;1103 1165 11B4;B361;1103 1165 11B4; # (덡; 덡; 덡; 덡; 덡; ) HANGUL SYLLABLE DEOLT +B362;B362;1103 1165 11B5;B362;1103 1165 11B5; # (덢; 덢; 덢; 덢; 덢; ) HANGUL SYLLABLE DEOLP +B363;B363;1103 1165 11B6;B363;1103 1165 11B6; # (덣; 덣; 덣; 덣; 덣; ) HANGUL SYLLABLE DEOLH +B364;B364;1103 1165 11B7;B364;1103 1165 11B7; # (덤; 덤; 덤; 덤; 덤; ) HANGUL SYLLABLE DEOM +B365;B365;1103 1165 11B8;B365;1103 1165 11B8; # (덥; 덥; 덥; 덥; 덥; ) HANGUL SYLLABLE DEOB +B366;B366;1103 1165 11B9;B366;1103 1165 11B9; # (덦; 덦; 덦; 덦; 덦; ) HANGUL SYLLABLE DEOBS +B367;B367;1103 1165 11BA;B367;1103 1165 11BA; # (덧; 덧; 덧; 덧; 덧; ) HANGUL SYLLABLE DEOS +B368;B368;1103 1165 11BB;B368;1103 1165 11BB; # (덨; 덨; 덨; 덨; 덨; ) HANGUL SYLLABLE DEOSS +B369;B369;1103 1165 11BC;B369;1103 1165 11BC; # (덩; 덩; 덩; 덩; 덩; ) HANGUL SYLLABLE DEONG +B36A;B36A;1103 1165 11BD;B36A;1103 1165 11BD; # (덪; 덪; 덪; 덪; 덪; ) HANGUL SYLLABLE DEOJ +B36B;B36B;1103 1165 11BE;B36B;1103 1165 11BE; # (덫; 덫; 덫; 덫; 덫; ) HANGUL SYLLABLE DEOC +B36C;B36C;1103 1165 11BF;B36C;1103 1165 11BF; # (덬; 덬; 덬; 덬; 덬; ) HANGUL SYLLABLE DEOK +B36D;B36D;1103 1165 11C0;B36D;1103 1165 11C0; # (덭; 덭; 덭; 덭; 덭; ) HANGUL SYLLABLE DEOT +B36E;B36E;1103 1165 11C1;B36E;1103 1165 11C1; # (덮; 덮; 덮; 덮; 덮; ) HANGUL SYLLABLE DEOP +B36F;B36F;1103 1165 11C2;B36F;1103 1165 11C2; # (덯; 덯; 덯; 덯; 덯; ) HANGUL SYLLABLE DEOH +B370;B370;1103 1166;B370;1103 1166; # (데; 데; 데; 데; 데; ) HANGUL SYLLABLE DE +B371;B371;1103 1166 11A8;B371;1103 1166 11A8; # (덱; 덱; 덱; 덱; 덱; ) HANGUL SYLLABLE DEG +B372;B372;1103 1166 11A9;B372;1103 1166 11A9; # (덲; 덲; 덲; 덲; 덲; ) HANGUL SYLLABLE DEGG +B373;B373;1103 1166 11AA;B373;1103 1166 11AA; # (덳; 덳; 덳; 덳; 덳; ) HANGUL SYLLABLE DEGS +B374;B374;1103 1166 11AB;B374;1103 1166 11AB; # (덴; 덴; 덴; 덴; 덴; ) HANGUL SYLLABLE DEN +B375;B375;1103 1166 11AC;B375;1103 1166 11AC; # (덵; 덵; 덵; 덵; 덵; ) HANGUL SYLLABLE DENJ +B376;B376;1103 1166 11AD;B376;1103 1166 11AD; # (덶; 덶; 덶; 덶; 덶; ) HANGUL SYLLABLE DENH +B377;B377;1103 1166 11AE;B377;1103 1166 11AE; # (덷; 덷; 덷; 덷; 덷; ) HANGUL SYLLABLE DED +B378;B378;1103 1166 11AF;B378;1103 1166 11AF; # (델; 델; 델; 델; 델; ) HANGUL SYLLABLE DEL +B379;B379;1103 1166 11B0;B379;1103 1166 11B0; # (덹; 덹; 덹; 덹; 덹; ) HANGUL SYLLABLE DELG +B37A;B37A;1103 1166 11B1;B37A;1103 1166 11B1; # (덺; 덺; 덺; 덺; 덺; ) HANGUL SYLLABLE DELM +B37B;B37B;1103 1166 11B2;B37B;1103 1166 11B2; # (덻; 덻; 덻; 덻; 덻; ) HANGUL SYLLABLE DELB +B37C;B37C;1103 1166 11B3;B37C;1103 1166 11B3; # (덼; 덼; 덼; 덼; 덼; ) HANGUL SYLLABLE DELS +B37D;B37D;1103 1166 11B4;B37D;1103 1166 11B4; # (덽; 덽; 덽; 덽; 덽; ) HANGUL SYLLABLE DELT +B37E;B37E;1103 1166 11B5;B37E;1103 1166 11B5; # (덾; 덾; 덾; 덾; 덾; ) HANGUL SYLLABLE DELP +B37F;B37F;1103 1166 11B6;B37F;1103 1166 11B6; # (덿; 덿; 덿; 덿; 덿; ) HANGUL SYLLABLE DELH +B380;B380;1103 1166 11B7;B380;1103 1166 11B7; # (뎀; 뎀; 뎀; 뎀; 뎀; ) HANGUL SYLLABLE DEM +B381;B381;1103 1166 11B8;B381;1103 1166 11B8; # (뎁; 뎁; 뎁; 뎁; 뎁; ) HANGUL SYLLABLE DEB +B382;B382;1103 1166 11B9;B382;1103 1166 11B9; # (뎂; 뎂; 뎂; 뎂; 뎂; ) HANGUL SYLLABLE DEBS +B383;B383;1103 1166 11BA;B383;1103 1166 11BA; # (뎃; 뎃; 뎃; 뎃; 뎃; ) HANGUL SYLLABLE DES +B384;B384;1103 1166 11BB;B384;1103 1166 11BB; # (뎄; 뎄; 뎄; 뎄; 뎄; ) HANGUL SYLLABLE DESS +B385;B385;1103 1166 11BC;B385;1103 1166 11BC; # (뎅; 뎅; 뎅; 뎅; 뎅; ) HANGUL SYLLABLE DENG +B386;B386;1103 1166 11BD;B386;1103 1166 11BD; # (뎆; 뎆; 뎆; 뎆; 뎆; ) HANGUL SYLLABLE DEJ +B387;B387;1103 1166 11BE;B387;1103 1166 11BE; # (뎇; 뎇; 뎇; 뎇; 뎇; ) HANGUL SYLLABLE DEC +B388;B388;1103 1166 11BF;B388;1103 1166 11BF; # (뎈; 뎈; 뎈; 뎈; 뎈; ) HANGUL SYLLABLE DEK +B389;B389;1103 1166 11C0;B389;1103 1166 11C0; # (뎉; 뎉; 뎉; 뎉; 뎉; ) HANGUL SYLLABLE DET +B38A;B38A;1103 1166 11C1;B38A;1103 1166 11C1; # (뎊; 뎊; 뎊; 뎊; 뎊; ) HANGUL SYLLABLE DEP +B38B;B38B;1103 1166 11C2;B38B;1103 1166 11C2; # (뎋; 뎋; 뎋; 뎋; 뎋; ) HANGUL SYLLABLE DEH +B38C;B38C;1103 1167;B38C;1103 1167; # (뎌; 뎌; 뎌; 뎌; 뎌; ) HANGUL SYLLABLE DYEO +B38D;B38D;1103 1167 11A8;B38D;1103 1167 11A8; # (뎍; 뎍; 뎍; 뎍; 뎍; ) HANGUL SYLLABLE DYEOG +B38E;B38E;1103 1167 11A9;B38E;1103 1167 11A9; # (뎎; 뎎; 뎎; 뎎; 뎎; ) HANGUL SYLLABLE DYEOGG +B38F;B38F;1103 1167 11AA;B38F;1103 1167 11AA; # (뎏; 뎏; 뎏; 뎏; 뎏; ) HANGUL SYLLABLE DYEOGS +B390;B390;1103 1167 11AB;B390;1103 1167 11AB; # (뎐; 뎐; 뎐; 뎐; 뎐; ) HANGUL SYLLABLE DYEON +B391;B391;1103 1167 11AC;B391;1103 1167 11AC; # (뎑; 뎑; 뎑; 뎑; 뎑; ) HANGUL SYLLABLE DYEONJ +B392;B392;1103 1167 11AD;B392;1103 1167 11AD; # (뎒; 뎒; 뎒; 뎒; 뎒; ) HANGUL SYLLABLE DYEONH +B393;B393;1103 1167 11AE;B393;1103 1167 11AE; # (뎓; 뎓; 뎓; 뎓; 뎓; ) HANGUL SYLLABLE DYEOD +B394;B394;1103 1167 11AF;B394;1103 1167 11AF; # (뎔; 뎔; 뎔; 뎔; 뎔; ) HANGUL SYLLABLE DYEOL +B395;B395;1103 1167 11B0;B395;1103 1167 11B0; # (뎕; 뎕; 뎕; 뎕; 뎕; ) HANGUL SYLLABLE DYEOLG +B396;B396;1103 1167 11B1;B396;1103 1167 11B1; # (뎖; 뎖; 뎖; 뎖; 뎖; ) HANGUL SYLLABLE DYEOLM +B397;B397;1103 1167 11B2;B397;1103 1167 11B2; # (뎗; 뎗; 뎗; 뎗; 뎗; ) HANGUL SYLLABLE DYEOLB +B398;B398;1103 1167 11B3;B398;1103 1167 11B3; # (뎘; 뎘; 뎘; 뎘; 뎘; ) HANGUL SYLLABLE DYEOLS +B399;B399;1103 1167 11B4;B399;1103 1167 11B4; # (뎙; 뎙; 뎙; 뎙; 뎙; ) HANGUL SYLLABLE DYEOLT +B39A;B39A;1103 1167 11B5;B39A;1103 1167 11B5; # (뎚; 뎚; 뎚; 뎚; 뎚; ) HANGUL SYLLABLE DYEOLP +B39B;B39B;1103 1167 11B6;B39B;1103 1167 11B6; # (뎛; 뎛; 뎛; 뎛; 뎛; ) HANGUL SYLLABLE DYEOLH +B39C;B39C;1103 1167 11B7;B39C;1103 1167 11B7; # (뎜; 뎜; 뎜; 뎜; 뎜; ) HANGUL SYLLABLE DYEOM +B39D;B39D;1103 1167 11B8;B39D;1103 1167 11B8; # (뎝; 뎝; 뎝; 뎝; 뎝; ) HANGUL SYLLABLE DYEOB +B39E;B39E;1103 1167 11B9;B39E;1103 1167 11B9; # (뎞; 뎞; 뎞; 뎞; 뎞; ) HANGUL SYLLABLE DYEOBS +B39F;B39F;1103 1167 11BA;B39F;1103 1167 11BA; # (뎟; 뎟; 뎟; 뎟; 뎟; ) HANGUL SYLLABLE DYEOS +B3A0;B3A0;1103 1167 11BB;B3A0;1103 1167 11BB; # (뎠; 뎠; 뎠; 뎠; 뎠; ) HANGUL SYLLABLE DYEOSS +B3A1;B3A1;1103 1167 11BC;B3A1;1103 1167 11BC; # (뎡; 뎡; 뎡; 뎡; 뎡; ) HANGUL SYLLABLE DYEONG +B3A2;B3A2;1103 1167 11BD;B3A2;1103 1167 11BD; # (뎢; 뎢; 뎢; 뎢; 뎢; ) HANGUL SYLLABLE DYEOJ +B3A3;B3A3;1103 1167 11BE;B3A3;1103 1167 11BE; # (뎣; 뎣; 뎣; 뎣; 뎣; ) HANGUL SYLLABLE DYEOC +B3A4;B3A4;1103 1167 11BF;B3A4;1103 1167 11BF; # (뎤; 뎤; 뎤; 뎤; 뎤; ) HANGUL SYLLABLE DYEOK +B3A5;B3A5;1103 1167 11C0;B3A5;1103 1167 11C0; # (뎥; 뎥; 뎥; 뎥; 뎥; ) HANGUL SYLLABLE DYEOT +B3A6;B3A6;1103 1167 11C1;B3A6;1103 1167 11C1; # (뎦; 뎦; 뎦; 뎦; 뎦; ) HANGUL SYLLABLE DYEOP +B3A7;B3A7;1103 1167 11C2;B3A7;1103 1167 11C2; # (뎧; 뎧; 뎧; 뎧; 뎧; ) HANGUL SYLLABLE DYEOH +B3A8;B3A8;1103 1168;B3A8;1103 1168; # (뎨; 뎨; 뎨; 뎨; 뎨; ) HANGUL SYLLABLE DYE +B3A9;B3A9;1103 1168 11A8;B3A9;1103 1168 11A8; # (뎩; 뎩; 뎩; 뎩; 뎩; ) HANGUL SYLLABLE DYEG +B3AA;B3AA;1103 1168 11A9;B3AA;1103 1168 11A9; # (뎪; 뎪; 뎪; 뎪; 뎪; ) HANGUL SYLLABLE DYEGG +B3AB;B3AB;1103 1168 11AA;B3AB;1103 1168 11AA; # (뎫; 뎫; 뎫; 뎫; 뎫; ) HANGUL SYLLABLE DYEGS +B3AC;B3AC;1103 1168 11AB;B3AC;1103 1168 11AB; # (뎬; 뎬; 뎬; 뎬; 뎬; ) HANGUL SYLLABLE DYEN +B3AD;B3AD;1103 1168 11AC;B3AD;1103 1168 11AC; # (뎭; 뎭; 뎭; 뎭; 뎭; ) HANGUL SYLLABLE DYENJ +B3AE;B3AE;1103 1168 11AD;B3AE;1103 1168 11AD; # (뎮; 뎮; 뎮; 뎮; 뎮; ) HANGUL SYLLABLE DYENH +B3AF;B3AF;1103 1168 11AE;B3AF;1103 1168 11AE; # (뎯; 뎯; 뎯; 뎯; 뎯; ) HANGUL SYLLABLE DYED +B3B0;B3B0;1103 1168 11AF;B3B0;1103 1168 11AF; # (뎰; 뎰; 뎰; 뎰; 뎰; ) HANGUL SYLLABLE DYEL +B3B1;B3B1;1103 1168 11B0;B3B1;1103 1168 11B0; # (뎱; 뎱; 뎱; 뎱; 뎱; ) HANGUL SYLLABLE DYELG +B3B2;B3B2;1103 1168 11B1;B3B2;1103 1168 11B1; # (뎲; 뎲; 뎲; 뎲; 뎲; ) HANGUL SYLLABLE DYELM +B3B3;B3B3;1103 1168 11B2;B3B3;1103 1168 11B2; # (뎳; 뎳; 뎳; 뎳; 뎳; ) HANGUL SYLLABLE DYELB +B3B4;B3B4;1103 1168 11B3;B3B4;1103 1168 11B3; # (뎴; 뎴; 뎴; 뎴; 뎴; ) HANGUL SYLLABLE DYELS +B3B5;B3B5;1103 1168 11B4;B3B5;1103 1168 11B4; # (뎵; 뎵; 뎵; 뎵; 뎵; ) HANGUL SYLLABLE DYELT +B3B6;B3B6;1103 1168 11B5;B3B6;1103 1168 11B5; # (뎶; 뎶; 뎶; 뎶; 뎶; ) HANGUL SYLLABLE DYELP +B3B7;B3B7;1103 1168 11B6;B3B7;1103 1168 11B6; # (뎷; 뎷; 뎷; 뎷; 뎷; ) HANGUL SYLLABLE DYELH +B3B8;B3B8;1103 1168 11B7;B3B8;1103 1168 11B7; # (뎸; 뎸; 뎸; 뎸; 뎸; ) HANGUL SYLLABLE DYEM +B3B9;B3B9;1103 1168 11B8;B3B9;1103 1168 11B8; # (뎹; 뎹; 뎹; 뎹; 뎹; ) HANGUL SYLLABLE DYEB +B3BA;B3BA;1103 1168 11B9;B3BA;1103 1168 11B9; # (뎺; 뎺; 뎺; 뎺; 뎺; ) HANGUL SYLLABLE DYEBS +B3BB;B3BB;1103 1168 11BA;B3BB;1103 1168 11BA; # (뎻; 뎻; 뎻; 뎻; 뎻; ) HANGUL SYLLABLE DYES +B3BC;B3BC;1103 1168 11BB;B3BC;1103 1168 11BB; # (뎼; 뎼; 뎼; 뎼; 뎼; ) HANGUL SYLLABLE DYESS +B3BD;B3BD;1103 1168 11BC;B3BD;1103 1168 11BC; # (뎽; 뎽; 뎽; 뎽; 뎽; ) HANGUL SYLLABLE DYENG +B3BE;B3BE;1103 1168 11BD;B3BE;1103 1168 11BD; # (뎾; 뎾; 뎾; 뎾; 뎾; ) HANGUL SYLLABLE DYEJ +B3BF;B3BF;1103 1168 11BE;B3BF;1103 1168 11BE; # (뎿; 뎿; 뎿; 뎿; 뎿; ) HANGUL SYLLABLE DYEC +B3C0;B3C0;1103 1168 11BF;B3C0;1103 1168 11BF; # (돀; 돀; 돀; 돀; 돀; ) HANGUL SYLLABLE DYEK +B3C1;B3C1;1103 1168 11C0;B3C1;1103 1168 11C0; # (돁; 돁; 돁; 돁; 돁; ) HANGUL SYLLABLE DYET +B3C2;B3C2;1103 1168 11C1;B3C2;1103 1168 11C1; # (돂; 돂; 돂; 돂; 돂; ) HANGUL SYLLABLE DYEP +B3C3;B3C3;1103 1168 11C2;B3C3;1103 1168 11C2; # (돃; 돃; 돃; 돃; 돃; ) HANGUL SYLLABLE DYEH +B3C4;B3C4;1103 1169;B3C4;1103 1169; # (도; 도; 도; 도; 도; ) HANGUL SYLLABLE DO +B3C5;B3C5;1103 1169 11A8;B3C5;1103 1169 11A8; # (독; 독; 독; 독; 독; ) HANGUL SYLLABLE DOG +B3C6;B3C6;1103 1169 11A9;B3C6;1103 1169 11A9; # (돆; 돆; 돆; 돆; 돆; ) HANGUL SYLLABLE DOGG +B3C7;B3C7;1103 1169 11AA;B3C7;1103 1169 11AA; # (돇; 돇; 돇; 돇; 돇; ) HANGUL SYLLABLE DOGS +B3C8;B3C8;1103 1169 11AB;B3C8;1103 1169 11AB; # (돈; 돈; 돈; 돈; 돈; ) HANGUL SYLLABLE DON +B3C9;B3C9;1103 1169 11AC;B3C9;1103 1169 11AC; # (돉; 돉; 돉; 돉; 돉; ) HANGUL SYLLABLE DONJ +B3CA;B3CA;1103 1169 11AD;B3CA;1103 1169 11AD; # (돊; 돊; 돊; 돊; 돊; ) HANGUL SYLLABLE DONH +B3CB;B3CB;1103 1169 11AE;B3CB;1103 1169 11AE; # (돋; 돋; 돋; 돋; 돋; ) HANGUL SYLLABLE DOD +B3CC;B3CC;1103 1169 11AF;B3CC;1103 1169 11AF; # (돌; 돌; 돌; 돌; 돌; ) HANGUL SYLLABLE DOL +B3CD;B3CD;1103 1169 11B0;B3CD;1103 1169 11B0; # (돍; 돍; 돍; 돍; 돍; ) HANGUL SYLLABLE DOLG +B3CE;B3CE;1103 1169 11B1;B3CE;1103 1169 11B1; # (돎; 돎; 돎; 돎; 돎; ) HANGUL SYLLABLE DOLM +B3CF;B3CF;1103 1169 11B2;B3CF;1103 1169 11B2; # (돏; 돏; 돏; 돏; 돏; ) HANGUL SYLLABLE DOLB +B3D0;B3D0;1103 1169 11B3;B3D0;1103 1169 11B3; # (돐; 돐; 돐; 돐; 돐; ) HANGUL SYLLABLE DOLS +B3D1;B3D1;1103 1169 11B4;B3D1;1103 1169 11B4; # (돑; 돑; 돑; 돑; 돑; ) HANGUL SYLLABLE DOLT +B3D2;B3D2;1103 1169 11B5;B3D2;1103 1169 11B5; # (돒; 돒; 돒; 돒; 돒; ) HANGUL SYLLABLE DOLP +B3D3;B3D3;1103 1169 11B6;B3D3;1103 1169 11B6; # (돓; 돓; 돓; 돓; 돓; ) HANGUL SYLLABLE DOLH +B3D4;B3D4;1103 1169 11B7;B3D4;1103 1169 11B7; # (돔; 돔; 돔; 돔; 돔; ) HANGUL SYLLABLE DOM +B3D5;B3D5;1103 1169 11B8;B3D5;1103 1169 11B8; # (돕; 돕; 돕; 돕; 돕; ) HANGUL SYLLABLE DOB +B3D6;B3D6;1103 1169 11B9;B3D6;1103 1169 11B9; # (돖; 돖; 돖; 돖; 돖; ) HANGUL SYLLABLE DOBS +B3D7;B3D7;1103 1169 11BA;B3D7;1103 1169 11BA; # (돗; 돗; 돗; 돗; 돗; ) HANGUL SYLLABLE DOS +B3D8;B3D8;1103 1169 11BB;B3D8;1103 1169 11BB; # (돘; 돘; 돘; 돘; 돘; ) HANGUL SYLLABLE DOSS +B3D9;B3D9;1103 1169 11BC;B3D9;1103 1169 11BC; # (동; 동; 동; 동; 동; ) HANGUL SYLLABLE DONG +B3DA;B3DA;1103 1169 11BD;B3DA;1103 1169 11BD; # (돚; 돚; 돚; 돚; 돚; ) HANGUL SYLLABLE DOJ +B3DB;B3DB;1103 1169 11BE;B3DB;1103 1169 11BE; # (돛; 돛; 돛; 돛; 돛; ) HANGUL SYLLABLE DOC +B3DC;B3DC;1103 1169 11BF;B3DC;1103 1169 11BF; # (돜; 돜; 돜; 돜; 돜; ) HANGUL SYLLABLE DOK +B3DD;B3DD;1103 1169 11C0;B3DD;1103 1169 11C0; # (돝; 돝; 돝; 돝; 돝; ) HANGUL SYLLABLE DOT +B3DE;B3DE;1103 1169 11C1;B3DE;1103 1169 11C1; # (돞; 돞; 돞; 돞; 돞; ) HANGUL SYLLABLE DOP +B3DF;B3DF;1103 1169 11C2;B3DF;1103 1169 11C2; # (돟; 돟; 돟; 돟; 돟; ) HANGUL SYLLABLE DOH +B3E0;B3E0;1103 116A;B3E0;1103 116A; # (돠; 돠; 돠; 돠; 돠; ) HANGUL SYLLABLE DWA +B3E1;B3E1;1103 116A 11A8;B3E1;1103 116A 11A8; # (돡; 돡; 돡; 돡; 돡; ) HANGUL SYLLABLE DWAG +B3E2;B3E2;1103 116A 11A9;B3E2;1103 116A 11A9; # (돢; 돢; 돢; 돢; 돢; ) HANGUL SYLLABLE DWAGG +B3E3;B3E3;1103 116A 11AA;B3E3;1103 116A 11AA; # (돣; 돣; 돣; 돣; 돣; ) HANGUL SYLLABLE DWAGS +B3E4;B3E4;1103 116A 11AB;B3E4;1103 116A 11AB; # (돤; 돤; 돤; 돤; 돤; ) HANGUL SYLLABLE DWAN +B3E5;B3E5;1103 116A 11AC;B3E5;1103 116A 11AC; # (돥; 돥; 돥; 돥; 돥; ) HANGUL SYLLABLE DWANJ +B3E6;B3E6;1103 116A 11AD;B3E6;1103 116A 11AD; # (돦; 돦; 돦; 돦; 돦; ) HANGUL SYLLABLE DWANH +B3E7;B3E7;1103 116A 11AE;B3E7;1103 116A 11AE; # (돧; 돧; 돧; 돧; 돧; ) HANGUL SYLLABLE DWAD +B3E8;B3E8;1103 116A 11AF;B3E8;1103 116A 11AF; # (돨; 돨; 돨; 돨; 돨; ) HANGUL SYLLABLE DWAL +B3E9;B3E9;1103 116A 11B0;B3E9;1103 116A 11B0; # (돩; 돩; 돩; 돩; 돩; ) HANGUL SYLLABLE DWALG +B3EA;B3EA;1103 116A 11B1;B3EA;1103 116A 11B1; # (돪; 돪; 돪; 돪; 돪; ) HANGUL SYLLABLE DWALM +B3EB;B3EB;1103 116A 11B2;B3EB;1103 116A 11B2; # (돫; 돫; 돫; 돫; 돫; ) HANGUL SYLLABLE DWALB +B3EC;B3EC;1103 116A 11B3;B3EC;1103 116A 11B3; # (돬; 돬; 돬; 돬; 돬; ) HANGUL SYLLABLE DWALS +B3ED;B3ED;1103 116A 11B4;B3ED;1103 116A 11B4; # (돭; 돭; 돭; 돭; 돭; ) HANGUL SYLLABLE DWALT +B3EE;B3EE;1103 116A 11B5;B3EE;1103 116A 11B5; # (돮; 돮; 돮; 돮; 돮; ) HANGUL SYLLABLE DWALP +B3EF;B3EF;1103 116A 11B6;B3EF;1103 116A 11B6; # (돯; 돯; 돯; 돯; 돯; ) HANGUL SYLLABLE DWALH +B3F0;B3F0;1103 116A 11B7;B3F0;1103 116A 11B7; # (돰; 돰; 돰; 돰; 돰; ) HANGUL SYLLABLE DWAM +B3F1;B3F1;1103 116A 11B8;B3F1;1103 116A 11B8; # (돱; 돱; 돱; 돱; 돱; ) HANGUL SYLLABLE DWAB +B3F2;B3F2;1103 116A 11B9;B3F2;1103 116A 11B9; # (돲; 돲; 돲; 돲; 돲; ) HANGUL SYLLABLE DWABS +B3F3;B3F3;1103 116A 11BA;B3F3;1103 116A 11BA; # (돳; 돳; 돳; 돳; 돳; ) HANGUL SYLLABLE DWAS +B3F4;B3F4;1103 116A 11BB;B3F4;1103 116A 11BB; # (돴; 돴; 돴; 돴; 돴; ) HANGUL SYLLABLE DWASS +B3F5;B3F5;1103 116A 11BC;B3F5;1103 116A 11BC; # (돵; 돵; 돵; 돵; 돵; ) HANGUL SYLLABLE DWANG +B3F6;B3F6;1103 116A 11BD;B3F6;1103 116A 11BD; # (돶; 돶; 돶; 돶; 돶; ) HANGUL SYLLABLE DWAJ +B3F7;B3F7;1103 116A 11BE;B3F7;1103 116A 11BE; # (돷; 돷; 돷; 돷; 돷; ) HANGUL SYLLABLE DWAC +B3F8;B3F8;1103 116A 11BF;B3F8;1103 116A 11BF; # (돸; 돸; 돸; 돸; 돸; ) HANGUL SYLLABLE DWAK +B3F9;B3F9;1103 116A 11C0;B3F9;1103 116A 11C0; # (돹; 돹; 돹; 돹; 돹; ) HANGUL SYLLABLE DWAT +B3FA;B3FA;1103 116A 11C1;B3FA;1103 116A 11C1; # (돺; 돺; 돺; 돺; 돺; ) HANGUL SYLLABLE DWAP +B3FB;B3FB;1103 116A 11C2;B3FB;1103 116A 11C2; # (돻; 돻; 돻; 돻; 돻; ) HANGUL SYLLABLE DWAH +B3FC;B3FC;1103 116B;B3FC;1103 116B; # (돼; 돼; 돼; 돼; 돼; ) HANGUL SYLLABLE DWAE +B3FD;B3FD;1103 116B 11A8;B3FD;1103 116B 11A8; # (돽; 돽; 돽; 돽; 돽; ) HANGUL SYLLABLE DWAEG +B3FE;B3FE;1103 116B 11A9;B3FE;1103 116B 11A9; # (돾; 돾; 돾; 돾; 돾; ) HANGUL SYLLABLE DWAEGG +B3FF;B3FF;1103 116B 11AA;B3FF;1103 116B 11AA; # (돿; 돿; 돿; 돿; 돿; ) HANGUL SYLLABLE DWAEGS +B400;B400;1103 116B 11AB;B400;1103 116B 11AB; # (됀; 됀; 됀; 됀; 됀; ) HANGUL SYLLABLE DWAEN +B401;B401;1103 116B 11AC;B401;1103 116B 11AC; # (됁; 됁; 됁; 됁; 됁; ) HANGUL SYLLABLE DWAENJ +B402;B402;1103 116B 11AD;B402;1103 116B 11AD; # (됂; 됂; 됂; 됂; 됂; ) HANGUL SYLLABLE DWAENH +B403;B403;1103 116B 11AE;B403;1103 116B 11AE; # (됃; 됃; 됃; 됃; 됃; ) HANGUL SYLLABLE DWAED +B404;B404;1103 116B 11AF;B404;1103 116B 11AF; # (됄; 됄; 됄; 됄; 됄; ) HANGUL SYLLABLE DWAEL +B405;B405;1103 116B 11B0;B405;1103 116B 11B0; # (됅; 됅; 됅; 됅; 됅; ) HANGUL SYLLABLE DWAELG +B406;B406;1103 116B 11B1;B406;1103 116B 11B1; # (됆; 됆; 됆; 됆; 됆; ) HANGUL SYLLABLE DWAELM +B407;B407;1103 116B 11B2;B407;1103 116B 11B2; # (됇; 됇; 됇; 됇; 됇; ) HANGUL SYLLABLE DWAELB +B408;B408;1103 116B 11B3;B408;1103 116B 11B3; # (됈; 됈; 됈; 됈; 됈; ) HANGUL SYLLABLE DWAELS +B409;B409;1103 116B 11B4;B409;1103 116B 11B4; # (됉; 됉; 됉; 됉; 됉; ) HANGUL SYLLABLE DWAELT +B40A;B40A;1103 116B 11B5;B40A;1103 116B 11B5; # (됊; 됊; 됊; 됊; 됊; ) HANGUL SYLLABLE DWAELP +B40B;B40B;1103 116B 11B6;B40B;1103 116B 11B6; # (됋; 됋; 됋; 됋; 됋; ) HANGUL SYLLABLE DWAELH +B40C;B40C;1103 116B 11B7;B40C;1103 116B 11B7; # (됌; 됌; 됌; 됌; 됌; ) HANGUL SYLLABLE DWAEM +B40D;B40D;1103 116B 11B8;B40D;1103 116B 11B8; # (됍; 됍; 됍; 됍; 됍; ) HANGUL SYLLABLE DWAEB +B40E;B40E;1103 116B 11B9;B40E;1103 116B 11B9; # (됎; 됎; 됎; 됎; 됎; ) HANGUL SYLLABLE DWAEBS +B40F;B40F;1103 116B 11BA;B40F;1103 116B 11BA; # (됏; 됏; 됏; 됏; 됏; ) HANGUL SYLLABLE DWAES +B410;B410;1103 116B 11BB;B410;1103 116B 11BB; # (됐; 됐; 됐; 됐; 됐; ) HANGUL SYLLABLE DWAESS +B411;B411;1103 116B 11BC;B411;1103 116B 11BC; # (됑; 됑; 됑; 됑; 됑; ) HANGUL SYLLABLE DWAENG +B412;B412;1103 116B 11BD;B412;1103 116B 11BD; # (됒; 됒; 됒; 됒; 됒; ) HANGUL SYLLABLE DWAEJ +B413;B413;1103 116B 11BE;B413;1103 116B 11BE; # (됓; 됓; 됓; 됓; 됓; ) HANGUL SYLLABLE DWAEC +B414;B414;1103 116B 11BF;B414;1103 116B 11BF; # (됔; 됔; 됔; 됔; 됔; ) HANGUL SYLLABLE DWAEK +B415;B415;1103 116B 11C0;B415;1103 116B 11C0; # (됕; 됕; 됕; 됕; 됕; ) HANGUL SYLLABLE DWAET +B416;B416;1103 116B 11C1;B416;1103 116B 11C1; # (됖; 됖; 됖; 됖; 됖; ) HANGUL SYLLABLE DWAEP +B417;B417;1103 116B 11C2;B417;1103 116B 11C2; # (됗; 됗; 됗; 됗; 됗; ) HANGUL SYLLABLE DWAEH +B418;B418;1103 116C;B418;1103 116C; # (되; 되; 되; 되; 되; ) HANGUL SYLLABLE DOE +B419;B419;1103 116C 11A8;B419;1103 116C 11A8; # (됙; 됙; 됙; 됙; 됙; ) HANGUL SYLLABLE DOEG +B41A;B41A;1103 116C 11A9;B41A;1103 116C 11A9; # (됚; 됚; 됚; 됚; 됚; ) HANGUL SYLLABLE DOEGG +B41B;B41B;1103 116C 11AA;B41B;1103 116C 11AA; # (됛; 됛; 됛; 됛; 됛; ) HANGUL SYLLABLE DOEGS +B41C;B41C;1103 116C 11AB;B41C;1103 116C 11AB; # (된; 된; 된; 된; 된; ) HANGUL SYLLABLE DOEN +B41D;B41D;1103 116C 11AC;B41D;1103 116C 11AC; # (됝; 됝; 됝; 됝; 됝; ) HANGUL SYLLABLE DOENJ +B41E;B41E;1103 116C 11AD;B41E;1103 116C 11AD; # (됞; 됞; 됞; 됞; 됞; ) HANGUL SYLLABLE DOENH +B41F;B41F;1103 116C 11AE;B41F;1103 116C 11AE; # (됟; 됟; 됟; 됟; 됟; ) HANGUL SYLLABLE DOED +B420;B420;1103 116C 11AF;B420;1103 116C 11AF; # (될; 될; 될; 될; 될; ) HANGUL SYLLABLE DOEL +B421;B421;1103 116C 11B0;B421;1103 116C 11B0; # (됡; 됡; 됡; 됡; 됡; ) HANGUL SYLLABLE DOELG +B422;B422;1103 116C 11B1;B422;1103 116C 11B1; # (됢; 됢; 됢; 됢; 됢; ) HANGUL SYLLABLE DOELM +B423;B423;1103 116C 11B2;B423;1103 116C 11B2; # (됣; 됣; 됣; 됣; 됣; ) HANGUL SYLLABLE DOELB +B424;B424;1103 116C 11B3;B424;1103 116C 11B3; # (됤; 됤; 됤; 됤; 됤; ) HANGUL SYLLABLE DOELS +B425;B425;1103 116C 11B4;B425;1103 116C 11B4; # (됥; 됥; 됥; 됥; 됥; ) HANGUL SYLLABLE DOELT +B426;B426;1103 116C 11B5;B426;1103 116C 11B5; # (됦; 됦; 됦; 됦; 됦; ) HANGUL SYLLABLE DOELP +B427;B427;1103 116C 11B6;B427;1103 116C 11B6; # (됧; 됧; 됧; 됧; 됧; ) HANGUL SYLLABLE DOELH +B428;B428;1103 116C 11B7;B428;1103 116C 11B7; # (됨; 됨; 됨; 됨; 됨; ) HANGUL SYLLABLE DOEM +B429;B429;1103 116C 11B8;B429;1103 116C 11B8; # (됩; 됩; 됩; 됩; 됩; ) HANGUL SYLLABLE DOEB +B42A;B42A;1103 116C 11B9;B42A;1103 116C 11B9; # (됪; 됪; 됪; 됪; 됪; ) HANGUL SYLLABLE DOEBS +B42B;B42B;1103 116C 11BA;B42B;1103 116C 11BA; # (됫; 됫; 됫; 됫; 됫; ) HANGUL SYLLABLE DOES +B42C;B42C;1103 116C 11BB;B42C;1103 116C 11BB; # (됬; 됬; 됬; 됬; 됬; ) HANGUL SYLLABLE DOESS +B42D;B42D;1103 116C 11BC;B42D;1103 116C 11BC; # (됭; 됭; 됭; 됭; 됭; ) HANGUL SYLLABLE DOENG +B42E;B42E;1103 116C 11BD;B42E;1103 116C 11BD; # (됮; 됮; 됮; 됮; 됮; ) HANGUL SYLLABLE DOEJ +B42F;B42F;1103 116C 11BE;B42F;1103 116C 11BE; # (됯; 됯; 됯; 됯; 됯; ) HANGUL SYLLABLE DOEC +B430;B430;1103 116C 11BF;B430;1103 116C 11BF; # (됰; 됰; 됰; 됰; 됰; ) HANGUL SYLLABLE DOEK +B431;B431;1103 116C 11C0;B431;1103 116C 11C0; # (됱; 됱; 됱; 됱; 됱; ) HANGUL SYLLABLE DOET +B432;B432;1103 116C 11C1;B432;1103 116C 11C1; # (됲; 됲; 됲; 됲; 됲; ) HANGUL SYLLABLE DOEP +B433;B433;1103 116C 11C2;B433;1103 116C 11C2; # (됳; 됳; 됳; 됳; 됳; ) HANGUL SYLLABLE DOEH +B434;B434;1103 116D;B434;1103 116D; # (됴; 됴; 됴; 됴; 됴; ) HANGUL SYLLABLE DYO +B435;B435;1103 116D 11A8;B435;1103 116D 11A8; # (됵; 됵; 됵; 됵; 됵; ) HANGUL SYLLABLE DYOG +B436;B436;1103 116D 11A9;B436;1103 116D 11A9; # (됶; 됶; 됶; 됶; 됶; ) HANGUL SYLLABLE DYOGG +B437;B437;1103 116D 11AA;B437;1103 116D 11AA; # (됷; 됷; 됷; 됷; 됷; ) HANGUL SYLLABLE DYOGS +B438;B438;1103 116D 11AB;B438;1103 116D 11AB; # (됸; 됸; 됸; 됸; 됸; ) HANGUL SYLLABLE DYON +B439;B439;1103 116D 11AC;B439;1103 116D 11AC; # (됹; 됹; 됹; 됹; 됹; ) HANGUL SYLLABLE DYONJ +B43A;B43A;1103 116D 11AD;B43A;1103 116D 11AD; # (됺; 됺; 됺; 됺; 됺; ) HANGUL SYLLABLE DYONH +B43B;B43B;1103 116D 11AE;B43B;1103 116D 11AE; # (됻; 됻; 됻; 됻; 됻; ) HANGUL SYLLABLE DYOD +B43C;B43C;1103 116D 11AF;B43C;1103 116D 11AF; # (됼; 됼; 됼; 됼; 됼; ) HANGUL SYLLABLE DYOL +B43D;B43D;1103 116D 11B0;B43D;1103 116D 11B0; # (됽; 됽; 됽; 됽; 됽; ) HANGUL SYLLABLE DYOLG +B43E;B43E;1103 116D 11B1;B43E;1103 116D 11B1; # (됾; 됾; 됾; 됾; 됾; ) HANGUL SYLLABLE DYOLM +B43F;B43F;1103 116D 11B2;B43F;1103 116D 11B2; # (됿; 됿; 됿; 됿; 됿; ) HANGUL SYLLABLE DYOLB +B440;B440;1103 116D 11B3;B440;1103 116D 11B3; # (둀; 둀; 둀; 둀; 둀; ) HANGUL SYLLABLE DYOLS +B441;B441;1103 116D 11B4;B441;1103 116D 11B4; # (둁; 둁; 둁; 둁; 둁; ) HANGUL SYLLABLE DYOLT +B442;B442;1103 116D 11B5;B442;1103 116D 11B5; # (둂; 둂; 둂; 둂; 둂; ) HANGUL SYLLABLE DYOLP +B443;B443;1103 116D 11B6;B443;1103 116D 11B6; # (둃; 둃; 둃; 둃; 둃; ) HANGUL SYLLABLE DYOLH +B444;B444;1103 116D 11B7;B444;1103 116D 11B7; # (둄; 둄; 둄; 둄; 둄; ) HANGUL SYLLABLE DYOM +B445;B445;1103 116D 11B8;B445;1103 116D 11B8; # (둅; 둅; 둅; 둅; 둅; ) HANGUL SYLLABLE DYOB +B446;B446;1103 116D 11B9;B446;1103 116D 11B9; # (둆; 둆; 둆; 둆; 둆; ) HANGUL SYLLABLE DYOBS +B447;B447;1103 116D 11BA;B447;1103 116D 11BA; # (둇; 둇; 둇; 둇; 둇; ) HANGUL SYLLABLE DYOS +B448;B448;1103 116D 11BB;B448;1103 116D 11BB; # (둈; 둈; 둈; 둈; 둈; ) HANGUL SYLLABLE DYOSS +B449;B449;1103 116D 11BC;B449;1103 116D 11BC; # (둉; 둉; 둉; 둉; 둉; ) HANGUL SYLLABLE DYONG +B44A;B44A;1103 116D 11BD;B44A;1103 116D 11BD; # (둊; 둊; 둊; 둊; 둊; ) HANGUL SYLLABLE DYOJ +B44B;B44B;1103 116D 11BE;B44B;1103 116D 11BE; # (둋; 둋; 둋; 둋; 둋; ) HANGUL SYLLABLE DYOC +B44C;B44C;1103 116D 11BF;B44C;1103 116D 11BF; # (둌; 둌; 둌; 둌; 둌; ) HANGUL SYLLABLE DYOK +B44D;B44D;1103 116D 11C0;B44D;1103 116D 11C0; # (둍; 둍; 둍; 둍; 둍; ) HANGUL SYLLABLE DYOT +B44E;B44E;1103 116D 11C1;B44E;1103 116D 11C1; # (둎; 둎; 둎; 둎; 둎; ) HANGUL SYLLABLE DYOP +B44F;B44F;1103 116D 11C2;B44F;1103 116D 11C2; # (둏; 둏; 둏; 둏; 둏; ) HANGUL SYLLABLE DYOH +B450;B450;1103 116E;B450;1103 116E; # (두; 두; 두; 두; 두; ) HANGUL SYLLABLE DU +B451;B451;1103 116E 11A8;B451;1103 116E 11A8; # (둑; 둑; 둑; 둑; 둑; ) HANGUL SYLLABLE DUG +B452;B452;1103 116E 11A9;B452;1103 116E 11A9; # (둒; 둒; 둒; 둒; 둒; ) HANGUL SYLLABLE DUGG +B453;B453;1103 116E 11AA;B453;1103 116E 11AA; # (둓; 둓; 둓; 둓; 둓; ) HANGUL SYLLABLE DUGS +B454;B454;1103 116E 11AB;B454;1103 116E 11AB; # (둔; 둔; 둔; 둔; 둔; ) HANGUL SYLLABLE DUN +B455;B455;1103 116E 11AC;B455;1103 116E 11AC; # (둕; 둕; 둕; 둕; 둕; ) HANGUL SYLLABLE DUNJ +B456;B456;1103 116E 11AD;B456;1103 116E 11AD; # (둖; 둖; 둖; 둖; 둖; ) HANGUL SYLLABLE DUNH +B457;B457;1103 116E 11AE;B457;1103 116E 11AE; # (둗; 둗; 둗; 둗; 둗; ) HANGUL SYLLABLE DUD +B458;B458;1103 116E 11AF;B458;1103 116E 11AF; # (둘; 둘; 둘; 둘; 둘; ) HANGUL SYLLABLE DUL +B459;B459;1103 116E 11B0;B459;1103 116E 11B0; # (둙; 둙; 둙; 둙; 둙; ) HANGUL SYLLABLE DULG +B45A;B45A;1103 116E 11B1;B45A;1103 116E 11B1; # (둚; 둚; 둚; 둚; 둚; ) HANGUL SYLLABLE DULM +B45B;B45B;1103 116E 11B2;B45B;1103 116E 11B2; # (둛; 둛; 둛; 둛; 둛; ) HANGUL SYLLABLE DULB +B45C;B45C;1103 116E 11B3;B45C;1103 116E 11B3; # (둜; 둜; 둜; 둜; 둜; ) HANGUL SYLLABLE DULS +B45D;B45D;1103 116E 11B4;B45D;1103 116E 11B4; # (둝; 둝; 둝; 둝; 둝; ) HANGUL SYLLABLE DULT +B45E;B45E;1103 116E 11B5;B45E;1103 116E 11B5; # (둞; 둞; 둞; 둞; 둞; ) HANGUL SYLLABLE DULP +B45F;B45F;1103 116E 11B6;B45F;1103 116E 11B6; # (둟; 둟; 둟; 둟; 둟; ) HANGUL SYLLABLE DULH +B460;B460;1103 116E 11B7;B460;1103 116E 11B7; # (둠; 둠; 둠; 둠; 둠; ) HANGUL SYLLABLE DUM +B461;B461;1103 116E 11B8;B461;1103 116E 11B8; # (둡; 둡; 둡; 둡; 둡; ) HANGUL SYLLABLE DUB +B462;B462;1103 116E 11B9;B462;1103 116E 11B9; # (둢; 둢; 둢; 둢; 둢; ) HANGUL SYLLABLE DUBS +B463;B463;1103 116E 11BA;B463;1103 116E 11BA; # (둣; 둣; 둣; 둣; 둣; ) HANGUL SYLLABLE DUS +B464;B464;1103 116E 11BB;B464;1103 116E 11BB; # (둤; 둤; 둤; 둤; 둤; ) HANGUL SYLLABLE DUSS +B465;B465;1103 116E 11BC;B465;1103 116E 11BC; # (둥; 둥; 둥; 둥; 둥; ) HANGUL SYLLABLE DUNG +B466;B466;1103 116E 11BD;B466;1103 116E 11BD; # (둦; 둦; 둦; 둦; 둦; ) HANGUL SYLLABLE DUJ +B467;B467;1103 116E 11BE;B467;1103 116E 11BE; # (둧; 둧; 둧; 둧; 둧; ) HANGUL SYLLABLE DUC +B468;B468;1103 116E 11BF;B468;1103 116E 11BF; # (둨; 둨; 둨; 둨; 둨; ) HANGUL SYLLABLE DUK +B469;B469;1103 116E 11C0;B469;1103 116E 11C0; # (둩; 둩; 둩; 둩; 둩; ) HANGUL SYLLABLE DUT +B46A;B46A;1103 116E 11C1;B46A;1103 116E 11C1; # (둪; 둪; 둪; 둪; 둪; ) HANGUL SYLLABLE DUP +B46B;B46B;1103 116E 11C2;B46B;1103 116E 11C2; # (둫; 둫; 둫; 둫; 둫; ) HANGUL SYLLABLE DUH +B46C;B46C;1103 116F;B46C;1103 116F; # (둬; 둬; 둬; 둬; 둬; ) HANGUL SYLLABLE DWEO +B46D;B46D;1103 116F 11A8;B46D;1103 116F 11A8; # (둭; 둭; 둭; 둭; 둭; ) HANGUL SYLLABLE DWEOG +B46E;B46E;1103 116F 11A9;B46E;1103 116F 11A9; # (둮; 둮; 둮; 둮; 둮; ) HANGUL SYLLABLE DWEOGG +B46F;B46F;1103 116F 11AA;B46F;1103 116F 11AA; # (둯; 둯; 둯; 둯; 둯; ) HANGUL SYLLABLE DWEOGS +B470;B470;1103 116F 11AB;B470;1103 116F 11AB; # (둰; 둰; 둰; 둰; 둰; ) HANGUL SYLLABLE DWEON +B471;B471;1103 116F 11AC;B471;1103 116F 11AC; # (둱; 둱; 둱; 둱; 둱; ) HANGUL SYLLABLE DWEONJ +B472;B472;1103 116F 11AD;B472;1103 116F 11AD; # (둲; 둲; 둲; 둲; 둲; ) HANGUL SYLLABLE DWEONH +B473;B473;1103 116F 11AE;B473;1103 116F 11AE; # (둳; 둳; 둳; 둳; 둳; ) HANGUL SYLLABLE DWEOD +B474;B474;1103 116F 11AF;B474;1103 116F 11AF; # (둴; 둴; 둴; 둴; 둴; ) HANGUL SYLLABLE DWEOL +B475;B475;1103 116F 11B0;B475;1103 116F 11B0; # (둵; 둵; 둵; 둵; 둵; ) HANGUL SYLLABLE DWEOLG +B476;B476;1103 116F 11B1;B476;1103 116F 11B1; # (둶; 둶; 둶; 둶; 둶; ) HANGUL SYLLABLE DWEOLM +B477;B477;1103 116F 11B2;B477;1103 116F 11B2; # (둷; 둷; 둷; 둷; 둷; ) HANGUL SYLLABLE DWEOLB +B478;B478;1103 116F 11B3;B478;1103 116F 11B3; # (둸; 둸; 둸; 둸; 둸; ) HANGUL SYLLABLE DWEOLS +B479;B479;1103 116F 11B4;B479;1103 116F 11B4; # (둹; 둹; 둹; 둹; 둹; ) HANGUL SYLLABLE DWEOLT +B47A;B47A;1103 116F 11B5;B47A;1103 116F 11B5; # (둺; 둺; 둺; 둺; 둺; ) HANGUL SYLLABLE DWEOLP +B47B;B47B;1103 116F 11B6;B47B;1103 116F 11B6; # (둻; 둻; 둻; 둻; 둻; ) HANGUL SYLLABLE DWEOLH +B47C;B47C;1103 116F 11B7;B47C;1103 116F 11B7; # (둼; 둼; 둼; 둼; 둼; ) HANGUL SYLLABLE DWEOM +B47D;B47D;1103 116F 11B8;B47D;1103 116F 11B8; # (둽; 둽; 둽; 둽; 둽; ) HANGUL SYLLABLE DWEOB +B47E;B47E;1103 116F 11B9;B47E;1103 116F 11B9; # (둾; 둾; 둾; 둾; 둾; ) HANGUL SYLLABLE DWEOBS +B47F;B47F;1103 116F 11BA;B47F;1103 116F 11BA; # (둿; 둿; 둿; 둿; 둿; ) HANGUL SYLLABLE DWEOS +B480;B480;1103 116F 11BB;B480;1103 116F 11BB; # (뒀; 뒀; 뒀; 뒀; 뒀; ) HANGUL SYLLABLE DWEOSS +B481;B481;1103 116F 11BC;B481;1103 116F 11BC; # (뒁; 뒁; 뒁; 뒁; 뒁; ) HANGUL SYLLABLE DWEONG +B482;B482;1103 116F 11BD;B482;1103 116F 11BD; # (뒂; 뒂; 뒂; 뒂; 뒂; ) HANGUL SYLLABLE DWEOJ +B483;B483;1103 116F 11BE;B483;1103 116F 11BE; # (뒃; 뒃; 뒃; 뒃; 뒃; ) HANGUL SYLLABLE DWEOC +B484;B484;1103 116F 11BF;B484;1103 116F 11BF; # (뒄; 뒄; 뒄; 뒄; 뒄; ) HANGUL SYLLABLE DWEOK +B485;B485;1103 116F 11C0;B485;1103 116F 11C0; # (뒅; 뒅; 뒅; 뒅; 뒅; ) HANGUL SYLLABLE DWEOT +B486;B486;1103 116F 11C1;B486;1103 116F 11C1; # (뒆; 뒆; 뒆; 뒆; 뒆; ) HANGUL SYLLABLE DWEOP +B487;B487;1103 116F 11C2;B487;1103 116F 11C2; # (뒇; 뒇; 뒇; 뒇; 뒇; ) HANGUL SYLLABLE DWEOH +B488;B488;1103 1170;B488;1103 1170; # (뒈; 뒈; 뒈; 뒈; 뒈; ) HANGUL SYLLABLE DWE +B489;B489;1103 1170 11A8;B489;1103 1170 11A8; # (뒉; 뒉; 뒉; 뒉; 뒉; ) HANGUL SYLLABLE DWEG +B48A;B48A;1103 1170 11A9;B48A;1103 1170 11A9; # (뒊; 뒊; 뒊; 뒊; 뒊; ) HANGUL SYLLABLE DWEGG +B48B;B48B;1103 1170 11AA;B48B;1103 1170 11AA; # (뒋; 뒋; 뒋; 뒋; 뒋; ) HANGUL SYLLABLE DWEGS +B48C;B48C;1103 1170 11AB;B48C;1103 1170 11AB; # (뒌; 뒌; 뒌; 뒌; 뒌; ) HANGUL SYLLABLE DWEN +B48D;B48D;1103 1170 11AC;B48D;1103 1170 11AC; # (뒍; 뒍; 뒍; 뒍; 뒍; ) HANGUL SYLLABLE DWENJ +B48E;B48E;1103 1170 11AD;B48E;1103 1170 11AD; # (뒎; 뒎; 뒎; 뒎; 뒎; ) HANGUL SYLLABLE DWENH +B48F;B48F;1103 1170 11AE;B48F;1103 1170 11AE; # (뒏; 뒏; 뒏; 뒏; 뒏; ) HANGUL SYLLABLE DWED +B490;B490;1103 1170 11AF;B490;1103 1170 11AF; # (뒐; 뒐; 뒐; 뒐; 뒐; ) HANGUL SYLLABLE DWEL +B491;B491;1103 1170 11B0;B491;1103 1170 11B0; # (뒑; 뒑; 뒑; 뒑; 뒑; ) HANGUL SYLLABLE DWELG +B492;B492;1103 1170 11B1;B492;1103 1170 11B1; # (뒒; 뒒; 뒒; 뒒; 뒒; ) HANGUL SYLLABLE DWELM +B493;B493;1103 1170 11B2;B493;1103 1170 11B2; # (뒓; 뒓; 뒓; 뒓; 뒓; ) HANGUL SYLLABLE DWELB +B494;B494;1103 1170 11B3;B494;1103 1170 11B3; # (뒔; 뒔; 뒔; 뒔; 뒔; ) HANGUL SYLLABLE DWELS +B495;B495;1103 1170 11B4;B495;1103 1170 11B4; # (뒕; 뒕; 뒕; 뒕; 뒕; ) HANGUL SYLLABLE DWELT +B496;B496;1103 1170 11B5;B496;1103 1170 11B5; # (뒖; 뒖; 뒖; 뒖; 뒖; ) HANGUL SYLLABLE DWELP +B497;B497;1103 1170 11B6;B497;1103 1170 11B6; # (뒗; 뒗; 뒗; 뒗; 뒗; ) HANGUL SYLLABLE DWELH +B498;B498;1103 1170 11B7;B498;1103 1170 11B7; # (뒘; 뒘; 뒘; 뒘; 뒘; ) HANGUL SYLLABLE DWEM +B499;B499;1103 1170 11B8;B499;1103 1170 11B8; # (뒙; 뒙; 뒙; 뒙; 뒙; ) HANGUL SYLLABLE DWEB +B49A;B49A;1103 1170 11B9;B49A;1103 1170 11B9; # (뒚; 뒚; 뒚; 뒚; 뒚; ) HANGUL SYLLABLE DWEBS +B49B;B49B;1103 1170 11BA;B49B;1103 1170 11BA; # (뒛; 뒛; 뒛; 뒛; 뒛; ) HANGUL SYLLABLE DWES +B49C;B49C;1103 1170 11BB;B49C;1103 1170 11BB; # (뒜; 뒜; 뒜; 뒜; 뒜; ) HANGUL SYLLABLE DWESS +B49D;B49D;1103 1170 11BC;B49D;1103 1170 11BC; # (뒝; 뒝; 뒝; 뒝; 뒝; ) HANGUL SYLLABLE DWENG +B49E;B49E;1103 1170 11BD;B49E;1103 1170 11BD; # (뒞; 뒞; 뒞; 뒞; 뒞; ) HANGUL SYLLABLE DWEJ +B49F;B49F;1103 1170 11BE;B49F;1103 1170 11BE; # (뒟; 뒟; 뒟; 뒟; 뒟; ) HANGUL SYLLABLE DWEC +B4A0;B4A0;1103 1170 11BF;B4A0;1103 1170 11BF; # (뒠; 뒠; 뒠; 뒠; 뒠; ) HANGUL SYLLABLE DWEK +B4A1;B4A1;1103 1170 11C0;B4A1;1103 1170 11C0; # (뒡; 뒡; 뒡; 뒡; 뒡; ) HANGUL SYLLABLE DWET +B4A2;B4A2;1103 1170 11C1;B4A2;1103 1170 11C1; # (뒢; 뒢; 뒢; 뒢; 뒢; ) HANGUL SYLLABLE DWEP +B4A3;B4A3;1103 1170 11C2;B4A3;1103 1170 11C2; # (뒣; 뒣; 뒣; 뒣; 뒣; ) HANGUL SYLLABLE DWEH +B4A4;B4A4;1103 1171;B4A4;1103 1171; # (뒤; 뒤; 뒤; 뒤; 뒤; ) HANGUL SYLLABLE DWI +B4A5;B4A5;1103 1171 11A8;B4A5;1103 1171 11A8; # (뒥; 뒥; 뒥; 뒥; 뒥; ) HANGUL SYLLABLE DWIG +B4A6;B4A6;1103 1171 11A9;B4A6;1103 1171 11A9; # (뒦; 뒦; 뒦; 뒦; 뒦; ) HANGUL SYLLABLE DWIGG +B4A7;B4A7;1103 1171 11AA;B4A7;1103 1171 11AA; # (뒧; 뒧; 뒧; 뒧; 뒧; ) HANGUL SYLLABLE DWIGS +B4A8;B4A8;1103 1171 11AB;B4A8;1103 1171 11AB; # (뒨; 뒨; 뒨; 뒨; 뒨; ) HANGUL SYLLABLE DWIN +B4A9;B4A9;1103 1171 11AC;B4A9;1103 1171 11AC; # (뒩; 뒩; 뒩; 뒩; 뒩; ) HANGUL SYLLABLE DWINJ +B4AA;B4AA;1103 1171 11AD;B4AA;1103 1171 11AD; # (뒪; 뒪; 뒪; 뒪; 뒪; ) HANGUL SYLLABLE DWINH +B4AB;B4AB;1103 1171 11AE;B4AB;1103 1171 11AE; # (뒫; 뒫; 뒫; 뒫; 뒫; ) HANGUL SYLLABLE DWID +B4AC;B4AC;1103 1171 11AF;B4AC;1103 1171 11AF; # (뒬; 뒬; 뒬; 뒬; 뒬; ) HANGUL SYLLABLE DWIL +B4AD;B4AD;1103 1171 11B0;B4AD;1103 1171 11B0; # (뒭; 뒭; 뒭; 뒭; 뒭; ) HANGUL SYLLABLE DWILG +B4AE;B4AE;1103 1171 11B1;B4AE;1103 1171 11B1; # (뒮; 뒮; 뒮; 뒮; 뒮; ) HANGUL SYLLABLE DWILM +B4AF;B4AF;1103 1171 11B2;B4AF;1103 1171 11B2; # (뒯; 뒯; 뒯; 뒯; 뒯; ) HANGUL SYLLABLE DWILB +B4B0;B4B0;1103 1171 11B3;B4B0;1103 1171 11B3; # (뒰; 뒰; 뒰; 뒰; 뒰; ) HANGUL SYLLABLE DWILS +B4B1;B4B1;1103 1171 11B4;B4B1;1103 1171 11B4; # (뒱; 뒱; 뒱; 뒱; 뒱; ) HANGUL SYLLABLE DWILT +B4B2;B4B2;1103 1171 11B5;B4B2;1103 1171 11B5; # (뒲; 뒲; 뒲; 뒲; 뒲; ) HANGUL SYLLABLE DWILP +B4B3;B4B3;1103 1171 11B6;B4B3;1103 1171 11B6; # (뒳; 뒳; 뒳; 뒳; 뒳; ) HANGUL SYLLABLE DWILH +B4B4;B4B4;1103 1171 11B7;B4B4;1103 1171 11B7; # (뒴; 뒴; 뒴; 뒴; 뒴; ) HANGUL SYLLABLE DWIM +B4B5;B4B5;1103 1171 11B8;B4B5;1103 1171 11B8; # (뒵; 뒵; 뒵; 뒵; 뒵; ) HANGUL SYLLABLE DWIB +B4B6;B4B6;1103 1171 11B9;B4B6;1103 1171 11B9; # (뒶; 뒶; 뒶; 뒶; 뒶; ) HANGUL SYLLABLE DWIBS +B4B7;B4B7;1103 1171 11BA;B4B7;1103 1171 11BA; # (뒷; 뒷; 뒷; 뒷; 뒷; ) HANGUL SYLLABLE DWIS +B4B8;B4B8;1103 1171 11BB;B4B8;1103 1171 11BB; # (뒸; 뒸; 뒸; 뒸; 뒸; ) HANGUL SYLLABLE DWISS +B4B9;B4B9;1103 1171 11BC;B4B9;1103 1171 11BC; # (뒹; 뒹; 뒹; 뒹; 뒹; ) HANGUL SYLLABLE DWING +B4BA;B4BA;1103 1171 11BD;B4BA;1103 1171 11BD; # (뒺; 뒺; 뒺; 뒺; 뒺; ) HANGUL SYLLABLE DWIJ +B4BB;B4BB;1103 1171 11BE;B4BB;1103 1171 11BE; # (뒻; 뒻; 뒻; 뒻; 뒻; ) HANGUL SYLLABLE DWIC +B4BC;B4BC;1103 1171 11BF;B4BC;1103 1171 11BF; # (뒼; 뒼; 뒼; 뒼; 뒼; ) HANGUL SYLLABLE DWIK +B4BD;B4BD;1103 1171 11C0;B4BD;1103 1171 11C0; # (뒽; 뒽; 뒽; 뒽; 뒽; ) HANGUL SYLLABLE DWIT +B4BE;B4BE;1103 1171 11C1;B4BE;1103 1171 11C1; # (뒾; 뒾; 뒾; 뒾; 뒾; ) HANGUL SYLLABLE DWIP +B4BF;B4BF;1103 1171 11C2;B4BF;1103 1171 11C2; # (뒿; 뒿; 뒿; 뒿; 뒿; ) HANGUL SYLLABLE DWIH +B4C0;B4C0;1103 1172;B4C0;1103 1172; # (듀; 듀; 듀; 듀; 듀; ) HANGUL SYLLABLE DYU +B4C1;B4C1;1103 1172 11A8;B4C1;1103 1172 11A8; # (듁; 듁; 듁; 듁; 듁; ) HANGUL SYLLABLE DYUG +B4C2;B4C2;1103 1172 11A9;B4C2;1103 1172 11A9; # (듂; 듂; 듂; 듂; 듂; ) HANGUL SYLLABLE DYUGG +B4C3;B4C3;1103 1172 11AA;B4C3;1103 1172 11AA; # (듃; 듃; 듃; 듃; 듃; ) HANGUL SYLLABLE DYUGS +B4C4;B4C4;1103 1172 11AB;B4C4;1103 1172 11AB; # (듄; 듄; 듄; 듄; 듄; ) HANGUL SYLLABLE DYUN +B4C5;B4C5;1103 1172 11AC;B4C5;1103 1172 11AC; # (듅; 듅; 듅; 듅; 듅; ) HANGUL SYLLABLE DYUNJ +B4C6;B4C6;1103 1172 11AD;B4C6;1103 1172 11AD; # (듆; 듆; 듆; 듆; 듆; ) HANGUL SYLLABLE DYUNH +B4C7;B4C7;1103 1172 11AE;B4C7;1103 1172 11AE; # (듇; 듇; 듇; 듇; 듇; ) HANGUL SYLLABLE DYUD +B4C8;B4C8;1103 1172 11AF;B4C8;1103 1172 11AF; # (듈; 듈; 듈; 듈; 듈; ) HANGUL SYLLABLE DYUL +B4C9;B4C9;1103 1172 11B0;B4C9;1103 1172 11B0; # (듉; 듉; 듉; 듉; 듉; ) HANGUL SYLLABLE DYULG +B4CA;B4CA;1103 1172 11B1;B4CA;1103 1172 11B1; # (듊; 듊; 듊; 듊; 듊; ) HANGUL SYLLABLE DYULM +B4CB;B4CB;1103 1172 11B2;B4CB;1103 1172 11B2; # (듋; 듋; 듋; 듋; 듋; ) HANGUL SYLLABLE DYULB +B4CC;B4CC;1103 1172 11B3;B4CC;1103 1172 11B3; # (듌; 듌; 듌; 듌; 듌; ) HANGUL SYLLABLE DYULS +B4CD;B4CD;1103 1172 11B4;B4CD;1103 1172 11B4; # (듍; 듍; 듍; 듍; 듍; ) HANGUL SYLLABLE DYULT +B4CE;B4CE;1103 1172 11B5;B4CE;1103 1172 11B5; # (듎; 듎; 듎; 듎; 듎; ) HANGUL SYLLABLE DYULP +B4CF;B4CF;1103 1172 11B6;B4CF;1103 1172 11B6; # (듏; 듏; 듏; 듏; 듏; ) HANGUL SYLLABLE DYULH +B4D0;B4D0;1103 1172 11B7;B4D0;1103 1172 11B7; # (듐; 듐; 듐; 듐; 듐; ) HANGUL SYLLABLE DYUM +B4D1;B4D1;1103 1172 11B8;B4D1;1103 1172 11B8; # (듑; 듑; 듑; 듑; 듑; ) HANGUL SYLLABLE DYUB +B4D2;B4D2;1103 1172 11B9;B4D2;1103 1172 11B9; # (듒; 듒; 듒; 듒; 듒; ) HANGUL SYLLABLE DYUBS +B4D3;B4D3;1103 1172 11BA;B4D3;1103 1172 11BA; # (듓; 듓; 듓; 듓; 듓; ) HANGUL SYLLABLE DYUS +B4D4;B4D4;1103 1172 11BB;B4D4;1103 1172 11BB; # (듔; 듔; 듔; 듔; 듔; ) HANGUL SYLLABLE DYUSS +B4D5;B4D5;1103 1172 11BC;B4D5;1103 1172 11BC; # (듕; 듕; 듕; 듕; 듕; ) HANGUL SYLLABLE DYUNG +B4D6;B4D6;1103 1172 11BD;B4D6;1103 1172 11BD; # (듖; 듖; 듖; 듖; 듖; ) HANGUL SYLLABLE DYUJ +B4D7;B4D7;1103 1172 11BE;B4D7;1103 1172 11BE; # (듗; 듗; 듗; 듗; 듗; ) HANGUL SYLLABLE DYUC +B4D8;B4D8;1103 1172 11BF;B4D8;1103 1172 11BF; # (듘; 듘; 듘; 듘; 듘; ) HANGUL SYLLABLE DYUK +B4D9;B4D9;1103 1172 11C0;B4D9;1103 1172 11C0; # (듙; 듙; 듙; 듙; 듙; ) HANGUL SYLLABLE DYUT +B4DA;B4DA;1103 1172 11C1;B4DA;1103 1172 11C1; # (듚; 듚; 듚; 듚; 듚; ) HANGUL SYLLABLE DYUP +B4DB;B4DB;1103 1172 11C2;B4DB;1103 1172 11C2; # (듛; 듛; 듛; 듛; 듛; ) HANGUL SYLLABLE DYUH +B4DC;B4DC;1103 1173;B4DC;1103 1173; # (드; 드; 드; 드; 드; ) HANGUL SYLLABLE DEU +B4DD;B4DD;1103 1173 11A8;B4DD;1103 1173 11A8; # (득; 득; 득; 득; 득; ) HANGUL SYLLABLE DEUG +B4DE;B4DE;1103 1173 11A9;B4DE;1103 1173 11A9; # (듞; 듞; 듞; 듞; 듞; ) HANGUL SYLLABLE DEUGG +B4DF;B4DF;1103 1173 11AA;B4DF;1103 1173 11AA; # (듟; 듟; 듟; 듟; 듟; ) HANGUL SYLLABLE DEUGS +B4E0;B4E0;1103 1173 11AB;B4E0;1103 1173 11AB; # (든; 든; 든; 든; 든; ) HANGUL SYLLABLE DEUN +B4E1;B4E1;1103 1173 11AC;B4E1;1103 1173 11AC; # (듡; 듡; 듡; 듡; 듡; ) HANGUL SYLLABLE DEUNJ +B4E2;B4E2;1103 1173 11AD;B4E2;1103 1173 11AD; # (듢; 듢; 듢; 듢; 듢; ) HANGUL SYLLABLE DEUNH +B4E3;B4E3;1103 1173 11AE;B4E3;1103 1173 11AE; # (듣; 듣; 듣; 듣; 듣; ) HANGUL SYLLABLE DEUD +B4E4;B4E4;1103 1173 11AF;B4E4;1103 1173 11AF; # (들; 들; 들; 들; 들; ) HANGUL SYLLABLE DEUL +B4E5;B4E5;1103 1173 11B0;B4E5;1103 1173 11B0; # (듥; 듥; 듥; 듥; 듥; ) HANGUL SYLLABLE DEULG +B4E6;B4E6;1103 1173 11B1;B4E6;1103 1173 11B1; # (듦; 듦; 듦; 듦; 듦; ) HANGUL SYLLABLE DEULM +B4E7;B4E7;1103 1173 11B2;B4E7;1103 1173 11B2; # (듧; 듧; 듧; 듧; 듧; ) HANGUL SYLLABLE DEULB +B4E8;B4E8;1103 1173 11B3;B4E8;1103 1173 11B3; # (듨; 듨; 듨; 듨; 듨; ) HANGUL SYLLABLE DEULS +B4E9;B4E9;1103 1173 11B4;B4E9;1103 1173 11B4; # (듩; 듩; 듩; 듩; 듩; ) HANGUL SYLLABLE DEULT +B4EA;B4EA;1103 1173 11B5;B4EA;1103 1173 11B5; # (듪; 듪; 듪; 듪; 듪; ) HANGUL SYLLABLE DEULP +B4EB;B4EB;1103 1173 11B6;B4EB;1103 1173 11B6; # (듫; 듫; 듫; 듫; 듫; ) HANGUL SYLLABLE DEULH +B4EC;B4EC;1103 1173 11B7;B4EC;1103 1173 11B7; # (듬; 듬; 듬; 듬; 듬; ) HANGUL SYLLABLE DEUM +B4ED;B4ED;1103 1173 11B8;B4ED;1103 1173 11B8; # (듭; 듭; 듭; 듭; 듭; ) HANGUL SYLLABLE DEUB +B4EE;B4EE;1103 1173 11B9;B4EE;1103 1173 11B9; # (듮; 듮; 듮; 듮; 듮; ) HANGUL SYLLABLE DEUBS +B4EF;B4EF;1103 1173 11BA;B4EF;1103 1173 11BA; # (듯; 듯; 듯; 듯; 듯; ) HANGUL SYLLABLE DEUS +B4F0;B4F0;1103 1173 11BB;B4F0;1103 1173 11BB; # (듰; 듰; 듰; 듰; 듰; ) HANGUL SYLLABLE DEUSS +B4F1;B4F1;1103 1173 11BC;B4F1;1103 1173 11BC; # (등; 등; 등; 등; 등; ) HANGUL SYLLABLE DEUNG +B4F2;B4F2;1103 1173 11BD;B4F2;1103 1173 11BD; # (듲; 듲; 듲; 듲; 듲; ) HANGUL SYLLABLE DEUJ +B4F3;B4F3;1103 1173 11BE;B4F3;1103 1173 11BE; # (듳; 듳; 듳; 듳; 듳; ) HANGUL SYLLABLE DEUC +B4F4;B4F4;1103 1173 11BF;B4F4;1103 1173 11BF; # (듴; 듴; 듴; 듴; 듴; ) HANGUL SYLLABLE DEUK +B4F5;B4F5;1103 1173 11C0;B4F5;1103 1173 11C0; # (듵; 듵; 듵; 듵; 듵; ) HANGUL SYLLABLE DEUT +B4F6;B4F6;1103 1173 11C1;B4F6;1103 1173 11C1; # (듶; 듶; 듶; 듶; 듶; ) HANGUL SYLLABLE DEUP +B4F7;B4F7;1103 1173 11C2;B4F7;1103 1173 11C2; # (듷; 듷; 듷; 듷; 듷; ) HANGUL SYLLABLE DEUH +B4F8;B4F8;1103 1174;B4F8;1103 1174; # (듸; 듸; 듸; 듸; 듸; ) HANGUL SYLLABLE DYI +B4F9;B4F9;1103 1174 11A8;B4F9;1103 1174 11A8; # (듹; 듹; 듹; 듹; 듹; ) HANGUL SYLLABLE DYIG +B4FA;B4FA;1103 1174 11A9;B4FA;1103 1174 11A9; # (듺; 듺; 듺; 듺; 듺; ) HANGUL SYLLABLE DYIGG +B4FB;B4FB;1103 1174 11AA;B4FB;1103 1174 11AA; # (듻; 듻; 듻; 듻; 듻; ) HANGUL SYLLABLE DYIGS +B4FC;B4FC;1103 1174 11AB;B4FC;1103 1174 11AB; # (듼; 듼; 듼; 듼; 듼; ) HANGUL SYLLABLE DYIN +B4FD;B4FD;1103 1174 11AC;B4FD;1103 1174 11AC; # (듽; 듽; 듽; 듽; 듽; ) HANGUL SYLLABLE DYINJ +B4FE;B4FE;1103 1174 11AD;B4FE;1103 1174 11AD; # (듾; 듾; 듾; 듾; 듾; ) HANGUL SYLLABLE DYINH +B4FF;B4FF;1103 1174 11AE;B4FF;1103 1174 11AE; # (듿; 듿; 듿; 듿; 듿; ) HANGUL SYLLABLE DYID +B500;B500;1103 1174 11AF;B500;1103 1174 11AF; # (딀; 딀; 딀; 딀; 딀; ) HANGUL SYLLABLE DYIL +B501;B501;1103 1174 11B0;B501;1103 1174 11B0; # (딁; 딁; 딁; 딁; 딁; ) HANGUL SYLLABLE DYILG +B502;B502;1103 1174 11B1;B502;1103 1174 11B1; # (딂; 딂; 딂; 딂; 딂; ) HANGUL SYLLABLE DYILM +B503;B503;1103 1174 11B2;B503;1103 1174 11B2; # (딃; 딃; 딃; 딃; 딃; ) HANGUL SYLLABLE DYILB +B504;B504;1103 1174 11B3;B504;1103 1174 11B3; # (딄; 딄; 딄; 딄; 딄; ) HANGUL SYLLABLE DYILS +B505;B505;1103 1174 11B4;B505;1103 1174 11B4; # (딅; 딅; 딅; 딅; 딅; ) HANGUL SYLLABLE DYILT +B506;B506;1103 1174 11B5;B506;1103 1174 11B5; # (딆; 딆; 딆; 딆; 딆; ) HANGUL SYLLABLE DYILP +B507;B507;1103 1174 11B6;B507;1103 1174 11B6; # (딇; 딇; 딇; 딇; 딇; ) HANGUL SYLLABLE DYILH +B508;B508;1103 1174 11B7;B508;1103 1174 11B7; # (딈; 딈; 딈; 딈; 딈; ) HANGUL SYLLABLE DYIM +B509;B509;1103 1174 11B8;B509;1103 1174 11B8; # (딉; 딉; 딉; 딉; 딉; ) HANGUL SYLLABLE DYIB +B50A;B50A;1103 1174 11B9;B50A;1103 1174 11B9; # (딊; 딊; 딊; 딊; 딊; ) HANGUL SYLLABLE DYIBS +B50B;B50B;1103 1174 11BA;B50B;1103 1174 11BA; # (딋; 딋; 딋; 딋; 딋; ) HANGUL SYLLABLE DYIS +B50C;B50C;1103 1174 11BB;B50C;1103 1174 11BB; # (딌; 딌; 딌; 딌; 딌; ) HANGUL SYLLABLE DYISS +B50D;B50D;1103 1174 11BC;B50D;1103 1174 11BC; # (딍; 딍; 딍; 딍; 딍; ) HANGUL SYLLABLE DYING +B50E;B50E;1103 1174 11BD;B50E;1103 1174 11BD; # (딎; 딎; 딎; 딎; 딎; ) HANGUL SYLLABLE DYIJ +B50F;B50F;1103 1174 11BE;B50F;1103 1174 11BE; # (딏; 딏; 딏; 딏; 딏; ) HANGUL SYLLABLE DYIC +B510;B510;1103 1174 11BF;B510;1103 1174 11BF; # (딐; 딐; 딐; 딐; 딐; ) HANGUL SYLLABLE DYIK +B511;B511;1103 1174 11C0;B511;1103 1174 11C0; # (딑; 딑; 딑; 딑; 딑; ) HANGUL SYLLABLE DYIT +B512;B512;1103 1174 11C1;B512;1103 1174 11C1; # (딒; 딒; 딒; 딒; 딒; ) HANGUL SYLLABLE DYIP +B513;B513;1103 1174 11C2;B513;1103 1174 11C2; # (딓; 딓; 딓; 딓; 딓; ) HANGUL SYLLABLE DYIH +B514;B514;1103 1175;B514;1103 1175; # (디; 디; 디; 디; 디; ) HANGUL SYLLABLE DI +B515;B515;1103 1175 11A8;B515;1103 1175 11A8; # (딕; 딕; 딕; 딕; 딕; ) HANGUL SYLLABLE DIG +B516;B516;1103 1175 11A9;B516;1103 1175 11A9; # (딖; 딖; 딖; 딖; 딖; ) HANGUL SYLLABLE DIGG +B517;B517;1103 1175 11AA;B517;1103 1175 11AA; # (딗; 딗; 딗; 딗; 딗; ) HANGUL SYLLABLE DIGS +B518;B518;1103 1175 11AB;B518;1103 1175 11AB; # (딘; 딘; 딘; 딘; 딘; ) HANGUL SYLLABLE DIN +B519;B519;1103 1175 11AC;B519;1103 1175 11AC; # (딙; 딙; 딙; 딙; 딙; ) HANGUL SYLLABLE DINJ +B51A;B51A;1103 1175 11AD;B51A;1103 1175 11AD; # (딚; 딚; 딚; 딚; 딚; ) HANGUL SYLLABLE DINH +B51B;B51B;1103 1175 11AE;B51B;1103 1175 11AE; # (딛; 딛; 딛; 딛; 딛; ) HANGUL SYLLABLE DID +B51C;B51C;1103 1175 11AF;B51C;1103 1175 11AF; # (딜; 딜; 딜; 딜; 딜; ) HANGUL SYLLABLE DIL +B51D;B51D;1103 1175 11B0;B51D;1103 1175 11B0; # (딝; 딝; 딝; 딝; 딝; ) HANGUL SYLLABLE DILG +B51E;B51E;1103 1175 11B1;B51E;1103 1175 11B1; # (딞; 딞; 딞; 딞; 딞; ) HANGUL SYLLABLE DILM +B51F;B51F;1103 1175 11B2;B51F;1103 1175 11B2; # (딟; 딟; 딟; 딟; 딟; ) HANGUL SYLLABLE DILB +B520;B520;1103 1175 11B3;B520;1103 1175 11B3; # (딠; 딠; 딠; 딠; 딠; ) HANGUL SYLLABLE DILS +B521;B521;1103 1175 11B4;B521;1103 1175 11B4; # (딡; 딡; 딡; 딡; 딡; ) HANGUL SYLLABLE DILT +B522;B522;1103 1175 11B5;B522;1103 1175 11B5; # (딢; 딢; 딢; 딢; 딢; ) HANGUL SYLLABLE DILP +B523;B523;1103 1175 11B6;B523;1103 1175 11B6; # (딣; 딣; 딣; 딣; 딣; ) HANGUL SYLLABLE DILH +B524;B524;1103 1175 11B7;B524;1103 1175 11B7; # (딤; 딤; 딤; 딤; 딤; ) HANGUL SYLLABLE DIM +B525;B525;1103 1175 11B8;B525;1103 1175 11B8; # (딥; 딥; 딥; 딥; 딥; ) HANGUL SYLLABLE DIB +B526;B526;1103 1175 11B9;B526;1103 1175 11B9; # (딦; 딦; 딦; 딦; 딦; ) HANGUL SYLLABLE DIBS +B527;B527;1103 1175 11BA;B527;1103 1175 11BA; # (딧; 딧; 딧; 딧; 딧; ) HANGUL SYLLABLE DIS +B528;B528;1103 1175 11BB;B528;1103 1175 11BB; # (딨; 딨; 딨; 딨; 딨; ) HANGUL SYLLABLE DISS +B529;B529;1103 1175 11BC;B529;1103 1175 11BC; # (딩; 딩; 딩; 딩; 딩; ) HANGUL SYLLABLE DING +B52A;B52A;1103 1175 11BD;B52A;1103 1175 11BD; # (딪; 딪; 딪; 딪; 딪; ) HANGUL SYLLABLE DIJ +B52B;B52B;1103 1175 11BE;B52B;1103 1175 11BE; # (딫; 딫; 딫; 딫; 딫; ) HANGUL SYLLABLE DIC +B52C;B52C;1103 1175 11BF;B52C;1103 1175 11BF; # (딬; 딬; 딬; 딬; 딬; ) HANGUL SYLLABLE DIK +B52D;B52D;1103 1175 11C0;B52D;1103 1175 11C0; # (딭; 딭; 딭; 딭; 딭; ) HANGUL SYLLABLE DIT +B52E;B52E;1103 1175 11C1;B52E;1103 1175 11C1; # (딮; 딮; 딮; 딮; 딮; ) HANGUL SYLLABLE DIP +B52F;B52F;1103 1175 11C2;B52F;1103 1175 11C2; # (딯; 딯; 딯; 딯; 딯; ) HANGUL SYLLABLE DIH +B530;B530;1104 1161;B530;1104 1161; # (따; 따; 따; 따; 따; ) HANGUL SYLLABLE DDA +B531;B531;1104 1161 11A8;B531;1104 1161 11A8; # (딱; 딱; 딱; 딱; 딱; ) HANGUL SYLLABLE DDAG +B532;B532;1104 1161 11A9;B532;1104 1161 11A9; # (딲; 딲; 딲; 딲; 딲; ) HANGUL SYLLABLE DDAGG +B533;B533;1104 1161 11AA;B533;1104 1161 11AA; # (딳; 딳; 딳; 딳; 딳; ) HANGUL SYLLABLE DDAGS +B534;B534;1104 1161 11AB;B534;1104 1161 11AB; # (딴; 딴; 딴; 딴; 딴; ) HANGUL SYLLABLE DDAN +B535;B535;1104 1161 11AC;B535;1104 1161 11AC; # (딵; 딵; 딵; 딵; 딵; ) HANGUL SYLLABLE DDANJ +B536;B536;1104 1161 11AD;B536;1104 1161 11AD; # (딶; 딶; 딶; 딶; 딶; ) HANGUL SYLLABLE DDANH +B537;B537;1104 1161 11AE;B537;1104 1161 11AE; # (딷; 딷; 딷; 딷; 딷; ) HANGUL SYLLABLE DDAD +B538;B538;1104 1161 11AF;B538;1104 1161 11AF; # (딸; 딸; 딸; 딸; 딸; ) HANGUL SYLLABLE DDAL +B539;B539;1104 1161 11B0;B539;1104 1161 11B0; # (딹; 딹; 딹; 딹; 딹; ) HANGUL SYLLABLE DDALG +B53A;B53A;1104 1161 11B1;B53A;1104 1161 11B1; # (딺; 딺; 딺; 딺; 딺; ) HANGUL SYLLABLE DDALM +B53B;B53B;1104 1161 11B2;B53B;1104 1161 11B2; # (딻; 딻; 딻; 딻; 딻; ) HANGUL SYLLABLE DDALB +B53C;B53C;1104 1161 11B3;B53C;1104 1161 11B3; # (딼; 딼; 딼; 딼; 딼; ) HANGUL SYLLABLE DDALS +B53D;B53D;1104 1161 11B4;B53D;1104 1161 11B4; # (딽; 딽; 딽; 딽; 딽; ) HANGUL SYLLABLE DDALT +B53E;B53E;1104 1161 11B5;B53E;1104 1161 11B5; # (딾; 딾; 딾; 딾; 딾; ) HANGUL SYLLABLE DDALP +B53F;B53F;1104 1161 11B6;B53F;1104 1161 11B6; # (딿; 딿; 딿; 딿; 딿; ) HANGUL SYLLABLE DDALH +B540;B540;1104 1161 11B7;B540;1104 1161 11B7; # (땀; 땀; 땀; 땀; 땀; ) HANGUL SYLLABLE DDAM +B541;B541;1104 1161 11B8;B541;1104 1161 11B8; # (땁; 땁; 땁; 땁; 땁; ) HANGUL SYLLABLE DDAB +B542;B542;1104 1161 11B9;B542;1104 1161 11B9; # (땂; 땂; 땂; 땂; 땂; ) HANGUL SYLLABLE DDABS +B543;B543;1104 1161 11BA;B543;1104 1161 11BA; # (땃; 땃; 땃; 땃; 땃; ) HANGUL SYLLABLE DDAS +B544;B544;1104 1161 11BB;B544;1104 1161 11BB; # (땄; 땄; 땄; 땄; 땄; ) HANGUL SYLLABLE DDASS +B545;B545;1104 1161 11BC;B545;1104 1161 11BC; # (땅; 땅; 땅; 땅; 땅; ) HANGUL SYLLABLE DDANG +B546;B546;1104 1161 11BD;B546;1104 1161 11BD; # (땆; 땆; 땆; 땆; 땆; ) HANGUL SYLLABLE DDAJ +B547;B547;1104 1161 11BE;B547;1104 1161 11BE; # (땇; 땇; 땇; 땇; 땇; ) HANGUL SYLLABLE DDAC +B548;B548;1104 1161 11BF;B548;1104 1161 11BF; # (땈; 땈; 땈; 땈; 땈; ) HANGUL SYLLABLE DDAK +B549;B549;1104 1161 11C0;B549;1104 1161 11C0; # (땉; 땉; 땉; 땉; 땉; ) HANGUL SYLLABLE DDAT +B54A;B54A;1104 1161 11C1;B54A;1104 1161 11C1; # (땊; 땊; 땊; 땊; 땊; ) HANGUL SYLLABLE DDAP +B54B;B54B;1104 1161 11C2;B54B;1104 1161 11C2; # (땋; 땋; 땋; 땋; 땋; ) HANGUL SYLLABLE DDAH +B54C;B54C;1104 1162;B54C;1104 1162; # (때; 때; 때; 때; 때; ) HANGUL SYLLABLE DDAE +B54D;B54D;1104 1162 11A8;B54D;1104 1162 11A8; # (땍; 땍; 땍; 땍; 땍; ) HANGUL SYLLABLE DDAEG +B54E;B54E;1104 1162 11A9;B54E;1104 1162 11A9; # (땎; 땎; 땎; 땎; 땎; ) HANGUL SYLLABLE DDAEGG +B54F;B54F;1104 1162 11AA;B54F;1104 1162 11AA; # (땏; 땏; 땏; 땏; 땏; ) HANGUL SYLLABLE DDAEGS +B550;B550;1104 1162 11AB;B550;1104 1162 11AB; # (땐; 땐; 땐; 땐; 땐; ) HANGUL SYLLABLE DDAEN +B551;B551;1104 1162 11AC;B551;1104 1162 11AC; # (땑; 땑; 땑; 땑; 땑; ) HANGUL SYLLABLE DDAENJ +B552;B552;1104 1162 11AD;B552;1104 1162 11AD; # (땒; 땒; 땒; 땒; 땒; ) HANGUL SYLLABLE DDAENH +B553;B553;1104 1162 11AE;B553;1104 1162 11AE; # (땓; 땓; 땓; 땓; 땓; ) HANGUL SYLLABLE DDAED +B554;B554;1104 1162 11AF;B554;1104 1162 11AF; # (땔; 땔; 땔; 땔; 땔; ) HANGUL SYLLABLE DDAEL +B555;B555;1104 1162 11B0;B555;1104 1162 11B0; # (땕; 땕; 땕; 땕; 땕; ) HANGUL SYLLABLE DDAELG +B556;B556;1104 1162 11B1;B556;1104 1162 11B1; # (땖; 땖; 땖; 땖; 땖; ) HANGUL SYLLABLE DDAELM +B557;B557;1104 1162 11B2;B557;1104 1162 11B2; # (땗; 땗; 땗; 땗; 땗; ) HANGUL SYLLABLE DDAELB +B558;B558;1104 1162 11B3;B558;1104 1162 11B3; # (땘; 땘; 땘; 땘; 땘; ) HANGUL SYLLABLE DDAELS +B559;B559;1104 1162 11B4;B559;1104 1162 11B4; # (땙; 땙; 땙; 땙; 땙; ) HANGUL SYLLABLE DDAELT +B55A;B55A;1104 1162 11B5;B55A;1104 1162 11B5; # (땚; 땚; 땚; 땚; 땚; ) HANGUL SYLLABLE DDAELP +B55B;B55B;1104 1162 11B6;B55B;1104 1162 11B6; # (땛; 땛; 땛; 땛; 땛; ) HANGUL SYLLABLE DDAELH +B55C;B55C;1104 1162 11B7;B55C;1104 1162 11B7; # (땜; 땜; 땜; 땜; 땜; ) HANGUL SYLLABLE DDAEM +B55D;B55D;1104 1162 11B8;B55D;1104 1162 11B8; # (땝; 땝; 땝; 땝; 땝; ) HANGUL SYLLABLE DDAEB +B55E;B55E;1104 1162 11B9;B55E;1104 1162 11B9; # (땞; 땞; 땞; 땞; 땞; ) HANGUL SYLLABLE DDAEBS +B55F;B55F;1104 1162 11BA;B55F;1104 1162 11BA; # (땟; 땟; 땟; 땟; 땟; ) HANGUL SYLLABLE DDAES +B560;B560;1104 1162 11BB;B560;1104 1162 11BB; # (땠; 땠; 땠; 땠; 땠; ) HANGUL SYLLABLE DDAESS +B561;B561;1104 1162 11BC;B561;1104 1162 11BC; # (땡; 땡; 땡; 땡; 땡; ) HANGUL SYLLABLE DDAENG +B562;B562;1104 1162 11BD;B562;1104 1162 11BD; # (땢; 땢; 땢; 땢; 땢; ) HANGUL SYLLABLE DDAEJ +B563;B563;1104 1162 11BE;B563;1104 1162 11BE; # (땣; 땣; 땣; 땣; 땣; ) HANGUL SYLLABLE DDAEC +B564;B564;1104 1162 11BF;B564;1104 1162 11BF; # (땤; 땤; 땤; 땤; 땤; ) HANGUL SYLLABLE DDAEK +B565;B565;1104 1162 11C0;B565;1104 1162 11C0; # (땥; 땥; 땥; 땥; 땥; ) HANGUL SYLLABLE DDAET +B566;B566;1104 1162 11C1;B566;1104 1162 11C1; # (땦; 땦; 땦; 땦; 땦; ) HANGUL SYLLABLE DDAEP +B567;B567;1104 1162 11C2;B567;1104 1162 11C2; # (땧; 땧; 땧; 땧; 땧; ) HANGUL SYLLABLE DDAEH +B568;B568;1104 1163;B568;1104 1163; # (땨; 땨; 땨; 땨; 땨; ) HANGUL SYLLABLE DDYA +B569;B569;1104 1163 11A8;B569;1104 1163 11A8; # (땩; 땩; 땩; 땩; 땩; ) HANGUL SYLLABLE DDYAG +B56A;B56A;1104 1163 11A9;B56A;1104 1163 11A9; # (땪; 땪; 땪; 땪; 땪; ) HANGUL SYLLABLE DDYAGG +B56B;B56B;1104 1163 11AA;B56B;1104 1163 11AA; # (땫; 땫; 땫; 땫; 땫; ) HANGUL SYLLABLE DDYAGS +B56C;B56C;1104 1163 11AB;B56C;1104 1163 11AB; # (땬; 땬; 땬; 땬; 땬; ) HANGUL SYLLABLE DDYAN +B56D;B56D;1104 1163 11AC;B56D;1104 1163 11AC; # (땭; 땭; 땭; 땭; 땭; ) HANGUL SYLLABLE DDYANJ +B56E;B56E;1104 1163 11AD;B56E;1104 1163 11AD; # (땮; 땮; 땮; 땮; 땮; ) HANGUL SYLLABLE DDYANH +B56F;B56F;1104 1163 11AE;B56F;1104 1163 11AE; # (땯; 땯; 땯; 땯; 땯; ) HANGUL SYLLABLE DDYAD +B570;B570;1104 1163 11AF;B570;1104 1163 11AF; # (땰; 땰; 땰; 땰; 땰; ) HANGUL SYLLABLE DDYAL +B571;B571;1104 1163 11B0;B571;1104 1163 11B0; # (땱; 땱; 땱; 땱; 땱; ) HANGUL SYLLABLE DDYALG +B572;B572;1104 1163 11B1;B572;1104 1163 11B1; # (땲; 땲; 땲; 땲; 땲; ) HANGUL SYLLABLE DDYALM +B573;B573;1104 1163 11B2;B573;1104 1163 11B2; # (땳; 땳; 땳; 땳; 땳; ) HANGUL SYLLABLE DDYALB +B574;B574;1104 1163 11B3;B574;1104 1163 11B3; # (땴; 땴; 땴; 땴; 땴; ) HANGUL SYLLABLE DDYALS +B575;B575;1104 1163 11B4;B575;1104 1163 11B4; # (땵; 땵; 땵; 땵; 땵; ) HANGUL SYLLABLE DDYALT +B576;B576;1104 1163 11B5;B576;1104 1163 11B5; # (땶; 땶; 땶; 땶; 땶; ) HANGUL SYLLABLE DDYALP +B577;B577;1104 1163 11B6;B577;1104 1163 11B6; # (땷; 땷; 땷; 땷; 땷; ) HANGUL SYLLABLE DDYALH +B578;B578;1104 1163 11B7;B578;1104 1163 11B7; # (땸; 땸; 땸; 땸; 땸; ) HANGUL SYLLABLE DDYAM +B579;B579;1104 1163 11B8;B579;1104 1163 11B8; # (땹; 땹; 땹; 땹; 땹; ) HANGUL SYLLABLE DDYAB +B57A;B57A;1104 1163 11B9;B57A;1104 1163 11B9; # (땺; 땺; 땺; 땺; 땺; ) HANGUL SYLLABLE DDYABS +B57B;B57B;1104 1163 11BA;B57B;1104 1163 11BA; # (땻; 땻; 땻; 땻; 땻; ) HANGUL SYLLABLE DDYAS +B57C;B57C;1104 1163 11BB;B57C;1104 1163 11BB; # (땼; 땼; 땼; 땼; 땼; ) HANGUL SYLLABLE DDYASS +B57D;B57D;1104 1163 11BC;B57D;1104 1163 11BC; # (땽; 땽; 땽; 땽; 땽; ) HANGUL SYLLABLE DDYANG +B57E;B57E;1104 1163 11BD;B57E;1104 1163 11BD; # (땾; 땾; 땾; 땾; 땾; ) HANGUL SYLLABLE DDYAJ +B57F;B57F;1104 1163 11BE;B57F;1104 1163 11BE; # (땿; 땿; 땿; 땿; 땿; ) HANGUL SYLLABLE DDYAC +B580;B580;1104 1163 11BF;B580;1104 1163 11BF; # (떀; 떀; 떀; 떀; 떀; ) HANGUL SYLLABLE DDYAK +B581;B581;1104 1163 11C0;B581;1104 1163 11C0; # (떁; 떁; 떁; 떁; 떁; ) HANGUL SYLLABLE DDYAT +B582;B582;1104 1163 11C1;B582;1104 1163 11C1; # (떂; 떂; 떂; 떂; 떂; ) HANGUL SYLLABLE DDYAP +B583;B583;1104 1163 11C2;B583;1104 1163 11C2; # (떃; 떃; 떃; 떃; 떃; ) HANGUL SYLLABLE DDYAH +B584;B584;1104 1164;B584;1104 1164; # (떄; 떄; 떄; 떄; 떄; ) HANGUL SYLLABLE DDYAE +B585;B585;1104 1164 11A8;B585;1104 1164 11A8; # (떅; 떅; 떅; 떅; 떅; ) HANGUL SYLLABLE DDYAEG +B586;B586;1104 1164 11A9;B586;1104 1164 11A9; # (떆; 떆; 떆; 떆; 떆; ) HANGUL SYLLABLE DDYAEGG +B587;B587;1104 1164 11AA;B587;1104 1164 11AA; # (떇; 떇; 떇; 떇; 떇; ) HANGUL SYLLABLE DDYAEGS +B588;B588;1104 1164 11AB;B588;1104 1164 11AB; # (떈; 떈; 떈; 떈; 떈; ) HANGUL SYLLABLE DDYAEN +B589;B589;1104 1164 11AC;B589;1104 1164 11AC; # (떉; 떉; 떉; 떉; 떉; ) HANGUL SYLLABLE DDYAENJ +B58A;B58A;1104 1164 11AD;B58A;1104 1164 11AD; # (떊; 떊; 떊; 떊; 떊; ) HANGUL SYLLABLE DDYAENH +B58B;B58B;1104 1164 11AE;B58B;1104 1164 11AE; # (떋; 떋; 떋; 떋; 떋; ) HANGUL SYLLABLE DDYAED +B58C;B58C;1104 1164 11AF;B58C;1104 1164 11AF; # (떌; 떌; 떌; 떌; 떌; ) HANGUL SYLLABLE DDYAEL +B58D;B58D;1104 1164 11B0;B58D;1104 1164 11B0; # (떍; 떍; 떍; 떍; 떍; ) HANGUL SYLLABLE DDYAELG +B58E;B58E;1104 1164 11B1;B58E;1104 1164 11B1; # (떎; 떎; 떎; 떎; 떎; ) HANGUL SYLLABLE DDYAELM +B58F;B58F;1104 1164 11B2;B58F;1104 1164 11B2; # (떏; 떏; 떏; 떏; 떏; ) HANGUL SYLLABLE DDYAELB +B590;B590;1104 1164 11B3;B590;1104 1164 11B3; # (떐; 떐; 떐; 떐; 떐; ) HANGUL SYLLABLE DDYAELS +B591;B591;1104 1164 11B4;B591;1104 1164 11B4; # (떑; 떑; 떑; 떑; 떑; ) HANGUL SYLLABLE DDYAELT +B592;B592;1104 1164 11B5;B592;1104 1164 11B5; # (떒; 떒; 떒; 떒; 떒; ) HANGUL SYLLABLE DDYAELP +B593;B593;1104 1164 11B6;B593;1104 1164 11B6; # (떓; 떓; 떓; 떓; 떓; ) HANGUL SYLLABLE DDYAELH +B594;B594;1104 1164 11B7;B594;1104 1164 11B7; # (떔; 떔; 떔; 떔; 떔; ) HANGUL SYLLABLE DDYAEM +B595;B595;1104 1164 11B8;B595;1104 1164 11B8; # (떕; 떕; 떕; 떕; 떕; ) HANGUL SYLLABLE DDYAEB +B596;B596;1104 1164 11B9;B596;1104 1164 11B9; # (떖; 떖; 떖; 떖; 떖; ) HANGUL SYLLABLE DDYAEBS +B597;B597;1104 1164 11BA;B597;1104 1164 11BA; # (떗; 떗; 떗; 떗; 떗; ) HANGUL SYLLABLE DDYAES +B598;B598;1104 1164 11BB;B598;1104 1164 11BB; # (떘; 떘; 떘; 떘; 떘; ) HANGUL SYLLABLE DDYAESS +B599;B599;1104 1164 11BC;B599;1104 1164 11BC; # (떙; 떙; 떙; 떙; 떙; ) HANGUL SYLLABLE DDYAENG +B59A;B59A;1104 1164 11BD;B59A;1104 1164 11BD; # (떚; 떚; 떚; 떚; 떚; ) HANGUL SYLLABLE DDYAEJ +B59B;B59B;1104 1164 11BE;B59B;1104 1164 11BE; # (떛; 떛; 떛; 떛; 떛; ) HANGUL SYLLABLE DDYAEC +B59C;B59C;1104 1164 11BF;B59C;1104 1164 11BF; # (떜; 떜; 떜; 떜; 떜; ) HANGUL SYLLABLE DDYAEK +B59D;B59D;1104 1164 11C0;B59D;1104 1164 11C0; # (떝; 떝; 떝; 떝; 떝; ) HANGUL SYLLABLE DDYAET +B59E;B59E;1104 1164 11C1;B59E;1104 1164 11C1; # (떞; 떞; 떞; 떞; 떞; ) HANGUL SYLLABLE DDYAEP +B59F;B59F;1104 1164 11C2;B59F;1104 1164 11C2; # (떟; 떟; 떟; 떟; 떟; ) HANGUL SYLLABLE DDYAEH +B5A0;B5A0;1104 1165;B5A0;1104 1165; # (떠; 떠; 떠; 떠; 떠; ) HANGUL SYLLABLE DDEO +B5A1;B5A1;1104 1165 11A8;B5A1;1104 1165 11A8; # (떡; 떡; 떡; 떡; 떡; ) HANGUL SYLLABLE DDEOG +B5A2;B5A2;1104 1165 11A9;B5A2;1104 1165 11A9; # (떢; 떢; 떢; 떢; 떢; ) HANGUL SYLLABLE DDEOGG +B5A3;B5A3;1104 1165 11AA;B5A3;1104 1165 11AA; # (떣; 떣; 떣; 떣; 떣; ) HANGUL SYLLABLE DDEOGS +B5A4;B5A4;1104 1165 11AB;B5A4;1104 1165 11AB; # (떤; 떤; 떤; 떤; 떤; ) HANGUL SYLLABLE DDEON +B5A5;B5A5;1104 1165 11AC;B5A5;1104 1165 11AC; # (떥; 떥; 떥; 떥; 떥; ) HANGUL SYLLABLE DDEONJ +B5A6;B5A6;1104 1165 11AD;B5A6;1104 1165 11AD; # (떦; 떦; 떦; 떦; 떦; ) HANGUL SYLLABLE DDEONH +B5A7;B5A7;1104 1165 11AE;B5A7;1104 1165 11AE; # (떧; 떧; 떧; 떧; 떧; ) HANGUL SYLLABLE DDEOD +B5A8;B5A8;1104 1165 11AF;B5A8;1104 1165 11AF; # (떨; 떨; 떨; 떨; 떨; ) HANGUL SYLLABLE DDEOL +B5A9;B5A9;1104 1165 11B0;B5A9;1104 1165 11B0; # (떩; 떩; 떩; 떩; 떩; ) HANGUL SYLLABLE DDEOLG +B5AA;B5AA;1104 1165 11B1;B5AA;1104 1165 11B1; # (떪; 떪; 떪; 떪; 떪; ) HANGUL SYLLABLE DDEOLM +B5AB;B5AB;1104 1165 11B2;B5AB;1104 1165 11B2; # (떫; 떫; 떫; 떫; 떫; ) HANGUL SYLLABLE DDEOLB +B5AC;B5AC;1104 1165 11B3;B5AC;1104 1165 11B3; # (떬; 떬; 떬; 떬; 떬; ) HANGUL SYLLABLE DDEOLS +B5AD;B5AD;1104 1165 11B4;B5AD;1104 1165 11B4; # (떭; 떭; 떭; 떭; 떭; ) HANGUL SYLLABLE DDEOLT +B5AE;B5AE;1104 1165 11B5;B5AE;1104 1165 11B5; # (떮; 떮; 떮; 떮; 떮; ) HANGUL SYLLABLE DDEOLP +B5AF;B5AF;1104 1165 11B6;B5AF;1104 1165 11B6; # (떯; 떯; 떯; 떯; 떯; ) HANGUL SYLLABLE DDEOLH +B5B0;B5B0;1104 1165 11B7;B5B0;1104 1165 11B7; # (떰; 떰; 떰; 떰; 떰; ) HANGUL SYLLABLE DDEOM +B5B1;B5B1;1104 1165 11B8;B5B1;1104 1165 11B8; # (떱; 떱; 떱; 떱; 떱; ) HANGUL SYLLABLE DDEOB +B5B2;B5B2;1104 1165 11B9;B5B2;1104 1165 11B9; # (떲; 떲; 떲; 떲; 떲; ) HANGUL SYLLABLE DDEOBS +B5B3;B5B3;1104 1165 11BA;B5B3;1104 1165 11BA; # (떳; 떳; 떳; 떳; 떳; ) HANGUL SYLLABLE DDEOS +B5B4;B5B4;1104 1165 11BB;B5B4;1104 1165 11BB; # (떴; 떴; 떴; 떴; 떴; ) HANGUL SYLLABLE DDEOSS +B5B5;B5B5;1104 1165 11BC;B5B5;1104 1165 11BC; # (떵; 떵; 떵; 떵; 떵; ) HANGUL SYLLABLE DDEONG +B5B6;B5B6;1104 1165 11BD;B5B6;1104 1165 11BD; # (떶; 떶; 떶; 떶; 떶; ) HANGUL SYLLABLE DDEOJ +B5B7;B5B7;1104 1165 11BE;B5B7;1104 1165 11BE; # (떷; 떷; 떷; 떷; 떷; ) HANGUL SYLLABLE DDEOC +B5B8;B5B8;1104 1165 11BF;B5B8;1104 1165 11BF; # (떸; 떸; 떸; 떸; 떸; ) HANGUL SYLLABLE DDEOK +B5B9;B5B9;1104 1165 11C0;B5B9;1104 1165 11C0; # (떹; 떹; 떹; 떹; 떹; ) HANGUL SYLLABLE DDEOT +B5BA;B5BA;1104 1165 11C1;B5BA;1104 1165 11C1; # (떺; 떺; 떺; 떺; 떺; ) HANGUL SYLLABLE DDEOP +B5BB;B5BB;1104 1165 11C2;B5BB;1104 1165 11C2; # (떻; 떻; 떻; 떻; 떻; ) HANGUL SYLLABLE DDEOH +B5BC;B5BC;1104 1166;B5BC;1104 1166; # (떼; 떼; 떼; 떼; 떼; ) HANGUL SYLLABLE DDE +B5BD;B5BD;1104 1166 11A8;B5BD;1104 1166 11A8; # (떽; 떽; 떽; 떽; 떽; ) HANGUL SYLLABLE DDEG +B5BE;B5BE;1104 1166 11A9;B5BE;1104 1166 11A9; # (떾; 떾; 떾; 떾; 떾; ) HANGUL SYLLABLE DDEGG +B5BF;B5BF;1104 1166 11AA;B5BF;1104 1166 11AA; # (떿; 떿; 떿; 떿; 떿; ) HANGUL SYLLABLE DDEGS +B5C0;B5C0;1104 1166 11AB;B5C0;1104 1166 11AB; # (뗀; 뗀; 뗀; 뗀; 뗀; ) HANGUL SYLLABLE DDEN +B5C1;B5C1;1104 1166 11AC;B5C1;1104 1166 11AC; # (뗁; 뗁; 뗁; 뗁; 뗁; ) HANGUL SYLLABLE DDENJ +B5C2;B5C2;1104 1166 11AD;B5C2;1104 1166 11AD; # (뗂; 뗂; 뗂; 뗂; 뗂; ) HANGUL SYLLABLE DDENH +B5C3;B5C3;1104 1166 11AE;B5C3;1104 1166 11AE; # (뗃; 뗃; 뗃; 뗃; 뗃; ) HANGUL SYLLABLE DDED +B5C4;B5C4;1104 1166 11AF;B5C4;1104 1166 11AF; # (뗄; 뗄; 뗄; 뗄; 뗄; ) HANGUL SYLLABLE DDEL +B5C5;B5C5;1104 1166 11B0;B5C5;1104 1166 11B0; # (뗅; 뗅; 뗅; 뗅; 뗅; ) HANGUL SYLLABLE DDELG +B5C6;B5C6;1104 1166 11B1;B5C6;1104 1166 11B1; # (뗆; 뗆; 뗆; 뗆; 뗆; ) HANGUL SYLLABLE DDELM +B5C7;B5C7;1104 1166 11B2;B5C7;1104 1166 11B2; # (뗇; 뗇; 뗇; 뗇; 뗇; ) HANGUL SYLLABLE DDELB +B5C8;B5C8;1104 1166 11B3;B5C8;1104 1166 11B3; # (뗈; 뗈; 뗈; 뗈; 뗈; ) HANGUL SYLLABLE DDELS +B5C9;B5C9;1104 1166 11B4;B5C9;1104 1166 11B4; # (뗉; 뗉; 뗉; 뗉; 뗉; ) HANGUL SYLLABLE DDELT +B5CA;B5CA;1104 1166 11B5;B5CA;1104 1166 11B5; # (뗊; 뗊; 뗊; 뗊; 뗊; ) HANGUL SYLLABLE DDELP +B5CB;B5CB;1104 1166 11B6;B5CB;1104 1166 11B6; # (뗋; 뗋; 뗋; 뗋; 뗋; ) HANGUL SYLLABLE DDELH +B5CC;B5CC;1104 1166 11B7;B5CC;1104 1166 11B7; # (뗌; 뗌; 뗌; 뗌; 뗌; ) HANGUL SYLLABLE DDEM +B5CD;B5CD;1104 1166 11B8;B5CD;1104 1166 11B8; # (뗍; 뗍; 뗍; 뗍; 뗍; ) HANGUL SYLLABLE DDEB +B5CE;B5CE;1104 1166 11B9;B5CE;1104 1166 11B9; # (뗎; 뗎; 뗎; 뗎; 뗎; ) HANGUL SYLLABLE DDEBS +B5CF;B5CF;1104 1166 11BA;B5CF;1104 1166 11BA; # (뗏; 뗏; 뗏; 뗏; 뗏; ) HANGUL SYLLABLE DDES +B5D0;B5D0;1104 1166 11BB;B5D0;1104 1166 11BB; # (뗐; 뗐; 뗐; 뗐; 뗐; ) HANGUL SYLLABLE DDESS +B5D1;B5D1;1104 1166 11BC;B5D1;1104 1166 11BC; # (뗑; 뗑; 뗑; 뗑; 뗑; ) HANGUL SYLLABLE DDENG +B5D2;B5D2;1104 1166 11BD;B5D2;1104 1166 11BD; # (뗒; 뗒; 뗒; 뗒; 뗒; ) HANGUL SYLLABLE DDEJ +B5D3;B5D3;1104 1166 11BE;B5D3;1104 1166 11BE; # (뗓; 뗓; 뗓; 뗓; 뗓; ) HANGUL SYLLABLE DDEC +B5D4;B5D4;1104 1166 11BF;B5D4;1104 1166 11BF; # (뗔; 뗔; 뗔; 뗔; 뗔; ) HANGUL SYLLABLE DDEK +B5D5;B5D5;1104 1166 11C0;B5D5;1104 1166 11C0; # (뗕; 뗕; 뗕; 뗕; 뗕; ) HANGUL SYLLABLE DDET +B5D6;B5D6;1104 1166 11C1;B5D6;1104 1166 11C1; # (뗖; 뗖; 뗖; 뗖; 뗖; ) HANGUL SYLLABLE DDEP +B5D7;B5D7;1104 1166 11C2;B5D7;1104 1166 11C2; # (뗗; 뗗; 뗗; 뗗; 뗗; ) HANGUL SYLLABLE DDEH +B5D8;B5D8;1104 1167;B5D8;1104 1167; # (뗘; 뗘; 뗘; 뗘; 뗘; ) HANGUL SYLLABLE DDYEO +B5D9;B5D9;1104 1167 11A8;B5D9;1104 1167 11A8; # (뗙; 뗙; 뗙; 뗙; 뗙; ) HANGUL SYLLABLE DDYEOG +B5DA;B5DA;1104 1167 11A9;B5DA;1104 1167 11A9; # (뗚; 뗚; 뗚; 뗚; 뗚; ) HANGUL SYLLABLE DDYEOGG +B5DB;B5DB;1104 1167 11AA;B5DB;1104 1167 11AA; # (뗛; 뗛; 뗛; 뗛; 뗛; ) HANGUL SYLLABLE DDYEOGS +B5DC;B5DC;1104 1167 11AB;B5DC;1104 1167 11AB; # (뗜; 뗜; 뗜; 뗜; 뗜; ) HANGUL SYLLABLE DDYEON +B5DD;B5DD;1104 1167 11AC;B5DD;1104 1167 11AC; # (뗝; 뗝; 뗝; 뗝; 뗝; ) HANGUL SYLLABLE DDYEONJ +B5DE;B5DE;1104 1167 11AD;B5DE;1104 1167 11AD; # (뗞; 뗞; 뗞; 뗞; 뗞; ) HANGUL SYLLABLE DDYEONH +B5DF;B5DF;1104 1167 11AE;B5DF;1104 1167 11AE; # (뗟; 뗟; 뗟; 뗟; 뗟; ) HANGUL SYLLABLE DDYEOD +B5E0;B5E0;1104 1167 11AF;B5E0;1104 1167 11AF; # (뗠; 뗠; 뗠; 뗠; 뗠; ) HANGUL SYLLABLE DDYEOL +B5E1;B5E1;1104 1167 11B0;B5E1;1104 1167 11B0; # (뗡; 뗡; 뗡; 뗡; 뗡; ) HANGUL SYLLABLE DDYEOLG +B5E2;B5E2;1104 1167 11B1;B5E2;1104 1167 11B1; # (뗢; 뗢; 뗢; 뗢; 뗢; ) HANGUL SYLLABLE DDYEOLM +B5E3;B5E3;1104 1167 11B2;B5E3;1104 1167 11B2; # (뗣; 뗣; 뗣; 뗣; 뗣; ) HANGUL SYLLABLE DDYEOLB +B5E4;B5E4;1104 1167 11B3;B5E4;1104 1167 11B3; # (뗤; 뗤; 뗤; 뗤; 뗤; ) HANGUL SYLLABLE DDYEOLS +B5E5;B5E5;1104 1167 11B4;B5E5;1104 1167 11B4; # (뗥; 뗥; 뗥; 뗥; 뗥; ) HANGUL SYLLABLE DDYEOLT +B5E6;B5E6;1104 1167 11B5;B5E6;1104 1167 11B5; # (뗦; 뗦; 뗦; 뗦; 뗦; ) HANGUL SYLLABLE DDYEOLP +B5E7;B5E7;1104 1167 11B6;B5E7;1104 1167 11B6; # (뗧; 뗧; 뗧; 뗧; 뗧; ) HANGUL SYLLABLE DDYEOLH +B5E8;B5E8;1104 1167 11B7;B5E8;1104 1167 11B7; # (뗨; 뗨; 뗨; 뗨; 뗨; ) HANGUL SYLLABLE DDYEOM +B5E9;B5E9;1104 1167 11B8;B5E9;1104 1167 11B8; # (뗩; 뗩; 뗩; 뗩; 뗩; ) HANGUL SYLLABLE DDYEOB +B5EA;B5EA;1104 1167 11B9;B5EA;1104 1167 11B9; # (뗪; 뗪; 뗪; 뗪; 뗪; ) HANGUL SYLLABLE DDYEOBS +B5EB;B5EB;1104 1167 11BA;B5EB;1104 1167 11BA; # (뗫; 뗫; 뗫; 뗫; 뗫; ) HANGUL SYLLABLE DDYEOS +B5EC;B5EC;1104 1167 11BB;B5EC;1104 1167 11BB; # (뗬; 뗬; 뗬; 뗬; 뗬; ) HANGUL SYLLABLE DDYEOSS +B5ED;B5ED;1104 1167 11BC;B5ED;1104 1167 11BC; # (뗭; 뗭; 뗭; 뗭; 뗭; ) HANGUL SYLLABLE DDYEONG +B5EE;B5EE;1104 1167 11BD;B5EE;1104 1167 11BD; # (뗮; 뗮; 뗮; 뗮; 뗮; ) HANGUL SYLLABLE DDYEOJ +B5EF;B5EF;1104 1167 11BE;B5EF;1104 1167 11BE; # (뗯; 뗯; 뗯; 뗯; 뗯; ) HANGUL SYLLABLE DDYEOC +B5F0;B5F0;1104 1167 11BF;B5F0;1104 1167 11BF; # (뗰; 뗰; 뗰; 뗰; 뗰; ) HANGUL SYLLABLE DDYEOK +B5F1;B5F1;1104 1167 11C0;B5F1;1104 1167 11C0; # (뗱; 뗱; 뗱; 뗱; 뗱; ) HANGUL SYLLABLE DDYEOT +B5F2;B5F2;1104 1167 11C1;B5F2;1104 1167 11C1; # (뗲; 뗲; 뗲; 뗲; 뗲; ) HANGUL SYLLABLE DDYEOP +B5F3;B5F3;1104 1167 11C2;B5F3;1104 1167 11C2; # (뗳; 뗳; 뗳; 뗳; 뗳; ) HANGUL SYLLABLE DDYEOH +B5F4;B5F4;1104 1168;B5F4;1104 1168; # (뗴; 뗴; 뗴; 뗴; 뗴; ) HANGUL SYLLABLE DDYE +B5F5;B5F5;1104 1168 11A8;B5F5;1104 1168 11A8; # (뗵; 뗵; 뗵; 뗵; 뗵; ) HANGUL SYLLABLE DDYEG +B5F6;B5F6;1104 1168 11A9;B5F6;1104 1168 11A9; # (뗶; 뗶; 뗶; 뗶; 뗶; ) HANGUL SYLLABLE DDYEGG +B5F7;B5F7;1104 1168 11AA;B5F7;1104 1168 11AA; # (뗷; 뗷; 뗷; 뗷; 뗷; ) HANGUL SYLLABLE DDYEGS +B5F8;B5F8;1104 1168 11AB;B5F8;1104 1168 11AB; # (뗸; 뗸; 뗸; 뗸; 뗸; ) HANGUL SYLLABLE DDYEN +B5F9;B5F9;1104 1168 11AC;B5F9;1104 1168 11AC; # (뗹; 뗹; 뗹; 뗹; 뗹; ) HANGUL SYLLABLE DDYENJ +B5FA;B5FA;1104 1168 11AD;B5FA;1104 1168 11AD; # (뗺; 뗺; 뗺; 뗺; 뗺; ) HANGUL SYLLABLE DDYENH +B5FB;B5FB;1104 1168 11AE;B5FB;1104 1168 11AE; # (뗻; 뗻; 뗻; 뗻; 뗻; ) HANGUL SYLLABLE DDYED +B5FC;B5FC;1104 1168 11AF;B5FC;1104 1168 11AF; # (뗼; 뗼; 뗼; 뗼; 뗼; ) HANGUL SYLLABLE DDYEL +B5FD;B5FD;1104 1168 11B0;B5FD;1104 1168 11B0; # (뗽; 뗽; 뗽; 뗽; 뗽; ) HANGUL SYLLABLE DDYELG +B5FE;B5FE;1104 1168 11B1;B5FE;1104 1168 11B1; # (뗾; 뗾; 뗾; 뗾; 뗾; ) HANGUL SYLLABLE DDYELM +B5FF;B5FF;1104 1168 11B2;B5FF;1104 1168 11B2; # (뗿; 뗿; 뗿; 뗿; 뗿; ) HANGUL SYLLABLE DDYELB +B600;B600;1104 1168 11B3;B600;1104 1168 11B3; # (똀; 똀; 똀; 똀; 똀; ) HANGUL SYLLABLE DDYELS +B601;B601;1104 1168 11B4;B601;1104 1168 11B4; # (똁; 똁; 똁; 똁; 똁; ) HANGUL SYLLABLE DDYELT +B602;B602;1104 1168 11B5;B602;1104 1168 11B5; # (똂; 똂; 똂; 똂; 똂; ) HANGUL SYLLABLE DDYELP +B603;B603;1104 1168 11B6;B603;1104 1168 11B6; # (똃; 똃; 똃; 똃; 똃; ) HANGUL SYLLABLE DDYELH +B604;B604;1104 1168 11B7;B604;1104 1168 11B7; # (똄; 똄; 똄; 똄; 똄; ) HANGUL SYLLABLE DDYEM +B605;B605;1104 1168 11B8;B605;1104 1168 11B8; # (똅; 똅; 똅; 똅; 똅; ) HANGUL SYLLABLE DDYEB +B606;B606;1104 1168 11B9;B606;1104 1168 11B9; # (똆; 똆; 똆; 똆; 똆; ) HANGUL SYLLABLE DDYEBS +B607;B607;1104 1168 11BA;B607;1104 1168 11BA; # (똇; 똇; 똇; 똇; 똇; ) HANGUL SYLLABLE DDYES +B608;B608;1104 1168 11BB;B608;1104 1168 11BB; # (똈; 똈; 똈; 똈; 똈; ) HANGUL SYLLABLE DDYESS +B609;B609;1104 1168 11BC;B609;1104 1168 11BC; # (똉; 똉; 똉; 똉; 똉; ) HANGUL SYLLABLE DDYENG +B60A;B60A;1104 1168 11BD;B60A;1104 1168 11BD; # (똊; 똊; 똊; 똊; 똊; ) HANGUL SYLLABLE DDYEJ +B60B;B60B;1104 1168 11BE;B60B;1104 1168 11BE; # (똋; 똋; 똋; 똋; 똋; ) HANGUL SYLLABLE DDYEC +B60C;B60C;1104 1168 11BF;B60C;1104 1168 11BF; # (똌; 똌; 똌; 똌; 똌; ) HANGUL SYLLABLE DDYEK +B60D;B60D;1104 1168 11C0;B60D;1104 1168 11C0; # (똍; 똍; 똍; 똍; 똍; ) HANGUL SYLLABLE DDYET +B60E;B60E;1104 1168 11C1;B60E;1104 1168 11C1; # (똎; 똎; 똎; 똎; 똎; ) HANGUL SYLLABLE DDYEP +B60F;B60F;1104 1168 11C2;B60F;1104 1168 11C2; # (똏; 똏; 똏; 똏; 똏; ) HANGUL SYLLABLE DDYEH +B610;B610;1104 1169;B610;1104 1169; # (또; 또; 또; 또; 또; ) HANGUL SYLLABLE DDO +B611;B611;1104 1169 11A8;B611;1104 1169 11A8; # (똑; 똑; 똑; 똑; 똑; ) HANGUL SYLLABLE DDOG +B612;B612;1104 1169 11A9;B612;1104 1169 11A9; # (똒; 똒; 똒; 똒; 똒; ) HANGUL SYLLABLE DDOGG +B613;B613;1104 1169 11AA;B613;1104 1169 11AA; # (똓; 똓; 똓; 똓; 똓; ) HANGUL SYLLABLE DDOGS +B614;B614;1104 1169 11AB;B614;1104 1169 11AB; # (똔; 똔; 똔; 똔; 똔; ) HANGUL SYLLABLE DDON +B615;B615;1104 1169 11AC;B615;1104 1169 11AC; # (똕; 똕; 똕; 똕; 똕; ) HANGUL SYLLABLE DDONJ +B616;B616;1104 1169 11AD;B616;1104 1169 11AD; # (똖; 똖; 똖; 똖; 똖; ) HANGUL SYLLABLE DDONH +B617;B617;1104 1169 11AE;B617;1104 1169 11AE; # (똗; 똗; 똗; 똗; 똗; ) HANGUL SYLLABLE DDOD +B618;B618;1104 1169 11AF;B618;1104 1169 11AF; # (똘; 똘; 똘; 똘; 똘; ) HANGUL SYLLABLE DDOL +B619;B619;1104 1169 11B0;B619;1104 1169 11B0; # (똙; 똙; 똙; 똙; 똙; ) HANGUL SYLLABLE DDOLG +B61A;B61A;1104 1169 11B1;B61A;1104 1169 11B1; # (똚; 똚; 똚; 똚; 똚; ) HANGUL SYLLABLE DDOLM +B61B;B61B;1104 1169 11B2;B61B;1104 1169 11B2; # (똛; 똛; 똛; 똛; 똛; ) HANGUL SYLLABLE DDOLB +B61C;B61C;1104 1169 11B3;B61C;1104 1169 11B3; # (똜; 똜; 똜; 똜; 똜; ) HANGUL SYLLABLE DDOLS +B61D;B61D;1104 1169 11B4;B61D;1104 1169 11B4; # (똝; 똝; 똝; 똝; 똝; ) HANGUL SYLLABLE DDOLT +B61E;B61E;1104 1169 11B5;B61E;1104 1169 11B5; # (똞; 똞; 똞; 똞; 똞; ) HANGUL SYLLABLE DDOLP +B61F;B61F;1104 1169 11B6;B61F;1104 1169 11B6; # (똟; 똟; 똟; 똟; 똟; ) HANGUL SYLLABLE DDOLH +B620;B620;1104 1169 11B7;B620;1104 1169 11B7; # (똠; 똠; 똠; 똠; 똠; ) HANGUL SYLLABLE DDOM +B621;B621;1104 1169 11B8;B621;1104 1169 11B8; # (똡; 똡; 똡; 똡; 똡; ) HANGUL SYLLABLE DDOB +B622;B622;1104 1169 11B9;B622;1104 1169 11B9; # (똢; 똢; 똢; 똢; 똢; ) HANGUL SYLLABLE DDOBS +B623;B623;1104 1169 11BA;B623;1104 1169 11BA; # (똣; 똣; 똣; 똣; 똣; ) HANGUL SYLLABLE DDOS +B624;B624;1104 1169 11BB;B624;1104 1169 11BB; # (똤; 똤; 똤; 똤; 똤; ) HANGUL SYLLABLE DDOSS +B625;B625;1104 1169 11BC;B625;1104 1169 11BC; # (똥; 똥; 똥; 똥; 똥; ) HANGUL SYLLABLE DDONG +B626;B626;1104 1169 11BD;B626;1104 1169 11BD; # (똦; 똦; 똦; 똦; 똦; ) HANGUL SYLLABLE DDOJ +B627;B627;1104 1169 11BE;B627;1104 1169 11BE; # (똧; 똧; 똧; 똧; 똧; ) HANGUL SYLLABLE DDOC +B628;B628;1104 1169 11BF;B628;1104 1169 11BF; # (똨; 똨; 똨; 똨; 똨; ) HANGUL SYLLABLE DDOK +B629;B629;1104 1169 11C0;B629;1104 1169 11C0; # (똩; 똩; 똩; 똩; 똩; ) HANGUL SYLLABLE DDOT +B62A;B62A;1104 1169 11C1;B62A;1104 1169 11C1; # (똪; 똪; 똪; 똪; 똪; ) HANGUL SYLLABLE DDOP +B62B;B62B;1104 1169 11C2;B62B;1104 1169 11C2; # (똫; 똫; 똫; 똫; 똫; ) HANGUL SYLLABLE DDOH +B62C;B62C;1104 116A;B62C;1104 116A; # (똬; 똬; 똬; 똬; 똬; ) HANGUL SYLLABLE DDWA +B62D;B62D;1104 116A 11A8;B62D;1104 116A 11A8; # (똭; 똭; 똭; 똭; 똭; ) HANGUL SYLLABLE DDWAG +B62E;B62E;1104 116A 11A9;B62E;1104 116A 11A9; # (똮; 똮; 똮; 똮; 똮; ) HANGUL SYLLABLE DDWAGG +B62F;B62F;1104 116A 11AA;B62F;1104 116A 11AA; # (똯; 똯; 똯; 똯; 똯; ) HANGUL SYLLABLE DDWAGS +B630;B630;1104 116A 11AB;B630;1104 116A 11AB; # (똰; 똰; 똰; 똰; 똰; ) HANGUL SYLLABLE DDWAN +B631;B631;1104 116A 11AC;B631;1104 116A 11AC; # (똱; 똱; 똱; 똱; 똱; ) HANGUL SYLLABLE DDWANJ +B632;B632;1104 116A 11AD;B632;1104 116A 11AD; # (똲; 똲; 똲; 똲; 똲; ) HANGUL SYLLABLE DDWANH +B633;B633;1104 116A 11AE;B633;1104 116A 11AE; # (똳; 똳; 똳; 똳; 똳; ) HANGUL SYLLABLE DDWAD +B634;B634;1104 116A 11AF;B634;1104 116A 11AF; # (똴; 똴; 똴; 똴; 똴; ) HANGUL SYLLABLE DDWAL +B635;B635;1104 116A 11B0;B635;1104 116A 11B0; # (똵; 똵; 똵; 똵; 똵; ) HANGUL SYLLABLE DDWALG +B636;B636;1104 116A 11B1;B636;1104 116A 11B1; # (똶; 똶; 똶; 똶; 똶; ) HANGUL SYLLABLE DDWALM +B637;B637;1104 116A 11B2;B637;1104 116A 11B2; # (똷; 똷; 똷; 똷; 똷; ) HANGUL SYLLABLE DDWALB +B638;B638;1104 116A 11B3;B638;1104 116A 11B3; # (똸; 똸; 똸; 똸; 똸; ) HANGUL SYLLABLE DDWALS +B639;B639;1104 116A 11B4;B639;1104 116A 11B4; # (똹; 똹; 똹; 똹; 똹; ) HANGUL SYLLABLE DDWALT +B63A;B63A;1104 116A 11B5;B63A;1104 116A 11B5; # (똺; 똺; 똺; 똺; 똺; ) HANGUL SYLLABLE DDWALP +B63B;B63B;1104 116A 11B6;B63B;1104 116A 11B6; # (똻; 똻; 똻; 똻; 똻; ) HANGUL SYLLABLE DDWALH +B63C;B63C;1104 116A 11B7;B63C;1104 116A 11B7; # (똼; 똼; 똼; 똼; 똼; ) HANGUL SYLLABLE DDWAM +B63D;B63D;1104 116A 11B8;B63D;1104 116A 11B8; # (똽; 똽; 똽; 똽; 똽; ) HANGUL SYLLABLE DDWAB +B63E;B63E;1104 116A 11B9;B63E;1104 116A 11B9; # (똾; 똾; 똾; 똾; 똾; ) HANGUL SYLLABLE DDWABS +B63F;B63F;1104 116A 11BA;B63F;1104 116A 11BA; # (똿; 똿; 똿; 똿; 똿; ) HANGUL SYLLABLE DDWAS +B640;B640;1104 116A 11BB;B640;1104 116A 11BB; # (뙀; 뙀; 뙀; 뙀; 뙀; ) HANGUL SYLLABLE DDWASS +B641;B641;1104 116A 11BC;B641;1104 116A 11BC; # (뙁; 뙁; 뙁; 뙁; 뙁; ) HANGUL SYLLABLE DDWANG +B642;B642;1104 116A 11BD;B642;1104 116A 11BD; # (뙂; 뙂; 뙂; 뙂; 뙂; ) HANGUL SYLLABLE DDWAJ +B643;B643;1104 116A 11BE;B643;1104 116A 11BE; # (뙃; 뙃; 뙃; 뙃; 뙃; ) HANGUL SYLLABLE DDWAC +B644;B644;1104 116A 11BF;B644;1104 116A 11BF; # (뙄; 뙄; 뙄; 뙄; 뙄; ) HANGUL SYLLABLE DDWAK +B645;B645;1104 116A 11C0;B645;1104 116A 11C0; # (뙅; 뙅; 뙅; 뙅; 뙅; ) HANGUL SYLLABLE DDWAT +B646;B646;1104 116A 11C1;B646;1104 116A 11C1; # (뙆; 뙆; 뙆; 뙆; 뙆; ) HANGUL SYLLABLE DDWAP +B647;B647;1104 116A 11C2;B647;1104 116A 11C2; # (뙇; 뙇; 뙇; 뙇; 뙇; ) HANGUL SYLLABLE DDWAH +B648;B648;1104 116B;B648;1104 116B; # (뙈; 뙈; 뙈; 뙈; 뙈; ) HANGUL SYLLABLE DDWAE +B649;B649;1104 116B 11A8;B649;1104 116B 11A8; # (뙉; 뙉; 뙉; 뙉; 뙉; ) HANGUL SYLLABLE DDWAEG +B64A;B64A;1104 116B 11A9;B64A;1104 116B 11A9; # (뙊; 뙊; 뙊; 뙊; 뙊; ) HANGUL SYLLABLE DDWAEGG +B64B;B64B;1104 116B 11AA;B64B;1104 116B 11AA; # (뙋; 뙋; 뙋; 뙋; 뙋; ) HANGUL SYLLABLE DDWAEGS +B64C;B64C;1104 116B 11AB;B64C;1104 116B 11AB; # (뙌; 뙌; 뙌; 뙌; 뙌; ) HANGUL SYLLABLE DDWAEN +B64D;B64D;1104 116B 11AC;B64D;1104 116B 11AC; # (뙍; 뙍; 뙍; 뙍; 뙍; ) HANGUL SYLLABLE DDWAENJ +B64E;B64E;1104 116B 11AD;B64E;1104 116B 11AD; # (뙎; 뙎; 뙎; 뙎; 뙎; ) HANGUL SYLLABLE DDWAENH +B64F;B64F;1104 116B 11AE;B64F;1104 116B 11AE; # (뙏; 뙏; 뙏; 뙏; 뙏; ) HANGUL SYLLABLE DDWAED +B650;B650;1104 116B 11AF;B650;1104 116B 11AF; # (뙐; 뙐; 뙐; 뙐; 뙐; ) HANGUL SYLLABLE DDWAEL +B651;B651;1104 116B 11B0;B651;1104 116B 11B0; # (뙑; 뙑; 뙑; 뙑; 뙑; ) HANGUL SYLLABLE DDWAELG +B652;B652;1104 116B 11B1;B652;1104 116B 11B1; # (뙒; 뙒; 뙒; 뙒; 뙒; ) HANGUL SYLLABLE DDWAELM +B653;B653;1104 116B 11B2;B653;1104 116B 11B2; # (뙓; 뙓; 뙓; 뙓; 뙓; ) HANGUL SYLLABLE DDWAELB +B654;B654;1104 116B 11B3;B654;1104 116B 11B3; # (뙔; 뙔; 뙔; 뙔; 뙔; ) HANGUL SYLLABLE DDWAELS +B655;B655;1104 116B 11B4;B655;1104 116B 11B4; # (뙕; 뙕; 뙕; 뙕; 뙕; ) HANGUL SYLLABLE DDWAELT +B656;B656;1104 116B 11B5;B656;1104 116B 11B5; # (뙖; 뙖; 뙖; 뙖; 뙖; ) HANGUL SYLLABLE DDWAELP +B657;B657;1104 116B 11B6;B657;1104 116B 11B6; # (뙗; 뙗; 뙗; 뙗; 뙗; ) HANGUL SYLLABLE DDWAELH +B658;B658;1104 116B 11B7;B658;1104 116B 11B7; # (뙘; 뙘; 뙘; 뙘; 뙘; ) HANGUL SYLLABLE DDWAEM +B659;B659;1104 116B 11B8;B659;1104 116B 11B8; # (뙙; 뙙; 뙙; 뙙; 뙙; ) HANGUL SYLLABLE DDWAEB +B65A;B65A;1104 116B 11B9;B65A;1104 116B 11B9; # (뙚; 뙚; 뙚; 뙚; 뙚; ) HANGUL SYLLABLE DDWAEBS +B65B;B65B;1104 116B 11BA;B65B;1104 116B 11BA; # (뙛; 뙛; 뙛; 뙛; 뙛; ) HANGUL SYLLABLE DDWAES +B65C;B65C;1104 116B 11BB;B65C;1104 116B 11BB; # (뙜; 뙜; 뙜; 뙜; 뙜; ) HANGUL SYLLABLE DDWAESS +B65D;B65D;1104 116B 11BC;B65D;1104 116B 11BC; # (뙝; 뙝; 뙝; 뙝; 뙝; ) HANGUL SYLLABLE DDWAENG +B65E;B65E;1104 116B 11BD;B65E;1104 116B 11BD; # (뙞; 뙞; 뙞; 뙞; 뙞; ) HANGUL SYLLABLE DDWAEJ +B65F;B65F;1104 116B 11BE;B65F;1104 116B 11BE; # (뙟; 뙟; 뙟; 뙟; 뙟; ) HANGUL SYLLABLE DDWAEC +B660;B660;1104 116B 11BF;B660;1104 116B 11BF; # (뙠; 뙠; 뙠; 뙠; 뙠; ) HANGUL SYLLABLE DDWAEK +B661;B661;1104 116B 11C0;B661;1104 116B 11C0; # (뙡; 뙡; 뙡; 뙡; 뙡; ) HANGUL SYLLABLE DDWAET +B662;B662;1104 116B 11C1;B662;1104 116B 11C1; # (뙢; 뙢; 뙢; 뙢; 뙢; ) HANGUL SYLLABLE DDWAEP +B663;B663;1104 116B 11C2;B663;1104 116B 11C2; # (뙣; 뙣; 뙣; 뙣; 뙣; ) HANGUL SYLLABLE DDWAEH +B664;B664;1104 116C;B664;1104 116C; # (뙤; 뙤; 뙤; 뙤; 뙤; ) HANGUL SYLLABLE DDOE +B665;B665;1104 116C 11A8;B665;1104 116C 11A8; # (뙥; 뙥; 뙥; 뙥; 뙥; ) HANGUL SYLLABLE DDOEG +B666;B666;1104 116C 11A9;B666;1104 116C 11A9; # (뙦; 뙦; 뙦; 뙦; 뙦; ) HANGUL SYLLABLE DDOEGG +B667;B667;1104 116C 11AA;B667;1104 116C 11AA; # (뙧; 뙧; 뙧; 뙧; 뙧; ) HANGUL SYLLABLE DDOEGS +B668;B668;1104 116C 11AB;B668;1104 116C 11AB; # (뙨; 뙨; 뙨; 뙨; 뙨; ) HANGUL SYLLABLE DDOEN +B669;B669;1104 116C 11AC;B669;1104 116C 11AC; # (뙩; 뙩; 뙩; 뙩; 뙩; ) HANGUL SYLLABLE DDOENJ +B66A;B66A;1104 116C 11AD;B66A;1104 116C 11AD; # (뙪; 뙪; 뙪; 뙪; 뙪; ) HANGUL SYLLABLE DDOENH +B66B;B66B;1104 116C 11AE;B66B;1104 116C 11AE; # (뙫; 뙫; 뙫; 뙫; 뙫; ) HANGUL SYLLABLE DDOED +B66C;B66C;1104 116C 11AF;B66C;1104 116C 11AF; # (뙬; 뙬; 뙬; 뙬; 뙬; ) HANGUL SYLLABLE DDOEL +B66D;B66D;1104 116C 11B0;B66D;1104 116C 11B0; # (뙭; 뙭; 뙭; 뙭; 뙭; ) HANGUL SYLLABLE DDOELG +B66E;B66E;1104 116C 11B1;B66E;1104 116C 11B1; # (뙮; 뙮; 뙮; 뙮; 뙮; ) HANGUL SYLLABLE DDOELM +B66F;B66F;1104 116C 11B2;B66F;1104 116C 11B2; # (뙯; 뙯; 뙯; 뙯; 뙯; ) HANGUL SYLLABLE DDOELB +B670;B670;1104 116C 11B3;B670;1104 116C 11B3; # (뙰; 뙰; 뙰; 뙰; 뙰; ) HANGUL SYLLABLE DDOELS +B671;B671;1104 116C 11B4;B671;1104 116C 11B4; # (뙱; 뙱; 뙱; 뙱; 뙱; ) HANGUL SYLLABLE DDOELT +B672;B672;1104 116C 11B5;B672;1104 116C 11B5; # (뙲; 뙲; 뙲; 뙲; 뙲; ) HANGUL SYLLABLE DDOELP +B673;B673;1104 116C 11B6;B673;1104 116C 11B6; # (뙳; 뙳; 뙳; 뙳; 뙳; ) HANGUL SYLLABLE DDOELH +B674;B674;1104 116C 11B7;B674;1104 116C 11B7; # (뙴; 뙴; 뙴; 뙴; 뙴; ) HANGUL SYLLABLE DDOEM +B675;B675;1104 116C 11B8;B675;1104 116C 11B8; # (뙵; 뙵; 뙵; 뙵; 뙵; ) HANGUL SYLLABLE DDOEB +B676;B676;1104 116C 11B9;B676;1104 116C 11B9; # (뙶; 뙶; 뙶; 뙶; 뙶; ) HANGUL SYLLABLE DDOEBS +B677;B677;1104 116C 11BA;B677;1104 116C 11BA; # (뙷; 뙷; 뙷; 뙷; 뙷; ) HANGUL SYLLABLE DDOES +B678;B678;1104 116C 11BB;B678;1104 116C 11BB; # (뙸; 뙸; 뙸; 뙸; 뙸; ) HANGUL SYLLABLE DDOESS +B679;B679;1104 116C 11BC;B679;1104 116C 11BC; # (뙹; 뙹; 뙹; 뙹; 뙹; ) HANGUL SYLLABLE DDOENG +B67A;B67A;1104 116C 11BD;B67A;1104 116C 11BD; # (뙺; 뙺; 뙺; 뙺; 뙺; ) HANGUL SYLLABLE DDOEJ +B67B;B67B;1104 116C 11BE;B67B;1104 116C 11BE; # (뙻; 뙻; 뙻; 뙻; 뙻; ) HANGUL SYLLABLE DDOEC +B67C;B67C;1104 116C 11BF;B67C;1104 116C 11BF; # (뙼; 뙼; 뙼; 뙼; 뙼; ) HANGUL SYLLABLE DDOEK +B67D;B67D;1104 116C 11C0;B67D;1104 116C 11C0; # (뙽; 뙽; 뙽; 뙽; 뙽; ) HANGUL SYLLABLE DDOET +B67E;B67E;1104 116C 11C1;B67E;1104 116C 11C1; # (뙾; 뙾; 뙾; 뙾; 뙾; ) HANGUL SYLLABLE DDOEP +B67F;B67F;1104 116C 11C2;B67F;1104 116C 11C2; # (뙿; 뙿; 뙿; 뙿; 뙿; ) HANGUL SYLLABLE DDOEH +B680;B680;1104 116D;B680;1104 116D; # (뚀; 뚀; 뚀; 뚀; 뚀; ) HANGUL SYLLABLE DDYO +B681;B681;1104 116D 11A8;B681;1104 116D 11A8; # (뚁; 뚁; 뚁; 뚁; 뚁; ) HANGUL SYLLABLE DDYOG +B682;B682;1104 116D 11A9;B682;1104 116D 11A9; # (뚂; 뚂; 뚂; 뚂; 뚂; ) HANGUL SYLLABLE DDYOGG +B683;B683;1104 116D 11AA;B683;1104 116D 11AA; # (뚃; 뚃; 뚃; 뚃; 뚃; ) HANGUL SYLLABLE DDYOGS +B684;B684;1104 116D 11AB;B684;1104 116D 11AB; # (뚄; 뚄; 뚄; 뚄; 뚄; ) HANGUL SYLLABLE DDYON +B685;B685;1104 116D 11AC;B685;1104 116D 11AC; # (뚅; 뚅; 뚅; 뚅; 뚅; ) HANGUL SYLLABLE DDYONJ +B686;B686;1104 116D 11AD;B686;1104 116D 11AD; # (뚆; 뚆; 뚆; 뚆; 뚆; ) HANGUL SYLLABLE DDYONH +B687;B687;1104 116D 11AE;B687;1104 116D 11AE; # (뚇; 뚇; 뚇; 뚇; 뚇; ) HANGUL SYLLABLE DDYOD +B688;B688;1104 116D 11AF;B688;1104 116D 11AF; # (뚈; 뚈; 뚈; 뚈; 뚈; ) HANGUL SYLLABLE DDYOL +B689;B689;1104 116D 11B0;B689;1104 116D 11B0; # (뚉; 뚉; 뚉; 뚉; 뚉; ) HANGUL SYLLABLE DDYOLG +B68A;B68A;1104 116D 11B1;B68A;1104 116D 11B1; # (뚊; 뚊; 뚊; 뚊; 뚊; ) HANGUL SYLLABLE DDYOLM +B68B;B68B;1104 116D 11B2;B68B;1104 116D 11B2; # (뚋; 뚋; 뚋; 뚋; 뚋; ) HANGUL SYLLABLE DDYOLB +B68C;B68C;1104 116D 11B3;B68C;1104 116D 11B3; # (뚌; 뚌; 뚌; 뚌; 뚌; ) HANGUL SYLLABLE DDYOLS +B68D;B68D;1104 116D 11B4;B68D;1104 116D 11B4; # (뚍; 뚍; 뚍; 뚍; 뚍; ) HANGUL SYLLABLE DDYOLT +B68E;B68E;1104 116D 11B5;B68E;1104 116D 11B5; # (뚎; 뚎; 뚎; 뚎; 뚎; ) HANGUL SYLLABLE DDYOLP +B68F;B68F;1104 116D 11B6;B68F;1104 116D 11B6; # (뚏; 뚏; 뚏; 뚏; 뚏; ) HANGUL SYLLABLE DDYOLH +B690;B690;1104 116D 11B7;B690;1104 116D 11B7; # (뚐; 뚐; 뚐; 뚐; 뚐; ) HANGUL SYLLABLE DDYOM +B691;B691;1104 116D 11B8;B691;1104 116D 11B8; # (뚑; 뚑; 뚑; 뚑; 뚑; ) HANGUL SYLLABLE DDYOB +B692;B692;1104 116D 11B9;B692;1104 116D 11B9; # (뚒; 뚒; 뚒; 뚒; 뚒; ) HANGUL SYLLABLE DDYOBS +B693;B693;1104 116D 11BA;B693;1104 116D 11BA; # (뚓; 뚓; 뚓; 뚓; 뚓; ) HANGUL SYLLABLE DDYOS +B694;B694;1104 116D 11BB;B694;1104 116D 11BB; # (뚔; 뚔; 뚔; 뚔; 뚔; ) HANGUL SYLLABLE DDYOSS +B695;B695;1104 116D 11BC;B695;1104 116D 11BC; # (뚕; 뚕; 뚕; 뚕; 뚕; ) HANGUL SYLLABLE DDYONG +B696;B696;1104 116D 11BD;B696;1104 116D 11BD; # (뚖; 뚖; 뚖; 뚖; 뚖; ) HANGUL SYLLABLE DDYOJ +B697;B697;1104 116D 11BE;B697;1104 116D 11BE; # (뚗; 뚗; 뚗; 뚗; 뚗; ) HANGUL SYLLABLE DDYOC +B698;B698;1104 116D 11BF;B698;1104 116D 11BF; # (뚘; 뚘; 뚘; 뚘; 뚘; ) HANGUL SYLLABLE DDYOK +B699;B699;1104 116D 11C0;B699;1104 116D 11C0; # (뚙; 뚙; 뚙; 뚙; 뚙; ) HANGUL SYLLABLE DDYOT +B69A;B69A;1104 116D 11C1;B69A;1104 116D 11C1; # (뚚; 뚚; 뚚; 뚚; 뚚; ) HANGUL SYLLABLE DDYOP +B69B;B69B;1104 116D 11C2;B69B;1104 116D 11C2; # (뚛; 뚛; 뚛; 뚛; 뚛; ) HANGUL SYLLABLE DDYOH +B69C;B69C;1104 116E;B69C;1104 116E; # (뚜; 뚜; 뚜; 뚜; 뚜; ) HANGUL SYLLABLE DDU +B69D;B69D;1104 116E 11A8;B69D;1104 116E 11A8; # (뚝; 뚝; 뚝; 뚝; 뚝; ) HANGUL SYLLABLE DDUG +B69E;B69E;1104 116E 11A9;B69E;1104 116E 11A9; # (뚞; 뚞; 뚞; 뚞; 뚞; ) HANGUL SYLLABLE DDUGG +B69F;B69F;1104 116E 11AA;B69F;1104 116E 11AA; # (뚟; 뚟; 뚟; 뚟; 뚟; ) HANGUL SYLLABLE DDUGS +B6A0;B6A0;1104 116E 11AB;B6A0;1104 116E 11AB; # (뚠; 뚠; 뚠; 뚠; 뚠; ) HANGUL SYLLABLE DDUN +B6A1;B6A1;1104 116E 11AC;B6A1;1104 116E 11AC; # (뚡; 뚡; 뚡; 뚡; 뚡; ) HANGUL SYLLABLE DDUNJ +B6A2;B6A2;1104 116E 11AD;B6A2;1104 116E 11AD; # (뚢; 뚢; 뚢; 뚢; 뚢; ) HANGUL SYLLABLE DDUNH +B6A3;B6A3;1104 116E 11AE;B6A3;1104 116E 11AE; # (뚣; 뚣; 뚣; 뚣; 뚣; ) HANGUL SYLLABLE DDUD +B6A4;B6A4;1104 116E 11AF;B6A4;1104 116E 11AF; # (뚤; 뚤; 뚤; 뚤; 뚤; ) HANGUL SYLLABLE DDUL +B6A5;B6A5;1104 116E 11B0;B6A5;1104 116E 11B0; # (뚥; 뚥; 뚥; 뚥; 뚥; ) HANGUL SYLLABLE DDULG +B6A6;B6A6;1104 116E 11B1;B6A6;1104 116E 11B1; # (뚦; 뚦; 뚦; 뚦; 뚦; ) HANGUL SYLLABLE DDULM +B6A7;B6A7;1104 116E 11B2;B6A7;1104 116E 11B2; # (뚧; 뚧; 뚧; 뚧; 뚧; ) HANGUL SYLLABLE DDULB +B6A8;B6A8;1104 116E 11B3;B6A8;1104 116E 11B3; # (뚨; 뚨; 뚨; 뚨; 뚨; ) HANGUL SYLLABLE DDULS +B6A9;B6A9;1104 116E 11B4;B6A9;1104 116E 11B4; # (뚩; 뚩; 뚩; 뚩; 뚩; ) HANGUL SYLLABLE DDULT +B6AA;B6AA;1104 116E 11B5;B6AA;1104 116E 11B5; # (뚪; 뚪; 뚪; 뚪; 뚪; ) HANGUL SYLLABLE DDULP +B6AB;B6AB;1104 116E 11B6;B6AB;1104 116E 11B6; # (뚫; 뚫; 뚫; 뚫; 뚫; ) HANGUL SYLLABLE DDULH +B6AC;B6AC;1104 116E 11B7;B6AC;1104 116E 11B7; # (뚬; 뚬; 뚬; 뚬; 뚬; ) HANGUL SYLLABLE DDUM +B6AD;B6AD;1104 116E 11B8;B6AD;1104 116E 11B8; # (뚭; 뚭; 뚭; 뚭; 뚭; ) HANGUL SYLLABLE DDUB +B6AE;B6AE;1104 116E 11B9;B6AE;1104 116E 11B9; # (뚮; 뚮; 뚮; 뚮; 뚮; ) HANGUL SYLLABLE DDUBS +B6AF;B6AF;1104 116E 11BA;B6AF;1104 116E 11BA; # (뚯; 뚯; 뚯; 뚯; 뚯; ) HANGUL SYLLABLE DDUS +B6B0;B6B0;1104 116E 11BB;B6B0;1104 116E 11BB; # (뚰; 뚰; 뚰; 뚰; 뚰; ) HANGUL SYLLABLE DDUSS +B6B1;B6B1;1104 116E 11BC;B6B1;1104 116E 11BC; # (뚱; 뚱; 뚱; 뚱; 뚱; ) HANGUL SYLLABLE DDUNG +B6B2;B6B2;1104 116E 11BD;B6B2;1104 116E 11BD; # (뚲; 뚲; 뚲; 뚲; 뚲; ) HANGUL SYLLABLE DDUJ +B6B3;B6B3;1104 116E 11BE;B6B3;1104 116E 11BE; # (뚳; 뚳; 뚳; 뚳; 뚳; ) HANGUL SYLLABLE DDUC +B6B4;B6B4;1104 116E 11BF;B6B4;1104 116E 11BF; # (뚴; 뚴; 뚴; 뚴; 뚴; ) HANGUL SYLLABLE DDUK +B6B5;B6B5;1104 116E 11C0;B6B5;1104 116E 11C0; # (뚵; 뚵; 뚵; 뚵; 뚵; ) HANGUL SYLLABLE DDUT +B6B6;B6B6;1104 116E 11C1;B6B6;1104 116E 11C1; # (뚶; 뚶; 뚶; 뚶; 뚶; ) HANGUL SYLLABLE DDUP +B6B7;B6B7;1104 116E 11C2;B6B7;1104 116E 11C2; # (뚷; 뚷; 뚷; 뚷; 뚷; ) HANGUL SYLLABLE DDUH +B6B8;B6B8;1104 116F;B6B8;1104 116F; # (뚸; 뚸; 뚸; 뚸; 뚸; ) HANGUL SYLLABLE DDWEO +B6B9;B6B9;1104 116F 11A8;B6B9;1104 116F 11A8; # (뚹; 뚹; 뚹; 뚹; 뚹; ) HANGUL SYLLABLE DDWEOG +B6BA;B6BA;1104 116F 11A9;B6BA;1104 116F 11A9; # (뚺; 뚺; 뚺; 뚺; 뚺; ) HANGUL SYLLABLE DDWEOGG +B6BB;B6BB;1104 116F 11AA;B6BB;1104 116F 11AA; # (뚻; 뚻; 뚻; 뚻; 뚻; ) HANGUL SYLLABLE DDWEOGS +B6BC;B6BC;1104 116F 11AB;B6BC;1104 116F 11AB; # (뚼; 뚼; 뚼; 뚼; 뚼; ) HANGUL SYLLABLE DDWEON +B6BD;B6BD;1104 116F 11AC;B6BD;1104 116F 11AC; # (뚽; 뚽; 뚽; 뚽; 뚽; ) HANGUL SYLLABLE DDWEONJ +B6BE;B6BE;1104 116F 11AD;B6BE;1104 116F 11AD; # (뚾; 뚾; 뚾; 뚾; 뚾; ) HANGUL SYLLABLE DDWEONH +B6BF;B6BF;1104 116F 11AE;B6BF;1104 116F 11AE; # (뚿; 뚿; 뚿; 뚿; 뚿; ) HANGUL SYLLABLE DDWEOD +B6C0;B6C0;1104 116F 11AF;B6C0;1104 116F 11AF; # (뛀; 뛀; 뛀; 뛀; 뛀; ) HANGUL SYLLABLE DDWEOL +B6C1;B6C1;1104 116F 11B0;B6C1;1104 116F 11B0; # (뛁; 뛁; 뛁; 뛁; 뛁; ) HANGUL SYLLABLE DDWEOLG +B6C2;B6C2;1104 116F 11B1;B6C2;1104 116F 11B1; # (뛂; 뛂; 뛂; 뛂; 뛂; ) HANGUL SYLLABLE DDWEOLM +B6C3;B6C3;1104 116F 11B2;B6C3;1104 116F 11B2; # (뛃; 뛃; 뛃; 뛃; 뛃; ) HANGUL SYLLABLE DDWEOLB +B6C4;B6C4;1104 116F 11B3;B6C4;1104 116F 11B3; # (뛄; 뛄; 뛄; 뛄; 뛄; ) HANGUL SYLLABLE DDWEOLS +B6C5;B6C5;1104 116F 11B4;B6C5;1104 116F 11B4; # (뛅; 뛅; 뛅; 뛅; 뛅; ) HANGUL SYLLABLE DDWEOLT +B6C6;B6C6;1104 116F 11B5;B6C6;1104 116F 11B5; # (뛆; 뛆; 뛆; 뛆; 뛆; ) HANGUL SYLLABLE DDWEOLP +B6C7;B6C7;1104 116F 11B6;B6C7;1104 116F 11B6; # (뛇; 뛇; 뛇; 뛇; 뛇; ) HANGUL SYLLABLE DDWEOLH +B6C8;B6C8;1104 116F 11B7;B6C8;1104 116F 11B7; # (뛈; 뛈; 뛈; 뛈; 뛈; ) HANGUL SYLLABLE DDWEOM +B6C9;B6C9;1104 116F 11B8;B6C9;1104 116F 11B8; # (뛉; 뛉; 뛉; 뛉; 뛉; ) HANGUL SYLLABLE DDWEOB +B6CA;B6CA;1104 116F 11B9;B6CA;1104 116F 11B9; # (뛊; 뛊; 뛊; 뛊; 뛊; ) HANGUL SYLLABLE DDWEOBS +B6CB;B6CB;1104 116F 11BA;B6CB;1104 116F 11BA; # (뛋; 뛋; 뛋; 뛋; 뛋; ) HANGUL SYLLABLE DDWEOS +B6CC;B6CC;1104 116F 11BB;B6CC;1104 116F 11BB; # (뛌; 뛌; 뛌; 뛌; 뛌; ) HANGUL SYLLABLE DDWEOSS +B6CD;B6CD;1104 116F 11BC;B6CD;1104 116F 11BC; # (뛍; 뛍; 뛍; 뛍; 뛍; ) HANGUL SYLLABLE DDWEONG +B6CE;B6CE;1104 116F 11BD;B6CE;1104 116F 11BD; # (뛎; 뛎; 뛎; 뛎; 뛎; ) HANGUL SYLLABLE DDWEOJ +B6CF;B6CF;1104 116F 11BE;B6CF;1104 116F 11BE; # (뛏; 뛏; 뛏; 뛏; 뛏; ) HANGUL SYLLABLE DDWEOC +B6D0;B6D0;1104 116F 11BF;B6D0;1104 116F 11BF; # (뛐; 뛐; 뛐; 뛐; 뛐; ) HANGUL SYLLABLE DDWEOK +B6D1;B6D1;1104 116F 11C0;B6D1;1104 116F 11C0; # (뛑; 뛑; 뛑; 뛑; 뛑; ) HANGUL SYLLABLE DDWEOT +B6D2;B6D2;1104 116F 11C1;B6D2;1104 116F 11C1; # (뛒; 뛒; 뛒; 뛒; 뛒; ) HANGUL SYLLABLE DDWEOP +B6D3;B6D3;1104 116F 11C2;B6D3;1104 116F 11C2; # (뛓; 뛓; 뛓; 뛓; 뛓; ) HANGUL SYLLABLE DDWEOH +B6D4;B6D4;1104 1170;B6D4;1104 1170; # (뛔; 뛔; 뛔; 뛔; 뛔; ) HANGUL SYLLABLE DDWE +B6D5;B6D5;1104 1170 11A8;B6D5;1104 1170 11A8; # (뛕; 뛕; 뛕; 뛕; 뛕; ) HANGUL SYLLABLE DDWEG +B6D6;B6D6;1104 1170 11A9;B6D6;1104 1170 11A9; # (뛖; 뛖; 뛖; 뛖; 뛖; ) HANGUL SYLLABLE DDWEGG +B6D7;B6D7;1104 1170 11AA;B6D7;1104 1170 11AA; # (뛗; 뛗; 뛗; 뛗; 뛗; ) HANGUL SYLLABLE DDWEGS +B6D8;B6D8;1104 1170 11AB;B6D8;1104 1170 11AB; # (뛘; 뛘; 뛘; 뛘; 뛘; ) HANGUL SYLLABLE DDWEN +B6D9;B6D9;1104 1170 11AC;B6D9;1104 1170 11AC; # (뛙; 뛙; 뛙; 뛙; 뛙; ) HANGUL SYLLABLE DDWENJ +B6DA;B6DA;1104 1170 11AD;B6DA;1104 1170 11AD; # (뛚; 뛚; 뛚; 뛚; 뛚; ) HANGUL SYLLABLE DDWENH +B6DB;B6DB;1104 1170 11AE;B6DB;1104 1170 11AE; # (뛛; 뛛; 뛛; 뛛; 뛛; ) HANGUL SYLLABLE DDWED +B6DC;B6DC;1104 1170 11AF;B6DC;1104 1170 11AF; # (뛜; 뛜; 뛜; 뛜; 뛜; ) HANGUL SYLLABLE DDWEL +B6DD;B6DD;1104 1170 11B0;B6DD;1104 1170 11B0; # (뛝; 뛝; 뛝; 뛝; 뛝; ) HANGUL SYLLABLE DDWELG +B6DE;B6DE;1104 1170 11B1;B6DE;1104 1170 11B1; # (뛞; 뛞; 뛞; 뛞; 뛞; ) HANGUL SYLLABLE DDWELM +B6DF;B6DF;1104 1170 11B2;B6DF;1104 1170 11B2; # (뛟; 뛟; 뛟; 뛟; 뛟; ) HANGUL SYLLABLE DDWELB +B6E0;B6E0;1104 1170 11B3;B6E0;1104 1170 11B3; # (뛠; 뛠; 뛠; 뛠; 뛠; ) HANGUL SYLLABLE DDWELS +B6E1;B6E1;1104 1170 11B4;B6E1;1104 1170 11B4; # (뛡; 뛡; 뛡; 뛡; 뛡; ) HANGUL SYLLABLE DDWELT +B6E2;B6E2;1104 1170 11B5;B6E2;1104 1170 11B5; # (뛢; 뛢; 뛢; 뛢; 뛢; ) HANGUL SYLLABLE DDWELP +B6E3;B6E3;1104 1170 11B6;B6E3;1104 1170 11B6; # (뛣; 뛣; 뛣; 뛣; 뛣; ) HANGUL SYLLABLE DDWELH +B6E4;B6E4;1104 1170 11B7;B6E4;1104 1170 11B7; # (뛤; 뛤; 뛤; 뛤; 뛤; ) HANGUL SYLLABLE DDWEM +B6E5;B6E5;1104 1170 11B8;B6E5;1104 1170 11B8; # (뛥; 뛥; 뛥; 뛥; 뛥; ) HANGUL SYLLABLE DDWEB +B6E6;B6E6;1104 1170 11B9;B6E6;1104 1170 11B9; # (뛦; 뛦; 뛦; 뛦; 뛦; ) HANGUL SYLLABLE DDWEBS +B6E7;B6E7;1104 1170 11BA;B6E7;1104 1170 11BA; # (뛧; 뛧; 뛧; 뛧; 뛧; ) HANGUL SYLLABLE DDWES +B6E8;B6E8;1104 1170 11BB;B6E8;1104 1170 11BB; # (뛨; 뛨; 뛨; 뛨; 뛨; ) HANGUL SYLLABLE DDWESS +B6E9;B6E9;1104 1170 11BC;B6E9;1104 1170 11BC; # (뛩; 뛩; 뛩; 뛩; 뛩; ) HANGUL SYLLABLE DDWENG +B6EA;B6EA;1104 1170 11BD;B6EA;1104 1170 11BD; # (뛪; 뛪; 뛪; 뛪; 뛪; ) HANGUL SYLLABLE DDWEJ +B6EB;B6EB;1104 1170 11BE;B6EB;1104 1170 11BE; # (뛫; 뛫; 뛫; 뛫; 뛫; ) HANGUL SYLLABLE DDWEC +B6EC;B6EC;1104 1170 11BF;B6EC;1104 1170 11BF; # (뛬; 뛬; 뛬; 뛬; 뛬; ) HANGUL SYLLABLE DDWEK +B6ED;B6ED;1104 1170 11C0;B6ED;1104 1170 11C0; # (뛭; 뛭; 뛭; 뛭; 뛭; ) HANGUL SYLLABLE DDWET +B6EE;B6EE;1104 1170 11C1;B6EE;1104 1170 11C1; # (뛮; 뛮; 뛮; 뛮; 뛮; ) HANGUL SYLLABLE DDWEP +B6EF;B6EF;1104 1170 11C2;B6EF;1104 1170 11C2; # (뛯; 뛯; 뛯; 뛯; 뛯; ) HANGUL SYLLABLE DDWEH +B6F0;B6F0;1104 1171;B6F0;1104 1171; # (뛰; 뛰; 뛰; 뛰; 뛰; ) HANGUL SYLLABLE DDWI +B6F1;B6F1;1104 1171 11A8;B6F1;1104 1171 11A8; # (뛱; 뛱; 뛱; 뛱; 뛱; ) HANGUL SYLLABLE DDWIG +B6F2;B6F2;1104 1171 11A9;B6F2;1104 1171 11A9; # (뛲; 뛲; 뛲; 뛲; 뛲; ) HANGUL SYLLABLE DDWIGG +B6F3;B6F3;1104 1171 11AA;B6F3;1104 1171 11AA; # (뛳; 뛳; 뛳; 뛳; 뛳; ) HANGUL SYLLABLE DDWIGS +B6F4;B6F4;1104 1171 11AB;B6F4;1104 1171 11AB; # (뛴; 뛴; 뛴; 뛴; 뛴; ) HANGUL SYLLABLE DDWIN +B6F5;B6F5;1104 1171 11AC;B6F5;1104 1171 11AC; # (뛵; 뛵; 뛵; 뛵; 뛵; ) HANGUL SYLLABLE DDWINJ +B6F6;B6F6;1104 1171 11AD;B6F6;1104 1171 11AD; # (뛶; 뛶; 뛶; 뛶; 뛶; ) HANGUL SYLLABLE DDWINH +B6F7;B6F7;1104 1171 11AE;B6F7;1104 1171 11AE; # (뛷; 뛷; 뛷; 뛷; 뛷; ) HANGUL SYLLABLE DDWID +B6F8;B6F8;1104 1171 11AF;B6F8;1104 1171 11AF; # (뛸; 뛸; 뛸; 뛸; 뛸; ) HANGUL SYLLABLE DDWIL +B6F9;B6F9;1104 1171 11B0;B6F9;1104 1171 11B0; # (뛹; 뛹; 뛹; 뛹; 뛹; ) HANGUL SYLLABLE DDWILG +B6FA;B6FA;1104 1171 11B1;B6FA;1104 1171 11B1; # (뛺; 뛺; 뛺; 뛺; 뛺; ) HANGUL SYLLABLE DDWILM +B6FB;B6FB;1104 1171 11B2;B6FB;1104 1171 11B2; # (뛻; 뛻; 뛻; 뛻; 뛻; ) HANGUL SYLLABLE DDWILB +B6FC;B6FC;1104 1171 11B3;B6FC;1104 1171 11B3; # (뛼; 뛼; 뛼; 뛼; 뛼; ) HANGUL SYLLABLE DDWILS +B6FD;B6FD;1104 1171 11B4;B6FD;1104 1171 11B4; # (뛽; 뛽; 뛽; 뛽; 뛽; ) HANGUL SYLLABLE DDWILT +B6FE;B6FE;1104 1171 11B5;B6FE;1104 1171 11B5; # (뛾; 뛾; 뛾; 뛾; 뛾; ) HANGUL SYLLABLE DDWILP +B6FF;B6FF;1104 1171 11B6;B6FF;1104 1171 11B6; # (뛿; 뛿; 뛿; 뛿; 뛿; ) HANGUL SYLLABLE DDWILH +B700;B700;1104 1171 11B7;B700;1104 1171 11B7; # (뜀; 뜀; 뜀; 뜀; 뜀; ) HANGUL SYLLABLE DDWIM +B701;B701;1104 1171 11B8;B701;1104 1171 11B8; # (뜁; 뜁; 뜁; 뜁; 뜁; ) HANGUL SYLLABLE DDWIB +B702;B702;1104 1171 11B9;B702;1104 1171 11B9; # (뜂; 뜂; 뜂; 뜂; 뜂; ) HANGUL SYLLABLE DDWIBS +B703;B703;1104 1171 11BA;B703;1104 1171 11BA; # (뜃; 뜃; 뜃; 뜃; 뜃; ) HANGUL SYLLABLE DDWIS +B704;B704;1104 1171 11BB;B704;1104 1171 11BB; # (뜄; 뜄; 뜄; 뜄; 뜄; ) HANGUL SYLLABLE DDWISS +B705;B705;1104 1171 11BC;B705;1104 1171 11BC; # (뜅; 뜅; 뜅; 뜅; 뜅; ) HANGUL SYLLABLE DDWING +B706;B706;1104 1171 11BD;B706;1104 1171 11BD; # (뜆; 뜆; 뜆; 뜆; 뜆; ) HANGUL SYLLABLE DDWIJ +B707;B707;1104 1171 11BE;B707;1104 1171 11BE; # (뜇; 뜇; 뜇; 뜇; 뜇; ) HANGUL SYLLABLE DDWIC +B708;B708;1104 1171 11BF;B708;1104 1171 11BF; # (뜈; 뜈; 뜈; 뜈; 뜈; ) HANGUL SYLLABLE DDWIK +B709;B709;1104 1171 11C0;B709;1104 1171 11C0; # (뜉; 뜉; 뜉; 뜉; 뜉; ) HANGUL SYLLABLE DDWIT +B70A;B70A;1104 1171 11C1;B70A;1104 1171 11C1; # (뜊; 뜊; 뜊; 뜊; 뜊; ) HANGUL SYLLABLE DDWIP +B70B;B70B;1104 1171 11C2;B70B;1104 1171 11C2; # (뜋; 뜋; 뜋; 뜋; 뜋; ) HANGUL SYLLABLE DDWIH +B70C;B70C;1104 1172;B70C;1104 1172; # (뜌; 뜌; 뜌; 뜌; 뜌; ) HANGUL SYLLABLE DDYU +B70D;B70D;1104 1172 11A8;B70D;1104 1172 11A8; # (뜍; 뜍; 뜍; 뜍; 뜍; ) HANGUL SYLLABLE DDYUG +B70E;B70E;1104 1172 11A9;B70E;1104 1172 11A9; # (뜎; 뜎; 뜎; 뜎; 뜎; ) HANGUL SYLLABLE DDYUGG +B70F;B70F;1104 1172 11AA;B70F;1104 1172 11AA; # (뜏; 뜏; 뜏; 뜏; 뜏; ) HANGUL SYLLABLE DDYUGS +B710;B710;1104 1172 11AB;B710;1104 1172 11AB; # (뜐; 뜐; 뜐; 뜐; 뜐; ) HANGUL SYLLABLE DDYUN +B711;B711;1104 1172 11AC;B711;1104 1172 11AC; # (뜑; 뜑; 뜑; 뜑; 뜑; ) HANGUL SYLLABLE DDYUNJ +B712;B712;1104 1172 11AD;B712;1104 1172 11AD; # (뜒; 뜒; 뜒; 뜒; 뜒; ) HANGUL SYLLABLE DDYUNH +B713;B713;1104 1172 11AE;B713;1104 1172 11AE; # (뜓; 뜓; 뜓; 뜓; 뜓; ) HANGUL SYLLABLE DDYUD +B714;B714;1104 1172 11AF;B714;1104 1172 11AF; # (뜔; 뜔; 뜔; 뜔; 뜔; ) HANGUL SYLLABLE DDYUL +B715;B715;1104 1172 11B0;B715;1104 1172 11B0; # (뜕; 뜕; 뜕; 뜕; 뜕; ) HANGUL SYLLABLE DDYULG +B716;B716;1104 1172 11B1;B716;1104 1172 11B1; # (뜖; 뜖; 뜖; 뜖; 뜖; ) HANGUL SYLLABLE DDYULM +B717;B717;1104 1172 11B2;B717;1104 1172 11B2; # (뜗; 뜗; 뜗; 뜗; 뜗; ) HANGUL SYLLABLE DDYULB +B718;B718;1104 1172 11B3;B718;1104 1172 11B3; # (뜘; 뜘; 뜘; 뜘; 뜘; ) HANGUL SYLLABLE DDYULS +B719;B719;1104 1172 11B4;B719;1104 1172 11B4; # (뜙; 뜙; 뜙; 뜙; 뜙; ) HANGUL SYLLABLE DDYULT +B71A;B71A;1104 1172 11B5;B71A;1104 1172 11B5; # (뜚; 뜚; 뜚; 뜚; 뜚; ) HANGUL SYLLABLE DDYULP +B71B;B71B;1104 1172 11B6;B71B;1104 1172 11B6; # (뜛; 뜛; 뜛; 뜛; 뜛; ) HANGUL SYLLABLE DDYULH +B71C;B71C;1104 1172 11B7;B71C;1104 1172 11B7; # (뜜; 뜜; 뜜; 뜜; 뜜; ) HANGUL SYLLABLE DDYUM +B71D;B71D;1104 1172 11B8;B71D;1104 1172 11B8; # (뜝; 뜝; 뜝; 뜝; 뜝; ) HANGUL SYLLABLE DDYUB +B71E;B71E;1104 1172 11B9;B71E;1104 1172 11B9; # (뜞; 뜞; 뜞; 뜞; 뜞; ) HANGUL SYLLABLE DDYUBS +B71F;B71F;1104 1172 11BA;B71F;1104 1172 11BA; # (뜟; 뜟; 뜟; 뜟; 뜟; ) HANGUL SYLLABLE DDYUS +B720;B720;1104 1172 11BB;B720;1104 1172 11BB; # (뜠; 뜠; 뜠; 뜠; 뜠; ) HANGUL SYLLABLE DDYUSS +B721;B721;1104 1172 11BC;B721;1104 1172 11BC; # (뜡; 뜡; 뜡; 뜡; 뜡; ) HANGUL SYLLABLE DDYUNG +B722;B722;1104 1172 11BD;B722;1104 1172 11BD; # (뜢; 뜢; 뜢; 뜢; 뜢; ) HANGUL SYLLABLE DDYUJ +B723;B723;1104 1172 11BE;B723;1104 1172 11BE; # (뜣; 뜣; 뜣; 뜣; 뜣; ) HANGUL SYLLABLE DDYUC +B724;B724;1104 1172 11BF;B724;1104 1172 11BF; # (뜤; 뜤; 뜤; 뜤; 뜤; ) HANGUL SYLLABLE DDYUK +B725;B725;1104 1172 11C0;B725;1104 1172 11C0; # (뜥; 뜥; 뜥; 뜥; 뜥; ) HANGUL SYLLABLE DDYUT +B726;B726;1104 1172 11C1;B726;1104 1172 11C1; # (뜦; 뜦; 뜦; 뜦; 뜦; ) HANGUL SYLLABLE DDYUP +B727;B727;1104 1172 11C2;B727;1104 1172 11C2; # (뜧; 뜧; 뜧; 뜧; 뜧; ) HANGUL SYLLABLE DDYUH +B728;B728;1104 1173;B728;1104 1173; # (뜨; 뜨; 뜨; 뜨; 뜨; ) HANGUL SYLLABLE DDEU +B729;B729;1104 1173 11A8;B729;1104 1173 11A8; # (뜩; 뜩; 뜩; 뜩; 뜩; ) HANGUL SYLLABLE DDEUG +B72A;B72A;1104 1173 11A9;B72A;1104 1173 11A9; # (뜪; 뜪; 뜪; 뜪; 뜪; ) HANGUL SYLLABLE DDEUGG +B72B;B72B;1104 1173 11AA;B72B;1104 1173 11AA; # (뜫; 뜫; 뜫; 뜫; 뜫; ) HANGUL SYLLABLE DDEUGS +B72C;B72C;1104 1173 11AB;B72C;1104 1173 11AB; # (뜬; 뜬; 뜬; 뜬; 뜬; ) HANGUL SYLLABLE DDEUN +B72D;B72D;1104 1173 11AC;B72D;1104 1173 11AC; # (뜭; 뜭; 뜭; 뜭; 뜭; ) HANGUL SYLLABLE DDEUNJ +B72E;B72E;1104 1173 11AD;B72E;1104 1173 11AD; # (뜮; 뜮; 뜮; 뜮; 뜮; ) HANGUL SYLLABLE DDEUNH +B72F;B72F;1104 1173 11AE;B72F;1104 1173 11AE; # (뜯; 뜯; 뜯; 뜯; 뜯; ) HANGUL SYLLABLE DDEUD +B730;B730;1104 1173 11AF;B730;1104 1173 11AF; # (뜰; 뜰; 뜰; 뜰; 뜰; ) HANGUL SYLLABLE DDEUL +B731;B731;1104 1173 11B0;B731;1104 1173 11B0; # (뜱; 뜱; 뜱; 뜱; 뜱; ) HANGUL SYLLABLE DDEULG +B732;B732;1104 1173 11B1;B732;1104 1173 11B1; # (뜲; 뜲; 뜲; 뜲; 뜲; ) HANGUL SYLLABLE DDEULM +B733;B733;1104 1173 11B2;B733;1104 1173 11B2; # (뜳; 뜳; 뜳; 뜳; 뜳; ) HANGUL SYLLABLE DDEULB +B734;B734;1104 1173 11B3;B734;1104 1173 11B3; # (뜴; 뜴; 뜴; 뜴; 뜴; ) HANGUL SYLLABLE DDEULS +B735;B735;1104 1173 11B4;B735;1104 1173 11B4; # (뜵; 뜵; 뜵; 뜵; 뜵; ) HANGUL SYLLABLE DDEULT +B736;B736;1104 1173 11B5;B736;1104 1173 11B5; # (뜶; 뜶; 뜶; 뜶; 뜶; ) HANGUL SYLLABLE DDEULP +B737;B737;1104 1173 11B6;B737;1104 1173 11B6; # (뜷; 뜷; 뜷; 뜷; 뜷; ) HANGUL SYLLABLE DDEULH +B738;B738;1104 1173 11B7;B738;1104 1173 11B7; # (뜸; 뜸; 뜸; 뜸; 뜸; ) HANGUL SYLLABLE DDEUM +B739;B739;1104 1173 11B8;B739;1104 1173 11B8; # (뜹; 뜹; 뜹; 뜹; 뜹; ) HANGUL SYLLABLE DDEUB +B73A;B73A;1104 1173 11B9;B73A;1104 1173 11B9; # (뜺; 뜺; 뜺; 뜺; 뜺; ) HANGUL SYLLABLE DDEUBS +B73B;B73B;1104 1173 11BA;B73B;1104 1173 11BA; # (뜻; 뜻; 뜻; 뜻; 뜻; ) HANGUL SYLLABLE DDEUS +B73C;B73C;1104 1173 11BB;B73C;1104 1173 11BB; # (뜼; 뜼; 뜼; 뜼; 뜼; ) HANGUL SYLLABLE DDEUSS +B73D;B73D;1104 1173 11BC;B73D;1104 1173 11BC; # (뜽; 뜽; 뜽; 뜽; 뜽; ) HANGUL SYLLABLE DDEUNG +B73E;B73E;1104 1173 11BD;B73E;1104 1173 11BD; # (뜾; 뜾; 뜾; 뜾; 뜾; ) HANGUL SYLLABLE DDEUJ +B73F;B73F;1104 1173 11BE;B73F;1104 1173 11BE; # (뜿; 뜿; 뜿; 뜿; 뜿; ) HANGUL SYLLABLE DDEUC +B740;B740;1104 1173 11BF;B740;1104 1173 11BF; # (띀; 띀; 띀; 띀; 띀; ) HANGUL SYLLABLE DDEUK +B741;B741;1104 1173 11C0;B741;1104 1173 11C0; # (띁; 띁; 띁; 띁; 띁; ) HANGUL SYLLABLE DDEUT +B742;B742;1104 1173 11C1;B742;1104 1173 11C1; # (띂; 띂; 띂; 띂; 띂; ) HANGUL SYLLABLE DDEUP +B743;B743;1104 1173 11C2;B743;1104 1173 11C2; # (띃; 띃; 띃; 띃; 띃; ) HANGUL SYLLABLE DDEUH +B744;B744;1104 1174;B744;1104 1174; # (띄; 띄; 띄; 띄; 띄; ) HANGUL SYLLABLE DDYI +B745;B745;1104 1174 11A8;B745;1104 1174 11A8; # (띅; 띅; 띅; 띅; 띅; ) HANGUL SYLLABLE DDYIG +B746;B746;1104 1174 11A9;B746;1104 1174 11A9; # (띆; 띆; 띆; 띆; 띆; ) HANGUL SYLLABLE DDYIGG +B747;B747;1104 1174 11AA;B747;1104 1174 11AA; # (띇; 띇; 띇; 띇; 띇; ) HANGUL SYLLABLE DDYIGS +B748;B748;1104 1174 11AB;B748;1104 1174 11AB; # (띈; 띈; 띈; 띈; 띈; ) HANGUL SYLLABLE DDYIN +B749;B749;1104 1174 11AC;B749;1104 1174 11AC; # (띉; 띉; 띉; 띉; 띉; ) HANGUL SYLLABLE DDYINJ +B74A;B74A;1104 1174 11AD;B74A;1104 1174 11AD; # (띊; 띊; 띊; 띊; 띊; ) HANGUL SYLLABLE DDYINH +B74B;B74B;1104 1174 11AE;B74B;1104 1174 11AE; # (띋; 띋; 띋; 띋; 띋; ) HANGUL SYLLABLE DDYID +B74C;B74C;1104 1174 11AF;B74C;1104 1174 11AF; # (띌; 띌; 띌; 띌; 띌; ) HANGUL SYLLABLE DDYIL +B74D;B74D;1104 1174 11B0;B74D;1104 1174 11B0; # (띍; 띍; 띍; 띍; 띍; ) HANGUL SYLLABLE DDYILG +B74E;B74E;1104 1174 11B1;B74E;1104 1174 11B1; # (띎; 띎; 띎; 띎; 띎; ) HANGUL SYLLABLE DDYILM +B74F;B74F;1104 1174 11B2;B74F;1104 1174 11B2; # (띏; 띏; 띏; 띏; 띏; ) HANGUL SYLLABLE DDYILB +B750;B750;1104 1174 11B3;B750;1104 1174 11B3; # (띐; 띐; 띐; 띐; 띐; ) HANGUL SYLLABLE DDYILS +B751;B751;1104 1174 11B4;B751;1104 1174 11B4; # (띑; 띑; 띑; 띑; 띑; ) HANGUL SYLLABLE DDYILT +B752;B752;1104 1174 11B5;B752;1104 1174 11B5; # (띒; 띒; 띒; 띒; 띒; ) HANGUL SYLLABLE DDYILP +B753;B753;1104 1174 11B6;B753;1104 1174 11B6; # (띓; 띓; 띓; 띓; 띓; ) HANGUL SYLLABLE DDYILH +B754;B754;1104 1174 11B7;B754;1104 1174 11B7; # (띔; 띔; 띔; 띔; 띔; ) HANGUL SYLLABLE DDYIM +B755;B755;1104 1174 11B8;B755;1104 1174 11B8; # (띕; 띕; 띕; 띕; 띕; ) HANGUL SYLLABLE DDYIB +B756;B756;1104 1174 11B9;B756;1104 1174 11B9; # (띖; 띖; 띖; 띖; 띖; ) HANGUL SYLLABLE DDYIBS +B757;B757;1104 1174 11BA;B757;1104 1174 11BA; # (띗; 띗; 띗; 띗; 띗; ) HANGUL SYLLABLE DDYIS +B758;B758;1104 1174 11BB;B758;1104 1174 11BB; # (띘; 띘; 띘; 띘; 띘; ) HANGUL SYLLABLE DDYISS +B759;B759;1104 1174 11BC;B759;1104 1174 11BC; # (띙; 띙; 띙; 띙; 띙; ) HANGUL SYLLABLE DDYING +B75A;B75A;1104 1174 11BD;B75A;1104 1174 11BD; # (띚; 띚; 띚; 띚; 띚; ) HANGUL SYLLABLE DDYIJ +B75B;B75B;1104 1174 11BE;B75B;1104 1174 11BE; # (띛; 띛; 띛; 띛; 띛; ) HANGUL SYLLABLE DDYIC +B75C;B75C;1104 1174 11BF;B75C;1104 1174 11BF; # (띜; 띜; 띜; 띜; 띜; ) HANGUL SYLLABLE DDYIK +B75D;B75D;1104 1174 11C0;B75D;1104 1174 11C0; # (띝; 띝; 띝; 띝; 띝; ) HANGUL SYLLABLE DDYIT +B75E;B75E;1104 1174 11C1;B75E;1104 1174 11C1; # (띞; 띞; 띞; 띞; 띞; ) HANGUL SYLLABLE DDYIP +B75F;B75F;1104 1174 11C2;B75F;1104 1174 11C2; # (띟; 띟; 띟; 띟; 띟; ) HANGUL SYLLABLE DDYIH +B760;B760;1104 1175;B760;1104 1175; # (띠; 띠; 띠; 띠; 띠; ) HANGUL SYLLABLE DDI +B761;B761;1104 1175 11A8;B761;1104 1175 11A8; # (띡; 띡; 띡; 띡; 띡; ) HANGUL SYLLABLE DDIG +B762;B762;1104 1175 11A9;B762;1104 1175 11A9; # (띢; 띢; 띢; 띢; 띢; ) HANGUL SYLLABLE DDIGG +B763;B763;1104 1175 11AA;B763;1104 1175 11AA; # (띣; 띣; 띣; 띣; 띣; ) HANGUL SYLLABLE DDIGS +B764;B764;1104 1175 11AB;B764;1104 1175 11AB; # (띤; 띤; 띤; 띤; 띤; ) HANGUL SYLLABLE DDIN +B765;B765;1104 1175 11AC;B765;1104 1175 11AC; # (띥; 띥; 띥; 띥; 띥; ) HANGUL SYLLABLE DDINJ +B766;B766;1104 1175 11AD;B766;1104 1175 11AD; # (띦; 띦; 띦; 띦; 띦; ) HANGUL SYLLABLE DDINH +B767;B767;1104 1175 11AE;B767;1104 1175 11AE; # (띧; 띧; 띧; 띧; 띧; ) HANGUL SYLLABLE DDID +B768;B768;1104 1175 11AF;B768;1104 1175 11AF; # (띨; 띨; 띨; 띨; 띨; ) HANGUL SYLLABLE DDIL +B769;B769;1104 1175 11B0;B769;1104 1175 11B0; # (띩; 띩; 띩; 띩; 띩; ) HANGUL SYLLABLE DDILG +B76A;B76A;1104 1175 11B1;B76A;1104 1175 11B1; # (띪; 띪; 띪; 띪; 띪; ) HANGUL SYLLABLE DDILM +B76B;B76B;1104 1175 11B2;B76B;1104 1175 11B2; # (띫; 띫; 띫; 띫; 띫; ) HANGUL SYLLABLE DDILB +B76C;B76C;1104 1175 11B3;B76C;1104 1175 11B3; # (띬; 띬; 띬; 띬; 띬; ) HANGUL SYLLABLE DDILS +B76D;B76D;1104 1175 11B4;B76D;1104 1175 11B4; # (띭; 띭; 띭; 띭; 띭; ) HANGUL SYLLABLE DDILT +B76E;B76E;1104 1175 11B5;B76E;1104 1175 11B5; # (띮; 띮; 띮; 띮; 띮; ) HANGUL SYLLABLE DDILP +B76F;B76F;1104 1175 11B6;B76F;1104 1175 11B6; # (띯; 띯; 띯; 띯; 띯; ) HANGUL SYLLABLE DDILH +B770;B770;1104 1175 11B7;B770;1104 1175 11B7; # (띰; 띰; 띰; 띰; 띰; ) HANGUL SYLLABLE DDIM +B771;B771;1104 1175 11B8;B771;1104 1175 11B8; # (띱; 띱; 띱; 띱; 띱; ) HANGUL SYLLABLE DDIB +B772;B772;1104 1175 11B9;B772;1104 1175 11B9; # (띲; 띲; 띲; 띲; 띲; ) HANGUL SYLLABLE DDIBS +B773;B773;1104 1175 11BA;B773;1104 1175 11BA; # (띳; 띳; 띳; 띳; 띳; ) HANGUL SYLLABLE DDIS +B774;B774;1104 1175 11BB;B774;1104 1175 11BB; # (띴; 띴; 띴; 띴; 띴; ) HANGUL SYLLABLE DDISS +B775;B775;1104 1175 11BC;B775;1104 1175 11BC; # (띵; 띵; 띵; 띵; 띵; ) HANGUL SYLLABLE DDING +B776;B776;1104 1175 11BD;B776;1104 1175 11BD; # (띶; 띶; 띶; 띶; 띶; ) HANGUL SYLLABLE DDIJ +B777;B777;1104 1175 11BE;B777;1104 1175 11BE; # (띷; 띷; 띷; 띷; 띷; ) HANGUL SYLLABLE DDIC +B778;B778;1104 1175 11BF;B778;1104 1175 11BF; # (띸; 띸; 띸; 띸; 띸; ) HANGUL SYLLABLE DDIK +B779;B779;1104 1175 11C0;B779;1104 1175 11C0; # (띹; 띹; 띹; 띹; 띹; ) HANGUL SYLLABLE DDIT +B77A;B77A;1104 1175 11C1;B77A;1104 1175 11C1; # (띺; 띺; 띺; 띺; 띺; ) HANGUL SYLLABLE DDIP +B77B;B77B;1104 1175 11C2;B77B;1104 1175 11C2; # (띻; 띻; 띻; 띻; 띻; ) HANGUL SYLLABLE DDIH +B77C;B77C;1105 1161;B77C;1105 1161; # (라; 라; 라; 라; 라; ) HANGUL SYLLABLE RA +B77D;B77D;1105 1161 11A8;B77D;1105 1161 11A8; # (락; 락; 락; 락; 락; ) HANGUL SYLLABLE RAG +B77E;B77E;1105 1161 11A9;B77E;1105 1161 11A9; # (띾; 띾; 띾; 띾; 띾; ) HANGUL SYLLABLE RAGG +B77F;B77F;1105 1161 11AA;B77F;1105 1161 11AA; # (띿; 띿; 띿; 띿; 띿; ) HANGUL SYLLABLE RAGS +B780;B780;1105 1161 11AB;B780;1105 1161 11AB; # (란; 란; 란; 란; 란; ) HANGUL SYLLABLE RAN +B781;B781;1105 1161 11AC;B781;1105 1161 11AC; # (랁; 랁; 랁; 랁; 랁; ) HANGUL SYLLABLE RANJ +B782;B782;1105 1161 11AD;B782;1105 1161 11AD; # (랂; 랂; 랂; 랂; 랂; ) HANGUL SYLLABLE RANH +B783;B783;1105 1161 11AE;B783;1105 1161 11AE; # (랃; 랃; 랃; 랃; 랃; ) HANGUL SYLLABLE RAD +B784;B784;1105 1161 11AF;B784;1105 1161 11AF; # (랄; 랄; 랄; 랄; 랄; ) HANGUL SYLLABLE RAL +B785;B785;1105 1161 11B0;B785;1105 1161 11B0; # (랅; 랅; 랅; 랅; 랅; ) HANGUL SYLLABLE RALG +B786;B786;1105 1161 11B1;B786;1105 1161 11B1; # (랆; 랆; 랆; 랆; 랆; ) HANGUL SYLLABLE RALM +B787;B787;1105 1161 11B2;B787;1105 1161 11B2; # (랇; 랇; 랇; 랇; 랇; ) HANGUL SYLLABLE RALB +B788;B788;1105 1161 11B3;B788;1105 1161 11B3; # (랈; 랈; 랈; 랈; 랈; ) HANGUL SYLLABLE RALS +B789;B789;1105 1161 11B4;B789;1105 1161 11B4; # (랉; 랉; 랉; 랉; 랉; ) HANGUL SYLLABLE RALT +B78A;B78A;1105 1161 11B5;B78A;1105 1161 11B5; # (랊; 랊; 랊; 랊; 랊; ) HANGUL SYLLABLE RALP +B78B;B78B;1105 1161 11B6;B78B;1105 1161 11B6; # (랋; 랋; 랋; 랋; 랋; ) HANGUL SYLLABLE RALH +B78C;B78C;1105 1161 11B7;B78C;1105 1161 11B7; # (람; 람; 람; 람; 람; ) HANGUL SYLLABLE RAM +B78D;B78D;1105 1161 11B8;B78D;1105 1161 11B8; # (랍; 랍; 랍; 랍; 랍; ) HANGUL SYLLABLE RAB +B78E;B78E;1105 1161 11B9;B78E;1105 1161 11B9; # (랎; 랎; 랎; 랎; 랎; ) HANGUL SYLLABLE RABS +B78F;B78F;1105 1161 11BA;B78F;1105 1161 11BA; # (랏; 랏; 랏; 랏; 랏; ) HANGUL SYLLABLE RAS +B790;B790;1105 1161 11BB;B790;1105 1161 11BB; # (랐; 랐; 랐; 랐; 랐; ) HANGUL SYLLABLE RASS +B791;B791;1105 1161 11BC;B791;1105 1161 11BC; # (랑; 랑; 랑; 랑; 랑; ) HANGUL SYLLABLE RANG +B792;B792;1105 1161 11BD;B792;1105 1161 11BD; # (랒; 랒; 랒; 랒; 랒; ) HANGUL SYLLABLE RAJ +B793;B793;1105 1161 11BE;B793;1105 1161 11BE; # (랓; 랓; 랓; 랓; 랓; ) HANGUL SYLLABLE RAC +B794;B794;1105 1161 11BF;B794;1105 1161 11BF; # (랔; 랔; 랔; 랔; 랔; ) HANGUL SYLLABLE RAK +B795;B795;1105 1161 11C0;B795;1105 1161 11C0; # (랕; 랕; 랕; 랕; 랕; ) HANGUL SYLLABLE RAT +B796;B796;1105 1161 11C1;B796;1105 1161 11C1; # (랖; 랖; 랖; 랖; 랖; ) HANGUL SYLLABLE RAP +B797;B797;1105 1161 11C2;B797;1105 1161 11C2; # (랗; 랗; 랗; 랗; 랗; ) HANGUL SYLLABLE RAH +B798;B798;1105 1162;B798;1105 1162; # (래; 래; 래; 래; 래; ) HANGUL SYLLABLE RAE +B799;B799;1105 1162 11A8;B799;1105 1162 11A8; # (랙; 랙; 랙; 랙; 랙; ) HANGUL SYLLABLE RAEG +B79A;B79A;1105 1162 11A9;B79A;1105 1162 11A9; # (랚; 랚; 랚; 랚; 랚; ) HANGUL SYLLABLE RAEGG +B79B;B79B;1105 1162 11AA;B79B;1105 1162 11AA; # (랛; 랛; 랛; 랛; 랛; ) HANGUL SYLLABLE RAEGS +B79C;B79C;1105 1162 11AB;B79C;1105 1162 11AB; # (랜; 랜; 랜; 랜; 랜; ) HANGUL SYLLABLE RAEN +B79D;B79D;1105 1162 11AC;B79D;1105 1162 11AC; # (랝; 랝; 랝; 랝; 랝; ) HANGUL SYLLABLE RAENJ +B79E;B79E;1105 1162 11AD;B79E;1105 1162 11AD; # (랞; 랞; 랞; 랞; 랞; ) HANGUL SYLLABLE RAENH +B79F;B79F;1105 1162 11AE;B79F;1105 1162 11AE; # (랟; 랟; 랟; 랟; 랟; ) HANGUL SYLLABLE RAED +B7A0;B7A0;1105 1162 11AF;B7A0;1105 1162 11AF; # (랠; 랠; 랠; 랠; 랠; ) HANGUL SYLLABLE RAEL +B7A1;B7A1;1105 1162 11B0;B7A1;1105 1162 11B0; # (랡; 랡; 랡; 랡; 랡; ) HANGUL SYLLABLE RAELG +B7A2;B7A2;1105 1162 11B1;B7A2;1105 1162 11B1; # (랢; 랢; 랢; 랢; 랢; ) HANGUL SYLLABLE RAELM +B7A3;B7A3;1105 1162 11B2;B7A3;1105 1162 11B2; # (랣; 랣; 랣; 랣; 랣; ) HANGUL SYLLABLE RAELB +B7A4;B7A4;1105 1162 11B3;B7A4;1105 1162 11B3; # (랤; 랤; 랤; 랤; 랤; ) HANGUL SYLLABLE RAELS +B7A5;B7A5;1105 1162 11B4;B7A5;1105 1162 11B4; # (랥; 랥; 랥; 랥; 랥; ) HANGUL SYLLABLE RAELT +B7A6;B7A6;1105 1162 11B5;B7A6;1105 1162 11B5; # (랦; 랦; 랦; 랦; 랦; ) HANGUL SYLLABLE RAELP +B7A7;B7A7;1105 1162 11B6;B7A7;1105 1162 11B6; # (랧; 랧; 랧; 랧; 랧; ) HANGUL SYLLABLE RAELH +B7A8;B7A8;1105 1162 11B7;B7A8;1105 1162 11B7; # (램; 램; 램; 램; 램; ) HANGUL SYLLABLE RAEM +B7A9;B7A9;1105 1162 11B8;B7A9;1105 1162 11B8; # (랩; 랩; 랩; 랩; 랩; ) HANGUL SYLLABLE RAEB +B7AA;B7AA;1105 1162 11B9;B7AA;1105 1162 11B9; # (랪; 랪; 랪; 랪; 랪; ) HANGUL SYLLABLE RAEBS +B7AB;B7AB;1105 1162 11BA;B7AB;1105 1162 11BA; # (랫; 랫; 랫; 랫; 랫; ) HANGUL SYLLABLE RAES +B7AC;B7AC;1105 1162 11BB;B7AC;1105 1162 11BB; # (랬; 랬; 랬; 랬; 랬; ) HANGUL SYLLABLE RAESS +B7AD;B7AD;1105 1162 11BC;B7AD;1105 1162 11BC; # (랭; 랭; 랭; 랭; 랭; ) HANGUL SYLLABLE RAENG +B7AE;B7AE;1105 1162 11BD;B7AE;1105 1162 11BD; # (랮; 랮; 랮; 랮; 랮; ) HANGUL SYLLABLE RAEJ +B7AF;B7AF;1105 1162 11BE;B7AF;1105 1162 11BE; # (랯; 랯; 랯; 랯; 랯; ) HANGUL SYLLABLE RAEC +B7B0;B7B0;1105 1162 11BF;B7B0;1105 1162 11BF; # (랰; 랰; 랰; 랰; 랰; ) HANGUL SYLLABLE RAEK +B7B1;B7B1;1105 1162 11C0;B7B1;1105 1162 11C0; # (랱; 랱; 랱; 랱; 랱; ) HANGUL SYLLABLE RAET +B7B2;B7B2;1105 1162 11C1;B7B2;1105 1162 11C1; # (랲; 랲; 랲; 랲; 랲; ) HANGUL SYLLABLE RAEP +B7B3;B7B3;1105 1162 11C2;B7B3;1105 1162 11C2; # (랳; 랳; 랳; 랳; 랳; ) HANGUL SYLLABLE RAEH +B7B4;B7B4;1105 1163;B7B4;1105 1163; # (랴; 랴; 랴; 랴; 랴; ) HANGUL SYLLABLE RYA +B7B5;B7B5;1105 1163 11A8;B7B5;1105 1163 11A8; # (략; 략; 략; 략; 략; ) HANGUL SYLLABLE RYAG +B7B6;B7B6;1105 1163 11A9;B7B6;1105 1163 11A9; # (랶; 랶; 랶; 랶; 랶; ) HANGUL SYLLABLE RYAGG +B7B7;B7B7;1105 1163 11AA;B7B7;1105 1163 11AA; # (랷; 랷; 랷; 랷; 랷; ) HANGUL SYLLABLE RYAGS +B7B8;B7B8;1105 1163 11AB;B7B8;1105 1163 11AB; # (랸; 랸; 랸; 랸; 랸; ) HANGUL SYLLABLE RYAN +B7B9;B7B9;1105 1163 11AC;B7B9;1105 1163 11AC; # (랹; 랹; 랹; 랹; 랹; ) HANGUL SYLLABLE RYANJ +B7BA;B7BA;1105 1163 11AD;B7BA;1105 1163 11AD; # (랺; 랺; 랺; 랺; 랺; ) HANGUL SYLLABLE RYANH +B7BB;B7BB;1105 1163 11AE;B7BB;1105 1163 11AE; # (랻; 랻; 랻; 랻; 랻; ) HANGUL SYLLABLE RYAD +B7BC;B7BC;1105 1163 11AF;B7BC;1105 1163 11AF; # (랼; 랼; 랼; 랼; 랼; ) HANGUL SYLLABLE RYAL +B7BD;B7BD;1105 1163 11B0;B7BD;1105 1163 11B0; # (랽; 랽; 랽; 랽; 랽; ) HANGUL SYLLABLE RYALG +B7BE;B7BE;1105 1163 11B1;B7BE;1105 1163 11B1; # (랾; 랾; 랾; 랾; 랾; ) HANGUL SYLLABLE RYALM +B7BF;B7BF;1105 1163 11B2;B7BF;1105 1163 11B2; # (랿; 랿; 랿; 랿; 랿; ) HANGUL SYLLABLE RYALB +B7C0;B7C0;1105 1163 11B3;B7C0;1105 1163 11B3; # (럀; 럀; 럀; 럀; 럀; ) HANGUL SYLLABLE RYALS +B7C1;B7C1;1105 1163 11B4;B7C1;1105 1163 11B4; # (럁; 럁; 럁; 럁; 럁; ) HANGUL SYLLABLE RYALT +B7C2;B7C2;1105 1163 11B5;B7C2;1105 1163 11B5; # (럂; 럂; 럂; 럂; 럂; ) HANGUL SYLLABLE RYALP +B7C3;B7C3;1105 1163 11B6;B7C3;1105 1163 11B6; # (럃; 럃; 럃; 럃; 럃; ) HANGUL SYLLABLE RYALH +B7C4;B7C4;1105 1163 11B7;B7C4;1105 1163 11B7; # (럄; 럄; 럄; 럄; 럄; ) HANGUL SYLLABLE RYAM +B7C5;B7C5;1105 1163 11B8;B7C5;1105 1163 11B8; # (럅; 럅; 럅; 럅; 럅; ) HANGUL SYLLABLE RYAB +B7C6;B7C6;1105 1163 11B9;B7C6;1105 1163 11B9; # (럆; 럆; 럆; 럆; 럆; ) HANGUL SYLLABLE RYABS +B7C7;B7C7;1105 1163 11BA;B7C7;1105 1163 11BA; # (럇; 럇; 럇; 럇; 럇; ) HANGUL SYLLABLE RYAS +B7C8;B7C8;1105 1163 11BB;B7C8;1105 1163 11BB; # (럈; 럈; 럈; 럈; 럈; ) HANGUL SYLLABLE RYASS +B7C9;B7C9;1105 1163 11BC;B7C9;1105 1163 11BC; # (량; 량; 량; 량; 량; ) HANGUL SYLLABLE RYANG +B7CA;B7CA;1105 1163 11BD;B7CA;1105 1163 11BD; # (럊; 럊; 럊; 럊; 럊; ) HANGUL SYLLABLE RYAJ +B7CB;B7CB;1105 1163 11BE;B7CB;1105 1163 11BE; # (럋; 럋; 럋; 럋; 럋; ) HANGUL SYLLABLE RYAC +B7CC;B7CC;1105 1163 11BF;B7CC;1105 1163 11BF; # (럌; 럌; 럌; 럌; 럌; ) HANGUL SYLLABLE RYAK +B7CD;B7CD;1105 1163 11C0;B7CD;1105 1163 11C0; # (럍; 럍; 럍; 럍; 럍; ) HANGUL SYLLABLE RYAT +B7CE;B7CE;1105 1163 11C1;B7CE;1105 1163 11C1; # (럎; 럎; 럎; 럎; 럎; ) HANGUL SYLLABLE RYAP +B7CF;B7CF;1105 1163 11C2;B7CF;1105 1163 11C2; # (럏; 럏; 럏; 럏; 럏; ) HANGUL SYLLABLE RYAH +B7D0;B7D0;1105 1164;B7D0;1105 1164; # (럐; 럐; 럐; 럐; 럐; ) HANGUL SYLLABLE RYAE +B7D1;B7D1;1105 1164 11A8;B7D1;1105 1164 11A8; # (럑; 럑; 럑; 럑; 럑; ) HANGUL SYLLABLE RYAEG +B7D2;B7D2;1105 1164 11A9;B7D2;1105 1164 11A9; # (럒; 럒; 럒; 럒; 럒; ) HANGUL SYLLABLE RYAEGG +B7D3;B7D3;1105 1164 11AA;B7D3;1105 1164 11AA; # (럓; 럓; 럓; 럓; 럓; ) HANGUL SYLLABLE RYAEGS +B7D4;B7D4;1105 1164 11AB;B7D4;1105 1164 11AB; # (럔; 럔; 럔; 럔; 럔; ) HANGUL SYLLABLE RYAEN +B7D5;B7D5;1105 1164 11AC;B7D5;1105 1164 11AC; # (럕; 럕; 럕; 럕; 럕; ) HANGUL SYLLABLE RYAENJ +B7D6;B7D6;1105 1164 11AD;B7D6;1105 1164 11AD; # (럖; 럖; 럖; 럖; 럖; ) HANGUL SYLLABLE RYAENH +B7D7;B7D7;1105 1164 11AE;B7D7;1105 1164 11AE; # (럗; 럗; 럗; 럗; 럗; ) HANGUL SYLLABLE RYAED +B7D8;B7D8;1105 1164 11AF;B7D8;1105 1164 11AF; # (럘; 럘; 럘; 럘; 럘; ) HANGUL SYLLABLE RYAEL +B7D9;B7D9;1105 1164 11B0;B7D9;1105 1164 11B0; # (럙; 럙; 럙; 럙; 럙; ) HANGUL SYLLABLE RYAELG +B7DA;B7DA;1105 1164 11B1;B7DA;1105 1164 11B1; # (럚; 럚; 럚; 럚; 럚; ) HANGUL SYLLABLE RYAELM +B7DB;B7DB;1105 1164 11B2;B7DB;1105 1164 11B2; # (럛; 럛; 럛; 럛; 럛; ) HANGUL SYLLABLE RYAELB +B7DC;B7DC;1105 1164 11B3;B7DC;1105 1164 11B3; # (럜; 럜; 럜; 럜; 럜; ) HANGUL SYLLABLE RYAELS +B7DD;B7DD;1105 1164 11B4;B7DD;1105 1164 11B4; # (럝; 럝; 럝; 럝; 럝; ) HANGUL SYLLABLE RYAELT +B7DE;B7DE;1105 1164 11B5;B7DE;1105 1164 11B5; # (럞; 럞; 럞; 럞; 럞; ) HANGUL SYLLABLE RYAELP +B7DF;B7DF;1105 1164 11B6;B7DF;1105 1164 11B6; # (럟; 럟; 럟; 럟; 럟; ) HANGUL SYLLABLE RYAELH +B7E0;B7E0;1105 1164 11B7;B7E0;1105 1164 11B7; # (럠; 럠; 럠; 럠; 럠; ) HANGUL SYLLABLE RYAEM +B7E1;B7E1;1105 1164 11B8;B7E1;1105 1164 11B8; # (럡; 럡; 럡; 럡; 럡; ) HANGUL SYLLABLE RYAEB +B7E2;B7E2;1105 1164 11B9;B7E2;1105 1164 11B9; # (럢; 럢; 럢; 럢; 럢; ) HANGUL SYLLABLE RYAEBS +B7E3;B7E3;1105 1164 11BA;B7E3;1105 1164 11BA; # (럣; 럣; 럣; 럣; 럣; ) HANGUL SYLLABLE RYAES +B7E4;B7E4;1105 1164 11BB;B7E4;1105 1164 11BB; # (럤; 럤; 럤; 럤; 럤; ) HANGUL SYLLABLE RYAESS +B7E5;B7E5;1105 1164 11BC;B7E5;1105 1164 11BC; # (럥; 럥; 럥; 럥; 럥; ) HANGUL SYLLABLE RYAENG +B7E6;B7E6;1105 1164 11BD;B7E6;1105 1164 11BD; # (럦; 럦; 럦; 럦; 럦; ) HANGUL SYLLABLE RYAEJ +B7E7;B7E7;1105 1164 11BE;B7E7;1105 1164 11BE; # (럧; 럧; 럧; 럧; 럧; ) HANGUL SYLLABLE RYAEC +B7E8;B7E8;1105 1164 11BF;B7E8;1105 1164 11BF; # (럨; 럨; 럨; 럨; 럨; ) HANGUL SYLLABLE RYAEK +B7E9;B7E9;1105 1164 11C0;B7E9;1105 1164 11C0; # (럩; 럩; 럩; 럩; 럩; ) HANGUL SYLLABLE RYAET +B7EA;B7EA;1105 1164 11C1;B7EA;1105 1164 11C1; # (럪; 럪; 럪; 럪; 럪; ) HANGUL SYLLABLE RYAEP +B7EB;B7EB;1105 1164 11C2;B7EB;1105 1164 11C2; # (럫; 럫; 럫; 럫; 럫; ) HANGUL SYLLABLE RYAEH +B7EC;B7EC;1105 1165;B7EC;1105 1165; # (러; 러; 러; 러; 러; ) HANGUL SYLLABLE REO +B7ED;B7ED;1105 1165 11A8;B7ED;1105 1165 11A8; # (럭; 럭; 럭; 럭; 럭; ) HANGUL SYLLABLE REOG +B7EE;B7EE;1105 1165 11A9;B7EE;1105 1165 11A9; # (럮; 럮; 럮; 럮; 럮; ) HANGUL SYLLABLE REOGG +B7EF;B7EF;1105 1165 11AA;B7EF;1105 1165 11AA; # (럯; 럯; 럯; 럯; 럯; ) HANGUL SYLLABLE REOGS +B7F0;B7F0;1105 1165 11AB;B7F0;1105 1165 11AB; # (런; 런; 런; 런; 런; ) HANGUL SYLLABLE REON +B7F1;B7F1;1105 1165 11AC;B7F1;1105 1165 11AC; # (럱; 럱; 럱; 럱; 럱; ) HANGUL SYLLABLE REONJ +B7F2;B7F2;1105 1165 11AD;B7F2;1105 1165 11AD; # (럲; 럲; 럲; 럲; 럲; ) HANGUL SYLLABLE REONH +B7F3;B7F3;1105 1165 11AE;B7F3;1105 1165 11AE; # (럳; 럳; 럳; 럳; 럳; ) HANGUL SYLLABLE REOD +B7F4;B7F4;1105 1165 11AF;B7F4;1105 1165 11AF; # (럴; 럴; 럴; 럴; 럴; ) HANGUL SYLLABLE REOL +B7F5;B7F5;1105 1165 11B0;B7F5;1105 1165 11B0; # (럵; 럵; 럵; 럵; 럵; ) HANGUL SYLLABLE REOLG +B7F6;B7F6;1105 1165 11B1;B7F6;1105 1165 11B1; # (럶; 럶; 럶; 럶; 럶; ) HANGUL SYLLABLE REOLM +B7F7;B7F7;1105 1165 11B2;B7F7;1105 1165 11B2; # (럷; 럷; 럷; 럷; 럷; ) HANGUL SYLLABLE REOLB +B7F8;B7F8;1105 1165 11B3;B7F8;1105 1165 11B3; # (럸; 럸; 럸; 럸; 럸; ) HANGUL SYLLABLE REOLS +B7F9;B7F9;1105 1165 11B4;B7F9;1105 1165 11B4; # (럹; 럹; 럹; 럹; 럹; ) HANGUL SYLLABLE REOLT +B7FA;B7FA;1105 1165 11B5;B7FA;1105 1165 11B5; # (럺; 럺; 럺; 럺; 럺; ) HANGUL SYLLABLE REOLP +B7FB;B7FB;1105 1165 11B6;B7FB;1105 1165 11B6; # (럻; 럻; 럻; 럻; 럻; ) HANGUL SYLLABLE REOLH +B7FC;B7FC;1105 1165 11B7;B7FC;1105 1165 11B7; # (럼; 럼; 럼; 럼; 럼; ) HANGUL SYLLABLE REOM +B7FD;B7FD;1105 1165 11B8;B7FD;1105 1165 11B8; # (럽; 럽; 럽; 럽; 럽; ) HANGUL SYLLABLE REOB +B7FE;B7FE;1105 1165 11B9;B7FE;1105 1165 11B9; # (럾; 럾; 럾; 럾; 럾; ) HANGUL SYLLABLE REOBS +B7FF;B7FF;1105 1165 11BA;B7FF;1105 1165 11BA; # (럿; 럿; 럿; 럿; 럿; ) HANGUL SYLLABLE REOS +B800;B800;1105 1165 11BB;B800;1105 1165 11BB; # (렀; 렀; 렀; 렀; 렀; ) HANGUL SYLLABLE REOSS +B801;B801;1105 1165 11BC;B801;1105 1165 11BC; # (렁; 렁; 렁; 렁; 렁; ) HANGUL SYLLABLE REONG +B802;B802;1105 1165 11BD;B802;1105 1165 11BD; # (렂; 렂; 렂; 렂; 렂; ) HANGUL SYLLABLE REOJ +B803;B803;1105 1165 11BE;B803;1105 1165 11BE; # (렃; 렃; 렃; 렃; 렃; ) HANGUL SYLLABLE REOC +B804;B804;1105 1165 11BF;B804;1105 1165 11BF; # (렄; 렄; 렄; 렄; 렄; ) HANGUL SYLLABLE REOK +B805;B805;1105 1165 11C0;B805;1105 1165 11C0; # (렅; 렅; 렅; 렅; 렅; ) HANGUL SYLLABLE REOT +B806;B806;1105 1165 11C1;B806;1105 1165 11C1; # (렆; 렆; 렆; 렆; 렆; ) HANGUL SYLLABLE REOP +B807;B807;1105 1165 11C2;B807;1105 1165 11C2; # (렇; 렇; 렇; 렇; 렇; ) HANGUL SYLLABLE REOH +B808;B808;1105 1166;B808;1105 1166; # (레; 레; 레; 레; 레; ) HANGUL SYLLABLE RE +B809;B809;1105 1166 11A8;B809;1105 1166 11A8; # (렉; 렉; 렉; 렉; 렉; ) HANGUL SYLLABLE REG +B80A;B80A;1105 1166 11A9;B80A;1105 1166 11A9; # (렊; 렊; 렊; 렊; 렊; ) HANGUL SYLLABLE REGG +B80B;B80B;1105 1166 11AA;B80B;1105 1166 11AA; # (렋; 렋; 렋; 렋; 렋; ) HANGUL SYLLABLE REGS +B80C;B80C;1105 1166 11AB;B80C;1105 1166 11AB; # (렌; 렌; 렌; 렌; 렌; ) HANGUL SYLLABLE REN +B80D;B80D;1105 1166 11AC;B80D;1105 1166 11AC; # (렍; 렍; 렍; 렍; 렍; ) HANGUL SYLLABLE RENJ +B80E;B80E;1105 1166 11AD;B80E;1105 1166 11AD; # (렎; 렎; 렎; 렎; 렎; ) HANGUL SYLLABLE RENH +B80F;B80F;1105 1166 11AE;B80F;1105 1166 11AE; # (렏; 렏; 렏; 렏; 렏; ) HANGUL SYLLABLE RED +B810;B810;1105 1166 11AF;B810;1105 1166 11AF; # (렐; 렐; 렐; 렐; 렐; ) HANGUL SYLLABLE REL +B811;B811;1105 1166 11B0;B811;1105 1166 11B0; # (렑; 렑; 렑; 렑; 렑; ) HANGUL SYLLABLE RELG +B812;B812;1105 1166 11B1;B812;1105 1166 11B1; # (렒; 렒; 렒; 렒; 렒; ) HANGUL SYLLABLE RELM +B813;B813;1105 1166 11B2;B813;1105 1166 11B2; # (렓; 렓; 렓; 렓; 렓; ) HANGUL SYLLABLE RELB +B814;B814;1105 1166 11B3;B814;1105 1166 11B3; # (렔; 렔; 렔; 렔; 렔; ) HANGUL SYLLABLE RELS +B815;B815;1105 1166 11B4;B815;1105 1166 11B4; # (렕; 렕; 렕; 렕; 렕; ) HANGUL SYLLABLE RELT +B816;B816;1105 1166 11B5;B816;1105 1166 11B5; # (렖; 렖; 렖; 렖; 렖; ) HANGUL SYLLABLE RELP +B817;B817;1105 1166 11B6;B817;1105 1166 11B6; # (렗; 렗; 렗; 렗; 렗; ) HANGUL SYLLABLE RELH +B818;B818;1105 1166 11B7;B818;1105 1166 11B7; # (렘; 렘; 렘; 렘; 렘; ) HANGUL SYLLABLE REM +B819;B819;1105 1166 11B8;B819;1105 1166 11B8; # (렙; 렙; 렙; 렙; 렙; ) HANGUL SYLLABLE REB +B81A;B81A;1105 1166 11B9;B81A;1105 1166 11B9; # (렚; 렚; 렚; 렚; 렚; ) HANGUL SYLLABLE REBS +B81B;B81B;1105 1166 11BA;B81B;1105 1166 11BA; # (렛; 렛; 렛; 렛; 렛; ) HANGUL SYLLABLE RES +B81C;B81C;1105 1166 11BB;B81C;1105 1166 11BB; # (렜; 렜; 렜; 렜; 렜; ) HANGUL SYLLABLE RESS +B81D;B81D;1105 1166 11BC;B81D;1105 1166 11BC; # (렝; 렝; 렝; 렝; 렝; ) HANGUL SYLLABLE RENG +B81E;B81E;1105 1166 11BD;B81E;1105 1166 11BD; # (렞; 렞; 렞; 렞; 렞; ) HANGUL SYLLABLE REJ +B81F;B81F;1105 1166 11BE;B81F;1105 1166 11BE; # (렟; 렟; 렟; 렟; 렟; ) HANGUL SYLLABLE REC +B820;B820;1105 1166 11BF;B820;1105 1166 11BF; # (렠; 렠; 렠; 렠; 렠; ) HANGUL SYLLABLE REK +B821;B821;1105 1166 11C0;B821;1105 1166 11C0; # (렡; 렡; 렡; 렡; 렡; ) HANGUL SYLLABLE RET +B822;B822;1105 1166 11C1;B822;1105 1166 11C1; # (렢; 렢; 렢; 렢; 렢; ) HANGUL SYLLABLE REP +B823;B823;1105 1166 11C2;B823;1105 1166 11C2; # (렣; 렣; 렣; 렣; 렣; ) HANGUL SYLLABLE REH +B824;B824;1105 1167;B824;1105 1167; # (려; 려; 려; 려; 려; ) HANGUL SYLLABLE RYEO +B825;B825;1105 1167 11A8;B825;1105 1167 11A8; # (력; 력; 력; 력; 력; ) HANGUL SYLLABLE RYEOG +B826;B826;1105 1167 11A9;B826;1105 1167 11A9; # (렦; 렦; 렦; 렦; 렦; ) HANGUL SYLLABLE RYEOGG +B827;B827;1105 1167 11AA;B827;1105 1167 11AA; # (렧; 렧; 렧; 렧; 렧; ) HANGUL SYLLABLE RYEOGS +B828;B828;1105 1167 11AB;B828;1105 1167 11AB; # (련; 련; 련; 련; 련; ) HANGUL SYLLABLE RYEON +B829;B829;1105 1167 11AC;B829;1105 1167 11AC; # (렩; 렩; 렩; 렩; 렩; ) HANGUL SYLLABLE RYEONJ +B82A;B82A;1105 1167 11AD;B82A;1105 1167 11AD; # (렪; 렪; 렪; 렪; 렪; ) HANGUL SYLLABLE RYEONH +B82B;B82B;1105 1167 11AE;B82B;1105 1167 11AE; # (렫; 렫; 렫; 렫; 렫; ) HANGUL SYLLABLE RYEOD +B82C;B82C;1105 1167 11AF;B82C;1105 1167 11AF; # (렬; 렬; 렬; 렬; 렬; ) HANGUL SYLLABLE RYEOL +B82D;B82D;1105 1167 11B0;B82D;1105 1167 11B0; # (렭; 렭; 렭; 렭; 렭; ) HANGUL SYLLABLE RYEOLG +B82E;B82E;1105 1167 11B1;B82E;1105 1167 11B1; # (렮; 렮; 렮; 렮; 렮; ) HANGUL SYLLABLE RYEOLM +B82F;B82F;1105 1167 11B2;B82F;1105 1167 11B2; # (렯; 렯; 렯; 렯; 렯; ) HANGUL SYLLABLE RYEOLB +B830;B830;1105 1167 11B3;B830;1105 1167 11B3; # (렰; 렰; 렰; 렰; 렰; ) HANGUL SYLLABLE RYEOLS +B831;B831;1105 1167 11B4;B831;1105 1167 11B4; # (렱; 렱; 렱; 렱; 렱; ) HANGUL SYLLABLE RYEOLT +B832;B832;1105 1167 11B5;B832;1105 1167 11B5; # (렲; 렲; 렲; 렲; 렲; ) HANGUL SYLLABLE RYEOLP +B833;B833;1105 1167 11B6;B833;1105 1167 11B6; # (렳; 렳; 렳; 렳; 렳; ) HANGUL SYLLABLE RYEOLH +B834;B834;1105 1167 11B7;B834;1105 1167 11B7; # (렴; 렴; 렴; 렴; 렴; ) HANGUL SYLLABLE RYEOM +B835;B835;1105 1167 11B8;B835;1105 1167 11B8; # (렵; 렵; 렵; 렵; 렵; ) HANGUL SYLLABLE RYEOB +B836;B836;1105 1167 11B9;B836;1105 1167 11B9; # (렶; 렶; 렶; 렶; 렶; ) HANGUL SYLLABLE RYEOBS +B837;B837;1105 1167 11BA;B837;1105 1167 11BA; # (렷; 렷; 렷; 렷; 렷; ) HANGUL SYLLABLE RYEOS +B838;B838;1105 1167 11BB;B838;1105 1167 11BB; # (렸; 렸; 렸; 렸; 렸; ) HANGUL SYLLABLE RYEOSS +B839;B839;1105 1167 11BC;B839;1105 1167 11BC; # (령; 령; 령; 령; 령; ) HANGUL SYLLABLE RYEONG +B83A;B83A;1105 1167 11BD;B83A;1105 1167 11BD; # (렺; 렺; 렺; 렺; 렺; ) HANGUL SYLLABLE RYEOJ +B83B;B83B;1105 1167 11BE;B83B;1105 1167 11BE; # (렻; 렻; 렻; 렻; 렻; ) HANGUL SYLLABLE RYEOC +B83C;B83C;1105 1167 11BF;B83C;1105 1167 11BF; # (렼; 렼; 렼; 렼; 렼; ) HANGUL SYLLABLE RYEOK +B83D;B83D;1105 1167 11C0;B83D;1105 1167 11C0; # (렽; 렽; 렽; 렽; 렽; ) HANGUL SYLLABLE RYEOT +B83E;B83E;1105 1167 11C1;B83E;1105 1167 11C1; # (렾; 렾; 렾; 렾; 렾; ) HANGUL SYLLABLE RYEOP +B83F;B83F;1105 1167 11C2;B83F;1105 1167 11C2; # (렿; 렿; 렿; 렿; 렿; ) HANGUL SYLLABLE RYEOH +B840;B840;1105 1168;B840;1105 1168; # (례; 례; 례; 례; 례; ) HANGUL SYLLABLE RYE +B841;B841;1105 1168 11A8;B841;1105 1168 11A8; # (롁; 롁; 롁; 롁; 롁; ) HANGUL SYLLABLE RYEG +B842;B842;1105 1168 11A9;B842;1105 1168 11A9; # (롂; 롂; 롂; 롂; 롂; ) HANGUL SYLLABLE RYEGG +B843;B843;1105 1168 11AA;B843;1105 1168 11AA; # (롃; 롃; 롃; 롃; 롃; ) HANGUL SYLLABLE RYEGS +B844;B844;1105 1168 11AB;B844;1105 1168 11AB; # (롄; 롄; 롄; 롄; 롄; ) HANGUL SYLLABLE RYEN +B845;B845;1105 1168 11AC;B845;1105 1168 11AC; # (롅; 롅; 롅; 롅; 롅; ) HANGUL SYLLABLE RYENJ +B846;B846;1105 1168 11AD;B846;1105 1168 11AD; # (롆; 롆; 롆; 롆; 롆; ) HANGUL SYLLABLE RYENH +B847;B847;1105 1168 11AE;B847;1105 1168 11AE; # (롇; 롇; 롇; 롇; 롇; ) HANGUL SYLLABLE RYED +B848;B848;1105 1168 11AF;B848;1105 1168 11AF; # (롈; 롈; 롈; 롈; 롈; ) HANGUL SYLLABLE RYEL +B849;B849;1105 1168 11B0;B849;1105 1168 11B0; # (롉; 롉; 롉; 롉; 롉; ) HANGUL SYLLABLE RYELG +B84A;B84A;1105 1168 11B1;B84A;1105 1168 11B1; # (롊; 롊; 롊; 롊; 롊; ) HANGUL SYLLABLE RYELM +B84B;B84B;1105 1168 11B2;B84B;1105 1168 11B2; # (롋; 롋; 롋; 롋; 롋; ) HANGUL SYLLABLE RYELB +B84C;B84C;1105 1168 11B3;B84C;1105 1168 11B3; # (롌; 롌; 롌; 롌; 롌; ) HANGUL SYLLABLE RYELS +B84D;B84D;1105 1168 11B4;B84D;1105 1168 11B4; # (롍; 롍; 롍; 롍; 롍; ) HANGUL SYLLABLE RYELT +B84E;B84E;1105 1168 11B5;B84E;1105 1168 11B5; # (롎; 롎; 롎; 롎; 롎; ) HANGUL SYLLABLE RYELP +B84F;B84F;1105 1168 11B6;B84F;1105 1168 11B6; # (롏; 롏; 롏; 롏; 롏; ) HANGUL SYLLABLE RYELH +B850;B850;1105 1168 11B7;B850;1105 1168 11B7; # (롐; 롐; 롐; 롐; 롐; ) HANGUL SYLLABLE RYEM +B851;B851;1105 1168 11B8;B851;1105 1168 11B8; # (롑; 롑; 롑; 롑; 롑; ) HANGUL SYLLABLE RYEB +B852;B852;1105 1168 11B9;B852;1105 1168 11B9; # (롒; 롒; 롒; 롒; 롒; ) HANGUL SYLLABLE RYEBS +B853;B853;1105 1168 11BA;B853;1105 1168 11BA; # (롓; 롓; 롓; 롓; 롓; ) HANGUL SYLLABLE RYES +B854;B854;1105 1168 11BB;B854;1105 1168 11BB; # (롔; 롔; 롔; 롔; 롔; ) HANGUL SYLLABLE RYESS +B855;B855;1105 1168 11BC;B855;1105 1168 11BC; # (롕; 롕; 롕; 롕; 롕; ) HANGUL SYLLABLE RYENG +B856;B856;1105 1168 11BD;B856;1105 1168 11BD; # (롖; 롖; 롖; 롖; 롖; ) HANGUL SYLLABLE RYEJ +B857;B857;1105 1168 11BE;B857;1105 1168 11BE; # (롗; 롗; 롗; 롗; 롗; ) HANGUL SYLLABLE RYEC +B858;B858;1105 1168 11BF;B858;1105 1168 11BF; # (롘; 롘; 롘; 롘; 롘; ) HANGUL SYLLABLE RYEK +B859;B859;1105 1168 11C0;B859;1105 1168 11C0; # (롙; 롙; 롙; 롙; 롙; ) HANGUL SYLLABLE RYET +B85A;B85A;1105 1168 11C1;B85A;1105 1168 11C1; # (롚; 롚; 롚; 롚; 롚; ) HANGUL SYLLABLE RYEP +B85B;B85B;1105 1168 11C2;B85B;1105 1168 11C2; # (롛; 롛; 롛; 롛; 롛; ) HANGUL SYLLABLE RYEH +B85C;B85C;1105 1169;B85C;1105 1169; # (로; 로; 로; 로; 로; ) HANGUL SYLLABLE RO +B85D;B85D;1105 1169 11A8;B85D;1105 1169 11A8; # (록; 록; 록; 록; 록; ) HANGUL SYLLABLE ROG +B85E;B85E;1105 1169 11A9;B85E;1105 1169 11A9; # (롞; 롞; 롞; 롞; 롞; ) HANGUL SYLLABLE ROGG +B85F;B85F;1105 1169 11AA;B85F;1105 1169 11AA; # (롟; 롟; 롟; 롟; 롟; ) HANGUL SYLLABLE ROGS +B860;B860;1105 1169 11AB;B860;1105 1169 11AB; # (론; 론; 론; 론; 론; ) HANGUL SYLLABLE RON +B861;B861;1105 1169 11AC;B861;1105 1169 11AC; # (롡; 롡; 롡; 롡; 롡; ) HANGUL SYLLABLE RONJ +B862;B862;1105 1169 11AD;B862;1105 1169 11AD; # (롢; 롢; 롢; 롢; 롢; ) HANGUL SYLLABLE RONH +B863;B863;1105 1169 11AE;B863;1105 1169 11AE; # (롣; 롣; 롣; 롣; 롣; ) HANGUL SYLLABLE ROD +B864;B864;1105 1169 11AF;B864;1105 1169 11AF; # (롤; 롤; 롤; 롤; 롤; ) HANGUL SYLLABLE ROL +B865;B865;1105 1169 11B0;B865;1105 1169 11B0; # (롥; 롥; 롥; 롥; 롥; ) HANGUL SYLLABLE ROLG +B866;B866;1105 1169 11B1;B866;1105 1169 11B1; # (롦; 롦; 롦; 롦; 롦; ) HANGUL SYLLABLE ROLM +B867;B867;1105 1169 11B2;B867;1105 1169 11B2; # (롧; 롧; 롧; 롧; 롧; ) HANGUL SYLLABLE ROLB +B868;B868;1105 1169 11B3;B868;1105 1169 11B3; # (롨; 롨; 롨; 롨; 롨; ) HANGUL SYLLABLE ROLS +B869;B869;1105 1169 11B4;B869;1105 1169 11B4; # (롩; 롩; 롩; 롩; 롩; ) HANGUL SYLLABLE ROLT +B86A;B86A;1105 1169 11B5;B86A;1105 1169 11B5; # (롪; 롪; 롪; 롪; 롪; ) HANGUL SYLLABLE ROLP +B86B;B86B;1105 1169 11B6;B86B;1105 1169 11B6; # (롫; 롫; 롫; 롫; 롫; ) HANGUL SYLLABLE ROLH +B86C;B86C;1105 1169 11B7;B86C;1105 1169 11B7; # (롬; 롬; 롬; 롬; 롬; ) HANGUL SYLLABLE ROM +B86D;B86D;1105 1169 11B8;B86D;1105 1169 11B8; # (롭; 롭; 롭; 롭; 롭; ) HANGUL SYLLABLE ROB +B86E;B86E;1105 1169 11B9;B86E;1105 1169 11B9; # (롮; 롮; 롮; 롮; 롮; ) HANGUL SYLLABLE ROBS +B86F;B86F;1105 1169 11BA;B86F;1105 1169 11BA; # (롯; 롯; 롯; 롯; 롯; ) HANGUL SYLLABLE ROS +B870;B870;1105 1169 11BB;B870;1105 1169 11BB; # (롰; 롰; 롰; 롰; 롰; ) HANGUL SYLLABLE ROSS +B871;B871;1105 1169 11BC;B871;1105 1169 11BC; # (롱; 롱; 롱; 롱; 롱; ) HANGUL SYLLABLE RONG +B872;B872;1105 1169 11BD;B872;1105 1169 11BD; # (롲; 롲; 롲; 롲; 롲; ) HANGUL SYLLABLE ROJ +B873;B873;1105 1169 11BE;B873;1105 1169 11BE; # (롳; 롳; 롳; 롳; 롳; ) HANGUL SYLLABLE ROC +B874;B874;1105 1169 11BF;B874;1105 1169 11BF; # (롴; 롴; 롴; 롴; 롴; ) HANGUL SYLLABLE ROK +B875;B875;1105 1169 11C0;B875;1105 1169 11C0; # (롵; 롵; 롵; 롵; 롵; ) HANGUL SYLLABLE ROT +B876;B876;1105 1169 11C1;B876;1105 1169 11C1; # (롶; 롶; 롶; 롶; 롶; ) HANGUL SYLLABLE ROP +B877;B877;1105 1169 11C2;B877;1105 1169 11C2; # (롷; 롷; 롷; 롷; 롷; ) HANGUL SYLLABLE ROH +B878;B878;1105 116A;B878;1105 116A; # (롸; 롸; 롸; 롸; 롸; ) HANGUL SYLLABLE RWA +B879;B879;1105 116A 11A8;B879;1105 116A 11A8; # (롹; 롹; 롹; 롹; 롹; ) HANGUL SYLLABLE RWAG +B87A;B87A;1105 116A 11A9;B87A;1105 116A 11A9; # (롺; 롺; 롺; 롺; 롺; ) HANGUL SYLLABLE RWAGG +B87B;B87B;1105 116A 11AA;B87B;1105 116A 11AA; # (롻; 롻; 롻; 롻; 롻; ) HANGUL SYLLABLE RWAGS +B87C;B87C;1105 116A 11AB;B87C;1105 116A 11AB; # (롼; 롼; 롼; 롼; 롼; ) HANGUL SYLLABLE RWAN +B87D;B87D;1105 116A 11AC;B87D;1105 116A 11AC; # (롽; 롽; 롽; 롽; 롽; ) HANGUL SYLLABLE RWANJ +B87E;B87E;1105 116A 11AD;B87E;1105 116A 11AD; # (롾; 롾; 롾; 롾; 롾; ) HANGUL SYLLABLE RWANH +B87F;B87F;1105 116A 11AE;B87F;1105 116A 11AE; # (롿; 롿; 롿; 롿; 롿; ) HANGUL SYLLABLE RWAD +B880;B880;1105 116A 11AF;B880;1105 116A 11AF; # (뢀; 뢀; 뢀; 뢀; 뢀; ) HANGUL SYLLABLE RWAL +B881;B881;1105 116A 11B0;B881;1105 116A 11B0; # (뢁; 뢁; 뢁; 뢁; 뢁; ) HANGUL SYLLABLE RWALG +B882;B882;1105 116A 11B1;B882;1105 116A 11B1; # (뢂; 뢂; 뢂; 뢂; 뢂; ) HANGUL SYLLABLE RWALM +B883;B883;1105 116A 11B2;B883;1105 116A 11B2; # (뢃; 뢃; 뢃; 뢃; 뢃; ) HANGUL SYLLABLE RWALB +B884;B884;1105 116A 11B3;B884;1105 116A 11B3; # (뢄; 뢄; 뢄; 뢄; 뢄; ) HANGUL SYLLABLE RWALS +B885;B885;1105 116A 11B4;B885;1105 116A 11B4; # (뢅; 뢅; 뢅; 뢅; 뢅; ) HANGUL SYLLABLE RWALT +B886;B886;1105 116A 11B5;B886;1105 116A 11B5; # (뢆; 뢆; 뢆; 뢆; 뢆; ) HANGUL SYLLABLE RWALP +B887;B887;1105 116A 11B6;B887;1105 116A 11B6; # (뢇; 뢇; 뢇; 뢇; 뢇; ) HANGUL SYLLABLE RWALH +B888;B888;1105 116A 11B7;B888;1105 116A 11B7; # (뢈; 뢈; 뢈; 뢈; 뢈; ) HANGUL SYLLABLE RWAM +B889;B889;1105 116A 11B8;B889;1105 116A 11B8; # (뢉; 뢉; 뢉; 뢉; 뢉; ) HANGUL SYLLABLE RWAB +B88A;B88A;1105 116A 11B9;B88A;1105 116A 11B9; # (뢊; 뢊; 뢊; 뢊; 뢊; ) HANGUL SYLLABLE RWABS +B88B;B88B;1105 116A 11BA;B88B;1105 116A 11BA; # (뢋; 뢋; 뢋; 뢋; 뢋; ) HANGUL SYLLABLE RWAS +B88C;B88C;1105 116A 11BB;B88C;1105 116A 11BB; # (뢌; 뢌; 뢌; 뢌; 뢌; ) HANGUL SYLLABLE RWASS +B88D;B88D;1105 116A 11BC;B88D;1105 116A 11BC; # (뢍; 뢍; 뢍; 뢍; 뢍; ) HANGUL SYLLABLE RWANG +B88E;B88E;1105 116A 11BD;B88E;1105 116A 11BD; # (뢎; 뢎; 뢎; 뢎; 뢎; ) HANGUL SYLLABLE RWAJ +B88F;B88F;1105 116A 11BE;B88F;1105 116A 11BE; # (뢏; 뢏; 뢏; 뢏; 뢏; ) HANGUL SYLLABLE RWAC +B890;B890;1105 116A 11BF;B890;1105 116A 11BF; # (뢐; 뢐; 뢐; 뢐; 뢐; ) HANGUL SYLLABLE RWAK +B891;B891;1105 116A 11C0;B891;1105 116A 11C0; # (뢑; 뢑; 뢑; 뢑; 뢑; ) HANGUL SYLLABLE RWAT +B892;B892;1105 116A 11C1;B892;1105 116A 11C1; # (뢒; 뢒; 뢒; 뢒; 뢒; ) HANGUL SYLLABLE RWAP +B893;B893;1105 116A 11C2;B893;1105 116A 11C2; # (뢓; 뢓; 뢓; 뢓; 뢓; ) HANGUL SYLLABLE RWAH +B894;B894;1105 116B;B894;1105 116B; # (뢔; 뢔; 뢔; 뢔; 뢔; ) HANGUL SYLLABLE RWAE +B895;B895;1105 116B 11A8;B895;1105 116B 11A8; # (뢕; 뢕; 뢕; 뢕; 뢕; ) HANGUL SYLLABLE RWAEG +B896;B896;1105 116B 11A9;B896;1105 116B 11A9; # (뢖; 뢖; 뢖; 뢖; 뢖; ) HANGUL SYLLABLE RWAEGG +B897;B897;1105 116B 11AA;B897;1105 116B 11AA; # (뢗; 뢗; 뢗; 뢗; 뢗; ) HANGUL SYLLABLE RWAEGS +B898;B898;1105 116B 11AB;B898;1105 116B 11AB; # (뢘; 뢘; 뢘; 뢘; 뢘; ) HANGUL SYLLABLE RWAEN +B899;B899;1105 116B 11AC;B899;1105 116B 11AC; # (뢙; 뢙; 뢙; 뢙; 뢙; ) HANGUL SYLLABLE RWAENJ +B89A;B89A;1105 116B 11AD;B89A;1105 116B 11AD; # (뢚; 뢚; 뢚; 뢚; 뢚; ) HANGUL SYLLABLE RWAENH +B89B;B89B;1105 116B 11AE;B89B;1105 116B 11AE; # (뢛; 뢛; 뢛; 뢛; 뢛; ) HANGUL SYLLABLE RWAED +B89C;B89C;1105 116B 11AF;B89C;1105 116B 11AF; # (뢜; 뢜; 뢜; 뢜; 뢜; ) HANGUL SYLLABLE RWAEL +B89D;B89D;1105 116B 11B0;B89D;1105 116B 11B0; # (뢝; 뢝; 뢝; 뢝; 뢝; ) HANGUL SYLLABLE RWAELG +B89E;B89E;1105 116B 11B1;B89E;1105 116B 11B1; # (뢞; 뢞; 뢞; 뢞; 뢞; ) HANGUL SYLLABLE RWAELM +B89F;B89F;1105 116B 11B2;B89F;1105 116B 11B2; # (뢟; 뢟; 뢟; 뢟; 뢟; ) HANGUL SYLLABLE RWAELB +B8A0;B8A0;1105 116B 11B3;B8A0;1105 116B 11B3; # (뢠; 뢠; 뢠; 뢠; 뢠; ) HANGUL SYLLABLE RWAELS +B8A1;B8A1;1105 116B 11B4;B8A1;1105 116B 11B4; # (뢡; 뢡; 뢡; 뢡; 뢡; ) HANGUL SYLLABLE RWAELT +B8A2;B8A2;1105 116B 11B5;B8A2;1105 116B 11B5; # (뢢; 뢢; 뢢; 뢢; 뢢; ) HANGUL SYLLABLE RWAELP +B8A3;B8A3;1105 116B 11B6;B8A3;1105 116B 11B6; # (뢣; 뢣; 뢣; 뢣; 뢣; ) HANGUL SYLLABLE RWAELH +B8A4;B8A4;1105 116B 11B7;B8A4;1105 116B 11B7; # (뢤; 뢤; 뢤; 뢤; 뢤; ) HANGUL SYLLABLE RWAEM +B8A5;B8A5;1105 116B 11B8;B8A5;1105 116B 11B8; # (뢥; 뢥; 뢥; 뢥; 뢥; ) HANGUL SYLLABLE RWAEB +B8A6;B8A6;1105 116B 11B9;B8A6;1105 116B 11B9; # (뢦; 뢦; 뢦; 뢦; 뢦; ) HANGUL SYLLABLE RWAEBS +B8A7;B8A7;1105 116B 11BA;B8A7;1105 116B 11BA; # (뢧; 뢧; 뢧; 뢧; 뢧; ) HANGUL SYLLABLE RWAES +B8A8;B8A8;1105 116B 11BB;B8A8;1105 116B 11BB; # (뢨; 뢨; 뢨; 뢨; 뢨; ) HANGUL SYLLABLE RWAESS +B8A9;B8A9;1105 116B 11BC;B8A9;1105 116B 11BC; # (뢩; 뢩; 뢩; 뢩; 뢩; ) HANGUL SYLLABLE RWAENG +B8AA;B8AA;1105 116B 11BD;B8AA;1105 116B 11BD; # (뢪; 뢪; 뢪; 뢪; 뢪; ) HANGUL SYLLABLE RWAEJ +B8AB;B8AB;1105 116B 11BE;B8AB;1105 116B 11BE; # (뢫; 뢫; 뢫; 뢫; 뢫; ) HANGUL SYLLABLE RWAEC +B8AC;B8AC;1105 116B 11BF;B8AC;1105 116B 11BF; # (뢬; 뢬; 뢬; 뢬; 뢬; ) HANGUL SYLLABLE RWAEK +B8AD;B8AD;1105 116B 11C0;B8AD;1105 116B 11C0; # (뢭; 뢭; 뢭; 뢭; 뢭; ) HANGUL SYLLABLE RWAET +B8AE;B8AE;1105 116B 11C1;B8AE;1105 116B 11C1; # (뢮; 뢮; 뢮; 뢮; 뢮; ) HANGUL SYLLABLE RWAEP +B8AF;B8AF;1105 116B 11C2;B8AF;1105 116B 11C2; # (뢯; 뢯; 뢯; 뢯; 뢯; ) HANGUL SYLLABLE RWAEH +B8B0;B8B0;1105 116C;B8B0;1105 116C; # (뢰; 뢰; 뢰; 뢰; 뢰; ) HANGUL SYLLABLE ROE +B8B1;B8B1;1105 116C 11A8;B8B1;1105 116C 11A8; # (뢱; 뢱; 뢱; 뢱; 뢱; ) HANGUL SYLLABLE ROEG +B8B2;B8B2;1105 116C 11A9;B8B2;1105 116C 11A9; # (뢲; 뢲; 뢲; 뢲; 뢲; ) HANGUL SYLLABLE ROEGG +B8B3;B8B3;1105 116C 11AA;B8B3;1105 116C 11AA; # (뢳; 뢳; 뢳; 뢳; 뢳; ) HANGUL SYLLABLE ROEGS +B8B4;B8B4;1105 116C 11AB;B8B4;1105 116C 11AB; # (뢴; 뢴; 뢴; 뢴; 뢴; ) HANGUL SYLLABLE ROEN +B8B5;B8B5;1105 116C 11AC;B8B5;1105 116C 11AC; # (뢵; 뢵; 뢵; 뢵; 뢵; ) HANGUL SYLLABLE ROENJ +B8B6;B8B6;1105 116C 11AD;B8B6;1105 116C 11AD; # (뢶; 뢶; 뢶; 뢶; 뢶; ) HANGUL SYLLABLE ROENH +B8B7;B8B7;1105 116C 11AE;B8B7;1105 116C 11AE; # (뢷; 뢷; 뢷; 뢷; 뢷; ) HANGUL SYLLABLE ROED +B8B8;B8B8;1105 116C 11AF;B8B8;1105 116C 11AF; # (뢸; 뢸; 뢸; 뢸; 뢸; ) HANGUL SYLLABLE ROEL +B8B9;B8B9;1105 116C 11B0;B8B9;1105 116C 11B0; # (뢹; 뢹; 뢹; 뢹; 뢹; ) HANGUL SYLLABLE ROELG +B8BA;B8BA;1105 116C 11B1;B8BA;1105 116C 11B1; # (뢺; 뢺; 뢺; 뢺; 뢺; ) HANGUL SYLLABLE ROELM +B8BB;B8BB;1105 116C 11B2;B8BB;1105 116C 11B2; # (뢻; 뢻; 뢻; 뢻; 뢻; ) HANGUL SYLLABLE ROELB +B8BC;B8BC;1105 116C 11B3;B8BC;1105 116C 11B3; # (뢼; 뢼; 뢼; 뢼; 뢼; ) HANGUL SYLLABLE ROELS +B8BD;B8BD;1105 116C 11B4;B8BD;1105 116C 11B4; # (뢽; 뢽; 뢽; 뢽; 뢽; ) HANGUL SYLLABLE ROELT +B8BE;B8BE;1105 116C 11B5;B8BE;1105 116C 11B5; # (뢾; 뢾; 뢾; 뢾; 뢾; ) HANGUL SYLLABLE ROELP +B8BF;B8BF;1105 116C 11B6;B8BF;1105 116C 11B6; # (뢿; 뢿; 뢿; 뢿; 뢿; ) HANGUL SYLLABLE ROELH +B8C0;B8C0;1105 116C 11B7;B8C0;1105 116C 11B7; # (룀; 룀; 룀; 룀; 룀; ) HANGUL SYLLABLE ROEM +B8C1;B8C1;1105 116C 11B8;B8C1;1105 116C 11B8; # (룁; 룁; 룁; 룁; 룁; ) HANGUL SYLLABLE ROEB +B8C2;B8C2;1105 116C 11B9;B8C2;1105 116C 11B9; # (룂; 룂; 룂; 룂; 룂; ) HANGUL SYLLABLE ROEBS +B8C3;B8C3;1105 116C 11BA;B8C3;1105 116C 11BA; # (룃; 룃; 룃; 룃; 룃; ) HANGUL SYLLABLE ROES +B8C4;B8C4;1105 116C 11BB;B8C4;1105 116C 11BB; # (룄; 룄; 룄; 룄; 룄; ) HANGUL SYLLABLE ROESS +B8C5;B8C5;1105 116C 11BC;B8C5;1105 116C 11BC; # (룅; 룅; 룅; 룅; 룅; ) HANGUL SYLLABLE ROENG +B8C6;B8C6;1105 116C 11BD;B8C6;1105 116C 11BD; # (룆; 룆; 룆; 룆; 룆; ) HANGUL SYLLABLE ROEJ +B8C7;B8C7;1105 116C 11BE;B8C7;1105 116C 11BE; # (룇; 룇; 룇; 룇; 룇; ) HANGUL SYLLABLE ROEC +B8C8;B8C8;1105 116C 11BF;B8C8;1105 116C 11BF; # (룈; 룈; 룈; 룈; 룈; ) HANGUL SYLLABLE ROEK +B8C9;B8C9;1105 116C 11C0;B8C9;1105 116C 11C0; # (룉; 룉; 룉; 룉; 룉; ) HANGUL SYLLABLE ROET +B8CA;B8CA;1105 116C 11C1;B8CA;1105 116C 11C1; # (룊; 룊; 룊; 룊; 룊; ) HANGUL SYLLABLE ROEP +B8CB;B8CB;1105 116C 11C2;B8CB;1105 116C 11C2; # (룋; 룋; 룋; 룋; 룋; ) HANGUL SYLLABLE ROEH +B8CC;B8CC;1105 116D;B8CC;1105 116D; # (료; 료; 료; 료; 료; ) HANGUL SYLLABLE RYO +B8CD;B8CD;1105 116D 11A8;B8CD;1105 116D 11A8; # (룍; 룍; 룍; 룍; 룍; ) HANGUL SYLLABLE RYOG +B8CE;B8CE;1105 116D 11A9;B8CE;1105 116D 11A9; # (룎; 룎; 룎; 룎; 룎; ) HANGUL SYLLABLE RYOGG +B8CF;B8CF;1105 116D 11AA;B8CF;1105 116D 11AA; # (룏; 룏; 룏; 룏; 룏; ) HANGUL SYLLABLE RYOGS +B8D0;B8D0;1105 116D 11AB;B8D0;1105 116D 11AB; # (룐; 룐; 룐; 룐; 룐; ) HANGUL SYLLABLE RYON +B8D1;B8D1;1105 116D 11AC;B8D1;1105 116D 11AC; # (룑; 룑; 룑; 룑; 룑; ) HANGUL SYLLABLE RYONJ +B8D2;B8D2;1105 116D 11AD;B8D2;1105 116D 11AD; # (룒; 룒; 룒; 룒; 룒; ) HANGUL SYLLABLE RYONH +B8D3;B8D3;1105 116D 11AE;B8D3;1105 116D 11AE; # (룓; 룓; 룓; 룓; 룓; ) HANGUL SYLLABLE RYOD +B8D4;B8D4;1105 116D 11AF;B8D4;1105 116D 11AF; # (룔; 룔; 룔; 룔; 룔; ) HANGUL SYLLABLE RYOL +B8D5;B8D5;1105 116D 11B0;B8D5;1105 116D 11B0; # (룕; 룕; 룕; 룕; 룕; ) HANGUL SYLLABLE RYOLG +B8D6;B8D6;1105 116D 11B1;B8D6;1105 116D 11B1; # (룖; 룖; 룖; 룖; 룖; ) HANGUL SYLLABLE RYOLM +B8D7;B8D7;1105 116D 11B2;B8D7;1105 116D 11B2; # (룗; 룗; 룗; 룗; 룗; ) HANGUL SYLLABLE RYOLB +B8D8;B8D8;1105 116D 11B3;B8D8;1105 116D 11B3; # (룘; 룘; 룘; 룘; 룘; ) HANGUL SYLLABLE RYOLS +B8D9;B8D9;1105 116D 11B4;B8D9;1105 116D 11B4; # (룙; 룙; 룙; 룙; 룙; ) HANGUL SYLLABLE RYOLT +B8DA;B8DA;1105 116D 11B5;B8DA;1105 116D 11B5; # (룚; 룚; 룚; 룚; 룚; ) HANGUL SYLLABLE RYOLP +B8DB;B8DB;1105 116D 11B6;B8DB;1105 116D 11B6; # (룛; 룛; 룛; 룛; 룛; ) HANGUL SYLLABLE RYOLH +B8DC;B8DC;1105 116D 11B7;B8DC;1105 116D 11B7; # (룜; 룜; 룜; 룜; 룜; ) HANGUL SYLLABLE RYOM +B8DD;B8DD;1105 116D 11B8;B8DD;1105 116D 11B8; # (룝; 룝; 룝; 룝; 룝; ) HANGUL SYLLABLE RYOB +B8DE;B8DE;1105 116D 11B9;B8DE;1105 116D 11B9; # (룞; 룞; 룞; 룞; 룞; ) HANGUL SYLLABLE RYOBS +B8DF;B8DF;1105 116D 11BA;B8DF;1105 116D 11BA; # (룟; 룟; 룟; 룟; 룟; ) HANGUL SYLLABLE RYOS +B8E0;B8E0;1105 116D 11BB;B8E0;1105 116D 11BB; # (룠; 룠; 룠; 룠; 룠; ) HANGUL SYLLABLE RYOSS +B8E1;B8E1;1105 116D 11BC;B8E1;1105 116D 11BC; # (룡; 룡; 룡; 룡; 룡; ) HANGUL SYLLABLE RYONG +B8E2;B8E2;1105 116D 11BD;B8E2;1105 116D 11BD; # (룢; 룢; 룢; 룢; 룢; ) HANGUL SYLLABLE RYOJ +B8E3;B8E3;1105 116D 11BE;B8E3;1105 116D 11BE; # (룣; 룣; 룣; 룣; 룣; ) HANGUL SYLLABLE RYOC +B8E4;B8E4;1105 116D 11BF;B8E4;1105 116D 11BF; # (룤; 룤; 룤; 룤; 룤; ) HANGUL SYLLABLE RYOK +B8E5;B8E5;1105 116D 11C0;B8E5;1105 116D 11C0; # (룥; 룥; 룥; 룥; 룥; ) HANGUL SYLLABLE RYOT +B8E6;B8E6;1105 116D 11C1;B8E6;1105 116D 11C1; # (룦; 룦; 룦; 룦; 룦; ) HANGUL SYLLABLE RYOP +B8E7;B8E7;1105 116D 11C2;B8E7;1105 116D 11C2; # (룧; 룧; 룧; 룧; 룧; ) HANGUL SYLLABLE RYOH +B8E8;B8E8;1105 116E;B8E8;1105 116E; # (루; 루; 루; 루; 루; ) HANGUL SYLLABLE RU +B8E9;B8E9;1105 116E 11A8;B8E9;1105 116E 11A8; # (룩; 룩; 룩; 룩; 룩; ) HANGUL SYLLABLE RUG +B8EA;B8EA;1105 116E 11A9;B8EA;1105 116E 11A9; # (룪; 룪; 룪; 룪; 룪; ) HANGUL SYLLABLE RUGG +B8EB;B8EB;1105 116E 11AA;B8EB;1105 116E 11AA; # (룫; 룫; 룫; 룫; 룫; ) HANGUL SYLLABLE RUGS +B8EC;B8EC;1105 116E 11AB;B8EC;1105 116E 11AB; # (룬; 룬; 룬; 룬; 룬; ) HANGUL SYLLABLE RUN +B8ED;B8ED;1105 116E 11AC;B8ED;1105 116E 11AC; # (룭; 룭; 룭; 룭; 룭; ) HANGUL SYLLABLE RUNJ +B8EE;B8EE;1105 116E 11AD;B8EE;1105 116E 11AD; # (룮; 룮; 룮; 룮; 룮; ) HANGUL SYLLABLE RUNH +B8EF;B8EF;1105 116E 11AE;B8EF;1105 116E 11AE; # (룯; 룯; 룯; 룯; 룯; ) HANGUL SYLLABLE RUD +B8F0;B8F0;1105 116E 11AF;B8F0;1105 116E 11AF; # (룰; 룰; 룰; 룰; 룰; ) HANGUL SYLLABLE RUL +B8F1;B8F1;1105 116E 11B0;B8F1;1105 116E 11B0; # (룱; 룱; 룱; 룱; 룱; ) HANGUL SYLLABLE RULG +B8F2;B8F2;1105 116E 11B1;B8F2;1105 116E 11B1; # (룲; 룲; 룲; 룲; 룲; ) HANGUL SYLLABLE RULM +B8F3;B8F3;1105 116E 11B2;B8F3;1105 116E 11B2; # (룳; 룳; 룳; 룳; 룳; ) HANGUL SYLLABLE RULB +B8F4;B8F4;1105 116E 11B3;B8F4;1105 116E 11B3; # (룴; 룴; 룴; 룴; 룴; ) HANGUL SYLLABLE RULS +B8F5;B8F5;1105 116E 11B4;B8F5;1105 116E 11B4; # (룵; 룵; 룵; 룵; 룵; ) HANGUL SYLLABLE RULT +B8F6;B8F6;1105 116E 11B5;B8F6;1105 116E 11B5; # (룶; 룶; 룶; 룶; 룶; ) HANGUL SYLLABLE RULP +B8F7;B8F7;1105 116E 11B6;B8F7;1105 116E 11B6; # (룷; 룷; 룷; 룷; 룷; ) HANGUL SYLLABLE RULH +B8F8;B8F8;1105 116E 11B7;B8F8;1105 116E 11B7; # (룸; 룸; 룸; 룸; 룸; ) HANGUL SYLLABLE RUM +B8F9;B8F9;1105 116E 11B8;B8F9;1105 116E 11B8; # (룹; 룹; 룹; 룹; 룹; ) HANGUL SYLLABLE RUB +B8FA;B8FA;1105 116E 11B9;B8FA;1105 116E 11B9; # (룺; 룺; 룺; 룺; 룺; ) HANGUL SYLLABLE RUBS +B8FB;B8FB;1105 116E 11BA;B8FB;1105 116E 11BA; # (룻; 룻; 룻; 룻; 룻; ) HANGUL SYLLABLE RUS +B8FC;B8FC;1105 116E 11BB;B8FC;1105 116E 11BB; # (룼; 룼; 룼; 룼; 룼; ) HANGUL SYLLABLE RUSS +B8FD;B8FD;1105 116E 11BC;B8FD;1105 116E 11BC; # (룽; 룽; 룽; 룽; 룽; ) HANGUL SYLLABLE RUNG +B8FE;B8FE;1105 116E 11BD;B8FE;1105 116E 11BD; # (룾; 룾; 룾; 룾; 룾; ) HANGUL SYLLABLE RUJ +B8FF;B8FF;1105 116E 11BE;B8FF;1105 116E 11BE; # (룿; 룿; 룿; 룿; 룿; ) HANGUL SYLLABLE RUC +B900;B900;1105 116E 11BF;B900;1105 116E 11BF; # (뤀; 뤀; 뤀; 뤀; 뤀; ) HANGUL SYLLABLE RUK +B901;B901;1105 116E 11C0;B901;1105 116E 11C0; # (뤁; 뤁; 뤁; 뤁; 뤁; ) HANGUL SYLLABLE RUT +B902;B902;1105 116E 11C1;B902;1105 116E 11C1; # (뤂; 뤂; 뤂; 뤂; 뤂; ) HANGUL SYLLABLE RUP +B903;B903;1105 116E 11C2;B903;1105 116E 11C2; # (뤃; 뤃; 뤃; 뤃; 뤃; ) HANGUL SYLLABLE RUH +B904;B904;1105 116F;B904;1105 116F; # (뤄; 뤄; 뤄; 뤄; 뤄; ) HANGUL SYLLABLE RWEO +B905;B905;1105 116F 11A8;B905;1105 116F 11A8; # (뤅; 뤅; 뤅; 뤅; 뤅; ) HANGUL SYLLABLE RWEOG +B906;B906;1105 116F 11A9;B906;1105 116F 11A9; # (뤆; 뤆; 뤆; 뤆; 뤆; ) HANGUL SYLLABLE RWEOGG +B907;B907;1105 116F 11AA;B907;1105 116F 11AA; # (뤇; 뤇; 뤇; 뤇; 뤇; ) HANGUL SYLLABLE RWEOGS +B908;B908;1105 116F 11AB;B908;1105 116F 11AB; # (뤈; 뤈; 뤈; 뤈; 뤈; ) HANGUL SYLLABLE RWEON +B909;B909;1105 116F 11AC;B909;1105 116F 11AC; # (뤉; 뤉; 뤉; 뤉; 뤉; ) HANGUL SYLLABLE RWEONJ +B90A;B90A;1105 116F 11AD;B90A;1105 116F 11AD; # (뤊; 뤊; 뤊; 뤊; 뤊; ) HANGUL SYLLABLE RWEONH +B90B;B90B;1105 116F 11AE;B90B;1105 116F 11AE; # (뤋; 뤋; 뤋; 뤋; 뤋; ) HANGUL SYLLABLE RWEOD +B90C;B90C;1105 116F 11AF;B90C;1105 116F 11AF; # (뤌; 뤌; 뤌; 뤌; 뤌; ) HANGUL SYLLABLE RWEOL +B90D;B90D;1105 116F 11B0;B90D;1105 116F 11B0; # (뤍; 뤍; 뤍; 뤍; 뤍; ) HANGUL SYLLABLE RWEOLG +B90E;B90E;1105 116F 11B1;B90E;1105 116F 11B1; # (뤎; 뤎; 뤎; 뤎; 뤎; ) HANGUL SYLLABLE RWEOLM +B90F;B90F;1105 116F 11B2;B90F;1105 116F 11B2; # (뤏; 뤏; 뤏; 뤏; 뤏; ) HANGUL SYLLABLE RWEOLB +B910;B910;1105 116F 11B3;B910;1105 116F 11B3; # (뤐; 뤐; 뤐; 뤐; 뤐; ) HANGUL SYLLABLE RWEOLS +B911;B911;1105 116F 11B4;B911;1105 116F 11B4; # (뤑; 뤑; 뤑; 뤑; 뤑; ) HANGUL SYLLABLE RWEOLT +B912;B912;1105 116F 11B5;B912;1105 116F 11B5; # (뤒; 뤒; 뤒; 뤒; 뤒; ) HANGUL SYLLABLE RWEOLP +B913;B913;1105 116F 11B6;B913;1105 116F 11B6; # (뤓; 뤓; 뤓; 뤓; 뤓; ) HANGUL SYLLABLE RWEOLH +B914;B914;1105 116F 11B7;B914;1105 116F 11B7; # (뤔; 뤔; 뤔; 뤔; 뤔; ) HANGUL SYLLABLE RWEOM +B915;B915;1105 116F 11B8;B915;1105 116F 11B8; # (뤕; 뤕; 뤕; 뤕; 뤕; ) HANGUL SYLLABLE RWEOB +B916;B916;1105 116F 11B9;B916;1105 116F 11B9; # (뤖; 뤖; 뤖; 뤖; 뤖; ) HANGUL SYLLABLE RWEOBS +B917;B917;1105 116F 11BA;B917;1105 116F 11BA; # (뤗; 뤗; 뤗; 뤗; 뤗; ) HANGUL SYLLABLE RWEOS +B918;B918;1105 116F 11BB;B918;1105 116F 11BB; # (뤘; 뤘; 뤘; 뤘; 뤘; ) HANGUL SYLLABLE RWEOSS +B919;B919;1105 116F 11BC;B919;1105 116F 11BC; # (뤙; 뤙; 뤙; 뤙; 뤙; ) HANGUL SYLLABLE RWEONG +B91A;B91A;1105 116F 11BD;B91A;1105 116F 11BD; # (뤚; 뤚; 뤚; 뤚; 뤚; ) HANGUL SYLLABLE RWEOJ +B91B;B91B;1105 116F 11BE;B91B;1105 116F 11BE; # (뤛; 뤛; 뤛; 뤛; 뤛; ) HANGUL SYLLABLE RWEOC +B91C;B91C;1105 116F 11BF;B91C;1105 116F 11BF; # (뤜; 뤜; 뤜; 뤜; 뤜; ) HANGUL SYLLABLE RWEOK +B91D;B91D;1105 116F 11C0;B91D;1105 116F 11C0; # (뤝; 뤝; 뤝; 뤝; 뤝; ) HANGUL SYLLABLE RWEOT +B91E;B91E;1105 116F 11C1;B91E;1105 116F 11C1; # (뤞; 뤞; 뤞; 뤞; 뤞; ) HANGUL SYLLABLE RWEOP +B91F;B91F;1105 116F 11C2;B91F;1105 116F 11C2; # (뤟; 뤟; 뤟; 뤟; 뤟; ) HANGUL SYLLABLE RWEOH +B920;B920;1105 1170;B920;1105 1170; # (뤠; 뤠; 뤠; 뤠; 뤠; ) HANGUL SYLLABLE RWE +B921;B921;1105 1170 11A8;B921;1105 1170 11A8; # (뤡; 뤡; 뤡; 뤡; 뤡; ) HANGUL SYLLABLE RWEG +B922;B922;1105 1170 11A9;B922;1105 1170 11A9; # (뤢; 뤢; 뤢; 뤢; 뤢; ) HANGUL SYLLABLE RWEGG +B923;B923;1105 1170 11AA;B923;1105 1170 11AA; # (뤣; 뤣; 뤣; 뤣; 뤣; ) HANGUL SYLLABLE RWEGS +B924;B924;1105 1170 11AB;B924;1105 1170 11AB; # (뤤; 뤤; 뤤; 뤤; 뤤; ) HANGUL SYLLABLE RWEN +B925;B925;1105 1170 11AC;B925;1105 1170 11AC; # (뤥; 뤥; 뤥; 뤥; 뤥; ) HANGUL SYLLABLE RWENJ +B926;B926;1105 1170 11AD;B926;1105 1170 11AD; # (뤦; 뤦; 뤦; 뤦; 뤦; ) HANGUL SYLLABLE RWENH +B927;B927;1105 1170 11AE;B927;1105 1170 11AE; # (뤧; 뤧; 뤧; 뤧; 뤧; ) HANGUL SYLLABLE RWED +B928;B928;1105 1170 11AF;B928;1105 1170 11AF; # (뤨; 뤨; 뤨; 뤨; 뤨; ) HANGUL SYLLABLE RWEL +B929;B929;1105 1170 11B0;B929;1105 1170 11B0; # (뤩; 뤩; 뤩; 뤩; 뤩; ) HANGUL SYLLABLE RWELG +B92A;B92A;1105 1170 11B1;B92A;1105 1170 11B1; # (뤪; 뤪; 뤪; 뤪; 뤪; ) HANGUL SYLLABLE RWELM +B92B;B92B;1105 1170 11B2;B92B;1105 1170 11B2; # (뤫; 뤫; 뤫; 뤫; 뤫; ) HANGUL SYLLABLE RWELB +B92C;B92C;1105 1170 11B3;B92C;1105 1170 11B3; # (뤬; 뤬; 뤬; 뤬; 뤬; ) HANGUL SYLLABLE RWELS +B92D;B92D;1105 1170 11B4;B92D;1105 1170 11B4; # (뤭; 뤭; 뤭; 뤭; 뤭; ) HANGUL SYLLABLE RWELT +B92E;B92E;1105 1170 11B5;B92E;1105 1170 11B5; # (뤮; 뤮; 뤮; 뤮; 뤮; ) HANGUL SYLLABLE RWELP +B92F;B92F;1105 1170 11B6;B92F;1105 1170 11B6; # (뤯; 뤯; 뤯; 뤯; 뤯; ) HANGUL SYLLABLE RWELH +B930;B930;1105 1170 11B7;B930;1105 1170 11B7; # (뤰; 뤰; 뤰; 뤰; 뤰; ) HANGUL SYLLABLE RWEM +B931;B931;1105 1170 11B8;B931;1105 1170 11B8; # (뤱; 뤱; 뤱; 뤱; 뤱; ) HANGUL SYLLABLE RWEB +B932;B932;1105 1170 11B9;B932;1105 1170 11B9; # (뤲; 뤲; 뤲; 뤲; 뤲; ) HANGUL SYLLABLE RWEBS +B933;B933;1105 1170 11BA;B933;1105 1170 11BA; # (뤳; 뤳; 뤳; 뤳; 뤳; ) HANGUL SYLLABLE RWES +B934;B934;1105 1170 11BB;B934;1105 1170 11BB; # (뤴; 뤴; 뤴; 뤴; 뤴; ) HANGUL SYLLABLE RWESS +B935;B935;1105 1170 11BC;B935;1105 1170 11BC; # (뤵; 뤵; 뤵; 뤵; 뤵; ) HANGUL SYLLABLE RWENG +B936;B936;1105 1170 11BD;B936;1105 1170 11BD; # (뤶; 뤶; 뤶; 뤶; 뤶; ) HANGUL SYLLABLE RWEJ +B937;B937;1105 1170 11BE;B937;1105 1170 11BE; # (뤷; 뤷; 뤷; 뤷; 뤷; ) HANGUL SYLLABLE RWEC +B938;B938;1105 1170 11BF;B938;1105 1170 11BF; # (뤸; 뤸; 뤸; 뤸; 뤸; ) HANGUL SYLLABLE RWEK +B939;B939;1105 1170 11C0;B939;1105 1170 11C0; # (뤹; 뤹; 뤹; 뤹; 뤹; ) HANGUL SYLLABLE RWET +B93A;B93A;1105 1170 11C1;B93A;1105 1170 11C1; # (뤺; 뤺; 뤺; 뤺; 뤺; ) HANGUL SYLLABLE RWEP +B93B;B93B;1105 1170 11C2;B93B;1105 1170 11C2; # (뤻; 뤻; 뤻; 뤻; 뤻; ) HANGUL SYLLABLE RWEH +B93C;B93C;1105 1171;B93C;1105 1171; # (뤼; 뤼; 뤼; 뤼; 뤼; ) HANGUL SYLLABLE RWI +B93D;B93D;1105 1171 11A8;B93D;1105 1171 11A8; # (뤽; 뤽; 뤽; 뤽; 뤽; ) HANGUL SYLLABLE RWIG +B93E;B93E;1105 1171 11A9;B93E;1105 1171 11A9; # (뤾; 뤾; 뤾; 뤾; 뤾; ) HANGUL SYLLABLE RWIGG +B93F;B93F;1105 1171 11AA;B93F;1105 1171 11AA; # (뤿; 뤿; 뤿; 뤿; 뤿; ) HANGUL SYLLABLE RWIGS +B940;B940;1105 1171 11AB;B940;1105 1171 11AB; # (륀; 륀; 륀; 륀; 륀; ) HANGUL SYLLABLE RWIN +B941;B941;1105 1171 11AC;B941;1105 1171 11AC; # (륁; 륁; 륁; 륁; 륁; ) HANGUL SYLLABLE RWINJ +B942;B942;1105 1171 11AD;B942;1105 1171 11AD; # (륂; 륂; 륂; 륂; 륂; ) HANGUL SYLLABLE RWINH +B943;B943;1105 1171 11AE;B943;1105 1171 11AE; # (륃; 륃; 륃; 륃; 륃; ) HANGUL SYLLABLE RWID +B944;B944;1105 1171 11AF;B944;1105 1171 11AF; # (륄; 륄; 륄; 륄; 륄; ) HANGUL SYLLABLE RWIL +B945;B945;1105 1171 11B0;B945;1105 1171 11B0; # (륅; 륅; 륅; 륅; 륅; ) HANGUL SYLLABLE RWILG +B946;B946;1105 1171 11B1;B946;1105 1171 11B1; # (륆; 륆; 륆; 륆; 륆; ) HANGUL SYLLABLE RWILM +B947;B947;1105 1171 11B2;B947;1105 1171 11B2; # (륇; 륇; 륇; 륇; 륇; ) HANGUL SYLLABLE RWILB +B948;B948;1105 1171 11B3;B948;1105 1171 11B3; # (륈; 륈; 륈; 륈; 륈; ) HANGUL SYLLABLE RWILS +B949;B949;1105 1171 11B4;B949;1105 1171 11B4; # (륉; 륉; 륉; 륉; 륉; ) HANGUL SYLLABLE RWILT +B94A;B94A;1105 1171 11B5;B94A;1105 1171 11B5; # (륊; 륊; 륊; 륊; 륊; ) HANGUL SYLLABLE RWILP +B94B;B94B;1105 1171 11B6;B94B;1105 1171 11B6; # (륋; 륋; 륋; 륋; 륋; ) HANGUL SYLLABLE RWILH +B94C;B94C;1105 1171 11B7;B94C;1105 1171 11B7; # (륌; 륌; 륌; 륌; 륌; ) HANGUL SYLLABLE RWIM +B94D;B94D;1105 1171 11B8;B94D;1105 1171 11B8; # (륍; 륍; 륍; 륍; 륍; ) HANGUL SYLLABLE RWIB +B94E;B94E;1105 1171 11B9;B94E;1105 1171 11B9; # (륎; 륎; 륎; 륎; 륎; ) HANGUL SYLLABLE RWIBS +B94F;B94F;1105 1171 11BA;B94F;1105 1171 11BA; # (륏; 륏; 륏; 륏; 륏; ) HANGUL SYLLABLE RWIS +B950;B950;1105 1171 11BB;B950;1105 1171 11BB; # (륐; 륐; 륐; 륐; 륐; ) HANGUL SYLLABLE RWISS +B951;B951;1105 1171 11BC;B951;1105 1171 11BC; # (륑; 륑; 륑; 륑; 륑; ) HANGUL SYLLABLE RWING +B952;B952;1105 1171 11BD;B952;1105 1171 11BD; # (륒; 륒; 륒; 륒; 륒; ) HANGUL SYLLABLE RWIJ +B953;B953;1105 1171 11BE;B953;1105 1171 11BE; # (륓; 륓; 륓; 륓; 륓; ) HANGUL SYLLABLE RWIC +B954;B954;1105 1171 11BF;B954;1105 1171 11BF; # (륔; 륔; 륔; 륔; 륔; ) HANGUL SYLLABLE RWIK +B955;B955;1105 1171 11C0;B955;1105 1171 11C0; # (륕; 륕; 륕; 륕; 륕; ) HANGUL SYLLABLE RWIT +B956;B956;1105 1171 11C1;B956;1105 1171 11C1; # (륖; 륖; 륖; 륖; 륖; ) HANGUL SYLLABLE RWIP +B957;B957;1105 1171 11C2;B957;1105 1171 11C2; # (륗; 륗; 륗; 륗; 륗; ) HANGUL SYLLABLE RWIH +B958;B958;1105 1172;B958;1105 1172; # (류; 류; 류; 류; 류; ) HANGUL SYLLABLE RYU +B959;B959;1105 1172 11A8;B959;1105 1172 11A8; # (륙; 륙; 륙; 륙; 륙; ) HANGUL SYLLABLE RYUG +B95A;B95A;1105 1172 11A9;B95A;1105 1172 11A9; # (륚; 륚; 륚; 륚; 륚; ) HANGUL SYLLABLE RYUGG +B95B;B95B;1105 1172 11AA;B95B;1105 1172 11AA; # (륛; 륛; 륛; 륛; 륛; ) HANGUL SYLLABLE RYUGS +B95C;B95C;1105 1172 11AB;B95C;1105 1172 11AB; # (륜; 륜; 륜; 륜; 륜; ) HANGUL SYLLABLE RYUN +B95D;B95D;1105 1172 11AC;B95D;1105 1172 11AC; # (륝; 륝; 륝; 륝; 륝; ) HANGUL SYLLABLE RYUNJ +B95E;B95E;1105 1172 11AD;B95E;1105 1172 11AD; # (륞; 륞; 륞; 륞; 륞; ) HANGUL SYLLABLE RYUNH +B95F;B95F;1105 1172 11AE;B95F;1105 1172 11AE; # (륟; 륟; 륟; 륟; 륟; ) HANGUL SYLLABLE RYUD +B960;B960;1105 1172 11AF;B960;1105 1172 11AF; # (률; 률; 률; 률; 률; ) HANGUL SYLLABLE RYUL +B961;B961;1105 1172 11B0;B961;1105 1172 11B0; # (륡; 륡; 륡; 륡; 륡; ) HANGUL SYLLABLE RYULG +B962;B962;1105 1172 11B1;B962;1105 1172 11B1; # (륢; 륢; 륢; 륢; 륢; ) HANGUL SYLLABLE RYULM +B963;B963;1105 1172 11B2;B963;1105 1172 11B2; # (륣; 륣; 륣; 륣; 륣; ) HANGUL SYLLABLE RYULB +B964;B964;1105 1172 11B3;B964;1105 1172 11B3; # (륤; 륤; 륤; 륤; 륤; ) HANGUL SYLLABLE RYULS +B965;B965;1105 1172 11B4;B965;1105 1172 11B4; # (륥; 륥; 륥; 륥; 륥; ) HANGUL SYLLABLE RYULT +B966;B966;1105 1172 11B5;B966;1105 1172 11B5; # (륦; 륦; 륦; 륦; 륦; ) HANGUL SYLLABLE RYULP +B967;B967;1105 1172 11B6;B967;1105 1172 11B6; # (륧; 륧; 륧; 륧; 륧; ) HANGUL SYLLABLE RYULH +B968;B968;1105 1172 11B7;B968;1105 1172 11B7; # (륨; 륨; 륨; 륨; 륨; ) HANGUL SYLLABLE RYUM +B969;B969;1105 1172 11B8;B969;1105 1172 11B8; # (륩; 륩; 륩; 륩; 륩; ) HANGUL SYLLABLE RYUB +B96A;B96A;1105 1172 11B9;B96A;1105 1172 11B9; # (륪; 륪; 륪; 륪; 륪; ) HANGUL SYLLABLE RYUBS +B96B;B96B;1105 1172 11BA;B96B;1105 1172 11BA; # (륫; 륫; 륫; 륫; 륫; ) HANGUL SYLLABLE RYUS +B96C;B96C;1105 1172 11BB;B96C;1105 1172 11BB; # (륬; 륬; 륬; 륬; 륬; ) HANGUL SYLLABLE RYUSS +B96D;B96D;1105 1172 11BC;B96D;1105 1172 11BC; # (륭; 륭; 륭; 륭; 륭; ) HANGUL SYLLABLE RYUNG +B96E;B96E;1105 1172 11BD;B96E;1105 1172 11BD; # (륮; 륮; 륮; 륮; 륮; ) HANGUL SYLLABLE RYUJ +B96F;B96F;1105 1172 11BE;B96F;1105 1172 11BE; # (륯; 륯; 륯; 륯; 륯; ) HANGUL SYLLABLE RYUC +B970;B970;1105 1172 11BF;B970;1105 1172 11BF; # (륰; 륰; 륰; 륰; 륰; ) HANGUL SYLLABLE RYUK +B971;B971;1105 1172 11C0;B971;1105 1172 11C0; # (륱; 륱; 륱; 륱; 륱; ) HANGUL SYLLABLE RYUT +B972;B972;1105 1172 11C1;B972;1105 1172 11C1; # (륲; 륲; 륲; 륲; 륲; ) HANGUL SYLLABLE RYUP +B973;B973;1105 1172 11C2;B973;1105 1172 11C2; # (륳; 륳; 륳; 륳; 륳; ) HANGUL SYLLABLE RYUH +B974;B974;1105 1173;B974;1105 1173; # (르; 르; 르; 르; 르; ) HANGUL SYLLABLE REU +B975;B975;1105 1173 11A8;B975;1105 1173 11A8; # (륵; 륵; 륵; 륵; 륵; ) HANGUL SYLLABLE REUG +B976;B976;1105 1173 11A9;B976;1105 1173 11A9; # (륶; 륶; 륶; 륶; 륶; ) HANGUL SYLLABLE REUGG +B977;B977;1105 1173 11AA;B977;1105 1173 11AA; # (륷; 륷; 륷; 륷; 륷; ) HANGUL SYLLABLE REUGS +B978;B978;1105 1173 11AB;B978;1105 1173 11AB; # (른; 른; 른; 른; 른; ) HANGUL SYLLABLE REUN +B979;B979;1105 1173 11AC;B979;1105 1173 11AC; # (륹; 륹; 륹; 륹; 륹; ) HANGUL SYLLABLE REUNJ +B97A;B97A;1105 1173 11AD;B97A;1105 1173 11AD; # (륺; 륺; 륺; 륺; 륺; ) HANGUL SYLLABLE REUNH +B97B;B97B;1105 1173 11AE;B97B;1105 1173 11AE; # (륻; 륻; 륻; 륻; 륻; ) HANGUL SYLLABLE REUD +B97C;B97C;1105 1173 11AF;B97C;1105 1173 11AF; # (를; 를; 를; 를; 를; ) HANGUL SYLLABLE REUL +B97D;B97D;1105 1173 11B0;B97D;1105 1173 11B0; # (륽; 륽; 륽; 륽; 륽; ) HANGUL SYLLABLE REULG +B97E;B97E;1105 1173 11B1;B97E;1105 1173 11B1; # (륾; 륾; 륾; 륾; 륾; ) HANGUL SYLLABLE REULM +B97F;B97F;1105 1173 11B2;B97F;1105 1173 11B2; # (륿; 륿; 륿; 륿; 륿; ) HANGUL SYLLABLE REULB +B980;B980;1105 1173 11B3;B980;1105 1173 11B3; # (릀; 릀; 릀; 릀; 릀; ) HANGUL SYLLABLE REULS +B981;B981;1105 1173 11B4;B981;1105 1173 11B4; # (릁; 릁; 릁; 릁; 릁; ) HANGUL SYLLABLE REULT +B982;B982;1105 1173 11B5;B982;1105 1173 11B5; # (릂; 릂; 릂; 릂; 릂; ) HANGUL SYLLABLE REULP +B983;B983;1105 1173 11B6;B983;1105 1173 11B6; # (릃; 릃; 릃; 릃; 릃; ) HANGUL SYLLABLE REULH +B984;B984;1105 1173 11B7;B984;1105 1173 11B7; # (름; 름; 름; 름; 름; ) HANGUL SYLLABLE REUM +B985;B985;1105 1173 11B8;B985;1105 1173 11B8; # (릅; 릅; 릅; 릅; 릅; ) HANGUL SYLLABLE REUB +B986;B986;1105 1173 11B9;B986;1105 1173 11B9; # (릆; 릆; 릆; 릆; 릆; ) HANGUL SYLLABLE REUBS +B987;B987;1105 1173 11BA;B987;1105 1173 11BA; # (릇; 릇; 릇; 릇; 릇; ) HANGUL SYLLABLE REUS +B988;B988;1105 1173 11BB;B988;1105 1173 11BB; # (릈; 릈; 릈; 릈; 릈; ) HANGUL SYLLABLE REUSS +B989;B989;1105 1173 11BC;B989;1105 1173 11BC; # (릉; 릉; 릉; 릉; 릉; ) HANGUL SYLLABLE REUNG +B98A;B98A;1105 1173 11BD;B98A;1105 1173 11BD; # (릊; 릊; 릊; 릊; 릊; ) HANGUL SYLLABLE REUJ +B98B;B98B;1105 1173 11BE;B98B;1105 1173 11BE; # (릋; 릋; 릋; 릋; 릋; ) HANGUL SYLLABLE REUC +B98C;B98C;1105 1173 11BF;B98C;1105 1173 11BF; # (릌; 릌; 릌; 릌; 릌; ) HANGUL SYLLABLE REUK +B98D;B98D;1105 1173 11C0;B98D;1105 1173 11C0; # (릍; 릍; 릍; 릍; 릍; ) HANGUL SYLLABLE REUT +B98E;B98E;1105 1173 11C1;B98E;1105 1173 11C1; # (릎; 릎; 릎; 릎; 릎; ) HANGUL SYLLABLE REUP +B98F;B98F;1105 1173 11C2;B98F;1105 1173 11C2; # (릏; 릏; 릏; 릏; 릏; ) HANGUL SYLLABLE REUH +B990;B990;1105 1174;B990;1105 1174; # (릐; 릐; 릐; 릐; 릐; ) HANGUL SYLLABLE RYI +B991;B991;1105 1174 11A8;B991;1105 1174 11A8; # (릑; 릑; 릑; 릑; 릑; ) HANGUL SYLLABLE RYIG +B992;B992;1105 1174 11A9;B992;1105 1174 11A9; # (릒; 릒; 릒; 릒; 릒; ) HANGUL SYLLABLE RYIGG +B993;B993;1105 1174 11AA;B993;1105 1174 11AA; # (릓; 릓; 릓; 릓; 릓; ) HANGUL SYLLABLE RYIGS +B994;B994;1105 1174 11AB;B994;1105 1174 11AB; # (릔; 릔; 릔; 릔; 릔; ) HANGUL SYLLABLE RYIN +B995;B995;1105 1174 11AC;B995;1105 1174 11AC; # (릕; 릕; 릕; 릕; 릕; ) HANGUL SYLLABLE RYINJ +B996;B996;1105 1174 11AD;B996;1105 1174 11AD; # (릖; 릖; 릖; 릖; 릖; ) HANGUL SYLLABLE RYINH +B997;B997;1105 1174 11AE;B997;1105 1174 11AE; # (릗; 릗; 릗; 릗; 릗; ) HANGUL SYLLABLE RYID +B998;B998;1105 1174 11AF;B998;1105 1174 11AF; # (릘; 릘; 릘; 릘; 릘; ) HANGUL SYLLABLE RYIL +B999;B999;1105 1174 11B0;B999;1105 1174 11B0; # (릙; 릙; 릙; 릙; 릙; ) HANGUL SYLLABLE RYILG +B99A;B99A;1105 1174 11B1;B99A;1105 1174 11B1; # (릚; 릚; 릚; 릚; 릚; ) HANGUL SYLLABLE RYILM +B99B;B99B;1105 1174 11B2;B99B;1105 1174 11B2; # (릛; 릛; 릛; 릛; 릛; ) HANGUL SYLLABLE RYILB +B99C;B99C;1105 1174 11B3;B99C;1105 1174 11B3; # (릜; 릜; 릜; 릜; 릜; ) HANGUL SYLLABLE RYILS +B99D;B99D;1105 1174 11B4;B99D;1105 1174 11B4; # (릝; 릝; 릝; 릝; 릝; ) HANGUL SYLLABLE RYILT +B99E;B99E;1105 1174 11B5;B99E;1105 1174 11B5; # (릞; 릞; 릞; 릞; 릞; ) HANGUL SYLLABLE RYILP +B99F;B99F;1105 1174 11B6;B99F;1105 1174 11B6; # (릟; 릟; 릟; 릟; 릟; ) HANGUL SYLLABLE RYILH +B9A0;B9A0;1105 1174 11B7;B9A0;1105 1174 11B7; # (릠; 릠; 릠; 릠; 릠; ) HANGUL SYLLABLE RYIM +B9A1;B9A1;1105 1174 11B8;B9A1;1105 1174 11B8; # (릡; 릡; 릡; 릡; 릡; ) HANGUL SYLLABLE RYIB +B9A2;B9A2;1105 1174 11B9;B9A2;1105 1174 11B9; # (릢; 릢; 릢; 릢; 릢; ) HANGUL SYLLABLE RYIBS +B9A3;B9A3;1105 1174 11BA;B9A3;1105 1174 11BA; # (릣; 릣; 릣; 릣; 릣; ) HANGUL SYLLABLE RYIS +B9A4;B9A4;1105 1174 11BB;B9A4;1105 1174 11BB; # (릤; 릤; 릤; 릤; 릤; ) HANGUL SYLLABLE RYISS +B9A5;B9A5;1105 1174 11BC;B9A5;1105 1174 11BC; # (릥; 릥; 릥; 릥; 릥; ) HANGUL SYLLABLE RYING +B9A6;B9A6;1105 1174 11BD;B9A6;1105 1174 11BD; # (릦; 릦; 릦; 릦; 릦; ) HANGUL SYLLABLE RYIJ +B9A7;B9A7;1105 1174 11BE;B9A7;1105 1174 11BE; # (릧; 릧; 릧; 릧; 릧; ) HANGUL SYLLABLE RYIC +B9A8;B9A8;1105 1174 11BF;B9A8;1105 1174 11BF; # (릨; 릨; 릨; 릨; 릨; ) HANGUL SYLLABLE RYIK +B9A9;B9A9;1105 1174 11C0;B9A9;1105 1174 11C0; # (릩; 릩; 릩; 릩; 릩; ) HANGUL SYLLABLE RYIT +B9AA;B9AA;1105 1174 11C1;B9AA;1105 1174 11C1; # (릪; 릪; 릪; 릪; 릪; ) HANGUL SYLLABLE RYIP +B9AB;B9AB;1105 1174 11C2;B9AB;1105 1174 11C2; # (릫; 릫; 릫; 릫; 릫; ) HANGUL SYLLABLE RYIH +B9AC;B9AC;1105 1175;B9AC;1105 1175; # (리; 리; 리; 리; 리; ) HANGUL SYLLABLE RI +B9AD;B9AD;1105 1175 11A8;B9AD;1105 1175 11A8; # (릭; 릭; 릭; 릭; 릭; ) HANGUL SYLLABLE RIG +B9AE;B9AE;1105 1175 11A9;B9AE;1105 1175 11A9; # (릮; 릮; 릮; 릮; 릮; ) HANGUL SYLLABLE RIGG +B9AF;B9AF;1105 1175 11AA;B9AF;1105 1175 11AA; # (릯; 릯; 릯; 릯; 릯; ) HANGUL SYLLABLE RIGS +B9B0;B9B0;1105 1175 11AB;B9B0;1105 1175 11AB; # (린; 린; 린; 린; 린; ) HANGUL SYLLABLE RIN +B9B1;B9B1;1105 1175 11AC;B9B1;1105 1175 11AC; # (릱; 릱; 릱; 릱; 릱; ) HANGUL SYLLABLE RINJ +B9B2;B9B2;1105 1175 11AD;B9B2;1105 1175 11AD; # (릲; 릲; 릲; 릲; 릲; ) HANGUL SYLLABLE RINH +B9B3;B9B3;1105 1175 11AE;B9B3;1105 1175 11AE; # (릳; 릳; 릳; 릳; 릳; ) HANGUL SYLLABLE RID +B9B4;B9B4;1105 1175 11AF;B9B4;1105 1175 11AF; # (릴; 릴; 릴; 릴; 릴; ) HANGUL SYLLABLE RIL +B9B5;B9B5;1105 1175 11B0;B9B5;1105 1175 11B0; # (릵; 릵; 릵; 릵; 릵; ) HANGUL SYLLABLE RILG +B9B6;B9B6;1105 1175 11B1;B9B6;1105 1175 11B1; # (릶; 릶; 릶; 릶; 릶; ) HANGUL SYLLABLE RILM +B9B7;B9B7;1105 1175 11B2;B9B7;1105 1175 11B2; # (릷; 릷; 릷; 릷; 릷; ) HANGUL SYLLABLE RILB +B9B8;B9B8;1105 1175 11B3;B9B8;1105 1175 11B3; # (릸; 릸; 릸; 릸; 릸; ) HANGUL SYLLABLE RILS +B9B9;B9B9;1105 1175 11B4;B9B9;1105 1175 11B4; # (릹; 릹; 릹; 릹; 릹; ) HANGUL SYLLABLE RILT +B9BA;B9BA;1105 1175 11B5;B9BA;1105 1175 11B5; # (릺; 릺; 릺; 릺; 릺; ) HANGUL SYLLABLE RILP +B9BB;B9BB;1105 1175 11B6;B9BB;1105 1175 11B6; # (릻; 릻; 릻; 릻; 릻; ) HANGUL SYLLABLE RILH +B9BC;B9BC;1105 1175 11B7;B9BC;1105 1175 11B7; # (림; 림; 림; 림; 림; ) HANGUL SYLLABLE RIM +B9BD;B9BD;1105 1175 11B8;B9BD;1105 1175 11B8; # (립; 립; 립; 립; 립; ) HANGUL SYLLABLE RIB +B9BE;B9BE;1105 1175 11B9;B9BE;1105 1175 11B9; # (릾; 릾; 릾; 릾; 릾; ) HANGUL SYLLABLE RIBS +B9BF;B9BF;1105 1175 11BA;B9BF;1105 1175 11BA; # (릿; 릿; 릿; 릿; 릿; ) HANGUL SYLLABLE RIS +B9C0;B9C0;1105 1175 11BB;B9C0;1105 1175 11BB; # (맀; 맀; 맀; 맀; 맀; ) HANGUL SYLLABLE RISS +B9C1;B9C1;1105 1175 11BC;B9C1;1105 1175 11BC; # (링; 링; 링; 링; 링; ) HANGUL SYLLABLE RING +B9C2;B9C2;1105 1175 11BD;B9C2;1105 1175 11BD; # (맂; 맂; 맂; 맂; 맂; ) HANGUL SYLLABLE RIJ +B9C3;B9C3;1105 1175 11BE;B9C3;1105 1175 11BE; # (맃; 맃; 맃; 맃; 맃; ) HANGUL SYLLABLE RIC +B9C4;B9C4;1105 1175 11BF;B9C4;1105 1175 11BF; # (맄; 맄; 맄; 맄; 맄; ) HANGUL SYLLABLE RIK +B9C5;B9C5;1105 1175 11C0;B9C5;1105 1175 11C0; # (맅; 맅; 맅; 맅; 맅; ) HANGUL SYLLABLE RIT +B9C6;B9C6;1105 1175 11C1;B9C6;1105 1175 11C1; # (맆; 맆; 맆; 맆; 맆; ) HANGUL SYLLABLE RIP +B9C7;B9C7;1105 1175 11C2;B9C7;1105 1175 11C2; # (맇; 맇; 맇; 맇; 맇; ) HANGUL SYLLABLE RIH +B9C8;B9C8;1106 1161;B9C8;1106 1161; # (마; 마; 마; 마; 마; ) HANGUL SYLLABLE MA +B9C9;B9C9;1106 1161 11A8;B9C9;1106 1161 11A8; # (막; 막; 막; 막; 막; ) HANGUL SYLLABLE MAG +B9CA;B9CA;1106 1161 11A9;B9CA;1106 1161 11A9; # (맊; 맊; 맊; 맊; 맊; ) HANGUL SYLLABLE MAGG +B9CB;B9CB;1106 1161 11AA;B9CB;1106 1161 11AA; # (맋; 맋; 맋; 맋; 맋; ) HANGUL SYLLABLE MAGS +B9CC;B9CC;1106 1161 11AB;B9CC;1106 1161 11AB; # (만; 만; 만; 만; 만; ) HANGUL SYLLABLE MAN +B9CD;B9CD;1106 1161 11AC;B9CD;1106 1161 11AC; # (맍; 맍; 맍; 맍; 맍; ) HANGUL SYLLABLE MANJ +B9CE;B9CE;1106 1161 11AD;B9CE;1106 1161 11AD; # (많; 많; 많; 많; 많; ) HANGUL SYLLABLE MANH +B9CF;B9CF;1106 1161 11AE;B9CF;1106 1161 11AE; # (맏; 맏; 맏; 맏; 맏; ) HANGUL SYLLABLE MAD +B9D0;B9D0;1106 1161 11AF;B9D0;1106 1161 11AF; # (말; 말; 말; 말; 말; ) HANGUL SYLLABLE MAL +B9D1;B9D1;1106 1161 11B0;B9D1;1106 1161 11B0; # (맑; 맑; 맑; 맑; 맑; ) HANGUL SYLLABLE MALG +B9D2;B9D2;1106 1161 11B1;B9D2;1106 1161 11B1; # (맒; 맒; 맒; 맒; 맒; ) HANGUL SYLLABLE MALM +B9D3;B9D3;1106 1161 11B2;B9D3;1106 1161 11B2; # (맓; 맓; 맓; 맓; 맓; ) HANGUL SYLLABLE MALB +B9D4;B9D4;1106 1161 11B3;B9D4;1106 1161 11B3; # (맔; 맔; 맔; 맔; 맔; ) HANGUL SYLLABLE MALS +B9D5;B9D5;1106 1161 11B4;B9D5;1106 1161 11B4; # (맕; 맕; 맕; 맕; 맕; ) HANGUL SYLLABLE MALT +B9D6;B9D6;1106 1161 11B5;B9D6;1106 1161 11B5; # (맖; 맖; 맖; 맖; 맖; ) HANGUL SYLLABLE MALP +B9D7;B9D7;1106 1161 11B6;B9D7;1106 1161 11B6; # (맗; 맗; 맗; 맗; 맗; ) HANGUL SYLLABLE MALH +B9D8;B9D8;1106 1161 11B7;B9D8;1106 1161 11B7; # (맘; 맘; 맘; 맘; 맘; ) HANGUL SYLLABLE MAM +B9D9;B9D9;1106 1161 11B8;B9D9;1106 1161 11B8; # (맙; 맙; 맙; 맙; 맙; ) HANGUL SYLLABLE MAB +B9DA;B9DA;1106 1161 11B9;B9DA;1106 1161 11B9; # (맚; 맚; 맚; 맚; 맚; ) HANGUL SYLLABLE MABS +B9DB;B9DB;1106 1161 11BA;B9DB;1106 1161 11BA; # (맛; 맛; 맛; 맛; 맛; ) HANGUL SYLLABLE MAS +B9DC;B9DC;1106 1161 11BB;B9DC;1106 1161 11BB; # (맜; 맜; 맜; 맜; 맜; ) HANGUL SYLLABLE MASS +B9DD;B9DD;1106 1161 11BC;B9DD;1106 1161 11BC; # (망; 망; 망; 망; 망; ) HANGUL SYLLABLE MANG +B9DE;B9DE;1106 1161 11BD;B9DE;1106 1161 11BD; # (맞; 맞; 맞; 맞; 맞; ) HANGUL SYLLABLE MAJ +B9DF;B9DF;1106 1161 11BE;B9DF;1106 1161 11BE; # (맟; 맟; 맟; 맟; 맟; ) HANGUL SYLLABLE MAC +B9E0;B9E0;1106 1161 11BF;B9E0;1106 1161 11BF; # (맠; 맠; 맠; 맠; 맠; ) HANGUL SYLLABLE MAK +B9E1;B9E1;1106 1161 11C0;B9E1;1106 1161 11C0; # (맡; 맡; 맡; 맡; 맡; ) HANGUL SYLLABLE MAT +B9E2;B9E2;1106 1161 11C1;B9E2;1106 1161 11C1; # (맢; 맢; 맢; 맢; 맢; ) HANGUL SYLLABLE MAP +B9E3;B9E3;1106 1161 11C2;B9E3;1106 1161 11C2; # (맣; 맣; 맣; 맣; 맣; ) HANGUL SYLLABLE MAH +B9E4;B9E4;1106 1162;B9E4;1106 1162; # (매; 매; 매; 매; 매; ) HANGUL SYLLABLE MAE +B9E5;B9E5;1106 1162 11A8;B9E5;1106 1162 11A8; # (맥; 맥; 맥; 맥; 맥; ) HANGUL SYLLABLE MAEG +B9E6;B9E6;1106 1162 11A9;B9E6;1106 1162 11A9; # (맦; 맦; 맦; 맦; 맦; ) HANGUL SYLLABLE MAEGG +B9E7;B9E7;1106 1162 11AA;B9E7;1106 1162 11AA; # (맧; 맧; 맧; 맧; 맧; ) HANGUL SYLLABLE MAEGS +B9E8;B9E8;1106 1162 11AB;B9E8;1106 1162 11AB; # (맨; 맨; 맨; 맨; 맨; ) HANGUL SYLLABLE MAEN +B9E9;B9E9;1106 1162 11AC;B9E9;1106 1162 11AC; # (맩; 맩; 맩; 맩; 맩; ) HANGUL SYLLABLE MAENJ +B9EA;B9EA;1106 1162 11AD;B9EA;1106 1162 11AD; # (맪; 맪; 맪; 맪; 맪; ) HANGUL SYLLABLE MAENH +B9EB;B9EB;1106 1162 11AE;B9EB;1106 1162 11AE; # (맫; 맫; 맫; 맫; 맫; ) HANGUL SYLLABLE MAED +B9EC;B9EC;1106 1162 11AF;B9EC;1106 1162 11AF; # (맬; 맬; 맬; 맬; 맬; ) HANGUL SYLLABLE MAEL +B9ED;B9ED;1106 1162 11B0;B9ED;1106 1162 11B0; # (맭; 맭; 맭; 맭; 맭; ) HANGUL SYLLABLE MAELG +B9EE;B9EE;1106 1162 11B1;B9EE;1106 1162 11B1; # (맮; 맮; 맮; 맮; 맮; ) HANGUL SYLLABLE MAELM +B9EF;B9EF;1106 1162 11B2;B9EF;1106 1162 11B2; # (맯; 맯; 맯; 맯; 맯; ) HANGUL SYLLABLE MAELB +B9F0;B9F0;1106 1162 11B3;B9F0;1106 1162 11B3; # (맰; 맰; 맰; 맰; 맰; ) HANGUL SYLLABLE MAELS +B9F1;B9F1;1106 1162 11B4;B9F1;1106 1162 11B4; # (맱; 맱; 맱; 맱; 맱; ) HANGUL SYLLABLE MAELT +B9F2;B9F2;1106 1162 11B5;B9F2;1106 1162 11B5; # (맲; 맲; 맲; 맲; 맲; ) HANGUL SYLLABLE MAELP +B9F3;B9F3;1106 1162 11B6;B9F3;1106 1162 11B6; # (맳; 맳; 맳; 맳; 맳; ) HANGUL SYLLABLE MAELH +B9F4;B9F4;1106 1162 11B7;B9F4;1106 1162 11B7; # (맴; 맴; 맴; 맴; 맴; ) HANGUL SYLLABLE MAEM +B9F5;B9F5;1106 1162 11B8;B9F5;1106 1162 11B8; # (맵; 맵; 맵; 맵; 맵; ) HANGUL SYLLABLE MAEB +B9F6;B9F6;1106 1162 11B9;B9F6;1106 1162 11B9; # (맶; 맶; 맶; 맶; 맶; ) HANGUL SYLLABLE MAEBS +B9F7;B9F7;1106 1162 11BA;B9F7;1106 1162 11BA; # (맷; 맷; 맷; 맷; 맷; ) HANGUL SYLLABLE MAES +B9F8;B9F8;1106 1162 11BB;B9F8;1106 1162 11BB; # (맸; 맸; 맸; 맸; 맸; ) HANGUL SYLLABLE MAESS +B9F9;B9F9;1106 1162 11BC;B9F9;1106 1162 11BC; # (맹; 맹; 맹; 맹; 맹; ) HANGUL SYLLABLE MAENG +B9FA;B9FA;1106 1162 11BD;B9FA;1106 1162 11BD; # (맺; 맺; 맺; 맺; 맺; ) HANGUL SYLLABLE MAEJ +B9FB;B9FB;1106 1162 11BE;B9FB;1106 1162 11BE; # (맻; 맻; 맻; 맻; 맻; ) HANGUL SYLLABLE MAEC +B9FC;B9FC;1106 1162 11BF;B9FC;1106 1162 11BF; # (맼; 맼; 맼; 맼; 맼; ) HANGUL SYLLABLE MAEK +B9FD;B9FD;1106 1162 11C0;B9FD;1106 1162 11C0; # (맽; 맽; 맽; 맽; 맽; ) HANGUL SYLLABLE MAET +B9FE;B9FE;1106 1162 11C1;B9FE;1106 1162 11C1; # (맾; 맾; 맾; 맾; 맾; ) HANGUL SYLLABLE MAEP +B9FF;B9FF;1106 1162 11C2;B9FF;1106 1162 11C2; # (맿; 맿; 맿; 맿; 맿; ) HANGUL SYLLABLE MAEH +BA00;BA00;1106 1163;BA00;1106 1163; # (먀; 먀; 먀; 먀; 먀; ) HANGUL SYLLABLE MYA +BA01;BA01;1106 1163 11A8;BA01;1106 1163 11A8; # (먁; 먁; 먁; 먁; 먁; ) HANGUL SYLLABLE MYAG +BA02;BA02;1106 1163 11A9;BA02;1106 1163 11A9; # (먂; 먂; 먂; 먂; 먂; ) HANGUL SYLLABLE MYAGG +BA03;BA03;1106 1163 11AA;BA03;1106 1163 11AA; # (먃; 먃; 먃; 먃; 먃; ) HANGUL SYLLABLE MYAGS +BA04;BA04;1106 1163 11AB;BA04;1106 1163 11AB; # (먄; 먄; 먄; 먄; 먄; ) HANGUL SYLLABLE MYAN +BA05;BA05;1106 1163 11AC;BA05;1106 1163 11AC; # (먅; 먅; 먅; 먅; 먅; ) HANGUL SYLLABLE MYANJ +BA06;BA06;1106 1163 11AD;BA06;1106 1163 11AD; # (먆; 먆; 먆; 먆; 먆; ) HANGUL SYLLABLE MYANH +BA07;BA07;1106 1163 11AE;BA07;1106 1163 11AE; # (먇; 먇; 먇; 먇; 먇; ) HANGUL SYLLABLE MYAD +BA08;BA08;1106 1163 11AF;BA08;1106 1163 11AF; # (먈; 먈; 먈; 먈; 먈; ) HANGUL SYLLABLE MYAL +BA09;BA09;1106 1163 11B0;BA09;1106 1163 11B0; # (먉; 먉; 먉; 먉; 먉; ) HANGUL SYLLABLE MYALG +BA0A;BA0A;1106 1163 11B1;BA0A;1106 1163 11B1; # (먊; 먊; 먊; 먊; 먊; ) HANGUL SYLLABLE MYALM +BA0B;BA0B;1106 1163 11B2;BA0B;1106 1163 11B2; # (먋; 먋; 먋; 먋; 먋; ) HANGUL SYLLABLE MYALB +BA0C;BA0C;1106 1163 11B3;BA0C;1106 1163 11B3; # (먌; 먌; 먌; 먌; 먌; ) HANGUL SYLLABLE MYALS +BA0D;BA0D;1106 1163 11B4;BA0D;1106 1163 11B4; # (먍; 먍; 먍; 먍; 먍; ) HANGUL SYLLABLE MYALT +BA0E;BA0E;1106 1163 11B5;BA0E;1106 1163 11B5; # (먎; 먎; 먎; 먎; 먎; ) HANGUL SYLLABLE MYALP +BA0F;BA0F;1106 1163 11B6;BA0F;1106 1163 11B6; # (먏; 먏; 먏; 먏; 먏; ) HANGUL SYLLABLE MYALH +BA10;BA10;1106 1163 11B7;BA10;1106 1163 11B7; # (먐; 먐; 먐; 먐; 먐; ) HANGUL SYLLABLE MYAM +BA11;BA11;1106 1163 11B8;BA11;1106 1163 11B8; # (먑; 먑; 먑; 먑; 먑; ) HANGUL SYLLABLE MYAB +BA12;BA12;1106 1163 11B9;BA12;1106 1163 11B9; # (먒; 먒; 먒; 먒; 먒; ) HANGUL SYLLABLE MYABS +BA13;BA13;1106 1163 11BA;BA13;1106 1163 11BA; # (먓; 먓; 먓; 먓; 먓; ) HANGUL SYLLABLE MYAS +BA14;BA14;1106 1163 11BB;BA14;1106 1163 11BB; # (먔; 먔; 먔; 먔; 먔; ) HANGUL SYLLABLE MYASS +BA15;BA15;1106 1163 11BC;BA15;1106 1163 11BC; # (먕; 먕; 먕; 먕; 먕; ) HANGUL SYLLABLE MYANG +BA16;BA16;1106 1163 11BD;BA16;1106 1163 11BD; # (먖; 먖; 먖; 먖; 먖; ) HANGUL SYLLABLE MYAJ +BA17;BA17;1106 1163 11BE;BA17;1106 1163 11BE; # (먗; 먗; 먗; 먗; 먗; ) HANGUL SYLLABLE MYAC +BA18;BA18;1106 1163 11BF;BA18;1106 1163 11BF; # (먘; 먘; 먘; 먘; 먘; ) HANGUL SYLLABLE MYAK +BA19;BA19;1106 1163 11C0;BA19;1106 1163 11C0; # (먙; 먙; 먙; 먙; 먙; ) HANGUL SYLLABLE MYAT +BA1A;BA1A;1106 1163 11C1;BA1A;1106 1163 11C1; # (먚; 먚; 먚; 먚; 먚; ) HANGUL SYLLABLE MYAP +BA1B;BA1B;1106 1163 11C2;BA1B;1106 1163 11C2; # (먛; 먛; 먛; 먛; 먛; ) HANGUL SYLLABLE MYAH +BA1C;BA1C;1106 1164;BA1C;1106 1164; # (먜; 먜; 먜; 먜; 먜; ) HANGUL SYLLABLE MYAE +BA1D;BA1D;1106 1164 11A8;BA1D;1106 1164 11A8; # (먝; 먝; 먝; 먝; 먝; ) HANGUL SYLLABLE MYAEG +BA1E;BA1E;1106 1164 11A9;BA1E;1106 1164 11A9; # (먞; 먞; 먞; 먞; 먞; ) HANGUL SYLLABLE MYAEGG +BA1F;BA1F;1106 1164 11AA;BA1F;1106 1164 11AA; # (먟; 먟; 먟; 먟; 먟; ) HANGUL SYLLABLE MYAEGS +BA20;BA20;1106 1164 11AB;BA20;1106 1164 11AB; # (먠; 먠; 먠; 먠; 먠; ) HANGUL SYLLABLE MYAEN +BA21;BA21;1106 1164 11AC;BA21;1106 1164 11AC; # (먡; 먡; 먡; 먡; 먡; ) HANGUL SYLLABLE MYAENJ +BA22;BA22;1106 1164 11AD;BA22;1106 1164 11AD; # (먢; 먢; 먢; 먢; 먢; ) HANGUL SYLLABLE MYAENH +BA23;BA23;1106 1164 11AE;BA23;1106 1164 11AE; # (먣; 먣; 먣; 먣; 먣; ) HANGUL SYLLABLE MYAED +BA24;BA24;1106 1164 11AF;BA24;1106 1164 11AF; # (먤; 먤; 먤; 먤; 먤; ) HANGUL SYLLABLE MYAEL +BA25;BA25;1106 1164 11B0;BA25;1106 1164 11B0; # (먥; 먥; 먥; 먥; 먥; ) HANGUL SYLLABLE MYAELG +BA26;BA26;1106 1164 11B1;BA26;1106 1164 11B1; # (먦; 먦; 먦; 먦; 먦; ) HANGUL SYLLABLE MYAELM +BA27;BA27;1106 1164 11B2;BA27;1106 1164 11B2; # (먧; 먧; 먧; 먧; 먧; ) HANGUL SYLLABLE MYAELB +BA28;BA28;1106 1164 11B3;BA28;1106 1164 11B3; # (먨; 먨; 먨; 먨; 먨; ) HANGUL SYLLABLE MYAELS +BA29;BA29;1106 1164 11B4;BA29;1106 1164 11B4; # (먩; 먩; 먩; 먩; 먩; ) HANGUL SYLLABLE MYAELT +BA2A;BA2A;1106 1164 11B5;BA2A;1106 1164 11B5; # (먪; 먪; 먪; 먪; 먪; ) HANGUL SYLLABLE MYAELP +BA2B;BA2B;1106 1164 11B6;BA2B;1106 1164 11B6; # (먫; 먫; 먫; 먫; 먫; ) HANGUL SYLLABLE MYAELH +BA2C;BA2C;1106 1164 11B7;BA2C;1106 1164 11B7; # (먬; 먬; 먬; 먬; 먬; ) HANGUL SYLLABLE MYAEM +BA2D;BA2D;1106 1164 11B8;BA2D;1106 1164 11B8; # (먭; 먭; 먭; 먭; 먭; ) HANGUL SYLLABLE MYAEB +BA2E;BA2E;1106 1164 11B9;BA2E;1106 1164 11B9; # (먮; 먮; 먮; 먮; 먮; ) HANGUL SYLLABLE MYAEBS +BA2F;BA2F;1106 1164 11BA;BA2F;1106 1164 11BA; # (먯; 먯; 먯; 먯; 먯; ) HANGUL SYLLABLE MYAES +BA30;BA30;1106 1164 11BB;BA30;1106 1164 11BB; # (먰; 먰; 먰; 먰; 먰; ) HANGUL SYLLABLE MYAESS +BA31;BA31;1106 1164 11BC;BA31;1106 1164 11BC; # (먱; 먱; 먱; 먱; 먱; ) HANGUL SYLLABLE MYAENG +BA32;BA32;1106 1164 11BD;BA32;1106 1164 11BD; # (먲; 먲; 먲; 먲; 먲; ) HANGUL SYLLABLE MYAEJ +BA33;BA33;1106 1164 11BE;BA33;1106 1164 11BE; # (먳; 먳; 먳; 먳; 먳; ) HANGUL SYLLABLE MYAEC +BA34;BA34;1106 1164 11BF;BA34;1106 1164 11BF; # (먴; 먴; 먴; 먴; 먴; ) HANGUL SYLLABLE MYAEK +BA35;BA35;1106 1164 11C0;BA35;1106 1164 11C0; # (먵; 먵; 먵; 먵; 먵; ) HANGUL SYLLABLE MYAET +BA36;BA36;1106 1164 11C1;BA36;1106 1164 11C1; # (먶; 먶; 먶; 먶; 먶; ) HANGUL SYLLABLE MYAEP +BA37;BA37;1106 1164 11C2;BA37;1106 1164 11C2; # (먷; 먷; 먷; 먷; 먷; ) HANGUL SYLLABLE MYAEH +BA38;BA38;1106 1165;BA38;1106 1165; # (머; 머; 머; 머; 머; ) HANGUL SYLLABLE MEO +BA39;BA39;1106 1165 11A8;BA39;1106 1165 11A8; # (먹; 먹; 먹; 먹; 먹; ) HANGUL SYLLABLE MEOG +BA3A;BA3A;1106 1165 11A9;BA3A;1106 1165 11A9; # (먺; 먺; 먺; 먺; 먺; ) HANGUL SYLLABLE MEOGG +BA3B;BA3B;1106 1165 11AA;BA3B;1106 1165 11AA; # (먻; 먻; 먻; 먻; 먻; ) HANGUL SYLLABLE MEOGS +BA3C;BA3C;1106 1165 11AB;BA3C;1106 1165 11AB; # (먼; 먼; 먼; 먼; 먼; ) HANGUL SYLLABLE MEON +BA3D;BA3D;1106 1165 11AC;BA3D;1106 1165 11AC; # (먽; 먽; 먽; 먽; 먽; ) HANGUL SYLLABLE MEONJ +BA3E;BA3E;1106 1165 11AD;BA3E;1106 1165 11AD; # (먾; 먾; 먾; 먾; 먾; ) HANGUL SYLLABLE MEONH +BA3F;BA3F;1106 1165 11AE;BA3F;1106 1165 11AE; # (먿; 먿; 먿; 먿; 먿; ) HANGUL SYLLABLE MEOD +BA40;BA40;1106 1165 11AF;BA40;1106 1165 11AF; # (멀; 멀; 멀; 멀; 멀; ) HANGUL SYLLABLE MEOL +BA41;BA41;1106 1165 11B0;BA41;1106 1165 11B0; # (멁; 멁; 멁; 멁; 멁; ) HANGUL SYLLABLE MEOLG +BA42;BA42;1106 1165 11B1;BA42;1106 1165 11B1; # (멂; 멂; 멂; 멂; 멂; ) HANGUL SYLLABLE MEOLM +BA43;BA43;1106 1165 11B2;BA43;1106 1165 11B2; # (멃; 멃; 멃; 멃; 멃; ) HANGUL SYLLABLE MEOLB +BA44;BA44;1106 1165 11B3;BA44;1106 1165 11B3; # (멄; 멄; 멄; 멄; 멄; ) HANGUL SYLLABLE MEOLS +BA45;BA45;1106 1165 11B4;BA45;1106 1165 11B4; # (멅; 멅; 멅; 멅; 멅; ) HANGUL SYLLABLE MEOLT +BA46;BA46;1106 1165 11B5;BA46;1106 1165 11B5; # (멆; 멆; 멆; 멆; 멆; ) HANGUL SYLLABLE MEOLP +BA47;BA47;1106 1165 11B6;BA47;1106 1165 11B6; # (멇; 멇; 멇; 멇; 멇; ) HANGUL SYLLABLE MEOLH +BA48;BA48;1106 1165 11B7;BA48;1106 1165 11B7; # (멈; 멈; 멈; 멈; 멈; ) HANGUL SYLLABLE MEOM +BA49;BA49;1106 1165 11B8;BA49;1106 1165 11B8; # (멉; 멉; 멉; 멉; 멉; ) HANGUL SYLLABLE MEOB +BA4A;BA4A;1106 1165 11B9;BA4A;1106 1165 11B9; # (멊; 멊; 멊; 멊; 멊; ) HANGUL SYLLABLE MEOBS +BA4B;BA4B;1106 1165 11BA;BA4B;1106 1165 11BA; # (멋; 멋; 멋; 멋; 멋; ) HANGUL SYLLABLE MEOS +BA4C;BA4C;1106 1165 11BB;BA4C;1106 1165 11BB; # (멌; 멌; 멌; 멌; 멌; ) HANGUL SYLLABLE MEOSS +BA4D;BA4D;1106 1165 11BC;BA4D;1106 1165 11BC; # (멍; 멍; 멍; 멍; 멍; ) HANGUL SYLLABLE MEONG +BA4E;BA4E;1106 1165 11BD;BA4E;1106 1165 11BD; # (멎; 멎; 멎; 멎; 멎; ) HANGUL SYLLABLE MEOJ +BA4F;BA4F;1106 1165 11BE;BA4F;1106 1165 11BE; # (멏; 멏; 멏; 멏; 멏; ) HANGUL SYLLABLE MEOC +BA50;BA50;1106 1165 11BF;BA50;1106 1165 11BF; # (멐; 멐; 멐; 멐; 멐; ) HANGUL SYLLABLE MEOK +BA51;BA51;1106 1165 11C0;BA51;1106 1165 11C0; # (멑; 멑; 멑; 멑; 멑; ) HANGUL SYLLABLE MEOT +BA52;BA52;1106 1165 11C1;BA52;1106 1165 11C1; # (멒; 멒; 멒; 멒; 멒; ) HANGUL SYLLABLE MEOP +BA53;BA53;1106 1165 11C2;BA53;1106 1165 11C2; # (멓; 멓; 멓; 멓; 멓; ) HANGUL SYLLABLE MEOH +BA54;BA54;1106 1166;BA54;1106 1166; # (메; 메; 메; 메; 메; ) HANGUL SYLLABLE ME +BA55;BA55;1106 1166 11A8;BA55;1106 1166 11A8; # (멕; 멕; 멕; 멕; 멕; ) HANGUL SYLLABLE MEG +BA56;BA56;1106 1166 11A9;BA56;1106 1166 11A9; # (멖; 멖; 멖; 멖; 멖; ) HANGUL SYLLABLE MEGG +BA57;BA57;1106 1166 11AA;BA57;1106 1166 11AA; # (멗; 멗; 멗; 멗; 멗; ) HANGUL SYLLABLE MEGS +BA58;BA58;1106 1166 11AB;BA58;1106 1166 11AB; # (멘; 멘; 멘; 멘; 멘; ) HANGUL SYLLABLE MEN +BA59;BA59;1106 1166 11AC;BA59;1106 1166 11AC; # (멙; 멙; 멙; 멙; 멙; ) HANGUL SYLLABLE MENJ +BA5A;BA5A;1106 1166 11AD;BA5A;1106 1166 11AD; # (멚; 멚; 멚; 멚; 멚; ) HANGUL SYLLABLE MENH +BA5B;BA5B;1106 1166 11AE;BA5B;1106 1166 11AE; # (멛; 멛; 멛; 멛; 멛; ) HANGUL SYLLABLE MED +BA5C;BA5C;1106 1166 11AF;BA5C;1106 1166 11AF; # (멜; 멜; 멜; 멜; 멜; ) HANGUL SYLLABLE MEL +BA5D;BA5D;1106 1166 11B0;BA5D;1106 1166 11B0; # (멝; 멝; 멝; 멝; 멝; ) HANGUL SYLLABLE MELG +BA5E;BA5E;1106 1166 11B1;BA5E;1106 1166 11B1; # (멞; 멞; 멞; 멞; 멞; ) HANGUL SYLLABLE MELM +BA5F;BA5F;1106 1166 11B2;BA5F;1106 1166 11B2; # (멟; 멟; 멟; 멟; 멟; ) HANGUL SYLLABLE MELB +BA60;BA60;1106 1166 11B3;BA60;1106 1166 11B3; # (멠; 멠; 멠; 멠; 멠; ) HANGUL SYLLABLE MELS +BA61;BA61;1106 1166 11B4;BA61;1106 1166 11B4; # (멡; 멡; 멡; 멡; 멡; ) HANGUL SYLLABLE MELT +BA62;BA62;1106 1166 11B5;BA62;1106 1166 11B5; # (멢; 멢; 멢; 멢; 멢; ) HANGUL SYLLABLE MELP +BA63;BA63;1106 1166 11B6;BA63;1106 1166 11B6; # (멣; 멣; 멣; 멣; 멣; ) HANGUL SYLLABLE MELH +BA64;BA64;1106 1166 11B7;BA64;1106 1166 11B7; # (멤; 멤; 멤; 멤; 멤; ) HANGUL SYLLABLE MEM +BA65;BA65;1106 1166 11B8;BA65;1106 1166 11B8; # (멥; 멥; 멥; 멥; 멥; ) HANGUL SYLLABLE MEB +BA66;BA66;1106 1166 11B9;BA66;1106 1166 11B9; # (멦; 멦; 멦; 멦; 멦; ) HANGUL SYLLABLE MEBS +BA67;BA67;1106 1166 11BA;BA67;1106 1166 11BA; # (멧; 멧; 멧; 멧; 멧; ) HANGUL SYLLABLE MES +BA68;BA68;1106 1166 11BB;BA68;1106 1166 11BB; # (멨; 멨; 멨; 멨; 멨; ) HANGUL SYLLABLE MESS +BA69;BA69;1106 1166 11BC;BA69;1106 1166 11BC; # (멩; 멩; 멩; 멩; 멩; ) HANGUL SYLLABLE MENG +BA6A;BA6A;1106 1166 11BD;BA6A;1106 1166 11BD; # (멪; 멪; 멪; 멪; 멪; ) HANGUL SYLLABLE MEJ +BA6B;BA6B;1106 1166 11BE;BA6B;1106 1166 11BE; # (멫; 멫; 멫; 멫; 멫; ) HANGUL SYLLABLE MEC +BA6C;BA6C;1106 1166 11BF;BA6C;1106 1166 11BF; # (멬; 멬; 멬; 멬; 멬; ) HANGUL SYLLABLE MEK +BA6D;BA6D;1106 1166 11C0;BA6D;1106 1166 11C0; # (멭; 멭; 멭; 멭; 멭; ) HANGUL SYLLABLE MET +BA6E;BA6E;1106 1166 11C1;BA6E;1106 1166 11C1; # (멮; 멮; 멮; 멮; 멮; ) HANGUL SYLLABLE MEP +BA6F;BA6F;1106 1166 11C2;BA6F;1106 1166 11C2; # (멯; 멯; 멯; 멯; 멯; ) HANGUL SYLLABLE MEH +BA70;BA70;1106 1167;BA70;1106 1167; # (며; 며; 며; 며; 며; ) HANGUL SYLLABLE MYEO +BA71;BA71;1106 1167 11A8;BA71;1106 1167 11A8; # (멱; 멱; 멱; 멱; 멱; ) HANGUL SYLLABLE MYEOG +BA72;BA72;1106 1167 11A9;BA72;1106 1167 11A9; # (멲; 멲; 멲; 멲; 멲; ) HANGUL SYLLABLE MYEOGG +BA73;BA73;1106 1167 11AA;BA73;1106 1167 11AA; # (멳; 멳; 멳; 멳; 멳; ) HANGUL SYLLABLE MYEOGS +BA74;BA74;1106 1167 11AB;BA74;1106 1167 11AB; # (면; 면; 면; 면; 면; ) HANGUL SYLLABLE MYEON +BA75;BA75;1106 1167 11AC;BA75;1106 1167 11AC; # (멵; 멵; 멵; 멵; 멵; ) HANGUL SYLLABLE MYEONJ +BA76;BA76;1106 1167 11AD;BA76;1106 1167 11AD; # (멶; 멶; 멶; 멶; 멶; ) HANGUL SYLLABLE MYEONH +BA77;BA77;1106 1167 11AE;BA77;1106 1167 11AE; # (멷; 멷; 멷; 멷; 멷; ) HANGUL SYLLABLE MYEOD +BA78;BA78;1106 1167 11AF;BA78;1106 1167 11AF; # (멸; 멸; 멸; 멸; 멸; ) HANGUL SYLLABLE MYEOL +BA79;BA79;1106 1167 11B0;BA79;1106 1167 11B0; # (멹; 멹; 멹; 멹; 멹; ) HANGUL SYLLABLE MYEOLG +BA7A;BA7A;1106 1167 11B1;BA7A;1106 1167 11B1; # (멺; 멺; 멺; 멺; 멺; ) HANGUL SYLLABLE MYEOLM +BA7B;BA7B;1106 1167 11B2;BA7B;1106 1167 11B2; # (멻; 멻; 멻; 멻; 멻; ) HANGUL SYLLABLE MYEOLB +BA7C;BA7C;1106 1167 11B3;BA7C;1106 1167 11B3; # (멼; 멼; 멼; 멼; 멼; ) HANGUL SYLLABLE MYEOLS +BA7D;BA7D;1106 1167 11B4;BA7D;1106 1167 11B4; # (멽; 멽; 멽; 멽; 멽; ) HANGUL SYLLABLE MYEOLT +BA7E;BA7E;1106 1167 11B5;BA7E;1106 1167 11B5; # (멾; 멾; 멾; 멾; 멾; ) HANGUL SYLLABLE MYEOLP +BA7F;BA7F;1106 1167 11B6;BA7F;1106 1167 11B6; # (멿; 멿; 멿; 멿; 멿; ) HANGUL SYLLABLE MYEOLH +BA80;BA80;1106 1167 11B7;BA80;1106 1167 11B7; # (몀; 몀; 몀; 몀; 몀; ) HANGUL SYLLABLE MYEOM +BA81;BA81;1106 1167 11B8;BA81;1106 1167 11B8; # (몁; 몁; 몁; 몁; 몁; ) HANGUL SYLLABLE MYEOB +BA82;BA82;1106 1167 11B9;BA82;1106 1167 11B9; # (몂; 몂; 몂; 몂; 몂; ) HANGUL SYLLABLE MYEOBS +BA83;BA83;1106 1167 11BA;BA83;1106 1167 11BA; # (몃; 몃; 몃; 몃; 몃; ) HANGUL SYLLABLE MYEOS +BA84;BA84;1106 1167 11BB;BA84;1106 1167 11BB; # (몄; 몄; 몄; 몄; 몄; ) HANGUL SYLLABLE MYEOSS +BA85;BA85;1106 1167 11BC;BA85;1106 1167 11BC; # (명; 명; 명; 명; 명; ) HANGUL SYLLABLE MYEONG +BA86;BA86;1106 1167 11BD;BA86;1106 1167 11BD; # (몆; 몆; 몆; 몆; 몆; ) HANGUL SYLLABLE MYEOJ +BA87;BA87;1106 1167 11BE;BA87;1106 1167 11BE; # (몇; 몇; 몇; 몇; 몇; ) HANGUL SYLLABLE MYEOC +BA88;BA88;1106 1167 11BF;BA88;1106 1167 11BF; # (몈; 몈; 몈; 몈; 몈; ) HANGUL SYLLABLE MYEOK +BA89;BA89;1106 1167 11C0;BA89;1106 1167 11C0; # (몉; 몉; 몉; 몉; 몉; ) HANGUL SYLLABLE MYEOT +BA8A;BA8A;1106 1167 11C1;BA8A;1106 1167 11C1; # (몊; 몊; 몊; 몊; 몊; ) HANGUL SYLLABLE MYEOP +BA8B;BA8B;1106 1167 11C2;BA8B;1106 1167 11C2; # (몋; 몋; 몋; 몋; 몋; ) HANGUL SYLLABLE MYEOH +BA8C;BA8C;1106 1168;BA8C;1106 1168; # (몌; 몌; 몌; 몌; 몌; ) HANGUL SYLLABLE MYE +BA8D;BA8D;1106 1168 11A8;BA8D;1106 1168 11A8; # (몍; 몍; 몍; 몍; 몍; ) HANGUL SYLLABLE MYEG +BA8E;BA8E;1106 1168 11A9;BA8E;1106 1168 11A9; # (몎; 몎; 몎; 몎; 몎; ) HANGUL SYLLABLE MYEGG +BA8F;BA8F;1106 1168 11AA;BA8F;1106 1168 11AA; # (몏; 몏; 몏; 몏; 몏; ) HANGUL SYLLABLE MYEGS +BA90;BA90;1106 1168 11AB;BA90;1106 1168 11AB; # (몐; 몐; 몐; 몐; 몐; ) HANGUL SYLLABLE MYEN +BA91;BA91;1106 1168 11AC;BA91;1106 1168 11AC; # (몑; 몑; 몑; 몑; 몑; ) HANGUL SYLLABLE MYENJ +BA92;BA92;1106 1168 11AD;BA92;1106 1168 11AD; # (몒; 몒; 몒; 몒; 몒; ) HANGUL SYLLABLE MYENH +BA93;BA93;1106 1168 11AE;BA93;1106 1168 11AE; # (몓; 몓; 몓; 몓; 몓; ) HANGUL SYLLABLE MYED +BA94;BA94;1106 1168 11AF;BA94;1106 1168 11AF; # (몔; 몔; 몔; 몔; 몔; ) HANGUL SYLLABLE MYEL +BA95;BA95;1106 1168 11B0;BA95;1106 1168 11B0; # (몕; 몕; 몕; 몕; 몕; ) HANGUL SYLLABLE MYELG +BA96;BA96;1106 1168 11B1;BA96;1106 1168 11B1; # (몖; 몖; 몖; 몖; 몖; ) HANGUL SYLLABLE MYELM +BA97;BA97;1106 1168 11B2;BA97;1106 1168 11B2; # (몗; 몗; 몗; 몗; 몗; ) HANGUL SYLLABLE MYELB +BA98;BA98;1106 1168 11B3;BA98;1106 1168 11B3; # (몘; 몘; 몘; 몘; 몘; ) HANGUL SYLLABLE MYELS +BA99;BA99;1106 1168 11B4;BA99;1106 1168 11B4; # (몙; 몙; 몙; 몙; 몙; ) HANGUL SYLLABLE MYELT +BA9A;BA9A;1106 1168 11B5;BA9A;1106 1168 11B5; # (몚; 몚; 몚; 몚; 몚; ) HANGUL SYLLABLE MYELP +BA9B;BA9B;1106 1168 11B6;BA9B;1106 1168 11B6; # (몛; 몛; 몛; 몛; 몛; ) HANGUL SYLLABLE MYELH +BA9C;BA9C;1106 1168 11B7;BA9C;1106 1168 11B7; # (몜; 몜; 몜; 몜; 몜; ) HANGUL SYLLABLE MYEM +BA9D;BA9D;1106 1168 11B8;BA9D;1106 1168 11B8; # (몝; 몝; 몝; 몝; 몝; ) HANGUL SYLLABLE MYEB +BA9E;BA9E;1106 1168 11B9;BA9E;1106 1168 11B9; # (몞; 몞; 몞; 몞; 몞; ) HANGUL SYLLABLE MYEBS +BA9F;BA9F;1106 1168 11BA;BA9F;1106 1168 11BA; # (몟; 몟; 몟; 몟; 몟; ) HANGUL SYLLABLE MYES +BAA0;BAA0;1106 1168 11BB;BAA0;1106 1168 11BB; # (몠; 몠; 몠; 몠; 몠; ) HANGUL SYLLABLE MYESS +BAA1;BAA1;1106 1168 11BC;BAA1;1106 1168 11BC; # (몡; 몡; 몡; 몡; 몡; ) HANGUL SYLLABLE MYENG +BAA2;BAA2;1106 1168 11BD;BAA2;1106 1168 11BD; # (몢; 몢; 몢; 몢; 몢; ) HANGUL SYLLABLE MYEJ +BAA3;BAA3;1106 1168 11BE;BAA3;1106 1168 11BE; # (몣; 몣; 몣; 몣; 몣; ) HANGUL SYLLABLE MYEC +BAA4;BAA4;1106 1168 11BF;BAA4;1106 1168 11BF; # (몤; 몤; 몤; 몤; 몤; ) HANGUL SYLLABLE MYEK +BAA5;BAA5;1106 1168 11C0;BAA5;1106 1168 11C0; # (몥; 몥; 몥; 몥; 몥; ) HANGUL SYLLABLE MYET +BAA6;BAA6;1106 1168 11C1;BAA6;1106 1168 11C1; # (몦; 몦; 몦; 몦; 몦; ) HANGUL SYLLABLE MYEP +BAA7;BAA7;1106 1168 11C2;BAA7;1106 1168 11C2; # (몧; 몧; 몧; 몧; 몧; ) HANGUL SYLLABLE MYEH +BAA8;BAA8;1106 1169;BAA8;1106 1169; # (모; 모; 모; 모; 모; ) HANGUL SYLLABLE MO +BAA9;BAA9;1106 1169 11A8;BAA9;1106 1169 11A8; # (목; 목; 목; 목; 목; ) HANGUL SYLLABLE MOG +BAAA;BAAA;1106 1169 11A9;BAAA;1106 1169 11A9; # (몪; 몪; 몪; 몪; 몪; ) HANGUL SYLLABLE MOGG +BAAB;BAAB;1106 1169 11AA;BAAB;1106 1169 11AA; # (몫; 몫; 몫; 몫; 몫; ) HANGUL SYLLABLE MOGS +BAAC;BAAC;1106 1169 11AB;BAAC;1106 1169 11AB; # (몬; 몬; 몬; 몬; 몬; ) HANGUL SYLLABLE MON +BAAD;BAAD;1106 1169 11AC;BAAD;1106 1169 11AC; # (몭; 몭; 몭; 몭; 몭; ) HANGUL SYLLABLE MONJ +BAAE;BAAE;1106 1169 11AD;BAAE;1106 1169 11AD; # (몮; 몮; 몮; 몮; 몮; ) HANGUL SYLLABLE MONH +BAAF;BAAF;1106 1169 11AE;BAAF;1106 1169 11AE; # (몯; 몯; 몯; 몯; 몯; ) HANGUL SYLLABLE MOD +BAB0;BAB0;1106 1169 11AF;BAB0;1106 1169 11AF; # (몰; 몰; 몰; 몰; 몰; ) HANGUL SYLLABLE MOL +BAB1;BAB1;1106 1169 11B0;BAB1;1106 1169 11B0; # (몱; 몱; 몱; 몱; 몱; ) HANGUL SYLLABLE MOLG +BAB2;BAB2;1106 1169 11B1;BAB2;1106 1169 11B1; # (몲; 몲; 몲; 몲; 몲; ) HANGUL SYLLABLE MOLM +BAB3;BAB3;1106 1169 11B2;BAB3;1106 1169 11B2; # (몳; 몳; 몳; 몳; 몳; ) HANGUL SYLLABLE MOLB +BAB4;BAB4;1106 1169 11B3;BAB4;1106 1169 11B3; # (몴; 몴; 몴; 몴; 몴; ) HANGUL SYLLABLE MOLS +BAB5;BAB5;1106 1169 11B4;BAB5;1106 1169 11B4; # (몵; 몵; 몵; 몵; 몵; ) HANGUL SYLLABLE MOLT +BAB6;BAB6;1106 1169 11B5;BAB6;1106 1169 11B5; # (몶; 몶; 몶; 몶; 몶; ) HANGUL SYLLABLE MOLP +BAB7;BAB7;1106 1169 11B6;BAB7;1106 1169 11B6; # (몷; 몷; 몷; 몷; 몷; ) HANGUL SYLLABLE MOLH +BAB8;BAB8;1106 1169 11B7;BAB8;1106 1169 11B7; # (몸; 몸; 몸; 몸; 몸; ) HANGUL SYLLABLE MOM +BAB9;BAB9;1106 1169 11B8;BAB9;1106 1169 11B8; # (몹; 몹; 몹; 몹; 몹; ) HANGUL SYLLABLE MOB +BABA;BABA;1106 1169 11B9;BABA;1106 1169 11B9; # (몺; 몺; 몺; 몺; 몺; ) HANGUL SYLLABLE MOBS +BABB;BABB;1106 1169 11BA;BABB;1106 1169 11BA; # (못; 못; 못; 못; 못; ) HANGUL SYLLABLE MOS +BABC;BABC;1106 1169 11BB;BABC;1106 1169 11BB; # (몼; 몼; 몼; 몼; 몼; ) HANGUL SYLLABLE MOSS +BABD;BABD;1106 1169 11BC;BABD;1106 1169 11BC; # (몽; 몽; 몽; 몽; 몽; ) HANGUL SYLLABLE MONG +BABE;BABE;1106 1169 11BD;BABE;1106 1169 11BD; # (몾; 몾; 몾; 몾; 몾; ) HANGUL SYLLABLE MOJ +BABF;BABF;1106 1169 11BE;BABF;1106 1169 11BE; # (몿; 몿; 몿; 몿; 몿; ) HANGUL SYLLABLE MOC +BAC0;BAC0;1106 1169 11BF;BAC0;1106 1169 11BF; # (뫀; 뫀; 뫀; 뫀; 뫀; ) HANGUL SYLLABLE MOK +BAC1;BAC1;1106 1169 11C0;BAC1;1106 1169 11C0; # (뫁; 뫁; 뫁; 뫁; 뫁; ) HANGUL SYLLABLE MOT +BAC2;BAC2;1106 1169 11C1;BAC2;1106 1169 11C1; # (뫂; 뫂; 뫂; 뫂; 뫂; ) HANGUL SYLLABLE MOP +BAC3;BAC3;1106 1169 11C2;BAC3;1106 1169 11C2; # (뫃; 뫃; 뫃; 뫃; 뫃; ) HANGUL SYLLABLE MOH +BAC4;BAC4;1106 116A;BAC4;1106 116A; # (뫄; 뫄; 뫄; 뫄; 뫄; ) HANGUL SYLLABLE MWA +BAC5;BAC5;1106 116A 11A8;BAC5;1106 116A 11A8; # (뫅; 뫅; 뫅; 뫅; 뫅; ) HANGUL SYLLABLE MWAG +BAC6;BAC6;1106 116A 11A9;BAC6;1106 116A 11A9; # (뫆; 뫆; 뫆; 뫆; 뫆; ) HANGUL SYLLABLE MWAGG +BAC7;BAC7;1106 116A 11AA;BAC7;1106 116A 11AA; # (뫇; 뫇; 뫇; 뫇; 뫇; ) HANGUL SYLLABLE MWAGS +BAC8;BAC8;1106 116A 11AB;BAC8;1106 116A 11AB; # (뫈; 뫈; 뫈; 뫈; 뫈; ) HANGUL SYLLABLE MWAN +BAC9;BAC9;1106 116A 11AC;BAC9;1106 116A 11AC; # (뫉; 뫉; 뫉; 뫉; 뫉; ) HANGUL SYLLABLE MWANJ +BACA;BACA;1106 116A 11AD;BACA;1106 116A 11AD; # (뫊; 뫊; 뫊; 뫊; 뫊; ) HANGUL SYLLABLE MWANH +BACB;BACB;1106 116A 11AE;BACB;1106 116A 11AE; # (뫋; 뫋; 뫋; 뫋; 뫋; ) HANGUL SYLLABLE MWAD +BACC;BACC;1106 116A 11AF;BACC;1106 116A 11AF; # (뫌; 뫌; 뫌; 뫌; 뫌; ) HANGUL SYLLABLE MWAL +BACD;BACD;1106 116A 11B0;BACD;1106 116A 11B0; # (뫍; 뫍; 뫍; 뫍; 뫍; ) HANGUL SYLLABLE MWALG +BACE;BACE;1106 116A 11B1;BACE;1106 116A 11B1; # (뫎; 뫎; 뫎; 뫎; 뫎; ) HANGUL SYLLABLE MWALM +BACF;BACF;1106 116A 11B2;BACF;1106 116A 11B2; # (뫏; 뫏; 뫏; 뫏; 뫏; ) HANGUL SYLLABLE MWALB +BAD0;BAD0;1106 116A 11B3;BAD0;1106 116A 11B3; # (뫐; 뫐; 뫐; 뫐; 뫐; ) HANGUL SYLLABLE MWALS +BAD1;BAD1;1106 116A 11B4;BAD1;1106 116A 11B4; # (뫑; 뫑; 뫑; 뫑; 뫑; ) HANGUL SYLLABLE MWALT +BAD2;BAD2;1106 116A 11B5;BAD2;1106 116A 11B5; # (뫒; 뫒; 뫒; 뫒; 뫒; ) HANGUL SYLLABLE MWALP +BAD3;BAD3;1106 116A 11B6;BAD3;1106 116A 11B6; # (뫓; 뫓; 뫓; 뫓; 뫓; ) HANGUL SYLLABLE MWALH +BAD4;BAD4;1106 116A 11B7;BAD4;1106 116A 11B7; # (뫔; 뫔; 뫔; 뫔; 뫔; ) HANGUL SYLLABLE MWAM +BAD5;BAD5;1106 116A 11B8;BAD5;1106 116A 11B8; # (뫕; 뫕; 뫕; 뫕; 뫕; ) HANGUL SYLLABLE MWAB +BAD6;BAD6;1106 116A 11B9;BAD6;1106 116A 11B9; # (뫖; 뫖; 뫖; 뫖; 뫖; ) HANGUL SYLLABLE MWABS +BAD7;BAD7;1106 116A 11BA;BAD7;1106 116A 11BA; # (뫗; 뫗; 뫗; 뫗; 뫗; ) HANGUL SYLLABLE MWAS +BAD8;BAD8;1106 116A 11BB;BAD8;1106 116A 11BB; # (뫘; 뫘; 뫘; 뫘; 뫘; ) HANGUL SYLLABLE MWASS +BAD9;BAD9;1106 116A 11BC;BAD9;1106 116A 11BC; # (뫙; 뫙; 뫙; 뫙; 뫙; ) HANGUL SYLLABLE MWANG +BADA;BADA;1106 116A 11BD;BADA;1106 116A 11BD; # (뫚; 뫚; 뫚; 뫚; 뫚; ) HANGUL SYLLABLE MWAJ +BADB;BADB;1106 116A 11BE;BADB;1106 116A 11BE; # (뫛; 뫛; 뫛; 뫛; 뫛; ) HANGUL SYLLABLE MWAC +BADC;BADC;1106 116A 11BF;BADC;1106 116A 11BF; # (뫜; 뫜; 뫜; 뫜; 뫜; ) HANGUL SYLLABLE MWAK +BADD;BADD;1106 116A 11C0;BADD;1106 116A 11C0; # (뫝; 뫝; 뫝; 뫝; 뫝; ) HANGUL SYLLABLE MWAT +BADE;BADE;1106 116A 11C1;BADE;1106 116A 11C1; # (뫞; 뫞; 뫞; 뫞; 뫞; ) HANGUL SYLLABLE MWAP +BADF;BADF;1106 116A 11C2;BADF;1106 116A 11C2; # (뫟; 뫟; 뫟; 뫟; 뫟; ) HANGUL SYLLABLE MWAH +BAE0;BAE0;1106 116B;BAE0;1106 116B; # (뫠; 뫠; 뫠; 뫠; 뫠; ) HANGUL SYLLABLE MWAE +BAE1;BAE1;1106 116B 11A8;BAE1;1106 116B 11A8; # (뫡; 뫡; 뫡; 뫡; 뫡; ) HANGUL SYLLABLE MWAEG +BAE2;BAE2;1106 116B 11A9;BAE2;1106 116B 11A9; # (뫢; 뫢; 뫢; 뫢; 뫢; ) HANGUL SYLLABLE MWAEGG +BAE3;BAE3;1106 116B 11AA;BAE3;1106 116B 11AA; # (뫣; 뫣; 뫣; 뫣; 뫣; ) HANGUL SYLLABLE MWAEGS +BAE4;BAE4;1106 116B 11AB;BAE4;1106 116B 11AB; # (뫤; 뫤; 뫤; 뫤; 뫤; ) HANGUL SYLLABLE MWAEN +BAE5;BAE5;1106 116B 11AC;BAE5;1106 116B 11AC; # (뫥; 뫥; 뫥; 뫥; 뫥; ) HANGUL SYLLABLE MWAENJ +BAE6;BAE6;1106 116B 11AD;BAE6;1106 116B 11AD; # (뫦; 뫦; 뫦; 뫦; 뫦; ) HANGUL SYLLABLE MWAENH +BAE7;BAE7;1106 116B 11AE;BAE7;1106 116B 11AE; # (뫧; 뫧; 뫧; 뫧; 뫧; ) HANGUL SYLLABLE MWAED +BAE8;BAE8;1106 116B 11AF;BAE8;1106 116B 11AF; # (뫨; 뫨; 뫨; 뫨; 뫨; ) HANGUL SYLLABLE MWAEL +BAE9;BAE9;1106 116B 11B0;BAE9;1106 116B 11B0; # (뫩; 뫩; 뫩; 뫩; 뫩; ) HANGUL SYLLABLE MWAELG +BAEA;BAEA;1106 116B 11B1;BAEA;1106 116B 11B1; # (뫪; 뫪; 뫪; 뫪; 뫪; ) HANGUL SYLLABLE MWAELM +BAEB;BAEB;1106 116B 11B2;BAEB;1106 116B 11B2; # (뫫; 뫫; 뫫; 뫫; 뫫; ) HANGUL SYLLABLE MWAELB +BAEC;BAEC;1106 116B 11B3;BAEC;1106 116B 11B3; # (뫬; 뫬; 뫬; 뫬; 뫬; ) HANGUL SYLLABLE MWAELS +BAED;BAED;1106 116B 11B4;BAED;1106 116B 11B4; # (뫭; 뫭; 뫭; 뫭; 뫭; ) HANGUL SYLLABLE MWAELT +BAEE;BAEE;1106 116B 11B5;BAEE;1106 116B 11B5; # (뫮; 뫮; 뫮; 뫮; 뫮; ) HANGUL SYLLABLE MWAELP +BAEF;BAEF;1106 116B 11B6;BAEF;1106 116B 11B6; # (뫯; 뫯; 뫯; 뫯; 뫯; ) HANGUL SYLLABLE MWAELH +BAF0;BAF0;1106 116B 11B7;BAF0;1106 116B 11B7; # (뫰; 뫰; 뫰; 뫰; 뫰; ) HANGUL SYLLABLE MWAEM +BAF1;BAF1;1106 116B 11B8;BAF1;1106 116B 11B8; # (뫱; 뫱; 뫱; 뫱; 뫱; ) HANGUL SYLLABLE MWAEB +BAF2;BAF2;1106 116B 11B9;BAF2;1106 116B 11B9; # (뫲; 뫲; 뫲; 뫲; 뫲; ) HANGUL SYLLABLE MWAEBS +BAF3;BAF3;1106 116B 11BA;BAF3;1106 116B 11BA; # (뫳; 뫳; 뫳; 뫳; 뫳; ) HANGUL SYLLABLE MWAES +BAF4;BAF4;1106 116B 11BB;BAF4;1106 116B 11BB; # (뫴; 뫴; 뫴; 뫴; 뫴; ) HANGUL SYLLABLE MWAESS +BAF5;BAF5;1106 116B 11BC;BAF5;1106 116B 11BC; # (뫵; 뫵; 뫵; 뫵; 뫵; ) HANGUL SYLLABLE MWAENG +BAF6;BAF6;1106 116B 11BD;BAF6;1106 116B 11BD; # (뫶; 뫶; 뫶; 뫶; 뫶; ) HANGUL SYLLABLE MWAEJ +BAF7;BAF7;1106 116B 11BE;BAF7;1106 116B 11BE; # (뫷; 뫷; 뫷; 뫷; 뫷; ) HANGUL SYLLABLE MWAEC +BAF8;BAF8;1106 116B 11BF;BAF8;1106 116B 11BF; # (뫸; 뫸; 뫸; 뫸; 뫸; ) HANGUL SYLLABLE MWAEK +BAF9;BAF9;1106 116B 11C0;BAF9;1106 116B 11C0; # (뫹; 뫹; 뫹; 뫹; 뫹; ) HANGUL SYLLABLE MWAET +BAFA;BAFA;1106 116B 11C1;BAFA;1106 116B 11C1; # (뫺; 뫺; 뫺; 뫺; 뫺; ) HANGUL SYLLABLE MWAEP +BAFB;BAFB;1106 116B 11C2;BAFB;1106 116B 11C2; # (뫻; 뫻; 뫻; 뫻; 뫻; ) HANGUL SYLLABLE MWAEH +BAFC;BAFC;1106 116C;BAFC;1106 116C; # (뫼; 뫼; 뫼; 뫼; 뫼; ) HANGUL SYLLABLE MOE +BAFD;BAFD;1106 116C 11A8;BAFD;1106 116C 11A8; # (뫽; 뫽; 뫽; 뫽; 뫽; ) HANGUL SYLLABLE MOEG +BAFE;BAFE;1106 116C 11A9;BAFE;1106 116C 11A9; # (뫾; 뫾; 뫾; 뫾; 뫾; ) HANGUL SYLLABLE MOEGG +BAFF;BAFF;1106 116C 11AA;BAFF;1106 116C 11AA; # (뫿; 뫿; 뫿; 뫿; 뫿; ) HANGUL SYLLABLE MOEGS +BB00;BB00;1106 116C 11AB;BB00;1106 116C 11AB; # (묀; 묀; 묀; 묀; 묀; ) HANGUL SYLLABLE MOEN +BB01;BB01;1106 116C 11AC;BB01;1106 116C 11AC; # (묁; 묁; 묁; 묁; 묁; ) HANGUL SYLLABLE MOENJ +BB02;BB02;1106 116C 11AD;BB02;1106 116C 11AD; # (묂; 묂; 묂; 묂; 묂; ) HANGUL SYLLABLE MOENH +BB03;BB03;1106 116C 11AE;BB03;1106 116C 11AE; # (묃; 묃; 묃; 묃; 묃; ) HANGUL SYLLABLE MOED +BB04;BB04;1106 116C 11AF;BB04;1106 116C 11AF; # (묄; 묄; 묄; 묄; 묄; ) HANGUL SYLLABLE MOEL +BB05;BB05;1106 116C 11B0;BB05;1106 116C 11B0; # (묅; 묅; 묅; 묅; 묅; ) HANGUL SYLLABLE MOELG +BB06;BB06;1106 116C 11B1;BB06;1106 116C 11B1; # (묆; 묆; 묆; 묆; 묆; ) HANGUL SYLLABLE MOELM +BB07;BB07;1106 116C 11B2;BB07;1106 116C 11B2; # (묇; 묇; 묇; 묇; 묇; ) HANGUL SYLLABLE MOELB +BB08;BB08;1106 116C 11B3;BB08;1106 116C 11B3; # (묈; 묈; 묈; 묈; 묈; ) HANGUL SYLLABLE MOELS +BB09;BB09;1106 116C 11B4;BB09;1106 116C 11B4; # (묉; 묉; 묉; 묉; 묉; ) HANGUL SYLLABLE MOELT +BB0A;BB0A;1106 116C 11B5;BB0A;1106 116C 11B5; # (묊; 묊; 묊; 묊; 묊; ) HANGUL SYLLABLE MOELP +BB0B;BB0B;1106 116C 11B6;BB0B;1106 116C 11B6; # (묋; 묋; 묋; 묋; 묋; ) HANGUL SYLLABLE MOELH +BB0C;BB0C;1106 116C 11B7;BB0C;1106 116C 11B7; # (묌; 묌; 묌; 묌; 묌; ) HANGUL SYLLABLE MOEM +BB0D;BB0D;1106 116C 11B8;BB0D;1106 116C 11B8; # (묍; 묍; 묍; 묍; 묍; ) HANGUL SYLLABLE MOEB +BB0E;BB0E;1106 116C 11B9;BB0E;1106 116C 11B9; # (묎; 묎; 묎; 묎; 묎; ) HANGUL SYLLABLE MOEBS +BB0F;BB0F;1106 116C 11BA;BB0F;1106 116C 11BA; # (묏; 묏; 묏; 묏; 묏; ) HANGUL SYLLABLE MOES +BB10;BB10;1106 116C 11BB;BB10;1106 116C 11BB; # (묐; 묐; 묐; 묐; 묐; ) HANGUL SYLLABLE MOESS +BB11;BB11;1106 116C 11BC;BB11;1106 116C 11BC; # (묑; 묑; 묑; 묑; 묑; ) HANGUL SYLLABLE MOENG +BB12;BB12;1106 116C 11BD;BB12;1106 116C 11BD; # (묒; 묒; 묒; 묒; 묒; ) HANGUL SYLLABLE MOEJ +BB13;BB13;1106 116C 11BE;BB13;1106 116C 11BE; # (묓; 묓; 묓; 묓; 묓; ) HANGUL SYLLABLE MOEC +BB14;BB14;1106 116C 11BF;BB14;1106 116C 11BF; # (묔; 묔; 묔; 묔; 묔; ) HANGUL SYLLABLE MOEK +BB15;BB15;1106 116C 11C0;BB15;1106 116C 11C0; # (묕; 묕; 묕; 묕; 묕; ) HANGUL SYLLABLE MOET +BB16;BB16;1106 116C 11C1;BB16;1106 116C 11C1; # (묖; 묖; 묖; 묖; 묖; ) HANGUL SYLLABLE MOEP +BB17;BB17;1106 116C 11C2;BB17;1106 116C 11C2; # (묗; 묗; 묗; 묗; 묗; ) HANGUL SYLLABLE MOEH +BB18;BB18;1106 116D;BB18;1106 116D; # (묘; 묘; 묘; 묘; 묘; ) HANGUL SYLLABLE MYO +BB19;BB19;1106 116D 11A8;BB19;1106 116D 11A8; # (묙; 묙; 묙; 묙; 묙; ) HANGUL SYLLABLE MYOG +BB1A;BB1A;1106 116D 11A9;BB1A;1106 116D 11A9; # (묚; 묚; 묚; 묚; 묚; ) HANGUL SYLLABLE MYOGG +BB1B;BB1B;1106 116D 11AA;BB1B;1106 116D 11AA; # (묛; 묛; 묛; 묛; 묛; ) HANGUL SYLLABLE MYOGS +BB1C;BB1C;1106 116D 11AB;BB1C;1106 116D 11AB; # (묜; 묜; 묜; 묜; 묜; ) HANGUL SYLLABLE MYON +BB1D;BB1D;1106 116D 11AC;BB1D;1106 116D 11AC; # (묝; 묝; 묝; 묝; 묝; ) HANGUL SYLLABLE MYONJ +BB1E;BB1E;1106 116D 11AD;BB1E;1106 116D 11AD; # (묞; 묞; 묞; 묞; 묞; ) HANGUL SYLLABLE MYONH +BB1F;BB1F;1106 116D 11AE;BB1F;1106 116D 11AE; # (묟; 묟; 묟; 묟; 묟; ) HANGUL SYLLABLE MYOD +BB20;BB20;1106 116D 11AF;BB20;1106 116D 11AF; # (묠; 묠; 묠; 묠; 묠; ) HANGUL SYLLABLE MYOL +BB21;BB21;1106 116D 11B0;BB21;1106 116D 11B0; # (묡; 묡; 묡; 묡; 묡; ) HANGUL SYLLABLE MYOLG +BB22;BB22;1106 116D 11B1;BB22;1106 116D 11B1; # (묢; 묢; 묢; 묢; 묢; ) HANGUL SYLLABLE MYOLM +BB23;BB23;1106 116D 11B2;BB23;1106 116D 11B2; # (묣; 묣; 묣; 묣; 묣; ) HANGUL SYLLABLE MYOLB +BB24;BB24;1106 116D 11B3;BB24;1106 116D 11B3; # (묤; 묤; 묤; 묤; 묤; ) HANGUL SYLLABLE MYOLS +BB25;BB25;1106 116D 11B4;BB25;1106 116D 11B4; # (묥; 묥; 묥; 묥; 묥; ) HANGUL SYLLABLE MYOLT +BB26;BB26;1106 116D 11B5;BB26;1106 116D 11B5; # (묦; 묦; 묦; 묦; 묦; ) HANGUL SYLLABLE MYOLP +BB27;BB27;1106 116D 11B6;BB27;1106 116D 11B6; # (묧; 묧; 묧; 묧; 묧; ) HANGUL SYLLABLE MYOLH +BB28;BB28;1106 116D 11B7;BB28;1106 116D 11B7; # (묨; 묨; 묨; 묨; 묨; ) HANGUL SYLLABLE MYOM +BB29;BB29;1106 116D 11B8;BB29;1106 116D 11B8; # (묩; 묩; 묩; 묩; 묩; ) HANGUL SYLLABLE MYOB +BB2A;BB2A;1106 116D 11B9;BB2A;1106 116D 11B9; # (묪; 묪; 묪; 묪; 묪; ) HANGUL SYLLABLE MYOBS +BB2B;BB2B;1106 116D 11BA;BB2B;1106 116D 11BA; # (묫; 묫; 묫; 묫; 묫; ) HANGUL SYLLABLE MYOS +BB2C;BB2C;1106 116D 11BB;BB2C;1106 116D 11BB; # (묬; 묬; 묬; 묬; 묬; ) HANGUL SYLLABLE MYOSS +BB2D;BB2D;1106 116D 11BC;BB2D;1106 116D 11BC; # (묭; 묭; 묭; 묭; 묭; ) HANGUL SYLLABLE MYONG +BB2E;BB2E;1106 116D 11BD;BB2E;1106 116D 11BD; # (묮; 묮; 묮; 묮; 묮; ) HANGUL SYLLABLE MYOJ +BB2F;BB2F;1106 116D 11BE;BB2F;1106 116D 11BE; # (묯; 묯; 묯; 묯; 묯; ) HANGUL SYLLABLE MYOC +BB30;BB30;1106 116D 11BF;BB30;1106 116D 11BF; # (묰; 묰; 묰; 묰; 묰; ) HANGUL SYLLABLE MYOK +BB31;BB31;1106 116D 11C0;BB31;1106 116D 11C0; # (묱; 묱; 묱; 묱; 묱; ) HANGUL SYLLABLE MYOT +BB32;BB32;1106 116D 11C1;BB32;1106 116D 11C1; # (묲; 묲; 묲; 묲; 묲; ) HANGUL SYLLABLE MYOP +BB33;BB33;1106 116D 11C2;BB33;1106 116D 11C2; # (묳; 묳; 묳; 묳; 묳; ) HANGUL SYLLABLE MYOH +BB34;BB34;1106 116E;BB34;1106 116E; # (무; 무; 무; 무; 무; ) HANGUL SYLLABLE MU +BB35;BB35;1106 116E 11A8;BB35;1106 116E 11A8; # (묵; 묵; 묵; 묵; 묵; ) HANGUL SYLLABLE MUG +BB36;BB36;1106 116E 11A9;BB36;1106 116E 11A9; # (묶; 묶; 묶; 묶; 묶; ) HANGUL SYLLABLE MUGG +BB37;BB37;1106 116E 11AA;BB37;1106 116E 11AA; # (묷; 묷; 묷; 묷; 묷; ) HANGUL SYLLABLE MUGS +BB38;BB38;1106 116E 11AB;BB38;1106 116E 11AB; # (문; 문; 문; 문; 문; ) HANGUL SYLLABLE MUN +BB39;BB39;1106 116E 11AC;BB39;1106 116E 11AC; # (묹; 묹; 묹; 묹; 묹; ) HANGUL SYLLABLE MUNJ +BB3A;BB3A;1106 116E 11AD;BB3A;1106 116E 11AD; # (묺; 묺; 묺; 묺; 묺; ) HANGUL SYLLABLE MUNH +BB3B;BB3B;1106 116E 11AE;BB3B;1106 116E 11AE; # (묻; 묻; 묻; 묻; 묻; ) HANGUL SYLLABLE MUD +BB3C;BB3C;1106 116E 11AF;BB3C;1106 116E 11AF; # (물; 물; 물; 물; 물; ) HANGUL SYLLABLE MUL +BB3D;BB3D;1106 116E 11B0;BB3D;1106 116E 11B0; # (묽; 묽; 묽; 묽; 묽; ) HANGUL SYLLABLE MULG +BB3E;BB3E;1106 116E 11B1;BB3E;1106 116E 11B1; # (묾; 묾; 묾; 묾; 묾; ) HANGUL SYLLABLE MULM +BB3F;BB3F;1106 116E 11B2;BB3F;1106 116E 11B2; # (묿; 묿; 묿; 묿; 묿; ) HANGUL SYLLABLE MULB +BB40;BB40;1106 116E 11B3;BB40;1106 116E 11B3; # (뭀; 뭀; 뭀; 뭀; 뭀; ) HANGUL SYLLABLE MULS +BB41;BB41;1106 116E 11B4;BB41;1106 116E 11B4; # (뭁; 뭁; 뭁; 뭁; 뭁; ) HANGUL SYLLABLE MULT +BB42;BB42;1106 116E 11B5;BB42;1106 116E 11B5; # (뭂; 뭂; 뭂; 뭂; 뭂; ) HANGUL SYLLABLE MULP +BB43;BB43;1106 116E 11B6;BB43;1106 116E 11B6; # (뭃; 뭃; 뭃; 뭃; 뭃; ) HANGUL SYLLABLE MULH +BB44;BB44;1106 116E 11B7;BB44;1106 116E 11B7; # (뭄; 뭄; 뭄; 뭄; 뭄; ) HANGUL SYLLABLE MUM +BB45;BB45;1106 116E 11B8;BB45;1106 116E 11B8; # (뭅; 뭅; 뭅; 뭅; 뭅; ) HANGUL SYLLABLE MUB +BB46;BB46;1106 116E 11B9;BB46;1106 116E 11B9; # (뭆; 뭆; 뭆; 뭆; 뭆; ) HANGUL SYLLABLE MUBS +BB47;BB47;1106 116E 11BA;BB47;1106 116E 11BA; # (뭇; 뭇; 뭇; 뭇; 뭇; ) HANGUL SYLLABLE MUS +BB48;BB48;1106 116E 11BB;BB48;1106 116E 11BB; # (뭈; 뭈; 뭈; 뭈; 뭈; ) HANGUL SYLLABLE MUSS +BB49;BB49;1106 116E 11BC;BB49;1106 116E 11BC; # (뭉; 뭉; 뭉; 뭉; 뭉; ) HANGUL SYLLABLE MUNG +BB4A;BB4A;1106 116E 11BD;BB4A;1106 116E 11BD; # (뭊; 뭊; 뭊; 뭊; 뭊; ) HANGUL SYLLABLE MUJ +BB4B;BB4B;1106 116E 11BE;BB4B;1106 116E 11BE; # (뭋; 뭋; 뭋; 뭋; 뭋; ) HANGUL SYLLABLE MUC +BB4C;BB4C;1106 116E 11BF;BB4C;1106 116E 11BF; # (뭌; 뭌; 뭌; 뭌; 뭌; ) HANGUL SYLLABLE MUK +BB4D;BB4D;1106 116E 11C0;BB4D;1106 116E 11C0; # (뭍; 뭍; 뭍; 뭍; 뭍; ) HANGUL SYLLABLE MUT +BB4E;BB4E;1106 116E 11C1;BB4E;1106 116E 11C1; # (뭎; 뭎; 뭎; 뭎; 뭎; ) HANGUL SYLLABLE MUP +BB4F;BB4F;1106 116E 11C2;BB4F;1106 116E 11C2; # (뭏; 뭏; 뭏; 뭏; 뭏; ) HANGUL SYLLABLE MUH +BB50;BB50;1106 116F;BB50;1106 116F; # (뭐; 뭐; 뭐; 뭐; 뭐; ) HANGUL SYLLABLE MWEO +BB51;BB51;1106 116F 11A8;BB51;1106 116F 11A8; # (뭑; 뭑; 뭑; 뭑; 뭑; ) HANGUL SYLLABLE MWEOG +BB52;BB52;1106 116F 11A9;BB52;1106 116F 11A9; # (뭒; 뭒; 뭒; 뭒; 뭒; ) HANGUL SYLLABLE MWEOGG +BB53;BB53;1106 116F 11AA;BB53;1106 116F 11AA; # (뭓; 뭓; 뭓; 뭓; 뭓; ) HANGUL SYLLABLE MWEOGS +BB54;BB54;1106 116F 11AB;BB54;1106 116F 11AB; # (뭔; 뭔; 뭔; 뭔; 뭔; ) HANGUL SYLLABLE MWEON +BB55;BB55;1106 116F 11AC;BB55;1106 116F 11AC; # (뭕; 뭕; 뭕; 뭕; 뭕; ) HANGUL SYLLABLE MWEONJ +BB56;BB56;1106 116F 11AD;BB56;1106 116F 11AD; # (뭖; 뭖; 뭖; 뭖; 뭖; ) HANGUL SYLLABLE MWEONH +BB57;BB57;1106 116F 11AE;BB57;1106 116F 11AE; # (뭗; 뭗; 뭗; 뭗; 뭗; ) HANGUL SYLLABLE MWEOD +BB58;BB58;1106 116F 11AF;BB58;1106 116F 11AF; # (뭘; 뭘; 뭘; 뭘; 뭘; ) HANGUL SYLLABLE MWEOL +BB59;BB59;1106 116F 11B0;BB59;1106 116F 11B0; # (뭙; 뭙; 뭙; 뭙; 뭙; ) HANGUL SYLLABLE MWEOLG +BB5A;BB5A;1106 116F 11B1;BB5A;1106 116F 11B1; # (뭚; 뭚; 뭚; 뭚; 뭚; ) HANGUL SYLLABLE MWEOLM +BB5B;BB5B;1106 116F 11B2;BB5B;1106 116F 11B2; # (뭛; 뭛; 뭛; 뭛; 뭛; ) HANGUL SYLLABLE MWEOLB +BB5C;BB5C;1106 116F 11B3;BB5C;1106 116F 11B3; # (뭜; 뭜; 뭜; 뭜; 뭜; ) HANGUL SYLLABLE MWEOLS +BB5D;BB5D;1106 116F 11B4;BB5D;1106 116F 11B4; # (뭝; 뭝; 뭝; 뭝; 뭝; ) HANGUL SYLLABLE MWEOLT +BB5E;BB5E;1106 116F 11B5;BB5E;1106 116F 11B5; # (뭞; 뭞; 뭞; 뭞; 뭞; ) HANGUL SYLLABLE MWEOLP +BB5F;BB5F;1106 116F 11B6;BB5F;1106 116F 11B6; # (뭟; 뭟; 뭟; 뭟; 뭟; ) HANGUL SYLLABLE MWEOLH +BB60;BB60;1106 116F 11B7;BB60;1106 116F 11B7; # (뭠; 뭠; 뭠; 뭠; 뭠; ) HANGUL SYLLABLE MWEOM +BB61;BB61;1106 116F 11B8;BB61;1106 116F 11B8; # (뭡; 뭡; 뭡; 뭡; 뭡; ) HANGUL SYLLABLE MWEOB +BB62;BB62;1106 116F 11B9;BB62;1106 116F 11B9; # (뭢; 뭢; 뭢; 뭢; 뭢; ) HANGUL SYLLABLE MWEOBS +BB63;BB63;1106 116F 11BA;BB63;1106 116F 11BA; # (뭣; 뭣; 뭣; 뭣; 뭣; ) HANGUL SYLLABLE MWEOS +BB64;BB64;1106 116F 11BB;BB64;1106 116F 11BB; # (뭤; 뭤; 뭤; 뭤; 뭤; ) HANGUL SYLLABLE MWEOSS +BB65;BB65;1106 116F 11BC;BB65;1106 116F 11BC; # (뭥; 뭥; 뭥; 뭥; 뭥; ) HANGUL SYLLABLE MWEONG +BB66;BB66;1106 116F 11BD;BB66;1106 116F 11BD; # (뭦; 뭦; 뭦; 뭦; 뭦; ) HANGUL SYLLABLE MWEOJ +BB67;BB67;1106 116F 11BE;BB67;1106 116F 11BE; # (뭧; 뭧; 뭧; 뭧; 뭧; ) HANGUL SYLLABLE MWEOC +BB68;BB68;1106 116F 11BF;BB68;1106 116F 11BF; # (뭨; 뭨; 뭨; 뭨; 뭨; ) HANGUL SYLLABLE MWEOK +BB69;BB69;1106 116F 11C0;BB69;1106 116F 11C0; # (뭩; 뭩; 뭩; 뭩; 뭩; ) HANGUL SYLLABLE MWEOT +BB6A;BB6A;1106 116F 11C1;BB6A;1106 116F 11C1; # (뭪; 뭪; 뭪; 뭪; 뭪; ) HANGUL SYLLABLE MWEOP +BB6B;BB6B;1106 116F 11C2;BB6B;1106 116F 11C2; # (뭫; 뭫; 뭫; 뭫; 뭫; ) HANGUL SYLLABLE MWEOH +BB6C;BB6C;1106 1170;BB6C;1106 1170; # (뭬; 뭬; 뭬; 뭬; 뭬; ) HANGUL SYLLABLE MWE +BB6D;BB6D;1106 1170 11A8;BB6D;1106 1170 11A8; # (뭭; 뭭; 뭭; 뭭; 뭭; ) HANGUL SYLLABLE MWEG +BB6E;BB6E;1106 1170 11A9;BB6E;1106 1170 11A9; # (뭮; 뭮; 뭮; 뭮; 뭮; ) HANGUL SYLLABLE MWEGG +BB6F;BB6F;1106 1170 11AA;BB6F;1106 1170 11AA; # (뭯; 뭯; 뭯; 뭯; 뭯; ) HANGUL SYLLABLE MWEGS +BB70;BB70;1106 1170 11AB;BB70;1106 1170 11AB; # (뭰; 뭰; 뭰; 뭰; 뭰; ) HANGUL SYLLABLE MWEN +BB71;BB71;1106 1170 11AC;BB71;1106 1170 11AC; # (뭱; 뭱; 뭱; 뭱; 뭱; ) HANGUL SYLLABLE MWENJ +BB72;BB72;1106 1170 11AD;BB72;1106 1170 11AD; # (뭲; 뭲; 뭲; 뭲; 뭲; ) HANGUL SYLLABLE MWENH +BB73;BB73;1106 1170 11AE;BB73;1106 1170 11AE; # (뭳; 뭳; 뭳; 뭳; 뭳; ) HANGUL SYLLABLE MWED +BB74;BB74;1106 1170 11AF;BB74;1106 1170 11AF; # (뭴; 뭴; 뭴; 뭴; 뭴; ) HANGUL SYLLABLE MWEL +BB75;BB75;1106 1170 11B0;BB75;1106 1170 11B0; # (뭵; 뭵; 뭵; 뭵; 뭵; ) HANGUL SYLLABLE MWELG +BB76;BB76;1106 1170 11B1;BB76;1106 1170 11B1; # (뭶; 뭶; 뭶; 뭶; 뭶; ) HANGUL SYLLABLE MWELM +BB77;BB77;1106 1170 11B2;BB77;1106 1170 11B2; # (뭷; 뭷; 뭷; 뭷; 뭷; ) HANGUL SYLLABLE MWELB +BB78;BB78;1106 1170 11B3;BB78;1106 1170 11B3; # (뭸; 뭸; 뭸; 뭸; 뭸; ) HANGUL SYLLABLE MWELS +BB79;BB79;1106 1170 11B4;BB79;1106 1170 11B4; # (뭹; 뭹; 뭹; 뭹; 뭹; ) HANGUL SYLLABLE MWELT +BB7A;BB7A;1106 1170 11B5;BB7A;1106 1170 11B5; # (뭺; 뭺; 뭺; 뭺; 뭺; ) HANGUL SYLLABLE MWELP +BB7B;BB7B;1106 1170 11B6;BB7B;1106 1170 11B6; # (뭻; 뭻; 뭻; 뭻; 뭻; ) HANGUL SYLLABLE MWELH +BB7C;BB7C;1106 1170 11B7;BB7C;1106 1170 11B7; # (뭼; 뭼; 뭼; 뭼; 뭼; ) HANGUL SYLLABLE MWEM +BB7D;BB7D;1106 1170 11B8;BB7D;1106 1170 11B8; # (뭽; 뭽; 뭽; 뭽; 뭽; ) HANGUL SYLLABLE MWEB +BB7E;BB7E;1106 1170 11B9;BB7E;1106 1170 11B9; # (뭾; 뭾; 뭾; 뭾; 뭾; ) HANGUL SYLLABLE MWEBS +BB7F;BB7F;1106 1170 11BA;BB7F;1106 1170 11BA; # (뭿; 뭿; 뭿; 뭿; 뭿; ) HANGUL SYLLABLE MWES +BB80;BB80;1106 1170 11BB;BB80;1106 1170 11BB; # (뮀; 뮀; 뮀; 뮀; 뮀; ) HANGUL SYLLABLE MWESS +BB81;BB81;1106 1170 11BC;BB81;1106 1170 11BC; # (뮁; 뮁; 뮁; 뮁; 뮁; ) HANGUL SYLLABLE MWENG +BB82;BB82;1106 1170 11BD;BB82;1106 1170 11BD; # (뮂; 뮂; 뮂; 뮂; 뮂; ) HANGUL SYLLABLE MWEJ +BB83;BB83;1106 1170 11BE;BB83;1106 1170 11BE; # (뮃; 뮃; 뮃; 뮃; 뮃; ) HANGUL SYLLABLE MWEC +BB84;BB84;1106 1170 11BF;BB84;1106 1170 11BF; # (뮄; 뮄; 뮄; 뮄; 뮄; ) HANGUL SYLLABLE MWEK +BB85;BB85;1106 1170 11C0;BB85;1106 1170 11C0; # (뮅; 뮅; 뮅; 뮅; 뮅; ) HANGUL SYLLABLE MWET +BB86;BB86;1106 1170 11C1;BB86;1106 1170 11C1; # (뮆; 뮆; 뮆; 뮆; 뮆; ) HANGUL SYLLABLE MWEP +BB87;BB87;1106 1170 11C2;BB87;1106 1170 11C2; # (뮇; 뮇; 뮇; 뮇; 뮇; ) HANGUL SYLLABLE MWEH +BB88;BB88;1106 1171;BB88;1106 1171; # (뮈; 뮈; 뮈; 뮈; 뮈; ) HANGUL SYLLABLE MWI +BB89;BB89;1106 1171 11A8;BB89;1106 1171 11A8; # (뮉; 뮉; 뮉; 뮉; 뮉; ) HANGUL SYLLABLE MWIG +BB8A;BB8A;1106 1171 11A9;BB8A;1106 1171 11A9; # (뮊; 뮊; 뮊; 뮊; 뮊; ) HANGUL SYLLABLE MWIGG +BB8B;BB8B;1106 1171 11AA;BB8B;1106 1171 11AA; # (뮋; 뮋; 뮋; 뮋; 뮋; ) HANGUL SYLLABLE MWIGS +BB8C;BB8C;1106 1171 11AB;BB8C;1106 1171 11AB; # (뮌; 뮌; 뮌; 뮌; 뮌; ) HANGUL SYLLABLE MWIN +BB8D;BB8D;1106 1171 11AC;BB8D;1106 1171 11AC; # (뮍; 뮍; 뮍; 뮍; 뮍; ) HANGUL SYLLABLE MWINJ +BB8E;BB8E;1106 1171 11AD;BB8E;1106 1171 11AD; # (뮎; 뮎; 뮎; 뮎; 뮎; ) HANGUL SYLLABLE MWINH +BB8F;BB8F;1106 1171 11AE;BB8F;1106 1171 11AE; # (뮏; 뮏; 뮏; 뮏; 뮏; ) HANGUL SYLLABLE MWID +BB90;BB90;1106 1171 11AF;BB90;1106 1171 11AF; # (뮐; 뮐; 뮐; 뮐; 뮐; ) HANGUL SYLLABLE MWIL +BB91;BB91;1106 1171 11B0;BB91;1106 1171 11B0; # (뮑; 뮑; 뮑; 뮑; 뮑; ) HANGUL SYLLABLE MWILG +BB92;BB92;1106 1171 11B1;BB92;1106 1171 11B1; # (뮒; 뮒; 뮒; 뮒; 뮒; ) HANGUL SYLLABLE MWILM +BB93;BB93;1106 1171 11B2;BB93;1106 1171 11B2; # (뮓; 뮓; 뮓; 뮓; 뮓; ) HANGUL SYLLABLE MWILB +BB94;BB94;1106 1171 11B3;BB94;1106 1171 11B3; # (뮔; 뮔; 뮔; 뮔; 뮔; ) HANGUL SYLLABLE MWILS +BB95;BB95;1106 1171 11B4;BB95;1106 1171 11B4; # (뮕; 뮕; 뮕; 뮕; 뮕; ) HANGUL SYLLABLE MWILT +BB96;BB96;1106 1171 11B5;BB96;1106 1171 11B5; # (뮖; 뮖; 뮖; 뮖; 뮖; ) HANGUL SYLLABLE MWILP +BB97;BB97;1106 1171 11B6;BB97;1106 1171 11B6; # (뮗; 뮗; 뮗; 뮗; 뮗; ) HANGUL SYLLABLE MWILH +BB98;BB98;1106 1171 11B7;BB98;1106 1171 11B7; # (뮘; 뮘; 뮘; 뮘; 뮘; ) HANGUL SYLLABLE MWIM +BB99;BB99;1106 1171 11B8;BB99;1106 1171 11B8; # (뮙; 뮙; 뮙; 뮙; 뮙; ) HANGUL SYLLABLE MWIB +BB9A;BB9A;1106 1171 11B9;BB9A;1106 1171 11B9; # (뮚; 뮚; 뮚; 뮚; 뮚; ) HANGUL SYLLABLE MWIBS +BB9B;BB9B;1106 1171 11BA;BB9B;1106 1171 11BA; # (뮛; 뮛; 뮛; 뮛; 뮛; ) HANGUL SYLLABLE MWIS +BB9C;BB9C;1106 1171 11BB;BB9C;1106 1171 11BB; # (뮜; 뮜; 뮜; 뮜; 뮜; ) HANGUL SYLLABLE MWISS +BB9D;BB9D;1106 1171 11BC;BB9D;1106 1171 11BC; # (뮝; 뮝; 뮝; 뮝; 뮝; ) HANGUL SYLLABLE MWING +BB9E;BB9E;1106 1171 11BD;BB9E;1106 1171 11BD; # (뮞; 뮞; 뮞; 뮞; 뮞; ) HANGUL SYLLABLE MWIJ +BB9F;BB9F;1106 1171 11BE;BB9F;1106 1171 11BE; # (뮟; 뮟; 뮟; 뮟; 뮟; ) HANGUL SYLLABLE MWIC +BBA0;BBA0;1106 1171 11BF;BBA0;1106 1171 11BF; # (뮠; 뮠; 뮠; 뮠; 뮠; ) HANGUL SYLLABLE MWIK +BBA1;BBA1;1106 1171 11C0;BBA1;1106 1171 11C0; # (뮡; 뮡; 뮡; 뮡; 뮡; ) HANGUL SYLLABLE MWIT +BBA2;BBA2;1106 1171 11C1;BBA2;1106 1171 11C1; # (뮢; 뮢; 뮢; 뮢; 뮢; ) HANGUL SYLLABLE MWIP +BBA3;BBA3;1106 1171 11C2;BBA3;1106 1171 11C2; # (뮣; 뮣; 뮣; 뮣; 뮣; ) HANGUL SYLLABLE MWIH +BBA4;BBA4;1106 1172;BBA4;1106 1172; # (뮤; 뮤; 뮤; 뮤; 뮤; ) HANGUL SYLLABLE MYU +BBA5;BBA5;1106 1172 11A8;BBA5;1106 1172 11A8; # (뮥; 뮥; 뮥; 뮥; 뮥; ) HANGUL SYLLABLE MYUG +BBA6;BBA6;1106 1172 11A9;BBA6;1106 1172 11A9; # (뮦; 뮦; 뮦; 뮦; 뮦; ) HANGUL SYLLABLE MYUGG +BBA7;BBA7;1106 1172 11AA;BBA7;1106 1172 11AA; # (뮧; 뮧; 뮧; 뮧; 뮧; ) HANGUL SYLLABLE MYUGS +BBA8;BBA8;1106 1172 11AB;BBA8;1106 1172 11AB; # (뮨; 뮨; 뮨; 뮨; 뮨; ) HANGUL SYLLABLE MYUN +BBA9;BBA9;1106 1172 11AC;BBA9;1106 1172 11AC; # (뮩; 뮩; 뮩; 뮩; 뮩; ) HANGUL SYLLABLE MYUNJ +BBAA;BBAA;1106 1172 11AD;BBAA;1106 1172 11AD; # (뮪; 뮪; 뮪; 뮪; 뮪; ) HANGUL SYLLABLE MYUNH +BBAB;BBAB;1106 1172 11AE;BBAB;1106 1172 11AE; # (뮫; 뮫; 뮫; 뮫; 뮫; ) HANGUL SYLLABLE MYUD +BBAC;BBAC;1106 1172 11AF;BBAC;1106 1172 11AF; # (뮬; 뮬; 뮬; 뮬; 뮬; ) HANGUL SYLLABLE MYUL +BBAD;BBAD;1106 1172 11B0;BBAD;1106 1172 11B0; # (뮭; 뮭; 뮭; 뮭; 뮭; ) HANGUL SYLLABLE MYULG +BBAE;BBAE;1106 1172 11B1;BBAE;1106 1172 11B1; # (뮮; 뮮; 뮮; 뮮; 뮮; ) HANGUL SYLLABLE MYULM +BBAF;BBAF;1106 1172 11B2;BBAF;1106 1172 11B2; # (뮯; 뮯; 뮯; 뮯; 뮯; ) HANGUL SYLLABLE MYULB +BBB0;BBB0;1106 1172 11B3;BBB0;1106 1172 11B3; # (뮰; 뮰; 뮰; 뮰; 뮰; ) HANGUL SYLLABLE MYULS +BBB1;BBB1;1106 1172 11B4;BBB1;1106 1172 11B4; # (뮱; 뮱; 뮱; 뮱; 뮱; ) HANGUL SYLLABLE MYULT +BBB2;BBB2;1106 1172 11B5;BBB2;1106 1172 11B5; # (뮲; 뮲; 뮲; 뮲; 뮲; ) HANGUL SYLLABLE MYULP +BBB3;BBB3;1106 1172 11B6;BBB3;1106 1172 11B6; # (뮳; 뮳; 뮳; 뮳; 뮳; ) HANGUL SYLLABLE MYULH +BBB4;BBB4;1106 1172 11B7;BBB4;1106 1172 11B7; # (뮴; 뮴; 뮴; 뮴; 뮴; ) HANGUL SYLLABLE MYUM +BBB5;BBB5;1106 1172 11B8;BBB5;1106 1172 11B8; # (뮵; 뮵; 뮵; 뮵; 뮵; ) HANGUL SYLLABLE MYUB +BBB6;BBB6;1106 1172 11B9;BBB6;1106 1172 11B9; # (뮶; 뮶; 뮶; 뮶; 뮶; ) HANGUL SYLLABLE MYUBS +BBB7;BBB7;1106 1172 11BA;BBB7;1106 1172 11BA; # (뮷; 뮷; 뮷; 뮷; 뮷; ) HANGUL SYLLABLE MYUS +BBB8;BBB8;1106 1172 11BB;BBB8;1106 1172 11BB; # (뮸; 뮸; 뮸; 뮸; 뮸; ) HANGUL SYLLABLE MYUSS +BBB9;BBB9;1106 1172 11BC;BBB9;1106 1172 11BC; # (뮹; 뮹; 뮹; 뮹; 뮹; ) HANGUL SYLLABLE MYUNG +BBBA;BBBA;1106 1172 11BD;BBBA;1106 1172 11BD; # (뮺; 뮺; 뮺; 뮺; 뮺; ) HANGUL SYLLABLE MYUJ +BBBB;BBBB;1106 1172 11BE;BBBB;1106 1172 11BE; # (뮻; 뮻; 뮻; 뮻; 뮻; ) HANGUL SYLLABLE MYUC +BBBC;BBBC;1106 1172 11BF;BBBC;1106 1172 11BF; # (뮼; 뮼; 뮼; 뮼; 뮼; ) HANGUL SYLLABLE MYUK +BBBD;BBBD;1106 1172 11C0;BBBD;1106 1172 11C0; # (뮽; 뮽; 뮽; 뮽; 뮽; ) HANGUL SYLLABLE MYUT +BBBE;BBBE;1106 1172 11C1;BBBE;1106 1172 11C1; # (뮾; 뮾; 뮾; 뮾; 뮾; ) HANGUL SYLLABLE MYUP +BBBF;BBBF;1106 1172 11C2;BBBF;1106 1172 11C2; # (뮿; 뮿; 뮿; 뮿; 뮿; ) HANGUL SYLLABLE MYUH +BBC0;BBC0;1106 1173;BBC0;1106 1173; # (므; 므; 므; 므; 므; ) HANGUL SYLLABLE MEU +BBC1;BBC1;1106 1173 11A8;BBC1;1106 1173 11A8; # (믁; 믁; 믁; 믁; 믁; ) HANGUL SYLLABLE MEUG +BBC2;BBC2;1106 1173 11A9;BBC2;1106 1173 11A9; # (믂; 믂; 믂; 믂; 믂; ) HANGUL SYLLABLE MEUGG +BBC3;BBC3;1106 1173 11AA;BBC3;1106 1173 11AA; # (믃; 믃; 믃; 믃; 믃; ) HANGUL SYLLABLE MEUGS +BBC4;BBC4;1106 1173 11AB;BBC4;1106 1173 11AB; # (믄; 믄; 믄; 믄; 믄; ) HANGUL SYLLABLE MEUN +BBC5;BBC5;1106 1173 11AC;BBC5;1106 1173 11AC; # (믅; 믅; 믅; 믅; 믅; ) HANGUL SYLLABLE MEUNJ +BBC6;BBC6;1106 1173 11AD;BBC6;1106 1173 11AD; # (믆; 믆; 믆; 믆; 믆; ) HANGUL SYLLABLE MEUNH +BBC7;BBC7;1106 1173 11AE;BBC7;1106 1173 11AE; # (믇; 믇; 믇; 믇; 믇; ) HANGUL SYLLABLE MEUD +BBC8;BBC8;1106 1173 11AF;BBC8;1106 1173 11AF; # (믈; 믈; 믈; 믈; 믈; ) HANGUL SYLLABLE MEUL +BBC9;BBC9;1106 1173 11B0;BBC9;1106 1173 11B0; # (믉; 믉; 믉; 믉; 믉; ) HANGUL SYLLABLE MEULG +BBCA;BBCA;1106 1173 11B1;BBCA;1106 1173 11B1; # (믊; 믊; 믊; 믊; 믊; ) HANGUL SYLLABLE MEULM +BBCB;BBCB;1106 1173 11B2;BBCB;1106 1173 11B2; # (믋; 믋; 믋; 믋; 믋; ) HANGUL SYLLABLE MEULB +BBCC;BBCC;1106 1173 11B3;BBCC;1106 1173 11B3; # (믌; 믌; 믌; 믌; 믌; ) HANGUL SYLLABLE MEULS +BBCD;BBCD;1106 1173 11B4;BBCD;1106 1173 11B4; # (믍; 믍; 믍; 믍; 믍; ) HANGUL SYLLABLE MEULT +BBCE;BBCE;1106 1173 11B5;BBCE;1106 1173 11B5; # (믎; 믎; 믎; 믎; 믎; ) HANGUL SYLLABLE MEULP +BBCF;BBCF;1106 1173 11B6;BBCF;1106 1173 11B6; # (믏; 믏; 믏; 믏; 믏; ) HANGUL SYLLABLE MEULH +BBD0;BBD0;1106 1173 11B7;BBD0;1106 1173 11B7; # (믐; 믐; 믐; 믐; 믐; ) HANGUL SYLLABLE MEUM +BBD1;BBD1;1106 1173 11B8;BBD1;1106 1173 11B8; # (믑; 믑; 믑; 믑; 믑; ) HANGUL SYLLABLE MEUB +BBD2;BBD2;1106 1173 11B9;BBD2;1106 1173 11B9; # (믒; 믒; 믒; 믒; 믒; ) HANGUL SYLLABLE MEUBS +BBD3;BBD3;1106 1173 11BA;BBD3;1106 1173 11BA; # (믓; 믓; 믓; 믓; 믓; ) HANGUL SYLLABLE MEUS +BBD4;BBD4;1106 1173 11BB;BBD4;1106 1173 11BB; # (믔; 믔; 믔; 믔; 믔; ) HANGUL SYLLABLE MEUSS +BBD5;BBD5;1106 1173 11BC;BBD5;1106 1173 11BC; # (믕; 믕; 믕; 믕; 믕; ) HANGUL SYLLABLE MEUNG +BBD6;BBD6;1106 1173 11BD;BBD6;1106 1173 11BD; # (믖; 믖; 믖; 믖; 믖; ) HANGUL SYLLABLE MEUJ +BBD7;BBD7;1106 1173 11BE;BBD7;1106 1173 11BE; # (믗; 믗; 믗; 믗; 믗; ) HANGUL SYLLABLE MEUC +BBD8;BBD8;1106 1173 11BF;BBD8;1106 1173 11BF; # (믘; 믘; 믘; 믘; 믘; ) HANGUL SYLLABLE MEUK +BBD9;BBD9;1106 1173 11C0;BBD9;1106 1173 11C0; # (믙; 믙; 믙; 믙; 믙; ) HANGUL SYLLABLE MEUT +BBDA;BBDA;1106 1173 11C1;BBDA;1106 1173 11C1; # (믚; 믚; 믚; 믚; 믚; ) HANGUL SYLLABLE MEUP +BBDB;BBDB;1106 1173 11C2;BBDB;1106 1173 11C2; # (믛; 믛; 믛; 믛; 믛; ) HANGUL SYLLABLE MEUH +BBDC;BBDC;1106 1174;BBDC;1106 1174; # (믜; 믜; 믜; 믜; 믜; ) HANGUL SYLLABLE MYI +BBDD;BBDD;1106 1174 11A8;BBDD;1106 1174 11A8; # (믝; 믝; 믝; 믝; 믝; ) HANGUL SYLLABLE MYIG +BBDE;BBDE;1106 1174 11A9;BBDE;1106 1174 11A9; # (믞; 믞; 믞; 믞; 믞; ) HANGUL SYLLABLE MYIGG +BBDF;BBDF;1106 1174 11AA;BBDF;1106 1174 11AA; # (믟; 믟; 믟; 믟; 믟; ) HANGUL SYLLABLE MYIGS +BBE0;BBE0;1106 1174 11AB;BBE0;1106 1174 11AB; # (믠; 믠; 믠; 믠; 믠; ) HANGUL SYLLABLE MYIN +BBE1;BBE1;1106 1174 11AC;BBE1;1106 1174 11AC; # (믡; 믡; 믡; 믡; 믡; ) HANGUL SYLLABLE MYINJ +BBE2;BBE2;1106 1174 11AD;BBE2;1106 1174 11AD; # (믢; 믢; 믢; 믢; 믢; ) HANGUL SYLLABLE MYINH +BBE3;BBE3;1106 1174 11AE;BBE3;1106 1174 11AE; # (믣; 믣; 믣; 믣; 믣; ) HANGUL SYLLABLE MYID +BBE4;BBE4;1106 1174 11AF;BBE4;1106 1174 11AF; # (믤; 믤; 믤; 믤; 믤; ) HANGUL SYLLABLE MYIL +BBE5;BBE5;1106 1174 11B0;BBE5;1106 1174 11B0; # (믥; 믥; 믥; 믥; 믥; ) HANGUL SYLLABLE MYILG +BBE6;BBE6;1106 1174 11B1;BBE6;1106 1174 11B1; # (믦; 믦; 믦; 믦; 믦; ) HANGUL SYLLABLE MYILM +BBE7;BBE7;1106 1174 11B2;BBE7;1106 1174 11B2; # (믧; 믧; 믧; 믧; 믧; ) HANGUL SYLLABLE MYILB +BBE8;BBE8;1106 1174 11B3;BBE8;1106 1174 11B3; # (믨; 믨; 믨; 믨; 믨; ) HANGUL SYLLABLE MYILS +BBE9;BBE9;1106 1174 11B4;BBE9;1106 1174 11B4; # (믩; 믩; 믩; 믩; 믩; ) HANGUL SYLLABLE MYILT +BBEA;BBEA;1106 1174 11B5;BBEA;1106 1174 11B5; # (믪; 믪; 믪; 믪; 믪; ) HANGUL SYLLABLE MYILP +BBEB;BBEB;1106 1174 11B6;BBEB;1106 1174 11B6; # (믫; 믫; 믫; 믫; 믫; ) HANGUL SYLLABLE MYILH +BBEC;BBEC;1106 1174 11B7;BBEC;1106 1174 11B7; # (믬; 믬; 믬; 믬; 믬; ) HANGUL SYLLABLE MYIM +BBED;BBED;1106 1174 11B8;BBED;1106 1174 11B8; # (믭; 믭; 믭; 믭; 믭; ) HANGUL SYLLABLE MYIB +BBEE;BBEE;1106 1174 11B9;BBEE;1106 1174 11B9; # (믮; 믮; 믮; 믮; 믮; ) HANGUL SYLLABLE MYIBS +BBEF;BBEF;1106 1174 11BA;BBEF;1106 1174 11BA; # (믯; 믯; 믯; 믯; 믯; ) HANGUL SYLLABLE MYIS +BBF0;BBF0;1106 1174 11BB;BBF0;1106 1174 11BB; # (믰; 믰; 믰; 믰; 믰; ) HANGUL SYLLABLE MYISS +BBF1;BBF1;1106 1174 11BC;BBF1;1106 1174 11BC; # (믱; 믱; 믱; 믱; 믱; ) HANGUL SYLLABLE MYING +BBF2;BBF2;1106 1174 11BD;BBF2;1106 1174 11BD; # (믲; 믲; 믲; 믲; 믲; ) HANGUL SYLLABLE MYIJ +BBF3;BBF3;1106 1174 11BE;BBF3;1106 1174 11BE; # (믳; 믳; 믳; 믳; 믳; ) HANGUL SYLLABLE MYIC +BBF4;BBF4;1106 1174 11BF;BBF4;1106 1174 11BF; # (믴; 믴; 믴; 믴; 믴; ) HANGUL SYLLABLE MYIK +BBF5;BBF5;1106 1174 11C0;BBF5;1106 1174 11C0; # (믵; 믵; 믵; 믵; 믵; ) HANGUL SYLLABLE MYIT +BBF6;BBF6;1106 1174 11C1;BBF6;1106 1174 11C1; # (믶; 믶; 믶; 믶; 믶; ) HANGUL SYLLABLE MYIP +BBF7;BBF7;1106 1174 11C2;BBF7;1106 1174 11C2; # (믷; 믷; 믷; 믷; 믷; ) HANGUL SYLLABLE MYIH +BBF8;BBF8;1106 1175;BBF8;1106 1175; # (미; 미; 미; 미; 미; ) HANGUL SYLLABLE MI +BBF9;BBF9;1106 1175 11A8;BBF9;1106 1175 11A8; # (믹; 믹; 믹; 믹; 믹; ) HANGUL SYLLABLE MIG +BBFA;BBFA;1106 1175 11A9;BBFA;1106 1175 11A9; # (믺; 믺; 믺; 믺; 믺; ) HANGUL SYLLABLE MIGG +BBFB;BBFB;1106 1175 11AA;BBFB;1106 1175 11AA; # (믻; 믻; 믻; 믻; 믻; ) HANGUL SYLLABLE MIGS +BBFC;BBFC;1106 1175 11AB;BBFC;1106 1175 11AB; # (민; 민; 민; 민; 민; ) HANGUL SYLLABLE MIN +BBFD;BBFD;1106 1175 11AC;BBFD;1106 1175 11AC; # (믽; 믽; 믽; 믽; 믽; ) HANGUL SYLLABLE MINJ +BBFE;BBFE;1106 1175 11AD;BBFE;1106 1175 11AD; # (믾; 믾; 믾; 믾; 믾; ) HANGUL SYLLABLE MINH +BBFF;BBFF;1106 1175 11AE;BBFF;1106 1175 11AE; # (믿; 믿; 믿; 믿; 믿; ) HANGUL SYLLABLE MID +BC00;BC00;1106 1175 11AF;BC00;1106 1175 11AF; # (밀; 밀; 밀; 밀; 밀; ) HANGUL SYLLABLE MIL +BC01;BC01;1106 1175 11B0;BC01;1106 1175 11B0; # (밁; 밁; 밁; 밁; 밁; ) HANGUL SYLLABLE MILG +BC02;BC02;1106 1175 11B1;BC02;1106 1175 11B1; # (밂; 밂; 밂; 밂; 밂; ) HANGUL SYLLABLE MILM +BC03;BC03;1106 1175 11B2;BC03;1106 1175 11B2; # (밃; 밃; 밃; 밃; 밃; ) HANGUL SYLLABLE MILB +BC04;BC04;1106 1175 11B3;BC04;1106 1175 11B3; # (밄; 밄; 밄; 밄; 밄; ) HANGUL SYLLABLE MILS +BC05;BC05;1106 1175 11B4;BC05;1106 1175 11B4; # (밅; 밅; 밅; 밅; 밅; ) HANGUL SYLLABLE MILT +BC06;BC06;1106 1175 11B5;BC06;1106 1175 11B5; # (밆; 밆; 밆; 밆; 밆; ) HANGUL SYLLABLE MILP +BC07;BC07;1106 1175 11B6;BC07;1106 1175 11B6; # (밇; 밇; 밇; 밇; 밇; ) HANGUL SYLLABLE MILH +BC08;BC08;1106 1175 11B7;BC08;1106 1175 11B7; # (밈; 밈; 밈; 밈; 밈; ) HANGUL SYLLABLE MIM +BC09;BC09;1106 1175 11B8;BC09;1106 1175 11B8; # (밉; 밉; 밉; 밉; 밉; ) HANGUL SYLLABLE MIB +BC0A;BC0A;1106 1175 11B9;BC0A;1106 1175 11B9; # (밊; 밊; 밊; 밊; 밊; ) HANGUL SYLLABLE MIBS +BC0B;BC0B;1106 1175 11BA;BC0B;1106 1175 11BA; # (밋; 밋; 밋; 밋; 밋; ) HANGUL SYLLABLE MIS +BC0C;BC0C;1106 1175 11BB;BC0C;1106 1175 11BB; # (밌; 밌; 밌; 밌; 밌; ) HANGUL SYLLABLE MISS +BC0D;BC0D;1106 1175 11BC;BC0D;1106 1175 11BC; # (밍; 밍; 밍; 밍; 밍; ) HANGUL SYLLABLE MING +BC0E;BC0E;1106 1175 11BD;BC0E;1106 1175 11BD; # (밎; 밎; 밎; 밎; 밎; ) HANGUL SYLLABLE MIJ +BC0F;BC0F;1106 1175 11BE;BC0F;1106 1175 11BE; # (및; 및; 및; 및; 및; ) HANGUL SYLLABLE MIC +BC10;BC10;1106 1175 11BF;BC10;1106 1175 11BF; # (밐; 밐; 밐; 밐; 밐; ) HANGUL SYLLABLE MIK +BC11;BC11;1106 1175 11C0;BC11;1106 1175 11C0; # (밑; 밑; 밑; 밑; 밑; ) HANGUL SYLLABLE MIT +BC12;BC12;1106 1175 11C1;BC12;1106 1175 11C1; # (밒; 밒; 밒; 밒; 밒; ) HANGUL SYLLABLE MIP +BC13;BC13;1106 1175 11C2;BC13;1106 1175 11C2; # (밓; 밓; 밓; 밓; 밓; ) HANGUL SYLLABLE MIH +BC14;BC14;1107 1161;BC14;1107 1161; # (바; 바; 바; 바; 바; ) HANGUL SYLLABLE BA +BC15;BC15;1107 1161 11A8;BC15;1107 1161 11A8; # (박; 박; 박; 박; 박; ) HANGUL SYLLABLE BAG +BC16;BC16;1107 1161 11A9;BC16;1107 1161 11A9; # (밖; 밖; 밖; 밖; 밖; ) HANGUL SYLLABLE BAGG +BC17;BC17;1107 1161 11AA;BC17;1107 1161 11AA; # (밗; 밗; 밗; 밗; 밗; ) HANGUL SYLLABLE BAGS +BC18;BC18;1107 1161 11AB;BC18;1107 1161 11AB; # (반; 반; 반; 반; 반; ) HANGUL SYLLABLE BAN +BC19;BC19;1107 1161 11AC;BC19;1107 1161 11AC; # (밙; 밙; 밙; 밙; 밙; ) HANGUL SYLLABLE BANJ +BC1A;BC1A;1107 1161 11AD;BC1A;1107 1161 11AD; # (밚; 밚; 밚; 밚; 밚; ) HANGUL SYLLABLE BANH +BC1B;BC1B;1107 1161 11AE;BC1B;1107 1161 11AE; # (받; 받; 받; 받; 받; ) HANGUL SYLLABLE BAD +BC1C;BC1C;1107 1161 11AF;BC1C;1107 1161 11AF; # (발; 발; 발; 발; 발; ) HANGUL SYLLABLE BAL +BC1D;BC1D;1107 1161 11B0;BC1D;1107 1161 11B0; # (밝; 밝; 밝; 밝; 밝; ) HANGUL SYLLABLE BALG +BC1E;BC1E;1107 1161 11B1;BC1E;1107 1161 11B1; # (밞; 밞; 밞; 밞; 밞; ) HANGUL SYLLABLE BALM +BC1F;BC1F;1107 1161 11B2;BC1F;1107 1161 11B2; # (밟; 밟; 밟; 밟; 밟; ) HANGUL SYLLABLE BALB +BC20;BC20;1107 1161 11B3;BC20;1107 1161 11B3; # (밠; 밠; 밠; 밠; 밠; ) HANGUL SYLLABLE BALS +BC21;BC21;1107 1161 11B4;BC21;1107 1161 11B4; # (밡; 밡; 밡; 밡; 밡; ) HANGUL SYLLABLE BALT +BC22;BC22;1107 1161 11B5;BC22;1107 1161 11B5; # (밢; 밢; 밢; 밢; 밢; ) HANGUL SYLLABLE BALP +BC23;BC23;1107 1161 11B6;BC23;1107 1161 11B6; # (밣; 밣; 밣; 밣; 밣; ) HANGUL SYLLABLE BALH +BC24;BC24;1107 1161 11B7;BC24;1107 1161 11B7; # (밤; 밤; 밤; 밤; 밤; ) HANGUL SYLLABLE BAM +BC25;BC25;1107 1161 11B8;BC25;1107 1161 11B8; # (밥; 밥; 밥; 밥; 밥; ) HANGUL SYLLABLE BAB +BC26;BC26;1107 1161 11B9;BC26;1107 1161 11B9; # (밦; 밦; 밦; 밦; 밦; ) HANGUL SYLLABLE BABS +BC27;BC27;1107 1161 11BA;BC27;1107 1161 11BA; # (밧; 밧; 밧; 밧; 밧; ) HANGUL SYLLABLE BAS +BC28;BC28;1107 1161 11BB;BC28;1107 1161 11BB; # (밨; 밨; 밨; 밨; 밨; ) HANGUL SYLLABLE BASS +BC29;BC29;1107 1161 11BC;BC29;1107 1161 11BC; # (방; 방; 방; 방; 방; ) HANGUL SYLLABLE BANG +BC2A;BC2A;1107 1161 11BD;BC2A;1107 1161 11BD; # (밪; 밪; 밪; 밪; 밪; ) HANGUL SYLLABLE BAJ +BC2B;BC2B;1107 1161 11BE;BC2B;1107 1161 11BE; # (밫; 밫; 밫; 밫; 밫; ) HANGUL SYLLABLE BAC +BC2C;BC2C;1107 1161 11BF;BC2C;1107 1161 11BF; # (밬; 밬; 밬; 밬; 밬; ) HANGUL SYLLABLE BAK +BC2D;BC2D;1107 1161 11C0;BC2D;1107 1161 11C0; # (밭; 밭; 밭; 밭; 밭; ) HANGUL SYLLABLE BAT +BC2E;BC2E;1107 1161 11C1;BC2E;1107 1161 11C1; # (밮; 밮; 밮; 밮; 밮; ) HANGUL SYLLABLE BAP +BC2F;BC2F;1107 1161 11C2;BC2F;1107 1161 11C2; # (밯; 밯; 밯; 밯; 밯; ) HANGUL SYLLABLE BAH +BC30;BC30;1107 1162;BC30;1107 1162; # (배; 배; 배; 배; 배; ) HANGUL SYLLABLE BAE +BC31;BC31;1107 1162 11A8;BC31;1107 1162 11A8; # (백; 백; 백; 백; 백; ) HANGUL SYLLABLE BAEG +BC32;BC32;1107 1162 11A9;BC32;1107 1162 11A9; # (밲; 밲; 밲; 밲; 밲; ) HANGUL SYLLABLE BAEGG +BC33;BC33;1107 1162 11AA;BC33;1107 1162 11AA; # (밳; 밳; 밳; 밳; 밳; ) HANGUL SYLLABLE BAEGS +BC34;BC34;1107 1162 11AB;BC34;1107 1162 11AB; # (밴; 밴; 밴; 밴; 밴; ) HANGUL SYLLABLE BAEN +BC35;BC35;1107 1162 11AC;BC35;1107 1162 11AC; # (밵; 밵; 밵; 밵; 밵; ) HANGUL SYLLABLE BAENJ +BC36;BC36;1107 1162 11AD;BC36;1107 1162 11AD; # (밶; 밶; 밶; 밶; 밶; ) HANGUL SYLLABLE BAENH +BC37;BC37;1107 1162 11AE;BC37;1107 1162 11AE; # (밷; 밷; 밷; 밷; 밷; ) HANGUL SYLLABLE BAED +BC38;BC38;1107 1162 11AF;BC38;1107 1162 11AF; # (밸; 밸; 밸; 밸; 밸; ) HANGUL SYLLABLE BAEL +BC39;BC39;1107 1162 11B0;BC39;1107 1162 11B0; # (밹; 밹; 밹; 밹; 밹; ) HANGUL SYLLABLE BAELG +BC3A;BC3A;1107 1162 11B1;BC3A;1107 1162 11B1; # (밺; 밺; 밺; 밺; 밺; ) HANGUL SYLLABLE BAELM +BC3B;BC3B;1107 1162 11B2;BC3B;1107 1162 11B2; # (밻; 밻; 밻; 밻; 밻; ) HANGUL SYLLABLE BAELB +BC3C;BC3C;1107 1162 11B3;BC3C;1107 1162 11B3; # (밼; 밼; 밼; 밼; 밼; ) HANGUL SYLLABLE BAELS +BC3D;BC3D;1107 1162 11B4;BC3D;1107 1162 11B4; # (밽; 밽; 밽; 밽; 밽; ) HANGUL SYLLABLE BAELT +BC3E;BC3E;1107 1162 11B5;BC3E;1107 1162 11B5; # (밾; 밾; 밾; 밾; 밾; ) HANGUL SYLLABLE BAELP +BC3F;BC3F;1107 1162 11B6;BC3F;1107 1162 11B6; # (밿; 밿; 밿; 밿; 밿; ) HANGUL SYLLABLE BAELH +BC40;BC40;1107 1162 11B7;BC40;1107 1162 11B7; # (뱀; 뱀; 뱀; 뱀; 뱀; ) HANGUL SYLLABLE BAEM +BC41;BC41;1107 1162 11B8;BC41;1107 1162 11B8; # (뱁; 뱁; 뱁; 뱁; 뱁; ) HANGUL SYLLABLE BAEB +BC42;BC42;1107 1162 11B9;BC42;1107 1162 11B9; # (뱂; 뱂; 뱂; 뱂; 뱂; ) HANGUL SYLLABLE BAEBS +BC43;BC43;1107 1162 11BA;BC43;1107 1162 11BA; # (뱃; 뱃; 뱃; 뱃; 뱃; ) HANGUL SYLLABLE BAES +BC44;BC44;1107 1162 11BB;BC44;1107 1162 11BB; # (뱄; 뱄; 뱄; 뱄; 뱄; ) HANGUL SYLLABLE BAESS +BC45;BC45;1107 1162 11BC;BC45;1107 1162 11BC; # (뱅; 뱅; 뱅; 뱅; 뱅; ) HANGUL SYLLABLE BAENG +BC46;BC46;1107 1162 11BD;BC46;1107 1162 11BD; # (뱆; 뱆; 뱆; 뱆; 뱆; ) HANGUL SYLLABLE BAEJ +BC47;BC47;1107 1162 11BE;BC47;1107 1162 11BE; # (뱇; 뱇; 뱇; 뱇; 뱇; ) HANGUL SYLLABLE BAEC +BC48;BC48;1107 1162 11BF;BC48;1107 1162 11BF; # (뱈; 뱈; 뱈; 뱈; 뱈; ) HANGUL SYLLABLE BAEK +BC49;BC49;1107 1162 11C0;BC49;1107 1162 11C0; # (뱉; 뱉; 뱉; 뱉; 뱉; ) HANGUL SYLLABLE BAET +BC4A;BC4A;1107 1162 11C1;BC4A;1107 1162 11C1; # (뱊; 뱊; 뱊; 뱊; 뱊; ) HANGUL SYLLABLE BAEP +BC4B;BC4B;1107 1162 11C2;BC4B;1107 1162 11C2; # (뱋; 뱋; 뱋; 뱋; 뱋; ) HANGUL SYLLABLE BAEH +BC4C;BC4C;1107 1163;BC4C;1107 1163; # (뱌; 뱌; 뱌; 뱌; 뱌; ) HANGUL SYLLABLE BYA +BC4D;BC4D;1107 1163 11A8;BC4D;1107 1163 11A8; # (뱍; 뱍; 뱍; 뱍; 뱍; ) HANGUL SYLLABLE BYAG +BC4E;BC4E;1107 1163 11A9;BC4E;1107 1163 11A9; # (뱎; 뱎; 뱎; 뱎; 뱎; ) HANGUL SYLLABLE BYAGG +BC4F;BC4F;1107 1163 11AA;BC4F;1107 1163 11AA; # (뱏; 뱏; 뱏; 뱏; 뱏; ) HANGUL SYLLABLE BYAGS +BC50;BC50;1107 1163 11AB;BC50;1107 1163 11AB; # (뱐; 뱐; 뱐; 뱐; 뱐; ) HANGUL SYLLABLE BYAN +BC51;BC51;1107 1163 11AC;BC51;1107 1163 11AC; # (뱑; 뱑; 뱑; 뱑; 뱑; ) HANGUL SYLLABLE BYANJ +BC52;BC52;1107 1163 11AD;BC52;1107 1163 11AD; # (뱒; 뱒; 뱒; 뱒; 뱒; ) HANGUL SYLLABLE BYANH +BC53;BC53;1107 1163 11AE;BC53;1107 1163 11AE; # (뱓; 뱓; 뱓; 뱓; 뱓; ) HANGUL SYLLABLE BYAD +BC54;BC54;1107 1163 11AF;BC54;1107 1163 11AF; # (뱔; 뱔; 뱔; 뱔; 뱔; ) HANGUL SYLLABLE BYAL +BC55;BC55;1107 1163 11B0;BC55;1107 1163 11B0; # (뱕; 뱕; 뱕; 뱕; 뱕; ) HANGUL SYLLABLE BYALG +BC56;BC56;1107 1163 11B1;BC56;1107 1163 11B1; # (뱖; 뱖; 뱖; 뱖; 뱖; ) HANGUL SYLLABLE BYALM +BC57;BC57;1107 1163 11B2;BC57;1107 1163 11B2; # (뱗; 뱗; 뱗; 뱗; 뱗; ) HANGUL SYLLABLE BYALB +BC58;BC58;1107 1163 11B3;BC58;1107 1163 11B3; # (뱘; 뱘; 뱘; 뱘; 뱘; ) HANGUL SYLLABLE BYALS +BC59;BC59;1107 1163 11B4;BC59;1107 1163 11B4; # (뱙; 뱙; 뱙; 뱙; 뱙; ) HANGUL SYLLABLE BYALT +BC5A;BC5A;1107 1163 11B5;BC5A;1107 1163 11B5; # (뱚; 뱚; 뱚; 뱚; 뱚; ) HANGUL SYLLABLE BYALP +BC5B;BC5B;1107 1163 11B6;BC5B;1107 1163 11B6; # (뱛; 뱛; 뱛; 뱛; 뱛; ) HANGUL SYLLABLE BYALH +BC5C;BC5C;1107 1163 11B7;BC5C;1107 1163 11B7; # (뱜; 뱜; 뱜; 뱜; 뱜; ) HANGUL SYLLABLE BYAM +BC5D;BC5D;1107 1163 11B8;BC5D;1107 1163 11B8; # (뱝; 뱝; 뱝; 뱝; 뱝; ) HANGUL SYLLABLE BYAB +BC5E;BC5E;1107 1163 11B9;BC5E;1107 1163 11B9; # (뱞; 뱞; 뱞; 뱞; 뱞; ) HANGUL SYLLABLE BYABS +BC5F;BC5F;1107 1163 11BA;BC5F;1107 1163 11BA; # (뱟; 뱟; 뱟; 뱟; 뱟; ) HANGUL SYLLABLE BYAS +BC60;BC60;1107 1163 11BB;BC60;1107 1163 11BB; # (뱠; 뱠; 뱠; 뱠; 뱠; ) HANGUL SYLLABLE BYASS +BC61;BC61;1107 1163 11BC;BC61;1107 1163 11BC; # (뱡; 뱡; 뱡; 뱡; 뱡; ) HANGUL SYLLABLE BYANG +BC62;BC62;1107 1163 11BD;BC62;1107 1163 11BD; # (뱢; 뱢; 뱢; 뱢; 뱢; ) HANGUL SYLLABLE BYAJ +BC63;BC63;1107 1163 11BE;BC63;1107 1163 11BE; # (뱣; 뱣; 뱣; 뱣; 뱣; ) HANGUL SYLLABLE BYAC +BC64;BC64;1107 1163 11BF;BC64;1107 1163 11BF; # (뱤; 뱤; 뱤; 뱤; 뱤; ) HANGUL SYLLABLE BYAK +BC65;BC65;1107 1163 11C0;BC65;1107 1163 11C0; # (뱥; 뱥; 뱥; 뱥; 뱥; ) HANGUL SYLLABLE BYAT +BC66;BC66;1107 1163 11C1;BC66;1107 1163 11C1; # (뱦; 뱦; 뱦; 뱦; 뱦; ) HANGUL SYLLABLE BYAP +BC67;BC67;1107 1163 11C2;BC67;1107 1163 11C2; # (뱧; 뱧; 뱧; 뱧; 뱧; ) HANGUL SYLLABLE BYAH +BC68;BC68;1107 1164;BC68;1107 1164; # (뱨; 뱨; 뱨; 뱨; 뱨; ) HANGUL SYLLABLE BYAE +BC69;BC69;1107 1164 11A8;BC69;1107 1164 11A8; # (뱩; 뱩; 뱩; 뱩; 뱩; ) HANGUL SYLLABLE BYAEG +BC6A;BC6A;1107 1164 11A9;BC6A;1107 1164 11A9; # (뱪; 뱪; 뱪; 뱪; 뱪; ) HANGUL SYLLABLE BYAEGG +BC6B;BC6B;1107 1164 11AA;BC6B;1107 1164 11AA; # (뱫; 뱫; 뱫; 뱫; 뱫; ) HANGUL SYLLABLE BYAEGS +BC6C;BC6C;1107 1164 11AB;BC6C;1107 1164 11AB; # (뱬; 뱬; 뱬; 뱬; 뱬; ) HANGUL SYLLABLE BYAEN +BC6D;BC6D;1107 1164 11AC;BC6D;1107 1164 11AC; # (뱭; 뱭; 뱭; 뱭; 뱭; ) HANGUL SYLLABLE BYAENJ +BC6E;BC6E;1107 1164 11AD;BC6E;1107 1164 11AD; # (뱮; 뱮; 뱮; 뱮; 뱮; ) HANGUL SYLLABLE BYAENH +BC6F;BC6F;1107 1164 11AE;BC6F;1107 1164 11AE; # (뱯; 뱯; 뱯; 뱯; 뱯; ) HANGUL SYLLABLE BYAED +BC70;BC70;1107 1164 11AF;BC70;1107 1164 11AF; # (뱰; 뱰; 뱰; 뱰; 뱰; ) HANGUL SYLLABLE BYAEL +BC71;BC71;1107 1164 11B0;BC71;1107 1164 11B0; # (뱱; 뱱; 뱱; 뱱; 뱱; ) HANGUL SYLLABLE BYAELG +BC72;BC72;1107 1164 11B1;BC72;1107 1164 11B1; # (뱲; 뱲; 뱲; 뱲; 뱲; ) HANGUL SYLLABLE BYAELM +BC73;BC73;1107 1164 11B2;BC73;1107 1164 11B2; # (뱳; 뱳; 뱳; 뱳; 뱳; ) HANGUL SYLLABLE BYAELB +BC74;BC74;1107 1164 11B3;BC74;1107 1164 11B3; # (뱴; 뱴; 뱴; 뱴; 뱴; ) HANGUL SYLLABLE BYAELS +BC75;BC75;1107 1164 11B4;BC75;1107 1164 11B4; # (뱵; 뱵; 뱵; 뱵; 뱵; ) HANGUL SYLLABLE BYAELT +BC76;BC76;1107 1164 11B5;BC76;1107 1164 11B5; # (뱶; 뱶; 뱶; 뱶; 뱶; ) HANGUL SYLLABLE BYAELP +BC77;BC77;1107 1164 11B6;BC77;1107 1164 11B6; # (뱷; 뱷; 뱷; 뱷; 뱷; ) HANGUL SYLLABLE BYAELH +BC78;BC78;1107 1164 11B7;BC78;1107 1164 11B7; # (뱸; 뱸; 뱸; 뱸; 뱸; ) HANGUL SYLLABLE BYAEM +BC79;BC79;1107 1164 11B8;BC79;1107 1164 11B8; # (뱹; 뱹; 뱹; 뱹; 뱹; ) HANGUL SYLLABLE BYAEB +BC7A;BC7A;1107 1164 11B9;BC7A;1107 1164 11B9; # (뱺; 뱺; 뱺; 뱺; 뱺; ) HANGUL SYLLABLE BYAEBS +BC7B;BC7B;1107 1164 11BA;BC7B;1107 1164 11BA; # (뱻; 뱻; 뱻; 뱻; 뱻; ) HANGUL SYLLABLE BYAES +BC7C;BC7C;1107 1164 11BB;BC7C;1107 1164 11BB; # (뱼; 뱼; 뱼; 뱼; 뱼; ) HANGUL SYLLABLE BYAESS +BC7D;BC7D;1107 1164 11BC;BC7D;1107 1164 11BC; # (뱽; 뱽; 뱽; 뱽; 뱽; ) HANGUL SYLLABLE BYAENG +BC7E;BC7E;1107 1164 11BD;BC7E;1107 1164 11BD; # (뱾; 뱾; 뱾; 뱾; 뱾; ) HANGUL SYLLABLE BYAEJ +BC7F;BC7F;1107 1164 11BE;BC7F;1107 1164 11BE; # (뱿; 뱿; 뱿; 뱿; 뱿; ) HANGUL SYLLABLE BYAEC +BC80;BC80;1107 1164 11BF;BC80;1107 1164 11BF; # (벀; 벀; 벀; 벀; 벀; ) HANGUL SYLLABLE BYAEK +BC81;BC81;1107 1164 11C0;BC81;1107 1164 11C0; # (벁; 벁; 벁; 벁; 벁; ) HANGUL SYLLABLE BYAET +BC82;BC82;1107 1164 11C1;BC82;1107 1164 11C1; # (벂; 벂; 벂; 벂; 벂; ) HANGUL SYLLABLE BYAEP +BC83;BC83;1107 1164 11C2;BC83;1107 1164 11C2; # (벃; 벃; 벃; 벃; 벃; ) HANGUL SYLLABLE BYAEH +BC84;BC84;1107 1165;BC84;1107 1165; # (버; 버; 버; 버; 버; ) HANGUL SYLLABLE BEO +BC85;BC85;1107 1165 11A8;BC85;1107 1165 11A8; # (벅; 벅; 벅; 벅; 벅; ) HANGUL SYLLABLE BEOG +BC86;BC86;1107 1165 11A9;BC86;1107 1165 11A9; # (벆; 벆; 벆; 벆; 벆; ) HANGUL SYLLABLE BEOGG +BC87;BC87;1107 1165 11AA;BC87;1107 1165 11AA; # (벇; 벇; 벇; 벇; 벇; ) HANGUL SYLLABLE BEOGS +BC88;BC88;1107 1165 11AB;BC88;1107 1165 11AB; # (번; 번; 번; 번; 번; ) HANGUL SYLLABLE BEON +BC89;BC89;1107 1165 11AC;BC89;1107 1165 11AC; # (벉; 벉; 벉; 벉; 벉; ) HANGUL SYLLABLE BEONJ +BC8A;BC8A;1107 1165 11AD;BC8A;1107 1165 11AD; # (벊; 벊; 벊; 벊; 벊; ) HANGUL SYLLABLE BEONH +BC8B;BC8B;1107 1165 11AE;BC8B;1107 1165 11AE; # (벋; 벋; 벋; 벋; 벋; ) HANGUL SYLLABLE BEOD +BC8C;BC8C;1107 1165 11AF;BC8C;1107 1165 11AF; # (벌; 벌; 벌; 벌; 벌; ) HANGUL SYLLABLE BEOL +BC8D;BC8D;1107 1165 11B0;BC8D;1107 1165 11B0; # (벍; 벍; 벍; 벍; 벍; ) HANGUL SYLLABLE BEOLG +BC8E;BC8E;1107 1165 11B1;BC8E;1107 1165 11B1; # (벎; 벎; 벎; 벎; 벎; ) HANGUL SYLLABLE BEOLM +BC8F;BC8F;1107 1165 11B2;BC8F;1107 1165 11B2; # (벏; 벏; 벏; 벏; 벏; ) HANGUL SYLLABLE BEOLB +BC90;BC90;1107 1165 11B3;BC90;1107 1165 11B3; # (벐; 벐; 벐; 벐; 벐; ) HANGUL SYLLABLE BEOLS +BC91;BC91;1107 1165 11B4;BC91;1107 1165 11B4; # (벑; 벑; 벑; 벑; 벑; ) HANGUL SYLLABLE BEOLT +BC92;BC92;1107 1165 11B5;BC92;1107 1165 11B5; # (벒; 벒; 벒; 벒; 벒; ) HANGUL SYLLABLE BEOLP +BC93;BC93;1107 1165 11B6;BC93;1107 1165 11B6; # (벓; 벓; 벓; 벓; 벓; ) HANGUL SYLLABLE BEOLH +BC94;BC94;1107 1165 11B7;BC94;1107 1165 11B7; # (범; 범; 범; 범; 범; ) HANGUL SYLLABLE BEOM +BC95;BC95;1107 1165 11B8;BC95;1107 1165 11B8; # (법; 법; 법; 법; 법; ) HANGUL SYLLABLE BEOB +BC96;BC96;1107 1165 11B9;BC96;1107 1165 11B9; # (벖; 벖; 벖; 벖; 벖; ) HANGUL SYLLABLE BEOBS +BC97;BC97;1107 1165 11BA;BC97;1107 1165 11BA; # (벗; 벗; 벗; 벗; 벗; ) HANGUL SYLLABLE BEOS +BC98;BC98;1107 1165 11BB;BC98;1107 1165 11BB; # (벘; 벘; 벘; 벘; 벘; ) HANGUL SYLLABLE BEOSS +BC99;BC99;1107 1165 11BC;BC99;1107 1165 11BC; # (벙; 벙; 벙; 벙; 벙; ) HANGUL SYLLABLE BEONG +BC9A;BC9A;1107 1165 11BD;BC9A;1107 1165 11BD; # (벚; 벚; 벚; 벚; 벚; ) HANGUL SYLLABLE BEOJ +BC9B;BC9B;1107 1165 11BE;BC9B;1107 1165 11BE; # (벛; 벛; 벛; 벛; 벛; ) HANGUL SYLLABLE BEOC +BC9C;BC9C;1107 1165 11BF;BC9C;1107 1165 11BF; # (벜; 벜; 벜; 벜; 벜; ) HANGUL SYLLABLE BEOK +BC9D;BC9D;1107 1165 11C0;BC9D;1107 1165 11C0; # (벝; 벝; 벝; 벝; 벝; ) HANGUL SYLLABLE BEOT +BC9E;BC9E;1107 1165 11C1;BC9E;1107 1165 11C1; # (벞; 벞; 벞; 벞; 벞; ) HANGUL SYLLABLE BEOP +BC9F;BC9F;1107 1165 11C2;BC9F;1107 1165 11C2; # (벟; 벟; 벟; 벟; 벟; ) HANGUL SYLLABLE BEOH +BCA0;BCA0;1107 1166;BCA0;1107 1166; # (베; 베; 베; 베; 베; ) HANGUL SYLLABLE BE +BCA1;BCA1;1107 1166 11A8;BCA1;1107 1166 11A8; # (벡; 벡; 벡; 벡; 벡; ) HANGUL SYLLABLE BEG +BCA2;BCA2;1107 1166 11A9;BCA2;1107 1166 11A9; # (벢; 벢; 벢; 벢; 벢; ) HANGUL SYLLABLE BEGG +BCA3;BCA3;1107 1166 11AA;BCA3;1107 1166 11AA; # (벣; 벣; 벣; 벣; 벣; ) HANGUL SYLLABLE BEGS +BCA4;BCA4;1107 1166 11AB;BCA4;1107 1166 11AB; # (벤; 벤; 벤; 벤; 벤; ) HANGUL SYLLABLE BEN +BCA5;BCA5;1107 1166 11AC;BCA5;1107 1166 11AC; # (벥; 벥; 벥; 벥; 벥; ) HANGUL SYLLABLE BENJ +BCA6;BCA6;1107 1166 11AD;BCA6;1107 1166 11AD; # (벦; 벦; 벦; 벦; 벦; ) HANGUL SYLLABLE BENH +BCA7;BCA7;1107 1166 11AE;BCA7;1107 1166 11AE; # (벧; 벧; 벧; 벧; 벧; ) HANGUL SYLLABLE BED +BCA8;BCA8;1107 1166 11AF;BCA8;1107 1166 11AF; # (벨; 벨; 벨; 벨; 벨; ) HANGUL SYLLABLE BEL +BCA9;BCA9;1107 1166 11B0;BCA9;1107 1166 11B0; # (벩; 벩; 벩; 벩; 벩; ) HANGUL SYLLABLE BELG +BCAA;BCAA;1107 1166 11B1;BCAA;1107 1166 11B1; # (벪; 벪; 벪; 벪; 벪; ) HANGUL SYLLABLE BELM +BCAB;BCAB;1107 1166 11B2;BCAB;1107 1166 11B2; # (벫; 벫; 벫; 벫; 벫; ) HANGUL SYLLABLE BELB +BCAC;BCAC;1107 1166 11B3;BCAC;1107 1166 11B3; # (벬; 벬; 벬; 벬; 벬; ) HANGUL SYLLABLE BELS +BCAD;BCAD;1107 1166 11B4;BCAD;1107 1166 11B4; # (벭; 벭; 벭; 벭; 벭; ) HANGUL SYLLABLE BELT +BCAE;BCAE;1107 1166 11B5;BCAE;1107 1166 11B5; # (벮; 벮; 벮; 벮; 벮; ) HANGUL SYLLABLE BELP +BCAF;BCAF;1107 1166 11B6;BCAF;1107 1166 11B6; # (벯; 벯; 벯; 벯; 벯; ) HANGUL SYLLABLE BELH +BCB0;BCB0;1107 1166 11B7;BCB0;1107 1166 11B7; # (벰; 벰; 벰; 벰; 벰; ) HANGUL SYLLABLE BEM +BCB1;BCB1;1107 1166 11B8;BCB1;1107 1166 11B8; # (벱; 벱; 벱; 벱; 벱; ) HANGUL SYLLABLE BEB +BCB2;BCB2;1107 1166 11B9;BCB2;1107 1166 11B9; # (벲; 벲; 벲; 벲; 벲; ) HANGUL SYLLABLE BEBS +BCB3;BCB3;1107 1166 11BA;BCB3;1107 1166 11BA; # (벳; 벳; 벳; 벳; 벳; ) HANGUL SYLLABLE BES +BCB4;BCB4;1107 1166 11BB;BCB4;1107 1166 11BB; # (벴; 벴; 벴; 벴; 벴; ) HANGUL SYLLABLE BESS +BCB5;BCB5;1107 1166 11BC;BCB5;1107 1166 11BC; # (벵; 벵; 벵; 벵; 벵; ) HANGUL SYLLABLE BENG +BCB6;BCB6;1107 1166 11BD;BCB6;1107 1166 11BD; # (벶; 벶; 벶; 벶; 벶; ) HANGUL SYLLABLE BEJ +BCB7;BCB7;1107 1166 11BE;BCB7;1107 1166 11BE; # (벷; 벷; 벷; 벷; 벷; ) HANGUL SYLLABLE BEC +BCB8;BCB8;1107 1166 11BF;BCB8;1107 1166 11BF; # (벸; 벸; 벸; 벸; 벸; ) HANGUL SYLLABLE BEK +BCB9;BCB9;1107 1166 11C0;BCB9;1107 1166 11C0; # (벹; 벹; 벹; 벹; 벹; ) HANGUL SYLLABLE BET +BCBA;BCBA;1107 1166 11C1;BCBA;1107 1166 11C1; # (벺; 벺; 벺; 벺; 벺; ) HANGUL SYLLABLE BEP +BCBB;BCBB;1107 1166 11C2;BCBB;1107 1166 11C2; # (벻; 벻; 벻; 벻; 벻; ) HANGUL SYLLABLE BEH +BCBC;BCBC;1107 1167;BCBC;1107 1167; # (벼; 벼; 벼; 벼; 벼; ) HANGUL SYLLABLE BYEO +BCBD;BCBD;1107 1167 11A8;BCBD;1107 1167 11A8; # (벽; 벽; 벽; 벽; 벽; ) HANGUL SYLLABLE BYEOG +BCBE;BCBE;1107 1167 11A9;BCBE;1107 1167 11A9; # (벾; 벾; 벾; 벾; 벾; ) HANGUL SYLLABLE BYEOGG +BCBF;BCBF;1107 1167 11AA;BCBF;1107 1167 11AA; # (벿; 벿; 벿; 벿; 벿; ) HANGUL SYLLABLE BYEOGS +BCC0;BCC0;1107 1167 11AB;BCC0;1107 1167 11AB; # (변; 변; 변; 변; 변; ) HANGUL SYLLABLE BYEON +BCC1;BCC1;1107 1167 11AC;BCC1;1107 1167 11AC; # (볁; 볁; 볁; 볁; 볁; ) HANGUL SYLLABLE BYEONJ +BCC2;BCC2;1107 1167 11AD;BCC2;1107 1167 11AD; # (볂; 볂; 볂; 볂; 볂; ) HANGUL SYLLABLE BYEONH +BCC3;BCC3;1107 1167 11AE;BCC3;1107 1167 11AE; # (볃; 볃; 볃; 볃; 볃; ) HANGUL SYLLABLE BYEOD +BCC4;BCC4;1107 1167 11AF;BCC4;1107 1167 11AF; # (별; 별; 별; 별; 별; ) HANGUL SYLLABLE BYEOL +BCC5;BCC5;1107 1167 11B0;BCC5;1107 1167 11B0; # (볅; 볅; 볅; 볅; 볅; ) HANGUL SYLLABLE BYEOLG +BCC6;BCC6;1107 1167 11B1;BCC6;1107 1167 11B1; # (볆; 볆; 볆; 볆; 볆; ) HANGUL SYLLABLE BYEOLM +BCC7;BCC7;1107 1167 11B2;BCC7;1107 1167 11B2; # (볇; 볇; 볇; 볇; 볇; ) HANGUL SYLLABLE BYEOLB +BCC8;BCC8;1107 1167 11B3;BCC8;1107 1167 11B3; # (볈; 볈; 볈; 볈; 볈; ) HANGUL SYLLABLE BYEOLS +BCC9;BCC9;1107 1167 11B4;BCC9;1107 1167 11B4; # (볉; 볉; 볉; 볉; 볉; ) HANGUL SYLLABLE BYEOLT +BCCA;BCCA;1107 1167 11B5;BCCA;1107 1167 11B5; # (볊; 볊; 볊; 볊; 볊; ) HANGUL SYLLABLE BYEOLP +BCCB;BCCB;1107 1167 11B6;BCCB;1107 1167 11B6; # (볋; 볋; 볋; 볋; 볋; ) HANGUL SYLLABLE BYEOLH +BCCC;BCCC;1107 1167 11B7;BCCC;1107 1167 11B7; # (볌; 볌; 볌; 볌; 볌; ) HANGUL SYLLABLE BYEOM +BCCD;BCCD;1107 1167 11B8;BCCD;1107 1167 11B8; # (볍; 볍; 볍; 볍; 볍; ) HANGUL SYLLABLE BYEOB +BCCE;BCCE;1107 1167 11B9;BCCE;1107 1167 11B9; # (볎; 볎; 볎; 볎; 볎; ) HANGUL SYLLABLE BYEOBS +BCCF;BCCF;1107 1167 11BA;BCCF;1107 1167 11BA; # (볏; 볏; 볏; 볏; 볏; ) HANGUL SYLLABLE BYEOS +BCD0;BCD0;1107 1167 11BB;BCD0;1107 1167 11BB; # (볐; 볐; 볐; 볐; 볐; ) HANGUL SYLLABLE BYEOSS +BCD1;BCD1;1107 1167 11BC;BCD1;1107 1167 11BC; # (병; 병; 병; 병; 병; ) HANGUL SYLLABLE BYEONG +BCD2;BCD2;1107 1167 11BD;BCD2;1107 1167 11BD; # (볒; 볒; 볒; 볒; 볒; ) HANGUL SYLLABLE BYEOJ +BCD3;BCD3;1107 1167 11BE;BCD3;1107 1167 11BE; # (볓; 볓; 볓; 볓; 볓; ) HANGUL SYLLABLE BYEOC +BCD4;BCD4;1107 1167 11BF;BCD4;1107 1167 11BF; # (볔; 볔; 볔; 볔; 볔; ) HANGUL SYLLABLE BYEOK +BCD5;BCD5;1107 1167 11C0;BCD5;1107 1167 11C0; # (볕; 볕; 볕; 볕; 볕; ) HANGUL SYLLABLE BYEOT +BCD6;BCD6;1107 1167 11C1;BCD6;1107 1167 11C1; # (볖; 볖; 볖; 볖; 볖; ) HANGUL SYLLABLE BYEOP +BCD7;BCD7;1107 1167 11C2;BCD7;1107 1167 11C2; # (볗; 볗; 볗; 볗; 볗; ) HANGUL SYLLABLE BYEOH +BCD8;BCD8;1107 1168;BCD8;1107 1168; # (볘; 볘; 볘; 볘; 볘; ) HANGUL SYLLABLE BYE +BCD9;BCD9;1107 1168 11A8;BCD9;1107 1168 11A8; # (볙; 볙; 볙; 볙; 볙; ) HANGUL SYLLABLE BYEG +BCDA;BCDA;1107 1168 11A9;BCDA;1107 1168 11A9; # (볚; 볚; 볚; 볚; 볚; ) HANGUL SYLLABLE BYEGG +BCDB;BCDB;1107 1168 11AA;BCDB;1107 1168 11AA; # (볛; 볛; 볛; 볛; 볛; ) HANGUL SYLLABLE BYEGS +BCDC;BCDC;1107 1168 11AB;BCDC;1107 1168 11AB; # (볜; 볜; 볜; 볜; 볜; ) HANGUL SYLLABLE BYEN +BCDD;BCDD;1107 1168 11AC;BCDD;1107 1168 11AC; # (볝; 볝; 볝; 볝; 볝; ) HANGUL SYLLABLE BYENJ +BCDE;BCDE;1107 1168 11AD;BCDE;1107 1168 11AD; # (볞; 볞; 볞; 볞; 볞; ) HANGUL SYLLABLE BYENH +BCDF;BCDF;1107 1168 11AE;BCDF;1107 1168 11AE; # (볟; 볟; 볟; 볟; 볟; ) HANGUL SYLLABLE BYED +BCE0;BCE0;1107 1168 11AF;BCE0;1107 1168 11AF; # (볠; 볠; 볠; 볠; 볠; ) HANGUL SYLLABLE BYEL +BCE1;BCE1;1107 1168 11B0;BCE1;1107 1168 11B0; # (볡; 볡; 볡; 볡; 볡; ) HANGUL SYLLABLE BYELG +BCE2;BCE2;1107 1168 11B1;BCE2;1107 1168 11B1; # (볢; 볢; 볢; 볢; 볢; ) HANGUL SYLLABLE BYELM +BCE3;BCE3;1107 1168 11B2;BCE3;1107 1168 11B2; # (볣; 볣; 볣; 볣; 볣; ) HANGUL SYLLABLE BYELB +BCE4;BCE4;1107 1168 11B3;BCE4;1107 1168 11B3; # (볤; 볤; 볤; 볤; 볤; ) HANGUL SYLLABLE BYELS +BCE5;BCE5;1107 1168 11B4;BCE5;1107 1168 11B4; # (볥; 볥; 볥; 볥; 볥; ) HANGUL SYLLABLE BYELT +BCE6;BCE6;1107 1168 11B5;BCE6;1107 1168 11B5; # (볦; 볦; 볦; 볦; 볦; ) HANGUL SYLLABLE BYELP +BCE7;BCE7;1107 1168 11B6;BCE7;1107 1168 11B6; # (볧; 볧; 볧; 볧; 볧; ) HANGUL SYLLABLE BYELH +BCE8;BCE8;1107 1168 11B7;BCE8;1107 1168 11B7; # (볨; 볨; 볨; 볨; 볨; ) HANGUL SYLLABLE BYEM +BCE9;BCE9;1107 1168 11B8;BCE9;1107 1168 11B8; # (볩; 볩; 볩; 볩; 볩; ) HANGUL SYLLABLE BYEB +BCEA;BCEA;1107 1168 11B9;BCEA;1107 1168 11B9; # (볪; 볪; 볪; 볪; 볪; ) HANGUL SYLLABLE BYEBS +BCEB;BCEB;1107 1168 11BA;BCEB;1107 1168 11BA; # (볫; 볫; 볫; 볫; 볫; ) HANGUL SYLLABLE BYES +BCEC;BCEC;1107 1168 11BB;BCEC;1107 1168 11BB; # (볬; 볬; 볬; 볬; 볬; ) HANGUL SYLLABLE BYESS +BCED;BCED;1107 1168 11BC;BCED;1107 1168 11BC; # (볭; 볭; 볭; 볭; 볭; ) HANGUL SYLLABLE BYENG +BCEE;BCEE;1107 1168 11BD;BCEE;1107 1168 11BD; # (볮; 볮; 볮; 볮; 볮; ) HANGUL SYLLABLE BYEJ +BCEF;BCEF;1107 1168 11BE;BCEF;1107 1168 11BE; # (볯; 볯; 볯; 볯; 볯; ) HANGUL SYLLABLE BYEC +BCF0;BCF0;1107 1168 11BF;BCF0;1107 1168 11BF; # (볰; 볰; 볰; 볰; 볰; ) HANGUL SYLLABLE BYEK +BCF1;BCF1;1107 1168 11C0;BCF1;1107 1168 11C0; # (볱; 볱; 볱; 볱; 볱; ) HANGUL SYLLABLE BYET +BCF2;BCF2;1107 1168 11C1;BCF2;1107 1168 11C1; # (볲; 볲; 볲; 볲; 볲; ) HANGUL SYLLABLE BYEP +BCF3;BCF3;1107 1168 11C2;BCF3;1107 1168 11C2; # (볳; 볳; 볳; 볳; 볳; ) HANGUL SYLLABLE BYEH +BCF4;BCF4;1107 1169;BCF4;1107 1169; # (보; 보; 보; 보; 보; ) HANGUL SYLLABLE BO +BCF5;BCF5;1107 1169 11A8;BCF5;1107 1169 11A8; # (복; 복; 복; 복; 복; ) HANGUL SYLLABLE BOG +BCF6;BCF6;1107 1169 11A9;BCF6;1107 1169 11A9; # (볶; 볶; 볶; 볶; 볶; ) HANGUL SYLLABLE BOGG +BCF7;BCF7;1107 1169 11AA;BCF7;1107 1169 11AA; # (볷; 볷; 볷; 볷; 볷; ) HANGUL SYLLABLE BOGS +BCF8;BCF8;1107 1169 11AB;BCF8;1107 1169 11AB; # (본; 본; 본; 본; 본; ) HANGUL SYLLABLE BON +BCF9;BCF9;1107 1169 11AC;BCF9;1107 1169 11AC; # (볹; 볹; 볹; 볹; 볹; ) HANGUL SYLLABLE BONJ +BCFA;BCFA;1107 1169 11AD;BCFA;1107 1169 11AD; # (볺; 볺; 볺; 볺; 볺; ) HANGUL SYLLABLE BONH +BCFB;BCFB;1107 1169 11AE;BCFB;1107 1169 11AE; # (볻; 볻; 볻; 볻; 볻; ) HANGUL SYLLABLE BOD +BCFC;BCFC;1107 1169 11AF;BCFC;1107 1169 11AF; # (볼; 볼; 볼; 볼; 볼; ) HANGUL SYLLABLE BOL +BCFD;BCFD;1107 1169 11B0;BCFD;1107 1169 11B0; # (볽; 볽; 볽; 볽; 볽; ) HANGUL SYLLABLE BOLG +BCFE;BCFE;1107 1169 11B1;BCFE;1107 1169 11B1; # (볾; 볾; 볾; 볾; 볾; ) HANGUL SYLLABLE BOLM +BCFF;BCFF;1107 1169 11B2;BCFF;1107 1169 11B2; # (볿; 볿; 볿; 볿; 볿; ) HANGUL SYLLABLE BOLB +BD00;BD00;1107 1169 11B3;BD00;1107 1169 11B3; # (봀; 봀; 봀; 봀; 봀; ) HANGUL SYLLABLE BOLS +BD01;BD01;1107 1169 11B4;BD01;1107 1169 11B4; # (봁; 봁; 봁; 봁; 봁; ) HANGUL SYLLABLE BOLT +BD02;BD02;1107 1169 11B5;BD02;1107 1169 11B5; # (봂; 봂; 봂; 봂; 봂; ) HANGUL SYLLABLE BOLP +BD03;BD03;1107 1169 11B6;BD03;1107 1169 11B6; # (봃; 봃; 봃; 봃; 봃; ) HANGUL SYLLABLE BOLH +BD04;BD04;1107 1169 11B7;BD04;1107 1169 11B7; # (봄; 봄; 봄; 봄; 봄; ) HANGUL SYLLABLE BOM +BD05;BD05;1107 1169 11B8;BD05;1107 1169 11B8; # (봅; 봅; 봅; 봅; 봅; ) HANGUL SYLLABLE BOB +BD06;BD06;1107 1169 11B9;BD06;1107 1169 11B9; # (봆; 봆; 봆; 봆; 봆; ) HANGUL SYLLABLE BOBS +BD07;BD07;1107 1169 11BA;BD07;1107 1169 11BA; # (봇; 봇; 봇; 봇; 봇; ) HANGUL SYLLABLE BOS +BD08;BD08;1107 1169 11BB;BD08;1107 1169 11BB; # (봈; 봈; 봈; 봈; 봈; ) HANGUL SYLLABLE BOSS +BD09;BD09;1107 1169 11BC;BD09;1107 1169 11BC; # (봉; 봉; 봉; 봉; 봉; ) HANGUL SYLLABLE BONG +BD0A;BD0A;1107 1169 11BD;BD0A;1107 1169 11BD; # (봊; 봊; 봊; 봊; 봊; ) HANGUL SYLLABLE BOJ +BD0B;BD0B;1107 1169 11BE;BD0B;1107 1169 11BE; # (봋; 봋; 봋; 봋; 봋; ) HANGUL SYLLABLE BOC +BD0C;BD0C;1107 1169 11BF;BD0C;1107 1169 11BF; # (봌; 봌; 봌; 봌; 봌; ) HANGUL SYLLABLE BOK +BD0D;BD0D;1107 1169 11C0;BD0D;1107 1169 11C0; # (봍; 봍; 봍; 봍; 봍; ) HANGUL SYLLABLE BOT +BD0E;BD0E;1107 1169 11C1;BD0E;1107 1169 11C1; # (봎; 봎; 봎; 봎; 봎; ) HANGUL SYLLABLE BOP +BD0F;BD0F;1107 1169 11C2;BD0F;1107 1169 11C2; # (봏; 봏; 봏; 봏; 봏; ) HANGUL SYLLABLE BOH +BD10;BD10;1107 116A;BD10;1107 116A; # (봐; 봐; 봐; 봐; 봐; ) HANGUL SYLLABLE BWA +BD11;BD11;1107 116A 11A8;BD11;1107 116A 11A8; # (봑; 봑; 봑; 봑; 봑; ) HANGUL SYLLABLE BWAG +BD12;BD12;1107 116A 11A9;BD12;1107 116A 11A9; # (봒; 봒; 봒; 봒; 봒; ) HANGUL SYLLABLE BWAGG +BD13;BD13;1107 116A 11AA;BD13;1107 116A 11AA; # (봓; 봓; 봓; 봓; 봓; ) HANGUL SYLLABLE BWAGS +BD14;BD14;1107 116A 11AB;BD14;1107 116A 11AB; # (봔; 봔; 봔; 봔; 봔; ) HANGUL SYLLABLE BWAN +BD15;BD15;1107 116A 11AC;BD15;1107 116A 11AC; # (봕; 봕; 봕; 봕; 봕; ) HANGUL SYLLABLE BWANJ +BD16;BD16;1107 116A 11AD;BD16;1107 116A 11AD; # (봖; 봖; 봖; 봖; 봖; ) HANGUL SYLLABLE BWANH +BD17;BD17;1107 116A 11AE;BD17;1107 116A 11AE; # (봗; 봗; 봗; 봗; 봗; ) HANGUL SYLLABLE BWAD +BD18;BD18;1107 116A 11AF;BD18;1107 116A 11AF; # (봘; 봘; 봘; 봘; 봘; ) HANGUL SYLLABLE BWAL +BD19;BD19;1107 116A 11B0;BD19;1107 116A 11B0; # (봙; 봙; 봙; 봙; 봙; ) HANGUL SYLLABLE BWALG +BD1A;BD1A;1107 116A 11B1;BD1A;1107 116A 11B1; # (봚; 봚; 봚; 봚; 봚; ) HANGUL SYLLABLE BWALM +BD1B;BD1B;1107 116A 11B2;BD1B;1107 116A 11B2; # (봛; 봛; 봛; 봛; 봛; ) HANGUL SYLLABLE BWALB +BD1C;BD1C;1107 116A 11B3;BD1C;1107 116A 11B3; # (봜; 봜; 봜; 봜; 봜; ) HANGUL SYLLABLE BWALS +BD1D;BD1D;1107 116A 11B4;BD1D;1107 116A 11B4; # (봝; 봝; 봝; 봝; 봝; ) HANGUL SYLLABLE BWALT +BD1E;BD1E;1107 116A 11B5;BD1E;1107 116A 11B5; # (봞; 봞; 봞; 봞; 봞; ) HANGUL SYLLABLE BWALP +BD1F;BD1F;1107 116A 11B6;BD1F;1107 116A 11B6; # (봟; 봟; 봟; 봟; 봟; ) HANGUL SYLLABLE BWALH +BD20;BD20;1107 116A 11B7;BD20;1107 116A 11B7; # (봠; 봠; 봠; 봠; 봠; ) HANGUL SYLLABLE BWAM +BD21;BD21;1107 116A 11B8;BD21;1107 116A 11B8; # (봡; 봡; 봡; 봡; 봡; ) HANGUL SYLLABLE BWAB +BD22;BD22;1107 116A 11B9;BD22;1107 116A 11B9; # (봢; 봢; 봢; 봢; 봢; ) HANGUL SYLLABLE BWABS +BD23;BD23;1107 116A 11BA;BD23;1107 116A 11BA; # (봣; 봣; 봣; 봣; 봣; ) HANGUL SYLLABLE BWAS +BD24;BD24;1107 116A 11BB;BD24;1107 116A 11BB; # (봤; 봤; 봤; 봤; 봤; ) HANGUL SYLLABLE BWASS +BD25;BD25;1107 116A 11BC;BD25;1107 116A 11BC; # (봥; 봥; 봥; 봥; 봥; ) HANGUL SYLLABLE BWANG +BD26;BD26;1107 116A 11BD;BD26;1107 116A 11BD; # (봦; 봦; 봦; 봦; 봦; ) HANGUL SYLLABLE BWAJ +BD27;BD27;1107 116A 11BE;BD27;1107 116A 11BE; # (봧; 봧; 봧; 봧; 봧; ) HANGUL SYLLABLE BWAC +BD28;BD28;1107 116A 11BF;BD28;1107 116A 11BF; # (봨; 봨; 봨; 봨; 봨; ) HANGUL SYLLABLE BWAK +BD29;BD29;1107 116A 11C0;BD29;1107 116A 11C0; # (봩; 봩; 봩; 봩; 봩; ) HANGUL SYLLABLE BWAT +BD2A;BD2A;1107 116A 11C1;BD2A;1107 116A 11C1; # (봪; 봪; 봪; 봪; 봪; ) HANGUL SYLLABLE BWAP +BD2B;BD2B;1107 116A 11C2;BD2B;1107 116A 11C2; # (봫; 봫; 봫; 봫; 봫; ) HANGUL SYLLABLE BWAH +BD2C;BD2C;1107 116B;BD2C;1107 116B; # (봬; 봬; 봬; 봬; 봬; ) HANGUL SYLLABLE BWAE +BD2D;BD2D;1107 116B 11A8;BD2D;1107 116B 11A8; # (봭; 봭; 봭; 봭; 봭; ) HANGUL SYLLABLE BWAEG +BD2E;BD2E;1107 116B 11A9;BD2E;1107 116B 11A9; # (봮; 봮; 봮; 봮; 봮; ) HANGUL SYLLABLE BWAEGG +BD2F;BD2F;1107 116B 11AA;BD2F;1107 116B 11AA; # (봯; 봯; 봯; 봯; 봯; ) HANGUL SYLLABLE BWAEGS +BD30;BD30;1107 116B 11AB;BD30;1107 116B 11AB; # (봰; 봰; 봰; 봰; 봰; ) HANGUL SYLLABLE BWAEN +BD31;BD31;1107 116B 11AC;BD31;1107 116B 11AC; # (봱; 봱; 봱; 봱; 봱; ) HANGUL SYLLABLE BWAENJ +BD32;BD32;1107 116B 11AD;BD32;1107 116B 11AD; # (봲; 봲; 봲; 봲; 봲; ) HANGUL SYLLABLE BWAENH +BD33;BD33;1107 116B 11AE;BD33;1107 116B 11AE; # (봳; 봳; 봳; 봳; 봳; ) HANGUL SYLLABLE BWAED +BD34;BD34;1107 116B 11AF;BD34;1107 116B 11AF; # (봴; 봴; 봴; 봴; 봴; ) HANGUL SYLLABLE BWAEL +BD35;BD35;1107 116B 11B0;BD35;1107 116B 11B0; # (봵; 봵; 봵; 봵; 봵; ) HANGUL SYLLABLE BWAELG +BD36;BD36;1107 116B 11B1;BD36;1107 116B 11B1; # (봶; 봶; 봶; 봶; 봶; ) HANGUL SYLLABLE BWAELM +BD37;BD37;1107 116B 11B2;BD37;1107 116B 11B2; # (봷; 봷; 봷; 봷; 봷; ) HANGUL SYLLABLE BWAELB +BD38;BD38;1107 116B 11B3;BD38;1107 116B 11B3; # (봸; 봸; 봸; 봸; 봸; ) HANGUL SYLLABLE BWAELS +BD39;BD39;1107 116B 11B4;BD39;1107 116B 11B4; # (봹; 봹; 봹; 봹; 봹; ) HANGUL SYLLABLE BWAELT +BD3A;BD3A;1107 116B 11B5;BD3A;1107 116B 11B5; # (봺; 봺; 봺; 봺; 봺; ) HANGUL SYLLABLE BWAELP +BD3B;BD3B;1107 116B 11B6;BD3B;1107 116B 11B6; # (봻; 봻; 봻; 봻; 봻; ) HANGUL SYLLABLE BWAELH +BD3C;BD3C;1107 116B 11B7;BD3C;1107 116B 11B7; # (봼; 봼; 봼; 봼; 봼; ) HANGUL SYLLABLE BWAEM +BD3D;BD3D;1107 116B 11B8;BD3D;1107 116B 11B8; # (봽; 봽; 봽; 봽; 봽; ) HANGUL SYLLABLE BWAEB +BD3E;BD3E;1107 116B 11B9;BD3E;1107 116B 11B9; # (봾; 봾; 봾; 봾; 봾; ) HANGUL SYLLABLE BWAEBS +BD3F;BD3F;1107 116B 11BA;BD3F;1107 116B 11BA; # (봿; 봿; 봿; 봿; 봿; ) HANGUL SYLLABLE BWAES +BD40;BD40;1107 116B 11BB;BD40;1107 116B 11BB; # (뵀; 뵀; 뵀; 뵀; 뵀; ) HANGUL SYLLABLE BWAESS +BD41;BD41;1107 116B 11BC;BD41;1107 116B 11BC; # (뵁; 뵁; 뵁; 뵁; 뵁; ) HANGUL SYLLABLE BWAENG +BD42;BD42;1107 116B 11BD;BD42;1107 116B 11BD; # (뵂; 뵂; 뵂; 뵂; 뵂; ) HANGUL SYLLABLE BWAEJ +BD43;BD43;1107 116B 11BE;BD43;1107 116B 11BE; # (뵃; 뵃; 뵃; 뵃; 뵃; ) HANGUL SYLLABLE BWAEC +BD44;BD44;1107 116B 11BF;BD44;1107 116B 11BF; # (뵄; 뵄; 뵄; 뵄; 뵄; ) HANGUL SYLLABLE BWAEK +BD45;BD45;1107 116B 11C0;BD45;1107 116B 11C0; # (뵅; 뵅; 뵅; 뵅; 뵅; ) HANGUL SYLLABLE BWAET +BD46;BD46;1107 116B 11C1;BD46;1107 116B 11C1; # (뵆; 뵆; 뵆; 뵆; 뵆; ) HANGUL SYLLABLE BWAEP +BD47;BD47;1107 116B 11C2;BD47;1107 116B 11C2; # (뵇; 뵇; 뵇; 뵇; 뵇; ) HANGUL SYLLABLE BWAEH +BD48;BD48;1107 116C;BD48;1107 116C; # (뵈; 뵈; 뵈; 뵈; 뵈; ) HANGUL SYLLABLE BOE +BD49;BD49;1107 116C 11A8;BD49;1107 116C 11A8; # (뵉; 뵉; 뵉; 뵉; 뵉; ) HANGUL SYLLABLE BOEG +BD4A;BD4A;1107 116C 11A9;BD4A;1107 116C 11A9; # (뵊; 뵊; 뵊; 뵊; 뵊; ) HANGUL SYLLABLE BOEGG +BD4B;BD4B;1107 116C 11AA;BD4B;1107 116C 11AA; # (뵋; 뵋; 뵋; 뵋; 뵋; ) HANGUL SYLLABLE BOEGS +BD4C;BD4C;1107 116C 11AB;BD4C;1107 116C 11AB; # (뵌; 뵌; 뵌; 뵌; 뵌; ) HANGUL SYLLABLE BOEN +BD4D;BD4D;1107 116C 11AC;BD4D;1107 116C 11AC; # (뵍; 뵍; 뵍; 뵍; 뵍; ) HANGUL SYLLABLE BOENJ +BD4E;BD4E;1107 116C 11AD;BD4E;1107 116C 11AD; # (뵎; 뵎; 뵎; 뵎; 뵎; ) HANGUL SYLLABLE BOENH +BD4F;BD4F;1107 116C 11AE;BD4F;1107 116C 11AE; # (뵏; 뵏; 뵏; 뵏; 뵏; ) HANGUL SYLLABLE BOED +BD50;BD50;1107 116C 11AF;BD50;1107 116C 11AF; # (뵐; 뵐; 뵐; 뵐; 뵐; ) HANGUL SYLLABLE BOEL +BD51;BD51;1107 116C 11B0;BD51;1107 116C 11B0; # (뵑; 뵑; 뵑; 뵑; 뵑; ) HANGUL SYLLABLE BOELG +BD52;BD52;1107 116C 11B1;BD52;1107 116C 11B1; # (뵒; 뵒; 뵒; 뵒; 뵒; ) HANGUL SYLLABLE BOELM +BD53;BD53;1107 116C 11B2;BD53;1107 116C 11B2; # (뵓; 뵓; 뵓; 뵓; 뵓; ) HANGUL SYLLABLE BOELB +BD54;BD54;1107 116C 11B3;BD54;1107 116C 11B3; # (뵔; 뵔; 뵔; 뵔; 뵔; ) HANGUL SYLLABLE BOELS +BD55;BD55;1107 116C 11B4;BD55;1107 116C 11B4; # (뵕; 뵕; 뵕; 뵕; 뵕; ) HANGUL SYLLABLE BOELT +BD56;BD56;1107 116C 11B5;BD56;1107 116C 11B5; # (뵖; 뵖; 뵖; 뵖; 뵖; ) HANGUL SYLLABLE BOELP +BD57;BD57;1107 116C 11B6;BD57;1107 116C 11B6; # (뵗; 뵗; 뵗; 뵗; 뵗; ) HANGUL SYLLABLE BOELH +BD58;BD58;1107 116C 11B7;BD58;1107 116C 11B7; # (뵘; 뵘; 뵘; 뵘; 뵘; ) HANGUL SYLLABLE BOEM +BD59;BD59;1107 116C 11B8;BD59;1107 116C 11B8; # (뵙; 뵙; 뵙; 뵙; 뵙; ) HANGUL SYLLABLE BOEB +BD5A;BD5A;1107 116C 11B9;BD5A;1107 116C 11B9; # (뵚; 뵚; 뵚; 뵚; 뵚; ) HANGUL SYLLABLE BOEBS +BD5B;BD5B;1107 116C 11BA;BD5B;1107 116C 11BA; # (뵛; 뵛; 뵛; 뵛; 뵛; ) HANGUL SYLLABLE BOES +BD5C;BD5C;1107 116C 11BB;BD5C;1107 116C 11BB; # (뵜; 뵜; 뵜; 뵜; 뵜; ) HANGUL SYLLABLE BOESS +BD5D;BD5D;1107 116C 11BC;BD5D;1107 116C 11BC; # (뵝; 뵝; 뵝; 뵝; 뵝; ) HANGUL SYLLABLE BOENG +BD5E;BD5E;1107 116C 11BD;BD5E;1107 116C 11BD; # (뵞; 뵞; 뵞; 뵞; 뵞; ) HANGUL SYLLABLE BOEJ +BD5F;BD5F;1107 116C 11BE;BD5F;1107 116C 11BE; # (뵟; 뵟; 뵟; 뵟; 뵟; ) HANGUL SYLLABLE BOEC +BD60;BD60;1107 116C 11BF;BD60;1107 116C 11BF; # (뵠; 뵠; 뵠; 뵠; 뵠; ) HANGUL SYLLABLE BOEK +BD61;BD61;1107 116C 11C0;BD61;1107 116C 11C0; # (뵡; 뵡; 뵡; 뵡; 뵡; ) HANGUL SYLLABLE BOET +BD62;BD62;1107 116C 11C1;BD62;1107 116C 11C1; # (뵢; 뵢; 뵢; 뵢; 뵢; ) HANGUL SYLLABLE BOEP +BD63;BD63;1107 116C 11C2;BD63;1107 116C 11C2; # (뵣; 뵣; 뵣; 뵣; 뵣; ) HANGUL SYLLABLE BOEH +BD64;BD64;1107 116D;BD64;1107 116D; # (뵤; 뵤; 뵤; 뵤; 뵤; ) HANGUL SYLLABLE BYO +BD65;BD65;1107 116D 11A8;BD65;1107 116D 11A8; # (뵥; 뵥; 뵥; 뵥; 뵥; ) HANGUL SYLLABLE BYOG +BD66;BD66;1107 116D 11A9;BD66;1107 116D 11A9; # (뵦; 뵦; 뵦; 뵦; 뵦; ) HANGUL SYLLABLE BYOGG +BD67;BD67;1107 116D 11AA;BD67;1107 116D 11AA; # (뵧; 뵧; 뵧; 뵧; 뵧; ) HANGUL SYLLABLE BYOGS +BD68;BD68;1107 116D 11AB;BD68;1107 116D 11AB; # (뵨; 뵨; 뵨; 뵨; 뵨; ) HANGUL SYLLABLE BYON +BD69;BD69;1107 116D 11AC;BD69;1107 116D 11AC; # (뵩; 뵩; 뵩; 뵩; 뵩; ) HANGUL SYLLABLE BYONJ +BD6A;BD6A;1107 116D 11AD;BD6A;1107 116D 11AD; # (뵪; 뵪; 뵪; 뵪; 뵪; ) HANGUL SYLLABLE BYONH +BD6B;BD6B;1107 116D 11AE;BD6B;1107 116D 11AE; # (뵫; 뵫; 뵫; 뵫; 뵫; ) HANGUL SYLLABLE BYOD +BD6C;BD6C;1107 116D 11AF;BD6C;1107 116D 11AF; # (뵬; 뵬; 뵬; 뵬; 뵬; ) HANGUL SYLLABLE BYOL +BD6D;BD6D;1107 116D 11B0;BD6D;1107 116D 11B0; # (뵭; 뵭; 뵭; 뵭; 뵭; ) HANGUL SYLLABLE BYOLG +BD6E;BD6E;1107 116D 11B1;BD6E;1107 116D 11B1; # (뵮; 뵮; 뵮; 뵮; 뵮; ) HANGUL SYLLABLE BYOLM +BD6F;BD6F;1107 116D 11B2;BD6F;1107 116D 11B2; # (뵯; 뵯; 뵯; 뵯; 뵯; ) HANGUL SYLLABLE BYOLB +BD70;BD70;1107 116D 11B3;BD70;1107 116D 11B3; # (뵰; 뵰; 뵰; 뵰; 뵰; ) HANGUL SYLLABLE BYOLS +BD71;BD71;1107 116D 11B4;BD71;1107 116D 11B4; # (뵱; 뵱; 뵱; 뵱; 뵱; ) HANGUL SYLLABLE BYOLT +BD72;BD72;1107 116D 11B5;BD72;1107 116D 11B5; # (뵲; 뵲; 뵲; 뵲; 뵲; ) HANGUL SYLLABLE BYOLP +BD73;BD73;1107 116D 11B6;BD73;1107 116D 11B6; # (뵳; 뵳; 뵳; 뵳; 뵳; ) HANGUL SYLLABLE BYOLH +BD74;BD74;1107 116D 11B7;BD74;1107 116D 11B7; # (뵴; 뵴; 뵴; 뵴; 뵴; ) HANGUL SYLLABLE BYOM +BD75;BD75;1107 116D 11B8;BD75;1107 116D 11B8; # (뵵; 뵵; 뵵; 뵵; 뵵; ) HANGUL SYLLABLE BYOB +BD76;BD76;1107 116D 11B9;BD76;1107 116D 11B9; # (뵶; 뵶; 뵶; 뵶; 뵶; ) HANGUL SYLLABLE BYOBS +BD77;BD77;1107 116D 11BA;BD77;1107 116D 11BA; # (뵷; 뵷; 뵷; 뵷; 뵷; ) HANGUL SYLLABLE BYOS +BD78;BD78;1107 116D 11BB;BD78;1107 116D 11BB; # (뵸; 뵸; 뵸; 뵸; 뵸; ) HANGUL SYLLABLE BYOSS +BD79;BD79;1107 116D 11BC;BD79;1107 116D 11BC; # (뵹; 뵹; 뵹; 뵹; 뵹; ) HANGUL SYLLABLE BYONG +BD7A;BD7A;1107 116D 11BD;BD7A;1107 116D 11BD; # (뵺; 뵺; 뵺; 뵺; 뵺; ) HANGUL SYLLABLE BYOJ +BD7B;BD7B;1107 116D 11BE;BD7B;1107 116D 11BE; # (뵻; 뵻; 뵻; 뵻; 뵻; ) HANGUL SYLLABLE BYOC +BD7C;BD7C;1107 116D 11BF;BD7C;1107 116D 11BF; # (뵼; 뵼; 뵼; 뵼; 뵼; ) HANGUL SYLLABLE BYOK +BD7D;BD7D;1107 116D 11C0;BD7D;1107 116D 11C0; # (뵽; 뵽; 뵽; 뵽; 뵽; ) HANGUL SYLLABLE BYOT +BD7E;BD7E;1107 116D 11C1;BD7E;1107 116D 11C1; # (뵾; 뵾; 뵾; 뵾; 뵾; ) HANGUL SYLLABLE BYOP +BD7F;BD7F;1107 116D 11C2;BD7F;1107 116D 11C2; # (뵿; 뵿; 뵿; 뵿; 뵿; ) HANGUL SYLLABLE BYOH +BD80;BD80;1107 116E;BD80;1107 116E; # (부; 부; 부; 부; 부; ) HANGUL SYLLABLE BU +BD81;BD81;1107 116E 11A8;BD81;1107 116E 11A8; # (북; 북; 북; 북; 북; ) HANGUL SYLLABLE BUG +BD82;BD82;1107 116E 11A9;BD82;1107 116E 11A9; # (붂; 붂; 붂; 붂; 붂; ) HANGUL SYLLABLE BUGG +BD83;BD83;1107 116E 11AA;BD83;1107 116E 11AA; # (붃; 붃; 붃; 붃; 붃; ) HANGUL SYLLABLE BUGS +BD84;BD84;1107 116E 11AB;BD84;1107 116E 11AB; # (분; 분; 분; 분; 분; ) HANGUL SYLLABLE BUN +BD85;BD85;1107 116E 11AC;BD85;1107 116E 11AC; # (붅; 붅; 붅; 붅; 붅; ) HANGUL SYLLABLE BUNJ +BD86;BD86;1107 116E 11AD;BD86;1107 116E 11AD; # (붆; 붆; 붆; 붆; 붆; ) HANGUL SYLLABLE BUNH +BD87;BD87;1107 116E 11AE;BD87;1107 116E 11AE; # (붇; 붇; 붇; 붇; 붇; ) HANGUL SYLLABLE BUD +BD88;BD88;1107 116E 11AF;BD88;1107 116E 11AF; # (불; 불; 불; 불; 불; ) HANGUL SYLLABLE BUL +BD89;BD89;1107 116E 11B0;BD89;1107 116E 11B0; # (붉; 붉; 붉; 붉; 붉; ) HANGUL SYLLABLE BULG +BD8A;BD8A;1107 116E 11B1;BD8A;1107 116E 11B1; # (붊; 붊; 붊; 붊; 붊; ) HANGUL SYLLABLE BULM +BD8B;BD8B;1107 116E 11B2;BD8B;1107 116E 11B2; # (붋; 붋; 붋; 붋; 붋; ) HANGUL SYLLABLE BULB +BD8C;BD8C;1107 116E 11B3;BD8C;1107 116E 11B3; # (붌; 붌; 붌; 붌; 붌; ) HANGUL SYLLABLE BULS +BD8D;BD8D;1107 116E 11B4;BD8D;1107 116E 11B4; # (붍; 붍; 붍; 붍; 붍; ) HANGUL SYLLABLE BULT +BD8E;BD8E;1107 116E 11B5;BD8E;1107 116E 11B5; # (붎; 붎; 붎; 붎; 붎; ) HANGUL SYLLABLE BULP +BD8F;BD8F;1107 116E 11B6;BD8F;1107 116E 11B6; # (붏; 붏; 붏; 붏; 붏; ) HANGUL SYLLABLE BULH +BD90;BD90;1107 116E 11B7;BD90;1107 116E 11B7; # (붐; 붐; 붐; 붐; 붐; ) HANGUL SYLLABLE BUM +BD91;BD91;1107 116E 11B8;BD91;1107 116E 11B8; # (붑; 붑; 붑; 붑; 붑; ) HANGUL SYLLABLE BUB +BD92;BD92;1107 116E 11B9;BD92;1107 116E 11B9; # (붒; 붒; 붒; 붒; 붒; ) HANGUL SYLLABLE BUBS +BD93;BD93;1107 116E 11BA;BD93;1107 116E 11BA; # (붓; 붓; 붓; 붓; 붓; ) HANGUL SYLLABLE BUS +BD94;BD94;1107 116E 11BB;BD94;1107 116E 11BB; # (붔; 붔; 붔; 붔; 붔; ) HANGUL SYLLABLE BUSS +BD95;BD95;1107 116E 11BC;BD95;1107 116E 11BC; # (붕; 붕; 붕; 붕; 붕; ) HANGUL SYLLABLE BUNG +BD96;BD96;1107 116E 11BD;BD96;1107 116E 11BD; # (붖; 붖; 붖; 붖; 붖; ) HANGUL SYLLABLE BUJ +BD97;BD97;1107 116E 11BE;BD97;1107 116E 11BE; # (붗; 붗; 붗; 붗; 붗; ) HANGUL SYLLABLE BUC +BD98;BD98;1107 116E 11BF;BD98;1107 116E 11BF; # (붘; 붘; 붘; 붘; 붘; ) HANGUL SYLLABLE BUK +BD99;BD99;1107 116E 11C0;BD99;1107 116E 11C0; # (붙; 붙; 붙; 붙; 붙; ) HANGUL SYLLABLE BUT +BD9A;BD9A;1107 116E 11C1;BD9A;1107 116E 11C1; # (붚; 붚; 붚; 붚; 붚; ) HANGUL SYLLABLE BUP +BD9B;BD9B;1107 116E 11C2;BD9B;1107 116E 11C2; # (붛; 붛; 붛; 붛; 붛; ) HANGUL SYLLABLE BUH +BD9C;BD9C;1107 116F;BD9C;1107 116F; # (붜; 붜; 붜; 붜; 붜; ) HANGUL SYLLABLE BWEO +BD9D;BD9D;1107 116F 11A8;BD9D;1107 116F 11A8; # (붝; 붝; 붝; 붝; 붝; ) HANGUL SYLLABLE BWEOG +BD9E;BD9E;1107 116F 11A9;BD9E;1107 116F 11A9; # (붞; 붞; 붞; 붞; 붞; ) HANGUL SYLLABLE BWEOGG +BD9F;BD9F;1107 116F 11AA;BD9F;1107 116F 11AA; # (붟; 붟; 붟; 붟; 붟; ) HANGUL SYLLABLE BWEOGS +BDA0;BDA0;1107 116F 11AB;BDA0;1107 116F 11AB; # (붠; 붠; 붠; 붠; 붠; ) HANGUL SYLLABLE BWEON +BDA1;BDA1;1107 116F 11AC;BDA1;1107 116F 11AC; # (붡; 붡; 붡; 붡; 붡; ) HANGUL SYLLABLE BWEONJ +BDA2;BDA2;1107 116F 11AD;BDA2;1107 116F 11AD; # (붢; 붢; 붢; 붢; 붢; ) HANGUL SYLLABLE BWEONH +BDA3;BDA3;1107 116F 11AE;BDA3;1107 116F 11AE; # (붣; 붣; 붣; 붣; 붣; ) HANGUL SYLLABLE BWEOD +BDA4;BDA4;1107 116F 11AF;BDA4;1107 116F 11AF; # (붤; 붤; 붤; 붤; 붤; ) HANGUL SYLLABLE BWEOL +BDA5;BDA5;1107 116F 11B0;BDA5;1107 116F 11B0; # (붥; 붥; 붥; 붥; 붥; ) HANGUL SYLLABLE BWEOLG +BDA6;BDA6;1107 116F 11B1;BDA6;1107 116F 11B1; # (붦; 붦; 붦; 붦; 붦; ) HANGUL SYLLABLE BWEOLM +BDA7;BDA7;1107 116F 11B2;BDA7;1107 116F 11B2; # (붧; 붧; 붧; 붧; 붧; ) HANGUL SYLLABLE BWEOLB +BDA8;BDA8;1107 116F 11B3;BDA8;1107 116F 11B3; # (붨; 붨; 붨; 붨; 붨; ) HANGUL SYLLABLE BWEOLS +BDA9;BDA9;1107 116F 11B4;BDA9;1107 116F 11B4; # (붩; 붩; 붩; 붩; 붩; ) HANGUL SYLLABLE BWEOLT +BDAA;BDAA;1107 116F 11B5;BDAA;1107 116F 11B5; # (붪; 붪; 붪; 붪; 붪; ) HANGUL SYLLABLE BWEOLP +BDAB;BDAB;1107 116F 11B6;BDAB;1107 116F 11B6; # (붫; 붫; 붫; 붫; 붫; ) HANGUL SYLLABLE BWEOLH +BDAC;BDAC;1107 116F 11B7;BDAC;1107 116F 11B7; # (붬; 붬; 붬; 붬; 붬; ) HANGUL SYLLABLE BWEOM +BDAD;BDAD;1107 116F 11B8;BDAD;1107 116F 11B8; # (붭; 붭; 붭; 붭; 붭; ) HANGUL SYLLABLE BWEOB +BDAE;BDAE;1107 116F 11B9;BDAE;1107 116F 11B9; # (붮; 붮; 붮; 붮; 붮; ) HANGUL SYLLABLE BWEOBS +BDAF;BDAF;1107 116F 11BA;BDAF;1107 116F 11BA; # (붯; 붯; 붯; 붯; 붯; ) HANGUL SYLLABLE BWEOS +BDB0;BDB0;1107 116F 11BB;BDB0;1107 116F 11BB; # (붰; 붰; 붰; 붰; 붰; ) HANGUL SYLLABLE BWEOSS +BDB1;BDB1;1107 116F 11BC;BDB1;1107 116F 11BC; # (붱; 붱; 붱; 붱; 붱; ) HANGUL SYLLABLE BWEONG +BDB2;BDB2;1107 116F 11BD;BDB2;1107 116F 11BD; # (붲; 붲; 붲; 붲; 붲; ) HANGUL SYLLABLE BWEOJ +BDB3;BDB3;1107 116F 11BE;BDB3;1107 116F 11BE; # (붳; 붳; 붳; 붳; 붳; ) HANGUL SYLLABLE BWEOC +BDB4;BDB4;1107 116F 11BF;BDB4;1107 116F 11BF; # (붴; 붴; 붴; 붴; 붴; ) HANGUL SYLLABLE BWEOK +BDB5;BDB5;1107 116F 11C0;BDB5;1107 116F 11C0; # (붵; 붵; 붵; 붵; 붵; ) HANGUL SYLLABLE BWEOT +BDB6;BDB6;1107 116F 11C1;BDB6;1107 116F 11C1; # (붶; 붶; 붶; 붶; 붶; ) HANGUL SYLLABLE BWEOP +BDB7;BDB7;1107 116F 11C2;BDB7;1107 116F 11C2; # (붷; 붷; 붷; 붷; 붷; ) HANGUL SYLLABLE BWEOH +BDB8;BDB8;1107 1170;BDB8;1107 1170; # (붸; 붸; 붸; 붸; 붸; ) HANGUL SYLLABLE BWE +BDB9;BDB9;1107 1170 11A8;BDB9;1107 1170 11A8; # (붹; 붹; 붹; 붹; 붹; ) HANGUL SYLLABLE BWEG +BDBA;BDBA;1107 1170 11A9;BDBA;1107 1170 11A9; # (붺; 붺; 붺; 붺; 붺; ) HANGUL SYLLABLE BWEGG +BDBB;BDBB;1107 1170 11AA;BDBB;1107 1170 11AA; # (붻; 붻; 붻; 붻; 붻; ) HANGUL SYLLABLE BWEGS +BDBC;BDBC;1107 1170 11AB;BDBC;1107 1170 11AB; # (붼; 붼; 붼; 붼; 붼; ) HANGUL SYLLABLE BWEN +BDBD;BDBD;1107 1170 11AC;BDBD;1107 1170 11AC; # (붽; 붽; 붽; 붽; 붽; ) HANGUL SYLLABLE BWENJ +BDBE;BDBE;1107 1170 11AD;BDBE;1107 1170 11AD; # (붾; 붾; 붾; 붾; 붾; ) HANGUL SYLLABLE BWENH +BDBF;BDBF;1107 1170 11AE;BDBF;1107 1170 11AE; # (붿; 붿; 붿; 붿; 붿; ) HANGUL SYLLABLE BWED +BDC0;BDC0;1107 1170 11AF;BDC0;1107 1170 11AF; # (뷀; 뷀; 뷀; 뷀; 뷀; ) HANGUL SYLLABLE BWEL +BDC1;BDC1;1107 1170 11B0;BDC1;1107 1170 11B0; # (뷁; 뷁; 뷁; 뷁; 뷁; ) HANGUL SYLLABLE BWELG +BDC2;BDC2;1107 1170 11B1;BDC2;1107 1170 11B1; # (뷂; 뷂; 뷂; 뷂; 뷂; ) HANGUL SYLLABLE BWELM +BDC3;BDC3;1107 1170 11B2;BDC3;1107 1170 11B2; # (뷃; 뷃; 뷃; 뷃; 뷃; ) HANGUL SYLLABLE BWELB +BDC4;BDC4;1107 1170 11B3;BDC4;1107 1170 11B3; # (뷄; 뷄; 뷄; 뷄; 뷄; ) HANGUL SYLLABLE BWELS +BDC5;BDC5;1107 1170 11B4;BDC5;1107 1170 11B4; # (뷅; 뷅; 뷅; 뷅; 뷅; ) HANGUL SYLLABLE BWELT +BDC6;BDC6;1107 1170 11B5;BDC6;1107 1170 11B5; # (뷆; 뷆; 뷆; 뷆; 뷆; ) HANGUL SYLLABLE BWELP +BDC7;BDC7;1107 1170 11B6;BDC7;1107 1170 11B6; # (뷇; 뷇; 뷇; 뷇; 뷇; ) HANGUL SYLLABLE BWELH +BDC8;BDC8;1107 1170 11B7;BDC8;1107 1170 11B7; # (뷈; 뷈; 뷈; 뷈; 뷈; ) HANGUL SYLLABLE BWEM +BDC9;BDC9;1107 1170 11B8;BDC9;1107 1170 11B8; # (뷉; 뷉; 뷉; 뷉; 뷉; ) HANGUL SYLLABLE BWEB +BDCA;BDCA;1107 1170 11B9;BDCA;1107 1170 11B9; # (뷊; 뷊; 뷊; 뷊; 뷊; ) HANGUL SYLLABLE BWEBS +BDCB;BDCB;1107 1170 11BA;BDCB;1107 1170 11BA; # (뷋; 뷋; 뷋; 뷋; 뷋; ) HANGUL SYLLABLE BWES +BDCC;BDCC;1107 1170 11BB;BDCC;1107 1170 11BB; # (뷌; 뷌; 뷌; 뷌; 뷌; ) HANGUL SYLLABLE BWESS +BDCD;BDCD;1107 1170 11BC;BDCD;1107 1170 11BC; # (뷍; 뷍; 뷍; 뷍; 뷍; ) HANGUL SYLLABLE BWENG +BDCE;BDCE;1107 1170 11BD;BDCE;1107 1170 11BD; # (뷎; 뷎; 뷎; 뷎; 뷎; ) HANGUL SYLLABLE BWEJ +BDCF;BDCF;1107 1170 11BE;BDCF;1107 1170 11BE; # (뷏; 뷏; 뷏; 뷏; 뷏; ) HANGUL SYLLABLE BWEC +BDD0;BDD0;1107 1170 11BF;BDD0;1107 1170 11BF; # (뷐; 뷐; 뷐; 뷐; 뷐; ) HANGUL SYLLABLE BWEK +BDD1;BDD1;1107 1170 11C0;BDD1;1107 1170 11C0; # (뷑; 뷑; 뷑; 뷑; 뷑; ) HANGUL SYLLABLE BWET +BDD2;BDD2;1107 1170 11C1;BDD2;1107 1170 11C1; # (뷒; 뷒; 뷒; 뷒; 뷒; ) HANGUL SYLLABLE BWEP +BDD3;BDD3;1107 1170 11C2;BDD3;1107 1170 11C2; # (뷓; 뷓; 뷓; 뷓; 뷓; ) HANGUL SYLLABLE BWEH +BDD4;BDD4;1107 1171;BDD4;1107 1171; # (뷔; 뷔; 뷔; 뷔; 뷔; ) HANGUL SYLLABLE BWI +BDD5;BDD5;1107 1171 11A8;BDD5;1107 1171 11A8; # (뷕; 뷕; 뷕; 뷕; 뷕; ) HANGUL SYLLABLE BWIG +BDD6;BDD6;1107 1171 11A9;BDD6;1107 1171 11A9; # (뷖; 뷖; 뷖; 뷖; 뷖; ) HANGUL SYLLABLE BWIGG +BDD7;BDD7;1107 1171 11AA;BDD7;1107 1171 11AA; # (뷗; 뷗; 뷗; 뷗; 뷗; ) HANGUL SYLLABLE BWIGS +BDD8;BDD8;1107 1171 11AB;BDD8;1107 1171 11AB; # (뷘; 뷘; 뷘; 뷘; 뷘; ) HANGUL SYLLABLE BWIN +BDD9;BDD9;1107 1171 11AC;BDD9;1107 1171 11AC; # (뷙; 뷙; 뷙; 뷙; 뷙; ) HANGUL SYLLABLE BWINJ +BDDA;BDDA;1107 1171 11AD;BDDA;1107 1171 11AD; # (뷚; 뷚; 뷚; 뷚; 뷚; ) HANGUL SYLLABLE BWINH +BDDB;BDDB;1107 1171 11AE;BDDB;1107 1171 11AE; # (뷛; 뷛; 뷛; 뷛; 뷛; ) HANGUL SYLLABLE BWID +BDDC;BDDC;1107 1171 11AF;BDDC;1107 1171 11AF; # (뷜; 뷜; 뷜; 뷜; 뷜; ) HANGUL SYLLABLE BWIL +BDDD;BDDD;1107 1171 11B0;BDDD;1107 1171 11B0; # (뷝; 뷝; 뷝; 뷝; 뷝; ) HANGUL SYLLABLE BWILG +BDDE;BDDE;1107 1171 11B1;BDDE;1107 1171 11B1; # (뷞; 뷞; 뷞; 뷞; 뷞; ) HANGUL SYLLABLE BWILM +BDDF;BDDF;1107 1171 11B2;BDDF;1107 1171 11B2; # (뷟; 뷟; 뷟; 뷟; 뷟; ) HANGUL SYLLABLE BWILB +BDE0;BDE0;1107 1171 11B3;BDE0;1107 1171 11B3; # (뷠; 뷠; 뷠; 뷠; 뷠; ) HANGUL SYLLABLE BWILS +BDE1;BDE1;1107 1171 11B4;BDE1;1107 1171 11B4; # (뷡; 뷡; 뷡; 뷡; 뷡; ) HANGUL SYLLABLE BWILT +BDE2;BDE2;1107 1171 11B5;BDE2;1107 1171 11B5; # (뷢; 뷢; 뷢; 뷢; 뷢; ) HANGUL SYLLABLE BWILP +BDE3;BDE3;1107 1171 11B6;BDE3;1107 1171 11B6; # (뷣; 뷣; 뷣; 뷣; 뷣; ) HANGUL SYLLABLE BWILH +BDE4;BDE4;1107 1171 11B7;BDE4;1107 1171 11B7; # (뷤; 뷤; 뷤; 뷤; 뷤; ) HANGUL SYLLABLE BWIM +BDE5;BDE5;1107 1171 11B8;BDE5;1107 1171 11B8; # (뷥; 뷥; 뷥; 뷥; 뷥; ) HANGUL SYLLABLE BWIB +BDE6;BDE6;1107 1171 11B9;BDE6;1107 1171 11B9; # (뷦; 뷦; 뷦; 뷦; 뷦; ) HANGUL SYLLABLE BWIBS +BDE7;BDE7;1107 1171 11BA;BDE7;1107 1171 11BA; # (뷧; 뷧; 뷧; 뷧; 뷧; ) HANGUL SYLLABLE BWIS +BDE8;BDE8;1107 1171 11BB;BDE8;1107 1171 11BB; # (뷨; 뷨; 뷨; 뷨; 뷨; ) HANGUL SYLLABLE BWISS +BDE9;BDE9;1107 1171 11BC;BDE9;1107 1171 11BC; # (뷩; 뷩; 뷩; 뷩; 뷩; ) HANGUL SYLLABLE BWING +BDEA;BDEA;1107 1171 11BD;BDEA;1107 1171 11BD; # (뷪; 뷪; 뷪; 뷪; 뷪; ) HANGUL SYLLABLE BWIJ +BDEB;BDEB;1107 1171 11BE;BDEB;1107 1171 11BE; # (뷫; 뷫; 뷫; 뷫; 뷫; ) HANGUL SYLLABLE BWIC +BDEC;BDEC;1107 1171 11BF;BDEC;1107 1171 11BF; # (뷬; 뷬; 뷬; 뷬; 뷬; ) HANGUL SYLLABLE BWIK +BDED;BDED;1107 1171 11C0;BDED;1107 1171 11C0; # (뷭; 뷭; 뷭; 뷭; 뷭; ) HANGUL SYLLABLE BWIT +BDEE;BDEE;1107 1171 11C1;BDEE;1107 1171 11C1; # (뷮; 뷮; 뷮; 뷮; 뷮; ) HANGUL SYLLABLE BWIP +BDEF;BDEF;1107 1171 11C2;BDEF;1107 1171 11C2; # (뷯; 뷯; 뷯; 뷯; 뷯; ) HANGUL SYLLABLE BWIH +BDF0;BDF0;1107 1172;BDF0;1107 1172; # (뷰; 뷰; 뷰; 뷰; 뷰; ) HANGUL SYLLABLE BYU +BDF1;BDF1;1107 1172 11A8;BDF1;1107 1172 11A8; # (뷱; 뷱; 뷱; 뷱; 뷱; ) HANGUL SYLLABLE BYUG +BDF2;BDF2;1107 1172 11A9;BDF2;1107 1172 11A9; # (뷲; 뷲; 뷲; 뷲; 뷲; ) HANGUL SYLLABLE BYUGG +BDF3;BDF3;1107 1172 11AA;BDF3;1107 1172 11AA; # (뷳; 뷳; 뷳; 뷳; 뷳; ) HANGUL SYLLABLE BYUGS +BDF4;BDF4;1107 1172 11AB;BDF4;1107 1172 11AB; # (뷴; 뷴; 뷴; 뷴; 뷴; ) HANGUL SYLLABLE BYUN +BDF5;BDF5;1107 1172 11AC;BDF5;1107 1172 11AC; # (뷵; 뷵; 뷵; 뷵; 뷵; ) HANGUL SYLLABLE BYUNJ +BDF6;BDF6;1107 1172 11AD;BDF6;1107 1172 11AD; # (뷶; 뷶; 뷶; 뷶; 뷶; ) HANGUL SYLLABLE BYUNH +BDF7;BDF7;1107 1172 11AE;BDF7;1107 1172 11AE; # (뷷; 뷷; 뷷; 뷷; 뷷; ) HANGUL SYLLABLE BYUD +BDF8;BDF8;1107 1172 11AF;BDF8;1107 1172 11AF; # (뷸; 뷸; 뷸; 뷸; 뷸; ) HANGUL SYLLABLE BYUL +BDF9;BDF9;1107 1172 11B0;BDF9;1107 1172 11B0; # (뷹; 뷹; 뷹; 뷹; 뷹; ) HANGUL SYLLABLE BYULG +BDFA;BDFA;1107 1172 11B1;BDFA;1107 1172 11B1; # (뷺; 뷺; 뷺; 뷺; 뷺; ) HANGUL SYLLABLE BYULM +BDFB;BDFB;1107 1172 11B2;BDFB;1107 1172 11B2; # (뷻; 뷻; 뷻; 뷻; 뷻; ) HANGUL SYLLABLE BYULB +BDFC;BDFC;1107 1172 11B3;BDFC;1107 1172 11B3; # (뷼; 뷼; 뷼; 뷼; 뷼; ) HANGUL SYLLABLE BYULS +BDFD;BDFD;1107 1172 11B4;BDFD;1107 1172 11B4; # (뷽; 뷽; 뷽; 뷽; 뷽; ) HANGUL SYLLABLE BYULT +BDFE;BDFE;1107 1172 11B5;BDFE;1107 1172 11B5; # (뷾; 뷾; 뷾; 뷾; 뷾; ) HANGUL SYLLABLE BYULP +BDFF;BDFF;1107 1172 11B6;BDFF;1107 1172 11B6; # (뷿; 뷿; 뷿; 뷿; 뷿; ) HANGUL SYLLABLE BYULH +BE00;BE00;1107 1172 11B7;BE00;1107 1172 11B7; # (븀; 븀; 븀; 븀; 븀; ) HANGUL SYLLABLE BYUM +BE01;BE01;1107 1172 11B8;BE01;1107 1172 11B8; # (븁; 븁; 븁; 븁; 븁; ) HANGUL SYLLABLE BYUB +BE02;BE02;1107 1172 11B9;BE02;1107 1172 11B9; # (븂; 븂; 븂; 븂; 븂; ) HANGUL SYLLABLE BYUBS +BE03;BE03;1107 1172 11BA;BE03;1107 1172 11BA; # (븃; 븃; 븃; 븃; 븃; ) HANGUL SYLLABLE BYUS +BE04;BE04;1107 1172 11BB;BE04;1107 1172 11BB; # (븄; 븄; 븄; 븄; 븄; ) HANGUL SYLLABLE BYUSS +BE05;BE05;1107 1172 11BC;BE05;1107 1172 11BC; # (븅; 븅; 븅; 븅; 븅; ) HANGUL SYLLABLE BYUNG +BE06;BE06;1107 1172 11BD;BE06;1107 1172 11BD; # (븆; 븆; 븆; 븆; 븆; ) HANGUL SYLLABLE BYUJ +BE07;BE07;1107 1172 11BE;BE07;1107 1172 11BE; # (븇; 븇; 븇; 븇; 븇; ) HANGUL SYLLABLE BYUC +BE08;BE08;1107 1172 11BF;BE08;1107 1172 11BF; # (븈; 븈; 븈; 븈; 븈; ) HANGUL SYLLABLE BYUK +BE09;BE09;1107 1172 11C0;BE09;1107 1172 11C0; # (븉; 븉; 븉; 븉; 븉; ) HANGUL SYLLABLE BYUT +BE0A;BE0A;1107 1172 11C1;BE0A;1107 1172 11C1; # (븊; 븊; 븊; 븊; 븊; ) HANGUL SYLLABLE BYUP +BE0B;BE0B;1107 1172 11C2;BE0B;1107 1172 11C2; # (븋; 븋; 븋; 븋; 븋; ) HANGUL SYLLABLE BYUH +BE0C;BE0C;1107 1173;BE0C;1107 1173; # (브; 브; 브; 브; 브; ) HANGUL SYLLABLE BEU +BE0D;BE0D;1107 1173 11A8;BE0D;1107 1173 11A8; # (븍; 븍; 븍; 븍; 븍; ) HANGUL SYLLABLE BEUG +BE0E;BE0E;1107 1173 11A9;BE0E;1107 1173 11A9; # (븎; 븎; 븎; 븎; 븎; ) HANGUL SYLLABLE BEUGG +BE0F;BE0F;1107 1173 11AA;BE0F;1107 1173 11AA; # (븏; 븏; 븏; 븏; 븏; ) HANGUL SYLLABLE BEUGS +BE10;BE10;1107 1173 11AB;BE10;1107 1173 11AB; # (븐; 븐; 븐; 븐; 븐; ) HANGUL SYLLABLE BEUN +BE11;BE11;1107 1173 11AC;BE11;1107 1173 11AC; # (븑; 븑; 븑; 븑; 븑; ) HANGUL SYLLABLE BEUNJ +BE12;BE12;1107 1173 11AD;BE12;1107 1173 11AD; # (븒; 븒; 븒; 븒; 븒; ) HANGUL SYLLABLE BEUNH +BE13;BE13;1107 1173 11AE;BE13;1107 1173 11AE; # (븓; 븓; 븓; 븓; 븓; ) HANGUL SYLLABLE BEUD +BE14;BE14;1107 1173 11AF;BE14;1107 1173 11AF; # (블; 블; 블; 블; 블; ) HANGUL SYLLABLE BEUL +BE15;BE15;1107 1173 11B0;BE15;1107 1173 11B0; # (븕; 븕; 븕; 븕; 븕; ) HANGUL SYLLABLE BEULG +BE16;BE16;1107 1173 11B1;BE16;1107 1173 11B1; # (븖; 븖; 븖; 븖; 븖; ) HANGUL SYLLABLE BEULM +BE17;BE17;1107 1173 11B2;BE17;1107 1173 11B2; # (븗; 븗; 븗; 븗; 븗; ) HANGUL SYLLABLE BEULB +BE18;BE18;1107 1173 11B3;BE18;1107 1173 11B3; # (븘; 븘; 븘; 븘; 븘; ) HANGUL SYLLABLE BEULS +BE19;BE19;1107 1173 11B4;BE19;1107 1173 11B4; # (븙; 븙; 븙; 븙; 븙; ) HANGUL SYLLABLE BEULT +BE1A;BE1A;1107 1173 11B5;BE1A;1107 1173 11B5; # (븚; 븚; 븚; 븚; 븚; ) HANGUL SYLLABLE BEULP +BE1B;BE1B;1107 1173 11B6;BE1B;1107 1173 11B6; # (븛; 븛; 븛; 븛; 븛; ) HANGUL SYLLABLE BEULH +BE1C;BE1C;1107 1173 11B7;BE1C;1107 1173 11B7; # (븜; 븜; 븜; 븜; 븜; ) HANGUL SYLLABLE BEUM +BE1D;BE1D;1107 1173 11B8;BE1D;1107 1173 11B8; # (븝; 븝; 븝; 븝; 븝; ) HANGUL SYLLABLE BEUB +BE1E;BE1E;1107 1173 11B9;BE1E;1107 1173 11B9; # (븞; 븞; 븞; 븞; 븞; ) HANGUL SYLLABLE BEUBS +BE1F;BE1F;1107 1173 11BA;BE1F;1107 1173 11BA; # (븟; 븟; 븟; 븟; 븟; ) HANGUL SYLLABLE BEUS +BE20;BE20;1107 1173 11BB;BE20;1107 1173 11BB; # (븠; 븠; 븠; 븠; 븠; ) HANGUL SYLLABLE BEUSS +BE21;BE21;1107 1173 11BC;BE21;1107 1173 11BC; # (븡; 븡; 븡; 븡; 븡; ) HANGUL SYLLABLE BEUNG +BE22;BE22;1107 1173 11BD;BE22;1107 1173 11BD; # (븢; 븢; 븢; 븢; 븢; ) HANGUL SYLLABLE BEUJ +BE23;BE23;1107 1173 11BE;BE23;1107 1173 11BE; # (븣; 븣; 븣; 븣; 븣; ) HANGUL SYLLABLE BEUC +BE24;BE24;1107 1173 11BF;BE24;1107 1173 11BF; # (븤; 븤; 븤; 븤; 븤; ) HANGUL SYLLABLE BEUK +BE25;BE25;1107 1173 11C0;BE25;1107 1173 11C0; # (븥; 븥; 븥; 븥; 븥; ) HANGUL SYLLABLE BEUT +BE26;BE26;1107 1173 11C1;BE26;1107 1173 11C1; # (븦; 븦; 븦; 븦; 븦; ) HANGUL SYLLABLE BEUP +BE27;BE27;1107 1173 11C2;BE27;1107 1173 11C2; # (븧; 븧; 븧; 븧; 븧; ) HANGUL SYLLABLE BEUH +BE28;BE28;1107 1174;BE28;1107 1174; # (븨; 븨; 븨; 븨; 븨; ) HANGUL SYLLABLE BYI +BE29;BE29;1107 1174 11A8;BE29;1107 1174 11A8; # (븩; 븩; 븩; 븩; 븩; ) HANGUL SYLLABLE BYIG +BE2A;BE2A;1107 1174 11A9;BE2A;1107 1174 11A9; # (븪; 븪; 븪; 븪; 븪; ) HANGUL SYLLABLE BYIGG +BE2B;BE2B;1107 1174 11AA;BE2B;1107 1174 11AA; # (븫; 븫; 븫; 븫; 븫; ) HANGUL SYLLABLE BYIGS +BE2C;BE2C;1107 1174 11AB;BE2C;1107 1174 11AB; # (븬; 븬; 븬; 븬; 븬; ) HANGUL SYLLABLE BYIN +BE2D;BE2D;1107 1174 11AC;BE2D;1107 1174 11AC; # (븭; 븭; 븭; 븭; 븭; ) HANGUL SYLLABLE BYINJ +BE2E;BE2E;1107 1174 11AD;BE2E;1107 1174 11AD; # (븮; 븮; 븮; 븮; 븮; ) HANGUL SYLLABLE BYINH +BE2F;BE2F;1107 1174 11AE;BE2F;1107 1174 11AE; # (븯; 븯; 븯; 븯; 븯; ) HANGUL SYLLABLE BYID +BE30;BE30;1107 1174 11AF;BE30;1107 1174 11AF; # (븰; 븰; 븰; 븰; 븰; ) HANGUL SYLLABLE BYIL +BE31;BE31;1107 1174 11B0;BE31;1107 1174 11B0; # (븱; 븱; 븱; 븱; 븱; ) HANGUL SYLLABLE BYILG +BE32;BE32;1107 1174 11B1;BE32;1107 1174 11B1; # (븲; 븲; 븲; 븲; 븲; ) HANGUL SYLLABLE BYILM +BE33;BE33;1107 1174 11B2;BE33;1107 1174 11B2; # (븳; 븳; 븳; 븳; 븳; ) HANGUL SYLLABLE BYILB +BE34;BE34;1107 1174 11B3;BE34;1107 1174 11B3; # (븴; 븴; 븴; 븴; 븴; ) HANGUL SYLLABLE BYILS +BE35;BE35;1107 1174 11B4;BE35;1107 1174 11B4; # (븵; 븵; 븵; 븵; 븵; ) HANGUL SYLLABLE BYILT +BE36;BE36;1107 1174 11B5;BE36;1107 1174 11B5; # (븶; 븶; 븶; 븶; 븶; ) HANGUL SYLLABLE BYILP +BE37;BE37;1107 1174 11B6;BE37;1107 1174 11B6; # (븷; 븷; 븷; 븷; 븷; ) HANGUL SYLLABLE BYILH +BE38;BE38;1107 1174 11B7;BE38;1107 1174 11B7; # (븸; 븸; 븸; 븸; 븸; ) HANGUL SYLLABLE BYIM +BE39;BE39;1107 1174 11B8;BE39;1107 1174 11B8; # (븹; 븹; 븹; 븹; 븹; ) HANGUL SYLLABLE BYIB +BE3A;BE3A;1107 1174 11B9;BE3A;1107 1174 11B9; # (븺; 븺; 븺; 븺; 븺; ) HANGUL SYLLABLE BYIBS +BE3B;BE3B;1107 1174 11BA;BE3B;1107 1174 11BA; # (븻; 븻; 븻; 븻; 븻; ) HANGUL SYLLABLE BYIS +BE3C;BE3C;1107 1174 11BB;BE3C;1107 1174 11BB; # (븼; 븼; 븼; 븼; 븼; ) HANGUL SYLLABLE BYISS +BE3D;BE3D;1107 1174 11BC;BE3D;1107 1174 11BC; # (븽; 븽; 븽; 븽; 븽; ) HANGUL SYLLABLE BYING +BE3E;BE3E;1107 1174 11BD;BE3E;1107 1174 11BD; # (븾; 븾; 븾; 븾; 븾; ) HANGUL SYLLABLE BYIJ +BE3F;BE3F;1107 1174 11BE;BE3F;1107 1174 11BE; # (븿; 븿; 븿; 븿; 븿; ) HANGUL SYLLABLE BYIC +BE40;BE40;1107 1174 11BF;BE40;1107 1174 11BF; # (빀; 빀; 빀; 빀; 빀; ) HANGUL SYLLABLE BYIK +BE41;BE41;1107 1174 11C0;BE41;1107 1174 11C0; # (빁; 빁; 빁; 빁; 빁; ) HANGUL SYLLABLE BYIT +BE42;BE42;1107 1174 11C1;BE42;1107 1174 11C1; # (빂; 빂; 빂; 빂; 빂; ) HANGUL SYLLABLE BYIP +BE43;BE43;1107 1174 11C2;BE43;1107 1174 11C2; # (빃; 빃; 빃; 빃; 빃; ) HANGUL SYLLABLE BYIH +BE44;BE44;1107 1175;BE44;1107 1175; # (비; 비; 비; 비; 비; ) HANGUL SYLLABLE BI +BE45;BE45;1107 1175 11A8;BE45;1107 1175 11A8; # (빅; 빅; 빅; 빅; 빅; ) HANGUL SYLLABLE BIG +BE46;BE46;1107 1175 11A9;BE46;1107 1175 11A9; # (빆; 빆; 빆; 빆; 빆; ) HANGUL SYLLABLE BIGG +BE47;BE47;1107 1175 11AA;BE47;1107 1175 11AA; # (빇; 빇; 빇; 빇; 빇; ) HANGUL SYLLABLE BIGS +BE48;BE48;1107 1175 11AB;BE48;1107 1175 11AB; # (빈; 빈; 빈; 빈; 빈; ) HANGUL SYLLABLE BIN +BE49;BE49;1107 1175 11AC;BE49;1107 1175 11AC; # (빉; 빉; 빉; 빉; 빉; ) HANGUL SYLLABLE BINJ +BE4A;BE4A;1107 1175 11AD;BE4A;1107 1175 11AD; # (빊; 빊; 빊; 빊; 빊; ) HANGUL SYLLABLE BINH +BE4B;BE4B;1107 1175 11AE;BE4B;1107 1175 11AE; # (빋; 빋; 빋; 빋; 빋; ) HANGUL SYLLABLE BID +BE4C;BE4C;1107 1175 11AF;BE4C;1107 1175 11AF; # (빌; 빌; 빌; 빌; 빌; ) HANGUL SYLLABLE BIL +BE4D;BE4D;1107 1175 11B0;BE4D;1107 1175 11B0; # (빍; 빍; 빍; 빍; 빍; ) HANGUL SYLLABLE BILG +BE4E;BE4E;1107 1175 11B1;BE4E;1107 1175 11B1; # (빎; 빎; 빎; 빎; 빎; ) HANGUL SYLLABLE BILM +BE4F;BE4F;1107 1175 11B2;BE4F;1107 1175 11B2; # (빏; 빏; 빏; 빏; 빏; ) HANGUL SYLLABLE BILB +BE50;BE50;1107 1175 11B3;BE50;1107 1175 11B3; # (빐; 빐; 빐; 빐; 빐; ) HANGUL SYLLABLE BILS +BE51;BE51;1107 1175 11B4;BE51;1107 1175 11B4; # (빑; 빑; 빑; 빑; 빑; ) HANGUL SYLLABLE BILT +BE52;BE52;1107 1175 11B5;BE52;1107 1175 11B5; # (빒; 빒; 빒; 빒; 빒; ) HANGUL SYLLABLE BILP +BE53;BE53;1107 1175 11B6;BE53;1107 1175 11B6; # (빓; 빓; 빓; 빓; 빓; ) HANGUL SYLLABLE BILH +BE54;BE54;1107 1175 11B7;BE54;1107 1175 11B7; # (빔; 빔; 빔; 빔; 빔; ) HANGUL SYLLABLE BIM +BE55;BE55;1107 1175 11B8;BE55;1107 1175 11B8; # (빕; 빕; 빕; 빕; 빕; ) HANGUL SYLLABLE BIB +BE56;BE56;1107 1175 11B9;BE56;1107 1175 11B9; # (빖; 빖; 빖; 빖; 빖; ) HANGUL SYLLABLE BIBS +BE57;BE57;1107 1175 11BA;BE57;1107 1175 11BA; # (빗; 빗; 빗; 빗; 빗; ) HANGUL SYLLABLE BIS +BE58;BE58;1107 1175 11BB;BE58;1107 1175 11BB; # (빘; 빘; 빘; 빘; 빘; ) HANGUL SYLLABLE BISS +BE59;BE59;1107 1175 11BC;BE59;1107 1175 11BC; # (빙; 빙; 빙; 빙; 빙; ) HANGUL SYLLABLE BING +BE5A;BE5A;1107 1175 11BD;BE5A;1107 1175 11BD; # (빚; 빚; 빚; 빚; 빚; ) HANGUL SYLLABLE BIJ +BE5B;BE5B;1107 1175 11BE;BE5B;1107 1175 11BE; # (빛; 빛; 빛; 빛; 빛; ) HANGUL SYLLABLE BIC +BE5C;BE5C;1107 1175 11BF;BE5C;1107 1175 11BF; # (빜; 빜; 빜; 빜; 빜; ) HANGUL SYLLABLE BIK +BE5D;BE5D;1107 1175 11C0;BE5D;1107 1175 11C0; # (빝; 빝; 빝; 빝; 빝; ) HANGUL SYLLABLE BIT +BE5E;BE5E;1107 1175 11C1;BE5E;1107 1175 11C1; # (빞; 빞; 빞; 빞; 빞; ) HANGUL SYLLABLE BIP +BE5F;BE5F;1107 1175 11C2;BE5F;1107 1175 11C2; # (빟; 빟; 빟; 빟; 빟; ) HANGUL SYLLABLE BIH +BE60;BE60;1108 1161;BE60;1108 1161; # (빠; 빠; 빠; 빠; 빠; ) HANGUL SYLLABLE BBA +BE61;BE61;1108 1161 11A8;BE61;1108 1161 11A8; # (빡; 빡; 빡; 빡; 빡; ) HANGUL SYLLABLE BBAG +BE62;BE62;1108 1161 11A9;BE62;1108 1161 11A9; # (빢; 빢; 빢; 빢; 빢; ) HANGUL SYLLABLE BBAGG +BE63;BE63;1108 1161 11AA;BE63;1108 1161 11AA; # (빣; 빣; 빣; 빣; 빣; ) HANGUL SYLLABLE BBAGS +BE64;BE64;1108 1161 11AB;BE64;1108 1161 11AB; # (빤; 빤; 빤; 빤; 빤; ) HANGUL SYLLABLE BBAN +BE65;BE65;1108 1161 11AC;BE65;1108 1161 11AC; # (빥; 빥; 빥; 빥; 빥; ) HANGUL SYLLABLE BBANJ +BE66;BE66;1108 1161 11AD;BE66;1108 1161 11AD; # (빦; 빦; 빦; 빦; 빦; ) HANGUL SYLLABLE BBANH +BE67;BE67;1108 1161 11AE;BE67;1108 1161 11AE; # (빧; 빧; 빧; 빧; 빧; ) HANGUL SYLLABLE BBAD +BE68;BE68;1108 1161 11AF;BE68;1108 1161 11AF; # (빨; 빨; 빨; 빨; 빨; ) HANGUL SYLLABLE BBAL +BE69;BE69;1108 1161 11B0;BE69;1108 1161 11B0; # (빩; 빩; 빩; 빩; 빩; ) HANGUL SYLLABLE BBALG +BE6A;BE6A;1108 1161 11B1;BE6A;1108 1161 11B1; # (빪; 빪; 빪; 빪; 빪; ) HANGUL SYLLABLE BBALM +BE6B;BE6B;1108 1161 11B2;BE6B;1108 1161 11B2; # (빫; 빫; 빫; 빫; 빫; ) HANGUL SYLLABLE BBALB +BE6C;BE6C;1108 1161 11B3;BE6C;1108 1161 11B3; # (빬; 빬; 빬; 빬; 빬; ) HANGUL SYLLABLE BBALS +BE6D;BE6D;1108 1161 11B4;BE6D;1108 1161 11B4; # (빭; 빭; 빭; 빭; 빭; ) HANGUL SYLLABLE BBALT +BE6E;BE6E;1108 1161 11B5;BE6E;1108 1161 11B5; # (빮; 빮; 빮; 빮; 빮; ) HANGUL SYLLABLE BBALP +BE6F;BE6F;1108 1161 11B6;BE6F;1108 1161 11B6; # (빯; 빯; 빯; 빯; 빯; ) HANGUL SYLLABLE BBALH +BE70;BE70;1108 1161 11B7;BE70;1108 1161 11B7; # (빰; 빰; 빰; 빰; 빰; ) HANGUL SYLLABLE BBAM +BE71;BE71;1108 1161 11B8;BE71;1108 1161 11B8; # (빱; 빱; 빱; 빱; 빱; ) HANGUL SYLLABLE BBAB +BE72;BE72;1108 1161 11B9;BE72;1108 1161 11B9; # (빲; 빲; 빲; 빲; 빲; ) HANGUL SYLLABLE BBABS +BE73;BE73;1108 1161 11BA;BE73;1108 1161 11BA; # (빳; 빳; 빳; 빳; 빳; ) HANGUL SYLLABLE BBAS +BE74;BE74;1108 1161 11BB;BE74;1108 1161 11BB; # (빴; 빴; 빴; 빴; 빴; ) HANGUL SYLLABLE BBASS +BE75;BE75;1108 1161 11BC;BE75;1108 1161 11BC; # (빵; 빵; 빵; 빵; 빵; ) HANGUL SYLLABLE BBANG +BE76;BE76;1108 1161 11BD;BE76;1108 1161 11BD; # (빶; 빶; 빶; 빶; 빶; ) HANGUL SYLLABLE BBAJ +BE77;BE77;1108 1161 11BE;BE77;1108 1161 11BE; # (빷; 빷; 빷; 빷; 빷; ) HANGUL SYLLABLE BBAC +BE78;BE78;1108 1161 11BF;BE78;1108 1161 11BF; # (빸; 빸; 빸; 빸; 빸; ) HANGUL SYLLABLE BBAK +BE79;BE79;1108 1161 11C0;BE79;1108 1161 11C0; # (빹; 빹; 빹; 빹; 빹; ) HANGUL SYLLABLE BBAT +BE7A;BE7A;1108 1161 11C1;BE7A;1108 1161 11C1; # (빺; 빺; 빺; 빺; 빺; ) HANGUL SYLLABLE BBAP +BE7B;BE7B;1108 1161 11C2;BE7B;1108 1161 11C2; # (빻; 빻; 빻; 빻; 빻; ) HANGUL SYLLABLE BBAH +BE7C;BE7C;1108 1162;BE7C;1108 1162; # (빼; 빼; 빼; 빼; 빼; ) HANGUL SYLLABLE BBAE +BE7D;BE7D;1108 1162 11A8;BE7D;1108 1162 11A8; # (빽; 빽; 빽; 빽; 빽; ) HANGUL SYLLABLE BBAEG +BE7E;BE7E;1108 1162 11A9;BE7E;1108 1162 11A9; # (빾; 빾; 빾; 빾; 빾; ) HANGUL SYLLABLE BBAEGG +BE7F;BE7F;1108 1162 11AA;BE7F;1108 1162 11AA; # (빿; 빿; 빿; 빿; 빿; ) HANGUL SYLLABLE BBAEGS +BE80;BE80;1108 1162 11AB;BE80;1108 1162 11AB; # (뺀; 뺀; 뺀; 뺀; 뺀; ) HANGUL SYLLABLE BBAEN +BE81;BE81;1108 1162 11AC;BE81;1108 1162 11AC; # (뺁; 뺁; 뺁; 뺁; 뺁; ) HANGUL SYLLABLE BBAENJ +BE82;BE82;1108 1162 11AD;BE82;1108 1162 11AD; # (뺂; 뺂; 뺂; 뺂; 뺂; ) HANGUL SYLLABLE BBAENH +BE83;BE83;1108 1162 11AE;BE83;1108 1162 11AE; # (뺃; 뺃; 뺃; 뺃; 뺃; ) HANGUL SYLLABLE BBAED +BE84;BE84;1108 1162 11AF;BE84;1108 1162 11AF; # (뺄; 뺄; 뺄; 뺄; 뺄; ) HANGUL SYLLABLE BBAEL +BE85;BE85;1108 1162 11B0;BE85;1108 1162 11B0; # (뺅; 뺅; 뺅; 뺅; 뺅; ) HANGUL SYLLABLE BBAELG +BE86;BE86;1108 1162 11B1;BE86;1108 1162 11B1; # (뺆; 뺆; 뺆; 뺆; 뺆; ) HANGUL SYLLABLE BBAELM +BE87;BE87;1108 1162 11B2;BE87;1108 1162 11B2; # (뺇; 뺇; 뺇; 뺇; 뺇; ) HANGUL SYLLABLE BBAELB +BE88;BE88;1108 1162 11B3;BE88;1108 1162 11B3; # (뺈; 뺈; 뺈; 뺈; 뺈; ) HANGUL SYLLABLE BBAELS +BE89;BE89;1108 1162 11B4;BE89;1108 1162 11B4; # (뺉; 뺉; 뺉; 뺉; 뺉; ) HANGUL SYLLABLE BBAELT +BE8A;BE8A;1108 1162 11B5;BE8A;1108 1162 11B5; # (뺊; 뺊; 뺊; 뺊; 뺊; ) HANGUL SYLLABLE BBAELP +BE8B;BE8B;1108 1162 11B6;BE8B;1108 1162 11B6; # (뺋; 뺋; 뺋; 뺋; 뺋; ) HANGUL SYLLABLE BBAELH +BE8C;BE8C;1108 1162 11B7;BE8C;1108 1162 11B7; # (뺌; 뺌; 뺌; 뺌; 뺌; ) HANGUL SYLLABLE BBAEM +BE8D;BE8D;1108 1162 11B8;BE8D;1108 1162 11B8; # (뺍; 뺍; 뺍; 뺍; 뺍; ) HANGUL SYLLABLE BBAEB +BE8E;BE8E;1108 1162 11B9;BE8E;1108 1162 11B9; # (뺎; 뺎; 뺎; 뺎; 뺎; ) HANGUL SYLLABLE BBAEBS +BE8F;BE8F;1108 1162 11BA;BE8F;1108 1162 11BA; # (뺏; 뺏; 뺏; 뺏; 뺏; ) HANGUL SYLLABLE BBAES +BE90;BE90;1108 1162 11BB;BE90;1108 1162 11BB; # (뺐; 뺐; 뺐; 뺐; 뺐; ) HANGUL SYLLABLE BBAESS +BE91;BE91;1108 1162 11BC;BE91;1108 1162 11BC; # (뺑; 뺑; 뺑; 뺑; 뺑; ) HANGUL SYLLABLE BBAENG +BE92;BE92;1108 1162 11BD;BE92;1108 1162 11BD; # (뺒; 뺒; 뺒; 뺒; 뺒; ) HANGUL SYLLABLE BBAEJ +BE93;BE93;1108 1162 11BE;BE93;1108 1162 11BE; # (뺓; 뺓; 뺓; 뺓; 뺓; ) HANGUL SYLLABLE BBAEC +BE94;BE94;1108 1162 11BF;BE94;1108 1162 11BF; # (뺔; 뺔; 뺔; 뺔; 뺔; ) HANGUL SYLLABLE BBAEK +BE95;BE95;1108 1162 11C0;BE95;1108 1162 11C0; # (뺕; 뺕; 뺕; 뺕; 뺕; ) HANGUL SYLLABLE BBAET +BE96;BE96;1108 1162 11C1;BE96;1108 1162 11C1; # (뺖; 뺖; 뺖; 뺖; 뺖; ) HANGUL SYLLABLE BBAEP +BE97;BE97;1108 1162 11C2;BE97;1108 1162 11C2; # (뺗; 뺗; 뺗; 뺗; 뺗; ) HANGUL SYLLABLE BBAEH +BE98;BE98;1108 1163;BE98;1108 1163; # (뺘; 뺘; 뺘; 뺘; 뺘; ) HANGUL SYLLABLE BBYA +BE99;BE99;1108 1163 11A8;BE99;1108 1163 11A8; # (뺙; 뺙; 뺙; 뺙; 뺙; ) HANGUL SYLLABLE BBYAG +BE9A;BE9A;1108 1163 11A9;BE9A;1108 1163 11A9; # (뺚; 뺚; 뺚; 뺚; 뺚; ) HANGUL SYLLABLE BBYAGG +BE9B;BE9B;1108 1163 11AA;BE9B;1108 1163 11AA; # (뺛; 뺛; 뺛; 뺛; 뺛; ) HANGUL SYLLABLE BBYAGS +BE9C;BE9C;1108 1163 11AB;BE9C;1108 1163 11AB; # (뺜; 뺜; 뺜; 뺜; 뺜; ) HANGUL SYLLABLE BBYAN +BE9D;BE9D;1108 1163 11AC;BE9D;1108 1163 11AC; # (뺝; 뺝; 뺝; 뺝; 뺝; ) HANGUL SYLLABLE BBYANJ +BE9E;BE9E;1108 1163 11AD;BE9E;1108 1163 11AD; # (뺞; 뺞; 뺞; 뺞; 뺞; ) HANGUL SYLLABLE BBYANH +BE9F;BE9F;1108 1163 11AE;BE9F;1108 1163 11AE; # (뺟; 뺟; 뺟; 뺟; 뺟; ) HANGUL SYLLABLE BBYAD +BEA0;BEA0;1108 1163 11AF;BEA0;1108 1163 11AF; # (뺠; 뺠; 뺠; 뺠; 뺠; ) HANGUL SYLLABLE BBYAL +BEA1;BEA1;1108 1163 11B0;BEA1;1108 1163 11B0; # (뺡; 뺡; 뺡; 뺡; 뺡; ) HANGUL SYLLABLE BBYALG +BEA2;BEA2;1108 1163 11B1;BEA2;1108 1163 11B1; # (뺢; 뺢; 뺢; 뺢; 뺢; ) HANGUL SYLLABLE BBYALM +BEA3;BEA3;1108 1163 11B2;BEA3;1108 1163 11B2; # (뺣; 뺣; 뺣; 뺣; 뺣; ) HANGUL SYLLABLE BBYALB +BEA4;BEA4;1108 1163 11B3;BEA4;1108 1163 11B3; # (뺤; 뺤; 뺤; 뺤; 뺤; ) HANGUL SYLLABLE BBYALS +BEA5;BEA5;1108 1163 11B4;BEA5;1108 1163 11B4; # (뺥; 뺥; 뺥; 뺥; 뺥; ) HANGUL SYLLABLE BBYALT +BEA6;BEA6;1108 1163 11B5;BEA6;1108 1163 11B5; # (뺦; 뺦; 뺦; 뺦; 뺦; ) HANGUL SYLLABLE BBYALP +BEA7;BEA7;1108 1163 11B6;BEA7;1108 1163 11B6; # (뺧; 뺧; 뺧; 뺧; 뺧; ) HANGUL SYLLABLE BBYALH +BEA8;BEA8;1108 1163 11B7;BEA8;1108 1163 11B7; # (뺨; 뺨; 뺨; 뺨; 뺨; ) HANGUL SYLLABLE BBYAM +BEA9;BEA9;1108 1163 11B8;BEA9;1108 1163 11B8; # (뺩; 뺩; 뺩; 뺩; 뺩; ) HANGUL SYLLABLE BBYAB +BEAA;BEAA;1108 1163 11B9;BEAA;1108 1163 11B9; # (뺪; 뺪; 뺪; 뺪; 뺪; ) HANGUL SYLLABLE BBYABS +BEAB;BEAB;1108 1163 11BA;BEAB;1108 1163 11BA; # (뺫; 뺫; 뺫; 뺫; 뺫; ) HANGUL SYLLABLE BBYAS +BEAC;BEAC;1108 1163 11BB;BEAC;1108 1163 11BB; # (뺬; 뺬; 뺬; 뺬; 뺬; ) HANGUL SYLLABLE BBYASS +BEAD;BEAD;1108 1163 11BC;BEAD;1108 1163 11BC; # (뺭; 뺭; 뺭; 뺭; 뺭; ) HANGUL SYLLABLE BBYANG +BEAE;BEAE;1108 1163 11BD;BEAE;1108 1163 11BD; # (뺮; 뺮; 뺮; 뺮; 뺮; ) HANGUL SYLLABLE BBYAJ +BEAF;BEAF;1108 1163 11BE;BEAF;1108 1163 11BE; # (뺯; 뺯; 뺯; 뺯; 뺯; ) HANGUL SYLLABLE BBYAC +BEB0;BEB0;1108 1163 11BF;BEB0;1108 1163 11BF; # (뺰; 뺰; 뺰; 뺰; 뺰; ) HANGUL SYLLABLE BBYAK +BEB1;BEB1;1108 1163 11C0;BEB1;1108 1163 11C0; # (뺱; 뺱; 뺱; 뺱; 뺱; ) HANGUL SYLLABLE BBYAT +BEB2;BEB2;1108 1163 11C1;BEB2;1108 1163 11C1; # (뺲; 뺲; 뺲; 뺲; 뺲; ) HANGUL SYLLABLE BBYAP +BEB3;BEB3;1108 1163 11C2;BEB3;1108 1163 11C2; # (뺳; 뺳; 뺳; 뺳; 뺳; ) HANGUL SYLLABLE BBYAH +BEB4;BEB4;1108 1164;BEB4;1108 1164; # (뺴; 뺴; 뺴; 뺴; 뺴; ) HANGUL SYLLABLE BBYAE +BEB5;BEB5;1108 1164 11A8;BEB5;1108 1164 11A8; # (뺵; 뺵; 뺵; 뺵; 뺵; ) HANGUL SYLLABLE BBYAEG +BEB6;BEB6;1108 1164 11A9;BEB6;1108 1164 11A9; # (뺶; 뺶; 뺶; 뺶; 뺶; ) HANGUL SYLLABLE BBYAEGG +BEB7;BEB7;1108 1164 11AA;BEB7;1108 1164 11AA; # (뺷; 뺷; 뺷; 뺷; 뺷; ) HANGUL SYLLABLE BBYAEGS +BEB8;BEB8;1108 1164 11AB;BEB8;1108 1164 11AB; # (뺸; 뺸; 뺸; 뺸; 뺸; ) HANGUL SYLLABLE BBYAEN +BEB9;BEB9;1108 1164 11AC;BEB9;1108 1164 11AC; # (뺹; 뺹; 뺹; 뺹; 뺹; ) HANGUL SYLLABLE BBYAENJ +BEBA;BEBA;1108 1164 11AD;BEBA;1108 1164 11AD; # (뺺; 뺺; 뺺; 뺺; 뺺; ) HANGUL SYLLABLE BBYAENH +BEBB;BEBB;1108 1164 11AE;BEBB;1108 1164 11AE; # (뺻; 뺻; 뺻; 뺻; 뺻; ) HANGUL SYLLABLE BBYAED +BEBC;BEBC;1108 1164 11AF;BEBC;1108 1164 11AF; # (뺼; 뺼; 뺼; 뺼; 뺼; ) HANGUL SYLLABLE BBYAEL +BEBD;BEBD;1108 1164 11B0;BEBD;1108 1164 11B0; # (뺽; 뺽; 뺽; 뺽; 뺽; ) HANGUL SYLLABLE BBYAELG +BEBE;BEBE;1108 1164 11B1;BEBE;1108 1164 11B1; # (뺾; 뺾; 뺾; 뺾; 뺾; ) HANGUL SYLLABLE BBYAELM +BEBF;BEBF;1108 1164 11B2;BEBF;1108 1164 11B2; # (뺿; 뺿; 뺿; 뺿; 뺿; ) HANGUL SYLLABLE BBYAELB +BEC0;BEC0;1108 1164 11B3;BEC0;1108 1164 11B3; # (뻀; 뻀; 뻀; 뻀; 뻀; ) HANGUL SYLLABLE BBYAELS +BEC1;BEC1;1108 1164 11B4;BEC1;1108 1164 11B4; # (뻁; 뻁; 뻁; 뻁; 뻁; ) HANGUL SYLLABLE BBYAELT +BEC2;BEC2;1108 1164 11B5;BEC2;1108 1164 11B5; # (뻂; 뻂; 뻂; 뻂; 뻂; ) HANGUL SYLLABLE BBYAELP +BEC3;BEC3;1108 1164 11B6;BEC3;1108 1164 11B6; # (뻃; 뻃; 뻃; 뻃; 뻃; ) HANGUL SYLLABLE BBYAELH +BEC4;BEC4;1108 1164 11B7;BEC4;1108 1164 11B7; # (뻄; 뻄; 뻄; 뻄; 뻄; ) HANGUL SYLLABLE BBYAEM +BEC5;BEC5;1108 1164 11B8;BEC5;1108 1164 11B8; # (뻅; 뻅; 뻅; 뻅; 뻅; ) HANGUL SYLLABLE BBYAEB +BEC6;BEC6;1108 1164 11B9;BEC6;1108 1164 11B9; # (뻆; 뻆; 뻆; 뻆; 뻆; ) HANGUL SYLLABLE BBYAEBS +BEC7;BEC7;1108 1164 11BA;BEC7;1108 1164 11BA; # (뻇; 뻇; 뻇; 뻇; 뻇; ) HANGUL SYLLABLE BBYAES +BEC8;BEC8;1108 1164 11BB;BEC8;1108 1164 11BB; # (뻈; 뻈; 뻈; 뻈; 뻈; ) HANGUL SYLLABLE BBYAESS +BEC9;BEC9;1108 1164 11BC;BEC9;1108 1164 11BC; # (뻉; 뻉; 뻉; 뻉; 뻉; ) HANGUL SYLLABLE BBYAENG +BECA;BECA;1108 1164 11BD;BECA;1108 1164 11BD; # (뻊; 뻊; 뻊; 뻊; 뻊; ) HANGUL SYLLABLE BBYAEJ +BECB;BECB;1108 1164 11BE;BECB;1108 1164 11BE; # (뻋; 뻋; 뻋; 뻋; 뻋; ) HANGUL SYLLABLE BBYAEC +BECC;BECC;1108 1164 11BF;BECC;1108 1164 11BF; # (뻌; 뻌; 뻌; 뻌; 뻌; ) HANGUL SYLLABLE BBYAEK +BECD;BECD;1108 1164 11C0;BECD;1108 1164 11C0; # (뻍; 뻍; 뻍; 뻍; 뻍; ) HANGUL SYLLABLE BBYAET +BECE;BECE;1108 1164 11C1;BECE;1108 1164 11C1; # (뻎; 뻎; 뻎; 뻎; 뻎; ) HANGUL SYLLABLE BBYAEP +BECF;BECF;1108 1164 11C2;BECF;1108 1164 11C2; # (뻏; 뻏; 뻏; 뻏; 뻏; ) HANGUL SYLLABLE BBYAEH +BED0;BED0;1108 1165;BED0;1108 1165; # (뻐; 뻐; 뻐; 뻐; 뻐; ) HANGUL SYLLABLE BBEO +BED1;BED1;1108 1165 11A8;BED1;1108 1165 11A8; # (뻑; 뻑; 뻑; 뻑; 뻑; ) HANGUL SYLLABLE BBEOG +BED2;BED2;1108 1165 11A9;BED2;1108 1165 11A9; # (뻒; 뻒; 뻒; 뻒; 뻒; ) HANGUL SYLLABLE BBEOGG +BED3;BED3;1108 1165 11AA;BED3;1108 1165 11AA; # (뻓; 뻓; 뻓; 뻓; 뻓; ) HANGUL SYLLABLE BBEOGS +BED4;BED4;1108 1165 11AB;BED4;1108 1165 11AB; # (뻔; 뻔; 뻔; 뻔; 뻔; ) HANGUL SYLLABLE BBEON +BED5;BED5;1108 1165 11AC;BED5;1108 1165 11AC; # (뻕; 뻕; 뻕; 뻕; 뻕; ) HANGUL SYLLABLE BBEONJ +BED6;BED6;1108 1165 11AD;BED6;1108 1165 11AD; # (뻖; 뻖; 뻖; 뻖; 뻖; ) HANGUL SYLLABLE BBEONH +BED7;BED7;1108 1165 11AE;BED7;1108 1165 11AE; # (뻗; 뻗; 뻗; 뻗; 뻗; ) HANGUL SYLLABLE BBEOD +BED8;BED8;1108 1165 11AF;BED8;1108 1165 11AF; # (뻘; 뻘; 뻘; 뻘; 뻘; ) HANGUL SYLLABLE BBEOL +BED9;BED9;1108 1165 11B0;BED9;1108 1165 11B0; # (뻙; 뻙; 뻙; 뻙; 뻙; ) HANGUL SYLLABLE BBEOLG +BEDA;BEDA;1108 1165 11B1;BEDA;1108 1165 11B1; # (뻚; 뻚; 뻚; 뻚; 뻚; ) HANGUL SYLLABLE BBEOLM +BEDB;BEDB;1108 1165 11B2;BEDB;1108 1165 11B2; # (뻛; 뻛; 뻛; 뻛; 뻛; ) HANGUL SYLLABLE BBEOLB +BEDC;BEDC;1108 1165 11B3;BEDC;1108 1165 11B3; # (뻜; 뻜; 뻜; 뻜; 뻜; ) HANGUL SYLLABLE BBEOLS +BEDD;BEDD;1108 1165 11B4;BEDD;1108 1165 11B4; # (뻝; 뻝; 뻝; 뻝; 뻝; ) HANGUL SYLLABLE BBEOLT +BEDE;BEDE;1108 1165 11B5;BEDE;1108 1165 11B5; # (뻞; 뻞; 뻞; 뻞; 뻞; ) HANGUL SYLLABLE BBEOLP +BEDF;BEDF;1108 1165 11B6;BEDF;1108 1165 11B6; # (뻟; 뻟; 뻟; 뻟; 뻟; ) HANGUL SYLLABLE BBEOLH +BEE0;BEE0;1108 1165 11B7;BEE0;1108 1165 11B7; # (뻠; 뻠; 뻠; 뻠; 뻠; ) HANGUL SYLLABLE BBEOM +BEE1;BEE1;1108 1165 11B8;BEE1;1108 1165 11B8; # (뻡; 뻡; 뻡; 뻡; 뻡; ) HANGUL SYLLABLE BBEOB +BEE2;BEE2;1108 1165 11B9;BEE2;1108 1165 11B9; # (뻢; 뻢; 뻢; 뻢; 뻢; ) HANGUL SYLLABLE BBEOBS +BEE3;BEE3;1108 1165 11BA;BEE3;1108 1165 11BA; # (뻣; 뻣; 뻣; 뻣; 뻣; ) HANGUL SYLLABLE BBEOS +BEE4;BEE4;1108 1165 11BB;BEE4;1108 1165 11BB; # (뻤; 뻤; 뻤; 뻤; 뻤; ) HANGUL SYLLABLE BBEOSS +BEE5;BEE5;1108 1165 11BC;BEE5;1108 1165 11BC; # (뻥; 뻥; 뻥; 뻥; 뻥; ) HANGUL SYLLABLE BBEONG +BEE6;BEE6;1108 1165 11BD;BEE6;1108 1165 11BD; # (뻦; 뻦; 뻦; 뻦; 뻦; ) HANGUL SYLLABLE BBEOJ +BEE7;BEE7;1108 1165 11BE;BEE7;1108 1165 11BE; # (뻧; 뻧; 뻧; 뻧; 뻧; ) HANGUL SYLLABLE BBEOC +BEE8;BEE8;1108 1165 11BF;BEE8;1108 1165 11BF; # (뻨; 뻨; 뻨; 뻨; 뻨; ) HANGUL SYLLABLE BBEOK +BEE9;BEE9;1108 1165 11C0;BEE9;1108 1165 11C0; # (뻩; 뻩; 뻩; 뻩; 뻩; ) HANGUL SYLLABLE BBEOT +BEEA;BEEA;1108 1165 11C1;BEEA;1108 1165 11C1; # (뻪; 뻪; 뻪; 뻪; 뻪; ) HANGUL SYLLABLE BBEOP +BEEB;BEEB;1108 1165 11C2;BEEB;1108 1165 11C2; # (뻫; 뻫; 뻫; 뻫; 뻫; ) HANGUL SYLLABLE BBEOH +BEEC;BEEC;1108 1166;BEEC;1108 1166; # (뻬; 뻬; 뻬; 뻬; 뻬; ) HANGUL SYLLABLE BBE +BEED;BEED;1108 1166 11A8;BEED;1108 1166 11A8; # (뻭; 뻭; 뻭; 뻭; 뻭; ) HANGUL SYLLABLE BBEG +BEEE;BEEE;1108 1166 11A9;BEEE;1108 1166 11A9; # (뻮; 뻮; 뻮; 뻮; 뻮; ) HANGUL SYLLABLE BBEGG +BEEF;BEEF;1108 1166 11AA;BEEF;1108 1166 11AA; # (뻯; 뻯; 뻯; 뻯; 뻯; ) HANGUL SYLLABLE BBEGS +BEF0;BEF0;1108 1166 11AB;BEF0;1108 1166 11AB; # (뻰; 뻰; 뻰; 뻰; 뻰; ) HANGUL SYLLABLE BBEN +BEF1;BEF1;1108 1166 11AC;BEF1;1108 1166 11AC; # (뻱; 뻱; 뻱; 뻱; 뻱; ) HANGUL SYLLABLE BBENJ +BEF2;BEF2;1108 1166 11AD;BEF2;1108 1166 11AD; # (뻲; 뻲; 뻲; 뻲; 뻲; ) HANGUL SYLLABLE BBENH +BEF3;BEF3;1108 1166 11AE;BEF3;1108 1166 11AE; # (뻳; 뻳; 뻳; 뻳; 뻳; ) HANGUL SYLLABLE BBED +BEF4;BEF4;1108 1166 11AF;BEF4;1108 1166 11AF; # (뻴; 뻴; 뻴; 뻴; 뻴; ) HANGUL SYLLABLE BBEL +BEF5;BEF5;1108 1166 11B0;BEF5;1108 1166 11B0; # (뻵; 뻵; 뻵; 뻵; 뻵; ) HANGUL SYLLABLE BBELG +BEF6;BEF6;1108 1166 11B1;BEF6;1108 1166 11B1; # (뻶; 뻶; 뻶; 뻶; 뻶; ) HANGUL SYLLABLE BBELM +BEF7;BEF7;1108 1166 11B2;BEF7;1108 1166 11B2; # (뻷; 뻷; 뻷; 뻷; 뻷; ) HANGUL SYLLABLE BBELB +BEF8;BEF8;1108 1166 11B3;BEF8;1108 1166 11B3; # (뻸; 뻸; 뻸; 뻸; 뻸; ) HANGUL SYLLABLE BBELS +BEF9;BEF9;1108 1166 11B4;BEF9;1108 1166 11B4; # (뻹; 뻹; 뻹; 뻹; 뻹; ) HANGUL SYLLABLE BBELT +BEFA;BEFA;1108 1166 11B5;BEFA;1108 1166 11B5; # (뻺; 뻺; 뻺; 뻺; 뻺; ) HANGUL SYLLABLE BBELP +BEFB;BEFB;1108 1166 11B6;BEFB;1108 1166 11B6; # (뻻; 뻻; 뻻; 뻻; 뻻; ) HANGUL SYLLABLE BBELH +BEFC;BEFC;1108 1166 11B7;BEFC;1108 1166 11B7; # (뻼; 뻼; 뻼; 뻼; 뻼; ) HANGUL SYLLABLE BBEM +BEFD;BEFD;1108 1166 11B8;BEFD;1108 1166 11B8; # (뻽; 뻽; 뻽; 뻽; 뻽; ) HANGUL SYLLABLE BBEB +BEFE;BEFE;1108 1166 11B9;BEFE;1108 1166 11B9; # (뻾; 뻾; 뻾; 뻾; 뻾; ) HANGUL SYLLABLE BBEBS +BEFF;BEFF;1108 1166 11BA;BEFF;1108 1166 11BA; # (뻿; 뻿; 뻿; 뻿; 뻿; ) HANGUL SYLLABLE BBES +BF00;BF00;1108 1166 11BB;BF00;1108 1166 11BB; # (뼀; 뼀; 뼀; 뼀; 뼀; ) HANGUL SYLLABLE BBESS +BF01;BF01;1108 1166 11BC;BF01;1108 1166 11BC; # (뼁; 뼁; 뼁; 뼁; 뼁; ) HANGUL SYLLABLE BBENG +BF02;BF02;1108 1166 11BD;BF02;1108 1166 11BD; # (뼂; 뼂; 뼂; 뼂; 뼂; ) HANGUL SYLLABLE BBEJ +BF03;BF03;1108 1166 11BE;BF03;1108 1166 11BE; # (뼃; 뼃; 뼃; 뼃; 뼃; ) HANGUL SYLLABLE BBEC +BF04;BF04;1108 1166 11BF;BF04;1108 1166 11BF; # (뼄; 뼄; 뼄; 뼄; 뼄; ) HANGUL SYLLABLE BBEK +BF05;BF05;1108 1166 11C0;BF05;1108 1166 11C0; # (뼅; 뼅; 뼅; 뼅; 뼅; ) HANGUL SYLLABLE BBET +BF06;BF06;1108 1166 11C1;BF06;1108 1166 11C1; # (뼆; 뼆; 뼆; 뼆; 뼆; ) HANGUL SYLLABLE BBEP +BF07;BF07;1108 1166 11C2;BF07;1108 1166 11C2; # (뼇; 뼇; 뼇; 뼇; 뼇; ) HANGUL SYLLABLE BBEH +BF08;BF08;1108 1167;BF08;1108 1167; # (뼈; 뼈; 뼈; 뼈; 뼈; ) HANGUL SYLLABLE BBYEO +BF09;BF09;1108 1167 11A8;BF09;1108 1167 11A8; # (뼉; 뼉; 뼉; 뼉; 뼉; ) HANGUL SYLLABLE BBYEOG +BF0A;BF0A;1108 1167 11A9;BF0A;1108 1167 11A9; # (뼊; 뼊; 뼊; 뼊; 뼊; ) HANGUL SYLLABLE BBYEOGG +BF0B;BF0B;1108 1167 11AA;BF0B;1108 1167 11AA; # (뼋; 뼋; 뼋; 뼋; 뼋; ) HANGUL SYLLABLE BBYEOGS +BF0C;BF0C;1108 1167 11AB;BF0C;1108 1167 11AB; # (뼌; 뼌; 뼌; 뼌; 뼌; ) HANGUL SYLLABLE BBYEON +BF0D;BF0D;1108 1167 11AC;BF0D;1108 1167 11AC; # (뼍; 뼍; 뼍; 뼍; 뼍; ) HANGUL SYLLABLE BBYEONJ +BF0E;BF0E;1108 1167 11AD;BF0E;1108 1167 11AD; # (뼎; 뼎; 뼎; 뼎; 뼎; ) HANGUL SYLLABLE BBYEONH +BF0F;BF0F;1108 1167 11AE;BF0F;1108 1167 11AE; # (뼏; 뼏; 뼏; 뼏; 뼏; ) HANGUL SYLLABLE BBYEOD +BF10;BF10;1108 1167 11AF;BF10;1108 1167 11AF; # (뼐; 뼐; 뼐; 뼐; 뼐; ) HANGUL SYLLABLE BBYEOL +BF11;BF11;1108 1167 11B0;BF11;1108 1167 11B0; # (뼑; 뼑; 뼑; 뼑; 뼑; ) HANGUL SYLLABLE BBYEOLG +BF12;BF12;1108 1167 11B1;BF12;1108 1167 11B1; # (뼒; 뼒; 뼒; 뼒; 뼒; ) HANGUL SYLLABLE BBYEOLM +BF13;BF13;1108 1167 11B2;BF13;1108 1167 11B2; # (뼓; 뼓; 뼓; 뼓; 뼓; ) HANGUL SYLLABLE BBYEOLB +BF14;BF14;1108 1167 11B3;BF14;1108 1167 11B3; # (뼔; 뼔; 뼔; 뼔; 뼔; ) HANGUL SYLLABLE BBYEOLS +BF15;BF15;1108 1167 11B4;BF15;1108 1167 11B4; # (뼕; 뼕; 뼕; 뼕; 뼕; ) HANGUL SYLLABLE BBYEOLT +BF16;BF16;1108 1167 11B5;BF16;1108 1167 11B5; # (뼖; 뼖; 뼖; 뼖; 뼖; ) HANGUL SYLLABLE BBYEOLP +BF17;BF17;1108 1167 11B6;BF17;1108 1167 11B6; # (뼗; 뼗; 뼗; 뼗; 뼗; ) HANGUL SYLLABLE BBYEOLH +BF18;BF18;1108 1167 11B7;BF18;1108 1167 11B7; # (뼘; 뼘; 뼘; 뼘; 뼘; ) HANGUL SYLLABLE BBYEOM +BF19;BF19;1108 1167 11B8;BF19;1108 1167 11B8; # (뼙; 뼙; 뼙; 뼙; 뼙; ) HANGUL SYLLABLE BBYEOB +BF1A;BF1A;1108 1167 11B9;BF1A;1108 1167 11B9; # (뼚; 뼚; 뼚; 뼚; 뼚; ) HANGUL SYLLABLE BBYEOBS +BF1B;BF1B;1108 1167 11BA;BF1B;1108 1167 11BA; # (뼛; 뼛; 뼛; 뼛; 뼛; ) HANGUL SYLLABLE BBYEOS +BF1C;BF1C;1108 1167 11BB;BF1C;1108 1167 11BB; # (뼜; 뼜; 뼜; 뼜; 뼜; ) HANGUL SYLLABLE BBYEOSS +BF1D;BF1D;1108 1167 11BC;BF1D;1108 1167 11BC; # (뼝; 뼝; 뼝; 뼝; 뼝; ) HANGUL SYLLABLE BBYEONG +BF1E;BF1E;1108 1167 11BD;BF1E;1108 1167 11BD; # (뼞; 뼞; 뼞; 뼞; 뼞; ) HANGUL SYLLABLE BBYEOJ +BF1F;BF1F;1108 1167 11BE;BF1F;1108 1167 11BE; # (뼟; 뼟; 뼟; 뼟; 뼟; ) HANGUL SYLLABLE BBYEOC +BF20;BF20;1108 1167 11BF;BF20;1108 1167 11BF; # (뼠; 뼠; 뼠; 뼠; 뼠; ) HANGUL SYLLABLE BBYEOK +BF21;BF21;1108 1167 11C0;BF21;1108 1167 11C0; # (뼡; 뼡; 뼡; 뼡; 뼡; ) HANGUL SYLLABLE BBYEOT +BF22;BF22;1108 1167 11C1;BF22;1108 1167 11C1; # (뼢; 뼢; 뼢; 뼢; 뼢; ) HANGUL SYLLABLE BBYEOP +BF23;BF23;1108 1167 11C2;BF23;1108 1167 11C2; # (뼣; 뼣; 뼣; 뼣; 뼣; ) HANGUL SYLLABLE BBYEOH +BF24;BF24;1108 1168;BF24;1108 1168; # (뼤; 뼤; 뼤; 뼤; 뼤; ) HANGUL SYLLABLE BBYE +BF25;BF25;1108 1168 11A8;BF25;1108 1168 11A8; # (뼥; 뼥; 뼥; 뼥; 뼥; ) HANGUL SYLLABLE BBYEG +BF26;BF26;1108 1168 11A9;BF26;1108 1168 11A9; # (뼦; 뼦; 뼦; 뼦; 뼦; ) HANGUL SYLLABLE BBYEGG +BF27;BF27;1108 1168 11AA;BF27;1108 1168 11AA; # (뼧; 뼧; 뼧; 뼧; 뼧; ) HANGUL SYLLABLE BBYEGS +BF28;BF28;1108 1168 11AB;BF28;1108 1168 11AB; # (뼨; 뼨; 뼨; 뼨; 뼨; ) HANGUL SYLLABLE BBYEN +BF29;BF29;1108 1168 11AC;BF29;1108 1168 11AC; # (뼩; 뼩; 뼩; 뼩; 뼩; ) HANGUL SYLLABLE BBYENJ +BF2A;BF2A;1108 1168 11AD;BF2A;1108 1168 11AD; # (뼪; 뼪; 뼪; 뼪; 뼪; ) HANGUL SYLLABLE BBYENH +BF2B;BF2B;1108 1168 11AE;BF2B;1108 1168 11AE; # (뼫; 뼫; 뼫; 뼫; 뼫; ) HANGUL SYLLABLE BBYED +BF2C;BF2C;1108 1168 11AF;BF2C;1108 1168 11AF; # (뼬; 뼬; 뼬; 뼬; 뼬; ) HANGUL SYLLABLE BBYEL +BF2D;BF2D;1108 1168 11B0;BF2D;1108 1168 11B0; # (뼭; 뼭; 뼭; 뼭; 뼭; ) HANGUL SYLLABLE BBYELG +BF2E;BF2E;1108 1168 11B1;BF2E;1108 1168 11B1; # (뼮; 뼮; 뼮; 뼮; 뼮; ) HANGUL SYLLABLE BBYELM +BF2F;BF2F;1108 1168 11B2;BF2F;1108 1168 11B2; # (뼯; 뼯; 뼯; 뼯; 뼯; ) HANGUL SYLLABLE BBYELB +BF30;BF30;1108 1168 11B3;BF30;1108 1168 11B3; # (뼰; 뼰; 뼰; 뼰; 뼰; ) HANGUL SYLLABLE BBYELS +BF31;BF31;1108 1168 11B4;BF31;1108 1168 11B4; # (뼱; 뼱; 뼱; 뼱; 뼱; ) HANGUL SYLLABLE BBYELT +BF32;BF32;1108 1168 11B5;BF32;1108 1168 11B5; # (뼲; 뼲; 뼲; 뼲; 뼲; ) HANGUL SYLLABLE BBYELP +BF33;BF33;1108 1168 11B6;BF33;1108 1168 11B6; # (뼳; 뼳; 뼳; 뼳; 뼳; ) HANGUL SYLLABLE BBYELH +BF34;BF34;1108 1168 11B7;BF34;1108 1168 11B7; # (뼴; 뼴; 뼴; 뼴; 뼴; ) HANGUL SYLLABLE BBYEM +BF35;BF35;1108 1168 11B8;BF35;1108 1168 11B8; # (뼵; 뼵; 뼵; 뼵; 뼵; ) HANGUL SYLLABLE BBYEB +BF36;BF36;1108 1168 11B9;BF36;1108 1168 11B9; # (뼶; 뼶; 뼶; 뼶; 뼶; ) HANGUL SYLLABLE BBYEBS +BF37;BF37;1108 1168 11BA;BF37;1108 1168 11BA; # (뼷; 뼷; 뼷; 뼷; 뼷; ) HANGUL SYLLABLE BBYES +BF38;BF38;1108 1168 11BB;BF38;1108 1168 11BB; # (뼸; 뼸; 뼸; 뼸; 뼸; ) HANGUL SYLLABLE BBYESS +BF39;BF39;1108 1168 11BC;BF39;1108 1168 11BC; # (뼹; 뼹; 뼹; 뼹; 뼹; ) HANGUL SYLLABLE BBYENG +BF3A;BF3A;1108 1168 11BD;BF3A;1108 1168 11BD; # (뼺; 뼺; 뼺; 뼺; 뼺; ) HANGUL SYLLABLE BBYEJ +BF3B;BF3B;1108 1168 11BE;BF3B;1108 1168 11BE; # (뼻; 뼻; 뼻; 뼻; 뼻; ) HANGUL SYLLABLE BBYEC +BF3C;BF3C;1108 1168 11BF;BF3C;1108 1168 11BF; # (뼼; 뼼; 뼼; 뼼; 뼼; ) HANGUL SYLLABLE BBYEK +BF3D;BF3D;1108 1168 11C0;BF3D;1108 1168 11C0; # (뼽; 뼽; 뼽; 뼽; 뼽; ) HANGUL SYLLABLE BBYET +BF3E;BF3E;1108 1168 11C1;BF3E;1108 1168 11C1; # (뼾; 뼾; 뼾; 뼾; 뼾; ) HANGUL SYLLABLE BBYEP +BF3F;BF3F;1108 1168 11C2;BF3F;1108 1168 11C2; # (뼿; 뼿; 뼿; 뼿; 뼿; ) HANGUL SYLLABLE BBYEH +BF40;BF40;1108 1169;BF40;1108 1169; # (뽀; 뽀; 뽀; 뽀; 뽀; ) HANGUL SYLLABLE BBO +BF41;BF41;1108 1169 11A8;BF41;1108 1169 11A8; # (뽁; 뽁; 뽁; 뽁; 뽁; ) HANGUL SYLLABLE BBOG +BF42;BF42;1108 1169 11A9;BF42;1108 1169 11A9; # (뽂; 뽂; 뽂; 뽂; 뽂; ) HANGUL SYLLABLE BBOGG +BF43;BF43;1108 1169 11AA;BF43;1108 1169 11AA; # (뽃; 뽃; 뽃; 뽃; 뽃; ) HANGUL SYLLABLE BBOGS +BF44;BF44;1108 1169 11AB;BF44;1108 1169 11AB; # (뽄; 뽄; 뽄; 뽄; 뽄; ) HANGUL SYLLABLE BBON +BF45;BF45;1108 1169 11AC;BF45;1108 1169 11AC; # (뽅; 뽅; 뽅; 뽅; 뽅; ) HANGUL SYLLABLE BBONJ +BF46;BF46;1108 1169 11AD;BF46;1108 1169 11AD; # (뽆; 뽆; 뽆; 뽆; 뽆; ) HANGUL SYLLABLE BBONH +BF47;BF47;1108 1169 11AE;BF47;1108 1169 11AE; # (뽇; 뽇; 뽇; 뽇; 뽇; ) HANGUL SYLLABLE BBOD +BF48;BF48;1108 1169 11AF;BF48;1108 1169 11AF; # (뽈; 뽈; 뽈; 뽈; 뽈; ) HANGUL SYLLABLE BBOL +BF49;BF49;1108 1169 11B0;BF49;1108 1169 11B0; # (뽉; 뽉; 뽉; 뽉; 뽉; ) HANGUL SYLLABLE BBOLG +BF4A;BF4A;1108 1169 11B1;BF4A;1108 1169 11B1; # (뽊; 뽊; 뽊; 뽊; 뽊; ) HANGUL SYLLABLE BBOLM +BF4B;BF4B;1108 1169 11B2;BF4B;1108 1169 11B2; # (뽋; 뽋; 뽋; 뽋; 뽋; ) HANGUL SYLLABLE BBOLB +BF4C;BF4C;1108 1169 11B3;BF4C;1108 1169 11B3; # (뽌; 뽌; 뽌; 뽌; 뽌; ) HANGUL SYLLABLE BBOLS +BF4D;BF4D;1108 1169 11B4;BF4D;1108 1169 11B4; # (뽍; 뽍; 뽍; 뽍; 뽍; ) HANGUL SYLLABLE BBOLT +BF4E;BF4E;1108 1169 11B5;BF4E;1108 1169 11B5; # (뽎; 뽎; 뽎; 뽎; 뽎; ) HANGUL SYLLABLE BBOLP +BF4F;BF4F;1108 1169 11B6;BF4F;1108 1169 11B6; # (뽏; 뽏; 뽏; 뽏; 뽏; ) HANGUL SYLLABLE BBOLH +BF50;BF50;1108 1169 11B7;BF50;1108 1169 11B7; # (뽐; 뽐; 뽐; 뽐; 뽐; ) HANGUL SYLLABLE BBOM +BF51;BF51;1108 1169 11B8;BF51;1108 1169 11B8; # (뽑; 뽑; 뽑; 뽑; 뽑; ) HANGUL SYLLABLE BBOB +BF52;BF52;1108 1169 11B9;BF52;1108 1169 11B9; # (뽒; 뽒; 뽒; 뽒; 뽒; ) HANGUL SYLLABLE BBOBS +BF53;BF53;1108 1169 11BA;BF53;1108 1169 11BA; # (뽓; 뽓; 뽓; 뽓; 뽓; ) HANGUL SYLLABLE BBOS +BF54;BF54;1108 1169 11BB;BF54;1108 1169 11BB; # (뽔; 뽔; 뽔; 뽔; 뽔; ) HANGUL SYLLABLE BBOSS +BF55;BF55;1108 1169 11BC;BF55;1108 1169 11BC; # (뽕; 뽕; 뽕; 뽕; 뽕; ) HANGUL SYLLABLE BBONG +BF56;BF56;1108 1169 11BD;BF56;1108 1169 11BD; # (뽖; 뽖; 뽖; 뽖; 뽖; ) HANGUL SYLLABLE BBOJ +BF57;BF57;1108 1169 11BE;BF57;1108 1169 11BE; # (뽗; 뽗; 뽗; 뽗; 뽗; ) HANGUL SYLLABLE BBOC +BF58;BF58;1108 1169 11BF;BF58;1108 1169 11BF; # (뽘; 뽘; 뽘; 뽘; 뽘; ) HANGUL SYLLABLE BBOK +BF59;BF59;1108 1169 11C0;BF59;1108 1169 11C0; # (뽙; 뽙; 뽙; 뽙; 뽙; ) HANGUL SYLLABLE BBOT +BF5A;BF5A;1108 1169 11C1;BF5A;1108 1169 11C1; # (뽚; 뽚; 뽚; 뽚; 뽚; ) HANGUL SYLLABLE BBOP +BF5B;BF5B;1108 1169 11C2;BF5B;1108 1169 11C2; # (뽛; 뽛; 뽛; 뽛; 뽛; ) HANGUL SYLLABLE BBOH +BF5C;BF5C;1108 116A;BF5C;1108 116A; # (뽜; 뽜; 뽜; 뽜; 뽜; ) HANGUL SYLLABLE BBWA +BF5D;BF5D;1108 116A 11A8;BF5D;1108 116A 11A8; # (뽝; 뽝; 뽝; 뽝; 뽝; ) HANGUL SYLLABLE BBWAG +BF5E;BF5E;1108 116A 11A9;BF5E;1108 116A 11A9; # (뽞; 뽞; 뽞; 뽞; 뽞; ) HANGUL SYLLABLE BBWAGG +BF5F;BF5F;1108 116A 11AA;BF5F;1108 116A 11AA; # (뽟; 뽟; 뽟; 뽟; 뽟; ) HANGUL SYLLABLE BBWAGS +BF60;BF60;1108 116A 11AB;BF60;1108 116A 11AB; # (뽠; 뽠; 뽠; 뽠; 뽠; ) HANGUL SYLLABLE BBWAN +BF61;BF61;1108 116A 11AC;BF61;1108 116A 11AC; # (뽡; 뽡; 뽡; 뽡; 뽡; ) HANGUL SYLLABLE BBWANJ +BF62;BF62;1108 116A 11AD;BF62;1108 116A 11AD; # (뽢; 뽢; 뽢; 뽢; 뽢; ) HANGUL SYLLABLE BBWANH +BF63;BF63;1108 116A 11AE;BF63;1108 116A 11AE; # (뽣; 뽣; 뽣; 뽣; 뽣; ) HANGUL SYLLABLE BBWAD +BF64;BF64;1108 116A 11AF;BF64;1108 116A 11AF; # (뽤; 뽤; 뽤; 뽤; 뽤; ) HANGUL SYLLABLE BBWAL +BF65;BF65;1108 116A 11B0;BF65;1108 116A 11B0; # (뽥; 뽥; 뽥; 뽥; 뽥; ) HANGUL SYLLABLE BBWALG +BF66;BF66;1108 116A 11B1;BF66;1108 116A 11B1; # (뽦; 뽦; 뽦; 뽦; 뽦; ) HANGUL SYLLABLE BBWALM +BF67;BF67;1108 116A 11B2;BF67;1108 116A 11B2; # (뽧; 뽧; 뽧; 뽧; 뽧; ) HANGUL SYLLABLE BBWALB +BF68;BF68;1108 116A 11B3;BF68;1108 116A 11B3; # (뽨; 뽨; 뽨; 뽨; 뽨; ) HANGUL SYLLABLE BBWALS +BF69;BF69;1108 116A 11B4;BF69;1108 116A 11B4; # (뽩; 뽩; 뽩; 뽩; 뽩; ) HANGUL SYLLABLE BBWALT +BF6A;BF6A;1108 116A 11B5;BF6A;1108 116A 11B5; # (뽪; 뽪; 뽪; 뽪; 뽪; ) HANGUL SYLLABLE BBWALP +BF6B;BF6B;1108 116A 11B6;BF6B;1108 116A 11B6; # (뽫; 뽫; 뽫; 뽫; 뽫; ) HANGUL SYLLABLE BBWALH +BF6C;BF6C;1108 116A 11B7;BF6C;1108 116A 11B7; # (뽬; 뽬; 뽬; 뽬; 뽬; ) HANGUL SYLLABLE BBWAM +BF6D;BF6D;1108 116A 11B8;BF6D;1108 116A 11B8; # (뽭; 뽭; 뽭; 뽭; 뽭; ) HANGUL SYLLABLE BBWAB +BF6E;BF6E;1108 116A 11B9;BF6E;1108 116A 11B9; # (뽮; 뽮; 뽮; 뽮; 뽮; ) HANGUL SYLLABLE BBWABS +BF6F;BF6F;1108 116A 11BA;BF6F;1108 116A 11BA; # (뽯; 뽯; 뽯; 뽯; 뽯; ) HANGUL SYLLABLE BBWAS +BF70;BF70;1108 116A 11BB;BF70;1108 116A 11BB; # (뽰; 뽰; 뽰; 뽰; 뽰; ) HANGUL SYLLABLE BBWASS +BF71;BF71;1108 116A 11BC;BF71;1108 116A 11BC; # (뽱; 뽱; 뽱; 뽱; 뽱; ) HANGUL SYLLABLE BBWANG +BF72;BF72;1108 116A 11BD;BF72;1108 116A 11BD; # (뽲; 뽲; 뽲; 뽲; 뽲; ) HANGUL SYLLABLE BBWAJ +BF73;BF73;1108 116A 11BE;BF73;1108 116A 11BE; # (뽳; 뽳; 뽳; 뽳; 뽳; ) HANGUL SYLLABLE BBWAC +BF74;BF74;1108 116A 11BF;BF74;1108 116A 11BF; # (뽴; 뽴; 뽴; 뽴; 뽴; ) HANGUL SYLLABLE BBWAK +BF75;BF75;1108 116A 11C0;BF75;1108 116A 11C0; # (뽵; 뽵; 뽵; 뽵; 뽵; ) HANGUL SYLLABLE BBWAT +BF76;BF76;1108 116A 11C1;BF76;1108 116A 11C1; # (뽶; 뽶; 뽶; 뽶; 뽶; ) HANGUL SYLLABLE BBWAP +BF77;BF77;1108 116A 11C2;BF77;1108 116A 11C2; # (뽷; 뽷; 뽷; 뽷; 뽷; ) HANGUL SYLLABLE BBWAH +BF78;BF78;1108 116B;BF78;1108 116B; # (뽸; 뽸; 뽸; 뽸; 뽸; ) HANGUL SYLLABLE BBWAE +BF79;BF79;1108 116B 11A8;BF79;1108 116B 11A8; # (뽹; 뽹; 뽹; 뽹; 뽹; ) HANGUL SYLLABLE BBWAEG +BF7A;BF7A;1108 116B 11A9;BF7A;1108 116B 11A9; # (뽺; 뽺; 뽺; 뽺; 뽺; ) HANGUL SYLLABLE BBWAEGG +BF7B;BF7B;1108 116B 11AA;BF7B;1108 116B 11AA; # (뽻; 뽻; 뽻; 뽻; 뽻; ) HANGUL SYLLABLE BBWAEGS +BF7C;BF7C;1108 116B 11AB;BF7C;1108 116B 11AB; # (뽼; 뽼; 뽼; 뽼; 뽼; ) HANGUL SYLLABLE BBWAEN +BF7D;BF7D;1108 116B 11AC;BF7D;1108 116B 11AC; # (뽽; 뽽; 뽽; 뽽; 뽽; ) HANGUL SYLLABLE BBWAENJ +BF7E;BF7E;1108 116B 11AD;BF7E;1108 116B 11AD; # (뽾; 뽾; 뽾; 뽾; 뽾; ) HANGUL SYLLABLE BBWAENH +BF7F;BF7F;1108 116B 11AE;BF7F;1108 116B 11AE; # (뽿; 뽿; 뽿; 뽿; 뽿; ) HANGUL SYLLABLE BBWAED +BF80;BF80;1108 116B 11AF;BF80;1108 116B 11AF; # (뾀; 뾀; 뾀; 뾀; 뾀; ) HANGUL SYLLABLE BBWAEL +BF81;BF81;1108 116B 11B0;BF81;1108 116B 11B0; # (뾁; 뾁; 뾁; 뾁; 뾁; ) HANGUL SYLLABLE BBWAELG +BF82;BF82;1108 116B 11B1;BF82;1108 116B 11B1; # (뾂; 뾂; 뾂; 뾂; 뾂; ) HANGUL SYLLABLE BBWAELM +BF83;BF83;1108 116B 11B2;BF83;1108 116B 11B2; # (뾃; 뾃; 뾃; 뾃; 뾃; ) HANGUL SYLLABLE BBWAELB +BF84;BF84;1108 116B 11B3;BF84;1108 116B 11B3; # (뾄; 뾄; 뾄; 뾄; 뾄; ) HANGUL SYLLABLE BBWAELS +BF85;BF85;1108 116B 11B4;BF85;1108 116B 11B4; # (뾅; 뾅; 뾅; 뾅; 뾅; ) HANGUL SYLLABLE BBWAELT +BF86;BF86;1108 116B 11B5;BF86;1108 116B 11B5; # (뾆; 뾆; 뾆; 뾆; 뾆; ) HANGUL SYLLABLE BBWAELP +BF87;BF87;1108 116B 11B6;BF87;1108 116B 11B6; # (뾇; 뾇; 뾇; 뾇; 뾇; ) HANGUL SYLLABLE BBWAELH +BF88;BF88;1108 116B 11B7;BF88;1108 116B 11B7; # (뾈; 뾈; 뾈; 뾈; 뾈; ) HANGUL SYLLABLE BBWAEM +BF89;BF89;1108 116B 11B8;BF89;1108 116B 11B8; # (뾉; 뾉; 뾉; 뾉; 뾉; ) HANGUL SYLLABLE BBWAEB +BF8A;BF8A;1108 116B 11B9;BF8A;1108 116B 11B9; # (뾊; 뾊; 뾊; 뾊; 뾊; ) HANGUL SYLLABLE BBWAEBS +BF8B;BF8B;1108 116B 11BA;BF8B;1108 116B 11BA; # (뾋; 뾋; 뾋; 뾋; 뾋; ) HANGUL SYLLABLE BBWAES +BF8C;BF8C;1108 116B 11BB;BF8C;1108 116B 11BB; # (뾌; 뾌; 뾌; 뾌; 뾌; ) HANGUL SYLLABLE BBWAESS +BF8D;BF8D;1108 116B 11BC;BF8D;1108 116B 11BC; # (뾍; 뾍; 뾍; 뾍; 뾍; ) HANGUL SYLLABLE BBWAENG +BF8E;BF8E;1108 116B 11BD;BF8E;1108 116B 11BD; # (뾎; 뾎; 뾎; 뾎; 뾎; ) HANGUL SYLLABLE BBWAEJ +BF8F;BF8F;1108 116B 11BE;BF8F;1108 116B 11BE; # (뾏; 뾏; 뾏; 뾏; 뾏; ) HANGUL SYLLABLE BBWAEC +BF90;BF90;1108 116B 11BF;BF90;1108 116B 11BF; # (뾐; 뾐; 뾐; 뾐; 뾐; ) HANGUL SYLLABLE BBWAEK +BF91;BF91;1108 116B 11C0;BF91;1108 116B 11C0; # (뾑; 뾑; 뾑; 뾑; 뾑; ) HANGUL SYLLABLE BBWAET +BF92;BF92;1108 116B 11C1;BF92;1108 116B 11C1; # (뾒; 뾒; 뾒; 뾒; 뾒; ) HANGUL SYLLABLE BBWAEP +BF93;BF93;1108 116B 11C2;BF93;1108 116B 11C2; # (뾓; 뾓; 뾓; 뾓; 뾓; ) HANGUL SYLLABLE BBWAEH +BF94;BF94;1108 116C;BF94;1108 116C; # (뾔; 뾔; 뾔; 뾔; 뾔; ) HANGUL SYLLABLE BBOE +BF95;BF95;1108 116C 11A8;BF95;1108 116C 11A8; # (뾕; 뾕; 뾕; 뾕; 뾕; ) HANGUL SYLLABLE BBOEG +BF96;BF96;1108 116C 11A9;BF96;1108 116C 11A9; # (뾖; 뾖; 뾖; 뾖; 뾖; ) HANGUL SYLLABLE BBOEGG +BF97;BF97;1108 116C 11AA;BF97;1108 116C 11AA; # (뾗; 뾗; 뾗; 뾗; 뾗; ) HANGUL SYLLABLE BBOEGS +BF98;BF98;1108 116C 11AB;BF98;1108 116C 11AB; # (뾘; 뾘; 뾘; 뾘; 뾘; ) HANGUL SYLLABLE BBOEN +BF99;BF99;1108 116C 11AC;BF99;1108 116C 11AC; # (뾙; 뾙; 뾙; 뾙; 뾙; ) HANGUL SYLLABLE BBOENJ +BF9A;BF9A;1108 116C 11AD;BF9A;1108 116C 11AD; # (뾚; 뾚; 뾚; 뾚; 뾚; ) HANGUL SYLLABLE BBOENH +BF9B;BF9B;1108 116C 11AE;BF9B;1108 116C 11AE; # (뾛; 뾛; 뾛; 뾛; 뾛; ) HANGUL SYLLABLE BBOED +BF9C;BF9C;1108 116C 11AF;BF9C;1108 116C 11AF; # (뾜; 뾜; 뾜; 뾜; 뾜; ) HANGUL SYLLABLE BBOEL +BF9D;BF9D;1108 116C 11B0;BF9D;1108 116C 11B0; # (뾝; 뾝; 뾝; 뾝; 뾝; ) HANGUL SYLLABLE BBOELG +BF9E;BF9E;1108 116C 11B1;BF9E;1108 116C 11B1; # (뾞; 뾞; 뾞; 뾞; 뾞; ) HANGUL SYLLABLE BBOELM +BF9F;BF9F;1108 116C 11B2;BF9F;1108 116C 11B2; # (뾟; 뾟; 뾟; 뾟; 뾟; ) HANGUL SYLLABLE BBOELB +BFA0;BFA0;1108 116C 11B3;BFA0;1108 116C 11B3; # (뾠; 뾠; 뾠; 뾠; 뾠; ) HANGUL SYLLABLE BBOELS +BFA1;BFA1;1108 116C 11B4;BFA1;1108 116C 11B4; # (뾡; 뾡; 뾡; 뾡; 뾡; ) HANGUL SYLLABLE BBOELT +BFA2;BFA2;1108 116C 11B5;BFA2;1108 116C 11B5; # (뾢; 뾢; 뾢; 뾢; 뾢; ) HANGUL SYLLABLE BBOELP +BFA3;BFA3;1108 116C 11B6;BFA3;1108 116C 11B6; # (뾣; 뾣; 뾣; 뾣; 뾣; ) HANGUL SYLLABLE BBOELH +BFA4;BFA4;1108 116C 11B7;BFA4;1108 116C 11B7; # (뾤; 뾤; 뾤; 뾤; 뾤; ) HANGUL SYLLABLE BBOEM +BFA5;BFA5;1108 116C 11B8;BFA5;1108 116C 11B8; # (뾥; 뾥; 뾥; 뾥; 뾥; ) HANGUL SYLLABLE BBOEB +BFA6;BFA6;1108 116C 11B9;BFA6;1108 116C 11B9; # (뾦; 뾦; 뾦; 뾦; 뾦; ) HANGUL SYLLABLE BBOEBS +BFA7;BFA7;1108 116C 11BA;BFA7;1108 116C 11BA; # (뾧; 뾧; 뾧; 뾧; 뾧; ) HANGUL SYLLABLE BBOES +BFA8;BFA8;1108 116C 11BB;BFA8;1108 116C 11BB; # (뾨; 뾨; 뾨; 뾨; 뾨; ) HANGUL SYLLABLE BBOESS +BFA9;BFA9;1108 116C 11BC;BFA9;1108 116C 11BC; # (뾩; 뾩; 뾩; 뾩; 뾩; ) HANGUL SYLLABLE BBOENG +BFAA;BFAA;1108 116C 11BD;BFAA;1108 116C 11BD; # (뾪; 뾪; 뾪; 뾪; 뾪; ) HANGUL SYLLABLE BBOEJ +BFAB;BFAB;1108 116C 11BE;BFAB;1108 116C 11BE; # (뾫; 뾫; 뾫; 뾫; 뾫; ) HANGUL SYLLABLE BBOEC +BFAC;BFAC;1108 116C 11BF;BFAC;1108 116C 11BF; # (뾬; 뾬; 뾬; 뾬; 뾬; ) HANGUL SYLLABLE BBOEK +BFAD;BFAD;1108 116C 11C0;BFAD;1108 116C 11C0; # (뾭; 뾭; 뾭; 뾭; 뾭; ) HANGUL SYLLABLE BBOET +BFAE;BFAE;1108 116C 11C1;BFAE;1108 116C 11C1; # (뾮; 뾮; 뾮; 뾮; 뾮; ) HANGUL SYLLABLE BBOEP +BFAF;BFAF;1108 116C 11C2;BFAF;1108 116C 11C2; # (뾯; 뾯; 뾯; 뾯; 뾯; ) HANGUL SYLLABLE BBOEH +BFB0;BFB0;1108 116D;BFB0;1108 116D; # (뾰; 뾰; 뾰; 뾰; 뾰; ) HANGUL SYLLABLE BBYO +BFB1;BFB1;1108 116D 11A8;BFB1;1108 116D 11A8; # (뾱; 뾱; 뾱; 뾱; 뾱; ) HANGUL SYLLABLE BBYOG +BFB2;BFB2;1108 116D 11A9;BFB2;1108 116D 11A9; # (뾲; 뾲; 뾲; 뾲; 뾲; ) HANGUL SYLLABLE BBYOGG +BFB3;BFB3;1108 116D 11AA;BFB3;1108 116D 11AA; # (뾳; 뾳; 뾳; 뾳; 뾳; ) HANGUL SYLLABLE BBYOGS +BFB4;BFB4;1108 116D 11AB;BFB4;1108 116D 11AB; # (뾴; 뾴; 뾴; 뾴; 뾴; ) HANGUL SYLLABLE BBYON +BFB5;BFB5;1108 116D 11AC;BFB5;1108 116D 11AC; # (뾵; 뾵; 뾵; 뾵; 뾵; ) HANGUL SYLLABLE BBYONJ +BFB6;BFB6;1108 116D 11AD;BFB6;1108 116D 11AD; # (뾶; 뾶; 뾶; 뾶; 뾶; ) HANGUL SYLLABLE BBYONH +BFB7;BFB7;1108 116D 11AE;BFB7;1108 116D 11AE; # (뾷; 뾷; 뾷; 뾷; 뾷; ) HANGUL SYLLABLE BBYOD +BFB8;BFB8;1108 116D 11AF;BFB8;1108 116D 11AF; # (뾸; 뾸; 뾸; 뾸; 뾸; ) HANGUL SYLLABLE BBYOL +BFB9;BFB9;1108 116D 11B0;BFB9;1108 116D 11B0; # (뾹; 뾹; 뾹; 뾹; 뾹; ) HANGUL SYLLABLE BBYOLG +BFBA;BFBA;1108 116D 11B1;BFBA;1108 116D 11B1; # (뾺; 뾺; 뾺; 뾺; 뾺; ) HANGUL SYLLABLE BBYOLM +BFBB;BFBB;1108 116D 11B2;BFBB;1108 116D 11B2; # (뾻; 뾻; 뾻; 뾻; 뾻; ) HANGUL SYLLABLE BBYOLB +BFBC;BFBC;1108 116D 11B3;BFBC;1108 116D 11B3; # (뾼; 뾼; 뾼; 뾼; 뾼; ) HANGUL SYLLABLE BBYOLS +BFBD;BFBD;1108 116D 11B4;BFBD;1108 116D 11B4; # (뾽; 뾽; 뾽; 뾽; 뾽; ) HANGUL SYLLABLE BBYOLT +BFBE;BFBE;1108 116D 11B5;BFBE;1108 116D 11B5; # (뾾; 뾾; 뾾; 뾾; 뾾; ) HANGUL SYLLABLE BBYOLP +BFBF;BFBF;1108 116D 11B6;BFBF;1108 116D 11B6; # (뾿; 뾿; 뾿; 뾿; 뾿; ) HANGUL SYLLABLE BBYOLH +BFC0;BFC0;1108 116D 11B7;BFC0;1108 116D 11B7; # (뿀; 뿀; 뿀; 뿀; 뿀; ) HANGUL SYLLABLE BBYOM +BFC1;BFC1;1108 116D 11B8;BFC1;1108 116D 11B8; # (뿁; 뿁; 뿁; 뿁; 뿁; ) HANGUL SYLLABLE BBYOB +BFC2;BFC2;1108 116D 11B9;BFC2;1108 116D 11B9; # (뿂; 뿂; 뿂; 뿂; 뿂; ) HANGUL SYLLABLE BBYOBS +BFC3;BFC3;1108 116D 11BA;BFC3;1108 116D 11BA; # (뿃; 뿃; 뿃; 뿃; 뿃; ) HANGUL SYLLABLE BBYOS +BFC4;BFC4;1108 116D 11BB;BFC4;1108 116D 11BB; # (뿄; 뿄; 뿄; 뿄; 뿄; ) HANGUL SYLLABLE BBYOSS +BFC5;BFC5;1108 116D 11BC;BFC5;1108 116D 11BC; # (뿅; 뿅; 뿅; 뿅; 뿅; ) HANGUL SYLLABLE BBYONG +BFC6;BFC6;1108 116D 11BD;BFC6;1108 116D 11BD; # (뿆; 뿆; 뿆; 뿆; 뿆; ) HANGUL SYLLABLE BBYOJ +BFC7;BFC7;1108 116D 11BE;BFC7;1108 116D 11BE; # (뿇; 뿇; 뿇; 뿇; 뿇; ) HANGUL SYLLABLE BBYOC +BFC8;BFC8;1108 116D 11BF;BFC8;1108 116D 11BF; # (뿈; 뿈; 뿈; 뿈; 뿈; ) HANGUL SYLLABLE BBYOK +BFC9;BFC9;1108 116D 11C0;BFC9;1108 116D 11C0; # (뿉; 뿉; 뿉; 뿉; 뿉; ) HANGUL SYLLABLE BBYOT +BFCA;BFCA;1108 116D 11C1;BFCA;1108 116D 11C1; # (뿊; 뿊; 뿊; 뿊; 뿊; ) HANGUL SYLLABLE BBYOP +BFCB;BFCB;1108 116D 11C2;BFCB;1108 116D 11C2; # (뿋; 뿋; 뿋; 뿋; 뿋; ) HANGUL SYLLABLE BBYOH +BFCC;BFCC;1108 116E;BFCC;1108 116E; # (뿌; 뿌; 뿌; 뿌; 뿌; ) HANGUL SYLLABLE BBU +BFCD;BFCD;1108 116E 11A8;BFCD;1108 116E 11A8; # (뿍; 뿍; 뿍; 뿍; 뿍; ) HANGUL SYLLABLE BBUG +BFCE;BFCE;1108 116E 11A9;BFCE;1108 116E 11A9; # (뿎; 뿎; 뿎; 뿎; 뿎; ) HANGUL SYLLABLE BBUGG +BFCF;BFCF;1108 116E 11AA;BFCF;1108 116E 11AA; # (뿏; 뿏; 뿏; 뿏; 뿏; ) HANGUL SYLLABLE BBUGS +BFD0;BFD0;1108 116E 11AB;BFD0;1108 116E 11AB; # (뿐; 뿐; 뿐; 뿐; 뿐; ) HANGUL SYLLABLE BBUN +BFD1;BFD1;1108 116E 11AC;BFD1;1108 116E 11AC; # (뿑; 뿑; 뿑; 뿑; 뿑; ) HANGUL SYLLABLE BBUNJ +BFD2;BFD2;1108 116E 11AD;BFD2;1108 116E 11AD; # (뿒; 뿒; 뿒; 뿒; 뿒; ) HANGUL SYLLABLE BBUNH +BFD3;BFD3;1108 116E 11AE;BFD3;1108 116E 11AE; # (뿓; 뿓; 뿓; 뿓; 뿓; ) HANGUL SYLLABLE BBUD +BFD4;BFD4;1108 116E 11AF;BFD4;1108 116E 11AF; # (뿔; 뿔; 뿔; 뿔; 뿔; ) HANGUL SYLLABLE BBUL +BFD5;BFD5;1108 116E 11B0;BFD5;1108 116E 11B0; # (뿕; 뿕; 뿕; 뿕; 뿕; ) HANGUL SYLLABLE BBULG +BFD6;BFD6;1108 116E 11B1;BFD6;1108 116E 11B1; # (뿖; 뿖; 뿖; 뿖; 뿖; ) HANGUL SYLLABLE BBULM +BFD7;BFD7;1108 116E 11B2;BFD7;1108 116E 11B2; # (뿗; 뿗; 뿗; 뿗; 뿗; ) HANGUL SYLLABLE BBULB +BFD8;BFD8;1108 116E 11B3;BFD8;1108 116E 11B3; # (뿘; 뿘; 뿘; 뿘; 뿘; ) HANGUL SYLLABLE BBULS +BFD9;BFD9;1108 116E 11B4;BFD9;1108 116E 11B4; # (뿙; 뿙; 뿙; 뿙; 뿙; ) HANGUL SYLLABLE BBULT +BFDA;BFDA;1108 116E 11B5;BFDA;1108 116E 11B5; # (뿚; 뿚; 뿚; 뿚; 뿚; ) HANGUL SYLLABLE BBULP +BFDB;BFDB;1108 116E 11B6;BFDB;1108 116E 11B6; # (뿛; 뿛; 뿛; 뿛; 뿛; ) HANGUL SYLLABLE BBULH +BFDC;BFDC;1108 116E 11B7;BFDC;1108 116E 11B7; # (뿜; 뿜; 뿜; 뿜; 뿜; ) HANGUL SYLLABLE BBUM +BFDD;BFDD;1108 116E 11B8;BFDD;1108 116E 11B8; # (뿝; 뿝; 뿝; 뿝; 뿝; ) HANGUL SYLLABLE BBUB +BFDE;BFDE;1108 116E 11B9;BFDE;1108 116E 11B9; # (뿞; 뿞; 뿞; 뿞; 뿞; ) HANGUL SYLLABLE BBUBS +BFDF;BFDF;1108 116E 11BA;BFDF;1108 116E 11BA; # (뿟; 뿟; 뿟; 뿟; 뿟; ) HANGUL SYLLABLE BBUS +BFE0;BFE0;1108 116E 11BB;BFE0;1108 116E 11BB; # (뿠; 뿠; 뿠; 뿠; 뿠; ) HANGUL SYLLABLE BBUSS +BFE1;BFE1;1108 116E 11BC;BFE1;1108 116E 11BC; # (뿡; 뿡; 뿡; 뿡; 뿡; ) HANGUL SYLLABLE BBUNG +BFE2;BFE2;1108 116E 11BD;BFE2;1108 116E 11BD; # (뿢; 뿢; 뿢; 뿢; 뿢; ) HANGUL SYLLABLE BBUJ +BFE3;BFE3;1108 116E 11BE;BFE3;1108 116E 11BE; # (뿣; 뿣; 뿣; 뿣; 뿣; ) HANGUL SYLLABLE BBUC +BFE4;BFE4;1108 116E 11BF;BFE4;1108 116E 11BF; # (뿤; 뿤; 뿤; 뿤; 뿤; ) HANGUL SYLLABLE BBUK +BFE5;BFE5;1108 116E 11C0;BFE5;1108 116E 11C0; # (뿥; 뿥; 뿥; 뿥; 뿥; ) HANGUL SYLLABLE BBUT +BFE6;BFE6;1108 116E 11C1;BFE6;1108 116E 11C1; # (뿦; 뿦; 뿦; 뿦; 뿦; ) HANGUL SYLLABLE BBUP +BFE7;BFE7;1108 116E 11C2;BFE7;1108 116E 11C2; # (뿧; 뿧; 뿧; 뿧; 뿧; ) HANGUL SYLLABLE BBUH +BFE8;BFE8;1108 116F;BFE8;1108 116F; # (뿨; 뿨; 뿨; 뿨; 뿨; ) HANGUL SYLLABLE BBWEO +BFE9;BFE9;1108 116F 11A8;BFE9;1108 116F 11A8; # (뿩; 뿩; 뿩; 뿩; 뿩; ) HANGUL SYLLABLE BBWEOG +BFEA;BFEA;1108 116F 11A9;BFEA;1108 116F 11A9; # (뿪; 뿪; 뿪; 뿪; 뿪; ) HANGUL SYLLABLE BBWEOGG +BFEB;BFEB;1108 116F 11AA;BFEB;1108 116F 11AA; # (뿫; 뿫; 뿫; 뿫; 뿫; ) HANGUL SYLLABLE BBWEOGS +BFEC;BFEC;1108 116F 11AB;BFEC;1108 116F 11AB; # (뿬; 뿬; 뿬; 뿬; 뿬; ) HANGUL SYLLABLE BBWEON +BFED;BFED;1108 116F 11AC;BFED;1108 116F 11AC; # (뿭; 뿭; 뿭; 뿭; 뿭; ) HANGUL SYLLABLE BBWEONJ +BFEE;BFEE;1108 116F 11AD;BFEE;1108 116F 11AD; # (뿮; 뿮; 뿮; 뿮; 뿮; ) HANGUL SYLLABLE BBWEONH +BFEF;BFEF;1108 116F 11AE;BFEF;1108 116F 11AE; # (뿯; 뿯; 뿯; 뿯; 뿯; ) HANGUL SYLLABLE BBWEOD +BFF0;BFF0;1108 116F 11AF;BFF0;1108 116F 11AF; # (뿰; 뿰; 뿰; 뿰; 뿰; ) HANGUL SYLLABLE BBWEOL +BFF1;BFF1;1108 116F 11B0;BFF1;1108 116F 11B0; # (뿱; 뿱; 뿱; 뿱; 뿱; ) HANGUL SYLLABLE BBWEOLG +BFF2;BFF2;1108 116F 11B1;BFF2;1108 116F 11B1; # (뿲; 뿲; 뿲; 뿲; 뿲; ) HANGUL SYLLABLE BBWEOLM +BFF3;BFF3;1108 116F 11B2;BFF3;1108 116F 11B2; # (뿳; 뿳; 뿳; 뿳; 뿳; ) HANGUL SYLLABLE BBWEOLB +BFF4;BFF4;1108 116F 11B3;BFF4;1108 116F 11B3; # (뿴; 뿴; 뿴; 뿴; 뿴; ) HANGUL SYLLABLE BBWEOLS +BFF5;BFF5;1108 116F 11B4;BFF5;1108 116F 11B4; # (뿵; 뿵; 뿵; 뿵; 뿵; ) HANGUL SYLLABLE BBWEOLT +BFF6;BFF6;1108 116F 11B5;BFF6;1108 116F 11B5; # (뿶; 뿶; 뿶; 뿶; 뿶; ) HANGUL SYLLABLE BBWEOLP +BFF7;BFF7;1108 116F 11B6;BFF7;1108 116F 11B6; # (뿷; 뿷; 뿷; 뿷; 뿷; ) HANGUL SYLLABLE BBWEOLH +BFF8;BFF8;1108 116F 11B7;BFF8;1108 116F 11B7; # (뿸; 뿸; 뿸; 뿸; 뿸; ) HANGUL SYLLABLE BBWEOM +BFF9;BFF9;1108 116F 11B8;BFF9;1108 116F 11B8; # (뿹; 뿹; 뿹; 뿹; 뿹; ) HANGUL SYLLABLE BBWEOB +BFFA;BFFA;1108 116F 11B9;BFFA;1108 116F 11B9; # (뿺; 뿺; 뿺; 뿺; 뿺; ) HANGUL SYLLABLE BBWEOBS +BFFB;BFFB;1108 116F 11BA;BFFB;1108 116F 11BA; # (뿻; 뿻; 뿻; 뿻; 뿻; ) HANGUL SYLLABLE BBWEOS +BFFC;BFFC;1108 116F 11BB;BFFC;1108 116F 11BB; # (뿼; 뿼; 뿼; 뿼; 뿼; ) HANGUL SYLLABLE BBWEOSS +BFFD;BFFD;1108 116F 11BC;BFFD;1108 116F 11BC; # (뿽; 뿽; 뿽; 뿽; 뿽; ) HANGUL SYLLABLE BBWEONG +BFFE;BFFE;1108 116F 11BD;BFFE;1108 116F 11BD; # (뿾; 뿾; 뿾; 뿾; 뿾; ) HANGUL SYLLABLE BBWEOJ +BFFF;BFFF;1108 116F 11BE;BFFF;1108 116F 11BE; # (뿿; 뿿; 뿿; 뿿; 뿿; ) HANGUL SYLLABLE BBWEOC +C000;C000;1108 116F 11BF;C000;1108 116F 11BF; # (쀀; 쀀; 쀀; 쀀; 쀀; ) HANGUL SYLLABLE BBWEOK +C001;C001;1108 116F 11C0;C001;1108 116F 11C0; # (쀁; 쀁; 쀁; 쀁; 쀁; ) HANGUL SYLLABLE BBWEOT +C002;C002;1108 116F 11C1;C002;1108 116F 11C1; # (쀂; 쀂; 쀂; 쀂; 쀂; ) HANGUL SYLLABLE BBWEOP +C003;C003;1108 116F 11C2;C003;1108 116F 11C2; # (쀃; 쀃; 쀃; 쀃; 쀃; ) HANGUL SYLLABLE BBWEOH +C004;C004;1108 1170;C004;1108 1170; # (쀄; 쀄; 쀄; 쀄; 쀄; ) HANGUL SYLLABLE BBWE +C005;C005;1108 1170 11A8;C005;1108 1170 11A8; # (쀅; 쀅; 쀅; 쀅; 쀅; ) HANGUL SYLLABLE BBWEG +C006;C006;1108 1170 11A9;C006;1108 1170 11A9; # (쀆; 쀆; 쀆; 쀆; 쀆; ) HANGUL SYLLABLE BBWEGG +C007;C007;1108 1170 11AA;C007;1108 1170 11AA; # (쀇; 쀇; 쀇; 쀇; 쀇; ) HANGUL SYLLABLE BBWEGS +C008;C008;1108 1170 11AB;C008;1108 1170 11AB; # (쀈; 쀈; 쀈; 쀈; 쀈; ) HANGUL SYLLABLE BBWEN +C009;C009;1108 1170 11AC;C009;1108 1170 11AC; # (쀉; 쀉; 쀉; 쀉; 쀉; ) HANGUL SYLLABLE BBWENJ +C00A;C00A;1108 1170 11AD;C00A;1108 1170 11AD; # (쀊; 쀊; 쀊; 쀊; 쀊; ) HANGUL SYLLABLE BBWENH +C00B;C00B;1108 1170 11AE;C00B;1108 1170 11AE; # (쀋; 쀋; 쀋; 쀋; 쀋; ) HANGUL SYLLABLE BBWED +C00C;C00C;1108 1170 11AF;C00C;1108 1170 11AF; # (쀌; 쀌; 쀌; 쀌; 쀌; ) HANGUL SYLLABLE BBWEL +C00D;C00D;1108 1170 11B0;C00D;1108 1170 11B0; # (쀍; 쀍; 쀍; 쀍; 쀍; ) HANGUL SYLLABLE BBWELG +C00E;C00E;1108 1170 11B1;C00E;1108 1170 11B1; # (쀎; 쀎; 쀎; 쀎; 쀎; ) HANGUL SYLLABLE BBWELM +C00F;C00F;1108 1170 11B2;C00F;1108 1170 11B2; # (쀏; 쀏; 쀏; 쀏; 쀏; ) HANGUL SYLLABLE BBWELB +C010;C010;1108 1170 11B3;C010;1108 1170 11B3; # (쀐; 쀐; 쀐; 쀐; 쀐; ) HANGUL SYLLABLE BBWELS +C011;C011;1108 1170 11B4;C011;1108 1170 11B4; # (쀑; 쀑; 쀑; 쀑; 쀑; ) HANGUL SYLLABLE BBWELT +C012;C012;1108 1170 11B5;C012;1108 1170 11B5; # (쀒; 쀒; 쀒; 쀒; 쀒; ) HANGUL SYLLABLE BBWELP +C013;C013;1108 1170 11B6;C013;1108 1170 11B6; # (쀓; 쀓; 쀓; 쀓; 쀓; ) HANGUL SYLLABLE BBWELH +C014;C014;1108 1170 11B7;C014;1108 1170 11B7; # (쀔; 쀔; 쀔; 쀔; 쀔; ) HANGUL SYLLABLE BBWEM +C015;C015;1108 1170 11B8;C015;1108 1170 11B8; # (쀕; 쀕; 쀕; 쀕; 쀕; ) HANGUL SYLLABLE BBWEB +C016;C016;1108 1170 11B9;C016;1108 1170 11B9; # (쀖; 쀖; 쀖; 쀖; 쀖; ) HANGUL SYLLABLE BBWEBS +C017;C017;1108 1170 11BA;C017;1108 1170 11BA; # (쀗; 쀗; 쀗; 쀗; 쀗; ) HANGUL SYLLABLE BBWES +C018;C018;1108 1170 11BB;C018;1108 1170 11BB; # (쀘; 쀘; 쀘; 쀘; 쀘; ) HANGUL SYLLABLE BBWESS +C019;C019;1108 1170 11BC;C019;1108 1170 11BC; # (쀙; 쀙; 쀙; 쀙; 쀙; ) HANGUL SYLLABLE BBWENG +C01A;C01A;1108 1170 11BD;C01A;1108 1170 11BD; # (쀚; 쀚; 쀚; 쀚; 쀚; ) HANGUL SYLLABLE BBWEJ +C01B;C01B;1108 1170 11BE;C01B;1108 1170 11BE; # (쀛; 쀛; 쀛; 쀛; 쀛; ) HANGUL SYLLABLE BBWEC +C01C;C01C;1108 1170 11BF;C01C;1108 1170 11BF; # (쀜; 쀜; 쀜; 쀜; 쀜; ) HANGUL SYLLABLE BBWEK +C01D;C01D;1108 1170 11C0;C01D;1108 1170 11C0; # (쀝; 쀝; 쀝; 쀝; 쀝; ) HANGUL SYLLABLE BBWET +C01E;C01E;1108 1170 11C1;C01E;1108 1170 11C1; # (쀞; 쀞; 쀞; 쀞; 쀞; ) HANGUL SYLLABLE BBWEP +C01F;C01F;1108 1170 11C2;C01F;1108 1170 11C2; # (쀟; 쀟; 쀟; 쀟; 쀟; ) HANGUL SYLLABLE BBWEH +C020;C020;1108 1171;C020;1108 1171; # (쀠; 쀠; 쀠; 쀠; 쀠; ) HANGUL SYLLABLE BBWI +C021;C021;1108 1171 11A8;C021;1108 1171 11A8; # (쀡; 쀡; 쀡; 쀡; 쀡; ) HANGUL SYLLABLE BBWIG +C022;C022;1108 1171 11A9;C022;1108 1171 11A9; # (쀢; 쀢; 쀢; 쀢; 쀢; ) HANGUL SYLLABLE BBWIGG +C023;C023;1108 1171 11AA;C023;1108 1171 11AA; # (쀣; 쀣; 쀣; 쀣; 쀣; ) HANGUL SYLLABLE BBWIGS +C024;C024;1108 1171 11AB;C024;1108 1171 11AB; # (쀤; 쀤; 쀤; 쀤; 쀤; ) HANGUL SYLLABLE BBWIN +C025;C025;1108 1171 11AC;C025;1108 1171 11AC; # (쀥; 쀥; 쀥; 쀥; 쀥; ) HANGUL SYLLABLE BBWINJ +C026;C026;1108 1171 11AD;C026;1108 1171 11AD; # (쀦; 쀦; 쀦; 쀦; 쀦; ) HANGUL SYLLABLE BBWINH +C027;C027;1108 1171 11AE;C027;1108 1171 11AE; # (쀧; 쀧; 쀧; 쀧; 쀧; ) HANGUL SYLLABLE BBWID +C028;C028;1108 1171 11AF;C028;1108 1171 11AF; # (쀨; 쀨; 쀨; 쀨; 쀨; ) HANGUL SYLLABLE BBWIL +C029;C029;1108 1171 11B0;C029;1108 1171 11B0; # (쀩; 쀩; 쀩; 쀩; 쀩; ) HANGUL SYLLABLE BBWILG +C02A;C02A;1108 1171 11B1;C02A;1108 1171 11B1; # (쀪; 쀪; 쀪; 쀪; 쀪; ) HANGUL SYLLABLE BBWILM +C02B;C02B;1108 1171 11B2;C02B;1108 1171 11B2; # (쀫; 쀫; 쀫; 쀫; 쀫; ) HANGUL SYLLABLE BBWILB +C02C;C02C;1108 1171 11B3;C02C;1108 1171 11B3; # (쀬; 쀬; 쀬; 쀬; 쀬; ) HANGUL SYLLABLE BBWILS +C02D;C02D;1108 1171 11B4;C02D;1108 1171 11B4; # (쀭; 쀭; 쀭; 쀭; 쀭; ) HANGUL SYLLABLE BBWILT +C02E;C02E;1108 1171 11B5;C02E;1108 1171 11B5; # (쀮; 쀮; 쀮; 쀮; 쀮; ) HANGUL SYLLABLE BBWILP +C02F;C02F;1108 1171 11B6;C02F;1108 1171 11B6; # (쀯; 쀯; 쀯; 쀯; 쀯; ) HANGUL SYLLABLE BBWILH +C030;C030;1108 1171 11B7;C030;1108 1171 11B7; # (쀰; 쀰; 쀰; 쀰; 쀰; ) HANGUL SYLLABLE BBWIM +C031;C031;1108 1171 11B8;C031;1108 1171 11B8; # (쀱; 쀱; 쀱; 쀱; 쀱; ) HANGUL SYLLABLE BBWIB +C032;C032;1108 1171 11B9;C032;1108 1171 11B9; # (쀲; 쀲; 쀲; 쀲; 쀲; ) HANGUL SYLLABLE BBWIBS +C033;C033;1108 1171 11BA;C033;1108 1171 11BA; # (쀳; 쀳; 쀳; 쀳; 쀳; ) HANGUL SYLLABLE BBWIS +C034;C034;1108 1171 11BB;C034;1108 1171 11BB; # (쀴; 쀴; 쀴; 쀴; 쀴; ) HANGUL SYLLABLE BBWISS +C035;C035;1108 1171 11BC;C035;1108 1171 11BC; # (쀵; 쀵; 쀵; 쀵; 쀵; ) HANGUL SYLLABLE BBWING +C036;C036;1108 1171 11BD;C036;1108 1171 11BD; # (쀶; 쀶; 쀶; 쀶; 쀶; ) HANGUL SYLLABLE BBWIJ +C037;C037;1108 1171 11BE;C037;1108 1171 11BE; # (쀷; 쀷; 쀷; 쀷; 쀷; ) HANGUL SYLLABLE BBWIC +C038;C038;1108 1171 11BF;C038;1108 1171 11BF; # (쀸; 쀸; 쀸; 쀸; 쀸; ) HANGUL SYLLABLE BBWIK +C039;C039;1108 1171 11C0;C039;1108 1171 11C0; # (쀹; 쀹; 쀹; 쀹; 쀹; ) HANGUL SYLLABLE BBWIT +C03A;C03A;1108 1171 11C1;C03A;1108 1171 11C1; # (쀺; 쀺; 쀺; 쀺; 쀺; ) HANGUL SYLLABLE BBWIP +C03B;C03B;1108 1171 11C2;C03B;1108 1171 11C2; # (쀻; 쀻; 쀻; 쀻; 쀻; ) HANGUL SYLLABLE BBWIH +C03C;C03C;1108 1172;C03C;1108 1172; # (쀼; 쀼; 쀼; 쀼; 쀼; ) HANGUL SYLLABLE BBYU +C03D;C03D;1108 1172 11A8;C03D;1108 1172 11A8; # (쀽; 쀽; 쀽; 쀽; 쀽; ) HANGUL SYLLABLE BBYUG +C03E;C03E;1108 1172 11A9;C03E;1108 1172 11A9; # (쀾; 쀾; 쀾; 쀾; 쀾; ) HANGUL SYLLABLE BBYUGG +C03F;C03F;1108 1172 11AA;C03F;1108 1172 11AA; # (쀿; 쀿; 쀿; 쀿; 쀿; ) HANGUL SYLLABLE BBYUGS +C040;C040;1108 1172 11AB;C040;1108 1172 11AB; # (쁀; 쁀; 쁀; 쁀; 쁀; ) HANGUL SYLLABLE BBYUN +C041;C041;1108 1172 11AC;C041;1108 1172 11AC; # (쁁; 쁁; 쁁; 쁁; 쁁; ) HANGUL SYLLABLE BBYUNJ +C042;C042;1108 1172 11AD;C042;1108 1172 11AD; # (쁂; 쁂; 쁂; 쁂; 쁂; ) HANGUL SYLLABLE BBYUNH +C043;C043;1108 1172 11AE;C043;1108 1172 11AE; # (쁃; 쁃; 쁃; 쁃; 쁃; ) HANGUL SYLLABLE BBYUD +C044;C044;1108 1172 11AF;C044;1108 1172 11AF; # (쁄; 쁄; 쁄; 쁄; 쁄; ) HANGUL SYLLABLE BBYUL +C045;C045;1108 1172 11B0;C045;1108 1172 11B0; # (쁅; 쁅; 쁅; 쁅; 쁅; ) HANGUL SYLLABLE BBYULG +C046;C046;1108 1172 11B1;C046;1108 1172 11B1; # (쁆; 쁆; 쁆; 쁆; 쁆; ) HANGUL SYLLABLE BBYULM +C047;C047;1108 1172 11B2;C047;1108 1172 11B2; # (쁇; 쁇; 쁇; 쁇; 쁇; ) HANGUL SYLLABLE BBYULB +C048;C048;1108 1172 11B3;C048;1108 1172 11B3; # (쁈; 쁈; 쁈; 쁈; 쁈; ) HANGUL SYLLABLE BBYULS +C049;C049;1108 1172 11B4;C049;1108 1172 11B4; # (쁉; 쁉; 쁉; 쁉; 쁉; ) HANGUL SYLLABLE BBYULT +C04A;C04A;1108 1172 11B5;C04A;1108 1172 11B5; # (쁊; 쁊; 쁊; 쁊; 쁊; ) HANGUL SYLLABLE BBYULP +C04B;C04B;1108 1172 11B6;C04B;1108 1172 11B6; # (쁋; 쁋; 쁋; 쁋; 쁋; ) HANGUL SYLLABLE BBYULH +C04C;C04C;1108 1172 11B7;C04C;1108 1172 11B7; # (쁌; 쁌; 쁌; 쁌; 쁌; ) HANGUL SYLLABLE BBYUM +C04D;C04D;1108 1172 11B8;C04D;1108 1172 11B8; # (쁍; 쁍; 쁍; 쁍; 쁍; ) HANGUL SYLLABLE BBYUB +C04E;C04E;1108 1172 11B9;C04E;1108 1172 11B9; # (쁎; 쁎; 쁎; 쁎; 쁎; ) HANGUL SYLLABLE BBYUBS +C04F;C04F;1108 1172 11BA;C04F;1108 1172 11BA; # (쁏; 쁏; 쁏; 쁏; 쁏; ) HANGUL SYLLABLE BBYUS +C050;C050;1108 1172 11BB;C050;1108 1172 11BB; # (쁐; 쁐; 쁐; 쁐; 쁐; ) HANGUL SYLLABLE BBYUSS +C051;C051;1108 1172 11BC;C051;1108 1172 11BC; # (쁑; 쁑; 쁑; 쁑; 쁑; ) HANGUL SYLLABLE BBYUNG +C052;C052;1108 1172 11BD;C052;1108 1172 11BD; # (쁒; 쁒; 쁒; 쁒; 쁒; ) HANGUL SYLLABLE BBYUJ +C053;C053;1108 1172 11BE;C053;1108 1172 11BE; # (쁓; 쁓; 쁓; 쁓; 쁓; ) HANGUL SYLLABLE BBYUC +C054;C054;1108 1172 11BF;C054;1108 1172 11BF; # (쁔; 쁔; 쁔; 쁔; 쁔; ) HANGUL SYLLABLE BBYUK +C055;C055;1108 1172 11C0;C055;1108 1172 11C0; # (쁕; 쁕; 쁕; 쁕; 쁕; ) HANGUL SYLLABLE BBYUT +C056;C056;1108 1172 11C1;C056;1108 1172 11C1; # (쁖; 쁖; 쁖; 쁖; 쁖; ) HANGUL SYLLABLE BBYUP +C057;C057;1108 1172 11C2;C057;1108 1172 11C2; # (쁗; 쁗; 쁗; 쁗; 쁗; ) HANGUL SYLLABLE BBYUH +C058;C058;1108 1173;C058;1108 1173; # (쁘; 쁘; 쁘; 쁘; 쁘; ) HANGUL SYLLABLE BBEU +C059;C059;1108 1173 11A8;C059;1108 1173 11A8; # (쁙; 쁙; 쁙; 쁙; 쁙; ) HANGUL SYLLABLE BBEUG +C05A;C05A;1108 1173 11A9;C05A;1108 1173 11A9; # (쁚; 쁚; 쁚; 쁚; 쁚; ) HANGUL SYLLABLE BBEUGG +C05B;C05B;1108 1173 11AA;C05B;1108 1173 11AA; # (쁛; 쁛; 쁛; 쁛; 쁛; ) HANGUL SYLLABLE BBEUGS +C05C;C05C;1108 1173 11AB;C05C;1108 1173 11AB; # (쁜; 쁜; 쁜; 쁜; 쁜; ) HANGUL SYLLABLE BBEUN +C05D;C05D;1108 1173 11AC;C05D;1108 1173 11AC; # (쁝; 쁝; 쁝; 쁝; 쁝; ) HANGUL SYLLABLE BBEUNJ +C05E;C05E;1108 1173 11AD;C05E;1108 1173 11AD; # (쁞; 쁞; 쁞; 쁞; 쁞; ) HANGUL SYLLABLE BBEUNH +C05F;C05F;1108 1173 11AE;C05F;1108 1173 11AE; # (쁟; 쁟; 쁟; 쁟; 쁟; ) HANGUL SYLLABLE BBEUD +C060;C060;1108 1173 11AF;C060;1108 1173 11AF; # (쁠; 쁠; 쁠; 쁠; 쁠; ) HANGUL SYLLABLE BBEUL +C061;C061;1108 1173 11B0;C061;1108 1173 11B0; # (쁡; 쁡; 쁡; 쁡; 쁡; ) HANGUL SYLLABLE BBEULG +C062;C062;1108 1173 11B1;C062;1108 1173 11B1; # (쁢; 쁢; 쁢; 쁢; 쁢; ) HANGUL SYLLABLE BBEULM +C063;C063;1108 1173 11B2;C063;1108 1173 11B2; # (쁣; 쁣; 쁣; 쁣; 쁣; ) HANGUL SYLLABLE BBEULB +C064;C064;1108 1173 11B3;C064;1108 1173 11B3; # (쁤; 쁤; 쁤; 쁤; 쁤; ) HANGUL SYLLABLE BBEULS +C065;C065;1108 1173 11B4;C065;1108 1173 11B4; # (쁥; 쁥; 쁥; 쁥; 쁥; ) HANGUL SYLLABLE BBEULT +C066;C066;1108 1173 11B5;C066;1108 1173 11B5; # (쁦; 쁦; 쁦; 쁦; 쁦; ) HANGUL SYLLABLE BBEULP +C067;C067;1108 1173 11B6;C067;1108 1173 11B6; # (쁧; 쁧; 쁧; 쁧; 쁧; ) HANGUL SYLLABLE BBEULH +C068;C068;1108 1173 11B7;C068;1108 1173 11B7; # (쁨; 쁨; 쁨; 쁨; 쁨; ) HANGUL SYLLABLE BBEUM +C069;C069;1108 1173 11B8;C069;1108 1173 11B8; # (쁩; 쁩; 쁩; 쁩; 쁩; ) HANGUL SYLLABLE BBEUB +C06A;C06A;1108 1173 11B9;C06A;1108 1173 11B9; # (쁪; 쁪; 쁪; 쁪; 쁪; ) HANGUL SYLLABLE BBEUBS +C06B;C06B;1108 1173 11BA;C06B;1108 1173 11BA; # (쁫; 쁫; 쁫; 쁫; 쁫; ) HANGUL SYLLABLE BBEUS +C06C;C06C;1108 1173 11BB;C06C;1108 1173 11BB; # (쁬; 쁬; 쁬; 쁬; 쁬; ) HANGUL SYLLABLE BBEUSS +C06D;C06D;1108 1173 11BC;C06D;1108 1173 11BC; # (쁭; 쁭; 쁭; 쁭; 쁭; ) HANGUL SYLLABLE BBEUNG +C06E;C06E;1108 1173 11BD;C06E;1108 1173 11BD; # (쁮; 쁮; 쁮; 쁮; 쁮; ) HANGUL SYLLABLE BBEUJ +C06F;C06F;1108 1173 11BE;C06F;1108 1173 11BE; # (쁯; 쁯; 쁯; 쁯; 쁯; ) HANGUL SYLLABLE BBEUC +C070;C070;1108 1173 11BF;C070;1108 1173 11BF; # (쁰; 쁰; 쁰; 쁰; 쁰; ) HANGUL SYLLABLE BBEUK +C071;C071;1108 1173 11C0;C071;1108 1173 11C0; # (쁱; 쁱; 쁱; 쁱; 쁱; ) HANGUL SYLLABLE BBEUT +C072;C072;1108 1173 11C1;C072;1108 1173 11C1; # (쁲; 쁲; 쁲; 쁲; 쁲; ) HANGUL SYLLABLE BBEUP +C073;C073;1108 1173 11C2;C073;1108 1173 11C2; # (쁳; 쁳; 쁳; 쁳; 쁳; ) HANGUL SYLLABLE BBEUH +C074;C074;1108 1174;C074;1108 1174; # (쁴; 쁴; 쁴; 쁴; 쁴; ) HANGUL SYLLABLE BBYI +C075;C075;1108 1174 11A8;C075;1108 1174 11A8; # (쁵; 쁵; 쁵; 쁵; 쁵; ) HANGUL SYLLABLE BBYIG +C076;C076;1108 1174 11A9;C076;1108 1174 11A9; # (쁶; 쁶; 쁶; 쁶; 쁶; ) HANGUL SYLLABLE BBYIGG +C077;C077;1108 1174 11AA;C077;1108 1174 11AA; # (쁷; 쁷; 쁷; 쁷; 쁷; ) HANGUL SYLLABLE BBYIGS +C078;C078;1108 1174 11AB;C078;1108 1174 11AB; # (쁸; 쁸; 쁸; 쁸; 쁸; ) HANGUL SYLLABLE BBYIN +C079;C079;1108 1174 11AC;C079;1108 1174 11AC; # (쁹; 쁹; 쁹; 쁹; 쁹; ) HANGUL SYLLABLE BBYINJ +C07A;C07A;1108 1174 11AD;C07A;1108 1174 11AD; # (쁺; 쁺; 쁺; 쁺; 쁺; ) HANGUL SYLLABLE BBYINH +C07B;C07B;1108 1174 11AE;C07B;1108 1174 11AE; # (쁻; 쁻; 쁻; 쁻; 쁻; ) HANGUL SYLLABLE BBYID +C07C;C07C;1108 1174 11AF;C07C;1108 1174 11AF; # (쁼; 쁼; 쁼; 쁼; 쁼; ) HANGUL SYLLABLE BBYIL +C07D;C07D;1108 1174 11B0;C07D;1108 1174 11B0; # (쁽; 쁽; 쁽; 쁽; 쁽; ) HANGUL SYLLABLE BBYILG +C07E;C07E;1108 1174 11B1;C07E;1108 1174 11B1; # (쁾; 쁾; 쁾; 쁾; 쁾; ) HANGUL SYLLABLE BBYILM +C07F;C07F;1108 1174 11B2;C07F;1108 1174 11B2; # (쁿; 쁿; 쁿; 쁿; 쁿; ) HANGUL SYLLABLE BBYILB +C080;C080;1108 1174 11B3;C080;1108 1174 11B3; # (삀; 삀; 삀; 삀; 삀; ) HANGUL SYLLABLE BBYILS +C081;C081;1108 1174 11B4;C081;1108 1174 11B4; # (삁; 삁; 삁; 삁; 삁; ) HANGUL SYLLABLE BBYILT +C082;C082;1108 1174 11B5;C082;1108 1174 11B5; # (삂; 삂; 삂; 삂; 삂; ) HANGUL SYLLABLE BBYILP +C083;C083;1108 1174 11B6;C083;1108 1174 11B6; # (삃; 삃; 삃; 삃; 삃; ) HANGUL SYLLABLE BBYILH +C084;C084;1108 1174 11B7;C084;1108 1174 11B7; # (삄; 삄; 삄; 삄; 삄; ) HANGUL SYLLABLE BBYIM +C085;C085;1108 1174 11B8;C085;1108 1174 11B8; # (삅; 삅; 삅; 삅; 삅; ) HANGUL SYLLABLE BBYIB +C086;C086;1108 1174 11B9;C086;1108 1174 11B9; # (삆; 삆; 삆; 삆; 삆; ) HANGUL SYLLABLE BBYIBS +C087;C087;1108 1174 11BA;C087;1108 1174 11BA; # (삇; 삇; 삇; 삇; 삇; ) HANGUL SYLLABLE BBYIS +C088;C088;1108 1174 11BB;C088;1108 1174 11BB; # (삈; 삈; 삈; 삈; 삈; ) HANGUL SYLLABLE BBYISS +C089;C089;1108 1174 11BC;C089;1108 1174 11BC; # (삉; 삉; 삉; 삉; 삉; ) HANGUL SYLLABLE BBYING +C08A;C08A;1108 1174 11BD;C08A;1108 1174 11BD; # (삊; 삊; 삊; 삊; 삊; ) HANGUL SYLLABLE BBYIJ +C08B;C08B;1108 1174 11BE;C08B;1108 1174 11BE; # (삋; 삋; 삋; 삋; 삋; ) HANGUL SYLLABLE BBYIC +C08C;C08C;1108 1174 11BF;C08C;1108 1174 11BF; # (삌; 삌; 삌; 삌; 삌; ) HANGUL SYLLABLE BBYIK +C08D;C08D;1108 1174 11C0;C08D;1108 1174 11C0; # (삍; 삍; 삍; 삍; 삍; ) HANGUL SYLLABLE BBYIT +C08E;C08E;1108 1174 11C1;C08E;1108 1174 11C1; # (삎; 삎; 삎; 삎; 삎; ) HANGUL SYLLABLE BBYIP +C08F;C08F;1108 1174 11C2;C08F;1108 1174 11C2; # (삏; 삏; 삏; 삏; 삏; ) HANGUL SYLLABLE BBYIH +C090;C090;1108 1175;C090;1108 1175; # (삐; 삐; 삐; 삐; 삐; ) HANGUL SYLLABLE BBI +C091;C091;1108 1175 11A8;C091;1108 1175 11A8; # (삑; 삑; 삑; 삑; 삑; ) HANGUL SYLLABLE BBIG +C092;C092;1108 1175 11A9;C092;1108 1175 11A9; # (삒; 삒; 삒; 삒; 삒; ) HANGUL SYLLABLE BBIGG +C093;C093;1108 1175 11AA;C093;1108 1175 11AA; # (삓; 삓; 삓; 삓; 삓; ) HANGUL SYLLABLE BBIGS +C094;C094;1108 1175 11AB;C094;1108 1175 11AB; # (삔; 삔; 삔; 삔; 삔; ) HANGUL SYLLABLE BBIN +C095;C095;1108 1175 11AC;C095;1108 1175 11AC; # (삕; 삕; 삕; 삕; 삕; ) HANGUL SYLLABLE BBINJ +C096;C096;1108 1175 11AD;C096;1108 1175 11AD; # (삖; 삖; 삖; 삖; 삖; ) HANGUL SYLLABLE BBINH +C097;C097;1108 1175 11AE;C097;1108 1175 11AE; # (삗; 삗; 삗; 삗; 삗; ) HANGUL SYLLABLE BBID +C098;C098;1108 1175 11AF;C098;1108 1175 11AF; # (삘; 삘; 삘; 삘; 삘; ) HANGUL SYLLABLE BBIL +C099;C099;1108 1175 11B0;C099;1108 1175 11B0; # (삙; 삙; 삙; 삙; 삙; ) HANGUL SYLLABLE BBILG +C09A;C09A;1108 1175 11B1;C09A;1108 1175 11B1; # (삚; 삚; 삚; 삚; 삚; ) HANGUL SYLLABLE BBILM +C09B;C09B;1108 1175 11B2;C09B;1108 1175 11B2; # (삛; 삛; 삛; 삛; 삛; ) HANGUL SYLLABLE BBILB +C09C;C09C;1108 1175 11B3;C09C;1108 1175 11B3; # (삜; 삜; 삜; 삜; 삜; ) HANGUL SYLLABLE BBILS +C09D;C09D;1108 1175 11B4;C09D;1108 1175 11B4; # (삝; 삝; 삝; 삝; 삝; ) HANGUL SYLLABLE BBILT +C09E;C09E;1108 1175 11B5;C09E;1108 1175 11B5; # (삞; 삞; 삞; 삞; 삞; ) HANGUL SYLLABLE BBILP +C09F;C09F;1108 1175 11B6;C09F;1108 1175 11B6; # (삟; 삟; 삟; 삟; 삟; ) HANGUL SYLLABLE BBILH +C0A0;C0A0;1108 1175 11B7;C0A0;1108 1175 11B7; # (삠; 삠; 삠; 삠; 삠; ) HANGUL SYLLABLE BBIM +C0A1;C0A1;1108 1175 11B8;C0A1;1108 1175 11B8; # (삡; 삡; 삡; 삡; 삡; ) HANGUL SYLLABLE BBIB +C0A2;C0A2;1108 1175 11B9;C0A2;1108 1175 11B9; # (삢; 삢; 삢; 삢; 삢; ) HANGUL SYLLABLE BBIBS +C0A3;C0A3;1108 1175 11BA;C0A3;1108 1175 11BA; # (삣; 삣; 삣; 삣; 삣; ) HANGUL SYLLABLE BBIS +C0A4;C0A4;1108 1175 11BB;C0A4;1108 1175 11BB; # (삤; 삤; 삤; 삤; 삤; ) HANGUL SYLLABLE BBISS +C0A5;C0A5;1108 1175 11BC;C0A5;1108 1175 11BC; # (삥; 삥; 삥; 삥; 삥; ) HANGUL SYLLABLE BBING +C0A6;C0A6;1108 1175 11BD;C0A6;1108 1175 11BD; # (삦; 삦; 삦; 삦; 삦; ) HANGUL SYLLABLE BBIJ +C0A7;C0A7;1108 1175 11BE;C0A7;1108 1175 11BE; # (삧; 삧; 삧; 삧; 삧; ) HANGUL SYLLABLE BBIC +C0A8;C0A8;1108 1175 11BF;C0A8;1108 1175 11BF; # (삨; 삨; 삨; 삨; 삨; ) HANGUL SYLLABLE BBIK +C0A9;C0A9;1108 1175 11C0;C0A9;1108 1175 11C0; # (삩; 삩; 삩; 삩; 삩; ) HANGUL SYLLABLE BBIT +C0AA;C0AA;1108 1175 11C1;C0AA;1108 1175 11C1; # (삪; 삪; 삪; 삪; 삪; ) HANGUL SYLLABLE BBIP +C0AB;C0AB;1108 1175 11C2;C0AB;1108 1175 11C2; # (삫; 삫; 삫; 삫; 삫; ) HANGUL SYLLABLE BBIH +C0AC;C0AC;1109 1161;C0AC;1109 1161; # (사; 사; 사; 사; 사; ) HANGUL SYLLABLE SA +C0AD;C0AD;1109 1161 11A8;C0AD;1109 1161 11A8; # (삭; 삭; 삭; 삭; 삭; ) HANGUL SYLLABLE SAG +C0AE;C0AE;1109 1161 11A9;C0AE;1109 1161 11A9; # (삮; 삮; 삮; 삮; 삮; ) HANGUL SYLLABLE SAGG +C0AF;C0AF;1109 1161 11AA;C0AF;1109 1161 11AA; # (삯; 삯; 삯; 삯; 삯; ) HANGUL SYLLABLE SAGS +C0B0;C0B0;1109 1161 11AB;C0B0;1109 1161 11AB; # (산; 산; 산; 산; 산; ) HANGUL SYLLABLE SAN +C0B1;C0B1;1109 1161 11AC;C0B1;1109 1161 11AC; # (삱; 삱; 삱; 삱; 삱; ) HANGUL SYLLABLE SANJ +C0B2;C0B2;1109 1161 11AD;C0B2;1109 1161 11AD; # (삲; 삲; 삲; 삲; 삲; ) HANGUL SYLLABLE SANH +C0B3;C0B3;1109 1161 11AE;C0B3;1109 1161 11AE; # (삳; 삳; 삳; 삳; 삳; ) HANGUL SYLLABLE SAD +C0B4;C0B4;1109 1161 11AF;C0B4;1109 1161 11AF; # (살; 살; 살; 살; 살; ) HANGUL SYLLABLE SAL +C0B5;C0B5;1109 1161 11B0;C0B5;1109 1161 11B0; # (삵; 삵; 삵; 삵; 삵; ) HANGUL SYLLABLE SALG +C0B6;C0B6;1109 1161 11B1;C0B6;1109 1161 11B1; # (삶; 삶; 삶; 삶; 삶; ) HANGUL SYLLABLE SALM +C0B7;C0B7;1109 1161 11B2;C0B7;1109 1161 11B2; # (삷; 삷; 삷; 삷; 삷; ) HANGUL SYLLABLE SALB +C0B8;C0B8;1109 1161 11B3;C0B8;1109 1161 11B3; # (삸; 삸; 삸; 삸; 삸; ) HANGUL SYLLABLE SALS +C0B9;C0B9;1109 1161 11B4;C0B9;1109 1161 11B4; # (삹; 삹; 삹; 삹; 삹; ) HANGUL SYLLABLE SALT +C0BA;C0BA;1109 1161 11B5;C0BA;1109 1161 11B5; # (삺; 삺; 삺; 삺; 삺; ) HANGUL SYLLABLE SALP +C0BB;C0BB;1109 1161 11B6;C0BB;1109 1161 11B6; # (삻; 삻; 삻; 삻; 삻; ) HANGUL SYLLABLE SALH +C0BC;C0BC;1109 1161 11B7;C0BC;1109 1161 11B7; # (삼; 삼; 삼; 삼; 삼; ) HANGUL SYLLABLE SAM +C0BD;C0BD;1109 1161 11B8;C0BD;1109 1161 11B8; # (삽; 삽; 삽; 삽; 삽; ) HANGUL SYLLABLE SAB +C0BE;C0BE;1109 1161 11B9;C0BE;1109 1161 11B9; # (삾; 삾; 삾; 삾; 삾; ) HANGUL SYLLABLE SABS +C0BF;C0BF;1109 1161 11BA;C0BF;1109 1161 11BA; # (삿; 삿; 삿; 삿; 삿; ) HANGUL SYLLABLE SAS +C0C0;C0C0;1109 1161 11BB;C0C0;1109 1161 11BB; # (샀; 샀; 샀; 샀; 샀; ) HANGUL SYLLABLE SASS +C0C1;C0C1;1109 1161 11BC;C0C1;1109 1161 11BC; # (상; 상; 상; 상; 상; ) HANGUL SYLLABLE SANG +C0C2;C0C2;1109 1161 11BD;C0C2;1109 1161 11BD; # (샂; 샂; 샂; 샂; 샂; ) HANGUL SYLLABLE SAJ +C0C3;C0C3;1109 1161 11BE;C0C3;1109 1161 11BE; # (샃; 샃; 샃; 샃; 샃; ) HANGUL SYLLABLE SAC +C0C4;C0C4;1109 1161 11BF;C0C4;1109 1161 11BF; # (샄; 샄; 샄; 샄; 샄; ) HANGUL SYLLABLE SAK +C0C5;C0C5;1109 1161 11C0;C0C5;1109 1161 11C0; # (샅; 샅; 샅; 샅; 샅; ) HANGUL SYLLABLE SAT +C0C6;C0C6;1109 1161 11C1;C0C6;1109 1161 11C1; # (샆; 샆; 샆; 샆; 샆; ) HANGUL SYLLABLE SAP +C0C7;C0C7;1109 1161 11C2;C0C7;1109 1161 11C2; # (샇; 샇; 샇; 샇; 샇; ) HANGUL SYLLABLE SAH +C0C8;C0C8;1109 1162;C0C8;1109 1162; # (새; 새; 새; 새; 새; ) HANGUL SYLLABLE SAE +C0C9;C0C9;1109 1162 11A8;C0C9;1109 1162 11A8; # (색; 색; 색; 색; 색; ) HANGUL SYLLABLE SAEG +C0CA;C0CA;1109 1162 11A9;C0CA;1109 1162 11A9; # (샊; 샊; 샊; 샊; 샊; ) HANGUL SYLLABLE SAEGG +C0CB;C0CB;1109 1162 11AA;C0CB;1109 1162 11AA; # (샋; 샋; 샋; 샋; 샋; ) HANGUL SYLLABLE SAEGS +C0CC;C0CC;1109 1162 11AB;C0CC;1109 1162 11AB; # (샌; 샌; 샌; 샌; 샌; ) HANGUL SYLLABLE SAEN +C0CD;C0CD;1109 1162 11AC;C0CD;1109 1162 11AC; # (샍; 샍; 샍; 샍; 샍; ) HANGUL SYLLABLE SAENJ +C0CE;C0CE;1109 1162 11AD;C0CE;1109 1162 11AD; # (샎; 샎; 샎; 샎; 샎; ) HANGUL SYLLABLE SAENH +C0CF;C0CF;1109 1162 11AE;C0CF;1109 1162 11AE; # (샏; 샏; 샏; 샏; 샏; ) HANGUL SYLLABLE SAED +C0D0;C0D0;1109 1162 11AF;C0D0;1109 1162 11AF; # (샐; 샐; 샐; 샐; 샐; ) HANGUL SYLLABLE SAEL +C0D1;C0D1;1109 1162 11B0;C0D1;1109 1162 11B0; # (샑; 샑; 샑; 샑; 샑; ) HANGUL SYLLABLE SAELG +C0D2;C0D2;1109 1162 11B1;C0D2;1109 1162 11B1; # (샒; 샒; 샒; 샒; 샒; ) HANGUL SYLLABLE SAELM +C0D3;C0D3;1109 1162 11B2;C0D3;1109 1162 11B2; # (샓; 샓; 샓; 샓; 샓; ) HANGUL SYLLABLE SAELB +C0D4;C0D4;1109 1162 11B3;C0D4;1109 1162 11B3; # (샔; 샔; 샔; 샔; 샔; ) HANGUL SYLLABLE SAELS +C0D5;C0D5;1109 1162 11B4;C0D5;1109 1162 11B4; # (샕; 샕; 샕; 샕; 샕; ) HANGUL SYLLABLE SAELT +C0D6;C0D6;1109 1162 11B5;C0D6;1109 1162 11B5; # (샖; 샖; 샖; 샖; 샖; ) HANGUL SYLLABLE SAELP +C0D7;C0D7;1109 1162 11B6;C0D7;1109 1162 11B6; # (샗; 샗; 샗; 샗; 샗; ) HANGUL SYLLABLE SAELH +C0D8;C0D8;1109 1162 11B7;C0D8;1109 1162 11B7; # (샘; 샘; 샘; 샘; 샘; ) HANGUL SYLLABLE SAEM +C0D9;C0D9;1109 1162 11B8;C0D9;1109 1162 11B8; # (샙; 샙; 샙; 샙; 샙; ) HANGUL SYLLABLE SAEB +C0DA;C0DA;1109 1162 11B9;C0DA;1109 1162 11B9; # (샚; 샚; 샚; 샚; 샚; ) HANGUL SYLLABLE SAEBS +C0DB;C0DB;1109 1162 11BA;C0DB;1109 1162 11BA; # (샛; 샛; 샛; 샛; 샛; ) HANGUL SYLLABLE SAES +C0DC;C0DC;1109 1162 11BB;C0DC;1109 1162 11BB; # (샜; 샜; 샜; 샜; 샜; ) HANGUL SYLLABLE SAESS +C0DD;C0DD;1109 1162 11BC;C0DD;1109 1162 11BC; # (생; 생; 생; 생; 생; ) HANGUL SYLLABLE SAENG +C0DE;C0DE;1109 1162 11BD;C0DE;1109 1162 11BD; # (샞; 샞; 샞; 샞; 샞; ) HANGUL SYLLABLE SAEJ +C0DF;C0DF;1109 1162 11BE;C0DF;1109 1162 11BE; # (샟; 샟; 샟; 샟; 샟; ) HANGUL SYLLABLE SAEC +C0E0;C0E0;1109 1162 11BF;C0E0;1109 1162 11BF; # (샠; 샠; 샠; 샠; 샠; ) HANGUL SYLLABLE SAEK +C0E1;C0E1;1109 1162 11C0;C0E1;1109 1162 11C0; # (샡; 샡; 샡; 샡; 샡; ) HANGUL SYLLABLE SAET +C0E2;C0E2;1109 1162 11C1;C0E2;1109 1162 11C1; # (샢; 샢; 샢; 샢; 샢; ) HANGUL SYLLABLE SAEP +C0E3;C0E3;1109 1162 11C2;C0E3;1109 1162 11C2; # (샣; 샣; 샣; 샣; 샣; ) HANGUL SYLLABLE SAEH +C0E4;C0E4;1109 1163;C0E4;1109 1163; # (샤; 샤; 샤; 샤; 샤; ) HANGUL SYLLABLE SYA +C0E5;C0E5;1109 1163 11A8;C0E5;1109 1163 11A8; # (샥; 샥; 샥; 샥; 샥; ) HANGUL SYLLABLE SYAG +C0E6;C0E6;1109 1163 11A9;C0E6;1109 1163 11A9; # (샦; 샦; 샦; 샦; 샦; ) HANGUL SYLLABLE SYAGG +C0E7;C0E7;1109 1163 11AA;C0E7;1109 1163 11AA; # (샧; 샧; 샧; 샧; 샧; ) HANGUL SYLLABLE SYAGS +C0E8;C0E8;1109 1163 11AB;C0E8;1109 1163 11AB; # (샨; 샨; 샨; 샨; 샨; ) HANGUL SYLLABLE SYAN +C0E9;C0E9;1109 1163 11AC;C0E9;1109 1163 11AC; # (샩; 샩; 샩; 샩; 샩; ) HANGUL SYLLABLE SYANJ +C0EA;C0EA;1109 1163 11AD;C0EA;1109 1163 11AD; # (샪; 샪; 샪; 샪; 샪; ) HANGUL SYLLABLE SYANH +C0EB;C0EB;1109 1163 11AE;C0EB;1109 1163 11AE; # (샫; 샫; 샫; 샫; 샫; ) HANGUL SYLLABLE SYAD +C0EC;C0EC;1109 1163 11AF;C0EC;1109 1163 11AF; # (샬; 샬; 샬; 샬; 샬; ) HANGUL SYLLABLE SYAL +C0ED;C0ED;1109 1163 11B0;C0ED;1109 1163 11B0; # (샭; 샭; 샭; 샭; 샭; ) HANGUL SYLLABLE SYALG +C0EE;C0EE;1109 1163 11B1;C0EE;1109 1163 11B1; # (샮; 샮; 샮; 샮; 샮; ) HANGUL SYLLABLE SYALM +C0EF;C0EF;1109 1163 11B2;C0EF;1109 1163 11B2; # (샯; 샯; 샯; 샯; 샯; ) HANGUL SYLLABLE SYALB +C0F0;C0F0;1109 1163 11B3;C0F0;1109 1163 11B3; # (샰; 샰; 샰; 샰; 샰; ) HANGUL SYLLABLE SYALS +C0F1;C0F1;1109 1163 11B4;C0F1;1109 1163 11B4; # (샱; 샱; 샱; 샱; 샱; ) HANGUL SYLLABLE SYALT +C0F2;C0F2;1109 1163 11B5;C0F2;1109 1163 11B5; # (샲; 샲; 샲; 샲; 샲; ) HANGUL SYLLABLE SYALP +C0F3;C0F3;1109 1163 11B6;C0F3;1109 1163 11B6; # (샳; 샳; 샳; 샳; 샳; ) HANGUL SYLLABLE SYALH +C0F4;C0F4;1109 1163 11B7;C0F4;1109 1163 11B7; # (샴; 샴; 샴; 샴; 샴; ) HANGUL SYLLABLE SYAM +C0F5;C0F5;1109 1163 11B8;C0F5;1109 1163 11B8; # (샵; 샵; 샵; 샵; 샵; ) HANGUL SYLLABLE SYAB +C0F6;C0F6;1109 1163 11B9;C0F6;1109 1163 11B9; # (샶; 샶; 샶; 샶; 샶; ) HANGUL SYLLABLE SYABS +C0F7;C0F7;1109 1163 11BA;C0F7;1109 1163 11BA; # (샷; 샷; 샷; 샷; 샷; ) HANGUL SYLLABLE SYAS +C0F8;C0F8;1109 1163 11BB;C0F8;1109 1163 11BB; # (샸; 샸; 샸; 샸; 샸; ) HANGUL SYLLABLE SYASS +C0F9;C0F9;1109 1163 11BC;C0F9;1109 1163 11BC; # (샹; 샹; 샹; 샹; 샹; ) HANGUL SYLLABLE SYANG +C0FA;C0FA;1109 1163 11BD;C0FA;1109 1163 11BD; # (샺; 샺; 샺; 샺; 샺; ) HANGUL SYLLABLE SYAJ +C0FB;C0FB;1109 1163 11BE;C0FB;1109 1163 11BE; # (샻; 샻; 샻; 샻; 샻; ) HANGUL SYLLABLE SYAC +C0FC;C0FC;1109 1163 11BF;C0FC;1109 1163 11BF; # (샼; 샼; 샼; 샼; 샼; ) HANGUL SYLLABLE SYAK +C0FD;C0FD;1109 1163 11C0;C0FD;1109 1163 11C0; # (샽; 샽; 샽; 샽; 샽; ) HANGUL SYLLABLE SYAT +C0FE;C0FE;1109 1163 11C1;C0FE;1109 1163 11C1; # (샾; 샾; 샾; 샾; 샾; ) HANGUL SYLLABLE SYAP +C0FF;C0FF;1109 1163 11C2;C0FF;1109 1163 11C2; # (샿; 샿; 샿; 샿; 샿; ) HANGUL SYLLABLE SYAH +C100;C100;1109 1164;C100;1109 1164; # (섀; 섀; 섀; 섀; 섀; ) HANGUL SYLLABLE SYAE +C101;C101;1109 1164 11A8;C101;1109 1164 11A8; # (섁; 섁; 섁; 섁; 섁; ) HANGUL SYLLABLE SYAEG +C102;C102;1109 1164 11A9;C102;1109 1164 11A9; # (섂; 섂; 섂; 섂; 섂; ) HANGUL SYLLABLE SYAEGG +C103;C103;1109 1164 11AA;C103;1109 1164 11AA; # (섃; 섃; 섃; 섃; 섃; ) HANGUL SYLLABLE SYAEGS +C104;C104;1109 1164 11AB;C104;1109 1164 11AB; # (섄; 섄; 섄; 섄; 섄; ) HANGUL SYLLABLE SYAEN +C105;C105;1109 1164 11AC;C105;1109 1164 11AC; # (섅; 섅; 섅; 섅; 섅; ) HANGUL SYLLABLE SYAENJ +C106;C106;1109 1164 11AD;C106;1109 1164 11AD; # (섆; 섆; 섆; 섆; 섆; ) HANGUL SYLLABLE SYAENH +C107;C107;1109 1164 11AE;C107;1109 1164 11AE; # (섇; 섇; 섇; 섇; 섇; ) HANGUL SYLLABLE SYAED +C108;C108;1109 1164 11AF;C108;1109 1164 11AF; # (섈; 섈; 섈; 섈; 섈; ) HANGUL SYLLABLE SYAEL +C109;C109;1109 1164 11B0;C109;1109 1164 11B0; # (섉; 섉; 섉; 섉; 섉; ) HANGUL SYLLABLE SYAELG +C10A;C10A;1109 1164 11B1;C10A;1109 1164 11B1; # (섊; 섊; 섊; 섊; 섊; ) HANGUL SYLLABLE SYAELM +C10B;C10B;1109 1164 11B2;C10B;1109 1164 11B2; # (섋; 섋; 섋; 섋; 섋; ) HANGUL SYLLABLE SYAELB +C10C;C10C;1109 1164 11B3;C10C;1109 1164 11B3; # (섌; 섌; 섌; 섌; 섌; ) HANGUL SYLLABLE SYAELS +C10D;C10D;1109 1164 11B4;C10D;1109 1164 11B4; # (섍; 섍; 섍; 섍; 섍; ) HANGUL SYLLABLE SYAELT +C10E;C10E;1109 1164 11B5;C10E;1109 1164 11B5; # (섎; 섎; 섎; 섎; 섎; ) HANGUL SYLLABLE SYAELP +C10F;C10F;1109 1164 11B6;C10F;1109 1164 11B6; # (섏; 섏; 섏; 섏; 섏; ) HANGUL SYLLABLE SYAELH +C110;C110;1109 1164 11B7;C110;1109 1164 11B7; # (섐; 섐; 섐; 섐; 섐; ) HANGUL SYLLABLE SYAEM +C111;C111;1109 1164 11B8;C111;1109 1164 11B8; # (섑; 섑; 섑; 섑; 섑; ) HANGUL SYLLABLE SYAEB +C112;C112;1109 1164 11B9;C112;1109 1164 11B9; # (섒; 섒; 섒; 섒; 섒; ) HANGUL SYLLABLE SYAEBS +C113;C113;1109 1164 11BA;C113;1109 1164 11BA; # (섓; 섓; 섓; 섓; 섓; ) HANGUL SYLLABLE SYAES +C114;C114;1109 1164 11BB;C114;1109 1164 11BB; # (섔; 섔; 섔; 섔; 섔; ) HANGUL SYLLABLE SYAESS +C115;C115;1109 1164 11BC;C115;1109 1164 11BC; # (섕; 섕; 섕; 섕; 섕; ) HANGUL SYLLABLE SYAENG +C116;C116;1109 1164 11BD;C116;1109 1164 11BD; # (섖; 섖; 섖; 섖; 섖; ) HANGUL SYLLABLE SYAEJ +C117;C117;1109 1164 11BE;C117;1109 1164 11BE; # (섗; 섗; 섗; 섗; 섗; ) HANGUL SYLLABLE SYAEC +C118;C118;1109 1164 11BF;C118;1109 1164 11BF; # (섘; 섘; 섘; 섘; 섘; ) HANGUL SYLLABLE SYAEK +C119;C119;1109 1164 11C0;C119;1109 1164 11C0; # (섙; 섙; 섙; 섙; 섙; ) HANGUL SYLLABLE SYAET +C11A;C11A;1109 1164 11C1;C11A;1109 1164 11C1; # (섚; 섚; 섚; 섚; 섚; ) HANGUL SYLLABLE SYAEP +C11B;C11B;1109 1164 11C2;C11B;1109 1164 11C2; # (섛; 섛; 섛; 섛; 섛; ) HANGUL SYLLABLE SYAEH +C11C;C11C;1109 1165;C11C;1109 1165; # (서; 서; 서; 서; 서; ) HANGUL SYLLABLE SEO +C11D;C11D;1109 1165 11A8;C11D;1109 1165 11A8; # (석; 석; 석; 석; 석; ) HANGUL SYLLABLE SEOG +C11E;C11E;1109 1165 11A9;C11E;1109 1165 11A9; # (섞; 섞; 섞; 섞; 섞; ) HANGUL SYLLABLE SEOGG +C11F;C11F;1109 1165 11AA;C11F;1109 1165 11AA; # (섟; 섟; 섟; 섟; 섟; ) HANGUL SYLLABLE SEOGS +C120;C120;1109 1165 11AB;C120;1109 1165 11AB; # (선; 선; 선; 선; 선; ) HANGUL SYLLABLE SEON +C121;C121;1109 1165 11AC;C121;1109 1165 11AC; # (섡; 섡; 섡; 섡; 섡; ) HANGUL SYLLABLE SEONJ +C122;C122;1109 1165 11AD;C122;1109 1165 11AD; # (섢; 섢; 섢; 섢; 섢; ) HANGUL SYLLABLE SEONH +C123;C123;1109 1165 11AE;C123;1109 1165 11AE; # (섣; 섣; 섣; 섣; 섣; ) HANGUL SYLLABLE SEOD +C124;C124;1109 1165 11AF;C124;1109 1165 11AF; # (설; 설; 설; 설; 설; ) HANGUL SYLLABLE SEOL +C125;C125;1109 1165 11B0;C125;1109 1165 11B0; # (섥; 섥; 섥; 섥; 섥; ) HANGUL SYLLABLE SEOLG +C126;C126;1109 1165 11B1;C126;1109 1165 11B1; # (섦; 섦; 섦; 섦; 섦; ) HANGUL SYLLABLE SEOLM +C127;C127;1109 1165 11B2;C127;1109 1165 11B2; # (섧; 섧; 섧; 섧; 섧; ) HANGUL SYLLABLE SEOLB +C128;C128;1109 1165 11B3;C128;1109 1165 11B3; # (섨; 섨; 섨; 섨; 섨; ) HANGUL SYLLABLE SEOLS +C129;C129;1109 1165 11B4;C129;1109 1165 11B4; # (섩; 섩; 섩; 섩; 섩; ) HANGUL SYLLABLE SEOLT +C12A;C12A;1109 1165 11B5;C12A;1109 1165 11B5; # (섪; 섪; 섪; 섪; 섪; ) HANGUL SYLLABLE SEOLP +C12B;C12B;1109 1165 11B6;C12B;1109 1165 11B6; # (섫; 섫; 섫; 섫; 섫; ) HANGUL SYLLABLE SEOLH +C12C;C12C;1109 1165 11B7;C12C;1109 1165 11B7; # (섬; 섬; 섬; 섬; 섬; ) HANGUL SYLLABLE SEOM +C12D;C12D;1109 1165 11B8;C12D;1109 1165 11B8; # (섭; 섭; 섭; 섭; 섭; ) HANGUL SYLLABLE SEOB +C12E;C12E;1109 1165 11B9;C12E;1109 1165 11B9; # (섮; 섮; 섮; 섮; 섮; ) HANGUL SYLLABLE SEOBS +C12F;C12F;1109 1165 11BA;C12F;1109 1165 11BA; # (섯; 섯; 섯; 섯; 섯; ) HANGUL SYLLABLE SEOS +C130;C130;1109 1165 11BB;C130;1109 1165 11BB; # (섰; 섰; 섰; 섰; 섰; ) HANGUL SYLLABLE SEOSS +C131;C131;1109 1165 11BC;C131;1109 1165 11BC; # (성; 성; 성; 성; 성; ) HANGUL SYLLABLE SEONG +C132;C132;1109 1165 11BD;C132;1109 1165 11BD; # (섲; 섲; 섲; 섲; 섲; ) HANGUL SYLLABLE SEOJ +C133;C133;1109 1165 11BE;C133;1109 1165 11BE; # (섳; 섳; 섳; 섳; 섳; ) HANGUL SYLLABLE SEOC +C134;C134;1109 1165 11BF;C134;1109 1165 11BF; # (섴; 섴; 섴; 섴; 섴; ) HANGUL SYLLABLE SEOK +C135;C135;1109 1165 11C0;C135;1109 1165 11C0; # (섵; 섵; 섵; 섵; 섵; ) HANGUL SYLLABLE SEOT +C136;C136;1109 1165 11C1;C136;1109 1165 11C1; # (섶; 섶; 섶; 섶; 섶; ) HANGUL SYLLABLE SEOP +C137;C137;1109 1165 11C2;C137;1109 1165 11C2; # (섷; 섷; 섷; 섷; 섷; ) HANGUL SYLLABLE SEOH +C138;C138;1109 1166;C138;1109 1166; # (세; 세; 세; 세; 세; ) HANGUL SYLLABLE SE +C139;C139;1109 1166 11A8;C139;1109 1166 11A8; # (섹; 섹; 섹; 섹; 섹; ) HANGUL SYLLABLE SEG +C13A;C13A;1109 1166 11A9;C13A;1109 1166 11A9; # (섺; 섺; 섺; 섺; 섺; ) HANGUL SYLLABLE SEGG +C13B;C13B;1109 1166 11AA;C13B;1109 1166 11AA; # (섻; 섻; 섻; 섻; 섻; ) HANGUL SYLLABLE SEGS +C13C;C13C;1109 1166 11AB;C13C;1109 1166 11AB; # (센; 센; 센; 센; 센; ) HANGUL SYLLABLE SEN +C13D;C13D;1109 1166 11AC;C13D;1109 1166 11AC; # (섽; 섽; 섽; 섽; 섽; ) HANGUL SYLLABLE SENJ +C13E;C13E;1109 1166 11AD;C13E;1109 1166 11AD; # (섾; 섾; 섾; 섾; 섾; ) HANGUL SYLLABLE SENH +C13F;C13F;1109 1166 11AE;C13F;1109 1166 11AE; # (섿; 섿; 섿; 섿; 섿; ) HANGUL SYLLABLE SED +C140;C140;1109 1166 11AF;C140;1109 1166 11AF; # (셀; 셀; 셀; 셀; 셀; ) HANGUL SYLLABLE SEL +C141;C141;1109 1166 11B0;C141;1109 1166 11B0; # (셁; 셁; 셁; 셁; 셁; ) HANGUL SYLLABLE SELG +C142;C142;1109 1166 11B1;C142;1109 1166 11B1; # (셂; 셂; 셂; 셂; 셂; ) HANGUL SYLLABLE SELM +C143;C143;1109 1166 11B2;C143;1109 1166 11B2; # (셃; 셃; 셃; 셃; 셃; ) HANGUL SYLLABLE SELB +C144;C144;1109 1166 11B3;C144;1109 1166 11B3; # (셄; 셄; 셄; 셄; 셄; ) HANGUL SYLLABLE SELS +C145;C145;1109 1166 11B4;C145;1109 1166 11B4; # (셅; 셅; 셅; 셅; 셅; ) HANGUL SYLLABLE SELT +C146;C146;1109 1166 11B5;C146;1109 1166 11B5; # (셆; 셆; 셆; 셆; 셆; ) HANGUL SYLLABLE SELP +C147;C147;1109 1166 11B6;C147;1109 1166 11B6; # (셇; 셇; 셇; 셇; 셇; ) HANGUL SYLLABLE SELH +C148;C148;1109 1166 11B7;C148;1109 1166 11B7; # (셈; 셈; 셈; 셈; 셈; ) HANGUL SYLLABLE SEM +C149;C149;1109 1166 11B8;C149;1109 1166 11B8; # (셉; 셉; 셉; 셉; 셉; ) HANGUL SYLLABLE SEB +C14A;C14A;1109 1166 11B9;C14A;1109 1166 11B9; # (셊; 셊; 셊; 셊; 셊; ) HANGUL SYLLABLE SEBS +C14B;C14B;1109 1166 11BA;C14B;1109 1166 11BA; # (셋; 셋; 셋; 셋; 셋; ) HANGUL SYLLABLE SES +C14C;C14C;1109 1166 11BB;C14C;1109 1166 11BB; # (셌; 셌; 셌; 셌; 셌; ) HANGUL SYLLABLE SESS +C14D;C14D;1109 1166 11BC;C14D;1109 1166 11BC; # (셍; 셍; 셍; 셍; 셍; ) HANGUL SYLLABLE SENG +C14E;C14E;1109 1166 11BD;C14E;1109 1166 11BD; # (셎; 셎; 셎; 셎; 셎; ) HANGUL SYLLABLE SEJ +C14F;C14F;1109 1166 11BE;C14F;1109 1166 11BE; # (셏; 셏; 셏; 셏; 셏; ) HANGUL SYLLABLE SEC +C150;C150;1109 1166 11BF;C150;1109 1166 11BF; # (셐; 셐; 셐; 셐; 셐; ) HANGUL SYLLABLE SEK +C151;C151;1109 1166 11C0;C151;1109 1166 11C0; # (셑; 셑; 셑; 셑; 셑; ) HANGUL SYLLABLE SET +C152;C152;1109 1166 11C1;C152;1109 1166 11C1; # (셒; 셒; 셒; 셒; 셒; ) HANGUL SYLLABLE SEP +C153;C153;1109 1166 11C2;C153;1109 1166 11C2; # (셓; 셓; 셓; 셓; 셓; ) HANGUL SYLLABLE SEH +C154;C154;1109 1167;C154;1109 1167; # (셔; 셔; 셔; 셔; 셔; ) HANGUL SYLLABLE SYEO +C155;C155;1109 1167 11A8;C155;1109 1167 11A8; # (셕; 셕; 셕; 셕; 셕; ) HANGUL SYLLABLE SYEOG +C156;C156;1109 1167 11A9;C156;1109 1167 11A9; # (셖; 셖; 셖; 셖; 셖; ) HANGUL SYLLABLE SYEOGG +C157;C157;1109 1167 11AA;C157;1109 1167 11AA; # (셗; 셗; 셗; 셗; 셗; ) HANGUL SYLLABLE SYEOGS +C158;C158;1109 1167 11AB;C158;1109 1167 11AB; # (션; 션; 션; 션; 션; ) HANGUL SYLLABLE SYEON +C159;C159;1109 1167 11AC;C159;1109 1167 11AC; # (셙; 셙; 셙; 셙; 셙; ) HANGUL SYLLABLE SYEONJ +C15A;C15A;1109 1167 11AD;C15A;1109 1167 11AD; # (셚; 셚; 셚; 셚; 셚; ) HANGUL SYLLABLE SYEONH +C15B;C15B;1109 1167 11AE;C15B;1109 1167 11AE; # (셛; 셛; 셛; 셛; 셛; ) HANGUL SYLLABLE SYEOD +C15C;C15C;1109 1167 11AF;C15C;1109 1167 11AF; # (셜; 셜; 셜; 셜; 셜; ) HANGUL SYLLABLE SYEOL +C15D;C15D;1109 1167 11B0;C15D;1109 1167 11B0; # (셝; 셝; 셝; 셝; 셝; ) HANGUL SYLLABLE SYEOLG +C15E;C15E;1109 1167 11B1;C15E;1109 1167 11B1; # (셞; 셞; 셞; 셞; 셞; ) HANGUL SYLLABLE SYEOLM +C15F;C15F;1109 1167 11B2;C15F;1109 1167 11B2; # (셟; 셟; 셟; 셟; 셟; ) HANGUL SYLLABLE SYEOLB +C160;C160;1109 1167 11B3;C160;1109 1167 11B3; # (셠; 셠; 셠; 셠; 셠; ) HANGUL SYLLABLE SYEOLS +C161;C161;1109 1167 11B4;C161;1109 1167 11B4; # (셡; 셡; 셡; 셡; 셡; ) HANGUL SYLLABLE SYEOLT +C162;C162;1109 1167 11B5;C162;1109 1167 11B5; # (셢; 셢; 셢; 셢; 셢; ) HANGUL SYLLABLE SYEOLP +C163;C163;1109 1167 11B6;C163;1109 1167 11B6; # (셣; 셣; 셣; 셣; 셣; ) HANGUL SYLLABLE SYEOLH +C164;C164;1109 1167 11B7;C164;1109 1167 11B7; # (셤; 셤; 셤; 셤; 셤; ) HANGUL SYLLABLE SYEOM +C165;C165;1109 1167 11B8;C165;1109 1167 11B8; # (셥; 셥; 셥; 셥; 셥; ) HANGUL SYLLABLE SYEOB +C166;C166;1109 1167 11B9;C166;1109 1167 11B9; # (셦; 셦; 셦; 셦; 셦; ) HANGUL SYLLABLE SYEOBS +C167;C167;1109 1167 11BA;C167;1109 1167 11BA; # (셧; 셧; 셧; 셧; 셧; ) HANGUL SYLLABLE SYEOS +C168;C168;1109 1167 11BB;C168;1109 1167 11BB; # (셨; 셨; 셨; 셨; 셨; ) HANGUL SYLLABLE SYEOSS +C169;C169;1109 1167 11BC;C169;1109 1167 11BC; # (셩; 셩; 셩; 셩; 셩; ) HANGUL SYLLABLE SYEONG +C16A;C16A;1109 1167 11BD;C16A;1109 1167 11BD; # (셪; 셪; 셪; 셪; 셪; ) HANGUL SYLLABLE SYEOJ +C16B;C16B;1109 1167 11BE;C16B;1109 1167 11BE; # (셫; 셫; 셫; 셫; 셫; ) HANGUL SYLLABLE SYEOC +C16C;C16C;1109 1167 11BF;C16C;1109 1167 11BF; # (셬; 셬; 셬; 셬; 셬; ) HANGUL SYLLABLE SYEOK +C16D;C16D;1109 1167 11C0;C16D;1109 1167 11C0; # (셭; 셭; 셭; 셭; 셭; ) HANGUL SYLLABLE SYEOT +C16E;C16E;1109 1167 11C1;C16E;1109 1167 11C1; # (셮; 셮; 셮; 셮; 셮; ) HANGUL SYLLABLE SYEOP +C16F;C16F;1109 1167 11C2;C16F;1109 1167 11C2; # (셯; 셯; 셯; 셯; 셯; ) HANGUL SYLLABLE SYEOH +C170;C170;1109 1168;C170;1109 1168; # (셰; 셰; 셰; 셰; 셰; ) HANGUL SYLLABLE SYE +C171;C171;1109 1168 11A8;C171;1109 1168 11A8; # (셱; 셱; 셱; 셱; 셱; ) HANGUL SYLLABLE SYEG +C172;C172;1109 1168 11A9;C172;1109 1168 11A9; # (셲; 셲; 셲; 셲; 셲; ) HANGUL SYLLABLE SYEGG +C173;C173;1109 1168 11AA;C173;1109 1168 11AA; # (셳; 셳; 셳; 셳; 셳; ) HANGUL SYLLABLE SYEGS +C174;C174;1109 1168 11AB;C174;1109 1168 11AB; # (셴; 셴; 셴; 셴; 셴; ) HANGUL SYLLABLE SYEN +C175;C175;1109 1168 11AC;C175;1109 1168 11AC; # (셵; 셵; 셵; 셵; 셵; ) HANGUL SYLLABLE SYENJ +C176;C176;1109 1168 11AD;C176;1109 1168 11AD; # (셶; 셶; 셶; 셶; 셶; ) HANGUL SYLLABLE SYENH +C177;C177;1109 1168 11AE;C177;1109 1168 11AE; # (셷; 셷; 셷; 셷; 셷; ) HANGUL SYLLABLE SYED +C178;C178;1109 1168 11AF;C178;1109 1168 11AF; # (셸; 셸; 셸; 셸; 셸; ) HANGUL SYLLABLE SYEL +C179;C179;1109 1168 11B0;C179;1109 1168 11B0; # (셹; 셹; 셹; 셹; 셹; ) HANGUL SYLLABLE SYELG +C17A;C17A;1109 1168 11B1;C17A;1109 1168 11B1; # (셺; 셺; 셺; 셺; 셺; ) HANGUL SYLLABLE SYELM +C17B;C17B;1109 1168 11B2;C17B;1109 1168 11B2; # (셻; 셻; 셻; 셻; 셻; ) HANGUL SYLLABLE SYELB +C17C;C17C;1109 1168 11B3;C17C;1109 1168 11B3; # (셼; 셼; 셼; 셼; 셼; ) HANGUL SYLLABLE SYELS +C17D;C17D;1109 1168 11B4;C17D;1109 1168 11B4; # (셽; 셽; 셽; 셽; 셽; ) HANGUL SYLLABLE SYELT +C17E;C17E;1109 1168 11B5;C17E;1109 1168 11B5; # (셾; 셾; 셾; 셾; 셾; ) HANGUL SYLLABLE SYELP +C17F;C17F;1109 1168 11B6;C17F;1109 1168 11B6; # (셿; 셿; 셿; 셿; 셿; ) HANGUL SYLLABLE SYELH +C180;C180;1109 1168 11B7;C180;1109 1168 11B7; # (솀; 솀; 솀; 솀; 솀; ) HANGUL SYLLABLE SYEM +C181;C181;1109 1168 11B8;C181;1109 1168 11B8; # (솁; 솁; 솁; 솁; 솁; ) HANGUL SYLLABLE SYEB +C182;C182;1109 1168 11B9;C182;1109 1168 11B9; # (솂; 솂; 솂; 솂; 솂; ) HANGUL SYLLABLE SYEBS +C183;C183;1109 1168 11BA;C183;1109 1168 11BA; # (솃; 솃; 솃; 솃; 솃; ) HANGUL SYLLABLE SYES +C184;C184;1109 1168 11BB;C184;1109 1168 11BB; # (솄; 솄; 솄; 솄; 솄; ) HANGUL SYLLABLE SYESS +C185;C185;1109 1168 11BC;C185;1109 1168 11BC; # (솅; 솅; 솅; 솅; 솅; ) HANGUL SYLLABLE SYENG +C186;C186;1109 1168 11BD;C186;1109 1168 11BD; # (솆; 솆; 솆; 솆; 솆; ) HANGUL SYLLABLE SYEJ +C187;C187;1109 1168 11BE;C187;1109 1168 11BE; # (솇; 솇; 솇; 솇; 솇; ) HANGUL SYLLABLE SYEC +C188;C188;1109 1168 11BF;C188;1109 1168 11BF; # (솈; 솈; 솈; 솈; 솈; ) HANGUL SYLLABLE SYEK +C189;C189;1109 1168 11C0;C189;1109 1168 11C0; # (솉; 솉; 솉; 솉; 솉; ) HANGUL SYLLABLE SYET +C18A;C18A;1109 1168 11C1;C18A;1109 1168 11C1; # (솊; 솊; 솊; 솊; 솊; ) HANGUL SYLLABLE SYEP +C18B;C18B;1109 1168 11C2;C18B;1109 1168 11C2; # (솋; 솋; 솋; 솋; 솋; ) HANGUL SYLLABLE SYEH +C18C;C18C;1109 1169;C18C;1109 1169; # (소; 소; 소; 소; 소; ) HANGUL SYLLABLE SO +C18D;C18D;1109 1169 11A8;C18D;1109 1169 11A8; # (속; 속; 속; 속; 속; ) HANGUL SYLLABLE SOG +C18E;C18E;1109 1169 11A9;C18E;1109 1169 11A9; # (솎; 솎; 솎; 솎; 솎; ) HANGUL SYLLABLE SOGG +C18F;C18F;1109 1169 11AA;C18F;1109 1169 11AA; # (솏; 솏; 솏; 솏; 솏; ) HANGUL SYLLABLE SOGS +C190;C190;1109 1169 11AB;C190;1109 1169 11AB; # (손; 손; 손; 손; 손; ) HANGUL SYLLABLE SON +C191;C191;1109 1169 11AC;C191;1109 1169 11AC; # (솑; 솑; 솑; 솑; 솑; ) HANGUL SYLLABLE SONJ +C192;C192;1109 1169 11AD;C192;1109 1169 11AD; # (솒; 솒; 솒; 솒; 솒; ) HANGUL SYLLABLE SONH +C193;C193;1109 1169 11AE;C193;1109 1169 11AE; # (솓; 솓; 솓; 솓; 솓; ) HANGUL SYLLABLE SOD +C194;C194;1109 1169 11AF;C194;1109 1169 11AF; # (솔; 솔; 솔; 솔; 솔; ) HANGUL SYLLABLE SOL +C195;C195;1109 1169 11B0;C195;1109 1169 11B0; # (솕; 솕; 솕; 솕; 솕; ) HANGUL SYLLABLE SOLG +C196;C196;1109 1169 11B1;C196;1109 1169 11B1; # (솖; 솖; 솖; 솖; 솖; ) HANGUL SYLLABLE SOLM +C197;C197;1109 1169 11B2;C197;1109 1169 11B2; # (솗; 솗; 솗; 솗; 솗; ) HANGUL SYLLABLE SOLB +C198;C198;1109 1169 11B3;C198;1109 1169 11B3; # (솘; 솘; 솘; 솘; 솘; ) HANGUL SYLLABLE SOLS +C199;C199;1109 1169 11B4;C199;1109 1169 11B4; # (솙; 솙; 솙; 솙; 솙; ) HANGUL SYLLABLE SOLT +C19A;C19A;1109 1169 11B5;C19A;1109 1169 11B5; # (솚; 솚; 솚; 솚; 솚; ) HANGUL SYLLABLE SOLP +C19B;C19B;1109 1169 11B6;C19B;1109 1169 11B6; # (솛; 솛; 솛; 솛; 솛; ) HANGUL SYLLABLE SOLH +C19C;C19C;1109 1169 11B7;C19C;1109 1169 11B7; # (솜; 솜; 솜; 솜; 솜; ) HANGUL SYLLABLE SOM +C19D;C19D;1109 1169 11B8;C19D;1109 1169 11B8; # (솝; 솝; 솝; 솝; 솝; ) HANGUL SYLLABLE SOB +C19E;C19E;1109 1169 11B9;C19E;1109 1169 11B9; # (솞; 솞; 솞; 솞; 솞; ) HANGUL SYLLABLE SOBS +C19F;C19F;1109 1169 11BA;C19F;1109 1169 11BA; # (솟; 솟; 솟; 솟; 솟; ) HANGUL SYLLABLE SOS +C1A0;C1A0;1109 1169 11BB;C1A0;1109 1169 11BB; # (솠; 솠; 솠; 솠; 솠; ) HANGUL SYLLABLE SOSS +C1A1;C1A1;1109 1169 11BC;C1A1;1109 1169 11BC; # (송; 송; 송; 송; 송; ) HANGUL SYLLABLE SONG +C1A2;C1A2;1109 1169 11BD;C1A2;1109 1169 11BD; # (솢; 솢; 솢; 솢; 솢; ) HANGUL SYLLABLE SOJ +C1A3;C1A3;1109 1169 11BE;C1A3;1109 1169 11BE; # (솣; 솣; 솣; 솣; 솣; ) HANGUL SYLLABLE SOC +C1A4;C1A4;1109 1169 11BF;C1A4;1109 1169 11BF; # (솤; 솤; 솤; 솤; 솤; ) HANGUL SYLLABLE SOK +C1A5;C1A5;1109 1169 11C0;C1A5;1109 1169 11C0; # (솥; 솥; 솥; 솥; 솥; ) HANGUL SYLLABLE SOT +C1A6;C1A6;1109 1169 11C1;C1A6;1109 1169 11C1; # (솦; 솦; 솦; 솦; 솦; ) HANGUL SYLLABLE SOP +C1A7;C1A7;1109 1169 11C2;C1A7;1109 1169 11C2; # (솧; 솧; 솧; 솧; 솧; ) HANGUL SYLLABLE SOH +C1A8;C1A8;1109 116A;C1A8;1109 116A; # (솨; 솨; 솨; 솨; 솨; ) HANGUL SYLLABLE SWA +C1A9;C1A9;1109 116A 11A8;C1A9;1109 116A 11A8; # (솩; 솩; 솩; 솩; 솩; ) HANGUL SYLLABLE SWAG +C1AA;C1AA;1109 116A 11A9;C1AA;1109 116A 11A9; # (솪; 솪; 솪; 솪; 솪; ) HANGUL SYLLABLE SWAGG +C1AB;C1AB;1109 116A 11AA;C1AB;1109 116A 11AA; # (솫; 솫; 솫; 솫; 솫; ) HANGUL SYLLABLE SWAGS +C1AC;C1AC;1109 116A 11AB;C1AC;1109 116A 11AB; # (솬; 솬; 솬; 솬; 솬; ) HANGUL SYLLABLE SWAN +C1AD;C1AD;1109 116A 11AC;C1AD;1109 116A 11AC; # (솭; 솭; 솭; 솭; 솭; ) HANGUL SYLLABLE SWANJ +C1AE;C1AE;1109 116A 11AD;C1AE;1109 116A 11AD; # (솮; 솮; 솮; 솮; 솮; ) HANGUL SYLLABLE SWANH +C1AF;C1AF;1109 116A 11AE;C1AF;1109 116A 11AE; # (솯; 솯; 솯; 솯; 솯; ) HANGUL SYLLABLE SWAD +C1B0;C1B0;1109 116A 11AF;C1B0;1109 116A 11AF; # (솰; 솰; 솰; 솰; 솰; ) HANGUL SYLLABLE SWAL +C1B1;C1B1;1109 116A 11B0;C1B1;1109 116A 11B0; # (솱; 솱; 솱; 솱; 솱; ) HANGUL SYLLABLE SWALG +C1B2;C1B2;1109 116A 11B1;C1B2;1109 116A 11B1; # (솲; 솲; 솲; 솲; 솲; ) HANGUL SYLLABLE SWALM +C1B3;C1B3;1109 116A 11B2;C1B3;1109 116A 11B2; # (솳; 솳; 솳; 솳; 솳; ) HANGUL SYLLABLE SWALB +C1B4;C1B4;1109 116A 11B3;C1B4;1109 116A 11B3; # (솴; 솴; 솴; 솴; 솴; ) HANGUL SYLLABLE SWALS +C1B5;C1B5;1109 116A 11B4;C1B5;1109 116A 11B4; # (솵; 솵; 솵; 솵; 솵; ) HANGUL SYLLABLE SWALT +C1B6;C1B6;1109 116A 11B5;C1B6;1109 116A 11B5; # (솶; 솶; 솶; 솶; 솶; ) HANGUL SYLLABLE SWALP +C1B7;C1B7;1109 116A 11B6;C1B7;1109 116A 11B6; # (솷; 솷; 솷; 솷; 솷; ) HANGUL SYLLABLE SWALH +C1B8;C1B8;1109 116A 11B7;C1B8;1109 116A 11B7; # (솸; 솸; 솸; 솸; 솸; ) HANGUL SYLLABLE SWAM +C1B9;C1B9;1109 116A 11B8;C1B9;1109 116A 11B8; # (솹; 솹; 솹; 솹; 솹; ) HANGUL SYLLABLE SWAB +C1BA;C1BA;1109 116A 11B9;C1BA;1109 116A 11B9; # (솺; 솺; 솺; 솺; 솺; ) HANGUL SYLLABLE SWABS +C1BB;C1BB;1109 116A 11BA;C1BB;1109 116A 11BA; # (솻; 솻; 솻; 솻; 솻; ) HANGUL SYLLABLE SWAS +C1BC;C1BC;1109 116A 11BB;C1BC;1109 116A 11BB; # (솼; 솼; 솼; 솼; 솼; ) HANGUL SYLLABLE SWASS +C1BD;C1BD;1109 116A 11BC;C1BD;1109 116A 11BC; # (솽; 솽; 솽; 솽; 솽; ) HANGUL SYLLABLE SWANG +C1BE;C1BE;1109 116A 11BD;C1BE;1109 116A 11BD; # (솾; 솾; 솾; 솾; 솾; ) HANGUL SYLLABLE SWAJ +C1BF;C1BF;1109 116A 11BE;C1BF;1109 116A 11BE; # (솿; 솿; 솿; 솿; 솿; ) HANGUL SYLLABLE SWAC +C1C0;C1C0;1109 116A 11BF;C1C0;1109 116A 11BF; # (쇀; 쇀; 쇀; 쇀; 쇀; ) HANGUL SYLLABLE SWAK +C1C1;C1C1;1109 116A 11C0;C1C1;1109 116A 11C0; # (쇁; 쇁; 쇁; 쇁; 쇁; ) HANGUL SYLLABLE SWAT +C1C2;C1C2;1109 116A 11C1;C1C2;1109 116A 11C1; # (쇂; 쇂; 쇂; 쇂; 쇂; ) HANGUL SYLLABLE SWAP +C1C3;C1C3;1109 116A 11C2;C1C3;1109 116A 11C2; # (쇃; 쇃; 쇃; 쇃; 쇃; ) HANGUL SYLLABLE SWAH +C1C4;C1C4;1109 116B;C1C4;1109 116B; # (쇄; 쇄; 쇄; 쇄; 쇄; ) HANGUL SYLLABLE SWAE +C1C5;C1C5;1109 116B 11A8;C1C5;1109 116B 11A8; # (쇅; 쇅; 쇅; 쇅; 쇅; ) HANGUL SYLLABLE SWAEG +C1C6;C1C6;1109 116B 11A9;C1C6;1109 116B 11A9; # (쇆; 쇆; 쇆; 쇆; 쇆; ) HANGUL SYLLABLE SWAEGG +C1C7;C1C7;1109 116B 11AA;C1C7;1109 116B 11AA; # (쇇; 쇇; 쇇; 쇇; 쇇; ) HANGUL SYLLABLE SWAEGS +C1C8;C1C8;1109 116B 11AB;C1C8;1109 116B 11AB; # (쇈; 쇈; 쇈; 쇈; 쇈; ) HANGUL SYLLABLE SWAEN +C1C9;C1C9;1109 116B 11AC;C1C9;1109 116B 11AC; # (쇉; 쇉; 쇉; 쇉; 쇉; ) HANGUL SYLLABLE SWAENJ +C1CA;C1CA;1109 116B 11AD;C1CA;1109 116B 11AD; # (쇊; 쇊; 쇊; 쇊; 쇊; ) HANGUL SYLLABLE SWAENH +C1CB;C1CB;1109 116B 11AE;C1CB;1109 116B 11AE; # (쇋; 쇋; 쇋; 쇋; 쇋; ) HANGUL SYLLABLE SWAED +C1CC;C1CC;1109 116B 11AF;C1CC;1109 116B 11AF; # (쇌; 쇌; 쇌; 쇌; 쇌; ) HANGUL SYLLABLE SWAEL +C1CD;C1CD;1109 116B 11B0;C1CD;1109 116B 11B0; # (쇍; 쇍; 쇍; 쇍; 쇍; ) HANGUL SYLLABLE SWAELG +C1CE;C1CE;1109 116B 11B1;C1CE;1109 116B 11B1; # (쇎; 쇎; 쇎; 쇎; 쇎; ) HANGUL SYLLABLE SWAELM +C1CF;C1CF;1109 116B 11B2;C1CF;1109 116B 11B2; # (쇏; 쇏; 쇏; 쇏; 쇏; ) HANGUL SYLLABLE SWAELB +C1D0;C1D0;1109 116B 11B3;C1D0;1109 116B 11B3; # (쇐; 쇐; 쇐; 쇐; 쇐; ) HANGUL SYLLABLE SWAELS +C1D1;C1D1;1109 116B 11B4;C1D1;1109 116B 11B4; # (쇑; 쇑; 쇑; 쇑; 쇑; ) HANGUL SYLLABLE SWAELT +C1D2;C1D2;1109 116B 11B5;C1D2;1109 116B 11B5; # (쇒; 쇒; 쇒; 쇒; 쇒; ) HANGUL SYLLABLE SWAELP +C1D3;C1D3;1109 116B 11B6;C1D3;1109 116B 11B6; # (쇓; 쇓; 쇓; 쇓; 쇓; ) HANGUL SYLLABLE SWAELH +C1D4;C1D4;1109 116B 11B7;C1D4;1109 116B 11B7; # (쇔; 쇔; 쇔; 쇔; 쇔; ) HANGUL SYLLABLE SWAEM +C1D5;C1D5;1109 116B 11B8;C1D5;1109 116B 11B8; # (쇕; 쇕; 쇕; 쇕; 쇕; ) HANGUL SYLLABLE SWAEB +C1D6;C1D6;1109 116B 11B9;C1D6;1109 116B 11B9; # (쇖; 쇖; 쇖; 쇖; 쇖; ) HANGUL SYLLABLE SWAEBS +C1D7;C1D7;1109 116B 11BA;C1D7;1109 116B 11BA; # (쇗; 쇗; 쇗; 쇗; 쇗; ) HANGUL SYLLABLE SWAES +C1D8;C1D8;1109 116B 11BB;C1D8;1109 116B 11BB; # (쇘; 쇘; 쇘; 쇘; 쇘; ) HANGUL SYLLABLE SWAESS +C1D9;C1D9;1109 116B 11BC;C1D9;1109 116B 11BC; # (쇙; 쇙; 쇙; 쇙; 쇙; ) HANGUL SYLLABLE SWAENG +C1DA;C1DA;1109 116B 11BD;C1DA;1109 116B 11BD; # (쇚; 쇚; 쇚; 쇚; 쇚; ) HANGUL SYLLABLE SWAEJ +C1DB;C1DB;1109 116B 11BE;C1DB;1109 116B 11BE; # (쇛; 쇛; 쇛; 쇛; 쇛; ) HANGUL SYLLABLE SWAEC +C1DC;C1DC;1109 116B 11BF;C1DC;1109 116B 11BF; # (쇜; 쇜; 쇜; 쇜; 쇜; ) HANGUL SYLLABLE SWAEK +C1DD;C1DD;1109 116B 11C0;C1DD;1109 116B 11C0; # (쇝; 쇝; 쇝; 쇝; 쇝; ) HANGUL SYLLABLE SWAET +C1DE;C1DE;1109 116B 11C1;C1DE;1109 116B 11C1; # (쇞; 쇞; 쇞; 쇞; 쇞; ) HANGUL SYLLABLE SWAEP +C1DF;C1DF;1109 116B 11C2;C1DF;1109 116B 11C2; # (쇟; 쇟; 쇟; 쇟; 쇟; ) HANGUL SYLLABLE SWAEH +C1E0;C1E0;1109 116C;C1E0;1109 116C; # (쇠; 쇠; 쇠; 쇠; 쇠; ) HANGUL SYLLABLE SOE +C1E1;C1E1;1109 116C 11A8;C1E1;1109 116C 11A8; # (쇡; 쇡; 쇡; 쇡; 쇡; ) HANGUL SYLLABLE SOEG +C1E2;C1E2;1109 116C 11A9;C1E2;1109 116C 11A9; # (쇢; 쇢; 쇢; 쇢; 쇢; ) HANGUL SYLLABLE SOEGG +C1E3;C1E3;1109 116C 11AA;C1E3;1109 116C 11AA; # (쇣; 쇣; 쇣; 쇣; 쇣; ) HANGUL SYLLABLE SOEGS +C1E4;C1E4;1109 116C 11AB;C1E4;1109 116C 11AB; # (쇤; 쇤; 쇤; 쇤; 쇤; ) HANGUL SYLLABLE SOEN +C1E5;C1E5;1109 116C 11AC;C1E5;1109 116C 11AC; # (쇥; 쇥; 쇥; 쇥; 쇥; ) HANGUL SYLLABLE SOENJ +C1E6;C1E6;1109 116C 11AD;C1E6;1109 116C 11AD; # (쇦; 쇦; 쇦; 쇦; 쇦; ) HANGUL SYLLABLE SOENH +C1E7;C1E7;1109 116C 11AE;C1E7;1109 116C 11AE; # (쇧; 쇧; 쇧; 쇧; 쇧; ) HANGUL SYLLABLE SOED +C1E8;C1E8;1109 116C 11AF;C1E8;1109 116C 11AF; # (쇨; 쇨; 쇨; 쇨; 쇨; ) HANGUL SYLLABLE SOEL +C1E9;C1E9;1109 116C 11B0;C1E9;1109 116C 11B0; # (쇩; 쇩; 쇩; 쇩; 쇩; ) HANGUL SYLLABLE SOELG +C1EA;C1EA;1109 116C 11B1;C1EA;1109 116C 11B1; # (쇪; 쇪; 쇪; 쇪; 쇪; ) HANGUL SYLLABLE SOELM +C1EB;C1EB;1109 116C 11B2;C1EB;1109 116C 11B2; # (쇫; 쇫; 쇫; 쇫; 쇫; ) HANGUL SYLLABLE SOELB +C1EC;C1EC;1109 116C 11B3;C1EC;1109 116C 11B3; # (쇬; 쇬; 쇬; 쇬; 쇬; ) HANGUL SYLLABLE SOELS +C1ED;C1ED;1109 116C 11B4;C1ED;1109 116C 11B4; # (쇭; 쇭; 쇭; 쇭; 쇭; ) HANGUL SYLLABLE SOELT +C1EE;C1EE;1109 116C 11B5;C1EE;1109 116C 11B5; # (쇮; 쇮; 쇮; 쇮; 쇮; ) HANGUL SYLLABLE SOELP +C1EF;C1EF;1109 116C 11B6;C1EF;1109 116C 11B6; # (쇯; 쇯; 쇯; 쇯; 쇯; ) HANGUL SYLLABLE SOELH +C1F0;C1F0;1109 116C 11B7;C1F0;1109 116C 11B7; # (쇰; 쇰; 쇰; 쇰; 쇰; ) HANGUL SYLLABLE SOEM +C1F1;C1F1;1109 116C 11B8;C1F1;1109 116C 11B8; # (쇱; 쇱; 쇱; 쇱; 쇱; ) HANGUL SYLLABLE SOEB +C1F2;C1F2;1109 116C 11B9;C1F2;1109 116C 11B9; # (쇲; 쇲; 쇲; 쇲; 쇲; ) HANGUL SYLLABLE SOEBS +C1F3;C1F3;1109 116C 11BA;C1F3;1109 116C 11BA; # (쇳; 쇳; 쇳; 쇳; 쇳; ) HANGUL SYLLABLE SOES +C1F4;C1F4;1109 116C 11BB;C1F4;1109 116C 11BB; # (쇴; 쇴; 쇴; 쇴; 쇴; ) HANGUL SYLLABLE SOESS +C1F5;C1F5;1109 116C 11BC;C1F5;1109 116C 11BC; # (쇵; 쇵; 쇵; 쇵; 쇵; ) HANGUL SYLLABLE SOENG +C1F6;C1F6;1109 116C 11BD;C1F6;1109 116C 11BD; # (쇶; 쇶; 쇶; 쇶; 쇶; ) HANGUL SYLLABLE SOEJ +C1F7;C1F7;1109 116C 11BE;C1F7;1109 116C 11BE; # (쇷; 쇷; 쇷; 쇷; 쇷; ) HANGUL SYLLABLE SOEC +C1F8;C1F8;1109 116C 11BF;C1F8;1109 116C 11BF; # (쇸; 쇸; 쇸; 쇸; 쇸; ) HANGUL SYLLABLE SOEK +C1F9;C1F9;1109 116C 11C0;C1F9;1109 116C 11C0; # (쇹; 쇹; 쇹; 쇹; 쇹; ) HANGUL SYLLABLE SOET +C1FA;C1FA;1109 116C 11C1;C1FA;1109 116C 11C1; # (쇺; 쇺; 쇺; 쇺; 쇺; ) HANGUL SYLLABLE SOEP +C1FB;C1FB;1109 116C 11C2;C1FB;1109 116C 11C2; # (쇻; 쇻; 쇻; 쇻; 쇻; ) HANGUL SYLLABLE SOEH +C1FC;C1FC;1109 116D;C1FC;1109 116D; # (쇼; 쇼; 쇼; 쇼; 쇼; ) HANGUL SYLLABLE SYO +C1FD;C1FD;1109 116D 11A8;C1FD;1109 116D 11A8; # (쇽; 쇽; 쇽; 쇽; 쇽; ) HANGUL SYLLABLE SYOG +C1FE;C1FE;1109 116D 11A9;C1FE;1109 116D 11A9; # (쇾; 쇾; 쇾; 쇾; 쇾; ) HANGUL SYLLABLE SYOGG +C1FF;C1FF;1109 116D 11AA;C1FF;1109 116D 11AA; # (쇿; 쇿; 쇿; 쇿; 쇿; ) HANGUL SYLLABLE SYOGS +C200;C200;1109 116D 11AB;C200;1109 116D 11AB; # (숀; 숀; 숀; 숀; 숀; ) HANGUL SYLLABLE SYON +C201;C201;1109 116D 11AC;C201;1109 116D 11AC; # (숁; 숁; 숁; 숁; 숁; ) HANGUL SYLLABLE SYONJ +C202;C202;1109 116D 11AD;C202;1109 116D 11AD; # (숂; 숂; 숂; 숂; 숂; ) HANGUL SYLLABLE SYONH +C203;C203;1109 116D 11AE;C203;1109 116D 11AE; # (숃; 숃; 숃; 숃; 숃; ) HANGUL SYLLABLE SYOD +C204;C204;1109 116D 11AF;C204;1109 116D 11AF; # (숄; 숄; 숄; 숄; 숄; ) HANGUL SYLLABLE SYOL +C205;C205;1109 116D 11B0;C205;1109 116D 11B0; # (숅; 숅; 숅; 숅; 숅; ) HANGUL SYLLABLE SYOLG +C206;C206;1109 116D 11B1;C206;1109 116D 11B1; # (숆; 숆; 숆; 숆; 숆; ) HANGUL SYLLABLE SYOLM +C207;C207;1109 116D 11B2;C207;1109 116D 11B2; # (숇; 숇; 숇; 숇; 숇; ) HANGUL SYLLABLE SYOLB +C208;C208;1109 116D 11B3;C208;1109 116D 11B3; # (숈; 숈; 숈; 숈; 숈; ) HANGUL SYLLABLE SYOLS +C209;C209;1109 116D 11B4;C209;1109 116D 11B4; # (숉; 숉; 숉; 숉; 숉; ) HANGUL SYLLABLE SYOLT +C20A;C20A;1109 116D 11B5;C20A;1109 116D 11B5; # (숊; 숊; 숊; 숊; 숊; ) HANGUL SYLLABLE SYOLP +C20B;C20B;1109 116D 11B6;C20B;1109 116D 11B6; # (숋; 숋; 숋; 숋; 숋; ) HANGUL SYLLABLE SYOLH +C20C;C20C;1109 116D 11B7;C20C;1109 116D 11B7; # (숌; 숌; 숌; 숌; 숌; ) HANGUL SYLLABLE SYOM +C20D;C20D;1109 116D 11B8;C20D;1109 116D 11B8; # (숍; 숍; 숍; 숍; 숍; ) HANGUL SYLLABLE SYOB +C20E;C20E;1109 116D 11B9;C20E;1109 116D 11B9; # (숎; 숎; 숎; 숎; 숎; ) HANGUL SYLLABLE SYOBS +C20F;C20F;1109 116D 11BA;C20F;1109 116D 11BA; # (숏; 숏; 숏; 숏; 숏; ) HANGUL SYLLABLE SYOS +C210;C210;1109 116D 11BB;C210;1109 116D 11BB; # (숐; 숐; 숐; 숐; 숐; ) HANGUL SYLLABLE SYOSS +C211;C211;1109 116D 11BC;C211;1109 116D 11BC; # (숑; 숑; 숑; 숑; 숑; ) HANGUL SYLLABLE SYONG +C212;C212;1109 116D 11BD;C212;1109 116D 11BD; # (숒; 숒; 숒; 숒; 숒; ) HANGUL SYLLABLE SYOJ +C213;C213;1109 116D 11BE;C213;1109 116D 11BE; # (숓; 숓; 숓; 숓; 숓; ) HANGUL SYLLABLE SYOC +C214;C214;1109 116D 11BF;C214;1109 116D 11BF; # (숔; 숔; 숔; 숔; 숔; ) HANGUL SYLLABLE SYOK +C215;C215;1109 116D 11C0;C215;1109 116D 11C0; # (숕; 숕; 숕; 숕; 숕; ) HANGUL SYLLABLE SYOT +C216;C216;1109 116D 11C1;C216;1109 116D 11C1; # (숖; 숖; 숖; 숖; 숖; ) HANGUL SYLLABLE SYOP +C217;C217;1109 116D 11C2;C217;1109 116D 11C2; # (숗; 숗; 숗; 숗; 숗; ) HANGUL SYLLABLE SYOH +C218;C218;1109 116E;C218;1109 116E; # (수; 수; 수; 수; 수; ) HANGUL SYLLABLE SU +C219;C219;1109 116E 11A8;C219;1109 116E 11A8; # (숙; 숙; 숙; 숙; 숙; ) HANGUL SYLLABLE SUG +C21A;C21A;1109 116E 11A9;C21A;1109 116E 11A9; # (숚; 숚; 숚; 숚; 숚; ) HANGUL SYLLABLE SUGG +C21B;C21B;1109 116E 11AA;C21B;1109 116E 11AA; # (숛; 숛; 숛; 숛; 숛; ) HANGUL SYLLABLE SUGS +C21C;C21C;1109 116E 11AB;C21C;1109 116E 11AB; # (순; 순; 순; 순; 순; ) HANGUL SYLLABLE SUN +C21D;C21D;1109 116E 11AC;C21D;1109 116E 11AC; # (숝; 숝; 숝; 숝; 숝; ) HANGUL SYLLABLE SUNJ +C21E;C21E;1109 116E 11AD;C21E;1109 116E 11AD; # (숞; 숞; 숞; 숞; 숞; ) HANGUL SYLLABLE SUNH +C21F;C21F;1109 116E 11AE;C21F;1109 116E 11AE; # (숟; 숟; 숟; 숟; 숟; ) HANGUL SYLLABLE SUD +C220;C220;1109 116E 11AF;C220;1109 116E 11AF; # (술; 술; 술; 술; 술; ) HANGUL SYLLABLE SUL +C221;C221;1109 116E 11B0;C221;1109 116E 11B0; # (숡; 숡; 숡; 숡; 숡; ) HANGUL SYLLABLE SULG +C222;C222;1109 116E 11B1;C222;1109 116E 11B1; # (숢; 숢; 숢; 숢; 숢; ) HANGUL SYLLABLE SULM +C223;C223;1109 116E 11B2;C223;1109 116E 11B2; # (숣; 숣; 숣; 숣; 숣; ) HANGUL SYLLABLE SULB +C224;C224;1109 116E 11B3;C224;1109 116E 11B3; # (숤; 숤; 숤; 숤; 숤; ) HANGUL SYLLABLE SULS +C225;C225;1109 116E 11B4;C225;1109 116E 11B4; # (숥; 숥; 숥; 숥; 숥; ) HANGUL SYLLABLE SULT +C226;C226;1109 116E 11B5;C226;1109 116E 11B5; # (숦; 숦; 숦; 숦; 숦; ) HANGUL SYLLABLE SULP +C227;C227;1109 116E 11B6;C227;1109 116E 11B6; # (숧; 숧; 숧; 숧; 숧; ) HANGUL SYLLABLE SULH +C228;C228;1109 116E 11B7;C228;1109 116E 11B7; # (숨; 숨; 숨; 숨; 숨; ) HANGUL SYLLABLE SUM +C229;C229;1109 116E 11B8;C229;1109 116E 11B8; # (숩; 숩; 숩; 숩; 숩; ) HANGUL SYLLABLE SUB +C22A;C22A;1109 116E 11B9;C22A;1109 116E 11B9; # (숪; 숪; 숪; 숪; 숪; ) HANGUL SYLLABLE SUBS +C22B;C22B;1109 116E 11BA;C22B;1109 116E 11BA; # (숫; 숫; 숫; 숫; 숫; ) HANGUL SYLLABLE SUS +C22C;C22C;1109 116E 11BB;C22C;1109 116E 11BB; # (숬; 숬; 숬; 숬; 숬; ) HANGUL SYLLABLE SUSS +C22D;C22D;1109 116E 11BC;C22D;1109 116E 11BC; # (숭; 숭; 숭; 숭; 숭; ) HANGUL SYLLABLE SUNG +C22E;C22E;1109 116E 11BD;C22E;1109 116E 11BD; # (숮; 숮; 숮; 숮; 숮; ) HANGUL SYLLABLE SUJ +C22F;C22F;1109 116E 11BE;C22F;1109 116E 11BE; # (숯; 숯; 숯; 숯; 숯; ) HANGUL SYLLABLE SUC +C230;C230;1109 116E 11BF;C230;1109 116E 11BF; # (숰; 숰; 숰; 숰; 숰; ) HANGUL SYLLABLE SUK +C231;C231;1109 116E 11C0;C231;1109 116E 11C0; # (숱; 숱; 숱; 숱; 숱; ) HANGUL SYLLABLE SUT +C232;C232;1109 116E 11C1;C232;1109 116E 11C1; # (숲; 숲; 숲; 숲; 숲; ) HANGUL SYLLABLE SUP +C233;C233;1109 116E 11C2;C233;1109 116E 11C2; # (숳; 숳; 숳; 숳; 숳; ) HANGUL SYLLABLE SUH +C234;C234;1109 116F;C234;1109 116F; # (숴; 숴; 숴; 숴; 숴; ) HANGUL SYLLABLE SWEO +C235;C235;1109 116F 11A8;C235;1109 116F 11A8; # (숵; 숵; 숵; 숵; 숵; ) HANGUL SYLLABLE SWEOG +C236;C236;1109 116F 11A9;C236;1109 116F 11A9; # (숶; 숶; 숶; 숶; 숶; ) HANGUL SYLLABLE SWEOGG +C237;C237;1109 116F 11AA;C237;1109 116F 11AA; # (숷; 숷; 숷; 숷; 숷; ) HANGUL SYLLABLE SWEOGS +C238;C238;1109 116F 11AB;C238;1109 116F 11AB; # (숸; 숸; 숸; 숸; 숸; ) HANGUL SYLLABLE SWEON +C239;C239;1109 116F 11AC;C239;1109 116F 11AC; # (숹; 숹; 숹; 숹; 숹; ) HANGUL SYLLABLE SWEONJ +C23A;C23A;1109 116F 11AD;C23A;1109 116F 11AD; # (숺; 숺; 숺; 숺; 숺; ) HANGUL SYLLABLE SWEONH +C23B;C23B;1109 116F 11AE;C23B;1109 116F 11AE; # (숻; 숻; 숻; 숻; 숻; ) HANGUL SYLLABLE SWEOD +C23C;C23C;1109 116F 11AF;C23C;1109 116F 11AF; # (숼; 숼; 숼; 숼; 숼; ) HANGUL SYLLABLE SWEOL +C23D;C23D;1109 116F 11B0;C23D;1109 116F 11B0; # (숽; 숽; 숽; 숽; 숽; ) HANGUL SYLLABLE SWEOLG +C23E;C23E;1109 116F 11B1;C23E;1109 116F 11B1; # (숾; 숾; 숾; 숾; 숾; ) HANGUL SYLLABLE SWEOLM +C23F;C23F;1109 116F 11B2;C23F;1109 116F 11B2; # (숿; 숿; 숿; 숿; 숿; ) HANGUL SYLLABLE SWEOLB +C240;C240;1109 116F 11B3;C240;1109 116F 11B3; # (쉀; 쉀; 쉀; 쉀; 쉀; ) HANGUL SYLLABLE SWEOLS +C241;C241;1109 116F 11B4;C241;1109 116F 11B4; # (쉁; 쉁; 쉁; 쉁; 쉁; ) HANGUL SYLLABLE SWEOLT +C242;C242;1109 116F 11B5;C242;1109 116F 11B5; # (쉂; 쉂; 쉂; 쉂; 쉂; ) HANGUL SYLLABLE SWEOLP +C243;C243;1109 116F 11B6;C243;1109 116F 11B6; # (쉃; 쉃; 쉃; 쉃; 쉃; ) HANGUL SYLLABLE SWEOLH +C244;C244;1109 116F 11B7;C244;1109 116F 11B7; # (쉄; 쉄; 쉄; 쉄; 쉄; ) HANGUL SYLLABLE SWEOM +C245;C245;1109 116F 11B8;C245;1109 116F 11B8; # (쉅; 쉅; 쉅; 쉅; 쉅; ) HANGUL SYLLABLE SWEOB +C246;C246;1109 116F 11B9;C246;1109 116F 11B9; # (쉆; 쉆; 쉆; 쉆; 쉆; ) HANGUL SYLLABLE SWEOBS +C247;C247;1109 116F 11BA;C247;1109 116F 11BA; # (쉇; 쉇; 쉇; 쉇; 쉇; ) HANGUL SYLLABLE SWEOS +C248;C248;1109 116F 11BB;C248;1109 116F 11BB; # (쉈; 쉈; 쉈; 쉈; 쉈; ) HANGUL SYLLABLE SWEOSS +C249;C249;1109 116F 11BC;C249;1109 116F 11BC; # (쉉; 쉉; 쉉; 쉉; 쉉; ) HANGUL SYLLABLE SWEONG +C24A;C24A;1109 116F 11BD;C24A;1109 116F 11BD; # (쉊; 쉊; 쉊; 쉊; 쉊; ) HANGUL SYLLABLE SWEOJ +C24B;C24B;1109 116F 11BE;C24B;1109 116F 11BE; # (쉋; 쉋; 쉋; 쉋; 쉋; ) HANGUL SYLLABLE SWEOC +C24C;C24C;1109 116F 11BF;C24C;1109 116F 11BF; # (쉌; 쉌; 쉌; 쉌; 쉌; ) HANGUL SYLLABLE SWEOK +C24D;C24D;1109 116F 11C0;C24D;1109 116F 11C0; # (쉍; 쉍; 쉍; 쉍; 쉍; ) HANGUL SYLLABLE SWEOT +C24E;C24E;1109 116F 11C1;C24E;1109 116F 11C1; # (쉎; 쉎; 쉎; 쉎; 쉎; ) HANGUL SYLLABLE SWEOP +C24F;C24F;1109 116F 11C2;C24F;1109 116F 11C2; # (쉏; 쉏; 쉏; 쉏; 쉏; ) HANGUL SYLLABLE SWEOH +C250;C250;1109 1170;C250;1109 1170; # (쉐; 쉐; 쉐; 쉐; 쉐; ) HANGUL SYLLABLE SWE +C251;C251;1109 1170 11A8;C251;1109 1170 11A8; # (쉑; 쉑; 쉑; 쉑; 쉑; ) HANGUL SYLLABLE SWEG +C252;C252;1109 1170 11A9;C252;1109 1170 11A9; # (쉒; 쉒; 쉒; 쉒; 쉒; ) HANGUL SYLLABLE SWEGG +C253;C253;1109 1170 11AA;C253;1109 1170 11AA; # (쉓; 쉓; 쉓; 쉓; 쉓; ) HANGUL SYLLABLE SWEGS +C254;C254;1109 1170 11AB;C254;1109 1170 11AB; # (쉔; 쉔; 쉔; 쉔; 쉔; ) HANGUL SYLLABLE SWEN +C255;C255;1109 1170 11AC;C255;1109 1170 11AC; # (쉕; 쉕; 쉕; 쉕; 쉕; ) HANGUL SYLLABLE SWENJ +C256;C256;1109 1170 11AD;C256;1109 1170 11AD; # (쉖; 쉖; 쉖; 쉖; 쉖; ) HANGUL SYLLABLE SWENH +C257;C257;1109 1170 11AE;C257;1109 1170 11AE; # (쉗; 쉗; 쉗; 쉗; 쉗; ) HANGUL SYLLABLE SWED +C258;C258;1109 1170 11AF;C258;1109 1170 11AF; # (쉘; 쉘; 쉘; 쉘; 쉘; ) HANGUL SYLLABLE SWEL +C259;C259;1109 1170 11B0;C259;1109 1170 11B0; # (쉙; 쉙; 쉙; 쉙; 쉙; ) HANGUL SYLLABLE SWELG +C25A;C25A;1109 1170 11B1;C25A;1109 1170 11B1; # (쉚; 쉚; 쉚; 쉚; 쉚; ) HANGUL SYLLABLE SWELM +C25B;C25B;1109 1170 11B2;C25B;1109 1170 11B2; # (쉛; 쉛; 쉛; 쉛; 쉛; ) HANGUL SYLLABLE SWELB +C25C;C25C;1109 1170 11B3;C25C;1109 1170 11B3; # (쉜; 쉜; 쉜; 쉜; 쉜; ) HANGUL SYLLABLE SWELS +C25D;C25D;1109 1170 11B4;C25D;1109 1170 11B4; # (쉝; 쉝; 쉝; 쉝; 쉝; ) HANGUL SYLLABLE SWELT +C25E;C25E;1109 1170 11B5;C25E;1109 1170 11B5; # (쉞; 쉞; 쉞; 쉞; 쉞; ) HANGUL SYLLABLE SWELP +C25F;C25F;1109 1170 11B6;C25F;1109 1170 11B6; # (쉟; 쉟; 쉟; 쉟; 쉟; ) HANGUL SYLLABLE SWELH +C260;C260;1109 1170 11B7;C260;1109 1170 11B7; # (쉠; 쉠; 쉠; 쉠; 쉠; ) HANGUL SYLLABLE SWEM +C261;C261;1109 1170 11B8;C261;1109 1170 11B8; # (쉡; 쉡; 쉡; 쉡; 쉡; ) HANGUL SYLLABLE SWEB +C262;C262;1109 1170 11B9;C262;1109 1170 11B9; # (쉢; 쉢; 쉢; 쉢; 쉢; ) HANGUL SYLLABLE SWEBS +C263;C263;1109 1170 11BA;C263;1109 1170 11BA; # (쉣; 쉣; 쉣; 쉣; 쉣; ) HANGUL SYLLABLE SWES +C264;C264;1109 1170 11BB;C264;1109 1170 11BB; # (쉤; 쉤; 쉤; 쉤; 쉤; ) HANGUL SYLLABLE SWESS +C265;C265;1109 1170 11BC;C265;1109 1170 11BC; # (쉥; 쉥; 쉥; 쉥; 쉥; ) HANGUL SYLLABLE SWENG +C266;C266;1109 1170 11BD;C266;1109 1170 11BD; # (쉦; 쉦; 쉦; 쉦; 쉦; ) HANGUL SYLLABLE SWEJ +C267;C267;1109 1170 11BE;C267;1109 1170 11BE; # (쉧; 쉧; 쉧; 쉧; 쉧; ) HANGUL SYLLABLE SWEC +C268;C268;1109 1170 11BF;C268;1109 1170 11BF; # (쉨; 쉨; 쉨; 쉨; 쉨; ) HANGUL SYLLABLE SWEK +C269;C269;1109 1170 11C0;C269;1109 1170 11C0; # (쉩; 쉩; 쉩; 쉩; 쉩; ) HANGUL SYLLABLE SWET +C26A;C26A;1109 1170 11C1;C26A;1109 1170 11C1; # (쉪; 쉪; 쉪; 쉪; 쉪; ) HANGUL SYLLABLE SWEP +C26B;C26B;1109 1170 11C2;C26B;1109 1170 11C2; # (쉫; 쉫; 쉫; 쉫; 쉫; ) HANGUL SYLLABLE SWEH +C26C;C26C;1109 1171;C26C;1109 1171; # (쉬; 쉬; 쉬; 쉬; 쉬; ) HANGUL SYLLABLE SWI +C26D;C26D;1109 1171 11A8;C26D;1109 1171 11A8; # (쉭; 쉭; 쉭; 쉭; 쉭; ) HANGUL SYLLABLE SWIG +C26E;C26E;1109 1171 11A9;C26E;1109 1171 11A9; # (쉮; 쉮; 쉮; 쉮; 쉮; ) HANGUL SYLLABLE SWIGG +C26F;C26F;1109 1171 11AA;C26F;1109 1171 11AA; # (쉯; 쉯; 쉯; 쉯; 쉯; ) HANGUL SYLLABLE SWIGS +C270;C270;1109 1171 11AB;C270;1109 1171 11AB; # (쉰; 쉰; 쉰; 쉰; 쉰; ) HANGUL SYLLABLE SWIN +C271;C271;1109 1171 11AC;C271;1109 1171 11AC; # (쉱; 쉱; 쉱; 쉱; 쉱; ) HANGUL SYLLABLE SWINJ +C272;C272;1109 1171 11AD;C272;1109 1171 11AD; # (쉲; 쉲; 쉲; 쉲; 쉲; ) HANGUL SYLLABLE SWINH +C273;C273;1109 1171 11AE;C273;1109 1171 11AE; # (쉳; 쉳; 쉳; 쉳; 쉳; ) HANGUL SYLLABLE SWID +C274;C274;1109 1171 11AF;C274;1109 1171 11AF; # (쉴; 쉴; 쉴; 쉴; 쉴; ) HANGUL SYLLABLE SWIL +C275;C275;1109 1171 11B0;C275;1109 1171 11B0; # (쉵; 쉵; 쉵; 쉵; 쉵; ) HANGUL SYLLABLE SWILG +C276;C276;1109 1171 11B1;C276;1109 1171 11B1; # (쉶; 쉶; 쉶; 쉶; 쉶; ) HANGUL SYLLABLE SWILM +C277;C277;1109 1171 11B2;C277;1109 1171 11B2; # (쉷; 쉷; 쉷; 쉷; 쉷; ) HANGUL SYLLABLE SWILB +C278;C278;1109 1171 11B3;C278;1109 1171 11B3; # (쉸; 쉸; 쉸; 쉸; 쉸; ) HANGUL SYLLABLE SWILS +C279;C279;1109 1171 11B4;C279;1109 1171 11B4; # (쉹; 쉹; 쉹; 쉹; 쉹; ) HANGUL SYLLABLE SWILT +C27A;C27A;1109 1171 11B5;C27A;1109 1171 11B5; # (쉺; 쉺; 쉺; 쉺; 쉺; ) HANGUL SYLLABLE SWILP +C27B;C27B;1109 1171 11B6;C27B;1109 1171 11B6; # (쉻; 쉻; 쉻; 쉻; 쉻; ) HANGUL SYLLABLE SWILH +C27C;C27C;1109 1171 11B7;C27C;1109 1171 11B7; # (쉼; 쉼; 쉼; 쉼; 쉼; ) HANGUL SYLLABLE SWIM +C27D;C27D;1109 1171 11B8;C27D;1109 1171 11B8; # (쉽; 쉽; 쉽; 쉽; 쉽; ) HANGUL SYLLABLE SWIB +C27E;C27E;1109 1171 11B9;C27E;1109 1171 11B9; # (쉾; 쉾; 쉾; 쉾; 쉾; ) HANGUL SYLLABLE SWIBS +C27F;C27F;1109 1171 11BA;C27F;1109 1171 11BA; # (쉿; 쉿; 쉿; 쉿; 쉿; ) HANGUL SYLLABLE SWIS +C280;C280;1109 1171 11BB;C280;1109 1171 11BB; # (슀; 슀; 슀; 슀; 슀; ) HANGUL SYLLABLE SWISS +C281;C281;1109 1171 11BC;C281;1109 1171 11BC; # (슁; 슁; 슁; 슁; 슁; ) HANGUL SYLLABLE SWING +C282;C282;1109 1171 11BD;C282;1109 1171 11BD; # (슂; 슂; 슂; 슂; 슂; ) HANGUL SYLLABLE SWIJ +C283;C283;1109 1171 11BE;C283;1109 1171 11BE; # (슃; 슃; 슃; 슃; 슃; ) HANGUL SYLLABLE SWIC +C284;C284;1109 1171 11BF;C284;1109 1171 11BF; # (슄; 슄; 슄; 슄; 슄; ) HANGUL SYLLABLE SWIK +C285;C285;1109 1171 11C0;C285;1109 1171 11C0; # (슅; 슅; 슅; 슅; 슅; ) HANGUL SYLLABLE SWIT +C286;C286;1109 1171 11C1;C286;1109 1171 11C1; # (슆; 슆; 슆; 슆; 슆; ) HANGUL SYLLABLE SWIP +C287;C287;1109 1171 11C2;C287;1109 1171 11C2; # (슇; 슇; 슇; 슇; 슇; ) HANGUL SYLLABLE SWIH +C288;C288;1109 1172;C288;1109 1172; # (슈; 슈; 슈; 슈; 슈; ) HANGUL SYLLABLE SYU +C289;C289;1109 1172 11A8;C289;1109 1172 11A8; # (슉; 슉; 슉; 슉; 슉; ) HANGUL SYLLABLE SYUG +C28A;C28A;1109 1172 11A9;C28A;1109 1172 11A9; # (슊; 슊; 슊; 슊; 슊; ) HANGUL SYLLABLE SYUGG +C28B;C28B;1109 1172 11AA;C28B;1109 1172 11AA; # (슋; 슋; 슋; 슋; 슋; ) HANGUL SYLLABLE SYUGS +C28C;C28C;1109 1172 11AB;C28C;1109 1172 11AB; # (슌; 슌; 슌; 슌; 슌; ) HANGUL SYLLABLE SYUN +C28D;C28D;1109 1172 11AC;C28D;1109 1172 11AC; # (슍; 슍; 슍; 슍; 슍; ) HANGUL SYLLABLE SYUNJ +C28E;C28E;1109 1172 11AD;C28E;1109 1172 11AD; # (슎; 슎; 슎; 슎; 슎; ) HANGUL SYLLABLE SYUNH +C28F;C28F;1109 1172 11AE;C28F;1109 1172 11AE; # (슏; 슏; 슏; 슏; 슏; ) HANGUL SYLLABLE SYUD +C290;C290;1109 1172 11AF;C290;1109 1172 11AF; # (슐; 슐; 슐; 슐; 슐; ) HANGUL SYLLABLE SYUL +C291;C291;1109 1172 11B0;C291;1109 1172 11B0; # (슑; 슑; 슑; 슑; 슑; ) HANGUL SYLLABLE SYULG +C292;C292;1109 1172 11B1;C292;1109 1172 11B1; # (슒; 슒; 슒; 슒; 슒; ) HANGUL SYLLABLE SYULM +C293;C293;1109 1172 11B2;C293;1109 1172 11B2; # (슓; 슓; 슓; 슓; 슓; ) HANGUL SYLLABLE SYULB +C294;C294;1109 1172 11B3;C294;1109 1172 11B3; # (슔; 슔; 슔; 슔; 슔; ) HANGUL SYLLABLE SYULS +C295;C295;1109 1172 11B4;C295;1109 1172 11B4; # (슕; 슕; 슕; 슕; 슕; ) HANGUL SYLLABLE SYULT +C296;C296;1109 1172 11B5;C296;1109 1172 11B5; # (슖; 슖; 슖; 슖; 슖; ) HANGUL SYLLABLE SYULP +C297;C297;1109 1172 11B6;C297;1109 1172 11B6; # (슗; 슗; 슗; 슗; 슗; ) HANGUL SYLLABLE SYULH +C298;C298;1109 1172 11B7;C298;1109 1172 11B7; # (슘; 슘; 슘; 슘; 슘; ) HANGUL SYLLABLE SYUM +C299;C299;1109 1172 11B8;C299;1109 1172 11B8; # (슙; 슙; 슙; 슙; 슙; ) HANGUL SYLLABLE SYUB +C29A;C29A;1109 1172 11B9;C29A;1109 1172 11B9; # (슚; 슚; 슚; 슚; 슚; ) HANGUL SYLLABLE SYUBS +C29B;C29B;1109 1172 11BA;C29B;1109 1172 11BA; # (슛; 슛; 슛; 슛; 슛; ) HANGUL SYLLABLE SYUS +C29C;C29C;1109 1172 11BB;C29C;1109 1172 11BB; # (슜; 슜; 슜; 슜; 슜; ) HANGUL SYLLABLE SYUSS +C29D;C29D;1109 1172 11BC;C29D;1109 1172 11BC; # (슝; 슝; 슝; 슝; 슝; ) HANGUL SYLLABLE SYUNG +C29E;C29E;1109 1172 11BD;C29E;1109 1172 11BD; # (슞; 슞; 슞; 슞; 슞; ) HANGUL SYLLABLE SYUJ +C29F;C29F;1109 1172 11BE;C29F;1109 1172 11BE; # (슟; 슟; 슟; 슟; 슟; ) HANGUL SYLLABLE SYUC +C2A0;C2A0;1109 1172 11BF;C2A0;1109 1172 11BF; # (슠; 슠; 슠; 슠; 슠; ) HANGUL SYLLABLE SYUK +C2A1;C2A1;1109 1172 11C0;C2A1;1109 1172 11C0; # (슡; 슡; 슡; 슡; 슡; ) HANGUL SYLLABLE SYUT +C2A2;C2A2;1109 1172 11C1;C2A2;1109 1172 11C1; # (슢; 슢; 슢; 슢; 슢; ) HANGUL SYLLABLE SYUP +C2A3;C2A3;1109 1172 11C2;C2A3;1109 1172 11C2; # (슣; 슣; 슣; 슣; 슣; ) HANGUL SYLLABLE SYUH +C2A4;C2A4;1109 1173;C2A4;1109 1173; # (스; 스; 스; 스; 스; ) HANGUL SYLLABLE SEU +C2A5;C2A5;1109 1173 11A8;C2A5;1109 1173 11A8; # (슥; 슥; 슥; 슥; 슥; ) HANGUL SYLLABLE SEUG +C2A6;C2A6;1109 1173 11A9;C2A6;1109 1173 11A9; # (슦; 슦; 슦; 슦; 슦; ) HANGUL SYLLABLE SEUGG +C2A7;C2A7;1109 1173 11AA;C2A7;1109 1173 11AA; # (슧; 슧; 슧; 슧; 슧; ) HANGUL SYLLABLE SEUGS +C2A8;C2A8;1109 1173 11AB;C2A8;1109 1173 11AB; # (슨; 슨; 슨; 슨; 슨; ) HANGUL SYLLABLE SEUN +C2A9;C2A9;1109 1173 11AC;C2A9;1109 1173 11AC; # (슩; 슩; 슩; 슩; 슩; ) HANGUL SYLLABLE SEUNJ +C2AA;C2AA;1109 1173 11AD;C2AA;1109 1173 11AD; # (슪; 슪; 슪; 슪; 슪; ) HANGUL SYLLABLE SEUNH +C2AB;C2AB;1109 1173 11AE;C2AB;1109 1173 11AE; # (슫; 슫; 슫; 슫; 슫; ) HANGUL SYLLABLE SEUD +C2AC;C2AC;1109 1173 11AF;C2AC;1109 1173 11AF; # (슬; 슬; 슬; 슬; 슬; ) HANGUL SYLLABLE SEUL +C2AD;C2AD;1109 1173 11B0;C2AD;1109 1173 11B0; # (슭; 슭; 슭; 슭; 슭; ) HANGUL SYLLABLE SEULG +C2AE;C2AE;1109 1173 11B1;C2AE;1109 1173 11B1; # (슮; 슮; 슮; 슮; 슮; ) HANGUL SYLLABLE SEULM +C2AF;C2AF;1109 1173 11B2;C2AF;1109 1173 11B2; # (슯; 슯; 슯; 슯; 슯; ) HANGUL SYLLABLE SEULB +C2B0;C2B0;1109 1173 11B3;C2B0;1109 1173 11B3; # (슰; 슰; 슰; 슰; 슰; ) HANGUL SYLLABLE SEULS +C2B1;C2B1;1109 1173 11B4;C2B1;1109 1173 11B4; # (슱; 슱; 슱; 슱; 슱; ) HANGUL SYLLABLE SEULT +C2B2;C2B2;1109 1173 11B5;C2B2;1109 1173 11B5; # (슲; 슲; 슲; 슲; 슲; ) HANGUL SYLLABLE SEULP +C2B3;C2B3;1109 1173 11B6;C2B3;1109 1173 11B6; # (슳; 슳; 슳; 슳; 슳; ) HANGUL SYLLABLE SEULH +C2B4;C2B4;1109 1173 11B7;C2B4;1109 1173 11B7; # (슴; 슴; 슴; 슴; 슴; ) HANGUL SYLLABLE SEUM +C2B5;C2B5;1109 1173 11B8;C2B5;1109 1173 11B8; # (습; 습; 습; 습; 습; ) HANGUL SYLLABLE SEUB +C2B6;C2B6;1109 1173 11B9;C2B6;1109 1173 11B9; # (슶; 슶; 슶; 슶; 슶; ) HANGUL SYLLABLE SEUBS +C2B7;C2B7;1109 1173 11BA;C2B7;1109 1173 11BA; # (슷; 슷; 슷; 슷; 슷; ) HANGUL SYLLABLE SEUS +C2B8;C2B8;1109 1173 11BB;C2B8;1109 1173 11BB; # (슸; 슸; 슸; 슸; 슸; ) HANGUL SYLLABLE SEUSS +C2B9;C2B9;1109 1173 11BC;C2B9;1109 1173 11BC; # (승; 승; 승; 승; 승; ) HANGUL SYLLABLE SEUNG +C2BA;C2BA;1109 1173 11BD;C2BA;1109 1173 11BD; # (슺; 슺; 슺; 슺; 슺; ) HANGUL SYLLABLE SEUJ +C2BB;C2BB;1109 1173 11BE;C2BB;1109 1173 11BE; # (슻; 슻; 슻; 슻; 슻; ) HANGUL SYLLABLE SEUC +C2BC;C2BC;1109 1173 11BF;C2BC;1109 1173 11BF; # (슼; 슼; 슼; 슼; 슼; ) HANGUL SYLLABLE SEUK +C2BD;C2BD;1109 1173 11C0;C2BD;1109 1173 11C0; # (슽; 슽; 슽; 슽; 슽; ) HANGUL SYLLABLE SEUT +C2BE;C2BE;1109 1173 11C1;C2BE;1109 1173 11C1; # (슾; 슾; 슾; 슾; 슾; ) HANGUL SYLLABLE SEUP +C2BF;C2BF;1109 1173 11C2;C2BF;1109 1173 11C2; # (슿; 슿; 슿; 슿; 슿; ) HANGUL SYLLABLE SEUH +C2C0;C2C0;1109 1174;C2C0;1109 1174; # (싀; 싀; 싀; 싀; 싀; ) HANGUL SYLLABLE SYI +C2C1;C2C1;1109 1174 11A8;C2C1;1109 1174 11A8; # (싁; 싁; 싁; 싁; 싁; ) HANGUL SYLLABLE SYIG +C2C2;C2C2;1109 1174 11A9;C2C2;1109 1174 11A9; # (싂; 싂; 싂; 싂; 싂; ) HANGUL SYLLABLE SYIGG +C2C3;C2C3;1109 1174 11AA;C2C3;1109 1174 11AA; # (싃; 싃; 싃; 싃; 싃; ) HANGUL SYLLABLE SYIGS +C2C4;C2C4;1109 1174 11AB;C2C4;1109 1174 11AB; # (싄; 싄; 싄; 싄; 싄; ) HANGUL SYLLABLE SYIN +C2C5;C2C5;1109 1174 11AC;C2C5;1109 1174 11AC; # (싅; 싅; 싅; 싅; 싅; ) HANGUL SYLLABLE SYINJ +C2C6;C2C6;1109 1174 11AD;C2C6;1109 1174 11AD; # (싆; 싆; 싆; 싆; 싆; ) HANGUL SYLLABLE SYINH +C2C7;C2C7;1109 1174 11AE;C2C7;1109 1174 11AE; # (싇; 싇; 싇; 싇; 싇; ) HANGUL SYLLABLE SYID +C2C8;C2C8;1109 1174 11AF;C2C8;1109 1174 11AF; # (싈; 싈; 싈; 싈; 싈; ) HANGUL SYLLABLE SYIL +C2C9;C2C9;1109 1174 11B0;C2C9;1109 1174 11B0; # (싉; 싉; 싉; 싉; 싉; ) HANGUL SYLLABLE SYILG +C2CA;C2CA;1109 1174 11B1;C2CA;1109 1174 11B1; # (싊; 싊; 싊; 싊; 싊; ) HANGUL SYLLABLE SYILM +C2CB;C2CB;1109 1174 11B2;C2CB;1109 1174 11B2; # (싋; 싋; 싋; 싋; 싋; ) HANGUL SYLLABLE SYILB +C2CC;C2CC;1109 1174 11B3;C2CC;1109 1174 11B3; # (싌; 싌; 싌; 싌; 싌; ) HANGUL SYLLABLE SYILS +C2CD;C2CD;1109 1174 11B4;C2CD;1109 1174 11B4; # (싍; 싍; 싍; 싍; 싍; ) HANGUL SYLLABLE SYILT +C2CE;C2CE;1109 1174 11B5;C2CE;1109 1174 11B5; # (싎; 싎; 싎; 싎; 싎; ) HANGUL SYLLABLE SYILP +C2CF;C2CF;1109 1174 11B6;C2CF;1109 1174 11B6; # (싏; 싏; 싏; 싏; 싏; ) HANGUL SYLLABLE SYILH +C2D0;C2D0;1109 1174 11B7;C2D0;1109 1174 11B7; # (싐; 싐; 싐; 싐; 싐; ) HANGUL SYLLABLE SYIM +C2D1;C2D1;1109 1174 11B8;C2D1;1109 1174 11B8; # (싑; 싑; 싑; 싑; 싑; ) HANGUL SYLLABLE SYIB +C2D2;C2D2;1109 1174 11B9;C2D2;1109 1174 11B9; # (싒; 싒; 싒; 싒; 싒; ) HANGUL SYLLABLE SYIBS +C2D3;C2D3;1109 1174 11BA;C2D3;1109 1174 11BA; # (싓; 싓; 싓; 싓; 싓; ) HANGUL SYLLABLE SYIS +C2D4;C2D4;1109 1174 11BB;C2D4;1109 1174 11BB; # (싔; 싔; 싔; 싔; 싔; ) HANGUL SYLLABLE SYISS +C2D5;C2D5;1109 1174 11BC;C2D5;1109 1174 11BC; # (싕; 싕; 싕; 싕; 싕; ) HANGUL SYLLABLE SYING +C2D6;C2D6;1109 1174 11BD;C2D6;1109 1174 11BD; # (싖; 싖; 싖; 싖; 싖; ) HANGUL SYLLABLE SYIJ +C2D7;C2D7;1109 1174 11BE;C2D7;1109 1174 11BE; # (싗; 싗; 싗; 싗; 싗; ) HANGUL SYLLABLE SYIC +C2D8;C2D8;1109 1174 11BF;C2D8;1109 1174 11BF; # (싘; 싘; 싘; 싘; 싘; ) HANGUL SYLLABLE SYIK +C2D9;C2D9;1109 1174 11C0;C2D9;1109 1174 11C0; # (싙; 싙; 싙; 싙; 싙; ) HANGUL SYLLABLE SYIT +C2DA;C2DA;1109 1174 11C1;C2DA;1109 1174 11C1; # (싚; 싚; 싚; 싚; 싚; ) HANGUL SYLLABLE SYIP +C2DB;C2DB;1109 1174 11C2;C2DB;1109 1174 11C2; # (싛; 싛; 싛; 싛; 싛; ) HANGUL SYLLABLE SYIH +C2DC;C2DC;1109 1175;C2DC;1109 1175; # (시; 시; 시; 시; 시; ) HANGUL SYLLABLE SI +C2DD;C2DD;1109 1175 11A8;C2DD;1109 1175 11A8; # (식; 식; 식; 식; 식; ) HANGUL SYLLABLE SIG +C2DE;C2DE;1109 1175 11A9;C2DE;1109 1175 11A9; # (싞; 싞; 싞; 싞; 싞; ) HANGUL SYLLABLE SIGG +C2DF;C2DF;1109 1175 11AA;C2DF;1109 1175 11AA; # (싟; 싟; 싟; 싟; 싟; ) HANGUL SYLLABLE SIGS +C2E0;C2E0;1109 1175 11AB;C2E0;1109 1175 11AB; # (신; 신; 신; 신; 신; ) HANGUL SYLLABLE SIN +C2E1;C2E1;1109 1175 11AC;C2E1;1109 1175 11AC; # (싡; 싡; 싡; 싡; 싡; ) HANGUL SYLLABLE SINJ +C2E2;C2E2;1109 1175 11AD;C2E2;1109 1175 11AD; # (싢; 싢; 싢; 싢; 싢; ) HANGUL SYLLABLE SINH +C2E3;C2E3;1109 1175 11AE;C2E3;1109 1175 11AE; # (싣; 싣; 싣; 싣; 싣; ) HANGUL SYLLABLE SID +C2E4;C2E4;1109 1175 11AF;C2E4;1109 1175 11AF; # (실; 실; 실; 실; 실; ) HANGUL SYLLABLE SIL +C2E5;C2E5;1109 1175 11B0;C2E5;1109 1175 11B0; # (싥; 싥; 싥; 싥; 싥; ) HANGUL SYLLABLE SILG +C2E6;C2E6;1109 1175 11B1;C2E6;1109 1175 11B1; # (싦; 싦; 싦; 싦; 싦; ) HANGUL SYLLABLE SILM +C2E7;C2E7;1109 1175 11B2;C2E7;1109 1175 11B2; # (싧; 싧; 싧; 싧; 싧; ) HANGUL SYLLABLE SILB +C2E8;C2E8;1109 1175 11B3;C2E8;1109 1175 11B3; # (싨; 싨; 싨; 싨; 싨; ) HANGUL SYLLABLE SILS +C2E9;C2E9;1109 1175 11B4;C2E9;1109 1175 11B4; # (싩; 싩; 싩; 싩; 싩; ) HANGUL SYLLABLE SILT +C2EA;C2EA;1109 1175 11B5;C2EA;1109 1175 11B5; # (싪; 싪; 싪; 싪; 싪; ) HANGUL SYLLABLE SILP +C2EB;C2EB;1109 1175 11B6;C2EB;1109 1175 11B6; # (싫; 싫; 싫; 싫; 싫; ) HANGUL SYLLABLE SILH +C2EC;C2EC;1109 1175 11B7;C2EC;1109 1175 11B7; # (심; 심; 심; 심; 심; ) HANGUL SYLLABLE SIM +C2ED;C2ED;1109 1175 11B8;C2ED;1109 1175 11B8; # (십; 십; 십; 십; 십; ) HANGUL SYLLABLE SIB +C2EE;C2EE;1109 1175 11B9;C2EE;1109 1175 11B9; # (싮; 싮; 싮; 싮; 싮; ) HANGUL SYLLABLE SIBS +C2EF;C2EF;1109 1175 11BA;C2EF;1109 1175 11BA; # (싯; 싯; 싯; 싯; 싯; ) HANGUL SYLLABLE SIS +C2F0;C2F0;1109 1175 11BB;C2F0;1109 1175 11BB; # (싰; 싰; 싰; 싰; 싰; ) HANGUL SYLLABLE SISS +C2F1;C2F1;1109 1175 11BC;C2F1;1109 1175 11BC; # (싱; 싱; 싱; 싱; 싱; ) HANGUL SYLLABLE SING +C2F2;C2F2;1109 1175 11BD;C2F2;1109 1175 11BD; # (싲; 싲; 싲; 싲; 싲; ) HANGUL SYLLABLE SIJ +C2F3;C2F3;1109 1175 11BE;C2F3;1109 1175 11BE; # (싳; 싳; 싳; 싳; 싳; ) HANGUL SYLLABLE SIC +C2F4;C2F4;1109 1175 11BF;C2F4;1109 1175 11BF; # (싴; 싴; 싴; 싴; 싴; ) HANGUL SYLLABLE SIK +C2F5;C2F5;1109 1175 11C0;C2F5;1109 1175 11C0; # (싵; 싵; 싵; 싵; 싵; ) HANGUL SYLLABLE SIT +C2F6;C2F6;1109 1175 11C1;C2F6;1109 1175 11C1; # (싶; 싶; 싶; 싶; 싶; ) HANGUL SYLLABLE SIP +C2F7;C2F7;1109 1175 11C2;C2F7;1109 1175 11C2; # (싷; 싷; 싷; 싷; 싷; ) HANGUL SYLLABLE SIH +C2F8;C2F8;110A 1161;C2F8;110A 1161; # (싸; 싸; 싸; 싸; 싸; ) HANGUL SYLLABLE SSA +C2F9;C2F9;110A 1161 11A8;C2F9;110A 1161 11A8; # (싹; 싹; 싹; 싹; 싹; ) HANGUL SYLLABLE SSAG +C2FA;C2FA;110A 1161 11A9;C2FA;110A 1161 11A9; # (싺; 싺; 싺; 싺; 싺; ) HANGUL SYLLABLE SSAGG +C2FB;C2FB;110A 1161 11AA;C2FB;110A 1161 11AA; # (싻; 싻; 싻; 싻; 싻; ) HANGUL SYLLABLE SSAGS +C2FC;C2FC;110A 1161 11AB;C2FC;110A 1161 11AB; # (싼; 싼; 싼; 싼; 싼; ) HANGUL SYLLABLE SSAN +C2FD;C2FD;110A 1161 11AC;C2FD;110A 1161 11AC; # (싽; 싽; 싽; 싽; 싽; ) HANGUL SYLLABLE SSANJ +C2FE;C2FE;110A 1161 11AD;C2FE;110A 1161 11AD; # (싾; 싾; 싾; 싾; 싾; ) HANGUL SYLLABLE SSANH +C2FF;C2FF;110A 1161 11AE;C2FF;110A 1161 11AE; # (싿; 싿; 싿; 싿; 싿; ) HANGUL SYLLABLE SSAD +C300;C300;110A 1161 11AF;C300;110A 1161 11AF; # (쌀; 쌀; 쌀; 쌀; 쌀; ) HANGUL SYLLABLE SSAL +C301;C301;110A 1161 11B0;C301;110A 1161 11B0; # (쌁; 쌁; 쌁; 쌁; 쌁; ) HANGUL SYLLABLE SSALG +C302;C302;110A 1161 11B1;C302;110A 1161 11B1; # (쌂; 쌂; 쌂; 쌂; 쌂; ) HANGUL SYLLABLE SSALM +C303;C303;110A 1161 11B2;C303;110A 1161 11B2; # (쌃; 쌃; 쌃; 쌃; 쌃; ) HANGUL SYLLABLE SSALB +C304;C304;110A 1161 11B3;C304;110A 1161 11B3; # (쌄; 쌄; 쌄; 쌄; 쌄; ) HANGUL SYLLABLE SSALS +C305;C305;110A 1161 11B4;C305;110A 1161 11B4; # (쌅; 쌅; 쌅; 쌅; 쌅; ) HANGUL SYLLABLE SSALT +C306;C306;110A 1161 11B5;C306;110A 1161 11B5; # (쌆; 쌆; 쌆; 쌆; 쌆; ) HANGUL SYLLABLE SSALP +C307;C307;110A 1161 11B6;C307;110A 1161 11B6; # (쌇; 쌇; 쌇; 쌇; 쌇; ) HANGUL SYLLABLE SSALH +C308;C308;110A 1161 11B7;C308;110A 1161 11B7; # (쌈; 쌈; 쌈; 쌈; 쌈; ) HANGUL SYLLABLE SSAM +C309;C309;110A 1161 11B8;C309;110A 1161 11B8; # (쌉; 쌉; 쌉; 쌉; 쌉; ) HANGUL SYLLABLE SSAB +C30A;C30A;110A 1161 11B9;C30A;110A 1161 11B9; # (쌊; 쌊; 쌊; 쌊; 쌊; ) HANGUL SYLLABLE SSABS +C30B;C30B;110A 1161 11BA;C30B;110A 1161 11BA; # (쌋; 쌋; 쌋; 쌋; 쌋; ) HANGUL SYLLABLE SSAS +C30C;C30C;110A 1161 11BB;C30C;110A 1161 11BB; # (쌌; 쌌; 쌌; 쌌; 쌌; ) HANGUL SYLLABLE SSASS +C30D;C30D;110A 1161 11BC;C30D;110A 1161 11BC; # (쌍; 쌍; 쌍; 쌍; 쌍; ) HANGUL SYLLABLE SSANG +C30E;C30E;110A 1161 11BD;C30E;110A 1161 11BD; # (쌎; 쌎; 쌎; 쌎; 쌎; ) HANGUL SYLLABLE SSAJ +C30F;C30F;110A 1161 11BE;C30F;110A 1161 11BE; # (쌏; 쌏; 쌏; 쌏; 쌏; ) HANGUL SYLLABLE SSAC +C310;C310;110A 1161 11BF;C310;110A 1161 11BF; # (쌐; 쌐; 쌐; 쌐; 쌐; ) HANGUL SYLLABLE SSAK +C311;C311;110A 1161 11C0;C311;110A 1161 11C0; # (쌑; 쌑; 쌑; 쌑; 쌑; ) HANGUL SYLLABLE SSAT +C312;C312;110A 1161 11C1;C312;110A 1161 11C1; # (쌒; 쌒; 쌒; 쌒; 쌒; ) HANGUL SYLLABLE SSAP +C313;C313;110A 1161 11C2;C313;110A 1161 11C2; # (쌓; 쌓; 쌓; 쌓; 쌓; ) HANGUL SYLLABLE SSAH +C314;C314;110A 1162;C314;110A 1162; # (쌔; 쌔; 쌔; 쌔; 쌔; ) HANGUL SYLLABLE SSAE +C315;C315;110A 1162 11A8;C315;110A 1162 11A8; # (쌕; 쌕; 쌕; 쌕; 쌕; ) HANGUL SYLLABLE SSAEG +C316;C316;110A 1162 11A9;C316;110A 1162 11A9; # (쌖; 쌖; 쌖; 쌖; 쌖; ) HANGUL SYLLABLE SSAEGG +C317;C317;110A 1162 11AA;C317;110A 1162 11AA; # (쌗; 쌗; 쌗; 쌗; 쌗; ) HANGUL SYLLABLE SSAEGS +C318;C318;110A 1162 11AB;C318;110A 1162 11AB; # (쌘; 쌘; 쌘; 쌘; 쌘; ) HANGUL SYLLABLE SSAEN +C319;C319;110A 1162 11AC;C319;110A 1162 11AC; # (쌙; 쌙; 쌙; 쌙; 쌙; ) HANGUL SYLLABLE SSAENJ +C31A;C31A;110A 1162 11AD;C31A;110A 1162 11AD; # (쌚; 쌚; 쌚; 쌚; 쌚; ) HANGUL SYLLABLE SSAENH +C31B;C31B;110A 1162 11AE;C31B;110A 1162 11AE; # (쌛; 쌛; 쌛; 쌛; 쌛; ) HANGUL SYLLABLE SSAED +C31C;C31C;110A 1162 11AF;C31C;110A 1162 11AF; # (쌜; 쌜; 쌜; 쌜; 쌜; ) HANGUL SYLLABLE SSAEL +C31D;C31D;110A 1162 11B0;C31D;110A 1162 11B0; # (쌝; 쌝; 쌝; 쌝; 쌝; ) HANGUL SYLLABLE SSAELG +C31E;C31E;110A 1162 11B1;C31E;110A 1162 11B1; # (쌞; 쌞; 쌞; 쌞; 쌞; ) HANGUL SYLLABLE SSAELM +C31F;C31F;110A 1162 11B2;C31F;110A 1162 11B2; # (쌟; 쌟; 쌟; 쌟; 쌟; ) HANGUL SYLLABLE SSAELB +C320;C320;110A 1162 11B3;C320;110A 1162 11B3; # (쌠; 쌠; 쌠; 쌠; 쌠; ) HANGUL SYLLABLE SSAELS +C321;C321;110A 1162 11B4;C321;110A 1162 11B4; # (쌡; 쌡; 쌡; 쌡; 쌡; ) HANGUL SYLLABLE SSAELT +C322;C322;110A 1162 11B5;C322;110A 1162 11B5; # (쌢; 쌢; 쌢; 쌢; 쌢; ) HANGUL SYLLABLE SSAELP +C323;C323;110A 1162 11B6;C323;110A 1162 11B6; # (쌣; 쌣; 쌣; 쌣; 쌣; ) HANGUL SYLLABLE SSAELH +C324;C324;110A 1162 11B7;C324;110A 1162 11B7; # (쌤; 쌤; 쌤; 쌤; 쌤; ) HANGUL SYLLABLE SSAEM +C325;C325;110A 1162 11B8;C325;110A 1162 11B8; # (쌥; 쌥; 쌥; 쌥; 쌥; ) HANGUL SYLLABLE SSAEB +C326;C326;110A 1162 11B9;C326;110A 1162 11B9; # (쌦; 쌦; 쌦; 쌦; 쌦; ) HANGUL SYLLABLE SSAEBS +C327;C327;110A 1162 11BA;C327;110A 1162 11BA; # (쌧; 쌧; 쌧; 쌧; 쌧; ) HANGUL SYLLABLE SSAES +C328;C328;110A 1162 11BB;C328;110A 1162 11BB; # (쌨; 쌨; 쌨; 쌨; 쌨; ) HANGUL SYLLABLE SSAESS +C329;C329;110A 1162 11BC;C329;110A 1162 11BC; # (쌩; 쌩; 쌩; 쌩; 쌩; ) HANGUL SYLLABLE SSAENG +C32A;C32A;110A 1162 11BD;C32A;110A 1162 11BD; # (쌪; 쌪; 쌪; 쌪; 쌪; ) HANGUL SYLLABLE SSAEJ +C32B;C32B;110A 1162 11BE;C32B;110A 1162 11BE; # (쌫; 쌫; 쌫; 쌫; 쌫; ) HANGUL SYLLABLE SSAEC +C32C;C32C;110A 1162 11BF;C32C;110A 1162 11BF; # (쌬; 쌬; 쌬; 쌬; 쌬; ) HANGUL SYLLABLE SSAEK +C32D;C32D;110A 1162 11C0;C32D;110A 1162 11C0; # (쌭; 쌭; 쌭; 쌭; 쌭; ) HANGUL SYLLABLE SSAET +C32E;C32E;110A 1162 11C1;C32E;110A 1162 11C1; # (쌮; 쌮; 쌮; 쌮; 쌮; ) HANGUL SYLLABLE SSAEP +C32F;C32F;110A 1162 11C2;C32F;110A 1162 11C2; # (쌯; 쌯; 쌯; 쌯; 쌯; ) HANGUL SYLLABLE SSAEH +C330;C330;110A 1163;C330;110A 1163; # (쌰; 쌰; 쌰; 쌰; 쌰; ) HANGUL SYLLABLE SSYA +C331;C331;110A 1163 11A8;C331;110A 1163 11A8; # (쌱; 쌱; 쌱; 쌱; 쌱; ) HANGUL SYLLABLE SSYAG +C332;C332;110A 1163 11A9;C332;110A 1163 11A9; # (쌲; 쌲; 쌲; 쌲; 쌲; ) HANGUL SYLLABLE SSYAGG +C333;C333;110A 1163 11AA;C333;110A 1163 11AA; # (쌳; 쌳; 쌳; 쌳; 쌳; ) HANGUL SYLLABLE SSYAGS +C334;C334;110A 1163 11AB;C334;110A 1163 11AB; # (쌴; 쌴; 쌴; 쌴; 쌴; ) HANGUL SYLLABLE SSYAN +C335;C335;110A 1163 11AC;C335;110A 1163 11AC; # (쌵; 쌵; 쌵; 쌵; 쌵; ) HANGUL SYLLABLE SSYANJ +C336;C336;110A 1163 11AD;C336;110A 1163 11AD; # (쌶; 쌶; 쌶; 쌶; 쌶; ) HANGUL SYLLABLE SSYANH +C337;C337;110A 1163 11AE;C337;110A 1163 11AE; # (쌷; 쌷; 쌷; 쌷; 쌷; ) HANGUL SYLLABLE SSYAD +C338;C338;110A 1163 11AF;C338;110A 1163 11AF; # (쌸; 쌸; 쌸; 쌸; 쌸; ) HANGUL SYLLABLE SSYAL +C339;C339;110A 1163 11B0;C339;110A 1163 11B0; # (쌹; 쌹; 쌹; 쌹; 쌹; ) HANGUL SYLLABLE SSYALG +C33A;C33A;110A 1163 11B1;C33A;110A 1163 11B1; # (쌺; 쌺; 쌺; 쌺; 쌺; ) HANGUL SYLLABLE SSYALM +C33B;C33B;110A 1163 11B2;C33B;110A 1163 11B2; # (쌻; 쌻; 쌻; 쌻; 쌻; ) HANGUL SYLLABLE SSYALB +C33C;C33C;110A 1163 11B3;C33C;110A 1163 11B3; # (쌼; 쌼; 쌼; 쌼; 쌼; ) HANGUL SYLLABLE SSYALS +C33D;C33D;110A 1163 11B4;C33D;110A 1163 11B4; # (쌽; 쌽; 쌽; 쌽; 쌽; ) HANGUL SYLLABLE SSYALT +C33E;C33E;110A 1163 11B5;C33E;110A 1163 11B5; # (쌾; 쌾; 쌾; 쌾; 쌾; ) HANGUL SYLLABLE SSYALP +C33F;C33F;110A 1163 11B6;C33F;110A 1163 11B6; # (쌿; 쌿; 쌿; 쌿; 쌿; ) HANGUL SYLLABLE SSYALH +C340;C340;110A 1163 11B7;C340;110A 1163 11B7; # (썀; 썀; 썀; 썀; 썀; ) HANGUL SYLLABLE SSYAM +C341;C341;110A 1163 11B8;C341;110A 1163 11B8; # (썁; 썁; 썁; 썁; 썁; ) HANGUL SYLLABLE SSYAB +C342;C342;110A 1163 11B9;C342;110A 1163 11B9; # (썂; 썂; 썂; 썂; 썂; ) HANGUL SYLLABLE SSYABS +C343;C343;110A 1163 11BA;C343;110A 1163 11BA; # (썃; 썃; 썃; 썃; 썃; ) HANGUL SYLLABLE SSYAS +C344;C344;110A 1163 11BB;C344;110A 1163 11BB; # (썄; 썄; 썄; 썄; 썄; ) HANGUL SYLLABLE SSYASS +C345;C345;110A 1163 11BC;C345;110A 1163 11BC; # (썅; 썅; 썅; 썅; 썅; ) HANGUL SYLLABLE SSYANG +C346;C346;110A 1163 11BD;C346;110A 1163 11BD; # (썆; 썆; 썆; 썆; 썆; ) HANGUL SYLLABLE SSYAJ +C347;C347;110A 1163 11BE;C347;110A 1163 11BE; # (썇; 썇; 썇; 썇; 썇; ) HANGUL SYLLABLE SSYAC +C348;C348;110A 1163 11BF;C348;110A 1163 11BF; # (썈; 썈; 썈; 썈; 썈; ) HANGUL SYLLABLE SSYAK +C349;C349;110A 1163 11C0;C349;110A 1163 11C0; # (썉; 썉; 썉; 썉; 썉; ) HANGUL SYLLABLE SSYAT +C34A;C34A;110A 1163 11C1;C34A;110A 1163 11C1; # (썊; 썊; 썊; 썊; 썊; ) HANGUL SYLLABLE SSYAP +C34B;C34B;110A 1163 11C2;C34B;110A 1163 11C2; # (썋; 썋; 썋; 썋; 썋; ) HANGUL SYLLABLE SSYAH +C34C;C34C;110A 1164;C34C;110A 1164; # (썌; 썌; 썌; 썌; 썌; ) HANGUL SYLLABLE SSYAE +C34D;C34D;110A 1164 11A8;C34D;110A 1164 11A8; # (썍; 썍; 썍; 썍; 썍; ) HANGUL SYLLABLE SSYAEG +C34E;C34E;110A 1164 11A9;C34E;110A 1164 11A9; # (썎; 썎; 썎; 썎; 썎; ) HANGUL SYLLABLE SSYAEGG +C34F;C34F;110A 1164 11AA;C34F;110A 1164 11AA; # (썏; 썏; 썏; 썏; 썏; ) HANGUL SYLLABLE SSYAEGS +C350;C350;110A 1164 11AB;C350;110A 1164 11AB; # (썐; 썐; 썐; 썐; 썐; ) HANGUL SYLLABLE SSYAEN +C351;C351;110A 1164 11AC;C351;110A 1164 11AC; # (썑; 썑; 썑; 썑; 썑; ) HANGUL SYLLABLE SSYAENJ +C352;C352;110A 1164 11AD;C352;110A 1164 11AD; # (썒; 썒; 썒; 썒; 썒; ) HANGUL SYLLABLE SSYAENH +C353;C353;110A 1164 11AE;C353;110A 1164 11AE; # (썓; 썓; 썓; 썓; 썓; ) HANGUL SYLLABLE SSYAED +C354;C354;110A 1164 11AF;C354;110A 1164 11AF; # (썔; 썔; 썔; 썔; 썔; ) HANGUL SYLLABLE SSYAEL +C355;C355;110A 1164 11B0;C355;110A 1164 11B0; # (썕; 썕; 썕; 썕; 썕; ) HANGUL SYLLABLE SSYAELG +C356;C356;110A 1164 11B1;C356;110A 1164 11B1; # (썖; 썖; 썖; 썖; 썖; ) HANGUL SYLLABLE SSYAELM +C357;C357;110A 1164 11B2;C357;110A 1164 11B2; # (썗; 썗; 썗; 썗; 썗; ) HANGUL SYLLABLE SSYAELB +C358;C358;110A 1164 11B3;C358;110A 1164 11B3; # (썘; 썘; 썘; 썘; 썘; ) HANGUL SYLLABLE SSYAELS +C359;C359;110A 1164 11B4;C359;110A 1164 11B4; # (썙; 썙; 썙; 썙; 썙; ) HANGUL SYLLABLE SSYAELT +C35A;C35A;110A 1164 11B5;C35A;110A 1164 11B5; # (썚; 썚; 썚; 썚; 썚; ) HANGUL SYLLABLE SSYAELP +C35B;C35B;110A 1164 11B6;C35B;110A 1164 11B6; # (썛; 썛; 썛; 썛; 썛; ) HANGUL SYLLABLE SSYAELH +C35C;C35C;110A 1164 11B7;C35C;110A 1164 11B7; # (썜; 썜; 썜; 썜; 썜; ) HANGUL SYLLABLE SSYAEM +C35D;C35D;110A 1164 11B8;C35D;110A 1164 11B8; # (썝; 썝; 썝; 썝; 썝; ) HANGUL SYLLABLE SSYAEB +C35E;C35E;110A 1164 11B9;C35E;110A 1164 11B9; # (썞; 썞; 썞; 썞; 썞; ) HANGUL SYLLABLE SSYAEBS +C35F;C35F;110A 1164 11BA;C35F;110A 1164 11BA; # (썟; 썟; 썟; 썟; 썟; ) HANGUL SYLLABLE SSYAES +C360;C360;110A 1164 11BB;C360;110A 1164 11BB; # (썠; 썠; 썠; 썠; 썠; ) HANGUL SYLLABLE SSYAESS +C361;C361;110A 1164 11BC;C361;110A 1164 11BC; # (썡; 썡; 썡; 썡; 썡; ) HANGUL SYLLABLE SSYAENG +C362;C362;110A 1164 11BD;C362;110A 1164 11BD; # (썢; 썢; 썢; 썢; 썢; ) HANGUL SYLLABLE SSYAEJ +C363;C363;110A 1164 11BE;C363;110A 1164 11BE; # (썣; 썣; 썣; 썣; 썣; ) HANGUL SYLLABLE SSYAEC +C364;C364;110A 1164 11BF;C364;110A 1164 11BF; # (썤; 썤; 썤; 썤; 썤; ) HANGUL SYLLABLE SSYAEK +C365;C365;110A 1164 11C0;C365;110A 1164 11C0; # (썥; 썥; 썥; 썥; 썥; ) HANGUL SYLLABLE SSYAET +C366;C366;110A 1164 11C1;C366;110A 1164 11C1; # (썦; 썦; 썦; 썦; 썦; ) HANGUL SYLLABLE SSYAEP +C367;C367;110A 1164 11C2;C367;110A 1164 11C2; # (썧; 썧; 썧; 썧; 썧; ) HANGUL SYLLABLE SSYAEH +C368;C368;110A 1165;C368;110A 1165; # (써; 써; 써; 써; 써; ) HANGUL SYLLABLE SSEO +C369;C369;110A 1165 11A8;C369;110A 1165 11A8; # (썩; 썩; 썩; 썩; 썩; ) HANGUL SYLLABLE SSEOG +C36A;C36A;110A 1165 11A9;C36A;110A 1165 11A9; # (썪; 썪; 썪; 썪; 썪; ) HANGUL SYLLABLE SSEOGG +C36B;C36B;110A 1165 11AA;C36B;110A 1165 11AA; # (썫; 썫; 썫; 썫; 썫; ) HANGUL SYLLABLE SSEOGS +C36C;C36C;110A 1165 11AB;C36C;110A 1165 11AB; # (썬; 썬; 썬; 썬; 썬; ) HANGUL SYLLABLE SSEON +C36D;C36D;110A 1165 11AC;C36D;110A 1165 11AC; # (썭; 썭; 썭; 썭; 썭; ) HANGUL SYLLABLE SSEONJ +C36E;C36E;110A 1165 11AD;C36E;110A 1165 11AD; # (썮; 썮; 썮; 썮; 썮; ) HANGUL SYLLABLE SSEONH +C36F;C36F;110A 1165 11AE;C36F;110A 1165 11AE; # (썯; 썯; 썯; 썯; 썯; ) HANGUL SYLLABLE SSEOD +C370;C370;110A 1165 11AF;C370;110A 1165 11AF; # (썰; 썰; 썰; 썰; 썰; ) HANGUL SYLLABLE SSEOL +C371;C371;110A 1165 11B0;C371;110A 1165 11B0; # (썱; 썱; 썱; 썱; 썱; ) HANGUL SYLLABLE SSEOLG +C372;C372;110A 1165 11B1;C372;110A 1165 11B1; # (썲; 썲; 썲; 썲; 썲; ) HANGUL SYLLABLE SSEOLM +C373;C373;110A 1165 11B2;C373;110A 1165 11B2; # (썳; 썳; 썳; 썳; 썳; ) HANGUL SYLLABLE SSEOLB +C374;C374;110A 1165 11B3;C374;110A 1165 11B3; # (썴; 썴; 썴; 썴; 썴; ) HANGUL SYLLABLE SSEOLS +C375;C375;110A 1165 11B4;C375;110A 1165 11B4; # (썵; 썵; 썵; 썵; 썵; ) HANGUL SYLLABLE SSEOLT +C376;C376;110A 1165 11B5;C376;110A 1165 11B5; # (썶; 썶; 썶; 썶; 썶; ) HANGUL SYLLABLE SSEOLP +C377;C377;110A 1165 11B6;C377;110A 1165 11B6; # (썷; 썷; 썷; 썷; 썷; ) HANGUL SYLLABLE SSEOLH +C378;C378;110A 1165 11B7;C378;110A 1165 11B7; # (썸; 썸; 썸; 썸; 썸; ) HANGUL SYLLABLE SSEOM +C379;C379;110A 1165 11B8;C379;110A 1165 11B8; # (썹; 썹; 썹; 썹; 썹; ) HANGUL SYLLABLE SSEOB +C37A;C37A;110A 1165 11B9;C37A;110A 1165 11B9; # (썺; 썺; 썺; 썺; 썺; ) HANGUL SYLLABLE SSEOBS +C37B;C37B;110A 1165 11BA;C37B;110A 1165 11BA; # (썻; 썻; 썻; 썻; 썻; ) HANGUL SYLLABLE SSEOS +C37C;C37C;110A 1165 11BB;C37C;110A 1165 11BB; # (썼; 썼; 썼; 썼; 썼; ) HANGUL SYLLABLE SSEOSS +C37D;C37D;110A 1165 11BC;C37D;110A 1165 11BC; # (썽; 썽; 썽; 썽; 썽; ) HANGUL SYLLABLE SSEONG +C37E;C37E;110A 1165 11BD;C37E;110A 1165 11BD; # (썾; 썾; 썾; 썾; 썾; ) HANGUL SYLLABLE SSEOJ +C37F;C37F;110A 1165 11BE;C37F;110A 1165 11BE; # (썿; 썿; 썿; 썿; 썿; ) HANGUL SYLLABLE SSEOC +C380;C380;110A 1165 11BF;C380;110A 1165 11BF; # (쎀; 쎀; 쎀; 쎀; 쎀; ) HANGUL SYLLABLE SSEOK +C381;C381;110A 1165 11C0;C381;110A 1165 11C0; # (쎁; 쎁; 쎁; 쎁; 쎁; ) HANGUL SYLLABLE SSEOT +C382;C382;110A 1165 11C1;C382;110A 1165 11C1; # (쎂; 쎂; 쎂; 쎂; 쎂; ) HANGUL SYLLABLE SSEOP +C383;C383;110A 1165 11C2;C383;110A 1165 11C2; # (쎃; 쎃; 쎃; 쎃; 쎃; ) HANGUL SYLLABLE SSEOH +C384;C384;110A 1166;C384;110A 1166; # (쎄; 쎄; 쎄; 쎄; 쎄; ) HANGUL SYLLABLE SSE +C385;C385;110A 1166 11A8;C385;110A 1166 11A8; # (쎅; 쎅; 쎅; 쎅; 쎅; ) HANGUL SYLLABLE SSEG +C386;C386;110A 1166 11A9;C386;110A 1166 11A9; # (쎆; 쎆; 쎆; 쎆; 쎆; ) HANGUL SYLLABLE SSEGG +C387;C387;110A 1166 11AA;C387;110A 1166 11AA; # (쎇; 쎇; 쎇; 쎇; 쎇; ) HANGUL SYLLABLE SSEGS +C388;C388;110A 1166 11AB;C388;110A 1166 11AB; # (쎈; 쎈; 쎈; 쎈; 쎈; ) HANGUL SYLLABLE SSEN +C389;C389;110A 1166 11AC;C389;110A 1166 11AC; # (쎉; 쎉; 쎉; 쎉; 쎉; ) HANGUL SYLLABLE SSENJ +C38A;C38A;110A 1166 11AD;C38A;110A 1166 11AD; # (쎊; 쎊; 쎊; 쎊; 쎊; ) HANGUL SYLLABLE SSENH +C38B;C38B;110A 1166 11AE;C38B;110A 1166 11AE; # (쎋; 쎋; 쎋; 쎋; 쎋; ) HANGUL SYLLABLE SSED +C38C;C38C;110A 1166 11AF;C38C;110A 1166 11AF; # (쎌; 쎌; 쎌; 쎌; 쎌; ) HANGUL SYLLABLE SSEL +C38D;C38D;110A 1166 11B0;C38D;110A 1166 11B0; # (쎍; 쎍; 쎍; 쎍; 쎍; ) HANGUL SYLLABLE SSELG +C38E;C38E;110A 1166 11B1;C38E;110A 1166 11B1; # (쎎; 쎎; 쎎; 쎎; 쎎; ) HANGUL SYLLABLE SSELM +C38F;C38F;110A 1166 11B2;C38F;110A 1166 11B2; # (쎏; 쎏; 쎏; 쎏; 쎏; ) HANGUL SYLLABLE SSELB +C390;C390;110A 1166 11B3;C390;110A 1166 11B3; # (쎐; 쎐; 쎐; 쎐; 쎐; ) HANGUL SYLLABLE SSELS +C391;C391;110A 1166 11B4;C391;110A 1166 11B4; # (쎑; 쎑; 쎑; 쎑; 쎑; ) HANGUL SYLLABLE SSELT +C392;C392;110A 1166 11B5;C392;110A 1166 11B5; # (쎒; 쎒; 쎒; 쎒; 쎒; ) HANGUL SYLLABLE SSELP +C393;C393;110A 1166 11B6;C393;110A 1166 11B6; # (쎓; 쎓; 쎓; 쎓; 쎓; ) HANGUL SYLLABLE SSELH +C394;C394;110A 1166 11B7;C394;110A 1166 11B7; # (쎔; 쎔; 쎔; 쎔; 쎔; ) HANGUL SYLLABLE SSEM +C395;C395;110A 1166 11B8;C395;110A 1166 11B8; # (쎕; 쎕; 쎕; 쎕; 쎕; ) HANGUL SYLLABLE SSEB +C396;C396;110A 1166 11B9;C396;110A 1166 11B9; # (쎖; 쎖; 쎖; 쎖; 쎖; ) HANGUL SYLLABLE SSEBS +C397;C397;110A 1166 11BA;C397;110A 1166 11BA; # (쎗; 쎗; 쎗; 쎗; 쎗; ) HANGUL SYLLABLE SSES +C398;C398;110A 1166 11BB;C398;110A 1166 11BB; # (쎘; 쎘; 쎘; 쎘; 쎘; ) HANGUL SYLLABLE SSESS +C399;C399;110A 1166 11BC;C399;110A 1166 11BC; # (쎙; 쎙; 쎙; 쎙; 쎙; ) HANGUL SYLLABLE SSENG +C39A;C39A;110A 1166 11BD;C39A;110A 1166 11BD; # (쎚; 쎚; 쎚; 쎚; 쎚; ) HANGUL SYLLABLE SSEJ +C39B;C39B;110A 1166 11BE;C39B;110A 1166 11BE; # (쎛; 쎛; 쎛; 쎛; 쎛; ) HANGUL SYLLABLE SSEC +C39C;C39C;110A 1166 11BF;C39C;110A 1166 11BF; # (쎜; 쎜; 쎜; 쎜; 쎜; ) HANGUL SYLLABLE SSEK +C39D;C39D;110A 1166 11C0;C39D;110A 1166 11C0; # (쎝; 쎝; 쎝; 쎝; 쎝; ) HANGUL SYLLABLE SSET +C39E;C39E;110A 1166 11C1;C39E;110A 1166 11C1; # (쎞; 쎞; 쎞; 쎞; 쎞; ) HANGUL SYLLABLE SSEP +C39F;C39F;110A 1166 11C2;C39F;110A 1166 11C2; # (쎟; 쎟; 쎟; 쎟; 쎟; ) HANGUL SYLLABLE SSEH +C3A0;C3A0;110A 1167;C3A0;110A 1167; # (쎠; 쎠; 쎠; 쎠; 쎠; ) HANGUL SYLLABLE SSYEO +C3A1;C3A1;110A 1167 11A8;C3A1;110A 1167 11A8; # (쎡; 쎡; 쎡; 쎡; 쎡; ) HANGUL SYLLABLE SSYEOG +C3A2;C3A2;110A 1167 11A9;C3A2;110A 1167 11A9; # (쎢; 쎢; 쎢; 쎢; 쎢; ) HANGUL SYLLABLE SSYEOGG +C3A3;C3A3;110A 1167 11AA;C3A3;110A 1167 11AA; # (쎣; 쎣; 쎣; 쎣; 쎣; ) HANGUL SYLLABLE SSYEOGS +C3A4;C3A4;110A 1167 11AB;C3A4;110A 1167 11AB; # (쎤; 쎤; 쎤; 쎤; 쎤; ) HANGUL SYLLABLE SSYEON +C3A5;C3A5;110A 1167 11AC;C3A5;110A 1167 11AC; # (쎥; 쎥; 쎥; 쎥; 쎥; ) HANGUL SYLLABLE SSYEONJ +C3A6;C3A6;110A 1167 11AD;C3A6;110A 1167 11AD; # (쎦; 쎦; 쎦; 쎦; 쎦; ) HANGUL SYLLABLE SSYEONH +C3A7;C3A7;110A 1167 11AE;C3A7;110A 1167 11AE; # (쎧; 쎧; 쎧; 쎧; 쎧; ) HANGUL SYLLABLE SSYEOD +C3A8;C3A8;110A 1167 11AF;C3A8;110A 1167 11AF; # (쎨; 쎨; 쎨; 쎨; 쎨; ) HANGUL SYLLABLE SSYEOL +C3A9;C3A9;110A 1167 11B0;C3A9;110A 1167 11B0; # (쎩; 쎩; 쎩; 쎩; 쎩; ) HANGUL SYLLABLE SSYEOLG +C3AA;C3AA;110A 1167 11B1;C3AA;110A 1167 11B1; # (쎪; 쎪; 쎪; 쎪; 쎪; ) HANGUL SYLLABLE SSYEOLM +C3AB;C3AB;110A 1167 11B2;C3AB;110A 1167 11B2; # (쎫; 쎫; 쎫; 쎫; 쎫; ) HANGUL SYLLABLE SSYEOLB +C3AC;C3AC;110A 1167 11B3;C3AC;110A 1167 11B3; # (쎬; 쎬; 쎬; 쎬; 쎬; ) HANGUL SYLLABLE SSYEOLS +C3AD;C3AD;110A 1167 11B4;C3AD;110A 1167 11B4; # (쎭; 쎭; 쎭; 쎭; 쎭; ) HANGUL SYLLABLE SSYEOLT +C3AE;C3AE;110A 1167 11B5;C3AE;110A 1167 11B5; # (쎮; 쎮; 쎮; 쎮; 쎮; ) HANGUL SYLLABLE SSYEOLP +C3AF;C3AF;110A 1167 11B6;C3AF;110A 1167 11B6; # (쎯; 쎯; 쎯; 쎯; 쎯; ) HANGUL SYLLABLE SSYEOLH +C3B0;C3B0;110A 1167 11B7;C3B0;110A 1167 11B7; # (쎰; 쎰; 쎰; 쎰; 쎰; ) HANGUL SYLLABLE SSYEOM +C3B1;C3B1;110A 1167 11B8;C3B1;110A 1167 11B8; # (쎱; 쎱; 쎱; 쎱; 쎱; ) HANGUL SYLLABLE SSYEOB +C3B2;C3B2;110A 1167 11B9;C3B2;110A 1167 11B9; # (쎲; 쎲; 쎲; 쎲; 쎲; ) HANGUL SYLLABLE SSYEOBS +C3B3;C3B3;110A 1167 11BA;C3B3;110A 1167 11BA; # (쎳; 쎳; 쎳; 쎳; 쎳; ) HANGUL SYLLABLE SSYEOS +C3B4;C3B4;110A 1167 11BB;C3B4;110A 1167 11BB; # (쎴; 쎴; 쎴; 쎴; 쎴; ) HANGUL SYLLABLE SSYEOSS +C3B5;C3B5;110A 1167 11BC;C3B5;110A 1167 11BC; # (쎵; 쎵; 쎵; 쎵; 쎵; ) HANGUL SYLLABLE SSYEONG +C3B6;C3B6;110A 1167 11BD;C3B6;110A 1167 11BD; # (쎶; 쎶; 쎶; 쎶; 쎶; ) HANGUL SYLLABLE SSYEOJ +C3B7;C3B7;110A 1167 11BE;C3B7;110A 1167 11BE; # (쎷; 쎷; 쎷; 쎷; 쎷; ) HANGUL SYLLABLE SSYEOC +C3B8;C3B8;110A 1167 11BF;C3B8;110A 1167 11BF; # (쎸; 쎸; 쎸; 쎸; 쎸; ) HANGUL SYLLABLE SSYEOK +C3B9;C3B9;110A 1167 11C0;C3B9;110A 1167 11C0; # (쎹; 쎹; 쎹; 쎹; 쎹; ) HANGUL SYLLABLE SSYEOT +C3BA;C3BA;110A 1167 11C1;C3BA;110A 1167 11C1; # (쎺; 쎺; 쎺; 쎺; 쎺; ) HANGUL SYLLABLE SSYEOP +C3BB;C3BB;110A 1167 11C2;C3BB;110A 1167 11C2; # (쎻; 쎻; 쎻; 쎻; 쎻; ) HANGUL SYLLABLE SSYEOH +C3BC;C3BC;110A 1168;C3BC;110A 1168; # (쎼; 쎼; 쎼; 쎼; 쎼; ) HANGUL SYLLABLE SSYE +C3BD;C3BD;110A 1168 11A8;C3BD;110A 1168 11A8; # (쎽; 쎽; 쎽; 쎽; 쎽; ) HANGUL SYLLABLE SSYEG +C3BE;C3BE;110A 1168 11A9;C3BE;110A 1168 11A9; # (쎾; 쎾; 쎾; 쎾; 쎾; ) HANGUL SYLLABLE SSYEGG +C3BF;C3BF;110A 1168 11AA;C3BF;110A 1168 11AA; # (쎿; 쎿; 쎿; 쎿; 쎿; ) HANGUL SYLLABLE SSYEGS +C3C0;C3C0;110A 1168 11AB;C3C0;110A 1168 11AB; # (쏀; 쏀; 쏀; 쏀; 쏀; ) HANGUL SYLLABLE SSYEN +C3C1;C3C1;110A 1168 11AC;C3C1;110A 1168 11AC; # (쏁; 쏁; 쏁; 쏁; 쏁; ) HANGUL SYLLABLE SSYENJ +C3C2;C3C2;110A 1168 11AD;C3C2;110A 1168 11AD; # (쏂; 쏂; 쏂; 쏂; 쏂; ) HANGUL SYLLABLE SSYENH +C3C3;C3C3;110A 1168 11AE;C3C3;110A 1168 11AE; # (쏃; 쏃; 쏃; 쏃; 쏃; ) HANGUL SYLLABLE SSYED +C3C4;C3C4;110A 1168 11AF;C3C4;110A 1168 11AF; # (쏄; 쏄; 쏄; 쏄; 쏄; ) HANGUL SYLLABLE SSYEL +C3C5;C3C5;110A 1168 11B0;C3C5;110A 1168 11B0; # (쏅; 쏅; 쏅; 쏅; 쏅; ) HANGUL SYLLABLE SSYELG +C3C6;C3C6;110A 1168 11B1;C3C6;110A 1168 11B1; # (쏆; 쏆; 쏆; 쏆; 쏆; ) HANGUL SYLLABLE SSYELM +C3C7;C3C7;110A 1168 11B2;C3C7;110A 1168 11B2; # (쏇; 쏇; 쏇; 쏇; 쏇; ) HANGUL SYLLABLE SSYELB +C3C8;C3C8;110A 1168 11B3;C3C8;110A 1168 11B3; # (쏈; 쏈; 쏈; 쏈; 쏈; ) HANGUL SYLLABLE SSYELS +C3C9;C3C9;110A 1168 11B4;C3C9;110A 1168 11B4; # (쏉; 쏉; 쏉; 쏉; 쏉; ) HANGUL SYLLABLE SSYELT +C3CA;C3CA;110A 1168 11B5;C3CA;110A 1168 11B5; # (쏊; 쏊; 쏊; 쏊; 쏊; ) HANGUL SYLLABLE SSYELP +C3CB;C3CB;110A 1168 11B6;C3CB;110A 1168 11B6; # (쏋; 쏋; 쏋; 쏋; 쏋; ) HANGUL SYLLABLE SSYELH +C3CC;C3CC;110A 1168 11B7;C3CC;110A 1168 11B7; # (쏌; 쏌; 쏌; 쏌; 쏌; ) HANGUL SYLLABLE SSYEM +C3CD;C3CD;110A 1168 11B8;C3CD;110A 1168 11B8; # (쏍; 쏍; 쏍; 쏍; 쏍; ) HANGUL SYLLABLE SSYEB +C3CE;C3CE;110A 1168 11B9;C3CE;110A 1168 11B9; # (쏎; 쏎; 쏎; 쏎; 쏎; ) HANGUL SYLLABLE SSYEBS +C3CF;C3CF;110A 1168 11BA;C3CF;110A 1168 11BA; # (쏏; 쏏; 쏏; 쏏; 쏏; ) HANGUL SYLLABLE SSYES +C3D0;C3D0;110A 1168 11BB;C3D0;110A 1168 11BB; # (쏐; 쏐; 쏐; 쏐; 쏐; ) HANGUL SYLLABLE SSYESS +C3D1;C3D1;110A 1168 11BC;C3D1;110A 1168 11BC; # (쏑; 쏑; 쏑; 쏑; 쏑; ) HANGUL SYLLABLE SSYENG +C3D2;C3D2;110A 1168 11BD;C3D2;110A 1168 11BD; # (쏒; 쏒; 쏒; 쏒; 쏒; ) HANGUL SYLLABLE SSYEJ +C3D3;C3D3;110A 1168 11BE;C3D3;110A 1168 11BE; # (쏓; 쏓; 쏓; 쏓; 쏓; ) HANGUL SYLLABLE SSYEC +C3D4;C3D4;110A 1168 11BF;C3D4;110A 1168 11BF; # (쏔; 쏔; 쏔; 쏔; 쏔; ) HANGUL SYLLABLE SSYEK +C3D5;C3D5;110A 1168 11C0;C3D5;110A 1168 11C0; # (쏕; 쏕; 쏕; 쏕; 쏕; ) HANGUL SYLLABLE SSYET +C3D6;C3D6;110A 1168 11C1;C3D6;110A 1168 11C1; # (쏖; 쏖; 쏖; 쏖; 쏖; ) HANGUL SYLLABLE SSYEP +C3D7;C3D7;110A 1168 11C2;C3D7;110A 1168 11C2; # (쏗; 쏗; 쏗; 쏗; 쏗; ) HANGUL SYLLABLE SSYEH +C3D8;C3D8;110A 1169;C3D8;110A 1169; # (쏘; 쏘; 쏘; 쏘; 쏘; ) HANGUL SYLLABLE SSO +C3D9;C3D9;110A 1169 11A8;C3D9;110A 1169 11A8; # (쏙; 쏙; 쏙; 쏙; 쏙; ) HANGUL SYLLABLE SSOG +C3DA;C3DA;110A 1169 11A9;C3DA;110A 1169 11A9; # (쏚; 쏚; 쏚; 쏚; 쏚; ) HANGUL SYLLABLE SSOGG +C3DB;C3DB;110A 1169 11AA;C3DB;110A 1169 11AA; # (쏛; 쏛; 쏛; 쏛; 쏛; ) HANGUL SYLLABLE SSOGS +C3DC;C3DC;110A 1169 11AB;C3DC;110A 1169 11AB; # (쏜; 쏜; 쏜; 쏜; 쏜; ) HANGUL SYLLABLE SSON +C3DD;C3DD;110A 1169 11AC;C3DD;110A 1169 11AC; # (쏝; 쏝; 쏝; 쏝; 쏝; ) HANGUL SYLLABLE SSONJ +C3DE;C3DE;110A 1169 11AD;C3DE;110A 1169 11AD; # (쏞; 쏞; 쏞; 쏞; 쏞; ) HANGUL SYLLABLE SSONH +C3DF;C3DF;110A 1169 11AE;C3DF;110A 1169 11AE; # (쏟; 쏟; 쏟; 쏟; 쏟; ) HANGUL SYLLABLE SSOD +C3E0;C3E0;110A 1169 11AF;C3E0;110A 1169 11AF; # (쏠; 쏠; 쏠; 쏠; 쏠; ) HANGUL SYLLABLE SSOL +C3E1;C3E1;110A 1169 11B0;C3E1;110A 1169 11B0; # (쏡; 쏡; 쏡; 쏡; 쏡; ) HANGUL SYLLABLE SSOLG +C3E2;C3E2;110A 1169 11B1;C3E2;110A 1169 11B1; # (쏢; 쏢; 쏢; 쏢; 쏢; ) HANGUL SYLLABLE SSOLM +C3E3;C3E3;110A 1169 11B2;C3E3;110A 1169 11B2; # (쏣; 쏣; 쏣; 쏣; 쏣; ) HANGUL SYLLABLE SSOLB +C3E4;C3E4;110A 1169 11B3;C3E4;110A 1169 11B3; # (쏤; 쏤; 쏤; 쏤; 쏤; ) HANGUL SYLLABLE SSOLS +C3E5;C3E5;110A 1169 11B4;C3E5;110A 1169 11B4; # (쏥; 쏥; 쏥; 쏥; 쏥; ) HANGUL SYLLABLE SSOLT +C3E6;C3E6;110A 1169 11B5;C3E6;110A 1169 11B5; # (쏦; 쏦; 쏦; 쏦; 쏦; ) HANGUL SYLLABLE SSOLP +C3E7;C3E7;110A 1169 11B6;C3E7;110A 1169 11B6; # (쏧; 쏧; 쏧; 쏧; 쏧; ) HANGUL SYLLABLE SSOLH +C3E8;C3E8;110A 1169 11B7;C3E8;110A 1169 11B7; # (쏨; 쏨; 쏨; 쏨; 쏨; ) HANGUL SYLLABLE SSOM +C3E9;C3E9;110A 1169 11B8;C3E9;110A 1169 11B8; # (쏩; 쏩; 쏩; 쏩; 쏩; ) HANGUL SYLLABLE SSOB +C3EA;C3EA;110A 1169 11B9;C3EA;110A 1169 11B9; # (쏪; 쏪; 쏪; 쏪; 쏪; ) HANGUL SYLLABLE SSOBS +C3EB;C3EB;110A 1169 11BA;C3EB;110A 1169 11BA; # (쏫; 쏫; 쏫; 쏫; 쏫; ) HANGUL SYLLABLE SSOS +C3EC;C3EC;110A 1169 11BB;C3EC;110A 1169 11BB; # (쏬; 쏬; 쏬; 쏬; 쏬; ) HANGUL SYLLABLE SSOSS +C3ED;C3ED;110A 1169 11BC;C3ED;110A 1169 11BC; # (쏭; 쏭; 쏭; 쏭; 쏭; ) HANGUL SYLLABLE SSONG +C3EE;C3EE;110A 1169 11BD;C3EE;110A 1169 11BD; # (쏮; 쏮; 쏮; 쏮; 쏮; ) HANGUL SYLLABLE SSOJ +C3EF;C3EF;110A 1169 11BE;C3EF;110A 1169 11BE; # (쏯; 쏯; 쏯; 쏯; 쏯; ) HANGUL SYLLABLE SSOC +C3F0;C3F0;110A 1169 11BF;C3F0;110A 1169 11BF; # (쏰; 쏰; 쏰; 쏰; 쏰; ) HANGUL SYLLABLE SSOK +C3F1;C3F1;110A 1169 11C0;C3F1;110A 1169 11C0; # (쏱; 쏱; 쏱; 쏱; 쏱; ) HANGUL SYLLABLE SSOT +C3F2;C3F2;110A 1169 11C1;C3F2;110A 1169 11C1; # (쏲; 쏲; 쏲; 쏲; 쏲; ) HANGUL SYLLABLE SSOP +C3F3;C3F3;110A 1169 11C2;C3F3;110A 1169 11C2; # (쏳; 쏳; 쏳; 쏳; 쏳; ) HANGUL SYLLABLE SSOH +C3F4;C3F4;110A 116A;C3F4;110A 116A; # (쏴; 쏴; 쏴; 쏴; 쏴; ) HANGUL SYLLABLE SSWA +C3F5;C3F5;110A 116A 11A8;C3F5;110A 116A 11A8; # (쏵; 쏵; 쏵; 쏵; 쏵; ) HANGUL SYLLABLE SSWAG +C3F6;C3F6;110A 116A 11A9;C3F6;110A 116A 11A9; # (쏶; 쏶; 쏶; 쏶; 쏶; ) HANGUL SYLLABLE SSWAGG +C3F7;C3F7;110A 116A 11AA;C3F7;110A 116A 11AA; # (쏷; 쏷; 쏷; 쏷; 쏷; ) HANGUL SYLLABLE SSWAGS +C3F8;C3F8;110A 116A 11AB;C3F8;110A 116A 11AB; # (쏸; 쏸; 쏸; 쏸; 쏸; ) HANGUL SYLLABLE SSWAN +C3F9;C3F9;110A 116A 11AC;C3F9;110A 116A 11AC; # (쏹; 쏹; 쏹; 쏹; 쏹; ) HANGUL SYLLABLE SSWANJ +C3FA;C3FA;110A 116A 11AD;C3FA;110A 116A 11AD; # (쏺; 쏺; 쏺; 쏺; 쏺; ) HANGUL SYLLABLE SSWANH +C3FB;C3FB;110A 116A 11AE;C3FB;110A 116A 11AE; # (쏻; 쏻; 쏻; 쏻; 쏻; ) HANGUL SYLLABLE SSWAD +C3FC;C3FC;110A 116A 11AF;C3FC;110A 116A 11AF; # (쏼; 쏼; 쏼; 쏼; 쏼; ) HANGUL SYLLABLE SSWAL +C3FD;C3FD;110A 116A 11B0;C3FD;110A 116A 11B0; # (쏽; 쏽; 쏽; 쏽; 쏽; ) HANGUL SYLLABLE SSWALG +C3FE;C3FE;110A 116A 11B1;C3FE;110A 116A 11B1; # (쏾; 쏾; 쏾; 쏾; 쏾; ) HANGUL SYLLABLE SSWALM +C3FF;C3FF;110A 116A 11B2;C3FF;110A 116A 11B2; # (쏿; 쏿; 쏿; 쏿; 쏿; ) HANGUL SYLLABLE SSWALB +C400;C400;110A 116A 11B3;C400;110A 116A 11B3; # (쐀; 쐀; 쐀; 쐀; 쐀; ) HANGUL SYLLABLE SSWALS +C401;C401;110A 116A 11B4;C401;110A 116A 11B4; # (쐁; 쐁; 쐁; 쐁; 쐁; ) HANGUL SYLLABLE SSWALT +C402;C402;110A 116A 11B5;C402;110A 116A 11B5; # (쐂; 쐂; 쐂; 쐂; 쐂; ) HANGUL SYLLABLE SSWALP +C403;C403;110A 116A 11B6;C403;110A 116A 11B6; # (쐃; 쐃; 쐃; 쐃; 쐃; ) HANGUL SYLLABLE SSWALH +C404;C404;110A 116A 11B7;C404;110A 116A 11B7; # (쐄; 쐄; 쐄; 쐄; 쐄; ) HANGUL SYLLABLE SSWAM +C405;C405;110A 116A 11B8;C405;110A 116A 11B8; # (쐅; 쐅; 쐅; 쐅; 쐅; ) HANGUL SYLLABLE SSWAB +C406;C406;110A 116A 11B9;C406;110A 116A 11B9; # (쐆; 쐆; 쐆; 쐆; 쐆; ) HANGUL SYLLABLE SSWABS +C407;C407;110A 116A 11BA;C407;110A 116A 11BA; # (쐇; 쐇; 쐇; 쐇; 쐇; ) HANGUL SYLLABLE SSWAS +C408;C408;110A 116A 11BB;C408;110A 116A 11BB; # (쐈; 쐈; 쐈; 쐈; 쐈; ) HANGUL SYLLABLE SSWASS +C409;C409;110A 116A 11BC;C409;110A 116A 11BC; # (쐉; 쐉; 쐉; 쐉; 쐉; ) HANGUL SYLLABLE SSWANG +C40A;C40A;110A 116A 11BD;C40A;110A 116A 11BD; # (쐊; 쐊; 쐊; 쐊; 쐊; ) HANGUL SYLLABLE SSWAJ +C40B;C40B;110A 116A 11BE;C40B;110A 116A 11BE; # (쐋; 쐋; 쐋; 쐋; 쐋; ) HANGUL SYLLABLE SSWAC +C40C;C40C;110A 116A 11BF;C40C;110A 116A 11BF; # (쐌; 쐌; 쐌; 쐌; 쐌; ) HANGUL SYLLABLE SSWAK +C40D;C40D;110A 116A 11C0;C40D;110A 116A 11C0; # (쐍; 쐍; 쐍; 쐍; 쐍; ) HANGUL SYLLABLE SSWAT +C40E;C40E;110A 116A 11C1;C40E;110A 116A 11C1; # (쐎; 쐎; 쐎; 쐎; 쐎; ) HANGUL SYLLABLE SSWAP +C40F;C40F;110A 116A 11C2;C40F;110A 116A 11C2; # (쐏; 쐏; 쐏; 쐏; 쐏; ) HANGUL SYLLABLE SSWAH +C410;C410;110A 116B;C410;110A 116B; # (쐐; 쐐; 쐐; 쐐; 쐐; ) HANGUL SYLLABLE SSWAE +C411;C411;110A 116B 11A8;C411;110A 116B 11A8; # (쐑; 쐑; 쐑; 쐑; 쐑; ) HANGUL SYLLABLE SSWAEG +C412;C412;110A 116B 11A9;C412;110A 116B 11A9; # (쐒; 쐒; 쐒; 쐒; 쐒; ) HANGUL SYLLABLE SSWAEGG +C413;C413;110A 116B 11AA;C413;110A 116B 11AA; # (쐓; 쐓; 쐓; 쐓; 쐓; ) HANGUL SYLLABLE SSWAEGS +C414;C414;110A 116B 11AB;C414;110A 116B 11AB; # (쐔; 쐔; 쐔; 쐔; 쐔; ) HANGUL SYLLABLE SSWAEN +C415;C415;110A 116B 11AC;C415;110A 116B 11AC; # (쐕; 쐕; 쐕; 쐕; 쐕; ) HANGUL SYLLABLE SSWAENJ +C416;C416;110A 116B 11AD;C416;110A 116B 11AD; # (쐖; 쐖; 쐖; 쐖; 쐖; ) HANGUL SYLLABLE SSWAENH +C417;C417;110A 116B 11AE;C417;110A 116B 11AE; # (쐗; 쐗; 쐗; 쐗; 쐗; ) HANGUL SYLLABLE SSWAED +C418;C418;110A 116B 11AF;C418;110A 116B 11AF; # (쐘; 쐘; 쐘; 쐘; 쐘; ) HANGUL SYLLABLE SSWAEL +C419;C419;110A 116B 11B0;C419;110A 116B 11B0; # (쐙; 쐙; 쐙; 쐙; 쐙; ) HANGUL SYLLABLE SSWAELG +C41A;C41A;110A 116B 11B1;C41A;110A 116B 11B1; # (쐚; 쐚; 쐚; 쐚; 쐚; ) HANGUL SYLLABLE SSWAELM +C41B;C41B;110A 116B 11B2;C41B;110A 116B 11B2; # (쐛; 쐛; 쐛; 쐛; 쐛; ) HANGUL SYLLABLE SSWAELB +C41C;C41C;110A 116B 11B3;C41C;110A 116B 11B3; # (쐜; 쐜; 쐜; 쐜; 쐜; ) HANGUL SYLLABLE SSWAELS +C41D;C41D;110A 116B 11B4;C41D;110A 116B 11B4; # (쐝; 쐝; 쐝; 쐝; 쐝; ) HANGUL SYLLABLE SSWAELT +C41E;C41E;110A 116B 11B5;C41E;110A 116B 11B5; # (쐞; 쐞; 쐞; 쐞; 쐞; ) HANGUL SYLLABLE SSWAELP +C41F;C41F;110A 116B 11B6;C41F;110A 116B 11B6; # (쐟; 쐟; 쐟; 쐟; 쐟; ) HANGUL SYLLABLE SSWAELH +C420;C420;110A 116B 11B7;C420;110A 116B 11B7; # (쐠; 쐠; 쐠; 쐠; 쐠; ) HANGUL SYLLABLE SSWAEM +C421;C421;110A 116B 11B8;C421;110A 116B 11B8; # (쐡; 쐡; 쐡; 쐡; 쐡; ) HANGUL SYLLABLE SSWAEB +C422;C422;110A 116B 11B9;C422;110A 116B 11B9; # (쐢; 쐢; 쐢; 쐢; 쐢; ) HANGUL SYLLABLE SSWAEBS +C423;C423;110A 116B 11BA;C423;110A 116B 11BA; # (쐣; 쐣; 쐣; 쐣; 쐣; ) HANGUL SYLLABLE SSWAES +C424;C424;110A 116B 11BB;C424;110A 116B 11BB; # (쐤; 쐤; 쐤; 쐤; 쐤; ) HANGUL SYLLABLE SSWAESS +C425;C425;110A 116B 11BC;C425;110A 116B 11BC; # (쐥; 쐥; 쐥; 쐥; 쐥; ) HANGUL SYLLABLE SSWAENG +C426;C426;110A 116B 11BD;C426;110A 116B 11BD; # (쐦; 쐦; 쐦; 쐦; 쐦; ) HANGUL SYLLABLE SSWAEJ +C427;C427;110A 116B 11BE;C427;110A 116B 11BE; # (쐧; 쐧; 쐧; 쐧; 쐧; ) HANGUL SYLLABLE SSWAEC +C428;C428;110A 116B 11BF;C428;110A 116B 11BF; # (쐨; 쐨; 쐨; 쐨; 쐨; ) HANGUL SYLLABLE SSWAEK +C429;C429;110A 116B 11C0;C429;110A 116B 11C0; # (쐩; 쐩; 쐩; 쐩; 쐩; ) HANGUL SYLLABLE SSWAET +C42A;C42A;110A 116B 11C1;C42A;110A 116B 11C1; # (쐪; 쐪; 쐪; 쐪; 쐪; ) HANGUL SYLLABLE SSWAEP +C42B;C42B;110A 116B 11C2;C42B;110A 116B 11C2; # (쐫; 쐫; 쐫; 쐫; 쐫; ) HANGUL SYLLABLE SSWAEH +C42C;C42C;110A 116C;C42C;110A 116C; # (쐬; 쐬; 쐬; 쐬; 쐬; ) HANGUL SYLLABLE SSOE +C42D;C42D;110A 116C 11A8;C42D;110A 116C 11A8; # (쐭; 쐭; 쐭; 쐭; 쐭; ) HANGUL SYLLABLE SSOEG +C42E;C42E;110A 116C 11A9;C42E;110A 116C 11A9; # (쐮; 쐮; 쐮; 쐮; 쐮; ) HANGUL SYLLABLE SSOEGG +C42F;C42F;110A 116C 11AA;C42F;110A 116C 11AA; # (쐯; 쐯; 쐯; 쐯; 쐯; ) HANGUL SYLLABLE SSOEGS +C430;C430;110A 116C 11AB;C430;110A 116C 11AB; # (쐰; 쐰; 쐰; 쐰; 쐰; ) HANGUL SYLLABLE SSOEN +C431;C431;110A 116C 11AC;C431;110A 116C 11AC; # (쐱; 쐱; 쐱; 쐱; 쐱; ) HANGUL SYLLABLE SSOENJ +C432;C432;110A 116C 11AD;C432;110A 116C 11AD; # (쐲; 쐲; 쐲; 쐲; 쐲; ) HANGUL SYLLABLE SSOENH +C433;C433;110A 116C 11AE;C433;110A 116C 11AE; # (쐳; 쐳; 쐳; 쐳; 쐳; ) HANGUL SYLLABLE SSOED +C434;C434;110A 116C 11AF;C434;110A 116C 11AF; # (쐴; 쐴; 쐴; 쐴; 쐴; ) HANGUL SYLLABLE SSOEL +C435;C435;110A 116C 11B0;C435;110A 116C 11B0; # (쐵; 쐵; 쐵; 쐵; 쐵; ) HANGUL SYLLABLE SSOELG +C436;C436;110A 116C 11B1;C436;110A 116C 11B1; # (쐶; 쐶; 쐶; 쐶; 쐶; ) HANGUL SYLLABLE SSOELM +C437;C437;110A 116C 11B2;C437;110A 116C 11B2; # (쐷; 쐷; 쐷; 쐷; 쐷; ) HANGUL SYLLABLE SSOELB +C438;C438;110A 116C 11B3;C438;110A 116C 11B3; # (쐸; 쐸; 쐸; 쐸; 쐸; ) HANGUL SYLLABLE SSOELS +C439;C439;110A 116C 11B4;C439;110A 116C 11B4; # (쐹; 쐹; 쐹; 쐹; 쐹; ) HANGUL SYLLABLE SSOELT +C43A;C43A;110A 116C 11B5;C43A;110A 116C 11B5; # (쐺; 쐺; 쐺; 쐺; 쐺; ) HANGUL SYLLABLE SSOELP +C43B;C43B;110A 116C 11B6;C43B;110A 116C 11B6; # (쐻; 쐻; 쐻; 쐻; 쐻; ) HANGUL SYLLABLE SSOELH +C43C;C43C;110A 116C 11B7;C43C;110A 116C 11B7; # (쐼; 쐼; 쐼; 쐼; 쐼; ) HANGUL SYLLABLE SSOEM +C43D;C43D;110A 116C 11B8;C43D;110A 116C 11B8; # (쐽; 쐽; 쐽; 쐽; 쐽; ) HANGUL SYLLABLE SSOEB +C43E;C43E;110A 116C 11B9;C43E;110A 116C 11B9; # (쐾; 쐾; 쐾; 쐾; 쐾; ) HANGUL SYLLABLE SSOEBS +C43F;C43F;110A 116C 11BA;C43F;110A 116C 11BA; # (쐿; 쐿; 쐿; 쐿; 쐿; ) HANGUL SYLLABLE SSOES +C440;C440;110A 116C 11BB;C440;110A 116C 11BB; # (쑀; 쑀; 쑀; 쑀; 쑀; ) HANGUL SYLLABLE SSOESS +C441;C441;110A 116C 11BC;C441;110A 116C 11BC; # (쑁; 쑁; 쑁; 쑁; 쑁; ) HANGUL SYLLABLE SSOENG +C442;C442;110A 116C 11BD;C442;110A 116C 11BD; # (쑂; 쑂; 쑂; 쑂; 쑂; ) HANGUL SYLLABLE SSOEJ +C443;C443;110A 116C 11BE;C443;110A 116C 11BE; # (쑃; 쑃; 쑃; 쑃; 쑃; ) HANGUL SYLLABLE SSOEC +C444;C444;110A 116C 11BF;C444;110A 116C 11BF; # (쑄; 쑄; 쑄; 쑄; 쑄; ) HANGUL SYLLABLE SSOEK +C445;C445;110A 116C 11C0;C445;110A 116C 11C0; # (쑅; 쑅; 쑅; 쑅; 쑅; ) HANGUL SYLLABLE SSOET +C446;C446;110A 116C 11C1;C446;110A 116C 11C1; # (쑆; 쑆; 쑆; 쑆; 쑆; ) HANGUL SYLLABLE SSOEP +C447;C447;110A 116C 11C2;C447;110A 116C 11C2; # (쑇; 쑇; 쑇; 쑇; 쑇; ) HANGUL SYLLABLE SSOEH +C448;C448;110A 116D;C448;110A 116D; # (쑈; 쑈; 쑈; 쑈; 쑈; ) HANGUL SYLLABLE SSYO +C449;C449;110A 116D 11A8;C449;110A 116D 11A8; # (쑉; 쑉; 쑉; 쑉; 쑉; ) HANGUL SYLLABLE SSYOG +C44A;C44A;110A 116D 11A9;C44A;110A 116D 11A9; # (쑊; 쑊; 쑊; 쑊; 쑊; ) HANGUL SYLLABLE SSYOGG +C44B;C44B;110A 116D 11AA;C44B;110A 116D 11AA; # (쑋; 쑋; 쑋; 쑋; 쑋; ) HANGUL SYLLABLE SSYOGS +C44C;C44C;110A 116D 11AB;C44C;110A 116D 11AB; # (쑌; 쑌; 쑌; 쑌; 쑌; ) HANGUL SYLLABLE SSYON +C44D;C44D;110A 116D 11AC;C44D;110A 116D 11AC; # (쑍; 쑍; 쑍; 쑍; 쑍; ) HANGUL SYLLABLE SSYONJ +C44E;C44E;110A 116D 11AD;C44E;110A 116D 11AD; # (쑎; 쑎; 쑎; 쑎; 쑎; ) HANGUL SYLLABLE SSYONH +C44F;C44F;110A 116D 11AE;C44F;110A 116D 11AE; # (쑏; 쑏; 쑏; 쑏; 쑏; ) HANGUL SYLLABLE SSYOD +C450;C450;110A 116D 11AF;C450;110A 116D 11AF; # (쑐; 쑐; 쑐; 쑐; 쑐; ) HANGUL SYLLABLE SSYOL +C451;C451;110A 116D 11B0;C451;110A 116D 11B0; # (쑑; 쑑; 쑑; 쑑; 쑑; ) HANGUL SYLLABLE SSYOLG +C452;C452;110A 116D 11B1;C452;110A 116D 11B1; # (쑒; 쑒; 쑒; 쑒; 쑒; ) HANGUL SYLLABLE SSYOLM +C453;C453;110A 116D 11B2;C453;110A 116D 11B2; # (쑓; 쑓; 쑓; 쑓; 쑓; ) HANGUL SYLLABLE SSYOLB +C454;C454;110A 116D 11B3;C454;110A 116D 11B3; # (쑔; 쑔; 쑔; 쑔; 쑔; ) HANGUL SYLLABLE SSYOLS +C455;C455;110A 116D 11B4;C455;110A 116D 11B4; # (쑕; 쑕; 쑕; 쑕; 쑕; ) HANGUL SYLLABLE SSYOLT +C456;C456;110A 116D 11B5;C456;110A 116D 11B5; # (쑖; 쑖; 쑖; 쑖; 쑖; ) HANGUL SYLLABLE SSYOLP +C457;C457;110A 116D 11B6;C457;110A 116D 11B6; # (쑗; 쑗; 쑗; 쑗; 쑗; ) HANGUL SYLLABLE SSYOLH +C458;C458;110A 116D 11B7;C458;110A 116D 11B7; # (쑘; 쑘; 쑘; 쑘; 쑘; ) HANGUL SYLLABLE SSYOM +C459;C459;110A 116D 11B8;C459;110A 116D 11B8; # (쑙; 쑙; 쑙; 쑙; 쑙; ) HANGUL SYLLABLE SSYOB +C45A;C45A;110A 116D 11B9;C45A;110A 116D 11B9; # (쑚; 쑚; 쑚; 쑚; 쑚; ) HANGUL SYLLABLE SSYOBS +C45B;C45B;110A 116D 11BA;C45B;110A 116D 11BA; # (쑛; 쑛; 쑛; 쑛; 쑛; ) HANGUL SYLLABLE SSYOS +C45C;C45C;110A 116D 11BB;C45C;110A 116D 11BB; # (쑜; 쑜; 쑜; 쑜; 쑜; ) HANGUL SYLLABLE SSYOSS +C45D;C45D;110A 116D 11BC;C45D;110A 116D 11BC; # (쑝; 쑝; 쑝; 쑝; 쑝; ) HANGUL SYLLABLE SSYONG +C45E;C45E;110A 116D 11BD;C45E;110A 116D 11BD; # (쑞; 쑞; 쑞; 쑞; 쑞; ) HANGUL SYLLABLE SSYOJ +C45F;C45F;110A 116D 11BE;C45F;110A 116D 11BE; # (쑟; 쑟; 쑟; 쑟; 쑟; ) HANGUL SYLLABLE SSYOC +C460;C460;110A 116D 11BF;C460;110A 116D 11BF; # (쑠; 쑠; 쑠; 쑠; 쑠; ) HANGUL SYLLABLE SSYOK +C461;C461;110A 116D 11C0;C461;110A 116D 11C0; # (쑡; 쑡; 쑡; 쑡; 쑡; ) HANGUL SYLLABLE SSYOT +C462;C462;110A 116D 11C1;C462;110A 116D 11C1; # (쑢; 쑢; 쑢; 쑢; 쑢; ) HANGUL SYLLABLE SSYOP +C463;C463;110A 116D 11C2;C463;110A 116D 11C2; # (쑣; 쑣; 쑣; 쑣; 쑣; ) HANGUL SYLLABLE SSYOH +C464;C464;110A 116E;C464;110A 116E; # (쑤; 쑤; 쑤; 쑤; 쑤; ) HANGUL SYLLABLE SSU +C465;C465;110A 116E 11A8;C465;110A 116E 11A8; # (쑥; 쑥; 쑥; 쑥; 쑥; ) HANGUL SYLLABLE SSUG +C466;C466;110A 116E 11A9;C466;110A 116E 11A9; # (쑦; 쑦; 쑦; 쑦; 쑦; ) HANGUL SYLLABLE SSUGG +C467;C467;110A 116E 11AA;C467;110A 116E 11AA; # (쑧; 쑧; 쑧; 쑧; 쑧; ) HANGUL SYLLABLE SSUGS +C468;C468;110A 116E 11AB;C468;110A 116E 11AB; # (쑨; 쑨; 쑨; 쑨; 쑨; ) HANGUL SYLLABLE SSUN +C469;C469;110A 116E 11AC;C469;110A 116E 11AC; # (쑩; 쑩; 쑩; 쑩; 쑩; ) HANGUL SYLLABLE SSUNJ +C46A;C46A;110A 116E 11AD;C46A;110A 116E 11AD; # (쑪; 쑪; 쑪; 쑪; 쑪; ) HANGUL SYLLABLE SSUNH +C46B;C46B;110A 116E 11AE;C46B;110A 116E 11AE; # (쑫; 쑫; 쑫; 쑫; 쑫; ) HANGUL SYLLABLE SSUD +C46C;C46C;110A 116E 11AF;C46C;110A 116E 11AF; # (쑬; 쑬; 쑬; 쑬; 쑬; ) HANGUL SYLLABLE SSUL +C46D;C46D;110A 116E 11B0;C46D;110A 116E 11B0; # (쑭; 쑭; 쑭; 쑭; 쑭; ) HANGUL SYLLABLE SSULG +C46E;C46E;110A 116E 11B1;C46E;110A 116E 11B1; # (쑮; 쑮; 쑮; 쑮; 쑮; ) HANGUL SYLLABLE SSULM +C46F;C46F;110A 116E 11B2;C46F;110A 116E 11B2; # (쑯; 쑯; 쑯; 쑯; 쑯; ) HANGUL SYLLABLE SSULB +C470;C470;110A 116E 11B3;C470;110A 116E 11B3; # (쑰; 쑰; 쑰; 쑰; 쑰; ) HANGUL SYLLABLE SSULS +C471;C471;110A 116E 11B4;C471;110A 116E 11B4; # (쑱; 쑱; 쑱; 쑱; 쑱; ) HANGUL SYLLABLE SSULT +C472;C472;110A 116E 11B5;C472;110A 116E 11B5; # (쑲; 쑲; 쑲; 쑲; 쑲; ) HANGUL SYLLABLE SSULP +C473;C473;110A 116E 11B6;C473;110A 116E 11B6; # (쑳; 쑳; 쑳; 쑳; 쑳; ) HANGUL SYLLABLE SSULH +C474;C474;110A 116E 11B7;C474;110A 116E 11B7; # (쑴; 쑴; 쑴; 쑴; 쑴; ) HANGUL SYLLABLE SSUM +C475;C475;110A 116E 11B8;C475;110A 116E 11B8; # (쑵; 쑵; 쑵; 쑵; 쑵; ) HANGUL SYLLABLE SSUB +C476;C476;110A 116E 11B9;C476;110A 116E 11B9; # (쑶; 쑶; 쑶; 쑶; 쑶; ) HANGUL SYLLABLE SSUBS +C477;C477;110A 116E 11BA;C477;110A 116E 11BA; # (쑷; 쑷; 쑷; 쑷; 쑷; ) HANGUL SYLLABLE SSUS +C478;C478;110A 116E 11BB;C478;110A 116E 11BB; # (쑸; 쑸; 쑸; 쑸; 쑸; ) HANGUL SYLLABLE SSUSS +C479;C479;110A 116E 11BC;C479;110A 116E 11BC; # (쑹; 쑹; 쑹; 쑹; 쑹; ) HANGUL SYLLABLE SSUNG +C47A;C47A;110A 116E 11BD;C47A;110A 116E 11BD; # (쑺; 쑺; 쑺; 쑺; 쑺; ) HANGUL SYLLABLE SSUJ +C47B;C47B;110A 116E 11BE;C47B;110A 116E 11BE; # (쑻; 쑻; 쑻; 쑻; 쑻; ) HANGUL SYLLABLE SSUC +C47C;C47C;110A 116E 11BF;C47C;110A 116E 11BF; # (쑼; 쑼; 쑼; 쑼; 쑼; ) HANGUL SYLLABLE SSUK +C47D;C47D;110A 116E 11C0;C47D;110A 116E 11C0; # (쑽; 쑽; 쑽; 쑽; 쑽; ) HANGUL SYLLABLE SSUT +C47E;C47E;110A 116E 11C1;C47E;110A 116E 11C1; # (쑾; 쑾; 쑾; 쑾; 쑾; ) HANGUL SYLLABLE SSUP +C47F;C47F;110A 116E 11C2;C47F;110A 116E 11C2; # (쑿; 쑿; 쑿; 쑿; 쑿; ) HANGUL SYLLABLE SSUH +C480;C480;110A 116F;C480;110A 116F; # (쒀; 쒀; 쒀; 쒀; 쒀; ) HANGUL SYLLABLE SSWEO +C481;C481;110A 116F 11A8;C481;110A 116F 11A8; # (쒁; 쒁; 쒁; 쒁; 쒁; ) HANGUL SYLLABLE SSWEOG +C482;C482;110A 116F 11A9;C482;110A 116F 11A9; # (쒂; 쒂; 쒂; 쒂; 쒂; ) HANGUL SYLLABLE SSWEOGG +C483;C483;110A 116F 11AA;C483;110A 116F 11AA; # (쒃; 쒃; 쒃; 쒃; 쒃; ) HANGUL SYLLABLE SSWEOGS +C484;C484;110A 116F 11AB;C484;110A 116F 11AB; # (쒄; 쒄; 쒄; 쒄; 쒄; ) HANGUL SYLLABLE SSWEON +C485;C485;110A 116F 11AC;C485;110A 116F 11AC; # (쒅; 쒅; 쒅; 쒅; 쒅; ) HANGUL SYLLABLE SSWEONJ +C486;C486;110A 116F 11AD;C486;110A 116F 11AD; # (쒆; 쒆; 쒆; 쒆; 쒆; ) HANGUL SYLLABLE SSWEONH +C487;C487;110A 116F 11AE;C487;110A 116F 11AE; # (쒇; 쒇; 쒇; 쒇; 쒇; ) HANGUL SYLLABLE SSWEOD +C488;C488;110A 116F 11AF;C488;110A 116F 11AF; # (쒈; 쒈; 쒈; 쒈; 쒈; ) HANGUL SYLLABLE SSWEOL +C489;C489;110A 116F 11B0;C489;110A 116F 11B0; # (쒉; 쒉; 쒉; 쒉; 쒉; ) HANGUL SYLLABLE SSWEOLG +C48A;C48A;110A 116F 11B1;C48A;110A 116F 11B1; # (쒊; 쒊; 쒊; 쒊; 쒊; ) HANGUL SYLLABLE SSWEOLM +C48B;C48B;110A 116F 11B2;C48B;110A 116F 11B2; # (쒋; 쒋; 쒋; 쒋; 쒋; ) HANGUL SYLLABLE SSWEOLB +C48C;C48C;110A 116F 11B3;C48C;110A 116F 11B3; # (쒌; 쒌; 쒌; 쒌; 쒌; ) HANGUL SYLLABLE SSWEOLS +C48D;C48D;110A 116F 11B4;C48D;110A 116F 11B4; # (쒍; 쒍; 쒍; 쒍; 쒍; ) HANGUL SYLLABLE SSWEOLT +C48E;C48E;110A 116F 11B5;C48E;110A 116F 11B5; # (쒎; 쒎; 쒎; 쒎; 쒎; ) HANGUL SYLLABLE SSWEOLP +C48F;C48F;110A 116F 11B6;C48F;110A 116F 11B6; # (쒏; 쒏; 쒏; 쒏; 쒏; ) HANGUL SYLLABLE SSWEOLH +C490;C490;110A 116F 11B7;C490;110A 116F 11B7; # (쒐; 쒐; 쒐; 쒐; 쒐; ) HANGUL SYLLABLE SSWEOM +C491;C491;110A 116F 11B8;C491;110A 116F 11B8; # (쒑; 쒑; 쒑; 쒑; 쒑; ) HANGUL SYLLABLE SSWEOB +C492;C492;110A 116F 11B9;C492;110A 116F 11B9; # (쒒; 쒒; 쒒; 쒒; 쒒; ) HANGUL SYLLABLE SSWEOBS +C493;C493;110A 116F 11BA;C493;110A 116F 11BA; # (쒓; 쒓; 쒓; 쒓; 쒓; ) HANGUL SYLLABLE SSWEOS +C494;C494;110A 116F 11BB;C494;110A 116F 11BB; # (쒔; 쒔; 쒔; 쒔; 쒔; ) HANGUL SYLLABLE SSWEOSS +C495;C495;110A 116F 11BC;C495;110A 116F 11BC; # (쒕; 쒕; 쒕; 쒕; 쒕; ) HANGUL SYLLABLE SSWEONG +C496;C496;110A 116F 11BD;C496;110A 116F 11BD; # (쒖; 쒖; 쒖; 쒖; 쒖; ) HANGUL SYLLABLE SSWEOJ +C497;C497;110A 116F 11BE;C497;110A 116F 11BE; # (쒗; 쒗; 쒗; 쒗; 쒗; ) HANGUL SYLLABLE SSWEOC +C498;C498;110A 116F 11BF;C498;110A 116F 11BF; # (쒘; 쒘; 쒘; 쒘; 쒘; ) HANGUL SYLLABLE SSWEOK +C499;C499;110A 116F 11C0;C499;110A 116F 11C0; # (쒙; 쒙; 쒙; 쒙; 쒙; ) HANGUL SYLLABLE SSWEOT +C49A;C49A;110A 116F 11C1;C49A;110A 116F 11C1; # (쒚; 쒚; 쒚; 쒚; 쒚; ) HANGUL SYLLABLE SSWEOP +C49B;C49B;110A 116F 11C2;C49B;110A 116F 11C2; # (쒛; 쒛; 쒛; 쒛; 쒛; ) HANGUL SYLLABLE SSWEOH +C49C;C49C;110A 1170;C49C;110A 1170; # (쒜; 쒜; 쒜; 쒜; 쒜; ) HANGUL SYLLABLE SSWE +C49D;C49D;110A 1170 11A8;C49D;110A 1170 11A8; # (쒝; 쒝; 쒝; 쒝; 쒝; ) HANGUL SYLLABLE SSWEG +C49E;C49E;110A 1170 11A9;C49E;110A 1170 11A9; # (쒞; 쒞; 쒞; 쒞; 쒞; ) HANGUL SYLLABLE SSWEGG +C49F;C49F;110A 1170 11AA;C49F;110A 1170 11AA; # (쒟; 쒟; 쒟; 쒟; 쒟; ) HANGUL SYLLABLE SSWEGS +C4A0;C4A0;110A 1170 11AB;C4A0;110A 1170 11AB; # (쒠; 쒠; 쒠; 쒠; 쒠; ) HANGUL SYLLABLE SSWEN +C4A1;C4A1;110A 1170 11AC;C4A1;110A 1170 11AC; # (쒡; 쒡; 쒡; 쒡; 쒡; ) HANGUL SYLLABLE SSWENJ +C4A2;C4A2;110A 1170 11AD;C4A2;110A 1170 11AD; # (쒢; 쒢; 쒢; 쒢; 쒢; ) HANGUL SYLLABLE SSWENH +C4A3;C4A3;110A 1170 11AE;C4A3;110A 1170 11AE; # (쒣; 쒣; 쒣; 쒣; 쒣; ) HANGUL SYLLABLE SSWED +C4A4;C4A4;110A 1170 11AF;C4A4;110A 1170 11AF; # (쒤; 쒤; 쒤; 쒤; 쒤; ) HANGUL SYLLABLE SSWEL +C4A5;C4A5;110A 1170 11B0;C4A5;110A 1170 11B0; # (쒥; 쒥; 쒥; 쒥; 쒥; ) HANGUL SYLLABLE SSWELG +C4A6;C4A6;110A 1170 11B1;C4A6;110A 1170 11B1; # (쒦; 쒦; 쒦; 쒦; 쒦; ) HANGUL SYLLABLE SSWELM +C4A7;C4A7;110A 1170 11B2;C4A7;110A 1170 11B2; # (쒧; 쒧; 쒧; 쒧; 쒧; ) HANGUL SYLLABLE SSWELB +C4A8;C4A8;110A 1170 11B3;C4A8;110A 1170 11B3; # (쒨; 쒨; 쒨; 쒨; 쒨; ) HANGUL SYLLABLE SSWELS +C4A9;C4A9;110A 1170 11B4;C4A9;110A 1170 11B4; # (쒩; 쒩; 쒩; 쒩; 쒩; ) HANGUL SYLLABLE SSWELT +C4AA;C4AA;110A 1170 11B5;C4AA;110A 1170 11B5; # (쒪; 쒪; 쒪; 쒪; 쒪; ) HANGUL SYLLABLE SSWELP +C4AB;C4AB;110A 1170 11B6;C4AB;110A 1170 11B6; # (쒫; 쒫; 쒫; 쒫; 쒫; ) HANGUL SYLLABLE SSWELH +C4AC;C4AC;110A 1170 11B7;C4AC;110A 1170 11B7; # (쒬; 쒬; 쒬; 쒬; 쒬; ) HANGUL SYLLABLE SSWEM +C4AD;C4AD;110A 1170 11B8;C4AD;110A 1170 11B8; # (쒭; 쒭; 쒭; 쒭; 쒭; ) HANGUL SYLLABLE SSWEB +C4AE;C4AE;110A 1170 11B9;C4AE;110A 1170 11B9; # (쒮; 쒮; 쒮; 쒮; 쒮; ) HANGUL SYLLABLE SSWEBS +C4AF;C4AF;110A 1170 11BA;C4AF;110A 1170 11BA; # (쒯; 쒯; 쒯; 쒯; 쒯; ) HANGUL SYLLABLE SSWES +C4B0;C4B0;110A 1170 11BB;C4B0;110A 1170 11BB; # (쒰; 쒰; 쒰; 쒰; 쒰; ) HANGUL SYLLABLE SSWESS +C4B1;C4B1;110A 1170 11BC;C4B1;110A 1170 11BC; # (쒱; 쒱; 쒱; 쒱; 쒱; ) HANGUL SYLLABLE SSWENG +C4B2;C4B2;110A 1170 11BD;C4B2;110A 1170 11BD; # (쒲; 쒲; 쒲; 쒲; 쒲; ) HANGUL SYLLABLE SSWEJ +C4B3;C4B3;110A 1170 11BE;C4B3;110A 1170 11BE; # (쒳; 쒳; 쒳; 쒳; 쒳; ) HANGUL SYLLABLE SSWEC +C4B4;C4B4;110A 1170 11BF;C4B4;110A 1170 11BF; # (쒴; 쒴; 쒴; 쒴; 쒴; ) HANGUL SYLLABLE SSWEK +C4B5;C4B5;110A 1170 11C0;C4B5;110A 1170 11C0; # (쒵; 쒵; 쒵; 쒵; 쒵; ) HANGUL SYLLABLE SSWET +C4B6;C4B6;110A 1170 11C1;C4B6;110A 1170 11C1; # (쒶; 쒶; 쒶; 쒶; 쒶; ) HANGUL SYLLABLE SSWEP +C4B7;C4B7;110A 1170 11C2;C4B7;110A 1170 11C2; # (쒷; 쒷; 쒷; 쒷; 쒷; ) HANGUL SYLLABLE SSWEH +C4B8;C4B8;110A 1171;C4B8;110A 1171; # (쒸; 쒸; 쒸; 쒸; 쒸; ) HANGUL SYLLABLE SSWI +C4B9;C4B9;110A 1171 11A8;C4B9;110A 1171 11A8; # (쒹; 쒹; 쒹; 쒹; 쒹; ) HANGUL SYLLABLE SSWIG +C4BA;C4BA;110A 1171 11A9;C4BA;110A 1171 11A9; # (쒺; 쒺; 쒺; 쒺; 쒺; ) HANGUL SYLLABLE SSWIGG +C4BB;C4BB;110A 1171 11AA;C4BB;110A 1171 11AA; # (쒻; 쒻; 쒻; 쒻; 쒻; ) HANGUL SYLLABLE SSWIGS +C4BC;C4BC;110A 1171 11AB;C4BC;110A 1171 11AB; # (쒼; 쒼; 쒼; 쒼; 쒼; ) HANGUL SYLLABLE SSWIN +C4BD;C4BD;110A 1171 11AC;C4BD;110A 1171 11AC; # (쒽; 쒽; 쒽; 쒽; 쒽; ) HANGUL SYLLABLE SSWINJ +C4BE;C4BE;110A 1171 11AD;C4BE;110A 1171 11AD; # (쒾; 쒾; 쒾; 쒾; 쒾; ) HANGUL SYLLABLE SSWINH +C4BF;C4BF;110A 1171 11AE;C4BF;110A 1171 11AE; # (쒿; 쒿; 쒿; 쒿; 쒿; ) HANGUL SYLLABLE SSWID +C4C0;C4C0;110A 1171 11AF;C4C0;110A 1171 11AF; # (쓀; 쓀; 쓀; 쓀; 쓀; ) HANGUL SYLLABLE SSWIL +C4C1;C4C1;110A 1171 11B0;C4C1;110A 1171 11B0; # (쓁; 쓁; 쓁; 쓁; 쓁; ) HANGUL SYLLABLE SSWILG +C4C2;C4C2;110A 1171 11B1;C4C2;110A 1171 11B1; # (쓂; 쓂; 쓂; 쓂; 쓂; ) HANGUL SYLLABLE SSWILM +C4C3;C4C3;110A 1171 11B2;C4C3;110A 1171 11B2; # (쓃; 쓃; 쓃; 쓃; 쓃; ) HANGUL SYLLABLE SSWILB +C4C4;C4C4;110A 1171 11B3;C4C4;110A 1171 11B3; # (쓄; 쓄; 쓄; 쓄; 쓄; ) HANGUL SYLLABLE SSWILS +C4C5;C4C5;110A 1171 11B4;C4C5;110A 1171 11B4; # (쓅; 쓅; 쓅; 쓅; 쓅; ) HANGUL SYLLABLE SSWILT +C4C6;C4C6;110A 1171 11B5;C4C6;110A 1171 11B5; # (쓆; 쓆; 쓆; 쓆; 쓆; ) HANGUL SYLLABLE SSWILP +C4C7;C4C7;110A 1171 11B6;C4C7;110A 1171 11B6; # (쓇; 쓇; 쓇; 쓇; 쓇; ) HANGUL SYLLABLE SSWILH +C4C8;C4C8;110A 1171 11B7;C4C8;110A 1171 11B7; # (쓈; 쓈; 쓈; 쓈; 쓈; ) HANGUL SYLLABLE SSWIM +C4C9;C4C9;110A 1171 11B8;C4C9;110A 1171 11B8; # (쓉; 쓉; 쓉; 쓉; 쓉; ) HANGUL SYLLABLE SSWIB +C4CA;C4CA;110A 1171 11B9;C4CA;110A 1171 11B9; # (쓊; 쓊; 쓊; 쓊; 쓊; ) HANGUL SYLLABLE SSWIBS +C4CB;C4CB;110A 1171 11BA;C4CB;110A 1171 11BA; # (쓋; 쓋; 쓋; 쓋; 쓋; ) HANGUL SYLLABLE SSWIS +C4CC;C4CC;110A 1171 11BB;C4CC;110A 1171 11BB; # (쓌; 쓌; 쓌; 쓌; 쓌; ) HANGUL SYLLABLE SSWISS +C4CD;C4CD;110A 1171 11BC;C4CD;110A 1171 11BC; # (쓍; 쓍; 쓍; 쓍; 쓍; ) HANGUL SYLLABLE SSWING +C4CE;C4CE;110A 1171 11BD;C4CE;110A 1171 11BD; # (쓎; 쓎; 쓎; 쓎; 쓎; ) HANGUL SYLLABLE SSWIJ +C4CF;C4CF;110A 1171 11BE;C4CF;110A 1171 11BE; # (쓏; 쓏; 쓏; 쓏; 쓏; ) HANGUL SYLLABLE SSWIC +C4D0;C4D0;110A 1171 11BF;C4D0;110A 1171 11BF; # (쓐; 쓐; 쓐; 쓐; 쓐; ) HANGUL SYLLABLE SSWIK +C4D1;C4D1;110A 1171 11C0;C4D1;110A 1171 11C0; # (쓑; 쓑; 쓑; 쓑; 쓑; ) HANGUL SYLLABLE SSWIT +C4D2;C4D2;110A 1171 11C1;C4D2;110A 1171 11C1; # (쓒; 쓒; 쓒; 쓒; 쓒; ) HANGUL SYLLABLE SSWIP +C4D3;C4D3;110A 1171 11C2;C4D3;110A 1171 11C2; # (쓓; 쓓; 쓓; 쓓; 쓓; ) HANGUL SYLLABLE SSWIH +C4D4;C4D4;110A 1172;C4D4;110A 1172; # (쓔; 쓔; 쓔; 쓔; 쓔; ) HANGUL SYLLABLE SSYU +C4D5;C4D5;110A 1172 11A8;C4D5;110A 1172 11A8; # (쓕; 쓕; 쓕; 쓕; 쓕; ) HANGUL SYLLABLE SSYUG +C4D6;C4D6;110A 1172 11A9;C4D6;110A 1172 11A9; # (쓖; 쓖; 쓖; 쓖; 쓖; ) HANGUL SYLLABLE SSYUGG +C4D7;C4D7;110A 1172 11AA;C4D7;110A 1172 11AA; # (쓗; 쓗; 쓗; 쓗; 쓗; ) HANGUL SYLLABLE SSYUGS +C4D8;C4D8;110A 1172 11AB;C4D8;110A 1172 11AB; # (쓘; 쓘; 쓘; 쓘; 쓘; ) HANGUL SYLLABLE SSYUN +C4D9;C4D9;110A 1172 11AC;C4D9;110A 1172 11AC; # (쓙; 쓙; 쓙; 쓙; 쓙; ) HANGUL SYLLABLE SSYUNJ +C4DA;C4DA;110A 1172 11AD;C4DA;110A 1172 11AD; # (쓚; 쓚; 쓚; 쓚; 쓚; ) HANGUL SYLLABLE SSYUNH +C4DB;C4DB;110A 1172 11AE;C4DB;110A 1172 11AE; # (쓛; 쓛; 쓛; 쓛; 쓛; ) HANGUL SYLLABLE SSYUD +C4DC;C4DC;110A 1172 11AF;C4DC;110A 1172 11AF; # (쓜; 쓜; 쓜; 쓜; 쓜; ) HANGUL SYLLABLE SSYUL +C4DD;C4DD;110A 1172 11B0;C4DD;110A 1172 11B0; # (쓝; 쓝; 쓝; 쓝; 쓝; ) HANGUL SYLLABLE SSYULG +C4DE;C4DE;110A 1172 11B1;C4DE;110A 1172 11B1; # (쓞; 쓞; 쓞; 쓞; 쓞; ) HANGUL SYLLABLE SSYULM +C4DF;C4DF;110A 1172 11B2;C4DF;110A 1172 11B2; # (쓟; 쓟; 쓟; 쓟; 쓟; ) HANGUL SYLLABLE SSYULB +C4E0;C4E0;110A 1172 11B3;C4E0;110A 1172 11B3; # (쓠; 쓠; 쓠; 쓠; 쓠; ) HANGUL SYLLABLE SSYULS +C4E1;C4E1;110A 1172 11B4;C4E1;110A 1172 11B4; # (쓡; 쓡; 쓡; 쓡; 쓡; ) HANGUL SYLLABLE SSYULT +C4E2;C4E2;110A 1172 11B5;C4E2;110A 1172 11B5; # (쓢; 쓢; 쓢; 쓢; 쓢; ) HANGUL SYLLABLE SSYULP +C4E3;C4E3;110A 1172 11B6;C4E3;110A 1172 11B6; # (쓣; 쓣; 쓣; 쓣; 쓣; ) HANGUL SYLLABLE SSYULH +C4E4;C4E4;110A 1172 11B7;C4E4;110A 1172 11B7; # (쓤; 쓤; 쓤; 쓤; 쓤; ) HANGUL SYLLABLE SSYUM +C4E5;C4E5;110A 1172 11B8;C4E5;110A 1172 11B8; # (쓥; 쓥; 쓥; 쓥; 쓥; ) HANGUL SYLLABLE SSYUB +C4E6;C4E6;110A 1172 11B9;C4E6;110A 1172 11B9; # (쓦; 쓦; 쓦; 쓦; 쓦; ) HANGUL SYLLABLE SSYUBS +C4E7;C4E7;110A 1172 11BA;C4E7;110A 1172 11BA; # (쓧; 쓧; 쓧; 쓧; 쓧; ) HANGUL SYLLABLE SSYUS +C4E8;C4E8;110A 1172 11BB;C4E8;110A 1172 11BB; # (쓨; 쓨; 쓨; 쓨; 쓨; ) HANGUL SYLLABLE SSYUSS +C4E9;C4E9;110A 1172 11BC;C4E9;110A 1172 11BC; # (쓩; 쓩; 쓩; 쓩; 쓩; ) HANGUL SYLLABLE SSYUNG +C4EA;C4EA;110A 1172 11BD;C4EA;110A 1172 11BD; # (쓪; 쓪; 쓪; 쓪; 쓪; ) HANGUL SYLLABLE SSYUJ +C4EB;C4EB;110A 1172 11BE;C4EB;110A 1172 11BE; # (쓫; 쓫; 쓫; 쓫; 쓫; ) HANGUL SYLLABLE SSYUC +C4EC;C4EC;110A 1172 11BF;C4EC;110A 1172 11BF; # (쓬; 쓬; 쓬; 쓬; 쓬; ) HANGUL SYLLABLE SSYUK +C4ED;C4ED;110A 1172 11C0;C4ED;110A 1172 11C0; # (쓭; 쓭; 쓭; 쓭; 쓭; ) HANGUL SYLLABLE SSYUT +C4EE;C4EE;110A 1172 11C1;C4EE;110A 1172 11C1; # (쓮; 쓮; 쓮; 쓮; 쓮; ) HANGUL SYLLABLE SSYUP +C4EF;C4EF;110A 1172 11C2;C4EF;110A 1172 11C2; # (쓯; 쓯; 쓯; 쓯; 쓯; ) HANGUL SYLLABLE SSYUH +C4F0;C4F0;110A 1173;C4F0;110A 1173; # (쓰; 쓰; 쓰; 쓰; 쓰; ) HANGUL SYLLABLE SSEU +C4F1;C4F1;110A 1173 11A8;C4F1;110A 1173 11A8; # (쓱; 쓱; 쓱; 쓱; 쓱; ) HANGUL SYLLABLE SSEUG +C4F2;C4F2;110A 1173 11A9;C4F2;110A 1173 11A9; # (쓲; 쓲; 쓲; 쓲; 쓲; ) HANGUL SYLLABLE SSEUGG +C4F3;C4F3;110A 1173 11AA;C4F3;110A 1173 11AA; # (쓳; 쓳; 쓳; 쓳; 쓳; ) HANGUL SYLLABLE SSEUGS +C4F4;C4F4;110A 1173 11AB;C4F4;110A 1173 11AB; # (쓴; 쓴; 쓴; 쓴; 쓴; ) HANGUL SYLLABLE SSEUN +C4F5;C4F5;110A 1173 11AC;C4F5;110A 1173 11AC; # (쓵; 쓵; 쓵; 쓵; 쓵; ) HANGUL SYLLABLE SSEUNJ +C4F6;C4F6;110A 1173 11AD;C4F6;110A 1173 11AD; # (쓶; 쓶; 쓶; 쓶; 쓶; ) HANGUL SYLLABLE SSEUNH +C4F7;C4F7;110A 1173 11AE;C4F7;110A 1173 11AE; # (쓷; 쓷; 쓷; 쓷; 쓷; ) HANGUL SYLLABLE SSEUD +C4F8;C4F8;110A 1173 11AF;C4F8;110A 1173 11AF; # (쓸; 쓸; 쓸; 쓸; 쓸; ) HANGUL SYLLABLE SSEUL +C4F9;C4F9;110A 1173 11B0;C4F9;110A 1173 11B0; # (쓹; 쓹; 쓹; 쓹; 쓹; ) HANGUL SYLLABLE SSEULG +C4FA;C4FA;110A 1173 11B1;C4FA;110A 1173 11B1; # (쓺; 쓺; 쓺; 쓺; 쓺; ) HANGUL SYLLABLE SSEULM +C4FB;C4FB;110A 1173 11B2;C4FB;110A 1173 11B2; # (쓻; 쓻; 쓻; 쓻; 쓻; ) HANGUL SYLLABLE SSEULB +C4FC;C4FC;110A 1173 11B3;C4FC;110A 1173 11B3; # (쓼; 쓼; 쓼; 쓼; 쓼; ) HANGUL SYLLABLE SSEULS +C4FD;C4FD;110A 1173 11B4;C4FD;110A 1173 11B4; # (쓽; 쓽; 쓽; 쓽; 쓽; ) HANGUL SYLLABLE SSEULT +C4FE;C4FE;110A 1173 11B5;C4FE;110A 1173 11B5; # (쓾; 쓾; 쓾; 쓾; 쓾; ) HANGUL SYLLABLE SSEULP +C4FF;C4FF;110A 1173 11B6;C4FF;110A 1173 11B6; # (쓿; 쓿; 쓿; 쓿; 쓿; ) HANGUL SYLLABLE SSEULH +C500;C500;110A 1173 11B7;C500;110A 1173 11B7; # (씀; 씀; 씀; 씀; 씀; ) HANGUL SYLLABLE SSEUM +C501;C501;110A 1173 11B8;C501;110A 1173 11B8; # (씁; 씁; 씁; 씁; 씁; ) HANGUL SYLLABLE SSEUB +C502;C502;110A 1173 11B9;C502;110A 1173 11B9; # (씂; 씂; 씂; 씂; 씂; ) HANGUL SYLLABLE SSEUBS +C503;C503;110A 1173 11BA;C503;110A 1173 11BA; # (씃; 씃; 씃; 씃; 씃; ) HANGUL SYLLABLE SSEUS +C504;C504;110A 1173 11BB;C504;110A 1173 11BB; # (씄; 씄; 씄; 씄; 씄; ) HANGUL SYLLABLE SSEUSS +C505;C505;110A 1173 11BC;C505;110A 1173 11BC; # (씅; 씅; 씅; 씅; 씅; ) HANGUL SYLLABLE SSEUNG +C506;C506;110A 1173 11BD;C506;110A 1173 11BD; # (씆; 씆; 씆; 씆; 씆; ) HANGUL SYLLABLE SSEUJ +C507;C507;110A 1173 11BE;C507;110A 1173 11BE; # (씇; 씇; 씇; 씇; 씇; ) HANGUL SYLLABLE SSEUC +C508;C508;110A 1173 11BF;C508;110A 1173 11BF; # (씈; 씈; 씈; 씈; 씈; ) HANGUL SYLLABLE SSEUK +C509;C509;110A 1173 11C0;C509;110A 1173 11C0; # (씉; 씉; 씉; 씉; 씉; ) HANGUL SYLLABLE SSEUT +C50A;C50A;110A 1173 11C1;C50A;110A 1173 11C1; # (씊; 씊; 씊; 씊; 씊; ) HANGUL SYLLABLE SSEUP +C50B;C50B;110A 1173 11C2;C50B;110A 1173 11C2; # (씋; 씋; 씋; 씋; 씋; ) HANGUL SYLLABLE SSEUH +C50C;C50C;110A 1174;C50C;110A 1174; # (씌; 씌; 씌; 씌; 씌; ) HANGUL SYLLABLE SSYI +C50D;C50D;110A 1174 11A8;C50D;110A 1174 11A8; # (씍; 씍; 씍; 씍; 씍; ) HANGUL SYLLABLE SSYIG +C50E;C50E;110A 1174 11A9;C50E;110A 1174 11A9; # (씎; 씎; 씎; 씎; 씎; ) HANGUL SYLLABLE SSYIGG +C50F;C50F;110A 1174 11AA;C50F;110A 1174 11AA; # (씏; 씏; 씏; 씏; 씏; ) HANGUL SYLLABLE SSYIGS +C510;C510;110A 1174 11AB;C510;110A 1174 11AB; # (씐; 씐; 씐; 씐; 씐; ) HANGUL SYLLABLE SSYIN +C511;C511;110A 1174 11AC;C511;110A 1174 11AC; # (씑; 씑; 씑; 씑; 씑; ) HANGUL SYLLABLE SSYINJ +C512;C512;110A 1174 11AD;C512;110A 1174 11AD; # (씒; 씒; 씒; 씒; 씒; ) HANGUL SYLLABLE SSYINH +C513;C513;110A 1174 11AE;C513;110A 1174 11AE; # (씓; 씓; 씓; 씓; 씓; ) HANGUL SYLLABLE SSYID +C514;C514;110A 1174 11AF;C514;110A 1174 11AF; # (씔; 씔; 씔; 씔; 씔; ) HANGUL SYLLABLE SSYIL +C515;C515;110A 1174 11B0;C515;110A 1174 11B0; # (씕; 씕; 씕; 씕; 씕; ) HANGUL SYLLABLE SSYILG +C516;C516;110A 1174 11B1;C516;110A 1174 11B1; # (씖; 씖; 씖; 씖; 씖; ) HANGUL SYLLABLE SSYILM +C517;C517;110A 1174 11B2;C517;110A 1174 11B2; # (씗; 씗; 씗; 씗; 씗; ) HANGUL SYLLABLE SSYILB +C518;C518;110A 1174 11B3;C518;110A 1174 11B3; # (씘; 씘; 씘; 씘; 씘; ) HANGUL SYLLABLE SSYILS +C519;C519;110A 1174 11B4;C519;110A 1174 11B4; # (씙; 씙; 씙; 씙; 씙; ) HANGUL SYLLABLE SSYILT +C51A;C51A;110A 1174 11B5;C51A;110A 1174 11B5; # (씚; 씚; 씚; 씚; 씚; ) HANGUL SYLLABLE SSYILP +C51B;C51B;110A 1174 11B6;C51B;110A 1174 11B6; # (씛; 씛; 씛; 씛; 씛; ) HANGUL SYLLABLE SSYILH +C51C;C51C;110A 1174 11B7;C51C;110A 1174 11B7; # (씜; 씜; 씜; 씜; 씜; ) HANGUL SYLLABLE SSYIM +C51D;C51D;110A 1174 11B8;C51D;110A 1174 11B8; # (씝; 씝; 씝; 씝; 씝; ) HANGUL SYLLABLE SSYIB +C51E;C51E;110A 1174 11B9;C51E;110A 1174 11B9; # (씞; 씞; 씞; 씞; 씞; ) HANGUL SYLLABLE SSYIBS +C51F;C51F;110A 1174 11BA;C51F;110A 1174 11BA; # (씟; 씟; 씟; 씟; 씟; ) HANGUL SYLLABLE SSYIS +C520;C520;110A 1174 11BB;C520;110A 1174 11BB; # (씠; 씠; 씠; 씠; 씠; ) HANGUL SYLLABLE SSYISS +C521;C521;110A 1174 11BC;C521;110A 1174 11BC; # (씡; 씡; 씡; 씡; 씡; ) HANGUL SYLLABLE SSYING +C522;C522;110A 1174 11BD;C522;110A 1174 11BD; # (씢; 씢; 씢; 씢; 씢; ) HANGUL SYLLABLE SSYIJ +C523;C523;110A 1174 11BE;C523;110A 1174 11BE; # (씣; 씣; 씣; 씣; 씣; ) HANGUL SYLLABLE SSYIC +C524;C524;110A 1174 11BF;C524;110A 1174 11BF; # (씤; 씤; 씤; 씤; 씤; ) HANGUL SYLLABLE SSYIK +C525;C525;110A 1174 11C0;C525;110A 1174 11C0; # (씥; 씥; 씥; 씥; 씥; ) HANGUL SYLLABLE SSYIT +C526;C526;110A 1174 11C1;C526;110A 1174 11C1; # (씦; 씦; 씦; 씦; 씦; ) HANGUL SYLLABLE SSYIP +C527;C527;110A 1174 11C2;C527;110A 1174 11C2; # (씧; 씧; 씧; 씧; 씧; ) HANGUL SYLLABLE SSYIH +C528;C528;110A 1175;C528;110A 1175; # (씨; 씨; 씨; 씨; 씨; ) HANGUL SYLLABLE SSI +C529;C529;110A 1175 11A8;C529;110A 1175 11A8; # (씩; 씩; 씩; 씩; 씩; ) HANGUL SYLLABLE SSIG +C52A;C52A;110A 1175 11A9;C52A;110A 1175 11A9; # (씪; 씪; 씪; 씪; 씪; ) HANGUL SYLLABLE SSIGG +C52B;C52B;110A 1175 11AA;C52B;110A 1175 11AA; # (씫; 씫; 씫; 씫; 씫; ) HANGUL SYLLABLE SSIGS +C52C;C52C;110A 1175 11AB;C52C;110A 1175 11AB; # (씬; 씬; 씬; 씬; 씬; ) HANGUL SYLLABLE SSIN +C52D;C52D;110A 1175 11AC;C52D;110A 1175 11AC; # (씭; 씭; 씭; 씭; 씭; ) HANGUL SYLLABLE SSINJ +C52E;C52E;110A 1175 11AD;C52E;110A 1175 11AD; # (씮; 씮; 씮; 씮; 씮; ) HANGUL SYLLABLE SSINH +C52F;C52F;110A 1175 11AE;C52F;110A 1175 11AE; # (씯; 씯; 씯; 씯; 씯; ) HANGUL SYLLABLE SSID +C530;C530;110A 1175 11AF;C530;110A 1175 11AF; # (씰; 씰; 씰; 씰; 씰; ) HANGUL SYLLABLE SSIL +C531;C531;110A 1175 11B0;C531;110A 1175 11B0; # (씱; 씱; 씱; 씱; 씱; ) HANGUL SYLLABLE SSILG +C532;C532;110A 1175 11B1;C532;110A 1175 11B1; # (씲; 씲; 씲; 씲; 씲; ) HANGUL SYLLABLE SSILM +C533;C533;110A 1175 11B2;C533;110A 1175 11B2; # (씳; 씳; 씳; 씳; 씳; ) HANGUL SYLLABLE SSILB +C534;C534;110A 1175 11B3;C534;110A 1175 11B3; # (씴; 씴; 씴; 씴; 씴; ) HANGUL SYLLABLE SSILS +C535;C535;110A 1175 11B4;C535;110A 1175 11B4; # (씵; 씵; 씵; 씵; 씵; ) HANGUL SYLLABLE SSILT +C536;C536;110A 1175 11B5;C536;110A 1175 11B5; # (씶; 씶; 씶; 씶; 씶; ) HANGUL SYLLABLE SSILP +C537;C537;110A 1175 11B6;C537;110A 1175 11B6; # (씷; 씷; 씷; 씷; 씷; ) HANGUL SYLLABLE SSILH +C538;C538;110A 1175 11B7;C538;110A 1175 11B7; # (씸; 씸; 씸; 씸; 씸; ) HANGUL SYLLABLE SSIM +C539;C539;110A 1175 11B8;C539;110A 1175 11B8; # (씹; 씹; 씹; 씹; 씹; ) HANGUL SYLLABLE SSIB +C53A;C53A;110A 1175 11B9;C53A;110A 1175 11B9; # (씺; 씺; 씺; 씺; 씺; ) HANGUL SYLLABLE SSIBS +C53B;C53B;110A 1175 11BA;C53B;110A 1175 11BA; # (씻; 씻; 씻; 씻; 씻; ) HANGUL SYLLABLE SSIS +C53C;C53C;110A 1175 11BB;C53C;110A 1175 11BB; # (씼; 씼; 씼; 씼; 씼; ) HANGUL SYLLABLE SSISS +C53D;C53D;110A 1175 11BC;C53D;110A 1175 11BC; # (씽; 씽; 씽; 씽; 씽; ) HANGUL SYLLABLE SSING +C53E;C53E;110A 1175 11BD;C53E;110A 1175 11BD; # (씾; 씾; 씾; 씾; 씾; ) HANGUL SYLLABLE SSIJ +C53F;C53F;110A 1175 11BE;C53F;110A 1175 11BE; # (씿; 씿; 씿; 씿; 씿; ) HANGUL SYLLABLE SSIC +C540;C540;110A 1175 11BF;C540;110A 1175 11BF; # (앀; 앀; 앀; 앀; 앀; ) HANGUL SYLLABLE SSIK +C541;C541;110A 1175 11C0;C541;110A 1175 11C0; # (앁; 앁; 앁; 앁; 앁; ) HANGUL SYLLABLE SSIT +C542;C542;110A 1175 11C1;C542;110A 1175 11C1; # (앂; 앂; 앂; 앂; 앂; ) HANGUL SYLLABLE SSIP +C543;C543;110A 1175 11C2;C543;110A 1175 11C2; # (앃; 앃; 앃; 앃; 앃; ) HANGUL SYLLABLE SSIH +C544;C544;110B 1161;C544;110B 1161; # (아; 아; 아; 아; 아; ) HANGUL SYLLABLE A +C545;C545;110B 1161 11A8;C545;110B 1161 11A8; # (악; 악; 악; 악; 악; ) HANGUL SYLLABLE AG +C546;C546;110B 1161 11A9;C546;110B 1161 11A9; # (앆; 앆; 앆; 앆; 앆; ) HANGUL SYLLABLE AGG +C547;C547;110B 1161 11AA;C547;110B 1161 11AA; # (앇; 앇; 앇; 앇; 앇; ) HANGUL SYLLABLE AGS +C548;C548;110B 1161 11AB;C548;110B 1161 11AB; # (안; 안; 안; 안; 안; ) HANGUL SYLLABLE AN +C549;C549;110B 1161 11AC;C549;110B 1161 11AC; # (앉; 앉; 앉; 앉; 앉; ) HANGUL SYLLABLE ANJ +C54A;C54A;110B 1161 11AD;C54A;110B 1161 11AD; # (않; 않; 않; 않; 않; ) HANGUL SYLLABLE ANH +C54B;C54B;110B 1161 11AE;C54B;110B 1161 11AE; # (앋; 앋; 앋; 앋; 앋; ) HANGUL SYLLABLE AD +C54C;C54C;110B 1161 11AF;C54C;110B 1161 11AF; # (알; 알; 알; 알; 알; ) HANGUL SYLLABLE AL +C54D;C54D;110B 1161 11B0;C54D;110B 1161 11B0; # (앍; 앍; 앍; 앍; 앍; ) HANGUL SYLLABLE ALG +C54E;C54E;110B 1161 11B1;C54E;110B 1161 11B1; # (앎; 앎; 앎; 앎; 앎; ) HANGUL SYLLABLE ALM +C54F;C54F;110B 1161 11B2;C54F;110B 1161 11B2; # (앏; 앏; 앏; 앏; 앏; ) HANGUL SYLLABLE ALB +C550;C550;110B 1161 11B3;C550;110B 1161 11B3; # (앐; 앐; 앐; 앐; 앐; ) HANGUL SYLLABLE ALS +C551;C551;110B 1161 11B4;C551;110B 1161 11B4; # (앑; 앑; 앑; 앑; 앑; ) HANGUL SYLLABLE ALT +C552;C552;110B 1161 11B5;C552;110B 1161 11B5; # (앒; 앒; 앒; 앒; 앒; ) HANGUL SYLLABLE ALP +C553;C553;110B 1161 11B6;C553;110B 1161 11B6; # (앓; 앓; 앓; 앓; 앓; ) HANGUL SYLLABLE ALH +C554;C554;110B 1161 11B7;C554;110B 1161 11B7; # (암; 암; 암; 암; 암; ) HANGUL SYLLABLE AM +C555;C555;110B 1161 11B8;C555;110B 1161 11B8; # (압; 압; 압; 압; 압; ) HANGUL SYLLABLE AB +C556;C556;110B 1161 11B9;C556;110B 1161 11B9; # (앖; 앖; 앖; 앖; 앖; ) HANGUL SYLLABLE ABS +C557;C557;110B 1161 11BA;C557;110B 1161 11BA; # (앗; 앗; 앗; 앗; 앗; ) HANGUL SYLLABLE AS +C558;C558;110B 1161 11BB;C558;110B 1161 11BB; # (았; 았; 았; 았; 았; ) HANGUL SYLLABLE ASS +C559;C559;110B 1161 11BC;C559;110B 1161 11BC; # (앙; 앙; 앙; 앙; 앙; ) HANGUL SYLLABLE ANG +C55A;C55A;110B 1161 11BD;C55A;110B 1161 11BD; # (앚; 앚; 앚; 앚; 앚; ) HANGUL SYLLABLE AJ +C55B;C55B;110B 1161 11BE;C55B;110B 1161 11BE; # (앛; 앛; 앛; 앛; 앛; ) HANGUL SYLLABLE AC +C55C;C55C;110B 1161 11BF;C55C;110B 1161 11BF; # (앜; 앜; 앜; 앜; 앜; ) HANGUL SYLLABLE AK +C55D;C55D;110B 1161 11C0;C55D;110B 1161 11C0; # (앝; 앝; 앝; 앝; 앝; ) HANGUL SYLLABLE AT +C55E;C55E;110B 1161 11C1;C55E;110B 1161 11C1; # (앞; 앞; 앞; 앞; 앞; ) HANGUL SYLLABLE AP +C55F;C55F;110B 1161 11C2;C55F;110B 1161 11C2; # (앟; 앟; 앟; 앟; 앟; ) HANGUL SYLLABLE AH +C560;C560;110B 1162;C560;110B 1162; # (애; 애; 애; 애; 애; ) HANGUL SYLLABLE AE +C561;C561;110B 1162 11A8;C561;110B 1162 11A8; # (액; 액; 액; 액; 액; ) HANGUL SYLLABLE AEG +C562;C562;110B 1162 11A9;C562;110B 1162 11A9; # (앢; 앢; 앢; 앢; 앢; ) HANGUL SYLLABLE AEGG +C563;C563;110B 1162 11AA;C563;110B 1162 11AA; # (앣; 앣; 앣; 앣; 앣; ) HANGUL SYLLABLE AEGS +C564;C564;110B 1162 11AB;C564;110B 1162 11AB; # (앤; 앤; 앤; 앤; 앤; ) HANGUL SYLLABLE AEN +C565;C565;110B 1162 11AC;C565;110B 1162 11AC; # (앥; 앥; 앥; 앥; 앥; ) HANGUL SYLLABLE AENJ +C566;C566;110B 1162 11AD;C566;110B 1162 11AD; # (앦; 앦; 앦; 앦; 앦; ) HANGUL SYLLABLE AENH +C567;C567;110B 1162 11AE;C567;110B 1162 11AE; # (앧; 앧; 앧; 앧; 앧; ) HANGUL SYLLABLE AED +C568;C568;110B 1162 11AF;C568;110B 1162 11AF; # (앨; 앨; 앨; 앨; 앨; ) HANGUL SYLLABLE AEL +C569;C569;110B 1162 11B0;C569;110B 1162 11B0; # (앩; 앩; 앩; 앩; 앩; ) HANGUL SYLLABLE AELG +C56A;C56A;110B 1162 11B1;C56A;110B 1162 11B1; # (앪; 앪; 앪; 앪; 앪; ) HANGUL SYLLABLE AELM +C56B;C56B;110B 1162 11B2;C56B;110B 1162 11B2; # (앫; 앫; 앫; 앫; 앫; ) HANGUL SYLLABLE AELB +C56C;C56C;110B 1162 11B3;C56C;110B 1162 11B3; # (앬; 앬; 앬; 앬; 앬; ) HANGUL SYLLABLE AELS +C56D;C56D;110B 1162 11B4;C56D;110B 1162 11B4; # (앭; 앭; 앭; 앭; 앭; ) HANGUL SYLLABLE AELT +C56E;C56E;110B 1162 11B5;C56E;110B 1162 11B5; # (앮; 앮; 앮; 앮; 앮; ) HANGUL SYLLABLE AELP +C56F;C56F;110B 1162 11B6;C56F;110B 1162 11B6; # (앯; 앯; 앯; 앯; 앯; ) HANGUL SYLLABLE AELH +C570;C570;110B 1162 11B7;C570;110B 1162 11B7; # (앰; 앰; 앰; 앰; 앰; ) HANGUL SYLLABLE AEM +C571;C571;110B 1162 11B8;C571;110B 1162 11B8; # (앱; 앱; 앱; 앱; 앱; ) HANGUL SYLLABLE AEB +C572;C572;110B 1162 11B9;C572;110B 1162 11B9; # (앲; 앲; 앲; 앲; 앲; ) HANGUL SYLLABLE AEBS +C573;C573;110B 1162 11BA;C573;110B 1162 11BA; # (앳; 앳; 앳; 앳; 앳; ) HANGUL SYLLABLE AES +C574;C574;110B 1162 11BB;C574;110B 1162 11BB; # (앴; 앴; 앴; 앴; 앴; ) HANGUL SYLLABLE AESS +C575;C575;110B 1162 11BC;C575;110B 1162 11BC; # (앵; 앵; 앵; 앵; 앵; ) HANGUL SYLLABLE AENG +C576;C576;110B 1162 11BD;C576;110B 1162 11BD; # (앶; 앶; 앶; 앶; 앶; ) HANGUL SYLLABLE AEJ +C577;C577;110B 1162 11BE;C577;110B 1162 11BE; # (앷; 앷; 앷; 앷; 앷; ) HANGUL SYLLABLE AEC +C578;C578;110B 1162 11BF;C578;110B 1162 11BF; # (앸; 앸; 앸; 앸; 앸; ) HANGUL SYLLABLE AEK +C579;C579;110B 1162 11C0;C579;110B 1162 11C0; # (앹; 앹; 앹; 앹; 앹; ) HANGUL SYLLABLE AET +C57A;C57A;110B 1162 11C1;C57A;110B 1162 11C1; # (앺; 앺; 앺; 앺; 앺; ) HANGUL SYLLABLE AEP +C57B;C57B;110B 1162 11C2;C57B;110B 1162 11C2; # (앻; 앻; 앻; 앻; 앻; ) HANGUL SYLLABLE AEH +C57C;C57C;110B 1163;C57C;110B 1163; # (야; 야; 야; 야; 야; ) HANGUL SYLLABLE YA +C57D;C57D;110B 1163 11A8;C57D;110B 1163 11A8; # (약; 약; 약; 약; 약; ) HANGUL SYLLABLE YAG +C57E;C57E;110B 1163 11A9;C57E;110B 1163 11A9; # (앾; 앾; 앾; 앾; 앾; ) HANGUL SYLLABLE YAGG +C57F;C57F;110B 1163 11AA;C57F;110B 1163 11AA; # (앿; 앿; 앿; 앿; 앿; ) HANGUL SYLLABLE YAGS +C580;C580;110B 1163 11AB;C580;110B 1163 11AB; # (얀; 얀; 얀; 얀; 얀; ) HANGUL SYLLABLE YAN +C581;C581;110B 1163 11AC;C581;110B 1163 11AC; # (얁; 얁; 얁; 얁; 얁; ) HANGUL SYLLABLE YANJ +C582;C582;110B 1163 11AD;C582;110B 1163 11AD; # (얂; 얂; 얂; 얂; 얂; ) HANGUL SYLLABLE YANH +C583;C583;110B 1163 11AE;C583;110B 1163 11AE; # (얃; 얃; 얃; 얃; 얃; ) HANGUL SYLLABLE YAD +C584;C584;110B 1163 11AF;C584;110B 1163 11AF; # (얄; 얄; 얄; 얄; 얄; ) HANGUL SYLLABLE YAL +C585;C585;110B 1163 11B0;C585;110B 1163 11B0; # (얅; 얅; 얅; 얅; 얅; ) HANGUL SYLLABLE YALG +C586;C586;110B 1163 11B1;C586;110B 1163 11B1; # (얆; 얆; 얆; 얆; 얆; ) HANGUL SYLLABLE YALM +C587;C587;110B 1163 11B2;C587;110B 1163 11B2; # (얇; 얇; 얇; 얇; 얇; ) HANGUL SYLLABLE YALB +C588;C588;110B 1163 11B3;C588;110B 1163 11B3; # (얈; 얈; 얈; 얈; 얈; ) HANGUL SYLLABLE YALS +C589;C589;110B 1163 11B4;C589;110B 1163 11B4; # (얉; 얉; 얉; 얉; 얉; ) HANGUL SYLLABLE YALT +C58A;C58A;110B 1163 11B5;C58A;110B 1163 11B5; # (얊; 얊; 얊; 얊; 얊; ) HANGUL SYLLABLE YALP +C58B;C58B;110B 1163 11B6;C58B;110B 1163 11B6; # (얋; 얋; 얋; 얋; 얋; ) HANGUL SYLLABLE YALH +C58C;C58C;110B 1163 11B7;C58C;110B 1163 11B7; # (얌; 얌; 얌; 얌; 얌; ) HANGUL SYLLABLE YAM +C58D;C58D;110B 1163 11B8;C58D;110B 1163 11B8; # (얍; 얍; 얍; 얍; 얍; ) HANGUL SYLLABLE YAB +C58E;C58E;110B 1163 11B9;C58E;110B 1163 11B9; # (얎; 얎; 얎; 얎; 얎; ) HANGUL SYLLABLE YABS +C58F;C58F;110B 1163 11BA;C58F;110B 1163 11BA; # (얏; 얏; 얏; 얏; 얏; ) HANGUL SYLLABLE YAS +C590;C590;110B 1163 11BB;C590;110B 1163 11BB; # (얐; 얐; 얐; 얐; 얐; ) HANGUL SYLLABLE YASS +C591;C591;110B 1163 11BC;C591;110B 1163 11BC; # (양; 양; 양; 양; 양; ) HANGUL SYLLABLE YANG +C592;C592;110B 1163 11BD;C592;110B 1163 11BD; # (얒; 얒; 얒; 얒; 얒; ) HANGUL SYLLABLE YAJ +C593;C593;110B 1163 11BE;C593;110B 1163 11BE; # (얓; 얓; 얓; 얓; 얓; ) HANGUL SYLLABLE YAC +C594;C594;110B 1163 11BF;C594;110B 1163 11BF; # (얔; 얔; 얔; 얔; 얔; ) HANGUL SYLLABLE YAK +C595;C595;110B 1163 11C0;C595;110B 1163 11C0; # (얕; 얕; 얕; 얕; 얕; ) HANGUL SYLLABLE YAT +C596;C596;110B 1163 11C1;C596;110B 1163 11C1; # (얖; 얖; 얖; 얖; 얖; ) HANGUL SYLLABLE YAP +C597;C597;110B 1163 11C2;C597;110B 1163 11C2; # (얗; 얗; 얗; 얗; 얗; ) HANGUL SYLLABLE YAH +C598;C598;110B 1164;C598;110B 1164; # (얘; 얘; 얘; 얘; 얘; ) HANGUL SYLLABLE YAE +C599;C599;110B 1164 11A8;C599;110B 1164 11A8; # (얙; 얙; 얙; 얙; 얙; ) HANGUL SYLLABLE YAEG +C59A;C59A;110B 1164 11A9;C59A;110B 1164 11A9; # (얚; 얚; 얚; 얚; 얚; ) HANGUL SYLLABLE YAEGG +C59B;C59B;110B 1164 11AA;C59B;110B 1164 11AA; # (얛; 얛; 얛; 얛; 얛; ) HANGUL SYLLABLE YAEGS +C59C;C59C;110B 1164 11AB;C59C;110B 1164 11AB; # (얜; 얜; 얜; 얜; 얜; ) HANGUL SYLLABLE YAEN +C59D;C59D;110B 1164 11AC;C59D;110B 1164 11AC; # (얝; 얝; 얝; 얝; 얝; ) HANGUL SYLLABLE YAENJ +C59E;C59E;110B 1164 11AD;C59E;110B 1164 11AD; # (얞; 얞; 얞; 얞; 얞; ) HANGUL SYLLABLE YAENH +C59F;C59F;110B 1164 11AE;C59F;110B 1164 11AE; # (얟; 얟; 얟; 얟; 얟; ) HANGUL SYLLABLE YAED +C5A0;C5A0;110B 1164 11AF;C5A0;110B 1164 11AF; # (얠; 얠; 얠; 얠; 얠; ) HANGUL SYLLABLE YAEL +C5A1;C5A1;110B 1164 11B0;C5A1;110B 1164 11B0; # (얡; 얡; 얡; 얡; 얡; ) HANGUL SYLLABLE YAELG +C5A2;C5A2;110B 1164 11B1;C5A2;110B 1164 11B1; # (얢; 얢; 얢; 얢; 얢; ) HANGUL SYLLABLE YAELM +C5A3;C5A3;110B 1164 11B2;C5A3;110B 1164 11B2; # (얣; 얣; 얣; 얣; 얣; ) HANGUL SYLLABLE YAELB +C5A4;C5A4;110B 1164 11B3;C5A4;110B 1164 11B3; # (얤; 얤; 얤; 얤; 얤; ) HANGUL SYLLABLE YAELS +C5A5;C5A5;110B 1164 11B4;C5A5;110B 1164 11B4; # (얥; 얥; 얥; 얥; 얥; ) HANGUL SYLLABLE YAELT +C5A6;C5A6;110B 1164 11B5;C5A6;110B 1164 11B5; # (얦; 얦; 얦; 얦; 얦; ) HANGUL SYLLABLE YAELP +C5A7;C5A7;110B 1164 11B6;C5A7;110B 1164 11B6; # (얧; 얧; 얧; 얧; 얧; ) HANGUL SYLLABLE YAELH +C5A8;C5A8;110B 1164 11B7;C5A8;110B 1164 11B7; # (얨; 얨; 얨; 얨; 얨; ) HANGUL SYLLABLE YAEM +C5A9;C5A9;110B 1164 11B8;C5A9;110B 1164 11B8; # (얩; 얩; 얩; 얩; 얩; ) HANGUL SYLLABLE YAEB +C5AA;C5AA;110B 1164 11B9;C5AA;110B 1164 11B9; # (얪; 얪; 얪; 얪; 얪; ) HANGUL SYLLABLE YAEBS +C5AB;C5AB;110B 1164 11BA;C5AB;110B 1164 11BA; # (얫; 얫; 얫; 얫; 얫; ) HANGUL SYLLABLE YAES +C5AC;C5AC;110B 1164 11BB;C5AC;110B 1164 11BB; # (얬; 얬; 얬; 얬; 얬; ) HANGUL SYLLABLE YAESS +C5AD;C5AD;110B 1164 11BC;C5AD;110B 1164 11BC; # (얭; 얭; 얭; 얭; 얭; ) HANGUL SYLLABLE YAENG +C5AE;C5AE;110B 1164 11BD;C5AE;110B 1164 11BD; # (얮; 얮; 얮; 얮; 얮; ) HANGUL SYLLABLE YAEJ +C5AF;C5AF;110B 1164 11BE;C5AF;110B 1164 11BE; # (얯; 얯; 얯; 얯; 얯; ) HANGUL SYLLABLE YAEC +C5B0;C5B0;110B 1164 11BF;C5B0;110B 1164 11BF; # (얰; 얰; 얰; 얰; 얰; ) HANGUL SYLLABLE YAEK +C5B1;C5B1;110B 1164 11C0;C5B1;110B 1164 11C0; # (얱; 얱; 얱; 얱; 얱; ) HANGUL SYLLABLE YAET +C5B2;C5B2;110B 1164 11C1;C5B2;110B 1164 11C1; # (얲; 얲; 얲; 얲; 얲; ) HANGUL SYLLABLE YAEP +C5B3;C5B3;110B 1164 11C2;C5B3;110B 1164 11C2; # (얳; 얳; 얳; 얳; 얳; ) HANGUL SYLLABLE YAEH +C5B4;C5B4;110B 1165;C5B4;110B 1165; # (어; 어; 어; 어; 어; ) HANGUL SYLLABLE EO +C5B5;C5B5;110B 1165 11A8;C5B5;110B 1165 11A8; # (억; 억; 억; 억; 억; ) HANGUL SYLLABLE EOG +C5B6;C5B6;110B 1165 11A9;C5B6;110B 1165 11A9; # (얶; 얶; 얶; 얶; 얶; ) HANGUL SYLLABLE EOGG +C5B7;C5B7;110B 1165 11AA;C5B7;110B 1165 11AA; # (얷; 얷; 얷; 얷; 얷; ) HANGUL SYLLABLE EOGS +C5B8;C5B8;110B 1165 11AB;C5B8;110B 1165 11AB; # (언; 언; 언; 언; 언; ) HANGUL SYLLABLE EON +C5B9;C5B9;110B 1165 11AC;C5B9;110B 1165 11AC; # (얹; 얹; 얹; 얹; 얹; ) HANGUL SYLLABLE EONJ +C5BA;C5BA;110B 1165 11AD;C5BA;110B 1165 11AD; # (얺; 얺; 얺; 얺; 얺; ) HANGUL SYLLABLE EONH +C5BB;C5BB;110B 1165 11AE;C5BB;110B 1165 11AE; # (얻; 얻; 얻; 얻; 얻; ) HANGUL SYLLABLE EOD +C5BC;C5BC;110B 1165 11AF;C5BC;110B 1165 11AF; # (얼; 얼; 얼; 얼; 얼; ) HANGUL SYLLABLE EOL +C5BD;C5BD;110B 1165 11B0;C5BD;110B 1165 11B0; # (얽; 얽; 얽; 얽; 얽; ) HANGUL SYLLABLE EOLG +C5BE;C5BE;110B 1165 11B1;C5BE;110B 1165 11B1; # (얾; 얾; 얾; 얾; 얾; ) HANGUL SYLLABLE EOLM +C5BF;C5BF;110B 1165 11B2;C5BF;110B 1165 11B2; # (얿; 얿; 얿; 얿; 얿; ) HANGUL SYLLABLE EOLB +C5C0;C5C0;110B 1165 11B3;C5C0;110B 1165 11B3; # (엀; 엀; 엀; 엀; 엀; ) HANGUL SYLLABLE EOLS +C5C1;C5C1;110B 1165 11B4;C5C1;110B 1165 11B4; # (엁; 엁; 엁; 엁; 엁; ) HANGUL SYLLABLE EOLT +C5C2;C5C2;110B 1165 11B5;C5C2;110B 1165 11B5; # (엂; 엂; 엂; 엂; 엂; ) HANGUL SYLLABLE EOLP +C5C3;C5C3;110B 1165 11B6;C5C3;110B 1165 11B6; # (엃; 엃; 엃; 엃; 엃; ) HANGUL SYLLABLE EOLH +C5C4;C5C4;110B 1165 11B7;C5C4;110B 1165 11B7; # (엄; 엄; 엄; 엄; 엄; ) HANGUL SYLLABLE EOM +C5C5;C5C5;110B 1165 11B8;C5C5;110B 1165 11B8; # (업; 업; 업; 업; 업; ) HANGUL SYLLABLE EOB +C5C6;C5C6;110B 1165 11B9;C5C6;110B 1165 11B9; # (없; 없; 없; 없; 없; ) HANGUL SYLLABLE EOBS +C5C7;C5C7;110B 1165 11BA;C5C7;110B 1165 11BA; # (엇; 엇; 엇; 엇; 엇; ) HANGUL SYLLABLE EOS +C5C8;C5C8;110B 1165 11BB;C5C8;110B 1165 11BB; # (었; 었; 었; 었; 었; ) HANGUL SYLLABLE EOSS +C5C9;C5C9;110B 1165 11BC;C5C9;110B 1165 11BC; # (엉; 엉; 엉; 엉; 엉; ) HANGUL SYLLABLE EONG +C5CA;C5CA;110B 1165 11BD;C5CA;110B 1165 11BD; # (엊; 엊; 엊; 엊; 엊; ) HANGUL SYLLABLE EOJ +C5CB;C5CB;110B 1165 11BE;C5CB;110B 1165 11BE; # (엋; 엋; 엋; 엋; 엋; ) HANGUL SYLLABLE EOC +C5CC;C5CC;110B 1165 11BF;C5CC;110B 1165 11BF; # (엌; 엌; 엌; 엌; 엌; ) HANGUL SYLLABLE EOK +C5CD;C5CD;110B 1165 11C0;C5CD;110B 1165 11C0; # (엍; 엍; 엍; 엍; 엍; ) HANGUL SYLLABLE EOT +C5CE;C5CE;110B 1165 11C1;C5CE;110B 1165 11C1; # (엎; 엎; 엎; 엎; 엎; ) HANGUL SYLLABLE EOP +C5CF;C5CF;110B 1165 11C2;C5CF;110B 1165 11C2; # (엏; 엏; 엏; 엏; 엏; ) HANGUL SYLLABLE EOH +C5D0;C5D0;110B 1166;C5D0;110B 1166; # (에; 에; 에; 에; 에; ) HANGUL SYLLABLE E +C5D1;C5D1;110B 1166 11A8;C5D1;110B 1166 11A8; # (엑; 엑; 엑; 엑; 엑; ) HANGUL SYLLABLE EG +C5D2;C5D2;110B 1166 11A9;C5D2;110B 1166 11A9; # (엒; 엒; 엒; 엒; 엒; ) HANGUL SYLLABLE EGG +C5D3;C5D3;110B 1166 11AA;C5D3;110B 1166 11AA; # (엓; 엓; 엓; 엓; 엓; ) HANGUL SYLLABLE EGS +C5D4;C5D4;110B 1166 11AB;C5D4;110B 1166 11AB; # (엔; 엔; 엔; 엔; 엔; ) HANGUL SYLLABLE EN +C5D5;C5D5;110B 1166 11AC;C5D5;110B 1166 11AC; # (엕; 엕; 엕; 엕; 엕; ) HANGUL SYLLABLE ENJ +C5D6;C5D6;110B 1166 11AD;C5D6;110B 1166 11AD; # (엖; 엖; 엖; 엖; 엖; ) HANGUL SYLLABLE ENH +C5D7;C5D7;110B 1166 11AE;C5D7;110B 1166 11AE; # (엗; 엗; 엗; 엗; 엗; ) HANGUL SYLLABLE ED +C5D8;C5D8;110B 1166 11AF;C5D8;110B 1166 11AF; # (엘; 엘; 엘; 엘; 엘; ) HANGUL SYLLABLE EL +C5D9;C5D9;110B 1166 11B0;C5D9;110B 1166 11B0; # (엙; 엙; 엙; 엙; 엙; ) HANGUL SYLLABLE ELG +C5DA;C5DA;110B 1166 11B1;C5DA;110B 1166 11B1; # (엚; 엚; 엚; 엚; 엚; ) HANGUL SYLLABLE ELM +C5DB;C5DB;110B 1166 11B2;C5DB;110B 1166 11B2; # (엛; 엛; 엛; 엛; 엛; ) HANGUL SYLLABLE ELB +C5DC;C5DC;110B 1166 11B3;C5DC;110B 1166 11B3; # (엜; 엜; 엜; 엜; 엜; ) HANGUL SYLLABLE ELS +C5DD;C5DD;110B 1166 11B4;C5DD;110B 1166 11B4; # (엝; 엝; 엝; 엝; 엝; ) HANGUL SYLLABLE ELT +C5DE;C5DE;110B 1166 11B5;C5DE;110B 1166 11B5; # (엞; 엞; 엞; 엞; 엞; ) HANGUL SYLLABLE ELP +C5DF;C5DF;110B 1166 11B6;C5DF;110B 1166 11B6; # (엟; 엟; 엟; 엟; 엟; ) HANGUL SYLLABLE ELH +C5E0;C5E0;110B 1166 11B7;C5E0;110B 1166 11B7; # (엠; 엠; 엠; 엠; 엠; ) HANGUL SYLLABLE EM +C5E1;C5E1;110B 1166 11B8;C5E1;110B 1166 11B8; # (엡; 엡; 엡; 엡; 엡; ) HANGUL SYLLABLE EB +C5E2;C5E2;110B 1166 11B9;C5E2;110B 1166 11B9; # (엢; 엢; 엢; 엢; 엢; ) HANGUL SYLLABLE EBS +C5E3;C5E3;110B 1166 11BA;C5E3;110B 1166 11BA; # (엣; 엣; 엣; 엣; 엣; ) HANGUL SYLLABLE ES +C5E4;C5E4;110B 1166 11BB;C5E4;110B 1166 11BB; # (엤; 엤; 엤; 엤; 엤; ) HANGUL SYLLABLE ESS +C5E5;C5E5;110B 1166 11BC;C5E5;110B 1166 11BC; # (엥; 엥; 엥; 엥; 엥; ) HANGUL SYLLABLE ENG +C5E6;C5E6;110B 1166 11BD;C5E6;110B 1166 11BD; # (엦; 엦; 엦; 엦; 엦; ) HANGUL SYLLABLE EJ +C5E7;C5E7;110B 1166 11BE;C5E7;110B 1166 11BE; # (엧; 엧; 엧; 엧; 엧; ) HANGUL SYLLABLE EC +C5E8;C5E8;110B 1166 11BF;C5E8;110B 1166 11BF; # (엨; 엨; 엨; 엨; 엨; ) HANGUL SYLLABLE EK +C5E9;C5E9;110B 1166 11C0;C5E9;110B 1166 11C0; # (엩; 엩; 엩; 엩; 엩; ) HANGUL SYLLABLE ET +C5EA;C5EA;110B 1166 11C1;C5EA;110B 1166 11C1; # (엪; 엪; 엪; 엪; 엪; ) HANGUL SYLLABLE EP +C5EB;C5EB;110B 1166 11C2;C5EB;110B 1166 11C2; # (엫; 엫; 엫; 엫; 엫; ) HANGUL SYLLABLE EH +C5EC;C5EC;110B 1167;C5EC;110B 1167; # (여; 여; 여; 여; 여; ) HANGUL SYLLABLE YEO +C5ED;C5ED;110B 1167 11A8;C5ED;110B 1167 11A8; # (역; 역; 역; 역; 역; ) HANGUL SYLLABLE YEOG +C5EE;C5EE;110B 1167 11A9;C5EE;110B 1167 11A9; # (엮; 엮; 엮; 엮; 엮; ) HANGUL SYLLABLE YEOGG +C5EF;C5EF;110B 1167 11AA;C5EF;110B 1167 11AA; # (엯; 엯; 엯; 엯; 엯; ) HANGUL SYLLABLE YEOGS +C5F0;C5F0;110B 1167 11AB;C5F0;110B 1167 11AB; # (연; 연; 연; 연; 연; ) HANGUL SYLLABLE YEON +C5F1;C5F1;110B 1167 11AC;C5F1;110B 1167 11AC; # (엱; 엱; 엱; 엱; 엱; ) HANGUL SYLLABLE YEONJ +C5F2;C5F2;110B 1167 11AD;C5F2;110B 1167 11AD; # (엲; 엲; 엲; 엲; 엲; ) HANGUL SYLLABLE YEONH +C5F3;C5F3;110B 1167 11AE;C5F3;110B 1167 11AE; # (엳; 엳; 엳; 엳; 엳; ) HANGUL SYLLABLE YEOD +C5F4;C5F4;110B 1167 11AF;C5F4;110B 1167 11AF; # (열; 열; 열; 열; 열; ) HANGUL SYLLABLE YEOL +C5F5;C5F5;110B 1167 11B0;C5F5;110B 1167 11B0; # (엵; 엵; 엵; 엵; 엵; ) HANGUL SYLLABLE YEOLG +C5F6;C5F6;110B 1167 11B1;C5F6;110B 1167 11B1; # (엶; 엶; 엶; 엶; 엶; ) HANGUL SYLLABLE YEOLM +C5F7;C5F7;110B 1167 11B2;C5F7;110B 1167 11B2; # (엷; 엷; 엷; 엷; 엷; ) HANGUL SYLLABLE YEOLB +C5F8;C5F8;110B 1167 11B3;C5F8;110B 1167 11B3; # (엸; 엸; 엸; 엸; 엸; ) HANGUL SYLLABLE YEOLS +C5F9;C5F9;110B 1167 11B4;C5F9;110B 1167 11B4; # (엹; 엹; 엹; 엹; 엹; ) HANGUL SYLLABLE YEOLT +C5FA;C5FA;110B 1167 11B5;C5FA;110B 1167 11B5; # (엺; 엺; 엺; 엺; 엺; ) HANGUL SYLLABLE YEOLP +C5FB;C5FB;110B 1167 11B6;C5FB;110B 1167 11B6; # (엻; 엻; 엻; 엻; 엻; ) HANGUL SYLLABLE YEOLH +C5FC;C5FC;110B 1167 11B7;C5FC;110B 1167 11B7; # (염; 염; 염; 염; 염; ) HANGUL SYLLABLE YEOM +C5FD;C5FD;110B 1167 11B8;C5FD;110B 1167 11B8; # (엽; 엽; 엽; 엽; 엽; ) HANGUL SYLLABLE YEOB +C5FE;C5FE;110B 1167 11B9;C5FE;110B 1167 11B9; # (엾; 엾; 엾; 엾; 엾; ) HANGUL SYLLABLE YEOBS +C5FF;C5FF;110B 1167 11BA;C5FF;110B 1167 11BA; # (엿; 엿; 엿; 엿; 엿; ) HANGUL SYLLABLE YEOS +C600;C600;110B 1167 11BB;C600;110B 1167 11BB; # (였; 였; 였; 였; 였; ) HANGUL SYLLABLE YEOSS +C601;C601;110B 1167 11BC;C601;110B 1167 11BC; # (영; 영; 영; 영; 영; ) HANGUL SYLLABLE YEONG +C602;C602;110B 1167 11BD;C602;110B 1167 11BD; # (옂; 옂; 옂; 옂; 옂; ) HANGUL SYLLABLE YEOJ +C603;C603;110B 1167 11BE;C603;110B 1167 11BE; # (옃; 옃; 옃; 옃; 옃; ) HANGUL SYLLABLE YEOC +C604;C604;110B 1167 11BF;C604;110B 1167 11BF; # (옄; 옄; 옄; 옄; 옄; ) HANGUL SYLLABLE YEOK +C605;C605;110B 1167 11C0;C605;110B 1167 11C0; # (옅; 옅; 옅; 옅; 옅; ) HANGUL SYLLABLE YEOT +C606;C606;110B 1167 11C1;C606;110B 1167 11C1; # (옆; 옆; 옆; 옆; 옆; ) HANGUL SYLLABLE YEOP +C607;C607;110B 1167 11C2;C607;110B 1167 11C2; # (옇; 옇; 옇; 옇; 옇; ) HANGUL SYLLABLE YEOH +C608;C608;110B 1168;C608;110B 1168; # (예; 예; 예; 예; 예; ) HANGUL SYLLABLE YE +C609;C609;110B 1168 11A8;C609;110B 1168 11A8; # (옉; 옉; 옉; 옉; 옉; ) HANGUL SYLLABLE YEG +C60A;C60A;110B 1168 11A9;C60A;110B 1168 11A9; # (옊; 옊; 옊; 옊; 옊; ) HANGUL SYLLABLE YEGG +C60B;C60B;110B 1168 11AA;C60B;110B 1168 11AA; # (옋; 옋; 옋; 옋; 옋; ) HANGUL SYLLABLE YEGS +C60C;C60C;110B 1168 11AB;C60C;110B 1168 11AB; # (옌; 옌; 옌; 옌; 옌; ) HANGUL SYLLABLE YEN +C60D;C60D;110B 1168 11AC;C60D;110B 1168 11AC; # (옍; 옍; 옍; 옍; 옍; ) HANGUL SYLLABLE YENJ +C60E;C60E;110B 1168 11AD;C60E;110B 1168 11AD; # (옎; 옎; 옎; 옎; 옎; ) HANGUL SYLLABLE YENH +C60F;C60F;110B 1168 11AE;C60F;110B 1168 11AE; # (옏; 옏; 옏; 옏; 옏; ) HANGUL SYLLABLE YED +C610;C610;110B 1168 11AF;C610;110B 1168 11AF; # (옐; 옐; 옐; 옐; 옐; ) HANGUL SYLLABLE YEL +C611;C611;110B 1168 11B0;C611;110B 1168 11B0; # (옑; 옑; 옑; 옑; 옑; ) HANGUL SYLLABLE YELG +C612;C612;110B 1168 11B1;C612;110B 1168 11B1; # (옒; 옒; 옒; 옒; 옒; ) HANGUL SYLLABLE YELM +C613;C613;110B 1168 11B2;C613;110B 1168 11B2; # (옓; 옓; 옓; 옓; 옓; ) HANGUL SYLLABLE YELB +C614;C614;110B 1168 11B3;C614;110B 1168 11B3; # (옔; 옔; 옔; 옔; 옔; ) HANGUL SYLLABLE YELS +C615;C615;110B 1168 11B4;C615;110B 1168 11B4; # (옕; 옕; 옕; 옕; 옕; ) HANGUL SYLLABLE YELT +C616;C616;110B 1168 11B5;C616;110B 1168 11B5; # (옖; 옖; 옖; 옖; 옖; ) HANGUL SYLLABLE YELP +C617;C617;110B 1168 11B6;C617;110B 1168 11B6; # (옗; 옗; 옗; 옗; 옗; ) HANGUL SYLLABLE YELH +C618;C618;110B 1168 11B7;C618;110B 1168 11B7; # (옘; 옘; 옘; 옘; 옘; ) HANGUL SYLLABLE YEM +C619;C619;110B 1168 11B8;C619;110B 1168 11B8; # (옙; 옙; 옙; 옙; 옙; ) HANGUL SYLLABLE YEB +C61A;C61A;110B 1168 11B9;C61A;110B 1168 11B9; # (옚; 옚; 옚; 옚; 옚; ) HANGUL SYLLABLE YEBS +C61B;C61B;110B 1168 11BA;C61B;110B 1168 11BA; # (옛; 옛; 옛; 옛; 옛; ) HANGUL SYLLABLE YES +C61C;C61C;110B 1168 11BB;C61C;110B 1168 11BB; # (옜; 옜; 옜; 옜; 옜; ) HANGUL SYLLABLE YESS +C61D;C61D;110B 1168 11BC;C61D;110B 1168 11BC; # (옝; 옝; 옝; 옝; 옝; ) HANGUL SYLLABLE YENG +C61E;C61E;110B 1168 11BD;C61E;110B 1168 11BD; # (옞; 옞; 옞; 옞; 옞; ) HANGUL SYLLABLE YEJ +C61F;C61F;110B 1168 11BE;C61F;110B 1168 11BE; # (옟; 옟; 옟; 옟; 옟; ) HANGUL SYLLABLE YEC +C620;C620;110B 1168 11BF;C620;110B 1168 11BF; # (옠; 옠; 옠; 옠; 옠; ) HANGUL SYLLABLE YEK +C621;C621;110B 1168 11C0;C621;110B 1168 11C0; # (옡; 옡; 옡; 옡; 옡; ) HANGUL SYLLABLE YET +C622;C622;110B 1168 11C1;C622;110B 1168 11C1; # (옢; 옢; 옢; 옢; 옢; ) HANGUL SYLLABLE YEP +C623;C623;110B 1168 11C2;C623;110B 1168 11C2; # (옣; 옣; 옣; 옣; 옣; ) HANGUL SYLLABLE YEH +C624;C624;110B 1169;C624;110B 1169; # (오; 오; 오; 오; 오; ) HANGUL SYLLABLE O +C625;C625;110B 1169 11A8;C625;110B 1169 11A8; # (옥; 옥; 옥; 옥; 옥; ) HANGUL SYLLABLE OG +C626;C626;110B 1169 11A9;C626;110B 1169 11A9; # (옦; 옦; 옦; 옦; 옦; ) HANGUL SYLLABLE OGG +C627;C627;110B 1169 11AA;C627;110B 1169 11AA; # (옧; 옧; 옧; 옧; 옧; ) HANGUL SYLLABLE OGS +C628;C628;110B 1169 11AB;C628;110B 1169 11AB; # (온; 온; 온; 온; 온; ) HANGUL SYLLABLE ON +C629;C629;110B 1169 11AC;C629;110B 1169 11AC; # (옩; 옩; 옩; 옩; 옩; ) HANGUL SYLLABLE ONJ +C62A;C62A;110B 1169 11AD;C62A;110B 1169 11AD; # (옪; 옪; 옪; 옪; 옪; ) HANGUL SYLLABLE ONH +C62B;C62B;110B 1169 11AE;C62B;110B 1169 11AE; # (옫; 옫; 옫; 옫; 옫; ) HANGUL SYLLABLE OD +C62C;C62C;110B 1169 11AF;C62C;110B 1169 11AF; # (올; 올; 올; 올; 올; ) HANGUL SYLLABLE OL +C62D;C62D;110B 1169 11B0;C62D;110B 1169 11B0; # (옭; 옭; 옭; 옭; 옭; ) HANGUL SYLLABLE OLG +C62E;C62E;110B 1169 11B1;C62E;110B 1169 11B1; # (옮; 옮; 옮; 옮; 옮; ) HANGUL SYLLABLE OLM +C62F;C62F;110B 1169 11B2;C62F;110B 1169 11B2; # (옯; 옯; 옯; 옯; 옯; ) HANGUL SYLLABLE OLB +C630;C630;110B 1169 11B3;C630;110B 1169 11B3; # (옰; 옰; 옰; 옰; 옰; ) HANGUL SYLLABLE OLS +C631;C631;110B 1169 11B4;C631;110B 1169 11B4; # (옱; 옱; 옱; 옱; 옱; ) HANGUL SYLLABLE OLT +C632;C632;110B 1169 11B5;C632;110B 1169 11B5; # (옲; 옲; 옲; 옲; 옲; ) HANGUL SYLLABLE OLP +C633;C633;110B 1169 11B6;C633;110B 1169 11B6; # (옳; 옳; 옳; 옳; 옳; ) HANGUL SYLLABLE OLH +C634;C634;110B 1169 11B7;C634;110B 1169 11B7; # (옴; 옴; 옴; 옴; 옴; ) HANGUL SYLLABLE OM +C635;C635;110B 1169 11B8;C635;110B 1169 11B8; # (옵; 옵; 옵; 옵; 옵; ) HANGUL SYLLABLE OB +C636;C636;110B 1169 11B9;C636;110B 1169 11B9; # (옶; 옶; 옶; 옶; 옶; ) HANGUL SYLLABLE OBS +C637;C637;110B 1169 11BA;C637;110B 1169 11BA; # (옷; 옷; 옷; 옷; 옷; ) HANGUL SYLLABLE OS +C638;C638;110B 1169 11BB;C638;110B 1169 11BB; # (옸; 옸; 옸; 옸; 옸; ) HANGUL SYLLABLE OSS +C639;C639;110B 1169 11BC;C639;110B 1169 11BC; # (옹; 옹; 옹; 옹; 옹; ) HANGUL SYLLABLE ONG +C63A;C63A;110B 1169 11BD;C63A;110B 1169 11BD; # (옺; 옺; 옺; 옺; 옺; ) HANGUL SYLLABLE OJ +C63B;C63B;110B 1169 11BE;C63B;110B 1169 11BE; # (옻; 옻; 옻; 옻; 옻; ) HANGUL SYLLABLE OC +C63C;C63C;110B 1169 11BF;C63C;110B 1169 11BF; # (옼; 옼; 옼; 옼; 옼; ) HANGUL SYLLABLE OK +C63D;C63D;110B 1169 11C0;C63D;110B 1169 11C0; # (옽; 옽; 옽; 옽; 옽; ) HANGUL SYLLABLE OT +C63E;C63E;110B 1169 11C1;C63E;110B 1169 11C1; # (옾; 옾; 옾; 옾; 옾; ) HANGUL SYLLABLE OP +C63F;C63F;110B 1169 11C2;C63F;110B 1169 11C2; # (옿; 옿; 옿; 옿; 옿; ) HANGUL SYLLABLE OH +C640;C640;110B 116A;C640;110B 116A; # (와; 와; 와; 와; 와; ) HANGUL SYLLABLE WA +C641;C641;110B 116A 11A8;C641;110B 116A 11A8; # (왁; 왁; 왁; 왁; 왁; ) HANGUL SYLLABLE WAG +C642;C642;110B 116A 11A9;C642;110B 116A 11A9; # (왂; 왂; 왂; 왂; 왂; ) HANGUL SYLLABLE WAGG +C643;C643;110B 116A 11AA;C643;110B 116A 11AA; # (왃; 왃; 왃; 왃; 왃; ) HANGUL SYLLABLE WAGS +C644;C644;110B 116A 11AB;C644;110B 116A 11AB; # (완; 완; 완; 완; 완; ) HANGUL SYLLABLE WAN +C645;C645;110B 116A 11AC;C645;110B 116A 11AC; # (왅; 왅; 왅; 왅; 왅; ) HANGUL SYLLABLE WANJ +C646;C646;110B 116A 11AD;C646;110B 116A 11AD; # (왆; 왆; 왆; 왆; 왆; ) HANGUL SYLLABLE WANH +C647;C647;110B 116A 11AE;C647;110B 116A 11AE; # (왇; 왇; 왇; 왇; 왇; ) HANGUL SYLLABLE WAD +C648;C648;110B 116A 11AF;C648;110B 116A 11AF; # (왈; 왈; 왈; 왈; 왈; ) HANGUL SYLLABLE WAL +C649;C649;110B 116A 11B0;C649;110B 116A 11B0; # (왉; 왉; 왉; 왉; 왉; ) HANGUL SYLLABLE WALG +C64A;C64A;110B 116A 11B1;C64A;110B 116A 11B1; # (왊; 왊; 왊; 왊; 왊; ) HANGUL SYLLABLE WALM +C64B;C64B;110B 116A 11B2;C64B;110B 116A 11B2; # (왋; 왋; 왋; 왋; 왋; ) HANGUL SYLLABLE WALB +C64C;C64C;110B 116A 11B3;C64C;110B 116A 11B3; # (왌; 왌; 왌; 왌; 왌; ) HANGUL SYLLABLE WALS +C64D;C64D;110B 116A 11B4;C64D;110B 116A 11B4; # (왍; 왍; 왍; 왍; 왍; ) HANGUL SYLLABLE WALT +C64E;C64E;110B 116A 11B5;C64E;110B 116A 11B5; # (왎; 왎; 왎; 왎; 왎; ) HANGUL SYLLABLE WALP +C64F;C64F;110B 116A 11B6;C64F;110B 116A 11B6; # (왏; 왏; 왏; 왏; 왏; ) HANGUL SYLLABLE WALH +C650;C650;110B 116A 11B7;C650;110B 116A 11B7; # (왐; 왐; 왐; 왐; 왐; ) HANGUL SYLLABLE WAM +C651;C651;110B 116A 11B8;C651;110B 116A 11B8; # (왑; 왑; 왑; 왑; 왑; ) HANGUL SYLLABLE WAB +C652;C652;110B 116A 11B9;C652;110B 116A 11B9; # (왒; 왒; 왒; 왒; 왒; ) HANGUL SYLLABLE WABS +C653;C653;110B 116A 11BA;C653;110B 116A 11BA; # (왓; 왓; 왓; 왓; 왓; ) HANGUL SYLLABLE WAS +C654;C654;110B 116A 11BB;C654;110B 116A 11BB; # (왔; 왔; 왔; 왔; 왔; ) HANGUL SYLLABLE WASS +C655;C655;110B 116A 11BC;C655;110B 116A 11BC; # (왕; 왕; 왕; 왕; 왕; ) HANGUL SYLLABLE WANG +C656;C656;110B 116A 11BD;C656;110B 116A 11BD; # (왖; 왖; 왖; 왖; 왖; ) HANGUL SYLLABLE WAJ +C657;C657;110B 116A 11BE;C657;110B 116A 11BE; # (왗; 왗; 왗; 왗; 왗; ) HANGUL SYLLABLE WAC +C658;C658;110B 116A 11BF;C658;110B 116A 11BF; # (왘; 왘; 왘; 왘; 왘; ) HANGUL SYLLABLE WAK +C659;C659;110B 116A 11C0;C659;110B 116A 11C0; # (왙; 왙; 왙; 왙; 왙; ) HANGUL SYLLABLE WAT +C65A;C65A;110B 116A 11C1;C65A;110B 116A 11C1; # (왚; 왚; 왚; 왚; 왚; ) HANGUL SYLLABLE WAP +C65B;C65B;110B 116A 11C2;C65B;110B 116A 11C2; # (왛; 왛; 왛; 왛; 왛; ) HANGUL SYLLABLE WAH +C65C;C65C;110B 116B;C65C;110B 116B; # (왜; 왜; 왜; 왜; 왜; ) HANGUL SYLLABLE WAE +C65D;C65D;110B 116B 11A8;C65D;110B 116B 11A8; # (왝; 왝; 왝; 왝; 왝; ) HANGUL SYLLABLE WAEG +C65E;C65E;110B 116B 11A9;C65E;110B 116B 11A9; # (왞; 왞; 왞; 왞; 왞; ) HANGUL SYLLABLE WAEGG +C65F;C65F;110B 116B 11AA;C65F;110B 116B 11AA; # (왟; 왟; 왟; 왟; 왟; ) HANGUL SYLLABLE WAEGS +C660;C660;110B 116B 11AB;C660;110B 116B 11AB; # (왠; 왠; 왠; 왠; 왠; ) HANGUL SYLLABLE WAEN +C661;C661;110B 116B 11AC;C661;110B 116B 11AC; # (왡; 왡; 왡; 왡; 왡; ) HANGUL SYLLABLE WAENJ +C662;C662;110B 116B 11AD;C662;110B 116B 11AD; # (왢; 왢; 왢; 왢; 왢; ) HANGUL SYLLABLE WAENH +C663;C663;110B 116B 11AE;C663;110B 116B 11AE; # (왣; 왣; 왣; 왣; 왣; ) HANGUL SYLLABLE WAED +C664;C664;110B 116B 11AF;C664;110B 116B 11AF; # (왤; 왤; 왤; 왤; 왤; ) HANGUL SYLLABLE WAEL +C665;C665;110B 116B 11B0;C665;110B 116B 11B0; # (왥; 왥; 왥; 왥; 왥; ) HANGUL SYLLABLE WAELG +C666;C666;110B 116B 11B1;C666;110B 116B 11B1; # (왦; 왦; 왦; 왦; 왦; ) HANGUL SYLLABLE WAELM +C667;C667;110B 116B 11B2;C667;110B 116B 11B2; # (왧; 왧; 왧; 왧; 왧; ) HANGUL SYLLABLE WAELB +C668;C668;110B 116B 11B3;C668;110B 116B 11B3; # (왨; 왨; 왨; 왨; 왨; ) HANGUL SYLLABLE WAELS +C669;C669;110B 116B 11B4;C669;110B 116B 11B4; # (왩; 왩; 왩; 왩; 왩; ) HANGUL SYLLABLE WAELT +C66A;C66A;110B 116B 11B5;C66A;110B 116B 11B5; # (왪; 왪; 왪; 왪; 왪; ) HANGUL SYLLABLE WAELP +C66B;C66B;110B 116B 11B6;C66B;110B 116B 11B6; # (왫; 왫; 왫; 왫; 왫; ) HANGUL SYLLABLE WAELH +C66C;C66C;110B 116B 11B7;C66C;110B 116B 11B7; # (왬; 왬; 왬; 왬; 왬; ) HANGUL SYLLABLE WAEM +C66D;C66D;110B 116B 11B8;C66D;110B 116B 11B8; # (왭; 왭; 왭; 왭; 왭; ) HANGUL SYLLABLE WAEB +C66E;C66E;110B 116B 11B9;C66E;110B 116B 11B9; # (왮; 왮; 왮; 왮; 왮; ) HANGUL SYLLABLE WAEBS +C66F;C66F;110B 116B 11BA;C66F;110B 116B 11BA; # (왯; 왯; 왯; 왯; 왯; ) HANGUL SYLLABLE WAES +C670;C670;110B 116B 11BB;C670;110B 116B 11BB; # (왰; 왰; 왰; 왰; 왰; ) HANGUL SYLLABLE WAESS +C671;C671;110B 116B 11BC;C671;110B 116B 11BC; # (왱; 왱; 왱; 왱; 왱; ) HANGUL SYLLABLE WAENG +C672;C672;110B 116B 11BD;C672;110B 116B 11BD; # (왲; 왲; 왲; 왲; 왲; ) HANGUL SYLLABLE WAEJ +C673;C673;110B 116B 11BE;C673;110B 116B 11BE; # (왳; 왳; 왳; 왳; 왳; ) HANGUL SYLLABLE WAEC +C674;C674;110B 116B 11BF;C674;110B 116B 11BF; # (왴; 왴; 왴; 왴; 왴; ) HANGUL SYLLABLE WAEK +C675;C675;110B 116B 11C0;C675;110B 116B 11C0; # (왵; 왵; 왵; 왵; 왵; ) HANGUL SYLLABLE WAET +C676;C676;110B 116B 11C1;C676;110B 116B 11C1; # (왶; 왶; 왶; 왶; 왶; ) HANGUL SYLLABLE WAEP +C677;C677;110B 116B 11C2;C677;110B 116B 11C2; # (왷; 왷; 왷; 왷; 왷; ) HANGUL SYLLABLE WAEH +C678;C678;110B 116C;C678;110B 116C; # (외; 외; 외; 외; 외; ) HANGUL SYLLABLE OE +C679;C679;110B 116C 11A8;C679;110B 116C 11A8; # (왹; 왹; 왹; 왹; 왹; ) HANGUL SYLLABLE OEG +C67A;C67A;110B 116C 11A9;C67A;110B 116C 11A9; # (왺; 왺; 왺; 왺; 왺; ) HANGUL SYLLABLE OEGG +C67B;C67B;110B 116C 11AA;C67B;110B 116C 11AA; # (왻; 왻; 왻; 왻; 왻; ) HANGUL SYLLABLE OEGS +C67C;C67C;110B 116C 11AB;C67C;110B 116C 11AB; # (왼; 왼; 왼; 왼; 왼; ) HANGUL SYLLABLE OEN +C67D;C67D;110B 116C 11AC;C67D;110B 116C 11AC; # (왽; 왽; 왽; 왽; 왽; ) HANGUL SYLLABLE OENJ +C67E;C67E;110B 116C 11AD;C67E;110B 116C 11AD; # (왾; 왾; 왾; 왾; 왾; ) HANGUL SYLLABLE OENH +C67F;C67F;110B 116C 11AE;C67F;110B 116C 11AE; # (왿; 왿; 왿; 왿; 왿; ) HANGUL SYLLABLE OED +C680;C680;110B 116C 11AF;C680;110B 116C 11AF; # (욀; 욀; 욀; 욀; 욀; ) HANGUL SYLLABLE OEL +C681;C681;110B 116C 11B0;C681;110B 116C 11B0; # (욁; 욁; 욁; 욁; 욁; ) HANGUL SYLLABLE OELG +C682;C682;110B 116C 11B1;C682;110B 116C 11B1; # (욂; 욂; 욂; 욂; 욂; ) HANGUL SYLLABLE OELM +C683;C683;110B 116C 11B2;C683;110B 116C 11B2; # (욃; 욃; 욃; 욃; 욃; ) HANGUL SYLLABLE OELB +C684;C684;110B 116C 11B3;C684;110B 116C 11B3; # (욄; 욄; 욄; 욄; 욄; ) HANGUL SYLLABLE OELS +C685;C685;110B 116C 11B4;C685;110B 116C 11B4; # (욅; 욅; 욅; 욅; 욅; ) HANGUL SYLLABLE OELT +C686;C686;110B 116C 11B5;C686;110B 116C 11B5; # (욆; 욆; 욆; 욆; 욆; ) HANGUL SYLLABLE OELP +C687;C687;110B 116C 11B6;C687;110B 116C 11B6; # (욇; 욇; 욇; 욇; 욇; ) HANGUL SYLLABLE OELH +C688;C688;110B 116C 11B7;C688;110B 116C 11B7; # (욈; 욈; 욈; 욈; 욈; ) HANGUL SYLLABLE OEM +C689;C689;110B 116C 11B8;C689;110B 116C 11B8; # (욉; 욉; 욉; 욉; 욉; ) HANGUL SYLLABLE OEB +C68A;C68A;110B 116C 11B9;C68A;110B 116C 11B9; # (욊; 욊; 욊; 욊; 욊; ) HANGUL SYLLABLE OEBS +C68B;C68B;110B 116C 11BA;C68B;110B 116C 11BA; # (욋; 욋; 욋; 욋; 욋; ) HANGUL SYLLABLE OES +C68C;C68C;110B 116C 11BB;C68C;110B 116C 11BB; # (욌; 욌; 욌; 욌; 욌; ) HANGUL SYLLABLE OESS +C68D;C68D;110B 116C 11BC;C68D;110B 116C 11BC; # (욍; 욍; 욍; 욍; 욍; ) HANGUL SYLLABLE OENG +C68E;C68E;110B 116C 11BD;C68E;110B 116C 11BD; # (욎; 욎; 욎; 욎; 욎; ) HANGUL SYLLABLE OEJ +C68F;C68F;110B 116C 11BE;C68F;110B 116C 11BE; # (욏; 욏; 욏; 욏; 욏; ) HANGUL SYLLABLE OEC +C690;C690;110B 116C 11BF;C690;110B 116C 11BF; # (욐; 욐; 욐; 욐; 욐; ) HANGUL SYLLABLE OEK +C691;C691;110B 116C 11C0;C691;110B 116C 11C0; # (욑; 욑; 욑; 욑; 욑; ) HANGUL SYLLABLE OET +C692;C692;110B 116C 11C1;C692;110B 116C 11C1; # (욒; 욒; 욒; 욒; 욒; ) HANGUL SYLLABLE OEP +C693;C693;110B 116C 11C2;C693;110B 116C 11C2; # (욓; 욓; 욓; 욓; 욓; ) HANGUL SYLLABLE OEH +C694;C694;110B 116D;C694;110B 116D; # (요; 요; 요; 요; 요; ) HANGUL SYLLABLE YO +C695;C695;110B 116D 11A8;C695;110B 116D 11A8; # (욕; 욕; 욕; 욕; 욕; ) HANGUL SYLLABLE YOG +C696;C696;110B 116D 11A9;C696;110B 116D 11A9; # (욖; 욖; 욖; 욖; 욖; ) HANGUL SYLLABLE YOGG +C697;C697;110B 116D 11AA;C697;110B 116D 11AA; # (욗; 욗; 욗; 욗; 욗; ) HANGUL SYLLABLE YOGS +C698;C698;110B 116D 11AB;C698;110B 116D 11AB; # (욘; 욘; 욘; 욘; 욘; ) HANGUL SYLLABLE YON +C699;C699;110B 116D 11AC;C699;110B 116D 11AC; # (욙; 욙; 욙; 욙; 욙; ) HANGUL SYLLABLE YONJ +C69A;C69A;110B 116D 11AD;C69A;110B 116D 11AD; # (욚; 욚; 욚; 욚; 욚; ) HANGUL SYLLABLE YONH +C69B;C69B;110B 116D 11AE;C69B;110B 116D 11AE; # (욛; 욛; 욛; 욛; 욛; ) HANGUL SYLLABLE YOD +C69C;C69C;110B 116D 11AF;C69C;110B 116D 11AF; # (욜; 욜; 욜; 욜; 욜; ) HANGUL SYLLABLE YOL +C69D;C69D;110B 116D 11B0;C69D;110B 116D 11B0; # (욝; 욝; 욝; 욝; 욝; ) HANGUL SYLLABLE YOLG +C69E;C69E;110B 116D 11B1;C69E;110B 116D 11B1; # (욞; 욞; 욞; 욞; 욞; ) HANGUL SYLLABLE YOLM +C69F;C69F;110B 116D 11B2;C69F;110B 116D 11B2; # (욟; 욟; 욟; 욟; 욟; ) HANGUL SYLLABLE YOLB +C6A0;C6A0;110B 116D 11B3;C6A0;110B 116D 11B3; # (욠; 욠; 욠; 욠; 욠; ) HANGUL SYLLABLE YOLS +C6A1;C6A1;110B 116D 11B4;C6A1;110B 116D 11B4; # (욡; 욡; 욡; 욡; 욡; ) HANGUL SYLLABLE YOLT +C6A2;C6A2;110B 116D 11B5;C6A2;110B 116D 11B5; # (욢; 욢; 욢; 욢; 욢; ) HANGUL SYLLABLE YOLP +C6A3;C6A3;110B 116D 11B6;C6A3;110B 116D 11B6; # (욣; 욣; 욣; 욣; 욣; ) HANGUL SYLLABLE YOLH +C6A4;C6A4;110B 116D 11B7;C6A4;110B 116D 11B7; # (욤; 욤; 욤; 욤; 욤; ) HANGUL SYLLABLE YOM +C6A5;C6A5;110B 116D 11B8;C6A5;110B 116D 11B8; # (욥; 욥; 욥; 욥; 욥; ) HANGUL SYLLABLE YOB +C6A6;C6A6;110B 116D 11B9;C6A6;110B 116D 11B9; # (욦; 욦; 욦; 욦; 욦; ) HANGUL SYLLABLE YOBS +C6A7;C6A7;110B 116D 11BA;C6A7;110B 116D 11BA; # (욧; 욧; 욧; 욧; 욧; ) HANGUL SYLLABLE YOS +C6A8;C6A8;110B 116D 11BB;C6A8;110B 116D 11BB; # (욨; 욨; 욨; 욨; 욨; ) HANGUL SYLLABLE YOSS +C6A9;C6A9;110B 116D 11BC;C6A9;110B 116D 11BC; # (용; 용; 용; 용; 용; ) HANGUL SYLLABLE YONG +C6AA;C6AA;110B 116D 11BD;C6AA;110B 116D 11BD; # (욪; 욪; 욪; 욪; 욪; ) HANGUL SYLLABLE YOJ +C6AB;C6AB;110B 116D 11BE;C6AB;110B 116D 11BE; # (욫; 욫; 욫; 욫; 욫; ) HANGUL SYLLABLE YOC +C6AC;C6AC;110B 116D 11BF;C6AC;110B 116D 11BF; # (욬; 욬; 욬; 욬; 욬; ) HANGUL SYLLABLE YOK +C6AD;C6AD;110B 116D 11C0;C6AD;110B 116D 11C0; # (욭; 욭; 욭; 욭; 욭; ) HANGUL SYLLABLE YOT +C6AE;C6AE;110B 116D 11C1;C6AE;110B 116D 11C1; # (욮; 욮; 욮; 욮; 욮; ) HANGUL SYLLABLE YOP +C6AF;C6AF;110B 116D 11C2;C6AF;110B 116D 11C2; # (욯; 욯; 욯; 욯; 욯; ) HANGUL SYLLABLE YOH +C6B0;C6B0;110B 116E;C6B0;110B 116E; # (우; 우; 우; 우; 우; ) HANGUL SYLLABLE U +C6B1;C6B1;110B 116E 11A8;C6B1;110B 116E 11A8; # (욱; 욱; 욱; 욱; 욱; ) HANGUL SYLLABLE UG +C6B2;C6B2;110B 116E 11A9;C6B2;110B 116E 11A9; # (욲; 욲; 욲; 욲; 욲; ) HANGUL SYLLABLE UGG +C6B3;C6B3;110B 116E 11AA;C6B3;110B 116E 11AA; # (욳; 욳; 욳; 욳; 욳; ) HANGUL SYLLABLE UGS +C6B4;C6B4;110B 116E 11AB;C6B4;110B 116E 11AB; # (운; 운; 운; 운; 운; ) HANGUL SYLLABLE UN +C6B5;C6B5;110B 116E 11AC;C6B5;110B 116E 11AC; # (욵; 욵; 욵; 욵; 욵; ) HANGUL SYLLABLE UNJ +C6B6;C6B6;110B 116E 11AD;C6B6;110B 116E 11AD; # (욶; 욶; 욶; 욶; 욶; ) HANGUL SYLLABLE UNH +C6B7;C6B7;110B 116E 11AE;C6B7;110B 116E 11AE; # (욷; 욷; 욷; 욷; 욷; ) HANGUL SYLLABLE UD +C6B8;C6B8;110B 116E 11AF;C6B8;110B 116E 11AF; # (울; 울; 울; 울; 울; ) HANGUL SYLLABLE UL +C6B9;C6B9;110B 116E 11B0;C6B9;110B 116E 11B0; # (욹; 욹; 욹; 욹; 욹; ) HANGUL SYLLABLE ULG +C6BA;C6BA;110B 116E 11B1;C6BA;110B 116E 11B1; # (욺; 욺; 욺; 욺; 욺; ) HANGUL SYLLABLE ULM +C6BB;C6BB;110B 116E 11B2;C6BB;110B 116E 11B2; # (욻; 욻; 욻; 욻; 욻; ) HANGUL SYLLABLE ULB +C6BC;C6BC;110B 116E 11B3;C6BC;110B 116E 11B3; # (욼; 욼; 욼; 욼; 욼; ) HANGUL SYLLABLE ULS +C6BD;C6BD;110B 116E 11B4;C6BD;110B 116E 11B4; # (욽; 욽; 욽; 욽; 욽; ) HANGUL SYLLABLE ULT +C6BE;C6BE;110B 116E 11B5;C6BE;110B 116E 11B5; # (욾; 욾; 욾; 욾; 욾; ) HANGUL SYLLABLE ULP +C6BF;C6BF;110B 116E 11B6;C6BF;110B 116E 11B6; # (욿; 욿; 욿; 욿; 욿; ) HANGUL SYLLABLE ULH +C6C0;C6C0;110B 116E 11B7;C6C0;110B 116E 11B7; # (움; 움; 움; 움; 움; ) HANGUL SYLLABLE UM +C6C1;C6C1;110B 116E 11B8;C6C1;110B 116E 11B8; # (웁; 웁; 웁; 웁; 웁; ) HANGUL SYLLABLE UB +C6C2;C6C2;110B 116E 11B9;C6C2;110B 116E 11B9; # (웂; 웂; 웂; 웂; 웂; ) HANGUL SYLLABLE UBS +C6C3;C6C3;110B 116E 11BA;C6C3;110B 116E 11BA; # (웃; 웃; 웃; 웃; 웃; ) HANGUL SYLLABLE US +C6C4;C6C4;110B 116E 11BB;C6C4;110B 116E 11BB; # (웄; 웄; 웄; 웄; 웄; ) HANGUL SYLLABLE USS +C6C5;C6C5;110B 116E 11BC;C6C5;110B 116E 11BC; # (웅; 웅; 웅; 웅; 웅; ) HANGUL SYLLABLE UNG +C6C6;C6C6;110B 116E 11BD;C6C6;110B 116E 11BD; # (웆; 웆; 웆; 웆; 웆; ) HANGUL SYLLABLE UJ +C6C7;C6C7;110B 116E 11BE;C6C7;110B 116E 11BE; # (웇; 웇; 웇; 웇; 웇; ) HANGUL SYLLABLE UC +C6C8;C6C8;110B 116E 11BF;C6C8;110B 116E 11BF; # (웈; 웈; 웈; 웈; 웈; ) HANGUL SYLLABLE UK +C6C9;C6C9;110B 116E 11C0;C6C9;110B 116E 11C0; # (웉; 웉; 웉; 웉; 웉; ) HANGUL SYLLABLE UT +C6CA;C6CA;110B 116E 11C1;C6CA;110B 116E 11C1; # (웊; 웊; 웊; 웊; 웊; ) HANGUL SYLLABLE UP +C6CB;C6CB;110B 116E 11C2;C6CB;110B 116E 11C2; # (웋; 웋; 웋; 웋; 웋; ) HANGUL SYLLABLE UH +C6CC;C6CC;110B 116F;C6CC;110B 116F; # (워; 워; 워; 워; 워; ) HANGUL SYLLABLE WEO +C6CD;C6CD;110B 116F 11A8;C6CD;110B 116F 11A8; # (웍; 웍; 웍; 웍; 웍; ) HANGUL SYLLABLE WEOG +C6CE;C6CE;110B 116F 11A9;C6CE;110B 116F 11A9; # (웎; 웎; 웎; 웎; 웎; ) HANGUL SYLLABLE WEOGG +C6CF;C6CF;110B 116F 11AA;C6CF;110B 116F 11AA; # (웏; 웏; 웏; 웏; 웏; ) HANGUL SYLLABLE WEOGS +C6D0;C6D0;110B 116F 11AB;C6D0;110B 116F 11AB; # (원; 원; 원; 원; 원; ) HANGUL SYLLABLE WEON +C6D1;C6D1;110B 116F 11AC;C6D1;110B 116F 11AC; # (웑; 웑; 웑; 웑; 웑; ) HANGUL SYLLABLE WEONJ +C6D2;C6D2;110B 116F 11AD;C6D2;110B 116F 11AD; # (웒; 웒; 웒; 웒; 웒; ) HANGUL SYLLABLE WEONH +C6D3;C6D3;110B 116F 11AE;C6D3;110B 116F 11AE; # (웓; 웓; 웓; 웓; 웓; ) HANGUL SYLLABLE WEOD +C6D4;C6D4;110B 116F 11AF;C6D4;110B 116F 11AF; # (월; 월; 월; 월; 월; ) HANGUL SYLLABLE WEOL +C6D5;C6D5;110B 116F 11B0;C6D5;110B 116F 11B0; # (웕; 웕; 웕; 웕; 웕; ) HANGUL SYLLABLE WEOLG +C6D6;C6D6;110B 116F 11B1;C6D6;110B 116F 11B1; # (웖; 웖; 웖; 웖; 웖; ) HANGUL SYLLABLE WEOLM +C6D7;C6D7;110B 116F 11B2;C6D7;110B 116F 11B2; # (웗; 웗; 웗; 웗; 웗; ) HANGUL SYLLABLE WEOLB +C6D8;C6D8;110B 116F 11B3;C6D8;110B 116F 11B3; # (웘; 웘; 웘; 웘; 웘; ) HANGUL SYLLABLE WEOLS +C6D9;C6D9;110B 116F 11B4;C6D9;110B 116F 11B4; # (웙; 웙; 웙; 웙; 웙; ) HANGUL SYLLABLE WEOLT +C6DA;C6DA;110B 116F 11B5;C6DA;110B 116F 11B5; # (웚; 웚; 웚; 웚; 웚; ) HANGUL SYLLABLE WEOLP +C6DB;C6DB;110B 116F 11B6;C6DB;110B 116F 11B6; # (웛; 웛; 웛; 웛; 웛; ) HANGUL SYLLABLE WEOLH +C6DC;C6DC;110B 116F 11B7;C6DC;110B 116F 11B7; # (웜; 웜; 웜; 웜; 웜; ) HANGUL SYLLABLE WEOM +C6DD;C6DD;110B 116F 11B8;C6DD;110B 116F 11B8; # (웝; 웝; 웝; 웝; 웝; ) HANGUL SYLLABLE WEOB +C6DE;C6DE;110B 116F 11B9;C6DE;110B 116F 11B9; # (웞; 웞; 웞; 웞; 웞; ) HANGUL SYLLABLE WEOBS +C6DF;C6DF;110B 116F 11BA;C6DF;110B 116F 11BA; # (웟; 웟; 웟; 웟; 웟; ) HANGUL SYLLABLE WEOS +C6E0;C6E0;110B 116F 11BB;C6E0;110B 116F 11BB; # (웠; 웠; 웠; 웠; 웠; ) HANGUL SYLLABLE WEOSS +C6E1;C6E1;110B 116F 11BC;C6E1;110B 116F 11BC; # (웡; 웡; 웡; 웡; 웡; ) HANGUL SYLLABLE WEONG +C6E2;C6E2;110B 116F 11BD;C6E2;110B 116F 11BD; # (웢; 웢; 웢; 웢; 웢; ) HANGUL SYLLABLE WEOJ +C6E3;C6E3;110B 116F 11BE;C6E3;110B 116F 11BE; # (웣; 웣; 웣; 웣; 웣; ) HANGUL SYLLABLE WEOC +C6E4;C6E4;110B 116F 11BF;C6E4;110B 116F 11BF; # (웤; 웤; 웤; 웤; 웤; ) HANGUL SYLLABLE WEOK +C6E5;C6E5;110B 116F 11C0;C6E5;110B 116F 11C0; # (웥; 웥; 웥; 웥; 웥; ) HANGUL SYLLABLE WEOT +C6E6;C6E6;110B 116F 11C1;C6E6;110B 116F 11C1; # (웦; 웦; 웦; 웦; 웦; ) HANGUL SYLLABLE WEOP +C6E7;C6E7;110B 116F 11C2;C6E7;110B 116F 11C2; # (웧; 웧; 웧; 웧; 웧; ) HANGUL SYLLABLE WEOH +C6E8;C6E8;110B 1170;C6E8;110B 1170; # (웨; 웨; 웨; 웨; 웨; ) HANGUL SYLLABLE WE +C6E9;C6E9;110B 1170 11A8;C6E9;110B 1170 11A8; # (웩; 웩; 웩; 웩; 웩; ) HANGUL SYLLABLE WEG +C6EA;C6EA;110B 1170 11A9;C6EA;110B 1170 11A9; # (웪; 웪; 웪; 웪; 웪; ) HANGUL SYLLABLE WEGG +C6EB;C6EB;110B 1170 11AA;C6EB;110B 1170 11AA; # (웫; 웫; 웫; 웫; 웫; ) HANGUL SYLLABLE WEGS +C6EC;C6EC;110B 1170 11AB;C6EC;110B 1170 11AB; # (웬; 웬; 웬; 웬; 웬; ) HANGUL SYLLABLE WEN +C6ED;C6ED;110B 1170 11AC;C6ED;110B 1170 11AC; # (웭; 웭; 웭; 웭; 웭; ) HANGUL SYLLABLE WENJ +C6EE;C6EE;110B 1170 11AD;C6EE;110B 1170 11AD; # (웮; 웮; 웮; 웮; 웮; ) HANGUL SYLLABLE WENH +C6EF;C6EF;110B 1170 11AE;C6EF;110B 1170 11AE; # (웯; 웯; 웯; 웯; 웯; ) HANGUL SYLLABLE WED +C6F0;C6F0;110B 1170 11AF;C6F0;110B 1170 11AF; # (웰; 웰; 웰; 웰; 웰; ) HANGUL SYLLABLE WEL +C6F1;C6F1;110B 1170 11B0;C6F1;110B 1170 11B0; # (웱; 웱; 웱; 웱; 웱; ) HANGUL SYLLABLE WELG +C6F2;C6F2;110B 1170 11B1;C6F2;110B 1170 11B1; # (웲; 웲; 웲; 웲; 웲; ) HANGUL SYLLABLE WELM +C6F3;C6F3;110B 1170 11B2;C6F3;110B 1170 11B2; # (웳; 웳; 웳; 웳; 웳; ) HANGUL SYLLABLE WELB +C6F4;C6F4;110B 1170 11B3;C6F4;110B 1170 11B3; # (웴; 웴; 웴; 웴; 웴; ) HANGUL SYLLABLE WELS +C6F5;C6F5;110B 1170 11B4;C6F5;110B 1170 11B4; # (웵; 웵; 웵; 웵; 웵; ) HANGUL SYLLABLE WELT +C6F6;C6F6;110B 1170 11B5;C6F6;110B 1170 11B5; # (웶; 웶; 웶; 웶; 웶; ) HANGUL SYLLABLE WELP +C6F7;C6F7;110B 1170 11B6;C6F7;110B 1170 11B6; # (웷; 웷; 웷; 웷; 웷; ) HANGUL SYLLABLE WELH +C6F8;C6F8;110B 1170 11B7;C6F8;110B 1170 11B7; # (웸; 웸; 웸; 웸; 웸; ) HANGUL SYLLABLE WEM +C6F9;C6F9;110B 1170 11B8;C6F9;110B 1170 11B8; # (웹; 웹; 웹; 웹; 웹; ) HANGUL SYLLABLE WEB +C6FA;C6FA;110B 1170 11B9;C6FA;110B 1170 11B9; # (웺; 웺; 웺; 웺; 웺; ) HANGUL SYLLABLE WEBS +C6FB;C6FB;110B 1170 11BA;C6FB;110B 1170 11BA; # (웻; 웻; 웻; 웻; 웻; ) HANGUL SYLLABLE WES +C6FC;C6FC;110B 1170 11BB;C6FC;110B 1170 11BB; # (웼; 웼; 웼; 웼; 웼; ) HANGUL SYLLABLE WESS +C6FD;C6FD;110B 1170 11BC;C6FD;110B 1170 11BC; # (웽; 웽; 웽; 웽; 웽; ) HANGUL SYLLABLE WENG +C6FE;C6FE;110B 1170 11BD;C6FE;110B 1170 11BD; # (웾; 웾; 웾; 웾; 웾; ) HANGUL SYLLABLE WEJ +C6FF;C6FF;110B 1170 11BE;C6FF;110B 1170 11BE; # (웿; 웿; 웿; 웿; 웿; ) HANGUL SYLLABLE WEC +C700;C700;110B 1170 11BF;C700;110B 1170 11BF; # (윀; 윀; 윀; 윀; 윀; ) HANGUL SYLLABLE WEK +C701;C701;110B 1170 11C0;C701;110B 1170 11C0; # (윁; 윁; 윁; 윁; 윁; ) HANGUL SYLLABLE WET +C702;C702;110B 1170 11C1;C702;110B 1170 11C1; # (윂; 윂; 윂; 윂; 윂; ) HANGUL SYLLABLE WEP +C703;C703;110B 1170 11C2;C703;110B 1170 11C2; # (윃; 윃; 윃; 윃; 윃; ) HANGUL SYLLABLE WEH +C704;C704;110B 1171;C704;110B 1171; # (위; 위; 위; 위; 위; ) HANGUL SYLLABLE WI +C705;C705;110B 1171 11A8;C705;110B 1171 11A8; # (윅; 윅; 윅; 윅; 윅; ) HANGUL SYLLABLE WIG +C706;C706;110B 1171 11A9;C706;110B 1171 11A9; # (윆; 윆; 윆; 윆; 윆; ) HANGUL SYLLABLE WIGG +C707;C707;110B 1171 11AA;C707;110B 1171 11AA; # (윇; 윇; 윇; 윇; 윇; ) HANGUL SYLLABLE WIGS +C708;C708;110B 1171 11AB;C708;110B 1171 11AB; # (윈; 윈; 윈; 윈; 윈; ) HANGUL SYLLABLE WIN +C709;C709;110B 1171 11AC;C709;110B 1171 11AC; # (윉; 윉; 윉; 윉; 윉; ) HANGUL SYLLABLE WINJ +C70A;C70A;110B 1171 11AD;C70A;110B 1171 11AD; # (윊; 윊; 윊; 윊; 윊; ) HANGUL SYLLABLE WINH +C70B;C70B;110B 1171 11AE;C70B;110B 1171 11AE; # (윋; 윋; 윋; 윋; 윋; ) HANGUL SYLLABLE WID +C70C;C70C;110B 1171 11AF;C70C;110B 1171 11AF; # (윌; 윌; 윌; 윌; 윌; ) HANGUL SYLLABLE WIL +C70D;C70D;110B 1171 11B0;C70D;110B 1171 11B0; # (윍; 윍; 윍; 윍; 윍; ) HANGUL SYLLABLE WILG +C70E;C70E;110B 1171 11B1;C70E;110B 1171 11B1; # (윎; 윎; 윎; 윎; 윎; ) HANGUL SYLLABLE WILM +C70F;C70F;110B 1171 11B2;C70F;110B 1171 11B2; # (윏; 윏; 윏; 윏; 윏; ) HANGUL SYLLABLE WILB +C710;C710;110B 1171 11B3;C710;110B 1171 11B3; # (윐; 윐; 윐; 윐; 윐; ) HANGUL SYLLABLE WILS +C711;C711;110B 1171 11B4;C711;110B 1171 11B4; # (윑; 윑; 윑; 윑; 윑; ) HANGUL SYLLABLE WILT +C712;C712;110B 1171 11B5;C712;110B 1171 11B5; # (윒; 윒; 윒; 윒; 윒; ) HANGUL SYLLABLE WILP +C713;C713;110B 1171 11B6;C713;110B 1171 11B6; # (윓; 윓; 윓; 윓; 윓; ) HANGUL SYLLABLE WILH +C714;C714;110B 1171 11B7;C714;110B 1171 11B7; # (윔; 윔; 윔; 윔; 윔; ) HANGUL SYLLABLE WIM +C715;C715;110B 1171 11B8;C715;110B 1171 11B8; # (윕; 윕; 윕; 윕; 윕; ) HANGUL SYLLABLE WIB +C716;C716;110B 1171 11B9;C716;110B 1171 11B9; # (윖; 윖; 윖; 윖; 윖; ) HANGUL SYLLABLE WIBS +C717;C717;110B 1171 11BA;C717;110B 1171 11BA; # (윗; 윗; 윗; 윗; 윗; ) HANGUL SYLLABLE WIS +C718;C718;110B 1171 11BB;C718;110B 1171 11BB; # (윘; 윘; 윘; 윘; 윘; ) HANGUL SYLLABLE WISS +C719;C719;110B 1171 11BC;C719;110B 1171 11BC; # (윙; 윙; 윙; 윙; 윙; ) HANGUL SYLLABLE WING +C71A;C71A;110B 1171 11BD;C71A;110B 1171 11BD; # (윚; 윚; 윚; 윚; 윚; ) HANGUL SYLLABLE WIJ +C71B;C71B;110B 1171 11BE;C71B;110B 1171 11BE; # (윛; 윛; 윛; 윛; 윛; ) HANGUL SYLLABLE WIC +C71C;C71C;110B 1171 11BF;C71C;110B 1171 11BF; # (윜; 윜; 윜; 윜; 윜; ) HANGUL SYLLABLE WIK +C71D;C71D;110B 1171 11C0;C71D;110B 1171 11C0; # (윝; 윝; 윝; 윝; 윝; ) HANGUL SYLLABLE WIT +C71E;C71E;110B 1171 11C1;C71E;110B 1171 11C1; # (윞; 윞; 윞; 윞; 윞; ) HANGUL SYLLABLE WIP +C71F;C71F;110B 1171 11C2;C71F;110B 1171 11C2; # (윟; 윟; 윟; 윟; 윟; ) HANGUL SYLLABLE WIH +C720;C720;110B 1172;C720;110B 1172; # (유; 유; 유; 유; 유; ) HANGUL SYLLABLE YU +C721;C721;110B 1172 11A8;C721;110B 1172 11A8; # (육; 육; 육; 육; 육; ) HANGUL SYLLABLE YUG +C722;C722;110B 1172 11A9;C722;110B 1172 11A9; # (윢; 윢; 윢; 윢; 윢; ) HANGUL SYLLABLE YUGG +C723;C723;110B 1172 11AA;C723;110B 1172 11AA; # (윣; 윣; 윣; 윣; 윣; ) HANGUL SYLLABLE YUGS +C724;C724;110B 1172 11AB;C724;110B 1172 11AB; # (윤; 윤; 윤; 윤; 윤; ) HANGUL SYLLABLE YUN +C725;C725;110B 1172 11AC;C725;110B 1172 11AC; # (윥; 윥; 윥; 윥; 윥; ) HANGUL SYLLABLE YUNJ +C726;C726;110B 1172 11AD;C726;110B 1172 11AD; # (윦; 윦; 윦; 윦; 윦; ) HANGUL SYLLABLE YUNH +C727;C727;110B 1172 11AE;C727;110B 1172 11AE; # (윧; 윧; 윧; 윧; 윧; ) HANGUL SYLLABLE YUD +C728;C728;110B 1172 11AF;C728;110B 1172 11AF; # (율; 율; 율; 율; 율; ) HANGUL SYLLABLE YUL +C729;C729;110B 1172 11B0;C729;110B 1172 11B0; # (윩; 윩; 윩; 윩; 윩; ) HANGUL SYLLABLE YULG +C72A;C72A;110B 1172 11B1;C72A;110B 1172 11B1; # (윪; 윪; 윪; 윪; 윪; ) HANGUL SYLLABLE YULM +C72B;C72B;110B 1172 11B2;C72B;110B 1172 11B2; # (윫; 윫; 윫; 윫; 윫; ) HANGUL SYLLABLE YULB +C72C;C72C;110B 1172 11B3;C72C;110B 1172 11B3; # (윬; 윬; 윬; 윬; 윬; ) HANGUL SYLLABLE YULS +C72D;C72D;110B 1172 11B4;C72D;110B 1172 11B4; # (윭; 윭; 윭; 윭; 윭; ) HANGUL SYLLABLE YULT +C72E;C72E;110B 1172 11B5;C72E;110B 1172 11B5; # (윮; 윮; 윮; 윮; 윮; ) HANGUL SYLLABLE YULP +C72F;C72F;110B 1172 11B6;C72F;110B 1172 11B6; # (윯; 윯; 윯; 윯; 윯; ) HANGUL SYLLABLE YULH +C730;C730;110B 1172 11B7;C730;110B 1172 11B7; # (윰; 윰; 윰; 윰; 윰; ) HANGUL SYLLABLE YUM +C731;C731;110B 1172 11B8;C731;110B 1172 11B8; # (윱; 윱; 윱; 윱; 윱; ) HANGUL SYLLABLE YUB +C732;C732;110B 1172 11B9;C732;110B 1172 11B9; # (윲; 윲; 윲; 윲; 윲; ) HANGUL SYLLABLE YUBS +C733;C733;110B 1172 11BA;C733;110B 1172 11BA; # (윳; 윳; 윳; 윳; 윳; ) HANGUL SYLLABLE YUS +C734;C734;110B 1172 11BB;C734;110B 1172 11BB; # (윴; 윴; 윴; 윴; 윴; ) HANGUL SYLLABLE YUSS +C735;C735;110B 1172 11BC;C735;110B 1172 11BC; # (융; 융; 융; 융; 융; ) HANGUL SYLLABLE YUNG +C736;C736;110B 1172 11BD;C736;110B 1172 11BD; # (윶; 윶; 윶; 윶; 윶; ) HANGUL SYLLABLE YUJ +C737;C737;110B 1172 11BE;C737;110B 1172 11BE; # (윷; 윷; 윷; 윷; 윷; ) HANGUL SYLLABLE YUC +C738;C738;110B 1172 11BF;C738;110B 1172 11BF; # (윸; 윸; 윸; 윸; 윸; ) HANGUL SYLLABLE YUK +C739;C739;110B 1172 11C0;C739;110B 1172 11C0; # (윹; 윹; 윹; 윹; 윹; ) HANGUL SYLLABLE YUT +C73A;C73A;110B 1172 11C1;C73A;110B 1172 11C1; # (윺; 윺; 윺; 윺; 윺; ) HANGUL SYLLABLE YUP +C73B;C73B;110B 1172 11C2;C73B;110B 1172 11C2; # (윻; 윻; 윻; 윻; 윻; ) HANGUL SYLLABLE YUH +C73C;C73C;110B 1173;C73C;110B 1173; # (으; 으; 으; 으; 으; ) HANGUL SYLLABLE EU +C73D;C73D;110B 1173 11A8;C73D;110B 1173 11A8; # (윽; 윽; 윽; 윽; 윽; ) HANGUL SYLLABLE EUG +C73E;C73E;110B 1173 11A9;C73E;110B 1173 11A9; # (윾; 윾; 윾; 윾; 윾; ) HANGUL SYLLABLE EUGG +C73F;C73F;110B 1173 11AA;C73F;110B 1173 11AA; # (윿; 윿; 윿; 윿; 윿; ) HANGUL SYLLABLE EUGS +C740;C740;110B 1173 11AB;C740;110B 1173 11AB; # (은; 은; 은; 은; 은; ) HANGUL SYLLABLE EUN +C741;C741;110B 1173 11AC;C741;110B 1173 11AC; # (읁; 읁; 읁; 읁; 읁; ) HANGUL SYLLABLE EUNJ +C742;C742;110B 1173 11AD;C742;110B 1173 11AD; # (읂; 읂; 읂; 읂; 읂; ) HANGUL SYLLABLE EUNH +C743;C743;110B 1173 11AE;C743;110B 1173 11AE; # (읃; 읃; 읃; 읃; 읃; ) HANGUL SYLLABLE EUD +C744;C744;110B 1173 11AF;C744;110B 1173 11AF; # (을; 을; 을; 을; 을; ) HANGUL SYLLABLE EUL +C745;C745;110B 1173 11B0;C745;110B 1173 11B0; # (읅; 읅; 읅; 읅; 읅; ) HANGUL SYLLABLE EULG +C746;C746;110B 1173 11B1;C746;110B 1173 11B1; # (읆; 읆; 읆; 읆; 읆; ) HANGUL SYLLABLE EULM +C747;C747;110B 1173 11B2;C747;110B 1173 11B2; # (읇; 읇; 읇; 읇; 읇; ) HANGUL SYLLABLE EULB +C748;C748;110B 1173 11B3;C748;110B 1173 11B3; # (읈; 읈; 읈; 읈; 읈; ) HANGUL SYLLABLE EULS +C749;C749;110B 1173 11B4;C749;110B 1173 11B4; # (읉; 읉; 읉; 읉; 읉; ) HANGUL SYLLABLE EULT +C74A;C74A;110B 1173 11B5;C74A;110B 1173 11B5; # (읊; 읊; 읊; 읊; 읊; ) HANGUL SYLLABLE EULP +C74B;C74B;110B 1173 11B6;C74B;110B 1173 11B6; # (읋; 읋; 읋; 읋; 읋; ) HANGUL SYLLABLE EULH +C74C;C74C;110B 1173 11B7;C74C;110B 1173 11B7; # (음; 음; 음; 음; 음; ) HANGUL SYLLABLE EUM +C74D;C74D;110B 1173 11B8;C74D;110B 1173 11B8; # (읍; 읍; 읍; 읍; 읍; ) HANGUL SYLLABLE EUB +C74E;C74E;110B 1173 11B9;C74E;110B 1173 11B9; # (읎; 읎; 읎; 읎; 읎; ) HANGUL SYLLABLE EUBS +C74F;C74F;110B 1173 11BA;C74F;110B 1173 11BA; # (읏; 읏; 읏; 읏; 읏; ) HANGUL SYLLABLE EUS +C750;C750;110B 1173 11BB;C750;110B 1173 11BB; # (읐; 읐; 읐; 읐; 읐; ) HANGUL SYLLABLE EUSS +C751;C751;110B 1173 11BC;C751;110B 1173 11BC; # (응; 응; 응; 응; 응; ) HANGUL SYLLABLE EUNG +C752;C752;110B 1173 11BD;C752;110B 1173 11BD; # (읒; 읒; 읒; 읒; 읒; ) HANGUL SYLLABLE EUJ +C753;C753;110B 1173 11BE;C753;110B 1173 11BE; # (읓; 읓; 읓; 읓; 읓; ) HANGUL SYLLABLE EUC +C754;C754;110B 1173 11BF;C754;110B 1173 11BF; # (읔; 읔; 읔; 읔; 읔; ) HANGUL SYLLABLE EUK +C755;C755;110B 1173 11C0;C755;110B 1173 11C0; # (읕; 읕; 읕; 읕; 읕; ) HANGUL SYLLABLE EUT +C756;C756;110B 1173 11C1;C756;110B 1173 11C1; # (읖; 읖; 읖; 읖; 읖; ) HANGUL SYLLABLE EUP +C757;C757;110B 1173 11C2;C757;110B 1173 11C2; # (읗; 읗; 읗; 읗; 읗; ) HANGUL SYLLABLE EUH +C758;C758;110B 1174;C758;110B 1174; # (의; 의; 의; 의; 의; ) HANGUL SYLLABLE YI +C759;C759;110B 1174 11A8;C759;110B 1174 11A8; # (읙; 읙; 읙; 읙; 읙; ) HANGUL SYLLABLE YIG +C75A;C75A;110B 1174 11A9;C75A;110B 1174 11A9; # (읚; 읚; 읚; 읚; 읚; ) HANGUL SYLLABLE YIGG +C75B;C75B;110B 1174 11AA;C75B;110B 1174 11AA; # (읛; 읛; 읛; 읛; 읛; ) HANGUL SYLLABLE YIGS +C75C;C75C;110B 1174 11AB;C75C;110B 1174 11AB; # (읜; 읜; 읜; 읜; 읜; ) HANGUL SYLLABLE YIN +C75D;C75D;110B 1174 11AC;C75D;110B 1174 11AC; # (읝; 읝; 읝; 읝; 읝; ) HANGUL SYLLABLE YINJ +C75E;C75E;110B 1174 11AD;C75E;110B 1174 11AD; # (읞; 읞; 읞; 읞; 읞; ) HANGUL SYLLABLE YINH +C75F;C75F;110B 1174 11AE;C75F;110B 1174 11AE; # (읟; 읟; 읟; 읟; 읟; ) HANGUL SYLLABLE YID +C760;C760;110B 1174 11AF;C760;110B 1174 11AF; # (읠; 읠; 읠; 읠; 읠; ) HANGUL SYLLABLE YIL +C761;C761;110B 1174 11B0;C761;110B 1174 11B0; # (읡; 읡; 읡; 읡; 읡; ) HANGUL SYLLABLE YILG +C762;C762;110B 1174 11B1;C762;110B 1174 11B1; # (읢; 읢; 읢; 읢; 읢; ) HANGUL SYLLABLE YILM +C763;C763;110B 1174 11B2;C763;110B 1174 11B2; # (읣; 읣; 읣; 읣; 읣; ) HANGUL SYLLABLE YILB +C764;C764;110B 1174 11B3;C764;110B 1174 11B3; # (읤; 읤; 읤; 읤; 읤; ) HANGUL SYLLABLE YILS +C765;C765;110B 1174 11B4;C765;110B 1174 11B4; # (읥; 읥; 읥; 읥; 읥; ) HANGUL SYLLABLE YILT +C766;C766;110B 1174 11B5;C766;110B 1174 11B5; # (읦; 읦; 읦; 읦; 읦; ) HANGUL SYLLABLE YILP +C767;C767;110B 1174 11B6;C767;110B 1174 11B6; # (읧; 읧; 읧; 읧; 읧; ) HANGUL SYLLABLE YILH +C768;C768;110B 1174 11B7;C768;110B 1174 11B7; # (읨; 읨; 읨; 읨; 읨; ) HANGUL SYLLABLE YIM +C769;C769;110B 1174 11B8;C769;110B 1174 11B8; # (읩; 읩; 읩; 읩; 읩; ) HANGUL SYLLABLE YIB +C76A;C76A;110B 1174 11B9;C76A;110B 1174 11B9; # (읪; 읪; 읪; 읪; 읪; ) HANGUL SYLLABLE YIBS +C76B;C76B;110B 1174 11BA;C76B;110B 1174 11BA; # (읫; 읫; 읫; 읫; 읫; ) HANGUL SYLLABLE YIS +C76C;C76C;110B 1174 11BB;C76C;110B 1174 11BB; # (읬; 읬; 읬; 읬; 읬; ) HANGUL SYLLABLE YISS +C76D;C76D;110B 1174 11BC;C76D;110B 1174 11BC; # (읭; 읭; 읭; 읭; 읭; ) HANGUL SYLLABLE YING +C76E;C76E;110B 1174 11BD;C76E;110B 1174 11BD; # (읮; 읮; 읮; 읮; 읮; ) HANGUL SYLLABLE YIJ +C76F;C76F;110B 1174 11BE;C76F;110B 1174 11BE; # (읯; 읯; 읯; 읯; 읯; ) HANGUL SYLLABLE YIC +C770;C770;110B 1174 11BF;C770;110B 1174 11BF; # (읰; 읰; 읰; 읰; 읰; ) HANGUL SYLLABLE YIK +C771;C771;110B 1174 11C0;C771;110B 1174 11C0; # (읱; 읱; 읱; 읱; 읱; ) HANGUL SYLLABLE YIT +C772;C772;110B 1174 11C1;C772;110B 1174 11C1; # (읲; 읲; 읲; 읲; 읲; ) HANGUL SYLLABLE YIP +C773;C773;110B 1174 11C2;C773;110B 1174 11C2; # (읳; 읳; 읳; 읳; 읳; ) HANGUL SYLLABLE YIH +C774;C774;110B 1175;C774;110B 1175; # (이; 이; 이; 이; 이; ) HANGUL SYLLABLE I +C775;C775;110B 1175 11A8;C775;110B 1175 11A8; # (익; 익; 익; 익; 익; ) HANGUL SYLLABLE IG +C776;C776;110B 1175 11A9;C776;110B 1175 11A9; # (읶; 읶; 읶; 읶; 읶; ) HANGUL SYLLABLE IGG +C777;C777;110B 1175 11AA;C777;110B 1175 11AA; # (읷; 읷; 읷; 읷; 읷; ) HANGUL SYLLABLE IGS +C778;C778;110B 1175 11AB;C778;110B 1175 11AB; # (인; 인; 인; 인; 인; ) HANGUL SYLLABLE IN +C779;C779;110B 1175 11AC;C779;110B 1175 11AC; # (읹; 읹; 읹; 읹; 읹; ) HANGUL SYLLABLE INJ +C77A;C77A;110B 1175 11AD;C77A;110B 1175 11AD; # (읺; 읺; 읺; 읺; 읺; ) HANGUL SYLLABLE INH +C77B;C77B;110B 1175 11AE;C77B;110B 1175 11AE; # (읻; 읻; 읻; 읻; 읻; ) HANGUL SYLLABLE ID +C77C;C77C;110B 1175 11AF;C77C;110B 1175 11AF; # (일; 일; 일; 일; 일; ) HANGUL SYLLABLE IL +C77D;C77D;110B 1175 11B0;C77D;110B 1175 11B0; # (읽; 읽; 읽; 읽; 읽; ) HANGUL SYLLABLE ILG +C77E;C77E;110B 1175 11B1;C77E;110B 1175 11B1; # (읾; 읾; 읾; 읾; 읾; ) HANGUL SYLLABLE ILM +C77F;C77F;110B 1175 11B2;C77F;110B 1175 11B2; # (읿; 읿; 읿; 읿; 읿; ) HANGUL SYLLABLE ILB +C780;C780;110B 1175 11B3;C780;110B 1175 11B3; # (잀; 잀; 잀; 잀; 잀; ) HANGUL SYLLABLE ILS +C781;C781;110B 1175 11B4;C781;110B 1175 11B4; # (잁; 잁; 잁; 잁; 잁; ) HANGUL SYLLABLE ILT +C782;C782;110B 1175 11B5;C782;110B 1175 11B5; # (잂; 잂; 잂; 잂; 잂; ) HANGUL SYLLABLE ILP +C783;C783;110B 1175 11B6;C783;110B 1175 11B6; # (잃; 잃; 잃; 잃; 잃; ) HANGUL SYLLABLE ILH +C784;C784;110B 1175 11B7;C784;110B 1175 11B7; # (임; 임; 임; 임; 임; ) HANGUL SYLLABLE IM +C785;C785;110B 1175 11B8;C785;110B 1175 11B8; # (입; 입; 입; 입; 입; ) HANGUL SYLLABLE IB +C786;C786;110B 1175 11B9;C786;110B 1175 11B9; # (잆; 잆; 잆; 잆; 잆; ) HANGUL SYLLABLE IBS +C787;C787;110B 1175 11BA;C787;110B 1175 11BA; # (잇; 잇; 잇; 잇; 잇; ) HANGUL SYLLABLE IS +C788;C788;110B 1175 11BB;C788;110B 1175 11BB; # (있; 있; 있; 있; 있; ) HANGUL SYLLABLE ISS +C789;C789;110B 1175 11BC;C789;110B 1175 11BC; # (잉; 잉; 잉; 잉; 잉; ) HANGUL SYLLABLE ING +C78A;C78A;110B 1175 11BD;C78A;110B 1175 11BD; # (잊; 잊; 잊; 잊; 잊; ) HANGUL SYLLABLE IJ +C78B;C78B;110B 1175 11BE;C78B;110B 1175 11BE; # (잋; 잋; 잋; 잋; 잋; ) HANGUL SYLLABLE IC +C78C;C78C;110B 1175 11BF;C78C;110B 1175 11BF; # (잌; 잌; 잌; 잌; 잌; ) HANGUL SYLLABLE IK +C78D;C78D;110B 1175 11C0;C78D;110B 1175 11C0; # (잍; 잍; 잍; 잍; 잍; ) HANGUL SYLLABLE IT +C78E;C78E;110B 1175 11C1;C78E;110B 1175 11C1; # (잎; 잎; 잎; 잎; 잎; ) HANGUL SYLLABLE IP +C78F;C78F;110B 1175 11C2;C78F;110B 1175 11C2; # (잏; 잏; 잏; 잏; 잏; ) HANGUL SYLLABLE IH +C790;C790;110C 1161;C790;110C 1161; # (자; 자; 자; 자; 자; ) HANGUL SYLLABLE JA +C791;C791;110C 1161 11A8;C791;110C 1161 11A8; # (작; 작; 작; 작; 작; ) HANGUL SYLLABLE JAG +C792;C792;110C 1161 11A9;C792;110C 1161 11A9; # (잒; 잒; 잒; 잒; 잒; ) HANGUL SYLLABLE JAGG +C793;C793;110C 1161 11AA;C793;110C 1161 11AA; # (잓; 잓; 잓; 잓; 잓; ) HANGUL SYLLABLE JAGS +C794;C794;110C 1161 11AB;C794;110C 1161 11AB; # (잔; 잔; 잔; 잔; 잔; ) HANGUL SYLLABLE JAN +C795;C795;110C 1161 11AC;C795;110C 1161 11AC; # (잕; 잕; 잕; 잕; 잕; ) HANGUL SYLLABLE JANJ +C796;C796;110C 1161 11AD;C796;110C 1161 11AD; # (잖; 잖; 잖; 잖; 잖; ) HANGUL SYLLABLE JANH +C797;C797;110C 1161 11AE;C797;110C 1161 11AE; # (잗; 잗; 잗; 잗; 잗; ) HANGUL SYLLABLE JAD +C798;C798;110C 1161 11AF;C798;110C 1161 11AF; # (잘; 잘; 잘; 잘; 잘; ) HANGUL SYLLABLE JAL +C799;C799;110C 1161 11B0;C799;110C 1161 11B0; # (잙; 잙; 잙; 잙; 잙; ) HANGUL SYLLABLE JALG +C79A;C79A;110C 1161 11B1;C79A;110C 1161 11B1; # (잚; 잚; 잚; 잚; 잚; ) HANGUL SYLLABLE JALM +C79B;C79B;110C 1161 11B2;C79B;110C 1161 11B2; # (잛; 잛; 잛; 잛; 잛; ) HANGUL SYLLABLE JALB +C79C;C79C;110C 1161 11B3;C79C;110C 1161 11B3; # (잜; 잜; 잜; 잜; 잜; ) HANGUL SYLLABLE JALS +C79D;C79D;110C 1161 11B4;C79D;110C 1161 11B4; # (잝; 잝; 잝; 잝; 잝; ) HANGUL SYLLABLE JALT +C79E;C79E;110C 1161 11B5;C79E;110C 1161 11B5; # (잞; 잞; 잞; 잞; 잞; ) HANGUL SYLLABLE JALP +C79F;C79F;110C 1161 11B6;C79F;110C 1161 11B6; # (잟; 잟; 잟; 잟; 잟; ) HANGUL SYLLABLE JALH +C7A0;C7A0;110C 1161 11B7;C7A0;110C 1161 11B7; # (잠; 잠; 잠; 잠; 잠; ) HANGUL SYLLABLE JAM +C7A1;C7A1;110C 1161 11B8;C7A1;110C 1161 11B8; # (잡; 잡; 잡; 잡; 잡; ) HANGUL SYLLABLE JAB +C7A2;C7A2;110C 1161 11B9;C7A2;110C 1161 11B9; # (잢; 잢; 잢; 잢; 잢; ) HANGUL SYLLABLE JABS +C7A3;C7A3;110C 1161 11BA;C7A3;110C 1161 11BA; # (잣; 잣; 잣; 잣; 잣; ) HANGUL SYLLABLE JAS +C7A4;C7A4;110C 1161 11BB;C7A4;110C 1161 11BB; # (잤; 잤; 잤; 잤; 잤; ) HANGUL SYLLABLE JASS +C7A5;C7A5;110C 1161 11BC;C7A5;110C 1161 11BC; # (장; 장; 장; 장; 장; ) HANGUL SYLLABLE JANG +C7A6;C7A6;110C 1161 11BD;C7A6;110C 1161 11BD; # (잦; 잦; 잦; 잦; 잦; ) HANGUL SYLLABLE JAJ +C7A7;C7A7;110C 1161 11BE;C7A7;110C 1161 11BE; # (잧; 잧; 잧; 잧; 잧; ) HANGUL SYLLABLE JAC +C7A8;C7A8;110C 1161 11BF;C7A8;110C 1161 11BF; # (잨; 잨; 잨; 잨; 잨; ) HANGUL SYLLABLE JAK +C7A9;C7A9;110C 1161 11C0;C7A9;110C 1161 11C0; # (잩; 잩; 잩; 잩; 잩; ) HANGUL SYLLABLE JAT +C7AA;C7AA;110C 1161 11C1;C7AA;110C 1161 11C1; # (잪; 잪; 잪; 잪; 잪; ) HANGUL SYLLABLE JAP +C7AB;C7AB;110C 1161 11C2;C7AB;110C 1161 11C2; # (잫; 잫; 잫; 잫; 잫; ) HANGUL SYLLABLE JAH +C7AC;C7AC;110C 1162;C7AC;110C 1162; # (재; 재; 재; 재; 재; ) HANGUL SYLLABLE JAE +C7AD;C7AD;110C 1162 11A8;C7AD;110C 1162 11A8; # (잭; 잭; 잭; 잭; 잭; ) HANGUL SYLLABLE JAEG +C7AE;C7AE;110C 1162 11A9;C7AE;110C 1162 11A9; # (잮; 잮; 잮; 잮; 잮; ) HANGUL SYLLABLE JAEGG +C7AF;C7AF;110C 1162 11AA;C7AF;110C 1162 11AA; # (잯; 잯; 잯; 잯; 잯; ) HANGUL SYLLABLE JAEGS +C7B0;C7B0;110C 1162 11AB;C7B0;110C 1162 11AB; # (잰; 잰; 잰; 잰; 잰; ) HANGUL SYLLABLE JAEN +C7B1;C7B1;110C 1162 11AC;C7B1;110C 1162 11AC; # (잱; 잱; 잱; 잱; 잱; ) HANGUL SYLLABLE JAENJ +C7B2;C7B2;110C 1162 11AD;C7B2;110C 1162 11AD; # (잲; 잲; 잲; 잲; 잲; ) HANGUL SYLLABLE JAENH +C7B3;C7B3;110C 1162 11AE;C7B3;110C 1162 11AE; # (잳; 잳; 잳; 잳; 잳; ) HANGUL SYLLABLE JAED +C7B4;C7B4;110C 1162 11AF;C7B4;110C 1162 11AF; # (잴; 잴; 잴; 잴; 잴; ) HANGUL SYLLABLE JAEL +C7B5;C7B5;110C 1162 11B0;C7B5;110C 1162 11B0; # (잵; 잵; 잵; 잵; 잵; ) HANGUL SYLLABLE JAELG +C7B6;C7B6;110C 1162 11B1;C7B6;110C 1162 11B1; # (잶; 잶; 잶; 잶; 잶; ) HANGUL SYLLABLE JAELM +C7B7;C7B7;110C 1162 11B2;C7B7;110C 1162 11B2; # (잷; 잷; 잷; 잷; 잷; ) HANGUL SYLLABLE JAELB +C7B8;C7B8;110C 1162 11B3;C7B8;110C 1162 11B3; # (잸; 잸; 잸; 잸; 잸; ) HANGUL SYLLABLE JAELS +C7B9;C7B9;110C 1162 11B4;C7B9;110C 1162 11B4; # (잹; 잹; 잹; 잹; 잹; ) HANGUL SYLLABLE JAELT +C7BA;C7BA;110C 1162 11B5;C7BA;110C 1162 11B5; # (잺; 잺; 잺; 잺; 잺; ) HANGUL SYLLABLE JAELP +C7BB;C7BB;110C 1162 11B6;C7BB;110C 1162 11B6; # (잻; 잻; 잻; 잻; 잻; ) HANGUL SYLLABLE JAELH +C7BC;C7BC;110C 1162 11B7;C7BC;110C 1162 11B7; # (잼; 잼; 잼; 잼; 잼; ) HANGUL SYLLABLE JAEM +C7BD;C7BD;110C 1162 11B8;C7BD;110C 1162 11B8; # (잽; 잽; 잽; 잽; 잽; ) HANGUL SYLLABLE JAEB +C7BE;C7BE;110C 1162 11B9;C7BE;110C 1162 11B9; # (잾; 잾; 잾; 잾; 잾; ) HANGUL SYLLABLE JAEBS +C7BF;C7BF;110C 1162 11BA;C7BF;110C 1162 11BA; # (잿; 잿; 잿; 잿; 잿; ) HANGUL SYLLABLE JAES +C7C0;C7C0;110C 1162 11BB;C7C0;110C 1162 11BB; # (쟀; 쟀; 쟀; 쟀; 쟀; ) HANGUL SYLLABLE JAESS +C7C1;C7C1;110C 1162 11BC;C7C1;110C 1162 11BC; # (쟁; 쟁; 쟁; 쟁; 쟁; ) HANGUL SYLLABLE JAENG +C7C2;C7C2;110C 1162 11BD;C7C2;110C 1162 11BD; # (쟂; 쟂; 쟂; 쟂; 쟂; ) HANGUL SYLLABLE JAEJ +C7C3;C7C3;110C 1162 11BE;C7C3;110C 1162 11BE; # (쟃; 쟃; 쟃; 쟃; 쟃; ) HANGUL SYLLABLE JAEC +C7C4;C7C4;110C 1162 11BF;C7C4;110C 1162 11BF; # (쟄; 쟄; 쟄; 쟄; 쟄; ) HANGUL SYLLABLE JAEK +C7C5;C7C5;110C 1162 11C0;C7C5;110C 1162 11C0; # (쟅; 쟅; 쟅; 쟅; 쟅; ) HANGUL SYLLABLE JAET +C7C6;C7C6;110C 1162 11C1;C7C6;110C 1162 11C1; # (쟆; 쟆; 쟆; 쟆; 쟆; ) HANGUL SYLLABLE JAEP +C7C7;C7C7;110C 1162 11C2;C7C7;110C 1162 11C2; # (쟇; 쟇; 쟇; 쟇; 쟇; ) HANGUL SYLLABLE JAEH +C7C8;C7C8;110C 1163;C7C8;110C 1163; # (쟈; 쟈; 쟈; 쟈; 쟈; ) HANGUL SYLLABLE JYA +C7C9;C7C9;110C 1163 11A8;C7C9;110C 1163 11A8; # (쟉; 쟉; 쟉; 쟉; 쟉; ) HANGUL SYLLABLE JYAG +C7CA;C7CA;110C 1163 11A9;C7CA;110C 1163 11A9; # (쟊; 쟊; 쟊; 쟊; 쟊; ) HANGUL SYLLABLE JYAGG +C7CB;C7CB;110C 1163 11AA;C7CB;110C 1163 11AA; # (쟋; 쟋; 쟋; 쟋; 쟋; ) HANGUL SYLLABLE JYAGS +C7CC;C7CC;110C 1163 11AB;C7CC;110C 1163 11AB; # (쟌; 쟌; 쟌; 쟌; 쟌; ) HANGUL SYLLABLE JYAN +C7CD;C7CD;110C 1163 11AC;C7CD;110C 1163 11AC; # (쟍; 쟍; 쟍; 쟍; 쟍; ) HANGUL SYLLABLE JYANJ +C7CE;C7CE;110C 1163 11AD;C7CE;110C 1163 11AD; # (쟎; 쟎; 쟎; 쟎; 쟎; ) HANGUL SYLLABLE JYANH +C7CF;C7CF;110C 1163 11AE;C7CF;110C 1163 11AE; # (쟏; 쟏; 쟏; 쟏; 쟏; ) HANGUL SYLLABLE JYAD +C7D0;C7D0;110C 1163 11AF;C7D0;110C 1163 11AF; # (쟐; 쟐; 쟐; 쟐; 쟐; ) HANGUL SYLLABLE JYAL +C7D1;C7D1;110C 1163 11B0;C7D1;110C 1163 11B0; # (쟑; 쟑; 쟑; 쟑; 쟑; ) HANGUL SYLLABLE JYALG +C7D2;C7D2;110C 1163 11B1;C7D2;110C 1163 11B1; # (쟒; 쟒; 쟒; 쟒; 쟒; ) HANGUL SYLLABLE JYALM +C7D3;C7D3;110C 1163 11B2;C7D3;110C 1163 11B2; # (쟓; 쟓; 쟓; 쟓; 쟓; ) HANGUL SYLLABLE JYALB +C7D4;C7D4;110C 1163 11B3;C7D4;110C 1163 11B3; # (쟔; 쟔; 쟔; 쟔; 쟔; ) HANGUL SYLLABLE JYALS +C7D5;C7D5;110C 1163 11B4;C7D5;110C 1163 11B4; # (쟕; 쟕; 쟕; 쟕; 쟕; ) HANGUL SYLLABLE JYALT +C7D6;C7D6;110C 1163 11B5;C7D6;110C 1163 11B5; # (쟖; 쟖; 쟖; 쟖; 쟖; ) HANGUL SYLLABLE JYALP +C7D7;C7D7;110C 1163 11B6;C7D7;110C 1163 11B6; # (쟗; 쟗; 쟗; 쟗; 쟗; ) HANGUL SYLLABLE JYALH +C7D8;C7D8;110C 1163 11B7;C7D8;110C 1163 11B7; # (쟘; 쟘; 쟘; 쟘; 쟘; ) HANGUL SYLLABLE JYAM +C7D9;C7D9;110C 1163 11B8;C7D9;110C 1163 11B8; # (쟙; 쟙; 쟙; 쟙; 쟙; ) HANGUL SYLLABLE JYAB +C7DA;C7DA;110C 1163 11B9;C7DA;110C 1163 11B9; # (쟚; 쟚; 쟚; 쟚; 쟚; ) HANGUL SYLLABLE JYABS +C7DB;C7DB;110C 1163 11BA;C7DB;110C 1163 11BA; # (쟛; 쟛; 쟛; 쟛; 쟛; ) HANGUL SYLLABLE JYAS +C7DC;C7DC;110C 1163 11BB;C7DC;110C 1163 11BB; # (쟜; 쟜; 쟜; 쟜; 쟜; ) HANGUL SYLLABLE JYASS +C7DD;C7DD;110C 1163 11BC;C7DD;110C 1163 11BC; # (쟝; 쟝; 쟝; 쟝; 쟝; ) HANGUL SYLLABLE JYANG +C7DE;C7DE;110C 1163 11BD;C7DE;110C 1163 11BD; # (쟞; 쟞; 쟞; 쟞; 쟞; ) HANGUL SYLLABLE JYAJ +C7DF;C7DF;110C 1163 11BE;C7DF;110C 1163 11BE; # (쟟; 쟟; 쟟; 쟟; 쟟; ) HANGUL SYLLABLE JYAC +C7E0;C7E0;110C 1163 11BF;C7E0;110C 1163 11BF; # (쟠; 쟠; 쟠; 쟠; 쟠; ) HANGUL SYLLABLE JYAK +C7E1;C7E1;110C 1163 11C0;C7E1;110C 1163 11C0; # (쟡; 쟡; 쟡; 쟡; 쟡; ) HANGUL SYLLABLE JYAT +C7E2;C7E2;110C 1163 11C1;C7E2;110C 1163 11C1; # (쟢; 쟢; 쟢; 쟢; 쟢; ) HANGUL SYLLABLE JYAP +C7E3;C7E3;110C 1163 11C2;C7E3;110C 1163 11C2; # (쟣; 쟣; 쟣; 쟣; 쟣; ) HANGUL SYLLABLE JYAH +C7E4;C7E4;110C 1164;C7E4;110C 1164; # (쟤; 쟤; 쟤; 쟤; 쟤; ) HANGUL SYLLABLE JYAE +C7E5;C7E5;110C 1164 11A8;C7E5;110C 1164 11A8; # (쟥; 쟥; 쟥; 쟥; 쟥; ) HANGUL SYLLABLE JYAEG +C7E6;C7E6;110C 1164 11A9;C7E6;110C 1164 11A9; # (쟦; 쟦; 쟦; 쟦; 쟦; ) HANGUL SYLLABLE JYAEGG +C7E7;C7E7;110C 1164 11AA;C7E7;110C 1164 11AA; # (쟧; 쟧; 쟧; 쟧; 쟧; ) HANGUL SYLLABLE JYAEGS +C7E8;C7E8;110C 1164 11AB;C7E8;110C 1164 11AB; # (쟨; 쟨; 쟨; 쟨; 쟨; ) HANGUL SYLLABLE JYAEN +C7E9;C7E9;110C 1164 11AC;C7E9;110C 1164 11AC; # (쟩; 쟩; 쟩; 쟩; 쟩; ) HANGUL SYLLABLE JYAENJ +C7EA;C7EA;110C 1164 11AD;C7EA;110C 1164 11AD; # (쟪; 쟪; 쟪; 쟪; 쟪; ) HANGUL SYLLABLE JYAENH +C7EB;C7EB;110C 1164 11AE;C7EB;110C 1164 11AE; # (쟫; 쟫; 쟫; 쟫; 쟫; ) HANGUL SYLLABLE JYAED +C7EC;C7EC;110C 1164 11AF;C7EC;110C 1164 11AF; # (쟬; 쟬; 쟬; 쟬; 쟬; ) HANGUL SYLLABLE JYAEL +C7ED;C7ED;110C 1164 11B0;C7ED;110C 1164 11B0; # (쟭; 쟭; 쟭; 쟭; 쟭; ) HANGUL SYLLABLE JYAELG +C7EE;C7EE;110C 1164 11B1;C7EE;110C 1164 11B1; # (쟮; 쟮; 쟮; 쟮; 쟮; ) HANGUL SYLLABLE JYAELM +C7EF;C7EF;110C 1164 11B2;C7EF;110C 1164 11B2; # (쟯; 쟯; 쟯; 쟯; 쟯; ) HANGUL SYLLABLE JYAELB +C7F0;C7F0;110C 1164 11B3;C7F0;110C 1164 11B3; # (쟰; 쟰; 쟰; 쟰; 쟰; ) HANGUL SYLLABLE JYAELS +C7F1;C7F1;110C 1164 11B4;C7F1;110C 1164 11B4; # (쟱; 쟱; 쟱; 쟱; 쟱; ) HANGUL SYLLABLE JYAELT +C7F2;C7F2;110C 1164 11B5;C7F2;110C 1164 11B5; # (쟲; 쟲; 쟲; 쟲; 쟲; ) HANGUL SYLLABLE JYAELP +C7F3;C7F3;110C 1164 11B6;C7F3;110C 1164 11B6; # (쟳; 쟳; 쟳; 쟳; 쟳; ) HANGUL SYLLABLE JYAELH +C7F4;C7F4;110C 1164 11B7;C7F4;110C 1164 11B7; # (쟴; 쟴; 쟴; 쟴; 쟴; ) HANGUL SYLLABLE JYAEM +C7F5;C7F5;110C 1164 11B8;C7F5;110C 1164 11B8; # (쟵; 쟵; 쟵; 쟵; 쟵; ) HANGUL SYLLABLE JYAEB +C7F6;C7F6;110C 1164 11B9;C7F6;110C 1164 11B9; # (쟶; 쟶; 쟶; 쟶; 쟶; ) HANGUL SYLLABLE JYAEBS +C7F7;C7F7;110C 1164 11BA;C7F7;110C 1164 11BA; # (쟷; 쟷; 쟷; 쟷; 쟷; ) HANGUL SYLLABLE JYAES +C7F8;C7F8;110C 1164 11BB;C7F8;110C 1164 11BB; # (쟸; 쟸; 쟸; 쟸; 쟸; ) HANGUL SYLLABLE JYAESS +C7F9;C7F9;110C 1164 11BC;C7F9;110C 1164 11BC; # (쟹; 쟹; 쟹; 쟹; 쟹; ) HANGUL SYLLABLE JYAENG +C7FA;C7FA;110C 1164 11BD;C7FA;110C 1164 11BD; # (쟺; 쟺; 쟺; 쟺; 쟺; ) HANGUL SYLLABLE JYAEJ +C7FB;C7FB;110C 1164 11BE;C7FB;110C 1164 11BE; # (쟻; 쟻; 쟻; 쟻; 쟻; ) HANGUL SYLLABLE JYAEC +C7FC;C7FC;110C 1164 11BF;C7FC;110C 1164 11BF; # (쟼; 쟼; 쟼; 쟼; 쟼; ) HANGUL SYLLABLE JYAEK +C7FD;C7FD;110C 1164 11C0;C7FD;110C 1164 11C0; # (쟽; 쟽; 쟽; 쟽; 쟽; ) HANGUL SYLLABLE JYAET +C7FE;C7FE;110C 1164 11C1;C7FE;110C 1164 11C1; # (쟾; 쟾; 쟾; 쟾; 쟾; ) HANGUL SYLLABLE JYAEP +C7FF;C7FF;110C 1164 11C2;C7FF;110C 1164 11C2; # (쟿; 쟿; 쟿; 쟿; 쟿; ) HANGUL SYLLABLE JYAEH +C800;C800;110C 1165;C800;110C 1165; # (저; 저; 저; 저; 저; ) HANGUL SYLLABLE JEO +C801;C801;110C 1165 11A8;C801;110C 1165 11A8; # (적; 적; 적; 적; 적; ) HANGUL SYLLABLE JEOG +C802;C802;110C 1165 11A9;C802;110C 1165 11A9; # (젂; 젂; 젂; 젂; 젂; ) HANGUL SYLLABLE JEOGG +C803;C803;110C 1165 11AA;C803;110C 1165 11AA; # (젃; 젃; 젃; 젃; 젃; ) HANGUL SYLLABLE JEOGS +C804;C804;110C 1165 11AB;C804;110C 1165 11AB; # (전; 전; 전; 전; 전; ) HANGUL SYLLABLE JEON +C805;C805;110C 1165 11AC;C805;110C 1165 11AC; # (젅; 젅; 젅; 젅; 젅; ) HANGUL SYLLABLE JEONJ +C806;C806;110C 1165 11AD;C806;110C 1165 11AD; # (젆; 젆; 젆; 젆; 젆; ) HANGUL SYLLABLE JEONH +C807;C807;110C 1165 11AE;C807;110C 1165 11AE; # (젇; 젇; 젇; 젇; 젇; ) HANGUL SYLLABLE JEOD +C808;C808;110C 1165 11AF;C808;110C 1165 11AF; # (절; 절; 절; 절; 절; ) HANGUL SYLLABLE JEOL +C809;C809;110C 1165 11B0;C809;110C 1165 11B0; # (젉; 젉; 젉; 젉; 젉; ) HANGUL SYLLABLE JEOLG +C80A;C80A;110C 1165 11B1;C80A;110C 1165 11B1; # (젊; 젊; 젊; 젊; 젊; ) HANGUL SYLLABLE JEOLM +C80B;C80B;110C 1165 11B2;C80B;110C 1165 11B2; # (젋; 젋; 젋; 젋; 젋; ) HANGUL SYLLABLE JEOLB +C80C;C80C;110C 1165 11B3;C80C;110C 1165 11B3; # (젌; 젌; 젌; 젌; 젌; ) HANGUL SYLLABLE JEOLS +C80D;C80D;110C 1165 11B4;C80D;110C 1165 11B4; # (젍; 젍; 젍; 젍; 젍; ) HANGUL SYLLABLE JEOLT +C80E;C80E;110C 1165 11B5;C80E;110C 1165 11B5; # (젎; 젎; 젎; 젎; 젎; ) HANGUL SYLLABLE JEOLP +C80F;C80F;110C 1165 11B6;C80F;110C 1165 11B6; # (젏; 젏; 젏; 젏; 젏; ) HANGUL SYLLABLE JEOLH +C810;C810;110C 1165 11B7;C810;110C 1165 11B7; # (점; 점; 점; 점; 점; ) HANGUL SYLLABLE JEOM +C811;C811;110C 1165 11B8;C811;110C 1165 11B8; # (접; 접; 접; 접; 접; ) HANGUL SYLLABLE JEOB +C812;C812;110C 1165 11B9;C812;110C 1165 11B9; # (젒; 젒; 젒; 젒; 젒; ) HANGUL SYLLABLE JEOBS +C813;C813;110C 1165 11BA;C813;110C 1165 11BA; # (젓; 젓; 젓; 젓; 젓; ) HANGUL SYLLABLE JEOS +C814;C814;110C 1165 11BB;C814;110C 1165 11BB; # (젔; 젔; 젔; 젔; 젔; ) HANGUL SYLLABLE JEOSS +C815;C815;110C 1165 11BC;C815;110C 1165 11BC; # (정; 정; 정; 정; 정; ) HANGUL SYLLABLE JEONG +C816;C816;110C 1165 11BD;C816;110C 1165 11BD; # (젖; 젖; 젖; 젖; 젖; ) HANGUL SYLLABLE JEOJ +C817;C817;110C 1165 11BE;C817;110C 1165 11BE; # (젗; 젗; 젗; 젗; 젗; ) HANGUL SYLLABLE JEOC +C818;C818;110C 1165 11BF;C818;110C 1165 11BF; # (젘; 젘; 젘; 젘; 젘; ) HANGUL SYLLABLE JEOK +C819;C819;110C 1165 11C0;C819;110C 1165 11C0; # (젙; 젙; 젙; 젙; 젙; ) HANGUL SYLLABLE JEOT +C81A;C81A;110C 1165 11C1;C81A;110C 1165 11C1; # (젚; 젚; 젚; 젚; 젚; ) HANGUL SYLLABLE JEOP +C81B;C81B;110C 1165 11C2;C81B;110C 1165 11C2; # (젛; 젛; 젛; 젛; 젛; ) HANGUL SYLLABLE JEOH +C81C;C81C;110C 1166;C81C;110C 1166; # (제; 제; 제; 제; 제; ) HANGUL SYLLABLE JE +C81D;C81D;110C 1166 11A8;C81D;110C 1166 11A8; # (젝; 젝; 젝; 젝; 젝; ) HANGUL SYLLABLE JEG +C81E;C81E;110C 1166 11A9;C81E;110C 1166 11A9; # (젞; 젞; 젞; 젞; 젞; ) HANGUL SYLLABLE JEGG +C81F;C81F;110C 1166 11AA;C81F;110C 1166 11AA; # (젟; 젟; 젟; 젟; 젟; ) HANGUL SYLLABLE JEGS +C820;C820;110C 1166 11AB;C820;110C 1166 11AB; # (젠; 젠; 젠; 젠; 젠; ) HANGUL SYLLABLE JEN +C821;C821;110C 1166 11AC;C821;110C 1166 11AC; # (젡; 젡; 젡; 젡; 젡; ) HANGUL SYLLABLE JENJ +C822;C822;110C 1166 11AD;C822;110C 1166 11AD; # (젢; 젢; 젢; 젢; 젢; ) HANGUL SYLLABLE JENH +C823;C823;110C 1166 11AE;C823;110C 1166 11AE; # (젣; 젣; 젣; 젣; 젣; ) HANGUL SYLLABLE JED +C824;C824;110C 1166 11AF;C824;110C 1166 11AF; # (젤; 젤; 젤; 젤; 젤; ) HANGUL SYLLABLE JEL +C825;C825;110C 1166 11B0;C825;110C 1166 11B0; # (젥; 젥; 젥; 젥; 젥; ) HANGUL SYLLABLE JELG +C826;C826;110C 1166 11B1;C826;110C 1166 11B1; # (젦; 젦; 젦; 젦; 젦; ) HANGUL SYLLABLE JELM +C827;C827;110C 1166 11B2;C827;110C 1166 11B2; # (젧; 젧; 젧; 젧; 젧; ) HANGUL SYLLABLE JELB +C828;C828;110C 1166 11B3;C828;110C 1166 11B3; # (젨; 젨; 젨; 젨; 젨; ) HANGUL SYLLABLE JELS +C829;C829;110C 1166 11B4;C829;110C 1166 11B4; # (젩; 젩; 젩; 젩; 젩; ) HANGUL SYLLABLE JELT +C82A;C82A;110C 1166 11B5;C82A;110C 1166 11B5; # (젪; 젪; 젪; 젪; 젪; ) HANGUL SYLLABLE JELP +C82B;C82B;110C 1166 11B6;C82B;110C 1166 11B6; # (젫; 젫; 젫; 젫; 젫; ) HANGUL SYLLABLE JELH +C82C;C82C;110C 1166 11B7;C82C;110C 1166 11B7; # (젬; 젬; 젬; 젬; 젬; ) HANGUL SYLLABLE JEM +C82D;C82D;110C 1166 11B8;C82D;110C 1166 11B8; # (젭; 젭; 젭; 젭; 젭; ) HANGUL SYLLABLE JEB +C82E;C82E;110C 1166 11B9;C82E;110C 1166 11B9; # (젮; 젮; 젮; 젮; 젮; ) HANGUL SYLLABLE JEBS +C82F;C82F;110C 1166 11BA;C82F;110C 1166 11BA; # (젯; 젯; 젯; 젯; 젯; ) HANGUL SYLLABLE JES +C830;C830;110C 1166 11BB;C830;110C 1166 11BB; # (젰; 젰; 젰; 젰; 젰; ) HANGUL SYLLABLE JESS +C831;C831;110C 1166 11BC;C831;110C 1166 11BC; # (젱; 젱; 젱; 젱; 젱; ) HANGUL SYLLABLE JENG +C832;C832;110C 1166 11BD;C832;110C 1166 11BD; # (젲; 젲; 젲; 젲; 젲; ) HANGUL SYLLABLE JEJ +C833;C833;110C 1166 11BE;C833;110C 1166 11BE; # (젳; 젳; 젳; 젳; 젳; ) HANGUL SYLLABLE JEC +C834;C834;110C 1166 11BF;C834;110C 1166 11BF; # (젴; 젴; 젴; 젴; 젴; ) HANGUL SYLLABLE JEK +C835;C835;110C 1166 11C0;C835;110C 1166 11C0; # (젵; 젵; 젵; 젵; 젵; ) HANGUL SYLLABLE JET +C836;C836;110C 1166 11C1;C836;110C 1166 11C1; # (젶; 젶; 젶; 젶; 젶; ) HANGUL SYLLABLE JEP +C837;C837;110C 1166 11C2;C837;110C 1166 11C2; # (젷; 젷; 젷; 젷; 젷; ) HANGUL SYLLABLE JEH +C838;C838;110C 1167;C838;110C 1167; # (져; 져; 져; 져; 져; ) HANGUL SYLLABLE JYEO +C839;C839;110C 1167 11A8;C839;110C 1167 11A8; # (젹; 젹; 젹; 젹; 젹; ) HANGUL SYLLABLE JYEOG +C83A;C83A;110C 1167 11A9;C83A;110C 1167 11A9; # (젺; 젺; 젺; 젺; 젺; ) HANGUL SYLLABLE JYEOGG +C83B;C83B;110C 1167 11AA;C83B;110C 1167 11AA; # (젻; 젻; 젻; 젻; 젻; ) HANGUL SYLLABLE JYEOGS +C83C;C83C;110C 1167 11AB;C83C;110C 1167 11AB; # (젼; 젼; 젼; 젼; 젼; ) HANGUL SYLLABLE JYEON +C83D;C83D;110C 1167 11AC;C83D;110C 1167 11AC; # (젽; 젽; 젽; 젽; 젽; ) HANGUL SYLLABLE JYEONJ +C83E;C83E;110C 1167 11AD;C83E;110C 1167 11AD; # (젾; 젾; 젾; 젾; 젾; ) HANGUL SYLLABLE JYEONH +C83F;C83F;110C 1167 11AE;C83F;110C 1167 11AE; # (젿; 젿; 젿; 젿; 젿; ) HANGUL SYLLABLE JYEOD +C840;C840;110C 1167 11AF;C840;110C 1167 11AF; # (졀; 졀; 졀; 졀; 졀; ) HANGUL SYLLABLE JYEOL +C841;C841;110C 1167 11B0;C841;110C 1167 11B0; # (졁; 졁; 졁; 졁; 졁; ) HANGUL SYLLABLE JYEOLG +C842;C842;110C 1167 11B1;C842;110C 1167 11B1; # (졂; 졂; 졂; 졂; 졂; ) HANGUL SYLLABLE JYEOLM +C843;C843;110C 1167 11B2;C843;110C 1167 11B2; # (졃; 졃; 졃; 졃; 졃; ) HANGUL SYLLABLE JYEOLB +C844;C844;110C 1167 11B3;C844;110C 1167 11B3; # (졄; 졄; 졄; 졄; 졄; ) HANGUL SYLLABLE JYEOLS +C845;C845;110C 1167 11B4;C845;110C 1167 11B4; # (졅; 졅; 졅; 졅; 졅; ) HANGUL SYLLABLE JYEOLT +C846;C846;110C 1167 11B5;C846;110C 1167 11B5; # (졆; 졆; 졆; 졆; 졆; ) HANGUL SYLLABLE JYEOLP +C847;C847;110C 1167 11B6;C847;110C 1167 11B6; # (졇; 졇; 졇; 졇; 졇; ) HANGUL SYLLABLE JYEOLH +C848;C848;110C 1167 11B7;C848;110C 1167 11B7; # (졈; 졈; 졈; 졈; 졈; ) HANGUL SYLLABLE JYEOM +C849;C849;110C 1167 11B8;C849;110C 1167 11B8; # (졉; 졉; 졉; 졉; 졉; ) HANGUL SYLLABLE JYEOB +C84A;C84A;110C 1167 11B9;C84A;110C 1167 11B9; # (졊; 졊; 졊; 졊; 졊; ) HANGUL SYLLABLE JYEOBS +C84B;C84B;110C 1167 11BA;C84B;110C 1167 11BA; # (졋; 졋; 졋; 졋; 졋; ) HANGUL SYLLABLE JYEOS +C84C;C84C;110C 1167 11BB;C84C;110C 1167 11BB; # (졌; 졌; 졌; 졌; 졌; ) HANGUL SYLLABLE JYEOSS +C84D;C84D;110C 1167 11BC;C84D;110C 1167 11BC; # (졍; 졍; 졍; 졍; 졍; ) HANGUL SYLLABLE JYEONG +C84E;C84E;110C 1167 11BD;C84E;110C 1167 11BD; # (졎; 졎; 졎; 졎; 졎; ) HANGUL SYLLABLE JYEOJ +C84F;C84F;110C 1167 11BE;C84F;110C 1167 11BE; # (졏; 졏; 졏; 졏; 졏; ) HANGUL SYLLABLE JYEOC +C850;C850;110C 1167 11BF;C850;110C 1167 11BF; # (졐; 졐; 졐; 졐; 졐; ) HANGUL SYLLABLE JYEOK +C851;C851;110C 1167 11C0;C851;110C 1167 11C0; # (졑; 졑; 졑; 졑; 졑; ) HANGUL SYLLABLE JYEOT +C852;C852;110C 1167 11C1;C852;110C 1167 11C1; # (졒; 졒; 졒; 졒; 졒; ) HANGUL SYLLABLE JYEOP +C853;C853;110C 1167 11C2;C853;110C 1167 11C2; # (졓; 졓; 졓; 졓; 졓; ) HANGUL SYLLABLE JYEOH +C854;C854;110C 1168;C854;110C 1168; # (졔; 졔; 졔; 졔; 졔; ) HANGUL SYLLABLE JYE +C855;C855;110C 1168 11A8;C855;110C 1168 11A8; # (졕; 졕; 졕; 졕; 졕; ) HANGUL SYLLABLE JYEG +C856;C856;110C 1168 11A9;C856;110C 1168 11A9; # (졖; 졖; 졖; 졖; 졖; ) HANGUL SYLLABLE JYEGG +C857;C857;110C 1168 11AA;C857;110C 1168 11AA; # (졗; 졗; 졗; 졗; 졗; ) HANGUL SYLLABLE JYEGS +C858;C858;110C 1168 11AB;C858;110C 1168 11AB; # (졘; 졘; 졘; 졘; 졘; ) HANGUL SYLLABLE JYEN +C859;C859;110C 1168 11AC;C859;110C 1168 11AC; # (졙; 졙; 졙; 졙; 졙; ) HANGUL SYLLABLE JYENJ +C85A;C85A;110C 1168 11AD;C85A;110C 1168 11AD; # (졚; 졚; 졚; 졚; 졚; ) HANGUL SYLLABLE JYENH +C85B;C85B;110C 1168 11AE;C85B;110C 1168 11AE; # (졛; 졛; 졛; 졛; 졛; ) HANGUL SYLLABLE JYED +C85C;C85C;110C 1168 11AF;C85C;110C 1168 11AF; # (졜; 졜; 졜; 졜; 졜; ) HANGUL SYLLABLE JYEL +C85D;C85D;110C 1168 11B0;C85D;110C 1168 11B0; # (졝; 졝; 졝; 졝; 졝; ) HANGUL SYLLABLE JYELG +C85E;C85E;110C 1168 11B1;C85E;110C 1168 11B1; # (졞; 졞; 졞; 졞; 졞; ) HANGUL SYLLABLE JYELM +C85F;C85F;110C 1168 11B2;C85F;110C 1168 11B2; # (졟; 졟; 졟; 졟; 졟; ) HANGUL SYLLABLE JYELB +C860;C860;110C 1168 11B3;C860;110C 1168 11B3; # (졠; 졠; 졠; 졠; 졠; ) HANGUL SYLLABLE JYELS +C861;C861;110C 1168 11B4;C861;110C 1168 11B4; # (졡; 졡; 졡; 졡; 졡; ) HANGUL SYLLABLE JYELT +C862;C862;110C 1168 11B5;C862;110C 1168 11B5; # (졢; 졢; 졢; 졢; 졢; ) HANGUL SYLLABLE JYELP +C863;C863;110C 1168 11B6;C863;110C 1168 11B6; # (졣; 졣; 졣; 졣; 졣; ) HANGUL SYLLABLE JYELH +C864;C864;110C 1168 11B7;C864;110C 1168 11B7; # (졤; 졤; 졤; 졤; 졤; ) HANGUL SYLLABLE JYEM +C865;C865;110C 1168 11B8;C865;110C 1168 11B8; # (졥; 졥; 졥; 졥; 졥; ) HANGUL SYLLABLE JYEB +C866;C866;110C 1168 11B9;C866;110C 1168 11B9; # (졦; 졦; 졦; 졦; 졦; ) HANGUL SYLLABLE JYEBS +C867;C867;110C 1168 11BA;C867;110C 1168 11BA; # (졧; 졧; 졧; 졧; 졧; ) HANGUL SYLLABLE JYES +C868;C868;110C 1168 11BB;C868;110C 1168 11BB; # (졨; 졨; 졨; 졨; 졨; ) HANGUL SYLLABLE JYESS +C869;C869;110C 1168 11BC;C869;110C 1168 11BC; # (졩; 졩; 졩; 졩; 졩; ) HANGUL SYLLABLE JYENG +C86A;C86A;110C 1168 11BD;C86A;110C 1168 11BD; # (졪; 졪; 졪; 졪; 졪; ) HANGUL SYLLABLE JYEJ +C86B;C86B;110C 1168 11BE;C86B;110C 1168 11BE; # (졫; 졫; 졫; 졫; 졫; ) HANGUL SYLLABLE JYEC +C86C;C86C;110C 1168 11BF;C86C;110C 1168 11BF; # (졬; 졬; 졬; 졬; 졬; ) HANGUL SYLLABLE JYEK +C86D;C86D;110C 1168 11C0;C86D;110C 1168 11C0; # (졭; 졭; 졭; 졭; 졭; ) HANGUL SYLLABLE JYET +C86E;C86E;110C 1168 11C1;C86E;110C 1168 11C1; # (졮; 졮; 졮; 졮; 졮; ) HANGUL SYLLABLE JYEP +C86F;C86F;110C 1168 11C2;C86F;110C 1168 11C2; # (졯; 졯; 졯; 졯; 졯; ) HANGUL SYLLABLE JYEH +C870;C870;110C 1169;C870;110C 1169; # (조; 조; 조; 조; 조; ) HANGUL SYLLABLE JO +C871;C871;110C 1169 11A8;C871;110C 1169 11A8; # (족; 족; 족; 족; 족; ) HANGUL SYLLABLE JOG +C872;C872;110C 1169 11A9;C872;110C 1169 11A9; # (졲; 졲; 졲; 졲; 졲; ) HANGUL SYLLABLE JOGG +C873;C873;110C 1169 11AA;C873;110C 1169 11AA; # (졳; 졳; 졳; 졳; 졳; ) HANGUL SYLLABLE JOGS +C874;C874;110C 1169 11AB;C874;110C 1169 11AB; # (존; 존; 존; 존; 존; ) HANGUL SYLLABLE JON +C875;C875;110C 1169 11AC;C875;110C 1169 11AC; # (졵; 졵; 졵; 졵; 졵; ) HANGUL SYLLABLE JONJ +C876;C876;110C 1169 11AD;C876;110C 1169 11AD; # (졶; 졶; 졶; 졶; 졶; ) HANGUL SYLLABLE JONH +C877;C877;110C 1169 11AE;C877;110C 1169 11AE; # (졷; 졷; 졷; 졷; 졷; ) HANGUL SYLLABLE JOD +C878;C878;110C 1169 11AF;C878;110C 1169 11AF; # (졸; 졸; 졸; 졸; 졸; ) HANGUL SYLLABLE JOL +C879;C879;110C 1169 11B0;C879;110C 1169 11B0; # (졹; 졹; 졹; 졹; 졹; ) HANGUL SYLLABLE JOLG +C87A;C87A;110C 1169 11B1;C87A;110C 1169 11B1; # (졺; 졺; 졺; 졺; 졺; ) HANGUL SYLLABLE JOLM +C87B;C87B;110C 1169 11B2;C87B;110C 1169 11B2; # (졻; 졻; 졻; 졻; 졻; ) HANGUL SYLLABLE JOLB +C87C;C87C;110C 1169 11B3;C87C;110C 1169 11B3; # (졼; 졼; 졼; 졼; 졼; ) HANGUL SYLLABLE JOLS +C87D;C87D;110C 1169 11B4;C87D;110C 1169 11B4; # (졽; 졽; 졽; 졽; 졽; ) HANGUL SYLLABLE JOLT +C87E;C87E;110C 1169 11B5;C87E;110C 1169 11B5; # (졾; 졾; 졾; 졾; 졾; ) HANGUL SYLLABLE JOLP +C87F;C87F;110C 1169 11B6;C87F;110C 1169 11B6; # (졿; 졿; 졿; 졿; 졿; ) HANGUL SYLLABLE JOLH +C880;C880;110C 1169 11B7;C880;110C 1169 11B7; # (좀; 좀; 좀; 좀; 좀; ) HANGUL SYLLABLE JOM +C881;C881;110C 1169 11B8;C881;110C 1169 11B8; # (좁; 좁; 좁; 좁; 좁; ) HANGUL SYLLABLE JOB +C882;C882;110C 1169 11B9;C882;110C 1169 11B9; # (좂; 좂; 좂; 좂; 좂; ) HANGUL SYLLABLE JOBS +C883;C883;110C 1169 11BA;C883;110C 1169 11BA; # (좃; 좃; 좃; 좃; 좃; ) HANGUL SYLLABLE JOS +C884;C884;110C 1169 11BB;C884;110C 1169 11BB; # (좄; 좄; 좄; 좄; 좄; ) HANGUL SYLLABLE JOSS +C885;C885;110C 1169 11BC;C885;110C 1169 11BC; # (종; 종; 종; 종; 종; ) HANGUL SYLLABLE JONG +C886;C886;110C 1169 11BD;C886;110C 1169 11BD; # (좆; 좆; 좆; 좆; 좆; ) HANGUL SYLLABLE JOJ +C887;C887;110C 1169 11BE;C887;110C 1169 11BE; # (좇; 좇; 좇; 좇; 좇; ) HANGUL SYLLABLE JOC +C888;C888;110C 1169 11BF;C888;110C 1169 11BF; # (좈; 좈; 좈; 좈; 좈; ) HANGUL SYLLABLE JOK +C889;C889;110C 1169 11C0;C889;110C 1169 11C0; # (좉; 좉; 좉; 좉; 좉; ) HANGUL SYLLABLE JOT +C88A;C88A;110C 1169 11C1;C88A;110C 1169 11C1; # (좊; 좊; 좊; 좊; 좊; ) HANGUL SYLLABLE JOP +C88B;C88B;110C 1169 11C2;C88B;110C 1169 11C2; # (좋; 좋; 좋; 좋; 좋; ) HANGUL SYLLABLE JOH +C88C;C88C;110C 116A;C88C;110C 116A; # (좌; 좌; 좌; 좌; 좌; ) HANGUL SYLLABLE JWA +C88D;C88D;110C 116A 11A8;C88D;110C 116A 11A8; # (좍; 좍; 좍; 좍; 좍; ) HANGUL SYLLABLE JWAG +C88E;C88E;110C 116A 11A9;C88E;110C 116A 11A9; # (좎; 좎; 좎; 좎; 좎; ) HANGUL SYLLABLE JWAGG +C88F;C88F;110C 116A 11AA;C88F;110C 116A 11AA; # (좏; 좏; 좏; 좏; 좏; ) HANGUL SYLLABLE JWAGS +C890;C890;110C 116A 11AB;C890;110C 116A 11AB; # (좐; 좐; 좐; 좐; 좐; ) HANGUL SYLLABLE JWAN +C891;C891;110C 116A 11AC;C891;110C 116A 11AC; # (좑; 좑; 좑; 좑; 좑; ) HANGUL SYLLABLE JWANJ +C892;C892;110C 116A 11AD;C892;110C 116A 11AD; # (좒; 좒; 좒; 좒; 좒; ) HANGUL SYLLABLE JWANH +C893;C893;110C 116A 11AE;C893;110C 116A 11AE; # (좓; 좓; 좓; 좓; 좓; ) HANGUL SYLLABLE JWAD +C894;C894;110C 116A 11AF;C894;110C 116A 11AF; # (좔; 좔; 좔; 좔; 좔; ) HANGUL SYLLABLE JWAL +C895;C895;110C 116A 11B0;C895;110C 116A 11B0; # (좕; 좕; 좕; 좕; 좕; ) HANGUL SYLLABLE JWALG +C896;C896;110C 116A 11B1;C896;110C 116A 11B1; # (좖; 좖; 좖; 좖; 좖; ) HANGUL SYLLABLE JWALM +C897;C897;110C 116A 11B2;C897;110C 116A 11B2; # (좗; 좗; 좗; 좗; 좗; ) HANGUL SYLLABLE JWALB +C898;C898;110C 116A 11B3;C898;110C 116A 11B3; # (좘; 좘; 좘; 좘; 좘; ) HANGUL SYLLABLE JWALS +C899;C899;110C 116A 11B4;C899;110C 116A 11B4; # (좙; 좙; 좙; 좙; 좙; ) HANGUL SYLLABLE JWALT +C89A;C89A;110C 116A 11B5;C89A;110C 116A 11B5; # (좚; 좚; 좚; 좚; 좚; ) HANGUL SYLLABLE JWALP +C89B;C89B;110C 116A 11B6;C89B;110C 116A 11B6; # (좛; 좛; 좛; 좛; 좛; ) HANGUL SYLLABLE JWALH +C89C;C89C;110C 116A 11B7;C89C;110C 116A 11B7; # (좜; 좜; 좜; 좜; 좜; ) HANGUL SYLLABLE JWAM +C89D;C89D;110C 116A 11B8;C89D;110C 116A 11B8; # (좝; 좝; 좝; 좝; 좝; ) HANGUL SYLLABLE JWAB +C89E;C89E;110C 116A 11B9;C89E;110C 116A 11B9; # (좞; 좞; 좞; 좞; 좞; ) HANGUL SYLLABLE JWABS +C89F;C89F;110C 116A 11BA;C89F;110C 116A 11BA; # (좟; 좟; 좟; 좟; 좟; ) HANGUL SYLLABLE JWAS +C8A0;C8A0;110C 116A 11BB;C8A0;110C 116A 11BB; # (좠; 좠; 좠; 좠; 좠; ) HANGUL SYLLABLE JWASS +C8A1;C8A1;110C 116A 11BC;C8A1;110C 116A 11BC; # (좡; 좡; 좡; 좡; 좡; ) HANGUL SYLLABLE JWANG +C8A2;C8A2;110C 116A 11BD;C8A2;110C 116A 11BD; # (좢; 좢; 좢; 좢; 좢; ) HANGUL SYLLABLE JWAJ +C8A3;C8A3;110C 116A 11BE;C8A3;110C 116A 11BE; # (좣; 좣; 좣; 좣; 좣; ) HANGUL SYLLABLE JWAC +C8A4;C8A4;110C 116A 11BF;C8A4;110C 116A 11BF; # (좤; 좤; 좤; 좤; 좤; ) HANGUL SYLLABLE JWAK +C8A5;C8A5;110C 116A 11C0;C8A5;110C 116A 11C0; # (좥; 좥; 좥; 좥; 좥; ) HANGUL SYLLABLE JWAT +C8A6;C8A6;110C 116A 11C1;C8A6;110C 116A 11C1; # (좦; 좦; 좦; 좦; 좦; ) HANGUL SYLLABLE JWAP +C8A7;C8A7;110C 116A 11C2;C8A7;110C 116A 11C2; # (좧; 좧; 좧; 좧; 좧; ) HANGUL SYLLABLE JWAH +C8A8;C8A8;110C 116B;C8A8;110C 116B; # (좨; 좨; 좨; 좨; 좨; ) HANGUL SYLLABLE JWAE +C8A9;C8A9;110C 116B 11A8;C8A9;110C 116B 11A8; # (좩; 좩; 좩; 좩; 좩; ) HANGUL SYLLABLE JWAEG +C8AA;C8AA;110C 116B 11A9;C8AA;110C 116B 11A9; # (좪; 좪; 좪; 좪; 좪; ) HANGUL SYLLABLE JWAEGG +C8AB;C8AB;110C 116B 11AA;C8AB;110C 116B 11AA; # (좫; 좫; 좫; 좫; 좫; ) HANGUL SYLLABLE JWAEGS +C8AC;C8AC;110C 116B 11AB;C8AC;110C 116B 11AB; # (좬; 좬; 좬; 좬; 좬; ) HANGUL SYLLABLE JWAEN +C8AD;C8AD;110C 116B 11AC;C8AD;110C 116B 11AC; # (좭; 좭; 좭; 좭; 좭; ) HANGUL SYLLABLE JWAENJ +C8AE;C8AE;110C 116B 11AD;C8AE;110C 116B 11AD; # (좮; 좮; 좮; 좮; 좮; ) HANGUL SYLLABLE JWAENH +C8AF;C8AF;110C 116B 11AE;C8AF;110C 116B 11AE; # (좯; 좯; 좯; 좯; 좯; ) HANGUL SYLLABLE JWAED +C8B0;C8B0;110C 116B 11AF;C8B0;110C 116B 11AF; # (좰; 좰; 좰; 좰; 좰; ) HANGUL SYLLABLE JWAEL +C8B1;C8B1;110C 116B 11B0;C8B1;110C 116B 11B0; # (좱; 좱; 좱; 좱; 좱; ) HANGUL SYLLABLE JWAELG +C8B2;C8B2;110C 116B 11B1;C8B2;110C 116B 11B1; # (좲; 좲; 좲; 좲; 좲; ) HANGUL SYLLABLE JWAELM +C8B3;C8B3;110C 116B 11B2;C8B3;110C 116B 11B2; # (좳; 좳; 좳; 좳; 좳; ) HANGUL SYLLABLE JWAELB +C8B4;C8B4;110C 116B 11B3;C8B4;110C 116B 11B3; # (좴; 좴; 좴; 좴; 좴; ) HANGUL SYLLABLE JWAELS +C8B5;C8B5;110C 116B 11B4;C8B5;110C 116B 11B4; # (좵; 좵; 좵; 좵; 좵; ) HANGUL SYLLABLE JWAELT +C8B6;C8B6;110C 116B 11B5;C8B6;110C 116B 11B5; # (좶; 좶; 좶; 좶; 좶; ) HANGUL SYLLABLE JWAELP +C8B7;C8B7;110C 116B 11B6;C8B7;110C 116B 11B6; # (좷; 좷; 좷; 좷; 좷; ) HANGUL SYLLABLE JWAELH +C8B8;C8B8;110C 116B 11B7;C8B8;110C 116B 11B7; # (좸; 좸; 좸; 좸; 좸; ) HANGUL SYLLABLE JWAEM +C8B9;C8B9;110C 116B 11B8;C8B9;110C 116B 11B8; # (좹; 좹; 좹; 좹; 좹; ) HANGUL SYLLABLE JWAEB +C8BA;C8BA;110C 116B 11B9;C8BA;110C 116B 11B9; # (좺; 좺; 좺; 좺; 좺; ) HANGUL SYLLABLE JWAEBS +C8BB;C8BB;110C 116B 11BA;C8BB;110C 116B 11BA; # (좻; 좻; 좻; 좻; 좻; ) HANGUL SYLLABLE JWAES +C8BC;C8BC;110C 116B 11BB;C8BC;110C 116B 11BB; # (좼; 좼; 좼; 좼; 좼; ) HANGUL SYLLABLE JWAESS +C8BD;C8BD;110C 116B 11BC;C8BD;110C 116B 11BC; # (좽; 좽; 좽; 좽; 좽; ) HANGUL SYLLABLE JWAENG +C8BE;C8BE;110C 116B 11BD;C8BE;110C 116B 11BD; # (좾; 좾; 좾; 좾; 좾; ) HANGUL SYLLABLE JWAEJ +C8BF;C8BF;110C 116B 11BE;C8BF;110C 116B 11BE; # (좿; 좿; 좿; 좿; 좿; ) HANGUL SYLLABLE JWAEC +C8C0;C8C0;110C 116B 11BF;C8C0;110C 116B 11BF; # (죀; 죀; 죀; 죀; 죀; ) HANGUL SYLLABLE JWAEK +C8C1;C8C1;110C 116B 11C0;C8C1;110C 116B 11C0; # (죁; 죁; 죁; 죁; 죁; ) HANGUL SYLLABLE JWAET +C8C2;C8C2;110C 116B 11C1;C8C2;110C 116B 11C1; # (죂; 죂; 죂; 죂; 죂; ) HANGUL SYLLABLE JWAEP +C8C3;C8C3;110C 116B 11C2;C8C3;110C 116B 11C2; # (죃; 죃; 죃; 죃; 죃; ) HANGUL SYLLABLE JWAEH +C8C4;C8C4;110C 116C;C8C4;110C 116C; # (죄; 죄; 죄; 죄; 죄; ) HANGUL SYLLABLE JOE +C8C5;C8C5;110C 116C 11A8;C8C5;110C 116C 11A8; # (죅; 죅; 죅; 죅; 죅; ) HANGUL SYLLABLE JOEG +C8C6;C8C6;110C 116C 11A9;C8C6;110C 116C 11A9; # (죆; 죆; 죆; 죆; 죆; ) HANGUL SYLLABLE JOEGG +C8C7;C8C7;110C 116C 11AA;C8C7;110C 116C 11AA; # (죇; 죇; 죇; 죇; 죇; ) HANGUL SYLLABLE JOEGS +C8C8;C8C8;110C 116C 11AB;C8C8;110C 116C 11AB; # (죈; 죈; 죈; 죈; 죈; ) HANGUL SYLLABLE JOEN +C8C9;C8C9;110C 116C 11AC;C8C9;110C 116C 11AC; # (죉; 죉; 죉; 죉; 죉; ) HANGUL SYLLABLE JOENJ +C8CA;C8CA;110C 116C 11AD;C8CA;110C 116C 11AD; # (죊; 죊; 죊; 죊; 죊; ) HANGUL SYLLABLE JOENH +C8CB;C8CB;110C 116C 11AE;C8CB;110C 116C 11AE; # (죋; 죋; 죋; 죋; 죋; ) HANGUL SYLLABLE JOED +C8CC;C8CC;110C 116C 11AF;C8CC;110C 116C 11AF; # (죌; 죌; 죌; 죌; 죌; ) HANGUL SYLLABLE JOEL +C8CD;C8CD;110C 116C 11B0;C8CD;110C 116C 11B0; # (죍; 죍; 죍; 죍; 죍; ) HANGUL SYLLABLE JOELG +C8CE;C8CE;110C 116C 11B1;C8CE;110C 116C 11B1; # (죎; 죎; 죎; 죎; 죎; ) HANGUL SYLLABLE JOELM +C8CF;C8CF;110C 116C 11B2;C8CF;110C 116C 11B2; # (죏; 죏; 죏; 죏; 죏; ) HANGUL SYLLABLE JOELB +C8D0;C8D0;110C 116C 11B3;C8D0;110C 116C 11B3; # (죐; 죐; 죐; 죐; 죐; ) HANGUL SYLLABLE JOELS +C8D1;C8D1;110C 116C 11B4;C8D1;110C 116C 11B4; # (죑; 죑; 죑; 죑; 죑; ) HANGUL SYLLABLE JOELT +C8D2;C8D2;110C 116C 11B5;C8D2;110C 116C 11B5; # (죒; 죒; 죒; 죒; 죒; ) HANGUL SYLLABLE JOELP +C8D3;C8D3;110C 116C 11B6;C8D3;110C 116C 11B6; # (죓; 죓; 죓; 죓; 죓; ) HANGUL SYLLABLE JOELH +C8D4;C8D4;110C 116C 11B7;C8D4;110C 116C 11B7; # (죔; 죔; 죔; 죔; 죔; ) HANGUL SYLLABLE JOEM +C8D5;C8D5;110C 116C 11B8;C8D5;110C 116C 11B8; # (죕; 죕; 죕; 죕; 죕; ) HANGUL SYLLABLE JOEB +C8D6;C8D6;110C 116C 11B9;C8D6;110C 116C 11B9; # (죖; 죖; 죖; 죖; 죖; ) HANGUL SYLLABLE JOEBS +C8D7;C8D7;110C 116C 11BA;C8D7;110C 116C 11BA; # (죗; 죗; 죗; 죗; 죗; ) HANGUL SYLLABLE JOES +C8D8;C8D8;110C 116C 11BB;C8D8;110C 116C 11BB; # (죘; 죘; 죘; 죘; 죘; ) HANGUL SYLLABLE JOESS +C8D9;C8D9;110C 116C 11BC;C8D9;110C 116C 11BC; # (죙; 죙; 죙; 죙; 죙; ) HANGUL SYLLABLE JOENG +C8DA;C8DA;110C 116C 11BD;C8DA;110C 116C 11BD; # (죚; 죚; 죚; 죚; 죚; ) HANGUL SYLLABLE JOEJ +C8DB;C8DB;110C 116C 11BE;C8DB;110C 116C 11BE; # (죛; 죛; 죛; 죛; 죛; ) HANGUL SYLLABLE JOEC +C8DC;C8DC;110C 116C 11BF;C8DC;110C 116C 11BF; # (죜; 죜; 죜; 죜; 죜; ) HANGUL SYLLABLE JOEK +C8DD;C8DD;110C 116C 11C0;C8DD;110C 116C 11C0; # (죝; 죝; 죝; 죝; 죝; ) HANGUL SYLLABLE JOET +C8DE;C8DE;110C 116C 11C1;C8DE;110C 116C 11C1; # (죞; 죞; 죞; 죞; 죞; ) HANGUL SYLLABLE JOEP +C8DF;C8DF;110C 116C 11C2;C8DF;110C 116C 11C2; # (죟; 죟; 죟; 죟; 죟; ) HANGUL SYLLABLE JOEH +C8E0;C8E0;110C 116D;C8E0;110C 116D; # (죠; 죠; 죠; 죠; 죠; ) HANGUL SYLLABLE JYO +C8E1;C8E1;110C 116D 11A8;C8E1;110C 116D 11A8; # (죡; 죡; 죡; 죡; 죡; ) HANGUL SYLLABLE JYOG +C8E2;C8E2;110C 116D 11A9;C8E2;110C 116D 11A9; # (죢; 죢; 죢; 죢; 죢; ) HANGUL SYLLABLE JYOGG +C8E3;C8E3;110C 116D 11AA;C8E3;110C 116D 11AA; # (죣; 죣; 죣; 죣; 죣; ) HANGUL SYLLABLE JYOGS +C8E4;C8E4;110C 116D 11AB;C8E4;110C 116D 11AB; # (죤; 죤; 죤; 죤; 죤; ) HANGUL SYLLABLE JYON +C8E5;C8E5;110C 116D 11AC;C8E5;110C 116D 11AC; # (죥; 죥; 죥; 죥; 죥; ) HANGUL SYLLABLE JYONJ +C8E6;C8E6;110C 116D 11AD;C8E6;110C 116D 11AD; # (죦; 죦; 죦; 죦; 죦; ) HANGUL SYLLABLE JYONH +C8E7;C8E7;110C 116D 11AE;C8E7;110C 116D 11AE; # (죧; 죧; 죧; 죧; 죧; ) HANGUL SYLLABLE JYOD +C8E8;C8E8;110C 116D 11AF;C8E8;110C 116D 11AF; # (죨; 죨; 죨; 죨; 죨; ) HANGUL SYLLABLE JYOL +C8E9;C8E9;110C 116D 11B0;C8E9;110C 116D 11B0; # (죩; 죩; 죩; 죩; 죩; ) HANGUL SYLLABLE JYOLG +C8EA;C8EA;110C 116D 11B1;C8EA;110C 116D 11B1; # (죪; 죪; 죪; 죪; 죪; ) HANGUL SYLLABLE JYOLM +C8EB;C8EB;110C 116D 11B2;C8EB;110C 116D 11B2; # (죫; 죫; 죫; 죫; 죫; ) HANGUL SYLLABLE JYOLB +C8EC;C8EC;110C 116D 11B3;C8EC;110C 116D 11B3; # (죬; 죬; 죬; 죬; 죬; ) HANGUL SYLLABLE JYOLS +C8ED;C8ED;110C 116D 11B4;C8ED;110C 116D 11B4; # (죭; 죭; 죭; 죭; 죭; ) HANGUL SYLLABLE JYOLT +C8EE;C8EE;110C 116D 11B5;C8EE;110C 116D 11B5; # (죮; 죮; 죮; 죮; 죮; ) HANGUL SYLLABLE JYOLP +C8EF;C8EF;110C 116D 11B6;C8EF;110C 116D 11B6; # (죯; 죯; 죯; 죯; 죯; ) HANGUL SYLLABLE JYOLH +C8F0;C8F0;110C 116D 11B7;C8F0;110C 116D 11B7; # (죰; 죰; 죰; 죰; 죰; ) HANGUL SYLLABLE JYOM +C8F1;C8F1;110C 116D 11B8;C8F1;110C 116D 11B8; # (죱; 죱; 죱; 죱; 죱; ) HANGUL SYLLABLE JYOB +C8F2;C8F2;110C 116D 11B9;C8F2;110C 116D 11B9; # (죲; 죲; 죲; 죲; 죲; ) HANGUL SYLLABLE JYOBS +C8F3;C8F3;110C 116D 11BA;C8F3;110C 116D 11BA; # (죳; 죳; 죳; 죳; 죳; ) HANGUL SYLLABLE JYOS +C8F4;C8F4;110C 116D 11BB;C8F4;110C 116D 11BB; # (죴; 죴; 죴; 죴; 죴; ) HANGUL SYLLABLE JYOSS +C8F5;C8F5;110C 116D 11BC;C8F5;110C 116D 11BC; # (죵; 죵; 죵; 죵; 죵; ) HANGUL SYLLABLE JYONG +C8F6;C8F6;110C 116D 11BD;C8F6;110C 116D 11BD; # (죶; 죶; 죶; 죶; 죶; ) HANGUL SYLLABLE JYOJ +C8F7;C8F7;110C 116D 11BE;C8F7;110C 116D 11BE; # (죷; 죷; 죷; 죷; 죷; ) HANGUL SYLLABLE JYOC +C8F8;C8F8;110C 116D 11BF;C8F8;110C 116D 11BF; # (죸; 죸; 죸; 죸; 죸; ) HANGUL SYLLABLE JYOK +C8F9;C8F9;110C 116D 11C0;C8F9;110C 116D 11C0; # (죹; 죹; 죹; 죹; 죹; ) HANGUL SYLLABLE JYOT +C8FA;C8FA;110C 116D 11C1;C8FA;110C 116D 11C1; # (죺; 죺; 죺; 죺; 죺; ) HANGUL SYLLABLE JYOP +C8FB;C8FB;110C 116D 11C2;C8FB;110C 116D 11C2; # (죻; 죻; 죻; 죻; 죻; ) HANGUL SYLLABLE JYOH +C8FC;C8FC;110C 116E;C8FC;110C 116E; # (주; 주; 주; 주; 주; ) HANGUL SYLLABLE JU +C8FD;C8FD;110C 116E 11A8;C8FD;110C 116E 11A8; # (죽; 죽; 죽; 죽; 죽; ) HANGUL SYLLABLE JUG +C8FE;C8FE;110C 116E 11A9;C8FE;110C 116E 11A9; # (죾; 죾; 죾; 죾; 죾; ) HANGUL SYLLABLE JUGG +C8FF;C8FF;110C 116E 11AA;C8FF;110C 116E 11AA; # (죿; 죿; 죿; 죿; 죿; ) HANGUL SYLLABLE JUGS +C900;C900;110C 116E 11AB;C900;110C 116E 11AB; # (준; 준; 준; 준; 준; ) HANGUL SYLLABLE JUN +C901;C901;110C 116E 11AC;C901;110C 116E 11AC; # (줁; 줁; 줁; 줁; 줁; ) HANGUL SYLLABLE JUNJ +C902;C902;110C 116E 11AD;C902;110C 116E 11AD; # (줂; 줂; 줂; 줂; 줂; ) HANGUL SYLLABLE JUNH +C903;C903;110C 116E 11AE;C903;110C 116E 11AE; # (줃; 줃; 줃; 줃; 줃; ) HANGUL SYLLABLE JUD +C904;C904;110C 116E 11AF;C904;110C 116E 11AF; # (줄; 줄; 줄; 줄; 줄; ) HANGUL SYLLABLE JUL +C905;C905;110C 116E 11B0;C905;110C 116E 11B0; # (줅; 줅; 줅; 줅; 줅; ) HANGUL SYLLABLE JULG +C906;C906;110C 116E 11B1;C906;110C 116E 11B1; # (줆; 줆; 줆; 줆; 줆; ) HANGUL SYLLABLE JULM +C907;C907;110C 116E 11B2;C907;110C 116E 11B2; # (줇; 줇; 줇; 줇; 줇; ) HANGUL SYLLABLE JULB +C908;C908;110C 116E 11B3;C908;110C 116E 11B3; # (줈; 줈; 줈; 줈; 줈; ) HANGUL SYLLABLE JULS +C909;C909;110C 116E 11B4;C909;110C 116E 11B4; # (줉; 줉; 줉; 줉; 줉; ) HANGUL SYLLABLE JULT +C90A;C90A;110C 116E 11B5;C90A;110C 116E 11B5; # (줊; 줊; 줊; 줊; 줊; ) HANGUL SYLLABLE JULP +C90B;C90B;110C 116E 11B6;C90B;110C 116E 11B6; # (줋; 줋; 줋; 줋; 줋; ) HANGUL SYLLABLE JULH +C90C;C90C;110C 116E 11B7;C90C;110C 116E 11B7; # (줌; 줌; 줌; 줌; 줌; ) HANGUL SYLLABLE JUM +C90D;C90D;110C 116E 11B8;C90D;110C 116E 11B8; # (줍; 줍; 줍; 줍; 줍; ) HANGUL SYLLABLE JUB +C90E;C90E;110C 116E 11B9;C90E;110C 116E 11B9; # (줎; 줎; 줎; 줎; 줎; ) HANGUL SYLLABLE JUBS +C90F;C90F;110C 116E 11BA;C90F;110C 116E 11BA; # (줏; 줏; 줏; 줏; 줏; ) HANGUL SYLLABLE JUS +C910;C910;110C 116E 11BB;C910;110C 116E 11BB; # (줐; 줐; 줐; 줐; 줐; ) HANGUL SYLLABLE JUSS +C911;C911;110C 116E 11BC;C911;110C 116E 11BC; # (중; 중; 중; 중; 중; ) HANGUL SYLLABLE JUNG +C912;C912;110C 116E 11BD;C912;110C 116E 11BD; # (줒; 줒; 줒; 줒; 줒; ) HANGUL SYLLABLE JUJ +C913;C913;110C 116E 11BE;C913;110C 116E 11BE; # (줓; 줓; 줓; 줓; 줓; ) HANGUL SYLLABLE JUC +C914;C914;110C 116E 11BF;C914;110C 116E 11BF; # (줔; 줔; 줔; 줔; 줔; ) HANGUL SYLLABLE JUK +C915;C915;110C 116E 11C0;C915;110C 116E 11C0; # (줕; 줕; 줕; 줕; 줕; ) HANGUL SYLLABLE JUT +C916;C916;110C 116E 11C1;C916;110C 116E 11C1; # (줖; 줖; 줖; 줖; 줖; ) HANGUL SYLLABLE JUP +C917;C917;110C 116E 11C2;C917;110C 116E 11C2; # (줗; 줗; 줗; 줗; 줗; ) HANGUL SYLLABLE JUH +C918;C918;110C 116F;C918;110C 116F; # (줘; 줘; 줘; 줘; 줘; ) HANGUL SYLLABLE JWEO +C919;C919;110C 116F 11A8;C919;110C 116F 11A8; # (줙; 줙; 줙; 줙; 줙; ) HANGUL SYLLABLE JWEOG +C91A;C91A;110C 116F 11A9;C91A;110C 116F 11A9; # (줚; 줚; 줚; 줚; 줚; ) HANGUL SYLLABLE JWEOGG +C91B;C91B;110C 116F 11AA;C91B;110C 116F 11AA; # (줛; 줛; 줛; 줛; 줛; ) HANGUL SYLLABLE JWEOGS +C91C;C91C;110C 116F 11AB;C91C;110C 116F 11AB; # (줜; 줜; 줜; 줜; 줜; ) HANGUL SYLLABLE JWEON +C91D;C91D;110C 116F 11AC;C91D;110C 116F 11AC; # (줝; 줝; 줝; 줝; 줝; ) HANGUL SYLLABLE JWEONJ +C91E;C91E;110C 116F 11AD;C91E;110C 116F 11AD; # (줞; 줞; 줞; 줞; 줞; ) HANGUL SYLLABLE JWEONH +C91F;C91F;110C 116F 11AE;C91F;110C 116F 11AE; # (줟; 줟; 줟; 줟; 줟; ) HANGUL SYLLABLE JWEOD +C920;C920;110C 116F 11AF;C920;110C 116F 11AF; # (줠; 줠; 줠; 줠; 줠; ) HANGUL SYLLABLE JWEOL +C921;C921;110C 116F 11B0;C921;110C 116F 11B0; # (줡; 줡; 줡; 줡; 줡; ) HANGUL SYLLABLE JWEOLG +C922;C922;110C 116F 11B1;C922;110C 116F 11B1; # (줢; 줢; 줢; 줢; 줢; ) HANGUL SYLLABLE JWEOLM +C923;C923;110C 116F 11B2;C923;110C 116F 11B2; # (줣; 줣; 줣; 줣; 줣; ) HANGUL SYLLABLE JWEOLB +C924;C924;110C 116F 11B3;C924;110C 116F 11B3; # (줤; 줤; 줤; 줤; 줤; ) HANGUL SYLLABLE JWEOLS +C925;C925;110C 116F 11B4;C925;110C 116F 11B4; # (줥; 줥; 줥; 줥; 줥; ) HANGUL SYLLABLE JWEOLT +C926;C926;110C 116F 11B5;C926;110C 116F 11B5; # (줦; 줦; 줦; 줦; 줦; ) HANGUL SYLLABLE JWEOLP +C927;C927;110C 116F 11B6;C927;110C 116F 11B6; # (줧; 줧; 줧; 줧; 줧; ) HANGUL SYLLABLE JWEOLH +C928;C928;110C 116F 11B7;C928;110C 116F 11B7; # (줨; 줨; 줨; 줨; 줨; ) HANGUL SYLLABLE JWEOM +C929;C929;110C 116F 11B8;C929;110C 116F 11B8; # (줩; 줩; 줩; 줩; 줩; ) HANGUL SYLLABLE JWEOB +C92A;C92A;110C 116F 11B9;C92A;110C 116F 11B9; # (줪; 줪; 줪; 줪; 줪; ) HANGUL SYLLABLE JWEOBS +C92B;C92B;110C 116F 11BA;C92B;110C 116F 11BA; # (줫; 줫; 줫; 줫; 줫; ) HANGUL SYLLABLE JWEOS +C92C;C92C;110C 116F 11BB;C92C;110C 116F 11BB; # (줬; 줬; 줬; 줬; 줬; ) HANGUL SYLLABLE JWEOSS +C92D;C92D;110C 116F 11BC;C92D;110C 116F 11BC; # (줭; 줭; 줭; 줭; 줭; ) HANGUL SYLLABLE JWEONG +C92E;C92E;110C 116F 11BD;C92E;110C 116F 11BD; # (줮; 줮; 줮; 줮; 줮; ) HANGUL SYLLABLE JWEOJ +C92F;C92F;110C 116F 11BE;C92F;110C 116F 11BE; # (줯; 줯; 줯; 줯; 줯; ) HANGUL SYLLABLE JWEOC +C930;C930;110C 116F 11BF;C930;110C 116F 11BF; # (줰; 줰; 줰; 줰; 줰; ) HANGUL SYLLABLE JWEOK +C931;C931;110C 116F 11C0;C931;110C 116F 11C0; # (줱; 줱; 줱; 줱; 줱; ) HANGUL SYLLABLE JWEOT +C932;C932;110C 116F 11C1;C932;110C 116F 11C1; # (줲; 줲; 줲; 줲; 줲; ) HANGUL SYLLABLE JWEOP +C933;C933;110C 116F 11C2;C933;110C 116F 11C2; # (줳; 줳; 줳; 줳; 줳; ) HANGUL SYLLABLE JWEOH +C934;C934;110C 1170;C934;110C 1170; # (줴; 줴; 줴; 줴; 줴; ) HANGUL SYLLABLE JWE +C935;C935;110C 1170 11A8;C935;110C 1170 11A8; # (줵; 줵; 줵; 줵; 줵; ) HANGUL SYLLABLE JWEG +C936;C936;110C 1170 11A9;C936;110C 1170 11A9; # (줶; 줶; 줶; 줶; 줶; ) HANGUL SYLLABLE JWEGG +C937;C937;110C 1170 11AA;C937;110C 1170 11AA; # (줷; 줷; 줷; 줷; 줷; ) HANGUL SYLLABLE JWEGS +C938;C938;110C 1170 11AB;C938;110C 1170 11AB; # (줸; 줸; 줸; 줸; 줸; ) HANGUL SYLLABLE JWEN +C939;C939;110C 1170 11AC;C939;110C 1170 11AC; # (줹; 줹; 줹; 줹; 줹; ) HANGUL SYLLABLE JWENJ +C93A;C93A;110C 1170 11AD;C93A;110C 1170 11AD; # (줺; 줺; 줺; 줺; 줺; ) HANGUL SYLLABLE JWENH +C93B;C93B;110C 1170 11AE;C93B;110C 1170 11AE; # (줻; 줻; 줻; 줻; 줻; ) HANGUL SYLLABLE JWED +C93C;C93C;110C 1170 11AF;C93C;110C 1170 11AF; # (줼; 줼; 줼; 줼; 줼; ) HANGUL SYLLABLE JWEL +C93D;C93D;110C 1170 11B0;C93D;110C 1170 11B0; # (줽; 줽; 줽; 줽; 줽; ) HANGUL SYLLABLE JWELG +C93E;C93E;110C 1170 11B1;C93E;110C 1170 11B1; # (줾; 줾; 줾; 줾; 줾; ) HANGUL SYLLABLE JWELM +C93F;C93F;110C 1170 11B2;C93F;110C 1170 11B2; # (줿; 줿; 줿; 줿; 줿; ) HANGUL SYLLABLE JWELB +C940;C940;110C 1170 11B3;C940;110C 1170 11B3; # (쥀; 쥀; 쥀; 쥀; 쥀; ) HANGUL SYLLABLE JWELS +C941;C941;110C 1170 11B4;C941;110C 1170 11B4; # (쥁; 쥁; 쥁; 쥁; 쥁; ) HANGUL SYLLABLE JWELT +C942;C942;110C 1170 11B5;C942;110C 1170 11B5; # (쥂; 쥂; 쥂; 쥂; 쥂; ) HANGUL SYLLABLE JWELP +C943;C943;110C 1170 11B6;C943;110C 1170 11B6; # (쥃; 쥃; 쥃; 쥃; 쥃; ) HANGUL SYLLABLE JWELH +C944;C944;110C 1170 11B7;C944;110C 1170 11B7; # (쥄; 쥄; 쥄; 쥄; 쥄; ) HANGUL SYLLABLE JWEM +C945;C945;110C 1170 11B8;C945;110C 1170 11B8; # (쥅; 쥅; 쥅; 쥅; 쥅; ) HANGUL SYLLABLE JWEB +C946;C946;110C 1170 11B9;C946;110C 1170 11B9; # (쥆; 쥆; 쥆; 쥆; 쥆; ) HANGUL SYLLABLE JWEBS +C947;C947;110C 1170 11BA;C947;110C 1170 11BA; # (쥇; 쥇; 쥇; 쥇; 쥇; ) HANGUL SYLLABLE JWES +C948;C948;110C 1170 11BB;C948;110C 1170 11BB; # (쥈; 쥈; 쥈; 쥈; 쥈; ) HANGUL SYLLABLE JWESS +C949;C949;110C 1170 11BC;C949;110C 1170 11BC; # (쥉; 쥉; 쥉; 쥉; 쥉; ) HANGUL SYLLABLE JWENG +C94A;C94A;110C 1170 11BD;C94A;110C 1170 11BD; # (쥊; 쥊; 쥊; 쥊; 쥊; ) HANGUL SYLLABLE JWEJ +C94B;C94B;110C 1170 11BE;C94B;110C 1170 11BE; # (쥋; 쥋; 쥋; 쥋; 쥋; ) HANGUL SYLLABLE JWEC +C94C;C94C;110C 1170 11BF;C94C;110C 1170 11BF; # (쥌; 쥌; 쥌; 쥌; 쥌; ) HANGUL SYLLABLE JWEK +C94D;C94D;110C 1170 11C0;C94D;110C 1170 11C0; # (쥍; 쥍; 쥍; 쥍; 쥍; ) HANGUL SYLLABLE JWET +C94E;C94E;110C 1170 11C1;C94E;110C 1170 11C1; # (쥎; 쥎; 쥎; 쥎; 쥎; ) HANGUL SYLLABLE JWEP +C94F;C94F;110C 1170 11C2;C94F;110C 1170 11C2; # (쥏; 쥏; 쥏; 쥏; 쥏; ) HANGUL SYLLABLE JWEH +C950;C950;110C 1171;C950;110C 1171; # (쥐; 쥐; 쥐; 쥐; 쥐; ) HANGUL SYLLABLE JWI +C951;C951;110C 1171 11A8;C951;110C 1171 11A8; # (쥑; 쥑; 쥑; 쥑; 쥑; ) HANGUL SYLLABLE JWIG +C952;C952;110C 1171 11A9;C952;110C 1171 11A9; # (쥒; 쥒; 쥒; 쥒; 쥒; ) HANGUL SYLLABLE JWIGG +C953;C953;110C 1171 11AA;C953;110C 1171 11AA; # (쥓; 쥓; 쥓; 쥓; 쥓; ) HANGUL SYLLABLE JWIGS +C954;C954;110C 1171 11AB;C954;110C 1171 11AB; # (쥔; 쥔; 쥔; 쥔; 쥔; ) HANGUL SYLLABLE JWIN +C955;C955;110C 1171 11AC;C955;110C 1171 11AC; # (쥕; 쥕; 쥕; 쥕; 쥕; ) HANGUL SYLLABLE JWINJ +C956;C956;110C 1171 11AD;C956;110C 1171 11AD; # (쥖; 쥖; 쥖; 쥖; 쥖; ) HANGUL SYLLABLE JWINH +C957;C957;110C 1171 11AE;C957;110C 1171 11AE; # (쥗; 쥗; 쥗; 쥗; 쥗; ) HANGUL SYLLABLE JWID +C958;C958;110C 1171 11AF;C958;110C 1171 11AF; # (쥘; 쥘; 쥘; 쥘; 쥘; ) HANGUL SYLLABLE JWIL +C959;C959;110C 1171 11B0;C959;110C 1171 11B0; # (쥙; 쥙; 쥙; 쥙; 쥙; ) HANGUL SYLLABLE JWILG +C95A;C95A;110C 1171 11B1;C95A;110C 1171 11B1; # (쥚; 쥚; 쥚; 쥚; 쥚; ) HANGUL SYLLABLE JWILM +C95B;C95B;110C 1171 11B2;C95B;110C 1171 11B2; # (쥛; 쥛; 쥛; 쥛; 쥛; ) HANGUL SYLLABLE JWILB +C95C;C95C;110C 1171 11B3;C95C;110C 1171 11B3; # (쥜; 쥜; 쥜; 쥜; 쥜; ) HANGUL SYLLABLE JWILS +C95D;C95D;110C 1171 11B4;C95D;110C 1171 11B4; # (쥝; 쥝; 쥝; 쥝; 쥝; ) HANGUL SYLLABLE JWILT +C95E;C95E;110C 1171 11B5;C95E;110C 1171 11B5; # (쥞; 쥞; 쥞; 쥞; 쥞; ) HANGUL SYLLABLE JWILP +C95F;C95F;110C 1171 11B6;C95F;110C 1171 11B6; # (쥟; 쥟; 쥟; 쥟; 쥟; ) HANGUL SYLLABLE JWILH +C960;C960;110C 1171 11B7;C960;110C 1171 11B7; # (쥠; 쥠; 쥠; 쥠; 쥠; ) HANGUL SYLLABLE JWIM +C961;C961;110C 1171 11B8;C961;110C 1171 11B8; # (쥡; 쥡; 쥡; 쥡; 쥡; ) HANGUL SYLLABLE JWIB +C962;C962;110C 1171 11B9;C962;110C 1171 11B9; # (쥢; 쥢; 쥢; 쥢; 쥢; ) HANGUL SYLLABLE JWIBS +C963;C963;110C 1171 11BA;C963;110C 1171 11BA; # (쥣; 쥣; 쥣; 쥣; 쥣; ) HANGUL SYLLABLE JWIS +C964;C964;110C 1171 11BB;C964;110C 1171 11BB; # (쥤; 쥤; 쥤; 쥤; 쥤; ) HANGUL SYLLABLE JWISS +C965;C965;110C 1171 11BC;C965;110C 1171 11BC; # (쥥; 쥥; 쥥; 쥥; 쥥; ) HANGUL SYLLABLE JWING +C966;C966;110C 1171 11BD;C966;110C 1171 11BD; # (쥦; 쥦; 쥦; 쥦; 쥦; ) HANGUL SYLLABLE JWIJ +C967;C967;110C 1171 11BE;C967;110C 1171 11BE; # (쥧; 쥧; 쥧; 쥧; 쥧; ) HANGUL SYLLABLE JWIC +C968;C968;110C 1171 11BF;C968;110C 1171 11BF; # (쥨; 쥨; 쥨; 쥨; 쥨; ) HANGUL SYLLABLE JWIK +C969;C969;110C 1171 11C0;C969;110C 1171 11C0; # (쥩; 쥩; 쥩; 쥩; 쥩; ) HANGUL SYLLABLE JWIT +C96A;C96A;110C 1171 11C1;C96A;110C 1171 11C1; # (쥪; 쥪; 쥪; 쥪; 쥪; ) HANGUL SYLLABLE JWIP +C96B;C96B;110C 1171 11C2;C96B;110C 1171 11C2; # (쥫; 쥫; 쥫; 쥫; 쥫; ) HANGUL SYLLABLE JWIH +C96C;C96C;110C 1172;C96C;110C 1172; # (쥬; 쥬; 쥬; 쥬; 쥬; ) HANGUL SYLLABLE JYU +C96D;C96D;110C 1172 11A8;C96D;110C 1172 11A8; # (쥭; 쥭; 쥭; 쥭; 쥭; ) HANGUL SYLLABLE JYUG +C96E;C96E;110C 1172 11A9;C96E;110C 1172 11A9; # (쥮; 쥮; 쥮; 쥮; 쥮; ) HANGUL SYLLABLE JYUGG +C96F;C96F;110C 1172 11AA;C96F;110C 1172 11AA; # (쥯; 쥯; 쥯; 쥯; 쥯; ) HANGUL SYLLABLE JYUGS +C970;C970;110C 1172 11AB;C970;110C 1172 11AB; # (쥰; 쥰; 쥰; 쥰; 쥰; ) HANGUL SYLLABLE JYUN +C971;C971;110C 1172 11AC;C971;110C 1172 11AC; # (쥱; 쥱; 쥱; 쥱; 쥱; ) HANGUL SYLLABLE JYUNJ +C972;C972;110C 1172 11AD;C972;110C 1172 11AD; # (쥲; 쥲; 쥲; 쥲; 쥲; ) HANGUL SYLLABLE JYUNH +C973;C973;110C 1172 11AE;C973;110C 1172 11AE; # (쥳; 쥳; 쥳; 쥳; 쥳; ) HANGUL SYLLABLE JYUD +C974;C974;110C 1172 11AF;C974;110C 1172 11AF; # (쥴; 쥴; 쥴; 쥴; 쥴; ) HANGUL SYLLABLE JYUL +C975;C975;110C 1172 11B0;C975;110C 1172 11B0; # (쥵; 쥵; 쥵; 쥵; 쥵; ) HANGUL SYLLABLE JYULG +C976;C976;110C 1172 11B1;C976;110C 1172 11B1; # (쥶; 쥶; 쥶; 쥶; 쥶; ) HANGUL SYLLABLE JYULM +C977;C977;110C 1172 11B2;C977;110C 1172 11B2; # (쥷; 쥷; 쥷; 쥷; 쥷; ) HANGUL SYLLABLE JYULB +C978;C978;110C 1172 11B3;C978;110C 1172 11B3; # (쥸; 쥸; 쥸; 쥸; 쥸; ) HANGUL SYLLABLE JYULS +C979;C979;110C 1172 11B4;C979;110C 1172 11B4; # (쥹; 쥹; 쥹; 쥹; 쥹; ) HANGUL SYLLABLE JYULT +C97A;C97A;110C 1172 11B5;C97A;110C 1172 11B5; # (쥺; 쥺; 쥺; 쥺; 쥺; ) HANGUL SYLLABLE JYULP +C97B;C97B;110C 1172 11B6;C97B;110C 1172 11B6; # (쥻; 쥻; 쥻; 쥻; 쥻; ) HANGUL SYLLABLE JYULH +C97C;C97C;110C 1172 11B7;C97C;110C 1172 11B7; # (쥼; 쥼; 쥼; 쥼; 쥼; ) HANGUL SYLLABLE JYUM +C97D;C97D;110C 1172 11B8;C97D;110C 1172 11B8; # (쥽; 쥽; 쥽; 쥽; 쥽; ) HANGUL SYLLABLE JYUB +C97E;C97E;110C 1172 11B9;C97E;110C 1172 11B9; # (쥾; 쥾; 쥾; 쥾; 쥾; ) HANGUL SYLLABLE JYUBS +C97F;C97F;110C 1172 11BA;C97F;110C 1172 11BA; # (쥿; 쥿; 쥿; 쥿; 쥿; ) HANGUL SYLLABLE JYUS +C980;C980;110C 1172 11BB;C980;110C 1172 11BB; # (즀; 즀; 즀; 즀; 즀; ) HANGUL SYLLABLE JYUSS +C981;C981;110C 1172 11BC;C981;110C 1172 11BC; # (즁; 즁; 즁; 즁; 즁; ) HANGUL SYLLABLE JYUNG +C982;C982;110C 1172 11BD;C982;110C 1172 11BD; # (즂; 즂; 즂; 즂; 즂; ) HANGUL SYLLABLE JYUJ +C983;C983;110C 1172 11BE;C983;110C 1172 11BE; # (즃; 즃; 즃; 즃; 즃; ) HANGUL SYLLABLE JYUC +C984;C984;110C 1172 11BF;C984;110C 1172 11BF; # (즄; 즄; 즄; 즄; 즄; ) HANGUL SYLLABLE JYUK +C985;C985;110C 1172 11C0;C985;110C 1172 11C0; # (즅; 즅; 즅; 즅; 즅; ) HANGUL SYLLABLE JYUT +C986;C986;110C 1172 11C1;C986;110C 1172 11C1; # (즆; 즆; 즆; 즆; 즆; ) HANGUL SYLLABLE JYUP +C987;C987;110C 1172 11C2;C987;110C 1172 11C2; # (즇; 즇; 즇; 즇; 즇; ) HANGUL SYLLABLE JYUH +C988;C988;110C 1173;C988;110C 1173; # (즈; 즈; 즈; 즈; 즈; ) HANGUL SYLLABLE JEU +C989;C989;110C 1173 11A8;C989;110C 1173 11A8; # (즉; 즉; 즉; 즉; 즉; ) HANGUL SYLLABLE JEUG +C98A;C98A;110C 1173 11A9;C98A;110C 1173 11A9; # (즊; 즊; 즊; 즊; 즊; ) HANGUL SYLLABLE JEUGG +C98B;C98B;110C 1173 11AA;C98B;110C 1173 11AA; # (즋; 즋; 즋; 즋; 즋; ) HANGUL SYLLABLE JEUGS +C98C;C98C;110C 1173 11AB;C98C;110C 1173 11AB; # (즌; 즌; 즌; 즌; 즌; ) HANGUL SYLLABLE JEUN +C98D;C98D;110C 1173 11AC;C98D;110C 1173 11AC; # (즍; 즍; 즍; 즍; 즍; ) HANGUL SYLLABLE JEUNJ +C98E;C98E;110C 1173 11AD;C98E;110C 1173 11AD; # (즎; 즎; 즎; 즎; 즎; ) HANGUL SYLLABLE JEUNH +C98F;C98F;110C 1173 11AE;C98F;110C 1173 11AE; # (즏; 즏; 즏; 즏; 즏; ) HANGUL SYLLABLE JEUD +C990;C990;110C 1173 11AF;C990;110C 1173 11AF; # (즐; 즐; 즐; 즐; 즐; ) HANGUL SYLLABLE JEUL +C991;C991;110C 1173 11B0;C991;110C 1173 11B0; # (즑; 즑; 즑; 즑; 즑; ) HANGUL SYLLABLE JEULG +C992;C992;110C 1173 11B1;C992;110C 1173 11B1; # (즒; 즒; 즒; 즒; 즒; ) HANGUL SYLLABLE JEULM +C993;C993;110C 1173 11B2;C993;110C 1173 11B2; # (즓; 즓; 즓; 즓; 즓; ) HANGUL SYLLABLE JEULB +C994;C994;110C 1173 11B3;C994;110C 1173 11B3; # (즔; 즔; 즔; 즔; 즔; ) HANGUL SYLLABLE JEULS +C995;C995;110C 1173 11B4;C995;110C 1173 11B4; # (즕; 즕; 즕; 즕; 즕; ) HANGUL SYLLABLE JEULT +C996;C996;110C 1173 11B5;C996;110C 1173 11B5; # (즖; 즖; 즖; 즖; 즖; ) HANGUL SYLLABLE JEULP +C997;C997;110C 1173 11B6;C997;110C 1173 11B6; # (즗; 즗; 즗; 즗; 즗; ) HANGUL SYLLABLE JEULH +C998;C998;110C 1173 11B7;C998;110C 1173 11B7; # (즘; 즘; 즘; 즘; 즘; ) HANGUL SYLLABLE JEUM +C999;C999;110C 1173 11B8;C999;110C 1173 11B8; # (즙; 즙; 즙; 즙; 즙; ) HANGUL SYLLABLE JEUB +C99A;C99A;110C 1173 11B9;C99A;110C 1173 11B9; # (즚; 즚; 즚; 즚; 즚; ) HANGUL SYLLABLE JEUBS +C99B;C99B;110C 1173 11BA;C99B;110C 1173 11BA; # (즛; 즛; 즛; 즛; 즛; ) HANGUL SYLLABLE JEUS +C99C;C99C;110C 1173 11BB;C99C;110C 1173 11BB; # (즜; 즜; 즜; 즜; 즜; ) HANGUL SYLLABLE JEUSS +C99D;C99D;110C 1173 11BC;C99D;110C 1173 11BC; # (증; 증; 증; 증; 증; ) HANGUL SYLLABLE JEUNG +C99E;C99E;110C 1173 11BD;C99E;110C 1173 11BD; # (즞; 즞; 즞; 즞; 즞; ) HANGUL SYLLABLE JEUJ +C99F;C99F;110C 1173 11BE;C99F;110C 1173 11BE; # (즟; 즟; 즟; 즟; 즟; ) HANGUL SYLLABLE JEUC +C9A0;C9A0;110C 1173 11BF;C9A0;110C 1173 11BF; # (즠; 즠; 즠; 즠; 즠; ) HANGUL SYLLABLE JEUK +C9A1;C9A1;110C 1173 11C0;C9A1;110C 1173 11C0; # (즡; 즡; 즡; 즡; 즡; ) HANGUL SYLLABLE JEUT +C9A2;C9A2;110C 1173 11C1;C9A2;110C 1173 11C1; # (즢; 즢; 즢; 즢; 즢; ) HANGUL SYLLABLE JEUP +C9A3;C9A3;110C 1173 11C2;C9A3;110C 1173 11C2; # (즣; 즣; 즣; 즣; 즣; ) HANGUL SYLLABLE JEUH +C9A4;C9A4;110C 1174;C9A4;110C 1174; # (즤; 즤; 즤; 즤; 즤; ) HANGUL SYLLABLE JYI +C9A5;C9A5;110C 1174 11A8;C9A5;110C 1174 11A8; # (즥; 즥; 즥; 즥; 즥; ) HANGUL SYLLABLE JYIG +C9A6;C9A6;110C 1174 11A9;C9A6;110C 1174 11A9; # (즦; 즦; 즦; 즦; 즦; ) HANGUL SYLLABLE JYIGG +C9A7;C9A7;110C 1174 11AA;C9A7;110C 1174 11AA; # (즧; 즧; 즧; 즧; 즧; ) HANGUL SYLLABLE JYIGS +C9A8;C9A8;110C 1174 11AB;C9A8;110C 1174 11AB; # (즨; 즨; 즨; 즨; 즨; ) HANGUL SYLLABLE JYIN +C9A9;C9A9;110C 1174 11AC;C9A9;110C 1174 11AC; # (즩; 즩; 즩; 즩; 즩; ) HANGUL SYLLABLE JYINJ +C9AA;C9AA;110C 1174 11AD;C9AA;110C 1174 11AD; # (즪; 즪; 즪; 즪; 즪; ) HANGUL SYLLABLE JYINH +C9AB;C9AB;110C 1174 11AE;C9AB;110C 1174 11AE; # (즫; 즫; 즫; 즫; 즫; ) HANGUL SYLLABLE JYID +C9AC;C9AC;110C 1174 11AF;C9AC;110C 1174 11AF; # (즬; 즬; 즬; 즬; 즬; ) HANGUL SYLLABLE JYIL +C9AD;C9AD;110C 1174 11B0;C9AD;110C 1174 11B0; # (즭; 즭; 즭; 즭; 즭; ) HANGUL SYLLABLE JYILG +C9AE;C9AE;110C 1174 11B1;C9AE;110C 1174 11B1; # (즮; 즮; 즮; 즮; 즮; ) HANGUL SYLLABLE JYILM +C9AF;C9AF;110C 1174 11B2;C9AF;110C 1174 11B2; # (즯; 즯; 즯; 즯; 즯; ) HANGUL SYLLABLE JYILB +C9B0;C9B0;110C 1174 11B3;C9B0;110C 1174 11B3; # (즰; 즰; 즰; 즰; 즰; ) HANGUL SYLLABLE JYILS +C9B1;C9B1;110C 1174 11B4;C9B1;110C 1174 11B4; # (즱; 즱; 즱; 즱; 즱; ) HANGUL SYLLABLE JYILT +C9B2;C9B2;110C 1174 11B5;C9B2;110C 1174 11B5; # (즲; 즲; 즲; 즲; 즲; ) HANGUL SYLLABLE JYILP +C9B3;C9B3;110C 1174 11B6;C9B3;110C 1174 11B6; # (즳; 즳; 즳; 즳; 즳; ) HANGUL SYLLABLE JYILH +C9B4;C9B4;110C 1174 11B7;C9B4;110C 1174 11B7; # (즴; 즴; 즴; 즴; 즴; ) HANGUL SYLLABLE JYIM +C9B5;C9B5;110C 1174 11B8;C9B5;110C 1174 11B8; # (즵; 즵; 즵; 즵; 즵; ) HANGUL SYLLABLE JYIB +C9B6;C9B6;110C 1174 11B9;C9B6;110C 1174 11B9; # (즶; 즶; 즶; 즶; 즶; ) HANGUL SYLLABLE JYIBS +C9B7;C9B7;110C 1174 11BA;C9B7;110C 1174 11BA; # (즷; 즷; 즷; 즷; 즷; ) HANGUL SYLLABLE JYIS +C9B8;C9B8;110C 1174 11BB;C9B8;110C 1174 11BB; # (즸; 즸; 즸; 즸; 즸; ) HANGUL SYLLABLE JYISS +C9B9;C9B9;110C 1174 11BC;C9B9;110C 1174 11BC; # (즹; 즹; 즹; 즹; 즹; ) HANGUL SYLLABLE JYING +C9BA;C9BA;110C 1174 11BD;C9BA;110C 1174 11BD; # (즺; 즺; 즺; 즺; 즺; ) HANGUL SYLLABLE JYIJ +C9BB;C9BB;110C 1174 11BE;C9BB;110C 1174 11BE; # (즻; 즻; 즻; 즻; 즻; ) HANGUL SYLLABLE JYIC +C9BC;C9BC;110C 1174 11BF;C9BC;110C 1174 11BF; # (즼; 즼; 즼; 즼; 즼; ) HANGUL SYLLABLE JYIK +C9BD;C9BD;110C 1174 11C0;C9BD;110C 1174 11C0; # (즽; 즽; 즽; 즽; 즽; ) HANGUL SYLLABLE JYIT +C9BE;C9BE;110C 1174 11C1;C9BE;110C 1174 11C1; # (즾; 즾; 즾; 즾; 즾; ) HANGUL SYLLABLE JYIP +C9BF;C9BF;110C 1174 11C2;C9BF;110C 1174 11C2; # (즿; 즿; 즿; 즿; 즿; ) HANGUL SYLLABLE JYIH +C9C0;C9C0;110C 1175;C9C0;110C 1175; # (지; 지; 지; 지; 지; ) HANGUL SYLLABLE JI +C9C1;C9C1;110C 1175 11A8;C9C1;110C 1175 11A8; # (직; 직; 직; 직; 직; ) HANGUL SYLLABLE JIG +C9C2;C9C2;110C 1175 11A9;C9C2;110C 1175 11A9; # (짂; 짂; 짂; 짂; 짂; ) HANGUL SYLLABLE JIGG +C9C3;C9C3;110C 1175 11AA;C9C3;110C 1175 11AA; # (짃; 짃; 짃; 짃; 짃; ) HANGUL SYLLABLE JIGS +C9C4;C9C4;110C 1175 11AB;C9C4;110C 1175 11AB; # (진; 진; 진; 진; 진; ) HANGUL SYLLABLE JIN +C9C5;C9C5;110C 1175 11AC;C9C5;110C 1175 11AC; # (짅; 짅; 짅; 짅; 짅; ) HANGUL SYLLABLE JINJ +C9C6;C9C6;110C 1175 11AD;C9C6;110C 1175 11AD; # (짆; 짆; 짆; 짆; 짆; ) HANGUL SYLLABLE JINH +C9C7;C9C7;110C 1175 11AE;C9C7;110C 1175 11AE; # (짇; 짇; 짇; 짇; 짇; ) HANGUL SYLLABLE JID +C9C8;C9C8;110C 1175 11AF;C9C8;110C 1175 11AF; # (질; 질; 질; 질; 질; ) HANGUL SYLLABLE JIL +C9C9;C9C9;110C 1175 11B0;C9C9;110C 1175 11B0; # (짉; 짉; 짉; 짉; 짉; ) HANGUL SYLLABLE JILG +C9CA;C9CA;110C 1175 11B1;C9CA;110C 1175 11B1; # (짊; 짊; 짊; 짊; 짊; ) HANGUL SYLLABLE JILM +C9CB;C9CB;110C 1175 11B2;C9CB;110C 1175 11B2; # (짋; 짋; 짋; 짋; 짋; ) HANGUL SYLLABLE JILB +C9CC;C9CC;110C 1175 11B3;C9CC;110C 1175 11B3; # (짌; 짌; 짌; 짌; 짌; ) HANGUL SYLLABLE JILS +C9CD;C9CD;110C 1175 11B4;C9CD;110C 1175 11B4; # (짍; 짍; 짍; 짍; 짍; ) HANGUL SYLLABLE JILT +C9CE;C9CE;110C 1175 11B5;C9CE;110C 1175 11B5; # (짎; 짎; 짎; 짎; 짎; ) HANGUL SYLLABLE JILP +C9CF;C9CF;110C 1175 11B6;C9CF;110C 1175 11B6; # (짏; 짏; 짏; 짏; 짏; ) HANGUL SYLLABLE JILH +C9D0;C9D0;110C 1175 11B7;C9D0;110C 1175 11B7; # (짐; 짐; 짐; 짐; 짐; ) HANGUL SYLLABLE JIM +C9D1;C9D1;110C 1175 11B8;C9D1;110C 1175 11B8; # (집; 집; 집; 집; 집; ) HANGUL SYLLABLE JIB +C9D2;C9D2;110C 1175 11B9;C9D2;110C 1175 11B9; # (짒; 짒; 짒; 짒; 짒; ) HANGUL SYLLABLE JIBS +C9D3;C9D3;110C 1175 11BA;C9D3;110C 1175 11BA; # (짓; 짓; 짓; 짓; 짓; ) HANGUL SYLLABLE JIS +C9D4;C9D4;110C 1175 11BB;C9D4;110C 1175 11BB; # (짔; 짔; 짔; 짔; 짔; ) HANGUL SYLLABLE JISS +C9D5;C9D5;110C 1175 11BC;C9D5;110C 1175 11BC; # (징; 징; 징; 징; 징; ) HANGUL SYLLABLE JING +C9D6;C9D6;110C 1175 11BD;C9D6;110C 1175 11BD; # (짖; 짖; 짖; 짖; 짖; ) HANGUL SYLLABLE JIJ +C9D7;C9D7;110C 1175 11BE;C9D7;110C 1175 11BE; # (짗; 짗; 짗; 짗; 짗; ) HANGUL SYLLABLE JIC +C9D8;C9D8;110C 1175 11BF;C9D8;110C 1175 11BF; # (짘; 짘; 짘; 짘; 짘; ) HANGUL SYLLABLE JIK +C9D9;C9D9;110C 1175 11C0;C9D9;110C 1175 11C0; # (짙; 짙; 짙; 짙; 짙; ) HANGUL SYLLABLE JIT +C9DA;C9DA;110C 1175 11C1;C9DA;110C 1175 11C1; # (짚; 짚; 짚; 짚; 짚; ) HANGUL SYLLABLE JIP +C9DB;C9DB;110C 1175 11C2;C9DB;110C 1175 11C2; # (짛; 짛; 짛; 짛; 짛; ) HANGUL SYLLABLE JIH +C9DC;C9DC;110D 1161;C9DC;110D 1161; # (짜; 짜; 짜; 짜; 짜; ) HANGUL SYLLABLE JJA +C9DD;C9DD;110D 1161 11A8;C9DD;110D 1161 11A8; # (짝; 짝; 짝; 짝; 짝; ) HANGUL SYLLABLE JJAG +C9DE;C9DE;110D 1161 11A9;C9DE;110D 1161 11A9; # (짞; 짞; 짞; 짞; 짞; ) HANGUL SYLLABLE JJAGG +C9DF;C9DF;110D 1161 11AA;C9DF;110D 1161 11AA; # (짟; 짟; 짟; 짟; 짟; ) HANGUL SYLLABLE JJAGS +C9E0;C9E0;110D 1161 11AB;C9E0;110D 1161 11AB; # (짠; 짠; 짠; 짠; 짠; ) HANGUL SYLLABLE JJAN +C9E1;C9E1;110D 1161 11AC;C9E1;110D 1161 11AC; # (짡; 짡; 짡; 짡; 짡; ) HANGUL SYLLABLE JJANJ +C9E2;C9E2;110D 1161 11AD;C9E2;110D 1161 11AD; # (짢; 짢; 짢; 짢; 짢; ) HANGUL SYLLABLE JJANH +C9E3;C9E3;110D 1161 11AE;C9E3;110D 1161 11AE; # (짣; 짣; 짣; 짣; 짣; ) HANGUL SYLLABLE JJAD +C9E4;C9E4;110D 1161 11AF;C9E4;110D 1161 11AF; # (짤; 짤; 짤; 짤; 짤; ) HANGUL SYLLABLE JJAL +C9E5;C9E5;110D 1161 11B0;C9E5;110D 1161 11B0; # (짥; 짥; 짥; 짥; 짥; ) HANGUL SYLLABLE JJALG +C9E6;C9E6;110D 1161 11B1;C9E6;110D 1161 11B1; # (짦; 짦; 짦; 짦; 짦; ) HANGUL SYLLABLE JJALM +C9E7;C9E7;110D 1161 11B2;C9E7;110D 1161 11B2; # (짧; 짧; 짧; 짧; 짧; ) HANGUL SYLLABLE JJALB +C9E8;C9E8;110D 1161 11B3;C9E8;110D 1161 11B3; # (짨; 짨; 짨; 짨; 짨; ) HANGUL SYLLABLE JJALS +C9E9;C9E9;110D 1161 11B4;C9E9;110D 1161 11B4; # (짩; 짩; 짩; 짩; 짩; ) HANGUL SYLLABLE JJALT +C9EA;C9EA;110D 1161 11B5;C9EA;110D 1161 11B5; # (짪; 짪; 짪; 짪; 짪; ) HANGUL SYLLABLE JJALP +C9EB;C9EB;110D 1161 11B6;C9EB;110D 1161 11B6; # (짫; 짫; 짫; 짫; 짫; ) HANGUL SYLLABLE JJALH +C9EC;C9EC;110D 1161 11B7;C9EC;110D 1161 11B7; # (짬; 짬; 짬; 짬; 짬; ) HANGUL SYLLABLE JJAM +C9ED;C9ED;110D 1161 11B8;C9ED;110D 1161 11B8; # (짭; 짭; 짭; 짭; 짭; ) HANGUL SYLLABLE JJAB +C9EE;C9EE;110D 1161 11B9;C9EE;110D 1161 11B9; # (짮; 짮; 짮; 짮; 짮; ) HANGUL SYLLABLE JJABS +C9EF;C9EF;110D 1161 11BA;C9EF;110D 1161 11BA; # (짯; 짯; 짯; 짯; 짯; ) HANGUL SYLLABLE JJAS +C9F0;C9F0;110D 1161 11BB;C9F0;110D 1161 11BB; # (짰; 짰; 짰; 짰; 짰; ) HANGUL SYLLABLE JJASS +C9F1;C9F1;110D 1161 11BC;C9F1;110D 1161 11BC; # (짱; 짱; 짱; 짱; 짱; ) HANGUL SYLLABLE JJANG +C9F2;C9F2;110D 1161 11BD;C9F2;110D 1161 11BD; # (짲; 짲; 짲; 짲; 짲; ) HANGUL SYLLABLE JJAJ +C9F3;C9F3;110D 1161 11BE;C9F3;110D 1161 11BE; # (짳; 짳; 짳; 짳; 짳; ) HANGUL SYLLABLE JJAC +C9F4;C9F4;110D 1161 11BF;C9F4;110D 1161 11BF; # (짴; 짴; 짴; 짴; 짴; ) HANGUL SYLLABLE JJAK +C9F5;C9F5;110D 1161 11C0;C9F5;110D 1161 11C0; # (짵; 짵; 짵; 짵; 짵; ) HANGUL SYLLABLE JJAT +C9F6;C9F6;110D 1161 11C1;C9F6;110D 1161 11C1; # (짶; 짶; 짶; 짶; 짶; ) HANGUL SYLLABLE JJAP +C9F7;C9F7;110D 1161 11C2;C9F7;110D 1161 11C2; # (짷; 짷; 짷; 짷; 짷; ) HANGUL SYLLABLE JJAH +C9F8;C9F8;110D 1162;C9F8;110D 1162; # (째; 째; 째; 째; 째; ) HANGUL SYLLABLE JJAE +C9F9;C9F9;110D 1162 11A8;C9F9;110D 1162 11A8; # (짹; 짹; 짹; 짹; 짹; ) HANGUL SYLLABLE JJAEG +C9FA;C9FA;110D 1162 11A9;C9FA;110D 1162 11A9; # (짺; 짺; 짺; 짺; 짺; ) HANGUL SYLLABLE JJAEGG +C9FB;C9FB;110D 1162 11AA;C9FB;110D 1162 11AA; # (짻; 짻; 짻; 짻; 짻; ) HANGUL SYLLABLE JJAEGS +C9FC;C9FC;110D 1162 11AB;C9FC;110D 1162 11AB; # (짼; 짼; 짼; 짼; 짼; ) HANGUL SYLLABLE JJAEN +C9FD;C9FD;110D 1162 11AC;C9FD;110D 1162 11AC; # (짽; 짽; 짽; 짽; 짽; ) HANGUL SYLLABLE JJAENJ +C9FE;C9FE;110D 1162 11AD;C9FE;110D 1162 11AD; # (짾; 짾; 짾; 짾; 짾; ) HANGUL SYLLABLE JJAENH +C9FF;C9FF;110D 1162 11AE;C9FF;110D 1162 11AE; # (짿; 짿; 짿; 짿; 짿; ) HANGUL SYLLABLE JJAED +CA00;CA00;110D 1162 11AF;CA00;110D 1162 11AF; # (쨀; 쨀; 쨀; 쨀; 쨀; ) HANGUL SYLLABLE JJAEL +CA01;CA01;110D 1162 11B0;CA01;110D 1162 11B0; # (쨁; 쨁; 쨁; 쨁; 쨁; ) HANGUL SYLLABLE JJAELG +CA02;CA02;110D 1162 11B1;CA02;110D 1162 11B1; # (쨂; 쨂; 쨂; 쨂; 쨂; ) HANGUL SYLLABLE JJAELM +CA03;CA03;110D 1162 11B2;CA03;110D 1162 11B2; # (쨃; 쨃; 쨃; 쨃; 쨃; ) HANGUL SYLLABLE JJAELB +CA04;CA04;110D 1162 11B3;CA04;110D 1162 11B3; # (쨄; 쨄; 쨄; 쨄; 쨄; ) HANGUL SYLLABLE JJAELS +CA05;CA05;110D 1162 11B4;CA05;110D 1162 11B4; # (쨅; 쨅; 쨅; 쨅; 쨅; ) HANGUL SYLLABLE JJAELT +CA06;CA06;110D 1162 11B5;CA06;110D 1162 11B5; # (쨆; 쨆; 쨆; 쨆; 쨆; ) HANGUL SYLLABLE JJAELP +CA07;CA07;110D 1162 11B6;CA07;110D 1162 11B6; # (쨇; 쨇; 쨇; 쨇; 쨇; ) HANGUL SYLLABLE JJAELH +CA08;CA08;110D 1162 11B7;CA08;110D 1162 11B7; # (쨈; 쨈; 쨈; 쨈; 쨈; ) HANGUL SYLLABLE JJAEM +CA09;CA09;110D 1162 11B8;CA09;110D 1162 11B8; # (쨉; 쨉; 쨉; 쨉; 쨉; ) HANGUL SYLLABLE JJAEB +CA0A;CA0A;110D 1162 11B9;CA0A;110D 1162 11B9; # (쨊; 쨊; 쨊; 쨊; 쨊; ) HANGUL SYLLABLE JJAEBS +CA0B;CA0B;110D 1162 11BA;CA0B;110D 1162 11BA; # (쨋; 쨋; 쨋; 쨋; 쨋; ) HANGUL SYLLABLE JJAES +CA0C;CA0C;110D 1162 11BB;CA0C;110D 1162 11BB; # (쨌; 쨌; 쨌; 쨌; 쨌; ) HANGUL SYLLABLE JJAESS +CA0D;CA0D;110D 1162 11BC;CA0D;110D 1162 11BC; # (쨍; 쨍; 쨍; 쨍; 쨍; ) HANGUL SYLLABLE JJAENG +CA0E;CA0E;110D 1162 11BD;CA0E;110D 1162 11BD; # (쨎; 쨎; 쨎; 쨎; 쨎; ) HANGUL SYLLABLE JJAEJ +CA0F;CA0F;110D 1162 11BE;CA0F;110D 1162 11BE; # (쨏; 쨏; 쨏; 쨏; 쨏; ) HANGUL SYLLABLE JJAEC +CA10;CA10;110D 1162 11BF;CA10;110D 1162 11BF; # (쨐; 쨐; 쨐; 쨐; 쨐; ) HANGUL SYLLABLE JJAEK +CA11;CA11;110D 1162 11C0;CA11;110D 1162 11C0; # (쨑; 쨑; 쨑; 쨑; 쨑; ) HANGUL SYLLABLE JJAET +CA12;CA12;110D 1162 11C1;CA12;110D 1162 11C1; # (쨒; 쨒; 쨒; 쨒; 쨒; ) HANGUL SYLLABLE JJAEP +CA13;CA13;110D 1162 11C2;CA13;110D 1162 11C2; # (쨓; 쨓; 쨓; 쨓; 쨓; ) HANGUL SYLLABLE JJAEH +CA14;CA14;110D 1163;CA14;110D 1163; # (쨔; 쨔; 쨔; 쨔; 쨔; ) HANGUL SYLLABLE JJYA +CA15;CA15;110D 1163 11A8;CA15;110D 1163 11A8; # (쨕; 쨕; 쨕; 쨕; 쨕; ) HANGUL SYLLABLE JJYAG +CA16;CA16;110D 1163 11A9;CA16;110D 1163 11A9; # (쨖; 쨖; 쨖; 쨖; 쨖; ) HANGUL SYLLABLE JJYAGG +CA17;CA17;110D 1163 11AA;CA17;110D 1163 11AA; # (쨗; 쨗; 쨗; 쨗; 쨗; ) HANGUL SYLLABLE JJYAGS +CA18;CA18;110D 1163 11AB;CA18;110D 1163 11AB; # (쨘; 쨘; 쨘; 쨘; 쨘; ) HANGUL SYLLABLE JJYAN +CA19;CA19;110D 1163 11AC;CA19;110D 1163 11AC; # (쨙; 쨙; 쨙; 쨙; 쨙; ) HANGUL SYLLABLE JJYANJ +CA1A;CA1A;110D 1163 11AD;CA1A;110D 1163 11AD; # (쨚; 쨚; 쨚; 쨚; 쨚; ) HANGUL SYLLABLE JJYANH +CA1B;CA1B;110D 1163 11AE;CA1B;110D 1163 11AE; # (쨛; 쨛; 쨛; 쨛; 쨛; ) HANGUL SYLLABLE JJYAD +CA1C;CA1C;110D 1163 11AF;CA1C;110D 1163 11AF; # (쨜; 쨜; 쨜; 쨜; 쨜; ) HANGUL SYLLABLE JJYAL +CA1D;CA1D;110D 1163 11B0;CA1D;110D 1163 11B0; # (쨝; 쨝; 쨝; 쨝; 쨝; ) HANGUL SYLLABLE JJYALG +CA1E;CA1E;110D 1163 11B1;CA1E;110D 1163 11B1; # (쨞; 쨞; 쨞; 쨞; 쨞; ) HANGUL SYLLABLE JJYALM +CA1F;CA1F;110D 1163 11B2;CA1F;110D 1163 11B2; # (쨟; 쨟; 쨟; 쨟; 쨟; ) HANGUL SYLLABLE JJYALB +CA20;CA20;110D 1163 11B3;CA20;110D 1163 11B3; # (쨠; 쨠; 쨠; 쨠; 쨠; ) HANGUL SYLLABLE JJYALS +CA21;CA21;110D 1163 11B4;CA21;110D 1163 11B4; # (쨡; 쨡; 쨡; 쨡; 쨡; ) HANGUL SYLLABLE JJYALT +CA22;CA22;110D 1163 11B5;CA22;110D 1163 11B5; # (쨢; 쨢; 쨢; 쨢; 쨢; ) HANGUL SYLLABLE JJYALP +CA23;CA23;110D 1163 11B6;CA23;110D 1163 11B6; # (쨣; 쨣; 쨣; 쨣; 쨣; ) HANGUL SYLLABLE JJYALH +CA24;CA24;110D 1163 11B7;CA24;110D 1163 11B7; # (쨤; 쨤; 쨤; 쨤; 쨤; ) HANGUL SYLLABLE JJYAM +CA25;CA25;110D 1163 11B8;CA25;110D 1163 11B8; # (쨥; 쨥; 쨥; 쨥; 쨥; ) HANGUL SYLLABLE JJYAB +CA26;CA26;110D 1163 11B9;CA26;110D 1163 11B9; # (쨦; 쨦; 쨦; 쨦; 쨦; ) HANGUL SYLLABLE JJYABS +CA27;CA27;110D 1163 11BA;CA27;110D 1163 11BA; # (쨧; 쨧; 쨧; 쨧; 쨧; ) HANGUL SYLLABLE JJYAS +CA28;CA28;110D 1163 11BB;CA28;110D 1163 11BB; # (쨨; 쨨; 쨨; 쨨; 쨨; ) HANGUL SYLLABLE JJYASS +CA29;CA29;110D 1163 11BC;CA29;110D 1163 11BC; # (쨩; 쨩; 쨩; 쨩; 쨩; ) HANGUL SYLLABLE JJYANG +CA2A;CA2A;110D 1163 11BD;CA2A;110D 1163 11BD; # (쨪; 쨪; 쨪; 쨪; 쨪; ) HANGUL SYLLABLE JJYAJ +CA2B;CA2B;110D 1163 11BE;CA2B;110D 1163 11BE; # (쨫; 쨫; 쨫; 쨫; 쨫; ) HANGUL SYLLABLE JJYAC +CA2C;CA2C;110D 1163 11BF;CA2C;110D 1163 11BF; # (쨬; 쨬; 쨬; 쨬; 쨬; ) HANGUL SYLLABLE JJYAK +CA2D;CA2D;110D 1163 11C0;CA2D;110D 1163 11C0; # (쨭; 쨭; 쨭; 쨭; 쨭; ) HANGUL SYLLABLE JJYAT +CA2E;CA2E;110D 1163 11C1;CA2E;110D 1163 11C1; # (쨮; 쨮; 쨮; 쨮; 쨮; ) HANGUL SYLLABLE JJYAP +CA2F;CA2F;110D 1163 11C2;CA2F;110D 1163 11C2; # (쨯; 쨯; 쨯; 쨯; 쨯; ) HANGUL SYLLABLE JJYAH +CA30;CA30;110D 1164;CA30;110D 1164; # (쨰; 쨰; 쨰; 쨰; 쨰; ) HANGUL SYLLABLE JJYAE +CA31;CA31;110D 1164 11A8;CA31;110D 1164 11A8; # (쨱; 쨱; 쨱; 쨱; 쨱; ) HANGUL SYLLABLE JJYAEG +CA32;CA32;110D 1164 11A9;CA32;110D 1164 11A9; # (쨲; 쨲; 쨲; 쨲; 쨲; ) HANGUL SYLLABLE JJYAEGG +CA33;CA33;110D 1164 11AA;CA33;110D 1164 11AA; # (쨳; 쨳; 쨳; 쨳; 쨳; ) HANGUL SYLLABLE JJYAEGS +CA34;CA34;110D 1164 11AB;CA34;110D 1164 11AB; # (쨴; 쨴; 쨴; 쨴; 쨴; ) HANGUL SYLLABLE JJYAEN +CA35;CA35;110D 1164 11AC;CA35;110D 1164 11AC; # (쨵; 쨵; 쨵; 쨵; 쨵; ) HANGUL SYLLABLE JJYAENJ +CA36;CA36;110D 1164 11AD;CA36;110D 1164 11AD; # (쨶; 쨶; 쨶; 쨶; 쨶; ) HANGUL SYLLABLE JJYAENH +CA37;CA37;110D 1164 11AE;CA37;110D 1164 11AE; # (쨷; 쨷; 쨷; 쨷; 쨷; ) HANGUL SYLLABLE JJYAED +CA38;CA38;110D 1164 11AF;CA38;110D 1164 11AF; # (쨸; 쨸; 쨸; 쨸; 쨸; ) HANGUL SYLLABLE JJYAEL +CA39;CA39;110D 1164 11B0;CA39;110D 1164 11B0; # (쨹; 쨹; 쨹; 쨹; 쨹; ) HANGUL SYLLABLE JJYAELG +CA3A;CA3A;110D 1164 11B1;CA3A;110D 1164 11B1; # (쨺; 쨺; 쨺; 쨺; 쨺; ) HANGUL SYLLABLE JJYAELM +CA3B;CA3B;110D 1164 11B2;CA3B;110D 1164 11B2; # (쨻; 쨻; 쨻; 쨻; 쨻; ) HANGUL SYLLABLE JJYAELB +CA3C;CA3C;110D 1164 11B3;CA3C;110D 1164 11B3; # (쨼; 쨼; 쨼; 쨼; 쨼; ) HANGUL SYLLABLE JJYAELS +CA3D;CA3D;110D 1164 11B4;CA3D;110D 1164 11B4; # (쨽; 쨽; 쨽; 쨽; 쨽; ) HANGUL SYLLABLE JJYAELT +CA3E;CA3E;110D 1164 11B5;CA3E;110D 1164 11B5; # (쨾; 쨾; 쨾; 쨾; 쨾; ) HANGUL SYLLABLE JJYAELP +CA3F;CA3F;110D 1164 11B6;CA3F;110D 1164 11B6; # (쨿; 쨿; 쨿; 쨿; 쨿; ) HANGUL SYLLABLE JJYAELH +CA40;CA40;110D 1164 11B7;CA40;110D 1164 11B7; # (쩀; 쩀; 쩀; 쩀; 쩀; ) HANGUL SYLLABLE JJYAEM +CA41;CA41;110D 1164 11B8;CA41;110D 1164 11B8; # (쩁; 쩁; 쩁; 쩁; 쩁; ) HANGUL SYLLABLE JJYAEB +CA42;CA42;110D 1164 11B9;CA42;110D 1164 11B9; # (쩂; 쩂; 쩂; 쩂; 쩂; ) HANGUL SYLLABLE JJYAEBS +CA43;CA43;110D 1164 11BA;CA43;110D 1164 11BA; # (쩃; 쩃; 쩃; 쩃; 쩃; ) HANGUL SYLLABLE JJYAES +CA44;CA44;110D 1164 11BB;CA44;110D 1164 11BB; # (쩄; 쩄; 쩄; 쩄; 쩄; ) HANGUL SYLLABLE JJYAESS +CA45;CA45;110D 1164 11BC;CA45;110D 1164 11BC; # (쩅; 쩅; 쩅; 쩅; 쩅; ) HANGUL SYLLABLE JJYAENG +CA46;CA46;110D 1164 11BD;CA46;110D 1164 11BD; # (쩆; 쩆; 쩆; 쩆; 쩆; ) HANGUL SYLLABLE JJYAEJ +CA47;CA47;110D 1164 11BE;CA47;110D 1164 11BE; # (쩇; 쩇; 쩇; 쩇; 쩇; ) HANGUL SYLLABLE JJYAEC +CA48;CA48;110D 1164 11BF;CA48;110D 1164 11BF; # (쩈; 쩈; 쩈; 쩈; 쩈; ) HANGUL SYLLABLE JJYAEK +CA49;CA49;110D 1164 11C0;CA49;110D 1164 11C0; # (쩉; 쩉; 쩉; 쩉; 쩉; ) HANGUL SYLLABLE JJYAET +CA4A;CA4A;110D 1164 11C1;CA4A;110D 1164 11C1; # (쩊; 쩊; 쩊; 쩊; 쩊; ) HANGUL SYLLABLE JJYAEP +CA4B;CA4B;110D 1164 11C2;CA4B;110D 1164 11C2; # (쩋; 쩋; 쩋; 쩋; 쩋; ) HANGUL SYLLABLE JJYAEH +CA4C;CA4C;110D 1165;CA4C;110D 1165; # (쩌; 쩌; 쩌; 쩌; 쩌; ) HANGUL SYLLABLE JJEO +CA4D;CA4D;110D 1165 11A8;CA4D;110D 1165 11A8; # (쩍; 쩍; 쩍; 쩍; 쩍; ) HANGUL SYLLABLE JJEOG +CA4E;CA4E;110D 1165 11A9;CA4E;110D 1165 11A9; # (쩎; 쩎; 쩎; 쩎; 쩎; ) HANGUL SYLLABLE JJEOGG +CA4F;CA4F;110D 1165 11AA;CA4F;110D 1165 11AA; # (쩏; 쩏; 쩏; 쩏; 쩏; ) HANGUL SYLLABLE JJEOGS +CA50;CA50;110D 1165 11AB;CA50;110D 1165 11AB; # (쩐; 쩐; 쩐; 쩐; 쩐; ) HANGUL SYLLABLE JJEON +CA51;CA51;110D 1165 11AC;CA51;110D 1165 11AC; # (쩑; 쩑; 쩑; 쩑; 쩑; ) HANGUL SYLLABLE JJEONJ +CA52;CA52;110D 1165 11AD;CA52;110D 1165 11AD; # (쩒; 쩒; 쩒; 쩒; 쩒; ) HANGUL SYLLABLE JJEONH +CA53;CA53;110D 1165 11AE;CA53;110D 1165 11AE; # (쩓; 쩓; 쩓; 쩓; 쩓; ) HANGUL SYLLABLE JJEOD +CA54;CA54;110D 1165 11AF;CA54;110D 1165 11AF; # (쩔; 쩔; 쩔; 쩔; 쩔; ) HANGUL SYLLABLE JJEOL +CA55;CA55;110D 1165 11B0;CA55;110D 1165 11B0; # (쩕; 쩕; 쩕; 쩕; 쩕; ) HANGUL SYLLABLE JJEOLG +CA56;CA56;110D 1165 11B1;CA56;110D 1165 11B1; # (쩖; 쩖; 쩖; 쩖; 쩖; ) HANGUL SYLLABLE JJEOLM +CA57;CA57;110D 1165 11B2;CA57;110D 1165 11B2; # (쩗; 쩗; 쩗; 쩗; 쩗; ) HANGUL SYLLABLE JJEOLB +CA58;CA58;110D 1165 11B3;CA58;110D 1165 11B3; # (쩘; 쩘; 쩘; 쩘; 쩘; ) HANGUL SYLLABLE JJEOLS +CA59;CA59;110D 1165 11B4;CA59;110D 1165 11B4; # (쩙; 쩙; 쩙; 쩙; 쩙; ) HANGUL SYLLABLE JJEOLT +CA5A;CA5A;110D 1165 11B5;CA5A;110D 1165 11B5; # (쩚; 쩚; 쩚; 쩚; 쩚; ) HANGUL SYLLABLE JJEOLP +CA5B;CA5B;110D 1165 11B6;CA5B;110D 1165 11B6; # (쩛; 쩛; 쩛; 쩛; 쩛; ) HANGUL SYLLABLE JJEOLH +CA5C;CA5C;110D 1165 11B7;CA5C;110D 1165 11B7; # (쩜; 쩜; 쩜; 쩜; 쩜; ) HANGUL SYLLABLE JJEOM +CA5D;CA5D;110D 1165 11B8;CA5D;110D 1165 11B8; # (쩝; 쩝; 쩝; 쩝; 쩝; ) HANGUL SYLLABLE JJEOB +CA5E;CA5E;110D 1165 11B9;CA5E;110D 1165 11B9; # (쩞; 쩞; 쩞; 쩞; 쩞; ) HANGUL SYLLABLE JJEOBS +CA5F;CA5F;110D 1165 11BA;CA5F;110D 1165 11BA; # (쩟; 쩟; 쩟; 쩟; 쩟; ) HANGUL SYLLABLE JJEOS +CA60;CA60;110D 1165 11BB;CA60;110D 1165 11BB; # (쩠; 쩠; 쩠; 쩠; 쩠; ) HANGUL SYLLABLE JJEOSS +CA61;CA61;110D 1165 11BC;CA61;110D 1165 11BC; # (쩡; 쩡; 쩡; 쩡; 쩡; ) HANGUL SYLLABLE JJEONG +CA62;CA62;110D 1165 11BD;CA62;110D 1165 11BD; # (쩢; 쩢; 쩢; 쩢; 쩢; ) HANGUL SYLLABLE JJEOJ +CA63;CA63;110D 1165 11BE;CA63;110D 1165 11BE; # (쩣; 쩣; 쩣; 쩣; 쩣; ) HANGUL SYLLABLE JJEOC +CA64;CA64;110D 1165 11BF;CA64;110D 1165 11BF; # (쩤; 쩤; 쩤; 쩤; 쩤; ) HANGUL SYLLABLE JJEOK +CA65;CA65;110D 1165 11C0;CA65;110D 1165 11C0; # (쩥; 쩥; 쩥; 쩥; 쩥; ) HANGUL SYLLABLE JJEOT +CA66;CA66;110D 1165 11C1;CA66;110D 1165 11C1; # (쩦; 쩦; 쩦; 쩦; 쩦; ) HANGUL SYLLABLE JJEOP +CA67;CA67;110D 1165 11C2;CA67;110D 1165 11C2; # (쩧; 쩧; 쩧; 쩧; 쩧; ) HANGUL SYLLABLE JJEOH +CA68;CA68;110D 1166;CA68;110D 1166; # (쩨; 쩨; 쩨; 쩨; 쩨; ) HANGUL SYLLABLE JJE +CA69;CA69;110D 1166 11A8;CA69;110D 1166 11A8; # (쩩; 쩩; 쩩; 쩩; 쩩; ) HANGUL SYLLABLE JJEG +CA6A;CA6A;110D 1166 11A9;CA6A;110D 1166 11A9; # (쩪; 쩪; 쩪; 쩪; 쩪; ) HANGUL SYLLABLE JJEGG +CA6B;CA6B;110D 1166 11AA;CA6B;110D 1166 11AA; # (쩫; 쩫; 쩫; 쩫; 쩫; ) HANGUL SYLLABLE JJEGS +CA6C;CA6C;110D 1166 11AB;CA6C;110D 1166 11AB; # (쩬; 쩬; 쩬; 쩬; 쩬; ) HANGUL SYLLABLE JJEN +CA6D;CA6D;110D 1166 11AC;CA6D;110D 1166 11AC; # (쩭; 쩭; 쩭; 쩭; 쩭; ) HANGUL SYLLABLE JJENJ +CA6E;CA6E;110D 1166 11AD;CA6E;110D 1166 11AD; # (쩮; 쩮; 쩮; 쩮; 쩮; ) HANGUL SYLLABLE JJENH +CA6F;CA6F;110D 1166 11AE;CA6F;110D 1166 11AE; # (쩯; 쩯; 쩯; 쩯; 쩯; ) HANGUL SYLLABLE JJED +CA70;CA70;110D 1166 11AF;CA70;110D 1166 11AF; # (쩰; 쩰; 쩰; 쩰; 쩰; ) HANGUL SYLLABLE JJEL +CA71;CA71;110D 1166 11B0;CA71;110D 1166 11B0; # (쩱; 쩱; 쩱; 쩱; 쩱; ) HANGUL SYLLABLE JJELG +CA72;CA72;110D 1166 11B1;CA72;110D 1166 11B1; # (쩲; 쩲; 쩲; 쩲; 쩲; ) HANGUL SYLLABLE JJELM +CA73;CA73;110D 1166 11B2;CA73;110D 1166 11B2; # (쩳; 쩳; 쩳; 쩳; 쩳; ) HANGUL SYLLABLE JJELB +CA74;CA74;110D 1166 11B3;CA74;110D 1166 11B3; # (쩴; 쩴; 쩴; 쩴; 쩴; ) HANGUL SYLLABLE JJELS +CA75;CA75;110D 1166 11B4;CA75;110D 1166 11B4; # (쩵; 쩵; 쩵; 쩵; 쩵; ) HANGUL SYLLABLE JJELT +CA76;CA76;110D 1166 11B5;CA76;110D 1166 11B5; # (쩶; 쩶; 쩶; 쩶; 쩶; ) HANGUL SYLLABLE JJELP +CA77;CA77;110D 1166 11B6;CA77;110D 1166 11B6; # (쩷; 쩷; 쩷; 쩷; 쩷; ) HANGUL SYLLABLE JJELH +CA78;CA78;110D 1166 11B7;CA78;110D 1166 11B7; # (쩸; 쩸; 쩸; 쩸; 쩸; ) HANGUL SYLLABLE JJEM +CA79;CA79;110D 1166 11B8;CA79;110D 1166 11B8; # (쩹; 쩹; 쩹; 쩹; 쩹; ) HANGUL SYLLABLE JJEB +CA7A;CA7A;110D 1166 11B9;CA7A;110D 1166 11B9; # (쩺; 쩺; 쩺; 쩺; 쩺; ) HANGUL SYLLABLE JJEBS +CA7B;CA7B;110D 1166 11BA;CA7B;110D 1166 11BA; # (쩻; 쩻; 쩻; 쩻; 쩻; ) HANGUL SYLLABLE JJES +CA7C;CA7C;110D 1166 11BB;CA7C;110D 1166 11BB; # (쩼; 쩼; 쩼; 쩼; 쩼; ) HANGUL SYLLABLE JJESS +CA7D;CA7D;110D 1166 11BC;CA7D;110D 1166 11BC; # (쩽; 쩽; 쩽; 쩽; 쩽; ) HANGUL SYLLABLE JJENG +CA7E;CA7E;110D 1166 11BD;CA7E;110D 1166 11BD; # (쩾; 쩾; 쩾; 쩾; 쩾; ) HANGUL SYLLABLE JJEJ +CA7F;CA7F;110D 1166 11BE;CA7F;110D 1166 11BE; # (쩿; 쩿; 쩿; 쩿; 쩿; ) HANGUL SYLLABLE JJEC +CA80;CA80;110D 1166 11BF;CA80;110D 1166 11BF; # (쪀; 쪀; 쪀; 쪀; 쪀; ) HANGUL SYLLABLE JJEK +CA81;CA81;110D 1166 11C0;CA81;110D 1166 11C0; # (쪁; 쪁; 쪁; 쪁; 쪁; ) HANGUL SYLLABLE JJET +CA82;CA82;110D 1166 11C1;CA82;110D 1166 11C1; # (쪂; 쪂; 쪂; 쪂; 쪂; ) HANGUL SYLLABLE JJEP +CA83;CA83;110D 1166 11C2;CA83;110D 1166 11C2; # (쪃; 쪃; 쪃; 쪃; 쪃; ) HANGUL SYLLABLE JJEH +CA84;CA84;110D 1167;CA84;110D 1167; # (쪄; 쪄; 쪄; 쪄; 쪄; ) HANGUL SYLLABLE JJYEO +CA85;CA85;110D 1167 11A8;CA85;110D 1167 11A8; # (쪅; 쪅; 쪅; 쪅; 쪅; ) HANGUL SYLLABLE JJYEOG +CA86;CA86;110D 1167 11A9;CA86;110D 1167 11A9; # (쪆; 쪆; 쪆; 쪆; 쪆; ) HANGUL SYLLABLE JJYEOGG +CA87;CA87;110D 1167 11AA;CA87;110D 1167 11AA; # (쪇; 쪇; 쪇; 쪇; 쪇; ) HANGUL SYLLABLE JJYEOGS +CA88;CA88;110D 1167 11AB;CA88;110D 1167 11AB; # (쪈; 쪈; 쪈; 쪈; 쪈; ) HANGUL SYLLABLE JJYEON +CA89;CA89;110D 1167 11AC;CA89;110D 1167 11AC; # (쪉; 쪉; 쪉; 쪉; 쪉; ) HANGUL SYLLABLE JJYEONJ +CA8A;CA8A;110D 1167 11AD;CA8A;110D 1167 11AD; # (쪊; 쪊; 쪊; 쪊; 쪊; ) HANGUL SYLLABLE JJYEONH +CA8B;CA8B;110D 1167 11AE;CA8B;110D 1167 11AE; # (쪋; 쪋; 쪋; 쪋; 쪋; ) HANGUL SYLLABLE JJYEOD +CA8C;CA8C;110D 1167 11AF;CA8C;110D 1167 11AF; # (쪌; 쪌; 쪌; 쪌; 쪌; ) HANGUL SYLLABLE JJYEOL +CA8D;CA8D;110D 1167 11B0;CA8D;110D 1167 11B0; # (쪍; 쪍; 쪍; 쪍; 쪍; ) HANGUL SYLLABLE JJYEOLG +CA8E;CA8E;110D 1167 11B1;CA8E;110D 1167 11B1; # (쪎; 쪎; 쪎; 쪎; 쪎; ) HANGUL SYLLABLE JJYEOLM +CA8F;CA8F;110D 1167 11B2;CA8F;110D 1167 11B2; # (쪏; 쪏; 쪏; 쪏; 쪏; ) HANGUL SYLLABLE JJYEOLB +CA90;CA90;110D 1167 11B3;CA90;110D 1167 11B3; # (쪐; 쪐; 쪐; 쪐; 쪐; ) HANGUL SYLLABLE JJYEOLS +CA91;CA91;110D 1167 11B4;CA91;110D 1167 11B4; # (쪑; 쪑; 쪑; 쪑; 쪑; ) HANGUL SYLLABLE JJYEOLT +CA92;CA92;110D 1167 11B5;CA92;110D 1167 11B5; # (쪒; 쪒; 쪒; 쪒; 쪒; ) HANGUL SYLLABLE JJYEOLP +CA93;CA93;110D 1167 11B6;CA93;110D 1167 11B6; # (쪓; 쪓; 쪓; 쪓; 쪓; ) HANGUL SYLLABLE JJYEOLH +CA94;CA94;110D 1167 11B7;CA94;110D 1167 11B7; # (쪔; 쪔; 쪔; 쪔; 쪔; ) HANGUL SYLLABLE JJYEOM +CA95;CA95;110D 1167 11B8;CA95;110D 1167 11B8; # (쪕; 쪕; 쪕; 쪕; 쪕; ) HANGUL SYLLABLE JJYEOB +CA96;CA96;110D 1167 11B9;CA96;110D 1167 11B9; # (쪖; 쪖; 쪖; 쪖; 쪖; ) HANGUL SYLLABLE JJYEOBS +CA97;CA97;110D 1167 11BA;CA97;110D 1167 11BA; # (쪗; 쪗; 쪗; 쪗; 쪗; ) HANGUL SYLLABLE JJYEOS +CA98;CA98;110D 1167 11BB;CA98;110D 1167 11BB; # (쪘; 쪘; 쪘; 쪘; 쪘; ) HANGUL SYLLABLE JJYEOSS +CA99;CA99;110D 1167 11BC;CA99;110D 1167 11BC; # (쪙; 쪙; 쪙; 쪙; 쪙; ) HANGUL SYLLABLE JJYEONG +CA9A;CA9A;110D 1167 11BD;CA9A;110D 1167 11BD; # (쪚; 쪚; 쪚; 쪚; 쪚; ) HANGUL SYLLABLE JJYEOJ +CA9B;CA9B;110D 1167 11BE;CA9B;110D 1167 11BE; # (쪛; 쪛; 쪛; 쪛; 쪛; ) HANGUL SYLLABLE JJYEOC +CA9C;CA9C;110D 1167 11BF;CA9C;110D 1167 11BF; # (쪜; 쪜; 쪜; 쪜; 쪜; ) HANGUL SYLLABLE JJYEOK +CA9D;CA9D;110D 1167 11C0;CA9D;110D 1167 11C0; # (쪝; 쪝; 쪝; 쪝; 쪝; ) HANGUL SYLLABLE JJYEOT +CA9E;CA9E;110D 1167 11C1;CA9E;110D 1167 11C1; # (쪞; 쪞; 쪞; 쪞; 쪞; ) HANGUL SYLLABLE JJYEOP +CA9F;CA9F;110D 1167 11C2;CA9F;110D 1167 11C2; # (쪟; 쪟; 쪟; 쪟; 쪟; ) HANGUL SYLLABLE JJYEOH +CAA0;CAA0;110D 1168;CAA0;110D 1168; # (쪠; 쪠; 쪠; 쪠; 쪠; ) HANGUL SYLLABLE JJYE +CAA1;CAA1;110D 1168 11A8;CAA1;110D 1168 11A8; # (쪡; 쪡; 쪡; 쪡; 쪡; ) HANGUL SYLLABLE JJYEG +CAA2;CAA2;110D 1168 11A9;CAA2;110D 1168 11A9; # (쪢; 쪢; 쪢; 쪢; 쪢; ) HANGUL SYLLABLE JJYEGG +CAA3;CAA3;110D 1168 11AA;CAA3;110D 1168 11AA; # (쪣; 쪣; 쪣; 쪣; 쪣; ) HANGUL SYLLABLE JJYEGS +CAA4;CAA4;110D 1168 11AB;CAA4;110D 1168 11AB; # (쪤; 쪤; 쪤; 쪤; 쪤; ) HANGUL SYLLABLE JJYEN +CAA5;CAA5;110D 1168 11AC;CAA5;110D 1168 11AC; # (쪥; 쪥; 쪥; 쪥; 쪥; ) HANGUL SYLLABLE JJYENJ +CAA6;CAA6;110D 1168 11AD;CAA6;110D 1168 11AD; # (쪦; 쪦; 쪦; 쪦; 쪦; ) HANGUL SYLLABLE JJYENH +CAA7;CAA7;110D 1168 11AE;CAA7;110D 1168 11AE; # (쪧; 쪧; 쪧; 쪧; 쪧; ) HANGUL SYLLABLE JJYED +CAA8;CAA8;110D 1168 11AF;CAA8;110D 1168 11AF; # (쪨; 쪨; 쪨; 쪨; 쪨; ) HANGUL SYLLABLE JJYEL +CAA9;CAA9;110D 1168 11B0;CAA9;110D 1168 11B0; # (쪩; 쪩; 쪩; 쪩; 쪩; ) HANGUL SYLLABLE JJYELG +CAAA;CAAA;110D 1168 11B1;CAAA;110D 1168 11B1; # (쪪; 쪪; 쪪; 쪪; 쪪; ) HANGUL SYLLABLE JJYELM +CAAB;CAAB;110D 1168 11B2;CAAB;110D 1168 11B2; # (쪫; 쪫; 쪫; 쪫; 쪫; ) HANGUL SYLLABLE JJYELB +CAAC;CAAC;110D 1168 11B3;CAAC;110D 1168 11B3; # (쪬; 쪬; 쪬; 쪬; 쪬; ) HANGUL SYLLABLE JJYELS +CAAD;CAAD;110D 1168 11B4;CAAD;110D 1168 11B4; # (쪭; 쪭; 쪭; 쪭; 쪭; ) HANGUL SYLLABLE JJYELT +CAAE;CAAE;110D 1168 11B5;CAAE;110D 1168 11B5; # (쪮; 쪮; 쪮; 쪮; 쪮; ) HANGUL SYLLABLE JJYELP +CAAF;CAAF;110D 1168 11B6;CAAF;110D 1168 11B6; # (쪯; 쪯; 쪯; 쪯; 쪯; ) HANGUL SYLLABLE JJYELH +CAB0;CAB0;110D 1168 11B7;CAB0;110D 1168 11B7; # (쪰; 쪰; 쪰; 쪰; 쪰; ) HANGUL SYLLABLE JJYEM +CAB1;CAB1;110D 1168 11B8;CAB1;110D 1168 11B8; # (쪱; 쪱; 쪱; 쪱; 쪱; ) HANGUL SYLLABLE JJYEB +CAB2;CAB2;110D 1168 11B9;CAB2;110D 1168 11B9; # (쪲; 쪲; 쪲; 쪲; 쪲; ) HANGUL SYLLABLE JJYEBS +CAB3;CAB3;110D 1168 11BA;CAB3;110D 1168 11BA; # (쪳; 쪳; 쪳; 쪳; 쪳; ) HANGUL SYLLABLE JJYES +CAB4;CAB4;110D 1168 11BB;CAB4;110D 1168 11BB; # (쪴; 쪴; 쪴; 쪴; 쪴; ) HANGUL SYLLABLE JJYESS +CAB5;CAB5;110D 1168 11BC;CAB5;110D 1168 11BC; # (쪵; 쪵; 쪵; 쪵; 쪵; ) HANGUL SYLLABLE JJYENG +CAB6;CAB6;110D 1168 11BD;CAB6;110D 1168 11BD; # (쪶; 쪶; 쪶; 쪶; 쪶; ) HANGUL SYLLABLE JJYEJ +CAB7;CAB7;110D 1168 11BE;CAB7;110D 1168 11BE; # (쪷; 쪷; 쪷; 쪷; 쪷; ) HANGUL SYLLABLE JJYEC +CAB8;CAB8;110D 1168 11BF;CAB8;110D 1168 11BF; # (쪸; 쪸; 쪸; 쪸; 쪸; ) HANGUL SYLLABLE JJYEK +CAB9;CAB9;110D 1168 11C0;CAB9;110D 1168 11C0; # (쪹; 쪹; 쪹; 쪹; 쪹; ) HANGUL SYLLABLE JJYET +CABA;CABA;110D 1168 11C1;CABA;110D 1168 11C1; # (쪺; 쪺; 쪺; 쪺; 쪺; ) HANGUL SYLLABLE JJYEP +CABB;CABB;110D 1168 11C2;CABB;110D 1168 11C2; # (쪻; 쪻; 쪻; 쪻; 쪻; ) HANGUL SYLLABLE JJYEH +CABC;CABC;110D 1169;CABC;110D 1169; # (쪼; 쪼; 쪼; 쪼; 쪼; ) HANGUL SYLLABLE JJO +CABD;CABD;110D 1169 11A8;CABD;110D 1169 11A8; # (쪽; 쪽; 쪽; 쪽; 쪽; ) HANGUL SYLLABLE JJOG +CABE;CABE;110D 1169 11A9;CABE;110D 1169 11A9; # (쪾; 쪾; 쪾; 쪾; 쪾; ) HANGUL SYLLABLE JJOGG +CABF;CABF;110D 1169 11AA;CABF;110D 1169 11AA; # (쪿; 쪿; 쪿; 쪿; 쪿; ) HANGUL SYLLABLE JJOGS +CAC0;CAC0;110D 1169 11AB;CAC0;110D 1169 11AB; # (쫀; 쫀; 쫀; 쫀; 쫀; ) HANGUL SYLLABLE JJON +CAC1;CAC1;110D 1169 11AC;CAC1;110D 1169 11AC; # (쫁; 쫁; 쫁; 쫁; 쫁; ) HANGUL SYLLABLE JJONJ +CAC2;CAC2;110D 1169 11AD;CAC2;110D 1169 11AD; # (쫂; 쫂; 쫂; 쫂; 쫂; ) HANGUL SYLLABLE JJONH +CAC3;CAC3;110D 1169 11AE;CAC3;110D 1169 11AE; # (쫃; 쫃; 쫃; 쫃; 쫃; ) HANGUL SYLLABLE JJOD +CAC4;CAC4;110D 1169 11AF;CAC4;110D 1169 11AF; # (쫄; 쫄; 쫄; 쫄; 쫄; ) HANGUL SYLLABLE JJOL +CAC5;CAC5;110D 1169 11B0;CAC5;110D 1169 11B0; # (쫅; 쫅; 쫅; 쫅; 쫅; ) HANGUL SYLLABLE JJOLG +CAC6;CAC6;110D 1169 11B1;CAC6;110D 1169 11B1; # (쫆; 쫆; 쫆; 쫆; 쫆; ) HANGUL SYLLABLE JJOLM +CAC7;CAC7;110D 1169 11B2;CAC7;110D 1169 11B2; # (쫇; 쫇; 쫇; 쫇; 쫇; ) HANGUL SYLLABLE JJOLB +CAC8;CAC8;110D 1169 11B3;CAC8;110D 1169 11B3; # (쫈; 쫈; 쫈; 쫈; 쫈; ) HANGUL SYLLABLE JJOLS +CAC9;CAC9;110D 1169 11B4;CAC9;110D 1169 11B4; # (쫉; 쫉; 쫉; 쫉; 쫉; ) HANGUL SYLLABLE JJOLT +CACA;CACA;110D 1169 11B5;CACA;110D 1169 11B5; # (쫊; 쫊; 쫊; 쫊; 쫊; ) HANGUL SYLLABLE JJOLP +CACB;CACB;110D 1169 11B6;CACB;110D 1169 11B6; # (쫋; 쫋; 쫋; 쫋; 쫋; ) HANGUL SYLLABLE JJOLH +CACC;CACC;110D 1169 11B7;CACC;110D 1169 11B7; # (쫌; 쫌; 쫌; 쫌; 쫌; ) HANGUL SYLLABLE JJOM +CACD;CACD;110D 1169 11B8;CACD;110D 1169 11B8; # (쫍; 쫍; 쫍; 쫍; 쫍; ) HANGUL SYLLABLE JJOB +CACE;CACE;110D 1169 11B9;CACE;110D 1169 11B9; # (쫎; 쫎; 쫎; 쫎; 쫎; ) HANGUL SYLLABLE JJOBS +CACF;CACF;110D 1169 11BA;CACF;110D 1169 11BA; # (쫏; 쫏; 쫏; 쫏; 쫏; ) HANGUL SYLLABLE JJOS +CAD0;CAD0;110D 1169 11BB;CAD0;110D 1169 11BB; # (쫐; 쫐; 쫐; 쫐; 쫐; ) HANGUL SYLLABLE JJOSS +CAD1;CAD1;110D 1169 11BC;CAD1;110D 1169 11BC; # (쫑; 쫑; 쫑; 쫑; 쫑; ) HANGUL SYLLABLE JJONG +CAD2;CAD2;110D 1169 11BD;CAD2;110D 1169 11BD; # (쫒; 쫒; 쫒; 쫒; 쫒; ) HANGUL SYLLABLE JJOJ +CAD3;CAD3;110D 1169 11BE;CAD3;110D 1169 11BE; # (쫓; 쫓; 쫓; 쫓; 쫓; ) HANGUL SYLLABLE JJOC +CAD4;CAD4;110D 1169 11BF;CAD4;110D 1169 11BF; # (쫔; 쫔; 쫔; 쫔; 쫔; ) HANGUL SYLLABLE JJOK +CAD5;CAD5;110D 1169 11C0;CAD5;110D 1169 11C0; # (쫕; 쫕; 쫕; 쫕; 쫕; ) HANGUL SYLLABLE JJOT +CAD6;CAD6;110D 1169 11C1;CAD6;110D 1169 11C1; # (쫖; 쫖; 쫖; 쫖; 쫖; ) HANGUL SYLLABLE JJOP +CAD7;CAD7;110D 1169 11C2;CAD7;110D 1169 11C2; # (쫗; 쫗; 쫗; 쫗; 쫗; ) HANGUL SYLLABLE JJOH +CAD8;CAD8;110D 116A;CAD8;110D 116A; # (쫘; 쫘; 쫘; 쫘; 쫘; ) HANGUL SYLLABLE JJWA +CAD9;CAD9;110D 116A 11A8;CAD9;110D 116A 11A8; # (쫙; 쫙; 쫙; 쫙; 쫙; ) HANGUL SYLLABLE JJWAG +CADA;CADA;110D 116A 11A9;CADA;110D 116A 11A9; # (쫚; 쫚; 쫚; 쫚; 쫚; ) HANGUL SYLLABLE JJWAGG +CADB;CADB;110D 116A 11AA;CADB;110D 116A 11AA; # (쫛; 쫛; 쫛; 쫛; 쫛; ) HANGUL SYLLABLE JJWAGS +CADC;CADC;110D 116A 11AB;CADC;110D 116A 11AB; # (쫜; 쫜; 쫜; 쫜; 쫜; ) HANGUL SYLLABLE JJWAN +CADD;CADD;110D 116A 11AC;CADD;110D 116A 11AC; # (쫝; 쫝; 쫝; 쫝; 쫝; ) HANGUL SYLLABLE JJWANJ +CADE;CADE;110D 116A 11AD;CADE;110D 116A 11AD; # (쫞; 쫞; 쫞; 쫞; 쫞; ) HANGUL SYLLABLE JJWANH +CADF;CADF;110D 116A 11AE;CADF;110D 116A 11AE; # (쫟; 쫟; 쫟; 쫟; 쫟; ) HANGUL SYLLABLE JJWAD +CAE0;CAE0;110D 116A 11AF;CAE0;110D 116A 11AF; # (쫠; 쫠; 쫠; 쫠; 쫠; ) HANGUL SYLLABLE JJWAL +CAE1;CAE1;110D 116A 11B0;CAE1;110D 116A 11B0; # (쫡; 쫡; 쫡; 쫡; 쫡; ) HANGUL SYLLABLE JJWALG +CAE2;CAE2;110D 116A 11B1;CAE2;110D 116A 11B1; # (쫢; 쫢; 쫢; 쫢; 쫢; ) HANGUL SYLLABLE JJWALM +CAE3;CAE3;110D 116A 11B2;CAE3;110D 116A 11B2; # (쫣; 쫣; 쫣; 쫣; 쫣; ) HANGUL SYLLABLE JJWALB +CAE4;CAE4;110D 116A 11B3;CAE4;110D 116A 11B3; # (쫤; 쫤; 쫤; 쫤; 쫤; ) HANGUL SYLLABLE JJWALS +CAE5;CAE5;110D 116A 11B4;CAE5;110D 116A 11B4; # (쫥; 쫥; 쫥; 쫥; 쫥; ) HANGUL SYLLABLE JJWALT +CAE6;CAE6;110D 116A 11B5;CAE6;110D 116A 11B5; # (쫦; 쫦; 쫦; 쫦; 쫦; ) HANGUL SYLLABLE JJWALP +CAE7;CAE7;110D 116A 11B6;CAE7;110D 116A 11B6; # (쫧; 쫧; 쫧; 쫧; 쫧; ) HANGUL SYLLABLE JJWALH +CAE8;CAE8;110D 116A 11B7;CAE8;110D 116A 11B7; # (쫨; 쫨; 쫨; 쫨; 쫨; ) HANGUL SYLLABLE JJWAM +CAE9;CAE9;110D 116A 11B8;CAE9;110D 116A 11B8; # (쫩; 쫩; 쫩; 쫩; 쫩; ) HANGUL SYLLABLE JJWAB +CAEA;CAEA;110D 116A 11B9;CAEA;110D 116A 11B9; # (쫪; 쫪; 쫪; 쫪; 쫪; ) HANGUL SYLLABLE JJWABS +CAEB;CAEB;110D 116A 11BA;CAEB;110D 116A 11BA; # (쫫; 쫫; 쫫; 쫫; 쫫; ) HANGUL SYLLABLE JJWAS +CAEC;CAEC;110D 116A 11BB;CAEC;110D 116A 11BB; # (쫬; 쫬; 쫬; 쫬; 쫬; ) HANGUL SYLLABLE JJWASS +CAED;CAED;110D 116A 11BC;CAED;110D 116A 11BC; # (쫭; 쫭; 쫭; 쫭; 쫭; ) HANGUL SYLLABLE JJWANG +CAEE;CAEE;110D 116A 11BD;CAEE;110D 116A 11BD; # (쫮; 쫮; 쫮; 쫮; 쫮; ) HANGUL SYLLABLE JJWAJ +CAEF;CAEF;110D 116A 11BE;CAEF;110D 116A 11BE; # (쫯; 쫯; 쫯; 쫯; 쫯; ) HANGUL SYLLABLE JJWAC +CAF0;CAF0;110D 116A 11BF;CAF0;110D 116A 11BF; # (쫰; 쫰; 쫰; 쫰; 쫰; ) HANGUL SYLLABLE JJWAK +CAF1;CAF1;110D 116A 11C0;CAF1;110D 116A 11C0; # (쫱; 쫱; 쫱; 쫱; 쫱; ) HANGUL SYLLABLE JJWAT +CAF2;CAF2;110D 116A 11C1;CAF2;110D 116A 11C1; # (쫲; 쫲; 쫲; 쫲; 쫲; ) HANGUL SYLLABLE JJWAP +CAF3;CAF3;110D 116A 11C2;CAF3;110D 116A 11C2; # (쫳; 쫳; 쫳; 쫳; 쫳; ) HANGUL SYLLABLE JJWAH +CAF4;CAF4;110D 116B;CAF4;110D 116B; # (쫴; 쫴; 쫴; 쫴; 쫴; ) HANGUL SYLLABLE JJWAE +CAF5;CAF5;110D 116B 11A8;CAF5;110D 116B 11A8; # (쫵; 쫵; 쫵; 쫵; 쫵; ) HANGUL SYLLABLE JJWAEG +CAF6;CAF6;110D 116B 11A9;CAF6;110D 116B 11A9; # (쫶; 쫶; 쫶; 쫶; 쫶; ) HANGUL SYLLABLE JJWAEGG +CAF7;CAF7;110D 116B 11AA;CAF7;110D 116B 11AA; # (쫷; 쫷; 쫷; 쫷; 쫷; ) HANGUL SYLLABLE JJWAEGS +CAF8;CAF8;110D 116B 11AB;CAF8;110D 116B 11AB; # (쫸; 쫸; 쫸; 쫸; 쫸; ) HANGUL SYLLABLE JJWAEN +CAF9;CAF9;110D 116B 11AC;CAF9;110D 116B 11AC; # (쫹; 쫹; 쫹; 쫹; 쫹; ) HANGUL SYLLABLE JJWAENJ +CAFA;CAFA;110D 116B 11AD;CAFA;110D 116B 11AD; # (쫺; 쫺; 쫺; 쫺; 쫺; ) HANGUL SYLLABLE JJWAENH +CAFB;CAFB;110D 116B 11AE;CAFB;110D 116B 11AE; # (쫻; 쫻; 쫻; 쫻; 쫻; ) HANGUL SYLLABLE JJWAED +CAFC;CAFC;110D 116B 11AF;CAFC;110D 116B 11AF; # (쫼; 쫼; 쫼; 쫼; 쫼; ) HANGUL SYLLABLE JJWAEL +CAFD;CAFD;110D 116B 11B0;CAFD;110D 116B 11B0; # (쫽; 쫽; 쫽; 쫽; 쫽; ) HANGUL SYLLABLE JJWAELG +CAFE;CAFE;110D 116B 11B1;CAFE;110D 116B 11B1; # (쫾; 쫾; 쫾; 쫾; 쫾; ) HANGUL SYLLABLE JJWAELM +CAFF;CAFF;110D 116B 11B2;CAFF;110D 116B 11B2; # (쫿; 쫿; 쫿; 쫿; 쫿; ) HANGUL SYLLABLE JJWAELB +CB00;CB00;110D 116B 11B3;CB00;110D 116B 11B3; # (쬀; 쬀; 쬀; 쬀; 쬀; ) HANGUL SYLLABLE JJWAELS +CB01;CB01;110D 116B 11B4;CB01;110D 116B 11B4; # (쬁; 쬁; 쬁; 쬁; 쬁; ) HANGUL SYLLABLE JJWAELT +CB02;CB02;110D 116B 11B5;CB02;110D 116B 11B5; # (쬂; 쬂; 쬂; 쬂; 쬂; ) HANGUL SYLLABLE JJWAELP +CB03;CB03;110D 116B 11B6;CB03;110D 116B 11B6; # (쬃; 쬃; 쬃; 쬃; 쬃; ) HANGUL SYLLABLE JJWAELH +CB04;CB04;110D 116B 11B7;CB04;110D 116B 11B7; # (쬄; 쬄; 쬄; 쬄; 쬄; ) HANGUL SYLLABLE JJWAEM +CB05;CB05;110D 116B 11B8;CB05;110D 116B 11B8; # (쬅; 쬅; 쬅; 쬅; 쬅; ) HANGUL SYLLABLE JJWAEB +CB06;CB06;110D 116B 11B9;CB06;110D 116B 11B9; # (쬆; 쬆; 쬆; 쬆; 쬆; ) HANGUL SYLLABLE JJWAEBS +CB07;CB07;110D 116B 11BA;CB07;110D 116B 11BA; # (쬇; 쬇; 쬇; 쬇; 쬇; ) HANGUL SYLLABLE JJWAES +CB08;CB08;110D 116B 11BB;CB08;110D 116B 11BB; # (쬈; 쬈; 쬈; 쬈; 쬈; ) HANGUL SYLLABLE JJWAESS +CB09;CB09;110D 116B 11BC;CB09;110D 116B 11BC; # (쬉; 쬉; 쬉; 쬉; 쬉; ) HANGUL SYLLABLE JJWAENG +CB0A;CB0A;110D 116B 11BD;CB0A;110D 116B 11BD; # (쬊; 쬊; 쬊; 쬊; 쬊; ) HANGUL SYLLABLE JJWAEJ +CB0B;CB0B;110D 116B 11BE;CB0B;110D 116B 11BE; # (쬋; 쬋; 쬋; 쬋; 쬋; ) HANGUL SYLLABLE JJWAEC +CB0C;CB0C;110D 116B 11BF;CB0C;110D 116B 11BF; # (쬌; 쬌; 쬌; 쬌; 쬌; ) HANGUL SYLLABLE JJWAEK +CB0D;CB0D;110D 116B 11C0;CB0D;110D 116B 11C0; # (쬍; 쬍; 쬍; 쬍; 쬍; ) HANGUL SYLLABLE JJWAET +CB0E;CB0E;110D 116B 11C1;CB0E;110D 116B 11C1; # (쬎; 쬎; 쬎; 쬎; 쬎; ) HANGUL SYLLABLE JJWAEP +CB0F;CB0F;110D 116B 11C2;CB0F;110D 116B 11C2; # (쬏; 쬏; 쬏; 쬏; 쬏; ) HANGUL SYLLABLE JJWAEH +CB10;CB10;110D 116C;CB10;110D 116C; # (쬐; 쬐; 쬐; 쬐; 쬐; ) HANGUL SYLLABLE JJOE +CB11;CB11;110D 116C 11A8;CB11;110D 116C 11A8; # (쬑; 쬑; 쬑; 쬑; 쬑; ) HANGUL SYLLABLE JJOEG +CB12;CB12;110D 116C 11A9;CB12;110D 116C 11A9; # (쬒; 쬒; 쬒; 쬒; 쬒; ) HANGUL SYLLABLE JJOEGG +CB13;CB13;110D 116C 11AA;CB13;110D 116C 11AA; # (쬓; 쬓; 쬓; 쬓; 쬓; ) HANGUL SYLLABLE JJOEGS +CB14;CB14;110D 116C 11AB;CB14;110D 116C 11AB; # (쬔; 쬔; 쬔; 쬔; 쬔; ) HANGUL SYLLABLE JJOEN +CB15;CB15;110D 116C 11AC;CB15;110D 116C 11AC; # (쬕; 쬕; 쬕; 쬕; 쬕; ) HANGUL SYLLABLE JJOENJ +CB16;CB16;110D 116C 11AD;CB16;110D 116C 11AD; # (쬖; 쬖; 쬖; 쬖; 쬖; ) HANGUL SYLLABLE JJOENH +CB17;CB17;110D 116C 11AE;CB17;110D 116C 11AE; # (쬗; 쬗; 쬗; 쬗; 쬗; ) HANGUL SYLLABLE JJOED +CB18;CB18;110D 116C 11AF;CB18;110D 116C 11AF; # (쬘; 쬘; 쬘; 쬘; 쬘; ) HANGUL SYLLABLE JJOEL +CB19;CB19;110D 116C 11B0;CB19;110D 116C 11B0; # (쬙; 쬙; 쬙; 쬙; 쬙; ) HANGUL SYLLABLE JJOELG +CB1A;CB1A;110D 116C 11B1;CB1A;110D 116C 11B1; # (쬚; 쬚; 쬚; 쬚; 쬚; ) HANGUL SYLLABLE JJOELM +CB1B;CB1B;110D 116C 11B2;CB1B;110D 116C 11B2; # (쬛; 쬛; 쬛; 쬛; 쬛; ) HANGUL SYLLABLE JJOELB +CB1C;CB1C;110D 116C 11B3;CB1C;110D 116C 11B3; # (쬜; 쬜; 쬜; 쬜; 쬜; ) HANGUL SYLLABLE JJOELS +CB1D;CB1D;110D 116C 11B4;CB1D;110D 116C 11B4; # (쬝; 쬝; 쬝; 쬝; 쬝; ) HANGUL SYLLABLE JJOELT +CB1E;CB1E;110D 116C 11B5;CB1E;110D 116C 11B5; # (쬞; 쬞; 쬞; 쬞; 쬞; ) HANGUL SYLLABLE JJOELP +CB1F;CB1F;110D 116C 11B6;CB1F;110D 116C 11B6; # (쬟; 쬟; 쬟; 쬟; 쬟; ) HANGUL SYLLABLE JJOELH +CB20;CB20;110D 116C 11B7;CB20;110D 116C 11B7; # (쬠; 쬠; 쬠; 쬠; 쬠; ) HANGUL SYLLABLE JJOEM +CB21;CB21;110D 116C 11B8;CB21;110D 116C 11B8; # (쬡; 쬡; 쬡; 쬡; 쬡; ) HANGUL SYLLABLE JJOEB +CB22;CB22;110D 116C 11B9;CB22;110D 116C 11B9; # (쬢; 쬢; 쬢; 쬢; 쬢; ) HANGUL SYLLABLE JJOEBS +CB23;CB23;110D 116C 11BA;CB23;110D 116C 11BA; # (쬣; 쬣; 쬣; 쬣; 쬣; ) HANGUL SYLLABLE JJOES +CB24;CB24;110D 116C 11BB;CB24;110D 116C 11BB; # (쬤; 쬤; 쬤; 쬤; 쬤; ) HANGUL SYLLABLE JJOESS +CB25;CB25;110D 116C 11BC;CB25;110D 116C 11BC; # (쬥; 쬥; 쬥; 쬥; 쬥; ) HANGUL SYLLABLE JJOENG +CB26;CB26;110D 116C 11BD;CB26;110D 116C 11BD; # (쬦; 쬦; 쬦; 쬦; 쬦; ) HANGUL SYLLABLE JJOEJ +CB27;CB27;110D 116C 11BE;CB27;110D 116C 11BE; # (쬧; 쬧; 쬧; 쬧; 쬧; ) HANGUL SYLLABLE JJOEC +CB28;CB28;110D 116C 11BF;CB28;110D 116C 11BF; # (쬨; 쬨; 쬨; 쬨; 쬨; ) HANGUL SYLLABLE JJOEK +CB29;CB29;110D 116C 11C0;CB29;110D 116C 11C0; # (쬩; 쬩; 쬩; 쬩; 쬩; ) HANGUL SYLLABLE JJOET +CB2A;CB2A;110D 116C 11C1;CB2A;110D 116C 11C1; # (쬪; 쬪; 쬪; 쬪; 쬪; ) HANGUL SYLLABLE JJOEP +CB2B;CB2B;110D 116C 11C2;CB2B;110D 116C 11C2; # (쬫; 쬫; 쬫; 쬫; 쬫; ) HANGUL SYLLABLE JJOEH +CB2C;CB2C;110D 116D;CB2C;110D 116D; # (쬬; 쬬; 쬬; 쬬; 쬬; ) HANGUL SYLLABLE JJYO +CB2D;CB2D;110D 116D 11A8;CB2D;110D 116D 11A8; # (쬭; 쬭; 쬭; 쬭; 쬭; ) HANGUL SYLLABLE JJYOG +CB2E;CB2E;110D 116D 11A9;CB2E;110D 116D 11A9; # (쬮; 쬮; 쬮; 쬮; 쬮; ) HANGUL SYLLABLE JJYOGG +CB2F;CB2F;110D 116D 11AA;CB2F;110D 116D 11AA; # (쬯; 쬯; 쬯; 쬯; 쬯; ) HANGUL SYLLABLE JJYOGS +CB30;CB30;110D 116D 11AB;CB30;110D 116D 11AB; # (쬰; 쬰; 쬰; 쬰; 쬰; ) HANGUL SYLLABLE JJYON +CB31;CB31;110D 116D 11AC;CB31;110D 116D 11AC; # (쬱; 쬱; 쬱; 쬱; 쬱; ) HANGUL SYLLABLE JJYONJ +CB32;CB32;110D 116D 11AD;CB32;110D 116D 11AD; # (쬲; 쬲; 쬲; 쬲; 쬲; ) HANGUL SYLLABLE JJYONH +CB33;CB33;110D 116D 11AE;CB33;110D 116D 11AE; # (쬳; 쬳; 쬳; 쬳; 쬳; ) HANGUL SYLLABLE JJYOD +CB34;CB34;110D 116D 11AF;CB34;110D 116D 11AF; # (쬴; 쬴; 쬴; 쬴; 쬴; ) HANGUL SYLLABLE JJYOL +CB35;CB35;110D 116D 11B0;CB35;110D 116D 11B0; # (쬵; 쬵; 쬵; 쬵; 쬵; ) HANGUL SYLLABLE JJYOLG +CB36;CB36;110D 116D 11B1;CB36;110D 116D 11B1; # (쬶; 쬶; 쬶; 쬶; 쬶; ) HANGUL SYLLABLE JJYOLM +CB37;CB37;110D 116D 11B2;CB37;110D 116D 11B2; # (쬷; 쬷; 쬷; 쬷; 쬷; ) HANGUL SYLLABLE JJYOLB +CB38;CB38;110D 116D 11B3;CB38;110D 116D 11B3; # (쬸; 쬸; 쬸; 쬸; 쬸; ) HANGUL SYLLABLE JJYOLS +CB39;CB39;110D 116D 11B4;CB39;110D 116D 11B4; # (쬹; 쬹; 쬹; 쬹; 쬹; ) HANGUL SYLLABLE JJYOLT +CB3A;CB3A;110D 116D 11B5;CB3A;110D 116D 11B5; # (쬺; 쬺; 쬺; 쬺; 쬺; ) HANGUL SYLLABLE JJYOLP +CB3B;CB3B;110D 116D 11B6;CB3B;110D 116D 11B6; # (쬻; 쬻; 쬻; 쬻; 쬻; ) HANGUL SYLLABLE JJYOLH +CB3C;CB3C;110D 116D 11B7;CB3C;110D 116D 11B7; # (쬼; 쬼; 쬼; 쬼; 쬼; ) HANGUL SYLLABLE JJYOM +CB3D;CB3D;110D 116D 11B8;CB3D;110D 116D 11B8; # (쬽; 쬽; 쬽; 쬽; 쬽; ) HANGUL SYLLABLE JJYOB +CB3E;CB3E;110D 116D 11B9;CB3E;110D 116D 11B9; # (쬾; 쬾; 쬾; 쬾; 쬾; ) HANGUL SYLLABLE JJYOBS +CB3F;CB3F;110D 116D 11BA;CB3F;110D 116D 11BA; # (쬿; 쬿; 쬿; 쬿; 쬿; ) HANGUL SYLLABLE JJYOS +CB40;CB40;110D 116D 11BB;CB40;110D 116D 11BB; # (쭀; 쭀; 쭀; 쭀; 쭀; ) HANGUL SYLLABLE JJYOSS +CB41;CB41;110D 116D 11BC;CB41;110D 116D 11BC; # (쭁; 쭁; 쭁; 쭁; 쭁; ) HANGUL SYLLABLE JJYONG +CB42;CB42;110D 116D 11BD;CB42;110D 116D 11BD; # (쭂; 쭂; 쭂; 쭂; 쭂; ) HANGUL SYLLABLE JJYOJ +CB43;CB43;110D 116D 11BE;CB43;110D 116D 11BE; # (쭃; 쭃; 쭃; 쭃; 쭃; ) HANGUL SYLLABLE JJYOC +CB44;CB44;110D 116D 11BF;CB44;110D 116D 11BF; # (쭄; 쭄; 쭄; 쭄; 쭄; ) HANGUL SYLLABLE JJYOK +CB45;CB45;110D 116D 11C0;CB45;110D 116D 11C0; # (쭅; 쭅; 쭅; 쭅; 쭅; ) HANGUL SYLLABLE JJYOT +CB46;CB46;110D 116D 11C1;CB46;110D 116D 11C1; # (쭆; 쭆; 쭆; 쭆; 쭆; ) HANGUL SYLLABLE JJYOP +CB47;CB47;110D 116D 11C2;CB47;110D 116D 11C2; # (쭇; 쭇; 쭇; 쭇; 쭇; ) HANGUL SYLLABLE JJYOH +CB48;CB48;110D 116E;CB48;110D 116E; # (쭈; 쭈; 쭈; 쭈; 쭈; ) HANGUL SYLLABLE JJU +CB49;CB49;110D 116E 11A8;CB49;110D 116E 11A8; # (쭉; 쭉; 쭉; 쭉; 쭉; ) HANGUL SYLLABLE JJUG +CB4A;CB4A;110D 116E 11A9;CB4A;110D 116E 11A9; # (쭊; 쭊; 쭊; 쭊; 쭊; ) HANGUL SYLLABLE JJUGG +CB4B;CB4B;110D 116E 11AA;CB4B;110D 116E 11AA; # (쭋; 쭋; 쭋; 쭋; 쭋; ) HANGUL SYLLABLE JJUGS +CB4C;CB4C;110D 116E 11AB;CB4C;110D 116E 11AB; # (쭌; 쭌; 쭌; 쭌; 쭌; ) HANGUL SYLLABLE JJUN +CB4D;CB4D;110D 116E 11AC;CB4D;110D 116E 11AC; # (쭍; 쭍; 쭍; 쭍; 쭍; ) HANGUL SYLLABLE JJUNJ +CB4E;CB4E;110D 116E 11AD;CB4E;110D 116E 11AD; # (쭎; 쭎; 쭎; 쭎; 쭎; ) HANGUL SYLLABLE JJUNH +CB4F;CB4F;110D 116E 11AE;CB4F;110D 116E 11AE; # (쭏; 쭏; 쭏; 쭏; 쭏; ) HANGUL SYLLABLE JJUD +CB50;CB50;110D 116E 11AF;CB50;110D 116E 11AF; # (쭐; 쭐; 쭐; 쭐; 쭐; ) HANGUL SYLLABLE JJUL +CB51;CB51;110D 116E 11B0;CB51;110D 116E 11B0; # (쭑; 쭑; 쭑; 쭑; 쭑; ) HANGUL SYLLABLE JJULG +CB52;CB52;110D 116E 11B1;CB52;110D 116E 11B1; # (쭒; 쭒; 쭒; 쭒; 쭒; ) HANGUL SYLLABLE JJULM +CB53;CB53;110D 116E 11B2;CB53;110D 116E 11B2; # (쭓; 쭓; 쭓; 쭓; 쭓; ) HANGUL SYLLABLE JJULB +CB54;CB54;110D 116E 11B3;CB54;110D 116E 11B3; # (쭔; 쭔; 쭔; 쭔; 쭔; ) HANGUL SYLLABLE JJULS +CB55;CB55;110D 116E 11B4;CB55;110D 116E 11B4; # (쭕; 쭕; 쭕; 쭕; 쭕; ) HANGUL SYLLABLE JJULT +CB56;CB56;110D 116E 11B5;CB56;110D 116E 11B5; # (쭖; 쭖; 쭖; 쭖; 쭖; ) HANGUL SYLLABLE JJULP +CB57;CB57;110D 116E 11B6;CB57;110D 116E 11B6; # (쭗; 쭗; 쭗; 쭗; 쭗; ) HANGUL SYLLABLE JJULH +CB58;CB58;110D 116E 11B7;CB58;110D 116E 11B7; # (쭘; 쭘; 쭘; 쭘; 쭘; ) HANGUL SYLLABLE JJUM +CB59;CB59;110D 116E 11B8;CB59;110D 116E 11B8; # (쭙; 쭙; 쭙; 쭙; 쭙; ) HANGUL SYLLABLE JJUB +CB5A;CB5A;110D 116E 11B9;CB5A;110D 116E 11B9; # (쭚; 쭚; 쭚; 쭚; 쭚; ) HANGUL SYLLABLE JJUBS +CB5B;CB5B;110D 116E 11BA;CB5B;110D 116E 11BA; # (쭛; 쭛; 쭛; 쭛; 쭛; ) HANGUL SYLLABLE JJUS +CB5C;CB5C;110D 116E 11BB;CB5C;110D 116E 11BB; # (쭜; 쭜; 쭜; 쭜; 쭜; ) HANGUL SYLLABLE JJUSS +CB5D;CB5D;110D 116E 11BC;CB5D;110D 116E 11BC; # (쭝; 쭝; 쭝; 쭝; 쭝; ) HANGUL SYLLABLE JJUNG +CB5E;CB5E;110D 116E 11BD;CB5E;110D 116E 11BD; # (쭞; 쭞; 쭞; 쭞; 쭞; ) HANGUL SYLLABLE JJUJ +CB5F;CB5F;110D 116E 11BE;CB5F;110D 116E 11BE; # (쭟; 쭟; 쭟; 쭟; 쭟; ) HANGUL SYLLABLE JJUC +CB60;CB60;110D 116E 11BF;CB60;110D 116E 11BF; # (쭠; 쭠; 쭠; 쭠; 쭠; ) HANGUL SYLLABLE JJUK +CB61;CB61;110D 116E 11C0;CB61;110D 116E 11C0; # (쭡; 쭡; 쭡; 쭡; 쭡; ) HANGUL SYLLABLE JJUT +CB62;CB62;110D 116E 11C1;CB62;110D 116E 11C1; # (쭢; 쭢; 쭢; 쭢; 쭢; ) HANGUL SYLLABLE JJUP +CB63;CB63;110D 116E 11C2;CB63;110D 116E 11C2; # (쭣; 쭣; 쭣; 쭣; 쭣; ) HANGUL SYLLABLE JJUH +CB64;CB64;110D 116F;CB64;110D 116F; # (쭤; 쭤; 쭤; 쭤; 쭤; ) HANGUL SYLLABLE JJWEO +CB65;CB65;110D 116F 11A8;CB65;110D 116F 11A8; # (쭥; 쭥; 쭥; 쭥; 쭥; ) HANGUL SYLLABLE JJWEOG +CB66;CB66;110D 116F 11A9;CB66;110D 116F 11A9; # (쭦; 쭦; 쭦; 쭦; 쭦; ) HANGUL SYLLABLE JJWEOGG +CB67;CB67;110D 116F 11AA;CB67;110D 116F 11AA; # (쭧; 쭧; 쭧; 쭧; 쭧; ) HANGUL SYLLABLE JJWEOGS +CB68;CB68;110D 116F 11AB;CB68;110D 116F 11AB; # (쭨; 쭨; 쭨; 쭨; 쭨; ) HANGUL SYLLABLE JJWEON +CB69;CB69;110D 116F 11AC;CB69;110D 116F 11AC; # (쭩; 쭩; 쭩; 쭩; 쭩; ) HANGUL SYLLABLE JJWEONJ +CB6A;CB6A;110D 116F 11AD;CB6A;110D 116F 11AD; # (쭪; 쭪; 쭪; 쭪; 쭪; ) HANGUL SYLLABLE JJWEONH +CB6B;CB6B;110D 116F 11AE;CB6B;110D 116F 11AE; # (쭫; 쭫; 쭫; 쭫; 쭫; ) HANGUL SYLLABLE JJWEOD +CB6C;CB6C;110D 116F 11AF;CB6C;110D 116F 11AF; # (쭬; 쭬; 쭬; 쭬; 쭬; ) HANGUL SYLLABLE JJWEOL +CB6D;CB6D;110D 116F 11B0;CB6D;110D 116F 11B0; # (쭭; 쭭; 쭭; 쭭; 쭭; ) HANGUL SYLLABLE JJWEOLG +CB6E;CB6E;110D 116F 11B1;CB6E;110D 116F 11B1; # (쭮; 쭮; 쭮; 쭮; 쭮; ) HANGUL SYLLABLE JJWEOLM +CB6F;CB6F;110D 116F 11B2;CB6F;110D 116F 11B2; # (쭯; 쭯; 쭯; 쭯; 쭯; ) HANGUL SYLLABLE JJWEOLB +CB70;CB70;110D 116F 11B3;CB70;110D 116F 11B3; # (쭰; 쭰; 쭰; 쭰; 쭰; ) HANGUL SYLLABLE JJWEOLS +CB71;CB71;110D 116F 11B4;CB71;110D 116F 11B4; # (쭱; 쭱; 쭱; 쭱; 쭱; ) HANGUL SYLLABLE JJWEOLT +CB72;CB72;110D 116F 11B5;CB72;110D 116F 11B5; # (쭲; 쭲; 쭲; 쭲; 쭲; ) HANGUL SYLLABLE JJWEOLP +CB73;CB73;110D 116F 11B6;CB73;110D 116F 11B6; # (쭳; 쭳; 쭳; 쭳; 쭳; ) HANGUL SYLLABLE JJWEOLH +CB74;CB74;110D 116F 11B7;CB74;110D 116F 11B7; # (쭴; 쭴; 쭴; 쭴; 쭴; ) HANGUL SYLLABLE JJWEOM +CB75;CB75;110D 116F 11B8;CB75;110D 116F 11B8; # (쭵; 쭵; 쭵; 쭵; 쭵; ) HANGUL SYLLABLE JJWEOB +CB76;CB76;110D 116F 11B9;CB76;110D 116F 11B9; # (쭶; 쭶; 쭶; 쭶; 쭶; ) HANGUL SYLLABLE JJWEOBS +CB77;CB77;110D 116F 11BA;CB77;110D 116F 11BA; # (쭷; 쭷; 쭷; 쭷; 쭷; ) HANGUL SYLLABLE JJWEOS +CB78;CB78;110D 116F 11BB;CB78;110D 116F 11BB; # (쭸; 쭸; 쭸; 쭸; 쭸; ) HANGUL SYLLABLE JJWEOSS +CB79;CB79;110D 116F 11BC;CB79;110D 116F 11BC; # (쭹; 쭹; 쭹; 쭹; 쭹; ) HANGUL SYLLABLE JJWEONG +CB7A;CB7A;110D 116F 11BD;CB7A;110D 116F 11BD; # (쭺; 쭺; 쭺; 쭺; 쭺; ) HANGUL SYLLABLE JJWEOJ +CB7B;CB7B;110D 116F 11BE;CB7B;110D 116F 11BE; # (쭻; 쭻; 쭻; 쭻; 쭻; ) HANGUL SYLLABLE JJWEOC +CB7C;CB7C;110D 116F 11BF;CB7C;110D 116F 11BF; # (쭼; 쭼; 쭼; 쭼; 쭼; ) HANGUL SYLLABLE JJWEOK +CB7D;CB7D;110D 116F 11C0;CB7D;110D 116F 11C0; # (쭽; 쭽; 쭽; 쭽; 쭽; ) HANGUL SYLLABLE JJWEOT +CB7E;CB7E;110D 116F 11C1;CB7E;110D 116F 11C1; # (쭾; 쭾; 쭾; 쭾; 쭾; ) HANGUL SYLLABLE JJWEOP +CB7F;CB7F;110D 116F 11C2;CB7F;110D 116F 11C2; # (쭿; 쭿; 쭿; 쭿; 쭿; ) HANGUL SYLLABLE JJWEOH +CB80;CB80;110D 1170;CB80;110D 1170; # (쮀; 쮀; 쮀; 쮀; 쮀; ) HANGUL SYLLABLE JJWE +CB81;CB81;110D 1170 11A8;CB81;110D 1170 11A8; # (쮁; 쮁; 쮁; 쮁; 쮁; ) HANGUL SYLLABLE JJWEG +CB82;CB82;110D 1170 11A9;CB82;110D 1170 11A9; # (쮂; 쮂; 쮂; 쮂; 쮂; ) HANGUL SYLLABLE JJWEGG +CB83;CB83;110D 1170 11AA;CB83;110D 1170 11AA; # (쮃; 쮃; 쮃; 쮃; 쮃; ) HANGUL SYLLABLE JJWEGS +CB84;CB84;110D 1170 11AB;CB84;110D 1170 11AB; # (쮄; 쮄; 쮄; 쮄; 쮄; ) HANGUL SYLLABLE JJWEN +CB85;CB85;110D 1170 11AC;CB85;110D 1170 11AC; # (쮅; 쮅; 쮅; 쮅; 쮅; ) HANGUL SYLLABLE JJWENJ +CB86;CB86;110D 1170 11AD;CB86;110D 1170 11AD; # (쮆; 쮆; 쮆; 쮆; 쮆; ) HANGUL SYLLABLE JJWENH +CB87;CB87;110D 1170 11AE;CB87;110D 1170 11AE; # (쮇; 쮇; 쮇; 쮇; 쮇; ) HANGUL SYLLABLE JJWED +CB88;CB88;110D 1170 11AF;CB88;110D 1170 11AF; # (쮈; 쮈; 쮈; 쮈; 쮈; ) HANGUL SYLLABLE JJWEL +CB89;CB89;110D 1170 11B0;CB89;110D 1170 11B0; # (쮉; 쮉; 쮉; 쮉; 쮉; ) HANGUL SYLLABLE JJWELG +CB8A;CB8A;110D 1170 11B1;CB8A;110D 1170 11B1; # (쮊; 쮊; 쮊; 쮊; 쮊; ) HANGUL SYLLABLE JJWELM +CB8B;CB8B;110D 1170 11B2;CB8B;110D 1170 11B2; # (쮋; 쮋; 쮋; 쮋; 쮋; ) HANGUL SYLLABLE JJWELB +CB8C;CB8C;110D 1170 11B3;CB8C;110D 1170 11B3; # (쮌; 쮌; 쮌; 쮌; 쮌; ) HANGUL SYLLABLE JJWELS +CB8D;CB8D;110D 1170 11B4;CB8D;110D 1170 11B4; # (쮍; 쮍; 쮍; 쮍; 쮍; ) HANGUL SYLLABLE JJWELT +CB8E;CB8E;110D 1170 11B5;CB8E;110D 1170 11B5; # (쮎; 쮎; 쮎; 쮎; 쮎; ) HANGUL SYLLABLE JJWELP +CB8F;CB8F;110D 1170 11B6;CB8F;110D 1170 11B6; # (쮏; 쮏; 쮏; 쮏; 쮏; ) HANGUL SYLLABLE JJWELH +CB90;CB90;110D 1170 11B7;CB90;110D 1170 11B7; # (쮐; 쮐; 쮐; 쮐; 쮐; ) HANGUL SYLLABLE JJWEM +CB91;CB91;110D 1170 11B8;CB91;110D 1170 11B8; # (쮑; 쮑; 쮑; 쮑; 쮑; ) HANGUL SYLLABLE JJWEB +CB92;CB92;110D 1170 11B9;CB92;110D 1170 11B9; # (쮒; 쮒; 쮒; 쮒; 쮒; ) HANGUL SYLLABLE JJWEBS +CB93;CB93;110D 1170 11BA;CB93;110D 1170 11BA; # (쮓; 쮓; 쮓; 쮓; 쮓; ) HANGUL SYLLABLE JJWES +CB94;CB94;110D 1170 11BB;CB94;110D 1170 11BB; # (쮔; 쮔; 쮔; 쮔; 쮔; ) HANGUL SYLLABLE JJWESS +CB95;CB95;110D 1170 11BC;CB95;110D 1170 11BC; # (쮕; 쮕; 쮕; 쮕; 쮕; ) HANGUL SYLLABLE JJWENG +CB96;CB96;110D 1170 11BD;CB96;110D 1170 11BD; # (쮖; 쮖; 쮖; 쮖; 쮖; ) HANGUL SYLLABLE JJWEJ +CB97;CB97;110D 1170 11BE;CB97;110D 1170 11BE; # (쮗; 쮗; 쮗; 쮗; 쮗; ) HANGUL SYLLABLE JJWEC +CB98;CB98;110D 1170 11BF;CB98;110D 1170 11BF; # (쮘; 쮘; 쮘; 쮘; 쮘; ) HANGUL SYLLABLE JJWEK +CB99;CB99;110D 1170 11C0;CB99;110D 1170 11C0; # (쮙; 쮙; 쮙; 쮙; 쮙; ) HANGUL SYLLABLE JJWET +CB9A;CB9A;110D 1170 11C1;CB9A;110D 1170 11C1; # (쮚; 쮚; 쮚; 쮚; 쮚; ) HANGUL SYLLABLE JJWEP +CB9B;CB9B;110D 1170 11C2;CB9B;110D 1170 11C2; # (쮛; 쮛; 쮛; 쮛; 쮛; ) HANGUL SYLLABLE JJWEH +CB9C;CB9C;110D 1171;CB9C;110D 1171; # (쮜; 쮜; 쮜; 쮜; 쮜; ) HANGUL SYLLABLE JJWI +CB9D;CB9D;110D 1171 11A8;CB9D;110D 1171 11A8; # (쮝; 쮝; 쮝; 쮝; 쮝; ) HANGUL SYLLABLE JJWIG +CB9E;CB9E;110D 1171 11A9;CB9E;110D 1171 11A9; # (쮞; 쮞; 쮞; 쮞; 쮞; ) HANGUL SYLLABLE JJWIGG +CB9F;CB9F;110D 1171 11AA;CB9F;110D 1171 11AA; # (쮟; 쮟; 쮟; 쮟; 쮟; ) HANGUL SYLLABLE JJWIGS +CBA0;CBA0;110D 1171 11AB;CBA0;110D 1171 11AB; # (쮠; 쮠; 쮠; 쮠; 쮠; ) HANGUL SYLLABLE JJWIN +CBA1;CBA1;110D 1171 11AC;CBA1;110D 1171 11AC; # (쮡; 쮡; 쮡; 쮡; 쮡; ) HANGUL SYLLABLE JJWINJ +CBA2;CBA2;110D 1171 11AD;CBA2;110D 1171 11AD; # (쮢; 쮢; 쮢; 쮢; 쮢; ) HANGUL SYLLABLE JJWINH +CBA3;CBA3;110D 1171 11AE;CBA3;110D 1171 11AE; # (쮣; 쮣; 쮣; 쮣; 쮣; ) HANGUL SYLLABLE JJWID +CBA4;CBA4;110D 1171 11AF;CBA4;110D 1171 11AF; # (쮤; 쮤; 쮤; 쮤; 쮤; ) HANGUL SYLLABLE JJWIL +CBA5;CBA5;110D 1171 11B0;CBA5;110D 1171 11B0; # (쮥; 쮥; 쮥; 쮥; 쮥; ) HANGUL SYLLABLE JJWILG +CBA6;CBA6;110D 1171 11B1;CBA6;110D 1171 11B1; # (쮦; 쮦; 쮦; 쮦; 쮦; ) HANGUL SYLLABLE JJWILM +CBA7;CBA7;110D 1171 11B2;CBA7;110D 1171 11B2; # (쮧; 쮧; 쮧; 쮧; 쮧; ) HANGUL SYLLABLE JJWILB +CBA8;CBA8;110D 1171 11B3;CBA8;110D 1171 11B3; # (쮨; 쮨; 쮨; 쮨; 쮨; ) HANGUL SYLLABLE JJWILS +CBA9;CBA9;110D 1171 11B4;CBA9;110D 1171 11B4; # (쮩; 쮩; 쮩; 쮩; 쮩; ) HANGUL SYLLABLE JJWILT +CBAA;CBAA;110D 1171 11B5;CBAA;110D 1171 11B5; # (쮪; 쮪; 쮪; 쮪; 쮪; ) HANGUL SYLLABLE JJWILP +CBAB;CBAB;110D 1171 11B6;CBAB;110D 1171 11B6; # (쮫; 쮫; 쮫; 쮫; 쮫; ) HANGUL SYLLABLE JJWILH +CBAC;CBAC;110D 1171 11B7;CBAC;110D 1171 11B7; # (쮬; 쮬; 쮬; 쮬; 쮬; ) HANGUL SYLLABLE JJWIM +CBAD;CBAD;110D 1171 11B8;CBAD;110D 1171 11B8; # (쮭; 쮭; 쮭; 쮭; 쮭; ) HANGUL SYLLABLE JJWIB +CBAE;CBAE;110D 1171 11B9;CBAE;110D 1171 11B9; # (쮮; 쮮; 쮮; 쮮; 쮮; ) HANGUL SYLLABLE JJWIBS +CBAF;CBAF;110D 1171 11BA;CBAF;110D 1171 11BA; # (쮯; 쮯; 쮯; 쮯; 쮯; ) HANGUL SYLLABLE JJWIS +CBB0;CBB0;110D 1171 11BB;CBB0;110D 1171 11BB; # (쮰; 쮰; 쮰; 쮰; 쮰; ) HANGUL SYLLABLE JJWISS +CBB1;CBB1;110D 1171 11BC;CBB1;110D 1171 11BC; # (쮱; 쮱; 쮱; 쮱; 쮱; ) HANGUL SYLLABLE JJWING +CBB2;CBB2;110D 1171 11BD;CBB2;110D 1171 11BD; # (쮲; 쮲; 쮲; 쮲; 쮲; ) HANGUL SYLLABLE JJWIJ +CBB3;CBB3;110D 1171 11BE;CBB3;110D 1171 11BE; # (쮳; 쮳; 쮳; 쮳; 쮳; ) HANGUL SYLLABLE JJWIC +CBB4;CBB4;110D 1171 11BF;CBB4;110D 1171 11BF; # (쮴; 쮴; 쮴; 쮴; 쮴; ) HANGUL SYLLABLE JJWIK +CBB5;CBB5;110D 1171 11C0;CBB5;110D 1171 11C0; # (쮵; 쮵; 쮵; 쮵; 쮵; ) HANGUL SYLLABLE JJWIT +CBB6;CBB6;110D 1171 11C1;CBB6;110D 1171 11C1; # (쮶; 쮶; 쮶; 쮶; 쮶; ) HANGUL SYLLABLE JJWIP +CBB7;CBB7;110D 1171 11C2;CBB7;110D 1171 11C2; # (쮷; 쮷; 쮷; 쮷; 쮷; ) HANGUL SYLLABLE JJWIH +CBB8;CBB8;110D 1172;CBB8;110D 1172; # (쮸; 쮸; 쮸; 쮸; 쮸; ) HANGUL SYLLABLE JJYU +CBB9;CBB9;110D 1172 11A8;CBB9;110D 1172 11A8; # (쮹; 쮹; 쮹; 쮹; 쮹; ) HANGUL SYLLABLE JJYUG +CBBA;CBBA;110D 1172 11A9;CBBA;110D 1172 11A9; # (쮺; 쮺; 쮺; 쮺; 쮺; ) HANGUL SYLLABLE JJYUGG +CBBB;CBBB;110D 1172 11AA;CBBB;110D 1172 11AA; # (쮻; 쮻; 쮻; 쮻; 쮻; ) HANGUL SYLLABLE JJYUGS +CBBC;CBBC;110D 1172 11AB;CBBC;110D 1172 11AB; # (쮼; 쮼; 쮼; 쮼; 쮼; ) HANGUL SYLLABLE JJYUN +CBBD;CBBD;110D 1172 11AC;CBBD;110D 1172 11AC; # (쮽; 쮽; 쮽; 쮽; 쮽; ) HANGUL SYLLABLE JJYUNJ +CBBE;CBBE;110D 1172 11AD;CBBE;110D 1172 11AD; # (쮾; 쮾; 쮾; 쮾; 쮾; ) HANGUL SYLLABLE JJYUNH +CBBF;CBBF;110D 1172 11AE;CBBF;110D 1172 11AE; # (쮿; 쮿; 쮿; 쮿; 쮿; ) HANGUL SYLLABLE JJYUD +CBC0;CBC0;110D 1172 11AF;CBC0;110D 1172 11AF; # (쯀; 쯀; 쯀; 쯀; 쯀; ) HANGUL SYLLABLE JJYUL +CBC1;CBC1;110D 1172 11B0;CBC1;110D 1172 11B0; # (쯁; 쯁; 쯁; 쯁; 쯁; ) HANGUL SYLLABLE JJYULG +CBC2;CBC2;110D 1172 11B1;CBC2;110D 1172 11B1; # (쯂; 쯂; 쯂; 쯂; 쯂; ) HANGUL SYLLABLE JJYULM +CBC3;CBC3;110D 1172 11B2;CBC3;110D 1172 11B2; # (쯃; 쯃; 쯃; 쯃; 쯃; ) HANGUL SYLLABLE JJYULB +CBC4;CBC4;110D 1172 11B3;CBC4;110D 1172 11B3; # (쯄; 쯄; 쯄; 쯄; 쯄; ) HANGUL SYLLABLE JJYULS +CBC5;CBC5;110D 1172 11B4;CBC5;110D 1172 11B4; # (쯅; 쯅; 쯅; 쯅; 쯅; ) HANGUL SYLLABLE JJYULT +CBC6;CBC6;110D 1172 11B5;CBC6;110D 1172 11B5; # (쯆; 쯆; 쯆; 쯆; 쯆; ) HANGUL SYLLABLE JJYULP +CBC7;CBC7;110D 1172 11B6;CBC7;110D 1172 11B6; # (쯇; 쯇; 쯇; 쯇; 쯇; ) HANGUL SYLLABLE JJYULH +CBC8;CBC8;110D 1172 11B7;CBC8;110D 1172 11B7; # (쯈; 쯈; 쯈; 쯈; 쯈; ) HANGUL SYLLABLE JJYUM +CBC9;CBC9;110D 1172 11B8;CBC9;110D 1172 11B8; # (쯉; 쯉; 쯉; 쯉; 쯉; ) HANGUL SYLLABLE JJYUB +CBCA;CBCA;110D 1172 11B9;CBCA;110D 1172 11B9; # (쯊; 쯊; 쯊; 쯊; 쯊; ) HANGUL SYLLABLE JJYUBS +CBCB;CBCB;110D 1172 11BA;CBCB;110D 1172 11BA; # (쯋; 쯋; 쯋; 쯋; 쯋; ) HANGUL SYLLABLE JJYUS +CBCC;CBCC;110D 1172 11BB;CBCC;110D 1172 11BB; # (쯌; 쯌; 쯌; 쯌; 쯌; ) HANGUL SYLLABLE JJYUSS +CBCD;CBCD;110D 1172 11BC;CBCD;110D 1172 11BC; # (쯍; 쯍; 쯍; 쯍; 쯍; ) HANGUL SYLLABLE JJYUNG +CBCE;CBCE;110D 1172 11BD;CBCE;110D 1172 11BD; # (쯎; 쯎; 쯎; 쯎; 쯎; ) HANGUL SYLLABLE JJYUJ +CBCF;CBCF;110D 1172 11BE;CBCF;110D 1172 11BE; # (쯏; 쯏; 쯏; 쯏; 쯏; ) HANGUL SYLLABLE JJYUC +CBD0;CBD0;110D 1172 11BF;CBD0;110D 1172 11BF; # (쯐; 쯐; 쯐; 쯐; 쯐; ) HANGUL SYLLABLE JJYUK +CBD1;CBD1;110D 1172 11C0;CBD1;110D 1172 11C0; # (쯑; 쯑; 쯑; 쯑; 쯑; ) HANGUL SYLLABLE JJYUT +CBD2;CBD2;110D 1172 11C1;CBD2;110D 1172 11C1; # (쯒; 쯒; 쯒; 쯒; 쯒; ) HANGUL SYLLABLE JJYUP +CBD3;CBD3;110D 1172 11C2;CBD3;110D 1172 11C2; # (쯓; 쯓; 쯓; 쯓; 쯓; ) HANGUL SYLLABLE JJYUH +CBD4;CBD4;110D 1173;CBD4;110D 1173; # (쯔; 쯔; 쯔; 쯔; 쯔; ) HANGUL SYLLABLE JJEU +CBD5;CBD5;110D 1173 11A8;CBD5;110D 1173 11A8; # (쯕; 쯕; 쯕; 쯕; 쯕; ) HANGUL SYLLABLE JJEUG +CBD6;CBD6;110D 1173 11A9;CBD6;110D 1173 11A9; # (쯖; 쯖; 쯖; 쯖; 쯖; ) HANGUL SYLLABLE JJEUGG +CBD7;CBD7;110D 1173 11AA;CBD7;110D 1173 11AA; # (쯗; 쯗; 쯗; 쯗; 쯗; ) HANGUL SYLLABLE JJEUGS +CBD8;CBD8;110D 1173 11AB;CBD8;110D 1173 11AB; # (쯘; 쯘; 쯘; 쯘; 쯘; ) HANGUL SYLLABLE JJEUN +CBD9;CBD9;110D 1173 11AC;CBD9;110D 1173 11AC; # (쯙; 쯙; 쯙; 쯙; 쯙; ) HANGUL SYLLABLE JJEUNJ +CBDA;CBDA;110D 1173 11AD;CBDA;110D 1173 11AD; # (쯚; 쯚; 쯚; 쯚; 쯚; ) HANGUL SYLLABLE JJEUNH +CBDB;CBDB;110D 1173 11AE;CBDB;110D 1173 11AE; # (쯛; 쯛; 쯛; 쯛; 쯛; ) HANGUL SYLLABLE JJEUD +CBDC;CBDC;110D 1173 11AF;CBDC;110D 1173 11AF; # (쯜; 쯜; 쯜; 쯜; 쯜; ) HANGUL SYLLABLE JJEUL +CBDD;CBDD;110D 1173 11B0;CBDD;110D 1173 11B0; # (쯝; 쯝; 쯝; 쯝; 쯝; ) HANGUL SYLLABLE JJEULG +CBDE;CBDE;110D 1173 11B1;CBDE;110D 1173 11B1; # (쯞; 쯞; 쯞; 쯞; 쯞; ) HANGUL SYLLABLE JJEULM +CBDF;CBDF;110D 1173 11B2;CBDF;110D 1173 11B2; # (쯟; 쯟; 쯟; 쯟; 쯟; ) HANGUL SYLLABLE JJEULB +CBE0;CBE0;110D 1173 11B3;CBE0;110D 1173 11B3; # (쯠; 쯠; 쯠; 쯠; 쯠; ) HANGUL SYLLABLE JJEULS +CBE1;CBE1;110D 1173 11B4;CBE1;110D 1173 11B4; # (쯡; 쯡; 쯡; 쯡; 쯡; ) HANGUL SYLLABLE JJEULT +CBE2;CBE2;110D 1173 11B5;CBE2;110D 1173 11B5; # (쯢; 쯢; 쯢; 쯢; 쯢; ) HANGUL SYLLABLE JJEULP +CBE3;CBE3;110D 1173 11B6;CBE3;110D 1173 11B6; # (쯣; 쯣; 쯣; 쯣; 쯣; ) HANGUL SYLLABLE JJEULH +CBE4;CBE4;110D 1173 11B7;CBE4;110D 1173 11B7; # (쯤; 쯤; 쯤; 쯤; 쯤; ) HANGUL SYLLABLE JJEUM +CBE5;CBE5;110D 1173 11B8;CBE5;110D 1173 11B8; # (쯥; 쯥; 쯥; 쯥; 쯥; ) HANGUL SYLLABLE JJEUB +CBE6;CBE6;110D 1173 11B9;CBE6;110D 1173 11B9; # (쯦; 쯦; 쯦; 쯦; 쯦; ) HANGUL SYLLABLE JJEUBS +CBE7;CBE7;110D 1173 11BA;CBE7;110D 1173 11BA; # (쯧; 쯧; 쯧; 쯧; 쯧; ) HANGUL SYLLABLE JJEUS +CBE8;CBE8;110D 1173 11BB;CBE8;110D 1173 11BB; # (쯨; 쯨; 쯨; 쯨; 쯨; ) HANGUL SYLLABLE JJEUSS +CBE9;CBE9;110D 1173 11BC;CBE9;110D 1173 11BC; # (쯩; 쯩; 쯩; 쯩; 쯩; ) HANGUL SYLLABLE JJEUNG +CBEA;CBEA;110D 1173 11BD;CBEA;110D 1173 11BD; # (쯪; 쯪; 쯪; 쯪; 쯪; ) HANGUL SYLLABLE JJEUJ +CBEB;CBEB;110D 1173 11BE;CBEB;110D 1173 11BE; # (쯫; 쯫; 쯫; 쯫; 쯫; ) HANGUL SYLLABLE JJEUC +CBEC;CBEC;110D 1173 11BF;CBEC;110D 1173 11BF; # (쯬; 쯬; 쯬; 쯬; 쯬; ) HANGUL SYLLABLE JJEUK +CBED;CBED;110D 1173 11C0;CBED;110D 1173 11C0; # (쯭; 쯭; 쯭; 쯭; 쯭; ) HANGUL SYLLABLE JJEUT +CBEE;CBEE;110D 1173 11C1;CBEE;110D 1173 11C1; # (쯮; 쯮; 쯮; 쯮; 쯮; ) HANGUL SYLLABLE JJEUP +CBEF;CBEF;110D 1173 11C2;CBEF;110D 1173 11C2; # (쯯; 쯯; 쯯; 쯯; 쯯; ) HANGUL SYLLABLE JJEUH +CBF0;CBF0;110D 1174;CBF0;110D 1174; # (쯰; 쯰; 쯰; 쯰; 쯰; ) HANGUL SYLLABLE JJYI +CBF1;CBF1;110D 1174 11A8;CBF1;110D 1174 11A8; # (쯱; 쯱; 쯱; 쯱; 쯱; ) HANGUL SYLLABLE JJYIG +CBF2;CBF2;110D 1174 11A9;CBF2;110D 1174 11A9; # (쯲; 쯲; 쯲; 쯲; 쯲; ) HANGUL SYLLABLE JJYIGG +CBF3;CBF3;110D 1174 11AA;CBF3;110D 1174 11AA; # (쯳; 쯳; 쯳; 쯳; 쯳; ) HANGUL SYLLABLE JJYIGS +CBF4;CBF4;110D 1174 11AB;CBF4;110D 1174 11AB; # (쯴; 쯴; 쯴; 쯴; 쯴; ) HANGUL SYLLABLE JJYIN +CBF5;CBF5;110D 1174 11AC;CBF5;110D 1174 11AC; # (쯵; 쯵; 쯵; 쯵; 쯵; ) HANGUL SYLLABLE JJYINJ +CBF6;CBF6;110D 1174 11AD;CBF6;110D 1174 11AD; # (쯶; 쯶; 쯶; 쯶; 쯶; ) HANGUL SYLLABLE JJYINH +CBF7;CBF7;110D 1174 11AE;CBF7;110D 1174 11AE; # (쯷; 쯷; 쯷; 쯷; 쯷; ) HANGUL SYLLABLE JJYID +CBF8;CBF8;110D 1174 11AF;CBF8;110D 1174 11AF; # (쯸; 쯸; 쯸; 쯸; 쯸; ) HANGUL SYLLABLE JJYIL +CBF9;CBF9;110D 1174 11B0;CBF9;110D 1174 11B0; # (쯹; 쯹; 쯹; 쯹; 쯹; ) HANGUL SYLLABLE JJYILG +CBFA;CBFA;110D 1174 11B1;CBFA;110D 1174 11B1; # (쯺; 쯺; 쯺; 쯺; 쯺; ) HANGUL SYLLABLE JJYILM +CBFB;CBFB;110D 1174 11B2;CBFB;110D 1174 11B2; # (쯻; 쯻; 쯻; 쯻; 쯻; ) HANGUL SYLLABLE JJYILB +CBFC;CBFC;110D 1174 11B3;CBFC;110D 1174 11B3; # (쯼; 쯼; 쯼; 쯼; 쯼; ) HANGUL SYLLABLE JJYILS +CBFD;CBFD;110D 1174 11B4;CBFD;110D 1174 11B4; # (쯽; 쯽; 쯽; 쯽; 쯽; ) HANGUL SYLLABLE JJYILT +CBFE;CBFE;110D 1174 11B5;CBFE;110D 1174 11B5; # (쯾; 쯾; 쯾; 쯾; 쯾; ) HANGUL SYLLABLE JJYILP +CBFF;CBFF;110D 1174 11B6;CBFF;110D 1174 11B6; # (쯿; 쯿; 쯿; 쯿; 쯿; ) HANGUL SYLLABLE JJYILH +CC00;CC00;110D 1174 11B7;CC00;110D 1174 11B7; # (찀; 찀; 찀; 찀; 찀; ) HANGUL SYLLABLE JJYIM +CC01;CC01;110D 1174 11B8;CC01;110D 1174 11B8; # (찁; 찁; 찁; 찁; 찁; ) HANGUL SYLLABLE JJYIB +CC02;CC02;110D 1174 11B9;CC02;110D 1174 11B9; # (찂; 찂; 찂; 찂; 찂; ) HANGUL SYLLABLE JJYIBS +CC03;CC03;110D 1174 11BA;CC03;110D 1174 11BA; # (찃; 찃; 찃; 찃; 찃; ) HANGUL SYLLABLE JJYIS +CC04;CC04;110D 1174 11BB;CC04;110D 1174 11BB; # (찄; 찄; 찄; 찄; 찄; ) HANGUL SYLLABLE JJYISS +CC05;CC05;110D 1174 11BC;CC05;110D 1174 11BC; # (찅; 찅; 찅; 찅; 찅; ) HANGUL SYLLABLE JJYING +CC06;CC06;110D 1174 11BD;CC06;110D 1174 11BD; # (찆; 찆; 찆; 찆; 찆; ) HANGUL SYLLABLE JJYIJ +CC07;CC07;110D 1174 11BE;CC07;110D 1174 11BE; # (찇; 찇; 찇; 찇; 찇; ) HANGUL SYLLABLE JJYIC +CC08;CC08;110D 1174 11BF;CC08;110D 1174 11BF; # (찈; 찈; 찈; 찈; 찈; ) HANGUL SYLLABLE JJYIK +CC09;CC09;110D 1174 11C0;CC09;110D 1174 11C0; # (찉; 찉; 찉; 찉; 찉; ) HANGUL SYLLABLE JJYIT +CC0A;CC0A;110D 1174 11C1;CC0A;110D 1174 11C1; # (찊; 찊; 찊; 찊; 찊; ) HANGUL SYLLABLE JJYIP +CC0B;CC0B;110D 1174 11C2;CC0B;110D 1174 11C2; # (찋; 찋; 찋; 찋; 찋; ) HANGUL SYLLABLE JJYIH +CC0C;CC0C;110D 1175;CC0C;110D 1175; # (찌; 찌; 찌; 찌; 찌; ) HANGUL SYLLABLE JJI +CC0D;CC0D;110D 1175 11A8;CC0D;110D 1175 11A8; # (찍; 찍; 찍; 찍; 찍; ) HANGUL SYLLABLE JJIG +CC0E;CC0E;110D 1175 11A9;CC0E;110D 1175 11A9; # (찎; 찎; 찎; 찎; 찎; ) HANGUL SYLLABLE JJIGG +CC0F;CC0F;110D 1175 11AA;CC0F;110D 1175 11AA; # (찏; 찏; 찏; 찏; 찏; ) HANGUL SYLLABLE JJIGS +CC10;CC10;110D 1175 11AB;CC10;110D 1175 11AB; # (찐; 찐; 찐; 찐; 찐; ) HANGUL SYLLABLE JJIN +CC11;CC11;110D 1175 11AC;CC11;110D 1175 11AC; # (찑; 찑; 찑; 찑; 찑; ) HANGUL SYLLABLE JJINJ +CC12;CC12;110D 1175 11AD;CC12;110D 1175 11AD; # (찒; 찒; 찒; 찒; 찒; ) HANGUL SYLLABLE JJINH +CC13;CC13;110D 1175 11AE;CC13;110D 1175 11AE; # (찓; 찓; 찓; 찓; 찓; ) HANGUL SYLLABLE JJID +CC14;CC14;110D 1175 11AF;CC14;110D 1175 11AF; # (찔; 찔; 찔; 찔; 찔; ) HANGUL SYLLABLE JJIL +CC15;CC15;110D 1175 11B0;CC15;110D 1175 11B0; # (찕; 찕; 찕; 찕; 찕; ) HANGUL SYLLABLE JJILG +CC16;CC16;110D 1175 11B1;CC16;110D 1175 11B1; # (찖; 찖; 찖; 찖; 찖; ) HANGUL SYLLABLE JJILM +CC17;CC17;110D 1175 11B2;CC17;110D 1175 11B2; # (찗; 찗; 찗; 찗; 찗; ) HANGUL SYLLABLE JJILB +CC18;CC18;110D 1175 11B3;CC18;110D 1175 11B3; # (찘; 찘; 찘; 찘; 찘; ) HANGUL SYLLABLE JJILS +CC19;CC19;110D 1175 11B4;CC19;110D 1175 11B4; # (찙; 찙; 찙; 찙; 찙; ) HANGUL SYLLABLE JJILT +CC1A;CC1A;110D 1175 11B5;CC1A;110D 1175 11B5; # (찚; 찚; 찚; 찚; 찚; ) HANGUL SYLLABLE JJILP +CC1B;CC1B;110D 1175 11B6;CC1B;110D 1175 11B6; # (찛; 찛; 찛; 찛; 찛; ) HANGUL SYLLABLE JJILH +CC1C;CC1C;110D 1175 11B7;CC1C;110D 1175 11B7; # (찜; 찜; 찜; 찜; 찜; ) HANGUL SYLLABLE JJIM +CC1D;CC1D;110D 1175 11B8;CC1D;110D 1175 11B8; # (찝; 찝; 찝; 찝; 찝; ) HANGUL SYLLABLE JJIB +CC1E;CC1E;110D 1175 11B9;CC1E;110D 1175 11B9; # (찞; 찞; 찞; 찞; 찞; ) HANGUL SYLLABLE JJIBS +CC1F;CC1F;110D 1175 11BA;CC1F;110D 1175 11BA; # (찟; 찟; 찟; 찟; 찟; ) HANGUL SYLLABLE JJIS +CC20;CC20;110D 1175 11BB;CC20;110D 1175 11BB; # (찠; 찠; 찠; 찠; 찠; ) HANGUL SYLLABLE JJISS +CC21;CC21;110D 1175 11BC;CC21;110D 1175 11BC; # (찡; 찡; 찡; 찡; 찡; ) HANGUL SYLLABLE JJING +CC22;CC22;110D 1175 11BD;CC22;110D 1175 11BD; # (찢; 찢; 찢; 찢; 찢; ) HANGUL SYLLABLE JJIJ +CC23;CC23;110D 1175 11BE;CC23;110D 1175 11BE; # (찣; 찣; 찣; 찣; 찣; ) HANGUL SYLLABLE JJIC +CC24;CC24;110D 1175 11BF;CC24;110D 1175 11BF; # (찤; 찤; 찤; 찤; 찤; ) HANGUL SYLLABLE JJIK +CC25;CC25;110D 1175 11C0;CC25;110D 1175 11C0; # (찥; 찥; 찥; 찥; 찥; ) HANGUL SYLLABLE JJIT +CC26;CC26;110D 1175 11C1;CC26;110D 1175 11C1; # (찦; 찦; 찦; 찦; 찦; ) HANGUL SYLLABLE JJIP +CC27;CC27;110D 1175 11C2;CC27;110D 1175 11C2; # (찧; 찧; 찧; 찧; 찧; ) HANGUL SYLLABLE JJIH +CC28;CC28;110E 1161;CC28;110E 1161; # (차; 차; 차; 차; 차; ) HANGUL SYLLABLE CA +CC29;CC29;110E 1161 11A8;CC29;110E 1161 11A8; # (착; 착; 착; 착; 착; ) HANGUL SYLLABLE CAG +CC2A;CC2A;110E 1161 11A9;CC2A;110E 1161 11A9; # (찪; 찪; 찪; 찪; 찪; ) HANGUL SYLLABLE CAGG +CC2B;CC2B;110E 1161 11AA;CC2B;110E 1161 11AA; # (찫; 찫; 찫; 찫; 찫; ) HANGUL SYLLABLE CAGS +CC2C;CC2C;110E 1161 11AB;CC2C;110E 1161 11AB; # (찬; 찬; 찬; 찬; 찬; ) HANGUL SYLLABLE CAN +CC2D;CC2D;110E 1161 11AC;CC2D;110E 1161 11AC; # (찭; 찭; 찭; 찭; 찭; ) HANGUL SYLLABLE CANJ +CC2E;CC2E;110E 1161 11AD;CC2E;110E 1161 11AD; # (찮; 찮; 찮; 찮; 찮; ) HANGUL SYLLABLE CANH +CC2F;CC2F;110E 1161 11AE;CC2F;110E 1161 11AE; # (찯; 찯; 찯; 찯; 찯; ) HANGUL SYLLABLE CAD +CC30;CC30;110E 1161 11AF;CC30;110E 1161 11AF; # (찰; 찰; 찰; 찰; 찰; ) HANGUL SYLLABLE CAL +CC31;CC31;110E 1161 11B0;CC31;110E 1161 11B0; # (찱; 찱; 찱; 찱; 찱; ) HANGUL SYLLABLE CALG +CC32;CC32;110E 1161 11B1;CC32;110E 1161 11B1; # (찲; 찲; 찲; 찲; 찲; ) HANGUL SYLLABLE CALM +CC33;CC33;110E 1161 11B2;CC33;110E 1161 11B2; # (찳; 찳; 찳; 찳; 찳; ) HANGUL SYLLABLE CALB +CC34;CC34;110E 1161 11B3;CC34;110E 1161 11B3; # (찴; 찴; 찴; 찴; 찴; ) HANGUL SYLLABLE CALS +CC35;CC35;110E 1161 11B4;CC35;110E 1161 11B4; # (찵; 찵; 찵; 찵; 찵; ) HANGUL SYLLABLE CALT +CC36;CC36;110E 1161 11B5;CC36;110E 1161 11B5; # (찶; 찶; 찶; 찶; 찶; ) HANGUL SYLLABLE CALP +CC37;CC37;110E 1161 11B6;CC37;110E 1161 11B6; # (찷; 찷; 찷; 찷; 찷; ) HANGUL SYLLABLE CALH +CC38;CC38;110E 1161 11B7;CC38;110E 1161 11B7; # (참; 참; 참; 참; 참; ) HANGUL SYLLABLE CAM +CC39;CC39;110E 1161 11B8;CC39;110E 1161 11B8; # (찹; 찹; 찹; 찹; 찹; ) HANGUL SYLLABLE CAB +CC3A;CC3A;110E 1161 11B9;CC3A;110E 1161 11B9; # (찺; 찺; 찺; 찺; 찺; ) HANGUL SYLLABLE CABS +CC3B;CC3B;110E 1161 11BA;CC3B;110E 1161 11BA; # (찻; 찻; 찻; 찻; 찻; ) HANGUL SYLLABLE CAS +CC3C;CC3C;110E 1161 11BB;CC3C;110E 1161 11BB; # (찼; 찼; 찼; 찼; 찼; ) HANGUL SYLLABLE CASS +CC3D;CC3D;110E 1161 11BC;CC3D;110E 1161 11BC; # (창; 창; 창; 창; 창; ) HANGUL SYLLABLE CANG +CC3E;CC3E;110E 1161 11BD;CC3E;110E 1161 11BD; # (찾; 찾; 찾; 찾; 찾; ) HANGUL SYLLABLE CAJ +CC3F;CC3F;110E 1161 11BE;CC3F;110E 1161 11BE; # (찿; 찿; 찿; 찿; 찿; ) HANGUL SYLLABLE CAC +CC40;CC40;110E 1161 11BF;CC40;110E 1161 11BF; # (챀; 챀; 챀; 챀; 챀; ) HANGUL SYLLABLE CAK +CC41;CC41;110E 1161 11C0;CC41;110E 1161 11C0; # (챁; 챁; 챁; 챁; 챁; ) HANGUL SYLLABLE CAT +CC42;CC42;110E 1161 11C1;CC42;110E 1161 11C1; # (챂; 챂; 챂; 챂; 챂; ) HANGUL SYLLABLE CAP +CC43;CC43;110E 1161 11C2;CC43;110E 1161 11C2; # (챃; 챃; 챃; 챃; 챃; ) HANGUL SYLLABLE CAH +CC44;CC44;110E 1162;CC44;110E 1162; # (채; 채; 채; 채; 채; ) HANGUL SYLLABLE CAE +CC45;CC45;110E 1162 11A8;CC45;110E 1162 11A8; # (책; 책; 책; 책; 책; ) HANGUL SYLLABLE CAEG +CC46;CC46;110E 1162 11A9;CC46;110E 1162 11A9; # (챆; 챆; 챆; 챆; 챆; ) HANGUL SYLLABLE CAEGG +CC47;CC47;110E 1162 11AA;CC47;110E 1162 11AA; # (챇; 챇; 챇; 챇; 챇; ) HANGUL SYLLABLE CAEGS +CC48;CC48;110E 1162 11AB;CC48;110E 1162 11AB; # (챈; 챈; 챈; 챈; 챈; ) HANGUL SYLLABLE CAEN +CC49;CC49;110E 1162 11AC;CC49;110E 1162 11AC; # (챉; 챉; 챉; 챉; 챉; ) HANGUL SYLLABLE CAENJ +CC4A;CC4A;110E 1162 11AD;CC4A;110E 1162 11AD; # (챊; 챊; 챊; 챊; 챊; ) HANGUL SYLLABLE CAENH +CC4B;CC4B;110E 1162 11AE;CC4B;110E 1162 11AE; # (챋; 챋; 챋; 챋; 챋; ) HANGUL SYLLABLE CAED +CC4C;CC4C;110E 1162 11AF;CC4C;110E 1162 11AF; # (챌; 챌; 챌; 챌; 챌; ) HANGUL SYLLABLE CAEL +CC4D;CC4D;110E 1162 11B0;CC4D;110E 1162 11B0; # (챍; 챍; 챍; 챍; 챍; ) HANGUL SYLLABLE CAELG +CC4E;CC4E;110E 1162 11B1;CC4E;110E 1162 11B1; # (챎; 챎; 챎; 챎; 챎; ) HANGUL SYLLABLE CAELM +CC4F;CC4F;110E 1162 11B2;CC4F;110E 1162 11B2; # (챏; 챏; 챏; 챏; 챏; ) HANGUL SYLLABLE CAELB +CC50;CC50;110E 1162 11B3;CC50;110E 1162 11B3; # (챐; 챐; 챐; 챐; 챐; ) HANGUL SYLLABLE CAELS +CC51;CC51;110E 1162 11B4;CC51;110E 1162 11B4; # (챑; 챑; 챑; 챑; 챑; ) HANGUL SYLLABLE CAELT +CC52;CC52;110E 1162 11B5;CC52;110E 1162 11B5; # (챒; 챒; 챒; 챒; 챒; ) HANGUL SYLLABLE CAELP +CC53;CC53;110E 1162 11B6;CC53;110E 1162 11B6; # (챓; 챓; 챓; 챓; 챓; ) HANGUL SYLLABLE CAELH +CC54;CC54;110E 1162 11B7;CC54;110E 1162 11B7; # (챔; 챔; 챔; 챔; 챔; ) HANGUL SYLLABLE CAEM +CC55;CC55;110E 1162 11B8;CC55;110E 1162 11B8; # (챕; 챕; 챕; 챕; 챕; ) HANGUL SYLLABLE CAEB +CC56;CC56;110E 1162 11B9;CC56;110E 1162 11B9; # (챖; 챖; 챖; 챖; 챖; ) HANGUL SYLLABLE CAEBS +CC57;CC57;110E 1162 11BA;CC57;110E 1162 11BA; # (챗; 챗; 챗; 챗; 챗; ) HANGUL SYLLABLE CAES +CC58;CC58;110E 1162 11BB;CC58;110E 1162 11BB; # (챘; 챘; 챘; 챘; 챘; ) HANGUL SYLLABLE CAESS +CC59;CC59;110E 1162 11BC;CC59;110E 1162 11BC; # (챙; 챙; 챙; 챙; 챙; ) HANGUL SYLLABLE CAENG +CC5A;CC5A;110E 1162 11BD;CC5A;110E 1162 11BD; # (챚; 챚; 챚; 챚; 챚; ) HANGUL SYLLABLE CAEJ +CC5B;CC5B;110E 1162 11BE;CC5B;110E 1162 11BE; # (챛; 챛; 챛; 챛; 챛; ) HANGUL SYLLABLE CAEC +CC5C;CC5C;110E 1162 11BF;CC5C;110E 1162 11BF; # (챜; 챜; 챜; 챜; 챜; ) HANGUL SYLLABLE CAEK +CC5D;CC5D;110E 1162 11C0;CC5D;110E 1162 11C0; # (챝; 챝; 챝; 챝; 챝; ) HANGUL SYLLABLE CAET +CC5E;CC5E;110E 1162 11C1;CC5E;110E 1162 11C1; # (챞; 챞; 챞; 챞; 챞; ) HANGUL SYLLABLE CAEP +CC5F;CC5F;110E 1162 11C2;CC5F;110E 1162 11C2; # (챟; 챟; 챟; 챟; 챟; ) HANGUL SYLLABLE CAEH +CC60;CC60;110E 1163;CC60;110E 1163; # (챠; 챠; 챠; 챠; 챠; ) HANGUL SYLLABLE CYA +CC61;CC61;110E 1163 11A8;CC61;110E 1163 11A8; # (챡; 챡; 챡; 챡; 챡; ) HANGUL SYLLABLE CYAG +CC62;CC62;110E 1163 11A9;CC62;110E 1163 11A9; # (챢; 챢; 챢; 챢; 챢; ) HANGUL SYLLABLE CYAGG +CC63;CC63;110E 1163 11AA;CC63;110E 1163 11AA; # (챣; 챣; 챣; 챣; 챣; ) HANGUL SYLLABLE CYAGS +CC64;CC64;110E 1163 11AB;CC64;110E 1163 11AB; # (챤; 챤; 챤; 챤; 챤; ) HANGUL SYLLABLE CYAN +CC65;CC65;110E 1163 11AC;CC65;110E 1163 11AC; # (챥; 챥; 챥; 챥; 챥; ) HANGUL SYLLABLE CYANJ +CC66;CC66;110E 1163 11AD;CC66;110E 1163 11AD; # (챦; 챦; 챦; 챦; 챦; ) HANGUL SYLLABLE CYANH +CC67;CC67;110E 1163 11AE;CC67;110E 1163 11AE; # (챧; 챧; 챧; 챧; 챧; ) HANGUL SYLLABLE CYAD +CC68;CC68;110E 1163 11AF;CC68;110E 1163 11AF; # (챨; 챨; 챨; 챨; 챨; ) HANGUL SYLLABLE CYAL +CC69;CC69;110E 1163 11B0;CC69;110E 1163 11B0; # (챩; 챩; 챩; 챩; 챩; ) HANGUL SYLLABLE CYALG +CC6A;CC6A;110E 1163 11B1;CC6A;110E 1163 11B1; # (챪; 챪; 챪; 챪; 챪; ) HANGUL SYLLABLE CYALM +CC6B;CC6B;110E 1163 11B2;CC6B;110E 1163 11B2; # (챫; 챫; 챫; 챫; 챫; ) HANGUL SYLLABLE CYALB +CC6C;CC6C;110E 1163 11B3;CC6C;110E 1163 11B3; # (챬; 챬; 챬; 챬; 챬; ) HANGUL SYLLABLE CYALS +CC6D;CC6D;110E 1163 11B4;CC6D;110E 1163 11B4; # (챭; 챭; 챭; 챭; 챭; ) HANGUL SYLLABLE CYALT +CC6E;CC6E;110E 1163 11B5;CC6E;110E 1163 11B5; # (챮; 챮; 챮; 챮; 챮; ) HANGUL SYLLABLE CYALP +CC6F;CC6F;110E 1163 11B6;CC6F;110E 1163 11B6; # (챯; 챯; 챯; 챯; 챯; ) HANGUL SYLLABLE CYALH +CC70;CC70;110E 1163 11B7;CC70;110E 1163 11B7; # (챰; 챰; 챰; 챰; 챰; ) HANGUL SYLLABLE CYAM +CC71;CC71;110E 1163 11B8;CC71;110E 1163 11B8; # (챱; 챱; 챱; 챱; 챱; ) HANGUL SYLLABLE CYAB +CC72;CC72;110E 1163 11B9;CC72;110E 1163 11B9; # (챲; 챲; 챲; 챲; 챲; ) HANGUL SYLLABLE CYABS +CC73;CC73;110E 1163 11BA;CC73;110E 1163 11BA; # (챳; 챳; 챳; 챳; 챳; ) HANGUL SYLLABLE CYAS +CC74;CC74;110E 1163 11BB;CC74;110E 1163 11BB; # (챴; 챴; 챴; 챴; 챴; ) HANGUL SYLLABLE CYASS +CC75;CC75;110E 1163 11BC;CC75;110E 1163 11BC; # (챵; 챵; 챵; 챵; 챵; ) HANGUL SYLLABLE CYANG +CC76;CC76;110E 1163 11BD;CC76;110E 1163 11BD; # (챶; 챶; 챶; 챶; 챶; ) HANGUL SYLLABLE CYAJ +CC77;CC77;110E 1163 11BE;CC77;110E 1163 11BE; # (챷; 챷; 챷; 챷; 챷; ) HANGUL SYLLABLE CYAC +CC78;CC78;110E 1163 11BF;CC78;110E 1163 11BF; # (챸; 챸; 챸; 챸; 챸; ) HANGUL SYLLABLE CYAK +CC79;CC79;110E 1163 11C0;CC79;110E 1163 11C0; # (챹; 챹; 챹; 챹; 챹; ) HANGUL SYLLABLE CYAT +CC7A;CC7A;110E 1163 11C1;CC7A;110E 1163 11C1; # (챺; 챺; 챺; 챺; 챺; ) HANGUL SYLLABLE CYAP +CC7B;CC7B;110E 1163 11C2;CC7B;110E 1163 11C2; # (챻; 챻; 챻; 챻; 챻; ) HANGUL SYLLABLE CYAH +CC7C;CC7C;110E 1164;CC7C;110E 1164; # (챼; 챼; 챼; 챼; 챼; ) HANGUL SYLLABLE CYAE +CC7D;CC7D;110E 1164 11A8;CC7D;110E 1164 11A8; # (챽; 챽; 챽; 챽; 챽; ) HANGUL SYLLABLE CYAEG +CC7E;CC7E;110E 1164 11A9;CC7E;110E 1164 11A9; # (챾; 챾; 챾; 챾; 챾; ) HANGUL SYLLABLE CYAEGG +CC7F;CC7F;110E 1164 11AA;CC7F;110E 1164 11AA; # (챿; 챿; 챿; 챿; 챿; ) HANGUL SYLLABLE CYAEGS +CC80;CC80;110E 1164 11AB;CC80;110E 1164 11AB; # (첀; 첀; 첀; 첀; 첀; ) HANGUL SYLLABLE CYAEN +CC81;CC81;110E 1164 11AC;CC81;110E 1164 11AC; # (첁; 첁; 첁; 첁; 첁; ) HANGUL SYLLABLE CYAENJ +CC82;CC82;110E 1164 11AD;CC82;110E 1164 11AD; # (첂; 첂; 첂; 첂; 첂; ) HANGUL SYLLABLE CYAENH +CC83;CC83;110E 1164 11AE;CC83;110E 1164 11AE; # (첃; 첃; 첃; 첃; 첃; ) HANGUL SYLLABLE CYAED +CC84;CC84;110E 1164 11AF;CC84;110E 1164 11AF; # (첄; 첄; 첄; 첄; 첄; ) HANGUL SYLLABLE CYAEL +CC85;CC85;110E 1164 11B0;CC85;110E 1164 11B0; # (첅; 첅; 첅; 첅; 첅; ) HANGUL SYLLABLE CYAELG +CC86;CC86;110E 1164 11B1;CC86;110E 1164 11B1; # (첆; 첆; 첆; 첆; 첆; ) HANGUL SYLLABLE CYAELM +CC87;CC87;110E 1164 11B2;CC87;110E 1164 11B2; # (첇; 첇; 첇; 첇; 첇; ) HANGUL SYLLABLE CYAELB +CC88;CC88;110E 1164 11B3;CC88;110E 1164 11B3; # (첈; 첈; 첈; 첈; 첈; ) HANGUL SYLLABLE CYAELS +CC89;CC89;110E 1164 11B4;CC89;110E 1164 11B4; # (첉; 첉; 첉; 첉; 첉; ) HANGUL SYLLABLE CYAELT +CC8A;CC8A;110E 1164 11B5;CC8A;110E 1164 11B5; # (첊; 첊; 첊; 첊; 첊; ) HANGUL SYLLABLE CYAELP +CC8B;CC8B;110E 1164 11B6;CC8B;110E 1164 11B6; # (첋; 첋; 첋; 첋; 첋; ) HANGUL SYLLABLE CYAELH +CC8C;CC8C;110E 1164 11B7;CC8C;110E 1164 11B7; # (첌; 첌; 첌; 첌; 첌; ) HANGUL SYLLABLE CYAEM +CC8D;CC8D;110E 1164 11B8;CC8D;110E 1164 11B8; # (첍; 첍; 첍; 첍; 첍; ) HANGUL SYLLABLE CYAEB +CC8E;CC8E;110E 1164 11B9;CC8E;110E 1164 11B9; # (첎; 첎; 첎; 첎; 첎; ) HANGUL SYLLABLE CYAEBS +CC8F;CC8F;110E 1164 11BA;CC8F;110E 1164 11BA; # (첏; 첏; 첏; 첏; 첏; ) HANGUL SYLLABLE CYAES +CC90;CC90;110E 1164 11BB;CC90;110E 1164 11BB; # (첐; 첐; 첐; 첐; 첐; ) HANGUL SYLLABLE CYAESS +CC91;CC91;110E 1164 11BC;CC91;110E 1164 11BC; # (첑; 첑; 첑; 첑; 첑; ) HANGUL SYLLABLE CYAENG +CC92;CC92;110E 1164 11BD;CC92;110E 1164 11BD; # (첒; 첒; 첒; 첒; 첒; ) HANGUL SYLLABLE CYAEJ +CC93;CC93;110E 1164 11BE;CC93;110E 1164 11BE; # (첓; 첓; 첓; 첓; 첓; ) HANGUL SYLLABLE CYAEC +CC94;CC94;110E 1164 11BF;CC94;110E 1164 11BF; # (첔; 첔; 첔; 첔; 첔; ) HANGUL SYLLABLE CYAEK +CC95;CC95;110E 1164 11C0;CC95;110E 1164 11C0; # (첕; 첕; 첕; 첕; 첕; ) HANGUL SYLLABLE CYAET +CC96;CC96;110E 1164 11C1;CC96;110E 1164 11C1; # (첖; 첖; 첖; 첖; 첖; ) HANGUL SYLLABLE CYAEP +CC97;CC97;110E 1164 11C2;CC97;110E 1164 11C2; # (첗; 첗; 첗; 첗; 첗; ) HANGUL SYLLABLE CYAEH +CC98;CC98;110E 1165;CC98;110E 1165; # (처; 처; 처; 처; 처; ) HANGUL SYLLABLE CEO +CC99;CC99;110E 1165 11A8;CC99;110E 1165 11A8; # (척; 척; 척; 척; 척; ) HANGUL SYLLABLE CEOG +CC9A;CC9A;110E 1165 11A9;CC9A;110E 1165 11A9; # (첚; 첚; 첚; 첚; 첚; ) HANGUL SYLLABLE CEOGG +CC9B;CC9B;110E 1165 11AA;CC9B;110E 1165 11AA; # (첛; 첛; 첛; 첛; 첛; ) HANGUL SYLLABLE CEOGS +CC9C;CC9C;110E 1165 11AB;CC9C;110E 1165 11AB; # (천; 천; 천; 천; 천; ) HANGUL SYLLABLE CEON +CC9D;CC9D;110E 1165 11AC;CC9D;110E 1165 11AC; # (첝; 첝; 첝; 첝; 첝; ) HANGUL SYLLABLE CEONJ +CC9E;CC9E;110E 1165 11AD;CC9E;110E 1165 11AD; # (첞; 첞; 첞; 첞; 첞; ) HANGUL SYLLABLE CEONH +CC9F;CC9F;110E 1165 11AE;CC9F;110E 1165 11AE; # (첟; 첟; 첟; 첟; 첟; ) HANGUL SYLLABLE CEOD +CCA0;CCA0;110E 1165 11AF;CCA0;110E 1165 11AF; # (철; 철; 철; 철; 철; ) HANGUL SYLLABLE CEOL +CCA1;CCA1;110E 1165 11B0;CCA1;110E 1165 11B0; # (첡; 첡; 첡; 첡; 첡; ) HANGUL SYLLABLE CEOLG +CCA2;CCA2;110E 1165 11B1;CCA2;110E 1165 11B1; # (첢; 첢; 첢; 첢; 첢; ) HANGUL SYLLABLE CEOLM +CCA3;CCA3;110E 1165 11B2;CCA3;110E 1165 11B2; # (첣; 첣; 첣; 첣; 첣; ) HANGUL SYLLABLE CEOLB +CCA4;CCA4;110E 1165 11B3;CCA4;110E 1165 11B3; # (첤; 첤; 첤; 첤; 첤; ) HANGUL SYLLABLE CEOLS +CCA5;CCA5;110E 1165 11B4;CCA5;110E 1165 11B4; # (첥; 첥; 첥; 첥; 첥; ) HANGUL SYLLABLE CEOLT +CCA6;CCA6;110E 1165 11B5;CCA6;110E 1165 11B5; # (첦; 첦; 첦; 첦; 첦; ) HANGUL SYLLABLE CEOLP +CCA7;CCA7;110E 1165 11B6;CCA7;110E 1165 11B6; # (첧; 첧; 첧; 첧; 첧; ) HANGUL SYLLABLE CEOLH +CCA8;CCA8;110E 1165 11B7;CCA8;110E 1165 11B7; # (첨; 첨; 첨; 첨; 첨; ) HANGUL SYLLABLE CEOM +CCA9;CCA9;110E 1165 11B8;CCA9;110E 1165 11B8; # (첩; 첩; 첩; 첩; 첩; ) HANGUL SYLLABLE CEOB +CCAA;CCAA;110E 1165 11B9;CCAA;110E 1165 11B9; # (첪; 첪; 첪; 첪; 첪; ) HANGUL SYLLABLE CEOBS +CCAB;CCAB;110E 1165 11BA;CCAB;110E 1165 11BA; # (첫; 첫; 첫; 첫; 첫; ) HANGUL SYLLABLE CEOS +CCAC;CCAC;110E 1165 11BB;CCAC;110E 1165 11BB; # (첬; 첬; 첬; 첬; 첬; ) HANGUL SYLLABLE CEOSS +CCAD;CCAD;110E 1165 11BC;CCAD;110E 1165 11BC; # (청; 청; 청; 청; 청; ) HANGUL SYLLABLE CEONG +CCAE;CCAE;110E 1165 11BD;CCAE;110E 1165 11BD; # (첮; 첮; 첮; 첮; 첮; ) HANGUL SYLLABLE CEOJ +CCAF;CCAF;110E 1165 11BE;CCAF;110E 1165 11BE; # (첯; 첯; 첯; 첯; 첯; ) HANGUL SYLLABLE CEOC +CCB0;CCB0;110E 1165 11BF;CCB0;110E 1165 11BF; # (첰; 첰; 첰; 첰; 첰; ) HANGUL SYLLABLE CEOK +CCB1;CCB1;110E 1165 11C0;CCB1;110E 1165 11C0; # (첱; 첱; 첱; 첱; 첱; ) HANGUL SYLLABLE CEOT +CCB2;CCB2;110E 1165 11C1;CCB2;110E 1165 11C1; # (첲; 첲; 첲; 첲; 첲; ) HANGUL SYLLABLE CEOP +CCB3;CCB3;110E 1165 11C2;CCB3;110E 1165 11C2; # (첳; 첳; 첳; 첳; 첳; ) HANGUL SYLLABLE CEOH +CCB4;CCB4;110E 1166;CCB4;110E 1166; # (체; 체; 체; 체; 체; ) HANGUL SYLLABLE CE +CCB5;CCB5;110E 1166 11A8;CCB5;110E 1166 11A8; # (첵; 첵; 첵; 첵; 첵; ) HANGUL SYLLABLE CEG +CCB6;CCB6;110E 1166 11A9;CCB6;110E 1166 11A9; # (첶; 첶; 첶; 첶; 첶; ) HANGUL SYLLABLE CEGG +CCB7;CCB7;110E 1166 11AA;CCB7;110E 1166 11AA; # (첷; 첷; 첷; 첷; 첷; ) HANGUL SYLLABLE CEGS +CCB8;CCB8;110E 1166 11AB;CCB8;110E 1166 11AB; # (첸; 첸; 첸; 첸; 첸; ) HANGUL SYLLABLE CEN +CCB9;CCB9;110E 1166 11AC;CCB9;110E 1166 11AC; # (첹; 첹; 첹; 첹; 첹; ) HANGUL SYLLABLE CENJ +CCBA;CCBA;110E 1166 11AD;CCBA;110E 1166 11AD; # (첺; 첺; 첺; 첺; 첺; ) HANGUL SYLLABLE CENH +CCBB;CCBB;110E 1166 11AE;CCBB;110E 1166 11AE; # (첻; 첻; 첻; 첻; 첻; ) HANGUL SYLLABLE CED +CCBC;CCBC;110E 1166 11AF;CCBC;110E 1166 11AF; # (첼; 첼; 첼; 첼; 첼; ) HANGUL SYLLABLE CEL +CCBD;CCBD;110E 1166 11B0;CCBD;110E 1166 11B0; # (첽; 첽; 첽; 첽; 첽; ) HANGUL SYLLABLE CELG +CCBE;CCBE;110E 1166 11B1;CCBE;110E 1166 11B1; # (첾; 첾; 첾; 첾; 첾; ) HANGUL SYLLABLE CELM +CCBF;CCBF;110E 1166 11B2;CCBF;110E 1166 11B2; # (첿; 첿; 첿; 첿; 첿; ) HANGUL SYLLABLE CELB +CCC0;CCC0;110E 1166 11B3;CCC0;110E 1166 11B3; # (쳀; 쳀; 쳀; 쳀; 쳀; ) HANGUL SYLLABLE CELS +CCC1;CCC1;110E 1166 11B4;CCC1;110E 1166 11B4; # (쳁; 쳁; 쳁; 쳁; 쳁; ) HANGUL SYLLABLE CELT +CCC2;CCC2;110E 1166 11B5;CCC2;110E 1166 11B5; # (쳂; 쳂; 쳂; 쳂; 쳂; ) HANGUL SYLLABLE CELP +CCC3;CCC3;110E 1166 11B6;CCC3;110E 1166 11B6; # (쳃; 쳃; 쳃; 쳃; 쳃; ) HANGUL SYLLABLE CELH +CCC4;CCC4;110E 1166 11B7;CCC4;110E 1166 11B7; # (쳄; 쳄; 쳄; 쳄; 쳄; ) HANGUL SYLLABLE CEM +CCC5;CCC5;110E 1166 11B8;CCC5;110E 1166 11B8; # (쳅; 쳅; 쳅; 쳅; 쳅; ) HANGUL SYLLABLE CEB +CCC6;CCC6;110E 1166 11B9;CCC6;110E 1166 11B9; # (쳆; 쳆; 쳆; 쳆; 쳆; ) HANGUL SYLLABLE CEBS +CCC7;CCC7;110E 1166 11BA;CCC7;110E 1166 11BA; # (쳇; 쳇; 쳇; 쳇; 쳇; ) HANGUL SYLLABLE CES +CCC8;CCC8;110E 1166 11BB;CCC8;110E 1166 11BB; # (쳈; 쳈; 쳈; 쳈; 쳈; ) HANGUL SYLLABLE CESS +CCC9;CCC9;110E 1166 11BC;CCC9;110E 1166 11BC; # (쳉; 쳉; 쳉; 쳉; 쳉; ) HANGUL SYLLABLE CENG +CCCA;CCCA;110E 1166 11BD;CCCA;110E 1166 11BD; # (쳊; 쳊; 쳊; 쳊; 쳊; ) HANGUL SYLLABLE CEJ +CCCB;CCCB;110E 1166 11BE;CCCB;110E 1166 11BE; # (쳋; 쳋; 쳋; 쳋; 쳋; ) HANGUL SYLLABLE CEC +CCCC;CCCC;110E 1166 11BF;CCCC;110E 1166 11BF; # (쳌; 쳌; 쳌; 쳌; 쳌; ) HANGUL SYLLABLE CEK +CCCD;CCCD;110E 1166 11C0;CCCD;110E 1166 11C0; # (쳍; 쳍; 쳍; 쳍; 쳍; ) HANGUL SYLLABLE CET +CCCE;CCCE;110E 1166 11C1;CCCE;110E 1166 11C1; # (쳎; 쳎; 쳎; 쳎; 쳎; ) HANGUL SYLLABLE CEP +CCCF;CCCF;110E 1166 11C2;CCCF;110E 1166 11C2; # (쳏; 쳏; 쳏; 쳏; 쳏; ) HANGUL SYLLABLE CEH +CCD0;CCD0;110E 1167;CCD0;110E 1167; # (쳐; 쳐; 쳐; 쳐; 쳐; ) HANGUL SYLLABLE CYEO +CCD1;CCD1;110E 1167 11A8;CCD1;110E 1167 11A8; # (쳑; 쳑; 쳑; 쳑; 쳑; ) HANGUL SYLLABLE CYEOG +CCD2;CCD2;110E 1167 11A9;CCD2;110E 1167 11A9; # (쳒; 쳒; 쳒; 쳒; 쳒; ) HANGUL SYLLABLE CYEOGG +CCD3;CCD3;110E 1167 11AA;CCD3;110E 1167 11AA; # (쳓; 쳓; 쳓; 쳓; 쳓; ) HANGUL SYLLABLE CYEOGS +CCD4;CCD4;110E 1167 11AB;CCD4;110E 1167 11AB; # (쳔; 쳔; 쳔; 쳔; 쳔; ) HANGUL SYLLABLE CYEON +CCD5;CCD5;110E 1167 11AC;CCD5;110E 1167 11AC; # (쳕; 쳕; 쳕; 쳕; 쳕; ) HANGUL SYLLABLE CYEONJ +CCD6;CCD6;110E 1167 11AD;CCD6;110E 1167 11AD; # (쳖; 쳖; 쳖; 쳖; 쳖; ) HANGUL SYLLABLE CYEONH +CCD7;CCD7;110E 1167 11AE;CCD7;110E 1167 11AE; # (쳗; 쳗; 쳗; 쳗; 쳗; ) HANGUL SYLLABLE CYEOD +CCD8;CCD8;110E 1167 11AF;CCD8;110E 1167 11AF; # (쳘; 쳘; 쳘; 쳘; 쳘; ) HANGUL SYLLABLE CYEOL +CCD9;CCD9;110E 1167 11B0;CCD9;110E 1167 11B0; # (쳙; 쳙; 쳙; 쳙; 쳙; ) HANGUL SYLLABLE CYEOLG +CCDA;CCDA;110E 1167 11B1;CCDA;110E 1167 11B1; # (쳚; 쳚; 쳚; 쳚; 쳚; ) HANGUL SYLLABLE CYEOLM +CCDB;CCDB;110E 1167 11B2;CCDB;110E 1167 11B2; # (쳛; 쳛; 쳛; 쳛; 쳛; ) HANGUL SYLLABLE CYEOLB +CCDC;CCDC;110E 1167 11B3;CCDC;110E 1167 11B3; # (쳜; 쳜; 쳜; 쳜; 쳜; ) HANGUL SYLLABLE CYEOLS +CCDD;CCDD;110E 1167 11B4;CCDD;110E 1167 11B4; # (쳝; 쳝; 쳝; 쳝; 쳝; ) HANGUL SYLLABLE CYEOLT +CCDE;CCDE;110E 1167 11B5;CCDE;110E 1167 11B5; # (쳞; 쳞; 쳞; 쳞; 쳞; ) HANGUL SYLLABLE CYEOLP +CCDF;CCDF;110E 1167 11B6;CCDF;110E 1167 11B6; # (쳟; 쳟; 쳟; 쳟; 쳟; ) HANGUL SYLLABLE CYEOLH +CCE0;CCE0;110E 1167 11B7;CCE0;110E 1167 11B7; # (쳠; 쳠; 쳠; 쳠; 쳠; ) HANGUL SYLLABLE CYEOM +CCE1;CCE1;110E 1167 11B8;CCE1;110E 1167 11B8; # (쳡; 쳡; 쳡; 쳡; 쳡; ) HANGUL SYLLABLE CYEOB +CCE2;CCE2;110E 1167 11B9;CCE2;110E 1167 11B9; # (쳢; 쳢; 쳢; 쳢; 쳢; ) HANGUL SYLLABLE CYEOBS +CCE3;CCE3;110E 1167 11BA;CCE3;110E 1167 11BA; # (쳣; 쳣; 쳣; 쳣; 쳣; ) HANGUL SYLLABLE CYEOS +CCE4;CCE4;110E 1167 11BB;CCE4;110E 1167 11BB; # (쳤; 쳤; 쳤; 쳤; 쳤; ) HANGUL SYLLABLE CYEOSS +CCE5;CCE5;110E 1167 11BC;CCE5;110E 1167 11BC; # (쳥; 쳥; 쳥; 쳥; 쳥; ) HANGUL SYLLABLE CYEONG +CCE6;CCE6;110E 1167 11BD;CCE6;110E 1167 11BD; # (쳦; 쳦; 쳦; 쳦; 쳦; ) HANGUL SYLLABLE CYEOJ +CCE7;CCE7;110E 1167 11BE;CCE7;110E 1167 11BE; # (쳧; 쳧; 쳧; 쳧; 쳧; ) HANGUL SYLLABLE CYEOC +CCE8;CCE8;110E 1167 11BF;CCE8;110E 1167 11BF; # (쳨; 쳨; 쳨; 쳨; 쳨; ) HANGUL SYLLABLE CYEOK +CCE9;CCE9;110E 1167 11C0;CCE9;110E 1167 11C0; # (쳩; 쳩; 쳩; 쳩; 쳩; ) HANGUL SYLLABLE CYEOT +CCEA;CCEA;110E 1167 11C1;CCEA;110E 1167 11C1; # (쳪; 쳪; 쳪; 쳪; 쳪; ) HANGUL SYLLABLE CYEOP +CCEB;CCEB;110E 1167 11C2;CCEB;110E 1167 11C2; # (쳫; 쳫; 쳫; 쳫; 쳫; ) HANGUL SYLLABLE CYEOH +CCEC;CCEC;110E 1168;CCEC;110E 1168; # (쳬; 쳬; 쳬; 쳬; 쳬; ) HANGUL SYLLABLE CYE +CCED;CCED;110E 1168 11A8;CCED;110E 1168 11A8; # (쳭; 쳭; 쳭; 쳭; 쳭; ) HANGUL SYLLABLE CYEG +CCEE;CCEE;110E 1168 11A9;CCEE;110E 1168 11A9; # (쳮; 쳮; 쳮; 쳮; 쳮; ) HANGUL SYLLABLE CYEGG +CCEF;CCEF;110E 1168 11AA;CCEF;110E 1168 11AA; # (쳯; 쳯; 쳯; 쳯; 쳯; ) HANGUL SYLLABLE CYEGS +CCF0;CCF0;110E 1168 11AB;CCF0;110E 1168 11AB; # (쳰; 쳰; 쳰; 쳰; 쳰; ) HANGUL SYLLABLE CYEN +CCF1;CCF1;110E 1168 11AC;CCF1;110E 1168 11AC; # (쳱; 쳱; 쳱; 쳱; 쳱; ) HANGUL SYLLABLE CYENJ +CCF2;CCF2;110E 1168 11AD;CCF2;110E 1168 11AD; # (쳲; 쳲; 쳲; 쳲; 쳲; ) HANGUL SYLLABLE CYENH +CCF3;CCF3;110E 1168 11AE;CCF3;110E 1168 11AE; # (쳳; 쳳; 쳳; 쳳; 쳳; ) HANGUL SYLLABLE CYED +CCF4;CCF4;110E 1168 11AF;CCF4;110E 1168 11AF; # (쳴; 쳴; 쳴; 쳴; 쳴; ) HANGUL SYLLABLE CYEL +CCF5;CCF5;110E 1168 11B0;CCF5;110E 1168 11B0; # (쳵; 쳵; 쳵; 쳵; 쳵; ) HANGUL SYLLABLE CYELG +CCF6;CCF6;110E 1168 11B1;CCF6;110E 1168 11B1; # (쳶; 쳶; 쳶; 쳶; 쳶; ) HANGUL SYLLABLE CYELM +CCF7;CCF7;110E 1168 11B2;CCF7;110E 1168 11B2; # (쳷; 쳷; 쳷; 쳷; 쳷; ) HANGUL SYLLABLE CYELB +CCF8;CCF8;110E 1168 11B3;CCF8;110E 1168 11B3; # (쳸; 쳸; 쳸; 쳸; 쳸; ) HANGUL SYLLABLE CYELS +CCF9;CCF9;110E 1168 11B4;CCF9;110E 1168 11B4; # (쳹; 쳹; 쳹; 쳹; 쳹; ) HANGUL SYLLABLE CYELT +CCFA;CCFA;110E 1168 11B5;CCFA;110E 1168 11B5; # (쳺; 쳺; 쳺; 쳺; 쳺; ) HANGUL SYLLABLE CYELP +CCFB;CCFB;110E 1168 11B6;CCFB;110E 1168 11B6; # (쳻; 쳻; 쳻; 쳻; 쳻; ) HANGUL SYLLABLE CYELH +CCFC;CCFC;110E 1168 11B7;CCFC;110E 1168 11B7; # (쳼; 쳼; 쳼; 쳼; 쳼; ) HANGUL SYLLABLE CYEM +CCFD;CCFD;110E 1168 11B8;CCFD;110E 1168 11B8; # (쳽; 쳽; 쳽; 쳽; 쳽; ) HANGUL SYLLABLE CYEB +CCFE;CCFE;110E 1168 11B9;CCFE;110E 1168 11B9; # (쳾; 쳾; 쳾; 쳾; 쳾; ) HANGUL SYLLABLE CYEBS +CCFF;CCFF;110E 1168 11BA;CCFF;110E 1168 11BA; # (쳿; 쳿; 쳿; 쳿; 쳿; ) HANGUL SYLLABLE CYES +CD00;CD00;110E 1168 11BB;CD00;110E 1168 11BB; # (촀; 촀; 촀; 촀; 촀; ) HANGUL SYLLABLE CYESS +CD01;CD01;110E 1168 11BC;CD01;110E 1168 11BC; # (촁; 촁; 촁; 촁; 촁; ) HANGUL SYLLABLE CYENG +CD02;CD02;110E 1168 11BD;CD02;110E 1168 11BD; # (촂; 촂; 촂; 촂; 촂; ) HANGUL SYLLABLE CYEJ +CD03;CD03;110E 1168 11BE;CD03;110E 1168 11BE; # (촃; 촃; 촃; 촃; 촃; ) HANGUL SYLLABLE CYEC +CD04;CD04;110E 1168 11BF;CD04;110E 1168 11BF; # (촄; 촄; 촄; 촄; 촄; ) HANGUL SYLLABLE CYEK +CD05;CD05;110E 1168 11C0;CD05;110E 1168 11C0; # (촅; 촅; 촅; 촅; 촅; ) HANGUL SYLLABLE CYET +CD06;CD06;110E 1168 11C1;CD06;110E 1168 11C1; # (촆; 촆; 촆; 촆; 촆; ) HANGUL SYLLABLE CYEP +CD07;CD07;110E 1168 11C2;CD07;110E 1168 11C2; # (촇; 촇; 촇; 촇; 촇; ) HANGUL SYLLABLE CYEH +CD08;CD08;110E 1169;CD08;110E 1169; # (초; 초; 초; 초; 초; ) HANGUL SYLLABLE CO +CD09;CD09;110E 1169 11A8;CD09;110E 1169 11A8; # (촉; 촉; 촉; 촉; 촉; ) HANGUL SYLLABLE COG +CD0A;CD0A;110E 1169 11A9;CD0A;110E 1169 11A9; # (촊; 촊; 촊; 촊; 촊; ) HANGUL SYLLABLE COGG +CD0B;CD0B;110E 1169 11AA;CD0B;110E 1169 11AA; # (촋; 촋; 촋; 촋; 촋; ) HANGUL SYLLABLE COGS +CD0C;CD0C;110E 1169 11AB;CD0C;110E 1169 11AB; # (촌; 촌; 촌; 촌; 촌; ) HANGUL SYLLABLE CON +CD0D;CD0D;110E 1169 11AC;CD0D;110E 1169 11AC; # (촍; 촍; 촍; 촍; 촍; ) HANGUL SYLLABLE CONJ +CD0E;CD0E;110E 1169 11AD;CD0E;110E 1169 11AD; # (촎; 촎; 촎; 촎; 촎; ) HANGUL SYLLABLE CONH +CD0F;CD0F;110E 1169 11AE;CD0F;110E 1169 11AE; # (촏; 촏; 촏; 촏; 촏; ) HANGUL SYLLABLE COD +CD10;CD10;110E 1169 11AF;CD10;110E 1169 11AF; # (촐; 촐; 촐; 촐; 촐; ) HANGUL SYLLABLE COL +CD11;CD11;110E 1169 11B0;CD11;110E 1169 11B0; # (촑; 촑; 촑; 촑; 촑; ) HANGUL SYLLABLE COLG +CD12;CD12;110E 1169 11B1;CD12;110E 1169 11B1; # (촒; 촒; 촒; 촒; 촒; ) HANGUL SYLLABLE COLM +CD13;CD13;110E 1169 11B2;CD13;110E 1169 11B2; # (촓; 촓; 촓; 촓; 촓; ) HANGUL SYLLABLE COLB +CD14;CD14;110E 1169 11B3;CD14;110E 1169 11B3; # (촔; 촔; 촔; 촔; 촔; ) HANGUL SYLLABLE COLS +CD15;CD15;110E 1169 11B4;CD15;110E 1169 11B4; # (촕; 촕; 촕; 촕; 촕; ) HANGUL SYLLABLE COLT +CD16;CD16;110E 1169 11B5;CD16;110E 1169 11B5; # (촖; 촖; 촖; 촖; 촖; ) HANGUL SYLLABLE COLP +CD17;CD17;110E 1169 11B6;CD17;110E 1169 11B6; # (촗; 촗; 촗; 촗; 촗; ) HANGUL SYLLABLE COLH +CD18;CD18;110E 1169 11B7;CD18;110E 1169 11B7; # (촘; 촘; 촘; 촘; 촘; ) HANGUL SYLLABLE COM +CD19;CD19;110E 1169 11B8;CD19;110E 1169 11B8; # (촙; 촙; 촙; 촙; 촙; ) HANGUL SYLLABLE COB +CD1A;CD1A;110E 1169 11B9;CD1A;110E 1169 11B9; # (촚; 촚; 촚; 촚; 촚; ) HANGUL SYLLABLE COBS +CD1B;CD1B;110E 1169 11BA;CD1B;110E 1169 11BA; # (촛; 촛; 촛; 촛; 촛; ) HANGUL SYLLABLE COS +CD1C;CD1C;110E 1169 11BB;CD1C;110E 1169 11BB; # (촜; 촜; 촜; 촜; 촜; ) HANGUL SYLLABLE COSS +CD1D;CD1D;110E 1169 11BC;CD1D;110E 1169 11BC; # (총; 총; 총; 총; 총; ) HANGUL SYLLABLE CONG +CD1E;CD1E;110E 1169 11BD;CD1E;110E 1169 11BD; # (촞; 촞; 촞; 촞; 촞; ) HANGUL SYLLABLE COJ +CD1F;CD1F;110E 1169 11BE;CD1F;110E 1169 11BE; # (촟; 촟; 촟; 촟; 촟; ) HANGUL SYLLABLE COC +CD20;CD20;110E 1169 11BF;CD20;110E 1169 11BF; # (촠; 촠; 촠; 촠; 촠; ) HANGUL SYLLABLE COK +CD21;CD21;110E 1169 11C0;CD21;110E 1169 11C0; # (촡; 촡; 촡; 촡; 촡; ) HANGUL SYLLABLE COT +CD22;CD22;110E 1169 11C1;CD22;110E 1169 11C1; # (촢; 촢; 촢; 촢; 촢; ) HANGUL SYLLABLE COP +CD23;CD23;110E 1169 11C2;CD23;110E 1169 11C2; # (촣; 촣; 촣; 촣; 촣; ) HANGUL SYLLABLE COH +CD24;CD24;110E 116A;CD24;110E 116A; # (촤; 촤; 촤; 촤; 촤; ) HANGUL SYLLABLE CWA +CD25;CD25;110E 116A 11A8;CD25;110E 116A 11A8; # (촥; 촥; 촥; 촥; 촥; ) HANGUL SYLLABLE CWAG +CD26;CD26;110E 116A 11A9;CD26;110E 116A 11A9; # (촦; 촦; 촦; 촦; 촦; ) HANGUL SYLLABLE CWAGG +CD27;CD27;110E 116A 11AA;CD27;110E 116A 11AA; # (촧; 촧; 촧; 촧; 촧; ) HANGUL SYLLABLE CWAGS +CD28;CD28;110E 116A 11AB;CD28;110E 116A 11AB; # (촨; 촨; 촨; 촨; 촨; ) HANGUL SYLLABLE CWAN +CD29;CD29;110E 116A 11AC;CD29;110E 116A 11AC; # (촩; 촩; 촩; 촩; 촩; ) HANGUL SYLLABLE CWANJ +CD2A;CD2A;110E 116A 11AD;CD2A;110E 116A 11AD; # (촪; 촪; 촪; 촪; 촪; ) HANGUL SYLLABLE CWANH +CD2B;CD2B;110E 116A 11AE;CD2B;110E 116A 11AE; # (촫; 촫; 촫; 촫; 촫; ) HANGUL SYLLABLE CWAD +CD2C;CD2C;110E 116A 11AF;CD2C;110E 116A 11AF; # (촬; 촬; 촬; 촬; 촬; ) HANGUL SYLLABLE CWAL +CD2D;CD2D;110E 116A 11B0;CD2D;110E 116A 11B0; # (촭; 촭; 촭; 촭; 촭; ) HANGUL SYLLABLE CWALG +CD2E;CD2E;110E 116A 11B1;CD2E;110E 116A 11B1; # (촮; 촮; 촮; 촮; 촮; ) HANGUL SYLLABLE CWALM +CD2F;CD2F;110E 116A 11B2;CD2F;110E 116A 11B2; # (촯; 촯; 촯; 촯; 촯; ) HANGUL SYLLABLE CWALB +CD30;CD30;110E 116A 11B3;CD30;110E 116A 11B3; # (촰; 촰; 촰; 촰; 촰; ) HANGUL SYLLABLE CWALS +CD31;CD31;110E 116A 11B4;CD31;110E 116A 11B4; # (촱; 촱; 촱; 촱; 촱; ) HANGUL SYLLABLE CWALT +CD32;CD32;110E 116A 11B5;CD32;110E 116A 11B5; # (촲; 촲; 촲; 촲; 촲; ) HANGUL SYLLABLE CWALP +CD33;CD33;110E 116A 11B6;CD33;110E 116A 11B6; # (촳; 촳; 촳; 촳; 촳; ) HANGUL SYLLABLE CWALH +CD34;CD34;110E 116A 11B7;CD34;110E 116A 11B7; # (촴; 촴; 촴; 촴; 촴; ) HANGUL SYLLABLE CWAM +CD35;CD35;110E 116A 11B8;CD35;110E 116A 11B8; # (촵; 촵; 촵; 촵; 촵; ) HANGUL SYLLABLE CWAB +CD36;CD36;110E 116A 11B9;CD36;110E 116A 11B9; # (촶; 촶; 촶; 촶; 촶; ) HANGUL SYLLABLE CWABS +CD37;CD37;110E 116A 11BA;CD37;110E 116A 11BA; # (촷; 촷; 촷; 촷; 촷; ) HANGUL SYLLABLE CWAS +CD38;CD38;110E 116A 11BB;CD38;110E 116A 11BB; # (촸; 촸; 촸; 촸; 촸; ) HANGUL SYLLABLE CWASS +CD39;CD39;110E 116A 11BC;CD39;110E 116A 11BC; # (촹; 촹; 촹; 촹; 촹; ) HANGUL SYLLABLE CWANG +CD3A;CD3A;110E 116A 11BD;CD3A;110E 116A 11BD; # (촺; 촺; 촺; 촺; 촺; ) HANGUL SYLLABLE CWAJ +CD3B;CD3B;110E 116A 11BE;CD3B;110E 116A 11BE; # (촻; 촻; 촻; 촻; 촻; ) HANGUL SYLLABLE CWAC +CD3C;CD3C;110E 116A 11BF;CD3C;110E 116A 11BF; # (촼; 촼; 촼; 촼; 촼; ) HANGUL SYLLABLE CWAK +CD3D;CD3D;110E 116A 11C0;CD3D;110E 116A 11C0; # (촽; 촽; 촽; 촽; 촽; ) HANGUL SYLLABLE CWAT +CD3E;CD3E;110E 116A 11C1;CD3E;110E 116A 11C1; # (촾; 촾; 촾; 촾; 촾; ) HANGUL SYLLABLE CWAP +CD3F;CD3F;110E 116A 11C2;CD3F;110E 116A 11C2; # (촿; 촿; 촿; 촿; 촿; ) HANGUL SYLLABLE CWAH +CD40;CD40;110E 116B;CD40;110E 116B; # (쵀; 쵀; 쵀; 쵀; 쵀; ) HANGUL SYLLABLE CWAE +CD41;CD41;110E 116B 11A8;CD41;110E 116B 11A8; # (쵁; 쵁; 쵁; 쵁; 쵁; ) HANGUL SYLLABLE CWAEG +CD42;CD42;110E 116B 11A9;CD42;110E 116B 11A9; # (쵂; 쵂; 쵂; 쵂; 쵂; ) HANGUL SYLLABLE CWAEGG +CD43;CD43;110E 116B 11AA;CD43;110E 116B 11AA; # (쵃; 쵃; 쵃; 쵃; 쵃; ) HANGUL SYLLABLE CWAEGS +CD44;CD44;110E 116B 11AB;CD44;110E 116B 11AB; # (쵄; 쵄; 쵄; 쵄; 쵄; ) HANGUL SYLLABLE CWAEN +CD45;CD45;110E 116B 11AC;CD45;110E 116B 11AC; # (쵅; 쵅; 쵅; 쵅; 쵅; ) HANGUL SYLLABLE CWAENJ +CD46;CD46;110E 116B 11AD;CD46;110E 116B 11AD; # (쵆; 쵆; 쵆; 쵆; 쵆; ) HANGUL SYLLABLE CWAENH +CD47;CD47;110E 116B 11AE;CD47;110E 116B 11AE; # (쵇; 쵇; 쵇; 쵇; 쵇; ) HANGUL SYLLABLE CWAED +CD48;CD48;110E 116B 11AF;CD48;110E 116B 11AF; # (쵈; 쵈; 쵈; 쵈; 쵈; ) HANGUL SYLLABLE CWAEL +CD49;CD49;110E 116B 11B0;CD49;110E 116B 11B0; # (쵉; 쵉; 쵉; 쵉; 쵉; ) HANGUL SYLLABLE CWAELG +CD4A;CD4A;110E 116B 11B1;CD4A;110E 116B 11B1; # (쵊; 쵊; 쵊; 쵊; 쵊; ) HANGUL SYLLABLE CWAELM +CD4B;CD4B;110E 116B 11B2;CD4B;110E 116B 11B2; # (쵋; 쵋; 쵋; 쵋; 쵋; ) HANGUL SYLLABLE CWAELB +CD4C;CD4C;110E 116B 11B3;CD4C;110E 116B 11B3; # (쵌; 쵌; 쵌; 쵌; 쵌; ) HANGUL SYLLABLE CWAELS +CD4D;CD4D;110E 116B 11B4;CD4D;110E 116B 11B4; # (쵍; 쵍; 쵍; 쵍; 쵍; ) HANGUL SYLLABLE CWAELT +CD4E;CD4E;110E 116B 11B5;CD4E;110E 116B 11B5; # (쵎; 쵎; 쵎; 쵎; 쵎; ) HANGUL SYLLABLE CWAELP +CD4F;CD4F;110E 116B 11B6;CD4F;110E 116B 11B6; # (쵏; 쵏; 쵏; 쵏; 쵏; ) HANGUL SYLLABLE CWAELH +CD50;CD50;110E 116B 11B7;CD50;110E 116B 11B7; # (쵐; 쵐; 쵐; 쵐; 쵐; ) HANGUL SYLLABLE CWAEM +CD51;CD51;110E 116B 11B8;CD51;110E 116B 11B8; # (쵑; 쵑; 쵑; 쵑; 쵑; ) HANGUL SYLLABLE CWAEB +CD52;CD52;110E 116B 11B9;CD52;110E 116B 11B9; # (쵒; 쵒; 쵒; 쵒; 쵒; ) HANGUL SYLLABLE CWAEBS +CD53;CD53;110E 116B 11BA;CD53;110E 116B 11BA; # (쵓; 쵓; 쵓; 쵓; 쵓; ) HANGUL SYLLABLE CWAES +CD54;CD54;110E 116B 11BB;CD54;110E 116B 11BB; # (쵔; 쵔; 쵔; 쵔; 쵔; ) HANGUL SYLLABLE CWAESS +CD55;CD55;110E 116B 11BC;CD55;110E 116B 11BC; # (쵕; 쵕; 쵕; 쵕; 쵕; ) HANGUL SYLLABLE CWAENG +CD56;CD56;110E 116B 11BD;CD56;110E 116B 11BD; # (쵖; 쵖; 쵖; 쵖; 쵖; ) HANGUL SYLLABLE CWAEJ +CD57;CD57;110E 116B 11BE;CD57;110E 116B 11BE; # (쵗; 쵗; 쵗; 쵗; 쵗; ) HANGUL SYLLABLE CWAEC +CD58;CD58;110E 116B 11BF;CD58;110E 116B 11BF; # (쵘; 쵘; 쵘; 쵘; 쵘; ) HANGUL SYLLABLE CWAEK +CD59;CD59;110E 116B 11C0;CD59;110E 116B 11C0; # (쵙; 쵙; 쵙; 쵙; 쵙; ) HANGUL SYLLABLE CWAET +CD5A;CD5A;110E 116B 11C1;CD5A;110E 116B 11C1; # (쵚; 쵚; 쵚; 쵚; 쵚; ) HANGUL SYLLABLE CWAEP +CD5B;CD5B;110E 116B 11C2;CD5B;110E 116B 11C2; # (쵛; 쵛; 쵛; 쵛; 쵛; ) HANGUL SYLLABLE CWAEH +CD5C;CD5C;110E 116C;CD5C;110E 116C; # (최; 최; 최; 최; 최; ) HANGUL SYLLABLE COE +CD5D;CD5D;110E 116C 11A8;CD5D;110E 116C 11A8; # (쵝; 쵝; 쵝; 쵝; 쵝; ) HANGUL SYLLABLE COEG +CD5E;CD5E;110E 116C 11A9;CD5E;110E 116C 11A9; # (쵞; 쵞; 쵞; 쵞; 쵞; ) HANGUL SYLLABLE COEGG +CD5F;CD5F;110E 116C 11AA;CD5F;110E 116C 11AA; # (쵟; 쵟; 쵟; 쵟; 쵟; ) HANGUL SYLLABLE COEGS +CD60;CD60;110E 116C 11AB;CD60;110E 116C 11AB; # (쵠; 쵠; 쵠; 쵠; 쵠; ) HANGUL SYLLABLE COEN +CD61;CD61;110E 116C 11AC;CD61;110E 116C 11AC; # (쵡; 쵡; 쵡; 쵡; 쵡; ) HANGUL SYLLABLE COENJ +CD62;CD62;110E 116C 11AD;CD62;110E 116C 11AD; # (쵢; 쵢; 쵢; 쵢; 쵢; ) HANGUL SYLLABLE COENH +CD63;CD63;110E 116C 11AE;CD63;110E 116C 11AE; # (쵣; 쵣; 쵣; 쵣; 쵣; ) HANGUL SYLLABLE COED +CD64;CD64;110E 116C 11AF;CD64;110E 116C 11AF; # (쵤; 쵤; 쵤; 쵤; 쵤; ) HANGUL SYLLABLE COEL +CD65;CD65;110E 116C 11B0;CD65;110E 116C 11B0; # (쵥; 쵥; 쵥; 쵥; 쵥; ) HANGUL SYLLABLE COELG +CD66;CD66;110E 116C 11B1;CD66;110E 116C 11B1; # (쵦; 쵦; 쵦; 쵦; 쵦; ) HANGUL SYLLABLE COELM +CD67;CD67;110E 116C 11B2;CD67;110E 116C 11B2; # (쵧; 쵧; 쵧; 쵧; 쵧; ) HANGUL SYLLABLE COELB +CD68;CD68;110E 116C 11B3;CD68;110E 116C 11B3; # (쵨; 쵨; 쵨; 쵨; 쵨; ) HANGUL SYLLABLE COELS +CD69;CD69;110E 116C 11B4;CD69;110E 116C 11B4; # (쵩; 쵩; 쵩; 쵩; 쵩; ) HANGUL SYLLABLE COELT +CD6A;CD6A;110E 116C 11B5;CD6A;110E 116C 11B5; # (쵪; 쵪; 쵪; 쵪; 쵪; ) HANGUL SYLLABLE COELP +CD6B;CD6B;110E 116C 11B6;CD6B;110E 116C 11B6; # (쵫; 쵫; 쵫; 쵫; 쵫; ) HANGUL SYLLABLE COELH +CD6C;CD6C;110E 116C 11B7;CD6C;110E 116C 11B7; # (쵬; 쵬; 쵬; 쵬; 쵬; ) HANGUL SYLLABLE COEM +CD6D;CD6D;110E 116C 11B8;CD6D;110E 116C 11B8; # (쵭; 쵭; 쵭; 쵭; 쵭; ) HANGUL SYLLABLE COEB +CD6E;CD6E;110E 116C 11B9;CD6E;110E 116C 11B9; # (쵮; 쵮; 쵮; 쵮; 쵮; ) HANGUL SYLLABLE COEBS +CD6F;CD6F;110E 116C 11BA;CD6F;110E 116C 11BA; # (쵯; 쵯; 쵯; 쵯; 쵯; ) HANGUL SYLLABLE COES +CD70;CD70;110E 116C 11BB;CD70;110E 116C 11BB; # (쵰; 쵰; 쵰; 쵰; 쵰; ) HANGUL SYLLABLE COESS +CD71;CD71;110E 116C 11BC;CD71;110E 116C 11BC; # (쵱; 쵱; 쵱; 쵱; 쵱; ) HANGUL SYLLABLE COENG +CD72;CD72;110E 116C 11BD;CD72;110E 116C 11BD; # (쵲; 쵲; 쵲; 쵲; 쵲; ) HANGUL SYLLABLE COEJ +CD73;CD73;110E 116C 11BE;CD73;110E 116C 11BE; # (쵳; 쵳; 쵳; 쵳; 쵳; ) HANGUL SYLLABLE COEC +CD74;CD74;110E 116C 11BF;CD74;110E 116C 11BF; # (쵴; 쵴; 쵴; 쵴; 쵴; ) HANGUL SYLLABLE COEK +CD75;CD75;110E 116C 11C0;CD75;110E 116C 11C0; # (쵵; 쵵; 쵵; 쵵; 쵵; ) HANGUL SYLLABLE COET +CD76;CD76;110E 116C 11C1;CD76;110E 116C 11C1; # (쵶; 쵶; 쵶; 쵶; 쵶; ) HANGUL SYLLABLE COEP +CD77;CD77;110E 116C 11C2;CD77;110E 116C 11C2; # (쵷; 쵷; 쵷; 쵷; 쵷; ) HANGUL SYLLABLE COEH +CD78;CD78;110E 116D;CD78;110E 116D; # (쵸; 쵸; 쵸; 쵸; 쵸; ) HANGUL SYLLABLE CYO +CD79;CD79;110E 116D 11A8;CD79;110E 116D 11A8; # (쵹; 쵹; 쵹; 쵹; 쵹; ) HANGUL SYLLABLE CYOG +CD7A;CD7A;110E 116D 11A9;CD7A;110E 116D 11A9; # (쵺; 쵺; 쵺; 쵺; 쵺; ) HANGUL SYLLABLE CYOGG +CD7B;CD7B;110E 116D 11AA;CD7B;110E 116D 11AA; # (쵻; 쵻; 쵻; 쵻; 쵻; ) HANGUL SYLLABLE CYOGS +CD7C;CD7C;110E 116D 11AB;CD7C;110E 116D 11AB; # (쵼; 쵼; 쵼; 쵼; 쵼; ) HANGUL SYLLABLE CYON +CD7D;CD7D;110E 116D 11AC;CD7D;110E 116D 11AC; # (쵽; 쵽; 쵽; 쵽; 쵽; ) HANGUL SYLLABLE CYONJ +CD7E;CD7E;110E 116D 11AD;CD7E;110E 116D 11AD; # (쵾; 쵾; 쵾; 쵾; 쵾; ) HANGUL SYLLABLE CYONH +CD7F;CD7F;110E 116D 11AE;CD7F;110E 116D 11AE; # (쵿; 쵿; 쵿; 쵿; 쵿; ) HANGUL SYLLABLE CYOD +CD80;CD80;110E 116D 11AF;CD80;110E 116D 11AF; # (춀; 춀; 춀; 춀; 춀; ) HANGUL SYLLABLE CYOL +CD81;CD81;110E 116D 11B0;CD81;110E 116D 11B0; # (춁; 춁; 춁; 춁; 춁; ) HANGUL SYLLABLE CYOLG +CD82;CD82;110E 116D 11B1;CD82;110E 116D 11B1; # (춂; 춂; 춂; 춂; 춂; ) HANGUL SYLLABLE CYOLM +CD83;CD83;110E 116D 11B2;CD83;110E 116D 11B2; # (춃; 춃; 춃; 춃; 춃; ) HANGUL SYLLABLE CYOLB +CD84;CD84;110E 116D 11B3;CD84;110E 116D 11B3; # (춄; 춄; 춄; 춄; 춄; ) HANGUL SYLLABLE CYOLS +CD85;CD85;110E 116D 11B4;CD85;110E 116D 11B4; # (춅; 춅; 춅; 춅; 춅; ) HANGUL SYLLABLE CYOLT +CD86;CD86;110E 116D 11B5;CD86;110E 116D 11B5; # (춆; 춆; 춆; 춆; 춆; ) HANGUL SYLLABLE CYOLP +CD87;CD87;110E 116D 11B6;CD87;110E 116D 11B6; # (춇; 춇; 춇; 춇; 춇; ) HANGUL SYLLABLE CYOLH +CD88;CD88;110E 116D 11B7;CD88;110E 116D 11B7; # (춈; 춈; 춈; 춈; 춈; ) HANGUL SYLLABLE CYOM +CD89;CD89;110E 116D 11B8;CD89;110E 116D 11B8; # (춉; 춉; 춉; 춉; 춉; ) HANGUL SYLLABLE CYOB +CD8A;CD8A;110E 116D 11B9;CD8A;110E 116D 11B9; # (춊; 춊; 춊; 춊; 춊; ) HANGUL SYLLABLE CYOBS +CD8B;CD8B;110E 116D 11BA;CD8B;110E 116D 11BA; # (춋; 춋; 춋; 춋; 춋; ) HANGUL SYLLABLE CYOS +CD8C;CD8C;110E 116D 11BB;CD8C;110E 116D 11BB; # (춌; 춌; 춌; 춌; 춌; ) HANGUL SYLLABLE CYOSS +CD8D;CD8D;110E 116D 11BC;CD8D;110E 116D 11BC; # (춍; 춍; 춍; 춍; 춍; ) HANGUL SYLLABLE CYONG +CD8E;CD8E;110E 116D 11BD;CD8E;110E 116D 11BD; # (춎; 춎; 춎; 춎; 춎; ) HANGUL SYLLABLE CYOJ +CD8F;CD8F;110E 116D 11BE;CD8F;110E 116D 11BE; # (춏; 춏; 춏; 춏; 춏; ) HANGUL SYLLABLE CYOC +CD90;CD90;110E 116D 11BF;CD90;110E 116D 11BF; # (춐; 춐; 춐; 춐; 춐; ) HANGUL SYLLABLE CYOK +CD91;CD91;110E 116D 11C0;CD91;110E 116D 11C0; # (춑; 춑; 춑; 춑; 춑; ) HANGUL SYLLABLE CYOT +CD92;CD92;110E 116D 11C1;CD92;110E 116D 11C1; # (춒; 춒; 춒; 춒; 춒; ) HANGUL SYLLABLE CYOP +CD93;CD93;110E 116D 11C2;CD93;110E 116D 11C2; # (춓; 춓; 춓; 춓; 춓; ) HANGUL SYLLABLE CYOH +CD94;CD94;110E 116E;CD94;110E 116E; # (추; 추; 추; 추; 추; ) HANGUL SYLLABLE CU +CD95;CD95;110E 116E 11A8;CD95;110E 116E 11A8; # (축; 축; 축; 축; 축; ) HANGUL SYLLABLE CUG +CD96;CD96;110E 116E 11A9;CD96;110E 116E 11A9; # (춖; 춖; 춖; 춖; 춖; ) HANGUL SYLLABLE CUGG +CD97;CD97;110E 116E 11AA;CD97;110E 116E 11AA; # (춗; 춗; 춗; 춗; 춗; ) HANGUL SYLLABLE CUGS +CD98;CD98;110E 116E 11AB;CD98;110E 116E 11AB; # (춘; 춘; 춘; 춘; 춘; ) HANGUL SYLLABLE CUN +CD99;CD99;110E 116E 11AC;CD99;110E 116E 11AC; # (춙; 춙; 춙; 춙; 춙; ) HANGUL SYLLABLE CUNJ +CD9A;CD9A;110E 116E 11AD;CD9A;110E 116E 11AD; # (춚; 춚; 춚; 춚; 춚; ) HANGUL SYLLABLE CUNH +CD9B;CD9B;110E 116E 11AE;CD9B;110E 116E 11AE; # (춛; 춛; 춛; 춛; 춛; ) HANGUL SYLLABLE CUD +CD9C;CD9C;110E 116E 11AF;CD9C;110E 116E 11AF; # (출; 출; 출; 출; 출; ) HANGUL SYLLABLE CUL +CD9D;CD9D;110E 116E 11B0;CD9D;110E 116E 11B0; # (춝; 춝; 춝; 춝; 춝; ) HANGUL SYLLABLE CULG +CD9E;CD9E;110E 116E 11B1;CD9E;110E 116E 11B1; # (춞; 춞; 춞; 춞; 춞; ) HANGUL SYLLABLE CULM +CD9F;CD9F;110E 116E 11B2;CD9F;110E 116E 11B2; # (춟; 춟; 춟; 춟; 춟; ) HANGUL SYLLABLE CULB +CDA0;CDA0;110E 116E 11B3;CDA0;110E 116E 11B3; # (춠; 춠; 춠; 춠; 춠; ) HANGUL SYLLABLE CULS +CDA1;CDA1;110E 116E 11B4;CDA1;110E 116E 11B4; # (춡; 춡; 춡; 춡; 춡; ) HANGUL SYLLABLE CULT +CDA2;CDA2;110E 116E 11B5;CDA2;110E 116E 11B5; # (춢; 춢; 춢; 춢; 춢; ) HANGUL SYLLABLE CULP +CDA3;CDA3;110E 116E 11B6;CDA3;110E 116E 11B6; # (춣; 춣; 춣; 춣; 춣; ) HANGUL SYLLABLE CULH +CDA4;CDA4;110E 116E 11B7;CDA4;110E 116E 11B7; # (춤; 춤; 춤; 춤; 춤; ) HANGUL SYLLABLE CUM +CDA5;CDA5;110E 116E 11B8;CDA5;110E 116E 11B8; # (춥; 춥; 춥; 춥; 춥; ) HANGUL SYLLABLE CUB +CDA6;CDA6;110E 116E 11B9;CDA6;110E 116E 11B9; # (춦; 춦; 춦; 춦; 춦; ) HANGUL SYLLABLE CUBS +CDA7;CDA7;110E 116E 11BA;CDA7;110E 116E 11BA; # (춧; 춧; 춧; 춧; 춧; ) HANGUL SYLLABLE CUS +CDA8;CDA8;110E 116E 11BB;CDA8;110E 116E 11BB; # (춨; 춨; 춨; 춨; 춨; ) HANGUL SYLLABLE CUSS +CDA9;CDA9;110E 116E 11BC;CDA9;110E 116E 11BC; # (충; 충; 충; 충; 충; ) HANGUL SYLLABLE CUNG +CDAA;CDAA;110E 116E 11BD;CDAA;110E 116E 11BD; # (춪; 춪; 춪; 춪; 춪; ) HANGUL SYLLABLE CUJ +CDAB;CDAB;110E 116E 11BE;CDAB;110E 116E 11BE; # (춫; 춫; 춫; 춫; 춫; ) HANGUL SYLLABLE CUC +CDAC;CDAC;110E 116E 11BF;CDAC;110E 116E 11BF; # (춬; 춬; 춬; 춬; 춬; ) HANGUL SYLLABLE CUK +CDAD;CDAD;110E 116E 11C0;CDAD;110E 116E 11C0; # (춭; 춭; 춭; 춭; 춭; ) HANGUL SYLLABLE CUT +CDAE;CDAE;110E 116E 11C1;CDAE;110E 116E 11C1; # (춮; 춮; 춮; 춮; 춮; ) HANGUL SYLLABLE CUP +CDAF;CDAF;110E 116E 11C2;CDAF;110E 116E 11C2; # (춯; 춯; 춯; 춯; 춯; ) HANGUL SYLLABLE CUH +CDB0;CDB0;110E 116F;CDB0;110E 116F; # (춰; 춰; 춰; 춰; 춰; ) HANGUL SYLLABLE CWEO +CDB1;CDB1;110E 116F 11A8;CDB1;110E 116F 11A8; # (춱; 춱; 춱; 춱; 춱; ) HANGUL SYLLABLE CWEOG +CDB2;CDB2;110E 116F 11A9;CDB2;110E 116F 11A9; # (춲; 춲; 춲; 춲; 춲; ) HANGUL SYLLABLE CWEOGG +CDB3;CDB3;110E 116F 11AA;CDB3;110E 116F 11AA; # (춳; 춳; 춳; 춳; 춳; ) HANGUL SYLLABLE CWEOGS +CDB4;CDB4;110E 116F 11AB;CDB4;110E 116F 11AB; # (춴; 춴; 춴; 춴; 춴; ) HANGUL SYLLABLE CWEON +CDB5;CDB5;110E 116F 11AC;CDB5;110E 116F 11AC; # (춵; 춵; 춵; 춵; 춵; ) HANGUL SYLLABLE CWEONJ +CDB6;CDB6;110E 116F 11AD;CDB6;110E 116F 11AD; # (춶; 춶; 춶; 춶; 춶; ) HANGUL SYLLABLE CWEONH +CDB7;CDB7;110E 116F 11AE;CDB7;110E 116F 11AE; # (춷; 춷; 춷; 춷; 춷; ) HANGUL SYLLABLE CWEOD +CDB8;CDB8;110E 116F 11AF;CDB8;110E 116F 11AF; # (춸; 춸; 춸; 춸; 춸; ) HANGUL SYLLABLE CWEOL +CDB9;CDB9;110E 116F 11B0;CDB9;110E 116F 11B0; # (춹; 춹; 춹; 춹; 춹; ) HANGUL SYLLABLE CWEOLG +CDBA;CDBA;110E 116F 11B1;CDBA;110E 116F 11B1; # (춺; 춺; 춺; 춺; 춺; ) HANGUL SYLLABLE CWEOLM +CDBB;CDBB;110E 116F 11B2;CDBB;110E 116F 11B2; # (춻; 춻; 춻; 춻; 춻; ) HANGUL SYLLABLE CWEOLB +CDBC;CDBC;110E 116F 11B3;CDBC;110E 116F 11B3; # (춼; 춼; 춼; 춼; 춼; ) HANGUL SYLLABLE CWEOLS +CDBD;CDBD;110E 116F 11B4;CDBD;110E 116F 11B4; # (춽; 춽; 춽; 춽; 춽; ) HANGUL SYLLABLE CWEOLT +CDBE;CDBE;110E 116F 11B5;CDBE;110E 116F 11B5; # (춾; 춾; 춾; 춾; 춾; ) HANGUL SYLLABLE CWEOLP +CDBF;CDBF;110E 116F 11B6;CDBF;110E 116F 11B6; # (춿; 춿; 춿; 춿; 춿; ) HANGUL SYLLABLE CWEOLH +CDC0;CDC0;110E 116F 11B7;CDC0;110E 116F 11B7; # (췀; 췀; 췀; 췀; 췀; ) HANGUL SYLLABLE CWEOM +CDC1;CDC1;110E 116F 11B8;CDC1;110E 116F 11B8; # (췁; 췁; 췁; 췁; 췁; ) HANGUL SYLLABLE CWEOB +CDC2;CDC2;110E 116F 11B9;CDC2;110E 116F 11B9; # (췂; 췂; 췂; 췂; 췂; ) HANGUL SYLLABLE CWEOBS +CDC3;CDC3;110E 116F 11BA;CDC3;110E 116F 11BA; # (췃; 췃; 췃; 췃; 췃; ) HANGUL SYLLABLE CWEOS +CDC4;CDC4;110E 116F 11BB;CDC4;110E 116F 11BB; # (췄; 췄; 췄; 췄; 췄; ) HANGUL SYLLABLE CWEOSS +CDC5;CDC5;110E 116F 11BC;CDC5;110E 116F 11BC; # (췅; 췅; 췅; 췅; 췅; ) HANGUL SYLLABLE CWEONG +CDC6;CDC6;110E 116F 11BD;CDC6;110E 116F 11BD; # (췆; 췆; 췆; 췆; 췆; ) HANGUL SYLLABLE CWEOJ +CDC7;CDC7;110E 116F 11BE;CDC7;110E 116F 11BE; # (췇; 췇; 췇; 췇; 췇; ) HANGUL SYLLABLE CWEOC +CDC8;CDC8;110E 116F 11BF;CDC8;110E 116F 11BF; # (췈; 췈; 췈; 췈; 췈; ) HANGUL SYLLABLE CWEOK +CDC9;CDC9;110E 116F 11C0;CDC9;110E 116F 11C0; # (췉; 췉; 췉; 췉; 췉; ) HANGUL SYLLABLE CWEOT +CDCA;CDCA;110E 116F 11C1;CDCA;110E 116F 11C1; # (췊; 췊; 췊; 췊; 췊; ) HANGUL SYLLABLE CWEOP +CDCB;CDCB;110E 116F 11C2;CDCB;110E 116F 11C2; # (췋; 췋; 췋; 췋; 췋; ) HANGUL SYLLABLE CWEOH +CDCC;CDCC;110E 1170;CDCC;110E 1170; # (췌; 췌; 췌; 췌; 췌; ) HANGUL SYLLABLE CWE +CDCD;CDCD;110E 1170 11A8;CDCD;110E 1170 11A8; # (췍; 췍; 췍; 췍; 췍; ) HANGUL SYLLABLE CWEG +CDCE;CDCE;110E 1170 11A9;CDCE;110E 1170 11A9; # (췎; 췎; 췎; 췎; 췎; ) HANGUL SYLLABLE CWEGG +CDCF;CDCF;110E 1170 11AA;CDCF;110E 1170 11AA; # (췏; 췏; 췏; 췏; 췏; ) HANGUL SYLLABLE CWEGS +CDD0;CDD0;110E 1170 11AB;CDD0;110E 1170 11AB; # (췐; 췐; 췐; 췐; 췐; ) HANGUL SYLLABLE CWEN +CDD1;CDD1;110E 1170 11AC;CDD1;110E 1170 11AC; # (췑; 췑; 췑; 췑; 췑; ) HANGUL SYLLABLE CWENJ +CDD2;CDD2;110E 1170 11AD;CDD2;110E 1170 11AD; # (췒; 췒; 췒; 췒; 췒; ) HANGUL SYLLABLE CWENH +CDD3;CDD3;110E 1170 11AE;CDD3;110E 1170 11AE; # (췓; 췓; 췓; 췓; 췓; ) HANGUL SYLLABLE CWED +CDD4;CDD4;110E 1170 11AF;CDD4;110E 1170 11AF; # (췔; 췔; 췔; 췔; 췔; ) HANGUL SYLLABLE CWEL +CDD5;CDD5;110E 1170 11B0;CDD5;110E 1170 11B0; # (췕; 췕; 췕; 췕; 췕; ) HANGUL SYLLABLE CWELG +CDD6;CDD6;110E 1170 11B1;CDD6;110E 1170 11B1; # (췖; 췖; 췖; 췖; 췖; ) HANGUL SYLLABLE CWELM +CDD7;CDD7;110E 1170 11B2;CDD7;110E 1170 11B2; # (췗; 췗; 췗; 췗; 췗; ) HANGUL SYLLABLE CWELB +CDD8;CDD8;110E 1170 11B3;CDD8;110E 1170 11B3; # (췘; 췘; 췘; 췘; 췘; ) HANGUL SYLLABLE CWELS +CDD9;CDD9;110E 1170 11B4;CDD9;110E 1170 11B4; # (췙; 췙; 췙; 췙; 췙; ) HANGUL SYLLABLE CWELT +CDDA;CDDA;110E 1170 11B5;CDDA;110E 1170 11B5; # (췚; 췚; 췚; 췚; 췚; ) HANGUL SYLLABLE CWELP +CDDB;CDDB;110E 1170 11B6;CDDB;110E 1170 11B6; # (췛; 췛; 췛; 췛; 췛; ) HANGUL SYLLABLE CWELH +CDDC;CDDC;110E 1170 11B7;CDDC;110E 1170 11B7; # (췜; 췜; 췜; 췜; 췜; ) HANGUL SYLLABLE CWEM +CDDD;CDDD;110E 1170 11B8;CDDD;110E 1170 11B8; # (췝; 췝; 췝; 췝; 췝; ) HANGUL SYLLABLE CWEB +CDDE;CDDE;110E 1170 11B9;CDDE;110E 1170 11B9; # (췞; 췞; 췞; 췞; 췞; ) HANGUL SYLLABLE CWEBS +CDDF;CDDF;110E 1170 11BA;CDDF;110E 1170 11BA; # (췟; 췟; 췟; 췟; 췟; ) HANGUL SYLLABLE CWES +CDE0;CDE0;110E 1170 11BB;CDE0;110E 1170 11BB; # (췠; 췠; 췠; 췠; 췠; ) HANGUL SYLLABLE CWESS +CDE1;CDE1;110E 1170 11BC;CDE1;110E 1170 11BC; # (췡; 췡; 췡; 췡; 췡; ) HANGUL SYLLABLE CWENG +CDE2;CDE2;110E 1170 11BD;CDE2;110E 1170 11BD; # (췢; 췢; 췢; 췢; 췢; ) HANGUL SYLLABLE CWEJ +CDE3;CDE3;110E 1170 11BE;CDE3;110E 1170 11BE; # (췣; 췣; 췣; 췣; 췣; ) HANGUL SYLLABLE CWEC +CDE4;CDE4;110E 1170 11BF;CDE4;110E 1170 11BF; # (췤; 췤; 췤; 췤; 췤; ) HANGUL SYLLABLE CWEK +CDE5;CDE5;110E 1170 11C0;CDE5;110E 1170 11C0; # (췥; 췥; 췥; 췥; 췥; ) HANGUL SYLLABLE CWET +CDE6;CDE6;110E 1170 11C1;CDE6;110E 1170 11C1; # (췦; 췦; 췦; 췦; 췦; ) HANGUL SYLLABLE CWEP +CDE7;CDE7;110E 1170 11C2;CDE7;110E 1170 11C2; # (췧; 췧; 췧; 췧; 췧; ) HANGUL SYLLABLE CWEH +CDE8;CDE8;110E 1171;CDE8;110E 1171; # (취; 취; 취; 취; 취; ) HANGUL SYLLABLE CWI +CDE9;CDE9;110E 1171 11A8;CDE9;110E 1171 11A8; # (췩; 췩; 췩; 췩; 췩; ) HANGUL SYLLABLE CWIG +CDEA;CDEA;110E 1171 11A9;CDEA;110E 1171 11A9; # (췪; 췪; 췪; 췪; 췪; ) HANGUL SYLLABLE CWIGG +CDEB;CDEB;110E 1171 11AA;CDEB;110E 1171 11AA; # (췫; 췫; 췫; 췫; 췫; ) HANGUL SYLLABLE CWIGS +CDEC;CDEC;110E 1171 11AB;CDEC;110E 1171 11AB; # (췬; 췬; 췬; 췬; 췬; ) HANGUL SYLLABLE CWIN +CDED;CDED;110E 1171 11AC;CDED;110E 1171 11AC; # (췭; 췭; 췭; 췭; 췭; ) HANGUL SYLLABLE CWINJ +CDEE;CDEE;110E 1171 11AD;CDEE;110E 1171 11AD; # (췮; 췮; 췮; 췮; 췮; ) HANGUL SYLLABLE CWINH +CDEF;CDEF;110E 1171 11AE;CDEF;110E 1171 11AE; # (췯; 췯; 췯; 췯; 췯; ) HANGUL SYLLABLE CWID +CDF0;CDF0;110E 1171 11AF;CDF0;110E 1171 11AF; # (췰; 췰; 췰; 췰; 췰; ) HANGUL SYLLABLE CWIL +CDF1;CDF1;110E 1171 11B0;CDF1;110E 1171 11B0; # (췱; 췱; 췱; 췱; 췱; ) HANGUL SYLLABLE CWILG +CDF2;CDF2;110E 1171 11B1;CDF2;110E 1171 11B1; # (췲; 췲; 췲; 췲; 췲; ) HANGUL SYLLABLE CWILM +CDF3;CDF3;110E 1171 11B2;CDF3;110E 1171 11B2; # (췳; 췳; 췳; 췳; 췳; ) HANGUL SYLLABLE CWILB +CDF4;CDF4;110E 1171 11B3;CDF4;110E 1171 11B3; # (췴; 췴; 췴; 췴; 췴; ) HANGUL SYLLABLE CWILS +CDF5;CDF5;110E 1171 11B4;CDF5;110E 1171 11B4; # (췵; 췵; 췵; 췵; 췵; ) HANGUL SYLLABLE CWILT +CDF6;CDF6;110E 1171 11B5;CDF6;110E 1171 11B5; # (췶; 췶; 췶; 췶; 췶; ) HANGUL SYLLABLE CWILP +CDF7;CDF7;110E 1171 11B6;CDF7;110E 1171 11B6; # (췷; 췷; 췷; 췷; 췷; ) HANGUL SYLLABLE CWILH +CDF8;CDF8;110E 1171 11B7;CDF8;110E 1171 11B7; # (췸; 췸; 췸; 췸; 췸; ) HANGUL SYLLABLE CWIM +CDF9;CDF9;110E 1171 11B8;CDF9;110E 1171 11B8; # (췹; 췹; 췹; 췹; 췹; ) HANGUL SYLLABLE CWIB +CDFA;CDFA;110E 1171 11B9;CDFA;110E 1171 11B9; # (췺; 췺; 췺; 췺; 췺; ) HANGUL SYLLABLE CWIBS +CDFB;CDFB;110E 1171 11BA;CDFB;110E 1171 11BA; # (췻; 췻; 췻; 췻; 췻; ) HANGUL SYLLABLE CWIS +CDFC;CDFC;110E 1171 11BB;CDFC;110E 1171 11BB; # (췼; 췼; 췼; 췼; 췼; ) HANGUL SYLLABLE CWISS +CDFD;CDFD;110E 1171 11BC;CDFD;110E 1171 11BC; # (췽; 췽; 췽; 췽; 췽; ) HANGUL SYLLABLE CWING +CDFE;CDFE;110E 1171 11BD;CDFE;110E 1171 11BD; # (췾; 췾; 췾; 췾; 췾; ) HANGUL SYLLABLE CWIJ +CDFF;CDFF;110E 1171 11BE;CDFF;110E 1171 11BE; # (췿; 췿; 췿; 췿; 췿; ) HANGUL SYLLABLE CWIC +CE00;CE00;110E 1171 11BF;CE00;110E 1171 11BF; # (츀; 츀; 츀; 츀; 츀; ) HANGUL SYLLABLE CWIK +CE01;CE01;110E 1171 11C0;CE01;110E 1171 11C0; # (츁; 츁; 츁; 츁; 츁; ) HANGUL SYLLABLE CWIT +CE02;CE02;110E 1171 11C1;CE02;110E 1171 11C1; # (츂; 츂; 츂; 츂; 츂; ) HANGUL SYLLABLE CWIP +CE03;CE03;110E 1171 11C2;CE03;110E 1171 11C2; # (츃; 츃; 츃; 츃; 츃; ) HANGUL SYLLABLE CWIH +CE04;CE04;110E 1172;CE04;110E 1172; # (츄; 츄; 츄; 츄; 츄; ) HANGUL SYLLABLE CYU +CE05;CE05;110E 1172 11A8;CE05;110E 1172 11A8; # (츅; 츅; 츅; 츅; 츅; ) HANGUL SYLLABLE CYUG +CE06;CE06;110E 1172 11A9;CE06;110E 1172 11A9; # (츆; 츆; 츆; 츆; 츆; ) HANGUL SYLLABLE CYUGG +CE07;CE07;110E 1172 11AA;CE07;110E 1172 11AA; # (츇; 츇; 츇; 츇; 츇; ) HANGUL SYLLABLE CYUGS +CE08;CE08;110E 1172 11AB;CE08;110E 1172 11AB; # (츈; 츈; 츈; 츈; 츈; ) HANGUL SYLLABLE CYUN +CE09;CE09;110E 1172 11AC;CE09;110E 1172 11AC; # (츉; 츉; 츉; 츉; 츉; ) HANGUL SYLLABLE CYUNJ +CE0A;CE0A;110E 1172 11AD;CE0A;110E 1172 11AD; # (츊; 츊; 츊; 츊; 츊; ) HANGUL SYLLABLE CYUNH +CE0B;CE0B;110E 1172 11AE;CE0B;110E 1172 11AE; # (츋; 츋; 츋; 츋; 츋; ) HANGUL SYLLABLE CYUD +CE0C;CE0C;110E 1172 11AF;CE0C;110E 1172 11AF; # (츌; 츌; 츌; 츌; 츌; ) HANGUL SYLLABLE CYUL +CE0D;CE0D;110E 1172 11B0;CE0D;110E 1172 11B0; # (츍; 츍; 츍; 츍; 츍; ) HANGUL SYLLABLE CYULG +CE0E;CE0E;110E 1172 11B1;CE0E;110E 1172 11B1; # (츎; 츎; 츎; 츎; 츎; ) HANGUL SYLLABLE CYULM +CE0F;CE0F;110E 1172 11B2;CE0F;110E 1172 11B2; # (츏; 츏; 츏; 츏; 츏; ) HANGUL SYLLABLE CYULB +CE10;CE10;110E 1172 11B3;CE10;110E 1172 11B3; # (츐; 츐; 츐; 츐; 츐; ) HANGUL SYLLABLE CYULS +CE11;CE11;110E 1172 11B4;CE11;110E 1172 11B4; # (츑; 츑; 츑; 츑; 츑; ) HANGUL SYLLABLE CYULT +CE12;CE12;110E 1172 11B5;CE12;110E 1172 11B5; # (츒; 츒; 츒; 츒; 츒; ) HANGUL SYLLABLE CYULP +CE13;CE13;110E 1172 11B6;CE13;110E 1172 11B6; # (츓; 츓; 츓; 츓; 츓; ) HANGUL SYLLABLE CYULH +CE14;CE14;110E 1172 11B7;CE14;110E 1172 11B7; # (츔; 츔; 츔; 츔; 츔; ) HANGUL SYLLABLE CYUM +CE15;CE15;110E 1172 11B8;CE15;110E 1172 11B8; # (츕; 츕; 츕; 츕; 츕; ) HANGUL SYLLABLE CYUB +CE16;CE16;110E 1172 11B9;CE16;110E 1172 11B9; # (츖; 츖; 츖; 츖; 츖; ) HANGUL SYLLABLE CYUBS +CE17;CE17;110E 1172 11BA;CE17;110E 1172 11BA; # (츗; 츗; 츗; 츗; 츗; ) HANGUL SYLLABLE CYUS +CE18;CE18;110E 1172 11BB;CE18;110E 1172 11BB; # (츘; 츘; 츘; 츘; 츘; ) HANGUL SYLLABLE CYUSS +CE19;CE19;110E 1172 11BC;CE19;110E 1172 11BC; # (츙; 츙; 츙; 츙; 츙; ) HANGUL SYLLABLE CYUNG +CE1A;CE1A;110E 1172 11BD;CE1A;110E 1172 11BD; # (츚; 츚; 츚; 츚; 츚; ) HANGUL SYLLABLE CYUJ +CE1B;CE1B;110E 1172 11BE;CE1B;110E 1172 11BE; # (츛; 츛; 츛; 츛; 츛; ) HANGUL SYLLABLE CYUC +CE1C;CE1C;110E 1172 11BF;CE1C;110E 1172 11BF; # (츜; 츜; 츜; 츜; 츜; ) HANGUL SYLLABLE CYUK +CE1D;CE1D;110E 1172 11C0;CE1D;110E 1172 11C0; # (츝; 츝; 츝; 츝; 츝; ) HANGUL SYLLABLE CYUT +CE1E;CE1E;110E 1172 11C1;CE1E;110E 1172 11C1; # (츞; 츞; 츞; 츞; 츞; ) HANGUL SYLLABLE CYUP +CE1F;CE1F;110E 1172 11C2;CE1F;110E 1172 11C2; # (츟; 츟; 츟; 츟; 츟; ) HANGUL SYLLABLE CYUH +CE20;CE20;110E 1173;CE20;110E 1173; # (츠; 츠; 츠; 츠; 츠; ) HANGUL SYLLABLE CEU +CE21;CE21;110E 1173 11A8;CE21;110E 1173 11A8; # (측; 측; 측; 측; 측; ) HANGUL SYLLABLE CEUG +CE22;CE22;110E 1173 11A9;CE22;110E 1173 11A9; # (츢; 츢; 츢; 츢; 츢; ) HANGUL SYLLABLE CEUGG +CE23;CE23;110E 1173 11AA;CE23;110E 1173 11AA; # (츣; 츣; 츣; 츣; 츣; ) HANGUL SYLLABLE CEUGS +CE24;CE24;110E 1173 11AB;CE24;110E 1173 11AB; # (츤; 츤; 츤; 츤; 츤; ) HANGUL SYLLABLE CEUN +CE25;CE25;110E 1173 11AC;CE25;110E 1173 11AC; # (츥; 츥; 츥; 츥; 츥; ) HANGUL SYLLABLE CEUNJ +CE26;CE26;110E 1173 11AD;CE26;110E 1173 11AD; # (츦; 츦; 츦; 츦; 츦; ) HANGUL SYLLABLE CEUNH +CE27;CE27;110E 1173 11AE;CE27;110E 1173 11AE; # (츧; 츧; 츧; 츧; 츧; ) HANGUL SYLLABLE CEUD +CE28;CE28;110E 1173 11AF;CE28;110E 1173 11AF; # (츨; 츨; 츨; 츨; 츨; ) HANGUL SYLLABLE CEUL +CE29;CE29;110E 1173 11B0;CE29;110E 1173 11B0; # (츩; 츩; 츩; 츩; 츩; ) HANGUL SYLLABLE CEULG +CE2A;CE2A;110E 1173 11B1;CE2A;110E 1173 11B1; # (츪; 츪; 츪; 츪; 츪; ) HANGUL SYLLABLE CEULM +CE2B;CE2B;110E 1173 11B2;CE2B;110E 1173 11B2; # (츫; 츫; 츫; 츫; 츫; ) HANGUL SYLLABLE CEULB +CE2C;CE2C;110E 1173 11B3;CE2C;110E 1173 11B3; # (츬; 츬; 츬; 츬; 츬; ) HANGUL SYLLABLE CEULS +CE2D;CE2D;110E 1173 11B4;CE2D;110E 1173 11B4; # (츭; 츭; 츭; 츭; 츭; ) HANGUL SYLLABLE CEULT +CE2E;CE2E;110E 1173 11B5;CE2E;110E 1173 11B5; # (츮; 츮; 츮; 츮; 츮; ) HANGUL SYLLABLE CEULP +CE2F;CE2F;110E 1173 11B6;CE2F;110E 1173 11B6; # (츯; 츯; 츯; 츯; 츯; ) HANGUL SYLLABLE CEULH +CE30;CE30;110E 1173 11B7;CE30;110E 1173 11B7; # (츰; 츰; 츰; 츰; 츰; ) HANGUL SYLLABLE CEUM +CE31;CE31;110E 1173 11B8;CE31;110E 1173 11B8; # (츱; 츱; 츱; 츱; 츱; ) HANGUL SYLLABLE CEUB +CE32;CE32;110E 1173 11B9;CE32;110E 1173 11B9; # (츲; 츲; 츲; 츲; 츲; ) HANGUL SYLLABLE CEUBS +CE33;CE33;110E 1173 11BA;CE33;110E 1173 11BA; # (츳; 츳; 츳; 츳; 츳; ) HANGUL SYLLABLE CEUS +CE34;CE34;110E 1173 11BB;CE34;110E 1173 11BB; # (츴; 츴; 츴; 츴; 츴; ) HANGUL SYLLABLE CEUSS +CE35;CE35;110E 1173 11BC;CE35;110E 1173 11BC; # (층; 층; 층; 층; 층; ) HANGUL SYLLABLE CEUNG +CE36;CE36;110E 1173 11BD;CE36;110E 1173 11BD; # (츶; 츶; 츶; 츶; 츶; ) HANGUL SYLLABLE CEUJ +CE37;CE37;110E 1173 11BE;CE37;110E 1173 11BE; # (츷; 츷; 츷; 츷; 츷; ) HANGUL SYLLABLE CEUC +CE38;CE38;110E 1173 11BF;CE38;110E 1173 11BF; # (츸; 츸; 츸; 츸; 츸; ) HANGUL SYLLABLE CEUK +CE39;CE39;110E 1173 11C0;CE39;110E 1173 11C0; # (츹; 츹; 츹; 츹; 츹; ) HANGUL SYLLABLE CEUT +CE3A;CE3A;110E 1173 11C1;CE3A;110E 1173 11C1; # (츺; 츺; 츺; 츺; 츺; ) HANGUL SYLLABLE CEUP +CE3B;CE3B;110E 1173 11C2;CE3B;110E 1173 11C2; # (츻; 츻; 츻; 츻; 츻; ) HANGUL SYLLABLE CEUH +CE3C;CE3C;110E 1174;CE3C;110E 1174; # (츼; 츼; 츼; 츼; 츼; ) HANGUL SYLLABLE CYI +CE3D;CE3D;110E 1174 11A8;CE3D;110E 1174 11A8; # (츽; 츽; 츽; 츽; 츽; ) HANGUL SYLLABLE CYIG +CE3E;CE3E;110E 1174 11A9;CE3E;110E 1174 11A9; # (츾; 츾; 츾; 츾; 츾; ) HANGUL SYLLABLE CYIGG +CE3F;CE3F;110E 1174 11AA;CE3F;110E 1174 11AA; # (츿; 츿; 츿; 츿; 츿; ) HANGUL SYLLABLE CYIGS +CE40;CE40;110E 1174 11AB;CE40;110E 1174 11AB; # (칀; 칀; 칀; 칀; 칀; ) HANGUL SYLLABLE CYIN +CE41;CE41;110E 1174 11AC;CE41;110E 1174 11AC; # (칁; 칁; 칁; 칁; 칁; ) HANGUL SYLLABLE CYINJ +CE42;CE42;110E 1174 11AD;CE42;110E 1174 11AD; # (칂; 칂; 칂; 칂; 칂; ) HANGUL SYLLABLE CYINH +CE43;CE43;110E 1174 11AE;CE43;110E 1174 11AE; # (칃; 칃; 칃; 칃; 칃; ) HANGUL SYLLABLE CYID +CE44;CE44;110E 1174 11AF;CE44;110E 1174 11AF; # (칄; 칄; 칄; 칄; 칄; ) HANGUL SYLLABLE CYIL +CE45;CE45;110E 1174 11B0;CE45;110E 1174 11B0; # (칅; 칅; 칅; 칅; 칅; ) HANGUL SYLLABLE CYILG +CE46;CE46;110E 1174 11B1;CE46;110E 1174 11B1; # (칆; 칆; 칆; 칆; 칆; ) HANGUL SYLLABLE CYILM +CE47;CE47;110E 1174 11B2;CE47;110E 1174 11B2; # (칇; 칇; 칇; 칇; 칇; ) HANGUL SYLLABLE CYILB +CE48;CE48;110E 1174 11B3;CE48;110E 1174 11B3; # (칈; 칈; 칈; 칈; 칈; ) HANGUL SYLLABLE CYILS +CE49;CE49;110E 1174 11B4;CE49;110E 1174 11B4; # (칉; 칉; 칉; 칉; 칉; ) HANGUL SYLLABLE CYILT +CE4A;CE4A;110E 1174 11B5;CE4A;110E 1174 11B5; # (칊; 칊; 칊; 칊; 칊; ) HANGUL SYLLABLE CYILP +CE4B;CE4B;110E 1174 11B6;CE4B;110E 1174 11B6; # (칋; 칋; 칋; 칋; 칋; ) HANGUL SYLLABLE CYILH +CE4C;CE4C;110E 1174 11B7;CE4C;110E 1174 11B7; # (칌; 칌; 칌; 칌; 칌; ) HANGUL SYLLABLE CYIM +CE4D;CE4D;110E 1174 11B8;CE4D;110E 1174 11B8; # (칍; 칍; 칍; 칍; 칍; ) HANGUL SYLLABLE CYIB +CE4E;CE4E;110E 1174 11B9;CE4E;110E 1174 11B9; # (칎; 칎; 칎; 칎; 칎; ) HANGUL SYLLABLE CYIBS +CE4F;CE4F;110E 1174 11BA;CE4F;110E 1174 11BA; # (칏; 칏; 칏; 칏; 칏; ) HANGUL SYLLABLE CYIS +CE50;CE50;110E 1174 11BB;CE50;110E 1174 11BB; # (칐; 칐; 칐; 칐; 칐; ) HANGUL SYLLABLE CYISS +CE51;CE51;110E 1174 11BC;CE51;110E 1174 11BC; # (칑; 칑; 칑; 칑; 칑; ) HANGUL SYLLABLE CYING +CE52;CE52;110E 1174 11BD;CE52;110E 1174 11BD; # (칒; 칒; 칒; 칒; 칒; ) HANGUL SYLLABLE CYIJ +CE53;CE53;110E 1174 11BE;CE53;110E 1174 11BE; # (칓; 칓; 칓; 칓; 칓; ) HANGUL SYLLABLE CYIC +CE54;CE54;110E 1174 11BF;CE54;110E 1174 11BF; # (칔; 칔; 칔; 칔; 칔; ) HANGUL SYLLABLE CYIK +CE55;CE55;110E 1174 11C0;CE55;110E 1174 11C0; # (칕; 칕; 칕; 칕; 칕; ) HANGUL SYLLABLE CYIT +CE56;CE56;110E 1174 11C1;CE56;110E 1174 11C1; # (칖; 칖; 칖; 칖; 칖; ) HANGUL SYLLABLE CYIP +CE57;CE57;110E 1174 11C2;CE57;110E 1174 11C2; # (칗; 칗; 칗; 칗; 칗; ) HANGUL SYLLABLE CYIH +CE58;CE58;110E 1175;CE58;110E 1175; # (치; 치; 치; 치; 치; ) HANGUL SYLLABLE CI +CE59;CE59;110E 1175 11A8;CE59;110E 1175 11A8; # (칙; 칙; 칙; 칙; 칙; ) HANGUL SYLLABLE CIG +CE5A;CE5A;110E 1175 11A9;CE5A;110E 1175 11A9; # (칚; 칚; 칚; 칚; 칚; ) HANGUL SYLLABLE CIGG +CE5B;CE5B;110E 1175 11AA;CE5B;110E 1175 11AA; # (칛; 칛; 칛; 칛; 칛; ) HANGUL SYLLABLE CIGS +CE5C;CE5C;110E 1175 11AB;CE5C;110E 1175 11AB; # (친; 친; 친; 친; 친; ) HANGUL SYLLABLE CIN +CE5D;CE5D;110E 1175 11AC;CE5D;110E 1175 11AC; # (칝; 칝; 칝; 칝; 칝; ) HANGUL SYLLABLE CINJ +CE5E;CE5E;110E 1175 11AD;CE5E;110E 1175 11AD; # (칞; 칞; 칞; 칞; 칞; ) HANGUL SYLLABLE CINH +CE5F;CE5F;110E 1175 11AE;CE5F;110E 1175 11AE; # (칟; 칟; 칟; 칟; 칟; ) HANGUL SYLLABLE CID +CE60;CE60;110E 1175 11AF;CE60;110E 1175 11AF; # (칠; 칠; 칠; 칠; 칠; ) HANGUL SYLLABLE CIL +CE61;CE61;110E 1175 11B0;CE61;110E 1175 11B0; # (칡; 칡; 칡; 칡; 칡; ) HANGUL SYLLABLE CILG +CE62;CE62;110E 1175 11B1;CE62;110E 1175 11B1; # (칢; 칢; 칢; 칢; 칢; ) HANGUL SYLLABLE CILM +CE63;CE63;110E 1175 11B2;CE63;110E 1175 11B2; # (칣; 칣; 칣; 칣; 칣; ) HANGUL SYLLABLE CILB +CE64;CE64;110E 1175 11B3;CE64;110E 1175 11B3; # (칤; 칤; 칤; 칤; 칤; ) HANGUL SYLLABLE CILS +CE65;CE65;110E 1175 11B4;CE65;110E 1175 11B4; # (칥; 칥; 칥; 칥; 칥; ) HANGUL SYLLABLE CILT +CE66;CE66;110E 1175 11B5;CE66;110E 1175 11B5; # (칦; 칦; 칦; 칦; 칦; ) HANGUL SYLLABLE CILP +CE67;CE67;110E 1175 11B6;CE67;110E 1175 11B6; # (칧; 칧; 칧; 칧; 칧; ) HANGUL SYLLABLE CILH +CE68;CE68;110E 1175 11B7;CE68;110E 1175 11B7; # (침; 침; 침; 침; 침; ) HANGUL SYLLABLE CIM +CE69;CE69;110E 1175 11B8;CE69;110E 1175 11B8; # (칩; 칩; 칩; 칩; 칩; ) HANGUL SYLLABLE CIB +CE6A;CE6A;110E 1175 11B9;CE6A;110E 1175 11B9; # (칪; 칪; 칪; 칪; 칪; ) HANGUL SYLLABLE CIBS +CE6B;CE6B;110E 1175 11BA;CE6B;110E 1175 11BA; # (칫; 칫; 칫; 칫; 칫; ) HANGUL SYLLABLE CIS +CE6C;CE6C;110E 1175 11BB;CE6C;110E 1175 11BB; # (칬; 칬; 칬; 칬; 칬; ) HANGUL SYLLABLE CISS +CE6D;CE6D;110E 1175 11BC;CE6D;110E 1175 11BC; # (칭; 칭; 칭; 칭; 칭; ) HANGUL SYLLABLE CING +CE6E;CE6E;110E 1175 11BD;CE6E;110E 1175 11BD; # (칮; 칮; 칮; 칮; 칮; ) HANGUL SYLLABLE CIJ +CE6F;CE6F;110E 1175 11BE;CE6F;110E 1175 11BE; # (칯; 칯; 칯; 칯; 칯; ) HANGUL SYLLABLE CIC +CE70;CE70;110E 1175 11BF;CE70;110E 1175 11BF; # (칰; 칰; 칰; 칰; 칰; ) HANGUL SYLLABLE CIK +CE71;CE71;110E 1175 11C0;CE71;110E 1175 11C0; # (칱; 칱; 칱; 칱; 칱; ) HANGUL SYLLABLE CIT +CE72;CE72;110E 1175 11C1;CE72;110E 1175 11C1; # (칲; 칲; 칲; 칲; 칲; ) HANGUL SYLLABLE CIP +CE73;CE73;110E 1175 11C2;CE73;110E 1175 11C2; # (칳; 칳; 칳; 칳; 칳; ) HANGUL SYLLABLE CIH +CE74;CE74;110F 1161;CE74;110F 1161; # (카; 카; 카; 카; 카; ) HANGUL SYLLABLE KA +CE75;CE75;110F 1161 11A8;CE75;110F 1161 11A8; # (칵; 칵; 칵; 칵; 칵; ) HANGUL SYLLABLE KAG +CE76;CE76;110F 1161 11A9;CE76;110F 1161 11A9; # (칶; 칶; 칶; 칶; 칶; ) HANGUL SYLLABLE KAGG +CE77;CE77;110F 1161 11AA;CE77;110F 1161 11AA; # (칷; 칷; 칷; 칷; 칷; ) HANGUL SYLLABLE KAGS +CE78;CE78;110F 1161 11AB;CE78;110F 1161 11AB; # (칸; 칸; 칸; 칸; 칸; ) HANGUL SYLLABLE KAN +CE79;CE79;110F 1161 11AC;CE79;110F 1161 11AC; # (칹; 칹; 칹; 칹; 칹; ) HANGUL SYLLABLE KANJ +CE7A;CE7A;110F 1161 11AD;CE7A;110F 1161 11AD; # (칺; 칺; 칺; 칺; 칺; ) HANGUL SYLLABLE KANH +CE7B;CE7B;110F 1161 11AE;CE7B;110F 1161 11AE; # (칻; 칻; 칻; 칻; 칻; ) HANGUL SYLLABLE KAD +CE7C;CE7C;110F 1161 11AF;CE7C;110F 1161 11AF; # (칼; 칼; 칼; 칼; 칼; ) HANGUL SYLLABLE KAL +CE7D;CE7D;110F 1161 11B0;CE7D;110F 1161 11B0; # (칽; 칽; 칽; 칽; 칽; ) HANGUL SYLLABLE KALG +CE7E;CE7E;110F 1161 11B1;CE7E;110F 1161 11B1; # (칾; 칾; 칾; 칾; 칾; ) HANGUL SYLLABLE KALM +CE7F;CE7F;110F 1161 11B2;CE7F;110F 1161 11B2; # (칿; 칿; 칿; 칿; 칿; ) HANGUL SYLLABLE KALB +CE80;CE80;110F 1161 11B3;CE80;110F 1161 11B3; # (캀; 캀; 캀; 캀; 캀; ) HANGUL SYLLABLE KALS +CE81;CE81;110F 1161 11B4;CE81;110F 1161 11B4; # (캁; 캁; 캁; 캁; 캁; ) HANGUL SYLLABLE KALT +CE82;CE82;110F 1161 11B5;CE82;110F 1161 11B5; # (캂; 캂; 캂; 캂; 캂; ) HANGUL SYLLABLE KALP +CE83;CE83;110F 1161 11B6;CE83;110F 1161 11B6; # (캃; 캃; 캃; 캃; 캃; ) HANGUL SYLLABLE KALH +CE84;CE84;110F 1161 11B7;CE84;110F 1161 11B7; # (캄; 캄; 캄; 캄; 캄; ) HANGUL SYLLABLE KAM +CE85;CE85;110F 1161 11B8;CE85;110F 1161 11B8; # (캅; 캅; 캅; 캅; 캅; ) HANGUL SYLLABLE KAB +CE86;CE86;110F 1161 11B9;CE86;110F 1161 11B9; # (캆; 캆; 캆; 캆; 캆; ) HANGUL SYLLABLE KABS +CE87;CE87;110F 1161 11BA;CE87;110F 1161 11BA; # (캇; 캇; 캇; 캇; 캇; ) HANGUL SYLLABLE KAS +CE88;CE88;110F 1161 11BB;CE88;110F 1161 11BB; # (캈; 캈; 캈; 캈; 캈; ) HANGUL SYLLABLE KASS +CE89;CE89;110F 1161 11BC;CE89;110F 1161 11BC; # (캉; 캉; 캉; 캉; 캉; ) HANGUL SYLLABLE KANG +CE8A;CE8A;110F 1161 11BD;CE8A;110F 1161 11BD; # (캊; 캊; 캊; 캊; 캊; ) HANGUL SYLLABLE KAJ +CE8B;CE8B;110F 1161 11BE;CE8B;110F 1161 11BE; # (캋; 캋; 캋; 캋; 캋; ) HANGUL SYLLABLE KAC +CE8C;CE8C;110F 1161 11BF;CE8C;110F 1161 11BF; # (캌; 캌; 캌; 캌; 캌; ) HANGUL SYLLABLE KAK +CE8D;CE8D;110F 1161 11C0;CE8D;110F 1161 11C0; # (캍; 캍; 캍; 캍; 캍; ) HANGUL SYLLABLE KAT +CE8E;CE8E;110F 1161 11C1;CE8E;110F 1161 11C1; # (캎; 캎; 캎; 캎; 캎; ) HANGUL SYLLABLE KAP +CE8F;CE8F;110F 1161 11C2;CE8F;110F 1161 11C2; # (캏; 캏; 캏; 캏; 캏; ) HANGUL SYLLABLE KAH +CE90;CE90;110F 1162;CE90;110F 1162; # (캐; 캐; 캐; 캐; 캐; ) HANGUL SYLLABLE KAE +CE91;CE91;110F 1162 11A8;CE91;110F 1162 11A8; # (캑; 캑; 캑; 캑; 캑; ) HANGUL SYLLABLE KAEG +CE92;CE92;110F 1162 11A9;CE92;110F 1162 11A9; # (캒; 캒; 캒; 캒; 캒; ) HANGUL SYLLABLE KAEGG +CE93;CE93;110F 1162 11AA;CE93;110F 1162 11AA; # (캓; 캓; 캓; 캓; 캓; ) HANGUL SYLLABLE KAEGS +CE94;CE94;110F 1162 11AB;CE94;110F 1162 11AB; # (캔; 캔; 캔; 캔; 캔; ) HANGUL SYLLABLE KAEN +CE95;CE95;110F 1162 11AC;CE95;110F 1162 11AC; # (캕; 캕; 캕; 캕; 캕; ) HANGUL SYLLABLE KAENJ +CE96;CE96;110F 1162 11AD;CE96;110F 1162 11AD; # (캖; 캖; 캖; 캖; 캖; ) HANGUL SYLLABLE KAENH +CE97;CE97;110F 1162 11AE;CE97;110F 1162 11AE; # (캗; 캗; 캗; 캗; 캗; ) HANGUL SYLLABLE KAED +CE98;CE98;110F 1162 11AF;CE98;110F 1162 11AF; # (캘; 캘; 캘; 캘; 캘; ) HANGUL SYLLABLE KAEL +CE99;CE99;110F 1162 11B0;CE99;110F 1162 11B0; # (캙; 캙; 캙; 캙; 캙; ) HANGUL SYLLABLE KAELG +CE9A;CE9A;110F 1162 11B1;CE9A;110F 1162 11B1; # (캚; 캚; 캚; 캚; 캚; ) HANGUL SYLLABLE KAELM +CE9B;CE9B;110F 1162 11B2;CE9B;110F 1162 11B2; # (캛; 캛; 캛; 캛; 캛; ) HANGUL SYLLABLE KAELB +CE9C;CE9C;110F 1162 11B3;CE9C;110F 1162 11B3; # (캜; 캜; 캜; 캜; 캜; ) HANGUL SYLLABLE KAELS +CE9D;CE9D;110F 1162 11B4;CE9D;110F 1162 11B4; # (캝; 캝; 캝; 캝; 캝; ) HANGUL SYLLABLE KAELT +CE9E;CE9E;110F 1162 11B5;CE9E;110F 1162 11B5; # (캞; 캞; 캞; 캞; 캞; ) HANGUL SYLLABLE KAELP +CE9F;CE9F;110F 1162 11B6;CE9F;110F 1162 11B6; # (캟; 캟; 캟; 캟; 캟; ) HANGUL SYLLABLE KAELH +CEA0;CEA0;110F 1162 11B7;CEA0;110F 1162 11B7; # (캠; 캠; 캠; 캠; 캠; ) HANGUL SYLLABLE KAEM +CEA1;CEA1;110F 1162 11B8;CEA1;110F 1162 11B8; # (캡; 캡; 캡; 캡; 캡; ) HANGUL SYLLABLE KAEB +CEA2;CEA2;110F 1162 11B9;CEA2;110F 1162 11B9; # (캢; 캢; 캢; 캢; 캢; ) HANGUL SYLLABLE KAEBS +CEA3;CEA3;110F 1162 11BA;CEA3;110F 1162 11BA; # (캣; 캣; 캣; 캣; 캣; ) HANGUL SYLLABLE KAES +CEA4;CEA4;110F 1162 11BB;CEA4;110F 1162 11BB; # (캤; 캤; 캤; 캤; 캤; ) HANGUL SYLLABLE KAESS +CEA5;CEA5;110F 1162 11BC;CEA5;110F 1162 11BC; # (캥; 캥; 캥; 캥; 캥; ) HANGUL SYLLABLE KAENG +CEA6;CEA6;110F 1162 11BD;CEA6;110F 1162 11BD; # (캦; 캦; 캦; 캦; 캦; ) HANGUL SYLLABLE KAEJ +CEA7;CEA7;110F 1162 11BE;CEA7;110F 1162 11BE; # (캧; 캧; 캧; 캧; 캧; ) HANGUL SYLLABLE KAEC +CEA8;CEA8;110F 1162 11BF;CEA8;110F 1162 11BF; # (캨; 캨; 캨; 캨; 캨; ) HANGUL SYLLABLE KAEK +CEA9;CEA9;110F 1162 11C0;CEA9;110F 1162 11C0; # (캩; 캩; 캩; 캩; 캩; ) HANGUL SYLLABLE KAET +CEAA;CEAA;110F 1162 11C1;CEAA;110F 1162 11C1; # (캪; 캪; 캪; 캪; 캪; ) HANGUL SYLLABLE KAEP +CEAB;CEAB;110F 1162 11C2;CEAB;110F 1162 11C2; # (캫; 캫; 캫; 캫; 캫; ) HANGUL SYLLABLE KAEH +CEAC;CEAC;110F 1163;CEAC;110F 1163; # (캬; 캬; 캬; 캬; 캬; ) HANGUL SYLLABLE KYA +CEAD;CEAD;110F 1163 11A8;CEAD;110F 1163 11A8; # (캭; 캭; 캭; 캭; 캭; ) HANGUL SYLLABLE KYAG +CEAE;CEAE;110F 1163 11A9;CEAE;110F 1163 11A9; # (캮; 캮; 캮; 캮; 캮; ) HANGUL SYLLABLE KYAGG +CEAF;CEAF;110F 1163 11AA;CEAF;110F 1163 11AA; # (캯; 캯; 캯; 캯; 캯; ) HANGUL SYLLABLE KYAGS +CEB0;CEB0;110F 1163 11AB;CEB0;110F 1163 11AB; # (캰; 캰; 캰; 캰; 캰; ) HANGUL SYLLABLE KYAN +CEB1;CEB1;110F 1163 11AC;CEB1;110F 1163 11AC; # (캱; 캱; 캱; 캱; 캱; ) HANGUL SYLLABLE KYANJ +CEB2;CEB2;110F 1163 11AD;CEB2;110F 1163 11AD; # (캲; 캲; 캲; 캲; 캲; ) HANGUL SYLLABLE KYANH +CEB3;CEB3;110F 1163 11AE;CEB3;110F 1163 11AE; # (캳; 캳; 캳; 캳; 캳; ) HANGUL SYLLABLE KYAD +CEB4;CEB4;110F 1163 11AF;CEB4;110F 1163 11AF; # (캴; 캴; 캴; 캴; 캴; ) HANGUL SYLLABLE KYAL +CEB5;CEB5;110F 1163 11B0;CEB5;110F 1163 11B0; # (캵; 캵; 캵; 캵; 캵; ) HANGUL SYLLABLE KYALG +CEB6;CEB6;110F 1163 11B1;CEB6;110F 1163 11B1; # (캶; 캶; 캶; 캶; 캶; ) HANGUL SYLLABLE KYALM +CEB7;CEB7;110F 1163 11B2;CEB7;110F 1163 11B2; # (캷; 캷; 캷; 캷; 캷; ) HANGUL SYLLABLE KYALB +CEB8;CEB8;110F 1163 11B3;CEB8;110F 1163 11B3; # (캸; 캸; 캸; 캸; 캸; ) HANGUL SYLLABLE KYALS +CEB9;CEB9;110F 1163 11B4;CEB9;110F 1163 11B4; # (캹; 캹; 캹; 캹; 캹; ) HANGUL SYLLABLE KYALT +CEBA;CEBA;110F 1163 11B5;CEBA;110F 1163 11B5; # (캺; 캺; 캺; 캺; 캺; ) HANGUL SYLLABLE KYALP +CEBB;CEBB;110F 1163 11B6;CEBB;110F 1163 11B6; # (캻; 캻; 캻; 캻; 캻; ) HANGUL SYLLABLE KYALH +CEBC;CEBC;110F 1163 11B7;CEBC;110F 1163 11B7; # (캼; 캼; 캼; 캼; 캼; ) HANGUL SYLLABLE KYAM +CEBD;CEBD;110F 1163 11B8;CEBD;110F 1163 11B8; # (캽; 캽; 캽; 캽; 캽; ) HANGUL SYLLABLE KYAB +CEBE;CEBE;110F 1163 11B9;CEBE;110F 1163 11B9; # (캾; 캾; 캾; 캾; 캾; ) HANGUL SYLLABLE KYABS +CEBF;CEBF;110F 1163 11BA;CEBF;110F 1163 11BA; # (캿; 캿; 캿; 캿; 캿; ) HANGUL SYLLABLE KYAS +CEC0;CEC0;110F 1163 11BB;CEC0;110F 1163 11BB; # (컀; 컀; 컀; 컀; 컀; ) HANGUL SYLLABLE KYASS +CEC1;CEC1;110F 1163 11BC;CEC1;110F 1163 11BC; # (컁; 컁; 컁; 컁; 컁; ) HANGUL SYLLABLE KYANG +CEC2;CEC2;110F 1163 11BD;CEC2;110F 1163 11BD; # (컂; 컂; 컂; 컂; 컂; ) HANGUL SYLLABLE KYAJ +CEC3;CEC3;110F 1163 11BE;CEC3;110F 1163 11BE; # (컃; 컃; 컃; 컃; 컃; ) HANGUL SYLLABLE KYAC +CEC4;CEC4;110F 1163 11BF;CEC4;110F 1163 11BF; # (컄; 컄; 컄; 컄; 컄; ) HANGUL SYLLABLE KYAK +CEC5;CEC5;110F 1163 11C0;CEC5;110F 1163 11C0; # (컅; 컅; 컅; 컅; 컅; ) HANGUL SYLLABLE KYAT +CEC6;CEC6;110F 1163 11C1;CEC6;110F 1163 11C1; # (컆; 컆; 컆; 컆; 컆; ) HANGUL SYLLABLE KYAP +CEC7;CEC7;110F 1163 11C2;CEC7;110F 1163 11C2; # (컇; 컇; 컇; 컇; 컇; ) HANGUL SYLLABLE KYAH +CEC8;CEC8;110F 1164;CEC8;110F 1164; # (컈; 컈; 컈; 컈; 컈; ) HANGUL SYLLABLE KYAE +CEC9;CEC9;110F 1164 11A8;CEC9;110F 1164 11A8; # (컉; 컉; 컉; 컉; 컉; ) HANGUL SYLLABLE KYAEG +CECA;CECA;110F 1164 11A9;CECA;110F 1164 11A9; # (컊; 컊; 컊; 컊; 컊; ) HANGUL SYLLABLE KYAEGG +CECB;CECB;110F 1164 11AA;CECB;110F 1164 11AA; # (컋; 컋; 컋; 컋; 컋; ) HANGUL SYLLABLE KYAEGS +CECC;CECC;110F 1164 11AB;CECC;110F 1164 11AB; # (컌; 컌; 컌; 컌; 컌; ) HANGUL SYLLABLE KYAEN +CECD;CECD;110F 1164 11AC;CECD;110F 1164 11AC; # (컍; 컍; 컍; 컍; 컍; ) HANGUL SYLLABLE KYAENJ +CECE;CECE;110F 1164 11AD;CECE;110F 1164 11AD; # (컎; 컎; 컎; 컎; 컎; ) HANGUL SYLLABLE KYAENH +CECF;CECF;110F 1164 11AE;CECF;110F 1164 11AE; # (컏; 컏; 컏; 컏; 컏; ) HANGUL SYLLABLE KYAED +CED0;CED0;110F 1164 11AF;CED0;110F 1164 11AF; # (컐; 컐; 컐; 컐; 컐; ) HANGUL SYLLABLE KYAEL +CED1;CED1;110F 1164 11B0;CED1;110F 1164 11B0; # (컑; 컑; 컑; 컑; 컑; ) HANGUL SYLLABLE KYAELG +CED2;CED2;110F 1164 11B1;CED2;110F 1164 11B1; # (컒; 컒; 컒; 컒; 컒; ) HANGUL SYLLABLE KYAELM +CED3;CED3;110F 1164 11B2;CED3;110F 1164 11B2; # (컓; 컓; 컓; 컓; 컓; ) HANGUL SYLLABLE KYAELB +CED4;CED4;110F 1164 11B3;CED4;110F 1164 11B3; # (컔; 컔; 컔; 컔; 컔; ) HANGUL SYLLABLE KYAELS +CED5;CED5;110F 1164 11B4;CED5;110F 1164 11B4; # (컕; 컕; 컕; 컕; 컕; ) HANGUL SYLLABLE KYAELT +CED6;CED6;110F 1164 11B5;CED6;110F 1164 11B5; # (컖; 컖; 컖; 컖; 컖; ) HANGUL SYLLABLE KYAELP +CED7;CED7;110F 1164 11B6;CED7;110F 1164 11B6; # (컗; 컗; 컗; 컗; 컗; ) HANGUL SYLLABLE KYAELH +CED8;CED8;110F 1164 11B7;CED8;110F 1164 11B7; # (컘; 컘; 컘; 컘; 컘; ) HANGUL SYLLABLE KYAEM +CED9;CED9;110F 1164 11B8;CED9;110F 1164 11B8; # (컙; 컙; 컙; 컙; 컙; ) HANGUL SYLLABLE KYAEB +CEDA;CEDA;110F 1164 11B9;CEDA;110F 1164 11B9; # (컚; 컚; 컚; 컚; 컚; ) HANGUL SYLLABLE KYAEBS +CEDB;CEDB;110F 1164 11BA;CEDB;110F 1164 11BA; # (컛; 컛; 컛; 컛; 컛; ) HANGUL SYLLABLE KYAES +CEDC;CEDC;110F 1164 11BB;CEDC;110F 1164 11BB; # (컜; 컜; 컜; 컜; 컜; ) HANGUL SYLLABLE KYAESS +CEDD;CEDD;110F 1164 11BC;CEDD;110F 1164 11BC; # (컝; 컝; 컝; 컝; 컝; ) HANGUL SYLLABLE KYAENG +CEDE;CEDE;110F 1164 11BD;CEDE;110F 1164 11BD; # (컞; 컞; 컞; 컞; 컞; ) HANGUL SYLLABLE KYAEJ +CEDF;CEDF;110F 1164 11BE;CEDF;110F 1164 11BE; # (컟; 컟; 컟; 컟; 컟; ) HANGUL SYLLABLE KYAEC +CEE0;CEE0;110F 1164 11BF;CEE0;110F 1164 11BF; # (컠; 컠; 컠; 컠; 컠; ) HANGUL SYLLABLE KYAEK +CEE1;CEE1;110F 1164 11C0;CEE1;110F 1164 11C0; # (컡; 컡; 컡; 컡; 컡; ) HANGUL SYLLABLE KYAET +CEE2;CEE2;110F 1164 11C1;CEE2;110F 1164 11C1; # (컢; 컢; 컢; 컢; 컢; ) HANGUL SYLLABLE KYAEP +CEE3;CEE3;110F 1164 11C2;CEE3;110F 1164 11C2; # (컣; 컣; 컣; 컣; 컣; ) HANGUL SYLLABLE KYAEH +CEE4;CEE4;110F 1165;CEE4;110F 1165; # (커; 커; 커; 커; 커; ) HANGUL SYLLABLE KEO +CEE5;CEE5;110F 1165 11A8;CEE5;110F 1165 11A8; # (컥; 컥; 컥; 컥; 컥; ) HANGUL SYLLABLE KEOG +CEE6;CEE6;110F 1165 11A9;CEE6;110F 1165 11A9; # (컦; 컦; 컦; 컦; 컦; ) HANGUL SYLLABLE KEOGG +CEE7;CEE7;110F 1165 11AA;CEE7;110F 1165 11AA; # (컧; 컧; 컧; 컧; 컧; ) HANGUL SYLLABLE KEOGS +CEE8;CEE8;110F 1165 11AB;CEE8;110F 1165 11AB; # (컨; 컨; 컨; 컨; 컨; ) HANGUL SYLLABLE KEON +CEE9;CEE9;110F 1165 11AC;CEE9;110F 1165 11AC; # (컩; 컩; 컩; 컩; 컩; ) HANGUL SYLLABLE KEONJ +CEEA;CEEA;110F 1165 11AD;CEEA;110F 1165 11AD; # (컪; 컪; 컪; 컪; 컪; ) HANGUL SYLLABLE KEONH +CEEB;CEEB;110F 1165 11AE;CEEB;110F 1165 11AE; # (컫; 컫; 컫; 컫; 컫; ) HANGUL SYLLABLE KEOD +CEEC;CEEC;110F 1165 11AF;CEEC;110F 1165 11AF; # (컬; 컬; 컬; 컬; 컬; ) HANGUL SYLLABLE KEOL +CEED;CEED;110F 1165 11B0;CEED;110F 1165 11B0; # (컭; 컭; 컭; 컭; 컭; ) HANGUL SYLLABLE KEOLG +CEEE;CEEE;110F 1165 11B1;CEEE;110F 1165 11B1; # (컮; 컮; 컮; 컮; 컮; ) HANGUL SYLLABLE KEOLM +CEEF;CEEF;110F 1165 11B2;CEEF;110F 1165 11B2; # (컯; 컯; 컯; 컯; 컯; ) HANGUL SYLLABLE KEOLB +CEF0;CEF0;110F 1165 11B3;CEF0;110F 1165 11B3; # (컰; 컰; 컰; 컰; 컰; ) HANGUL SYLLABLE KEOLS +CEF1;CEF1;110F 1165 11B4;CEF1;110F 1165 11B4; # (컱; 컱; 컱; 컱; 컱; ) HANGUL SYLLABLE KEOLT +CEF2;CEF2;110F 1165 11B5;CEF2;110F 1165 11B5; # (컲; 컲; 컲; 컲; 컲; ) HANGUL SYLLABLE KEOLP +CEF3;CEF3;110F 1165 11B6;CEF3;110F 1165 11B6; # (컳; 컳; 컳; 컳; 컳; ) HANGUL SYLLABLE KEOLH +CEF4;CEF4;110F 1165 11B7;CEF4;110F 1165 11B7; # (컴; 컴; 컴; 컴; 컴; ) HANGUL SYLLABLE KEOM +CEF5;CEF5;110F 1165 11B8;CEF5;110F 1165 11B8; # (컵; 컵; 컵; 컵; 컵; ) HANGUL SYLLABLE KEOB +CEF6;CEF6;110F 1165 11B9;CEF6;110F 1165 11B9; # (컶; 컶; 컶; 컶; 컶; ) HANGUL SYLLABLE KEOBS +CEF7;CEF7;110F 1165 11BA;CEF7;110F 1165 11BA; # (컷; 컷; 컷; 컷; 컷; ) HANGUL SYLLABLE KEOS +CEF8;CEF8;110F 1165 11BB;CEF8;110F 1165 11BB; # (컸; 컸; 컸; 컸; 컸; ) HANGUL SYLLABLE KEOSS +CEF9;CEF9;110F 1165 11BC;CEF9;110F 1165 11BC; # (컹; 컹; 컹; 컹; 컹; ) HANGUL SYLLABLE KEONG +CEFA;CEFA;110F 1165 11BD;CEFA;110F 1165 11BD; # (컺; 컺; 컺; 컺; 컺; ) HANGUL SYLLABLE KEOJ +CEFB;CEFB;110F 1165 11BE;CEFB;110F 1165 11BE; # (컻; 컻; 컻; 컻; 컻; ) HANGUL SYLLABLE KEOC +CEFC;CEFC;110F 1165 11BF;CEFC;110F 1165 11BF; # (컼; 컼; 컼; 컼; 컼; ) HANGUL SYLLABLE KEOK +CEFD;CEFD;110F 1165 11C0;CEFD;110F 1165 11C0; # (컽; 컽; 컽; 컽; 컽; ) HANGUL SYLLABLE KEOT +CEFE;CEFE;110F 1165 11C1;CEFE;110F 1165 11C1; # (컾; 컾; 컾; 컾; 컾; ) HANGUL SYLLABLE KEOP +CEFF;CEFF;110F 1165 11C2;CEFF;110F 1165 11C2; # (컿; 컿; 컿; 컿; 컿; ) HANGUL SYLLABLE KEOH +CF00;CF00;110F 1166;CF00;110F 1166; # (케; 케; 케; 케; 케; ) HANGUL SYLLABLE KE +CF01;CF01;110F 1166 11A8;CF01;110F 1166 11A8; # (켁; 켁; 켁; 켁; 켁; ) HANGUL SYLLABLE KEG +CF02;CF02;110F 1166 11A9;CF02;110F 1166 11A9; # (켂; 켂; 켂; 켂; 켂; ) HANGUL SYLLABLE KEGG +CF03;CF03;110F 1166 11AA;CF03;110F 1166 11AA; # (켃; 켃; 켃; 켃; 켃; ) HANGUL SYLLABLE KEGS +CF04;CF04;110F 1166 11AB;CF04;110F 1166 11AB; # (켄; 켄; 켄; 켄; 켄; ) HANGUL SYLLABLE KEN +CF05;CF05;110F 1166 11AC;CF05;110F 1166 11AC; # (켅; 켅; 켅; 켅; 켅; ) HANGUL SYLLABLE KENJ +CF06;CF06;110F 1166 11AD;CF06;110F 1166 11AD; # (켆; 켆; 켆; 켆; 켆; ) HANGUL SYLLABLE KENH +CF07;CF07;110F 1166 11AE;CF07;110F 1166 11AE; # (켇; 켇; 켇; 켇; 켇; ) HANGUL SYLLABLE KED +CF08;CF08;110F 1166 11AF;CF08;110F 1166 11AF; # (켈; 켈; 켈; 켈; 켈; ) HANGUL SYLLABLE KEL +CF09;CF09;110F 1166 11B0;CF09;110F 1166 11B0; # (켉; 켉; 켉; 켉; 켉; ) HANGUL SYLLABLE KELG +CF0A;CF0A;110F 1166 11B1;CF0A;110F 1166 11B1; # (켊; 켊; 켊; 켊; 켊; ) HANGUL SYLLABLE KELM +CF0B;CF0B;110F 1166 11B2;CF0B;110F 1166 11B2; # (켋; 켋; 켋; 켋; 켋; ) HANGUL SYLLABLE KELB +CF0C;CF0C;110F 1166 11B3;CF0C;110F 1166 11B3; # (켌; 켌; 켌; 켌; 켌; ) HANGUL SYLLABLE KELS +CF0D;CF0D;110F 1166 11B4;CF0D;110F 1166 11B4; # (켍; 켍; 켍; 켍; 켍; ) HANGUL SYLLABLE KELT +CF0E;CF0E;110F 1166 11B5;CF0E;110F 1166 11B5; # (켎; 켎; 켎; 켎; 켎; ) HANGUL SYLLABLE KELP +CF0F;CF0F;110F 1166 11B6;CF0F;110F 1166 11B6; # (켏; 켏; 켏; 켏; 켏; ) HANGUL SYLLABLE KELH +CF10;CF10;110F 1166 11B7;CF10;110F 1166 11B7; # (켐; 켐; 켐; 켐; 켐; ) HANGUL SYLLABLE KEM +CF11;CF11;110F 1166 11B8;CF11;110F 1166 11B8; # (켑; 켑; 켑; 켑; 켑; ) HANGUL SYLLABLE KEB +CF12;CF12;110F 1166 11B9;CF12;110F 1166 11B9; # (켒; 켒; 켒; 켒; 켒; ) HANGUL SYLLABLE KEBS +CF13;CF13;110F 1166 11BA;CF13;110F 1166 11BA; # (켓; 켓; 켓; 켓; 켓; ) HANGUL SYLLABLE KES +CF14;CF14;110F 1166 11BB;CF14;110F 1166 11BB; # (켔; 켔; 켔; 켔; 켔; ) HANGUL SYLLABLE KESS +CF15;CF15;110F 1166 11BC;CF15;110F 1166 11BC; # (켕; 켕; 켕; 켕; 켕; ) HANGUL SYLLABLE KENG +CF16;CF16;110F 1166 11BD;CF16;110F 1166 11BD; # (켖; 켖; 켖; 켖; 켖; ) HANGUL SYLLABLE KEJ +CF17;CF17;110F 1166 11BE;CF17;110F 1166 11BE; # (켗; 켗; 켗; 켗; 켗; ) HANGUL SYLLABLE KEC +CF18;CF18;110F 1166 11BF;CF18;110F 1166 11BF; # (켘; 켘; 켘; 켘; 켘; ) HANGUL SYLLABLE KEK +CF19;CF19;110F 1166 11C0;CF19;110F 1166 11C0; # (켙; 켙; 켙; 켙; 켙; ) HANGUL SYLLABLE KET +CF1A;CF1A;110F 1166 11C1;CF1A;110F 1166 11C1; # (켚; 켚; 켚; 켚; 켚; ) HANGUL SYLLABLE KEP +CF1B;CF1B;110F 1166 11C2;CF1B;110F 1166 11C2; # (켛; 켛; 켛; 켛; 켛; ) HANGUL SYLLABLE KEH +CF1C;CF1C;110F 1167;CF1C;110F 1167; # (켜; 켜; 켜; 켜; 켜; ) HANGUL SYLLABLE KYEO +CF1D;CF1D;110F 1167 11A8;CF1D;110F 1167 11A8; # (켝; 켝; 켝; 켝; 켝; ) HANGUL SYLLABLE KYEOG +CF1E;CF1E;110F 1167 11A9;CF1E;110F 1167 11A9; # (켞; 켞; 켞; 켞; 켞; ) HANGUL SYLLABLE KYEOGG +CF1F;CF1F;110F 1167 11AA;CF1F;110F 1167 11AA; # (켟; 켟; 켟; 켟; 켟; ) HANGUL SYLLABLE KYEOGS +CF20;CF20;110F 1167 11AB;CF20;110F 1167 11AB; # (켠; 켠; 켠; 켠; 켠; ) HANGUL SYLLABLE KYEON +CF21;CF21;110F 1167 11AC;CF21;110F 1167 11AC; # (켡; 켡; 켡; 켡; 켡; ) HANGUL SYLLABLE KYEONJ +CF22;CF22;110F 1167 11AD;CF22;110F 1167 11AD; # (켢; 켢; 켢; 켢; 켢; ) HANGUL SYLLABLE KYEONH +CF23;CF23;110F 1167 11AE;CF23;110F 1167 11AE; # (켣; 켣; 켣; 켣; 켣; ) HANGUL SYLLABLE KYEOD +CF24;CF24;110F 1167 11AF;CF24;110F 1167 11AF; # (켤; 켤; 켤; 켤; 켤; ) HANGUL SYLLABLE KYEOL +CF25;CF25;110F 1167 11B0;CF25;110F 1167 11B0; # (켥; 켥; 켥; 켥; 켥; ) HANGUL SYLLABLE KYEOLG +CF26;CF26;110F 1167 11B1;CF26;110F 1167 11B1; # (켦; 켦; 켦; 켦; 켦; ) HANGUL SYLLABLE KYEOLM +CF27;CF27;110F 1167 11B2;CF27;110F 1167 11B2; # (켧; 켧; 켧; 켧; 켧; ) HANGUL SYLLABLE KYEOLB +CF28;CF28;110F 1167 11B3;CF28;110F 1167 11B3; # (켨; 켨; 켨; 켨; 켨; ) HANGUL SYLLABLE KYEOLS +CF29;CF29;110F 1167 11B4;CF29;110F 1167 11B4; # (켩; 켩; 켩; 켩; 켩; ) HANGUL SYLLABLE KYEOLT +CF2A;CF2A;110F 1167 11B5;CF2A;110F 1167 11B5; # (켪; 켪; 켪; 켪; 켪; ) HANGUL SYLLABLE KYEOLP +CF2B;CF2B;110F 1167 11B6;CF2B;110F 1167 11B6; # (켫; 켫; 켫; 켫; 켫; ) HANGUL SYLLABLE KYEOLH +CF2C;CF2C;110F 1167 11B7;CF2C;110F 1167 11B7; # (켬; 켬; 켬; 켬; 켬; ) HANGUL SYLLABLE KYEOM +CF2D;CF2D;110F 1167 11B8;CF2D;110F 1167 11B8; # (켭; 켭; 켭; 켭; 켭; ) HANGUL SYLLABLE KYEOB +CF2E;CF2E;110F 1167 11B9;CF2E;110F 1167 11B9; # (켮; 켮; 켮; 켮; 켮; ) HANGUL SYLLABLE KYEOBS +CF2F;CF2F;110F 1167 11BA;CF2F;110F 1167 11BA; # (켯; 켯; 켯; 켯; 켯; ) HANGUL SYLLABLE KYEOS +CF30;CF30;110F 1167 11BB;CF30;110F 1167 11BB; # (켰; 켰; 켰; 켰; 켰; ) HANGUL SYLLABLE KYEOSS +CF31;CF31;110F 1167 11BC;CF31;110F 1167 11BC; # (켱; 켱; 켱; 켱; 켱; ) HANGUL SYLLABLE KYEONG +CF32;CF32;110F 1167 11BD;CF32;110F 1167 11BD; # (켲; 켲; 켲; 켲; 켲; ) HANGUL SYLLABLE KYEOJ +CF33;CF33;110F 1167 11BE;CF33;110F 1167 11BE; # (켳; 켳; 켳; 켳; 켳; ) HANGUL SYLLABLE KYEOC +CF34;CF34;110F 1167 11BF;CF34;110F 1167 11BF; # (켴; 켴; 켴; 켴; 켴; ) HANGUL SYLLABLE KYEOK +CF35;CF35;110F 1167 11C0;CF35;110F 1167 11C0; # (켵; 켵; 켵; 켵; 켵; ) HANGUL SYLLABLE KYEOT +CF36;CF36;110F 1167 11C1;CF36;110F 1167 11C1; # (켶; 켶; 켶; 켶; 켶; ) HANGUL SYLLABLE KYEOP +CF37;CF37;110F 1167 11C2;CF37;110F 1167 11C2; # (켷; 켷; 켷; 켷; 켷; ) HANGUL SYLLABLE KYEOH +CF38;CF38;110F 1168;CF38;110F 1168; # (켸; 켸; 켸; 켸; 켸; ) HANGUL SYLLABLE KYE +CF39;CF39;110F 1168 11A8;CF39;110F 1168 11A8; # (켹; 켹; 켹; 켹; 켹; ) HANGUL SYLLABLE KYEG +CF3A;CF3A;110F 1168 11A9;CF3A;110F 1168 11A9; # (켺; 켺; 켺; 켺; 켺; ) HANGUL SYLLABLE KYEGG +CF3B;CF3B;110F 1168 11AA;CF3B;110F 1168 11AA; # (켻; 켻; 켻; 켻; 켻; ) HANGUL SYLLABLE KYEGS +CF3C;CF3C;110F 1168 11AB;CF3C;110F 1168 11AB; # (켼; 켼; 켼; 켼; 켼; ) HANGUL SYLLABLE KYEN +CF3D;CF3D;110F 1168 11AC;CF3D;110F 1168 11AC; # (켽; 켽; 켽; 켽; 켽; ) HANGUL SYLLABLE KYENJ +CF3E;CF3E;110F 1168 11AD;CF3E;110F 1168 11AD; # (켾; 켾; 켾; 켾; 켾; ) HANGUL SYLLABLE KYENH +CF3F;CF3F;110F 1168 11AE;CF3F;110F 1168 11AE; # (켿; 켿; 켿; 켿; 켿; ) HANGUL SYLLABLE KYED +CF40;CF40;110F 1168 11AF;CF40;110F 1168 11AF; # (콀; 콀; 콀; 콀; 콀; ) HANGUL SYLLABLE KYEL +CF41;CF41;110F 1168 11B0;CF41;110F 1168 11B0; # (콁; 콁; 콁; 콁; 콁; ) HANGUL SYLLABLE KYELG +CF42;CF42;110F 1168 11B1;CF42;110F 1168 11B1; # (콂; 콂; 콂; 콂; 콂; ) HANGUL SYLLABLE KYELM +CF43;CF43;110F 1168 11B2;CF43;110F 1168 11B2; # (콃; 콃; 콃; 콃; 콃; ) HANGUL SYLLABLE KYELB +CF44;CF44;110F 1168 11B3;CF44;110F 1168 11B3; # (콄; 콄; 콄; 콄; 콄; ) HANGUL SYLLABLE KYELS +CF45;CF45;110F 1168 11B4;CF45;110F 1168 11B4; # (콅; 콅; 콅; 콅; 콅; ) HANGUL SYLLABLE KYELT +CF46;CF46;110F 1168 11B5;CF46;110F 1168 11B5; # (콆; 콆; 콆; 콆; 콆; ) HANGUL SYLLABLE KYELP +CF47;CF47;110F 1168 11B6;CF47;110F 1168 11B6; # (콇; 콇; 콇; 콇; 콇; ) HANGUL SYLLABLE KYELH +CF48;CF48;110F 1168 11B7;CF48;110F 1168 11B7; # (콈; 콈; 콈; 콈; 콈; ) HANGUL SYLLABLE KYEM +CF49;CF49;110F 1168 11B8;CF49;110F 1168 11B8; # (콉; 콉; 콉; 콉; 콉; ) HANGUL SYLLABLE KYEB +CF4A;CF4A;110F 1168 11B9;CF4A;110F 1168 11B9; # (콊; 콊; 콊; 콊; 콊; ) HANGUL SYLLABLE KYEBS +CF4B;CF4B;110F 1168 11BA;CF4B;110F 1168 11BA; # (콋; 콋; 콋; 콋; 콋; ) HANGUL SYLLABLE KYES +CF4C;CF4C;110F 1168 11BB;CF4C;110F 1168 11BB; # (콌; 콌; 콌; 콌; 콌; ) HANGUL SYLLABLE KYESS +CF4D;CF4D;110F 1168 11BC;CF4D;110F 1168 11BC; # (콍; 콍; 콍; 콍; 콍; ) HANGUL SYLLABLE KYENG +CF4E;CF4E;110F 1168 11BD;CF4E;110F 1168 11BD; # (콎; 콎; 콎; 콎; 콎; ) HANGUL SYLLABLE KYEJ +CF4F;CF4F;110F 1168 11BE;CF4F;110F 1168 11BE; # (콏; 콏; 콏; 콏; 콏; ) HANGUL SYLLABLE KYEC +CF50;CF50;110F 1168 11BF;CF50;110F 1168 11BF; # (콐; 콐; 콐; 콐; 콐; ) HANGUL SYLLABLE KYEK +CF51;CF51;110F 1168 11C0;CF51;110F 1168 11C0; # (콑; 콑; 콑; 콑; 콑; ) HANGUL SYLLABLE KYET +CF52;CF52;110F 1168 11C1;CF52;110F 1168 11C1; # (콒; 콒; 콒; 콒; 콒; ) HANGUL SYLLABLE KYEP +CF53;CF53;110F 1168 11C2;CF53;110F 1168 11C2; # (콓; 콓; 콓; 콓; 콓; ) HANGUL SYLLABLE KYEH +CF54;CF54;110F 1169;CF54;110F 1169; # (코; 코; 코; 코; 코; ) HANGUL SYLLABLE KO +CF55;CF55;110F 1169 11A8;CF55;110F 1169 11A8; # (콕; 콕; 콕; 콕; 콕; ) HANGUL SYLLABLE KOG +CF56;CF56;110F 1169 11A9;CF56;110F 1169 11A9; # (콖; 콖; 콖; 콖; 콖; ) HANGUL SYLLABLE KOGG +CF57;CF57;110F 1169 11AA;CF57;110F 1169 11AA; # (콗; 콗; 콗; 콗; 콗; ) HANGUL SYLLABLE KOGS +CF58;CF58;110F 1169 11AB;CF58;110F 1169 11AB; # (콘; 콘; 콘; 콘; 콘; ) HANGUL SYLLABLE KON +CF59;CF59;110F 1169 11AC;CF59;110F 1169 11AC; # (콙; 콙; 콙; 콙; 콙; ) HANGUL SYLLABLE KONJ +CF5A;CF5A;110F 1169 11AD;CF5A;110F 1169 11AD; # (콚; 콚; 콚; 콚; 콚; ) HANGUL SYLLABLE KONH +CF5B;CF5B;110F 1169 11AE;CF5B;110F 1169 11AE; # (콛; 콛; 콛; 콛; 콛; ) HANGUL SYLLABLE KOD +CF5C;CF5C;110F 1169 11AF;CF5C;110F 1169 11AF; # (콜; 콜; 콜; 콜; 콜; ) HANGUL SYLLABLE KOL +CF5D;CF5D;110F 1169 11B0;CF5D;110F 1169 11B0; # (콝; 콝; 콝; 콝; 콝; ) HANGUL SYLLABLE KOLG +CF5E;CF5E;110F 1169 11B1;CF5E;110F 1169 11B1; # (콞; 콞; 콞; 콞; 콞; ) HANGUL SYLLABLE KOLM +CF5F;CF5F;110F 1169 11B2;CF5F;110F 1169 11B2; # (콟; 콟; 콟; 콟; 콟; ) HANGUL SYLLABLE KOLB +CF60;CF60;110F 1169 11B3;CF60;110F 1169 11B3; # (콠; 콠; 콠; 콠; 콠; ) HANGUL SYLLABLE KOLS +CF61;CF61;110F 1169 11B4;CF61;110F 1169 11B4; # (콡; 콡; 콡; 콡; 콡; ) HANGUL SYLLABLE KOLT +CF62;CF62;110F 1169 11B5;CF62;110F 1169 11B5; # (콢; 콢; 콢; 콢; 콢; ) HANGUL SYLLABLE KOLP +CF63;CF63;110F 1169 11B6;CF63;110F 1169 11B6; # (콣; 콣; 콣; 콣; 콣; ) HANGUL SYLLABLE KOLH +CF64;CF64;110F 1169 11B7;CF64;110F 1169 11B7; # (콤; 콤; 콤; 콤; 콤; ) HANGUL SYLLABLE KOM +CF65;CF65;110F 1169 11B8;CF65;110F 1169 11B8; # (콥; 콥; 콥; 콥; 콥; ) HANGUL SYLLABLE KOB +CF66;CF66;110F 1169 11B9;CF66;110F 1169 11B9; # (콦; 콦; 콦; 콦; 콦; ) HANGUL SYLLABLE KOBS +CF67;CF67;110F 1169 11BA;CF67;110F 1169 11BA; # (콧; 콧; 콧; 콧; 콧; ) HANGUL SYLLABLE KOS +CF68;CF68;110F 1169 11BB;CF68;110F 1169 11BB; # (콨; 콨; 콨; 콨; 콨; ) HANGUL SYLLABLE KOSS +CF69;CF69;110F 1169 11BC;CF69;110F 1169 11BC; # (콩; 콩; 콩; 콩; 콩; ) HANGUL SYLLABLE KONG +CF6A;CF6A;110F 1169 11BD;CF6A;110F 1169 11BD; # (콪; 콪; 콪; 콪; 콪; ) HANGUL SYLLABLE KOJ +CF6B;CF6B;110F 1169 11BE;CF6B;110F 1169 11BE; # (콫; 콫; 콫; 콫; 콫; ) HANGUL SYLLABLE KOC +CF6C;CF6C;110F 1169 11BF;CF6C;110F 1169 11BF; # (콬; 콬; 콬; 콬; 콬; ) HANGUL SYLLABLE KOK +CF6D;CF6D;110F 1169 11C0;CF6D;110F 1169 11C0; # (콭; 콭; 콭; 콭; 콭; ) HANGUL SYLLABLE KOT +CF6E;CF6E;110F 1169 11C1;CF6E;110F 1169 11C1; # (콮; 콮; 콮; 콮; 콮; ) HANGUL SYLLABLE KOP +CF6F;CF6F;110F 1169 11C2;CF6F;110F 1169 11C2; # (콯; 콯; 콯; 콯; 콯; ) HANGUL SYLLABLE KOH +CF70;CF70;110F 116A;CF70;110F 116A; # (콰; 콰; 콰; 콰; 콰; ) HANGUL SYLLABLE KWA +CF71;CF71;110F 116A 11A8;CF71;110F 116A 11A8; # (콱; 콱; 콱; 콱; 콱; ) HANGUL SYLLABLE KWAG +CF72;CF72;110F 116A 11A9;CF72;110F 116A 11A9; # (콲; 콲; 콲; 콲; 콲; ) HANGUL SYLLABLE KWAGG +CF73;CF73;110F 116A 11AA;CF73;110F 116A 11AA; # (콳; 콳; 콳; 콳; 콳; ) HANGUL SYLLABLE KWAGS +CF74;CF74;110F 116A 11AB;CF74;110F 116A 11AB; # (콴; 콴; 콴; 콴; 콴; ) HANGUL SYLLABLE KWAN +CF75;CF75;110F 116A 11AC;CF75;110F 116A 11AC; # (콵; 콵; 콵; 콵; 콵; ) HANGUL SYLLABLE KWANJ +CF76;CF76;110F 116A 11AD;CF76;110F 116A 11AD; # (콶; 콶; 콶; 콶; 콶; ) HANGUL SYLLABLE KWANH +CF77;CF77;110F 116A 11AE;CF77;110F 116A 11AE; # (콷; 콷; 콷; 콷; 콷; ) HANGUL SYLLABLE KWAD +CF78;CF78;110F 116A 11AF;CF78;110F 116A 11AF; # (콸; 콸; 콸; 콸; 콸; ) HANGUL SYLLABLE KWAL +CF79;CF79;110F 116A 11B0;CF79;110F 116A 11B0; # (콹; 콹; 콹; 콹; 콹; ) HANGUL SYLLABLE KWALG +CF7A;CF7A;110F 116A 11B1;CF7A;110F 116A 11B1; # (콺; 콺; 콺; 콺; 콺; ) HANGUL SYLLABLE KWALM +CF7B;CF7B;110F 116A 11B2;CF7B;110F 116A 11B2; # (콻; 콻; 콻; 콻; 콻; ) HANGUL SYLLABLE KWALB +CF7C;CF7C;110F 116A 11B3;CF7C;110F 116A 11B3; # (콼; 콼; 콼; 콼; 콼; ) HANGUL SYLLABLE KWALS +CF7D;CF7D;110F 116A 11B4;CF7D;110F 116A 11B4; # (콽; 콽; 콽; 콽; 콽; ) HANGUL SYLLABLE KWALT +CF7E;CF7E;110F 116A 11B5;CF7E;110F 116A 11B5; # (콾; 콾; 콾; 콾; 콾; ) HANGUL SYLLABLE KWALP +CF7F;CF7F;110F 116A 11B6;CF7F;110F 116A 11B6; # (콿; 콿; 콿; 콿; 콿; ) HANGUL SYLLABLE KWALH +CF80;CF80;110F 116A 11B7;CF80;110F 116A 11B7; # (쾀; 쾀; 쾀; 쾀; 쾀; ) HANGUL SYLLABLE KWAM +CF81;CF81;110F 116A 11B8;CF81;110F 116A 11B8; # (쾁; 쾁; 쾁; 쾁; 쾁; ) HANGUL SYLLABLE KWAB +CF82;CF82;110F 116A 11B9;CF82;110F 116A 11B9; # (쾂; 쾂; 쾂; 쾂; 쾂; ) HANGUL SYLLABLE KWABS +CF83;CF83;110F 116A 11BA;CF83;110F 116A 11BA; # (쾃; 쾃; 쾃; 쾃; 쾃; ) HANGUL SYLLABLE KWAS +CF84;CF84;110F 116A 11BB;CF84;110F 116A 11BB; # (쾄; 쾄; 쾄; 쾄; 쾄; ) HANGUL SYLLABLE KWASS +CF85;CF85;110F 116A 11BC;CF85;110F 116A 11BC; # (쾅; 쾅; 쾅; 쾅; 쾅; ) HANGUL SYLLABLE KWANG +CF86;CF86;110F 116A 11BD;CF86;110F 116A 11BD; # (쾆; 쾆; 쾆; 쾆; 쾆; ) HANGUL SYLLABLE KWAJ +CF87;CF87;110F 116A 11BE;CF87;110F 116A 11BE; # (쾇; 쾇; 쾇; 쾇; 쾇; ) HANGUL SYLLABLE KWAC +CF88;CF88;110F 116A 11BF;CF88;110F 116A 11BF; # (쾈; 쾈; 쾈; 쾈; 쾈; ) HANGUL SYLLABLE KWAK +CF89;CF89;110F 116A 11C0;CF89;110F 116A 11C0; # (쾉; 쾉; 쾉; 쾉; 쾉; ) HANGUL SYLLABLE KWAT +CF8A;CF8A;110F 116A 11C1;CF8A;110F 116A 11C1; # (쾊; 쾊; 쾊; 쾊; 쾊; ) HANGUL SYLLABLE KWAP +CF8B;CF8B;110F 116A 11C2;CF8B;110F 116A 11C2; # (쾋; 쾋; 쾋; 쾋; 쾋; ) HANGUL SYLLABLE KWAH +CF8C;CF8C;110F 116B;CF8C;110F 116B; # (쾌; 쾌; 쾌; 쾌; 쾌; ) HANGUL SYLLABLE KWAE +CF8D;CF8D;110F 116B 11A8;CF8D;110F 116B 11A8; # (쾍; 쾍; 쾍; 쾍; 쾍; ) HANGUL SYLLABLE KWAEG +CF8E;CF8E;110F 116B 11A9;CF8E;110F 116B 11A9; # (쾎; 쾎; 쾎; 쾎; 쾎; ) HANGUL SYLLABLE KWAEGG +CF8F;CF8F;110F 116B 11AA;CF8F;110F 116B 11AA; # (쾏; 쾏; 쾏; 쾏; 쾏; ) HANGUL SYLLABLE KWAEGS +CF90;CF90;110F 116B 11AB;CF90;110F 116B 11AB; # (쾐; 쾐; 쾐; 쾐; 쾐; ) HANGUL SYLLABLE KWAEN +CF91;CF91;110F 116B 11AC;CF91;110F 116B 11AC; # (쾑; 쾑; 쾑; 쾑; 쾑; ) HANGUL SYLLABLE KWAENJ +CF92;CF92;110F 116B 11AD;CF92;110F 116B 11AD; # (쾒; 쾒; 쾒; 쾒; 쾒; ) HANGUL SYLLABLE KWAENH +CF93;CF93;110F 116B 11AE;CF93;110F 116B 11AE; # (쾓; 쾓; 쾓; 쾓; 쾓; ) HANGUL SYLLABLE KWAED +CF94;CF94;110F 116B 11AF;CF94;110F 116B 11AF; # (쾔; 쾔; 쾔; 쾔; 쾔; ) HANGUL SYLLABLE KWAEL +CF95;CF95;110F 116B 11B0;CF95;110F 116B 11B0; # (쾕; 쾕; 쾕; 쾕; 쾕; ) HANGUL SYLLABLE KWAELG +CF96;CF96;110F 116B 11B1;CF96;110F 116B 11B1; # (쾖; 쾖; 쾖; 쾖; 쾖; ) HANGUL SYLLABLE KWAELM +CF97;CF97;110F 116B 11B2;CF97;110F 116B 11B2; # (쾗; 쾗; 쾗; 쾗; 쾗; ) HANGUL SYLLABLE KWAELB +CF98;CF98;110F 116B 11B3;CF98;110F 116B 11B3; # (쾘; 쾘; 쾘; 쾘; 쾘; ) HANGUL SYLLABLE KWAELS +CF99;CF99;110F 116B 11B4;CF99;110F 116B 11B4; # (쾙; 쾙; 쾙; 쾙; 쾙; ) HANGUL SYLLABLE KWAELT +CF9A;CF9A;110F 116B 11B5;CF9A;110F 116B 11B5; # (쾚; 쾚; 쾚; 쾚; 쾚; ) HANGUL SYLLABLE KWAELP +CF9B;CF9B;110F 116B 11B6;CF9B;110F 116B 11B6; # (쾛; 쾛; 쾛; 쾛; 쾛; ) HANGUL SYLLABLE KWAELH +CF9C;CF9C;110F 116B 11B7;CF9C;110F 116B 11B7; # (쾜; 쾜; 쾜; 쾜; 쾜; ) HANGUL SYLLABLE KWAEM +CF9D;CF9D;110F 116B 11B8;CF9D;110F 116B 11B8; # (쾝; 쾝; 쾝; 쾝; 쾝; ) HANGUL SYLLABLE KWAEB +CF9E;CF9E;110F 116B 11B9;CF9E;110F 116B 11B9; # (쾞; 쾞; 쾞; 쾞; 쾞; ) HANGUL SYLLABLE KWAEBS +CF9F;CF9F;110F 116B 11BA;CF9F;110F 116B 11BA; # (쾟; 쾟; 쾟; 쾟; 쾟; ) HANGUL SYLLABLE KWAES +CFA0;CFA0;110F 116B 11BB;CFA0;110F 116B 11BB; # (쾠; 쾠; 쾠; 쾠; 쾠; ) HANGUL SYLLABLE KWAESS +CFA1;CFA1;110F 116B 11BC;CFA1;110F 116B 11BC; # (쾡; 쾡; 쾡; 쾡; 쾡; ) HANGUL SYLLABLE KWAENG +CFA2;CFA2;110F 116B 11BD;CFA2;110F 116B 11BD; # (쾢; 쾢; 쾢; 쾢; 쾢; ) HANGUL SYLLABLE KWAEJ +CFA3;CFA3;110F 116B 11BE;CFA3;110F 116B 11BE; # (쾣; 쾣; 쾣; 쾣; 쾣; ) HANGUL SYLLABLE KWAEC +CFA4;CFA4;110F 116B 11BF;CFA4;110F 116B 11BF; # (쾤; 쾤; 쾤; 쾤; 쾤; ) HANGUL SYLLABLE KWAEK +CFA5;CFA5;110F 116B 11C0;CFA5;110F 116B 11C0; # (쾥; 쾥; 쾥; 쾥; 쾥; ) HANGUL SYLLABLE KWAET +CFA6;CFA6;110F 116B 11C1;CFA6;110F 116B 11C1; # (쾦; 쾦; 쾦; 쾦; 쾦; ) HANGUL SYLLABLE KWAEP +CFA7;CFA7;110F 116B 11C2;CFA7;110F 116B 11C2; # (쾧; 쾧; 쾧; 쾧; 쾧; ) HANGUL SYLLABLE KWAEH +CFA8;CFA8;110F 116C;CFA8;110F 116C; # (쾨; 쾨; 쾨; 쾨; 쾨; ) HANGUL SYLLABLE KOE +CFA9;CFA9;110F 116C 11A8;CFA9;110F 116C 11A8; # (쾩; 쾩; 쾩; 쾩; 쾩; ) HANGUL SYLLABLE KOEG +CFAA;CFAA;110F 116C 11A9;CFAA;110F 116C 11A9; # (쾪; 쾪; 쾪; 쾪; 쾪; ) HANGUL SYLLABLE KOEGG +CFAB;CFAB;110F 116C 11AA;CFAB;110F 116C 11AA; # (쾫; 쾫; 쾫; 쾫; 쾫; ) HANGUL SYLLABLE KOEGS +CFAC;CFAC;110F 116C 11AB;CFAC;110F 116C 11AB; # (쾬; 쾬; 쾬; 쾬; 쾬; ) HANGUL SYLLABLE KOEN +CFAD;CFAD;110F 116C 11AC;CFAD;110F 116C 11AC; # (쾭; 쾭; 쾭; 쾭; 쾭; ) HANGUL SYLLABLE KOENJ +CFAE;CFAE;110F 116C 11AD;CFAE;110F 116C 11AD; # (쾮; 쾮; 쾮; 쾮; 쾮; ) HANGUL SYLLABLE KOENH +CFAF;CFAF;110F 116C 11AE;CFAF;110F 116C 11AE; # (쾯; 쾯; 쾯; 쾯; 쾯; ) HANGUL SYLLABLE KOED +CFB0;CFB0;110F 116C 11AF;CFB0;110F 116C 11AF; # (쾰; 쾰; 쾰; 쾰; 쾰; ) HANGUL SYLLABLE KOEL +CFB1;CFB1;110F 116C 11B0;CFB1;110F 116C 11B0; # (쾱; 쾱; 쾱; 쾱; 쾱; ) HANGUL SYLLABLE KOELG +CFB2;CFB2;110F 116C 11B1;CFB2;110F 116C 11B1; # (쾲; 쾲; 쾲; 쾲; 쾲; ) HANGUL SYLLABLE KOELM +CFB3;CFB3;110F 116C 11B2;CFB3;110F 116C 11B2; # (쾳; 쾳; 쾳; 쾳; 쾳; ) HANGUL SYLLABLE KOELB +CFB4;CFB4;110F 116C 11B3;CFB4;110F 116C 11B3; # (쾴; 쾴; 쾴; 쾴; 쾴; ) HANGUL SYLLABLE KOELS +CFB5;CFB5;110F 116C 11B4;CFB5;110F 116C 11B4; # (쾵; 쾵; 쾵; 쾵; 쾵; ) HANGUL SYLLABLE KOELT +CFB6;CFB6;110F 116C 11B5;CFB6;110F 116C 11B5; # (쾶; 쾶; 쾶; 쾶; 쾶; ) HANGUL SYLLABLE KOELP +CFB7;CFB7;110F 116C 11B6;CFB7;110F 116C 11B6; # (쾷; 쾷; 쾷; 쾷; 쾷; ) HANGUL SYLLABLE KOELH +CFB8;CFB8;110F 116C 11B7;CFB8;110F 116C 11B7; # (쾸; 쾸; 쾸; 쾸; 쾸; ) HANGUL SYLLABLE KOEM +CFB9;CFB9;110F 116C 11B8;CFB9;110F 116C 11B8; # (쾹; 쾹; 쾹; 쾹; 쾹; ) HANGUL SYLLABLE KOEB +CFBA;CFBA;110F 116C 11B9;CFBA;110F 116C 11B9; # (쾺; 쾺; 쾺; 쾺; 쾺; ) HANGUL SYLLABLE KOEBS +CFBB;CFBB;110F 116C 11BA;CFBB;110F 116C 11BA; # (쾻; 쾻; 쾻; 쾻; 쾻; ) HANGUL SYLLABLE KOES +CFBC;CFBC;110F 116C 11BB;CFBC;110F 116C 11BB; # (쾼; 쾼; 쾼; 쾼; 쾼; ) HANGUL SYLLABLE KOESS +CFBD;CFBD;110F 116C 11BC;CFBD;110F 116C 11BC; # (쾽; 쾽; 쾽; 쾽; 쾽; ) HANGUL SYLLABLE KOENG +CFBE;CFBE;110F 116C 11BD;CFBE;110F 116C 11BD; # (쾾; 쾾; 쾾; 쾾; 쾾; ) HANGUL SYLLABLE KOEJ +CFBF;CFBF;110F 116C 11BE;CFBF;110F 116C 11BE; # (쾿; 쾿; 쾿; 쾿; 쾿; ) HANGUL SYLLABLE KOEC +CFC0;CFC0;110F 116C 11BF;CFC0;110F 116C 11BF; # (쿀; 쿀; 쿀; 쿀; 쿀; ) HANGUL SYLLABLE KOEK +CFC1;CFC1;110F 116C 11C0;CFC1;110F 116C 11C0; # (쿁; 쿁; 쿁; 쿁; 쿁; ) HANGUL SYLLABLE KOET +CFC2;CFC2;110F 116C 11C1;CFC2;110F 116C 11C1; # (쿂; 쿂; 쿂; 쿂; 쿂; ) HANGUL SYLLABLE KOEP +CFC3;CFC3;110F 116C 11C2;CFC3;110F 116C 11C2; # (쿃; 쿃; 쿃; 쿃; 쿃; ) HANGUL SYLLABLE KOEH +CFC4;CFC4;110F 116D;CFC4;110F 116D; # (쿄; 쿄; 쿄; 쿄; 쿄; ) HANGUL SYLLABLE KYO +CFC5;CFC5;110F 116D 11A8;CFC5;110F 116D 11A8; # (쿅; 쿅; 쿅; 쿅; 쿅; ) HANGUL SYLLABLE KYOG +CFC6;CFC6;110F 116D 11A9;CFC6;110F 116D 11A9; # (쿆; 쿆; 쿆; 쿆; 쿆; ) HANGUL SYLLABLE KYOGG +CFC7;CFC7;110F 116D 11AA;CFC7;110F 116D 11AA; # (쿇; 쿇; 쿇; 쿇; 쿇; ) HANGUL SYLLABLE KYOGS +CFC8;CFC8;110F 116D 11AB;CFC8;110F 116D 11AB; # (쿈; 쿈; 쿈; 쿈; 쿈; ) HANGUL SYLLABLE KYON +CFC9;CFC9;110F 116D 11AC;CFC9;110F 116D 11AC; # (쿉; 쿉; 쿉; 쿉; 쿉; ) HANGUL SYLLABLE KYONJ +CFCA;CFCA;110F 116D 11AD;CFCA;110F 116D 11AD; # (쿊; 쿊; 쿊; 쿊; 쿊; ) HANGUL SYLLABLE KYONH +CFCB;CFCB;110F 116D 11AE;CFCB;110F 116D 11AE; # (쿋; 쿋; 쿋; 쿋; 쿋; ) HANGUL SYLLABLE KYOD +CFCC;CFCC;110F 116D 11AF;CFCC;110F 116D 11AF; # (쿌; 쿌; 쿌; 쿌; 쿌; ) HANGUL SYLLABLE KYOL +CFCD;CFCD;110F 116D 11B0;CFCD;110F 116D 11B0; # (쿍; 쿍; 쿍; 쿍; 쿍; ) HANGUL SYLLABLE KYOLG +CFCE;CFCE;110F 116D 11B1;CFCE;110F 116D 11B1; # (쿎; 쿎; 쿎; 쿎; 쿎; ) HANGUL SYLLABLE KYOLM +CFCF;CFCF;110F 116D 11B2;CFCF;110F 116D 11B2; # (쿏; 쿏; 쿏; 쿏; 쿏; ) HANGUL SYLLABLE KYOLB +CFD0;CFD0;110F 116D 11B3;CFD0;110F 116D 11B3; # (쿐; 쿐; 쿐; 쿐; 쿐; ) HANGUL SYLLABLE KYOLS +CFD1;CFD1;110F 116D 11B4;CFD1;110F 116D 11B4; # (쿑; 쿑; 쿑; 쿑; 쿑; ) HANGUL SYLLABLE KYOLT +CFD2;CFD2;110F 116D 11B5;CFD2;110F 116D 11B5; # (쿒; 쿒; 쿒; 쿒; 쿒; ) HANGUL SYLLABLE KYOLP +CFD3;CFD3;110F 116D 11B6;CFD3;110F 116D 11B6; # (쿓; 쿓; 쿓; 쿓; 쿓; ) HANGUL SYLLABLE KYOLH +CFD4;CFD4;110F 116D 11B7;CFD4;110F 116D 11B7; # (쿔; 쿔; 쿔; 쿔; 쿔; ) HANGUL SYLLABLE KYOM +CFD5;CFD5;110F 116D 11B8;CFD5;110F 116D 11B8; # (쿕; 쿕; 쿕; 쿕; 쿕; ) HANGUL SYLLABLE KYOB +CFD6;CFD6;110F 116D 11B9;CFD6;110F 116D 11B9; # (쿖; 쿖; 쿖; 쿖; 쿖; ) HANGUL SYLLABLE KYOBS +CFD7;CFD7;110F 116D 11BA;CFD7;110F 116D 11BA; # (쿗; 쿗; 쿗; 쿗; 쿗; ) HANGUL SYLLABLE KYOS +CFD8;CFD8;110F 116D 11BB;CFD8;110F 116D 11BB; # (쿘; 쿘; 쿘; 쿘; 쿘; ) HANGUL SYLLABLE KYOSS +CFD9;CFD9;110F 116D 11BC;CFD9;110F 116D 11BC; # (쿙; 쿙; 쿙; 쿙; 쿙; ) HANGUL SYLLABLE KYONG +CFDA;CFDA;110F 116D 11BD;CFDA;110F 116D 11BD; # (쿚; 쿚; 쿚; 쿚; 쿚; ) HANGUL SYLLABLE KYOJ +CFDB;CFDB;110F 116D 11BE;CFDB;110F 116D 11BE; # (쿛; 쿛; 쿛; 쿛; 쿛; ) HANGUL SYLLABLE KYOC +CFDC;CFDC;110F 116D 11BF;CFDC;110F 116D 11BF; # (쿜; 쿜; 쿜; 쿜; 쿜; ) HANGUL SYLLABLE KYOK +CFDD;CFDD;110F 116D 11C0;CFDD;110F 116D 11C0; # (쿝; 쿝; 쿝; 쿝; 쿝; ) HANGUL SYLLABLE KYOT +CFDE;CFDE;110F 116D 11C1;CFDE;110F 116D 11C1; # (쿞; 쿞; 쿞; 쿞; 쿞; ) HANGUL SYLLABLE KYOP +CFDF;CFDF;110F 116D 11C2;CFDF;110F 116D 11C2; # (쿟; 쿟; 쿟; 쿟; 쿟; ) HANGUL SYLLABLE KYOH +CFE0;CFE0;110F 116E;CFE0;110F 116E; # (쿠; 쿠; 쿠; 쿠; 쿠; ) HANGUL SYLLABLE KU +CFE1;CFE1;110F 116E 11A8;CFE1;110F 116E 11A8; # (쿡; 쿡; 쿡; 쿡; 쿡; ) HANGUL SYLLABLE KUG +CFE2;CFE2;110F 116E 11A9;CFE2;110F 116E 11A9; # (쿢; 쿢; 쿢; 쿢; 쿢; ) HANGUL SYLLABLE KUGG +CFE3;CFE3;110F 116E 11AA;CFE3;110F 116E 11AA; # (쿣; 쿣; 쿣; 쿣; 쿣; ) HANGUL SYLLABLE KUGS +CFE4;CFE4;110F 116E 11AB;CFE4;110F 116E 11AB; # (쿤; 쿤; 쿤; 쿤; 쿤; ) HANGUL SYLLABLE KUN +CFE5;CFE5;110F 116E 11AC;CFE5;110F 116E 11AC; # (쿥; 쿥; 쿥; 쿥; 쿥; ) HANGUL SYLLABLE KUNJ +CFE6;CFE6;110F 116E 11AD;CFE6;110F 116E 11AD; # (쿦; 쿦; 쿦; 쿦; 쿦; ) HANGUL SYLLABLE KUNH +CFE7;CFE7;110F 116E 11AE;CFE7;110F 116E 11AE; # (쿧; 쿧; 쿧; 쿧; 쿧; ) HANGUL SYLLABLE KUD +CFE8;CFE8;110F 116E 11AF;CFE8;110F 116E 11AF; # (쿨; 쿨; 쿨; 쿨; 쿨; ) HANGUL SYLLABLE KUL +CFE9;CFE9;110F 116E 11B0;CFE9;110F 116E 11B0; # (쿩; 쿩; 쿩; 쿩; 쿩; ) HANGUL SYLLABLE KULG +CFEA;CFEA;110F 116E 11B1;CFEA;110F 116E 11B1; # (쿪; 쿪; 쿪; 쿪; 쿪; ) HANGUL SYLLABLE KULM +CFEB;CFEB;110F 116E 11B2;CFEB;110F 116E 11B2; # (쿫; 쿫; 쿫; 쿫; 쿫; ) HANGUL SYLLABLE KULB +CFEC;CFEC;110F 116E 11B3;CFEC;110F 116E 11B3; # (쿬; 쿬; 쿬; 쿬; 쿬; ) HANGUL SYLLABLE KULS +CFED;CFED;110F 116E 11B4;CFED;110F 116E 11B4; # (쿭; 쿭; 쿭; 쿭; 쿭; ) HANGUL SYLLABLE KULT +CFEE;CFEE;110F 116E 11B5;CFEE;110F 116E 11B5; # (쿮; 쿮; 쿮; 쿮; 쿮; ) HANGUL SYLLABLE KULP +CFEF;CFEF;110F 116E 11B6;CFEF;110F 116E 11B6; # (쿯; 쿯; 쿯; 쿯; 쿯; ) HANGUL SYLLABLE KULH +CFF0;CFF0;110F 116E 11B7;CFF0;110F 116E 11B7; # (쿰; 쿰; 쿰; 쿰; 쿰; ) HANGUL SYLLABLE KUM +CFF1;CFF1;110F 116E 11B8;CFF1;110F 116E 11B8; # (쿱; 쿱; 쿱; 쿱; 쿱; ) HANGUL SYLLABLE KUB +CFF2;CFF2;110F 116E 11B9;CFF2;110F 116E 11B9; # (쿲; 쿲; 쿲; 쿲; 쿲; ) HANGUL SYLLABLE KUBS +CFF3;CFF3;110F 116E 11BA;CFF3;110F 116E 11BA; # (쿳; 쿳; 쿳; 쿳; 쿳; ) HANGUL SYLLABLE KUS +CFF4;CFF4;110F 116E 11BB;CFF4;110F 116E 11BB; # (쿴; 쿴; 쿴; 쿴; 쿴; ) HANGUL SYLLABLE KUSS +CFF5;CFF5;110F 116E 11BC;CFF5;110F 116E 11BC; # (쿵; 쿵; 쿵; 쿵; 쿵; ) HANGUL SYLLABLE KUNG +CFF6;CFF6;110F 116E 11BD;CFF6;110F 116E 11BD; # (쿶; 쿶; 쿶; 쿶; 쿶; ) HANGUL SYLLABLE KUJ +CFF7;CFF7;110F 116E 11BE;CFF7;110F 116E 11BE; # (쿷; 쿷; 쿷; 쿷; 쿷; ) HANGUL SYLLABLE KUC +CFF8;CFF8;110F 116E 11BF;CFF8;110F 116E 11BF; # (쿸; 쿸; 쿸; 쿸; 쿸; ) HANGUL SYLLABLE KUK +CFF9;CFF9;110F 116E 11C0;CFF9;110F 116E 11C0; # (쿹; 쿹; 쿹; 쿹; 쿹; ) HANGUL SYLLABLE KUT +CFFA;CFFA;110F 116E 11C1;CFFA;110F 116E 11C1; # (쿺; 쿺; 쿺; 쿺; 쿺; ) HANGUL SYLLABLE KUP +CFFB;CFFB;110F 116E 11C2;CFFB;110F 116E 11C2; # (쿻; 쿻; 쿻; 쿻; 쿻; ) HANGUL SYLLABLE KUH +CFFC;CFFC;110F 116F;CFFC;110F 116F; # (쿼; 쿼; 쿼; 쿼; 쿼; ) HANGUL SYLLABLE KWEO +CFFD;CFFD;110F 116F 11A8;CFFD;110F 116F 11A8; # (쿽; 쿽; 쿽; 쿽; 쿽; ) HANGUL SYLLABLE KWEOG +CFFE;CFFE;110F 116F 11A9;CFFE;110F 116F 11A9; # (쿾; 쿾; 쿾; 쿾; 쿾; ) HANGUL SYLLABLE KWEOGG +CFFF;CFFF;110F 116F 11AA;CFFF;110F 116F 11AA; # (쿿; 쿿; 쿿; 쿿; 쿿; ) HANGUL SYLLABLE KWEOGS +D000;D000;110F 116F 11AB;D000;110F 116F 11AB; # (퀀; 퀀; 퀀; 퀀; 퀀; ) HANGUL SYLLABLE KWEON +D001;D001;110F 116F 11AC;D001;110F 116F 11AC; # (퀁; 퀁; 퀁; 퀁; 퀁; ) HANGUL SYLLABLE KWEONJ +D002;D002;110F 116F 11AD;D002;110F 116F 11AD; # (퀂; 퀂; 퀂; 퀂; 퀂; ) HANGUL SYLLABLE KWEONH +D003;D003;110F 116F 11AE;D003;110F 116F 11AE; # (퀃; 퀃; 퀃; 퀃; 퀃; ) HANGUL SYLLABLE KWEOD +D004;D004;110F 116F 11AF;D004;110F 116F 11AF; # (퀄; 퀄; 퀄; 퀄; 퀄; ) HANGUL SYLLABLE KWEOL +D005;D005;110F 116F 11B0;D005;110F 116F 11B0; # (퀅; 퀅; 퀅; 퀅; 퀅; ) HANGUL SYLLABLE KWEOLG +D006;D006;110F 116F 11B1;D006;110F 116F 11B1; # (퀆; 퀆; 퀆; 퀆; 퀆; ) HANGUL SYLLABLE KWEOLM +D007;D007;110F 116F 11B2;D007;110F 116F 11B2; # (퀇; 퀇; 퀇; 퀇; 퀇; ) HANGUL SYLLABLE KWEOLB +D008;D008;110F 116F 11B3;D008;110F 116F 11B3; # (퀈; 퀈; 퀈; 퀈; 퀈; ) HANGUL SYLLABLE KWEOLS +D009;D009;110F 116F 11B4;D009;110F 116F 11B4; # (퀉; 퀉; 퀉; 퀉; 퀉; ) HANGUL SYLLABLE KWEOLT +D00A;D00A;110F 116F 11B5;D00A;110F 116F 11B5; # (퀊; 퀊; 퀊; 퀊; 퀊; ) HANGUL SYLLABLE KWEOLP +D00B;D00B;110F 116F 11B6;D00B;110F 116F 11B6; # (퀋; 퀋; 퀋; 퀋; 퀋; ) HANGUL SYLLABLE KWEOLH +D00C;D00C;110F 116F 11B7;D00C;110F 116F 11B7; # (퀌; 퀌; 퀌; 퀌; 퀌; ) HANGUL SYLLABLE KWEOM +D00D;D00D;110F 116F 11B8;D00D;110F 116F 11B8; # (퀍; 퀍; 퀍; 퀍; 퀍; ) HANGUL SYLLABLE KWEOB +D00E;D00E;110F 116F 11B9;D00E;110F 116F 11B9; # (퀎; 퀎; 퀎; 퀎; 퀎; ) HANGUL SYLLABLE KWEOBS +D00F;D00F;110F 116F 11BA;D00F;110F 116F 11BA; # (퀏; 퀏; 퀏; 퀏; 퀏; ) HANGUL SYLLABLE KWEOS +D010;D010;110F 116F 11BB;D010;110F 116F 11BB; # (퀐; 퀐; 퀐; 퀐; 퀐; ) HANGUL SYLLABLE KWEOSS +D011;D011;110F 116F 11BC;D011;110F 116F 11BC; # (퀑; 퀑; 퀑; 퀑; 퀑; ) HANGUL SYLLABLE KWEONG +D012;D012;110F 116F 11BD;D012;110F 116F 11BD; # (퀒; 퀒; 퀒; 퀒; 퀒; ) HANGUL SYLLABLE KWEOJ +D013;D013;110F 116F 11BE;D013;110F 116F 11BE; # (퀓; 퀓; 퀓; 퀓; 퀓; ) HANGUL SYLLABLE KWEOC +D014;D014;110F 116F 11BF;D014;110F 116F 11BF; # (퀔; 퀔; 퀔; 퀔; 퀔; ) HANGUL SYLLABLE KWEOK +D015;D015;110F 116F 11C0;D015;110F 116F 11C0; # (퀕; 퀕; 퀕; 퀕; 퀕; ) HANGUL SYLLABLE KWEOT +D016;D016;110F 116F 11C1;D016;110F 116F 11C1; # (퀖; 퀖; 퀖; 퀖; 퀖; ) HANGUL SYLLABLE KWEOP +D017;D017;110F 116F 11C2;D017;110F 116F 11C2; # (퀗; 퀗; 퀗; 퀗; 퀗; ) HANGUL SYLLABLE KWEOH +D018;D018;110F 1170;D018;110F 1170; # (퀘; 퀘; 퀘; 퀘; 퀘; ) HANGUL SYLLABLE KWE +D019;D019;110F 1170 11A8;D019;110F 1170 11A8; # (퀙; 퀙; 퀙; 퀙; 퀙; ) HANGUL SYLLABLE KWEG +D01A;D01A;110F 1170 11A9;D01A;110F 1170 11A9; # (퀚; 퀚; 퀚; 퀚; 퀚; ) HANGUL SYLLABLE KWEGG +D01B;D01B;110F 1170 11AA;D01B;110F 1170 11AA; # (퀛; 퀛; 퀛; 퀛; 퀛; ) HANGUL SYLLABLE KWEGS +D01C;D01C;110F 1170 11AB;D01C;110F 1170 11AB; # (퀜; 퀜; 퀜; 퀜; 퀜; ) HANGUL SYLLABLE KWEN +D01D;D01D;110F 1170 11AC;D01D;110F 1170 11AC; # (퀝; 퀝; 퀝; 퀝; 퀝; ) HANGUL SYLLABLE KWENJ +D01E;D01E;110F 1170 11AD;D01E;110F 1170 11AD; # (퀞; 퀞; 퀞; 퀞; 퀞; ) HANGUL SYLLABLE KWENH +D01F;D01F;110F 1170 11AE;D01F;110F 1170 11AE; # (퀟; 퀟; 퀟; 퀟; 퀟; ) HANGUL SYLLABLE KWED +D020;D020;110F 1170 11AF;D020;110F 1170 11AF; # (퀠; 퀠; 퀠; 퀠; 퀠; ) HANGUL SYLLABLE KWEL +D021;D021;110F 1170 11B0;D021;110F 1170 11B0; # (퀡; 퀡; 퀡; 퀡; 퀡; ) HANGUL SYLLABLE KWELG +D022;D022;110F 1170 11B1;D022;110F 1170 11B1; # (퀢; 퀢; 퀢; 퀢; 퀢; ) HANGUL SYLLABLE KWELM +D023;D023;110F 1170 11B2;D023;110F 1170 11B2; # (퀣; 퀣; 퀣; 퀣; 퀣; ) HANGUL SYLLABLE KWELB +D024;D024;110F 1170 11B3;D024;110F 1170 11B3; # (퀤; 퀤; 퀤; 퀤; 퀤; ) HANGUL SYLLABLE KWELS +D025;D025;110F 1170 11B4;D025;110F 1170 11B4; # (퀥; 퀥; 퀥; 퀥; 퀥; ) HANGUL SYLLABLE KWELT +D026;D026;110F 1170 11B5;D026;110F 1170 11B5; # (퀦; 퀦; 퀦; 퀦; 퀦; ) HANGUL SYLLABLE KWELP +D027;D027;110F 1170 11B6;D027;110F 1170 11B6; # (퀧; 퀧; 퀧; 퀧; 퀧; ) HANGUL SYLLABLE KWELH +D028;D028;110F 1170 11B7;D028;110F 1170 11B7; # (퀨; 퀨; 퀨; 퀨; 퀨; ) HANGUL SYLLABLE KWEM +D029;D029;110F 1170 11B8;D029;110F 1170 11B8; # (퀩; 퀩; 퀩; 퀩; 퀩; ) HANGUL SYLLABLE KWEB +D02A;D02A;110F 1170 11B9;D02A;110F 1170 11B9; # (퀪; 퀪; 퀪; 퀪; 퀪; ) HANGUL SYLLABLE KWEBS +D02B;D02B;110F 1170 11BA;D02B;110F 1170 11BA; # (퀫; 퀫; 퀫; 퀫; 퀫; ) HANGUL SYLLABLE KWES +D02C;D02C;110F 1170 11BB;D02C;110F 1170 11BB; # (퀬; 퀬; 퀬; 퀬; 퀬; ) HANGUL SYLLABLE KWESS +D02D;D02D;110F 1170 11BC;D02D;110F 1170 11BC; # (퀭; 퀭; 퀭; 퀭; 퀭; ) HANGUL SYLLABLE KWENG +D02E;D02E;110F 1170 11BD;D02E;110F 1170 11BD; # (퀮; 퀮; 퀮; 퀮; 퀮; ) HANGUL SYLLABLE KWEJ +D02F;D02F;110F 1170 11BE;D02F;110F 1170 11BE; # (퀯; 퀯; 퀯; 퀯; 퀯; ) HANGUL SYLLABLE KWEC +D030;D030;110F 1170 11BF;D030;110F 1170 11BF; # (퀰; 퀰; 퀰; 퀰; 퀰; ) HANGUL SYLLABLE KWEK +D031;D031;110F 1170 11C0;D031;110F 1170 11C0; # (퀱; 퀱; 퀱; 퀱; 퀱; ) HANGUL SYLLABLE KWET +D032;D032;110F 1170 11C1;D032;110F 1170 11C1; # (퀲; 퀲; 퀲; 퀲; 퀲; ) HANGUL SYLLABLE KWEP +D033;D033;110F 1170 11C2;D033;110F 1170 11C2; # (퀳; 퀳; 퀳; 퀳; 퀳; ) HANGUL SYLLABLE KWEH +D034;D034;110F 1171;D034;110F 1171; # (퀴; 퀴; 퀴; 퀴; 퀴; ) HANGUL SYLLABLE KWI +D035;D035;110F 1171 11A8;D035;110F 1171 11A8; # (퀵; 퀵; 퀵; 퀵; 퀵; ) HANGUL SYLLABLE KWIG +D036;D036;110F 1171 11A9;D036;110F 1171 11A9; # (퀶; 퀶; 퀶; 퀶; 퀶; ) HANGUL SYLLABLE KWIGG +D037;D037;110F 1171 11AA;D037;110F 1171 11AA; # (퀷; 퀷; 퀷; 퀷; 퀷; ) HANGUL SYLLABLE KWIGS +D038;D038;110F 1171 11AB;D038;110F 1171 11AB; # (퀸; 퀸; 퀸; 퀸; 퀸; ) HANGUL SYLLABLE KWIN +D039;D039;110F 1171 11AC;D039;110F 1171 11AC; # (퀹; 퀹; 퀹; 퀹; 퀹; ) HANGUL SYLLABLE KWINJ +D03A;D03A;110F 1171 11AD;D03A;110F 1171 11AD; # (퀺; 퀺; 퀺; 퀺; 퀺; ) HANGUL SYLLABLE KWINH +D03B;D03B;110F 1171 11AE;D03B;110F 1171 11AE; # (퀻; 퀻; 퀻; 퀻; 퀻; ) HANGUL SYLLABLE KWID +D03C;D03C;110F 1171 11AF;D03C;110F 1171 11AF; # (퀼; 퀼; 퀼; 퀼; 퀼; ) HANGUL SYLLABLE KWIL +D03D;D03D;110F 1171 11B0;D03D;110F 1171 11B0; # (퀽; 퀽; 퀽; 퀽; 퀽; ) HANGUL SYLLABLE KWILG +D03E;D03E;110F 1171 11B1;D03E;110F 1171 11B1; # (퀾; 퀾; 퀾; 퀾; 퀾; ) HANGUL SYLLABLE KWILM +D03F;D03F;110F 1171 11B2;D03F;110F 1171 11B2; # (퀿; 퀿; 퀿; 퀿; 퀿; ) HANGUL SYLLABLE KWILB +D040;D040;110F 1171 11B3;D040;110F 1171 11B3; # (큀; 큀; 큀; 큀; 큀; ) HANGUL SYLLABLE KWILS +D041;D041;110F 1171 11B4;D041;110F 1171 11B4; # (큁; 큁; 큁; 큁; 큁; ) HANGUL SYLLABLE KWILT +D042;D042;110F 1171 11B5;D042;110F 1171 11B5; # (큂; 큂; 큂; 큂; 큂; ) HANGUL SYLLABLE KWILP +D043;D043;110F 1171 11B6;D043;110F 1171 11B6; # (큃; 큃; 큃; 큃; 큃; ) HANGUL SYLLABLE KWILH +D044;D044;110F 1171 11B7;D044;110F 1171 11B7; # (큄; 큄; 큄; 큄; 큄; ) HANGUL SYLLABLE KWIM +D045;D045;110F 1171 11B8;D045;110F 1171 11B8; # (큅; 큅; 큅; 큅; 큅; ) HANGUL SYLLABLE KWIB +D046;D046;110F 1171 11B9;D046;110F 1171 11B9; # (큆; 큆; 큆; 큆; 큆; ) HANGUL SYLLABLE KWIBS +D047;D047;110F 1171 11BA;D047;110F 1171 11BA; # (큇; 큇; 큇; 큇; 큇; ) HANGUL SYLLABLE KWIS +D048;D048;110F 1171 11BB;D048;110F 1171 11BB; # (큈; 큈; 큈; 큈; 큈; ) HANGUL SYLLABLE KWISS +D049;D049;110F 1171 11BC;D049;110F 1171 11BC; # (큉; 큉; 큉; 큉; 큉; ) HANGUL SYLLABLE KWING +D04A;D04A;110F 1171 11BD;D04A;110F 1171 11BD; # (큊; 큊; 큊; 큊; 큊; ) HANGUL SYLLABLE KWIJ +D04B;D04B;110F 1171 11BE;D04B;110F 1171 11BE; # (큋; 큋; 큋; 큋; 큋; ) HANGUL SYLLABLE KWIC +D04C;D04C;110F 1171 11BF;D04C;110F 1171 11BF; # (큌; 큌; 큌; 큌; 큌; ) HANGUL SYLLABLE KWIK +D04D;D04D;110F 1171 11C0;D04D;110F 1171 11C0; # (큍; 큍; 큍; 큍; 큍; ) HANGUL SYLLABLE KWIT +D04E;D04E;110F 1171 11C1;D04E;110F 1171 11C1; # (큎; 큎; 큎; 큎; 큎; ) HANGUL SYLLABLE KWIP +D04F;D04F;110F 1171 11C2;D04F;110F 1171 11C2; # (큏; 큏; 큏; 큏; 큏; ) HANGUL SYLLABLE KWIH +D050;D050;110F 1172;D050;110F 1172; # (큐; 큐; 큐; 큐; 큐; ) HANGUL SYLLABLE KYU +D051;D051;110F 1172 11A8;D051;110F 1172 11A8; # (큑; 큑; 큑; 큑; 큑; ) HANGUL SYLLABLE KYUG +D052;D052;110F 1172 11A9;D052;110F 1172 11A9; # (큒; 큒; 큒; 큒; 큒; ) HANGUL SYLLABLE KYUGG +D053;D053;110F 1172 11AA;D053;110F 1172 11AA; # (큓; 큓; 큓; 큓; 큓; ) HANGUL SYLLABLE KYUGS +D054;D054;110F 1172 11AB;D054;110F 1172 11AB; # (큔; 큔; 큔; 큔; 큔; ) HANGUL SYLLABLE KYUN +D055;D055;110F 1172 11AC;D055;110F 1172 11AC; # (큕; 큕; 큕; 큕; 큕; ) HANGUL SYLLABLE KYUNJ +D056;D056;110F 1172 11AD;D056;110F 1172 11AD; # (큖; 큖; 큖; 큖; 큖; ) HANGUL SYLLABLE KYUNH +D057;D057;110F 1172 11AE;D057;110F 1172 11AE; # (큗; 큗; 큗; 큗; 큗; ) HANGUL SYLLABLE KYUD +D058;D058;110F 1172 11AF;D058;110F 1172 11AF; # (큘; 큘; 큘; 큘; 큘; ) HANGUL SYLLABLE KYUL +D059;D059;110F 1172 11B0;D059;110F 1172 11B0; # (큙; 큙; 큙; 큙; 큙; ) HANGUL SYLLABLE KYULG +D05A;D05A;110F 1172 11B1;D05A;110F 1172 11B1; # (큚; 큚; 큚; 큚; 큚; ) HANGUL SYLLABLE KYULM +D05B;D05B;110F 1172 11B2;D05B;110F 1172 11B2; # (큛; 큛; 큛; 큛; 큛; ) HANGUL SYLLABLE KYULB +D05C;D05C;110F 1172 11B3;D05C;110F 1172 11B3; # (큜; 큜; 큜; 큜; 큜; ) HANGUL SYLLABLE KYULS +D05D;D05D;110F 1172 11B4;D05D;110F 1172 11B4; # (큝; 큝; 큝; 큝; 큝; ) HANGUL SYLLABLE KYULT +D05E;D05E;110F 1172 11B5;D05E;110F 1172 11B5; # (큞; 큞; 큞; 큞; 큞; ) HANGUL SYLLABLE KYULP +D05F;D05F;110F 1172 11B6;D05F;110F 1172 11B6; # (큟; 큟; 큟; 큟; 큟; ) HANGUL SYLLABLE KYULH +D060;D060;110F 1172 11B7;D060;110F 1172 11B7; # (큠; 큠; 큠; 큠; 큠; ) HANGUL SYLLABLE KYUM +D061;D061;110F 1172 11B8;D061;110F 1172 11B8; # (큡; 큡; 큡; 큡; 큡; ) HANGUL SYLLABLE KYUB +D062;D062;110F 1172 11B9;D062;110F 1172 11B9; # (큢; 큢; 큢; 큢; 큢; ) HANGUL SYLLABLE KYUBS +D063;D063;110F 1172 11BA;D063;110F 1172 11BA; # (큣; 큣; 큣; 큣; 큣; ) HANGUL SYLLABLE KYUS +D064;D064;110F 1172 11BB;D064;110F 1172 11BB; # (큤; 큤; 큤; 큤; 큤; ) HANGUL SYLLABLE KYUSS +D065;D065;110F 1172 11BC;D065;110F 1172 11BC; # (큥; 큥; 큥; 큥; 큥; ) HANGUL SYLLABLE KYUNG +D066;D066;110F 1172 11BD;D066;110F 1172 11BD; # (큦; 큦; 큦; 큦; 큦; ) HANGUL SYLLABLE KYUJ +D067;D067;110F 1172 11BE;D067;110F 1172 11BE; # (큧; 큧; 큧; 큧; 큧; ) HANGUL SYLLABLE KYUC +D068;D068;110F 1172 11BF;D068;110F 1172 11BF; # (큨; 큨; 큨; 큨; 큨; ) HANGUL SYLLABLE KYUK +D069;D069;110F 1172 11C0;D069;110F 1172 11C0; # (큩; 큩; 큩; 큩; 큩; ) HANGUL SYLLABLE KYUT +D06A;D06A;110F 1172 11C1;D06A;110F 1172 11C1; # (큪; 큪; 큪; 큪; 큪; ) HANGUL SYLLABLE KYUP +D06B;D06B;110F 1172 11C2;D06B;110F 1172 11C2; # (큫; 큫; 큫; 큫; 큫; ) HANGUL SYLLABLE KYUH +D06C;D06C;110F 1173;D06C;110F 1173; # (크; 크; 크; 크; 크; ) HANGUL SYLLABLE KEU +D06D;D06D;110F 1173 11A8;D06D;110F 1173 11A8; # (큭; 큭; 큭; 큭; 큭; ) HANGUL SYLLABLE KEUG +D06E;D06E;110F 1173 11A9;D06E;110F 1173 11A9; # (큮; 큮; 큮; 큮; 큮; ) HANGUL SYLLABLE KEUGG +D06F;D06F;110F 1173 11AA;D06F;110F 1173 11AA; # (큯; 큯; 큯; 큯; 큯; ) HANGUL SYLLABLE KEUGS +D070;D070;110F 1173 11AB;D070;110F 1173 11AB; # (큰; 큰; 큰; 큰; 큰; ) HANGUL SYLLABLE KEUN +D071;D071;110F 1173 11AC;D071;110F 1173 11AC; # (큱; 큱; 큱; 큱; 큱; ) HANGUL SYLLABLE KEUNJ +D072;D072;110F 1173 11AD;D072;110F 1173 11AD; # (큲; 큲; 큲; 큲; 큲; ) HANGUL SYLLABLE KEUNH +D073;D073;110F 1173 11AE;D073;110F 1173 11AE; # (큳; 큳; 큳; 큳; 큳; ) HANGUL SYLLABLE KEUD +D074;D074;110F 1173 11AF;D074;110F 1173 11AF; # (클; 클; 클; 클; 클; ) HANGUL SYLLABLE KEUL +D075;D075;110F 1173 11B0;D075;110F 1173 11B0; # (큵; 큵; 큵; 큵; 큵; ) HANGUL SYLLABLE KEULG +D076;D076;110F 1173 11B1;D076;110F 1173 11B1; # (큶; 큶; 큶; 큶; 큶; ) HANGUL SYLLABLE KEULM +D077;D077;110F 1173 11B2;D077;110F 1173 11B2; # (큷; 큷; 큷; 큷; 큷; ) HANGUL SYLLABLE KEULB +D078;D078;110F 1173 11B3;D078;110F 1173 11B3; # (큸; 큸; 큸; 큸; 큸; ) HANGUL SYLLABLE KEULS +D079;D079;110F 1173 11B4;D079;110F 1173 11B4; # (큹; 큹; 큹; 큹; 큹; ) HANGUL SYLLABLE KEULT +D07A;D07A;110F 1173 11B5;D07A;110F 1173 11B5; # (큺; 큺; 큺; 큺; 큺; ) HANGUL SYLLABLE KEULP +D07B;D07B;110F 1173 11B6;D07B;110F 1173 11B6; # (큻; 큻; 큻; 큻; 큻; ) HANGUL SYLLABLE KEULH +D07C;D07C;110F 1173 11B7;D07C;110F 1173 11B7; # (큼; 큼; 큼; 큼; 큼; ) HANGUL SYLLABLE KEUM +D07D;D07D;110F 1173 11B8;D07D;110F 1173 11B8; # (큽; 큽; 큽; 큽; 큽; ) HANGUL SYLLABLE KEUB +D07E;D07E;110F 1173 11B9;D07E;110F 1173 11B9; # (큾; 큾; 큾; 큾; 큾; ) HANGUL SYLLABLE KEUBS +D07F;D07F;110F 1173 11BA;D07F;110F 1173 11BA; # (큿; 큿; 큿; 큿; 큿; ) HANGUL SYLLABLE KEUS +D080;D080;110F 1173 11BB;D080;110F 1173 11BB; # (킀; 킀; 킀; 킀; 킀; ) HANGUL SYLLABLE KEUSS +D081;D081;110F 1173 11BC;D081;110F 1173 11BC; # (킁; 킁; 킁; 킁; 킁; ) HANGUL SYLLABLE KEUNG +D082;D082;110F 1173 11BD;D082;110F 1173 11BD; # (킂; 킂; 킂; 킂; 킂; ) HANGUL SYLLABLE KEUJ +D083;D083;110F 1173 11BE;D083;110F 1173 11BE; # (킃; 킃; 킃; 킃; 킃; ) HANGUL SYLLABLE KEUC +D084;D084;110F 1173 11BF;D084;110F 1173 11BF; # (킄; 킄; 킄; 킄; 킄; ) HANGUL SYLLABLE KEUK +D085;D085;110F 1173 11C0;D085;110F 1173 11C0; # (킅; 킅; 킅; 킅; 킅; ) HANGUL SYLLABLE KEUT +D086;D086;110F 1173 11C1;D086;110F 1173 11C1; # (킆; 킆; 킆; 킆; 킆; ) HANGUL SYLLABLE KEUP +D087;D087;110F 1173 11C2;D087;110F 1173 11C2; # (킇; 킇; 킇; 킇; 킇; ) HANGUL SYLLABLE KEUH +D088;D088;110F 1174;D088;110F 1174; # (킈; 킈; 킈; 킈; 킈; ) HANGUL SYLLABLE KYI +D089;D089;110F 1174 11A8;D089;110F 1174 11A8; # (킉; 킉; 킉; 킉; 킉; ) HANGUL SYLLABLE KYIG +D08A;D08A;110F 1174 11A9;D08A;110F 1174 11A9; # (킊; 킊; 킊; 킊; 킊; ) HANGUL SYLLABLE KYIGG +D08B;D08B;110F 1174 11AA;D08B;110F 1174 11AA; # (킋; 킋; 킋; 킋; 킋; ) HANGUL SYLLABLE KYIGS +D08C;D08C;110F 1174 11AB;D08C;110F 1174 11AB; # (킌; 킌; 킌; 킌; 킌; ) HANGUL SYLLABLE KYIN +D08D;D08D;110F 1174 11AC;D08D;110F 1174 11AC; # (킍; 킍; 킍; 킍; 킍; ) HANGUL SYLLABLE KYINJ +D08E;D08E;110F 1174 11AD;D08E;110F 1174 11AD; # (킎; 킎; 킎; 킎; 킎; ) HANGUL SYLLABLE KYINH +D08F;D08F;110F 1174 11AE;D08F;110F 1174 11AE; # (킏; 킏; 킏; 킏; 킏; ) HANGUL SYLLABLE KYID +D090;D090;110F 1174 11AF;D090;110F 1174 11AF; # (킐; 킐; 킐; 킐; 킐; ) HANGUL SYLLABLE KYIL +D091;D091;110F 1174 11B0;D091;110F 1174 11B0; # (킑; 킑; 킑; 킑; 킑; ) HANGUL SYLLABLE KYILG +D092;D092;110F 1174 11B1;D092;110F 1174 11B1; # (킒; 킒; 킒; 킒; 킒; ) HANGUL SYLLABLE KYILM +D093;D093;110F 1174 11B2;D093;110F 1174 11B2; # (킓; 킓; 킓; 킓; 킓; ) HANGUL SYLLABLE KYILB +D094;D094;110F 1174 11B3;D094;110F 1174 11B3; # (킔; 킔; 킔; 킔; 킔; ) HANGUL SYLLABLE KYILS +D095;D095;110F 1174 11B4;D095;110F 1174 11B4; # (킕; 킕; 킕; 킕; 킕; ) HANGUL SYLLABLE KYILT +D096;D096;110F 1174 11B5;D096;110F 1174 11B5; # (킖; 킖; 킖; 킖; 킖; ) HANGUL SYLLABLE KYILP +D097;D097;110F 1174 11B6;D097;110F 1174 11B6; # (킗; 킗; 킗; 킗; 킗; ) HANGUL SYLLABLE KYILH +D098;D098;110F 1174 11B7;D098;110F 1174 11B7; # (킘; 킘; 킘; 킘; 킘; ) HANGUL SYLLABLE KYIM +D099;D099;110F 1174 11B8;D099;110F 1174 11B8; # (킙; 킙; 킙; 킙; 킙; ) HANGUL SYLLABLE KYIB +D09A;D09A;110F 1174 11B9;D09A;110F 1174 11B9; # (킚; 킚; 킚; 킚; 킚; ) HANGUL SYLLABLE KYIBS +D09B;D09B;110F 1174 11BA;D09B;110F 1174 11BA; # (킛; 킛; 킛; 킛; 킛; ) HANGUL SYLLABLE KYIS +D09C;D09C;110F 1174 11BB;D09C;110F 1174 11BB; # (킜; 킜; 킜; 킜; 킜; ) HANGUL SYLLABLE KYISS +D09D;D09D;110F 1174 11BC;D09D;110F 1174 11BC; # (킝; 킝; 킝; 킝; 킝; ) HANGUL SYLLABLE KYING +D09E;D09E;110F 1174 11BD;D09E;110F 1174 11BD; # (킞; 킞; 킞; 킞; 킞; ) HANGUL SYLLABLE KYIJ +D09F;D09F;110F 1174 11BE;D09F;110F 1174 11BE; # (킟; 킟; 킟; 킟; 킟; ) HANGUL SYLLABLE KYIC +D0A0;D0A0;110F 1174 11BF;D0A0;110F 1174 11BF; # (킠; 킠; 킠; 킠; 킠; ) HANGUL SYLLABLE KYIK +D0A1;D0A1;110F 1174 11C0;D0A1;110F 1174 11C0; # (킡; 킡; 킡; 킡; 킡; ) HANGUL SYLLABLE KYIT +D0A2;D0A2;110F 1174 11C1;D0A2;110F 1174 11C1; # (킢; 킢; 킢; 킢; 킢; ) HANGUL SYLLABLE KYIP +D0A3;D0A3;110F 1174 11C2;D0A3;110F 1174 11C2; # (킣; 킣; 킣; 킣; 킣; ) HANGUL SYLLABLE KYIH +D0A4;D0A4;110F 1175;D0A4;110F 1175; # (키; 키; 키; 키; 키; ) HANGUL SYLLABLE KI +D0A5;D0A5;110F 1175 11A8;D0A5;110F 1175 11A8; # (킥; 킥; 킥; 킥; 킥; ) HANGUL SYLLABLE KIG +D0A6;D0A6;110F 1175 11A9;D0A6;110F 1175 11A9; # (킦; 킦; 킦; 킦; 킦; ) HANGUL SYLLABLE KIGG +D0A7;D0A7;110F 1175 11AA;D0A7;110F 1175 11AA; # (킧; 킧; 킧; 킧; 킧; ) HANGUL SYLLABLE KIGS +D0A8;D0A8;110F 1175 11AB;D0A8;110F 1175 11AB; # (킨; 킨; 킨; 킨; 킨; ) HANGUL SYLLABLE KIN +D0A9;D0A9;110F 1175 11AC;D0A9;110F 1175 11AC; # (킩; 킩; 킩; 킩; 킩; ) HANGUL SYLLABLE KINJ +D0AA;D0AA;110F 1175 11AD;D0AA;110F 1175 11AD; # (킪; 킪; 킪; 킪; 킪; ) HANGUL SYLLABLE KINH +D0AB;D0AB;110F 1175 11AE;D0AB;110F 1175 11AE; # (킫; 킫; 킫; 킫; 킫; ) HANGUL SYLLABLE KID +D0AC;D0AC;110F 1175 11AF;D0AC;110F 1175 11AF; # (킬; 킬; 킬; 킬; 킬; ) HANGUL SYLLABLE KIL +D0AD;D0AD;110F 1175 11B0;D0AD;110F 1175 11B0; # (킭; 킭; 킭; 킭; 킭; ) HANGUL SYLLABLE KILG +D0AE;D0AE;110F 1175 11B1;D0AE;110F 1175 11B1; # (킮; 킮; 킮; 킮; 킮; ) HANGUL SYLLABLE KILM +D0AF;D0AF;110F 1175 11B2;D0AF;110F 1175 11B2; # (킯; 킯; 킯; 킯; 킯; ) HANGUL SYLLABLE KILB +D0B0;D0B0;110F 1175 11B3;D0B0;110F 1175 11B3; # (킰; 킰; 킰; 킰; 킰; ) HANGUL SYLLABLE KILS +D0B1;D0B1;110F 1175 11B4;D0B1;110F 1175 11B4; # (킱; 킱; 킱; 킱; 킱; ) HANGUL SYLLABLE KILT +D0B2;D0B2;110F 1175 11B5;D0B2;110F 1175 11B5; # (킲; 킲; 킲; 킲; 킲; ) HANGUL SYLLABLE KILP +D0B3;D0B3;110F 1175 11B6;D0B3;110F 1175 11B6; # (킳; 킳; 킳; 킳; 킳; ) HANGUL SYLLABLE KILH +D0B4;D0B4;110F 1175 11B7;D0B4;110F 1175 11B7; # (킴; 킴; 킴; 킴; 킴; ) HANGUL SYLLABLE KIM +D0B5;D0B5;110F 1175 11B8;D0B5;110F 1175 11B8; # (킵; 킵; 킵; 킵; 킵; ) HANGUL SYLLABLE KIB +D0B6;D0B6;110F 1175 11B9;D0B6;110F 1175 11B9; # (킶; 킶; 킶; 킶; 킶; ) HANGUL SYLLABLE KIBS +D0B7;D0B7;110F 1175 11BA;D0B7;110F 1175 11BA; # (킷; 킷; 킷; 킷; 킷; ) HANGUL SYLLABLE KIS +D0B8;D0B8;110F 1175 11BB;D0B8;110F 1175 11BB; # (킸; 킸; 킸; 킸; 킸; ) HANGUL SYLLABLE KISS +D0B9;D0B9;110F 1175 11BC;D0B9;110F 1175 11BC; # (킹; 킹; 킹; 킹; 킹; ) HANGUL SYLLABLE KING +D0BA;D0BA;110F 1175 11BD;D0BA;110F 1175 11BD; # (킺; 킺; 킺; 킺; 킺; ) HANGUL SYLLABLE KIJ +D0BB;D0BB;110F 1175 11BE;D0BB;110F 1175 11BE; # (킻; 킻; 킻; 킻; 킻; ) HANGUL SYLLABLE KIC +D0BC;D0BC;110F 1175 11BF;D0BC;110F 1175 11BF; # (킼; 킼; 킼; 킼; 킼; ) HANGUL SYLLABLE KIK +D0BD;D0BD;110F 1175 11C0;D0BD;110F 1175 11C0; # (킽; 킽; 킽; 킽; 킽; ) HANGUL SYLLABLE KIT +D0BE;D0BE;110F 1175 11C1;D0BE;110F 1175 11C1; # (킾; 킾; 킾; 킾; 킾; ) HANGUL SYLLABLE KIP +D0BF;D0BF;110F 1175 11C2;D0BF;110F 1175 11C2; # (킿; 킿; 킿; 킿; 킿; ) HANGUL SYLLABLE KIH +D0C0;D0C0;1110 1161;D0C0;1110 1161; # (타; 타; 타; 타; 타; ) HANGUL SYLLABLE TA +D0C1;D0C1;1110 1161 11A8;D0C1;1110 1161 11A8; # (탁; 탁; 탁; 탁; 탁; ) HANGUL SYLLABLE TAG +D0C2;D0C2;1110 1161 11A9;D0C2;1110 1161 11A9; # (탂; 탂; 탂; 탂; 탂; ) HANGUL SYLLABLE TAGG +D0C3;D0C3;1110 1161 11AA;D0C3;1110 1161 11AA; # (탃; 탃; 탃; 탃; 탃; ) HANGUL SYLLABLE TAGS +D0C4;D0C4;1110 1161 11AB;D0C4;1110 1161 11AB; # (탄; 탄; 탄; 탄; 탄; ) HANGUL SYLLABLE TAN +D0C5;D0C5;1110 1161 11AC;D0C5;1110 1161 11AC; # (탅; 탅; 탅; 탅; 탅; ) HANGUL SYLLABLE TANJ +D0C6;D0C6;1110 1161 11AD;D0C6;1110 1161 11AD; # (탆; 탆; 탆; 탆; 탆; ) HANGUL SYLLABLE TANH +D0C7;D0C7;1110 1161 11AE;D0C7;1110 1161 11AE; # (탇; 탇; 탇; 탇; 탇; ) HANGUL SYLLABLE TAD +D0C8;D0C8;1110 1161 11AF;D0C8;1110 1161 11AF; # (탈; 탈; 탈; 탈; 탈; ) HANGUL SYLLABLE TAL +D0C9;D0C9;1110 1161 11B0;D0C9;1110 1161 11B0; # (탉; 탉; 탉; 탉; 탉; ) HANGUL SYLLABLE TALG +D0CA;D0CA;1110 1161 11B1;D0CA;1110 1161 11B1; # (탊; 탊; 탊; 탊; 탊; ) HANGUL SYLLABLE TALM +D0CB;D0CB;1110 1161 11B2;D0CB;1110 1161 11B2; # (탋; 탋; 탋; 탋; 탋; ) HANGUL SYLLABLE TALB +D0CC;D0CC;1110 1161 11B3;D0CC;1110 1161 11B3; # (탌; 탌; 탌; 탌; 탌; ) HANGUL SYLLABLE TALS +D0CD;D0CD;1110 1161 11B4;D0CD;1110 1161 11B4; # (탍; 탍; 탍; 탍; 탍; ) HANGUL SYLLABLE TALT +D0CE;D0CE;1110 1161 11B5;D0CE;1110 1161 11B5; # (탎; 탎; 탎; 탎; 탎; ) HANGUL SYLLABLE TALP +D0CF;D0CF;1110 1161 11B6;D0CF;1110 1161 11B6; # (탏; 탏; 탏; 탏; 탏; ) HANGUL SYLLABLE TALH +D0D0;D0D0;1110 1161 11B7;D0D0;1110 1161 11B7; # (탐; 탐; 탐; 탐; 탐; ) HANGUL SYLLABLE TAM +D0D1;D0D1;1110 1161 11B8;D0D1;1110 1161 11B8; # (탑; 탑; 탑; 탑; 탑; ) HANGUL SYLLABLE TAB +D0D2;D0D2;1110 1161 11B9;D0D2;1110 1161 11B9; # (탒; 탒; 탒; 탒; 탒; ) HANGUL SYLLABLE TABS +D0D3;D0D3;1110 1161 11BA;D0D3;1110 1161 11BA; # (탓; 탓; 탓; 탓; 탓; ) HANGUL SYLLABLE TAS +D0D4;D0D4;1110 1161 11BB;D0D4;1110 1161 11BB; # (탔; 탔; 탔; 탔; 탔; ) HANGUL SYLLABLE TASS +D0D5;D0D5;1110 1161 11BC;D0D5;1110 1161 11BC; # (탕; 탕; 탕; 탕; 탕; ) HANGUL SYLLABLE TANG +D0D6;D0D6;1110 1161 11BD;D0D6;1110 1161 11BD; # (탖; 탖; 탖; 탖; 탖; ) HANGUL SYLLABLE TAJ +D0D7;D0D7;1110 1161 11BE;D0D7;1110 1161 11BE; # (탗; 탗; 탗; 탗; 탗; ) HANGUL SYLLABLE TAC +D0D8;D0D8;1110 1161 11BF;D0D8;1110 1161 11BF; # (탘; 탘; 탘; 탘; 탘; ) HANGUL SYLLABLE TAK +D0D9;D0D9;1110 1161 11C0;D0D9;1110 1161 11C0; # (탙; 탙; 탙; 탙; 탙; ) HANGUL SYLLABLE TAT +D0DA;D0DA;1110 1161 11C1;D0DA;1110 1161 11C1; # (탚; 탚; 탚; 탚; 탚; ) HANGUL SYLLABLE TAP +D0DB;D0DB;1110 1161 11C2;D0DB;1110 1161 11C2; # (탛; 탛; 탛; 탛; 탛; ) HANGUL SYLLABLE TAH +D0DC;D0DC;1110 1162;D0DC;1110 1162; # (태; 태; 태; 태; 태; ) HANGUL SYLLABLE TAE +D0DD;D0DD;1110 1162 11A8;D0DD;1110 1162 11A8; # (택; 택; 택; 택; 택; ) HANGUL SYLLABLE TAEG +D0DE;D0DE;1110 1162 11A9;D0DE;1110 1162 11A9; # (탞; 탞; 탞; 탞; 탞; ) HANGUL SYLLABLE TAEGG +D0DF;D0DF;1110 1162 11AA;D0DF;1110 1162 11AA; # (탟; 탟; 탟; 탟; 탟; ) HANGUL SYLLABLE TAEGS +D0E0;D0E0;1110 1162 11AB;D0E0;1110 1162 11AB; # (탠; 탠; 탠; 탠; 탠; ) HANGUL SYLLABLE TAEN +D0E1;D0E1;1110 1162 11AC;D0E1;1110 1162 11AC; # (탡; 탡; 탡; 탡; 탡; ) HANGUL SYLLABLE TAENJ +D0E2;D0E2;1110 1162 11AD;D0E2;1110 1162 11AD; # (탢; 탢; 탢; 탢; 탢; ) HANGUL SYLLABLE TAENH +D0E3;D0E3;1110 1162 11AE;D0E3;1110 1162 11AE; # (탣; 탣; 탣; 탣; 탣; ) HANGUL SYLLABLE TAED +D0E4;D0E4;1110 1162 11AF;D0E4;1110 1162 11AF; # (탤; 탤; 탤; 탤; 탤; ) HANGUL SYLLABLE TAEL +D0E5;D0E5;1110 1162 11B0;D0E5;1110 1162 11B0; # (탥; 탥; 탥; 탥; 탥; ) HANGUL SYLLABLE TAELG +D0E6;D0E6;1110 1162 11B1;D0E6;1110 1162 11B1; # (탦; 탦; 탦; 탦; 탦; ) HANGUL SYLLABLE TAELM +D0E7;D0E7;1110 1162 11B2;D0E7;1110 1162 11B2; # (탧; 탧; 탧; 탧; 탧; ) HANGUL SYLLABLE TAELB +D0E8;D0E8;1110 1162 11B3;D0E8;1110 1162 11B3; # (탨; 탨; 탨; 탨; 탨; ) HANGUL SYLLABLE TAELS +D0E9;D0E9;1110 1162 11B4;D0E9;1110 1162 11B4; # (탩; 탩; 탩; 탩; 탩; ) HANGUL SYLLABLE TAELT +D0EA;D0EA;1110 1162 11B5;D0EA;1110 1162 11B5; # (탪; 탪; 탪; 탪; 탪; ) HANGUL SYLLABLE TAELP +D0EB;D0EB;1110 1162 11B6;D0EB;1110 1162 11B6; # (탫; 탫; 탫; 탫; 탫; ) HANGUL SYLLABLE TAELH +D0EC;D0EC;1110 1162 11B7;D0EC;1110 1162 11B7; # (탬; 탬; 탬; 탬; 탬; ) HANGUL SYLLABLE TAEM +D0ED;D0ED;1110 1162 11B8;D0ED;1110 1162 11B8; # (탭; 탭; 탭; 탭; 탭; ) HANGUL SYLLABLE TAEB +D0EE;D0EE;1110 1162 11B9;D0EE;1110 1162 11B9; # (탮; 탮; 탮; 탮; 탮; ) HANGUL SYLLABLE TAEBS +D0EF;D0EF;1110 1162 11BA;D0EF;1110 1162 11BA; # (탯; 탯; 탯; 탯; 탯; ) HANGUL SYLLABLE TAES +D0F0;D0F0;1110 1162 11BB;D0F0;1110 1162 11BB; # (탰; 탰; 탰; 탰; 탰; ) HANGUL SYLLABLE TAESS +D0F1;D0F1;1110 1162 11BC;D0F1;1110 1162 11BC; # (탱; 탱; 탱; 탱; 탱; ) HANGUL SYLLABLE TAENG +D0F2;D0F2;1110 1162 11BD;D0F2;1110 1162 11BD; # (탲; 탲; 탲; 탲; 탲; ) HANGUL SYLLABLE TAEJ +D0F3;D0F3;1110 1162 11BE;D0F3;1110 1162 11BE; # (탳; 탳; 탳; 탳; 탳; ) HANGUL SYLLABLE TAEC +D0F4;D0F4;1110 1162 11BF;D0F4;1110 1162 11BF; # (탴; 탴; 탴; 탴; 탴; ) HANGUL SYLLABLE TAEK +D0F5;D0F5;1110 1162 11C0;D0F5;1110 1162 11C0; # (탵; 탵; 탵; 탵; 탵; ) HANGUL SYLLABLE TAET +D0F6;D0F6;1110 1162 11C1;D0F6;1110 1162 11C1; # (탶; 탶; 탶; 탶; 탶; ) HANGUL SYLLABLE TAEP +D0F7;D0F7;1110 1162 11C2;D0F7;1110 1162 11C2; # (탷; 탷; 탷; 탷; 탷; ) HANGUL SYLLABLE TAEH +D0F8;D0F8;1110 1163;D0F8;1110 1163; # (탸; 탸; 탸; 탸; 탸; ) HANGUL SYLLABLE TYA +D0F9;D0F9;1110 1163 11A8;D0F9;1110 1163 11A8; # (탹; 탹; 탹; 탹; 탹; ) HANGUL SYLLABLE TYAG +D0FA;D0FA;1110 1163 11A9;D0FA;1110 1163 11A9; # (탺; 탺; 탺; 탺; 탺; ) HANGUL SYLLABLE TYAGG +D0FB;D0FB;1110 1163 11AA;D0FB;1110 1163 11AA; # (탻; 탻; 탻; 탻; 탻; ) HANGUL SYLLABLE TYAGS +D0FC;D0FC;1110 1163 11AB;D0FC;1110 1163 11AB; # (탼; 탼; 탼; 탼; 탼; ) HANGUL SYLLABLE TYAN +D0FD;D0FD;1110 1163 11AC;D0FD;1110 1163 11AC; # (탽; 탽; 탽; 탽; 탽; ) HANGUL SYLLABLE TYANJ +D0FE;D0FE;1110 1163 11AD;D0FE;1110 1163 11AD; # (탾; 탾; 탾; 탾; 탾; ) HANGUL SYLLABLE TYANH +D0FF;D0FF;1110 1163 11AE;D0FF;1110 1163 11AE; # (탿; 탿; 탿; 탿; 탿; ) HANGUL SYLLABLE TYAD +D100;D100;1110 1163 11AF;D100;1110 1163 11AF; # (턀; 턀; 턀; 턀; 턀; ) HANGUL SYLLABLE TYAL +D101;D101;1110 1163 11B0;D101;1110 1163 11B0; # (턁; 턁; 턁; 턁; 턁; ) HANGUL SYLLABLE TYALG +D102;D102;1110 1163 11B1;D102;1110 1163 11B1; # (턂; 턂; 턂; 턂; 턂; ) HANGUL SYLLABLE TYALM +D103;D103;1110 1163 11B2;D103;1110 1163 11B2; # (턃; 턃; 턃; 턃; 턃; ) HANGUL SYLLABLE TYALB +D104;D104;1110 1163 11B3;D104;1110 1163 11B3; # (턄; 턄; 턄; 턄; 턄; ) HANGUL SYLLABLE TYALS +D105;D105;1110 1163 11B4;D105;1110 1163 11B4; # (턅; 턅; 턅; 턅; 턅; ) HANGUL SYLLABLE TYALT +D106;D106;1110 1163 11B5;D106;1110 1163 11B5; # (턆; 턆; 턆; 턆; 턆; ) HANGUL SYLLABLE TYALP +D107;D107;1110 1163 11B6;D107;1110 1163 11B6; # (턇; 턇; 턇; 턇; 턇; ) HANGUL SYLLABLE TYALH +D108;D108;1110 1163 11B7;D108;1110 1163 11B7; # (턈; 턈; 턈; 턈; 턈; ) HANGUL SYLLABLE TYAM +D109;D109;1110 1163 11B8;D109;1110 1163 11B8; # (턉; 턉; 턉; 턉; 턉; ) HANGUL SYLLABLE TYAB +D10A;D10A;1110 1163 11B9;D10A;1110 1163 11B9; # (턊; 턊; 턊; 턊; 턊; ) HANGUL SYLLABLE TYABS +D10B;D10B;1110 1163 11BA;D10B;1110 1163 11BA; # (턋; 턋; 턋; 턋; 턋; ) HANGUL SYLLABLE TYAS +D10C;D10C;1110 1163 11BB;D10C;1110 1163 11BB; # (턌; 턌; 턌; 턌; 턌; ) HANGUL SYLLABLE TYASS +D10D;D10D;1110 1163 11BC;D10D;1110 1163 11BC; # (턍; 턍; 턍; 턍; 턍; ) HANGUL SYLLABLE TYANG +D10E;D10E;1110 1163 11BD;D10E;1110 1163 11BD; # (턎; 턎; 턎; 턎; 턎; ) HANGUL SYLLABLE TYAJ +D10F;D10F;1110 1163 11BE;D10F;1110 1163 11BE; # (턏; 턏; 턏; 턏; 턏; ) HANGUL SYLLABLE TYAC +D110;D110;1110 1163 11BF;D110;1110 1163 11BF; # (턐; 턐; 턐; 턐; 턐; ) HANGUL SYLLABLE TYAK +D111;D111;1110 1163 11C0;D111;1110 1163 11C0; # (턑; 턑; 턑; 턑; 턑; ) HANGUL SYLLABLE TYAT +D112;D112;1110 1163 11C1;D112;1110 1163 11C1; # (턒; 턒; 턒; 턒; 턒; ) HANGUL SYLLABLE TYAP +D113;D113;1110 1163 11C2;D113;1110 1163 11C2; # (턓; 턓; 턓; 턓; 턓; ) HANGUL SYLLABLE TYAH +D114;D114;1110 1164;D114;1110 1164; # (턔; 턔; 턔; 턔; 턔; ) HANGUL SYLLABLE TYAE +D115;D115;1110 1164 11A8;D115;1110 1164 11A8; # (턕; 턕; 턕; 턕; 턕; ) HANGUL SYLLABLE TYAEG +D116;D116;1110 1164 11A9;D116;1110 1164 11A9; # (턖; 턖; 턖; 턖; 턖; ) HANGUL SYLLABLE TYAEGG +D117;D117;1110 1164 11AA;D117;1110 1164 11AA; # (턗; 턗; 턗; 턗; 턗; ) HANGUL SYLLABLE TYAEGS +D118;D118;1110 1164 11AB;D118;1110 1164 11AB; # (턘; 턘; 턘; 턘; 턘; ) HANGUL SYLLABLE TYAEN +D119;D119;1110 1164 11AC;D119;1110 1164 11AC; # (턙; 턙; 턙; 턙; 턙; ) HANGUL SYLLABLE TYAENJ +D11A;D11A;1110 1164 11AD;D11A;1110 1164 11AD; # (턚; 턚; 턚; 턚; 턚; ) HANGUL SYLLABLE TYAENH +D11B;D11B;1110 1164 11AE;D11B;1110 1164 11AE; # (턛; 턛; 턛; 턛; 턛; ) HANGUL SYLLABLE TYAED +D11C;D11C;1110 1164 11AF;D11C;1110 1164 11AF; # (턜; 턜; 턜; 턜; 턜; ) HANGUL SYLLABLE TYAEL +D11D;D11D;1110 1164 11B0;D11D;1110 1164 11B0; # (턝; 턝; 턝; 턝; 턝; ) HANGUL SYLLABLE TYAELG +D11E;D11E;1110 1164 11B1;D11E;1110 1164 11B1; # (턞; 턞; 턞; 턞; 턞; ) HANGUL SYLLABLE TYAELM +D11F;D11F;1110 1164 11B2;D11F;1110 1164 11B2; # (턟; 턟; 턟; 턟; 턟; ) HANGUL SYLLABLE TYAELB +D120;D120;1110 1164 11B3;D120;1110 1164 11B3; # (턠; 턠; 턠; 턠; 턠; ) HANGUL SYLLABLE TYAELS +D121;D121;1110 1164 11B4;D121;1110 1164 11B4; # (턡; 턡; 턡; 턡; 턡; ) HANGUL SYLLABLE TYAELT +D122;D122;1110 1164 11B5;D122;1110 1164 11B5; # (턢; 턢; 턢; 턢; 턢; ) HANGUL SYLLABLE TYAELP +D123;D123;1110 1164 11B6;D123;1110 1164 11B6; # (턣; 턣; 턣; 턣; 턣; ) HANGUL SYLLABLE TYAELH +D124;D124;1110 1164 11B7;D124;1110 1164 11B7; # (턤; 턤; 턤; 턤; 턤; ) HANGUL SYLLABLE TYAEM +D125;D125;1110 1164 11B8;D125;1110 1164 11B8; # (턥; 턥; 턥; 턥; 턥; ) HANGUL SYLLABLE TYAEB +D126;D126;1110 1164 11B9;D126;1110 1164 11B9; # (턦; 턦; 턦; 턦; 턦; ) HANGUL SYLLABLE TYAEBS +D127;D127;1110 1164 11BA;D127;1110 1164 11BA; # (턧; 턧; 턧; 턧; 턧; ) HANGUL SYLLABLE TYAES +D128;D128;1110 1164 11BB;D128;1110 1164 11BB; # (턨; 턨; 턨; 턨; 턨; ) HANGUL SYLLABLE TYAESS +D129;D129;1110 1164 11BC;D129;1110 1164 11BC; # (턩; 턩; 턩; 턩; 턩; ) HANGUL SYLLABLE TYAENG +D12A;D12A;1110 1164 11BD;D12A;1110 1164 11BD; # (턪; 턪; 턪; 턪; 턪; ) HANGUL SYLLABLE TYAEJ +D12B;D12B;1110 1164 11BE;D12B;1110 1164 11BE; # (턫; 턫; 턫; 턫; 턫; ) HANGUL SYLLABLE TYAEC +D12C;D12C;1110 1164 11BF;D12C;1110 1164 11BF; # (턬; 턬; 턬; 턬; 턬; ) HANGUL SYLLABLE TYAEK +D12D;D12D;1110 1164 11C0;D12D;1110 1164 11C0; # (턭; 턭; 턭; 턭; 턭; ) HANGUL SYLLABLE TYAET +D12E;D12E;1110 1164 11C1;D12E;1110 1164 11C1; # (턮; 턮; 턮; 턮; 턮; ) HANGUL SYLLABLE TYAEP +D12F;D12F;1110 1164 11C2;D12F;1110 1164 11C2; # (턯; 턯; 턯; 턯; 턯; ) HANGUL SYLLABLE TYAEH +D130;D130;1110 1165;D130;1110 1165; # (터; 터; 터; 터; 터; ) HANGUL SYLLABLE TEO +D131;D131;1110 1165 11A8;D131;1110 1165 11A8; # (턱; 턱; 턱; 턱; 턱; ) HANGUL SYLLABLE TEOG +D132;D132;1110 1165 11A9;D132;1110 1165 11A9; # (턲; 턲; 턲; 턲; 턲; ) HANGUL SYLLABLE TEOGG +D133;D133;1110 1165 11AA;D133;1110 1165 11AA; # (턳; 턳; 턳; 턳; 턳; ) HANGUL SYLLABLE TEOGS +D134;D134;1110 1165 11AB;D134;1110 1165 11AB; # (턴; 턴; 턴; 턴; 턴; ) HANGUL SYLLABLE TEON +D135;D135;1110 1165 11AC;D135;1110 1165 11AC; # (턵; 턵; 턵; 턵; 턵; ) HANGUL SYLLABLE TEONJ +D136;D136;1110 1165 11AD;D136;1110 1165 11AD; # (턶; 턶; 턶; 턶; 턶; ) HANGUL SYLLABLE TEONH +D137;D137;1110 1165 11AE;D137;1110 1165 11AE; # (턷; 턷; 턷; 턷; 턷; ) HANGUL SYLLABLE TEOD +D138;D138;1110 1165 11AF;D138;1110 1165 11AF; # (털; 털; 털; 털; 털; ) HANGUL SYLLABLE TEOL +D139;D139;1110 1165 11B0;D139;1110 1165 11B0; # (턹; 턹; 턹; 턹; 턹; ) HANGUL SYLLABLE TEOLG +D13A;D13A;1110 1165 11B1;D13A;1110 1165 11B1; # (턺; 턺; 턺; 턺; 턺; ) HANGUL SYLLABLE TEOLM +D13B;D13B;1110 1165 11B2;D13B;1110 1165 11B2; # (턻; 턻; 턻; 턻; 턻; ) HANGUL SYLLABLE TEOLB +D13C;D13C;1110 1165 11B3;D13C;1110 1165 11B3; # (턼; 턼; 턼; 턼; 턼; ) HANGUL SYLLABLE TEOLS +D13D;D13D;1110 1165 11B4;D13D;1110 1165 11B4; # (턽; 턽; 턽; 턽; 턽; ) HANGUL SYLLABLE TEOLT +D13E;D13E;1110 1165 11B5;D13E;1110 1165 11B5; # (턾; 턾; 턾; 턾; 턾; ) HANGUL SYLLABLE TEOLP +D13F;D13F;1110 1165 11B6;D13F;1110 1165 11B6; # (턿; 턿; 턿; 턿; 턿; ) HANGUL SYLLABLE TEOLH +D140;D140;1110 1165 11B7;D140;1110 1165 11B7; # (텀; 텀; 텀; 텀; 텀; ) HANGUL SYLLABLE TEOM +D141;D141;1110 1165 11B8;D141;1110 1165 11B8; # (텁; 텁; 텁; 텁; 텁; ) HANGUL SYLLABLE TEOB +D142;D142;1110 1165 11B9;D142;1110 1165 11B9; # (텂; 텂; 텂; 텂; 텂; ) HANGUL SYLLABLE TEOBS +D143;D143;1110 1165 11BA;D143;1110 1165 11BA; # (텃; 텃; 텃; 텃; 텃; ) HANGUL SYLLABLE TEOS +D144;D144;1110 1165 11BB;D144;1110 1165 11BB; # (텄; 텄; 텄; 텄; 텄; ) HANGUL SYLLABLE TEOSS +D145;D145;1110 1165 11BC;D145;1110 1165 11BC; # (텅; 텅; 텅; 텅; 텅; ) HANGUL SYLLABLE TEONG +D146;D146;1110 1165 11BD;D146;1110 1165 11BD; # (텆; 텆; 텆; 텆; 텆; ) HANGUL SYLLABLE TEOJ +D147;D147;1110 1165 11BE;D147;1110 1165 11BE; # (텇; 텇; 텇; 텇; 텇; ) HANGUL SYLLABLE TEOC +D148;D148;1110 1165 11BF;D148;1110 1165 11BF; # (텈; 텈; 텈; 텈; 텈; ) HANGUL SYLLABLE TEOK +D149;D149;1110 1165 11C0;D149;1110 1165 11C0; # (텉; 텉; 텉; 텉; 텉; ) HANGUL SYLLABLE TEOT +D14A;D14A;1110 1165 11C1;D14A;1110 1165 11C1; # (텊; 텊; 텊; 텊; 텊; ) HANGUL SYLLABLE TEOP +D14B;D14B;1110 1165 11C2;D14B;1110 1165 11C2; # (텋; 텋; 텋; 텋; 텋; ) HANGUL SYLLABLE TEOH +D14C;D14C;1110 1166;D14C;1110 1166; # (테; 테; 테; 테; 테; ) HANGUL SYLLABLE TE +D14D;D14D;1110 1166 11A8;D14D;1110 1166 11A8; # (텍; 텍; 텍; 텍; 텍; ) HANGUL SYLLABLE TEG +D14E;D14E;1110 1166 11A9;D14E;1110 1166 11A9; # (텎; 텎; 텎; 텎; 텎; ) HANGUL SYLLABLE TEGG +D14F;D14F;1110 1166 11AA;D14F;1110 1166 11AA; # (텏; 텏; 텏; 텏; 텏; ) HANGUL SYLLABLE TEGS +D150;D150;1110 1166 11AB;D150;1110 1166 11AB; # (텐; 텐; 텐; 텐; 텐; ) HANGUL SYLLABLE TEN +D151;D151;1110 1166 11AC;D151;1110 1166 11AC; # (텑; 텑; 텑; 텑; 텑; ) HANGUL SYLLABLE TENJ +D152;D152;1110 1166 11AD;D152;1110 1166 11AD; # (텒; 텒; 텒; 텒; 텒; ) HANGUL SYLLABLE TENH +D153;D153;1110 1166 11AE;D153;1110 1166 11AE; # (텓; 텓; 텓; 텓; 텓; ) HANGUL SYLLABLE TED +D154;D154;1110 1166 11AF;D154;1110 1166 11AF; # (텔; 텔; 텔; 텔; 텔; ) HANGUL SYLLABLE TEL +D155;D155;1110 1166 11B0;D155;1110 1166 11B0; # (텕; 텕; 텕; 텕; 텕; ) HANGUL SYLLABLE TELG +D156;D156;1110 1166 11B1;D156;1110 1166 11B1; # (텖; 텖; 텖; 텖; 텖; ) HANGUL SYLLABLE TELM +D157;D157;1110 1166 11B2;D157;1110 1166 11B2; # (텗; 텗; 텗; 텗; 텗; ) HANGUL SYLLABLE TELB +D158;D158;1110 1166 11B3;D158;1110 1166 11B3; # (텘; 텘; 텘; 텘; 텘; ) HANGUL SYLLABLE TELS +D159;D159;1110 1166 11B4;D159;1110 1166 11B4; # (텙; 텙; 텙; 텙; 텙; ) HANGUL SYLLABLE TELT +D15A;D15A;1110 1166 11B5;D15A;1110 1166 11B5; # (텚; 텚; 텚; 텚; 텚; ) HANGUL SYLLABLE TELP +D15B;D15B;1110 1166 11B6;D15B;1110 1166 11B6; # (텛; 텛; 텛; 텛; 텛; ) HANGUL SYLLABLE TELH +D15C;D15C;1110 1166 11B7;D15C;1110 1166 11B7; # (템; 템; 템; 템; 템; ) HANGUL SYLLABLE TEM +D15D;D15D;1110 1166 11B8;D15D;1110 1166 11B8; # (텝; 텝; 텝; 텝; 텝; ) HANGUL SYLLABLE TEB +D15E;D15E;1110 1166 11B9;D15E;1110 1166 11B9; # (텞; 텞; 텞; 텞; 텞; ) HANGUL SYLLABLE TEBS +D15F;D15F;1110 1166 11BA;D15F;1110 1166 11BA; # (텟; 텟; 텟; 텟; 텟; ) HANGUL SYLLABLE TES +D160;D160;1110 1166 11BB;D160;1110 1166 11BB; # (텠; 텠; 텠; 텠; 텠; ) HANGUL SYLLABLE TESS +D161;D161;1110 1166 11BC;D161;1110 1166 11BC; # (텡; 텡; 텡; 텡; 텡; ) HANGUL SYLLABLE TENG +D162;D162;1110 1166 11BD;D162;1110 1166 11BD; # (텢; 텢; 텢; 텢; 텢; ) HANGUL SYLLABLE TEJ +D163;D163;1110 1166 11BE;D163;1110 1166 11BE; # (텣; 텣; 텣; 텣; 텣; ) HANGUL SYLLABLE TEC +D164;D164;1110 1166 11BF;D164;1110 1166 11BF; # (텤; 텤; 텤; 텤; 텤; ) HANGUL SYLLABLE TEK +D165;D165;1110 1166 11C0;D165;1110 1166 11C0; # (텥; 텥; 텥; 텥; 텥; ) HANGUL SYLLABLE TET +D166;D166;1110 1166 11C1;D166;1110 1166 11C1; # (텦; 텦; 텦; 텦; 텦; ) HANGUL SYLLABLE TEP +D167;D167;1110 1166 11C2;D167;1110 1166 11C2; # (텧; 텧; 텧; 텧; 텧; ) HANGUL SYLLABLE TEH +D168;D168;1110 1167;D168;1110 1167; # (텨; 텨; 텨; 텨; 텨; ) HANGUL SYLLABLE TYEO +D169;D169;1110 1167 11A8;D169;1110 1167 11A8; # (텩; 텩; 텩; 텩; 텩; ) HANGUL SYLLABLE TYEOG +D16A;D16A;1110 1167 11A9;D16A;1110 1167 11A9; # (텪; 텪; 텪; 텪; 텪; ) HANGUL SYLLABLE TYEOGG +D16B;D16B;1110 1167 11AA;D16B;1110 1167 11AA; # (텫; 텫; 텫; 텫; 텫; ) HANGUL SYLLABLE TYEOGS +D16C;D16C;1110 1167 11AB;D16C;1110 1167 11AB; # (텬; 텬; 텬; 텬; 텬; ) HANGUL SYLLABLE TYEON +D16D;D16D;1110 1167 11AC;D16D;1110 1167 11AC; # (텭; 텭; 텭; 텭; 텭; ) HANGUL SYLLABLE TYEONJ +D16E;D16E;1110 1167 11AD;D16E;1110 1167 11AD; # (텮; 텮; 텮; 텮; 텮; ) HANGUL SYLLABLE TYEONH +D16F;D16F;1110 1167 11AE;D16F;1110 1167 11AE; # (텯; 텯; 텯; 텯; 텯; ) HANGUL SYLLABLE TYEOD +D170;D170;1110 1167 11AF;D170;1110 1167 11AF; # (텰; 텰; 텰; 텰; 텰; ) HANGUL SYLLABLE TYEOL +D171;D171;1110 1167 11B0;D171;1110 1167 11B0; # (텱; 텱; 텱; 텱; 텱; ) HANGUL SYLLABLE TYEOLG +D172;D172;1110 1167 11B1;D172;1110 1167 11B1; # (텲; 텲; 텲; 텲; 텲; ) HANGUL SYLLABLE TYEOLM +D173;D173;1110 1167 11B2;D173;1110 1167 11B2; # (텳; 텳; 텳; 텳; 텳; ) HANGUL SYLLABLE TYEOLB +D174;D174;1110 1167 11B3;D174;1110 1167 11B3; # (텴; 텴; 텴; 텴; 텴; ) HANGUL SYLLABLE TYEOLS +D175;D175;1110 1167 11B4;D175;1110 1167 11B4; # (텵; 텵; 텵; 텵; 텵; ) HANGUL SYLLABLE TYEOLT +D176;D176;1110 1167 11B5;D176;1110 1167 11B5; # (텶; 텶; 텶; 텶; 텶; ) HANGUL SYLLABLE TYEOLP +D177;D177;1110 1167 11B6;D177;1110 1167 11B6; # (텷; 텷; 텷; 텷; 텷; ) HANGUL SYLLABLE TYEOLH +D178;D178;1110 1167 11B7;D178;1110 1167 11B7; # (텸; 텸; 텸; 텸; 텸; ) HANGUL SYLLABLE TYEOM +D179;D179;1110 1167 11B8;D179;1110 1167 11B8; # (텹; 텹; 텹; 텹; 텹; ) HANGUL SYLLABLE TYEOB +D17A;D17A;1110 1167 11B9;D17A;1110 1167 11B9; # (텺; 텺; 텺; 텺; 텺; ) HANGUL SYLLABLE TYEOBS +D17B;D17B;1110 1167 11BA;D17B;1110 1167 11BA; # (텻; 텻; 텻; 텻; 텻; ) HANGUL SYLLABLE TYEOS +D17C;D17C;1110 1167 11BB;D17C;1110 1167 11BB; # (텼; 텼; 텼; 텼; 텼; ) HANGUL SYLLABLE TYEOSS +D17D;D17D;1110 1167 11BC;D17D;1110 1167 11BC; # (텽; 텽; 텽; 텽; 텽; ) HANGUL SYLLABLE TYEONG +D17E;D17E;1110 1167 11BD;D17E;1110 1167 11BD; # (텾; 텾; 텾; 텾; 텾; ) HANGUL SYLLABLE TYEOJ +D17F;D17F;1110 1167 11BE;D17F;1110 1167 11BE; # (텿; 텿; 텿; 텿; 텿; ) HANGUL SYLLABLE TYEOC +D180;D180;1110 1167 11BF;D180;1110 1167 11BF; # (톀; 톀; 톀; 톀; 톀; ) HANGUL SYLLABLE TYEOK +D181;D181;1110 1167 11C0;D181;1110 1167 11C0; # (톁; 톁; 톁; 톁; 톁; ) HANGUL SYLLABLE TYEOT +D182;D182;1110 1167 11C1;D182;1110 1167 11C1; # (톂; 톂; 톂; 톂; 톂; ) HANGUL SYLLABLE TYEOP +D183;D183;1110 1167 11C2;D183;1110 1167 11C2; # (톃; 톃; 톃; 톃; 톃; ) HANGUL SYLLABLE TYEOH +D184;D184;1110 1168;D184;1110 1168; # (톄; 톄; 톄; 톄; 톄; ) HANGUL SYLLABLE TYE +D185;D185;1110 1168 11A8;D185;1110 1168 11A8; # (톅; 톅; 톅; 톅; 톅; ) HANGUL SYLLABLE TYEG +D186;D186;1110 1168 11A9;D186;1110 1168 11A9; # (톆; 톆; 톆; 톆; 톆; ) HANGUL SYLLABLE TYEGG +D187;D187;1110 1168 11AA;D187;1110 1168 11AA; # (톇; 톇; 톇; 톇; 톇; ) HANGUL SYLLABLE TYEGS +D188;D188;1110 1168 11AB;D188;1110 1168 11AB; # (톈; 톈; 톈; 톈; 톈; ) HANGUL SYLLABLE TYEN +D189;D189;1110 1168 11AC;D189;1110 1168 11AC; # (톉; 톉; 톉; 톉; 톉; ) HANGUL SYLLABLE TYENJ +D18A;D18A;1110 1168 11AD;D18A;1110 1168 11AD; # (톊; 톊; 톊; 톊; 톊; ) HANGUL SYLLABLE TYENH +D18B;D18B;1110 1168 11AE;D18B;1110 1168 11AE; # (톋; 톋; 톋; 톋; 톋; ) HANGUL SYLLABLE TYED +D18C;D18C;1110 1168 11AF;D18C;1110 1168 11AF; # (톌; 톌; 톌; 톌; 톌; ) HANGUL SYLLABLE TYEL +D18D;D18D;1110 1168 11B0;D18D;1110 1168 11B0; # (톍; 톍; 톍; 톍; 톍; ) HANGUL SYLLABLE TYELG +D18E;D18E;1110 1168 11B1;D18E;1110 1168 11B1; # (톎; 톎; 톎; 톎; 톎; ) HANGUL SYLLABLE TYELM +D18F;D18F;1110 1168 11B2;D18F;1110 1168 11B2; # (톏; 톏; 톏; 톏; 톏; ) HANGUL SYLLABLE TYELB +D190;D190;1110 1168 11B3;D190;1110 1168 11B3; # (톐; 톐; 톐; 톐; 톐; ) HANGUL SYLLABLE TYELS +D191;D191;1110 1168 11B4;D191;1110 1168 11B4; # (톑; 톑; 톑; 톑; 톑; ) HANGUL SYLLABLE TYELT +D192;D192;1110 1168 11B5;D192;1110 1168 11B5; # (톒; 톒; 톒; 톒; 톒; ) HANGUL SYLLABLE TYELP +D193;D193;1110 1168 11B6;D193;1110 1168 11B6; # (톓; 톓; 톓; 톓; 톓; ) HANGUL SYLLABLE TYELH +D194;D194;1110 1168 11B7;D194;1110 1168 11B7; # (톔; 톔; 톔; 톔; 톔; ) HANGUL SYLLABLE TYEM +D195;D195;1110 1168 11B8;D195;1110 1168 11B8; # (톕; 톕; 톕; 톕; 톕; ) HANGUL SYLLABLE TYEB +D196;D196;1110 1168 11B9;D196;1110 1168 11B9; # (톖; 톖; 톖; 톖; 톖; ) HANGUL SYLLABLE TYEBS +D197;D197;1110 1168 11BA;D197;1110 1168 11BA; # (톗; 톗; 톗; 톗; 톗; ) HANGUL SYLLABLE TYES +D198;D198;1110 1168 11BB;D198;1110 1168 11BB; # (톘; 톘; 톘; 톘; 톘; ) HANGUL SYLLABLE TYESS +D199;D199;1110 1168 11BC;D199;1110 1168 11BC; # (톙; 톙; 톙; 톙; 톙; ) HANGUL SYLLABLE TYENG +D19A;D19A;1110 1168 11BD;D19A;1110 1168 11BD; # (톚; 톚; 톚; 톚; 톚; ) HANGUL SYLLABLE TYEJ +D19B;D19B;1110 1168 11BE;D19B;1110 1168 11BE; # (톛; 톛; 톛; 톛; 톛; ) HANGUL SYLLABLE TYEC +D19C;D19C;1110 1168 11BF;D19C;1110 1168 11BF; # (톜; 톜; 톜; 톜; 톜; ) HANGUL SYLLABLE TYEK +D19D;D19D;1110 1168 11C0;D19D;1110 1168 11C0; # (톝; 톝; 톝; 톝; 톝; ) HANGUL SYLLABLE TYET +D19E;D19E;1110 1168 11C1;D19E;1110 1168 11C1; # (톞; 톞; 톞; 톞; 톞; ) HANGUL SYLLABLE TYEP +D19F;D19F;1110 1168 11C2;D19F;1110 1168 11C2; # (톟; 톟; 톟; 톟; 톟; ) HANGUL SYLLABLE TYEH +D1A0;D1A0;1110 1169;D1A0;1110 1169; # (토; 토; 토; 토; 토; ) HANGUL SYLLABLE TO +D1A1;D1A1;1110 1169 11A8;D1A1;1110 1169 11A8; # (톡; 톡; 톡; 톡; 톡; ) HANGUL SYLLABLE TOG +D1A2;D1A2;1110 1169 11A9;D1A2;1110 1169 11A9; # (톢; 톢; 톢; 톢; 톢; ) HANGUL SYLLABLE TOGG +D1A3;D1A3;1110 1169 11AA;D1A3;1110 1169 11AA; # (톣; 톣; 톣; 톣; 톣; ) HANGUL SYLLABLE TOGS +D1A4;D1A4;1110 1169 11AB;D1A4;1110 1169 11AB; # (톤; 톤; 톤; 톤; 톤; ) HANGUL SYLLABLE TON +D1A5;D1A5;1110 1169 11AC;D1A5;1110 1169 11AC; # (톥; 톥; 톥; 톥; 톥; ) HANGUL SYLLABLE TONJ +D1A6;D1A6;1110 1169 11AD;D1A6;1110 1169 11AD; # (톦; 톦; 톦; 톦; 톦; ) HANGUL SYLLABLE TONH +D1A7;D1A7;1110 1169 11AE;D1A7;1110 1169 11AE; # (톧; 톧; 톧; 톧; 톧; ) HANGUL SYLLABLE TOD +D1A8;D1A8;1110 1169 11AF;D1A8;1110 1169 11AF; # (톨; 톨; 톨; 톨; 톨; ) HANGUL SYLLABLE TOL +D1A9;D1A9;1110 1169 11B0;D1A9;1110 1169 11B0; # (톩; 톩; 톩; 톩; 톩; ) HANGUL SYLLABLE TOLG +D1AA;D1AA;1110 1169 11B1;D1AA;1110 1169 11B1; # (톪; 톪; 톪; 톪; 톪; ) HANGUL SYLLABLE TOLM +D1AB;D1AB;1110 1169 11B2;D1AB;1110 1169 11B2; # (톫; 톫; 톫; 톫; 톫; ) HANGUL SYLLABLE TOLB +D1AC;D1AC;1110 1169 11B3;D1AC;1110 1169 11B3; # (톬; 톬; 톬; 톬; 톬; ) HANGUL SYLLABLE TOLS +D1AD;D1AD;1110 1169 11B4;D1AD;1110 1169 11B4; # (톭; 톭; 톭; 톭; 톭; ) HANGUL SYLLABLE TOLT +D1AE;D1AE;1110 1169 11B5;D1AE;1110 1169 11B5; # (톮; 톮; 톮; 톮; 톮; ) HANGUL SYLLABLE TOLP +D1AF;D1AF;1110 1169 11B6;D1AF;1110 1169 11B6; # (톯; 톯; 톯; 톯; 톯; ) HANGUL SYLLABLE TOLH +D1B0;D1B0;1110 1169 11B7;D1B0;1110 1169 11B7; # (톰; 톰; 톰; 톰; 톰; ) HANGUL SYLLABLE TOM +D1B1;D1B1;1110 1169 11B8;D1B1;1110 1169 11B8; # (톱; 톱; 톱; 톱; 톱; ) HANGUL SYLLABLE TOB +D1B2;D1B2;1110 1169 11B9;D1B2;1110 1169 11B9; # (톲; 톲; 톲; 톲; 톲; ) HANGUL SYLLABLE TOBS +D1B3;D1B3;1110 1169 11BA;D1B3;1110 1169 11BA; # (톳; 톳; 톳; 톳; 톳; ) HANGUL SYLLABLE TOS +D1B4;D1B4;1110 1169 11BB;D1B4;1110 1169 11BB; # (톴; 톴; 톴; 톴; 톴; ) HANGUL SYLLABLE TOSS +D1B5;D1B5;1110 1169 11BC;D1B5;1110 1169 11BC; # (통; 통; 통; 통; 통; ) HANGUL SYLLABLE TONG +D1B6;D1B6;1110 1169 11BD;D1B6;1110 1169 11BD; # (톶; 톶; 톶; 톶; 톶; ) HANGUL SYLLABLE TOJ +D1B7;D1B7;1110 1169 11BE;D1B7;1110 1169 11BE; # (톷; 톷; 톷; 톷; 톷; ) HANGUL SYLLABLE TOC +D1B8;D1B8;1110 1169 11BF;D1B8;1110 1169 11BF; # (톸; 톸; 톸; 톸; 톸; ) HANGUL SYLLABLE TOK +D1B9;D1B9;1110 1169 11C0;D1B9;1110 1169 11C0; # (톹; 톹; 톹; 톹; 톹; ) HANGUL SYLLABLE TOT +D1BA;D1BA;1110 1169 11C1;D1BA;1110 1169 11C1; # (톺; 톺; 톺; 톺; 톺; ) HANGUL SYLLABLE TOP +D1BB;D1BB;1110 1169 11C2;D1BB;1110 1169 11C2; # (톻; 톻; 톻; 톻; 톻; ) HANGUL SYLLABLE TOH +D1BC;D1BC;1110 116A;D1BC;1110 116A; # (톼; 톼; 톼; 톼; 톼; ) HANGUL SYLLABLE TWA +D1BD;D1BD;1110 116A 11A8;D1BD;1110 116A 11A8; # (톽; 톽; 톽; 톽; 톽; ) HANGUL SYLLABLE TWAG +D1BE;D1BE;1110 116A 11A9;D1BE;1110 116A 11A9; # (톾; 톾; 톾; 톾; 톾; ) HANGUL SYLLABLE TWAGG +D1BF;D1BF;1110 116A 11AA;D1BF;1110 116A 11AA; # (톿; 톿; 톿; 톿; 톿; ) HANGUL SYLLABLE TWAGS +D1C0;D1C0;1110 116A 11AB;D1C0;1110 116A 11AB; # (퇀; 퇀; 퇀; 퇀; 퇀; ) HANGUL SYLLABLE TWAN +D1C1;D1C1;1110 116A 11AC;D1C1;1110 116A 11AC; # (퇁; 퇁; 퇁; 퇁; 퇁; ) HANGUL SYLLABLE TWANJ +D1C2;D1C2;1110 116A 11AD;D1C2;1110 116A 11AD; # (퇂; 퇂; 퇂; 퇂; 퇂; ) HANGUL SYLLABLE TWANH +D1C3;D1C3;1110 116A 11AE;D1C3;1110 116A 11AE; # (퇃; 퇃; 퇃; 퇃; 퇃; ) HANGUL SYLLABLE TWAD +D1C4;D1C4;1110 116A 11AF;D1C4;1110 116A 11AF; # (퇄; 퇄; 퇄; 퇄; 퇄; ) HANGUL SYLLABLE TWAL +D1C5;D1C5;1110 116A 11B0;D1C5;1110 116A 11B0; # (퇅; 퇅; 퇅; 퇅; 퇅; ) HANGUL SYLLABLE TWALG +D1C6;D1C6;1110 116A 11B1;D1C6;1110 116A 11B1; # (퇆; 퇆; 퇆; 퇆; 퇆; ) HANGUL SYLLABLE TWALM +D1C7;D1C7;1110 116A 11B2;D1C7;1110 116A 11B2; # (퇇; 퇇; 퇇; 퇇; 퇇; ) HANGUL SYLLABLE TWALB +D1C8;D1C8;1110 116A 11B3;D1C8;1110 116A 11B3; # (퇈; 퇈; 퇈; 퇈; 퇈; ) HANGUL SYLLABLE TWALS +D1C9;D1C9;1110 116A 11B4;D1C9;1110 116A 11B4; # (퇉; 퇉; 퇉; 퇉; 퇉; ) HANGUL SYLLABLE TWALT +D1CA;D1CA;1110 116A 11B5;D1CA;1110 116A 11B5; # (퇊; 퇊; 퇊; 퇊; 퇊; ) HANGUL SYLLABLE TWALP +D1CB;D1CB;1110 116A 11B6;D1CB;1110 116A 11B6; # (퇋; 퇋; 퇋; 퇋; 퇋; ) HANGUL SYLLABLE TWALH +D1CC;D1CC;1110 116A 11B7;D1CC;1110 116A 11B7; # (퇌; 퇌; 퇌; 퇌; 퇌; ) HANGUL SYLLABLE TWAM +D1CD;D1CD;1110 116A 11B8;D1CD;1110 116A 11B8; # (퇍; 퇍; 퇍; 퇍; 퇍; ) HANGUL SYLLABLE TWAB +D1CE;D1CE;1110 116A 11B9;D1CE;1110 116A 11B9; # (퇎; 퇎; 퇎; 퇎; 퇎; ) HANGUL SYLLABLE TWABS +D1CF;D1CF;1110 116A 11BA;D1CF;1110 116A 11BA; # (퇏; 퇏; 퇏; 퇏; 퇏; ) HANGUL SYLLABLE TWAS +D1D0;D1D0;1110 116A 11BB;D1D0;1110 116A 11BB; # (퇐; 퇐; 퇐; 퇐; 퇐; ) HANGUL SYLLABLE TWASS +D1D1;D1D1;1110 116A 11BC;D1D1;1110 116A 11BC; # (퇑; 퇑; 퇑; 퇑; 퇑; ) HANGUL SYLLABLE TWANG +D1D2;D1D2;1110 116A 11BD;D1D2;1110 116A 11BD; # (퇒; 퇒; 퇒; 퇒; 퇒; ) HANGUL SYLLABLE TWAJ +D1D3;D1D3;1110 116A 11BE;D1D3;1110 116A 11BE; # (퇓; 퇓; 퇓; 퇓; 퇓; ) HANGUL SYLLABLE TWAC +D1D4;D1D4;1110 116A 11BF;D1D4;1110 116A 11BF; # (퇔; 퇔; 퇔; 퇔; 퇔; ) HANGUL SYLLABLE TWAK +D1D5;D1D5;1110 116A 11C0;D1D5;1110 116A 11C0; # (퇕; 퇕; 퇕; 퇕; 퇕; ) HANGUL SYLLABLE TWAT +D1D6;D1D6;1110 116A 11C1;D1D6;1110 116A 11C1; # (퇖; 퇖; 퇖; 퇖; 퇖; ) HANGUL SYLLABLE TWAP +D1D7;D1D7;1110 116A 11C2;D1D7;1110 116A 11C2; # (퇗; 퇗; 퇗; 퇗; 퇗; ) HANGUL SYLLABLE TWAH +D1D8;D1D8;1110 116B;D1D8;1110 116B; # (퇘; 퇘; 퇘; 퇘; 퇘; ) HANGUL SYLLABLE TWAE +D1D9;D1D9;1110 116B 11A8;D1D9;1110 116B 11A8; # (퇙; 퇙; 퇙; 퇙; 퇙; ) HANGUL SYLLABLE TWAEG +D1DA;D1DA;1110 116B 11A9;D1DA;1110 116B 11A9; # (퇚; 퇚; 퇚; 퇚; 퇚; ) HANGUL SYLLABLE TWAEGG +D1DB;D1DB;1110 116B 11AA;D1DB;1110 116B 11AA; # (퇛; 퇛; 퇛; 퇛; 퇛; ) HANGUL SYLLABLE TWAEGS +D1DC;D1DC;1110 116B 11AB;D1DC;1110 116B 11AB; # (퇜; 퇜; 퇜; 퇜; 퇜; ) HANGUL SYLLABLE TWAEN +D1DD;D1DD;1110 116B 11AC;D1DD;1110 116B 11AC; # (퇝; 퇝; 퇝; 퇝; 퇝; ) HANGUL SYLLABLE TWAENJ +D1DE;D1DE;1110 116B 11AD;D1DE;1110 116B 11AD; # (퇞; 퇞; 퇞; 퇞; 퇞; ) HANGUL SYLLABLE TWAENH +D1DF;D1DF;1110 116B 11AE;D1DF;1110 116B 11AE; # (퇟; 퇟; 퇟; 퇟; 퇟; ) HANGUL SYLLABLE TWAED +D1E0;D1E0;1110 116B 11AF;D1E0;1110 116B 11AF; # (퇠; 퇠; 퇠; 퇠; 퇠; ) HANGUL SYLLABLE TWAEL +D1E1;D1E1;1110 116B 11B0;D1E1;1110 116B 11B0; # (퇡; 퇡; 퇡; 퇡; 퇡; ) HANGUL SYLLABLE TWAELG +D1E2;D1E2;1110 116B 11B1;D1E2;1110 116B 11B1; # (퇢; 퇢; 퇢; 퇢; 퇢; ) HANGUL SYLLABLE TWAELM +D1E3;D1E3;1110 116B 11B2;D1E3;1110 116B 11B2; # (퇣; 퇣; 퇣; 퇣; 퇣; ) HANGUL SYLLABLE TWAELB +D1E4;D1E4;1110 116B 11B3;D1E4;1110 116B 11B3; # (퇤; 퇤; 퇤; 퇤; 퇤; ) HANGUL SYLLABLE TWAELS +D1E5;D1E5;1110 116B 11B4;D1E5;1110 116B 11B4; # (퇥; 퇥; 퇥; 퇥; 퇥; ) HANGUL SYLLABLE TWAELT +D1E6;D1E6;1110 116B 11B5;D1E6;1110 116B 11B5; # (퇦; 퇦; 퇦; 퇦; 퇦; ) HANGUL SYLLABLE TWAELP +D1E7;D1E7;1110 116B 11B6;D1E7;1110 116B 11B6; # (퇧; 퇧; 퇧; 퇧; 퇧; ) HANGUL SYLLABLE TWAELH +D1E8;D1E8;1110 116B 11B7;D1E8;1110 116B 11B7; # (퇨; 퇨; 퇨; 퇨; 퇨; ) HANGUL SYLLABLE TWAEM +D1E9;D1E9;1110 116B 11B8;D1E9;1110 116B 11B8; # (퇩; 퇩; 퇩; 퇩; 퇩; ) HANGUL SYLLABLE TWAEB +D1EA;D1EA;1110 116B 11B9;D1EA;1110 116B 11B9; # (퇪; 퇪; 퇪; 퇪; 퇪; ) HANGUL SYLLABLE TWAEBS +D1EB;D1EB;1110 116B 11BA;D1EB;1110 116B 11BA; # (퇫; 퇫; 퇫; 퇫; 퇫; ) HANGUL SYLLABLE TWAES +D1EC;D1EC;1110 116B 11BB;D1EC;1110 116B 11BB; # (퇬; 퇬; 퇬; 퇬; 퇬; ) HANGUL SYLLABLE TWAESS +D1ED;D1ED;1110 116B 11BC;D1ED;1110 116B 11BC; # (퇭; 퇭; 퇭; 퇭; 퇭; ) HANGUL SYLLABLE TWAENG +D1EE;D1EE;1110 116B 11BD;D1EE;1110 116B 11BD; # (퇮; 퇮; 퇮; 퇮; 퇮; ) HANGUL SYLLABLE TWAEJ +D1EF;D1EF;1110 116B 11BE;D1EF;1110 116B 11BE; # (퇯; 퇯; 퇯; 퇯; 퇯; ) HANGUL SYLLABLE TWAEC +D1F0;D1F0;1110 116B 11BF;D1F0;1110 116B 11BF; # (퇰; 퇰; 퇰; 퇰; 퇰; ) HANGUL SYLLABLE TWAEK +D1F1;D1F1;1110 116B 11C0;D1F1;1110 116B 11C0; # (퇱; 퇱; 퇱; 퇱; 퇱; ) HANGUL SYLLABLE TWAET +D1F2;D1F2;1110 116B 11C1;D1F2;1110 116B 11C1; # (퇲; 퇲; 퇲; 퇲; 퇲; ) HANGUL SYLLABLE TWAEP +D1F3;D1F3;1110 116B 11C2;D1F3;1110 116B 11C2; # (퇳; 퇳; 퇳; 퇳; 퇳; ) HANGUL SYLLABLE TWAEH +D1F4;D1F4;1110 116C;D1F4;1110 116C; # (퇴; 퇴; 퇴; 퇴; 퇴; ) HANGUL SYLLABLE TOE +D1F5;D1F5;1110 116C 11A8;D1F5;1110 116C 11A8; # (퇵; 퇵; 퇵; 퇵; 퇵; ) HANGUL SYLLABLE TOEG +D1F6;D1F6;1110 116C 11A9;D1F6;1110 116C 11A9; # (퇶; 퇶; 퇶; 퇶; 퇶; ) HANGUL SYLLABLE TOEGG +D1F7;D1F7;1110 116C 11AA;D1F7;1110 116C 11AA; # (퇷; 퇷; 퇷; 퇷; 퇷; ) HANGUL SYLLABLE TOEGS +D1F8;D1F8;1110 116C 11AB;D1F8;1110 116C 11AB; # (퇸; 퇸; 퇸; 퇸; 퇸; ) HANGUL SYLLABLE TOEN +D1F9;D1F9;1110 116C 11AC;D1F9;1110 116C 11AC; # (퇹; 퇹; 퇹; 퇹; 퇹; ) HANGUL SYLLABLE TOENJ +D1FA;D1FA;1110 116C 11AD;D1FA;1110 116C 11AD; # (퇺; 퇺; 퇺; 퇺; 퇺; ) HANGUL SYLLABLE TOENH +D1FB;D1FB;1110 116C 11AE;D1FB;1110 116C 11AE; # (퇻; 퇻; 퇻; 퇻; 퇻; ) HANGUL SYLLABLE TOED +D1FC;D1FC;1110 116C 11AF;D1FC;1110 116C 11AF; # (퇼; 퇼; 퇼; 퇼; 퇼; ) HANGUL SYLLABLE TOEL +D1FD;D1FD;1110 116C 11B0;D1FD;1110 116C 11B0; # (퇽; 퇽; 퇽; 퇽; 퇽; ) HANGUL SYLLABLE TOELG +D1FE;D1FE;1110 116C 11B1;D1FE;1110 116C 11B1; # (퇾; 퇾; 퇾; 퇾; 퇾; ) HANGUL SYLLABLE TOELM +D1FF;D1FF;1110 116C 11B2;D1FF;1110 116C 11B2; # (퇿; 퇿; 퇿; 퇿; 퇿; ) HANGUL SYLLABLE TOELB +D200;D200;1110 116C 11B3;D200;1110 116C 11B3; # (툀; 툀; 툀; 툀; 툀; ) HANGUL SYLLABLE TOELS +D201;D201;1110 116C 11B4;D201;1110 116C 11B4; # (툁; 툁; 툁; 툁; 툁; ) HANGUL SYLLABLE TOELT +D202;D202;1110 116C 11B5;D202;1110 116C 11B5; # (툂; 툂; 툂; 툂; 툂; ) HANGUL SYLLABLE TOELP +D203;D203;1110 116C 11B6;D203;1110 116C 11B6; # (툃; 툃; 툃; 툃; 툃; ) HANGUL SYLLABLE TOELH +D204;D204;1110 116C 11B7;D204;1110 116C 11B7; # (툄; 툄; 툄; 툄; 툄; ) HANGUL SYLLABLE TOEM +D205;D205;1110 116C 11B8;D205;1110 116C 11B8; # (툅; 툅; 툅; 툅; 툅; ) HANGUL SYLLABLE TOEB +D206;D206;1110 116C 11B9;D206;1110 116C 11B9; # (툆; 툆; 툆; 툆; 툆; ) HANGUL SYLLABLE TOEBS +D207;D207;1110 116C 11BA;D207;1110 116C 11BA; # (툇; 툇; 툇; 툇; 툇; ) HANGUL SYLLABLE TOES +D208;D208;1110 116C 11BB;D208;1110 116C 11BB; # (툈; 툈; 툈; 툈; 툈; ) HANGUL SYLLABLE TOESS +D209;D209;1110 116C 11BC;D209;1110 116C 11BC; # (툉; 툉; 툉; 툉; 툉; ) HANGUL SYLLABLE TOENG +D20A;D20A;1110 116C 11BD;D20A;1110 116C 11BD; # (툊; 툊; 툊; 툊; 툊; ) HANGUL SYLLABLE TOEJ +D20B;D20B;1110 116C 11BE;D20B;1110 116C 11BE; # (툋; 툋; 툋; 툋; 툋; ) HANGUL SYLLABLE TOEC +D20C;D20C;1110 116C 11BF;D20C;1110 116C 11BF; # (툌; 툌; 툌; 툌; 툌; ) HANGUL SYLLABLE TOEK +D20D;D20D;1110 116C 11C0;D20D;1110 116C 11C0; # (툍; 툍; 툍; 툍; 툍; ) HANGUL SYLLABLE TOET +D20E;D20E;1110 116C 11C1;D20E;1110 116C 11C1; # (툎; 툎; 툎; 툎; 툎; ) HANGUL SYLLABLE TOEP +D20F;D20F;1110 116C 11C2;D20F;1110 116C 11C2; # (툏; 툏; 툏; 툏; 툏; ) HANGUL SYLLABLE TOEH +D210;D210;1110 116D;D210;1110 116D; # (툐; 툐; 툐; 툐; 툐; ) HANGUL SYLLABLE TYO +D211;D211;1110 116D 11A8;D211;1110 116D 11A8; # (툑; 툑; 툑; 툑; 툑; ) HANGUL SYLLABLE TYOG +D212;D212;1110 116D 11A9;D212;1110 116D 11A9; # (툒; 툒; 툒; 툒; 툒; ) HANGUL SYLLABLE TYOGG +D213;D213;1110 116D 11AA;D213;1110 116D 11AA; # (툓; 툓; 툓; 툓; 툓; ) HANGUL SYLLABLE TYOGS +D214;D214;1110 116D 11AB;D214;1110 116D 11AB; # (툔; 툔; 툔; 툔; 툔; ) HANGUL SYLLABLE TYON +D215;D215;1110 116D 11AC;D215;1110 116D 11AC; # (툕; 툕; 툕; 툕; 툕; ) HANGUL SYLLABLE TYONJ +D216;D216;1110 116D 11AD;D216;1110 116D 11AD; # (툖; 툖; 툖; 툖; 툖; ) HANGUL SYLLABLE TYONH +D217;D217;1110 116D 11AE;D217;1110 116D 11AE; # (툗; 툗; 툗; 툗; 툗; ) HANGUL SYLLABLE TYOD +D218;D218;1110 116D 11AF;D218;1110 116D 11AF; # (툘; 툘; 툘; 툘; 툘; ) HANGUL SYLLABLE TYOL +D219;D219;1110 116D 11B0;D219;1110 116D 11B0; # (툙; 툙; 툙; 툙; 툙; ) HANGUL SYLLABLE TYOLG +D21A;D21A;1110 116D 11B1;D21A;1110 116D 11B1; # (툚; 툚; 툚; 툚; 툚; ) HANGUL SYLLABLE TYOLM +D21B;D21B;1110 116D 11B2;D21B;1110 116D 11B2; # (툛; 툛; 툛; 툛; 툛; ) HANGUL SYLLABLE TYOLB +D21C;D21C;1110 116D 11B3;D21C;1110 116D 11B3; # (툜; 툜; 툜; 툜; 툜; ) HANGUL SYLLABLE TYOLS +D21D;D21D;1110 116D 11B4;D21D;1110 116D 11B4; # (툝; 툝; 툝; 툝; 툝; ) HANGUL SYLLABLE TYOLT +D21E;D21E;1110 116D 11B5;D21E;1110 116D 11B5; # (툞; 툞; 툞; 툞; 툞; ) HANGUL SYLLABLE TYOLP +D21F;D21F;1110 116D 11B6;D21F;1110 116D 11B6; # (툟; 툟; 툟; 툟; 툟; ) HANGUL SYLLABLE TYOLH +D220;D220;1110 116D 11B7;D220;1110 116D 11B7; # (툠; 툠; 툠; 툠; 툠; ) HANGUL SYLLABLE TYOM +D221;D221;1110 116D 11B8;D221;1110 116D 11B8; # (툡; 툡; 툡; 툡; 툡; ) HANGUL SYLLABLE TYOB +D222;D222;1110 116D 11B9;D222;1110 116D 11B9; # (툢; 툢; 툢; 툢; 툢; ) HANGUL SYLLABLE TYOBS +D223;D223;1110 116D 11BA;D223;1110 116D 11BA; # (툣; 툣; 툣; 툣; 툣; ) HANGUL SYLLABLE TYOS +D224;D224;1110 116D 11BB;D224;1110 116D 11BB; # (툤; 툤; 툤; 툤; 툤; ) HANGUL SYLLABLE TYOSS +D225;D225;1110 116D 11BC;D225;1110 116D 11BC; # (툥; 툥; 툥; 툥; 툥; ) HANGUL SYLLABLE TYONG +D226;D226;1110 116D 11BD;D226;1110 116D 11BD; # (툦; 툦; 툦; 툦; 툦; ) HANGUL SYLLABLE TYOJ +D227;D227;1110 116D 11BE;D227;1110 116D 11BE; # (툧; 툧; 툧; 툧; 툧; ) HANGUL SYLLABLE TYOC +D228;D228;1110 116D 11BF;D228;1110 116D 11BF; # (툨; 툨; 툨; 툨; 툨; ) HANGUL SYLLABLE TYOK +D229;D229;1110 116D 11C0;D229;1110 116D 11C0; # (툩; 툩; 툩; 툩; 툩; ) HANGUL SYLLABLE TYOT +D22A;D22A;1110 116D 11C1;D22A;1110 116D 11C1; # (툪; 툪; 툪; 툪; 툪; ) HANGUL SYLLABLE TYOP +D22B;D22B;1110 116D 11C2;D22B;1110 116D 11C2; # (툫; 툫; 툫; 툫; 툫; ) HANGUL SYLLABLE TYOH +D22C;D22C;1110 116E;D22C;1110 116E; # (투; 투; 투; 투; 투; ) HANGUL SYLLABLE TU +D22D;D22D;1110 116E 11A8;D22D;1110 116E 11A8; # (툭; 툭; 툭; 툭; 툭; ) HANGUL SYLLABLE TUG +D22E;D22E;1110 116E 11A9;D22E;1110 116E 11A9; # (툮; 툮; 툮; 툮; 툮; ) HANGUL SYLLABLE TUGG +D22F;D22F;1110 116E 11AA;D22F;1110 116E 11AA; # (툯; 툯; 툯; 툯; 툯; ) HANGUL SYLLABLE TUGS +D230;D230;1110 116E 11AB;D230;1110 116E 11AB; # (툰; 툰; 툰; 툰; 툰; ) HANGUL SYLLABLE TUN +D231;D231;1110 116E 11AC;D231;1110 116E 11AC; # (툱; 툱; 툱; 툱; 툱; ) HANGUL SYLLABLE TUNJ +D232;D232;1110 116E 11AD;D232;1110 116E 11AD; # (툲; 툲; 툲; 툲; 툲; ) HANGUL SYLLABLE TUNH +D233;D233;1110 116E 11AE;D233;1110 116E 11AE; # (툳; 툳; 툳; 툳; 툳; ) HANGUL SYLLABLE TUD +D234;D234;1110 116E 11AF;D234;1110 116E 11AF; # (툴; 툴; 툴; 툴; 툴; ) HANGUL SYLLABLE TUL +D235;D235;1110 116E 11B0;D235;1110 116E 11B0; # (툵; 툵; 툵; 툵; 툵; ) HANGUL SYLLABLE TULG +D236;D236;1110 116E 11B1;D236;1110 116E 11B1; # (툶; 툶; 툶; 툶; 툶; ) HANGUL SYLLABLE TULM +D237;D237;1110 116E 11B2;D237;1110 116E 11B2; # (툷; 툷; 툷; 툷; 툷; ) HANGUL SYLLABLE TULB +D238;D238;1110 116E 11B3;D238;1110 116E 11B3; # (툸; 툸; 툸; 툸; 툸; ) HANGUL SYLLABLE TULS +D239;D239;1110 116E 11B4;D239;1110 116E 11B4; # (툹; 툹; 툹; 툹; 툹; ) HANGUL SYLLABLE TULT +D23A;D23A;1110 116E 11B5;D23A;1110 116E 11B5; # (툺; 툺; 툺; 툺; 툺; ) HANGUL SYLLABLE TULP +D23B;D23B;1110 116E 11B6;D23B;1110 116E 11B6; # (툻; 툻; 툻; 툻; 툻; ) HANGUL SYLLABLE TULH +D23C;D23C;1110 116E 11B7;D23C;1110 116E 11B7; # (툼; 툼; 툼; 툼; 툼; ) HANGUL SYLLABLE TUM +D23D;D23D;1110 116E 11B8;D23D;1110 116E 11B8; # (툽; 툽; 툽; 툽; 툽; ) HANGUL SYLLABLE TUB +D23E;D23E;1110 116E 11B9;D23E;1110 116E 11B9; # (툾; 툾; 툾; 툾; 툾; ) HANGUL SYLLABLE TUBS +D23F;D23F;1110 116E 11BA;D23F;1110 116E 11BA; # (툿; 툿; 툿; 툿; 툿; ) HANGUL SYLLABLE TUS +D240;D240;1110 116E 11BB;D240;1110 116E 11BB; # (퉀; 퉀; 퉀; 퉀; 퉀; ) HANGUL SYLLABLE TUSS +D241;D241;1110 116E 11BC;D241;1110 116E 11BC; # (퉁; 퉁; 퉁; 퉁; 퉁; ) HANGUL SYLLABLE TUNG +D242;D242;1110 116E 11BD;D242;1110 116E 11BD; # (퉂; 퉂; 퉂; 퉂; 퉂; ) HANGUL SYLLABLE TUJ +D243;D243;1110 116E 11BE;D243;1110 116E 11BE; # (퉃; 퉃; 퉃; 퉃; 퉃; ) HANGUL SYLLABLE TUC +D244;D244;1110 116E 11BF;D244;1110 116E 11BF; # (퉄; 퉄; 퉄; 퉄; 퉄; ) HANGUL SYLLABLE TUK +D245;D245;1110 116E 11C0;D245;1110 116E 11C0; # (퉅; 퉅; 퉅; 퉅; 퉅; ) HANGUL SYLLABLE TUT +D246;D246;1110 116E 11C1;D246;1110 116E 11C1; # (퉆; 퉆; 퉆; 퉆; 퉆; ) HANGUL SYLLABLE TUP +D247;D247;1110 116E 11C2;D247;1110 116E 11C2; # (퉇; 퉇; 퉇; 퉇; 퉇; ) HANGUL SYLLABLE TUH +D248;D248;1110 116F;D248;1110 116F; # (퉈; 퉈; 퉈; 퉈; 퉈; ) HANGUL SYLLABLE TWEO +D249;D249;1110 116F 11A8;D249;1110 116F 11A8; # (퉉; 퉉; 퉉; 퉉; 퉉; ) HANGUL SYLLABLE TWEOG +D24A;D24A;1110 116F 11A9;D24A;1110 116F 11A9; # (퉊; 퉊; 퉊; 퉊; 퉊; ) HANGUL SYLLABLE TWEOGG +D24B;D24B;1110 116F 11AA;D24B;1110 116F 11AA; # (퉋; 퉋; 퉋; 퉋; 퉋; ) HANGUL SYLLABLE TWEOGS +D24C;D24C;1110 116F 11AB;D24C;1110 116F 11AB; # (퉌; 퉌; 퉌; 퉌; 퉌; ) HANGUL SYLLABLE TWEON +D24D;D24D;1110 116F 11AC;D24D;1110 116F 11AC; # (퉍; 퉍; 퉍; 퉍; 퉍; ) HANGUL SYLLABLE TWEONJ +D24E;D24E;1110 116F 11AD;D24E;1110 116F 11AD; # (퉎; 퉎; 퉎; 퉎; 퉎; ) HANGUL SYLLABLE TWEONH +D24F;D24F;1110 116F 11AE;D24F;1110 116F 11AE; # (퉏; 퉏; 퉏; 퉏; 퉏; ) HANGUL SYLLABLE TWEOD +D250;D250;1110 116F 11AF;D250;1110 116F 11AF; # (퉐; 퉐; 퉐; 퉐; 퉐; ) HANGUL SYLLABLE TWEOL +D251;D251;1110 116F 11B0;D251;1110 116F 11B0; # (퉑; 퉑; 퉑; 퉑; 퉑; ) HANGUL SYLLABLE TWEOLG +D252;D252;1110 116F 11B1;D252;1110 116F 11B1; # (퉒; 퉒; 퉒; 퉒; 퉒; ) HANGUL SYLLABLE TWEOLM +D253;D253;1110 116F 11B2;D253;1110 116F 11B2; # (퉓; 퉓; 퉓; 퉓; 퉓; ) HANGUL SYLLABLE TWEOLB +D254;D254;1110 116F 11B3;D254;1110 116F 11B3; # (퉔; 퉔; 퉔; 퉔; 퉔; ) HANGUL SYLLABLE TWEOLS +D255;D255;1110 116F 11B4;D255;1110 116F 11B4; # (퉕; 퉕; 퉕; 퉕; 퉕; ) HANGUL SYLLABLE TWEOLT +D256;D256;1110 116F 11B5;D256;1110 116F 11B5; # (퉖; 퉖; 퉖; 퉖; 퉖; ) HANGUL SYLLABLE TWEOLP +D257;D257;1110 116F 11B6;D257;1110 116F 11B6; # (퉗; 퉗; 퉗; 퉗; 퉗; ) HANGUL SYLLABLE TWEOLH +D258;D258;1110 116F 11B7;D258;1110 116F 11B7; # (퉘; 퉘; 퉘; 퉘; 퉘; ) HANGUL SYLLABLE TWEOM +D259;D259;1110 116F 11B8;D259;1110 116F 11B8; # (퉙; 퉙; 퉙; 퉙; 퉙; ) HANGUL SYLLABLE TWEOB +D25A;D25A;1110 116F 11B9;D25A;1110 116F 11B9; # (퉚; 퉚; 퉚; 퉚; 퉚; ) HANGUL SYLLABLE TWEOBS +D25B;D25B;1110 116F 11BA;D25B;1110 116F 11BA; # (퉛; 퉛; 퉛; 퉛; 퉛; ) HANGUL SYLLABLE TWEOS +D25C;D25C;1110 116F 11BB;D25C;1110 116F 11BB; # (퉜; 퉜; 퉜; 퉜; 퉜; ) HANGUL SYLLABLE TWEOSS +D25D;D25D;1110 116F 11BC;D25D;1110 116F 11BC; # (퉝; 퉝; 퉝; 퉝; 퉝; ) HANGUL SYLLABLE TWEONG +D25E;D25E;1110 116F 11BD;D25E;1110 116F 11BD; # (퉞; 퉞; 퉞; 퉞; 퉞; ) HANGUL SYLLABLE TWEOJ +D25F;D25F;1110 116F 11BE;D25F;1110 116F 11BE; # (퉟; 퉟; 퉟; 퉟; 퉟; ) HANGUL SYLLABLE TWEOC +D260;D260;1110 116F 11BF;D260;1110 116F 11BF; # (퉠; 퉠; 퉠; 퉠; 퉠; ) HANGUL SYLLABLE TWEOK +D261;D261;1110 116F 11C0;D261;1110 116F 11C0; # (퉡; 퉡; 퉡; 퉡; 퉡; ) HANGUL SYLLABLE TWEOT +D262;D262;1110 116F 11C1;D262;1110 116F 11C1; # (퉢; 퉢; 퉢; 퉢; 퉢; ) HANGUL SYLLABLE TWEOP +D263;D263;1110 116F 11C2;D263;1110 116F 11C2; # (퉣; 퉣; 퉣; 퉣; 퉣; ) HANGUL SYLLABLE TWEOH +D264;D264;1110 1170;D264;1110 1170; # (퉤; 퉤; 퉤; 퉤; 퉤; ) HANGUL SYLLABLE TWE +D265;D265;1110 1170 11A8;D265;1110 1170 11A8; # (퉥; 퉥; 퉥; 퉥; 퉥; ) HANGUL SYLLABLE TWEG +D266;D266;1110 1170 11A9;D266;1110 1170 11A9; # (퉦; 퉦; 퉦; 퉦; 퉦; ) HANGUL SYLLABLE TWEGG +D267;D267;1110 1170 11AA;D267;1110 1170 11AA; # (퉧; 퉧; 퉧; 퉧; 퉧; ) HANGUL SYLLABLE TWEGS +D268;D268;1110 1170 11AB;D268;1110 1170 11AB; # (퉨; 퉨; 퉨; 퉨; 퉨; ) HANGUL SYLLABLE TWEN +D269;D269;1110 1170 11AC;D269;1110 1170 11AC; # (퉩; 퉩; 퉩; 퉩; 퉩; ) HANGUL SYLLABLE TWENJ +D26A;D26A;1110 1170 11AD;D26A;1110 1170 11AD; # (퉪; 퉪; 퉪; 퉪; 퉪; ) HANGUL SYLLABLE TWENH +D26B;D26B;1110 1170 11AE;D26B;1110 1170 11AE; # (퉫; 퉫; 퉫; 퉫; 퉫; ) HANGUL SYLLABLE TWED +D26C;D26C;1110 1170 11AF;D26C;1110 1170 11AF; # (퉬; 퉬; 퉬; 퉬; 퉬; ) HANGUL SYLLABLE TWEL +D26D;D26D;1110 1170 11B0;D26D;1110 1170 11B0; # (퉭; 퉭; 퉭; 퉭; 퉭; ) HANGUL SYLLABLE TWELG +D26E;D26E;1110 1170 11B1;D26E;1110 1170 11B1; # (퉮; 퉮; 퉮; 퉮; 퉮; ) HANGUL SYLLABLE TWELM +D26F;D26F;1110 1170 11B2;D26F;1110 1170 11B2; # (퉯; 퉯; 퉯; 퉯; 퉯; ) HANGUL SYLLABLE TWELB +D270;D270;1110 1170 11B3;D270;1110 1170 11B3; # (퉰; 퉰; 퉰; 퉰; 퉰; ) HANGUL SYLLABLE TWELS +D271;D271;1110 1170 11B4;D271;1110 1170 11B4; # (퉱; 퉱; 퉱; 퉱; 퉱; ) HANGUL SYLLABLE TWELT +D272;D272;1110 1170 11B5;D272;1110 1170 11B5; # (퉲; 퉲; 퉲; 퉲; 퉲; ) HANGUL SYLLABLE TWELP +D273;D273;1110 1170 11B6;D273;1110 1170 11B6; # (퉳; 퉳; 퉳; 퉳; 퉳; ) HANGUL SYLLABLE TWELH +D274;D274;1110 1170 11B7;D274;1110 1170 11B7; # (퉴; 퉴; 퉴; 퉴; 퉴; ) HANGUL SYLLABLE TWEM +D275;D275;1110 1170 11B8;D275;1110 1170 11B8; # (퉵; 퉵; 퉵; 퉵; 퉵; ) HANGUL SYLLABLE TWEB +D276;D276;1110 1170 11B9;D276;1110 1170 11B9; # (퉶; 퉶; 퉶; 퉶; 퉶; ) HANGUL SYLLABLE TWEBS +D277;D277;1110 1170 11BA;D277;1110 1170 11BA; # (퉷; 퉷; 퉷; 퉷; 퉷; ) HANGUL SYLLABLE TWES +D278;D278;1110 1170 11BB;D278;1110 1170 11BB; # (퉸; 퉸; 퉸; 퉸; 퉸; ) HANGUL SYLLABLE TWESS +D279;D279;1110 1170 11BC;D279;1110 1170 11BC; # (퉹; 퉹; 퉹; 퉹; 퉹; ) HANGUL SYLLABLE TWENG +D27A;D27A;1110 1170 11BD;D27A;1110 1170 11BD; # (퉺; 퉺; 퉺; 퉺; 퉺; ) HANGUL SYLLABLE TWEJ +D27B;D27B;1110 1170 11BE;D27B;1110 1170 11BE; # (퉻; 퉻; 퉻; 퉻; 퉻; ) HANGUL SYLLABLE TWEC +D27C;D27C;1110 1170 11BF;D27C;1110 1170 11BF; # (퉼; 퉼; 퉼; 퉼; 퉼; ) HANGUL SYLLABLE TWEK +D27D;D27D;1110 1170 11C0;D27D;1110 1170 11C0; # (퉽; 퉽; 퉽; 퉽; 퉽; ) HANGUL SYLLABLE TWET +D27E;D27E;1110 1170 11C1;D27E;1110 1170 11C1; # (퉾; 퉾; 퉾; 퉾; 퉾; ) HANGUL SYLLABLE TWEP +D27F;D27F;1110 1170 11C2;D27F;1110 1170 11C2; # (퉿; 퉿; 퉿; 퉿; 퉿; ) HANGUL SYLLABLE TWEH +D280;D280;1110 1171;D280;1110 1171; # (튀; 튀; 튀; 튀; 튀; ) HANGUL SYLLABLE TWI +D281;D281;1110 1171 11A8;D281;1110 1171 11A8; # (튁; 튁; 튁; 튁; 튁; ) HANGUL SYLLABLE TWIG +D282;D282;1110 1171 11A9;D282;1110 1171 11A9; # (튂; 튂; 튂; 튂; 튂; ) HANGUL SYLLABLE TWIGG +D283;D283;1110 1171 11AA;D283;1110 1171 11AA; # (튃; 튃; 튃; 튃; 튃; ) HANGUL SYLLABLE TWIGS +D284;D284;1110 1171 11AB;D284;1110 1171 11AB; # (튄; 튄; 튄; 튄; 튄; ) HANGUL SYLLABLE TWIN +D285;D285;1110 1171 11AC;D285;1110 1171 11AC; # (튅; 튅; 튅; 튅; 튅; ) HANGUL SYLLABLE TWINJ +D286;D286;1110 1171 11AD;D286;1110 1171 11AD; # (튆; 튆; 튆; 튆; 튆; ) HANGUL SYLLABLE TWINH +D287;D287;1110 1171 11AE;D287;1110 1171 11AE; # (튇; 튇; 튇; 튇; 튇; ) HANGUL SYLLABLE TWID +D288;D288;1110 1171 11AF;D288;1110 1171 11AF; # (튈; 튈; 튈; 튈; 튈; ) HANGUL SYLLABLE TWIL +D289;D289;1110 1171 11B0;D289;1110 1171 11B0; # (튉; 튉; 튉; 튉; 튉; ) HANGUL SYLLABLE TWILG +D28A;D28A;1110 1171 11B1;D28A;1110 1171 11B1; # (튊; 튊; 튊; 튊; 튊; ) HANGUL SYLLABLE TWILM +D28B;D28B;1110 1171 11B2;D28B;1110 1171 11B2; # (튋; 튋; 튋; 튋; 튋; ) HANGUL SYLLABLE TWILB +D28C;D28C;1110 1171 11B3;D28C;1110 1171 11B3; # (튌; 튌; 튌; 튌; 튌; ) HANGUL SYLLABLE TWILS +D28D;D28D;1110 1171 11B4;D28D;1110 1171 11B4; # (튍; 튍; 튍; 튍; 튍; ) HANGUL SYLLABLE TWILT +D28E;D28E;1110 1171 11B5;D28E;1110 1171 11B5; # (튎; 튎; 튎; 튎; 튎; ) HANGUL SYLLABLE TWILP +D28F;D28F;1110 1171 11B6;D28F;1110 1171 11B6; # (튏; 튏; 튏; 튏; 튏; ) HANGUL SYLLABLE TWILH +D290;D290;1110 1171 11B7;D290;1110 1171 11B7; # (튐; 튐; 튐; 튐; 튐; ) HANGUL SYLLABLE TWIM +D291;D291;1110 1171 11B8;D291;1110 1171 11B8; # (튑; 튑; 튑; 튑; 튑; ) HANGUL SYLLABLE TWIB +D292;D292;1110 1171 11B9;D292;1110 1171 11B9; # (튒; 튒; 튒; 튒; 튒; ) HANGUL SYLLABLE TWIBS +D293;D293;1110 1171 11BA;D293;1110 1171 11BA; # (튓; 튓; 튓; 튓; 튓; ) HANGUL SYLLABLE TWIS +D294;D294;1110 1171 11BB;D294;1110 1171 11BB; # (튔; 튔; 튔; 튔; 튔; ) HANGUL SYLLABLE TWISS +D295;D295;1110 1171 11BC;D295;1110 1171 11BC; # (튕; 튕; 튕; 튕; 튕; ) HANGUL SYLLABLE TWING +D296;D296;1110 1171 11BD;D296;1110 1171 11BD; # (튖; 튖; 튖; 튖; 튖; ) HANGUL SYLLABLE TWIJ +D297;D297;1110 1171 11BE;D297;1110 1171 11BE; # (튗; 튗; 튗; 튗; 튗; ) HANGUL SYLLABLE TWIC +D298;D298;1110 1171 11BF;D298;1110 1171 11BF; # (튘; 튘; 튘; 튘; 튘; ) HANGUL SYLLABLE TWIK +D299;D299;1110 1171 11C0;D299;1110 1171 11C0; # (튙; 튙; 튙; 튙; 튙; ) HANGUL SYLLABLE TWIT +D29A;D29A;1110 1171 11C1;D29A;1110 1171 11C1; # (튚; 튚; 튚; 튚; 튚; ) HANGUL SYLLABLE TWIP +D29B;D29B;1110 1171 11C2;D29B;1110 1171 11C2; # (튛; 튛; 튛; 튛; 튛; ) HANGUL SYLLABLE TWIH +D29C;D29C;1110 1172;D29C;1110 1172; # (튜; 튜; 튜; 튜; 튜; ) HANGUL SYLLABLE TYU +D29D;D29D;1110 1172 11A8;D29D;1110 1172 11A8; # (튝; 튝; 튝; 튝; 튝; ) HANGUL SYLLABLE TYUG +D29E;D29E;1110 1172 11A9;D29E;1110 1172 11A9; # (튞; 튞; 튞; 튞; 튞; ) HANGUL SYLLABLE TYUGG +D29F;D29F;1110 1172 11AA;D29F;1110 1172 11AA; # (튟; 튟; 튟; 튟; 튟; ) HANGUL SYLLABLE TYUGS +D2A0;D2A0;1110 1172 11AB;D2A0;1110 1172 11AB; # (튠; 튠; 튠; 튠; 튠; ) HANGUL SYLLABLE TYUN +D2A1;D2A1;1110 1172 11AC;D2A1;1110 1172 11AC; # (튡; 튡; 튡; 튡; 튡; ) HANGUL SYLLABLE TYUNJ +D2A2;D2A2;1110 1172 11AD;D2A2;1110 1172 11AD; # (튢; 튢; 튢; 튢; 튢; ) HANGUL SYLLABLE TYUNH +D2A3;D2A3;1110 1172 11AE;D2A3;1110 1172 11AE; # (튣; 튣; 튣; 튣; 튣; ) HANGUL SYLLABLE TYUD +D2A4;D2A4;1110 1172 11AF;D2A4;1110 1172 11AF; # (튤; 튤; 튤; 튤; 튤; ) HANGUL SYLLABLE TYUL +D2A5;D2A5;1110 1172 11B0;D2A5;1110 1172 11B0; # (튥; 튥; 튥; 튥; 튥; ) HANGUL SYLLABLE TYULG +D2A6;D2A6;1110 1172 11B1;D2A6;1110 1172 11B1; # (튦; 튦; 튦; 튦; 튦; ) HANGUL SYLLABLE TYULM +D2A7;D2A7;1110 1172 11B2;D2A7;1110 1172 11B2; # (튧; 튧; 튧; 튧; 튧; ) HANGUL SYLLABLE TYULB +D2A8;D2A8;1110 1172 11B3;D2A8;1110 1172 11B3; # (튨; 튨; 튨; 튨; 튨; ) HANGUL SYLLABLE TYULS +D2A9;D2A9;1110 1172 11B4;D2A9;1110 1172 11B4; # (튩; 튩; 튩; 튩; 튩; ) HANGUL SYLLABLE TYULT +D2AA;D2AA;1110 1172 11B5;D2AA;1110 1172 11B5; # (튪; 튪; 튪; 튪; 튪; ) HANGUL SYLLABLE TYULP +D2AB;D2AB;1110 1172 11B6;D2AB;1110 1172 11B6; # (튫; 튫; 튫; 튫; 튫; ) HANGUL SYLLABLE TYULH +D2AC;D2AC;1110 1172 11B7;D2AC;1110 1172 11B7; # (튬; 튬; 튬; 튬; 튬; ) HANGUL SYLLABLE TYUM +D2AD;D2AD;1110 1172 11B8;D2AD;1110 1172 11B8; # (튭; 튭; 튭; 튭; 튭; ) HANGUL SYLLABLE TYUB +D2AE;D2AE;1110 1172 11B9;D2AE;1110 1172 11B9; # (튮; 튮; 튮; 튮; 튮; ) HANGUL SYLLABLE TYUBS +D2AF;D2AF;1110 1172 11BA;D2AF;1110 1172 11BA; # (튯; 튯; 튯; 튯; 튯; ) HANGUL SYLLABLE TYUS +D2B0;D2B0;1110 1172 11BB;D2B0;1110 1172 11BB; # (튰; 튰; 튰; 튰; 튰; ) HANGUL SYLLABLE TYUSS +D2B1;D2B1;1110 1172 11BC;D2B1;1110 1172 11BC; # (튱; 튱; 튱; 튱; 튱; ) HANGUL SYLLABLE TYUNG +D2B2;D2B2;1110 1172 11BD;D2B2;1110 1172 11BD; # (튲; 튲; 튲; 튲; 튲; ) HANGUL SYLLABLE TYUJ +D2B3;D2B3;1110 1172 11BE;D2B3;1110 1172 11BE; # (튳; 튳; 튳; 튳; 튳; ) HANGUL SYLLABLE TYUC +D2B4;D2B4;1110 1172 11BF;D2B4;1110 1172 11BF; # (튴; 튴; 튴; 튴; 튴; ) HANGUL SYLLABLE TYUK +D2B5;D2B5;1110 1172 11C0;D2B5;1110 1172 11C0; # (튵; 튵; 튵; 튵; 튵; ) HANGUL SYLLABLE TYUT +D2B6;D2B6;1110 1172 11C1;D2B6;1110 1172 11C1; # (튶; 튶; 튶; 튶; 튶; ) HANGUL SYLLABLE TYUP +D2B7;D2B7;1110 1172 11C2;D2B7;1110 1172 11C2; # (튷; 튷; 튷; 튷; 튷; ) HANGUL SYLLABLE TYUH +D2B8;D2B8;1110 1173;D2B8;1110 1173; # (트; 트; 트; 트; 트; ) HANGUL SYLLABLE TEU +D2B9;D2B9;1110 1173 11A8;D2B9;1110 1173 11A8; # (특; 특; 특; 특; 특; ) HANGUL SYLLABLE TEUG +D2BA;D2BA;1110 1173 11A9;D2BA;1110 1173 11A9; # (튺; 튺; 튺; 튺; 튺; ) HANGUL SYLLABLE TEUGG +D2BB;D2BB;1110 1173 11AA;D2BB;1110 1173 11AA; # (튻; 튻; 튻; 튻; 튻; ) HANGUL SYLLABLE TEUGS +D2BC;D2BC;1110 1173 11AB;D2BC;1110 1173 11AB; # (튼; 튼; 튼; 튼; 튼; ) HANGUL SYLLABLE TEUN +D2BD;D2BD;1110 1173 11AC;D2BD;1110 1173 11AC; # (튽; 튽; 튽; 튽; 튽; ) HANGUL SYLLABLE TEUNJ +D2BE;D2BE;1110 1173 11AD;D2BE;1110 1173 11AD; # (튾; 튾; 튾; 튾; 튾; ) HANGUL SYLLABLE TEUNH +D2BF;D2BF;1110 1173 11AE;D2BF;1110 1173 11AE; # (튿; 튿; 튿; 튿; 튿; ) HANGUL SYLLABLE TEUD +D2C0;D2C0;1110 1173 11AF;D2C0;1110 1173 11AF; # (틀; 틀; 틀; 틀; 틀; ) HANGUL SYLLABLE TEUL +D2C1;D2C1;1110 1173 11B0;D2C1;1110 1173 11B0; # (틁; 틁; 틁; 틁; 틁; ) HANGUL SYLLABLE TEULG +D2C2;D2C2;1110 1173 11B1;D2C2;1110 1173 11B1; # (틂; 틂; 틂; 틂; 틂; ) HANGUL SYLLABLE TEULM +D2C3;D2C3;1110 1173 11B2;D2C3;1110 1173 11B2; # (틃; 틃; 틃; 틃; 틃; ) HANGUL SYLLABLE TEULB +D2C4;D2C4;1110 1173 11B3;D2C4;1110 1173 11B3; # (틄; 틄; 틄; 틄; 틄; ) HANGUL SYLLABLE TEULS +D2C5;D2C5;1110 1173 11B4;D2C5;1110 1173 11B4; # (틅; 틅; 틅; 틅; 틅; ) HANGUL SYLLABLE TEULT +D2C6;D2C6;1110 1173 11B5;D2C6;1110 1173 11B5; # (틆; 틆; 틆; 틆; 틆; ) HANGUL SYLLABLE TEULP +D2C7;D2C7;1110 1173 11B6;D2C7;1110 1173 11B6; # (틇; 틇; 틇; 틇; 틇; ) HANGUL SYLLABLE TEULH +D2C8;D2C8;1110 1173 11B7;D2C8;1110 1173 11B7; # (틈; 틈; 틈; 틈; 틈; ) HANGUL SYLLABLE TEUM +D2C9;D2C9;1110 1173 11B8;D2C9;1110 1173 11B8; # (틉; 틉; 틉; 틉; 틉; ) HANGUL SYLLABLE TEUB +D2CA;D2CA;1110 1173 11B9;D2CA;1110 1173 11B9; # (틊; 틊; 틊; 틊; 틊; ) HANGUL SYLLABLE TEUBS +D2CB;D2CB;1110 1173 11BA;D2CB;1110 1173 11BA; # (틋; 틋; 틋; 틋; 틋; ) HANGUL SYLLABLE TEUS +D2CC;D2CC;1110 1173 11BB;D2CC;1110 1173 11BB; # (틌; 틌; 틌; 틌; 틌; ) HANGUL SYLLABLE TEUSS +D2CD;D2CD;1110 1173 11BC;D2CD;1110 1173 11BC; # (틍; 틍; 틍; 틍; 틍; ) HANGUL SYLLABLE TEUNG +D2CE;D2CE;1110 1173 11BD;D2CE;1110 1173 11BD; # (틎; 틎; 틎; 틎; 틎; ) HANGUL SYLLABLE TEUJ +D2CF;D2CF;1110 1173 11BE;D2CF;1110 1173 11BE; # (틏; 틏; 틏; 틏; 틏; ) HANGUL SYLLABLE TEUC +D2D0;D2D0;1110 1173 11BF;D2D0;1110 1173 11BF; # (틐; 틐; 틐; 틐; 틐; ) HANGUL SYLLABLE TEUK +D2D1;D2D1;1110 1173 11C0;D2D1;1110 1173 11C0; # (틑; 틑; 틑; 틑; 틑; ) HANGUL SYLLABLE TEUT +D2D2;D2D2;1110 1173 11C1;D2D2;1110 1173 11C1; # (틒; 틒; 틒; 틒; 틒; ) HANGUL SYLLABLE TEUP +D2D3;D2D3;1110 1173 11C2;D2D3;1110 1173 11C2; # (틓; 틓; 틓; 틓; 틓; ) HANGUL SYLLABLE TEUH +D2D4;D2D4;1110 1174;D2D4;1110 1174; # (틔; 틔; 틔; 틔; 틔; ) HANGUL SYLLABLE TYI +D2D5;D2D5;1110 1174 11A8;D2D5;1110 1174 11A8; # (틕; 틕; 틕; 틕; 틕; ) HANGUL SYLLABLE TYIG +D2D6;D2D6;1110 1174 11A9;D2D6;1110 1174 11A9; # (틖; 틖; 틖; 틖; 틖; ) HANGUL SYLLABLE TYIGG +D2D7;D2D7;1110 1174 11AA;D2D7;1110 1174 11AA; # (틗; 틗; 틗; 틗; 틗; ) HANGUL SYLLABLE TYIGS +D2D8;D2D8;1110 1174 11AB;D2D8;1110 1174 11AB; # (틘; 틘; 틘; 틘; 틘; ) HANGUL SYLLABLE TYIN +D2D9;D2D9;1110 1174 11AC;D2D9;1110 1174 11AC; # (틙; 틙; 틙; 틙; 틙; ) HANGUL SYLLABLE TYINJ +D2DA;D2DA;1110 1174 11AD;D2DA;1110 1174 11AD; # (틚; 틚; 틚; 틚; 틚; ) HANGUL SYLLABLE TYINH +D2DB;D2DB;1110 1174 11AE;D2DB;1110 1174 11AE; # (틛; 틛; 틛; 틛; 틛; ) HANGUL SYLLABLE TYID +D2DC;D2DC;1110 1174 11AF;D2DC;1110 1174 11AF; # (틜; 틜; 틜; 틜; 틜; ) HANGUL SYLLABLE TYIL +D2DD;D2DD;1110 1174 11B0;D2DD;1110 1174 11B0; # (틝; 틝; 틝; 틝; 틝; ) HANGUL SYLLABLE TYILG +D2DE;D2DE;1110 1174 11B1;D2DE;1110 1174 11B1; # (틞; 틞; 틞; 틞; 틞; ) HANGUL SYLLABLE TYILM +D2DF;D2DF;1110 1174 11B2;D2DF;1110 1174 11B2; # (틟; 틟; 틟; 틟; 틟; ) HANGUL SYLLABLE TYILB +D2E0;D2E0;1110 1174 11B3;D2E0;1110 1174 11B3; # (틠; 틠; 틠; 틠; 틠; ) HANGUL SYLLABLE TYILS +D2E1;D2E1;1110 1174 11B4;D2E1;1110 1174 11B4; # (틡; 틡; 틡; 틡; 틡; ) HANGUL SYLLABLE TYILT +D2E2;D2E2;1110 1174 11B5;D2E2;1110 1174 11B5; # (틢; 틢; 틢; 틢; 틢; ) HANGUL SYLLABLE TYILP +D2E3;D2E3;1110 1174 11B6;D2E3;1110 1174 11B6; # (틣; 틣; 틣; 틣; 틣; ) HANGUL SYLLABLE TYILH +D2E4;D2E4;1110 1174 11B7;D2E4;1110 1174 11B7; # (틤; 틤; 틤; 틤; 틤; ) HANGUL SYLLABLE TYIM +D2E5;D2E5;1110 1174 11B8;D2E5;1110 1174 11B8; # (틥; 틥; 틥; 틥; 틥; ) HANGUL SYLLABLE TYIB +D2E6;D2E6;1110 1174 11B9;D2E6;1110 1174 11B9; # (틦; 틦; 틦; 틦; 틦; ) HANGUL SYLLABLE TYIBS +D2E7;D2E7;1110 1174 11BA;D2E7;1110 1174 11BA; # (틧; 틧; 틧; 틧; 틧; ) HANGUL SYLLABLE TYIS +D2E8;D2E8;1110 1174 11BB;D2E8;1110 1174 11BB; # (틨; 틨; 틨; 틨; 틨; ) HANGUL SYLLABLE TYISS +D2E9;D2E9;1110 1174 11BC;D2E9;1110 1174 11BC; # (틩; 틩; 틩; 틩; 틩; ) HANGUL SYLLABLE TYING +D2EA;D2EA;1110 1174 11BD;D2EA;1110 1174 11BD; # (틪; 틪; 틪; 틪; 틪; ) HANGUL SYLLABLE TYIJ +D2EB;D2EB;1110 1174 11BE;D2EB;1110 1174 11BE; # (틫; 틫; 틫; 틫; 틫; ) HANGUL SYLLABLE TYIC +D2EC;D2EC;1110 1174 11BF;D2EC;1110 1174 11BF; # (틬; 틬; 틬; 틬; 틬; ) HANGUL SYLLABLE TYIK +D2ED;D2ED;1110 1174 11C0;D2ED;1110 1174 11C0; # (틭; 틭; 틭; 틭; 틭; ) HANGUL SYLLABLE TYIT +D2EE;D2EE;1110 1174 11C1;D2EE;1110 1174 11C1; # (틮; 틮; 틮; 틮; 틮; ) HANGUL SYLLABLE TYIP +D2EF;D2EF;1110 1174 11C2;D2EF;1110 1174 11C2; # (틯; 틯; 틯; 틯; 틯; ) HANGUL SYLLABLE TYIH +D2F0;D2F0;1110 1175;D2F0;1110 1175; # (티; 티; 티; 티; 티; ) HANGUL SYLLABLE TI +D2F1;D2F1;1110 1175 11A8;D2F1;1110 1175 11A8; # (틱; 틱; 틱; 틱; 틱; ) HANGUL SYLLABLE TIG +D2F2;D2F2;1110 1175 11A9;D2F2;1110 1175 11A9; # (틲; 틲; 틲; 틲; 틲; ) HANGUL SYLLABLE TIGG +D2F3;D2F3;1110 1175 11AA;D2F3;1110 1175 11AA; # (틳; 틳; 틳; 틳; 틳; ) HANGUL SYLLABLE TIGS +D2F4;D2F4;1110 1175 11AB;D2F4;1110 1175 11AB; # (틴; 틴; 틴; 틴; 틴; ) HANGUL SYLLABLE TIN +D2F5;D2F5;1110 1175 11AC;D2F5;1110 1175 11AC; # (틵; 틵; 틵; 틵; 틵; ) HANGUL SYLLABLE TINJ +D2F6;D2F6;1110 1175 11AD;D2F6;1110 1175 11AD; # (틶; 틶; 틶; 틶; 틶; ) HANGUL SYLLABLE TINH +D2F7;D2F7;1110 1175 11AE;D2F7;1110 1175 11AE; # (틷; 틷; 틷; 틷; 틷; ) HANGUL SYLLABLE TID +D2F8;D2F8;1110 1175 11AF;D2F8;1110 1175 11AF; # (틸; 틸; 틸; 틸; 틸; ) HANGUL SYLLABLE TIL +D2F9;D2F9;1110 1175 11B0;D2F9;1110 1175 11B0; # (틹; 틹; 틹; 틹; 틹; ) HANGUL SYLLABLE TILG +D2FA;D2FA;1110 1175 11B1;D2FA;1110 1175 11B1; # (틺; 틺; 틺; 틺; 틺; ) HANGUL SYLLABLE TILM +D2FB;D2FB;1110 1175 11B2;D2FB;1110 1175 11B2; # (틻; 틻; 틻; 틻; 틻; ) HANGUL SYLLABLE TILB +D2FC;D2FC;1110 1175 11B3;D2FC;1110 1175 11B3; # (틼; 틼; 틼; 틼; 틼; ) HANGUL SYLLABLE TILS +D2FD;D2FD;1110 1175 11B4;D2FD;1110 1175 11B4; # (틽; 틽; 틽; 틽; 틽; ) HANGUL SYLLABLE TILT +D2FE;D2FE;1110 1175 11B5;D2FE;1110 1175 11B5; # (틾; 틾; 틾; 틾; 틾; ) HANGUL SYLLABLE TILP +D2FF;D2FF;1110 1175 11B6;D2FF;1110 1175 11B6; # (틿; 틿; 틿; 틿; 틿; ) HANGUL SYLLABLE TILH +D300;D300;1110 1175 11B7;D300;1110 1175 11B7; # (팀; 팀; 팀; 팀; 팀; ) HANGUL SYLLABLE TIM +D301;D301;1110 1175 11B8;D301;1110 1175 11B8; # (팁; 팁; 팁; 팁; 팁; ) HANGUL SYLLABLE TIB +D302;D302;1110 1175 11B9;D302;1110 1175 11B9; # (팂; 팂; 팂; 팂; 팂; ) HANGUL SYLLABLE TIBS +D303;D303;1110 1175 11BA;D303;1110 1175 11BA; # (팃; 팃; 팃; 팃; 팃; ) HANGUL SYLLABLE TIS +D304;D304;1110 1175 11BB;D304;1110 1175 11BB; # (팄; 팄; 팄; 팄; 팄; ) HANGUL SYLLABLE TISS +D305;D305;1110 1175 11BC;D305;1110 1175 11BC; # (팅; 팅; 팅; 팅; 팅; ) HANGUL SYLLABLE TING +D306;D306;1110 1175 11BD;D306;1110 1175 11BD; # (팆; 팆; 팆; 팆; 팆; ) HANGUL SYLLABLE TIJ +D307;D307;1110 1175 11BE;D307;1110 1175 11BE; # (팇; 팇; 팇; 팇; 팇; ) HANGUL SYLLABLE TIC +D308;D308;1110 1175 11BF;D308;1110 1175 11BF; # (팈; 팈; 팈; 팈; 팈; ) HANGUL SYLLABLE TIK +D309;D309;1110 1175 11C0;D309;1110 1175 11C0; # (팉; 팉; 팉; 팉; 팉; ) HANGUL SYLLABLE TIT +D30A;D30A;1110 1175 11C1;D30A;1110 1175 11C1; # (팊; 팊; 팊; 팊; 팊; ) HANGUL SYLLABLE TIP +D30B;D30B;1110 1175 11C2;D30B;1110 1175 11C2; # (팋; 팋; 팋; 팋; 팋; ) HANGUL SYLLABLE TIH +D30C;D30C;1111 1161;D30C;1111 1161; # (파; 파; 파; 파; 파; ) HANGUL SYLLABLE PA +D30D;D30D;1111 1161 11A8;D30D;1111 1161 11A8; # (팍; 팍; 팍; 팍; 팍; ) HANGUL SYLLABLE PAG +D30E;D30E;1111 1161 11A9;D30E;1111 1161 11A9; # (팎; 팎; 팎; 팎; 팎; ) HANGUL SYLLABLE PAGG +D30F;D30F;1111 1161 11AA;D30F;1111 1161 11AA; # (팏; 팏; 팏; 팏; 팏; ) HANGUL SYLLABLE PAGS +D310;D310;1111 1161 11AB;D310;1111 1161 11AB; # (판; 판; 판; 판; 판; ) HANGUL SYLLABLE PAN +D311;D311;1111 1161 11AC;D311;1111 1161 11AC; # (팑; 팑; 팑; 팑; 팑; ) HANGUL SYLLABLE PANJ +D312;D312;1111 1161 11AD;D312;1111 1161 11AD; # (팒; 팒; 팒; 팒; 팒; ) HANGUL SYLLABLE PANH +D313;D313;1111 1161 11AE;D313;1111 1161 11AE; # (팓; 팓; 팓; 팓; 팓; ) HANGUL SYLLABLE PAD +D314;D314;1111 1161 11AF;D314;1111 1161 11AF; # (팔; 팔; 팔; 팔; 팔; ) HANGUL SYLLABLE PAL +D315;D315;1111 1161 11B0;D315;1111 1161 11B0; # (팕; 팕; 팕; 팕; 팕; ) HANGUL SYLLABLE PALG +D316;D316;1111 1161 11B1;D316;1111 1161 11B1; # (팖; 팖; 팖; 팖; 팖; ) HANGUL SYLLABLE PALM +D317;D317;1111 1161 11B2;D317;1111 1161 11B2; # (팗; 팗; 팗; 팗; 팗; ) HANGUL SYLLABLE PALB +D318;D318;1111 1161 11B3;D318;1111 1161 11B3; # (팘; 팘; 팘; 팘; 팘; ) HANGUL SYLLABLE PALS +D319;D319;1111 1161 11B4;D319;1111 1161 11B4; # (팙; 팙; 팙; 팙; 팙; ) HANGUL SYLLABLE PALT +D31A;D31A;1111 1161 11B5;D31A;1111 1161 11B5; # (팚; 팚; 팚; 팚; 팚; ) HANGUL SYLLABLE PALP +D31B;D31B;1111 1161 11B6;D31B;1111 1161 11B6; # (팛; 팛; 팛; 팛; 팛; ) HANGUL SYLLABLE PALH +D31C;D31C;1111 1161 11B7;D31C;1111 1161 11B7; # (팜; 팜; 팜; 팜; 팜; ) HANGUL SYLLABLE PAM +D31D;D31D;1111 1161 11B8;D31D;1111 1161 11B8; # (팝; 팝; 팝; 팝; 팝; ) HANGUL SYLLABLE PAB +D31E;D31E;1111 1161 11B9;D31E;1111 1161 11B9; # (팞; 팞; 팞; 팞; 팞; ) HANGUL SYLLABLE PABS +D31F;D31F;1111 1161 11BA;D31F;1111 1161 11BA; # (팟; 팟; 팟; 팟; 팟; ) HANGUL SYLLABLE PAS +D320;D320;1111 1161 11BB;D320;1111 1161 11BB; # (팠; 팠; 팠; 팠; 팠; ) HANGUL SYLLABLE PASS +D321;D321;1111 1161 11BC;D321;1111 1161 11BC; # (팡; 팡; 팡; 팡; 팡; ) HANGUL SYLLABLE PANG +D322;D322;1111 1161 11BD;D322;1111 1161 11BD; # (팢; 팢; 팢; 팢; 팢; ) HANGUL SYLLABLE PAJ +D323;D323;1111 1161 11BE;D323;1111 1161 11BE; # (팣; 팣; 팣; 팣; 팣; ) HANGUL SYLLABLE PAC +D324;D324;1111 1161 11BF;D324;1111 1161 11BF; # (팤; 팤; 팤; 팤; 팤; ) HANGUL SYLLABLE PAK +D325;D325;1111 1161 11C0;D325;1111 1161 11C0; # (팥; 팥; 팥; 팥; 팥; ) HANGUL SYLLABLE PAT +D326;D326;1111 1161 11C1;D326;1111 1161 11C1; # (팦; 팦; 팦; 팦; 팦; ) HANGUL SYLLABLE PAP +D327;D327;1111 1161 11C2;D327;1111 1161 11C2; # (팧; 팧; 팧; 팧; 팧; ) HANGUL SYLLABLE PAH +D328;D328;1111 1162;D328;1111 1162; # (패; 패; 패; 패; 패; ) HANGUL SYLLABLE PAE +D329;D329;1111 1162 11A8;D329;1111 1162 11A8; # (팩; 팩; 팩; 팩; 팩; ) HANGUL SYLLABLE PAEG +D32A;D32A;1111 1162 11A9;D32A;1111 1162 11A9; # (팪; 팪; 팪; 팪; 팪; ) HANGUL SYLLABLE PAEGG +D32B;D32B;1111 1162 11AA;D32B;1111 1162 11AA; # (팫; 팫; 팫; 팫; 팫; ) HANGUL SYLLABLE PAEGS +D32C;D32C;1111 1162 11AB;D32C;1111 1162 11AB; # (팬; 팬; 팬; 팬; 팬; ) HANGUL SYLLABLE PAEN +D32D;D32D;1111 1162 11AC;D32D;1111 1162 11AC; # (팭; 팭; 팭; 팭; 팭; ) HANGUL SYLLABLE PAENJ +D32E;D32E;1111 1162 11AD;D32E;1111 1162 11AD; # (팮; 팮; 팮; 팮; 팮; ) HANGUL SYLLABLE PAENH +D32F;D32F;1111 1162 11AE;D32F;1111 1162 11AE; # (팯; 팯; 팯; 팯; 팯; ) HANGUL SYLLABLE PAED +D330;D330;1111 1162 11AF;D330;1111 1162 11AF; # (팰; 팰; 팰; 팰; 팰; ) HANGUL SYLLABLE PAEL +D331;D331;1111 1162 11B0;D331;1111 1162 11B0; # (팱; 팱; 팱; 팱; 팱; ) HANGUL SYLLABLE PAELG +D332;D332;1111 1162 11B1;D332;1111 1162 11B1; # (팲; 팲; 팲; 팲; 팲; ) HANGUL SYLLABLE PAELM +D333;D333;1111 1162 11B2;D333;1111 1162 11B2; # (팳; 팳; 팳; 팳; 팳; ) HANGUL SYLLABLE PAELB +D334;D334;1111 1162 11B3;D334;1111 1162 11B3; # (팴; 팴; 팴; 팴; 팴; ) HANGUL SYLLABLE PAELS +D335;D335;1111 1162 11B4;D335;1111 1162 11B4; # (팵; 팵; 팵; 팵; 팵; ) HANGUL SYLLABLE PAELT +D336;D336;1111 1162 11B5;D336;1111 1162 11B5; # (팶; 팶; 팶; 팶; 팶; ) HANGUL SYLLABLE PAELP +D337;D337;1111 1162 11B6;D337;1111 1162 11B6; # (팷; 팷; 팷; 팷; 팷; ) HANGUL SYLLABLE PAELH +D338;D338;1111 1162 11B7;D338;1111 1162 11B7; # (팸; 팸; 팸; 팸; 팸; ) HANGUL SYLLABLE PAEM +D339;D339;1111 1162 11B8;D339;1111 1162 11B8; # (팹; 팹; 팹; 팹; 팹; ) HANGUL SYLLABLE PAEB +D33A;D33A;1111 1162 11B9;D33A;1111 1162 11B9; # (팺; 팺; 팺; 팺; 팺; ) HANGUL SYLLABLE PAEBS +D33B;D33B;1111 1162 11BA;D33B;1111 1162 11BA; # (팻; 팻; 팻; 팻; 팻; ) HANGUL SYLLABLE PAES +D33C;D33C;1111 1162 11BB;D33C;1111 1162 11BB; # (팼; 팼; 팼; 팼; 팼; ) HANGUL SYLLABLE PAESS +D33D;D33D;1111 1162 11BC;D33D;1111 1162 11BC; # (팽; 팽; 팽; 팽; 팽; ) HANGUL SYLLABLE PAENG +D33E;D33E;1111 1162 11BD;D33E;1111 1162 11BD; # (팾; 팾; 팾; 팾; 팾; ) HANGUL SYLLABLE PAEJ +D33F;D33F;1111 1162 11BE;D33F;1111 1162 11BE; # (팿; 팿; 팿; 팿; 팿; ) HANGUL SYLLABLE PAEC +D340;D340;1111 1162 11BF;D340;1111 1162 11BF; # (퍀; 퍀; 퍀; 퍀; 퍀; ) HANGUL SYLLABLE PAEK +D341;D341;1111 1162 11C0;D341;1111 1162 11C0; # (퍁; 퍁; 퍁; 퍁; 퍁; ) HANGUL SYLLABLE PAET +D342;D342;1111 1162 11C1;D342;1111 1162 11C1; # (퍂; 퍂; 퍂; 퍂; 퍂; ) HANGUL SYLLABLE PAEP +D343;D343;1111 1162 11C2;D343;1111 1162 11C2; # (퍃; 퍃; 퍃; 퍃; 퍃; ) HANGUL SYLLABLE PAEH +D344;D344;1111 1163;D344;1111 1163; # (퍄; 퍄; 퍄; 퍄; 퍄; ) HANGUL SYLLABLE PYA +D345;D345;1111 1163 11A8;D345;1111 1163 11A8; # (퍅; 퍅; 퍅; 퍅; 퍅; ) HANGUL SYLLABLE PYAG +D346;D346;1111 1163 11A9;D346;1111 1163 11A9; # (퍆; 퍆; 퍆; 퍆; 퍆; ) HANGUL SYLLABLE PYAGG +D347;D347;1111 1163 11AA;D347;1111 1163 11AA; # (퍇; 퍇; 퍇; 퍇; 퍇; ) HANGUL SYLLABLE PYAGS +D348;D348;1111 1163 11AB;D348;1111 1163 11AB; # (퍈; 퍈; 퍈; 퍈; 퍈; ) HANGUL SYLLABLE PYAN +D349;D349;1111 1163 11AC;D349;1111 1163 11AC; # (퍉; 퍉; 퍉; 퍉; 퍉; ) HANGUL SYLLABLE PYANJ +D34A;D34A;1111 1163 11AD;D34A;1111 1163 11AD; # (퍊; 퍊; 퍊; 퍊; 퍊; ) HANGUL SYLLABLE PYANH +D34B;D34B;1111 1163 11AE;D34B;1111 1163 11AE; # (퍋; 퍋; 퍋; 퍋; 퍋; ) HANGUL SYLLABLE PYAD +D34C;D34C;1111 1163 11AF;D34C;1111 1163 11AF; # (퍌; 퍌; 퍌; 퍌; 퍌; ) HANGUL SYLLABLE PYAL +D34D;D34D;1111 1163 11B0;D34D;1111 1163 11B0; # (퍍; 퍍; 퍍; 퍍; 퍍; ) HANGUL SYLLABLE PYALG +D34E;D34E;1111 1163 11B1;D34E;1111 1163 11B1; # (퍎; 퍎; 퍎; 퍎; 퍎; ) HANGUL SYLLABLE PYALM +D34F;D34F;1111 1163 11B2;D34F;1111 1163 11B2; # (퍏; 퍏; 퍏; 퍏; 퍏; ) HANGUL SYLLABLE PYALB +D350;D350;1111 1163 11B3;D350;1111 1163 11B3; # (퍐; 퍐; 퍐; 퍐; 퍐; ) HANGUL SYLLABLE PYALS +D351;D351;1111 1163 11B4;D351;1111 1163 11B4; # (퍑; 퍑; 퍑; 퍑; 퍑; ) HANGUL SYLLABLE PYALT +D352;D352;1111 1163 11B5;D352;1111 1163 11B5; # (퍒; 퍒; 퍒; 퍒; 퍒; ) HANGUL SYLLABLE PYALP +D353;D353;1111 1163 11B6;D353;1111 1163 11B6; # (퍓; 퍓; 퍓; 퍓; 퍓; ) HANGUL SYLLABLE PYALH +D354;D354;1111 1163 11B7;D354;1111 1163 11B7; # (퍔; 퍔; 퍔; 퍔; 퍔; ) HANGUL SYLLABLE PYAM +D355;D355;1111 1163 11B8;D355;1111 1163 11B8; # (퍕; 퍕; 퍕; 퍕; 퍕; ) HANGUL SYLLABLE PYAB +D356;D356;1111 1163 11B9;D356;1111 1163 11B9; # (퍖; 퍖; 퍖; 퍖; 퍖; ) HANGUL SYLLABLE PYABS +D357;D357;1111 1163 11BA;D357;1111 1163 11BA; # (퍗; 퍗; 퍗; 퍗; 퍗; ) HANGUL SYLLABLE PYAS +D358;D358;1111 1163 11BB;D358;1111 1163 11BB; # (퍘; 퍘; 퍘; 퍘; 퍘; ) HANGUL SYLLABLE PYASS +D359;D359;1111 1163 11BC;D359;1111 1163 11BC; # (퍙; 퍙; 퍙; 퍙; 퍙; ) HANGUL SYLLABLE PYANG +D35A;D35A;1111 1163 11BD;D35A;1111 1163 11BD; # (퍚; 퍚; 퍚; 퍚; 퍚; ) HANGUL SYLLABLE PYAJ +D35B;D35B;1111 1163 11BE;D35B;1111 1163 11BE; # (퍛; 퍛; 퍛; 퍛; 퍛; ) HANGUL SYLLABLE PYAC +D35C;D35C;1111 1163 11BF;D35C;1111 1163 11BF; # (퍜; 퍜; 퍜; 퍜; 퍜; ) HANGUL SYLLABLE PYAK +D35D;D35D;1111 1163 11C0;D35D;1111 1163 11C0; # (퍝; 퍝; 퍝; 퍝; 퍝; ) HANGUL SYLLABLE PYAT +D35E;D35E;1111 1163 11C1;D35E;1111 1163 11C1; # (퍞; 퍞; 퍞; 퍞; 퍞; ) HANGUL SYLLABLE PYAP +D35F;D35F;1111 1163 11C2;D35F;1111 1163 11C2; # (퍟; 퍟; 퍟; 퍟; 퍟; ) HANGUL SYLLABLE PYAH +D360;D360;1111 1164;D360;1111 1164; # (퍠; 퍠; 퍠; 퍠; 퍠; ) HANGUL SYLLABLE PYAE +D361;D361;1111 1164 11A8;D361;1111 1164 11A8; # (퍡; 퍡; 퍡; 퍡; 퍡; ) HANGUL SYLLABLE PYAEG +D362;D362;1111 1164 11A9;D362;1111 1164 11A9; # (퍢; 퍢; 퍢; 퍢; 퍢; ) HANGUL SYLLABLE PYAEGG +D363;D363;1111 1164 11AA;D363;1111 1164 11AA; # (퍣; 퍣; 퍣; 퍣; 퍣; ) HANGUL SYLLABLE PYAEGS +D364;D364;1111 1164 11AB;D364;1111 1164 11AB; # (퍤; 퍤; 퍤; 퍤; 퍤; ) HANGUL SYLLABLE PYAEN +D365;D365;1111 1164 11AC;D365;1111 1164 11AC; # (퍥; 퍥; 퍥; 퍥; 퍥; ) HANGUL SYLLABLE PYAENJ +D366;D366;1111 1164 11AD;D366;1111 1164 11AD; # (퍦; 퍦; 퍦; 퍦; 퍦; ) HANGUL SYLLABLE PYAENH +D367;D367;1111 1164 11AE;D367;1111 1164 11AE; # (퍧; 퍧; 퍧; 퍧; 퍧; ) HANGUL SYLLABLE PYAED +D368;D368;1111 1164 11AF;D368;1111 1164 11AF; # (퍨; 퍨; 퍨; 퍨; 퍨; ) HANGUL SYLLABLE PYAEL +D369;D369;1111 1164 11B0;D369;1111 1164 11B0; # (퍩; 퍩; 퍩; 퍩; 퍩; ) HANGUL SYLLABLE PYAELG +D36A;D36A;1111 1164 11B1;D36A;1111 1164 11B1; # (퍪; 퍪; 퍪; 퍪; 퍪; ) HANGUL SYLLABLE PYAELM +D36B;D36B;1111 1164 11B2;D36B;1111 1164 11B2; # (퍫; 퍫; 퍫; 퍫; 퍫; ) HANGUL SYLLABLE PYAELB +D36C;D36C;1111 1164 11B3;D36C;1111 1164 11B3; # (퍬; 퍬; 퍬; 퍬; 퍬; ) HANGUL SYLLABLE PYAELS +D36D;D36D;1111 1164 11B4;D36D;1111 1164 11B4; # (퍭; 퍭; 퍭; 퍭; 퍭; ) HANGUL SYLLABLE PYAELT +D36E;D36E;1111 1164 11B5;D36E;1111 1164 11B5; # (퍮; 퍮; 퍮; 퍮; 퍮; ) HANGUL SYLLABLE PYAELP +D36F;D36F;1111 1164 11B6;D36F;1111 1164 11B6; # (퍯; 퍯; 퍯; 퍯; 퍯; ) HANGUL SYLLABLE PYAELH +D370;D370;1111 1164 11B7;D370;1111 1164 11B7; # (퍰; 퍰; 퍰; 퍰; 퍰; ) HANGUL SYLLABLE PYAEM +D371;D371;1111 1164 11B8;D371;1111 1164 11B8; # (퍱; 퍱; 퍱; 퍱; 퍱; ) HANGUL SYLLABLE PYAEB +D372;D372;1111 1164 11B9;D372;1111 1164 11B9; # (퍲; 퍲; 퍲; 퍲; 퍲; ) HANGUL SYLLABLE PYAEBS +D373;D373;1111 1164 11BA;D373;1111 1164 11BA; # (퍳; 퍳; 퍳; 퍳; 퍳; ) HANGUL SYLLABLE PYAES +D374;D374;1111 1164 11BB;D374;1111 1164 11BB; # (퍴; 퍴; 퍴; 퍴; 퍴; ) HANGUL SYLLABLE PYAESS +D375;D375;1111 1164 11BC;D375;1111 1164 11BC; # (퍵; 퍵; 퍵; 퍵; 퍵; ) HANGUL SYLLABLE PYAENG +D376;D376;1111 1164 11BD;D376;1111 1164 11BD; # (퍶; 퍶; 퍶; 퍶; 퍶; ) HANGUL SYLLABLE PYAEJ +D377;D377;1111 1164 11BE;D377;1111 1164 11BE; # (퍷; 퍷; 퍷; 퍷; 퍷; ) HANGUL SYLLABLE PYAEC +D378;D378;1111 1164 11BF;D378;1111 1164 11BF; # (퍸; 퍸; 퍸; 퍸; 퍸; ) HANGUL SYLLABLE PYAEK +D379;D379;1111 1164 11C0;D379;1111 1164 11C0; # (퍹; 퍹; 퍹; 퍹; 퍹; ) HANGUL SYLLABLE PYAET +D37A;D37A;1111 1164 11C1;D37A;1111 1164 11C1; # (퍺; 퍺; 퍺; 퍺; 퍺; ) HANGUL SYLLABLE PYAEP +D37B;D37B;1111 1164 11C2;D37B;1111 1164 11C2; # (퍻; 퍻; 퍻; 퍻; 퍻; ) HANGUL SYLLABLE PYAEH +D37C;D37C;1111 1165;D37C;1111 1165; # (퍼; 퍼; 퍼; 퍼; 퍼; ) HANGUL SYLLABLE PEO +D37D;D37D;1111 1165 11A8;D37D;1111 1165 11A8; # (퍽; 퍽; 퍽; 퍽; 퍽; ) HANGUL SYLLABLE PEOG +D37E;D37E;1111 1165 11A9;D37E;1111 1165 11A9; # (퍾; 퍾; 퍾; 퍾; 퍾; ) HANGUL SYLLABLE PEOGG +D37F;D37F;1111 1165 11AA;D37F;1111 1165 11AA; # (퍿; 퍿; 퍿; 퍿; 퍿; ) HANGUL SYLLABLE PEOGS +D380;D380;1111 1165 11AB;D380;1111 1165 11AB; # (펀; 펀; 펀; 펀; 펀; ) HANGUL SYLLABLE PEON +D381;D381;1111 1165 11AC;D381;1111 1165 11AC; # (펁; 펁; 펁; 펁; 펁; ) HANGUL SYLLABLE PEONJ +D382;D382;1111 1165 11AD;D382;1111 1165 11AD; # (펂; 펂; 펂; 펂; 펂; ) HANGUL SYLLABLE PEONH +D383;D383;1111 1165 11AE;D383;1111 1165 11AE; # (펃; 펃; 펃; 펃; 펃; ) HANGUL SYLLABLE PEOD +D384;D384;1111 1165 11AF;D384;1111 1165 11AF; # (펄; 펄; 펄; 펄; 펄; ) HANGUL SYLLABLE PEOL +D385;D385;1111 1165 11B0;D385;1111 1165 11B0; # (펅; 펅; 펅; 펅; 펅; ) HANGUL SYLLABLE PEOLG +D386;D386;1111 1165 11B1;D386;1111 1165 11B1; # (펆; 펆; 펆; 펆; 펆; ) HANGUL SYLLABLE PEOLM +D387;D387;1111 1165 11B2;D387;1111 1165 11B2; # (펇; 펇; 펇; 펇; 펇; ) HANGUL SYLLABLE PEOLB +D388;D388;1111 1165 11B3;D388;1111 1165 11B3; # (펈; 펈; 펈; 펈; 펈; ) HANGUL SYLLABLE PEOLS +D389;D389;1111 1165 11B4;D389;1111 1165 11B4; # (펉; 펉; 펉; 펉; 펉; ) HANGUL SYLLABLE PEOLT +D38A;D38A;1111 1165 11B5;D38A;1111 1165 11B5; # (펊; 펊; 펊; 펊; 펊; ) HANGUL SYLLABLE PEOLP +D38B;D38B;1111 1165 11B6;D38B;1111 1165 11B6; # (펋; 펋; 펋; 펋; 펋; ) HANGUL SYLLABLE PEOLH +D38C;D38C;1111 1165 11B7;D38C;1111 1165 11B7; # (펌; 펌; 펌; 펌; 펌; ) HANGUL SYLLABLE PEOM +D38D;D38D;1111 1165 11B8;D38D;1111 1165 11B8; # (펍; 펍; 펍; 펍; 펍; ) HANGUL SYLLABLE PEOB +D38E;D38E;1111 1165 11B9;D38E;1111 1165 11B9; # (펎; 펎; 펎; 펎; 펎; ) HANGUL SYLLABLE PEOBS +D38F;D38F;1111 1165 11BA;D38F;1111 1165 11BA; # (펏; 펏; 펏; 펏; 펏; ) HANGUL SYLLABLE PEOS +D390;D390;1111 1165 11BB;D390;1111 1165 11BB; # (펐; 펐; 펐; 펐; 펐; ) HANGUL SYLLABLE PEOSS +D391;D391;1111 1165 11BC;D391;1111 1165 11BC; # (펑; 펑; 펑; 펑; 펑; ) HANGUL SYLLABLE PEONG +D392;D392;1111 1165 11BD;D392;1111 1165 11BD; # (펒; 펒; 펒; 펒; 펒; ) HANGUL SYLLABLE PEOJ +D393;D393;1111 1165 11BE;D393;1111 1165 11BE; # (펓; 펓; 펓; 펓; 펓; ) HANGUL SYLLABLE PEOC +D394;D394;1111 1165 11BF;D394;1111 1165 11BF; # (펔; 펔; 펔; 펔; 펔; ) HANGUL SYLLABLE PEOK +D395;D395;1111 1165 11C0;D395;1111 1165 11C0; # (펕; 펕; 펕; 펕; 펕; ) HANGUL SYLLABLE PEOT +D396;D396;1111 1165 11C1;D396;1111 1165 11C1; # (펖; 펖; 펖; 펖; 펖; ) HANGUL SYLLABLE PEOP +D397;D397;1111 1165 11C2;D397;1111 1165 11C2; # (펗; 펗; 펗; 펗; 펗; ) HANGUL SYLLABLE PEOH +D398;D398;1111 1166;D398;1111 1166; # (페; 페; 페; 페; 페; ) HANGUL SYLLABLE PE +D399;D399;1111 1166 11A8;D399;1111 1166 11A8; # (펙; 펙; 펙; 펙; 펙; ) HANGUL SYLLABLE PEG +D39A;D39A;1111 1166 11A9;D39A;1111 1166 11A9; # (펚; 펚; 펚; 펚; 펚; ) HANGUL SYLLABLE PEGG +D39B;D39B;1111 1166 11AA;D39B;1111 1166 11AA; # (펛; 펛; 펛; 펛; 펛; ) HANGUL SYLLABLE PEGS +D39C;D39C;1111 1166 11AB;D39C;1111 1166 11AB; # (펜; 펜; 펜; 펜; 펜; ) HANGUL SYLLABLE PEN +D39D;D39D;1111 1166 11AC;D39D;1111 1166 11AC; # (펝; 펝; 펝; 펝; 펝; ) HANGUL SYLLABLE PENJ +D39E;D39E;1111 1166 11AD;D39E;1111 1166 11AD; # (펞; 펞; 펞; 펞; 펞; ) HANGUL SYLLABLE PENH +D39F;D39F;1111 1166 11AE;D39F;1111 1166 11AE; # (펟; 펟; 펟; 펟; 펟; ) HANGUL SYLLABLE PED +D3A0;D3A0;1111 1166 11AF;D3A0;1111 1166 11AF; # (펠; 펠; 펠; 펠; 펠; ) HANGUL SYLLABLE PEL +D3A1;D3A1;1111 1166 11B0;D3A1;1111 1166 11B0; # (펡; 펡; 펡; 펡; 펡; ) HANGUL SYLLABLE PELG +D3A2;D3A2;1111 1166 11B1;D3A2;1111 1166 11B1; # (펢; 펢; 펢; 펢; 펢; ) HANGUL SYLLABLE PELM +D3A3;D3A3;1111 1166 11B2;D3A3;1111 1166 11B2; # (펣; 펣; 펣; 펣; 펣; ) HANGUL SYLLABLE PELB +D3A4;D3A4;1111 1166 11B3;D3A4;1111 1166 11B3; # (펤; 펤; 펤; 펤; 펤; ) HANGUL SYLLABLE PELS +D3A5;D3A5;1111 1166 11B4;D3A5;1111 1166 11B4; # (펥; 펥; 펥; 펥; 펥; ) HANGUL SYLLABLE PELT +D3A6;D3A6;1111 1166 11B5;D3A6;1111 1166 11B5; # (펦; 펦; 펦; 펦; 펦; ) HANGUL SYLLABLE PELP +D3A7;D3A7;1111 1166 11B6;D3A7;1111 1166 11B6; # (펧; 펧; 펧; 펧; 펧; ) HANGUL SYLLABLE PELH +D3A8;D3A8;1111 1166 11B7;D3A8;1111 1166 11B7; # (펨; 펨; 펨; 펨; 펨; ) HANGUL SYLLABLE PEM +D3A9;D3A9;1111 1166 11B8;D3A9;1111 1166 11B8; # (펩; 펩; 펩; 펩; 펩; ) HANGUL SYLLABLE PEB +D3AA;D3AA;1111 1166 11B9;D3AA;1111 1166 11B9; # (펪; 펪; 펪; 펪; 펪; ) HANGUL SYLLABLE PEBS +D3AB;D3AB;1111 1166 11BA;D3AB;1111 1166 11BA; # (펫; 펫; 펫; 펫; 펫; ) HANGUL SYLLABLE PES +D3AC;D3AC;1111 1166 11BB;D3AC;1111 1166 11BB; # (펬; 펬; 펬; 펬; 펬; ) HANGUL SYLLABLE PESS +D3AD;D3AD;1111 1166 11BC;D3AD;1111 1166 11BC; # (펭; 펭; 펭; 펭; 펭; ) HANGUL SYLLABLE PENG +D3AE;D3AE;1111 1166 11BD;D3AE;1111 1166 11BD; # (펮; 펮; 펮; 펮; 펮; ) HANGUL SYLLABLE PEJ +D3AF;D3AF;1111 1166 11BE;D3AF;1111 1166 11BE; # (펯; 펯; 펯; 펯; 펯; ) HANGUL SYLLABLE PEC +D3B0;D3B0;1111 1166 11BF;D3B0;1111 1166 11BF; # (펰; 펰; 펰; 펰; 펰; ) HANGUL SYLLABLE PEK +D3B1;D3B1;1111 1166 11C0;D3B1;1111 1166 11C0; # (펱; 펱; 펱; 펱; 펱; ) HANGUL SYLLABLE PET +D3B2;D3B2;1111 1166 11C1;D3B2;1111 1166 11C1; # (펲; 펲; 펲; 펲; 펲; ) HANGUL SYLLABLE PEP +D3B3;D3B3;1111 1166 11C2;D3B3;1111 1166 11C2; # (펳; 펳; 펳; 펳; 펳; ) HANGUL SYLLABLE PEH +D3B4;D3B4;1111 1167;D3B4;1111 1167; # (펴; 펴; 펴; 펴; 펴; ) HANGUL SYLLABLE PYEO +D3B5;D3B5;1111 1167 11A8;D3B5;1111 1167 11A8; # (펵; 펵; 펵; 펵; 펵; ) HANGUL SYLLABLE PYEOG +D3B6;D3B6;1111 1167 11A9;D3B6;1111 1167 11A9; # (펶; 펶; 펶; 펶; 펶; ) HANGUL SYLLABLE PYEOGG +D3B7;D3B7;1111 1167 11AA;D3B7;1111 1167 11AA; # (펷; 펷; 펷; 펷; 펷; ) HANGUL SYLLABLE PYEOGS +D3B8;D3B8;1111 1167 11AB;D3B8;1111 1167 11AB; # (편; 편; 편; 편; 편; ) HANGUL SYLLABLE PYEON +D3B9;D3B9;1111 1167 11AC;D3B9;1111 1167 11AC; # (펹; 펹; 펹; 펹; 펹; ) HANGUL SYLLABLE PYEONJ +D3BA;D3BA;1111 1167 11AD;D3BA;1111 1167 11AD; # (펺; 펺; 펺; 펺; 펺; ) HANGUL SYLLABLE PYEONH +D3BB;D3BB;1111 1167 11AE;D3BB;1111 1167 11AE; # (펻; 펻; 펻; 펻; 펻; ) HANGUL SYLLABLE PYEOD +D3BC;D3BC;1111 1167 11AF;D3BC;1111 1167 11AF; # (펼; 펼; 펼; 펼; 펼; ) HANGUL SYLLABLE PYEOL +D3BD;D3BD;1111 1167 11B0;D3BD;1111 1167 11B0; # (펽; 펽; 펽; 펽; 펽; ) HANGUL SYLLABLE PYEOLG +D3BE;D3BE;1111 1167 11B1;D3BE;1111 1167 11B1; # (펾; 펾; 펾; 펾; 펾; ) HANGUL SYLLABLE PYEOLM +D3BF;D3BF;1111 1167 11B2;D3BF;1111 1167 11B2; # (펿; 펿; 펿; 펿; 펿; ) HANGUL SYLLABLE PYEOLB +D3C0;D3C0;1111 1167 11B3;D3C0;1111 1167 11B3; # (폀; 폀; 폀; 폀; 폀; ) HANGUL SYLLABLE PYEOLS +D3C1;D3C1;1111 1167 11B4;D3C1;1111 1167 11B4; # (폁; 폁; 폁; 폁; 폁; ) HANGUL SYLLABLE PYEOLT +D3C2;D3C2;1111 1167 11B5;D3C2;1111 1167 11B5; # (폂; 폂; 폂; 폂; 폂; ) HANGUL SYLLABLE PYEOLP +D3C3;D3C3;1111 1167 11B6;D3C3;1111 1167 11B6; # (폃; 폃; 폃; 폃; 폃; ) HANGUL SYLLABLE PYEOLH +D3C4;D3C4;1111 1167 11B7;D3C4;1111 1167 11B7; # (폄; 폄; 폄; 폄; 폄; ) HANGUL SYLLABLE PYEOM +D3C5;D3C5;1111 1167 11B8;D3C5;1111 1167 11B8; # (폅; 폅; 폅; 폅; 폅; ) HANGUL SYLLABLE PYEOB +D3C6;D3C6;1111 1167 11B9;D3C6;1111 1167 11B9; # (폆; 폆; 폆; 폆; 폆; ) HANGUL SYLLABLE PYEOBS +D3C7;D3C7;1111 1167 11BA;D3C7;1111 1167 11BA; # (폇; 폇; 폇; 폇; 폇; ) HANGUL SYLLABLE PYEOS +D3C8;D3C8;1111 1167 11BB;D3C8;1111 1167 11BB; # (폈; 폈; 폈; 폈; 폈; ) HANGUL SYLLABLE PYEOSS +D3C9;D3C9;1111 1167 11BC;D3C9;1111 1167 11BC; # (평; 평; 평; 평; 평; ) HANGUL SYLLABLE PYEONG +D3CA;D3CA;1111 1167 11BD;D3CA;1111 1167 11BD; # (폊; 폊; 폊; 폊; 폊; ) HANGUL SYLLABLE PYEOJ +D3CB;D3CB;1111 1167 11BE;D3CB;1111 1167 11BE; # (폋; 폋; 폋; 폋; 폋; ) HANGUL SYLLABLE PYEOC +D3CC;D3CC;1111 1167 11BF;D3CC;1111 1167 11BF; # (폌; 폌; 폌; 폌; 폌; ) HANGUL SYLLABLE PYEOK +D3CD;D3CD;1111 1167 11C0;D3CD;1111 1167 11C0; # (폍; 폍; 폍; 폍; 폍; ) HANGUL SYLLABLE PYEOT +D3CE;D3CE;1111 1167 11C1;D3CE;1111 1167 11C1; # (폎; 폎; 폎; 폎; 폎; ) HANGUL SYLLABLE PYEOP +D3CF;D3CF;1111 1167 11C2;D3CF;1111 1167 11C2; # (폏; 폏; 폏; 폏; 폏; ) HANGUL SYLLABLE PYEOH +D3D0;D3D0;1111 1168;D3D0;1111 1168; # (폐; 폐; 폐; 폐; 폐; ) HANGUL SYLLABLE PYE +D3D1;D3D1;1111 1168 11A8;D3D1;1111 1168 11A8; # (폑; 폑; 폑; 폑; 폑; ) HANGUL SYLLABLE PYEG +D3D2;D3D2;1111 1168 11A9;D3D2;1111 1168 11A9; # (폒; 폒; 폒; 폒; 폒; ) HANGUL SYLLABLE PYEGG +D3D3;D3D3;1111 1168 11AA;D3D3;1111 1168 11AA; # (폓; 폓; 폓; 폓; 폓; ) HANGUL SYLLABLE PYEGS +D3D4;D3D4;1111 1168 11AB;D3D4;1111 1168 11AB; # (폔; 폔; 폔; 폔; 폔; ) HANGUL SYLLABLE PYEN +D3D5;D3D5;1111 1168 11AC;D3D5;1111 1168 11AC; # (폕; 폕; 폕; 폕; 폕; ) HANGUL SYLLABLE PYENJ +D3D6;D3D6;1111 1168 11AD;D3D6;1111 1168 11AD; # (폖; 폖; 폖; 폖; 폖; ) HANGUL SYLLABLE PYENH +D3D7;D3D7;1111 1168 11AE;D3D7;1111 1168 11AE; # (폗; 폗; 폗; 폗; 폗; ) HANGUL SYLLABLE PYED +D3D8;D3D8;1111 1168 11AF;D3D8;1111 1168 11AF; # (폘; 폘; 폘; 폘; 폘; ) HANGUL SYLLABLE PYEL +D3D9;D3D9;1111 1168 11B0;D3D9;1111 1168 11B0; # (폙; 폙; 폙; 폙; 폙; ) HANGUL SYLLABLE PYELG +D3DA;D3DA;1111 1168 11B1;D3DA;1111 1168 11B1; # (폚; 폚; 폚; 폚; 폚; ) HANGUL SYLLABLE PYELM +D3DB;D3DB;1111 1168 11B2;D3DB;1111 1168 11B2; # (폛; 폛; 폛; 폛; 폛; ) HANGUL SYLLABLE PYELB +D3DC;D3DC;1111 1168 11B3;D3DC;1111 1168 11B3; # (폜; 폜; 폜; 폜; 폜; ) HANGUL SYLLABLE PYELS +D3DD;D3DD;1111 1168 11B4;D3DD;1111 1168 11B4; # (폝; 폝; 폝; 폝; 폝; ) HANGUL SYLLABLE PYELT +D3DE;D3DE;1111 1168 11B5;D3DE;1111 1168 11B5; # (폞; 폞; 폞; 폞; 폞; ) HANGUL SYLLABLE PYELP +D3DF;D3DF;1111 1168 11B6;D3DF;1111 1168 11B6; # (폟; 폟; 폟; 폟; 폟; ) HANGUL SYLLABLE PYELH +D3E0;D3E0;1111 1168 11B7;D3E0;1111 1168 11B7; # (폠; 폠; 폠; 폠; 폠; ) HANGUL SYLLABLE PYEM +D3E1;D3E1;1111 1168 11B8;D3E1;1111 1168 11B8; # (폡; 폡; 폡; 폡; 폡; ) HANGUL SYLLABLE PYEB +D3E2;D3E2;1111 1168 11B9;D3E2;1111 1168 11B9; # (폢; 폢; 폢; 폢; 폢; ) HANGUL SYLLABLE PYEBS +D3E3;D3E3;1111 1168 11BA;D3E3;1111 1168 11BA; # (폣; 폣; 폣; 폣; 폣; ) HANGUL SYLLABLE PYES +D3E4;D3E4;1111 1168 11BB;D3E4;1111 1168 11BB; # (폤; 폤; 폤; 폤; 폤; ) HANGUL SYLLABLE PYESS +D3E5;D3E5;1111 1168 11BC;D3E5;1111 1168 11BC; # (폥; 폥; 폥; 폥; 폥; ) HANGUL SYLLABLE PYENG +D3E6;D3E6;1111 1168 11BD;D3E6;1111 1168 11BD; # (폦; 폦; 폦; 폦; 폦; ) HANGUL SYLLABLE PYEJ +D3E7;D3E7;1111 1168 11BE;D3E7;1111 1168 11BE; # (폧; 폧; 폧; 폧; 폧; ) HANGUL SYLLABLE PYEC +D3E8;D3E8;1111 1168 11BF;D3E8;1111 1168 11BF; # (폨; 폨; 폨; 폨; 폨; ) HANGUL SYLLABLE PYEK +D3E9;D3E9;1111 1168 11C0;D3E9;1111 1168 11C0; # (폩; 폩; 폩; 폩; 폩; ) HANGUL SYLLABLE PYET +D3EA;D3EA;1111 1168 11C1;D3EA;1111 1168 11C1; # (폪; 폪; 폪; 폪; 폪; ) HANGUL SYLLABLE PYEP +D3EB;D3EB;1111 1168 11C2;D3EB;1111 1168 11C2; # (폫; 폫; 폫; 폫; 폫; ) HANGUL SYLLABLE PYEH +D3EC;D3EC;1111 1169;D3EC;1111 1169; # (포; 포; 포; 포; 포; ) HANGUL SYLLABLE PO +D3ED;D3ED;1111 1169 11A8;D3ED;1111 1169 11A8; # (폭; 폭; 폭; 폭; 폭; ) HANGUL SYLLABLE POG +D3EE;D3EE;1111 1169 11A9;D3EE;1111 1169 11A9; # (폮; 폮; 폮; 폮; 폮; ) HANGUL SYLLABLE POGG +D3EF;D3EF;1111 1169 11AA;D3EF;1111 1169 11AA; # (폯; 폯; 폯; 폯; 폯; ) HANGUL SYLLABLE POGS +D3F0;D3F0;1111 1169 11AB;D3F0;1111 1169 11AB; # (폰; 폰; 폰; 폰; 폰; ) HANGUL SYLLABLE PON +D3F1;D3F1;1111 1169 11AC;D3F1;1111 1169 11AC; # (폱; 폱; 폱; 폱; 폱; ) HANGUL SYLLABLE PONJ +D3F2;D3F2;1111 1169 11AD;D3F2;1111 1169 11AD; # (폲; 폲; 폲; 폲; 폲; ) HANGUL SYLLABLE PONH +D3F3;D3F3;1111 1169 11AE;D3F3;1111 1169 11AE; # (폳; 폳; 폳; 폳; 폳; ) HANGUL SYLLABLE POD +D3F4;D3F4;1111 1169 11AF;D3F4;1111 1169 11AF; # (폴; 폴; 폴; 폴; 폴; ) HANGUL SYLLABLE POL +D3F5;D3F5;1111 1169 11B0;D3F5;1111 1169 11B0; # (폵; 폵; 폵; 폵; 폵; ) HANGUL SYLLABLE POLG +D3F6;D3F6;1111 1169 11B1;D3F6;1111 1169 11B1; # (폶; 폶; 폶; 폶; 폶; ) HANGUL SYLLABLE POLM +D3F7;D3F7;1111 1169 11B2;D3F7;1111 1169 11B2; # (폷; 폷; 폷; 폷; 폷; ) HANGUL SYLLABLE POLB +D3F8;D3F8;1111 1169 11B3;D3F8;1111 1169 11B3; # (폸; 폸; 폸; 폸; 폸; ) HANGUL SYLLABLE POLS +D3F9;D3F9;1111 1169 11B4;D3F9;1111 1169 11B4; # (폹; 폹; 폹; 폹; 폹; ) HANGUL SYLLABLE POLT +D3FA;D3FA;1111 1169 11B5;D3FA;1111 1169 11B5; # (폺; 폺; 폺; 폺; 폺; ) HANGUL SYLLABLE POLP +D3FB;D3FB;1111 1169 11B6;D3FB;1111 1169 11B6; # (폻; 폻; 폻; 폻; 폻; ) HANGUL SYLLABLE POLH +D3FC;D3FC;1111 1169 11B7;D3FC;1111 1169 11B7; # (폼; 폼; 폼; 폼; 폼; ) HANGUL SYLLABLE POM +D3FD;D3FD;1111 1169 11B8;D3FD;1111 1169 11B8; # (폽; 폽; 폽; 폽; 폽; ) HANGUL SYLLABLE POB +D3FE;D3FE;1111 1169 11B9;D3FE;1111 1169 11B9; # (폾; 폾; 폾; 폾; 폾; ) HANGUL SYLLABLE POBS +D3FF;D3FF;1111 1169 11BA;D3FF;1111 1169 11BA; # (폿; 폿; 폿; 폿; 폿; ) HANGUL SYLLABLE POS +D400;D400;1111 1169 11BB;D400;1111 1169 11BB; # (퐀; 퐀; 퐀; 퐀; 퐀; ) HANGUL SYLLABLE POSS +D401;D401;1111 1169 11BC;D401;1111 1169 11BC; # (퐁; 퐁; 퐁; 퐁; 퐁; ) HANGUL SYLLABLE PONG +D402;D402;1111 1169 11BD;D402;1111 1169 11BD; # (퐂; 퐂; 퐂; 퐂; 퐂; ) HANGUL SYLLABLE POJ +D403;D403;1111 1169 11BE;D403;1111 1169 11BE; # (퐃; 퐃; 퐃; 퐃; 퐃; ) HANGUL SYLLABLE POC +D404;D404;1111 1169 11BF;D404;1111 1169 11BF; # (퐄; 퐄; 퐄; 퐄; 퐄; ) HANGUL SYLLABLE POK +D405;D405;1111 1169 11C0;D405;1111 1169 11C0; # (퐅; 퐅; 퐅; 퐅; 퐅; ) HANGUL SYLLABLE POT +D406;D406;1111 1169 11C1;D406;1111 1169 11C1; # (퐆; 퐆; 퐆; 퐆; 퐆; ) HANGUL SYLLABLE POP +D407;D407;1111 1169 11C2;D407;1111 1169 11C2; # (퐇; 퐇; 퐇; 퐇; 퐇; ) HANGUL SYLLABLE POH +D408;D408;1111 116A;D408;1111 116A; # (퐈; 퐈; 퐈; 퐈; 퐈; ) HANGUL SYLLABLE PWA +D409;D409;1111 116A 11A8;D409;1111 116A 11A8; # (퐉; 퐉; 퐉; 퐉; 퐉; ) HANGUL SYLLABLE PWAG +D40A;D40A;1111 116A 11A9;D40A;1111 116A 11A9; # (퐊; 퐊; 퐊; 퐊; 퐊; ) HANGUL SYLLABLE PWAGG +D40B;D40B;1111 116A 11AA;D40B;1111 116A 11AA; # (퐋; 퐋; 퐋; 퐋; 퐋; ) HANGUL SYLLABLE PWAGS +D40C;D40C;1111 116A 11AB;D40C;1111 116A 11AB; # (퐌; 퐌; 퐌; 퐌; 퐌; ) HANGUL SYLLABLE PWAN +D40D;D40D;1111 116A 11AC;D40D;1111 116A 11AC; # (퐍; 퐍; 퐍; 퐍; 퐍; ) HANGUL SYLLABLE PWANJ +D40E;D40E;1111 116A 11AD;D40E;1111 116A 11AD; # (퐎; 퐎; 퐎; 퐎; 퐎; ) HANGUL SYLLABLE PWANH +D40F;D40F;1111 116A 11AE;D40F;1111 116A 11AE; # (퐏; 퐏; 퐏; 퐏; 퐏; ) HANGUL SYLLABLE PWAD +D410;D410;1111 116A 11AF;D410;1111 116A 11AF; # (퐐; 퐐; 퐐; 퐐; 퐐; ) HANGUL SYLLABLE PWAL +D411;D411;1111 116A 11B0;D411;1111 116A 11B0; # (퐑; 퐑; 퐑; 퐑; 퐑; ) HANGUL SYLLABLE PWALG +D412;D412;1111 116A 11B1;D412;1111 116A 11B1; # (퐒; 퐒; 퐒; 퐒; 퐒; ) HANGUL SYLLABLE PWALM +D413;D413;1111 116A 11B2;D413;1111 116A 11B2; # (퐓; 퐓; 퐓; 퐓; 퐓; ) HANGUL SYLLABLE PWALB +D414;D414;1111 116A 11B3;D414;1111 116A 11B3; # (퐔; 퐔; 퐔; 퐔; 퐔; ) HANGUL SYLLABLE PWALS +D415;D415;1111 116A 11B4;D415;1111 116A 11B4; # (퐕; 퐕; 퐕; 퐕; 퐕; ) HANGUL SYLLABLE PWALT +D416;D416;1111 116A 11B5;D416;1111 116A 11B5; # (퐖; 퐖; 퐖; 퐖; 퐖; ) HANGUL SYLLABLE PWALP +D417;D417;1111 116A 11B6;D417;1111 116A 11B6; # (퐗; 퐗; 퐗; 퐗; 퐗; ) HANGUL SYLLABLE PWALH +D418;D418;1111 116A 11B7;D418;1111 116A 11B7; # (퐘; 퐘; 퐘; 퐘; 퐘; ) HANGUL SYLLABLE PWAM +D419;D419;1111 116A 11B8;D419;1111 116A 11B8; # (퐙; 퐙; 퐙; 퐙; 퐙; ) HANGUL SYLLABLE PWAB +D41A;D41A;1111 116A 11B9;D41A;1111 116A 11B9; # (퐚; 퐚; 퐚; 퐚; 퐚; ) HANGUL SYLLABLE PWABS +D41B;D41B;1111 116A 11BA;D41B;1111 116A 11BA; # (퐛; 퐛; 퐛; 퐛; 퐛; ) HANGUL SYLLABLE PWAS +D41C;D41C;1111 116A 11BB;D41C;1111 116A 11BB; # (퐜; 퐜; 퐜; 퐜; 퐜; ) HANGUL SYLLABLE PWASS +D41D;D41D;1111 116A 11BC;D41D;1111 116A 11BC; # (퐝; 퐝; 퐝; 퐝; 퐝; ) HANGUL SYLLABLE PWANG +D41E;D41E;1111 116A 11BD;D41E;1111 116A 11BD; # (퐞; 퐞; 퐞; 퐞; 퐞; ) HANGUL SYLLABLE PWAJ +D41F;D41F;1111 116A 11BE;D41F;1111 116A 11BE; # (퐟; 퐟; 퐟; 퐟; 퐟; ) HANGUL SYLLABLE PWAC +D420;D420;1111 116A 11BF;D420;1111 116A 11BF; # (퐠; 퐠; 퐠; 퐠; 퐠; ) HANGUL SYLLABLE PWAK +D421;D421;1111 116A 11C0;D421;1111 116A 11C0; # (퐡; 퐡; 퐡; 퐡; 퐡; ) HANGUL SYLLABLE PWAT +D422;D422;1111 116A 11C1;D422;1111 116A 11C1; # (퐢; 퐢; 퐢; 퐢; 퐢; ) HANGUL SYLLABLE PWAP +D423;D423;1111 116A 11C2;D423;1111 116A 11C2; # (퐣; 퐣; 퐣; 퐣; 퐣; ) HANGUL SYLLABLE PWAH +D424;D424;1111 116B;D424;1111 116B; # (퐤; 퐤; 퐤; 퐤; 퐤; ) HANGUL SYLLABLE PWAE +D425;D425;1111 116B 11A8;D425;1111 116B 11A8; # (퐥; 퐥; 퐥; 퐥; 퐥; ) HANGUL SYLLABLE PWAEG +D426;D426;1111 116B 11A9;D426;1111 116B 11A9; # (퐦; 퐦; 퐦; 퐦; 퐦; ) HANGUL SYLLABLE PWAEGG +D427;D427;1111 116B 11AA;D427;1111 116B 11AA; # (퐧; 퐧; 퐧; 퐧; 퐧; ) HANGUL SYLLABLE PWAEGS +D428;D428;1111 116B 11AB;D428;1111 116B 11AB; # (퐨; 퐨; 퐨; 퐨; 퐨; ) HANGUL SYLLABLE PWAEN +D429;D429;1111 116B 11AC;D429;1111 116B 11AC; # (퐩; 퐩; 퐩; 퐩; 퐩; ) HANGUL SYLLABLE PWAENJ +D42A;D42A;1111 116B 11AD;D42A;1111 116B 11AD; # (퐪; 퐪; 퐪; 퐪; 퐪; ) HANGUL SYLLABLE PWAENH +D42B;D42B;1111 116B 11AE;D42B;1111 116B 11AE; # (퐫; 퐫; 퐫; 퐫; 퐫; ) HANGUL SYLLABLE PWAED +D42C;D42C;1111 116B 11AF;D42C;1111 116B 11AF; # (퐬; 퐬; 퐬; 퐬; 퐬; ) HANGUL SYLLABLE PWAEL +D42D;D42D;1111 116B 11B0;D42D;1111 116B 11B0; # (퐭; 퐭; 퐭; 퐭; 퐭; ) HANGUL SYLLABLE PWAELG +D42E;D42E;1111 116B 11B1;D42E;1111 116B 11B1; # (퐮; 퐮; 퐮; 퐮; 퐮; ) HANGUL SYLLABLE PWAELM +D42F;D42F;1111 116B 11B2;D42F;1111 116B 11B2; # (퐯; 퐯; 퐯; 퐯; 퐯; ) HANGUL SYLLABLE PWAELB +D430;D430;1111 116B 11B3;D430;1111 116B 11B3; # (퐰; 퐰; 퐰; 퐰; 퐰; ) HANGUL SYLLABLE PWAELS +D431;D431;1111 116B 11B4;D431;1111 116B 11B4; # (퐱; 퐱; 퐱; 퐱; 퐱; ) HANGUL SYLLABLE PWAELT +D432;D432;1111 116B 11B5;D432;1111 116B 11B5; # (퐲; 퐲; 퐲; 퐲; 퐲; ) HANGUL SYLLABLE PWAELP +D433;D433;1111 116B 11B6;D433;1111 116B 11B6; # (퐳; 퐳; 퐳; 퐳; 퐳; ) HANGUL SYLLABLE PWAELH +D434;D434;1111 116B 11B7;D434;1111 116B 11B7; # (퐴; 퐴; 퐴; 퐴; 퐴; ) HANGUL SYLLABLE PWAEM +D435;D435;1111 116B 11B8;D435;1111 116B 11B8; # (퐵; 퐵; 퐵; 퐵; 퐵; ) HANGUL SYLLABLE PWAEB +D436;D436;1111 116B 11B9;D436;1111 116B 11B9; # (퐶; 퐶; 퐶; 퐶; 퐶; ) HANGUL SYLLABLE PWAEBS +D437;D437;1111 116B 11BA;D437;1111 116B 11BA; # (퐷; 퐷; 퐷; 퐷; 퐷; ) HANGUL SYLLABLE PWAES +D438;D438;1111 116B 11BB;D438;1111 116B 11BB; # (퐸; 퐸; 퐸; 퐸; 퐸; ) HANGUL SYLLABLE PWAESS +D439;D439;1111 116B 11BC;D439;1111 116B 11BC; # (퐹; 퐹; 퐹; 퐹; 퐹; ) HANGUL SYLLABLE PWAENG +D43A;D43A;1111 116B 11BD;D43A;1111 116B 11BD; # (퐺; 퐺; 퐺; 퐺; 퐺; ) HANGUL SYLLABLE PWAEJ +D43B;D43B;1111 116B 11BE;D43B;1111 116B 11BE; # (퐻; 퐻; 퐻; 퐻; 퐻; ) HANGUL SYLLABLE PWAEC +D43C;D43C;1111 116B 11BF;D43C;1111 116B 11BF; # (퐼; 퐼; 퐼; 퐼; 퐼; ) HANGUL SYLLABLE PWAEK +D43D;D43D;1111 116B 11C0;D43D;1111 116B 11C0; # (퐽; 퐽; 퐽; 퐽; 퐽; ) HANGUL SYLLABLE PWAET +D43E;D43E;1111 116B 11C1;D43E;1111 116B 11C1; # (퐾; 퐾; 퐾; 퐾; 퐾; ) HANGUL SYLLABLE PWAEP +D43F;D43F;1111 116B 11C2;D43F;1111 116B 11C2; # (퐿; 퐿; 퐿; 퐿; 퐿; ) HANGUL SYLLABLE PWAEH +D440;D440;1111 116C;D440;1111 116C; # (푀; 푀; 푀; 푀; 푀; ) HANGUL SYLLABLE POE +D441;D441;1111 116C 11A8;D441;1111 116C 11A8; # (푁; 푁; 푁; 푁; 푁; ) HANGUL SYLLABLE POEG +D442;D442;1111 116C 11A9;D442;1111 116C 11A9; # (푂; 푂; 푂; 푂; 푂; ) HANGUL SYLLABLE POEGG +D443;D443;1111 116C 11AA;D443;1111 116C 11AA; # (푃; 푃; 푃; 푃; 푃; ) HANGUL SYLLABLE POEGS +D444;D444;1111 116C 11AB;D444;1111 116C 11AB; # (푄; 푄; 푄; 푄; 푄; ) HANGUL SYLLABLE POEN +D445;D445;1111 116C 11AC;D445;1111 116C 11AC; # (푅; 푅; 푅; 푅; 푅; ) HANGUL SYLLABLE POENJ +D446;D446;1111 116C 11AD;D446;1111 116C 11AD; # (푆; 푆; 푆; 푆; 푆; ) HANGUL SYLLABLE POENH +D447;D447;1111 116C 11AE;D447;1111 116C 11AE; # (푇; 푇; 푇; 푇; 푇; ) HANGUL SYLLABLE POED +D448;D448;1111 116C 11AF;D448;1111 116C 11AF; # (푈; 푈; 푈; 푈; 푈; ) HANGUL SYLLABLE POEL +D449;D449;1111 116C 11B0;D449;1111 116C 11B0; # (푉; 푉; 푉; 푉; 푉; ) HANGUL SYLLABLE POELG +D44A;D44A;1111 116C 11B1;D44A;1111 116C 11B1; # (푊; 푊; 푊; 푊; 푊; ) HANGUL SYLLABLE POELM +D44B;D44B;1111 116C 11B2;D44B;1111 116C 11B2; # (푋; 푋; 푋; 푋; 푋; ) HANGUL SYLLABLE POELB +D44C;D44C;1111 116C 11B3;D44C;1111 116C 11B3; # (푌; 푌; 푌; 푌; 푌; ) HANGUL SYLLABLE POELS +D44D;D44D;1111 116C 11B4;D44D;1111 116C 11B4; # (푍; 푍; 푍; 푍; 푍; ) HANGUL SYLLABLE POELT +D44E;D44E;1111 116C 11B5;D44E;1111 116C 11B5; # (푎; 푎; 푎; 푎; 푎; ) HANGUL SYLLABLE POELP +D44F;D44F;1111 116C 11B6;D44F;1111 116C 11B6; # (푏; 푏; 푏; 푏; 푏; ) HANGUL SYLLABLE POELH +D450;D450;1111 116C 11B7;D450;1111 116C 11B7; # (푐; 푐; 푐; 푐; 푐; ) HANGUL SYLLABLE POEM +D451;D451;1111 116C 11B8;D451;1111 116C 11B8; # (푑; 푑; 푑; 푑; 푑; ) HANGUL SYLLABLE POEB +D452;D452;1111 116C 11B9;D452;1111 116C 11B9; # (푒; 푒; 푒; 푒; 푒; ) HANGUL SYLLABLE POEBS +D453;D453;1111 116C 11BA;D453;1111 116C 11BA; # (푓; 푓; 푓; 푓; 푓; ) HANGUL SYLLABLE POES +D454;D454;1111 116C 11BB;D454;1111 116C 11BB; # (푔; 푔; 푔; 푔; 푔; ) HANGUL SYLLABLE POESS +D455;D455;1111 116C 11BC;D455;1111 116C 11BC; # (푕; 푕; 푕; 푕; 푕; ) HANGUL SYLLABLE POENG +D456;D456;1111 116C 11BD;D456;1111 116C 11BD; # (푖; 푖; 푖; 푖; 푖; ) HANGUL SYLLABLE POEJ +D457;D457;1111 116C 11BE;D457;1111 116C 11BE; # (푗; 푗; 푗; 푗; 푗; ) HANGUL SYLLABLE POEC +D458;D458;1111 116C 11BF;D458;1111 116C 11BF; # (푘; 푘; 푘; 푘; 푘; ) HANGUL SYLLABLE POEK +D459;D459;1111 116C 11C0;D459;1111 116C 11C0; # (푙; 푙; 푙; 푙; 푙; ) HANGUL SYLLABLE POET +D45A;D45A;1111 116C 11C1;D45A;1111 116C 11C1; # (푚; 푚; 푚; 푚; 푚; ) HANGUL SYLLABLE POEP +D45B;D45B;1111 116C 11C2;D45B;1111 116C 11C2; # (푛; 푛; 푛; 푛; 푛; ) HANGUL SYLLABLE POEH +D45C;D45C;1111 116D;D45C;1111 116D; # (표; 표; 표; 표; 표; ) HANGUL SYLLABLE PYO +D45D;D45D;1111 116D 11A8;D45D;1111 116D 11A8; # (푝; 푝; 푝; 푝; 푝; ) HANGUL SYLLABLE PYOG +D45E;D45E;1111 116D 11A9;D45E;1111 116D 11A9; # (푞; 푞; 푞; 푞; 푞; ) HANGUL SYLLABLE PYOGG +D45F;D45F;1111 116D 11AA;D45F;1111 116D 11AA; # (푟; 푟; 푟; 푟; 푟; ) HANGUL SYLLABLE PYOGS +D460;D460;1111 116D 11AB;D460;1111 116D 11AB; # (푠; 푠; 푠; 푠; 푠; ) HANGUL SYLLABLE PYON +D461;D461;1111 116D 11AC;D461;1111 116D 11AC; # (푡; 푡; 푡; 푡; 푡; ) HANGUL SYLLABLE PYONJ +D462;D462;1111 116D 11AD;D462;1111 116D 11AD; # (푢; 푢; 푢; 푢; 푢; ) HANGUL SYLLABLE PYONH +D463;D463;1111 116D 11AE;D463;1111 116D 11AE; # (푣; 푣; 푣; 푣; 푣; ) HANGUL SYLLABLE PYOD +D464;D464;1111 116D 11AF;D464;1111 116D 11AF; # (푤; 푤; 푤; 푤; 푤; ) HANGUL SYLLABLE PYOL +D465;D465;1111 116D 11B0;D465;1111 116D 11B0; # (푥; 푥; 푥; 푥; 푥; ) HANGUL SYLLABLE PYOLG +D466;D466;1111 116D 11B1;D466;1111 116D 11B1; # (푦; 푦; 푦; 푦; 푦; ) HANGUL SYLLABLE PYOLM +D467;D467;1111 116D 11B2;D467;1111 116D 11B2; # (푧; 푧; 푧; 푧; 푧; ) HANGUL SYLLABLE PYOLB +D468;D468;1111 116D 11B3;D468;1111 116D 11B3; # (푨; 푨; 푨; 푨; 푨; ) HANGUL SYLLABLE PYOLS +D469;D469;1111 116D 11B4;D469;1111 116D 11B4; # (푩; 푩; 푩; 푩; 푩; ) HANGUL SYLLABLE PYOLT +D46A;D46A;1111 116D 11B5;D46A;1111 116D 11B5; # (푪; 푪; 푪; 푪; 푪; ) HANGUL SYLLABLE PYOLP +D46B;D46B;1111 116D 11B6;D46B;1111 116D 11B6; # (푫; 푫; 푫; 푫; 푫; ) HANGUL SYLLABLE PYOLH +D46C;D46C;1111 116D 11B7;D46C;1111 116D 11B7; # (푬; 푬; 푬; 푬; 푬; ) HANGUL SYLLABLE PYOM +D46D;D46D;1111 116D 11B8;D46D;1111 116D 11B8; # (푭; 푭; 푭; 푭; 푭; ) HANGUL SYLLABLE PYOB +D46E;D46E;1111 116D 11B9;D46E;1111 116D 11B9; # (푮; 푮; 푮; 푮; 푮; ) HANGUL SYLLABLE PYOBS +D46F;D46F;1111 116D 11BA;D46F;1111 116D 11BA; # (푯; 푯; 푯; 푯; 푯; ) HANGUL SYLLABLE PYOS +D470;D470;1111 116D 11BB;D470;1111 116D 11BB; # (푰; 푰; 푰; 푰; 푰; ) HANGUL SYLLABLE PYOSS +D471;D471;1111 116D 11BC;D471;1111 116D 11BC; # (푱; 푱; 푱; 푱; 푱; ) HANGUL SYLLABLE PYONG +D472;D472;1111 116D 11BD;D472;1111 116D 11BD; # (푲; 푲; 푲; 푲; 푲; ) HANGUL SYLLABLE PYOJ +D473;D473;1111 116D 11BE;D473;1111 116D 11BE; # (푳; 푳; 푳; 푳; 푳; ) HANGUL SYLLABLE PYOC +D474;D474;1111 116D 11BF;D474;1111 116D 11BF; # (푴; 푴; 푴; 푴; 푴; ) HANGUL SYLLABLE PYOK +D475;D475;1111 116D 11C0;D475;1111 116D 11C0; # (푵; 푵; 푵; 푵; 푵; ) HANGUL SYLLABLE PYOT +D476;D476;1111 116D 11C1;D476;1111 116D 11C1; # (푶; 푶; 푶; 푶; 푶; ) HANGUL SYLLABLE PYOP +D477;D477;1111 116D 11C2;D477;1111 116D 11C2; # (푷; 푷; 푷; 푷; 푷; ) HANGUL SYLLABLE PYOH +D478;D478;1111 116E;D478;1111 116E; # (푸; 푸; 푸; 푸; 푸; ) HANGUL SYLLABLE PU +D479;D479;1111 116E 11A8;D479;1111 116E 11A8; # (푹; 푹; 푹; 푹; 푹; ) HANGUL SYLLABLE PUG +D47A;D47A;1111 116E 11A9;D47A;1111 116E 11A9; # (푺; 푺; 푺; 푺; 푺; ) HANGUL SYLLABLE PUGG +D47B;D47B;1111 116E 11AA;D47B;1111 116E 11AA; # (푻; 푻; 푻; 푻; 푻; ) HANGUL SYLLABLE PUGS +D47C;D47C;1111 116E 11AB;D47C;1111 116E 11AB; # (푼; 푼; 푼; 푼; 푼; ) HANGUL SYLLABLE PUN +D47D;D47D;1111 116E 11AC;D47D;1111 116E 11AC; # (푽; 푽; 푽; 푽; 푽; ) HANGUL SYLLABLE PUNJ +D47E;D47E;1111 116E 11AD;D47E;1111 116E 11AD; # (푾; 푾; 푾; 푾; 푾; ) HANGUL SYLLABLE PUNH +D47F;D47F;1111 116E 11AE;D47F;1111 116E 11AE; # (푿; 푿; 푿; 푿; 푿; ) HANGUL SYLLABLE PUD +D480;D480;1111 116E 11AF;D480;1111 116E 11AF; # (풀; 풀; 풀; 풀; 풀; ) HANGUL SYLLABLE PUL +D481;D481;1111 116E 11B0;D481;1111 116E 11B0; # (풁; 풁; 풁; 풁; 풁; ) HANGUL SYLLABLE PULG +D482;D482;1111 116E 11B1;D482;1111 116E 11B1; # (풂; 풂; 풂; 풂; 풂; ) HANGUL SYLLABLE PULM +D483;D483;1111 116E 11B2;D483;1111 116E 11B2; # (풃; 풃; 풃; 풃; 풃; ) HANGUL SYLLABLE PULB +D484;D484;1111 116E 11B3;D484;1111 116E 11B3; # (풄; 풄; 풄; 풄; 풄; ) HANGUL SYLLABLE PULS +D485;D485;1111 116E 11B4;D485;1111 116E 11B4; # (풅; 풅; 풅; 풅; 풅; ) HANGUL SYLLABLE PULT +D486;D486;1111 116E 11B5;D486;1111 116E 11B5; # (풆; 풆; 풆; 풆; 풆; ) HANGUL SYLLABLE PULP +D487;D487;1111 116E 11B6;D487;1111 116E 11B6; # (풇; 풇; 풇; 풇; 풇; ) HANGUL SYLLABLE PULH +D488;D488;1111 116E 11B7;D488;1111 116E 11B7; # (품; 품; 품; 품; 품; ) HANGUL SYLLABLE PUM +D489;D489;1111 116E 11B8;D489;1111 116E 11B8; # (풉; 풉; 풉; 풉; 풉; ) HANGUL SYLLABLE PUB +D48A;D48A;1111 116E 11B9;D48A;1111 116E 11B9; # (풊; 풊; 풊; 풊; 풊; ) HANGUL SYLLABLE PUBS +D48B;D48B;1111 116E 11BA;D48B;1111 116E 11BA; # (풋; 풋; 풋; 풋; 풋; ) HANGUL SYLLABLE PUS +D48C;D48C;1111 116E 11BB;D48C;1111 116E 11BB; # (풌; 풌; 풌; 풌; 풌; ) HANGUL SYLLABLE PUSS +D48D;D48D;1111 116E 11BC;D48D;1111 116E 11BC; # (풍; 풍; 풍; 풍; 풍; ) HANGUL SYLLABLE PUNG +D48E;D48E;1111 116E 11BD;D48E;1111 116E 11BD; # (풎; 풎; 풎; 풎; 풎; ) HANGUL SYLLABLE PUJ +D48F;D48F;1111 116E 11BE;D48F;1111 116E 11BE; # (풏; 풏; 풏; 풏; 풏; ) HANGUL SYLLABLE PUC +D490;D490;1111 116E 11BF;D490;1111 116E 11BF; # (풐; 풐; 풐; 풐; 풐; ) HANGUL SYLLABLE PUK +D491;D491;1111 116E 11C0;D491;1111 116E 11C0; # (풑; 풑; 풑; 풑; 풑; ) HANGUL SYLLABLE PUT +D492;D492;1111 116E 11C1;D492;1111 116E 11C1; # (풒; 풒; 풒; 풒; 풒; ) HANGUL SYLLABLE PUP +D493;D493;1111 116E 11C2;D493;1111 116E 11C2; # (풓; 풓; 풓; 풓; 풓; ) HANGUL SYLLABLE PUH +D494;D494;1111 116F;D494;1111 116F; # (풔; 풔; 풔; 풔; 풔; ) HANGUL SYLLABLE PWEO +D495;D495;1111 116F 11A8;D495;1111 116F 11A8; # (풕; 풕; 풕; 풕; 풕; ) HANGUL SYLLABLE PWEOG +D496;D496;1111 116F 11A9;D496;1111 116F 11A9; # (풖; 풖; 풖; 풖; 풖; ) HANGUL SYLLABLE PWEOGG +D497;D497;1111 116F 11AA;D497;1111 116F 11AA; # (풗; 풗; 풗; 풗; 풗; ) HANGUL SYLLABLE PWEOGS +D498;D498;1111 116F 11AB;D498;1111 116F 11AB; # (풘; 풘; 풘; 풘; 풘; ) HANGUL SYLLABLE PWEON +D499;D499;1111 116F 11AC;D499;1111 116F 11AC; # (풙; 풙; 풙; 풙; 풙; ) HANGUL SYLLABLE PWEONJ +D49A;D49A;1111 116F 11AD;D49A;1111 116F 11AD; # (풚; 풚; 풚; 풚; 풚; ) HANGUL SYLLABLE PWEONH +D49B;D49B;1111 116F 11AE;D49B;1111 116F 11AE; # (풛; 풛; 풛; 풛; 풛; ) HANGUL SYLLABLE PWEOD +D49C;D49C;1111 116F 11AF;D49C;1111 116F 11AF; # (풜; 풜; 풜; 풜; 풜; ) HANGUL SYLLABLE PWEOL +D49D;D49D;1111 116F 11B0;D49D;1111 116F 11B0; # (풝; 풝; 풝; 풝; 풝; ) HANGUL SYLLABLE PWEOLG +D49E;D49E;1111 116F 11B1;D49E;1111 116F 11B1; # (풞; 풞; 풞; 풞; 풞; ) HANGUL SYLLABLE PWEOLM +D49F;D49F;1111 116F 11B2;D49F;1111 116F 11B2; # (풟; 풟; 풟; 풟; 풟; ) HANGUL SYLLABLE PWEOLB +D4A0;D4A0;1111 116F 11B3;D4A0;1111 116F 11B3; # (풠; 풠; 풠; 풠; 풠; ) HANGUL SYLLABLE PWEOLS +D4A1;D4A1;1111 116F 11B4;D4A1;1111 116F 11B4; # (풡; 풡; 풡; 풡; 풡; ) HANGUL SYLLABLE PWEOLT +D4A2;D4A2;1111 116F 11B5;D4A2;1111 116F 11B5; # (풢; 풢; 풢; 풢; 풢; ) HANGUL SYLLABLE PWEOLP +D4A3;D4A3;1111 116F 11B6;D4A3;1111 116F 11B6; # (풣; 풣; 풣; 풣; 풣; ) HANGUL SYLLABLE PWEOLH +D4A4;D4A4;1111 116F 11B7;D4A4;1111 116F 11B7; # (풤; 풤; 풤; 풤; 풤; ) HANGUL SYLLABLE PWEOM +D4A5;D4A5;1111 116F 11B8;D4A5;1111 116F 11B8; # (풥; 풥; 풥; 풥; 풥; ) HANGUL SYLLABLE PWEOB +D4A6;D4A6;1111 116F 11B9;D4A6;1111 116F 11B9; # (풦; 풦; 풦; 풦; 풦; ) HANGUL SYLLABLE PWEOBS +D4A7;D4A7;1111 116F 11BA;D4A7;1111 116F 11BA; # (풧; 풧; 풧; 풧; 풧; ) HANGUL SYLLABLE PWEOS +D4A8;D4A8;1111 116F 11BB;D4A8;1111 116F 11BB; # (풨; 풨; 풨; 풨; 풨; ) HANGUL SYLLABLE PWEOSS +D4A9;D4A9;1111 116F 11BC;D4A9;1111 116F 11BC; # (풩; 풩; 풩; 풩; 풩; ) HANGUL SYLLABLE PWEONG +D4AA;D4AA;1111 116F 11BD;D4AA;1111 116F 11BD; # (풪; 풪; 풪; 풪; 풪; ) HANGUL SYLLABLE PWEOJ +D4AB;D4AB;1111 116F 11BE;D4AB;1111 116F 11BE; # (풫; 풫; 풫; 풫; 풫; ) HANGUL SYLLABLE PWEOC +D4AC;D4AC;1111 116F 11BF;D4AC;1111 116F 11BF; # (풬; 풬; 풬; 풬; 풬; ) HANGUL SYLLABLE PWEOK +D4AD;D4AD;1111 116F 11C0;D4AD;1111 116F 11C0; # (풭; 풭; 풭; 풭; 풭; ) HANGUL SYLLABLE PWEOT +D4AE;D4AE;1111 116F 11C1;D4AE;1111 116F 11C1; # (풮; 풮; 풮; 풮; 풮; ) HANGUL SYLLABLE PWEOP +D4AF;D4AF;1111 116F 11C2;D4AF;1111 116F 11C2; # (풯; 풯; 풯; 풯; 풯; ) HANGUL SYLLABLE PWEOH +D4B0;D4B0;1111 1170;D4B0;1111 1170; # (풰; 풰; 풰; 풰; 풰; ) HANGUL SYLLABLE PWE +D4B1;D4B1;1111 1170 11A8;D4B1;1111 1170 11A8; # (풱; 풱; 풱; 풱; 풱; ) HANGUL SYLLABLE PWEG +D4B2;D4B2;1111 1170 11A9;D4B2;1111 1170 11A9; # (풲; 풲; 풲; 풲; 풲; ) HANGUL SYLLABLE PWEGG +D4B3;D4B3;1111 1170 11AA;D4B3;1111 1170 11AA; # (풳; 풳; 풳; 풳; 풳; ) HANGUL SYLLABLE PWEGS +D4B4;D4B4;1111 1170 11AB;D4B4;1111 1170 11AB; # (풴; 풴; 풴; 풴; 풴; ) HANGUL SYLLABLE PWEN +D4B5;D4B5;1111 1170 11AC;D4B5;1111 1170 11AC; # (풵; 풵; 풵; 풵; 풵; ) HANGUL SYLLABLE PWENJ +D4B6;D4B6;1111 1170 11AD;D4B6;1111 1170 11AD; # (풶; 풶; 풶; 풶; 풶; ) HANGUL SYLLABLE PWENH +D4B7;D4B7;1111 1170 11AE;D4B7;1111 1170 11AE; # (풷; 풷; 풷; 풷; 풷; ) HANGUL SYLLABLE PWED +D4B8;D4B8;1111 1170 11AF;D4B8;1111 1170 11AF; # (풸; 풸; 풸; 풸; 풸; ) HANGUL SYLLABLE PWEL +D4B9;D4B9;1111 1170 11B0;D4B9;1111 1170 11B0; # (풹; 풹; 풹; 풹; 풹; ) HANGUL SYLLABLE PWELG +D4BA;D4BA;1111 1170 11B1;D4BA;1111 1170 11B1; # (풺; 풺; 풺; 풺; 풺; ) HANGUL SYLLABLE PWELM +D4BB;D4BB;1111 1170 11B2;D4BB;1111 1170 11B2; # (풻; 풻; 풻; 풻; 풻; ) HANGUL SYLLABLE PWELB +D4BC;D4BC;1111 1170 11B3;D4BC;1111 1170 11B3; # (풼; 풼; 풼; 풼; 풼; ) HANGUL SYLLABLE PWELS +D4BD;D4BD;1111 1170 11B4;D4BD;1111 1170 11B4; # (풽; 풽; 풽; 풽; 풽; ) HANGUL SYLLABLE PWELT +D4BE;D4BE;1111 1170 11B5;D4BE;1111 1170 11B5; # (풾; 풾; 풾; 풾; 풾; ) HANGUL SYLLABLE PWELP +D4BF;D4BF;1111 1170 11B6;D4BF;1111 1170 11B6; # (풿; 풿; 풿; 풿; 풿; ) HANGUL SYLLABLE PWELH +D4C0;D4C0;1111 1170 11B7;D4C0;1111 1170 11B7; # (퓀; 퓀; 퓀; 퓀; 퓀; ) HANGUL SYLLABLE PWEM +D4C1;D4C1;1111 1170 11B8;D4C1;1111 1170 11B8; # (퓁; 퓁; 퓁; 퓁; 퓁; ) HANGUL SYLLABLE PWEB +D4C2;D4C2;1111 1170 11B9;D4C2;1111 1170 11B9; # (퓂; 퓂; 퓂; 퓂; 퓂; ) HANGUL SYLLABLE PWEBS +D4C3;D4C3;1111 1170 11BA;D4C3;1111 1170 11BA; # (퓃; 퓃; 퓃; 퓃; 퓃; ) HANGUL SYLLABLE PWES +D4C4;D4C4;1111 1170 11BB;D4C4;1111 1170 11BB; # (퓄; 퓄; 퓄; 퓄; 퓄; ) HANGUL SYLLABLE PWESS +D4C5;D4C5;1111 1170 11BC;D4C5;1111 1170 11BC; # (퓅; 퓅; 퓅; 퓅; 퓅; ) HANGUL SYLLABLE PWENG +D4C6;D4C6;1111 1170 11BD;D4C6;1111 1170 11BD; # (퓆; 퓆; 퓆; 퓆; 퓆; ) HANGUL SYLLABLE PWEJ +D4C7;D4C7;1111 1170 11BE;D4C7;1111 1170 11BE; # (퓇; 퓇; 퓇; 퓇; 퓇; ) HANGUL SYLLABLE PWEC +D4C8;D4C8;1111 1170 11BF;D4C8;1111 1170 11BF; # (퓈; 퓈; 퓈; 퓈; 퓈; ) HANGUL SYLLABLE PWEK +D4C9;D4C9;1111 1170 11C0;D4C9;1111 1170 11C0; # (퓉; 퓉; 퓉; 퓉; 퓉; ) HANGUL SYLLABLE PWET +D4CA;D4CA;1111 1170 11C1;D4CA;1111 1170 11C1; # (퓊; 퓊; 퓊; 퓊; 퓊; ) HANGUL SYLLABLE PWEP +D4CB;D4CB;1111 1170 11C2;D4CB;1111 1170 11C2; # (퓋; 퓋; 퓋; 퓋; 퓋; ) HANGUL SYLLABLE PWEH +D4CC;D4CC;1111 1171;D4CC;1111 1171; # (퓌; 퓌; 퓌; 퓌; 퓌; ) HANGUL SYLLABLE PWI +D4CD;D4CD;1111 1171 11A8;D4CD;1111 1171 11A8; # (퓍; 퓍; 퓍; 퓍; 퓍; ) HANGUL SYLLABLE PWIG +D4CE;D4CE;1111 1171 11A9;D4CE;1111 1171 11A9; # (퓎; 퓎; 퓎; 퓎; 퓎; ) HANGUL SYLLABLE PWIGG +D4CF;D4CF;1111 1171 11AA;D4CF;1111 1171 11AA; # (퓏; 퓏; 퓏; 퓏; 퓏; ) HANGUL SYLLABLE PWIGS +D4D0;D4D0;1111 1171 11AB;D4D0;1111 1171 11AB; # (퓐; 퓐; 퓐; 퓐; 퓐; ) HANGUL SYLLABLE PWIN +D4D1;D4D1;1111 1171 11AC;D4D1;1111 1171 11AC; # (퓑; 퓑; 퓑; 퓑; 퓑; ) HANGUL SYLLABLE PWINJ +D4D2;D4D2;1111 1171 11AD;D4D2;1111 1171 11AD; # (퓒; 퓒; 퓒; 퓒; 퓒; ) HANGUL SYLLABLE PWINH +D4D3;D4D3;1111 1171 11AE;D4D3;1111 1171 11AE; # (퓓; 퓓; 퓓; 퓓; 퓓; ) HANGUL SYLLABLE PWID +D4D4;D4D4;1111 1171 11AF;D4D4;1111 1171 11AF; # (퓔; 퓔; 퓔; 퓔; 퓔; ) HANGUL SYLLABLE PWIL +D4D5;D4D5;1111 1171 11B0;D4D5;1111 1171 11B0; # (퓕; 퓕; 퓕; 퓕; 퓕; ) HANGUL SYLLABLE PWILG +D4D6;D4D6;1111 1171 11B1;D4D6;1111 1171 11B1; # (퓖; 퓖; 퓖; 퓖; 퓖; ) HANGUL SYLLABLE PWILM +D4D7;D4D7;1111 1171 11B2;D4D7;1111 1171 11B2; # (퓗; 퓗; 퓗; 퓗; 퓗; ) HANGUL SYLLABLE PWILB +D4D8;D4D8;1111 1171 11B3;D4D8;1111 1171 11B3; # (퓘; 퓘; 퓘; 퓘; 퓘; ) HANGUL SYLLABLE PWILS +D4D9;D4D9;1111 1171 11B4;D4D9;1111 1171 11B4; # (퓙; 퓙; 퓙; 퓙; 퓙; ) HANGUL SYLLABLE PWILT +D4DA;D4DA;1111 1171 11B5;D4DA;1111 1171 11B5; # (퓚; 퓚; 퓚; 퓚; 퓚; ) HANGUL SYLLABLE PWILP +D4DB;D4DB;1111 1171 11B6;D4DB;1111 1171 11B6; # (퓛; 퓛; 퓛; 퓛; 퓛; ) HANGUL SYLLABLE PWILH +D4DC;D4DC;1111 1171 11B7;D4DC;1111 1171 11B7; # (퓜; 퓜; 퓜; 퓜; 퓜; ) HANGUL SYLLABLE PWIM +D4DD;D4DD;1111 1171 11B8;D4DD;1111 1171 11B8; # (퓝; 퓝; 퓝; 퓝; 퓝; ) HANGUL SYLLABLE PWIB +D4DE;D4DE;1111 1171 11B9;D4DE;1111 1171 11B9; # (퓞; 퓞; 퓞; 퓞; 퓞; ) HANGUL SYLLABLE PWIBS +D4DF;D4DF;1111 1171 11BA;D4DF;1111 1171 11BA; # (퓟; 퓟; 퓟; 퓟; 퓟; ) HANGUL SYLLABLE PWIS +D4E0;D4E0;1111 1171 11BB;D4E0;1111 1171 11BB; # (퓠; 퓠; 퓠; 퓠; 퓠; ) HANGUL SYLLABLE PWISS +D4E1;D4E1;1111 1171 11BC;D4E1;1111 1171 11BC; # (퓡; 퓡; 퓡; 퓡; 퓡; ) HANGUL SYLLABLE PWING +D4E2;D4E2;1111 1171 11BD;D4E2;1111 1171 11BD; # (퓢; 퓢; 퓢; 퓢; 퓢; ) HANGUL SYLLABLE PWIJ +D4E3;D4E3;1111 1171 11BE;D4E3;1111 1171 11BE; # (퓣; 퓣; 퓣; 퓣; 퓣; ) HANGUL SYLLABLE PWIC +D4E4;D4E4;1111 1171 11BF;D4E4;1111 1171 11BF; # (퓤; 퓤; 퓤; 퓤; 퓤; ) HANGUL SYLLABLE PWIK +D4E5;D4E5;1111 1171 11C0;D4E5;1111 1171 11C0; # (퓥; 퓥; 퓥; 퓥; 퓥; ) HANGUL SYLLABLE PWIT +D4E6;D4E6;1111 1171 11C1;D4E6;1111 1171 11C1; # (퓦; 퓦; 퓦; 퓦; 퓦; ) HANGUL SYLLABLE PWIP +D4E7;D4E7;1111 1171 11C2;D4E7;1111 1171 11C2; # (퓧; 퓧; 퓧; 퓧; 퓧; ) HANGUL SYLLABLE PWIH +D4E8;D4E8;1111 1172;D4E8;1111 1172; # (퓨; 퓨; 퓨; 퓨; 퓨; ) HANGUL SYLLABLE PYU +D4E9;D4E9;1111 1172 11A8;D4E9;1111 1172 11A8; # (퓩; 퓩; 퓩; 퓩; 퓩; ) HANGUL SYLLABLE PYUG +D4EA;D4EA;1111 1172 11A9;D4EA;1111 1172 11A9; # (퓪; 퓪; 퓪; 퓪; 퓪; ) HANGUL SYLLABLE PYUGG +D4EB;D4EB;1111 1172 11AA;D4EB;1111 1172 11AA; # (퓫; 퓫; 퓫; 퓫; 퓫; ) HANGUL SYLLABLE PYUGS +D4EC;D4EC;1111 1172 11AB;D4EC;1111 1172 11AB; # (퓬; 퓬; 퓬; 퓬; 퓬; ) HANGUL SYLLABLE PYUN +D4ED;D4ED;1111 1172 11AC;D4ED;1111 1172 11AC; # (퓭; 퓭; 퓭; 퓭; 퓭; ) HANGUL SYLLABLE PYUNJ +D4EE;D4EE;1111 1172 11AD;D4EE;1111 1172 11AD; # (퓮; 퓮; 퓮; 퓮; 퓮; ) HANGUL SYLLABLE PYUNH +D4EF;D4EF;1111 1172 11AE;D4EF;1111 1172 11AE; # (퓯; 퓯; 퓯; 퓯; 퓯; ) HANGUL SYLLABLE PYUD +D4F0;D4F0;1111 1172 11AF;D4F0;1111 1172 11AF; # (퓰; 퓰; 퓰; 퓰; 퓰; ) HANGUL SYLLABLE PYUL +D4F1;D4F1;1111 1172 11B0;D4F1;1111 1172 11B0; # (퓱; 퓱; 퓱; 퓱; 퓱; ) HANGUL SYLLABLE PYULG +D4F2;D4F2;1111 1172 11B1;D4F2;1111 1172 11B1; # (퓲; 퓲; 퓲; 퓲; 퓲; ) HANGUL SYLLABLE PYULM +D4F3;D4F3;1111 1172 11B2;D4F3;1111 1172 11B2; # (퓳; 퓳; 퓳; 퓳; 퓳; ) HANGUL SYLLABLE PYULB +D4F4;D4F4;1111 1172 11B3;D4F4;1111 1172 11B3; # (퓴; 퓴; 퓴; 퓴; 퓴; ) HANGUL SYLLABLE PYULS +D4F5;D4F5;1111 1172 11B4;D4F5;1111 1172 11B4; # (퓵; 퓵; 퓵; 퓵; 퓵; ) HANGUL SYLLABLE PYULT +D4F6;D4F6;1111 1172 11B5;D4F6;1111 1172 11B5; # (퓶; 퓶; 퓶; 퓶; 퓶; ) HANGUL SYLLABLE PYULP +D4F7;D4F7;1111 1172 11B6;D4F7;1111 1172 11B6; # (퓷; 퓷; 퓷; 퓷; 퓷; ) HANGUL SYLLABLE PYULH +D4F8;D4F8;1111 1172 11B7;D4F8;1111 1172 11B7; # (퓸; 퓸; 퓸; 퓸; 퓸; ) HANGUL SYLLABLE PYUM +D4F9;D4F9;1111 1172 11B8;D4F9;1111 1172 11B8; # (퓹; 퓹; 퓹; 퓹; 퓹; ) HANGUL SYLLABLE PYUB +D4FA;D4FA;1111 1172 11B9;D4FA;1111 1172 11B9; # (퓺; 퓺; 퓺; 퓺; 퓺; ) HANGUL SYLLABLE PYUBS +D4FB;D4FB;1111 1172 11BA;D4FB;1111 1172 11BA; # (퓻; 퓻; 퓻; 퓻; 퓻; ) HANGUL SYLLABLE PYUS +D4FC;D4FC;1111 1172 11BB;D4FC;1111 1172 11BB; # (퓼; 퓼; 퓼; 퓼; 퓼; ) HANGUL SYLLABLE PYUSS +D4FD;D4FD;1111 1172 11BC;D4FD;1111 1172 11BC; # (퓽; 퓽; 퓽; 퓽; 퓽; ) HANGUL SYLLABLE PYUNG +D4FE;D4FE;1111 1172 11BD;D4FE;1111 1172 11BD; # (퓾; 퓾; 퓾; 퓾; 퓾; ) HANGUL SYLLABLE PYUJ +D4FF;D4FF;1111 1172 11BE;D4FF;1111 1172 11BE; # (퓿; 퓿; 퓿; 퓿; 퓿; ) HANGUL SYLLABLE PYUC +D500;D500;1111 1172 11BF;D500;1111 1172 11BF; # (픀; 픀; 픀; 픀; 픀; ) HANGUL SYLLABLE PYUK +D501;D501;1111 1172 11C0;D501;1111 1172 11C0; # (픁; 픁; 픁; 픁; 픁; ) HANGUL SYLLABLE PYUT +D502;D502;1111 1172 11C1;D502;1111 1172 11C1; # (픂; 픂; 픂; 픂; 픂; ) HANGUL SYLLABLE PYUP +D503;D503;1111 1172 11C2;D503;1111 1172 11C2; # (픃; 픃; 픃; 픃; 픃; ) HANGUL SYLLABLE PYUH +D504;D504;1111 1173;D504;1111 1173; # (프; 프; 프; 프; 프; ) HANGUL SYLLABLE PEU +D505;D505;1111 1173 11A8;D505;1111 1173 11A8; # (픅; 픅; 픅; 픅; 픅; ) HANGUL SYLLABLE PEUG +D506;D506;1111 1173 11A9;D506;1111 1173 11A9; # (픆; 픆; 픆; 픆; 픆; ) HANGUL SYLLABLE PEUGG +D507;D507;1111 1173 11AA;D507;1111 1173 11AA; # (픇; 픇; 픇; 픇; 픇; ) HANGUL SYLLABLE PEUGS +D508;D508;1111 1173 11AB;D508;1111 1173 11AB; # (픈; 픈; 픈; 픈; 픈; ) HANGUL SYLLABLE PEUN +D509;D509;1111 1173 11AC;D509;1111 1173 11AC; # (픉; 픉; 픉; 픉; 픉; ) HANGUL SYLLABLE PEUNJ +D50A;D50A;1111 1173 11AD;D50A;1111 1173 11AD; # (픊; 픊; 픊; 픊; 픊; ) HANGUL SYLLABLE PEUNH +D50B;D50B;1111 1173 11AE;D50B;1111 1173 11AE; # (픋; 픋; 픋; 픋; 픋; ) HANGUL SYLLABLE PEUD +D50C;D50C;1111 1173 11AF;D50C;1111 1173 11AF; # (플; 플; 플; 플; 플; ) HANGUL SYLLABLE PEUL +D50D;D50D;1111 1173 11B0;D50D;1111 1173 11B0; # (픍; 픍; 픍; 픍; 픍; ) HANGUL SYLLABLE PEULG +D50E;D50E;1111 1173 11B1;D50E;1111 1173 11B1; # (픎; 픎; 픎; 픎; 픎; ) HANGUL SYLLABLE PEULM +D50F;D50F;1111 1173 11B2;D50F;1111 1173 11B2; # (픏; 픏; 픏; 픏; 픏; ) HANGUL SYLLABLE PEULB +D510;D510;1111 1173 11B3;D510;1111 1173 11B3; # (픐; 픐; 픐; 픐; 픐; ) HANGUL SYLLABLE PEULS +D511;D511;1111 1173 11B4;D511;1111 1173 11B4; # (픑; 픑; 픑; 픑; 픑; ) HANGUL SYLLABLE PEULT +D512;D512;1111 1173 11B5;D512;1111 1173 11B5; # (픒; 픒; 픒; 픒; 픒; ) HANGUL SYLLABLE PEULP +D513;D513;1111 1173 11B6;D513;1111 1173 11B6; # (픓; 픓; 픓; 픓; 픓; ) HANGUL SYLLABLE PEULH +D514;D514;1111 1173 11B7;D514;1111 1173 11B7; # (픔; 픔; 픔; 픔; 픔; ) HANGUL SYLLABLE PEUM +D515;D515;1111 1173 11B8;D515;1111 1173 11B8; # (픕; 픕; 픕; 픕; 픕; ) HANGUL SYLLABLE PEUB +D516;D516;1111 1173 11B9;D516;1111 1173 11B9; # (픖; 픖; 픖; 픖; 픖; ) HANGUL SYLLABLE PEUBS +D517;D517;1111 1173 11BA;D517;1111 1173 11BA; # (픗; 픗; 픗; 픗; 픗; ) HANGUL SYLLABLE PEUS +D518;D518;1111 1173 11BB;D518;1111 1173 11BB; # (픘; 픘; 픘; 픘; 픘; ) HANGUL SYLLABLE PEUSS +D519;D519;1111 1173 11BC;D519;1111 1173 11BC; # (픙; 픙; 픙; 픙; 픙; ) HANGUL SYLLABLE PEUNG +D51A;D51A;1111 1173 11BD;D51A;1111 1173 11BD; # (픚; 픚; 픚; 픚; 픚; ) HANGUL SYLLABLE PEUJ +D51B;D51B;1111 1173 11BE;D51B;1111 1173 11BE; # (픛; 픛; 픛; 픛; 픛; ) HANGUL SYLLABLE PEUC +D51C;D51C;1111 1173 11BF;D51C;1111 1173 11BF; # (픜; 픜; 픜; 픜; 픜; ) HANGUL SYLLABLE PEUK +D51D;D51D;1111 1173 11C0;D51D;1111 1173 11C0; # (픝; 픝; 픝; 픝; 픝; ) HANGUL SYLLABLE PEUT +D51E;D51E;1111 1173 11C1;D51E;1111 1173 11C1; # (픞; 픞; 픞; 픞; 픞; ) HANGUL SYLLABLE PEUP +D51F;D51F;1111 1173 11C2;D51F;1111 1173 11C2; # (픟; 픟; 픟; 픟; 픟; ) HANGUL SYLLABLE PEUH +D520;D520;1111 1174;D520;1111 1174; # (픠; 픠; 픠; 픠; 픠; ) HANGUL SYLLABLE PYI +D521;D521;1111 1174 11A8;D521;1111 1174 11A8; # (픡; 픡; 픡; 픡; 픡; ) HANGUL SYLLABLE PYIG +D522;D522;1111 1174 11A9;D522;1111 1174 11A9; # (픢; 픢; 픢; 픢; 픢; ) HANGUL SYLLABLE PYIGG +D523;D523;1111 1174 11AA;D523;1111 1174 11AA; # (픣; 픣; 픣; 픣; 픣; ) HANGUL SYLLABLE PYIGS +D524;D524;1111 1174 11AB;D524;1111 1174 11AB; # (픤; 픤; 픤; 픤; 픤; ) HANGUL SYLLABLE PYIN +D525;D525;1111 1174 11AC;D525;1111 1174 11AC; # (픥; 픥; 픥; 픥; 픥; ) HANGUL SYLLABLE PYINJ +D526;D526;1111 1174 11AD;D526;1111 1174 11AD; # (픦; 픦; 픦; 픦; 픦; ) HANGUL SYLLABLE PYINH +D527;D527;1111 1174 11AE;D527;1111 1174 11AE; # (픧; 픧; 픧; 픧; 픧; ) HANGUL SYLLABLE PYID +D528;D528;1111 1174 11AF;D528;1111 1174 11AF; # (픨; 픨; 픨; 픨; 픨; ) HANGUL SYLLABLE PYIL +D529;D529;1111 1174 11B0;D529;1111 1174 11B0; # (픩; 픩; 픩; 픩; 픩; ) HANGUL SYLLABLE PYILG +D52A;D52A;1111 1174 11B1;D52A;1111 1174 11B1; # (픪; 픪; 픪; 픪; 픪; ) HANGUL SYLLABLE PYILM +D52B;D52B;1111 1174 11B2;D52B;1111 1174 11B2; # (픫; 픫; 픫; 픫; 픫; ) HANGUL SYLLABLE PYILB +D52C;D52C;1111 1174 11B3;D52C;1111 1174 11B3; # (픬; 픬; 픬; 픬; 픬; ) HANGUL SYLLABLE PYILS +D52D;D52D;1111 1174 11B4;D52D;1111 1174 11B4; # (픭; 픭; 픭; 픭; 픭; ) HANGUL SYLLABLE PYILT +D52E;D52E;1111 1174 11B5;D52E;1111 1174 11B5; # (픮; 픮; 픮; 픮; 픮; ) HANGUL SYLLABLE PYILP +D52F;D52F;1111 1174 11B6;D52F;1111 1174 11B6; # (픯; 픯; 픯; 픯; 픯; ) HANGUL SYLLABLE PYILH +D530;D530;1111 1174 11B7;D530;1111 1174 11B7; # (픰; 픰; 픰; 픰; 픰; ) HANGUL SYLLABLE PYIM +D531;D531;1111 1174 11B8;D531;1111 1174 11B8; # (픱; 픱; 픱; 픱; 픱; ) HANGUL SYLLABLE PYIB +D532;D532;1111 1174 11B9;D532;1111 1174 11B9; # (픲; 픲; 픲; 픲; 픲; ) HANGUL SYLLABLE PYIBS +D533;D533;1111 1174 11BA;D533;1111 1174 11BA; # (픳; 픳; 픳; 픳; 픳; ) HANGUL SYLLABLE PYIS +D534;D534;1111 1174 11BB;D534;1111 1174 11BB; # (픴; 픴; 픴; 픴; 픴; ) HANGUL SYLLABLE PYISS +D535;D535;1111 1174 11BC;D535;1111 1174 11BC; # (픵; 픵; 픵; 픵; 픵; ) HANGUL SYLLABLE PYING +D536;D536;1111 1174 11BD;D536;1111 1174 11BD; # (픶; 픶; 픶; 픶; 픶; ) HANGUL SYLLABLE PYIJ +D537;D537;1111 1174 11BE;D537;1111 1174 11BE; # (픷; 픷; 픷; 픷; 픷; ) HANGUL SYLLABLE PYIC +D538;D538;1111 1174 11BF;D538;1111 1174 11BF; # (픸; 픸; 픸; 픸; 픸; ) HANGUL SYLLABLE PYIK +D539;D539;1111 1174 11C0;D539;1111 1174 11C0; # (픹; 픹; 픹; 픹; 픹; ) HANGUL SYLLABLE PYIT +D53A;D53A;1111 1174 11C1;D53A;1111 1174 11C1; # (픺; 픺; 픺; 픺; 픺; ) HANGUL SYLLABLE PYIP +D53B;D53B;1111 1174 11C2;D53B;1111 1174 11C2; # (픻; 픻; 픻; 픻; 픻; ) HANGUL SYLLABLE PYIH +D53C;D53C;1111 1175;D53C;1111 1175; # (피; 피; 피; 피; 피; ) HANGUL SYLLABLE PI +D53D;D53D;1111 1175 11A8;D53D;1111 1175 11A8; # (픽; 픽; 픽; 픽; 픽; ) HANGUL SYLLABLE PIG +D53E;D53E;1111 1175 11A9;D53E;1111 1175 11A9; # (픾; 픾; 픾; 픾; 픾; ) HANGUL SYLLABLE PIGG +D53F;D53F;1111 1175 11AA;D53F;1111 1175 11AA; # (픿; 픿; 픿; 픿; 픿; ) HANGUL SYLLABLE PIGS +D540;D540;1111 1175 11AB;D540;1111 1175 11AB; # (핀; 핀; 핀; 핀; 핀; ) HANGUL SYLLABLE PIN +D541;D541;1111 1175 11AC;D541;1111 1175 11AC; # (핁; 핁; 핁; 핁; 핁; ) HANGUL SYLLABLE PINJ +D542;D542;1111 1175 11AD;D542;1111 1175 11AD; # (핂; 핂; 핂; 핂; 핂; ) HANGUL SYLLABLE PINH +D543;D543;1111 1175 11AE;D543;1111 1175 11AE; # (핃; 핃; 핃; 핃; 핃; ) HANGUL SYLLABLE PID +D544;D544;1111 1175 11AF;D544;1111 1175 11AF; # (필; 필; 필; 필; 필; ) HANGUL SYLLABLE PIL +D545;D545;1111 1175 11B0;D545;1111 1175 11B0; # (핅; 핅; 핅; 핅; 핅; ) HANGUL SYLLABLE PILG +D546;D546;1111 1175 11B1;D546;1111 1175 11B1; # (핆; 핆; 핆; 핆; 핆; ) HANGUL SYLLABLE PILM +D547;D547;1111 1175 11B2;D547;1111 1175 11B2; # (핇; 핇; 핇; 핇; 핇; ) HANGUL SYLLABLE PILB +D548;D548;1111 1175 11B3;D548;1111 1175 11B3; # (핈; 핈; 핈; 핈; 핈; ) HANGUL SYLLABLE PILS +D549;D549;1111 1175 11B4;D549;1111 1175 11B4; # (핉; 핉; 핉; 핉; 핉; ) HANGUL SYLLABLE PILT +D54A;D54A;1111 1175 11B5;D54A;1111 1175 11B5; # (핊; 핊; 핊; 핊; 핊; ) HANGUL SYLLABLE PILP +D54B;D54B;1111 1175 11B6;D54B;1111 1175 11B6; # (핋; 핋; 핋; 핋; 핋; ) HANGUL SYLLABLE PILH +D54C;D54C;1111 1175 11B7;D54C;1111 1175 11B7; # (핌; 핌; 핌; 핌; 핌; ) HANGUL SYLLABLE PIM +D54D;D54D;1111 1175 11B8;D54D;1111 1175 11B8; # (핍; 핍; 핍; 핍; 핍; ) HANGUL SYLLABLE PIB +D54E;D54E;1111 1175 11B9;D54E;1111 1175 11B9; # (핎; 핎; 핎; 핎; 핎; ) HANGUL SYLLABLE PIBS +D54F;D54F;1111 1175 11BA;D54F;1111 1175 11BA; # (핏; 핏; 핏; 핏; 핏; ) HANGUL SYLLABLE PIS +D550;D550;1111 1175 11BB;D550;1111 1175 11BB; # (핐; 핐; 핐; 핐; 핐; ) HANGUL SYLLABLE PISS +D551;D551;1111 1175 11BC;D551;1111 1175 11BC; # (핑; 핑; 핑; 핑; 핑; ) HANGUL SYLLABLE PING +D552;D552;1111 1175 11BD;D552;1111 1175 11BD; # (핒; 핒; 핒; 핒; 핒; ) HANGUL SYLLABLE PIJ +D553;D553;1111 1175 11BE;D553;1111 1175 11BE; # (핓; 핓; 핓; 핓; 핓; ) HANGUL SYLLABLE PIC +D554;D554;1111 1175 11BF;D554;1111 1175 11BF; # (핔; 핔; 핔; 핔; 핔; ) HANGUL SYLLABLE PIK +D555;D555;1111 1175 11C0;D555;1111 1175 11C0; # (핕; 핕; 핕; 핕; 핕; ) HANGUL SYLLABLE PIT +D556;D556;1111 1175 11C1;D556;1111 1175 11C1; # (핖; 핖; 핖; 핖; 핖; ) HANGUL SYLLABLE PIP +D557;D557;1111 1175 11C2;D557;1111 1175 11C2; # (핗; 핗; 핗; 핗; 핗; ) HANGUL SYLLABLE PIH +D558;D558;1112 1161;D558;1112 1161; # (하; 하; 하; 하; 하; ) HANGUL SYLLABLE HA +D559;D559;1112 1161 11A8;D559;1112 1161 11A8; # (학; 학; 학; 학; 학; ) HANGUL SYLLABLE HAG +D55A;D55A;1112 1161 11A9;D55A;1112 1161 11A9; # (핚; 핚; 핚; 핚; 핚; ) HANGUL SYLLABLE HAGG +D55B;D55B;1112 1161 11AA;D55B;1112 1161 11AA; # (핛; 핛; 핛; 핛; 핛; ) HANGUL SYLLABLE HAGS +D55C;D55C;1112 1161 11AB;D55C;1112 1161 11AB; # (한; 한; 한; 한; 한; ) HANGUL SYLLABLE HAN +D55D;D55D;1112 1161 11AC;D55D;1112 1161 11AC; # (핝; 핝; 핝; 핝; 핝; ) HANGUL SYLLABLE HANJ +D55E;D55E;1112 1161 11AD;D55E;1112 1161 11AD; # (핞; 핞; 핞; 핞; 핞; ) HANGUL SYLLABLE HANH +D55F;D55F;1112 1161 11AE;D55F;1112 1161 11AE; # (핟; 핟; 핟; 핟; 핟; ) HANGUL SYLLABLE HAD +D560;D560;1112 1161 11AF;D560;1112 1161 11AF; # (할; 할; 할; 할; 할; ) HANGUL SYLLABLE HAL +D561;D561;1112 1161 11B0;D561;1112 1161 11B0; # (핡; 핡; 핡; 핡; 핡; ) HANGUL SYLLABLE HALG +D562;D562;1112 1161 11B1;D562;1112 1161 11B1; # (핢; 핢; 핢; 핢; 핢; ) HANGUL SYLLABLE HALM +D563;D563;1112 1161 11B2;D563;1112 1161 11B2; # (핣; 핣; 핣; 핣; 핣; ) HANGUL SYLLABLE HALB +D564;D564;1112 1161 11B3;D564;1112 1161 11B3; # (핤; 핤; 핤; 핤; 핤; ) HANGUL SYLLABLE HALS +D565;D565;1112 1161 11B4;D565;1112 1161 11B4; # (핥; 핥; 핥; 핥; 핥; ) HANGUL SYLLABLE HALT +D566;D566;1112 1161 11B5;D566;1112 1161 11B5; # (핦; 핦; 핦; 핦; 핦; ) HANGUL SYLLABLE HALP +D567;D567;1112 1161 11B6;D567;1112 1161 11B6; # (핧; 핧; 핧; 핧; 핧; ) HANGUL SYLLABLE HALH +D568;D568;1112 1161 11B7;D568;1112 1161 11B7; # (함; 함; 함; 함; 함; ) HANGUL SYLLABLE HAM +D569;D569;1112 1161 11B8;D569;1112 1161 11B8; # (합; 합; 합; 합; 합; ) HANGUL SYLLABLE HAB +D56A;D56A;1112 1161 11B9;D56A;1112 1161 11B9; # (핪; 핪; 핪; 핪; 핪; ) HANGUL SYLLABLE HABS +D56B;D56B;1112 1161 11BA;D56B;1112 1161 11BA; # (핫; 핫; 핫; 핫; 핫; ) HANGUL SYLLABLE HAS +D56C;D56C;1112 1161 11BB;D56C;1112 1161 11BB; # (핬; 핬; 핬; 핬; 핬; ) HANGUL SYLLABLE HASS +D56D;D56D;1112 1161 11BC;D56D;1112 1161 11BC; # (항; 항; 항; 항; 항; ) HANGUL SYLLABLE HANG +D56E;D56E;1112 1161 11BD;D56E;1112 1161 11BD; # (핮; 핮; 핮; 핮; 핮; ) HANGUL SYLLABLE HAJ +D56F;D56F;1112 1161 11BE;D56F;1112 1161 11BE; # (핯; 핯; 핯; 핯; 핯; ) HANGUL SYLLABLE HAC +D570;D570;1112 1161 11BF;D570;1112 1161 11BF; # (핰; 핰; 핰; 핰; 핰; ) HANGUL SYLLABLE HAK +D571;D571;1112 1161 11C0;D571;1112 1161 11C0; # (핱; 핱; 핱; 핱; 핱; ) HANGUL SYLLABLE HAT +D572;D572;1112 1161 11C1;D572;1112 1161 11C1; # (핲; 핲; 핲; 핲; 핲; ) HANGUL SYLLABLE HAP +D573;D573;1112 1161 11C2;D573;1112 1161 11C2; # (핳; 핳; 핳; 핳; 핳; ) HANGUL SYLLABLE HAH +D574;D574;1112 1162;D574;1112 1162; # (해; 해; 해; 해; 해; ) HANGUL SYLLABLE HAE +D575;D575;1112 1162 11A8;D575;1112 1162 11A8; # (핵; 핵; 핵; 핵; 핵; ) HANGUL SYLLABLE HAEG +D576;D576;1112 1162 11A9;D576;1112 1162 11A9; # (핶; 핶; 핶; 핶; 핶; ) HANGUL SYLLABLE HAEGG +D577;D577;1112 1162 11AA;D577;1112 1162 11AA; # (핷; 핷; 핷; 핷; 핷; ) HANGUL SYLLABLE HAEGS +D578;D578;1112 1162 11AB;D578;1112 1162 11AB; # (핸; 핸; 핸; 핸; 핸; ) HANGUL SYLLABLE HAEN +D579;D579;1112 1162 11AC;D579;1112 1162 11AC; # (핹; 핹; 핹; 핹; 핹; ) HANGUL SYLLABLE HAENJ +D57A;D57A;1112 1162 11AD;D57A;1112 1162 11AD; # (핺; 핺; 핺; 핺; 핺; ) HANGUL SYLLABLE HAENH +D57B;D57B;1112 1162 11AE;D57B;1112 1162 11AE; # (핻; 핻; 핻; 핻; 핻; ) HANGUL SYLLABLE HAED +D57C;D57C;1112 1162 11AF;D57C;1112 1162 11AF; # (핼; 핼; 핼; 핼; 핼; ) HANGUL SYLLABLE HAEL +D57D;D57D;1112 1162 11B0;D57D;1112 1162 11B0; # (핽; 핽; 핽; 핽; 핽; ) HANGUL SYLLABLE HAELG +D57E;D57E;1112 1162 11B1;D57E;1112 1162 11B1; # (핾; 핾; 핾; 핾; 핾; ) HANGUL SYLLABLE HAELM +D57F;D57F;1112 1162 11B2;D57F;1112 1162 11B2; # (핿; 핿; 핿; 핿; 핿; ) HANGUL SYLLABLE HAELB +D580;D580;1112 1162 11B3;D580;1112 1162 11B3; # (햀; 햀; 햀; 햀; 햀; ) HANGUL SYLLABLE HAELS +D581;D581;1112 1162 11B4;D581;1112 1162 11B4; # (햁; 햁; 햁; 햁; 햁; ) HANGUL SYLLABLE HAELT +D582;D582;1112 1162 11B5;D582;1112 1162 11B5; # (햂; 햂; 햂; 햂; 햂; ) HANGUL SYLLABLE HAELP +D583;D583;1112 1162 11B6;D583;1112 1162 11B6; # (햃; 햃; 햃; 햃; 햃; ) HANGUL SYLLABLE HAELH +D584;D584;1112 1162 11B7;D584;1112 1162 11B7; # (햄; 햄; 햄; 햄; 햄; ) HANGUL SYLLABLE HAEM +D585;D585;1112 1162 11B8;D585;1112 1162 11B8; # (햅; 햅; 햅; 햅; 햅; ) HANGUL SYLLABLE HAEB +D586;D586;1112 1162 11B9;D586;1112 1162 11B9; # (햆; 햆; 햆; 햆; 햆; ) HANGUL SYLLABLE HAEBS +D587;D587;1112 1162 11BA;D587;1112 1162 11BA; # (햇; 햇; 햇; 햇; 햇; ) HANGUL SYLLABLE HAES +D588;D588;1112 1162 11BB;D588;1112 1162 11BB; # (했; 했; 했; 했; 했; ) HANGUL SYLLABLE HAESS +D589;D589;1112 1162 11BC;D589;1112 1162 11BC; # (행; 행; 행; 행; 행; ) HANGUL SYLLABLE HAENG +D58A;D58A;1112 1162 11BD;D58A;1112 1162 11BD; # (햊; 햊; 햊; 햊; 햊; ) HANGUL SYLLABLE HAEJ +D58B;D58B;1112 1162 11BE;D58B;1112 1162 11BE; # (햋; 햋; 햋; 햋; 햋; ) HANGUL SYLLABLE HAEC +D58C;D58C;1112 1162 11BF;D58C;1112 1162 11BF; # (햌; 햌; 햌; 햌; 햌; ) HANGUL SYLLABLE HAEK +D58D;D58D;1112 1162 11C0;D58D;1112 1162 11C0; # (햍; 햍; 햍; 햍; 햍; ) HANGUL SYLLABLE HAET +D58E;D58E;1112 1162 11C1;D58E;1112 1162 11C1; # (햎; 햎; 햎; 햎; 햎; ) HANGUL SYLLABLE HAEP +D58F;D58F;1112 1162 11C2;D58F;1112 1162 11C2; # (햏; 햏; 햏; 햏; 햏; ) HANGUL SYLLABLE HAEH +D590;D590;1112 1163;D590;1112 1163; # (햐; 햐; 햐; 햐; 햐; ) HANGUL SYLLABLE HYA +D591;D591;1112 1163 11A8;D591;1112 1163 11A8; # (햑; 햑; 햑; 햑; 햑; ) HANGUL SYLLABLE HYAG +D592;D592;1112 1163 11A9;D592;1112 1163 11A9; # (햒; 햒; 햒; 햒; 햒; ) HANGUL SYLLABLE HYAGG +D593;D593;1112 1163 11AA;D593;1112 1163 11AA; # (햓; 햓; 햓; 햓; 햓; ) HANGUL SYLLABLE HYAGS +D594;D594;1112 1163 11AB;D594;1112 1163 11AB; # (햔; 햔; 햔; 햔; 햔; ) HANGUL SYLLABLE HYAN +D595;D595;1112 1163 11AC;D595;1112 1163 11AC; # (햕; 햕; 햕; 햕; 햕; ) HANGUL SYLLABLE HYANJ +D596;D596;1112 1163 11AD;D596;1112 1163 11AD; # (햖; 햖; 햖; 햖; 햖; ) HANGUL SYLLABLE HYANH +D597;D597;1112 1163 11AE;D597;1112 1163 11AE; # (햗; 햗; 햗; 햗; 햗; ) HANGUL SYLLABLE HYAD +D598;D598;1112 1163 11AF;D598;1112 1163 11AF; # (햘; 햘; 햘; 햘; 햘; ) HANGUL SYLLABLE HYAL +D599;D599;1112 1163 11B0;D599;1112 1163 11B0; # (햙; 햙; 햙; 햙; 햙; ) HANGUL SYLLABLE HYALG +D59A;D59A;1112 1163 11B1;D59A;1112 1163 11B1; # (햚; 햚; 햚; 햚; 햚; ) HANGUL SYLLABLE HYALM +D59B;D59B;1112 1163 11B2;D59B;1112 1163 11B2; # (햛; 햛; 햛; 햛; 햛; ) HANGUL SYLLABLE HYALB +D59C;D59C;1112 1163 11B3;D59C;1112 1163 11B3; # (햜; 햜; 햜; 햜; 햜; ) HANGUL SYLLABLE HYALS +D59D;D59D;1112 1163 11B4;D59D;1112 1163 11B4; # (햝; 햝; 햝; 햝; 햝; ) HANGUL SYLLABLE HYALT +D59E;D59E;1112 1163 11B5;D59E;1112 1163 11B5; # (햞; 햞; 햞; 햞; 햞; ) HANGUL SYLLABLE HYALP +D59F;D59F;1112 1163 11B6;D59F;1112 1163 11B6; # (햟; 햟; 햟; 햟; 햟; ) HANGUL SYLLABLE HYALH +D5A0;D5A0;1112 1163 11B7;D5A0;1112 1163 11B7; # (햠; 햠; 햠; 햠; 햠; ) HANGUL SYLLABLE HYAM +D5A1;D5A1;1112 1163 11B8;D5A1;1112 1163 11B8; # (햡; 햡; 햡; 햡; 햡; ) HANGUL SYLLABLE HYAB +D5A2;D5A2;1112 1163 11B9;D5A2;1112 1163 11B9; # (햢; 햢; 햢; 햢; 햢; ) HANGUL SYLLABLE HYABS +D5A3;D5A3;1112 1163 11BA;D5A3;1112 1163 11BA; # (햣; 햣; 햣; 햣; 햣; ) HANGUL SYLLABLE HYAS +D5A4;D5A4;1112 1163 11BB;D5A4;1112 1163 11BB; # (햤; 햤; 햤; 햤; 햤; ) HANGUL SYLLABLE HYASS +D5A5;D5A5;1112 1163 11BC;D5A5;1112 1163 11BC; # (향; 향; 향; 향; 향; ) HANGUL SYLLABLE HYANG +D5A6;D5A6;1112 1163 11BD;D5A6;1112 1163 11BD; # (햦; 햦; 햦; 햦; 햦; ) HANGUL SYLLABLE HYAJ +D5A7;D5A7;1112 1163 11BE;D5A7;1112 1163 11BE; # (햧; 햧; 햧; 햧; 햧; ) HANGUL SYLLABLE HYAC +D5A8;D5A8;1112 1163 11BF;D5A8;1112 1163 11BF; # (햨; 햨; 햨; 햨; 햨; ) HANGUL SYLLABLE HYAK +D5A9;D5A9;1112 1163 11C0;D5A9;1112 1163 11C0; # (햩; 햩; 햩; 햩; 햩; ) HANGUL SYLLABLE HYAT +D5AA;D5AA;1112 1163 11C1;D5AA;1112 1163 11C1; # (햪; 햪; 햪; 햪; 햪; ) HANGUL SYLLABLE HYAP +D5AB;D5AB;1112 1163 11C2;D5AB;1112 1163 11C2; # (햫; 햫; 햫; 햫; 햫; ) HANGUL SYLLABLE HYAH +D5AC;D5AC;1112 1164;D5AC;1112 1164; # (햬; 햬; 햬; 햬; 햬; ) HANGUL SYLLABLE HYAE +D5AD;D5AD;1112 1164 11A8;D5AD;1112 1164 11A8; # (햭; 햭; 햭; 햭; 햭; ) HANGUL SYLLABLE HYAEG +D5AE;D5AE;1112 1164 11A9;D5AE;1112 1164 11A9; # (햮; 햮; 햮; 햮; 햮; ) HANGUL SYLLABLE HYAEGG +D5AF;D5AF;1112 1164 11AA;D5AF;1112 1164 11AA; # (햯; 햯; 햯; 햯; 햯; ) HANGUL SYLLABLE HYAEGS +D5B0;D5B0;1112 1164 11AB;D5B0;1112 1164 11AB; # (햰; 햰; 햰; 햰; 햰; ) HANGUL SYLLABLE HYAEN +D5B1;D5B1;1112 1164 11AC;D5B1;1112 1164 11AC; # (햱; 햱; 햱; 햱; 햱; ) HANGUL SYLLABLE HYAENJ +D5B2;D5B2;1112 1164 11AD;D5B2;1112 1164 11AD; # (햲; 햲; 햲; 햲; 햲; ) HANGUL SYLLABLE HYAENH +D5B3;D5B3;1112 1164 11AE;D5B3;1112 1164 11AE; # (햳; 햳; 햳; 햳; 햳; ) HANGUL SYLLABLE HYAED +D5B4;D5B4;1112 1164 11AF;D5B4;1112 1164 11AF; # (햴; 햴; 햴; 햴; 햴; ) HANGUL SYLLABLE HYAEL +D5B5;D5B5;1112 1164 11B0;D5B5;1112 1164 11B0; # (햵; 햵; 햵; 햵; 햵; ) HANGUL SYLLABLE HYAELG +D5B6;D5B6;1112 1164 11B1;D5B6;1112 1164 11B1; # (햶; 햶; 햶; 햶; 햶; ) HANGUL SYLLABLE HYAELM +D5B7;D5B7;1112 1164 11B2;D5B7;1112 1164 11B2; # (햷; 햷; 햷; 햷; 햷; ) HANGUL SYLLABLE HYAELB +D5B8;D5B8;1112 1164 11B3;D5B8;1112 1164 11B3; # (햸; 햸; 햸; 햸; 햸; ) HANGUL SYLLABLE HYAELS +D5B9;D5B9;1112 1164 11B4;D5B9;1112 1164 11B4; # (햹; 햹; 햹; 햹; 햹; ) HANGUL SYLLABLE HYAELT +D5BA;D5BA;1112 1164 11B5;D5BA;1112 1164 11B5; # (햺; 햺; 햺; 햺; 햺; ) HANGUL SYLLABLE HYAELP +D5BB;D5BB;1112 1164 11B6;D5BB;1112 1164 11B6; # (햻; 햻; 햻; 햻; 햻; ) HANGUL SYLLABLE HYAELH +D5BC;D5BC;1112 1164 11B7;D5BC;1112 1164 11B7; # (햼; 햼; 햼; 햼; 햼; ) HANGUL SYLLABLE HYAEM +D5BD;D5BD;1112 1164 11B8;D5BD;1112 1164 11B8; # (햽; 햽; 햽; 햽; 햽; ) HANGUL SYLLABLE HYAEB +D5BE;D5BE;1112 1164 11B9;D5BE;1112 1164 11B9; # (햾; 햾; 햾; 햾; 햾; ) HANGUL SYLLABLE HYAEBS +D5BF;D5BF;1112 1164 11BA;D5BF;1112 1164 11BA; # (햿; 햿; 햿; 햿; 햿; ) HANGUL SYLLABLE HYAES +D5C0;D5C0;1112 1164 11BB;D5C0;1112 1164 11BB; # (헀; 헀; 헀; 헀; 헀; ) HANGUL SYLLABLE HYAESS +D5C1;D5C1;1112 1164 11BC;D5C1;1112 1164 11BC; # (헁; 헁; 헁; 헁; 헁; ) HANGUL SYLLABLE HYAENG +D5C2;D5C2;1112 1164 11BD;D5C2;1112 1164 11BD; # (헂; 헂; 헂; 헂; 헂; ) HANGUL SYLLABLE HYAEJ +D5C3;D5C3;1112 1164 11BE;D5C3;1112 1164 11BE; # (헃; 헃; 헃; 헃; 헃; ) HANGUL SYLLABLE HYAEC +D5C4;D5C4;1112 1164 11BF;D5C4;1112 1164 11BF; # (헄; 헄; 헄; 헄; 헄; ) HANGUL SYLLABLE HYAEK +D5C5;D5C5;1112 1164 11C0;D5C5;1112 1164 11C0; # (헅; 헅; 헅; 헅; 헅; ) HANGUL SYLLABLE HYAET +D5C6;D5C6;1112 1164 11C1;D5C6;1112 1164 11C1; # (헆; 헆; 헆; 헆; 헆; ) HANGUL SYLLABLE HYAEP +D5C7;D5C7;1112 1164 11C2;D5C7;1112 1164 11C2; # (헇; 헇; 헇; 헇; 헇; ) HANGUL SYLLABLE HYAEH +D5C8;D5C8;1112 1165;D5C8;1112 1165; # (허; 허; 허; 허; 허; ) HANGUL SYLLABLE HEO +D5C9;D5C9;1112 1165 11A8;D5C9;1112 1165 11A8; # (헉; 헉; 헉; 헉; 헉; ) HANGUL SYLLABLE HEOG +D5CA;D5CA;1112 1165 11A9;D5CA;1112 1165 11A9; # (헊; 헊; 헊; 헊; 헊; ) HANGUL SYLLABLE HEOGG +D5CB;D5CB;1112 1165 11AA;D5CB;1112 1165 11AA; # (헋; 헋; 헋; 헋; 헋; ) HANGUL SYLLABLE HEOGS +D5CC;D5CC;1112 1165 11AB;D5CC;1112 1165 11AB; # (헌; 헌; 헌; 헌; 헌; ) HANGUL SYLLABLE HEON +D5CD;D5CD;1112 1165 11AC;D5CD;1112 1165 11AC; # (헍; 헍; 헍; 헍; 헍; ) HANGUL SYLLABLE HEONJ +D5CE;D5CE;1112 1165 11AD;D5CE;1112 1165 11AD; # (헎; 헎; 헎; 헎; 헎; ) HANGUL SYLLABLE HEONH +D5CF;D5CF;1112 1165 11AE;D5CF;1112 1165 11AE; # (헏; 헏; 헏; 헏; 헏; ) HANGUL SYLLABLE HEOD +D5D0;D5D0;1112 1165 11AF;D5D0;1112 1165 11AF; # (헐; 헐; 헐; 헐; 헐; ) HANGUL SYLLABLE HEOL +D5D1;D5D1;1112 1165 11B0;D5D1;1112 1165 11B0; # (헑; 헑; 헑; 헑; 헑; ) HANGUL SYLLABLE HEOLG +D5D2;D5D2;1112 1165 11B1;D5D2;1112 1165 11B1; # (헒; 헒; 헒; 헒; 헒; ) HANGUL SYLLABLE HEOLM +D5D3;D5D3;1112 1165 11B2;D5D3;1112 1165 11B2; # (헓; 헓; 헓; 헓; 헓; ) HANGUL SYLLABLE HEOLB +D5D4;D5D4;1112 1165 11B3;D5D4;1112 1165 11B3; # (헔; 헔; 헔; 헔; 헔; ) HANGUL SYLLABLE HEOLS +D5D5;D5D5;1112 1165 11B4;D5D5;1112 1165 11B4; # (헕; 헕; 헕; 헕; 헕; ) HANGUL SYLLABLE HEOLT +D5D6;D5D6;1112 1165 11B5;D5D6;1112 1165 11B5; # (헖; 헖; 헖; 헖; 헖; ) HANGUL SYLLABLE HEOLP +D5D7;D5D7;1112 1165 11B6;D5D7;1112 1165 11B6; # (헗; 헗; 헗; 헗; 헗; ) HANGUL SYLLABLE HEOLH +D5D8;D5D8;1112 1165 11B7;D5D8;1112 1165 11B7; # (험; 험; 험; 험; 험; ) HANGUL SYLLABLE HEOM +D5D9;D5D9;1112 1165 11B8;D5D9;1112 1165 11B8; # (헙; 헙; 헙; 헙; 헙; ) HANGUL SYLLABLE HEOB +D5DA;D5DA;1112 1165 11B9;D5DA;1112 1165 11B9; # (헚; 헚; 헚; 헚; 헚; ) HANGUL SYLLABLE HEOBS +D5DB;D5DB;1112 1165 11BA;D5DB;1112 1165 11BA; # (헛; 헛; 헛; 헛; 헛; ) HANGUL SYLLABLE HEOS +D5DC;D5DC;1112 1165 11BB;D5DC;1112 1165 11BB; # (헜; 헜; 헜; 헜; 헜; ) HANGUL SYLLABLE HEOSS +D5DD;D5DD;1112 1165 11BC;D5DD;1112 1165 11BC; # (헝; 헝; 헝; 헝; 헝; ) HANGUL SYLLABLE HEONG +D5DE;D5DE;1112 1165 11BD;D5DE;1112 1165 11BD; # (헞; 헞; 헞; 헞; 헞; ) HANGUL SYLLABLE HEOJ +D5DF;D5DF;1112 1165 11BE;D5DF;1112 1165 11BE; # (헟; 헟; 헟; 헟; 헟; ) HANGUL SYLLABLE HEOC +D5E0;D5E0;1112 1165 11BF;D5E0;1112 1165 11BF; # (헠; 헠; 헠; 헠; 헠; ) HANGUL SYLLABLE HEOK +D5E1;D5E1;1112 1165 11C0;D5E1;1112 1165 11C0; # (헡; 헡; 헡; 헡; 헡; ) HANGUL SYLLABLE HEOT +D5E2;D5E2;1112 1165 11C1;D5E2;1112 1165 11C1; # (헢; 헢; 헢; 헢; 헢; ) HANGUL SYLLABLE HEOP +D5E3;D5E3;1112 1165 11C2;D5E3;1112 1165 11C2; # (헣; 헣; 헣; 헣; 헣; ) HANGUL SYLLABLE HEOH +D5E4;D5E4;1112 1166;D5E4;1112 1166; # (헤; 헤; 헤; 헤; 헤; ) HANGUL SYLLABLE HE +D5E5;D5E5;1112 1166 11A8;D5E5;1112 1166 11A8; # (헥; 헥; 헥; 헥; 헥; ) HANGUL SYLLABLE HEG +D5E6;D5E6;1112 1166 11A9;D5E6;1112 1166 11A9; # (헦; 헦; 헦; 헦; 헦; ) HANGUL SYLLABLE HEGG +D5E7;D5E7;1112 1166 11AA;D5E7;1112 1166 11AA; # (헧; 헧; 헧; 헧; 헧; ) HANGUL SYLLABLE HEGS +D5E8;D5E8;1112 1166 11AB;D5E8;1112 1166 11AB; # (헨; 헨; 헨; 헨; 헨; ) HANGUL SYLLABLE HEN +D5E9;D5E9;1112 1166 11AC;D5E9;1112 1166 11AC; # (헩; 헩; 헩; 헩; 헩; ) HANGUL SYLLABLE HENJ +D5EA;D5EA;1112 1166 11AD;D5EA;1112 1166 11AD; # (헪; 헪; 헪; 헪; 헪; ) HANGUL SYLLABLE HENH +D5EB;D5EB;1112 1166 11AE;D5EB;1112 1166 11AE; # (헫; 헫; 헫; 헫; 헫; ) HANGUL SYLLABLE HED +D5EC;D5EC;1112 1166 11AF;D5EC;1112 1166 11AF; # (헬; 헬; 헬; 헬; 헬; ) HANGUL SYLLABLE HEL +D5ED;D5ED;1112 1166 11B0;D5ED;1112 1166 11B0; # (헭; 헭; 헭; 헭; 헭; ) HANGUL SYLLABLE HELG +D5EE;D5EE;1112 1166 11B1;D5EE;1112 1166 11B1; # (헮; 헮; 헮; 헮; 헮; ) HANGUL SYLLABLE HELM +D5EF;D5EF;1112 1166 11B2;D5EF;1112 1166 11B2; # (헯; 헯; 헯; 헯; 헯; ) HANGUL SYLLABLE HELB +D5F0;D5F0;1112 1166 11B3;D5F0;1112 1166 11B3; # (헰; 헰; 헰; 헰; 헰; ) HANGUL SYLLABLE HELS +D5F1;D5F1;1112 1166 11B4;D5F1;1112 1166 11B4; # (헱; 헱; 헱; 헱; 헱; ) HANGUL SYLLABLE HELT +D5F2;D5F2;1112 1166 11B5;D5F2;1112 1166 11B5; # (헲; 헲; 헲; 헲; 헲; ) HANGUL SYLLABLE HELP +D5F3;D5F3;1112 1166 11B6;D5F3;1112 1166 11B6; # (헳; 헳; 헳; 헳; 헳; ) HANGUL SYLLABLE HELH +D5F4;D5F4;1112 1166 11B7;D5F4;1112 1166 11B7; # (헴; 헴; 헴; 헴; 헴; ) HANGUL SYLLABLE HEM +D5F5;D5F5;1112 1166 11B8;D5F5;1112 1166 11B8; # (헵; 헵; 헵; 헵; 헵; ) HANGUL SYLLABLE HEB +D5F6;D5F6;1112 1166 11B9;D5F6;1112 1166 11B9; # (헶; 헶; 헶; 헶; 헶; ) HANGUL SYLLABLE HEBS +D5F7;D5F7;1112 1166 11BA;D5F7;1112 1166 11BA; # (헷; 헷; 헷; 헷; 헷; ) HANGUL SYLLABLE HES +D5F8;D5F8;1112 1166 11BB;D5F8;1112 1166 11BB; # (헸; 헸; 헸; 헸; 헸; ) HANGUL SYLLABLE HESS +D5F9;D5F9;1112 1166 11BC;D5F9;1112 1166 11BC; # (헹; 헹; 헹; 헹; 헹; ) HANGUL SYLLABLE HENG +D5FA;D5FA;1112 1166 11BD;D5FA;1112 1166 11BD; # (헺; 헺; 헺; 헺; 헺; ) HANGUL SYLLABLE HEJ +D5FB;D5FB;1112 1166 11BE;D5FB;1112 1166 11BE; # (헻; 헻; 헻; 헻; 헻; ) HANGUL SYLLABLE HEC +D5FC;D5FC;1112 1166 11BF;D5FC;1112 1166 11BF; # (헼; 헼; 헼; 헼; 헼; ) HANGUL SYLLABLE HEK +D5FD;D5FD;1112 1166 11C0;D5FD;1112 1166 11C0; # (헽; 헽; 헽; 헽; 헽; ) HANGUL SYLLABLE HET +D5FE;D5FE;1112 1166 11C1;D5FE;1112 1166 11C1; # (헾; 헾; 헾; 헾; 헾; ) HANGUL SYLLABLE HEP +D5FF;D5FF;1112 1166 11C2;D5FF;1112 1166 11C2; # (헿; 헿; 헿; 헿; 헿; ) HANGUL SYLLABLE HEH +D600;D600;1112 1167;D600;1112 1167; # (혀; 혀; 혀; 혀; 혀; ) HANGUL SYLLABLE HYEO +D601;D601;1112 1167 11A8;D601;1112 1167 11A8; # (혁; 혁; 혁; 혁; 혁; ) HANGUL SYLLABLE HYEOG +D602;D602;1112 1167 11A9;D602;1112 1167 11A9; # (혂; 혂; 혂; 혂; 혂; ) HANGUL SYLLABLE HYEOGG +D603;D603;1112 1167 11AA;D603;1112 1167 11AA; # (혃; 혃; 혃; 혃; 혃; ) HANGUL SYLLABLE HYEOGS +D604;D604;1112 1167 11AB;D604;1112 1167 11AB; # (현; 현; 현; 현; 현; ) HANGUL SYLLABLE HYEON +D605;D605;1112 1167 11AC;D605;1112 1167 11AC; # (혅; 혅; 혅; 혅; 혅; ) HANGUL SYLLABLE HYEONJ +D606;D606;1112 1167 11AD;D606;1112 1167 11AD; # (혆; 혆; 혆; 혆; 혆; ) HANGUL SYLLABLE HYEONH +D607;D607;1112 1167 11AE;D607;1112 1167 11AE; # (혇; 혇; 혇; 혇; 혇; ) HANGUL SYLLABLE HYEOD +D608;D608;1112 1167 11AF;D608;1112 1167 11AF; # (혈; 혈; 혈; 혈; 혈; ) HANGUL SYLLABLE HYEOL +D609;D609;1112 1167 11B0;D609;1112 1167 11B0; # (혉; 혉; 혉; 혉; 혉; ) HANGUL SYLLABLE HYEOLG +D60A;D60A;1112 1167 11B1;D60A;1112 1167 11B1; # (혊; 혊; 혊; 혊; 혊; ) HANGUL SYLLABLE HYEOLM +D60B;D60B;1112 1167 11B2;D60B;1112 1167 11B2; # (혋; 혋; 혋; 혋; 혋; ) HANGUL SYLLABLE HYEOLB +D60C;D60C;1112 1167 11B3;D60C;1112 1167 11B3; # (혌; 혌; 혌; 혌; 혌; ) HANGUL SYLLABLE HYEOLS +D60D;D60D;1112 1167 11B4;D60D;1112 1167 11B4; # (혍; 혍; 혍; 혍; 혍; ) HANGUL SYLLABLE HYEOLT +D60E;D60E;1112 1167 11B5;D60E;1112 1167 11B5; # (혎; 혎; 혎; 혎; 혎; ) HANGUL SYLLABLE HYEOLP +D60F;D60F;1112 1167 11B6;D60F;1112 1167 11B6; # (혏; 혏; 혏; 혏; 혏; ) HANGUL SYLLABLE HYEOLH +D610;D610;1112 1167 11B7;D610;1112 1167 11B7; # (혐; 혐; 혐; 혐; 혐; ) HANGUL SYLLABLE HYEOM +D611;D611;1112 1167 11B8;D611;1112 1167 11B8; # (협; 협; 협; 협; 협; ) HANGUL SYLLABLE HYEOB +D612;D612;1112 1167 11B9;D612;1112 1167 11B9; # (혒; 혒; 혒; 혒; 혒; ) HANGUL SYLLABLE HYEOBS +D613;D613;1112 1167 11BA;D613;1112 1167 11BA; # (혓; 혓; 혓; 혓; 혓; ) HANGUL SYLLABLE HYEOS +D614;D614;1112 1167 11BB;D614;1112 1167 11BB; # (혔; 혔; 혔; 혔; 혔; ) HANGUL SYLLABLE HYEOSS +D615;D615;1112 1167 11BC;D615;1112 1167 11BC; # (형; 형; 형; 형; 형; ) HANGUL SYLLABLE HYEONG +D616;D616;1112 1167 11BD;D616;1112 1167 11BD; # (혖; 혖; 혖; 혖; 혖; ) HANGUL SYLLABLE HYEOJ +D617;D617;1112 1167 11BE;D617;1112 1167 11BE; # (혗; 혗; 혗; 혗; 혗; ) HANGUL SYLLABLE HYEOC +D618;D618;1112 1167 11BF;D618;1112 1167 11BF; # (혘; 혘; 혘; 혘; 혘; ) HANGUL SYLLABLE HYEOK +D619;D619;1112 1167 11C0;D619;1112 1167 11C0; # (혙; 혙; 혙; 혙; 혙; ) HANGUL SYLLABLE HYEOT +D61A;D61A;1112 1167 11C1;D61A;1112 1167 11C1; # (혚; 혚; 혚; 혚; 혚; ) HANGUL SYLLABLE HYEOP +D61B;D61B;1112 1167 11C2;D61B;1112 1167 11C2; # (혛; 혛; 혛; 혛; 혛; ) HANGUL SYLLABLE HYEOH +D61C;D61C;1112 1168;D61C;1112 1168; # (혜; 혜; 혜; 혜; 혜; ) HANGUL SYLLABLE HYE +D61D;D61D;1112 1168 11A8;D61D;1112 1168 11A8; # (혝; 혝; 혝; 혝; 혝; ) HANGUL SYLLABLE HYEG +D61E;D61E;1112 1168 11A9;D61E;1112 1168 11A9; # (혞; 혞; 혞; 혞; 혞; ) HANGUL SYLLABLE HYEGG +D61F;D61F;1112 1168 11AA;D61F;1112 1168 11AA; # (혟; 혟; 혟; 혟; 혟; ) HANGUL SYLLABLE HYEGS +D620;D620;1112 1168 11AB;D620;1112 1168 11AB; # (혠; 혠; 혠; 혠; 혠; ) HANGUL SYLLABLE HYEN +D621;D621;1112 1168 11AC;D621;1112 1168 11AC; # (혡; 혡; 혡; 혡; 혡; ) HANGUL SYLLABLE HYENJ +D622;D622;1112 1168 11AD;D622;1112 1168 11AD; # (혢; 혢; 혢; 혢; 혢; ) HANGUL SYLLABLE HYENH +D623;D623;1112 1168 11AE;D623;1112 1168 11AE; # (혣; 혣; 혣; 혣; 혣; ) HANGUL SYLLABLE HYED +D624;D624;1112 1168 11AF;D624;1112 1168 11AF; # (혤; 혤; 혤; 혤; 혤; ) HANGUL SYLLABLE HYEL +D625;D625;1112 1168 11B0;D625;1112 1168 11B0; # (혥; 혥; 혥; 혥; 혥; ) HANGUL SYLLABLE HYELG +D626;D626;1112 1168 11B1;D626;1112 1168 11B1; # (혦; 혦; 혦; 혦; 혦; ) HANGUL SYLLABLE HYELM +D627;D627;1112 1168 11B2;D627;1112 1168 11B2; # (혧; 혧; 혧; 혧; 혧; ) HANGUL SYLLABLE HYELB +D628;D628;1112 1168 11B3;D628;1112 1168 11B3; # (혨; 혨; 혨; 혨; 혨; ) HANGUL SYLLABLE HYELS +D629;D629;1112 1168 11B4;D629;1112 1168 11B4; # (혩; 혩; 혩; 혩; 혩; ) HANGUL SYLLABLE HYELT +D62A;D62A;1112 1168 11B5;D62A;1112 1168 11B5; # (혪; 혪; 혪; 혪; 혪; ) HANGUL SYLLABLE HYELP +D62B;D62B;1112 1168 11B6;D62B;1112 1168 11B6; # (혫; 혫; 혫; 혫; 혫; ) HANGUL SYLLABLE HYELH +D62C;D62C;1112 1168 11B7;D62C;1112 1168 11B7; # (혬; 혬; 혬; 혬; 혬; ) HANGUL SYLLABLE HYEM +D62D;D62D;1112 1168 11B8;D62D;1112 1168 11B8; # (혭; 혭; 혭; 혭; 혭; ) HANGUL SYLLABLE HYEB +D62E;D62E;1112 1168 11B9;D62E;1112 1168 11B9; # (혮; 혮; 혮; 혮; 혮; ) HANGUL SYLLABLE HYEBS +D62F;D62F;1112 1168 11BA;D62F;1112 1168 11BA; # (혯; 혯; 혯; 혯; 혯; ) HANGUL SYLLABLE HYES +D630;D630;1112 1168 11BB;D630;1112 1168 11BB; # (혰; 혰; 혰; 혰; 혰; ) HANGUL SYLLABLE HYESS +D631;D631;1112 1168 11BC;D631;1112 1168 11BC; # (혱; 혱; 혱; 혱; 혱; ) HANGUL SYLLABLE HYENG +D632;D632;1112 1168 11BD;D632;1112 1168 11BD; # (혲; 혲; 혲; 혲; 혲; ) HANGUL SYLLABLE HYEJ +D633;D633;1112 1168 11BE;D633;1112 1168 11BE; # (혳; 혳; 혳; 혳; 혳; ) HANGUL SYLLABLE HYEC +D634;D634;1112 1168 11BF;D634;1112 1168 11BF; # (혴; 혴; 혴; 혴; 혴; ) HANGUL SYLLABLE HYEK +D635;D635;1112 1168 11C0;D635;1112 1168 11C0; # (혵; 혵; 혵; 혵; 혵; ) HANGUL SYLLABLE HYET +D636;D636;1112 1168 11C1;D636;1112 1168 11C1; # (혶; 혶; 혶; 혶; 혶; ) HANGUL SYLLABLE HYEP +D637;D637;1112 1168 11C2;D637;1112 1168 11C2; # (혷; 혷; 혷; 혷; 혷; ) HANGUL SYLLABLE HYEH +D638;D638;1112 1169;D638;1112 1169; # (호; 호; 호; 호; 호; ) HANGUL SYLLABLE HO +D639;D639;1112 1169 11A8;D639;1112 1169 11A8; # (혹; 혹; 혹; 혹; 혹; ) HANGUL SYLLABLE HOG +D63A;D63A;1112 1169 11A9;D63A;1112 1169 11A9; # (혺; 혺; 혺; 혺; 혺; ) HANGUL SYLLABLE HOGG +D63B;D63B;1112 1169 11AA;D63B;1112 1169 11AA; # (혻; 혻; 혻; 혻; 혻; ) HANGUL SYLLABLE HOGS +D63C;D63C;1112 1169 11AB;D63C;1112 1169 11AB; # (혼; 혼; 혼; 혼; 혼; ) HANGUL SYLLABLE HON +D63D;D63D;1112 1169 11AC;D63D;1112 1169 11AC; # (혽; 혽; 혽; 혽; 혽; ) HANGUL SYLLABLE HONJ +D63E;D63E;1112 1169 11AD;D63E;1112 1169 11AD; # (혾; 혾; 혾; 혾; 혾; ) HANGUL SYLLABLE HONH +D63F;D63F;1112 1169 11AE;D63F;1112 1169 11AE; # (혿; 혿; 혿; 혿; 혿; ) HANGUL SYLLABLE HOD +D640;D640;1112 1169 11AF;D640;1112 1169 11AF; # (홀; 홀; 홀; 홀; 홀; ) HANGUL SYLLABLE HOL +D641;D641;1112 1169 11B0;D641;1112 1169 11B0; # (홁; 홁; 홁; 홁; 홁; ) HANGUL SYLLABLE HOLG +D642;D642;1112 1169 11B1;D642;1112 1169 11B1; # (홂; 홂; 홂; 홂; 홂; ) HANGUL SYLLABLE HOLM +D643;D643;1112 1169 11B2;D643;1112 1169 11B2; # (홃; 홃; 홃; 홃; 홃; ) HANGUL SYLLABLE HOLB +D644;D644;1112 1169 11B3;D644;1112 1169 11B3; # (홄; 홄; 홄; 홄; 홄; ) HANGUL SYLLABLE HOLS +D645;D645;1112 1169 11B4;D645;1112 1169 11B4; # (홅; 홅; 홅; 홅; 홅; ) HANGUL SYLLABLE HOLT +D646;D646;1112 1169 11B5;D646;1112 1169 11B5; # (홆; 홆; 홆; 홆; 홆; ) HANGUL SYLLABLE HOLP +D647;D647;1112 1169 11B6;D647;1112 1169 11B6; # (홇; 홇; 홇; 홇; 홇; ) HANGUL SYLLABLE HOLH +D648;D648;1112 1169 11B7;D648;1112 1169 11B7; # (홈; 홈; 홈; 홈; 홈; ) HANGUL SYLLABLE HOM +D649;D649;1112 1169 11B8;D649;1112 1169 11B8; # (홉; 홉; 홉; 홉; 홉; ) HANGUL SYLLABLE HOB +D64A;D64A;1112 1169 11B9;D64A;1112 1169 11B9; # (홊; 홊; 홊; 홊; 홊; ) HANGUL SYLLABLE HOBS +D64B;D64B;1112 1169 11BA;D64B;1112 1169 11BA; # (홋; 홋; 홋; 홋; 홋; ) HANGUL SYLLABLE HOS +D64C;D64C;1112 1169 11BB;D64C;1112 1169 11BB; # (홌; 홌; 홌; 홌; 홌; ) HANGUL SYLLABLE HOSS +D64D;D64D;1112 1169 11BC;D64D;1112 1169 11BC; # (홍; 홍; 홍; 홍; 홍; ) HANGUL SYLLABLE HONG +D64E;D64E;1112 1169 11BD;D64E;1112 1169 11BD; # (홎; 홎; 홎; 홎; 홎; ) HANGUL SYLLABLE HOJ +D64F;D64F;1112 1169 11BE;D64F;1112 1169 11BE; # (홏; 홏; 홏; 홏; 홏; ) HANGUL SYLLABLE HOC +D650;D650;1112 1169 11BF;D650;1112 1169 11BF; # (홐; 홐; 홐; 홐; 홐; ) HANGUL SYLLABLE HOK +D651;D651;1112 1169 11C0;D651;1112 1169 11C0; # (홑; 홑; 홑; 홑; 홑; ) HANGUL SYLLABLE HOT +D652;D652;1112 1169 11C1;D652;1112 1169 11C1; # (홒; 홒; 홒; 홒; 홒; ) HANGUL SYLLABLE HOP +D653;D653;1112 1169 11C2;D653;1112 1169 11C2; # (홓; 홓; 홓; 홓; 홓; ) HANGUL SYLLABLE HOH +D654;D654;1112 116A;D654;1112 116A; # (화; 화; 화; 화; 화; ) HANGUL SYLLABLE HWA +D655;D655;1112 116A 11A8;D655;1112 116A 11A8; # (확; 확; 확; 확; 확; ) HANGUL SYLLABLE HWAG +D656;D656;1112 116A 11A9;D656;1112 116A 11A9; # (홖; 홖; 홖; 홖; 홖; ) HANGUL SYLLABLE HWAGG +D657;D657;1112 116A 11AA;D657;1112 116A 11AA; # (홗; 홗; 홗; 홗; 홗; ) HANGUL SYLLABLE HWAGS +D658;D658;1112 116A 11AB;D658;1112 116A 11AB; # (환; 환; 환; 환; 환; ) HANGUL SYLLABLE HWAN +D659;D659;1112 116A 11AC;D659;1112 116A 11AC; # (홙; 홙; 홙; 홙; 홙; ) HANGUL SYLLABLE HWANJ +D65A;D65A;1112 116A 11AD;D65A;1112 116A 11AD; # (홚; 홚; 홚; 홚; 홚; ) HANGUL SYLLABLE HWANH +D65B;D65B;1112 116A 11AE;D65B;1112 116A 11AE; # (홛; 홛; 홛; 홛; 홛; ) HANGUL SYLLABLE HWAD +D65C;D65C;1112 116A 11AF;D65C;1112 116A 11AF; # (활; 활; 활; 활; 활; ) HANGUL SYLLABLE HWAL +D65D;D65D;1112 116A 11B0;D65D;1112 116A 11B0; # (홝; 홝; 홝; 홝; 홝; ) HANGUL SYLLABLE HWALG +D65E;D65E;1112 116A 11B1;D65E;1112 116A 11B1; # (홞; 홞; 홞; 홞; 홞; ) HANGUL SYLLABLE HWALM +D65F;D65F;1112 116A 11B2;D65F;1112 116A 11B2; # (홟; 홟; 홟; 홟; 홟; ) HANGUL SYLLABLE HWALB +D660;D660;1112 116A 11B3;D660;1112 116A 11B3; # (홠; 홠; 홠; 홠; 홠; ) HANGUL SYLLABLE HWALS +D661;D661;1112 116A 11B4;D661;1112 116A 11B4; # (홡; 홡; 홡; 홡; 홡; ) HANGUL SYLLABLE HWALT +D662;D662;1112 116A 11B5;D662;1112 116A 11B5; # (홢; 홢; 홢; 홢; 홢; ) HANGUL SYLLABLE HWALP +D663;D663;1112 116A 11B6;D663;1112 116A 11B6; # (홣; 홣; 홣; 홣; 홣; ) HANGUL SYLLABLE HWALH +D664;D664;1112 116A 11B7;D664;1112 116A 11B7; # (홤; 홤; 홤; 홤; 홤; ) HANGUL SYLLABLE HWAM +D665;D665;1112 116A 11B8;D665;1112 116A 11B8; # (홥; 홥; 홥; 홥; 홥; ) HANGUL SYLLABLE HWAB +D666;D666;1112 116A 11B9;D666;1112 116A 11B9; # (홦; 홦; 홦; 홦; 홦; ) HANGUL SYLLABLE HWABS +D667;D667;1112 116A 11BA;D667;1112 116A 11BA; # (홧; 홧; 홧; 홧; 홧; ) HANGUL SYLLABLE HWAS +D668;D668;1112 116A 11BB;D668;1112 116A 11BB; # (홨; 홨; 홨; 홨; 홨; ) HANGUL SYLLABLE HWASS +D669;D669;1112 116A 11BC;D669;1112 116A 11BC; # (황; 황; 황; 황; 황; ) HANGUL SYLLABLE HWANG +D66A;D66A;1112 116A 11BD;D66A;1112 116A 11BD; # (홪; 홪; 홪; 홪; 홪; ) HANGUL SYLLABLE HWAJ +D66B;D66B;1112 116A 11BE;D66B;1112 116A 11BE; # (홫; 홫; 홫; 홫; 홫; ) HANGUL SYLLABLE HWAC +D66C;D66C;1112 116A 11BF;D66C;1112 116A 11BF; # (홬; 홬; 홬; 홬; 홬; ) HANGUL SYLLABLE HWAK +D66D;D66D;1112 116A 11C0;D66D;1112 116A 11C0; # (홭; 홭; 홭; 홭; 홭; ) HANGUL SYLLABLE HWAT +D66E;D66E;1112 116A 11C1;D66E;1112 116A 11C1; # (홮; 홮; 홮; 홮; 홮; ) HANGUL SYLLABLE HWAP +D66F;D66F;1112 116A 11C2;D66F;1112 116A 11C2; # (홯; 홯; 홯; 홯; 홯; ) HANGUL SYLLABLE HWAH +D670;D670;1112 116B;D670;1112 116B; # (홰; 홰; 홰; 홰; 홰; ) HANGUL SYLLABLE HWAE +D671;D671;1112 116B 11A8;D671;1112 116B 11A8; # (홱; 홱; 홱; 홱; 홱; ) HANGUL SYLLABLE HWAEG +D672;D672;1112 116B 11A9;D672;1112 116B 11A9; # (홲; 홲; 홲; 홲; 홲; ) HANGUL SYLLABLE HWAEGG +D673;D673;1112 116B 11AA;D673;1112 116B 11AA; # (홳; 홳; 홳; 홳; 홳; ) HANGUL SYLLABLE HWAEGS +D674;D674;1112 116B 11AB;D674;1112 116B 11AB; # (홴; 홴; 홴; 홴; 홴; ) HANGUL SYLLABLE HWAEN +D675;D675;1112 116B 11AC;D675;1112 116B 11AC; # (홵; 홵; 홵; 홵; 홵; ) HANGUL SYLLABLE HWAENJ +D676;D676;1112 116B 11AD;D676;1112 116B 11AD; # (홶; 홶; 홶; 홶; 홶; ) HANGUL SYLLABLE HWAENH +D677;D677;1112 116B 11AE;D677;1112 116B 11AE; # (홷; 홷; 홷; 홷; 홷; ) HANGUL SYLLABLE HWAED +D678;D678;1112 116B 11AF;D678;1112 116B 11AF; # (홸; 홸; 홸; 홸; 홸; ) HANGUL SYLLABLE HWAEL +D679;D679;1112 116B 11B0;D679;1112 116B 11B0; # (홹; 홹; 홹; 홹; 홹; ) HANGUL SYLLABLE HWAELG +D67A;D67A;1112 116B 11B1;D67A;1112 116B 11B1; # (홺; 홺; 홺; 홺; 홺; ) HANGUL SYLLABLE HWAELM +D67B;D67B;1112 116B 11B2;D67B;1112 116B 11B2; # (홻; 홻; 홻; 홻; 홻; ) HANGUL SYLLABLE HWAELB +D67C;D67C;1112 116B 11B3;D67C;1112 116B 11B3; # (홼; 홼; 홼; 홼; 홼; ) HANGUL SYLLABLE HWAELS +D67D;D67D;1112 116B 11B4;D67D;1112 116B 11B4; # (홽; 홽; 홽; 홽; 홽; ) HANGUL SYLLABLE HWAELT +D67E;D67E;1112 116B 11B5;D67E;1112 116B 11B5; # (홾; 홾; 홾; 홾; 홾; ) HANGUL SYLLABLE HWAELP +D67F;D67F;1112 116B 11B6;D67F;1112 116B 11B6; # (홿; 홿; 홿; 홿; 홿; ) HANGUL SYLLABLE HWAELH +D680;D680;1112 116B 11B7;D680;1112 116B 11B7; # (횀; 횀; 횀; 횀; 횀; ) HANGUL SYLLABLE HWAEM +D681;D681;1112 116B 11B8;D681;1112 116B 11B8; # (횁; 횁; 횁; 횁; 횁; ) HANGUL SYLLABLE HWAEB +D682;D682;1112 116B 11B9;D682;1112 116B 11B9; # (횂; 횂; 횂; 횂; 횂; ) HANGUL SYLLABLE HWAEBS +D683;D683;1112 116B 11BA;D683;1112 116B 11BA; # (횃; 횃; 횃; 횃; 횃; ) HANGUL SYLLABLE HWAES +D684;D684;1112 116B 11BB;D684;1112 116B 11BB; # (횄; 횄; 횄; 횄; 횄; ) HANGUL SYLLABLE HWAESS +D685;D685;1112 116B 11BC;D685;1112 116B 11BC; # (횅; 횅; 횅; 횅; 횅; ) HANGUL SYLLABLE HWAENG +D686;D686;1112 116B 11BD;D686;1112 116B 11BD; # (횆; 횆; 횆; 횆; 횆; ) HANGUL SYLLABLE HWAEJ +D687;D687;1112 116B 11BE;D687;1112 116B 11BE; # (횇; 횇; 횇; 횇; 횇; ) HANGUL SYLLABLE HWAEC +D688;D688;1112 116B 11BF;D688;1112 116B 11BF; # (횈; 횈; 횈; 횈; 횈; ) HANGUL SYLLABLE HWAEK +D689;D689;1112 116B 11C0;D689;1112 116B 11C0; # (횉; 횉; 횉; 횉; 횉; ) HANGUL SYLLABLE HWAET +D68A;D68A;1112 116B 11C1;D68A;1112 116B 11C1; # (횊; 횊; 횊; 횊; 횊; ) HANGUL SYLLABLE HWAEP +D68B;D68B;1112 116B 11C2;D68B;1112 116B 11C2; # (횋; 횋; 횋; 횋; 횋; ) HANGUL SYLLABLE HWAEH +D68C;D68C;1112 116C;D68C;1112 116C; # (회; 회; 회; 회; 회; ) HANGUL SYLLABLE HOE +D68D;D68D;1112 116C 11A8;D68D;1112 116C 11A8; # (획; 획; 획; 획; 획; ) HANGUL SYLLABLE HOEG +D68E;D68E;1112 116C 11A9;D68E;1112 116C 11A9; # (횎; 횎; 횎; 횎; 횎; ) HANGUL SYLLABLE HOEGG +D68F;D68F;1112 116C 11AA;D68F;1112 116C 11AA; # (횏; 횏; 횏; 횏; 횏; ) HANGUL SYLLABLE HOEGS +D690;D690;1112 116C 11AB;D690;1112 116C 11AB; # (횐; 횐; 횐; 횐; 횐; ) HANGUL SYLLABLE HOEN +D691;D691;1112 116C 11AC;D691;1112 116C 11AC; # (횑; 횑; 횑; 횑; 횑; ) HANGUL SYLLABLE HOENJ +D692;D692;1112 116C 11AD;D692;1112 116C 11AD; # (횒; 횒; 횒; 횒; 횒; ) HANGUL SYLLABLE HOENH +D693;D693;1112 116C 11AE;D693;1112 116C 11AE; # (횓; 횓; 횓; 횓; 횓; ) HANGUL SYLLABLE HOED +D694;D694;1112 116C 11AF;D694;1112 116C 11AF; # (횔; 횔; 횔; 횔; 횔; ) HANGUL SYLLABLE HOEL +D695;D695;1112 116C 11B0;D695;1112 116C 11B0; # (횕; 횕; 횕; 횕; 횕; ) HANGUL SYLLABLE HOELG +D696;D696;1112 116C 11B1;D696;1112 116C 11B1; # (횖; 횖; 횖; 횖; 횖; ) HANGUL SYLLABLE HOELM +D697;D697;1112 116C 11B2;D697;1112 116C 11B2; # (횗; 횗; 횗; 횗; 횗; ) HANGUL SYLLABLE HOELB +D698;D698;1112 116C 11B3;D698;1112 116C 11B3; # (횘; 횘; 횘; 횘; 횘; ) HANGUL SYLLABLE HOELS +D699;D699;1112 116C 11B4;D699;1112 116C 11B4; # (횙; 횙; 횙; 횙; 횙; ) HANGUL SYLLABLE HOELT +D69A;D69A;1112 116C 11B5;D69A;1112 116C 11B5; # (횚; 횚; 횚; 횚; 횚; ) HANGUL SYLLABLE HOELP +D69B;D69B;1112 116C 11B6;D69B;1112 116C 11B6; # (횛; 횛; 횛; 횛; 횛; ) HANGUL SYLLABLE HOELH +D69C;D69C;1112 116C 11B7;D69C;1112 116C 11B7; # (횜; 횜; 횜; 횜; 횜; ) HANGUL SYLLABLE HOEM +D69D;D69D;1112 116C 11B8;D69D;1112 116C 11B8; # (횝; 횝; 횝; 횝; 횝; ) HANGUL SYLLABLE HOEB +D69E;D69E;1112 116C 11B9;D69E;1112 116C 11B9; # (횞; 횞; 횞; 횞; 횞; ) HANGUL SYLLABLE HOEBS +D69F;D69F;1112 116C 11BA;D69F;1112 116C 11BA; # (횟; 횟; 횟; 횟; 횟; ) HANGUL SYLLABLE HOES +D6A0;D6A0;1112 116C 11BB;D6A0;1112 116C 11BB; # (횠; 횠; 횠; 횠; 횠; ) HANGUL SYLLABLE HOESS +D6A1;D6A1;1112 116C 11BC;D6A1;1112 116C 11BC; # (횡; 횡; 횡; 횡; 횡; ) HANGUL SYLLABLE HOENG +D6A2;D6A2;1112 116C 11BD;D6A2;1112 116C 11BD; # (횢; 횢; 횢; 횢; 횢; ) HANGUL SYLLABLE HOEJ +D6A3;D6A3;1112 116C 11BE;D6A3;1112 116C 11BE; # (횣; 횣; 횣; 횣; 횣; ) HANGUL SYLLABLE HOEC +D6A4;D6A4;1112 116C 11BF;D6A4;1112 116C 11BF; # (횤; 횤; 횤; 횤; 횤; ) HANGUL SYLLABLE HOEK +D6A5;D6A5;1112 116C 11C0;D6A5;1112 116C 11C0; # (횥; 횥; 횥; 횥; 횥; ) HANGUL SYLLABLE HOET +D6A6;D6A6;1112 116C 11C1;D6A6;1112 116C 11C1; # (횦; 횦; 횦; 횦; 횦; ) HANGUL SYLLABLE HOEP +D6A7;D6A7;1112 116C 11C2;D6A7;1112 116C 11C2; # (횧; 횧; 횧; 횧; 횧; ) HANGUL SYLLABLE HOEH +D6A8;D6A8;1112 116D;D6A8;1112 116D; # (효; 효; 효; 효; 효; ) HANGUL SYLLABLE HYO +D6A9;D6A9;1112 116D 11A8;D6A9;1112 116D 11A8; # (횩; 횩; 횩; 횩; 횩; ) HANGUL SYLLABLE HYOG +D6AA;D6AA;1112 116D 11A9;D6AA;1112 116D 11A9; # (횪; 횪; 횪; 횪; 횪; ) HANGUL SYLLABLE HYOGG +D6AB;D6AB;1112 116D 11AA;D6AB;1112 116D 11AA; # (횫; 횫; 횫; 횫; 횫; ) HANGUL SYLLABLE HYOGS +D6AC;D6AC;1112 116D 11AB;D6AC;1112 116D 11AB; # (횬; 횬; 횬; 횬; 횬; ) HANGUL SYLLABLE HYON +D6AD;D6AD;1112 116D 11AC;D6AD;1112 116D 11AC; # (횭; 횭; 횭; 횭; 횭; ) HANGUL SYLLABLE HYONJ +D6AE;D6AE;1112 116D 11AD;D6AE;1112 116D 11AD; # (횮; 횮; 횮; 횮; 횮; ) HANGUL SYLLABLE HYONH +D6AF;D6AF;1112 116D 11AE;D6AF;1112 116D 11AE; # (횯; 횯; 횯; 횯; 횯; ) HANGUL SYLLABLE HYOD +D6B0;D6B0;1112 116D 11AF;D6B0;1112 116D 11AF; # (횰; 횰; 횰; 횰; 횰; ) HANGUL SYLLABLE HYOL +D6B1;D6B1;1112 116D 11B0;D6B1;1112 116D 11B0; # (횱; 횱; 횱; 횱; 횱; ) HANGUL SYLLABLE HYOLG +D6B2;D6B2;1112 116D 11B1;D6B2;1112 116D 11B1; # (횲; 횲; 횲; 횲; 횲; ) HANGUL SYLLABLE HYOLM +D6B3;D6B3;1112 116D 11B2;D6B3;1112 116D 11B2; # (횳; 횳; 횳; 횳; 횳; ) HANGUL SYLLABLE HYOLB +D6B4;D6B4;1112 116D 11B3;D6B4;1112 116D 11B3; # (횴; 횴; 횴; 횴; 횴; ) HANGUL SYLLABLE HYOLS +D6B5;D6B5;1112 116D 11B4;D6B5;1112 116D 11B4; # (횵; 횵; 횵; 횵; 횵; ) HANGUL SYLLABLE HYOLT +D6B6;D6B6;1112 116D 11B5;D6B6;1112 116D 11B5; # (횶; 횶; 횶; 횶; 횶; ) HANGUL SYLLABLE HYOLP +D6B7;D6B7;1112 116D 11B6;D6B7;1112 116D 11B6; # (횷; 횷; 횷; 횷; 횷; ) HANGUL SYLLABLE HYOLH +D6B8;D6B8;1112 116D 11B7;D6B8;1112 116D 11B7; # (횸; 횸; 횸; 횸; 횸; ) HANGUL SYLLABLE HYOM +D6B9;D6B9;1112 116D 11B8;D6B9;1112 116D 11B8; # (횹; 횹; 횹; 횹; 횹; ) HANGUL SYLLABLE HYOB +D6BA;D6BA;1112 116D 11B9;D6BA;1112 116D 11B9; # (횺; 횺; 횺; 횺; 횺; ) HANGUL SYLLABLE HYOBS +D6BB;D6BB;1112 116D 11BA;D6BB;1112 116D 11BA; # (횻; 횻; 횻; 횻; 횻; ) HANGUL SYLLABLE HYOS +D6BC;D6BC;1112 116D 11BB;D6BC;1112 116D 11BB; # (횼; 횼; 횼; 횼; 횼; ) HANGUL SYLLABLE HYOSS +D6BD;D6BD;1112 116D 11BC;D6BD;1112 116D 11BC; # (횽; 횽; 횽; 횽; 횽; ) HANGUL SYLLABLE HYONG +D6BE;D6BE;1112 116D 11BD;D6BE;1112 116D 11BD; # (횾; 횾; 횾; 횾; 횾; ) HANGUL SYLLABLE HYOJ +D6BF;D6BF;1112 116D 11BE;D6BF;1112 116D 11BE; # (횿; 횿; 횿; 횿; 횿; ) HANGUL SYLLABLE HYOC +D6C0;D6C0;1112 116D 11BF;D6C0;1112 116D 11BF; # (훀; 훀; 훀; 훀; 훀; ) HANGUL SYLLABLE HYOK +D6C1;D6C1;1112 116D 11C0;D6C1;1112 116D 11C0; # (훁; 훁; 훁; 훁; 훁; ) HANGUL SYLLABLE HYOT +D6C2;D6C2;1112 116D 11C1;D6C2;1112 116D 11C1; # (훂; 훂; 훂; 훂; 훂; ) HANGUL SYLLABLE HYOP +D6C3;D6C3;1112 116D 11C2;D6C3;1112 116D 11C2; # (훃; 훃; 훃; 훃; 훃; ) HANGUL SYLLABLE HYOH +D6C4;D6C4;1112 116E;D6C4;1112 116E; # (후; 후; 후; 후; 후; ) HANGUL SYLLABLE HU +D6C5;D6C5;1112 116E 11A8;D6C5;1112 116E 11A8; # (훅; 훅; 훅; 훅; 훅; ) HANGUL SYLLABLE HUG +D6C6;D6C6;1112 116E 11A9;D6C6;1112 116E 11A9; # (훆; 훆; 훆; 훆; 훆; ) HANGUL SYLLABLE HUGG +D6C7;D6C7;1112 116E 11AA;D6C7;1112 116E 11AA; # (훇; 훇; 훇; 훇; 훇; ) HANGUL SYLLABLE HUGS +D6C8;D6C8;1112 116E 11AB;D6C8;1112 116E 11AB; # (훈; 훈; 훈; 훈; 훈; ) HANGUL SYLLABLE HUN +D6C9;D6C9;1112 116E 11AC;D6C9;1112 116E 11AC; # (훉; 훉; 훉; 훉; 훉; ) HANGUL SYLLABLE HUNJ +D6CA;D6CA;1112 116E 11AD;D6CA;1112 116E 11AD; # (훊; 훊; 훊; 훊; 훊; ) HANGUL SYLLABLE HUNH +D6CB;D6CB;1112 116E 11AE;D6CB;1112 116E 11AE; # (훋; 훋; 훋; 훋; 훋; ) HANGUL SYLLABLE HUD +D6CC;D6CC;1112 116E 11AF;D6CC;1112 116E 11AF; # (훌; 훌; 훌; 훌; 훌; ) HANGUL SYLLABLE HUL +D6CD;D6CD;1112 116E 11B0;D6CD;1112 116E 11B0; # (훍; 훍; 훍; 훍; 훍; ) HANGUL SYLLABLE HULG +D6CE;D6CE;1112 116E 11B1;D6CE;1112 116E 11B1; # (훎; 훎; 훎; 훎; 훎; ) HANGUL SYLLABLE HULM +D6CF;D6CF;1112 116E 11B2;D6CF;1112 116E 11B2; # (훏; 훏; 훏; 훏; 훏; ) HANGUL SYLLABLE HULB +D6D0;D6D0;1112 116E 11B3;D6D0;1112 116E 11B3; # (훐; 훐; 훐; 훐; 훐; ) HANGUL SYLLABLE HULS +D6D1;D6D1;1112 116E 11B4;D6D1;1112 116E 11B4; # (훑; 훑; 훑; 훑; 훑; ) HANGUL SYLLABLE HULT +D6D2;D6D2;1112 116E 11B5;D6D2;1112 116E 11B5; # (훒; 훒; 훒; 훒; 훒; ) HANGUL SYLLABLE HULP +D6D3;D6D3;1112 116E 11B6;D6D3;1112 116E 11B6; # (훓; 훓; 훓; 훓; 훓; ) HANGUL SYLLABLE HULH +D6D4;D6D4;1112 116E 11B7;D6D4;1112 116E 11B7; # (훔; 훔; 훔; 훔; 훔; ) HANGUL SYLLABLE HUM +D6D5;D6D5;1112 116E 11B8;D6D5;1112 116E 11B8; # (훕; 훕; 훕; 훕; 훕; ) HANGUL SYLLABLE HUB +D6D6;D6D6;1112 116E 11B9;D6D6;1112 116E 11B9; # (훖; 훖; 훖; 훖; 훖; ) HANGUL SYLLABLE HUBS +D6D7;D6D7;1112 116E 11BA;D6D7;1112 116E 11BA; # (훗; 훗; 훗; 훗; 훗; ) HANGUL SYLLABLE HUS +D6D8;D6D8;1112 116E 11BB;D6D8;1112 116E 11BB; # (훘; 훘; 훘; 훘; 훘; ) HANGUL SYLLABLE HUSS +D6D9;D6D9;1112 116E 11BC;D6D9;1112 116E 11BC; # (훙; 훙; 훙; 훙; 훙; ) HANGUL SYLLABLE HUNG +D6DA;D6DA;1112 116E 11BD;D6DA;1112 116E 11BD; # (훚; 훚; 훚; 훚; 훚; ) HANGUL SYLLABLE HUJ +D6DB;D6DB;1112 116E 11BE;D6DB;1112 116E 11BE; # (훛; 훛; 훛; 훛; 훛; ) HANGUL SYLLABLE HUC +D6DC;D6DC;1112 116E 11BF;D6DC;1112 116E 11BF; # (훜; 훜; 훜; 훜; 훜; ) HANGUL SYLLABLE HUK +D6DD;D6DD;1112 116E 11C0;D6DD;1112 116E 11C0; # (훝; 훝; 훝; 훝; 훝; ) HANGUL SYLLABLE HUT +D6DE;D6DE;1112 116E 11C1;D6DE;1112 116E 11C1; # (훞; 훞; 훞; 훞; 훞; ) HANGUL SYLLABLE HUP +D6DF;D6DF;1112 116E 11C2;D6DF;1112 116E 11C2; # (훟; 훟; 훟; 훟; 훟; ) HANGUL SYLLABLE HUH +D6E0;D6E0;1112 116F;D6E0;1112 116F; # (훠; 훠; 훠; 훠; 훠; ) HANGUL SYLLABLE HWEO +D6E1;D6E1;1112 116F 11A8;D6E1;1112 116F 11A8; # (훡; 훡; 훡; 훡; 훡; ) HANGUL SYLLABLE HWEOG +D6E2;D6E2;1112 116F 11A9;D6E2;1112 116F 11A9; # (훢; 훢; 훢; 훢; 훢; ) HANGUL SYLLABLE HWEOGG +D6E3;D6E3;1112 116F 11AA;D6E3;1112 116F 11AA; # (훣; 훣; 훣; 훣; 훣; ) HANGUL SYLLABLE HWEOGS +D6E4;D6E4;1112 116F 11AB;D6E4;1112 116F 11AB; # (훤; 훤; 훤; 훤; 훤; ) HANGUL SYLLABLE HWEON +D6E5;D6E5;1112 116F 11AC;D6E5;1112 116F 11AC; # (훥; 훥; 훥; 훥; 훥; ) HANGUL SYLLABLE HWEONJ +D6E6;D6E6;1112 116F 11AD;D6E6;1112 116F 11AD; # (훦; 훦; 훦; 훦; 훦; ) HANGUL SYLLABLE HWEONH +D6E7;D6E7;1112 116F 11AE;D6E7;1112 116F 11AE; # (훧; 훧; 훧; 훧; 훧; ) HANGUL SYLLABLE HWEOD +D6E8;D6E8;1112 116F 11AF;D6E8;1112 116F 11AF; # (훨; 훨; 훨; 훨; 훨; ) HANGUL SYLLABLE HWEOL +D6E9;D6E9;1112 116F 11B0;D6E9;1112 116F 11B0; # (훩; 훩; 훩; 훩; 훩; ) HANGUL SYLLABLE HWEOLG +D6EA;D6EA;1112 116F 11B1;D6EA;1112 116F 11B1; # (훪; 훪; 훪; 훪; 훪; ) HANGUL SYLLABLE HWEOLM +D6EB;D6EB;1112 116F 11B2;D6EB;1112 116F 11B2; # (훫; 훫; 훫; 훫; 훫; ) HANGUL SYLLABLE HWEOLB +D6EC;D6EC;1112 116F 11B3;D6EC;1112 116F 11B3; # (훬; 훬; 훬; 훬; 훬; ) HANGUL SYLLABLE HWEOLS +D6ED;D6ED;1112 116F 11B4;D6ED;1112 116F 11B4; # (훭; 훭; 훭; 훭; 훭; ) HANGUL SYLLABLE HWEOLT +D6EE;D6EE;1112 116F 11B5;D6EE;1112 116F 11B5; # (훮; 훮; 훮; 훮; 훮; ) HANGUL SYLLABLE HWEOLP +D6EF;D6EF;1112 116F 11B6;D6EF;1112 116F 11B6; # (훯; 훯; 훯; 훯; 훯; ) HANGUL SYLLABLE HWEOLH +D6F0;D6F0;1112 116F 11B7;D6F0;1112 116F 11B7; # (훰; 훰; 훰; 훰; 훰; ) HANGUL SYLLABLE HWEOM +D6F1;D6F1;1112 116F 11B8;D6F1;1112 116F 11B8; # (훱; 훱; 훱; 훱; 훱; ) HANGUL SYLLABLE HWEOB +D6F2;D6F2;1112 116F 11B9;D6F2;1112 116F 11B9; # (훲; 훲; 훲; 훲; 훲; ) HANGUL SYLLABLE HWEOBS +D6F3;D6F3;1112 116F 11BA;D6F3;1112 116F 11BA; # (훳; 훳; 훳; 훳; 훳; ) HANGUL SYLLABLE HWEOS +D6F4;D6F4;1112 116F 11BB;D6F4;1112 116F 11BB; # (훴; 훴; 훴; 훴; 훴; ) HANGUL SYLLABLE HWEOSS +D6F5;D6F5;1112 116F 11BC;D6F5;1112 116F 11BC; # (훵; 훵; 훵; 훵; 훵; ) HANGUL SYLLABLE HWEONG +D6F6;D6F6;1112 116F 11BD;D6F6;1112 116F 11BD; # (훶; 훶; 훶; 훶; 훶; ) HANGUL SYLLABLE HWEOJ +D6F7;D6F7;1112 116F 11BE;D6F7;1112 116F 11BE; # (훷; 훷; 훷; 훷; 훷; ) HANGUL SYLLABLE HWEOC +D6F8;D6F8;1112 116F 11BF;D6F8;1112 116F 11BF; # (훸; 훸; 훸; 훸; 훸; ) HANGUL SYLLABLE HWEOK +D6F9;D6F9;1112 116F 11C0;D6F9;1112 116F 11C0; # (훹; 훹; 훹; 훹; 훹; ) HANGUL SYLLABLE HWEOT +D6FA;D6FA;1112 116F 11C1;D6FA;1112 116F 11C1; # (훺; 훺; 훺; 훺; 훺; ) HANGUL SYLLABLE HWEOP +D6FB;D6FB;1112 116F 11C2;D6FB;1112 116F 11C2; # (훻; 훻; 훻; 훻; 훻; ) HANGUL SYLLABLE HWEOH +D6FC;D6FC;1112 1170;D6FC;1112 1170; # (훼; 훼; 훼; 훼; 훼; ) HANGUL SYLLABLE HWE +D6FD;D6FD;1112 1170 11A8;D6FD;1112 1170 11A8; # (훽; 훽; 훽; 훽; 훽; ) HANGUL SYLLABLE HWEG +D6FE;D6FE;1112 1170 11A9;D6FE;1112 1170 11A9; # (훾; 훾; 훾; 훾; 훾; ) HANGUL SYLLABLE HWEGG +D6FF;D6FF;1112 1170 11AA;D6FF;1112 1170 11AA; # (훿; 훿; 훿; 훿; 훿; ) HANGUL SYLLABLE HWEGS +D700;D700;1112 1170 11AB;D700;1112 1170 11AB; # (휀; 휀; 휀; 휀; 휀; ) HANGUL SYLLABLE HWEN +D701;D701;1112 1170 11AC;D701;1112 1170 11AC; # (휁; 휁; 휁; 휁; 휁; ) HANGUL SYLLABLE HWENJ +D702;D702;1112 1170 11AD;D702;1112 1170 11AD; # (휂; 휂; 휂; 휂; 휂; ) HANGUL SYLLABLE HWENH +D703;D703;1112 1170 11AE;D703;1112 1170 11AE; # (휃; 휃; 휃; 휃; 휃; ) HANGUL SYLLABLE HWED +D704;D704;1112 1170 11AF;D704;1112 1170 11AF; # (휄; 휄; 휄; 휄; 휄; ) HANGUL SYLLABLE HWEL +D705;D705;1112 1170 11B0;D705;1112 1170 11B0; # (휅; 휅; 휅; 휅; 휅; ) HANGUL SYLLABLE HWELG +D706;D706;1112 1170 11B1;D706;1112 1170 11B1; # (휆; 휆; 휆; 휆; 휆; ) HANGUL SYLLABLE HWELM +D707;D707;1112 1170 11B2;D707;1112 1170 11B2; # (휇; 휇; 휇; 휇; 휇; ) HANGUL SYLLABLE HWELB +D708;D708;1112 1170 11B3;D708;1112 1170 11B3; # (휈; 휈; 휈; 휈; 휈; ) HANGUL SYLLABLE HWELS +D709;D709;1112 1170 11B4;D709;1112 1170 11B4; # (휉; 휉; 휉; 휉; 휉; ) HANGUL SYLLABLE HWELT +D70A;D70A;1112 1170 11B5;D70A;1112 1170 11B5; # (휊; 휊; 휊; 휊; 휊; ) HANGUL SYLLABLE HWELP +D70B;D70B;1112 1170 11B6;D70B;1112 1170 11B6; # (휋; 휋; 휋; 휋; 휋; ) HANGUL SYLLABLE HWELH +D70C;D70C;1112 1170 11B7;D70C;1112 1170 11B7; # (휌; 휌; 휌; 휌; 휌; ) HANGUL SYLLABLE HWEM +D70D;D70D;1112 1170 11B8;D70D;1112 1170 11B8; # (휍; 휍; 휍; 휍; 휍; ) HANGUL SYLLABLE HWEB +D70E;D70E;1112 1170 11B9;D70E;1112 1170 11B9; # (휎; 휎; 휎; 휎; 휎; ) HANGUL SYLLABLE HWEBS +D70F;D70F;1112 1170 11BA;D70F;1112 1170 11BA; # (휏; 휏; 휏; 휏; 휏; ) HANGUL SYLLABLE HWES +D710;D710;1112 1170 11BB;D710;1112 1170 11BB; # (휐; 휐; 휐; 휐; 휐; ) HANGUL SYLLABLE HWESS +D711;D711;1112 1170 11BC;D711;1112 1170 11BC; # (휑; 휑; 휑; 휑; 휑; ) HANGUL SYLLABLE HWENG +D712;D712;1112 1170 11BD;D712;1112 1170 11BD; # (휒; 휒; 휒; 휒; 휒; ) HANGUL SYLLABLE HWEJ +D713;D713;1112 1170 11BE;D713;1112 1170 11BE; # (휓; 휓; 휓; 휓; 휓; ) HANGUL SYLLABLE HWEC +D714;D714;1112 1170 11BF;D714;1112 1170 11BF; # (휔; 휔; 휔; 휔; 휔; ) HANGUL SYLLABLE HWEK +D715;D715;1112 1170 11C0;D715;1112 1170 11C0; # (휕; 휕; 휕; 휕; 휕; ) HANGUL SYLLABLE HWET +D716;D716;1112 1170 11C1;D716;1112 1170 11C1; # (휖; 휖; 휖; 휖; 휖; ) HANGUL SYLLABLE HWEP +D717;D717;1112 1170 11C2;D717;1112 1170 11C2; # (휗; 휗; 휗; 휗; 휗; ) HANGUL SYLLABLE HWEH +D718;D718;1112 1171;D718;1112 1171; # (휘; 휘; 휘; 휘; 휘; ) HANGUL SYLLABLE HWI +D719;D719;1112 1171 11A8;D719;1112 1171 11A8; # (휙; 휙; 휙; 휙; 휙; ) HANGUL SYLLABLE HWIG +D71A;D71A;1112 1171 11A9;D71A;1112 1171 11A9; # (휚; 휚; 휚; 휚; 휚; ) HANGUL SYLLABLE HWIGG +D71B;D71B;1112 1171 11AA;D71B;1112 1171 11AA; # (휛; 휛; 휛; 휛; 휛; ) HANGUL SYLLABLE HWIGS +D71C;D71C;1112 1171 11AB;D71C;1112 1171 11AB; # (휜; 휜; 휜; 휜; 휜; ) HANGUL SYLLABLE HWIN +D71D;D71D;1112 1171 11AC;D71D;1112 1171 11AC; # (휝; 휝; 휝; 휝; 휝; ) HANGUL SYLLABLE HWINJ +D71E;D71E;1112 1171 11AD;D71E;1112 1171 11AD; # (휞; 휞; 휞; 휞; 휞; ) HANGUL SYLLABLE HWINH +D71F;D71F;1112 1171 11AE;D71F;1112 1171 11AE; # (휟; 휟; 휟; 휟; 휟; ) HANGUL SYLLABLE HWID +D720;D720;1112 1171 11AF;D720;1112 1171 11AF; # (휠; 휠; 휠; 휠; 휠; ) HANGUL SYLLABLE HWIL +D721;D721;1112 1171 11B0;D721;1112 1171 11B0; # (휡; 휡; 휡; 휡; 휡; ) HANGUL SYLLABLE HWILG +D722;D722;1112 1171 11B1;D722;1112 1171 11B1; # (휢; 휢; 휢; 휢; 휢; ) HANGUL SYLLABLE HWILM +D723;D723;1112 1171 11B2;D723;1112 1171 11B2; # (휣; 휣; 휣; 휣; 휣; ) HANGUL SYLLABLE HWILB +D724;D724;1112 1171 11B3;D724;1112 1171 11B3; # (휤; 휤; 휤; 휤; 휤; ) HANGUL SYLLABLE HWILS +D725;D725;1112 1171 11B4;D725;1112 1171 11B4; # (휥; 휥; 휥; 휥; 휥; ) HANGUL SYLLABLE HWILT +D726;D726;1112 1171 11B5;D726;1112 1171 11B5; # (휦; 휦; 휦; 휦; 휦; ) HANGUL SYLLABLE HWILP +D727;D727;1112 1171 11B6;D727;1112 1171 11B6; # (휧; 휧; 휧; 휧; 휧; ) HANGUL SYLLABLE HWILH +D728;D728;1112 1171 11B7;D728;1112 1171 11B7; # (휨; 휨; 휨; 휨; 휨; ) HANGUL SYLLABLE HWIM +D729;D729;1112 1171 11B8;D729;1112 1171 11B8; # (휩; 휩; 휩; 휩; 휩; ) HANGUL SYLLABLE HWIB +D72A;D72A;1112 1171 11B9;D72A;1112 1171 11B9; # (휪; 휪; 휪; 휪; 휪; ) HANGUL SYLLABLE HWIBS +D72B;D72B;1112 1171 11BA;D72B;1112 1171 11BA; # (휫; 휫; 휫; 휫; 휫; ) HANGUL SYLLABLE HWIS +D72C;D72C;1112 1171 11BB;D72C;1112 1171 11BB; # (휬; 휬; 휬; 휬; 휬; ) HANGUL SYLLABLE HWISS +D72D;D72D;1112 1171 11BC;D72D;1112 1171 11BC; # (휭; 휭; 휭; 휭; 휭; ) HANGUL SYLLABLE HWING +D72E;D72E;1112 1171 11BD;D72E;1112 1171 11BD; # (휮; 휮; 휮; 휮; 휮; ) HANGUL SYLLABLE HWIJ +D72F;D72F;1112 1171 11BE;D72F;1112 1171 11BE; # (휯; 휯; 휯; 휯; 휯; ) HANGUL SYLLABLE HWIC +D730;D730;1112 1171 11BF;D730;1112 1171 11BF; # (휰; 휰; 휰; 휰; 휰; ) HANGUL SYLLABLE HWIK +D731;D731;1112 1171 11C0;D731;1112 1171 11C0; # (휱; 휱; 휱; 휱; 휱; ) HANGUL SYLLABLE HWIT +D732;D732;1112 1171 11C1;D732;1112 1171 11C1; # (휲; 휲; 휲; 휲; 휲; ) HANGUL SYLLABLE HWIP +D733;D733;1112 1171 11C2;D733;1112 1171 11C2; # (휳; 휳; 휳; 휳; 휳; ) HANGUL SYLLABLE HWIH +D734;D734;1112 1172;D734;1112 1172; # (휴; 휴; 휴; 휴; 휴; ) HANGUL SYLLABLE HYU +D735;D735;1112 1172 11A8;D735;1112 1172 11A8; # (휵; 휵; 휵; 휵; 휵; ) HANGUL SYLLABLE HYUG +D736;D736;1112 1172 11A9;D736;1112 1172 11A9; # (휶; 휶; 휶; 휶; 휶; ) HANGUL SYLLABLE HYUGG +D737;D737;1112 1172 11AA;D737;1112 1172 11AA; # (휷; 휷; 휷; 휷; 휷; ) HANGUL SYLLABLE HYUGS +D738;D738;1112 1172 11AB;D738;1112 1172 11AB; # (휸; 휸; 휸; 휸; 휸; ) HANGUL SYLLABLE HYUN +D739;D739;1112 1172 11AC;D739;1112 1172 11AC; # (휹; 휹; 휹; 휹; 휹; ) HANGUL SYLLABLE HYUNJ +D73A;D73A;1112 1172 11AD;D73A;1112 1172 11AD; # (휺; 휺; 휺; 휺; 휺; ) HANGUL SYLLABLE HYUNH +D73B;D73B;1112 1172 11AE;D73B;1112 1172 11AE; # (휻; 휻; 휻; 휻; 휻; ) HANGUL SYLLABLE HYUD +D73C;D73C;1112 1172 11AF;D73C;1112 1172 11AF; # (휼; 휼; 휼; 휼; 휼; ) HANGUL SYLLABLE HYUL +D73D;D73D;1112 1172 11B0;D73D;1112 1172 11B0; # (휽; 휽; 휽; 휽; 휽; ) HANGUL SYLLABLE HYULG +D73E;D73E;1112 1172 11B1;D73E;1112 1172 11B1; # (휾; 휾; 휾; 휾; 휾; ) HANGUL SYLLABLE HYULM +D73F;D73F;1112 1172 11B2;D73F;1112 1172 11B2; # (휿; 휿; 휿; 휿; 휿; ) HANGUL SYLLABLE HYULB +D740;D740;1112 1172 11B3;D740;1112 1172 11B3; # (흀; 흀; 흀; 흀; 흀; ) HANGUL SYLLABLE HYULS +D741;D741;1112 1172 11B4;D741;1112 1172 11B4; # (흁; 흁; 흁; 흁; 흁; ) HANGUL SYLLABLE HYULT +D742;D742;1112 1172 11B5;D742;1112 1172 11B5; # (흂; 흂; 흂; 흂; 흂; ) HANGUL SYLLABLE HYULP +D743;D743;1112 1172 11B6;D743;1112 1172 11B6; # (흃; 흃; 흃; 흃; 흃; ) HANGUL SYLLABLE HYULH +D744;D744;1112 1172 11B7;D744;1112 1172 11B7; # (흄; 흄; 흄; 흄; 흄; ) HANGUL SYLLABLE HYUM +D745;D745;1112 1172 11B8;D745;1112 1172 11B8; # (흅; 흅; 흅; 흅; 흅; ) HANGUL SYLLABLE HYUB +D746;D746;1112 1172 11B9;D746;1112 1172 11B9; # (흆; 흆; 흆; 흆; 흆; ) HANGUL SYLLABLE HYUBS +D747;D747;1112 1172 11BA;D747;1112 1172 11BA; # (흇; 흇; 흇; 흇; 흇; ) HANGUL SYLLABLE HYUS +D748;D748;1112 1172 11BB;D748;1112 1172 11BB; # (흈; 흈; 흈; 흈; 흈; ) HANGUL SYLLABLE HYUSS +D749;D749;1112 1172 11BC;D749;1112 1172 11BC; # (흉; 흉; 흉; 흉; 흉; ) HANGUL SYLLABLE HYUNG +D74A;D74A;1112 1172 11BD;D74A;1112 1172 11BD; # (흊; 흊; 흊; 흊; 흊; ) HANGUL SYLLABLE HYUJ +D74B;D74B;1112 1172 11BE;D74B;1112 1172 11BE; # (흋; 흋; 흋; 흋; 흋; ) HANGUL SYLLABLE HYUC +D74C;D74C;1112 1172 11BF;D74C;1112 1172 11BF; # (흌; 흌; 흌; 흌; 흌; ) HANGUL SYLLABLE HYUK +D74D;D74D;1112 1172 11C0;D74D;1112 1172 11C0; # (흍; 흍; 흍; 흍; 흍; ) HANGUL SYLLABLE HYUT +D74E;D74E;1112 1172 11C1;D74E;1112 1172 11C1; # (흎; 흎; 흎; 흎; 흎; ) HANGUL SYLLABLE HYUP +D74F;D74F;1112 1172 11C2;D74F;1112 1172 11C2; # (흏; 흏; 흏; 흏; 흏; ) HANGUL SYLLABLE HYUH +D750;D750;1112 1173;D750;1112 1173; # (흐; 흐; 흐; 흐; 흐; ) HANGUL SYLLABLE HEU +D751;D751;1112 1173 11A8;D751;1112 1173 11A8; # (흑; 흑; 흑; 흑; 흑; ) HANGUL SYLLABLE HEUG +D752;D752;1112 1173 11A9;D752;1112 1173 11A9; # (흒; 흒; 흒; 흒; 흒; ) HANGUL SYLLABLE HEUGG +D753;D753;1112 1173 11AA;D753;1112 1173 11AA; # (흓; 흓; 흓; 흓; 흓; ) HANGUL SYLLABLE HEUGS +D754;D754;1112 1173 11AB;D754;1112 1173 11AB; # (흔; 흔; 흔; 흔; 흔; ) HANGUL SYLLABLE HEUN +D755;D755;1112 1173 11AC;D755;1112 1173 11AC; # (흕; 흕; 흕; 흕; 흕; ) HANGUL SYLLABLE HEUNJ +D756;D756;1112 1173 11AD;D756;1112 1173 11AD; # (흖; 흖; 흖; 흖; 흖; ) HANGUL SYLLABLE HEUNH +D757;D757;1112 1173 11AE;D757;1112 1173 11AE; # (흗; 흗; 흗; 흗; 흗; ) HANGUL SYLLABLE HEUD +D758;D758;1112 1173 11AF;D758;1112 1173 11AF; # (흘; 흘; 흘; 흘; 흘; ) HANGUL SYLLABLE HEUL +D759;D759;1112 1173 11B0;D759;1112 1173 11B0; # (흙; 흙; 흙; 흙; 흙; ) HANGUL SYLLABLE HEULG +D75A;D75A;1112 1173 11B1;D75A;1112 1173 11B1; # (흚; 흚; 흚; 흚; 흚; ) HANGUL SYLLABLE HEULM +D75B;D75B;1112 1173 11B2;D75B;1112 1173 11B2; # (흛; 흛; 흛; 흛; 흛; ) HANGUL SYLLABLE HEULB +D75C;D75C;1112 1173 11B3;D75C;1112 1173 11B3; # (흜; 흜; 흜; 흜; 흜; ) HANGUL SYLLABLE HEULS +D75D;D75D;1112 1173 11B4;D75D;1112 1173 11B4; # (흝; 흝; 흝; 흝; 흝; ) HANGUL SYLLABLE HEULT +D75E;D75E;1112 1173 11B5;D75E;1112 1173 11B5; # (흞; 흞; 흞; 흞; 흞; ) HANGUL SYLLABLE HEULP +D75F;D75F;1112 1173 11B6;D75F;1112 1173 11B6; # (흟; 흟; 흟; 흟; 흟; ) HANGUL SYLLABLE HEULH +D760;D760;1112 1173 11B7;D760;1112 1173 11B7; # (흠; 흠; 흠; 흠; 흠; ) HANGUL SYLLABLE HEUM +D761;D761;1112 1173 11B8;D761;1112 1173 11B8; # (흡; 흡; 흡; 흡; 흡; ) HANGUL SYLLABLE HEUB +D762;D762;1112 1173 11B9;D762;1112 1173 11B9; # (흢; 흢; 흢; 흢; 흢; ) HANGUL SYLLABLE HEUBS +D763;D763;1112 1173 11BA;D763;1112 1173 11BA; # (흣; 흣; 흣; 흣; 흣; ) HANGUL SYLLABLE HEUS +D764;D764;1112 1173 11BB;D764;1112 1173 11BB; # (흤; 흤; 흤; 흤; 흤; ) HANGUL SYLLABLE HEUSS +D765;D765;1112 1173 11BC;D765;1112 1173 11BC; # (흥; 흥; 흥; 흥; 흥; ) HANGUL SYLLABLE HEUNG +D766;D766;1112 1173 11BD;D766;1112 1173 11BD; # (흦; 흦; 흦; 흦; 흦; ) HANGUL SYLLABLE HEUJ +D767;D767;1112 1173 11BE;D767;1112 1173 11BE; # (흧; 흧; 흧; 흧; 흧; ) HANGUL SYLLABLE HEUC +D768;D768;1112 1173 11BF;D768;1112 1173 11BF; # (흨; 흨; 흨; 흨; 흨; ) HANGUL SYLLABLE HEUK +D769;D769;1112 1173 11C0;D769;1112 1173 11C0; # (흩; 흩; 흩; 흩; 흩; ) HANGUL SYLLABLE HEUT +D76A;D76A;1112 1173 11C1;D76A;1112 1173 11C1; # (흪; 흪; 흪; 흪; 흪; ) HANGUL SYLLABLE HEUP +D76B;D76B;1112 1173 11C2;D76B;1112 1173 11C2; # (흫; 흫; 흫; 흫; 흫; ) HANGUL SYLLABLE HEUH +D76C;D76C;1112 1174;D76C;1112 1174; # (희; 희; 희; 희; 희; ) HANGUL SYLLABLE HYI +D76D;D76D;1112 1174 11A8;D76D;1112 1174 11A8; # (흭; 흭; 흭; 흭; 흭; ) HANGUL SYLLABLE HYIG +D76E;D76E;1112 1174 11A9;D76E;1112 1174 11A9; # (흮; 흮; 흮; 흮; 흮; ) HANGUL SYLLABLE HYIGG +D76F;D76F;1112 1174 11AA;D76F;1112 1174 11AA; # (흯; 흯; 흯; 흯; 흯; ) HANGUL SYLLABLE HYIGS +D770;D770;1112 1174 11AB;D770;1112 1174 11AB; # (흰; 흰; 흰; 흰; 흰; ) HANGUL SYLLABLE HYIN +D771;D771;1112 1174 11AC;D771;1112 1174 11AC; # (흱; 흱; 흱; 흱; 흱; ) HANGUL SYLLABLE HYINJ +D772;D772;1112 1174 11AD;D772;1112 1174 11AD; # (흲; 흲; 흲; 흲; 흲; ) HANGUL SYLLABLE HYINH +D773;D773;1112 1174 11AE;D773;1112 1174 11AE; # (흳; 흳; 흳; 흳; 흳; ) HANGUL SYLLABLE HYID +D774;D774;1112 1174 11AF;D774;1112 1174 11AF; # (흴; 흴; 흴; 흴; 흴; ) HANGUL SYLLABLE HYIL +D775;D775;1112 1174 11B0;D775;1112 1174 11B0; # (흵; 흵; 흵; 흵; 흵; ) HANGUL SYLLABLE HYILG +D776;D776;1112 1174 11B1;D776;1112 1174 11B1; # (흶; 흶; 흶; 흶; 흶; ) HANGUL SYLLABLE HYILM +D777;D777;1112 1174 11B2;D777;1112 1174 11B2; # (흷; 흷; 흷; 흷; 흷; ) HANGUL SYLLABLE HYILB +D778;D778;1112 1174 11B3;D778;1112 1174 11B3; # (흸; 흸; 흸; 흸; 흸; ) HANGUL SYLLABLE HYILS +D779;D779;1112 1174 11B4;D779;1112 1174 11B4; # (흹; 흹; 흹; 흹; 흹; ) HANGUL SYLLABLE HYILT +D77A;D77A;1112 1174 11B5;D77A;1112 1174 11B5; # (흺; 흺; 흺; 흺; 흺; ) HANGUL SYLLABLE HYILP +D77B;D77B;1112 1174 11B6;D77B;1112 1174 11B6; # (흻; 흻; 흻; 흻; 흻; ) HANGUL SYLLABLE HYILH +D77C;D77C;1112 1174 11B7;D77C;1112 1174 11B7; # (흼; 흼; 흼; 흼; 흼; ) HANGUL SYLLABLE HYIM +D77D;D77D;1112 1174 11B8;D77D;1112 1174 11B8; # (흽; 흽; 흽; 흽; 흽; ) HANGUL SYLLABLE HYIB +D77E;D77E;1112 1174 11B9;D77E;1112 1174 11B9; # (흾; 흾; 흾; 흾; 흾; ) HANGUL SYLLABLE HYIBS +D77F;D77F;1112 1174 11BA;D77F;1112 1174 11BA; # (흿; 흿; 흿; 흿; 흿; ) HANGUL SYLLABLE HYIS +D780;D780;1112 1174 11BB;D780;1112 1174 11BB; # (힀; 힀; 힀; 힀; 힀; ) HANGUL SYLLABLE HYISS +D781;D781;1112 1174 11BC;D781;1112 1174 11BC; # (힁; 힁; 힁; 힁; 힁; ) HANGUL SYLLABLE HYING +D782;D782;1112 1174 11BD;D782;1112 1174 11BD; # (힂; 힂; 힂; 힂; 힂; ) HANGUL SYLLABLE HYIJ +D783;D783;1112 1174 11BE;D783;1112 1174 11BE; # (힃; 힃; 힃; 힃; 힃; ) HANGUL SYLLABLE HYIC +D784;D784;1112 1174 11BF;D784;1112 1174 11BF; # (힄; 힄; 힄; 힄; 힄; ) HANGUL SYLLABLE HYIK +D785;D785;1112 1174 11C0;D785;1112 1174 11C0; # (힅; 힅; 힅; 힅; 힅; ) HANGUL SYLLABLE HYIT +D786;D786;1112 1174 11C1;D786;1112 1174 11C1; # (힆; 힆; 힆; 힆; 힆; ) HANGUL SYLLABLE HYIP +D787;D787;1112 1174 11C2;D787;1112 1174 11C2; # (힇; 힇; 힇; 힇; 힇; ) HANGUL SYLLABLE HYIH +D788;D788;1112 1175;D788;1112 1175; # (히; 히; 히; 히; 히; ) HANGUL SYLLABLE HI +D789;D789;1112 1175 11A8;D789;1112 1175 11A8; # (힉; 힉; 힉; 힉; 힉; ) HANGUL SYLLABLE HIG +D78A;D78A;1112 1175 11A9;D78A;1112 1175 11A9; # (힊; 힊; 힊; 힊; 힊; ) HANGUL SYLLABLE HIGG +D78B;D78B;1112 1175 11AA;D78B;1112 1175 11AA; # (힋; 힋; 힋; 힋; 힋; ) HANGUL SYLLABLE HIGS +D78C;D78C;1112 1175 11AB;D78C;1112 1175 11AB; # (힌; 힌; 힌; 힌; 힌; ) HANGUL SYLLABLE HIN +D78D;D78D;1112 1175 11AC;D78D;1112 1175 11AC; # (힍; 힍; 힍; 힍; 힍; ) HANGUL SYLLABLE HINJ +D78E;D78E;1112 1175 11AD;D78E;1112 1175 11AD; # (힎; 힎; 힎; 힎; 힎; ) HANGUL SYLLABLE HINH +D78F;D78F;1112 1175 11AE;D78F;1112 1175 11AE; # (힏; 힏; 힏; 힏; 힏; ) HANGUL SYLLABLE HID +D790;D790;1112 1175 11AF;D790;1112 1175 11AF; # (힐; 힐; 힐; 힐; 힐; ) HANGUL SYLLABLE HIL +D791;D791;1112 1175 11B0;D791;1112 1175 11B0; # (힑; 힑; 힑; 힑; 힑; ) HANGUL SYLLABLE HILG +D792;D792;1112 1175 11B1;D792;1112 1175 11B1; # (힒; 힒; 힒; 힒; 힒; ) HANGUL SYLLABLE HILM +D793;D793;1112 1175 11B2;D793;1112 1175 11B2; # (힓; 힓; 힓; 힓; 힓; ) HANGUL SYLLABLE HILB +D794;D794;1112 1175 11B3;D794;1112 1175 11B3; # (힔; 힔; 힔; 힔; 힔; ) HANGUL SYLLABLE HILS +D795;D795;1112 1175 11B4;D795;1112 1175 11B4; # (힕; 힕; 힕; 힕; 힕; ) HANGUL SYLLABLE HILT +D796;D796;1112 1175 11B5;D796;1112 1175 11B5; # (힖; 힖; 힖; 힖; 힖; ) HANGUL SYLLABLE HILP +D797;D797;1112 1175 11B6;D797;1112 1175 11B6; # (힗; 힗; 힗; 힗; 힗; ) HANGUL SYLLABLE HILH +D798;D798;1112 1175 11B7;D798;1112 1175 11B7; # (힘; 힘; 힘; 힘; 힘; ) HANGUL SYLLABLE HIM +D799;D799;1112 1175 11B8;D799;1112 1175 11B8; # (힙; 힙; 힙; 힙; 힙; ) HANGUL SYLLABLE HIB +D79A;D79A;1112 1175 11B9;D79A;1112 1175 11B9; # (힚; 힚; 힚; 힚; 힚; ) HANGUL SYLLABLE HIBS +D79B;D79B;1112 1175 11BA;D79B;1112 1175 11BA; # (힛; 힛; 힛; 힛; 힛; ) HANGUL SYLLABLE HIS +D79C;D79C;1112 1175 11BB;D79C;1112 1175 11BB; # (힜; 힜; 힜; 힜; 힜; ) HANGUL SYLLABLE HISS +D79D;D79D;1112 1175 11BC;D79D;1112 1175 11BC; # (힝; 힝; 힝; 힝; 힝; ) HANGUL SYLLABLE HING +D79E;D79E;1112 1175 11BD;D79E;1112 1175 11BD; # (힞; 힞; 힞; 힞; 힞; ) HANGUL SYLLABLE HIJ +D79F;D79F;1112 1175 11BE;D79F;1112 1175 11BE; # (힟; 힟; 힟; 힟; 힟; ) HANGUL SYLLABLE HIC +D7A0;D7A0;1112 1175 11BF;D7A0;1112 1175 11BF; # (힠; 힠; 힠; 힠; 힠; ) HANGUL SYLLABLE HIK +D7A1;D7A1;1112 1175 11C0;D7A1;1112 1175 11C0; # (힡; 힡; 힡; 힡; 힡; ) HANGUL SYLLABLE HIT +D7A2;D7A2;1112 1175 11C1;D7A2;1112 1175 11C1; # (힢; 힢; 힢; 힢; 힢; ) HANGUL SYLLABLE HIP +D7A3;D7A3;1112 1175 11C2;D7A3;1112 1175 11C2; # (힣; 힣; 힣; 힣; 힣; ) HANGUL SYLLABLE HIH +F900;8C48;8C48;8C48;8C48; # (豈; 豈; 豈; 豈; 豈; ) CJK COMPATIBILITY IDEOGRAPH-F900 +F901;66F4;66F4;66F4;66F4; # (更; 更; 更; 更; 更; ) CJK COMPATIBILITY IDEOGRAPH-F901 +F902;8ECA;8ECA;8ECA;8ECA; # (車; 車; 車; 車; 車; ) CJK COMPATIBILITY IDEOGRAPH-F902 +F903;8CC8;8CC8;8CC8;8CC8; # (賈; 賈; 賈; 賈; 賈; ) CJK COMPATIBILITY IDEOGRAPH-F903 +F904;6ED1;6ED1;6ED1;6ED1; # (滑; 滑; 滑; 滑; 滑; ) CJK COMPATIBILITY IDEOGRAPH-F904 +F905;4E32;4E32;4E32;4E32; # (串; 串; 串; 串; 串; ) CJK COMPATIBILITY IDEOGRAPH-F905 +F906;53E5;53E5;53E5;53E5; # (句; 句; 句; 句; 句; ) CJK COMPATIBILITY IDEOGRAPH-F906 +F907;9F9C;9F9C;9F9C;9F9C; # (龜; 龜; 龜; 龜; 龜; ) CJK COMPATIBILITY IDEOGRAPH-F907 +F908;9F9C;9F9C;9F9C;9F9C; # (龜; 龜; 龜; 龜; 龜; ) CJK COMPATIBILITY IDEOGRAPH-F908 +F909;5951;5951;5951;5951; # (契; 契; 契; 契; 契; ) CJK COMPATIBILITY IDEOGRAPH-F909 +F90A;91D1;91D1;91D1;91D1; # (金; 金; 金; 金; 金; ) CJK COMPATIBILITY IDEOGRAPH-F90A +F90B;5587;5587;5587;5587; # (喇; 喇; 喇; 喇; 喇; ) CJK COMPATIBILITY IDEOGRAPH-F90B +F90C;5948;5948;5948;5948; # (奈; 奈; 奈; 奈; 奈; ) CJK COMPATIBILITY IDEOGRAPH-F90C +F90D;61F6;61F6;61F6;61F6; # (懶; 懶; 懶; 懶; 懶; ) CJK COMPATIBILITY IDEOGRAPH-F90D +F90E;7669;7669;7669;7669; # (癩; 癩; 癩; 癩; 癩; ) CJK COMPATIBILITY IDEOGRAPH-F90E +F90F;7F85;7F85;7F85;7F85; # (羅; 羅; 羅; 羅; 羅; ) CJK COMPATIBILITY IDEOGRAPH-F90F +F910;863F;863F;863F;863F; # (蘿; 蘿; 蘿; 蘿; 蘿; ) CJK COMPATIBILITY IDEOGRAPH-F910 +F911;87BA;87BA;87BA;87BA; # (螺; 螺; 螺; 螺; 螺; ) CJK COMPATIBILITY IDEOGRAPH-F911 +F912;88F8;88F8;88F8;88F8; # (裸; 裸; 裸; 裸; 裸; ) CJK COMPATIBILITY IDEOGRAPH-F912 +F913;908F;908F;908F;908F; # (邏; 邏; 邏; 邏; 邏; ) CJK COMPATIBILITY IDEOGRAPH-F913 +F914;6A02;6A02;6A02;6A02; # (樂; 樂; 樂; 樂; 樂; ) CJK COMPATIBILITY IDEOGRAPH-F914 +F915;6D1B;6D1B;6D1B;6D1B; # (洛; 洛; 洛; 洛; 洛; ) CJK COMPATIBILITY IDEOGRAPH-F915 +F916;70D9;70D9;70D9;70D9; # (烙; 烙; 烙; 烙; 烙; ) CJK COMPATIBILITY IDEOGRAPH-F916 +F917;73DE;73DE;73DE;73DE; # (珞; 珞; 珞; 珞; 珞; ) CJK COMPATIBILITY IDEOGRAPH-F917 +F918;843D;843D;843D;843D; # (落; 落; 落; 落; 落; ) CJK COMPATIBILITY IDEOGRAPH-F918 +F919;916A;916A;916A;916A; # (酪; 酪; 酪; 酪; 酪; ) CJK COMPATIBILITY IDEOGRAPH-F919 +F91A;99F1;99F1;99F1;99F1; # (駱; 駱; 駱; 駱; 駱; ) CJK COMPATIBILITY IDEOGRAPH-F91A +F91B;4E82;4E82;4E82;4E82; # (亂; 亂; 亂; 亂; 亂; ) CJK COMPATIBILITY IDEOGRAPH-F91B +F91C;5375;5375;5375;5375; # (卵; 卵; 卵; 卵; 卵; ) CJK COMPATIBILITY IDEOGRAPH-F91C +F91D;6B04;6B04;6B04;6B04; # (欄; 欄; 欄; 欄; 欄; ) CJK COMPATIBILITY IDEOGRAPH-F91D +F91E;721B;721B;721B;721B; # (爛; 爛; 爛; 爛; 爛; ) CJK COMPATIBILITY IDEOGRAPH-F91E +F91F;862D;862D;862D;862D; # (蘭; 蘭; 蘭; 蘭; 蘭; ) CJK COMPATIBILITY IDEOGRAPH-F91F +F920;9E1E;9E1E;9E1E;9E1E; # (鸞; 鸞; 鸞; 鸞; 鸞; ) CJK COMPATIBILITY IDEOGRAPH-F920 +F921;5D50;5D50;5D50;5D50; # (嵐; 嵐; 嵐; 嵐; 嵐; ) CJK COMPATIBILITY IDEOGRAPH-F921 +F922;6FEB;6FEB;6FEB;6FEB; # (濫; 濫; 濫; 濫; 濫; ) CJK COMPATIBILITY IDEOGRAPH-F922 +F923;85CD;85CD;85CD;85CD; # (藍; 藍; 藍; 藍; 藍; ) CJK COMPATIBILITY IDEOGRAPH-F923 +F924;8964;8964;8964;8964; # (襤; 襤; 襤; 襤; 襤; ) CJK COMPATIBILITY IDEOGRAPH-F924 +F925;62C9;62C9;62C9;62C9; # (拉; 拉; 拉; 拉; 拉; ) CJK COMPATIBILITY IDEOGRAPH-F925 +F926;81D8;81D8;81D8;81D8; # (臘; 臘; 臘; 臘; 臘; ) CJK COMPATIBILITY IDEOGRAPH-F926 +F927;881F;881F;881F;881F; # (蠟; 蠟; 蠟; 蠟; 蠟; ) CJK COMPATIBILITY IDEOGRAPH-F927 +F928;5ECA;5ECA;5ECA;5ECA; # (廊; 廊; 廊; 廊; 廊; ) CJK COMPATIBILITY IDEOGRAPH-F928 +F929;6717;6717;6717;6717; # (朗; 朗; 朗; 朗; 朗; ) CJK COMPATIBILITY IDEOGRAPH-F929 +F92A;6D6A;6D6A;6D6A;6D6A; # (浪; 浪; 浪; 浪; 浪; ) CJK COMPATIBILITY IDEOGRAPH-F92A +F92B;72FC;72FC;72FC;72FC; # (狼; 狼; 狼; 狼; 狼; ) CJK COMPATIBILITY IDEOGRAPH-F92B +F92C;90CE;90CE;90CE;90CE; # (郎; 郎; 郎; 郎; 郎; ) CJK COMPATIBILITY IDEOGRAPH-F92C +F92D;4F86;4F86;4F86;4F86; # (來; 來; 來; 來; 來; ) CJK COMPATIBILITY IDEOGRAPH-F92D +F92E;51B7;51B7;51B7;51B7; # (冷; 冷; 冷; 冷; 冷; ) CJK COMPATIBILITY IDEOGRAPH-F92E +F92F;52DE;52DE;52DE;52DE; # (勞; 勞; 勞; 勞; 勞; ) CJK COMPATIBILITY IDEOGRAPH-F92F +F930;64C4;64C4;64C4;64C4; # (擄; 擄; 擄; 擄; 擄; ) CJK COMPATIBILITY IDEOGRAPH-F930 +F931;6AD3;6AD3;6AD3;6AD3; # (櫓; 櫓; 櫓; 櫓; 櫓; ) CJK COMPATIBILITY IDEOGRAPH-F931 +F932;7210;7210;7210;7210; # (爐; 爐; 爐; 爐; 爐; ) CJK COMPATIBILITY IDEOGRAPH-F932 +F933;76E7;76E7;76E7;76E7; # (盧; 盧; 盧; 盧; 盧; ) CJK COMPATIBILITY IDEOGRAPH-F933 +F934;8001;8001;8001;8001; # (老; 老; 老; 老; 老; ) CJK COMPATIBILITY IDEOGRAPH-F934 +F935;8606;8606;8606;8606; # (蘆; 蘆; 蘆; 蘆; 蘆; ) CJK COMPATIBILITY IDEOGRAPH-F935 +F936;865C;865C;865C;865C; # (虜; 虜; 虜; 虜; 虜; ) CJK COMPATIBILITY IDEOGRAPH-F936 +F937;8DEF;8DEF;8DEF;8DEF; # (路; 路; 路; 路; 路; ) CJK COMPATIBILITY IDEOGRAPH-F937 +F938;9732;9732;9732;9732; # (露; 露; 露; 露; 露; ) CJK COMPATIBILITY IDEOGRAPH-F938 +F939;9B6F;9B6F;9B6F;9B6F; # (魯; 魯; 魯; 魯; 魯; ) CJK COMPATIBILITY IDEOGRAPH-F939 +F93A;9DFA;9DFA;9DFA;9DFA; # (鷺; 鷺; 鷺; 鷺; 鷺; ) CJK COMPATIBILITY IDEOGRAPH-F93A +F93B;788C;788C;788C;788C; # (碌; 碌; 碌; 碌; 碌; ) CJK COMPATIBILITY IDEOGRAPH-F93B +F93C;797F;797F;797F;797F; # (祿; 祿; 祿; 祿; 祿; ) CJK COMPATIBILITY IDEOGRAPH-F93C +F93D;7DA0;7DA0;7DA0;7DA0; # (綠; 綠; 綠; 綠; 綠; ) CJK COMPATIBILITY IDEOGRAPH-F93D +F93E;83C9;83C9;83C9;83C9; # (菉; 菉; 菉; 菉; 菉; ) CJK COMPATIBILITY IDEOGRAPH-F93E +F93F;9304;9304;9304;9304; # (錄; 錄; 錄; 錄; 錄; ) CJK COMPATIBILITY IDEOGRAPH-F93F +F940;9E7F;9E7F;9E7F;9E7F; # (鹿; 鹿; 鹿; 鹿; 鹿; ) CJK COMPATIBILITY IDEOGRAPH-F940 +F941;8AD6;8AD6;8AD6;8AD6; # (論; 論; 論; 論; 論; ) CJK COMPATIBILITY IDEOGRAPH-F941 +F942;58DF;58DF;58DF;58DF; # (壟; 壟; 壟; 壟; 壟; ) CJK COMPATIBILITY IDEOGRAPH-F942 +F943;5F04;5F04;5F04;5F04; # (弄; 弄; 弄; 弄; 弄; ) CJK COMPATIBILITY IDEOGRAPH-F943 +F944;7C60;7C60;7C60;7C60; # (籠; 籠; 籠; 籠; 籠; ) CJK COMPATIBILITY IDEOGRAPH-F944 +F945;807E;807E;807E;807E; # (聾; 聾; 聾; 聾; 聾; ) CJK COMPATIBILITY IDEOGRAPH-F945 +F946;7262;7262;7262;7262; # (牢; 牢; 牢; 牢; 牢; ) CJK COMPATIBILITY IDEOGRAPH-F946 +F947;78CA;78CA;78CA;78CA; # (磊; 磊; 磊; 磊; 磊; ) CJK COMPATIBILITY IDEOGRAPH-F947 +F948;8CC2;8CC2;8CC2;8CC2; # (賂; 賂; 賂; 賂; 賂; ) CJK COMPATIBILITY IDEOGRAPH-F948 +F949;96F7;96F7;96F7;96F7; # (雷; 雷; 雷; 雷; 雷; ) CJK COMPATIBILITY IDEOGRAPH-F949 +F94A;58D8;58D8;58D8;58D8; # (壘; 壘; 壘; 壘; 壘; ) CJK COMPATIBILITY IDEOGRAPH-F94A +F94B;5C62;5C62;5C62;5C62; # (屢; 屢; 屢; 屢; 屢; ) CJK COMPATIBILITY IDEOGRAPH-F94B +F94C;6A13;6A13;6A13;6A13; # (樓; 樓; 樓; 樓; 樓; ) CJK COMPATIBILITY IDEOGRAPH-F94C +F94D;6DDA;6DDA;6DDA;6DDA; # (淚; 淚; 淚; 淚; 淚; ) CJK COMPATIBILITY IDEOGRAPH-F94D +F94E;6F0F;6F0F;6F0F;6F0F; # (漏; 漏; 漏; 漏; 漏; ) CJK COMPATIBILITY IDEOGRAPH-F94E +F94F;7D2F;7D2F;7D2F;7D2F; # (累; 累; 累; 累; 累; ) CJK COMPATIBILITY IDEOGRAPH-F94F +F950;7E37;7E37;7E37;7E37; # (縷; 縷; 縷; 縷; 縷; ) CJK COMPATIBILITY IDEOGRAPH-F950 +F951;964B;964B;964B;964B; # (陋; 陋; 陋; 陋; 陋; ) CJK COMPATIBILITY IDEOGRAPH-F951 +F952;52D2;52D2;52D2;52D2; # (勒; 勒; 勒; 勒; 勒; ) CJK COMPATIBILITY IDEOGRAPH-F952 +F953;808B;808B;808B;808B; # (肋; 肋; 肋; 肋; 肋; ) CJK COMPATIBILITY IDEOGRAPH-F953 +F954;51DC;51DC;51DC;51DC; # (凜; 凜; 凜; 凜; 凜; ) CJK COMPATIBILITY IDEOGRAPH-F954 +F955;51CC;51CC;51CC;51CC; # (凌; 凌; 凌; 凌; 凌; ) CJK COMPATIBILITY IDEOGRAPH-F955 +F956;7A1C;7A1C;7A1C;7A1C; # (稜; 稜; 稜; 稜; 稜; ) CJK COMPATIBILITY IDEOGRAPH-F956 +F957;7DBE;7DBE;7DBE;7DBE; # (綾; 綾; 綾; 綾; 綾; ) CJK COMPATIBILITY IDEOGRAPH-F957 +F958;83F1;83F1;83F1;83F1; # (菱; 菱; 菱; 菱; 菱; ) CJK COMPATIBILITY IDEOGRAPH-F958 +F959;9675;9675;9675;9675; # (陵; 陵; 陵; 陵; 陵; ) CJK COMPATIBILITY IDEOGRAPH-F959 +F95A;8B80;8B80;8B80;8B80; # (讀; 讀; 讀; 讀; 讀; ) CJK COMPATIBILITY IDEOGRAPH-F95A +F95B;62CF;62CF;62CF;62CF; # (拏; 拏; 拏; 拏; 拏; ) CJK COMPATIBILITY IDEOGRAPH-F95B +F95C;6A02;6A02;6A02;6A02; # (樂; 樂; 樂; 樂; 樂; ) CJK COMPATIBILITY IDEOGRAPH-F95C +F95D;8AFE;8AFE;8AFE;8AFE; # (諾; 諾; 諾; 諾; 諾; ) CJK COMPATIBILITY IDEOGRAPH-F95D +F95E;4E39;4E39;4E39;4E39; # (丹; 丹; 丹; 丹; 丹; ) CJK COMPATIBILITY IDEOGRAPH-F95E +F95F;5BE7;5BE7;5BE7;5BE7; # (寧; 寧; 寧; 寧; 寧; ) CJK COMPATIBILITY IDEOGRAPH-F95F +F960;6012;6012;6012;6012; # (怒; 怒; 怒; 怒; 怒; ) CJK COMPATIBILITY IDEOGRAPH-F960 +F961;7387;7387;7387;7387; # (率; 率; 率; 率; 率; ) CJK COMPATIBILITY IDEOGRAPH-F961 +F962;7570;7570;7570;7570; # (異; 異; 異; 異; 異; ) CJK COMPATIBILITY IDEOGRAPH-F962 +F963;5317;5317;5317;5317; # (北; 北; 北; 北; 北; ) CJK COMPATIBILITY IDEOGRAPH-F963 +F964;78FB;78FB;78FB;78FB; # (磻; 磻; 磻; 磻; 磻; ) CJK COMPATIBILITY IDEOGRAPH-F964 +F965;4FBF;4FBF;4FBF;4FBF; # (便; 便; 便; 便; 便; ) CJK COMPATIBILITY IDEOGRAPH-F965 +F966;5FA9;5FA9;5FA9;5FA9; # (復; 復; 復; 復; 復; ) CJK COMPATIBILITY IDEOGRAPH-F966 +F967;4E0D;4E0D;4E0D;4E0D; # (不; 不; 不; 不; 不; ) CJK COMPATIBILITY IDEOGRAPH-F967 +F968;6CCC;6CCC;6CCC;6CCC; # (泌; 泌; 泌; 泌; 泌; ) CJK COMPATIBILITY IDEOGRAPH-F968 +F969;6578;6578;6578;6578; # (數; 數; 數; 數; 數; ) CJK COMPATIBILITY IDEOGRAPH-F969 +F96A;7D22;7D22;7D22;7D22; # (索; 索; 索; 索; 索; ) CJK COMPATIBILITY IDEOGRAPH-F96A +F96B;53C3;53C3;53C3;53C3; # (參; 參; 參; 參; 參; ) CJK COMPATIBILITY IDEOGRAPH-F96B +F96C;585E;585E;585E;585E; # (塞; 塞; 塞; 塞; 塞; ) CJK COMPATIBILITY IDEOGRAPH-F96C +F96D;7701;7701;7701;7701; # (省; 省; 省; 省; 省; ) CJK COMPATIBILITY IDEOGRAPH-F96D +F96E;8449;8449;8449;8449; # (葉; 葉; 葉; 葉; 葉; ) CJK COMPATIBILITY IDEOGRAPH-F96E +F96F;8AAA;8AAA;8AAA;8AAA; # (說; 說; 說; 說; 說; ) CJK COMPATIBILITY IDEOGRAPH-F96F +F970;6BBA;6BBA;6BBA;6BBA; # (殺; 殺; 殺; 殺; 殺; ) CJK COMPATIBILITY IDEOGRAPH-F970 +F971;8FB0;8FB0;8FB0;8FB0; # (辰; 辰; 辰; 辰; 辰; ) CJK COMPATIBILITY IDEOGRAPH-F971 +F972;6C88;6C88;6C88;6C88; # (沈; 沈; 沈; 沈; 沈; ) CJK COMPATIBILITY IDEOGRAPH-F972 +F973;62FE;62FE;62FE;62FE; # (拾; 拾; 拾; 拾; 拾; ) CJK COMPATIBILITY IDEOGRAPH-F973 +F974;82E5;82E5;82E5;82E5; # (若; 若; 若; 若; 若; ) CJK COMPATIBILITY IDEOGRAPH-F974 +F975;63A0;63A0;63A0;63A0; # (掠; 掠; 掠; 掠; 掠; ) CJK COMPATIBILITY IDEOGRAPH-F975 +F976;7565;7565;7565;7565; # (略; 略; 略; 略; 略; ) CJK COMPATIBILITY IDEOGRAPH-F976 +F977;4EAE;4EAE;4EAE;4EAE; # (亮; 亮; 亮; 亮; 亮; ) CJK COMPATIBILITY IDEOGRAPH-F977 +F978;5169;5169;5169;5169; # (兩; 兩; 兩; 兩; 兩; ) CJK COMPATIBILITY IDEOGRAPH-F978 +F979;51C9;51C9;51C9;51C9; # (凉; 凉; 凉; 凉; 凉; ) CJK COMPATIBILITY IDEOGRAPH-F979 +F97A;6881;6881;6881;6881; # (梁; 梁; 梁; 梁; 梁; ) CJK COMPATIBILITY IDEOGRAPH-F97A +F97B;7CE7;7CE7;7CE7;7CE7; # (糧; 糧; 糧; 糧; 糧; ) CJK COMPATIBILITY IDEOGRAPH-F97B +F97C;826F;826F;826F;826F; # (良; 良; 良; 良; 良; ) CJK COMPATIBILITY IDEOGRAPH-F97C +F97D;8AD2;8AD2;8AD2;8AD2; # (諒; 諒; 諒; 諒; 諒; ) CJK COMPATIBILITY IDEOGRAPH-F97D +F97E;91CF;91CF;91CF;91CF; # (量; 量; 量; 量; 量; ) CJK COMPATIBILITY IDEOGRAPH-F97E +F97F;52F5;52F5;52F5;52F5; # (勵; 勵; 勵; 勵; 勵; ) CJK COMPATIBILITY IDEOGRAPH-F97F +F980;5442;5442;5442;5442; # (呂; 呂; 呂; 呂; 呂; ) CJK COMPATIBILITY IDEOGRAPH-F980 +F981;5973;5973;5973;5973; # (女; 女; 女; 女; 女; ) CJK COMPATIBILITY IDEOGRAPH-F981 +F982;5EEC;5EEC;5EEC;5EEC; # (廬; 廬; 廬; 廬; 廬; ) CJK COMPATIBILITY IDEOGRAPH-F982 +F983;65C5;65C5;65C5;65C5; # (旅; 旅; 旅; 旅; 旅; ) CJK COMPATIBILITY IDEOGRAPH-F983 +F984;6FFE;6FFE;6FFE;6FFE; # (濾; 濾; 濾; 濾; 濾; ) CJK COMPATIBILITY IDEOGRAPH-F984 +F985;792A;792A;792A;792A; # (礪; 礪; 礪; 礪; 礪; ) CJK COMPATIBILITY IDEOGRAPH-F985 +F986;95AD;95AD;95AD;95AD; # (閭; 閭; 閭; 閭; 閭; ) CJK COMPATIBILITY IDEOGRAPH-F986 +F987;9A6A;9A6A;9A6A;9A6A; # (驪; 驪; 驪; 驪; 驪; ) CJK COMPATIBILITY IDEOGRAPH-F987 +F988;9E97;9E97;9E97;9E97; # (麗; 麗; 麗; 麗; 麗; ) CJK COMPATIBILITY IDEOGRAPH-F988 +F989;9ECE;9ECE;9ECE;9ECE; # (黎; 黎; 黎; 黎; 黎; ) CJK COMPATIBILITY IDEOGRAPH-F989 +F98A;529B;529B;529B;529B; # (力; 力; 力; 力; 力; ) CJK COMPATIBILITY IDEOGRAPH-F98A +F98B;66C6;66C6;66C6;66C6; # (曆; 曆; 曆; 曆; 曆; ) CJK COMPATIBILITY IDEOGRAPH-F98B +F98C;6B77;6B77;6B77;6B77; # (歷; 歷; 歷; 歷; 歷; ) CJK COMPATIBILITY IDEOGRAPH-F98C +F98D;8F62;8F62;8F62;8F62; # (轢; 轢; 轢; 轢; 轢; ) CJK COMPATIBILITY IDEOGRAPH-F98D +F98E;5E74;5E74;5E74;5E74; # (年; 年; 年; 年; 年; ) CJK COMPATIBILITY IDEOGRAPH-F98E +F98F;6190;6190;6190;6190; # (憐; 憐; 憐; 憐; 憐; ) CJK COMPATIBILITY IDEOGRAPH-F98F +F990;6200;6200;6200;6200; # (戀; 戀; 戀; 戀; 戀; ) CJK COMPATIBILITY IDEOGRAPH-F990 +F991;649A;649A;649A;649A; # (撚; 撚; 撚; 撚; 撚; ) CJK COMPATIBILITY IDEOGRAPH-F991 +F992;6F23;6F23;6F23;6F23; # (漣; 漣; 漣; 漣; 漣; ) CJK COMPATIBILITY IDEOGRAPH-F992 +F993;7149;7149;7149;7149; # (煉; 煉; 煉; 煉; 煉; ) CJK COMPATIBILITY IDEOGRAPH-F993 +F994;7489;7489;7489;7489; # (璉; 璉; 璉; 璉; 璉; ) CJK COMPATIBILITY IDEOGRAPH-F994 +F995;79CA;79CA;79CA;79CA; # (秊; 秊; 秊; 秊; 秊; ) CJK COMPATIBILITY IDEOGRAPH-F995 +F996;7DF4;7DF4;7DF4;7DF4; # (練; 練; 練; 練; 練; ) CJK COMPATIBILITY IDEOGRAPH-F996 +F997;806F;806F;806F;806F; # (聯; 聯; 聯; 聯; 聯; ) CJK COMPATIBILITY IDEOGRAPH-F997 +F998;8F26;8F26;8F26;8F26; # (輦; 輦; 輦; 輦; 輦; ) CJK COMPATIBILITY IDEOGRAPH-F998 +F999;84EE;84EE;84EE;84EE; # (蓮; 蓮; 蓮; 蓮; 蓮; ) CJK COMPATIBILITY IDEOGRAPH-F999 +F99A;9023;9023;9023;9023; # (連; 連; 連; 連; 連; ) CJK COMPATIBILITY IDEOGRAPH-F99A +F99B;934A;934A;934A;934A; # (鍊; 鍊; 鍊; 鍊; 鍊; ) CJK COMPATIBILITY IDEOGRAPH-F99B +F99C;5217;5217;5217;5217; # (列; 列; 列; 列; 列; ) CJK COMPATIBILITY IDEOGRAPH-F99C +F99D;52A3;52A3;52A3;52A3; # (劣; 劣; 劣; 劣; 劣; ) CJK COMPATIBILITY IDEOGRAPH-F99D +F99E;54BD;54BD;54BD;54BD; # (咽; 咽; 咽; 咽; 咽; ) CJK COMPATIBILITY IDEOGRAPH-F99E +F99F;70C8;70C8;70C8;70C8; # (烈; 烈; 烈; 烈; 烈; ) CJK COMPATIBILITY IDEOGRAPH-F99F +F9A0;88C2;88C2;88C2;88C2; # (裂; 裂; 裂; 裂; 裂; ) CJK COMPATIBILITY IDEOGRAPH-F9A0 +F9A1;8AAA;8AAA;8AAA;8AAA; # (說; 說; 說; 說; 說; ) CJK COMPATIBILITY IDEOGRAPH-F9A1 +F9A2;5EC9;5EC9;5EC9;5EC9; # (廉; 廉; 廉; 廉; 廉; ) CJK COMPATIBILITY IDEOGRAPH-F9A2 +F9A3;5FF5;5FF5;5FF5;5FF5; # (念; 念; 念; 念; 念; ) CJK COMPATIBILITY IDEOGRAPH-F9A3 +F9A4;637B;637B;637B;637B; # (捻; 捻; 捻; 捻; 捻; ) CJK COMPATIBILITY IDEOGRAPH-F9A4 +F9A5;6BAE;6BAE;6BAE;6BAE; # (殮; 殮; 殮; 殮; 殮; ) CJK COMPATIBILITY IDEOGRAPH-F9A5 +F9A6;7C3E;7C3E;7C3E;7C3E; # (簾; 簾; 簾; 簾; 簾; ) CJK COMPATIBILITY IDEOGRAPH-F9A6 +F9A7;7375;7375;7375;7375; # (獵; 獵; 獵; 獵; 獵; ) CJK COMPATIBILITY IDEOGRAPH-F9A7 +F9A8;4EE4;4EE4;4EE4;4EE4; # (令; 令; 令; 令; 令; ) CJK COMPATIBILITY IDEOGRAPH-F9A8 +F9A9;56F9;56F9;56F9;56F9; # (囹; 囹; 囹; 囹; 囹; ) CJK COMPATIBILITY IDEOGRAPH-F9A9 +F9AA;5BE7;5BE7;5BE7;5BE7; # (寧; 寧; 寧; 寧; 寧; ) CJK COMPATIBILITY IDEOGRAPH-F9AA +F9AB;5DBA;5DBA;5DBA;5DBA; # (嶺; 嶺; 嶺; 嶺; 嶺; ) CJK COMPATIBILITY IDEOGRAPH-F9AB +F9AC;601C;601C;601C;601C; # (怜; 怜; 怜; 怜; 怜; ) CJK COMPATIBILITY IDEOGRAPH-F9AC +F9AD;73B2;73B2;73B2;73B2; # (玲; 玲; 玲; 玲; 玲; ) CJK COMPATIBILITY IDEOGRAPH-F9AD +F9AE;7469;7469;7469;7469; # (瑩; 瑩; 瑩; 瑩; 瑩; ) CJK COMPATIBILITY IDEOGRAPH-F9AE +F9AF;7F9A;7F9A;7F9A;7F9A; # (羚; 羚; 羚; 羚; 羚; ) CJK COMPATIBILITY IDEOGRAPH-F9AF +F9B0;8046;8046;8046;8046; # (聆; 聆; 聆; 聆; 聆; ) CJK COMPATIBILITY IDEOGRAPH-F9B0 +F9B1;9234;9234;9234;9234; # (鈴; 鈴; 鈴; 鈴; 鈴; ) CJK COMPATIBILITY IDEOGRAPH-F9B1 +F9B2;96F6;96F6;96F6;96F6; # (零; 零; 零; 零; 零; ) CJK COMPATIBILITY IDEOGRAPH-F9B2 +F9B3;9748;9748;9748;9748; # (靈; 靈; 靈; 靈; 靈; ) CJK COMPATIBILITY IDEOGRAPH-F9B3 +F9B4;9818;9818;9818;9818; # (領; 領; 領; 領; 領; ) CJK COMPATIBILITY IDEOGRAPH-F9B4 +F9B5;4F8B;4F8B;4F8B;4F8B; # (例; 例; 例; 例; 例; ) CJK COMPATIBILITY IDEOGRAPH-F9B5 +F9B6;79AE;79AE;79AE;79AE; # (禮; 禮; 禮; 禮; 禮; ) CJK COMPATIBILITY IDEOGRAPH-F9B6 +F9B7;91B4;91B4;91B4;91B4; # (醴; 醴; 醴; 醴; 醴; ) CJK COMPATIBILITY IDEOGRAPH-F9B7 +F9B8;96B8;96B8;96B8;96B8; # (隸; 隸; 隸; 隸; 隸; ) CJK COMPATIBILITY IDEOGRAPH-F9B8 +F9B9;60E1;60E1;60E1;60E1; # (惡; 惡; 惡; 惡; 惡; ) CJK COMPATIBILITY IDEOGRAPH-F9B9 +F9BA;4E86;4E86;4E86;4E86; # (了; 了; 了; 了; 了; ) CJK COMPATIBILITY IDEOGRAPH-F9BA +F9BB;50DA;50DA;50DA;50DA; # (僚; 僚; 僚; 僚; 僚; ) CJK COMPATIBILITY IDEOGRAPH-F9BB +F9BC;5BEE;5BEE;5BEE;5BEE; # (寮; 寮; 寮; 寮; 寮; ) CJK COMPATIBILITY IDEOGRAPH-F9BC +F9BD;5C3F;5C3F;5C3F;5C3F; # (尿; 尿; 尿; 尿; 尿; ) CJK COMPATIBILITY IDEOGRAPH-F9BD +F9BE;6599;6599;6599;6599; # (料; 料; 料; 料; 料; ) CJK COMPATIBILITY IDEOGRAPH-F9BE +F9BF;6A02;6A02;6A02;6A02; # (樂; 樂; 樂; 樂; 樂; ) CJK COMPATIBILITY IDEOGRAPH-F9BF +F9C0;71CE;71CE;71CE;71CE; # (燎; 燎; 燎; 燎; 燎; ) CJK COMPATIBILITY IDEOGRAPH-F9C0 +F9C1;7642;7642;7642;7642; # (療; 療; 療; 療; 療; ) CJK COMPATIBILITY IDEOGRAPH-F9C1 +F9C2;84FC;84FC;84FC;84FC; # (蓼; 蓼; 蓼; 蓼; 蓼; ) CJK COMPATIBILITY IDEOGRAPH-F9C2 +F9C3;907C;907C;907C;907C; # (遼; 遼; 遼; 遼; 遼; ) CJK COMPATIBILITY IDEOGRAPH-F9C3 +F9C4;9F8D;9F8D;9F8D;9F8D; # (龍; 龍; 龍; 龍; 龍; ) CJK COMPATIBILITY IDEOGRAPH-F9C4 +F9C5;6688;6688;6688;6688; # (暈; 暈; 暈; 暈; 暈; ) CJK COMPATIBILITY IDEOGRAPH-F9C5 +F9C6;962E;962E;962E;962E; # (阮; 阮; 阮; 阮; 阮; ) CJK COMPATIBILITY IDEOGRAPH-F9C6 +F9C7;5289;5289;5289;5289; # (劉; 劉; 劉; 劉; 劉; ) CJK COMPATIBILITY IDEOGRAPH-F9C7 +F9C8;677B;677B;677B;677B; # (杻; 杻; 杻; 杻; 杻; ) CJK COMPATIBILITY IDEOGRAPH-F9C8 +F9C9;67F3;67F3;67F3;67F3; # (柳; 柳; 柳; 柳; 柳; ) CJK COMPATIBILITY IDEOGRAPH-F9C9 +F9CA;6D41;6D41;6D41;6D41; # (流; 流; 流; 流; 流; ) CJK COMPATIBILITY IDEOGRAPH-F9CA +F9CB;6E9C;6E9C;6E9C;6E9C; # (溜; 溜; 溜; 溜; 溜; ) CJK COMPATIBILITY IDEOGRAPH-F9CB +F9CC;7409;7409;7409;7409; # (琉; 琉; 琉; 琉; 琉; ) CJK COMPATIBILITY IDEOGRAPH-F9CC +F9CD;7559;7559;7559;7559; # (留; 留; 留; 留; 留; ) CJK COMPATIBILITY IDEOGRAPH-F9CD +F9CE;786B;786B;786B;786B; # (硫; 硫; 硫; 硫; 硫; ) CJK COMPATIBILITY IDEOGRAPH-F9CE +F9CF;7D10;7D10;7D10;7D10; # (紐; 紐; 紐; 紐; 紐; ) CJK COMPATIBILITY IDEOGRAPH-F9CF +F9D0;985E;985E;985E;985E; # (類; 類; 類; 類; 類; ) CJK COMPATIBILITY IDEOGRAPH-F9D0 +F9D1;516D;516D;516D;516D; # (六; 六; 六; 六; 六; ) CJK COMPATIBILITY IDEOGRAPH-F9D1 +F9D2;622E;622E;622E;622E; # (戮; 戮; 戮; 戮; 戮; ) CJK COMPATIBILITY IDEOGRAPH-F9D2 +F9D3;9678;9678;9678;9678; # (陸; 陸; 陸; 陸; 陸; ) CJK COMPATIBILITY IDEOGRAPH-F9D3 +F9D4;502B;502B;502B;502B; # (倫; 倫; 倫; 倫; 倫; ) CJK COMPATIBILITY IDEOGRAPH-F9D4 +F9D5;5D19;5D19;5D19;5D19; # (崙; 崙; 崙; 崙; 崙; ) CJK COMPATIBILITY IDEOGRAPH-F9D5 +F9D6;6DEA;6DEA;6DEA;6DEA; # (淪; 淪; 淪; 淪; 淪; ) CJK COMPATIBILITY IDEOGRAPH-F9D6 +F9D7;8F2A;8F2A;8F2A;8F2A; # (輪; 輪; 輪; 輪; 輪; ) CJK COMPATIBILITY IDEOGRAPH-F9D7 +F9D8;5F8B;5F8B;5F8B;5F8B; # (律; 律; 律; 律; 律; ) CJK COMPATIBILITY IDEOGRAPH-F9D8 +F9D9;6144;6144;6144;6144; # (慄; 慄; 慄; 慄; 慄; ) CJK COMPATIBILITY IDEOGRAPH-F9D9 +F9DA;6817;6817;6817;6817; # (栗; 栗; 栗; 栗; 栗; ) CJK COMPATIBILITY IDEOGRAPH-F9DA +F9DB;7387;7387;7387;7387; # (率; 率; 率; 率; 率; ) CJK COMPATIBILITY IDEOGRAPH-F9DB +F9DC;9686;9686;9686;9686; # (隆; 隆; 隆; 隆; 隆; ) CJK COMPATIBILITY IDEOGRAPH-F9DC +F9DD;5229;5229;5229;5229; # (利; 利; 利; 利; 利; ) CJK COMPATIBILITY IDEOGRAPH-F9DD +F9DE;540F;540F;540F;540F; # (吏; 吏; 吏; 吏; 吏; ) CJK COMPATIBILITY IDEOGRAPH-F9DE +F9DF;5C65;5C65;5C65;5C65; # (履; 履; 履; 履; 履; ) CJK COMPATIBILITY IDEOGRAPH-F9DF +F9E0;6613;6613;6613;6613; # (易; 易; 易; 易; 易; ) CJK COMPATIBILITY IDEOGRAPH-F9E0 +F9E1;674E;674E;674E;674E; # (李; 李; 李; 李; 李; ) CJK COMPATIBILITY IDEOGRAPH-F9E1 +F9E2;68A8;68A8;68A8;68A8; # (梨; 梨; 梨; 梨; 梨; ) CJK COMPATIBILITY IDEOGRAPH-F9E2 +F9E3;6CE5;6CE5;6CE5;6CE5; # (泥; 泥; 泥; 泥; 泥; ) CJK COMPATIBILITY IDEOGRAPH-F9E3 +F9E4;7406;7406;7406;7406; # (理; 理; 理; 理; 理; ) CJK COMPATIBILITY IDEOGRAPH-F9E4 +F9E5;75E2;75E2;75E2;75E2; # (痢; 痢; 痢; 痢; 痢; ) CJK COMPATIBILITY IDEOGRAPH-F9E5 +F9E6;7F79;7F79;7F79;7F79; # (罹; 罹; 罹; 罹; 罹; ) CJK COMPATIBILITY IDEOGRAPH-F9E6 +F9E7;88CF;88CF;88CF;88CF; # (裏; 裏; 裏; 裏; 裏; ) CJK COMPATIBILITY IDEOGRAPH-F9E7 +F9E8;88E1;88E1;88E1;88E1; # (裡; 裡; 裡; 裡; 裡; ) CJK COMPATIBILITY IDEOGRAPH-F9E8 +F9E9;91CC;91CC;91CC;91CC; # (里; 里; 里; 里; 里; ) CJK COMPATIBILITY IDEOGRAPH-F9E9 +F9EA;96E2;96E2;96E2;96E2; # (離; 離; 離; 離; 離; ) CJK COMPATIBILITY IDEOGRAPH-F9EA +F9EB;533F;533F;533F;533F; # (匿; 匿; 匿; 匿; 匿; ) CJK COMPATIBILITY IDEOGRAPH-F9EB +F9EC;6EBA;6EBA;6EBA;6EBA; # (溺; 溺; 溺; 溺; 溺; ) CJK COMPATIBILITY IDEOGRAPH-F9EC +F9ED;541D;541D;541D;541D; # (吝; 吝; 吝; 吝; 吝; ) CJK COMPATIBILITY IDEOGRAPH-F9ED +F9EE;71D0;71D0;71D0;71D0; # (燐; 燐; 燐; 燐; 燐; ) CJK COMPATIBILITY IDEOGRAPH-F9EE +F9EF;7498;7498;7498;7498; # (璘; 璘; 璘; 璘; 璘; ) CJK COMPATIBILITY IDEOGRAPH-F9EF +F9F0;85FA;85FA;85FA;85FA; # (藺; 藺; 藺; 藺; 藺; ) CJK COMPATIBILITY IDEOGRAPH-F9F0 +F9F1;96A3;96A3;96A3;96A3; # (隣; 隣; 隣; 隣; 隣; ) CJK COMPATIBILITY IDEOGRAPH-F9F1 +F9F2;9C57;9C57;9C57;9C57; # (鱗; 鱗; 鱗; 鱗; 鱗; ) CJK COMPATIBILITY IDEOGRAPH-F9F2 +F9F3;9E9F;9E9F;9E9F;9E9F; # (麟; 麟; 麟; 麟; 麟; ) CJK COMPATIBILITY IDEOGRAPH-F9F3 +F9F4;6797;6797;6797;6797; # (林; 林; 林; 林; 林; ) CJK COMPATIBILITY IDEOGRAPH-F9F4 +F9F5;6DCB;6DCB;6DCB;6DCB; # (淋; 淋; 淋; 淋; 淋; ) CJK COMPATIBILITY IDEOGRAPH-F9F5 +F9F6;81E8;81E8;81E8;81E8; # (臨; 臨; 臨; 臨; 臨; ) CJK COMPATIBILITY IDEOGRAPH-F9F6 +F9F7;7ACB;7ACB;7ACB;7ACB; # (立; 立; 立; 立; 立; ) CJK COMPATIBILITY IDEOGRAPH-F9F7 +F9F8;7B20;7B20;7B20;7B20; # (笠; 笠; 笠; 笠; 笠; ) CJK COMPATIBILITY IDEOGRAPH-F9F8 +F9F9;7C92;7C92;7C92;7C92; # (粒; 粒; 粒; 粒; 粒; ) CJK COMPATIBILITY IDEOGRAPH-F9F9 +F9FA;72C0;72C0;72C0;72C0; # (狀; 狀; 狀; 狀; 狀; ) CJK COMPATIBILITY IDEOGRAPH-F9FA +F9FB;7099;7099;7099;7099; # (炙; 炙; 炙; 炙; 炙; ) CJK COMPATIBILITY IDEOGRAPH-F9FB +F9FC;8B58;8B58;8B58;8B58; # (識; 識; 識; 識; 識; ) CJK COMPATIBILITY IDEOGRAPH-F9FC +F9FD;4EC0;4EC0;4EC0;4EC0; # (什; 什; 什; 什; 什; ) CJK COMPATIBILITY IDEOGRAPH-F9FD +F9FE;8336;8336;8336;8336; # (茶; 茶; 茶; 茶; 茶; ) CJK COMPATIBILITY IDEOGRAPH-F9FE +F9FF;523A;523A;523A;523A; # (刺; 刺; 刺; 刺; 刺; ) CJK COMPATIBILITY IDEOGRAPH-F9FF +FA00;5207;5207;5207;5207; # (切; 切; 切; 切; 切; ) CJK COMPATIBILITY IDEOGRAPH-FA00 +FA01;5EA6;5EA6;5EA6;5EA6; # (度; 度; 度; 度; 度; ) CJK COMPATIBILITY IDEOGRAPH-FA01 +FA02;62D3;62D3;62D3;62D3; # (拓; 拓; 拓; 拓; 拓; ) CJK COMPATIBILITY IDEOGRAPH-FA02 +FA03;7CD6;7CD6;7CD6;7CD6; # (糖; 糖; 糖; 糖; 糖; ) CJK COMPATIBILITY IDEOGRAPH-FA03 +FA04;5B85;5B85;5B85;5B85; # (宅; 宅; 宅; 宅; 宅; ) CJK COMPATIBILITY IDEOGRAPH-FA04 +FA05;6D1E;6D1E;6D1E;6D1E; # (洞; 洞; 洞; 洞; 洞; ) CJK COMPATIBILITY IDEOGRAPH-FA05 +FA06;66B4;66B4;66B4;66B4; # (暴; 暴; 暴; 暴; 暴; ) CJK COMPATIBILITY IDEOGRAPH-FA06 +FA07;8F3B;8F3B;8F3B;8F3B; # (輻; 輻; 輻; 輻; 輻; ) CJK COMPATIBILITY IDEOGRAPH-FA07 +FA08;884C;884C;884C;884C; # (行; 行; 行; 行; 行; ) CJK COMPATIBILITY IDEOGRAPH-FA08 +FA09;964D;964D;964D;964D; # (降; 降; 降; 降; 降; ) CJK COMPATIBILITY IDEOGRAPH-FA09 +FA0A;898B;898B;898B;898B; # (見; 見; 見; 見; 見; ) CJK COMPATIBILITY IDEOGRAPH-FA0A +FA0B;5ED3;5ED3;5ED3;5ED3; # (廓; 廓; 廓; 廓; 廓; ) CJK COMPATIBILITY IDEOGRAPH-FA0B +FA0C;5140;5140;5140;5140; # (兀; 兀; 兀; 兀; 兀; ) CJK COMPATIBILITY IDEOGRAPH-FA0C +FA0D;55C0;55C0;55C0;55C0; # (嗀; 嗀; 嗀; 嗀; 嗀; ) CJK COMPATIBILITY IDEOGRAPH-FA0D +FA10;585A;585A;585A;585A; # (塚; 塚; 塚; 塚; 塚; ) CJK COMPATIBILITY IDEOGRAPH-FA10 +FA12;6674;6674;6674;6674; # (晴; 晴; 晴; 晴; 晴; ) CJK COMPATIBILITY IDEOGRAPH-FA12 +FA15;51DE;51DE;51DE;51DE; # (凞; 凞; 凞; 凞; 凞; ) CJK COMPATIBILITY IDEOGRAPH-FA15 +FA16;732A;732A;732A;732A; # (猪; 猪; 猪; 猪; 猪; ) CJK COMPATIBILITY IDEOGRAPH-FA16 +FA17;76CA;76CA;76CA;76CA; # (益; 益; 益; 益; 益; ) CJK COMPATIBILITY IDEOGRAPH-FA17 +FA18;793C;793C;793C;793C; # (礼; 礼; 礼; 礼; 礼; ) CJK COMPATIBILITY IDEOGRAPH-FA18 +FA19;795E;795E;795E;795E; # (神; 神; 神; 神; 神; ) CJK COMPATIBILITY IDEOGRAPH-FA19 +FA1A;7965;7965;7965;7965; # (祥; 祥; 祥; 祥; 祥; ) CJK COMPATIBILITY IDEOGRAPH-FA1A +FA1B;798F;798F;798F;798F; # (福; 福; 福; 福; 福; ) CJK COMPATIBILITY IDEOGRAPH-FA1B +FA1C;9756;9756;9756;9756; # (靖; 靖; 靖; 靖; 靖; ) CJK COMPATIBILITY IDEOGRAPH-FA1C +FA1D;7CBE;7CBE;7CBE;7CBE; # (精; 精; 精; 精; 精; ) CJK COMPATIBILITY IDEOGRAPH-FA1D +FA1E;7FBD;7FBD;7FBD;7FBD; # (羽; 羽; 羽; 羽; 羽; ) CJK COMPATIBILITY IDEOGRAPH-FA1E +FA20;8612;8612;8612;8612; # (蘒; 蘒; 蘒; 蘒; 蘒; ) CJK COMPATIBILITY IDEOGRAPH-FA20 +FA22;8AF8;8AF8;8AF8;8AF8; # (諸; 諸; 諸; 諸; 諸; ) CJK COMPATIBILITY IDEOGRAPH-FA22 +FA25;9038;9038;9038;9038; # (逸; 逸; 逸; 逸; 逸; ) CJK COMPATIBILITY IDEOGRAPH-FA25 +FA26;90FD;90FD;90FD;90FD; # (都; 都; 都; 都; 都; ) CJK COMPATIBILITY IDEOGRAPH-FA26 +FA2A;98EF;98EF;98EF;98EF; # (飯; 飯; 飯; 飯; 飯; ) CJK COMPATIBILITY IDEOGRAPH-FA2A +FA2B;98FC;98FC;98FC;98FC; # (飼; 飼; 飼; 飼; 飼; ) CJK COMPATIBILITY IDEOGRAPH-FA2B +FA2C;9928;9928;9928;9928; # (館; 館; 館; 館; 館; ) CJK COMPATIBILITY IDEOGRAPH-FA2C +FA2D;9DB4;9DB4;9DB4;9DB4; # (鶴; 鶴; 鶴; 鶴; 鶴; ) CJK COMPATIBILITY IDEOGRAPH-FA2D +FA30;4FAE;4FAE;4FAE;4FAE; # (侮; 侮; 侮; 侮; 侮; ) CJK COMPATIBILITY IDEOGRAPH-FA30 +FA31;50E7;50E7;50E7;50E7; # (僧; 僧; 僧; 僧; 僧; ) CJK COMPATIBILITY IDEOGRAPH-FA31 +FA32;514D;514D;514D;514D; # (免; 免; 免; 免; 免; ) CJK COMPATIBILITY IDEOGRAPH-FA32 +FA33;52C9;52C9;52C9;52C9; # (勉; 勉; 勉; 勉; 勉; ) CJK COMPATIBILITY IDEOGRAPH-FA33 +FA34;52E4;52E4;52E4;52E4; # (勤; 勤; 勤; 勤; 勤; ) CJK COMPATIBILITY IDEOGRAPH-FA34 +FA35;5351;5351;5351;5351; # (卑; 卑; 卑; 卑; 卑; ) CJK COMPATIBILITY IDEOGRAPH-FA35 +FA36;559D;559D;559D;559D; # (喝; 喝; 喝; 喝; 喝; ) CJK COMPATIBILITY IDEOGRAPH-FA36 +FA37;5606;5606;5606;5606; # (嘆; 嘆; 嘆; 嘆; 嘆; ) CJK COMPATIBILITY IDEOGRAPH-FA37 +FA38;5668;5668;5668;5668; # (器; 器; 器; 器; 器; ) CJK COMPATIBILITY IDEOGRAPH-FA38 +FA39;5840;5840;5840;5840; # (塀; 塀; 塀; 塀; 塀; ) CJK COMPATIBILITY IDEOGRAPH-FA39 +FA3A;58A8;58A8;58A8;58A8; # (墨; 墨; 墨; 墨; 墨; ) CJK COMPATIBILITY IDEOGRAPH-FA3A +FA3B;5C64;5C64;5C64;5C64; # (層; 層; 層; 層; 層; ) CJK COMPATIBILITY IDEOGRAPH-FA3B +FA3C;5C6E;5C6E;5C6E;5C6E; # (屮; 屮; 屮; 屮; 屮; ) CJK COMPATIBILITY IDEOGRAPH-FA3C +FA3D;6094;6094;6094;6094; # (悔; 悔; 悔; 悔; 悔; ) CJK COMPATIBILITY IDEOGRAPH-FA3D +FA3E;6168;6168;6168;6168; # (慨; 慨; 慨; 慨; 慨; ) CJK COMPATIBILITY IDEOGRAPH-FA3E +FA3F;618E;618E;618E;618E; # (憎; 憎; 憎; 憎; 憎; ) CJK COMPATIBILITY IDEOGRAPH-FA3F +FA40;61F2;61F2;61F2;61F2; # (懲; 懲; 懲; 懲; 懲; ) CJK COMPATIBILITY IDEOGRAPH-FA40 +FA41;654F;654F;654F;654F; # (敏; 敏; 敏; 敏; 敏; ) CJK COMPATIBILITY IDEOGRAPH-FA41 +FA42;65E2;65E2;65E2;65E2; # (既; 既; 既; 既; 既; ) CJK COMPATIBILITY IDEOGRAPH-FA42 +FA43;6691;6691;6691;6691; # (暑; 暑; 暑; 暑; 暑; ) CJK COMPATIBILITY IDEOGRAPH-FA43 +FA44;6885;6885;6885;6885; # (梅; 梅; 梅; 梅; 梅; ) CJK COMPATIBILITY IDEOGRAPH-FA44 +FA45;6D77;6D77;6D77;6D77; # (海; 海; 海; 海; 海; ) CJK COMPATIBILITY IDEOGRAPH-FA45 +FA46;6E1A;6E1A;6E1A;6E1A; # (渚; 渚; 渚; 渚; 渚; ) CJK COMPATIBILITY IDEOGRAPH-FA46 +FA47;6F22;6F22;6F22;6F22; # (漢; 漢; 漢; 漢; 漢; ) CJK COMPATIBILITY IDEOGRAPH-FA47 +FA48;716E;716E;716E;716E; # (煮; 煮; 煮; 煮; 煮; ) CJK COMPATIBILITY IDEOGRAPH-FA48 +FA49;722B;722B;722B;722B; # (爫; 爫; 爫; 爫; 爫; ) CJK COMPATIBILITY IDEOGRAPH-FA49 +FA4A;7422;7422;7422;7422; # (琢; 琢; 琢; 琢; 琢; ) CJK COMPATIBILITY IDEOGRAPH-FA4A +FA4B;7891;7891;7891;7891; # (碑; 碑; 碑; 碑; 碑; ) CJK COMPATIBILITY IDEOGRAPH-FA4B +FA4C;793E;793E;793E;793E; # (社; 社; 社; 社; 社; ) CJK COMPATIBILITY IDEOGRAPH-FA4C +FA4D;7949;7949;7949;7949; # (祉; 祉; 祉; 祉; 祉; ) CJK COMPATIBILITY IDEOGRAPH-FA4D +FA4E;7948;7948;7948;7948; # (祈; 祈; 祈; 祈; 祈; ) CJK COMPATIBILITY IDEOGRAPH-FA4E +FA4F;7950;7950;7950;7950; # (祐; 祐; 祐; 祐; 祐; ) CJK COMPATIBILITY IDEOGRAPH-FA4F +FA50;7956;7956;7956;7956; # (祖; 祖; 祖; 祖; 祖; ) CJK COMPATIBILITY IDEOGRAPH-FA50 +FA51;795D;795D;795D;795D; # (祝; 祝; 祝; 祝; 祝; ) CJK COMPATIBILITY IDEOGRAPH-FA51 +FA52;798D;798D;798D;798D; # (禍; 禍; 禍; 禍; 禍; ) CJK COMPATIBILITY IDEOGRAPH-FA52 +FA53;798E;798E;798E;798E; # (禎; 禎; 禎; 禎; 禎; ) CJK COMPATIBILITY IDEOGRAPH-FA53 +FA54;7A40;7A40;7A40;7A40; # (穀; 穀; 穀; 穀; 穀; ) CJK COMPATIBILITY IDEOGRAPH-FA54 +FA55;7A81;7A81;7A81;7A81; # (突; 突; 突; 突; 突; ) CJK COMPATIBILITY IDEOGRAPH-FA55 +FA56;7BC0;7BC0;7BC0;7BC0; # (節; 節; 節; 節; 節; ) CJK COMPATIBILITY IDEOGRAPH-FA56 +FA57;7DF4;7DF4;7DF4;7DF4; # (練; 練; 練; 練; 練; ) CJK COMPATIBILITY IDEOGRAPH-FA57 +FA58;7E09;7E09;7E09;7E09; # (縉; 縉; 縉; 縉; 縉; ) CJK COMPATIBILITY IDEOGRAPH-FA58 +FA59;7E41;7E41;7E41;7E41; # (繁; 繁; 繁; 繁; 繁; ) CJK COMPATIBILITY IDEOGRAPH-FA59 +FA5A;7F72;7F72;7F72;7F72; # (署; 署; 署; 署; 署; ) CJK COMPATIBILITY IDEOGRAPH-FA5A +FA5B;8005;8005;8005;8005; # (者; 者; 者; 者; 者; ) CJK COMPATIBILITY IDEOGRAPH-FA5B +FA5C;81ED;81ED;81ED;81ED; # (臭; 臭; 臭; 臭; 臭; ) CJK COMPATIBILITY IDEOGRAPH-FA5C +FA5D;8279;8279;8279;8279; # (艹; 艹; 艹; 艹; 艹; ) CJK COMPATIBILITY IDEOGRAPH-FA5D +FA5E;8279;8279;8279;8279; # (艹; 艹; 艹; 艹; 艹; ) CJK COMPATIBILITY IDEOGRAPH-FA5E +FA5F;8457;8457;8457;8457; # (著; 著; 著; 著; 著; ) CJK COMPATIBILITY IDEOGRAPH-FA5F +FA60;8910;8910;8910;8910; # (褐; 褐; 褐; 褐; 褐; ) CJK COMPATIBILITY IDEOGRAPH-FA60 +FA61;8996;8996;8996;8996; # (視; 視; 視; 視; 視; ) CJK COMPATIBILITY IDEOGRAPH-FA61 +FA62;8B01;8B01;8B01;8B01; # (謁; 謁; 謁; 謁; 謁; ) CJK COMPATIBILITY IDEOGRAPH-FA62 +FA63;8B39;8B39;8B39;8B39; # (謹; 謹; 謹; 謹; 謹; ) CJK COMPATIBILITY IDEOGRAPH-FA63 +FA64;8CD3;8CD3;8CD3;8CD3; # (賓; 賓; 賓; 賓; 賓; ) CJK COMPATIBILITY IDEOGRAPH-FA64 +FA65;8D08;8D08;8D08;8D08; # (贈; 贈; 贈; 贈; 贈; ) CJK COMPATIBILITY IDEOGRAPH-FA65 +FA66;8FB6;8FB6;8FB6;8FB6; # (辶; 辶; 辶; 辶; 辶; ) CJK COMPATIBILITY IDEOGRAPH-FA66 +FA67;9038;9038;9038;9038; # (逸; 逸; 逸; 逸; 逸; ) CJK COMPATIBILITY IDEOGRAPH-FA67 +FA68;96E3;96E3;96E3;96E3; # (難; 難; 難; 難; 難; ) CJK COMPATIBILITY IDEOGRAPH-FA68 +FA69;97FF;97FF;97FF;97FF; # (響; 響; 響; 響; 響; ) CJK COMPATIBILITY IDEOGRAPH-FA69 +FA6A;983B;983B;983B;983B; # (頻; 頻; 頻; 頻; 頻; ) CJK COMPATIBILITY IDEOGRAPH-FA6A +FB00;FB00;FB00;0066 0066;0066 0066; # (ff; ff; ff; ff; ff; ) LATIN SMALL LIGATURE FF +FB01;FB01;FB01;0066 0069;0066 0069; # (fi; fi; fi; fi; fi; ) LATIN SMALL LIGATURE FI +FB02;FB02;FB02;0066 006C;0066 006C; # (fl; fl; fl; fl; fl; ) LATIN SMALL LIGATURE FL +FB03;FB03;FB03;0066 0066 0069;0066 0066 0069; # (ffi; ffi; ffi; ffi; ffi; ) LATIN SMALL LIGATURE FFI +FB04;FB04;FB04;0066 0066 006C;0066 0066 006C; # (ffl; ffl; ffl; ffl; ffl; ) LATIN SMALL LIGATURE FFL +FB05;FB05;FB05;0073 0074;0073 0074; # (ſt; ſt; ſt; st; st; ) LATIN SMALL LIGATURE LONG S T +FB06;FB06;FB06;0073 0074;0073 0074; # (st; st; st; st; st; ) LATIN SMALL LIGATURE ST +FB13;FB13;FB13;0574 0576;0574 0576; # (ﬓ; ﬓ; ﬓ; մն; մն; ) ARMENIAN SMALL LIGATURE MEN NOW +FB14;FB14;FB14;0574 0565;0574 0565; # (ﬔ; ﬔ; ﬔ; մե; մե; ) ARMENIAN SMALL LIGATURE MEN ECH +FB15;FB15;FB15;0574 056B;0574 056B; # (ﬕ; ﬕ; ﬕ; մի; մի; ) ARMENIAN SMALL LIGATURE MEN INI +FB16;FB16;FB16;057E 0576;057E 0576; # (ﬖ; ﬖ; ﬖ; վն; վն; ) ARMENIAN SMALL LIGATURE VEW NOW +FB17;FB17;FB17;0574 056D;0574 056D; # (ﬗ; ﬗ; ﬗ; մխ; մխ; ) ARMENIAN SMALL LIGATURE MEN XEH +FB1D;05D9 05B4;05D9 05B4;05D9 05B4;05D9 05B4; # (יִ; י◌ִ; י◌ִ; י◌ִ; י◌ִ; ) HEBREW LETTER YOD WITH HIRIQ +FB1F;05F2 05B7;05F2 05B7;05F2 05B7;05F2 05B7; # (ײַ; ײ◌ַ; ײ◌ַ; ײ◌ַ; ײ◌ַ; ) HEBREW LIGATURE YIDDISH YOD YOD PATAH +FB20;FB20;FB20;05E2;05E2; # (ﬠ; ﬠ; ﬠ; ע; ע; ) HEBREW LETTER ALTERNATIVE AYIN +FB21;FB21;FB21;05D0;05D0; # (ﬡ; ﬡ; ﬡ; א; א; ) HEBREW LETTER WIDE ALEF +FB22;FB22;FB22;05D3;05D3; # (ﬢ; ﬢ; ﬢ; ד; ד; ) HEBREW LETTER WIDE DALET +FB23;FB23;FB23;05D4;05D4; # (ﬣ; ﬣ; ﬣ; ה; ה; ) HEBREW LETTER WIDE HE +FB24;FB24;FB24;05DB;05DB; # (ﬤ; ﬤ; ﬤ; כ; כ; ) HEBREW LETTER WIDE KAF +FB25;FB25;FB25;05DC;05DC; # (ﬥ; ﬥ; ﬥ; ל; ל; ) HEBREW LETTER WIDE LAMED +FB26;FB26;FB26;05DD;05DD; # (ﬦ; ﬦ; ﬦ; ם; ם; ) HEBREW LETTER WIDE FINAL MEM +FB27;FB27;FB27;05E8;05E8; # (ﬧ; ﬧ; ﬧ; ר; ר; ) HEBREW LETTER WIDE RESH +FB28;FB28;FB28;05EA;05EA; # (ﬨ; ﬨ; ﬨ; ת; ת; ) HEBREW LETTER WIDE TAV +FB29;FB29;FB29;002B;002B; # (﬩; ﬩; ﬩; +; +; ) HEBREW LETTER ALTERNATIVE PLUS SIGN +FB2A;05E9 05C1;05E9 05C1;05E9 05C1;05E9 05C1; # (שׁ; ש◌ׁ; ש◌ׁ; ש◌ׁ; ש◌ׁ; ) HEBREW LETTER SHIN WITH SHIN DOT +FB2B;05E9 05C2;05E9 05C2;05E9 05C2;05E9 05C2; # (שׂ; ש◌ׂ; ש◌ׂ; ש◌ׂ; ש◌ׂ; ) HEBREW LETTER SHIN WITH SIN DOT +FB2C;05E9 05BC 05C1;05E9 05BC 05C1;05E9 05BC 05C1;05E9 05BC 05C1; # (שּׁ; ש◌ּ◌ׁ; ש◌ּ◌ׁ; ש◌ּ◌ׁ; ש◌ּ◌ׁ; ) HEBREW LETTER SHIN WITH DAGESH AND SHIN DOT +FB2D;05E9 05BC 05C2;05E9 05BC 05C2;05E9 05BC 05C2;05E9 05BC 05C2; # (שּׂ; ש◌ּ◌ׂ; ש◌ּ◌ׂ; ש◌ּ◌ׂ; ש◌ּ◌ׂ; ) HEBREW LETTER SHIN WITH DAGESH AND SIN DOT +FB2E;05D0 05B7;05D0 05B7;05D0 05B7;05D0 05B7; # (אַ; א◌ַ; א◌ַ; א◌ַ; א◌ַ; ) HEBREW LETTER ALEF WITH PATAH +FB2F;05D0 05B8;05D0 05B8;05D0 05B8;05D0 05B8; # (אָ; א◌ָ; א◌ָ; א◌ָ; א◌ָ; ) HEBREW LETTER ALEF WITH QAMATS +FB30;05D0 05BC;05D0 05BC;05D0 05BC;05D0 05BC; # (אּ; א◌ּ; א◌ּ; א◌ּ; א◌ּ; ) HEBREW LETTER ALEF WITH MAPIQ +FB31;05D1 05BC;05D1 05BC;05D1 05BC;05D1 05BC; # (בּ; ב◌ּ; ב◌ּ; ב◌ּ; ב◌ּ; ) HEBREW LETTER BET WITH DAGESH +FB32;05D2 05BC;05D2 05BC;05D2 05BC;05D2 05BC; # (גּ; ג◌ּ; ג◌ּ; ג◌ּ; ג◌ּ; ) HEBREW LETTER GIMEL WITH DAGESH +FB33;05D3 05BC;05D3 05BC;05D3 05BC;05D3 05BC; # (דּ; ד◌ּ; ד◌ּ; ד◌ּ; ד◌ּ; ) HEBREW LETTER DALET WITH DAGESH +FB34;05D4 05BC;05D4 05BC;05D4 05BC;05D4 05BC; # (הּ; ה◌ּ; ה◌ּ; ה◌ּ; ה◌ּ; ) HEBREW LETTER HE WITH MAPIQ +FB35;05D5 05BC;05D5 05BC;05D5 05BC;05D5 05BC; # (וּ; ו◌ּ; ו◌ּ; ו◌ּ; ו◌ּ; ) HEBREW LETTER VAV WITH DAGESH +FB36;05D6 05BC;05D6 05BC;05D6 05BC;05D6 05BC; # (זּ; ז◌ּ; ז◌ּ; ז◌ּ; ז◌ּ; ) HEBREW LETTER ZAYIN WITH DAGESH +FB38;05D8 05BC;05D8 05BC;05D8 05BC;05D8 05BC; # (טּ; ט◌ּ; ט◌ּ; ט◌ּ; ט◌ּ; ) HEBREW LETTER TET WITH DAGESH +FB39;05D9 05BC;05D9 05BC;05D9 05BC;05D9 05BC; # (יּ; י◌ּ; י◌ּ; י◌ּ; י◌ּ; ) HEBREW LETTER YOD WITH DAGESH +FB3A;05DA 05BC;05DA 05BC;05DA 05BC;05DA 05BC; # (ךּ; ך◌ּ; ך◌ּ; ך◌ּ; ך◌ּ; ) HEBREW LETTER FINAL KAF WITH DAGESH +FB3B;05DB 05BC;05DB 05BC;05DB 05BC;05DB 05BC; # (כּ; כ◌ּ; כ◌ּ; כ◌ּ; כ◌ּ; ) HEBREW LETTER KAF WITH DAGESH +FB3C;05DC 05BC;05DC 05BC;05DC 05BC;05DC 05BC; # (לּ; ל◌ּ; ל◌ּ; ל◌ּ; ל◌ּ; ) HEBREW LETTER LAMED WITH DAGESH +FB3E;05DE 05BC;05DE 05BC;05DE 05BC;05DE 05BC; # (מּ; מ◌ּ; מ◌ּ; מ◌ּ; מ◌ּ; ) HEBREW LETTER MEM WITH DAGESH +FB40;05E0 05BC;05E0 05BC;05E0 05BC;05E0 05BC; # (נּ; נ◌ּ; נ◌ּ; נ◌ּ; נ◌ּ; ) HEBREW LETTER NUN WITH DAGESH +FB41;05E1 05BC;05E1 05BC;05E1 05BC;05E1 05BC; # (סּ; ס◌ּ; ס◌ּ; ס◌ּ; ס◌ּ; ) HEBREW LETTER SAMEKH WITH DAGESH +FB43;05E3 05BC;05E3 05BC;05E3 05BC;05E3 05BC; # (ףּ; ף◌ּ; ף◌ּ; ף◌ּ; ף◌ּ; ) HEBREW LETTER FINAL PE WITH DAGESH +FB44;05E4 05BC;05E4 05BC;05E4 05BC;05E4 05BC; # (פּ; פ◌ּ; פ◌ּ; פ◌ּ; פ◌ּ; ) HEBREW LETTER PE WITH DAGESH +FB46;05E6 05BC;05E6 05BC;05E6 05BC;05E6 05BC; # (צּ; צ◌ּ; צ◌ּ; צ◌ּ; צ◌ּ; ) HEBREW LETTER TSADI WITH DAGESH +FB47;05E7 05BC;05E7 05BC;05E7 05BC;05E7 05BC; # (קּ; ק◌ּ; ק◌ּ; ק◌ּ; ק◌ּ; ) HEBREW LETTER QOF WITH DAGESH +FB48;05E8 05BC;05E8 05BC;05E8 05BC;05E8 05BC; # (רּ; ר◌ּ; ר◌ּ; ר◌ּ; ר◌ּ; ) HEBREW LETTER RESH WITH DAGESH +FB49;05E9 05BC;05E9 05BC;05E9 05BC;05E9 05BC; # (שּ; ש◌ּ; ש◌ּ; ש◌ּ; ש◌ּ; ) HEBREW LETTER SHIN WITH DAGESH +FB4A;05EA 05BC;05EA 05BC;05EA 05BC;05EA 05BC; # (תּ; ת◌ּ; ת◌ּ; ת◌ּ; ת◌ּ; ) HEBREW LETTER TAV WITH DAGESH +FB4B;05D5 05B9;05D5 05B9;05D5 05B9;05D5 05B9; # (וֹ; ו◌ֹ; ו◌ֹ; ו◌ֹ; ו◌ֹ; ) HEBREW LETTER VAV WITH HOLAM +FB4C;05D1 05BF;05D1 05BF;05D1 05BF;05D1 05BF; # (בֿ; ב◌ֿ; ב◌ֿ; ב◌ֿ; ב◌ֿ; ) HEBREW LETTER BET WITH RAFE +FB4D;05DB 05BF;05DB 05BF;05DB 05BF;05DB 05BF; # (כֿ; כ◌ֿ; כ◌ֿ; כ◌ֿ; כ◌ֿ; ) HEBREW LETTER KAF WITH RAFE +FB4E;05E4 05BF;05E4 05BF;05E4 05BF;05E4 05BF; # (פֿ; פ◌ֿ; פ◌ֿ; פ◌ֿ; פ◌ֿ; ) HEBREW LETTER PE WITH RAFE +FB4F;FB4F;FB4F;05D0 05DC;05D0 05DC; # (ﭏ; ﭏ; ﭏ; אל; אל; ) HEBREW LIGATURE ALEF LAMED +FB50;FB50;FB50;0671;0671; # (ﭐ; ﭐ; ﭐ; ٱ; ٱ; ) ARABIC LETTER ALEF WASLA ISOLATED FORM +FB51;FB51;FB51;0671;0671; # (ﭑ; ﭑ; ﭑ; ٱ; ٱ; ) ARABIC LETTER ALEF WASLA FINAL FORM +FB52;FB52;FB52;067B;067B; # (ﭒ; ﭒ; ﭒ; ٻ; ٻ; ) ARABIC LETTER BEEH ISOLATED FORM +FB53;FB53;FB53;067B;067B; # (ﭓ; ﭓ; ﭓ; ٻ; ٻ; ) ARABIC LETTER BEEH FINAL FORM +FB54;FB54;FB54;067B;067B; # (ﭔ; ﭔ; ﭔ; ٻ; ٻ; ) ARABIC LETTER BEEH INITIAL FORM +FB55;FB55;FB55;067B;067B; # (ﭕ; ﭕ; ﭕ; ٻ; ٻ; ) ARABIC LETTER BEEH MEDIAL FORM +FB56;FB56;FB56;067E;067E; # (ﭖ; ﭖ; ﭖ; پ; پ; ) ARABIC LETTER PEH ISOLATED FORM +FB57;FB57;FB57;067E;067E; # (ﭗ; ﭗ; ﭗ; پ; پ; ) ARABIC LETTER PEH FINAL FORM +FB58;FB58;FB58;067E;067E; # (ﭘ; ﭘ; ﭘ; پ; پ; ) ARABIC LETTER PEH INITIAL FORM +FB59;FB59;FB59;067E;067E; # (ﭙ; ﭙ; ﭙ; پ; پ; ) ARABIC LETTER PEH MEDIAL FORM +FB5A;FB5A;FB5A;0680;0680; # (ﭚ; ﭚ; ﭚ; ڀ; ڀ; ) ARABIC LETTER BEHEH ISOLATED FORM +FB5B;FB5B;FB5B;0680;0680; # (ﭛ; ﭛ; ﭛ; ڀ; ڀ; ) ARABIC LETTER BEHEH FINAL FORM +FB5C;FB5C;FB5C;0680;0680; # (ﭜ; ﭜ; ﭜ; ڀ; ڀ; ) ARABIC LETTER BEHEH INITIAL FORM +FB5D;FB5D;FB5D;0680;0680; # (ﭝ; ﭝ; ﭝ; ڀ; ڀ; ) ARABIC LETTER BEHEH MEDIAL FORM +FB5E;FB5E;FB5E;067A;067A; # (ﭞ; ﭞ; ﭞ; ٺ; ٺ; ) ARABIC LETTER TTEHEH ISOLATED FORM +FB5F;FB5F;FB5F;067A;067A; # (ﭟ; ﭟ; ﭟ; ٺ; ٺ; ) ARABIC LETTER TTEHEH FINAL FORM +FB60;FB60;FB60;067A;067A; # (ﭠ; ﭠ; ﭠ; ٺ; ٺ; ) ARABIC LETTER TTEHEH INITIAL FORM +FB61;FB61;FB61;067A;067A; # (ﭡ; ﭡ; ﭡ; ٺ; ٺ; ) ARABIC LETTER TTEHEH MEDIAL FORM +FB62;FB62;FB62;067F;067F; # (ﭢ; ﭢ; ﭢ; ٿ; ٿ; ) ARABIC LETTER TEHEH ISOLATED FORM +FB63;FB63;FB63;067F;067F; # (ﭣ; ﭣ; ﭣ; ٿ; ٿ; ) ARABIC LETTER TEHEH FINAL FORM +FB64;FB64;FB64;067F;067F; # (ﭤ; ﭤ; ﭤ; ٿ; ٿ; ) ARABIC LETTER TEHEH INITIAL FORM +FB65;FB65;FB65;067F;067F; # (ﭥ; ﭥ; ﭥ; ٿ; ٿ; ) ARABIC LETTER TEHEH MEDIAL FORM +FB66;FB66;FB66;0679;0679; # (ﭦ; ﭦ; ﭦ; ٹ; ٹ; ) ARABIC LETTER TTEH ISOLATED FORM +FB67;FB67;FB67;0679;0679; # (ﭧ; ﭧ; ﭧ; ٹ; ٹ; ) ARABIC LETTER TTEH FINAL FORM +FB68;FB68;FB68;0679;0679; # (ﭨ; ﭨ; ﭨ; ٹ; ٹ; ) ARABIC LETTER TTEH INITIAL FORM +FB69;FB69;FB69;0679;0679; # (ﭩ; ﭩ; ﭩ; ٹ; ٹ; ) ARABIC LETTER TTEH MEDIAL FORM +FB6A;FB6A;FB6A;06A4;06A4; # (ﭪ; ﭪ; ﭪ; ڤ; ڤ; ) ARABIC LETTER VEH ISOLATED FORM +FB6B;FB6B;FB6B;06A4;06A4; # (ﭫ; ﭫ; ﭫ; ڤ; ڤ; ) ARABIC LETTER VEH FINAL FORM +FB6C;FB6C;FB6C;06A4;06A4; # (ﭬ; ﭬ; ﭬ; ڤ; ڤ; ) ARABIC LETTER VEH INITIAL FORM +FB6D;FB6D;FB6D;06A4;06A4; # (ﭭ; ﭭ; ﭭ; ڤ; ڤ; ) ARABIC LETTER VEH MEDIAL FORM +FB6E;FB6E;FB6E;06A6;06A6; # (ﭮ; ﭮ; ﭮ; ڦ; ڦ; ) ARABIC LETTER PEHEH ISOLATED FORM +FB6F;FB6F;FB6F;06A6;06A6; # (ﭯ; ﭯ; ﭯ; ڦ; ڦ; ) ARABIC LETTER PEHEH FINAL FORM +FB70;FB70;FB70;06A6;06A6; # (ﭰ; ﭰ; ﭰ; ڦ; ڦ; ) ARABIC LETTER PEHEH INITIAL FORM +FB71;FB71;FB71;06A6;06A6; # (ﭱ; ﭱ; ﭱ; ڦ; ڦ; ) ARABIC LETTER PEHEH MEDIAL FORM +FB72;FB72;FB72;0684;0684; # (ﭲ; ﭲ; ﭲ; ڄ; ڄ; ) ARABIC LETTER DYEH ISOLATED FORM +FB73;FB73;FB73;0684;0684; # (ﭳ; ﭳ; ﭳ; ڄ; ڄ; ) ARABIC LETTER DYEH FINAL FORM +FB74;FB74;FB74;0684;0684; # (ﭴ; ﭴ; ﭴ; ڄ; ڄ; ) ARABIC LETTER DYEH INITIAL FORM +FB75;FB75;FB75;0684;0684; # (ﭵ; ﭵ; ﭵ; ڄ; ڄ; ) ARABIC LETTER DYEH MEDIAL FORM +FB76;FB76;FB76;0683;0683; # (ﭶ; ﭶ; ﭶ; ڃ; ڃ; ) ARABIC LETTER NYEH ISOLATED FORM +FB77;FB77;FB77;0683;0683; # (ﭷ; ﭷ; ﭷ; ڃ; ڃ; ) ARABIC LETTER NYEH FINAL FORM +FB78;FB78;FB78;0683;0683; # (ﭸ; ﭸ; ﭸ; ڃ; ڃ; ) ARABIC LETTER NYEH INITIAL FORM +FB79;FB79;FB79;0683;0683; # (ﭹ; ﭹ; ﭹ; ڃ; ڃ; ) ARABIC LETTER NYEH MEDIAL FORM +FB7A;FB7A;FB7A;0686;0686; # (ﭺ; ﭺ; ﭺ; چ; چ; ) ARABIC LETTER TCHEH ISOLATED FORM +FB7B;FB7B;FB7B;0686;0686; # (ﭻ; ﭻ; ﭻ; چ; چ; ) ARABIC LETTER TCHEH FINAL FORM +FB7C;FB7C;FB7C;0686;0686; # (ﭼ; ﭼ; ﭼ; چ; چ; ) ARABIC LETTER TCHEH INITIAL FORM +FB7D;FB7D;FB7D;0686;0686; # (ﭽ; ﭽ; ﭽ; چ; چ; ) ARABIC LETTER TCHEH MEDIAL FORM +FB7E;FB7E;FB7E;0687;0687; # (ﭾ; ﭾ; ﭾ; ڇ; ڇ; ) ARABIC LETTER TCHEHEH ISOLATED FORM +FB7F;FB7F;FB7F;0687;0687; # (ﭿ; ﭿ; ﭿ; ڇ; ڇ; ) ARABIC LETTER TCHEHEH FINAL FORM +FB80;FB80;FB80;0687;0687; # (ﮀ; ﮀ; ﮀ; ڇ; ڇ; ) ARABIC LETTER TCHEHEH INITIAL FORM +FB81;FB81;FB81;0687;0687; # (ﮁ; ﮁ; ﮁ; ڇ; ڇ; ) ARABIC LETTER TCHEHEH MEDIAL FORM +FB82;FB82;FB82;068D;068D; # (ﮂ; ﮂ; ﮂ; ڍ; ڍ; ) ARABIC LETTER DDAHAL ISOLATED FORM +FB83;FB83;FB83;068D;068D; # (ﮃ; ﮃ; ﮃ; ڍ; ڍ; ) ARABIC LETTER DDAHAL FINAL FORM +FB84;FB84;FB84;068C;068C; # (ﮄ; ﮄ; ﮄ; ڌ; ڌ; ) ARABIC LETTER DAHAL ISOLATED FORM +FB85;FB85;FB85;068C;068C; # (ﮅ; ﮅ; ﮅ; ڌ; ڌ; ) ARABIC LETTER DAHAL FINAL FORM +FB86;FB86;FB86;068E;068E; # (ﮆ; ﮆ; ﮆ; ڎ; ڎ; ) ARABIC LETTER DUL ISOLATED FORM +FB87;FB87;FB87;068E;068E; # (ﮇ; ﮇ; ﮇ; ڎ; ڎ; ) ARABIC LETTER DUL FINAL FORM +FB88;FB88;FB88;0688;0688; # (ﮈ; ﮈ; ﮈ; ڈ; ڈ; ) ARABIC LETTER DDAL ISOLATED FORM +FB89;FB89;FB89;0688;0688; # (ﮉ; ﮉ; ﮉ; ڈ; ڈ; ) ARABIC LETTER DDAL FINAL FORM +FB8A;FB8A;FB8A;0698;0698; # (ﮊ; ﮊ; ﮊ; ژ; ژ; ) ARABIC LETTER JEH ISOLATED FORM +FB8B;FB8B;FB8B;0698;0698; # (ﮋ; ﮋ; ﮋ; ژ; ژ; ) ARABIC LETTER JEH FINAL FORM +FB8C;FB8C;FB8C;0691;0691; # (ﮌ; ﮌ; ﮌ; ڑ; ڑ; ) ARABIC LETTER RREH ISOLATED FORM +FB8D;FB8D;FB8D;0691;0691; # (ﮍ; ﮍ; ﮍ; ڑ; ڑ; ) ARABIC LETTER RREH FINAL FORM +FB8E;FB8E;FB8E;06A9;06A9; # (ﮎ; ﮎ; ﮎ; ک; ک; ) ARABIC LETTER KEHEH ISOLATED FORM +FB8F;FB8F;FB8F;06A9;06A9; # (ﮏ; ﮏ; ﮏ; ک; ک; ) ARABIC LETTER KEHEH FINAL FORM +FB90;FB90;FB90;06A9;06A9; # (ﮐ; ﮐ; ﮐ; ک; ک; ) ARABIC LETTER KEHEH INITIAL FORM +FB91;FB91;FB91;06A9;06A9; # (ﮑ; ﮑ; ﮑ; ک; ک; ) ARABIC LETTER KEHEH MEDIAL FORM +FB92;FB92;FB92;06AF;06AF; # (ﮒ; ﮒ; ﮒ; گ; گ; ) ARABIC LETTER GAF ISOLATED FORM +FB93;FB93;FB93;06AF;06AF; # (ﮓ; ﮓ; ﮓ; گ; گ; ) ARABIC LETTER GAF FINAL FORM +FB94;FB94;FB94;06AF;06AF; # (ﮔ; ﮔ; ﮔ; گ; گ; ) ARABIC LETTER GAF INITIAL FORM +FB95;FB95;FB95;06AF;06AF; # (ﮕ; ﮕ; ﮕ; گ; گ; ) ARABIC LETTER GAF MEDIAL FORM +FB96;FB96;FB96;06B3;06B3; # (ﮖ; ﮖ; ﮖ; ڳ; ڳ; ) ARABIC LETTER GUEH ISOLATED FORM +FB97;FB97;FB97;06B3;06B3; # (ﮗ; ﮗ; ﮗ; ڳ; ڳ; ) ARABIC LETTER GUEH FINAL FORM +FB98;FB98;FB98;06B3;06B3; # (ﮘ; ﮘ; ﮘ; ڳ; ڳ; ) ARABIC LETTER GUEH INITIAL FORM +FB99;FB99;FB99;06B3;06B3; # (ﮙ; ﮙ; ﮙ; ڳ; ڳ; ) ARABIC LETTER GUEH MEDIAL FORM +FB9A;FB9A;FB9A;06B1;06B1; # (ﮚ; ﮚ; ﮚ; ڱ; ڱ; ) ARABIC LETTER NGOEH ISOLATED FORM +FB9B;FB9B;FB9B;06B1;06B1; # (ﮛ; ﮛ; ﮛ; ڱ; ڱ; ) ARABIC LETTER NGOEH FINAL FORM +FB9C;FB9C;FB9C;06B1;06B1; # (ﮜ; ﮜ; ﮜ; ڱ; ڱ; ) ARABIC LETTER NGOEH INITIAL FORM +FB9D;FB9D;FB9D;06B1;06B1; # (ﮝ; ﮝ; ﮝ; ڱ; ڱ; ) ARABIC LETTER NGOEH MEDIAL FORM +FB9E;FB9E;FB9E;06BA;06BA; # (ﮞ; ﮞ; ﮞ; ں; ں; ) ARABIC LETTER NOON GHUNNA ISOLATED FORM +FB9F;FB9F;FB9F;06BA;06BA; # (ﮟ; ﮟ; ﮟ; ں; ں; ) ARABIC LETTER NOON GHUNNA FINAL FORM +FBA0;FBA0;FBA0;06BB;06BB; # (ﮠ; ﮠ; ﮠ; ڻ; ڻ; ) ARABIC LETTER RNOON ISOLATED FORM +FBA1;FBA1;FBA1;06BB;06BB; # (ﮡ; ﮡ; ﮡ; ڻ; ڻ; ) ARABIC LETTER RNOON FINAL FORM +FBA2;FBA2;FBA2;06BB;06BB; # (ﮢ; ﮢ; ﮢ; ڻ; ڻ; ) ARABIC LETTER RNOON INITIAL FORM +FBA3;FBA3;FBA3;06BB;06BB; # (ﮣ; ﮣ; ﮣ; ڻ; ڻ; ) ARABIC LETTER RNOON MEDIAL FORM +FBA4;FBA4;FBA4;06C0;06D5 0654; # (ﮤ; ﮤ; ﮤ; ۀ; ە◌ٔ; ) ARABIC LETTER HEH WITH YEH ABOVE ISOLATED FORM +FBA5;FBA5;FBA5;06C0;06D5 0654; # (ﮥ; ﮥ; ﮥ; ۀ; ە◌ٔ; ) ARABIC LETTER HEH WITH YEH ABOVE FINAL FORM +FBA6;FBA6;FBA6;06C1;06C1; # (ﮦ; ﮦ; ﮦ; ہ; ہ; ) ARABIC LETTER HEH GOAL ISOLATED FORM +FBA7;FBA7;FBA7;06C1;06C1; # (ﮧ; ﮧ; ﮧ; ہ; ہ; ) ARABIC LETTER HEH GOAL FINAL FORM +FBA8;FBA8;FBA8;06C1;06C1; # (ﮨ; ﮨ; ﮨ; ہ; ہ; ) ARABIC LETTER HEH GOAL INITIAL FORM +FBA9;FBA9;FBA9;06C1;06C1; # (ﮩ; ﮩ; ﮩ; ہ; ہ; ) ARABIC LETTER HEH GOAL MEDIAL FORM +FBAA;FBAA;FBAA;06BE;06BE; # (ﮪ; ﮪ; ﮪ; ھ; ھ; ) ARABIC LETTER HEH DOACHASHMEE ISOLATED FORM +FBAB;FBAB;FBAB;06BE;06BE; # (ﮫ; ﮫ; ﮫ; ھ; ھ; ) ARABIC LETTER HEH DOACHASHMEE FINAL FORM +FBAC;FBAC;FBAC;06BE;06BE; # (ﮬ; ﮬ; ﮬ; ھ; ھ; ) ARABIC LETTER HEH DOACHASHMEE INITIAL FORM +FBAD;FBAD;FBAD;06BE;06BE; # (ﮭ; ﮭ; ﮭ; ھ; ھ; ) ARABIC LETTER HEH DOACHASHMEE MEDIAL FORM +FBAE;FBAE;FBAE;06D2;06D2; # (ﮮ; ﮮ; ﮮ; ے; ے; ) ARABIC LETTER YEH BARREE ISOLATED FORM +FBAF;FBAF;FBAF;06D2;06D2; # (ﮯ; ﮯ; ﮯ; ے; ے; ) ARABIC LETTER YEH BARREE FINAL FORM +FBB0;FBB0;FBB0;06D3;06D2 0654; # (ﮰ; ﮰ; ﮰ; ۓ; ے◌ٔ; ) ARABIC LETTER YEH BARREE WITH HAMZA ABOVE ISOLATED FORM +FBB1;FBB1;FBB1;06D3;06D2 0654; # (ﮱ; ﮱ; ﮱ; ۓ; ے◌ٔ; ) ARABIC LETTER YEH BARREE WITH HAMZA ABOVE FINAL FORM +FBD3;FBD3;FBD3;06AD;06AD; # (ﯓ; ﯓ; ﯓ; ڭ; ڭ; ) ARABIC LETTER NG ISOLATED FORM +FBD4;FBD4;FBD4;06AD;06AD; # (ﯔ; ﯔ; ﯔ; ڭ; ڭ; ) ARABIC LETTER NG FINAL FORM +FBD5;FBD5;FBD5;06AD;06AD; # (ﯕ; ﯕ; ﯕ; ڭ; ڭ; ) ARABIC LETTER NG INITIAL FORM +FBD6;FBD6;FBD6;06AD;06AD; # (ﯖ; ﯖ; ﯖ; ڭ; ڭ; ) ARABIC LETTER NG MEDIAL FORM +FBD7;FBD7;FBD7;06C7;06C7; # (ﯗ; ﯗ; ﯗ; ۇ; ۇ; ) ARABIC LETTER U ISOLATED FORM +FBD8;FBD8;FBD8;06C7;06C7; # (ﯘ; ﯘ; ﯘ; ۇ; ۇ; ) ARABIC LETTER U FINAL FORM +FBD9;FBD9;FBD9;06C6;06C6; # (ﯙ; ﯙ; ﯙ; ۆ; ۆ; ) ARABIC LETTER OE ISOLATED FORM +FBDA;FBDA;FBDA;06C6;06C6; # (ﯚ; ﯚ; ﯚ; ۆ; ۆ; ) ARABIC LETTER OE FINAL FORM +FBDB;FBDB;FBDB;06C8;06C8; # (ﯛ; ﯛ; ﯛ; ۈ; ۈ; ) ARABIC LETTER YU ISOLATED FORM +FBDC;FBDC;FBDC;06C8;06C8; # (ﯜ; ﯜ; ﯜ; ۈ; ۈ; ) ARABIC LETTER YU FINAL FORM +FBDD;FBDD;FBDD;06C7 0674;06C7 0674; # (ﯝ; ﯝ; ﯝ; ۇٴ; ۇٴ; ) ARABIC LETTER U WITH HAMZA ABOVE ISOLATED FORM +FBDE;FBDE;FBDE;06CB;06CB; # (ﯞ; ﯞ; ﯞ; ۋ; ۋ; ) ARABIC LETTER VE ISOLATED FORM +FBDF;FBDF;FBDF;06CB;06CB; # (ﯟ; ﯟ; ﯟ; ۋ; ۋ; ) ARABIC LETTER VE FINAL FORM +FBE0;FBE0;FBE0;06C5;06C5; # (ﯠ; ﯠ; ﯠ; ۅ; ۅ; ) ARABIC LETTER KIRGHIZ OE ISOLATED FORM +FBE1;FBE1;FBE1;06C5;06C5; # (ﯡ; ﯡ; ﯡ; ۅ; ۅ; ) ARABIC LETTER KIRGHIZ OE FINAL FORM +FBE2;FBE2;FBE2;06C9;06C9; # (ﯢ; ﯢ; ﯢ; ۉ; ۉ; ) ARABIC LETTER KIRGHIZ YU ISOLATED FORM +FBE3;FBE3;FBE3;06C9;06C9; # (ﯣ; ﯣ; ﯣ; ۉ; ۉ; ) ARABIC LETTER KIRGHIZ YU FINAL FORM +FBE4;FBE4;FBE4;06D0;06D0; # (ﯤ; ﯤ; ﯤ; ې; ې; ) ARABIC LETTER E ISOLATED FORM +FBE5;FBE5;FBE5;06D0;06D0; # (ﯥ; ﯥ; ﯥ; ې; ې; ) ARABIC LETTER E FINAL FORM +FBE6;FBE6;FBE6;06D0;06D0; # (ﯦ; ﯦ; ﯦ; ې; ې; ) ARABIC LETTER E INITIAL FORM +FBE7;FBE7;FBE7;06D0;06D0; # (ﯧ; ﯧ; ﯧ; ې; ې; ) ARABIC LETTER E MEDIAL FORM +FBE8;FBE8;FBE8;0649;0649; # (ﯨ; ﯨ; ﯨ; ى; ى; ) ARABIC LETTER UIGHUR KAZAKH KIRGHIZ ALEF MAKSURA INITIAL FORM +FBE9;FBE9;FBE9;0649;0649; # (ﯩ; ﯩ; ﯩ; ى; ى; ) ARABIC LETTER UIGHUR KAZAKH KIRGHIZ ALEF MAKSURA MEDIAL FORM +FBEA;FBEA;FBEA;0626 0627;064A 0654 0627; # (ﯪ; ﯪ; ﯪ; ئا; ي◌ٔا; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH ALEF ISOLATED FORM +FBEB;FBEB;FBEB;0626 0627;064A 0654 0627; # (ﯫ; ﯫ; ﯫ; ئا; ي◌ٔا; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH ALEF FINAL FORM +FBEC;FBEC;FBEC;0626 06D5;064A 0654 06D5; # (ﯬ; ﯬ; ﯬ; ئە; ي◌ٔە; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH AE ISOLATED FORM +FBED;FBED;FBED;0626 06D5;064A 0654 06D5; # (ﯭ; ﯭ; ﯭ; ئە; ي◌ٔە; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH AE FINAL FORM +FBEE;FBEE;FBEE;0626 0648;064A 0654 0648; # (ﯮ; ﯮ; ﯮ; ئو; ي◌ٔو; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH WAW ISOLATED FORM +FBEF;FBEF;FBEF;0626 0648;064A 0654 0648; # (ﯯ; ﯯ; ﯯ; ئو; ي◌ٔو; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH WAW FINAL FORM +FBF0;FBF0;FBF0;0626 06C7;064A 0654 06C7; # (ﯰ; ﯰ; ﯰ; ئۇ; ي◌ٔۇ; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH U ISOLATED FORM +FBF1;FBF1;FBF1;0626 06C7;064A 0654 06C7; # (ﯱ; ﯱ; ﯱ; ئۇ; ي◌ٔۇ; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH U FINAL FORM +FBF2;FBF2;FBF2;0626 06C6;064A 0654 06C6; # (ﯲ; ﯲ; ﯲ; ئۆ; ي◌ٔۆ; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH OE ISOLATED FORM +FBF3;FBF3;FBF3;0626 06C6;064A 0654 06C6; # (ﯳ; ﯳ; ﯳ; ئۆ; ي◌ٔۆ; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH OE FINAL FORM +FBF4;FBF4;FBF4;0626 06C8;064A 0654 06C8; # (ﯴ; ﯴ; ﯴ; ئۈ; ي◌ٔۈ; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH YU ISOLATED FORM +FBF5;FBF5;FBF5;0626 06C8;064A 0654 06C8; # (ﯵ; ﯵ; ﯵ; ئۈ; ي◌ٔۈ; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH YU FINAL FORM +FBF6;FBF6;FBF6;0626 06D0;064A 0654 06D0; # (ﯶ; ﯶ; ﯶ; ئې; ي◌ٔې; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH E ISOLATED FORM +FBF7;FBF7;FBF7;0626 06D0;064A 0654 06D0; # (ﯷ; ﯷ; ﯷ; ئې; ي◌ٔې; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH E FINAL FORM +FBF8;FBF8;FBF8;0626 06D0;064A 0654 06D0; # (ﯸ; ﯸ; ﯸ; ئې; ي◌ٔې; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH E INITIAL FORM +FBF9;FBF9;FBF9;0626 0649;064A 0654 0649; # (ﯹ; ﯹ; ﯹ; ئى; ي◌ٔى; ) ARABIC LIGATURE UIGHUR KIRGHIZ YEH WITH HAMZA ABOVE WITH ALEF MAKSURA ISOLATED FORM +FBFA;FBFA;FBFA;0626 0649;064A 0654 0649; # (ﯺ; ﯺ; ﯺ; ئى; ي◌ٔى; ) ARABIC LIGATURE UIGHUR KIRGHIZ YEH WITH HAMZA ABOVE WITH ALEF MAKSURA FINAL FORM +FBFB;FBFB;FBFB;0626 0649;064A 0654 0649; # (ﯻ; ﯻ; ﯻ; ئى; ي◌ٔى; ) ARABIC LIGATURE UIGHUR KIRGHIZ YEH WITH HAMZA ABOVE WITH ALEF MAKSURA INITIAL FORM +FBFC;FBFC;FBFC;06CC;06CC; # (ﯼ; ﯼ; ﯼ; ی; ی; ) ARABIC LETTER FARSI YEH ISOLATED FORM +FBFD;FBFD;FBFD;06CC;06CC; # (ﯽ; ﯽ; ﯽ; ی; ی; ) ARABIC LETTER FARSI YEH FINAL FORM +FBFE;FBFE;FBFE;06CC;06CC; # (ﯾ; ﯾ; ﯾ; ی; ی; ) ARABIC LETTER FARSI YEH INITIAL FORM +FBFF;FBFF;FBFF;06CC;06CC; # (ﯿ; ﯿ; ﯿ; ی; ی; ) ARABIC LETTER FARSI YEH MEDIAL FORM +FC00;FC00;FC00;0626 062C;064A 0654 062C; # (ﰀ; ﰀ; ﰀ; ئج; ي◌ٔج; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH JEEM ISOLATED FORM +FC01;FC01;FC01;0626 062D;064A 0654 062D; # (ﰁ; ﰁ; ﰁ; ئح; ي◌ٔح; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH HAH ISOLATED FORM +FC02;FC02;FC02;0626 0645;064A 0654 0645; # (ﰂ; ﰂ; ﰂ; ئم; ي◌ٔم; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH MEEM ISOLATED FORM +FC03;FC03;FC03;0626 0649;064A 0654 0649; # (ﰃ; ﰃ; ﰃ; ئى; ي◌ٔى; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH ALEF MAKSURA ISOLATED FORM +FC04;FC04;FC04;0626 064A;064A 0654 064A; # (ﰄ; ﰄ; ﰄ; ئي; ي◌ٔي; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH YEH ISOLATED FORM +FC05;FC05;FC05;0628 062C;0628 062C; # (ﰅ; ﰅ; ﰅ; بج; بج; ) ARABIC LIGATURE BEH WITH JEEM ISOLATED FORM +FC06;FC06;FC06;0628 062D;0628 062D; # (ﰆ; ﰆ; ﰆ; بح; بح; ) ARABIC LIGATURE BEH WITH HAH ISOLATED FORM +FC07;FC07;FC07;0628 062E;0628 062E; # (ﰇ; ﰇ; ﰇ; بخ; بخ; ) ARABIC LIGATURE BEH WITH KHAH ISOLATED FORM +FC08;FC08;FC08;0628 0645;0628 0645; # (ﰈ; ﰈ; ﰈ; بم; بم; ) ARABIC LIGATURE BEH WITH MEEM ISOLATED FORM +FC09;FC09;FC09;0628 0649;0628 0649; # (ﰉ; ﰉ; ﰉ; بى; بى; ) ARABIC LIGATURE BEH WITH ALEF MAKSURA ISOLATED FORM +FC0A;FC0A;FC0A;0628 064A;0628 064A; # (ﰊ; ﰊ; ﰊ; بي; بي; ) ARABIC LIGATURE BEH WITH YEH ISOLATED FORM +FC0B;FC0B;FC0B;062A 062C;062A 062C; # (ﰋ; ﰋ; ﰋ; تج; تج; ) ARABIC LIGATURE TEH WITH JEEM ISOLATED FORM +FC0C;FC0C;FC0C;062A 062D;062A 062D; # (ﰌ; ﰌ; ﰌ; تح; تح; ) ARABIC LIGATURE TEH WITH HAH ISOLATED FORM +FC0D;FC0D;FC0D;062A 062E;062A 062E; # (ﰍ; ﰍ; ﰍ; تخ; تخ; ) ARABIC LIGATURE TEH WITH KHAH ISOLATED FORM +FC0E;FC0E;FC0E;062A 0645;062A 0645; # (ﰎ; ﰎ; ﰎ; تم; تم; ) ARABIC LIGATURE TEH WITH MEEM ISOLATED FORM +FC0F;FC0F;FC0F;062A 0649;062A 0649; # (ﰏ; ﰏ; ﰏ; تى; تى; ) ARABIC LIGATURE TEH WITH ALEF MAKSURA ISOLATED FORM +FC10;FC10;FC10;062A 064A;062A 064A; # (ﰐ; ﰐ; ﰐ; تي; تي; ) ARABIC LIGATURE TEH WITH YEH ISOLATED FORM +FC11;FC11;FC11;062B 062C;062B 062C; # (ﰑ; ﰑ; ﰑ; ثج; ثج; ) ARABIC LIGATURE THEH WITH JEEM ISOLATED FORM +FC12;FC12;FC12;062B 0645;062B 0645; # (ﰒ; ﰒ; ﰒ; ثم; ثم; ) ARABIC LIGATURE THEH WITH MEEM ISOLATED FORM +FC13;FC13;FC13;062B 0649;062B 0649; # (ﰓ; ﰓ; ﰓ; ثى; ثى; ) ARABIC LIGATURE THEH WITH ALEF MAKSURA ISOLATED FORM +FC14;FC14;FC14;062B 064A;062B 064A; # (ﰔ; ﰔ; ﰔ; ثي; ثي; ) ARABIC LIGATURE THEH WITH YEH ISOLATED FORM +FC15;FC15;FC15;062C 062D;062C 062D; # (ﰕ; ﰕ; ﰕ; جح; جح; ) ARABIC LIGATURE JEEM WITH HAH ISOLATED FORM +FC16;FC16;FC16;062C 0645;062C 0645; # (ﰖ; ﰖ; ﰖ; جم; جم; ) ARABIC LIGATURE JEEM WITH MEEM ISOLATED FORM +FC17;FC17;FC17;062D 062C;062D 062C; # (ﰗ; ﰗ; ﰗ; حج; حج; ) ARABIC LIGATURE HAH WITH JEEM ISOLATED FORM +FC18;FC18;FC18;062D 0645;062D 0645; # (ﰘ; ﰘ; ﰘ; حم; حم; ) ARABIC LIGATURE HAH WITH MEEM ISOLATED FORM +FC19;FC19;FC19;062E 062C;062E 062C; # (ﰙ; ﰙ; ﰙ; خج; خج; ) ARABIC LIGATURE KHAH WITH JEEM ISOLATED FORM +FC1A;FC1A;FC1A;062E 062D;062E 062D; # (ﰚ; ﰚ; ﰚ; خح; خح; ) ARABIC LIGATURE KHAH WITH HAH ISOLATED FORM +FC1B;FC1B;FC1B;062E 0645;062E 0645; # (ﰛ; ﰛ; ﰛ; خم; خم; ) ARABIC LIGATURE KHAH WITH MEEM ISOLATED FORM +FC1C;FC1C;FC1C;0633 062C;0633 062C; # (ﰜ; ﰜ; ﰜ; سج; سج; ) ARABIC LIGATURE SEEN WITH JEEM ISOLATED FORM +FC1D;FC1D;FC1D;0633 062D;0633 062D; # (ﰝ; ﰝ; ﰝ; سح; سح; ) ARABIC LIGATURE SEEN WITH HAH ISOLATED FORM +FC1E;FC1E;FC1E;0633 062E;0633 062E; # (ﰞ; ﰞ; ﰞ; سخ; سخ; ) ARABIC LIGATURE SEEN WITH KHAH ISOLATED FORM +FC1F;FC1F;FC1F;0633 0645;0633 0645; # (ﰟ; ﰟ; ﰟ; سم; سم; ) ARABIC LIGATURE SEEN WITH MEEM ISOLATED FORM +FC20;FC20;FC20;0635 062D;0635 062D; # (ﰠ; ﰠ; ﰠ; صح; صح; ) ARABIC LIGATURE SAD WITH HAH ISOLATED FORM +FC21;FC21;FC21;0635 0645;0635 0645; # (ﰡ; ﰡ; ﰡ; صم; صم; ) ARABIC LIGATURE SAD WITH MEEM ISOLATED FORM +FC22;FC22;FC22;0636 062C;0636 062C; # (ﰢ; ﰢ; ﰢ; ضج; ضج; ) ARABIC LIGATURE DAD WITH JEEM ISOLATED FORM +FC23;FC23;FC23;0636 062D;0636 062D; # (ﰣ; ﰣ; ﰣ; ضح; ضح; ) ARABIC LIGATURE DAD WITH HAH ISOLATED FORM +FC24;FC24;FC24;0636 062E;0636 062E; # (ﰤ; ﰤ; ﰤ; ضخ; ضخ; ) ARABIC LIGATURE DAD WITH KHAH ISOLATED FORM +FC25;FC25;FC25;0636 0645;0636 0645; # (ﰥ; ﰥ; ﰥ; ضم; ضم; ) ARABIC LIGATURE DAD WITH MEEM ISOLATED FORM +FC26;FC26;FC26;0637 062D;0637 062D; # (ﰦ; ﰦ; ﰦ; طح; طح; ) ARABIC LIGATURE TAH WITH HAH ISOLATED FORM +FC27;FC27;FC27;0637 0645;0637 0645; # (ﰧ; ﰧ; ﰧ; طم; طم; ) ARABIC LIGATURE TAH WITH MEEM ISOLATED FORM +FC28;FC28;FC28;0638 0645;0638 0645; # (ﰨ; ﰨ; ﰨ; ظم; ظم; ) ARABIC LIGATURE ZAH WITH MEEM ISOLATED FORM +FC29;FC29;FC29;0639 062C;0639 062C; # (ﰩ; ﰩ; ﰩ; عج; عج; ) ARABIC LIGATURE AIN WITH JEEM ISOLATED FORM +FC2A;FC2A;FC2A;0639 0645;0639 0645; # (ﰪ; ﰪ; ﰪ; عم; عم; ) ARABIC LIGATURE AIN WITH MEEM ISOLATED FORM +FC2B;FC2B;FC2B;063A 062C;063A 062C; # (ﰫ; ﰫ; ﰫ; غج; غج; ) ARABIC LIGATURE GHAIN WITH JEEM ISOLATED FORM +FC2C;FC2C;FC2C;063A 0645;063A 0645; # (ﰬ; ﰬ; ﰬ; غم; غم; ) ARABIC LIGATURE GHAIN WITH MEEM ISOLATED FORM +FC2D;FC2D;FC2D;0641 062C;0641 062C; # (ﰭ; ﰭ; ﰭ; فج; فج; ) ARABIC LIGATURE FEH WITH JEEM ISOLATED FORM +FC2E;FC2E;FC2E;0641 062D;0641 062D; # (ﰮ; ﰮ; ﰮ; فح; فح; ) ARABIC LIGATURE FEH WITH HAH ISOLATED FORM +FC2F;FC2F;FC2F;0641 062E;0641 062E; # (ﰯ; ﰯ; ﰯ; فخ; فخ; ) ARABIC LIGATURE FEH WITH KHAH ISOLATED FORM +FC30;FC30;FC30;0641 0645;0641 0645; # (ﰰ; ﰰ; ﰰ; فم; فم; ) ARABIC LIGATURE FEH WITH MEEM ISOLATED FORM +FC31;FC31;FC31;0641 0649;0641 0649; # (ﰱ; ﰱ; ﰱ; فى; فى; ) ARABIC LIGATURE FEH WITH ALEF MAKSURA ISOLATED FORM +FC32;FC32;FC32;0641 064A;0641 064A; # (ﰲ; ﰲ; ﰲ; في; في; ) ARABIC LIGATURE FEH WITH YEH ISOLATED FORM +FC33;FC33;FC33;0642 062D;0642 062D; # (ﰳ; ﰳ; ﰳ; قح; قح; ) ARABIC LIGATURE QAF WITH HAH ISOLATED FORM +FC34;FC34;FC34;0642 0645;0642 0645; # (ﰴ; ﰴ; ﰴ; قم; قم; ) ARABIC LIGATURE QAF WITH MEEM ISOLATED FORM +FC35;FC35;FC35;0642 0649;0642 0649; # (ﰵ; ﰵ; ﰵ; قى; قى; ) ARABIC LIGATURE QAF WITH ALEF MAKSURA ISOLATED FORM +FC36;FC36;FC36;0642 064A;0642 064A; # (ﰶ; ﰶ; ﰶ; قي; قي; ) ARABIC LIGATURE QAF WITH YEH ISOLATED FORM +FC37;FC37;FC37;0643 0627;0643 0627; # (ﰷ; ﰷ; ﰷ; كا; كا; ) ARABIC LIGATURE KAF WITH ALEF ISOLATED FORM +FC38;FC38;FC38;0643 062C;0643 062C; # (ﰸ; ﰸ; ﰸ; كج; كج; ) ARABIC LIGATURE KAF WITH JEEM ISOLATED FORM +FC39;FC39;FC39;0643 062D;0643 062D; # (ﰹ; ﰹ; ﰹ; كح; كح; ) ARABIC LIGATURE KAF WITH HAH ISOLATED FORM +FC3A;FC3A;FC3A;0643 062E;0643 062E; # (ﰺ; ﰺ; ﰺ; كخ; كخ; ) ARABIC LIGATURE KAF WITH KHAH ISOLATED FORM +FC3B;FC3B;FC3B;0643 0644;0643 0644; # (ﰻ; ﰻ; ﰻ; كل; كل; ) ARABIC LIGATURE KAF WITH LAM ISOLATED FORM +FC3C;FC3C;FC3C;0643 0645;0643 0645; # (ﰼ; ﰼ; ﰼ; كم; كم; ) ARABIC LIGATURE KAF WITH MEEM ISOLATED FORM +FC3D;FC3D;FC3D;0643 0649;0643 0649; # (ﰽ; ﰽ; ﰽ; كى; كى; ) ARABIC LIGATURE KAF WITH ALEF MAKSURA ISOLATED FORM +FC3E;FC3E;FC3E;0643 064A;0643 064A; # (ﰾ; ﰾ; ﰾ; كي; كي; ) ARABIC LIGATURE KAF WITH YEH ISOLATED FORM +FC3F;FC3F;FC3F;0644 062C;0644 062C; # (ﰿ; ﰿ; ﰿ; لج; لج; ) ARABIC LIGATURE LAM WITH JEEM ISOLATED FORM +FC40;FC40;FC40;0644 062D;0644 062D; # (ﱀ; ﱀ; ﱀ; لح; لح; ) ARABIC LIGATURE LAM WITH HAH ISOLATED FORM +FC41;FC41;FC41;0644 062E;0644 062E; # (ﱁ; ﱁ; ﱁ; لخ; لخ; ) ARABIC LIGATURE LAM WITH KHAH ISOLATED FORM +FC42;FC42;FC42;0644 0645;0644 0645; # (ﱂ; ﱂ; ﱂ; لم; لم; ) ARABIC LIGATURE LAM WITH MEEM ISOLATED FORM +FC43;FC43;FC43;0644 0649;0644 0649; # (ﱃ; ﱃ; ﱃ; لى; لى; ) ARABIC LIGATURE LAM WITH ALEF MAKSURA ISOLATED FORM +FC44;FC44;FC44;0644 064A;0644 064A; # (ﱄ; ﱄ; ﱄ; لي; لي; ) ARABIC LIGATURE LAM WITH YEH ISOLATED FORM +FC45;FC45;FC45;0645 062C;0645 062C; # (ﱅ; ﱅ; ﱅ; مج; مج; ) ARABIC LIGATURE MEEM WITH JEEM ISOLATED FORM +FC46;FC46;FC46;0645 062D;0645 062D; # (ﱆ; ﱆ; ﱆ; مح; مح; ) ARABIC LIGATURE MEEM WITH HAH ISOLATED FORM +FC47;FC47;FC47;0645 062E;0645 062E; # (ﱇ; ﱇ; ﱇ; مخ; مخ; ) ARABIC LIGATURE MEEM WITH KHAH ISOLATED FORM +FC48;FC48;FC48;0645 0645;0645 0645; # (ﱈ; ﱈ; ﱈ; مم; مم; ) ARABIC LIGATURE MEEM WITH MEEM ISOLATED FORM +FC49;FC49;FC49;0645 0649;0645 0649; # (ﱉ; ﱉ; ﱉ; مى; مى; ) ARABIC LIGATURE MEEM WITH ALEF MAKSURA ISOLATED FORM +FC4A;FC4A;FC4A;0645 064A;0645 064A; # (ﱊ; ﱊ; ﱊ; مي; مي; ) ARABIC LIGATURE MEEM WITH YEH ISOLATED FORM +FC4B;FC4B;FC4B;0646 062C;0646 062C; # (ﱋ; ﱋ; ﱋ; نج; نج; ) ARABIC LIGATURE NOON WITH JEEM ISOLATED FORM +FC4C;FC4C;FC4C;0646 062D;0646 062D; # (ﱌ; ﱌ; ﱌ; نح; نح; ) ARABIC LIGATURE NOON WITH HAH ISOLATED FORM +FC4D;FC4D;FC4D;0646 062E;0646 062E; # (ﱍ; ﱍ; ﱍ; نخ; نخ; ) ARABIC LIGATURE NOON WITH KHAH ISOLATED FORM +FC4E;FC4E;FC4E;0646 0645;0646 0645; # (ﱎ; ﱎ; ﱎ; نم; نم; ) ARABIC LIGATURE NOON WITH MEEM ISOLATED FORM +FC4F;FC4F;FC4F;0646 0649;0646 0649; # (ﱏ; ﱏ; ﱏ; نى; نى; ) ARABIC LIGATURE NOON WITH ALEF MAKSURA ISOLATED FORM +FC50;FC50;FC50;0646 064A;0646 064A; # (ﱐ; ﱐ; ﱐ; ني; ني; ) ARABIC LIGATURE NOON WITH YEH ISOLATED FORM +FC51;FC51;FC51;0647 062C;0647 062C; # (ﱑ; ﱑ; ﱑ; هج; هج; ) ARABIC LIGATURE HEH WITH JEEM ISOLATED FORM +FC52;FC52;FC52;0647 0645;0647 0645; # (ﱒ; ﱒ; ﱒ; هم; هم; ) ARABIC LIGATURE HEH WITH MEEM ISOLATED FORM +FC53;FC53;FC53;0647 0649;0647 0649; # (ﱓ; ﱓ; ﱓ; هى; هى; ) ARABIC LIGATURE HEH WITH ALEF MAKSURA ISOLATED FORM +FC54;FC54;FC54;0647 064A;0647 064A; # (ﱔ; ﱔ; ﱔ; هي; هي; ) ARABIC LIGATURE HEH WITH YEH ISOLATED FORM +FC55;FC55;FC55;064A 062C;064A 062C; # (ﱕ; ﱕ; ﱕ; يج; يج; ) ARABIC LIGATURE YEH WITH JEEM ISOLATED FORM +FC56;FC56;FC56;064A 062D;064A 062D; # (ﱖ; ﱖ; ﱖ; يح; يح; ) ARABIC LIGATURE YEH WITH HAH ISOLATED FORM +FC57;FC57;FC57;064A 062E;064A 062E; # (ﱗ; ﱗ; ﱗ; يخ; يخ; ) ARABIC LIGATURE YEH WITH KHAH ISOLATED FORM +FC58;FC58;FC58;064A 0645;064A 0645; # (ﱘ; ﱘ; ﱘ; يم; يم; ) ARABIC LIGATURE YEH WITH MEEM ISOLATED FORM +FC59;FC59;FC59;064A 0649;064A 0649; # (ﱙ; ﱙ; ﱙ; يى; يى; ) ARABIC LIGATURE YEH WITH ALEF MAKSURA ISOLATED FORM +FC5A;FC5A;FC5A;064A 064A;064A 064A; # (ﱚ; ﱚ; ﱚ; يي; يي; ) ARABIC LIGATURE YEH WITH YEH ISOLATED FORM +FC5B;FC5B;FC5B;0630 0670;0630 0670; # (ﱛ; ﱛ; ﱛ; ذ◌ٰ; ذ◌ٰ; ) ARABIC LIGATURE THAL WITH SUPERSCRIPT ALEF ISOLATED FORM +FC5C;FC5C;FC5C;0631 0670;0631 0670; # (ﱜ; ﱜ; ﱜ; ر◌ٰ; ر◌ٰ; ) ARABIC LIGATURE REH WITH SUPERSCRIPT ALEF ISOLATED FORM +FC5D;FC5D;FC5D;0649 0670;0649 0670; # (ﱝ; ﱝ; ﱝ; ى◌ٰ; ى◌ٰ; ) ARABIC LIGATURE ALEF MAKSURA WITH SUPERSCRIPT ALEF ISOLATED FORM +FC5E;FC5E;FC5E;0020 064C 0651;0020 064C 0651; # (ﱞ; ﱞ; ﱞ; ◌ٌ◌ّ; ◌ٌ◌ّ; ) ARABIC LIGATURE SHADDA WITH DAMMATAN ISOLATED FORM +FC5F;FC5F;FC5F;0020 064D 0651;0020 064D 0651; # (ﱟ; ﱟ; ﱟ; ◌ٍ◌ّ; ◌ٍ◌ّ; ) ARABIC LIGATURE SHADDA WITH KASRATAN ISOLATED FORM +FC60;FC60;FC60;0020 064E 0651;0020 064E 0651; # (ﱠ; ﱠ; ﱠ; ◌َ◌ّ; ◌َ◌ّ; ) ARABIC LIGATURE SHADDA WITH FATHA ISOLATED FORM +FC61;FC61;FC61;0020 064F 0651;0020 064F 0651; # (ﱡ; ﱡ; ﱡ; ◌ُ◌ّ; ◌ُ◌ّ; ) ARABIC LIGATURE SHADDA WITH DAMMA ISOLATED FORM +FC62;FC62;FC62;0020 0650 0651;0020 0650 0651; # (ﱢ; ﱢ; ﱢ; ◌ِ◌ّ; ◌ِ◌ّ; ) ARABIC LIGATURE SHADDA WITH KASRA ISOLATED FORM +FC63;FC63;FC63;0020 0651 0670;0020 0651 0670; # (ﱣ; ﱣ; ﱣ; ◌ّ◌ٰ; ◌ّ◌ٰ; ) ARABIC LIGATURE SHADDA WITH SUPERSCRIPT ALEF ISOLATED FORM +FC64;FC64;FC64;0626 0631;064A 0654 0631; # (ﱤ; ﱤ; ﱤ; ئر; ي◌ٔر; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH REH FINAL FORM +FC65;FC65;FC65;0626 0632;064A 0654 0632; # (ﱥ; ﱥ; ﱥ; ئز; ي◌ٔز; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH ZAIN FINAL FORM +FC66;FC66;FC66;0626 0645;064A 0654 0645; # (ﱦ; ﱦ; ﱦ; ئم; ي◌ٔم; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH MEEM FINAL FORM +FC67;FC67;FC67;0626 0646;064A 0654 0646; # (ﱧ; ﱧ; ﱧ; ئن; ي◌ٔن; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH NOON FINAL FORM +FC68;FC68;FC68;0626 0649;064A 0654 0649; # (ﱨ; ﱨ; ﱨ; ئى; ي◌ٔى; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH ALEF MAKSURA FINAL FORM +FC69;FC69;FC69;0626 064A;064A 0654 064A; # (ﱩ; ﱩ; ﱩ; ئي; ي◌ٔي; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH YEH FINAL FORM +FC6A;FC6A;FC6A;0628 0631;0628 0631; # (ﱪ; ﱪ; ﱪ; بر; بر; ) ARABIC LIGATURE BEH WITH REH FINAL FORM +FC6B;FC6B;FC6B;0628 0632;0628 0632; # (ﱫ; ﱫ; ﱫ; بز; بز; ) ARABIC LIGATURE BEH WITH ZAIN FINAL FORM +FC6C;FC6C;FC6C;0628 0645;0628 0645; # (ﱬ; ﱬ; ﱬ; بم; بم; ) ARABIC LIGATURE BEH WITH MEEM FINAL FORM +FC6D;FC6D;FC6D;0628 0646;0628 0646; # (ﱭ; ﱭ; ﱭ; بن; بن; ) ARABIC LIGATURE BEH WITH NOON FINAL FORM +FC6E;FC6E;FC6E;0628 0649;0628 0649; # (ﱮ; ﱮ; ﱮ; بى; بى; ) ARABIC LIGATURE BEH WITH ALEF MAKSURA FINAL FORM +FC6F;FC6F;FC6F;0628 064A;0628 064A; # (ﱯ; ﱯ; ﱯ; بي; بي; ) ARABIC LIGATURE BEH WITH YEH FINAL FORM +FC70;FC70;FC70;062A 0631;062A 0631; # (ﱰ; ﱰ; ﱰ; تر; تر; ) ARABIC LIGATURE TEH WITH REH FINAL FORM +FC71;FC71;FC71;062A 0632;062A 0632; # (ﱱ; ﱱ; ﱱ; تز; تز; ) ARABIC LIGATURE TEH WITH ZAIN FINAL FORM +FC72;FC72;FC72;062A 0645;062A 0645; # (ﱲ; ﱲ; ﱲ; تم; تم; ) ARABIC LIGATURE TEH WITH MEEM FINAL FORM +FC73;FC73;FC73;062A 0646;062A 0646; # (ﱳ; ﱳ; ﱳ; تن; تن; ) ARABIC LIGATURE TEH WITH NOON FINAL FORM +FC74;FC74;FC74;062A 0649;062A 0649; # (ﱴ; ﱴ; ﱴ; تى; تى; ) ARABIC LIGATURE TEH WITH ALEF MAKSURA FINAL FORM +FC75;FC75;FC75;062A 064A;062A 064A; # (ﱵ; ﱵ; ﱵ; تي; تي; ) ARABIC LIGATURE TEH WITH YEH FINAL FORM +FC76;FC76;FC76;062B 0631;062B 0631; # (ﱶ; ﱶ; ﱶ; ثر; ثر; ) ARABIC LIGATURE THEH WITH REH FINAL FORM +FC77;FC77;FC77;062B 0632;062B 0632; # (ﱷ; ﱷ; ﱷ; ثز; ثز; ) ARABIC LIGATURE THEH WITH ZAIN FINAL FORM +FC78;FC78;FC78;062B 0645;062B 0645; # (ﱸ; ﱸ; ﱸ; ثم; ثم; ) ARABIC LIGATURE THEH WITH MEEM FINAL FORM +FC79;FC79;FC79;062B 0646;062B 0646; # (ﱹ; ﱹ; ﱹ; ثن; ثن; ) ARABIC LIGATURE THEH WITH NOON FINAL FORM +FC7A;FC7A;FC7A;062B 0649;062B 0649; # (ﱺ; ﱺ; ﱺ; ثى; ثى; ) ARABIC LIGATURE THEH WITH ALEF MAKSURA FINAL FORM +FC7B;FC7B;FC7B;062B 064A;062B 064A; # (ﱻ; ﱻ; ﱻ; ثي; ثي; ) ARABIC LIGATURE THEH WITH YEH FINAL FORM +FC7C;FC7C;FC7C;0641 0649;0641 0649; # (ﱼ; ﱼ; ﱼ; فى; فى; ) ARABIC LIGATURE FEH WITH ALEF MAKSURA FINAL FORM +FC7D;FC7D;FC7D;0641 064A;0641 064A; # (ﱽ; ﱽ; ﱽ; في; في; ) ARABIC LIGATURE FEH WITH YEH FINAL FORM +FC7E;FC7E;FC7E;0642 0649;0642 0649; # (ﱾ; ﱾ; ﱾ; قى; قى; ) ARABIC LIGATURE QAF WITH ALEF MAKSURA FINAL FORM +FC7F;FC7F;FC7F;0642 064A;0642 064A; # (ﱿ; ﱿ; ﱿ; قي; قي; ) ARABIC LIGATURE QAF WITH YEH FINAL FORM +FC80;FC80;FC80;0643 0627;0643 0627; # (ﲀ; ﲀ; ﲀ; كا; كا; ) ARABIC LIGATURE KAF WITH ALEF FINAL FORM +FC81;FC81;FC81;0643 0644;0643 0644; # (ﲁ; ﲁ; ﲁ; كل; كل; ) ARABIC LIGATURE KAF WITH LAM FINAL FORM +FC82;FC82;FC82;0643 0645;0643 0645; # (ﲂ; ﲂ; ﲂ; كم; كم; ) ARABIC LIGATURE KAF WITH MEEM FINAL FORM +FC83;FC83;FC83;0643 0649;0643 0649; # (ﲃ; ﲃ; ﲃ; كى; كى; ) ARABIC LIGATURE KAF WITH ALEF MAKSURA FINAL FORM +FC84;FC84;FC84;0643 064A;0643 064A; # (ﲄ; ﲄ; ﲄ; كي; كي; ) ARABIC LIGATURE KAF WITH YEH FINAL FORM +FC85;FC85;FC85;0644 0645;0644 0645; # (ﲅ; ﲅ; ﲅ; لم; لم; ) ARABIC LIGATURE LAM WITH MEEM FINAL FORM +FC86;FC86;FC86;0644 0649;0644 0649; # (ﲆ; ﲆ; ﲆ; لى; لى; ) ARABIC LIGATURE LAM WITH ALEF MAKSURA FINAL FORM +FC87;FC87;FC87;0644 064A;0644 064A; # (ﲇ; ﲇ; ﲇ; لي; لي; ) ARABIC LIGATURE LAM WITH YEH FINAL FORM +FC88;FC88;FC88;0645 0627;0645 0627; # (ﲈ; ﲈ; ﲈ; ما; ما; ) ARABIC LIGATURE MEEM WITH ALEF FINAL FORM +FC89;FC89;FC89;0645 0645;0645 0645; # (ﲉ; ﲉ; ﲉ; مم; مم; ) ARABIC LIGATURE MEEM WITH MEEM FINAL FORM +FC8A;FC8A;FC8A;0646 0631;0646 0631; # (ﲊ; ﲊ; ﲊ; نر; نر; ) ARABIC LIGATURE NOON WITH REH FINAL FORM +FC8B;FC8B;FC8B;0646 0632;0646 0632; # (ﲋ; ﲋ; ﲋ; نز; نز; ) ARABIC LIGATURE NOON WITH ZAIN FINAL FORM +FC8C;FC8C;FC8C;0646 0645;0646 0645; # (ﲌ; ﲌ; ﲌ; نم; نم; ) ARABIC LIGATURE NOON WITH MEEM FINAL FORM +FC8D;FC8D;FC8D;0646 0646;0646 0646; # (ﲍ; ﲍ; ﲍ; نن; نن; ) ARABIC LIGATURE NOON WITH NOON FINAL FORM +FC8E;FC8E;FC8E;0646 0649;0646 0649; # (ﲎ; ﲎ; ﲎ; نى; نى; ) ARABIC LIGATURE NOON WITH ALEF MAKSURA FINAL FORM +FC8F;FC8F;FC8F;0646 064A;0646 064A; # (ﲏ; ﲏ; ﲏ; ني; ني; ) ARABIC LIGATURE NOON WITH YEH FINAL FORM +FC90;FC90;FC90;0649 0670;0649 0670; # (ﲐ; ﲐ; ﲐ; ى◌ٰ; ى◌ٰ; ) ARABIC LIGATURE ALEF MAKSURA WITH SUPERSCRIPT ALEF FINAL FORM +FC91;FC91;FC91;064A 0631;064A 0631; # (ﲑ; ﲑ; ﲑ; ير; ير; ) ARABIC LIGATURE YEH WITH REH FINAL FORM +FC92;FC92;FC92;064A 0632;064A 0632; # (ﲒ; ﲒ; ﲒ; يز; يز; ) ARABIC LIGATURE YEH WITH ZAIN FINAL FORM +FC93;FC93;FC93;064A 0645;064A 0645; # (ﲓ; ﲓ; ﲓ; يم; يم; ) ARABIC LIGATURE YEH WITH MEEM FINAL FORM +FC94;FC94;FC94;064A 0646;064A 0646; # (ﲔ; ﲔ; ﲔ; ين; ين; ) ARABIC LIGATURE YEH WITH NOON FINAL FORM +FC95;FC95;FC95;064A 0649;064A 0649; # (ﲕ; ﲕ; ﲕ; يى; يى; ) ARABIC LIGATURE YEH WITH ALEF MAKSURA FINAL FORM +FC96;FC96;FC96;064A 064A;064A 064A; # (ﲖ; ﲖ; ﲖ; يي; يي; ) ARABIC LIGATURE YEH WITH YEH FINAL FORM +FC97;FC97;FC97;0626 062C;064A 0654 062C; # (ﲗ; ﲗ; ﲗ; ئج; ي◌ٔج; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH JEEM INITIAL FORM +FC98;FC98;FC98;0626 062D;064A 0654 062D; # (ﲘ; ﲘ; ﲘ; ئح; ي◌ٔح; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH HAH INITIAL FORM +FC99;FC99;FC99;0626 062E;064A 0654 062E; # (ﲙ; ﲙ; ﲙ; ئخ; ي◌ٔخ; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH KHAH INITIAL FORM +FC9A;FC9A;FC9A;0626 0645;064A 0654 0645; # (ﲚ; ﲚ; ﲚ; ئم; ي◌ٔم; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH MEEM INITIAL FORM +FC9B;FC9B;FC9B;0626 0647;064A 0654 0647; # (ﲛ; ﲛ; ﲛ; ئه; ي◌ٔه; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH HEH INITIAL FORM +FC9C;FC9C;FC9C;0628 062C;0628 062C; # (ﲜ; ﲜ; ﲜ; بج; بج; ) ARABIC LIGATURE BEH WITH JEEM INITIAL FORM +FC9D;FC9D;FC9D;0628 062D;0628 062D; # (ﲝ; ﲝ; ﲝ; بح; بح; ) ARABIC LIGATURE BEH WITH HAH INITIAL FORM +FC9E;FC9E;FC9E;0628 062E;0628 062E; # (ﲞ; ﲞ; ﲞ; بخ; بخ; ) ARABIC LIGATURE BEH WITH KHAH INITIAL FORM +FC9F;FC9F;FC9F;0628 0645;0628 0645; # (ﲟ; ﲟ; ﲟ; بم; بم; ) ARABIC LIGATURE BEH WITH MEEM INITIAL FORM +FCA0;FCA0;FCA0;0628 0647;0628 0647; # (ﲠ; ﲠ; ﲠ; به; به; ) ARABIC LIGATURE BEH WITH HEH INITIAL FORM +FCA1;FCA1;FCA1;062A 062C;062A 062C; # (ﲡ; ﲡ; ﲡ; تج; تج; ) ARABIC LIGATURE TEH WITH JEEM INITIAL FORM +FCA2;FCA2;FCA2;062A 062D;062A 062D; # (ﲢ; ﲢ; ﲢ; تح; تح; ) ARABIC LIGATURE TEH WITH HAH INITIAL FORM +FCA3;FCA3;FCA3;062A 062E;062A 062E; # (ﲣ; ﲣ; ﲣ; تخ; تخ; ) ARABIC LIGATURE TEH WITH KHAH INITIAL FORM +FCA4;FCA4;FCA4;062A 0645;062A 0645; # (ﲤ; ﲤ; ﲤ; تم; تم; ) ARABIC LIGATURE TEH WITH MEEM INITIAL FORM +FCA5;FCA5;FCA5;062A 0647;062A 0647; # (ﲥ; ﲥ; ﲥ; ته; ته; ) ARABIC LIGATURE TEH WITH HEH INITIAL FORM +FCA6;FCA6;FCA6;062B 0645;062B 0645; # (ﲦ; ﲦ; ﲦ; ثم; ثم; ) ARABIC LIGATURE THEH WITH MEEM INITIAL FORM +FCA7;FCA7;FCA7;062C 062D;062C 062D; # (ﲧ; ﲧ; ﲧ; جح; جح; ) ARABIC LIGATURE JEEM WITH HAH INITIAL FORM +FCA8;FCA8;FCA8;062C 0645;062C 0645; # (ﲨ; ﲨ; ﲨ; جم; جم; ) ARABIC LIGATURE JEEM WITH MEEM INITIAL FORM +FCA9;FCA9;FCA9;062D 062C;062D 062C; # (ﲩ; ﲩ; ﲩ; حج; حج; ) ARABIC LIGATURE HAH WITH JEEM INITIAL FORM +FCAA;FCAA;FCAA;062D 0645;062D 0645; # (ﲪ; ﲪ; ﲪ; حم; حم; ) ARABIC LIGATURE HAH WITH MEEM INITIAL FORM +FCAB;FCAB;FCAB;062E 062C;062E 062C; # (ﲫ; ﲫ; ﲫ; خج; خج; ) ARABIC LIGATURE KHAH WITH JEEM INITIAL FORM +FCAC;FCAC;FCAC;062E 0645;062E 0645; # (ﲬ; ﲬ; ﲬ; خم; خم; ) ARABIC LIGATURE KHAH WITH MEEM INITIAL FORM +FCAD;FCAD;FCAD;0633 062C;0633 062C; # (ﲭ; ﲭ; ﲭ; سج; سج; ) ARABIC LIGATURE SEEN WITH JEEM INITIAL FORM +FCAE;FCAE;FCAE;0633 062D;0633 062D; # (ﲮ; ﲮ; ﲮ; سح; سح; ) ARABIC LIGATURE SEEN WITH HAH INITIAL FORM +FCAF;FCAF;FCAF;0633 062E;0633 062E; # (ﲯ; ﲯ; ﲯ; سخ; سخ; ) ARABIC LIGATURE SEEN WITH KHAH INITIAL FORM +FCB0;FCB0;FCB0;0633 0645;0633 0645; # (ﲰ; ﲰ; ﲰ; سم; سم; ) ARABIC LIGATURE SEEN WITH MEEM INITIAL FORM +FCB1;FCB1;FCB1;0635 062D;0635 062D; # (ﲱ; ﲱ; ﲱ; صح; صح; ) ARABIC LIGATURE SAD WITH HAH INITIAL FORM +FCB2;FCB2;FCB2;0635 062E;0635 062E; # (ﲲ; ﲲ; ﲲ; صخ; صخ; ) ARABIC LIGATURE SAD WITH KHAH INITIAL FORM +FCB3;FCB3;FCB3;0635 0645;0635 0645; # (ﲳ; ﲳ; ﲳ; صم; صم; ) ARABIC LIGATURE SAD WITH MEEM INITIAL FORM +FCB4;FCB4;FCB4;0636 062C;0636 062C; # (ﲴ; ﲴ; ﲴ; ضج; ضج; ) ARABIC LIGATURE DAD WITH JEEM INITIAL FORM +FCB5;FCB5;FCB5;0636 062D;0636 062D; # (ﲵ; ﲵ; ﲵ; ضح; ضح; ) ARABIC LIGATURE DAD WITH HAH INITIAL FORM +FCB6;FCB6;FCB6;0636 062E;0636 062E; # (ﲶ; ﲶ; ﲶ; ضخ; ضخ; ) ARABIC LIGATURE DAD WITH KHAH INITIAL FORM +FCB7;FCB7;FCB7;0636 0645;0636 0645; # (ﲷ; ﲷ; ﲷ; ضم; ضم; ) ARABIC LIGATURE DAD WITH MEEM INITIAL FORM +FCB8;FCB8;FCB8;0637 062D;0637 062D; # (ﲸ; ﲸ; ﲸ; طح; طح; ) ARABIC LIGATURE TAH WITH HAH INITIAL FORM +FCB9;FCB9;FCB9;0638 0645;0638 0645; # (ﲹ; ﲹ; ﲹ; ظم; ظم; ) ARABIC LIGATURE ZAH WITH MEEM INITIAL FORM +FCBA;FCBA;FCBA;0639 062C;0639 062C; # (ﲺ; ﲺ; ﲺ; عج; عج; ) ARABIC LIGATURE AIN WITH JEEM INITIAL FORM +FCBB;FCBB;FCBB;0639 0645;0639 0645; # (ﲻ; ﲻ; ﲻ; عم; عم; ) ARABIC LIGATURE AIN WITH MEEM INITIAL FORM +FCBC;FCBC;FCBC;063A 062C;063A 062C; # (ﲼ; ﲼ; ﲼ; غج; غج; ) ARABIC LIGATURE GHAIN WITH JEEM INITIAL FORM +FCBD;FCBD;FCBD;063A 0645;063A 0645; # (ﲽ; ﲽ; ﲽ; غم; غم; ) ARABIC LIGATURE GHAIN WITH MEEM INITIAL FORM +FCBE;FCBE;FCBE;0641 062C;0641 062C; # (ﲾ; ﲾ; ﲾ; فج; فج; ) ARABIC LIGATURE FEH WITH JEEM INITIAL FORM +FCBF;FCBF;FCBF;0641 062D;0641 062D; # (ﲿ; ﲿ; ﲿ; فح; فح; ) ARABIC LIGATURE FEH WITH HAH INITIAL FORM +FCC0;FCC0;FCC0;0641 062E;0641 062E; # (ﳀ; ﳀ; ﳀ; فخ; فخ; ) ARABIC LIGATURE FEH WITH KHAH INITIAL FORM +FCC1;FCC1;FCC1;0641 0645;0641 0645; # (ﳁ; ﳁ; ﳁ; فم; فم; ) ARABIC LIGATURE FEH WITH MEEM INITIAL FORM +FCC2;FCC2;FCC2;0642 062D;0642 062D; # (ﳂ; ﳂ; ﳂ; قح; قح; ) ARABIC LIGATURE QAF WITH HAH INITIAL FORM +FCC3;FCC3;FCC3;0642 0645;0642 0645; # (ﳃ; ﳃ; ﳃ; قم; قم; ) ARABIC LIGATURE QAF WITH MEEM INITIAL FORM +FCC4;FCC4;FCC4;0643 062C;0643 062C; # (ﳄ; ﳄ; ﳄ; كج; كج; ) ARABIC LIGATURE KAF WITH JEEM INITIAL FORM +FCC5;FCC5;FCC5;0643 062D;0643 062D; # (ﳅ; ﳅ; ﳅ; كح; كح; ) ARABIC LIGATURE KAF WITH HAH INITIAL FORM +FCC6;FCC6;FCC6;0643 062E;0643 062E; # (ﳆ; ﳆ; ﳆ; كخ; كخ; ) ARABIC LIGATURE KAF WITH KHAH INITIAL FORM +FCC7;FCC7;FCC7;0643 0644;0643 0644; # (ﳇ; ﳇ; ﳇ; كل; كل; ) ARABIC LIGATURE KAF WITH LAM INITIAL FORM +FCC8;FCC8;FCC8;0643 0645;0643 0645; # (ﳈ; ﳈ; ﳈ; كم; كم; ) ARABIC LIGATURE KAF WITH MEEM INITIAL FORM +FCC9;FCC9;FCC9;0644 062C;0644 062C; # (ﳉ; ﳉ; ﳉ; لج; لج; ) ARABIC LIGATURE LAM WITH JEEM INITIAL FORM +FCCA;FCCA;FCCA;0644 062D;0644 062D; # (ﳊ; ﳊ; ﳊ; لح; لح; ) ARABIC LIGATURE LAM WITH HAH INITIAL FORM +FCCB;FCCB;FCCB;0644 062E;0644 062E; # (ﳋ; ﳋ; ﳋ; لخ; لخ; ) ARABIC LIGATURE LAM WITH KHAH INITIAL FORM +FCCC;FCCC;FCCC;0644 0645;0644 0645; # (ﳌ; ﳌ; ﳌ; لم; لم; ) ARABIC LIGATURE LAM WITH MEEM INITIAL FORM +FCCD;FCCD;FCCD;0644 0647;0644 0647; # (ﳍ; ﳍ; ﳍ; له; له; ) ARABIC LIGATURE LAM WITH HEH INITIAL FORM +FCCE;FCCE;FCCE;0645 062C;0645 062C; # (ﳎ; ﳎ; ﳎ; مج; مج; ) ARABIC LIGATURE MEEM WITH JEEM INITIAL FORM +FCCF;FCCF;FCCF;0645 062D;0645 062D; # (ﳏ; ﳏ; ﳏ; مح; مح; ) ARABIC LIGATURE MEEM WITH HAH INITIAL FORM +FCD0;FCD0;FCD0;0645 062E;0645 062E; # (ﳐ; ﳐ; ﳐ; مخ; مخ; ) ARABIC LIGATURE MEEM WITH KHAH INITIAL FORM +FCD1;FCD1;FCD1;0645 0645;0645 0645; # (ﳑ; ﳑ; ﳑ; مم; مم; ) ARABIC LIGATURE MEEM WITH MEEM INITIAL FORM +FCD2;FCD2;FCD2;0646 062C;0646 062C; # (ﳒ; ﳒ; ﳒ; نج; نج; ) ARABIC LIGATURE NOON WITH JEEM INITIAL FORM +FCD3;FCD3;FCD3;0646 062D;0646 062D; # (ﳓ; ﳓ; ﳓ; نح; نح; ) ARABIC LIGATURE NOON WITH HAH INITIAL FORM +FCD4;FCD4;FCD4;0646 062E;0646 062E; # (ﳔ; ﳔ; ﳔ; نخ; نخ; ) ARABIC LIGATURE NOON WITH KHAH INITIAL FORM +FCD5;FCD5;FCD5;0646 0645;0646 0645; # (ﳕ; ﳕ; ﳕ; نم; نم; ) ARABIC LIGATURE NOON WITH MEEM INITIAL FORM +FCD6;FCD6;FCD6;0646 0647;0646 0647; # (ﳖ; ﳖ; ﳖ; نه; نه; ) ARABIC LIGATURE NOON WITH HEH INITIAL FORM +FCD7;FCD7;FCD7;0647 062C;0647 062C; # (ﳗ; ﳗ; ﳗ; هج; هج; ) ARABIC LIGATURE HEH WITH JEEM INITIAL FORM +FCD8;FCD8;FCD8;0647 0645;0647 0645; # (ﳘ; ﳘ; ﳘ; هم; هم; ) ARABIC LIGATURE HEH WITH MEEM INITIAL FORM +FCD9;FCD9;FCD9;0647 0670;0647 0670; # (ﳙ; ﳙ; ﳙ; ه◌ٰ; ه◌ٰ; ) ARABIC LIGATURE HEH WITH SUPERSCRIPT ALEF INITIAL FORM +FCDA;FCDA;FCDA;064A 062C;064A 062C; # (ﳚ; ﳚ; ﳚ; يج; يج; ) ARABIC LIGATURE YEH WITH JEEM INITIAL FORM +FCDB;FCDB;FCDB;064A 062D;064A 062D; # (ﳛ; ﳛ; ﳛ; يح; يح; ) ARABIC LIGATURE YEH WITH HAH INITIAL FORM +FCDC;FCDC;FCDC;064A 062E;064A 062E; # (ﳜ; ﳜ; ﳜ; يخ; يخ; ) ARABIC LIGATURE YEH WITH KHAH INITIAL FORM +FCDD;FCDD;FCDD;064A 0645;064A 0645; # (ﳝ; ﳝ; ﳝ; يم; يم; ) ARABIC LIGATURE YEH WITH MEEM INITIAL FORM +FCDE;FCDE;FCDE;064A 0647;064A 0647; # (ﳞ; ﳞ; ﳞ; يه; يه; ) ARABIC LIGATURE YEH WITH HEH INITIAL FORM +FCDF;FCDF;FCDF;0626 0645;064A 0654 0645; # (ﳟ; ﳟ; ﳟ; ئم; ي◌ٔم; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH MEEM MEDIAL FORM +FCE0;FCE0;FCE0;0626 0647;064A 0654 0647; # (ﳠ; ﳠ; ﳠ; ئه; ي◌ٔه; ) ARABIC LIGATURE YEH WITH HAMZA ABOVE WITH HEH MEDIAL FORM +FCE1;FCE1;FCE1;0628 0645;0628 0645; # (ﳡ; ﳡ; ﳡ; بم; بم; ) ARABIC LIGATURE BEH WITH MEEM MEDIAL FORM +FCE2;FCE2;FCE2;0628 0647;0628 0647; # (ﳢ; ﳢ; ﳢ; به; به; ) ARABIC LIGATURE BEH WITH HEH MEDIAL FORM +FCE3;FCE3;FCE3;062A 0645;062A 0645; # (ﳣ; ﳣ; ﳣ; تم; تم; ) ARABIC LIGATURE TEH WITH MEEM MEDIAL FORM +FCE4;FCE4;FCE4;062A 0647;062A 0647; # (ﳤ; ﳤ; ﳤ; ته; ته; ) ARABIC LIGATURE TEH WITH HEH MEDIAL FORM +FCE5;FCE5;FCE5;062B 0645;062B 0645; # (ﳥ; ﳥ; ﳥ; ثم; ثم; ) ARABIC LIGATURE THEH WITH MEEM MEDIAL FORM +FCE6;FCE6;FCE6;062B 0647;062B 0647; # (ﳦ; ﳦ; ﳦ; ثه; ثه; ) ARABIC LIGATURE THEH WITH HEH MEDIAL FORM +FCE7;FCE7;FCE7;0633 0645;0633 0645; # (ﳧ; ﳧ; ﳧ; سم; سم; ) ARABIC LIGATURE SEEN WITH MEEM MEDIAL FORM +FCE8;FCE8;FCE8;0633 0647;0633 0647; # (ﳨ; ﳨ; ﳨ; سه; سه; ) ARABIC LIGATURE SEEN WITH HEH MEDIAL FORM +FCE9;FCE9;FCE9;0634 0645;0634 0645; # (ﳩ; ﳩ; ﳩ; شم; شم; ) ARABIC LIGATURE SHEEN WITH MEEM MEDIAL FORM +FCEA;FCEA;FCEA;0634 0647;0634 0647; # (ﳪ; ﳪ; ﳪ; شه; شه; ) ARABIC LIGATURE SHEEN WITH HEH MEDIAL FORM +FCEB;FCEB;FCEB;0643 0644;0643 0644; # (ﳫ; ﳫ; ﳫ; كل; كل; ) ARABIC LIGATURE KAF WITH LAM MEDIAL FORM +FCEC;FCEC;FCEC;0643 0645;0643 0645; # (ﳬ; ﳬ; ﳬ; كم; كم; ) ARABIC LIGATURE KAF WITH MEEM MEDIAL FORM +FCED;FCED;FCED;0644 0645;0644 0645; # (ﳭ; ﳭ; ﳭ; لم; لم; ) ARABIC LIGATURE LAM WITH MEEM MEDIAL FORM +FCEE;FCEE;FCEE;0646 0645;0646 0645; # (ﳮ; ﳮ; ﳮ; نم; نم; ) ARABIC LIGATURE NOON WITH MEEM MEDIAL FORM +FCEF;FCEF;FCEF;0646 0647;0646 0647; # (ﳯ; ﳯ; ﳯ; نه; نه; ) ARABIC LIGATURE NOON WITH HEH MEDIAL FORM +FCF0;FCF0;FCF0;064A 0645;064A 0645; # (ﳰ; ﳰ; ﳰ; يم; يم; ) ARABIC LIGATURE YEH WITH MEEM MEDIAL FORM +FCF1;FCF1;FCF1;064A 0647;064A 0647; # (ﳱ; ﳱ; ﳱ; يه; يه; ) ARABIC LIGATURE YEH WITH HEH MEDIAL FORM +FCF2;FCF2;FCF2;0640 064E 0651;0640 064E 0651; # (ﳲ; ﳲ; ﳲ; ـ◌َ◌ّ; ـ◌َ◌ّ; ) ARABIC LIGATURE SHADDA WITH FATHA MEDIAL FORM +FCF3;FCF3;FCF3;0640 064F 0651;0640 064F 0651; # (ﳳ; ﳳ; ﳳ; ـ◌ُ◌ّ; ـ◌ُ◌ّ; ) ARABIC LIGATURE SHADDA WITH DAMMA MEDIAL FORM +FCF4;FCF4;FCF4;0640 0650 0651;0640 0650 0651; # (ﳴ; ﳴ; ﳴ; ـ◌ِ◌ّ; ـ◌ِ◌ّ; ) ARABIC LIGATURE SHADDA WITH KASRA MEDIAL FORM +FCF5;FCF5;FCF5;0637 0649;0637 0649; # (ﳵ; ﳵ; ﳵ; طى; طى; ) ARABIC LIGATURE TAH WITH ALEF MAKSURA ISOLATED FORM +FCF6;FCF6;FCF6;0637 064A;0637 064A; # (ﳶ; ﳶ; ﳶ; طي; طي; ) ARABIC LIGATURE TAH WITH YEH ISOLATED FORM +FCF7;FCF7;FCF7;0639 0649;0639 0649; # (ﳷ; ﳷ; ﳷ; عى; عى; ) ARABIC LIGATURE AIN WITH ALEF MAKSURA ISOLATED FORM +FCF8;FCF8;FCF8;0639 064A;0639 064A; # (ﳸ; ﳸ; ﳸ; عي; عي; ) ARABIC LIGATURE AIN WITH YEH ISOLATED FORM +FCF9;FCF9;FCF9;063A 0649;063A 0649; # (ﳹ; ﳹ; ﳹ; غى; غى; ) ARABIC LIGATURE GHAIN WITH ALEF MAKSURA ISOLATED FORM +FCFA;FCFA;FCFA;063A 064A;063A 064A; # (ﳺ; ﳺ; ﳺ; غي; غي; ) ARABIC LIGATURE GHAIN WITH YEH ISOLATED FORM +FCFB;FCFB;FCFB;0633 0649;0633 0649; # (ﳻ; ﳻ; ﳻ; سى; سى; ) ARABIC LIGATURE SEEN WITH ALEF MAKSURA ISOLATED FORM +FCFC;FCFC;FCFC;0633 064A;0633 064A; # (ﳼ; ﳼ; ﳼ; سي; سي; ) ARABIC LIGATURE SEEN WITH YEH ISOLATED FORM +FCFD;FCFD;FCFD;0634 0649;0634 0649; # (ﳽ; ﳽ; ﳽ; شى; شى; ) ARABIC LIGATURE SHEEN WITH ALEF MAKSURA ISOLATED FORM +FCFE;FCFE;FCFE;0634 064A;0634 064A; # (ﳾ; ﳾ; ﳾ; شي; شي; ) ARABIC LIGATURE SHEEN WITH YEH ISOLATED FORM +FCFF;FCFF;FCFF;062D 0649;062D 0649; # (ﳿ; ﳿ; ﳿ; حى; حى; ) ARABIC LIGATURE HAH WITH ALEF MAKSURA ISOLATED FORM +FD00;FD00;FD00;062D 064A;062D 064A; # (ﴀ; ﴀ; ﴀ; حي; حي; ) ARABIC LIGATURE HAH WITH YEH ISOLATED FORM +FD01;FD01;FD01;062C 0649;062C 0649; # (ﴁ; ﴁ; ﴁ; جى; جى; ) ARABIC LIGATURE JEEM WITH ALEF MAKSURA ISOLATED FORM +FD02;FD02;FD02;062C 064A;062C 064A; # (ﴂ; ﴂ; ﴂ; جي; جي; ) ARABIC LIGATURE JEEM WITH YEH ISOLATED FORM +FD03;FD03;FD03;062E 0649;062E 0649; # (ﴃ; ﴃ; ﴃ; خى; خى; ) ARABIC LIGATURE KHAH WITH ALEF MAKSURA ISOLATED FORM +FD04;FD04;FD04;062E 064A;062E 064A; # (ﴄ; ﴄ; ﴄ; خي; خي; ) ARABIC LIGATURE KHAH WITH YEH ISOLATED FORM +FD05;FD05;FD05;0635 0649;0635 0649; # (ﴅ; ﴅ; ﴅ; صى; صى; ) ARABIC LIGATURE SAD WITH ALEF MAKSURA ISOLATED FORM +FD06;FD06;FD06;0635 064A;0635 064A; # (ﴆ; ﴆ; ﴆ; صي; صي; ) ARABIC LIGATURE SAD WITH YEH ISOLATED FORM +FD07;FD07;FD07;0636 0649;0636 0649; # (ﴇ; ﴇ; ﴇ; ضى; ضى; ) ARABIC LIGATURE DAD WITH ALEF MAKSURA ISOLATED FORM +FD08;FD08;FD08;0636 064A;0636 064A; # (ﴈ; ﴈ; ﴈ; ضي; ضي; ) ARABIC LIGATURE DAD WITH YEH ISOLATED FORM +FD09;FD09;FD09;0634 062C;0634 062C; # (ﴉ; ﴉ; ﴉ; شج; شج; ) ARABIC LIGATURE SHEEN WITH JEEM ISOLATED FORM +FD0A;FD0A;FD0A;0634 062D;0634 062D; # (ﴊ; ﴊ; ﴊ; شح; شح; ) ARABIC LIGATURE SHEEN WITH HAH ISOLATED FORM +FD0B;FD0B;FD0B;0634 062E;0634 062E; # (ﴋ; ﴋ; ﴋ; شخ; شخ; ) ARABIC LIGATURE SHEEN WITH KHAH ISOLATED FORM +FD0C;FD0C;FD0C;0634 0645;0634 0645; # (ﴌ; ﴌ; ﴌ; شم; شم; ) ARABIC LIGATURE SHEEN WITH MEEM ISOLATED FORM +FD0D;FD0D;FD0D;0634 0631;0634 0631; # (ﴍ; ﴍ; ﴍ; شر; شر; ) ARABIC LIGATURE SHEEN WITH REH ISOLATED FORM +FD0E;FD0E;FD0E;0633 0631;0633 0631; # (ﴎ; ﴎ; ﴎ; سر; سر; ) ARABIC LIGATURE SEEN WITH REH ISOLATED FORM +FD0F;FD0F;FD0F;0635 0631;0635 0631; # (ﴏ; ﴏ; ﴏ; صر; صر; ) ARABIC LIGATURE SAD WITH REH ISOLATED FORM +FD10;FD10;FD10;0636 0631;0636 0631; # (ﴐ; ﴐ; ﴐ; ضر; ضر; ) ARABIC LIGATURE DAD WITH REH ISOLATED FORM +FD11;FD11;FD11;0637 0649;0637 0649; # (ﴑ; ﴑ; ﴑ; طى; طى; ) ARABIC LIGATURE TAH WITH ALEF MAKSURA FINAL FORM +FD12;FD12;FD12;0637 064A;0637 064A; # (ﴒ; ﴒ; ﴒ; طي; طي; ) ARABIC LIGATURE TAH WITH YEH FINAL FORM +FD13;FD13;FD13;0639 0649;0639 0649; # (ﴓ; ﴓ; ﴓ; عى; عى; ) ARABIC LIGATURE AIN WITH ALEF MAKSURA FINAL FORM +FD14;FD14;FD14;0639 064A;0639 064A; # (ﴔ; ﴔ; ﴔ; عي; عي; ) ARABIC LIGATURE AIN WITH YEH FINAL FORM +FD15;FD15;FD15;063A 0649;063A 0649; # (ﴕ; ﴕ; ﴕ; غى; غى; ) ARABIC LIGATURE GHAIN WITH ALEF MAKSURA FINAL FORM +FD16;FD16;FD16;063A 064A;063A 064A; # (ﴖ; ﴖ; ﴖ; غي; غي; ) ARABIC LIGATURE GHAIN WITH YEH FINAL FORM +FD17;FD17;FD17;0633 0649;0633 0649; # (ﴗ; ﴗ; ﴗ; سى; سى; ) ARABIC LIGATURE SEEN WITH ALEF MAKSURA FINAL FORM +FD18;FD18;FD18;0633 064A;0633 064A; # (ﴘ; ﴘ; ﴘ; سي; سي; ) ARABIC LIGATURE SEEN WITH YEH FINAL FORM +FD19;FD19;FD19;0634 0649;0634 0649; # (ﴙ; ﴙ; ﴙ; شى; شى; ) ARABIC LIGATURE SHEEN WITH ALEF MAKSURA FINAL FORM +FD1A;FD1A;FD1A;0634 064A;0634 064A; # (ﴚ; ﴚ; ﴚ; شي; شي; ) ARABIC LIGATURE SHEEN WITH YEH FINAL FORM +FD1B;FD1B;FD1B;062D 0649;062D 0649; # (ﴛ; ﴛ; ﴛ; حى; حى; ) ARABIC LIGATURE HAH WITH ALEF MAKSURA FINAL FORM +FD1C;FD1C;FD1C;062D 064A;062D 064A; # (ﴜ; ﴜ; ﴜ; حي; حي; ) ARABIC LIGATURE HAH WITH YEH FINAL FORM +FD1D;FD1D;FD1D;062C 0649;062C 0649; # (ﴝ; ﴝ; ﴝ; جى; جى; ) ARABIC LIGATURE JEEM WITH ALEF MAKSURA FINAL FORM +FD1E;FD1E;FD1E;062C 064A;062C 064A; # (ﴞ; ﴞ; ﴞ; جي; جي; ) ARABIC LIGATURE JEEM WITH YEH FINAL FORM +FD1F;FD1F;FD1F;062E 0649;062E 0649; # (ﴟ; ﴟ; ﴟ; خى; خى; ) ARABIC LIGATURE KHAH WITH ALEF MAKSURA FINAL FORM +FD20;FD20;FD20;062E 064A;062E 064A; # (ﴠ; ﴠ; ﴠ; خي; خي; ) ARABIC LIGATURE KHAH WITH YEH FINAL FORM +FD21;FD21;FD21;0635 0649;0635 0649; # (ﴡ; ﴡ; ﴡ; صى; صى; ) ARABIC LIGATURE SAD WITH ALEF MAKSURA FINAL FORM +FD22;FD22;FD22;0635 064A;0635 064A; # (ﴢ; ﴢ; ﴢ; صي; صي; ) ARABIC LIGATURE SAD WITH YEH FINAL FORM +FD23;FD23;FD23;0636 0649;0636 0649; # (ﴣ; ﴣ; ﴣ; ضى; ضى; ) ARABIC LIGATURE DAD WITH ALEF MAKSURA FINAL FORM +FD24;FD24;FD24;0636 064A;0636 064A; # (ﴤ; ﴤ; ﴤ; ضي; ضي; ) ARABIC LIGATURE DAD WITH YEH FINAL FORM +FD25;FD25;FD25;0634 062C;0634 062C; # (ﴥ; ﴥ; ﴥ; شج; شج; ) ARABIC LIGATURE SHEEN WITH JEEM FINAL FORM +FD26;FD26;FD26;0634 062D;0634 062D; # (ﴦ; ﴦ; ﴦ; شح; شح; ) ARABIC LIGATURE SHEEN WITH HAH FINAL FORM +FD27;FD27;FD27;0634 062E;0634 062E; # (ﴧ; ﴧ; ﴧ; شخ; شخ; ) ARABIC LIGATURE SHEEN WITH KHAH FINAL FORM +FD28;FD28;FD28;0634 0645;0634 0645; # (ﴨ; ﴨ; ﴨ; شم; شم; ) ARABIC LIGATURE SHEEN WITH MEEM FINAL FORM +FD29;FD29;FD29;0634 0631;0634 0631; # (ﴩ; ﴩ; ﴩ; شر; شر; ) ARABIC LIGATURE SHEEN WITH REH FINAL FORM +FD2A;FD2A;FD2A;0633 0631;0633 0631; # (ﴪ; ﴪ; ﴪ; سر; سر; ) ARABIC LIGATURE SEEN WITH REH FINAL FORM +FD2B;FD2B;FD2B;0635 0631;0635 0631; # (ﴫ; ﴫ; ﴫ; صر; صر; ) ARABIC LIGATURE SAD WITH REH FINAL FORM +FD2C;FD2C;FD2C;0636 0631;0636 0631; # (ﴬ; ﴬ; ﴬ; ضر; ضر; ) ARABIC LIGATURE DAD WITH REH FINAL FORM +FD2D;FD2D;FD2D;0634 062C;0634 062C; # (ﴭ; ﴭ; ﴭ; شج; شج; ) ARABIC LIGATURE SHEEN WITH JEEM INITIAL FORM +FD2E;FD2E;FD2E;0634 062D;0634 062D; # (ﴮ; ﴮ; ﴮ; شح; شح; ) ARABIC LIGATURE SHEEN WITH HAH INITIAL FORM +FD2F;FD2F;FD2F;0634 062E;0634 062E; # (ﴯ; ﴯ; ﴯ; شخ; شخ; ) ARABIC LIGATURE SHEEN WITH KHAH INITIAL FORM +FD30;FD30;FD30;0634 0645;0634 0645; # (ﴰ; ﴰ; ﴰ; شم; شم; ) ARABIC LIGATURE SHEEN WITH MEEM INITIAL FORM +FD31;FD31;FD31;0633 0647;0633 0647; # (ﴱ; ﴱ; ﴱ; سه; سه; ) ARABIC LIGATURE SEEN WITH HEH INITIAL FORM +FD32;FD32;FD32;0634 0647;0634 0647; # (ﴲ; ﴲ; ﴲ; شه; شه; ) ARABIC LIGATURE SHEEN WITH HEH INITIAL FORM +FD33;FD33;FD33;0637 0645;0637 0645; # (ﴳ; ﴳ; ﴳ; طم; طم; ) ARABIC LIGATURE TAH WITH MEEM INITIAL FORM +FD34;FD34;FD34;0633 062C;0633 062C; # (ﴴ; ﴴ; ﴴ; سج; سج; ) ARABIC LIGATURE SEEN WITH JEEM MEDIAL FORM +FD35;FD35;FD35;0633 062D;0633 062D; # (ﴵ; ﴵ; ﴵ; سح; سح; ) ARABIC LIGATURE SEEN WITH HAH MEDIAL FORM +FD36;FD36;FD36;0633 062E;0633 062E; # (ﴶ; ﴶ; ﴶ; سخ; سخ; ) ARABIC LIGATURE SEEN WITH KHAH MEDIAL FORM +FD37;FD37;FD37;0634 062C;0634 062C; # (ﴷ; ﴷ; ﴷ; شج; شج; ) ARABIC LIGATURE SHEEN WITH JEEM MEDIAL FORM +FD38;FD38;FD38;0634 062D;0634 062D; # (ﴸ; ﴸ; ﴸ; شح; شح; ) ARABIC LIGATURE SHEEN WITH HAH MEDIAL FORM +FD39;FD39;FD39;0634 062E;0634 062E; # (ﴹ; ﴹ; ﴹ; شخ; شخ; ) ARABIC LIGATURE SHEEN WITH KHAH MEDIAL FORM +FD3A;FD3A;FD3A;0637 0645;0637 0645; # (ﴺ; ﴺ; ﴺ; طم; طم; ) ARABIC LIGATURE TAH WITH MEEM MEDIAL FORM +FD3B;FD3B;FD3B;0638 0645;0638 0645; # (ﴻ; ﴻ; ﴻ; ظم; ظم; ) ARABIC LIGATURE ZAH WITH MEEM MEDIAL FORM +FD3C;FD3C;FD3C;0627 064B;0627 064B; # (ﴼ; ﴼ; ﴼ; ا◌ً; ا◌ً; ) ARABIC LIGATURE ALEF WITH FATHATAN FINAL FORM +FD3D;FD3D;FD3D;0627 064B;0627 064B; # (ﴽ; ﴽ; ﴽ; ا◌ً; ا◌ً; ) ARABIC LIGATURE ALEF WITH FATHATAN ISOLATED FORM +FD50;FD50;FD50;062A 062C 0645;062A 062C 0645; # (ﵐ; ﵐ; ﵐ; تجم; تجم; ) ARABIC LIGATURE TEH WITH JEEM WITH MEEM INITIAL FORM +FD51;FD51;FD51;062A 062D 062C;062A 062D 062C; # (ﵑ; ﵑ; ﵑ; تحج; تحج; ) ARABIC LIGATURE TEH WITH HAH WITH JEEM FINAL FORM +FD52;FD52;FD52;062A 062D 062C;062A 062D 062C; # (ﵒ; ﵒ; ﵒ; تحج; تحج; ) ARABIC LIGATURE TEH WITH HAH WITH JEEM INITIAL FORM +FD53;FD53;FD53;062A 062D 0645;062A 062D 0645; # (ﵓ; ﵓ; ﵓ; تحم; تحم; ) ARABIC LIGATURE TEH WITH HAH WITH MEEM INITIAL FORM +FD54;FD54;FD54;062A 062E 0645;062A 062E 0645; # (ﵔ; ﵔ; ﵔ; تخم; تخم; ) ARABIC LIGATURE TEH WITH KHAH WITH MEEM INITIAL FORM +FD55;FD55;FD55;062A 0645 062C;062A 0645 062C; # (ﵕ; ﵕ; ﵕ; تمج; تمج; ) ARABIC LIGATURE TEH WITH MEEM WITH JEEM INITIAL FORM +FD56;FD56;FD56;062A 0645 062D;062A 0645 062D; # (ﵖ; ﵖ; ﵖ; تمح; تمح; ) ARABIC LIGATURE TEH WITH MEEM WITH HAH INITIAL FORM +FD57;FD57;FD57;062A 0645 062E;062A 0645 062E; # (ﵗ; ﵗ; ﵗ; تمخ; تمخ; ) ARABIC LIGATURE TEH WITH MEEM WITH KHAH INITIAL FORM +FD58;FD58;FD58;062C 0645 062D;062C 0645 062D; # (ﵘ; ﵘ; ﵘ; جمح; جمح; ) ARABIC LIGATURE JEEM WITH MEEM WITH HAH FINAL FORM +FD59;FD59;FD59;062C 0645 062D;062C 0645 062D; # (ﵙ; ﵙ; ﵙ; جمح; جمح; ) ARABIC LIGATURE JEEM WITH MEEM WITH HAH INITIAL FORM +FD5A;FD5A;FD5A;062D 0645 064A;062D 0645 064A; # (ﵚ; ﵚ; ﵚ; حمي; حمي; ) ARABIC LIGATURE HAH WITH MEEM WITH YEH FINAL FORM +FD5B;FD5B;FD5B;062D 0645 0649;062D 0645 0649; # (ﵛ; ﵛ; ﵛ; حمى; حمى; ) ARABIC LIGATURE HAH WITH MEEM WITH ALEF MAKSURA FINAL FORM +FD5C;FD5C;FD5C;0633 062D 062C;0633 062D 062C; # (ﵜ; ﵜ; ﵜ; سحج; سحج; ) ARABIC LIGATURE SEEN WITH HAH WITH JEEM INITIAL FORM +FD5D;FD5D;FD5D;0633 062C 062D;0633 062C 062D; # (ﵝ; ﵝ; ﵝ; سجح; سجح; ) ARABIC LIGATURE SEEN WITH JEEM WITH HAH INITIAL FORM +FD5E;FD5E;FD5E;0633 062C 0649;0633 062C 0649; # (ﵞ; ﵞ; ﵞ; سجى; سجى; ) ARABIC LIGATURE SEEN WITH JEEM WITH ALEF MAKSURA FINAL FORM +FD5F;FD5F;FD5F;0633 0645 062D;0633 0645 062D; # (ﵟ; ﵟ; ﵟ; سمح; سمح; ) ARABIC LIGATURE SEEN WITH MEEM WITH HAH FINAL FORM +FD60;FD60;FD60;0633 0645 062D;0633 0645 062D; # (ﵠ; ﵠ; ﵠ; سمح; سمح; ) ARABIC LIGATURE SEEN WITH MEEM WITH HAH INITIAL FORM +FD61;FD61;FD61;0633 0645 062C;0633 0645 062C; # (ﵡ; ﵡ; ﵡ; سمج; سمج; ) ARABIC LIGATURE SEEN WITH MEEM WITH JEEM INITIAL FORM +FD62;FD62;FD62;0633 0645 0645;0633 0645 0645; # (ﵢ; ﵢ; ﵢ; سمم; سمم; ) ARABIC LIGATURE SEEN WITH MEEM WITH MEEM FINAL FORM +FD63;FD63;FD63;0633 0645 0645;0633 0645 0645; # (ﵣ; ﵣ; ﵣ; سمم; سمم; ) ARABIC LIGATURE SEEN WITH MEEM WITH MEEM INITIAL FORM +FD64;FD64;FD64;0635 062D 062D;0635 062D 062D; # (ﵤ; ﵤ; ﵤ; صحح; صحح; ) ARABIC LIGATURE SAD WITH HAH WITH HAH FINAL FORM +FD65;FD65;FD65;0635 062D 062D;0635 062D 062D; # (ﵥ; ﵥ; ﵥ; صحح; صحح; ) ARABIC LIGATURE SAD WITH HAH WITH HAH INITIAL FORM +FD66;FD66;FD66;0635 0645 0645;0635 0645 0645; # (ﵦ; ﵦ; ﵦ; صمم; صمم; ) ARABIC LIGATURE SAD WITH MEEM WITH MEEM FINAL FORM +FD67;FD67;FD67;0634 062D 0645;0634 062D 0645; # (ﵧ; ﵧ; ﵧ; شحم; شحم; ) ARABIC LIGATURE SHEEN WITH HAH WITH MEEM FINAL FORM +FD68;FD68;FD68;0634 062D 0645;0634 062D 0645; # (ﵨ; ﵨ; ﵨ; شحم; شحم; ) ARABIC LIGATURE SHEEN WITH HAH WITH MEEM INITIAL FORM +FD69;FD69;FD69;0634 062C 064A;0634 062C 064A; # (ﵩ; ﵩ; ﵩ; شجي; شجي; ) ARABIC LIGATURE SHEEN WITH JEEM WITH YEH FINAL FORM +FD6A;FD6A;FD6A;0634 0645 062E;0634 0645 062E; # (ﵪ; ﵪ; ﵪ; شمخ; شمخ; ) ARABIC LIGATURE SHEEN WITH MEEM WITH KHAH FINAL FORM +FD6B;FD6B;FD6B;0634 0645 062E;0634 0645 062E; # (ﵫ; ﵫ; ﵫ; شمخ; شمخ; ) ARABIC LIGATURE SHEEN WITH MEEM WITH KHAH INITIAL FORM +FD6C;FD6C;FD6C;0634 0645 0645;0634 0645 0645; # (ﵬ; ﵬ; ﵬ; شمم; شمم; ) ARABIC LIGATURE SHEEN WITH MEEM WITH MEEM FINAL FORM +FD6D;FD6D;FD6D;0634 0645 0645;0634 0645 0645; # (ﵭ; ﵭ; ﵭ; شمم; شمم; ) ARABIC LIGATURE SHEEN WITH MEEM WITH MEEM INITIAL FORM +FD6E;FD6E;FD6E;0636 062D 0649;0636 062D 0649; # (ﵮ; ﵮ; ﵮ; ضحى; ضحى; ) ARABIC LIGATURE DAD WITH HAH WITH ALEF MAKSURA FINAL FORM +FD6F;FD6F;FD6F;0636 062E 0645;0636 062E 0645; # (ﵯ; ﵯ; ﵯ; ضخم; ضخم; ) ARABIC LIGATURE DAD WITH KHAH WITH MEEM FINAL FORM +FD70;FD70;FD70;0636 062E 0645;0636 062E 0645; # (ﵰ; ﵰ; ﵰ; ضخم; ضخم; ) ARABIC LIGATURE DAD WITH KHAH WITH MEEM INITIAL FORM +FD71;FD71;FD71;0637 0645 062D;0637 0645 062D; # (ﵱ; ﵱ; ﵱ; طمح; طمح; ) ARABIC LIGATURE TAH WITH MEEM WITH HAH FINAL FORM +FD72;FD72;FD72;0637 0645 062D;0637 0645 062D; # (ﵲ; ﵲ; ﵲ; طمح; طمح; ) ARABIC LIGATURE TAH WITH MEEM WITH HAH INITIAL FORM +FD73;FD73;FD73;0637 0645 0645;0637 0645 0645; # (ﵳ; ﵳ; ﵳ; طمم; طمم; ) ARABIC LIGATURE TAH WITH MEEM WITH MEEM INITIAL FORM +FD74;FD74;FD74;0637 0645 064A;0637 0645 064A; # (ﵴ; ﵴ; ﵴ; طمي; طمي; ) ARABIC LIGATURE TAH WITH MEEM WITH YEH FINAL FORM +FD75;FD75;FD75;0639 062C 0645;0639 062C 0645; # (ﵵ; ﵵ; ﵵ; عجم; عجم; ) ARABIC LIGATURE AIN WITH JEEM WITH MEEM FINAL FORM +FD76;FD76;FD76;0639 0645 0645;0639 0645 0645; # (ﵶ; ﵶ; ﵶ; عمم; عمم; ) ARABIC LIGATURE AIN WITH MEEM WITH MEEM FINAL FORM +FD77;FD77;FD77;0639 0645 0645;0639 0645 0645; # (ﵷ; ﵷ; ﵷ; عمم; عمم; ) ARABIC LIGATURE AIN WITH MEEM WITH MEEM INITIAL FORM +FD78;FD78;FD78;0639 0645 0649;0639 0645 0649; # (ﵸ; ﵸ; ﵸ; عمى; عمى; ) ARABIC LIGATURE AIN WITH MEEM WITH ALEF MAKSURA FINAL FORM +FD79;FD79;FD79;063A 0645 0645;063A 0645 0645; # (ﵹ; ﵹ; ﵹ; غمم; غمم; ) ARABIC LIGATURE GHAIN WITH MEEM WITH MEEM FINAL FORM +FD7A;FD7A;FD7A;063A 0645 064A;063A 0645 064A; # (ﵺ; ﵺ; ﵺ; غمي; غمي; ) ARABIC LIGATURE GHAIN WITH MEEM WITH YEH FINAL FORM +FD7B;FD7B;FD7B;063A 0645 0649;063A 0645 0649; # (ﵻ; ﵻ; ﵻ; غمى; غمى; ) ARABIC LIGATURE GHAIN WITH MEEM WITH ALEF MAKSURA FINAL FORM +FD7C;FD7C;FD7C;0641 062E 0645;0641 062E 0645; # (ﵼ; ﵼ; ﵼ; فخم; فخم; ) ARABIC LIGATURE FEH WITH KHAH WITH MEEM FINAL FORM +FD7D;FD7D;FD7D;0641 062E 0645;0641 062E 0645; # (ﵽ; ﵽ; ﵽ; فخم; فخم; ) ARABIC LIGATURE FEH WITH KHAH WITH MEEM INITIAL FORM +FD7E;FD7E;FD7E;0642 0645 062D;0642 0645 062D; # (ﵾ; ﵾ; ﵾ; قمح; قمح; ) ARABIC LIGATURE QAF WITH MEEM WITH HAH FINAL FORM +FD7F;FD7F;FD7F;0642 0645 0645;0642 0645 0645; # (ﵿ; ﵿ; ﵿ; قمم; قمم; ) ARABIC LIGATURE QAF WITH MEEM WITH MEEM FINAL FORM +FD80;FD80;FD80;0644 062D 0645;0644 062D 0645; # (ﶀ; ﶀ; ﶀ; لحم; لحم; ) ARABIC LIGATURE LAM WITH HAH WITH MEEM FINAL FORM +FD81;FD81;FD81;0644 062D 064A;0644 062D 064A; # (ﶁ; ﶁ; ﶁ; لحي; لحي; ) ARABIC LIGATURE LAM WITH HAH WITH YEH FINAL FORM +FD82;FD82;FD82;0644 062D 0649;0644 062D 0649; # (ﶂ; ﶂ; ﶂ; لحى; لحى; ) ARABIC LIGATURE LAM WITH HAH WITH ALEF MAKSURA FINAL FORM +FD83;FD83;FD83;0644 062C 062C;0644 062C 062C; # (ﶃ; ﶃ; ﶃ; لجج; لجج; ) ARABIC LIGATURE LAM WITH JEEM WITH JEEM INITIAL FORM +FD84;FD84;FD84;0644 062C 062C;0644 062C 062C; # (ﶄ; ﶄ; ﶄ; لجج; لجج; ) ARABIC LIGATURE LAM WITH JEEM WITH JEEM FINAL FORM +FD85;FD85;FD85;0644 062E 0645;0644 062E 0645; # (ﶅ; ﶅ; ﶅ; لخم; لخم; ) ARABIC LIGATURE LAM WITH KHAH WITH MEEM FINAL FORM +FD86;FD86;FD86;0644 062E 0645;0644 062E 0645; # (ﶆ; ﶆ; ﶆ; لخم; لخم; ) ARABIC LIGATURE LAM WITH KHAH WITH MEEM INITIAL FORM +FD87;FD87;FD87;0644 0645 062D;0644 0645 062D; # (ﶇ; ﶇ; ﶇ; لمح; لمح; ) ARABIC LIGATURE LAM WITH MEEM WITH HAH FINAL FORM +FD88;FD88;FD88;0644 0645 062D;0644 0645 062D; # (ﶈ; ﶈ; ﶈ; لمح; لمح; ) ARABIC LIGATURE LAM WITH MEEM WITH HAH INITIAL FORM +FD89;FD89;FD89;0645 062D 062C;0645 062D 062C; # (ﶉ; ﶉ; ﶉ; محج; محج; ) ARABIC LIGATURE MEEM WITH HAH WITH JEEM INITIAL FORM +FD8A;FD8A;FD8A;0645 062D 0645;0645 062D 0645; # (ﶊ; ﶊ; ﶊ; محم; محم; ) ARABIC LIGATURE MEEM WITH HAH WITH MEEM INITIAL FORM +FD8B;FD8B;FD8B;0645 062D 064A;0645 062D 064A; # (ﶋ; ﶋ; ﶋ; محي; محي; ) ARABIC LIGATURE MEEM WITH HAH WITH YEH FINAL FORM +FD8C;FD8C;FD8C;0645 062C 062D;0645 062C 062D; # (ﶌ; ﶌ; ﶌ; مجح; مجح; ) ARABIC LIGATURE MEEM WITH JEEM WITH HAH INITIAL FORM +FD8D;FD8D;FD8D;0645 062C 0645;0645 062C 0645; # (ﶍ; ﶍ; ﶍ; مجم; مجم; ) ARABIC LIGATURE MEEM WITH JEEM WITH MEEM INITIAL FORM +FD8E;FD8E;FD8E;0645 062E 062C;0645 062E 062C; # (ﶎ; ﶎ; ﶎ; مخج; مخج; ) ARABIC LIGATURE MEEM WITH KHAH WITH JEEM INITIAL FORM +FD8F;FD8F;FD8F;0645 062E 0645;0645 062E 0645; # (ﶏ; ﶏ; ﶏ; مخم; مخم; ) ARABIC LIGATURE MEEM WITH KHAH WITH MEEM INITIAL FORM +FD92;FD92;FD92;0645 062C 062E;0645 062C 062E; # (ﶒ; ﶒ; ﶒ; مجخ; مجخ; ) ARABIC LIGATURE MEEM WITH JEEM WITH KHAH INITIAL FORM +FD93;FD93;FD93;0647 0645 062C;0647 0645 062C; # (ﶓ; ﶓ; ﶓ; همج; همج; ) ARABIC LIGATURE HEH WITH MEEM WITH JEEM INITIAL FORM +FD94;FD94;FD94;0647 0645 0645;0647 0645 0645; # (ﶔ; ﶔ; ﶔ; همم; همم; ) ARABIC LIGATURE HEH WITH MEEM WITH MEEM INITIAL FORM +FD95;FD95;FD95;0646 062D 0645;0646 062D 0645; # (ﶕ; ﶕ; ﶕ; نحم; نحم; ) ARABIC LIGATURE NOON WITH HAH WITH MEEM INITIAL FORM +FD96;FD96;FD96;0646 062D 0649;0646 062D 0649; # (ﶖ; ﶖ; ﶖ; نحى; نحى; ) ARABIC LIGATURE NOON WITH HAH WITH ALEF MAKSURA FINAL FORM +FD97;FD97;FD97;0646 062C 0645;0646 062C 0645; # (ﶗ; ﶗ; ﶗ; نجم; نجم; ) ARABIC LIGATURE NOON WITH JEEM WITH MEEM FINAL FORM +FD98;FD98;FD98;0646 062C 0645;0646 062C 0645; # (ﶘ; ﶘ; ﶘ; نجم; نجم; ) ARABIC LIGATURE NOON WITH JEEM WITH MEEM INITIAL FORM +FD99;FD99;FD99;0646 062C 0649;0646 062C 0649; # (ﶙ; ﶙ; ﶙ; نجى; نجى; ) ARABIC LIGATURE NOON WITH JEEM WITH ALEF MAKSURA FINAL FORM +FD9A;FD9A;FD9A;0646 0645 064A;0646 0645 064A; # (ﶚ; ﶚ; ﶚ; نمي; نمي; ) ARABIC LIGATURE NOON WITH MEEM WITH YEH FINAL FORM +FD9B;FD9B;FD9B;0646 0645 0649;0646 0645 0649; # (ﶛ; ﶛ; ﶛ; نمى; نمى; ) ARABIC LIGATURE NOON WITH MEEM WITH ALEF MAKSURA FINAL FORM +FD9C;FD9C;FD9C;064A 0645 0645;064A 0645 0645; # (ﶜ; ﶜ; ﶜ; يمم; يمم; ) ARABIC LIGATURE YEH WITH MEEM WITH MEEM FINAL FORM +FD9D;FD9D;FD9D;064A 0645 0645;064A 0645 0645; # (ﶝ; ﶝ; ﶝ; يمم; يمم; ) ARABIC LIGATURE YEH WITH MEEM WITH MEEM INITIAL FORM +FD9E;FD9E;FD9E;0628 062E 064A;0628 062E 064A; # (ﶞ; ﶞ; ﶞ; بخي; بخي; ) ARABIC LIGATURE BEH WITH KHAH WITH YEH FINAL FORM +FD9F;FD9F;FD9F;062A 062C 064A;062A 062C 064A; # (ﶟ; ﶟ; ﶟ; تجي; تجي; ) ARABIC LIGATURE TEH WITH JEEM WITH YEH FINAL FORM +FDA0;FDA0;FDA0;062A 062C 0649;062A 062C 0649; # (ﶠ; ﶠ; ﶠ; تجى; تجى; ) ARABIC LIGATURE TEH WITH JEEM WITH ALEF MAKSURA FINAL FORM +FDA1;FDA1;FDA1;062A 062E 064A;062A 062E 064A; # (ﶡ; ﶡ; ﶡ; تخي; تخي; ) ARABIC LIGATURE TEH WITH KHAH WITH YEH FINAL FORM +FDA2;FDA2;FDA2;062A 062E 0649;062A 062E 0649; # (ﶢ; ﶢ; ﶢ; تخى; تخى; ) ARABIC LIGATURE TEH WITH KHAH WITH ALEF MAKSURA FINAL FORM +FDA3;FDA3;FDA3;062A 0645 064A;062A 0645 064A; # (ﶣ; ﶣ; ﶣ; تمي; تمي; ) ARABIC LIGATURE TEH WITH MEEM WITH YEH FINAL FORM +FDA4;FDA4;FDA4;062A 0645 0649;062A 0645 0649; # (ﶤ; ﶤ; ﶤ; تمى; تمى; ) ARABIC LIGATURE TEH WITH MEEM WITH ALEF MAKSURA FINAL FORM +FDA5;FDA5;FDA5;062C 0645 064A;062C 0645 064A; # (ﶥ; ﶥ; ﶥ; جمي; جمي; ) ARABIC LIGATURE JEEM WITH MEEM WITH YEH FINAL FORM +FDA6;FDA6;FDA6;062C 062D 0649;062C 062D 0649; # (ﶦ; ﶦ; ﶦ; جحى; جحى; ) ARABIC LIGATURE JEEM WITH HAH WITH ALEF MAKSURA FINAL FORM +FDA7;FDA7;FDA7;062C 0645 0649;062C 0645 0649; # (ﶧ; ﶧ; ﶧ; جمى; جمى; ) ARABIC LIGATURE JEEM WITH MEEM WITH ALEF MAKSURA FINAL FORM +FDA8;FDA8;FDA8;0633 062E 0649;0633 062E 0649; # (ﶨ; ﶨ; ﶨ; سخى; سخى; ) ARABIC LIGATURE SEEN WITH KHAH WITH ALEF MAKSURA FINAL FORM +FDA9;FDA9;FDA9;0635 062D 064A;0635 062D 064A; # (ﶩ; ﶩ; ﶩ; صحي; صحي; ) ARABIC LIGATURE SAD WITH HAH WITH YEH FINAL FORM +FDAA;FDAA;FDAA;0634 062D 064A;0634 062D 064A; # (ﶪ; ﶪ; ﶪ; شحي; شحي; ) ARABIC LIGATURE SHEEN WITH HAH WITH YEH FINAL FORM +FDAB;FDAB;FDAB;0636 062D 064A;0636 062D 064A; # (ﶫ; ﶫ; ﶫ; ضحي; ضحي; ) ARABIC LIGATURE DAD WITH HAH WITH YEH FINAL FORM +FDAC;FDAC;FDAC;0644 062C 064A;0644 062C 064A; # (ﶬ; ﶬ; ﶬ; لجي; لجي; ) ARABIC LIGATURE LAM WITH JEEM WITH YEH FINAL FORM +FDAD;FDAD;FDAD;0644 0645 064A;0644 0645 064A; # (ﶭ; ﶭ; ﶭ; لمي; لمي; ) ARABIC LIGATURE LAM WITH MEEM WITH YEH FINAL FORM +FDAE;FDAE;FDAE;064A 062D 064A;064A 062D 064A; # (ﶮ; ﶮ; ﶮ; يحي; يحي; ) ARABIC LIGATURE YEH WITH HAH WITH YEH FINAL FORM +FDAF;FDAF;FDAF;064A 062C 064A;064A 062C 064A; # (ﶯ; ﶯ; ﶯ; يجي; يجي; ) ARABIC LIGATURE YEH WITH JEEM WITH YEH FINAL FORM +FDB0;FDB0;FDB0;064A 0645 064A;064A 0645 064A; # (ﶰ; ﶰ; ﶰ; يمي; يمي; ) ARABIC LIGATURE YEH WITH MEEM WITH YEH FINAL FORM +FDB1;FDB1;FDB1;0645 0645 064A;0645 0645 064A; # (ﶱ; ﶱ; ﶱ; ممي; ممي; ) ARABIC LIGATURE MEEM WITH MEEM WITH YEH FINAL FORM +FDB2;FDB2;FDB2;0642 0645 064A;0642 0645 064A; # (ﶲ; ﶲ; ﶲ; قمي; قمي; ) ARABIC LIGATURE QAF WITH MEEM WITH YEH FINAL FORM +FDB3;FDB3;FDB3;0646 062D 064A;0646 062D 064A; # (ﶳ; ﶳ; ﶳ; نحي; نحي; ) ARABIC LIGATURE NOON WITH HAH WITH YEH FINAL FORM +FDB4;FDB4;FDB4;0642 0645 062D;0642 0645 062D; # (ﶴ; ﶴ; ﶴ; قمح; قمح; ) ARABIC LIGATURE QAF WITH MEEM WITH HAH INITIAL FORM +FDB5;FDB5;FDB5;0644 062D 0645;0644 062D 0645; # (ﶵ; ﶵ; ﶵ; لحم; لحم; ) ARABIC LIGATURE LAM WITH HAH WITH MEEM INITIAL FORM +FDB6;FDB6;FDB6;0639 0645 064A;0639 0645 064A; # (ﶶ; ﶶ; ﶶ; عمي; عمي; ) ARABIC LIGATURE AIN WITH MEEM WITH YEH FINAL FORM +FDB7;FDB7;FDB7;0643 0645 064A;0643 0645 064A; # (ﶷ; ﶷ; ﶷ; كمي; كمي; ) ARABIC LIGATURE KAF WITH MEEM WITH YEH FINAL FORM +FDB8;FDB8;FDB8;0646 062C 062D;0646 062C 062D; # (ﶸ; ﶸ; ﶸ; نجح; نجح; ) ARABIC LIGATURE NOON WITH JEEM WITH HAH INITIAL FORM +FDB9;FDB9;FDB9;0645 062E 064A;0645 062E 064A; # (ﶹ; ﶹ; ﶹ; مخي; مخي; ) ARABIC LIGATURE MEEM WITH KHAH WITH YEH FINAL FORM +FDBA;FDBA;FDBA;0644 062C 0645;0644 062C 0645; # (ﶺ; ﶺ; ﶺ; لجم; لجم; ) ARABIC LIGATURE LAM WITH JEEM WITH MEEM INITIAL FORM +FDBB;FDBB;FDBB;0643 0645 0645;0643 0645 0645; # (ﶻ; ﶻ; ﶻ; كمم; كمم; ) ARABIC LIGATURE KAF WITH MEEM WITH MEEM FINAL FORM +FDBC;FDBC;FDBC;0644 062C 0645;0644 062C 0645; # (ﶼ; ﶼ; ﶼ; لجم; لجم; ) ARABIC LIGATURE LAM WITH JEEM WITH MEEM FINAL FORM +FDBD;FDBD;FDBD;0646 062C 062D;0646 062C 062D; # (ﶽ; ﶽ; ﶽ; نجح; نجح; ) ARABIC LIGATURE NOON WITH JEEM WITH HAH FINAL FORM +FDBE;FDBE;FDBE;062C 062D 064A;062C 062D 064A; # (ﶾ; ﶾ; ﶾ; جحي; جحي; ) ARABIC LIGATURE JEEM WITH HAH WITH YEH FINAL FORM +FDBF;FDBF;FDBF;062D 062C 064A;062D 062C 064A; # (ﶿ; ﶿ; ﶿ; حجي; حجي; ) ARABIC LIGATURE HAH WITH JEEM WITH YEH FINAL FORM +FDC0;FDC0;FDC0;0645 062C 064A;0645 062C 064A; # (ﷀ; ﷀ; ﷀ; مجي; مجي; ) ARABIC LIGATURE MEEM WITH JEEM WITH YEH FINAL FORM +FDC1;FDC1;FDC1;0641 0645 064A;0641 0645 064A; # (ﷁ; ﷁ; ﷁ; فمي; فمي; ) ARABIC LIGATURE FEH WITH MEEM WITH YEH FINAL FORM +FDC2;FDC2;FDC2;0628 062D 064A;0628 062D 064A; # (ﷂ; ﷂ; ﷂ; بحي; بحي; ) ARABIC LIGATURE BEH WITH HAH WITH YEH FINAL FORM +FDC3;FDC3;FDC3;0643 0645 0645;0643 0645 0645; # (ﷃ; ﷃ; ﷃ; كمم; كمم; ) ARABIC LIGATURE KAF WITH MEEM WITH MEEM INITIAL FORM +FDC4;FDC4;FDC4;0639 062C 0645;0639 062C 0645; # (ﷄ; ﷄ; ﷄ; عجم; عجم; ) ARABIC LIGATURE AIN WITH JEEM WITH MEEM INITIAL FORM +FDC5;FDC5;FDC5;0635 0645 0645;0635 0645 0645; # (ﷅ; ﷅ; ﷅ; صمم; صمم; ) ARABIC LIGATURE SAD WITH MEEM WITH MEEM INITIAL FORM +FDC6;FDC6;FDC6;0633 062E 064A;0633 062E 064A; # (ﷆ; ﷆ; ﷆ; سخي; سخي; ) ARABIC LIGATURE SEEN WITH KHAH WITH YEH FINAL FORM +FDC7;FDC7;FDC7;0646 062C 064A;0646 062C 064A; # (ﷇ; ﷇ; ﷇ; نجي; نجي; ) ARABIC LIGATURE NOON WITH JEEM WITH YEH FINAL FORM +FDF0;FDF0;FDF0;0635 0644 06D2;0635 0644 06D2; # (ﷰ; ﷰ; ﷰ; صلے; صلے; ) ARABIC LIGATURE SALLA USED AS KORANIC STOP SIGN ISOLATED FORM +FDF1;FDF1;FDF1;0642 0644 06D2;0642 0644 06D2; # (ﷱ; ﷱ; ﷱ; قلے; قلے; ) ARABIC LIGATURE QALA USED AS KORANIC STOP SIGN ISOLATED FORM +FDF2;FDF2;FDF2;0627 0644 0644 0647;0627 0644 0644 0647; # (ﷲ; ﷲ; ﷲ; الله; الله; ) ARABIC LIGATURE ALLAH ISOLATED FORM +FDF3;FDF3;FDF3;0627 0643 0628 0631;0627 0643 0628 0631; # (ﷳ; ﷳ; ﷳ; اكبر; اكبر; ) ARABIC LIGATURE AKBAR ISOLATED FORM +FDF4;FDF4;FDF4;0645 062D 0645 062F;0645 062D 0645 062F; # (ﷴ; ﷴ; ﷴ; محمد; محمد; ) ARABIC LIGATURE MOHAMMAD ISOLATED FORM +FDF5;FDF5;FDF5;0635 0644 0639 0645;0635 0644 0639 0645; # (ﷵ; ﷵ; ﷵ; صلعم; صلعم; ) ARABIC LIGATURE SALAM ISOLATED FORM +FDF6;FDF6;FDF6;0631 0633 0648 0644;0631 0633 0648 0644; # (ﷶ; ﷶ; ﷶ; رسول; رسول; ) ARABIC LIGATURE RASOUL ISOLATED FORM +FDF7;FDF7;FDF7;0639 0644 064A 0647;0639 0644 064A 0647; # (ﷷ; ﷷ; ﷷ; عليه; عليه; ) ARABIC LIGATURE ALAYHE ISOLATED FORM +FDF8;FDF8;FDF8;0648 0633 0644 0645;0648 0633 0644 0645; # (ﷸ; ﷸ; ﷸ; وسلم; وسلم; ) ARABIC LIGATURE WASALLAM ISOLATED FORM +FDF9;FDF9;FDF9;0635 0644 0649;0635 0644 0649; # (ﷹ; ﷹ; ﷹ; صلى; صلى; ) ARABIC LIGATURE SALLA ISOLATED FORM +FDFA;FDFA;FDFA;0635 0644 0649 0020 0627 0644 0644 0647 0020 0639 0644 064A 0647 0020 0648 0633 0644 0645;0635 0644 0649 0020 0627 0644 0644 0647 0020 0639 0644 064A 0647 0020 0648 0633 0644 0645; # (ﷺ; ﷺ; ﷺ; صلى الله عليه وسلم; صلى الله عليه وسلم; ) ARABIC LIGATURE SALLALLAHOU ALAYHE WASALLAM +FDFB;FDFB;FDFB;062C 0644 0020 062C 0644 0627 0644 0647;062C 0644 0020 062C 0644 0627 0644 0647; # (ﷻ; ﷻ; ﷻ; جل جلاله; جل جلاله; ) ARABIC LIGATURE JALLAJALALOUHOU +FDFC;FDFC;FDFC;0631 06CC 0627 0644;0631 06CC 0627 0644; # (﷼; ﷼; ﷼; ریال; ریال; ) RIAL SIGN +FE30;FE30;FE30;002E 002E;002E 002E; # (︰; ︰; ︰; ..; ..; ) PRESENTATION FORM FOR VERTICAL TWO DOT LEADER +FE31;FE31;FE31;2014;2014; # (︱; ︱; ︱; —; —; ) PRESENTATION FORM FOR VERTICAL EM DASH +FE32;FE32;FE32;2013;2013; # (︲; ︲; ︲; –; –; ) PRESENTATION FORM FOR VERTICAL EN DASH +FE33;FE33;FE33;005F;005F; # (︳; ︳; ︳; _; _; ) PRESENTATION FORM FOR VERTICAL LOW LINE +FE34;FE34;FE34;005F;005F; # (︴; ︴; ︴; _; _; ) PRESENTATION FORM FOR VERTICAL WAVY LOW LINE +FE35;FE35;FE35;0028;0028; # (︵; ︵; ︵; (; (; ) PRESENTATION FORM FOR VERTICAL LEFT PARENTHESIS +FE36;FE36;FE36;0029;0029; # (︶; ︶; ︶; ); ); ) PRESENTATION FORM FOR VERTICAL RIGHT PARENTHESIS +FE37;FE37;FE37;007B;007B; # (︷; ︷; ︷; {; {; ) PRESENTATION FORM FOR VERTICAL LEFT CURLY BRACKET +FE38;FE38;FE38;007D;007D; # (︸; ︸; ︸; }; }; ) PRESENTATION FORM FOR VERTICAL RIGHT CURLY BRACKET +FE39;FE39;FE39;3014;3014; # (︹; ︹; ︹; 〔; 〔; ) PRESENTATION FORM FOR VERTICAL LEFT TORTOISE SHELL BRACKET +FE3A;FE3A;FE3A;3015;3015; # (︺; ︺; ︺; 〕; 〕; ) PRESENTATION FORM FOR VERTICAL RIGHT TORTOISE SHELL BRACKET +FE3B;FE3B;FE3B;3010;3010; # (︻; ︻; ︻; 【; 【; ) PRESENTATION FORM FOR VERTICAL LEFT BLACK LENTICULAR BRACKET +FE3C;FE3C;FE3C;3011;3011; # (︼; ︼; ︼; 】; 】; ) PRESENTATION FORM FOR VERTICAL RIGHT BLACK LENTICULAR BRACKET +FE3D;FE3D;FE3D;300A;300A; # (︽; ︽; ︽; 《; 《; ) PRESENTATION FORM FOR VERTICAL LEFT DOUBLE ANGLE BRACKET +FE3E;FE3E;FE3E;300B;300B; # (︾; ︾; ︾; 》; 》; ) PRESENTATION FORM FOR VERTICAL RIGHT DOUBLE ANGLE BRACKET +FE3F;FE3F;FE3F;3008;3008; # (︿; ︿; ︿; 〈; 〈; ) PRESENTATION FORM FOR VERTICAL LEFT ANGLE BRACKET +FE40;FE40;FE40;3009;3009; # (﹀; ﹀; ﹀; 〉; 〉; ) PRESENTATION FORM FOR VERTICAL RIGHT ANGLE BRACKET +FE41;FE41;FE41;300C;300C; # (﹁; ﹁; ﹁; 「; 「; ) PRESENTATION FORM FOR VERTICAL LEFT CORNER BRACKET +FE42;FE42;FE42;300D;300D; # (﹂; ﹂; ﹂; 」; 」; ) PRESENTATION FORM FOR VERTICAL RIGHT CORNER BRACKET +FE43;FE43;FE43;300E;300E; # (﹃; ﹃; ﹃; 『; 『; ) PRESENTATION FORM FOR VERTICAL LEFT WHITE CORNER BRACKET +FE44;FE44;FE44;300F;300F; # (﹄; ﹄; ﹄; 』; 』; ) PRESENTATION FORM FOR VERTICAL RIGHT WHITE CORNER BRACKET +FE49;FE49;FE49;0020 0305;0020 0305; # (﹉; ﹉; ﹉; ◌̅; ◌̅; ) DASHED OVERLINE +FE4A;FE4A;FE4A;0020 0305;0020 0305; # (﹊; ﹊; ﹊; ◌̅; ◌̅; ) CENTRELINE OVERLINE +FE4B;FE4B;FE4B;0020 0305;0020 0305; # (﹋; ﹋; ﹋; ◌̅; ◌̅; ) WAVY OVERLINE +FE4C;FE4C;FE4C;0020 0305;0020 0305; # (﹌; ﹌; ﹌; ◌̅; ◌̅; ) DOUBLE WAVY OVERLINE +FE4D;FE4D;FE4D;005F;005F; # (﹍; ﹍; ﹍; _; _; ) DASHED LOW LINE +FE4E;FE4E;FE4E;005F;005F; # (﹎; ﹎; ﹎; _; _; ) CENTRELINE LOW LINE +FE4F;FE4F;FE4F;005F;005F; # (﹏; ﹏; ﹏; _; _; ) WAVY LOW LINE +FE50;FE50;FE50;002C;002C; # (﹐; ﹐; ﹐; ,; ,; ) SMALL COMMA +FE51;FE51;FE51;3001;3001; # (﹑; ﹑; ﹑; 、; 、; ) SMALL IDEOGRAPHIC COMMA +FE52;FE52;FE52;002E;002E; # (﹒; ﹒; ﹒; .; .; ) SMALL FULL STOP +FE54;FE54;FE54;003B;003B; # (﹔; ﹔; ﹔; ;; ;; ) SMALL SEMICOLON +FE55;FE55;FE55;003A;003A; # (﹕; ﹕; ﹕; :; :; ) SMALL COLON +FE56;FE56;FE56;003F;003F; # (﹖; ﹖; ﹖; ?; ?; ) SMALL QUESTION MARK +FE57;FE57;FE57;0021;0021; # (﹗; ﹗; ﹗; !; !; ) SMALL EXCLAMATION MARK +FE58;FE58;FE58;2014;2014; # (﹘; ﹘; ﹘; —; —; ) SMALL EM DASH +FE59;FE59;FE59;0028;0028; # (﹙; ﹙; ﹙; (; (; ) SMALL LEFT PARENTHESIS +FE5A;FE5A;FE5A;0029;0029; # (﹚; ﹚; ﹚; ); ); ) SMALL RIGHT PARENTHESIS +FE5B;FE5B;FE5B;007B;007B; # (﹛; ﹛; ﹛; {; {; ) SMALL LEFT CURLY BRACKET +FE5C;FE5C;FE5C;007D;007D; # (﹜; ﹜; ﹜; }; }; ) SMALL RIGHT CURLY BRACKET +FE5D;FE5D;FE5D;3014;3014; # (﹝; ﹝; ﹝; 〔; 〔; ) SMALL LEFT TORTOISE SHELL BRACKET +FE5E;FE5E;FE5E;3015;3015; # (﹞; ﹞; ﹞; 〕; 〕; ) SMALL RIGHT TORTOISE SHELL BRACKET +FE5F;FE5F;FE5F;0023;0023; # (﹟; ﹟; ﹟; #; #; ) SMALL NUMBER SIGN +FE60;FE60;FE60;0026;0026; # (﹠; ﹠; ﹠; &; &; ) SMALL AMPERSAND +FE61;FE61;FE61;002A;002A; # (﹡; ﹡; ﹡; *; *; ) SMALL ASTERISK +FE62;FE62;FE62;002B;002B; # (﹢; ﹢; ﹢; +; +; ) SMALL PLUS SIGN +FE63;FE63;FE63;002D;002D; # (﹣; ﹣; ﹣; -; -; ) SMALL HYPHEN-MINUS +FE64;FE64;FE64;003C;003C; # (﹤; ﹤; ﹤; <; <; ) SMALL LESS-THAN SIGN +FE65;FE65;FE65;003E;003E; # (﹥; ﹥; ﹥; >; >; ) SMALL GREATER-THAN SIGN +FE66;FE66;FE66;003D;003D; # (﹦; ﹦; ﹦; =; =; ) SMALL EQUALS SIGN +FE68;FE68;FE68;005C;005C; # (﹨; ﹨; ﹨; \; \; ) SMALL REVERSE SOLIDUS +FE69;FE69;FE69;0024;0024; # (﹩; ﹩; ﹩; $; $; ) SMALL DOLLAR SIGN +FE6A;FE6A;FE6A;0025;0025; # (﹪; ﹪; ﹪; %; %; ) SMALL PERCENT SIGN +FE6B;FE6B;FE6B;0040;0040; # (﹫; ﹫; ﹫; @; @; ) SMALL COMMERCIAL AT +FE70;FE70;FE70;0020 064B;0020 064B; # (ﹰ; ﹰ; ﹰ; ◌ً; ◌ً; ) ARABIC FATHATAN ISOLATED FORM +FE71;FE71;FE71;0640 064B;0640 064B; # (ﹱ; ﹱ; ﹱ; ـ◌ً; ـ◌ً; ) ARABIC TATWEEL WITH FATHATAN ABOVE +FE72;FE72;FE72;0020 064C;0020 064C; # (ﹲ; ﹲ; ﹲ; ◌ٌ; ◌ٌ; ) ARABIC DAMMATAN ISOLATED FORM +FE74;FE74;FE74;0020 064D;0020 064D; # (ﹴ; ﹴ; ﹴ; ◌ٍ; ◌ٍ; ) ARABIC KASRATAN ISOLATED FORM +FE76;FE76;FE76;0020 064E;0020 064E; # (ﹶ; ﹶ; ﹶ; ◌َ; ◌َ; ) ARABIC FATHA ISOLATED FORM +FE77;FE77;FE77;0640 064E;0640 064E; # (ﹷ; ﹷ; ﹷ; ـ◌َ; ـ◌َ; ) ARABIC FATHA MEDIAL FORM +FE78;FE78;FE78;0020 064F;0020 064F; # (ﹸ; ﹸ; ﹸ; ◌ُ; ◌ُ; ) ARABIC DAMMA ISOLATED FORM +FE79;FE79;FE79;0640 064F;0640 064F; # (ﹹ; ﹹ; ﹹ; ـ◌ُ; ـ◌ُ; ) ARABIC DAMMA MEDIAL FORM +FE7A;FE7A;FE7A;0020 0650;0020 0650; # (ﹺ; ﹺ; ﹺ; ◌ِ; ◌ِ; ) ARABIC KASRA ISOLATED FORM +FE7B;FE7B;FE7B;0640 0650;0640 0650; # (ﹻ; ﹻ; ﹻ; ـ◌ِ; ـ◌ِ; ) ARABIC KASRA MEDIAL FORM +FE7C;FE7C;FE7C;0020 0651;0020 0651; # (ﹼ; ﹼ; ﹼ; ◌ّ; ◌ّ; ) ARABIC SHADDA ISOLATED FORM +FE7D;FE7D;FE7D;0640 0651;0640 0651; # (ﹽ; ﹽ; ﹽ; ـ◌ّ; ـ◌ّ; ) ARABIC SHADDA MEDIAL FORM +FE7E;FE7E;FE7E;0020 0652;0020 0652; # (ﹾ; ﹾ; ﹾ; ◌ْ; ◌ْ; ) ARABIC SUKUN ISOLATED FORM +FE7F;FE7F;FE7F;0640 0652;0640 0652; # (ﹿ; ﹿ; ﹿ; ـ◌ْ; ـ◌ْ; ) ARABIC SUKUN MEDIAL FORM +FE80;FE80;FE80;0621;0621; # (ﺀ; ﺀ; ﺀ; ء; ء; ) ARABIC LETTER HAMZA ISOLATED FORM +FE81;FE81;FE81;0622;0627 0653; # (ﺁ; ﺁ; ﺁ; آ; ا◌ٓ; ) ARABIC LETTER ALEF WITH MADDA ABOVE ISOLATED FORM +FE82;FE82;FE82;0622;0627 0653; # (ﺂ; ﺂ; ﺂ; آ; ا◌ٓ; ) ARABIC LETTER ALEF WITH MADDA ABOVE FINAL FORM +FE83;FE83;FE83;0623;0627 0654; # (ﺃ; ﺃ; ﺃ; أ; ا◌ٔ; ) ARABIC LETTER ALEF WITH HAMZA ABOVE ISOLATED FORM +FE84;FE84;FE84;0623;0627 0654; # (ﺄ; ﺄ; ﺄ; أ; ا◌ٔ; ) ARABIC LETTER ALEF WITH HAMZA ABOVE FINAL FORM +FE85;FE85;FE85;0624;0648 0654; # (ﺅ; ﺅ; ﺅ; ؤ; و◌ٔ; ) ARABIC LETTER WAW WITH HAMZA ABOVE ISOLATED FORM +FE86;FE86;FE86;0624;0648 0654; # (ﺆ; ﺆ; ﺆ; ؤ; و◌ٔ; ) ARABIC LETTER WAW WITH HAMZA ABOVE FINAL FORM +FE87;FE87;FE87;0625;0627 0655; # (ﺇ; ﺇ; ﺇ; إ; ا◌ٕ; ) ARABIC LETTER ALEF WITH HAMZA BELOW ISOLATED FORM +FE88;FE88;FE88;0625;0627 0655; # (ﺈ; ﺈ; ﺈ; إ; ا◌ٕ; ) ARABIC LETTER ALEF WITH HAMZA BELOW FINAL FORM +FE89;FE89;FE89;0626;064A 0654; # (ﺉ; ﺉ; ﺉ; ئ; ي◌ٔ; ) ARABIC LETTER YEH WITH HAMZA ABOVE ISOLATED FORM +FE8A;FE8A;FE8A;0626;064A 0654; # (ﺊ; ﺊ; ﺊ; ئ; ي◌ٔ; ) ARABIC LETTER YEH WITH HAMZA ABOVE FINAL FORM +FE8B;FE8B;FE8B;0626;064A 0654; # (ﺋ; ﺋ; ﺋ; ئ; ي◌ٔ; ) ARABIC LETTER YEH WITH HAMZA ABOVE INITIAL FORM +FE8C;FE8C;FE8C;0626;064A 0654; # (ﺌ; ﺌ; ﺌ; ئ; ي◌ٔ; ) ARABIC LETTER YEH WITH HAMZA ABOVE MEDIAL FORM +FE8D;FE8D;FE8D;0627;0627; # (ﺍ; ﺍ; ﺍ; ا; ا; ) ARABIC LETTER ALEF ISOLATED FORM +FE8E;FE8E;FE8E;0627;0627; # (ﺎ; ﺎ; ﺎ; ا; ا; ) ARABIC LETTER ALEF FINAL FORM +FE8F;FE8F;FE8F;0628;0628; # (ﺏ; ﺏ; ﺏ; ب; ب; ) ARABIC LETTER BEH ISOLATED FORM +FE90;FE90;FE90;0628;0628; # (ﺐ; ﺐ; ﺐ; ب; ب; ) ARABIC LETTER BEH FINAL FORM +FE91;FE91;FE91;0628;0628; # (ﺑ; ﺑ; ﺑ; ب; ب; ) ARABIC LETTER BEH INITIAL FORM +FE92;FE92;FE92;0628;0628; # (ﺒ; ﺒ; ﺒ; ب; ب; ) ARABIC LETTER BEH MEDIAL FORM +FE93;FE93;FE93;0629;0629; # (ﺓ; ﺓ; ﺓ; ة; ة; ) ARABIC LETTER TEH MARBUTA ISOLATED FORM +FE94;FE94;FE94;0629;0629; # (ﺔ; ﺔ; ﺔ; ة; ة; ) ARABIC LETTER TEH MARBUTA FINAL FORM +FE95;FE95;FE95;062A;062A; # (ﺕ; ﺕ; ﺕ; ت; ت; ) ARABIC LETTER TEH ISOLATED FORM +FE96;FE96;FE96;062A;062A; # (ﺖ; ﺖ; ﺖ; ت; ت; ) ARABIC LETTER TEH FINAL FORM +FE97;FE97;FE97;062A;062A; # (ﺗ; ﺗ; ﺗ; ت; ت; ) ARABIC LETTER TEH INITIAL FORM +FE98;FE98;FE98;062A;062A; # (ﺘ; ﺘ; ﺘ; ت; ت; ) ARABIC LETTER TEH MEDIAL FORM +FE99;FE99;FE99;062B;062B; # (ﺙ; ﺙ; ﺙ; ث; ث; ) ARABIC LETTER THEH ISOLATED FORM +FE9A;FE9A;FE9A;062B;062B; # (ﺚ; ﺚ; ﺚ; ث; ث; ) ARABIC LETTER THEH FINAL FORM +FE9B;FE9B;FE9B;062B;062B; # (ﺛ; ﺛ; ﺛ; ث; ث; ) ARABIC LETTER THEH INITIAL FORM +FE9C;FE9C;FE9C;062B;062B; # (ﺜ; ﺜ; ﺜ; ث; ث; ) ARABIC LETTER THEH MEDIAL FORM +FE9D;FE9D;FE9D;062C;062C; # (ﺝ; ﺝ; ﺝ; ج; ج; ) ARABIC LETTER JEEM ISOLATED FORM +FE9E;FE9E;FE9E;062C;062C; # (ﺞ; ﺞ; ﺞ; ج; ج; ) ARABIC LETTER JEEM FINAL FORM +FE9F;FE9F;FE9F;062C;062C; # (ﺟ; ﺟ; ﺟ; ج; ج; ) ARABIC LETTER JEEM INITIAL FORM +FEA0;FEA0;FEA0;062C;062C; # (ﺠ; ﺠ; ﺠ; ج; ج; ) ARABIC LETTER JEEM MEDIAL FORM +FEA1;FEA1;FEA1;062D;062D; # (ﺡ; ﺡ; ﺡ; ح; ح; ) ARABIC LETTER HAH ISOLATED FORM +FEA2;FEA2;FEA2;062D;062D; # (ﺢ; ﺢ; ﺢ; ح; ح; ) ARABIC LETTER HAH FINAL FORM +FEA3;FEA3;FEA3;062D;062D; # (ﺣ; ﺣ; ﺣ; ح; ح; ) ARABIC LETTER HAH INITIAL FORM +FEA4;FEA4;FEA4;062D;062D; # (ﺤ; ﺤ; ﺤ; ح; ح; ) ARABIC LETTER HAH MEDIAL FORM +FEA5;FEA5;FEA5;062E;062E; # (ﺥ; ﺥ; ﺥ; خ; خ; ) ARABIC LETTER KHAH ISOLATED FORM +FEA6;FEA6;FEA6;062E;062E; # (ﺦ; ﺦ; ﺦ; خ; خ; ) ARABIC LETTER KHAH FINAL FORM +FEA7;FEA7;FEA7;062E;062E; # (ﺧ; ﺧ; ﺧ; خ; خ; ) ARABIC LETTER KHAH INITIAL FORM +FEA8;FEA8;FEA8;062E;062E; # (ﺨ; ﺨ; ﺨ; خ; خ; ) ARABIC LETTER KHAH MEDIAL FORM +FEA9;FEA9;FEA9;062F;062F; # (ﺩ; ﺩ; ﺩ; د; د; ) ARABIC LETTER DAL ISOLATED FORM +FEAA;FEAA;FEAA;062F;062F; # (ﺪ; ﺪ; ﺪ; د; د; ) ARABIC LETTER DAL FINAL FORM +FEAB;FEAB;FEAB;0630;0630; # (ﺫ; ﺫ; ﺫ; ذ; ذ; ) ARABIC LETTER THAL ISOLATED FORM +FEAC;FEAC;FEAC;0630;0630; # (ﺬ; ﺬ; ﺬ; ذ; ذ; ) ARABIC LETTER THAL FINAL FORM +FEAD;FEAD;FEAD;0631;0631; # (ﺭ; ﺭ; ﺭ; ر; ر; ) ARABIC LETTER REH ISOLATED FORM +FEAE;FEAE;FEAE;0631;0631; # (ﺮ; ﺮ; ﺮ; ر; ر; ) ARABIC LETTER REH FINAL FORM +FEAF;FEAF;FEAF;0632;0632; # (ﺯ; ﺯ; ﺯ; ز; ز; ) ARABIC LETTER ZAIN ISOLATED FORM +FEB0;FEB0;FEB0;0632;0632; # (ﺰ; ﺰ; ﺰ; ز; ز; ) ARABIC LETTER ZAIN FINAL FORM +FEB1;FEB1;FEB1;0633;0633; # (ﺱ; ﺱ; ﺱ; س; س; ) ARABIC LETTER SEEN ISOLATED FORM +FEB2;FEB2;FEB2;0633;0633; # (ﺲ; ﺲ; ﺲ; س; س; ) ARABIC LETTER SEEN FINAL FORM +FEB3;FEB3;FEB3;0633;0633; # (ﺳ; ﺳ; ﺳ; س; س; ) ARABIC LETTER SEEN INITIAL FORM +FEB4;FEB4;FEB4;0633;0633; # (ﺴ; ﺴ; ﺴ; س; س; ) ARABIC LETTER SEEN MEDIAL FORM +FEB5;FEB5;FEB5;0634;0634; # (ﺵ; ﺵ; ﺵ; ش; ش; ) ARABIC LETTER SHEEN ISOLATED FORM +FEB6;FEB6;FEB6;0634;0634; # (ﺶ; ﺶ; ﺶ; ش; ش; ) ARABIC LETTER SHEEN FINAL FORM +FEB7;FEB7;FEB7;0634;0634; # (ﺷ; ﺷ; ﺷ; ش; ش; ) ARABIC LETTER SHEEN INITIAL FORM +FEB8;FEB8;FEB8;0634;0634; # (ﺸ; ﺸ; ﺸ; ش; ش; ) ARABIC LETTER SHEEN MEDIAL FORM +FEB9;FEB9;FEB9;0635;0635; # (ﺹ; ﺹ; ﺹ; ص; ص; ) ARABIC LETTER SAD ISOLATED FORM +FEBA;FEBA;FEBA;0635;0635; # (ﺺ; ﺺ; ﺺ; ص; ص; ) ARABIC LETTER SAD FINAL FORM +FEBB;FEBB;FEBB;0635;0635; # (ﺻ; ﺻ; ﺻ; ص; ص; ) ARABIC LETTER SAD INITIAL FORM +FEBC;FEBC;FEBC;0635;0635; # (ﺼ; ﺼ; ﺼ; ص; ص; ) ARABIC LETTER SAD MEDIAL FORM +FEBD;FEBD;FEBD;0636;0636; # (ﺽ; ﺽ; ﺽ; ض; ض; ) ARABIC LETTER DAD ISOLATED FORM +FEBE;FEBE;FEBE;0636;0636; # (ﺾ; ﺾ; ﺾ; ض; ض; ) ARABIC LETTER DAD FINAL FORM +FEBF;FEBF;FEBF;0636;0636; # (ﺿ; ﺿ; ﺿ; ض; ض; ) ARABIC LETTER DAD INITIAL FORM +FEC0;FEC0;FEC0;0636;0636; # (ﻀ; ﻀ; ﻀ; ض; ض; ) ARABIC LETTER DAD MEDIAL FORM +FEC1;FEC1;FEC1;0637;0637; # (ﻁ; ﻁ; ﻁ; ط; ط; ) ARABIC LETTER TAH ISOLATED FORM +FEC2;FEC2;FEC2;0637;0637; # (ﻂ; ﻂ; ﻂ; ط; ط; ) ARABIC LETTER TAH FINAL FORM +FEC3;FEC3;FEC3;0637;0637; # (ﻃ; ﻃ; ﻃ; ط; ط; ) ARABIC LETTER TAH INITIAL FORM +FEC4;FEC4;FEC4;0637;0637; # (ﻄ; ﻄ; ﻄ; ط; ط; ) ARABIC LETTER TAH MEDIAL FORM +FEC5;FEC5;FEC5;0638;0638; # (ﻅ; ﻅ; ﻅ; ظ; ظ; ) ARABIC LETTER ZAH ISOLATED FORM +FEC6;FEC6;FEC6;0638;0638; # (ﻆ; ﻆ; ﻆ; ظ; ظ; ) ARABIC LETTER ZAH FINAL FORM +FEC7;FEC7;FEC7;0638;0638; # (ﻇ; ﻇ; ﻇ; ظ; ظ; ) ARABIC LETTER ZAH INITIAL FORM +FEC8;FEC8;FEC8;0638;0638; # (ﻈ; ﻈ; ﻈ; ظ; ظ; ) ARABIC LETTER ZAH MEDIAL FORM +FEC9;FEC9;FEC9;0639;0639; # (ﻉ; ﻉ; ﻉ; ع; ع; ) ARABIC LETTER AIN ISOLATED FORM +FECA;FECA;FECA;0639;0639; # (ﻊ; ﻊ; ﻊ; ع; ع; ) ARABIC LETTER AIN FINAL FORM +FECB;FECB;FECB;0639;0639; # (ﻋ; ﻋ; ﻋ; ع; ع; ) ARABIC LETTER AIN INITIAL FORM +FECC;FECC;FECC;0639;0639; # (ﻌ; ﻌ; ﻌ; ع; ع; ) ARABIC LETTER AIN MEDIAL FORM +FECD;FECD;FECD;063A;063A; # (ﻍ; ﻍ; ﻍ; غ; غ; ) ARABIC LETTER GHAIN ISOLATED FORM +FECE;FECE;FECE;063A;063A; # (ﻎ; ﻎ; ﻎ; غ; غ; ) ARABIC LETTER GHAIN FINAL FORM +FECF;FECF;FECF;063A;063A; # (ﻏ; ﻏ; ﻏ; غ; غ; ) ARABIC LETTER GHAIN INITIAL FORM +FED0;FED0;FED0;063A;063A; # (ﻐ; ﻐ; ﻐ; غ; غ; ) ARABIC LETTER GHAIN MEDIAL FORM +FED1;FED1;FED1;0641;0641; # (ﻑ; ﻑ; ﻑ; ف; ف; ) ARABIC LETTER FEH ISOLATED FORM +FED2;FED2;FED2;0641;0641; # (ﻒ; ﻒ; ﻒ; ف; ف; ) ARABIC LETTER FEH FINAL FORM +FED3;FED3;FED3;0641;0641; # (ﻓ; ﻓ; ﻓ; ف; ف; ) ARABIC LETTER FEH INITIAL FORM +FED4;FED4;FED4;0641;0641; # (ﻔ; ﻔ; ﻔ; ف; ف; ) ARABIC LETTER FEH MEDIAL FORM +FED5;FED5;FED5;0642;0642; # (ﻕ; ﻕ; ﻕ; ق; ق; ) ARABIC LETTER QAF ISOLATED FORM +FED6;FED6;FED6;0642;0642; # (ﻖ; ﻖ; ﻖ; ق; ق; ) ARABIC LETTER QAF FINAL FORM +FED7;FED7;FED7;0642;0642; # (ﻗ; ﻗ; ﻗ; ق; ق; ) ARABIC LETTER QAF INITIAL FORM +FED8;FED8;FED8;0642;0642; # (ﻘ; ﻘ; ﻘ; ق; ق; ) ARABIC LETTER QAF MEDIAL FORM +FED9;FED9;FED9;0643;0643; # (ﻙ; ﻙ; ﻙ; ك; ك; ) ARABIC LETTER KAF ISOLATED FORM +FEDA;FEDA;FEDA;0643;0643; # (ﻚ; ﻚ; ﻚ; ك; ك; ) ARABIC LETTER KAF FINAL FORM +FEDB;FEDB;FEDB;0643;0643; # (ﻛ; ﻛ; ﻛ; ك; ك; ) ARABIC LETTER KAF INITIAL FORM +FEDC;FEDC;FEDC;0643;0643; # (ﻜ; ﻜ; ﻜ; ك; ك; ) ARABIC LETTER KAF MEDIAL FORM +FEDD;FEDD;FEDD;0644;0644; # (ﻝ; ﻝ; ﻝ; ل; ل; ) ARABIC LETTER LAM ISOLATED FORM +FEDE;FEDE;FEDE;0644;0644; # (ﻞ; ﻞ; ﻞ; ل; ل; ) ARABIC LETTER LAM FINAL FORM +FEDF;FEDF;FEDF;0644;0644; # (ﻟ; ﻟ; ﻟ; ل; ل; ) ARABIC LETTER LAM INITIAL FORM +FEE0;FEE0;FEE0;0644;0644; # (ﻠ; ﻠ; ﻠ; ل; ل; ) ARABIC LETTER LAM MEDIAL FORM +FEE1;FEE1;FEE1;0645;0645; # (ﻡ; ﻡ; ﻡ; م; م; ) ARABIC LETTER MEEM ISOLATED FORM +FEE2;FEE2;FEE2;0645;0645; # (ﻢ; ﻢ; ﻢ; م; م; ) ARABIC LETTER MEEM FINAL FORM +FEE3;FEE3;FEE3;0645;0645; # (ﻣ; ﻣ; ﻣ; م; م; ) ARABIC LETTER MEEM INITIAL FORM +FEE4;FEE4;FEE4;0645;0645; # (ﻤ; ﻤ; ﻤ; م; م; ) ARABIC LETTER MEEM MEDIAL FORM +FEE5;FEE5;FEE5;0646;0646; # (ﻥ; ﻥ; ﻥ; ن; ن; ) ARABIC LETTER NOON ISOLATED FORM +FEE6;FEE6;FEE6;0646;0646; # (ﻦ; ﻦ; ﻦ; ن; ن; ) ARABIC LETTER NOON FINAL FORM +FEE7;FEE7;FEE7;0646;0646; # (ﻧ; ﻧ; ﻧ; ن; ن; ) ARABIC LETTER NOON INITIAL FORM +FEE8;FEE8;FEE8;0646;0646; # (ﻨ; ﻨ; ﻨ; ن; ن; ) ARABIC LETTER NOON MEDIAL FORM +FEE9;FEE9;FEE9;0647;0647; # (ﻩ; ﻩ; ﻩ; ه; ه; ) ARABIC LETTER HEH ISOLATED FORM +FEEA;FEEA;FEEA;0647;0647; # (ﻪ; ﻪ; ﻪ; ه; ه; ) ARABIC LETTER HEH FINAL FORM +FEEB;FEEB;FEEB;0647;0647; # (ﻫ; ﻫ; ﻫ; ه; ه; ) ARABIC LETTER HEH INITIAL FORM +FEEC;FEEC;FEEC;0647;0647; # (ﻬ; ﻬ; ﻬ; ه; ه; ) ARABIC LETTER HEH MEDIAL FORM +FEED;FEED;FEED;0648;0648; # (ﻭ; ﻭ; ﻭ; و; و; ) ARABIC LETTER WAW ISOLATED FORM +FEEE;FEEE;FEEE;0648;0648; # (ﻮ; ﻮ; ﻮ; و; و; ) ARABIC LETTER WAW FINAL FORM +FEEF;FEEF;FEEF;0649;0649; # (ﻯ; ﻯ; ﻯ; ى; ى; ) ARABIC LETTER ALEF MAKSURA ISOLATED FORM +FEF0;FEF0;FEF0;0649;0649; # (ﻰ; ﻰ; ﻰ; ى; ى; ) ARABIC LETTER ALEF MAKSURA FINAL FORM +FEF1;FEF1;FEF1;064A;064A; # (ﻱ; ﻱ; ﻱ; ي; ي; ) ARABIC LETTER YEH ISOLATED FORM +FEF2;FEF2;FEF2;064A;064A; # (ﻲ; ﻲ; ﻲ; ي; ي; ) ARABIC LETTER YEH FINAL FORM +FEF3;FEF3;FEF3;064A;064A; # (ﻳ; ﻳ; ﻳ; ي; ي; ) ARABIC LETTER YEH INITIAL FORM +FEF4;FEF4;FEF4;064A;064A; # (ﻴ; ﻴ; ﻴ; ي; ي; ) ARABIC LETTER YEH MEDIAL FORM +FEF5;FEF5;FEF5;0644 0622;0644 0627 0653; # (ﻵ; ﻵ; ﻵ; لآ; لا◌ٓ; ) ARABIC LIGATURE LAM WITH ALEF WITH MADDA ABOVE ISOLATED FORM +FEF6;FEF6;FEF6;0644 0622;0644 0627 0653; # (ﻶ; ﻶ; ﻶ; لآ; لا◌ٓ; ) ARABIC LIGATURE LAM WITH ALEF WITH MADDA ABOVE FINAL FORM +FEF7;FEF7;FEF7;0644 0623;0644 0627 0654; # (ﻷ; ﻷ; ﻷ; لأ; لا◌ٔ; ) ARABIC LIGATURE LAM WITH ALEF WITH HAMZA ABOVE ISOLATED FORM +FEF8;FEF8;FEF8;0644 0623;0644 0627 0654; # (ﻸ; ﻸ; ﻸ; لأ; لا◌ٔ; ) ARABIC LIGATURE LAM WITH ALEF WITH HAMZA ABOVE FINAL FORM +FEF9;FEF9;FEF9;0644 0625;0644 0627 0655; # (ﻹ; ﻹ; ﻹ; لإ; لا◌ٕ; ) ARABIC LIGATURE LAM WITH ALEF WITH HAMZA BELOW ISOLATED FORM +FEFA;FEFA;FEFA;0644 0625;0644 0627 0655; # (ﻺ; ﻺ; ﻺ; لإ; لا◌ٕ; ) ARABIC LIGATURE LAM WITH ALEF WITH HAMZA BELOW FINAL FORM +FEFB;FEFB;FEFB;0644 0627;0644 0627; # (ﻻ; ﻻ; ﻻ; لا; لا; ) ARABIC LIGATURE LAM WITH ALEF ISOLATED FORM +FEFC;FEFC;FEFC;0644 0627;0644 0627; # (ﻼ; ﻼ; ﻼ; لا; لا; ) ARABIC LIGATURE LAM WITH ALEF FINAL FORM +FF01;FF01;FF01;0021;0021; # (!; !; !; !; !; ) FULLWIDTH EXCLAMATION MARK +FF02;FF02;FF02;0022;0022; # ("; "; "; "; "; ) FULLWIDTH QUOTATION MARK +FF03;FF03;FF03;0023;0023; # (#; #; #; #; #; ) FULLWIDTH NUMBER SIGN +FF04;FF04;FF04;0024;0024; # ($; $; $; $; $; ) FULLWIDTH DOLLAR SIGN +FF05;FF05;FF05;0025;0025; # (%; %; %; %; %; ) FULLWIDTH PERCENT SIGN +FF06;FF06;FF06;0026;0026; # (&; &; &; &; &; ) FULLWIDTH AMPERSAND +FF07;FF07;FF07;0027;0027; # ('; '; '; '; '; ) FULLWIDTH APOSTROPHE +FF08;FF08;FF08;0028;0028; # ((; (; (; (; (; ) FULLWIDTH LEFT PARENTHESIS +FF09;FF09;FF09;0029;0029; # (); ); ); ); ); ) FULLWIDTH RIGHT PARENTHESIS +FF0A;FF0A;FF0A;002A;002A; # (*; *; *; *; *; ) FULLWIDTH ASTERISK +FF0B;FF0B;FF0B;002B;002B; # (+; +; +; +; +; ) FULLWIDTH PLUS SIGN +FF0C;FF0C;FF0C;002C;002C; # (,; ,; ,; ,; ,; ) FULLWIDTH COMMA +FF0D;FF0D;FF0D;002D;002D; # (-; -; -; -; -; ) FULLWIDTH HYPHEN-MINUS +FF0E;FF0E;FF0E;002E;002E; # (.; .; .; .; .; ) FULLWIDTH FULL STOP +FF0F;FF0F;FF0F;002F;002F; # (/; /; /; /; /; ) FULLWIDTH SOLIDUS +FF10;FF10;FF10;0030;0030; # (0; 0; 0; 0; 0; ) FULLWIDTH DIGIT ZERO +FF11;FF11;FF11;0031;0031; # (1; 1; 1; 1; 1; ) FULLWIDTH DIGIT ONE +FF12;FF12;FF12;0032;0032; # (2; 2; 2; 2; 2; ) FULLWIDTH DIGIT TWO +FF13;FF13;FF13;0033;0033; # (3; 3; 3; 3; 3; ) FULLWIDTH DIGIT THREE +FF14;FF14;FF14;0034;0034; # (4; 4; 4; 4; 4; ) FULLWIDTH DIGIT FOUR +FF15;FF15;FF15;0035;0035; # (5; 5; 5; 5; 5; ) FULLWIDTH DIGIT FIVE +FF16;FF16;FF16;0036;0036; # (6; 6; 6; 6; 6; ) FULLWIDTH DIGIT SIX +FF17;FF17;FF17;0037;0037; # (7; 7; 7; 7; 7; ) FULLWIDTH DIGIT SEVEN +FF18;FF18;FF18;0038;0038; # (8; 8; 8; 8; 8; ) FULLWIDTH DIGIT EIGHT +FF19;FF19;FF19;0039;0039; # (9; 9; 9; 9; 9; ) FULLWIDTH DIGIT NINE +FF1A;FF1A;FF1A;003A;003A; # (:; :; :; :; :; ) FULLWIDTH COLON +FF1B;FF1B;FF1B;003B;003B; # (;; ;; ;; ;; ;; ) FULLWIDTH SEMICOLON +FF1C;FF1C;FF1C;003C;003C; # (<; <; <; <; <; ) FULLWIDTH LESS-THAN SIGN +FF1D;FF1D;FF1D;003D;003D; # (=; =; =; =; =; ) FULLWIDTH EQUALS SIGN +FF1E;FF1E;FF1E;003E;003E; # (>; >; >; >; >; ) FULLWIDTH GREATER-THAN SIGN +FF1F;FF1F;FF1F;003F;003F; # (?; ?; ?; ?; ?; ) FULLWIDTH QUESTION MARK +FF20;FF20;FF20;0040;0040; # (@; @; @; @; @; ) FULLWIDTH COMMERCIAL AT +FF21;FF21;FF21;0041;0041; # (A; A; A; A; A; ) FULLWIDTH LATIN CAPITAL LETTER A +FF22;FF22;FF22;0042;0042; # (B; B; B; B; B; ) FULLWIDTH LATIN CAPITAL LETTER B +FF23;FF23;FF23;0043;0043; # (C; C; C; C; C; ) FULLWIDTH LATIN CAPITAL LETTER C +FF24;FF24;FF24;0044;0044; # (D; D; D; D; D; ) FULLWIDTH LATIN CAPITAL LETTER D +FF25;FF25;FF25;0045;0045; # (E; E; E; E; E; ) FULLWIDTH LATIN CAPITAL LETTER E +FF26;FF26;FF26;0046;0046; # (F; F; F; F; F; ) FULLWIDTH LATIN CAPITAL LETTER F +FF27;FF27;FF27;0047;0047; # (G; G; G; G; G; ) FULLWIDTH LATIN CAPITAL LETTER G +FF28;FF28;FF28;0048;0048; # (H; H; H; H; H; ) FULLWIDTH LATIN CAPITAL LETTER H +FF29;FF29;FF29;0049;0049; # (I; I; I; I; I; ) FULLWIDTH LATIN CAPITAL LETTER I +FF2A;FF2A;FF2A;004A;004A; # (J; J; J; J; J; ) FULLWIDTH LATIN CAPITAL LETTER J +FF2B;FF2B;FF2B;004B;004B; # (K; K; K; K; K; ) FULLWIDTH LATIN CAPITAL LETTER K +FF2C;FF2C;FF2C;004C;004C; # (L; L; L; L; L; ) FULLWIDTH LATIN CAPITAL LETTER L +FF2D;FF2D;FF2D;004D;004D; # (M; M; M; M; M; ) FULLWIDTH LATIN CAPITAL LETTER M +FF2E;FF2E;FF2E;004E;004E; # (N; N; N; N; N; ) FULLWIDTH LATIN CAPITAL LETTER N +FF2F;FF2F;FF2F;004F;004F; # (O; O; O; O; O; ) FULLWIDTH LATIN CAPITAL LETTER O +FF30;FF30;FF30;0050;0050; # (P; P; P; P; P; ) FULLWIDTH LATIN CAPITAL LETTER P +FF31;FF31;FF31;0051;0051; # (Q; Q; Q; Q; Q; ) FULLWIDTH LATIN CAPITAL LETTER Q +FF32;FF32;FF32;0052;0052; # (R; R; R; R; R; ) FULLWIDTH LATIN CAPITAL LETTER R +FF33;FF33;FF33;0053;0053; # (S; S; S; S; S; ) FULLWIDTH LATIN CAPITAL LETTER S +FF34;FF34;FF34;0054;0054; # (T; T; T; T; T; ) FULLWIDTH LATIN CAPITAL LETTER T +FF35;FF35;FF35;0055;0055; # (U; U; U; U; U; ) FULLWIDTH LATIN CAPITAL LETTER U +FF36;FF36;FF36;0056;0056; # (V; V; V; V; V; ) FULLWIDTH LATIN CAPITAL LETTER V +FF37;FF37;FF37;0057;0057; # (W; W; W; W; W; ) FULLWIDTH LATIN CAPITAL LETTER W +FF38;FF38;FF38;0058;0058; # (X; X; X; X; X; ) FULLWIDTH LATIN CAPITAL LETTER X +FF39;FF39;FF39;0059;0059; # (Y; Y; Y; Y; Y; ) FULLWIDTH LATIN CAPITAL LETTER Y +FF3A;FF3A;FF3A;005A;005A; # (Z; Z; Z; Z; Z; ) FULLWIDTH LATIN CAPITAL LETTER Z +FF3B;FF3B;FF3B;005B;005B; # ([; [; [; [; [; ) FULLWIDTH LEFT SQUARE BRACKET +FF3C;FF3C;FF3C;005C;005C; # (\; \; \; \; \; ) FULLWIDTH REVERSE SOLIDUS +FF3D;FF3D;FF3D;005D;005D; # (]; ]; ]; ]; ]; ) FULLWIDTH RIGHT SQUARE BRACKET +FF3E;FF3E;FF3E;005E;005E; # (^; ^; ^; ^; ^; ) FULLWIDTH CIRCUMFLEX ACCENT +FF3F;FF3F;FF3F;005F;005F; # (_; _; _; _; _; ) FULLWIDTH LOW LINE +FF40;FF40;FF40;0060;0060; # (`; `; `; `; `; ) FULLWIDTH GRAVE ACCENT +FF41;FF41;FF41;0061;0061; # (a; a; a; a; a; ) FULLWIDTH LATIN SMALL LETTER A +FF42;FF42;FF42;0062;0062; # (b; b; b; b; b; ) FULLWIDTH LATIN SMALL LETTER B +FF43;FF43;FF43;0063;0063; # (c; c; c; c; c; ) FULLWIDTH LATIN SMALL LETTER C +FF44;FF44;FF44;0064;0064; # (d; d; d; d; d; ) FULLWIDTH LATIN SMALL LETTER D +FF45;FF45;FF45;0065;0065; # (e; e; e; e; e; ) FULLWIDTH LATIN SMALL LETTER E +FF46;FF46;FF46;0066;0066; # (f; f; f; f; f; ) FULLWIDTH LATIN SMALL LETTER F +FF47;FF47;FF47;0067;0067; # (g; g; g; g; g; ) FULLWIDTH LATIN SMALL LETTER G +FF48;FF48;FF48;0068;0068; # (h; h; h; h; h; ) FULLWIDTH LATIN SMALL LETTER H +FF49;FF49;FF49;0069;0069; # (i; i; i; i; i; ) FULLWIDTH LATIN SMALL LETTER I +FF4A;FF4A;FF4A;006A;006A; # (j; j; j; j; j; ) FULLWIDTH LATIN SMALL LETTER J +FF4B;FF4B;FF4B;006B;006B; # (k; k; k; k; k; ) FULLWIDTH LATIN SMALL LETTER K +FF4C;FF4C;FF4C;006C;006C; # (l; l; l; l; l; ) FULLWIDTH LATIN SMALL LETTER L +FF4D;FF4D;FF4D;006D;006D; # (m; m; m; m; m; ) FULLWIDTH LATIN SMALL LETTER M +FF4E;FF4E;FF4E;006E;006E; # (n; n; n; n; n; ) FULLWIDTH LATIN SMALL LETTER N +FF4F;FF4F;FF4F;006F;006F; # (o; o; o; o; o; ) FULLWIDTH LATIN SMALL LETTER O +FF50;FF50;FF50;0070;0070; # (p; p; p; p; p; ) FULLWIDTH LATIN SMALL LETTER P +FF51;FF51;FF51;0071;0071; # (q; q; q; q; q; ) FULLWIDTH LATIN SMALL LETTER Q +FF52;FF52;FF52;0072;0072; # (r; r; r; r; r; ) FULLWIDTH LATIN SMALL LETTER R +FF53;FF53;FF53;0073;0073; # (s; s; s; s; s; ) FULLWIDTH LATIN SMALL LETTER S +FF54;FF54;FF54;0074;0074; # (t; t; t; t; t; ) FULLWIDTH LATIN SMALL LETTER T +FF55;FF55;FF55;0075;0075; # (u; u; u; u; u; ) FULLWIDTH LATIN SMALL LETTER U +FF56;FF56;FF56;0076;0076; # (v; v; v; v; v; ) FULLWIDTH LATIN SMALL LETTER V +FF57;FF57;FF57;0077;0077; # (w; w; w; w; w; ) FULLWIDTH LATIN SMALL LETTER W +FF58;FF58;FF58;0078;0078; # (x; x; x; x; x; ) FULLWIDTH LATIN SMALL LETTER X +FF59;FF59;FF59;0079;0079; # (y; y; y; y; y; ) FULLWIDTH LATIN SMALL LETTER Y +FF5A;FF5A;FF5A;007A;007A; # (z; z; z; z; z; ) FULLWIDTH LATIN SMALL LETTER Z +FF5B;FF5B;FF5B;007B;007B; # ({; {; {; {; {; ) FULLWIDTH LEFT CURLY BRACKET +FF5C;FF5C;FF5C;007C;007C; # (|; |; |; |; |; ) FULLWIDTH VERTICAL LINE +FF5D;FF5D;FF5D;007D;007D; # (}; }; }; }; }; ) FULLWIDTH RIGHT CURLY BRACKET +FF5E;FF5E;FF5E;007E;007E; # (~; ~; ~; ~; ~; ) FULLWIDTH TILDE +FF5F;FF5F;FF5F;2985;2985; # (⦅; ⦅; ⦅; ⦅; ⦅; ) FULLWIDTH LEFT WHITE PARENTHESIS +FF60;FF60;FF60;2986;2986; # (⦆; ⦆; ⦆; ⦆; ⦆; ) FULLWIDTH RIGHT WHITE PARENTHESIS +FF61;FF61;FF61;3002;3002; # (。; 。; 。; 。; 。; ) HALFWIDTH IDEOGRAPHIC FULL STOP +FF62;FF62;FF62;300C;300C; # (「; 「; 「; 「; 「; ) HALFWIDTH LEFT CORNER BRACKET +FF63;FF63;FF63;300D;300D; # (」; 」; 」; 」; 」; ) HALFWIDTH RIGHT CORNER BRACKET +FF64;FF64;FF64;3001;3001; # (、; 、; 、; 、; 、; ) HALFWIDTH IDEOGRAPHIC COMMA +FF65;FF65;FF65;30FB;30FB; # (・; ・; ・; ・; ・; ) HALFWIDTH KATAKANA MIDDLE DOT +FF66;FF66;FF66;30F2;30F2; # (ヲ; ヲ; ヲ; ヲ; ヲ; ) HALFWIDTH KATAKANA LETTER WO +FF67;FF67;FF67;30A1;30A1; # (ァ; ァ; ァ; ァ; ァ; ) HALFWIDTH KATAKANA LETTER SMALL A +FF68;FF68;FF68;30A3;30A3; # (ィ; ィ; ィ; ィ; ィ; ) HALFWIDTH KATAKANA LETTER SMALL I +FF69;FF69;FF69;30A5;30A5; # (ゥ; ゥ; ゥ; ゥ; ゥ; ) HALFWIDTH KATAKANA LETTER SMALL U +FF6A;FF6A;FF6A;30A7;30A7; # (ェ; ェ; ェ; ェ; ェ; ) HALFWIDTH KATAKANA LETTER SMALL E +FF6B;FF6B;FF6B;30A9;30A9; # (ォ; ォ; ォ; ォ; ォ; ) HALFWIDTH KATAKANA LETTER SMALL O +FF6C;FF6C;FF6C;30E3;30E3; # (ャ; ャ; ャ; ャ; ャ; ) HALFWIDTH KATAKANA LETTER SMALL YA +FF6D;FF6D;FF6D;30E5;30E5; # (ュ; ュ; ュ; ュ; ュ; ) HALFWIDTH KATAKANA LETTER SMALL YU +FF6E;FF6E;FF6E;30E7;30E7; # (ョ; ョ; ョ; ョ; ョ; ) HALFWIDTH KATAKANA LETTER SMALL YO +FF6F;FF6F;FF6F;30C3;30C3; # (ッ; ッ; ッ; ッ; ッ; ) HALFWIDTH KATAKANA LETTER SMALL TU +FF70;FF70;FF70;30FC;30FC; # (ー; ー; ー; ー; ー; ) HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK +FF71;FF71;FF71;30A2;30A2; # (ア; ア; ア; ア; ア; ) HALFWIDTH KATAKANA LETTER A +FF72;FF72;FF72;30A4;30A4; # (イ; イ; イ; イ; イ; ) HALFWIDTH KATAKANA LETTER I +FF73;FF73;FF73;30A6;30A6; # (ウ; ウ; ウ; ウ; ウ; ) HALFWIDTH KATAKANA LETTER U +FF74;FF74;FF74;30A8;30A8; # (エ; エ; エ; エ; エ; ) HALFWIDTH KATAKANA LETTER E +FF75;FF75;FF75;30AA;30AA; # (オ; オ; オ; オ; オ; ) HALFWIDTH KATAKANA LETTER O +FF76;FF76;FF76;30AB;30AB; # (カ; カ; カ; カ; カ; ) HALFWIDTH KATAKANA LETTER KA +FF77;FF77;FF77;30AD;30AD; # (キ; キ; キ; キ; キ; ) HALFWIDTH KATAKANA LETTER KI +FF78;FF78;FF78;30AF;30AF; # (ク; ク; ク; ク; ク; ) HALFWIDTH KATAKANA LETTER KU +FF79;FF79;FF79;30B1;30B1; # (ケ; ケ; ケ; ケ; ケ; ) HALFWIDTH KATAKANA LETTER KE +FF7A;FF7A;FF7A;30B3;30B3; # (コ; コ; コ; コ; コ; ) HALFWIDTH KATAKANA LETTER KO +FF7B;FF7B;FF7B;30B5;30B5; # (サ; サ; サ; サ; サ; ) HALFWIDTH KATAKANA LETTER SA +FF7C;FF7C;FF7C;30B7;30B7; # (シ; シ; シ; シ; シ; ) HALFWIDTH KATAKANA LETTER SI +FF7D;FF7D;FF7D;30B9;30B9; # (ス; ス; ス; ス; ス; ) HALFWIDTH KATAKANA LETTER SU +FF7E;FF7E;FF7E;30BB;30BB; # (セ; セ; セ; セ; セ; ) HALFWIDTH KATAKANA LETTER SE +FF7F;FF7F;FF7F;30BD;30BD; # (ソ; ソ; ソ; ソ; ソ; ) HALFWIDTH KATAKANA LETTER SO +FF80;FF80;FF80;30BF;30BF; # (タ; タ; タ; タ; タ; ) HALFWIDTH KATAKANA LETTER TA +FF81;FF81;FF81;30C1;30C1; # (チ; チ; チ; チ; チ; ) HALFWIDTH KATAKANA LETTER TI +FF82;FF82;FF82;30C4;30C4; # (ツ; ツ; ツ; ツ; ツ; ) HALFWIDTH KATAKANA LETTER TU +FF83;FF83;FF83;30C6;30C6; # (テ; テ; テ; テ; テ; ) HALFWIDTH KATAKANA LETTER TE +FF84;FF84;FF84;30C8;30C8; # (ト; ト; ト; ト; ト; ) HALFWIDTH KATAKANA LETTER TO +FF85;FF85;FF85;30CA;30CA; # (ナ; ナ; ナ; ナ; ナ; ) HALFWIDTH KATAKANA LETTER NA +FF86;FF86;FF86;30CB;30CB; # (ニ; ニ; ニ; ニ; ニ; ) HALFWIDTH KATAKANA LETTER NI +FF87;FF87;FF87;30CC;30CC; # (ヌ; ヌ; ヌ; ヌ; ヌ; ) HALFWIDTH KATAKANA LETTER NU +FF88;FF88;FF88;30CD;30CD; # (ネ; ネ; ネ; ネ; ネ; ) HALFWIDTH KATAKANA LETTER NE +FF89;FF89;FF89;30CE;30CE; # (ノ; ノ; ノ; ノ; ノ; ) HALFWIDTH KATAKANA LETTER NO +FF8A;FF8A;FF8A;30CF;30CF; # (ハ; ハ; ハ; ハ; ハ; ) HALFWIDTH KATAKANA LETTER HA +FF8B;FF8B;FF8B;30D2;30D2; # (ヒ; ヒ; ヒ; ヒ; ヒ; ) HALFWIDTH KATAKANA LETTER HI +FF8C;FF8C;FF8C;30D5;30D5; # (フ; フ; フ; フ; フ; ) HALFWIDTH KATAKANA LETTER HU +FF8D;FF8D;FF8D;30D8;30D8; # (ヘ; ヘ; ヘ; ヘ; ヘ; ) HALFWIDTH KATAKANA LETTER HE +FF8E;FF8E;FF8E;30DB;30DB; # (ホ; ホ; ホ; ホ; ホ; ) HALFWIDTH KATAKANA LETTER HO +FF8F;FF8F;FF8F;30DE;30DE; # (マ; マ; マ; マ; マ; ) HALFWIDTH KATAKANA LETTER MA +FF90;FF90;FF90;30DF;30DF; # (ミ; ミ; ミ; ミ; ミ; ) HALFWIDTH KATAKANA LETTER MI +FF91;FF91;FF91;30E0;30E0; # (ム; ム; ム; ム; ム; ) HALFWIDTH KATAKANA LETTER MU +FF92;FF92;FF92;30E1;30E1; # (メ; メ; メ; メ; メ; ) HALFWIDTH KATAKANA LETTER ME +FF93;FF93;FF93;30E2;30E2; # (モ; モ; モ; モ; モ; ) HALFWIDTH KATAKANA LETTER MO +FF94;FF94;FF94;30E4;30E4; # (ヤ; ヤ; ヤ; ヤ; ヤ; ) HALFWIDTH KATAKANA LETTER YA +FF95;FF95;FF95;30E6;30E6; # (ユ; ユ; ユ; ユ; ユ; ) HALFWIDTH KATAKANA LETTER YU +FF96;FF96;FF96;30E8;30E8; # (ヨ; ヨ; ヨ; ヨ; ヨ; ) HALFWIDTH KATAKANA LETTER YO +FF97;FF97;FF97;30E9;30E9; # (ラ; ラ; ラ; ラ; ラ; ) HALFWIDTH KATAKANA LETTER RA +FF98;FF98;FF98;30EA;30EA; # (リ; リ; リ; リ; リ; ) HALFWIDTH KATAKANA LETTER RI +FF99;FF99;FF99;30EB;30EB; # (ル; ル; ル; ル; ル; ) HALFWIDTH KATAKANA LETTER RU +FF9A;FF9A;FF9A;30EC;30EC; # (レ; レ; レ; レ; レ; ) HALFWIDTH KATAKANA LETTER RE +FF9B;FF9B;FF9B;30ED;30ED; # (ロ; ロ; ロ; ロ; ロ; ) HALFWIDTH KATAKANA LETTER RO +FF9C;FF9C;FF9C;30EF;30EF; # (ワ; ワ; ワ; ワ; ワ; ) HALFWIDTH KATAKANA LETTER WA +FF9D;FF9D;FF9D;30F3;30F3; # (ン; ン; ン; ン; ン; ) HALFWIDTH KATAKANA LETTER N +FF9E;FF9E;FF9E;3099;3099; # (゙; ゙; ゙; ◌゙; ◌゙; ) HALFWIDTH KATAKANA VOICED SOUND MARK +FF9F;FF9F;FF9F;309A;309A; # (゚; ゚; ゚; ◌゚; ◌゚; ) HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK +FFA0;FFA0;FFA0;1160;1160; # (ᅠ; ᅠ; ᅠ; ᅠ; ᅠ; ) HALFWIDTH HANGUL FILLER +FFA1;FFA1;FFA1;1100;1100; # (ᄀ; ᄀ; ᄀ; ᄀ; ᄀ; ) HALFWIDTH HANGUL LETTER KIYEOK +FFA2;FFA2;FFA2;1101;1101; # (ᄁ; ᄁ; ᄁ; ᄁ; ᄁ; ) HALFWIDTH HANGUL LETTER SSANGKIYEOK +FFA3;FFA3;FFA3;11AA;11AA; # (ᆪ; ᆪ; ᆪ; ᆪ; ᆪ; ) HALFWIDTH HANGUL LETTER KIYEOK-SIOS +FFA4;FFA4;FFA4;1102;1102; # (ᄂ; ᄂ; ᄂ; ᄂ; ᄂ; ) HALFWIDTH HANGUL LETTER NIEUN +FFA5;FFA5;FFA5;11AC;11AC; # (ᆬ; ᆬ; ᆬ; ᆬ; ᆬ; ) HALFWIDTH HANGUL LETTER NIEUN-CIEUC +FFA6;FFA6;FFA6;11AD;11AD; # (ᆭ; ᆭ; ᆭ; ᆭ; ᆭ; ) HALFWIDTH HANGUL LETTER NIEUN-HIEUH +FFA7;FFA7;FFA7;1103;1103; # (ᄃ; ᄃ; ᄃ; ᄃ; ᄃ; ) HALFWIDTH HANGUL LETTER TIKEUT +FFA8;FFA8;FFA8;1104;1104; # (ᄄ; ᄄ; ᄄ; ᄄ; ᄄ; ) HALFWIDTH HANGUL LETTER SSANGTIKEUT +FFA9;FFA9;FFA9;1105;1105; # (ᄅ; ᄅ; ᄅ; ᄅ; ᄅ; ) HALFWIDTH HANGUL LETTER RIEUL +FFAA;FFAA;FFAA;11B0;11B0; # (ᆰ; ᆰ; ᆰ; ᆰ; ᆰ; ) HALFWIDTH HANGUL LETTER RIEUL-KIYEOK +FFAB;FFAB;FFAB;11B1;11B1; # (ᆱ; ᆱ; ᆱ; ᆱ; ᆱ; ) HALFWIDTH HANGUL LETTER RIEUL-MIEUM +FFAC;FFAC;FFAC;11B2;11B2; # (ᆲ; ᆲ; ᆲ; ᆲ; ᆲ; ) HALFWIDTH HANGUL LETTER RIEUL-PIEUP +FFAD;FFAD;FFAD;11B3;11B3; # (ᆳ; ᆳ; ᆳ; ᆳ; ᆳ; ) HALFWIDTH HANGUL LETTER RIEUL-SIOS +FFAE;FFAE;FFAE;11B4;11B4; # (ᆴ; ᆴ; ᆴ; ᆴ; ᆴ; ) HALFWIDTH HANGUL LETTER RIEUL-THIEUTH +FFAF;FFAF;FFAF;11B5;11B5; # (ᆵ; ᆵ; ᆵ; ᆵ; ᆵ; ) HALFWIDTH HANGUL LETTER RIEUL-PHIEUPH +FFB0;FFB0;FFB0;111A;111A; # (ᄚ; ᄚ; ᄚ; ᄚ; ᄚ; ) HALFWIDTH HANGUL LETTER RIEUL-HIEUH +FFB1;FFB1;FFB1;1106;1106; # (ᄆ; ᄆ; ᄆ; ᄆ; ᄆ; ) HALFWIDTH HANGUL LETTER MIEUM +FFB2;FFB2;FFB2;1107;1107; # (ᄇ; ᄇ; ᄇ; ᄇ; ᄇ; ) HALFWIDTH HANGUL LETTER PIEUP +FFB3;FFB3;FFB3;1108;1108; # (ᄈ; ᄈ; ᄈ; ᄈ; ᄈ; ) HALFWIDTH HANGUL LETTER SSANGPIEUP +FFB4;FFB4;FFB4;1121;1121; # (ᄡ; ᄡ; ᄡ; ᄡ; ᄡ; ) HALFWIDTH HANGUL LETTER PIEUP-SIOS +FFB5;FFB5;FFB5;1109;1109; # (ᄉ; ᄉ; ᄉ; ᄉ; ᄉ; ) HALFWIDTH HANGUL LETTER SIOS +FFB6;FFB6;FFB6;110A;110A; # (ᄊ; ᄊ; ᄊ; ᄊ; ᄊ; ) HALFWIDTH HANGUL LETTER SSANGSIOS +FFB7;FFB7;FFB7;110B;110B; # (ᄋ; ᄋ; ᄋ; ᄋ; ᄋ; ) HALFWIDTH HANGUL LETTER IEUNG +FFB8;FFB8;FFB8;110C;110C; # (ᄌ; ᄌ; ᄌ; ᄌ; ᄌ; ) HALFWIDTH HANGUL LETTER CIEUC +FFB9;FFB9;FFB9;110D;110D; # (ᄍ; ᄍ; ᄍ; ᄍ; ᄍ; ) HALFWIDTH HANGUL LETTER SSANGCIEUC +FFBA;FFBA;FFBA;110E;110E; # (ᄎ; ᄎ; ᄎ; ᄎ; ᄎ; ) HALFWIDTH HANGUL LETTER CHIEUCH +FFBB;FFBB;FFBB;110F;110F; # (ᄏ; ᄏ; ᄏ; ᄏ; ᄏ; ) HALFWIDTH HANGUL LETTER KHIEUKH +FFBC;FFBC;FFBC;1110;1110; # (ᄐ; ᄐ; ᄐ; ᄐ; ᄐ; ) HALFWIDTH HANGUL LETTER THIEUTH +FFBD;FFBD;FFBD;1111;1111; # (ᄑ; ᄑ; ᄑ; ᄑ; ᄑ; ) HALFWIDTH HANGUL LETTER PHIEUPH +FFBE;FFBE;FFBE;1112;1112; # (ᄒ; ᄒ; ᄒ; ᄒ; ᄒ; ) HALFWIDTH HANGUL LETTER HIEUH +FFC2;FFC2;FFC2;1161;1161; # (ᅡ; ᅡ; ᅡ; ᅡ; ᅡ; ) HALFWIDTH HANGUL LETTER A +FFC3;FFC3;FFC3;1162;1162; # (ᅢ; ᅢ; ᅢ; ᅢ; ᅢ; ) HALFWIDTH HANGUL LETTER AE +FFC4;FFC4;FFC4;1163;1163; # (ᅣ; ᅣ; ᅣ; ᅣ; ᅣ; ) HALFWIDTH HANGUL LETTER YA +FFC5;FFC5;FFC5;1164;1164; # (ᅤ; ᅤ; ᅤ; ᅤ; ᅤ; ) HALFWIDTH HANGUL LETTER YAE +FFC6;FFC6;FFC6;1165;1165; # (ᅥ; ᅥ; ᅥ; ᅥ; ᅥ; ) HALFWIDTH HANGUL LETTER EO +FFC7;FFC7;FFC7;1166;1166; # (ᅦ; ᅦ; ᅦ; ᅦ; ᅦ; ) HALFWIDTH HANGUL LETTER E +FFCA;FFCA;FFCA;1167;1167; # (ᅧ; ᅧ; ᅧ; ᅧ; ᅧ; ) HALFWIDTH HANGUL LETTER YEO +FFCB;FFCB;FFCB;1168;1168; # (ᅨ; ᅨ; ᅨ; ᅨ; ᅨ; ) HALFWIDTH HANGUL LETTER YE +FFCC;FFCC;FFCC;1169;1169; # (ᅩ; ᅩ; ᅩ; ᅩ; ᅩ; ) HALFWIDTH HANGUL LETTER O +FFCD;FFCD;FFCD;116A;116A; # (ᅪ; ᅪ; ᅪ; ᅪ; ᅪ; ) HALFWIDTH HANGUL LETTER WA +FFCE;FFCE;FFCE;116B;116B; # (ᅫ; ᅫ; ᅫ; ᅫ; ᅫ; ) HALFWIDTH HANGUL LETTER WAE +FFCF;FFCF;FFCF;116C;116C; # (ᅬ; ᅬ; ᅬ; ᅬ; ᅬ; ) HALFWIDTH HANGUL LETTER OE +FFD2;FFD2;FFD2;116D;116D; # (ᅭ; ᅭ; ᅭ; ᅭ; ᅭ; ) HALFWIDTH HANGUL LETTER YO +FFD3;FFD3;FFD3;116E;116E; # (ᅮ; ᅮ; ᅮ; ᅮ; ᅮ; ) HALFWIDTH HANGUL LETTER U +FFD4;FFD4;FFD4;116F;116F; # (ᅯ; ᅯ; ᅯ; ᅯ; ᅯ; ) HALFWIDTH HANGUL LETTER WEO +FFD5;FFD5;FFD5;1170;1170; # (ᅰ; ᅰ; ᅰ; ᅰ; ᅰ; ) HALFWIDTH HANGUL LETTER WE +FFD6;FFD6;FFD6;1171;1171; # (ᅱ; ᅱ; ᅱ; ᅱ; ᅱ; ) HALFWIDTH HANGUL LETTER WI +FFD7;FFD7;FFD7;1172;1172; # (ᅲ; ᅲ; ᅲ; ᅲ; ᅲ; ) HALFWIDTH HANGUL LETTER YU +FFDA;FFDA;FFDA;1173;1173; # (ᅳ; ᅳ; ᅳ; ᅳ; ᅳ; ) HALFWIDTH HANGUL LETTER EU +FFDB;FFDB;FFDB;1174;1174; # (ᅴ; ᅴ; ᅴ; ᅴ; ᅴ; ) HALFWIDTH HANGUL LETTER YI +FFDC;FFDC;FFDC;1175;1175; # (ᅵ; ᅵ; ᅵ; ᅵ; ᅵ; ) HALFWIDTH HANGUL LETTER I +FFE0;FFE0;FFE0;00A2;00A2; # (¢; ¢; ¢; ¢; ¢; ) FULLWIDTH CENT SIGN +FFE1;FFE1;FFE1;00A3;00A3; # (£; £; £; £; £; ) FULLWIDTH POUND SIGN +FFE2;FFE2;FFE2;00AC;00AC; # (¬; ¬; ¬; ¬; ¬; ) FULLWIDTH NOT SIGN +FFE3;FFE3;FFE3;0020 0304;0020 0304; # ( ̄;  ̄;  ̄; ◌̄; ◌̄; ) FULLWIDTH MACRON +FFE4;FFE4;FFE4;00A6;00A6; # (¦; ¦; ¦; ¦; ¦; ) FULLWIDTH BROKEN BAR +FFE5;FFE5;FFE5;00A5;00A5; # (¥; ¥; ¥; ¥; ¥; ) FULLWIDTH YEN SIGN +FFE6;FFE6;FFE6;20A9;20A9; # (₩; ₩; ₩; ₩; ₩; ) FULLWIDTH WON SIGN +FFE8;FFE8;FFE8;2502;2502; # (│; │; │; │; │; ) HALFWIDTH FORMS LIGHT VERTICAL +FFE9;FFE9;FFE9;2190;2190; # (←; ←; ←; ←; ←; ) HALFWIDTH LEFTWARDS ARROW +FFEA;FFEA;FFEA;2191;2191; # (↑; ↑; ↑; ↑; ↑; ) HALFWIDTH UPWARDS ARROW +FFEB;FFEB;FFEB;2192;2192; # (→; →; →; →; →; ) HALFWIDTH RIGHTWARDS ARROW +FFEC;FFEC;FFEC;2193;2193; # (↓; ↓; ↓; ↓; ↓; ) HALFWIDTH DOWNWARDS ARROW +FFED;FFED;FFED;25A0;25A0; # (■; ■; ■; ■; ■; ) HALFWIDTH BLACK SQUARE +FFEE;FFEE;FFEE;25CB;25CB; # (○; ○; ○; ○; ○; ) HALFWIDTH WHITE CIRCLE +1D15E;1D157 1D165;1D157 1D165;1D157 1D165;1D157 1D165; # (𝅗𝅥𝅗𝅥; 𝅗𝅗𝅥𝅥; 𝅗𝅗𝅥𝅥; 𝅗𝅗𝅥𝅥; 𝅗𝅗𝅥𝅥; ) MUSICAL SYMBOL HALF NOTE +1D15F;1D158 1D165;1D158 1D165;1D158 1D165;1D158 1D165; # (𝅘𝅥𝅘𝅥; 𝅘𝅘𝅥𝅥; 𝅘𝅘𝅥𝅥; 𝅘𝅘𝅥𝅥; 𝅘𝅘𝅥𝅥; ) MUSICAL SYMBOL QUARTER NOTE +1D160;1D158 1D165 1D16E;1D158 1D165 1D16E;1D158 1D165 1D16E;1D158 1D165 1D16E; # (𝅘𝅥𝅮𝅘𝅥𝅮; 𝅘𝅘𝅥𝅥𝅮𝅮; 𝅘𝅘𝅥𝅥𝅮𝅮; 𝅘𝅘𝅥𝅥𝅮𝅮; 𝅘𝅘𝅥𝅥𝅮𝅮; ) MUSICAL SYMBOL EIGHTH NOTE +1D161;1D158 1D165 1D16F;1D158 1D165 1D16F;1D158 1D165 1D16F;1D158 1D165 1D16F; # (𝅘𝅥𝅯𝅘𝅥𝅯; 𝅘𝅘𝅥𝅥𝅯𝅯; 𝅘𝅘𝅥𝅥𝅯𝅯; 𝅘𝅘𝅥𝅥𝅯𝅯; 𝅘𝅘𝅥𝅥𝅯𝅯; ) MUSICAL SYMBOL SIXTEENTH NOTE +1D162;1D158 1D165 1D170;1D158 1D165 1D170;1D158 1D165 1D170;1D158 1D165 1D170; # (𝅘𝅥𝅰𝅘𝅥𝅰; 𝅘𝅘𝅥𝅥𝅰𝅰; 𝅘𝅘𝅥𝅥𝅰𝅰; 𝅘𝅘𝅥𝅥𝅰𝅰; 𝅘𝅘𝅥𝅥𝅰𝅰; ) MUSICAL SYMBOL THIRTY-SECOND NOTE +1D163;1D158 1D165 1D171;1D158 1D165 1D171;1D158 1D165 1D171;1D158 1D165 1D171; # (𝅘𝅥𝅱𝅘𝅥𝅱; 𝅘𝅘𝅥𝅥𝅱𝅱; 𝅘𝅘𝅥𝅥𝅱𝅱; 𝅘𝅘𝅥𝅥𝅱𝅱; 𝅘𝅘𝅥𝅥𝅱𝅱; ) MUSICAL SYMBOL SIXTY-FOURTH NOTE +1D164;1D158 1D165 1D172;1D158 1D165 1D172;1D158 1D165 1D172;1D158 1D165 1D172; # (𝅘𝅥𝅲𝅘𝅥𝅲; 𝅘𝅘𝅥𝅥𝅲𝅲; 𝅘𝅘𝅥𝅥𝅲𝅲; 𝅘𝅘𝅥𝅥𝅲𝅲; 𝅘𝅘𝅥𝅥𝅲𝅲; ) MUSICAL SYMBOL ONE HUNDRED TWENTY-EIGHTH NOTE +1D1BB;1D1B9 1D165;1D1B9 1D165;1D1B9 1D165;1D1B9 1D165; # (𝆹𝅥𝆹𝅥; 𝆹𝆹𝅥𝅥; 𝆹𝆹𝅥𝅥; 𝆹𝆹𝅥𝅥; 𝆹𝆹𝅥𝅥; ) MUSICAL SYMBOL MINIMA +1D1BC;1D1BA 1D165;1D1BA 1D165;1D1BA 1D165;1D1BA 1D165; # (𝆺𝅥𝆺𝅥; 𝆺𝆺𝅥𝅥; 𝆺𝆺𝅥𝅥; 𝆺𝆺𝅥𝅥; 𝆺𝆺𝅥𝅥; ) MUSICAL SYMBOL MINIMA BLACK +1D1BD;1D1B9 1D165 1D16E;1D1B9 1D165 1D16E;1D1B9 1D165 1D16E;1D1B9 1D165 1D16E; # (𝆹𝅥𝅮𝆹𝅥𝅮; 𝆹𝆹𝅥𝅥𝅮𝅮; 𝆹𝆹𝅥𝅥𝅮𝅮; 𝆹𝆹𝅥𝅥𝅮𝅮; 𝆹𝆹𝅥𝅥𝅮𝅮; ) MUSICAL SYMBOL SEMIMINIMA WHITE +1D1BE;1D1BA 1D165 1D16E;1D1BA 1D165 1D16E;1D1BA 1D165 1D16E;1D1BA 1D165 1D16E; # (𝆺𝅥𝅮𝆺𝅥𝅮; 𝆺𝆺𝅥𝅥𝅮𝅮; 𝆺𝆺𝅥𝅥𝅮𝅮; 𝆺𝆺𝅥𝅥𝅮𝅮; 𝆺𝆺𝅥𝅥𝅮𝅮; ) MUSICAL SYMBOL SEMIMINIMA BLACK +1D1BF;1D1B9 1D165 1D16F;1D1B9 1D165 1D16F;1D1B9 1D165 1D16F;1D1B9 1D165 1D16F; # (𝆹𝅥𝅯𝆹𝅥𝅯; 𝆹𝆹𝅥𝅥𝅯𝅯; 𝆹𝆹𝅥𝅥𝅯𝅯; 𝆹𝆹𝅥𝅥𝅯𝅯; 𝆹𝆹𝅥𝅥𝅯𝅯; ) MUSICAL SYMBOL FUSA WHITE +1D1C0;1D1BA 1D165 1D16F;1D1BA 1D165 1D16F;1D1BA 1D165 1D16F;1D1BA 1D165 1D16F; # (𝆺𝅥𝅯𝆺𝅥𝅯; 𝆺𝆺𝅥𝅥𝅯𝅯; 𝆺𝆺𝅥𝅥𝅯𝅯; 𝆺𝆺𝅥𝅥𝅯𝅯; 𝆺𝆺𝅥𝅥𝅯𝅯; ) MUSICAL SYMBOL FUSA BLACK +1D400;1D400;1D400;0041;0041; # (𝐀𝐀; 𝐀𝐀; 𝐀𝐀; A; A; ) MATHEMATICAL BOLD CAPITAL A +1D401;1D401;1D401;0042;0042; # (𝐁𝐁; 𝐁𝐁; 𝐁𝐁; B; B; ) MATHEMATICAL BOLD CAPITAL B +1D402;1D402;1D402;0043;0043; # (𝐂𝐂; 𝐂𝐂; 𝐂𝐂; C; C; ) MATHEMATICAL BOLD CAPITAL C +1D403;1D403;1D403;0044;0044; # (𝐃𝐃; 𝐃𝐃; 𝐃𝐃; D; D; ) MATHEMATICAL BOLD CAPITAL D +1D404;1D404;1D404;0045;0045; # (𝐄𝐄; 𝐄𝐄; 𝐄𝐄; E; E; ) MATHEMATICAL BOLD CAPITAL E +1D405;1D405;1D405;0046;0046; # (𝐅𝐅; 𝐅𝐅; 𝐅𝐅; F; F; ) MATHEMATICAL BOLD CAPITAL F +1D406;1D406;1D406;0047;0047; # (𝐆𝐆; 𝐆𝐆; 𝐆𝐆; G; G; ) MATHEMATICAL BOLD CAPITAL G +1D407;1D407;1D407;0048;0048; # (𝐇𝐇; 𝐇𝐇; 𝐇𝐇; H; H; ) MATHEMATICAL BOLD CAPITAL H +1D408;1D408;1D408;0049;0049; # (𝐈𝐈; 𝐈𝐈; 𝐈𝐈; I; I; ) MATHEMATICAL BOLD CAPITAL I +1D409;1D409;1D409;004A;004A; # (𝐉𝐉; 𝐉𝐉; 𝐉𝐉; J; J; ) MATHEMATICAL BOLD CAPITAL J +1D40A;1D40A;1D40A;004B;004B; # (𝐊𝐊; 𝐊𝐊; 𝐊𝐊; K; K; ) MATHEMATICAL BOLD CAPITAL K +1D40B;1D40B;1D40B;004C;004C; # (𝐋𝐋; 𝐋𝐋; 𝐋𝐋; L; L; ) MATHEMATICAL BOLD CAPITAL L +1D40C;1D40C;1D40C;004D;004D; # (𝐌𝐌; 𝐌𝐌; 𝐌𝐌; M; M; ) MATHEMATICAL BOLD CAPITAL M +1D40D;1D40D;1D40D;004E;004E; # (𝐍𝐍; 𝐍𝐍; 𝐍𝐍; N; N; ) MATHEMATICAL BOLD CAPITAL N +1D40E;1D40E;1D40E;004F;004F; # (𝐎𝐎; 𝐎𝐎; 𝐎𝐎; O; O; ) MATHEMATICAL BOLD CAPITAL O +1D40F;1D40F;1D40F;0050;0050; # (𝐏𝐏; 𝐏𝐏; 𝐏𝐏; P; P; ) MATHEMATICAL BOLD CAPITAL P +1D410;1D410;1D410;0051;0051; # (𝐐𝐐; 𝐐𝐐; 𝐐𝐐; Q; Q; ) MATHEMATICAL BOLD CAPITAL Q +1D411;1D411;1D411;0052;0052; # (𝐑𝐑; 𝐑𝐑; 𝐑𝐑; R; R; ) MATHEMATICAL BOLD CAPITAL R +1D412;1D412;1D412;0053;0053; # (𝐒𝐒; 𝐒𝐒; 𝐒𝐒; S; S; ) MATHEMATICAL BOLD CAPITAL S +1D413;1D413;1D413;0054;0054; # (𝐓𝐓; 𝐓𝐓; 𝐓𝐓; T; T; ) MATHEMATICAL BOLD CAPITAL T +1D414;1D414;1D414;0055;0055; # (𝐔𝐔; 𝐔𝐔; 𝐔𝐔; U; U; ) MATHEMATICAL BOLD CAPITAL U +1D415;1D415;1D415;0056;0056; # (𝐕𝐕; 𝐕𝐕; 𝐕𝐕; V; V; ) MATHEMATICAL BOLD CAPITAL V +1D416;1D416;1D416;0057;0057; # (𝐖𝐖; 𝐖𝐖; 𝐖𝐖; W; W; ) MATHEMATICAL BOLD CAPITAL W +1D417;1D417;1D417;0058;0058; # (𝐗𝐗; 𝐗𝐗; 𝐗𝐗; X; X; ) MATHEMATICAL BOLD CAPITAL X +1D418;1D418;1D418;0059;0059; # (𝐘𝐘; 𝐘𝐘; 𝐘𝐘; Y; Y; ) MATHEMATICAL BOLD CAPITAL Y +1D419;1D419;1D419;005A;005A; # (𝐙𝐙; 𝐙𝐙; 𝐙𝐙; Z; Z; ) MATHEMATICAL BOLD CAPITAL Z +1D41A;1D41A;1D41A;0061;0061; # (𝐚𝐚; 𝐚𝐚; 𝐚𝐚; a; a; ) MATHEMATICAL BOLD SMALL A +1D41B;1D41B;1D41B;0062;0062; # (𝐛𝐛; 𝐛𝐛; 𝐛𝐛; b; b; ) MATHEMATICAL BOLD SMALL B +1D41C;1D41C;1D41C;0063;0063; # (𝐜𝐜; 𝐜𝐜; 𝐜𝐜; c; c; ) MATHEMATICAL BOLD SMALL C +1D41D;1D41D;1D41D;0064;0064; # (𝐝𝐝; 𝐝𝐝; 𝐝𝐝; d; d; ) MATHEMATICAL BOLD SMALL D +1D41E;1D41E;1D41E;0065;0065; # (𝐞𝐞; 𝐞𝐞; 𝐞𝐞; e; e; ) MATHEMATICAL BOLD SMALL E +1D41F;1D41F;1D41F;0066;0066; # (𝐟𝐟; 𝐟𝐟; 𝐟𝐟; f; f; ) MATHEMATICAL BOLD SMALL F +1D420;1D420;1D420;0067;0067; # (𝐠𝐠; 𝐠𝐠; 𝐠𝐠; g; g; ) MATHEMATICAL BOLD SMALL G +1D421;1D421;1D421;0068;0068; # (𝐡𝐡; 𝐡𝐡; 𝐡𝐡; h; h; ) MATHEMATICAL BOLD SMALL H +1D422;1D422;1D422;0069;0069; # (𝐢𝐢; 𝐢𝐢; 𝐢𝐢; i; i; ) MATHEMATICAL BOLD SMALL I +1D423;1D423;1D423;006A;006A; # (𝐣𝐣; 𝐣𝐣; 𝐣𝐣; j; j; ) MATHEMATICAL BOLD SMALL J +1D424;1D424;1D424;006B;006B; # (𝐤𝐤; 𝐤𝐤; 𝐤𝐤; k; k; ) MATHEMATICAL BOLD SMALL K +1D425;1D425;1D425;006C;006C; # (𝐥𝐥; 𝐥𝐥; 𝐥𝐥; l; l; ) MATHEMATICAL BOLD SMALL L +1D426;1D426;1D426;006D;006D; # (𝐦𝐦; 𝐦𝐦; 𝐦𝐦; m; m; ) MATHEMATICAL BOLD SMALL M +1D427;1D427;1D427;006E;006E; # (𝐧𝐧; 𝐧𝐧; 𝐧𝐧; n; n; ) MATHEMATICAL BOLD SMALL N +1D428;1D428;1D428;006F;006F; # (𝐨𝐨; 𝐨𝐨; 𝐨𝐨; o; o; ) MATHEMATICAL BOLD SMALL O +1D429;1D429;1D429;0070;0070; # (𝐩𝐩; 𝐩𝐩; 𝐩𝐩; p; p; ) MATHEMATICAL BOLD SMALL P +1D42A;1D42A;1D42A;0071;0071; # (𝐪𝐪; 𝐪𝐪; 𝐪𝐪; q; q; ) MATHEMATICAL BOLD SMALL Q +1D42B;1D42B;1D42B;0072;0072; # (𝐫𝐫; 𝐫𝐫; 𝐫𝐫; r; r; ) MATHEMATICAL BOLD SMALL R +1D42C;1D42C;1D42C;0073;0073; # (𝐬𝐬; 𝐬𝐬; 𝐬𝐬; s; s; ) MATHEMATICAL BOLD SMALL S +1D42D;1D42D;1D42D;0074;0074; # (𝐭𝐭; 𝐭𝐭; 𝐭𝐭; t; t; ) MATHEMATICAL BOLD SMALL T +1D42E;1D42E;1D42E;0075;0075; # (𝐮𝐮; 𝐮𝐮; 𝐮𝐮; u; u; ) MATHEMATICAL BOLD SMALL U +1D42F;1D42F;1D42F;0076;0076; # (𝐯𝐯; 𝐯𝐯; 𝐯𝐯; v; v; ) MATHEMATICAL BOLD SMALL V +1D430;1D430;1D430;0077;0077; # (𝐰𝐰; 𝐰𝐰; 𝐰𝐰; w; w; ) MATHEMATICAL BOLD SMALL W +1D431;1D431;1D431;0078;0078; # (𝐱𝐱; 𝐱𝐱; 𝐱𝐱; x; x; ) MATHEMATICAL BOLD SMALL X +1D432;1D432;1D432;0079;0079; # (𝐲𝐲; 𝐲𝐲; 𝐲𝐲; y; y; ) MATHEMATICAL BOLD SMALL Y +1D433;1D433;1D433;007A;007A; # (𝐳𝐳; 𝐳𝐳; 𝐳𝐳; z; z; ) MATHEMATICAL BOLD SMALL Z +1D434;1D434;1D434;0041;0041; # (𝐴𝐴; 𝐴𝐴; 𝐴𝐴; A; A; ) MATHEMATICAL ITALIC CAPITAL A +1D435;1D435;1D435;0042;0042; # (𝐵𝐵; 𝐵𝐵; 𝐵𝐵; B; B; ) MATHEMATICAL ITALIC CAPITAL B +1D436;1D436;1D436;0043;0043; # (𝐶𝐶; 𝐶𝐶; 𝐶𝐶; C; C; ) MATHEMATICAL ITALIC CAPITAL C +1D437;1D437;1D437;0044;0044; # (𝐷𝐷; 𝐷𝐷; 𝐷𝐷; D; D; ) MATHEMATICAL ITALIC CAPITAL D +1D438;1D438;1D438;0045;0045; # (𝐸𝐸; 𝐸𝐸; 𝐸𝐸; E; E; ) MATHEMATICAL ITALIC CAPITAL E +1D439;1D439;1D439;0046;0046; # (𝐹𝐹; 𝐹𝐹; 𝐹𝐹; F; F; ) MATHEMATICAL ITALIC CAPITAL F +1D43A;1D43A;1D43A;0047;0047; # (𝐺𝐺; 𝐺𝐺; 𝐺𝐺; G; G; ) MATHEMATICAL ITALIC CAPITAL G +1D43B;1D43B;1D43B;0048;0048; # (𝐻𝐻; 𝐻𝐻; 𝐻𝐻; H; H; ) MATHEMATICAL ITALIC CAPITAL H +1D43C;1D43C;1D43C;0049;0049; # (𝐼𝐼; 𝐼𝐼; 𝐼𝐼; I; I; ) MATHEMATICAL ITALIC CAPITAL I +1D43D;1D43D;1D43D;004A;004A; # (𝐽𝐽; 𝐽𝐽; 𝐽𝐽; J; J; ) MATHEMATICAL ITALIC CAPITAL J +1D43E;1D43E;1D43E;004B;004B; # (𝐾𝐾; 𝐾𝐾; 𝐾𝐾; K; K; ) MATHEMATICAL ITALIC CAPITAL K +1D43F;1D43F;1D43F;004C;004C; # (𝐿𝐿; 𝐿𝐿; 𝐿𝐿; L; L; ) MATHEMATICAL ITALIC CAPITAL L +1D440;1D440;1D440;004D;004D; # (𝑀𝑀; 𝑀𝑀; 𝑀𝑀; M; M; ) MATHEMATICAL ITALIC CAPITAL M +1D441;1D441;1D441;004E;004E; # (𝑁𝑁; 𝑁𝑁; 𝑁𝑁; N; N; ) MATHEMATICAL ITALIC CAPITAL N +1D442;1D442;1D442;004F;004F; # (𝑂𝑂; 𝑂𝑂; 𝑂𝑂; O; O; ) MATHEMATICAL ITALIC CAPITAL O +1D443;1D443;1D443;0050;0050; # (𝑃𝑃; 𝑃𝑃; 𝑃𝑃; P; P; ) MATHEMATICAL ITALIC CAPITAL P +1D444;1D444;1D444;0051;0051; # (𝑄𝑄; 𝑄𝑄; 𝑄𝑄; Q; Q; ) MATHEMATICAL ITALIC CAPITAL Q +1D445;1D445;1D445;0052;0052; # (𝑅𝑅; 𝑅𝑅; 𝑅𝑅; R; R; ) MATHEMATICAL ITALIC CAPITAL R +1D446;1D446;1D446;0053;0053; # (𝑆𝑆; 𝑆𝑆; 𝑆𝑆; S; S; ) MATHEMATICAL ITALIC CAPITAL S +1D447;1D447;1D447;0054;0054; # (𝑇𝑇; 𝑇𝑇; 𝑇𝑇; T; T; ) MATHEMATICAL ITALIC CAPITAL T +1D448;1D448;1D448;0055;0055; # (𝑈𝑈; 𝑈𝑈; 𝑈𝑈; U; U; ) MATHEMATICAL ITALIC CAPITAL U +1D449;1D449;1D449;0056;0056; # (𝑉𝑉; 𝑉𝑉; 𝑉𝑉; V; V; ) MATHEMATICAL ITALIC CAPITAL V +1D44A;1D44A;1D44A;0057;0057; # (𝑊𝑊; 𝑊𝑊; 𝑊𝑊; W; W; ) MATHEMATICAL ITALIC CAPITAL W +1D44B;1D44B;1D44B;0058;0058; # (𝑋𝑋; 𝑋𝑋; 𝑋𝑋; X; X; ) MATHEMATICAL ITALIC CAPITAL X +1D44C;1D44C;1D44C;0059;0059; # (𝑌𝑌; 𝑌𝑌; 𝑌𝑌; Y; Y; ) MATHEMATICAL ITALIC CAPITAL Y +1D44D;1D44D;1D44D;005A;005A; # (𝑍𝑍; 𝑍𝑍; 𝑍𝑍; Z; Z; ) MATHEMATICAL ITALIC CAPITAL Z +1D44E;1D44E;1D44E;0061;0061; # (𝑎𝑎; 𝑎𝑎; 𝑎𝑎; a; a; ) MATHEMATICAL ITALIC SMALL A +1D44F;1D44F;1D44F;0062;0062; # (𝑏𝑏; 𝑏𝑏; 𝑏𝑏; b; b; ) MATHEMATICAL ITALIC SMALL B +1D450;1D450;1D450;0063;0063; # (𝑐𝑐; 𝑐𝑐; 𝑐𝑐; c; c; ) MATHEMATICAL ITALIC SMALL C +1D451;1D451;1D451;0064;0064; # (𝑑𝑑; 𝑑𝑑; 𝑑𝑑; d; d; ) MATHEMATICAL ITALIC SMALL D +1D452;1D452;1D452;0065;0065; # (𝑒𝑒; 𝑒𝑒; 𝑒𝑒; e; e; ) MATHEMATICAL ITALIC SMALL E +1D453;1D453;1D453;0066;0066; # (𝑓𝑓; 𝑓𝑓; 𝑓𝑓; f; f; ) MATHEMATICAL ITALIC SMALL F +1D454;1D454;1D454;0067;0067; # (𝑔𝑔; 𝑔𝑔; 𝑔𝑔; g; g; ) MATHEMATICAL ITALIC SMALL G +1D456;1D456;1D456;0069;0069; # (𝑖𝑖; 𝑖𝑖; 𝑖𝑖; i; i; ) MATHEMATICAL ITALIC SMALL I +1D457;1D457;1D457;006A;006A; # (𝑗𝑗; 𝑗𝑗; 𝑗𝑗; j; j; ) MATHEMATICAL ITALIC SMALL J +1D458;1D458;1D458;006B;006B; # (𝑘𝑘; 𝑘𝑘; 𝑘𝑘; k; k; ) MATHEMATICAL ITALIC SMALL K +1D459;1D459;1D459;006C;006C; # (𝑙𝑙; 𝑙𝑙; 𝑙𝑙; l; l; ) MATHEMATICAL ITALIC SMALL L +1D45A;1D45A;1D45A;006D;006D; # (𝑚𝑚; 𝑚𝑚; 𝑚𝑚; m; m; ) MATHEMATICAL ITALIC SMALL M +1D45B;1D45B;1D45B;006E;006E; # (𝑛𝑛; 𝑛𝑛; 𝑛𝑛; n; n; ) MATHEMATICAL ITALIC SMALL N +1D45C;1D45C;1D45C;006F;006F; # (𝑜𝑜; 𝑜𝑜; 𝑜𝑜; o; o; ) MATHEMATICAL ITALIC SMALL O +1D45D;1D45D;1D45D;0070;0070; # (𝑝𝑝; 𝑝𝑝; 𝑝𝑝; p; p; ) MATHEMATICAL ITALIC SMALL P +1D45E;1D45E;1D45E;0071;0071; # (𝑞𝑞; 𝑞𝑞; 𝑞𝑞; q; q; ) MATHEMATICAL ITALIC SMALL Q +1D45F;1D45F;1D45F;0072;0072; # (𝑟𝑟; 𝑟𝑟; 𝑟𝑟; r; r; ) MATHEMATICAL ITALIC SMALL R +1D460;1D460;1D460;0073;0073; # (𝑠𝑠; 𝑠𝑠; 𝑠𝑠; s; s; ) MATHEMATICAL ITALIC SMALL S +1D461;1D461;1D461;0074;0074; # (𝑡𝑡; 𝑡𝑡; 𝑡𝑡; t; t; ) MATHEMATICAL ITALIC SMALL T +1D462;1D462;1D462;0075;0075; # (𝑢𝑢; 𝑢𝑢; 𝑢𝑢; u; u; ) MATHEMATICAL ITALIC SMALL U +1D463;1D463;1D463;0076;0076; # (𝑣𝑣; 𝑣𝑣; 𝑣𝑣; v; v; ) MATHEMATICAL ITALIC SMALL V +1D464;1D464;1D464;0077;0077; # (𝑤𝑤; 𝑤𝑤; 𝑤𝑤; w; w; ) MATHEMATICAL ITALIC SMALL W +1D465;1D465;1D465;0078;0078; # (𝑥𝑥; 𝑥𝑥; 𝑥𝑥; x; x; ) MATHEMATICAL ITALIC SMALL X +1D466;1D466;1D466;0079;0079; # (𝑦𝑦; 𝑦𝑦; 𝑦𝑦; y; y; ) MATHEMATICAL ITALIC SMALL Y +1D467;1D467;1D467;007A;007A; # (𝑧𝑧; 𝑧𝑧; 𝑧𝑧; z; z; ) MATHEMATICAL ITALIC SMALL Z +1D468;1D468;1D468;0041;0041; # (𝑨𝑨; 𝑨𝑨; 𝑨𝑨; A; A; ) MATHEMATICAL BOLD ITALIC CAPITAL A +1D469;1D469;1D469;0042;0042; # (𝑩𝑩; 𝑩𝑩; 𝑩𝑩; B; B; ) MATHEMATICAL BOLD ITALIC CAPITAL B +1D46A;1D46A;1D46A;0043;0043; # (𝑪𝑪; 𝑪𝑪; 𝑪𝑪; C; C; ) MATHEMATICAL BOLD ITALIC CAPITAL C +1D46B;1D46B;1D46B;0044;0044; # (𝑫𝑫; 𝑫𝑫; 𝑫𝑫; D; D; ) MATHEMATICAL BOLD ITALIC CAPITAL D +1D46C;1D46C;1D46C;0045;0045; # (𝑬𝑬; 𝑬𝑬; 𝑬𝑬; E; E; ) MATHEMATICAL BOLD ITALIC CAPITAL E +1D46D;1D46D;1D46D;0046;0046; # (𝑭𝑭; 𝑭𝑭; 𝑭𝑭; F; F; ) MATHEMATICAL BOLD ITALIC CAPITAL F +1D46E;1D46E;1D46E;0047;0047; # (𝑮𝑮; 𝑮𝑮; 𝑮𝑮; G; G; ) MATHEMATICAL BOLD ITALIC CAPITAL G +1D46F;1D46F;1D46F;0048;0048; # (𝑯𝑯; 𝑯𝑯; 𝑯𝑯; H; H; ) MATHEMATICAL BOLD ITALIC CAPITAL H +1D470;1D470;1D470;0049;0049; # (𝑰𝑰; 𝑰𝑰; 𝑰𝑰; I; I; ) MATHEMATICAL BOLD ITALIC CAPITAL I +1D471;1D471;1D471;004A;004A; # (𝑱𝑱; 𝑱𝑱; 𝑱𝑱; J; J; ) MATHEMATICAL BOLD ITALIC CAPITAL J +1D472;1D472;1D472;004B;004B; # (𝑲𝑲; 𝑲𝑲; 𝑲𝑲; K; K; ) MATHEMATICAL BOLD ITALIC CAPITAL K +1D473;1D473;1D473;004C;004C; # (𝑳𝑳; 𝑳𝑳; 𝑳𝑳; L; L; ) MATHEMATICAL BOLD ITALIC CAPITAL L +1D474;1D474;1D474;004D;004D; # (𝑴𝑴; 𝑴𝑴; 𝑴𝑴; M; M; ) MATHEMATICAL BOLD ITALIC CAPITAL M +1D475;1D475;1D475;004E;004E; # (𝑵𝑵; 𝑵𝑵; 𝑵𝑵; N; N; ) MATHEMATICAL BOLD ITALIC CAPITAL N +1D476;1D476;1D476;004F;004F; # (𝑶𝑶; 𝑶𝑶; 𝑶𝑶; O; O; ) MATHEMATICAL BOLD ITALIC CAPITAL O +1D477;1D477;1D477;0050;0050; # (𝑷𝑷; 𝑷𝑷; 𝑷𝑷; P; P; ) MATHEMATICAL BOLD ITALIC CAPITAL P +1D478;1D478;1D478;0051;0051; # (𝑸𝑸; 𝑸𝑸; 𝑸𝑸; Q; Q; ) MATHEMATICAL BOLD ITALIC CAPITAL Q +1D479;1D479;1D479;0052;0052; # (𝑹𝑹; 𝑹𝑹; 𝑹𝑹; R; R; ) MATHEMATICAL BOLD ITALIC CAPITAL R +1D47A;1D47A;1D47A;0053;0053; # (𝑺𝑺; 𝑺𝑺; 𝑺𝑺; S; S; ) MATHEMATICAL BOLD ITALIC CAPITAL S +1D47B;1D47B;1D47B;0054;0054; # (𝑻𝑻; 𝑻𝑻; 𝑻𝑻; T; T; ) MATHEMATICAL BOLD ITALIC CAPITAL T +1D47C;1D47C;1D47C;0055;0055; # (𝑼𝑼; 𝑼𝑼; 𝑼𝑼; U; U; ) MATHEMATICAL BOLD ITALIC CAPITAL U +1D47D;1D47D;1D47D;0056;0056; # (𝑽𝑽; 𝑽𝑽; 𝑽𝑽; V; V; ) MATHEMATICAL BOLD ITALIC CAPITAL V +1D47E;1D47E;1D47E;0057;0057; # (𝑾𝑾; 𝑾𝑾; 𝑾𝑾; W; W; ) MATHEMATICAL BOLD ITALIC CAPITAL W +1D47F;1D47F;1D47F;0058;0058; # (𝑿𝑿; 𝑿𝑿; 𝑿𝑿; X; X; ) MATHEMATICAL BOLD ITALIC CAPITAL X +1D480;1D480;1D480;0059;0059; # (𝒀𝒀; 𝒀𝒀; 𝒀𝒀; Y; Y; ) MATHEMATICAL BOLD ITALIC CAPITAL Y +1D481;1D481;1D481;005A;005A; # (𝒁𝒁; 𝒁𝒁; 𝒁𝒁; Z; Z; ) MATHEMATICAL BOLD ITALIC CAPITAL Z +1D482;1D482;1D482;0061;0061; # (𝒂𝒂; 𝒂𝒂; 𝒂𝒂; a; a; ) MATHEMATICAL BOLD ITALIC SMALL A +1D483;1D483;1D483;0062;0062; # (𝒃𝒃; 𝒃𝒃; 𝒃𝒃; b; b; ) MATHEMATICAL BOLD ITALIC SMALL B +1D484;1D484;1D484;0063;0063; # (𝒄𝒄; 𝒄𝒄; 𝒄𝒄; c; c; ) MATHEMATICAL BOLD ITALIC SMALL C +1D485;1D485;1D485;0064;0064; # (𝒅𝒅; 𝒅𝒅; 𝒅𝒅; d; d; ) MATHEMATICAL BOLD ITALIC SMALL D +1D486;1D486;1D486;0065;0065; # (𝒆𝒆; 𝒆𝒆; 𝒆𝒆; e; e; ) MATHEMATICAL BOLD ITALIC SMALL E +1D487;1D487;1D487;0066;0066; # (𝒇𝒇; 𝒇𝒇; 𝒇𝒇; f; f; ) MATHEMATICAL BOLD ITALIC SMALL F +1D488;1D488;1D488;0067;0067; # (𝒈𝒈; 𝒈𝒈; 𝒈𝒈; g; g; ) MATHEMATICAL BOLD ITALIC SMALL G +1D489;1D489;1D489;0068;0068; # (𝒉𝒉; 𝒉𝒉; 𝒉𝒉; h; h; ) MATHEMATICAL BOLD ITALIC SMALL H +1D48A;1D48A;1D48A;0069;0069; # (𝒊𝒊; 𝒊𝒊; 𝒊𝒊; i; i; ) MATHEMATICAL BOLD ITALIC SMALL I +1D48B;1D48B;1D48B;006A;006A; # (𝒋𝒋; 𝒋𝒋; 𝒋𝒋; j; j; ) MATHEMATICAL BOLD ITALIC SMALL J +1D48C;1D48C;1D48C;006B;006B; # (𝒌𝒌; 𝒌𝒌; 𝒌𝒌; k; k; ) MATHEMATICAL BOLD ITALIC SMALL K +1D48D;1D48D;1D48D;006C;006C; # (𝒍𝒍; 𝒍𝒍; 𝒍𝒍; l; l; ) MATHEMATICAL BOLD ITALIC SMALL L +1D48E;1D48E;1D48E;006D;006D; # (𝒎𝒎; 𝒎𝒎; 𝒎𝒎; m; m; ) MATHEMATICAL BOLD ITALIC SMALL M +1D48F;1D48F;1D48F;006E;006E; # (𝒏𝒏; 𝒏𝒏; 𝒏𝒏; n; n; ) MATHEMATICAL BOLD ITALIC SMALL N +1D490;1D490;1D490;006F;006F; # (𝒐𝒐; 𝒐𝒐; 𝒐𝒐; o; o; ) MATHEMATICAL BOLD ITALIC SMALL O +1D491;1D491;1D491;0070;0070; # (𝒑𝒑; 𝒑𝒑; 𝒑𝒑; p; p; ) MATHEMATICAL BOLD ITALIC SMALL P +1D492;1D492;1D492;0071;0071; # (𝒒𝒒; 𝒒𝒒; 𝒒𝒒; q; q; ) MATHEMATICAL BOLD ITALIC SMALL Q +1D493;1D493;1D493;0072;0072; # (𝒓𝒓; 𝒓𝒓; 𝒓𝒓; r; r; ) MATHEMATICAL BOLD ITALIC SMALL R +1D494;1D494;1D494;0073;0073; # (𝒔𝒔; 𝒔𝒔; 𝒔𝒔; s; s; ) MATHEMATICAL BOLD ITALIC SMALL S +1D495;1D495;1D495;0074;0074; # (𝒕𝒕; 𝒕𝒕; 𝒕𝒕; t; t; ) MATHEMATICAL BOLD ITALIC SMALL T +1D496;1D496;1D496;0075;0075; # (𝒖𝒖; 𝒖𝒖; 𝒖𝒖; u; u; ) MATHEMATICAL BOLD ITALIC SMALL U +1D497;1D497;1D497;0076;0076; # (𝒗𝒗; 𝒗𝒗; 𝒗𝒗; v; v; ) MATHEMATICAL BOLD ITALIC SMALL V +1D498;1D498;1D498;0077;0077; # (𝒘𝒘; 𝒘𝒘; 𝒘𝒘; w; w; ) MATHEMATICAL BOLD ITALIC SMALL W +1D499;1D499;1D499;0078;0078; # (𝒙𝒙; 𝒙𝒙; 𝒙𝒙; x; x; ) MATHEMATICAL BOLD ITALIC SMALL X +1D49A;1D49A;1D49A;0079;0079; # (𝒚𝒚; 𝒚𝒚; 𝒚𝒚; y; y; ) MATHEMATICAL BOLD ITALIC SMALL Y +1D49B;1D49B;1D49B;007A;007A; # (𝒛𝒛; 𝒛𝒛; 𝒛𝒛; z; z; ) MATHEMATICAL BOLD ITALIC SMALL Z +1D49C;1D49C;1D49C;0041;0041; # (𝒜𝒜; 𝒜𝒜; 𝒜𝒜; A; A; ) MATHEMATICAL SCRIPT CAPITAL A +1D49E;1D49E;1D49E;0043;0043; # (𝒞𝒞; 𝒞𝒞; 𝒞𝒞; C; C; ) MATHEMATICAL SCRIPT CAPITAL C +1D49F;1D49F;1D49F;0044;0044; # (𝒟𝒟; 𝒟𝒟; 𝒟𝒟; D; D; ) MATHEMATICAL SCRIPT CAPITAL D +1D4A2;1D4A2;1D4A2;0047;0047; # (𝒢𝒢; 𝒢𝒢; 𝒢𝒢; G; G; ) MATHEMATICAL SCRIPT CAPITAL G +1D4A5;1D4A5;1D4A5;004A;004A; # (𝒥𝒥; 𝒥𝒥; 𝒥𝒥; J; J; ) MATHEMATICAL SCRIPT CAPITAL J +1D4A6;1D4A6;1D4A6;004B;004B; # (𝒦𝒦; 𝒦𝒦; 𝒦𝒦; K; K; ) MATHEMATICAL SCRIPT CAPITAL K +1D4A9;1D4A9;1D4A9;004E;004E; # (𝒩𝒩; 𝒩𝒩; 𝒩𝒩; N; N; ) MATHEMATICAL SCRIPT CAPITAL N +1D4AA;1D4AA;1D4AA;004F;004F; # (𝒪𝒪; 𝒪𝒪; 𝒪𝒪; O; O; ) MATHEMATICAL SCRIPT CAPITAL O +1D4AB;1D4AB;1D4AB;0050;0050; # (𝒫𝒫; 𝒫𝒫; 𝒫𝒫; P; P; ) MATHEMATICAL SCRIPT CAPITAL P +1D4AC;1D4AC;1D4AC;0051;0051; # (𝒬𝒬; 𝒬𝒬; 𝒬𝒬; Q; Q; ) MATHEMATICAL SCRIPT CAPITAL Q +1D4AE;1D4AE;1D4AE;0053;0053; # (𝒮𝒮; 𝒮𝒮; 𝒮𝒮; S; S; ) MATHEMATICAL SCRIPT CAPITAL S +1D4AF;1D4AF;1D4AF;0054;0054; # (𝒯𝒯; 𝒯𝒯; 𝒯𝒯; T; T; ) MATHEMATICAL SCRIPT CAPITAL T +1D4B0;1D4B0;1D4B0;0055;0055; # (𝒰𝒰; 𝒰𝒰; 𝒰𝒰; U; U; ) MATHEMATICAL SCRIPT CAPITAL U +1D4B1;1D4B1;1D4B1;0056;0056; # (𝒱𝒱; 𝒱𝒱; 𝒱𝒱; V; V; ) MATHEMATICAL SCRIPT CAPITAL V +1D4B2;1D4B2;1D4B2;0057;0057; # (𝒲𝒲; 𝒲𝒲; 𝒲𝒲; W; W; ) MATHEMATICAL SCRIPT CAPITAL W +1D4B3;1D4B3;1D4B3;0058;0058; # (𝒳𝒳; 𝒳𝒳; 𝒳𝒳; X; X; ) MATHEMATICAL SCRIPT CAPITAL X +1D4B4;1D4B4;1D4B4;0059;0059; # (𝒴𝒴; 𝒴𝒴; 𝒴𝒴; Y; Y; ) MATHEMATICAL SCRIPT CAPITAL Y +1D4B5;1D4B5;1D4B5;005A;005A; # (𝒵𝒵; 𝒵𝒵; 𝒵𝒵; Z; Z; ) MATHEMATICAL SCRIPT CAPITAL Z +1D4B6;1D4B6;1D4B6;0061;0061; # (𝒶𝒶; 𝒶𝒶; 𝒶𝒶; a; a; ) MATHEMATICAL SCRIPT SMALL A +1D4B7;1D4B7;1D4B7;0062;0062; # (𝒷𝒷; 𝒷𝒷; 𝒷𝒷; b; b; ) MATHEMATICAL SCRIPT SMALL B +1D4B8;1D4B8;1D4B8;0063;0063; # (𝒸𝒸; 𝒸𝒸; 𝒸𝒸; c; c; ) MATHEMATICAL SCRIPT SMALL C +1D4B9;1D4B9;1D4B9;0064;0064; # (𝒹𝒹; 𝒹𝒹; 𝒹𝒹; d; d; ) MATHEMATICAL SCRIPT SMALL D +1D4BB;1D4BB;1D4BB;0066;0066; # (𝒻𝒻; 𝒻𝒻; 𝒻𝒻; f; f; ) MATHEMATICAL SCRIPT SMALL F +1D4BD;1D4BD;1D4BD;0068;0068; # (𝒽𝒽; 𝒽𝒽; 𝒽𝒽; h; h; ) MATHEMATICAL SCRIPT SMALL H +1D4BE;1D4BE;1D4BE;0069;0069; # (𝒾𝒾; 𝒾𝒾; 𝒾𝒾; i; i; ) MATHEMATICAL SCRIPT SMALL I +1D4BF;1D4BF;1D4BF;006A;006A; # (𝒿𝒿; 𝒿𝒿; 𝒿𝒿; j; j; ) MATHEMATICAL SCRIPT SMALL J +1D4C0;1D4C0;1D4C0;006B;006B; # (𝓀𝓀; 𝓀𝓀; 𝓀𝓀; k; k; ) MATHEMATICAL SCRIPT SMALL K +1D4C2;1D4C2;1D4C2;006D;006D; # (𝓂𝓂; 𝓂𝓂; 𝓂𝓂; m; m; ) MATHEMATICAL SCRIPT SMALL M +1D4C3;1D4C3;1D4C3;006E;006E; # (𝓃𝓃; 𝓃𝓃; 𝓃𝓃; n; n; ) MATHEMATICAL SCRIPT SMALL N +1D4C5;1D4C5;1D4C5;0070;0070; # (𝓅𝓅; 𝓅𝓅; 𝓅𝓅; p; p; ) MATHEMATICAL SCRIPT SMALL P +1D4C6;1D4C6;1D4C6;0071;0071; # (𝓆𝓆; 𝓆𝓆; 𝓆𝓆; q; q; ) MATHEMATICAL SCRIPT SMALL Q +1D4C7;1D4C7;1D4C7;0072;0072; # (𝓇𝓇; 𝓇𝓇; 𝓇𝓇; r; r; ) MATHEMATICAL SCRIPT SMALL R +1D4C8;1D4C8;1D4C8;0073;0073; # (𝓈𝓈; 𝓈𝓈; 𝓈𝓈; s; s; ) MATHEMATICAL SCRIPT SMALL S +1D4C9;1D4C9;1D4C9;0074;0074; # (𝓉𝓉; 𝓉𝓉; 𝓉𝓉; t; t; ) MATHEMATICAL SCRIPT SMALL T +1D4CA;1D4CA;1D4CA;0075;0075; # (𝓊𝓊; 𝓊𝓊; 𝓊𝓊; u; u; ) MATHEMATICAL SCRIPT SMALL U +1D4CB;1D4CB;1D4CB;0076;0076; # (𝓋𝓋; 𝓋𝓋; 𝓋𝓋; v; v; ) MATHEMATICAL SCRIPT SMALL V +1D4CC;1D4CC;1D4CC;0077;0077; # (𝓌𝓌; 𝓌𝓌; 𝓌𝓌; w; w; ) MATHEMATICAL SCRIPT SMALL W +1D4CD;1D4CD;1D4CD;0078;0078; # (𝓍𝓍; 𝓍𝓍; 𝓍𝓍; x; x; ) MATHEMATICAL SCRIPT SMALL X +1D4CE;1D4CE;1D4CE;0079;0079; # (𝓎𝓎; 𝓎𝓎; 𝓎𝓎; y; y; ) MATHEMATICAL SCRIPT SMALL Y +1D4CF;1D4CF;1D4CF;007A;007A; # (𝓏𝓏; 𝓏𝓏; 𝓏𝓏; z; z; ) MATHEMATICAL SCRIPT SMALL Z +1D4D0;1D4D0;1D4D0;0041;0041; # (𝓐𝓐; 𝓐𝓐; 𝓐𝓐; A; A; ) MATHEMATICAL BOLD SCRIPT CAPITAL A +1D4D1;1D4D1;1D4D1;0042;0042; # (𝓑𝓑; 𝓑𝓑; 𝓑𝓑; B; B; ) MATHEMATICAL BOLD SCRIPT CAPITAL B +1D4D2;1D4D2;1D4D2;0043;0043; # (𝓒𝓒; 𝓒𝓒; 𝓒𝓒; C; C; ) MATHEMATICAL BOLD SCRIPT CAPITAL C +1D4D3;1D4D3;1D4D3;0044;0044; # (𝓓𝓓; 𝓓𝓓; 𝓓𝓓; D; D; ) MATHEMATICAL BOLD SCRIPT CAPITAL D +1D4D4;1D4D4;1D4D4;0045;0045; # (𝓔𝓔; 𝓔𝓔; 𝓔𝓔; E; E; ) MATHEMATICAL BOLD SCRIPT CAPITAL E +1D4D5;1D4D5;1D4D5;0046;0046; # (𝓕𝓕; 𝓕𝓕; 𝓕𝓕; F; F; ) MATHEMATICAL BOLD SCRIPT CAPITAL F +1D4D6;1D4D6;1D4D6;0047;0047; # (𝓖𝓖; 𝓖𝓖; 𝓖𝓖; G; G; ) MATHEMATICAL BOLD SCRIPT CAPITAL G +1D4D7;1D4D7;1D4D7;0048;0048; # (𝓗𝓗; 𝓗𝓗; 𝓗𝓗; H; H; ) MATHEMATICAL BOLD SCRIPT CAPITAL H +1D4D8;1D4D8;1D4D8;0049;0049; # (𝓘𝓘; 𝓘𝓘; 𝓘𝓘; I; I; ) MATHEMATICAL BOLD SCRIPT CAPITAL I +1D4D9;1D4D9;1D4D9;004A;004A; # (𝓙𝓙; 𝓙𝓙; 𝓙𝓙; J; J; ) MATHEMATICAL BOLD SCRIPT CAPITAL J +1D4DA;1D4DA;1D4DA;004B;004B; # (𝓚𝓚; 𝓚𝓚; 𝓚𝓚; K; K; ) MATHEMATICAL BOLD SCRIPT CAPITAL K +1D4DB;1D4DB;1D4DB;004C;004C; # (𝓛𝓛; 𝓛𝓛; 𝓛𝓛; L; L; ) MATHEMATICAL BOLD SCRIPT CAPITAL L +1D4DC;1D4DC;1D4DC;004D;004D; # (𝓜𝓜; 𝓜𝓜; 𝓜𝓜; M; M; ) MATHEMATICAL BOLD SCRIPT CAPITAL M +1D4DD;1D4DD;1D4DD;004E;004E; # (𝓝𝓝; 𝓝𝓝; 𝓝𝓝; N; N; ) MATHEMATICAL BOLD SCRIPT CAPITAL N +1D4DE;1D4DE;1D4DE;004F;004F; # (𝓞𝓞; 𝓞𝓞; 𝓞𝓞; O; O; ) MATHEMATICAL BOLD SCRIPT CAPITAL O +1D4DF;1D4DF;1D4DF;0050;0050; # (𝓟𝓟; 𝓟𝓟; 𝓟𝓟; P; P; ) MATHEMATICAL BOLD SCRIPT CAPITAL P +1D4E0;1D4E0;1D4E0;0051;0051; # (𝓠𝓠; 𝓠𝓠; 𝓠𝓠; Q; Q; ) MATHEMATICAL BOLD SCRIPT CAPITAL Q +1D4E1;1D4E1;1D4E1;0052;0052; # (𝓡𝓡; 𝓡𝓡; 𝓡𝓡; R; R; ) MATHEMATICAL BOLD SCRIPT CAPITAL R +1D4E2;1D4E2;1D4E2;0053;0053; # (𝓢𝓢; 𝓢𝓢; 𝓢𝓢; S; S; ) MATHEMATICAL BOLD SCRIPT CAPITAL S +1D4E3;1D4E3;1D4E3;0054;0054; # (𝓣𝓣; 𝓣𝓣; 𝓣𝓣; T; T; ) MATHEMATICAL BOLD SCRIPT CAPITAL T +1D4E4;1D4E4;1D4E4;0055;0055; # (𝓤𝓤; 𝓤𝓤; 𝓤𝓤; U; U; ) MATHEMATICAL BOLD SCRIPT CAPITAL U +1D4E5;1D4E5;1D4E5;0056;0056; # (𝓥𝓥; 𝓥𝓥; 𝓥𝓥; V; V; ) MATHEMATICAL BOLD SCRIPT CAPITAL V +1D4E6;1D4E6;1D4E6;0057;0057; # (𝓦𝓦; 𝓦𝓦; 𝓦𝓦; W; W; ) MATHEMATICAL BOLD SCRIPT CAPITAL W +1D4E7;1D4E7;1D4E7;0058;0058; # (𝓧𝓧; 𝓧𝓧; 𝓧𝓧; X; X; ) MATHEMATICAL BOLD SCRIPT CAPITAL X +1D4E8;1D4E8;1D4E8;0059;0059; # (𝓨𝓨; 𝓨𝓨; 𝓨𝓨; Y; Y; ) MATHEMATICAL BOLD SCRIPT CAPITAL Y +1D4E9;1D4E9;1D4E9;005A;005A; # (𝓩𝓩; 𝓩𝓩; 𝓩𝓩; Z; Z; ) MATHEMATICAL BOLD SCRIPT CAPITAL Z +1D4EA;1D4EA;1D4EA;0061;0061; # (𝓪𝓪; 𝓪𝓪; 𝓪𝓪; a; a; ) MATHEMATICAL BOLD SCRIPT SMALL A +1D4EB;1D4EB;1D4EB;0062;0062; # (𝓫𝓫; 𝓫𝓫; 𝓫𝓫; b; b; ) MATHEMATICAL BOLD SCRIPT SMALL B +1D4EC;1D4EC;1D4EC;0063;0063; # (𝓬𝓬; 𝓬𝓬; 𝓬𝓬; c; c; ) MATHEMATICAL BOLD SCRIPT SMALL C +1D4ED;1D4ED;1D4ED;0064;0064; # (𝓭𝓭; 𝓭𝓭; 𝓭𝓭; d; d; ) MATHEMATICAL BOLD SCRIPT SMALL D +1D4EE;1D4EE;1D4EE;0065;0065; # (𝓮𝓮; 𝓮𝓮; 𝓮𝓮; e; e; ) MATHEMATICAL BOLD SCRIPT SMALL E +1D4EF;1D4EF;1D4EF;0066;0066; # (𝓯𝓯; 𝓯𝓯; 𝓯𝓯; f; f; ) MATHEMATICAL BOLD SCRIPT SMALL F +1D4F0;1D4F0;1D4F0;0067;0067; # (𝓰𝓰; 𝓰𝓰; 𝓰𝓰; g; g; ) MATHEMATICAL BOLD SCRIPT SMALL G +1D4F1;1D4F1;1D4F1;0068;0068; # (𝓱𝓱; 𝓱𝓱; 𝓱𝓱; h; h; ) MATHEMATICAL BOLD SCRIPT SMALL H +1D4F2;1D4F2;1D4F2;0069;0069; # (𝓲𝓲; 𝓲𝓲; 𝓲𝓲; i; i; ) MATHEMATICAL BOLD SCRIPT SMALL I +1D4F3;1D4F3;1D4F3;006A;006A; # (𝓳𝓳; 𝓳𝓳; 𝓳𝓳; j; j; ) MATHEMATICAL BOLD SCRIPT SMALL J +1D4F4;1D4F4;1D4F4;006B;006B; # (𝓴𝓴; 𝓴𝓴; 𝓴𝓴; k; k; ) MATHEMATICAL BOLD SCRIPT SMALL K +1D4F5;1D4F5;1D4F5;006C;006C; # (𝓵𝓵; 𝓵𝓵; 𝓵𝓵; l; l; ) MATHEMATICAL BOLD SCRIPT SMALL L +1D4F6;1D4F6;1D4F6;006D;006D; # (𝓶𝓶; 𝓶𝓶; 𝓶𝓶; m; m; ) MATHEMATICAL BOLD SCRIPT SMALL M +1D4F7;1D4F7;1D4F7;006E;006E; # (𝓷𝓷; 𝓷𝓷; 𝓷𝓷; n; n; ) MATHEMATICAL BOLD SCRIPT SMALL N +1D4F8;1D4F8;1D4F8;006F;006F; # (𝓸𝓸; 𝓸𝓸; 𝓸𝓸; o; o; ) MATHEMATICAL BOLD SCRIPT SMALL O +1D4F9;1D4F9;1D4F9;0070;0070; # (𝓹𝓹; 𝓹𝓹; 𝓹𝓹; p; p; ) MATHEMATICAL BOLD SCRIPT SMALL P +1D4FA;1D4FA;1D4FA;0071;0071; # (𝓺𝓺; 𝓺𝓺; 𝓺𝓺; q; q; ) MATHEMATICAL BOLD SCRIPT SMALL Q +1D4FB;1D4FB;1D4FB;0072;0072; # (𝓻𝓻; 𝓻𝓻; 𝓻𝓻; r; r; ) MATHEMATICAL BOLD SCRIPT SMALL R +1D4FC;1D4FC;1D4FC;0073;0073; # (𝓼𝓼; 𝓼𝓼; 𝓼𝓼; s; s; ) MATHEMATICAL BOLD SCRIPT SMALL S +1D4FD;1D4FD;1D4FD;0074;0074; # (𝓽𝓽; 𝓽𝓽; 𝓽𝓽; t; t; ) MATHEMATICAL BOLD SCRIPT SMALL T +1D4FE;1D4FE;1D4FE;0075;0075; # (𝓾𝓾; 𝓾𝓾; 𝓾𝓾; u; u; ) MATHEMATICAL BOLD SCRIPT SMALL U +1D4FF;1D4FF;1D4FF;0076;0076; # (𝓿𝓿; 𝓿𝓿; 𝓿𝓿; v; v; ) MATHEMATICAL BOLD SCRIPT SMALL V +1D500;1D500;1D500;0077;0077; # (𝔀𝔀; 𝔀𝔀; 𝔀𝔀; w; w; ) MATHEMATICAL BOLD SCRIPT SMALL W +1D501;1D501;1D501;0078;0078; # (𝔁𝔁; 𝔁𝔁; 𝔁𝔁; x; x; ) MATHEMATICAL BOLD SCRIPT SMALL X +1D502;1D502;1D502;0079;0079; # (𝔂𝔂; 𝔂𝔂; 𝔂𝔂; y; y; ) MATHEMATICAL BOLD SCRIPT SMALL Y +1D503;1D503;1D503;007A;007A; # (𝔃𝔃; 𝔃𝔃; 𝔃𝔃; z; z; ) MATHEMATICAL BOLD SCRIPT SMALL Z +1D504;1D504;1D504;0041;0041; # (𝔄𝔄; 𝔄𝔄; 𝔄𝔄; A; A; ) MATHEMATICAL FRAKTUR CAPITAL A +1D505;1D505;1D505;0042;0042; # (𝔅𝔅; 𝔅𝔅; 𝔅𝔅; B; B; ) MATHEMATICAL FRAKTUR CAPITAL B +1D507;1D507;1D507;0044;0044; # (𝔇𝔇; 𝔇𝔇; 𝔇𝔇; D; D; ) MATHEMATICAL FRAKTUR CAPITAL D +1D508;1D508;1D508;0045;0045; # (𝔈𝔈; 𝔈𝔈; 𝔈𝔈; E; E; ) MATHEMATICAL FRAKTUR CAPITAL E +1D509;1D509;1D509;0046;0046; # (𝔉𝔉; 𝔉𝔉; 𝔉𝔉; F; F; ) MATHEMATICAL FRAKTUR CAPITAL F +1D50A;1D50A;1D50A;0047;0047; # (𝔊𝔊; 𝔊𝔊; 𝔊𝔊; G; G; ) MATHEMATICAL FRAKTUR CAPITAL G +1D50D;1D50D;1D50D;004A;004A; # (𝔍𝔍; 𝔍𝔍; 𝔍𝔍; J; J; ) MATHEMATICAL FRAKTUR CAPITAL J +1D50E;1D50E;1D50E;004B;004B; # (𝔎𝔎; 𝔎𝔎; 𝔎𝔎; K; K; ) MATHEMATICAL FRAKTUR CAPITAL K +1D50F;1D50F;1D50F;004C;004C; # (𝔏𝔏; 𝔏𝔏; 𝔏𝔏; L; L; ) MATHEMATICAL FRAKTUR CAPITAL L +1D510;1D510;1D510;004D;004D; # (𝔐𝔐; 𝔐𝔐; 𝔐𝔐; M; M; ) MATHEMATICAL FRAKTUR CAPITAL M +1D511;1D511;1D511;004E;004E; # (𝔑𝔑; 𝔑𝔑; 𝔑𝔑; N; N; ) MATHEMATICAL FRAKTUR CAPITAL N +1D512;1D512;1D512;004F;004F; # (𝔒𝔒; 𝔒𝔒; 𝔒𝔒; O; O; ) MATHEMATICAL FRAKTUR CAPITAL O +1D513;1D513;1D513;0050;0050; # (𝔓𝔓; 𝔓𝔓; 𝔓𝔓; P; P; ) MATHEMATICAL FRAKTUR CAPITAL P +1D514;1D514;1D514;0051;0051; # (𝔔𝔔; 𝔔𝔔; 𝔔𝔔; Q; Q; ) MATHEMATICAL FRAKTUR CAPITAL Q +1D516;1D516;1D516;0053;0053; # (𝔖𝔖; 𝔖𝔖; 𝔖𝔖; S; S; ) MATHEMATICAL FRAKTUR CAPITAL S +1D517;1D517;1D517;0054;0054; # (𝔗𝔗; 𝔗𝔗; 𝔗𝔗; T; T; ) MATHEMATICAL FRAKTUR CAPITAL T +1D518;1D518;1D518;0055;0055; # (𝔘𝔘; 𝔘𝔘; 𝔘𝔘; U; U; ) MATHEMATICAL FRAKTUR CAPITAL U +1D519;1D519;1D519;0056;0056; # (𝔙𝔙; 𝔙𝔙; 𝔙𝔙; V; V; ) MATHEMATICAL FRAKTUR CAPITAL V +1D51A;1D51A;1D51A;0057;0057; # (𝔚𝔚; 𝔚𝔚; 𝔚𝔚; W; W; ) MATHEMATICAL FRAKTUR CAPITAL W +1D51B;1D51B;1D51B;0058;0058; # (𝔛𝔛; 𝔛𝔛; 𝔛𝔛; X; X; ) MATHEMATICAL FRAKTUR CAPITAL X +1D51C;1D51C;1D51C;0059;0059; # (𝔜𝔜; 𝔜𝔜; 𝔜𝔜; Y; Y; ) MATHEMATICAL FRAKTUR CAPITAL Y +1D51E;1D51E;1D51E;0061;0061; # (𝔞𝔞; 𝔞𝔞; 𝔞𝔞; a; a; ) MATHEMATICAL FRAKTUR SMALL A +1D51F;1D51F;1D51F;0062;0062; # (𝔟𝔟; 𝔟𝔟; 𝔟𝔟; b; b; ) MATHEMATICAL FRAKTUR SMALL B +1D520;1D520;1D520;0063;0063; # (𝔠𝔠; 𝔠𝔠; 𝔠𝔠; c; c; ) MATHEMATICAL FRAKTUR SMALL C +1D521;1D521;1D521;0064;0064; # (𝔡𝔡; 𝔡𝔡; 𝔡𝔡; d; d; ) MATHEMATICAL FRAKTUR SMALL D +1D522;1D522;1D522;0065;0065; # (𝔢𝔢; 𝔢𝔢; 𝔢𝔢; e; e; ) MATHEMATICAL FRAKTUR SMALL E +1D523;1D523;1D523;0066;0066; # (𝔣𝔣; 𝔣𝔣; 𝔣𝔣; f; f; ) MATHEMATICAL FRAKTUR SMALL F +1D524;1D524;1D524;0067;0067; # (𝔤𝔤; 𝔤𝔤; 𝔤𝔤; g; g; ) MATHEMATICAL FRAKTUR SMALL G +1D525;1D525;1D525;0068;0068; # (𝔥𝔥; 𝔥𝔥; 𝔥𝔥; h; h; ) MATHEMATICAL FRAKTUR SMALL H +1D526;1D526;1D526;0069;0069; # (𝔦𝔦; 𝔦𝔦; 𝔦𝔦; i; i; ) MATHEMATICAL FRAKTUR SMALL I +1D527;1D527;1D527;006A;006A; # (𝔧𝔧; 𝔧𝔧; 𝔧𝔧; j; j; ) MATHEMATICAL FRAKTUR SMALL J +1D528;1D528;1D528;006B;006B; # (𝔨𝔨; 𝔨𝔨; 𝔨𝔨; k; k; ) MATHEMATICAL FRAKTUR SMALL K +1D529;1D529;1D529;006C;006C; # (𝔩𝔩; 𝔩𝔩; 𝔩𝔩; l; l; ) MATHEMATICAL FRAKTUR SMALL L +1D52A;1D52A;1D52A;006D;006D; # (𝔪𝔪; 𝔪𝔪; 𝔪𝔪; m; m; ) MATHEMATICAL FRAKTUR SMALL M +1D52B;1D52B;1D52B;006E;006E; # (𝔫𝔫; 𝔫𝔫; 𝔫𝔫; n; n; ) MATHEMATICAL FRAKTUR SMALL N +1D52C;1D52C;1D52C;006F;006F; # (𝔬𝔬; 𝔬𝔬; 𝔬𝔬; o; o; ) MATHEMATICAL FRAKTUR SMALL O +1D52D;1D52D;1D52D;0070;0070; # (𝔭𝔭; 𝔭𝔭; 𝔭𝔭; p; p; ) MATHEMATICAL FRAKTUR SMALL P +1D52E;1D52E;1D52E;0071;0071; # (𝔮𝔮; 𝔮𝔮; 𝔮𝔮; q; q; ) MATHEMATICAL FRAKTUR SMALL Q +1D52F;1D52F;1D52F;0072;0072; # (𝔯𝔯; 𝔯𝔯; 𝔯𝔯; r; r; ) MATHEMATICAL FRAKTUR SMALL R +1D530;1D530;1D530;0073;0073; # (𝔰𝔰; 𝔰𝔰; 𝔰𝔰; s; s; ) MATHEMATICAL FRAKTUR SMALL S +1D531;1D531;1D531;0074;0074; # (𝔱𝔱; 𝔱𝔱; 𝔱𝔱; t; t; ) MATHEMATICAL FRAKTUR SMALL T +1D532;1D532;1D532;0075;0075; # (𝔲𝔲; 𝔲𝔲; 𝔲𝔲; u; u; ) MATHEMATICAL FRAKTUR SMALL U +1D533;1D533;1D533;0076;0076; # (𝔳𝔳; 𝔳𝔳; 𝔳𝔳; v; v; ) MATHEMATICAL FRAKTUR SMALL V +1D534;1D534;1D534;0077;0077; # (𝔴𝔴; 𝔴𝔴; 𝔴𝔴; w; w; ) MATHEMATICAL FRAKTUR SMALL W +1D535;1D535;1D535;0078;0078; # (𝔵𝔵; 𝔵𝔵; 𝔵𝔵; x; x; ) MATHEMATICAL FRAKTUR SMALL X +1D536;1D536;1D536;0079;0079; # (𝔶𝔶; 𝔶𝔶; 𝔶𝔶; y; y; ) MATHEMATICAL FRAKTUR SMALL Y +1D537;1D537;1D537;007A;007A; # (𝔷𝔷; 𝔷𝔷; 𝔷𝔷; z; z; ) MATHEMATICAL FRAKTUR SMALL Z +1D538;1D538;1D538;0041;0041; # (𝔸𝔸; 𝔸𝔸; 𝔸𝔸; A; A; ) MATHEMATICAL DOUBLE-STRUCK CAPITAL A +1D539;1D539;1D539;0042;0042; # (𝔹𝔹; 𝔹𝔹; 𝔹𝔹; B; B; ) MATHEMATICAL DOUBLE-STRUCK CAPITAL B +1D53B;1D53B;1D53B;0044;0044; # (𝔻𝔻; 𝔻𝔻; 𝔻𝔻; D; D; ) MATHEMATICAL DOUBLE-STRUCK CAPITAL D +1D53C;1D53C;1D53C;0045;0045; # (𝔼𝔼; 𝔼𝔼; 𝔼𝔼; E; E; ) MATHEMATICAL DOUBLE-STRUCK CAPITAL E +1D53D;1D53D;1D53D;0046;0046; # (𝔽𝔽; 𝔽𝔽; 𝔽𝔽; F; F; ) MATHEMATICAL DOUBLE-STRUCK CAPITAL F +1D53E;1D53E;1D53E;0047;0047; # (𝔾𝔾; 𝔾𝔾; 𝔾𝔾; G; G; ) MATHEMATICAL DOUBLE-STRUCK CAPITAL G +1D540;1D540;1D540;0049;0049; # (𝕀𝕀; 𝕀𝕀; 𝕀𝕀; I; I; ) MATHEMATICAL DOUBLE-STRUCK CAPITAL I +1D541;1D541;1D541;004A;004A; # (𝕁𝕁; 𝕁𝕁; 𝕁𝕁; J; J; ) MATHEMATICAL DOUBLE-STRUCK CAPITAL J +1D542;1D542;1D542;004B;004B; # (𝕂𝕂; 𝕂𝕂; 𝕂𝕂; K; K; ) MATHEMATICAL DOUBLE-STRUCK CAPITAL K +1D543;1D543;1D543;004C;004C; # (𝕃𝕃; 𝕃𝕃; 𝕃𝕃; L; L; ) MATHEMATICAL DOUBLE-STRUCK CAPITAL L +1D544;1D544;1D544;004D;004D; # (𝕄𝕄; 𝕄𝕄; 𝕄𝕄; M; M; ) MATHEMATICAL DOUBLE-STRUCK CAPITAL M +1D546;1D546;1D546;004F;004F; # (𝕆𝕆; 𝕆𝕆; 𝕆𝕆; O; O; ) MATHEMATICAL DOUBLE-STRUCK CAPITAL O +1D54A;1D54A;1D54A;0053;0053; # (𝕊𝕊; 𝕊𝕊; 𝕊𝕊; S; S; ) MATHEMATICAL DOUBLE-STRUCK CAPITAL S +1D54B;1D54B;1D54B;0054;0054; # (𝕋𝕋; 𝕋𝕋; 𝕋𝕋; T; T; ) MATHEMATICAL DOUBLE-STRUCK CAPITAL T +1D54C;1D54C;1D54C;0055;0055; # (𝕌𝕌; 𝕌𝕌; 𝕌𝕌; U; U; ) MATHEMATICAL DOUBLE-STRUCK CAPITAL U +1D54D;1D54D;1D54D;0056;0056; # (𝕍𝕍; 𝕍𝕍; 𝕍𝕍; V; V; ) MATHEMATICAL DOUBLE-STRUCK CAPITAL V +1D54E;1D54E;1D54E;0057;0057; # (𝕎𝕎; 𝕎𝕎; 𝕎𝕎; W; W; ) MATHEMATICAL DOUBLE-STRUCK CAPITAL W +1D54F;1D54F;1D54F;0058;0058; # (𝕏𝕏; 𝕏𝕏; 𝕏𝕏; X; X; ) MATHEMATICAL DOUBLE-STRUCK CAPITAL X +1D550;1D550;1D550;0059;0059; # (𝕐𝕐; 𝕐𝕐; 𝕐𝕐; Y; Y; ) MATHEMATICAL DOUBLE-STRUCK CAPITAL Y +1D552;1D552;1D552;0061;0061; # (𝕒𝕒; 𝕒𝕒; 𝕒𝕒; a; a; ) MATHEMATICAL DOUBLE-STRUCK SMALL A +1D553;1D553;1D553;0062;0062; # (𝕓𝕓; 𝕓𝕓; 𝕓𝕓; b; b; ) MATHEMATICAL DOUBLE-STRUCK SMALL B +1D554;1D554;1D554;0063;0063; # (𝕔𝕔; 𝕔𝕔; 𝕔𝕔; c; c; ) MATHEMATICAL DOUBLE-STRUCK SMALL C +1D555;1D555;1D555;0064;0064; # (𝕕𝕕; 𝕕𝕕; 𝕕𝕕; d; d; ) MATHEMATICAL DOUBLE-STRUCK SMALL D +1D556;1D556;1D556;0065;0065; # (𝕖𝕖; 𝕖𝕖; 𝕖𝕖; e; e; ) MATHEMATICAL DOUBLE-STRUCK SMALL E +1D557;1D557;1D557;0066;0066; # (𝕗𝕗; 𝕗𝕗; 𝕗𝕗; f; f; ) MATHEMATICAL DOUBLE-STRUCK SMALL F +1D558;1D558;1D558;0067;0067; # (𝕘𝕘; 𝕘𝕘; 𝕘𝕘; g; g; ) MATHEMATICAL DOUBLE-STRUCK SMALL G +1D559;1D559;1D559;0068;0068; # (𝕙𝕙; 𝕙𝕙; 𝕙𝕙; h; h; ) MATHEMATICAL DOUBLE-STRUCK SMALL H +1D55A;1D55A;1D55A;0069;0069; # (𝕚𝕚; 𝕚𝕚; 𝕚𝕚; i; i; ) MATHEMATICAL DOUBLE-STRUCK SMALL I +1D55B;1D55B;1D55B;006A;006A; # (𝕛𝕛; 𝕛𝕛; 𝕛𝕛; j; j; ) MATHEMATICAL DOUBLE-STRUCK SMALL J +1D55C;1D55C;1D55C;006B;006B; # (𝕜𝕜; 𝕜𝕜; 𝕜𝕜; k; k; ) MATHEMATICAL DOUBLE-STRUCK SMALL K +1D55D;1D55D;1D55D;006C;006C; # (𝕝𝕝; 𝕝𝕝; 𝕝𝕝; l; l; ) MATHEMATICAL DOUBLE-STRUCK SMALL L +1D55E;1D55E;1D55E;006D;006D; # (𝕞𝕞; 𝕞𝕞; 𝕞𝕞; m; m; ) MATHEMATICAL DOUBLE-STRUCK SMALL M +1D55F;1D55F;1D55F;006E;006E; # (𝕟𝕟; 𝕟𝕟; 𝕟𝕟; n; n; ) MATHEMATICAL DOUBLE-STRUCK SMALL N +1D560;1D560;1D560;006F;006F; # (𝕠𝕠; 𝕠𝕠; 𝕠𝕠; o; o; ) MATHEMATICAL DOUBLE-STRUCK SMALL O +1D561;1D561;1D561;0070;0070; # (𝕡𝕡; 𝕡𝕡; 𝕡𝕡; p; p; ) MATHEMATICAL DOUBLE-STRUCK SMALL P +1D562;1D562;1D562;0071;0071; # (𝕢𝕢; 𝕢𝕢; 𝕢𝕢; q; q; ) MATHEMATICAL DOUBLE-STRUCK SMALL Q +1D563;1D563;1D563;0072;0072; # (𝕣𝕣; 𝕣𝕣; 𝕣𝕣; r; r; ) MATHEMATICAL DOUBLE-STRUCK SMALL R +1D564;1D564;1D564;0073;0073; # (𝕤𝕤; 𝕤𝕤; 𝕤𝕤; s; s; ) MATHEMATICAL DOUBLE-STRUCK SMALL S +1D565;1D565;1D565;0074;0074; # (𝕥𝕥; 𝕥𝕥; 𝕥𝕥; t; t; ) MATHEMATICAL DOUBLE-STRUCK SMALL T +1D566;1D566;1D566;0075;0075; # (𝕦𝕦; 𝕦𝕦; 𝕦𝕦; u; u; ) MATHEMATICAL DOUBLE-STRUCK SMALL U +1D567;1D567;1D567;0076;0076; # (𝕧𝕧; 𝕧𝕧; 𝕧𝕧; v; v; ) MATHEMATICAL DOUBLE-STRUCK SMALL V +1D568;1D568;1D568;0077;0077; # (𝕨𝕨; 𝕨𝕨; 𝕨𝕨; w; w; ) MATHEMATICAL DOUBLE-STRUCK SMALL W +1D569;1D569;1D569;0078;0078; # (𝕩𝕩; 𝕩𝕩; 𝕩𝕩; x; x; ) MATHEMATICAL DOUBLE-STRUCK SMALL X +1D56A;1D56A;1D56A;0079;0079; # (𝕪𝕪; 𝕪𝕪; 𝕪𝕪; y; y; ) MATHEMATICAL DOUBLE-STRUCK SMALL Y +1D56B;1D56B;1D56B;007A;007A; # (𝕫𝕫; 𝕫𝕫; 𝕫𝕫; z; z; ) MATHEMATICAL DOUBLE-STRUCK SMALL Z +1D56C;1D56C;1D56C;0041;0041; # (𝕬𝕬; 𝕬𝕬; 𝕬𝕬; A; A; ) MATHEMATICAL BOLD FRAKTUR CAPITAL A +1D56D;1D56D;1D56D;0042;0042; # (𝕭𝕭; 𝕭𝕭; 𝕭𝕭; B; B; ) MATHEMATICAL BOLD FRAKTUR CAPITAL B +1D56E;1D56E;1D56E;0043;0043; # (𝕮𝕮; 𝕮𝕮; 𝕮𝕮; C; C; ) MATHEMATICAL BOLD FRAKTUR CAPITAL C +1D56F;1D56F;1D56F;0044;0044; # (𝕯𝕯; 𝕯𝕯; 𝕯𝕯; D; D; ) MATHEMATICAL BOLD FRAKTUR CAPITAL D +1D570;1D570;1D570;0045;0045; # (𝕰𝕰; 𝕰𝕰; 𝕰𝕰; E; E; ) MATHEMATICAL BOLD FRAKTUR CAPITAL E +1D571;1D571;1D571;0046;0046; # (𝕱𝕱; 𝕱𝕱; 𝕱𝕱; F; F; ) MATHEMATICAL BOLD FRAKTUR CAPITAL F +1D572;1D572;1D572;0047;0047; # (𝕲𝕲; 𝕲𝕲; 𝕲𝕲; G; G; ) MATHEMATICAL BOLD FRAKTUR CAPITAL G +1D573;1D573;1D573;0048;0048; # (𝕳𝕳; 𝕳𝕳; 𝕳𝕳; H; H; ) MATHEMATICAL BOLD FRAKTUR CAPITAL H +1D574;1D574;1D574;0049;0049; # (𝕴𝕴; 𝕴𝕴; 𝕴𝕴; I; I; ) MATHEMATICAL BOLD FRAKTUR CAPITAL I +1D575;1D575;1D575;004A;004A; # (𝕵𝕵; 𝕵𝕵; 𝕵𝕵; J; J; ) MATHEMATICAL BOLD FRAKTUR CAPITAL J +1D576;1D576;1D576;004B;004B; # (𝕶𝕶; 𝕶𝕶; 𝕶𝕶; K; K; ) MATHEMATICAL BOLD FRAKTUR CAPITAL K +1D577;1D577;1D577;004C;004C; # (𝕷𝕷; 𝕷𝕷; 𝕷𝕷; L; L; ) MATHEMATICAL BOLD FRAKTUR CAPITAL L +1D578;1D578;1D578;004D;004D; # (𝕸𝕸; 𝕸𝕸; 𝕸𝕸; M; M; ) MATHEMATICAL BOLD FRAKTUR CAPITAL M +1D579;1D579;1D579;004E;004E; # (𝕹𝕹; 𝕹𝕹; 𝕹𝕹; N; N; ) MATHEMATICAL BOLD FRAKTUR CAPITAL N +1D57A;1D57A;1D57A;004F;004F; # (𝕺𝕺; 𝕺𝕺; 𝕺𝕺; O; O; ) MATHEMATICAL BOLD FRAKTUR CAPITAL O +1D57B;1D57B;1D57B;0050;0050; # (𝕻𝕻; 𝕻𝕻; 𝕻𝕻; P; P; ) MATHEMATICAL BOLD FRAKTUR CAPITAL P +1D57C;1D57C;1D57C;0051;0051; # (𝕼𝕼; 𝕼𝕼; 𝕼𝕼; Q; Q; ) MATHEMATICAL BOLD FRAKTUR CAPITAL Q +1D57D;1D57D;1D57D;0052;0052; # (𝕽𝕽; 𝕽𝕽; 𝕽𝕽; R; R; ) MATHEMATICAL BOLD FRAKTUR CAPITAL R +1D57E;1D57E;1D57E;0053;0053; # (𝕾𝕾; 𝕾𝕾; 𝕾𝕾; S; S; ) MATHEMATICAL BOLD FRAKTUR CAPITAL S +1D57F;1D57F;1D57F;0054;0054; # (𝕿𝕿; 𝕿𝕿; 𝕿𝕿; T; T; ) MATHEMATICAL BOLD FRAKTUR CAPITAL T +1D580;1D580;1D580;0055;0055; # (𝖀𝖀; 𝖀𝖀; 𝖀𝖀; U; U; ) MATHEMATICAL BOLD FRAKTUR CAPITAL U +1D581;1D581;1D581;0056;0056; # (𝖁𝖁; 𝖁𝖁; 𝖁𝖁; V; V; ) MATHEMATICAL BOLD FRAKTUR CAPITAL V +1D582;1D582;1D582;0057;0057; # (𝖂𝖂; 𝖂𝖂; 𝖂𝖂; W; W; ) MATHEMATICAL BOLD FRAKTUR CAPITAL W +1D583;1D583;1D583;0058;0058; # (𝖃𝖃; 𝖃𝖃; 𝖃𝖃; X; X; ) MATHEMATICAL BOLD FRAKTUR CAPITAL X +1D584;1D584;1D584;0059;0059; # (𝖄𝖄; 𝖄𝖄; 𝖄𝖄; Y; Y; ) MATHEMATICAL BOLD FRAKTUR CAPITAL Y +1D585;1D585;1D585;005A;005A; # (𝖅𝖅; 𝖅𝖅; 𝖅𝖅; Z; Z; ) MATHEMATICAL BOLD FRAKTUR CAPITAL Z +1D586;1D586;1D586;0061;0061; # (𝖆𝖆; 𝖆𝖆; 𝖆𝖆; a; a; ) MATHEMATICAL BOLD FRAKTUR SMALL A +1D587;1D587;1D587;0062;0062; # (𝖇𝖇; 𝖇𝖇; 𝖇𝖇; b; b; ) MATHEMATICAL BOLD FRAKTUR SMALL B +1D588;1D588;1D588;0063;0063; # (𝖈𝖈; 𝖈𝖈; 𝖈𝖈; c; c; ) MATHEMATICAL BOLD FRAKTUR SMALL C +1D589;1D589;1D589;0064;0064; # (𝖉𝖉; 𝖉𝖉; 𝖉𝖉; d; d; ) MATHEMATICAL BOLD FRAKTUR SMALL D +1D58A;1D58A;1D58A;0065;0065; # (𝖊𝖊; 𝖊𝖊; 𝖊𝖊; e; e; ) MATHEMATICAL BOLD FRAKTUR SMALL E +1D58B;1D58B;1D58B;0066;0066; # (𝖋𝖋; 𝖋𝖋; 𝖋𝖋; f; f; ) MATHEMATICAL BOLD FRAKTUR SMALL F +1D58C;1D58C;1D58C;0067;0067; # (𝖌𝖌; 𝖌𝖌; 𝖌𝖌; g; g; ) MATHEMATICAL BOLD FRAKTUR SMALL G +1D58D;1D58D;1D58D;0068;0068; # (𝖍𝖍; 𝖍𝖍; 𝖍𝖍; h; h; ) MATHEMATICAL BOLD FRAKTUR SMALL H +1D58E;1D58E;1D58E;0069;0069; # (𝖎𝖎; 𝖎𝖎; 𝖎𝖎; i; i; ) MATHEMATICAL BOLD FRAKTUR SMALL I +1D58F;1D58F;1D58F;006A;006A; # (𝖏𝖏; 𝖏𝖏; 𝖏𝖏; j; j; ) MATHEMATICAL BOLD FRAKTUR SMALL J +1D590;1D590;1D590;006B;006B; # (𝖐𝖐; 𝖐𝖐; 𝖐𝖐; k; k; ) MATHEMATICAL BOLD FRAKTUR SMALL K +1D591;1D591;1D591;006C;006C; # (𝖑𝖑; 𝖑𝖑; 𝖑𝖑; l; l; ) MATHEMATICAL BOLD FRAKTUR SMALL L +1D592;1D592;1D592;006D;006D; # (𝖒𝖒; 𝖒𝖒; 𝖒𝖒; m; m; ) MATHEMATICAL BOLD FRAKTUR SMALL M +1D593;1D593;1D593;006E;006E; # (𝖓𝖓; 𝖓𝖓; 𝖓𝖓; n; n; ) MATHEMATICAL BOLD FRAKTUR SMALL N +1D594;1D594;1D594;006F;006F; # (𝖔𝖔; 𝖔𝖔; 𝖔𝖔; o; o; ) MATHEMATICAL BOLD FRAKTUR SMALL O +1D595;1D595;1D595;0070;0070; # (𝖕𝖕; 𝖕𝖕; 𝖕𝖕; p; p; ) MATHEMATICAL BOLD FRAKTUR SMALL P +1D596;1D596;1D596;0071;0071; # (𝖖𝖖; 𝖖𝖖; 𝖖𝖖; q; q; ) MATHEMATICAL BOLD FRAKTUR SMALL Q +1D597;1D597;1D597;0072;0072; # (𝖗𝖗; 𝖗𝖗; 𝖗𝖗; r; r; ) MATHEMATICAL BOLD FRAKTUR SMALL R +1D598;1D598;1D598;0073;0073; # (𝖘𝖘; 𝖘𝖘; 𝖘𝖘; s; s; ) MATHEMATICAL BOLD FRAKTUR SMALL S +1D599;1D599;1D599;0074;0074; # (𝖙𝖙; 𝖙𝖙; 𝖙𝖙; t; t; ) MATHEMATICAL BOLD FRAKTUR SMALL T +1D59A;1D59A;1D59A;0075;0075; # (𝖚𝖚; 𝖚𝖚; 𝖚𝖚; u; u; ) MATHEMATICAL BOLD FRAKTUR SMALL U +1D59B;1D59B;1D59B;0076;0076; # (𝖛𝖛; 𝖛𝖛; 𝖛𝖛; v; v; ) MATHEMATICAL BOLD FRAKTUR SMALL V +1D59C;1D59C;1D59C;0077;0077; # (𝖜𝖜; 𝖜𝖜; 𝖜𝖜; w; w; ) MATHEMATICAL BOLD FRAKTUR SMALL W +1D59D;1D59D;1D59D;0078;0078; # (𝖝𝖝; 𝖝𝖝; 𝖝𝖝; x; x; ) MATHEMATICAL BOLD FRAKTUR SMALL X +1D59E;1D59E;1D59E;0079;0079; # (𝖞𝖞; 𝖞𝖞; 𝖞𝖞; y; y; ) MATHEMATICAL BOLD FRAKTUR SMALL Y +1D59F;1D59F;1D59F;007A;007A; # (𝖟𝖟; 𝖟𝖟; 𝖟𝖟; z; z; ) MATHEMATICAL BOLD FRAKTUR SMALL Z +1D5A0;1D5A0;1D5A0;0041;0041; # (𝖠𝖠; 𝖠𝖠; 𝖠𝖠; A; A; ) MATHEMATICAL SANS-SERIF CAPITAL A +1D5A1;1D5A1;1D5A1;0042;0042; # (𝖡𝖡; 𝖡𝖡; 𝖡𝖡; B; B; ) MATHEMATICAL SANS-SERIF CAPITAL B +1D5A2;1D5A2;1D5A2;0043;0043; # (𝖢𝖢; 𝖢𝖢; 𝖢𝖢; C; C; ) MATHEMATICAL SANS-SERIF CAPITAL C +1D5A3;1D5A3;1D5A3;0044;0044; # (𝖣𝖣; 𝖣𝖣; 𝖣𝖣; D; D; ) MATHEMATICAL SANS-SERIF CAPITAL D +1D5A4;1D5A4;1D5A4;0045;0045; # (𝖤𝖤; 𝖤𝖤; 𝖤𝖤; E; E; ) MATHEMATICAL SANS-SERIF CAPITAL E +1D5A5;1D5A5;1D5A5;0046;0046; # (𝖥𝖥; 𝖥𝖥; 𝖥𝖥; F; F; ) MATHEMATICAL SANS-SERIF CAPITAL F +1D5A6;1D5A6;1D5A6;0047;0047; # (𝖦𝖦; 𝖦𝖦; 𝖦𝖦; G; G; ) MATHEMATICAL SANS-SERIF CAPITAL G +1D5A7;1D5A7;1D5A7;0048;0048; # (𝖧𝖧; 𝖧𝖧; 𝖧𝖧; H; H; ) MATHEMATICAL SANS-SERIF CAPITAL H +1D5A8;1D5A8;1D5A8;0049;0049; # (𝖨𝖨; 𝖨𝖨; 𝖨𝖨; I; I; ) MATHEMATICAL SANS-SERIF CAPITAL I +1D5A9;1D5A9;1D5A9;004A;004A; # (𝖩𝖩; 𝖩𝖩; 𝖩𝖩; J; J; ) MATHEMATICAL SANS-SERIF CAPITAL J +1D5AA;1D5AA;1D5AA;004B;004B; # (𝖪𝖪; 𝖪𝖪; 𝖪𝖪; K; K; ) MATHEMATICAL SANS-SERIF CAPITAL K +1D5AB;1D5AB;1D5AB;004C;004C; # (𝖫𝖫; 𝖫𝖫; 𝖫𝖫; L; L; ) MATHEMATICAL SANS-SERIF CAPITAL L +1D5AC;1D5AC;1D5AC;004D;004D; # (𝖬𝖬; 𝖬𝖬; 𝖬𝖬; M; M; ) MATHEMATICAL SANS-SERIF CAPITAL M +1D5AD;1D5AD;1D5AD;004E;004E; # (𝖭𝖭; 𝖭𝖭; 𝖭𝖭; N; N; ) MATHEMATICAL SANS-SERIF CAPITAL N +1D5AE;1D5AE;1D5AE;004F;004F; # (𝖮𝖮; 𝖮𝖮; 𝖮𝖮; O; O; ) MATHEMATICAL SANS-SERIF CAPITAL O +1D5AF;1D5AF;1D5AF;0050;0050; # (𝖯𝖯; 𝖯𝖯; 𝖯𝖯; P; P; ) MATHEMATICAL SANS-SERIF CAPITAL P +1D5B0;1D5B0;1D5B0;0051;0051; # (𝖰𝖰; 𝖰𝖰; 𝖰𝖰; Q; Q; ) MATHEMATICAL SANS-SERIF CAPITAL Q +1D5B1;1D5B1;1D5B1;0052;0052; # (𝖱𝖱; 𝖱𝖱; 𝖱𝖱; R; R; ) MATHEMATICAL SANS-SERIF CAPITAL R +1D5B2;1D5B2;1D5B2;0053;0053; # (𝖲𝖲; 𝖲𝖲; 𝖲𝖲; S; S; ) MATHEMATICAL SANS-SERIF CAPITAL S +1D5B3;1D5B3;1D5B3;0054;0054; # (𝖳𝖳; 𝖳𝖳; 𝖳𝖳; T; T; ) MATHEMATICAL SANS-SERIF CAPITAL T +1D5B4;1D5B4;1D5B4;0055;0055; # (𝖴𝖴; 𝖴𝖴; 𝖴𝖴; U; U; ) MATHEMATICAL SANS-SERIF CAPITAL U +1D5B5;1D5B5;1D5B5;0056;0056; # (𝖵𝖵; 𝖵𝖵; 𝖵𝖵; V; V; ) MATHEMATICAL SANS-SERIF CAPITAL V +1D5B6;1D5B6;1D5B6;0057;0057; # (𝖶𝖶; 𝖶𝖶; 𝖶𝖶; W; W; ) MATHEMATICAL SANS-SERIF CAPITAL W +1D5B7;1D5B7;1D5B7;0058;0058; # (𝖷𝖷; 𝖷𝖷; 𝖷𝖷; X; X; ) MATHEMATICAL SANS-SERIF CAPITAL X +1D5B8;1D5B8;1D5B8;0059;0059; # (𝖸𝖸; 𝖸𝖸; 𝖸𝖸; Y; Y; ) MATHEMATICAL SANS-SERIF CAPITAL Y +1D5B9;1D5B9;1D5B9;005A;005A; # (𝖹𝖹; 𝖹𝖹; 𝖹𝖹; Z; Z; ) MATHEMATICAL SANS-SERIF CAPITAL Z +1D5BA;1D5BA;1D5BA;0061;0061; # (𝖺𝖺; 𝖺𝖺; 𝖺𝖺; a; a; ) MATHEMATICAL SANS-SERIF SMALL A +1D5BB;1D5BB;1D5BB;0062;0062; # (𝖻𝖻; 𝖻𝖻; 𝖻𝖻; b; b; ) MATHEMATICAL SANS-SERIF SMALL B +1D5BC;1D5BC;1D5BC;0063;0063; # (𝖼𝖼; 𝖼𝖼; 𝖼𝖼; c; c; ) MATHEMATICAL SANS-SERIF SMALL C +1D5BD;1D5BD;1D5BD;0064;0064; # (𝖽𝖽; 𝖽𝖽; 𝖽𝖽; d; d; ) MATHEMATICAL SANS-SERIF SMALL D +1D5BE;1D5BE;1D5BE;0065;0065; # (𝖾𝖾; 𝖾𝖾; 𝖾𝖾; e; e; ) MATHEMATICAL SANS-SERIF SMALL E +1D5BF;1D5BF;1D5BF;0066;0066; # (𝖿𝖿; 𝖿𝖿; 𝖿𝖿; f; f; ) MATHEMATICAL SANS-SERIF SMALL F +1D5C0;1D5C0;1D5C0;0067;0067; # (𝗀𝗀; 𝗀𝗀; 𝗀𝗀; g; g; ) MATHEMATICAL SANS-SERIF SMALL G +1D5C1;1D5C1;1D5C1;0068;0068; # (𝗁𝗁; 𝗁𝗁; 𝗁𝗁; h; h; ) MATHEMATICAL SANS-SERIF SMALL H +1D5C2;1D5C2;1D5C2;0069;0069; # (𝗂𝗂; 𝗂𝗂; 𝗂𝗂; i; i; ) MATHEMATICAL SANS-SERIF SMALL I +1D5C3;1D5C3;1D5C3;006A;006A; # (𝗃𝗃; 𝗃𝗃; 𝗃𝗃; j; j; ) MATHEMATICAL SANS-SERIF SMALL J +1D5C4;1D5C4;1D5C4;006B;006B; # (𝗄𝗄; 𝗄𝗄; 𝗄𝗄; k; k; ) MATHEMATICAL SANS-SERIF SMALL K +1D5C5;1D5C5;1D5C5;006C;006C; # (𝗅𝗅; 𝗅𝗅; 𝗅𝗅; l; l; ) MATHEMATICAL SANS-SERIF SMALL L +1D5C6;1D5C6;1D5C6;006D;006D; # (𝗆𝗆; 𝗆𝗆; 𝗆𝗆; m; m; ) MATHEMATICAL SANS-SERIF SMALL M +1D5C7;1D5C7;1D5C7;006E;006E; # (𝗇𝗇; 𝗇𝗇; 𝗇𝗇; n; n; ) MATHEMATICAL SANS-SERIF SMALL N +1D5C8;1D5C8;1D5C8;006F;006F; # (𝗈𝗈; 𝗈𝗈; 𝗈𝗈; o; o; ) MATHEMATICAL SANS-SERIF SMALL O +1D5C9;1D5C9;1D5C9;0070;0070; # (𝗉𝗉; 𝗉𝗉; 𝗉𝗉; p; p; ) MATHEMATICAL SANS-SERIF SMALL P +1D5CA;1D5CA;1D5CA;0071;0071; # (𝗊𝗊; 𝗊𝗊; 𝗊𝗊; q; q; ) MATHEMATICAL SANS-SERIF SMALL Q +1D5CB;1D5CB;1D5CB;0072;0072; # (𝗋𝗋; 𝗋𝗋; 𝗋𝗋; r; r; ) MATHEMATICAL SANS-SERIF SMALL R +1D5CC;1D5CC;1D5CC;0073;0073; # (𝗌𝗌; 𝗌𝗌; 𝗌𝗌; s; s; ) MATHEMATICAL SANS-SERIF SMALL S +1D5CD;1D5CD;1D5CD;0074;0074; # (𝗍𝗍; 𝗍𝗍; 𝗍𝗍; t; t; ) MATHEMATICAL SANS-SERIF SMALL T +1D5CE;1D5CE;1D5CE;0075;0075; # (𝗎𝗎; 𝗎𝗎; 𝗎𝗎; u; u; ) MATHEMATICAL SANS-SERIF SMALL U +1D5CF;1D5CF;1D5CF;0076;0076; # (𝗏𝗏; 𝗏𝗏; 𝗏𝗏; v; v; ) MATHEMATICAL SANS-SERIF SMALL V +1D5D0;1D5D0;1D5D0;0077;0077; # (𝗐𝗐; 𝗐𝗐; 𝗐𝗐; w; w; ) MATHEMATICAL SANS-SERIF SMALL W +1D5D1;1D5D1;1D5D1;0078;0078; # (𝗑𝗑; 𝗑𝗑; 𝗑𝗑; x; x; ) MATHEMATICAL SANS-SERIF SMALL X +1D5D2;1D5D2;1D5D2;0079;0079; # (𝗒𝗒; 𝗒𝗒; 𝗒𝗒; y; y; ) MATHEMATICAL SANS-SERIF SMALL Y +1D5D3;1D5D3;1D5D3;007A;007A; # (𝗓𝗓; 𝗓𝗓; 𝗓𝗓; z; z; ) MATHEMATICAL SANS-SERIF SMALL Z +1D5D4;1D5D4;1D5D4;0041;0041; # (𝗔𝗔; 𝗔𝗔; 𝗔𝗔; A; A; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL A +1D5D5;1D5D5;1D5D5;0042;0042; # (𝗕𝗕; 𝗕𝗕; 𝗕𝗕; B; B; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL B +1D5D6;1D5D6;1D5D6;0043;0043; # (𝗖𝗖; 𝗖𝗖; 𝗖𝗖; C; C; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL C +1D5D7;1D5D7;1D5D7;0044;0044; # (𝗗𝗗; 𝗗𝗗; 𝗗𝗗; D; D; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL D +1D5D8;1D5D8;1D5D8;0045;0045; # (𝗘𝗘; 𝗘𝗘; 𝗘𝗘; E; E; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL E +1D5D9;1D5D9;1D5D9;0046;0046; # (𝗙𝗙; 𝗙𝗙; 𝗙𝗙; F; F; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL F +1D5DA;1D5DA;1D5DA;0047;0047; # (𝗚𝗚; 𝗚𝗚; 𝗚𝗚; G; G; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL G +1D5DB;1D5DB;1D5DB;0048;0048; # (𝗛𝗛; 𝗛𝗛; 𝗛𝗛; H; H; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL H +1D5DC;1D5DC;1D5DC;0049;0049; # (𝗜𝗜; 𝗜𝗜; 𝗜𝗜; I; I; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL I +1D5DD;1D5DD;1D5DD;004A;004A; # (𝗝𝗝; 𝗝𝗝; 𝗝𝗝; J; J; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL J +1D5DE;1D5DE;1D5DE;004B;004B; # (𝗞𝗞; 𝗞𝗞; 𝗞𝗞; K; K; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL K +1D5DF;1D5DF;1D5DF;004C;004C; # (𝗟𝗟; 𝗟𝗟; 𝗟𝗟; L; L; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL L +1D5E0;1D5E0;1D5E0;004D;004D; # (𝗠𝗠; 𝗠𝗠; 𝗠𝗠; M; M; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL M +1D5E1;1D5E1;1D5E1;004E;004E; # (𝗡𝗡; 𝗡𝗡; 𝗡𝗡; N; N; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL N +1D5E2;1D5E2;1D5E2;004F;004F; # (𝗢𝗢; 𝗢𝗢; 𝗢𝗢; O; O; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL O +1D5E3;1D5E3;1D5E3;0050;0050; # (𝗣𝗣; 𝗣𝗣; 𝗣𝗣; P; P; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL P +1D5E4;1D5E4;1D5E4;0051;0051; # (𝗤𝗤; 𝗤𝗤; 𝗤𝗤; Q; Q; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL Q +1D5E5;1D5E5;1D5E5;0052;0052; # (𝗥𝗥; 𝗥𝗥; 𝗥𝗥; R; R; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL R +1D5E6;1D5E6;1D5E6;0053;0053; # (𝗦𝗦; 𝗦𝗦; 𝗦𝗦; S; S; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL S +1D5E7;1D5E7;1D5E7;0054;0054; # (𝗧𝗧; 𝗧𝗧; 𝗧𝗧; T; T; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL T +1D5E8;1D5E8;1D5E8;0055;0055; # (𝗨𝗨; 𝗨𝗨; 𝗨𝗨; U; U; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL U +1D5E9;1D5E9;1D5E9;0056;0056; # (𝗩𝗩; 𝗩𝗩; 𝗩𝗩; V; V; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL V +1D5EA;1D5EA;1D5EA;0057;0057; # (𝗪𝗪; 𝗪𝗪; 𝗪𝗪; W; W; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL W +1D5EB;1D5EB;1D5EB;0058;0058; # (𝗫𝗫; 𝗫𝗫; 𝗫𝗫; X; X; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL X +1D5EC;1D5EC;1D5EC;0059;0059; # (𝗬𝗬; 𝗬𝗬; 𝗬𝗬; Y; Y; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL Y +1D5ED;1D5ED;1D5ED;005A;005A; # (𝗭𝗭; 𝗭𝗭; 𝗭𝗭; Z; Z; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL Z +1D5EE;1D5EE;1D5EE;0061;0061; # (𝗮𝗮; 𝗮𝗮; 𝗮𝗮; a; a; ) MATHEMATICAL SANS-SERIF BOLD SMALL A +1D5EF;1D5EF;1D5EF;0062;0062; # (𝗯𝗯; 𝗯𝗯; 𝗯𝗯; b; b; ) MATHEMATICAL SANS-SERIF BOLD SMALL B +1D5F0;1D5F0;1D5F0;0063;0063; # (𝗰𝗰; 𝗰𝗰; 𝗰𝗰; c; c; ) MATHEMATICAL SANS-SERIF BOLD SMALL C +1D5F1;1D5F1;1D5F1;0064;0064; # (𝗱𝗱; 𝗱𝗱; 𝗱𝗱; d; d; ) MATHEMATICAL SANS-SERIF BOLD SMALL D +1D5F2;1D5F2;1D5F2;0065;0065; # (𝗲𝗲; 𝗲𝗲; 𝗲𝗲; e; e; ) MATHEMATICAL SANS-SERIF BOLD SMALL E +1D5F3;1D5F3;1D5F3;0066;0066; # (𝗳𝗳; 𝗳𝗳; 𝗳𝗳; f; f; ) MATHEMATICAL SANS-SERIF BOLD SMALL F +1D5F4;1D5F4;1D5F4;0067;0067; # (𝗴𝗴; 𝗴𝗴; 𝗴𝗴; g; g; ) MATHEMATICAL SANS-SERIF BOLD SMALL G +1D5F5;1D5F5;1D5F5;0068;0068; # (𝗵𝗵; 𝗵𝗵; 𝗵𝗵; h; h; ) MATHEMATICAL SANS-SERIF BOLD SMALL H +1D5F6;1D5F6;1D5F6;0069;0069; # (𝗶𝗶; 𝗶𝗶; 𝗶𝗶; i; i; ) MATHEMATICAL SANS-SERIF BOLD SMALL I +1D5F7;1D5F7;1D5F7;006A;006A; # (𝗷𝗷; 𝗷𝗷; 𝗷𝗷; j; j; ) MATHEMATICAL SANS-SERIF BOLD SMALL J +1D5F8;1D5F8;1D5F8;006B;006B; # (𝗸𝗸; 𝗸𝗸; 𝗸𝗸; k; k; ) MATHEMATICAL SANS-SERIF BOLD SMALL K +1D5F9;1D5F9;1D5F9;006C;006C; # (𝗹𝗹; 𝗹𝗹; 𝗹𝗹; l; l; ) MATHEMATICAL SANS-SERIF BOLD SMALL L +1D5FA;1D5FA;1D5FA;006D;006D; # (𝗺𝗺; 𝗺𝗺; 𝗺𝗺; m; m; ) MATHEMATICAL SANS-SERIF BOLD SMALL M +1D5FB;1D5FB;1D5FB;006E;006E; # (𝗻𝗻; 𝗻𝗻; 𝗻𝗻; n; n; ) MATHEMATICAL SANS-SERIF BOLD SMALL N +1D5FC;1D5FC;1D5FC;006F;006F; # (𝗼𝗼; 𝗼𝗼; 𝗼𝗼; o; o; ) MATHEMATICAL SANS-SERIF BOLD SMALL O +1D5FD;1D5FD;1D5FD;0070;0070; # (𝗽𝗽; 𝗽𝗽; 𝗽𝗽; p; p; ) MATHEMATICAL SANS-SERIF BOLD SMALL P +1D5FE;1D5FE;1D5FE;0071;0071; # (𝗾𝗾; 𝗾𝗾; 𝗾𝗾; q; q; ) MATHEMATICAL SANS-SERIF BOLD SMALL Q +1D5FF;1D5FF;1D5FF;0072;0072; # (𝗿𝗿; 𝗿𝗿; 𝗿𝗿; r; r; ) MATHEMATICAL SANS-SERIF BOLD SMALL R +1D600;1D600;1D600;0073;0073; # (𝘀𝘀; 𝘀𝘀; 𝘀𝘀; s; s; ) MATHEMATICAL SANS-SERIF BOLD SMALL S +1D601;1D601;1D601;0074;0074; # (𝘁𝘁; 𝘁𝘁; 𝘁𝘁; t; t; ) MATHEMATICAL SANS-SERIF BOLD SMALL T +1D602;1D602;1D602;0075;0075; # (𝘂𝘂; 𝘂𝘂; 𝘂𝘂; u; u; ) MATHEMATICAL SANS-SERIF BOLD SMALL U +1D603;1D603;1D603;0076;0076; # (𝘃𝘃; 𝘃𝘃; 𝘃𝘃; v; v; ) MATHEMATICAL SANS-SERIF BOLD SMALL V +1D604;1D604;1D604;0077;0077; # (𝘄𝘄; 𝘄𝘄; 𝘄𝘄; w; w; ) MATHEMATICAL SANS-SERIF BOLD SMALL W +1D605;1D605;1D605;0078;0078; # (𝘅𝘅; 𝘅𝘅; 𝘅𝘅; x; x; ) MATHEMATICAL SANS-SERIF BOLD SMALL X +1D606;1D606;1D606;0079;0079; # (𝘆𝘆; 𝘆𝘆; 𝘆𝘆; y; y; ) MATHEMATICAL SANS-SERIF BOLD SMALL Y +1D607;1D607;1D607;007A;007A; # (𝘇𝘇; 𝘇𝘇; 𝘇𝘇; z; z; ) MATHEMATICAL SANS-SERIF BOLD SMALL Z +1D608;1D608;1D608;0041;0041; # (𝘈𝘈; 𝘈𝘈; 𝘈𝘈; A; A; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL A +1D609;1D609;1D609;0042;0042; # (𝘉𝘉; 𝘉𝘉; 𝘉𝘉; B; B; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL B +1D60A;1D60A;1D60A;0043;0043; # (𝘊𝘊; 𝘊𝘊; 𝘊𝘊; C; C; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL C +1D60B;1D60B;1D60B;0044;0044; # (𝘋𝘋; 𝘋𝘋; 𝘋𝘋; D; D; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL D +1D60C;1D60C;1D60C;0045;0045; # (𝘌𝘌; 𝘌𝘌; 𝘌𝘌; E; E; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL E +1D60D;1D60D;1D60D;0046;0046; # (𝘍𝘍; 𝘍𝘍; 𝘍𝘍; F; F; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL F +1D60E;1D60E;1D60E;0047;0047; # (𝘎𝘎; 𝘎𝘎; 𝘎𝘎; G; G; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL G +1D60F;1D60F;1D60F;0048;0048; # (𝘏𝘏; 𝘏𝘏; 𝘏𝘏; H; H; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL H +1D610;1D610;1D610;0049;0049; # (𝘐𝘐; 𝘐𝘐; 𝘐𝘐; I; I; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL I +1D611;1D611;1D611;004A;004A; # (𝘑𝘑; 𝘑𝘑; 𝘑𝘑; J; J; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL J +1D612;1D612;1D612;004B;004B; # (𝘒𝘒; 𝘒𝘒; 𝘒𝘒; K; K; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL K +1D613;1D613;1D613;004C;004C; # (𝘓𝘓; 𝘓𝘓; 𝘓𝘓; L; L; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL L +1D614;1D614;1D614;004D;004D; # (𝘔𝘔; 𝘔𝘔; 𝘔𝘔; M; M; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL M +1D615;1D615;1D615;004E;004E; # (𝘕𝘕; 𝘕𝘕; 𝘕𝘕; N; N; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL N +1D616;1D616;1D616;004F;004F; # (𝘖𝘖; 𝘖𝘖; 𝘖𝘖; O; O; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL O +1D617;1D617;1D617;0050;0050; # (𝘗𝘗; 𝘗𝘗; 𝘗𝘗; P; P; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL P +1D618;1D618;1D618;0051;0051; # (𝘘𝘘; 𝘘𝘘; 𝘘𝘘; Q; Q; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL Q +1D619;1D619;1D619;0052;0052; # (𝘙𝘙; 𝘙𝘙; 𝘙𝘙; R; R; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL R +1D61A;1D61A;1D61A;0053;0053; # (𝘚𝘚; 𝘚𝘚; 𝘚𝘚; S; S; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL S +1D61B;1D61B;1D61B;0054;0054; # (𝘛𝘛; 𝘛𝘛; 𝘛𝘛; T; T; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL T +1D61C;1D61C;1D61C;0055;0055; # (𝘜𝘜; 𝘜𝘜; 𝘜𝘜; U; U; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL U +1D61D;1D61D;1D61D;0056;0056; # (𝘝𝘝; 𝘝𝘝; 𝘝𝘝; V; V; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL V +1D61E;1D61E;1D61E;0057;0057; # (𝘞𝘞; 𝘞𝘞; 𝘞𝘞; W; W; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL W +1D61F;1D61F;1D61F;0058;0058; # (𝘟𝘟; 𝘟𝘟; 𝘟𝘟; X; X; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL X +1D620;1D620;1D620;0059;0059; # (𝘠𝘠; 𝘠𝘠; 𝘠𝘠; Y; Y; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL Y +1D621;1D621;1D621;005A;005A; # (𝘡𝘡; 𝘡𝘡; 𝘡𝘡; Z; Z; ) MATHEMATICAL SANS-SERIF ITALIC CAPITAL Z +1D622;1D622;1D622;0061;0061; # (𝘢𝘢; 𝘢𝘢; 𝘢𝘢; a; a; ) MATHEMATICAL SANS-SERIF ITALIC SMALL A +1D623;1D623;1D623;0062;0062; # (𝘣𝘣; 𝘣𝘣; 𝘣𝘣; b; b; ) MATHEMATICAL SANS-SERIF ITALIC SMALL B +1D624;1D624;1D624;0063;0063; # (𝘤𝘤; 𝘤𝘤; 𝘤𝘤; c; c; ) MATHEMATICAL SANS-SERIF ITALIC SMALL C +1D625;1D625;1D625;0064;0064; # (𝘥𝘥; 𝘥𝘥; 𝘥𝘥; d; d; ) MATHEMATICAL SANS-SERIF ITALIC SMALL D +1D626;1D626;1D626;0065;0065; # (𝘦𝘦; 𝘦𝘦; 𝘦𝘦; e; e; ) MATHEMATICAL SANS-SERIF ITALIC SMALL E +1D627;1D627;1D627;0066;0066; # (𝘧𝘧; 𝘧𝘧; 𝘧𝘧; f; f; ) MATHEMATICAL SANS-SERIF ITALIC SMALL F +1D628;1D628;1D628;0067;0067; # (𝘨𝘨; 𝘨𝘨; 𝘨𝘨; g; g; ) MATHEMATICAL SANS-SERIF ITALIC SMALL G +1D629;1D629;1D629;0068;0068; # (𝘩𝘩; 𝘩𝘩; 𝘩𝘩; h; h; ) MATHEMATICAL SANS-SERIF ITALIC SMALL H +1D62A;1D62A;1D62A;0069;0069; # (𝘪𝘪; 𝘪𝘪; 𝘪𝘪; i; i; ) MATHEMATICAL SANS-SERIF ITALIC SMALL I +1D62B;1D62B;1D62B;006A;006A; # (𝘫𝘫; 𝘫𝘫; 𝘫𝘫; j; j; ) MATHEMATICAL SANS-SERIF ITALIC SMALL J +1D62C;1D62C;1D62C;006B;006B; # (𝘬𝘬; 𝘬𝘬; 𝘬𝘬; k; k; ) MATHEMATICAL SANS-SERIF ITALIC SMALL K +1D62D;1D62D;1D62D;006C;006C; # (𝘭𝘭; 𝘭𝘭; 𝘭𝘭; l; l; ) MATHEMATICAL SANS-SERIF ITALIC SMALL L +1D62E;1D62E;1D62E;006D;006D; # (𝘮𝘮; 𝘮𝘮; 𝘮𝘮; m; m; ) MATHEMATICAL SANS-SERIF ITALIC SMALL M +1D62F;1D62F;1D62F;006E;006E; # (𝘯𝘯; 𝘯𝘯; 𝘯𝘯; n; n; ) MATHEMATICAL SANS-SERIF ITALIC SMALL N +1D630;1D630;1D630;006F;006F; # (𝘰𝘰; 𝘰𝘰; 𝘰𝘰; o; o; ) MATHEMATICAL SANS-SERIF ITALIC SMALL O +1D631;1D631;1D631;0070;0070; # (𝘱𝘱; 𝘱𝘱; 𝘱𝘱; p; p; ) MATHEMATICAL SANS-SERIF ITALIC SMALL P +1D632;1D632;1D632;0071;0071; # (𝘲𝘲; 𝘲𝘲; 𝘲𝘲; q; q; ) MATHEMATICAL SANS-SERIF ITALIC SMALL Q +1D633;1D633;1D633;0072;0072; # (𝘳𝘳; 𝘳𝘳; 𝘳𝘳; r; r; ) MATHEMATICAL SANS-SERIF ITALIC SMALL R +1D634;1D634;1D634;0073;0073; # (𝘴𝘴; 𝘴𝘴; 𝘴𝘴; s; s; ) MATHEMATICAL SANS-SERIF ITALIC SMALL S +1D635;1D635;1D635;0074;0074; # (𝘵𝘵; 𝘵𝘵; 𝘵𝘵; t; t; ) MATHEMATICAL SANS-SERIF ITALIC SMALL T +1D636;1D636;1D636;0075;0075; # (𝘶𝘶; 𝘶𝘶; 𝘶𝘶; u; u; ) MATHEMATICAL SANS-SERIF ITALIC SMALL U +1D637;1D637;1D637;0076;0076; # (𝘷𝘷; 𝘷𝘷; 𝘷𝘷; v; v; ) MATHEMATICAL SANS-SERIF ITALIC SMALL V +1D638;1D638;1D638;0077;0077; # (𝘸𝘸; 𝘸𝘸; 𝘸𝘸; w; w; ) MATHEMATICAL SANS-SERIF ITALIC SMALL W +1D639;1D639;1D639;0078;0078; # (𝘹𝘹; 𝘹𝘹; 𝘹𝘹; x; x; ) MATHEMATICAL SANS-SERIF ITALIC SMALL X +1D63A;1D63A;1D63A;0079;0079; # (𝘺𝘺; 𝘺𝘺; 𝘺𝘺; y; y; ) MATHEMATICAL SANS-SERIF ITALIC SMALL Y +1D63B;1D63B;1D63B;007A;007A; # (𝘻𝘻; 𝘻𝘻; 𝘻𝘻; z; z; ) MATHEMATICAL SANS-SERIF ITALIC SMALL Z +1D63C;1D63C;1D63C;0041;0041; # (𝘼𝘼; 𝘼𝘼; 𝘼𝘼; A; A; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL A +1D63D;1D63D;1D63D;0042;0042; # (𝘽𝘽; 𝘽𝘽; 𝘽𝘽; B; B; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL B +1D63E;1D63E;1D63E;0043;0043; # (𝘾𝘾; 𝘾𝘾; 𝘾𝘾; C; C; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL C +1D63F;1D63F;1D63F;0044;0044; # (𝘿𝘿; 𝘿𝘿; 𝘿𝘿; D; D; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL D +1D640;1D640;1D640;0045;0045; # (𝙀𝙀; 𝙀𝙀; 𝙀𝙀; E; E; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL E +1D641;1D641;1D641;0046;0046; # (𝙁𝙁; 𝙁𝙁; 𝙁𝙁; F; F; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL F +1D642;1D642;1D642;0047;0047; # (𝙂𝙂; 𝙂𝙂; 𝙂𝙂; G; G; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL G +1D643;1D643;1D643;0048;0048; # (𝙃𝙃; 𝙃𝙃; 𝙃𝙃; H; H; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL H +1D644;1D644;1D644;0049;0049; # (𝙄𝙄; 𝙄𝙄; 𝙄𝙄; I; I; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL I +1D645;1D645;1D645;004A;004A; # (𝙅𝙅; 𝙅𝙅; 𝙅𝙅; J; J; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL J +1D646;1D646;1D646;004B;004B; # (𝙆𝙆; 𝙆𝙆; 𝙆𝙆; K; K; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL K +1D647;1D647;1D647;004C;004C; # (𝙇𝙇; 𝙇𝙇; 𝙇𝙇; L; L; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL L +1D648;1D648;1D648;004D;004D; # (𝙈𝙈; 𝙈𝙈; 𝙈𝙈; M; M; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL M +1D649;1D649;1D649;004E;004E; # (𝙉𝙉; 𝙉𝙉; 𝙉𝙉; N; N; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL N +1D64A;1D64A;1D64A;004F;004F; # (𝙊𝙊; 𝙊𝙊; 𝙊𝙊; O; O; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL O +1D64B;1D64B;1D64B;0050;0050; # (𝙋𝙋; 𝙋𝙋; 𝙋𝙋; P; P; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL P +1D64C;1D64C;1D64C;0051;0051; # (𝙌𝙌; 𝙌𝙌; 𝙌𝙌; Q; Q; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL Q +1D64D;1D64D;1D64D;0052;0052; # (𝙍𝙍; 𝙍𝙍; 𝙍𝙍; R; R; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL R +1D64E;1D64E;1D64E;0053;0053; # (𝙎𝙎; 𝙎𝙎; 𝙎𝙎; S; S; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL S +1D64F;1D64F;1D64F;0054;0054; # (𝙏𝙏; 𝙏𝙏; 𝙏𝙏; T; T; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL T +1D650;1D650;1D650;0055;0055; # (𝙐𝙐; 𝙐𝙐; 𝙐𝙐; U; U; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL U +1D651;1D651;1D651;0056;0056; # (𝙑𝙑; 𝙑𝙑; 𝙑𝙑; V; V; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL V +1D652;1D652;1D652;0057;0057; # (𝙒𝙒; 𝙒𝙒; 𝙒𝙒; W; W; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL W +1D653;1D653;1D653;0058;0058; # (𝙓𝙓; 𝙓𝙓; 𝙓𝙓; X; X; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL X +1D654;1D654;1D654;0059;0059; # (𝙔𝙔; 𝙔𝙔; 𝙔𝙔; Y; Y; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL Y +1D655;1D655;1D655;005A;005A; # (𝙕𝙕; 𝙕𝙕; 𝙕𝙕; Z; Z; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL Z +1D656;1D656;1D656;0061;0061; # (𝙖𝙖; 𝙖𝙖; 𝙖𝙖; a; a; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL A +1D657;1D657;1D657;0062;0062; # (𝙗𝙗; 𝙗𝙗; 𝙗𝙗; b; b; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL B +1D658;1D658;1D658;0063;0063; # (𝙘𝙘; 𝙘𝙘; 𝙘𝙘; c; c; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL C +1D659;1D659;1D659;0064;0064; # (𝙙𝙙; 𝙙𝙙; 𝙙𝙙; d; d; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL D +1D65A;1D65A;1D65A;0065;0065; # (𝙚𝙚; 𝙚𝙚; 𝙚𝙚; e; e; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL E +1D65B;1D65B;1D65B;0066;0066; # (𝙛𝙛; 𝙛𝙛; 𝙛𝙛; f; f; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL F +1D65C;1D65C;1D65C;0067;0067; # (𝙜𝙜; 𝙜𝙜; 𝙜𝙜; g; g; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL G +1D65D;1D65D;1D65D;0068;0068; # (𝙝𝙝; 𝙝𝙝; 𝙝𝙝; h; h; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL H +1D65E;1D65E;1D65E;0069;0069; # (𝙞𝙞; 𝙞𝙞; 𝙞𝙞; i; i; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL I +1D65F;1D65F;1D65F;006A;006A; # (𝙟𝙟; 𝙟𝙟; 𝙟𝙟; j; j; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL J +1D660;1D660;1D660;006B;006B; # (𝙠𝙠; 𝙠𝙠; 𝙠𝙠; k; k; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL K +1D661;1D661;1D661;006C;006C; # (𝙡𝙡; 𝙡𝙡; 𝙡𝙡; l; l; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL L +1D662;1D662;1D662;006D;006D; # (𝙢𝙢; 𝙢𝙢; 𝙢𝙢; m; m; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL M +1D663;1D663;1D663;006E;006E; # (𝙣𝙣; 𝙣𝙣; 𝙣𝙣; n; n; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL N +1D664;1D664;1D664;006F;006F; # (𝙤𝙤; 𝙤𝙤; 𝙤𝙤; o; o; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL O +1D665;1D665;1D665;0070;0070; # (𝙥𝙥; 𝙥𝙥; 𝙥𝙥; p; p; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL P +1D666;1D666;1D666;0071;0071; # (𝙦𝙦; 𝙦𝙦; 𝙦𝙦; q; q; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL Q +1D667;1D667;1D667;0072;0072; # (𝙧𝙧; 𝙧𝙧; 𝙧𝙧; r; r; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL R +1D668;1D668;1D668;0073;0073; # (𝙨𝙨; 𝙨𝙨; 𝙨𝙨; s; s; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL S +1D669;1D669;1D669;0074;0074; # (𝙩𝙩; 𝙩𝙩; 𝙩𝙩; t; t; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL T +1D66A;1D66A;1D66A;0075;0075; # (𝙪𝙪; 𝙪𝙪; 𝙪𝙪; u; u; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL U +1D66B;1D66B;1D66B;0076;0076; # (𝙫𝙫; 𝙫𝙫; 𝙫𝙫; v; v; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL V +1D66C;1D66C;1D66C;0077;0077; # (𝙬𝙬; 𝙬𝙬; 𝙬𝙬; w; w; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL W +1D66D;1D66D;1D66D;0078;0078; # (𝙭𝙭; 𝙭𝙭; 𝙭𝙭; x; x; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL X +1D66E;1D66E;1D66E;0079;0079; # (𝙮𝙮; 𝙮𝙮; 𝙮𝙮; y; y; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL Y +1D66F;1D66F;1D66F;007A;007A; # (𝙯𝙯; 𝙯𝙯; 𝙯𝙯; z; z; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL Z +1D670;1D670;1D670;0041;0041; # (𝙰𝙰; 𝙰𝙰; 𝙰𝙰; A; A; ) MATHEMATICAL MONOSPACE CAPITAL A +1D671;1D671;1D671;0042;0042; # (𝙱𝙱; 𝙱𝙱; 𝙱𝙱; B; B; ) MATHEMATICAL MONOSPACE CAPITAL B +1D672;1D672;1D672;0043;0043; # (𝙲𝙲; 𝙲𝙲; 𝙲𝙲; C; C; ) MATHEMATICAL MONOSPACE CAPITAL C +1D673;1D673;1D673;0044;0044; # (𝙳𝙳; 𝙳𝙳; 𝙳𝙳; D; D; ) MATHEMATICAL MONOSPACE CAPITAL D +1D674;1D674;1D674;0045;0045; # (𝙴𝙴; 𝙴𝙴; 𝙴𝙴; E; E; ) MATHEMATICAL MONOSPACE CAPITAL E +1D675;1D675;1D675;0046;0046; # (𝙵𝙵; 𝙵𝙵; 𝙵𝙵; F; F; ) MATHEMATICAL MONOSPACE CAPITAL F +1D676;1D676;1D676;0047;0047; # (𝙶𝙶; 𝙶𝙶; 𝙶𝙶; G; G; ) MATHEMATICAL MONOSPACE CAPITAL G +1D677;1D677;1D677;0048;0048; # (𝙷𝙷; 𝙷𝙷; 𝙷𝙷; H; H; ) MATHEMATICAL MONOSPACE CAPITAL H +1D678;1D678;1D678;0049;0049; # (𝙸𝙸; 𝙸𝙸; 𝙸𝙸; I; I; ) MATHEMATICAL MONOSPACE CAPITAL I +1D679;1D679;1D679;004A;004A; # (𝙹𝙹; 𝙹𝙹; 𝙹𝙹; J; J; ) MATHEMATICAL MONOSPACE CAPITAL J +1D67A;1D67A;1D67A;004B;004B; # (𝙺𝙺; 𝙺𝙺; 𝙺𝙺; K; K; ) MATHEMATICAL MONOSPACE CAPITAL K +1D67B;1D67B;1D67B;004C;004C; # (𝙻𝙻; 𝙻𝙻; 𝙻𝙻; L; L; ) MATHEMATICAL MONOSPACE CAPITAL L +1D67C;1D67C;1D67C;004D;004D; # (𝙼𝙼; 𝙼𝙼; 𝙼𝙼; M; M; ) MATHEMATICAL MONOSPACE CAPITAL M +1D67D;1D67D;1D67D;004E;004E; # (𝙽𝙽; 𝙽𝙽; 𝙽𝙽; N; N; ) MATHEMATICAL MONOSPACE CAPITAL N +1D67E;1D67E;1D67E;004F;004F; # (𝙾𝙾; 𝙾𝙾; 𝙾𝙾; O; O; ) MATHEMATICAL MONOSPACE CAPITAL O +1D67F;1D67F;1D67F;0050;0050; # (𝙿𝙿; 𝙿𝙿; 𝙿𝙿; P; P; ) MATHEMATICAL MONOSPACE CAPITAL P +1D680;1D680;1D680;0051;0051; # (𝚀𝚀; 𝚀𝚀; 𝚀𝚀; Q; Q; ) MATHEMATICAL MONOSPACE CAPITAL Q +1D681;1D681;1D681;0052;0052; # (𝚁𝚁; 𝚁𝚁; 𝚁𝚁; R; R; ) MATHEMATICAL MONOSPACE CAPITAL R +1D682;1D682;1D682;0053;0053; # (𝚂𝚂; 𝚂𝚂; 𝚂𝚂; S; S; ) MATHEMATICAL MONOSPACE CAPITAL S +1D683;1D683;1D683;0054;0054; # (𝚃𝚃; 𝚃𝚃; 𝚃𝚃; T; T; ) MATHEMATICAL MONOSPACE CAPITAL T +1D684;1D684;1D684;0055;0055; # (𝚄𝚄; 𝚄𝚄; 𝚄𝚄; U; U; ) MATHEMATICAL MONOSPACE CAPITAL U +1D685;1D685;1D685;0056;0056; # (𝚅𝚅; 𝚅𝚅; 𝚅𝚅; V; V; ) MATHEMATICAL MONOSPACE CAPITAL V +1D686;1D686;1D686;0057;0057; # (𝚆𝚆; 𝚆𝚆; 𝚆𝚆; W; W; ) MATHEMATICAL MONOSPACE CAPITAL W +1D687;1D687;1D687;0058;0058; # (𝚇𝚇; 𝚇𝚇; 𝚇𝚇; X; X; ) MATHEMATICAL MONOSPACE CAPITAL X +1D688;1D688;1D688;0059;0059; # (𝚈𝚈; 𝚈𝚈; 𝚈𝚈; Y; Y; ) MATHEMATICAL MONOSPACE CAPITAL Y +1D689;1D689;1D689;005A;005A; # (𝚉𝚉; 𝚉𝚉; 𝚉𝚉; Z; Z; ) MATHEMATICAL MONOSPACE CAPITAL Z +1D68A;1D68A;1D68A;0061;0061; # (𝚊𝚊; 𝚊𝚊; 𝚊𝚊; a; a; ) MATHEMATICAL MONOSPACE SMALL A +1D68B;1D68B;1D68B;0062;0062; # (𝚋𝚋; 𝚋𝚋; 𝚋𝚋; b; b; ) MATHEMATICAL MONOSPACE SMALL B +1D68C;1D68C;1D68C;0063;0063; # (𝚌𝚌; 𝚌𝚌; 𝚌𝚌; c; c; ) MATHEMATICAL MONOSPACE SMALL C +1D68D;1D68D;1D68D;0064;0064; # (𝚍𝚍; 𝚍𝚍; 𝚍𝚍; d; d; ) MATHEMATICAL MONOSPACE SMALL D +1D68E;1D68E;1D68E;0065;0065; # (𝚎𝚎; 𝚎𝚎; 𝚎𝚎; e; e; ) MATHEMATICAL MONOSPACE SMALL E +1D68F;1D68F;1D68F;0066;0066; # (𝚏𝚏; 𝚏𝚏; 𝚏𝚏; f; f; ) MATHEMATICAL MONOSPACE SMALL F +1D690;1D690;1D690;0067;0067; # (𝚐𝚐; 𝚐𝚐; 𝚐𝚐; g; g; ) MATHEMATICAL MONOSPACE SMALL G +1D691;1D691;1D691;0068;0068; # (𝚑𝚑; 𝚑𝚑; 𝚑𝚑; h; h; ) MATHEMATICAL MONOSPACE SMALL H +1D692;1D692;1D692;0069;0069; # (𝚒𝚒; 𝚒𝚒; 𝚒𝚒; i; i; ) MATHEMATICAL MONOSPACE SMALL I +1D693;1D693;1D693;006A;006A; # (𝚓𝚓; 𝚓𝚓; 𝚓𝚓; j; j; ) MATHEMATICAL MONOSPACE SMALL J +1D694;1D694;1D694;006B;006B; # (𝚔𝚔; 𝚔𝚔; 𝚔𝚔; k; k; ) MATHEMATICAL MONOSPACE SMALL K +1D695;1D695;1D695;006C;006C; # (𝚕𝚕; 𝚕𝚕; 𝚕𝚕; l; l; ) MATHEMATICAL MONOSPACE SMALL L +1D696;1D696;1D696;006D;006D; # (𝚖𝚖; 𝚖𝚖; 𝚖𝚖; m; m; ) MATHEMATICAL MONOSPACE SMALL M +1D697;1D697;1D697;006E;006E; # (𝚗𝚗; 𝚗𝚗; 𝚗𝚗; n; n; ) MATHEMATICAL MONOSPACE SMALL N +1D698;1D698;1D698;006F;006F; # (𝚘𝚘; 𝚘𝚘; 𝚘𝚘; o; o; ) MATHEMATICAL MONOSPACE SMALL O +1D699;1D699;1D699;0070;0070; # (𝚙𝚙; 𝚙𝚙; 𝚙𝚙; p; p; ) MATHEMATICAL MONOSPACE SMALL P +1D69A;1D69A;1D69A;0071;0071; # (𝚚𝚚; 𝚚𝚚; 𝚚𝚚; q; q; ) MATHEMATICAL MONOSPACE SMALL Q +1D69B;1D69B;1D69B;0072;0072; # (𝚛𝚛; 𝚛𝚛; 𝚛𝚛; r; r; ) MATHEMATICAL MONOSPACE SMALL R +1D69C;1D69C;1D69C;0073;0073; # (𝚜𝚜; 𝚜𝚜; 𝚜𝚜; s; s; ) MATHEMATICAL MONOSPACE SMALL S +1D69D;1D69D;1D69D;0074;0074; # (𝚝𝚝; 𝚝𝚝; 𝚝𝚝; t; t; ) MATHEMATICAL MONOSPACE SMALL T +1D69E;1D69E;1D69E;0075;0075; # (𝚞𝚞; 𝚞𝚞; 𝚞𝚞; u; u; ) MATHEMATICAL MONOSPACE SMALL U +1D69F;1D69F;1D69F;0076;0076; # (𝚟𝚟; 𝚟𝚟; 𝚟𝚟; v; v; ) MATHEMATICAL MONOSPACE SMALL V +1D6A0;1D6A0;1D6A0;0077;0077; # (𝚠𝚠; 𝚠𝚠; 𝚠𝚠; w; w; ) MATHEMATICAL MONOSPACE SMALL W +1D6A1;1D6A1;1D6A1;0078;0078; # (𝚡𝚡; 𝚡𝚡; 𝚡𝚡; x; x; ) MATHEMATICAL MONOSPACE SMALL X +1D6A2;1D6A2;1D6A2;0079;0079; # (𝚢𝚢; 𝚢𝚢; 𝚢𝚢; y; y; ) MATHEMATICAL MONOSPACE SMALL Y +1D6A3;1D6A3;1D6A3;007A;007A; # (𝚣𝚣; 𝚣𝚣; 𝚣𝚣; z; z; ) MATHEMATICAL MONOSPACE SMALL Z +1D6A8;1D6A8;1D6A8;0391;0391; # (𝚨𝚨; 𝚨𝚨; 𝚨𝚨; Α; Α; ) MATHEMATICAL BOLD CAPITAL ALPHA +1D6A9;1D6A9;1D6A9;0392;0392; # (𝚩𝚩; 𝚩𝚩; 𝚩𝚩; Β; Β; ) MATHEMATICAL BOLD CAPITAL BETA +1D6AA;1D6AA;1D6AA;0393;0393; # (𝚪𝚪; 𝚪𝚪; 𝚪𝚪; Γ; Γ; ) MATHEMATICAL BOLD CAPITAL GAMMA +1D6AB;1D6AB;1D6AB;0394;0394; # (𝚫𝚫; 𝚫𝚫; 𝚫𝚫; Δ; Δ; ) MATHEMATICAL BOLD CAPITAL DELTA +1D6AC;1D6AC;1D6AC;0395;0395; # (𝚬𝚬; 𝚬𝚬; 𝚬𝚬; Ε; Ε; ) MATHEMATICAL BOLD CAPITAL EPSILON +1D6AD;1D6AD;1D6AD;0396;0396; # (𝚭𝚭; 𝚭𝚭; 𝚭𝚭; Ζ; Ζ; ) MATHEMATICAL BOLD CAPITAL ZETA +1D6AE;1D6AE;1D6AE;0397;0397; # (𝚮𝚮; 𝚮𝚮; 𝚮𝚮; Η; Η; ) MATHEMATICAL BOLD CAPITAL ETA +1D6AF;1D6AF;1D6AF;0398;0398; # (𝚯𝚯; 𝚯𝚯; 𝚯𝚯; Θ; Θ; ) MATHEMATICAL BOLD CAPITAL THETA +1D6B0;1D6B0;1D6B0;0399;0399; # (𝚰𝚰; 𝚰𝚰; 𝚰𝚰; Ι; Ι; ) MATHEMATICAL BOLD CAPITAL IOTA +1D6B1;1D6B1;1D6B1;039A;039A; # (𝚱𝚱; 𝚱𝚱; 𝚱𝚱; Κ; Κ; ) MATHEMATICAL BOLD CAPITAL KAPPA +1D6B2;1D6B2;1D6B2;039B;039B; # (𝚲𝚲; 𝚲𝚲; 𝚲𝚲; Λ; Λ; ) MATHEMATICAL BOLD CAPITAL LAMDA +1D6B3;1D6B3;1D6B3;039C;039C; # (𝚳𝚳; 𝚳𝚳; 𝚳𝚳; Μ; Μ; ) MATHEMATICAL BOLD CAPITAL MU +1D6B4;1D6B4;1D6B4;039D;039D; # (𝚴𝚴; 𝚴𝚴; 𝚴𝚴; Ν; Ν; ) MATHEMATICAL BOLD CAPITAL NU +1D6B5;1D6B5;1D6B5;039E;039E; # (𝚵𝚵; 𝚵𝚵; 𝚵𝚵; Ξ; Ξ; ) MATHEMATICAL BOLD CAPITAL XI +1D6B6;1D6B6;1D6B6;039F;039F; # (𝚶𝚶; 𝚶𝚶; 𝚶𝚶; Ο; Ο; ) MATHEMATICAL BOLD CAPITAL OMICRON +1D6B7;1D6B7;1D6B7;03A0;03A0; # (𝚷𝚷; 𝚷𝚷; 𝚷𝚷; Π; Π; ) MATHEMATICAL BOLD CAPITAL PI +1D6B8;1D6B8;1D6B8;03A1;03A1; # (𝚸𝚸; 𝚸𝚸; 𝚸𝚸; Ρ; Ρ; ) MATHEMATICAL BOLD CAPITAL RHO +1D6B9;1D6B9;1D6B9;0398;0398; # (𝚹𝚹; 𝚹𝚹; 𝚹𝚹; Θ; Θ; ) MATHEMATICAL BOLD CAPITAL THETA SYMBOL +1D6BA;1D6BA;1D6BA;03A3;03A3; # (𝚺𝚺; 𝚺𝚺; 𝚺𝚺; Σ; Σ; ) MATHEMATICAL BOLD CAPITAL SIGMA +1D6BB;1D6BB;1D6BB;03A4;03A4; # (𝚻𝚻; 𝚻𝚻; 𝚻𝚻; Τ; Τ; ) MATHEMATICAL BOLD CAPITAL TAU +1D6BC;1D6BC;1D6BC;03A5;03A5; # (𝚼𝚼; 𝚼𝚼; 𝚼𝚼; Υ; Υ; ) MATHEMATICAL BOLD CAPITAL UPSILON +1D6BD;1D6BD;1D6BD;03A6;03A6; # (𝚽𝚽; 𝚽𝚽; 𝚽𝚽; Φ; Φ; ) MATHEMATICAL BOLD CAPITAL PHI +1D6BE;1D6BE;1D6BE;03A7;03A7; # (𝚾𝚾; 𝚾𝚾; 𝚾𝚾; Χ; Χ; ) MATHEMATICAL BOLD CAPITAL CHI +1D6BF;1D6BF;1D6BF;03A8;03A8; # (𝚿𝚿; 𝚿𝚿; 𝚿𝚿; Ψ; Ψ; ) MATHEMATICAL BOLD CAPITAL PSI +1D6C0;1D6C0;1D6C0;03A9;03A9; # (𝛀𝛀; 𝛀𝛀; 𝛀𝛀; Ω; Ω; ) MATHEMATICAL BOLD CAPITAL OMEGA +1D6C1;1D6C1;1D6C1;2207;2207; # (𝛁𝛁; 𝛁𝛁; 𝛁𝛁; ∇; ∇; ) MATHEMATICAL BOLD NABLA +1D6C2;1D6C2;1D6C2;03B1;03B1; # (𝛂𝛂; 𝛂𝛂; 𝛂𝛂; α; α; ) MATHEMATICAL BOLD SMALL ALPHA +1D6C3;1D6C3;1D6C3;03B2;03B2; # (𝛃𝛃; 𝛃𝛃; 𝛃𝛃; β; β; ) MATHEMATICAL BOLD SMALL BETA +1D6C4;1D6C4;1D6C4;03B3;03B3; # (𝛄𝛄; 𝛄𝛄; 𝛄𝛄; γ; γ; ) MATHEMATICAL BOLD SMALL GAMMA +1D6C5;1D6C5;1D6C5;03B4;03B4; # (𝛅𝛅; 𝛅𝛅; 𝛅𝛅; δ; δ; ) MATHEMATICAL BOLD SMALL DELTA +1D6C6;1D6C6;1D6C6;03B5;03B5; # (𝛆𝛆; 𝛆𝛆; 𝛆𝛆; ε; ε; ) MATHEMATICAL BOLD SMALL EPSILON +1D6C7;1D6C7;1D6C7;03B6;03B6; # (𝛇𝛇; 𝛇𝛇; 𝛇𝛇; ζ; ζ; ) MATHEMATICAL BOLD SMALL ZETA +1D6C8;1D6C8;1D6C8;03B7;03B7; # (𝛈𝛈; 𝛈𝛈; 𝛈𝛈; η; η; ) MATHEMATICAL BOLD SMALL ETA +1D6C9;1D6C9;1D6C9;03B8;03B8; # (𝛉𝛉; 𝛉𝛉; 𝛉𝛉; θ; θ; ) MATHEMATICAL BOLD SMALL THETA +1D6CA;1D6CA;1D6CA;03B9;03B9; # (𝛊𝛊; 𝛊𝛊; 𝛊𝛊; ι; ι; ) MATHEMATICAL BOLD SMALL IOTA +1D6CB;1D6CB;1D6CB;03BA;03BA; # (𝛋𝛋; 𝛋𝛋; 𝛋𝛋; κ; κ; ) MATHEMATICAL BOLD SMALL KAPPA +1D6CC;1D6CC;1D6CC;03BB;03BB; # (𝛌𝛌; 𝛌𝛌; 𝛌𝛌; λ; λ; ) MATHEMATICAL BOLD SMALL LAMDA +1D6CD;1D6CD;1D6CD;03BC;03BC; # (𝛍𝛍; 𝛍𝛍; 𝛍𝛍; μ; μ; ) MATHEMATICAL BOLD SMALL MU +1D6CE;1D6CE;1D6CE;03BD;03BD; # (𝛎𝛎; 𝛎𝛎; 𝛎𝛎; ν; ν; ) MATHEMATICAL BOLD SMALL NU +1D6CF;1D6CF;1D6CF;03BE;03BE; # (𝛏𝛏; 𝛏𝛏; 𝛏𝛏; ξ; ξ; ) MATHEMATICAL BOLD SMALL XI +1D6D0;1D6D0;1D6D0;03BF;03BF; # (𝛐𝛐; 𝛐𝛐; 𝛐𝛐; ο; ο; ) MATHEMATICAL BOLD SMALL OMICRON +1D6D1;1D6D1;1D6D1;03C0;03C0; # (𝛑𝛑; 𝛑𝛑; 𝛑𝛑; π; π; ) MATHEMATICAL BOLD SMALL PI +1D6D2;1D6D2;1D6D2;03C1;03C1; # (𝛒𝛒; 𝛒𝛒; 𝛒𝛒; ρ; ρ; ) MATHEMATICAL BOLD SMALL RHO +1D6D3;1D6D3;1D6D3;03C2;03C2; # (𝛓𝛓; 𝛓𝛓; 𝛓𝛓; ς; ς; ) MATHEMATICAL BOLD SMALL FINAL SIGMA +1D6D4;1D6D4;1D6D4;03C3;03C3; # (𝛔𝛔; 𝛔𝛔; 𝛔𝛔; σ; σ; ) MATHEMATICAL BOLD SMALL SIGMA +1D6D5;1D6D5;1D6D5;03C4;03C4; # (𝛕𝛕; 𝛕𝛕; 𝛕𝛕; τ; τ; ) MATHEMATICAL BOLD SMALL TAU +1D6D6;1D6D6;1D6D6;03C5;03C5; # (𝛖𝛖; 𝛖𝛖; 𝛖𝛖; υ; υ; ) MATHEMATICAL BOLD SMALL UPSILON +1D6D7;1D6D7;1D6D7;03C6;03C6; # (𝛗𝛗; 𝛗𝛗; 𝛗𝛗; φ; φ; ) MATHEMATICAL BOLD SMALL PHI +1D6D8;1D6D8;1D6D8;03C7;03C7; # (𝛘𝛘; 𝛘𝛘; 𝛘𝛘; χ; χ; ) MATHEMATICAL BOLD SMALL CHI +1D6D9;1D6D9;1D6D9;03C8;03C8; # (𝛙𝛙; 𝛙𝛙; 𝛙𝛙; ψ; ψ; ) MATHEMATICAL BOLD SMALL PSI +1D6DA;1D6DA;1D6DA;03C9;03C9; # (𝛚𝛚; 𝛚𝛚; 𝛚𝛚; ω; ω; ) MATHEMATICAL BOLD SMALL OMEGA +1D6DB;1D6DB;1D6DB;2202;2202; # (𝛛𝛛; 𝛛𝛛; 𝛛𝛛; ∂; ∂; ) MATHEMATICAL BOLD PARTIAL DIFFERENTIAL +1D6DC;1D6DC;1D6DC;03B5;03B5; # (𝛜𝛜; 𝛜𝛜; 𝛜𝛜; ε; ε; ) MATHEMATICAL BOLD EPSILON SYMBOL +1D6DD;1D6DD;1D6DD;03B8;03B8; # (𝛝𝛝; 𝛝𝛝; 𝛝𝛝; θ; θ; ) MATHEMATICAL BOLD THETA SYMBOL +1D6DE;1D6DE;1D6DE;03BA;03BA; # (𝛞𝛞; 𝛞𝛞; 𝛞𝛞; κ; κ; ) MATHEMATICAL BOLD KAPPA SYMBOL +1D6DF;1D6DF;1D6DF;03C6;03C6; # (𝛟𝛟; 𝛟𝛟; 𝛟𝛟; φ; φ; ) MATHEMATICAL BOLD PHI SYMBOL +1D6E0;1D6E0;1D6E0;03C1;03C1; # (𝛠𝛠; 𝛠𝛠; 𝛠𝛠; ρ; ρ; ) MATHEMATICAL BOLD RHO SYMBOL +1D6E1;1D6E1;1D6E1;03C0;03C0; # (𝛡𝛡; 𝛡𝛡; 𝛡𝛡; π; π; ) MATHEMATICAL BOLD PI SYMBOL +1D6E2;1D6E2;1D6E2;0391;0391; # (𝛢𝛢; 𝛢𝛢; 𝛢𝛢; Α; Α; ) MATHEMATICAL ITALIC CAPITAL ALPHA +1D6E3;1D6E3;1D6E3;0392;0392; # (𝛣𝛣; 𝛣𝛣; 𝛣𝛣; Β; Β; ) MATHEMATICAL ITALIC CAPITAL BETA +1D6E4;1D6E4;1D6E4;0393;0393; # (𝛤𝛤; 𝛤𝛤; 𝛤𝛤; Γ; Γ; ) MATHEMATICAL ITALIC CAPITAL GAMMA +1D6E5;1D6E5;1D6E5;0394;0394; # (𝛥𝛥; 𝛥𝛥; 𝛥𝛥; Δ; Δ; ) MATHEMATICAL ITALIC CAPITAL DELTA +1D6E6;1D6E6;1D6E6;0395;0395; # (𝛦𝛦; 𝛦𝛦; 𝛦𝛦; Ε; Ε; ) MATHEMATICAL ITALIC CAPITAL EPSILON +1D6E7;1D6E7;1D6E7;0396;0396; # (𝛧𝛧; 𝛧𝛧; 𝛧𝛧; Ζ; Ζ; ) MATHEMATICAL ITALIC CAPITAL ZETA +1D6E8;1D6E8;1D6E8;0397;0397; # (𝛨𝛨; 𝛨𝛨; 𝛨𝛨; Η; Η; ) MATHEMATICAL ITALIC CAPITAL ETA +1D6E9;1D6E9;1D6E9;0398;0398; # (𝛩𝛩; 𝛩𝛩; 𝛩𝛩; Θ; Θ; ) MATHEMATICAL ITALIC CAPITAL THETA +1D6EA;1D6EA;1D6EA;0399;0399; # (𝛪𝛪; 𝛪𝛪; 𝛪𝛪; Ι; Ι; ) MATHEMATICAL ITALIC CAPITAL IOTA +1D6EB;1D6EB;1D6EB;039A;039A; # (𝛫𝛫; 𝛫𝛫; 𝛫𝛫; Κ; Κ; ) MATHEMATICAL ITALIC CAPITAL KAPPA +1D6EC;1D6EC;1D6EC;039B;039B; # (𝛬𝛬; 𝛬𝛬; 𝛬𝛬; Λ; Λ; ) MATHEMATICAL ITALIC CAPITAL LAMDA +1D6ED;1D6ED;1D6ED;039C;039C; # (𝛭𝛭; 𝛭𝛭; 𝛭𝛭; Μ; Μ; ) MATHEMATICAL ITALIC CAPITAL MU +1D6EE;1D6EE;1D6EE;039D;039D; # (𝛮𝛮; 𝛮𝛮; 𝛮𝛮; Ν; Ν; ) MATHEMATICAL ITALIC CAPITAL NU +1D6EF;1D6EF;1D6EF;039E;039E; # (𝛯𝛯; 𝛯𝛯; 𝛯𝛯; Ξ; Ξ; ) MATHEMATICAL ITALIC CAPITAL XI +1D6F0;1D6F0;1D6F0;039F;039F; # (𝛰𝛰; 𝛰𝛰; 𝛰𝛰; Ο; Ο; ) MATHEMATICAL ITALIC CAPITAL OMICRON +1D6F1;1D6F1;1D6F1;03A0;03A0; # (𝛱𝛱; 𝛱𝛱; 𝛱𝛱; Π; Π; ) MATHEMATICAL ITALIC CAPITAL PI +1D6F2;1D6F2;1D6F2;03A1;03A1; # (𝛲𝛲; 𝛲𝛲; 𝛲𝛲; Ρ; Ρ; ) MATHEMATICAL ITALIC CAPITAL RHO +1D6F3;1D6F3;1D6F3;0398;0398; # (𝛳𝛳; 𝛳𝛳; 𝛳𝛳; Θ; Θ; ) MATHEMATICAL ITALIC CAPITAL THETA SYMBOL +1D6F4;1D6F4;1D6F4;03A3;03A3; # (𝛴𝛴; 𝛴𝛴; 𝛴𝛴; Σ; Σ; ) MATHEMATICAL ITALIC CAPITAL SIGMA +1D6F5;1D6F5;1D6F5;03A4;03A4; # (𝛵𝛵; 𝛵𝛵; 𝛵𝛵; Τ; Τ; ) MATHEMATICAL ITALIC CAPITAL TAU +1D6F6;1D6F6;1D6F6;03A5;03A5; # (𝛶𝛶; 𝛶𝛶; 𝛶𝛶; Υ; Υ; ) MATHEMATICAL ITALIC CAPITAL UPSILON +1D6F7;1D6F7;1D6F7;03A6;03A6; # (𝛷𝛷; 𝛷𝛷; 𝛷𝛷; Φ; Φ; ) MATHEMATICAL ITALIC CAPITAL PHI +1D6F8;1D6F8;1D6F8;03A7;03A7; # (𝛸𝛸; 𝛸𝛸; 𝛸𝛸; Χ; Χ; ) MATHEMATICAL ITALIC CAPITAL CHI +1D6F9;1D6F9;1D6F9;03A8;03A8; # (𝛹𝛹; 𝛹𝛹; 𝛹𝛹; Ψ; Ψ; ) MATHEMATICAL ITALIC CAPITAL PSI +1D6FA;1D6FA;1D6FA;03A9;03A9; # (𝛺𝛺; 𝛺𝛺; 𝛺𝛺; Ω; Ω; ) MATHEMATICAL ITALIC CAPITAL OMEGA +1D6FB;1D6FB;1D6FB;2207;2207; # (𝛻𝛻; 𝛻𝛻; 𝛻𝛻; ∇; ∇; ) MATHEMATICAL ITALIC NABLA +1D6FC;1D6FC;1D6FC;03B1;03B1; # (𝛼𝛼; 𝛼𝛼; 𝛼𝛼; α; α; ) MATHEMATICAL ITALIC SMALL ALPHA +1D6FD;1D6FD;1D6FD;03B2;03B2; # (𝛽𝛽; 𝛽𝛽; 𝛽𝛽; β; β; ) MATHEMATICAL ITALIC SMALL BETA +1D6FE;1D6FE;1D6FE;03B3;03B3; # (𝛾𝛾; 𝛾𝛾; 𝛾𝛾; γ; γ; ) MATHEMATICAL ITALIC SMALL GAMMA +1D6FF;1D6FF;1D6FF;03B4;03B4; # (𝛿𝛿; 𝛿𝛿; 𝛿𝛿; δ; δ; ) MATHEMATICAL ITALIC SMALL DELTA +1D700;1D700;1D700;03B5;03B5; # (𝜀𝜀; 𝜀𝜀; 𝜀𝜀; ε; ε; ) MATHEMATICAL ITALIC SMALL EPSILON +1D701;1D701;1D701;03B6;03B6; # (𝜁𝜁; 𝜁𝜁; 𝜁𝜁; ζ; ζ; ) MATHEMATICAL ITALIC SMALL ZETA +1D702;1D702;1D702;03B7;03B7; # (𝜂𝜂; 𝜂𝜂; 𝜂𝜂; η; η; ) MATHEMATICAL ITALIC SMALL ETA +1D703;1D703;1D703;03B8;03B8; # (𝜃𝜃; 𝜃𝜃; 𝜃𝜃; θ; θ; ) MATHEMATICAL ITALIC SMALL THETA +1D704;1D704;1D704;03B9;03B9; # (𝜄𝜄; 𝜄𝜄; 𝜄𝜄; ι; ι; ) MATHEMATICAL ITALIC SMALL IOTA +1D705;1D705;1D705;03BA;03BA; # (𝜅𝜅; 𝜅𝜅; 𝜅𝜅; κ; κ; ) MATHEMATICAL ITALIC SMALL KAPPA +1D706;1D706;1D706;03BB;03BB; # (𝜆𝜆; 𝜆𝜆; 𝜆𝜆; λ; λ; ) MATHEMATICAL ITALIC SMALL LAMDA +1D707;1D707;1D707;03BC;03BC; # (𝜇𝜇; 𝜇𝜇; 𝜇𝜇; μ; μ; ) MATHEMATICAL ITALIC SMALL MU +1D708;1D708;1D708;03BD;03BD; # (𝜈𝜈; 𝜈𝜈; 𝜈𝜈; ν; ν; ) MATHEMATICAL ITALIC SMALL NU +1D709;1D709;1D709;03BE;03BE; # (𝜉𝜉; 𝜉𝜉; 𝜉𝜉; ξ; ξ; ) MATHEMATICAL ITALIC SMALL XI +1D70A;1D70A;1D70A;03BF;03BF; # (𝜊𝜊; 𝜊𝜊; 𝜊𝜊; ο; ο; ) MATHEMATICAL ITALIC SMALL OMICRON +1D70B;1D70B;1D70B;03C0;03C0; # (𝜋𝜋; 𝜋𝜋; 𝜋𝜋; π; π; ) MATHEMATICAL ITALIC SMALL PI +1D70C;1D70C;1D70C;03C1;03C1; # (𝜌𝜌; 𝜌𝜌; 𝜌𝜌; ρ; ρ; ) MATHEMATICAL ITALIC SMALL RHO +1D70D;1D70D;1D70D;03C2;03C2; # (𝜍𝜍; 𝜍𝜍; 𝜍𝜍; ς; ς; ) MATHEMATICAL ITALIC SMALL FINAL SIGMA +1D70E;1D70E;1D70E;03C3;03C3; # (𝜎𝜎; 𝜎𝜎; 𝜎𝜎; σ; σ; ) MATHEMATICAL ITALIC SMALL SIGMA +1D70F;1D70F;1D70F;03C4;03C4; # (𝜏𝜏; 𝜏𝜏; 𝜏𝜏; τ; τ; ) MATHEMATICAL ITALIC SMALL TAU +1D710;1D710;1D710;03C5;03C5; # (𝜐𝜐; 𝜐𝜐; 𝜐𝜐; υ; υ; ) MATHEMATICAL ITALIC SMALL UPSILON +1D711;1D711;1D711;03C6;03C6; # (𝜑𝜑; 𝜑𝜑; 𝜑𝜑; φ; φ; ) MATHEMATICAL ITALIC SMALL PHI +1D712;1D712;1D712;03C7;03C7; # (𝜒𝜒; 𝜒𝜒; 𝜒𝜒; χ; χ; ) MATHEMATICAL ITALIC SMALL CHI +1D713;1D713;1D713;03C8;03C8; # (𝜓𝜓; 𝜓𝜓; 𝜓𝜓; ψ; ψ; ) MATHEMATICAL ITALIC SMALL PSI +1D714;1D714;1D714;03C9;03C9; # (𝜔𝜔; 𝜔𝜔; 𝜔𝜔; ω; ω; ) MATHEMATICAL ITALIC SMALL OMEGA +1D715;1D715;1D715;2202;2202; # (𝜕𝜕; 𝜕𝜕; 𝜕𝜕; ∂; ∂; ) MATHEMATICAL ITALIC PARTIAL DIFFERENTIAL +1D716;1D716;1D716;03B5;03B5; # (𝜖𝜖; 𝜖𝜖; 𝜖𝜖; ε; ε; ) MATHEMATICAL ITALIC EPSILON SYMBOL +1D717;1D717;1D717;03B8;03B8; # (𝜗𝜗; 𝜗𝜗; 𝜗𝜗; θ; θ; ) MATHEMATICAL ITALIC THETA SYMBOL +1D718;1D718;1D718;03BA;03BA; # (𝜘𝜘; 𝜘𝜘; 𝜘𝜘; κ; κ; ) MATHEMATICAL ITALIC KAPPA SYMBOL +1D719;1D719;1D719;03C6;03C6; # (𝜙𝜙; 𝜙𝜙; 𝜙𝜙; φ; φ; ) MATHEMATICAL ITALIC PHI SYMBOL +1D71A;1D71A;1D71A;03C1;03C1; # (𝜚𝜚; 𝜚𝜚; 𝜚𝜚; ρ; ρ; ) MATHEMATICAL ITALIC RHO SYMBOL +1D71B;1D71B;1D71B;03C0;03C0; # (𝜛𝜛; 𝜛𝜛; 𝜛𝜛; π; π; ) MATHEMATICAL ITALIC PI SYMBOL +1D71C;1D71C;1D71C;0391;0391; # (𝜜𝜜; 𝜜𝜜; 𝜜𝜜; Α; Α; ) MATHEMATICAL BOLD ITALIC CAPITAL ALPHA +1D71D;1D71D;1D71D;0392;0392; # (𝜝𝜝; 𝜝𝜝; 𝜝𝜝; Β; Β; ) MATHEMATICAL BOLD ITALIC CAPITAL BETA +1D71E;1D71E;1D71E;0393;0393; # (𝜞𝜞; 𝜞𝜞; 𝜞𝜞; Γ; Γ; ) MATHEMATICAL BOLD ITALIC CAPITAL GAMMA +1D71F;1D71F;1D71F;0394;0394; # (𝜟𝜟; 𝜟𝜟; 𝜟𝜟; Δ; Δ; ) MATHEMATICAL BOLD ITALIC CAPITAL DELTA +1D720;1D720;1D720;0395;0395; # (𝜠𝜠; 𝜠𝜠; 𝜠𝜠; Ε; Ε; ) MATHEMATICAL BOLD ITALIC CAPITAL EPSILON +1D721;1D721;1D721;0396;0396; # (𝜡𝜡; 𝜡𝜡; 𝜡𝜡; Ζ; Ζ; ) MATHEMATICAL BOLD ITALIC CAPITAL ZETA +1D722;1D722;1D722;0397;0397; # (𝜢𝜢; 𝜢𝜢; 𝜢𝜢; Η; Η; ) MATHEMATICAL BOLD ITALIC CAPITAL ETA +1D723;1D723;1D723;0398;0398; # (𝜣𝜣; 𝜣𝜣; 𝜣𝜣; Θ; Θ; ) MATHEMATICAL BOLD ITALIC CAPITAL THETA +1D724;1D724;1D724;0399;0399; # (𝜤𝜤; 𝜤𝜤; 𝜤𝜤; Ι; Ι; ) MATHEMATICAL BOLD ITALIC CAPITAL IOTA +1D725;1D725;1D725;039A;039A; # (𝜥𝜥; 𝜥𝜥; 𝜥𝜥; Κ; Κ; ) MATHEMATICAL BOLD ITALIC CAPITAL KAPPA +1D726;1D726;1D726;039B;039B; # (𝜦𝜦; 𝜦𝜦; 𝜦𝜦; Λ; Λ; ) MATHEMATICAL BOLD ITALIC CAPITAL LAMDA +1D727;1D727;1D727;039C;039C; # (𝜧𝜧; 𝜧𝜧; 𝜧𝜧; Μ; Μ; ) MATHEMATICAL BOLD ITALIC CAPITAL MU +1D728;1D728;1D728;039D;039D; # (𝜨𝜨; 𝜨𝜨; 𝜨𝜨; Ν; Ν; ) MATHEMATICAL BOLD ITALIC CAPITAL NU +1D729;1D729;1D729;039E;039E; # (𝜩𝜩; 𝜩𝜩; 𝜩𝜩; Ξ; Ξ; ) MATHEMATICAL BOLD ITALIC CAPITAL XI +1D72A;1D72A;1D72A;039F;039F; # (𝜪𝜪; 𝜪𝜪; 𝜪𝜪; Ο; Ο; ) MATHEMATICAL BOLD ITALIC CAPITAL OMICRON +1D72B;1D72B;1D72B;03A0;03A0; # (𝜫𝜫; 𝜫𝜫; 𝜫𝜫; Π; Π; ) MATHEMATICAL BOLD ITALIC CAPITAL PI +1D72C;1D72C;1D72C;03A1;03A1; # (𝜬𝜬; 𝜬𝜬; 𝜬𝜬; Ρ; Ρ; ) MATHEMATICAL BOLD ITALIC CAPITAL RHO +1D72D;1D72D;1D72D;0398;0398; # (𝜭𝜭; 𝜭𝜭; 𝜭𝜭; Θ; Θ; ) MATHEMATICAL BOLD ITALIC CAPITAL THETA SYMBOL +1D72E;1D72E;1D72E;03A3;03A3; # (𝜮𝜮; 𝜮𝜮; 𝜮𝜮; Σ; Σ; ) MATHEMATICAL BOLD ITALIC CAPITAL SIGMA +1D72F;1D72F;1D72F;03A4;03A4; # (𝜯𝜯; 𝜯𝜯; 𝜯𝜯; Τ; Τ; ) MATHEMATICAL BOLD ITALIC CAPITAL TAU +1D730;1D730;1D730;03A5;03A5; # (𝜰𝜰; 𝜰𝜰; 𝜰𝜰; Υ; Υ; ) MATHEMATICAL BOLD ITALIC CAPITAL UPSILON +1D731;1D731;1D731;03A6;03A6; # (𝜱𝜱; 𝜱𝜱; 𝜱𝜱; Φ; Φ; ) MATHEMATICAL BOLD ITALIC CAPITAL PHI +1D732;1D732;1D732;03A7;03A7; # (𝜲𝜲; 𝜲𝜲; 𝜲𝜲; Χ; Χ; ) MATHEMATICAL BOLD ITALIC CAPITAL CHI +1D733;1D733;1D733;03A8;03A8; # (𝜳𝜳; 𝜳𝜳; 𝜳𝜳; Ψ; Ψ; ) MATHEMATICAL BOLD ITALIC CAPITAL PSI +1D734;1D734;1D734;03A9;03A9; # (𝜴𝜴; 𝜴𝜴; 𝜴𝜴; Ω; Ω; ) MATHEMATICAL BOLD ITALIC CAPITAL OMEGA +1D735;1D735;1D735;2207;2207; # (𝜵𝜵; 𝜵𝜵; 𝜵𝜵; ∇; ∇; ) MATHEMATICAL BOLD ITALIC NABLA +1D736;1D736;1D736;03B1;03B1; # (𝜶𝜶; 𝜶𝜶; 𝜶𝜶; α; α; ) MATHEMATICAL BOLD ITALIC SMALL ALPHA +1D737;1D737;1D737;03B2;03B2; # (𝜷𝜷; 𝜷𝜷; 𝜷𝜷; β; β; ) MATHEMATICAL BOLD ITALIC SMALL BETA +1D738;1D738;1D738;03B3;03B3; # (𝜸𝜸; 𝜸𝜸; 𝜸𝜸; γ; γ; ) MATHEMATICAL BOLD ITALIC SMALL GAMMA +1D739;1D739;1D739;03B4;03B4; # (𝜹𝜹; 𝜹𝜹; 𝜹𝜹; δ; δ; ) MATHEMATICAL BOLD ITALIC SMALL DELTA +1D73A;1D73A;1D73A;03B5;03B5; # (𝜺𝜺; 𝜺𝜺; 𝜺𝜺; ε; ε; ) MATHEMATICAL BOLD ITALIC SMALL EPSILON +1D73B;1D73B;1D73B;03B6;03B6; # (𝜻𝜻; 𝜻𝜻; 𝜻𝜻; ζ; ζ; ) MATHEMATICAL BOLD ITALIC SMALL ZETA +1D73C;1D73C;1D73C;03B7;03B7; # (𝜼𝜼; 𝜼𝜼; 𝜼𝜼; η; η; ) MATHEMATICAL BOLD ITALIC SMALL ETA +1D73D;1D73D;1D73D;03B8;03B8; # (𝜽𝜽; 𝜽𝜽; 𝜽𝜽; θ; θ; ) MATHEMATICAL BOLD ITALIC SMALL THETA +1D73E;1D73E;1D73E;03B9;03B9; # (𝜾𝜾; 𝜾𝜾; 𝜾𝜾; ι; ι; ) MATHEMATICAL BOLD ITALIC SMALL IOTA +1D73F;1D73F;1D73F;03BA;03BA; # (𝜿𝜿; 𝜿𝜿; 𝜿𝜿; κ; κ; ) MATHEMATICAL BOLD ITALIC SMALL KAPPA +1D740;1D740;1D740;03BB;03BB; # (𝝀𝝀; 𝝀𝝀; 𝝀𝝀; λ; λ; ) MATHEMATICAL BOLD ITALIC SMALL LAMDA +1D741;1D741;1D741;03BC;03BC; # (𝝁𝝁; 𝝁𝝁; 𝝁𝝁; μ; μ; ) MATHEMATICAL BOLD ITALIC SMALL MU +1D742;1D742;1D742;03BD;03BD; # (𝝂𝝂; 𝝂𝝂; 𝝂𝝂; ν; ν; ) MATHEMATICAL BOLD ITALIC SMALL NU +1D743;1D743;1D743;03BE;03BE; # (𝝃𝝃; 𝝃𝝃; 𝝃𝝃; ξ; ξ; ) MATHEMATICAL BOLD ITALIC SMALL XI +1D744;1D744;1D744;03BF;03BF; # (𝝄𝝄; 𝝄𝝄; 𝝄𝝄; ο; ο; ) MATHEMATICAL BOLD ITALIC SMALL OMICRON +1D745;1D745;1D745;03C0;03C0; # (𝝅𝝅; 𝝅𝝅; 𝝅𝝅; π; π; ) MATHEMATICAL BOLD ITALIC SMALL PI +1D746;1D746;1D746;03C1;03C1; # (𝝆𝝆; 𝝆𝝆; 𝝆𝝆; ρ; ρ; ) MATHEMATICAL BOLD ITALIC SMALL RHO +1D747;1D747;1D747;03C2;03C2; # (𝝇𝝇; 𝝇𝝇; 𝝇𝝇; ς; ς; ) MATHEMATICAL BOLD ITALIC SMALL FINAL SIGMA +1D748;1D748;1D748;03C3;03C3; # (𝝈𝝈; 𝝈𝝈; 𝝈𝝈; σ; σ; ) MATHEMATICAL BOLD ITALIC SMALL SIGMA +1D749;1D749;1D749;03C4;03C4; # (𝝉𝝉; 𝝉𝝉; 𝝉𝝉; τ; τ; ) MATHEMATICAL BOLD ITALIC SMALL TAU +1D74A;1D74A;1D74A;03C5;03C5; # (𝝊𝝊; 𝝊𝝊; 𝝊𝝊; υ; υ; ) MATHEMATICAL BOLD ITALIC SMALL UPSILON +1D74B;1D74B;1D74B;03C6;03C6; # (𝝋𝝋; 𝝋𝝋; 𝝋𝝋; φ; φ; ) MATHEMATICAL BOLD ITALIC SMALL PHI +1D74C;1D74C;1D74C;03C7;03C7; # (𝝌𝝌; 𝝌𝝌; 𝝌𝝌; χ; χ; ) MATHEMATICAL BOLD ITALIC SMALL CHI +1D74D;1D74D;1D74D;03C8;03C8; # (𝝍𝝍; 𝝍𝝍; 𝝍𝝍; ψ; ψ; ) MATHEMATICAL BOLD ITALIC SMALL PSI +1D74E;1D74E;1D74E;03C9;03C9; # (𝝎𝝎; 𝝎𝝎; 𝝎𝝎; ω; ω; ) MATHEMATICAL BOLD ITALIC SMALL OMEGA +1D74F;1D74F;1D74F;2202;2202; # (𝝏𝝏; 𝝏𝝏; 𝝏𝝏; ∂; ∂; ) MATHEMATICAL BOLD ITALIC PARTIAL DIFFERENTIAL +1D750;1D750;1D750;03B5;03B5; # (𝝐𝝐; 𝝐𝝐; 𝝐𝝐; ε; ε; ) MATHEMATICAL BOLD ITALIC EPSILON SYMBOL +1D751;1D751;1D751;03B8;03B8; # (𝝑𝝑; 𝝑𝝑; 𝝑𝝑; θ; θ; ) MATHEMATICAL BOLD ITALIC THETA SYMBOL +1D752;1D752;1D752;03BA;03BA; # (𝝒𝝒; 𝝒𝝒; 𝝒𝝒; κ; κ; ) MATHEMATICAL BOLD ITALIC KAPPA SYMBOL +1D753;1D753;1D753;03C6;03C6; # (𝝓𝝓; 𝝓𝝓; 𝝓𝝓; φ; φ; ) MATHEMATICAL BOLD ITALIC PHI SYMBOL +1D754;1D754;1D754;03C1;03C1; # (𝝔𝝔; 𝝔𝝔; 𝝔𝝔; ρ; ρ; ) MATHEMATICAL BOLD ITALIC RHO SYMBOL +1D755;1D755;1D755;03C0;03C0; # (𝝕𝝕; 𝝕𝝕; 𝝕𝝕; π; π; ) MATHEMATICAL BOLD ITALIC PI SYMBOL +1D756;1D756;1D756;0391;0391; # (𝝖𝝖; 𝝖𝝖; 𝝖𝝖; Α; Α; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL ALPHA +1D757;1D757;1D757;0392;0392; # (𝝗𝝗; 𝝗𝝗; 𝝗𝝗; Β; Β; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL BETA +1D758;1D758;1D758;0393;0393; # (𝝘𝝘; 𝝘𝝘; 𝝘𝝘; Γ; Γ; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL GAMMA +1D759;1D759;1D759;0394;0394; # (𝝙𝝙; 𝝙𝝙; 𝝙𝝙; Δ; Δ; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL DELTA +1D75A;1D75A;1D75A;0395;0395; # (𝝚𝝚; 𝝚𝝚; 𝝚𝝚; Ε; Ε; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL EPSILON +1D75B;1D75B;1D75B;0396;0396; # (𝝛𝝛; 𝝛𝝛; 𝝛𝝛; Ζ; Ζ; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL ZETA +1D75C;1D75C;1D75C;0397;0397; # (𝝜𝝜; 𝝜𝝜; 𝝜𝝜; Η; Η; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL ETA +1D75D;1D75D;1D75D;0398;0398; # (𝝝𝝝; 𝝝𝝝; 𝝝𝝝; Θ; Θ; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL THETA +1D75E;1D75E;1D75E;0399;0399; # (𝝞𝝞; 𝝞𝝞; 𝝞𝝞; Ι; Ι; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL IOTA +1D75F;1D75F;1D75F;039A;039A; # (𝝟𝝟; 𝝟𝝟; 𝝟𝝟; Κ; Κ; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL KAPPA +1D760;1D760;1D760;039B;039B; # (𝝠𝝠; 𝝠𝝠; 𝝠𝝠; Λ; Λ; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL LAMDA +1D761;1D761;1D761;039C;039C; # (𝝡𝝡; 𝝡𝝡; 𝝡𝝡; Μ; Μ; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL MU +1D762;1D762;1D762;039D;039D; # (𝝢𝝢; 𝝢𝝢; 𝝢𝝢; Ν; Ν; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL NU +1D763;1D763;1D763;039E;039E; # (𝝣𝝣; 𝝣𝝣; 𝝣𝝣; Ξ; Ξ; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL XI +1D764;1D764;1D764;039F;039F; # (𝝤𝝤; 𝝤𝝤; 𝝤𝝤; Ο; Ο; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL OMICRON +1D765;1D765;1D765;03A0;03A0; # (𝝥𝝥; 𝝥𝝥; 𝝥𝝥; Π; Π; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL PI +1D766;1D766;1D766;03A1;03A1; # (𝝦𝝦; 𝝦𝝦; 𝝦𝝦; Ρ; Ρ; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL RHO +1D767;1D767;1D767;0398;0398; # (𝝧𝝧; 𝝧𝝧; 𝝧𝝧; Θ; Θ; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL THETA SYMBOL +1D768;1D768;1D768;03A3;03A3; # (𝝨𝝨; 𝝨𝝨; 𝝨𝝨; Σ; Σ; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL SIGMA +1D769;1D769;1D769;03A4;03A4; # (𝝩𝝩; 𝝩𝝩; 𝝩𝝩; Τ; Τ; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL TAU +1D76A;1D76A;1D76A;03A5;03A5; # (𝝪𝝪; 𝝪𝝪; 𝝪𝝪; Υ; Υ; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL UPSILON +1D76B;1D76B;1D76B;03A6;03A6; # (𝝫𝝫; 𝝫𝝫; 𝝫𝝫; Φ; Φ; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL PHI +1D76C;1D76C;1D76C;03A7;03A7; # (𝝬𝝬; 𝝬𝝬; 𝝬𝝬; Χ; Χ; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL CHI +1D76D;1D76D;1D76D;03A8;03A8; # (𝝭𝝭; 𝝭𝝭; 𝝭𝝭; Ψ; Ψ; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL PSI +1D76E;1D76E;1D76E;03A9;03A9; # (𝝮𝝮; 𝝮𝝮; 𝝮𝝮; Ω; Ω; ) MATHEMATICAL SANS-SERIF BOLD CAPITAL OMEGA +1D76F;1D76F;1D76F;2207;2207; # (𝝯𝝯; 𝝯𝝯; 𝝯𝝯; ∇; ∇; ) MATHEMATICAL SANS-SERIF BOLD NABLA +1D770;1D770;1D770;03B1;03B1; # (𝝰𝝰; 𝝰𝝰; 𝝰𝝰; α; α; ) MATHEMATICAL SANS-SERIF BOLD SMALL ALPHA +1D771;1D771;1D771;03B2;03B2; # (𝝱𝝱; 𝝱𝝱; 𝝱𝝱; β; β; ) MATHEMATICAL SANS-SERIF BOLD SMALL BETA +1D772;1D772;1D772;03B3;03B3; # (𝝲𝝲; 𝝲𝝲; 𝝲𝝲; γ; γ; ) MATHEMATICAL SANS-SERIF BOLD SMALL GAMMA +1D773;1D773;1D773;03B4;03B4; # (𝝳𝝳; 𝝳𝝳; 𝝳𝝳; δ; δ; ) MATHEMATICAL SANS-SERIF BOLD SMALL DELTA +1D774;1D774;1D774;03B5;03B5; # (𝝴𝝴; 𝝴𝝴; 𝝴𝝴; ε; ε; ) MATHEMATICAL SANS-SERIF BOLD SMALL EPSILON +1D775;1D775;1D775;03B6;03B6; # (𝝵𝝵; 𝝵𝝵; 𝝵𝝵; ζ; ζ; ) MATHEMATICAL SANS-SERIF BOLD SMALL ZETA +1D776;1D776;1D776;03B7;03B7; # (𝝶𝝶; 𝝶𝝶; 𝝶𝝶; η; η; ) MATHEMATICAL SANS-SERIF BOLD SMALL ETA +1D777;1D777;1D777;03B8;03B8; # (𝝷𝝷; 𝝷𝝷; 𝝷𝝷; θ; θ; ) MATHEMATICAL SANS-SERIF BOLD SMALL THETA +1D778;1D778;1D778;03B9;03B9; # (𝝸𝝸; 𝝸𝝸; 𝝸𝝸; ι; ι; ) MATHEMATICAL SANS-SERIF BOLD SMALL IOTA +1D779;1D779;1D779;03BA;03BA; # (𝝹𝝹; 𝝹𝝹; 𝝹𝝹; κ; κ; ) MATHEMATICAL SANS-SERIF BOLD SMALL KAPPA +1D77A;1D77A;1D77A;03BB;03BB; # (𝝺𝝺; 𝝺𝝺; 𝝺𝝺; λ; λ; ) MATHEMATICAL SANS-SERIF BOLD SMALL LAMDA +1D77B;1D77B;1D77B;03BC;03BC; # (𝝻𝝻; 𝝻𝝻; 𝝻𝝻; μ; μ; ) MATHEMATICAL SANS-SERIF BOLD SMALL MU +1D77C;1D77C;1D77C;03BD;03BD; # (𝝼𝝼; 𝝼𝝼; 𝝼𝝼; ν; ν; ) MATHEMATICAL SANS-SERIF BOLD SMALL NU +1D77D;1D77D;1D77D;03BE;03BE; # (𝝽𝝽; 𝝽𝝽; 𝝽𝝽; ξ; ξ; ) MATHEMATICAL SANS-SERIF BOLD SMALL XI +1D77E;1D77E;1D77E;03BF;03BF; # (𝝾𝝾; 𝝾𝝾; 𝝾𝝾; ο; ο; ) MATHEMATICAL SANS-SERIF BOLD SMALL OMICRON +1D77F;1D77F;1D77F;03C0;03C0; # (𝝿𝝿; 𝝿𝝿; 𝝿𝝿; π; π; ) MATHEMATICAL SANS-SERIF BOLD SMALL PI +1D780;1D780;1D780;03C1;03C1; # (𝞀𝞀; 𝞀𝞀; 𝞀𝞀; ρ; ρ; ) MATHEMATICAL SANS-SERIF BOLD SMALL RHO +1D781;1D781;1D781;03C2;03C2; # (𝞁𝞁; 𝞁𝞁; 𝞁𝞁; ς; ς; ) MATHEMATICAL SANS-SERIF BOLD SMALL FINAL SIGMA +1D782;1D782;1D782;03C3;03C3; # (𝞂𝞂; 𝞂𝞂; 𝞂𝞂; σ; σ; ) MATHEMATICAL SANS-SERIF BOLD SMALL SIGMA +1D783;1D783;1D783;03C4;03C4; # (𝞃𝞃; 𝞃𝞃; 𝞃𝞃; τ; τ; ) MATHEMATICAL SANS-SERIF BOLD SMALL TAU +1D784;1D784;1D784;03C5;03C5; # (𝞄𝞄; 𝞄𝞄; 𝞄𝞄; υ; υ; ) MATHEMATICAL SANS-SERIF BOLD SMALL UPSILON +1D785;1D785;1D785;03C6;03C6; # (𝞅𝞅; 𝞅𝞅; 𝞅𝞅; φ; φ; ) MATHEMATICAL SANS-SERIF BOLD SMALL PHI +1D786;1D786;1D786;03C7;03C7; # (𝞆𝞆; 𝞆𝞆; 𝞆𝞆; χ; χ; ) MATHEMATICAL SANS-SERIF BOLD SMALL CHI +1D787;1D787;1D787;03C8;03C8; # (𝞇𝞇; 𝞇𝞇; 𝞇𝞇; ψ; ψ; ) MATHEMATICAL SANS-SERIF BOLD SMALL PSI +1D788;1D788;1D788;03C9;03C9; # (𝞈𝞈; 𝞈𝞈; 𝞈𝞈; ω; ω; ) MATHEMATICAL SANS-SERIF BOLD SMALL OMEGA +1D789;1D789;1D789;2202;2202; # (𝞉𝞉; 𝞉𝞉; 𝞉𝞉; ∂; ∂; ) MATHEMATICAL SANS-SERIF BOLD PARTIAL DIFFERENTIAL +1D78A;1D78A;1D78A;03B5;03B5; # (𝞊𝞊; 𝞊𝞊; 𝞊𝞊; ε; ε; ) MATHEMATICAL SANS-SERIF BOLD EPSILON SYMBOL +1D78B;1D78B;1D78B;03B8;03B8; # (𝞋𝞋; 𝞋𝞋; 𝞋𝞋; θ; θ; ) MATHEMATICAL SANS-SERIF BOLD THETA SYMBOL +1D78C;1D78C;1D78C;03BA;03BA; # (𝞌𝞌; 𝞌𝞌; 𝞌𝞌; κ; κ; ) MATHEMATICAL SANS-SERIF BOLD KAPPA SYMBOL +1D78D;1D78D;1D78D;03C6;03C6; # (𝞍𝞍; 𝞍𝞍; 𝞍𝞍; φ; φ; ) MATHEMATICAL SANS-SERIF BOLD PHI SYMBOL +1D78E;1D78E;1D78E;03C1;03C1; # (𝞎𝞎; 𝞎𝞎; 𝞎𝞎; ρ; ρ; ) MATHEMATICAL SANS-SERIF BOLD RHO SYMBOL +1D78F;1D78F;1D78F;03C0;03C0; # (𝞏𝞏; 𝞏𝞏; 𝞏𝞏; π; π; ) MATHEMATICAL SANS-SERIF BOLD PI SYMBOL +1D790;1D790;1D790;0391;0391; # (𝞐𝞐; 𝞐𝞐; 𝞐𝞐; Α; Α; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL ALPHA +1D791;1D791;1D791;0392;0392; # (𝞑𝞑; 𝞑𝞑; 𝞑𝞑; Β; Β; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL BETA +1D792;1D792;1D792;0393;0393; # (𝞒𝞒; 𝞒𝞒; 𝞒𝞒; Γ; Γ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL GAMMA +1D793;1D793;1D793;0394;0394; # (𝞓𝞓; 𝞓𝞓; 𝞓𝞓; Δ; Δ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL DELTA +1D794;1D794;1D794;0395;0395; # (𝞔𝞔; 𝞔𝞔; 𝞔𝞔; Ε; Ε; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL EPSILON +1D795;1D795;1D795;0396;0396; # (𝞕𝞕; 𝞕𝞕; 𝞕𝞕; Ζ; Ζ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL ZETA +1D796;1D796;1D796;0397;0397; # (𝞖𝞖; 𝞖𝞖; 𝞖𝞖; Η; Η; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL ETA +1D797;1D797;1D797;0398;0398; # (𝞗𝞗; 𝞗𝞗; 𝞗𝞗; Θ; Θ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL THETA +1D798;1D798;1D798;0399;0399; # (𝞘𝞘; 𝞘𝞘; 𝞘𝞘; Ι; Ι; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL IOTA +1D799;1D799;1D799;039A;039A; # (𝞙𝞙; 𝞙𝞙; 𝞙𝞙; Κ; Κ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL KAPPA +1D79A;1D79A;1D79A;039B;039B; # (𝞚𝞚; 𝞚𝞚; 𝞚𝞚; Λ; Λ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL LAMDA +1D79B;1D79B;1D79B;039C;039C; # (𝞛𝞛; 𝞛𝞛; 𝞛𝞛; Μ; Μ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL MU +1D79C;1D79C;1D79C;039D;039D; # (𝞜𝞜; 𝞜𝞜; 𝞜𝞜; Ν; Ν; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL NU +1D79D;1D79D;1D79D;039E;039E; # (𝞝𝞝; 𝞝𝞝; 𝞝𝞝; Ξ; Ξ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL XI +1D79E;1D79E;1D79E;039F;039F; # (𝞞𝞞; 𝞞𝞞; 𝞞𝞞; Ο; Ο; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMICRON +1D79F;1D79F;1D79F;03A0;03A0; # (𝞟𝞟; 𝞟𝞟; 𝞟𝞟; Π; Π; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL PI +1D7A0;1D7A0;1D7A0;03A1;03A1; # (𝞠𝞠; 𝞠𝞠; 𝞠𝞠; Ρ; Ρ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL RHO +1D7A1;1D7A1;1D7A1;0398;0398; # (𝞡𝞡; 𝞡𝞡; 𝞡𝞡; Θ; Θ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL THETA SYMBOL +1D7A2;1D7A2;1D7A2;03A3;03A3; # (𝞢𝞢; 𝞢𝞢; 𝞢𝞢; Σ; Σ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL SIGMA +1D7A3;1D7A3;1D7A3;03A4;03A4; # (𝞣𝞣; 𝞣𝞣; 𝞣𝞣; Τ; Τ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL TAU +1D7A4;1D7A4;1D7A4;03A5;03A5; # (𝞤𝞤; 𝞤𝞤; 𝞤𝞤; Υ; Υ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL UPSILON +1D7A5;1D7A5;1D7A5;03A6;03A6; # (𝞥𝞥; 𝞥𝞥; 𝞥𝞥; Φ; Φ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL PHI +1D7A6;1D7A6;1D7A6;03A7;03A7; # (𝞦𝞦; 𝞦𝞦; 𝞦𝞦; Χ; Χ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL CHI +1D7A7;1D7A7;1D7A7;03A8;03A8; # (𝞧𝞧; 𝞧𝞧; 𝞧𝞧; Ψ; Ψ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL PSI +1D7A8;1D7A8;1D7A8;03A9;03A9; # (𝞨𝞨; 𝞨𝞨; 𝞨𝞨; Ω; Ω; ) MATHEMATICAL SANS-SERIF BOLD ITALIC CAPITAL OMEGA +1D7A9;1D7A9;1D7A9;2207;2207; # (𝞩𝞩; 𝞩𝞩; 𝞩𝞩; ∇; ∇; ) MATHEMATICAL SANS-SERIF BOLD ITALIC NABLA +1D7AA;1D7AA;1D7AA;03B1;03B1; # (𝞪𝞪; 𝞪𝞪; 𝞪𝞪; α; α; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ALPHA +1D7AB;1D7AB;1D7AB;03B2;03B2; # (𝞫𝞫; 𝞫𝞫; 𝞫𝞫; β; β; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL BETA +1D7AC;1D7AC;1D7AC;03B3;03B3; # (𝞬𝞬; 𝞬𝞬; 𝞬𝞬; γ; γ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL GAMMA +1D7AD;1D7AD;1D7AD;03B4;03B4; # (𝞭𝞭; 𝞭𝞭; 𝞭𝞭; δ; δ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL DELTA +1D7AE;1D7AE;1D7AE;03B5;03B5; # (𝞮𝞮; 𝞮𝞮; 𝞮𝞮; ε; ε; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL EPSILON +1D7AF;1D7AF;1D7AF;03B6;03B6; # (𝞯𝞯; 𝞯𝞯; 𝞯𝞯; ζ; ζ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ZETA +1D7B0;1D7B0;1D7B0;03B7;03B7; # (𝞰𝞰; 𝞰𝞰; 𝞰𝞰; η; η; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL ETA +1D7B1;1D7B1;1D7B1;03B8;03B8; # (𝞱𝞱; 𝞱𝞱; 𝞱𝞱; θ; θ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL THETA +1D7B2;1D7B2;1D7B2;03B9;03B9; # (𝞲𝞲; 𝞲𝞲; 𝞲𝞲; ι; ι; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL IOTA +1D7B3;1D7B3;1D7B3;03BA;03BA; # (𝞳𝞳; 𝞳𝞳; 𝞳𝞳; κ; κ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL KAPPA +1D7B4;1D7B4;1D7B4;03BB;03BB; # (𝞴𝞴; 𝞴𝞴; 𝞴𝞴; λ; λ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL LAMDA +1D7B5;1D7B5;1D7B5;03BC;03BC; # (𝞵𝞵; 𝞵𝞵; 𝞵𝞵; μ; μ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL MU +1D7B6;1D7B6;1D7B6;03BD;03BD; # (𝞶𝞶; 𝞶𝞶; 𝞶𝞶; ν; ν; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL NU +1D7B7;1D7B7;1D7B7;03BE;03BE; # (𝞷𝞷; 𝞷𝞷; 𝞷𝞷; ξ; ξ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL XI +1D7B8;1D7B8;1D7B8;03BF;03BF; # (𝞸𝞸; 𝞸𝞸; 𝞸𝞸; ο; ο; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMICRON +1D7B9;1D7B9;1D7B9;03C0;03C0; # (𝞹𝞹; 𝞹𝞹; 𝞹𝞹; π; π; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL PI +1D7BA;1D7BA;1D7BA;03C1;03C1; # (𝞺𝞺; 𝞺𝞺; 𝞺𝞺; ρ; ρ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL RHO +1D7BB;1D7BB;1D7BB;03C2;03C2; # (𝞻𝞻; 𝞻𝞻; 𝞻𝞻; ς; ς; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL FINAL SIGMA +1D7BC;1D7BC;1D7BC;03C3;03C3; # (𝞼𝞼; 𝞼𝞼; 𝞼𝞼; σ; σ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL SIGMA +1D7BD;1D7BD;1D7BD;03C4;03C4; # (𝞽𝞽; 𝞽𝞽; 𝞽𝞽; τ; τ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL TAU +1D7BE;1D7BE;1D7BE;03C5;03C5; # (𝞾𝞾; 𝞾𝞾; 𝞾𝞾; υ; υ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL UPSILON +1D7BF;1D7BF;1D7BF;03C6;03C6; # (𝞿𝞿; 𝞿𝞿; 𝞿𝞿; φ; φ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL PHI +1D7C0;1D7C0;1D7C0;03C7;03C7; # (𝟀𝟀; 𝟀𝟀; 𝟀𝟀; χ; χ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL CHI +1D7C1;1D7C1;1D7C1;03C8;03C8; # (𝟁𝟁; 𝟁𝟁; 𝟁𝟁; ψ; ψ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL PSI +1D7C2;1D7C2;1D7C2;03C9;03C9; # (𝟂𝟂; 𝟂𝟂; 𝟂𝟂; ω; ω; ) MATHEMATICAL SANS-SERIF BOLD ITALIC SMALL OMEGA +1D7C3;1D7C3;1D7C3;2202;2202; # (𝟃𝟃; 𝟃𝟃; 𝟃𝟃; ∂; ∂; ) MATHEMATICAL SANS-SERIF BOLD ITALIC PARTIAL DIFFERENTIAL +1D7C4;1D7C4;1D7C4;03B5;03B5; # (𝟄𝟄; 𝟄𝟄; 𝟄𝟄; ε; ε; ) MATHEMATICAL SANS-SERIF BOLD ITALIC EPSILON SYMBOL +1D7C5;1D7C5;1D7C5;03B8;03B8; # (𝟅𝟅; 𝟅𝟅; 𝟅𝟅; θ; θ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC THETA SYMBOL +1D7C6;1D7C6;1D7C6;03BA;03BA; # (𝟆𝟆; 𝟆𝟆; 𝟆𝟆; κ; κ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC KAPPA SYMBOL +1D7C7;1D7C7;1D7C7;03C6;03C6; # (𝟇𝟇; 𝟇𝟇; 𝟇𝟇; φ; φ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC PHI SYMBOL +1D7C8;1D7C8;1D7C8;03C1;03C1; # (𝟈𝟈; 𝟈𝟈; 𝟈𝟈; ρ; ρ; ) MATHEMATICAL SANS-SERIF BOLD ITALIC RHO SYMBOL +1D7C9;1D7C9;1D7C9;03C0;03C0; # (𝟉𝟉; 𝟉𝟉; 𝟉𝟉; π; π; ) MATHEMATICAL SANS-SERIF BOLD ITALIC PI SYMBOL +1D7CE;1D7CE;1D7CE;0030;0030; # (𝟎𝟎; 𝟎𝟎; 𝟎𝟎; 0; 0; ) MATHEMATICAL BOLD DIGIT ZERO +1D7CF;1D7CF;1D7CF;0031;0031; # (𝟏𝟏; 𝟏𝟏; 𝟏𝟏; 1; 1; ) MATHEMATICAL BOLD DIGIT ONE +1D7D0;1D7D0;1D7D0;0032;0032; # (𝟐𝟐; 𝟐𝟐; 𝟐𝟐; 2; 2; ) MATHEMATICAL BOLD DIGIT TWO +1D7D1;1D7D1;1D7D1;0033;0033; # (𝟑𝟑; 𝟑𝟑; 𝟑𝟑; 3; 3; ) MATHEMATICAL BOLD DIGIT THREE +1D7D2;1D7D2;1D7D2;0034;0034; # (𝟒𝟒; 𝟒𝟒; 𝟒𝟒; 4; 4; ) MATHEMATICAL BOLD DIGIT FOUR +1D7D3;1D7D3;1D7D3;0035;0035; # (𝟓𝟓; 𝟓𝟓; 𝟓𝟓; 5; 5; ) MATHEMATICAL BOLD DIGIT FIVE +1D7D4;1D7D4;1D7D4;0036;0036; # (𝟔𝟔; 𝟔𝟔; 𝟔𝟔; 6; 6; ) MATHEMATICAL BOLD DIGIT SIX +1D7D5;1D7D5;1D7D5;0037;0037; # (𝟕𝟕; 𝟕𝟕; 𝟕𝟕; 7; 7; ) MATHEMATICAL BOLD DIGIT SEVEN +1D7D6;1D7D6;1D7D6;0038;0038; # (𝟖𝟖; 𝟖𝟖; 𝟖𝟖; 8; 8; ) MATHEMATICAL BOLD DIGIT EIGHT +1D7D7;1D7D7;1D7D7;0039;0039; # (𝟗𝟗; 𝟗𝟗; 𝟗𝟗; 9; 9; ) MATHEMATICAL BOLD DIGIT NINE +1D7D8;1D7D8;1D7D8;0030;0030; # (𝟘𝟘; 𝟘𝟘; 𝟘𝟘; 0; 0; ) MATHEMATICAL DOUBLE-STRUCK DIGIT ZERO +1D7D9;1D7D9;1D7D9;0031;0031; # (𝟙𝟙; 𝟙𝟙; 𝟙𝟙; 1; 1; ) MATHEMATICAL DOUBLE-STRUCK DIGIT ONE +1D7DA;1D7DA;1D7DA;0032;0032; # (𝟚𝟚; 𝟚𝟚; 𝟚𝟚; 2; 2; ) MATHEMATICAL DOUBLE-STRUCK DIGIT TWO +1D7DB;1D7DB;1D7DB;0033;0033; # (𝟛𝟛; 𝟛𝟛; 𝟛𝟛; 3; 3; ) MATHEMATICAL DOUBLE-STRUCK DIGIT THREE +1D7DC;1D7DC;1D7DC;0034;0034; # (𝟜𝟜; 𝟜𝟜; 𝟜𝟜; 4; 4; ) MATHEMATICAL DOUBLE-STRUCK DIGIT FOUR +1D7DD;1D7DD;1D7DD;0035;0035; # (𝟝𝟝; 𝟝𝟝; 𝟝𝟝; 5; 5; ) MATHEMATICAL DOUBLE-STRUCK DIGIT FIVE +1D7DE;1D7DE;1D7DE;0036;0036; # (𝟞𝟞; 𝟞𝟞; 𝟞𝟞; 6; 6; ) MATHEMATICAL DOUBLE-STRUCK DIGIT SIX +1D7DF;1D7DF;1D7DF;0037;0037; # (𝟟𝟟; 𝟟𝟟; 𝟟𝟟; 7; 7; ) MATHEMATICAL DOUBLE-STRUCK DIGIT SEVEN +1D7E0;1D7E0;1D7E0;0038;0038; # (𝟠𝟠; 𝟠𝟠; 𝟠𝟠; 8; 8; ) MATHEMATICAL DOUBLE-STRUCK DIGIT EIGHT +1D7E1;1D7E1;1D7E1;0039;0039; # (𝟡𝟡; 𝟡𝟡; 𝟡𝟡; 9; 9; ) MATHEMATICAL DOUBLE-STRUCK DIGIT NINE +1D7E2;1D7E2;1D7E2;0030;0030; # (𝟢𝟢; 𝟢𝟢; 𝟢𝟢; 0; 0; ) MATHEMATICAL SANS-SERIF DIGIT ZERO +1D7E3;1D7E3;1D7E3;0031;0031; # (𝟣𝟣; 𝟣𝟣; 𝟣𝟣; 1; 1; ) MATHEMATICAL SANS-SERIF DIGIT ONE +1D7E4;1D7E4;1D7E4;0032;0032; # (𝟤𝟤; 𝟤𝟤; 𝟤𝟤; 2; 2; ) MATHEMATICAL SANS-SERIF DIGIT TWO +1D7E5;1D7E5;1D7E5;0033;0033; # (𝟥𝟥; 𝟥𝟥; 𝟥𝟥; 3; 3; ) MATHEMATICAL SANS-SERIF DIGIT THREE +1D7E6;1D7E6;1D7E6;0034;0034; # (𝟦𝟦; 𝟦𝟦; 𝟦𝟦; 4; 4; ) MATHEMATICAL SANS-SERIF DIGIT FOUR +1D7E7;1D7E7;1D7E7;0035;0035; # (𝟧𝟧; 𝟧𝟧; 𝟧𝟧; 5; 5; ) MATHEMATICAL SANS-SERIF DIGIT FIVE +1D7E8;1D7E8;1D7E8;0036;0036; # (𝟨𝟨; 𝟨𝟨; 𝟨𝟨; 6; 6; ) MATHEMATICAL SANS-SERIF DIGIT SIX +1D7E9;1D7E9;1D7E9;0037;0037; # (𝟩𝟩; 𝟩𝟩; 𝟩𝟩; 7; 7; ) MATHEMATICAL SANS-SERIF DIGIT SEVEN +1D7EA;1D7EA;1D7EA;0038;0038; # (𝟪𝟪; 𝟪𝟪; 𝟪𝟪; 8; 8; ) MATHEMATICAL SANS-SERIF DIGIT EIGHT +1D7EB;1D7EB;1D7EB;0039;0039; # (𝟫𝟫; 𝟫𝟫; 𝟫𝟫; 9; 9; ) MATHEMATICAL SANS-SERIF DIGIT NINE +1D7EC;1D7EC;1D7EC;0030;0030; # (𝟬𝟬; 𝟬𝟬; 𝟬𝟬; 0; 0; ) MATHEMATICAL SANS-SERIF BOLD DIGIT ZERO +1D7ED;1D7ED;1D7ED;0031;0031; # (𝟭𝟭; 𝟭𝟭; 𝟭𝟭; 1; 1; ) MATHEMATICAL SANS-SERIF BOLD DIGIT ONE +1D7EE;1D7EE;1D7EE;0032;0032; # (𝟮𝟮; 𝟮𝟮; 𝟮𝟮; 2; 2; ) MATHEMATICAL SANS-SERIF BOLD DIGIT TWO +1D7EF;1D7EF;1D7EF;0033;0033; # (𝟯𝟯; 𝟯𝟯; 𝟯𝟯; 3; 3; ) MATHEMATICAL SANS-SERIF BOLD DIGIT THREE +1D7F0;1D7F0;1D7F0;0034;0034; # (𝟰𝟰; 𝟰𝟰; 𝟰𝟰; 4; 4; ) MATHEMATICAL SANS-SERIF BOLD DIGIT FOUR +1D7F1;1D7F1;1D7F1;0035;0035; # (𝟱𝟱; 𝟱𝟱; 𝟱𝟱; 5; 5; ) MATHEMATICAL SANS-SERIF BOLD DIGIT FIVE +1D7F2;1D7F2;1D7F2;0036;0036; # (𝟲𝟲; 𝟲𝟲; 𝟲𝟲; 6; 6; ) MATHEMATICAL SANS-SERIF BOLD DIGIT SIX +1D7F3;1D7F3;1D7F3;0037;0037; # (𝟳𝟳; 𝟳𝟳; 𝟳𝟳; 7; 7; ) MATHEMATICAL SANS-SERIF BOLD DIGIT SEVEN +1D7F4;1D7F4;1D7F4;0038;0038; # (𝟴𝟴; 𝟴𝟴; 𝟴𝟴; 8; 8; ) MATHEMATICAL SANS-SERIF BOLD DIGIT EIGHT +1D7F5;1D7F5;1D7F5;0039;0039; # (𝟵𝟵; 𝟵𝟵; 𝟵𝟵; 9; 9; ) MATHEMATICAL SANS-SERIF BOLD DIGIT NINE +1D7F6;1D7F6;1D7F6;0030;0030; # (𝟶𝟶; 𝟶𝟶; 𝟶𝟶; 0; 0; ) MATHEMATICAL MONOSPACE DIGIT ZERO +1D7F7;1D7F7;1D7F7;0031;0031; # (𝟷𝟷; 𝟷𝟷; 𝟷𝟷; 1; 1; ) MATHEMATICAL MONOSPACE DIGIT ONE +1D7F8;1D7F8;1D7F8;0032;0032; # (𝟸𝟸; 𝟸𝟸; 𝟸𝟸; 2; 2; ) MATHEMATICAL MONOSPACE DIGIT TWO +1D7F9;1D7F9;1D7F9;0033;0033; # (𝟹𝟹; 𝟹𝟹; 𝟹𝟹; 3; 3; ) MATHEMATICAL MONOSPACE DIGIT THREE +1D7FA;1D7FA;1D7FA;0034;0034; # (𝟺𝟺; 𝟺𝟺; 𝟺𝟺; 4; 4; ) MATHEMATICAL MONOSPACE DIGIT FOUR +1D7FB;1D7FB;1D7FB;0035;0035; # (𝟻𝟻; 𝟻𝟻; 𝟻𝟻; 5; 5; ) MATHEMATICAL MONOSPACE DIGIT FIVE +1D7FC;1D7FC;1D7FC;0036;0036; # (𝟼𝟼; 𝟼𝟼; 𝟼𝟼; 6; 6; ) MATHEMATICAL MONOSPACE DIGIT SIX +1D7FD;1D7FD;1D7FD;0037;0037; # (𝟽𝟽; 𝟽𝟽; 𝟽𝟽; 7; 7; ) MATHEMATICAL MONOSPACE DIGIT SEVEN +1D7FE;1D7FE;1D7FE;0038;0038; # (𝟾𝟾; 𝟾𝟾; 𝟾𝟾; 8; 8; ) MATHEMATICAL MONOSPACE DIGIT EIGHT +1D7FF;1D7FF;1D7FF;0039;0039; # (𝟿𝟿; 𝟿𝟿; 𝟿𝟿; 9; 9; ) MATHEMATICAL MONOSPACE DIGIT NINE +2F800;4E3D;4E3D;4E3D;4E3D; # (丽丽; 丽; 丽; 丽; 丽; ) CJK COMPATIBILITY IDEOGRAPH-2F800 +2F801;4E38;4E38;4E38;4E38; # (丸丸; 丸; 丸; 丸; 丸; ) CJK COMPATIBILITY IDEOGRAPH-2F801 +2F802;4E41;4E41;4E41;4E41; # (乁乁; 乁; 乁; 乁; 乁; ) CJK COMPATIBILITY IDEOGRAPH-2F802 +2F803;20122;20122;20122;20122; # (𠄢𠄢; 𠄢𠄢; 𠄢𠄢; 𠄢𠄢; 𠄢𠄢; ) CJK COMPATIBILITY IDEOGRAPH-2F803 +2F804;4F60;4F60;4F60;4F60; # (你你; 你; 你; 你; 你; ) CJK COMPATIBILITY IDEOGRAPH-2F804 +2F805;4FAE;4FAE;4FAE;4FAE; # (侮侮; 侮; 侮; 侮; 侮; ) CJK COMPATIBILITY IDEOGRAPH-2F805 +2F806;4FBB;4FBB;4FBB;4FBB; # (侻侻; 侻; 侻; 侻; 侻; ) CJK COMPATIBILITY IDEOGRAPH-2F806 +2F807;5002;5002;5002;5002; # (倂倂; 倂; 倂; 倂; 倂; ) CJK COMPATIBILITY IDEOGRAPH-2F807 +2F808;507A;507A;507A;507A; # (偺偺; 偺; 偺; 偺; 偺; ) CJK COMPATIBILITY IDEOGRAPH-2F808 +2F809;5099;5099;5099;5099; # (備備; 備; 備; 備; 備; ) CJK COMPATIBILITY IDEOGRAPH-2F809 +2F80A;50E7;50E7;50E7;50E7; # (僧僧; 僧; 僧; 僧; 僧; ) CJK COMPATIBILITY IDEOGRAPH-2F80A +2F80B;50CF;50CF;50CF;50CF; # (像像; 像; 像; 像; 像; ) CJK COMPATIBILITY IDEOGRAPH-2F80B +2F80C;349E;349E;349E;349E; # (㒞㒞; 㒞; 㒞; 㒞; 㒞; ) CJK COMPATIBILITY IDEOGRAPH-2F80C +2F80D;2063A;2063A;2063A;2063A; # (𠘺𠘺; 𠘺𠘺; 𠘺𠘺; 𠘺𠘺; 𠘺𠘺; ) CJK COMPATIBILITY IDEOGRAPH-2F80D +2F80E;514D;514D;514D;514D; # (免免; 免; 免; 免; 免; ) CJK COMPATIBILITY IDEOGRAPH-2F80E +2F80F;5154;5154;5154;5154; # (兔兔; 兔; 兔; 兔; 兔; ) CJK COMPATIBILITY IDEOGRAPH-2F80F +2F810;5164;5164;5164;5164; # (兤兤; 兤; 兤; 兤; 兤; ) CJK COMPATIBILITY IDEOGRAPH-2F810 +2F811;5177;5177;5177;5177; # (具具; 具; 具; 具; 具; ) CJK COMPATIBILITY IDEOGRAPH-2F811 +2F812;2051C;2051C;2051C;2051C; # (𠔜𠔜; 𠔜𠔜; 𠔜𠔜; 𠔜𠔜; 𠔜𠔜; ) CJK COMPATIBILITY IDEOGRAPH-2F812 +2F813;34B9;34B9;34B9;34B9; # (㒹㒹; 㒹; 㒹; 㒹; 㒹; ) CJK COMPATIBILITY IDEOGRAPH-2F813 +2F814;5167;5167;5167;5167; # (內內; 內; 內; 內; 內; ) CJK COMPATIBILITY IDEOGRAPH-2F814 +2F815;518D;518D;518D;518D; # (再再; 再; 再; 再; 再; ) CJK COMPATIBILITY IDEOGRAPH-2F815 +2F816;2054B;2054B;2054B;2054B; # (𠕋𠕋; 𠕋𠕋; 𠕋𠕋; 𠕋𠕋; 𠕋𠕋; ) CJK COMPATIBILITY IDEOGRAPH-2F816 +2F817;5197;5197;5197;5197; # (冗冗; 冗; 冗; 冗; 冗; ) CJK COMPATIBILITY IDEOGRAPH-2F817 +2F818;51A4;51A4;51A4;51A4; # (冤冤; 冤; 冤; 冤; 冤; ) CJK COMPATIBILITY IDEOGRAPH-2F818 +2F819;4ECC;4ECC;4ECC;4ECC; # (仌仌; 仌; 仌; 仌; 仌; ) CJK COMPATIBILITY IDEOGRAPH-2F819 +2F81A;51AC;51AC;51AC;51AC; # (冬冬; 冬; 冬; 冬; 冬; ) CJK COMPATIBILITY IDEOGRAPH-2F81A +2F81B;51B5;51B5;51B5;51B5; # (况况; 况; 况; 况; 况; ) CJK COMPATIBILITY IDEOGRAPH-2F81B +2F81C;291DF;291DF;291DF;291DF; # (𩇟𩇟; 𩇟𩇟; 𩇟𩇟; 𩇟𩇟; 𩇟𩇟; ) CJK COMPATIBILITY IDEOGRAPH-2F81C +2F81D;51F5;51F5;51F5;51F5; # (凵凵; 凵; 凵; 凵; 凵; ) CJK COMPATIBILITY IDEOGRAPH-2F81D +2F81E;5203;5203;5203;5203; # (刃刃; 刃; 刃; 刃; 刃; ) CJK COMPATIBILITY IDEOGRAPH-2F81E +2F81F;34DF;34DF;34DF;34DF; # (㓟㓟; 㓟; 㓟; 㓟; 㓟; ) CJK COMPATIBILITY IDEOGRAPH-2F81F +2F820;523B;523B;523B;523B; # (刻刻; 刻; 刻; 刻; 刻; ) CJK COMPATIBILITY IDEOGRAPH-2F820 +2F821;5246;5246;5246;5246; # (剆剆; 剆; 剆; 剆; 剆; ) CJK COMPATIBILITY IDEOGRAPH-2F821 +2F822;5272;5272;5272;5272; # (割割; 割; 割; 割; 割; ) CJK COMPATIBILITY IDEOGRAPH-2F822 +2F823;5277;5277;5277;5277; # (剷剷; 剷; 剷; 剷; 剷; ) CJK COMPATIBILITY IDEOGRAPH-2F823 +2F824;3515;3515;3515;3515; # (㔕㔕; 㔕; 㔕; 㔕; 㔕; ) CJK COMPATIBILITY IDEOGRAPH-2F824 +2F825;52C7;52C7;52C7;52C7; # (勇勇; 勇; 勇; 勇; 勇; ) CJK COMPATIBILITY IDEOGRAPH-2F825 +2F826;52C9;52C9;52C9;52C9; # (勉勉; 勉; 勉; 勉; 勉; ) CJK COMPATIBILITY IDEOGRAPH-2F826 +2F827;52E4;52E4;52E4;52E4; # (勤勤; 勤; 勤; 勤; 勤; ) CJK COMPATIBILITY IDEOGRAPH-2F827 +2F828;52FA;52FA;52FA;52FA; # (勺勺; 勺; 勺; 勺; 勺; ) CJK COMPATIBILITY IDEOGRAPH-2F828 +2F829;5305;5305;5305;5305; # (包包; 包; 包; 包; 包; ) CJK COMPATIBILITY IDEOGRAPH-2F829 +2F82A;5306;5306;5306;5306; # (匆匆; 匆; 匆; 匆; 匆; ) CJK COMPATIBILITY IDEOGRAPH-2F82A +2F82B;5317;5317;5317;5317; # (北北; 北; 北; 北; 北; ) CJK COMPATIBILITY IDEOGRAPH-2F82B +2F82C;5349;5349;5349;5349; # (卉卉; 卉; 卉; 卉; 卉; ) CJK COMPATIBILITY IDEOGRAPH-2F82C +2F82D;5351;5351;5351;5351; # (卑卑; 卑; 卑; 卑; 卑; ) CJK COMPATIBILITY IDEOGRAPH-2F82D +2F82E;535A;535A;535A;535A; # (博博; 博; 博; 博; 博; ) CJK COMPATIBILITY IDEOGRAPH-2F82E +2F82F;5373;5373;5373;5373; # (即即; 即; 即; 即; 即; ) CJK COMPATIBILITY IDEOGRAPH-2F82F +2F830;537D;537D;537D;537D; # (卽卽; 卽; 卽; 卽; 卽; ) CJK COMPATIBILITY IDEOGRAPH-2F830 +2F831;537F;537F;537F;537F; # (卿卿; 卿; 卿; 卿; 卿; ) CJK COMPATIBILITY IDEOGRAPH-2F831 +2F832;537F;537F;537F;537F; # (卿卿; 卿; 卿; 卿; 卿; ) CJK COMPATIBILITY IDEOGRAPH-2F832 +2F833;537F;537F;537F;537F; # (卿卿; 卿; 卿; 卿; 卿; ) CJK COMPATIBILITY IDEOGRAPH-2F833 +2F834;20A2C;20A2C;20A2C;20A2C; # (𠨬𠨬; 𠨬𠨬; 𠨬𠨬; 𠨬𠨬; 𠨬𠨬; ) CJK COMPATIBILITY IDEOGRAPH-2F834 +2F835;7070;7070;7070;7070; # (灰灰; 灰; 灰; 灰; 灰; ) CJK COMPATIBILITY IDEOGRAPH-2F835 +2F836;53CA;53CA;53CA;53CA; # (及及; 及; 及; 及; 及; ) CJK COMPATIBILITY IDEOGRAPH-2F836 +2F837;53DF;53DF;53DF;53DF; # (叟叟; 叟; 叟; 叟; 叟; ) CJK COMPATIBILITY IDEOGRAPH-2F837 +2F838;20B63;20B63;20B63;20B63; # (𠭣𠭣; 𠭣𠭣; 𠭣𠭣; 𠭣𠭣; 𠭣𠭣; ) CJK COMPATIBILITY IDEOGRAPH-2F838 +2F839;53EB;53EB;53EB;53EB; # (叫叫; 叫; 叫; 叫; 叫; ) CJK COMPATIBILITY IDEOGRAPH-2F839 +2F83A;53F1;53F1;53F1;53F1; # (叱叱; 叱; 叱; 叱; 叱; ) CJK COMPATIBILITY IDEOGRAPH-2F83A +2F83B;5406;5406;5406;5406; # (吆吆; 吆; 吆; 吆; 吆; ) CJK COMPATIBILITY IDEOGRAPH-2F83B +2F83C;549E;549E;549E;549E; # (咞咞; 咞; 咞; 咞; 咞; ) CJK COMPATIBILITY IDEOGRAPH-2F83C +2F83D;5438;5438;5438;5438; # (吸吸; 吸; 吸; 吸; 吸; ) CJK COMPATIBILITY IDEOGRAPH-2F83D +2F83E;5448;5448;5448;5448; # (呈呈; 呈; 呈; 呈; 呈; ) CJK COMPATIBILITY IDEOGRAPH-2F83E +2F83F;5468;5468;5468;5468; # (周周; 周; 周; 周; 周; ) CJK COMPATIBILITY IDEOGRAPH-2F83F +2F840;54A2;54A2;54A2;54A2; # (咢咢; 咢; 咢; 咢; 咢; ) CJK COMPATIBILITY IDEOGRAPH-2F840 +2F841;54F6;54F6;54F6;54F6; # (哶哶; 哶; 哶; 哶; 哶; ) CJK COMPATIBILITY IDEOGRAPH-2F841 +2F842;5510;5510;5510;5510; # (唐唐; 唐; 唐; 唐; 唐; ) CJK COMPATIBILITY IDEOGRAPH-2F842 +2F843;5553;5553;5553;5553; # (啓啓; 啓; 啓; 啓; 啓; ) CJK COMPATIBILITY IDEOGRAPH-2F843 +2F844;5563;5563;5563;5563; # (啣啣; 啣; 啣; 啣; 啣; ) CJK COMPATIBILITY IDEOGRAPH-2F844 +2F845;5584;5584;5584;5584; # (善善; 善; 善; 善; 善; ) CJK COMPATIBILITY IDEOGRAPH-2F845 +2F846;5584;5584;5584;5584; # (善善; 善; 善; 善; 善; ) CJK COMPATIBILITY IDEOGRAPH-2F846 +2F847;5599;5599;5599;5599; # (喙喙; 喙; 喙; 喙; 喙; ) CJK COMPATIBILITY IDEOGRAPH-2F847 +2F848;55AB;55AB;55AB;55AB; # (喫喫; 喫; 喫; 喫; 喫; ) CJK COMPATIBILITY IDEOGRAPH-2F848 +2F849;55B3;55B3;55B3;55B3; # (喳喳; 喳; 喳; 喳; 喳; ) CJK COMPATIBILITY IDEOGRAPH-2F849 +2F84A;55C2;55C2;55C2;55C2; # (嗂嗂; 嗂; 嗂; 嗂; 嗂; ) CJK COMPATIBILITY IDEOGRAPH-2F84A +2F84B;5716;5716;5716;5716; # (圖圖; 圖; 圖; 圖; 圖; ) CJK COMPATIBILITY IDEOGRAPH-2F84B +2F84C;5606;5606;5606;5606; # (嘆嘆; 嘆; 嘆; 嘆; 嘆; ) CJK COMPATIBILITY IDEOGRAPH-2F84C +2F84D;5717;5717;5717;5717; # (圗圗; 圗; 圗; 圗; 圗; ) CJK COMPATIBILITY IDEOGRAPH-2F84D +2F84E;5651;5651;5651;5651; # (噑噑; 噑; 噑; 噑; 噑; ) CJK COMPATIBILITY IDEOGRAPH-2F84E +2F84F;5674;5674;5674;5674; # (噴噴; 噴; 噴; 噴; 噴; ) CJK COMPATIBILITY IDEOGRAPH-2F84F +2F850;5207;5207;5207;5207; # (切切; 切; 切; 切; 切; ) CJK COMPATIBILITY IDEOGRAPH-2F850 +2F851;58EE;58EE;58EE;58EE; # (壮壮; 壮; 壮; 壮; 壮; ) CJK COMPATIBILITY IDEOGRAPH-2F851 +2F852;57CE;57CE;57CE;57CE; # (城城; 城; 城; 城; 城; ) CJK COMPATIBILITY IDEOGRAPH-2F852 +2F853;57F4;57F4;57F4;57F4; # (埴埴; 埴; 埴; 埴; 埴; ) CJK COMPATIBILITY IDEOGRAPH-2F853 +2F854;580D;580D;580D;580D; # (堍堍; 堍; 堍; 堍; 堍; ) CJK COMPATIBILITY IDEOGRAPH-2F854 +2F855;578B;578B;578B;578B; # (型型; 型; 型; 型; 型; ) CJK COMPATIBILITY IDEOGRAPH-2F855 +2F856;5832;5832;5832;5832; # (堲堲; 堲; 堲; 堲; 堲; ) CJK COMPATIBILITY IDEOGRAPH-2F856 +2F857;5831;5831;5831;5831; # (報報; 報; 報; 報; 報; ) CJK COMPATIBILITY IDEOGRAPH-2F857 +2F858;58AC;58AC;58AC;58AC; # (墬墬; 墬; 墬; 墬; 墬; ) CJK COMPATIBILITY IDEOGRAPH-2F858 +2F859;214E4;214E4;214E4;214E4; # (𡓤𡓤; 𡓤𡓤; 𡓤𡓤; 𡓤𡓤; 𡓤𡓤; ) CJK COMPATIBILITY IDEOGRAPH-2F859 +2F85A;58F2;58F2;58F2;58F2; # (売売; 売; 売; 売; 売; ) CJK COMPATIBILITY IDEOGRAPH-2F85A +2F85B;58F7;58F7;58F7;58F7; # (壷壷; 壷; 壷; 壷; 壷; ) CJK COMPATIBILITY IDEOGRAPH-2F85B +2F85C;5906;5906;5906;5906; # (夆夆; 夆; 夆; 夆; 夆; ) CJK COMPATIBILITY IDEOGRAPH-2F85C +2F85D;591A;591A;591A;591A; # (多多; 多; 多; 多; 多; ) CJK COMPATIBILITY IDEOGRAPH-2F85D +2F85E;5922;5922;5922;5922; # (夢夢; 夢; 夢; 夢; 夢; ) CJK COMPATIBILITY IDEOGRAPH-2F85E +2F85F;5962;5962;5962;5962; # (奢奢; 奢; 奢; 奢; 奢; ) CJK COMPATIBILITY IDEOGRAPH-2F85F +2F860;216A8;216A8;216A8;216A8; # (𡚨𡚨; 𡚨𡚨; 𡚨𡚨; 𡚨𡚨; 𡚨𡚨; ) CJK COMPATIBILITY IDEOGRAPH-2F860 +2F861;216EA;216EA;216EA;216EA; # (𡛪𡛪; 𡛪𡛪; 𡛪𡛪; 𡛪𡛪; 𡛪𡛪; ) CJK COMPATIBILITY IDEOGRAPH-2F861 +2F862;59EC;59EC;59EC;59EC; # (姬姬; 姬; 姬; 姬; 姬; ) CJK COMPATIBILITY IDEOGRAPH-2F862 +2F863;5A1B;5A1B;5A1B;5A1B; # (娛娛; 娛; 娛; 娛; 娛; ) CJK COMPATIBILITY IDEOGRAPH-2F863 +2F864;5A27;5A27;5A27;5A27; # (娧娧; 娧; 娧; 娧; 娧; ) CJK COMPATIBILITY IDEOGRAPH-2F864 +2F865;59D8;59D8;59D8;59D8; # (姘姘; 姘; 姘; 姘; 姘; ) CJK COMPATIBILITY IDEOGRAPH-2F865 +2F866;5A66;5A66;5A66;5A66; # (婦婦; 婦; 婦; 婦; 婦; ) CJK COMPATIBILITY IDEOGRAPH-2F866 +2F867;36EE;36EE;36EE;36EE; # (㛮㛮; 㛮; 㛮; 㛮; 㛮; ) CJK COMPATIBILITY IDEOGRAPH-2F867 +2F868;2136A;2136A;2136A;2136A; # (㛼㛼; 𡍪𡍪; 𡍪𡍪; 𡍪𡍪; 𡍪𡍪; ) CJK COMPATIBILITY IDEOGRAPH-2F868 +2F869;5B08;5B08;5B08;5B08; # (嬈嬈; 嬈; 嬈; 嬈; 嬈; ) CJK COMPATIBILITY IDEOGRAPH-2F869 +2F86A;5B3E;5B3E;5B3E;5B3E; # (嬾嬾; 嬾; 嬾; 嬾; 嬾; ) CJK COMPATIBILITY IDEOGRAPH-2F86A +2F86B;5B3E;5B3E;5B3E;5B3E; # (嬾嬾; 嬾; 嬾; 嬾; 嬾; ) CJK COMPATIBILITY IDEOGRAPH-2F86B +2F86C;219C8;219C8;219C8;219C8; # (𡧈𡧈; 𡧈𡧈; 𡧈𡧈; 𡧈𡧈; 𡧈𡧈; ) CJK COMPATIBILITY IDEOGRAPH-2F86C +2F86D;5BC3;5BC3;5BC3;5BC3; # (寃寃; 寃; 寃; 寃; 寃; ) CJK COMPATIBILITY IDEOGRAPH-2F86D +2F86E;5BD8;5BD8;5BD8;5BD8; # (寘寘; 寘; 寘; 寘; 寘; ) CJK COMPATIBILITY IDEOGRAPH-2F86E +2F86F;5BE7;5BE7;5BE7;5BE7; # (寧寧; 寧; 寧; 寧; 寧; ) CJK COMPATIBILITY IDEOGRAPH-2F86F +2F870;5BF3;5BF3;5BF3;5BF3; # (寳寳; 寳; 寳; 寳; 寳; ) CJK COMPATIBILITY IDEOGRAPH-2F870 +2F871;21B18;21B18;21B18;21B18; # (𡬘𡬘; 𡬘𡬘; 𡬘𡬘; 𡬘𡬘; 𡬘𡬘; ) CJK COMPATIBILITY IDEOGRAPH-2F871 +2F872;5BFF;5BFF;5BFF;5BFF; # (寿寿; 寿; 寿; 寿; 寿; ) CJK COMPATIBILITY IDEOGRAPH-2F872 +2F873;5C06;5C06;5C06;5C06; # (将将; 将; 将; 将; 将; ) CJK COMPATIBILITY IDEOGRAPH-2F873 +2F874;5F33;5F33;5F33;5F33; # (当当; 弳; 弳; 弳; 弳; ) CJK COMPATIBILITY IDEOGRAPH-2F874 +2F875;5C22;5C22;5C22;5C22; # (尢尢; 尢; 尢; 尢; 尢; ) CJK COMPATIBILITY IDEOGRAPH-2F875 +2F876;3781;3781;3781;3781; # (㞁㞁; 㞁; 㞁; 㞁; 㞁; ) CJK COMPATIBILITY IDEOGRAPH-2F876 +2F877;5C60;5C60;5C60;5C60; # (屠屠; 屠; 屠; 屠; 屠; ) CJK COMPATIBILITY IDEOGRAPH-2F877 +2F878;5C6E;5C6E;5C6E;5C6E; # (屮屮; 屮; 屮; 屮; 屮; ) CJK COMPATIBILITY IDEOGRAPH-2F878 +2F879;5CC0;5CC0;5CC0;5CC0; # (峀峀; 峀; 峀; 峀; 峀; ) CJK COMPATIBILITY IDEOGRAPH-2F879 +2F87A;5C8D;5C8D;5C8D;5C8D; # (岍岍; 岍; 岍; 岍; 岍; ) CJK COMPATIBILITY IDEOGRAPH-2F87A +2F87B;21DE4;21DE4;21DE4;21DE4; # (𡷤𡷤; 𡷤𡷤; 𡷤𡷤; 𡷤𡷤; 𡷤𡷤; ) CJK COMPATIBILITY IDEOGRAPH-2F87B +2F87C;5D43;5D43;5D43;5D43; # (嵃嵃; 嵃; 嵃; 嵃; 嵃; ) CJK COMPATIBILITY IDEOGRAPH-2F87C +2F87D;21DE6;21DE6;21DE6;21DE6; # (𡷦𡷦; 𡷦𡷦; 𡷦𡷦; 𡷦𡷦; 𡷦𡷦; ) CJK COMPATIBILITY IDEOGRAPH-2F87D +2F87E;5D6E;5D6E;5D6E;5D6E; # (嵮嵮; 嵮; 嵮; 嵮; 嵮; ) CJK COMPATIBILITY IDEOGRAPH-2F87E +2F87F;5D6B;5D6B;5D6B;5D6B; # (嵫嵫; 嵫; 嵫; 嵫; 嵫; ) CJK COMPATIBILITY IDEOGRAPH-2F87F +2F880;5D7C;5D7C;5D7C;5D7C; # (嵼嵼; 嵼; 嵼; 嵼; 嵼; ) CJK COMPATIBILITY IDEOGRAPH-2F880 +2F881;5DE1;5DE1;5DE1;5DE1; # (巡巡; 巡; 巡; 巡; 巡; ) CJK COMPATIBILITY IDEOGRAPH-2F881 +2F882;5DE2;5DE2;5DE2;5DE2; # (巢巢; 巢; 巢; 巢; 巢; ) CJK COMPATIBILITY IDEOGRAPH-2F882 +2F883;382F;382F;382F;382F; # (㠯㠯; 㠯; 㠯; 㠯; 㠯; ) CJK COMPATIBILITY IDEOGRAPH-2F883 +2F884;5DFD;5DFD;5DFD;5DFD; # (巽巽; 巽; 巽; 巽; 巽; ) CJK COMPATIBILITY IDEOGRAPH-2F884 +2F885;5E28;5E28;5E28;5E28; # (帨帨; 帨; 帨; 帨; 帨; ) CJK COMPATIBILITY IDEOGRAPH-2F885 +2F886;5E3D;5E3D;5E3D;5E3D; # (帽帽; 帽; 帽; 帽; 帽; ) CJK COMPATIBILITY IDEOGRAPH-2F886 +2F887;5E69;5E69;5E69;5E69; # (幩幩; 幩; 幩; 幩; 幩; ) CJK COMPATIBILITY IDEOGRAPH-2F887 +2F888;3862;3862;3862;3862; # (㡢㡢; 㡢; 㡢; 㡢; 㡢; ) CJK COMPATIBILITY IDEOGRAPH-2F888 +2F889;22183;22183;22183;22183; # (𢆃𢆃; 𢆃𢆃; 𢆃𢆃; 𢆃𢆃; 𢆃𢆃; ) CJK COMPATIBILITY IDEOGRAPH-2F889 +2F88A;387C;387C;387C;387C; # (㡼㡼; 㡼; 㡼; 㡼; 㡼; ) CJK COMPATIBILITY IDEOGRAPH-2F88A +2F88B;5EB0;5EB0;5EB0;5EB0; # (庰庰; 庰; 庰; 庰; 庰; ) CJK COMPATIBILITY IDEOGRAPH-2F88B +2F88C;5EB3;5EB3;5EB3;5EB3; # (庳庳; 庳; 庳; 庳; 庳; ) CJK COMPATIBILITY IDEOGRAPH-2F88C +2F88D;5EB6;5EB6;5EB6;5EB6; # (庶庶; 庶; 庶; 庶; 庶; ) CJK COMPATIBILITY IDEOGRAPH-2F88D +2F88E;5ECA;5ECA;5ECA;5ECA; # (廊廊; 廊; 廊; 廊; 廊; ) CJK COMPATIBILITY IDEOGRAPH-2F88E +2F88F;2A392;2A392;2A392;2A392; # (𪎒𪎒; 𪎒𪎒; 𪎒𪎒; 𪎒𪎒; 𪎒𪎒; ) CJK COMPATIBILITY IDEOGRAPH-2F88F +2F890;5EFE;5EFE;5EFE;5EFE; # (廾廾; 廾; 廾; 廾; 廾; ) CJK COMPATIBILITY IDEOGRAPH-2F890 +2F891;22331;22331;22331;22331; # (𢌱𢌱; 𢌱𢌱; 𢌱𢌱; 𢌱𢌱; 𢌱𢌱; ) CJK COMPATIBILITY IDEOGRAPH-2F891 +2F892;22331;22331;22331;22331; # (𢌱𢌱; 𢌱𢌱; 𢌱𢌱; 𢌱𢌱; 𢌱𢌱; ) CJK COMPATIBILITY IDEOGRAPH-2F892 +2F893;8201;8201;8201;8201; # (舁舁; 舁; 舁; 舁; 舁; ) CJK COMPATIBILITY IDEOGRAPH-2F893 +2F894;5F22;5F22;5F22;5F22; # (弢弢; 弢; 弢; 弢; 弢; ) CJK COMPATIBILITY IDEOGRAPH-2F894 +2F895;5F22;5F22;5F22;5F22; # (弢弢; 弢; 弢; 弢; 弢; ) CJK COMPATIBILITY IDEOGRAPH-2F895 +2F896;38C7;38C7;38C7;38C7; # (㣇㣇; 㣇; 㣇; 㣇; 㣇; ) CJK COMPATIBILITY IDEOGRAPH-2F896 +2F897;232B8;232B8;232B8;232B8; # (𣊸𣊸; 𣊸𣊸; 𣊸𣊸; 𣊸𣊸; 𣊸𣊸; ) CJK COMPATIBILITY IDEOGRAPH-2F897 +2F898;261DA;261DA;261DA;261DA; # (𦇚𦇚; 𦇚𦇚; 𦇚𦇚; 𦇚𦇚; 𦇚𦇚; ) CJK COMPATIBILITY IDEOGRAPH-2F898 +2F899;5F62;5F62;5F62;5F62; # (形形; 形; 形; 形; 形; ) CJK COMPATIBILITY IDEOGRAPH-2F899 +2F89A;5F6B;5F6B;5F6B;5F6B; # (彫彫; 彫; 彫; 彫; 彫; ) CJK COMPATIBILITY IDEOGRAPH-2F89A +2F89B;38E3;38E3;38E3;38E3; # (㣣㣣; 㣣; 㣣; 㣣; 㣣; ) CJK COMPATIBILITY IDEOGRAPH-2F89B +2F89C;5F9A;5F9A;5F9A;5F9A; # (徚徚; 徚; 徚; 徚; 徚; ) CJK COMPATIBILITY IDEOGRAPH-2F89C +2F89D;5FCD;5FCD;5FCD;5FCD; # (忍忍; 忍; 忍; 忍; 忍; ) CJK COMPATIBILITY IDEOGRAPH-2F89D +2F89E;5FD7;5FD7;5FD7;5FD7; # (志志; 志; 志; 志; 志; ) CJK COMPATIBILITY IDEOGRAPH-2F89E +2F89F;5FF9;5FF9;5FF9;5FF9; # (忹忹; 忹; 忹; 忹; 忹; ) CJK COMPATIBILITY IDEOGRAPH-2F89F +2F8A0;6081;6081;6081;6081; # (悁悁; 悁; 悁; 悁; 悁; ) CJK COMPATIBILITY IDEOGRAPH-2F8A0 +2F8A1;393A;393A;393A;393A; # (㤺㤺; 㤺; 㤺; 㤺; 㤺; ) CJK COMPATIBILITY IDEOGRAPH-2F8A1 +2F8A2;391C;391C;391C;391C; # (㤜㤜; 㤜; 㤜; 㤜; 㤜; ) CJK COMPATIBILITY IDEOGRAPH-2F8A2 +2F8A3;6094;6094;6094;6094; # (悔悔; 悔; 悔; 悔; 悔; ) CJK COMPATIBILITY IDEOGRAPH-2F8A3 +2F8A4;226D4;226D4;226D4;226D4; # (𢛔𢛔; 𢛔𢛔; 𢛔𢛔; 𢛔𢛔; 𢛔𢛔; ) CJK COMPATIBILITY IDEOGRAPH-2F8A4 +2F8A5;60C7;60C7;60C7;60C7; # (惇惇; 惇; 惇; 惇; 惇; ) CJK COMPATIBILITY IDEOGRAPH-2F8A5 +2F8A6;6148;6148;6148;6148; # (慈慈; 慈; 慈; 慈; 慈; ) CJK COMPATIBILITY IDEOGRAPH-2F8A6 +2F8A7;614C;614C;614C;614C; # (慌慌; 慌; 慌; 慌; 慌; ) CJK COMPATIBILITY IDEOGRAPH-2F8A7 +2F8A8;614E;614E;614E;614E; # (慎慎; 慎; 慎; 慎; 慎; ) CJK COMPATIBILITY IDEOGRAPH-2F8A8 +2F8A9;614C;614C;614C;614C; # (慌慌; 慌; 慌; 慌; 慌; ) CJK COMPATIBILITY IDEOGRAPH-2F8A9 +2F8AA;617A;617A;617A;617A; # (慺慺; 慺; 慺; 慺; 慺; ) CJK COMPATIBILITY IDEOGRAPH-2F8AA +2F8AB;618E;618E;618E;618E; # (憎憎; 憎; 憎; 憎; 憎; ) CJK COMPATIBILITY IDEOGRAPH-2F8AB +2F8AC;61B2;61B2;61B2;61B2; # (憲憲; 憲; 憲; 憲; 憲; ) CJK COMPATIBILITY IDEOGRAPH-2F8AC +2F8AD;61A4;61A4;61A4;61A4; # (憤憤; 憤; 憤; 憤; 憤; ) CJK COMPATIBILITY IDEOGRAPH-2F8AD +2F8AE;61AF;61AF;61AF;61AF; # (憯憯; 憯; 憯; 憯; 憯; ) CJK COMPATIBILITY IDEOGRAPH-2F8AE +2F8AF;61DE;61DE;61DE;61DE; # (懞懞; 懞; 懞; 懞; 懞; ) CJK COMPATIBILITY IDEOGRAPH-2F8AF +2F8B0;61F2;61F2;61F2;61F2; # (懲懲; 懲; 懲; 懲; 懲; ) CJK COMPATIBILITY IDEOGRAPH-2F8B0 +2F8B1;61F6;61F6;61F6;61F6; # (懶懶; 懶; 懶; 懶; 懶; ) CJK COMPATIBILITY IDEOGRAPH-2F8B1 +2F8B2;6210;6210;6210;6210; # (成成; 成; 成; 成; 成; ) CJK COMPATIBILITY IDEOGRAPH-2F8B2 +2F8B3;621B;621B;621B;621B; # (戛戛; 戛; 戛; 戛; 戛; ) CJK COMPATIBILITY IDEOGRAPH-2F8B3 +2F8B4;625D;625D;625D;625D; # (扝扝; 扝; 扝; 扝; 扝; ) CJK COMPATIBILITY IDEOGRAPH-2F8B4 +2F8B5;62B1;62B1;62B1;62B1; # (抱抱; 抱; 抱; 抱; 抱; ) CJK COMPATIBILITY IDEOGRAPH-2F8B5 +2F8B6;62D4;62D4;62D4;62D4; # (拔拔; 拔; 拔; 拔; 拔; ) CJK COMPATIBILITY IDEOGRAPH-2F8B6 +2F8B7;6350;6350;6350;6350; # (捐捐; 捐; 捐; 捐; 捐; ) CJK COMPATIBILITY IDEOGRAPH-2F8B7 +2F8B8;22B0C;22B0C;22B0C;22B0C; # (𢬌𢬌; 𢬌𢬌; 𢬌𢬌; 𢬌𢬌; 𢬌𢬌; ) CJK COMPATIBILITY IDEOGRAPH-2F8B8 +2F8B9;633D;633D;633D;633D; # (挽挽; 挽; 挽; 挽; 挽; ) CJK COMPATIBILITY IDEOGRAPH-2F8B9 +2F8BA;62FC;62FC;62FC;62FC; # (拼拼; 拼; 拼; 拼; 拼; ) CJK COMPATIBILITY IDEOGRAPH-2F8BA +2F8BB;6368;6368;6368;6368; # (捨捨; 捨; 捨; 捨; 捨; ) CJK COMPATIBILITY IDEOGRAPH-2F8BB +2F8BC;6383;6383;6383;6383; # (掃掃; 掃; 掃; 掃; 掃; ) CJK COMPATIBILITY IDEOGRAPH-2F8BC +2F8BD;63E4;63E4;63E4;63E4; # (揤揤; 揤; 揤; 揤; 揤; ) CJK COMPATIBILITY IDEOGRAPH-2F8BD +2F8BE;22BF1;22BF1;22BF1;22BF1; # (𢯱𢯱; 𢯱𢯱; 𢯱𢯱; 𢯱𢯱; 𢯱𢯱; ) CJK COMPATIBILITY IDEOGRAPH-2F8BE +2F8BF;6422;6422;6422;6422; # (搢搢; 搢; 搢; 搢; 搢; ) CJK COMPATIBILITY IDEOGRAPH-2F8BF +2F8C0;63C5;63C5;63C5;63C5; # (揅揅; 揅; 揅; 揅; 揅; ) CJK COMPATIBILITY IDEOGRAPH-2F8C0 +2F8C1;63A9;63A9;63A9;63A9; # (掩掩; 掩; 掩; 掩; 掩; ) CJK COMPATIBILITY IDEOGRAPH-2F8C1 +2F8C2;3A2E;3A2E;3A2E;3A2E; # (㨮㨮; 㨮; 㨮; 㨮; 㨮; ) CJK COMPATIBILITY IDEOGRAPH-2F8C2 +2F8C3;6469;6469;6469;6469; # (摩摩; 摩; 摩; 摩; 摩; ) CJK COMPATIBILITY IDEOGRAPH-2F8C3 +2F8C4;647E;647E;647E;647E; # (摾摾; 摾; 摾; 摾; 摾; ) CJK COMPATIBILITY IDEOGRAPH-2F8C4 +2F8C5;649D;649D;649D;649D; # (撝撝; 撝; 撝; 撝; 撝; ) CJK COMPATIBILITY IDEOGRAPH-2F8C5 +2F8C6;6477;6477;6477;6477; # (摷摷; 摷; 摷; 摷; 摷; ) CJK COMPATIBILITY IDEOGRAPH-2F8C6 +2F8C7;3A6C;3A6C;3A6C;3A6C; # (㩬㩬; 㩬; 㩬; 㩬; 㩬; ) CJK COMPATIBILITY IDEOGRAPH-2F8C7 +2F8C8;654F;654F;654F;654F; # (敏敏; 敏; 敏; 敏; 敏; ) CJK COMPATIBILITY IDEOGRAPH-2F8C8 +2F8C9;656C;656C;656C;656C; # (敬敬; 敬; 敬; 敬; 敬; ) CJK COMPATIBILITY IDEOGRAPH-2F8C9 +2F8CA;2300A;2300A;2300A;2300A; # (𣀊𣀊; 𣀊𣀊; 𣀊𣀊; 𣀊𣀊; 𣀊𣀊; ) CJK COMPATIBILITY IDEOGRAPH-2F8CA +2F8CB;65E3;65E3;65E3;65E3; # (旣旣; 旣; 旣; 旣; 旣; ) CJK COMPATIBILITY IDEOGRAPH-2F8CB +2F8CC;66F8;66F8;66F8;66F8; # (書書; 書; 書; 書; 書; ) CJK COMPATIBILITY IDEOGRAPH-2F8CC +2F8CD;6649;6649;6649;6649; # (晉晉; 晉; 晉; 晉; 晉; ) CJK COMPATIBILITY IDEOGRAPH-2F8CD +2F8CE;3B19;3B19;3B19;3B19; # (㬙㬙; 㬙; 㬙; 㬙; 㬙; ) CJK COMPATIBILITY IDEOGRAPH-2F8CE +2F8CF;6691;6691;6691;6691; # (暑暑; 暑; 暑; 暑; 暑; ) CJK COMPATIBILITY IDEOGRAPH-2F8CF +2F8D0;3B08;3B08;3B08;3B08; # (㬈㬈; 㬈; 㬈; 㬈; 㬈; ) CJK COMPATIBILITY IDEOGRAPH-2F8D0 +2F8D1;3AE4;3AE4;3AE4;3AE4; # (㫤㫤; 㫤; 㫤; 㫤; 㫤; ) CJK COMPATIBILITY IDEOGRAPH-2F8D1 +2F8D2;5192;5192;5192;5192; # (冒冒; 冒; 冒; 冒; 冒; ) CJK COMPATIBILITY IDEOGRAPH-2F8D2 +2F8D3;5195;5195;5195;5195; # (冕冕; 冕; 冕; 冕; 冕; ) CJK COMPATIBILITY IDEOGRAPH-2F8D3 +2F8D4;6700;6700;6700;6700; # (最最; 最; 最; 最; 最; ) CJK COMPATIBILITY IDEOGRAPH-2F8D4 +2F8D5;669C;669C;669C;669C; # (暜暜; 暜; 暜; 暜; 暜; ) CJK COMPATIBILITY IDEOGRAPH-2F8D5 +2F8D6;80AD;80AD;80AD;80AD; # (肭肭; 肭; 肭; 肭; 肭; ) CJK COMPATIBILITY IDEOGRAPH-2F8D6 +2F8D7;43D9;43D9;43D9;43D9; # (䏙䏙; 䏙; 䏙; 䏙; 䏙; ) CJK COMPATIBILITY IDEOGRAPH-2F8D7 +2F8D8;6717;6717;6717;6717; # (朗朗; 朗; 朗; 朗; 朗; ) CJK COMPATIBILITY IDEOGRAPH-2F8D8 +2F8D9;671B;671B;671B;671B; # (望望; 望; 望; 望; 望; ) CJK COMPATIBILITY IDEOGRAPH-2F8D9 +2F8DA;6721;6721;6721;6721; # (朡朡; 朡; 朡; 朡; 朡; ) CJK COMPATIBILITY IDEOGRAPH-2F8DA +2F8DB;675E;675E;675E;675E; # (杞杞; 杞; 杞; 杞; 杞; ) CJK COMPATIBILITY IDEOGRAPH-2F8DB +2F8DC;6753;6753;6753;6753; # (杓杓; 杓; 杓; 杓; 杓; ) CJK COMPATIBILITY IDEOGRAPH-2F8DC +2F8DD;233C3;233C3;233C3;233C3; # (𣏃𣏃; 𣏃𣏃; 𣏃𣏃; 𣏃𣏃; 𣏃𣏃; ) CJK COMPATIBILITY IDEOGRAPH-2F8DD +2F8DE;3B49;3B49;3B49;3B49; # (㭉㭉; 㭉; 㭉; 㭉; 㭉; ) CJK COMPATIBILITY IDEOGRAPH-2F8DE +2F8DF;67FA;67FA;67FA;67FA; # (柺柺; 柺; 柺; 柺; 柺; ) CJK COMPATIBILITY IDEOGRAPH-2F8DF +2F8E0;6785;6785;6785;6785; # (枅枅; 枅; 枅; 枅; 枅; ) CJK COMPATIBILITY IDEOGRAPH-2F8E0 +2F8E1;6852;6852;6852;6852; # (桒桒; 桒; 桒; 桒; 桒; ) CJK COMPATIBILITY IDEOGRAPH-2F8E1 +2F8E2;6885;6885;6885;6885; # (梅梅; 梅; 梅; 梅; 梅; ) CJK COMPATIBILITY IDEOGRAPH-2F8E2 +2F8E3;2346D;2346D;2346D;2346D; # (𣑭𣑭; 𣑭𣑭; 𣑭𣑭; 𣑭𣑭; 𣑭𣑭; ) CJK COMPATIBILITY IDEOGRAPH-2F8E3 +2F8E4;688E;688E;688E;688E; # (梎梎; 梎; 梎; 梎; 梎; ) CJK COMPATIBILITY IDEOGRAPH-2F8E4 +2F8E5;681F;681F;681F;681F; # (栟栟; 栟; 栟; 栟; 栟; ) CJK COMPATIBILITY IDEOGRAPH-2F8E5 +2F8E6;6914;6914;6914;6914; # (椔椔; 椔; 椔; 椔; 椔; ) CJK COMPATIBILITY IDEOGRAPH-2F8E6 +2F8E7;3B9D;3B9D;3B9D;3B9D; # (㮝㮝; 㮝; 㮝; 㮝; 㮝; ) CJK COMPATIBILITY IDEOGRAPH-2F8E7 +2F8E8;6942;6942;6942;6942; # (楂楂; 楂; 楂; 楂; 楂; ) CJK COMPATIBILITY IDEOGRAPH-2F8E8 +2F8E9;69A3;69A3;69A3;69A3; # (榣榣; 榣; 榣; 榣; 榣; ) CJK COMPATIBILITY IDEOGRAPH-2F8E9 +2F8EA;69EA;69EA;69EA;69EA; # (槪槪; 槪; 槪; 槪; 槪; ) CJK COMPATIBILITY IDEOGRAPH-2F8EA +2F8EB;6AA8;6AA8;6AA8;6AA8; # (檨檨; 檨; 檨; 檨; 檨; ) CJK COMPATIBILITY IDEOGRAPH-2F8EB +2F8EC;236A3;236A3;236A3;236A3; # (𣚣𣚣; 𣚣𣚣; 𣚣𣚣; 𣚣𣚣; 𣚣𣚣; ) CJK COMPATIBILITY IDEOGRAPH-2F8EC +2F8ED;6ADB;6ADB;6ADB;6ADB; # (櫛櫛; 櫛; 櫛; 櫛; 櫛; ) CJK COMPATIBILITY IDEOGRAPH-2F8ED +2F8EE;3C18;3C18;3C18;3C18; # (㰘㰘; 㰘; 㰘; 㰘; 㰘; ) CJK COMPATIBILITY IDEOGRAPH-2F8EE +2F8EF;6B21;6B21;6B21;6B21; # (次次; 次; 次; 次; 次; ) CJK COMPATIBILITY IDEOGRAPH-2F8EF +2F8F0;238A7;238A7;238A7;238A7; # (𣢧𣢧; 𣢧𣢧; 𣢧𣢧; 𣢧𣢧; 𣢧𣢧; ) CJK COMPATIBILITY IDEOGRAPH-2F8F0 +2F8F1;6B54;6B54;6B54;6B54; # (歔歔; 歔; 歔; 歔; 歔; ) CJK COMPATIBILITY IDEOGRAPH-2F8F1 +2F8F2;3C4E;3C4E;3C4E;3C4E; # (㱎㱎; 㱎; 㱎; 㱎; 㱎; ) CJK COMPATIBILITY IDEOGRAPH-2F8F2 +2F8F3;6B72;6B72;6B72;6B72; # (歲歲; 歲; 歲; 歲; 歲; ) CJK COMPATIBILITY IDEOGRAPH-2F8F3 +2F8F4;6B9F;6B9F;6B9F;6B9F; # (殟殟; 殟; 殟; 殟; 殟; ) CJK COMPATIBILITY IDEOGRAPH-2F8F4 +2F8F5;6BBA;6BBA;6BBA;6BBA; # (殺殺; 殺; 殺; 殺; 殺; ) CJK COMPATIBILITY IDEOGRAPH-2F8F5 +2F8F6;6BBB;6BBB;6BBB;6BBB; # (殻殻; 殻; 殻; 殻; 殻; ) CJK COMPATIBILITY IDEOGRAPH-2F8F6 +2F8F7;23A8D;23A8D;23A8D;23A8D; # (𣪍𣪍; 𣪍𣪍; 𣪍𣪍; 𣪍𣪍; 𣪍𣪍; ) CJK COMPATIBILITY IDEOGRAPH-2F8F7 +2F8F8;21D0B;21D0B;21D0B;21D0B; # (𡴋𡴋; 𡴋𡴋; 𡴋𡴋; 𡴋𡴋; 𡴋𡴋; ) CJK COMPATIBILITY IDEOGRAPH-2F8F8 +2F8F9;23AFA;23AFA;23AFA;23AFA; # (𣫺𣫺; 𣫺𣫺; 𣫺𣫺; 𣫺𣫺; 𣫺𣫺; ) CJK COMPATIBILITY IDEOGRAPH-2F8F9 +2F8FA;6C4E;6C4E;6C4E;6C4E; # (汎汎; 汎; 汎; 汎; 汎; ) CJK COMPATIBILITY IDEOGRAPH-2F8FA +2F8FB;23CBC;23CBC;23CBC;23CBC; # (𣲼𣲼; 𣲼𣲼; 𣲼𣲼; 𣲼𣲼; 𣲼𣲼; ) CJK COMPATIBILITY IDEOGRAPH-2F8FB +2F8FC;6CBF;6CBF;6CBF;6CBF; # (沿沿; 沿; 沿; 沿; 沿; ) CJK COMPATIBILITY IDEOGRAPH-2F8FC +2F8FD;6CCD;6CCD;6CCD;6CCD; # (泍泍; 泍; 泍; 泍; 泍; ) CJK COMPATIBILITY IDEOGRAPH-2F8FD +2F8FE;6C67;6C67;6C67;6C67; # (汧汧; 汧; 汧; 汧; 汧; ) CJK COMPATIBILITY IDEOGRAPH-2F8FE +2F8FF;6D16;6D16;6D16;6D16; # (洖洖; 洖; 洖; 洖; 洖; ) CJK COMPATIBILITY IDEOGRAPH-2F8FF +2F900;6D3E;6D3E;6D3E;6D3E; # (派派; 派; 派; 派; 派; ) CJK COMPATIBILITY IDEOGRAPH-2F900 +2F901;6D77;6D77;6D77;6D77; # (海海; 海; 海; 海; 海; ) CJK COMPATIBILITY IDEOGRAPH-2F901 +2F902;6D41;6D41;6D41;6D41; # (流流; 流; 流; 流; 流; ) CJK COMPATIBILITY IDEOGRAPH-2F902 +2F903;6D69;6D69;6D69;6D69; # (浩浩; 浩; 浩; 浩; 浩; ) CJK COMPATIBILITY IDEOGRAPH-2F903 +2F904;6D78;6D78;6D78;6D78; # (浸浸; 浸; 浸; 浸; 浸; ) CJK COMPATIBILITY IDEOGRAPH-2F904 +2F905;6D85;6D85;6D85;6D85; # (涅涅; 涅; 涅; 涅; 涅; ) CJK COMPATIBILITY IDEOGRAPH-2F905 +2F906;23D1E;23D1E;23D1E;23D1E; # (𣴞𣴞; 𣴞𣴞; 𣴞𣴞; 𣴞𣴞; 𣴞𣴞; ) CJK COMPATIBILITY IDEOGRAPH-2F906 +2F907;6D34;6D34;6D34;6D34; # (洴洴; 洴; 洴; 洴; 洴; ) CJK COMPATIBILITY IDEOGRAPH-2F907 +2F908;6E2F;6E2F;6E2F;6E2F; # (港港; 港; 港; 港; 港; ) CJK COMPATIBILITY IDEOGRAPH-2F908 +2F909;6E6E;6E6E;6E6E;6E6E; # (湮湮; 湮; 湮; 湮; 湮; ) CJK COMPATIBILITY IDEOGRAPH-2F909 +2F90A;3D33;3D33;3D33;3D33; # (㴳㴳; 㴳; 㴳; 㴳; 㴳; ) CJK COMPATIBILITY IDEOGRAPH-2F90A +2F90B;6ECB;6ECB;6ECB;6ECB; # (滋滋; 滋; 滋; 滋; 滋; ) CJK COMPATIBILITY IDEOGRAPH-2F90B +2F90C;6EC7;6EC7;6EC7;6EC7; # (滇滇; 滇; 滇; 滇; 滇; ) CJK COMPATIBILITY IDEOGRAPH-2F90C +2F90D;23ED1;23ED1;23ED1;23ED1; # (𣻑𣻑; 𣻑𣻑; 𣻑𣻑; 𣻑𣻑; 𣻑𣻑; ) CJK COMPATIBILITY IDEOGRAPH-2F90D +2F90E;6DF9;6DF9;6DF9;6DF9; # (淹淹; 淹; 淹; 淹; 淹; ) CJK COMPATIBILITY IDEOGRAPH-2F90E +2F90F;6F6E;6F6E;6F6E;6F6E; # (潮潮; 潮; 潮; 潮; 潮; ) CJK COMPATIBILITY IDEOGRAPH-2F90F +2F910;23F5E;23F5E;23F5E;23F5E; # (𣽞𣽞; 𣽞𣽞; 𣽞𣽞; 𣽞𣽞; 𣽞𣽞; ) CJK COMPATIBILITY IDEOGRAPH-2F910 +2F911;23F8E;23F8E;23F8E;23F8E; # (𣾎𣾎; 𣾎𣾎; 𣾎𣾎; 𣾎𣾎; 𣾎𣾎; ) CJK COMPATIBILITY IDEOGRAPH-2F911 +2F912;6FC6;6FC6;6FC6;6FC6; # (濆濆; 濆; 濆; 濆; 濆; ) CJK COMPATIBILITY IDEOGRAPH-2F912 +2F913;7039;7039;7039;7039; # (瀹瀹; 瀹; 瀹; 瀹; 瀹; ) CJK COMPATIBILITY IDEOGRAPH-2F913 +2F914;701E;701E;701E;701E; # (瀞瀞; 瀞; 瀞; 瀞; 瀞; ) CJK COMPATIBILITY IDEOGRAPH-2F914 +2F915;701B;701B;701B;701B; # (瀛瀛; 瀛; 瀛; 瀛; 瀛; ) CJK COMPATIBILITY IDEOGRAPH-2F915 +2F916;3D96;3D96;3D96;3D96; # (㶖㶖; 㶖; 㶖; 㶖; 㶖; ) CJK COMPATIBILITY IDEOGRAPH-2F916 +2F917;704A;704A;704A;704A; # (灊灊; 灊; 灊; 灊; 灊; ) CJK COMPATIBILITY IDEOGRAPH-2F917 +2F918;707D;707D;707D;707D; # (災災; 災; 災; 災; 災; ) CJK COMPATIBILITY IDEOGRAPH-2F918 +2F919;7077;7077;7077;7077; # (灷灷; 灷; 灷; 灷; 灷; ) CJK COMPATIBILITY IDEOGRAPH-2F919 +2F91A;70AD;70AD;70AD;70AD; # (炭炭; 炭; 炭; 炭; 炭; ) CJK COMPATIBILITY IDEOGRAPH-2F91A +2F91B;20525;20525;20525;20525; # (𠔥𠔥; 𠔥𠔥; 𠔥𠔥; 𠔥𠔥; 𠔥𠔥; ) CJK COMPATIBILITY IDEOGRAPH-2F91B +2F91C;7145;7145;7145;7145; # (煅煅; 煅; 煅; 煅; 煅; ) CJK COMPATIBILITY IDEOGRAPH-2F91C +2F91D;24263;24263;24263;24263; # (𤉣𤉣; 𤉣𤉣; 𤉣𤉣; 𤉣𤉣; 𤉣𤉣; ) CJK COMPATIBILITY IDEOGRAPH-2F91D +2F91E;719C;719C;719C;719C; # (熜熜; 熜; 熜; 熜; 熜; ) CJK COMPATIBILITY IDEOGRAPH-2F91E +2F91F;43AB;43AB;43AB;43AB; # (𤎫𤎫; 䎫; 䎫; 䎫; 䎫; ) CJK COMPATIBILITY IDEOGRAPH-2F91F +2F920;7228;7228;7228;7228; # (爨爨; 爨; 爨; 爨; 爨; ) CJK COMPATIBILITY IDEOGRAPH-2F920 +2F921;7235;7235;7235;7235; # (爵爵; 爵; 爵; 爵; 爵; ) CJK COMPATIBILITY IDEOGRAPH-2F921 +2F922;7250;7250;7250;7250; # (牐牐; 牐; 牐; 牐; 牐; ) CJK COMPATIBILITY IDEOGRAPH-2F922 +2F923;24608;24608;24608;24608; # (𤘈𤘈; 𤘈𤘈; 𤘈𤘈; 𤘈𤘈; 𤘈𤘈; ) CJK COMPATIBILITY IDEOGRAPH-2F923 +2F924;7280;7280;7280;7280; # (犀犀; 犀; 犀; 犀; 犀; ) CJK COMPATIBILITY IDEOGRAPH-2F924 +2F925;7295;7295;7295;7295; # (犕犕; 犕; 犕; 犕; 犕; ) CJK COMPATIBILITY IDEOGRAPH-2F925 +2F926;24735;24735;24735;24735; # (𤜵𤜵; 𤜵𤜵; 𤜵𤜵; 𤜵𤜵; 𤜵𤜵; ) CJK COMPATIBILITY IDEOGRAPH-2F926 +2F927;24814;24814;24814;24814; # (𤠔𤠔; 𤠔𤠔; 𤠔𤠔; 𤠔𤠔; 𤠔𤠔; ) CJK COMPATIBILITY IDEOGRAPH-2F927 +2F928;737A;737A;737A;737A; # (獺獺; 獺; 獺; 獺; 獺; ) CJK COMPATIBILITY IDEOGRAPH-2F928 +2F929;738B;738B;738B;738B; # (王王; 王; 王; 王; 王; ) CJK COMPATIBILITY IDEOGRAPH-2F929 +2F92A;3EAC;3EAC;3EAC;3EAC; # (㺬㺬; 㺬; 㺬; 㺬; 㺬; ) CJK COMPATIBILITY IDEOGRAPH-2F92A +2F92B;73A5;73A5;73A5;73A5; # (玥玥; 玥; 玥; 玥; 玥; ) CJK COMPATIBILITY IDEOGRAPH-2F92B +2F92C;3EB8;3EB8;3EB8;3EB8; # (㺸㺸; 㺸; 㺸; 㺸; 㺸; ) CJK COMPATIBILITY IDEOGRAPH-2F92C +2F92D;3EB8;3EB8;3EB8;3EB8; # (㺸㺸; 㺸; 㺸; 㺸; 㺸; ) CJK COMPATIBILITY IDEOGRAPH-2F92D +2F92E;7447;7447;7447;7447; # (瑇瑇; 瑇; 瑇; 瑇; 瑇; ) CJK COMPATIBILITY IDEOGRAPH-2F92E +2F92F;745C;745C;745C;745C; # (瑜瑜; 瑜; 瑜; 瑜; 瑜; ) CJK COMPATIBILITY IDEOGRAPH-2F92F +2F930;7471;7471;7471;7471; # (瑱瑱; 瑱; 瑱; 瑱; 瑱; ) CJK COMPATIBILITY IDEOGRAPH-2F930 +2F931;7485;7485;7485;7485; # (璅璅; 璅; 璅; 璅; 璅; ) CJK COMPATIBILITY IDEOGRAPH-2F931 +2F932;74CA;74CA;74CA;74CA; # (瓊瓊; 瓊; 瓊; 瓊; 瓊; ) CJK COMPATIBILITY IDEOGRAPH-2F932 +2F933;3F1B;3F1B;3F1B;3F1B; # (㼛㼛; 㼛; 㼛; 㼛; 㼛; ) CJK COMPATIBILITY IDEOGRAPH-2F933 +2F934;7524;7524;7524;7524; # (甤甤; 甤; 甤; 甤; 甤; ) CJK COMPATIBILITY IDEOGRAPH-2F934 +2F935;24C36;24C36;24C36;24C36; # (𤰶𤰶; 𤰶𤰶; 𤰶𤰶; 𤰶𤰶; 𤰶𤰶; ) CJK COMPATIBILITY IDEOGRAPH-2F935 +2F936;753E;753E;753E;753E; # (甾甾; 甾; 甾; 甾; 甾; ) CJK COMPATIBILITY IDEOGRAPH-2F936 +2F937;24C92;24C92;24C92;24C92; # (𤲒𤲒; 𤲒𤲒; 𤲒𤲒; 𤲒𤲒; 𤲒𤲒; ) CJK COMPATIBILITY IDEOGRAPH-2F937 +2F938;7570;7570;7570;7570; # (異異; 異; 異; 異; 異; ) CJK COMPATIBILITY IDEOGRAPH-2F938 +2F939;2219F;2219F;2219F;2219F; # (𢆟𢆟; 𢆟𢆟; 𢆟𢆟; 𢆟𢆟; 𢆟𢆟; ) CJK COMPATIBILITY IDEOGRAPH-2F939 +2F93A;7610;7610;7610;7610; # (瘐瘐; 瘐; 瘐; 瘐; 瘐; ) CJK COMPATIBILITY IDEOGRAPH-2F93A +2F93B;24FA1;24FA1;24FA1;24FA1; # (𤾡𤾡; 𤾡𤾡; 𤾡𤾡; 𤾡𤾡; 𤾡𤾡; ) CJK COMPATIBILITY IDEOGRAPH-2F93B +2F93C;24FB8;24FB8;24FB8;24FB8; # (𤾸𤾸; 𤾸𤾸; 𤾸𤾸; 𤾸𤾸; 𤾸𤾸; ) CJK COMPATIBILITY IDEOGRAPH-2F93C +2F93D;25044;25044;25044;25044; # (𥁄𥁄; 𥁄𥁄; 𥁄𥁄; 𥁄𥁄; 𥁄𥁄; ) CJK COMPATIBILITY IDEOGRAPH-2F93D +2F93E;3FFC;3FFC;3FFC;3FFC; # (㿼㿼; 㿼; 㿼; 㿼; 㿼; ) CJK COMPATIBILITY IDEOGRAPH-2F93E +2F93F;4008;4008;4008;4008; # (䀈䀈; 䀈; 䀈; 䀈; 䀈; ) CJK COMPATIBILITY IDEOGRAPH-2F93F +2F940;76F4;76F4;76F4;76F4; # (直直; 直; 直; 直; 直; ) CJK COMPATIBILITY IDEOGRAPH-2F940 +2F941;250F3;250F3;250F3;250F3; # (𥃳𥃳; 𥃳𥃳; 𥃳𥃳; 𥃳𥃳; 𥃳𥃳; ) CJK COMPATIBILITY IDEOGRAPH-2F941 +2F942;250F2;250F2;250F2;250F2; # (𥃲𥃲; 𥃲𥃲; 𥃲𥃲; 𥃲𥃲; 𥃲𥃲; ) CJK COMPATIBILITY IDEOGRAPH-2F942 +2F943;25119;25119;25119;25119; # (𥄙𥄙; 𥄙𥄙; 𥄙𥄙; 𥄙𥄙; 𥄙𥄙; ) CJK COMPATIBILITY IDEOGRAPH-2F943 +2F944;25133;25133;25133;25133; # (𥄳𥄳; 𥄳𥄳; 𥄳𥄳; 𥄳𥄳; 𥄳𥄳; ) CJK COMPATIBILITY IDEOGRAPH-2F944 +2F945;771E;771E;771E;771E; # (眞眞; 眞; 眞; 眞; 眞; ) CJK COMPATIBILITY IDEOGRAPH-2F945 +2F946;771F;771F;771F;771F; # (真真; 真; 真; 真; 真; ) CJK COMPATIBILITY IDEOGRAPH-2F946 +2F947;771F;771F;771F;771F; # (真真; 真; 真; 真; 真; ) CJK COMPATIBILITY IDEOGRAPH-2F947 +2F948;774A;774A;774A;774A; # (睊睊; 睊; 睊; 睊; 睊; ) CJK COMPATIBILITY IDEOGRAPH-2F948 +2F949;4039;4039;4039;4039; # (䀹䀹; 䀹; 䀹; 䀹; 䀹; ) CJK COMPATIBILITY IDEOGRAPH-2F949 +2F94A;778B;778B;778B;778B; # (瞋瞋; 瞋; 瞋; 瞋; 瞋; ) CJK COMPATIBILITY IDEOGRAPH-2F94A +2F94B;4046;4046;4046;4046; # (䁆䁆; 䁆; 䁆; 䁆; 䁆; ) CJK COMPATIBILITY IDEOGRAPH-2F94B +2F94C;4096;4096;4096;4096; # (䂖䂖; 䂖; 䂖; 䂖; 䂖; ) CJK COMPATIBILITY IDEOGRAPH-2F94C +2F94D;2541D;2541D;2541D;2541D; # (𥐝𥐝; 𥐝𥐝; 𥐝𥐝; 𥐝𥐝; 𥐝𥐝; ) CJK COMPATIBILITY IDEOGRAPH-2F94D +2F94E;784E;784E;784E;784E; # (硎硎; 硎; 硎; 硎; 硎; ) CJK COMPATIBILITY IDEOGRAPH-2F94E +2F94F;788C;788C;788C;788C; # (碌碌; 碌; 碌; 碌; 碌; ) CJK COMPATIBILITY IDEOGRAPH-2F94F +2F950;78CC;78CC;78CC;78CC; # (磌磌; 磌; 磌; 磌; 磌; ) CJK COMPATIBILITY IDEOGRAPH-2F950 +2F951;40E3;40E3;40E3;40E3; # (䃣䃣; 䃣; 䃣; 䃣; 䃣; ) CJK COMPATIBILITY IDEOGRAPH-2F951 +2F952;25626;25626;25626;25626; # (𥘦𥘦; 𥘦𥘦; 𥘦𥘦; 𥘦𥘦; 𥘦𥘦; ) CJK COMPATIBILITY IDEOGRAPH-2F952 +2F953;7956;7956;7956;7956; # (祖祖; 祖; 祖; 祖; 祖; ) CJK COMPATIBILITY IDEOGRAPH-2F953 +2F954;2569A;2569A;2569A;2569A; # (𥚚𥚚; 𥚚𥚚; 𥚚𥚚; 𥚚𥚚; 𥚚𥚚; ) CJK COMPATIBILITY IDEOGRAPH-2F954 +2F955;256C5;256C5;256C5;256C5; # (𥛅𥛅; 𥛅𥛅; 𥛅𥛅; 𥛅𥛅; 𥛅𥛅; ) CJK COMPATIBILITY IDEOGRAPH-2F955 +2F956;798F;798F;798F;798F; # (福福; 福; 福; 福; 福; ) CJK COMPATIBILITY IDEOGRAPH-2F956 +2F957;79EB;79EB;79EB;79EB; # (秫秫; 秫; 秫; 秫; 秫; ) CJK COMPATIBILITY IDEOGRAPH-2F957 +2F958;412F;412F;412F;412F; # (䄯䄯; 䄯; 䄯; 䄯; 䄯; ) CJK COMPATIBILITY IDEOGRAPH-2F958 +2F959;7A40;7A40;7A40;7A40; # (穀穀; 穀; 穀; 穀; 穀; ) CJK COMPATIBILITY IDEOGRAPH-2F959 +2F95A;7A4A;7A4A;7A4A;7A4A; # (穊穊; 穊; 穊; 穊; 穊; ) CJK COMPATIBILITY IDEOGRAPH-2F95A +2F95B;7A4F;7A4F;7A4F;7A4F; # (穏穏; 穏; 穏; 穏; 穏; ) CJK COMPATIBILITY IDEOGRAPH-2F95B +2F95C;2597C;2597C;2597C;2597C; # (𥥼𥥼; 𥥼𥥼; 𥥼𥥼; 𥥼𥥼; 𥥼𥥼; ) CJK COMPATIBILITY IDEOGRAPH-2F95C +2F95D;25AA7;25AA7;25AA7;25AA7; # (𥪧𥪧; 𥪧𥪧; 𥪧𥪧; 𥪧𥪧; 𥪧𥪧; ) CJK COMPATIBILITY IDEOGRAPH-2F95D +2F95E;25AA7;25AA7;25AA7;25AA7; # (𥪧𥪧; 𥪧𥪧; 𥪧𥪧; 𥪧𥪧; 𥪧𥪧; ) CJK COMPATIBILITY IDEOGRAPH-2F95E +2F95F;7AAE;7AAE;7AAE;7AAE; # (竮竮; 窮; 窮; 窮; 窮; ) CJK COMPATIBILITY IDEOGRAPH-2F95F +2F960;4202;4202;4202;4202; # (䈂䈂; 䈂; 䈂; 䈂; 䈂; ) CJK COMPATIBILITY IDEOGRAPH-2F960 +2F961;25BAB;25BAB;25BAB;25BAB; # (𥮫𥮫; 𥮫𥮫; 𥮫𥮫; 𥮫𥮫; 𥮫𥮫; ) CJK COMPATIBILITY IDEOGRAPH-2F961 +2F962;7BC6;7BC6;7BC6;7BC6; # (篆篆; 篆; 篆; 篆; 篆; ) CJK COMPATIBILITY IDEOGRAPH-2F962 +2F963;7BC9;7BC9;7BC9;7BC9; # (築築; 築; 築; 築; 築; ) CJK COMPATIBILITY IDEOGRAPH-2F963 +2F964;4227;4227;4227;4227; # (䈧䈧; 䈧; 䈧; 䈧; 䈧; ) CJK COMPATIBILITY IDEOGRAPH-2F964 +2F965;25C80;25C80;25C80;25C80; # (𥲀𥲀; 𥲀𥲀; 𥲀𥲀; 𥲀𥲀; 𥲀𥲀; ) CJK COMPATIBILITY IDEOGRAPH-2F965 +2F966;7CD2;7CD2;7CD2;7CD2; # (糒糒; 糒; 糒; 糒; 糒; ) CJK COMPATIBILITY IDEOGRAPH-2F966 +2F967;42A0;42A0;42A0;42A0; # (䊠䊠; 䊠; 䊠; 䊠; 䊠; ) CJK COMPATIBILITY IDEOGRAPH-2F967 +2F968;7CE8;7CE8;7CE8;7CE8; # (糨糨; 糨; 糨; 糨; 糨; ) CJK COMPATIBILITY IDEOGRAPH-2F968 +2F969;7CE3;7CE3;7CE3;7CE3; # (糣糣; 糣; 糣; 糣; 糣; ) CJK COMPATIBILITY IDEOGRAPH-2F969 +2F96A;7D00;7D00;7D00;7D00; # (紀紀; 紀; 紀; 紀; 紀; ) CJK COMPATIBILITY IDEOGRAPH-2F96A +2F96B;25F86;25F86;25F86;25F86; # (𥾆𥾆; 𥾆𥾆; 𥾆𥾆; 𥾆𥾆; 𥾆𥾆; ) CJK COMPATIBILITY IDEOGRAPH-2F96B +2F96C;7D63;7D63;7D63;7D63; # (絣絣; 絣; 絣; 絣; 絣; ) CJK COMPATIBILITY IDEOGRAPH-2F96C +2F96D;4301;4301;4301;4301; # (䌁䌁; 䌁; 䌁; 䌁; 䌁; ) CJK COMPATIBILITY IDEOGRAPH-2F96D +2F96E;7DC7;7DC7;7DC7;7DC7; # (緇緇; 緇; 緇; 緇; 緇; ) CJK COMPATIBILITY IDEOGRAPH-2F96E +2F96F;7E02;7E02;7E02;7E02; # (縂縂; 縂; 縂; 縂; 縂; ) CJK COMPATIBILITY IDEOGRAPH-2F96F +2F970;7E45;7E45;7E45;7E45; # (繅繅; 繅; 繅; 繅; 繅; ) CJK COMPATIBILITY IDEOGRAPH-2F970 +2F971;4334;4334;4334;4334; # (䌴䌴; 䌴; 䌴; 䌴; 䌴; ) CJK COMPATIBILITY IDEOGRAPH-2F971 +2F972;26228;26228;26228;26228; # (𦈨𦈨; 𦈨𦈨; 𦈨𦈨; 𦈨𦈨; 𦈨𦈨; ) CJK COMPATIBILITY IDEOGRAPH-2F972 +2F973;26247;26247;26247;26247; # (𦉇𦉇; 𦉇𦉇; 𦉇𦉇; 𦉇𦉇; 𦉇𦉇; ) CJK COMPATIBILITY IDEOGRAPH-2F973 +2F974;4359;4359;4359;4359; # (䍙䍙; 䍙; 䍙; 䍙; 䍙; ) CJK COMPATIBILITY IDEOGRAPH-2F974 +2F975;262D9;262D9;262D9;262D9; # (𦋙𦋙; 𦋙𦋙; 𦋙𦋙; 𦋙𦋙; 𦋙𦋙; ) CJK COMPATIBILITY IDEOGRAPH-2F975 +2F976;7F7A;7F7A;7F7A;7F7A; # (罺罺; 罺; 罺; 罺; 罺; ) CJK COMPATIBILITY IDEOGRAPH-2F976 +2F977;2633E;2633E;2633E;2633E; # (𦌾𦌾; 𦌾𦌾; 𦌾𦌾; 𦌾𦌾; 𦌾𦌾; ) CJK COMPATIBILITY IDEOGRAPH-2F977 +2F978;7F95;7F95;7F95;7F95; # (羕羕; 羕; 羕; 羕; 羕; ) CJK COMPATIBILITY IDEOGRAPH-2F978 +2F979;7FFA;7FFA;7FFA;7FFA; # (翺翺; 翺; 翺; 翺; 翺; ) CJK COMPATIBILITY IDEOGRAPH-2F979 +2F97A;8005;8005;8005;8005; # (者者; 者; 者; 者; 者; ) CJK COMPATIBILITY IDEOGRAPH-2F97A +2F97B;264DA;264DA;264DA;264DA; # (𦓚𦓚; 𦓚𦓚; 𦓚𦓚; 𦓚𦓚; 𦓚𦓚; ) CJK COMPATIBILITY IDEOGRAPH-2F97B +2F97C;26523;26523;26523;26523; # (𦔣𦔣; 𦔣𦔣; 𦔣𦔣; 𦔣𦔣; 𦔣𦔣; ) CJK COMPATIBILITY IDEOGRAPH-2F97C +2F97D;8060;8060;8060;8060; # (聠聠; 聠; 聠; 聠; 聠; ) CJK COMPATIBILITY IDEOGRAPH-2F97D +2F97E;265A8;265A8;265A8;265A8; # (𦖨𦖨; 𦖨𦖨; 𦖨𦖨; 𦖨𦖨; 𦖨𦖨; ) CJK COMPATIBILITY IDEOGRAPH-2F97E +2F97F;8070;8070;8070;8070; # (聰聰; 聰; 聰; 聰; 聰; ) CJK COMPATIBILITY IDEOGRAPH-2F97F +2F980;2335F;2335F;2335F;2335F; # (𣍟𣍟; 𣍟𣍟; 𣍟𣍟; 𣍟𣍟; 𣍟𣍟; ) CJK COMPATIBILITY IDEOGRAPH-2F980 +2F981;43D5;43D5;43D5;43D5; # (䏕䏕; 䏕; 䏕; 䏕; 䏕; ) CJK COMPATIBILITY IDEOGRAPH-2F981 +2F982;80B2;80B2;80B2;80B2; # (育育; 育; 育; 育; 育; ) CJK COMPATIBILITY IDEOGRAPH-2F982 +2F983;8103;8103;8103;8103; # (脃脃; 脃; 脃; 脃; 脃; ) CJK COMPATIBILITY IDEOGRAPH-2F983 +2F984;440B;440B;440B;440B; # (䐋䐋; 䐋; 䐋; 䐋; 䐋; ) CJK COMPATIBILITY IDEOGRAPH-2F984 +2F985;813E;813E;813E;813E; # (脾脾; 脾; 脾; 脾; 脾; ) CJK COMPATIBILITY IDEOGRAPH-2F985 +2F986;5AB5;5AB5;5AB5;5AB5; # (媵媵; 媵; 媵; 媵; 媵; ) CJK COMPATIBILITY IDEOGRAPH-2F986 +2F987;267A7;267A7;267A7;267A7; # (𦞧𦞧; 𦞧𦞧; 𦞧𦞧; 𦞧𦞧; 𦞧𦞧; ) CJK COMPATIBILITY IDEOGRAPH-2F987 +2F988;267B5;267B5;267B5;267B5; # (𦞵𦞵; 𦞵𦞵; 𦞵𦞵; 𦞵𦞵; 𦞵𦞵; ) CJK COMPATIBILITY IDEOGRAPH-2F988 +2F989;23393;23393;23393;23393; # (𣎓𣎓; 𣎓𣎓; 𣎓𣎓; 𣎓𣎓; 𣎓𣎓; ) CJK COMPATIBILITY IDEOGRAPH-2F989 +2F98A;2339C;2339C;2339C;2339C; # (𣎜𣎜; 𣎜𣎜; 𣎜𣎜; 𣎜𣎜; 𣎜𣎜; ) CJK COMPATIBILITY IDEOGRAPH-2F98A +2F98B;8201;8201;8201;8201; # (舁舁; 舁; 舁; 舁; 舁; ) CJK COMPATIBILITY IDEOGRAPH-2F98B +2F98C;8204;8204;8204;8204; # (舄舄; 舄; 舄; 舄; 舄; ) CJK COMPATIBILITY IDEOGRAPH-2F98C +2F98D;8F9E;8F9E;8F9E;8F9E; # (辞辞; 辞; 辞; 辞; 辞; ) CJK COMPATIBILITY IDEOGRAPH-2F98D +2F98E;446B;446B;446B;446B; # (䑫䑫; 䑫; 䑫; 䑫; 䑫; ) CJK COMPATIBILITY IDEOGRAPH-2F98E +2F98F;8291;8291;8291;8291; # (芑芑; 芑; 芑; 芑; 芑; ) CJK COMPATIBILITY IDEOGRAPH-2F98F +2F990;828B;828B;828B;828B; # (芋芋; 芋; 芋; 芋; 芋; ) CJK COMPATIBILITY IDEOGRAPH-2F990 +2F991;829D;829D;829D;829D; # (芝芝; 芝; 芝; 芝; 芝; ) CJK COMPATIBILITY IDEOGRAPH-2F991 +2F992;52B3;52B3;52B3;52B3; # (劳劳; 劳; 劳; 劳; 劳; ) CJK COMPATIBILITY IDEOGRAPH-2F992 +2F993;82B1;82B1;82B1;82B1; # (花花; 花; 花; 花; 花; ) CJK COMPATIBILITY IDEOGRAPH-2F993 +2F994;82B3;82B3;82B3;82B3; # (芳芳; 芳; 芳; 芳; 芳; ) CJK COMPATIBILITY IDEOGRAPH-2F994 +2F995;82BD;82BD;82BD;82BD; # (芽芽; 芽; 芽; 芽; 芽; ) CJK COMPATIBILITY IDEOGRAPH-2F995 +2F996;82E6;82E6;82E6;82E6; # (苦苦; 苦; 苦; 苦; 苦; ) CJK COMPATIBILITY IDEOGRAPH-2F996 +2F997;26B3C;26B3C;26B3C;26B3C; # (𦬼𦬼; 𦬼𦬼; 𦬼𦬼; 𦬼𦬼; 𦬼𦬼; ) CJK COMPATIBILITY IDEOGRAPH-2F997 +2F998;82E5;82E5;82E5;82E5; # (若若; 若; 若; 若; 若; ) CJK COMPATIBILITY IDEOGRAPH-2F998 +2F999;831D;831D;831D;831D; # (茝茝; 茝; 茝; 茝; 茝; ) CJK COMPATIBILITY IDEOGRAPH-2F999 +2F99A;8363;8363;8363;8363; # (荣荣; 荣; 荣; 荣; 荣; ) CJK COMPATIBILITY IDEOGRAPH-2F99A +2F99B;83AD;83AD;83AD;83AD; # (莭莭; 莭; 莭; 莭; 莭; ) CJK COMPATIBILITY IDEOGRAPH-2F99B +2F99C;8323;8323;8323;8323; # (茣茣; 茣; 茣; 茣; 茣; ) CJK COMPATIBILITY IDEOGRAPH-2F99C +2F99D;83BD;83BD;83BD;83BD; # (莽莽; 莽; 莽; 莽; 莽; ) CJK COMPATIBILITY IDEOGRAPH-2F99D +2F99E;83E7;83E7;83E7;83E7; # (菧菧; 菧; 菧; 菧; 菧; ) CJK COMPATIBILITY IDEOGRAPH-2F99E +2F99F;8457;8457;8457;8457; # (著著; 著; 著; 著; 著; ) CJK COMPATIBILITY IDEOGRAPH-2F99F +2F9A0;8353;8353;8353;8353; # (荓荓; 荓; 荓; 荓; 荓; ) CJK COMPATIBILITY IDEOGRAPH-2F9A0 +2F9A1;83CA;83CA;83CA;83CA; # (菊菊; 菊; 菊; 菊; 菊; ) CJK COMPATIBILITY IDEOGRAPH-2F9A1 +2F9A2;83CC;83CC;83CC;83CC; # (菌菌; 菌; 菌; 菌; 菌; ) CJK COMPATIBILITY IDEOGRAPH-2F9A2 +2F9A3;83DC;83DC;83DC;83DC; # (菜菜; 菜; 菜; 菜; 菜; ) CJK COMPATIBILITY IDEOGRAPH-2F9A3 +2F9A4;26C36;26C36;26C36;26C36; # (𦰶𦰶; 𦰶𦰶; 𦰶𦰶; 𦰶𦰶; 𦰶𦰶; ) CJK COMPATIBILITY IDEOGRAPH-2F9A4 +2F9A5;26D6B;26D6B;26D6B;26D6B; # (𦵫𦵫; 𦵫𦵫; 𦵫𦵫; 𦵫𦵫; 𦵫𦵫; ) CJK COMPATIBILITY IDEOGRAPH-2F9A5 +2F9A6;26CD5;26CD5;26CD5;26CD5; # (𦳕𦳕; 𦳕𦳕; 𦳕𦳕; 𦳕𦳕; 𦳕𦳕; ) CJK COMPATIBILITY IDEOGRAPH-2F9A6 +2F9A7;452B;452B;452B;452B; # (䔫䔫; 䔫; 䔫; 䔫; 䔫; ) CJK COMPATIBILITY IDEOGRAPH-2F9A7 +2F9A8;84F1;84F1;84F1;84F1; # (蓱蓱; 蓱; 蓱; 蓱; 蓱; ) CJK COMPATIBILITY IDEOGRAPH-2F9A8 +2F9A9;84F3;84F3;84F3;84F3; # (蓳蓳; 蓳; 蓳; 蓳; 蓳; ) CJK COMPATIBILITY IDEOGRAPH-2F9A9 +2F9AA;8516;8516;8516;8516; # (蔖蔖; 蔖; 蔖; 蔖; 蔖; ) CJK COMPATIBILITY IDEOGRAPH-2F9AA +2F9AB;273CA;273CA;273CA;273CA; # (𧏊𧏊; 𧏊𧏊; 𧏊𧏊; 𧏊𧏊; 𧏊𧏊; ) CJK COMPATIBILITY IDEOGRAPH-2F9AB +2F9AC;8564;8564;8564;8564; # (蕤蕤; 蕤; 蕤; 蕤; 蕤; ) CJK COMPATIBILITY IDEOGRAPH-2F9AC +2F9AD;26F2C;26F2C;26F2C;26F2C; # (𦼬𦼬; 𦼬𦼬; 𦼬𦼬; 𦼬𦼬; 𦼬𦼬; ) CJK COMPATIBILITY IDEOGRAPH-2F9AD +2F9AE;455D;455D;455D;455D; # (䕝䕝; 䕝; 䕝; 䕝; 䕝; ) CJK COMPATIBILITY IDEOGRAPH-2F9AE +2F9AF;4561;4561;4561;4561; # (䕡䕡; 䕡; 䕡; 䕡; 䕡; ) CJK COMPATIBILITY IDEOGRAPH-2F9AF +2F9B0;26FB1;26FB1;26FB1;26FB1; # (𦾱𦾱; 𦾱𦾱; 𦾱𦾱; 𦾱𦾱; 𦾱𦾱; ) CJK COMPATIBILITY IDEOGRAPH-2F9B0 +2F9B1;270D2;270D2;270D2;270D2; # (𧃒𧃒; 𧃒𧃒; 𧃒𧃒; 𧃒𧃒; 𧃒𧃒; ) CJK COMPATIBILITY IDEOGRAPH-2F9B1 +2F9B2;456B;456B;456B;456B; # (䕫䕫; 䕫; 䕫; 䕫; 䕫; ) CJK COMPATIBILITY IDEOGRAPH-2F9B2 +2F9B3;8650;8650;8650;8650; # (虐虐; 虐; 虐; 虐; 虐; ) CJK COMPATIBILITY IDEOGRAPH-2F9B3 +2F9B4;865C;865C;865C;865C; # (虜虜; 虜; 虜; 虜; 虜; ) CJK COMPATIBILITY IDEOGRAPH-2F9B4 +2F9B5;8667;8667;8667;8667; # (虧虧; 虧; 虧; 虧; 虧; ) CJK COMPATIBILITY IDEOGRAPH-2F9B5 +2F9B6;8669;8669;8669;8669; # (虩虩; 虩; 虩; 虩; 虩; ) CJK COMPATIBILITY IDEOGRAPH-2F9B6 +2F9B7;86A9;86A9;86A9;86A9; # (蚩蚩; 蚩; 蚩; 蚩; 蚩; ) CJK COMPATIBILITY IDEOGRAPH-2F9B7 +2F9B8;8688;8688;8688;8688; # (蚈蚈; 蚈; 蚈; 蚈; 蚈; ) CJK COMPATIBILITY IDEOGRAPH-2F9B8 +2F9B9;870E;870E;870E;870E; # (蜎蜎; 蜎; 蜎; 蜎; 蜎; ) CJK COMPATIBILITY IDEOGRAPH-2F9B9 +2F9BA;86E2;86E2;86E2;86E2; # (蛢蛢; 蛢; 蛢; 蛢; 蛢; ) CJK COMPATIBILITY IDEOGRAPH-2F9BA +2F9BB;8779;8779;8779;8779; # (蝹蝹; 蝹; 蝹; 蝹; 蝹; ) CJK COMPATIBILITY IDEOGRAPH-2F9BB +2F9BC;8728;8728;8728;8728; # (蜨蜨; 蜨; 蜨; 蜨; 蜨; ) CJK COMPATIBILITY IDEOGRAPH-2F9BC +2F9BD;876B;876B;876B;876B; # (蝫蝫; 蝫; 蝫; 蝫; 蝫; ) CJK COMPATIBILITY IDEOGRAPH-2F9BD +2F9BE;8786;8786;8786;8786; # (螆螆; 螆; 螆; 螆; 螆; ) CJK COMPATIBILITY IDEOGRAPH-2F9BE +2F9BF;4D57;4D57;4D57;4D57; # (䗗䗗; 䵗; 䵗; 䵗; 䵗; ) CJK COMPATIBILITY IDEOGRAPH-2F9BF +2F9C0;87E1;87E1;87E1;87E1; # (蟡蟡; 蟡; 蟡; 蟡; 蟡; ) CJK COMPATIBILITY IDEOGRAPH-2F9C0 +2F9C1;8801;8801;8801;8801; # (蠁蠁; 蠁; 蠁; 蠁; 蠁; ) CJK COMPATIBILITY IDEOGRAPH-2F9C1 +2F9C2;45F9;45F9;45F9;45F9; # (䗹䗹; 䗹; 䗹; 䗹; 䗹; ) CJK COMPATIBILITY IDEOGRAPH-2F9C2 +2F9C3;8860;8860;8860;8860; # (衠衠; 衠; 衠; 衠; 衠; ) CJK COMPATIBILITY IDEOGRAPH-2F9C3 +2F9C4;8863;8863;8863;8863; # (衣衣; 衣; 衣; 衣; 衣; ) CJK COMPATIBILITY IDEOGRAPH-2F9C4 +2F9C5;27667;27667;27667;27667; # (𧙧𧙧; 𧙧𧙧; 𧙧𧙧; 𧙧𧙧; 𧙧𧙧; ) CJK COMPATIBILITY IDEOGRAPH-2F9C5 +2F9C6;88D7;88D7;88D7;88D7; # (裗裗; 裗; 裗; 裗; 裗; ) CJK COMPATIBILITY IDEOGRAPH-2F9C6 +2F9C7;88DE;88DE;88DE;88DE; # (裞裞; 裞; 裞; 裞; 裞; ) CJK COMPATIBILITY IDEOGRAPH-2F9C7 +2F9C8;4635;4635;4635;4635; # (䘵䘵; 䘵; 䘵; 䘵; 䘵; ) CJK COMPATIBILITY IDEOGRAPH-2F9C8 +2F9C9;88FA;88FA;88FA;88FA; # (裺裺; 裺; 裺; 裺; 裺; ) CJK COMPATIBILITY IDEOGRAPH-2F9C9 +2F9CA;34BB;34BB;34BB;34BB; # (㒻㒻; 㒻; 㒻; 㒻; 㒻; ) CJK COMPATIBILITY IDEOGRAPH-2F9CA +2F9CB;278AE;278AE;278AE;278AE; # (𧢮𧢮; 𧢮𧢮; 𧢮𧢮; 𧢮𧢮; 𧢮𧢮; ) CJK COMPATIBILITY IDEOGRAPH-2F9CB +2F9CC;27966;27966;27966;27966; # (𧥦𧥦; 𧥦𧥦; 𧥦𧥦; 𧥦𧥦; 𧥦𧥦; ) CJK COMPATIBILITY IDEOGRAPH-2F9CC +2F9CD;46BE;46BE;46BE;46BE; # (䚾䚾; 䚾; 䚾; 䚾; 䚾; ) CJK COMPATIBILITY IDEOGRAPH-2F9CD +2F9CE;46C7;46C7;46C7;46C7; # (䛇䛇; 䛇; 䛇; 䛇; 䛇; ) CJK COMPATIBILITY IDEOGRAPH-2F9CE +2F9CF;8AA0;8AA0;8AA0;8AA0; # (誠誠; 誠; 誠; 誠; 誠; ) CJK COMPATIBILITY IDEOGRAPH-2F9CF +2F9D0;8AED;8AED;8AED;8AED; # (諭諭; 諭; 諭; 諭; 諭; ) CJK COMPATIBILITY IDEOGRAPH-2F9D0 +2F9D1;8B8A;8B8A;8B8A;8B8A; # (變變; 變; 變; 變; 變; ) CJK COMPATIBILITY IDEOGRAPH-2F9D1 +2F9D2;8C55;8C55;8C55;8C55; # (豕豕; 豕; 豕; 豕; 豕; ) CJK COMPATIBILITY IDEOGRAPH-2F9D2 +2F9D3;27CA8;27CA8;27CA8;27CA8; # (𧲨𧲨; 𧲨𧲨; 𧲨𧲨; 𧲨𧲨; 𧲨𧲨; ) CJK COMPATIBILITY IDEOGRAPH-2F9D3 +2F9D4;8CAB;8CAB;8CAB;8CAB; # (貫貫; 貫; 貫; 貫; 貫; ) CJK COMPATIBILITY IDEOGRAPH-2F9D4 +2F9D5;8CC1;8CC1;8CC1;8CC1; # (賁賁; 賁; 賁; 賁; 賁; ) CJK COMPATIBILITY IDEOGRAPH-2F9D5 +2F9D6;8D1B;8D1B;8D1B;8D1B; # (贛贛; 贛; 贛; 贛; 贛; ) CJK COMPATIBILITY IDEOGRAPH-2F9D6 +2F9D7;8D77;8D77;8D77;8D77; # (起起; 起; 起; 起; 起; ) CJK COMPATIBILITY IDEOGRAPH-2F9D7 +2F9D8;27F2F;27F2F;27F2F;27F2F; # (𧼯𧼯; 𧼯𧼯; 𧼯𧼯; 𧼯𧼯; 𧼯𧼯; ) CJK COMPATIBILITY IDEOGRAPH-2F9D8 +2F9D9;20804;20804;20804;20804; # (𠠄𠠄; 𠠄𠠄; 𠠄𠠄; 𠠄𠠄; 𠠄𠠄; ) CJK COMPATIBILITY IDEOGRAPH-2F9D9 +2F9DA;8DCB;8DCB;8DCB;8DCB; # (跋跋; 跋; 跋; 跋; 跋; ) CJK COMPATIBILITY IDEOGRAPH-2F9DA +2F9DB;8DBC;8DBC;8DBC;8DBC; # (趼趼; 趼; 趼; 趼; 趼; ) CJK COMPATIBILITY IDEOGRAPH-2F9DB +2F9DC;8DF0;8DF0;8DF0;8DF0; # (跰跰; 跰; 跰; 跰; 跰; ) CJK COMPATIBILITY IDEOGRAPH-2F9DC +2F9DD;208DE;208DE;208DE;208DE; # (𠣞𠣞; 𠣞𠣞; 𠣞𠣞; 𠣞𠣞; 𠣞𠣞; ) CJK COMPATIBILITY IDEOGRAPH-2F9DD +2F9DE;8ED4;8ED4;8ED4;8ED4; # (軔軔; 軔; 軔; 軔; 軔; ) CJK COMPATIBILITY IDEOGRAPH-2F9DE +2F9DF;8F38;8F38;8F38;8F38; # (輸輸; 輸; 輸; 輸; 輸; ) CJK COMPATIBILITY IDEOGRAPH-2F9DF +2F9E0;285D2;285D2;285D2;285D2; # (𨗒𨗒; 𨗒𨗒; 𨗒𨗒; 𨗒𨗒; 𨗒𨗒; ) CJK COMPATIBILITY IDEOGRAPH-2F9E0 +2F9E1;285ED;285ED;285ED;285ED; # (𨗭𨗭; 𨗭𨗭; 𨗭𨗭; 𨗭𨗭; 𨗭𨗭; ) CJK COMPATIBILITY IDEOGRAPH-2F9E1 +2F9E2;9094;9094;9094;9094; # (邔邔; 邔; 邔; 邔; 邔; ) CJK COMPATIBILITY IDEOGRAPH-2F9E2 +2F9E3;90F1;90F1;90F1;90F1; # (郱郱; 郱; 郱; 郱; 郱; ) CJK COMPATIBILITY IDEOGRAPH-2F9E3 +2F9E4;9111;9111;9111;9111; # (鄑鄑; 鄑; 鄑; 鄑; 鄑; ) CJK COMPATIBILITY IDEOGRAPH-2F9E4 +2F9E5;2872E;2872E;2872E;2872E; # (𨜮𨜮; 𨜮𨜮; 𨜮𨜮; 𨜮𨜮; 𨜮𨜮; ) CJK COMPATIBILITY IDEOGRAPH-2F9E5 +2F9E6;911B;911B;911B;911B; # (鄛鄛; 鄛; 鄛; 鄛; 鄛; ) CJK COMPATIBILITY IDEOGRAPH-2F9E6 +2F9E7;9238;9238;9238;9238; # (鈸鈸; 鈸; 鈸; 鈸; 鈸; ) CJK COMPATIBILITY IDEOGRAPH-2F9E7 +2F9E8;92D7;92D7;92D7;92D7; # (鋗鋗; 鋗; 鋗; 鋗; 鋗; ) CJK COMPATIBILITY IDEOGRAPH-2F9E8 +2F9E9;92D8;92D8;92D8;92D8; # (鋘鋘; 鋘; 鋘; 鋘; 鋘; ) CJK COMPATIBILITY IDEOGRAPH-2F9E9 +2F9EA;927C;927C;927C;927C; # (鉼鉼; 鉼; 鉼; 鉼; 鉼; ) CJK COMPATIBILITY IDEOGRAPH-2F9EA +2F9EB;93F9;93F9;93F9;93F9; # (鏹鏹; 鏹; 鏹; 鏹; 鏹; ) CJK COMPATIBILITY IDEOGRAPH-2F9EB +2F9EC;9415;9415;9415;9415; # (鐕鐕; 鐕; 鐕; 鐕; 鐕; ) CJK COMPATIBILITY IDEOGRAPH-2F9EC +2F9ED;28BFA;28BFA;28BFA;28BFA; # (𨯺𨯺; 𨯺𨯺; 𨯺𨯺; 𨯺𨯺; 𨯺𨯺; ) CJK COMPATIBILITY IDEOGRAPH-2F9ED +2F9EE;958B;958B;958B;958B; # (開開; 開; 開; 開; 開; ) CJK COMPATIBILITY IDEOGRAPH-2F9EE +2F9EF;4995;4995;4995;4995; # (䦕䦕; 䦕; 䦕; 䦕; 䦕; ) CJK COMPATIBILITY IDEOGRAPH-2F9EF +2F9F0;95B7;95B7;95B7;95B7; # (閷閷; 閷; 閷; 閷; 閷; ) CJK COMPATIBILITY IDEOGRAPH-2F9F0 +2F9F1;28D77;28D77;28D77;28D77; # (𨵷𨵷; 𨵷𨵷; 𨵷𨵷; 𨵷𨵷; 𨵷𨵷; ) CJK COMPATIBILITY IDEOGRAPH-2F9F1 +2F9F2;49E6;49E6;49E6;49E6; # (䧦䧦; 䧦; 䧦; 䧦; 䧦; ) CJK COMPATIBILITY IDEOGRAPH-2F9F2 +2F9F3;96C3;96C3;96C3;96C3; # (雃雃; 雃; 雃; 雃; 雃; ) CJK COMPATIBILITY IDEOGRAPH-2F9F3 +2F9F4;5DB2;5DB2;5DB2;5DB2; # (嶲嶲; 嶲; 嶲; 嶲; 嶲; ) CJK COMPATIBILITY IDEOGRAPH-2F9F4 +2F9F5;9723;9723;9723;9723; # (霣霣; 霣; 霣; 霣; 霣; ) CJK COMPATIBILITY IDEOGRAPH-2F9F5 +2F9F6;29145;29145;29145;29145; # (𩅅𩅅; 𩅅𩅅; 𩅅𩅅; 𩅅𩅅; 𩅅𩅅; ) CJK COMPATIBILITY IDEOGRAPH-2F9F6 +2F9F7;2921A;2921A;2921A;2921A; # (𩈚𩈚; 𩈚𩈚; 𩈚𩈚; 𩈚𩈚; 𩈚𩈚; ) CJK COMPATIBILITY IDEOGRAPH-2F9F7 +2F9F8;4A6E;4A6E;4A6E;4A6E; # (䩮䩮; 䩮; 䩮; 䩮; 䩮; ) CJK COMPATIBILITY IDEOGRAPH-2F9F8 +2F9F9;4A76;4A76;4A76;4A76; # (䩶䩶; 䩶; 䩶; 䩶; 䩶; ) CJK COMPATIBILITY IDEOGRAPH-2F9F9 +2F9FA;97E0;97E0;97E0;97E0; # (韠韠; 韠; 韠; 韠; 韠; ) CJK COMPATIBILITY IDEOGRAPH-2F9FA +2F9FB;2940A;2940A;2940A;2940A; # (𩐊𩐊; 𩐊𩐊; 𩐊𩐊; 𩐊𩐊; 𩐊𩐊; ) CJK COMPATIBILITY IDEOGRAPH-2F9FB +2F9FC;4AB2;4AB2;4AB2;4AB2; # (䪲䪲; 䪲; 䪲; 䪲; 䪲; ) CJK COMPATIBILITY IDEOGRAPH-2F9FC +2F9FD;29496;29496;29496;29496; # (𩒖𩒖; 𩒖𩒖; 𩒖𩒖; 𩒖𩒖; 𩒖𩒖; ) CJK COMPATIBILITY IDEOGRAPH-2F9FD +2F9FE;980B;980B;980B;980B; # (頋頋; 頋; 頋; 頋; 頋; ) CJK COMPATIBILITY IDEOGRAPH-2F9FE +2F9FF;980B;980B;980B;980B; # (頋頋; 頋; 頋; 頋; 頋; ) CJK COMPATIBILITY IDEOGRAPH-2F9FF +2FA00;9829;9829;9829;9829; # (頩頩; 頩; 頩; 頩; 頩; ) CJK COMPATIBILITY IDEOGRAPH-2FA00 +2FA01;295B6;295B6;295B6;295B6; # (𩖶𩖶; 𩖶𩖶; 𩖶𩖶; 𩖶𩖶; 𩖶𩖶; ) CJK COMPATIBILITY IDEOGRAPH-2FA01 +2FA02;98E2;98E2;98E2;98E2; # (飢飢; 飢; 飢; 飢; 飢; ) CJK COMPATIBILITY IDEOGRAPH-2FA02 +2FA03;4B33;4B33;4B33;4B33; # (䬳䬳; 䬳; 䬳; 䬳; 䬳; ) CJK COMPATIBILITY IDEOGRAPH-2FA03 +2FA04;9929;9929;9929;9929; # (餩餩; 餩; 餩; 餩; 餩; ) CJK COMPATIBILITY IDEOGRAPH-2FA04 +2FA05;99A7;99A7;99A7;99A7; # (馧馧; 馧; 馧; 馧; 馧; ) CJK COMPATIBILITY IDEOGRAPH-2FA05 +2FA06;99C2;99C2;99C2;99C2; # (駂駂; 駂; 駂; 駂; 駂; ) CJK COMPATIBILITY IDEOGRAPH-2FA06 +2FA07;99FE;99FE;99FE;99FE; # (駾駾; 駾; 駾; 駾; 駾; ) CJK COMPATIBILITY IDEOGRAPH-2FA07 +2FA08;4BCE;4BCE;4BCE;4BCE; # (䯎䯎; 䯎; 䯎; 䯎; 䯎; ) CJK COMPATIBILITY IDEOGRAPH-2FA08 +2FA09;29B30;29B30;29B30;29B30; # (𩬰𩬰; 𩬰𩬰; 𩬰𩬰; 𩬰𩬰; 𩬰𩬰; ) CJK COMPATIBILITY IDEOGRAPH-2FA09 +2FA0A;9B12;9B12;9B12;9B12; # (鬒鬒; 鬒; 鬒; 鬒; 鬒; ) CJK COMPATIBILITY IDEOGRAPH-2FA0A +2FA0B;9C40;9C40;9C40;9C40; # (鱀鱀; 鱀; 鱀; 鱀; 鱀; ) CJK COMPATIBILITY IDEOGRAPH-2FA0B +2FA0C;9CFD;9CFD;9CFD;9CFD; # (鳽鳽; 鳽; 鳽; 鳽; 鳽; ) CJK COMPATIBILITY IDEOGRAPH-2FA0C +2FA0D;4CCE;4CCE;4CCE;4CCE; # (䳎䳎; 䳎; 䳎; 䳎; 䳎; ) CJK COMPATIBILITY IDEOGRAPH-2FA0D +2FA0E;4CED;4CED;4CED;4CED; # (䳭䳭; 䳭; 䳭; 䳭; 䳭; ) CJK COMPATIBILITY IDEOGRAPH-2FA0E +2FA0F;9D67;9D67;9D67;9D67; # (鵧鵧; 鵧; 鵧; 鵧; 鵧; ) CJK COMPATIBILITY IDEOGRAPH-2FA0F +2FA10;2A0CE;2A0CE;2A0CE;2A0CE; # (𪃎𪃎; 𪃎𪃎; 𪃎𪃎; 𪃎𪃎; 𪃎𪃎; ) CJK COMPATIBILITY IDEOGRAPH-2FA10 +2FA11;4CF8;4CF8;4CF8;4CF8; # (䳸䳸; 䳸; 䳸; 䳸; 䳸; ) CJK COMPATIBILITY IDEOGRAPH-2FA11 +2FA12;2A105;2A105;2A105;2A105; # (𪄅𪄅; 𪄅𪄅; 𪄅𪄅; 𪄅𪄅; 𪄅𪄅; ) CJK COMPATIBILITY IDEOGRAPH-2FA12 +2FA13;2A20E;2A20E;2A20E;2A20E; # (𪈎𪈎; 𪈎𪈎; 𪈎𪈎; 𪈎𪈎; 𪈎𪈎; ) CJK COMPATIBILITY IDEOGRAPH-2FA13 +2FA14;2A291;2A291;2A291;2A291; # (𪊑𪊑; 𪊑𪊑; 𪊑𪊑; 𪊑𪊑; 𪊑𪊑; ) CJK COMPATIBILITY IDEOGRAPH-2FA14 +2FA15;9EBB;9EBB;9EBB;9EBB; # (麻麻; 麻; 麻; 麻; 麻; ) CJK COMPATIBILITY IDEOGRAPH-2FA15 +2FA16;4D56;4D56;4D56;4D56; # (䵖䵖; 䵖; 䵖; 䵖; 䵖; ) CJK COMPATIBILITY IDEOGRAPH-2FA16 +2FA17;9EF9;9EF9;9EF9;9EF9; # (黹黹; 黹; 黹; 黹; 黹; ) CJK COMPATIBILITY IDEOGRAPH-2FA17 +2FA18;9EFE;9EFE;9EFE;9EFE; # (黾黾; 黾; 黾; 黾; 黾; ) CJK COMPATIBILITY IDEOGRAPH-2FA18 +2FA19;9F05;9F05;9F05;9F05; # (鼅鼅; 鼅; 鼅; 鼅; 鼅; ) CJK COMPATIBILITY IDEOGRAPH-2FA19 +2FA1A;9F0F;9F0F;9F0F;9F0F; # (鼏鼏; 鼏; 鼏; 鼏; 鼏; ) CJK COMPATIBILITY IDEOGRAPH-2FA1A +2FA1B;9F16;9F16;9F16;9F16; # (鼖鼖; 鼖; 鼖; 鼖; 鼖; ) CJK COMPATIBILITY IDEOGRAPH-2FA1B +2FA1C;9F3B;9F3B;9F3B;9F3B; # (鼻鼻; 鼻; 鼻; 鼻; 鼻; ) CJK COMPATIBILITY IDEOGRAPH-2FA1C +2FA1D;2A600;2A600;2A600;2A600; # (𪘀𪘀; 𪘀𪘀; 𪘀𪘀; 𪘀𪘀; 𪘀𪘀; ) CJK COMPATIBILITY IDEOGRAPH-2FA1D +# +@Part2 # Canonical Order Test +# +0061 0315 0300 05AE 0300 0062;00E0 05AE 0300 0315 0062;0061 05AE 0300 0300 0315 0062;00E0 05AE 0300 0315 0062;0061 05AE 0300 0300 0315 0062; # (a◌̕◌̀◌֮◌̀b; à◌֮◌̀◌̕b; a◌֮◌̀◌̀◌̕b; à◌֮◌̀◌̕b; a◌֮◌̀◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING GRAVE ACCENT, LATIN SMALL LETTER B +0061 0300 0315 0300 05AE 0062;00E0 05AE 0300 0315 0062;0061 05AE 0300 0300 0315 0062;00E0 05AE 0300 0315 0062;0061 05AE 0300 0300 0315 0062; # (a◌̀◌̕◌̀◌֮b; à◌֮◌̀◌̕b; a◌֮◌̀◌̀◌̕b; à◌֮◌̀◌̕b; a◌֮◌̀◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING GRAVE ACCENT, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0301 0062;00E0 05AE 0301 0315 0062;0061 05AE 0300 0301 0315 0062;00E0 05AE 0301 0315 0062;0061 05AE 0300 0301 0315 0062; # (a◌̕◌̀◌֮◌́b; à◌֮◌́◌̕b; a◌֮◌̀◌́◌̕b; à◌֮◌́◌̕b; a◌֮◌̀◌́◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING ACUTE ACCENT, LATIN SMALL LETTER B +0061 0301 0315 0300 05AE 0062;00E1 05AE 0300 0315 0062;0061 05AE 0301 0300 0315 0062;00E1 05AE 0300 0315 0062;0061 05AE 0301 0300 0315 0062; # (a◌́◌̕◌̀◌֮b; á◌֮◌̀◌̕b; a◌֮◌́◌̀◌̕b; á◌֮◌̀◌̕b; a◌֮◌́◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING ACUTE ACCENT, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0302 0062;00E0 05AE 0302 0315 0062;0061 05AE 0300 0302 0315 0062;00E0 05AE 0302 0315 0062;0061 05AE 0300 0302 0315 0062; # (a◌̕◌̀◌֮◌̂b; à◌֮◌̂◌̕b; a◌֮◌̀◌̂◌̕b; à◌֮◌̂◌̕b; a◌֮◌̀◌̂◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING CIRCUMFLEX ACCENT, LATIN SMALL LETTER B +0061 0302 0315 0300 05AE 0062;1EA7 05AE 0315 0062;0061 05AE 0302 0300 0315 0062;1EA7 05AE 0315 0062;0061 05AE 0302 0300 0315 0062; # (a◌̂◌̕◌̀◌֮b; ầ◌֮◌̕b; a◌֮◌̂◌̀◌̕b; ầ◌֮◌̕b; a◌֮◌̂◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING CIRCUMFLEX ACCENT, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0303 0062;00E0 05AE 0303 0315 0062;0061 05AE 0300 0303 0315 0062;00E0 05AE 0303 0315 0062;0061 05AE 0300 0303 0315 0062; # (a◌̕◌̀◌֮◌̃b; à◌֮◌̃◌̕b; a◌֮◌̀◌̃◌̕b; à◌֮◌̃◌̕b; a◌֮◌̀◌̃◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING TILDE, LATIN SMALL LETTER B +0061 0303 0315 0300 05AE 0062;00E3 05AE 0300 0315 0062;0061 05AE 0303 0300 0315 0062;00E3 05AE 0300 0315 0062;0061 05AE 0303 0300 0315 0062; # (a◌̃◌̕◌̀◌֮b; ã◌֮◌̀◌̕b; a◌֮◌̃◌̀◌̕b; ã◌֮◌̀◌̕b; a◌֮◌̃◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING TILDE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0304 0062;00E0 05AE 0304 0315 0062;0061 05AE 0300 0304 0315 0062;00E0 05AE 0304 0315 0062;0061 05AE 0300 0304 0315 0062; # (a◌̕◌̀◌֮◌̄b; à◌֮◌̄◌̕b; a◌֮◌̀◌̄◌̕b; à◌֮◌̄◌̕b; a◌֮◌̀◌̄◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING MACRON, LATIN SMALL LETTER B +0061 0304 0315 0300 05AE 0062;0101 05AE 0300 0315 0062;0061 05AE 0304 0300 0315 0062;0101 05AE 0300 0315 0062;0061 05AE 0304 0300 0315 0062; # (a◌̄◌̕◌̀◌֮b; ā◌֮◌̀◌̕b; a◌֮◌̄◌̀◌̕b; ā◌֮◌̀◌̕b; a◌֮◌̄◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING MACRON, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0305 0062;00E0 05AE 0305 0315 0062;0061 05AE 0300 0305 0315 0062;00E0 05AE 0305 0315 0062;0061 05AE 0300 0305 0315 0062; # (a◌̕◌̀◌֮◌̅b; à◌֮◌̅◌̕b; a◌֮◌̀◌̅◌̕b; à◌֮◌̅◌̕b; a◌֮◌̀◌̅◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING OVERLINE, LATIN SMALL LETTER B +0061 0305 0315 0300 05AE 0062;0061 05AE 0305 0300 0315 0062;0061 05AE 0305 0300 0315 0062;0061 05AE 0305 0300 0315 0062;0061 05AE 0305 0300 0315 0062; # (a◌̅◌̕◌̀◌֮b; a◌֮◌̅◌̀◌̕b; a◌֮◌̅◌̀◌̕b; a◌֮◌̅◌̀◌̕b; a◌֮◌̅◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING OVERLINE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0306 0062;00E0 05AE 0306 0315 0062;0061 05AE 0300 0306 0315 0062;00E0 05AE 0306 0315 0062;0061 05AE 0300 0306 0315 0062; # (a◌̕◌̀◌֮◌̆b; à◌֮◌̆◌̕b; a◌֮◌̀◌̆◌̕b; à◌֮◌̆◌̕b; a◌֮◌̀◌̆◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING BREVE, LATIN SMALL LETTER B +0061 0306 0315 0300 05AE 0062;1EB1 05AE 0315 0062;0061 05AE 0306 0300 0315 0062;1EB1 05AE 0315 0062;0061 05AE 0306 0300 0315 0062; # (a◌̆◌̕◌̀◌֮b; ằ◌֮◌̕b; a◌֮◌̆◌̀◌̕b; ằ◌֮◌̕b; a◌֮◌̆◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING BREVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0307 0062;00E0 05AE 0307 0315 0062;0061 05AE 0300 0307 0315 0062;00E0 05AE 0307 0315 0062;0061 05AE 0300 0307 0315 0062; # (a◌̕◌̀◌֮◌̇b; à◌֮◌̇◌̕b; a◌֮◌̀◌̇◌̕b; à◌֮◌̇◌̕b; a◌֮◌̀◌̇◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING DOT ABOVE, LATIN SMALL LETTER B +0061 0307 0315 0300 05AE 0062;0227 05AE 0300 0315 0062;0061 05AE 0307 0300 0315 0062;0227 05AE 0300 0315 0062;0061 05AE 0307 0300 0315 0062; # (a◌̇◌̕◌̀◌֮b; ȧ◌֮◌̀◌̕b; a◌֮◌̇◌̀◌̕b; ȧ◌֮◌̀◌̕b; a◌֮◌̇◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING DOT ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0308 0062;00E0 05AE 0308 0315 0062;0061 05AE 0300 0308 0315 0062;00E0 05AE 0308 0315 0062;0061 05AE 0300 0308 0315 0062; # (a◌̕◌̀◌֮◌̈b; à◌֮◌̈◌̕b; a◌֮◌̀◌̈◌̕b; à◌֮◌̈◌̕b; a◌֮◌̀◌̈◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING DIAERESIS, LATIN SMALL LETTER B +0061 0308 0315 0300 05AE 0062;00E4 05AE 0300 0315 0062;0061 05AE 0308 0300 0315 0062;00E4 05AE 0300 0315 0062;0061 05AE 0308 0300 0315 0062; # (a◌̈◌̕◌̀◌֮b; ä◌֮◌̀◌̕b; a◌֮◌̈◌̀◌̕b; ä◌֮◌̀◌̕b; a◌֮◌̈◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING DIAERESIS, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0309 0062;00E0 05AE 0309 0315 0062;0061 05AE 0300 0309 0315 0062;00E0 05AE 0309 0315 0062;0061 05AE 0300 0309 0315 0062; # (a◌̕◌̀◌֮◌̉b; à◌֮◌̉◌̕b; a◌֮◌̀◌̉◌̕b; à◌֮◌̉◌̕b; a◌֮◌̀◌̉◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING HOOK ABOVE, LATIN SMALL LETTER B +0061 0309 0315 0300 05AE 0062;1EA3 05AE 0300 0315 0062;0061 05AE 0309 0300 0315 0062;1EA3 05AE 0300 0315 0062;0061 05AE 0309 0300 0315 0062; # (a◌̉◌̕◌̀◌֮b; ả◌֮◌̀◌̕b; a◌֮◌̉◌̀◌̕b; ả◌֮◌̀◌̕b; a◌֮◌̉◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING HOOK ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 030A 0062;00E0 05AE 030A 0315 0062;0061 05AE 0300 030A 0315 0062;00E0 05AE 030A 0315 0062;0061 05AE 0300 030A 0315 0062; # (a◌̕◌̀◌֮◌̊b; à◌֮◌̊◌̕b; a◌֮◌̀◌̊◌̕b; à◌֮◌̊◌̕b; a◌֮◌̀◌̊◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING RING ABOVE, LATIN SMALL LETTER B +0061 030A 0315 0300 05AE 0062;00E5 05AE 0300 0315 0062;0061 05AE 030A 0300 0315 0062;00E5 05AE 0300 0315 0062;0061 05AE 030A 0300 0315 0062; # (a◌̊◌̕◌̀◌֮b; å◌֮◌̀◌̕b; a◌֮◌̊◌̀◌̕b; å◌֮◌̀◌̕b; a◌֮◌̊◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING RING ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 030B 0062;00E0 05AE 030B 0315 0062;0061 05AE 0300 030B 0315 0062;00E0 05AE 030B 0315 0062;0061 05AE 0300 030B 0315 0062; # (a◌̕◌̀◌֮◌̋b; à◌֮◌̋◌̕b; a◌֮◌̀◌̋◌̕b; à◌֮◌̋◌̕b; a◌֮◌̀◌̋◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING DOUBLE ACUTE ACCENT, LATIN SMALL LETTER B +0061 030B 0315 0300 05AE 0062;0061 05AE 030B 0300 0315 0062;0061 05AE 030B 0300 0315 0062;0061 05AE 030B 0300 0315 0062;0061 05AE 030B 0300 0315 0062; # (a◌̋◌̕◌̀◌֮b; a◌֮◌̋◌̀◌̕b; a◌֮◌̋◌̀◌̕b; a◌֮◌̋◌̀◌̕b; a◌֮◌̋◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING DOUBLE ACUTE ACCENT, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 030C 0062;00E0 05AE 030C 0315 0062;0061 05AE 0300 030C 0315 0062;00E0 05AE 030C 0315 0062;0061 05AE 0300 030C 0315 0062; # (a◌̕◌̀◌֮◌̌b; à◌֮◌̌◌̕b; a◌֮◌̀◌̌◌̕b; à◌֮◌̌◌̕b; a◌֮◌̀◌̌◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING CARON, LATIN SMALL LETTER B +0061 030C 0315 0300 05AE 0062;01CE 05AE 0300 0315 0062;0061 05AE 030C 0300 0315 0062;01CE 05AE 0300 0315 0062;0061 05AE 030C 0300 0315 0062; # (a◌̌◌̕◌̀◌֮b; ǎ◌֮◌̀◌̕b; a◌֮◌̌◌̀◌̕b; ǎ◌֮◌̀◌̕b; a◌֮◌̌◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING CARON, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 030D 0062;00E0 05AE 030D 0315 0062;0061 05AE 0300 030D 0315 0062;00E0 05AE 030D 0315 0062;0061 05AE 0300 030D 0315 0062; # (a◌̕◌̀◌֮◌̍b; à◌֮◌̍◌̕b; a◌֮◌̀◌̍◌̕b; à◌֮◌̍◌̕b; a◌֮◌̀◌̍◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING VERTICAL LINE ABOVE, LATIN SMALL LETTER B +0061 030D 0315 0300 05AE 0062;0061 05AE 030D 0300 0315 0062;0061 05AE 030D 0300 0315 0062;0061 05AE 030D 0300 0315 0062;0061 05AE 030D 0300 0315 0062; # (a◌̍◌̕◌̀◌֮b; a◌֮◌̍◌̀◌̕b; a◌֮◌̍◌̀◌̕b; a◌֮◌̍◌̀◌̕b; a◌֮◌̍◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING VERTICAL LINE ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 030E 0062;00E0 05AE 030E 0315 0062;0061 05AE 0300 030E 0315 0062;00E0 05AE 030E 0315 0062;0061 05AE 0300 030E 0315 0062; # (a◌̕◌̀◌֮◌̎b; à◌֮◌̎◌̕b; a◌֮◌̀◌̎◌̕b; à◌֮◌̎◌̕b; a◌֮◌̀◌̎◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING DOUBLE VERTICAL LINE ABOVE, LATIN SMALL LETTER B +0061 030E 0315 0300 05AE 0062;0061 05AE 030E 0300 0315 0062;0061 05AE 030E 0300 0315 0062;0061 05AE 030E 0300 0315 0062;0061 05AE 030E 0300 0315 0062; # (a◌̎◌̕◌̀◌֮b; a◌֮◌̎◌̀◌̕b; a◌֮◌̎◌̀◌̕b; a◌֮◌̎◌̀◌̕b; a◌֮◌̎◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING DOUBLE VERTICAL LINE ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 030F 0062;00E0 05AE 030F 0315 0062;0061 05AE 0300 030F 0315 0062;00E0 05AE 030F 0315 0062;0061 05AE 0300 030F 0315 0062; # (a◌̕◌̀◌֮◌̏b; à◌֮◌̏◌̕b; a◌֮◌̀◌̏◌̕b; à◌֮◌̏◌̕b; a◌֮◌̀◌̏◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING DOUBLE GRAVE ACCENT, LATIN SMALL LETTER B +0061 030F 0315 0300 05AE 0062;0201 05AE 0300 0315 0062;0061 05AE 030F 0300 0315 0062;0201 05AE 0300 0315 0062;0061 05AE 030F 0300 0315 0062; # (a◌̏◌̕◌̀◌֮b; ȁ◌֮◌̀◌̕b; a◌֮◌̏◌̀◌̕b; ȁ◌֮◌̀◌̕b; a◌֮◌̏◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING DOUBLE GRAVE ACCENT, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0310 0062;00E0 05AE 0310 0315 0062;0061 05AE 0300 0310 0315 0062;00E0 05AE 0310 0315 0062;0061 05AE 0300 0310 0315 0062; # (a◌̕◌̀◌֮◌̐b; à◌֮◌̐◌̕b; a◌֮◌̀◌̐◌̕b; à◌֮◌̐◌̕b; a◌֮◌̀◌̐◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING CANDRABINDU, LATIN SMALL LETTER B +0061 0310 0315 0300 05AE 0062;0061 05AE 0310 0300 0315 0062;0061 05AE 0310 0300 0315 0062;0061 05AE 0310 0300 0315 0062;0061 05AE 0310 0300 0315 0062; # (a◌̐◌̕◌̀◌֮b; a◌֮◌̐◌̀◌̕b; a◌֮◌̐◌̀◌̕b; a◌֮◌̐◌̀◌̕b; a◌֮◌̐◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING CANDRABINDU, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0311 0062;00E0 05AE 0311 0315 0062;0061 05AE 0300 0311 0315 0062;00E0 05AE 0311 0315 0062;0061 05AE 0300 0311 0315 0062; # (a◌̕◌̀◌֮◌̑b; à◌֮◌̑◌̕b; a◌֮◌̀◌̑◌̕b; à◌֮◌̑◌̕b; a◌֮◌̀◌̑◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING INVERTED BREVE, LATIN SMALL LETTER B +0061 0311 0315 0300 05AE 0062;0203 05AE 0300 0315 0062;0061 05AE 0311 0300 0315 0062;0203 05AE 0300 0315 0062;0061 05AE 0311 0300 0315 0062; # (a◌̑◌̕◌̀◌֮b; ȃ◌֮◌̀◌̕b; a◌֮◌̑◌̀◌̕b; ȃ◌֮◌̀◌̕b; a◌֮◌̑◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING INVERTED BREVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0312 0062;00E0 05AE 0312 0315 0062;0061 05AE 0300 0312 0315 0062;00E0 05AE 0312 0315 0062;0061 05AE 0300 0312 0315 0062; # (a◌̕◌̀◌֮◌̒b; à◌֮◌̒◌̕b; a◌֮◌̀◌̒◌̕b; à◌֮◌̒◌̕b; a◌֮◌̀◌̒◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING TURNED COMMA ABOVE, LATIN SMALL LETTER B +0061 0312 0315 0300 05AE 0062;0061 05AE 0312 0300 0315 0062;0061 05AE 0312 0300 0315 0062;0061 05AE 0312 0300 0315 0062;0061 05AE 0312 0300 0315 0062; # (a◌̒◌̕◌̀◌֮b; a◌֮◌̒◌̀◌̕b; a◌֮◌̒◌̀◌̕b; a◌֮◌̒◌̀◌̕b; a◌֮◌̒◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING TURNED COMMA ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0313 0062;00E0 05AE 0313 0315 0062;0061 05AE 0300 0313 0315 0062;00E0 05AE 0313 0315 0062;0061 05AE 0300 0313 0315 0062; # (a◌̕◌̀◌֮◌̓b; à◌֮◌̓◌̕b; a◌֮◌̀◌̓◌̕b; à◌֮◌̓◌̕b; a◌֮◌̀◌̓◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING COMMA ABOVE, LATIN SMALL LETTER B +0061 0313 0315 0300 05AE 0062;0061 05AE 0313 0300 0315 0062;0061 05AE 0313 0300 0315 0062;0061 05AE 0313 0300 0315 0062;0061 05AE 0313 0300 0315 0062; # (a◌̓◌̕◌̀◌֮b; a◌֮◌̓◌̀◌̕b; a◌֮◌̓◌̀◌̕b; a◌֮◌̓◌̀◌̕b; a◌֮◌̓◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0314 0062;00E0 05AE 0314 0315 0062;0061 05AE 0300 0314 0315 0062;00E0 05AE 0314 0315 0062;0061 05AE 0300 0314 0315 0062; # (a◌̕◌̀◌֮◌̔b; à◌֮◌̔◌̕b; a◌֮◌̀◌̔◌̕b; à◌֮◌̔◌̕b; a◌֮◌̀◌̔◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING REVERSED COMMA ABOVE, LATIN SMALL LETTER B +0061 0314 0315 0300 05AE 0062;0061 05AE 0314 0300 0315 0062;0061 05AE 0314 0300 0315 0062;0061 05AE 0314 0300 0315 0062;0061 05AE 0314 0300 0315 0062; # (a◌̔◌̕◌̀◌֮b; a◌֮◌̔◌̀◌̕b; a◌֮◌̔◌̀◌̕b; a◌֮◌̔◌̀◌̕b; a◌֮◌̔◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING REVERSED COMMA ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0362 0315 0300 0315 0062;00E0 0315 0315 0362 0062;0061 0300 0315 0315 0362 0062;00E0 0315 0315 0362 0062;0061 0300 0315 0315 0362 0062; # (a◌͢◌̕◌̀◌̕b; à◌̕◌̕◌͢b; a◌̀◌̕◌̕◌͢b; à◌̕◌̕◌͢b; a◌̀◌̕◌̕◌͢b; ) LATIN SMALL LETTER A, COMBINING DOUBLE RIGHTWARDS ARROW BELOW, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, COMBINING COMMA ABOVE RIGHT, LATIN SMALL LETTER B +0061 0315 0362 0315 0300 0062;00E0 0315 0315 0362 0062;0061 0300 0315 0315 0362 0062;00E0 0315 0315 0362 0062;0061 0300 0315 0315 0362 0062; # (a◌̕◌͢◌̕◌̀b; à◌̕◌̕◌͢b; a◌̀◌̕◌̕◌͢b; à◌̕◌̕◌͢b; a◌̀◌̕◌̕◌͢b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING DOUBLE RIGHTWARDS ARROW BELOW, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, LATIN SMALL LETTER B +0061 059A 0316 302A 0316 0062;0061 302A 0316 0316 059A 0062;0061 302A 0316 0316 059A 0062;0061 302A 0316 0316 059A 0062;0061 302A 0316 0316 059A 0062; # (a◌֚◌̖◌〪◌̖b; a◌〪◌̖◌̖◌֚b; a◌〪◌̖◌̖◌֚b; a◌〪◌̖◌̖◌֚b; a◌〪◌̖◌̖◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING GRAVE ACCENT BELOW, LATIN SMALL LETTER B +0061 0316 059A 0316 302A 0062;0061 302A 0316 0316 059A 0062;0061 302A 0316 0316 059A 0062;0061 302A 0316 0316 059A 0062;0061 302A 0316 0316 059A 0062; # (a◌̖◌֚◌̖◌〪b; a◌〪◌̖◌̖◌֚b; a◌〪◌̖◌̖◌֚b; a◌〪◌̖◌̖◌֚b; a◌〪◌̖◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING GRAVE ACCENT BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 0317 0062;0061 302A 0316 0317 059A 0062;0061 302A 0316 0317 059A 0062;0061 302A 0316 0317 059A 0062;0061 302A 0316 0317 059A 0062; # (a◌֚◌̖◌〪◌̗b; a◌〪◌̖◌̗◌֚b; a◌〪◌̖◌̗◌֚b; a◌〪◌̖◌̗◌֚b; a◌〪◌̖◌̗◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING ACUTE ACCENT BELOW, LATIN SMALL LETTER B +0061 0317 059A 0316 302A 0062;0061 302A 0317 0316 059A 0062;0061 302A 0317 0316 059A 0062;0061 302A 0317 0316 059A 0062;0061 302A 0317 0316 059A 0062; # (a◌̗◌֚◌̖◌〪b; a◌〪◌̗◌̖◌֚b; a◌〪◌̗◌̖◌֚b; a◌〪◌̗◌̖◌֚b; a◌〪◌̗◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING ACUTE ACCENT BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 0318 0062;0061 302A 0316 0318 059A 0062;0061 302A 0316 0318 059A 0062;0061 302A 0316 0318 059A 0062;0061 302A 0316 0318 059A 0062; # (a◌֚◌̖◌〪◌̘b; a◌〪◌̖◌̘◌֚b; a◌〪◌̖◌̘◌֚b; a◌〪◌̖◌̘◌֚b; a◌〪◌̖◌̘◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING LEFT TACK BELOW, LATIN SMALL LETTER B +0061 0318 059A 0316 302A 0062;0061 302A 0318 0316 059A 0062;0061 302A 0318 0316 059A 0062;0061 302A 0318 0316 059A 0062;0061 302A 0318 0316 059A 0062; # (a◌̘◌֚◌̖◌〪b; a◌〪◌̘◌̖◌֚b; a◌〪◌̘◌̖◌֚b; a◌〪◌̘◌̖◌֚b; a◌〪◌̘◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING LEFT TACK BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 0319 0062;0061 302A 0316 0319 059A 0062;0061 302A 0316 0319 059A 0062;0061 302A 0316 0319 059A 0062;0061 302A 0316 0319 059A 0062; # (a◌֚◌̖◌〪◌̙b; a◌〪◌̖◌̙◌֚b; a◌〪◌̖◌̙◌֚b; a◌〪◌̖◌̙◌֚b; a◌〪◌̖◌̙◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING RIGHT TACK BELOW, LATIN SMALL LETTER B +0061 0319 059A 0316 302A 0062;0061 302A 0319 0316 059A 0062;0061 302A 0319 0316 059A 0062;0061 302A 0319 0316 059A 0062;0061 302A 0319 0316 059A 0062; # (a◌̙◌֚◌̖◌〪b; a◌〪◌̙◌̖◌֚b; a◌〪◌̙◌̖◌֚b; a◌〪◌̙◌̖◌֚b; a◌〪◌̙◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING RIGHT TACK BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0362 0315 0300 031A 0062;00E0 0315 031A 0362 0062;0061 0300 0315 031A 0362 0062;00E0 0315 031A 0362 0062;0061 0300 0315 031A 0362 0062; # (a◌͢◌̕◌̀◌̚b; à◌̕◌̚◌͢b; a◌̀◌̕◌̚◌͢b; à◌̕◌̚◌͢b; a◌̀◌̕◌̚◌͢b; ) LATIN SMALL LETTER A, COMBINING DOUBLE RIGHTWARDS ARROW BELOW, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, COMBINING LEFT ANGLE ABOVE, LATIN SMALL LETTER B +0061 031A 0362 0315 0300 0062;00E0 031A 0315 0362 0062;0061 0300 031A 0315 0362 0062;00E0 031A 0315 0362 0062;0061 0300 031A 0315 0362 0062; # (a◌̚◌͢◌̕◌̀b; à◌̚◌̕◌͢b; a◌̀◌̚◌̕◌͢b; à◌̚◌̕◌͢b; a◌̀◌̚◌̕◌͢b; ) LATIN SMALL LETTER A, COMBINING LEFT ANGLE ABOVE, COMBINING DOUBLE RIGHTWARDS ARROW BELOW, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, LATIN SMALL LETTER B +0061 302A 031B 0321 031B 0062;0061 0321 031B 031B 302A 0062;0061 0321 031B 031B 302A 0062;0061 0321 031B 031B 302A 0062;0061 0321 031B 031B 302A 0062; # (a◌〪◌̛◌̡◌̛b; a◌̡◌̛◌̛◌〪b; a◌̡◌̛◌̛◌〪b; a◌̡◌̛◌̛◌〪b; a◌̡◌̛◌̛◌〪b; ) LATIN SMALL LETTER A, IDEOGRAPHIC LEVEL TONE MARK, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, COMBINING HORN, LATIN SMALL LETTER B +0061 031B 302A 031B 0321 0062;0061 0321 031B 031B 302A 0062;0061 0321 031B 031B 302A 0062;0061 0321 031B 031B 302A 0062;0061 0321 031B 031B 302A 0062; # (a◌̛◌〪◌̛◌̡b; a◌̡◌̛◌̛◌〪b; a◌̡◌̛◌̛◌〪b; a◌̡◌̛◌̛◌〪b; a◌̡◌̛◌̛◌〪b; ) LATIN SMALL LETTER A, COMBINING HORN, IDEOGRAPHIC LEVEL TONE MARK, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, LATIN SMALL LETTER B +0061 059A 0316 302A 031C 0062;0061 302A 0316 031C 059A 0062;0061 302A 0316 031C 059A 0062;0061 302A 0316 031C 059A 0062;0061 302A 0316 031C 059A 0062; # (a◌֚◌̖◌〪◌̜b; a◌〪◌̖◌̜◌֚b; a◌〪◌̖◌̜◌֚b; a◌〪◌̖◌̜◌֚b; a◌〪◌̖◌̜◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING LEFT HALF RING BELOW, LATIN SMALL LETTER B +0061 031C 059A 0316 302A 0062;0061 302A 031C 0316 059A 0062;0061 302A 031C 0316 059A 0062;0061 302A 031C 0316 059A 0062;0061 302A 031C 0316 059A 0062; # (a◌̜◌֚◌̖◌〪b; a◌〪◌̜◌̖◌֚b; a◌〪◌̜◌̖◌֚b; a◌〪◌̜◌̖◌֚b; a◌〪◌̜◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING LEFT HALF RING BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 031D 0062;0061 302A 0316 031D 059A 0062;0061 302A 0316 031D 059A 0062;0061 302A 0316 031D 059A 0062;0061 302A 0316 031D 059A 0062; # (a◌֚◌̖◌〪◌̝b; a◌〪◌̖◌̝◌֚b; a◌〪◌̖◌̝◌֚b; a◌〪◌̖◌̝◌֚b; a◌〪◌̖◌̝◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING UP TACK BELOW, LATIN SMALL LETTER B +0061 031D 059A 0316 302A 0062;0061 302A 031D 0316 059A 0062;0061 302A 031D 0316 059A 0062;0061 302A 031D 0316 059A 0062;0061 302A 031D 0316 059A 0062; # (a◌̝◌֚◌̖◌〪b; a◌〪◌̝◌̖◌֚b; a◌〪◌̝◌̖◌֚b; a◌〪◌̝◌̖◌֚b; a◌〪◌̝◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING UP TACK BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 031E 0062;0061 302A 0316 031E 059A 0062;0061 302A 0316 031E 059A 0062;0061 302A 0316 031E 059A 0062;0061 302A 0316 031E 059A 0062; # (a◌֚◌̖◌〪◌̞b; a◌〪◌̖◌̞◌֚b; a◌〪◌̖◌̞◌֚b; a◌〪◌̖◌̞◌֚b; a◌〪◌̖◌̞◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING DOWN TACK BELOW, LATIN SMALL LETTER B +0061 031E 059A 0316 302A 0062;0061 302A 031E 0316 059A 0062;0061 302A 031E 0316 059A 0062;0061 302A 031E 0316 059A 0062;0061 302A 031E 0316 059A 0062; # (a◌̞◌֚◌̖◌〪b; a◌〪◌̞◌̖◌֚b; a◌〪◌̞◌̖◌֚b; a◌〪◌̞◌̖◌֚b; a◌〪◌̞◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING DOWN TACK BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 031F 0062;0061 302A 0316 031F 059A 0062;0061 302A 0316 031F 059A 0062;0061 302A 0316 031F 059A 0062;0061 302A 0316 031F 059A 0062; # (a◌֚◌̖◌〪◌̟b; a◌〪◌̖◌̟◌֚b; a◌〪◌̖◌̟◌֚b; a◌〪◌̖◌̟◌֚b; a◌〪◌̖◌̟◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING PLUS SIGN BELOW, LATIN SMALL LETTER B +0061 031F 059A 0316 302A 0062;0061 302A 031F 0316 059A 0062;0061 302A 031F 0316 059A 0062;0061 302A 031F 0316 059A 0062;0061 302A 031F 0316 059A 0062; # (a◌̟◌֚◌̖◌〪b; a◌〪◌̟◌̖◌֚b; a◌〪◌̟◌̖◌֚b; a◌〪◌̟◌̖◌֚b; a◌〪◌̟◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING PLUS SIGN BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 0320 0062;0061 302A 0316 0320 059A 0062;0061 302A 0316 0320 059A 0062;0061 302A 0316 0320 059A 0062;0061 302A 0316 0320 059A 0062; # (a◌֚◌̖◌〪◌̠b; a◌〪◌̖◌̠◌֚b; a◌〪◌̖◌̠◌֚b; a◌〪◌̖◌̠◌֚b; a◌〪◌̖◌̠◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING MINUS SIGN BELOW, LATIN SMALL LETTER B +0061 0320 059A 0316 302A 0062;0061 302A 0320 0316 059A 0062;0061 302A 0320 0316 059A 0062;0061 302A 0320 0316 059A 0062;0061 302A 0320 0316 059A 0062; # (a◌̠◌֚◌̖◌〪b; a◌〪◌̠◌̖◌֚b; a◌〪◌̠◌̖◌֚b; a◌〪◌̠◌̖◌֚b; a◌〪◌̠◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING MINUS SIGN BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 031B 0321 0F74 0321 0062;0061 0F74 0321 0321 031B 0062;0061 0F74 0321 0321 031B 0062;0061 0F74 0321 0321 031B 0062;0061 0F74 0321 0321 031B 0062; # (a◌̛◌̡◌ུ◌̡b; a◌ུ◌̡◌̡◌̛b; a◌ུ◌̡◌̡◌̛b; a◌ུ◌̡◌̡◌̛b; a◌ུ◌̡◌̡◌̛b; ) LATIN SMALL LETTER A, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, TIBETAN VOWEL SIGN U, COMBINING PALATALIZED HOOK BELOW, LATIN SMALL LETTER B +0061 0321 031B 0321 0F74 0062;0061 0F74 0321 0321 031B 0062;0061 0F74 0321 0321 031B 0062;0061 0F74 0321 0321 031B 0062;0061 0F74 0321 0321 031B 0062; # (a◌̡◌̛◌̡◌ུb; a◌ུ◌̡◌̡◌̛b; a◌ུ◌̡◌̡◌̛b; a◌ུ◌̡◌̡◌̛b; a◌ུ◌̡◌̡◌̛b; ) LATIN SMALL LETTER A, COMBINING PALATALIZED HOOK BELOW, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, TIBETAN VOWEL SIGN U, LATIN SMALL LETTER B +0061 031B 0321 0F74 0322 0062;0061 0F74 0321 0322 031B 0062;0061 0F74 0321 0322 031B 0062;0061 0F74 0321 0322 031B 0062;0061 0F74 0321 0322 031B 0062; # (a◌̛◌̡◌ུ◌̢b; a◌ུ◌̡◌̢◌̛b; a◌ུ◌̡◌̢◌̛b; a◌ུ◌̡◌̢◌̛b; a◌ུ◌̡◌̢◌̛b; ) LATIN SMALL LETTER A, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, TIBETAN VOWEL SIGN U, COMBINING RETROFLEX HOOK BELOW, LATIN SMALL LETTER B +0061 0322 031B 0321 0F74 0062;0061 0F74 0322 0321 031B 0062;0061 0F74 0322 0321 031B 0062;0061 0F74 0322 0321 031B 0062;0061 0F74 0322 0321 031B 0062; # (a◌̢◌̛◌̡◌ུb; a◌ུ◌̢◌̡◌̛b; a◌ུ◌̢◌̡◌̛b; a◌ུ◌̢◌̡◌̛b; a◌ུ◌̢◌̡◌̛b; ) LATIN SMALL LETTER A, COMBINING RETROFLEX HOOK BELOW, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, TIBETAN VOWEL SIGN U, LATIN SMALL LETTER B +0061 059A 0316 302A 0323 0062;0061 302A 0316 0323 059A 0062;0061 302A 0316 0323 059A 0062;0061 302A 0316 0323 059A 0062;0061 302A 0316 0323 059A 0062; # (a◌֚◌̖◌〪◌̣b; a◌〪◌̖◌̣◌֚b; a◌〪◌̖◌̣◌֚b; a◌〪◌̖◌̣◌֚b; a◌〪◌̖◌̣◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING DOT BELOW, LATIN SMALL LETTER B +0061 0323 059A 0316 302A 0062;1EA1 302A 0316 059A 0062;0061 302A 0323 0316 059A 0062;1EA1 302A 0316 059A 0062;0061 302A 0323 0316 059A 0062; # (a◌̣◌֚◌̖◌〪b; ạ◌〪◌̖◌֚b; a◌〪◌̣◌̖◌֚b; ạ◌〪◌̖◌֚b; a◌〪◌̣◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING DOT BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 0324 0062;0061 302A 0316 0324 059A 0062;0061 302A 0316 0324 059A 0062;0061 302A 0316 0324 059A 0062;0061 302A 0316 0324 059A 0062; # (a◌֚◌̖◌〪◌̤b; a◌〪◌̖◌̤◌֚b; a◌〪◌̖◌̤◌֚b; a◌〪◌̖◌̤◌֚b; a◌〪◌̖◌̤◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING DIAERESIS BELOW, LATIN SMALL LETTER B +0061 0324 059A 0316 302A 0062;0061 302A 0324 0316 059A 0062;0061 302A 0324 0316 059A 0062;0061 302A 0324 0316 059A 0062;0061 302A 0324 0316 059A 0062; # (a◌̤◌֚◌̖◌〪b; a◌〪◌̤◌̖◌֚b; a◌〪◌̤◌̖◌֚b; a◌〪◌̤◌̖◌֚b; a◌〪◌̤◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING DIAERESIS BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 0325 0062;0061 302A 0316 0325 059A 0062;0061 302A 0316 0325 059A 0062;0061 302A 0316 0325 059A 0062;0061 302A 0316 0325 059A 0062; # (a◌֚◌̖◌〪◌̥b; a◌〪◌̖◌̥◌֚b; a◌〪◌̖◌̥◌֚b; a◌〪◌̖◌̥◌֚b; a◌〪◌̖◌̥◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING RING BELOW, LATIN SMALL LETTER B +0061 0325 059A 0316 302A 0062;1E01 302A 0316 059A 0062;0061 302A 0325 0316 059A 0062;1E01 302A 0316 059A 0062;0061 302A 0325 0316 059A 0062; # (a◌̥◌֚◌̖◌〪b; ḁ◌〪◌̖◌֚b; a◌〪◌̥◌̖◌֚b; ḁ◌〪◌̖◌֚b; a◌〪◌̥◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING RING BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 0326 0062;0061 302A 0316 0326 059A 0062;0061 302A 0316 0326 059A 0062;0061 302A 0316 0326 059A 0062;0061 302A 0316 0326 059A 0062; # (a◌֚◌̖◌〪◌̦b; a◌〪◌̖◌̦◌֚b; a◌〪◌̖◌̦◌֚b; a◌〪◌̖◌̦◌֚b; a◌〪◌̖◌̦◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING COMMA BELOW, LATIN SMALL LETTER B +0061 0326 059A 0316 302A 0062;0061 302A 0326 0316 059A 0062;0061 302A 0326 0316 059A 0062;0061 302A 0326 0316 059A 0062;0061 302A 0326 0316 059A 0062; # (a◌̦◌֚◌̖◌〪b; a◌〪◌̦◌̖◌֚b; a◌〪◌̦◌̖◌֚b; a◌〪◌̦◌̖◌֚b; a◌〪◌̦◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING COMMA BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 031B 0321 0F74 0327 0062;0061 0F74 0321 0327 031B 0062;0061 0F74 0321 0327 031B 0062;0061 0F74 0321 0327 031B 0062;0061 0F74 0321 0327 031B 0062; # (a◌̛◌̡◌ུ◌̧b; a◌ུ◌̡◌̧◌̛b; a◌ུ◌̡◌̧◌̛b; a◌ུ◌̡◌̧◌̛b; a◌ུ◌̡◌̧◌̛b; ) LATIN SMALL LETTER A, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, TIBETAN VOWEL SIGN U, COMBINING CEDILLA, LATIN SMALL LETTER B +0061 0327 031B 0321 0F74 0062;0061 0F74 0327 0321 031B 0062;0061 0F74 0327 0321 031B 0062;0061 0F74 0327 0321 031B 0062;0061 0F74 0327 0321 031B 0062; # (a◌̧◌̛◌̡◌ུb; a◌ུ◌̧◌̡◌̛b; a◌ུ◌̧◌̡◌̛b; a◌ུ◌̧◌̡◌̛b; a◌ུ◌̧◌̡◌̛b; ) LATIN SMALL LETTER A, COMBINING CEDILLA, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, TIBETAN VOWEL SIGN U, LATIN SMALL LETTER B +0061 031B 0321 0F74 0328 0062;0061 0F74 0321 0328 031B 0062;0061 0F74 0321 0328 031B 0062;0061 0F74 0321 0328 031B 0062;0061 0F74 0321 0328 031B 0062; # (a◌̛◌̡◌ུ◌̨b; a◌ུ◌̡◌̨◌̛b; a◌ུ◌̡◌̨◌̛b; a◌ུ◌̡◌̨◌̛b; a◌ུ◌̡◌̨◌̛b; ) LATIN SMALL LETTER A, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, TIBETAN VOWEL SIGN U, COMBINING OGONEK, LATIN SMALL LETTER B +0061 0328 031B 0321 0F74 0062;0105 0F74 0321 031B 0062;0061 0F74 0328 0321 031B 0062;0105 0F74 0321 031B 0062;0061 0F74 0328 0321 031B 0062; # (a◌̨◌̛◌̡◌ུb; ą◌ུ◌̡◌̛b; a◌ུ◌̨◌̡◌̛b; ą◌ུ◌̡◌̛b; a◌ུ◌̨◌̡◌̛b; ) LATIN SMALL LETTER A, COMBINING OGONEK, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, TIBETAN VOWEL SIGN U, LATIN SMALL LETTER B +0061 059A 0316 302A 0329 0062;0061 302A 0316 0329 059A 0062;0061 302A 0316 0329 059A 0062;0061 302A 0316 0329 059A 0062;0061 302A 0316 0329 059A 0062; # (a◌֚◌̖◌〪◌̩b; a◌〪◌̖◌̩◌֚b; a◌〪◌̖◌̩◌֚b; a◌〪◌̖◌̩◌֚b; a◌〪◌̖◌̩◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING VERTICAL LINE BELOW, LATIN SMALL LETTER B +0061 0329 059A 0316 302A 0062;0061 302A 0329 0316 059A 0062;0061 302A 0329 0316 059A 0062;0061 302A 0329 0316 059A 0062;0061 302A 0329 0316 059A 0062; # (a◌̩◌֚◌̖◌〪b; a◌〪◌̩◌̖◌֚b; a◌〪◌̩◌̖◌֚b; a◌〪◌̩◌̖◌֚b; a◌〪◌̩◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING VERTICAL LINE BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 032A 0062;0061 302A 0316 032A 059A 0062;0061 302A 0316 032A 059A 0062;0061 302A 0316 032A 059A 0062;0061 302A 0316 032A 059A 0062; # (a◌֚◌̖◌〪◌̪b; a◌〪◌̖◌̪◌֚b; a◌〪◌̖◌̪◌֚b; a◌〪◌̖◌̪◌֚b; a◌〪◌̖◌̪◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING BRIDGE BELOW, LATIN SMALL LETTER B +0061 032A 059A 0316 302A 0062;0061 302A 032A 0316 059A 0062;0061 302A 032A 0316 059A 0062;0061 302A 032A 0316 059A 0062;0061 302A 032A 0316 059A 0062; # (a◌̪◌֚◌̖◌〪b; a◌〪◌̪◌̖◌֚b; a◌〪◌̪◌̖◌֚b; a◌〪◌̪◌̖◌֚b; a◌〪◌̪◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING BRIDGE BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 032B 0062;0061 302A 0316 032B 059A 0062;0061 302A 0316 032B 059A 0062;0061 302A 0316 032B 059A 0062;0061 302A 0316 032B 059A 0062; # (a◌֚◌̖◌〪◌̫b; a◌〪◌̖◌̫◌֚b; a◌〪◌̖◌̫◌֚b; a◌〪◌̖◌̫◌֚b; a◌〪◌̖◌̫◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING INVERTED DOUBLE ARCH BELOW, LATIN SMALL LETTER B +0061 032B 059A 0316 302A 0062;0061 302A 032B 0316 059A 0062;0061 302A 032B 0316 059A 0062;0061 302A 032B 0316 059A 0062;0061 302A 032B 0316 059A 0062; # (a◌̫◌֚◌̖◌〪b; a◌〪◌̫◌̖◌֚b; a◌〪◌̫◌̖◌֚b; a◌〪◌̫◌̖◌֚b; a◌〪◌̫◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING INVERTED DOUBLE ARCH BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 032C 0062;0061 302A 0316 032C 059A 0062;0061 302A 0316 032C 059A 0062;0061 302A 0316 032C 059A 0062;0061 302A 0316 032C 059A 0062; # (a◌֚◌̖◌〪◌̬b; a◌〪◌̖◌̬◌֚b; a◌〪◌̖◌̬◌֚b; a◌〪◌̖◌̬◌֚b; a◌〪◌̖◌̬◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING CARON BELOW, LATIN SMALL LETTER B +0061 032C 059A 0316 302A 0062;0061 302A 032C 0316 059A 0062;0061 302A 032C 0316 059A 0062;0061 302A 032C 0316 059A 0062;0061 302A 032C 0316 059A 0062; # (a◌̬◌֚◌̖◌〪b; a◌〪◌̬◌̖◌֚b; a◌〪◌̬◌̖◌֚b; a◌〪◌̬◌̖◌֚b; a◌〪◌̬◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING CARON BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 032D 0062;0061 302A 0316 032D 059A 0062;0061 302A 0316 032D 059A 0062;0061 302A 0316 032D 059A 0062;0061 302A 0316 032D 059A 0062; # (a◌֚◌̖◌〪◌̭b; a◌〪◌̖◌̭◌֚b; a◌〪◌̖◌̭◌֚b; a◌〪◌̖◌̭◌֚b; a◌〪◌̖◌̭◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING CIRCUMFLEX ACCENT BELOW, LATIN SMALL LETTER B +0061 032D 059A 0316 302A 0062;0061 302A 032D 0316 059A 0062;0061 302A 032D 0316 059A 0062;0061 302A 032D 0316 059A 0062;0061 302A 032D 0316 059A 0062; # (a◌̭◌֚◌̖◌〪b; a◌〪◌̭◌̖◌֚b; a◌〪◌̭◌̖◌֚b; a◌〪◌̭◌̖◌֚b; a◌〪◌̭◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING CIRCUMFLEX ACCENT BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 032E 0062;0061 302A 0316 032E 059A 0062;0061 302A 0316 032E 059A 0062;0061 302A 0316 032E 059A 0062;0061 302A 0316 032E 059A 0062; # (a◌֚◌̖◌〪◌̮b; a◌〪◌̖◌̮◌֚b; a◌〪◌̖◌̮◌֚b; a◌〪◌̖◌̮◌֚b; a◌〪◌̖◌̮◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING BREVE BELOW, LATIN SMALL LETTER B +0061 032E 059A 0316 302A 0062;0061 302A 032E 0316 059A 0062;0061 302A 032E 0316 059A 0062;0061 302A 032E 0316 059A 0062;0061 302A 032E 0316 059A 0062; # (a◌̮◌֚◌̖◌〪b; a◌〪◌̮◌̖◌֚b; a◌〪◌̮◌̖◌֚b; a◌〪◌̮◌̖◌֚b; a◌〪◌̮◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING BREVE BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 032F 0062;0061 302A 0316 032F 059A 0062;0061 302A 0316 032F 059A 0062;0061 302A 0316 032F 059A 0062;0061 302A 0316 032F 059A 0062; # (a◌֚◌̖◌〪◌̯b; a◌〪◌̖◌̯◌֚b; a◌〪◌̖◌̯◌֚b; a◌〪◌̖◌̯◌֚b; a◌〪◌̖◌̯◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING INVERTED BREVE BELOW, LATIN SMALL LETTER B +0061 032F 059A 0316 302A 0062;0061 302A 032F 0316 059A 0062;0061 302A 032F 0316 059A 0062;0061 302A 032F 0316 059A 0062;0061 302A 032F 0316 059A 0062; # (a◌̯◌֚◌̖◌〪b; a◌〪◌̯◌̖◌֚b; a◌〪◌̯◌̖◌֚b; a◌〪◌̯◌̖◌֚b; a◌〪◌̯◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING INVERTED BREVE BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 0330 0062;0061 302A 0316 0330 059A 0062;0061 302A 0316 0330 059A 0062;0061 302A 0316 0330 059A 0062;0061 302A 0316 0330 059A 0062; # (a◌֚◌̖◌〪◌̰b; a◌〪◌̖◌̰◌֚b; a◌〪◌̖◌̰◌֚b; a◌〪◌̖◌̰◌֚b; a◌〪◌̖◌̰◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING TILDE BELOW, LATIN SMALL LETTER B +0061 0330 059A 0316 302A 0062;0061 302A 0330 0316 059A 0062;0061 302A 0330 0316 059A 0062;0061 302A 0330 0316 059A 0062;0061 302A 0330 0316 059A 0062; # (a◌̰◌֚◌̖◌〪b; a◌〪◌̰◌̖◌֚b; a◌〪◌̰◌̖◌֚b; a◌〪◌̰◌̖◌֚b; a◌〪◌̰◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING TILDE BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 0331 0062;0061 302A 0316 0331 059A 0062;0061 302A 0316 0331 059A 0062;0061 302A 0316 0331 059A 0062;0061 302A 0316 0331 059A 0062; # (a◌֚◌̖◌〪◌̱b; a◌〪◌̖◌̱◌֚b; a◌〪◌̖◌̱◌֚b; a◌〪◌̖◌̱◌֚b; a◌〪◌̖◌̱◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING MACRON BELOW, LATIN SMALL LETTER B +0061 0331 059A 0316 302A 0062;0061 302A 0331 0316 059A 0062;0061 302A 0331 0316 059A 0062;0061 302A 0331 0316 059A 0062;0061 302A 0331 0316 059A 0062; # (a◌̱◌֚◌̖◌〪b; a◌〪◌̱◌̖◌֚b; a◌〪◌̱◌̖◌֚b; a◌〪◌̱◌̖◌֚b; a◌〪◌̱◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING MACRON BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 0332 0062;0061 302A 0316 0332 059A 0062;0061 302A 0316 0332 059A 0062;0061 302A 0316 0332 059A 0062;0061 302A 0316 0332 059A 0062; # (a◌֚◌̖◌〪◌̲b; a◌〪◌̖◌̲◌֚b; a◌〪◌̖◌̲◌֚b; a◌〪◌̖◌̲◌֚b; a◌〪◌̖◌̲◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING LOW LINE, LATIN SMALL LETTER B +0061 0332 059A 0316 302A 0062;0061 302A 0332 0316 059A 0062;0061 302A 0332 0316 059A 0062;0061 302A 0332 0316 059A 0062;0061 302A 0332 0316 059A 0062; # (a◌̲◌֚◌̖◌〪b; a◌〪◌̲◌̖◌֚b; a◌〪◌̲◌̖◌֚b; a◌〪◌̲◌̖◌֚b; a◌〪◌̲◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING LOW LINE, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 0333 0062;0061 302A 0316 0333 059A 0062;0061 302A 0316 0333 059A 0062;0061 302A 0316 0333 059A 0062;0061 302A 0316 0333 059A 0062; # (a◌֚◌̖◌〪◌̳b; a◌〪◌̖◌̳◌֚b; a◌〪◌̖◌̳◌֚b; a◌〪◌̖◌̳◌֚b; a◌〪◌̖◌̳◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING DOUBLE LOW LINE, LATIN SMALL LETTER B +0061 0333 059A 0316 302A 0062;0061 302A 0333 0316 059A 0062;0061 302A 0333 0316 059A 0062;0061 302A 0333 0316 059A 0062;0061 302A 0333 0316 059A 0062; # (a◌̳◌֚◌̖◌〪b; a◌〪◌̳◌̖◌֚b; a◌〪◌̳◌̖◌֚b; a◌〪◌̳◌̖◌֚b; a◌〪◌̳◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING DOUBLE LOW LINE, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 093C 0334 0334 0062;0061 0334 0334 093C 0062;0061 0334 0334 093C 0062;0061 0334 0334 093C 0062;0061 0334 0334 093C 0062; # (a◌़◌̴◌̴b; a◌̴◌̴◌़b; a◌̴◌̴◌़b; a◌̴◌̴◌़b; a◌̴◌̴◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 0334 093C 0334 0062;0061 0334 0334 093C 0062;0061 0334 0334 093C 0062;0061 0334 0334 093C 0062;0061 0334 0334 093C 0062; # (a◌̴◌़◌̴b; a◌̴◌̴◌़b; a◌̴◌̴◌़b; a◌̴◌̴◌़b; a◌̴◌̴◌़b; ) LATIN SMALL LETTER A, COMBINING TILDE OVERLAY, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 093C 0334 0335 0062;0061 0334 0335 093C 0062;0061 0334 0335 093C 0062;0061 0334 0335 093C 0062;0061 0334 0335 093C 0062; # (a◌़◌̴◌̵b; a◌̴◌̵◌़b; a◌̴◌̵◌़b; a◌̴◌̵◌़b; a◌̴◌̵◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, COMBINING SHORT STROKE OVERLAY, LATIN SMALL LETTER B +0061 0335 093C 0334 0062;0061 0335 0334 093C 0062;0061 0335 0334 093C 0062;0061 0335 0334 093C 0062;0061 0335 0334 093C 0062; # (a◌̵◌़◌̴b; a◌̵◌̴◌़b; a◌̵◌̴◌़b; a◌̵◌̴◌़b; a◌̵◌̴◌़b; ) LATIN SMALL LETTER A, COMBINING SHORT STROKE OVERLAY, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 093C 0334 0336 0062;0061 0334 0336 093C 0062;0061 0334 0336 093C 0062;0061 0334 0336 093C 0062;0061 0334 0336 093C 0062; # (a◌़◌̴◌̶b; a◌̴◌̶◌़b; a◌̴◌̶◌़b; a◌̴◌̶◌़b; a◌̴◌̶◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, COMBINING LONG STROKE OVERLAY, LATIN SMALL LETTER B +0061 0336 093C 0334 0062;0061 0336 0334 093C 0062;0061 0336 0334 093C 0062;0061 0336 0334 093C 0062;0061 0336 0334 093C 0062; # (a◌̶◌़◌̴b; a◌̶◌̴◌़b; a◌̶◌̴◌़b; a◌̶◌̴◌़b; a◌̶◌̴◌़b; ) LATIN SMALL LETTER A, COMBINING LONG STROKE OVERLAY, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 093C 0334 0337 0062;0061 0334 0337 093C 0062;0061 0334 0337 093C 0062;0061 0334 0337 093C 0062;0061 0334 0337 093C 0062; # (a◌़◌̴◌̷b; a◌̴◌̷◌़b; a◌̴◌̷◌़b; a◌̴◌̷◌़b; a◌̴◌̷◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, COMBINING SHORT SOLIDUS OVERLAY, LATIN SMALL LETTER B +0061 0337 093C 0334 0062;0061 0337 0334 093C 0062;0061 0337 0334 093C 0062;0061 0337 0334 093C 0062;0061 0337 0334 093C 0062; # (a◌̷◌़◌̴b; a◌̷◌̴◌़b; a◌̷◌̴◌़b; a◌̷◌̴◌़b; a◌̷◌̴◌़b; ) LATIN SMALL LETTER A, COMBINING SHORT SOLIDUS OVERLAY, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 093C 0334 0338 0062;0061 0334 0338 093C 0062;0061 0334 0338 093C 0062;0061 0334 0338 093C 0062;0061 0334 0338 093C 0062; # (a◌़◌̴◌̸b; a◌̴◌̸◌़b; a◌̴◌̸◌़b; a◌̴◌̸◌़b; a◌̴◌̸◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, COMBINING LONG SOLIDUS OVERLAY, LATIN SMALL LETTER B +0061 0338 093C 0334 0062;0061 0338 0334 093C 0062;0061 0338 0334 093C 0062;0061 0338 0334 093C 0062;0061 0338 0334 093C 0062; # (a◌̸◌़◌̴b; a◌̸◌̴◌़b; a◌̸◌̴◌़b; a◌̸◌̴◌़b; a◌̸◌̴◌़b; ) LATIN SMALL LETTER A, COMBINING LONG SOLIDUS OVERLAY, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 059A 0316 302A 0339 0062;0061 302A 0316 0339 059A 0062;0061 302A 0316 0339 059A 0062;0061 302A 0316 0339 059A 0062;0061 302A 0316 0339 059A 0062; # (a◌֚◌̖◌〪◌̹b; a◌〪◌̖◌̹◌֚b; a◌〪◌̖◌̹◌֚b; a◌〪◌̖◌̹◌֚b; a◌〪◌̖◌̹◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING RIGHT HALF RING BELOW, LATIN SMALL LETTER B +0061 0339 059A 0316 302A 0062;0061 302A 0339 0316 059A 0062;0061 302A 0339 0316 059A 0062;0061 302A 0339 0316 059A 0062;0061 302A 0339 0316 059A 0062; # (a◌̹◌֚◌̖◌〪b; a◌〪◌̹◌̖◌֚b; a◌〪◌̹◌̖◌֚b; a◌〪◌̹◌̖◌֚b; a◌〪◌̹◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING RIGHT HALF RING BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 033A 0062;0061 302A 0316 033A 059A 0062;0061 302A 0316 033A 059A 0062;0061 302A 0316 033A 059A 0062;0061 302A 0316 033A 059A 0062; # (a◌֚◌̖◌〪◌̺b; a◌〪◌̖◌̺◌֚b; a◌〪◌̖◌̺◌֚b; a◌〪◌̖◌̺◌֚b; a◌〪◌̖◌̺◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING INVERTED BRIDGE BELOW, LATIN SMALL LETTER B +0061 033A 059A 0316 302A 0062;0061 302A 033A 0316 059A 0062;0061 302A 033A 0316 059A 0062;0061 302A 033A 0316 059A 0062;0061 302A 033A 0316 059A 0062; # (a◌̺◌֚◌̖◌〪b; a◌〪◌̺◌̖◌֚b; a◌〪◌̺◌̖◌֚b; a◌〪◌̺◌̖◌֚b; a◌〪◌̺◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING INVERTED BRIDGE BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 033B 0062;0061 302A 0316 033B 059A 0062;0061 302A 0316 033B 059A 0062;0061 302A 0316 033B 059A 0062;0061 302A 0316 033B 059A 0062; # (a◌֚◌̖◌〪◌̻b; a◌〪◌̖◌̻◌֚b; a◌〪◌̖◌̻◌֚b; a◌〪◌̖◌̻◌֚b; a◌〪◌̖◌̻◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING SQUARE BELOW, LATIN SMALL LETTER B +0061 033B 059A 0316 302A 0062;0061 302A 033B 0316 059A 0062;0061 302A 033B 0316 059A 0062;0061 302A 033B 0316 059A 0062;0061 302A 033B 0316 059A 0062; # (a◌̻◌֚◌̖◌〪b; a◌〪◌̻◌̖◌֚b; a◌〪◌̻◌̖◌֚b; a◌〪◌̻◌̖◌֚b; a◌〪◌̻◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING SQUARE BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 033C 0062;0061 302A 0316 033C 059A 0062;0061 302A 0316 033C 059A 0062;0061 302A 0316 033C 059A 0062;0061 302A 0316 033C 059A 0062; # (a◌֚◌̖◌〪◌̼b; a◌〪◌̖◌̼◌֚b; a◌〪◌̖◌̼◌֚b; a◌〪◌̖◌̼◌֚b; a◌〪◌̖◌̼◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING SEAGULL BELOW, LATIN SMALL LETTER B +0061 033C 059A 0316 302A 0062;0061 302A 033C 0316 059A 0062;0061 302A 033C 0316 059A 0062;0061 302A 033C 0316 059A 0062;0061 302A 033C 0316 059A 0062; # (a◌̼◌֚◌̖◌〪b; a◌〪◌̼◌̖◌֚b; a◌〪◌̼◌̖◌֚b; a◌〪◌̼◌̖◌֚b; a◌〪◌̼◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING SEAGULL BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 033D 0062;00E0 05AE 033D 0315 0062;0061 05AE 0300 033D 0315 0062;00E0 05AE 033D 0315 0062;0061 05AE 0300 033D 0315 0062; # (a◌̕◌̀◌֮◌̽b; à◌֮◌̽◌̕b; a◌֮◌̀◌̽◌̕b; à◌֮◌̽◌̕b; a◌֮◌̀◌̽◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING X ABOVE, LATIN SMALL LETTER B +0061 033D 0315 0300 05AE 0062;0061 05AE 033D 0300 0315 0062;0061 05AE 033D 0300 0315 0062;0061 05AE 033D 0300 0315 0062;0061 05AE 033D 0300 0315 0062; # (a◌̽◌̕◌̀◌֮b; a◌֮◌̽◌̀◌̕b; a◌֮◌̽◌̀◌̕b; a◌֮◌̽◌̀◌̕b; a◌֮◌̽◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING X ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 033E 0062;00E0 05AE 033E 0315 0062;0061 05AE 0300 033E 0315 0062;00E0 05AE 033E 0315 0062;0061 05AE 0300 033E 0315 0062; # (a◌̕◌̀◌֮◌̾b; à◌֮◌̾◌̕b; a◌֮◌̀◌̾◌̕b; à◌֮◌̾◌̕b; a◌֮◌̀◌̾◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING VERTICAL TILDE, LATIN SMALL LETTER B +0061 033E 0315 0300 05AE 0062;0061 05AE 033E 0300 0315 0062;0061 05AE 033E 0300 0315 0062;0061 05AE 033E 0300 0315 0062;0061 05AE 033E 0300 0315 0062; # (a◌̾◌̕◌̀◌֮b; a◌֮◌̾◌̀◌̕b; a◌֮◌̾◌̀◌̕b; a◌֮◌̾◌̀◌̕b; a◌֮◌̾◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING VERTICAL TILDE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 033F 0062;00E0 05AE 033F 0315 0062;0061 05AE 0300 033F 0315 0062;00E0 05AE 033F 0315 0062;0061 05AE 0300 033F 0315 0062; # (a◌̕◌̀◌֮◌̿b; à◌֮◌̿◌̕b; a◌֮◌̀◌̿◌̕b; à◌֮◌̿◌̕b; a◌֮◌̀◌̿◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING DOUBLE OVERLINE, LATIN SMALL LETTER B +0061 033F 0315 0300 05AE 0062;0061 05AE 033F 0300 0315 0062;0061 05AE 033F 0300 0315 0062;0061 05AE 033F 0300 0315 0062;0061 05AE 033F 0300 0315 0062; # (a◌̿◌̕◌̀◌֮b; a◌֮◌̿◌̀◌̕b; a◌֮◌̿◌̀◌̕b; a◌֮◌̿◌̀◌̕b; a◌֮◌̿◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING DOUBLE OVERLINE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0340 0062;00E0 05AE 0300 0315 0062;0061 05AE 0300 0300 0315 0062;00E0 05AE 0300 0315 0062;0061 05AE 0300 0300 0315 0062; # (a◌̕◌̀◌֮◌̀b; à◌֮◌̀◌̕b; a◌֮◌̀◌̀◌̕b; à◌֮◌̀◌̕b; a◌֮◌̀◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING GRAVE TONE MARK, LATIN SMALL LETTER B +0061 0340 0315 0300 05AE 0062;00E0 05AE 0300 0315 0062;0061 05AE 0300 0300 0315 0062;00E0 05AE 0300 0315 0062;0061 05AE 0300 0300 0315 0062; # (a◌̀◌̕◌̀◌֮b; à◌֮◌̀◌̕b; a◌֮◌̀◌̀◌̕b; à◌֮◌̀◌̕b; a◌֮◌̀◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING GRAVE TONE MARK, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0341 0062;00E0 05AE 0301 0315 0062;0061 05AE 0300 0301 0315 0062;00E0 05AE 0301 0315 0062;0061 05AE 0300 0301 0315 0062; # (a◌̕◌̀◌֮◌́b; à◌֮◌́◌̕b; a◌֮◌̀◌́◌̕b; à◌֮◌́◌̕b; a◌֮◌̀◌́◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING ACUTE TONE MARK, LATIN SMALL LETTER B +0061 0341 0315 0300 05AE 0062;00E1 05AE 0300 0315 0062;0061 05AE 0301 0300 0315 0062;00E1 05AE 0300 0315 0062;0061 05AE 0301 0300 0315 0062; # (a◌́◌̕◌̀◌֮b; á◌֮◌̀◌̕b; a◌֮◌́◌̀◌̕b; á◌֮◌̀◌̕b; a◌֮◌́◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING ACUTE TONE MARK, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0342 0062;00E0 05AE 0342 0315 0062;0061 05AE 0300 0342 0315 0062;00E0 05AE 0342 0315 0062;0061 05AE 0300 0342 0315 0062; # (a◌̕◌̀◌֮◌͂b; à◌֮◌͂◌̕b; a◌֮◌̀◌͂◌̕b; à◌֮◌͂◌̕b; a◌֮◌̀◌͂◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING GREEK PERISPOMENI, LATIN SMALL LETTER B +0061 0342 0315 0300 05AE 0062;0061 05AE 0342 0300 0315 0062;0061 05AE 0342 0300 0315 0062;0061 05AE 0342 0300 0315 0062;0061 05AE 0342 0300 0315 0062; # (a◌͂◌̕◌̀◌֮b; a◌֮◌͂◌̀◌̕b; a◌֮◌͂◌̀◌̕b; a◌֮◌͂◌̀◌̕b; a◌֮◌͂◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING GREEK PERISPOMENI, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0343 0062;00E0 05AE 0313 0315 0062;0061 05AE 0300 0313 0315 0062;00E0 05AE 0313 0315 0062;0061 05AE 0300 0313 0315 0062; # (a◌̕◌̀◌֮◌̓b; à◌֮◌̓◌̕b; a◌֮◌̀◌̓◌̕b; à◌֮◌̓◌̕b; a◌֮◌̀◌̓◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING GREEK KORONIS, LATIN SMALL LETTER B +0061 0343 0315 0300 05AE 0062;0061 05AE 0313 0300 0315 0062;0061 05AE 0313 0300 0315 0062;0061 05AE 0313 0300 0315 0062;0061 05AE 0313 0300 0315 0062; # (a◌̓◌̕◌̀◌֮b; a◌֮◌̓◌̀◌̕b; a◌֮◌̓◌̀◌̕b; a◌֮◌̓◌̀◌̕b; a◌֮◌̓◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING GREEK KORONIS, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0344 0062;00E0 05AE 0308 0301 0315 0062;0061 05AE 0300 0308 0301 0315 0062;00E0 05AE 0308 0301 0315 0062;0061 05AE 0300 0308 0301 0315 0062; # (a◌̕◌̀◌֮◌̈́b; à◌֮◌̈◌́◌̕b; a◌֮◌̀◌̈◌́◌̕b; à◌֮◌̈◌́◌̕b; a◌֮◌̀◌̈◌́◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING GREEK DIALYTIKA TONOS, LATIN SMALL LETTER B +0061 0344 0315 0300 05AE 0062;00E4 05AE 0301 0300 0315 0062;0061 05AE 0308 0301 0300 0315 0062;00E4 05AE 0301 0300 0315 0062;0061 05AE 0308 0301 0300 0315 0062; # (a◌̈́◌̕◌̀◌֮b; ä◌֮◌́◌̀◌̕b; a◌֮◌̈◌́◌̀◌̕b; ä◌֮◌́◌̀◌̕b; a◌֮◌̈◌́◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING GREEK DIALYTIKA TONOS, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0345 0360 0345 0062;0061 0360 0345 0345 0062;0061 0360 0345 0345 0062;0061 0360 0345 0345 0062;0061 0360 0345 0345 0062; # (a◌ͅ◌͠◌ͅb; a◌͠◌ͅ◌ͅb; a◌͠◌ͅ◌ͅb; a◌͠◌ͅ◌ͅb; a◌͠◌ͅ◌ͅb; ) LATIN SMALL LETTER A, COMBINING GREEK YPOGEGRAMMENI, COMBINING DOUBLE TILDE, COMBINING GREEK YPOGEGRAMMENI, LATIN SMALL LETTER B +0061 0345 0345 0360 0062;0061 0360 0345 0345 0062;0061 0360 0345 0345 0062;0061 0360 0345 0345 0062;0061 0360 0345 0345 0062; # (a◌ͅ◌ͅ◌͠b; a◌͠◌ͅ◌ͅb; a◌͠◌ͅ◌ͅb; a◌͠◌ͅ◌ͅb; a◌͠◌ͅ◌ͅb; ) LATIN SMALL LETTER A, COMBINING GREEK YPOGEGRAMMENI, COMBINING GREEK YPOGEGRAMMENI, COMBINING DOUBLE TILDE, LATIN SMALL LETTER B +0061 0315 0300 05AE 0346 0062;00E0 05AE 0346 0315 0062;0061 05AE 0300 0346 0315 0062;00E0 05AE 0346 0315 0062;0061 05AE 0300 0346 0315 0062; # (a◌̕◌̀◌֮◌͆b; à◌֮◌͆◌̕b; a◌֮◌̀◌͆◌̕b; à◌֮◌͆◌̕b; a◌֮◌̀◌͆◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING BRIDGE ABOVE, LATIN SMALL LETTER B +0061 0346 0315 0300 05AE 0062;0061 05AE 0346 0300 0315 0062;0061 05AE 0346 0300 0315 0062;0061 05AE 0346 0300 0315 0062;0061 05AE 0346 0300 0315 0062; # (a◌͆◌̕◌̀◌֮b; a◌֮◌͆◌̀◌̕b; a◌֮◌͆◌̀◌̕b; a◌֮◌͆◌̀◌̕b; a◌֮◌͆◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING BRIDGE ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 0347 0062;0061 302A 0316 0347 059A 0062;0061 302A 0316 0347 059A 0062;0061 302A 0316 0347 059A 0062;0061 302A 0316 0347 059A 0062; # (a◌֚◌̖◌〪◌͇b; a◌〪◌̖◌͇◌֚b; a◌〪◌̖◌͇◌֚b; a◌〪◌̖◌͇◌֚b; a◌〪◌̖◌͇◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING EQUALS SIGN BELOW, LATIN SMALL LETTER B +0061 0347 059A 0316 302A 0062;0061 302A 0347 0316 059A 0062;0061 302A 0347 0316 059A 0062;0061 302A 0347 0316 059A 0062;0061 302A 0347 0316 059A 0062; # (a◌͇◌֚◌̖◌〪b; a◌〪◌͇◌̖◌֚b; a◌〪◌͇◌̖◌֚b; a◌〪◌͇◌̖◌֚b; a◌〪◌͇◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING EQUALS SIGN BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 0348 0062;0061 302A 0316 0348 059A 0062;0061 302A 0316 0348 059A 0062;0061 302A 0316 0348 059A 0062;0061 302A 0316 0348 059A 0062; # (a◌֚◌̖◌〪◌͈b; a◌〪◌̖◌͈◌֚b; a◌〪◌̖◌͈◌֚b; a◌〪◌̖◌͈◌֚b; a◌〪◌̖◌͈◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING DOUBLE VERTICAL LINE BELOW, LATIN SMALL LETTER B +0061 0348 059A 0316 302A 0062;0061 302A 0348 0316 059A 0062;0061 302A 0348 0316 059A 0062;0061 302A 0348 0316 059A 0062;0061 302A 0348 0316 059A 0062; # (a◌͈◌֚◌̖◌〪b; a◌〪◌͈◌̖◌֚b; a◌〪◌͈◌̖◌֚b; a◌〪◌͈◌̖◌֚b; a◌〪◌͈◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING DOUBLE VERTICAL LINE BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 0349 0062;0061 302A 0316 0349 059A 0062;0061 302A 0316 0349 059A 0062;0061 302A 0316 0349 059A 0062;0061 302A 0316 0349 059A 0062; # (a◌֚◌̖◌〪◌͉b; a◌〪◌̖◌͉◌֚b; a◌〪◌̖◌͉◌֚b; a◌〪◌̖◌͉◌֚b; a◌〪◌̖◌͉◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING LEFT ANGLE BELOW, LATIN SMALL LETTER B +0061 0349 059A 0316 302A 0062;0061 302A 0349 0316 059A 0062;0061 302A 0349 0316 059A 0062;0061 302A 0349 0316 059A 0062;0061 302A 0349 0316 059A 0062; # (a◌͉◌֚◌̖◌〪b; a◌〪◌͉◌̖◌֚b; a◌〪◌͉◌̖◌֚b; a◌〪◌͉◌̖◌֚b; a◌〪◌͉◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING LEFT ANGLE BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 034A 0062;00E0 05AE 034A 0315 0062;0061 05AE 0300 034A 0315 0062;00E0 05AE 034A 0315 0062;0061 05AE 0300 034A 0315 0062; # (a◌̕◌̀◌֮◌͊b; à◌֮◌͊◌̕b; a◌֮◌̀◌͊◌̕b; à◌֮◌͊◌̕b; a◌֮◌̀◌͊◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING NOT TILDE ABOVE, LATIN SMALL LETTER B +0061 034A 0315 0300 05AE 0062;0061 05AE 034A 0300 0315 0062;0061 05AE 034A 0300 0315 0062;0061 05AE 034A 0300 0315 0062;0061 05AE 034A 0300 0315 0062; # (a◌͊◌̕◌̀◌֮b; a◌֮◌͊◌̀◌̕b; a◌֮◌͊◌̀◌̕b; a◌֮◌͊◌̀◌̕b; a◌֮◌͊◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING NOT TILDE ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 034B 0062;00E0 05AE 034B 0315 0062;0061 05AE 0300 034B 0315 0062;00E0 05AE 034B 0315 0062;0061 05AE 0300 034B 0315 0062; # (a◌̕◌̀◌֮◌͋b; à◌֮◌͋◌̕b; a◌֮◌̀◌͋◌̕b; à◌֮◌͋◌̕b; a◌֮◌̀◌͋◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING HOMOTHETIC ABOVE, LATIN SMALL LETTER B +0061 034B 0315 0300 05AE 0062;0061 05AE 034B 0300 0315 0062;0061 05AE 034B 0300 0315 0062;0061 05AE 034B 0300 0315 0062;0061 05AE 034B 0300 0315 0062; # (a◌͋◌̕◌̀◌֮b; a◌֮◌͋◌̀◌̕b; a◌֮◌͋◌̀◌̕b; a◌֮◌͋◌̀◌̕b; a◌֮◌͋◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING HOMOTHETIC ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 034C 0062;00E0 05AE 034C 0315 0062;0061 05AE 0300 034C 0315 0062;00E0 05AE 034C 0315 0062;0061 05AE 0300 034C 0315 0062; # (a◌̕◌̀◌֮◌͌b; à◌֮◌͌◌̕b; a◌֮◌̀◌͌◌̕b; à◌֮◌͌◌̕b; a◌֮◌̀◌͌◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING ALMOST EQUAL TO ABOVE, LATIN SMALL LETTER B +0061 034C 0315 0300 05AE 0062;0061 05AE 034C 0300 0315 0062;0061 05AE 034C 0300 0315 0062;0061 05AE 034C 0300 0315 0062;0061 05AE 034C 0300 0315 0062; # (a◌͌◌̕◌̀◌֮b; a◌֮◌͌◌̀◌̕b; a◌֮◌͌◌̀◌̕b; a◌֮◌͌◌̀◌̕b; a◌֮◌͌◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING ALMOST EQUAL TO ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 034D 0062;0061 302A 0316 034D 059A 0062;0061 302A 0316 034D 059A 0062;0061 302A 0316 034D 059A 0062;0061 302A 0316 034D 059A 0062; # (a◌֚◌̖◌〪◌͍b; a◌〪◌̖◌͍◌֚b; a◌〪◌̖◌͍◌֚b; a◌〪◌̖◌͍◌֚b; a◌〪◌̖◌͍◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING LEFT RIGHT ARROW BELOW, LATIN SMALL LETTER B +0061 034D 059A 0316 302A 0062;0061 302A 034D 0316 059A 0062;0061 302A 034D 0316 059A 0062;0061 302A 034D 0316 059A 0062;0061 302A 034D 0316 059A 0062; # (a◌͍◌֚◌̖◌〪b; a◌〪◌͍◌̖◌֚b; a◌〪◌͍◌̖◌֚b; a◌〪◌͍◌̖◌֚b; a◌〪◌͍◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING LEFT RIGHT ARROW BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 034E 0062;0061 302A 0316 034E 059A 0062;0061 302A 0316 034E 059A 0062;0061 302A 0316 034E 059A 0062;0061 302A 0316 034E 059A 0062; # (a◌֚◌̖◌〪◌͎b; a◌〪◌̖◌͎◌֚b; a◌〪◌̖◌͎◌֚b; a◌〪◌̖◌͎◌֚b; a◌〪◌̖◌͎◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING UPWARDS ARROW BELOW, LATIN SMALL LETTER B +0061 034E 059A 0316 302A 0062;0061 302A 034E 0316 059A 0062;0061 302A 034E 0316 059A 0062;0061 302A 034E 0316 059A 0062;0061 302A 034E 0316 059A 0062; # (a◌͎◌֚◌̖◌〪b; a◌〪◌͎◌̖◌֚b; a◌〪◌͎◌̖◌֚b; a◌〪◌͎◌̖◌֚b; a◌〪◌͎◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING UPWARDS ARROW BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0345 0360 0362 0360 0062;0061 0362 0360 0360 0345 0062;0061 0362 0360 0360 0345 0062;0061 0362 0360 0360 0345 0062;0061 0362 0360 0360 0345 0062; # (a◌ͅ◌͠◌͢◌͠b; a◌͢◌͠◌͠◌ͅb; a◌͢◌͠◌͠◌ͅb; a◌͢◌͠◌͠◌ͅb; a◌͢◌͠◌͠◌ͅb; ) LATIN SMALL LETTER A, COMBINING GREEK YPOGEGRAMMENI, COMBINING DOUBLE TILDE, COMBINING DOUBLE RIGHTWARDS ARROW BELOW, COMBINING DOUBLE TILDE, LATIN SMALL LETTER B +0061 0360 0345 0360 0362 0062;0061 0362 0360 0360 0345 0062;0061 0362 0360 0360 0345 0062;0061 0362 0360 0360 0345 0062;0061 0362 0360 0360 0345 0062; # (a◌͠◌ͅ◌͠◌͢b; a◌͢◌͠◌͠◌ͅb; a◌͢◌͠◌͠◌ͅb; a◌͢◌͠◌͠◌ͅb; a◌͢◌͠◌͠◌ͅb; ) LATIN SMALL LETTER A, COMBINING DOUBLE TILDE, COMBINING GREEK YPOGEGRAMMENI, COMBINING DOUBLE TILDE, COMBINING DOUBLE RIGHTWARDS ARROW BELOW, LATIN SMALL LETTER B +0061 0345 0360 0362 0361 0062;0061 0362 0360 0361 0345 0062;0061 0362 0360 0361 0345 0062;0061 0362 0360 0361 0345 0062;0061 0362 0360 0361 0345 0062; # (a◌ͅ◌͠◌͢◌͡b; a◌͢◌͠◌͡◌ͅb; a◌͢◌͠◌͡◌ͅb; a◌͢◌͠◌͡◌ͅb; a◌͢◌͠◌͡◌ͅb; ) LATIN SMALL LETTER A, COMBINING GREEK YPOGEGRAMMENI, COMBINING DOUBLE TILDE, COMBINING DOUBLE RIGHTWARDS ARROW BELOW, COMBINING DOUBLE INVERTED BREVE, LATIN SMALL LETTER B +0061 0361 0345 0360 0362 0062;0061 0362 0361 0360 0345 0062;0061 0362 0361 0360 0345 0062;0061 0362 0361 0360 0345 0062;0061 0362 0361 0360 0345 0062; # (a◌͡◌ͅ◌͠◌͢b; a◌͢◌͡◌͠◌ͅb; a◌͢◌͡◌͠◌ͅb; a◌͢◌͡◌͠◌ͅb; a◌͢◌͡◌͠◌ͅb; ) LATIN SMALL LETTER A, COMBINING DOUBLE INVERTED BREVE, COMBINING GREEK YPOGEGRAMMENI, COMBINING DOUBLE TILDE, COMBINING DOUBLE RIGHTWARDS ARROW BELOW, LATIN SMALL LETTER B +0061 0360 0362 0315 0362 0062;0061 0315 0362 0362 0360 0062;0061 0315 0362 0362 0360 0062;0061 0315 0362 0362 0360 0062;0061 0315 0362 0362 0360 0062; # (a◌͠◌͢◌̕◌͢b; a◌̕◌͢◌͢◌͠b; a◌̕◌͢◌͢◌͠b; a◌̕◌͢◌͢◌͠b; a◌̕◌͢◌͢◌͠b; ) LATIN SMALL LETTER A, COMBINING DOUBLE TILDE, COMBINING DOUBLE RIGHTWARDS ARROW BELOW, COMBINING COMMA ABOVE RIGHT, COMBINING DOUBLE RIGHTWARDS ARROW BELOW, LATIN SMALL LETTER B +0061 0362 0360 0362 0315 0062;0061 0315 0362 0362 0360 0062;0061 0315 0362 0362 0360 0062;0061 0315 0362 0362 0360 0062;0061 0315 0362 0362 0360 0062; # (a◌͢◌͠◌͢◌̕b; a◌̕◌͢◌͢◌͠b; a◌̕◌͢◌͢◌͠b; a◌̕◌͢◌͢◌͠b; a◌̕◌͢◌͢◌͠b; ) LATIN SMALL LETTER A, COMBINING DOUBLE RIGHTWARDS ARROW BELOW, COMBINING DOUBLE TILDE, COMBINING DOUBLE RIGHTWARDS ARROW BELOW, COMBINING COMMA ABOVE RIGHT, LATIN SMALL LETTER B +0061 0315 0300 05AE 0363 0062;00E0 05AE 0363 0315 0062;0061 05AE 0300 0363 0315 0062;00E0 05AE 0363 0315 0062;0061 05AE 0300 0363 0315 0062; # (a◌̕◌̀◌֮◌ͣb; à◌֮◌ͣ◌̕b; a◌֮◌̀◌ͣ◌̕b; à◌֮◌ͣ◌̕b; a◌֮◌̀◌ͣ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING LATIN SMALL LETTER A, LATIN SMALL LETTER B +0061 0363 0315 0300 05AE 0062;0061 05AE 0363 0300 0315 0062;0061 05AE 0363 0300 0315 0062;0061 05AE 0363 0300 0315 0062;0061 05AE 0363 0300 0315 0062; # (a◌ͣ◌̕◌̀◌֮b; a◌֮◌ͣ◌̀◌̕b; a◌֮◌ͣ◌̀◌̕b; a◌֮◌ͣ◌̀◌̕b; a◌֮◌ͣ◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0364 0062;00E0 05AE 0364 0315 0062;0061 05AE 0300 0364 0315 0062;00E0 05AE 0364 0315 0062;0061 05AE 0300 0364 0315 0062; # (a◌̕◌̀◌֮◌ͤb; à◌֮◌ͤ◌̕b; a◌֮◌̀◌ͤ◌̕b; à◌֮◌ͤ◌̕b; a◌֮◌̀◌ͤ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING LATIN SMALL LETTER E, LATIN SMALL LETTER B +0061 0364 0315 0300 05AE 0062;0061 05AE 0364 0300 0315 0062;0061 05AE 0364 0300 0315 0062;0061 05AE 0364 0300 0315 0062;0061 05AE 0364 0300 0315 0062; # (a◌ͤ◌̕◌̀◌֮b; a◌֮◌ͤ◌̀◌̕b; a◌֮◌ͤ◌̀◌̕b; a◌֮◌ͤ◌̀◌̕b; a◌֮◌ͤ◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING LATIN SMALL LETTER E, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0365 0062;00E0 05AE 0365 0315 0062;0061 05AE 0300 0365 0315 0062;00E0 05AE 0365 0315 0062;0061 05AE 0300 0365 0315 0062; # (a◌̕◌̀◌֮◌ͥb; à◌֮◌ͥ◌̕b; a◌֮◌̀◌ͥ◌̕b; à◌֮◌ͥ◌̕b; a◌֮◌̀◌ͥ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING LATIN SMALL LETTER I, LATIN SMALL LETTER B +0061 0365 0315 0300 05AE 0062;0061 05AE 0365 0300 0315 0062;0061 05AE 0365 0300 0315 0062;0061 05AE 0365 0300 0315 0062;0061 05AE 0365 0300 0315 0062; # (a◌ͥ◌̕◌̀◌֮b; a◌֮◌ͥ◌̀◌̕b; a◌֮◌ͥ◌̀◌̕b; a◌֮◌ͥ◌̀◌̕b; a◌֮◌ͥ◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING LATIN SMALL LETTER I, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0366 0062;00E0 05AE 0366 0315 0062;0061 05AE 0300 0366 0315 0062;00E0 05AE 0366 0315 0062;0061 05AE 0300 0366 0315 0062; # (a◌̕◌̀◌֮◌ͦb; à◌֮◌ͦ◌̕b; a◌֮◌̀◌ͦ◌̕b; à◌֮◌ͦ◌̕b; a◌֮◌̀◌ͦ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING LATIN SMALL LETTER O, LATIN SMALL LETTER B +0061 0366 0315 0300 05AE 0062;0061 05AE 0366 0300 0315 0062;0061 05AE 0366 0300 0315 0062;0061 05AE 0366 0300 0315 0062;0061 05AE 0366 0300 0315 0062; # (a◌ͦ◌̕◌̀◌֮b; a◌֮◌ͦ◌̀◌̕b; a◌֮◌ͦ◌̀◌̕b; a◌֮◌ͦ◌̀◌̕b; a◌֮◌ͦ◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING LATIN SMALL LETTER O, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0367 0062;00E0 05AE 0367 0315 0062;0061 05AE 0300 0367 0315 0062;00E0 05AE 0367 0315 0062;0061 05AE 0300 0367 0315 0062; # (a◌̕◌̀◌֮◌ͧb; à◌֮◌ͧ◌̕b; a◌֮◌̀◌ͧ◌̕b; à◌֮◌ͧ◌̕b; a◌֮◌̀◌ͧ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING LATIN SMALL LETTER U, LATIN SMALL LETTER B +0061 0367 0315 0300 05AE 0062;0061 05AE 0367 0300 0315 0062;0061 05AE 0367 0300 0315 0062;0061 05AE 0367 0300 0315 0062;0061 05AE 0367 0300 0315 0062; # (a◌ͧ◌̕◌̀◌֮b; a◌֮◌ͧ◌̀◌̕b; a◌֮◌ͧ◌̀◌̕b; a◌֮◌ͧ◌̀◌̕b; a◌֮◌ͧ◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING LATIN SMALL LETTER U, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0368 0062;00E0 05AE 0368 0315 0062;0061 05AE 0300 0368 0315 0062;00E0 05AE 0368 0315 0062;0061 05AE 0300 0368 0315 0062; # (a◌̕◌̀◌֮◌ͨb; à◌֮◌ͨ◌̕b; a◌֮◌̀◌ͨ◌̕b; à◌֮◌ͨ◌̕b; a◌֮◌̀◌ͨ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING LATIN SMALL LETTER C, LATIN SMALL LETTER B +0061 0368 0315 0300 05AE 0062;0061 05AE 0368 0300 0315 0062;0061 05AE 0368 0300 0315 0062;0061 05AE 0368 0300 0315 0062;0061 05AE 0368 0300 0315 0062; # (a◌ͨ◌̕◌̀◌֮b; a◌֮◌ͨ◌̀◌̕b; a◌֮◌ͨ◌̀◌̕b; a◌֮◌ͨ◌̀◌̕b; a◌֮◌ͨ◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING LATIN SMALL LETTER C, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0369 0062;00E0 05AE 0369 0315 0062;0061 05AE 0300 0369 0315 0062;00E0 05AE 0369 0315 0062;0061 05AE 0300 0369 0315 0062; # (a◌̕◌̀◌֮◌ͩb; à◌֮◌ͩ◌̕b; a◌֮◌̀◌ͩ◌̕b; à◌֮◌ͩ◌̕b; a◌֮◌̀◌ͩ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING LATIN SMALL LETTER D, LATIN SMALL LETTER B +0061 0369 0315 0300 05AE 0062;0061 05AE 0369 0300 0315 0062;0061 05AE 0369 0300 0315 0062;0061 05AE 0369 0300 0315 0062;0061 05AE 0369 0300 0315 0062; # (a◌ͩ◌̕◌̀◌֮b; a◌֮◌ͩ◌̀◌̕b; a◌֮◌ͩ◌̀◌̕b; a◌֮◌ͩ◌̀◌̕b; a◌֮◌ͩ◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING LATIN SMALL LETTER D, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 036A 0062;00E0 05AE 036A 0315 0062;0061 05AE 0300 036A 0315 0062;00E0 05AE 036A 0315 0062;0061 05AE 0300 036A 0315 0062; # (a◌̕◌̀◌֮◌ͪb; à◌֮◌ͪ◌̕b; a◌֮◌̀◌ͪ◌̕b; à◌֮◌ͪ◌̕b; a◌֮◌̀◌ͪ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING LATIN SMALL LETTER H, LATIN SMALL LETTER B +0061 036A 0315 0300 05AE 0062;0061 05AE 036A 0300 0315 0062;0061 05AE 036A 0300 0315 0062;0061 05AE 036A 0300 0315 0062;0061 05AE 036A 0300 0315 0062; # (a◌ͪ◌̕◌̀◌֮b; a◌֮◌ͪ◌̀◌̕b; a◌֮◌ͪ◌̀◌̕b; a◌֮◌ͪ◌̀◌̕b; a◌֮◌ͪ◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING LATIN SMALL LETTER H, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 036B 0062;00E0 05AE 036B 0315 0062;0061 05AE 0300 036B 0315 0062;00E0 05AE 036B 0315 0062;0061 05AE 0300 036B 0315 0062; # (a◌̕◌̀◌֮◌ͫb; à◌֮◌ͫ◌̕b; a◌֮◌̀◌ͫ◌̕b; à◌֮◌ͫ◌̕b; a◌֮◌̀◌ͫ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING LATIN SMALL LETTER M, LATIN SMALL LETTER B +0061 036B 0315 0300 05AE 0062;0061 05AE 036B 0300 0315 0062;0061 05AE 036B 0300 0315 0062;0061 05AE 036B 0300 0315 0062;0061 05AE 036B 0300 0315 0062; # (a◌ͫ◌̕◌̀◌֮b; a◌֮◌ͫ◌̀◌̕b; a◌֮◌ͫ◌̀◌̕b; a◌֮◌ͫ◌̀◌̕b; a◌֮◌ͫ◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING LATIN SMALL LETTER M, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 036C 0062;00E0 05AE 036C 0315 0062;0061 05AE 0300 036C 0315 0062;00E0 05AE 036C 0315 0062;0061 05AE 0300 036C 0315 0062; # (a◌̕◌̀◌֮◌ͬb; à◌֮◌ͬ◌̕b; a◌֮◌̀◌ͬ◌̕b; à◌֮◌ͬ◌̕b; a◌֮◌̀◌ͬ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING LATIN SMALL LETTER R, LATIN SMALL LETTER B +0061 036C 0315 0300 05AE 0062;0061 05AE 036C 0300 0315 0062;0061 05AE 036C 0300 0315 0062;0061 05AE 036C 0300 0315 0062;0061 05AE 036C 0300 0315 0062; # (a◌ͬ◌̕◌̀◌֮b; a◌֮◌ͬ◌̀◌̕b; a◌֮◌ͬ◌̀◌̕b; a◌֮◌ͬ◌̀◌̕b; a◌֮◌ͬ◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING LATIN SMALL LETTER R, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 036D 0062;00E0 05AE 036D 0315 0062;0061 05AE 0300 036D 0315 0062;00E0 05AE 036D 0315 0062;0061 05AE 0300 036D 0315 0062; # (a◌̕◌̀◌֮◌ͭb; à◌֮◌ͭ◌̕b; a◌֮◌̀◌ͭ◌̕b; à◌֮◌ͭ◌̕b; a◌֮◌̀◌ͭ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING LATIN SMALL LETTER T, LATIN SMALL LETTER B +0061 036D 0315 0300 05AE 0062;0061 05AE 036D 0300 0315 0062;0061 05AE 036D 0300 0315 0062;0061 05AE 036D 0300 0315 0062;0061 05AE 036D 0300 0315 0062; # (a◌ͭ◌̕◌̀◌֮b; a◌֮◌ͭ◌̀◌̕b; a◌֮◌ͭ◌̀◌̕b; a◌֮◌ͭ◌̀◌̕b; a◌֮◌ͭ◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING LATIN SMALL LETTER T, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 036E 0062;00E0 05AE 036E 0315 0062;0061 05AE 0300 036E 0315 0062;00E0 05AE 036E 0315 0062;0061 05AE 0300 036E 0315 0062; # (a◌̕◌̀◌֮◌ͮb; à◌֮◌ͮ◌̕b; a◌֮◌̀◌ͮ◌̕b; à◌֮◌ͮ◌̕b; a◌֮◌̀◌ͮ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING LATIN SMALL LETTER V, LATIN SMALL LETTER B +0061 036E 0315 0300 05AE 0062;0061 05AE 036E 0300 0315 0062;0061 05AE 036E 0300 0315 0062;0061 05AE 036E 0300 0315 0062;0061 05AE 036E 0300 0315 0062; # (a◌ͮ◌̕◌̀◌֮b; a◌֮◌ͮ◌̀◌̕b; a◌֮◌ͮ◌̀◌̕b; a◌֮◌ͮ◌̀◌̕b; a◌֮◌ͮ◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING LATIN SMALL LETTER V, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 036F 0062;00E0 05AE 036F 0315 0062;0061 05AE 0300 036F 0315 0062;00E0 05AE 036F 0315 0062;0061 05AE 0300 036F 0315 0062; # (a◌̕◌̀◌֮◌ͯb; à◌֮◌ͯ◌̕b; a◌֮◌̀◌ͯ◌̕b; à◌֮◌ͯ◌̕b; a◌֮◌̀◌ͯ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING LATIN SMALL LETTER X, LATIN SMALL LETTER B +0061 036F 0315 0300 05AE 0062;0061 05AE 036F 0300 0315 0062;0061 05AE 036F 0300 0315 0062;0061 05AE 036F 0300 0315 0062;0061 05AE 036F 0300 0315 0062; # (a◌ͯ◌̕◌̀◌֮b; a◌֮◌ͯ◌̀◌̕b; a◌֮◌ͯ◌̀◌̕b; a◌֮◌ͯ◌̀◌̕b; a◌֮◌ͯ◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING LATIN SMALL LETTER X, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0483 0062;00E0 05AE 0483 0315 0062;0061 05AE 0300 0483 0315 0062;00E0 05AE 0483 0315 0062;0061 05AE 0300 0483 0315 0062; # (a◌̕◌̀◌֮◌҃b; à◌֮◌҃◌̕b; a◌֮◌̀◌҃◌̕b; à◌֮◌҃◌̕b; a◌֮◌̀◌҃◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING CYRILLIC TITLO, LATIN SMALL LETTER B +0061 0483 0315 0300 05AE 0062;0061 05AE 0483 0300 0315 0062;0061 05AE 0483 0300 0315 0062;0061 05AE 0483 0300 0315 0062;0061 05AE 0483 0300 0315 0062; # (a◌҃◌̕◌̀◌֮b; a◌֮◌҃◌̀◌̕b; a◌֮◌҃◌̀◌̕b; a◌֮◌҃◌̀◌̕b; a◌֮◌҃◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING CYRILLIC TITLO, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0484 0062;00E0 05AE 0484 0315 0062;0061 05AE 0300 0484 0315 0062;00E0 05AE 0484 0315 0062;0061 05AE 0300 0484 0315 0062; # (a◌̕◌̀◌֮◌҄b; à◌֮◌҄◌̕b; a◌֮◌̀◌҄◌̕b; à◌֮◌҄◌̕b; a◌֮◌̀◌҄◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING CYRILLIC PALATALIZATION, LATIN SMALL LETTER B +0061 0484 0315 0300 05AE 0062;0061 05AE 0484 0300 0315 0062;0061 05AE 0484 0300 0315 0062;0061 05AE 0484 0300 0315 0062;0061 05AE 0484 0300 0315 0062; # (a◌҄◌̕◌̀◌֮b; a◌֮◌҄◌̀◌̕b; a◌֮◌҄◌̀◌̕b; a◌֮◌҄◌̀◌̕b; a◌֮◌҄◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING CYRILLIC PALATALIZATION, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0485 0062;00E0 05AE 0485 0315 0062;0061 05AE 0300 0485 0315 0062;00E0 05AE 0485 0315 0062;0061 05AE 0300 0485 0315 0062; # (a◌̕◌̀◌֮◌҅b; à◌֮◌҅◌̕b; a◌֮◌̀◌҅◌̕b; à◌֮◌҅◌̕b; a◌֮◌̀◌҅◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING CYRILLIC DASIA PNEUMATA, LATIN SMALL LETTER B +0061 0485 0315 0300 05AE 0062;0061 05AE 0485 0300 0315 0062;0061 05AE 0485 0300 0315 0062;0061 05AE 0485 0300 0315 0062;0061 05AE 0485 0300 0315 0062; # (a◌҅◌̕◌̀◌֮b; a◌֮◌҅◌̀◌̕b; a◌֮◌҅◌̀◌̕b; a◌֮◌҅◌̀◌̕b; a◌֮◌҅◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING CYRILLIC DASIA PNEUMATA, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0486 0062;00E0 05AE 0486 0315 0062;0061 05AE 0300 0486 0315 0062;00E0 05AE 0486 0315 0062;0061 05AE 0300 0486 0315 0062; # (a◌̕◌̀◌֮◌҆b; à◌֮◌҆◌̕b; a◌֮◌̀◌҆◌̕b; à◌֮◌҆◌̕b; a◌֮◌̀◌҆◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING CYRILLIC PSILI PNEUMATA, LATIN SMALL LETTER B +0061 0486 0315 0300 05AE 0062;0061 05AE 0486 0300 0315 0062;0061 05AE 0486 0300 0315 0062;0061 05AE 0486 0300 0315 0062;0061 05AE 0486 0300 0315 0062; # (a◌҆◌̕◌̀◌֮b; a◌֮◌҆◌̀◌̕b; a◌֮◌҆◌̀◌̕b; a◌֮◌҆◌̀◌̕b; a◌֮◌҆◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING CYRILLIC PSILI PNEUMATA, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 0591 0062;0061 302A 0316 0591 059A 0062;0061 302A 0316 0591 059A 0062;0061 302A 0316 0591 059A 0062;0061 302A 0316 0591 059A 0062; # (a◌֚◌̖◌〪◌֑b; a◌〪◌̖◌֑◌֚b; a◌〪◌̖◌֑◌֚b; a◌〪◌̖◌֑◌֚b; a◌〪◌̖◌֑◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, HEBREW ACCENT ETNAHTA, LATIN SMALL LETTER B +0061 0591 059A 0316 302A 0062;0061 302A 0591 0316 059A 0062;0061 302A 0591 0316 059A 0062;0061 302A 0591 0316 059A 0062;0061 302A 0591 0316 059A 0062; # (a◌֑◌֚◌̖◌〪b; a◌〪◌֑◌̖◌֚b; a◌〪◌֑◌̖◌֚b; a◌〪◌֑◌̖◌֚b; a◌〪◌֑◌̖◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT ETNAHTA, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 0592 0062;00E0 05AE 0592 0315 0062;0061 05AE 0300 0592 0315 0062;00E0 05AE 0592 0315 0062;0061 05AE 0300 0592 0315 0062; # (a◌̕◌̀◌֮◌֒b; à◌֮◌֒◌̕b; a◌֮◌̀◌֒◌̕b; à◌֮◌֒◌̕b; a◌֮◌̀◌֒◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HEBREW ACCENT SEGOL, LATIN SMALL LETTER B +0061 0592 0315 0300 05AE 0062;0061 05AE 0592 0300 0315 0062;0061 05AE 0592 0300 0315 0062;0061 05AE 0592 0300 0315 0062;0061 05AE 0592 0300 0315 0062; # (a◌֒◌̕◌̀◌֮b; a◌֮◌֒◌̀◌̕b; a◌֮◌֒◌̀◌̕b; a◌֮◌֒◌̀◌̕b; a◌֮◌֒◌̀◌̕b; ) LATIN SMALL LETTER A, HEBREW ACCENT SEGOL, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0593 0062;00E0 05AE 0593 0315 0062;0061 05AE 0300 0593 0315 0062;00E0 05AE 0593 0315 0062;0061 05AE 0300 0593 0315 0062; # (a◌̕◌̀◌֮◌֓b; à◌֮◌֓◌̕b; a◌֮◌̀◌֓◌̕b; à◌֮◌֓◌̕b; a◌֮◌̀◌֓◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HEBREW ACCENT SHALSHELET, LATIN SMALL LETTER B +0061 0593 0315 0300 05AE 0062;0061 05AE 0593 0300 0315 0062;0061 05AE 0593 0300 0315 0062;0061 05AE 0593 0300 0315 0062;0061 05AE 0593 0300 0315 0062; # (a◌֓◌̕◌̀◌֮b; a◌֮◌֓◌̀◌̕b; a◌֮◌֓◌̀◌̕b; a◌֮◌֓◌̀◌̕b; a◌֮◌֓◌̀◌̕b; ) LATIN SMALL LETTER A, HEBREW ACCENT SHALSHELET, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0594 0062;00E0 05AE 0594 0315 0062;0061 05AE 0300 0594 0315 0062;00E0 05AE 0594 0315 0062;0061 05AE 0300 0594 0315 0062; # (a◌̕◌̀◌֮◌֔b; à◌֮◌֔◌̕b; a◌֮◌̀◌֔◌̕b; à◌֮◌֔◌̕b; a◌֮◌̀◌֔◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HEBREW ACCENT ZAQEF QATAN, LATIN SMALL LETTER B +0061 0594 0315 0300 05AE 0062;0061 05AE 0594 0300 0315 0062;0061 05AE 0594 0300 0315 0062;0061 05AE 0594 0300 0315 0062;0061 05AE 0594 0300 0315 0062; # (a◌֔◌̕◌̀◌֮b; a◌֮◌֔◌̀◌̕b; a◌֮◌֔◌̀◌̕b; a◌֮◌֔◌̀◌̕b; a◌֮◌֔◌̀◌̕b; ) LATIN SMALL LETTER A, HEBREW ACCENT ZAQEF QATAN, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0595 0062;00E0 05AE 0595 0315 0062;0061 05AE 0300 0595 0315 0062;00E0 05AE 0595 0315 0062;0061 05AE 0300 0595 0315 0062; # (a◌̕◌̀◌֮◌֕b; à◌֮◌֕◌̕b; a◌֮◌̀◌֕◌̕b; à◌֮◌֕◌̕b; a◌֮◌̀◌֕◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HEBREW ACCENT ZAQEF GADOL, LATIN SMALL LETTER B +0061 0595 0315 0300 05AE 0062;0061 05AE 0595 0300 0315 0062;0061 05AE 0595 0300 0315 0062;0061 05AE 0595 0300 0315 0062;0061 05AE 0595 0300 0315 0062; # (a◌֕◌̕◌̀◌֮b; a◌֮◌֕◌̀◌̕b; a◌֮◌֕◌̀◌̕b; a◌֮◌֕◌̀◌̕b; a◌֮◌֕◌̀◌̕b; ) LATIN SMALL LETTER A, HEBREW ACCENT ZAQEF GADOL, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 0596 0062;0061 302A 0316 0596 059A 0062;0061 302A 0316 0596 059A 0062;0061 302A 0316 0596 059A 0062;0061 302A 0316 0596 059A 0062; # (a◌֚◌̖◌〪◌֖b; a◌〪◌̖◌֖◌֚b; a◌〪◌̖◌֖◌֚b; a◌〪◌̖◌֖◌֚b; a◌〪◌̖◌֖◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, HEBREW ACCENT TIPEHA, LATIN SMALL LETTER B +0061 0596 059A 0316 302A 0062;0061 302A 0596 0316 059A 0062;0061 302A 0596 0316 059A 0062;0061 302A 0596 0316 059A 0062;0061 302A 0596 0316 059A 0062; # (a◌֖◌֚◌̖◌〪b; a◌〪◌֖◌̖◌֚b; a◌〪◌֖◌̖◌֚b; a◌〪◌֖◌̖◌֚b; a◌〪◌֖◌̖◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT TIPEHA, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 0597 0062;00E0 05AE 0597 0315 0062;0061 05AE 0300 0597 0315 0062;00E0 05AE 0597 0315 0062;0061 05AE 0300 0597 0315 0062; # (a◌̕◌̀◌֮◌֗b; à◌֮◌֗◌̕b; a◌֮◌̀◌֗◌̕b; à◌֮◌֗◌̕b; a◌֮◌̀◌֗◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HEBREW ACCENT REVIA, LATIN SMALL LETTER B +0061 0597 0315 0300 05AE 0062;0061 05AE 0597 0300 0315 0062;0061 05AE 0597 0300 0315 0062;0061 05AE 0597 0300 0315 0062;0061 05AE 0597 0300 0315 0062; # (a◌֗◌̕◌̀◌֮b; a◌֮◌֗◌̀◌̕b; a◌֮◌֗◌̀◌̕b; a◌֮◌֗◌̀◌̕b; a◌֮◌֗◌̀◌̕b; ) LATIN SMALL LETTER A, HEBREW ACCENT REVIA, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0598 0062;00E0 05AE 0598 0315 0062;0061 05AE 0300 0598 0315 0062;00E0 05AE 0598 0315 0062;0061 05AE 0300 0598 0315 0062; # (a◌̕◌̀◌֮◌֘b; à◌֮◌֘◌̕b; a◌֮◌̀◌֘◌̕b; à◌֮◌֘◌̕b; a◌֮◌̀◌֘◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HEBREW ACCENT ZARQA, LATIN SMALL LETTER B +0061 0598 0315 0300 05AE 0062;0061 05AE 0598 0300 0315 0062;0061 05AE 0598 0300 0315 0062;0061 05AE 0598 0300 0315 0062;0061 05AE 0598 0300 0315 0062; # (a◌֘◌̕◌̀◌֮b; a◌֮◌֘◌̀◌̕b; a◌֮◌֘◌̀◌̕b; a◌֮◌֘◌̀◌̕b; a◌֮◌֘◌̀◌̕b; ) LATIN SMALL LETTER A, HEBREW ACCENT ZARQA, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0599 0062;00E0 05AE 0599 0315 0062;0061 05AE 0300 0599 0315 0062;00E0 05AE 0599 0315 0062;0061 05AE 0300 0599 0315 0062; # (a◌̕◌̀◌֮◌֙b; à◌֮◌֙◌̕b; a◌֮◌̀◌֙◌̕b; à◌֮◌֙◌̕b; a◌֮◌̀◌֙◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HEBREW ACCENT PASHTA, LATIN SMALL LETTER B +0061 0599 0315 0300 05AE 0062;0061 05AE 0599 0300 0315 0062;0061 05AE 0599 0300 0315 0062;0061 05AE 0599 0300 0315 0062;0061 05AE 0599 0300 0315 0062; # (a◌֙◌̕◌̀◌֮b; a◌֮◌֙◌̀◌̕b; a◌֮◌֙◌̀◌̕b; a◌֮◌֙◌̀◌̕b; a◌֮◌֙◌̀◌̕b; ) LATIN SMALL LETTER A, HEBREW ACCENT PASHTA, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 302E 059A 0316 059A 0062;0061 0316 059A 059A 302E 0062;0061 0316 059A 059A 302E 0062;0061 0316 059A 059A 302E 0062;0061 0316 059A 059A 302E 0062; # (a◌〮◌֚◌̖◌֚b; a◌̖◌֚◌֚◌〮b; a◌̖◌֚◌֚◌〮b; a◌̖◌֚◌֚◌〮b; a◌̖◌֚◌֚◌〮b; ) LATIN SMALL LETTER A, HANGUL SINGLE DOT TONE MARK, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, HEBREW ACCENT YETIV, LATIN SMALL LETTER B +0061 059A 302E 059A 0316 0062;0061 0316 059A 059A 302E 0062;0061 0316 059A 059A 302E 0062;0061 0316 059A 059A 302E 0062;0061 0316 059A 059A 302E 0062; # (a◌֚◌〮◌֚◌̖b; a◌̖◌֚◌֚◌〮b; a◌̖◌֚◌֚◌〮b; a◌̖◌֚◌֚◌〮b; a◌̖◌֚◌֚◌〮b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, HANGUL SINGLE DOT TONE MARK, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, LATIN SMALL LETTER B +0061 059A 0316 302A 059B 0062;0061 302A 0316 059B 059A 0062;0061 302A 0316 059B 059A 0062;0061 302A 0316 059B 059A 0062;0061 302A 0316 059B 059A 0062; # (a◌֚◌̖◌〪◌֛b; a◌〪◌̖◌֛◌֚b; a◌〪◌̖◌֛◌֚b; a◌〪◌̖◌֛◌֚b; a◌〪◌̖◌֛◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, HEBREW ACCENT TEVIR, LATIN SMALL LETTER B +0061 059B 059A 0316 302A 0062;0061 302A 059B 0316 059A 0062;0061 302A 059B 0316 059A 0062;0061 302A 059B 0316 059A 0062;0061 302A 059B 0316 059A 0062; # (a◌֛◌֚◌̖◌〪b; a◌〪◌֛◌̖◌֚b; a◌〪◌֛◌̖◌֚b; a◌〪◌֛◌̖◌֚b; a◌〪◌֛◌̖◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT TEVIR, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 059C 0062;00E0 05AE 059C 0315 0062;0061 05AE 0300 059C 0315 0062;00E0 05AE 059C 0315 0062;0061 05AE 0300 059C 0315 0062; # (a◌̕◌̀◌֮◌֜b; à◌֮◌֜◌̕b; a◌֮◌̀◌֜◌̕b; à◌֮◌֜◌̕b; a◌֮◌̀◌֜◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HEBREW ACCENT GERESH, LATIN SMALL LETTER B +0061 059C 0315 0300 05AE 0062;0061 05AE 059C 0300 0315 0062;0061 05AE 059C 0300 0315 0062;0061 05AE 059C 0300 0315 0062;0061 05AE 059C 0300 0315 0062; # (a◌֜◌̕◌̀◌֮b; a◌֮◌֜◌̀◌̕b; a◌֮◌֜◌̀◌̕b; a◌֮◌֜◌̀◌̕b; a◌֮◌֜◌̀◌̕b; ) LATIN SMALL LETTER A, HEBREW ACCENT GERESH, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 059D 0062;00E0 05AE 059D 0315 0062;0061 05AE 0300 059D 0315 0062;00E0 05AE 059D 0315 0062;0061 05AE 0300 059D 0315 0062; # (a◌̕◌̀◌֮◌֝b; à◌֮◌֝◌̕b; a◌֮◌̀◌֝◌̕b; à◌֮◌֝◌̕b; a◌֮◌̀◌֝◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HEBREW ACCENT GERESH MUQDAM, LATIN SMALL LETTER B +0061 059D 0315 0300 05AE 0062;0061 05AE 059D 0300 0315 0062;0061 05AE 059D 0300 0315 0062;0061 05AE 059D 0300 0315 0062;0061 05AE 059D 0300 0315 0062; # (a◌֝◌̕◌̀◌֮b; a◌֮◌֝◌̀◌̕b; a◌֮◌֝◌̀◌̕b; a◌֮◌֝◌̀◌̕b; a◌֮◌֝◌̀◌̕b; ) LATIN SMALL LETTER A, HEBREW ACCENT GERESH MUQDAM, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 059E 0062;00E0 05AE 059E 0315 0062;0061 05AE 0300 059E 0315 0062;00E0 05AE 059E 0315 0062;0061 05AE 0300 059E 0315 0062; # (a◌̕◌̀◌֮◌֞b; à◌֮◌֞◌̕b; a◌֮◌̀◌֞◌̕b; à◌֮◌֞◌̕b; a◌֮◌̀◌֞◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HEBREW ACCENT GERSHAYIM, LATIN SMALL LETTER B +0061 059E 0315 0300 05AE 0062;0061 05AE 059E 0300 0315 0062;0061 05AE 059E 0300 0315 0062;0061 05AE 059E 0300 0315 0062;0061 05AE 059E 0300 0315 0062; # (a◌֞◌̕◌̀◌֮b; a◌֮◌֞◌̀◌̕b; a◌֮◌֞◌̀◌̕b; a◌֮◌֞◌̀◌̕b; a◌֮◌֞◌̀◌̕b; ) LATIN SMALL LETTER A, HEBREW ACCENT GERSHAYIM, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 059F 0062;00E0 05AE 059F 0315 0062;0061 05AE 0300 059F 0315 0062;00E0 05AE 059F 0315 0062;0061 05AE 0300 059F 0315 0062; # (a◌̕◌̀◌֮◌֟b; à◌֮◌֟◌̕b; a◌֮◌̀◌֟◌̕b; à◌֮◌֟◌̕b; a◌֮◌̀◌֟◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HEBREW ACCENT QARNEY PARA, LATIN SMALL LETTER B +0061 059F 0315 0300 05AE 0062;0061 05AE 059F 0300 0315 0062;0061 05AE 059F 0300 0315 0062;0061 05AE 059F 0300 0315 0062;0061 05AE 059F 0300 0315 0062; # (a◌֟◌̕◌̀◌֮b; a◌֮◌֟◌̀◌̕b; a◌֮◌֟◌̀◌̕b; a◌֮◌֟◌̀◌̕b; a◌֮◌֟◌̀◌̕b; ) LATIN SMALL LETTER A, HEBREW ACCENT QARNEY PARA, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 05A0 0062;00E0 05AE 05A0 0315 0062;0061 05AE 0300 05A0 0315 0062;00E0 05AE 05A0 0315 0062;0061 05AE 0300 05A0 0315 0062; # (a◌̕◌̀◌֮◌֠b; à◌֮◌֠◌̕b; a◌֮◌̀◌֠◌̕b; à◌֮◌֠◌̕b; a◌֮◌̀◌֠◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HEBREW ACCENT TELISHA GEDOLA, LATIN SMALL LETTER B +0061 05A0 0315 0300 05AE 0062;0061 05AE 05A0 0300 0315 0062;0061 05AE 05A0 0300 0315 0062;0061 05AE 05A0 0300 0315 0062;0061 05AE 05A0 0300 0315 0062; # (a◌֠◌̕◌̀◌֮b; a◌֮◌֠◌̀◌̕b; a◌֮◌֠◌̀◌̕b; a◌֮◌֠◌̀◌̕b; a◌֮◌֠◌̀◌̕b; ) LATIN SMALL LETTER A, HEBREW ACCENT TELISHA GEDOLA, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 05A1 0062;00E0 05AE 05A1 0315 0062;0061 05AE 0300 05A1 0315 0062;00E0 05AE 05A1 0315 0062;0061 05AE 0300 05A1 0315 0062; # (a◌̕◌̀◌֮◌֡b; à◌֮◌֡◌̕b; a◌֮◌̀◌֡◌̕b; à◌֮◌֡◌̕b; a◌֮◌̀◌֡◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HEBREW ACCENT PAZER, LATIN SMALL LETTER B +0061 05A1 0315 0300 05AE 0062;0061 05AE 05A1 0300 0315 0062;0061 05AE 05A1 0300 0315 0062;0061 05AE 05A1 0300 0315 0062;0061 05AE 05A1 0300 0315 0062; # (a◌֡◌̕◌̀◌֮b; a◌֮◌֡◌̀◌̕b; a◌֮◌֡◌̀◌̕b; a◌֮◌֡◌̀◌̕b; a◌֮◌֡◌̀◌̕b; ) LATIN SMALL LETTER A, HEBREW ACCENT PAZER, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 05A3 0062;0061 302A 0316 05A3 059A 0062;0061 302A 0316 05A3 059A 0062;0061 302A 0316 05A3 059A 0062;0061 302A 0316 05A3 059A 0062; # (a◌֚◌̖◌〪◌֣b; a◌〪◌̖◌֣◌֚b; a◌〪◌̖◌֣◌֚b; a◌〪◌̖◌֣◌֚b; a◌〪◌̖◌֣◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, HEBREW ACCENT MUNAH, LATIN SMALL LETTER B +0061 05A3 059A 0316 302A 0062;0061 302A 05A3 0316 059A 0062;0061 302A 05A3 0316 059A 0062;0061 302A 05A3 0316 059A 0062;0061 302A 05A3 0316 059A 0062; # (a◌֣◌֚◌̖◌〪b; a◌〪◌֣◌̖◌֚b; a◌〪◌֣◌̖◌֚b; a◌〪◌֣◌̖◌֚b; a◌〪◌֣◌̖◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT MUNAH, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 05A4 0062;0061 302A 0316 05A4 059A 0062;0061 302A 0316 05A4 059A 0062;0061 302A 0316 05A4 059A 0062;0061 302A 0316 05A4 059A 0062; # (a◌֚◌̖◌〪◌֤b; a◌〪◌̖◌֤◌֚b; a◌〪◌̖◌֤◌֚b; a◌〪◌̖◌֤◌֚b; a◌〪◌̖◌֤◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, HEBREW ACCENT MAHAPAKH, LATIN SMALL LETTER B +0061 05A4 059A 0316 302A 0062;0061 302A 05A4 0316 059A 0062;0061 302A 05A4 0316 059A 0062;0061 302A 05A4 0316 059A 0062;0061 302A 05A4 0316 059A 0062; # (a◌֤◌֚◌̖◌〪b; a◌〪◌֤◌̖◌֚b; a◌〪◌֤◌̖◌֚b; a◌〪◌֤◌̖◌֚b; a◌〪◌֤◌̖◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT MAHAPAKH, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 05A5 0062;0061 302A 0316 05A5 059A 0062;0061 302A 0316 05A5 059A 0062;0061 302A 0316 05A5 059A 0062;0061 302A 0316 05A5 059A 0062; # (a◌֚◌̖◌〪◌֥b; a◌〪◌̖◌֥◌֚b; a◌〪◌̖◌֥◌֚b; a◌〪◌̖◌֥◌֚b; a◌〪◌̖◌֥◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, HEBREW ACCENT MERKHA, LATIN SMALL LETTER B +0061 05A5 059A 0316 302A 0062;0061 302A 05A5 0316 059A 0062;0061 302A 05A5 0316 059A 0062;0061 302A 05A5 0316 059A 0062;0061 302A 05A5 0316 059A 0062; # (a◌֥◌֚◌̖◌〪b; a◌〪◌֥◌̖◌֚b; a◌〪◌֥◌̖◌֚b; a◌〪◌֥◌̖◌֚b; a◌〪◌֥◌̖◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT MERKHA, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 05A6 0062;0061 302A 0316 05A6 059A 0062;0061 302A 0316 05A6 059A 0062;0061 302A 0316 05A6 059A 0062;0061 302A 0316 05A6 059A 0062; # (a◌֚◌̖◌〪◌֦b; a◌〪◌̖◌֦◌֚b; a◌〪◌̖◌֦◌֚b; a◌〪◌̖◌֦◌֚b; a◌〪◌̖◌֦◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, HEBREW ACCENT MERKHA KEFULA, LATIN SMALL LETTER B +0061 05A6 059A 0316 302A 0062;0061 302A 05A6 0316 059A 0062;0061 302A 05A6 0316 059A 0062;0061 302A 05A6 0316 059A 0062;0061 302A 05A6 0316 059A 0062; # (a◌֦◌֚◌̖◌〪b; a◌〪◌֦◌̖◌֚b; a◌〪◌֦◌̖◌֚b; a◌〪◌֦◌̖◌֚b; a◌〪◌֦◌̖◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT MERKHA KEFULA, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 05A7 0062;0061 302A 0316 05A7 059A 0062;0061 302A 0316 05A7 059A 0062;0061 302A 0316 05A7 059A 0062;0061 302A 0316 05A7 059A 0062; # (a◌֚◌̖◌〪◌֧b; a◌〪◌̖◌֧◌֚b; a◌〪◌̖◌֧◌֚b; a◌〪◌̖◌֧◌֚b; a◌〪◌̖◌֧◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, HEBREW ACCENT DARGA, LATIN SMALL LETTER B +0061 05A7 059A 0316 302A 0062;0061 302A 05A7 0316 059A 0062;0061 302A 05A7 0316 059A 0062;0061 302A 05A7 0316 059A 0062;0061 302A 05A7 0316 059A 0062; # (a◌֧◌֚◌̖◌〪b; a◌〪◌֧◌̖◌֚b; a◌〪◌֧◌̖◌֚b; a◌〪◌֧◌̖◌֚b; a◌〪◌֧◌̖◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT DARGA, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 05A8 0062;00E0 05AE 05A8 0315 0062;0061 05AE 0300 05A8 0315 0062;00E0 05AE 05A8 0315 0062;0061 05AE 0300 05A8 0315 0062; # (a◌̕◌̀◌֮◌֨b; à◌֮◌֨◌̕b; a◌֮◌̀◌֨◌̕b; à◌֮◌֨◌̕b; a◌֮◌̀◌֨◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HEBREW ACCENT QADMA, LATIN SMALL LETTER B +0061 05A8 0315 0300 05AE 0062;0061 05AE 05A8 0300 0315 0062;0061 05AE 05A8 0300 0315 0062;0061 05AE 05A8 0300 0315 0062;0061 05AE 05A8 0300 0315 0062; # (a◌֨◌̕◌̀◌֮b; a◌֮◌֨◌̀◌̕b; a◌֮◌֨◌̀◌̕b; a◌֮◌֨◌̀◌̕b; a◌֮◌֨◌̀◌̕b; ) LATIN SMALL LETTER A, HEBREW ACCENT QADMA, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 05A9 0062;00E0 05AE 05A9 0315 0062;0061 05AE 0300 05A9 0315 0062;00E0 05AE 05A9 0315 0062;0061 05AE 0300 05A9 0315 0062; # (a◌̕◌̀◌֮◌֩b; à◌֮◌֩◌̕b; a◌֮◌̀◌֩◌̕b; à◌֮◌֩◌̕b; a◌֮◌̀◌֩◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HEBREW ACCENT TELISHA QETANA, LATIN SMALL LETTER B +0061 05A9 0315 0300 05AE 0062;0061 05AE 05A9 0300 0315 0062;0061 05AE 05A9 0300 0315 0062;0061 05AE 05A9 0300 0315 0062;0061 05AE 05A9 0300 0315 0062; # (a◌֩◌̕◌̀◌֮b; a◌֮◌֩◌̀◌̕b; a◌֮◌֩◌̀◌̕b; a◌֮◌֩◌̀◌̕b; a◌֮◌֩◌̀◌̕b; ) LATIN SMALL LETTER A, HEBREW ACCENT TELISHA QETANA, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 05AA 0062;0061 302A 0316 05AA 059A 0062;0061 302A 0316 05AA 059A 0062;0061 302A 0316 05AA 059A 0062;0061 302A 0316 05AA 059A 0062; # (a◌֚◌̖◌〪◌֪b; a◌〪◌̖◌֪◌֚b; a◌〪◌̖◌֪◌֚b; a◌〪◌̖◌֪◌֚b; a◌〪◌̖◌֪◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, HEBREW ACCENT YERAH BEN YOMO, LATIN SMALL LETTER B +0061 05AA 059A 0316 302A 0062;0061 302A 05AA 0316 059A 0062;0061 302A 05AA 0316 059A 0062;0061 302A 05AA 0316 059A 0062;0061 302A 05AA 0316 059A 0062; # (a◌֪◌֚◌̖◌〪b; a◌〪◌֪◌̖◌֚b; a◌〪◌֪◌̖◌֚b; a◌〪◌֪◌̖◌֚b; a◌〪◌֪◌̖◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YERAH BEN YOMO, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 05AB 0062;00E0 05AE 05AB 0315 0062;0061 05AE 0300 05AB 0315 0062;00E0 05AE 05AB 0315 0062;0061 05AE 0300 05AB 0315 0062; # (a◌̕◌̀◌֮◌֫b; à◌֮◌֫◌̕b; a◌֮◌̀◌֫◌̕b; à◌֮◌֫◌̕b; a◌֮◌̀◌֫◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HEBREW ACCENT OLE, LATIN SMALL LETTER B +0061 05AB 0315 0300 05AE 0062;0061 05AE 05AB 0300 0315 0062;0061 05AE 05AB 0300 0315 0062;0061 05AE 05AB 0300 0315 0062;0061 05AE 05AB 0300 0315 0062; # (a◌֫◌̕◌̀◌֮b; a◌֮◌֫◌̀◌̕b; a◌֮◌֫◌̀◌̕b; a◌֮◌֫◌̀◌̕b; a◌֮◌֫◌̀◌̕b; ) LATIN SMALL LETTER A, HEBREW ACCENT OLE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 05AC 0062;00E0 05AE 05AC 0315 0062;0061 05AE 0300 05AC 0315 0062;00E0 05AE 05AC 0315 0062;0061 05AE 0300 05AC 0315 0062; # (a◌̕◌̀◌֮◌֬b; à◌֮◌֬◌̕b; a◌֮◌̀◌֬◌̕b; à◌֮◌֬◌̕b; a◌֮◌̀◌֬◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HEBREW ACCENT ILUY, LATIN SMALL LETTER B +0061 05AC 0315 0300 05AE 0062;0061 05AE 05AC 0300 0315 0062;0061 05AE 05AC 0300 0315 0062;0061 05AE 05AC 0300 0315 0062;0061 05AE 05AC 0300 0315 0062; # (a◌֬◌̕◌̀◌֮b; a◌֮◌֬◌̀◌̕b; a◌֮◌֬◌̀◌̕b; a◌֮◌֬◌̀◌̕b; a◌֮◌֬◌̀◌̕b; ) LATIN SMALL LETTER A, HEBREW ACCENT ILUY, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 302E 059A 0316 05AD 0062;0061 0316 059A 05AD 302E 0062;0061 0316 059A 05AD 302E 0062;0061 0316 059A 05AD 302E 0062;0061 0316 059A 05AD 302E 0062; # (a◌〮◌֚◌̖◌֭b; a◌̖◌֚◌֭◌〮b; a◌̖◌֚◌֭◌〮b; a◌̖◌֚◌֭◌〮b; a◌̖◌֚◌֭◌〮b; ) LATIN SMALL LETTER A, HANGUL SINGLE DOT TONE MARK, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, HEBREW ACCENT DEHI, LATIN SMALL LETTER B +0061 05AD 302E 059A 0316 0062;0061 0316 05AD 059A 302E 0062;0061 0316 05AD 059A 302E 0062;0061 0316 05AD 059A 302E 0062;0061 0316 05AD 059A 302E 0062; # (a◌֭◌〮◌֚◌̖b; a◌̖◌֭◌֚◌〮b; a◌̖◌֭◌֚◌〮b; a◌̖◌֭◌֚◌〮b; a◌̖◌֭◌֚◌〮b; ) LATIN SMALL LETTER A, HEBREW ACCENT DEHI, HANGUL SINGLE DOT TONE MARK, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, LATIN SMALL LETTER B +0061 0300 05AE 1D16D 05AE 0062;00E0 1D16D 05AE 05AE 0062;0061 1D16D 05AE 05AE 0300 0062;00E0 1D16D 05AE 05AE 0062;0061 1D16D 05AE 05AE 0300 0062; # (a◌̀◌𝅭𝅭֮◌֮b; à𝅭𝅭◌֮◌֮b; a𝅭𝅭◌֮◌֮◌̀b; à𝅭𝅭◌֮◌֮b; a𝅭𝅭◌֮◌֮◌̀b; ) LATIN SMALL LETTER A, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 05AE 0300 05AE 1D16D 0062;00E0 1D16D 05AE 05AE 0062;0061 1D16D 05AE 05AE 0300 0062;00E0 1D16D 05AE 05AE 0062;0061 1D16D 05AE 05AE 0300 0062; # (a◌֮◌̀◌𝅭𝅭֮b; à𝅭𝅭◌֮◌֮b; a𝅭𝅭◌֮◌֮◌̀b; à𝅭𝅭◌֮◌֮b; a𝅭𝅭◌֮◌֮◌̀b; ) LATIN SMALL LETTER A, HEBREW ACCENT ZINOR, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, LATIN SMALL LETTER B +0061 0315 0300 05AE 05AF 0062;00E0 05AE 05AF 0315 0062;0061 05AE 0300 05AF 0315 0062;00E0 05AE 05AF 0315 0062;0061 05AE 0300 05AF 0315 0062; # (a◌̕◌̀◌֮◌֯b; à◌֮◌֯◌̕b; a◌֮◌̀◌֯◌̕b; à◌֮◌֯◌̕b; a◌֮◌̀◌֯◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HEBREW MARK MASORA CIRCLE, LATIN SMALL LETTER B +0061 05AF 0315 0300 05AE 0062;0061 05AE 05AF 0300 0315 0062;0061 05AE 05AF 0300 0315 0062;0061 05AE 05AF 0300 0315 0062;0061 05AE 05AF 0300 0315 0062; # (a◌֯◌̕◌̀◌֮b; a◌֮◌֯◌̀◌̕b; a◌֮◌֯◌̀◌̕b; a◌֮◌֯◌̀◌̕b; a◌֮◌֯◌̀◌̕b; ) LATIN SMALL LETTER A, HEBREW MARK MASORA CIRCLE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 05B1 05B0 094D 05B0 0062;0061 094D 05B0 05B0 05B1 0062;0061 094D 05B0 05B0 05B1 0062;0061 094D 05B0 05B0 05B1 0062;0061 094D 05B0 05B0 05B1 0062; # (a◌ֱ◌ְ◌्◌ְb; a◌्◌ְ◌ְ◌ֱb; a◌्◌ְ◌ְ◌ֱb; a◌्◌ְ◌ְ◌ֱb; a◌्◌ְ◌ְ◌ֱb; ) LATIN SMALL LETTER A, HEBREW POINT HATAF SEGOL, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, HEBREW POINT SHEVA, LATIN SMALL LETTER B +0061 05B0 05B1 05B0 094D 0062;0061 094D 05B0 05B0 05B1 0062;0061 094D 05B0 05B0 05B1 0062;0061 094D 05B0 05B0 05B1 0062;0061 094D 05B0 05B0 05B1 0062; # (a◌ְ◌ֱ◌ְ◌्b; a◌्◌ְ◌ְ◌ֱb; a◌्◌ְ◌ְ◌ֱb; a◌्◌ְ◌ְ◌ֱb; a◌्◌ְ◌ְ◌ֱb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, HEBREW POINT HATAF SEGOL, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, LATIN SMALL LETTER B +0061 05B2 05B1 05B0 05B1 0062;0061 05B0 05B1 05B1 05B2 0062;0061 05B0 05B1 05B1 05B2 0062;0061 05B0 05B1 05B1 05B2 0062;0061 05B0 05B1 05B1 05B2 0062; # (a◌ֲ◌ֱ◌ְ◌ֱb; a◌ְ◌ֱ◌ֱ◌ֲb; a◌ְ◌ֱ◌ֱ◌ֲb; a◌ְ◌ֱ◌ֱ◌ֲb; a◌ְ◌ֱ◌ֱ◌ֲb; ) LATIN SMALL LETTER A, HEBREW POINT HATAF PATAH, HEBREW POINT HATAF SEGOL, HEBREW POINT SHEVA, HEBREW POINT HATAF SEGOL, LATIN SMALL LETTER B +0061 05B1 05B2 05B1 05B0 0062;0061 05B0 05B1 05B1 05B2 0062;0061 05B0 05B1 05B1 05B2 0062;0061 05B0 05B1 05B1 05B2 0062;0061 05B0 05B1 05B1 05B2 0062; # (a◌ֱ◌ֲ◌ֱ◌ְb; a◌ְ◌ֱ◌ֱ◌ֲb; a◌ְ◌ֱ◌ֱ◌ֲb; a◌ְ◌ֱ◌ֱ◌ֲb; a◌ְ◌ֱ◌ֱ◌ֲb; ) LATIN SMALL LETTER A, HEBREW POINT HATAF SEGOL, HEBREW POINT HATAF PATAH, HEBREW POINT HATAF SEGOL, HEBREW POINT SHEVA, LATIN SMALL LETTER B +0061 05B3 05B2 05B1 05B2 0062;0061 05B1 05B2 05B2 05B3 0062;0061 05B1 05B2 05B2 05B3 0062;0061 05B1 05B2 05B2 05B3 0062;0061 05B1 05B2 05B2 05B3 0062; # (a◌ֳ◌ֲ◌ֱ◌ֲb; a◌ֱ◌ֲ◌ֲ◌ֳb; a◌ֱ◌ֲ◌ֲ◌ֳb; a◌ֱ◌ֲ◌ֲ◌ֳb; a◌ֱ◌ֲ◌ֲ◌ֳb; ) LATIN SMALL LETTER A, HEBREW POINT HATAF QAMATS, HEBREW POINT HATAF PATAH, HEBREW POINT HATAF SEGOL, HEBREW POINT HATAF PATAH, LATIN SMALL LETTER B +0061 05B2 05B3 05B2 05B1 0062;0061 05B1 05B2 05B2 05B3 0062;0061 05B1 05B2 05B2 05B3 0062;0061 05B1 05B2 05B2 05B3 0062;0061 05B1 05B2 05B2 05B3 0062; # (a◌ֲ◌ֳ◌ֲ◌ֱb; a◌ֱ◌ֲ◌ֲ◌ֳb; a◌ֱ◌ֲ◌ֲ◌ֳb; a◌ֱ◌ֲ◌ֲ◌ֳb; a◌ֱ◌ֲ◌ֲ◌ֳb; ) LATIN SMALL LETTER A, HEBREW POINT HATAF PATAH, HEBREW POINT HATAF QAMATS, HEBREW POINT HATAF PATAH, HEBREW POINT HATAF SEGOL, LATIN SMALL LETTER B +0061 05B4 05B3 05B2 05B3 0062;0061 05B2 05B3 05B3 05B4 0062;0061 05B2 05B3 05B3 05B4 0062;0061 05B2 05B3 05B3 05B4 0062;0061 05B2 05B3 05B3 05B4 0062; # (a◌ִ◌ֳ◌ֲ◌ֳb; a◌ֲ◌ֳ◌ֳ◌ִb; a◌ֲ◌ֳ◌ֳ◌ִb; a◌ֲ◌ֳ◌ֳ◌ִb; a◌ֲ◌ֳ◌ֳ◌ִb; ) LATIN SMALL LETTER A, HEBREW POINT HIRIQ, HEBREW POINT HATAF QAMATS, HEBREW POINT HATAF PATAH, HEBREW POINT HATAF QAMATS, LATIN SMALL LETTER B +0061 05B3 05B4 05B3 05B2 0062;0061 05B2 05B3 05B3 05B4 0062;0061 05B2 05B3 05B3 05B4 0062;0061 05B2 05B3 05B3 05B4 0062;0061 05B2 05B3 05B3 05B4 0062; # (a◌ֳ◌ִ◌ֳ◌ֲb; a◌ֲ◌ֳ◌ֳ◌ִb; a◌ֲ◌ֳ◌ֳ◌ִb; a◌ֲ◌ֳ◌ֳ◌ִb; a◌ֲ◌ֳ◌ֳ◌ִb; ) LATIN SMALL LETTER A, HEBREW POINT HATAF QAMATS, HEBREW POINT HIRIQ, HEBREW POINT HATAF QAMATS, HEBREW POINT HATAF PATAH, LATIN SMALL LETTER B +0061 05B5 05B4 05B3 05B4 0062;0061 05B3 05B4 05B4 05B5 0062;0061 05B3 05B4 05B4 05B5 0062;0061 05B3 05B4 05B4 05B5 0062;0061 05B3 05B4 05B4 05B5 0062; # (a◌ֵ◌ִ◌ֳ◌ִb; a◌ֳ◌ִ◌ִ◌ֵb; a◌ֳ◌ִ◌ִ◌ֵb; a◌ֳ◌ִ◌ִ◌ֵb; a◌ֳ◌ִ◌ִ◌ֵb; ) LATIN SMALL LETTER A, HEBREW POINT TSERE, HEBREW POINT HIRIQ, HEBREW POINT HATAF QAMATS, HEBREW POINT HIRIQ, LATIN SMALL LETTER B +0061 05B4 05B5 05B4 05B3 0062;0061 05B3 05B4 05B4 05B5 0062;0061 05B3 05B4 05B4 05B5 0062;0061 05B3 05B4 05B4 05B5 0062;0061 05B3 05B4 05B4 05B5 0062; # (a◌ִ◌ֵ◌ִ◌ֳb; a◌ֳ◌ִ◌ִ◌ֵb; a◌ֳ◌ִ◌ִ◌ֵb; a◌ֳ◌ִ◌ִ◌ֵb; a◌ֳ◌ִ◌ִ◌ֵb; ) LATIN SMALL LETTER A, HEBREW POINT HIRIQ, HEBREW POINT TSERE, HEBREW POINT HIRIQ, HEBREW POINT HATAF QAMATS, LATIN SMALL LETTER B +0061 05B6 05B5 05B4 05B5 0062;0061 05B4 05B5 05B5 05B6 0062;0061 05B4 05B5 05B5 05B6 0062;0061 05B4 05B5 05B5 05B6 0062;0061 05B4 05B5 05B5 05B6 0062; # (a◌ֶ◌ֵ◌ִ◌ֵb; a◌ִ◌ֵ◌ֵ◌ֶb; a◌ִ◌ֵ◌ֵ◌ֶb; a◌ִ◌ֵ◌ֵ◌ֶb; a◌ִ◌ֵ◌ֵ◌ֶb; ) LATIN SMALL LETTER A, HEBREW POINT SEGOL, HEBREW POINT TSERE, HEBREW POINT HIRIQ, HEBREW POINT TSERE, LATIN SMALL LETTER B +0061 05B5 05B6 05B5 05B4 0062;0061 05B4 05B5 05B5 05B6 0062;0061 05B4 05B5 05B5 05B6 0062;0061 05B4 05B5 05B5 05B6 0062;0061 05B4 05B5 05B5 05B6 0062; # (a◌ֵ◌ֶ◌ֵ◌ִb; a◌ִ◌ֵ◌ֵ◌ֶb; a◌ִ◌ֵ◌ֵ◌ֶb; a◌ִ◌ֵ◌ֵ◌ֶb; a◌ִ◌ֵ◌ֵ◌ֶb; ) LATIN SMALL LETTER A, HEBREW POINT TSERE, HEBREW POINT SEGOL, HEBREW POINT TSERE, HEBREW POINT HIRIQ, LATIN SMALL LETTER B +0061 05B7 05B6 05B5 05B6 0062;0061 05B5 05B6 05B6 05B7 0062;0061 05B5 05B6 05B6 05B7 0062;0061 05B5 05B6 05B6 05B7 0062;0061 05B5 05B6 05B6 05B7 0062; # (a◌ַ◌ֶ◌ֵ◌ֶb; a◌ֵ◌ֶ◌ֶ◌ַb; a◌ֵ◌ֶ◌ֶ◌ַb; a◌ֵ◌ֶ◌ֶ◌ַb; a◌ֵ◌ֶ◌ֶ◌ַb; ) LATIN SMALL LETTER A, HEBREW POINT PATAH, HEBREW POINT SEGOL, HEBREW POINT TSERE, HEBREW POINT SEGOL, LATIN SMALL LETTER B +0061 05B6 05B7 05B6 05B5 0062;0061 05B5 05B6 05B6 05B7 0062;0061 05B5 05B6 05B6 05B7 0062;0061 05B5 05B6 05B6 05B7 0062;0061 05B5 05B6 05B6 05B7 0062; # (a◌ֶ◌ַ◌ֶ◌ֵb; a◌ֵ◌ֶ◌ֶ◌ַb; a◌ֵ◌ֶ◌ֶ◌ַb; a◌ֵ◌ֶ◌ֶ◌ַb; a◌ֵ◌ֶ◌ֶ◌ַb; ) LATIN SMALL LETTER A, HEBREW POINT SEGOL, HEBREW POINT PATAH, HEBREW POINT SEGOL, HEBREW POINT TSERE, LATIN SMALL LETTER B +0061 05B8 05B7 05B6 05B7 0062;0061 05B6 05B7 05B7 05B8 0062;0061 05B6 05B7 05B7 05B8 0062;0061 05B6 05B7 05B7 05B8 0062;0061 05B6 05B7 05B7 05B8 0062; # (a◌ָ◌ַ◌ֶ◌ַb; a◌ֶ◌ַ◌ַ◌ָb; a◌ֶ◌ַ◌ַ◌ָb; a◌ֶ◌ַ◌ַ◌ָb; a◌ֶ◌ַ◌ַ◌ָb; ) LATIN SMALL LETTER A, HEBREW POINT QAMATS, HEBREW POINT PATAH, HEBREW POINT SEGOL, HEBREW POINT PATAH, LATIN SMALL LETTER B +0061 05B7 05B8 05B7 05B6 0062;0061 05B6 05B7 05B7 05B8 0062;0061 05B6 05B7 05B7 05B8 0062;0061 05B6 05B7 05B7 05B8 0062;0061 05B6 05B7 05B7 05B8 0062; # (a◌ַ◌ָ◌ַ◌ֶb; a◌ֶ◌ַ◌ַ◌ָb; a◌ֶ◌ַ◌ַ◌ָb; a◌ֶ◌ַ◌ַ◌ָb; a◌ֶ◌ַ◌ַ◌ָb; ) LATIN SMALL LETTER A, HEBREW POINT PATAH, HEBREW POINT QAMATS, HEBREW POINT PATAH, HEBREW POINT SEGOL, LATIN SMALL LETTER B +0061 05B9 05B8 05B7 05B8 0062;0061 05B7 05B8 05B8 05B9 0062;0061 05B7 05B8 05B8 05B9 0062;0061 05B7 05B8 05B8 05B9 0062;0061 05B7 05B8 05B8 05B9 0062; # (a◌ֹ◌ָ◌ַ◌ָb; a◌ַ◌ָ◌ָ◌ֹb; a◌ַ◌ָ◌ָ◌ֹb; a◌ַ◌ָ◌ָ◌ֹb; a◌ַ◌ָ◌ָ◌ֹb; ) LATIN SMALL LETTER A, HEBREW POINT HOLAM, HEBREW POINT QAMATS, HEBREW POINT PATAH, HEBREW POINT QAMATS, LATIN SMALL LETTER B +0061 05B8 05B9 05B8 05B7 0062;0061 05B7 05B8 05B8 05B9 0062;0061 05B7 05B8 05B8 05B9 0062;0061 05B7 05B8 05B8 05B9 0062;0061 05B7 05B8 05B8 05B9 0062; # (a◌ָ◌ֹ◌ָ◌ַb; a◌ַ◌ָ◌ָ◌ֹb; a◌ַ◌ָ◌ָ◌ֹb; a◌ַ◌ָ◌ָ◌ֹb; a◌ַ◌ָ◌ָ◌ֹb; ) LATIN SMALL LETTER A, HEBREW POINT QAMATS, HEBREW POINT HOLAM, HEBREW POINT QAMATS, HEBREW POINT PATAH, LATIN SMALL LETTER B +0061 05BB 05B9 05B8 05B9 0062;0061 05B8 05B9 05B9 05BB 0062;0061 05B8 05B9 05B9 05BB 0062;0061 05B8 05B9 05B9 05BB 0062;0061 05B8 05B9 05B9 05BB 0062; # (a◌ֻ◌ֹ◌ָ◌ֹb; a◌ָ◌ֹ◌ֹ◌ֻb; a◌ָ◌ֹ◌ֹ◌ֻb; a◌ָ◌ֹ◌ֹ◌ֻb; a◌ָ◌ֹ◌ֹ◌ֻb; ) LATIN SMALL LETTER A, HEBREW POINT QUBUTS, HEBREW POINT HOLAM, HEBREW POINT QAMATS, HEBREW POINT HOLAM, LATIN SMALL LETTER B +0061 05B9 05BB 05B9 05B8 0062;0061 05B8 05B9 05B9 05BB 0062;0061 05B8 05B9 05B9 05BB 0062;0061 05B8 05B9 05B9 05BB 0062;0061 05B8 05B9 05B9 05BB 0062; # (a◌ֹ◌ֻ◌ֹ◌ָb; a◌ָ◌ֹ◌ֹ◌ֻb; a◌ָ◌ֹ◌ֹ◌ֻb; a◌ָ◌ֹ◌ֹ◌ֻb; a◌ָ◌ֹ◌ֹ◌ֻb; ) LATIN SMALL LETTER A, HEBREW POINT HOLAM, HEBREW POINT QUBUTS, HEBREW POINT HOLAM, HEBREW POINT QAMATS, LATIN SMALL LETTER B +0061 05BC 05BB 05B9 05BB 0062;0061 05B9 05BB 05BB 05BC 0062;0061 05B9 05BB 05BB 05BC 0062;0061 05B9 05BB 05BB 05BC 0062;0061 05B9 05BB 05BB 05BC 0062; # (a◌ּ◌ֻ◌ֹ◌ֻb; a◌ֹ◌ֻ◌ֻ◌ּb; a◌ֹ◌ֻ◌ֻ◌ּb; a◌ֹ◌ֻ◌ֻ◌ּb; a◌ֹ◌ֻ◌ֻ◌ּb; ) LATIN SMALL LETTER A, HEBREW POINT DAGESH OR MAPIQ, HEBREW POINT QUBUTS, HEBREW POINT HOLAM, HEBREW POINT QUBUTS, LATIN SMALL LETTER B +0061 05BB 05BC 05BB 05B9 0062;0061 05B9 05BB 05BB 05BC 0062;0061 05B9 05BB 05BB 05BC 0062;0061 05B9 05BB 05BB 05BC 0062;0061 05B9 05BB 05BB 05BC 0062; # (a◌ֻ◌ּ◌ֻ◌ֹb; a◌ֹ◌ֻ◌ֻ◌ּb; a◌ֹ◌ֻ◌ֻ◌ּb; a◌ֹ◌ֻ◌ֻ◌ּb; a◌ֹ◌ֻ◌ֻ◌ּb; ) LATIN SMALL LETTER A, HEBREW POINT QUBUTS, HEBREW POINT DAGESH OR MAPIQ, HEBREW POINT QUBUTS, HEBREW POINT HOLAM, LATIN SMALL LETTER B +0061 05BD 05BC 05BB 05BC 0062;0061 05BB 05BC 05BC 05BD 0062;0061 05BB 05BC 05BC 05BD 0062;0061 05BB 05BC 05BC 05BD 0062;0061 05BB 05BC 05BC 05BD 0062; # (a◌ֽ◌ּ◌ֻ◌ּb; a◌ֻ◌ּ◌ּ◌ֽb; a◌ֻ◌ּ◌ּ◌ֽb; a◌ֻ◌ּ◌ּ◌ֽb; a◌ֻ◌ּ◌ּ◌ֽb; ) LATIN SMALL LETTER A, HEBREW POINT METEG, HEBREW POINT DAGESH OR MAPIQ, HEBREW POINT QUBUTS, HEBREW POINT DAGESH OR MAPIQ, LATIN SMALL LETTER B +0061 05BC 05BD 05BC 05BB 0062;0061 05BB 05BC 05BC 05BD 0062;0061 05BB 05BC 05BC 05BD 0062;0061 05BB 05BC 05BC 05BD 0062;0061 05BB 05BC 05BC 05BD 0062; # (a◌ּ◌ֽ◌ּ◌ֻb; a◌ֻ◌ּ◌ּ◌ֽb; a◌ֻ◌ּ◌ּ◌ֽb; a◌ֻ◌ּ◌ּ◌ֽb; a◌ֻ◌ּ◌ּ◌ֽb; ) LATIN SMALL LETTER A, HEBREW POINT DAGESH OR MAPIQ, HEBREW POINT METEG, HEBREW POINT DAGESH OR MAPIQ, HEBREW POINT QUBUTS, LATIN SMALL LETTER B +0061 05BF 05BD 05BC 05BD 0062;0061 05BC 05BD 05BD 05BF 0062;0061 05BC 05BD 05BD 05BF 0062;0061 05BC 05BD 05BD 05BF 0062;0061 05BC 05BD 05BD 05BF 0062; # (a◌ֿ◌ֽ◌ּ◌ֽb; a◌ּ◌ֽ◌ֽ◌ֿb; a◌ּ◌ֽ◌ֽ◌ֿb; a◌ּ◌ֽ◌ֽ◌ֿb; a◌ּ◌ֽ◌ֽ◌ֿb; ) LATIN SMALL LETTER A, HEBREW POINT RAFE, HEBREW POINT METEG, HEBREW POINT DAGESH OR MAPIQ, HEBREW POINT METEG, LATIN SMALL LETTER B +0061 05BD 05BF 05BD 05BC 0062;0061 05BC 05BD 05BD 05BF 0062;0061 05BC 05BD 05BD 05BF 0062;0061 05BC 05BD 05BD 05BF 0062;0061 05BC 05BD 05BD 05BF 0062; # (a◌ֽ◌ֿ◌ֽ◌ּb; a◌ּ◌ֽ◌ֽ◌ֿb; a◌ּ◌ֽ◌ֽ◌ֿb; a◌ּ◌ֽ◌ֽ◌ֿb; a◌ּ◌ֽ◌ֽ◌ֿb; ) LATIN SMALL LETTER A, HEBREW POINT METEG, HEBREW POINT RAFE, HEBREW POINT METEG, HEBREW POINT DAGESH OR MAPIQ, LATIN SMALL LETTER B +0061 05C1 05BF 05BD 05BF 0062;0061 05BD 05BF 05BF 05C1 0062;0061 05BD 05BF 05BF 05C1 0062;0061 05BD 05BF 05BF 05C1 0062;0061 05BD 05BF 05BF 05C1 0062; # (a◌ׁ◌ֿ◌ֽ◌ֿb; a◌ֽ◌ֿ◌ֿ◌ׁb; a◌ֽ◌ֿ◌ֿ◌ׁb; a◌ֽ◌ֿ◌ֿ◌ׁb; a◌ֽ◌ֿ◌ֿ◌ׁb; ) LATIN SMALL LETTER A, HEBREW POINT SHIN DOT, HEBREW POINT RAFE, HEBREW POINT METEG, HEBREW POINT RAFE, LATIN SMALL LETTER B +0061 05BF 05C1 05BF 05BD 0062;0061 05BD 05BF 05BF 05C1 0062;0061 05BD 05BF 05BF 05C1 0062;0061 05BD 05BF 05BF 05C1 0062;0061 05BD 05BF 05BF 05C1 0062; # (a◌ֿ◌ׁ◌ֿ◌ֽb; a◌ֽ◌ֿ◌ֿ◌ׁb; a◌ֽ◌ֿ◌ֿ◌ׁb; a◌ֽ◌ֿ◌ֿ◌ׁb; a◌ֽ◌ֿ◌ֿ◌ׁb; ) LATIN SMALL LETTER A, HEBREW POINT RAFE, HEBREW POINT SHIN DOT, HEBREW POINT RAFE, HEBREW POINT METEG, LATIN SMALL LETTER B +0061 05C2 05C1 05BF 05C1 0062;0061 05BF 05C1 05C1 05C2 0062;0061 05BF 05C1 05C1 05C2 0062;0061 05BF 05C1 05C1 05C2 0062;0061 05BF 05C1 05C1 05C2 0062; # (a◌ׂ◌ׁ◌ֿ◌ׁb; a◌ֿ◌ׁ◌ׁ◌ׂb; a◌ֿ◌ׁ◌ׁ◌ׂb; a◌ֿ◌ׁ◌ׁ◌ׂb; a◌ֿ◌ׁ◌ׁ◌ׂb; ) LATIN SMALL LETTER A, HEBREW POINT SIN DOT, HEBREW POINT SHIN DOT, HEBREW POINT RAFE, HEBREW POINT SHIN DOT, LATIN SMALL LETTER B +0061 05C1 05C2 05C1 05BF 0062;0061 05BF 05C1 05C1 05C2 0062;0061 05BF 05C1 05C1 05C2 0062;0061 05BF 05C1 05C1 05C2 0062;0061 05BF 05C1 05C1 05C2 0062; # (a◌ׁ◌ׂ◌ׁ◌ֿb; a◌ֿ◌ׁ◌ׁ◌ׂb; a◌ֿ◌ׁ◌ׁ◌ׂb; a◌ֿ◌ׁ◌ׁ◌ׂb; a◌ֿ◌ׁ◌ׁ◌ׂb; ) LATIN SMALL LETTER A, HEBREW POINT SHIN DOT, HEBREW POINT SIN DOT, HEBREW POINT SHIN DOT, HEBREW POINT RAFE, LATIN SMALL LETTER B +0061 FB1E 05C2 05C1 05C2 0062;0061 05C1 05C2 05C2 FB1E 0062;0061 05C1 05C2 05C2 FB1E 0062;0061 05C1 05C2 05C2 FB1E 0062;0061 05C1 05C2 05C2 FB1E 0062; # (a◌ﬞ◌ׂ◌ׁ◌ׂb; a◌ׁ◌ׂ◌ׂ◌ﬞb; a◌ׁ◌ׂ◌ׂ◌ﬞb; a◌ׁ◌ׂ◌ׂ◌ﬞb; a◌ׁ◌ׂ◌ׂ◌ﬞb; ) LATIN SMALL LETTER A, HEBREW POINT JUDEO-SPANISH VARIKA, HEBREW POINT SIN DOT, HEBREW POINT SHIN DOT, HEBREW POINT SIN DOT, LATIN SMALL LETTER B +0061 05C2 FB1E 05C2 05C1 0062;0061 05C1 05C2 05C2 FB1E 0062;0061 05C1 05C2 05C2 FB1E 0062;0061 05C1 05C2 05C2 FB1E 0062;0061 05C1 05C2 05C2 FB1E 0062; # (a◌ׂ◌ﬞ◌ׂ◌ׁb; a◌ׁ◌ׂ◌ׂ◌ﬞb; a◌ׁ◌ׂ◌ׂ◌ﬞb; a◌ׁ◌ׂ◌ׂ◌ﬞb; a◌ׁ◌ׂ◌ׂ◌ﬞb; ) LATIN SMALL LETTER A, HEBREW POINT SIN DOT, HEBREW POINT JUDEO-SPANISH VARIKA, HEBREW POINT SIN DOT, HEBREW POINT SHIN DOT, LATIN SMALL LETTER B +0061 0315 0300 05AE 05C4 0062;00E0 05AE 05C4 0315 0062;0061 05AE 0300 05C4 0315 0062;00E0 05AE 05C4 0315 0062;0061 05AE 0300 05C4 0315 0062; # (a◌̕◌̀◌֮◌ׄb; à◌֮◌ׄ◌̕b; a◌֮◌̀◌ׄ◌̕b; à◌֮◌ׄ◌̕b; a◌֮◌̀◌ׄ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, HEBREW MARK UPPER DOT, LATIN SMALL LETTER B +0061 05C4 0315 0300 05AE 0062;0061 05AE 05C4 0300 0315 0062;0061 05AE 05C4 0300 0315 0062;0061 05AE 05C4 0300 0315 0062;0061 05AE 05C4 0300 0315 0062; # (a◌ׄ◌̕◌̀◌֮b; a◌֮◌ׄ◌̀◌̕b; a◌֮◌ׄ◌̀◌̕b; a◌֮◌ׄ◌̀◌̕b; a◌֮◌ׄ◌̀◌̕b; ) LATIN SMALL LETTER A, HEBREW MARK UPPER DOT, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 064C 064B FB1E 064B 0062;0061 FB1E 064B 064B 064C 0062;0061 FB1E 064B 064B 064C 0062;0061 FB1E 064B 064B 064C 0062;0061 FB1E 064B 064B 064C 0062; # (a◌ٌ◌ً◌ﬞ◌ًb; a◌ﬞ◌ً◌ً◌ٌb; a◌ﬞ◌ً◌ً◌ٌb; a◌ﬞ◌ً◌ً◌ٌb; a◌ﬞ◌ً◌ً◌ٌb; ) LATIN SMALL LETTER A, ARABIC DAMMATAN, ARABIC FATHATAN, HEBREW POINT JUDEO-SPANISH VARIKA, ARABIC FATHATAN, LATIN SMALL LETTER B +0061 064B 064C 064B FB1E 0062;0061 FB1E 064B 064B 064C 0062;0061 FB1E 064B 064B 064C 0062;0061 FB1E 064B 064B 064C 0062;0061 FB1E 064B 064B 064C 0062; # (a◌ً◌ٌ◌ً◌ﬞb; a◌ﬞ◌ً◌ً◌ٌb; a◌ﬞ◌ً◌ً◌ٌb; a◌ﬞ◌ً◌ً◌ٌb; a◌ﬞ◌ً◌ً◌ٌb; ) LATIN SMALL LETTER A, ARABIC FATHATAN, ARABIC DAMMATAN, ARABIC FATHATAN, HEBREW POINT JUDEO-SPANISH VARIKA, LATIN SMALL LETTER B +0061 064D 064C 064B 064C 0062;0061 064B 064C 064C 064D 0062;0061 064B 064C 064C 064D 0062;0061 064B 064C 064C 064D 0062;0061 064B 064C 064C 064D 0062; # (a◌ٍ◌ٌ◌ً◌ٌb; a◌ً◌ٌ◌ٌ◌ٍb; a◌ً◌ٌ◌ٌ◌ٍb; a◌ً◌ٌ◌ٌ◌ٍb; a◌ً◌ٌ◌ٌ◌ٍb; ) LATIN SMALL LETTER A, ARABIC KASRATAN, ARABIC DAMMATAN, ARABIC FATHATAN, ARABIC DAMMATAN, LATIN SMALL LETTER B +0061 064C 064D 064C 064B 0062;0061 064B 064C 064C 064D 0062;0061 064B 064C 064C 064D 0062;0061 064B 064C 064C 064D 0062;0061 064B 064C 064C 064D 0062; # (a◌ٌ◌ٍ◌ٌ◌ًb; a◌ً◌ٌ◌ٌ◌ٍb; a◌ً◌ٌ◌ٌ◌ٍb; a◌ً◌ٌ◌ٌ◌ٍb; a◌ً◌ٌ◌ٌ◌ٍb; ) LATIN SMALL LETTER A, ARABIC DAMMATAN, ARABIC KASRATAN, ARABIC DAMMATAN, ARABIC FATHATAN, LATIN SMALL LETTER B +0061 064E 064D 064C 064D 0062;0061 064C 064D 064D 064E 0062;0061 064C 064D 064D 064E 0062;0061 064C 064D 064D 064E 0062;0061 064C 064D 064D 064E 0062; # (a◌َ◌ٍ◌ٌ◌ٍb; a◌ٌ◌ٍ◌ٍ◌َb; a◌ٌ◌ٍ◌ٍ◌َb; a◌ٌ◌ٍ◌ٍ◌َb; a◌ٌ◌ٍ◌ٍ◌َb; ) LATIN SMALL LETTER A, ARABIC FATHA, ARABIC KASRATAN, ARABIC DAMMATAN, ARABIC KASRATAN, LATIN SMALL LETTER B +0061 064D 064E 064D 064C 0062;0061 064C 064D 064D 064E 0062;0061 064C 064D 064D 064E 0062;0061 064C 064D 064D 064E 0062;0061 064C 064D 064D 064E 0062; # (a◌ٍ◌َ◌ٍ◌ٌb; a◌ٌ◌ٍ◌ٍ◌َb; a◌ٌ◌ٍ◌ٍ◌َb; a◌ٌ◌ٍ◌ٍ◌َb; a◌ٌ◌ٍ◌ٍ◌َb; ) LATIN SMALL LETTER A, ARABIC KASRATAN, ARABIC FATHA, ARABIC KASRATAN, ARABIC DAMMATAN, LATIN SMALL LETTER B +0061 064F 064E 064D 064E 0062;0061 064D 064E 064E 064F 0062;0061 064D 064E 064E 064F 0062;0061 064D 064E 064E 064F 0062;0061 064D 064E 064E 064F 0062; # (a◌ُ◌َ◌ٍ◌َb; a◌ٍ◌َ◌َ◌ُb; a◌ٍ◌َ◌َ◌ُb; a◌ٍ◌َ◌َ◌ُb; a◌ٍ◌َ◌َ◌ُb; ) LATIN SMALL LETTER A, ARABIC DAMMA, ARABIC FATHA, ARABIC KASRATAN, ARABIC FATHA, LATIN SMALL LETTER B +0061 064E 064F 064E 064D 0062;0061 064D 064E 064E 064F 0062;0061 064D 064E 064E 064F 0062;0061 064D 064E 064E 064F 0062;0061 064D 064E 064E 064F 0062; # (a◌َ◌ُ◌َ◌ٍb; a◌ٍ◌َ◌َ◌ُb; a◌ٍ◌َ◌َ◌ُb; a◌ٍ◌َ◌َ◌ُb; a◌ٍ◌َ◌َ◌ُb; ) LATIN SMALL LETTER A, ARABIC FATHA, ARABIC DAMMA, ARABIC FATHA, ARABIC KASRATAN, LATIN SMALL LETTER B +0061 0650 064F 064E 064F 0062;0061 064E 064F 064F 0650 0062;0061 064E 064F 064F 0650 0062;0061 064E 064F 064F 0650 0062;0061 064E 064F 064F 0650 0062; # (a◌ِ◌ُ◌َ◌ُb; a◌َ◌ُ◌ُ◌ِb; a◌َ◌ُ◌ُ◌ِb; a◌َ◌ُ◌ُ◌ِb; a◌َ◌ُ◌ُ◌ِb; ) LATIN SMALL LETTER A, ARABIC KASRA, ARABIC DAMMA, ARABIC FATHA, ARABIC DAMMA, LATIN SMALL LETTER B +0061 064F 0650 064F 064E 0062;0061 064E 064F 064F 0650 0062;0061 064E 064F 064F 0650 0062;0061 064E 064F 064F 0650 0062;0061 064E 064F 064F 0650 0062; # (a◌ُ◌ِ◌ُ◌َb; a◌َ◌ُ◌ُ◌ِb; a◌َ◌ُ◌ُ◌ِb; a◌َ◌ُ◌ُ◌ِb; a◌َ◌ُ◌ُ◌ِb; ) LATIN SMALL LETTER A, ARABIC DAMMA, ARABIC KASRA, ARABIC DAMMA, ARABIC FATHA, LATIN SMALL LETTER B +0061 0651 0650 064F 0650 0062;0061 064F 0650 0650 0651 0062;0061 064F 0650 0650 0651 0062;0061 064F 0650 0650 0651 0062;0061 064F 0650 0650 0651 0062; # (a◌ّ◌ِ◌ُ◌ِb; a◌ُ◌ِ◌ِ◌ّb; a◌ُ◌ِ◌ِ◌ّb; a◌ُ◌ِ◌ِ◌ّb; a◌ُ◌ِ◌ِ◌ّb; ) LATIN SMALL LETTER A, ARABIC SHADDA, ARABIC KASRA, ARABIC DAMMA, ARABIC KASRA, LATIN SMALL LETTER B +0061 0650 0651 0650 064F 0062;0061 064F 0650 0650 0651 0062;0061 064F 0650 0650 0651 0062;0061 064F 0650 0650 0651 0062;0061 064F 0650 0650 0651 0062; # (a◌ِ◌ّ◌ِ◌ُb; a◌ُ◌ِ◌ِ◌ّb; a◌ُ◌ِ◌ِ◌ّb; a◌ُ◌ِ◌ِ◌ّb; a◌ُ◌ِ◌ِ◌ّb; ) LATIN SMALL LETTER A, ARABIC KASRA, ARABIC SHADDA, ARABIC KASRA, ARABIC DAMMA, LATIN SMALL LETTER B +0061 0652 0651 0650 0651 0062;0061 0650 0651 0651 0652 0062;0061 0650 0651 0651 0652 0062;0061 0650 0651 0651 0652 0062;0061 0650 0651 0651 0652 0062; # (a◌ْ◌ّ◌ِ◌ّb; a◌ِ◌ّ◌ّ◌ْb; a◌ِ◌ّ◌ّ◌ْb; a◌ِ◌ّ◌ّ◌ْb; a◌ِ◌ّ◌ّ◌ْb; ) LATIN SMALL LETTER A, ARABIC SUKUN, ARABIC SHADDA, ARABIC KASRA, ARABIC SHADDA, LATIN SMALL LETTER B +0061 0651 0652 0651 0650 0062;0061 0650 0651 0651 0652 0062;0061 0650 0651 0651 0652 0062;0061 0650 0651 0651 0652 0062;0061 0650 0651 0651 0652 0062; # (a◌ّ◌ْ◌ّ◌ِb; a◌ِ◌ّ◌ّ◌ْb; a◌ِ◌ّ◌ّ◌ْb; a◌ِ◌ّ◌ّ◌ْb; a◌ِ◌ّ◌ّ◌ْb; ) LATIN SMALL LETTER A, ARABIC SHADDA, ARABIC SUKUN, ARABIC SHADDA, ARABIC KASRA, LATIN SMALL LETTER B +0061 0670 0652 0651 0652 0062;0061 0651 0652 0652 0670 0062;0061 0651 0652 0652 0670 0062;0061 0651 0652 0652 0670 0062;0061 0651 0652 0652 0670 0062; # (a◌ٰ◌ْ◌ّ◌ْb; a◌ّ◌ْ◌ْ◌ٰb; a◌ّ◌ْ◌ْ◌ٰb; a◌ّ◌ْ◌ْ◌ٰb; a◌ّ◌ْ◌ْ◌ٰb; ) LATIN SMALL LETTER A, ARABIC LETTER SUPERSCRIPT ALEF, ARABIC SUKUN, ARABIC SHADDA, ARABIC SUKUN, LATIN SMALL LETTER B +0061 0652 0670 0652 0651 0062;0061 0651 0652 0652 0670 0062;0061 0651 0652 0652 0670 0062;0061 0651 0652 0652 0670 0062;0061 0651 0652 0652 0670 0062; # (a◌ْ◌ٰ◌ْ◌ّb; a◌ّ◌ْ◌ْ◌ٰb; a◌ّ◌ْ◌ْ◌ٰb; a◌ّ◌ْ◌ْ◌ٰb; a◌ّ◌ْ◌ْ◌ٰb; ) LATIN SMALL LETTER A, ARABIC SUKUN, ARABIC LETTER SUPERSCRIPT ALEF, ARABIC SUKUN, ARABIC SHADDA, LATIN SMALL LETTER B +0061 0315 0300 05AE 0653 0062;00E0 05AE 0653 0315 0062;0061 05AE 0300 0653 0315 0062;00E0 05AE 0653 0315 0062;0061 05AE 0300 0653 0315 0062; # (a◌̕◌̀◌֮◌ٓb; à◌֮◌ٓ◌̕b; a◌֮◌̀◌ٓ◌̕b; à◌֮◌ٓ◌̕b; a◌֮◌̀◌ٓ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, ARABIC MADDAH ABOVE, LATIN SMALL LETTER B +0061 0653 0315 0300 05AE 0062;0061 05AE 0653 0300 0315 0062;0061 05AE 0653 0300 0315 0062;0061 05AE 0653 0300 0315 0062;0061 05AE 0653 0300 0315 0062; # (a◌ٓ◌̕◌̀◌֮b; a◌֮◌ٓ◌̀◌̕b; a◌֮◌ٓ◌̀◌̕b; a◌֮◌ٓ◌̀◌̕b; a◌֮◌ٓ◌̀◌̕b; ) LATIN SMALL LETTER A, ARABIC MADDAH ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0654 0062;00E0 05AE 0654 0315 0062;0061 05AE 0300 0654 0315 0062;00E0 05AE 0654 0315 0062;0061 05AE 0300 0654 0315 0062; # (a◌̕◌̀◌֮◌ٔb; à◌֮◌ٔ◌̕b; a◌֮◌̀◌ٔ◌̕b; à◌֮◌ٔ◌̕b; a◌֮◌̀◌ٔ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, ARABIC HAMZA ABOVE, LATIN SMALL LETTER B +0061 0654 0315 0300 05AE 0062;0061 05AE 0654 0300 0315 0062;0061 05AE 0654 0300 0315 0062;0061 05AE 0654 0300 0315 0062;0061 05AE 0654 0300 0315 0062; # (a◌ٔ◌̕◌̀◌֮b; a◌֮◌ٔ◌̀◌̕b; a◌֮◌ٔ◌̀◌̕b; a◌֮◌ٔ◌̀◌̕b; a◌֮◌ٔ◌̀◌̕b; ) LATIN SMALL LETTER A, ARABIC HAMZA ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 0655 0062;0061 302A 0316 0655 059A 0062;0061 302A 0316 0655 059A 0062;0061 302A 0316 0655 059A 0062;0061 302A 0316 0655 059A 0062; # (a◌֚◌̖◌〪◌ٕb; a◌〪◌̖◌ٕ◌֚b; a◌〪◌̖◌ٕ◌֚b; a◌〪◌̖◌ٕ◌֚b; a◌〪◌̖◌ٕ◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, ARABIC HAMZA BELOW, LATIN SMALL LETTER B +0061 0655 059A 0316 302A 0062;0061 302A 0655 0316 059A 0062;0061 302A 0655 0316 059A 0062;0061 302A 0655 0316 059A 0062;0061 302A 0655 0316 059A 0062; # (a◌ٕ◌֚◌̖◌〪b; a◌〪◌ٕ◌̖◌֚b; a◌〪◌ٕ◌̖◌֚b; a◌〪◌ٕ◌̖◌֚b; a◌〪◌ٕ◌̖◌֚b; ) LATIN SMALL LETTER A, ARABIC HAMZA BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0711 0670 0652 0670 0062;0061 0652 0670 0670 0711 0062;0061 0652 0670 0670 0711 0062;0061 0652 0670 0670 0711 0062;0061 0652 0670 0670 0711 0062; # (a◌ܑ◌ٰ◌ْ◌ٰb; a◌ْ◌ٰ◌ٰ◌ܑb; a◌ْ◌ٰ◌ٰ◌ܑb; a◌ْ◌ٰ◌ٰ◌ܑb; a◌ْ◌ٰ◌ٰ◌ܑb; ) LATIN SMALL LETTER A, SYRIAC LETTER SUPERSCRIPT ALAPH, ARABIC LETTER SUPERSCRIPT ALEF, ARABIC SUKUN, ARABIC LETTER SUPERSCRIPT ALEF, LATIN SMALL LETTER B +0061 0670 0711 0670 0652 0062;0061 0652 0670 0670 0711 0062;0061 0652 0670 0670 0711 0062;0061 0652 0670 0670 0711 0062;0061 0652 0670 0670 0711 0062; # (a◌ٰ◌ܑ◌ٰ◌ْb; a◌ْ◌ٰ◌ٰ◌ܑb; a◌ْ◌ٰ◌ٰ◌ܑb; a◌ْ◌ٰ◌ٰ◌ܑb; a◌ْ◌ٰ◌ٰ◌ܑb; ) LATIN SMALL LETTER A, ARABIC LETTER SUPERSCRIPT ALEF, SYRIAC LETTER SUPERSCRIPT ALAPH, ARABIC LETTER SUPERSCRIPT ALEF, ARABIC SUKUN, LATIN SMALL LETTER B +0061 0315 0300 05AE 06D6 0062;00E0 05AE 06D6 0315 0062;0061 05AE 0300 06D6 0315 0062;00E0 05AE 06D6 0315 0062;0061 05AE 0300 06D6 0315 0062; # (a◌̕◌̀◌֮◌ۖb; à◌֮◌ۖ◌̕b; a◌֮◌̀◌ۖ◌̕b; à◌֮◌ۖ◌̕b; a◌֮◌̀◌ۖ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA, LATIN SMALL LETTER B +0061 06D6 0315 0300 05AE 0062;0061 05AE 06D6 0300 0315 0062;0061 05AE 06D6 0300 0315 0062;0061 05AE 06D6 0300 0315 0062;0061 05AE 06D6 0300 0315 0062; # (a◌ۖ◌̕◌̀◌֮b; a◌֮◌ۖ◌̀◌̕b; a◌֮◌ۖ◌̀◌̕b; a◌֮◌ۖ◌̀◌̕b; a◌֮◌ۖ◌̀◌̕b; ) LATIN SMALL LETTER A, ARABIC SMALL HIGH LIGATURE SAD WITH LAM WITH ALEF MAKSURA, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 06D7 0062;00E0 05AE 06D7 0315 0062;0061 05AE 0300 06D7 0315 0062;00E0 05AE 06D7 0315 0062;0061 05AE 0300 06D7 0315 0062; # (a◌̕◌̀◌֮◌ۗb; à◌֮◌ۗ◌̕b; a◌֮◌̀◌ۗ◌̕b; à◌֮◌ۗ◌̕b; a◌֮◌̀◌ۗ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, ARABIC SMALL HIGH LIGATURE QAF WITH LAM WITH ALEF MAKSURA, LATIN SMALL LETTER B +0061 06D7 0315 0300 05AE 0062;0061 05AE 06D7 0300 0315 0062;0061 05AE 06D7 0300 0315 0062;0061 05AE 06D7 0300 0315 0062;0061 05AE 06D7 0300 0315 0062; # (a◌ۗ◌̕◌̀◌֮b; a◌֮◌ۗ◌̀◌̕b; a◌֮◌ۗ◌̀◌̕b; a◌֮◌ۗ◌̀◌̕b; a◌֮◌ۗ◌̀◌̕b; ) LATIN SMALL LETTER A, ARABIC SMALL HIGH LIGATURE QAF WITH LAM WITH ALEF MAKSURA, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 06D8 0062;00E0 05AE 06D8 0315 0062;0061 05AE 0300 06D8 0315 0062;00E0 05AE 06D8 0315 0062;0061 05AE 0300 06D8 0315 0062; # (a◌̕◌̀◌֮◌ۘb; à◌֮◌ۘ◌̕b; a◌֮◌̀◌ۘ◌̕b; à◌֮◌ۘ◌̕b; a◌֮◌̀◌ۘ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, ARABIC SMALL HIGH MEEM INITIAL FORM, LATIN SMALL LETTER B +0061 06D8 0315 0300 05AE 0062;0061 05AE 06D8 0300 0315 0062;0061 05AE 06D8 0300 0315 0062;0061 05AE 06D8 0300 0315 0062;0061 05AE 06D8 0300 0315 0062; # (a◌ۘ◌̕◌̀◌֮b; a◌֮◌ۘ◌̀◌̕b; a◌֮◌ۘ◌̀◌̕b; a◌֮◌ۘ◌̀◌̕b; a◌֮◌ۘ◌̀◌̕b; ) LATIN SMALL LETTER A, ARABIC SMALL HIGH MEEM INITIAL FORM, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 06D9 0062;00E0 05AE 06D9 0315 0062;0061 05AE 0300 06D9 0315 0062;00E0 05AE 06D9 0315 0062;0061 05AE 0300 06D9 0315 0062; # (a◌̕◌̀◌֮◌ۙb; à◌֮◌ۙ◌̕b; a◌֮◌̀◌ۙ◌̕b; à◌֮◌ۙ◌̕b; a◌֮◌̀◌ۙ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, ARABIC SMALL HIGH LAM ALEF, LATIN SMALL LETTER B +0061 06D9 0315 0300 05AE 0062;0061 05AE 06D9 0300 0315 0062;0061 05AE 06D9 0300 0315 0062;0061 05AE 06D9 0300 0315 0062;0061 05AE 06D9 0300 0315 0062; # (a◌ۙ◌̕◌̀◌֮b; a◌֮◌ۙ◌̀◌̕b; a◌֮◌ۙ◌̀◌̕b; a◌֮◌ۙ◌̀◌̕b; a◌֮◌ۙ◌̀◌̕b; ) LATIN SMALL LETTER A, ARABIC SMALL HIGH LAM ALEF, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 06DA 0062;00E0 05AE 06DA 0315 0062;0061 05AE 0300 06DA 0315 0062;00E0 05AE 06DA 0315 0062;0061 05AE 0300 06DA 0315 0062; # (a◌̕◌̀◌֮◌ۚb; à◌֮◌ۚ◌̕b; a◌֮◌̀◌ۚ◌̕b; à◌֮◌ۚ◌̕b; a◌֮◌̀◌ۚ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, ARABIC SMALL HIGH JEEM, LATIN SMALL LETTER B +0061 06DA 0315 0300 05AE 0062;0061 05AE 06DA 0300 0315 0062;0061 05AE 06DA 0300 0315 0062;0061 05AE 06DA 0300 0315 0062;0061 05AE 06DA 0300 0315 0062; # (a◌ۚ◌̕◌̀◌֮b; a◌֮◌ۚ◌̀◌̕b; a◌֮◌ۚ◌̀◌̕b; a◌֮◌ۚ◌̀◌̕b; a◌֮◌ۚ◌̀◌̕b; ) LATIN SMALL LETTER A, ARABIC SMALL HIGH JEEM, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 06DB 0062;00E0 05AE 06DB 0315 0062;0061 05AE 0300 06DB 0315 0062;00E0 05AE 06DB 0315 0062;0061 05AE 0300 06DB 0315 0062; # (a◌̕◌̀◌֮◌ۛb; à◌֮◌ۛ◌̕b; a◌֮◌̀◌ۛ◌̕b; à◌֮◌ۛ◌̕b; a◌֮◌̀◌ۛ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, ARABIC SMALL HIGH THREE DOTS, LATIN SMALL LETTER B +0061 06DB 0315 0300 05AE 0062;0061 05AE 06DB 0300 0315 0062;0061 05AE 06DB 0300 0315 0062;0061 05AE 06DB 0300 0315 0062;0061 05AE 06DB 0300 0315 0062; # (a◌ۛ◌̕◌̀◌֮b; a◌֮◌ۛ◌̀◌̕b; a◌֮◌ۛ◌̀◌̕b; a◌֮◌ۛ◌̀◌̕b; a◌֮◌ۛ◌̀◌̕b; ) LATIN SMALL LETTER A, ARABIC SMALL HIGH THREE DOTS, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 06DC 0062;00E0 05AE 06DC 0315 0062;0061 05AE 0300 06DC 0315 0062;00E0 05AE 06DC 0315 0062;0061 05AE 0300 06DC 0315 0062; # (a◌̕◌̀◌֮◌ۜb; à◌֮◌ۜ◌̕b; a◌֮◌̀◌ۜ◌̕b; à◌֮◌ۜ◌̕b; a◌֮◌̀◌ۜ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, ARABIC SMALL HIGH SEEN, LATIN SMALL LETTER B +0061 06DC 0315 0300 05AE 0062;0061 05AE 06DC 0300 0315 0062;0061 05AE 06DC 0300 0315 0062;0061 05AE 06DC 0300 0315 0062;0061 05AE 06DC 0300 0315 0062; # (a◌ۜ◌̕◌̀◌֮b; a◌֮◌ۜ◌̀◌̕b; a◌֮◌ۜ◌̀◌̕b; a◌֮◌ۜ◌̀◌̕b; a◌֮◌ۜ◌̀◌̕b; ) LATIN SMALL LETTER A, ARABIC SMALL HIGH SEEN, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 06DF 0062;00E0 05AE 06DF 0315 0062;0061 05AE 0300 06DF 0315 0062;00E0 05AE 06DF 0315 0062;0061 05AE 0300 06DF 0315 0062; # (a◌̕◌̀◌֮◌۟b; à◌֮◌۟◌̕b; a◌֮◌̀◌۟◌̕b; à◌֮◌۟◌̕b; a◌֮◌̀◌۟◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, ARABIC SMALL HIGH ROUNDED ZERO, LATIN SMALL LETTER B +0061 06DF 0315 0300 05AE 0062;0061 05AE 06DF 0300 0315 0062;0061 05AE 06DF 0300 0315 0062;0061 05AE 06DF 0300 0315 0062;0061 05AE 06DF 0300 0315 0062; # (a◌۟◌̕◌̀◌֮b; a◌֮◌۟◌̀◌̕b; a◌֮◌۟◌̀◌̕b; a◌֮◌۟◌̀◌̕b; a◌֮◌۟◌̀◌̕b; ) LATIN SMALL LETTER A, ARABIC SMALL HIGH ROUNDED ZERO, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 06E0 0062;00E0 05AE 06E0 0315 0062;0061 05AE 0300 06E0 0315 0062;00E0 05AE 06E0 0315 0062;0061 05AE 0300 06E0 0315 0062; # (a◌̕◌̀◌֮◌۠b; à◌֮◌۠◌̕b; a◌֮◌̀◌۠◌̕b; à◌֮◌۠◌̕b; a◌֮◌̀◌۠◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, ARABIC SMALL HIGH UPRIGHT RECTANGULAR ZERO, LATIN SMALL LETTER B +0061 06E0 0315 0300 05AE 0062;0061 05AE 06E0 0300 0315 0062;0061 05AE 06E0 0300 0315 0062;0061 05AE 06E0 0300 0315 0062;0061 05AE 06E0 0300 0315 0062; # (a◌۠◌̕◌̀◌֮b; a◌֮◌۠◌̀◌̕b; a◌֮◌۠◌̀◌̕b; a◌֮◌۠◌̀◌̕b; a◌֮◌۠◌̀◌̕b; ) LATIN SMALL LETTER A, ARABIC SMALL HIGH UPRIGHT RECTANGULAR ZERO, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 06E1 0062;00E0 05AE 06E1 0315 0062;0061 05AE 0300 06E1 0315 0062;00E0 05AE 06E1 0315 0062;0061 05AE 0300 06E1 0315 0062; # (a◌̕◌̀◌֮◌ۡb; à◌֮◌ۡ◌̕b; a◌֮◌̀◌ۡ◌̕b; à◌֮◌ۡ◌̕b; a◌֮◌̀◌ۡ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, ARABIC SMALL HIGH DOTLESS HEAD OF KHAH, LATIN SMALL LETTER B +0061 06E1 0315 0300 05AE 0062;0061 05AE 06E1 0300 0315 0062;0061 05AE 06E1 0300 0315 0062;0061 05AE 06E1 0300 0315 0062;0061 05AE 06E1 0300 0315 0062; # (a◌ۡ◌̕◌̀◌֮b; a◌֮◌ۡ◌̀◌̕b; a◌֮◌ۡ◌̀◌̕b; a◌֮◌ۡ◌̀◌̕b; a◌֮◌ۡ◌̀◌̕b; ) LATIN SMALL LETTER A, ARABIC SMALL HIGH DOTLESS HEAD OF KHAH, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 06E2 0062;00E0 05AE 06E2 0315 0062;0061 05AE 0300 06E2 0315 0062;00E0 05AE 06E2 0315 0062;0061 05AE 0300 06E2 0315 0062; # (a◌̕◌̀◌֮◌ۢb; à◌֮◌ۢ◌̕b; a◌֮◌̀◌ۢ◌̕b; à◌֮◌ۢ◌̕b; a◌֮◌̀◌ۢ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, ARABIC SMALL HIGH MEEM ISOLATED FORM, LATIN SMALL LETTER B +0061 06E2 0315 0300 05AE 0062;0061 05AE 06E2 0300 0315 0062;0061 05AE 06E2 0300 0315 0062;0061 05AE 06E2 0300 0315 0062;0061 05AE 06E2 0300 0315 0062; # (a◌ۢ◌̕◌̀◌֮b; a◌֮◌ۢ◌̀◌̕b; a◌֮◌ۢ◌̀◌̕b; a◌֮◌ۢ◌̀◌̕b; a◌֮◌ۢ◌̀◌̕b; ) LATIN SMALL LETTER A, ARABIC SMALL HIGH MEEM ISOLATED FORM, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 06E3 0062;0061 302A 0316 06E3 059A 0062;0061 302A 0316 06E3 059A 0062;0061 302A 0316 06E3 059A 0062;0061 302A 0316 06E3 059A 0062; # (a◌֚◌̖◌〪◌ۣb; a◌〪◌̖◌ۣ◌֚b; a◌〪◌̖◌ۣ◌֚b; a◌〪◌̖◌ۣ◌֚b; a◌〪◌̖◌ۣ◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, ARABIC SMALL LOW SEEN, LATIN SMALL LETTER B +0061 06E3 059A 0316 302A 0062;0061 302A 06E3 0316 059A 0062;0061 302A 06E3 0316 059A 0062;0061 302A 06E3 0316 059A 0062;0061 302A 06E3 0316 059A 0062; # (a◌ۣ◌֚◌̖◌〪b; a◌〪◌ۣ◌̖◌֚b; a◌〪◌ۣ◌̖◌֚b; a◌〪◌ۣ◌̖◌֚b; a◌〪◌ۣ◌̖◌֚b; ) LATIN SMALL LETTER A, ARABIC SMALL LOW SEEN, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 06E4 0062;00E0 05AE 06E4 0315 0062;0061 05AE 0300 06E4 0315 0062;00E0 05AE 06E4 0315 0062;0061 05AE 0300 06E4 0315 0062; # (a◌̕◌̀◌֮◌ۤb; à◌֮◌ۤ◌̕b; a◌֮◌̀◌ۤ◌̕b; à◌֮◌ۤ◌̕b; a◌֮◌̀◌ۤ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, ARABIC SMALL HIGH MADDA, LATIN SMALL LETTER B +0061 06E4 0315 0300 05AE 0062;0061 05AE 06E4 0300 0315 0062;0061 05AE 06E4 0300 0315 0062;0061 05AE 06E4 0300 0315 0062;0061 05AE 06E4 0300 0315 0062; # (a◌ۤ◌̕◌̀◌֮b; a◌֮◌ۤ◌̀◌̕b; a◌֮◌ۤ◌̀◌̕b; a◌֮◌ۤ◌̀◌̕b; a◌֮◌ۤ◌̀◌̕b; ) LATIN SMALL LETTER A, ARABIC SMALL HIGH MADDA, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 06E7 0062;00E0 05AE 06E7 0315 0062;0061 05AE 0300 06E7 0315 0062;00E0 05AE 06E7 0315 0062;0061 05AE 0300 06E7 0315 0062; # (a◌̕◌̀◌֮◌ۧb; à◌֮◌ۧ◌̕b; a◌֮◌̀◌ۧ◌̕b; à◌֮◌ۧ◌̕b; a◌֮◌̀◌ۧ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, ARABIC SMALL HIGH YEH, LATIN SMALL LETTER B +0061 06E7 0315 0300 05AE 0062;0061 05AE 06E7 0300 0315 0062;0061 05AE 06E7 0300 0315 0062;0061 05AE 06E7 0300 0315 0062;0061 05AE 06E7 0300 0315 0062; # (a◌ۧ◌̕◌̀◌֮b; a◌֮◌ۧ◌̀◌̕b; a◌֮◌ۧ◌̀◌̕b; a◌֮◌ۧ◌̀◌̕b; a◌֮◌ۧ◌̀◌̕b; ) LATIN SMALL LETTER A, ARABIC SMALL HIGH YEH, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 06E8 0062;00E0 05AE 06E8 0315 0062;0061 05AE 0300 06E8 0315 0062;00E0 05AE 06E8 0315 0062;0061 05AE 0300 06E8 0315 0062; # (a◌̕◌̀◌֮◌ۨb; à◌֮◌ۨ◌̕b; a◌֮◌̀◌ۨ◌̕b; à◌֮◌ۨ◌̕b; a◌֮◌̀◌ۨ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, ARABIC SMALL HIGH NOON, LATIN SMALL LETTER B +0061 06E8 0315 0300 05AE 0062;0061 05AE 06E8 0300 0315 0062;0061 05AE 06E8 0300 0315 0062;0061 05AE 06E8 0300 0315 0062;0061 05AE 06E8 0300 0315 0062; # (a◌ۨ◌̕◌̀◌֮b; a◌֮◌ۨ◌̀◌̕b; a◌֮◌ۨ◌̀◌̕b; a◌֮◌ۨ◌̀◌̕b; a◌֮◌ۨ◌̀◌̕b; ) LATIN SMALL LETTER A, ARABIC SMALL HIGH NOON, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 06EA 0062;0061 302A 0316 06EA 059A 0062;0061 302A 0316 06EA 059A 0062;0061 302A 0316 06EA 059A 0062;0061 302A 0316 06EA 059A 0062; # (a◌֚◌̖◌〪◌۪b; a◌〪◌̖◌۪◌֚b; a◌〪◌̖◌۪◌֚b; a◌〪◌̖◌۪◌֚b; a◌〪◌̖◌۪◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, ARABIC EMPTY CENTRE LOW STOP, LATIN SMALL LETTER B +0061 06EA 059A 0316 302A 0062;0061 302A 06EA 0316 059A 0062;0061 302A 06EA 0316 059A 0062;0061 302A 06EA 0316 059A 0062;0061 302A 06EA 0316 059A 0062; # (a◌۪◌֚◌̖◌〪b; a◌〪◌۪◌̖◌֚b; a◌〪◌۪◌̖◌֚b; a◌〪◌۪◌̖◌֚b; a◌〪◌۪◌̖◌֚b; ) LATIN SMALL LETTER A, ARABIC EMPTY CENTRE LOW STOP, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 06EB 0062;00E0 05AE 06EB 0315 0062;0061 05AE 0300 06EB 0315 0062;00E0 05AE 06EB 0315 0062;0061 05AE 0300 06EB 0315 0062; # (a◌̕◌̀◌֮◌۫b; à◌֮◌۫◌̕b; a◌֮◌̀◌۫◌̕b; à◌֮◌۫◌̕b; a◌֮◌̀◌۫◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, ARABIC EMPTY CENTRE HIGH STOP, LATIN SMALL LETTER B +0061 06EB 0315 0300 05AE 0062;0061 05AE 06EB 0300 0315 0062;0061 05AE 06EB 0300 0315 0062;0061 05AE 06EB 0300 0315 0062;0061 05AE 06EB 0300 0315 0062; # (a◌۫◌̕◌̀◌֮b; a◌֮◌۫◌̀◌̕b; a◌֮◌۫◌̀◌̕b; a◌֮◌۫◌̀◌̕b; a◌֮◌۫◌̀◌̕b; ) LATIN SMALL LETTER A, ARABIC EMPTY CENTRE HIGH STOP, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 06EC 0062;00E0 05AE 06EC 0315 0062;0061 05AE 0300 06EC 0315 0062;00E0 05AE 06EC 0315 0062;0061 05AE 0300 06EC 0315 0062; # (a◌̕◌̀◌֮◌۬b; à◌֮◌۬◌̕b; a◌֮◌̀◌۬◌̕b; à◌֮◌۬◌̕b; a◌֮◌̀◌۬◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, ARABIC ROUNDED HIGH STOP WITH FILLED CENTRE, LATIN SMALL LETTER B +0061 06EC 0315 0300 05AE 0062;0061 05AE 06EC 0300 0315 0062;0061 05AE 06EC 0300 0315 0062;0061 05AE 06EC 0300 0315 0062;0061 05AE 06EC 0300 0315 0062; # (a◌۬◌̕◌̀◌֮b; a◌֮◌۬◌̀◌̕b; a◌֮◌۬◌̀◌̕b; a◌֮◌۬◌̀◌̕b; a◌֮◌۬◌̀◌̕b; ) LATIN SMALL LETTER A, ARABIC ROUNDED HIGH STOP WITH FILLED CENTRE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 06ED 0062;0061 302A 0316 06ED 059A 0062;0061 302A 0316 06ED 059A 0062;0061 302A 0316 06ED 059A 0062;0061 302A 0316 06ED 059A 0062; # (a◌֚◌̖◌〪◌ۭb; a◌〪◌̖◌ۭ◌֚b; a◌〪◌̖◌ۭ◌֚b; a◌〪◌̖◌ۭ◌֚b; a◌〪◌̖◌ۭ◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, ARABIC SMALL LOW MEEM, LATIN SMALL LETTER B +0061 06ED 059A 0316 302A 0062;0061 302A 06ED 0316 059A 0062;0061 302A 06ED 0316 059A 0062;0061 302A 06ED 0316 059A 0062;0061 302A 06ED 0316 059A 0062; # (a◌ۭ◌֚◌̖◌〪b; a◌〪◌ۭ◌̖◌֚b; a◌〪◌ۭ◌̖◌֚b; a◌〪◌ۭ◌̖◌֚b; a◌〪◌ۭ◌̖◌֚b; ) LATIN SMALL LETTER A, ARABIC SMALL LOW MEEM, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0C55 0711 0670 0711 0062;0061 0670 0711 0711 0C55 0062;0061 0670 0711 0711 0C55 0062;0061 0670 0711 0711 0C55 0062;0061 0670 0711 0711 0C55 0062; # (a◌ౕ◌ܑ◌ٰ◌ܑb; a◌ٰ◌ܑ◌ܑ◌ౕb; a◌ٰ◌ܑ◌ܑ◌ౕb; a◌ٰ◌ܑ◌ܑ◌ౕb; a◌ٰ◌ܑ◌ܑ◌ౕb; ) LATIN SMALL LETTER A, TELUGU LENGTH MARK, SYRIAC LETTER SUPERSCRIPT ALAPH, ARABIC LETTER SUPERSCRIPT ALEF, SYRIAC LETTER SUPERSCRIPT ALAPH, LATIN SMALL LETTER B +0061 0711 0C55 0711 0670 0062;0061 0670 0711 0711 0C55 0062;0061 0670 0711 0711 0C55 0062;0061 0670 0711 0711 0C55 0062;0061 0670 0711 0711 0C55 0062; # (a◌ܑ◌ౕ◌ܑ◌ٰb; a◌ٰ◌ܑ◌ܑ◌ౕb; a◌ٰ◌ܑ◌ܑ◌ౕb; a◌ٰ◌ܑ◌ܑ◌ౕb; a◌ٰ◌ܑ◌ܑ◌ౕb; ) LATIN SMALL LETTER A, SYRIAC LETTER SUPERSCRIPT ALAPH, TELUGU LENGTH MARK, SYRIAC LETTER SUPERSCRIPT ALAPH, ARABIC LETTER SUPERSCRIPT ALEF, LATIN SMALL LETTER B +0061 0315 0300 05AE 0730 0062;00E0 05AE 0730 0315 0062;0061 05AE 0300 0730 0315 0062;00E0 05AE 0730 0315 0062;0061 05AE 0300 0730 0315 0062; # (a◌̕◌̀◌֮◌ܰb; à◌֮◌ܰ◌̕b; a◌֮◌̀◌ܰ◌̕b; à◌֮◌ܰ◌̕b; a◌֮◌̀◌ܰ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, SYRIAC PTHAHA ABOVE, LATIN SMALL LETTER B +0061 0730 0315 0300 05AE 0062;0061 05AE 0730 0300 0315 0062;0061 05AE 0730 0300 0315 0062;0061 05AE 0730 0300 0315 0062;0061 05AE 0730 0300 0315 0062; # (a◌ܰ◌̕◌̀◌֮b; a◌֮◌ܰ◌̀◌̕b; a◌֮◌ܰ◌̀◌̕b; a◌֮◌ܰ◌̀◌̕b; a◌֮◌ܰ◌̀◌̕b; ) LATIN SMALL LETTER A, SYRIAC PTHAHA ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 0731 0062;0061 302A 0316 0731 059A 0062;0061 302A 0316 0731 059A 0062;0061 302A 0316 0731 059A 0062;0061 302A 0316 0731 059A 0062; # (a◌֚◌̖◌〪◌ܱb; a◌〪◌̖◌ܱ◌֚b; a◌〪◌̖◌ܱ◌֚b; a◌〪◌̖◌ܱ◌֚b; a◌〪◌̖◌ܱ◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, SYRIAC PTHAHA BELOW, LATIN SMALL LETTER B +0061 0731 059A 0316 302A 0062;0061 302A 0731 0316 059A 0062;0061 302A 0731 0316 059A 0062;0061 302A 0731 0316 059A 0062;0061 302A 0731 0316 059A 0062; # (a◌ܱ◌֚◌̖◌〪b; a◌〪◌ܱ◌̖◌֚b; a◌〪◌ܱ◌̖◌֚b; a◌〪◌ܱ◌̖◌֚b; a◌〪◌ܱ◌̖◌֚b; ) LATIN SMALL LETTER A, SYRIAC PTHAHA BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 0732 0062;00E0 05AE 0732 0315 0062;0061 05AE 0300 0732 0315 0062;00E0 05AE 0732 0315 0062;0061 05AE 0300 0732 0315 0062; # (a◌̕◌̀◌֮◌ܲb; à◌֮◌ܲ◌̕b; a◌֮◌̀◌ܲ◌̕b; à◌֮◌ܲ◌̕b; a◌֮◌̀◌ܲ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, SYRIAC PTHAHA DOTTED, LATIN SMALL LETTER B +0061 0732 0315 0300 05AE 0062;0061 05AE 0732 0300 0315 0062;0061 05AE 0732 0300 0315 0062;0061 05AE 0732 0300 0315 0062;0061 05AE 0732 0300 0315 0062; # (a◌ܲ◌̕◌̀◌֮b; a◌֮◌ܲ◌̀◌̕b; a◌֮◌ܲ◌̀◌̕b; a◌֮◌ܲ◌̀◌̕b; a◌֮◌ܲ◌̀◌̕b; ) LATIN SMALL LETTER A, SYRIAC PTHAHA DOTTED, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0733 0062;00E0 05AE 0733 0315 0062;0061 05AE 0300 0733 0315 0062;00E0 05AE 0733 0315 0062;0061 05AE 0300 0733 0315 0062; # (a◌̕◌̀◌֮◌ܳb; à◌֮◌ܳ◌̕b; a◌֮◌̀◌ܳ◌̕b; à◌֮◌ܳ◌̕b; a◌֮◌̀◌ܳ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, SYRIAC ZQAPHA ABOVE, LATIN SMALL LETTER B +0061 0733 0315 0300 05AE 0062;0061 05AE 0733 0300 0315 0062;0061 05AE 0733 0300 0315 0062;0061 05AE 0733 0300 0315 0062;0061 05AE 0733 0300 0315 0062; # (a◌ܳ◌̕◌̀◌֮b; a◌֮◌ܳ◌̀◌̕b; a◌֮◌ܳ◌̀◌̕b; a◌֮◌ܳ◌̀◌̕b; a◌֮◌ܳ◌̀◌̕b; ) LATIN SMALL LETTER A, SYRIAC ZQAPHA ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 0734 0062;0061 302A 0316 0734 059A 0062;0061 302A 0316 0734 059A 0062;0061 302A 0316 0734 059A 0062;0061 302A 0316 0734 059A 0062; # (a◌֚◌̖◌〪◌ܴb; a◌〪◌̖◌ܴ◌֚b; a◌〪◌̖◌ܴ◌֚b; a◌〪◌̖◌ܴ◌֚b; a◌〪◌̖◌ܴ◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, SYRIAC ZQAPHA BELOW, LATIN SMALL LETTER B +0061 0734 059A 0316 302A 0062;0061 302A 0734 0316 059A 0062;0061 302A 0734 0316 059A 0062;0061 302A 0734 0316 059A 0062;0061 302A 0734 0316 059A 0062; # (a◌ܴ◌֚◌̖◌〪b; a◌〪◌ܴ◌̖◌֚b; a◌〪◌ܴ◌̖◌֚b; a◌〪◌ܴ◌̖◌֚b; a◌〪◌ܴ◌̖◌֚b; ) LATIN SMALL LETTER A, SYRIAC ZQAPHA BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 0735 0062;00E0 05AE 0735 0315 0062;0061 05AE 0300 0735 0315 0062;00E0 05AE 0735 0315 0062;0061 05AE 0300 0735 0315 0062; # (a◌̕◌̀◌֮◌ܵb; à◌֮◌ܵ◌̕b; a◌֮◌̀◌ܵ◌̕b; à◌֮◌ܵ◌̕b; a◌֮◌̀◌ܵ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, SYRIAC ZQAPHA DOTTED, LATIN SMALL LETTER B +0061 0735 0315 0300 05AE 0062;0061 05AE 0735 0300 0315 0062;0061 05AE 0735 0300 0315 0062;0061 05AE 0735 0300 0315 0062;0061 05AE 0735 0300 0315 0062; # (a◌ܵ◌̕◌̀◌֮b; a◌֮◌ܵ◌̀◌̕b; a◌֮◌ܵ◌̀◌̕b; a◌֮◌ܵ◌̀◌̕b; a◌֮◌ܵ◌̀◌̕b; ) LATIN SMALL LETTER A, SYRIAC ZQAPHA DOTTED, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0736 0062;00E0 05AE 0736 0315 0062;0061 05AE 0300 0736 0315 0062;00E0 05AE 0736 0315 0062;0061 05AE 0300 0736 0315 0062; # (a◌̕◌̀◌֮◌ܶb; à◌֮◌ܶ◌̕b; a◌֮◌̀◌ܶ◌̕b; à◌֮◌ܶ◌̕b; a◌֮◌̀◌ܶ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, SYRIAC RBASA ABOVE, LATIN SMALL LETTER B +0061 0736 0315 0300 05AE 0062;0061 05AE 0736 0300 0315 0062;0061 05AE 0736 0300 0315 0062;0061 05AE 0736 0300 0315 0062;0061 05AE 0736 0300 0315 0062; # (a◌ܶ◌̕◌̀◌֮b; a◌֮◌ܶ◌̀◌̕b; a◌֮◌ܶ◌̀◌̕b; a◌֮◌ܶ◌̀◌̕b; a◌֮◌ܶ◌̀◌̕b; ) LATIN SMALL LETTER A, SYRIAC RBASA ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 0737 0062;0061 302A 0316 0737 059A 0062;0061 302A 0316 0737 059A 0062;0061 302A 0316 0737 059A 0062;0061 302A 0316 0737 059A 0062; # (a◌֚◌̖◌〪◌ܷb; a◌〪◌̖◌ܷ◌֚b; a◌〪◌̖◌ܷ◌֚b; a◌〪◌̖◌ܷ◌֚b; a◌〪◌̖◌ܷ◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, SYRIAC RBASA BELOW, LATIN SMALL LETTER B +0061 0737 059A 0316 302A 0062;0061 302A 0737 0316 059A 0062;0061 302A 0737 0316 059A 0062;0061 302A 0737 0316 059A 0062;0061 302A 0737 0316 059A 0062; # (a◌ܷ◌֚◌̖◌〪b; a◌〪◌ܷ◌̖◌֚b; a◌〪◌ܷ◌̖◌֚b; a◌〪◌ܷ◌̖◌֚b; a◌〪◌ܷ◌̖◌֚b; ) LATIN SMALL LETTER A, SYRIAC RBASA BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 0738 0062;0061 302A 0316 0738 059A 0062;0061 302A 0316 0738 059A 0062;0061 302A 0316 0738 059A 0062;0061 302A 0316 0738 059A 0062; # (a◌֚◌̖◌〪◌ܸb; a◌〪◌̖◌ܸ◌֚b; a◌〪◌̖◌ܸ◌֚b; a◌〪◌̖◌ܸ◌֚b; a◌〪◌̖◌ܸ◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, SYRIAC DOTTED ZLAMA HORIZONTAL, LATIN SMALL LETTER B +0061 0738 059A 0316 302A 0062;0061 302A 0738 0316 059A 0062;0061 302A 0738 0316 059A 0062;0061 302A 0738 0316 059A 0062;0061 302A 0738 0316 059A 0062; # (a◌ܸ◌֚◌̖◌〪b; a◌〪◌ܸ◌̖◌֚b; a◌〪◌ܸ◌̖◌֚b; a◌〪◌ܸ◌̖◌֚b; a◌〪◌ܸ◌̖◌֚b; ) LATIN SMALL LETTER A, SYRIAC DOTTED ZLAMA HORIZONTAL, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 0739 0062;0061 302A 0316 0739 059A 0062;0061 302A 0316 0739 059A 0062;0061 302A 0316 0739 059A 0062;0061 302A 0316 0739 059A 0062; # (a◌֚◌̖◌〪◌ܹb; a◌〪◌̖◌ܹ◌֚b; a◌〪◌̖◌ܹ◌֚b; a◌〪◌̖◌ܹ◌֚b; a◌〪◌̖◌ܹ◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, SYRIAC DOTTED ZLAMA ANGULAR, LATIN SMALL LETTER B +0061 0739 059A 0316 302A 0062;0061 302A 0739 0316 059A 0062;0061 302A 0739 0316 059A 0062;0061 302A 0739 0316 059A 0062;0061 302A 0739 0316 059A 0062; # (a◌ܹ◌֚◌̖◌〪b; a◌〪◌ܹ◌̖◌֚b; a◌〪◌ܹ◌̖◌֚b; a◌〪◌ܹ◌̖◌֚b; a◌〪◌ܹ◌̖◌֚b; ) LATIN SMALL LETTER A, SYRIAC DOTTED ZLAMA ANGULAR, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 073A 0062;00E0 05AE 073A 0315 0062;0061 05AE 0300 073A 0315 0062;00E0 05AE 073A 0315 0062;0061 05AE 0300 073A 0315 0062; # (a◌̕◌̀◌֮◌ܺb; à◌֮◌ܺ◌̕b; a◌֮◌̀◌ܺ◌̕b; à◌֮◌ܺ◌̕b; a◌֮◌̀◌ܺ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, SYRIAC HBASA ABOVE, LATIN SMALL LETTER B +0061 073A 0315 0300 05AE 0062;0061 05AE 073A 0300 0315 0062;0061 05AE 073A 0300 0315 0062;0061 05AE 073A 0300 0315 0062;0061 05AE 073A 0300 0315 0062; # (a◌ܺ◌̕◌̀◌֮b; a◌֮◌ܺ◌̀◌̕b; a◌֮◌ܺ◌̀◌̕b; a◌֮◌ܺ◌̀◌̕b; a◌֮◌ܺ◌̀◌̕b; ) LATIN SMALL LETTER A, SYRIAC HBASA ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 073B 0062;0061 302A 0316 073B 059A 0062;0061 302A 0316 073B 059A 0062;0061 302A 0316 073B 059A 0062;0061 302A 0316 073B 059A 0062; # (a◌֚◌̖◌〪◌ܻb; a◌〪◌̖◌ܻ◌֚b; a◌〪◌̖◌ܻ◌֚b; a◌〪◌̖◌ܻ◌֚b; a◌〪◌̖◌ܻ◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, SYRIAC HBASA BELOW, LATIN SMALL LETTER B +0061 073B 059A 0316 302A 0062;0061 302A 073B 0316 059A 0062;0061 302A 073B 0316 059A 0062;0061 302A 073B 0316 059A 0062;0061 302A 073B 0316 059A 0062; # (a◌ܻ◌֚◌̖◌〪b; a◌〪◌ܻ◌̖◌֚b; a◌〪◌ܻ◌̖◌֚b; a◌〪◌ܻ◌̖◌֚b; a◌〪◌ܻ◌̖◌֚b; ) LATIN SMALL LETTER A, SYRIAC HBASA BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 073C 0062;0061 302A 0316 073C 059A 0062;0061 302A 0316 073C 059A 0062;0061 302A 0316 073C 059A 0062;0061 302A 0316 073C 059A 0062; # (a◌֚◌̖◌〪◌ܼb; a◌〪◌̖◌ܼ◌֚b; a◌〪◌̖◌ܼ◌֚b; a◌〪◌̖◌ܼ◌֚b; a◌〪◌̖◌ܼ◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, SYRIAC HBASA-ESASA DOTTED, LATIN SMALL LETTER B +0061 073C 059A 0316 302A 0062;0061 302A 073C 0316 059A 0062;0061 302A 073C 0316 059A 0062;0061 302A 073C 0316 059A 0062;0061 302A 073C 0316 059A 0062; # (a◌ܼ◌֚◌̖◌〪b; a◌〪◌ܼ◌̖◌֚b; a◌〪◌ܼ◌̖◌֚b; a◌〪◌ܼ◌̖◌֚b; a◌〪◌ܼ◌̖◌֚b; ) LATIN SMALL LETTER A, SYRIAC HBASA-ESASA DOTTED, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 073D 0062;00E0 05AE 073D 0315 0062;0061 05AE 0300 073D 0315 0062;00E0 05AE 073D 0315 0062;0061 05AE 0300 073D 0315 0062; # (a◌̕◌̀◌֮◌ܽb; à◌֮◌ܽ◌̕b; a◌֮◌̀◌ܽ◌̕b; à◌֮◌ܽ◌̕b; a◌֮◌̀◌ܽ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, SYRIAC ESASA ABOVE, LATIN SMALL LETTER B +0061 073D 0315 0300 05AE 0062;0061 05AE 073D 0300 0315 0062;0061 05AE 073D 0300 0315 0062;0061 05AE 073D 0300 0315 0062;0061 05AE 073D 0300 0315 0062; # (a◌ܽ◌̕◌̀◌֮b; a◌֮◌ܽ◌̀◌̕b; a◌֮◌ܽ◌̀◌̕b; a◌֮◌ܽ◌̀◌̕b; a◌֮◌ܽ◌̀◌̕b; ) LATIN SMALL LETTER A, SYRIAC ESASA ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 073E 0062;0061 302A 0316 073E 059A 0062;0061 302A 0316 073E 059A 0062;0061 302A 0316 073E 059A 0062;0061 302A 0316 073E 059A 0062; # (a◌֚◌̖◌〪◌ܾb; a◌〪◌̖◌ܾ◌֚b; a◌〪◌̖◌ܾ◌֚b; a◌〪◌̖◌ܾ◌֚b; a◌〪◌̖◌ܾ◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, SYRIAC ESASA BELOW, LATIN SMALL LETTER B +0061 073E 059A 0316 302A 0062;0061 302A 073E 0316 059A 0062;0061 302A 073E 0316 059A 0062;0061 302A 073E 0316 059A 0062;0061 302A 073E 0316 059A 0062; # (a◌ܾ◌֚◌̖◌〪b; a◌〪◌ܾ◌̖◌֚b; a◌〪◌ܾ◌̖◌֚b; a◌〪◌ܾ◌̖◌֚b; a◌〪◌ܾ◌̖◌֚b; ) LATIN SMALL LETTER A, SYRIAC ESASA BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 073F 0062;00E0 05AE 073F 0315 0062;0061 05AE 0300 073F 0315 0062;00E0 05AE 073F 0315 0062;0061 05AE 0300 073F 0315 0062; # (a◌̕◌̀◌֮◌ܿb; à◌֮◌ܿ◌̕b; a◌֮◌̀◌ܿ◌̕b; à◌֮◌ܿ◌̕b; a◌֮◌̀◌ܿ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, SYRIAC RWAHA, LATIN SMALL LETTER B +0061 073F 0315 0300 05AE 0062;0061 05AE 073F 0300 0315 0062;0061 05AE 073F 0300 0315 0062;0061 05AE 073F 0300 0315 0062;0061 05AE 073F 0300 0315 0062; # (a◌ܿ◌̕◌̀◌֮b; a◌֮◌ܿ◌̀◌̕b; a◌֮◌ܿ◌̀◌̕b; a◌֮◌ܿ◌̀◌̕b; a◌֮◌ܿ◌̀◌̕b; ) LATIN SMALL LETTER A, SYRIAC RWAHA, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0740 0062;00E0 05AE 0740 0315 0062;0061 05AE 0300 0740 0315 0062;00E0 05AE 0740 0315 0062;0061 05AE 0300 0740 0315 0062; # (a◌̕◌̀◌֮◌݀b; à◌֮◌݀◌̕b; a◌֮◌̀◌݀◌̕b; à◌֮◌݀◌̕b; a◌֮◌̀◌݀◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, SYRIAC FEMININE DOT, LATIN SMALL LETTER B +0061 0740 0315 0300 05AE 0062;0061 05AE 0740 0300 0315 0062;0061 05AE 0740 0300 0315 0062;0061 05AE 0740 0300 0315 0062;0061 05AE 0740 0300 0315 0062; # (a◌݀◌̕◌̀◌֮b; a◌֮◌݀◌̀◌̕b; a◌֮◌݀◌̀◌̕b; a◌֮◌݀◌̀◌̕b; a◌֮◌݀◌̀◌̕b; ) LATIN SMALL LETTER A, SYRIAC FEMININE DOT, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0741 0062;00E0 05AE 0741 0315 0062;0061 05AE 0300 0741 0315 0062;00E0 05AE 0741 0315 0062;0061 05AE 0300 0741 0315 0062; # (a◌̕◌̀◌֮◌݁b; à◌֮◌݁◌̕b; a◌֮◌̀◌݁◌̕b; à◌֮◌݁◌̕b; a◌֮◌̀◌݁◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, SYRIAC QUSHSHAYA, LATIN SMALL LETTER B +0061 0741 0315 0300 05AE 0062;0061 05AE 0741 0300 0315 0062;0061 05AE 0741 0300 0315 0062;0061 05AE 0741 0300 0315 0062;0061 05AE 0741 0300 0315 0062; # (a◌݁◌̕◌̀◌֮b; a◌֮◌݁◌̀◌̕b; a◌֮◌݁◌̀◌̕b; a◌֮◌݁◌̀◌̕b; a◌֮◌݁◌̀◌̕b; ) LATIN SMALL LETTER A, SYRIAC QUSHSHAYA, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 0742 0062;0061 302A 0316 0742 059A 0062;0061 302A 0316 0742 059A 0062;0061 302A 0316 0742 059A 0062;0061 302A 0316 0742 059A 0062; # (a◌֚◌̖◌〪◌݂b; a◌〪◌̖◌݂◌֚b; a◌〪◌̖◌݂◌֚b; a◌〪◌̖◌݂◌֚b; a◌〪◌̖◌݂◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, SYRIAC RUKKAKHA, LATIN SMALL LETTER B +0061 0742 059A 0316 302A 0062;0061 302A 0742 0316 059A 0062;0061 302A 0742 0316 059A 0062;0061 302A 0742 0316 059A 0062;0061 302A 0742 0316 059A 0062; # (a◌݂◌֚◌̖◌〪b; a◌〪◌݂◌̖◌֚b; a◌〪◌݂◌̖◌֚b; a◌〪◌݂◌̖◌֚b; a◌〪◌݂◌̖◌֚b; ) LATIN SMALL LETTER A, SYRIAC RUKKAKHA, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 0743 0062;00E0 05AE 0743 0315 0062;0061 05AE 0300 0743 0315 0062;00E0 05AE 0743 0315 0062;0061 05AE 0300 0743 0315 0062; # (a◌̕◌̀◌֮◌݃b; à◌֮◌݃◌̕b; a◌֮◌̀◌݃◌̕b; à◌֮◌݃◌̕b; a◌֮◌̀◌݃◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, SYRIAC TWO VERTICAL DOTS ABOVE, LATIN SMALL LETTER B +0061 0743 0315 0300 05AE 0062;0061 05AE 0743 0300 0315 0062;0061 05AE 0743 0300 0315 0062;0061 05AE 0743 0300 0315 0062;0061 05AE 0743 0300 0315 0062; # (a◌݃◌̕◌̀◌֮b; a◌֮◌݃◌̀◌̕b; a◌֮◌݃◌̀◌̕b; a◌֮◌݃◌̀◌̕b; a◌֮◌݃◌̀◌̕b; ) LATIN SMALL LETTER A, SYRIAC TWO VERTICAL DOTS ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 0744 0062;0061 302A 0316 0744 059A 0062;0061 302A 0316 0744 059A 0062;0061 302A 0316 0744 059A 0062;0061 302A 0316 0744 059A 0062; # (a◌֚◌̖◌〪◌݄b; a◌〪◌̖◌݄◌֚b; a◌〪◌̖◌݄◌֚b; a◌〪◌̖◌݄◌֚b; a◌〪◌̖◌݄◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, SYRIAC TWO VERTICAL DOTS BELOW, LATIN SMALL LETTER B +0061 0744 059A 0316 302A 0062;0061 302A 0744 0316 059A 0062;0061 302A 0744 0316 059A 0062;0061 302A 0744 0316 059A 0062;0061 302A 0744 0316 059A 0062; # (a◌݄◌֚◌̖◌〪b; a◌〪◌݄◌̖◌֚b; a◌〪◌݄◌̖◌֚b; a◌〪◌݄◌̖◌֚b; a◌〪◌݄◌̖◌֚b; ) LATIN SMALL LETTER A, SYRIAC TWO VERTICAL DOTS BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 0745 0062;00E0 05AE 0745 0315 0062;0061 05AE 0300 0745 0315 0062;00E0 05AE 0745 0315 0062;0061 05AE 0300 0745 0315 0062; # (a◌̕◌̀◌֮◌݅b; à◌֮◌݅◌̕b; a◌֮◌̀◌݅◌̕b; à◌֮◌݅◌̕b; a◌֮◌̀◌݅◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, SYRIAC THREE DOTS ABOVE, LATIN SMALL LETTER B +0061 0745 0315 0300 05AE 0062;0061 05AE 0745 0300 0315 0062;0061 05AE 0745 0300 0315 0062;0061 05AE 0745 0300 0315 0062;0061 05AE 0745 0300 0315 0062; # (a◌݅◌̕◌̀◌֮b; a◌֮◌݅◌̀◌̕b; a◌֮◌݅◌̀◌̕b; a◌֮◌݅◌̀◌̕b; a◌֮◌݅◌̀◌̕b; ) LATIN SMALL LETTER A, SYRIAC THREE DOTS ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 0746 0062;0061 302A 0316 0746 059A 0062;0061 302A 0316 0746 059A 0062;0061 302A 0316 0746 059A 0062;0061 302A 0316 0746 059A 0062; # (a◌֚◌̖◌〪◌݆b; a◌〪◌̖◌݆◌֚b; a◌〪◌̖◌݆◌֚b; a◌〪◌̖◌݆◌֚b; a◌〪◌̖◌݆◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, SYRIAC THREE DOTS BELOW, LATIN SMALL LETTER B +0061 0746 059A 0316 302A 0062;0061 302A 0746 0316 059A 0062;0061 302A 0746 0316 059A 0062;0061 302A 0746 0316 059A 0062;0061 302A 0746 0316 059A 0062; # (a◌݆◌֚◌̖◌〪b; a◌〪◌݆◌̖◌֚b; a◌〪◌݆◌̖◌֚b; a◌〪◌݆◌̖◌֚b; a◌〪◌݆◌̖◌֚b; ) LATIN SMALL LETTER A, SYRIAC THREE DOTS BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 0747 0062;00E0 05AE 0747 0315 0062;0061 05AE 0300 0747 0315 0062;00E0 05AE 0747 0315 0062;0061 05AE 0300 0747 0315 0062; # (a◌̕◌̀◌֮◌݇b; à◌֮◌݇◌̕b; a◌֮◌̀◌݇◌̕b; à◌֮◌݇◌̕b; a◌֮◌̀◌݇◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, SYRIAC OBLIQUE LINE ABOVE, LATIN SMALL LETTER B +0061 0747 0315 0300 05AE 0062;0061 05AE 0747 0300 0315 0062;0061 05AE 0747 0300 0315 0062;0061 05AE 0747 0300 0315 0062;0061 05AE 0747 0300 0315 0062; # (a◌݇◌̕◌̀◌֮b; a◌֮◌݇◌̀◌̕b; a◌֮◌݇◌̀◌̕b; a◌֮◌݇◌̀◌̕b; a◌֮◌݇◌̀◌̕b; ) LATIN SMALL LETTER A, SYRIAC OBLIQUE LINE ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 0748 0062;0061 302A 0316 0748 059A 0062;0061 302A 0316 0748 059A 0062;0061 302A 0316 0748 059A 0062;0061 302A 0316 0748 059A 0062; # (a◌֚◌̖◌〪◌݈b; a◌〪◌̖◌݈◌֚b; a◌〪◌̖◌݈◌֚b; a◌〪◌̖◌݈◌֚b; a◌〪◌̖◌݈◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, SYRIAC OBLIQUE LINE BELOW, LATIN SMALL LETTER B +0061 0748 059A 0316 302A 0062;0061 302A 0748 0316 059A 0062;0061 302A 0748 0316 059A 0062;0061 302A 0748 0316 059A 0062;0061 302A 0748 0316 059A 0062; # (a◌݈◌֚◌̖◌〪b; a◌〪◌݈◌̖◌֚b; a◌〪◌݈◌̖◌֚b; a◌〪◌݈◌̖◌֚b; a◌〪◌݈◌̖◌֚b; ) LATIN SMALL LETTER A, SYRIAC OBLIQUE LINE BELOW, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 0749 0062;00E0 05AE 0749 0315 0062;0061 05AE 0300 0749 0315 0062;00E0 05AE 0749 0315 0062;0061 05AE 0300 0749 0315 0062; # (a◌̕◌̀◌֮◌݉b; à◌֮◌݉◌̕b; a◌֮◌̀◌݉◌̕b; à◌֮◌݉◌̕b; a◌֮◌̀◌݉◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, SYRIAC MUSIC, LATIN SMALL LETTER B +0061 0749 0315 0300 05AE 0062;0061 05AE 0749 0300 0315 0062;0061 05AE 0749 0300 0315 0062;0061 05AE 0749 0300 0315 0062;0061 05AE 0749 0300 0315 0062; # (a◌݉◌̕◌̀◌֮b; a◌֮◌݉◌̀◌̕b; a◌֮◌݉◌̀◌̕b; a◌֮◌݉◌̀◌̕b; a◌֮◌݉◌̀◌̕b; ) LATIN SMALL LETTER A, SYRIAC MUSIC, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 074A 0062;00E0 05AE 074A 0315 0062;0061 05AE 0300 074A 0315 0062;00E0 05AE 074A 0315 0062;0061 05AE 0300 074A 0315 0062; # (a◌̕◌̀◌֮◌݊b; à◌֮◌݊◌̕b; a◌֮◌̀◌݊◌̕b; à◌֮◌݊◌̕b; a◌֮◌̀◌݊◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, SYRIAC BARREKH, LATIN SMALL LETTER B +0061 074A 0315 0300 05AE 0062;0061 05AE 074A 0300 0315 0062;0061 05AE 074A 0300 0315 0062;0061 05AE 074A 0300 0315 0062;0061 05AE 074A 0300 0315 0062; # (a◌݊◌̕◌̀◌֮b; a◌֮◌݊◌̀◌̕b; a◌֮◌݊◌̀◌̕b; a◌֮◌݊◌̀◌̕b; a◌֮◌݊◌̀◌̕b; ) LATIN SMALL LETTER A, SYRIAC BARREKH, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 3099 093C 0334 093C 0062;0061 0334 093C 093C 3099 0062;0061 0334 093C 093C 3099 0062;0061 0334 093C 093C 3099 0062;0061 0334 093C 093C 3099 0062; # (a◌゙◌़◌̴◌़b; a◌̴◌़◌़◌゙b; a◌̴◌़◌़◌゙b; a◌̴◌़◌़◌゙b; a◌̴◌़◌़◌゙b; ) LATIN SMALL LETTER A, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, DEVANAGARI SIGN NUKTA, LATIN SMALL LETTER B +0061 093C 3099 093C 0334 0062;0061 0334 093C 093C 3099 0062;0061 0334 093C 093C 3099 0062;0061 0334 093C 093C 3099 0062;0061 0334 093C 093C 3099 0062; # (a◌़◌゙◌़◌̴b; a◌̴◌़◌़◌゙b; a◌̴◌़◌़◌゙b; a◌̴◌़◌़◌゙b; a◌̴◌़◌़◌゙b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 05B0 094D 3099 094D 0062;0061 3099 094D 094D 05B0 0062;0061 3099 094D 094D 05B0 0062;0061 3099 094D 094D 05B0 0062;0061 3099 094D 094D 05B0 0062; # (a◌ְ◌्◌゙◌्b; a◌゙◌्◌्◌ְb; a◌゙◌्◌्◌ְb; a◌゙◌्◌्◌ְb; a◌゙◌्◌्◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN VIRAMA, LATIN SMALL LETTER B +0061 094D 05B0 094D 3099 0062;0061 3099 094D 094D 05B0 0062;0061 3099 094D 094D 05B0 0062;0061 3099 094D 094D 05B0 0062;0061 3099 094D 094D 05B0 0062; # (a◌्◌ְ◌्◌゙b; a◌゙◌्◌्◌ְb; a◌゙◌्◌्◌ְb; a◌゙◌्◌्◌ְb; a◌゙◌्◌्◌ְb; ) LATIN SMALL LETTER A, DEVANAGARI SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 0951 0062;00E0 05AE 0951 0315 0062;0061 05AE 0300 0951 0315 0062;00E0 05AE 0951 0315 0062;0061 05AE 0300 0951 0315 0062; # (a◌̕◌̀◌֮◌॑b; à◌֮◌॑◌̕b; a◌֮◌̀◌॑◌̕b; à◌֮◌॑◌̕b; a◌֮◌̀◌॑◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, DEVANAGARI STRESS SIGN UDATTA, LATIN SMALL LETTER B +0061 0951 0315 0300 05AE 0062;0061 05AE 0951 0300 0315 0062;0061 05AE 0951 0300 0315 0062;0061 05AE 0951 0300 0315 0062;0061 05AE 0951 0300 0315 0062; # (a◌॑◌̕◌̀◌֮b; a◌֮◌॑◌̀◌̕b; a◌֮◌॑◌̀◌̕b; a◌֮◌॑◌̀◌̕b; a◌֮◌॑◌̀◌̕b; ) LATIN SMALL LETTER A, DEVANAGARI STRESS SIGN UDATTA, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 0952 0062;0061 302A 0316 0952 059A 0062;0061 302A 0316 0952 059A 0062;0061 302A 0316 0952 059A 0062;0061 302A 0316 0952 059A 0062; # (a◌֚◌̖◌〪◌॒b; a◌〪◌̖◌॒◌֚b; a◌〪◌̖◌॒◌֚b; a◌〪◌̖◌॒◌֚b; a◌〪◌̖◌॒◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, DEVANAGARI STRESS SIGN ANUDATTA, LATIN SMALL LETTER B +0061 0952 059A 0316 302A 0062;0061 302A 0952 0316 059A 0062;0061 302A 0952 0316 059A 0062;0061 302A 0952 0316 059A 0062;0061 302A 0952 0316 059A 0062; # (a◌॒◌֚◌̖◌〪b; a◌〪◌॒◌̖◌֚b; a◌〪◌॒◌̖◌֚b; a◌〪◌॒◌̖◌֚b; a◌〪◌॒◌̖◌֚b; ) LATIN SMALL LETTER A, DEVANAGARI STRESS SIGN ANUDATTA, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 0953 0062;00E0 05AE 0953 0315 0062;0061 05AE 0300 0953 0315 0062;00E0 05AE 0953 0315 0062;0061 05AE 0300 0953 0315 0062; # (a◌̕◌̀◌֮◌॓b; à◌֮◌॓◌̕b; a◌֮◌̀◌॓◌̕b; à◌֮◌॓◌̕b; a◌֮◌̀◌॓◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, DEVANAGARI GRAVE ACCENT, LATIN SMALL LETTER B +0061 0953 0315 0300 05AE 0062;0061 05AE 0953 0300 0315 0062;0061 05AE 0953 0300 0315 0062;0061 05AE 0953 0300 0315 0062;0061 05AE 0953 0300 0315 0062; # (a◌॓◌̕◌̀◌֮b; a◌֮◌॓◌̀◌̕b; a◌֮◌॓◌̀◌̕b; a◌֮◌॓◌̀◌̕b; a◌֮◌॓◌̀◌̕b; ) LATIN SMALL LETTER A, DEVANAGARI GRAVE ACCENT, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0954 0062;00E0 05AE 0954 0315 0062;0061 05AE 0300 0954 0315 0062;00E0 05AE 0954 0315 0062;0061 05AE 0300 0954 0315 0062; # (a◌̕◌̀◌֮◌॔b; à◌֮◌॔◌̕b; a◌֮◌̀◌॔◌̕b; à◌֮◌॔◌̕b; a◌֮◌̀◌॔◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, DEVANAGARI ACUTE ACCENT, LATIN SMALL LETTER B +0061 0954 0315 0300 05AE 0062;0061 05AE 0954 0300 0315 0062;0061 05AE 0954 0300 0315 0062;0061 05AE 0954 0300 0315 0062;0061 05AE 0954 0300 0315 0062; # (a◌॔◌̕◌̀◌֮b; a◌֮◌॔◌̀◌̕b; a◌֮◌॔◌̀◌̕b; a◌֮◌॔◌̀◌̕b; a◌֮◌॔◌̀◌̕b; ) LATIN SMALL LETTER A, DEVANAGARI ACUTE ACCENT, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 3099 093C 0334 09BC 0062;0061 0334 093C 09BC 3099 0062;0061 0334 093C 09BC 3099 0062;0061 0334 093C 09BC 3099 0062;0061 0334 093C 09BC 3099 0062; # (a◌゙◌़◌̴◌়b; a◌̴◌़◌়◌゙b; a◌̴◌़◌়◌゙b; a◌̴◌़◌়◌゙b; a◌̴◌़◌়◌゙b; ) LATIN SMALL LETTER A, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, BENGALI SIGN NUKTA, LATIN SMALL LETTER B +0061 09BC 3099 093C 0334 0062;0061 0334 09BC 093C 3099 0062;0061 0334 09BC 093C 3099 0062;0061 0334 09BC 093C 3099 0062;0061 0334 09BC 093C 3099 0062; # (a◌়◌゙◌़◌̴b; a◌̴◌়◌़◌゙b; a◌̴◌়◌़◌゙b; a◌̴◌়◌़◌゙b; a◌̴◌়◌़◌゙b; ) LATIN SMALL LETTER A, BENGALI SIGN NUKTA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 05B0 094D 3099 09CD 0062;0061 3099 094D 09CD 05B0 0062;0061 3099 094D 09CD 05B0 0062;0061 3099 094D 09CD 05B0 0062;0061 3099 094D 09CD 05B0 0062; # (a◌ְ◌्◌゙◌্b; a◌゙◌्◌্◌ְb; a◌゙◌्◌্◌ְb; a◌゙◌्◌্◌ְb; a◌゙◌्◌্◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, BENGALI SIGN VIRAMA, LATIN SMALL LETTER B +0061 09CD 05B0 094D 3099 0062;0061 3099 09CD 094D 05B0 0062;0061 3099 09CD 094D 05B0 0062;0061 3099 09CD 094D 05B0 0062;0061 3099 09CD 094D 05B0 0062; # (a◌্◌ְ◌्◌゙b; a◌゙◌্◌्◌ְb; a◌゙◌্◌्◌ְb; a◌゙◌্◌्◌ְb; a◌゙◌্◌्◌ְb; ) LATIN SMALL LETTER A, BENGALI SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 3099 093C 0334 0A3C 0062;0061 0334 093C 0A3C 3099 0062;0061 0334 093C 0A3C 3099 0062;0061 0334 093C 0A3C 3099 0062;0061 0334 093C 0A3C 3099 0062; # (a◌゙◌़◌̴◌਼b; a◌̴◌़◌਼◌゙b; a◌̴◌़◌਼◌゙b; a◌̴◌़◌਼◌゙b; a◌̴◌़◌਼◌゙b; ) LATIN SMALL LETTER A, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, GURMUKHI SIGN NUKTA, LATIN SMALL LETTER B +0061 0A3C 3099 093C 0334 0062;0061 0334 0A3C 093C 3099 0062;0061 0334 0A3C 093C 3099 0062;0061 0334 0A3C 093C 3099 0062;0061 0334 0A3C 093C 3099 0062; # (a◌਼◌゙◌़◌̴b; a◌̴◌਼◌़◌゙b; a◌̴◌਼◌़◌゙b; a◌̴◌਼◌़◌゙b; a◌̴◌਼◌़◌゙b; ) LATIN SMALL LETTER A, GURMUKHI SIGN NUKTA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 05B0 094D 3099 0A4D 0062;0061 3099 094D 0A4D 05B0 0062;0061 3099 094D 0A4D 05B0 0062;0061 3099 094D 0A4D 05B0 0062;0061 3099 094D 0A4D 05B0 0062; # (a◌ְ◌्◌゙◌੍b; a◌゙◌्◌੍◌ְb; a◌゙◌्◌੍◌ְb; a◌゙◌्◌੍◌ְb; a◌゙◌्◌੍◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, GURMUKHI SIGN VIRAMA, LATIN SMALL LETTER B +0061 0A4D 05B0 094D 3099 0062;0061 3099 0A4D 094D 05B0 0062;0061 3099 0A4D 094D 05B0 0062;0061 3099 0A4D 094D 05B0 0062;0061 3099 0A4D 094D 05B0 0062; # (a◌੍◌ְ◌्◌゙b; a◌゙◌੍◌्◌ְb; a◌゙◌੍◌्◌ְb; a◌゙◌੍◌्◌ְb; a◌゙◌੍◌्◌ְb; ) LATIN SMALL LETTER A, GURMUKHI SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 3099 093C 0334 0ABC 0062;0061 0334 093C 0ABC 3099 0062;0061 0334 093C 0ABC 3099 0062;0061 0334 093C 0ABC 3099 0062;0061 0334 093C 0ABC 3099 0062; # (a◌゙◌़◌̴◌઼b; a◌̴◌़◌઼◌゙b; a◌̴◌़◌઼◌゙b; a◌̴◌़◌઼◌゙b; a◌̴◌़◌઼◌゙b; ) LATIN SMALL LETTER A, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, GUJARATI SIGN NUKTA, LATIN SMALL LETTER B +0061 0ABC 3099 093C 0334 0062;0061 0334 0ABC 093C 3099 0062;0061 0334 0ABC 093C 3099 0062;0061 0334 0ABC 093C 3099 0062;0061 0334 0ABC 093C 3099 0062; # (a◌઼◌゙◌़◌̴b; a◌̴◌઼◌़◌゙b; a◌̴◌઼◌़◌゙b; a◌̴◌઼◌़◌゙b; a◌̴◌઼◌़◌゙b; ) LATIN SMALL LETTER A, GUJARATI SIGN NUKTA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 05B0 094D 3099 0ACD 0062;0061 3099 094D 0ACD 05B0 0062;0061 3099 094D 0ACD 05B0 0062;0061 3099 094D 0ACD 05B0 0062;0061 3099 094D 0ACD 05B0 0062; # (a◌ְ◌्◌゙◌્b; a◌゙◌्◌્◌ְb; a◌゙◌्◌્◌ְb; a◌゙◌्◌્◌ְb; a◌゙◌्◌્◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, GUJARATI SIGN VIRAMA, LATIN SMALL LETTER B +0061 0ACD 05B0 094D 3099 0062;0061 3099 0ACD 094D 05B0 0062;0061 3099 0ACD 094D 05B0 0062;0061 3099 0ACD 094D 05B0 0062;0061 3099 0ACD 094D 05B0 0062; # (a◌્◌ְ◌्◌゙b; a◌゙◌્◌्◌ְb; a◌゙◌્◌्◌ְb; a◌゙◌્◌्◌ְb; a◌゙◌્◌्◌ְb; ) LATIN SMALL LETTER A, GUJARATI SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 3099 093C 0334 0B3C 0062;0061 0334 093C 0B3C 3099 0062;0061 0334 093C 0B3C 3099 0062;0061 0334 093C 0B3C 3099 0062;0061 0334 093C 0B3C 3099 0062; # (a◌゙◌़◌̴◌଼b; a◌̴◌़◌଼◌゙b; a◌̴◌़◌଼◌゙b; a◌̴◌़◌଼◌゙b; a◌̴◌़◌଼◌゙b; ) LATIN SMALL LETTER A, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, ORIYA SIGN NUKTA, LATIN SMALL LETTER B +0061 0B3C 3099 093C 0334 0062;0061 0334 0B3C 093C 3099 0062;0061 0334 0B3C 093C 3099 0062;0061 0334 0B3C 093C 3099 0062;0061 0334 0B3C 093C 3099 0062; # (a◌଼◌゙◌़◌̴b; a◌̴◌଼◌़◌゙b; a◌̴◌଼◌़◌゙b; a◌̴◌଼◌़◌゙b; a◌̴◌଼◌़◌゙b; ) LATIN SMALL LETTER A, ORIYA SIGN NUKTA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 05B0 094D 3099 0B4D 0062;0061 3099 094D 0B4D 05B0 0062;0061 3099 094D 0B4D 05B0 0062;0061 3099 094D 0B4D 05B0 0062;0061 3099 094D 0B4D 05B0 0062; # (a◌ְ◌्◌゙◌୍b; a◌゙◌्◌୍◌ְb; a◌゙◌्◌୍◌ְb; a◌゙◌्◌୍◌ְb; a◌゙◌्◌୍◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, ORIYA SIGN VIRAMA, LATIN SMALL LETTER B +0061 0B4D 05B0 094D 3099 0062;0061 3099 0B4D 094D 05B0 0062;0061 3099 0B4D 094D 05B0 0062;0061 3099 0B4D 094D 05B0 0062;0061 3099 0B4D 094D 05B0 0062; # (a◌୍◌ְ◌्◌゙b; a◌゙◌୍◌्◌ְb; a◌゙◌୍◌्◌ְb; a◌゙◌୍◌्◌ְb; a◌゙◌୍◌्◌ְb; ) LATIN SMALL LETTER A, ORIYA SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 05B0 094D 3099 0BCD 0062;0061 3099 094D 0BCD 05B0 0062;0061 3099 094D 0BCD 05B0 0062;0061 3099 094D 0BCD 05B0 0062;0061 3099 094D 0BCD 05B0 0062; # (a◌ְ◌्◌゙◌்b; a◌゙◌्◌்◌ְb; a◌゙◌्◌்◌ְb; a◌゙◌्◌்◌ְb; a◌゙◌्◌்◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, TAMIL SIGN VIRAMA, LATIN SMALL LETTER B +0061 0BCD 05B0 094D 3099 0062;0061 3099 0BCD 094D 05B0 0062;0061 3099 0BCD 094D 05B0 0062;0061 3099 0BCD 094D 05B0 0062;0061 3099 0BCD 094D 05B0 0062; # (a◌்◌ְ◌्◌゙b; a◌゙◌்◌्◌ְb; a◌゙◌்◌्◌ְb; a◌゙◌்◌्◌ְb; a◌゙◌்◌्◌ְb; ) LATIN SMALL LETTER A, TAMIL SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 05B0 094D 3099 0C4D 0062;0061 3099 094D 0C4D 05B0 0062;0061 3099 094D 0C4D 05B0 0062;0061 3099 094D 0C4D 05B0 0062;0061 3099 094D 0C4D 05B0 0062; # (a◌ְ◌्◌゙◌్b; a◌゙◌्◌్◌ְb; a◌゙◌्◌్◌ְb; a◌゙◌्◌్◌ְb; a◌゙◌्◌్◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, TELUGU SIGN VIRAMA, LATIN SMALL LETTER B +0061 0C4D 05B0 094D 3099 0062;0061 3099 0C4D 094D 05B0 0062;0061 3099 0C4D 094D 05B0 0062;0061 3099 0C4D 094D 05B0 0062;0061 3099 0C4D 094D 05B0 0062; # (a◌్◌ְ◌्◌゙b; a◌゙◌్◌्◌ְb; a◌゙◌్◌्◌ְb; a◌゙◌్◌्◌ְb; a◌゙◌్◌्◌ְb; ) LATIN SMALL LETTER A, TELUGU SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 0C56 0C55 0711 0C55 0062;0061 0711 0C55 0C55 0C56 0062;0061 0711 0C55 0C55 0C56 0062;0061 0711 0C55 0C55 0C56 0062;0061 0711 0C55 0C55 0C56 0062; # (a◌ౖ◌ౕ◌ܑ◌ౕb; a◌ܑ◌ౕ◌ౕ◌ౖb; a◌ܑ◌ౕ◌ౕ◌ౖb; a◌ܑ◌ౕ◌ౕ◌ౖb; a◌ܑ◌ౕ◌ౕ◌ౖb; ) LATIN SMALL LETTER A, TELUGU AI LENGTH MARK, TELUGU LENGTH MARK, SYRIAC LETTER SUPERSCRIPT ALAPH, TELUGU LENGTH MARK, LATIN SMALL LETTER B +0061 0C55 0C56 0C55 0711 0062;0061 0711 0C55 0C55 0C56 0062;0061 0711 0C55 0C55 0C56 0062;0061 0711 0C55 0C55 0C56 0062;0061 0711 0C55 0C55 0C56 0062; # (a◌ౕ◌ౖ◌ౕ◌ܑb; a◌ܑ◌ౕ◌ౕ◌ౖb; a◌ܑ◌ౕ◌ౕ◌ౖb; a◌ܑ◌ౕ◌ౕ◌ౖb; a◌ܑ◌ౕ◌ౕ◌ౖb; ) LATIN SMALL LETTER A, TELUGU LENGTH MARK, TELUGU AI LENGTH MARK, TELUGU LENGTH MARK, SYRIAC LETTER SUPERSCRIPT ALAPH, LATIN SMALL LETTER B +0061 0E38 0C56 0C55 0C56 0062;0061 0C55 0C56 0C56 0E38 0062;0061 0C55 0C56 0C56 0E38 0062;0061 0C55 0C56 0C56 0E38 0062;0061 0C55 0C56 0C56 0E38 0062; # (a◌ุ◌ౖ◌ౕ◌ౖb; a◌ౕ◌ౖ◌ౖ◌ุb; a◌ౕ◌ౖ◌ౖ◌ุb; a◌ౕ◌ౖ◌ౖ◌ุb; a◌ౕ◌ౖ◌ౖ◌ุb; ) LATIN SMALL LETTER A, THAI CHARACTER SARA U, TELUGU AI LENGTH MARK, TELUGU LENGTH MARK, TELUGU AI LENGTH MARK, LATIN SMALL LETTER B +0061 0C56 0E38 0C56 0C55 0062;0061 0C55 0C56 0C56 0E38 0062;0061 0C55 0C56 0C56 0E38 0062;0061 0C55 0C56 0C56 0E38 0062;0061 0C55 0C56 0C56 0E38 0062; # (a◌ౖ◌ุ◌ౖ◌ౕb; a◌ౕ◌ౖ◌ౖ◌ุb; a◌ౕ◌ౖ◌ౖ◌ุb; a◌ౕ◌ౖ◌ౖ◌ุb; a◌ౕ◌ౖ◌ౖ◌ุb; ) LATIN SMALL LETTER A, TELUGU AI LENGTH MARK, THAI CHARACTER SARA U, TELUGU AI LENGTH MARK, TELUGU LENGTH MARK, LATIN SMALL LETTER B +0061 05B0 094D 3099 0CCD 0062;0061 3099 094D 0CCD 05B0 0062;0061 3099 094D 0CCD 05B0 0062;0061 3099 094D 0CCD 05B0 0062;0061 3099 094D 0CCD 05B0 0062; # (a◌ְ◌्◌゙◌್b; a◌゙◌्◌್◌ְb; a◌゙◌्◌್◌ְb; a◌゙◌्◌್◌ְb; a◌゙◌्◌್◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, KANNADA SIGN VIRAMA, LATIN SMALL LETTER B +0061 0CCD 05B0 094D 3099 0062;0061 3099 0CCD 094D 05B0 0062;0061 3099 0CCD 094D 05B0 0062;0061 3099 0CCD 094D 05B0 0062;0061 3099 0CCD 094D 05B0 0062; # (a◌್◌ְ◌्◌゙b; a◌゙◌್◌्◌ְb; a◌゙◌್◌्◌ְb; a◌゙◌್◌्◌ְb; a◌゙◌್◌्◌ְb; ) LATIN SMALL LETTER A, KANNADA SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 05B0 094D 3099 0D4D 0062;0061 3099 094D 0D4D 05B0 0062;0061 3099 094D 0D4D 05B0 0062;0061 3099 094D 0D4D 05B0 0062;0061 3099 094D 0D4D 05B0 0062; # (a◌ְ◌्◌゙◌്b; a◌゙◌्◌്◌ְb; a◌゙◌्◌്◌ְb; a◌゙◌्◌്◌ְb; a◌゙◌्◌്◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, MALAYALAM SIGN VIRAMA, LATIN SMALL LETTER B +0061 0D4D 05B0 094D 3099 0062;0061 3099 0D4D 094D 05B0 0062;0061 3099 0D4D 094D 05B0 0062;0061 3099 0D4D 094D 05B0 0062;0061 3099 0D4D 094D 05B0 0062; # (a◌്◌ְ◌्◌゙b; a◌゙◌്◌्◌ְb; a◌゙◌്◌्◌ְb; a◌゙◌്◌्◌ְb; a◌゙◌്◌्◌ְb; ) LATIN SMALL LETTER A, MALAYALAM SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 05B0 094D 3099 0DCA 0062;0061 3099 094D 0DCA 05B0 0062;0061 3099 094D 0DCA 05B0 0062;0061 3099 094D 0DCA 05B0 0062;0061 3099 094D 0DCA 05B0 0062; # (a◌ְ◌्◌゙◌්b; a◌゙◌्◌්◌ְb; a◌゙◌्◌්◌ְb; a◌゙◌्◌්◌ְb; a◌゙◌्◌්◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, SINHALA SIGN AL-LAKUNA, LATIN SMALL LETTER B +0061 0DCA 05B0 094D 3099 0062;0061 3099 0DCA 094D 05B0 0062;0061 3099 0DCA 094D 05B0 0062;0061 3099 0DCA 094D 05B0 0062;0061 3099 0DCA 094D 05B0 0062; # (a◌්◌ְ◌्◌゙b; a◌゙◌්◌्◌ְb; a◌゙◌්◌्◌ְb; a◌゙◌්◌्◌ְb; a◌゙◌්◌्◌ְb; ) LATIN SMALL LETTER A, SINHALA SIGN AL-LAKUNA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 0E48 0E38 0C56 0E38 0062;0061 0C56 0E38 0E38 0E48 0062;0061 0C56 0E38 0E38 0E48 0062;0061 0C56 0E38 0E38 0E48 0062;0061 0C56 0E38 0E38 0E48 0062; # (a◌่◌ุ◌ౖ◌ุb; a◌ౖ◌ุ◌ุ◌่b; a◌ౖ◌ุ◌ุ◌่b; a◌ౖ◌ุ◌ุ◌่b; a◌ౖ◌ุ◌ุ◌่b; ) LATIN SMALL LETTER A, THAI CHARACTER MAI EK, THAI CHARACTER SARA U, TELUGU AI LENGTH MARK, THAI CHARACTER SARA U, LATIN SMALL LETTER B +0061 0E38 0E48 0E38 0C56 0062;0061 0C56 0E38 0E38 0E48 0062;0061 0C56 0E38 0E38 0E48 0062;0061 0C56 0E38 0E38 0E48 0062;0061 0C56 0E38 0E38 0E48 0062; # (a◌ุ◌่◌ุ◌ౖb; a◌ౖ◌ุ◌ุ◌่b; a◌ౖ◌ุ◌ุ◌่b; a◌ౖ◌ุ◌ุ◌่b; a◌ౖ◌ุ◌ุ◌่b; ) LATIN SMALL LETTER A, THAI CHARACTER SARA U, THAI CHARACTER MAI EK, THAI CHARACTER SARA U, TELUGU AI LENGTH MARK, LATIN SMALL LETTER B +0061 0E48 0E38 0C56 0E39 0062;0061 0C56 0E38 0E39 0E48 0062;0061 0C56 0E38 0E39 0E48 0062;0061 0C56 0E38 0E39 0E48 0062;0061 0C56 0E38 0E39 0E48 0062; # (a◌่◌ุ◌ౖ◌ูb; a◌ౖ◌ุ◌ู◌่b; a◌ౖ◌ุ◌ู◌่b; a◌ౖ◌ุ◌ู◌่b; a◌ౖ◌ุ◌ู◌่b; ) LATIN SMALL LETTER A, THAI CHARACTER MAI EK, THAI CHARACTER SARA U, TELUGU AI LENGTH MARK, THAI CHARACTER SARA UU, LATIN SMALL LETTER B +0061 0E39 0E48 0E38 0C56 0062;0061 0C56 0E39 0E38 0E48 0062;0061 0C56 0E39 0E38 0E48 0062;0061 0C56 0E39 0E38 0E48 0062;0061 0C56 0E39 0E38 0E48 0062; # (a◌ู◌่◌ุ◌ౖb; a◌ౖ◌ู◌ุ◌่b; a◌ౖ◌ู◌ุ◌่b; a◌ౖ◌ู◌ุ◌่b; a◌ౖ◌ู◌ุ◌่b; ) LATIN SMALL LETTER A, THAI CHARACTER SARA UU, THAI CHARACTER MAI EK, THAI CHARACTER SARA U, TELUGU AI LENGTH MARK, LATIN SMALL LETTER B +0061 05B0 094D 3099 0E3A 0062;0061 3099 094D 0E3A 05B0 0062;0061 3099 094D 0E3A 05B0 0062;0061 3099 094D 0E3A 05B0 0062;0061 3099 094D 0E3A 05B0 0062; # (a◌ְ◌्◌゙◌ฺb; a◌゙◌्◌ฺ◌ְb; a◌゙◌्◌ฺ◌ְb; a◌゙◌्◌ฺ◌ְb; a◌゙◌्◌ฺ◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, THAI CHARACTER PHINTHU, LATIN SMALL LETTER B +0061 0E3A 05B0 094D 3099 0062;0061 3099 0E3A 094D 05B0 0062;0061 3099 0E3A 094D 05B0 0062;0061 3099 0E3A 094D 05B0 0062;0061 3099 0E3A 094D 05B0 0062; # (a◌ฺ◌ְ◌्◌゙b; a◌゙◌ฺ◌्◌ְb; a◌゙◌ฺ◌्◌ְb; a◌゙◌ฺ◌्◌ְb; a◌゙◌ฺ◌्◌ְb; ) LATIN SMALL LETTER A, THAI CHARACTER PHINTHU, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 0EB8 0E48 0E38 0E48 0062;0061 0E38 0E48 0E48 0EB8 0062;0061 0E38 0E48 0E48 0EB8 0062;0061 0E38 0E48 0E48 0EB8 0062;0061 0E38 0E48 0E48 0EB8 0062; # (a◌ຸ◌่◌ุ◌่b; a◌ุ◌่◌่◌ຸb; a◌ุ◌่◌่◌ຸb; a◌ุ◌่◌่◌ຸb; a◌ุ◌่◌่◌ຸb; ) LATIN SMALL LETTER A, LAO VOWEL SIGN U, THAI CHARACTER MAI EK, THAI CHARACTER SARA U, THAI CHARACTER MAI EK, LATIN SMALL LETTER B +0061 0E48 0EB8 0E48 0E38 0062;0061 0E38 0E48 0E48 0EB8 0062;0061 0E38 0E48 0E48 0EB8 0062;0061 0E38 0E48 0E48 0EB8 0062;0061 0E38 0E48 0E48 0EB8 0062; # (a◌่◌ຸ◌่◌ุb; a◌ุ◌่◌่◌ຸb; a◌ุ◌่◌่◌ຸb; a◌ุ◌่◌่◌ຸb; a◌ุ◌่◌่◌ຸb; ) LATIN SMALL LETTER A, THAI CHARACTER MAI EK, LAO VOWEL SIGN U, THAI CHARACTER MAI EK, THAI CHARACTER SARA U, LATIN SMALL LETTER B +0061 0EB8 0E48 0E38 0E49 0062;0061 0E38 0E48 0E49 0EB8 0062;0061 0E38 0E48 0E49 0EB8 0062;0061 0E38 0E48 0E49 0EB8 0062;0061 0E38 0E48 0E49 0EB8 0062; # (a◌ຸ◌่◌ุ◌้b; a◌ุ◌่◌้◌ຸb; a◌ุ◌่◌้◌ຸb; a◌ุ◌่◌้◌ຸb; a◌ุ◌่◌้◌ຸb; ) LATIN SMALL LETTER A, LAO VOWEL SIGN U, THAI CHARACTER MAI EK, THAI CHARACTER SARA U, THAI CHARACTER MAI THO, LATIN SMALL LETTER B +0061 0E49 0EB8 0E48 0E38 0062;0061 0E38 0E49 0E48 0EB8 0062;0061 0E38 0E49 0E48 0EB8 0062;0061 0E38 0E49 0E48 0EB8 0062;0061 0E38 0E49 0E48 0EB8 0062; # (a◌้◌ຸ◌่◌ุb; a◌ุ◌้◌่◌ຸb; a◌ุ◌้◌่◌ຸb; a◌ุ◌้◌่◌ຸb; a◌ุ◌้◌่◌ຸb; ) LATIN SMALL LETTER A, THAI CHARACTER MAI THO, LAO VOWEL SIGN U, THAI CHARACTER MAI EK, THAI CHARACTER SARA U, LATIN SMALL LETTER B +0061 0EB8 0E48 0E38 0E4A 0062;0061 0E38 0E48 0E4A 0EB8 0062;0061 0E38 0E48 0E4A 0EB8 0062;0061 0E38 0E48 0E4A 0EB8 0062;0061 0E38 0E48 0E4A 0EB8 0062; # (a◌ຸ◌่◌ุ◌๊b; a◌ุ◌่◌๊◌ຸb; a◌ุ◌่◌๊◌ຸb; a◌ุ◌่◌๊◌ຸb; a◌ุ◌่◌๊◌ຸb; ) LATIN SMALL LETTER A, LAO VOWEL SIGN U, THAI CHARACTER MAI EK, THAI CHARACTER SARA U, THAI CHARACTER MAI TRI, LATIN SMALL LETTER B +0061 0E4A 0EB8 0E48 0E38 0062;0061 0E38 0E4A 0E48 0EB8 0062;0061 0E38 0E4A 0E48 0EB8 0062;0061 0E38 0E4A 0E48 0EB8 0062;0061 0E38 0E4A 0E48 0EB8 0062; # (a◌๊◌ຸ◌่◌ุb; a◌ุ◌๊◌่◌ຸb; a◌ุ◌๊◌่◌ຸb; a◌ุ◌๊◌่◌ຸb; a◌ุ◌๊◌่◌ຸb; ) LATIN SMALL LETTER A, THAI CHARACTER MAI TRI, LAO VOWEL SIGN U, THAI CHARACTER MAI EK, THAI CHARACTER SARA U, LATIN SMALL LETTER B +0061 0EB8 0E48 0E38 0E4B 0062;0061 0E38 0E48 0E4B 0EB8 0062;0061 0E38 0E48 0E4B 0EB8 0062;0061 0E38 0E48 0E4B 0EB8 0062;0061 0E38 0E48 0E4B 0EB8 0062; # (a◌ຸ◌่◌ุ◌๋b; a◌ุ◌่◌๋◌ຸb; a◌ุ◌่◌๋◌ຸb; a◌ุ◌่◌๋◌ຸb; a◌ุ◌่◌๋◌ຸb; ) LATIN SMALL LETTER A, LAO VOWEL SIGN U, THAI CHARACTER MAI EK, THAI CHARACTER SARA U, THAI CHARACTER MAI CHATTAWA, LATIN SMALL LETTER B +0061 0E4B 0EB8 0E48 0E38 0062;0061 0E38 0E4B 0E48 0EB8 0062;0061 0E38 0E4B 0E48 0EB8 0062;0061 0E38 0E4B 0E48 0EB8 0062;0061 0E38 0E4B 0E48 0EB8 0062; # (a◌๋◌ຸ◌่◌ุb; a◌ุ◌๋◌่◌ຸb; a◌ุ◌๋◌่◌ຸb; a◌ุ◌๋◌่◌ຸb; a◌ุ◌๋◌่◌ຸb; ) LATIN SMALL LETTER A, THAI CHARACTER MAI CHATTAWA, LAO VOWEL SIGN U, THAI CHARACTER MAI EK, THAI CHARACTER SARA U, LATIN SMALL LETTER B +0061 0EC8 0EB8 0E48 0EB8 0062;0061 0E48 0EB8 0EB8 0EC8 0062;0061 0E48 0EB8 0EB8 0EC8 0062;0061 0E48 0EB8 0EB8 0EC8 0062;0061 0E48 0EB8 0EB8 0EC8 0062; # (a◌່◌ຸ◌่◌ຸb; a◌่◌ຸ◌ຸ◌່b; a◌่◌ຸ◌ຸ◌່b; a◌่◌ຸ◌ຸ◌່b; a◌่◌ຸ◌ຸ◌່b; ) LATIN SMALL LETTER A, LAO TONE MAI EK, LAO VOWEL SIGN U, THAI CHARACTER MAI EK, LAO VOWEL SIGN U, LATIN SMALL LETTER B +0061 0EB8 0EC8 0EB8 0E48 0062;0061 0E48 0EB8 0EB8 0EC8 0062;0061 0E48 0EB8 0EB8 0EC8 0062;0061 0E48 0EB8 0EB8 0EC8 0062;0061 0E48 0EB8 0EB8 0EC8 0062; # (a◌ຸ◌່◌ຸ◌่b; a◌่◌ຸ◌ຸ◌່b; a◌่◌ຸ◌ຸ◌່b; a◌่◌ຸ◌ຸ◌່b; a◌่◌ຸ◌ຸ◌່b; ) LATIN SMALL LETTER A, LAO VOWEL SIGN U, LAO TONE MAI EK, LAO VOWEL SIGN U, THAI CHARACTER MAI EK, LATIN SMALL LETTER B +0061 0EC8 0EB8 0E48 0EB9 0062;0061 0E48 0EB8 0EB9 0EC8 0062;0061 0E48 0EB8 0EB9 0EC8 0062;0061 0E48 0EB8 0EB9 0EC8 0062;0061 0E48 0EB8 0EB9 0EC8 0062; # (a◌່◌ຸ◌่◌ູb; a◌่◌ຸ◌ູ◌່b; a◌่◌ຸ◌ູ◌່b; a◌่◌ຸ◌ູ◌່b; a◌่◌ຸ◌ູ◌່b; ) LATIN SMALL LETTER A, LAO TONE MAI EK, LAO VOWEL SIGN U, THAI CHARACTER MAI EK, LAO VOWEL SIGN UU, LATIN SMALL LETTER B +0061 0EB9 0EC8 0EB8 0E48 0062;0061 0E48 0EB9 0EB8 0EC8 0062;0061 0E48 0EB9 0EB8 0EC8 0062;0061 0E48 0EB9 0EB8 0EC8 0062;0061 0E48 0EB9 0EB8 0EC8 0062; # (a◌ູ◌່◌ຸ◌่b; a◌่◌ູ◌ຸ◌່b; a◌่◌ູ◌ຸ◌່b; a◌่◌ູ◌ຸ◌່b; a◌่◌ູ◌ຸ◌່b; ) LATIN SMALL LETTER A, LAO VOWEL SIGN UU, LAO TONE MAI EK, LAO VOWEL SIGN U, THAI CHARACTER MAI EK, LATIN SMALL LETTER B +0061 0F71 0EC8 0EB8 0EC8 0062;0061 0EB8 0EC8 0EC8 0F71 0062;0061 0EB8 0EC8 0EC8 0F71 0062;0061 0EB8 0EC8 0EC8 0F71 0062;0061 0EB8 0EC8 0EC8 0F71 0062; # (a◌ཱ◌່◌ຸ◌່b; a◌ຸ◌່◌່◌ཱb; a◌ຸ◌່◌່◌ཱb; a◌ຸ◌່◌່◌ཱb; a◌ຸ◌່◌່◌ཱb; ) LATIN SMALL LETTER A, TIBETAN VOWEL SIGN AA, LAO TONE MAI EK, LAO VOWEL SIGN U, LAO TONE MAI EK, LATIN SMALL LETTER B +0061 0EC8 0F71 0EC8 0EB8 0062;0061 0EB8 0EC8 0EC8 0F71 0062;0061 0EB8 0EC8 0EC8 0F71 0062;0061 0EB8 0EC8 0EC8 0F71 0062;0061 0EB8 0EC8 0EC8 0F71 0062; # (a◌່◌ཱ◌່◌ຸb; a◌ຸ◌່◌່◌ཱb; a◌ຸ◌່◌່◌ཱb; a◌ຸ◌່◌່◌ཱb; a◌ຸ◌່◌່◌ཱb; ) LATIN SMALL LETTER A, LAO TONE MAI EK, TIBETAN VOWEL SIGN AA, LAO TONE MAI EK, LAO VOWEL SIGN U, LATIN SMALL LETTER B +0061 0F71 0EC8 0EB8 0EC9 0062;0061 0EB8 0EC8 0EC9 0F71 0062;0061 0EB8 0EC8 0EC9 0F71 0062;0061 0EB8 0EC8 0EC9 0F71 0062;0061 0EB8 0EC8 0EC9 0F71 0062; # (a◌ཱ◌່◌ຸ◌້b; a◌ຸ◌່◌້◌ཱb; a◌ຸ◌່◌້◌ཱb; a◌ຸ◌່◌້◌ཱb; a◌ຸ◌່◌້◌ཱb; ) LATIN SMALL LETTER A, TIBETAN VOWEL SIGN AA, LAO TONE MAI EK, LAO VOWEL SIGN U, LAO TONE MAI THO, LATIN SMALL LETTER B +0061 0EC9 0F71 0EC8 0EB8 0062;0061 0EB8 0EC9 0EC8 0F71 0062;0061 0EB8 0EC9 0EC8 0F71 0062;0061 0EB8 0EC9 0EC8 0F71 0062;0061 0EB8 0EC9 0EC8 0F71 0062; # (a◌້◌ཱ◌່◌ຸb; a◌ຸ◌້◌່◌ཱb; a◌ຸ◌້◌່◌ཱb; a◌ຸ◌້◌່◌ཱb; a◌ຸ◌້◌່◌ཱb; ) LATIN SMALL LETTER A, LAO TONE MAI THO, TIBETAN VOWEL SIGN AA, LAO TONE MAI EK, LAO VOWEL SIGN U, LATIN SMALL LETTER B +0061 0F71 0EC8 0EB8 0ECA 0062;0061 0EB8 0EC8 0ECA 0F71 0062;0061 0EB8 0EC8 0ECA 0F71 0062;0061 0EB8 0EC8 0ECA 0F71 0062;0061 0EB8 0EC8 0ECA 0F71 0062; # (a◌ཱ◌່◌ຸ◌໊b; a◌ຸ◌່◌໊◌ཱb; a◌ຸ◌່◌໊◌ཱb; a◌ຸ◌່◌໊◌ཱb; a◌ຸ◌່◌໊◌ཱb; ) LATIN SMALL LETTER A, TIBETAN VOWEL SIGN AA, LAO TONE MAI EK, LAO VOWEL SIGN U, LAO TONE MAI TI, LATIN SMALL LETTER B +0061 0ECA 0F71 0EC8 0EB8 0062;0061 0EB8 0ECA 0EC8 0F71 0062;0061 0EB8 0ECA 0EC8 0F71 0062;0061 0EB8 0ECA 0EC8 0F71 0062;0061 0EB8 0ECA 0EC8 0F71 0062; # (a◌໊◌ཱ◌່◌ຸb; a◌ຸ◌໊◌່◌ཱb; a◌ຸ◌໊◌່◌ཱb; a◌ຸ◌໊◌່◌ཱb; a◌ຸ◌໊◌່◌ཱb; ) LATIN SMALL LETTER A, LAO TONE MAI TI, TIBETAN VOWEL SIGN AA, LAO TONE MAI EK, LAO VOWEL SIGN U, LATIN SMALL LETTER B +0061 0F71 0EC8 0EB8 0ECB 0062;0061 0EB8 0EC8 0ECB 0F71 0062;0061 0EB8 0EC8 0ECB 0F71 0062;0061 0EB8 0EC8 0ECB 0F71 0062;0061 0EB8 0EC8 0ECB 0F71 0062; # (a◌ཱ◌່◌ຸ◌໋b; a◌ຸ◌່◌໋◌ཱb; a◌ຸ◌່◌໋◌ཱb; a◌ຸ◌່◌໋◌ཱb; a◌ຸ◌່◌໋◌ཱb; ) LATIN SMALL LETTER A, TIBETAN VOWEL SIGN AA, LAO TONE MAI EK, LAO VOWEL SIGN U, LAO TONE MAI CATAWA, LATIN SMALL LETTER B +0061 0ECB 0F71 0EC8 0EB8 0062;0061 0EB8 0ECB 0EC8 0F71 0062;0061 0EB8 0ECB 0EC8 0F71 0062;0061 0EB8 0ECB 0EC8 0F71 0062;0061 0EB8 0ECB 0EC8 0F71 0062; # (a◌໋◌ཱ◌່◌ຸb; a◌ຸ◌໋◌່◌ཱb; a◌ຸ◌໋◌່◌ཱb; a◌ຸ◌໋◌່◌ཱb; a◌ຸ◌໋◌່◌ཱb; ) LATIN SMALL LETTER A, LAO TONE MAI CATAWA, TIBETAN VOWEL SIGN AA, LAO TONE MAI EK, LAO VOWEL SIGN U, LATIN SMALL LETTER B +0061 059A 0316 302A 0F18 0062;0061 302A 0316 0F18 059A 0062;0061 302A 0316 0F18 059A 0062;0061 302A 0316 0F18 059A 0062;0061 302A 0316 0F18 059A 0062; # (a◌֚◌̖◌〪◌༘b; a◌〪◌̖◌༘◌֚b; a◌〪◌̖◌༘◌֚b; a◌〪◌̖◌༘◌֚b; a◌〪◌̖◌༘◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, TIBETAN ASTROLOGICAL SIGN -KHYUD PA, LATIN SMALL LETTER B +0061 0F18 059A 0316 302A 0062;0061 302A 0F18 0316 059A 0062;0061 302A 0F18 0316 059A 0062;0061 302A 0F18 0316 059A 0062;0061 302A 0F18 0316 059A 0062; # (a◌༘◌֚◌̖◌〪b; a◌〪◌༘◌̖◌֚b; a◌〪◌༘◌̖◌֚b; a◌〪◌༘◌̖◌֚b; a◌〪◌༘◌̖◌֚b; ) LATIN SMALL LETTER A, TIBETAN ASTROLOGICAL SIGN -KHYUD PA, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 0F19 0062;0061 302A 0316 0F19 059A 0062;0061 302A 0316 0F19 059A 0062;0061 302A 0316 0F19 059A 0062;0061 302A 0316 0F19 059A 0062; # (a◌֚◌̖◌〪◌༙b; a◌〪◌̖◌༙◌֚b; a◌〪◌̖◌༙◌֚b; a◌〪◌̖◌༙◌֚b; a◌〪◌̖◌༙◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS, LATIN SMALL LETTER B +0061 0F19 059A 0316 302A 0062;0061 302A 0F19 0316 059A 0062;0061 302A 0F19 0316 059A 0062;0061 302A 0F19 0316 059A 0062;0061 302A 0F19 0316 059A 0062; # (a◌༙◌֚◌̖◌〪b; a◌〪◌༙◌̖◌֚b; a◌〪◌༙◌̖◌֚b; a◌〪◌༙◌̖◌֚b; a◌〪◌༙◌̖◌֚b; ) LATIN SMALL LETTER A, TIBETAN ASTROLOGICAL SIGN SDONG TSHUGS, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 0F35 0062;0061 302A 0316 0F35 059A 0062;0061 302A 0316 0F35 059A 0062;0061 302A 0316 0F35 059A 0062;0061 302A 0316 0F35 059A 0062; # (a◌֚◌̖◌〪◌༵b; a◌〪◌̖◌༵◌֚b; a◌〪◌̖◌༵◌֚b; a◌〪◌̖◌༵◌֚b; a◌〪◌̖◌༵◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, TIBETAN MARK NGAS BZUNG NYI ZLA, LATIN SMALL LETTER B +0061 0F35 059A 0316 302A 0062;0061 302A 0F35 0316 059A 0062;0061 302A 0F35 0316 059A 0062;0061 302A 0F35 0316 059A 0062;0061 302A 0F35 0316 059A 0062; # (a◌༵◌֚◌̖◌〪b; a◌〪◌༵◌̖◌֚b; a◌〪◌༵◌̖◌֚b; a◌〪◌༵◌̖◌֚b; a◌〪◌༵◌̖◌֚b; ) LATIN SMALL LETTER A, TIBETAN MARK NGAS BZUNG NYI ZLA, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 0F37 0062;0061 302A 0316 0F37 059A 0062;0061 302A 0316 0F37 059A 0062;0061 302A 0316 0F37 059A 0062;0061 302A 0316 0F37 059A 0062; # (a◌֚◌̖◌〪◌༷b; a◌〪◌̖◌༷◌֚b; a◌〪◌̖◌༷◌֚b; a◌〪◌̖◌༷◌֚b; a◌〪◌̖◌༷◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, TIBETAN MARK NGAS BZUNG SGOR RTAGS, LATIN SMALL LETTER B +0061 0F37 059A 0316 302A 0062;0061 302A 0F37 0316 059A 0062;0061 302A 0F37 0316 059A 0062;0061 302A 0F37 0316 059A 0062;0061 302A 0F37 0316 059A 0062; # (a◌༷◌֚◌̖◌〪b; a◌〪◌༷◌̖◌֚b; a◌〪◌༷◌̖◌֚b; a◌〪◌༷◌̖◌֚b; a◌〪◌༷◌̖◌֚b; ) LATIN SMALL LETTER A, TIBETAN MARK NGAS BZUNG SGOR RTAGS, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 302A 031B 0321 0F39 0062;0061 0321 031B 0F39 302A 0062;0061 0321 031B 0F39 302A 0062;0061 0321 031B 0F39 302A 0062;0061 0321 031B 0F39 302A 0062; # (a◌〪◌̛◌̡◌༹b; a◌̡◌̛◌༹◌〪b; a◌̡◌̛◌༹◌〪b; a◌̡◌̛◌༹◌〪b; a◌̡◌̛◌༹◌〪b; ) LATIN SMALL LETTER A, IDEOGRAPHIC LEVEL TONE MARK, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, TIBETAN MARK TSA -PHRU, LATIN SMALL LETTER B +0061 0F39 302A 031B 0321 0062;0061 0321 0F39 031B 302A 0062;0061 0321 0F39 031B 302A 0062;0061 0321 0F39 031B 302A 0062;0061 0321 0F39 031B 302A 0062; # (a◌༹◌〪◌̛◌̡b; a◌̡◌༹◌̛◌〪b; a◌̡◌༹◌̛◌〪b; a◌̡◌༹◌̛◌〪b; a◌̡◌༹◌̛◌〪b; ) LATIN SMALL LETTER A, TIBETAN MARK TSA -PHRU, IDEOGRAPHIC LEVEL TONE MARK, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, LATIN SMALL LETTER B +0061 0F72 0F71 0EC8 0F71 0062;0061 0EC8 0F71 0F71 0F72 0062;0061 0EC8 0F71 0F71 0F72 0062;0061 0EC8 0F71 0F71 0F72 0062;0061 0EC8 0F71 0F71 0F72 0062; # (a◌ི◌ཱ◌່◌ཱb; a◌່◌ཱ◌ཱ◌ིb; a◌່◌ཱ◌ཱ◌ིb; a◌່◌ཱ◌ཱ◌ིb; a◌່◌ཱ◌ཱ◌ིb; ) LATIN SMALL LETTER A, TIBETAN VOWEL SIGN I, TIBETAN VOWEL SIGN AA, LAO TONE MAI EK, TIBETAN VOWEL SIGN AA, LATIN SMALL LETTER B +0061 0F71 0F72 0F71 0EC8 0062;0061 0EC8 0F71 0F71 0F72 0062;0061 0EC8 0F71 0F71 0F72 0062;0061 0EC8 0F71 0F71 0F72 0062;0061 0EC8 0F71 0F71 0F72 0062; # (a◌ཱ◌ི◌ཱ◌່b; a◌່◌ཱ◌ཱ◌ིb; a◌່◌ཱ◌ཱ◌ིb; a◌່◌ཱ◌ཱ◌ིb; a◌່◌ཱ◌ཱ◌ིb; ) LATIN SMALL LETTER A, TIBETAN VOWEL SIGN AA, TIBETAN VOWEL SIGN I, TIBETAN VOWEL SIGN AA, LAO TONE MAI EK, LATIN SMALL LETTER B +0061 0F74 0F72 0F71 0F72 0062;0061 0F71 0F72 0F72 0F74 0062;0061 0F71 0F72 0F72 0F74 0062;0061 0F71 0F72 0F72 0F74 0062;0061 0F71 0F72 0F72 0F74 0062; # (a◌ུ◌ི◌ཱ◌ིb; a◌ཱ◌ི◌ི◌ུb; a◌ཱ◌ི◌ི◌ུb; a◌ཱ◌ི◌ི◌ུb; a◌ཱ◌ི◌ི◌ུb; ) LATIN SMALL LETTER A, TIBETAN VOWEL SIGN U, TIBETAN VOWEL SIGN I, TIBETAN VOWEL SIGN AA, TIBETAN VOWEL SIGN I, LATIN SMALL LETTER B +0061 0F72 0F74 0F72 0F71 0062;0061 0F71 0F72 0F72 0F74 0062;0061 0F71 0F72 0F72 0F74 0062;0061 0F71 0F72 0F72 0F74 0062;0061 0F71 0F72 0F72 0F74 0062; # (a◌ི◌ུ◌ི◌ཱb; a◌ཱ◌ི◌ི◌ུb; a◌ཱ◌ི◌ི◌ུb; a◌ཱ◌ི◌ི◌ུb; a◌ཱ◌ི◌ི◌ུb; ) LATIN SMALL LETTER A, TIBETAN VOWEL SIGN I, TIBETAN VOWEL SIGN U, TIBETAN VOWEL SIGN I, TIBETAN VOWEL SIGN AA, LATIN SMALL LETTER B +0061 0321 0F74 0F72 0F74 0062;0061 0F72 0F74 0F74 0321 0062;0061 0F72 0F74 0F74 0321 0062;0061 0F72 0F74 0F74 0321 0062;0061 0F72 0F74 0F74 0321 0062; # (a◌̡◌ུ◌ི◌ུb; a◌ི◌ུ◌ུ◌̡b; a◌ི◌ུ◌ུ◌̡b; a◌ི◌ུ◌ུ◌̡b; a◌ི◌ུ◌ུ◌̡b; ) LATIN SMALL LETTER A, COMBINING PALATALIZED HOOK BELOW, TIBETAN VOWEL SIGN U, TIBETAN VOWEL SIGN I, TIBETAN VOWEL SIGN U, LATIN SMALL LETTER B +0061 0F74 0321 0F74 0F72 0062;0061 0F72 0F74 0F74 0321 0062;0061 0F72 0F74 0F74 0321 0062;0061 0F72 0F74 0F74 0321 0062;0061 0F72 0F74 0F74 0321 0062; # (a◌ུ◌̡◌ུ◌ིb; a◌ི◌ུ◌ུ◌̡b; a◌ི◌ུ◌ུ◌̡b; a◌ི◌ུ◌ུ◌̡b; a◌ི◌ུ◌ུ◌̡b; ) LATIN SMALL LETTER A, TIBETAN VOWEL SIGN U, COMBINING PALATALIZED HOOK BELOW, TIBETAN VOWEL SIGN U, TIBETAN VOWEL SIGN I, LATIN SMALL LETTER B +0061 0F74 0F72 0F71 0F7A 0062;0061 0F71 0F72 0F7A 0F74 0062;0061 0F71 0F72 0F7A 0F74 0062;0061 0F71 0F72 0F7A 0F74 0062;0061 0F71 0F72 0F7A 0F74 0062; # (a◌ུ◌ི◌ཱ◌ེb; a◌ཱ◌ི◌ེ◌ུb; a◌ཱ◌ི◌ེ◌ུb; a◌ཱ◌ི◌ེ◌ུb; a◌ཱ◌ི◌ེ◌ུb; ) LATIN SMALL LETTER A, TIBETAN VOWEL SIGN U, TIBETAN VOWEL SIGN I, TIBETAN VOWEL SIGN AA, TIBETAN VOWEL SIGN E, LATIN SMALL LETTER B +0061 0F7A 0F74 0F72 0F71 0062;0061 0F71 0F7A 0F72 0F74 0062;0061 0F71 0F7A 0F72 0F74 0062;0061 0F71 0F7A 0F72 0F74 0062;0061 0F71 0F7A 0F72 0F74 0062; # (a◌ེ◌ུ◌ི◌ཱb; a◌ཱ◌ེ◌ི◌ུb; a◌ཱ◌ེ◌ི◌ུb; a◌ཱ◌ེ◌ི◌ུb; a◌ཱ◌ེ◌ི◌ུb; ) LATIN SMALL LETTER A, TIBETAN VOWEL SIGN E, TIBETAN VOWEL SIGN U, TIBETAN VOWEL SIGN I, TIBETAN VOWEL SIGN AA, LATIN SMALL LETTER B +0061 0F74 0F72 0F71 0F7B 0062;0061 0F71 0F72 0F7B 0F74 0062;0061 0F71 0F72 0F7B 0F74 0062;0061 0F71 0F72 0F7B 0F74 0062;0061 0F71 0F72 0F7B 0F74 0062; # (a◌ུ◌ི◌ཱ◌ཻb; a◌ཱ◌ི◌ཻ◌ུb; a◌ཱ◌ི◌ཻ◌ུb; a◌ཱ◌ི◌ཻ◌ུb; a◌ཱ◌ི◌ཻ◌ུb; ) LATIN SMALL LETTER A, TIBETAN VOWEL SIGN U, TIBETAN VOWEL SIGN I, TIBETAN VOWEL SIGN AA, TIBETAN VOWEL SIGN EE, LATIN SMALL LETTER B +0061 0F7B 0F74 0F72 0F71 0062;0061 0F71 0F7B 0F72 0F74 0062;0061 0F71 0F7B 0F72 0F74 0062;0061 0F71 0F7B 0F72 0F74 0062;0061 0F71 0F7B 0F72 0F74 0062; # (a◌ཻ◌ུ◌ི◌ཱb; a◌ཱ◌ཻ◌ི◌ུb; a◌ཱ◌ཻ◌ི◌ུb; a◌ཱ◌ཻ◌ི◌ུb; a◌ཱ◌ཻ◌ི◌ུb; ) LATIN SMALL LETTER A, TIBETAN VOWEL SIGN EE, TIBETAN VOWEL SIGN U, TIBETAN VOWEL SIGN I, TIBETAN VOWEL SIGN AA, LATIN SMALL LETTER B +0061 0F74 0F72 0F71 0F7C 0062;0061 0F71 0F72 0F7C 0F74 0062;0061 0F71 0F72 0F7C 0F74 0062;0061 0F71 0F72 0F7C 0F74 0062;0061 0F71 0F72 0F7C 0F74 0062; # (a◌ུ◌ི◌ཱ◌ོb; a◌ཱ◌ི◌ོ◌ུb; a◌ཱ◌ི◌ོ◌ུb; a◌ཱ◌ི◌ོ◌ུb; a◌ཱ◌ི◌ོ◌ུb; ) LATIN SMALL LETTER A, TIBETAN VOWEL SIGN U, TIBETAN VOWEL SIGN I, TIBETAN VOWEL SIGN AA, TIBETAN VOWEL SIGN O, LATIN SMALL LETTER B +0061 0F7C 0F74 0F72 0F71 0062;0061 0F71 0F7C 0F72 0F74 0062;0061 0F71 0F7C 0F72 0F74 0062;0061 0F71 0F7C 0F72 0F74 0062;0061 0F71 0F7C 0F72 0F74 0062; # (a◌ོ◌ུ◌ི◌ཱb; a◌ཱ◌ོ◌ི◌ུb; a◌ཱ◌ོ◌ི◌ུb; a◌ཱ◌ོ◌ི◌ུb; a◌ཱ◌ོ◌ི◌ུb; ) LATIN SMALL LETTER A, TIBETAN VOWEL SIGN O, TIBETAN VOWEL SIGN U, TIBETAN VOWEL SIGN I, TIBETAN VOWEL SIGN AA, LATIN SMALL LETTER B +0061 0F74 0F72 0F71 0F7D 0062;0061 0F71 0F72 0F7D 0F74 0062;0061 0F71 0F72 0F7D 0F74 0062;0061 0F71 0F72 0F7D 0F74 0062;0061 0F71 0F72 0F7D 0F74 0062; # (a◌ུ◌ི◌ཱ◌ཽb; a◌ཱ◌ི◌ཽ◌ུb; a◌ཱ◌ི◌ཽ◌ུb; a◌ཱ◌ི◌ཽ◌ུb; a◌ཱ◌ི◌ཽ◌ུb; ) LATIN SMALL LETTER A, TIBETAN VOWEL SIGN U, TIBETAN VOWEL SIGN I, TIBETAN VOWEL SIGN AA, TIBETAN VOWEL SIGN OO, LATIN SMALL LETTER B +0061 0F7D 0F74 0F72 0F71 0062;0061 0F71 0F7D 0F72 0F74 0062;0061 0F71 0F7D 0F72 0F74 0062;0061 0F71 0F7D 0F72 0F74 0062;0061 0F71 0F7D 0F72 0F74 0062; # (a◌ཽ◌ུ◌ི◌ཱb; a◌ཱ◌ཽ◌ི◌ུb; a◌ཱ◌ཽ◌ི◌ུb; a◌ཱ◌ཽ◌ི◌ུb; a◌ཱ◌ཽ◌ི◌ུb; ) LATIN SMALL LETTER A, TIBETAN VOWEL SIGN OO, TIBETAN VOWEL SIGN U, TIBETAN VOWEL SIGN I, TIBETAN VOWEL SIGN AA, LATIN SMALL LETTER B +0061 0F74 0F72 0F71 0F80 0062;0061 0F71 0F72 0F80 0F74 0062;0061 0F71 0F72 0F80 0F74 0062;0061 0F71 0F72 0F80 0F74 0062;0061 0F71 0F72 0F80 0F74 0062; # (a◌ུ◌ི◌ཱ◌ྀb; a◌ཱ◌ི◌ྀ◌ུb; a◌ཱ◌ི◌ྀ◌ུb; a◌ཱ◌ི◌ྀ◌ུb; a◌ཱ◌ི◌ྀ◌ུb; ) LATIN SMALL LETTER A, TIBETAN VOWEL SIGN U, TIBETAN VOWEL SIGN I, TIBETAN VOWEL SIGN AA, TIBETAN VOWEL SIGN REVERSED I, LATIN SMALL LETTER B +0061 0F80 0F74 0F72 0F71 0062;0061 0F71 0F80 0F72 0F74 0062;0061 0F71 0F80 0F72 0F74 0062;0061 0F71 0F80 0F72 0F74 0062;0061 0F71 0F80 0F72 0F74 0062; # (a◌ྀ◌ུ◌ི◌ཱb; a◌ཱ◌ྀ◌ི◌ུb; a◌ཱ◌ྀ◌ི◌ུb; a◌ཱ◌ྀ◌ི◌ུb; a◌ཱ◌ྀ◌ི◌ུb; ) LATIN SMALL LETTER A, TIBETAN VOWEL SIGN REVERSED I, TIBETAN VOWEL SIGN U, TIBETAN VOWEL SIGN I, TIBETAN VOWEL SIGN AA, LATIN SMALL LETTER B +0061 0315 0300 05AE 0F82 0062;00E0 05AE 0F82 0315 0062;0061 05AE 0300 0F82 0315 0062;00E0 05AE 0F82 0315 0062;0061 05AE 0300 0F82 0315 0062; # (a◌̕◌̀◌֮◌ྂb; à◌֮◌ྂ◌̕b; a◌֮◌̀◌ྂ◌̕b; à◌֮◌ྂ◌̕b; a◌֮◌̀◌ྂ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, TIBETAN SIGN NYI ZLA NAA DA, LATIN SMALL LETTER B +0061 0F82 0315 0300 05AE 0062;0061 05AE 0F82 0300 0315 0062;0061 05AE 0F82 0300 0315 0062;0061 05AE 0F82 0300 0315 0062;0061 05AE 0F82 0300 0315 0062; # (a◌ྂ◌̕◌̀◌֮b; a◌֮◌ྂ◌̀◌̕b; a◌֮◌ྂ◌̀◌̕b; a◌֮◌ྂ◌̀◌̕b; a◌֮◌ྂ◌̀◌̕b; ) LATIN SMALL LETTER A, TIBETAN SIGN NYI ZLA NAA DA, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0F83 0062;00E0 05AE 0F83 0315 0062;0061 05AE 0300 0F83 0315 0062;00E0 05AE 0F83 0315 0062;0061 05AE 0300 0F83 0315 0062; # (a◌̕◌̀◌֮◌ྃb; à◌֮◌ྃ◌̕b; a◌֮◌̀◌ྃ◌̕b; à◌֮◌ྃ◌̕b; a◌֮◌̀◌ྃ◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, TIBETAN SIGN SNA LDAN, LATIN SMALL LETTER B +0061 0F83 0315 0300 05AE 0062;0061 05AE 0F83 0300 0315 0062;0061 05AE 0F83 0300 0315 0062;0061 05AE 0F83 0300 0315 0062;0061 05AE 0F83 0300 0315 0062; # (a◌ྃ◌̕◌̀◌֮b; a◌֮◌ྃ◌̀◌̕b; a◌֮◌ྃ◌̀◌̕b; a◌֮◌ྃ◌̀◌̕b; a◌֮◌ྃ◌̀◌̕b; ) LATIN SMALL LETTER A, TIBETAN SIGN SNA LDAN, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 05B0 094D 3099 0F84 0062;0061 3099 094D 0F84 05B0 0062;0061 3099 094D 0F84 05B0 0062;0061 3099 094D 0F84 05B0 0062;0061 3099 094D 0F84 05B0 0062; # (a◌ְ◌्◌゙◌྄b; a◌゙◌्◌྄◌ְb; a◌゙◌्◌྄◌ְb; a◌゙◌्◌྄◌ְb; a◌゙◌्◌྄◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, TIBETAN MARK HALANTA, LATIN SMALL LETTER B +0061 0F84 05B0 094D 3099 0062;0061 3099 0F84 094D 05B0 0062;0061 3099 0F84 094D 05B0 0062;0061 3099 0F84 094D 05B0 0062;0061 3099 0F84 094D 05B0 0062; # (a◌྄◌ְ◌्◌゙b; a◌゙◌྄◌्◌ְb; a◌゙◌྄◌्◌ְb; a◌゙◌྄◌्◌ְb; a◌゙◌྄◌्◌ְb; ) LATIN SMALL LETTER A, TIBETAN MARK HALANTA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 0F86 0062;00E0 05AE 0F86 0315 0062;0061 05AE 0300 0F86 0315 0062;00E0 05AE 0F86 0315 0062;0061 05AE 0300 0F86 0315 0062; # (a◌̕◌̀◌֮◌྆b; à◌֮◌྆◌̕b; a◌֮◌̀◌྆◌̕b; à◌֮◌྆◌̕b; a◌֮◌̀◌྆◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, TIBETAN SIGN LCI RTAGS, LATIN SMALL LETTER B +0061 0F86 0315 0300 05AE 0062;0061 05AE 0F86 0300 0315 0062;0061 05AE 0F86 0300 0315 0062;0061 05AE 0F86 0300 0315 0062;0061 05AE 0F86 0300 0315 0062; # (a◌྆◌̕◌̀◌֮b; a◌֮◌྆◌̀◌̕b; a◌֮◌྆◌̀◌̕b; a◌֮◌྆◌̀◌̕b; a◌֮◌྆◌̀◌̕b; ) LATIN SMALL LETTER A, TIBETAN SIGN LCI RTAGS, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 0F87 0062;00E0 05AE 0F87 0315 0062;0061 05AE 0300 0F87 0315 0062;00E0 05AE 0F87 0315 0062;0061 05AE 0300 0F87 0315 0062; # (a◌̕◌̀◌֮◌྇b; à◌֮◌྇◌̕b; a◌֮◌̀◌྇◌̕b; à◌֮◌྇◌̕b; a◌֮◌̀◌྇◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, TIBETAN SIGN YANG RTAGS, LATIN SMALL LETTER B +0061 0F87 0315 0300 05AE 0062;0061 05AE 0F87 0300 0315 0062;0061 05AE 0F87 0300 0315 0062;0061 05AE 0F87 0300 0315 0062;0061 05AE 0F87 0300 0315 0062; # (a◌྇◌̕◌̀◌֮b; a◌֮◌྇◌̀◌̕b; a◌֮◌྇◌̀◌̕b; a◌֮◌྇◌̀◌̕b; a◌֮◌྇◌̀◌̕b; ) LATIN SMALL LETTER A, TIBETAN SIGN YANG RTAGS, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 0FC6 0062;0061 302A 0316 0FC6 059A 0062;0061 302A 0316 0FC6 059A 0062;0061 302A 0316 0FC6 059A 0062;0061 302A 0316 0FC6 059A 0062; # (a◌֚◌̖◌〪◌࿆b; a◌〪◌̖◌࿆◌֚b; a◌〪◌̖◌࿆◌֚b; a◌〪◌̖◌࿆◌֚b; a◌〪◌̖◌࿆◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, TIBETAN SYMBOL PADMA GDAN, LATIN SMALL LETTER B +0061 0FC6 059A 0316 302A 0062;0061 302A 0FC6 0316 059A 0062;0061 302A 0FC6 0316 059A 0062;0061 302A 0FC6 0316 059A 0062;0061 302A 0FC6 0316 059A 0062; # (a◌࿆◌֚◌̖◌〪b; a◌〪◌࿆◌̖◌֚b; a◌〪◌࿆◌̖◌֚b; a◌〪◌࿆◌̖◌֚b; a◌〪◌࿆◌̖◌֚b; ) LATIN SMALL LETTER A, TIBETAN SYMBOL PADMA GDAN, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 3099 093C 0334 1037 0062;0061 0334 093C 1037 3099 0062;0061 0334 093C 1037 3099 0062;0061 0334 093C 1037 3099 0062;0061 0334 093C 1037 3099 0062; # (a◌゙◌़◌̴◌့b; a◌̴◌़◌့◌゙b; a◌̴◌़◌့◌゙b; a◌̴◌़◌့◌゙b; a◌̴◌़◌့◌゙b; ) LATIN SMALL LETTER A, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, MYANMAR SIGN DOT BELOW, LATIN SMALL LETTER B +0061 1037 3099 093C 0334 0062;0061 0334 1037 093C 3099 0062;0061 0334 1037 093C 3099 0062;0061 0334 1037 093C 3099 0062;0061 0334 1037 093C 3099 0062; # (a◌့◌゙◌़◌̴b; a◌̴◌့◌़◌゙b; a◌̴◌့◌़◌゙b; a◌̴◌့◌़◌゙b; a◌̴◌့◌़◌゙b; ) LATIN SMALL LETTER A, MYANMAR SIGN DOT BELOW, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 05B0 094D 3099 1039 0062;0061 3099 094D 1039 05B0 0062;0061 3099 094D 1039 05B0 0062;0061 3099 094D 1039 05B0 0062;0061 3099 094D 1039 05B0 0062; # (a◌ְ◌्◌゙◌္b; a◌゙◌्◌္◌ְb; a◌゙◌्◌္◌ְb; a◌゙◌्◌္◌ְb; a◌゙◌्◌္◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, MYANMAR SIGN VIRAMA, LATIN SMALL LETTER B +0061 1039 05B0 094D 3099 0062;0061 3099 1039 094D 05B0 0062;0061 3099 1039 094D 05B0 0062;0061 3099 1039 094D 05B0 0062;0061 3099 1039 094D 05B0 0062; # (a◌္◌ְ◌्◌゙b; a◌゙◌္◌्◌ְb; a◌゙◌္◌्◌ְb; a◌゙◌္◌्◌ְb; a◌゙◌္◌्◌ְb; ) LATIN SMALL LETTER A, MYANMAR SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 05B0 094D 3099 1714 0062;0061 3099 094D 1714 05B0 0062;0061 3099 094D 1714 05B0 0062;0061 3099 094D 1714 05B0 0062;0061 3099 094D 1714 05B0 0062; # (a◌ְ◌्◌゙◌᜔b; a◌゙◌्◌᜔◌ְb; a◌゙◌्◌᜔◌ְb; a◌゙◌्◌᜔◌ְb; a◌゙◌्◌᜔◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, TAGALOG SIGN VIRAMA, LATIN SMALL LETTER B +0061 1714 05B0 094D 3099 0062;0061 3099 1714 094D 05B0 0062;0061 3099 1714 094D 05B0 0062;0061 3099 1714 094D 05B0 0062;0061 3099 1714 094D 05B0 0062; # (a◌᜔◌ְ◌्◌゙b; a◌゙◌᜔◌्◌ְb; a◌゙◌᜔◌्◌ְb; a◌゙◌᜔◌्◌ְb; a◌゙◌᜔◌्◌ְb; ) LATIN SMALL LETTER A, TAGALOG SIGN VIRAMA, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 05B0 094D 3099 1734 0062;0061 3099 094D 1734 05B0 0062;0061 3099 094D 1734 05B0 0062;0061 3099 094D 1734 05B0 0062;0061 3099 094D 1734 05B0 0062; # (a◌ְ◌्◌゙◌᜴b; a◌゙◌्◌᜴◌ְb; a◌゙◌्◌᜴◌ְb; a◌゙◌्◌᜴◌ְb; a◌゙◌्◌᜴◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, HANUNOO SIGN PAMUDPOD, LATIN SMALL LETTER B +0061 1734 05B0 094D 3099 0062;0061 3099 1734 094D 05B0 0062;0061 3099 1734 094D 05B0 0062;0061 3099 1734 094D 05B0 0062;0061 3099 1734 094D 05B0 0062; # (a◌᜴◌ְ◌्◌゙b; a◌゙◌᜴◌्◌ְb; a◌゙◌᜴◌्◌ְb; a◌゙◌᜴◌्◌ְb; a◌゙◌᜴◌्◌ְb; ) LATIN SMALL LETTER A, HANUNOO SIGN PAMUDPOD, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 05B0 094D 3099 17D2 0062;0061 3099 094D 17D2 05B0 0062;0061 3099 094D 17D2 05B0 0062;0061 3099 094D 17D2 05B0 0062;0061 3099 094D 17D2 05B0 0062; # (a◌ְ◌्◌゙◌្b; a◌゙◌्◌្◌ְb; a◌゙◌्◌្◌ְb; a◌゙◌्◌្◌ְb; a◌゙◌्◌្◌ְb; ) LATIN SMALL LETTER A, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, KHMER SIGN COENG, LATIN SMALL LETTER B +0061 17D2 05B0 094D 3099 0062;0061 3099 17D2 094D 05B0 0062;0061 3099 17D2 094D 05B0 0062;0061 3099 17D2 094D 05B0 0062;0061 3099 17D2 094D 05B0 0062; # (a◌្◌ְ◌्◌゙b; a◌゙◌្◌्◌ְb; a◌゙◌្◌्◌ְb; a◌゙◌្◌्◌ְb; a◌゙◌្◌्◌ְb; ) LATIN SMALL LETTER A, KHMER SIGN COENG, HEBREW POINT SHEVA, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 0300 05AE 1D16D 18A9 0062;00E0 1D16D 05AE 18A9 0062;0061 1D16D 05AE 18A9 0300 0062;00E0 1D16D 05AE 18A9 0062;0061 1D16D 05AE 18A9 0300 0062; # (a◌̀◌𝅭𝅭֮◌ᢩb; à𝅭𝅭◌֮◌ᢩb; a𝅭𝅭◌֮◌ᢩ◌̀b; à𝅭𝅭◌֮◌ᢩb; a𝅭𝅭◌֮◌ᢩ◌̀b; ) LATIN SMALL LETTER A, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, MONGOLIAN LETTER ALI GALI DAGALGA, LATIN SMALL LETTER B +0061 18A9 0300 05AE 1D16D 0062;00E0 1D16D 18A9 05AE 0062;0061 1D16D 18A9 05AE 0300 0062;00E0 1D16D 18A9 05AE 0062;0061 1D16D 18A9 05AE 0300 0062; # (a◌ᢩ◌̀◌𝅭𝅭֮b; à𝅭𝅭◌ᢩ◌֮b; a𝅭𝅭◌ᢩ◌֮◌̀b; à𝅭𝅭◌ᢩ◌֮b; a𝅭𝅭◌ᢩ◌֮◌̀b; ) LATIN SMALL LETTER A, MONGOLIAN LETTER ALI GALI DAGALGA, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, LATIN SMALL LETTER B +0061 0315 0300 05AE 20D0 0062;00E0 05AE 20D0 0315 0062;0061 05AE 0300 20D0 0315 0062;00E0 05AE 20D0 0315 0062;0061 05AE 0300 20D0 0315 0062; # (a◌̕◌̀◌֮◌⃐b; à◌֮◌⃐◌̕b; a◌֮◌̀◌⃐◌̕b; à◌֮◌⃐◌̕b; a◌֮◌̀◌⃐◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING LEFT HARPOON ABOVE, LATIN SMALL LETTER B +0061 20D0 0315 0300 05AE 0062;0061 05AE 20D0 0300 0315 0062;0061 05AE 20D0 0300 0315 0062;0061 05AE 20D0 0300 0315 0062;0061 05AE 20D0 0300 0315 0062; # (a◌⃐◌̕◌̀◌֮b; a◌֮◌⃐◌̀◌̕b; a◌֮◌⃐◌̀◌̕b; a◌֮◌⃐◌̀◌̕b; a◌֮◌⃐◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING LEFT HARPOON ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 20D1 0062;00E0 05AE 20D1 0315 0062;0061 05AE 0300 20D1 0315 0062;00E0 05AE 20D1 0315 0062;0061 05AE 0300 20D1 0315 0062; # (a◌̕◌̀◌֮◌⃑b; à◌֮◌⃑◌̕b; a◌֮◌̀◌⃑◌̕b; à◌֮◌⃑◌̕b; a◌֮◌̀◌⃑◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING RIGHT HARPOON ABOVE, LATIN SMALL LETTER B +0061 20D1 0315 0300 05AE 0062;0061 05AE 20D1 0300 0315 0062;0061 05AE 20D1 0300 0315 0062;0061 05AE 20D1 0300 0315 0062;0061 05AE 20D1 0300 0315 0062; # (a◌⃑◌̕◌̀◌֮b; a◌֮◌⃑◌̀◌̕b; a◌֮◌⃑◌̀◌̕b; a◌֮◌⃑◌̀◌̕b; a◌֮◌⃑◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING RIGHT HARPOON ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 093C 0334 20D2 0062;0061 0334 20D2 093C 0062;0061 0334 20D2 093C 0062;0061 0334 20D2 093C 0062;0061 0334 20D2 093C 0062; # (a◌़◌̴◌⃒b; a◌̴◌⃒◌़b; a◌̴◌⃒◌़b; a◌̴◌⃒◌़b; a◌̴◌⃒◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, COMBINING LONG VERTICAL LINE OVERLAY, LATIN SMALL LETTER B +0061 20D2 093C 0334 0062;0061 20D2 0334 093C 0062;0061 20D2 0334 093C 0062;0061 20D2 0334 093C 0062;0061 20D2 0334 093C 0062; # (a◌⃒◌़◌̴b; a◌⃒◌̴◌़b; a◌⃒◌̴◌़b; a◌⃒◌̴◌़b; a◌⃒◌̴◌़b; ) LATIN SMALL LETTER A, COMBINING LONG VERTICAL LINE OVERLAY, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 093C 0334 20D3 0062;0061 0334 20D3 093C 0062;0061 0334 20D3 093C 0062;0061 0334 20D3 093C 0062;0061 0334 20D3 093C 0062; # (a◌़◌̴◌⃓b; a◌̴◌⃓◌़b; a◌̴◌⃓◌़b; a◌̴◌⃓◌़b; a◌̴◌⃓◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, COMBINING SHORT VERTICAL LINE OVERLAY, LATIN SMALL LETTER B +0061 20D3 093C 0334 0062;0061 20D3 0334 093C 0062;0061 20D3 0334 093C 0062;0061 20D3 0334 093C 0062;0061 20D3 0334 093C 0062; # (a◌⃓◌़◌̴b; a◌⃓◌̴◌़b; a◌⃓◌̴◌़b; a◌⃓◌̴◌़b; a◌⃓◌̴◌़b; ) LATIN SMALL LETTER A, COMBINING SHORT VERTICAL LINE OVERLAY, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 0315 0300 05AE 20D4 0062;00E0 05AE 20D4 0315 0062;0061 05AE 0300 20D4 0315 0062;00E0 05AE 20D4 0315 0062;0061 05AE 0300 20D4 0315 0062; # (a◌̕◌̀◌֮◌⃔b; à◌֮◌⃔◌̕b; a◌֮◌̀◌⃔◌̕b; à◌֮◌⃔◌̕b; a◌֮◌̀◌⃔◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING ANTICLOCKWISE ARROW ABOVE, LATIN SMALL LETTER B +0061 20D4 0315 0300 05AE 0062;0061 05AE 20D4 0300 0315 0062;0061 05AE 20D4 0300 0315 0062;0061 05AE 20D4 0300 0315 0062;0061 05AE 20D4 0300 0315 0062; # (a◌⃔◌̕◌̀◌֮b; a◌֮◌⃔◌̀◌̕b; a◌֮◌⃔◌̀◌̕b; a◌֮◌⃔◌̀◌̕b; a◌֮◌⃔◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING ANTICLOCKWISE ARROW ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 20D5 0062;00E0 05AE 20D5 0315 0062;0061 05AE 0300 20D5 0315 0062;00E0 05AE 20D5 0315 0062;0061 05AE 0300 20D5 0315 0062; # (a◌̕◌̀◌֮◌⃕b; à◌֮◌⃕◌̕b; a◌֮◌̀◌⃕◌̕b; à◌֮◌⃕◌̕b; a◌֮◌̀◌⃕◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING CLOCKWISE ARROW ABOVE, LATIN SMALL LETTER B +0061 20D5 0315 0300 05AE 0062;0061 05AE 20D5 0300 0315 0062;0061 05AE 20D5 0300 0315 0062;0061 05AE 20D5 0300 0315 0062;0061 05AE 20D5 0300 0315 0062; # (a◌⃕◌̕◌̀◌֮b; a◌֮◌⃕◌̀◌̕b; a◌֮◌⃕◌̀◌̕b; a◌֮◌⃕◌̀◌̕b; a◌֮◌⃕◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING CLOCKWISE ARROW ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 20D6 0062;00E0 05AE 20D6 0315 0062;0061 05AE 0300 20D6 0315 0062;00E0 05AE 20D6 0315 0062;0061 05AE 0300 20D6 0315 0062; # (a◌̕◌̀◌֮◌⃖b; à◌֮◌⃖◌̕b; a◌֮◌̀◌⃖◌̕b; à◌֮◌⃖◌̕b; a◌֮◌̀◌⃖◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING LEFT ARROW ABOVE, LATIN SMALL LETTER B +0061 20D6 0315 0300 05AE 0062;0061 05AE 20D6 0300 0315 0062;0061 05AE 20D6 0300 0315 0062;0061 05AE 20D6 0300 0315 0062;0061 05AE 20D6 0300 0315 0062; # (a◌⃖◌̕◌̀◌֮b; a◌֮◌⃖◌̀◌̕b; a◌֮◌⃖◌̀◌̕b; a◌֮◌⃖◌̀◌̕b; a◌֮◌⃖◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING LEFT ARROW ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 20D7 0062;00E0 05AE 20D7 0315 0062;0061 05AE 0300 20D7 0315 0062;00E0 05AE 20D7 0315 0062;0061 05AE 0300 20D7 0315 0062; # (a◌̕◌̀◌֮◌⃗b; à◌֮◌⃗◌̕b; a◌֮◌̀◌⃗◌̕b; à◌֮◌⃗◌̕b; a◌֮◌̀◌⃗◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING RIGHT ARROW ABOVE, LATIN SMALL LETTER B +0061 20D7 0315 0300 05AE 0062;0061 05AE 20D7 0300 0315 0062;0061 05AE 20D7 0300 0315 0062;0061 05AE 20D7 0300 0315 0062;0061 05AE 20D7 0300 0315 0062; # (a◌⃗◌̕◌̀◌֮b; a◌֮◌⃗◌̀◌̕b; a◌֮◌⃗◌̀◌̕b; a◌֮◌⃗◌̀◌̕b; a◌֮◌⃗◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING RIGHT ARROW ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 093C 0334 20D8 0062;0061 0334 20D8 093C 0062;0061 0334 20D8 093C 0062;0061 0334 20D8 093C 0062;0061 0334 20D8 093C 0062; # (a◌़◌̴◌⃘b; a◌̴◌⃘◌़b; a◌̴◌⃘◌़b; a◌̴◌⃘◌़b; a◌̴◌⃘◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, COMBINING RING OVERLAY, LATIN SMALL LETTER B +0061 20D8 093C 0334 0062;0061 20D8 0334 093C 0062;0061 20D8 0334 093C 0062;0061 20D8 0334 093C 0062;0061 20D8 0334 093C 0062; # (a◌⃘◌़◌̴b; a◌⃘◌̴◌़b; a◌⃘◌̴◌़b; a◌⃘◌̴◌़b; a◌⃘◌̴◌़b; ) LATIN SMALL LETTER A, COMBINING RING OVERLAY, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 093C 0334 20D9 0062;0061 0334 20D9 093C 0062;0061 0334 20D9 093C 0062;0061 0334 20D9 093C 0062;0061 0334 20D9 093C 0062; # (a◌़◌̴◌⃙b; a◌̴◌⃙◌़b; a◌̴◌⃙◌़b; a◌̴◌⃙◌़b; a◌̴◌⃙◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, COMBINING CLOCKWISE RING OVERLAY, LATIN SMALL LETTER B +0061 20D9 093C 0334 0062;0061 20D9 0334 093C 0062;0061 20D9 0334 093C 0062;0061 20D9 0334 093C 0062;0061 20D9 0334 093C 0062; # (a◌⃙◌़◌̴b; a◌⃙◌̴◌़b; a◌⃙◌̴◌़b; a◌⃙◌̴◌़b; a◌⃙◌̴◌़b; ) LATIN SMALL LETTER A, COMBINING CLOCKWISE RING OVERLAY, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 093C 0334 20DA 0062;0061 0334 20DA 093C 0062;0061 0334 20DA 093C 0062;0061 0334 20DA 093C 0062;0061 0334 20DA 093C 0062; # (a◌़◌̴◌⃚b; a◌̴◌⃚◌़b; a◌̴◌⃚◌़b; a◌̴◌⃚◌़b; a◌̴◌⃚◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, COMBINING ANTICLOCKWISE RING OVERLAY, LATIN SMALL LETTER B +0061 20DA 093C 0334 0062;0061 20DA 0334 093C 0062;0061 20DA 0334 093C 0062;0061 20DA 0334 093C 0062;0061 20DA 0334 093C 0062; # (a◌⃚◌़◌̴b; a◌⃚◌̴◌़b; a◌⃚◌̴◌़b; a◌⃚◌̴◌़b; a◌⃚◌̴◌़b; ) LATIN SMALL LETTER A, COMBINING ANTICLOCKWISE RING OVERLAY, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 0315 0300 05AE 20DB 0062;00E0 05AE 20DB 0315 0062;0061 05AE 0300 20DB 0315 0062;00E0 05AE 20DB 0315 0062;0061 05AE 0300 20DB 0315 0062; # (a◌̕◌̀◌֮◌⃛b; à◌֮◌⃛◌̕b; a◌֮◌̀◌⃛◌̕b; à◌֮◌⃛◌̕b; a◌֮◌̀◌⃛◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING THREE DOTS ABOVE, LATIN SMALL LETTER B +0061 20DB 0315 0300 05AE 0062;0061 05AE 20DB 0300 0315 0062;0061 05AE 20DB 0300 0315 0062;0061 05AE 20DB 0300 0315 0062;0061 05AE 20DB 0300 0315 0062; # (a◌⃛◌̕◌̀◌֮b; a◌֮◌⃛◌̀◌̕b; a◌֮◌⃛◌̀◌̕b; a◌֮◌⃛◌̀◌̕b; a◌֮◌⃛◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING THREE DOTS ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 20DC 0062;00E0 05AE 20DC 0315 0062;0061 05AE 0300 20DC 0315 0062;00E0 05AE 20DC 0315 0062;0061 05AE 0300 20DC 0315 0062; # (a◌̕◌̀◌֮◌⃜b; à◌֮◌⃜◌̕b; a◌֮◌̀◌⃜◌̕b; à◌֮◌⃜◌̕b; a◌֮◌̀◌⃜◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING FOUR DOTS ABOVE, LATIN SMALL LETTER B +0061 20DC 0315 0300 05AE 0062;0061 05AE 20DC 0300 0315 0062;0061 05AE 20DC 0300 0315 0062;0061 05AE 20DC 0300 0315 0062;0061 05AE 20DC 0300 0315 0062; # (a◌⃜◌̕◌̀◌֮b; a◌֮◌⃜◌̀◌̕b; a◌֮◌⃜◌̀◌̕b; a◌֮◌⃜◌̀◌̕b; a◌֮◌⃜◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING FOUR DOTS ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 20E1 0062;00E0 05AE 20E1 0315 0062;0061 05AE 0300 20E1 0315 0062;00E0 05AE 20E1 0315 0062;0061 05AE 0300 20E1 0315 0062; # (a◌̕◌̀◌֮◌⃡b; à◌֮◌⃡◌̕b; a◌֮◌̀◌⃡◌̕b; à◌֮◌⃡◌̕b; a◌֮◌̀◌⃡◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING LEFT RIGHT ARROW ABOVE, LATIN SMALL LETTER B +0061 20E1 0315 0300 05AE 0062;0061 05AE 20E1 0300 0315 0062;0061 05AE 20E1 0300 0315 0062;0061 05AE 20E1 0300 0315 0062;0061 05AE 20E1 0300 0315 0062; # (a◌⃡◌̕◌̀◌֮b; a◌֮◌⃡◌̀◌̕b; a◌֮◌⃡◌̀◌̕b; a◌֮◌⃡◌̀◌̕b; a◌֮◌⃡◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING LEFT RIGHT ARROW ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 093C 0334 20E5 0062;0061 0334 20E5 093C 0062;0061 0334 20E5 093C 0062;0061 0334 20E5 093C 0062;0061 0334 20E5 093C 0062; # (a◌़◌̴◌⃥b; a◌̴◌⃥◌़b; a◌̴◌⃥◌़b; a◌̴◌⃥◌़b; a◌̴◌⃥◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, COMBINING REVERSE SOLIDUS OVERLAY, LATIN SMALL LETTER B +0061 20E5 093C 0334 0062;0061 20E5 0334 093C 0062;0061 20E5 0334 093C 0062;0061 20E5 0334 093C 0062;0061 20E5 0334 093C 0062; # (a◌⃥◌़◌̴b; a◌⃥◌̴◌़b; a◌⃥◌̴◌़b; a◌⃥◌̴◌़b; a◌⃥◌̴◌़b; ) LATIN SMALL LETTER A, COMBINING REVERSE SOLIDUS OVERLAY, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 093C 0334 20E6 0062;0061 0334 20E6 093C 0062;0061 0334 20E6 093C 0062;0061 0334 20E6 093C 0062;0061 0334 20E6 093C 0062; # (a◌़◌̴◌⃦b; a◌̴◌⃦◌़b; a◌̴◌⃦◌़b; a◌̴◌⃦◌़b; a◌̴◌⃦◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, COMBINING DOUBLE VERTICAL STROKE OVERLAY, LATIN SMALL LETTER B +0061 20E6 093C 0334 0062;0061 20E6 0334 093C 0062;0061 20E6 0334 093C 0062;0061 20E6 0334 093C 0062;0061 20E6 0334 093C 0062; # (a◌⃦◌़◌̴b; a◌⃦◌̴◌़b; a◌⃦◌̴◌़b; a◌⃦◌̴◌़b; a◌⃦◌̴◌़b; ) LATIN SMALL LETTER A, COMBINING DOUBLE VERTICAL STROKE OVERLAY, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 0315 0300 05AE 20E7 0062;00E0 05AE 20E7 0315 0062;0061 05AE 0300 20E7 0315 0062;00E0 05AE 20E7 0315 0062;0061 05AE 0300 20E7 0315 0062; # (a◌̕◌̀◌֮◌⃧b; à◌֮◌⃧◌̕b; a◌֮◌̀◌⃧◌̕b; à◌֮◌⃧◌̕b; a◌֮◌̀◌⃧◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING ANNUITY SYMBOL, LATIN SMALL LETTER B +0061 20E7 0315 0300 05AE 0062;0061 05AE 20E7 0300 0315 0062;0061 05AE 20E7 0300 0315 0062;0061 05AE 20E7 0300 0315 0062;0061 05AE 20E7 0300 0315 0062; # (a◌⃧◌̕◌̀◌֮b; a◌֮◌⃧◌̀◌̕b; a◌֮◌⃧◌̀◌̕b; a◌֮◌⃧◌̀◌̕b; a◌֮◌⃧◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING ANNUITY SYMBOL, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 20E8 0062;0061 302A 0316 20E8 059A 0062;0061 302A 0316 20E8 059A 0062;0061 302A 0316 20E8 059A 0062;0061 302A 0316 20E8 059A 0062; # (a◌֚◌̖◌〪◌⃨b; a◌〪◌̖◌⃨◌֚b; a◌〪◌̖◌⃨◌֚b; a◌〪◌̖◌⃨◌֚b; a◌〪◌̖◌⃨◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING TRIPLE UNDERDOT, LATIN SMALL LETTER B +0061 20E8 059A 0316 302A 0062;0061 302A 20E8 0316 059A 0062;0061 302A 20E8 0316 059A 0062;0061 302A 20E8 0316 059A 0062;0061 302A 20E8 0316 059A 0062; # (a◌⃨◌֚◌̖◌〪b; a◌〪◌⃨◌̖◌֚b; a◌〪◌⃨◌̖◌֚b; a◌〪◌⃨◌̖◌֚b; a◌〪◌⃨◌̖◌֚b; ) LATIN SMALL LETTER A, COMBINING TRIPLE UNDERDOT, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 20E9 0062;00E0 05AE 20E9 0315 0062;0061 05AE 0300 20E9 0315 0062;00E0 05AE 20E9 0315 0062;0061 05AE 0300 20E9 0315 0062; # (a◌̕◌̀◌֮◌⃩b; à◌֮◌⃩◌̕b; a◌֮◌̀◌⃩◌̕b; à◌֮◌⃩◌̕b; a◌֮◌̀◌⃩◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING WIDE BRIDGE ABOVE, LATIN SMALL LETTER B +0061 20E9 0315 0300 05AE 0062;0061 05AE 20E9 0300 0315 0062;0061 05AE 20E9 0300 0315 0062;0061 05AE 20E9 0300 0315 0062;0061 05AE 20E9 0300 0315 0062; # (a◌⃩◌̕◌̀◌֮b; a◌֮◌⃩◌̀◌̕b; a◌֮◌⃩◌̀◌̕b; a◌֮◌⃩◌̀◌̕b; a◌֮◌⃩◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING WIDE BRIDGE ABOVE, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 093C 0334 20EA 0062;0061 0334 20EA 093C 0062;0061 0334 20EA 093C 0062;0061 0334 20EA 093C 0062;0061 0334 20EA 093C 0062; # (a◌़◌̴◌⃪b; a◌̴◌⃪◌़b; a◌̴◌⃪◌़b; a◌̴◌⃪◌़b; a◌̴◌⃪◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, COMBINING LEFTWARDS ARROW OVERLAY, LATIN SMALL LETTER B +0061 20EA 093C 0334 0062;0061 20EA 0334 093C 0062;0061 20EA 0334 093C 0062;0061 20EA 0334 093C 0062;0061 20EA 0334 093C 0062; # (a◌⃪◌़◌̴b; a◌⃪◌̴◌़b; a◌⃪◌̴◌़b; a◌⃪◌̴◌़b; a◌⃪◌̴◌़b; ) LATIN SMALL LETTER A, COMBINING LEFTWARDS ARROW OVERLAY, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 0316 302A 031B 302A 0062;0061 031B 302A 302A 0316 0062;0061 031B 302A 302A 0316 0062;0061 031B 302A 302A 0316 0062;0061 031B 302A 302A 0316 0062; # (a◌̖◌〪◌̛◌〪b; a◌̛◌〪◌〪◌̖b; a◌̛◌〪◌〪◌̖b; a◌̛◌〪◌〪◌̖b; a◌̛◌〪◌〪◌̖b; ) LATIN SMALL LETTER A, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING HORN, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 302A 0316 302A 031B 0062;0061 031B 302A 302A 0316 0062;0061 031B 302A 302A 0316 0062;0061 031B 302A 302A 0316 0062;0061 031B 302A 302A 0316 0062; # (a◌〪◌̖◌〪◌̛b; a◌̛◌〪◌〪◌̖b; a◌̛◌〪◌〪◌̖b; a◌̛◌〪◌〪◌̖b; a◌̛◌〪◌〪◌̖b; ) LATIN SMALL LETTER A, IDEOGRAPHIC LEVEL TONE MARK, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, COMBINING HORN, LATIN SMALL LETTER B +0061 0300 05AE 1D16D 302B 0062;00E0 1D16D 05AE 302B 0062;0061 1D16D 05AE 302B 0300 0062;00E0 1D16D 05AE 302B 0062;0061 1D16D 05AE 302B 0300 0062; # (a◌̀◌𝅭𝅭֮◌〫b; à𝅭𝅭◌֮◌〫b; a𝅭𝅭◌֮◌〫◌̀b; à𝅭𝅭◌֮◌〫b; a𝅭𝅭◌֮◌〫◌̀b; ) LATIN SMALL LETTER A, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, IDEOGRAPHIC RISING TONE MARK, LATIN SMALL LETTER B +0061 302B 0300 05AE 1D16D 0062;00E0 1D16D 302B 05AE 0062;0061 1D16D 302B 05AE 0300 0062;00E0 1D16D 302B 05AE 0062;0061 1D16D 302B 05AE 0300 0062; # (a◌〫◌̀◌𝅭𝅭֮b; à𝅭𝅭◌〫◌֮b; a𝅭𝅭◌〫◌֮◌̀b; à𝅭𝅭◌〫◌֮b; a𝅭𝅭◌〫◌֮◌̀b; ) LATIN SMALL LETTER A, IDEOGRAPHIC RISING TONE MARK, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, LATIN SMALL LETTER B +0061 0362 0315 0300 302C 0062;00E0 0315 302C 0362 0062;0061 0300 0315 302C 0362 0062;00E0 0315 302C 0362 0062;0061 0300 0315 302C 0362 0062; # (a◌͢◌̕◌̀◌〬b; à◌̕◌〬◌͢b; a◌̀◌̕◌〬◌͢b; à◌̕◌〬◌͢b; a◌̀◌̕◌〬◌͢b; ) LATIN SMALL LETTER A, COMBINING DOUBLE RIGHTWARDS ARROW BELOW, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, IDEOGRAPHIC DEPARTING TONE MARK, LATIN SMALL LETTER B +0061 302C 0362 0315 0300 0062;00E0 302C 0315 0362 0062;0061 0300 302C 0315 0362 0062;00E0 302C 0315 0362 0062;0061 0300 302C 0315 0362 0062; # (a◌〬◌͢◌̕◌̀b; à◌〬◌̕◌͢b; a◌̀◌〬◌̕◌͢b; à◌〬◌̕◌͢b; a◌̀◌〬◌̕◌͢b; ) LATIN SMALL LETTER A, IDEOGRAPHIC DEPARTING TONE MARK, COMBINING DOUBLE RIGHTWARDS ARROW BELOW, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, LATIN SMALL LETTER B +0061 302E 059A 0316 302D 0062;0061 0316 059A 302D 302E 0062;0061 0316 059A 302D 302E 0062;0061 0316 059A 302D 302E 0062;0061 0316 059A 302D 302E 0062; # (a◌〮◌֚◌̖◌〭b; a◌̖◌֚◌〭◌〮b; a◌̖◌֚◌〭◌〮b; a◌̖◌֚◌〭◌〮b; a◌̖◌֚◌〭◌〮b; ) LATIN SMALL LETTER A, HANGUL SINGLE DOT TONE MARK, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC ENTERING TONE MARK, LATIN SMALL LETTER B +0061 302D 302E 059A 0316 0062;0061 0316 302D 059A 302E 0062;0061 0316 302D 059A 302E 0062;0061 0316 302D 059A 302E 0062;0061 0316 302D 059A 302E 0062; # (a◌〭◌〮◌֚◌̖b; a◌̖◌〭◌֚◌〮b; a◌̖◌〭◌֚◌〮b; a◌̖◌〭◌֚◌〮b; a◌̖◌〭◌֚◌〮b; ) LATIN SMALL LETTER A, IDEOGRAPHIC ENTERING TONE MARK, HANGUL SINGLE DOT TONE MARK, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, LATIN SMALL LETTER B +0061 1D16D 302E 059A 302E 0062;0061 059A 302E 302E 1D16D 0062;0061 059A 302E 302E 1D16D 0062;0061 059A 302E 302E 1D16D 0062;0061 059A 302E 302E 1D16D 0062; # (a𝅭𝅭◌〮◌֚◌〮b; a◌֚◌〮◌〮𝅭𝅭b; a◌֚◌〮◌〮𝅭𝅭b; a◌֚◌〮◌〮𝅭𝅭b; a◌֚◌〮◌〮𝅭𝅭b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, HANGUL SINGLE DOT TONE MARK, HEBREW ACCENT YETIV, HANGUL SINGLE DOT TONE MARK, LATIN SMALL LETTER B +0061 302E 1D16D 302E 059A 0062;0061 059A 302E 302E 1D16D 0062;0061 059A 302E 302E 1D16D 0062;0061 059A 302E 302E 1D16D 0062;0061 059A 302E 302E 1D16D 0062; # (a◌〮𝅭𝅭◌〮◌֚b; a◌֚◌〮◌〮𝅭𝅭b; a◌֚◌〮◌〮𝅭𝅭b; a◌֚◌〮◌〮𝅭𝅭b; a◌֚◌〮◌〮𝅭𝅭b; ) LATIN SMALL LETTER A, HANGUL SINGLE DOT TONE MARK, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, HANGUL SINGLE DOT TONE MARK, HEBREW ACCENT YETIV, LATIN SMALL LETTER B +0061 1D16D 302E 059A 302F 0062;0061 059A 302E 302F 1D16D 0062;0061 059A 302E 302F 1D16D 0062;0061 059A 302E 302F 1D16D 0062;0061 059A 302E 302F 1D16D 0062; # (a𝅭𝅭◌〮◌֚◌〯b; a◌֚◌〮◌〯𝅭𝅭b; a◌֚◌〮◌〯𝅭𝅭b; a◌֚◌〮◌〯𝅭𝅭b; a◌֚◌〮◌〯𝅭𝅭b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, HANGUL SINGLE DOT TONE MARK, HEBREW ACCENT YETIV, HANGUL DOUBLE DOT TONE MARK, LATIN SMALL LETTER B +0061 302F 1D16D 302E 059A 0062;0061 059A 302F 302E 1D16D 0062;0061 059A 302F 302E 1D16D 0062;0061 059A 302F 302E 1D16D 0062;0061 059A 302F 302E 1D16D 0062; # (a◌〯𝅭𝅭◌〮◌֚b; a◌֚◌〯◌〮𝅭𝅭b; a◌֚◌〯◌〮𝅭𝅭b; a◌֚◌〯◌〮𝅭𝅭b; a◌֚◌〯◌〮𝅭𝅭b; ) LATIN SMALL LETTER A, HANGUL DOUBLE DOT TONE MARK, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, HANGUL SINGLE DOT TONE MARK, HEBREW ACCENT YETIV, LATIN SMALL LETTER B +0061 094D 3099 093C 3099 0062;0061 093C 3099 3099 094D 0062;0061 093C 3099 3099 094D 0062;0061 093C 3099 3099 094D 0062;0061 093C 3099 3099 094D 0062; # (a◌्◌゙◌़◌゙b; a◌़◌゙◌゙◌्b; a◌़◌゙◌゙◌्b; a◌़◌゙◌゙◌्b; a◌़◌゙◌゙◌्b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, LATIN SMALL LETTER B +0061 3099 094D 3099 093C 0062;0061 093C 3099 3099 094D 0062;0061 093C 3099 3099 094D 0062;0061 093C 3099 3099 094D 0062;0061 093C 3099 3099 094D 0062; # (a◌゙◌्◌゙◌़b; a◌़◌゙◌゙◌्b; a◌़◌゙◌゙◌्b; a◌़◌゙◌゙◌्b; a◌़◌゙◌゙◌्b; ) LATIN SMALL LETTER A, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, LATIN SMALL LETTER B +0061 094D 3099 093C 309A 0062;0061 093C 3099 309A 094D 0062;0061 093C 3099 309A 094D 0062;0061 093C 3099 309A 094D 0062;0061 093C 3099 309A 094D 0062; # (a◌्◌゙◌़◌゚b; a◌़◌゙◌゚◌्b; a◌़◌゙◌゚◌्b; a◌़◌゙◌゚◌्b; a◌़◌゙◌゚◌्b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK, LATIN SMALL LETTER B +0061 309A 094D 3099 093C 0062;0061 093C 309A 3099 094D 0062;0061 093C 309A 3099 094D 0062;0061 093C 309A 3099 094D 0062;0061 093C 309A 3099 094D 0062; # (a◌゚◌्◌゙◌़b; a◌़◌゚◌゙◌्b; a◌़◌゚◌゙◌्b; a◌़◌゚◌゙◌्b; a◌़◌゚◌゙◌्b; ) LATIN SMALL LETTER A, COMBINING KATAKANA-HIRAGANA SEMI-VOICED SOUND MARK, DEVANAGARI SIGN VIRAMA, COMBINING KATAKANA-HIRAGANA VOICED SOUND MARK, DEVANAGARI SIGN NUKTA, LATIN SMALL LETTER B +0061 064B FB1E 05C2 FB1E 0062;0061 05C2 FB1E FB1E 064B 0062;0061 05C2 FB1E FB1E 064B 0062;0061 05C2 FB1E FB1E 064B 0062;0061 05C2 FB1E FB1E 064B 0062; # (a◌ً◌ﬞ◌ׂ◌ﬞb; a◌ׂ◌ﬞ◌ﬞ◌ًb; a◌ׂ◌ﬞ◌ﬞ◌ًb; a◌ׂ◌ﬞ◌ﬞ◌ًb; a◌ׂ◌ﬞ◌ﬞ◌ًb; ) LATIN SMALL LETTER A, ARABIC FATHATAN, HEBREW POINT JUDEO-SPANISH VARIKA, HEBREW POINT SIN DOT, HEBREW POINT JUDEO-SPANISH VARIKA, LATIN SMALL LETTER B +0061 FB1E 064B FB1E 05C2 0062;0061 05C2 FB1E FB1E 064B 0062;0061 05C2 FB1E FB1E 064B 0062;0061 05C2 FB1E FB1E 064B 0062;0061 05C2 FB1E FB1E 064B 0062; # (a◌ﬞ◌ً◌ﬞ◌ׂb; a◌ׂ◌ﬞ◌ﬞ◌ًb; a◌ׂ◌ﬞ◌ﬞ◌ًb; a◌ׂ◌ﬞ◌ﬞ◌ًb; a◌ׂ◌ﬞ◌ﬞ◌ًb; ) LATIN SMALL LETTER A, HEBREW POINT JUDEO-SPANISH VARIKA, ARABIC FATHATAN, HEBREW POINT JUDEO-SPANISH VARIKA, HEBREW POINT SIN DOT, LATIN SMALL LETTER B +0061 0315 0300 05AE FE20 0062;00E0 05AE FE20 0315 0062;0061 05AE 0300 FE20 0315 0062;00E0 05AE FE20 0315 0062;0061 05AE 0300 FE20 0315 0062; # (a◌̕◌̀◌֮◌︠b; à◌֮◌︠◌̕b; a◌֮◌̀◌︠◌̕b; à◌֮◌︠◌̕b; a◌֮◌̀◌︠◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING LIGATURE LEFT HALF, LATIN SMALL LETTER B +0061 FE20 0315 0300 05AE 0062;0061 05AE FE20 0300 0315 0062;0061 05AE FE20 0300 0315 0062;0061 05AE FE20 0300 0315 0062;0061 05AE FE20 0300 0315 0062; # (a◌︠◌̕◌̀◌֮b; a◌֮◌︠◌̀◌̕b; a◌֮◌︠◌̀◌̕b; a◌֮◌︠◌̀◌̕b; a◌֮◌︠◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING LIGATURE LEFT HALF, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE FE21 0062;00E0 05AE FE21 0315 0062;0061 05AE 0300 FE21 0315 0062;00E0 05AE FE21 0315 0062;0061 05AE 0300 FE21 0315 0062; # (a◌̕◌̀◌֮◌︡b; à◌֮◌︡◌̕b; a◌֮◌̀◌︡◌̕b; à◌֮◌︡◌̕b; a◌֮◌̀◌︡◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING LIGATURE RIGHT HALF, LATIN SMALL LETTER B +0061 FE21 0315 0300 05AE 0062;0061 05AE FE21 0300 0315 0062;0061 05AE FE21 0300 0315 0062;0061 05AE FE21 0300 0315 0062;0061 05AE FE21 0300 0315 0062; # (a◌︡◌̕◌̀◌֮b; a◌֮◌︡◌̀◌̕b; a◌֮◌︡◌̀◌̕b; a◌֮◌︡◌̀◌̕b; a◌֮◌︡◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING LIGATURE RIGHT HALF, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE FE22 0062;00E0 05AE FE22 0315 0062;0061 05AE 0300 FE22 0315 0062;00E0 05AE FE22 0315 0062;0061 05AE 0300 FE22 0315 0062; # (a◌̕◌̀◌֮◌︢b; à◌֮◌︢◌̕b; a◌֮◌̀◌︢◌̕b; à◌֮◌︢◌̕b; a◌֮◌̀◌︢◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING DOUBLE TILDE LEFT HALF, LATIN SMALL LETTER B +0061 FE22 0315 0300 05AE 0062;0061 05AE FE22 0300 0315 0062;0061 05AE FE22 0300 0315 0062;0061 05AE FE22 0300 0315 0062;0061 05AE FE22 0300 0315 0062; # (a◌︢◌̕◌̀◌֮b; a◌֮◌︢◌̀◌̕b; a◌֮◌︢◌̀◌̕b; a◌֮◌︢◌̀◌̕b; a◌֮◌︢◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING DOUBLE TILDE LEFT HALF, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE FE23 0062;00E0 05AE FE23 0315 0062;0061 05AE 0300 FE23 0315 0062;00E0 05AE FE23 0315 0062;0061 05AE 0300 FE23 0315 0062; # (a◌̕◌̀◌֮◌︣b; à◌֮◌︣◌̕b; a◌֮◌̀◌︣◌̕b; à◌֮◌︣◌̕b; a◌֮◌̀◌︣◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, COMBINING DOUBLE TILDE RIGHT HALF, LATIN SMALL LETTER B +0061 FE23 0315 0300 05AE 0062;0061 05AE FE23 0300 0315 0062;0061 05AE FE23 0300 0315 0062;0061 05AE FE23 0300 0315 0062;0061 05AE FE23 0300 0315 0062; # (a◌︣◌̕◌̀◌֮b; a◌֮◌︣◌̀◌̕b; a◌֮◌︣◌̀◌̕b; a◌֮◌︣◌̀◌̕b; a◌֮◌︣◌̀◌̕b; ) LATIN SMALL LETTER A, COMBINING DOUBLE TILDE RIGHT HALF, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 302A 031B 0321 1D165 0062;0061 0321 031B 1D165 302A 0062;0061 0321 031B 1D165 302A 0062;0061 0321 031B 1D165 302A 0062;0061 0321 031B 1D165 302A 0062; # (a◌〪◌̛◌̡𝅥𝅥b; a◌̡◌̛𝅥𝅥◌〪b; a◌̡◌̛𝅥𝅥◌〪b; a◌̡◌̛𝅥𝅥◌〪b; a◌̡◌̛𝅥𝅥◌〪b; ) LATIN SMALL LETTER A, IDEOGRAPHIC LEVEL TONE MARK, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, MUSICAL SYMBOL COMBINING STEM, LATIN SMALL LETTER B +0061 1D165 302A 031B 0321 0062;0061 0321 1D165 031B 302A 0062;0061 0321 1D165 031B 302A 0062;0061 0321 1D165 031B 302A 0062;0061 0321 1D165 031B 302A 0062; # (a𝅥𝅥◌〪◌̛◌̡b; a◌̡𝅥𝅥◌̛◌〪b; a◌̡𝅥𝅥◌̛◌〪b; a◌̡𝅥𝅥◌̛◌〪b; a◌̡𝅥𝅥◌̛◌〪b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING STEM, IDEOGRAPHIC LEVEL TONE MARK, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, LATIN SMALL LETTER B +0061 302A 031B 0321 1D166 0062;0061 0321 031B 1D166 302A 0062;0061 0321 031B 1D166 302A 0062;0061 0321 031B 1D166 302A 0062;0061 0321 031B 1D166 302A 0062; # (a◌〪◌̛◌̡𝅦𝅦b; a◌̡◌̛𝅦𝅦◌〪b; a◌̡◌̛𝅦𝅦◌〪b; a◌̡◌̛𝅦𝅦◌〪b; a◌̡◌̛𝅦𝅦◌〪b; ) LATIN SMALL LETTER A, IDEOGRAPHIC LEVEL TONE MARK, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, MUSICAL SYMBOL COMBINING SPRECHGESANG STEM, LATIN SMALL LETTER B +0061 1D166 302A 031B 0321 0062;0061 0321 1D166 031B 302A 0062;0061 0321 1D166 031B 302A 0062;0061 0321 1D166 031B 302A 0062;0061 0321 1D166 031B 302A 0062; # (a𝅦𝅦◌〪◌̛◌̡b; a◌̡𝅦𝅦◌̛◌〪b; a◌̡𝅦𝅦◌̛◌〪b; a◌̡𝅦𝅦◌̛◌〪b; a◌̡𝅦𝅦◌̛◌〪b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING SPRECHGESANG STEM, IDEOGRAPHIC LEVEL TONE MARK, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, LATIN SMALL LETTER B +0061 093C 0334 1D167 0062;0061 0334 1D167 093C 0062;0061 0334 1D167 093C 0062;0061 0334 1D167 093C 0062;0061 0334 1D167 093C 0062; # (a◌़◌̴◌𝅧◌𝅧b; a◌̴◌𝅧◌𝅧◌़b; a◌̴◌𝅧◌𝅧◌़b; a◌̴◌𝅧◌𝅧◌़b; a◌̴◌𝅧◌𝅧◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, MUSICAL SYMBOL COMBINING TREMOLO-1, LATIN SMALL LETTER B +0061 1D167 093C 0334 0062;0061 1D167 0334 093C 0062;0061 1D167 0334 093C 0062;0061 1D167 0334 093C 0062;0061 1D167 0334 093C 0062; # (a◌𝅧◌𝅧◌़◌̴b; a◌𝅧◌𝅧◌̴◌़b; a◌𝅧◌𝅧◌̴◌़b; a◌𝅧◌𝅧◌̴◌़b; a◌𝅧◌𝅧◌̴◌़b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING TREMOLO-1, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 093C 0334 1D168 0062;0061 0334 1D168 093C 0062;0061 0334 1D168 093C 0062;0061 0334 1D168 093C 0062;0061 0334 1D168 093C 0062; # (a◌़◌̴◌𝅨◌𝅨b; a◌̴◌𝅨◌𝅨◌़b; a◌̴◌𝅨◌𝅨◌़b; a◌̴◌𝅨◌𝅨◌़b; a◌̴◌𝅨◌𝅨◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, MUSICAL SYMBOL COMBINING TREMOLO-2, LATIN SMALL LETTER B +0061 1D168 093C 0334 0062;0061 1D168 0334 093C 0062;0061 1D168 0334 093C 0062;0061 1D168 0334 093C 0062;0061 1D168 0334 093C 0062; # (a◌𝅨◌𝅨◌़◌̴b; a◌𝅨◌𝅨◌̴◌़b; a◌𝅨◌𝅨◌̴◌़b; a◌𝅨◌𝅨◌̴◌़b; a◌𝅨◌𝅨◌̴◌़b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING TREMOLO-2, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 093C 0334 1D169 0062;0061 0334 1D169 093C 0062;0061 0334 1D169 093C 0062;0061 0334 1D169 093C 0062;0061 0334 1D169 093C 0062; # (a◌़◌̴◌𝅩◌𝅩b; a◌̴◌𝅩◌𝅩◌़b; a◌̴◌𝅩◌𝅩◌़b; a◌̴◌𝅩◌𝅩◌़b; a◌̴◌𝅩◌𝅩◌़b; ) LATIN SMALL LETTER A, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, MUSICAL SYMBOL COMBINING TREMOLO-3, LATIN SMALL LETTER B +0061 1D169 093C 0334 0062;0061 1D169 0334 093C 0062;0061 1D169 0334 093C 0062;0061 1D169 0334 093C 0062;0061 1D169 0334 093C 0062; # (a◌𝅩◌𝅩◌़◌̴b; a◌𝅩◌𝅩◌̴◌़b; a◌𝅩◌𝅩◌̴◌़b; a◌𝅩◌𝅩◌̴◌़b; a◌𝅩◌𝅩◌̴◌़b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING TREMOLO-3, DEVANAGARI SIGN NUKTA, COMBINING TILDE OVERLAY, LATIN SMALL LETTER B +0061 05AE 1D16D 302E 1D16D 0062;0061 302E 1D16D 1D16D 05AE 0062;0061 302E 1D16D 1D16D 05AE 0062;0061 302E 1D16D 1D16D 05AE 0062;0061 302E 1D16D 1D16D 05AE 0062; # (a◌𝅭𝅭֮◌〮𝅭𝅭b; a◌〮𝅭𝅭𝅭𝅭◌֮b; a◌〮𝅭𝅭𝅭𝅭◌֮b; a◌〮𝅭𝅭𝅭𝅭◌֮b; a◌〮𝅭𝅭𝅭𝅭◌֮b; ) LATIN SMALL LETTER A, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, HANGUL SINGLE DOT TONE MARK, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, LATIN SMALL LETTER B +0061 1D16D 05AE 1D16D 302E 0062;0061 302E 1D16D 1D16D 05AE 0062;0061 302E 1D16D 1D16D 05AE 0062;0061 302E 1D16D 1D16D 05AE 0062;0061 302E 1D16D 1D16D 05AE 0062; # (a𝅭𝅭◌𝅭𝅭֮◌〮b; a◌〮𝅭𝅭𝅭𝅭◌֮b; a◌〮𝅭𝅭𝅭𝅭◌֮b; a◌〮𝅭𝅭𝅭𝅭◌֮b; a◌〮𝅭𝅭𝅭𝅭◌֮b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING AUGMENTATION DOT, HANGUL SINGLE DOT TONE MARK, LATIN SMALL LETTER B +0061 302A 031B 0321 1D16E 0062;0061 0321 031B 1D16E 302A 0062;0061 0321 031B 1D16E 302A 0062;0061 0321 031B 1D16E 302A 0062;0061 0321 031B 1D16E 302A 0062; # (a◌〪◌̛◌̡𝅮𝅮b; a◌̡◌̛𝅮𝅮◌〪b; a◌̡◌̛𝅮𝅮◌〪b; a◌̡◌̛𝅮𝅮◌〪b; a◌̡◌̛𝅮𝅮◌〪b; ) LATIN SMALL LETTER A, IDEOGRAPHIC LEVEL TONE MARK, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, MUSICAL SYMBOL COMBINING FLAG-1, LATIN SMALL LETTER B +0061 1D16E 302A 031B 0321 0062;0061 0321 1D16E 031B 302A 0062;0061 0321 1D16E 031B 302A 0062;0061 0321 1D16E 031B 302A 0062;0061 0321 1D16E 031B 302A 0062; # (a𝅮𝅮◌〪◌̛◌̡b; a◌̡𝅮𝅮◌̛◌〪b; a◌̡𝅮𝅮◌̛◌〪b; a◌̡𝅮𝅮◌̛◌〪b; a◌̡𝅮𝅮◌̛◌〪b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING FLAG-1, IDEOGRAPHIC LEVEL TONE MARK, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, LATIN SMALL LETTER B +0061 302A 031B 0321 1D16F 0062;0061 0321 031B 1D16F 302A 0062;0061 0321 031B 1D16F 302A 0062;0061 0321 031B 1D16F 302A 0062;0061 0321 031B 1D16F 302A 0062; # (a◌〪◌̛◌̡𝅯𝅯b; a◌̡◌̛𝅯𝅯◌〪b; a◌̡◌̛𝅯𝅯◌〪b; a◌̡◌̛𝅯𝅯◌〪b; a◌̡◌̛𝅯𝅯◌〪b; ) LATIN SMALL LETTER A, IDEOGRAPHIC LEVEL TONE MARK, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, MUSICAL SYMBOL COMBINING FLAG-2, LATIN SMALL LETTER B +0061 1D16F 302A 031B 0321 0062;0061 0321 1D16F 031B 302A 0062;0061 0321 1D16F 031B 302A 0062;0061 0321 1D16F 031B 302A 0062;0061 0321 1D16F 031B 302A 0062; # (a𝅯𝅯◌〪◌̛◌̡b; a◌̡𝅯𝅯◌̛◌〪b; a◌̡𝅯𝅯◌̛◌〪b; a◌̡𝅯𝅯◌̛◌〪b; a◌̡𝅯𝅯◌̛◌〪b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING FLAG-2, IDEOGRAPHIC LEVEL TONE MARK, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, LATIN SMALL LETTER B +0061 302A 031B 0321 1D170 0062;0061 0321 031B 1D170 302A 0062;0061 0321 031B 1D170 302A 0062;0061 0321 031B 1D170 302A 0062;0061 0321 031B 1D170 302A 0062; # (a◌〪◌̛◌̡𝅰𝅰b; a◌̡◌̛𝅰𝅰◌〪b; a◌̡◌̛𝅰𝅰◌〪b; a◌̡◌̛𝅰𝅰◌〪b; a◌̡◌̛𝅰𝅰◌〪b; ) LATIN SMALL LETTER A, IDEOGRAPHIC LEVEL TONE MARK, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, MUSICAL SYMBOL COMBINING FLAG-3, LATIN SMALL LETTER B +0061 1D170 302A 031B 0321 0062;0061 0321 1D170 031B 302A 0062;0061 0321 1D170 031B 302A 0062;0061 0321 1D170 031B 302A 0062;0061 0321 1D170 031B 302A 0062; # (a𝅰𝅰◌〪◌̛◌̡b; a◌̡𝅰𝅰◌̛◌〪b; a◌̡𝅰𝅰◌̛◌〪b; a◌̡𝅰𝅰◌̛◌〪b; a◌̡𝅰𝅰◌̛◌〪b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING FLAG-3, IDEOGRAPHIC LEVEL TONE MARK, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, LATIN SMALL LETTER B +0061 302A 031B 0321 1D171 0062;0061 0321 031B 1D171 302A 0062;0061 0321 031B 1D171 302A 0062;0061 0321 031B 1D171 302A 0062;0061 0321 031B 1D171 302A 0062; # (a◌〪◌̛◌̡𝅱𝅱b; a◌̡◌̛𝅱𝅱◌〪b; a◌̡◌̛𝅱𝅱◌〪b; a◌̡◌̛𝅱𝅱◌〪b; a◌̡◌̛𝅱𝅱◌〪b; ) LATIN SMALL LETTER A, IDEOGRAPHIC LEVEL TONE MARK, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, MUSICAL SYMBOL COMBINING FLAG-4, LATIN SMALL LETTER B +0061 1D171 302A 031B 0321 0062;0061 0321 1D171 031B 302A 0062;0061 0321 1D171 031B 302A 0062;0061 0321 1D171 031B 302A 0062;0061 0321 1D171 031B 302A 0062; # (a𝅱𝅱◌〪◌̛◌̡b; a◌̡𝅱𝅱◌̛◌〪b; a◌̡𝅱𝅱◌̛◌〪b; a◌̡𝅱𝅱◌̛◌〪b; a◌̡𝅱𝅱◌̛◌〪b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING FLAG-4, IDEOGRAPHIC LEVEL TONE MARK, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, LATIN SMALL LETTER B +0061 302A 031B 0321 1D172 0062;0061 0321 031B 1D172 302A 0062;0061 0321 031B 1D172 302A 0062;0061 0321 031B 1D172 302A 0062;0061 0321 031B 1D172 302A 0062; # (a◌〪◌̛◌̡𝅲𝅲b; a◌̡◌̛𝅲𝅲◌〪b; a◌̡◌̛𝅲𝅲◌〪b; a◌̡◌̛𝅲𝅲◌〪b; a◌̡◌̛𝅲𝅲◌〪b; ) LATIN SMALL LETTER A, IDEOGRAPHIC LEVEL TONE MARK, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, MUSICAL SYMBOL COMBINING FLAG-5, LATIN SMALL LETTER B +0061 1D172 302A 031B 0321 0062;0061 0321 1D172 031B 302A 0062;0061 0321 1D172 031B 302A 0062;0061 0321 1D172 031B 302A 0062;0061 0321 1D172 031B 302A 0062; # (a𝅲𝅲◌〪◌̛◌̡b; a◌̡𝅲𝅲◌̛◌〪b; a◌̡𝅲𝅲◌̛◌〪b; a◌̡𝅲𝅲◌̛◌〪b; a◌̡𝅲𝅲◌̛◌〪b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING FLAG-5, IDEOGRAPHIC LEVEL TONE MARK, COMBINING HORN, COMBINING PALATALIZED HOOK BELOW, LATIN SMALL LETTER B +0061 059A 0316 302A 1D17B 0062;0061 302A 0316 1D17B 059A 0062;0061 302A 0316 1D17B 059A 0062;0061 302A 0316 1D17B 059A 0062;0061 302A 0316 1D17B 059A 0062; # (a◌֚◌̖◌〪◌𝅻◌𝅻b; a◌〪◌̖◌𝅻◌𝅻◌֚b; a◌〪◌̖◌𝅻◌𝅻◌֚b; a◌〪◌̖◌𝅻◌𝅻◌֚b; a◌〪◌̖◌𝅻◌𝅻◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, MUSICAL SYMBOL COMBINING ACCENT, LATIN SMALL LETTER B +0061 1D17B 059A 0316 302A 0062;0061 302A 1D17B 0316 059A 0062;0061 302A 1D17B 0316 059A 0062;0061 302A 1D17B 0316 059A 0062;0061 302A 1D17B 0316 059A 0062; # (a◌𝅻◌𝅻◌֚◌̖◌〪b; a◌〪◌𝅻◌𝅻◌̖◌֚b; a◌〪◌𝅻◌𝅻◌̖◌֚b; a◌〪◌𝅻◌𝅻◌̖◌֚b; a◌〪◌𝅻◌𝅻◌̖◌֚b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING ACCENT, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 1D17C 0062;0061 302A 0316 1D17C 059A 0062;0061 302A 0316 1D17C 059A 0062;0061 302A 0316 1D17C 059A 0062;0061 302A 0316 1D17C 059A 0062; # (a◌֚◌̖◌〪◌𝅼◌𝅼b; a◌〪◌̖◌𝅼◌𝅼◌֚b; a◌〪◌̖◌𝅼◌𝅼◌֚b; a◌〪◌̖◌𝅼◌𝅼◌֚b; a◌〪◌̖◌𝅼◌𝅼◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, MUSICAL SYMBOL COMBINING STACCATO, LATIN SMALL LETTER B +0061 1D17C 059A 0316 302A 0062;0061 302A 1D17C 0316 059A 0062;0061 302A 1D17C 0316 059A 0062;0061 302A 1D17C 0316 059A 0062;0061 302A 1D17C 0316 059A 0062; # (a◌𝅼◌𝅼◌֚◌̖◌〪b; a◌〪◌𝅼◌𝅼◌̖◌֚b; a◌〪◌𝅼◌𝅼◌̖◌֚b; a◌〪◌𝅼◌𝅼◌̖◌֚b; a◌〪◌𝅼◌𝅼◌̖◌֚b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING STACCATO, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 1D17D 0062;0061 302A 0316 1D17D 059A 0062;0061 302A 0316 1D17D 059A 0062;0061 302A 0316 1D17D 059A 0062;0061 302A 0316 1D17D 059A 0062; # (a◌֚◌̖◌〪◌𝅽◌𝅽b; a◌〪◌̖◌𝅽◌𝅽◌֚b; a◌〪◌̖◌𝅽◌𝅽◌֚b; a◌〪◌̖◌𝅽◌𝅽◌֚b; a◌〪◌̖◌𝅽◌𝅽◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, MUSICAL SYMBOL COMBINING TENUTO, LATIN SMALL LETTER B +0061 1D17D 059A 0316 302A 0062;0061 302A 1D17D 0316 059A 0062;0061 302A 1D17D 0316 059A 0062;0061 302A 1D17D 0316 059A 0062;0061 302A 1D17D 0316 059A 0062; # (a◌𝅽◌𝅽◌֚◌̖◌〪b; a◌〪◌𝅽◌𝅽◌̖◌֚b; a◌〪◌𝅽◌𝅽◌̖◌֚b; a◌〪◌𝅽◌𝅽◌̖◌֚b; a◌〪◌𝅽◌𝅽◌̖◌֚b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING TENUTO, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 1D17E 0062;0061 302A 0316 1D17E 059A 0062;0061 302A 0316 1D17E 059A 0062;0061 302A 0316 1D17E 059A 0062;0061 302A 0316 1D17E 059A 0062; # (a◌֚◌̖◌〪◌𝅾◌𝅾b; a◌〪◌̖◌𝅾◌𝅾◌֚b; a◌〪◌̖◌𝅾◌𝅾◌֚b; a◌〪◌̖◌𝅾◌𝅾◌֚b; a◌〪◌̖◌𝅾◌𝅾◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, MUSICAL SYMBOL COMBINING STACCATISSIMO, LATIN SMALL LETTER B +0061 1D17E 059A 0316 302A 0062;0061 302A 1D17E 0316 059A 0062;0061 302A 1D17E 0316 059A 0062;0061 302A 1D17E 0316 059A 0062;0061 302A 1D17E 0316 059A 0062; # (a◌𝅾◌𝅾◌֚◌̖◌〪b; a◌〪◌𝅾◌𝅾◌̖◌֚b; a◌〪◌𝅾◌𝅾◌̖◌֚b; a◌〪◌𝅾◌𝅾◌̖◌֚b; a◌〪◌𝅾◌𝅾◌̖◌֚b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING STACCATISSIMO, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 1D17F 0062;0061 302A 0316 1D17F 059A 0062;0061 302A 0316 1D17F 059A 0062;0061 302A 0316 1D17F 059A 0062;0061 302A 0316 1D17F 059A 0062; # (a◌֚◌̖◌〪◌𝅿◌𝅿b; a◌〪◌̖◌𝅿◌𝅿◌֚b; a◌〪◌̖◌𝅿◌𝅿◌֚b; a◌〪◌̖◌𝅿◌𝅿◌֚b; a◌〪◌̖◌𝅿◌𝅿◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, MUSICAL SYMBOL COMBINING MARCATO, LATIN SMALL LETTER B +0061 1D17F 059A 0316 302A 0062;0061 302A 1D17F 0316 059A 0062;0061 302A 1D17F 0316 059A 0062;0061 302A 1D17F 0316 059A 0062;0061 302A 1D17F 0316 059A 0062; # (a◌𝅿◌𝅿◌֚◌̖◌〪b; a◌〪◌𝅿◌𝅿◌̖◌֚b; a◌〪◌𝅿◌𝅿◌̖◌֚b; a◌〪◌𝅿◌𝅿◌̖◌֚b; a◌〪◌𝅿◌𝅿◌̖◌֚b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING MARCATO, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 1D180 0062;0061 302A 0316 1D180 059A 0062;0061 302A 0316 1D180 059A 0062;0061 302A 0316 1D180 059A 0062;0061 302A 0316 1D180 059A 0062; # (a◌֚◌̖◌〪◌𝆀◌𝆀b; a◌〪◌̖◌𝆀◌𝆀◌֚b; a◌〪◌̖◌𝆀◌𝆀◌֚b; a◌〪◌̖◌𝆀◌𝆀◌֚b; a◌〪◌̖◌𝆀◌𝆀◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, MUSICAL SYMBOL COMBINING MARCATO-STACCATO, LATIN SMALL LETTER B +0061 1D180 059A 0316 302A 0062;0061 302A 1D180 0316 059A 0062;0061 302A 1D180 0316 059A 0062;0061 302A 1D180 0316 059A 0062;0061 302A 1D180 0316 059A 0062; # (a◌𝆀◌𝆀◌֚◌̖◌〪b; a◌〪◌𝆀◌𝆀◌̖◌֚b; a◌〪◌𝆀◌𝆀◌̖◌֚b; a◌〪◌𝆀◌𝆀◌̖◌֚b; a◌〪◌𝆀◌𝆀◌̖◌֚b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING MARCATO-STACCATO, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 1D181 0062;0061 302A 0316 1D181 059A 0062;0061 302A 0316 1D181 059A 0062;0061 302A 0316 1D181 059A 0062;0061 302A 0316 1D181 059A 0062; # (a◌֚◌̖◌〪◌𝆁◌𝆁b; a◌〪◌̖◌𝆁◌𝆁◌֚b; a◌〪◌̖◌𝆁◌𝆁◌֚b; a◌〪◌̖◌𝆁◌𝆁◌֚b; a◌〪◌̖◌𝆁◌𝆁◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, MUSICAL SYMBOL COMBINING ACCENT-STACCATO, LATIN SMALL LETTER B +0061 1D181 059A 0316 302A 0062;0061 302A 1D181 0316 059A 0062;0061 302A 1D181 0316 059A 0062;0061 302A 1D181 0316 059A 0062;0061 302A 1D181 0316 059A 0062; # (a◌𝆁◌𝆁◌֚◌̖◌〪b; a◌〪◌𝆁◌𝆁◌̖◌֚b; a◌〪◌𝆁◌𝆁◌̖◌֚b; a◌〪◌𝆁◌𝆁◌̖◌֚b; a◌〪◌𝆁◌𝆁◌̖◌֚b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING ACCENT-STACCATO, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 1D182 0062;0061 302A 0316 1D182 059A 0062;0061 302A 0316 1D182 059A 0062;0061 302A 0316 1D182 059A 0062;0061 302A 0316 1D182 059A 0062; # (a◌֚◌̖◌〪◌𝆂◌𝆂b; a◌〪◌̖◌𝆂◌𝆂◌֚b; a◌〪◌̖◌𝆂◌𝆂◌֚b; a◌〪◌̖◌𝆂◌𝆂◌֚b; a◌〪◌̖◌𝆂◌𝆂◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, MUSICAL SYMBOL COMBINING LOURE, LATIN SMALL LETTER B +0061 1D182 059A 0316 302A 0062;0061 302A 1D182 0316 059A 0062;0061 302A 1D182 0316 059A 0062;0061 302A 1D182 0316 059A 0062;0061 302A 1D182 0316 059A 0062; # (a◌𝆂◌𝆂◌֚◌̖◌〪b; a◌〪◌𝆂◌𝆂◌̖◌֚b; a◌〪◌𝆂◌𝆂◌̖◌֚b; a◌〪◌𝆂◌𝆂◌̖◌֚b; a◌〪◌𝆂◌𝆂◌̖◌֚b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING LOURE, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 1D185 0062;00E0 05AE 1D185 0315 0062;0061 05AE 0300 1D185 0315 0062;00E0 05AE 1D185 0315 0062;0061 05AE 0300 1D185 0315 0062; # (a◌̕◌̀◌֮◌𝆅◌𝆅b; à◌֮◌𝆅◌𝆅◌̕b; a◌֮◌̀◌𝆅◌𝆅◌̕b; à◌֮◌𝆅◌𝆅◌̕b; a◌֮◌̀◌𝆅◌𝆅◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING DOIT, LATIN SMALL LETTER B +0061 1D185 0315 0300 05AE 0062;0061 05AE 1D185 0300 0315 0062;0061 05AE 1D185 0300 0315 0062;0061 05AE 1D185 0300 0315 0062;0061 05AE 1D185 0300 0315 0062; # (a◌𝆅◌𝆅◌̕◌̀◌֮b; a◌֮◌𝆅◌𝆅◌̀◌̕b; a◌֮◌𝆅◌𝆅◌̀◌̕b; a◌֮◌𝆅◌𝆅◌̀◌̕b; a◌֮◌𝆅◌𝆅◌̀◌̕b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING DOIT, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 1D186 0062;00E0 05AE 1D186 0315 0062;0061 05AE 0300 1D186 0315 0062;00E0 05AE 1D186 0315 0062;0061 05AE 0300 1D186 0315 0062; # (a◌̕◌̀◌֮◌𝆆◌𝆆b; à◌֮◌𝆆◌𝆆◌̕b; a◌֮◌̀◌𝆆◌𝆆◌̕b; à◌֮◌𝆆◌𝆆◌̕b; a◌֮◌̀◌𝆆◌𝆆◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING RIP, LATIN SMALL LETTER B +0061 1D186 0315 0300 05AE 0062;0061 05AE 1D186 0300 0315 0062;0061 05AE 1D186 0300 0315 0062;0061 05AE 1D186 0300 0315 0062;0061 05AE 1D186 0300 0315 0062; # (a◌𝆆◌𝆆◌̕◌̀◌֮b; a◌֮◌𝆆◌𝆆◌̀◌̕b; a◌֮◌𝆆◌𝆆◌̀◌̕b; a◌֮◌𝆆◌𝆆◌̀◌̕b; a◌֮◌𝆆◌𝆆◌̀◌̕b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING RIP, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 1D187 0062;00E0 05AE 1D187 0315 0062;0061 05AE 0300 1D187 0315 0062;00E0 05AE 1D187 0315 0062;0061 05AE 0300 1D187 0315 0062; # (a◌̕◌̀◌֮◌𝆇◌𝆇b; à◌֮◌𝆇◌𝆇◌̕b; a◌֮◌̀◌𝆇◌𝆇◌̕b; à◌֮◌𝆇◌𝆇◌̕b; a◌֮◌̀◌𝆇◌𝆇◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING FLIP, LATIN SMALL LETTER B +0061 1D187 0315 0300 05AE 0062;0061 05AE 1D187 0300 0315 0062;0061 05AE 1D187 0300 0315 0062;0061 05AE 1D187 0300 0315 0062;0061 05AE 1D187 0300 0315 0062; # (a◌𝆇◌𝆇◌̕◌̀◌֮b; a◌֮◌𝆇◌𝆇◌̀◌̕b; a◌֮◌𝆇◌𝆇◌̀◌̕b; a◌֮◌𝆇◌𝆇◌̀◌̕b; a◌֮◌𝆇◌𝆇◌̀◌̕b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING FLIP, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 1D188 0062;00E0 05AE 1D188 0315 0062;0061 05AE 0300 1D188 0315 0062;00E0 05AE 1D188 0315 0062;0061 05AE 0300 1D188 0315 0062; # (a◌̕◌̀◌֮◌𝆈◌𝆈b; à◌֮◌𝆈◌𝆈◌̕b; a◌֮◌̀◌𝆈◌𝆈◌̕b; à◌֮◌𝆈◌𝆈◌̕b; a◌֮◌̀◌𝆈◌𝆈◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING SMEAR, LATIN SMALL LETTER B +0061 1D188 0315 0300 05AE 0062;0061 05AE 1D188 0300 0315 0062;0061 05AE 1D188 0300 0315 0062;0061 05AE 1D188 0300 0315 0062;0061 05AE 1D188 0300 0315 0062; # (a◌𝆈◌𝆈◌̕◌̀◌֮b; a◌֮◌𝆈◌𝆈◌̀◌̕b; a◌֮◌𝆈◌𝆈◌̀◌̕b; a◌֮◌𝆈◌𝆈◌̀◌̕b; a◌֮◌𝆈◌𝆈◌̀◌̕b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING SMEAR, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 1D189 0062;00E0 05AE 1D189 0315 0062;0061 05AE 0300 1D189 0315 0062;00E0 05AE 1D189 0315 0062;0061 05AE 0300 1D189 0315 0062; # (a◌̕◌̀◌֮◌𝆉◌𝆉b; à◌֮◌𝆉◌𝆉◌̕b; a◌֮◌̀◌𝆉◌𝆉◌̕b; à◌֮◌𝆉◌𝆉◌̕b; a◌֮◌̀◌𝆉◌𝆉◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING BEND, LATIN SMALL LETTER B +0061 1D189 0315 0300 05AE 0062;0061 05AE 1D189 0300 0315 0062;0061 05AE 1D189 0300 0315 0062;0061 05AE 1D189 0300 0315 0062;0061 05AE 1D189 0300 0315 0062; # (a◌𝆉◌𝆉◌̕◌̀◌֮b; a◌֮◌𝆉◌𝆉◌̀◌̕b; a◌֮◌𝆉◌𝆉◌̀◌̕b; a◌֮◌𝆉◌𝆉◌̀◌̕b; a◌֮◌𝆉◌𝆉◌̀◌̕b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING BEND, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 059A 0316 302A 1D18A 0062;0061 302A 0316 1D18A 059A 0062;0061 302A 0316 1D18A 059A 0062;0061 302A 0316 1D18A 059A 0062;0061 302A 0316 1D18A 059A 0062; # (a◌֚◌̖◌〪◌𝆊◌𝆊b; a◌〪◌̖◌𝆊◌𝆊◌֚b; a◌〪◌̖◌𝆊◌𝆊◌֚b; a◌〪◌̖◌𝆊◌𝆊◌֚b; a◌〪◌̖◌𝆊◌𝆊◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, MUSICAL SYMBOL COMBINING DOUBLE TONGUE, LATIN SMALL LETTER B +0061 1D18A 059A 0316 302A 0062;0061 302A 1D18A 0316 059A 0062;0061 302A 1D18A 0316 059A 0062;0061 302A 1D18A 0316 059A 0062;0061 302A 1D18A 0316 059A 0062; # (a◌𝆊◌𝆊◌֚◌̖◌〪b; a◌〪◌𝆊◌𝆊◌̖◌֚b; a◌〪◌𝆊◌𝆊◌̖◌֚b; a◌〪◌𝆊◌𝆊◌̖◌֚b; a◌〪◌𝆊◌𝆊◌̖◌֚b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING DOUBLE TONGUE, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 059A 0316 302A 1D18B 0062;0061 302A 0316 1D18B 059A 0062;0061 302A 0316 1D18B 059A 0062;0061 302A 0316 1D18B 059A 0062;0061 302A 0316 1D18B 059A 0062; # (a◌֚◌̖◌〪◌𝆋◌𝆋b; a◌〪◌̖◌𝆋◌𝆋◌֚b; a◌〪◌̖◌𝆋◌𝆋◌֚b; a◌〪◌̖◌𝆋◌𝆋◌֚b; a◌〪◌̖◌𝆋◌𝆋◌֚b; ) LATIN SMALL LETTER A, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, MUSICAL SYMBOL COMBINING TRIPLE TONGUE, LATIN SMALL LETTER B +0061 1D18B 059A 0316 302A 0062;0061 302A 1D18B 0316 059A 0062;0061 302A 1D18B 0316 059A 0062;0061 302A 1D18B 0316 059A 0062;0061 302A 1D18B 0316 059A 0062; # (a◌𝆋◌𝆋◌֚◌̖◌〪b; a◌〪◌𝆋◌𝆋◌̖◌֚b; a◌〪◌𝆋◌𝆋◌̖◌֚b; a◌〪◌𝆋◌𝆋◌̖◌֚b; a◌〪◌𝆋◌𝆋◌̖◌֚b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING TRIPLE TONGUE, HEBREW ACCENT YETIV, COMBINING GRAVE ACCENT BELOW, IDEOGRAPHIC LEVEL TONE MARK, LATIN SMALL LETTER B +0061 0315 0300 05AE 1D1AA 0062;00E0 05AE 1D1AA 0315 0062;0061 05AE 0300 1D1AA 0315 0062;00E0 05AE 1D1AA 0315 0062;0061 05AE 0300 1D1AA 0315 0062; # (a◌̕◌̀◌֮◌𝆪◌𝆪b; à◌֮◌𝆪◌𝆪◌̕b; a◌֮◌̀◌𝆪◌𝆪◌̕b; à◌֮◌𝆪◌𝆪◌̕b; a◌֮◌̀◌𝆪◌𝆪◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING DOWN BOW, LATIN SMALL LETTER B +0061 1D1AA 0315 0300 05AE 0062;0061 05AE 1D1AA 0300 0315 0062;0061 05AE 1D1AA 0300 0315 0062;0061 05AE 1D1AA 0300 0315 0062;0061 05AE 1D1AA 0300 0315 0062; # (a◌𝆪◌𝆪◌̕◌̀◌֮b; a◌֮◌𝆪◌𝆪◌̀◌̕b; a◌֮◌𝆪◌𝆪◌̀◌̕b; a◌֮◌𝆪◌𝆪◌̀◌̕b; a◌֮◌𝆪◌𝆪◌̀◌̕b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING DOWN BOW, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 1D1AB 0062;00E0 05AE 1D1AB 0315 0062;0061 05AE 0300 1D1AB 0315 0062;00E0 05AE 1D1AB 0315 0062;0061 05AE 0300 1D1AB 0315 0062; # (a◌̕◌̀◌֮◌𝆫◌𝆫b; à◌֮◌𝆫◌𝆫◌̕b; a◌֮◌̀◌𝆫◌𝆫◌̕b; à◌֮◌𝆫◌𝆫◌̕b; a◌֮◌̀◌𝆫◌𝆫◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING UP BOW, LATIN SMALL LETTER B +0061 1D1AB 0315 0300 05AE 0062;0061 05AE 1D1AB 0300 0315 0062;0061 05AE 1D1AB 0300 0315 0062;0061 05AE 1D1AB 0300 0315 0062;0061 05AE 1D1AB 0300 0315 0062; # (a◌𝆫◌𝆫◌̕◌̀◌֮b; a◌֮◌𝆫◌𝆫◌̀◌̕b; a◌֮◌𝆫◌𝆫◌̀◌̕b; a◌֮◌𝆫◌𝆫◌̀◌̕b; a◌֮◌𝆫◌𝆫◌̀◌̕b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING UP BOW, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 1D1AC 0062;00E0 05AE 1D1AC 0315 0062;0061 05AE 0300 1D1AC 0315 0062;00E0 05AE 1D1AC 0315 0062;0061 05AE 0300 1D1AC 0315 0062; # (a◌̕◌̀◌֮◌𝆬◌𝆬b; à◌֮◌𝆬◌𝆬◌̕b; a◌֮◌̀◌𝆬◌𝆬◌̕b; à◌֮◌𝆬◌𝆬◌̕b; a◌֮◌̀◌𝆬◌𝆬◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING HARMONIC, LATIN SMALL LETTER B +0061 1D1AC 0315 0300 05AE 0062;0061 05AE 1D1AC 0300 0315 0062;0061 05AE 1D1AC 0300 0315 0062;0061 05AE 1D1AC 0300 0315 0062;0061 05AE 1D1AC 0300 0315 0062; # (a◌𝆬◌𝆬◌̕◌̀◌֮b; a◌֮◌𝆬◌𝆬◌̀◌̕b; a◌֮◌𝆬◌𝆬◌̀◌̕b; a◌֮◌𝆬◌𝆬◌̀◌̕b; a◌֮◌𝆬◌𝆬◌̀◌̕b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING HARMONIC, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +0061 0315 0300 05AE 1D1AD 0062;00E0 05AE 1D1AD 0315 0062;0061 05AE 0300 1D1AD 0315 0062;00E0 05AE 1D1AD 0315 0062;0061 05AE 0300 1D1AD 0315 0062; # (a◌̕◌̀◌֮◌𝆭◌𝆭b; à◌֮◌𝆭◌𝆭◌̕b; a◌֮◌̀◌𝆭◌𝆭◌̕b; à◌֮◌𝆭◌𝆭◌̕b; a◌֮◌̀◌𝆭◌𝆭◌̕b; ) LATIN SMALL LETTER A, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, MUSICAL SYMBOL COMBINING SNAP PIZZICATO, LATIN SMALL LETTER B +0061 1D1AD 0315 0300 05AE 0062;0061 05AE 1D1AD 0300 0315 0062;0061 05AE 1D1AD 0300 0315 0062;0061 05AE 1D1AD 0300 0315 0062;0061 05AE 1D1AD 0300 0315 0062; # (a◌𝆭◌𝆭◌̕◌̀◌֮b; a◌֮◌𝆭◌𝆭◌̀◌̕b; a◌֮◌𝆭◌𝆭◌̀◌̕b; a◌֮◌𝆭◌𝆭◌̀◌̕b; a◌֮◌𝆭◌𝆭◌̀◌̕b; ) LATIN SMALL LETTER A, MUSICAL SYMBOL COMBINING SNAP PIZZICATO, COMBINING COMMA ABOVE RIGHT, COMBINING GRAVE ACCENT, HEBREW ACCENT ZINOR, LATIN SMALL LETTER B +# +# END OF FILE diff --git a/Lib/test/_code_definitions.py b/Lib/test/_code_definitions.py index c3daa0dccf5df31..70c44da2ec63822 100644 --- a/Lib/test/_code_definitions.py +++ b/Lib/test/_code_definitions.py @@ -1,4 +1,32 @@ +def simple_script(): + assert True + + +def complex_script(): + obj = 'a string' + pickle = __import__('pickle') + def spam_minimal(): + pass + spam_minimal() + data = pickle.dumps(obj) + res = pickle.loads(data) + assert res == obj, (res, obj) + + +def script_with_globals(): + obj1, obj2 = spam(42) + assert obj1 == 42 + assert obj2 is None + + +def script_with_explicit_empty_return(): + return None + + +def script_with_return(): + return True + def spam_minimal(): # no arg defaults or kwarg defaults @@ -29,6 +57,22 @@ def spam_with_globals_and_builtins(): print(res) +def spam_with_global_and_attr_same_name(): + try: + spam_minimal.spam_minimal + except AttributeError: + pass + + +def spam_full_args(a, b, /, c, d, *args, e, f, **kwargs): + return (a, b, c, d, e, f, args, kwargs) + + +def spam_full_args_with_defaults(a=-1, b=-2, /, c=-3, d=-4, *args, + e=-5, f=-6, **kwargs): + return (a, b, c, d, e, f, args, kwargs) + + def spam_args_attrs_and_builtins(a, b, /, c, d, *args, e, f, **kwargs): if args.__len__() > 2: return None @@ -39,6 +83,10 @@ def spam_returns_arg(x): return x +def spam_raises(): + raise Exception('spam!') + + def spam_with_inner_not_closure(): def eggs(): pass @@ -141,11 +189,20 @@ def ham_C_closure(z): TOP_FUNCTIONS = [ # shallow + simple_script, + complex_script, + script_with_globals, + script_with_explicit_empty_return, + script_with_return, spam_minimal, spam_with_builtins, spam_with_globals_and_builtins, + spam_with_global_and_attr_same_name, + spam_full_args, + spam_full_args_with_defaults, spam_args_attrs_and_builtins, spam_returns_arg, + spam_raises, spam_with_inner_not_closure, spam_with_inner_closure, spam_annotated, @@ -178,6 +235,58 @@ def ham_C_closure(z): *NESTED_FUNCTIONS, ] +STATELESS_FUNCTIONS = [ + simple_script, + complex_script, + script_with_explicit_empty_return, + script_with_return, + spam, + spam_minimal, + spam_with_builtins, + spam_full_args, + spam_args_attrs_and_builtins, + spam_returns_arg, + spam_raises, + spam_annotated, + spam_with_inner_not_closure, + spam_with_inner_closure, + spam_N, + spam_C, + spam_NN, + spam_NC, + spam_CN, + spam_CC, + eggs_nested, + eggs_nested_N, + ham_nested, + ham_C_nested +] +STATELESS_CODE = [ + *STATELESS_FUNCTIONS, + script_with_globals, + spam_full_args_with_defaults, + spam_with_globals_and_builtins, + spam_with_global_and_attr_same_name, + spam_full, +] + +PURE_SCRIPT_FUNCTIONS = [ + simple_script, + complex_script, + script_with_explicit_empty_return, + spam_minimal, + spam_with_builtins, + spam_raises, + spam_with_inner_not_closure, + spam_with_inner_closure, +] +SCRIPT_FUNCTIONS = [ + *PURE_SCRIPT_FUNCTIONS, + script_with_globals, + spam_with_globals_and_builtins, + spam_with_global_and_attr_same_name, +] + # generators diff --git a/Lib/test/_test_atexit.py b/Lib/test/_test_atexit.py index f618c1fcbca52b1..2e961d6a4854a0b 100644 --- a/Lib/test/_test_atexit.py +++ b/Lib/test/_test_atexit.py @@ -135,6 +135,53 @@ def func(): finally: atexit.unregister(func) + def test_eq_unregister_clear(self): + # Issue #112127: callback's __eq__ may call unregister or _clear + class Evil: + def __eq__(self, other): + action(other) + return NotImplemented + + for action in atexit.unregister, lambda o: atexit._clear(): + with self.subTest(action=action): + atexit.register(lambda: None) + atexit.unregister(Evil()) + atexit._clear() + + def test_eq_unregister(self): + # Issue #112127: callback's __eq__ may call unregister + def f1(): + log.append(1) + def f2(): + log.append(2) + def f3(): + log.append(3) + + class Pred: + def __eq__(self, other): + nonlocal cnt + cnt += 1 + if cnt == when: + atexit.unregister(what) + if other is f2: + return True + return False + + for what, expected in ( + (f1, [3]), + (f2, [3, 1]), + (f3, [1]), + ): + for when in range(1, 4): + with self.subTest(what=what.__name__, when=when): + cnt = 0 + log = [] + for f in (f1, f2, f3): + atexit.register(f) + atexit.unregister(Pred()) + atexit._run_exitfuncs() + self.assertEqual(log, expected) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/_test_eintr.py b/Lib/test/_test_eintr.py index 0ce42276bfe3d62..4a050792df73c4f 100644 --- a/Lib/test/_test_eintr.py +++ b/Lib/test/_test_eintr.py @@ -380,6 +380,8 @@ def os_open(self, path): @unittest.skipIf(sys.platform == "darwin", "hangs under macOS; see bpo-25234, bpo-35363") + @unittest.skipIf(sys.platform.startswith('netbsd'), + "hangs on NetBSD; see gh-137397") def test_os_open(self): self._test_open("fd = os.open(path, os.O_RDONLY)\nos.close(fd)", self.os_open) diff --git a/Lib/test/_test_embed_structseq.py b/Lib/test/_test_embed_structseq.py index 154662efce94124..4cac84d7a469acc 100644 --- a/Lib/test/_test_embed_structseq.py +++ b/Lib/test/_test_embed_structseq.py @@ -11,7 +11,7 @@ def check_structseq(self, obj_type): # ob_refcnt self.assertGreaterEqual(sys.getrefcount(obj_type), 1) # tp_base - self.assertTrue(issubclass(obj_type, tuple)) + self.assertIsSubclass(obj_type, tuple) # tp_bases self.assertEqual(obj_type.__bases__, (tuple,)) # tp_dict diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 4dc9a31d22f7716..f9e48c7c75378e1 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -39,7 +39,7 @@ from test.support import socket_helper from test.support import threading_helper from test.support import warnings_helper - +from test.support import subTests # Skip tests if _multiprocessing wasn't built. _multiprocessing = import_helper.import_module('_multiprocessing') @@ -206,10 +206,38 @@ def spawn_check_wrapper(*args, **kwargs): return decorator +def only_run_in_forkserver_testsuite(reason): + """Returns a decorator: raises SkipTest unless fork is supported + and the current start method is forkserver. + + Combines @support.requires_fork() with the single-run semantics of + only_run_in_spawn_testsuite(), but uses the forkserver testsuite as + the single-run target. Appropriate for tests that exercise + os.fork() directly (raw fork or mp.set_start_method("fork") in a + subprocess) and don't vary by start method, since forkserver is + only available on platforms that support fork. + """ + + def decorator(test_item): + + @functools.wraps(test_item) + def forkserver_check_wrapper(*args, **kwargs): + if not support.has_fork_support: + raise unittest.SkipTest("requires working os.fork()") + if (start_method := multiprocessing.get_start_method()) != "forkserver": + raise unittest.SkipTest( + f"{start_method=}, not 'forkserver'; {reason}") + return test_item(*args, **kwargs) + + return forkserver_check_wrapper + + return decorator + + class TestInternalDecorators(unittest.TestCase): """Logic within a test suite that could errantly skip tests? Test it!""" - @unittest.skipIf(sys.platform == "win32", "test requires that fork exists.") + @support.requires_fork() def test_only_run_in_spawn_testsuite(self): if multiprocessing.get_start_method() != "spawn": raise unittest.SkipTest("only run in test_multiprocessing_spawn.") @@ -233,6 +261,30 @@ def return_four_if_spawn(): finally: multiprocessing.set_start_method(orig_start_method, force=True) + @support.requires_fork() + def test_only_run_in_forkserver_testsuite(self): + if multiprocessing.get_start_method() != "forkserver": + raise unittest.SkipTest("only run in test_multiprocessing_forkserver.") + + try: + @only_run_in_forkserver_testsuite("testing this decorator") + def return_four_if_forkserver(): + return 4 + except Exception as err: + self.fail(f"expected decorated `def` not to raise; caught {err}") + + orig_start_method = multiprocessing.get_start_method(allow_none=True) + try: + multiprocessing.set_start_method("forkserver", force=True) + self.assertEqual(return_four_if_forkserver(), 4) + multiprocessing.set_start_method("spawn", force=True) + with self.assertRaises(unittest.SkipTest) as ctx: + return_four_if_forkserver() + self.assertIn("testing this decorator", str(ctx.exception)) + self.assertIn("start_method=", str(ctx.exception)) + finally: + multiprocessing.set_start_method(orig_start_method, force=True) + # # Creates a wrapper for a function which records the time it takes to finish @@ -513,9 +565,14 @@ def _sleep_some(cls): time.sleep(100) @classmethod - def _sleep_no_int_handler(cls): + def _sleep_some_event(cls, event): + event.set() + time.sleep(100) + + @classmethod + def _sleep_no_int_handler(cls, event): signal.signal(signal.SIGINT, signal.SIG_DFL) - cls._sleep_some() + cls._sleep_some_event(event) @classmethod def _test_sleep(cls, delay): @@ -525,7 +582,10 @@ def _kill_process(self, meth, target=None): if self.TYPE == 'threads': self.skipTest('test not appropriate for {}'.format(self.TYPE)) - p = self.Process(target=target or self._sleep_some) + event = self.Event() + if not target: + target = self._sleep_some_event + p = self.Process(target=target, args=(event,)) p.daemon = True p.start() @@ -543,8 +603,11 @@ def _kill_process(self, meth, target=None): self.assertTimingAlmostEqual(join.elapsed, 0.0) self.assertEqual(p.is_alive(), True) - # XXX maybe terminating too soon causes the problems on Gentoo... - time.sleep(1) + timeout = support.SHORT_TIMEOUT + if not event.wait(timeout): + p.terminate() + p.join() + self.fail(f"event not signaled in {timeout} seconds") meth(p) @@ -1169,7 +1232,7 @@ def test_put(self): @classmethod def _test_get(cls, queue, child_can_start, parent_can_continue): child_can_start.wait() - #queue.put(1) + queue.put(1) queue.put(2) queue.put(3) queue.put(4) @@ -1193,15 +1256,16 @@ def test_get(self): child_can_start.set() parent_can_continue.wait() - time.sleep(DELTA) + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if not queue_empty(queue): + break self.assertEqual(queue_empty(queue), False) - # Hangs unexpectedly, remove for now - #self.assertEqual(queue.get(), 1) + self.assertEqual(queue.get_nowait(), 1) self.assertEqual(queue.get(True, None), 2) self.assertEqual(queue.get(True), 3) self.assertEqual(queue.get(timeout=1), 4) - self.assertEqual(queue.get_nowait(), 5) + self.assertEqual(queue.get(), 5) self.assertEqual(queue_empty(queue), True) @@ -3300,6 +3364,7 @@ class _TestMyManager(BaseTestCase): ALLOWED_TYPES = ('manager',) + @support.skip_if_sanitizer('TSan: leaks threads', thread=True) def test_mymanager(self): manager = MyManager(shutdown_timeout=SHUTDOWN_TIMEOUT) manager.start() @@ -3311,6 +3376,7 @@ def test_mymanager(self): # which happens on slow buildbots. self.assertIn(manager._process.exitcode, (0, -signal.SIGTERM)) + @support.skip_if_sanitizer('TSan: leaks threads', thread=True) def test_mymanager_context(self): manager = MyManager(shutdown_timeout=SHUTDOWN_TIMEOUT) with manager: @@ -3320,6 +3386,7 @@ def test_mymanager_context(self): # which happens on slow buildbots. self.assertIn(manager._process.exitcode, (0, -signal.SIGTERM)) + @support.skip_if_sanitizer('TSan: leaks threads', thread=True) def test_mymanager_context_prestarted(self): manager = MyManager(shutdown_timeout=SHUTDOWN_TIMEOUT) manager.start() @@ -3390,6 +3457,7 @@ def _putter(cls, address, authkey): # Note that xmlrpclib will deserialize object as a list not a tuple queue.put(tuple(cls.values)) + @support.skip_if_sanitizer('TSan: leaks threads', thread=True) def test_remote(self): authkey = os.urandom(32) @@ -3431,6 +3499,7 @@ def _putter(cls, address, authkey): queue = manager.get_queue() queue.put('hello world') + @support.skip_if_sanitizer("TSan: leaks threads", thread=True) def test_rapid_restart(self): authkey = os.urandom(32) manager = QueueManager( @@ -4272,6 +4341,19 @@ def test_copy(self): self.assertEqual(bar.z, 2 ** 33) +def resource_tracker_format_subtests(func): + """Run given test using both resource tracker communication formats""" + def _inner(self, *args, **kwargs): + tracker = resource_tracker._resource_tracker + for use_simple_format in False, True: + with ( + self.subTest(use_simple_format=use_simple_format), + unittest.mock.patch.object( + tracker, '_use_simple_format', use_simple_format) + ): + func(self, *args, **kwargs) + return _inner + @unittest.skipUnless(HAS_SHMEM, "requires multiprocessing.shared_memory") @hashlib_helper.requires_hashdigest('sha256') class _TestSharedMemory(BaseTestCase): @@ -4549,6 +4631,7 @@ def test_shared_memory_SharedMemoryServer_ignores_sigint(self): smm.shutdown() @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + @resource_tracker_format_subtests def test_shared_memory_SharedMemoryManager_reuses_resource_tracker(self): # bpo-36867: test that a SharedMemoryManager uses the # same resource_tracker process as its parent. @@ -4799,6 +4882,7 @@ def test_shared_memory_cleaned_after_process_termination(self): "shared_memory objects to clean up at shutdown", err) @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + @resource_tracker_format_subtests def test_shared_memory_untracking(self): # gh-82300: When a separate Python process accesses shared memory # with track=False, it must not cause the memory to be deleted @@ -4826,6 +4910,7 @@ def test_shared_memory_untracking(self): mem.close() @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + @resource_tracker_format_subtests def test_shared_memory_tracking(self): # gh-82300: When a separate Python process accesses shared memory # with track=True, it must cause the memory to be deleted when @@ -5162,6 +5247,23 @@ def test_invalid_handles(self): multiprocessing.connection.Connection, -1) +# +# Regression tests for BaseProcess kwargs handling +# + +class TestBaseProcessKwargs(unittest.TestCase): + def test_default_kwargs_not_shared_between_instances(self): + # Creating multiple Process instances without passing kwargs + # must create independent empty dicts (no shared state). + p1 = multiprocessing.Process(target=lambda: None) + p2 = multiprocessing.Process(target=lambda: None) + self.assertIsInstance(p1._kwargs, dict) + self.assertIsInstance(p2._kwargs, dict) + self.assertIsNot(p1._kwargs, p2._kwargs) + # Mutating one should not affect the other + p1._kwargs['x'] = 1 + self.assertNotIn('x', p2._kwargs) + @hashlib_helper.requires_hashdigest('sha256') class OtherTest(unittest.TestCase): @@ -5800,6 +5902,39 @@ def test_context(self): self.assertRaises(ValueError, ctx.set_start_method, None) self.check_context(ctx) + @staticmethod + def _dummy_func(): + pass + + def test_spawn_dont_set_context(self): + # Run a process with spawn or forkserver context may change + # the global start method, see gh-109263. + for method in ('fork', 'spawn', 'forkserver'): + multiprocessing.set_start_method(None, force=True) + + try: + ctx = multiprocessing.get_context(method) + except ValueError: + continue + process = ctx.Process(target=self._dummy_func) + process.start() + process.join() + self.assertIsNone(multiprocessing.get_start_method(allow_none=True)) + + @only_run_in_spawn_testsuite("freeze_support is not start method specific") + def test_freeze_support_dont_set_context(self): + # gh-140814: freeze_support() should not set the start method + # as a side effect, so a later set_start_method() still works. + multiprocessing.set_start_method(None, force=True) + try: + multiprocessing.freeze_support() + self.assertIsNone( + multiprocessing.get_start_method(allow_none=True)) + # Should not raise "context has already been set" + multiprocessing.set_start_method('spawn') + finally: + multiprocessing.set_start_method(None, force=True) + def test_context_check_module_types(self): try: ctx = multiprocessing.get_context('forkserver') @@ -6047,8 +6182,9 @@ def test_resource_tracker_sigkill(self): def _is_resource_tracker_reused(conn, pid): from multiprocessing.resource_tracker import _resource_tracker _resource_tracker.ensure_running() - # The pid should be None in the child process, expect for the fork - # context. It should not be a new value. + # The pid should be None in the child (the at-fork handler clears + # it for fork; spawn/forkserver children never had it set). It + # should not be a new value. reused = _resource_tracker._pid in (None, pid) reused &= _resource_tracker._check_alive() conn.send(reused) @@ -6133,6 +6269,183 @@ def test_resource_tracker_blocked_signals(self): # restore sigmask to what it was before executing test signal.pthread_sigmask(signal.SIG_SETMASK, orig_sigmask) + @only_run_in_forkserver_testsuite("avoids redundant testing.") + def test_resource_tracker_fork_deadlock(self): + # gh-146313: ResourceTracker.__del__ used to deadlock if a forked + # child still held the pipe's write end open when the parent + # exited, because the parent would block in waitpid() waiting for + # the tracker to exit, but the tracker would never see EOF. + cmd = '''if 1: + import os, signal + from multiprocessing.resource_tracker import ensure_running + ensure_running() + if os.fork() == 0: + signal.pause() + os._exit(0) + # parent falls through and exits, triggering __del__ + ''' + proc = subprocess.Popen([sys.executable, '-c', cmd], + start_new_session=True) + try: + try: + proc.wait(timeout=support.SHORT_TIMEOUT) + except subprocess.TimeoutExpired: + self.fail( + "Parent process deadlocked in ResourceTracker.__del__" + ) + self.assertEqual(proc.returncode, 0) + finally: + try: + os.killpg(proc.pid, signal.SIGKILL) + except ProcessLookupError: + pass + proc.wait() + + @only_run_in_forkserver_testsuite("avoids redundant testing.") + def test_resource_tracker_mp_fork_reuse_and_prompt_reap(self): + # gh-146313 / gh-80849: A child started via multiprocessing.Process + # with the 'fork' start method should reuse the parent's resource + # tracker (the at-fork handler preserves the inherited pipe fd), + # *and* the parent should be able to reap the tracker promptly + # after joining the child, without hitting the waitpid timeout. + cmd = textwrap.dedent(''' + import multiprocessing as mp + from multiprocessing.resource_tracker import _resource_tracker + + def child(conn): + # Prove we can talk to the parent's tracker by registering + # and unregistering a dummy resource over the inherited fd. + # If the fd were closed, ensure_running would launch a new + # tracker and _pid would be non-None. + _resource_tracker.register("x", "dummy") + _resource_tracker.unregister("x", "dummy") + conn.send((_resource_tracker._fd is not None, + _resource_tracker._pid is None, + _resource_tracker._check_alive())) + + if __name__ == "__main__": + mp.set_start_method("fork") + _resource_tracker.ensure_running() + r, w = mp.Pipe(duplex=False) + p = mp.Process(target=child, args=(w,)) + p.start() + child_has_fd, child_pid_none, child_alive = r.recv() + p.join() + w.close(); r.close() + + # Now simulate __del__: the child has exited and released + # its fd copy, so the tracker should see EOF and exit + # promptly -- no timeout. + _resource_tracker._stop(wait_timeout=5.0) + print(child_has_fd, child_pid_none, child_alive, + _resource_tracker._waitpid_timed_out, + _resource_tracker._exitcode) + ''') + rc, out, err = script_helper.assert_python_ok('-c', cmd) + parts = out.decode().split() + self.assertEqual(parts, ['True', 'True', 'True', 'False', '0'], + f"unexpected: {parts!r} stderr={err!r}") + + @only_run_in_forkserver_testsuite("avoids redundant testing.") + def test_resource_tracker_raw_fork_prompt_reap(self): + # gh-146313: After a raw os.fork() the at-fork handler closes the + # child's inherited fd, so the parent can reap the tracker + # immediately -- even while the child is still alive -- rather + # than waiting out the 1s timeout. + cmd = textwrap.dedent(''' + import os, signal + from multiprocessing.resource_tracker import _resource_tracker + + _resource_tracker.ensure_running() + r, w = os.pipe() + pid = os.fork() + if pid == 0: + os.close(r) + # Report whether our fd was closed by the at-fork handler. + os.write(w, b"1" if _resource_tracker._fd is None else b"0") + os.close(w) + signal.pause() # stay alive so parent's reap is meaningful + os._exit(0) + os.close(w) + child_fd_closed = os.read(r, 1) == b"1" + os.close(r) + + # Child is still alive and paused. Because it closed its fd + # copy, our close below is the last one and the tracker exits. + _resource_tracker._stop(wait_timeout=5.0) + + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + print(child_fd_closed, + _resource_tracker._waitpid_timed_out, + _resource_tracker._exitcode) + ''') + rc, out, err = script_helper.assert_python_ok('-c', cmd) + parts = out.decode().split() + self.assertEqual(parts, ['True', 'False', '0'], + f"unexpected: {parts!r} stderr={err!r}") + + @only_run_in_forkserver_testsuite("avoids redundant testing.") + def test_resource_tracker_lock_reinit_after_fork(self): + # gh-146313: If a parent thread held the tracker's lock at fork + # time, the child would inherit the held lock and deadlock on + # its next ensure_running(). The at-fork handler reinits it. + cmd = textwrap.dedent(''' + import os, threading + from multiprocessing.resource_tracker import _resource_tracker + + held = threading.Event() + release = threading.Event() + def hold(): + with _resource_tracker._lock: + held.set() + release.wait() + t = threading.Thread(target=hold) + t.start() + held.wait() + + pid = os.fork() + if pid == 0: + ok = _resource_tracker._lock.acquire(timeout=5.0) + os._exit(0 if ok else 1) + + release.set() + t.join() + _, status = os.waitpid(pid, 0) + print(os.waitstatus_to_exitcode(status)) + ''') + rc, out, err = script_helper.assert_python_ok( + '-W', 'ignore::DeprecationWarning', '-c', cmd) + self.assertEqual(out.strip(), b'0', + f"child failed to acquire lock: stderr={err!r}") + + @only_run_in_forkserver_testsuite("avoids redundant testing.") + def test_resource_tracker_safety_net_timeout(self): + # gh-146313: When an mp.Process(fork) child holds the preserved + # fd and the parent calls _stop() without joining (simulating + # abnormal shutdown), the safety-net timeout should fire rather + # than deadlocking. + cmd = textwrap.dedent(''' + import multiprocessing as mp + import signal + from multiprocessing.resource_tracker import _resource_tracker + + if __name__ == "__main__": + mp.set_start_method("fork") + _resource_tracker.ensure_running() + p = mp.Process(target=signal.pause) + p.start() + # Stop WITHOUT joining -- child still holds preserved fd + _resource_tracker._stop(wait_timeout=0.5) + print(_resource_tracker._waitpid_timed_out) + p.terminate() + p.join() + ''') + rc, out, err = script_helper.assert_python_ok('-c', cmd) + self.assertEqual(out.strip(), b'True', + f"safety-net timeout did not fire: stderr={err!r}") + + class TestSimpleQueue(unittest.TestCase): @classmethod @@ -6778,6 +7091,20 @@ def test_child_sys_path(self): self.assertEqual(child_sys_path[1:], sys.path[1:]) self.assertIsNone(import_error, msg=f"child could not import {self._mod_name}") + def test_std_streams_flushed_after_preload(self): + # gh-135335: Check fork server flushes standard streams after + # preloading modules + if multiprocessing.get_start_method() != "forkserver": + self.skipTest("forkserver specific test") + + name = os.path.join(os.path.dirname(__file__), 'mp_preload_flush.py') + _, out, err = test.support.script_helper.assert_python_ok(name) + + # Check stderr first, as it is more likely to be useful to see in the + # event of a failure. + self.assertEqual(err.decode().rstrip(), '__main____mp_main__') + self.assertEqual(out.decode().rstrip(), '__main____mp_main__') + class MiscTestCase(unittest.TestCase): def test__all__(self): @@ -6821,6 +7148,77 @@ def f(x): return x*x self.assertEqual("332833500", out.decode('utf-8').strip()) self.assertFalse(err, msg=err.decode('utf-8')) + def test_forked_thread_not_started(self): + # gh-134381: Ensure that a thread that has not been started yet in + # the parent process can be started within a forked child process. + + if multiprocessing.get_start_method() != "fork": + self.skipTest("fork specific test") + + q = multiprocessing.Queue() + t = threading.Thread(target=lambda: q.put("done"), daemon=True) + + def child(): + t.start() + t.join() + + p = multiprocessing.Process(target=child) + p.start() + p.join(support.SHORT_TIMEOUT) + + self.assertEqual(p.exitcode, 0) + self.assertEqual(q.get_nowait(), "done") + close_queue(q) + + def test_preload_main(self): + # gh-126631: Check that __main__ can be pre-loaded + if multiprocessing.get_start_method() != "forkserver": + self.skipTest("forkserver specific test") + + name = os.path.join(os.path.dirname(__file__), 'mp_preload_main.py') + _, out, err = test.support.script_helper.assert_python_ok(name) + self.assertEqual(err, b'') + + # The trailing empty string comes from split() on output ending with \n + out = out.decode().split("\n") + self.assertEqual(out, ['__main__', '__mp_main__', 'f', 'f', '']) + + def test_preload_main_sys_argv(self): + # gh-143706: Check that sys.argv is set before __main__ is pre-loaded + if multiprocessing.get_start_method() != "forkserver": + self.skipTest("forkserver specific test") + + name = os.path.join(os.path.dirname(__file__), 'mp_preload_sysargv.py') + _, out, err = test.support.script_helper.assert_python_ok( + name, 'foo', 'bar') + self.assertEqual(err, b'') + + out = out.decode().split("\n") + expected_argv = "['foo', 'bar']" + self.assertEqual(out, [ + f"module:{expected_argv}", + f"fun:{expected_argv}", + f"module:{expected_argv}", + f"fun:{expected_argv}", + '', + ]) + + @only_run_in_forkserver_testsuite("forkserver specific test.") + def test_preload_main_large_sys_argv(self): + # gh-144503: a very large parent sys.argv must not prevent the + # forkserver from starting (it previously overflowed the OS + # per-argument length limit when repr'd into the -c command string). + name = os.path.join(os.path.dirname(__file__), + 'mp_preload_large_sysargv.py') + _, out, err = test.support.script_helper.assert_python_ok(name) + self.assertEqual(err, b'') + + out = out.decode().split("\n") + self.assertEqual(out, [ + 'preload:5002:sentinel', + 'worker:5002:sentinel', + '', + ]) # # Mixins @@ -7068,3 +7466,52 @@ class SemLock(_multiprocessing.SemLock): name = f'test_semlock_subclass-{os.getpid()}' s = SemLock(1, 0, 10, name, False) _multiprocessing.sem_unlink(name) + + +@unittest.skipUnless(HAS_SHMEM, "requires multiprocessing.shared_memory") +class TestSharedMemoryNames(unittest.TestCase): + @subTests('use_simple_format', (True, False)) + def test_that_shared_memory_name_with_colons_has_no_resource_tracker_errors( + self, use_simple_format): + # Test script that creates and cleans up shared memory with colon in name + test_script = textwrap.dedent(""" + import sys + from multiprocessing import shared_memory + from multiprocessing import resource_tracker + import time + + resource_tracker._resource_tracker._use_simple_format = %s + + # Test various patterns of colons in names + test_names = [ + "a:b", + "a:b:c", + "test:name:with:many:colons", + ":starts:with:colon", + "ends:with:colon:", + "::double::colons::", + "name\\nwithnewline", + "name-with-trailing-newline\\n", + "\\nname-starts-with-newline", + "colons:and\\nnewlines:mix", + "multi\\nline\\nname", + ] + + for name in test_names: + try: + shm = shared_memory.SharedMemory(create=True, size=100, name=name) + shm.buf[:5] = b'hello' # Write something to the shared memory + shm.close() + shm.unlink() + + except Exception as e: + print(f"Error with name '{name}': {e}", file=sys.stderr) + sys.exit(1) + + print("SUCCESS") + """ % use_simple_format) + + rc, out, err = script_helper.assert_python_ok("-c", test_script) + self.assertIn(b"SUCCESS", out) + self.assertNotIn(b"traceback", err.lower(), err) + self.assertNotIn(b"resource_tracker.py", err, err) diff --git a/Lib/test/audit-tests.py b/Lib/test/audit-tests.py index 08b638e4b8d5245..6884ac0dbe6ff03 100644 --- a/Lib/test/audit-tests.py +++ b/Lib/test/audit-tests.py @@ -643,6 +643,34 @@ def test_assert_unicode(): else: raise RuntimeError("Expected sys.audit(9) to fail.") +def test_sys_remote_exec(): + import tempfile + + pid = os.getpid() + event_pid = -1 + event_script_path = "" + remote_event_script_path = "" + def hook(event, args): + if event not in ["sys.remote_exec", "cpython.remote_debugger_script"]: + return + print(event, args) + match event: + case "sys.remote_exec": + nonlocal event_pid, event_script_path + event_pid = args[0] + event_script_path = args[1] + case "cpython.remote_debugger_script": + nonlocal remote_event_script_path + remote_event_script_path = args[0] + + sys.addaudithook(hook) + with tempfile.NamedTemporaryFile(mode='w+', delete=True) as tmp_file: + tmp_file.write("a = 1+1\n") + tmp_file.flush() + sys.remote_exec(pid, tmp_file.name) + assertEqual(event_pid, pid) + assertEqual(event_script_path, tmp_file.name) + assertEqual(remote_event_script_path, tmp_file.name) if __name__ == "__main__": from test.support import suppress_msvcrt_asserts diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 4a67fcd2c3e9b38..4cec427dbaa8850 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -530,19 +530,19 @@ test_char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; char a = 'A'; - char b = '\x07'; - char c = '\x08'; + char b = '\a'; + char c = '\b'; char d = '\t'; char e = '\n'; - char f = '\x0b'; - char g = '\x0c'; + char f = '\v'; + char g = '\f'; char h = '\r'; char i = '"'; char j = '\''; char k = '?'; char l = '\\'; - char m = '\x00'; - char n = '\xff'; + char m = '\0'; + char n = '\377'; if (!_PyArg_CheckPositional("test_char_converter", nargs, 0, 14)) { goto exit; @@ -936,7 +936,7 @@ static PyObject * test_char_converter_impl(PyObject *module, char a, char b, char c, char d, char e, char f, char g, char h, char i, char j, char k, char l, char m, char n) -/*[clinic end generated code: output=ff11e203248582df input=e42330417a44feac]*/ +/*[clinic end generated code: output=6503d15448e1d4c4 input=e42330417a44feac]*/ /*[clinic input] @@ -1173,14 +1173,14 @@ test_int_converter a: int = 12 b: int(accept={int}) = 34 - c: int(accept={str}) = 45 + c: int(accept={str}) = '-' d: int(type='myenum') = 67 / [clinic start generated code]*/ PyDoc_STRVAR(test_int_converter__doc__, -"test_int_converter($module, a=12, b=34, c=45, d=67, /)\n" +"test_int_converter($module, a=12, b=34, c=\'-\', d=67, /)\n" "--\n" "\n"); @@ -1196,7 +1196,7 @@ test_int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *return_value = NULL; int a = 12; int b = 34; - int c = 45; + int c = '-'; myenum d = 67; if (!_PyArg_CheckPositional("test_int_converter", nargs, 0, 4)) { @@ -1247,7 +1247,7 @@ test_int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) static PyObject * test_int_converter_impl(PyObject *module, int a, int b, int c, myenum d) -/*[clinic end generated code: output=fbcfb7554688663d input=d20541fc1ca0553e]*/ +/*[clinic end generated code: output=d5357b563bdb8789 input=5d8f4eb5899b24de]*/ /*[clinic input] diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 55844ec35a90c94..1b71b71c544c0cb 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -47,7 +47,11 @@ try: import _pydatetime except ImportError: - pass + _pydatetime = None +try: + import _datetime +except ImportError: + _datetime = None # pickle_loads = {pickle.loads, pickle._loads} @@ -183,7 +187,7 @@ class NotEnough(tzinfo): def __init__(self, offset, name): self.__offset = offset self.__name = name - self.assertTrue(issubclass(NotEnough, tzinfo)) + self.assertIsSubclass(NotEnough, tzinfo) ne = NotEnough(3, "NotByALongShot") self.assertIsInstance(ne, tzinfo) @@ -232,7 +236,7 @@ def test_pickling_subclass(self): self.assertIs(type(derived), otype) self.assertEqual(derived.utcoffset(None), offset) self.assertEqual(derived.tzname(None), oname) - self.assertFalse(hasattr(derived, 'spam')) + self.assertNotHasAttr(derived, 'spam') def test_issue23600(self): DSTDIFF = DSTOFFSET = timedelta(hours=1) @@ -773,6 +777,9 @@ def test_str(self): microseconds=999999)), "999999999 days, 23:59:59.999999") + # test the Doc/library/datetime.rst recipe + eq(f'-({-td(hours=-1)!s})', "-(1:00:00)") + def test_repr(self): name = 'datetime.' + self.theclass.__name__ self.assertEqual(repr(self.theclass(1)), @@ -810,7 +817,7 @@ def test_roundtrip(self): # Verify td -> string -> td identity. s = repr(td) - self.assertTrue(s.startswith('datetime.')) + self.assertStartsWith(s, 'datetime.') s = s[9:] td2 = eval(s) self.assertEqual(td, td2) @@ -1228,7 +1235,7 @@ def test_roundtrip(self): self.theclass.today()): # Verify dt -> string -> date identity. s = repr(dt) - self.assertTrue(s.startswith('datetime.')) + self.assertStartsWith(s, 'datetime.') s = s[9:] dt2 = eval(s) self.assertEqual(dt, dt2) @@ -1804,7 +1811,7 @@ def test_bool(self): self.assertTrue(self.theclass.min) self.assertTrue(self.theclass.max) - def test_strftime_y2k(self): + def check_strftime_y2k(self, specifier): # Test that years less than 1000 are 0-padded; note that the beginning # of an ISO 8601 year may fall in an ISO week of the year before, and # therefore needs an offset of -1 when formatting with '%G'. @@ -1818,22 +1825,28 @@ def test_strftime_y2k(self): (1000, 0), (1970, 0), ) - specifiers = 'YG' - if _time.strftime('%F', (1900, 1, 1, 0, 0, 0, 0, 1, 0)) == '1900-01-01': - specifiers += 'FC' for year, g_offset in dataset: - for specifier in specifiers: - with self.subTest(year=year, specifier=specifier): - d = self.theclass(year, 1, 1) - if specifier == 'G': - year += g_offset - if specifier == 'C': - expected = f"{year // 100:02d}" - else: - expected = f"{year:04d}" - if specifier == 'F': - expected += f"-01-01" - self.assertEqual(d.strftime(f"%{specifier}"), expected) + with self.subTest(year=year, specifier=specifier): + d = self.theclass(year, 1, 1) + if specifier == 'G': + year += g_offset + if specifier == 'C': + expected = f"{year // 100:02d}" + else: + expected = f"{year:04d}" + if specifier == 'F': + expected += f"-01-01" + self.assertEqual(d.strftime(f"%{specifier}"), expected) + + def test_strftime_y2k(self): + self.check_strftime_y2k('Y') + self.check_strftime_y2k('G') + + def test_strftime_y2k_c99(self): + # CPython requires C11; specifiers new in C99 must work. + # (Other implementations may want to disable this test.) + self.check_strftime_y2k('F') + self.check_strftime_y2k('C') def test_replace(self): cls = self.theclass @@ -2215,7 +2228,7 @@ def test_roundtrip(self): self.theclass.now()): # Verify dt -> string -> datetime identity. s = repr(dt) - self.assertTrue(s.startswith('datetime.')) + self.assertStartsWith(s, 'datetime.') s = s[9:] dt2 = eval(s) self.assertEqual(dt, dt2) @@ -3568,6 +3581,11 @@ def test_fromisoformat_fails_datetime(self): '2009-04-19T12:30:45.400 +02:30', # Space between ms and timezone (gh-130959) '2009-04-19T12:30:45.400 ', # Trailing space (gh-130959) '2009-04-19T12:30:45. 400', # Space before fraction (gh-130959) + '2020-2020', # Ambiguous 9-char date portion + '2009-04-19T12:30:45.+05:00', # Empty fraction before offset + '2009-04-19T12:30:45.-05:00', # Empty fraction before offset + '2009-04-19T12:30:45.Z', # Empty fraction before Z + '2009-04-19T12:30:45,+05:00', # Empty fraction (comma) before offset ] for bad_str in bad_strs: @@ -3584,6 +3602,7 @@ def test_fromisoformat_fails_datetime_valueerror(self): "2009-04-01T12:30:90", # Second out of range "2009-04-01T12:90:45", # Minute out of range "2009-04-01T25:30:45", # Hour out of range + "2009-00-01T24:00:00", # Month below range "2009-13-01T24:00:00", # Month out of range "9999-12-31T24:00:00", # Year out of range ] @@ -3612,6 +3631,32 @@ def test_fromisoformat_utc(self): self.assertIs(dt.tzinfo, timezone.utc) + def test_fromisoformat_utc_subsecond_offset(self): + # A UTC offset whose whole-second part is zero but with a non-zero + # microsecond part must be preserved, not collapsed to UTC. + for us in (1, -1, 999999, -999999): + with self.subTest(microseconds=us): + tz = timezone(timedelta(microseconds=us)) + dt = self.theclass(2020, 6, 15, 12, 34, 56, tzinfo=tz) + rt = self.theclass.fromisoformat(dt.isoformat()) + self.assertEqual(rt.utcoffset(), timedelta(microseconds=us)) + self.assertEqual(rt, dt) + self.assertIsNot(rt.tzinfo, timezone.utc) + + tz = timezone(timedelta(hours=5, minutes=30, seconds=15, + microseconds=123456)) + dt = self.theclass(2020, 6, 15, 12, 34, 56, tzinfo=tz) + rt = self.theclass.fromisoformat(dt.isoformat()) + self.assertEqual(rt.utcoffset(), tz.utcoffset(None)) + self.assertEqual(rt, dt) + + for tstr in ('2020-06-15T12:34:56+00:00', + '2020-06-15T12:34:56+00:00:00.000000', + '2020-06-15T12:34:56Z'): + with self.subTest(tstr=tstr): + self.assertIs(self.theclass.fromisoformat(tstr).tzinfo, + timezone.utc) + def test_fromisoformat_subclass(self): class DateTimeSubclass(self.theclass): pass @@ -3669,7 +3714,7 @@ def test_roundtrip(self): # Verify t -> string -> time identity. s = repr(t) - self.assertTrue(s.startswith('datetime.')) + self.assertStartsWith(s, 'datetime.') s = s[9:] t2 = eval(s) self.assertEqual(t, t2) @@ -3895,6 +3940,11 @@ def test_strftime_special(self): self.assertEqual(t.strftime('\0'*1000), '\0'*1000) self.assertEqual(t.strftime('\0%I%p%Z\0%X'), f'\0{s1}\0{s2}') self.assertEqual(t.strftime('%I%p%Z\0%X\0'), f'{s1}\0{s2}\0') + # gh-152305: the year directives must not raise on a time. + for directive, expected in (('%Y', '1900'), ('%G', '1900'), + ('%C', '19'), ('%F', '1900-01-01')): + with self.subTest(directive=directive): + self.assertEqual(t.strftime(directive), expected) def test_format(self): t = self.theclass(1, 2, 3, 4) @@ -4792,6 +4842,10 @@ def test_fromisoformat_fails(self): '12:30:45.400 +02:30', # Space between ms and timezone (gh-130959) '12:30:45.400 ', # Trailing space (gh-130959) '12:30:45. 400', # Space before fraction (gh-130959) + '12:30:45.+05:00', # Empty fraction before offset + '12:30:45.-05:00', # Empty fraction before offset + '12:30:45.Z', # Empty fraction before Z + '12:30:45,+05:00', # Empty fraction (comma) before offset ] for bad_str in bad_strs: @@ -6128,21 +6182,21 @@ def test_vilnius_1941_fromutc(self): gdt = datetime(1941, 6, 23, 20, 59, 59, tzinfo=timezone.utc) ldt = gdt.astimezone(Vilnius) - self.assertEqual(ldt.strftime("%c %Z%z"), + self.assertEqual(ldt.strftime("%a %b %d %H:%M:%S %Y %Z%z"), 'Mon Jun 23 23:59:59 1941 MSK+0300') self.assertEqual(ldt.fold, 0) self.assertFalse(ldt.dst()) gdt = datetime(1941, 6, 23, 21, tzinfo=timezone.utc) ldt = gdt.astimezone(Vilnius) - self.assertEqual(ldt.strftime("%c %Z%z"), + self.assertEqual(ldt.strftime("%a %b %d %H:%M:%S %Y %Z%z"), 'Mon Jun 23 23:00:00 1941 CEST+0200') self.assertEqual(ldt.fold, 1) self.assertTrue(ldt.dst()) gdt = datetime(1941, 6, 23, 22, tzinfo=timezone.utc) ldt = gdt.astimezone(Vilnius) - self.assertEqual(ldt.strftime("%c %Z%z"), + self.assertEqual(ldt.strftime("%a %b %d %H:%M:%S %Y %Z%z"), 'Tue Jun 24 00:00:00 1941 CEST+0200') self.assertEqual(ldt.fold, 0) self.assertTrue(ldt.dst()) @@ -6152,22 +6206,22 @@ def test_vilnius_1941_toutc(self): ldt = datetime(1941, 6, 23, 22, 59, 59, tzinfo=Vilnius) gdt = ldt.astimezone(timezone.utc) - self.assertEqual(gdt.strftime("%c %Z"), + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), 'Mon Jun 23 19:59:59 1941 UTC') ldt = datetime(1941, 6, 23, 23, 59, 59, tzinfo=Vilnius) gdt = ldt.astimezone(timezone.utc) - self.assertEqual(gdt.strftime("%c %Z"), + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), 'Mon Jun 23 20:59:59 1941 UTC') ldt = datetime(1941, 6, 23, 23, 59, 59, tzinfo=Vilnius, fold=1) gdt = ldt.astimezone(timezone.utc) - self.assertEqual(gdt.strftime("%c %Z"), + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), 'Mon Jun 23 21:59:59 1941 UTC') ldt = datetime(1941, 6, 24, 0, tzinfo=Vilnius) gdt = ldt.astimezone(timezone.utc) - self.assertEqual(gdt.strftime("%c %Z"), + self.assertEqual(gdt.strftime("%a %b %d %H:%M:%S %Y %Z"), 'Mon Jun 23 22:00:00 1941 UTC') def test_constructors(self): @@ -7272,6 +7326,64 @@ def test_update_type_cache(self): """) script_helper.assert_python_ok('-c', script) + def test_concurrent_initialization_subinterpreter(self): + # gh-136421: Concurrent initialization of _datetime across multiple + # interpreters wasn't thread-safe due to its static types. + + # Run in a subprocess to ensure we get a clean version of _datetime + script = """if True: + from concurrent.futures import InterpreterPoolExecutor + + def func(): + import _datetime + print('a', end='') + + with InterpreterPoolExecutor() as executor: + for _ in range(8): + executor.submit(func) + """ + rc, out, err = script_helper.assert_python_ok("-c", script) + self.assertEqual(rc, 0) + self.assertEqual(out, b"a" * 8) + self.assertEqual(err, b"") + + # Now test against concurrent reinitialization + script = "import _datetime\n" + script + rc, out, err = script_helper.assert_python_ok("-c", script) + self.assertEqual(rc, 0) + self.assertEqual(out, b"a" * 8) + self.assertEqual(err, b"") + + @support.cpython_only + @support.subTests(("setup", "call"), [ + ("obj = _datetime.timedelta", "obj(seconds=2)"), + ("obj = _datetime.timedelta(seconds=2)", "obj.total_seconds()"), + ("obj = _datetime.date(2026, 6, 7)", "obj.isocalendar()"), + ]) + def test_static_datetime_types_outlive_collected_module(self, setup, call): + # gh-151039: This code used to crash + script = f"""if True: + import sys, gc + import _datetime + + {setup} # static C type, survives the module + del sys.modules['_datetime'] + del _datetime + sys.modules['_datetime'] = None # block re-import + gc.collect() # module object is collected + + try: + {call} # used to be a segmentation fault + except ImportError: + pass + else: + raise AssertionError("ImportError not raised") + """ + rc, out, err = script_helper.assert_python_ok("-c", script) + self.assertEqual(rc, 0) + self.assertEqual(out, b'') + self.assertEqual(err, b'') + def load_tests(loader, standard_tests, pattern): standard_tests.addTest(ZoneInfoCompleteTest()) diff --git a/Lib/test/libregrtest/cmdline.py b/Lib/test/libregrtest/cmdline.py index 07681d75448e24d..2c404f6d80bcf39 100644 --- a/Lib/test/libregrtest/cmdline.py +++ b/Lib/test/libregrtest/cmdline.py @@ -130,6 +130,13 @@ tzdata - Run tests that require timezone data. + xpickle - Test pickle and _pickle against Python 3.6, 3.7, 3.8 + and 3.9 to test backwards compatibility. These tests + may take very long to complete. + + wantobjects - Allows to run Tkinter tests with the specified value of + tkinter.wantobjects. + To enable all resources except one, use '-uall,-'. For example, to run all the tests except for the gui tests, give the option '-uall,-gui'. @@ -158,7 +165,7 @@ def __init__(self, **kwargs) -> None: self.randomize = False self.fromfile = None self.fail_env_changed = False - self.use_resources: list[str] = [] + self.use_resources: dict[str, str | None] = {} self.trace = False self.coverdir = 'coverage' self.runleaks = False @@ -271,6 +278,9 @@ def _create_parser(): group = parser.add_argument_group('Selecting tests') group.add_argument('-r', '--randomize', action='store_true', help='randomize test execution order.' + more_details) + group.add_argument('--no-randomize', dest='no_randomize', action='store_true', + help='do not randomize test execution order, even if ' + 'it would be implied by another option') group.add_argument('--prioritize', metavar='TEST1,TEST2,...', action='append', type=priority_list, help='select these tests first, even if the order is' @@ -302,7 +312,7 @@ def _create_parser(): group.add_argument('-G', '--failfast', action='store_true', help='fail as soon as a test fails (only with -v or -W)') group.add_argument('-u', '--use', metavar='RES1,RES2,...', - action='append', type=resources_list, + action='extend', type=resources_list, help='specify which special resource intensive tests ' 'to run.' + more_details) group.add_argument('-M', '--memlimit', metavar='LIMIT', @@ -407,11 +417,18 @@ def huntrleaks(string): def resources_list(string): - u = [x.lower() for x in string.split(',')] - for r in u: + u = [] + for x in string.split(','): + r, eq, v = x.partition('=') + r = r.lower() + u.append((r, v if eq else None)) if r == 'all' or r == 'none': + if eq: + raise argparse.ArgumentTypeError('invalid resource: ' + x) continue if r[0] == '-': + if eq: + raise argparse.ArgumentTypeError('invalid resource: ' + x) r = r[1:] if r not in RESOURCE_NAMES: raise argparse.ArgumentTypeError('invalid resource: ' + r) @@ -461,7 +478,11 @@ def _parse_args(args, **kwargs): if ns.python is None: ns.rerun = True ns.print_slow = True - ns.verbose3 = True + if not ns.verbose: + ns.verbose3 = True + else: + # --verbose has the priority over --verbose3 + pass else: ns._add_python_opts = False @@ -475,14 +496,14 @@ def _parse_args(args, **kwargs): # Similar to: -u "all" --timeout=1200 if ns.use is None: ns.use = [] - ns.use.insert(0, ['all']) + ns.use[:0] = [('all', None)] if ns.timeout is None: ns.timeout = 1200 # 20 minutes elif ns.fast_ci: # Similar to: -u "all,-cpu" --timeout=600 if ns.use is None: ns.use = [] - ns.use.insert(0, ['all', '-cpu']) + ns.use[:0] = [('all', None), ('-cpu', None)] if ns.timeout is None: ns.timeout = 600 # 10 minutes @@ -520,25 +541,21 @@ def _parse_args(args, **kwargs): if ns.timeout <= 0: ns.timeout = None if ns.use: - for a in ns.use: - for r in a: - if r == 'all': - ns.use_resources[:] = ALL_RESOURCES - continue - if r == 'none': - del ns.use_resources[:] - continue - remove = False - if r[0] == '-': - remove = True - r = r[1:] - if remove: - if r in ns.use_resources: - ns.use_resources.remove(r) - elif r not in ns.use_resources: - ns.use_resources.append(r) + for r, v in ns.use: + if r == 'all': + for r in ALL_RESOURCES: + ns.use_resources[r] = None + elif r == 'none': + ns.use_resources.clear() + elif r[0] == '-': + r = r[1:] + ns.use_resources.pop(r, None) + else: + ns.use_resources[r] = v if ns.random_seed is not None: ns.randomize = True + if ns.no_randomize: + ns.randomize = False if ns.verbose: ns.header = True diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 713cbedb299706c..d8b9605ea498432 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -118,7 +118,7 @@ def __init__(self, ns: Namespace, _add_python_opts: bool = False): self.junit_filename: StrPath | None = ns.xmlpath self.memory_limit: str | None = ns.memlimit self.gc_threshold: int | None = ns.threshold - self.use_resources: tuple[str, ...] = tuple(ns.use_resources) + self.use_resources: dict[str, str | None] = dict(ns.use_resources) if ns.python: self.python_cmd: tuple[str, ...] | None = tuple(ns.python) else: @@ -190,6 +190,12 @@ def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList strip_py_suffix(tests) + exclude_tests = set() + if self.exclude: + for arg in self.cmdline_args: + exclude_tests.add(arg) + self.cmdline_args = [] + if self.pgo: # add default PGO tests if no tests are specified setup_pgo_tests(self.cmdline_args, self.pgo_extended) @@ -200,17 +206,15 @@ def find_tests(self, tests: TestList | None = None) -> tuple[TestTuple, TestList if self.tsan_parallel: setup_tsan_parallel_tests(self.cmdline_args) - exclude_tests = set() - if self.exclude: - for arg in self.cmdline_args: - exclude_tests.add(arg) - self.cmdline_args = [] - alltests = findtests(testdir=self.test_dir, exclude=exclude_tests) if not self.fromfile: selected = tests or self.cmdline_args + if exclude_tests: + # Support "--pgo/--tsan -x test_xxx" command + selected = [name for name in selected + if name not in exclude_tests] if selected: selected = split_test_packages(selected) else: @@ -543,8 +547,6 @@ def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int: self.first_runtests = runtests self.logger.set_tests(runtests) - setup_process() - if (runtests.hunt_refleak is not None) and (not self.num_workers): # gh-109739: WindowsLoadTracker thread interferes with refleak check use_load_tracker = False @@ -644,15 +646,23 @@ def _add_cross_compile_opts(self, regrtest_opts): return (environ, keep_environ) def _add_ci_python_opts(self, python_opts, keep_environ): - # --fast-ci and --slow-ci add options to Python: - # "-u -W default -bb -E" - - # Unbuffered stdout and stderr - if not sys.stdout.write_through: + # --fast-ci and --slow-ci add options to Python. + # + # Some platforms run tests in embedded mode and cannot change options + # after startup, so if this function changes, consider also updating: + # * gradle_task in Android/android.py + + # Unbuffered stdout and stderr. This isn't helpful on Android, because + # it would cause lines to be split into multiple log messages. + if not sys.stdout.write_through and sys.platform != "android": python_opts.append('-u') - # Add warnings filter 'error' - if 'default' not in sys.warnoptions: + # Add warnings filter 'error', unless the user specified a different + # filter. Ignore BytesWarning since it's controlled by '-b' below. + if not [ + opt for opt in sys.warnoptions + if not opt.endswith("::BytesWarning") + ]: python_opts.extend(('-W', 'error')) # Error on bytes/str comparison @@ -671,8 +681,12 @@ def _execute_python(self, cmd, environ): cmd_text = shlex.join(cmd) try: - print(f"+ {cmd_text}", flush=True) + # Android and iOS run tests in embedded mode. To update their + # Python options, see the comment in _add_ci_python_opts. + if not cmd[0]: + raise ValueError("No Python executable is present") + print(f"+ {cmd_text}", flush=True) if hasattr(os, 'execv') and not MS_WINDOWS: os.execv(cmd[0], cmd) # On success, execv() do no return. @@ -721,10 +735,7 @@ def _add_python_opts(self) -> None: self._execute_python(cmd, environ) def _init(self): - # Set sys.stdout encoder error handler to backslashreplace, - # similar to sys.stderr error handler, to avoid UnicodeEncodeError - # when printing a traceback or any other non-encodable character. - sys.stdout.reconfigure(errors="backslashreplace") + setup_process() if self.junit_filename and not os.path.isabs(self.junit_filename): self.junit_filename = os.path.abspath(self.junit_filename) diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py index 424085a0050eb59..4458e160705bca7 100644 --- a/Lib/test/libregrtest/run_workers.py +++ b/Lib/test/libregrtest/run_workers.py @@ -147,9 +147,12 @@ def _kill(self) -> None: use_killpg = USE_PROCESS_GROUP if use_killpg: - parent_sid = os.getsid(0) - sid = os.getsid(popen.pid) - use_killpg = (sid != parent_sid) + try: + use_killpg = (os.getsid(popen.pid) != os.getsid(0)) + except PermissionError: + # On OpenBSD getsid() is only allowed for a process in the + # same session, so the failure means that it is not. + use_killpg = True if use_killpg: what = f"{self} process group" diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py index 759f24fc25e38ca..e6d34d8e6a3be53 100644 --- a/Lib/test/libregrtest/runtests.py +++ b/Lib/test/libregrtest/runtests.py @@ -96,7 +96,7 @@ class RunTests: coverage: bool memory_limit: str | None gc_threshold: int | None - use_resources: tuple[str, ...] + use_resources: dict[str, str | None] python_cmd: tuple[str, ...] | None randomize: bool random_seed: int | str @@ -179,7 +179,14 @@ def bisect_cmd_args(self) -> list[str]: if self.gc_threshold: args.append(f"--threshold={self.gc_threshold}") if self.use_resources: - args.extend(("-u", ','.join(self.use_resources))) + simple = ','.join(resource + for resource, value in self.use_resources.items() + if value is None) + if simple: + args.extend(("-u", simple)) + for resource, value in self.use_resources.items(): + if value is not None: + args.extend(("-u", f"{resource}={value}")) if self.python_cmd: cmd = shlex.join(self.python_cmd) args.extend(("--python", cmd)) diff --git a/Lib/test/libregrtest/save_env.py b/Lib/test/libregrtest/save_env.py index ffc29fa8dc686ac..138465012a252c5 100644 --- a/Lib/test/libregrtest/save_env.py +++ b/Lib/test/libregrtest/save_env.py @@ -9,6 +9,13 @@ from .utils import print_warning +# Import termios to save and restore terminal echo. This is only available on +# Unix, and it's fine if the module can't be found. +try: + import termios # noqa: F401 +except ModuleNotFoundError: + pass + class SkipTestEnvironment(Exception): pass @@ -65,6 +72,7 @@ def __init__(self, test_name, verbose, quiet, *, pgo): 'shutil_archive_formats', 'shutil_unpack_formats', 'asyncio.events._event_loop_policy', 'urllib.requests._url_tempfiles', 'urllib.requests._opener', + 'stty_echo', ) def get_module(self, name): @@ -97,7 +105,7 @@ def get_asyncio_events__event_loop_policy(self): return support.maybe_get_event_loop_policy() def restore_asyncio_events__event_loop_policy(self, policy): asyncio = self.get_module('asyncio') - asyncio._set_event_loop_policy(policy) + asyncio.events._set_event_loop_policy(policy) def get_sys_argv(self): return id(sys.argv), sys.argv, sys.argv[:] @@ -292,6 +300,24 @@ def restore_warnings_showwarning(self, fxn): warnings = self.get_module('warnings') warnings.showwarning = fxn + def get_stty_echo(self): + termios = self.try_get_module('termios') + if not os.isatty(fd := sys.__stdin__.fileno()): + return None + attrs = termios.tcgetattr(fd) + lflags = attrs[3] + return bool(lflags & termios.ECHO) + def restore_stty_echo(self, echo): + termios = self.get_module('termios') + attrs = termios.tcgetattr(fd := sys.__stdin__.fileno()) + if echo: + # Turn echo on. + attrs[3] |= termios.ECHO + else: + # Turn echo off. + attrs[3] &= ~termios.ECHO + termios.tcsetattr(fd, termios.TCSADRAIN, attrs) + def resource_info(self): for name in self.resources: method_suffix = name.replace('.', '_') diff --git a/Lib/test/libregrtest/setup.py b/Lib/test/libregrtest/setup.py index c3d1f60a4006657..d62194acd9c29e5 100644 --- a/Lib/test/libregrtest/setup.py +++ b/Lib/test/libregrtest/setup.py @@ -1,5 +1,6 @@ import faulthandler import gc +import io import os import random import signal @@ -7,6 +8,7 @@ import unittest from test import support from test.support.os_helper import TESTFN_UNDECODABLE, FS_NONASCII +from _colorize import can_colorize # type: ignore[import-not-found] from .filter import set_match_tests from .runtests import RunTests @@ -48,10 +50,26 @@ def setup_process() -> None: for signum in signals: faulthandler.register(signum, chain=True, file=stderr_fd) + # Restore the default SIGINT handler if there is no Python-level + # handler. Python inherits the SIG_IGN disposition when the test + # suite runs as a shell background job; then asyncio.Runner does not + # install its own handler and _thread.interrupt_main() is a no-op, + # which makes some tests in test_asyncio and test_threading hang. + if signal.getsignal(signal.SIGINT) in (signal.SIG_IGN, signal.SIG_DFL): + signal.signal(signal.SIGINT, signal.default_int_handler) + adjust_rlimit_nofile() support.record_original_stdout(sys.stdout) + # Set sys.stdout encoder error handler to backslashreplace, + # similar to sys.stderr error handler, to avoid UnicodeEncodeError + # when printing a traceback or any other non-encodable character. + # + # Use an assertion to fix mypy error. + assert isinstance(sys.stdout, io.TextIOWrapper) + sys.stdout.reconfigure(errors="backslashreplace") + # Some times __path__ and __file__ are not absolute (e.g. while running from # Lib/) and, if we change the CWD to run the tests in a temporary dir, some # imports might fail. This affects only the modules imported before os.chdir(). @@ -130,3 +148,10 @@ def setup_tests(runtests: RunTests) -> None: gc.set_threshold(runtests.gc_threshold) random.seed(runtests.random_seed) + + # sys.stdout is redirected to a StringIO in single process mode on which + # color auto-detect fails as StringIO is not a TTY. If the original + # sys.stdout supports color pass that through with FORCE_COLOR so that when + # results are printed, such as with -W, they get color. + if can_colorize(file=sys.stdout): + os.environ['FORCE_COLOR'] = "1" diff --git a/Lib/test/libregrtest/single.py b/Lib/test/libregrtest/single.py index 57d7b649d2ef630..d0759d2626989d6 100644 --- a/Lib/test/libregrtest/single.py +++ b/Lib/test/libregrtest/single.py @@ -145,7 +145,7 @@ def regrtest_runner(result: TestResult, test_func, runtests: RunTests) -> None: # Storage of uncollectable GC objects (gc.garbage) -GC_GARBAGE = [] +GC_GARBAGE: list[object] = [] def _load_run_test(result: TestResult, runtests: RunTests) -> None: @@ -283,7 +283,7 @@ def _runtest(result: TestResult, runtests: RunTests) -> None: try: setup_tests(runtests) - if output_on_failure: + if output_on_failure or runtests.pgo: support.verbose = True stream = io.StringIO() diff --git a/Lib/test/libregrtest/tsan.py b/Lib/test/libregrtest/tsan.py index d984a735bdf92f9..534b419f1272bd4 100644 --- a/Lib/test/libregrtest/tsan.py +++ b/Lib/test/libregrtest/tsan.py @@ -10,6 +10,7 @@ 'test_ctypes', # 'test_concurrent_futures', # gh-130605: too many data races 'test_enum', + 'test_enumerate', 'test_functools', 'test_httpservers', 'test_imaplib', diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index 63a2e427d185f15..15caba75acfa3da 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -12,7 +12,7 @@ import sysconfig import tempfile import textwrap -from collections.abc import Callable, Iterable +from collections.abc import Callable from test import support from test.support import os_helper @@ -31,7 +31,7 @@ EXIT_TIMEOUT = 120.0 -ALL_RESOURCES = ('audio', 'curses', 'largefile', 'network', +ALL_RESOURCES = ('audio', 'console', 'curses', 'largefile', 'network', 'decimal', 'cpu', 'subprocess', 'urlfetch', 'gui', 'walltime') # Other resources excluded from --use=all: @@ -41,7 +41,7 @@ # - tzdata: while needed to validate fully test_datetime, it makes # test_datetime too slow (15-20 min on some buildbots) and so is disabled by # default (see bpo-30822). -RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata') +RESOURCE_NAMES = ALL_RESOURCES + ('extralargefile', 'tzdata', 'xpickle', 'wantobjects') # Types for types hints @@ -150,7 +150,7 @@ def setup_unraisable_hook() -> None: sys.unraisablehook = regrtest_unraisable_hook -orig_threading_excepthook: Callable[..., None] | None = None +orig_threading_excepthook: Callable[..., object] | None = None def regrtest_threading_excepthook(args) -> None: @@ -433,12 +433,6 @@ def get_temp_dir(tmp_dir: StrPath | None = None) -> StrPath: f"unexpectedly returned {tmp_dir!r} on WASI" ) tmp_dir = os.path.join(tmp_dir, 'build') - - # When get_temp_dir() is called in a worker process, - # get_temp_dir() path is different than in the parent process - # which is not a WASI process. So the parent does not create - # the same "tmp_dir" than the test worker process. - os.makedirs(tmp_dir, exist_ok=True) else: tmp_dir = tempfile.gettempdir() @@ -536,7 +530,7 @@ def normalize_test_name(test_full_name: str, *, if is_error and short_name in _TEST_LIFECYCLE_HOOKS: if test_full_name.startswith(('setUpModule (', 'tearDownModule (')): # if setUpModule() or tearDownModule() failed, don't filter - # tests with the test file name, don't use use filters. + # tests with the test file name, don't use filters. return None # This means that we have a failure in a life-cycle hook, @@ -588,21 +582,30 @@ def is_cross_compiled() -> bool: return ('_PYTHON_HOST_PLATFORM' in os.environ) -def format_resources(use_resources: Iterable[str]) -> str: - use_resources = set(use_resources) +def format_resources(use_resources: dict[str, str | None]) -> str: all_resources = set(ALL_RESOURCES) + values = [] + for name in sorted(use_resources): + if use_resources[name] is not None: + values.append(f'{name}={use_resources[name]}') + # Express resources relative to "all" relative_all = ['all'] - for name in sorted(all_resources - use_resources): + for name in sorted(all_resources - set(use_resources)): relative_all.append(f'-{name}') - for name in sorted(use_resources - all_resources): - relative_all.append(f'{name}') - all_text = ','.join(relative_all) + for name in sorted(set(use_resources) - all_resources): + if use_resources[name] is None: + relative_all.append(name) + all_text = ','.join(relative_all + values) all_text = f"resources: {all_text}" # List of enabled resources - text = ','.join(sorted(use_resources)) + resources = [] + for name in sorted(use_resources): + if use_resources[name] is None: + resources.append(name) + text = ','.join(resources + values) text = f"resources ({len(use_resources)}): {text}" # Pick the shortest string (prefer relative to all if lengths are equal) @@ -612,7 +615,7 @@ def format_resources(use_resources: Iterable[str]) -> str: return text -def display_header(use_resources: tuple[str, ...], +def display_header(use_resources: dict[str, str | None], python_cmd: tuple[str, ...] | None) -> None: # Print basic platform information print("==", platform.python_implementation(), *sys.version.split()) diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py index 5d75bf7ae787ed2..4e69ab9d8fad05f 100644 --- a/Lib/test/libregrtest/worker.py +++ b/Lib/test/libregrtest/worker.py @@ -1,6 +1,7 @@ import subprocess import sys import os +from _colorize import can_colorize # type: ignore[import-not-found] from typing import Any, NoReturn from test.support import os_helper, Py_DEBUG @@ -32,6 +33,12 @@ def create_worker_process(runtests: WorkerRunTests, output_fd: int, env['TEMP'] = tmp_dir env['TMP'] = tmp_dir + # The subcommand is run with a temporary output which means it is not a TTY + # and won't auto-color. The test results are printed to stdout so if we can + # color that have the subprocess use color. + if can_colorize(file=sys.stdout): + env['FORCE_COLOR'] = '1' + # Running the child from the same working directory as regrtest's original # invocation ensures that TEMPDIR for the child is the same when # sysconfig.is_python_build() is true. See issue 15300. @@ -120,6 +127,9 @@ def main() -> NoReturn: worker_json = sys.argv[1] tmp_dir = get_temp_dir() + # get_temp_dir() can be different in the worker and the parent process. + # For example, if --tempdir option is used. + os.makedirs(tmp_dir, exist_ok=True) work_dir = get_work_dir(tmp_dir, worker=True) with exit_timeout(): diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index 68d6bad2094268a..ec2aa59f2cb8728 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -6,7 +6,7 @@ from functools import cmp_to_key from test import seq_tests -from test.support import ALWAYS_EQ, NEVER_EQ +from test.support import ALWAYS_EQ, NEVER_EQ, skip_if_huge_c_stack from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow @@ -32,13 +32,13 @@ def test_init(self): self.assertEqual(a, b) def test_getitem_error(self): - a = [] + a = self.type2test([]) msg = "list indices must be integers or slices" with self.assertRaisesRegex(TypeError, msg): a['a'] def test_setitem_error(self): - a = [] + a = self.type2test([]) msg = "list indices must be integers or slices" with self.assertRaisesRegex(TypeError, msg): a['a'] = "python" @@ -60,6 +60,7 @@ def test_repr(self): self.assertEqual(str(a2), "[0, 1, 2, [...], 3]") self.assertEqual(repr(a2), "[0, 1, 2, [...], 3]") + @skip_if_huge_c_stack(200_000) @skip_wasi_stack_overflow() @skip_emscripten_stack_overflow() def test_repr_deep(self): @@ -561,7 +562,7 @@ def test_constructor_exception_handling(self): class F(object): def __iter__(self): raise KeyboardInterrupt - self.assertRaises(KeyboardInterrupt, list, F()) + self.assertRaises(KeyboardInterrupt, self.type2test, F()) def test_exhausted_iterator(self): a = self.type2test([1, 2, 3]) diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index 009e04e9c0b5226..fb11f4828957ff3 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -332,6 +332,26 @@ class RLockTests(BaseLockTests): """ Tests for recursive locks. """ + def test_repr_count(self): + # see gh-134322: check that count values are correct: + # when a rlock is just created, + # in a second thread when rlock is acquired in the main thread. + lock = self.locktype() + self.assertIn("count=0", repr(lock)) + self.assertIn("= (3,): + # Syntax not compatible with Python 2 + exec(''' +class use_metaclass(object, metaclass=metaclass): + pass +''') +else: + class use_metaclass(object): + __metaclass__ = metaclass + + +# Test classes for reduce_ex + +class R: + def __init__(self, reduce=None): + self.reduce = reduce + def __reduce__(self, proto): + return self.reduce + +class REX: + def __init__(self, reduce_ex=None): + self.reduce_ex = reduce_ex + def __reduce_ex__(self, proto): + return self.reduce_ex + +class REX_one(object): + """No __reduce_ex__ here, but inheriting it from object""" + _reduce_called = 0 + def __reduce__(self): + self._reduce_called = 1 + return REX_one, () + +class REX_two(object): + """No __reduce__ here, but inheriting it from object""" + _proto = None + def __reduce_ex__(self, proto): + self._proto = proto + return REX_two, () + +class REX_three(object): + _proto = None + def __reduce_ex__(self, proto): + self._proto = proto + return REX_two, () + def __reduce__(self): + raise AssertionError("This __reduce__ shouldn't be called") + +class REX_four(object): + """Calling base class method should succeed""" + _proto = None + def __reduce_ex__(self, proto): + self._proto = proto + return object.__reduce_ex__(self, proto) + +class REX_five(object): + """This one used to fail with infinite recursion""" + _reduce_called = 0 + def __reduce__(self): + self._reduce_called = 1 + return object.__reduce__(self) + +class REX_six(object): + """This class is used to check the 4th argument (list iterator) of + the reduce protocol. + """ + def __init__(self, items=None): + self.items = items if items is not None else [] + def __eq__(self, other): + return type(self) is type(other) and self.items == other.items + def append(self, item): + self.items.append(item) + def __reduce__(self): + return type(self), (), None, iter(self.items), None + +class REX_seven(object): + """This class is used to check the 5th argument (dict iterator) of + the reduce protocol. + """ + def __init__(self, table=None): + self.table = table if table is not None else {} + def __eq__(self, other): + return type(self) is type(other) and self.table == other.table + def __setitem__(self, key, value): + self.table[key] = value + def __reduce__(self): + return type(self), (), None, None, iter(self.table.items()) + +class REX_state(object): + """This class is used to check the 3th argument (state) of + the reduce protocol. + """ + def __init__(self, state=None): + self.state = state + def __eq__(self, other): + return type(self) is type(other) and self.state == other.state + def __setstate__(self, state): + self.state = state + def __reduce__(self): + return type(self), (), self.state + +# For test_reduce_ex_None +class REX_None: + """ Setting __reduce_ex__ to None should fail """ + __reduce_ex__ = None + +# For test_reduce_None +class R_None: + """ Setting __reduce__ to None should fail """ + __reduce__ = None + +# For test_pickle_setstate_None +class C_None_setstate: + """ Setting __setstate__ to None should fail """ + def __getstate__(self): + return 1 + + __setstate__ = None + + +# Test classes for newobj + +# For test_newobj_generic and test_newobj_proxies + +class MyInt(int): + sample = 1 + +if sys.version_info >= (3,): + class MyLong(int): + sample = 1 +else: + class MyLong(long): + sample = long(1) + +class MyFloat(float): + sample = 1.0 + +class MyComplex(complex): + sample = 1.0 + 0.0j + +class MyStr(str): + sample = "hello" + +if sys.version_info >= (3,): + class MyUnicode(str): + sample = "hello \u1234" +else: + class MyUnicode(unicode): + sample = unicode(r"hello \u1234", "raw-unicode-escape") + +class MyTuple(tuple): + sample = (1, 2, 3) + +class MyList(list): + sample = [1, 2, 3] + +class MyDict(dict): + sample = {"a": 1, "b": 2} + +class MySet(set): + sample = {"a", "b"} + +class MyFrozenSet(frozenset): + sample = frozenset({"a", "b"}) + +myclasses = [MyInt, MyLong, MyFloat, + MyComplex, + MyStr, MyUnicode, + MyTuple, MyList, MyDict, MySet, MyFrozenSet] + +# For test_newobj_overridden_new +class MyIntWithNew(int): + def __new__(cls, value): + raise AssertionError + +class MyIntWithNew2(MyIntWithNew): + __new__ = int.__new__ + + +# For test_newobj_list_slots +class SlotList(MyList): + __slots__ = ["foo"] + +# Ruff "redefined while unused" false positive here due to `global` variables +# being assigned (and then restored) from within test methods earlier in the file +class SimpleNewObj(int): # noqa: F811 + def __init__(self, *args, **kwargs): + # raise an error, to make sure this isn't called + raise TypeError("SimpleNewObj.__init__() didn't expect to get called") + def __eq__(self, other): + return int(self) == int(other) and self.__dict__ == other.__dict__ + +class ComplexNewObj(SimpleNewObj): + def __getnewargs__(self): + return ('%X' % self, 16) + +class ComplexNewObjEx(SimpleNewObj): + def __getnewargs_ex__(self): + return ('%X' % self,), {'base': 16} + + +class ZeroCopyBytes(bytes): + readonly = True + c_contiguous = True + f_contiguous = True + zero_copy_reconstruct = True + + def __reduce_ex__(self, protocol): + if protocol >= 5: + import pickle + return type(self)._reconstruct, (pickle.PickleBuffer(self),), None + else: + return type(self)._reconstruct, (bytes(self),) + + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, bytes(self)) + + __str__ = __repr__ + + @classmethod + def _reconstruct(cls, obj): + with memoryview(obj) as m: + obj = m.obj + if type(obj) is cls: + # Zero-copy + return obj + else: + return cls(obj) + + +class ZeroCopyBytearray(bytearray): + readonly = False + c_contiguous = True + f_contiguous = True + zero_copy_reconstruct = True + + def __reduce_ex__(self, protocol): + if protocol >= 5: + import pickle + return type(self)._reconstruct, (pickle.PickleBuffer(self),), None + else: + return type(self)._reconstruct, (bytes(self),) + + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, bytes(self)) + + __str__ = __repr__ + + @classmethod + def _reconstruct(cls, obj): + with memoryview(obj) as m: + obj = m.obj + if type(obj) is cls: + # Zero-copy + return obj + else: + return cls(obj) + + +# For test_nested_names +class Nested: + class A: + class B: + class C: + pass + +# For test_py_methods +class PyMethodsTest: + @staticmethod + def cheese(): + return "cheese" + @classmethod + def wine(cls): + assert cls is PyMethodsTest + return "wine" + def biscuits(self): + assert isinstance(self, PyMethodsTest) + return "biscuits" + class Nested: + "Nested class" + @staticmethod + def ketchup(): + return "ketchup" + @classmethod + def maple(cls): + assert cls is PyMethodsTest.Nested + return "maple" + def pie(self): + assert isinstance(self, PyMethodsTest.Nested) + return "pie" + +# For test_c_methods +class Subclass(tuple): + class Nested(str): + pass diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index bdc7ef62943a282..0dc4749ac5fa33d 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -15,6 +15,7 @@ import types import unittest import weakref +import __main__ from textwrap import dedent from http.cookies import SimpleCookie @@ -26,13 +27,15 @@ from test import support from test.support import os_helper from test.support import ( - TestFailed, run_with_locales, no_tracing, + run_with_locales, no_tracing, _2G, _4G, bigmemtest ) from test.support.import_helper import forget from test.support.os_helper import TESTFN from test.support import threading_helper from test.support.warnings_helper import save_restore_warnings_filters +from test import picklecommon +from test.picklecommon import * from pickle import bytes_types @@ -54,6 +57,8 @@ # kind of outer loop. protocols = range(pickle.HIGHEST_PROTOCOL + 1) +FAST_NESTING_LIMIT = 50 + # Return True if opcode code appears in the pickle, else False. def opcode_in_pickle(code, pickle): @@ -132,58 +137,6 @@ def restore(self): if pair is not None: copyreg.add_extension(pair[0], pair[1], code) -class C: - def __eq__(self, other): - return self.__dict__ == other.__dict__ - -class D(C): - def __init__(self, arg): - pass - -class E(C): - def __getinitargs__(self): - return () - -import __main__ -__main__.C = C -C.__module__ = "__main__" -__main__.D = D -D.__module__ = "__main__" -__main__.E = E -E.__module__ = "__main__" - -# Simple mutable object. -class Object: - pass - -# Hashable immutable key object containing unheshable mutable data. -class K: - def __init__(self, value): - self.value = value - - def __reduce__(self): - # Shouldn't support the recursion itself - return K, (self.value,) - -class myint(int): - def __init__(self, x): - self.str = str(x) - -class initarg(C): - - def __init__(self, a, b): - self.a = a - self.b = b - - def __getinitargs__(self): - return self.a, self.b - -class metaclass(type): - pass - -class use_metaclass(object, metaclass=metaclass): - pass - class pickling_metaclass(type): def __eq__(self, other): return (type(self) == type(other) and @@ -198,62 +151,6 @@ def create_dynamic_class(name, bases): return result -class ZeroCopyBytes(bytes): - readonly = True - c_contiguous = True - f_contiguous = True - zero_copy_reconstruct = True - - def __reduce_ex__(self, protocol): - if protocol >= 5: - return type(self)._reconstruct, (pickle.PickleBuffer(self),), None - else: - return type(self)._reconstruct, (bytes(self),) - - def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, bytes(self)) - - __str__ = __repr__ - - @classmethod - def _reconstruct(cls, obj): - with memoryview(obj) as m: - obj = m.obj - if type(obj) is cls: - # Zero-copy - return obj - else: - return cls(obj) - - -class ZeroCopyBytearray(bytearray): - readonly = False - c_contiguous = True - f_contiguous = True - zero_copy_reconstruct = True - - def __reduce_ex__(self, protocol): - if protocol >= 5: - return type(self)._reconstruct, (pickle.PickleBuffer(self),), None - else: - return type(self)._reconstruct, (bytes(self),) - - def __repr__(self): - return "{}({!r})".format(self.__class__.__name__, bytes(self)) - - __str__ = __repr__ - - @classmethod - def _reconstruct(cls, obj): - with memoryview(obj) as m: - obj = m.obj - if type(obj) is cls: - # Zero-copy - return obj - else: - return cls(obj) - - if _testbuffer is not None: class PicklableNDArray: @@ -298,9 +195,10 @@ def __ne__(self, other): return not (self == other) def __repr__(self): - return (f"{type(self)}(shape={self.array.shape}," - f"strides={self.array.strides}, " - f"bytes={self.array.tobytes()})") + return ("{name}(shape={array.shape}," + "strides={array.strides}, " + "bytes={array.tobytes()})").format( + name=type(self).__name__, array=self.array.shape) def __reduce_ex__(self, protocol): if not self.array.contiguous: @@ -1100,6 +998,11 @@ def test_large_32b_binunicode8(self): self.check_unpickling_error((pickle.UnpicklingError, OverflowError), dumped) + def test_large_binstring(self): + errmsg = 'BINSTRING pickle has negative byte count' + with self.assertRaisesRegex(pickle.UnpicklingError, errmsg): + self.loads(b'T\0\0\0\x80') + def test_get(self): pickled = b'((lp100000\ng100000\nt.' unpickled = self.loads(pickled) @@ -1354,9 +1257,9 @@ def check(key, exc): self.loads(b'\x82\x01.') check(None, ValueError) check((), ValueError) - check((__name__,), (TypeError, ValueError)) - check((__name__, "MyList", "x"), (TypeError, ValueError)) - check((__name__, None), (TypeError, ValueError)) + check((MyList.__module__,), (TypeError, ValueError)) + check((MyList.__module__, "MyList", "x"), (TypeError, ValueError)) + check((MyList.__module__, None), (TypeError, ValueError)) check((None, "MyList"), (TypeError, ValueError)) def test_bad_reduce(self): @@ -1670,7 +1573,7 @@ def test_bad_reduce_result(self): self.assertEqual(str(cm.exception), '__reduce__ must return a string or tuple, not list') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) obj = REX((print,)) for proto in protocols: @@ -1680,7 +1583,7 @@ def test_bad_reduce_result(self): self.assertEqual(str(cm.exception), 'tuple returned by __reduce__ must contain 2 through 6 elements') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) obj = REX((print, (), None, None, None, None, None)) for proto in protocols: @@ -1690,7 +1593,7 @@ def test_bad_reduce_result(self): self.assertEqual(str(cm.exception), 'tuple returned by __reduce__ must contain 2 through 6 elements') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_bad_reconstructor(self): obj = REX((42, ())) @@ -1702,7 +1605,7 @@ def test_bad_reconstructor(self): 'first item of the tuple returned by __reduce__ ' 'must be callable, not int') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_unpickleable_reconstructor(self): obj = REX((UnpickleableCallable(), ())) @@ -1711,8 +1614,8 @@ def test_unpickleable_reconstructor(self): with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX reconstructor', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX reconstructor', + f'when serializing {REX.__module__}.REX object']) def test_bad_reconstructor_args(self): obj = REX((print, [])) @@ -1724,7 +1627,7 @@ def test_bad_reconstructor_args(self): 'second item of the tuple returned by __reduce__ ' 'must be a tuple, not list') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_unpickleable_reconstructor_args(self): obj = REX((print, (1, 2, UNPICKLEABLE))) @@ -1734,8 +1637,8 @@ def test_unpickleable_reconstructor_args(self): self.dumps(obj, proto) self.assertEqual(cm.exception.__notes__, [ 'when serializing tuple item 2', - 'when serializing test.pickletester.REX reconstructor arguments', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX reconstructor arguments', + f'when serializing {REX.__module__}.REX object']) def test_bad_newobj_args(self): obj = REX((copyreg.__newobj__, ())) @@ -1747,7 +1650,7 @@ def test_bad_newobj_args(self): 'tuple index out of range', '__newobj__ expected at least 1 argument, got 0'}) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) obj = REX((copyreg.__newobj__, [REX])) for proto in protocols[2:]: @@ -1758,7 +1661,7 @@ def test_bad_newobj_args(self): 'second item of the tuple returned by __reduce__ ' 'must be a tuple, not list') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_bad_newobj_class(self): obj = REX((copyreg.__newobj__, (NoNew(),))) @@ -1768,9 +1671,9 @@ def test_bad_newobj_class(self): self.dumps(obj, proto) self.assertIn(str(cm.exception), { 'first argument to __newobj__() has no __new__', - f'first argument to __newobj__() must be a class, not {__name__}.NoNew'}) + f'first argument to __newobj__() must be a class, not {NoNew.__module__}.NoNew'}) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_wrong_newobj_class(self): obj = REX((copyreg.__newobj__, (str,))) @@ -1781,7 +1684,7 @@ def test_wrong_newobj_class(self): self.assertEqual(str(cm.exception), f'first argument to __newobj__() must be {REX!r}, not {str!r}') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_unpickleable_newobj_class(self): class LocalREX(REX): pass @@ -1809,13 +1712,13 @@ def test_unpickleable_newobj_args(self): if proto >= 2: self.assertEqual(cm.exception.__notes__, [ 'when serializing tuple item 2', - 'when serializing test.pickletester.REX __new__ arguments', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX __new__ arguments', + f'when serializing {REX.__module__}.REX object']) else: self.assertEqual(cm.exception.__notes__, [ 'when serializing tuple item 3', - 'when serializing test.pickletester.REX reconstructor arguments', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX reconstructor arguments', + f'when serializing {REX.__module__}.REX object']) def test_bad_newobj_ex_args(self): obj = REX((copyreg.__newobj_ex__, ())) @@ -1827,7 +1730,7 @@ def test_bad_newobj_ex_args(self): 'not enough values to unpack (expected 3, got 0)', '__newobj_ex__ expected 3 arguments, got 0'}) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) obj = REX((copyreg.__newobj_ex__, 42)) for proto in protocols[2:]: @@ -1838,7 +1741,7 @@ def test_bad_newobj_ex_args(self): 'second item of the tuple returned by __reduce__ ' 'must be a tuple, not int') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) obj = REX((copyreg.__newobj_ex__, (REX, 42, {}))) if self.pickler is pickle._Pickler: @@ -1849,7 +1752,7 @@ def test_bad_newobj_ex_args(self): self.assertEqual(str(cm.exception), 'Value after * must be an iterable, not int') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) else: for proto in protocols[2:]: with self.subTest(proto=proto): @@ -1858,7 +1761,7 @@ def test_bad_newobj_ex_args(self): self.assertEqual(str(cm.exception), 'second argument to __newobj_ex__() must be a tuple, not int') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) obj = REX((copyreg.__newobj_ex__, (REX, (), []))) if self.pickler is pickle._Pickler: @@ -1869,7 +1772,7 @@ def test_bad_newobj_ex_args(self): self.assertEqual(str(cm.exception), 'functools.partial() argument after ** must be a mapping, not list') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) else: for proto in protocols[2:]: with self.subTest(proto=proto): @@ -1878,7 +1781,7 @@ def test_bad_newobj_ex_args(self): self.assertEqual(str(cm.exception), 'third argument to __newobj_ex__() must be a dict, not list') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_bad_newobj_ex__class(self): obj = REX((copyreg.__newobj_ex__, (NoNew(), (), {}))) @@ -1888,9 +1791,9 @@ def test_bad_newobj_ex__class(self): self.dumps(obj, proto) self.assertIn(str(cm.exception), { 'first argument to __newobj_ex__() has no __new__', - f'first argument to __newobj_ex__() must be a class, not {__name__}.NoNew'}) + f'first argument to __newobj_ex__() must be a class, not {NoNew.__module__}.NoNew'}) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_wrong_newobj_ex_class(self): if self.pickler is not pickle._Pickler: @@ -1903,7 +1806,7 @@ def test_wrong_newobj_ex_class(self): self.assertEqual(str(cm.exception), f'first argument to __newobj_ex__() must be {REX}, not {str}') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_unpickleable_newobj_ex_class(self): class LocalREX(REX): pass @@ -1939,22 +1842,22 @@ def test_unpickleable_newobj_ex_args(self): if proto >= 4: self.assertEqual(cm.exception.__notes__, [ 'when serializing tuple item 2', - 'when serializing test.pickletester.REX __new__ arguments', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX __new__ arguments', + f'when serializing {REX.__module__}.REX object']) elif proto >= 2: self.assertEqual(cm.exception.__notes__, [ 'when serializing tuple item 3', 'when serializing tuple item 1', 'when serializing functools.partial state', 'when serializing functools.partial object', - 'when serializing test.pickletester.REX reconstructor', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX reconstructor', + f'when serializing {REX.__module__}.REX object']) else: self.assertEqual(cm.exception.__notes__, [ 'when serializing tuple item 2', 'when serializing tuple item 1', - 'when serializing test.pickletester.REX reconstructor arguments', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX reconstructor arguments', + f'when serializing {REX.__module__}.REX object']) def test_unpickleable_newobj_ex_kwargs(self): obj = REX((copyreg.__newobj_ex__, (REX, (), {'a': UNPICKLEABLE}))) @@ -1965,22 +1868,22 @@ def test_unpickleable_newobj_ex_kwargs(self): if proto >= 4: self.assertEqual(cm.exception.__notes__, [ "when serializing dict item 'a'", - 'when serializing test.pickletester.REX __new__ arguments', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX __new__ arguments', + f'when serializing {REX.__module__}.REX object']) elif proto >= 2: self.assertEqual(cm.exception.__notes__, [ "when serializing dict item 'a'", 'when serializing tuple item 2', 'when serializing functools.partial state', 'when serializing functools.partial object', - 'when serializing test.pickletester.REX reconstructor', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX reconstructor', + f'when serializing {REX.__module__}.REX object']) else: self.assertEqual(cm.exception.__notes__, [ "when serializing dict item 'a'", 'when serializing tuple item 2', - 'when serializing test.pickletester.REX reconstructor arguments', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX reconstructor arguments', + f'when serializing {REX.__module__}.REX object']) def test_unpickleable_state(self): obj = REX_state(UNPICKLEABLE) @@ -1989,8 +1892,8 @@ def test_unpickleable_state(self): with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX_state state', - 'when serializing test.pickletester.REX_state object']) + f'when serializing {REX_state.__module__}.REX_state state', + f'when serializing {REX_state.__module__}.REX_state object']) def test_bad_state_setter(self): if self.pickler is pickle._Pickler: @@ -2004,7 +1907,7 @@ def test_bad_state_setter(self): 'sixth item of the tuple returned by __reduce__ ' 'must be callable, not int') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_unpickleable_state_setter(self): obj = REX((print, (), 'state', None, None, UnpickleableCallable())) @@ -2013,8 +1916,8 @@ def test_unpickleable_state_setter(self): with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX state setter', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX state setter', + f'when serializing {REX.__module__}.REX object']) def test_unpickleable_state_with_state_setter(self): obj = REX((print, (), UNPICKLEABLE, None, None, print)) @@ -2023,8 +1926,8 @@ def test_unpickleable_state_with_state_setter(self): with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX state', - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX state', + f'when serializing {REX.__module__}.REX object']) def test_bad_object_list_items(self): # Issue4176: crash when 4th and 5th items of __reduce__() @@ -2039,7 +1942,7 @@ def test_bad_object_list_items(self): 'fourth item of the tuple returned by __reduce__ ' 'must be an iterator, not int'}) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) if self.pickler is not pickle._Pickler: # Python implementation is less strict and also accepts iterables. @@ -2052,7 +1955,7 @@ def test_bad_object_list_items(self): 'fourth item of the tuple returned by __reduce__ ' 'must be an iterator, not int') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_unpickleable_object_list_items(self): obj = REX_six([1, 2, UNPICKLEABLE]) @@ -2061,8 +1964,8 @@ def test_unpickleable_object_list_items(self): with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX_six item 2', - 'when serializing test.pickletester.REX_six object']) + f'when serializing {REX_six.__module__}.REX_six item 2', + f'when serializing {REX_six.__module__}.REX_six object']) def test_bad_object_dict_items(self): # Issue4176: crash when 4th and 5th items of __reduce__() @@ -2077,7 +1980,7 @@ def test_bad_object_dict_items(self): 'fifth item of the tuple returned by __reduce__ ' 'must be an iterator, not int'}) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) for proto in protocols: obj = REX((dict, (), None, None, iter([('a',)]))) @@ -2088,7 +1991,7 @@ def test_bad_object_dict_items(self): 'not enough values to unpack (expected 2, got 1)', 'dict items iterator must return 2-tuples'}) self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) if self.pickler is not pickle._Pickler: # Python implementation is less strict and also accepts iterables. @@ -2100,7 +2003,7 @@ def test_bad_object_dict_items(self): self.assertEqual(str(cm.exception), 'dict items iterator must return 2-tuples') self.assertEqual(cm.exception.__notes__, [ - 'when serializing test.pickletester.REX object']) + f'when serializing {REX.__module__}.REX object']) def test_unpickleable_object_dict_items(self): obj = REX_seven({'a': UNPICKLEABLE}) @@ -2109,8 +2012,8 @@ def test_unpickleable_object_dict_items(self): with self.assertRaises(CustomError) as cm: self.dumps(obj, proto) self.assertEqual(cm.exception.__notes__, [ - "when serializing test.pickletester.REX_seven item 'a'", - 'when serializing test.pickletester.REX_seven object']) + f"when serializing {REX_seven.__module__}.REX_seven item 'a'", + f'when serializing {REX_seven.__module__}.REX_seven object']) def test_unpickleable_list_items(self): obj = [1, [2, 3, UNPICKLEABLE]] @@ -2203,15 +2106,15 @@ def test_unpickleable_frozenset_items(self): def test_global_lookup_error(self): # Global name does not exist obj = REX('spam') - obj.__module__ = __name__ + obj.__module__ = 'test.picklecommon' for proto in protocols: with self.subTest(proto=proto): with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) self.assertEqual(str(cm.exception), - f"Can't pickle {obj!r}: it's not found as {__name__}.spam") + f"Can't pickle {obj!r}: it's not found as test.picklecommon.spam") self.assertEqual(str(cm.exception.__context__), - f"module '{__name__}' has no attribute 'spam'") + "module 'test.picklecommon' has no attribute 'spam'") obj.__module__ = 'nonexisting' for proto in protocols: @@ -2272,7 +2175,11 @@ def test_nonencodable_module_name_error(self): def test_nested_lookup_error(self): # Nested name does not exist - obj = REX('AbstractPickleTests.spam') + global TestGlobal + class TestGlobal: + class A: + pass + obj = REX('TestGlobal.A.B.C') obj.__module__ = __name__ for proto in protocols: with self.subTest(proto=proto): @@ -2280,9 +2187,9 @@ def test_nested_lookup_error(self): self.dumps(obj, proto) self.assertEqual(str(cm.exception), f"Can't pickle {obj!r}: " - f"it's not found as {__name__}.AbstractPickleTests.spam") + f"it's not found as {__name__}.TestGlobal.A.B.C") self.assertEqual(str(cm.exception.__context__), - "type object 'AbstractPickleTests' has no attribute 'spam'") + "type object 'A' has no attribute 'B'") obj.__module__ = None for proto in protocols: @@ -2290,21 +2197,25 @@ def test_nested_lookup_error(self): with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) self.assertEqual(str(cm.exception), - f"Can't pickle {obj!r}: it's not found as __main__.AbstractPickleTests.spam") + f"Can't pickle {obj!r}: " + f"it's not found as __main__.TestGlobal.A.B.C") self.assertEqual(str(cm.exception.__context__), - "module '__main__' has no attribute 'AbstractPickleTests'") + "module '__main__' has no attribute 'TestGlobal'") def test_wrong_object_lookup_error(self): # Name is bound to different object - obj = REX('AbstractPickleTests') + global TestGlobal + class TestGlobal: + pass + obj = REX('TestGlobal') obj.__module__ = __name__ - AbstractPickleTests.ham = [] for proto in protocols: with self.subTest(proto=proto): with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) self.assertEqual(str(cm.exception), - f"Can't pickle {obj!r}: it's not the same object as {__name__}.AbstractPickleTests") + f"Can't pickle {obj!r}: " + f"it's not the same object as {__name__}.TestGlobal") self.assertIsNone(cm.exception.__context__) obj.__module__ = None @@ -2313,9 +2224,10 @@ def test_wrong_object_lookup_error(self): with self.assertRaises(pickle.PicklingError) as cm: self.dumps(obj, proto) self.assertEqual(str(cm.exception), - f"Can't pickle {obj!r}: it's not found as __main__.AbstractPickleTests") + f"Can't pickle {obj!r}: " + f"it's not found as __main__.TestGlobal") self.assertEqual(str(cm.exception.__context__), - "module '__main__' has no attribute 'AbstractPickleTests'") + "module '__main__' has no attribute 'TestGlobal'") def test_local_lookup_error(self): # Test that whichmodule() errors out cleanly when looking up @@ -2357,6 +2269,7 @@ def test_reduce_None(self): with self.assertRaises(TypeError): self.dumps(c) + @support.skip_if_unlimited_stack_size @no_tracing def test_bad_getattr(self): # Issue #3514: crash when there is an infinite loop in __getattr__ @@ -2423,7 +2336,7 @@ def persistent_id(self, obj): def test_bad_ext_code(self): # This should never happen in normal circumstances, because the type # and the value of the extension code is checked in copyreg.add_extension(). - key = (__name__, 'MyList') + key = (MyList.__module__, 'MyList') def check(code, exc): assert key not in copyreg._extension_registry assert code not in copyreg._inverted_registry @@ -2445,6 +2358,7 @@ def check(code, exc): class AbstractPickleTests: # Subclass must define self.dumps, self.loads. + py_version = sys.version_info # for test_xpickle optimized = False _testdata = AbstractUnpickleTests._testdata @@ -2457,24 +2371,33 @@ def setUp(self): def test_misc(self): # test various datatypes not tested by testdata for proto in protocols: - x = myint(4) - s = self.dumps(x, proto) - y = self.loads(s) - self.assert_is_copy(x, y) + with self.subTest('myint', proto=proto): + if self.py_version < (3, 0) and proto < 2: + self.skipTest('int subclasses are not interoperable with Python 2') + x = myint(4) + s = self.dumps(x, proto) + y = self.loads(s) + self.assert_is_copy(x, y) - x = (1, ()) - s = self.dumps(x, proto) - y = self.loads(s) - self.assert_is_copy(x, y) + with self.subTest('tuple', proto=proto): + x = (1, ()) + s = self.dumps(x, proto) + y = self.loads(s) + self.assert_is_copy(x, y) - x = initarg(1, x) - s = self.dumps(x, proto) - y = self.loads(s) - self.assert_is_copy(x, y) + with self.subTest('initarg', proto=proto): + if self.py_version < (3, 0): + self.skipTest('"classic" classes are not interoperable with Python 2') + x = initarg(1, x) + s = self.dumps(x, proto) + y = self.loads(s) + self.assert_is_copy(x, y) # XXX test __reduce__ protocol? def test_roundtrip_equality(self): + if self.py_version < (3, 0): + self.skipTest('"classic" classes are not interoperable with Python 2') expected = self._testdata for proto in protocols: s = self.dumps(expected, proto) @@ -2671,6 +2594,8 @@ def test_recursive_tuple_and_dict_like_key(self): self._test_recursive_tuple_and_dict_key(REX_seven, asdict=lambda x: x.table) def test_recursive_set(self): + if self.py_version < (3, 4): + self.skipTest('not supported in Python < 3.4') # Set containing an immutable object containing the original set. y = set() y.add(K(y)) @@ -2694,6 +2619,8 @@ def test_recursive_set(self): def test_recursive_inst(self): # Mutable object containing itself. + if self.py_version < (3, 0): + self.skipTest('"classic" classes are not interoperable with Python 2') i = Object() i.attr = i for proto in protocols: @@ -2704,6 +2631,8 @@ def test_recursive_inst(self): self.assertIs(x.attr, x) def test_recursive_multi(self): + if self.py_version < (3, 0): + self.skipTest('"classic" classes are not interoperable with Python 2') l = [] d = {1:l} i = Object() @@ -2718,39 +2647,49 @@ def test_recursive_multi(self): self.assertEqual(list(x[0].attr.keys()), [1]) self.assertIs(x[0].attr[1], x) - def _test_recursive_collection_and_inst(self, factory): + def _test_recursive_collection_and_inst(self, factory, oldminproto=None): + if self.py_version < (3, 0): + self.skipTest('"classic" classes are not interoperable with Python 2') # Mutable object containing a collection containing the original # object. o = Object() o.attr = factory([o]) t = type(o.attr) - for proto in protocols: - s = self.dumps(o, proto) - x = self.loads(s) - self.assertIsInstance(x.attr, t) - self.assertEqual(len(x.attr), 1) - self.assertIsInstance(list(x.attr)[0], Object) - self.assertIs(list(x.attr)[0], x) + with self.subTest('obj -> {t.__name__} -> obj'): + for proto in protocols: + with self.subTest(proto=proto): + s = self.dumps(o, proto) + x = self.loads(s) + self.assertIsInstance(x.attr, t) + self.assertEqual(len(x.attr), 1) + self.assertIsInstance(list(x.attr)[0], Object) + self.assertIs(list(x.attr)[0], x) # Collection containing a mutable object containing the original # collection. o = o.attr - for proto in protocols: - s = self.dumps(o, proto) - x = self.loads(s) - self.assertIsInstance(x, t) - self.assertEqual(len(x), 1) - self.assertIsInstance(list(x)[0], Object) - self.assertIs(list(x)[0].attr, x) + with self.subTest(f'{t.__name__} -> obj -> {t.__name__}'): + if self.py_version < (3, 4) and oldminproto is None: + self.skipTest('not supported in Python < 3.4') + for proto in protocols: + with self.subTest(proto=proto): + if self.py_version < (3, 4) and proto < oldminproto: + self.skipTest(f'requires protocol {oldminproto} in Python < 3.4') + s = self.dumps(o, proto) + x = self.loads(s) + self.assertIsInstance(x, t) + self.assertEqual(len(x), 1) + self.assertIsInstance(list(x)[0], Object) + self.assertIs(list(x)[0].attr, x) def test_recursive_list_and_inst(self): - self._test_recursive_collection_and_inst(list) + self._test_recursive_collection_and_inst(list, oldminproto=0) def test_recursive_tuple_and_inst(self): - self._test_recursive_collection_and_inst(tuple) + self._test_recursive_collection_and_inst(tuple, oldminproto=0) def test_recursive_dict_and_inst(self): - self._test_recursive_collection_and_inst(dict.fromkeys) + self._test_recursive_collection_and_inst(dict.fromkeys, oldminproto=0) def test_recursive_set_and_inst(self): self._test_recursive_collection_and_inst(set) @@ -2759,13 +2698,13 @@ def test_recursive_frozenset_and_inst(self): self._test_recursive_collection_and_inst(frozenset) def test_recursive_list_subclass_and_inst(self): - self._test_recursive_collection_and_inst(MyList) + self._test_recursive_collection_and_inst(MyList, oldminproto=2) def test_recursive_tuple_subclass_and_inst(self): self._test_recursive_collection_and_inst(MyTuple) def test_recursive_dict_subclass_and_inst(self): - self._test_recursive_collection_and_inst(MyDict.fromkeys) + self._test_recursive_collection_and_inst(MyDict.fromkeys, oldminproto=2) def test_recursive_set_subclass_and_inst(self): self._test_recursive_collection_and_inst(MySet) @@ -2825,6 +2764,8 @@ def test_unicode_high_plane(self): def test_unicode_memoization(self): # Repeated str is re-used (even when escapes added). + if self.py_version < (3, 0): + self.skipTest('not supported in Python < 3.0') for proto in protocols: for s in '', 'xyz', 'xyz\n', 'x\\yz', 'x\xa1yz\r': p = self.dumps((s, s), proto) @@ -2844,23 +2785,27 @@ def test_bytes(self): self.assert_is_copy(s, self.loads(p)) def test_bytes_memoization(self): + array_types = [bytes] + if self.py_version >= (3, 4): + array_types += [ZeroCopyBytes] for proto in protocols: - for array_type in [bytes, ZeroCopyBytes]: + for array_type in array_types: for s in b'', b'xyz', b'xyz'*100: + b = array_type(s) + expected = (b, b) if self.py_version >= (3, 0) else (b.decode(),)*2 with self.subTest(proto=proto, array_type=array_type, s=s, independent=False): - b = array_type(s) p = self.dumps((b, b), proto) x, y = self.loads(p) self.assertIs(x, y) - self.assert_is_copy((b, b), (x, y)) + self.assert_is_copy(expected, (x, y)) + b2 = array_type(s) with self.subTest(proto=proto, array_type=array_type, s=s, independent=True): - b1, b2 = array_type(s), array_type(s) - p = self.dumps((b1, b2), proto) - # Note that (b1, b2) = self.loads(p) might have identical - # components, i.e., b1 is b2, but this is not always the + p = self.dumps((b, b2), proto) + # Note that (b, b2) = self.loads(p) might have identical + # components, i.e., b is b2, but this is not always the # case if the content is large (equality still holds). - self.assert_is_copy((b1, b2), self.loads(p)) + self.assert_is_copy(expected, self.loads(p)) def test_bytearray(self): for proto in protocols: @@ -2882,8 +2827,11 @@ def test_bytearray(self): self.assertTrue(opcode_in_pickle(pickle.BYTEARRAY8, p)) def test_bytearray_memoization(self): + array_types = [bytearray] + if self.py_version >= (3, 4): + array_types += [ZeroCopyBytearray] for proto in protocols: - for array_type in [bytearray, ZeroCopyBytearray]: + for array_type in array_types: for s in b'', b'xyz', b'xyz'*100: with self.subTest(proto=proto, array_type=array_type, s=s, independent=False): b = array_type(s) @@ -2907,6 +2855,51 @@ def test_bytearray_memoization(self): self.assertIsNot(b2a, b2b) self.assert_is_copy(b2a, b2b) + def test_picklebuffer_memoization(self): + if self.py_version < (3, 8): + self.skipTest('not supported in Python < 3.8') + array_types = [bytes, bytearray] + for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): + for array_type in array_types: + for s in b'', b'xyz', b'xyz'*100: + with self.subTest(proto=proto, array_type=array_type, s=s, independent=False): + b = pickle.PickleBuffer(array_type(s)) + p = self.dumps((b, b), proto) + b1, b2 = self.loads(p) + self.assertIs(b1, b2) + + with self.subTest(proto=proto, array_type=array_type, s=s, independent=True): + b = array_type(s) + b1a = pickle.PickleBuffer(b) + b2a = pickle.PickleBuffer(b) + p = self.dumps((b1a, b2a), proto) + b1b, b2b = self.loads(p) + if array_type is not bytes: + self.assertIsNot(b1b, b2b) + self.assert_is_copy(b1b, b) + self.assert_is_copy(b2b, b) + + def test_empty_picklebuffer_memoization(self): + # gh-148914: Empty writable PickleBuffer memoized an empty bytearray + # with the id of b'' (a singleton in CPython). + if self.py_version < (3, 8): + self.skipTest('not supported in Python < 3.8') + for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): + for readonly in False, True: + with self.subTest(proto=proto, readonly=readonly): + b = b'' + ba = bytearray() + buf = pickle.PickleBuffer(b if readonly else ba) + p = self.dumps((buf, b, ba), proto) + buf, b, ba = self.loads(p) + array_type = bytes if readonly else bytearray + self.assertIsInstance(buf, array_type) + self.assertIsInstance(b, bytes) + self.assertIsInstance(ba, bytearray) + self.assertEqual(buf, b'') + self.assertEqual(b, b'') + self.assertEqual(ba, b'') + def test_ints(self): for proto in protocols: n = sys.maxsize @@ -2957,12 +2950,17 @@ def test_float_format(self): def test_reduce(self): for proto in protocols: - inst = AAA() - dumped = self.dumps(inst, proto) - loaded = self.loads(dumped) - self.assertEqual(loaded, REDUCE_A) + with self.subTest(proto=proto): + if self.py_version < (3, 4) and proto < 3: + self.skipTest('str is not interoperable with Python < 3.4') + inst = AAA() + dumped = self.dumps(inst, proto) + loaded = self.loads(dumped) + self.assertEqual(loaded, REDUCE_A) def test_getinitargs(self): + if self.py_version < (3, 0): + self.skipTest('"classic" classes are not interoperable with Python 2') for proto in protocols: inst = initarg(1, 2) dumped = self.dumps(inst, proto) @@ -2970,6 +2968,7 @@ def test_getinitargs(self): self.assert_is_copy(inst, loaded) def test_metaclass(self): + self.assertEqual(type(use_metaclass), metaclass) a = use_metaclass() for proto in protocols: s = self.dumps(a, proto) @@ -2994,6 +2993,10 @@ def test_structseq(self): s = self.dumps(t, proto) u = self.loads(s) self.assert_is_copy(t, u) + if self.py_version < (3, 4): + # module 'os' has no attributes '_make_stat_result' and + # '_make_statvfs_result' + continue t = os.stat(os.curdir) s = self.dumps(t, proto) u = self.loads(s) @@ -3005,52 +3008,111 @@ def test_structseq(self): self.assert_is_copy(t, u) def test_ellipsis(self): + if self.py_version < (3, 3): + self.skipTest('not supported in Python < 3.3') for proto in protocols: - s = self.dumps(..., proto) - u = self.loads(s) - self.assertIs(..., u) + with self.subTest(proto=proto): + s = self.dumps(..., proto) + u = self.loads(s) + self.assertIs(..., u) def test_notimplemented(self): + if self.py_version < (3, 3): + self.skipTest('not supported in Python < 3.3') for proto in protocols: - s = self.dumps(NotImplemented, proto) - u = self.loads(s) - self.assertIs(NotImplemented, u) + with self.subTest(proto=proto): + s = self.dumps(NotImplemented, proto) + u = self.loads(s) + self.assertIs(NotImplemented, u) def test_singleton_types(self): # Issue #6477: Test that types of built-in singletons can be pickled. + if self.py_version < (3, 3): + self.skipTest('not supported in Python < 3.3') singletons = [None, ..., NotImplemented] for singleton in singletons: + t = type(singleton) for proto in protocols: - s = self.dumps(type(singleton), proto) - u = self.loads(s) - self.assertIs(type(singleton), u) + with self.subTest(name=t.__name__, proto=proto): + s = self.dumps(t, proto) + u = self.loads(s) + self.assertIs(t, u) def test_builtin_types(self): + new_names = { + 'bytes': (3, 0), + 'BuiltinImporter': (3, 3), + 'str': (3, 4), # not interoperable with Python < 3.4 + } for t in builtins.__dict__.values(): if isinstance(t, type) and not issubclass(t, BaseException): + if t.__name__ in new_names and self.py_version < new_names[t.__name__]: + continue for proto in protocols: - s = self.dumps(t, proto) - self.assertIs(self.loads(s), t) + with self.subTest(name=t.__name__, proto=proto): + s = self.dumps(t, proto) + self.assertIs(self.loads(s), t) def test_builtin_exceptions(self): + new_names = { + 'BlockingIOError': (3, 3), + 'BrokenPipeError': (3, 3), + 'ChildProcessError': (3, 3), + 'ConnectionError': (3, 3), + 'ConnectionAbortedError': (3, 3), + 'ConnectionRefusedError': (3, 3), + 'ConnectionResetError': (3, 3), + 'FileExistsError': (3, 3), + 'FileNotFoundError': (3, 3), + 'InterruptedError': (3, 3), + 'IsADirectoryError': (3, 3), + 'NotADirectoryError': (3, 3), + 'PermissionError': (3, 3), + 'ProcessLookupError': (3, 3), + 'TimeoutError': (3, 3), + 'RecursionError': (3, 5), + 'StopAsyncIteration': (3, 5), + 'ModuleNotFoundError': (3, 6), + 'EncodingWarning': (3, 10), + 'BaseExceptionGroup': (3, 11), + 'ExceptionGroup': (3, 11), + '_IncompleteInputError': (3, 13), + 'PythonFinalizationError': (3, 13), + } for t in builtins.__dict__.values(): if isinstance(t, type) and issubclass(t, BaseException): + if t.__name__ in new_names and self.py_version < new_names[t.__name__]: + continue for proto in protocols: - s = self.dumps(t, proto) - u = self.loads(s) - if proto <= 2 and issubclass(t, OSError) and t is not BlockingIOError: - self.assertIs(u, OSError) - elif proto <= 2 and issubclass(t, ImportError): - self.assertIs(u, ImportError) - else: - self.assertIs(u, t) + with self.subTest(name=t.__name__, proto=proto): + if self.py_version < (3, 3) and proto < 3: + self.skipTest('exception classes are not interoperable with Python < 3.3') + s = self.dumps(t, proto) + u = self.loads(s) + if proto <= 2 and issubclass(t, OSError) and t is not BlockingIOError: + self.assertIs(u, OSError) + elif proto <= 2 and issubclass(t, ImportError): + self.assertIs(u, ImportError) + else: + self.assertIs(u, t) def test_builtin_functions(self): + new_names = { + '__build_class__': (3, 0), + 'ascii': (3, 0), + 'exec': (3, 0), + 'breakpoint': (3, 7), + 'aiter': (3, 10), + 'anext': (3, 10), + } for t in builtins.__dict__.values(): if isinstance(t, types.BuiltinFunctionType): + if t.__name__ in new_names and self.py_version < new_names[t.__name__]: + continue for proto in protocols: - s = self.dumps(t, proto) - self.assertIs(self.loads(s), t) + with self.subTest(name=t.__name__, proto=proto): + s = self.dumps(t, proto) + self.assertIs(self.loads(s), t) # Tests for protocol 2 @@ -3059,10 +3121,13 @@ def test_proto(self): pickled = self.dumps(None, proto) if proto >= 2: proto_header = pickle.PROTO + bytes([proto]) - self.assertTrue(pickled.startswith(proto_header)) + self.assertStartsWith(pickled, proto_header) else: self.assertEqual(count_opcode(pickle.PROTO, pickled), 0) + def test_bad_proto(self): + if self.py_version < (3, 8): + self.skipTest('no protocol validation in Python < 3.8') oob = protocols[-1] + 1 # a future protocol build_none = pickle.NONE + pickle.STOP badpickle = pickle.PROTO + bytes([oob]) + build_none @@ -3174,57 +3239,65 @@ def test_newobj_list(self): def test_newobj_generic(self): for proto in protocols: for C in myclasses: - B = C.__base__ - x = C(C.sample) - x.foo = 42 - s = self.dumps(x, proto) - y = self.loads(s) - detail = (proto, C, B, x, y, type(y)) - self.assert_is_copy(x, y) # XXX revisit - self.assertEqual(B(x), B(y), detail) - self.assertEqual(x.__dict__, y.__dict__, detail) + with self.subTest(proto=proto, C=C): + if self.py_version < (3, 0) and proto < 2 and C in (MyInt, MyStr): + self.skipTest('int and str subclasses are not interoperable with Python 2') + if (3, 0) <= self.py_version < (3, 4) and proto < 2 and C in (MyStr, MyUnicode): + self.skipTest('str subclasses are not interoperable with Python < 3.4') + B = C.__base__ + x = C(C.sample) + x.foo = 42 + s = self.dumps(x, proto) + y = self.loads(s) + detail = (proto, C, B, x, y, type(y)) + self.assert_is_copy(x, y) # XXX revisit + self.assertEqual(B(x), B(y), detail) + self.assertEqual(x.__dict__, y.__dict__, detail) def test_newobj_proxies(self): # NEWOBJ should use the __class__ rather than the raw type classes = myclasses[:] # Cannot create weakproxies to these classes - for c in (MyInt, MyTuple): + for c in (MyInt, MyLong, MyTuple): classes.remove(c) for proto in protocols: for C in classes: - B = C.__base__ - x = C(C.sample) - x.foo = 42 - p = weakref.proxy(x) - s = self.dumps(p, proto) - y = self.loads(s) - self.assertEqual(type(y), type(x)) # rather than type(p) - detail = (proto, C, B, x, y, type(y)) - self.assertEqual(B(x), B(y), detail) - self.assertEqual(x.__dict__, y.__dict__, detail) + with self.subTest(proto=proto, C=C): + if self.py_version < (3, 4) and proto < 3 and C in (MyStr, MyUnicode): + self.skipTest('str subclasses are not interoperable with Python < 3.4') + B = C.__base__ + x = C(C.sample) + x.foo = 42 + p = weakref.proxy(x) + s = self.dumps(p, proto) + y = self.loads(s) + self.assertEqual(type(y), type(x)) # rather than type(p) + detail = (proto, C, B, x, y, type(y)) + self.assertEqual(B(x), B(y), detail) + self.assertEqual(x.__dict__, y.__dict__, detail) def test_newobj_overridden_new(self): # Test that Python class with C implemented __new__ is pickleable for proto in protocols: - x = MyIntWithNew2(1) - x.foo = 42 - s = self.dumps(x, proto) - y = self.loads(s) - self.assertIs(type(y), MyIntWithNew2) - self.assertEqual(int(y), 1) - self.assertEqual(y.foo, 42) + with self.subTest(proto=proto): + if self.py_version < (3, 0) and proto < 2: + self.skipTest('int subclasses are not interoperable with Python 2') + x = MyIntWithNew2(1) + x.foo = 42 + s = self.dumps(x, proto) + y = self.loads(s) + self.assertIs(type(y), MyIntWithNew2) + self.assertEqual(int(y), 1) + self.assertEqual(y.foo, 42) def test_newobj_not_class(self): # Issue 24552 - global SimpleNewObj - save = SimpleNewObj + if self.py_version < (3, 4): + self.skipTest('not supported in Python < 3.4') o = SimpleNewObj.__new__(SimpleNewObj) b = self.dumps(o, 4) - try: - SimpleNewObj = 42 + with support.swap_attr(picklecommon, 'SimpleNewObj', 42): self.assertRaises((TypeError, pickle.UnpicklingError), self.loads, b) - finally: - SimpleNewObj = save # Register a type with copyreg, with extension code extcode. Pickle # an object of that type. Check that the resulting pickle uses opcode @@ -3233,14 +3306,14 @@ def test_newobj_not_class(self): def produce_global_ext(self, extcode, opcode): e = ExtensionSaver(extcode) try: - copyreg.add_extension(__name__, "MyList", extcode) + copyreg.add_extension(MyList.__module__, "MyList", extcode) x = MyList([1, 2, 3]) x.foo = 42 x.bar = "hello" # Dump using protocol 1 for comparison. s1 = self.dumps(x, 1) - self.assertIn(__name__.encode("utf-8"), s1) + self.assertIn(MyList.__module__.encode(), s1) self.assertIn(b"MyList", s1) self.assertFalse(opcode_in_pickle(opcode, s1)) @@ -3249,7 +3322,7 @@ def produce_global_ext(self, extcode, opcode): # Dump using protocol 2 for test. s2 = self.dumps(x, 2) - self.assertNotIn(__name__.encode("utf-8"), s2) + self.assertNotIn(MyList.__module__.encode(), s2) self.assertNotIn(b"MyList", s2) self.assertEqual(opcode_in_pickle(opcode, s2), True, repr(s2)) @@ -3347,14 +3420,20 @@ def test_simple_newobj(self): x.abc = 666 for proto in protocols: with self.subTest(proto=proto): + if self.py_version < (3, 0) and proto < 2: + self.skipTest('int subclasses are not interoperable with Python 2') s = self.dumps(x, proto) if proto < 1: - self.assertIn(b'\nI64206', s) # INT + if self.py_version >= (3, 7): + self.assertIn(b'\nI64206', s) # INT + else: # for test_xpickle + self.assertIn(b'64206', s) # INT or LONG else: self.assertIn(b'M\xce\xfa', s) # BININT2 - self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), - 2 <= proto) - self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s)) + if not (self.py_version < (3, 5) and proto == 4): + self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), + 2 <= proto) + self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s)) y = self.loads(s) # will raise TypeError if __init__ called self.assert_is_copy(x, y) @@ -3363,29 +3442,45 @@ def test_complex_newobj(self): x.abc = 666 for proto in protocols: with self.subTest(proto=proto): + if self.py_version < (3, 0) and proto < 2: + self.skipTest('int subclasses are not interoperable with Python 2') s = self.dumps(x, proto) if proto < 1: - self.assertIn(b'\nI64206', s) # INT + if self.py_version >= (3, 7): + self.assertIn(b'\nI64206', s) # INT + else: # for test_xpickle + self.assertIn(b'64206', s) # INT or LONG elif proto < 2: self.assertIn(b'M\xce\xfa', s) # BININT2 elif proto < 4: - self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE + if self.py_version >= (3, 0): + self.assertIn(b'X\x04\x00\x00\x00FACE', s) # BINUNICODE + else: # for test_xpickle + self.assertIn(b'U\x04FACE', s) # SHORT_BINSTRING else: self.assertIn(b'\x8c\x04FACE', s) # SHORT_BINUNICODE - self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), - 2 <= proto) - self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s)) + if not (self.py_version < (3, 5) and proto == 4): + self.assertEqual(opcode_in_pickle(pickle.NEWOBJ, s), + 2 <= proto) + self.assertFalse(opcode_in_pickle(pickle.NEWOBJ_EX, s)) y = self.loads(s) # will raise TypeError if __init__ called self.assert_is_copy(x, y) def test_complex_newobj_ex(self): + if self.py_version < (3, 4): + self.skipTest('not supported in Python < 3.4') x = ComplexNewObjEx.__new__(ComplexNewObjEx, 0xface) # avoid __init__ x.abc = 666 for proto in protocols: with self.subTest(proto=proto): + if self.py_version < (3, 6) and proto < 4: + self.skipTest('requires protocol 4 in Python < 3.6') s = self.dumps(x, proto) if proto < 1: - self.assertIn(b'\nI64206', s) # INT + if self.py_version >= (3, 7): + self.assertIn(b'\nI64206', s) # INT + else: # for test_xpickle + self.assertIn(b'64206', s) # INT or LONG elif proto < 2: self.assertIn(b'M\xce\xfa', s) # BININT2 elif proto < 4: @@ -3473,6 +3568,8 @@ def test_many_puts_and_gets(self): def test_attribute_name_interning(self): # Test that attribute names of pickled objects are interned when # unpickling. + if self.py_version < (3, 0): + self.skipTest('"classic" classes are not interoperable with Python 2') for proto in protocols: x = C() x.foo = 42 @@ -3502,10 +3599,14 @@ def test_large_pickles(self): dumped = self.dumps(data, proto) loaded = self.loads(dumped) self.assertEqual(len(loaded), len(data)) + if self.py_version < (3, 0): + data = (1, min, 'xy' * (30 * 1024), len) self.assertEqual(loaded, data) def test_int_pickling_efficiency(self): # Test compacity of int representation (see issue #12744) + if self.py_version < (3, 3): + self.skipTest('not supported in Python < 3.3') for proto in protocols: with self.subTest(proto=proto): pickles = [self.dumps(2**n, proto) for n in range(70)] @@ -3526,10 +3627,13 @@ def test_appends_on_non_lists(self): # Issue #17720 obj = REX_six([1, 2, 3]) for proto in protocols: - if proto == 0: - self._check_pickling_with_opcode(obj, pickle.APPEND, proto) - else: - self._check_pickling_with_opcode(obj, pickle.APPENDS, proto) + with self.subTest(proto=proto): + if proto == 0: + self._check_pickling_with_opcode(obj, pickle.APPEND, proto) + else: + if self.py_version < (3, 0): + self.skipTest('not supported in Python 2') + self._check_pickling_with_opcode(obj, pickle.APPENDS, proto) def test_setitems_on_non_dicts(self): obj = REX_seven({1: -1, 2: -2, 3: -3}) @@ -3594,6 +3698,8 @@ def check_frame_opcodes(self, pickled): @support.skip_if_pgo_task @support.requires_resource('cpu') def test_framing_many_objects(self): + if self.py_version < (3, 4): + self.skipTest('not supported in Python < 3.4') obj = list(range(10**5)) for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): with self.subTest(proto=proto): @@ -3609,6 +3715,8 @@ def test_framing_many_objects(self): self.check_frame_opcodes(pickled) def test_framing_large_objects(self): + if self.py_version < (3, 4): + self.skipTest('not supported in Python < 3.4') N = 1024 * 1024 small_items = [[i] for i in range(10)] obj = [b'x' * N, *small_items, b'y' * N, 'z' * N] @@ -3634,15 +3742,16 @@ def test_framing_large_objects(self): [len(x) for x in unpickled]) # Perform full equality check if the lengths match. self.assertEqual(obj, unpickled) - n_frames = count_opcode(pickle.FRAME, pickled) - # A single frame for small objects between - # first two large objects. - self.assertEqual(n_frames, 1) - self.check_frame_opcodes(pickled) + if self.py_version >= (3, 7): + n_frames = count_opcode(pickle.FRAME, pickled) + # A single frame for small objects between + # first two large objects. + self.assertEqual(n_frames, 1) + self.check_frame_opcodes(pickled) def test_optional_frames(self): - if pickle.HIGHEST_PROTOCOL < 4: - return + if self.py_version < (3, 4): + self.skipTest('not supported in Python < 3.4') def remove_frames(pickled, keep_frame=None): """Remove frame opcodes from the given pickle.""" @@ -3684,6 +3793,9 @@ def remove_frames(pickled, keep_frame=None): @support.skip_if_pgo_task def test_framed_write_sizes_with_delayed_writer(self): + if self.py_version < (3, 4): + self.skipTest('not supported in Python < 3.4') + class ChunkAccumulator: """Accumulate pickler output in a list of raw chunks.""" def __init__(self): @@ -3749,13 +3861,12 @@ def concatenate_chunks(self): chunk_sizes) def test_nested_names(self): - global Nested - class Nested: - class A: - class B: - class C: - pass + if self.py_version < (3, 4): + self.skipTest('not supported in Python < 3.4') + # required protocol 4 in Python 3.4 for proto in range(pickle.HIGHEST_PROTOCOL + 1): + if self.py_version < (3, 5) and proto < 4: + continue for obj in [Nested.A, Nested.A.B, Nested.A.B.C]: with self.subTest(proto=proto, obj=obj): unpickled = self.loads(self.dumps(obj, proto)) @@ -3786,35 +3897,21 @@ class Recursive: del Recursive.ref # break reference loop def test_py_methods(self): - global PyMethodsTest - class PyMethodsTest: - @staticmethod - def cheese(): - return "cheese" - @classmethod - def wine(cls): - assert cls is PyMethodsTest - return "wine" - def biscuits(self): - assert isinstance(self, PyMethodsTest) - return "biscuits" - class Nested: - "Nested class" - @staticmethod - def ketchup(): - return "ketchup" - @classmethod - def maple(cls): - assert cls is PyMethodsTest.Nested - return "maple" - def pie(self): - assert isinstance(self, PyMethodsTest.Nested) - return "pie" - + if self.py_version < (3, 4): + self.skipTest('not supported in Python < 3.4') py_methods = ( - PyMethodsTest.cheese, PyMethodsTest.wine, PyMethodsTest().biscuits, + ) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + for method in py_methods: + with self.subTest(proto=proto, method=method): + unpickled = self.loads(self.dumps(method, proto)) + self.assertEqual(method(), unpickled()) + + # required protocol 4 in Python 3.4 + py_methods = ( + PyMethodsTest.cheese, PyMethodsTest.Nested.ketchup, PyMethodsTest.Nested.maple, PyMethodsTest.Nested().pie @@ -3824,6 +3921,8 @@ def pie(self): (PyMethodsTest.Nested.pie, PyMethodsTest.Nested) ) for proto in range(pickle.HIGHEST_PROTOCOL + 1): + if self.py_version < (3, 5) and proto < 4: + continue for method in py_methods: with self.subTest(proto=proto, method=method): unpickled = self.loads(self.dumps(method, proto)) @@ -3844,11 +3943,8 @@ def pie(self): self.assertRaises(TypeError, self.dumps, descr, proto) def test_c_methods(self): - global Subclass - class Subclass(tuple): - class Nested(str): - pass - + if self.py_version < (3, 4): + self.skipTest('not supported in Python < 3.4') c_methods = ( # bound built-in method ("abcd".index, ("c",)), @@ -3869,7 +3965,6 @@ class Nested(str): # subclass methods (Subclass([1,2,2]).count, (2,)), (Subclass.count, (Subclass([1,2,2]), 2)), - (Subclass.Nested("sweet").count, ("e",)), (Subclass.Nested.count, (Subclass.Nested("sweet"), "e")), ) for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -3878,6 +3973,18 @@ class Nested(str): unpickled = self.loads(self.dumps(method, proto)) self.assertEqual(method(*args), unpickled(*args)) + # required protocol 4 in Python 3.4 + c_methods = ( + (Subclass.Nested("sweet").count, ("e",)), + ) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + if self.py_version < (3, 5) and proto < 4: + continue + for method, args in c_methods: + with self.subTest(proto=proto, method=method): + unpickled = self.loads(self.dumps(method, proto)) + self.assertEqual(method(*args), unpickled(*args)) + descriptors = ( bytearray.__dict__['maketrans'], # built-in static method descriptor dict.__dict__['fromkeys'], # built-in class method descriptor @@ -3887,7 +3994,85 @@ class Nested(str): with self.subTest(proto=proto, descr=descr): self.assertRaises(TypeError, self.dumps, descr, proto) + def test_object_with_attrs(self): + obj = Object() + obj.a = 1 + for proto in protocols: + with self.subTest(proto=proto): + unpickled = self.loads(self.dumps(obj, proto)) + self.assertEqual(unpickled.a, obj.a) + + def test_object_with_slots(self): + obj = WithSlots() + obj.a = 1 + self.assertRaises(TypeError, self.dumps, obj, 0) + self.assertRaises(TypeError, self.dumps, obj, 1) + for proto in protocols[2:]: + with self.subTest(proto=proto): + unpickled = self.loads(self.dumps(obj, proto)) + self.assertEqual(unpickled.a, obj.a) + self.assertNotHasAttr(unpickled, 'b') + + obj = WithSlotsSubclass() + obj.a = 1 + obj.c = 2 + self.assertRaises(TypeError, self.dumps, obj, 0) + self.assertRaises(TypeError, self.dumps, obj, 1) + for proto in protocols[2:]: + with self.subTest(proto=proto): + unpickled = self.loads(self.dumps(obj, proto)) + self.assertEqual(unpickled.a, obj.a) + self.assertEqual(unpickled.c, obj.c) + self.assertNotHasAttr(unpickled, 'b') + + obj = WithSlotsAndDict() + obj.a = 1 + obj.c = 2 + self.assertRaises(TypeError, self.dumps, obj, 0) + self.assertRaises(TypeError, self.dumps, obj, 1) + for proto in protocols[2:]: + with self.subTest(proto=proto): + unpickled = self.loads(self.dumps(obj, proto)) + self.assertEqual(unpickled.a, obj.a) + self.assertEqual(unpickled.c, obj.c) + self.assertEqual(unpickled.__dict__, obj.__dict__) + self.assertNotHasAttr(unpickled, 'b') + + def test_object_with_private_attrs(self): + obj = WithPrivateAttrs(1) + for proto in protocols: + with self.subTest(proto=proto): + unpickled = self.loads(self.dumps(obj, proto)) + self.assertEqual(unpickled.get(), obj.get()) + + obj = WithPrivateAttrsSubclass(1, 2) + for proto in protocols: + with self.subTest(proto=proto): + unpickled = self.loads(self.dumps(obj, proto)) + self.assertEqual(unpickled.get(), obj.get()) + self.assertEqual(unpickled.get2(), obj.get2()) + + def test_object_with_private_slots(self): + obj = WithPrivateSlots(1) + self.assertRaises(TypeError, self.dumps, obj, 0) + self.assertRaises(TypeError, self.dumps, obj, 1) + for proto in protocols[2:]: + with self.subTest(proto=proto): + unpickled = self.loads(self.dumps(obj, proto)) + self.assertEqual(unpickled.get(), obj.get()) + + obj = WithPrivateSlotsSubclass(1, 2) + self.assertRaises(TypeError, self.dumps, obj, 0) + self.assertRaises(TypeError, self.dumps, obj, 1) + for proto in protocols[2:]: + with self.subTest(proto=proto): + unpickled = self.loads(self.dumps(obj, proto)) + self.assertEqual(unpickled.get(), obj.get()) + self.assertEqual(unpickled.get2(), obj.get2()) + def test_compat_pickle(self): + if self.py_version < (3, 4): + self.skipTest("doesn't work in Python < 3.4'") tests = [ (range(1, 7), '__builtin__', 'xrange'), (map(int, '123'), 'itertools', 'imap'), @@ -4076,6 +4261,35 @@ def check_array(arr): # 2-D, non-contiguous check_array(arr[::2]) + def test_concurrent_mutation_in_buffer_with_bytearray(self): + def factory(): + s = b"a" * 16 + return bytearray(s), s + self.do_test_concurrent_mutation_in_buffer_callback(factory) + + def test_concurrent_mutation_in_buffer_with_memoryview(self): + def factory(): + obj = memoryview(b"a" * 32)[10:26] + sub = b"a" * len(obj) + return obj, sub + self.do_test_concurrent_mutation_in_buffer_callback(factory) + + def do_test_concurrent_mutation_in_buffer_callback(self, factory): + # See: https://github.com/python/cpython/issues/143308. + class R: + def __bool__(self): + buf.release() + return True + + for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): + obj, sub = factory() + buf = pickle.PickleBuffer(obj) + buffer_callback = lambda _: R() + + with self.subTest(proto=proto, obj=obj, sub=sub): + res = self.dumps(buf, proto, buffer_callback=buffer_callback) + self.assertIn(sub, res) + def test_evil_class_mutating_dict(self): # https://github.com/python/cpython/issues/92930 from random import getrandbits @@ -4106,6 +4320,94 @@ def __reduce__(self): expected = "changed size during iteration" self.assertIn(expected, str(e)) + def fast_save_enter(self, create_data, minprotocol=0): + # gh-146059: Check that fast_save_leave() is called when + # fast_save_enter() is called. + if not hasattr(self, "pickler"): + self.skipTest("need Pickler class") + + data = [create_data(i) for i in range(FAST_NESTING_LIMIT * 2)] + protocols = range(minprotocol, pickle.HIGHEST_PROTOCOL + 1) + for proto in protocols: + with self.subTest(proto=proto): + buf = io.BytesIO() + pickler = self.pickler(buf, protocol=proto) + # Enable fast mode (disables memo, enables cycle detection) + pickler.fast = 1 + pickler.dump(data) + + buf.seek(0) + data2 = self.unpickler(buf).load() + self.assertEqual(data2, data) + + def test_fast_save_enter_tuple(self): + self.fast_save_enter(lambda i: (i,)) + + def test_fast_save_enter_list(self): + self.fast_save_enter(lambda i: [i]) + + def test_fast_save_enter_frozenset(self): + self.fast_save_enter(lambda i: frozenset([i])) + + def test_fast_save_enter_set(self): + self.fast_save_enter(lambda i: set([i])) + + def test_fast_save_enter_frozendict(self): + if self.py_version < (3, 15): + self.skipTest('need frozendict') + self.fast_save_enter(lambda i: frozendict(key=i), minprotocol=2) + + def test_fast_save_enter_dict(self): + self.fast_save_enter(lambda i: {"key": i}) + + def deep_nested_struct(self, create_nested, + minprotocol=0, compare_equal=True, + depth=FAST_NESTING_LIMIT * 2): + # gh-146059: Check that fast_save_leave() is called when + # fast_save_enter() is called. + if not hasattr(self, "pickler"): + self.skipTest("need Pickler class") + + data = None + for i in range(depth): + data = create_nested(data) + protocols = range(minprotocol, pickle.HIGHEST_PROTOCOL + 1) + for proto in protocols: + with self.subTest(proto=proto): + buf = io.BytesIO() + pickler = self.pickler(buf, protocol=proto) + # Enable fast mode (disables memo, enables cycle detection) + pickler.fast = 1 + pickler.dump(data) + + buf.seek(0) + data2 = self.unpickler(buf).load() + if compare_equal: + self.assertEqual(data2, data) + + def test_deep_nested_struct_tuple(self): + self.deep_nested_struct(lambda data: (data,)) + + def test_deep_nested_struct_list(self): + self.deep_nested_struct(lambda data: [data]) + + def test_deep_nested_struct_frozenset(self): + self.deep_nested_struct(lambda data: frozenset((1, data))) + + def test_deep_nested_struct_set(self): + self.deep_nested_struct(lambda data: {K(data)}, + depth=FAST_NESTING_LIMIT+1, + compare_equal=False) + + def test_deep_nested_struct_frozendict(self): + if self.py_version < (3, 15): + self.skipTest('need frozendict') + self.deep_nested_struct(lambda data: frozendict(x=data), + minprotocol=2) + + def test_deep_nested_struct_dict(self): + self.deep_nested_struct(lambda data: {'x': data}) + class BigmemPickleTests: @@ -4234,110 +4536,6 @@ def test_huge_str_64b(self, size): data = None -# Test classes for reduce_ex - -class R: - def __init__(self, reduce=None): - self.reduce = reduce - def __reduce__(self, proto): - return self.reduce - -class REX: - def __init__(self, reduce_ex=None): - self.reduce_ex = reduce_ex - def __reduce_ex__(self, proto): - return self.reduce_ex - -class REX_one(object): - """No __reduce_ex__ here, but inheriting it from object""" - _reduce_called = 0 - def __reduce__(self): - self._reduce_called = 1 - return REX_one, () - -class REX_two(object): - """No __reduce__ here, but inheriting it from object""" - _proto = None - def __reduce_ex__(self, proto): - self._proto = proto - return REX_two, () - -class REX_three(object): - _proto = None - def __reduce_ex__(self, proto): - self._proto = proto - return REX_two, () - def __reduce__(self): - raise TestFailed("This __reduce__ shouldn't be called") - -class REX_four(object): - """Calling base class method should succeed""" - _proto = None - def __reduce_ex__(self, proto): - self._proto = proto - return object.__reduce_ex__(self, proto) - -class REX_five(object): - """This one used to fail with infinite recursion""" - _reduce_called = 0 - def __reduce__(self): - self._reduce_called = 1 - return object.__reduce__(self) - -class REX_six(object): - """This class is used to check the 4th argument (list iterator) of - the reduce protocol. - """ - def __init__(self, items=None): - self.items = items if items is not None else [] - def __eq__(self, other): - return type(self) is type(other) and self.items == other.items - def append(self, item): - self.items.append(item) - def __reduce__(self): - return type(self), (), None, iter(self.items), None - -class REX_seven(object): - """This class is used to check the 5th argument (dict iterator) of - the reduce protocol. - """ - def __init__(self, table=None): - self.table = table if table is not None else {} - def __eq__(self, other): - return type(self) is type(other) and self.table == other.table - def __setitem__(self, key, value): - self.table[key] = value - def __reduce__(self): - return type(self), (), None, None, iter(self.table.items()) - -class REX_state(object): - """This class is used to check the 3th argument (state) of - the reduce protocol. - """ - def __init__(self, state=None): - self.state = state - def __eq__(self, other): - return type(self) is type(other) and self.state == other.state - def __setstate__(self, state): - self.state = state - def __reduce__(self): - return type(self), (), self.state - -class REX_None: - """ Setting __reduce_ex__ to None should fail """ - __reduce_ex__ = None - -class R_None: - """ Setting __reduce__ to None should fail """ - __reduce__ = None - -class C_None_setstate: - """ Setting __setstate__ to None should fail """ - def __getstate__(self): - return 1 - - __setstate__ = None - class CustomError(Exception): pass @@ -4347,80 +4545,17 @@ def __reduce__(self): UNPICKLEABLE = Unpickleable() +# For test_unpickleable_reconstructor and test_unpickleable_state_setter class UnpickleableCallable(Unpickleable): def __call__(self, *args, **kwargs): pass - -# Test classes for newobj - -class MyInt(int): - sample = 1 - -class MyFloat(float): - sample = 1.0 - -class MyComplex(complex): - sample = 1.0 + 0.0j - -class MyStr(str): - sample = "hello" - -class MyUnicode(str): - sample = "hello \u1234" - -class MyTuple(tuple): - sample = (1, 2, 3) - -class MyList(list): - sample = [1, 2, 3] - -class MyDict(dict): - sample = {"a": 1, "b": 2} - -class MySet(set): - sample = {"a", "b"} - -class MyFrozenSet(frozenset): - sample = frozenset({"a", "b"}) - -myclasses = [MyInt, MyFloat, - MyComplex, - MyStr, MyUnicode, - MyTuple, MyList, MyDict, MySet, MyFrozenSet] - -class MyIntWithNew(int): - def __new__(cls, value): - raise AssertionError - -class MyIntWithNew2(MyIntWithNew): - __new__ = int.__new__ - - -class SlotList(MyList): - __slots__ = ["foo"] - -# Ruff "redefined while unused" false positive here due to `global` variables -# being assigned (and then restored) from within test methods earlier in the file -class SimpleNewObj(int): # noqa: F811 - def __init__(self, *args, **kwargs): - # raise an error, to make sure this isn't called - raise TypeError("SimpleNewObj.__init__() didn't expect to get called") - def __eq__(self, other): - return int(self) == int(other) and self.__dict__ == other.__dict__ - -class ComplexNewObj(SimpleNewObj): - def __getnewargs__(self): - return ('%X' % self, 16) - -class ComplexNewObjEx(SimpleNewObj): - def __getnewargs_ex__(self): - return ('%X' % self,), {'base': 16} - +# For test_bad_getattr class BadGetattr: def __getattr__(self, key): self.foo +# For test_bad_newobj_class and test_bad_newobj_ex__class class NoNew: def __getattribute__(self, name): if name == '__new__': @@ -4998,7 +5133,7 @@ def test_default_dispatch_table(self): p = self.pickler_class(f, 0) with self.assertRaises(AttributeError): p.dispatch_table - self.assertFalse(hasattr(p, 'dispatch_table')) + self.assertNotHasAttr(p, 'dispatch_table') def test_class_dispatch_table(self): # A dispatch_table attribute can be specified class-wide diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 682815c3fdd6e0e..4b000284526e5f0 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -8,6 +8,12 @@ import warnings +MS_WINDOWS = (sys.platform == "win32") +APPLE = (sys.platform in ("darwin", "ios", "tvos", "watchos")) + +COMMAND_TIMEOUT = 60.0 + + def normalize_text(text): if text is None: return None @@ -16,6 +22,25 @@ def normalize_text(text): return text.strip() +def first_line(text): + # Get the first line. Return text unchanged if it's empty. + lines = text.splitlines() + if lines: + return lines[0] + else: + # text is an empty string + return text + + +def read_first_line(filename): + # Get the first line of a text file and strip trailing spaces + try: + with open(filename, encoding="utf-8") as fp: + return fp.readline().rstrip() + except OSError: + return '' + + class PythonInfo: def __init__(self): self.info = {} @@ -281,9 +306,11 @@ def format_groups(groups): "BUILDPYTHON", "CC", "CFLAGS", + "CI", "COLUMNS", "COMPUTERNAME", "COMSPEC", + "CONTAINER", "CPP", "CPPFLAGS", "DISPLAY", @@ -291,11 +318,14 @@ def format_groups(groups): "DISTUTILS_USE_SDK", "DYLD_LIBRARY_PATH", "ENSUREPIP_OPTIONS", + "FORCE_COLOR", + "GITHUB_ACTIONS", "HISTORY_FILE", "HOME", "HOMEDRIVE", "HOMEPATH", "IDLESTARTUP", + "IMAGE_OS_VERSION", "IPHONEOS_DEPLOYMENT_TARGET", "LANG", "LDFLAGS", @@ -307,11 +337,13 @@ def format_groups(groups): "MAKEFLAGS", "MIXERDEV", "MSSDK", + "NO_COLOR", "PATH", "PATHEXT", "PIP_CONFIG_FILE", "PLAT", "POSIXLY_CORRECT", + "PYTHON_COLORS", "PY_SAX_PARSER", "ProgramFiles", "ProgramFiles(x86)", @@ -418,24 +450,53 @@ def format_attr(attr, value): info_add('readline.library', 'GNU readline') -def collect_gdb(info_add): +def run_command(cmd, check=True, **kwargs): import subprocess + from test.support import has_subprocess_support + + if not has_subprocess_support: + # subprocess is not supported by the current platform + return '' + timeout = COMMAND_TIMEOUT + + cmd_str = ' '.join(cmd) try: - proc = subprocess.Popen(["gdb", "-nx", "--version"], + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True) - version = proc.communicate()[0] - if proc.returncode: - # ignore gdb failure: test_gdb will log the error - return - except OSError: - return + stderr=subprocess.DEVNULL, + text=True, + **kwargs) + with proc: + try: + stdout = proc.communicate(timeout=timeout)[0] + except: + proc.kill() + proc.communicate() + raise + + if check and proc.returncode: + print(f"Command {cmd_str} failed with exit code {proc.returncode}") + return '' - # Only keep the first line - version = version.splitlines()[0] - info_add('gdb_version', version) + # Strip trailing spaces and newlines + stdout = stdout.rstrip() + return stdout + except FileNotFoundError: + return '' + except OSError as exc: + print(f"Command {cmd_str} failed with: {exc!r}") + return '' + except subprocess.TimeoutExpired: + print(f"Command {cmd_str}: timeout!") + return '' + + +def collect_gdb(info_add): + version = run_command(["gdb", "-nx", "--version"]) + if version: + # Only keep the first line + info_add('gdb_version', first_line(version)) def collect_tkinter(info_add): @@ -735,6 +796,10 @@ def collect_test_socket(info_add): if name.startswith('HAVE_')] copy_attributes(info_add, test_socket, 'test_socket.%s', attributes) + # Get IOCTL_VM_SOCKETS_GET_LOCAL_CID of /dev/vsock + cid = test_socket.get_cid() + info_add('test_socket.get_cid', cid) + def collect_support(info_add): try: @@ -752,6 +817,7 @@ def collect_support(info_add): 'is_emscripten', 'is_jython', 'is_wasi', + 'is_wasm32', ) copy_attributes(info_add, support, 'support.%s', attributes) @@ -814,7 +880,6 @@ def collect_support_threading_helper(info_add): def collect_cc(info_add): - import subprocess import sysconfig CC = sysconfig.get_config_var('CC') @@ -827,23 +892,17 @@ def collect_cc(info_add): except ImportError: args = CC.split() args.append('--version') - try: - proc = subprocess.Popen(args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - universal_newlines=True) - except OSError: + + stdout = run_command(args) + if not stdout: # Cannot run the compiler, for example when Python has been # cross-compiled and installed on the target platform where the # compiler is missing. - return - - stdout = proc.communicate()[0] - if proc.returncode: + # # CC --version failed: ignore error return - text = stdout.splitlines()[0] + text = first_line(stdout) text = normalize_text(text) info_add('CC.version', text) @@ -876,8 +935,48 @@ def collect_subprocess(info_add): copy_attributes(info_add, subprocess, 'subprocess.%s', ('_USE_POSIX_SPAWN',)) +def winreg_query(path): + try: + import winreg + except ImportError: + return None + + key, path = path.split('\\', 1) + sub_key, value = path.rsplit('\\', 1) + if key == "HKEY_LOCAL_MACHINE": + key = winreg.HKEY_LOCAL_MACHINE + else: + raise ValueError(f"unknown key {key!r}") + + try: + access = winreg.KEY_READ | winreg.KEY_WOW64_64KEY + with winreg.OpenKey(key, sub_key, access=access) as key_handle: + result, _ = winreg.QueryValueEx(key_handle, value) + return result + except OSError: + return None + + +def wmi_query(query): + try: + import _wmi + except ImportError: + return {} + + try: + data = _wmi.exec_query(query) + except OSError: + return {} + + dict_data = {} + for item in data.split("\0"): + key, _, value = item.partition("=") + dict_data[key] = value + return dict_data + + def collect_windows(info_add): - if sys.platform != "win32": + if not MS_WINDOWS: # Code specific to Windows return @@ -915,66 +1014,27 @@ def collect_windows(info_add): except (ImportError, AttributeError): pass - # windows.version_caption: "wmic os get Caption,Version /value" command - import subprocess - try: - # When wmic.exe output is redirected to a pipe, - # it uses the OEM code page - proc = subprocess.Popen(["wmic", "os", "get", "Caption,Version", "/value"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - encoding="oem", - text=True) - output, stderr = proc.communicate() - if proc.returncode: - output = "" - except OSError: - pass - else: - for line in output.splitlines(): - line = line.strip() - if line.startswith('Caption='): - line = line.removeprefix('Caption=').strip() - if line: - info_add('windows.version_caption', line) - elif line.startswith('Version='): - line = line.removeprefix('Version=').strip() - if line: - info_add('windows.version', line) + # Get operating system caption and version using WMI + data = wmi_query("SELECT Caption, Version FROM Win32_OperatingSystem") + caption = data.get('Caption', '') + if caption: + info_add('windows.version_caption', caption) + version = data.get('Version', '') + if version: + info_add('windows.version', version) # windows.ver: "ver" command - try: - proc = subprocess.Popen(["ver"], shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True) - output = proc.communicate()[0] - if proc.returncode == 0xc0000142: - return - if proc.returncode: - output = "" - except OSError: - return - else: - output = output.strip() - line = output.splitlines()[0] - if line: - info_add('windows.ver', line) + output = run_command(["ver"], shell=True) + # "ver" output starts with an empty line: remove it + output = output.strip() + if output: + info_add('windows.ver', first_line(output)) # windows.developer_mode: get AllowDevelopmentWithoutDevLicense registry - import winreg - try: - key = winreg.OpenKey( - winreg.HKEY_LOCAL_MACHINE, - r"SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock") - subkey = "AllowDevelopmentWithoutDevLicense" - try: - value, value_type = winreg.QueryValueEx(key, subkey) - finally: - winreg.CloseKey(key) - except OSError: - pass - else: + value = winreg_query(r"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows" + r"\CurrentVersion\AppModelUnlock" + r"\AllowDevelopmentWithoutDevLicense") + if value is not None: info_add('windows.developer_mode', "enabled" if value else "disabled") @@ -987,14 +1047,9 @@ def collect_fips(info_add): if _hashlib is not None: call_func(info_add, 'fips.openssl_fips_mode', _hashlib, 'get_fips_mode') - try: - with open("/proc/sys/crypto/fips_enabled", encoding="utf-8") as fp: - line = fp.readline().rstrip() - - if line: - info_add('fips.linux_crypto_fips_enabled', line) - except OSError: - pass + fips_enabled = read_first_line("/proc/sys/crypto/fips_enabled") + if fips_enabled: + info_add('fips.linux_crypto_fips_enabled', fips_enabled) def collect_tempfile(info_add): @@ -1012,6 +1067,236 @@ def collect_libregrtest_utils(info_add): info_add('libregrtests.build_info', ' '.join(utils.get_build_info())) +def uptime_boottime(): + # Use CLOCK_BOOTTIME + import time + try: + return time.clock_gettime(time.CLOCK_BOOTTIME) + except (AttributeError, OSError): + return None + + +def uptime_linux(): + # Parse the first member of /proc/uptime + line = read_first_line("/proc/uptime") + if not line: + return + try: + parts = line.split() + if not parts: + return + return float(parts[0]) + except ValueError: + return + + +def uptime_bsd(): + # Get sysctlbyname("kern.boottime") + try: + import _testcapi + except ImportError: + return None + try: + return _testcapi.uptime_bsd() + except (AttributeError, OSError): + return None + + +def uptime_windows(): + try: + import _winapi + except ImportError: + return None + else: + return _winapi.GetTickCount64() / 1000. + + +def get_uptime(): + for func in (uptime_boottime, uptime_linux, uptime_bsd, uptime_windows): + uptime = func() + if uptime is not None: + return uptime + return None + + +def get_machine_id(): + if MS_WINDOWS: + machine_guid = winreg_query(r"HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft" + r"\Cryptography\MachineGuid") + if machine_guid: + return machine_guid + + for filename in ( + # https://www.freedesktop.org/software/systemd/man/latest/machine-id.html + "/etc/machine-id", + # BSD + "/etc/hostid", + ): + machine_id = read_first_line(filename) + if machine_id: + return machine_id + + return None + + +def detect_virt_windows(info_add): + # On Windows, use WMI to detect the virtualization. + # + # Microsoft Hyper-V: + # - Win32_Bios.Version = 'VRTUAL - 12001807' + # - Win32_Bios.Manufacturer = 'American Megatrends Inc.' + # - Win32_ComputerSystem.Model = 'Virtual Machine' + # - Win32_ComputerSystem.Manufacturer = 'Microsoft Corporation' + # + # VMware: + # - Win32_ComputerSystem.Model = 'VMware' + # - Win32_ComputerSystem.Manufacturer = 'VMWare' (uppercase W in Ware) + # - Win32_Bios.SerialNumber starts with 'VMware-' + # + # QEMU: + # - Win32_ComputerSystem.Manufacturer = 'QEMU' + # - Win32_ComputerSystem.Model = 'Standard PC (Q35 + ICH9, 2009)' + # - Win32_Bios.Version = 'BOCHS - 1' + # - Win32_Bios.Manufacturer = 'EDK II' + # + # Parallels: + # - Win32_Bios.Version = 'PARALLELS' + # + # VirtualBox: + # - Win32_Bios.Version = 'VBOX' + # - Win32_ComputerSystem.Model = 'VirtualBox' + # - Win32_ComputerSystem.Manufacturer = 'innotek GmbH' + # + # Amazon EC2: + # - Win32_Bios.Version = 'AMAZON - 1' + # - Win32_Bios.Manufacturer = 'Amazon EC2' + # - Win32_ComputerSystem.Model = 'm7i.4xlarge' + # - Win32_ComputerSystem.Manufacturer = 'Amazon EC2' + + KNOWN_VIRT = ( + 'Amazon EC2', + 'QEMU', + 'VMware', + 'VirtualBox', + 'Xen', + 'oVirt', + ) + KNOWN_BIOS_VERSIONS = { + 'PARALLELS': 'Parallels', + 'VBOX': 'VirtualBox', + } + + computer = wmi_query('SELECT Model, Manufacturer FROM Win32_ComputerSystem') + computer_model = computer.get('Model', '') + computer_manufacturer = computer.get('Manufacturer', '') + if computer_manufacturer == 'Amazon EC2': + # Log the VM model (ex: 'm7i.4xlarge') + info_add('system.computer.model', computer_model) + return computer_manufacturer + if computer_model in KNOWN_VIRT: + return computer_model + if computer_manufacturer in KNOWN_VIRT: + return computer_manufacturer + + bios = wmi_query('SELECT Version, Manufacturer FROM Win32_Bios') + + bios_version = bios.get('Version', '') + if bios_version in KNOWN_VIRT: + return bios_version + if (bios_version.startswith('VRTUAL - ') + and computer_manufacturer == 'Microsoft Corporation'): + return 'Microsoft Hyper-V' + try: + return KNOWN_BIOS_VERSIONS[bios_version] + except KeyError: + pass + + bios_manufacturer = bios.get('Manufacturer', '') + if bios_manufacturer in KNOWN_VIRT: + return bios_manufacturer + + # Log the values to update the code if a new VM is discovered + if computer_model: + info_add('system.computer.model', computer_model) + if computer_manufacturer: + info_add('system.computer.manufacturer', computer_manufacturer) + if bios_version: + info_add('system.bios.version', bios_version) + if bios_manufacturer: + info_add('system.bios.manufacturer', bios_manufacturer) + + +def detect_virt(info_add): + if MS_WINDOWS: + return detect_virt_windows(info_add) + + # Run systemd-detect-virt command + virt = run_command(["systemd-detect-virt"], check=False) + if virt and virt != "none": + return virt + + # Check if the process in running in a container + import os.path + if os.path.exists('/.dockerenv'): + return 'docker' + if os.path.exists('/run/.containerenv'): + return 'podman' + + container = read_first_line('/run/systemd/container') + if container: + return container + + if APPLE: + hv_vmm_present = run_command(['sysctl', '-n', 'kern.hv_vmm_present']) + if hv_vmm_present == '1': + return 'run in a VM (kern.hv_vmm_present is 1)' + + # Other ways to check if running in a container: + # * Parse /proc/1/mounts or /proc/1/mountinfo (check "/" filesystem). + # * Parse /proc/1/cgroup. + # * Parse the first line of /proc/1/sched (check process name is different + # than "init" and "systemd"). + # * Check / inode. + # * On systems using SELinux (Fedora/CentOS/RHEL), check for "container_t" + # label, for example of /proc/1/attr/current. + # * Check for "container" variable in /proc/1/environ + # (only root can read this file). + # * Check for "container" environment variable. + # * Set a specific env var when creating the container image. + # * Run virt-what, need to install the script, and must be run as root. + # * Check for "GITHUB_ACTIONS" environmant variable (GitHub Action). + + +def collect_system(info_add): + boot_id = read_first_line("/proc/sys/kernel/random/boot_id") + if boot_id: + info_add('system.boot_id', boot_id) + + machine_id = get_machine_id() + if machine_id: + info_add('system.machine_id', machine_id) + + uptime = get_uptime() + if uptime is not None: + # truncate microseconds + uptime = int(uptime) + try: + import datetime + uptime = str(datetime.timedelta(seconds=uptime)) + except ImportError: + uptime = f'{uptime} sec' + info_add('system.uptime', uptime) + + virt = detect_virt(info_add) + if virt: + info_add('system.virt', virt) + + if APPLE: + hardware = run_command(['sysctl', '-n', 'hw.model']) + if hardware: + info_add('system.hardware', hardware) + + def collect_info(info): error = False info_add = info.add @@ -1052,6 +1337,7 @@ def collect_info(info): collect_windows, collect_zlib, collect_libregrtest_utils, + collect_system, # Collecting from tests should be last as they have side effects. collect_test_socket, diff --git a/Lib/test/seq_tests.py b/Lib/test/seq_tests.py index a41970d8f3f55ac..b7875fe8f2f75df 100644 --- a/Lib/test/seq_tests.py +++ b/Lib/test/seq_tests.py @@ -261,23 +261,20 @@ def test_minmax(self): self.assertEqual(min(u), 0) self.assertEqual(max(u), 2) - def test_addmul(self): + def test_add(self): u1 = self.type2test([0]) u2 = self.type2test([0, 1]) self.assertEqual(u1, u1 + self.type2test()) self.assertEqual(u1, self.type2test() + u1) self.assertEqual(u1 + self.type2test([1]), u2) self.assertEqual(self.type2test([-1]) + u1, self.type2test([-1, 0])) - self.assertEqual(self.type2test(), u2*0) - self.assertEqual(self.type2test(), 0*u2) + + def test_mul(self): + u2 = self.type2test([0, 1]) self.assertEqual(self.type2test(), u2*0) self.assertEqual(self.type2test(), 0*u2) self.assertEqual(u2, u2*1) self.assertEqual(u2, 1*u2) - self.assertEqual(u2, u2*1) - self.assertEqual(u2, 1*u2) - self.assertEqual(u2+u2, u2*2) - self.assertEqual(u2+u2, 2*u2) self.assertEqual(u2+u2, u2*2) self.assertEqual(u2+u2, 2*u2) self.assertEqual(u2+u2+u2, u2*3) @@ -286,8 +283,9 @@ def test_addmul(self): class subclass(self.type2test): pass u3 = subclass([0, 1]) - self.assertEqual(u3, u3*1) - self.assertIsNot(u3, u3*1) + r = u3*1 + self.assertEqual(r, u3) + self.assertIsNot(r, u3) def test_iadd(self): u = self.type2test([0, 1]) @@ -348,6 +346,21 @@ def test_subscript(self): self.assertRaises(ValueError, a.__getitem__, slice(0, 10, 0)) self.assertRaises(TypeError, a.__getitem__, 'x') + def _assert_cmp(self, a, b, r): + self.assertIs(a == b, r == 0) + self.assertIs(a != b, r != 0) + self.assertIs(a > b, r > 0) + self.assertIs(a <= b, r <= 0) + self.assertIs(a < b, r < 0) + self.assertIs(a >= b, r >= 0) + + def test_cmp(self): + a = self.type2test([0, 1]) + self._assert_cmp(a, a, 0) + self._assert_cmp(a, self.type2test([0, 1]), 0) + self._assert_cmp(a, self.type2test([0]), 1) + self._assert_cmp(a, self.type2test([0, 2]), -1) + def test_count(self): a = self.type2test([0, 1, 2])*3 self.assertEqual(a.count(0), 3) diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index 4b82d51b4508ac6..f66051531faaa16 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -90,6 +90,55 @@ def checkcall(self, obj, methodname, *args): args = self.fixtype(args) getattr(obj, methodname)(*args) + def _get_teststrings(self, charset, digits): + base = len(charset) + teststrings = set() + for i in range(base ** digits): + entry = [] + for j in range(digits): + i, m = divmod(i, base) + entry.append(charset[m]) + teststrings.add(''.join(entry)) + teststrings = [self.fixtype(ts) for ts in teststrings] + return teststrings + + def test_add(self): + s = self.fixtype('ab') + self.assertEqual(s + self.fixtype(''), s) + self.assertEqual(self.fixtype('') + s, s) + self.assertEqual(s + self.fixtype('cd'), self.fixtype('abcd')) + + def test_mul(self): + s = self.fixtype('ab') + self.assertEqual(s*0, self.fixtype('')) + self.assertEqual(0*s, self.fixtype('')) + self.assertEqual(s*1, s) + self.assertEqual(1*s, s) + self.assertEqual(s*2, self.fixtype('abab')) + self.assertEqual(2*s, self.fixtype('abab')) + + class subclass(self.type2test): + pass + s = subclass(self.fixtype('ab')) + r = s*1 + self.assertEqual(r, s) + self.assertIsNot(r, s) + + def _assert_cmp(self, a, b, r): + self.assertIs(a == b, r == 0) + self.assertIs(a != b, r != 0) + self.assertIs(a > b, r > 0) + self.assertIs(a <= b, r <= 0) + self.assertIs(a < b, r < 0) + self.assertIs(a >= b, r >= 0) + + def test_cmp(self): + a = self.fixtype('ab') + self._assert_cmp(a, a, 0) + self._assert_cmp(a, self.fixtype('ab'), 0) + self._assert_cmp(a, self.fixtype('a'), 1) + self._assert_cmp(a, self.fixtype('ac'), -1) + def test_count(self): self.checkequal(3, 'aaa', 'count', 'a') self.checkequal(0, 'aaa', 'count', 'b') @@ -130,17 +179,7 @@ def test_count(self): # For a variety of combinations, # verify that str.count() matches an equivalent function # replacing all occurrences and then differencing the string lengths - charset = ['', 'a', 'b'] - digits = 7 - base = len(charset) - teststrings = set() - for i in range(base ** digits): - entry = [] - for j in range(digits): - i, m = divmod(i, base) - entry.append(charset[m]) - teststrings.add(''.join(entry)) - teststrings = [self.fixtype(ts) for ts in teststrings] + teststrings = self._get_teststrings(['', 'a', 'b'], 7) for i in teststrings: n = len(i) for j in teststrings: @@ -197,17 +236,7 @@ def test_find(self): # For a variety of combinations, # verify that str.find() matches __contains__ # and that the found substring is really at that location - charset = ['', 'a', 'b', 'c'] - digits = 5 - base = len(charset) - teststrings = set() - for i in range(base ** digits): - entry = [] - for j in range(digits): - i, m = divmod(i, base) - entry.append(charset[m]) - teststrings.add(''.join(entry)) - teststrings = [self.fixtype(ts) for ts in teststrings] + teststrings = self._get_teststrings(['', 'a', 'b', 'c'], 5) for i in teststrings: for j in teststrings: loc = i.find(j) @@ -244,17 +273,7 @@ def test_rfind(self): # For a variety of combinations, # verify that str.rfind() matches __contains__ # and that the found substring is really at that location - charset = ['', 'a', 'b', 'c'] - digits = 5 - base = len(charset) - teststrings = set() - for i in range(base ** digits): - entry = [] - for j in range(digits): - i, m = divmod(i, base) - entry.append(charset[m]) - teststrings.add(''.join(entry)) - teststrings = [self.fixtype(ts) for ts in teststrings] + teststrings = self._get_teststrings(['', 'a', 'b', 'c'], 5) for i in teststrings: for j in teststrings: loc = i.rfind(j) @@ -295,6 +314,19 @@ def test_index(self): else: self.checkraises(TypeError, 'hello', 'index', 42) + # For a variety of combinations, + # verify that str.index() matches __contains__ + # and that the found substring is really at that location + teststrings = self._get_teststrings(['', 'a', 'b', 'c'], 5) + for i in teststrings: + for j in teststrings: + if j in i: + loc = i.index(j) + self.assertGreaterEqual(loc, 0) + self.assertEqual(i[loc:loc+len(j)], j) + else: + self.assertRaises(ValueError, i.index, j) + def test_rindex(self): self.checkequal(12, 'abcdefghiabc', 'rindex', '') self.checkequal(3, 'abcdefghiabc', 'rindex', 'def') @@ -321,6 +353,19 @@ def test_rindex(self): else: self.checkraises(TypeError, 'hello', 'rindex', 42) + # For a variety of combinations, + # verify that str.rindex() matches __contains__ + # and that the found substring is really at that location + teststrings = self._get_teststrings(['', 'a', 'b', 'c'], 5) + for i in teststrings: + for j in teststrings: + if j in i: + loc = i.rindex(j) + self.assertGreaterEqual(loc, 0) + self.assertEqual(i[loc:loc+len(j)], j) + else: + self.assertRaises(ValueError, i.rindex, j) + def test_find_periodic_pattern(self): """Cover the special path for periodic patterns.""" def reference_find(p, s): @@ -767,6 +812,15 @@ def test_replace(self): self.checkraises(TypeError, 'hello', 'replace', 42, 'h') self.checkraises(TypeError, 'hello', 'replace', 'h', 42) + def test_replacement_on_buffer_boundary(self): + # gh-127971: Check we don't read past the end of the buffer when a + # potential match misses on the last character. + any_3_nonblank_codepoints = '!!!' + seven_codepoints = any_3_nonblank_codepoints + ' ' + any_3_nonblank_codepoints + a = (' ' * 243) + seven_codepoints + (' ' * 7) + b = ' ' * 6 + chr(256) + a.replace(seven_codepoints, b) + def test_replace_uses_two_way_maxcount(self): # Test that maxcount works in _two_way_count in fastsearch.h A, B = "A"*1000, "B"*1000 @@ -1287,6 +1341,7 @@ def test_extended_getslice(self): slice(start, stop, step)) def test_mul(self): + super().test_mul() self.checkequal('', 'abc', '__mul__', -1) self.checkequal('', 'abc', '__mul__', 0) self.checkequal('abc', 'abc', '__mul__', 1) diff --git a/Lib/test/subprocessdata/fd_status.py b/Lib/test/subprocessdata/fd_status.py index d12bd95abee61ce..90e785981aeab0f 100644 --- a/Lib/test/subprocessdata/fd_status.py +++ b/Lib/test/subprocessdata/fd_status.py @@ -2,7 +2,7 @@ file descriptors on stdout. Usage: -fd_stats.py: check all file descriptors +fd_status.py: check all file descriptors (up to 255) fd_status.py fd1 fd2 ...: check only specified file descriptors """ @@ -18,7 +18,7 @@ _MAXFD = os.sysconf("SC_OPEN_MAX") except: _MAXFD = 256 - test_fds = range(0, _MAXFD) + test_fds = range(0, min(_MAXFD, 256)) else: test_fds = map(int, sys.argv[1:]) for fd in test_fds: diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index c74c3a3190947ba..005ab19a93d941c 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -30,11 +30,12 @@ "record_original_stdout", "get_original_stdout", "captured_stdout", "captured_stdin", "captured_stderr", "captured_output", # unittest - "is_resource_enabled", "requires", "requires_freebsd_version", + "is_resource_enabled", "get_resource_value", "requires", "requires_resource", + "requires_freebsd_version", "requires_gil_enabled", "requires_linux_version", "requires_mac_ver", "check_syntax_error", "requires_gzip", "requires_bz2", "requires_lzma", "requires_zstd", - "bigmemtest", "bigaddrspacetest", "cpython_only", "get_attribute", + "bigmemtest", "nomemtest", "bigaddrspacetest", "cpython_only", "get_attribute", "requires_IEEE_754", "requires_zlib", "has_fork_support", "requires_fork", "has_subprocess_support", "requires_subprocess", @@ -43,9 +44,11 @@ "check__all__", "skip_if_buggy_ucrt_strfptime", "check_disallow_instantiation", "check_sanitizer", "skip_if_sanitizer", "requires_limited_api", "requires_specialization", "thread_unsafe", + "skip_if_unlimited_stack_size", "skip_if_huge_c_stack", # sys "MS_WINDOWS", "is_jython", "is_android", "is_emscripten", "is_wasi", "is_apple_mobile", "check_impl_detail", "unix_shell", "setswitchinterval", + "support_remote_exec_only", # os "get_pagesize", # network @@ -67,7 +70,7 @@ "BrokenIter", "in_systemd_nspawn_sync_suppressed", "run_no_yield_async_fn", "run_yielding_async_fn", "async_yield", - "reset_code", + "reset_code", "on_github_actions" ] @@ -183,7 +186,7 @@ def get_attribute(obj, name): return attribute verbose = 1 # Flag set to 0 by regrtest.py -use_resources = None # Flag set to [] by regrtest.py +use_resources = None # Flag set to {} by regrtest.py max_memuse = 0 # Disable bigmem tests (they will still be run with # small sizes, to make sure they work.) real_max_memuse = 0 @@ -298,6 +301,16 @@ def is_resource_enabled(resource): """ return use_resources is None or resource in use_resources +def get_resource_value(resource): + """Test whether a resource is enabled. + + Known resources are set by regrtest.py. If not running under regrtest.py, + all resources are assumed enabled unless use_resources has been set. + """ + if use_resources is None: + return None + return use_resources.get(resource) + def requires(resource, msg=None): """Raise ResourceDenied if the specified resource is not available.""" if not is_resource_enabled(resource): @@ -309,6 +322,16 @@ def requires(resource, msg=None): if resource == 'gui' and not _is_gui_available(): raise ResourceDenied(_is_gui_available.reason) +def _get_kernel_version(sysname="Linux"): + import platform + if platform.system() != sysname: + return None + version_txt = platform.release().split('-', 1)[0] + try: + return tuple(map(int, version_txt.split('.'))) + except ValueError: + return None + def _requires_unix_version(sysname, min_version): """Decorator raising SkipTest if the OS is `sysname` and the version is less than `min_version`. @@ -540,10 +563,14 @@ def has_no_debug_ranges(): except ImportError: raise unittest.SkipTest("_testinternalcapi required") return not _testcapi.config_get('code_debug_ranges') - return not bool(config['code_debug_ranges']) def requires_debug_ranges(reason='requires co_positions / debug_ranges'): - return unittest.skipIf(has_no_debug_ranges(), reason) + try: + skip = has_no_debug_ranges() + except unittest.SkipTest as e: + skip = True + reason = e.args[0] if e.args else reason + return unittest.skipIf(skip, reason) MS_WINDOWS = (sys.platform == 'win32') @@ -568,8 +595,11 @@ def skip_android_selinux(name): is_emscripten = sys.platform == "emscripten" is_wasi = sys.platform == "wasi" +# Use is_wasm32 as a generic check for WebAssembly platforms. +is_wasm32 = is_emscripten or is_wasi + def skip_emscripten_stack_overflow(): - return unittest.skipIf(is_emscripten, "Exhausts limited stack on Emscripten") + return unittest.skipIf(is_emscripten, "Exhausts stack on Emscripten") def skip_wasi_stack_overflow(): return unittest.skipIf(is_wasi, "Exhausts stack on WASI") @@ -696,9 +726,11 @@ def sortdict(dict): return "{%s}" % withcommas -def run_code(code: str) -> dict[str, object]: +def run_code(code: str, extra_names: dict[str, object] | None = None) -> dict[str, object]: """Run a piece of code after dedenting it, and return its global namespace.""" ns = {} + if extra_names: + ns.update(extra_names) exec(textwrap.dedent(code), ns) return ns @@ -943,6 +975,31 @@ def check_sizeof(test, o, size): % (type(o), result, size) test.assertEqual(result, size, msg) +def subTests(arg_names, arg_values, /, *, _do_cleanups=False): + """Run multiple subtests with different parameters. + """ + single_param = False + if isinstance(arg_names, str): + arg_names = arg_names.replace(',',' ').split() + if len(arg_names) == 1: + single_param = True + arg_values = tuple(arg_values) + def decorator(func): + if isinstance(func, type): + raise TypeError('subTests() can only decorate methods, not classes') + @functools.wraps(func) + def wrapper(self, /, *args, **kwargs): + for values in arg_values: + if single_param: + values = (values,) + subtest_kwargs = dict(zip(arg_names, values)) + with self.subTest(**subtest_kwargs): + func(self, *args, **kwargs, **subtest_kwargs) + if _do_cleanups: + self.doCleanups() + return wrapper + return decorator + #======================================================================= # Decorator/context manager for running a code in a different locale, # correctly resetting it afterwards. @@ -1082,7 +1139,7 @@ def set_memlimit(limit: str) -> None: global real_max_memuse memlimit = _parse_memlimit(limit) if memlimit < _2G - 1: - raise ValueError('Memory limit {limit!r} too low to be useful') + raise ValueError(f'Memory limit {limit!r} too low to be useful') real_max_memuse = memlimit memlimit = min(memlimit, MAX_Py_ssize_t) @@ -1169,6 +1226,22 @@ def wrapper(self): return wrapper return decorator +def nomemtest(f): + """Check that we can use this test with `_testcapi.set_nomemory`.""" + from .import_helper import import_module + + @functools.wraps(f) + def internal(*args, **kwargs): + import_module('_testcapi') + return f(*args, **kwargs) + + return unittest.skipIf( + # Python built with Py_TRACE_REFS fail with a fatal error in + # _PyRefchain_Trace() on memory allocation error. + Py_TRACE_REFS, + 'cannot test Py_TRACE_REFS build', + )(cpython_only(internal)) + def bigaddrspacetest(f): """Decorator for tests that fill the address space.""" def wrapper(self): @@ -1325,6 +1398,7 @@ def reset_code(f: types.FunctionType) -> types.FunctionType: f.__code__ = f.__code__.replace() return f +on_github_actions = "GITHUB_ACTIONS" in os.environ #======================================================================= # Check for the presence of docstrings. @@ -1627,6 +1701,25 @@ def skip_if_pgo_task(test): return test if ok else unittest.skip(msg)(test) +def skip_if_unlimited_stack_size(test): + """Skip decorator for tests not run when an unlimited stack size is configured. + + Tests using support.infinite_recursion([...]) may otherwise run into + an infinite loop, running until the memory on the system is filled and + crashing due to OOM. + + See https://github.com/python/cpython/issues/143460. + """ + if is_emscripten or is_wasi or os.name == "nt": + return test + + import resource + curlim, maxlim = resource.getrlimit(resource.RLIMIT_STACK) + unlimited_stack_size_cond = curlim == maxlim and curlim in (-1, 0xFFFF_FFFF_FFFF_FFFF) + reason = "Not run due to unlimited stack size" + return unittest.skipIf(unlimited_stack_size_cond, reason)(test) + + def detect_api_mismatch(ref_api, other_api, *, ignore=()): """Returns the set of items in ref_api not in other_api, except for a defined list of items to be ignored in this check. @@ -1651,7 +1744,7 @@ def check__all__(test_case, module, name_of_module=None, extra=(), 'module'. The 'name_of_module' argument can specify (as a string or tuple thereof) - what module(s) an API could be defined in in order to be detected as a + what module(s) an API could be defined in order to be detected as a public API. One case for this is when 'module' imports part of its public API from other modules, possibly a C backend (like 'csv' and its '_csv'). @@ -2306,6 +2399,7 @@ def check_disallow_instantiation(testcase, tp, *args, **kwds): qualname = f"{name}" msg = f"cannot create '{re.escape(qualname)}' instances" testcase.assertRaisesRegex(TypeError, msg, tp, *args, **kwds) + testcase.assertRaisesRegex(TypeError, msg, tp.__new__, tp, *args, **kwds) def get_recursion_depth(): """Get the recursion depth of the caller function. @@ -2357,7 +2451,7 @@ def infinite_recursion(max_depth=None): # very deep recursion. max_depth = 20_000 elif max_depth < 3: - raise ValueError("max_depth must be at least 3, got {max_depth}") + raise ValueError(f"max_depth must be at least 3, got {max_depth}") depth = get_recursion_depth() depth = max(depth - 1, 1) # Ignore infinite_recursion() frame. limit = depth + max_depth @@ -2649,6 +2743,32 @@ def exceeds_recursion_limit(): return 150_000 +def skip_if_huge_c_stack(depth=150_000): + """Skip decorator for tests which cannot overflow the C stack. + + Tests exhausting the C stack with *depth* recursive calls cannot + trigger the recursion protection if the C stack is too large (e.g. + with a large or unlimited RLIMIT_STACK), and either fail, or run + for a very long time, or crash, or consume all memory. + """ + try: + from _testinternalcapi import get_c_recursion_remaining + except ImportError: + # Fall back to checking for an unlimited stack size. + huge = False + if not (is_emscripten or is_wasi) and os.name != "nt": + import resource + soft, hard = resource.getrlimit(resource.RLIMIT_STACK) + huge = soft == hard and soft in (-1, 0xFFFF_FFFF_FFFF_FFFF) + else: + remaining = get_c_recursion_remaining() + # A negative value means integer overflow in the estimate + # (e.g. with an unlimited RLIMIT_STACK). + huge = remaining >= depth or remaining < 0 + return unittest.skipIf( + huge, f"the C stack is large enough for {depth} recursive calls") + + # Windows doesn't have os.uname() but it doesn't support s390x. is_s390x = hasattr(os, 'uname') and os.uname().machine == 's390x' skip_on_s390x = unittest.skipIf(is_s390x, 'skipped on s390x') @@ -2863,7 +2983,7 @@ def force_color(color: bool): from .os_helper import EnvironmentVarGuard with ( - swap_attr(_colorize, "can_colorize", lambda file=None: color), + swap_attr(_colorize, "can_colorize", lambda *, file=None: color), EnvironmentVarGuard() as env, ): env.unset("FORCE_COLOR", "NO_COLOR", "PYTHON_COLORS") @@ -2927,12 +3047,6 @@ def make_clean_env() -> dict[str, str]: return clean_env -def initialized_with_pyrepl(): - """Detect whether PyREPL was used during Python initialization.""" - # If the main module has a __file__ attribute it's a Python module, which means PyREPL. - return hasattr(sys.modules["__main__"], "__file__") - - WINDOWS_STATUS = { 0xC0000005: "STATUS_ACCESS_VIOLATION", 0xC00000FD: "STATUS_STACK_OVERFLOW", @@ -2962,6 +3076,10 @@ def get_signal_name(exitcode): except KeyError: pass + # Format Windows exit status as hexadecimal + if 0xC0000000 <= exitcode: + return f"0x{exitcode:X}" + return None class BrokenIter: @@ -2997,7 +3115,7 @@ def in_systemd_nspawn_sync_suppressed() -> bool: with open("/run/systemd/container", "rb") as fp: if fp.read().rstrip() != b"systemd-nspawn": return False - except FileNotFoundError: + except (FileNotFoundError, PermissionError): return False # If systemd-nspawn is used, O_SYNC flag will immediately @@ -3049,6 +3167,27 @@ def is_libssl_fips_mode(): return False # more of a maybe, unless we add this to the _ssl module. return get_fips_mode() != 0 +def _supports_remote_attaching(): + PROCESS_VM_READV_SUPPORTED = False + + try: + from _remote_debugging import PROCESS_VM_READV_SUPPORTED + except ImportError: + pass + + return PROCESS_VM_READV_SUPPORTED + +def _support_remote_exec_only_impl(): + if not sys.is_remote_debug_enabled(): + return unittest.skip("Remote debugging is not enabled") + if sys.platform not in ("darwin", "linux", "win32"): + return unittest.skip("Test only runs on Linux, Windows and macOS") + if sys.platform == "linux" and not _supports_remote_attaching(): + return unittest.skip("Test only runs on Linux with process_vm_readv support") + return _id + +def support_remote_exec_only(test): + return _support_remote_exec_only_impl()(test) class EqualToForwardRef: """Helper to ease use of annotationlib.ForwardRef in tests. @@ -3105,7 +3244,7 @@ def linked_to_musl(): # emscripten (at least as far as we're concerned) and wasi use musl, # but platform doesn't know how to get the version, so set it to zero. - if is_emscripten or is_wasi: + if is_wasm32: _linked_to_musl = (0, 0, 0) return _linked_to_musl @@ -3123,3 +3262,25 @@ def linked_to_musl(): return _linked_to_musl _linked_to_musl = tuple(map(int, version.split('.'))) return _linked_to_musl + + +def control_characters_c0() -> list[str]: + """Returns a list of C0 control characters as strings. + C0 control characters defined as the byte range 0x00-0x1F, and 0x7F. + """ + return [chr(c) for c in range(0x00, 0x20)] + ["\x7F"] + + +STATUS_DLL_INIT_FAILED = 0xC0000142 +def skip_on_low_desktop_heap_memory_subprocess(returncode): + if sys.platform not in ('win32', 'cygwin'): + return + # On Windows, STATUS_DLL_INIT_FAILED is a generic error code that could + # come from any of the DLLs being loaded when a new Python process is + # created. In practice, it's likely a memory allocation failure in the + # desktop heap memory which caused the DLL init failure, especially on + # process created with CREATE_NEW_CONSOLE creation flag. See the article: + # https://learn.microsoft.com/en-us/troubleshoot/windows-server/performance/desktop-heap-limitation-out-of-memory + if returncode == STATUS_DLL_INIT_FAILED: + raise unittest.SkipTest('gh-150436: DLL init failed, likely because ' + 'of low desktop heap memory') diff --git a/Lib/test/support/interpreters/channels.py b/Lib/test/support/channels.py similarity index 74% rename from Lib/test/support/interpreters/channels.py rename to Lib/test/support/channels.py index d2bd93d77f71699..5352f7d4da3af0f 100644 --- a/Lib/test/support/interpreters/channels.py +++ b/Lib/test/support/channels.py @@ -2,14 +2,14 @@ import time import _interpchannels as _channels -from . import _crossinterp +from concurrent.interpreters import _crossinterp # aliases: from _interpchannels import ( ChannelError, ChannelNotFoundError, ChannelClosedError, ChannelEmptyError, ChannelNotEmptyError, ) -from ._crossinterp import ( +from concurrent.interpreters._crossinterp import ( UNBOUND_ERROR, UNBOUND_REMOVE, ) @@ -55,15 +55,23 @@ def create(*, unbounditems=UNBOUND): """ unbound = _serialize_unbound(unbounditems) unboundop, = unbound - cid = _channels.create(unboundop) - recv, send = RecvChannel(cid), SendChannel(cid, _unbound=unbound) + cid = _channels.create(unboundop, -1) + recv, send = RecvChannel(cid), SendChannel(cid) + send._set_unbound(unboundop, unbounditems) return recv, send def list_all(): """Return a list of (recv, send) for all open channels.""" - return [(RecvChannel(cid), SendChannel(cid, _unbound=unbound)) - for cid, unbound in _channels.list_all()] + channels = [] + for cid, unboundop, _ in _channels.list_all(): + chan = _, send = RecvChannel(cid), SendChannel(cid) + if not hasattr(send, '_unboundop'): + send._set_unbound(unboundop) + else: + assert send._unbound[0] == unboundop + channels.append(chan) + return channels class _ChannelEnd: @@ -97,12 +105,8 @@ def __eq__(self, other): return other._id == self._id # for pickling: - def __getnewargs__(self): - return (int(self._id),) - - # for pickling: - def __getstate__(self): - return None + def __reduce__(self): + return (type(self), (int(self._id),)) @property def id(self): @@ -175,16 +179,33 @@ class SendChannel(_ChannelEnd): _end = 'send' - def __new__(cls, cid, *, _unbound=None): - if _unbound is None: - try: - op = _channels.get_channel_defaults(cid) - _unbound = (op,) - except ChannelNotFoundError: - _unbound = _serialize_unbound(UNBOUND) - self = super().__new__(cls, cid) - self._unbound = _unbound - return self +# def __new__(cls, cid, *, _unbound=None): +# if _unbound is None: +# try: +# op = _channels.get_channel_defaults(cid) +# _unbound = (op,) +# except ChannelNotFoundError: +# _unbound = _serialize_unbound(UNBOUND) +# self = super().__new__(cls, cid) +# self._unbound = _unbound +# return self + + def _set_unbound(self, op, items=None): + assert not hasattr(self, '_unbound') + if items is None: + items = _resolve_unbound(op) + unbound = (op, items) + self._unbound = unbound + return unbound + + @property + def unbounditems(self): + try: + _, items = self._unbound + except AttributeError: + op, _ = _channels.get_queue_defaults(self._id) + _, items = self._set_unbound(op) + return items @property def is_closed(self): @@ -192,61 +213,61 @@ def is_closed(self): return info.closed or info.closing def send(self, obj, timeout=None, *, - unbound=None, + unbounditems=None, ): """Send the object (i.e. its data) to the channel's receiving end. This blocks until the object is received. """ - if unbound is None: - unboundop, = self._unbound + if unbounditems is None: + unboundop = -1 else: - unboundop, = _serialize_unbound(unbound) + unboundop, = _serialize_unbound(unbounditems) _channels.send(self._id, obj, unboundop, timeout=timeout, blocking=True) def send_nowait(self, obj, *, - unbound=None, + unbounditems=None, ): """Send the object to the channel's receiving end. If the object is immediately received then return True (else False). Otherwise this is the same as send(). """ - if unbound is None: - unboundop, = self._unbound + if unbounditems is None: + unboundop = -1 else: - unboundop, = _serialize_unbound(unbound) + unboundop, = _serialize_unbound(unbounditems) # XXX Note that at the moment channel_send() only ever returns # None. This should be fixed when channel_send_wait() is added. # See bpo-32604 and gh-19829. return _channels.send(self._id, obj, unboundop, blocking=False) def send_buffer(self, obj, timeout=None, *, - unbound=None, + unbounditems=None, ): """Send the object's buffer to the channel's receiving end. This blocks until the object is received. """ - if unbound is None: - unboundop, = self._unbound + if unbounditems is None: + unboundop = -1 else: - unboundop, = _serialize_unbound(unbound) + unboundop, = _serialize_unbound(unbounditems) _channels.send_buffer(self._id, obj, unboundop, timeout=timeout, blocking=True) def send_buffer_nowait(self, obj, *, - unbound=None, + unbounditems=None, ): """Send the object's buffer to the channel's receiving end. If the object is immediately received then return True (else False). Otherwise this is the same as send(). """ - if unbound is None: - unboundop, = self._unbound + if unbounditems is None: + unboundop = -1 else: - unboundop, = _serialize_unbound(unbound) + unboundop, = _serialize_unbound(unbounditems) return _channels.send_buffer(self._id, obj, unboundop, blocking=False) def close(self): diff --git a/Lib/test/support/hashlib_helper.py b/Lib/test/support/hashlib_helper.py index 5043f08dd93de65..7032257b06877a6 100644 --- a/Lib/test/support/hashlib_helper.py +++ b/Lib/test/support/hashlib_helper.py @@ -23,6 +23,22 @@ def requires_builtin_hmac(): return unittest.skipIf(_hmac is None, "requires _hmac") +def _missing_hash(digestname, implementation=None, *, exc=None): + parts = ["missing", implementation, f"hash algorithm: {digestname!r}"] + msg = " ".join(filter(None, parts)) + raise unittest.SkipTest(msg) from exc + + +def _openssl_availabillity(digestname, *, usedforsecurity): + try: + _hashlib.new(digestname, usedforsecurity=usedforsecurity) + except AttributeError: + assert _hashlib is None + _missing_hash(digestname, "OpenSSL") + except ValueError as exc: + _missing_hash(digestname, "OpenSSL", exc=exc) + + def _decorate_func_or_class(func_or_class, decorator_func): if not isinstance(func_or_class, type): return decorator_func(func_or_class) @@ -71,8 +87,7 @@ def wrapper(*args, **kwargs): try: test_availability() except ValueError as exc: - msg = f"missing hash algorithm: {digestname!r}" - raise unittest.SkipTest(msg) from exc + _missing_hash(digestname, exc=exc) return func(*args, **kwargs) return wrapper @@ -87,14 +102,44 @@ def requires_openssl_hashdigest(digestname, *, usedforsecurity=True): The hashing algorithm may be missing or blocked by a strict crypto policy. """ def decorator_func(func): - @requires_hashlib() + @requires_hashlib() # avoid checking at each call @functools.wraps(func) def wrapper(*args, **kwargs): + _openssl_availabillity(digestname, usedforsecurity=usedforsecurity) + return func(*args, **kwargs) + return wrapper + + def decorator(func_or_class): + return _decorate_func_or_class(func_or_class, decorator_func) + return decorator + + +def find_openssl_hashdigest_constructor(digestname, *, usedforsecurity=True): + """Find the OpenSSL hash function constructor by its name.""" + assert isinstance(digestname, str), digestname + _openssl_availabillity(digestname, usedforsecurity=usedforsecurity) + # This returns a function of the form _hashlib.openssl_ and + # not a lambda function as it is rejected by _hashlib.hmac_new(). + return getattr(_hashlib, f"openssl_{digestname}") + + +def requires_builtin_hashdigest( + module_name, digestname, *, usedforsecurity=True +): + """Decorator raising SkipTest if a HACL* hashing algorithm is missing. + + - The *module_name* is the C extension module name based on HACL*. + - The *digestname* is one of its member, e.g., 'md5'. + """ + def decorator_func(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + module = import_module(module_name) try: - _hashlib.new(digestname, usedforsecurity=usedforsecurity) - except ValueError: - msg = f"missing OpenSSL hash algorithm: {digestname!r}" - raise unittest.SkipTest(msg) + getattr(module, digestname) + except AttributeError: + fullname = f'{module_name}.{digestname}' + _missing_hash(fullname, implementation="HACL") return func(*args, **kwargs) return wrapper @@ -103,6 +148,168 @@ def decorator(func_or_class): return decorator +def find_builtin_hashdigest_constructor( + module_name, digestname, *, usedforsecurity=True +): + """Find the HACL* hash function constructor. + + - The *module_name* is the C extension module name based on HACL*. + - The *digestname* is one of its member, e.g., 'md5'. + """ + module = import_module(module_name) + try: + constructor = getattr(module, digestname) + constructor(b'', usedforsecurity=usedforsecurity) + except (AttributeError, TypeError, ValueError): + _missing_hash(f'{module_name}.{digestname}', implementation="HACL") + return constructor + + +class HashFunctionsTrait: + """Mixin trait class containing hash functions. + + This class is assumed to have all unitest.TestCase methods but should + not directly inherit from it to prevent the test suite being run on it. + + Subclasses should implement the hash functions by returning an object + that can be recognized as a valid digestmod parameter for both hashlib + and HMAC. In particular, it cannot be a lambda function as it will not + be recognized by hashlib (it will still be accepted by the pure Python + implementation of HMAC). + """ + + ALGORITHMS = [ + 'md5', 'sha1', + 'sha224', 'sha256', 'sha384', 'sha512', + 'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', + ] + + # Default 'usedforsecurity' to use when looking up a hash function. + usedforsecurity = True + + def _find_constructor(self, name): + # By default, a missing algorithm skips the test that uses it. + self.assertIn(name, self.ALGORITHMS) + self.skipTest(f"missing hash function: {name}") + + @property + def md5(self): + return self._find_constructor("md5") + + @property + def sha1(self): + return self._find_constructor("sha1") + + @property + def sha224(self): + return self._find_constructor("sha224") + + @property + def sha256(self): + return self._find_constructor("sha256") + + @property + def sha384(self): + return self._find_constructor("sha384") + + @property + def sha512(self): + return self._find_constructor("sha512") + + @property + def sha3_224(self): + return self._find_constructor("sha3_224") + + @property + def sha3_256(self): + return self._find_constructor("sha3_256") + + @property + def sha3_384(self): + return self._find_constructor("sha3_384") + + @property + def sha3_512(self): + return self._find_constructor("sha3_512") + + +class NamedHashFunctionsTrait(HashFunctionsTrait): + """Trait containing named hash functions. + + Hash functions are available if and only if they are available in hashlib. + """ + + def _find_constructor(self, name): + self.assertIn(name, self.ALGORITHMS) + return name + + +class OpenSSLHashFunctionsTrait(HashFunctionsTrait): + """Trait containing OpenSSL hash functions. + + Hash functions are available if and only if they are available in _hashlib. + """ + + def _find_constructor(self, name): + self.assertIn(name, self.ALGORITHMS) + return find_openssl_hashdigest_constructor( + name, usedforsecurity=self.usedforsecurity + ) + + +class BuiltinHashFunctionsTrait(HashFunctionsTrait): + """Trait containing HACL* hash functions. + + Hash functions are available if and only if they are available in C. + In particular, HACL* HMAC-MD5 may be available even though HACL* md5 + is not since the former is unconditionally built. + """ + + def _find_constructor_in(self, module, name): + self.assertIn(name, self.ALGORITHMS) + return find_builtin_hashdigest_constructor(module, name) + + @property + def md5(self): + return self._find_constructor_in("_md5", "md5") + + @property + def sha1(self): + return self._find_constructor_in("_sha1", "sha1") + + @property + def sha224(self): + return self._find_constructor_in("_sha2", "sha224") + + @property + def sha256(self): + return self._find_constructor_in("_sha2", "sha256") + + @property + def sha384(self): + return self._find_constructor_in("_sha2", "sha384") + + @property + def sha512(self): + return self._find_constructor_in("_sha2", "sha512") + + @property + def sha3_224(self): + return self._find_constructor_in("_sha3", "sha3_224") + + @property + def sha3_256(self): + return self._find_constructor_in("_sha3","sha3_256") + + @property + def sha3_384(self): + return self._find_constructor_in("_sha3","sha3_384") + + @property + def sha3_512(self): + return self._find_constructor_in("_sha3","sha3_512") + + def find_gil_minsize(modules_names, default=2048): """Get the largest GIL_MINSIZE value for the given cryptographic modules. diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index edb734d294f2872..4c7eac0b7eb674f 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -305,8 +305,8 @@ def ready_to_import(name=None, source=""): try: sys.path.insert(0, tempdir) yield name, path - sys.path.remove(tempdir) finally: + sys.path.remove(tempdir) if old_module is not None: sys.modules[name] = old_module else: @@ -438,5 +438,5 @@ def ensure_module_imported(name, *, clearnone=True): if sys.modules.get(name) is not None: mod = sys.modules[name] else: - mod, _, _ = _force_import(name, False, True, clearnone) + mod, _, _ = _ensure_module(name, False, True, clearnone) return mod diff --git a/Lib/test/support/os_helper.py b/Lib/test/support/os_helper.py index 2c45fe2369ec36b..daf6060940e97f0 100644 --- a/Lib/test/support/os_helper.py +++ b/Lib/test/support/os_helper.py @@ -467,12 +467,23 @@ def _rmtree(path): def _rmtree_inner(path): from test.support import _force_run + # Clear file flags (e.g. UF_IMMUTABLE, UF_NOUNLINK on BSD). + if hasattr(os, 'chflags'): + try: + os.chflags(path, 0) + except OSError: + pass for name in _force_run(path, os.listdir, path): fullname = os.path.join(path, name) try: mode = os.lstat(fullname).st_mode except OSError: mode = 0 + if hasattr(os, 'lchflags'): + try: + os.lchflags(fullname, 0) + except OSError: + pass if stat.S_ISDIR(mode): _rmtree_inner(fullname) _force_run(path, os.rmdir, fullname) diff --git a/Lib/test/support/pty_helper.py b/Lib/test/support/pty_helper.py index 6587fd40333c512..dbe7fa429096fc6 100644 --- a/Lib/test/support/pty_helper.py +++ b/Lib/test/support/pty_helper.py @@ -15,6 +15,14 @@ def run_pty(script, input=b"dummy input\r", env=None): output = bytearray() [master, slave] = pty.openpty() args = (sys.executable, '-c', script) + + # Isolate readline from personal init files by setting INPUTRC + # to an empty file. See also GH-142353. + if env is None: + env = {**os.environ.copy(), "INPUTRC": os.devnull} + else: + env.setdefault("INPUTRC", os.devnull) + proc = subprocess.Popen(args, stdin=slave, stdout=slave, stderr=slave, env=env) os.close(slave) with ExitStack() as cleanup: diff --git a/Lib/test/support/socket_helper.py b/Lib/test/support/socket_helper.py index 87941ee1791b4e7..a41e487f3e4bc52 100644 --- a/Lib/test/support/socket_helper.py +++ b/Lib/test/support/socket_helper.py @@ -259,6 +259,10 @@ def filter_error(err): # raise OSError('socket error', msg) from msg elif len(a) >= 2 and isinstance(a[1], OSError): err = a[1] + # The error can also be wrapped as __cause__: + # raise URLError(f"ftp error: {exp}") from exp + elif isinstance(err, urllib.error.URLError) and err.__cause__: + err = err.__cause__ else: break filter_error(err) diff --git a/Lib/test/support/strace_helper.py b/Lib/test/support/strace_helper.py index 798d6c6886962fa..cf95f7bdc7d2cad 100644 --- a/Lib/test/support/strace_helper.py +++ b/Lib/test/support/strace_helper.py @@ -38,7 +38,7 @@ def events(self): This assumes the program under inspection doesn't print any non-utf8 strings which would mix into the strace output.""" - decoded_events = self.event_bytes.decode('utf-8') + decoded_events = self.event_bytes.decode('utf-8', 'surrogateescape') matches = [ _syscall_regex.match(event) for event in decoded_events.splitlines() @@ -178,7 +178,10 @@ def get_syscalls(code, strace_flags, prelude="", cleanup="", # Moderately expensive (spawns a subprocess), so share results when possible. @cache def _can_strace(): - res = strace_python("import sys; sys.exit(0)", [], check=False) + res = strace_python("import sys; sys.exit(0)", + # --trace option needs strace 5.5 (gh-133741) + ["--trace=%process"], + check=False) if res.strace_returncode == 0 and res.python_returncode == 0: assert res.events(), "Should have parsed multiple calls" return True diff --git a/Lib/test/support/threading_helper.py b/Lib/test/support/threading_helper.py index afa25a76f638295..cf87233f0e2e938 100644 --- a/Lib/test/support/threading_helper.py +++ b/Lib/test/support/threading_helper.py @@ -248,3 +248,38 @@ def requires_working_threading(*, module=False): raise unittest.SkipTest(msg) else: return unittest.skipUnless(can_start_thread, msg) + + +def run_concurrently(worker_func, nthreads=None, args=(), kwargs={}): + """ + Run the worker function(s) concurrently in multiple threads. + + If `worker_func` is a single callable, it is used for all threads. + If it is a list of callables, each callable is used for one thread. + """ + from collections.abc import Iterable + + if nthreads is None: + nthreads = len(worker_func) + if not isinstance(worker_func, Iterable): + worker_func = [worker_func] * nthreads + assert len(worker_func) == nthreads + + barrier = threading.Barrier(nthreads) + + def wrapper_func(func, *args, **kwargs): + # Wait for all threads to reach this point before proceeding. + barrier.wait() + func(*args, **kwargs) + + with catch_threading_exception() as cm: + workers = [ + threading.Thread(target=wrapper_func, args=(func, *args), kwargs=kwargs) + for func in worker_func + ] + with start_threads(workers): + pass + + # If a worker thread raises an exception, re-raise it. + if cm.exc_value is not None: + raise cm.exc_value diff --git a/Lib/test/support/warnings_helper.py b/Lib/test/support/warnings_helper.py index a6e43dff2003b75..5f6f14afd74a6ee 100644 --- a/Lib/test/support/warnings_helper.py +++ b/Lib/test/support/warnings_helper.py @@ -23,8 +23,7 @@ def check_syntax_warning(testcase, statement, errtext='', testcase.assertEqual(len(warns), 1, warns) warn, = warns - testcase.assertTrue(issubclass(warn.category, SyntaxWarning), - warn.category) + testcase.assertIsSubclass(warn.category, SyntaxWarning) if errtext: testcase.assertRegex(str(warn.message), errtext) testcase.assertEqual(warn.filename, '') diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index f35b11943082622..8ded9f99248372a 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -72,6 +72,8 @@ def check_all(self, modname): all_set = set(all_list) self.assertCountEqual(all_set, all_list, "in module {}".format(modname)) self.assertEqual(keys, all_set, "in module {}".format(modname)) + # Verify __dir__ is non-empty and doesn't produce an error + self.assertTrue(dir(sys.modules[modname])) def walk_modules(self, basedir, modpath): for fn in sorted(os.listdir(basedir)): diff --git a/Lib/test/test__colorize.py b/Lib/test/test__colorize.py index b2f0bb1386fe5b4..026277267e04ce0 100644 --- a/Lib/test/test__colorize.py +++ b/Lib/test/test__colorize.py @@ -1,4 +1,5 @@ import contextlib +import dataclasses import io import sys import unittest @@ -21,6 +22,41 @@ def supports_virtual_terminal(): return contextlib.nullcontext() +class TestTheme(unittest.TestCase): + + def test_attributes(self): + # only theme configurations attributes by default + for field in dataclasses.fields(_colorize.Theme): + with self.subTest(field.name): + self.assertIsSubclass(field.type, _colorize.ThemeSection) + self.assertIsNotNone(field.default_factory) + + def test_copy_with(self): + theme = _colorize.Theme() + + copy = theme.copy_with() + self.assertEqual(theme, copy) + + unittest_no_colors = _colorize.Unittest.no_colors() + copy = theme.copy_with(unittest=unittest_no_colors) + self.assertEqual(copy.argparse, theme.argparse) + self.assertEqual(copy.syntax, theme.syntax) + self.assertEqual(copy.traceback, theme.traceback) + self.assertEqual(copy.unittest, unittest_no_colors) + + def test_no_colors(self): + # idempotence test + theme_no_colors = _colorize.Theme().no_colors() + theme_no_colors_no_colors = theme_no_colors.no_colors() + self.assertEqual(theme_no_colors, theme_no_colors_no_colors) + + # attributes check + for section in dataclasses.fields(_colorize.Theme): + with self.subTest(section.name): + section_theme = getattr(theme_no_colors, section.name) + self.assertEqual(section_theme, section.type.no_colors()) + + class TestColorizeFunction(unittest.TestCase): def test_colorized_detection_checks_for_environment_variables(self): def check(env, fallback, expected): @@ -129,6 +165,17 @@ def test_colorized_detection_checks_for_file(self): file.isatty.return_value = False self.assertEqual(_colorize.can_colorize(file=file), False) + # The documentation for file.fileno says: + # > An OSError is raised if the IO object does not use a file descriptor. + # gh-141570: Check OSError is caught and handled + with unittest.mock.patch("os.isatty", side_effect=ZeroDivisionError): + file = unittest.mock.MagicMock() + file.fileno.side_effect = OSError + file.isatty.return_value = True + self.assertEqual(_colorize.can_colorize(file=file), True) + file.isatty.return_value = False + self.assertEqual(_colorize.can_colorize(file=file), False) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test__interpchannels.py b/Lib/test/test__interpchannels.py index e4c1ad854514ed7..2b0aba42896c06a 100644 --- a/Lib/test/test__interpchannels.py +++ b/Lib/test/test__interpchannels.py @@ -6,10 +6,10 @@ import time import unittest -from test.support import import_helper, skip_if_sanitizer +from test.support import import_helper _channels = import_helper.import_module('_interpchannels') -from test.support.interpreters import _crossinterp +from concurrent.interpreters import _crossinterp from test.test__interpreters import ( _interpreters, _run_output, @@ -247,7 +247,7 @@ def _run_action(cid, action, end, state): def clean_up_channels(): - for cid, _ in _channels.list_all(): + for cid, _, _ in _channels.list_all(): try: _channels.destroy(cid) except _channels.ChannelNotFoundError: @@ -365,7 +365,6 @@ def test_shareable(self): #self.assertIsNot(got, obj) -@skip_if_sanitizer('gh-129824: race on _waiting_release', thread=True) class ChannelTests(TestBase): def test_create_cid(self): @@ -373,16 +372,48 @@ def test_create_cid(self): self.assertIsInstance(cid, _channels.ChannelID) def test_sequential_ids(self): - before = [cid for cid, _ in _channels.list_all()] + before = [cid for cid, _, _ in _channels.list_all()] id1 = _channels.create(REPLACE) id2 = _channels.create(REPLACE) id3 = _channels.create(REPLACE) - after = [cid for cid, _ in _channels.list_all()] + after = [cid for cid, _, _ in _channels.list_all()] self.assertEqual(id2, int(id1) + 1) self.assertEqual(id3, int(id2) + 1) self.assertEqual(set(after) - set(before), {id1, id2, id3}) + def test_channel_list_all_closed(self): + id1 = _channels.create() + id2 = _channels.create() + id3 = _channels.create() + before = _channels.list_all() + expected = [info for info in before if info[0] != id2] + _channels.close(id2, force=True) + after = _channels.list_all() + self.assertEqual(set(after), set(expected)) + self.assertEqual(len(after), len(before) - 1) + + def test_channel_list_all_destroyed(self): + id1 = _channels.create() + id2 = _channels.create() + id3 = _channels.create() + before = _channels.list_all() + expected = [info for info in before if info[0] != id2] + _channels.destroy(id2) + after = _channels.list_all() + self.assertEqual(set(after), set(expected)) + self.assertEqual(len(after), len(before) - 1) + + def test_channel_list_all_released(self): + id1 = _channels.create() + id2 = _channels.create() + id3 = _channels.create() + before = _channels.list_all() + _channels.release(id2, send=True, recv=True) + after = _channels.list_all() + self.assertEqual(set(after), set(before)) + self.assertEqual(len(after), len(before)) + def test_ids_global(self): id1 = _interpreters.create() out = _run_output(id1, dedent(""" diff --git a/Lib/test/test__interpreters.py b/Lib/test/test__interpreters.py index 0c43f46300f67de..a32d5d81d2bf2da 100644 --- a/Lib/test/test__interpreters.py +++ b/Lib/test/test__interpreters.py @@ -474,15 +474,32 @@ def setUp(self): def test_signatures(self): # See https://github.com/python/cpython/issues/126654 - msg = "expected 'shared' to be a dict" + msg = r'_interpreters.exec\(\) argument 3 must be dict, not int' with self.assertRaisesRegex(TypeError, msg): _interpreters.exec(self.id, 'a', 1) with self.assertRaisesRegex(TypeError, msg): _interpreters.exec(self.id, 'a', shared=1) + msg = r'_interpreters.run_string\(\) argument 3 must be dict, not int' with self.assertRaisesRegex(TypeError, msg): _interpreters.run_string(self.id, 'a', shared=1) + msg = r'_interpreters.run_func\(\) argument 3 must be dict, not int' with self.assertRaisesRegex(TypeError, msg): _interpreters.run_func(self.id, lambda: None, shared=1) + # See https://github.com/python/cpython/issues/135855 + msg = r'_interpreters.set___main___attrs\(\) argument 2 must be dict, not int' + with self.assertRaisesRegex(TypeError, msg): + _interpreters.set___main___attrs(self.id, 1) + + def test_invalid_shared_none(self): + msg = r'must be dict, not None' + with self.assertRaisesRegex(TypeError, msg): + _interpreters.exec(self.id, 'a', shared=None) + with self.assertRaisesRegex(TypeError, msg): + _interpreters.run_string(self.id, 'a', shared=None) + with self.assertRaisesRegex(TypeError, msg): + _interpreters.run_func(self.id, lambda: None, shared=None) + with self.assertRaisesRegex(TypeError, msg): + _interpreters.set___main___attrs(self.id, None) def test_invalid_shared_encoding(self): # See https://github.com/python/cpython/issues/127196 @@ -952,7 +969,8 @@ def test_invalid_syntax(self): """) with self.subTest('script'): - self.assert_run_failed(SyntaxError, script) + with self.assertRaises(SyntaxError): + _interpreters.run_string(self.id, script) with self.subTest('module'): modname = 'spam_spam_spam' @@ -1019,12 +1037,19 @@ def script(): with open(w, 'w', encoding="utf-8") as spipe: with contextlib.redirect_stdout(spipe): print('it worked!', end='') + failed = None def f(): - _interpreters.set___main___attrs(self.id, dict(w=w)) - _interpreters.run_func(self.id, script) + nonlocal failed + try: + _interpreters.set___main___attrs(self.id, dict(w=w)) + _interpreters.run_func(self.id, script) + except Exception as exc: + failed = exc t = threading.Thread(target=f) t.start() t.join() + if failed: + raise Exception from failed with open(r, encoding="utf-8") as outfile: out = outfile.read() @@ -1053,18 +1078,16 @@ def test_closure(self): spam = True def script(): assert spam - with self.assertRaises(ValueError): _interpreters.run_func(self.id, script) - # XXX This hasn't been fixed yet. - @unittest.expectedFailure def test_return_value(self): def script(): return 'spam' with self.assertRaises(ValueError): _interpreters.run_func(self.id, script) +# @unittest.skip("we're not quite there yet") def test_args(self): with self.subTest('args'): def script(a, b=0): diff --git a/Lib/test/test__osx_support.py b/Lib/test/test__osx_support.py index 53aa26620a6475e..0813c4804c1cdca 100644 --- a/Lib/test/test__osx_support.py +++ b/Lib/test/test__osx_support.py @@ -66,8 +66,8 @@ def test__find_build_tool(self): 'cc not found - check xcode-select') def test__get_system_version(self): - self.assertTrue(platform.mac_ver()[0].startswith( - _osx_support._get_system_version())) + self.assertStartsWith(platform.mac_ver()[0], + _osx_support._get_system_version()) def test__remove_original_values(self): config_vars = { diff --git a/Lib/test/test_abstract_numbers.py b/Lib/test/test_abstract_numbers.py index 72232b670cdb89e..cf071d2c933dd2a 100644 --- a/Lib/test/test_abstract_numbers.py +++ b/Lib/test/test_abstract_numbers.py @@ -24,11 +24,11 @@ def not_implemented(*args, **kwargs): class TestNumbers(unittest.TestCase): def test_int(self): - self.assertTrue(issubclass(int, Integral)) - self.assertTrue(issubclass(int, Rational)) - self.assertTrue(issubclass(int, Real)) - self.assertTrue(issubclass(int, Complex)) - self.assertTrue(issubclass(int, Number)) + self.assertIsSubclass(int, Integral) + self.assertIsSubclass(int, Rational) + self.assertIsSubclass(int, Real) + self.assertIsSubclass(int, Complex) + self.assertIsSubclass(int, Number) self.assertEqual(7, int(7).real) self.assertEqual(0, int(7).imag) @@ -38,11 +38,11 @@ def test_int(self): self.assertEqual(1, int(7).denominator) def test_float(self): - self.assertFalse(issubclass(float, Integral)) - self.assertFalse(issubclass(float, Rational)) - self.assertTrue(issubclass(float, Real)) - self.assertTrue(issubclass(float, Complex)) - self.assertTrue(issubclass(float, Number)) + self.assertNotIsSubclass(float, Integral) + self.assertNotIsSubclass(float, Rational) + self.assertIsSubclass(float, Real) + self.assertIsSubclass(float, Complex) + self.assertIsSubclass(float, Number) self.assertEqual(7.3, float(7.3).real) self.assertEqual(0, float(7.3).imag) @@ -50,11 +50,11 @@ def test_float(self): self.assertEqual(-7.3, float(-7.3).conjugate()) def test_complex(self): - self.assertFalse(issubclass(complex, Integral)) - self.assertFalse(issubclass(complex, Rational)) - self.assertFalse(issubclass(complex, Real)) - self.assertTrue(issubclass(complex, Complex)) - self.assertTrue(issubclass(complex, Number)) + self.assertNotIsSubclass(complex, Integral) + self.assertNotIsSubclass(complex, Rational) + self.assertNotIsSubclass(complex, Real) + self.assertIsSubclass(complex, Complex) + self.assertIsSubclass(complex, Number) c1, c2 = complex(3, 2), complex(4,1) # XXX: This is not ideal, but see the comment in math_trunc(). diff --git a/Lib/test/test_android.py b/Lib/test/test_android.py index de83ce081c2790f..31daafbc3d63009 100644 --- a/Lib/test/test_android.py +++ b/Lib/test/test_android.py @@ -1,5 +1,4 @@ import io -import platform import queue import re import subprocess @@ -17,8 +16,6 @@ if sys.platform != "android": raise unittest.SkipTest("Android-specific") -api_level = platform.android_ver().api_level - # (name, level, fileno) STREAM_INFO = [("stdout", "I", 1), ("stderr", "W", 2)] @@ -91,34 +88,38 @@ def tearDown(self): self.logcat_thread = None @contextmanager - def unbuffered(self, stream): - stream.reconfigure(write_through=True) + def reconfigure(self, stream, **settings): + original_settings = {key: getattr(stream, key, None) for key in settings.keys()} + stream.reconfigure(**settings) try: yield finally: - stream.reconfigure(write_through=False) + stream.reconfigure(**original_settings) - # In --verbose3 mode, sys.stdout and sys.stderr are captured, so we can't - # test them directly. Detect this mode and use some temporary streams with - # the same properties. def stream_context(self, stream_name, level): - # https://developer.android.com/ndk/reference/group/logging - prio = {"I": 4, "W": 5}[level] - stack = ExitStack() stack.enter_context(self.subTest(stream_name)) + + # In --verbose3 mode, sys.stdout and sys.stderr are captured, so we can't + # test them directly. Detect this mode and use some temporary streams with + # the same properties. stream = getattr(sys, stream_name) native_stream = getattr(sys, f"__{stream_name}__") if isinstance(stream, io.StringIO): + # https://developer.android.com/ndk/reference/group/logging + prio = {"I": 4, "W": 5}[level] stack.enter_context( patch( f"sys.{stream_name}", - TextLogStream( - prio, f"python.{stream_name}", native_stream.fileno(), - errors="backslashreplace" + stream := TextLogStream( + prio, f"python.{stream_name}", native_stream, ), ) ) + + # The tests assume the stream is initially buffered. + stack.enter_context(self.reconfigure(stream, write_through=False)) + return stack def test_str(self): @@ -145,7 +146,7 @@ def write(s, lines=None, *, write_len=None): self.assert_logs(level, tag, lines) # Single-line messages, - with self.unbuffered(stream): + with self.reconfigure(stream, write_through=True): write("", []) write("a") @@ -175,14 +176,18 @@ def write(s, lines=None, *, write_len=None): # Multi-line messages. Avoid identical consecutive lines, as # they may activate "chatty" filtering and break the tests. - write("\nx", [""]) + # + # Additional spaces will appear in the output where necessary to + # protect leading newlines. + write("\nx", [" "]) write("\na\n", ["x", "a"]) - write("\n", [""]) + write("\n", [" "]) + write("\n\n", [" ", " "]) write("b\n", ["b"]) - write("c\n\n", ["c", ""]) + write("c\n\n", ["c", " "]) write("d\ne", ["d"]) write("xx", []) - write("f\n\ng", ["exxf", ""]) + write("f\n\ng", ["exxf", " "]) write("\n", ["g"]) # Since this is a line-based logging system, line buffering @@ -192,16 +197,17 @@ def write(s, lines=None, *, write_len=None): # However, buffering can be turned off completely if you want a # flush after every write. - with self.unbuffered(stream): - write("\nx", ["", "x"]) - write("\na\n", ["", "a"]) - write("\n", [""]) + with self.reconfigure(stream, write_through=True): + write("\nx", [" ", "x"]) + write("\na\n", [" ", "a"]) + write("\n", [" "]) + write("\n\n", [" ", " "]) write("b\n", ["b"]) - write("c\n\n", ["c", ""]) + write("c\n\n", ["c", " "]) write("d\ne", ["d", "e"]) write("xx", ["xx"]) - write("f\n\ng", ["f", "", "g"]) - write("\n", [""]) + write("f\n\ng", ["f", " ", "g"]) + write("\n", [" "]) # "\r\n" should be translated into "\n". write("hello\r\n", ["hello"]) @@ -321,19 +327,16 @@ def write(b, lines=None, *, write_len=None): # currently use `logcat -v tag`, which shows each line as if it # was a separate log entry, but strips a single trailing # newline. - # - # On newer versions of Android, all three of the above tools (or - # maybe Logcat itself) will also strip any number of leading - # newlines. - write(b"\nx", ["", "x"] if api_level < 30 else ["x"]) - write(b"\na\n", ["", "a"] if api_level < 30 else ["a"]) - write(b"\n", [""]) + write(b"\nx", [" ", "x"]) + write(b"\na\n", [" ", "a"]) + write(b"\n", [" "]) + write(b"\n\n", [" ", ""]) write(b"b\n", ["b"]) write(b"c\n\n", ["c", ""]) write(b"d\ne", ["d", "e"]) write(b"xx", ["xx"]) write(b"f\n\ng", ["f", "", "g"]) - write(b"\n", [""]) + write(b"\n", [" "]) # "\r\n" should be translated into "\n". write(b"hello\r\n", ["hello"]) diff --git a/Lib/test/test_annotationlib.py b/Lib/test/test_annotationlib.py index c3c245ddaf86d1b..5087c3ca425f1fc 100644 --- a/Lib/test/test_annotationlib.py +++ b/Lib/test/test_annotationlib.py @@ -7,7 +7,10 @@ import functools import itertools import pickle +from string.templatelib import Template, Interpolation +import types import typing +import sys import unittest from annotationlib import ( Format, @@ -72,6 +75,30 @@ def inner(arg: x): anno = get_annotations(inner, format=Format.FORWARDREF) self.assertEqual(anno["arg"], x) + def test_multiple_closure(self): + def inner(arg: x[y]): + pass + + fwdref = get_annotations(inner, format=Format.FORWARDREF)["arg"] + self.assertIsInstance(fwdref, ForwardRef) + self.assertEqual(fwdref.__forward_arg__, "x[y]") + with self.assertRaises(NameError): + fwdref.evaluate() + + y = str + fwdref = get_annotations(inner, format=Format.FORWARDREF)["arg"] + self.assertIsInstance(fwdref, ForwardRef) + extra_name, extra_val = next(iter(fwdref.__extra_names__.items())) + self.assertEqual(fwdref.__forward_arg__.replace(extra_name, extra_val.__name__), "x[str]") + with self.assertRaises(NameError): + fwdref.evaluate() + + x = list + self.assertEqual(fwdref.evaluate(), x[y]) + + fwdref = get_annotations(inner, format=Format.FORWARDREF)["arg"] + self.assertEqual(fwdref, x[y]) + def test_function(self): def f(x: int, y: doesntexist): pass @@ -93,6 +120,10 @@ def f( alpha: some | obj, beta: +some, gamma: some < obj, + delta: some | {obj: module}, + epsilon: some | {obj}, + zeta: some | [obj, module], + eta: some | (), ): pass @@ -121,6 +152,69 @@ def f( self.assertIsInstance(gamma_anno, ForwardRef) self.assertEqual(gamma_anno, support.EqualToForwardRef("some < obj", owner=f)) + delta_anno = anno["delta"] + self.assertIsInstance(delta_anno, ForwardRef) + self.assertEqual(delta_anno, support.EqualToForwardRef("some | {obj: module}", owner=f)) + + epsilon_anno = anno["epsilon"] + self.assertIsInstance(epsilon_anno, ForwardRef) + self.assertEqual(epsilon_anno, support.EqualToForwardRef("some | {obj}", owner=f)) + + zeta_anno = anno["zeta"] + self.assertIsInstance(zeta_anno, ForwardRef) + self.assertEqual(zeta_anno, support.EqualToForwardRef("some | [obj, module]", owner=f)) + + eta_anno = anno["eta"] + self.assertIsInstance(eta_anno, ForwardRef) + self.assertEqual(eta_anno, support.EqualToForwardRef("some | ()", owner=f)) + + def test_partially_nonexistent(self): + # These annotations start with a non-existent variable and then use + # global types with defined values. This partially evaluates by putting + # those globals into `fwdref.__extra_names__`. + def f( + x: obj | int, + y: container[int:obj, int], + z: dict_val | {str: int}, + alpha: set_val | {str, int}, + beta: obj | bool | int, + gamma: obj | call_func(int, kwd=bool), + ): + pass + + def func(*args, **kwargs): + return Union[*args, *(kwargs.values())] + + anno = get_annotations(f, format=Format.FORWARDREF) + globals_ = { + "obj": str, "container": list, "dict_val": {1: 2}, "set_val": {1, 2}, + "call_func": func + } + + x_anno = anno["x"] + self.assertIsInstance(x_anno, ForwardRef) + self.assertEqual(x_anno.evaluate(globals=globals_), str | int) + + y_anno = anno["y"] + self.assertIsInstance(y_anno, ForwardRef) + self.assertEqual(y_anno.evaluate(globals=globals_), list[int:str, int]) + + z_anno = anno["z"] + self.assertIsInstance(z_anno, ForwardRef) + self.assertEqual(z_anno.evaluate(globals=globals_), {1: 2} | {str: int}) + + alpha_anno = anno["alpha"] + self.assertIsInstance(alpha_anno, ForwardRef) + self.assertEqual(alpha_anno.evaluate(globals=globals_), {1, 2} | {str, int}) + + beta_anno = anno["beta"] + self.assertIsInstance(beta_anno, ForwardRef) + self.assertEqual(beta_anno.evaluate(globals=globals_), str | bool | int) + + gamma_anno = anno["gamma"] + self.assertIsInstance(gamma_anno, ForwardRef) + self.assertEqual(gamma_anno.evaluate(globals=globals_), str | func(int, kwd=bool)) + def test_partially_nonexistent_union(self): # Test unions with '|' syntax equal unions with typing.Union[] with some forwardrefs class UnionForwardrefs: @@ -273,6 +367,45 @@ def f( }, ) + def test_template_str(self): + def f( + x: t"{a}", + y: list[t"{a}"], + z: t"{a:b} {c!r} {d!s:t}", + a: t"a{b}c{d}e{f}g", + b: t"{a:{1}}", + c: t"{a | b * c}", + gh138558: t"{ 0}", + ): pass + + annos = get_annotations(f, format=Format.STRING) + self.assertEqual(annos, { + "x": "t'{a}'", + "y": "list[t'{a}']", + "z": "t'{a:b} {c!r} {d!s:t}'", + "a": "t'a{b}c{d}e{f}g'", + # interpolations in the format spec are eagerly evaluated so we can't recover the source + "b": "t'{a:1}'", + "c": "t'{a | b * c}'", + "gh138558": "t'{ 0}'", + }) + + def g( + x: t"{a}", + ): ... + + annos = get_annotations(g, format=Format.FORWARDREF) + templ = annos["x"] + # Template and Interpolation don't have __eq__ so we have to compare manually + self.assertIsInstance(templ, Template) + self.assertEqual(templ.strings, ("", "")) + self.assertEqual(len(templ.interpolations), 1) + interp = templ.interpolations[0] + self.assertEqual(interp.value, support.EqualToForwardRef("a", owner=g)) + self.assertEqual(interp.expression, "a") + self.assertIsNone(interp.conversion) + self.assertEqual(interp.format_spec, "") + def test_getitem(self): def f(x: undef1[str, undef2]): pass @@ -340,7 +473,7 @@ def f(x: a[[int, str], float]): def g( w: a[[int, str], float], - x: a[{int, str}, 3], + x: a[{int}, 3], y: a[{int: str}, 4], z: a[(int, str), 5], ): @@ -350,7 +483,7 @@ def g( anno, { "w": "a[[int, str], float]", - "x": "a[{int, str}, 3]", + "x": "a[{int}, 3]", "y": "a[{int: str}, 4]", "z": "a[(int, str), 5]", }, @@ -513,6 +646,31 @@ def foo(): get_annotations(foo, format=Format.FORWARDREF, eval_str=True) get_annotations(foo, format=Format.STRING, eval_str=True) + def test_eval_str_wrapped_cycle_self(self): + # gh-146556: self-referential __wrapped__ cycle must not hang. + def f(x: 'int') -> 'str': ... + f.__wrapped__ = f + # Cycle is detected and broken; globals from f itself are used. + result = get_annotations(f, eval_str=True) + self.assertEqual(result, {'x': int, 'return': str}) + + def test_eval_str_wrapped_cycle_mutual(self): + # gh-146556: mutual __wrapped__ cycle (a -> b -> a) must not hang. + def a(x: 'int'): ... + def b(): ... + a.__wrapped__ = b + b.__wrapped__ = a + result = get_annotations(a, eval_str=True) + self.assertEqual(result, {'x': int}) + + def test_eval_str_wrapped_chain_no_cycle(self): + # gh-146556: a valid (non-cyclic) __wrapped__ chain must still work. + def inner(x: 'int'): ... + def outer(x: 'int'): ... + outer.__wrapped__ = inner + result = get_annotations(outer, eval_str=True) + self.assertEqual(result, {'x': int}) + def test_stock_annotations(self): def foo(a: int, b: str): pass @@ -715,6 +873,8 @@ def test_stringized_annotations_in_module(self): for kwargs in [ {"eval_str": True}, + {"eval_str": True, "globals": isa.__dict__, "locals": {}}, + {"eval_str": True, "globals": {}, "locals": isa.__dict__}, {"format": Format.VALUE, "eval_str": True}, ]: with self.subTest(**kwargs): @@ -747,6 +907,15 @@ def test_stringized_annotations_in_empty_module(self): self.assertEqual(get_annotations(isa2, eval_str=True), {}) self.assertEqual(get_annotations(isa2, eval_str=False), {}) + def test_stringized_annotations_with_star_unpack(self): + def f(*args: "*tuple[int, ...]"): ... + self.assertEqual(get_annotations(f, eval_str=True), + {'args': (*tuple[int, ...],)[0]}) + def f(*args: " *tuple[int, ...]"): ... + self.assertEqual(get_annotations(f, eval_str=True), + {'args': (*tuple[int, ...],)[0]}) + + def test_stringized_annotations_on_wrapper(self): isa = inspect_stringized_annotations wrapped = times_three(isa.function) @@ -765,6 +934,44 @@ def test_stringized_annotations_on_wrapper(self): {"a": "int", "b": "str", "return": "MyClass"}, ) + def test_stringized_annotations_on_partial_wrapper(self): + isa = inspect_stringized_annotations + + def times_three_str(fn: typing.Callable[[str], isa.MyClass]): + @functools.wraps(fn) + def wrapper(b: "str") -> "MyClass": + return fn(b * 3) + + return wrapper + + wrapped = times_three_str(functools.partial(isa.function, 1)) + self.assertEqual(wrapped("x"), isa.MyClass(1, "xxx")) + self.assertIsNot(wrapped.__globals__, isa.function.__globals__) + self.assertEqual( + get_annotations(wrapped, eval_str=True), + {"b": str, "return": isa.MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=False), + {"b": "str", "return": "MyClass"}, + ) + + # If functools is not loaded, names will be evaluated in the current + # module instead of being unwrapped to the original. + functools_mod = sys.modules["functools"] + del sys.modules["functools"] + + self.assertEqual( + get_annotations(wrapped, eval_str=True), + {"b": str, "return": MyClass}, + ) + self.assertEqual( + get_annotations(wrapped, eval_str=False), + {"b": "str", "return": "MyClass"}, + ) + + sys.modules["functools"] = functools_mod + def test_stringized_annotations_on_class(self): isa = inspect_stringized_annotations # test that local namespace lookups work @@ -777,6 +984,80 @@ def test_stringized_annotations_on_class(self): {"x": int}, ) + def test_stringized_annotations_on_custom_object(self): + class HasAnnotations: + @property + def __annotations__(self): + return {"x": "int"} + + ha = HasAnnotations() + self.assertEqual(get_annotations(ha), {"x": "int"}) + self.assertEqual(get_annotations(ha, eval_str=True), {"x": int}) + + def test_stringized_annotation_permutations(self): + def define_class(name, has_future, has_annos, base_text, extra_names=None): + lines = [] + if has_future: + lines.append("from __future__ import annotations") + lines.append(f"class {name}({base_text}):") + if has_annos: + lines.append(f" {name}_attr: int") + else: + lines.append(" pass") + code = "\n".join(lines) + ns = support.run_code(code, extra_names=extra_names) + return ns[name] + + def check_annotations(cls, has_future, has_annos): + if has_annos: + if has_future: + anno = "int" + else: + anno = int + self.assertEqual(get_annotations(cls), {f"{cls.__name__}_attr": anno}) + else: + self.assertEqual(get_annotations(cls), {}) + + for meta_future, base_future, child_future, meta_has_annos, base_has_annos, child_has_annos in itertools.product( + (False, True), + (False, True), + (False, True), + (False, True), + (False, True), + (False, True), + ): + with self.subTest( + meta_future=meta_future, + base_future=base_future, + child_future=child_future, + meta_has_annos=meta_has_annos, + base_has_annos=base_has_annos, + child_has_annos=child_has_annos, + ): + meta = define_class( + "Meta", + has_future=meta_future, + has_annos=meta_has_annos, + base_text="type", + ) + base = define_class( + "Base", + has_future=base_future, + has_annos=base_has_annos, + base_text="metaclass=Meta", + extra_names={"Meta": meta}, + ) + child = define_class( + "Child", + has_future=child_future, + has_annos=child_has_annos, + base_text="Base", + extra_names={"Base": base}, + ) + check_annotations(meta, meta_future, meta_has_annos) + check_annotations(base, base_future, base_has_annos) + check_annotations(child, child_future, child_has_annos) + def test_modify_annotations(self): def f(x: int): pass @@ -880,6 +1161,23 @@ def __annotate__(self): {"x": "int"}, ) + def test_non_dict_annotate(self): + class WeirdAnnotate: + def __annotate__(self, *args, **kwargs): + return "not a dict" + + wa = WeirdAnnotate() + for format in Format: + if format == Format.VALUE_WITH_FAKE_GLOBALS: + continue + with ( + self.subTest(format=format), + self.assertRaisesRegex( + ValueError, r".*__annotate__ returned a non-dict" + ), + ): + get_annotations(wa, format=format) + def test_no_annotations(self): class CustomClass: pass @@ -1084,6 +1382,40 @@ class RaisesAttributeError: }, ) + def test_nonlocal_in_annotation_scope(self): + class Demo: + nonlocal sequence_b + x: sequence_b + y: sequence_b[int] + + fwdrefs = get_annotations(Demo, format=Format.FORWARDREF) + + self.assertIsInstance(fwdrefs["x"], ForwardRef) + self.assertIsInstance(fwdrefs["y"], ForwardRef) + + sequence_b = list + self.assertIs(fwdrefs["x"].evaluate(), list) + self.assertEqual(fwdrefs["y"].evaluate(), list[int]) + + def test_raises_error_from_value(self): + # test that if VALUE is the only supported format, but raises an error + # that error is propagated from get_annotations + class DemoException(Exception): ... + + def annotate(format, /): + if format == Format.VALUE: + raise DemoException() + else: + raise NotImplementedError(format) + + def f(): ... + + f.__annotate__ = annotate + + for fmt in [Format.VALUE, Format.FORWARDREF, Format.STRING]: + with self.assertRaises(DemoException): + get_annotations(f, format=fmt) + class TestCallEvaluateFunction(unittest.TestCase): def test_evaluation(self): @@ -1103,6 +1435,284 @@ def evaluate(format, exc=NotImplementedError): "undefined", ) + def test_fake_global_evaluation(self): + # This will raise an AttributeError + def evaluate_union(format, exc=NotImplementedError): + if format == Format.VALUE_WITH_FAKE_GLOBALS: + # Return a ForwardRef + return builtins.undefined | list[int] + raise exc + + self.assertEqual( + annotationlib.call_evaluate_function(evaluate_union, Format.FORWARDREF), + support.EqualToForwardRef("builtins.undefined | list[int]"), + ) + + # This will raise an AttributeError + def evaluate_intermediate(format, exc=NotImplementedError): + if format == Format.VALUE_WITH_FAKE_GLOBALS: + intermediate = builtins.undefined + # Return a literal + return intermediate is None + raise exc + + self.assertIs( + annotationlib.call_evaluate_function(evaluate_intermediate, Format.FORWARDREF), + False, + ) + + +class TestCallAnnotateFunction(unittest.TestCase): + # Tests for user defined annotate functions. + + # Format and NotImplementedError are provided as arguments so they exist in + # the fake globals namespace. + # This avoids non-matching conditions passing by being converted to stringifiers. + # See: https://github.com/python/cpython/issues/138764 + + def test_user_annotate_value(self): + def annotate(format, /): + if format == Format.VALUE: + return {"x": str} + else: + raise NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.VALUE, + ) + + self.assertEqual(annotations, {"x": str}) + + def test_user_annotate_forwardref_supported(self): + # If Format.FORWARDREF is supported prefer it over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + elif format == __Format.FORWARDREF: + return {'x': float} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF + ) + + self.assertEqual(annotations, {"x": float}) + + def test_user_annotate_forwardref_fakeglobals(self): + # If Format.FORWARDREF is not supported, use Format.VALUE_WITH_FAKE_GLOBALS + # before falling back to Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF + ) + + self.assertEqual(annotations, {"x": int}) + + def test_user_annotate_forwardref_value_fallback(self): + # If Format.FORWARDREF and Format.VALUE_WITH_FAKE_GLOBALS are not supported + # use Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {"x": str} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.FORWARDREF, + ) + + self.assertEqual(annotations, {"x": str}) + + def test_user_annotate_string_supported(self): + # If Format.STRING is supported prefer it over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + elif format == __Format.STRING: + return {'x': "float"} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + self.assertEqual(annotations, {"x": "float"}) + + def test_user_annotate_string_fakeglobals(self): + # If Format.STRING is not supported but Format.VALUE_WITH_FAKE_GLOBALS is + # prefer that over Format.VALUE + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + self.assertEqual(annotations, {"x": "int"}) + + def test_user_annotate_string_value_fallback(self): + # If Format.STRING and Format.VALUE_WITH_FAKE_GLOBALS are not + # supported fall back to Format.VALUE and convert to strings + def annotate(format, /, __Format=Format, __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {"x": str} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function( + annotate, + Format.STRING, + ) + + self.assertEqual(annotations, {"x": "str"}) + + def test_condition_not_stringified(self): + # Make sure the first condition isn't evaluated as True by being converted + # to a _Stringifier + def annotate(format, /): + if format == Format.FORWARDREF: + return {"x": str} + else: + raise NotImplementedError(format) + + with self.assertRaises(NotImplementedError): + annotationlib.call_annotate_function(annotate, Format.STRING) + + def test_unsupported_formats(self): + def annotate(format, /): + if format == Format.FORWARDREF: + return {"x": str} + else: + raise NotImplementedError(format) + + with self.assertRaises(ValueError): + annotationlib.call_annotate_function(annotate, Format.VALUE_WITH_FAKE_GLOBALS) + + with self.assertRaises(RuntimeError): + annotationlib.call_annotate_function(annotate, Format.VALUE) + + with self.assertRaises(ValueError): + # Some non-Format value + annotationlib.call_annotate_function(annotate, 7) + + def test_basic_non_function_annotate(self): + class Annotate: + def __call__(self, format, /, __Format=Format, + __NotImplementedError=NotImplementedError): + if format == __Format.VALUE: + return {'x': str} + elif format == __Format.VALUE_WITH_FAKE_GLOBALS: + return {'x': int} + elif format == __Format.STRING: + return {'x': "float"} + else: + raise __NotImplementedError(format) + + annotations = annotationlib.call_annotate_function(Annotate(), Format.VALUE) + self.assertEqual(annotations, {"x": str}) + + annotations = annotationlib.call_annotate_function(Annotate(), Format.STRING) + self.assertEqual(annotations, {"x": "float"}) + + with self.assertRaises(AttributeError) as cm: + annotations = annotationlib.call_annotate_function( + Annotate(), Format.FORWARDREF + ) + + self.assertEqual(cm.exception.name, "__builtins__") + self.assertIsInstance(cm.exception.obj, Annotate) + + def test_full_non_function_annotate(self): + def outer(): + local = str + + class Annotate: + called_formats = [] + + def __call__(self, format=None, *, _self=None): + nonlocal local + if _self is not None: + self, format = _self, self + + self.called_formats.append(format) + if format == 1: # VALUE + return {"x": MyClass, "y": int, "z": local} + if format == 2: # VALUE_WITH_FAKE_GLOBALS + return {"w": unknown, "x": MyClass, "y": int, "z": local} + raise NotImplementedError + + __globals__ = {"MyClass": MyClass} + __builtins__ = {"int": int} + __closure__ = (types.CellType(str),) + __defaults__ = (None,) + + __kwdefaults__ = property(lambda self: dict(_self=self)) + __code__ = property(lambda self: self.__call__.__code__) + + return Annotate() + + annotate = outer() + + self.assertEqual( + annotationlib.call_annotate_function(annotate, Format.VALUE), + {"x": MyClass, "y": int, "z": str} + ) + self.assertEqual(annotate.called_formats[-1], Format.VALUE) + + self.assertEqual( + annotationlib.call_annotate_function(annotate, Format.STRING), + {"w": "unknown", "x": "MyClass", "y": "int", "z": "local"} + ) + self.assertIn(Format.STRING, annotate.called_formats) + self.assertEqual(annotate.called_formats[-1], Format.VALUE_WITH_FAKE_GLOBALS) + + self.assertEqual( + annotationlib.call_annotate_function(annotate, Format.FORWARDREF), + {"w": support.EqualToForwardRef("unknown"), "x": MyClass, "y": int, "z": str} + ) + self.assertIn(Format.FORWARDREF, annotate.called_formats) + self.assertEqual(annotate.called_formats[-1], Format.VALUE_WITH_FAKE_GLOBALS) + + def test_error_from_value_raised(self): + # Test that the error from format.VALUE is raised + # if all formats fail + + class DemoException(Exception): ... + + def annotate(format, /): + if format == Format.VALUE: + raise DemoException() + else: + raise NotImplementedError(format) + + for fmt in [Format.VALUE, Format.FORWARDREF, Format.STRING]: + with self.assertRaises(DemoException): + annotationlib.call_annotate_function(annotate, format=fmt) + class MetaclassTests(unittest.TestCase): def test_annotated_meta(self): @@ -1248,6 +1858,24 @@ def nested(): self.assertEqual(type_repr("1"), "'1'") self.assertEqual(type_repr(Format.VALUE), repr(Format.VALUE)) self.assertEqual(type_repr(MyClass()), "my repr") + # gh138558 tests + self.assertEqual(type_repr(t'''{ 0 + & 1 + | 2 + }'''), 't"""{ 0\n & 1\n | 2}"""') + self.assertEqual( + type_repr(Template("hi", Interpolation(42, "42"))), "t'hi{42}'" + ) + self.assertEqual( + type_repr(Template("hi", Interpolation(42))), + "Template('hi', Interpolation(42, '', None, ''))", + ) + self.assertEqual( + type_repr(Template("hi", Interpolation(42, " "))), + "Template('hi', Interpolation(42, ' ', None, ''))", + ) + # gh138558: perhaps in the future, we can improve this behavior: + self.assertEqual(type_repr(Template(Interpolation(42, "99"))), "t'{99}'") class TestAnnotationsToString(unittest.TestCase): @@ -1263,6 +1891,11 @@ def test_annotations_to_string(self): class A: pass +TypeParamsAlias1 = int + +class TypeParamsSample[TypeParamsAlias1, TypeParamsAlias2]: + TypeParamsAlias2 = str + class TestForwardRefClass(unittest.TestCase): def test_forwardref_instance_type_error(self): @@ -1333,6 +1966,39 @@ def foo(a: c1_gth, b: c2_gth): self.assertNotEqual(hash(c3), hash(c4)) self.assertEqual(hash(c3), hash(ForwardRef("int", module=__name__))) + def test_forward_equality_and_hash_with_cells(self): + """Regression test for GH-143831.""" + class A: + def one(_) -> C1: + """One cell.""" + + one_f = ForwardRef("C1", owner=one) + one_f_ga1 = get_annotations(one, format=Format.FORWARDREF)["return"] + one_f_ga2 = get_annotations(one, format=Format.FORWARDREF)["return"] + self.assertIsInstance(one_f_ga1.__cell__, types.CellType) + self.assertIs(one_f_ga1.__cell__, one_f_ga2.__cell__) + + def two(_) -> C1 | C2: + """Two cells.""" + + two_f_ga1 = get_annotations(two, format=Format.FORWARDREF)["return"] + two_f_ga2 = get_annotations(two, format=Format.FORWARDREF)["return"] + self.assertIsNot(two_f_ga1.__cell__, two_f_ga2.__cell__) + self.assertIsInstance(two_f_ga1.__cell__, dict) + self.assertIsInstance(two_f_ga2.__cell__, dict) + + type C1 = None + type C2 = None + + self.assertNotEqual(A.one_f, A.one_f_ga1) + self.assertNotEqual(hash(A.one_f), hash(A.one_f_ga1)) + + self.assertEqual(A.one_f_ga1, A.one_f_ga2) + self.assertEqual(hash(A.one_f_ga1), hash(A.one_f_ga2)) + + self.assertEqual(A.two_f_ga1, A.two_f_ga2) + self.assertEqual(hash(A.two_f_ga1), hash(A.two_f_ga2)) + def test_forward_equality_namespace(self): def namespace1(): a = ForwardRef("A") @@ -1364,6 +2030,23 @@ def test_forward_repr(self): repr(List[ForwardRef("int", module="mod")]), "typing.List[ForwardRef('int', module='mod')]", ) + self.assertEqual( + repr(List[ForwardRef("int", module="mod", is_class=True)]), + "typing.List[ForwardRef('int', module='mod', is_class=True)]", + ) + self.assertEqual( + repr(List[ForwardRef("int", owner="class")]), + "typing.List[ForwardRef('int', owner='class')]", + ) + + def test_forward_repr_extra_names(self): + def f(a: undefined | str): ... + + annos = get_annotations(f, format=Format.FORWARDREF) + + self.assertRegex( + repr(annos['a']), r"ForwardRef\('undefined \| str'.*\)" + ) def test_forward_recursion_actually(self): def namespace1(): @@ -1441,6 +2124,17 @@ def test_evaluate_string_format(self): fr = ForwardRef("set[Any]") self.assertEqual(fr.evaluate(format=Format.STRING), "set[Any]") + def test_evaluate_string_format_extra_names(self): + # Test that internal extra_names are replaced when evaluating as strings + def f(a: unknown | str | int | list[str] | tuple[int, ...]): ... + + fr = get_annotations(f, format=Format.FORWARDREF)['a'] + # Test the cache is not populated before access + self.assertIsNone(fr.__resolved_str_cache__) + + self.assertEqual(fr.evaluate(format=Format.STRING), "unknown | str | int | list[str] | tuple[int, ...]") + self.assertEqual(fr.__resolved_str_cache__, "unknown | str | int | list[str] | tuple[int, ...]") + def test_evaluate_forwardref_format(self): fr = ForwardRef("undef") evaluated = fr.evaluate(format=Format.FORWARDREF) @@ -1469,6 +2163,19 @@ def test_evaluate_forwardref_format(self): support.EqualToForwardRef('"a" + 1'), ) + def test_evaluate_notimplemented_format(self): + class C: + x: alias + + fwdref = get_annotations(C, format=Format.FORWARDREF)["x"] + + with self.assertRaises(NotImplementedError): + fwdref.evaluate(format=Format.VALUE_WITH_FAKE_GLOBALS) + + with self.assertRaises(NotImplementedError): + # Some other unsupported value + fwdref.evaluate(format=7) + def test_evaluate_with_type_params(self): class Gen[T]: alias = int @@ -1495,6 +2202,21 @@ class Gen[T]: ForwardRef("alias").evaluate(owner=Gen, locals={"alias": str}), str ) + def test_evaluate_with_type_params_and_scope_conflict(self): + for is_class in (False, True): + with self.subTest(is_class=is_class): + fwdref1 = ForwardRef("TypeParamsAlias1", owner=TypeParamsSample, is_class=is_class) + fwdref2 = ForwardRef("TypeParamsAlias2", owner=TypeParamsSample, is_class=is_class) + + self.assertIs( + fwdref1.evaluate(), + TypeParamsSample.__type_params__[0], + ) + self.assertIs( + fwdref2.evaluate(), + TypeParamsSample.TypeParamsAlias2, + ) + def test_fwdref_with_module(self): self.assertIs(ForwardRef("Format", module="annotationlib").evaluate(), Format) self.assertIs( @@ -1548,9 +2270,37 @@ def test_name_lookup_without_eval(self): with support.swap_attr(builtins, "int", dict): self.assertIs(ForwardRef("int").evaluate(), dict) - with self.assertRaises(NameError): + with self.assertRaises(NameError, msg="name 'doesntexist' is not defined") as exc: ForwardRef("doesntexist").evaluate() + self.assertEqual(exc.exception.name, "doesntexist") + + def test_evaluate_undefined_generic(self): + # Test the codepath where have to eval() with undefined variables. + class C: + x: alias[int, undef] + + generic = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + format=Format.FORWARDREF, + globals={"alias": dict} + ) + self.assertNotIsInstance(generic, ForwardRef) + self.assertIs(generic.__origin__, dict) + self.assertEqual(len(generic.__args__), 2) + self.assertIs(generic.__args__[0], int) + self.assertIsInstance(generic.__args__[1], ForwardRef) + + generic = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + format=Format.FORWARDREF, + globals={"alias": Union}, + locals={"alias": dict} + ) + self.assertNotIsInstance(generic, ForwardRef) + self.assertIs(generic.__origin__, dict) + self.assertEqual(len(generic.__args__), 2) + self.assertIs(generic.__args__[0], int) + self.assertIsInstance(generic.__args__[1], ForwardRef) + def test_fwdref_invalid_syntax(self): fr = ForwardRef("if") with self.assertRaises(SyntaxError): @@ -1559,6 +2309,56 @@ def test_fwdref_invalid_syntax(self): with self.assertRaises(SyntaxError): fr.evaluate() + def test_re_evaluate_generics(self): + global global_alias + + # If we've already run this test before, + # ensure the variable is still undefined + if "global_alias" in globals(): + del global_alias + + class C: + x: global_alias[int] + + # Evaluate the ForwardRef once + evaluated = get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + format=Format.FORWARDREF + ) + + # Now define the global and ensure that the ForwardRef evaluates + global_alias = list + self.assertEqual(evaluated.evaluate(), list[int]) + + def test_fwdref_evaluate_argument_mutation(self): + class C[T]: + nonlocal alias + x: alias[T] + + # Mutable arguments + globals_ = globals() + globals_copy = globals_.copy() + locals_ = locals() + locals_copy = locals_.copy() + + # Evaluate the ForwardRef, ensuring we use __cell__ and type params + get_annotations(C, format=Format.FORWARDREF)["x"].evaluate( + globals=globals_, + locals=locals_, + type_params=C.__type_params__, + format=Format.FORWARDREF, + ) + + # Check if the passed in mutable arguments equal the originals + self.assertEqual(globals_, globals_copy) + self.assertEqual(locals_, locals_copy) + + alias = list + + def test_fwdref_final_class(self): + with self.assertRaises(TypeError): + class C(ForwardRef): + pass + class TestAnnotationLib(unittest.TestCase): def test__all__(self): diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 5a6be1180c1a3ec..627e395c49b194f 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -18,7 +18,12 @@ import warnings from enum import StrEnum -from test.support import captured_stderr +from test.support import ( + captured_stderr, + force_not_colorized, + force_not_colorized_test_class, + swap_attr, +) from test.support import import_helper from test.support import os_helper from test.support import script_helper @@ -76,6 +81,27 @@ def test_skip_invalid_stdout(self): self.assertRegex(mocked_stderr.getvalue(), r'usage:') +class TestArgumentParserPickleable(unittest.TestCase): + + @force_not_colorized + def test_pickle_roundtrip(self): + import pickle + parser = argparse.ArgumentParser(exit_on_error=False) + parser.add_argument('--foo', type=int, default=42) + parser.add_argument('bar', nargs='?', default='baz') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + # Try to pickle and unpickle the parser + parser2 = pickle.loads(pickle.dumps(parser, protocol=proto)) + # Check that the round-tripped parser still works + ns = parser2.parse_args(['--foo', '123', 'quux']) + self.assertEqual(ns.foo, 123) + self.assertEqual(ns.bar, 'quux') + ns2 = parser2.parse_args([]) + self.assertEqual(ns2.foo, 42) + self.assertEqual(ns2.bar, 'baz') + + class TestCase(unittest.TestCase): def setUp(self): @@ -1007,6 +1033,7 @@ def test_parse_enum_value(self): args = parser.parse_args(['--color', 'red']) self.assertEqual(args.color, self.Color.RED) + @force_not_colorized def test_help_message_contains_enum_choices(self): parser = argparse.ArgumentParser() parser.add_argument('--color', choices=self.Color, help='Choose a color') @@ -1018,7 +1045,7 @@ def test_invalid_enum_value_raises_error(self): parser.add_argument('--color', choices=self.Color) self.assertRaisesRegex( argparse.ArgumentError, - r"invalid choice: 'yellow' \(choose from red, green, blue\)", + r"invalid choice: 'yellow' \(choose from 'red', 'green', 'blue'\)", parser.parse_args, ['--color', 'yellow'], ) @@ -2286,7 +2313,7 @@ def test_wrong_argument_error_with_suggestions(self): with self.assertRaises(ArgumentParserError) as excinfo: parser.parse_args(('bazz',)) self.assertIn( - "error: argument foo: invalid choice: 'bazz', maybe you meant 'baz'? (choose from bar, baz)", + "error: argument foo: invalid choice: 'bazz', maybe you meant 'baz'? (choose from 'bar', 'baz')", excinfo.exception.stderr ) @@ -2296,7 +2323,7 @@ def test_wrong_argument_error_no_suggestions(self): with self.assertRaises(ArgumentParserError) as excinfo: parser.parse_args(('bazz',)) self.assertIn( - "error: argument foo: invalid choice: 'bazz' (choose from bar, baz)", + "error: argument foo: invalid choice: 'bazz' (choose from 'bar', 'baz')", excinfo.exception.stderr, ) @@ -2309,7 +2336,7 @@ def test_wrong_argument_subparsers_with_suggestions(self): parser.parse_args(('baz',)) self.assertIn( "error: argument {foo,bar}: invalid choice: 'baz', maybe you meant" - " 'bar'? (choose from foo, bar)", + " 'bar'? (choose from 'foo', 'bar')", excinfo.exception.stderr, ) @@ -2321,7 +2348,7 @@ def test_wrong_argument_subparsers_no_suggestions(self): with self.assertRaises(ArgumentParserError) as excinfo: parser.parse_args(('baz',)) self.assertIn( - "error: argument {foo,bar}: invalid choice: 'baz' (choose from foo, bar)", + "error: argument {foo,bar}: invalid choice: 'baz' (choose from 'foo', 'bar')", excinfo.exception.stderr, ) @@ -2331,7 +2358,7 @@ def test_wrong_argument_no_suggestion_implicit(self): with self.assertRaises(ArgumentParserError) as excinfo: parser.parse_args(('bazz',)) self.assertIn( - "error: argument foo: invalid choice: 'bazz' (choose from bar, baz)", + "error: argument foo: invalid choice: 'bazz' (choose from 'bar', 'baz')", excinfo.exception.stderr, ) @@ -2351,7 +2378,7 @@ def test_suggestions_choices_int(self): with self.assertRaises(ArgumentParserError) as excinfo: parser.parse_args(('3',)) self.assertIn( - "error: argument foo: invalid choice: '3' (choose from 1, 2)", + "error: argument foo: invalid choice: '3' (choose from '1', '2')", excinfo.exception.stderr, ) @@ -2361,7 +2388,7 @@ def test_suggestions_choices_mixed_types(self): with self.assertRaises(ArgumentParserError) as excinfo: parser.parse_args(('3',)) self.assertIn( - "error: argument foo: invalid choice: '3' (choose from 1, 2)", + "error: argument foo: invalid choice: '3' (choose from '1', '2')", excinfo.exception.stderr, ) @@ -2403,6 +2430,7 @@ def test_modified_invalid_action(self): # Subparsers tests # ================ +@force_not_colorized_test_class class TestAddSubparsers(TestCase): """Test the add_subparsers method""" @@ -3009,6 +3037,7 @@ def test_nested_argument_group(self): # Parent parser tests # =================== +@force_not_colorized_test_class class TestParentParsers(TestCase): """Tests that parsers can be created with parent parsers""" @@ -3216,6 +3245,7 @@ def test_mutex_groups_parents(self): # Mutually exclusive group tests # ============================== +@force_not_colorized_test_class class TestMutuallyExclusiveGroupErrors(TestCase): def test_invalid_add_argument_group(self): @@ -3298,12 +3328,11 @@ def test_help_subparser_all_mutually_exclusive_group_members_suppressed(self): ''' self.assertEqual(cmd_foo.format_help(), textwrap.dedent(expected)) - def test_empty_group(self): + def test_usage_empty_group(self): # See issue 26952 - parser = argparse.ArgumentParser() + parser = ErrorRaisingArgumentParser(prog='PROG') group = parser.add_mutually_exclusive_group() - with self.assertRaises(ValueError): - parser.parse_args(['-h']) + self.assertEqual(parser.format_usage(), 'usage: PROG [-h]\n') def test_nested_mutex_groups(self): parser = argparse.ArgumentParser(prog='PROG') @@ -3344,21 +3373,25 @@ def test_successes_when_required(self): actual_ns = parse_args(args_string.split()) self.assertEqual(actual_ns, expected_ns) + @force_not_colorized def test_usage_when_not_required(self): format_usage = self.get_parser(required=False).format_usage expected_usage = self.usage_when_not_required self.assertEqual(format_usage(), textwrap.dedent(expected_usage)) + @force_not_colorized def test_usage_when_required(self): format_usage = self.get_parser(required=True).format_usage expected_usage = self.usage_when_required self.assertEqual(format_usage(), textwrap.dedent(expected_usage)) + @force_not_colorized def test_help_when_not_required(self): format_help = self.get_parser(required=False).format_help help = self.usage_when_not_required + self.help self.assertEqual(format_help(), textwrap.dedent(help)) + @force_not_colorized def test_help_when_required(self): format_help = self.get_parser(required=True).format_help help = self.usage_when_required + self.help @@ -3567,25 +3600,29 @@ def get_parser(self, required): group.add_argument('-b', action='store_true', help='b help') parser.add_argument('-y', action='store_true', help='y help') group.add_argument('-c', action='store_true', help='c help') + parser.add_argument('-z', action='store_true', help='z help') return parser failures = ['-a -b', '-b -c', '-a -c', '-a -b -c'] successes = [ - ('-a', NS(a=True, b=False, c=False, x=False, y=False)), - ('-b', NS(a=False, b=True, c=False, x=False, y=False)), - ('-c', NS(a=False, b=False, c=True, x=False, y=False)), - ('-a -x', NS(a=True, b=False, c=False, x=True, y=False)), - ('-y -b', NS(a=False, b=True, c=False, x=False, y=True)), - ('-x -y -c', NS(a=False, b=False, c=True, x=True, y=True)), + ('-a', NS(a=True, b=False, c=False, x=False, y=False, z=False)), + ('-b', NS(a=False, b=True, c=False, x=False, y=False, z=False)), + ('-c', NS(a=False, b=False, c=True, x=False, y=False, z=False)), + ('-a -x', NS(a=True, b=False, c=False, x=True, y=False, z=False)), + ('-y -b', NS(a=False, b=True, c=False, x=False, y=True, z=False)), + ('-x -y -c', NS(a=False, b=False, c=True, x=True, y=True, z=False)), ] successes_when_not_required = [ - ('', NS(a=False, b=False, c=False, x=False, y=False)), - ('-x', NS(a=False, b=False, c=False, x=True, y=False)), - ('-y', NS(a=False, b=False, c=False, x=False, y=True)), + ('', NS(a=False, b=False, c=False, x=False, y=False, z=False)), + ('-x', NS(a=False, b=False, c=False, x=True, y=False, z=False)), + ('-y', NS(a=False, b=False, c=False, x=False, y=True, z=False)), ] - usage_when_required = usage_when_not_required = '''\ - usage: PROG [-h] [-x] [-a] [-b] [-y] [-c] + usage_when_not_required = '''\ + usage: PROG [-h] [-x] [-a | -b | -c] [-y] [-z] + ''' + usage_when_required = '''\ + usage: PROG [-h] [-x] (-a | -b | -c) [-y] [-z] ''' help = '''\ @@ -3596,6 +3633,7 @@ def get_parser(self, required): -b b help -y y help -c c help + -z z help ''' @@ -3649,23 +3687,27 @@ def get_parser(self, required): group.add_argument('a', nargs='?', help='a help') group.add_argument('-b', action='store_true', help='b help') group.add_argument('-c', action='store_true', help='c help') + parser.add_argument('-z', action='store_true', help='z help') return parser failures = ['X A -b', '-b -c', '-c X A'] successes = [ - ('X A', NS(a='A', b=False, c=False, x='X', y=False)), - ('X -b', NS(a=None, b=True, c=False, x='X', y=False)), - ('X -c', NS(a=None, b=False, c=True, x='X', y=False)), - ('X A -y', NS(a='A', b=False, c=False, x='X', y=True)), - ('X -y -b', NS(a=None, b=True, c=False, x='X', y=True)), + ('X A', NS(a='A', b=False, c=False, x='X', y=False, z=False)), + ('X -b', NS(a=None, b=True, c=False, x='X', y=False, z=False)), + ('X -c', NS(a=None, b=False, c=True, x='X', y=False, z=False)), + ('X A -y', NS(a='A', b=False, c=False, x='X', y=True, z=False)), + ('X -y -b', NS(a=None, b=True, c=False, x='X', y=True, z=False)), ] successes_when_not_required = [ - ('X', NS(a=None, b=False, c=False, x='X', y=False)), - ('X -y', NS(a=None, b=False, c=False, x='X', y=True)), + ('X', NS(a=None, b=False, c=False, x='X', y=False, z=False)), + ('X -y', NS(a=None, b=False, c=False, x='X', y=True, z=False)), ] - usage_when_required = usage_when_not_required = '''\ - usage: PROG [-h] [-y] [-b] [-c] x [a] + usage_when_not_required = '''\ + usage: PROG [-h] [-y] [-z] x [-b | -c | a] + ''' + usage_when_required = '''\ + usage: PROG [-h] [-y] [-z] x (-b | -c | a) ''' help = '''\ @@ -3678,6 +3720,7 @@ def get_parser(self, required): -y y help -b b help -c c help + -z z help ''' @@ -4030,11 +4073,13 @@ def _test(self, tester, parser_text): tester.maxDiff = None tester.assertEqual(expected_text, parser_text) + @force_not_colorized def test_format(self, tester): parser = self._get_parser(tester) format = getattr(parser, 'format_%s' % self.func_suffix) self._test(tester, format()) + @force_not_colorized def test_print(self, tester): parser = self._get_parser(tester) print_ = getattr(parser, 'print_%s' % self.func_suffix) @@ -4047,6 +4092,7 @@ def test_print(self, tester): setattr(sys, self.std_name, old_stream) self._test(tester, parser_text) + @force_not_colorized def test_print_file(self, tester): parser = self._get_parser(tester) print_ = getattr(parser, 'print_%s' % self.func_suffix) @@ -4788,6 +4834,7 @@ class TestHelpUsageMetavarsSpacesParentheses(HelpTestCase): version = '' +@force_not_colorized_test_class class TestHelpUsageNoWhitespaceCrash(TestCase): def test_all_suppressed_mutex_followed_by_long_arg(self): @@ -4868,6 +4915,25 @@ def test_long_mutex_groups_wrap(self): ''') self.assertEqual(parser.format_usage(), usage) + def test_mutex_groups_with_mixed_optionals_positionals_wrap(self): + # https://github.com/python/cpython/issues/75949 + # Mutually exclusive groups containing both optionals and positionals + # should preserve pipe separators when the usage line wraps. + parser = argparse.ArgumentParser(prog='PROG') + g = parser.add_mutually_exclusive_group() + g.add_argument('-v', '--verbose', action='store_true') + g.add_argument('-q', '--quiet', action='store_true') + g.add_argument('-x', '--extra-long-option-name', nargs='?') + g.add_argument('-y', '--yet-another-long-option', nargs='?') + g.add_argument('positional', nargs='?') + + usage = textwrap.dedent('''\ + usage: PROG [-h] + [-v | -q | -x [EXTRA_LONG_OPTION_NAME] | + -y [YET_ANOTHER_LONG_OPTION] | positional] + ''') + self.assertEqual(parser.format_usage(), usage) + class TestHelpVariableExpansion(HelpTestCase): """Test that variables are expanded properly in help messages""" @@ -5283,6 +5349,7 @@ class TestHelpArgumentDefaults(HelpTestCase): argument_signatures = [ Sig('--foo', help='foo help - oh and by the way, %(default)s'), Sig('--bar', action='store_true', help='bar help'), + Sig('--required', required=True, help='some help'), Sig('--taz', action=argparse.BooleanOptionalAction, help='Whether to taz it', default=True), Sig('--corge', action=argparse.BooleanOptionalAction, @@ -5296,8 +5363,8 @@ class TestHelpArgumentDefaults(HelpTestCase): [Sig('--baz', type=int, default=42, help='baz help')]), ] usage = '''\ - usage: PROG [-h] [--foo FOO] [--bar] [--taz | --no-taz] [--corge | --no-corge] - [--quux QUUX] [--baz BAZ] + usage: PROG [-h] [--foo FOO] [--bar] --required REQUIRED [--taz | --no-taz] + [--corge | --no-corge] [--quux QUUX] [--baz BAZ] spam [badger] ''' help = usage + '''\ @@ -5312,6 +5379,7 @@ class TestHelpArgumentDefaults(HelpTestCase): -h, --help show this help message and exit --foo FOO foo help - oh and by the way, None --bar bar help (default: False) + --required REQUIRED some help --taz, --no-taz Whether to taz it (default: True) --corge, --no-corge Whether to corge it --quux QUUX Set the quux (default: 42) @@ -5469,11 +5537,61 @@ def custom_type(string): version = '' -class TestHelpUsageLongSubparserCommand(TestCase): - """Test that subparser commands are formatted correctly in help""" +@force_not_colorized_test_class +class TestHelpCustomHelpFormatter(TestCase): maxDiff = None - def test_parent_help(self): + def test_custom_formatter_function(self): + def custom_formatter(prog): + return argparse.RawTextHelpFormatter(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog='PROG', + prefix_chars='-+', + formatter_class=custom_formatter + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent('''\ + usage: PROG [-h] [+f FOO] spam + + positional arguments: + spam spam help + + options: + -h, --help show this help message and exit + +f, ++foo FOO foo help + ''')) + + def test_custom_formatter_class(self): + class CustomFormatter(argparse.RawTextHelpFormatter): + def __init__(self, prog): + super().__init__(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog='PROG', + prefix_chars='-+', + formatter_class=CustomFormatter + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent('''\ + usage: PROG [-h] [+f FOO] spam + + positional arguments: + spam spam help + + options: + -h, --help show this help message and exit + +f, ++foo FOO foo help + ''')) + + def test_usage_long_subparser_command(self): + """Test that subparser commands are formatted correctly in help""" def custom_formatter(prog): return argparse.RawTextHelpFormatter(prog, max_help_position=50) @@ -5716,6 +5834,7 @@ def test_conflict_error(self): self.assertRaises(argparse.ArgumentError, parser.add_argument, '--spam') + @force_not_colorized def test_resolve_error(self): get_parser = argparse.ArgumentParser parser = get_parser(prog='PROG', conflict_handler='resolve') @@ -5763,6 +5882,7 @@ def test_subparser_conflict(self): # Help and Version option tests # ============================= +@force_not_colorized_test_class class TestOptionalsHelpVersionActions(TestCase): """Test the help and version actions""" @@ -5982,6 +6102,7 @@ def test_argument_error(self): class TestArgumentTypeError(TestCase): + @force_not_colorized def test_argument_type_error(self): def spam(string): @@ -6756,7 +6877,7 @@ class TestImportStar(TestCase): def test(self): for name in argparse.__all__: - self.assertTrue(hasattr(argparse, name)) + self.assertHasAttr(argparse, name) def test_all_exports_everything_but_modules(self): items = [ @@ -6780,6 +6901,7 @@ def setUp(self): metavar = '' self.parser.add_argument('--proxy', metavar=metavar) + @force_not_colorized def test_help_with_metavar(self): help_text = self.parser.format_help() self.assertEqual(help_text, textwrap.dedent('''\ @@ -6945,6 +7067,7 @@ def test_os_error(self): self.parser.parse_args, ['@no-such-file']) +@force_not_colorized_test_class class TestProgName(TestCase): source = textwrap.dedent('''\ import argparse @@ -6973,7 +7096,7 @@ def make_zip_script(self, script_name, name_in_zip=None): def check_usage(self, expected, *args, **kwargs): res = script_helper.assert_python_ok('-Xutf8', *args, '-h', **kwargs) - self.assertEqual(res.out.splitlines()[0].decode(), + self.assertEqual(os.fsdecode(res.out.splitlines()[0]), f'usage: {expected} [-h]') def test_script(self, compiled=False): @@ -7053,11 +7176,13 @@ def test_translations(self): class TestColorized(TestCase): + maxDiff = None def setUp(self): super().setUp() # Ensure color even if ran with NO_COLOR=1 - _colorize.can_colorize = lambda *args, **kwargs: True + self.enterContext(swap_attr(_colorize, 'can_colorize', + lambda *args, **kwargs: True)) self.theme = _colorize.get_theme(force_color=True).argparse def test_argparse_color(self): @@ -7182,7 +7307,28 @@ def test_argparse_color(self): ), ) - def test_argparse_color_usage(self): + def test_argparse_color_mutually_exclusive_group_usage(self): + parser = argparse.ArgumentParser(color=True, prog="PROG") + group = parser.add_mutually_exclusive_group() + group.add_argument('--foo', action='store_true', help='FOO') + group.add_argument('--spam', help='SPAM') + group.add_argument('badger', nargs='*', help='BADGER') + + prog = self.theme.prog + heading = self.theme.heading + long = self.theme.summary_long_option + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + reset = self.theme.reset + + self.assertEqual(parser.format_usage(), + f"{heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] " + f"[{long}--foo{reset} | " + f"{long}--spam {label}SPAM{reset} | " + f"{pos}badger ...{reset}]\n") + + def test_argparse_color_custom_usage(self): # Arrange parser = argparse.ArgumentParser( add_help=False, @@ -7211,6 +7357,90 @@ def test_argparse_color_usage(self): ), ) + def test_custom_formatter_function(self): + def custom_formatter(prog): + return argparse.RawTextHelpFormatter(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog="PROG", + prefix_chars="-+", + formatter_class=custom_formatter, + color=True, + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + prog = self.theme.prog + heading = self.theme.heading + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + long_b = self.theme.long_option + short_b = self.theme.short_option + label_b = self.theme.label + pos_b = self.theme.action + reset = self.theme.reset + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent(f'''\ + {heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] [{short}+f {label}FOO{reset}] {pos}spam{reset} + + {heading}positional arguments:{reset} + {pos_b}spam{reset} spam help + + {heading}options:{reset} + {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit + {short_b}+f{reset}, {long_b}++foo{reset} {label_b}FOO{reset} foo help + ''')) + + def test_custom_formatter_class(self): + class CustomFormatter(argparse.RawTextHelpFormatter): + def __init__(self, prog): + super().__init__(prog, indent_increment=5) + + parser = argparse.ArgumentParser( + prog="PROG", + prefix_chars="-+", + formatter_class=CustomFormatter, + color=True, + ) + parser.add_argument('+f', '++foo', help="foo help") + parser.add_argument('spam', help="spam help") + + prog = self.theme.prog + heading = self.theme.heading + short = self.theme.summary_short_option + label = self.theme.summary_label + pos = self.theme.summary_action + long_b = self.theme.long_option + short_b = self.theme.short_option + label_b = self.theme.label + pos_b = self.theme.action + reset = self.theme.reset + + parser_help = parser.format_help() + self.assertEqual(parser_help, textwrap.dedent(f'''\ + {heading}usage: {reset}{prog}PROG{reset} [{short}-h{reset}] [{short}+f {label}FOO{reset}] {pos}spam{reset} + + {heading}positional arguments:{reset} + {pos_b}spam{reset} spam help + + {heading}options:{reset} + {short_b}-h{reset}, {long_b}--help{reset} show this help message and exit + {short_b}+f{reset}, {long_b}++foo{reset} {label_b}FOO{reset} foo help + ''')) + + def test_subparser_prog_is_stored_without_color(self): + parser = argparse.ArgumentParser(prog='complex', color=True) + sub = parser.add_subparsers(dest='command') + demo_parser = sub.add_parser('demo') + + self.assertNotIn('\x1b[', demo_parser.prog) + + demo_parser.color = False + help_text = demo_parser.format_help() + self.assertNotIn('\x1b[', help_text) + def tearDownModule(): # Remove global references to avoid looking like we have refleaks. diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 58ea89c4fac833b..8cf669935f923e5 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -5,9 +5,10 @@ import collections.abc import unittest from test import support -from test.support import import_helper +from test.support import import_helper, script_helper from test.support import os_helper from test.support import _2G +from test.support import subTests import weakref import pickle import operator @@ -34,6 +35,11 @@ def __init__(self, typecode, newarg=None): class MiscTest(unittest.TestCase): + def test_array_type_importable(self): + from array import ArrayType + + self.assertIs(array.array, ArrayType) + def test_array_is_sequence(self): self.assertIsInstance(array.array("B"), collections.abc.MutableSequence) self.assertIsInstance(array.array("B"), collections.abc.Reversible) @@ -44,6 +50,23 @@ def test_bad_constructor(self): self.assertRaises(TypeError, array.array, 'xx') self.assertRaises(ValueError, array.array, 'x') + @support.cpython_only + def test_does_not_crash_on_broken_imports(self): + # gh-153210 + code = """if 1: + import collections.abc + + del collections.abc.MutableSequence + + try: + import array # it used to crash before + except AttributeError: + pass + else: + raise AssertionError('AttributeError was not raised') + """ + script_helper.assert_python_ok('-c', code) + @support.cpython_only def test_disallow_instantiation(self): my_array = array.array("I") @@ -1255,6 +1278,14 @@ def test_typecode_u_deprecation(self): with self.assertWarns(DeprecationWarning): array.array("u") + def test_empty_string_mem_leak_gh140474(self): + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + for _ in range(1000): + a = array.array('u', '') + self.assertEqual(len(a), 0) + self.assertEqual(a.typecode, 'u') + class UCS4Test(UnicodeTest): typecode = 'w' @@ -1672,6 +1703,45 @@ def test_gh_128961(self): it.__setstate__(0) self.assertRaises(StopIteration, next, it) + # Tests for NULL pointer dereference in array.__setitem__ + # when the index conversion mutates the array. + # See: https://github.com/python/cpython/issues/142555. + + @subTests("dtype", ["b", "B", "h", "H", "i", "l", "q", "I", "L", "Q"]) + def test_setitem_use_after_clear_with_int_data(self, dtype): + victim = array.array(dtype, list(range(64))) + + class Index: + def __index__(self): + victim.clear() + return 0 + + self.assertRaises(IndexError, victim.__setitem__, 1, Index()) + self.assertEqual(len(victim), 0) + + def test_setitem_use_after_shrink_with_int_data(self): + victim = array.array('b', [1, 2, 3]) + + class Index: + def __index__(self): + victim.pop() + victim.pop() + return 0 + + self.assertRaises(IndexError, victim.__setitem__, 1, Index()) + + @subTests("dtype", ["f", "d"]) + def test_setitem_use_after_clear_with_float_data(self, dtype): + victim = array.array(dtype, [1.0, 2.0, 3.0]) + + class Float: + def __float__(self): + victim.clear() + return 0.0 + + self.assertRaises(IndexError, victim.__setitem__, 1, Float()) + self.assertEqual(len(victim), 0) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py index 09cf3186e05cc17..3f607a00d2508e0 100644 --- a/Lib/test/test_ast/test_ast.py +++ b/Lib/test/test_ast/test_ast.py @@ -24,7 +24,9 @@ from test import support from test.support import os_helper -from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow +from test.support import (skip_emscripten_stack_overflow, + skip_wasi_stack_overflow, + skip_if_unlimited_stack_size, skip_if_huge_c_stack) from test.support.ast_helper import ASTTestMixin from test.support.import_helper import ensure_lazy_imports from test.test_ast.utils import to_tuple @@ -220,6 +222,131 @@ def test_negative_locations_for_compile(self): # This also must not crash: ast.parse(tree, optimize=2) + def test_docstring_optimization_single_node(self): + # https://github.com/python/cpython/issues/137308 + class_example1 = textwrap.dedent(''' + class A: + """Docstring""" + ''') + class_example2 = textwrap.dedent(''' + class A: + """ + Docstring""" + ''') + def_example1 = textwrap.dedent(''' + def some(): + """Docstring""" + ''') + def_example2 = textwrap.dedent(''' + def some(): + """Docstring + """ + ''') + async_def_example1 = textwrap.dedent(''' + async def some(): + """Docstring""" + ''') + async_def_example2 = textwrap.dedent(''' + async def some(): + """ + Docstring + """ + ''') + for code in [ + class_example1, + class_example2, + def_example1, + def_example2, + async_def_example1, + async_def_example2, + ]: + for opt_level in [0, 1, 2]: + with self.subTest(code=code, opt_level=opt_level): + mod = ast.parse(code, optimize=opt_level) + self.assertEqual(len(mod.body[0].body), 1) + if opt_level == 2: + pass_stmt = mod.body[0].body[0] + self.assertIsInstance(pass_stmt, ast.Pass) + self.assertEqual( + vars(pass_stmt), + { + 'lineno': 3, + 'col_offset': 4, + 'end_lineno': 3, + 'end_col_offset': 8, + }, + ) + else: + self.assertIsInstance(mod.body[0].body[0], ast.Expr) + self.assertIsInstance( + mod.body[0].body[0].value, + ast.Constant, + ) + + compile(code, "a", "exec") + compile(code, "a", "exec", optimize=opt_level) + compile(mod, "a", "exec") + compile(mod, "a", "exec", optimize=opt_level) + + def test_docstring_optimization_multiple_nodes(self): + # https://github.com/python/cpython/issues/137308 + class_example = textwrap.dedent( + """ + class A: + ''' + Docstring + ''' + x = 1 + """ + ) + + def_example = textwrap.dedent( + """ + def some(): + ''' + Docstring + + ''' + x = 1 + """ + ) + + async_def_example = textwrap.dedent( + """ + async def some(): + + '''Docstring + + ''' + x = 1 + """ + ) + + for code in [ + class_example, + def_example, + async_def_example, + ]: + for opt_level in [0, 1, 2]: + with self.subTest(code=code, opt_level=opt_level): + mod = ast.parse(code, optimize=opt_level) + if opt_level == 2: + self.assertNotIsInstance( + mod.body[0].body[0], + (ast.Pass, ast.Expr), + ) + else: + self.assertIsInstance(mod.body[0].body[0], ast.Expr) + self.assertIsInstance( + mod.body[0].body[0].value, + ast.Constant, + ) + + compile(code, "a", "exec") + compile(code, "a", "exec", optimize=opt_level) + compile(mod, "a", "exec") + compile(mod, "a", "exec", optimize=opt_level) + def test_slice(self): slc = ast.parse("x[::]").body[0].value.slice self.assertIsNone(slc.upper) @@ -275,12 +402,12 @@ def test_alias(self): self.assertEqual(alias.end_col_offset, 17) def test_base_classes(self): - self.assertTrue(issubclass(ast.For, ast.stmt)) - self.assertTrue(issubclass(ast.Name, ast.expr)) - self.assertTrue(issubclass(ast.stmt, ast.AST)) - self.assertTrue(issubclass(ast.expr, ast.AST)) - self.assertTrue(issubclass(ast.comprehension, ast.AST)) - self.assertTrue(issubclass(ast.Gt, ast.AST)) + self.assertIsSubclass(ast.For, ast.stmt) + self.assertIsSubclass(ast.Name, ast.expr) + self.assertIsSubclass(ast.stmt, ast.AST) + self.assertIsSubclass(ast.expr, ast.AST) + self.assertIsSubclass(ast.comprehension, ast.AST) + self.assertIsSubclass(ast.Gt, ast.AST) def test_field_attr_existence(self): for name, item in ast.__dict__.items(): @@ -821,6 +948,17 @@ def test_constant_as_name(self): with self.assertRaisesRegex(ValueError, f"identifier field can't represent '{constant}' constant"): compile(expr, "", "eval") + def test_constant_as_unicode_name(self): + constants = [ + ("True", b"Tru\xe1\xb5\x89"), + ("False", b"Fal\xc5\xbfe"), + ("None", b"N\xc2\xbane"), + ] + for constant in constants: + with self.assertRaisesRegex(ValueError, + f"identifier field can't represent '{constant[0]}' constant"): + ast.parse(constant[1], mode="eval") + def test_precedence_enum(self): class _Precedence(enum.IntEnum): """Precedence table that originated from python grammar.""" @@ -852,10 +990,12 @@ def next(self): enum._test_simple_enum(_Precedence, _ast_unparse._Precedence) @support.cpython_only + @support.skip_if_huge_c_stack(100_000 if sys.platform == "android" else 500_000) @skip_wasi_stack_overflow() @skip_emscripten_stack_overflow() def test_ast_recursion_limit(self): - crash_depth = 500_000 + # Android test devices have less memory. + crash_depth = 100_000 if sys.platform == "android" else 500_000 success_depth = 200 if _testinternalcapi is not None: remaining = _testinternalcapi.get_c_recursion_remaining() @@ -921,61 +1061,6 @@ def test_repr_large_input_crash(self): r"Exceeds the limit \(\d+ digits\)"): repr(ast.Constant(value=eval(source))) - def test_pep_765_warnings(self): - srcs = [ - textwrap.dedent(""" - def f(): - try: - pass - finally: - return 42 - """), - textwrap.dedent(""" - for x in y: - try: - pass - finally: - break - """), - textwrap.dedent(""" - for x in y: - try: - pass - finally: - continue - """), - ] - for src in srcs: - with self.assertWarnsRegex(SyntaxWarning, 'finally'): - ast.parse(src) - - def test_pep_765_no_warnings(self): - srcs = [ - textwrap.dedent(""" - try: - pass - finally: - def f(): - return 42 - """), - textwrap.dedent(""" - try: - pass - finally: - for x in y: - break - """), - textwrap.dedent(""" - try: - pass - finally: - for x in y: - continue - """), - ] - for src in srcs: - ast.parse(src) - def test_tstring(self): # Test AST structure for simple t-string tree = ast.parse('t"Hello"') @@ -988,13 +1073,6 @@ def test_tstring(self): self.assertIsInstance(tree.body[0].value.values[0], ast.Constant) self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation) - # Test AST for implicit concat of t-string with f-string - tree = ast.parse('t"Hello {name}" f"{name}"') - self.assertIsInstance(tree.body[0].value, ast.TemplateStr) - self.assertIsInstance(tree.body[0].value.values[0], ast.Constant) - self.assertIsInstance(tree.body[0].value.values[1], ast.Interpolation) - self.assertIsInstance(tree.body[0].value.values[2], ast.FormattedValue) - class CopyTests(unittest.TestCase): """Test copying and pickling AST nodes.""" @@ -1028,6 +1106,7 @@ def test_pickling(self): ast2 = pickle.loads(pickle.dumps(tree, protocol)) self.assertEqual(to_tuple(ast2), to_tuple(tree)) + @skip_if_unlimited_stack_size def test_copy_with_parents(self): # gh-120108 code = """ @@ -1090,7 +1169,7 @@ def test_copy_with_parents(self): def test_replace_interface(self): for klass in self.iter_ast_classes(): with self.subTest(klass=klass): - self.assertTrue(hasattr(klass, '__replace__')) + self.assertHasAttr(klass, '__replace__') fields = set(klass._fields) with self.subTest(klass=klass, fields=fields): @@ -1304,13 +1383,22 @@ def test_replace_reject_missing_field(self): self.assertIs(repl.id, 'y') self.assertIs(repl.ctx, context) + def test_replace_accept_missing_field_with_default(self): + node = ast.FunctionDef(name="foo", args=ast.arguments()) + self.assertIs(node.returns, None) + self.assertEqual(node.decorator_list, []) + node2 = copy.replace(node, name="bar") + self.assertEqual(node2.name, "bar") + self.assertIs(node2.returns, None) + self.assertEqual(node2.decorator_list, []) + def test_replace_reject_known_custom_instance_fields_commits(self): node = ast.parse('x').body[0].value node.extra = extra = object() # add instance 'extra' field context = node.ctx # explicit rejection of known instance fields - self.assertTrue(hasattr(node, 'extra')) + self.assertHasAttr(node, 'extra') msg = "Name.__replace__ got an unexpected keyword argument 'extra'." with self.assertRaisesRegex(TypeError, re.escape(msg)): copy.replace(node, extra=1) @@ -1333,6 +1421,13 @@ def test_replace_reject_unknown_instance_fields(self): self.assertIs(node.ctx, context) self.assertRaises(AttributeError, getattr, node, 'unknown') + def test_replace_non_str_kwarg(self): + node = ast.Name(id="x") + errmsg = "got an unexpected keyword argument ", "eval") + @skip_if_huge_c_stack() @skip_emscripten_stack_overflow() def test_recursion_indirect(self): e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) @@ -3051,7 +3172,7 @@ def test_FunctionDef(self): with self.assertWarnsRegex(DeprecationWarning, r"FunctionDef\.__init__ missing 1 required positional argument: 'name'"): node = ast.FunctionDef(args=args) - self.assertFalse(hasattr(node, "name")) + self.assertNotHasAttr(node, "name") self.assertEqual(node.decorator_list, []) node = ast.FunctionDef(name='foo', args=args) self.assertEqual(node.name, 'foo') @@ -3147,6 +3268,15 @@ class MoreFieldsThanTypes(ast.AST): self.assertEqual(obj.a, 1) self.assertEqual(obj.b, 2) + def test_malformed_fields_with_bytes(self): + class BadFields(ast.AST): + _fields = (b'\xff'*64,) + _field_types = {'a': int} + + # This should not crash + with self.assertWarnsRegex(DeprecationWarning, r"Field b'\\xff\\xff.*' .*"): + obj = BadFields() + def test_complete_field_types(self): class _AllFieldTypes(ast.AST): _fields = ('a', 'b') @@ -3160,6 +3290,27 @@ class _AllFieldTypes(ast.AST): self.assertIs(obj.a, None) self.assertEqual(obj.b, []) + def test_non_str_kwarg(self): + warn_msg = "got an unexpected keyword argument None: diff --git a/Lib/test/test_asyncio/test_sslproto.py b/Lib/test/test_asyncio/test_sslproto.py index aa248c5786f634c..3e304c166425b0b 100644 --- a/Lib/test/test_asyncio/test_sslproto.py +++ b/Lib/test/test_asyncio/test_sslproto.py @@ -21,7 +21,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) @unittest.skipIf(ssl is None, 'No ssl module') diff --git a/Lib/test/test_asyncio/test_staggered.py b/Lib/test/test_asyncio/test_staggered.py index ad34aa6da01f546..32e4817b70d7173 100644 --- a/Lib/test/test_asyncio/test_staggered.py +++ b/Lib/test/test_asyncio/test_staggered.py @@ -8,7 +8,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class StaggeredTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_streams.py b/Lib/test/test_asyncio/test_streams.py index 4fa4384346f9e9e..cae8c7c6f7c94c9 100644 --- a/Lib/test/test_asyncio/test_streams.py +++ b/Lib/test/test_asyncio/test_streams.py @@ -18,7 +18,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class StreamTests(test_utils.TestCase): @@ -819,6 +819,48 @@ async def client(addr): self.assertEqual(msg1, b"hello world 1!\n") self.assertEqual(msg2, b"hello world 2!\n") + @unittest.skipIf(ssl is None, 'No ssl module') + def test_start_tls_buffered_data(self): + # gh-142352: test start_tls() with buffered data + + async def server_handler(client_reader, client_writer): + # Wait for TLS ClientHello to be buffered before start_tls(). + await client_reader._wait_for_data('test_start_tls_buffered_data'), + self.assertTrue(client_reader._buffer) + await client_writer.start_tls(test_utils.simple_server_sslcontext()) + + line = await client_reader.readline() + self.assertEqual(line, b"ping\n") + client_writer.write(b"pong\n") + await client_writer.drain() + client_writer.close() + await client_writer.wait_closed() + + async def client(addr): + reader, writer = await asyncio.open_connection(*addr) + await writer.start_tls(test_utils.simple_client_sslcontext()) + + writer.write(b"ping\n") + await writer.drain() + line = await reader.readline() + self.assertEqual(line, b"pong\n") + writer.close() + await writer.wait_closed() + + async def run_test(): + server = await asyncio.start_server( + server_handler, socket_helper.HOSTv4, 0) + server_addr = server.sockets[0].getsockname() + + await client(server_addr) + server.close() + await server.wait_closed() + + messages = [] + self.loop.set_exception_handler(lambda loop, ctx: messages.append(ctx)) + self.loop.run_until_complete(run_test()) + self.assertEqual(messages, []) + def test_streamreader_constructor_without_loop(self): with self.assertRaisesRegex(RuntimeError, 'no current event loop'): asyncio.StreamReader() diff --git a/Lib/test/test_asyncio/test_subprocess.py b/Lib/test/test_asyncio/test_subprocess.py index 341e3e979e002b1..aab92db872d6a98 100644 --- a/Lib/test/test_asyncio/test_subprocess.py +++ b/Lib/test/test_asyncio/test_subprocess.py @@ -11,7 +11,7 @@ from asyncio import subprocess from test.test_asyncio import utils as test_utils from test import support -from test.support import os_helper +from test.support import os_helper, gc_collect if not support.has_subprocess_support: raise unittest.SkipTest("test module requires subprocess") @@ -37,7 +37,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TestSubprocessTransport(base_subprocess.BaseSubprocessTransport): @@ -219,7 +219,7 @@ def test_kill_issue43884(self): # kills the process and all its children. creationflags = CREATE_NEW_PROCESS_GROUP proc = self.loop.run_until_complete( - asyncio.create_subprocess_shell(blocking_shell_command, stdout=asyncio.subprocess.PIPE, + asyncio.create_subprocess_shell(blocking_shell_command, creationflags=creationflags) ) self.loop.run_until_complete(asyncio.sleep(1)) @@ -404,6 +404,46 @@ async def len_message(message): self.assertEqual(output.rstrip(), b'3') self.assertEqual(exitcode, 0) + def test_wait_even_if_pipe_is_open(self): + # gh-119710: Process.wait() must return once the process exits even + # if its stdout pipe is inherited by a grandchild that keeps it open, + # so the pipe never reaches EOF. Otherwise wait() hangs forever + # despite the returncode being known. + + async def run(): + # The grandchild inherits the child's stdin and stdout pipes and + # keeps both open after the child is killed. It writes "ready" + # so we know it has started, and exits once its stdin hits EOF. + code = textwrap.dedent("""\ + import subprocess, sys + subprocess.run([sys.executable, "-c", + "import sys; sys.stdout.write('ready');" + " sys.stdout.flush(); sys.stdin.read()"]) + """) + + proc = await asyncio.create_subprocess_exec( + sys.executable, "-c", code, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + try: + wait_proc = asyncio.create_task(proc.wait()) + # Wait until the grandchild holds the inherited pipes; this + # also lets the wait() task register its waiter. + await proc.stdout.readexactly(5) + proc.kill() + returncode = await asyncio.wait_for( + wait_proc, timeout=support.SHORT_TIMEOUT) + if sys.platform == 'win32': + self.assertIsInstance(returncode, int) + else: + self.assertEqual(-signal.SIGKILL, returncode) + finally: + proc.stdin.close() # let the grandchild exit + await proc.stdout.read() + + self.loop.run_until_complete(run()) + def test_empty_input(self): async def empty_input(): @@ -879,6 +919,41 @@ async def main(): self.loop.run_until_complete(main()) + def test_subprocess_read_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_read_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED, stderr=asyncio.subprocess.PIPE) + + asyncio.run(main()) + gc_collect() + + def test_subprocess_write_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_write_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec(*PROGRAM_BLOCKED, stdin=asyncio.subprocess.PIPE) + + asyncio.run(main()) + gc_collect() + + def test_subprocess_read_write_pipe_cancelled(self): + async def main(): + loop = asyncio.get_running_loop() + loop.connect_read_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + loop.connect_write_pipe = mock.AsyncMock(side_effect=asyncio.CancelledError) + with self.assertRaises(asyncio.CancelledError): + await asyncio.create_subprocess_exec( + *PROGRAM_BLOCKED, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + + asyncio.run(main()) + gc_collect() if sys.platform != 'win32': # Unix @@ -897,6 +972,45 @@ def test_watcher_implementation(self): else: self.assertIsInstance(watcher, unix_events._ThreadedChildWatcher) + @unittest.skipUnless(hasattr(os, 'waitid'), 'needs os.waitid()') + def test_send_signal_never_targets_reaped_pid(self): + # gh-127049: there must be no window between the child watcher + # reaping the child and asyncio publishing the exit in which + # send_signal() signals the freed (possibly recycled) PID. + reaped_pids = set() + stale_kills = [] + orig_waitpid = os.waitpid + orig_kill = os.kill + + def waitpid(pid, options): + res = orig_waitpid(pid, options) + if res[0] != 0: + reaped_pids.add(res[0]) + return res + + def kill(pid, sig): + if pid in reaped_pids: + stale_kills.append(pid) + return + orig_kill(pid, sig) + + async def run(): + with mock.patch('os.waitpid', waitpid), \ + mock.patch('os.kill', kill): + proc = await asyncio.create_subprocess_exec( + *PROGRAM_BLOCKED) + proc.kill() + deadline = self.loop.time() + support.SHORT_TIMEOUT + while proc.returncode is None: + if self.loop.time() > deadline: + self.fail('child exit was not published in time') + proc.kill() + await asyncio.sleep(0) + await proc.wait() + self.assertEqual(stale_kills, []) + + self.loop.run_until_complete(run()) + class SubprocessThreadedWatcherTests(SubprocessWatcherMixin, test_utils.TestCase): @@ -910,6 +1024,35 @@ def tearDown(self): unix_events.can_use_pidfd = self._original_can_use_pidfd return super().tearDown() + @unittest.skipUnless(hasattr(os, 'waitid'), 'needs os.waitid()') + def test_pid_not_reaped_before_exit_published(self): + # gh-127049: the watcher thread must wait for the child + # without reaping it: the PID must stay reserved until the + # event loop thread reaps it and publishes the exit as one + # atomic step. + async def run(): + proc = await asyncio.create_subprocess_exec( + *PROGRAM_BLOCKED) + thread = self.loop._watcher._threads.get(proc.pid) + self.assertIsNotNone(thread) + proc.kill() + # Wait for the watcher thread to observe the exit while + # the event loop cannot process the notification yet. + thread.join(support.SHORT_TIMEOUT) + self.assertFalse(thread.is_alive()) + # The exit has not been published yet... + self.assertIsNone(proc.returncode) + # ...so the child must still be an unreaped zombie and + # signalling its PID must still be safe. waitid() raises + # ChildProcessError if the PID was already reaped (and + # possibly recycled by the kernel). + os.waitid(os.P_PID, proc.pid, + os.WEXITED | os.WNOWAIT | os.WNOHANG) + proc.kill() + self.assertEqual(await proc.wait(), -signal.SIGKILL) + + self.loop.run_until_complete(run()) + @unittest.skipUnless( unix_events.can_use_pidfd(), "operating system does not support pidfds", diff --git a/Lib/test/test_asyncio/test_taskgroups.py b/Lib/test/test_asyncio/test_taskgroups.py index 0d69a436fdb8404..91f6b03b4597a50 100644 --- a/Lib/test/test_asyncio/test_taskgroups.py +++ b/Lib/test/test_asyncio/test_taskgroups.py @@ -15,7 +15,7 @@ # To prevent a warning "test altered the execution environment" def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class MyExc(Exception): diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index 44498ef790e450d..f084395b5c5b782 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -24,7 +24,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) async def coroutine_function(): @@ -1218,6 +1218,19 @@ def gen(): loop.advance_time(10) loop.run_until_complete(asyncio.wait([a, b])) + def test_wait_discards_awaited_by_for_pending(self): + # gh-152569: wait() must remove itself from the await-graph of every + # future once it returns, including futures that never resolved. + async def coro(): + immortal = self.loop.create_future() + done = self.new_task(self.loop, asyncio.sleep(0)) + await asyncio.wait({done, immortal}, + return_when=asyncio.FIRST_COMPLETED) + self.assertFalse(immortal._asyncio_awaited_by) + immortal.cancel() + + self.loop.run_until_complete(self.new_task(self.loop, coro())) + def test_wait_really_done(self): # there is possibility that some tasks in the pending list # became done but their callbacks haven't all been called yet @@ -2116,6 +2129,46 @@ def test_shield_cancel_outer(self): self.assertTrue(outer.cancelled()) self.assertEqual(0, 0 if outer._callbacks is None else len(outer._callbacks)) + def test_shield_cancel_outer_result(self): + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + inner.set_result(1) + test_utils.run_briefly(self.loop) + mock_handler.assert_not_called() + + def test_shield_cancel_outer_exception(self): + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + inner.set_exception(Exception('foo')) + test_utils.run_briefly(self.loop) + mock_handler.assert_called_once() + + def test_shield_duplicate_log_once(self): + mock_handler = mock.Mock() + self.loop.set_exception_handler(mock_handler) + inner = self.new_future(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + outer = asyncio.shield(inner) + test_utils.run_briefly(self.loop) + outer.cancel() + test_utils.run_briefly(self.loop) + inner.set_exception(Exception('foo')) + test_utils.run_briefly(self.loop) + mock_handler.assert_called_once() + def test_shield_shortcut(self): fut = self.new_future(self.loop) fut.set_result(42) @@ -3005,6 +3058,26 @@ class BaseTaskIntrospectionTests: _enter_task = None _leave_task = None all_tasks = None + Task = None + + def test_register_task_resurrection(self): + register_task = self._register_task + class EvilLoop: + def get_debug(self): + return False + + def call_exception_handler(self, context): + register_task(context["task"]) + + async def coro_fn (): + pass + + coro = coro_fn() + self.addCleanup(coro.close) + loop = EvilLoop() + with self.assertRaises(AttributeError): + self.Task(coro, loop=loop) + def test__register_task_1(self): class TaskLike: @@ -3135,6 +3208,7 @@ class PyIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests): _leave_task = staticmethod(tasks._py_leave_task) all_tasks = staticmethod(tasks._py_all_tasks) current_task = staticmethod(tasks._py_current_task) + Task = tasks._PyTask @unittest.skipUnless(hasattr(tasks, '_c_register_task'), @@ -3147,10 +3221,12 @@ class CIntrospectionTests(test_utils.TestCase, BaseTaskIntrospectionTests): _leave_task = staticmethod(tasks._c_leave_task) all_tasks = staticmethod(tasks._c_all_tasks) current_task = staticmethod(tasks._c_current_task) + Task = tasks._CTask else: _register_task = _unregister_task = _enter_task = _leave_task = None + class BaseCurrentLoopTests: current_task = None @@ -3640,6 +3716,30 @@ def task_factory(loop, coro): (loop, context), kwargs = callback.call_args self.assertEqual(context['exception'], exc_context.exception) + def test_run_coroutine_threadsafe_and_cancel(self): + task = None + thread_future = None + # Use a custom task factory to capture the created Task + def task_factory(loop, coro): + nonlocal task + task = asyncio.Task(coro, loop=loop) + return task + + self.addCleanup(self.loop.set_task_factory, + self.loop.get_task_factory()) + + async def target(): + nonlocal thread_future + self.loop.set_task_factory(task_factory) + thread_future = asyncio.run_coroutine_threadsafe(asyncio.sleep(10), self.loop) + await asyncio.sleep(0) + + thread_future.cancel() + + self.loop.run_until_complete(target()) + self.assertTrue(task.cancelled()) + self.assertTrue(thread_future.cancelled()) + class SleepTests(test_utils.TestCase): def setUp(self): diff --git a/Lib/test/test_asyncio/test_threads.py b/Lib/test/test_asyncio/test_threads.py index c98c9a9b395ff94..8ad5f9b2c9e7505 100644 --- a/Lib/test/test_asyncio/test_threads.py +++ b/Lib/test/test_asyncio/test_threads.py @@ -8,7 +8,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class ToThreadTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_timeouts.py b/Lib/test/test_asyncio/test_timeouts.py index 3ba84d63b2ca5fd..f60722c48b76179 100644 --- a/Lib/test/test_asyncio/test_timeouts.py +++ b/Lib/test/test_asyncio/test_timeouts.py @@ -9,7 +9,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TimeoutTests(unittest.IsolatedAsyncioTestCase): diff --git a/Lib/test/test_asyncio/test_tools.py b/Lib/test/test_asyncio/test_tools.py index 0413e236c27dfab..34e94830204cd8c 100644 --- a/Lib/test/test_asyncio/test_tools.py +++ b/Lib/test/test_asyncio/test_tools.py @@ -2,6 +2,13 @@ from asyncio import tools +from collections import namedtuple + +FrameInfo = namedtuple('FrameInfo', ['funcname', 'filename', 'lineno']) +CoroInfo = namedtuple('CoroInfo', ['call_stack', 'task_name']) +TaskInfo = namedtuple('TaskInfo', ['task_id', 'task_name', 'coroutine_stack', 'awaited_by']) +AwaitedInfo = namedtuple('AwaitedInfo', ['thread_id', 'awaited_by']) + # mock output of get_all_awaited_by function. TEST_INPUTS_TREE = [ @@ -10,81 +17,151 @@ # different subtasks part of a TaskGroup (root1 and root2) which call # awaiter functions. ( - ( - 1, - [ - (2, "Task-1", []), - ( - 3, - "timer", - [ - [[("awaiter3", "/path/to/app.py", 130), - ("awaiter2", "/path/to/app.py", 120), - ("awaiter", "/path/to/app.py", 110)], 4], - [[("awaiterB3", "/path/to/app.py", 190), - ("awaiterB2", "/path/to/app.py", 180), - ("awaiterB", "/path/to/app.py", 170)], 5], - [[("awaiterB3", "/path/to/app.py", 190), - ("awaiterB2", "/path/to/app.py", 180), - ("awaiterB", "/path/to/app.py", 170)], 6], - [[("awaiter3", "/path/to/app.py", 130), - ("awaiter2", "/path/to/app.py", 120), - ("awaiter", "/path/to/app.py", 110)], 7], - ], - ), - ( - 8, - "root1", - [[["_aexit", "__aexit__", "main"], 2]], - ), - ( - 9, - "root2", - [[["_aexit", "__aexit__", "main"], 2]], - ), - ( - 4, - "child1_1", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 8, - ] - ], - ), - ( - 6, - "child2_1", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 8, - ] - ], - ), - ( - 7, - "child1_2", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 9, - ] - ], - ), - ( - 5, - "child2_2", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 9, - ] - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] ), - ], + TaskInfo( + task_id=3, + task_name="timer", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "/path/to/app.py", 130), + FrameInfo("awaiter2", "/path/to/app.py", 120), + FrameInfo("awaiter", "/path/to/app.py", 110) + ], + task_name=4 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiterB3", "/path/to/app.py", 190), + FrameInfo("awaiterB2", "/path/to/app.py", 180), + FrameInfo("awaiterB", "/path/to/app.py", 170) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiterB3", "/path/to/app.py", 190), + FrameInfo("awaiterB2", "/path/to/app.py", 180), + FrameInfo("awaiterB", "/path/to/app.py", 170) + ], + task_name=6 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "/path/to/app.py", 130), + FrameInfo("awaiter2", "/path/to/app.py", 120), + FrameInfo("awaiter", "/path/to/app.py", 110) + ], + task_name=7 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="root1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=9, + task_name="root2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="child1_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="child2_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="child1_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ), + TaskInfo( + task_id=5, + task_name="child2_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ) + ] ), - (0, []), + AwaitedInfo(thread_id=0, awaited_by=[]) ), ( [ @@ -130,26 +207,96 @@ [ # test case containing two roots ( - ( - 9, - [ - (5, "Task-5", []), - (6, "Task-6", [[["main2"], 5]]), - (7, "Task-7", [[["main2"], 5]]), - (8, "Task-8", [[["main2"], 5]]), - ], + AwaitedInfo( + thread_id=9, + awaited_by=[ + TaskInfo( + task_id=5, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=6, + task_name="Task-6", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-7", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="Task-8", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ) + ] ), - ( - 10, - [ - (1, "Task-1", []), - (2, "Task-2", [[["main"], 1]]), - (3, "Task-3", [[["main"], 1]]), - (4, "Task-4", [[["main"], 1]]), - ], + AwaitedInfo( + thread_id=10, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=2, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ) + ] ), - (11, []), - (0, []), + AwaitedInfo(thread_id=11, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) ), ( [ @@ -174,18 +321,63 @@ # test case containing two roots, one of them without subtasks ( [ - (1, [(2, "Task-5", [])]), - ( - 3, - [ - (4, "Task-1", []), - (5, "Task-2", [[["main"], 4]]), - (6, "Task-3", [[["main"], 4]]), - (7, "Task-4", [[["main"], 4]]), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ) + ] ), - (8, []), - (0, []), + AwaitedInfo( + thread_id=3, + awaited_by=[ + TaskInfo( + task_id=4, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=5, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=8, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) ] ), ( @@ -208,19 +400,44 @@ # this test case contains a cycle: two tasks awaiting each other. ( [ - ( - 1, - [ - (2, "Task-1", []), - ( - 3, - "a", - [[["awaiter2"], 4], [["main"], 2]], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] ), - (4, "b", [[["awaiter"], 3]]), - ], + TaskInfo( + task_id=3, + task_name="a", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter2", "", 0)], + task_name=4 + ), + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="b", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter", "", 0)], + task_name=3 + ) + ] + ) + ] ), - (0, []), + AwaitedInfo(thread_id=0, awaited_by=[]) ] ), ([[4, 3, 4]]), @@ -229,32 +446,85 @@ # this test case contains two cycles ( [ - ( - 1, - [ - (2, "Task-1", []), - ( - 3, - "A", - [[["nested", "nested", "task_b"], 4]], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] ), - ( - 4, - "B", - [ - [["nested", "nested", "task_c"], 5], - [["nested", "nested", "task_a"], 3], - ], + TaskInfo( + task_id=3, + task_name="A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] ), - (5, "C", [[["nested", "nested"], 6]]), - ( - 6, - "Task-2", - [[["nested", "nested", "task_b"], 4]], + TaskInfo( + task_id=4, + task_name="B", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_c", "", 0) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_a", "", 0) + ], + task_name=3 + ) + ] ), - ], + TaskInfo( + task_id=5, + task_name="C", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0) + ], + task_name=6 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] + ) + ] ), - (0, []), + AwaitedInfo(thread_id=0, awaited_by=[]) ] ), ([[4, 3, 4], [4, 6, 5, 4]]), @@ -267,81 +537,160 @@ # different subtasks part of a TaskGroup (root1 and root2) which call # awaiter functions. ( - ( - 1, - [ - (2, "Task-1", []), - ( - 3, - "timer", - [ - [["awaiter3", "awaiter2", "awaiter"], 4], - [["awaiter1_3", "awaiter1_2", "awaiter1"], 5], - [["awaiter1_3", "awaiter1_2", "awaiter1"], 6], - [["awaiter3", "awaiter2", "awaiter"], 7], - ], - ), - ( - 8, - "root1", - [[["_aexit", "__aexit__", "main"], 2]], - ), - ( - 9, - "root2", - [[["_aexit", "__aexit__", "main"], 2]], - ), - ( - 4, - "child1_1", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 8, - ] - ], - ), - ( - 6, - "child2_1", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 8, - ] - ], - ), - ( - 7, - "child1_2", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 9, - ] - ], - ), - ( - 5, - "child2_2", - [ - [ - ["_aexit", "__aexit__", "blocho_caller", "bloch"], - 9, - ] - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] ), - ], + TaskInfo( + task_id=3, + task_name="timer", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "", 0), + FrameInfo("awaiter2", "", 0), + FrameInfo("awaiter", "", 0) + ], + task_name=4 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter1_3", "", 0), + FrameInfo("awaiter1_2", "", 0), + FrameInfo("awaiter1", "", 0) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter1_3", "", 0), + FrameInfo("awaiter1_2", "", 0), + FrameInfo("awaiter1", "", 0) + ], + task_name=6 + ), + CoroInfo( + call_stack=[ + FrameInfo("awaiter3", "", 0), + FrameInfo("awaiter2", "", 0), + FrameInfo("awaiter", "", 0) + ], + task_name=7 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="root1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=9, + task_name="root2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("main", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="child1_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="child2_1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=8 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="child1_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ), + TaskInfo( + task_id=5, + task_name="child2_2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("_aexit", "", 0), + FrameInfo("__aexit__", "", 0), + FrameInfo("blocho_caller", "", 0), + FrameInfo("bloch", "", 0) + ], + task_name=9 + ) + ] + ) + ] ), - (0, []), + AwaitedInfo(thread_id=0, awaited_by=[]) ), ( [ - [1, "0x2", "Task-1", "", "", "0x0"], + [1, "0x2", "Task-1", "", "", "", "0x0"], [ 1, "0x3", "timer", + "", "awaiter3 -> awaiter2 -> awaiter", "child1_1", "0x4", @@ -350,6 +699,7 @@ 1, "0x3", "timer", + "", "awaiter1_3 -> awaiter1_2 -> awaiter1", "child2_2", "0x5", @@ -358,6 +708,7 @@ 1, "0x3", "timer", + "", "awaiter1_3 -> awaiter1_2 -> awaiter1", "child2_1", "0x6", @@ -366,6 +717,7 @@ 1, "0x3", "timer", + "", "awaiter3 -> awaiter2 -> awaiter", "child1_2", "0x7", @@ -374,6 +726,7 @@ 1, "0x8", "root1", + "", "_aexit -> __aexit__ -> main", "Task-1", "0x2", @@ -382,6 +735,7 @@ 1, "0x9", "root2", + "", "_aexit -> __aexit__ -> main", "Task-1", "0x2", @@ -390,6 +744,7 @@ 1, "0x4", "child1_1", + "", "_aexit -> __aexit__ -> blocho_caller -> bloch", "root1", "0x8", @@ -398,6 +753,7 @@ 1, "0x6", "child2_1", + "", "_aexit -> __aexit__ -> blocho_caller -> bloch", "root1", "0x8", @@ -406,6 +762,7 @@ 1, "0x7", "child1_2", + "", "_aexit -> __aexit__ -> blocho_caller -> bloch", "root2", "0x9", @@ -414,6 +771,7 @@ 1, "0x5", "child2_2", + "", "_aexit -> __aexit__ -> blocho_caller -> bloch", "root2", "0x9", @@ -424,37 +782,107 @@ [ # test case containing two roots ( - ( - 9, - [ - (5, "Task-5", []), - (6, "Task-6", [[["main2"], 5]]), - (7, "Task-7", [[["main2"], 5]]), - (8, "Task-8", [[["main2"], 5]]), - ], + AwaitedInfo( + thread_id=9, + awaited_by=[ + TaskInfo( + task_id=5, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=6, + task_name="Task-6", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-7", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ), + TaskInfo( + task_id=8, + task_name="Task-8", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main2", "", 0)], + task_name=5 + ) + ] + ) + ] ), - ( - 10, - [ - (1, "Task-1", []), - (2, "Task-2", [[["main"], 1]]), - (3, "Task-3", [[["main"], 1]]), - (4, "Task-4", [[["main"], 1]]), - ], + AwaitedInfo( + thread_id=10, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=2, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=1 + ) + ] + ) + ] ), - (11, []), - (0, []), + AwaitedInfo(thread_id=11, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) ), ( [ - [9, "0x5", "Task-5", "", "", "0x0"], - [9, "0x6", "Task-6", "main2", "Task-5", "0x5"], - [9, "0x7", "Task-7", "main2", "Task-5", "0x5"], - [9, "0x8", "Task-8", "main2", "Task-5", "0x5"], - [10, "0x1", "Task-1", "", "", "0x0"], - [10, "0x2", "Task-2", "main", "Task-1", "0x1"], - [10, "0x3", "Task-3", "main", "Task-1", "0x1"], - [10, "0x4", "Task-4", "main", "Task-1", "0x1"], + [9, "0x5", "Task-5", "", "", "", "0x0"], + [9, "0x6", "Task-6", "", "main2", "Task-5", "0x5"], + [9, "0x7", "Task-7", "", "main2", "Task-5", "0x5"], + [9, "0x8", "Task-8", "", "main2", "Task-5", "0x5"], + [10, "0x1", "Task-1", "", "", "", "0x0"], + [10, "0x2", "Task-2", "", "main", "Task-1", "0x1"], + [10, "0x3", "Task-3", "", "main", "Task-1", "0x1"], + [10, "0x4", "Task-4", "", "main", "Task-1", "0x1"], ] ), ], @@ -462,27 +890,72 @@ # test case containing two roots, one of them without subtasks ( [ - (1, [(2, "Task-5", [])]), - ( - 3, - [ - (4, "Task-1", []), - (5, "Task-2", [[["main"], 4]]), - (6, "Task-3", [[["main"], 4]]), - (7, "Task-4", [[["main"], 4]]), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-5", + coroutine_stack=[], + awaited_by=[] + ) + ] ), - (8, []), - (0, []), + AwaitedInfo( + thread_id=3, + awaited_by=[ + TaskInfo( + task_id=4, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=5, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=6, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ), + TaskInfo( + task_id=7, + task_name="Task-4", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=4 + ) + ] + ) + ] + ), + AwaitedInfo(thread_id=8, awaited_by=[]), + AwaitedInfo(thread_id=0, awaited_by=[]) ] ), ( [ - [1, "0x2", "Task-5", "", "", "0x0"], - [3, "0x4", "Task-1", "", "", "0x0"], - [3, "0x5", "Task-2", "main", "Task-1", "0x4"], - [3, "0x6", "Task-3", "main", "Task-1", "0x4"], - [3, "0x7", "Task-4", "main", "Task-1", "0x4"], + [1, "0x2", "Task-5", "", "", "", "0x0"], + [3, "0x4", "Task-1", "", "", "", "0x0"], + [3, "0x5", "Task-2", "", "main", "Task-1", "0x4"], + [3, "0x6", "Task-3", "", "main", "Task-1", "0x4"], + [3, "0x7", "Task-4", "", "main", "Task-1", "0x4"], ] ), ], @@ -491,27 +964,52 @@ # this test case contains a cycle: two tasks awaiting each other. ( [ - ( - 1, - [ - (2, "Task-1", []), - ( - 3, - "a", - [[["awaiter2"], 4], [["main"], 2]], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] ), - (4, "b", [[["awaiter"], 3]]), - ], + TaskInfo( + task_id=3, + task_name="a", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter2", "", 0)], + task_name=4 + ), + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="b", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("awaiter", "", 0)], + task_name=3 + ) + ] + ) + ] ), - (0, []), + AwaitedInfo(thread_id=0, awaited_by=[]) ] ), ( [ - [1, "0x2", "Task-1", "", "", "0x0"], - [1, "0x3", "a", "awaiter2", "b", "0x4"], - [1, "0x3", "a", "main", "Task-1", "0x2"], - [1, "0x4", "b", "awaiter", "a", "0x3"], + [1, "0x2", "Task-1", "", "", "", "0x0"], + [1, "0x3", "a", "", "awaiter2", "b", "0x4"], + [1, "0x3", "a", "", "main", "Task-1", "0x2"], + [1, "0x4", "b", "", "awaiter", "a", "0x3"], ] ), ], @@ -519,41 +1017,95 @@ # this test case contains two cycles ( [ - ( - 1, - [ - (2, "Task-1", []), - ( - 3, - "A", - [[["nested", "nested", "task_b"], 4]], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] ), - ( - 4, - "B", - [ - [["nested", "nested", "task_c"], 5], - [["nested", "nested", "task_a"], 3], - ], + TaskInfo( + task_id=4, + task_name="B", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_c", "", 0) + ], + task_name=5 + ), + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_a", "", 0) + ], + task_name=3 + ) + ] ), - (5, "C", [[["nested", "nested"], 6]]), - ( - 6, - "Task-2", - [[["nested", "nested", "task_b"], 4]], + TaskInfo( + task_id=5, + task_name="C", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0) + ], + task_name=6 + ) + ] ), - ], + TaskInfo( + task_id=6, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("nested", "", 0), + FrameInfo("nested", "", 0), + FrameInfo("task_b", "", 0) + ], + task_name=4 + ) + ] + ) + ] ), - (0, []), + AwaitedInfo(thread_id=0, awaited_by=[]) ] ), ( [ - [1, "0x2", "Task-1", "", "", "0x0"], + [1, "0x2", "Task-1", "", "", "", "0x0"], [ 1, "0x3", "A", + "", "nested -> nested -> task_b", "B", "0x4", @@ -562,6 +1114,7 @@ 1, "0x4", "B", + "", "nested -> nested -> task_c", "C", "0x5", @@ -570,6 +1123,7 @@ 1, "0x4", "B", + "", "nested -> nested -> task_a", "A", "0x3", @@ -578,6 +1132,7 @@ 1, "0x5", "C", + "", "nested -> nested", "Task-2", "0x6", @@ -586,6 +1141,7 @@ 1, "0x6", "Task-2", + "", "nested -> nested -> task_b", "B", "0x4", @@ -600,7 +1156,8 @@ class TestAsyncioToolsTree(unittest.TestCase): def test_asyncio_utils(self): for input_, tree in TEST_INPUTS_TREE: with self.subTest(input_): - self.assertEqual(tools.build_async_tree(input_), tree) + result = tools.build_async_tree(input_) + self.assertEqual(result, tree) def test_asyncio_utils_cycles(self): for input_, cycles in TEST_INPUTS_CYCLES_TREE: @@ -615,7 +1172,8 @@ class TestAsyncioToolsTable(unittest.TestCase): def test_asyncio_utils(self): for input_, table in TEST_INPUTS_TABLE: with self.subTest(input_): - self.assertEqual(tools.build_task_table(input_), table) + result = tools.build_task_table(input_) + self.assertEqual(result, table) class TestAsyncioToolsBasic(unittest.TestCase): @@ -632,26 +1190,67 @@ def test_empty_input_table(self): self.assertEqual(tools.build_task_table(result), expected_output) def test_only_independent_tasks_tree(self): - input_ = [(1, [(10, "taskA", []), (11, "taskB", [])])] + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=10, + task_name="taskA", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=11, + task_name="taskB", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] expected = [["└── (T) taskA"], ["└── (T) taskB"]] result = tools.build_async_tree(input_) self.assertEqual(sorted(result), sorted(expected)) def test_only_independent_tasks_table(self): - input_ = [(1, [(10, "taskA", []), (11, "taskB", [])])] + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=10, + task_name="taskA", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=11, + task_name="taskB", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] self.assertEqual( tools.build_task_table(input_), - [[1, "0xa", "taskA", "", "", "0x0"], [1, "0xb", "taskB", "", "", "0x0"]], + [[1, '0xa', 'taskA', '', '', '', '0x0'], [1, '0xb', 'taskB', '', '', '', '0x0']] ) def test_single_task_tree(self): """Test build_async_tree with a single task and no awaits.""" result = [ - ( - 1, - [ - (2, "Task-1", []), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ) + ] ) ] expected_output = [ @@ -664,25 +1263,50 @@ def test_single_task_tree(self): def test_single_task_table(self): """Test build_task_table with a single task and no awaits.""" result = [ - ( - 1, - [ - (2, "Task-1", []), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ) + ] ) ] - expected_output = [[1, "0x2", "Task-1", "", "", "0x0"]] + expected_output = [[1, '0x2', 'Task-1', '', '', '', '0x0']] self.assertEqual(tools.build_task_table(result), expected_output) def test_cycle_detection(self): """Test build_async_tree raises CycleFoundException for cyclic input.""" result = [ - ( - 1, - [ - (2, "Task-1", [[["main"], 3]]), - (3, "Task-2", [[["main"], 2]]), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ) + ] ) ] with self.assertRaises(tools.CycleFoundException) as context: @@ -692,13 +1316,38 @@ def test_cycle_detection(self): def test_complex_tree(self): """Test build_async_tree with a more complex tree structure.""" result = [ - ( - 1, - [ - (2, "Task-1", []), - (3, "Task-2", [[["main"], 2]]), - (4, "Task-3", [[["main"], 3]]), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=3 + ) + ] + ) + ] ) ] expected_output = [ @@ -715,30 +1364,76 @@ def test_complex_tree(self): def test_complex_table(self): """Test build_task_table with a more complex tree structure.""" result = [ - ( - 1, - [ - (2, "Task-1", []), - (3, "Task-2", [[["main"], 2]]), - (4, "Task-3", [[["main"], 3]]), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=2, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=4, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("main", "", 0)], + task_name=3 + ) + ] + ) + ] ) ] expected_output = [ - [1, "0x2", "Task-1", "", "", "0x0"], - [1, "0x3", "Task-2", "main", "Task-1", "0x2"], - [1, "0x4", "Task-3", "main", "Task-2", "0x3"], + [1, '0x2', 'Task-1', '', '', '', '0x0'], + [1, '0x3', 'Task-2', '', 'main', 'Task-1', '0x2'], + [1, '0x4', 'Task-3', '', 'main', 'Task-2', '0x3'] ] self.assertEqual(tools.build_task_table(result), expected_output) def test_deep_coroutine_chain(self): input_ = [ - ( - 1, - [ - (10, "leaf", [[["c1", "c2", "c3", "c4", "c5"], 11]]), - (11, "root", []), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=10, + task_name="leaf", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("c1", "", 0), + FrameInfo("c2", "", 0), + FrameInfo("c3", "", 0), + FrameInfo("c4", "", 0), + FrameInfo("c5", "", 0) + ], + task_name=11 + ) + ] + ), + TaskInfo( + task_id=11, + task_name="root", + coroutine_stack=[], + awaited_by=[] + ) + ] ) ] expected = [ @@ -757,13 +1452,47 @@ def test_deep_coroutine_chain(self): def test_multiple_cycles_same_node(self): input_ = [ - ( - 1, - [ - (1, "Task-A", [[["call1"], 2]]), - (2, "Task-B", [[["call2"], 3]]), - (3, "Task-C", [[["call3"], 1], [["call4"], 2]]), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("call1", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="Task-B", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("call2", "", 0)], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=3, + task_name="Task-C", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("call3", "", 0)], + task_name=1 + ), + CoroInfo( + call_stack=[FrameInfo("call4", "", 0)], + task_name=2 + ) + ] + ) + ] ) ] with self.assertRaises(tools.CycleFoundException) as ctx: @@ -772,48 +1501,130 @@ def test_multiple_cycles_same_node(self): self.assertTrue(any(set(c) == {1, 2, 3} for c in cycles)) def test_table_output_format(self): - input_ = [(1, [(1, "Task-A", [[["foo"], 2]]), (2, "Task-B", [])])] + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("foo", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="Task-B", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] table = tools.build_task_table(input_) for row in table: - self.assertEqual(len(row), 6) + self.assertEqual(len(row), 7) self.assertIsInstance(row[0], int) # thread ID self.assertTrue( isinstance(row[1], str) and row[1].startswith("0x") ) # hex task ID self.assertIsInstance(row[2], str) # task name - self.assertIsInstance(row[3], str) # coroutine chain - self.assertIsInstance(row[4], str) # awaiter name + self.assertIsInstance(row[3], str) # coroutine stack + self.assertIsInstance(row[4], str) # coroutine chain + self.assertIsInstance(row[5], str) # awaiter name self.assertTrue( - isinstance(row[5], str) and row[5].startswith("0x") + isinstance(row[6], str) and row[6].startswith("0x") ) # hex awaiter ID class TestAsyncioToolsEdgeCases(unittest.TestCase): def test_task_awaits_self(self): - """A task directly awaits itself – should raise a cycle.""" - input_ = [(1, [(1, "Self-Awaiter", [[["loopback"], 1]])])] + """A task directly awaits itself - should raise a cycle.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Self-Awaiter", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("loopback", "", 0)], + task_name=1 + ) + ] + ) + ] + ) + ] with self.assertRaises(tools.CycleFoundException) as ctx: tools.build_async_tree(input_) self.assertIn([1, 1], ctx.exception.cycles) def test_task_with_missing_awaiter_id(self): - """Awaiter ID not in task list – should not crash, just show 'Unknown'.""" - input_ = [(1, [(1, "Task-A", [[["coro"], 999]])])] # 999 not defined + """Awaiter ID not in task list - should not crash, just show 'Unknown'.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-A", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("coro", "", 0)], + task_name=999 + ) + ] + ) + ] + ) + ] table = tools.build_task_table(input_) self.assertEqual(len(table), 1) - self.assertEqual(table[0][4], "Unknown") + self.assertEqual(table[0][5], "Unknown") def test_duplicate_coroutine_frames(self): - """Same coroutine frame repeated under a parent – should deduplicate.""" + """Same coroutine frame repeated under a parent - should deduplicate.""" input_ = [ - ( - 1, - [ - (1, "Task-1", [[["frameA"], 2], [["frameA"], 3]]), - (2, "Task-2", []), - (3, "Task-3", []), - ], + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="Task-1", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("frameA", "", 0)], + task_name=2 + ), + CoroInfo( + call_stack=[FrameInfo("frameA", "", 0)], + task_name=3 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="Task-2", + coroutine_stack=[], + awaited_by=[] + ), + TaskInfo( + task_id=3, + task_name="Task-3", + coroutine_stack=[], + awaited_by=[] + ) + ] ) ] tree = tools.build_async_tree(input_) @@ -829,15 +1640,64 @@ def test_duplicate_coroutine_frames(self): self.assertIn("Task-1", flat) def test_task_with_no_name(self): - """Task with no name in id2name – should still render with fallback.""" - input_ = [(1, [(1, "root", [[["f1"], 2]]), (2, None, [])])] + """Task with no name in id2name - should still render with fallback.""" + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="root", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[FrameInfo("f1", "", 0)], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name=None, + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] # If name is None, fallback to string should not crash tree = tools.build_async_tree(input_) self.assertIn("(T) None", "\n".join(tree[0])) def test_tree_rendering_with_custom_emojis(self): """Pass custom emojis to the tree renderer.""" - input_ = [(1, [(1, "MainTask", [[["f1", "f2"], 2]]), (2, "SubTask", [])])] + input_ = [ + AwaitedInfo( + thread_id=1, + awaited_by=[ + TaskInfo( + task_id=1, + task_name="MainTask", + coroutine_stack=[], + awaited_by=[ + CoroInfo( + call_stack=[ + FrameInfo("f1", "", 0), + FrameInfo("f2", "", 0) + ], + task_name=2 + ) + ] + ), + TaskInfo( + task_id=2, + task_name="SubTask", + coroutine_stack=[], + awaited_by=[] + ) + ] + ) + ] tree = tools.build_async_tree(input_, task_emoji="🧵", cor_emoji="🔁") flat = "\n".join(tree[0]) self.assertIn("🧵 MainTask", flat) diff --git a/Lib/test/test_asyncio/test_transports.py b/Lib/test/test_asyncio/test_transports.py index af10d3dc2a80df0..dbb572e2e1536b3 100644 --- a/Lib/test/test_asyncio/test_transports.py +++ b/Lib/test/test_asyncio/test_transports.py @@ -10,7 +10,7 @@ def tearDownModule(): # not needed for the test file but added for uniformness with all other # asyncio test files for the sake of unified cleanup - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TransportTests(unittest.TestCase): diff --git a/Lib/test/test_asyncio/test_unix_events.py b/Lib/test/test_asyncio/test_unix_events.py index e020c1f3e4f6770..520f5c733c3bf27 100644 --- a/Lib/test/test_asyncio/test_unix_events.py +++ b/Lib/test/test_asyncio/test_unix_events.py @@ -30,7 +30,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) MOCK_ANY = mock.ANY @@ -1180,11 +1180,46 @@ async def runner(): @support.requires_fork() -class TestFork(unittest.IsolatedAsyncioTestCase): +class TestFork(unittest.TestCase): + + def test_fork_not_share_current_task(self): + loop = object() + task = object() + asyncio._set_running_loop(loop) + self.addCleanup(asyncio._set_running_loop, None) + asyncio.tasks._enter_task(loop, task) + self.addCleanup(asyncio.tasks._leave_task, loop, task) + self.assertIs(asyncio.current_task(), task) + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + pid = os.fork() + if pid == 0: + # child + try: + asyncio._set_running_loop(loop) + current_task = asyncio.current_task() + if current_task is None: + os.write(w, b'NO TASK') + else: + os.write(w, b'TASK:' + str(id(current_task)).encode()) + except BaseException as e: + os.write(w, b'ERROR:' + ascii(e).encode()) + finally: + asyncio._set_running_loop(None) + os._exit(0) + else: + # parent + result = os.read(r, 100) + self.assertEqual(result, b'NO TASK') + wait_process(pid, exitcode=0) - async def test_fork_not_share_event_loop(self): + def test_fork_not_share_event_loop(self): # The forked process should not share the event loop with the parent - loop = asyncio.get_running_loop() + loop = object() + asyncio._set_running_loop(loop) + self.assertIs(asyncio.get_running_loop(), loop) + self.addCleanup(asyncio._set_running_loop, None) r, w = os.pipe() self.addCleanup(os.close, r) self.addCleanup(os.close, w) diff --git a/Lib/test/test_asyncio/test_waitfor.py b/Lib/test/test_asyncio/test_waitfor.py index d083f6b4d2a535c..dedc6bf69d770e4 100644 --- a/Lib/test/test_asyncio/test_waitfor.py +++ b/Lib/test/test_asyncio/test_waitfor.py @@ -5,7 +5,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) # The following value can be used as a very small timeout: diff --git a/Lib/test/test_asyncio/test_windows_events.py b/Lib/test/test_asyncio/test_windows_events.py index 69e9905205eee0a..4a85b1a09ee0e24 100644 --- a/Lib/test/test_asyncio/test_windows_events.py +++ b/Lib/test/test_asyncio/test_windows_events.py @@ -5,6 +5,7 @@ import time import threading import unittest +import warnings from unittest import mock if sys.platform != 'win32': @@ -15,11 +16,14 @@ import asyncio from asyncio import windows_events +from asyncio import windows_utils +from test import support +from test.support import os_helper from test.test_asyncio import utils as test_utils def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class UpperProto(asyncio.Protocol): @@ -330,16 +334,16 @@ def test_selector_win_policy(self): async def main(): self.assertIsInstance(asyncio.get_running_loop(), asyncio.SelectorEventLoop) - old_policy = asyncio._get_event_loop_policy() + old_policy = asyncio.events._get_event_loop_policy() try: with self.assertWarnsRegex( DeprecationWarning, "'asyncio.WindowsSelectorEventLoopPolicy' is deprecated", ): - asyncio._set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + asyncio.events._set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) asyncio.run(main()) finally: - asyncio._set_event_loop_policy(old_policy) + asyncio.events._set_event_loop_policy(old_policy) def test_proactor_win_policy(self): async def main(): @@ -347,16 +351,121 @@ async def main(): asyncio.get_running_loop(), asyncio.ProactorEventLoop) - old_policy = asyncio._get_event_loop_policy() + old_policy = asyncio.events._get_event_loop_policy() try: with self.assertWarnsRegex( DeprecationWarning, "'asyncio.WindowsProactorEventLoopPolicy' is deprecated", ): - asyncio._set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) + asyncio.events._set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) asyncio.run(main()) finally: - asyncio._set_event_loop_policy(old_policy) + asyncio.events._set_event_loop_policy(old_policy) + + +class ProactorPipeObjectSupportTests(unittest.TestCase): + + def setUp(self): + super().setUp() + self.loop = asyncio.ProactorEventLoop() + self.addCleanup(self.loop.close) + self.errors = [] + self.loop.set_exception_handler( + lambda loop, context: self.errors.append(context)) + + def check_read_rejected(self, pipe): + self.addCleanup(pipe.close) + lost = self.loop.create_future() + + class Proto(asyncio.Protocol): + def connection_lost(self, exc): + if not lost.done(): + lost.set_result(exc) + + async def run(): + transport, _ = await self.loop.connect_read_pipe(Proto, pipe) + exc = await lost + transport.close() + return exc + + exc = self.loop.run_until_complete(run()) + self.assertIsInstance(exc, OSError) + + def check_write_rejected(self, pipe): + self.addCleanup(pipe.close) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', ResourceWarning) + with self.assertRaises(OSError): + self.loop.run_until_complete( + self.loop.connect_write_pipe(asyncio.BaseProtocol, pipe)) + support.gc_collect() + + def check_accepted(self, rpipe, wpipe): + self.addCleanup(rpipe.close) + self.addCleanup(wpipe.close) + + chunks = [] + lost = self.loop.create_future() + + class ReadProto(asyncio.Protocol): + def data_received(self, data): + chunks.append(data) + + def connection_lost(self, exc): + if not lost.done(): + lost.set_result(exc) + + async def run(): + rtransport, _ = await self.loop.connect_read_pipe(ReadProto, rpipe) + wtransport, _ = await self.loop.connect_write_pipe( + asyncio.BaseProtocol, wpipe) + wtransport.write(b'spam') + wtransport.close() + await lost + rtransport.close() + + self.loop.run_until_complete(run()) + self.assertEqual(b''.join(chunks), b'spam') + self.assertFalse(self.errors) + + def test_overlapped_pipe(self): + rhandle, whandle = windows_utils.pipe(duplex=True, overlapped=(True, True)) + self.check_accepted(windows_utils.PipeHandle(rhandle), + windows_utils.PipeHandle(whandle)) + + def test_socketpair(self): + rsock, wsock = socket.socketpair() + self.check_accepted(rsock, wsock) + + def test_read_non_overlapped_pipe(self): + rhandle, whandle = windows_utils.pipe(overlapped=(False, False)) + self.addCleanup(windows_utils.PipeHandle(whandle).close) + self.check_read_rejected(windows_utils.PipeHandle(rhandle)) + + def test_write_non_overlapped_pipe(self): + rhandle, whandle = windows_utils.pipe(overlapped=(False, False)) + self.addCleanup(windows_utils.PipeHandle(rhandle).close) + self.check_write_rejected(windows_utils.PipeHandle(whandle)) + + def test_read_regular_file(self): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with open(os_helper.TESTFN, 'wb') as f: + f.write(b'spam') + self.check_read_rejected(open(os_helper.TESTFN, 'rb', 0)) + + def test_write_regular_file(self): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + self.check_write_rejected(open(os_helper.TESTFN, 'wb', 0)) + + def test_read_os_pipe(self): + rfd, wfd = os.pipe() + self.addCleanup(os.close, wfd) + self.check_read_rejected(open(rfd, 'rb', 0)) + + def test_write_os_pipe(self): + rfd, wfd = os.pipe() + self.addCleanup(os.close, rfd) + self.check_write_rejected(open(wfd, 'wb', 0)) if __name__ == '__main__': diff --git a/Lib/test/test_asyncio/test_windows_utils.py b/Lib/test/test_asyncio/test_windows_utils.py index a6b207567c4f005..509697613475953 100644 --- a/Lib/test/test_asyncio/test_windows_utils.py +++ b/Lib/test/test_asyncio/test_windows_utils.py @@ -16,7 +16,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class PipeTests(unittest.TestCase): @@ -77,6 +77,30 @@ def test_pipe_handle(self): else: raise RuntimeError('expected ERROR_INVALID_HANDLE') + def test_pipe_handle_close_after_external_close(self): + # gh-149388: PipeHandle.close() must clear ``_handle`` before calling + # CloseHandle so that if CloseHandle raises on a stale handle the + # PipeHandle is still marked closed and __del__ / subsequent close() + # calls are silent no-ops. + h1, h2 = windows_utils.pipe(overlapped=(False, False)) + try: + p = windows_utils.PipeHandle(h1) + # Simulate an external close of the underlying handle (e.g. + # a finalizer race or a concurrent close on the same object). + _winapi.CloseHandle(p.handle) + # First close() still propagates the OSError from CloseHandle, + # but must clear ``_handle`` first. + with self.assertRaises(OSError): + p.close() + self.assertIsNone(p.handle) + # Second close() is a no-op. + p.close() + # __del__ through GC is also a silent no-op — no unraisable. + del p + support.gc_collect() + finally: + _winapi.CloseHandle(h2) + class PopenTests(unittest.TestCase): @@ -129,5 +153,25 @@ def test_popen(self): pass +class OverlappedRefleakTests(unittest.TestCase): + + def test_wsasendto_failure(self): + ov = _overlapped.Overlapped() + buf = bytearray(4096) + with self.assertRaises(OSError): + ov.WSASendTo(0x1234, buf, 0, ("127.0.0.1", 1)) + + def test_wsarecvfrom_failure(self): + ov = _overlapped.Overlapped() + with self.assertRaises(OSError): + ov.WSARecvFrom(0x1234, 1024, 0) + + def test_wsarecvfrominto_failure(self): + ov = _overlapped.Overlapped() + buf = bytearray(4096) + with self.assertRaises(OSError): + ov.WSARecvFromInto(0x1234, buf, len(buf), 0) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index 0a96573a81c1737..a480e16e81bb91a 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -601,3 +601,9 @@ def func(): await asyncio.sleep(0) if exc is not None: raise exc + + +if sys.platform == 'win32': + DefaultEventLoopPolicy = asyncio.windows_events._DefaultEventLoopPolicy +else: + DefaultEventLoopPolicy = asyncio.unix_events._DefaultEventLoopPolicy diff --git a/Lib/test/test_atexit.py b/Lib/test/test_atexit.py index eb01da6e88a8bcd..d6d39e7683363a5 100644 --- a/Lib/test/test_atexit.py +++ b/Lib/test/test_atexit.py @@ -1,9 +1,11 @@ import atexit import os +import subprocess import textwrap import unittest +from test.support import os_helper from test import support -from test.support import script_helper +from test.support import SuppressCrashReport, script_helper from test.support import threading_helper class GeneralTest(unittest.TestCase): @@ -133,6 +135,35 @@ def callback(): self.assertEqual(os.read(r, len(expected)), expected) os.close(r) + @support.nomemtest + def test_atexit_with_low_memory(self): + # gh-140080: Test that setting low memory after registering an atexit + # callback doesn't cause an infinite loop during finalization. + code = textwrap.dedent(""" + import atexit + import _testcapi + + def callback(): + print("hello") + + atexit.register(callback) + # Simulate low memory condition + _testcapi.set_nomemory(0) + """) + + with os_helper.temp_dir() as temp_dir: + script = script_helper.make_script(temp_dir, 'test_atexit_script', code) + with SuppressCrashReport(): + with script_helper.spawn_python(script, + stderr=subprocess.PIPE) as proc: + proc.wait() + stdout = proc.stdout.read() + stderr = proc.stderr.read() + + self.assertIn(proc.returncode, (0, 1)) + self.assertNotIn(b"hello", stdout) + self.assertIn(b"MemoryError", stderr) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_audit.py b/Lib/test/test_audit.py index 2b24b5d79275fa7..077765fcda210a6 100644 --- a/Lib/test/test_audit.py +++ b/Lib/test/test_audit.py @@ -134,7 +134,7 @@ def test_socket(self): self.assertEqual(events[0][0], "socket.gethostname") self.assertEqual(events[1][0], "socket.__new__") self.assertEqual(events[2][0], "socket.bind") - self.assertTrue(events[2][2].endswith("('127.0.0.1', 8080)")) + self.assertEndsWith(events[2][2], "('127.0.0.1', 8080)") def test_gc(self): returncode, events, stderr = self.run_python("test_gc") @@ -322,6 +322,14 @@ def test_assert_unicode(self): if returncode: self.fail(stderr) + @support.support_remote_exec_only + @support.cpython_only + def test_sys_remote_exec(self): + returncode, events, stderr = self.run_python("test_sys_remote_exec") + self.assertTrue(any(["sys.remote_exec" in event for event in events])) + self.assertTrue(any(["cpython.remote_debugger_script" in event for event in events])) + if returncode: + self.fail(stderr) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_base64.py b/Lib/test/test_base64.py index 9efebc43d911c45..9fe69f4f692d8f5 100644 --- a/Lib/test/test_base64.py +++ b/Lib/test/test_base64.py @@ -208,18 +208,6 @@ def test_b64decode(self): self.check_other_types(base64.b64decode, b"YWJj", b"abc") self.check_decode_type_errors(base64.b64decode) - # Test with arbitrary alternative characters - tests_altchars = {(b'01a*b$cd', b'*$'): b'\xd3V\xbeo\xf7\x1d', - } - for (data, altchars), res in tests_altchars.items(): - data_str = data.decode('ascii') - altchars_str = altchars.decode('ascii') - - eq(base64.b64decode(data, altchars=altchars), res) - eq(base64.b64decode(data_str, altchars=altchars), res) - eq(base64.b64decode(data, altchars=altchars_str), res) - eq(base64.b64decode(data_str, altchars=altchars_str), res) - # Test standard alphabet for data, res in tests.items(): eq(base64.standard_b64decode(data), res) @@ -240,6 +228,20 @@ def test_b64decode(self): b'\xd3V\xbeo\xf7\x1d') self.check_decode_type_errors(base64.urlsafe_b64decode) + def test_b64decode_altchars(self): + # Test with arbitrary alternative characters + eq = self.assertEqual + res = b'\xd3V\xbeo\xf7\x1d' + for altchars in b'*$', b'+/', b'/+', b'+_', b'-+', b'-/', b'/_': + data = b'01a%cb%ccd' % tuple(altchars) + data_str = data.decode('ascii') + altchars_str = altchars.decode('ascii') + + eq(base64.b64decode(data, altchars=altchars), res) + eq(base64.b64decode(data_str, altchars=altchars), res) + eq(base64.b64decode(data, altchars=altchars_str), res) + eq(base64.b64decode(data_str, altchars=altchars_str), res) + def test_b64decode_padding_error(self): self.assertRaises(binascii.Error, base64.b64decode, b'abc') self.assertRaises(binascii.Error, base64.b64decode, 'abc') @@ -272,9 +274,12 @@ def test_b64decode_invalid_chars(self): base64.b64decode(bstr.decode('ascii'), validate=True) # Normal alphabet characters not discarded when alternative given - res = b'\xFB\xEF\xBE\xFF\xFF\xFF' - self.assertEqual(base64.b64decode(b'++[[//]]', b'[]'), res) - self.assertEqual(base64.urlsafe_b64decode(b'++--//__'), res) + res = b'\xfb\xef\xff' + self.assertEqual(base64.b64decode(b'++//', validate=True), res) + self.assertEqual(base64.b64decode(b'++//', '-_', validate=True), res) + self.assertEqual(base64.b64decode(b'--__', '-_', validate=True), res) + self.assertEqual(base64.urlsafe_b64decode(b'++//'), res) + self.assertEqual(base64.urlsafe_b64decode(b'--__'), res) def test_b32encode(self): eq = self.assertEqual @@ -329,23 +334,33 @@ def test_b32decode_casefold(self): self.assertRaises(binascii.Error, base64.b32decode, b'me======') self.assertRaises(binascii.Error, base64.b32decode, 'me======') + def test_b32decode_map01(self): # Mapping zero and one - eq(base64.b32decode(b'MLO23456'), b'b\xdd\xad\xf3\xbe') - eq(base64.b32decode('MLO23456'), b'b\xdd\xad\xf3\xbe') - - map_tests = {(b'M1023456', b'L'): b'b\xdd\xad\xf3\xbe', - (b'M1023456', b'I'): b'b\x1d\xad\xf3\xbe', - } - for (data, map01), res in map_tests.items(): - data_str = data.decode('ascii') + eq = self.assertEqual + res_L = b'b\xdd\xad\xf3\xbe' + res_I = b'b\x1d\xad\xf3\xbe' + eq(base64.b32decode(b'MLO23456'), res_L) + eq(base64.b32decode('MLO23456'), res_L) + eq(base64.b32decode(b'MIO23456'), res_I) + eq(base64.b32decode('MIO23456'), res_I) + self.assertRaises(binascii.Error, base64.b32decode, b'M1023456') + self.assertRaises(binascii.Error, base64.b32decode, b'M1O23456') + self.assertRaises(binascii.Error, base64.b32decode, b'ML023456') + self.assertRaises(binascii.Error, base64.b32decode, b'MI023456') + + data = b'M1023456' + data_str = data.decode('ascii') + for map01, res in [(b'L', res_L), (b'I', res_I)]: map01_str = map01.decode('ascii') eq(base64.b32decode(data, map01=map01), res) eq(base64.b32decode(data_str, map01=map01), res) eq(base64.b32decode(data, map01=map01_str), res) eq(base64.b32decode(data_str, map01=map01_str), res) - self.assertRaises(binascii.Error, base64.b32decode, data) - self.assertRaises(binascii.Error, base64.b32decode, data_str) + + eq(base64.b32decode(b'M1O23456', map01=map01), res) + eq(base64.b32decode(b'M%c023456' % map01, map01=map01), res) + eq(base64.b32decode(b'M%cO23456' % map01, map01=map01), res) def test_b32decode_error(self): tests = [b'abc', b'ABCDEF==', b'==ABCDEF'] @@ -770,6 +785,19 @@ def test_a85decode_errors(self): self.assertRaises(ValueError, base64.a85decode, b'aaaay', foldspaces=True) + self.assertEqual(base64.a85decode(b"a b\nc", ignorechars=b" \n"), + b'\xc9\x89') + with self.assertRaises(ValueError): + base64.a85decode(b"a b\nc", ignorechars=b"") + with self.assertRaises(ValueError): + base64.a85decode(b"a b\nc", ignorechars=b" ") + with self.assertRaises(ValueError): + base64.a85decode(b"a b\nc", ignorechars=b"\n") + with self.assertRaises(TypeError): + base64.a85decode(b"a b\nc", ignorechars=" \n") + with self.assertRaises(TypeError): + base64.a85decode(b"a b\nc", ignorechars=None) + def test_b85decode_errors(self): illegal = list(range(33)) + \ list(b'"\',./:[\\]') + \ @@ -812,7 +840,7 @@ def test_decode_nonascii_str(self): self.assertRaises(ValueError, f, 'with non-ascii \xcb') def test_ErrorHeritage(self): - self.assertTrue(issubclass(binascii.Error, ValueError)) + self.assertIsSubclass(binascii.Error, ValueError) def test_RFC4648_test_cases(self): # test cases from RFC 4648 section 10 diff --git a/Lib/test/test_baseexception.py b/Lib/test/test_baseexception.py index e599b02c17d9c01..12d4088842b119e 100644 --- a/Lib/test/test_baseexception.py +++ b/Lib/test/test_baseexception.py @@ -10,13 +10,11 @@ class ExceptionClassTests(unittest.TestCase): inheritance hierarchy)""" def test_builtins_new_style(self): - self.assertTrue(issubclass(Exception, object)) + self.assertIsSubclass(Exception, object) def verify_instance_interface(self, ins): for attr in ("args", "__str__", "__repr__"): - self.assertTrue(hasattr(ins, attr), - "%s missing %s attribute" % - (ins.__class__.__name__, attr)) + self.assertHasAttr(ins, attr) def test_inheritance(self): # Make sure the inheritance hierarchy matches the documentation @@ -65,7 +63,7 @@ def test_inheritance(self): elif last_depth > depth: while superclasses[-1][0] >= depth: superclasses.pop() - self.assertTrue(issubclass(exc, superclasses[-1][1]), + self.assertIsSubclass(exc, superclasses[-1][1], "%s is not a subclass of %s" % (exc.__name__, superclasses[-1][1].__name__)) try: # Some exceptions require arguments; just skip them diff --git a/Lib/test/test_bigmem.py b/Lib/test/test_bigmem.py index 6d0879ad65d53c6..7f53379a952dfc1 100644 --- a/Lib/test/test_bigmem.py +++ b/Lib/test/test_bigmem.py @@ -10,7 +10,6 @@ from test import support from test.support import bigmemtest, _1G, _2G, _4G, import_helper -_testcapi = import_helper.import_module('_testcapi') import unittest import operator @@ -1264,6 +1263,7 @@ class ImmortalityTest(unittest.TestCase): def test_stickiness(self, size): """Check that immortality is "sticky", so that once an object is immortal it remains so.""" + _testcapi = import_helper.import_module('_testcapi') if size < _2G: # Not enough memory to cause immortality on overflow return diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index 1f3b6746ce4a626..82eabf4bb06b2ff 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -38,13 +38,13 @@ def assertConversion(self, original, converted, restored, **kwargs): def test_exceptions(self): # Check module exceptions - self.assertTrue(issubclass(binascii.Error, Exception)) - self.assertTrue(issubclass(binascii.Incomplete, Exception)) + self.assertIsSubclass(binascii.Error, Exception) + self.assertIsSubclass(binascii.Incomplete, Exception) def test_functions(self): # Check presence of all functions for name in all_functions: - self.assertTrue(hasattr(getattr(binascii, name), '__call__')) + self.assertHasAttr(getattr(binascii, name), '__call__') self.assertRaises(TypeError, getattr(binascii, name)) def test_returned_value(self): @@ -143,17 +143,16 @@ def assertExcessPadding(data, non_strict_mode_expected_result: bytes): _assertRegexTemplate(r'(?i)Excess padding', data, non_strict_mode_expected_result) # Test excess data exceptions - assertExcessData(b'ab==a', b'i') - assertExcessData(b'ab===', b'i') - assertExcessData(b'ab====', b'i') - assertExcessData(b'ab==:', b'i') - assertExcessData(b'abc=a', b'i\xb7') - assertExcessData(b'abc=:', b'i\xb7') - assertExcessData(b'ab==\n', b'i') - assertExcessData(b'abc==', b'i\xb7') - assertExcessData(b'abc===', b'i\xb7') - assertExcessData(b'abc====', b'i\xb7') - assertExcessData(b'abc=====', b'i\xb7') + assertExcessPadding(b'ab===', b'i') + assertExcessPadding(b'ab====', b'i') + assertNonBase64Data(b'ab==:', b'i') + assertExcessData(b'abc=a', b'i\xb7\x1a') + assertNonBase64Data(b'abc=:', b'i\xb7') + assertNonBase64Data(b'ab==\n', b'i') + assertExcessPadding(b'abc==', b'i\xb7') + assertExcessPadding(b'abc===', b'i\xb7') + assertExcessPadding(b'abc====', b'i\xb7') + assertExcessPadding(b'abc=====', b'i\xb7') # Test non-base64 data exceptions assertNonBase64Data(b'\nab==', b'i') @@ -175,6 +174,20 @@ def assertExcessPadding(data, non_strict_mode_expected_result: bytes): assertExcessPadding(b'abcd====', b'i\xb7\x1d') assertExcessPadding(b'abcd=====', b'i\xb7\x1d') + def test_base64_excess_data(self): + # Test excess data exceptions + def assertExcessData(data, expected): + assert_regex = r'(?i)Excess data' + data = self.type2test(data) + with self.assertRaisesRegex(binascii.Error, assert_regex): + binascii.a2b_base64(data, strict_mode=True) + self.assertEqual(binascii.a2b_base64(data, strict_mode=False), + expected) + self.assertEqual(binascii.a2b_base64(data), expected) + + assertExcessData(b'ab==c=', b'i\xb7') + assertExcessData(b'ab==cd', b'i\xb7\x1d') + assertExcessData(b'abc=d', b'i\xb7\x1d') def test_base64errors(self): # Test base64 with invalid padding @@ -227,6 +240,10 @@ def test_uu(self): self.assertEqual(binascii.a2b_uu(b"\xff"), b"\x00"*31) self.assertRaises(binascii.Error, binascii.a2b_uu, b"\xff\x00") self.assertRaises(binascii.Error, binascii.a2b_uu, b"!!!!") + self.assertRaises(binascii.Error, binascii.a2b_uu, + self.type2test(b"")) + self.assertRaises(binascii.Error, binascii.a2b_uu, + self.type2test(b"#86)C")[:0]) self.assertRaises(binascii.Error, binascii.b2a_uu, 46*b"!") # Issue #7701 (crash on a pydebug build) @@ -434,6 +451,9 @@ def test_empty_string(self): binascii.crc_hqx(empty, 0) continue f = getattr(binascii, func) + if func == 'a2b_uu': + self.assertRaises(binascii.Error, f, empty) + continue try: f(empty) except Exception as err: diff --git a/Lib/test/test_binop.py b/Lib/test/test_binop.py index 299af09c4983dfd..b224c3d4e6078e8 100644 --- a/Lib/test/test_binop.py +++ b/Lib/test/test_binop.py @@ -383,7 +383,7 @@ def test_comparison_orders(self): self.assertEqual(op_sequence(le, B, C), ['C.__ge__', 'B.__le__']) self.assertEqual(op_sequence(le, C, B), ['C.__le__', 'B.__ge__']) - self.assertTrue(issubclass(V, B)) + self.assertIsSubclass(V, B) self.assertEqual(op_sequence(eq, B, V), ['B.__eq__', 'V.__eq__']) self.assertEqual(op_sequence(le, B, V), ['B.__le__', 'V.__ge__']) diff --git a/Lib/test/test_bisect.py b/Lib/test/test_bisect.py index 97204d4cad38716..a7e1f533ff2adc1 100644 --- a/Lib/test/test_bisect.py +++ b/Lib/test/test_bisect.py @@ -391,9 +391,9 @@ class TestErrorHandlingC(TestErrorHandling, unittest.TestCase): class TestDocExample: def test_grades(self): - def grade(score, breakpoints=[60, 70, 80, 90], grades='FDCBA'): - i = self.module.bisect(breakpoints, score) - return grades[i] + def grade(score): + i = self.module.bisect([60, 70, 80, 90], score) + return "FDCBA"[i] result = [grade(score) for score in [33, 99, 77, 70, 89, 90, 100]] self.assertEqual(result, ['F', 'A', 'C', 'C', 'B', 'A', 'A']) diff --git a/Lib/test/test_buffer.py b/Lib/test/test_buffer.py index 61921e93e85e632..19582e757161fc1 100644 --- a/Lib/test/test_buffer.py +++ b/Lib/test/test_buffer.py @@ -2879,11 +2879,11 @@ def test_memoryview_tolist(self): def test_memoryview_repr(self): m = memoryview(bytearray(9)) r = m.__repr__() - self.assertTrue(r.startswith(" +).resolve() +MODULE_PATH = BASE_PATH / 'Tools' / 'build' / 'generate-build-details.py' + +try: + # Import "generate-build-details.py" as "generate_build_details" + spec = importlib.util.spec_from_file_location( + "generate_build_details", MODULE_PATH + ) + generate_build_details = importlib.util.module_from_spec(spec) + sys.modules["generate_build_details"] = generate_build_details + spec.loader.exec_module(generate_build_details) +except (FileNotFoundError, ImportError): + generate_build_details = None class FormatTestsBase: @@ -31,16 +53,15 @@ def key(self, name): value = value[part] return value - def test_parse(self): - self.data - def test_top_level_container(self): self.assertIsInstance(self.data, dict) for key, value in self.data.items(): with self.subTest(key=key): - if key in ('schema_version', 'base_prefix', 'base_interpreter', 'platform'): + if key in ('schema_version', 'base_prefix', 'base_interpreter', + 'platform'): self.assertIsInstance(value, str) - elif key in ('language', 'implementation', 'abi', 'suffixes', 'libpython', 'c_api', 'arbitrary_data'): + elif key in ('language', 'implementation', 'abi', 'suffixes', + 'libpython', 'c_api', 'arbitrary_data'): self.assertIsInstance(value, dict) def test_base_prefix(self): @@ -71,15 +92,20 @@ def test_language_version_info(self): self.assertEqual(len(value), sys.version_info.n_fields) for part_name, part_value in value.items(): with self.subTest(part=part_name): - self.assertEqual(part_value, getattr(sys.version_info, part_name)) + sys_version_value = getattr(sys.version_info, part_name) + self.assertEqual(part_value, sys_version_value) def test_implementation(self): + impl_ver = sys.implementation.version for key, value in self.key('implementation').items(): with self.subTest(part=key): if key == 'version': - self.assertEqual(len(value), len(sys.implementation.version)) + self.assertEqual(len(value), len(impl_ver)) for part_name, part_value in value.items(): - self.assertEqual(getattr(sys.implementation.version, part_name), part_value) + self.assertFalse(isinstance(sys.implementation.version, dict)) + getattr(sys.implementation.version, part_name) + sys_implementation_value = getattr(impl_ver, part_name) + self.assertEqual(sys_implementation_value, part_value) else: self.assertEqual(getattr(sys.implementation, key), value) @@ -91,7 +117,7 @@ def test_implementation(self): @unittest.skipIf(os.name != 'posix', 'Feature only implemented on POSIX right now') -@unittest.skipIf(is_wasi or is_emscripten, 'Feature not available on WebAssembly builds') +@unittest.skipIf(is_wasm32, 'Feature not available on WebAssembly builds') class CPythonBuildDetailsTests(unittest.TestCase, FormatTestsBase): """Test CPython's install details file implementation.""" @@ -99,7 +125,8 @@ class CPythonBuildDetailsTests(unittest.TestCase, FormatTestsBase): def location(self): if sysconfig.is_python_build(): projectdir = sysconfig.get_config_var('projectbase') - with open(os.path.join(projectdir, 'pybuilddir.txt')) as f: + pybuilddir = os.path.join(projectdir, 'pybuilddir.txt') + with open(pybuilddir, encoding='utf-8') as f: dirname = os.path.join(projectdir, f.read()) else: dirname = sysconfig.get_path('stdlib') @@ -107,7 +134,7 @@ def location(self): @property def contents(self): - with open(self.location, 'r') as f: + with open(self.location, 'r', encoding='utf-8') as f: return f.read() @needs_installed_python @@ -117,12 +144,94 @@ def test_location(self): # Override generic format tests with tests for our specific implemenation. @needs_installed_python - @unittest.skipIf(is_android or is_apple_mobile, 'Android and iOS run tests via a custom testbed method that changes sys.executable') + @unittest.skipIf( + is_android or is_apple_mobile, + 'Android and iOS run tests via a custom testbed method that changes sys.executable' + ) def test_base_interpreter(self): value = self.key('base_interpreter') + # Skip check if installation is relocated + if sysconfig._installation_is_relocated(): + self.skipTest("Installation is relocated") + self.assertEqual(os.path.realpath(value), os.path.realpath(sys.executable)) + @needs_installed_python + @unittest.skipIf( + is_android or is_apple_mobile, + "Android and iOS run tests via a custom testbed method that doesn't ship headers" + ) + def test_c_api(self): + value = self.key('c_api') + + # Skip check if installation is relocated + if sysconfig._installation_is_relocated(): + self.skipTest("Installation is relocated") + + self.assertTrue(os.path.exists(os.path.join(value['headers'], 'Python.h'))) + version = sysconfig.get_config_var('VERSION') + self.assertTrue(os.path.exists(os.path.join(value['pkgconfig_path'], f'python-{version}.pc'))) + + +@unittest.skipIf( + generate_build_details is None, + "Failed to import generate-build-details" +) +@unittest.skipIf(os.name != 'posix', 'Feature only implemented on POSIX right now') +@unittest.skipIf(is_wasm32, 'Feature not available on WebAssembly builds') +class BuildDetailsRelativePathsTests(unittest.TestCase): + @property + def build_details_absolute_paths(self): + data = generate_build_details.generate_data(schema_version='1.0') + return json.loads(json.dumps(data)) + + @property + def build_details_relative_paths(self): + data = self.build_details_absolute_paths + generate_build_details.make_paths_relative(data, config_path=None) + return data + + def test_round_trip(self): + data_abs_path = self.build_details_absolute_paths + data_rel_path = self.build_details_relative_paths + + self.assertEqual(data_abs_path['base_prefix'], + data_rel_path['base_prefix']) + + base_prefix = data_abs_path['base_prefix'] + + top_level_keys = ('base_interpreter',) + for key in top_level_keys: + self.assertEqual(key in data_abs_path, key in data_rel_path) + if key not in data_abs_path: + continue + + abs_rel_path = os.path.join(base_prefix, data_rel_path[key]) + abs_rel_path = os.path.normpath(abs_rel_path) + self.assertEqual(data_abs_path[key], abs_rel_path) + + second_level_keys = ( + ('libpython', 'dynamic'), + ('libpython', 'dynamic_stableabi'), + ('libpython', 'static'), + ('c_api', 'headers'), + ('c_api', 'pkgconfig_path'), + + ) + for part, key in second_level_keys: + self.assertEqual(part in data_abs_path, part in data_rel_path) + if part not in data_abs_path: + continue + self.assertEqual(key in data_abs_path[part], + key in data_rel_path[part]) + if key not in data_abs_path[part]: + continue + + abs_rel_path = os.path.join(base_prefix, data_rel_path[part][key]) + abs_rel_path = os.path.normpath(abs_rel_path) + self.assertEqual(data_abs_path[part][key], abs_rel_path) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 31597a320d418ad..54043eac2909169 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -293,6 +293,27 @@ def f_tuple(): self.assertEqual(overridden_outputs, ['all', 'any', 'tuple']) + def test_builtin_call_async_genexpr_no_crash(self): + async def f_all(): + return all(await 2 for _ in []) + + async def f_any(): + return any(await 2 for _ in []) + + async def f_tuple(): + return tuple(await 2 for _ in []) + + async def f_list(): + return list(await 2 for _ in []) + + async def f_set(): + return set(await 2 for _ in []) + + for f in (f_all, f_any, f_tuple, f_list, f_set): + with self.subTest(func=f.__name__): + with self.assertRaises(TypeError): + run_yielding_async_fn(f) + def test_ascii(self): self.assertEqual(ascii(''), '\'\'') self.assertEqual(ascii(0), '0') @@ -393,7 +414,7 @@ def test_chr(self): self.assertRaises(ValueError, chr, -2**1000) def test_cmp(self): - self.assertTrue(not hasattr(builtins, "cmp")) + self.assertNotHasAttr(builtins, "cmp") def test_compile(self): compile('print(1)\n', '', 'exec') @@ -436,7 +457,7 @@ def f(): """doc""" # test both direct compilation and compilation via AST codeobjs = [] codeobjs.append(compile(codestr, "", "exec", optimize=optval)) - tree = ast.parse(codestr) + tree = ast.parse(codestr, optimize=optval) codeobjs.append(compile(tree, "", "exec", optimize=optval)) for code in codeobjs: ns = {} @@ -624,7 +645,7 @@ def test_compile_ast(self): for opt in [opt1, opt2]: opt_right = opt.value.right self.assertIsInstance(opt_right, ast.Constant) - self.assertEqual(opt_right.value, True) + self.assertEqual(opt_right.value, __debug__) def test_delattr(self): sys.spam = 1 @@ -1120,6 +1141,7 @@ def test_filter_pickle(self): self.check_iter_pickle(f1, list(f2), proto) @support.skip_wasi_stack_overflow() + @support.skip_emscripten_stack_overflow() @support.requires_resource('cpu') def test_filter_dealloc(self): # Tests recursive deallocation of nested filter objects using the @@ -1182,6 +1204,16 @@ def __hash__(self): return self self.assertEqual(hash(Z(42)), hash(42)) + def test_invalid_hash_typeerror(self): + # GH-140406: The returned object from __hash__() would leak if it + # wasn't an integer. + class A: + def __hash__(self): + return 1.0 + + with self.assertRaises(TypeError): + hash(A()) + def test_hex(self): self.assertEqual(hex(16), '0x10') self.assertEqual(hex(-16), '-0x10') @@ -1373,6 +1405,22 @@ def test_map_strict(self): self.assertRaises(ValueError, tuple, map(pack, (1, 2), (1, 2), 'abc', strict=True)) + # gh-140517: Testing refleaks with mortal objects. + t1 = (None, object()) + t2 = (object(), object()) + t3 = (object(),) + + self.assertRaises(ValueError, tuple, + map(pack, t1, 'a', strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, t1, t2, 'a', strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, t1, t2, t3, strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, 'a', t1, strict=True)) + self.assertRaises(ValueError, tuple, + map(pack, 'a', t2, t3, strict=True)) + def test_map_strict_iterators(self): x = iter(range(5)) y = [0] @@ -2303,7 +2351,7 @@ def __format__(self, format_spec): # tests for object.__format__ really belong elsewhere, but # there's no good place to put them x = object().__format__('') - self.assertTrue(x.startswith(' None: + view.release() + def __index__(self): + self.ba.clear() + return ord("A") + + def make_case(): + ba = bytearray(b"A") + return ba, Evil(ba) + + for name in ("find", "count", "index", "rindex", "rfind"): + ba, evil = make_case() + with self.subTest(name): + with self.assertRaises(BufferError): + getattr(ba, name)(evil) + + ba, evil = make_case() + with self.assertRaises(BufferError): + evil in ba + with self.assertRaises(BufferError): + ba.split(evil) + with self.assertRaises(BufferError): + ba.rsplit(evil) + + def test_extend_empty_buffer_overflow(self): + # gh-143003 + class EvilIter: + def __iter__(self): + return self + def __next__(self): + return next(source) + def __length_hint__(self): + return 0 + + # Use ASCII digits so float() takes the fast path that expects a NUL terminator. + source = iter(b'42') + ba = bytearray() + ba.extend(EvilIter()) + + self.assertRaises(ValueError, float, bytearray()) + + def test_hex_use_after_free(self): + # Prevent UAF in bytearray.hex(sep) with re-entrant sep.__len__. + # Regression test for https://github.com/python/cpython/issues/143195. + ba = bytearray(b'\xAA') + + class S(bytes): + def __len__(self): + ba.clear() + return 1 + + self.assertRaises(BufferError, ba.hex, S(b':')) + class AssortedBytesTest(unittest.TestCase): # # Test various combinations of bytes and bytearray # + def test_bytes_repr(self, f=repr): + self.assertEqual(f(b''), "b''") + self.assertEqual(f(b"abc"), "b'abc'") + self.assertEqual(f(bytes([92])), r"b'\\'") + self.assertEqual(f(bytes([0, 1, 254, 255])), r"b'\x00\x01\xfe\xff'") + self.assertEqual(f(b'\a\b\t\n\v\f\r'), r"b'\x07\x08\t\n\x0b\x0c\r'") + self.assertEqual(f(b'"'), """b'"'""") # '"' + self.assertEqual(f(b"'"), '''b"'"''') # "'" + self.assertEqual(f(b"'\""), r"""b'\'"'""") # '\'"' + self.assertEqual(f(b"\"'\""), r"""b'"\'"'""") # '"\'"' + self.assertEqual(f(b"'\"'"), r"""b'\'"\''""") # '\'"\'' + self.assertEqual(f(BytesSubclass(b"abc")), "b'abc'") + + def test_bytearray_repr(self, f=repr): + self.assertEqual(f(bytearray()), "bytearray(b'')") + self.assertEqual(f(bytearray(b'abc')), "bytearray(b'abc')") + self.assertEqual(f(bytearray([92])), r"bytearray(b'\\')") + self.assertEqual(f(bytearray([0, 1, 254, 255])), + r"bytearray(b'\x00\x01\xfe\xff')") + self.assertEqual(f(bytearray([7, 8, 9, 10, 11, 12, 13])), + r"bytearray(b'\x07\x08\t\n\x0b\x0c\r')") + self.assertEqual(f(bytearray(b'"')), """bytearray(b'"')""") # '"' + self.assertEqual(f(bytearray(b"'")), r'''bytearray(b"\'")''') # "\'" + self.assertEqual(f(bytearray(b"'\"")), r"""bytearray(b'\'"')""") # '\'"' + self.assertEqual(f(bytearray(b"\"'\"")), r"""bytearray(b'"\'"')""") # '"\'"' + self.assertEqual(f(bytearray(b'\'"\'')), r"""bytearray(b'\'"\'')""") # '\'"\'' + self.assertEqual(f(ByteArraySubclass(b"abc")), "ByteArraySubclass(b'abc')") + self.assertEqual(f(ByteArraySubclass.Nested(b"abc")), "Nested(b'abc')") + self.assertEqual(f(ByteArraySubclass.Ŭñıçöđë(b"abc")), "Ŭñıçöđë(b'abc')") + @check_bytes_warnings - def test_repr_str(self): - for f in str, repr: - self.assertEqual(f(bytearray()), "bytearray(b'')") - self.assertEqual(f(bytearray([0])), "bytearray(b'\\x00')") - self.assertEqual(f(bytearray([0, 1, 254, 255])), - "bytearray(b'\\x00\\x01\\xfe\\xff')") - self.assertEqual(f(b"abc"), "b'abc'") - self.assertEqual(f(b"'"), '''b"'"''') # ''' - self.assertEqual(f(b"'\""), r"""b'\'"'""") # ' + def test_bytes_str(self): + self.test_bytes_repr(str) + + @check_bytes_warnings + def test_bytearray_str(self): + self.test_bytearray_repr(str) @check_bytes_warnings def test_format(self): @@ -1974,9 +2153,9 @@ def test_compare_bytes_to_bytearray(self): @test.support.requires_docstrings def test_doc(self): self.assertIsNotNone(bytearray.__doc__) - self.assertTrue(bytearray.__doc__.startswith("bytearray("), bytearray.__doc__) + self.assertStartsWith(bytearray.__doc__, "bytearray(") self.assertIsNotNone(bytes.__doc__) - self.assertTrue(bytes.__doc__.startswith("bytes("), bytes.__doc__) + self.assertStartsWith(bytes.__doc__, "bytes(") def test_from_bytearray(self): sample = bytes(b"Hello world\n\x80\x81\xfe\xff") @@ -1984,15 +2163,6 @@ def test_from_bytearray(self): b = bytearray(buf) self.assertEqual(b, bytearray(sample)) - @check_bytes_warnings - def test_to_str(self): - self.assertEqual(str(b''), "b''") - self.assertEqual(str(b'x'), "b'x'") - self.assertEqual(str(b'\x80'), "b'\\x80'") - self.assertEqual(str(bytearray(b'')), "bytearray(b'')") - self.assertEqual(str(bytearray(b'x')), "bytearray(b'x')") - self.assertEqual(str(bytearray(b'\x80')), "bytearray(b'\\x80')") - def test_literal(self): tests = [ (b"Wonderful spam", "Wonderful spam"), @@ -2097,17 +2267,24 @@ def fixtype(self, obj): contains_bytes = True + def test_mixed_cmp(self): + a = self.type2test(b'ab') + for t in bytes, bytearray, BytesSubclass, ByteArraySubclass: + with self.subTest(t.__name__): + self._assert_cmp(a, t(b'ab'), 0) + self._assert_cmp(a, t(b'a'), 1) + self._assert_cmp(a, t(b'ac'), -1) + class ByteArrayAsStringTest(FixedStringTest, unittest.TestCase): type2test = bytearray class BytesAsStringTest(FixedStringTest, unittest.TestCase): type2test = bytes - class SubclassTest: def test_basic(self): - self.assertTrue(issubclass(self.type2test, self.basetype)) + self.assertIsSubclass(self.type2test, self.basetype) self.assertIsInstance(self.type2test(), self.basetype) a, b = b"abcd", b"efgh" @@ -2155,7 +2332,7 @@ def test_pickle(self): self.assertEqual(a.z, b.z) self.assertEqual(type(a), type(b)) self.assertEqual(type(a.z), type(b.z)) - self.assertFalse(hasattr(b, 'y')) + self.assertNotHasAttr(b, 'y') def test_copy(self): a = self.type2test(b"abcd") @@ -2169,7 +2346,7 @@ def test_copy(self): self.assertEqual(a.z, b.z) self.assertEqual(type(a), type(b)) self.assertEqual(type(a.z), type(b.z)) - self.assertFalse(hasattr(b, 'y')) + self.assertNotHasAttr(b, 'y') def test_fromhex(self): b = self.type2test.fromhex('1a2B30') @@ -2200,7 +2377,10 @@ def __init__(me, *args, **kwargs): class ByteArraySubclass(bytearray): - pass + class Nested(bytearray): + pass + class Ŭñıçöđë(bytearray): + pass class ByteArraySubclassWithSlots(bytearray): __slots__ = ('x', 'y', '__dict__') @@ -2417,10 +2597,6 @@ def iconcat(b, a): # MODIFIES! b.wait() a += c - def irepeat(b, a): # MODIFIES! - b.wait() - a *= 2 - def subscript(b, a): b.wait() try: assert a[0] != 0xdd @@ -2501,6 +2677,10 @@ def zfill(b, a): c = a.zfill(0x400000) assert not c or c[-1] not in (0xdd, 0xcd) + def resize(b, a): # MODIFIES! + b.wait() + a.resize(10) + def check(funcs, a=None, *args): if a is None: a = bytearray(b'0' * 0x400000) @@ -2538,9 +2718,10 @@ def check(funcs, a=None, *args): check([clear] + [repeat] * 10) check([clear] + [iconcat] * 10) - check([clear] + [irepeat] * 10) check([clear] + [ass_subscript] * 10) check([clear] + [repr_] * 10) + # gh-148605: Do not test "a *= 2" since it allocates up to 4 GiB using + # 10 threads # value errors @@ -2562,6 +2743,8 @@ def check(funcs, a=None, *args): check([clear] + [startswith] * 10) check([clear] + [strip] * 10) + check([clear] + [resize] * 10) + check([clear] + [contains] * 10) check([clear] + [subscript] * 10) check([clear2] + [ass_subscript2] * 10, None, bytearray(b'0' * 0x400000)) @@ -2617,6 +2800,22 @@ def check(funcs, it): check([iter_next] + [iter_reduce] * 10, iter(ba)) # for tsan check([iter_next] + [iter_setstate] * 10, iter(ba)) # for tsan + @unittest.skipUnless(support.Py_GIL_DISABLED, 'this test can only possibly fail with GIL disabled') + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_free_threading_bytearray_resize(self): + def resize_stress(ba): + for _ in range(1000): + try: + ba.resize(1000) + ba.resize(1) + except (BufferError, ValueError): + pass + + ba = bytearray(100) + threads = [threading.Thread(target=resize_stress, args=(ba,)) for _ in range(4)] + with threading_helper.start_threads(threads): + pass if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index f32b24b39bad00f..64293d757331d75 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -66,18 +66,28 @@ class BaseTest(unittest.TestCase): EMPTY_DATA = b'BZh9\x17rE8P\x90\x00\x00\x00\x00' BAD_DATA = b'this is not a valid bzip2 file' - # Some tests need more than one block of uncompressed data. Since one block - # is at least 100,000 bytes, we gather some data dynamically and compress it. - # Note that this assumes that compression works correctly, so we cannot - # simply use the bigger test data for all tests. + # Some tests need more than one block of data. The bz2 module does not + # support flushing a block during compression, so we must read in data until + # there are at least 2 blocks. Since different orderings of Python files may + # be compressed differently, we need to check the compression output for + # more than one bzip2 block header magic, a hex encoding of Pi + # (0x314159265359) + bz2_block_magic = bytes.fromhex('314159265359') test_size = 0 - BIG_TEXT = bytearray(128*1024) + BIG_TEXT = b'' + BIG_DATA = b'' + compressor = BZ2Compressor(1) for fname in glob.glob(os.path.join(glob.escape(os.path.dirname(__file__)), '*.py')): with open(fname, 'rb') as fh: - test_size += fh.readinto(memoryview(BIG_TEXT)[test_size:]) - if test_size > 128*1024: + data = fh.read() + BIG_DATA += compressor.compress(data) + BIG_TEXT += data + # TODO(emmatyping): if it is impossible for a block header to cross + # multiple outputs, we can just search the output of each compress call + # which should be more efficient + if BIG_DATA.count(bz2_block_magic) > 1: + BIG_DATA += compressor.flush() break - BIG_DATA = bz2.compress(BIG_TEXT, compresslevel=1) def setUp(self): fd, self.filename = tempfile.mkstemp() @@ -184,7 +194,7 @@ def testPeek(self): with BZ2File(self.filename) as bz2f: pdata = bz2f.peek() self.assertNotEqual(len(pdata), 0) - self.assertTrue(self.TEXT.startswith(pdata)) + self.assertStartsWith(self.TEXT, pdata) self.assertEqual(bz2f.read(), self.TEXT) def testReadInto(self): @@ -768,7 +778,7 @@ def testPeekBytesIO(self): with BZ2File(bio) as bz2f: pdata = bz2f.peek() self.assertNotEqual(len(pdata), 0) - self.assertTrue(self.TEXT.startswith(pdata)) + self.assertStartsWith(self.TEXT, pdata) self.assertEqual(bz2f.read(), self.TEXT) def testWriteBytesIO(self): @@ -1022,6 +1032,21 @@ def test_failure(self): # Previously, a second call could crash due to internal inconsistency self.assertRaises(Exception, bzd.decompress, self.BAD_DATA * 30) + def test_decompress_after_data_error(self): + data = bytes.fromhex( + "425a6839314159265359000000000000007fffff000000000000000000000000" + "00000000000000000000000000000000000000e0370000000000000000000000" + "000000000000000000000000000000000000000000000000000083f3" + ) + bzd = BZ2Decompressor() + with self.assertRaisesRegex(OSError, "Invalid data stream"): + bzd.decompress(data) + # Previously, a second call could crash due to internal inconsistency + self.assertFalse(bzd.needs_input) + self.assertFalse(bzd.eof) + with self.assertRaisesRegex(ValueError, "previous error"): + bzd.decompress(b'\x00' * 18) + @support.refcount_test def test_refleaks_in___init__(self): gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') diff --git a/Lib/test/test_c_locale_coercion.py b/Lib/test/test_c_locale_coercion.py index e4b0b8c451fd456..10f8ba2255228b8 100644 --- a/Lib/test/test_c_locale_coercion.py +++ b/Lib/test/test_c_locale_coercion.py @@ -12,7 +12,9 @@ from test.support.script_helper import run_python_until_end -# Set the list of ways we expect to be able to ask for the "C" locale +# Set the list of ways we expect to be able to ask for the "C" locale. +# 'invalid.ascii' is an invalid LOCALE name and so should get turned in to the +# default locale, which is traditionally C. EXPECTED_C_LOCALE_EQUIVALENTS = ["C", "invalid.ascii"] # Set our expectation for the default encoding used in the C locale @@ -21,6 +23,7 @@ EXPECTED_C_LOCALE_FS_ENCODING = "ascii" # Set our expectation for the default locale used when none is specified +DEFAULT_LOCALE_IS_C = True EXPECT_COERCION_IN_DEFAULT_LOCALE = True TARGET_LOCALES = ["C.UTF-8", "C.utf8", "UTF-8"] @@ -30,12 +33,12 @@ # Android defaults to using UTF-8 for all system interfaces EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8" EXPECTED_C_LOCALE_FS_ENCODING = "utf-8" -elif sys.platform.startswith("linux"): - # Linux distros typically alias the POSIX locale directly to the C - # locale. - # TODO: Once https://bugs.python.org/issue30672 is addressed, we'll be - # able to check this case unconditionally - EXPECTED_C_LOCALE_EQUIVALENTS.append("POSIX") +elif support.linked_to_musl(): + # MUSL defaults to utf-8 unless the C locale is set explicitly. + EXPECTED_C_LOCALE_EQUIVALENTS = ["C"] + DEFAULT_LOCALE_IS_C = False + DEFAULT_ENCODING = 'utf-8' + EXPECT_COERCION_IN_DEFAULT_LOCALE = False elif sys.platform.startswith("aix"): # AIX uses iso8859-1 in the C locale, other *nix platforms use ASCII EXPECTED_C_LOCALE_STREAM_ENCODING = "iso8859-1" @@ -52,6 +55,11 @@ # VxWorks defaults to using UTF-8 for all system interfaces EXPECTED_C_LOCALE_STREAM_ENCODING = "utf-8" EXPECTED_C_LOCALE_FS_ENCODING = "utf-8" +if sys.platform.startswith("linux"): + # Linux recognizes POSIX as a synonym for C. Python will always coerce + # if the locale is set to POSIX, but not all platforms will use the + # C locale encodings if POSIX is set, so we'll only test it on linux. + EXPECTED_C_LOCALE_EQUIVALENTS.append("POSIX") # Note that the above expectations are still wrong in some cases, such as: # * Windows when PYTHONLEGACYWINDOWSFSENCODING is set @@ -362,9 +370,14 @@ def _check_c_locale_coercion(self, base_var_dict["PYTHONCOERCECLOCALE"] = coerce_c_locale # Check behaviour for the default locale + _fs_encoding = fs_encoding + _stream_encoding = stream_encoding + if not DEFAULT_LOCALE_IS_C and 'LC_ALL' not in extra_vars: + _fs_encoding = _stream_encoding = DEFAULT_ENCODING with self.subTest(default_locale=True, PYTHONCOERCECLOCALE=coerce_c_locale): - if EXPECT_COERCION_IN_DEFAULT_LOCALE: + if (EXPECT_COERCION_IN_DEFAULT_LOCALE + or (not DEFAULT_LOCALE_IS_C and 'LC_ALL' in extra_vars)): _expected_warnings = expected_warnings _coercion_expected = coercion_expected else: @@ -378,8 +391,8 @@ def _check_c_locale_coercion(self, _expected_warnings == [CLI_COERCION_WARNING]): _expected_warnings = None self._check_child_encoding_details(base_var_dict, - fs_encoding, - stream_encoding, + _fs_encoding, + _stream_encoding, None, _expected_warnings, _coercion_expected) diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index cbfee604b7a8b59..50ccf66b8dae6b4 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -457,12 +457,17 @@ def test_formatmonth(self): calendar.TextCalendar().formatmonth(0, 2), result_0_02_text ) + def test_formatmonth_with_invalid_month(self): with self.assertRaises(calendar.IllegalMonthError): calendar.TextCalendar().formatmonth(2017, 13) with self.assertRaises(calendar.IllegalMonthError): calendar.TextCalendar().formatmonth(2017, -1) + def test_illegal_month_error_bases(self): + self.assertIsSubclass(calendar.IllegalMonthError, ValueError) + self.assertIsSubclass(calendar.IllegalMonthError, IndexError) + def test_formatmonthname_with_year(self): self.assertEqual( calendar.HTMLCalendar().formatmonthname(2004, 1, withyear=True), @@ -504,6 +509,11 @@ def test_deprecation_warning(self): "The 'January' attribute is deprecated, use 'JANUARY' instead" ): calendar.January + with self.assertWarnsRegex( + DeprecationWarning, + "The 'February' attribute is deprecated, use 'FEBRUARY' instead" + ): + calendar.February def test_isleap(self): # Make sure that the return is right for a few years, and @@ -1098,7 +1108,7 @@ def test_option_type(self): output = run('--type', 'text', '2004') self.assertEqual(output, conv(result_2004_text)) output = run('--type', 'html', '2004') - self.assertEqual(output[:6], b'Calendar for 2004', output) def test_html_output_current_year(self): diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 185ae84dc4d19f1..d3af0eed893874e 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -3,7 +3,7 @@ set_recursion_limit, skip_on_s390x, skip_emscripten_stack_overflow, skip_wasi_stack_overflow, skip_if_sanitizer, - import_helper) + skip_if_huge_c_stack, import_helper) try: import _testcapi except ImportError: @@ -12,6 +12,10 @@ import _testlimitedcapi except ImportError: _testlimitedcapi = None +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None import struct import collections import itertools @@ -695,8 +699,8 @@ class DerivedType(SuperType): UnaffectedType2 = _testcapi.make_vectorcall_class(SuperType) # Aside: Quickly check that the C helper actually made derived types - self.assertTrue(issubclass(UnaffectedType1, DerivedType)) - self.assertTrue(issubclass(UnaffectedType2, SuperType)) + self.assertIsSubclass(UnaffectedType1, DerivedType) + self.assertIsSubclass(UnaffectedType2, SuperType) # Initial state: tp_call self.assertEqual(instance(), "tp_call") @@ -1037,10 +1041,28 @@ def test_unexpected_keyword_suggestion_via_getargs(self): @cpython_only class TestRecursion(unittest.TestCase): + def test_margin_is_sufficient(self): + + def get_sp(): + return _testinternalcapi.get_stack_pointer() + + this_sp = _testinternalcapi.get_stack_pointer() + lower_sp = _testcapi.pyobject_vectorcall(get_sp, (), ()) + if _testcapi._Py_STACK_GROWS_DOWN: + self.assertLess(lower_sp, this_sp) + safe_margin = this_sp - lower_sp + else: + self.assertLess(this_sp, lower_sp) + safe_margin = lower_sp - this_sp + # Add an (arbitrary) extra 20% for safety + safe_margin *= 6 / 5 + self.assertLess(safe_margin, _testinternalcapi.get_stack_margin()) + @skip_on_s390x @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") @skip_if_sanitizer("requires deep stack", thread=True) @unittest.skipIf(_testcapi is None, "requires _testcapi") + @skip_if_huge_c_stack(90_000) @skip_emscripten_stack_overflow() @skip_wasi_stack_overflow() def test_super_deep(self): @@ -1074,6 +1096,14 @@ def c_py_recurse(m): with self.assertRaises(RecursionError): c_py_recurse(100_000) + def test_recursion_with_kwargs(self): + # GH-137883: The interpreter forgot to check the recursion limit when + # calling with keywords. + def recurse_kw(a=0): + recurse_kw(a=0) + with self.assertRaises(RecursionError): + recurse_kw() + class TestFunctionWithManyArgs(unittest.TestCase): def test_function_with_many_args(self): diff --git a/Lib/test/test_capi/test_bytearray.py b/Lib/test/test_capi/test_bytearray.py index dfa98de9f007d88..52565ea34c61b89 100644 --- a/Lib/test/test_capi/test_bytearray.py +++ b/Lib/test/test_capi/test_bytearray.py @@ -66,6 +66,7 @@ def test_fromobject(self): # Test PyByteArray_FromObject() fromobject = _testlimitedcapi.bytearray_fromobject + self.assertEqual(fromobject(b''), bytearray(b'')) self.assertEqual(fromobject(b'abc'), bytearray(b'abc')) self.assertEqual(fromobject(bytearray(b'abc')), bytearray(b'abc')) self.assertEqual(fromobject(ByteArraySubclass(b'abc')), bytearray(b'abc')) @@ -115,6 +116,7 @@ def test_concat(self): self.assertEqual(concat(b'abc', bytearray(b'def')), bytearray(b'abcdef')) self.assertEqual(concat(bytearray(b'abc'), b''), bytearray(b'abc')) self.assertEqual(concat(b'', bytearray(b'def')), bytearray(b'def')) + self.assertEqual(concat(bytearray(b''), bytearray(b'')), bytearray(b'')) self.assertEqual(concat(memoryview(b'xabcy')[1:4], b'def'), bytearray(b'abcdef')) self.assertEqual(concat(b'abc', memoryview(b'xdefy')[1:4]), @@ -150,6 +152,10 @@ def test_resize(self): self.assertEqual(resize(ba, 0), 0) self.assertEqual(ba, bytearray()) + ba = bytearray(b'') + self.assertEqual(resize(ba, 0), 0) + self.assertEqual(ba, bytearray()) + ba = ByteArraySubclass(b'abcdef') self.assertEqual(resize(ba, 3), 0) self.assertEqual(ba, bytearray(b'abc')) diff --git a/Lib/test/test_capi/test_bytes.py b/Lib/test/test_capi/test_bytes.py index 5b61c73381542dc..bc820bd68d9e21d 100644 --- a/Lib/test/test_capi/test_bytes.py +++ b/Lib/test/test_capi/test_bytes.py @@ -22,6 +22,7 @@ def test_check(self): # Test PyBytes_Check() check = _testlimitedcapi.bytes_check self.assertTrue(check(b'abc')) + self.assertTrue(check(b'')) self.assertFalse(check('abc')) self.assertFalse(check(bytearray(b'abc'))) self.assertTrue(check(BytesSubclass(b'abc'))) @@ -36,6 +37,7 @@ def test_checkexact(self): # Test PyBytes_CheckExact() check = _testlimitedcapi.bytes_checkexact self.assertTrue(check(b'abc')) + self.assertTrue(check(b'')) self.assertFalse(check('abc')) self.assertFalse(check(bytearray(b'abc'))) self.assertFalse(check(BytesSubclass(b'abc'))) @@ -79,6 +81,7 @@ def test_fromobject(self): # Test PyBytes_FromObject() fromobject = _testlimitedcapi.bytes_fromobject + self.assertEqual(fromobject(b''), b'') self.assertEqual(fromobject(b'abc'), b'abc') self.assertEqual(fromobject(bytearray(b'abc')), b'abc') self.assertEqual(fromobject(BytesSubclass(b'abc')), b'abc') @@ -108,6 +111,7 @@ def test_asstring(self): self.assertEqual(asstring(b'abc', 4), b'abc\0') self.assertEqual(asstring(b'abc\0def', 8), b'abc\0def\0') + self.assertEqual(asstring(b'', 1), b'\0') self.assertRaises(TypeError, asstring, 'abc', 0) self.assertRaises(TypeError, asstring, object(), 0) @@ -120,6 +124,7 @@ def test_asstringandsize(self): self.assertEqual(asstringandsize(b'abc', 4), (b'abc\0', 3)) self.assertEqual(asstringandsize(b'abc\0def', 8), (b'abc\0def\0', 7)) + self.assertEqual(asstringandsize(b'', 1), (b'\0', 0)) self.assertEqual(asstringandsize_null(b'abc', 4), b'abc\0') self.assertRaises(ValueError, asstringandsize_null, b'abc\0def', 8) self.assertRaises(TypeError, asstringandsize, 'abc', 0) @@ -134,6 +139,7 @@ def test_repr(self): # Test PyBytes_Repr() bytes_repr = _testlimitedcapi.bytes_repr + self.assertEqual(bytes_repr(b'', 0), r"""b''""") self.assertEqual(bytes_repr(b'''abc''', 0), r"""b'abc'""") self.assertEqual(bytes_repr(b'''abc''', 1), r"""b'abc'""") self.assertEqual(bytes_repr(b'''a'b"c"d''', 0), r"""b'a\'b"c"d'""") @@ -163,6 +169,7 @@ def test_concat(self, concat=None): self.assertEqual(concat(b'', bytearray(b'def')), b'def') self.assertEqual(concat(memoryview(b'xabcy')[1:4], b'def'), b'abcdef') self.assertEqual(concat(b'abc', memoryview(b'xdefy')[1:4]), b'abcdef') + self.assertEqual(concat(b'', b''), b'') self.assertEqual(concat(b'abc', b'def', True), b'abcdef') self.assertEqual(concat(b'abc', bytearray(b'def'), True), b'abcdef') @@ -192,6 +199,7 @@ def test_decodeescape(self): """Test PyBytes_DecodeEscape()""" decodeescape = _testlimitedcapi.bytes_decodeescape + self.assertEqual(decodeescape(b''), b'') self.assertEqual(decodeescape(b'abc'), b'abc') self.assertEqual(decodeescape(br'\t\n\r\x0b\x0c\x00\\\'\"'), b'''\t\n\r\v\f\0\\'"''') diff --git a/Lib/test/test_capi/test_capsule.py b/Lib/test/test_capi/test_capsule.py new file mode 100644 index 000000000000000..981caf3fad426bd --- /dev/null +++ b/Lib/test/test_capi/test_capsule.py @@ -0,0 +1,181 @@ +import importlib +import os +import sys +import textwrap +import unittest +from test.support import import_helper, os_helper + +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') + + +class CapsuleImportTests(unittest.TestCase): + """Tests for PyCapsule_Import().""" + + @classmethod + def setUpClass(cls): + tmp = cls.tmp = cls.enterClassContext(os_helper.temp_dir()) + cls.enterClassContext(import_helper.DirsOnSysPath(tmp)) + cls.write_file(os.path.join(tmp, 'capsule_mod.py'), ''' + import _testlimitedcapi + + capsule = _testlimitedcapi.capsule_new('capsule_mod.capsule') + капсула = _testlimitedcapi.capsule_new('capsule_mod.капсула') + mismatched = _testlimitedcapi.capsule_new('other.name') + nonutf8 = _testlimitedcapi.capsule_new(b'capsule_mod.nonutf8\\xff') + nullname = _testlimitedcapi.capsule_new(None) + not_capsule = 42 + + class ns: + nested = _testlimitedcapi.capsule_new('capsule_mod.ns.nested') + + def __getattr__(name): + if name == 'bad_attr': + raise FloatingPointError('bad attribute') + raise AttributeError(name) + ''') + pkg = os.path.join(tmp, 'capsule_pkg') + os.mkdir(pkg) + cls.write_file(os.path.join(pkg, '__init__.py'), '') + cls.write_file(os.path.join(pkg, 'sub.py'), ''' + import _testlimitedcapi + + capsule = _testlimitedcapi.capsule_new('capsule_pkg.sub.capsule') + ''') + autopkg = os.path.join(tmp, 'capsule_autopkg') + os.mkdir(autopkg) + cls.write_file(os.path.join(autopkg, '__init__.py'), 'from . import sub\n') + cls.write_file(os.path.join(autopkg, 'sub.py'), ''' + import _testlimitedcapi + + capsule = _testlimitedcapi.capsule_new('capsule_autopkg.sub.capsule') + ''') + cls.write_file(os.path.join(tmp, 'capsule_broken.py'), '1/0\n') + importlib.invalidate_caches() + + def setUp(self): + for name in ('capsule_mod', 'capsule_pkg.sub', 'capsule_pkg', + 'capsule_autopkg.sub', 'capsule_autopkg', + 'capsule_broken'): + self.addCleanup(import_helper.unload, name) + + @staticmethod + def write_file(path, source): + with open(path, 'w', encoding='utf-8') as f: + f.write(textwrap.dedent(source)) + + def check_import(self, name, no_block=0): + # _testlimitedcapi.PyCapsule_Import() returns the name stored as the + # pointer by _testlimitedcapi.capsule_new(). + self.assertEqual(_testlimitedcapi.PyCapsule_Import(name, no_block), name) + + def test_import(self): + # The module is imported if not already imported. + self.assertNotIn('capsule_mod', sys.modules) + self.check_import('capsule_mod.capsule') + # Attributes after the first component are plain attribute lookups. + self.check_import('capsule_mod.ns.nested') + # Non-ASCII capsule and attribute name. + self.check_import('capsule_mod.капсула') + # The no_block argument is ignored. + self.check_import('capsule_mod.capsule', 1) + + @unittest.skipUnless(os_helper.TESTFN_NONASCII, + 'requires non-ASCII file name support') + def test_non_ascii_module_name(self): + name = os_helper.TESTFN_NONASCII + self.write_file(os.path.join(self.tmp, name + '.py'), f''' + import _testlimitedcapi + + capsule = _testlimitedcapi.capsule_new('{name}.capsule') + ''') + importlib.invalidate_caches() + self.addCleanup(import_helper.unload, name) + self.check_import(f'{name}.capsule') + + def test_submodule(self): + # Only the first component is imported; a submodule not imported + # by its package is not found. + self.assertRaises(AttributeError, + _testlimitedcapi.PyCapsule_Import, 'capsule_pkg.sub.capsule') + # It is found after explicit import. + importlib.import_module('capsule_pkg.sub') + self.check_import('capsule_pkg.sub.capsule') + # A submodule imported by its package is found. + self.check_import('capsule_autopkg.sub.capsule') + + def test_invalid_name(self): + pycapsule_import = _testlimitedcapi.PyCapsule_Import + # Non-existing module. + self.assertRaisesRegex(ImportError, + 'PyCapsule_Import could not import module "capsule_nonexistent"', + pycapsule_import, 'capsule_nonexistent.capsule') + # Non-UTF-8 module name. + self.assertRaisesRegex(ImportError, + 'PyCapsule_Import could not import module', + pycapsule_import, b'\xff\xfe.capsule') + # Empty module name. + self.assertRaisesRegex(ImportError, + 'PyCapsule_Import could not import module ""', + pycapsule_import, '.capsule_mod.capsule') + # Empty name. + self.assertRaisesRegex(ImportError, + 'PyCapsule_Import could not import module ""', + pycapsule_import, '') + # Only a dot. + self.assertRaisesRegex(ImportError, + 'PyCapsule_Import could not import module ""', + pycapsule_import, '.') + # Non-existing attribute. + self.assertRaises(AttributeError, + pycapsule_import, 'capsule_mod.nonexistent') + # Empty attribute name. + self.assertRaises(AttributeError, pycapsule_import, 'capsule_mod.') + # Consecutive dots. + self.assertRaises(AttributeError, + pycapsule_import, 'capsule_mod..capsule') + # Attribute of an object which is not a module. + self.assertRaises(AttributeError, + pycapsule_import, 'capsule_mod.not_capsule.capsule') + # No attribute name. + self.assertRaisesRegex(AttributeError, 'is not valid', + pycapsule_import, 'capsule_mod') + + # CRASHES pycapsule_import(NULL) + + def test_invalid_capsule(self): + pycapsule_import = _testlimitedcapi.PyCapsule_Import + # The attribute is not a capsule. + self.assertRaisesRegex(AttributeError, 'is not valid', + pycapsule_import, 'capsule_mod.not_capsule') + # The capsule name does not match the requested name. + self.assertRaisesRegex(AttributeError, 'is not valid', + pycapsule_import, 'capsule_mod.mismatched') + # The capsule name contains a byte not decodable from UTF-8. + self.assertRaisesRegex(AttributeError, 'is not valid', + pycapsule_import, 'capsule_mod.nonutf8') + # Even the exactly matching name fails: the attribute lookup + # requires a name decodable from UTF-8. + self.assertRaises(UnicodeDecodeError, + pycapsule_import, b'capsule_mod.nonutf8\xff') + # The capsule name is NULL. + self.assertRaisesRegex(AttributeError, 'is not valid', + pycapsule_import, 'capsule_mod.nullname') + + def test_error_from_import(self): + # The exception raised during importing the module is replaced + # with generic ImportError. + with self.assertRaises(ImportError) as cm: + _testlimitedcapi.PyCapsule_Import('capsule_broken.capsule') + self.assertEqual(str(cm.exception), + 'PyCapsule_Import could not import ' + 'module "capsule_broken"') + + def test_error_from_attribute_lookup(self): + self.assertRaises(FloatingPointError, + _testlimitedcapi.PyCapsule_Import, 'capsule_mod.bad_attr') + self.assertRaises(FloatingPointError, + _testlimitedcapi.PyCapsule_Import, 'capsule_mod.bad_attr.capsule') + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_capi/test_config.py b/Lib/test/test_capi/test_config.py index a2d70dd3af482df..ef155c070b202d8 100644 --- a/Lib/test/test_capi/test_config.py +++ b/Lib/test/test_capi/test_config.py @@ -10,6 +10,7 @@ from test.support import import_helper _testcapi = import_helper.import_module('_testcapi') +_testinternalcapi = import_helper.import_module('_testinternalcapi') # Is the Py_STATS macro defined? @@ -355,9 +356,11 @@ def expect_bool_not(value): for value in new_values: expected, expect_flag = expect_func(value) + old_flags = sys.flags config_set(name, value) self.assertEqual(config_get(name), expected) self.assertEqual(getattr(sys.flags, sys_flag), expect_flag) + self.assertIsNot(sys.flags, old_flags) if name == "write_bytecode": self.assertEqual(getattr(sys, "dont_write_bytecode"), expect_flag) @@ -375,6 +378,46 @@ def expect_bool_not(value): finally: config_set(name, old_value) + def test_config_set_global_vars(self): + # Test PyConfig_Set() with global configuration variables + config_get = _testcapi.config_get + config_set = _testcapi.config_set + get_configs = _testinternalcapi.get_configs + new_values = (0, 1, 5) + + for name, global_name, not_value in ( + ('bytes_warning', 'Py_BytesWarningFlag', False), + ('inspect', 'Py_InspectFlag', False), + ('interactive', 'Py_InteractiveFlag', False), + ('optimization_level', 'Py_OptimizeFlag', False), + ('parser_debug', 'Py_DebugFlag', False), + ('quiet', 'Py_QuietFlag', False), + ('use_environment', 'Py_IgnoreEnvironmentFlag', True), + ('verbose', 'Py_VerboseFlag', False), + ('write_bytecode', 'Py_DontWriteBytecodeFlag', True), + + # Read-only variables + #('buffered_stdio', 'Py_UnbufferedStdioFlag', True), + #('isolated', 'Py_IsolatedFlag', False), + #('pathconfig_warnings', 'Py_FrozenFlag', True), + #('site_import', 'Py_NoSiteFlag', True), + #('user_site_directory', 'Py_NoUserSiteDirectory', True), + # Windows only + #('legacy_windows_stdio', 'Py_LegacyWindowsStdioFlag', False) + ): + with self.subTest(name=name): + old_value = config_get(name) + try: + for value in new_values: + config_set(name, value) + global_config = get_configs()['global_config'] + expected = value + if not_value: + expected = int(not value) + self.assertEqual(global_config[global_name], expected) + finally: + config_set(name, old_value) + def test_config_set_cpu_count(self): config_get = _testcapi.config_get config_set = _testcapi.config_set diff --git a/Lib/test/test_capi/test_emscripten.py b/Lib/test/test_capi/test_emscripten.py new file mode 100644 index 000000000000000..272d9a10ceb950a --- /dev/null +++ b/Lib/test/test_capi/test_emscripten.py @@ -0,0 +1,25 @@ +import unittest +from test.support import is_emscripten + +if not is_emscripten: + raise unittest.SkipTest("Emscripten-only test") + +from _testinternalcapi import emscripten_set_up_async_input_device +from pathlib import Path + + +class EmscriptenAsyncInputDeviceTest(unittest.TestCase): + def test_emscripten_async_input_device(self): + jspi_supported = emscripten_set_up_async_input_device() + p = Path("/dev/blah") + self.addCleanup(p.unlink) + if not jspi_supported: + with open(p, "r") as f: + self.assertRaises(OSError, f.readline) + return + + with open(p, "r") as f: + for _ in range(10): + self.assertEqual(f.readline().strip(), "ab") + self.assertEqual(f.readline().strip(), "fi") + self.assertEqual(f.readline().strip(), "xy") diff --git a/Lib/test/test_capi/test_exceptions.py b/Lib/test/test_capi/test_exceptions.py index ade55338e63b69f..6a32201cce5e4aa 100644 --- a/Lib/test/test_capi/test_exceptions.py +++ b/Lib/test/test_capi/test_exceptions.py @@ -13,8 +13,9 @@ from .test_misc import decode_stderr -# Skip this test if the _testcapi module isn't available. +# Skip this test if the _testcapi or _testinternalcapi module isn't available. _testcapi = import_helper.import_module('_testcapi') +_testinternalcapi = import_helper.import_module('_testinternalcapi') NULL = None @@ -108,6 +109,26 @@ def __del__(self): b':7: RuntimeWarning: Testing PyErr_WarnEx', ]) + def test__pyerr_setkeyerror(self): + # Test _PyErr_SetKeyError() + _pyerr_setkeyerror = _testinternalcapi._pyerr_setkeyerror + for arg in ( + "key", + # check that a tuple argument is not unpacked + (1, 2, 3), + # PyErr_SetObject(exc_type, exc_value) uses exc_value if it's + # already an exception, but _PyErr_SetKeyError() always creates a + # new KeyError. + KeyError('arg'), + ): + with self.subTest(arg=arg): + with self.assertRaises(KeyError) as cm: + # Test calling _PyErr_SetKeyError() with an exception set + # to check that the function overrides the current + # exception. + _pyerr_setkeyerror(arg) + self.assertEqual(cm.exception.args, (arg,)) + class Test_FatalError(unittest.TestCase): diff --git a/Lib/test/test_capi/test_float.py b/Lib/test/test_capi/test_float.py index f7efe0d02549a3b..8b25607b6d504f7 100644 --- a/Lib/test/test_capi/test_float.py +++ b/Lib/test/test_capi/test_float.py @@ -28,6 +28,23 @@ NAN = float("nan") +def make_nan(size, sign, quiet, payload=None): + if size == 8: + payload_mask = 0x7ffffffffffff + i = (sign << 63) + (0x7ff << 52) + (quiet << 51) + elif size == 4: + payload_mask = 0x3fffff + i = (sign << 31) + (0xff << 23) + (quiet << 22) + elif size == 2: + payload_mask = 0x1ff + i = (sign << 15) + (0x1f << 10) + (quiet << 9) + else: + raise ValueError("size must be either 2, 4, or 8") + if payload is None: + payload = random.randint(not quiet, payload_mask) + return i + payload + + class CAPIFloatTest(unittest.TestCase): def test_check(self): # Test PyFloat_Check() @@ -197,16 +214,11 @@ def test_pack_unpack_roundtrip_for_nans(self): # PyFloat_Pack/Unpack*() API. See also gh-130317 and # e.g. https://developercommunity.visualstudio.com/t/155064 signaling = 0 - quiet = int(not signaling) - if size == 8: - payload = random.randint(signaling, 0x7ffffffffffff) - i = (sign << 63) + (0x7ff << 52) + (quiet << 51) + payload - elif size == 4: - payload = random.randint(signaling, 0x3fffff) - i = (sign << 31) + (0xff << 23) + (quiet << 22) + payload - elif size == 2: - payload = random.randint(signaling, 0x1ff) - i = (sign << 15) + (0x1f << 10) + (quiet << 9) + payload + if _testcapi.nan_msb_is_signaling: + # HP PA RISC and some MIPS CPUs use 0 for quiet, see: + # https://en.wikipedia.org/wiki/NaN#Encoding + signaling = 1 + i = make_nan(size, sign, not signaling) data = bytes.fromhex(f'{i:x}') for endian in (BIG_ENDIAN, LITTLE_ENDIAN): with self.subTest(data=data, size=size, endian=endian): @@ -216,6 +228,32 @@ def test_pack_unpack_roundtrip_for_nans(self): self.assertTrue(math.isnan(value)) self.assertEqual(data1, data2) + @unittest.skipUnless(HAVE_IEEE_754, "requires IEEE 754") + @unittest.skipUnless(sys.maxsize != 2147483647, "requires 64-bit mode") + def test_pack_unpack_nans_for_different_formats(self): + pack = _testcapi.float_pack + unpack = _testcapi.float_unpack + + for endian in (BIG_ENDIAN, LITTLE_ENDIAN): + with self.subTest(endian=endian): + byteorder = "big" if endian == BIG_ENDIAN else "little" + + # Convert sNaN to qNaN, if payload got truncated + data = make_nan(8, 0, False, 0x80001).to_bytes(8, byteorder) + snan_low = unpack(data, endian) + qnan4 = make_nan(4, 0, True, 0).to_bytes(4, byteorder) + qnan2 = make_nan(2, 0, True, 0).to_bytes(2, byteorder) + self.assertEqual(pack(4, snan_low, endian), qnan4) + self.assertEqual(pack(2, snan_low, endian), qnan2) + + # Preserve NaN type, if payload not truncated + data = make_nan(8, 0, False, 0x80000000001).to_bytes(8, byteorder) + snan_high = unpack(data, endian) + snan4 = make_nan(4, 0, False, 16384).to_bytes(4, byteorder) + snan2 = make_nan(2, 0, False, 2).to_bytes(2, byteorder) + self.assertEqual(pack(4, snan_high, endian), snan4) + self.assertEqual(pack(2, snan_high, endian), snan2) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_function.py b/Lib/test/test_capi/test_function.py index 9dca377e28ba42f..c1a278e5d4da916 100644 --- a/Lib/test/test_capi/test_function.py +++ b/Lib/test/test_capi/test_function.py @@ -307,10 +307,27 @@ def function_without_closure(): ... _testcapi.function_get_closure(function_without_closure), (1, 2)) self.assertEqual(function_without_closure.__closure__, (1, 2)) + def test_function_get_annotations(self): + # Test PyFunction_GetAnnotations() + def normal(): + pass + + def annofn(arg: int) -> str: + return f'arg = {arg}' + + annotations = _testcapi.function_get_annotations(normal) + self.assertIsNone(annotations) + + annotations = _testcapi.function_get_annotations(annofn) + self.assertIsInstance(annotations, dict) + self.assertEqual(annotations, annofn.__annotations__) + + with self.assertRaises(SystemError): + _testcapi.function_get_annotations(None) + # TODO: test PyFunction_New() # TODO: test PyFunction_NewWithQualName() # TODO: test PyFunction_SetVectorcall() - # TODO: test PyFunction_GetAnnotations() # TODO: test PyFunction_SetAnnotations() # TODO: test PyClassMethod_New() # TODO: test PyStaticMethod_New() diff --git a/Lib/test/test_capi/test_getargs.py b/Lib/test/test_capi/test_getargs.py index 67a8da7599511fb..2f047200dafa0b5 100644 --- a/Lib/test/test_capi/test_getargs.py +++ b/Lib/test/test_capi/test_getargs.py @@ -13,7 +13,7 @@ try: import _testinternalcapi except ImportError: - _testinternalcapi = NULL + _testinternalcapi = None # > How about the following counterproposal. This also changes some of # > the other format codes to be a little more regular. @@ -1389,123 +1389,6 @@ def test_nested_sequence(self): "argument 1 must be sequence of length 1, not 0"): parse(([],), {}, '(' + f + ')', ['a']) - def test_specific_type_errors(self): - parse = _testcapi.parse_tuple_and_keywords - - def check(format, arg, expected, got='list'): - errmsg = f'must be {expected}, not {got}' - with self.assertRaisesRegex(TypeError, errmsg): - parse((arg,), {}, format, ['a']) - - check('k', [], 'int') - check('k?', [], 'int or None') - check('K', [], 'int') - check('K?', [], 'int or None') - check('c', [], 'a byte string of length 1') - check('c?', [], 'a byte string of length 1 or None') - check('c', b'abc', 'a byte string of length 1', - 'a bytes object of length 3') - check('c?', b'abc', 'a byte string of length 1 or None', - 'a bytes object of length 3') - check('c', bytearray(b'abc'), 'a byte string of length 1', - 'a bytearray object of length 3') - check('c?', bytearray(b'abc'), 'a byte string of length 1 or None', - 'a bytearray object of length 3') - check('C', [], 'a unicode character') - check('C?', [], 'a unicode character or None') - check('C', 'abc', 'a unicode character', - 'a string of length 3') - check('C?', 'abc', 'a unicode character or None', - 'a string of length 3') - check('s', [], 'str') - check('s?', [], 'str or None') - check('z', [], 'str or None') - check('z?', [], 'str or None') - check('es', [], 'str') - check('es?', [], 'str or None') - check('es#', [], 'str') - check('es#?', [], 'str or None') - check('et', [], 'str, bytes or bytearray') - check('et?', [], 'str, bytes, bytearray or None') - check('et#', [], 'str, bytes or bytearray') - check('et#?', [], 'str, bytes, bytearray or None') - check('w*', [], 'read-write bytes-like object') - check('w*?', [], 'read-write bytes-like object or None') - check('S', [], 'bytes') - check('S?', [], 'bytes or None') - check('U', [], 'str') - check('U?', [], 'str or None') - check('Y', [], 'bytearray') - check('Y?', [], 'bytearray or None') - check('(OO)', 42, '2-item tuple', 'int') - check('(OO)?', 42, '2-item tuple or None', 'int') - check('(OO)', (1, 2, 3), 'tuple of length 2', '3') - - def test_nullable(self): - parse = _testcapi.parse_tuple_and_keywords - - def check(format, arg, allows_none=False): - # Because some format units (such as y*) require cleanup, - # we force the parsing code to perform the cleanup by adding - # an argument that always fails. - # By checking for an exception, we ensure that the parsing - # of the first argument was successful. - self.assertRaises(OverflowError, parse, - (arg, 256), {}, format + '?b', ['a', 'b']) - self.assertRaises(OverflowError, parse, - (None, 256), {}, format + '?b', ['a', 'b']) - self.assertRaises(OverflowError, parse, - (arg, 256), {}, format + 'b', ['a', 'b']) - self.assertRaises(OverflowError if allows_none else TypeError, parse, - (None, 256), {}, format + 'b', ['a', 'b']) - - check('b', 42) - check('B', 42) - check('h', 42) - check('H', 42) - check('i', 42) - check('I', 42) - check('n', 42) - check('l', 42) - check('k', 42) - check('L', 42) - check('K', 42) - check('f', 2.5) - check('d', 2.5) - check('D', 2.5j) - check('c', b'a') - check('C', 'a') - check('p', True, allows_none=True) - check('y', b'buffer') - check('y*', b'buffer') - check('y#', b'buffer') - check('s', 'string') - check('s*', 'string') - check('s#', 'string') - check('z', 'string', allows_none=True) - check('z*', 'string', allows_none=True) - check('z#', 'string', allows_none=True) - check('w*', bytearray(b'buffer')) - check('U', 'string') - check('S', b'bytes') - check('Y', bytearray(b'bytearray')) - check('O', object, allows_none=True) - - check('(OO)', (1, 2)) - self.assertEqual(parse((((1, 2), 3),), {}, '((OO)?O)', ['a']), (1, 2, 3)) - self.assertEqual(parse(((None, 3),), {}, '((OO)?O)', ['a']), (NULL, NULL, 3)) - self.assertEqual(parse((((1, 2), 3),), {}, '((OO)O)', ['a']), (1, 2, 3)) - self.assertRaises(TypeError, parse, ((None, 3),), {}, '((OO)O)', ['a']) - - parse((None,), {}, 'es?', ['a']) - parse((None,), {}, 'es#?', ['a']) - parse((None,), {}, 'et?', ['a']) - parse((None,), {}, 'et#?', ['a']) - parse((None,), {}, 'O!?', ['a']) - parse((None,), {}, 'O&?', ['a']) - - # TODO: More tests for es?, es#?, et?, et#?, O!, O& - @unittest.skipIf(_testinternalcapi is None, 'needs _testinternalcapi') def test_gh_119213(self): rc, out, err = script_helper.assert_python_ok("-c", """if True: diff --git a/Lib/test/test_capi/test_list.py b/Lib/test/test_capi/test_list.py index 67ed5d0b4f8722b..b95b8ba960bd8b2 100644 --- a/Lib/test/test_capi/test_list.py +++ b/Lib/test/test_capi/test_list.py @@ -350,6 +350,10 @@ def test_list_extend(self): # CRASHES list_extend(NULL, []) # CRASHES list_extend([], NULL) + def test_uninitialized_list_repr(self): + lst = _testlimitedcapi.list_new(3) + self.assertEqual(repr(lst), '[, , ]') + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_long.py b/Lib/test/test_capi/test_long.py index d3156645eeec2d6..fc0454b71cb780d 100644 --- a/Lib/test/test_capi/test_long.py +++ b/Lib/test/test_capi/test_long.py @@ -803,6 +803,16 @@ def to_digits(num): self.assertEqual(pylongwriter_create(negative, digits), num, (negative, digits)) + def test_bug_143050(self): + with support.adjust_int_max_str_digits(0): + # Bug coming from using _pylong.int_from_string(), that + # currently requires > 6000 decimal digits. + int('-' + '0' * 7000, 10) + _testcapi.test_immortal_small_ints() + # Test also nonzero small int + int('-' + '0' * 7000 + '123', 10) + _testcapi.test_immortal_small_ints() + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_mem.py b/Lib/test/test_capi/test_mem.py index 5035b2b4829bf64..e6389b4e0c1e924 100644 --- a/Lib/test/test_capi/test_mem.py +++ b/Lib/test/test_capi/test_mem.py @@ -24,9 +24,12 @@ def check(self, code): out = assert_python_failure( '-c', code, PYTHONMALLOC=self.PYTHONMALLOC, - # FreeBSD: instruct jemalloc to not fill freed() memory - # with junk byte 0x5a, see JEMALLOC(3) + # Instruct the system allocator to not fill freed() memory + # with junk bytes: + # FreeBSD: jemalloc, see JEMALLOC(3). MALLOC_CONF="junk:false", + # OpenBSD: see MALLOC.CONF(5). + MALLOC_OPTIONS="j", ) stderr = out.err return stderr.decode('ascii', 'replace') @@ -102,7 +105,9 @@ def check_pyobject_is_freed(self, func_name): assert_python_ok( '-c', code, PYTHONMALLOC=self.PYTHONMALLOC, + # See the comment in check() above. MALLOC_CONF="junk:false", + MALLOC_OPTIONS="j", ) def test_pyobject_null_is_freed(self): @@ -117,9 +122,7 @@ def test_pyobject_forbidden_bytes_is_freed(self): def test_pyobject_freed_is_freed(self): self.check_pyobject_is_freed('check_pyobject_freed_is_freed') - # Python built with Py_TRACE_REFS fail with a fatal error in - # _PyRefchain_Trace() on memory allocation error. - @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + @support.nomemtest def test_set_nomemory(self): code = """if 1: import _testcapi diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index a597f23a992e7b2..e71c1f6f5541f97 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -412,10 +412,14 @@ def test_trashcan_subclass(self): L = MyList((L,)) @support.requires_resource('cpu') + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_trashcan_python_class1(self): self.do_test_trashcan_python_class(list) @support.requires_resource('cpu') + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_trashcan_python_class2(self): from _testcapi import MyList self.do_test_trashcan_python_class(MyList) @@ -911,6 +915,18 @@ def genf(): yield gen = genf() self.assertEqual(_testcapi.gen_get_code(gen), gen.gi_code) + def test_tp_bases_slot(self): + cls = _testcapi.HeapCTypeWithBasesSlot + self.assertEqual(cls.__bases__, (int,)) + self.assertEqual(cls.__base__, int) + + def test_tp_bases_slot_none(self): + self.assertRaisesRegex( + SystemError, + "Py_tp_bases is not a tuple", + _testcapi.create_heapctype_with_none_bases_slot + ) + @requires_limited_api class TestHeapTypeRelative(unittest.TestCase): diff --git a/Lib/test/test_capi/test_object.py b/Lib/test/test_capi/test_object.py index 127862546b1bcef..0b188fc90597744 100644 --- a/Lib/test/test_capi/test_object.py +++ b/Lib/test/test_capi/test_object.py @@ -144,11 +144,17 @@ class EnableDeferredRefcountingTest(unittest.TestCase): @support.requires_resource("cpu") def test_enable_deferred_refcount(self): from threading import Thread + import gc self.assertEqual(_testcapi.pyobject_enable_deferred_refcount("not tracked"), 0) foo = [] self.assertEqual(_testcapi.pyobject_enable_deferred_refcount(foo), int(support.Py_GIL_DISABLED)) + # The object must be tracked by the GC + not_gc_tracked = tuple([1, 2]) + self.assertFalse(gc.is_tracked(not_gc_tracked)) + self.assertEqual(_testcapi.pyobject_enable_deferred_refcount(not_gc_tracked), 0) + # Make sure reference counting works on foo now self.assertEqual(foo, []) if support.Py_GIL_DISABLED: @@ -221,6 +227,7 @@ def test_decref_freed_object(self): """ self.check_negative_refcount(code) + @support.requires_resource('cpu') def test_decref_delayed(self): # gh-130519: Test that _PyObject_XDecRefDelayed() and QSBR code path # handles destructors that are possibly re-entrant or trigger a GC. @@ -246,5 +253,12 @@ def func(x): func(object()) + # Test that a newly created object in C is not considered + # a uniquely referenced temporary, because it's not on the stack. + # gh-142586: do the test in a loop over a list to test for handling + # tagged ints on the stack. + for i in [0, 1, 2]: + self.assertFalse(_testcapi.pyobject_is_unique_temporary_new_object()) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index ba7bcb4540a4517..be3f8895db5d093 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -1942,6 +1942,77 @@ def testfunc(n): self.assertNotIn("_COMPARE_OP_INT", uops) self.assertNotIn("_GUARD_IS_TRUE_POP", uops) + def test_attr_promotion_failure(self): + # We're not testing for any specific uops here, just + # testing it doesn't crash. + script_helper.assert_python_ok('-c', textwrap.dedent(""" + import _testinternalcapi + import _opcode + import email + + def get_first_executor(func): + code = func.__code__ + co_code = code.co_code + for i in range(0, len(co_code), 2): + try: + return _opcode.get_executor(code, i) + except ValueError: + pass + return None + + def testfunc(n): + for _ in range(n): + email.jit_testing = None + prompt = email.jit_testing + del email.jit_testing + + + testfunc(_testinternalcapi.TIER2_THRESHOLD) + ex = get_first_executor(testfunc) + assert ex is not None + """)) + + def test_interpreter_finalization_with_generator_alive(self): + script_helper.assert_python_ok("-c", textwrap.dedent(""" + import sys + t = tuple(range(%d)) + def simple_for(): + for x in t: + x + + def gen(): + try: + yield + except: + simple_for() + + sys.settrace(lambda *args: None) + simple_for() + g = gen() + next(g) + """ % _testinternalcapi.SPECIALIZATION_THRESHOLD)) + + def test_next_instr_for_exception_handler_set(self): + # gh-140104: We just want the exception to be caught properly. + def f(): + for i in range(TIER2_THRESHOLD + 3): + try: + undefined_variable(i) + except Exception: + pass + + f() + + def test_next_instr_for_exception_handler_set_lasts_instr(self): + # gh-140104: We just want the exception to be caught properly. + def f(): + a_list = [] + for _ in range(TIER2_THRESHOLD + 3): + try: + a_list[""] = 0 + except Exception: + pass + def global_identity(x): return x diff --git a/Lib/test/test_capi/test_run.py b/Lib/test/test_capi/test_run.py index 894f66b437a39c2..7796c589b557c72 100644 --- a/Lib/test/test_capi/test_run.py +++ b/Lib/test/test_capi/test_run.py @@ -1,48 +1,82 @@ +import ast import os +import sys +import tempfile +import textwrap import unittest from collections import UserDict +from test import support from test.support import import_helper from test.support.os_helper import unlink, TESTFN, TESTFN_ASCII, TESTFN_UNDECODABLE -NULL = None _testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') + + +NULL = None Py_single_input = _testcapi.Py_single_input Py_file_input = _testcapi.Py_file_input Py_eval_input = _testcapi.Py_eval_input +INVALID_START = Py_single_input - 1 +STDIN = '' +STDERR_FD = 2 +PyCF_ONLY_AST = _testcapi.PyCF_ONLY_AST +PyCF_IGNORE_COOKIE = _testcapi.PyCF_IGNORE_COOKIE + +# Code raising a SyntaxError +SYNTAX_ERROR = 'True = 1' + +# Raise a SystemExit with exit code 42 +CODE_EXIT_42 = 'raise SystemExit(42)' + + +def create_text_file(filename, text): + with open(filename, 'w', encoding='utf8') as fp: + fp.write(text) class DictSubclass(dict): pass -class CAPITest(unittest.TestCase): - # TODO: Test the following functions: - # - # PyRun_SimpleStringFlags - # PyRun_AnyFileExFlags - # PyRun_SimpleFileExFlags - # PyRun_InteractiveOneFlags - # PyRun_InteractiveOneObject - # PyRun_InteractiveLoopFlags - # PyRun_String (may be a macro) - # PyRun_AnyFile (may be a macro) - # PyRun_AnyFileEx (may be a macro) - # PyRun_AnyFileFlags (may be a macro) - # PyRun_SimpleString (may be a macro) - # PyRun_SimpleFile (may be a macro) - # PyRun_SimpleFileEx (may be a macro) - # PyRun_InteractiveOne (may be a macro) - # PyRun_InteractiveLoop (may be a macro) - # PyRun_File (may be a macro) - # PyRun_FileEx (may be a macro) - # PyRun_FileFlags (may be a macro) +class capture_excepthook: + def __init__(self): + self.exc = None + self._old_hook = None - def test_run_stringflags(self): - # Test PyRun_StringFlags(). - # XXX: fopen() uses different path encoding than Python on Windows. + def _hook(self, exc_type, exc_value, exc_tb): + # Storing the exception instance creates a reference cycle. + self.exc = exc_value + + def __enter__(self): + self._old_hook = sys.excepthook + sys.excepthook = self._hook + return self + + def __exit__(self, *exc_info): + sys.excepthook = self._old_hook + self.exc = None + + +class CAPITest(unittest.TestCase): + def check_run_string(self, func): def run(s, *args): - return _testcapi.run_stringflags(s, Py_file_input, *args) - source = b'a\n' + return func(s, Py_file_input, *args) + + ns = {} + self.assertIsNone(run(b'x = 1', ns, ns)) + self.assertEqual(ns['x'], 1) + + with self.assertRaises(SyntaxError) as cm: + run(SYNTAX_ERROR.encode(), {}) + self.assertEqual(cm.exception.filename, '') + + with self.assertRaises(ValueError) as cm: + run(b'raise ValueError("BUG")', {}) + self.assertEqual(str(cm.exception), 'BUG') + + with self.assertRaises(ValueError): + func(b'x = 1', INVALID_START, {}) self.assertIsNone(run(b'a\n', dict(a=1))) self.assertIsNone(run(b'a\n', dict(a=1), {})) @@ -70,20 +104,32 @@ def run(s, *args): # CRASHES run(NULL, {}) - def test_run_fileexflags(self): - # Test PyRun_FileExFlags(). + def test_run_string(self): + # Test PyRun_String() + self.check_run_string(_testcapi.run_string) + + def test_run_stringflags(self): + # Test PyRun_StringFlags(). + self.check_run_string(_testcapi.run_stringflags) + + def check_run_file(self, func, support_closeit): + # XXX: fopen() uses different path encoding than Python on Windows. filename = os.fsencode(TESTFN if os.name != 'nt' else TESTFN_ASCII) - with open(filename, 'wb') as fp: - fp.write(b'a\n') + create_text_file(filename, 'a\n') self.addCleanup(unlink, filename) def run(*args): - return _testcapi.run_fileexflags(filename, Py_file_input, *args) + return func(filename, Py_file_input, *args) self.assertIsNone(run(dict(a=1))) self.assertIsNone(run(dict(a=1), {})) self.assertIsNone(run({}, dict(a=1))) self.assertIsNone(run({}, UserDict(a=1))) - self.assertIsNone(run(dict(a=1), {}, 1)) # closeit = True + if support_closeit: + closeit = 1 + self.assertIsNone(run(dict(a=1), {}, closeit)) + + with self.assertRaises(ValueError): + func(filename, INVALID_START, {}) self.assertRaises(NameError, run, {}) self.assertRaises(NameError, run, {}, {}) @@ -101,17 +147,289 @@ def run(*args): self.assertRaises(SystemError, run, UserDict(), {}) self.assertRaises(SystemError, run, UserDict(), dict(a=1)) - @unittest.skipUnless(TESTFN_UNDECODABLE, 'only works if there are undecodable paths') - @unittest.skipIf(os.name == 'nt', 'does not work on Windows') - def test_run_fileexflags_with_undecodable_filename(self): - run = _testcapi.run_fileexflags + # syntax error + create_text_file(filename, SYNTAX_ERROR) + with self.assertRaises(SyntaxError) as cm: + self.assertIsNone(run({})) + self.assertEqual(cm.exception.filename, os.fsdecode(filename)) + + # raise exception + create_text_file(filename, 'raise ValueError("BUG")') + with self.assertRaises(ValueError) as cm: + self.assertIsNone(run({})) + self.assertEqual(str(cm.exception), 'BUG') + + # Test undecodable filename + if TESTFN_UNDECODABLE and os.name != 'nt': + try: + create_text_file(TESTFN_UNDECODABLE, 'a\n') + self.addCleanup(unlink, TESTFN_UNDECODABLE) + except OSError: + # undecodable paths are not supported + pass + else: + self.assertIsNone(func(TESTFN_UNDECODABLE, Py_file_input, dict(a=1))) + + def test_run_file(self): + # Test PyRun_File(). + self.check_run_file(_testcapi.run_file, False) + + def test_run_fileex(self): + # Test PyRun_FileEx(). + self.check_run_file(_testcapi.run_fileex, True) + + def test_run_fileflags(self): + # Test PyRun_FileFlags(). + self.check_run_file(_testcapi.run_fileflags, True) + + def test_run_fileexflags(self): + # Test PyRun_FileExFlags(). + self.check_run_file(_testcapi.run_fileexflags, True) + + def check_run_simplefile(self, func_name, have_closeit): + run = getattr(_testcapi, func_name) + + open_filename = TESTFN + with open(open_filename, 'w') as fp: + print('import sys', file=fp) + print('print(__file__)', file=fp) + print('mod = sys.modules["__main__"]', file=fp) + print('print(type(mod.__loader__).__name__)', file=fp) + self.addCleanup(unlink, open_filename) + + class MockLoader: + pass + loader = MockLoader() + + for closeit in (0, 1): + for set_file in (False, True): + for filename in (open_filename, 'custom filename', STDIN): + set_loader = (filename != STDIN) + with self.subTest(closeit=closeit, + set_file=set_file, + filename=filename): + mod = sys.modules['__main__'] + mod_file = mod.__file__ + with (support.captured_stdout() as stdout, + support.swap_attr(mod, '__loader__', loader)): + try: + if set_file: + del mod.__file__ + filename_arg = os.fsencode(filename) + if have_closeit: + res = run(open_filename, filename_arg, closeit) + else: + res = run(open_filename, filename_arg) + if set_file: + self.assertFalse(hasattr(mod, '__file__')) + finally: + mod.__file__ = mod_file + + self.assertEqual(res, 0) + expected = [filename if set_file else mod_file] + if set_loader: + expected.append('SourceFileLoader') + else: + expected.append('MockLoader') + self.assertEqual(stdout.getvalue().splitlines(), + expected) + + def test_run_simplefile(self): + # Test PyRun_SimpleFile() + self.check_run_simplefile('run_simplefile', False) + + def test_run_simplefileex(self): + # Test PyRun_SimpleFileEx() + self.check_run_simplefile('run_simplefileex', True) + + def test_run_simplefileexflags(self): + # Test PyRun_SimpleFileExFlags() + self.check_run_simplefile('run_simplefileexflags', True) + + def test_run_anyfile(self): + # Test PyRun_AnyFile() + self.check_run_simplefile('run_anyfile', False) + + def test_run_anyfileex(self): + # Test PyRun_AnyFileEx() + self.check_run_simplefile('run_anyfileex', True) + + def test_run_anyfileflags(self): + # Test PyRun_AnyFileFlags() + self.check_run_simplefile('run_anyfileflags', True) + + def _check_run_interactive(self, run, encode_filename, use_loop): + open_filename = TESTFN + if use_loop: + with open(open_filename, 'w') as fp: + print('welcome = "hello REPL"', file=fp) + print('print(welcome)', file=fp) + else: + create_text_file(open_filename, 'print("hello REPL")') + self.addCleanup(unlink, open_filename) + + for filename in ('filename', STDIN): + with self.subTest(filename=filename): + with support.captured_stdout() as stdout: + if encode_filename: + filename = filename.encode() + self.assertEqual(run(open_filename, filename), 0) + self.assertEqual(stdout.getvalue(), 'hello REPL\n') + + create_text_file(open_filename, SYNTAX_ERROR) + filename = 'custom filename' + if encode_filename: + filename_arg = filename.encode() + else: + filename_arg = filename + with capture_excepthook() as excepthook: + expected = (0 if use_loop else -1) + self.assertEqual(run(open_filename, filename_arg), expected) + self.assertIsInstance(excepthook.exc, SyntaxError) + self.assertEqual(excepthook.exc.filename, filename) + + create_text_file(open_filename, 'raise ValueError("BUG")') + with capture_excepthook() as excepthook: + expected = (0 if use_loop else -1) + self.assertEqual(run(open_filename, filename_arg), expected) + self.assertIsInstance(excepthook.exc, ValueError) + self.assertEqual(str(excepthook.exc), 'BUG') + + if not encode_filename: + # wrong type for the second parameter (filename) + with capture_excepthook() as excepthook: + self.assertEqual(run(open_filename, b'bytes'), -1) + self.assertIsInstance(excepthook.exc, TypeError) + + def check_run_interactive(self, run, encode_filename, use_loop=False): + # Redirect stderr to a temporary file to hide '>>> ' from the REPL try: - with open(TESTFN_UNDECODABLE, 'wb') as fp: - fp.write(b'a\n') - self.addCleanup(unlink, TESTFN_UNDECODABLE) + stderr_copy = os.dup(STDERR_FD) except OSError: - self.skipTest('undecodable paths are not supported') - self.assertIsNone(run(TESTFN_UNDECODABLE, Py_file_input, dict(a=1))) + # On WASI, dup(STDERR_FD) fails with "OSError: [Errno 58] Not + # supported". In this case, run the test without redirecting + # stderr to a temporary file. + self._check_run_interactive(run, encode_filename, use_loop) + return + + with tempfile.TemporaryFile() as tmp: + try: + os.dup2(tmp.fileno(), STDERR_FD) + self._check_run_interactive(run, encode_filename, use_loop) + finally: + os.dup2(stderr_copy, STDERR_FD) + os.close(stderr_copy) + + def test_run_interactiveone(self): + # Test PyRun_InteractiveOne() + run = _testcapi.run_interactiveone + self.check_run_interactive(run, True) + + def test_run_interactiveoneflags(self): + # Test PyRun_InteractiveOneFlags() + run = _testcapi.run_interactiveoneflags + self.check_run_interactive(run, True) + + def test_run_interactiveoneobject(self): + # Test PyRun_InteractiveOneObject() + run = _testcapi.run_interactiveoneobject + self.check_run_interactive(run, False) + + def test_run_interactiveloop(self): + # Test PyRun_InteractiveLoop() + run = _testcapi.run_interactiveloop + self.check_run_interactive(run, True, use_loop=True) + + def test_run_interactiveloopflags(self): + # Test PyRun_InteractiveLoopFlags() + run = _testcapi.run_interactiveloopflags + self.check_run_interactive(run, True, use_loop=True) + + def test_run_anyfileexflags(self): + # Test PyRun_AnyFileExFlags() + self.check_run_simplefile('run_anyfileexflags', True) + + def check_run_simplestring(self, run): + with support.captured_stdout() as stdout: + run(b'print("simple hello")') + self.assertEqual(stdout.getvalue(), 'simple hello\n') + + with capture_excepthook() as excepthook: + self.assertEqual(run(SYNTAX_ERROR.encode()), -1) + self.assertIsInstance(excepthook.exc, SyntaxError) + self.assertEqual(excepthook.exc.filename, '') + + with capture_excepthook() as excepthook: + self.assertEqual(run(b'raise ValueError("BUG")'), -1) + self.assertIsInstance(excepthook.exc, ValueError) + self.assertEqual(str(excepthook.exc), 'BUG') + + def test_run_simplestring(self): + # Test PyRun_SimpleString() + self.check_run_simplestring(_testcapi.run_simplestring) + + def test_run_simplestringflags(self): + # Test PyRun_SimpleStringFlags() + self.check_run_simplestring(_testcapi.run_simplestringflags) + + def check_compilestring(self, compilestring, has_flags, encode_filename=True): + filename_str = TESTFN + if encode_filename: + filename = os.fsencode(filename_str) + else: + filename = filename_str + + def check_code(co, name, value): + ns = {} + exec(co, ns, ns) + self.assertEqual(ns[name], value) + + co = compilestring(b'x = 1', filename, Py_file_input) + self.assertEqual(co.co_filename, filename_str) + check_code(co, 'x', 1) + + if has_flags: + code = textwrap.dedent(""" + # encoding: latin1 + x = 'a\xe9' + """) + co = compilestring(code.encode(), filename, Py_file_input, PyCF_IGNORE_COOKIE) + self.assertEqual(co.co_filename, filename_str) + check_code(co, 'x', 'a\xe9') + + co = compilestring(code.encode(), filename, Py_file_input) + self.assertEqual(co.co_filename, filename_str) + check_code(co, 'x', 'a\xc3\xa9') + + tree = compilestring(b'x = 1', filename, Py_file_input, PyCF_ONLY_AST) + self.assertIsInstance(tree, ast.AST) + + co = compilestring(b'raise ValueError("BUG")', filename, Py_file_input) + with self.assertRaises(ValueError): + exec(co, {}) + + with self.assertRaises(SyntaxError) as cm: + compilestring(SYNTAX_ERROR.encode(), filename, Py_file_input) + + with self.assertRaises(ValueError): + compilestring(b'x = 1', filename, INVALID_START) + + def test_compilestring(self): + # Test Py_CompileString() + self.check_compilestring(_testlimitedcapi.run_compilestring, False) + + def test_compilestringflags(self): + # Test Py_CompileStringFlags() + self.check_compilestring(_testcapi.run_compilestringflags, True) + + def test_compilestringexflags(self): + # Test Py_CompileStringExFlags() + self.check_compilestring(_testcapi.run_compilestringexflags, True) + + def test_compilestringobject(self): + # Test Py_CompileStringObject() + self.check_compilestring(_testcapi.run_compilestringobject, True, + encode_filename=False) if __name__ == '__main__': diff --git a/Lib/test/test_capi/test_tuple.py b/Lib/test/test_capi/test_tuple.py index 7c07bc64e247c50..3683c008dbcaaf5 100644 --- a/Lib/test/test_capi/test_tuple.py +++ b/Lib/test/test_capi/test_tuple.py @@ -14,6 +14,12 @@ class TupleSubclass(tuple): class CAPITest(unittest.TestCase): + def _not_tracked(self, t): + self.assertFalse(gc.is_tracked(t), t) + + def _tracked(self, t): + self.assertTrue(gc.is_tracked(t), t) + def test_check(self): # Test PyTuple_Check() check = _testlimitedcapi.tuple_check @@ -52,11 +58,14 @@ def test_tuple_new(self): self.assertEqual(tup1, ()) self.assertEqual(size(tup1), 0) self.assertIs(type(tup1), tuple) + self._not_tracked(tup1) + tup2 = tuple_new(1) self.assertIs(type(tup2), tuple) self.assertEqual(size(tup2), 1) self.assertIsNot(tup2, tup1) self.assertTrue(checknull(tup2, 0)) + self._tracked(tup2) self.assertRaises(SystemError, tuple_new, -1) self.assertRaises(SystemError, tuple_new, PY_SSIZE_T_MIN) @@ -70,6 +79,10 @@ def test_tuple_pack(self): self.assertEqual(pack(1, [1]), ([1],)) self.assertEqual(pack(2, [1], [2]), ([1], [2])) + self._tracked(pack(1, [1])) + self._tracked(pack(2, [1], b'abc')) + self._not_tracked(pack(2, 42, b'abc')) + self.assertRaises(SystemError, pack, PY_SSIZE_T_MIN) self.assertRaises(SystemError, pack, -1) self.assertRaises(MemoryError, pack, PY_SSIZE_T_MAX) @@ -279,6 +292,10 @@ def my_iter(): self.assertEqual(tuple(my_iter()), (TAG, *range(10))) self.assertEqual(tuples, []) + def test_uninitialized_tuple_repr(self): + tup = _testlimitedcapi.tuple_new(3) + self.assertEqual(repr(tup), '(, , )') + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_capi/test_type.py b/Lib/test/test_capi/test_type.py index 3c9974c73873882..15fb4a93e2ad746 100644 --- a/Lib/test/test_capi/test_type.py +++ b/Lib/test/test_capi/test_type.py @@ -264,3 +264,13 @@ def test_manual_heap_type(self): ManualHeapType = _testcapi.ManualHeapType for i in range(100): self.assertIsInstance(ManualHeapType(), ManualHeapType) + + def test_extension_managed_dict_type(self): + ManagedDictType = _testcapi.ManagedDictType + obj = ManagedDictType() + obj.foo = 42 + self.assertEqual(obj.foo, 42) + self.assertEqual(obj.__dict__, {'foo': 42}) + obj.__dict__ = {'bar': 3} + self.assertEqual(obj.__dict__, {'bar': 3}) + self.assertEqual(obj.bar, 3) diff --git a/Lib/test/test_capi/test_unicode.py b/Lib/test/test_capi/test_unicode.py index 3408c10f4260586..d612b43bd7b5c05 100644 --- a/Lib/test/test_capi/test_unicode.py +++ b/Lib/test/test_capi/test_unicode.py @@ -842,9 +842,7 @@ def test_fromwidechar(self): if SIZEOF_WCHAR_T == 2: self.assertEqual(fromwidechar('a\U0001f600'.encode(encoding), 2), 'a\ud83d') - self.assertRaises(MemoryError, fromwidechar, b'', PY_SSIZE_T_MAX) self.assertRaises(SystemError, fromwidechar, b'\0'*SIZEOF_WCHAR_T, -2) - self.assertRaises(SystemError, fromwidechar, b'\0'*SIZEOF_WCHAR_T, PY_SSIZE_T_MIN) self.assertEqual(fromwidechar(NULL, 0), '') self.assertRaises(SystemError, fromwidechar, NULL, 1) self.assertRaises(SystemError, fromwidechar, NULL, PY_SSIZE_T_MAX) @@ -852,6 +850,10 @@ def test_fromwidechar(self): self.assertRaises(SystemError, fromwidechar, NULL, -2) self.assertRaises(SystemError, fromwidechar, NULL, PY_SSIZE_T_MIN) + # The following tests are skipped since they rely on undefined behavior + #self.assertRaises(MemoryError, fromwidechar, b'', PY_SSIZE_T_MAX) + #self.assertRaises(SystemError, fromwidechar, b'\0'*SIZEOF_WCHAR_T, PY_SSIZE_T_MIN) + @support.cpython_only @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_aswidechar(self): @@ -1751,13 +1753,15 @@ def test_basic(self): writer.write_utf8(b'var', -1) # test PyUnicodeWriter_WriteChar() - writer.write_char('=') + writer.write_char(ord('=')) # test PyUnicodeWriter_WriteSubstring() writer.write_substring("[long]", 1, 5) + # CRASHES writer.write_substring(NULL, 0, 0) # test PyUnicodeWriter_WriteStr() writer.write_str(" value ") + # CRASHES writer.write_str(NULL) # test PyUnicodeWriter_WriteRepr() writer.write_repr("repr") @@ -1765,17 +1769,48 @@ def test_basic(self): self.assertEqual(writer.finish(), "var=long value 'repr'") + def test_repr_null(self): + writer = self.create_writer(0) + writer.write_utf8(b'var=', -1) + writer.write_repr(NULL) + self.assertEqual(writer.finish(), + "var=") + + def test_write_char(self): + writer = self.create_writer(0) + writer.write_char(0) + writer.write_char(ord('$')) + writer.write_char(0x20ac) + writer.write_char(0x10_ffff) + self.assertRaises(ValueError, writer.write_char, 0x11_0000) + self.assertRaises(ValueError, writer.write_char, 0xFFFF_FFFF) + self.assertEqual(writer.finish(), + "\0$\u20AC\U0010FFFF") + def test_utf8(self): writer = self.create_writer(0) writer.write_utf8(b"ascii", -1) - writer.write_char('-') + writer.write_char(ord('-')) writer.write_utf8(b"latin1=\xC3\xA9", -1) - writer.write_char('-') + writer.write_char(ord('-')) writer.write_utf8(b"euro=\xE2\x82\xAC", -1) - writer.write_char('.') + writer.write_char(ord('.')) + writer.write_utf8(NULL, 0) + # CRASHES writer.write_utf8(NULL, 1) + # CRASHES writer.write_utf8(NULL, -1) self.assertEqual(writer.finish(), "ascii-latin1=\xE9-euro=\u20AC.") + def test_ascii(self): + writer = self.create_writer(0) + writer.write_ascii(b"Hello ", -1) + writer.write_ascii(b"", 0) + writer.write_ascii(NULL, 0) + # CRASHES writer.write_ascii(NULL, 1) + # CRASHES writer.write_ascii(NULL, -1) + writer.write_ascii(b"Python! ", 6) + self.assertEqual(writer.finish(), "Hello Python") + def test_invalid_utf8(self): writer = self.create_writer(0) with self.assertRaises(UnicodeDecodeError): @@ -1789,6 +1824,9 @@ def test_recover_utf8_error(self): # write fails with an invalid string with self.assertRaises(UnicodeDecodeError): writer.write_utf8(b"invalid\xFF", -1) + with self.assertRaises(UnicodeDecodeError): + s = "truncated\u20AC".encode() + writer.write_utf8(s, len(s) - 1) # retry write with a valid string writer.write_utf8(b"valid", -1) @@ -1800,13 +1838,19 @@ def test_decode_utf8(self): # test PyUnicodeWriter_DecodeUTF8Stateful() writer = self.create_writer(0) writer.decodeutf8stateful(b"ign\xFFore", -1, b"ignore") - writer.write_char('-') + writer.write_char(ord('-')) writer.decodeutf8stateful(b"replace\xFF", -1, b"replace") - writer.write_char('-') + writer.write_char(ord('-')) # incomplete trailing UTF-8 sequence writer.decodeutf8stateful(b"incomplete\xC3", -1, b"replace") + writer.decodeutf8stateful(NULL, 0, b"replace") + # CRASHES writer.decodeutf8stateful(NULL, 1, b"replace") + # CRASHES writer.decodeutf8stateful(NULL, -1, b"replace") + with self.assertRaises(UnicodeDecodeError): + writer.decodeutf8stateful(b"default\xFF", -1, NULL) + self.assertEqual(writer.finish(), "ignore-replace\uFFFD-incomplete\uFFFD") @@ -1817,12 +1861,12 @@ def test_decode_utf8_consumed(self): # valid string consumed = writer.decodeutf8stateful(b"text", -1, b"strict", True) self.assertEqual(consumed, 4) - writer.write_char('-') + writer.write_char(ord('-')) # non-ASCII consumed = writer.decodeutf8stateful(b"\xC3\xA9-\xE2\x82\xAC", 6, b"strict", True) self.assertEqual(consumed, 6) - writer.write_char('-') + writer.write_char(ord('-')) # invalid UTF-8 (consumed is 0 on error) with self.assertRaises(UnicodeDecodeError): @@ -1831,54 +1875,92 @@ def test_decode_utf8_consumed(self): # ignore error handler consumed = writer.decodeutf8stateful(b"more\xFF", -1, b"ignore", True) self.assertEqual(consumed, 5) - writer.write_char('-') + writer.write_char(ord('-')) # incomplete trailing UTF-8 sequence consumed = writer.decodeutf8stateful(b"incomplete\xC3", -1, b"ignore", True) self.assertEqual(consumed, 10) + writer.write_char(ord('-')) + + consumed = writer.decodeutf8stateful(NULL, 0, b"replace", True) + self.assertEqual(consumed, 0) + # CRASHES writer.decodeutf8stateful(NULL, 1, b"replace", True) + # CRASHES writer.decodeutf8stateful(NULL, -1, b"replace", True) + consumed = writer.decodeutf8stateful(b"default\xC3", -1, NULL, True) + self.assertEqual(consumed, 7) - self.assertEqual(writer.finish(), "text-\xE9-\u20AC-more-incomplete") + self.assertEqual(writer.finish(), "text-\xE9-\u20AC-more-incomplete-default") def test_widechar(self): + from _testcapi import SIZEOF_WCHAR_T + + if SIZEOF_WCHAR_T == 2: + encoding = 'utf-16le' if sys.byteorder == 'little' else 'utf-16be' + elif SIZEOF_WCHAR_T == 4: + encoding = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be' + writer = self.create_writer(0) - writer.write_widechar("latin1=\xE9") - writer.write_widechar("-") - writer.write_widechar("euro=\u20AC") - writer.write_char("-") - writer.write_widechar("max=\U0010ffff") - writer.write_char('.') + writer.write_widechar("latin1=\xE9".encode(encoding)) + writer.write_char(ord("-")) + writer.write_widechar("euro=\u20AC".encode(encoding)) + writer.write_char(ord("-")) + writer.write_widechar("max=\U0010ffff".encode(encoding)) + writer.write_char(ord("-")) + writer.write_widechar("zeroes=".encode(encoding).ljust(SIZEOF_WCHAR_T * 10, b'\0'), + 10) + writer.write_char(ord('.')) + + if SIZEOF_WCHAR_T == 4: + invalid = (b'\x00\x00\x11\x00' if sys.byteorder == 'little' else + b'\x00\x11\x00\x00') + with self.assertRaises(ValueError): + writer.write_widechar("invalid=".encode(encoding) + invalid) + writer.write_widechar(b'', -5) + writer.write_widechar(NULL, 0) + # CRASHES writer.write_widechar(NULL, 1) + # CRASHES writer.write_widechar(NULL, -1) + self.assertEqual(writer.finish(), - "latin1=\xE9-euro=\u20AC-max=\U0010ffff.") + "latin1=\xE9-euro=\u20AC-max=\U0010ffff-zeroes=\0\0\0.") def test_ucs4(self): + encoding = 'utf-32le' if sys.byteorder == 'little' else 'utf-32be' + writer = self.create_writer(0) - writer.write_ucs4("ascii IGNORED", 5) - writer.write_char("-") - writer.write_ucs4("latin1=\xe9", 8) - writer.write_char("-") - writer.write_ucs4("euro=\u20ac", 6) - writer.write_char("-") - writer.write_ucs4("max=\U0010ffff", 5) - writer.write_char(".") + writer.write_ucs4("ascii IGNORED".encode(encoding), 5) + writer.write_char(ord("-")) + writer.write_ucs4("latin1=\xe9".encode(encoding)) + writer.write_char(ord("-")) + writer.write_ucs4("euro=\u20ac".encode(encoding)) + writer.write_char(ord("-")) + writer.write_ucs4("max=\U0010ffff".encode(encoding)) + writer.write_char(ord(".")) self.assertEqual(writer.finish(), "ascii-latin1=\xE9-euro=\u20AC-max=\U0010ffff.") # Test some special characters writer = self.create_writer(0) # Lone surrogate character - writer.write_ucs4("lone\uDC80", 5) - writer.write_char("-") + writer.write_ucs4("lone\uDC80".encode(encoding, 'surrogatepass')) + writer.write_char(ord("-")) # Surrogate pair - writer.write_ucs4("pair\uDBFF\uDFFF", 5) - writer.write_char("-") - writer.write_ucs4("null[\0]", 7) + writer.write_ucs4("pair\uD83D\uDC0D".encode(encoding, 'surrogatepass')) + writer.write_char(ord("-")) + writer.write_ucs4("null[\0]".encode(encoding), 7) + invalid = (b'\x00\x00\x11\x00' if sys.byteorder == 'little' else + b'\x00\x11\x00\x00') + # CRASHES writer.write_ucs4("invalid".encode(encoding) + invalid) + writer.write_ucs4(NULL, 0) + # CRASHES writer.write_ucs4(NULL, 1) self.assertEqual(writer.finish(), - "lone\udc80-pair\udbff-null[\0]") + "lone\udc80-pair\ud83d\udc0d-null[\x00]") # invalid size writer = self.create_writer(0) with self.assertRaises(ValueError): - writer.write_ucs4("text", -1) + writer.write_ucs4("text".encode(encoding), -1) + self.assertRaises(ValueError, writer.write_ucs4, b'', -1) + self.assertRaises(ValueError, writer.write_ucs4, NULL, -1) def test_substring_empty(self): writer = self.create_writer(0) @@ -1904,7 +1986,7 @@ def test_format(self): from ctypes import c_int writer = self.create_writer(0) self.writer_format(writer, b'%s %i', b'abc', c_int(123)) - writer.write_char('.') + writer.write_char(ord('.')) self.assertEqual(writer.finish(), 'abc 123.') def test_recover_error(self): diff --git a/Lib/test/test_capi/test_weakref.py b/Lib/test/test_capi/test_weakref.py new file mode 100644 index 000000000000000..86ebe92da8d95db --- /dev/null +++ b/Lib/test/test_capi/test_weakref.py @@ -0,0 +1,136 @@ +import weakref +import unittest +from test.support import import_helper + +_testcapi = import_helper.import_module('_testcapi') +_testlimitedcapi = import_helper.import_module('_testlimitedcapi') +NULL = None + +class Object: + pass + +class Ref(weakref.ReferenceType): + pass + + +class CAPIWeakrefTest(unittest.TestCase): + def test_pyweakref_check(self): + # Test PyWeakref_Check() + check = _testlimitedcapi.pyweakref_check + obj = Object() + self.assertEqual(check(obj), 0) + self.assertEqual(check(weakref.ref(obj)), 1) + self.assertEqual(check(Ref(obj)), 1) + self.assertEqual(check(weakref.proxy(obj)), 1) + + # CRASHES check(NULL) + + def test_pyweakref_checkref(self): + # Test PyWeakref_CheckRef() + checkref = _testlimitedcapi.pyweakref_checkref + obj = Object() + self.assertEqual(checkref(obj), 0) + self.assertEqual(checkref(weakref.ref(obj)), 1) + self.assertEqual(checkref(Ref(obj)), 1) + self.assertEqual(checkref(weakref.proxy(obj)), 0) + + # CRASHES checkref(NULL) + + def test_pyweakref_checkrefexact(self): + # Test PyWeakref_CheckRefExact() + checkrefexact = _testlimitedcapi.pyweakref_checkrefexact + obj = Object() + self.assertEqual(checkrefexact(obj), 0) + self.assertEqual(checkrefexact(weakref.ref(obj)), 1) + self.assertEqual(checkrefexact(Ref(obj)), 0) + self.assertEqual(checkrefexact(weakref.proxy(obj)), 0) + + # CRASHES checkrefexact(NULL) + + def test_pyweakref_checkproxy(self): + # Test PyWeakref_CheckProxy() + checkproxy = _testlimitedcapi.pyweakref_checkproxy + obj = Object() + self.assertEqual(checkproxy(obj), 0) + self.assertEqual(checkproxy(weakref.ref(obj)), 0) + self.assertEqual(checkproxy(Ref(obj)), 0) + self.assertEqual(checkproxy(weakref.proxy(obj)), 1) + + # CRASHES checkproxy(NULL) + + def test_pyweakref_getref(self): + # Test PyWeakref_GetRef() + getref = _testcapi.pyweakref_getref + obj = Object() + wr = weakref.ref(obj) + wp = weakref.proxy(obj) + self.assertEqual(getref(wr), (1, obj)) + self.assertEqual(getref(wp), (1, obj)) + del obj + self.assertEqual(getref(wr), 0) + self.assertEqual(getref(wp), 0) + + self.assertRaises(TypeError, getref, 42) + self.assertRaises(SystemError, getref, NULL) + + def test_pyweakref_isdead(self): + # Test PyWeakref_IsDead() + isdead = _testcapi.pyweakref_isdead + obj = Object() + wr = weakref.ref(obj) + wp = weakref.proxy(obj) + self.assertEqual(isdead(wr), 0) + self.assertEqual(isdead(wp), 0) + del obj + self.assertEqual(isdead(wr), 1) + self.assertEqual(isdead(wp), 1) + + self.assertRaises(TypeError, isdead, 42) + self.assertRaises(SystemError, isdead, NULL) + + def test_pyweakref_newref(self): + # Test PyWeakref_NewRef() + newref = _testlimitedcapi.pyweakref_newref + obj = Object() + wr = newref(obj) + self.assertIs(type(wr), weakref.ReferenceType) + # PyWeakref_NewRef() handles None callback as NULL callback + wr = newref(obj, None) + self.assertIs(type(wr), weakref.ReferenceType) + log = [] + wr = newref(obj, log.append) + self.assertIs(type(wr), weakref.ReferenceType) + self.assertEqual(log, []) + del obj + self.assertEqual(log, [wr]) + + self.assertRaises(TypeError, newref, []) + # CRASHES newref(NULL) + + def test_pyweakref_newproxy(self): + # Test PyWeakref_NewProxy() + newproxy = _testlimitedcapi.pyweakref_newproxy + obj = Object() + wp = newproxy(obj) + self.assertIs(type(wp), weakref.ProxyType) + # PyWeakref_NewProxy() handles None callback as NULL callback + wp = newproxy(obj, None) + self.assertIs(type(wp), weakref.ProxyType) + log = [] + wp = newproxy(obj, log.append) + self.assertIs(type(wp), weakref.ProxyType) + self.assertEqual(log, []) + del obj + self.assertEqual(log, [wp]) + + def func(): + pass + wp = newproxy(func) + self.assertIs(type(wp), weakref.CallableProxyType) + + self.assertRaises(TypeError, newproxy, []) + # CRASHES newproxy(NULL) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_cext/__init__.py b/Lib/test/test_cext/__init__.py index 46fde541494aa30..f75257f3d889f73 100644 --- a/Lib/test/test_cext/__init__.py +++ b/Lib/test/test_cext/__init__.py @@ -12,7 +12,9 @@ from test import support -SOURCE = os.path.join(os.path.dirname(__file__), 'extension.c') +SOURCES = [ + os.path.join(os.path.dirname(__file__), 'extension.c'), +] SETUP = os.path.join(os.path.dirname(__file__), 'setup.py') @@ -28,29 +30,13 @@ @support.requires_venv_with_pip() @support.requires_subprocess() @support.requires_resource('cpu') -class TestExt(unittest.TestCase): +class BaseTests: + TEST_INTERNAL_C_API = False + # Default build with no options def test_build(self): self.check_build('_test_cext') - def test_build_c11(self): - self.check_build('_test_c11_cext', std='c11') - - @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99") - def test_build_c99(self): - # In public docs, we say C API is compatible with C11. However, - # in practice we do maintain C99 compatibility in public headers. - # Please ask the C API WG before adding a new C11-only feature. - self.check_build('_test_c99_cext', std='c99') - - @support.requires_gil_enabled('incompatible with Free Threading') - def test_build_limited(self): - self.check_build('_test_limited_cext', limited=True) - - @support.requires_gil_enabled('broken for now with Free Threading') - def test_build_limited_c11(self): - self.check_build('_test_limited_c11_cext', limited=True, std='c11') - def check_build(self, extension_name, std=None, limited=False): venv_dir = 'env' with support.setup_venv_with_pip_setuptools(venv_dir) as python_exe: @@ -61,7 +47,9 @@ def _check_build(self, extension_name, python_exe, std, limited): pkg_dir = 'pkg' os.mkdir(pkg_dir) shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) - shutil.copy(SOURCE, os.path.join(pkg_dir, os.path.basename(SOURCE))) + for source in SOURCES: + dest = os.path.join(pkg_dir, os.path.basename(source)) + shutil.copy(source, dest) def run_cmd(operation, cmd): env = os.environ.copy() @@ -70,6 +58,7 @@ def run_cmd(operation, cmd): if limited: env['CPYTHON_TEST_LIMITED'] = '1' env['CPYTHON_TEST_EXT_NAME'] = extension_name + env['TEST_INTERNAL_C_API'] = str(int(self.TEST_INTERNAL_C_API)) if support.verbose: print('Run:', ' '.join(map(shlex.quote, cmd))) subprocess.run(cmd, check=True, env=env) @@ -110,5 +99,29 @@ def run_cmd(operation, cmd): run_cmd('Import', cmd) +class TestPublicCAPI(BaseTests, unittest.TestCase): + @support.requires_gil_enabled('incompatible with Free Threading') + def test_build_limited(self): + self.check_build('_test_limited_cext', limited=True) + + @support.requires_gil_enabled('broken for now with Free Threading') + def test_build_limited_c11(self): + self.check_build('_test_limited_c11_cext', limited=True, std='c11') + + def test_build_c11(self): + self.check_build('_test_c11_cext', std='c11') + + @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c99") + def test_build_c99(self): + # In public docs, we say C API is compatible with C11. However, + # in practice we do maintain C99 compatibility in public headers. + # Please ask the C API WG before adding a new C11-only feature. + self.check_build('_test_c99_cext', std='c99') + + +class TestInteralCAPI(BaseTests, unittest.TestCase): + TEST_INTERNAL_C_API = True + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_cext/extension.c b/Lib/test/test_cext/extension.c index 64629c5a6da8cd2..56f40b354c69135 100644 --- a/Lib/test/test_cext/extension.c +++ b/Lib/test/test_cext/extension.c @@ -1,10 +1,34 @@ // gh-116869: Basic C test extension to check that the Python C API // does not emit C compiler warnings. +// +// Test also the internal C API if the TEST_INTERNAL_C_API macro is defined. // Always enable assertions #undef NDEBUG +#ifdef TEST_INTERNAL_C_API +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" +#include "datetime.h" + +#ifdef TEST_INTERNAL_C_API + // gh-135906: Check for compiler warnings in the internal C API. + // - Cython uses pycore_critical_section.h, pycore_frame.h and + // pycore_template.h. + // - greenlet uses pycore_frame.h, pycore_interpframe_structs.h and + // pycore_interpframe.h. +# include "internal/pycore_critical_section.h" +# include "internal/pycore_frame.h" +# include "internal/pycore_gc.h" +# include "internal/pycore_interp.h" +# include "internal/pycore_interpframe.h" +# include "internal/pycore_interpframe_structs.h" +# include "internal/pycore_object.h" +# include "internal/pycore_pystate.h" +# include "internal/pycore_template.h" +#endif #ifndef MODULE_NAME # error "MODULE_NAME macro must be defined" @@ -30,27 +54,43 @@ _testcext_add(PyObject *Py_UNUSED(module), PyObject *args) } +static PyObject * +test_datetime(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + // datetime.h is excluded from the limited C API +#ifndef Py_LIMITED_API + PyDateTime_IMPORT; + if (PyErr_Occurred()) { + return NULL; + } +#endif + + Py_RETURN_NONE; +} + + static PyMethodDef _testcext_methods[] = { {"add", _testcext_add, METH_VARARGS, _testcext_add_doc}, + {"test_datetime", test_datetime, METH_NOARGS, NULL}, {NULL, NULL, 0, NULL} // sentinel }; static int -_testcext_exec( -#ifdef __STDC_VERSION__ - PyObject *module -#else - PyObject *Py_UNUSED(module) -#endif - ) +_testcext_exec(PyObject *module) { + PyObject *result; + #ifdef __STDC_VERSION__ if (PyModule_AddIntMacro(module, __STDC_VERSION__) < 0) { return -1; } #endif + result = PyObject_CallMethod(module, "test_datetime", ""); + if (!result) return -1; + Py_DECREF(result); + // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR() Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int)); assert(Py_BUILD_ASSERT_EXPR(sizeof(int) == sizeof(unsigned int)) == 0); diff --git a/Lib/test/test_cext/setup.py b/Lib/test/test_cext/setup.py index 1275282983f7ffd..f2a8c9f9381e0f4 100644 --- a/Lib/test/test_cext/setup.py +++ b/Lib/test/test_cext/setup.py @@ -14,10 +14,15 @@ if not support.MS_WINDOWS: # C compiler flags for GCC and clang - CFLAGS = [ + BASE_CFLAGS = [ # The purpose of test_cext extension is to check that building a C # extension using the Python C API does not emit C compiler warnings. '-Werror', + ] + + # C compiler flags for GCC and clang + PUBLIC_CFLAGS = [ + *BASE_CFLAGS, # gh-120593: Check the 'const' qualifier '-Wcast-qual', @@ -26,27 +31,42 @@ '-pedantic-errors', ] if not support.Py_GIL_DISABLED: - CFLAGS.append( + PUBLIC_CFLAGS.append( # gh-116869: The Python C API must be compatible with building # with the -Werror=declaration-after-statement compiler flag. '-Werror=declaration-after-statement', ) + INTERNAL_CFLAGS = [*BASE_CFLAGS] else: # MSVC compiler flags - CFLAGS = [ - # Display warnings level 1 to 4 - '/W4', + BASE_CFLAGS = [ # Treat all compiler warnings as compiler errors '/WX', ] + PUBLIC_CFLAGS = [ + *BASE_CFLAGS, + # Display warnings level 1 to 4 + '/W4', + ] + INTERNAL_CFLAGS = [ + *BASE_CFLAGS, + # Display warnings level 1 to 3 + '/W3', + ] def main(): std = os.environ.get("CPYTHON_TEST_STD", "") module_name = os.environ["CPYTHON_TEST_EXT_NAME"] limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", "")) + internal = bool(int(os.environ.get("TEST_INTERNAL_C_API", "0"))) - cflags = list(CFLAGS) + sources = [SOURCE] + + if not internal: + cflags = list(PUBLIC_CFLAGS) + else: + cflags = list(INTERNAL_CFLAGS) cflags.append(f'-DMODULE_NAME={module_name}') # Add -std=STD or /std:STD (MSVC) compiler flag @@ -75,6 +95,9 @@ def main(): version = sys.hexversion cflags.append(f'-DPy_LIMITED_API={version:#x}') + if internal: + cflags.append('-DTEST_INTERNAL_C_API=1') + # On Windows, add PCbuild\amd64\ to include and library directories include_dirs = [] library_dirs = [] @@ -90,7 +113,7 @@ def main(): print(f"Add PCbuild directory: {pcbuild}") # Display information to help debugging - for env_name in ('CC', 'CFLAGS'): + for env_name in ('CC', 'CFLAGS', 'CPPFLAGS'): if env_name in os.environ: print(f"{env_name} env var: {os.environ[env_name]!r}") else: @@ -99,7 +122,7 @@ def main(): ext = Extension( module_name, - sources=[SOURCE], + sources=sources, extra_compile_args=cflags, include_dirs=include_dirs, library_dirs=library_dirs) diff --git a/Lib/test/test_class.py b/Lib/test/test_class.py index 4c12d43556fc2a5..df46c0de00d60aa 100644 --- a/Lib/test/test_class.py +++ b/Lib/test/test_class.py @@ -449,7 +449,6 @@ def __delattr__(self, *args): def testHasAttrString(self): import sys - from test.support import import_helper _testlimitedcapi = import_helper.import_module('_testlimitedcapi') class A: @@ -556,6 +555,7 @@ class Custom: self.assertFalse(hasattr(o, "__call__")) self.assertFalse(hasattr(c, "__call__")) + @support.skip_if_huge_c_stack() @support.skip_emscripten_stack_overflow() @support.skip_wasi_stack_overflow() def testSFBug532646(self): @@ -858,7 +858,12 @@ def __init__(self, arg): from _testinternalcapi import has_inline_values -Py_TPFLAGS_MANAGED_DICT = (1 << 2) +Py_TPFLAGS_INLINE_VALUES = (1 << 2) +Py_TPFLAGS_MANAGED_DICT = (1 << 4) + +class NoManagedDict: + __slots__ = ('a',) + class Plain: pass @@ -873,11 +878,31 @@ def __init__(self): self.d = 4 +class VarSizedSubclass(tuple): + pass + + class TestInlineValues(unittest.TestCase): - def test_flags(self): - self.assertEqual(Plain.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) - self.assertEqual(WithAttrs.__flags__ & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + def test_no_flags_for_slots_class(self): + flags = NoManagedDict.__flags__ + self.assertEqual(flags & Py_TPFLAGS_MANAGED_DICT, 0) + self.assertEqual(flags & Py_TPFLAGS_INLINE_VALUES, 0) + self.assertFalse(has_inline_values(NoManagedDict())) + + def test_both_flags_for_regular_class(self): + for cls in (Plain, WithAttrs): + with self.subTest(cls=cls.__name__): + flags = cls.__flags__ + self.assertEqual(flags & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + self.assertEqual(flags & Py_TPFLAGS_INLINE_VALUES, Py_TPFLAGS_INLINE_VALUES) + self.assertTrue(has_inline_values(cls())) + + def test_managed_dict_only_for_varsized_subclass(self): + flags = VarSizedSubclass.__flags__ + self.assertEqual(flags & Py_TPFLAGS_MANAGED_DICT, Py_TPFLAGS_MANAGED_DICT) + self.assertEqual(flags & Py_TPFLAGS_INLINE_VALUES, 0) + self.assertFalse(has_inline_values(VarSizedSubclass())) def test_has_inline_values(self): c = Plain() @@ -987,11 +1012,8 @@ class C: C.a = X() C.a = X() - @cpython_only + @support.nomemtest def test_detach_materialized_dict_no_memory(self): - # Skip test if _testcapi is not available: - import_helper.import_module('_testcapi') - code = """if 1: import test.support import _testcapi diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 0c99620e27cde4e..73bb942af7c0a18 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -238,11 +238,11 @@ def test_directive_output_print(self): # The generated output will differ for every run, but we can check that # it starts with the clinic block, we check that it contains all the # expected fields, and we check that it contains the checksum line. - self.assertTrue(out.startswith(dedent(""" + self.assertStartsWith(out, dedent(""" /*[clinic input] output print 'I told you once.' [clinic start generated code]*/ - """))) + """)) fields = { "cpp_endif", "cpp_if", @@ -259,9 +259,7 @@ def test_directive_output_print(self): with self.subTest(field=field): self.assertIn(field, out) last_line = out.rstrip().split("\n")[-1] - self.assertTrue( - last_line.startswith("/*[clinic end generated code: output=") - ) + self.assertStartsWith(last_line, "/*[clinic end generated code: output=") def test_directive_wrong_arg_number(self): raw = dedent(""" @@ -1046,6 +1044,187 @@ def test_param_with_continuations(self): p = function.parameters['follow_symlinks'] self.assertEqual(True, p.default) + def test_param_default_none(self): + function = self.parse_function(r""" + module test + test.func + obj: object = None + str: str(accept={str, NoneType}) = None + buf: Py_buffer(accept={str, buffer, NoneType}) = None + """) + p = function.parameters['obj'] + self.assertIs(p.default, None) + self.assertEqual(p.converter.py_default, 'None') + self.assertEqual(p.converter.c_default, 'Py_None') + + p = function.parameters['str'] + self.assertIs(p.default, None) + self.assertEqual(p.converter.py_default, 'None') + self.assertEqual(p.converter.c_default, 'NULL') + + p = function.parameters['buf'] + self.assertIs(p.default, None) + self.assertEqual(p.converter.py_default, 'None') + self.assertEqual(p.converter.c_default, '{NULL, NULL}') + + def test_param_default_null(self): + function = self.parse_function(r""" + module test + test.func + obj: object = NULL + str: str = NULL + buf: Py_buffer = NULL + fsencoded: unicode_fs_encoded = NULL + fsdecoded: unicode_fs_decoded = NULL + """) + p = function.parameters['obj'] + self.assertIs(p.default, NULL) + self.assertEqual(p.converter.py_default, '') + self.assertEqual(p.converter.c_default, 'NULL') + + p = function.parameters['str'] + self.assertIs(p.default, NULL) + self.assertEqual(p.converter.py_default, '') + self.assertEqual(p.converter.c_default, 'NULL') + + p = function.parameters['buf'] + self.assertIs(p.default, NULL) + self.assertEqual(p.converter.py_default, '') + self.assertEqual(p.converter.c_default, '{NULL, NULL}') + + p = function.parameters['fsencoded'] + self.assertIs(p.default, NULL) + self.assertEqual(p.converter.py_default, '') + self.assertEqual(p.converter.c_default, 'NULL') + + p = function.parameters['fsdecoded'] + self.assertIs(p.default, NULL) + self.assertEqual(p.converter.py_default, '') + self.assertEqual(p.converter.c_default, 'NULL') + + def test_param_default_str_literal(self): + function = self.parse_function(r""" + module test + test.func + str: str = ' \t\n\r\v\f\xa0' + buf: Py_buffer(accept={str, buffer}) = ' \t\n\r\v\f\xa0' + """) + p = function.parameters['str'] + self.assertEqual(p.default, ' \t\n\r\v\f\xa0') + self.assertEqual(p.converter.py_default, r"' \t\n\r\x0b\x0c\xa0'") + self.assertEqual(p.converter.c_default, r'" \t\n\r\v\f\u00a0"') + + p = function.parameters['buf'] + self.assertEqual(p.default, ' \t\n\r\v\f\xa0') + self.assertEqual(p.converter.py_default, r"' \t\n\r\x0b\x0c\xa0'") + self.assertEqual(p.converter.c_default, + r'{.buf = " \t\n\r\v\f\302\240", .obj = NULL, .len = 8}') + + def test_param_default_bytes_literal(self): + function = self.parse_function(r""" + module test + test.func + str: str(accept={robuffer}) = b' \t\n\r\v\f\xa0' + buf: Py_buffer = b' \t\n\r\v\f\xa0' + """) + p = function.parameters['str'] + self.assertEqual(p.default, b' \t\n\r\v\f\xa0') + self.assertEqual(p.converter.py_default, r"b' \t\n\r\x0b\x0c\xa0'") + self.assertEqual(p.converter.c_default, r'" \t\n\r\v\f\240"') + + p = function.parameters['buf'] + self.assertEqual(p.default, b' \t\n\r\v\f\xa0') + self.assertEqual(p.converter.py_default, r"b' \t\n\r\x0b\x0c\xa0'") + self.assertEqual(p.converter.c_default, + r'{.buf = " \t\n\r\v\f\240", .obj = NULL, .len = 7}') + + def test_param_default_byte_literal(self): + function = self.parse_function(r""" + module test + test.func + zero: char = b'\0' + one: char = b'\1' + lf: char = b'\n' + nbsp: char = b'\xa0' + """) + p = function.parameters['zero'] + self.assertEqual(p.default, b'\0') + self.assertEqual(p.converter.py_default, r"b'\x00'") + self.assertEqual(p.converter.c_default, r"'\0'") + + p = function.parameters['one'] + self.assertEqual(p.default, b'\1') + self.assertEqual(p.converter.py_default, r"b'\x01'") + self.assertEqual(p.converter.c_default, r"'\001'") + + p = function.parameters['lf'] + self.assertEqual(p.default, b'\n') + self.assertEqual(p.converter.py_default, r"b'\n'") + self.assertEqual(p.converter.c_default, r"'\n'") + + p = function.parameters['nbsp'] + self.assertEqual(p.default, b'\xa0') + self.assertEqual(p.converter.py_default, r"b'\xa0'") + self.assertEqual(p.converter.c_default, r"'\240'") + + def test_param_default_unicode_char(self): + function = self.parse_function(r""" + module test + test.func + zero: int(accept={str}) = '\0' + one: int(accept={str}) = '\1' + lf: int(accept={str}) = '\n' + nbsp: int(accept={str}) = '\xa0' + snake: int(accept={str}) = '\U0001f40d' + """) + p = function.parameters['zero'] + self.assertEqual(p.default, '\0') + self.assertEqual(p.converter.py_default, r"'\x00'") + self.assertEqual(p.converter.c_default, '0') + + p = function.parameters['one'] + self.assertEqual(p.default, '\1') + self.assertEqual(p.converter.py_default, r"'\x01'") + self.assertEqual(p.converter.c_default, '0x01') + + p = function.parameters['lf'] + self.assertEqual(p.default, '\n') + self.assertEqual(p.converter.py_default, r"'\n'") + self.assertEqual(p.converter.c_default, r"'\n'") + + p = function.parameters['nbsp'] + self.assertEqual(p.default, '\xa0') + self.assertEqual(p.converter.py_default, r"'\xa0'") + self.assertEqual(p.converter.c_default, '0xa0') + + p = function.parameters['snake'] + self.assertEqual(p.default, '\U0001f40d') + self.assertEqual(p.converter.py_default, "'\U0001f40d'") + self.assertEqual(p.converter.c_default, '0x1f40d') + + def test_param_default_bool(self): + function = self.parse_function(r""" + module test + test.func + bool: bool = True + intbool: bool(accept={int}) = True + intbool2: bool(accept={int}) = 2 + """) + p = function.parameters['bool'] + self.assertIs(p.default, True) + self.assertEqual(p.converter.py_default, 'True') + self.assertEqual(p.converter.c_default, '1') + + p = function.parameters['intbool'] + self.assertIs(p.default, True) + self.assertEqual(p.converter.py_default, 'True') + self.assertEqual(p.converter.c_default, '1') + + p = function.parameters['intbool2'] + self.assertEqual(p.default, 2) + self.assertEqual(p.converter.py_default, '2') + self.assertEqual(p.converter.c_default, '2') + def test_param_default_expr_named_constant(self): function = self.parse_function(""" module os @@ -1279,12 +1458,8 @@ def test_base_invalid_syntax(self): os.stat invalid syntax: int = 42 """ - err = dedent(r""" - Function 'stat' has an invalid parameter declaration: - \s+'invalid syntax: int = 42' - """).strip() - with self.assertRaisesRegex(ClinicError, err): - self.parse_function(block) + err = "Function 'stat' has an invalid parameter declaration: 'invalid syntax: int = 42'" + self.expect_failure(block, err, lineno=2) def test_param_default_invalid_syntax(self): block = """ @@ -1292,7 +1467,7 @@ def test_param_default_invalid_syntax(self): os.stat x: int = invalid syntax """ - err = r"Syntax error: 'x = invalid syntax\n'" + err = "Function 'stat' has an invalid parameter declaration:" self.expect_failure(block, err, lineno=2) def test_cloning_nonexistent_function_correctly_fails(self): @@ -2512,7 +2687,7 @@ def test_cannot_specify_pydefault_without_default(self): self.expect_failure(block, err, lineno=1) def test_vararg_cannot_take_default_value(self): - err = "Vararg can't take a default value!" + err = "Function 'fn' has an invalid parameter declaration:" block = """ fn *args: tuple = None @@ -2705,8 +2880,7 @@ def test_cli_force(self): # Note, we cannot check the entire fail msg, because the path to # the tmp file will change for every run. _, err = self.expect_failure(fn) - self.assertTrue(err.endswith(fail_msg), - f"{err!r} does not end with {fail_msg!r}") + self.assertEndsWith(err, fail_msg) # Then, force regeneration; success expected. out = self.expect_success("-f", fn) self.assertEqual(out, "") @@ -2717,8 +2891,7 @@ def test_cli_force(self): ) with open(fn, encoding='utf-8') as f: generated = f.read() - self.assertTrue(generated.endswith(checksum), - (generated, checksum)) + self.assertEndsWith(generated, checksum) def test_cli_make(self): c_code = dedent(""" @@ -2800,6 +2973,7 @@ def test_cli_verbose(self): out = self.expect_success("-v", fn) self.assertEqual(out.strip(), fn) + @support.force_not_colorized def test_cli_help(self): out = self.expect_success("-h") self.assertIn("usage: clinic.py", out) @@ -2835,7 +3009,13 @@ def test_cli_converters(self): "size_t", "slice_index", "str", + "uint16", + "uint32", + "uint64", + "uint8", "unicode", + "unicode_fs_decoded", + "unicode_fs_encoded", "unsigned_char", "unsigned_int", "unsigned_long", @@ -2863,8 +3043,8 @@ def test_cli_converters(self): # param may change (it's a set, thus unordered). So, let's compare the # start and end of the expected output, and then assert that the # converters appear lined up in alphabetical order. - self.assertTrue(out.startswith(prelude), out) - self.assertTrue(out.endswith(finale), out) + self.assertStartsWith(out, prelude) + self.assertEndsWith(out, finale) out = out.removeprefix(prelude) out = out.removesuffix(finale) @@ -2872,10 +3052,7 @@ def test_cli_converters(self): for converter, line in zip(expected_converters, lines): line = line.lstrip() with self.subTest(converter=converter): - self.assertTrue( - line.startswith(converter), - f"expected converter {converter!r}, got {line!r}" - ) + self.assertStartsWith(line, converter) def test_cli_fail_converters_and_filename(self): _, err = self.expect_failure("--converters", "test.c") @@ -4213,6 +4390,56 @@ def test_format_escape(self): out = libclinic.format_escape(line) self.assertEqual(out, expected) + def test_c_bytes_repr(self): + c_bytes_repr = libclinic.c_bytes_repr + self.assertEqual(c_bytes_repr(b''), '""') + self.assertEqual(c_bytes_repr(b'abc'), '"abc"') + self.assertEqual(c_bytes_repr(b'\a\b\f\n\r\t\v'), r'"\a\b\f\n\r\t\v"') + self.assertEqual(c_bytes_repr(b' \0\x7f'), r'" \000\177"') + self.assertEqual(c_bytes_repr(b'"'), r'"\""') + self.assertEqual(c_bytes_repr(b"'"), r'''"'"''') + self.assertEqual(c_bytes_repr(b'\\'), r'"\\"') + self.assertEqual(c_bytes_repr(b'??/'), r'"?\?/"') + self.assertEqual(c_bytes_repr(b'???/'), r'"?\?\?/"') + self.assertEqual(c_bytes_repr(b'/*****/ /*/ */*'), r'"/\*****\/ /\*\/ *\/\*"') + self.assertEqual(c_bytes_repr(b'\xa0'), r'"\240"') + self.assertEqual(c_bytes_repr(b'\xff'), r'"\377"') + + def test_c_str_repr(self): + c_str_repr = libclinic.c_str_repr + self.assertEqual(c_str_repr(''), '""') + self.assertEqual(c_str_repr('abc'), '"abc"') + self.assertEqual(c_str_repr('\a\b\f\n\r\t\v'), r'"\a\b\f\n\r\t\v"') + self.assertEqual(c_str_repr(' \0\x7f'), r'" \000\177"') + self.assertEqual(c_str_repr('"'), r'"\""') + self.assertEqual(c_str_repr("'"), r'''"'"''') + self.assertEqual(c_str_repr('\\'), r'"\\"') + self.assertEqual(c_str_repr('??/'), r'"?\?/"') + self.assertEqual(c_str_repr('???/'), r'"?\?\?/"') + self.assertEqual(c_str_repr('/*****/ /*/ */*'), r'"/\*****\/ /\*\/ *\/\*"') + self.assertEqual(c_str_repr('\xa0'), r'"\u00a0"') + self.assertEqual(c_str_repr('\xff'), r'"\u00ff"') + self.assertEqual(c_str_repr('\u20ac'), r'"\u20ac"') + self.assertEqual(c_str_repr('\U0001f40d'), r'"\U0001f40d"') + + def test_c_unichar_repr(self): + c_unichar_repr = libclinic.c_unichar_repr + self.assertEqual(c_unichar_repr('a'), "'a'") + self.assertEqual(c_unichar_repr('\n'), r"'\n'") + self.assertEqual(c_unichar_repr('\b'), r"'\b'") + self.assertEqual(c_unichar_repr('\0'), '0') + self.assertEqual(c_unichar_repr('\1'), '0x01') + self.assertEqual(c_unichar_repr('\x7f'), '0x7f') + self.assertEqual(c_unichar_repr(' '), "' '") + self.assertEqual(c_unichar_repr('"'), """'"'""") + self.assertEqual(c_unichar_repr("'"), r"'\''") + self.assertEqual(c_unichar_repr('\\'), r"'\\'") + self.assertEqual(c_unichar_repr('?'), "'?'") + self.assertEqual(c_unichar_repr('\xa0'), '0xa0') + self.assertEqual(c_unichar_repr('\xff'), '0xff') + self.assertEqual(c_unichar_repr('\u20ac'), '0x20ac') + self.assertEqual(c_unichar_repr('\U0001f40d'), '0x1f40d') + def test_indent_all_lines(self): # Blank lines are expected to be unchanged. self.assertEqual(libclinic.indent_all_lines("", prefix="bar"), "") diff --git a/Lib/test/test_cmath.py b/Lib/test/test_cmath.py index a96a5780b31b6f1..389a3fa0e0a1eb6 100644 --- a/Lib/test/test_cmath.py +++ b/Lib/test/test_cmath.py @@ -406,6 +406,8 @@ def polar_with_errno_set(z): _testcapi.set_errno(0) self.check_polar(polar_with_errno_set) + @unittest.skipIf(sys.platform.startswith("sunos"), + "skipping, see gh-138573") def test_phase(self): self.assertAlmostEqual(phase(0), 0.) self.assertAlmostEqual(phase(1.), 0.) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 1b40e0d05fe3bca..2f4fee0443cff38 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -3,13 +3,13 @@ # See test_cmd_line_script.py for testing of script execution import os +import re import subprocess import sys import sysconfig import tempfile import textwrap import unittest -import warnings from test import support from test.support import os_helper from test.support import force_not_colorized @@ -39,7 +39,8 @@ def test_directories(self): def verify_valid_flag(self, cmd_line): rc, out, err = assert_python_ok(cmd_line) - self.assertTrue(out == b'' or out.endswith(b'\n')) + if out != b'': + self.assertEndsWith(out, b'\n') self.assertNotIn(b'Traceback', out) self.assertNotIn(b'Traceback', err) return out @@ -59,11 +60,22 @@ def test_help(self): def test_help_env(self): out = self.verify_valid_flag('--help-env') self.assertIn(b'PYTHONHOME', out) + # Env vars in each section should be sorted alphabetically + # (ignoring underscores so PYTHON_FOO and PYTHONFOO intermix naturally) + sort_key = lambda name: name.replace(b'_', b'').lower() + sections = out.split(b'These variables have equivalent') + for section in sections: + envvars = re.findall(rb'^(PYTHON\w+)', section, re.MULTILINE) + self.assertEqual(envvars, sorted(envvars, key=sort_key), + "env vars should be sorted alphabetically") @support.cpython_only def test_help_xoptions(self): out = self.verify_valid_flag('--help-xoptions') self.assertIn(b'-X dev', out) + options = re.findall(rb'^-X (\w+)', out, re.MULTILINE) + self.assertEqual(options, sorted(options), + "options should be sorted alphabetically") @support.cpython_only def test_help_all(self): @@ -89,8 +101,8 @@ def test_version(self): version = ('Python %d.%d' % sys.version_info[:2]).encode("ascii") for switch in '-V', '--version', '-VV': rc, out, err = assert_python_ok(switch) - self.assertFalse(err.startswith(version)) - self.assertTrue(out.startswith(version)) + self.assertNotStartsWith(err, version) + self.assertStartsWith(out, version) def test_verbose(self): # -v causes imports to write to stderr. If the write to @@ -200,6 +212,14 @@ def test_run_module_bug1764407(self): self.assertTrue(data.find(b'1 loop') != -1) self.assertTrue(data.find(b'__main__.Timer') != -1) + @support.cpython_only + def test_null_byte_in_interactive_mode(self): + # gh-140594: Fix an out of bounds read when a single NUL character + # is read from the standard input in interactive mode. + proc = spawn_python('-i') + proc.communicate(b'\x00', timeout=support.SHORT_TIMEOUT) + self.assertEqual(proc.returncode, 0) + def test_relativedir_bug46421(self): # Test `python -m unittest` with a relative directory beginning with ./ # Note: We have to switch to the project's top module's directory, as per @@ -380,7 +400,7 @@ def test_unbuffered_input(self): p.stdin.flush() data, rc = _kill_python_and_exit_code(p) self.assertEqual(rc, 0) - self.assertTrue(data.startswith(b'x'), data) + self.assertStartsWith(data, b'x') def test_large_PYTHONPATH(self): path1 = "ABCDE" * 100 @@ -936,21 +956,15 @@ def test_python_asyncio_debug(self): @unittest.skipUnless(sysconfig.get_config_var('Py_TRACE_REFS'), "Requires --with-trace-refs build option") def test_python_dump_refs(self): - code = 'import sys; sys._clear_type_cache()' - # TODO: Remove warnings context manager once sys._clear_type_cache is removed - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - rc, out, err = assert_python_ok('-c', code, PYTHONDUMPREFS='1') + code = 'import sys; sys._clear_internal_caches()' + rc, out, err = assert_python_ok('-c', code, PYTHONDUMPREFS='1') self.assertEqual(rc, 0) @unittest.skipUnless(sysconfig.get_config_var('Py_TRACE_REFS'), "Requires --with-trace-refs build option") def test_python_dump_refs_file(self): with tempfile.NamedTemporaryFile() as dump_file: - code = 'import sys; sys._clear_type_cache()' - # TODO: Remove warnings context manager once sys._clear_type_cache is removed - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - rc, out, err = assert_python_ok('-c', code, PYTHONDUMPREFSFILE=dump_file.name) + code = 'import sys; sys._clear_internal_caches()' + rc, out, err = assert_python_ok('-c', code, PYTHONDUMPREFSFILE=dump_file.name) self.assertEqual(rc, 0) with open(dump_file.name, 'r') as file: contents = file.read() @@ -972,10 +986,27 @@ def test_python_legacy_windows_fs_encoding(self): @unittest.skipUnless(support.MS_WINDOWS, 'Test only applicable on Windows') def test_python_legacy_windows_stdio(self): - code = "import sys; print(sys.stdin.encoding, sys.stdout.encoding)" - expected = 'cp' - rc, out, err = assert_python_ok('-c', code, PYTHONLEGACYWINDOWSSTDIO='1') - self.assertIn(expected.encode(), out) + # Test that _WindowsConsoleIO is used when PYTHONLEGACYWINDOWSSTDIO + # is not set. + # We cannot use PIPE becase it prevents creating new console. + # So we use exit code. + code = "import sys; sys.exit(type(sys.stdout.buffer.raw).__name__ != '_WindowsConsoleIO')" + env = os.environ.copy() + env["PYTHONLEGACYWINDOWSSTDIO"] = "" + p = subprocess.run([sys.executable, "-c", code], + creationflags=subprocess.CREATE_NEW_CONSOLE, + env=env) + support.skip_on_low_desktop_heap_memory_subprocess(p.returncode) + self.assertEqual(p.returncode, 0) + + # Then test that FIleIO is used when PYTHONLEGACYWINDOWSSTDIO is set. + code = "import sys; sys.exit(type(sys.stdout.buffer.raw).__name__ != 'FileIO')" + env["PYTHONLEGACYWINDOWSSTDIO"] = "1" + p = subprocess.run([sys.executable, "-c", code], + creationflags=subprocess.CREATE_NEW_CONSOLE, + env=env) + support.skip_on_low_desktop_heap_memory_subprocess(p.returncode) + self.assertEqual(p.returncode, 0) @unittest.skipIf("-fsanitize" in sysconfig.get_config_vars().get('PY_CFLAGS', ()), "PYTHONMALLOCSTATS doesn't work with ASAN") @@ -1024,7 +1055,7 @@ def test_parsing_error(self): stderr=subprocess.PIPE, text=True) err_msg = "Unknown option: --unknown-option\nusage: " - self.assertTrue(proc.stderr.startswith(err_msg), proc.stderr) + self.assertStartsWith(proc.stderr, err_msg) self.assertNotEqual(proc.returncode, 0) def test_int_max_str_digits(self): @@ -1231,6 +1262,17 @@ def test_invalid_thread_local_bytecode(self): rc, out, err = assert_python_failure(PYTHON_TLBC="2") self.assertIn(b"PYTHON_TLBC=N: N is missing or invalid", err) + def test_dump_path_config(self): + # gh-151253: At the first import (import encodings) during Python + # startup, if the import fails, dump the Python path configuration. + nonexistent = '/nonexistent-python-path' + # Use -X frozen_modules=off to disable frozen encodings module + # on release build. + cmd = ["-X", "frozen_modules=off", "-c", "pass"] + proc = assert_python_failure(*cmd, PYTHONHOME=nonexistent) + self.assertIn(b'Python path configuration:', proc.err) + self.assertIn(f"PYTHONHOME = '{nonexistent}'".encode(), proc.err) + @unittest.skipIf(interpreter_requires_environment(), 'Cannot run -I tests when PYTHON env vars are required.') diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 53dc9b1a7effb54..784c45aa96f8a7a 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -553,9 +553,9 @@ def test_pep_409_verbiage(self): exitcode, stdout, stderr = assert_python_failure(script_name) text = stderr.decode('ascii').split('\n') self.assertEqual(len(text), 5) - self.assertTrue(text[0].startswith('Traceback')) - self.assertTrue(text[1].startswith(' File ')) - self.assertTrue(text[3].startswith('NameError')) + self.assertStartsWith(text[0], 'Traceback') + self.assertStartsWith(text[1], ' File ') + self.assertStartsWith(text[3], 'NameError') def test_non_ascii(self): # Apple platforms deny the creation of a file with an invalid UTF-8 name. @@ -708,9 +708,8 @@ def test_syntaxerror_does_not_crash(self): exitcode, stdout, stderr = assert_python_failure(script_name) text = io.TextIOWrapper(io.BytesIO(stderr), 'ascii').read() # It used to crash in https://github.com/python/cpython/issues/111132 - self.assertTrue(text.endswith( - 'SyntaxError: nonlocal declaration not allowed at module level\n', - ), text) + self.assertEndsWith(text, + 'SyntaxError: nonlocal declaration not allowed at module level\n') def test_consistent_sys_path_for_direct_execution(self): # This test case ensures that the following all give the same diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index b646042a3b84b05..908e0ab3fb3a02e 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -210,16 +210,16 @@ ctypes = None from test.support import (cpython_only, check_impl_detail, requires_debug_ranges, - gc_collect, Py_GIL_DISABLED) + gc_collect, Py_GIL_DISABLED, late_deletion) from test.support.script_helper import assert_python_ok from test.support import threading_helper, import_helper from test.support.bytecode_helper import instructions_with_positions from opcode import opmap, opname -from _testcapi import code_offset_to_line try: import _testinternalcapi except ModuleNotFoundError: _testinternalcapi = None +import test._code_definitions as defs COPY_FREE_VARS = opmap['COPY_FREE_VARS'] @@ -671,8 +671,21 @@ def test_local_kinds(self): VARARGS = CO_FAST_LOCAL | CO_FAST_ARG_VAR | CO_FAST_ARG_POS VARKWARGS = CO_FAST_LOCAL | CO_FAST_ARG_VAR | CO_FAST_ARG_KW - import test._code_definitions as defs funcs = { + defs.simple_script: {}, + defs.complex_script: { + 'obj': CO_FAST_LOCAL, + 'pickle': CO_FAST_LOCAL, + 'spam_minimal': CO_FAST_LOCAL, + 'data': CO_FAST_LOCAL, + 'res': CO_FAST_LOCAL, + }, + defs.script_with_globals: { + 'obj1': CO_FAST_LOCAL, + 'obj2': CO_FAST_LOCAL, + }, + defs.script_with_explicit_empty_return: {}, + defs.script_with_return: {}, defs.spam_minimal: {}, defs.spam_with_builtins: { 'x': CO_FAST_LOCAL, @@ -687,6 +700,27 @@ def test_local_kinds(self): 'checks': CO_FAST_LOCAL, 'res': CO_FAST_LOCAL, }, + defs.spam_with_global_and_attr_same_name: {}, + defs.spam_full_args: { + 'a': POSONLY, + 'b': POSONLY, + 'c': POSORKW, + 'd': POSORKW, + 'e': KWONLY, + 'f': KWONLY, + 'args': VARARGS, + 'kwargs': VARKWARGS, + }, + defs.spam_full_args_with_defaults: { + 'a': POSONLY, + 'b': POSONLY, + 'c': POSORKW, + 'd': POSORKW, + 'e': KWONLY, + 'f': KWONLY, + 'args': VARARGS, + 'kwargs': VARKWARGS, + }, defs.spam_args_attrs_and_builtins: { 'a': POSONLY, 'b': POSONLY, @@ -700,6 +734,7 @@ def test_local_kinds(self): defs.spam_returns_arg: { 'x': POSORKW, }, + defs.spam_raises: {}, defs.spam_with_inner_not_closure: { 'eggs': CO_FAST_LOCAL, }, @@ -897,8 +932,20 @@ def new_var_counts(*, }, } - import test._code_definitions as defs funcs = { + defs.simple_script: new_var_counts(), + defs.complex_script: new_var_counts( + purelocals=5, + globalvars=1, + attrs=2, + ), + defs.script_with_globals: new_var_counts( + purelocals=2, + globalvars=1, + ), + defs.script_with_explicit_empty_return: new_var_counts(), + defs.script_with_return: new_var_counts(), + defs.spam_minimal: new_var_counts(), defs.spam_minimal: new_var_counts(), defs.spam_with_builtins: new_var_counts( purelocals=4, @@ -908,6 +955,24 @@ def new_var_counts(*, purelocals=5, globalvars=6, ), + defs.spam_with_global_and_attr_same_name: new_var_counts( + globalvars=2, + attrs=1, + ), + defs.spam_full_args: new_var_counts( + posonly=2, + posorkw=2, + kwonly=2, + varargs=1, + varkwargs=1, + ), + defs.spam_full_args_with_defaults: new_var_counts( + posonly=2, + posorkw=2, + kwonly=2, + varargs=1, + varkwargs=1, + ), defs.spam_args_attrs_and_builtins: new_var_counts( posonly=2, posorkw=2, @@ -919,6 +984,9 @@ def new_var_counts(*, defs.spam_returns_arg: new_var_counts( posorkw=1, ), + defs.spam_raises: new_var_counts( + globalvars=1, + ), defs.spam_with_inner_not_closure: new_var_counts( purelocals=1, ), @@ -1025,42 +1093,35 @@ def new_var_counts(*, counts = _testinternalcapi.get_code_var_counts(func.__code__) self.assertEqual(counts, expected) - def func_with_globals_and_builtins(): - mod1 = _testinternalcapi - mod2 = dis - mods = (mod1, mod2) - checks = tuple(callable(m) for m in mods) - return callable(mod2), tuple(mods), list(mods), checks - - func = func_with_globals_and_builtins + func = defs.spam_with_globals_and_builtins with self.subTest(f'{func} code'): expected = new_var_counts( - purelocals=4, - globalvars=5, + purelocals=5, + globalvars=6, ) counts = _testinternalcapi.get_code_var_counts(func.__code__) self.assertEqual(counts, expected) with self.subTest(f'{func} with own globals and builtins'): expected = new_var_counts( - purelocals=4, - globalvars=(2, 3), + purelocals=5, + globalvars=(2, 4), ) counts = _testinternalcapi.get_code_var_counts(func) self.assertEqual(counts, expected) with self.subTest(f'{func} without globals'): expected = new_var_counts( - purelocals=4, - globalvars=(0, 3, 2), + purelocals=5, + globalvars=(0, 4, 2), ) counts = _testinternalcapi.get_code_var_counts(func, globalsns={}) self.assertEqual(counts, expected) with self.subTest(f'{func} without both'): expected = new_var_counts( - purelocals=4, - globalvars=5, + purelocals=5, + globalvars=6, ) counts = _testinternalcapi.get_code_var_counts(func, globalsns={}, builtinsns={}) @@ -1068,12 +1129,40 @@ def func_with_globals_and_builtins(): with self.subTest(f'{func} without builtins'): expected = new_var_counts( - purelocals=4, - globalvars=(2, 0, 3), + purelocals=5, + globalvars=(2, 0, 4), ) counts = _testinternalcapi.get_code_var_counts(func, builtinsns={}) self.assertEqual(counts, expected) + @unittest.skipIf(_testinternalcapi is None, "missing _testinternalcapi") + def test_stateless(self): + self.maxDiff = None + + STATELESS_FUNCTIONS = [ + *defs.STATELESS_FUNCTIONS, + # stateless with defaults + defs.spam_full_args_with_defaults, + ] + + for func in defs.STATELESS_CODE: + with self.subTest((func, '(code)')): + _testinternalcapi.verify_stateless_code(func.__code__) + for func in STATELESS_FUNCTIONS: + with self.subTest((func, '(func)')): + _testinternalcapi.verify_stateless_code(func) + + for func in defs.FUNCTIONS: + if func not in defs.STATELESS_CODE: + with self.subTest((func, '(code)')): + with self.assertRaises(Exception): + _testinternalcapi.verify_stateless_code(func.__code__) + + if func not in STATELESS_FUNCTIONS: + with self.subTest((func, '(func)')): + with self.assertRaises(Exception): + _testinternalcapi.verify_stateless_code(func) + def isinterned(s): return s is sys.intern(('_' + s + '_')[1:-1]) @@ -1395,6 +1484,8 @@ async def async_func(): rc, out, err = assert_python_ok('-OO', '-c', code) def test_co_branches(self): + _testcapi = import_helper.import_module("_testcapi") + code_offset_to_line = _testcapi.code_offset_to_line def get_line_branches(func): code = func.__code__ @@ -1465,6 +1556,11 @@ def myfree(ptr): FREE_FUNC = freefunc(myfree) FREE_INDEX = RequestCodeExtraIndex(FREE_FUNC) + # Make sure myfree sticks around at least as long as the interpreter, + # since we (currently) can't unregister the function and leaving a + # dangling pointer will cause a crash on deallocation of code objects if + # something else uses co_extras, like test_capi.test_misc. + late_deletion(myfree) class CoExtra(unittest.TestCase): def get_func(self): diff --git a/Lib/test/test_code_module.py b/Lib/test/test_code_module.py index 57fb130070b34ed..3642b47c2c1f03f 100644 --- a/Lib/test/test_code_module.py +++ b/Lib/test/test_code_module.py @@ -133,7 +133,7 @@ def test_unicode_error(self): output = ''.join(''.join(call[1]) for call in self.stderr.method_calls) output = output[output.index('(InteractiveConsole)'):] output = output[output.index('\n') + 1:] - self.assertTrue(output.startswith('UnicodeEncodeError: '), output) + self.assertStartsWith(output, 'UnicodeEncodeError: ') self.assertIs(self.sysmod.last_type, UnicodeEncodeError) self.assertIs(type(self.sysmod.last_value), UnicodeEncodeError) self.assertIsNone(self.sysmod.last_traceback) diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index 86e5e5c14746741..a767f67a02cf568 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -2,6 +2,7 @@ import codecs import html.entities import itertools +import re import sys import unicodedata import unittest @@ -1125,7 +1126,7 @@ def test_bug828737(self): text = 'abcghi'*n text.translate(charmap) - def test_mutatingdecodehandler(self): + def test_mutating_decode_handler(self): baddata = [ ("ascii", b"\xff"), ("utf-7", b"++"), @@ -1160,6 +1161,42 @@ def mutating(exc): for (encoding, data) in baddata: self.assertEqual(data.decode(encoding, "test.mutating"), "\u4242") + def test_mutating_decode_handler_unicode_escape(self): + decode = codecs.unicode_escape_decode + def mutating(exc): + if isinstance(exc, UnicodeDecodeError): + r = data.get(exc.object[:exc.end]) + if r is not None: + exc.object = r[0] + exc.object[exc.end:] + return ('\u0404', r[1]) + raise AssertionError("don't know how to handle %r" % exc) + + codecs.register_error('test.mutating2', mutating) + data = { + br'\x0': (b'\\', 0), + br'\x3': (b'xxx\\', 3), + br'\x5': (b'x\\', 1), + } + def check(input, expected, msg): + with self.assertWarns(DeprecationWarning) as cm: + self.assertEqual(decode(input, 'test.mutating2'), (expected, len(input))) + self.assertIn(msg, str(cm.warning)) + + check(br'\x0n\z', '\u0404\n\\z', r'"\z" is an invalid escape sequence') + check(br'\x0n\501', '\u0404\n\u0141', r'"\501" is an invalid octal escape sequence') + check(br'\x0z', '\u0404\\z', r'"\z" is an invalid escape sequence') + + check(br'\x3n\zr', '\u0404\n\\zr', r'"\z" is an invalid escape sequence') + check(br'\x3zr', '\u0404\\zr', r'"\z" is an invalid escape sequence') + check(br'\x3z5', '\u0404\\z5', r'"\z" is an invalid escape sequence') + check(memoryview(br'\x3z5x')[:-1], '\u0404\\z5', r'"\z" is an invalid escape sequence') + check(memoryview(br'\x3z5xy')[:-2], '\u0404\\z5', r'"\z" is an invalid escape sequence') + + check(br'\x5n\z', '\u0404\n\\z', r'"\z" is an invalid escape sequence') + check(br'\x5n\501', '\u0404\n\u0141', r'"\501" is an invalid octal escape sequence') + check(br'\x5z', '\u0404\\z', r'"\z" is an invalid escape sequence') + check(memoryview(br'\x5zy')[:-1], '\u0404\\z', r'"\z" is an invalid escape sequence') + # issue32583 def test_crashing_decode_handler(self): # better generating one more character to fill the extra space slot diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 94fcf98e75721f6..1533fdcc9d34833 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1,8 +1,10 @@ import codecs import contextlib import copy +import importlib import io import pickle +import os import sys import unittest import encodings @@ -1196,23 +1198,39 @@ def test_escape(self): check(br"[\1010]", b"[A0]") check(br"[\x41]", b"[A]") check(br"[\x410]", b"[A0]") + + def test_warnings(self): + decode = codecs.escape_decode + check = coding_checker(self, decode) for i in range(97, 123): b = bytes([i]) if b not in b'abfnrtvx': - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\%c" is an invalid escape sequence' % i): check(b"\\" + b, b"\\" + b) - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\%c" is an invalid escape sequence' % (i-32)): check(b"\\" + b.upper(), b"\\" + b.upper()) - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\8" is an invalid escape sequence'): check(br"\8", b"\\8") with self.assertWarns(DeprecationWarning): check(br"\9", b"\\9") - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\\xfa" is an invalid escape sequence') as cm: check(b"\\\xfa", b"\\\xfa") for i in range(0o400, 0o1000): - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\%o" is an invalid octal escape sequence' % i): check(rb'\%o' % i, bytes([i & 0o377])) + with self.assertWarnsRegex(DeprecationWarning, + r'"\\z" is an invalid escape sequence'): + self.assertEqual(decode(br'\x\z', 'ignore'), (b'\\z', 4)) + with self.assertWarnsRegex(DeprecationWarning, + r'"\\501" is an invalid octal escape sequence'): + self.assertEqual(decode(br'\x\501', 'ignore'), (b'A', 6)) + def test_errors(self): decode = codecs.escape_decode self.assertRaises(ValueError, decode, br"\x") @@ -2661,24 +2679,40 @@ def test_escape_decode(self): check(br"[\x410]", "[A0]") check(br"\u20ac", "\u20ac") check(br"\U0001d120", "\U0001d120") + + def test_decode_warnings(self): + decode = codecs.unicode_escape_decode + check = coding_checker(self, decode) for i in range(97, 123): b = bytes([i]) if b not in b'abfnrtuvx': - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\%c" is an invalid escape sequence' % i): check(b"\\" + b, "\\" + chr(i)) if b.upper() not in b'UN': - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\%c" is an invalid escape sequence' % (i-32)): check(b"\\" + b.upper(), "\\" + chr(i-32)) - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\8" is an invalid escape sequence'): check(br"\8", "\\8") with self.assertWarns(DeprecationWarning): check(br"\9", "\\9") - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\\xfa" is an invalid escape sequence') as cm: check(b"\\\xfa", "\\\xfa") for i in range(0o400, 0o1000): - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r'"\\%o" is an invalid octal escape sequence' % i): check(rb'\%o' % i, chr(i)) + with self.assertWarnsRegex(DeprecationWarning, + r'"\\z" is an invalid escape sequence'): + self.assertEqual(decode(br'\x\z', 'ignore'), ('\\z', 4)) + with self.assertWarnsRegex(DeprecationWarning, + r'"\\501" is an invalid octal escape sequence'): + self.assertEqual(decode(br'\x\501', 'ignore'), ('\u0141', 6)) + def test_decode_errors(self): decode = codecs.unicode_escape_decode for c, d in (b'x', 2), (b'u', 4), (b'U', 4): @@ -3075,6 +3109,13 @@ def test_aliases(self): info = codecs.lookup(alias) self.assertEqual(info.name, expected_name) + def test_alias_modules_exist(self): + encodings_dir = os.path.dirname(encodings.__file__) + for value in encodings.aliases.aliases.values(): + codec_mod = f"encodings.{value}" + self.assertIsNotNone(importlib.util.find_spec(codec_mod), + f"Codec module not found: {codec_mod}") + def test_quopri_stateless(self): # Should encode with quotetabs=True encoded = codecs.encode(b"space tab\teol \n", "quopri-codec") @@ -3252,7 +3293,7 @@ def test_code_page_name(self): codecs.code_page_encode, 932, '\xff') self.assertRaisesRegex(UnicodeDecodeError, 'cp932', codecs.code_page_decode, 932, b'\x81\x00', 'strict', True) - self.assertRaisesRegex(UnicodeDecodeError, 'CP_UTF8', + self.assertRaisesRegex(UnicodeDecodeError, 'cp65001', codecs.code_page_decode, self.CP_UTF8, b'\xff', 'strict', True) def check_decode(self, cp, tests): @@ -3762,7 +3803,7 @@ def check_decode_strings(self, errors): with self.assertRaises(RuntimeError) as cm: self.decode(encoded, errors) errmsg = str(cm.exception) - self.assertTrue(errmsg.startswith("decode error: "), errmsg) + self.assertStartsWith(errmsg, "decode error: ") else: decoded = self.decode(encoded, errors) self.assertEqual(decoded, expected) @@ -3866,5 +3907,16 @@ def test_encodings_normalize_encoding(self): self.assertEqual(normalize('utf...8'), 'utf...8') +class CodecCacheTest(unittest.TestCase): + def test_cache_bounded(self): + for i in range(encodings._MAXCACHE + 1000): + try: + b'x'.decode(f'nonexist_{i}') + except LookupError: + pass + + self.assertLessEqual(len(encodings._cache), encodings._MAXCACHE) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 1e93530398be79b..c1dadc4e274ec97 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -26,7 +26,7 @@ from collections.abc import Set, MutableSet from collections.abc import Mapping, MutableMapping, KeysView, ItemsView, ValuesView from collections.abc import Sequence, MutableSequence -from collections.abc import Buffer +from collections.abc import ByteString, Buffer class TestUserObjects(unittest.TestCase): @@ -542,6 +542,8 @@ def test_odd_sizes(self): self.assertEqual(Dot(1)._replace(d=999), (999,)) self.assertEqual(Dot(1)._fields, ('d',)) + @support.requires_resource('cpu') + def test_large_size(self): n = support.exceeds_recursion_limit() names = list(set(''.join([choice(string.ascii_letters) for j in range(10)]) for i in range(n))) @@ -734,7 +736,7 @@ def validate_abstract_methods(self, abc, *names): stubs = methodstubs.copy() del stubs[name] C = type('C', (abc,), stubs) - self.assertRaises(TypeError, C, name) + self.assertRaises(TypeError, C) def validate_isinstance(self, abc, name): stub = lambda s, *args: 0 @@ -961,7 +963,7 @@ class AnextOnly: async def __anext__(self): raise StopAsyncIteration self.assertNotIsInstance(AnextOnly(), AsyncIterator) - self.validate_abstract_methods(AsyncIterator, '__anext__', '__aiter__') + self.validate_abstract_methods(AsyncIterator, '__anext__') def test_Iterable(self): # Check some non-iterables @@ -1157,7 +1159,7 @@ def test_Iterator(self): for x in samples: self.assertIsInstance(x, Iterator) self.assertIsSubclass(type(x), Iterator) - self.validate_abstract_methods(Iterator, '__next__', '__iter__') + self.validate_abstract_methods(Iterator, '__next__') # Issue 10565 class NextOnly: @@ -1841,8 +1843,7 @@ def test_Mapping(self): for sample in [dict]: self.assertIsInstance(sample(), Mapping) self.assertIsSubclass(sample, Mapping) - self.validate_abstract_methods(Mapping, '__contains__', '__iter__', '__len__', - '__getitem__') + self.validate_abstract_methods(Mapping, '__iter__', '__len__', '__getitem__') class MyMapping(Mapping): def __len__(self): return 0 @@ -1857,7 +1858,7 @@ def test_MutableMapping(self): for sample in [dict]: self.assertIsInstance(sample(), MutableMapping) self.assertIsSubclass(sample, MutableMapping) - self.validate_abstract_methods(MutableMapping, '__contains__', '__iter__', '__len__', + self.validate_abstract_methods(MutableMapping, '__iter__', '__len__', '__getitem__', '__setitem__', '__delitem__') def test_MutableMapping_subclass(self): @@ -1896,8 +1897,7 @@ def test_Sequence(self): self.assertIsInstance(memoryview(b""), Sequence) self.assertIsSubclass(memoryview, Sequence) self.assertIsSubclass(str, Sequence) - self.validate_abstract_methods(Sequence, '__contains__', '__iter__', '__len__', - '__getitem__') + self.validate_abstract_methods(Sequence, '__len__', '__getitem__') def test_Sequence_mixins(self): class SequenceSubclass(Sequence): @@ -1934,6 +1934,28 @@ def assert_index_same(seq1, seq2, index_args): assert_index_same( nativeseq, seqseq, (letter, start, stop)) + def test_ByteString(self): + for sample in [bytes, bytearray]: + with self.assertWarns(DeprecationWarning): + self.assertIsInstance(sample(), ByteString) + self.assertTrue(issubclass(sample, ByteString)) + for sample in [str, list, tuple]: + with self.assertWarns(DeprecationWarning): + self.assertNotIsInstance(sample(), ByteString) + self.assertFalse(issubclass(sample, ByteString)) + with self.assertWarns(DeprecationWarning): + self.assertNotIsInstance(memoryview(b""), ByteString) + self.assertFalse(issubclass(memoryview, ByteString)) + with self.assertWarns(DeprecationWarning): + self.validate_abstract_methods(ByteString, '__getitem__', '__len__') + + with self.assertWarns(DeprecationWarning): + class X(ByteString): pass + + with self.assertWarns(DeprecationWarning): + # No metaclass conflict + class Z(ByteString, Awaitable): pass + def test_Buffer(self): for sample in [bytes, bytearray, memoryview]: self.assertIsInstance(sample(b"x"), Buffer) @@ -1952,8 +1974,8 @@ def test_MutableSequence(self): self.assertIsSubclass(sample, MutableSequence) self.assertIsSubclass(array.array, MutableSequence) self.assertNotIsSubclass(str, MutableSequence) - self.validate_abstract_methods(MutableSequence, '__contains__', '__iter__', - '__len__', '__getitem__', '__setitem__', '__delitem__', 'insert') + self.validate_abstract_methods(MutableSequence, '__len__', '__getitem__', + '__setitem__', '__delitem__', 'insert') def test_MutableSequence_mixins(self): # Test the mixins of MutableSequence by creating a minimal concrete @@ -2096,6 +2118,19 @@ def test_basics(self): self.assertEqual(c.setdefault('e', 5), 5) self.assertEqual(c['e'], 5) + def test_update_reentrant_add_clears_counter(self): + c = Counter() + key = object() + + class Evil(int): + def __add__(self, other): + c.clear() + return NotImplemented + + c[key] = Evil() + c.update([key]) + self.assertEqual(c[key], 1) + def test_init(self): self.assertEqual(list(Counter(self=42).items()), [('self', 42)]) self.assertEqual(list(Counter(iterable=42).items()), [('iterable', 42)]) diff --git a/Lib/test/test_colorsys.py b/Lib/test/test_colorsys.py index 74d76294b0b4d46..9dfaef44a6a4508 100644 --- a/Lib/test/test_colorsys.py +++ b/Lib/test/test_colorsys.py @@ -42,6 +42,10 @@ def test_hsv_values(self): self.assertTripleEqual(hsv, colorsys.rgb_to_hsv(*rgb)) self.assertTripleEqual(rgb, colorsys.hsv_to_rgb(*hsv)) + # test 360 phase shift in hue + h, s, v = hsv + self.assertTripleEqual(rgb, colorsys.hsv_to_rgb(h + 1.0, s, v)) + def test_hls_roundtrip(self): for r in frange(0.0, 1.0, 0.2): for g in frange(0.0, 1.0, 0.2): @@ -89,6 +93,18 @@ def test_yiq_roundtrip(self): colorsys.yiq_to_rgb(*colorsys.rgb_to_yiq(*rgb)) ) + def test_yiq_to_rgb_clamping(self): + values = [ + # rgb, yiq (invalid YIQ values clamped to RGB range) + ((1.0, 0.0, 1.0), (0.0, 0.5, 1.0)), + ((0.0, 1.0, 0.0), (0.25, -1.0, -1.0)), + ((0.0, 0.0, 1.0), (0.0, -1.0, 0.5)) + ] + + for (rgb, yiq) in values: + with self.subTest(rgb=rgb, yiq=yiq): + self.assertTripleEqual(rgb, colorsys.yiq_to_rgb(*yiq)) + def test_yiq_values(self): values = [ # rgb, yiq diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 57e5f29b015637f..db1b4c6aa95186d 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -249,8 +249,8 @@ def test_32_63_bit_values(self): d = -281474976710656 # 1 << 48 e = +4611686018427387904 # 1 << 62 f = -4611686018427387904 # 1 << 62 - g = +9223372036854775807 # 1 << 63 - 1 - h = -9223372036854775807 # 1 << 63 - 1 + g = +9223372036854775807 # (1 << 63) - 1 + h = -9223372036854775807 # (1 << 63) - 1 for variable in self.test_32_63_bit_values.__code__.co_consts: if variable is not None: @@ -651,6 +651,21 @@ def test_compile_filename(self): compile('pass', filename, 'exec') self.assertRaises(TypeError, compile, 'pass', list(b'file.py'), 'exec') + def test_compile_filename_refleak(self): + # Regression tests for reference leak in PyUnicode_FSDecoder. + # See https://github.com/python/cpython/issues/139748. + mortal_str = 'this is a mortal string' + # check error path when 'mode' AC conversion failed + self.assertRaises(TypeError, compile, b'', mortal_str, mode=1234) + # check error path when 'optimize' AC conversion failed + self.assertRaises(OverflowError, compile, b'', mortal_str, + 'exec', optimize=1 << 1000) + # check error path when 'dont_inherit' AC conversion failed + class EvilBool: + def __bool__(self): raise ValueError + self.assertRaises(ValueError, compile, b'', mortal_str, + 'exec', dont_inherit=EvilBool()) + @support.cpython_only def test_same_filename_used(self): s = """def f(): pass\ndef g(): pass""" @@ -709,11 +724,13 @@ def test_yet_more_evil_still_undecodable(self): @support.cpython_only @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") + @support.skip_if_huge_c_stack(100_000 if sys.platform == "android" else 500_000) @support.skip_emscripten_stack_overflow() def test_compiler_recursion_limit(self): # Compiler frames are small limit = 100 - crash_depth = limit * 5000 + # Android test devices have less memory. + crash_depth = limit * (1000 if sys.platform == "android" else 5000) success_depth = limit def check_limit(prefix, repeated, mode="single"): @@ -1015,11 +1032,13 @@ def test_path_like_objects(self): # An implicit test for PyUnicode_FSDecoder(). compile("42", FakePath("test_compile_pathlike"), "single") + # bpo-31113: Stack overflow when compile a long sequence of + # complex statements. @support.requires_resource('cpu') def test_stack_overflow(self): - # bpo-31113: Stack overflow when compile a long sequence of - # complex statements. - compile("if a: b\n" * 200000, "", "exec") + # Android test devices have less memory. + size = 100_000 if sys.platform == "android" else 200_000 + compile("if a: b\n" * size, "", "exec") # Multiple users rely on the fact that CPython does not generate # bytecode for dead code blocks. See bpo-37500 for more context. @@ -1647,22 +1666,21 @@ class WeirdDict(dict): self.assertRaises(NameError, ns['foo']) def test_compile_warnings(self): - # See gh-131927 - # Compile warnings originating from the same file and - # line are now only emitted once. + # Each invocation of compile() emits compiler warnings, even if they + # have the same message and line number. + source = textwrap.dedent(r""" + # tokenizer + 1or 0 # line 3 + # code generator + 1 is 1 # line 5 + """) with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("default") - compile('1 is 1', '', 'eval') - compile('1 is 1', '', 'eval') - - self.assertEqual(len(caught), 1) - - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - compile('1 is 1', '', 'eval') - compile('1 is 1', '', 'eval') + for i in range(2): + # Even if compile() is at the same line. + compile(source, '', 'exec') - self.assertEqual(len(caught), 2) + self.assertEqual([wm.lineno for wm in caught], [3, 5] * 2) def test_compile_warning_in_finally(self): # Ensure that warnings inside finally blocks are @@ -1673,16 +1691,107 @@ def test_compile_warning_in_finally(self): try: pass finally: - 1 is 1 + 1 is 1 # line 5 + try: + pass + finally: # nested + 1 is 1 # line 9 """) with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("default") + warnings.simplefilter("always") compile(source, '', 'exec') - self.assertEqual(len(caught), 1) - self.assertEqual(caught[0].category, SyntaxWarning) - self.assertIn("\"is\" with 'int' literal", str(caught[0].message)) + self.assertEqual(sorted(wm.lineno for wm in caught), [5, 9]) + for wm in caught: + self.assertEqual(wm.category, SyntaxWarning) + self.assertIn("\"is\" with 'int' literal", str(wm.message)) + + # Other code path is used for "try" with "except*". + source = textwrap.dedent(""" + try: + pass + except *Exception: + pass + finally: + 1 is 1 # line 7 + try: + pass + except *Exception: + pass + finally: # nested + 1 is 1 # line 13 + """) + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + compile(source, '', 'exec') + + self.assertEqual(sorted(wm.lineno for wm in caught), [7, 13]) + for wm in caught: + self.assertEqual(wm.category, SyntaxWarning) + self.assertIn("\"is\" with 'int' literal", str(wm.message)) + + @support.subTests('src', [ + textwrap.dedent(""" + def f(): + try: + pass + finally: + return 42 + """), + textwrap.dedent(""" + for x in y: + try: + pass + finally: + break + """), + textwrap.dedent(""" + for x in y: + try: + pass + finally: + continue + """), + ]) + def test_pep_765_warnings(self, src): + with self.assertWarnsRegex(SyntaxWarning, 'finally'): + compile(src, '', 'exec') + with warnings.catch_warnings(): + warnings.simplefilter("error") + tree = ast.parse(src) + with self.assertWarnsRegex(SyntaxWarning, 'finally'): + compile(tree, '', 'exec') + + @support.subTests('src', [ + textwrap.dedent(""" + try: + pass + finally: + def f(): + return 42 + """), + textwrap.dedent(""" + try: + pass + finally: + for x in y: + break + """), + textwrap.dedent(""" + try: + pass + finally: + for x in y: + continue + """), + ]) + def test_pep_765_no_warnings(self, src): + with warnings.catch_warnings(): + warnings.simplefilter("error") + compile(src, '', 'exec') + class TestBooleanExpression(unittest.TestCase): class Value: @@ -1723,6 +1832,21 @@ def test_compound(self): self.assertIs(res, v[3]) self.assertEqual([e.called for e in v], [1, 1, 0, 1, 0]) + def test_exception(self): + # See gh-137288 + class Foo: + def __bool__(self): + raise NotImplementedError() + + a = Foo() + b = Foo() + + with self.assertRaises(NotImplementedError): + bool(a) + + with self.assertRaises(NotImplementedError): + c = a or b + @requires_debug_ranges() class TestSourcePositions(unittest.TestCase): # Ensure that compiled code snippets have correct line and column numbers diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index a580a240d9f4746..e9caa82a700d2fe 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -316,7 +316,7 @@ def _test_ddir_only(self, *, ddir, parallel=True): self.assertTrue(mods) for mod in mods: - self.assertTrue(mod.startswith(self.directory), mod) + self.assertStartsWith(mod, self.directory) modcode = importlib.util.cache_from_source(mod) modpath = mod[len(self.directory+os.sep):] _, _, err = script_helper.assert_python_failure(modcode) @@ -550,6 +550,24 @@ def temporary_pycache_prefix(self): finally: sys.pycache_prefix = old_prefix + @contextlib.contextmanager + def no_pycache_prefix(self): + """Ignore any ambient pycache prefix for the duration of the test. + + Some tests assume bytecode is written next to the source in a + __pycache__ directory. When the test suite is run with + PYTHONPYCACHEPREFIX set, neutralize it both in this process (used by + cache_from_source) and in any spawned subprocesses. + """ + old_prefix = sys.pycache_prefix + sys.pycache_prefix = None + try: + with os_helper.EnvironmentVarGuard() as env: + env.unset('PYTHONPYCACHEPREFIX') + yield + finally: + sys.pycache_prefix = old_prefix + def _get_run_args(self, args): return [*support.optim_args_from_interpreter_flags(), '-S', '-m', 'compileall', @@ -646,15 +664,16 @@ def test_legacy_paths(self): def test_multiple_runs(self): # Bug 8527 reported that multiple calls produced empty # __pycache__/__pycache__ directories. - self.assertRunOK('-q', self.pkgdir) - # Verify the __pycache__ directory contents. - self.assertTrue(os.path.exists(self.pkgdir_cachedir)) - cachecachedir = os.path.join(self.pkgdir_cachedir, '__pycache__') - self.assertFalse(os.path.exists(cachecachedir)) - # Call compileall again. - self.assertRunOK('-q', self.pkgdir) - self.assertTrue(os.path.exists(self.pkgdir_cachedir)) - self.assertFalse(os.path.exists(cachecachedir)) + with self.no_pycache_prefix(): + self.assertRunOK('-q', self.pkgdir) + # Verify the __pycache__ directory contents. + self.assertTrue(os.path.exists(self.pkgdir_cachedir)) + cachecachedir = os.path.join(self.pkgdir_cachedir, '__pycache__') + self.assertFalse(os.path.exists(cachecachedir)) + # Call compileall again. + self.assertRunOK('-q', self.pkgdir) + self.assertTrue(os.path.exists(self.pkgdir_cachedir)) + self.assertFalse(os.path.exists(cachecachedir)) @without_source_date_epoch # timestamp invalidation test def test_force(self): @@ -727,10 +746,13 @@ def test_symlink_loop(self): script_helper.make_pkg(pkg) os.symlink('.', os.path.join(pkg, 'evil')) os.symlink('.', os.path.join(pkg, 'evil2')) - self.assertRunOK('-q', self.pkgdir) - self.assertCompiled(os.path.join( - self.pkgdir, 'spam', 'evil', 'evil2', '__init__.py' - )) + # This relies on the __pycache__ layout (shared across the symlinked + # paths), so neutralize any ambient PYTHONPYCACHEPREFIX. + with self.no_pycache_prefix(): + self.assertRunOK('-q', self.pkgdir) + self.assertCompiled(os.path.join( + self.pkgdir, 'spam', 'evil', 'evil2', '__init__.py' + )) def test_quiet(self): noisy = self.assertRunOK(self.pkgdir) @@ -817,13 +839,16 @@ def test_include_on_stdin(self): f2 = script_helper.make_script(self.pkgdir, 'f2', '') f3 = script_helper.make_script(self.pkgdir, 'f3', '') f4 = script_helper.make_script(self.pkgdir, 'f4', '') - p = script_helper.spawn_python(*(self._get_run_args(()) + ['-i', '-'])) - p.stdin.write((f3+os.linesep).encode('ascii')) - script_helper.kill_python(p) - self.assertNotCompiled(f1) - self.assertNotCompiled(f2) - self.assertCompiled(f3) - self.assertNotCompiled(f4) + # spawn_python() runs with -E, ignoring PYTHONPYCACHEPREFIX, so make + # cache_from_source() in this process agree by neutralizing it too. + with self.no_pycache_prefix(): + p = script_helper.spawn_python(*(self._get_run_args(()) + ['-i', '-'])) + p.stdin.write((f3+os.linesep).encode('ascii')) + script_helper.kill_python(p) + self.assertNotCompiled(f1) + self.assertNotCompiled(f2) + self.assertCompiled(f3) + self.assertNotCompiled(f4) def test_compiles_as_much_as_possible(self): bingfn = script_helper.make_script(self.pkgdir, 'bing', 'syntax(error') diff --git a/Lib/test/test_compiler_assemble.py b/Lib/test/test_compiler_assemble.py index c4962e3599986e1..99a11e99d564852 100644 --- a/Lib/test/test_compiler_assemble.py +++ b/Lib/test/test_compiler_assemble.py @@ -146,4 +146,4 @@ def test_exception_table(self): L1 to L2 -> L2 [0] L2 to L3 -> L3 [1] lasti """) - self.assertTrue(output.getvalue().endswith(exc_table)) + self.assertEndsWith(output.getvalue(), exc_table) diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index 0c7e7341f13d4ea..bee2aceb1870276 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -72,8 +72,8 @@ def assertAlmostEqual(self, a, b): else: unittest.TestCase.assertAlmostEqual(self, a, b) - def assertCloseAbs(self, x, y, eps=1e-9): - """Return true iff floats x and y "are close".""" + def assertClose(self, x, y, eps=1e-9): + """Return true iff complexes x and y "are close".""" # put the one with larger magnitude second if abs(x) > abs(y): x, y = y, x @@ -82,26 +82,15 @@ def assertCloseAbs(self, x, y, eps=1e-9): if x == 0: return abs(y) < eps # check that relative difference < eps - self.assertTrue(abs((x-y)/y) < eps) - - def assertClose(self, x, y, eps=1e-9): - """Return true iff complexes x and y "are close".""" - self.assertCloseAbs(x.real, y.real, eps) - self.assertCloseAbs(x.imag, y.imag, eps) + self.assertTrue(abs(x-y)/abs(y) < eps) def check_div(self, x, y): """Compute complex z=x*y, and check that z/x==y and z/y==x.""" z = x * y - if x != 0: - q = z / x - self.assertClose(q, y) - q = z.__truediv__(x) - self.assertClose(q, y) - if y != 0: - q = z / y - self.assertClose(q, x) - q = z.__truediv__(y) - self.assertClose(q, x) + if x: + self.assertClose(z / x, y) + if y: + self.assertClose(z / y, x) def test_truediv(self): simple_real = [float(i) for i in range(-5, 6)] @@ -115,10 +104,20 @@ def test_truediv(self): self.check_div(complex(1e200, 1e200), 1+0j) self.check_div(complex(1e-200, 1e-200), 1+0j) + # Smith's algorithm has several sources of inaccuracy + # for components of the result. In examples below, + # it's cancellation of digits in computation of sum. + self.check_div(1e-09+1j, 1+1j) + self.check_div(8.289760544677449e-09+0.13257307440728516j, + 0.9059966714925808+0.5054864708672686j) + # Just for fun. for i in range(100): - self.check_div(complex(random(), random()), - complex(random(), random())) + x = complex(random(), random()) + y = complex(random(), random()) + self.check_div(x, y) + y = complex(1e10*y.real, y.imag) + self.check_div(x, y) self.assertAlmostEqual(complex.__truediv__(2+0j, 1+1j), 1-1j) self.assertRaises(TypeError, operator.truediv, 1j, None) @@ -454,7 +453,7 @@ def test_boolcontext(self): self.assertTrue(1j) def test_conjugate(self): - self.assertClose(complex(5.3, 9.8).conjugate(), 5.3-9.8j) + self.assertEqual(complex(5.3, 9.8).conjugate(), 5.3-9.8j) def test_constructor(self): def check(z, x, y): diff --git a/Lib/test/test_concurrent_futures/test_init.py b/Lib/test/test_concurrent_futures/test_init.py index df6409293093184..3a12dfe2c74d5d8 100644 --- a/Lib/test/test_concurrent_futures/test_init.py +++ b/Lib/test/test_concurrent_futures/test_init.py @@ -1,5 +1,6 @@ import contextlib import logging +import multiprocessing import queue import time import unittest @@ -20,6 +21,10 @@ def init(x): global INITIALIZER_STATUS INITIALIZER_STATUS = x + # InterpreterPoolInitializerTest.test_initializer fails + # if we don't have a LOAD_GLOBAL. (It could be any global.) + # We will address this separately. + INITIALIZER_STATUS def get_init_status(): return INITIALIZER_STATUS @@ -140,6 +145,8 @@ def test_spawn(self): self._test(ProcessPoolSpawnFailingInitializerTest) @support.skip_if_sanitizer("TSAN doesn't support threads after fork", thread=True) + @unittest.skipUnless("forkserver" in multiprocessing.get_all_start_methods(), + "forkserver start method is not available") def test_forkserver(self): self._test(ProcessPoolForkserverFailingInitializerTest) diff --git a/Lib/test/test_concurrent_futures/test_interpreter_pool.py b/Lib/test/test_concurrent_futures/test_interpreter_pool.py index f6c62ae4b2021b2..5ae6563ef23ee3d 100644 --- a/Lib/test/test_concurrent_futures/test_interpreter_pool.py +++ b/Lib/test/test_concurrent_futures/test_interpreter_pool.py @@ -1,36 +1,82 @@ +import _thread import asyncio import contextlib import io import os -import pickle +import subprocess +import sys +import textwrap import time import unittest -from concurrent.futures.interpreter import ( - ExecutionFailed, BrokenInterpreterPool, -) +from concurrent.futures.interpreter import BrokenInterpreterPool +from concurrent import interpreters +from concurrent.interpreters import _queues as queues import _interpreters from test import support +from test.support import os_helper +from test.support import script_helper import test.test_asyncio.utils as testasyncio_utils -from test.support.interpreters import queues from .executor import ExecutorTest, mul from .util import BaseTestCase, InterpreterPoolMixin, setup_module +WINDOWS = sys.platform.startswith('win') + + +@contextlib.contextmanager +def nonblocking(fd): + blocking = os.get_blocking(fd) + if blocking: + os.set_blocking(fd, False) + try: + yield + finally: + if blocking: + os.set_blocking(fd, blocking) + + +def read_file_with_timeout(fd, nbytes, timeout): + with nonblocking(fd): + end = time.time() + timeout + try: + return os.read(fd, nbytes) + except BlockingIOError: + pass + while time.time() < end: + try: + return os.read(fd, nbytes) + except BlockingIOError: + continue + else: + raise TimeoutError('nothing to read') + + +if not WINDOWS: + import select + def read_file_with_timeout(fd, nbytes, timeout): + r, _, _ = select.select([fd], [], [], timeout) + if fd not in r: + raise TimeoutError('nothing to read') + return os.read(fd, nbytes) + + def noop(): pass def write_msg(fd, msg): + import os os.write(fd, msg + b'\0') -def read_msg(fd): +def read_msg(fd, timeout=10.0): msg = b'' - while ch := os.read(fd, 1): - if ch == b'\0': - return msg + ch = read_file_with_timeout(fd, 1, timeout) + while ch != b'\0': msg += ch + ch = os.read(fd, 1) + return msg def get_current_name(): @@ -83,12 +129,12 @@ def test_init_script(self): """ os.write(w, b'\0') - executor = self.executor_type(initializer=initscript) - before_init = os.read(r, 100) - fut = executor.submit(script) - after_init = read_msg(r) - fut.result() - after_run = read_msg(r) + with self.executor_type(initializer=initscript) as executor: + before_init = os.read(r, 100) + fut = executor.submit(script) + after_init = read_msg(r) + fut.result() + after_run = read_msg(r) self.assertEqual(before_init, b'\0') self.assertEqual(after_init, msg1) @@ -104,15 +150,47 @@ def test_init_func(self): r, w = self.pipe() os.write(w, b'\0') - executor = self.executor_type( - initializer=write_msg, initargs=(w, msg)) - before = os.read(r, 100) - executor.submit(mul, 10, 10) - after = read_msg(r) + with self.executor_type( + initializer=write_msg, initargs=(w, msg)) as executor: + before = os.read(r, 100) + executor.submit(mul, 10, 10) + after = read_msg(r) self.assertEqual(before, b'\0') self.assertEqual(after, msg) + def test_init_with___main___global(self): + # See https://github.com/python/cpython/pull/133957#issuecomment-2927415311. + text = """if True: + from concurrent.futures import InterpreterPoolExecutor + + INITIALIZER_STATUS = 'uninitialized' + + def init(x): + global INITIALIZER_STATUS + INITIALIZER_STATUS = x + INITIALIZER_STATUS + + def get_init_status(): + return INITIALIZER_STATUS + + if __name__ == "__main__": + exe = InterpreterPoolExecutor(initializer=init, + initargs=('initialized',)) + fut = exe.submit(get_init_status) + print(fut.result()) # 'initialized' + exe.shutdown(wait=True) + print(INITIALIZER_STATUS) # 'uninitialized' + """ + with os_helper.temp_dir() as tempdir: + filename = script_helper.make_script(tempdir, 'my-script', text) + res = script_helper.assert_python_ok(filename) + stdout = res.out.decode('utf-8').strip() + self.assertEqual(stdout.splitlines(), [ + 'initialized', + 'uninitialized', + ]) + def test_init_closure(self): count = 0 def init1(): @@ -121,10 +199,19 @@ def init2(): nonlocal count count += 1 - with self.assertRaises(pickle.PicklingError): - self.executor_type(initializer=init1) - with self.assertRaises(pickle.PicklingError): - self.executor_type(initializer=init2) + with contextlib.redirect_stderr(io.StringIO()) as stderr: + with self.executor_type(initializer=init1) as executor: + fut = executor.submit(lambda: None) + self.assertIn('NotShareableError', stderr.getvalue()) + with self.assertRaises(BrokenInterpreterPool): + fut.result() + + with contextlib.redirect_stderr(io.StringIO()) as stderr: + with self.executor_type(initializer=init2) as executor: + fut = executor.submit(lambda: None) + self.assertIn('NotShareableError', stderr.getvalue()) + with self.assertRaises(BrokenInterpreterPool): + fut.result() def test_init_instance_method(self): class Spam: @@ -132,26 +219,12 @@ def initializer(self): raise NotImplementedError spam = Spam() - with self.assertRaises(pickle.PicklingError): - self.executor_type(initializer=spam.initializer) - - def test_init_shared(self): - msg = b'eggs' - r, w = self.pipe() - script = f"""if True: - import os - if __name__ != '__main__': - import __main__ - spam = __main__.spam - os.write({w}, spam + b'\\0') - """ - - executor = self.executor_type(shared={'spam': msg}) - fut = executor.submit(exec, script) - fut.result() - after = read_msg(r) - - self.assertEqual(after, msg) + with contextlib.redirect_stderr(io.StringIO()) as stderr: + with self.executor_type(initializer=spam.initializer) as executor: + fut = executor.submit(lambda: None) + self.assertIn('NotShareableError', stderr.getvalue()) + with self.assertRaises(BrokenInterpreterPool): + fut.result() @unittest.expectedFailure def test_init_exception_in_script(self): @@ -178,8 +251,6 @@ def test_init_exception_in_func(self): stderr = stderr.getvalue() self.assertIn('ExecutionFailed: Exception: spam', stderr) self.assertIn('Uncaught in the interpreter:', stderr) - self.assertIn('The above exception was the direct cause of the following exception:', - stderr) @unittest.expectedFailure def test_submit_script(self): @@ -189,11 +260,10 @@ def test_submit_script(self): import os os.write({w}, __name__.encode('utf-8') + b'\\0') """ - executor = self.executor_type() - - fut = executor.submit(script) - res = fut.result() - after = read_msg(r) + with self.executor_type() as executor: + fut = executor.submit(script) + res = fut.result() + after = read_msg(r) self.assertEqual(after, b'__main__') self.assertIs(res, None) @@ -207,20 +277,24 @@ def task2(): spam += 1 return spam - executor = self.executor_type() - with self.assertRaises(pickle.PicklingError): - executor.submit(task1) - with self.assertRaises(pickle.PicklingError): - executor.submit(task2) + with self.executor_type() as executor: + fut = executor.submit(task1) + with self.assertRaises(_interpreters.NotShareableError): + fut.result() + + fut = executor.submit(task2) + with self.assertRaises(_interpreters.NotShareableError): + fut.result() def test_submit_local_instance(self): class Spam: def __init__(self): self.value = True - executor = self.executor_type() - with self.assertRaises(pickle.PicklingError): - executor.submit(Spam) + with self.executor_type() as executor: + fut = executor.submit(Spam) + with self.assertRaises(_interpreters.NotShareableError): + fut.result() def test_submit_instance_method(self): class Spam: @@ -228,27 +302,29 @@ def run(self): return True spam = Spam() - executor = self.executor_type() - with self.assertRaises(pickle.PicklingError): - executor.submit(spam.run) + with self.executor_type() as executor: + fut = executor.submit(spam.run) + with self.assertRaises(_interpreters.NotShareableError): + fut.result() def test_submit_func_globals(self): - executor = self.executor_type() - fut = executor.submit(get_current_name) - name = fut.result() + with self.executor_type() as executor: + fut = executor.submit(get_current_name) + name = fut.result() self.assertEqual(name, __name__) self.assertNotEqual(name, '__main__') @unittest.expectedFailure def test_submit_exception_in_script(self): + # Scripts are not supported currently. fut = self.executor.submit('raise Exception("spam")') with self.assertRaises(Exception) as captured: fut.result() self.assertIs(type(captured.exception), Exception) self.assertEqual(str(captured.exception), 'spam') cause = captured.exception.__cause__ - self.assertIs(type(cause), ExecutionFailed) + self.assertIs(type(cause), interpreters.ExecutionFailed) for attr in ('__name__', '__qualname__', '__module__'): self.assertEqual(getattr(cause.excinfo.type, attr), getattr(Exception, attr)) @@ -261,7 +337,7 @@ def test_submit_exception_in_func(self): self.assertIs(type(captured.exception), Exception) self.assertEqual(str(captured.exception), 'spam') cause = captured.exception.__cause__ - self.assertIs(type(cause), ExecutionFailed) + self.assertIs(type(cause), interpreters.ExecutionFailed) for attr in ('__name__', '__qualname__', '__module__'): self.assertEqual(getattr(cause.excinfo.type, attr), getattr(Exception, attr)) @@ -269,16 +345,93 @@ def test_submit_exception_in_func(self): def test_saturation(self): blocker = queues.create() - executor = self.executor_type(4, shared=dict(blocker=blocker)) + executor = self.executor_type(4) for i in range(15 * executor._max_workers): - executor.submit(exec, 'import __main__; __main__.blocker.get()') - #executor.submit('blocker.get()') + executor.submit(blocker.get) self.assertEqual(len(executor._threads), executor._max_workers) for i in range(15 * executor._max_workers): blocker.put_nowait(None) executor.shutdown(wait=True) + def test_blocking(self): + # There is no guarantee that a worker will be created for every + # submitted task. That's because there's a race between: + # + # * a new worker thread, created when task A was just submitted, + # becoming non-idle when it picks up task A + # * after task B is added to the queue, a new worker thread + # is started only if there are no idle workers + # (the check in ThreadPoolExecutor._adjust_thread_count()) + # + # That means we must not block waiting for *all* tasks to report + # "ready" before we unblock the known-ready workers. + ready = queues.create() + blocker = queues.create() + + def run(taskid, ready, blocker): + # There can't be any globals here. + ready.put_nowait(taskid) + blocker.get() # blocking + + numtasks = 10 + futures = [] + with self.executor_type() as executor: + # Request the jobs. + for i in range(numtasks): + fut = executor.submit(run, i, ready, blocker) + futures.append(fut) + pending = numtasks + while pending > 0: + # Wait for any to be ready. + done = 0 + for _ in range(pending): + try: + ready.get(timeout=1) # blocking + except interpreters.QueueEmpty: + pass + else: + done += 1 + pending -= done + # Unblock the workers. + for _ in range(done): + blocker.put_nowait(None) + + def test_blocking_with_limited_workers(self): + # This is essentially the same as test_blocking, + # but we explicitly force a limited number of workers, + # instead of it happening implicitly sometimes due to a race. + ready = queues.create() + blocker = queues.create() + + def run(taskid, ready, blocker): + # There can't be any globals here. + ready.put_nowait(taskid) + blocker.get() # blocking + + numtasks = 10 + futures = [] + with self.executor_type(4) as executor: + # Request the jobs. + for i in range(numtasks): + fut = executor.submit(run, i, ready, blocker) + futures.append(fut) + pending = numtasks + while pending > 0: + # Wait for any to be ready. + done = 0 + for _ in range(pending): + try: + ready.get(timeout=1) # blocking + except interpreters.QueueEmpty: + pass + else: + done += 1 + pending -= done + # Unblock the workers. + for _ in range(done): + blocker.put_nowait(None) + @support.requires_gil_enabled("gh-117344: test is flaky without the GIL") def test_idle_thread_reuse(self): executor = self.executor_type() @@ -289,13 +442,75 @@ def test_idle_thread_reuse(self): executor.shutdown(wait=True) def test_pickle_errors_propagate(self): - # GH-125864: Pickle errors happen before the script tries to execute, so the - # queue used to wait infinitely. - + # GH-125864: Pickle errors happen before the script tries to execute, + # so the queue used to wait infinitely. fut = self.executor.submit(PickleShenanigans(0)) - with self.assertRaisesRegex(RuntimeError, "gotcha"): + expected = interpreters.NotShareableError + with self.assertRaisesRegex(expected, 'args not shareable') as cm: fut.result() - + self.assertRegex(str(cm.exception.__cause__), 'unpickled') + + def test_no_stale_references(self): + # Weak references don't cross between interpreters. + raise unittest.SkipTest('not applicable') + + def test_free_reference(self): + # Weak references don't cross between interpreters. + raise unittest.SkipTest('not applicable') + + @support.requires_subprocess() + def test_import_interpreter_pool_executor(self): + # Test the import behavior normally if _interpreters is unavailable. + code = textwrap.dedent(""" + import sys + # Set it to None to emulate the case when _interpreter is unavailable. + sys.modules['_interpreters'] = None + from concurrent import futures + + try: + futures.InterpreterPoolExecutor + except AttributeError: + pass + else: + print('AttributeError not raised!', file=sys.stderr) + sys.exit(1) + + try: + from concurrent.futures import InterpreterPoolExecutor + except ImportError: + pass + else: + print('ImportError not raised!', file=sys.stderr) + sys.exit(1) + + from concurrent.futures import * + + if 'InterpreterPoolExecutor' in globals(): + print('InterpreterPoolExecutor should not be imported!', + file=sys.stderr) + sys.exit(1) + """) + + cmd = [sys.executable, '-c', code] + p = subprocess.run(cmd, capture_output=True) + self.assertEqual(p.returncode, 0, p.stderr.decode()) + self.assertEqual(p.stdout.decode(), '') + self.assertEqual(p.stderr.decode(), '') + + def test_thread_name_prefix(self): + self.assertStartsWith(self.executor._thread_name_prefix, + "InterpreterPoolExecutor-") + + @unittest.skipUnless(hasattr(_thread, '_get_name'), "missing _thread._get_name") + def test_thread_name_prefix_with_thread_get_name(self): + def get_thread_name(): + import _thread + return _thread._get_name() + + # Some platforms (Linux) are using 16 bytes to store the thread name, + # so only compare the first 15 bytes (without the trailing \n). + self.assertStartsWith(self.executor.submit(get_thread_name).result(), + "InterpreterPoolExecutor-"[:15]) class AsyncioTest(InterpretersMixin, testasyncio_utils.TestCase): @@ -310,7 +525,7 @@ def setUpClass(cls): # tests left a policy in place, just in case. policy = support.maybe_get_event_loop_policy() assert policy is None, policy - cls.addClassCleanup(lambda: asyncio._set_event_loop_policy(None)) + cls.addClassCleanup(lambda: asyncio.events._set_event_loop_policy(None)) def setUp(self): super().setUp() diff --git a/Lib/test/test_concurrent_futures/test_process_pool.py b/Lib/test/test_concurrent_futures/test_process_pool.py index 3f13a1900a4ca4c..9f79753fe20d05e 100644 --- a/Lib/test/test_concurrent_futures/test_process_pool.py +++ b/Lib/test/test_concurrent_futures/test_process_pool.py @@ -3,6 +3,7 @@ import sys import threading import time +import traceback import unittest import unittest.mock from concurrent import futures @@ -61,6 +62,31 @@ def test_killed_child(self): # Submitting other jobs fails as well. self.assertRaises(BrokenProcessPool, self.executor.submit, pow, 2, 8) + def test_broken_process_pool_traceback(self): + # When a child process is abruptly terminated, the whole pool gets + # "broken", and a BrokenProcessPool exception should be created + # for each future instead of sharing one exception among all futures. + event = self.create_event() + futures = [self.executor.submit(event.wait) for _ in range(3)] + p = next(iter(self.executor._processes.values())) + p.terminate() + for fut in futures: + # Don't use assertRaises(): it clears the traceback off exc. + try: + fut.result() + except BrokenProcessPool as exc: + tb = exc.__traceback__ + else: + self.fail("BrokenProcessPool not raised") + count = sum( + 1 + for frame_summary in traceback.extract_tb(tb) + if frame_summary.filename == __file__ + ) + # This code file should appear exactly once in the traceback. + # A shared exception would accumulate a frame per result() call. + self.assertEqual(count, 1) + def test_map_chunksize(self): def bad_map(): list(self.executor.map(pow, range(40), range(40), chunksize=-1)) @@ -193,6 +219,33 @@ def test_max_tasks_per_child_defaults_to_spawn_context(self): executor = self.executor_type(1, max_tasks_per_child=3) self.assertEqual(executor._mp_context.get_start_method(), "spawn") + def test_max_tasks_per_child_pending_tasks_gh115634(self): + # gh-115634: A worker exiting at its max_tasks_per_child limit left a + # stale token in the idle worker semaphore, so no replacement worker + # was spawned and the remaining queued tasks deadlocked. Submit more + # tasks than the pool can run at once so a backlog is queued while + # workers hit their task limit. + context = self.get_context() + if context.get_start_method(allow_none=False) == "fork": + raise unittest.SkipTest("Incompatible with the fork start method.") + + for max_workers, max_tasks, num_tasks in [(1, 2, 6), (2, 2, 8)]: + with self.subTest(max_workers=max_workers, max_tasks=max_tasks): + executor = self.executor_type( + max_workers, mp_context=context, + max_tasks_per_child=max_tasks) + try: + futures = [executor.submit(mul, i, 2) + for i in range(num_tasks)] + # If the deadlock regresses, the result() calls time out, + # and the shutdown below hangs until the test timeout. + results = [f.result(timeout=support.SHORT_TIMEOUT) + for f in futures] + self.assertEqual(results, + [i * 2 for i in range(num_tasks)]) + finally: + executor.shutdown(wait=True, cancel_futures=True) + def test_max_tasks_early_shutdown(self): context = self.get_context() if context.get_start_method(allow_none=False) == "fork": diff --git a/Lib/test/test_concurrent_futures/test_shutdown.py b/Lib/test/test_concurrent_futures/test_shutdown.py index 7a4065afd46fc84..b5cf83020056a08 100644 --- a/Lib/test/test_concurrent_futures/test_shutdown.py +++ b/Lib/test/test_concurrent_futures/test_shutdown.py @@ -111,20 +111,21 @@ def test_hang_gh83386(self): See https://github.com/python/cpython/issues/83386. """ - if self.executor_type == futures.ProcessPoolExecutor: - raise unittest.SkipTest( - "Hangs, see https://github.com/python/cpython/issues/83386") - rc, out, err = assert_python_ok('-c', """if True: from concurrent.futures import {executor_type} from test.test_concurrent_futures.test_shutdown import sleep_and_print if __name__ == "__main__": - if {context!r}: multiprocessing.set_start_method({context!r}) - t = {executor_type}(max_workers=3) + context = '{context}' + if context == "": + t = {executor_type}(max_workers=3) + else: + from multiprocessing import get_context + t = {executor_type}(max_workers=3, + mp_context=get_context(context)) t.submit(sleep_and_print, 1.0, "apple") t.shutdown(wait=False) """.format(executor_type=self.executor_type.__name__, - context=getattr(self, 'ctx', None))) + context=getattr(self, 'ctx', ''))) self.assertFalse(err) self.assertEqual(out.strip(), b"apple") @@ -234,8 +235,6 @@ def test_thread_names_default(self): t.join() def test_cancel_futures_wait_false(self): - # Can only be reliably tested for TPE, since PPE often hangs with - # `wait=False` (even without *cancel_futures*). rc, out, err = assert_python_ok('-c', """if True: from concurrent.futures import ThreadPoolExecutor from test.test_concurrent_futures.test_shutdown import sleep_and_print @@ -330,6 +329,64 @@ def test_shutdown_no_wait(self): # shutdown. assert all([r == abs(v) for r, v in zip(res, range(-5, 5))]) + @classmethod + def _failing_task_gh_132969(cls, n): + raise ValueError("failing task") + + @classmethod + def _good_task_gh_132969(cls, n): + time.sleep(0.1 * n) + return n + + def _run_test_issue_gh_132969(self, max_workers): + # max_workers=2 will repro exception + # max_workers=4 will repro exception and then hang + + # Repro conditions + # max_tasks_per_child=1 + # a task ends abnormally + # shutdown(wait=False) is called + start_method = self.get_context().get_start_method() + if (start_method == "fork" or + (start_method == "forkserver" and sys.platform.startswith("win"))): + self.skipTest(f"Skipping test for {start_method = }") + executor = futures.ProcessPoolExecutor( + max_workers=max_workers, + max_tasks_per_child=1, + mp_context=self.get_context()) + f1 = executor.submit(ProcessPoolShutdownTest._good_task_gh_132969, 1) + f2 = executor.submit(ProcessPoolShutdownTest._failing_task_gh_132969, 2) + f3 = executor.submit(ProcessPoolShutdownTest._good_task_gh_132969, 3) + result = 0 + try: + result += f1.result() + result += f2.result() + result += f3.result() + except ValueError: + # stop processing results upon first exception + pass + + # Ensure that the executor cleans up after called + # shutdown with wait=False + executor_manager_thread = executor._executor_manager_thread + executor.shutdown(wait=False) + time.sleep(0.2) + executor_manager_thread.join() + return result + + def test_shutdown_gh_132969_case_1(self): + # gh-132969: test that exception "object of type 'NoneType' has no len()" + # is not raised when shutdown(wait=False) is called. + result = self._run_test_issue_gh_132969(2) + self.assertEqual(result, 1) + + def test_shutdown_gh_132969_case_2(self): + # gh-132969: test that process does not hang and + # exception "object of type 'NoneType' has no len()" is not raised + # when shutdown(wait=False) is called. + result = self._run_test_issue_gh_132969(4) + self.assertEqual(result, 1) + create_executor_tests(globals(), ProcessPoolShutdownTest, executor_mixins=(ProcessPoolForkMixin, diff --git a/Lib/test/test_concurrent_futures/util.py b/Lib/test/test_concurrent_futures/util.py index b12940414d9142d..842fdb0431ed18a 100644 --- a/Lib/test/test_concurrent_futures/util.py +++ b/Lib/test/test_concurrent_futures/util.py @@ -138,6 +138,8 @@ def get_context(self): self.skipTest("require unix system") if support.check_sanitizer(thread=True): self.skipTest("TSAN doesn't support threads after fork") + if "forkserver" not in multiprocessing.get_all_start_methods(): + self.skipTest("forkserver start method is not available") return super().get_context() def create_event(self): diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index 23904d17d326d81..4783943f71a1092 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -526,6 +526,17 @@ def test_default_case_sensitivity(self): cf.get(self.default_section, "Foo"), "Bar", "could not locate option, expecting case-insensitive defaults") + def test_crlf_normalization(self): + cf = self.newconfig({"key1": "a\nb","key2": "a\rb", "key3": "a\r\nb", "key4": "a\r\nb"}) + buf = io.StringIO() + cf.write(buf) + cf_str = buf.getvalue() + self.assertNotIn("\r", cf_str) + self.assertNotIn("\r\n", cf_str) + self.assertEqual(cf_str.count("\n"), 10) + self.assertEqual(cf_str.count("\n\t"), 4) + self.assertTrue(cf_str.endswith("\n\n")) + def test_parse_errors(self): cf = self.newconfig() self.parse_error(cf, configparser.ParsingError, @@ -986,12 +997,12 @@ def test_add_section_default(self): def test_defaults_keyword(self): """bpo-23835 fix for ConfigParser""" - cf = self.newconfig(defaults={1: 2.4}) - self.assertEqual(cf[self.default_section]['1'], '2.4') - self.assertAlmostEqual(cf[self.default_section].getfloat('1'), 2.4) - cf = self.newconfig(defaults={"A": 5.2}) - self.assertEqual(cf[self.default_section]['a'], '5.2') - self.assertAlmostEqual(cf[self.default_section].getfloat('a'), 5.2) + cf = self.newconfig(defaults={1: 2.5}) + self.assertEqual(cf[self.default_section]['1'], '2.5') + self.assertAlmostEqual(cf[self.default_section].getfloat('1'), 2.5) + cf = self.newconfig(defaults={"A": 5.25}) + self.assertEqual(cf[self.default_section]['a'], '5.25') + self.assertAlmostEqual(cf[self.default_section].getfloat('a'), 5.25) class ConfigParserTestCaseNoInterpolation(BasicTestCase, unittest.TestCase): @@ -1729,6 +1740,19 @@ def test_error(self): self.assertEqual(e1.message, e2.message) self.assertEqual(repr(e1), repr(e2)) + def test_combine_error_linear_complexity(self): + # Ensure that ParsingError.combine() has linear complexity. + # See https://github.com/python/cpython/issues/148370. + n = 50000 + s = '[*]\n' + (err_line := '=\n') * n + p = configparser.ConfigParser(strict=False) + with self.assertRaises(configparser.ParsingError) as cm: + p.read_string(s) + errlines = cm.exception.message.splitlines() + self.assertEqual(len(errlines), n + 1) + self.assertStartsWith(errlines[0], "Source contains parsing errors: ") + self.assertEqual(errlines[42], f"\t[line {43:2d}]: {err_line!r}") + def test_nosectionerror(self): import pickle e1 = configparser.NoSectionError('section') @@ -2215,6 +2239,16 @@ def test_add_section(self): cfg.add_section(configparser.UNNAMED_SECTION) cfg.set(configparser.UNNAMED_SECTION, 'a', '1') self.assertEqual('1', cfg[configparser.UNNAMED_SECTION]['a']) + output = io.StringIO() + cfg.write(output) + self.assertEqual(output.getvalue(), 'a = 1\n\n') + + cfg = configparser.ConfigParser(allow_unnamed_section=True) + cfg[configparser.UNNAMED_SECTION] = {'a': '1'} + self.assertEqual('1', cfg[configparser.UNNAMED_SECTION]['a']) + output = io.StringIO() + cfg.write(output) + self.assertEqual(output.getvalue(), 'a = 1\n\n') def test_disabled_error(self): with self.assertRaises(configparser.MissingSectionHeaderError): @@ -2223,6 +2257,9 @@ def test_disabled_error(self): with self.assertRaises(configparser.UnnamedSectionDisabledError): configparser.ConfigParser().add_section(configparser.UNNAMED_SECTION) + with self.assertRaises(configparser.UnnamedSectionDisabledError): + configparser.ConfigParser()[configparser.UNNAMED_SECTION] = {'a': '1'} + def test_multiple_configs(self): cfg = configparser.ConfigParser(allow_unnamed_section=True) cfg.read_string('a = 1') @@ -2257,6 +2294,26 @@ def test_section_bracket_in_key(self): output.close() +class ReDoSTestCase(unittest.TestCase): + """Regression tests for quadratic regex backtracking (gh-146333).""" + + def test_option_regex_does_not_backtrack(self): + # A line with many spaces between non-delimiter characters + # should be parsed in linear time, not quadratic. + parser = configparser.RawConfigParser() + content = "[section]\n" + "x" + " " * 40000 + "y" + "\n" + # This should complete almost instantly. Before the fix, + # it would take over a minute due to catastrophic backtracking. + with self.assertRaises(configparser.ParsingError): + parser.read_string(content) + + def test_option_regex_no_value_does_not_backtrack(self): + parser = configparser.RawConfigParser(allow_no_value=True) + content = "[section]\n" + "x" + " " * 40000 + "y" + "\n" + parser.read_string(content) + self.assertTrue(parser.has_option("section", "x" + " " * 40000 + "y")) + + class MiscTestCase(unittest.TestCase): def test__all__(self): support.check__all__(self, configparser, not_exported={"Error"}) diff --git a/Lib/test/test_context.py b/Lib/test/test_context.py index a08038b5dbd4071..ef20495dcc01ea9 100644 --- a/Lib/test/test_context.py +++ b/Lib/test/test_context.py @@ -556,6 +556,36 @@ def fun(): ctx.run(fun) + def test_context_eq_reentrant_contextvar_set(self): + var = contextvars.ContextVar("v") + ctx1 = contextvars.Context() + ctx2 = contextvars.Context() + + class ReentrantEq: + def __eq__(self, other): + ctx1.run(lambda: var.set(object())) + return True + + ctx1.run(var.set, ReentrantEq()) + ctx2.run(var.set, object()) + ctx1 == ctx2 + + def test_context_eq_reentrant_contextvar_set_in_hash(self): + var = contextvars.ContextVar("v") + ctx1 = contextvars.Context() + ctx2 = contextvars.Context() + + class ReentrantHash: + def __hash__(self): + ctx1.run(lambda: var.set(object())) + return 0 + def __eq__(self, other): + return isinstance(other, ReentrantHash) + + ctx1.run(var.set, ReentrantHash()) + ctx2.run(var.set, ReentrantHash()) + ctx1 == ctx2 + # HAMT Tests diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index cf6519598037e9a..6a3329fa5aaacef 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -48,23 +48,23 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): return None - self.assertTrue(issubclass(ManagerFromScratch, AbstractContextManager)) + self.assertIsSubclass(ManagerFromScratch, AbstractContextManager) class DefaultEnter(AbstractContextManager): def __exit__(self, *args): super().__exit__(*args) - self.assertTrue(issubclass(DefaultEnter, AbstractContextManager)) + self.assertIsSubclass(DefaultEnter, AbstractContextManager) class NoEnter(ManagerFromScratch): __enter__ = None - self.assertFalse(issubclass(NoEnter, AbstractContextManager)) + self.assertNotIsSubclass(NoEnter, AbstractContextManager) class NoExit(ManagerFromScratch): __exit__ = None - self.assertFalse(issubclass(NoExit, AbstractContextManager)) + self.assertNotIsSubclass(NoExit, AbstractContextManager) class ContextManagerTestCase(unittest.TestCase): diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index 7750186e56a5ccb..dcd0072037950e1 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -77,23 +77,23 @@ async def __aenter__(self): async def __aexit__(self, exc_type, exc_value, traceback): return None - self.assertTrue(issubclass(ManagerFromScratch, AbstractAsyncContextManager)) + self.assertIsSubclass(ManagerFromScratch, AbstractAsyncContextManager) class DefaultEnter(AbstractAsyncContextManager): async def __aexit__(self, *args): await super().__aexit__(*args) - self.assertTrue(issubclass(DefaultEnter, AbstractAsyncContextManager)) + self.assertIsSubclass(DefaultEnter, AbstractAsyncContextManager) class NoneAenter(ManagerFromScratch): __aenter__ = None - self.assertFalse(issubclass(NoneAenter, AbstractAsyncContextManager)) + self.assertNotIsSubclass(NoneAenter, AbstractAsyncContextManager) class NoneAexit(ManagerFromScratch): __aexit__ = None - self.assertFalse(issubclass(NoneAexit, AbstractAsyncContextManager)) + self.assertNotIsSubclass(NoneAexit, AbstractAsyncContextManager) class AsyncContextManagerTestCase(unittest.TestCase): diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index d76341417e9bef3..b452ddbef5ccbee 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -19,7 +19,7 @@ class TestCopy(unittest.TestCase): def test_exceptions(self): self.assertIs(copy.Error, copy.error) - self.assertTrue(issubclass(copy.Error, Exception)) + self.assertIsSubclass(copy.Error, Exception) # The copy() method @@ -371,7 +371,9 @@ def test_deepcopy_list(self): self.assertIsNot(x, y) self.assertIsNot(x[0], y[0]) + @support.skip_if_huge_c_stack() @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_deepcopy_reflexive_list(self): x = [] x.append(x) @@ -399,7 +401,9 @@ def test_deepcopy_tuple_of_immutables(self): y = copy.deepcopy(x) self.assertIs(x, y) + @support.skip_if_huge_c_stack() @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_deepcopy_reflexive_tuple(self): x = ([],) x[0].append(x) @@ -417,7 +421,9 @@ def test_deepcopy_dict(self): self.assertIsNot(x, y) self.assertIsNot(x["foo"], y["foo"]) + @support.skip_if_huge_c_stack() @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_deepcopy_reflexive_dict(self): x = {} x['foo'] = x @@ -576,6 +582,7 @@ def __eq__(self, other): self.assertIsNot(y, x) self.assertIsNot(y.foo, x.foo) + @support.skip_if_huge_c_stack() def test_deepcopy_reflexive_inst(self): class C: pass @@ -638,6 +645,7 @@ def __eq__(self, other): self.assertEqual(y, x) self.assertIsNot(y.foo, x.foo) + @support.skip_if_huge_c_stack() def test_reconstruct_reflexive(self): class C(object): pass @@ -669,7 +677,7 @@ def __eq__(self, other): def test_reduce_5tuple(self): class C(dict): def __reduce__(self): - return (C, (), self.__dict__, None, self.items()) + return (C, (), self.__dict__, None, iter(self.items())) def __eq__(self, other): return (dict(self) == dict(other) and self.__dict__ == other.__dict__) diff --git a/Lib/test/test_coroutines.py b/Lib/test/test_coroutines.py index 761cb230277bd97..a515e0f5ca9b5f4 100644 --- a/Lib/test/test_coroutines.py +++ b/Lib/test/test_coroutines.py @@ -527,7 +527,7 @@ class CoroutineTest(unittest.TestCase): def test_gen_1(self): def gen(): yield - self.assertFalse(hasattr(gen, '__await__')) + self.assertNotHasAttr(gen, '__await__') def test_func_1(self): async def foo(): @@ -2265,6 +2265,36 @@ def c(): # before fixing, visible stack from throw would be shorter than from send. self.assertEqual(len_send, len_throw) + def test_call_aiter_once_in_comprehension(self): + + class AsyncIterator: + + def __init__(self): + self.val = 0 + + async def __anext__(self): + if self.val == 2: + raise StopAsyncIteration + self.val += 1 + return self.val + + # No __aiter__ method + + class C: + + def __aiter__(self): + return AsyncIterator() + + async def run_listcomp(): + return [i async for i in C()] + + async def run_asyncgen(): + ag = (i async for i in C()) + return [i async for i in ag] + + self.assertEqual(run_async(run_listcomp()), ([], [1, 2])) + self.assertEqual(run_async(run_asyncgen()), ([], [1, 2])) + @unittest.skipIf( support.is_emscripten or support.is_wasi, @@ -2307,7 +2337,7 @@ async def f(): pass finally: loop.close() - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) self.assertEqual(buffer, [1, 2, 'MyException']) diff --git a/Lib/test/test_cppext/__init__.py b/Lib/test/test_cppext/__init__.py index 2b7adac4bccd151..5b4c97c181bb6ae 100644 --- a/Lib/test/test_cppext/__init__.py +++ b/Lib/test/test_cppext/__init__.py @@ -1,9 +1,11 @@ # gh-91321: Build a basic C++ test extension to check that the Python C API is # compatible with C++ and does not emit C++ compiler warnings. import os.path +import platform import shlex import shutil import subprocess +import sys import unittest from test import support @@ -24,41 +26,19 @@ @support.requires_venv_with_pip() @support.requires_subprocess() @support.requires_resource('cpu') -class TestCPPExt(unittest.TestCase): - def test_build(self): - self.check_build('_testcppext') - - def test_build_cpp03(self): - # In public docs, we say C API is compatible with C++11. However, - # in practice we do maintain C++03 compatibility in public headers. - # Please ask the C API WG before adding a new C++11-only feature. - self.check_build('_testcpp03ext', std='c++03') - - @support.requires_gil_enabled('incompatible with Free Threading') - def test_build_limited_cpp03(self): - self.check_build('_test_limited_cpp03ext', std='c++03', limited=True) - - @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c++11") - def test_build_cpp11(self): - self.check_build('_testcpp11ext', std='c++11') - - # Only test C++14 on MSVC. - # On s390x RHEL7, GCC 4.8.5 doesn't support C++14. - @unittest.skipIf(not support.MS_WINDOWS, "need Windows") - def test_build_cpp14(self): - self.check_build('_testcpp14ext', std='c++14') +class BaseTests: + TEST_INTERNAL_C_API = False - @support.requires_gil_enabled('incompatible with Free Threading') - def test_build_limited(self): - self.check_build('_testcppext_limited', limited=True) - - def check_build(self, extension_name, std=None, limited=False): + def check_build(self, extension_name, std=None, limited=False, + extra_cflags=None): venv_dir = 'env' with support.setup_venv_with_pip_setuptools(venv_dir) as python_exe: self._check_build(extension_name, python_exe, - std=std, limited=limited) + std=std, limited=limited, + extra_cflags=extra_cflags) - def _check_build(self, extension_name, python_exe, std, limited): + def _check_build(self, extension_name, python_exe, std, limited, + extra_cflags=None): pkg_dir = 'pkg' os.mkdir(pkg_dir) shutil.copy(SETUP, os.path.join(pkg_dir, os.path.basename(SETUP))) @@ -71,6 +51,9 @@ def run_cmd(operation, cmd): if limited: env['CPYTHON_TEST_LIMITED'] = '1' env['CPYTHON_TEST_EXT_NAME'] = extension_name + env['TEST_INTERNAL_C_API'] = str(int(self.TEST_INTERNAL_C_API)) + if extra_cflags: + env['CPYTHON_TEST_EXTRA_CFLAGS'] = extra_cflags if support.verbose: print('Run:', ' '.join(map(shlex.quote, cmd))) subprocess.run(cmd, check=True, env=env) @@ -111,5 +94,53 @@ def run_cmd(operation, cmd): run_cmd('Import', cmd) +class TestPublicCAPI(BaseTests, unittest.TestCase): + def test_build(self): + self.check_build('_testcppext') + + @support.requires_gil_enabled('incompatible with Free Threading') + def test_build_limited_cpp03(self): + self.check_build('_test_limited_cpp03ext', std='c++03', limited=True) + + @support.requires_gil_enabled('incompatible with Free Threading') + def test_build_limited(self): + self.check_build('_testcppext_limited', limited=True) + + def test_build_cpp03(self): + # In public docs, we say C API is compatible with C++11. However, + # in practice we do maintain C++03 compatibility in public headers. + # Please ask the C API WG before adding a new C++11-only feature. + self.check_build('_testcpp03ext', std='c++03') + + @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support /std:c++11") + def test_build_cpp11(self): + self.check_build('_testcpp11ext', std='c++11') + + # Only test C++14 on MSVC. + # On s390x RHEL7, GCC 4.8.5 doesn't support C++14. + @unittest.skipIf(not support.MS_WINDOWS, "need Windows") + def test_build_cpp14(self): + self.check_build('_testcpp14ext', std='c++14') + + # Test that headers compile with Intel asm syntax, which may conflict + # with inline assembly in free-threading headers that use AT&T syntax. + @unittest.skipIf(support.MS_WINDOWS, "MSVC doesn't support -masm=intel") + @unittest.skipUnless(platform.machine() in ('x86_64', 'i686', 'AMD64'), + "x86-specific flag") + def test_build_intel_asm(self): + self.check_build('_testcppext_asm', extra_cflags='-masm=intel') + + +class TestInteralCAPI(BaseTests, unittest.TestCase): + TEST_INTERNAL_C_API = True + + def test_build(self): + kwargs = {} + if sys.platform == 'darwin': + # Old Apple clang++ default C++ std is gnu++98 + kwargs['std'] = 'c++11' + self.check_build('_testcppext_internal', **kwargs) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_cppext/extension.cpp b/Lib/test/test_cppext/extension.cpp index 5b3571b295bec3b..4db63df94f52334 100644 --- a/Lib/test/test_cppext/extension.cpp +++ b/Lib/test/test_cppext/extension.cpp @@ -6,7 +6,31 @@ // Always enable assertions #undef NDEBUG +#ifdef TEST_INTERNAL_C_API +# define Py_BUILD_CORE_MODULE 1 +#endif + #include "Python.h" +#include "datetime.h" + +#ifdef TEST_INTERNAL_C_API + // gh-135906: Check for compiler warnings in the internal C API + // - Cython uses pycore_critical_section.h, pycore_frame.h and + // pycore_template.h. + // - greenlet uses pycore_frame.h, pycore_interpframe_structs.h and + // pycore_interpframe.h. +# include "internal/pycore_frame.h" +# include "internal/pycore_interpframe_structs.h" +# include "internal/pycore_template.h" + + // mimalloc emits compiler warnings on Windows. +# if !defined(MS_WINDOWS) +# include "internal/pycore_backoff.h" +# include "internal/pycore_cell.h" +# include "internal/pycore_critical_section.h" +# include "internal/pycore_interpframe.h" +# endif +#endif #ifndef MODULE_NAME # error "MODULE_NAME macro must be defined" @@ -214,11 +238,26 @@ test_virtual_object(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) Py_RETURN_NONE; } +static PyObject * +test_datetime(PyObject *Py_UNUSED(module), PyObject *Py_UNUSED(args)) +{ + // datetime.h is excluded from the limited C API +#ifndef Py_LIMITED_API + PyDateTime_IMPORT; + if (PyErr_Occurred()) { + return NULL; + } +#endif + + Py_RETURN_NONE; +} + static PyMethodDef _testcppext_methods[] = { {"add", _testcppext_add, METH_VARARGS, _testcppext_add_doc}, {"test_api_casts", test_api_casts, METH_NOARGS, _Py_NULL}, {"test_unicode", test_unicode, METH_NOARGS, _Py_NULL}, {"test_virtual_object", test_virtual_object, METH_NOARGS, _Py_NULL}, + {"test_datetime", test_datetime, METH_NOARGS, _Py_NULL}, // Note: _testcppext_exec currently runs all test functions directly. // When adding a new one, add a call there. @@ -247,6 +286,10 @@ _testcppext_exec(PyObject *module) if (!result) return -1; Py_DECREF(result); + result = PyObject_CallMethod(module, "test_datetime", ""); + if (!result) return -1; + Py_DECREF(result); + // test Py_BUILD_ASSERT() and Py_BUILD_ASSERT_EXPR() Py_BUILD_ASSERT(sizeof(int) == sizeof(unsigned int)); assert(Py_BUILD_ASSERT_EXPR(sizeof(int) == sizeof(unsigned int)) == 0); diff --git a/Lib/test/test_cppext/setup.py b/Lib/test/test_cppext/setup.py index ea1ed64bf7ab0a2..14aeafefcaa8f72 100644 --- a/Lib/test/test_cppext/setup.py +++ b/Lib/test/test_cppext/setup.py @@ -47,6 +47,7 @@ def main(): std = os.environ.get("CPYTHON_TEST_CPP_STD", "") module_name = os.environ["CPYTHON_TEST_EXT_NAME"] limited = bool(os.environ.get("CPYTHON_TEST_LIMITED", "")) + internal = bool(int(os.environ.get("TEST_INTERNAL_C_API", "0"))) cppflags = list(CPPFLAGS) cppflags.append(f'-DMODULE_NAME={module_name}') @@ -58,7 +59,7 @@ def main(): else: cppflags.append(f'-std={std}') - if limited or (std != 'c++03'): + if limited or (std != 'c++03') and not internal: # See CPPFLAGS_PEDANTIC docstring cppflags.extend(CPPFLAGS_PEDANTIC) @@ -82,6 +83,13 @@ def main(): version = sys.hexversion cppflags.append(f'-DPy_LIMITED_API={version:#x}') + if internal: + cppflags.append('-DTEST_INTERNAL_C_API=1') + + extra_cflags = os.environ.get("CPYTHON_TEST_EXTRA_CFLAGS", "") + if extra_cflags: + cppflags.extend(shlex.split(extra_cflags)) + # On Windows, add PCbuild\amd64\ to include and library directories include_dirs = [] library_dirs = [] @@ -97,7 +105,7 @@ def main(): print(f"Add PCbuild directory: {pcbuild}") # Display information to help debugging - for env_name in ('CC', 'CFLAGS', 'CPPFLAGS'): + for env_name in ('CC', 'CXX', 'CFLAGS', 'CPPFLAGS', 'CXXFLAGS'): if env_name in os.environ: print(f"{env_name} env var: {os.environ[env_name]!r}") else: diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index 192c8eab26ebff3..c903ecd5f65756e 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -118,28 +118,30 @@ def test_second_profiler(self): pr = self.profilerclass() pr2 = self.profilerclass() pr.enable() - self.assertRaises(ValueError, pr2.enable) - pr.disable() + self.addCleanup(pr.disable) + msg = f"tool {sys.monitoring.PROFILER_ID} is already in use" + self.assertRaisesRegex(ValueError, msg, pr2.enable) def test_throw(self): """ gh-106152 generator.throw() should trigger a call in cProfile - In the any() call below, there should be two entries for the generator: - * one for the call to __next__ which gets a True and terminates any - * one when the generator is garbage collected which will effectively - do a throw. """ + + def gen(): + yield + pr = self.profilerclass() pr.enable() - any(a == 1 for a in (1, 2)) + g = gen() + try: + g.throw(SyntaxError) + except SyntaxError: + pass pr.disable() pr.create_stats() - for func, (cc, nc, _, _, _) in pr.stats.items(): - if func[2] == "": - self.assertEqual(cc, 1) - self.assertEqual(nc, 1) + self.assertTrue(any("throw" in func[2] for func in pr.stats.keys())), def test_bad_descriptor(self): # gh-132250 diff --git a/Lib/test/test_crossinterp.py b/Lib/test/test_crossinterp.py index 5ac0080db435a82..06cbe268348d1c5 100644 --- a/Lib/test/test_crossinterp.py +++ b/Lib/test/test_crossinterp.py @@ -3,9 +3,12 @@ import importlib.util import itertools import sys +import traceback import types import unittest +import warnings +from test import support from test.support import import_helper _testinternalcapi = import_helper.import_module('_testinternalcapi') @@ -16,13 +19,281 @@ from test import _crossinterp_definitions as defs -BUILTIN_TYPES = [o for _, o in __builtins__.items() - if isinstance(o, type)] -EXCEPTION_TYPES = [cls for cls in BUILTIN_TYPES +@contextlib.contextmanager +def ignore_byteswarning(): + with warnings.catch_warnings(): + warnings.filterwarnings('ignore', category=BytesWarning) + yield + + +# builtin types + +BUILTINS_TYPES = [o for _, o in __builtins__.items() if isinstance(o, type)] +EXCEPTION_TYPES = [cls for cls in BUILTINS_TYPES if issubclass(cls, BaseException)] OTHER_TYPES = [o for n, o in vars(types).items() if (isinstance(o, type) and - n not in ('DynamicClassAttribute', '_GeneratorWrapper'))] + n not in ('DynamicClassAttribute', '_GeneratorWrapper'))] +BUILTIN_TYPES = [ + *BUILTINS_TYPES, + *OTHER_TYPES, +] + +# builtin exceptions + +try: + raise Exception +except Exception as exc: + CAUGHT = exc +EXCEPTIONS_WITH_SPECIAL_SIG = { + BaseExceptionGroup: (lambda msg: (msg, [CAUGHT])), + ExceptionGroup: (lambda msg: (msg, [CAUGHT])), + UnicodeError: (lambda msg: (None, msg, None, None, None)), + UnicodeEncodeError: (lambda msg: ('utf-8', '', 1, 3, msg)), + UnicodeDecodeError: (lambda msg: ('utf-8', b'', 1, 3, msg)), + UnicodeTranslateError: (lambda msg: ('', 1, 3, msg)), +} +BUILTIN_EXCEPTIONS = [ + *(cls(*sig('error!')) for cls, sig in EXCEPTIONS_WITH_SPECIAL_SIG.items()), + *(cls('error!') for cls in EXCEPTION_TYPES + if cls not in EXCEPTIONS_WITH_SPECIAL_SIG), +] + +# other builtin objects + +METHOD = defs.SpamOkay().okay +BUILTIN_METHOD = [].append +METHOD_DESCRIPTOR_WRAPPER = str.join +METHOD_WRAPPER = object().__str__ +WRAPPER_DESCRIPTOR = object.__init__ +BUILTIN_WRAPPERS = { + METHOD: types.MethodType, + BUILTIN_METHOD: types.BuiltinMethodType, + dict.__dict__['fromkeys']: types.ClassMethodDescriptorType, + types.FunctionType.__code__: types.GetSetDescriptorType, + types.FunctionType.__globals__: types.MemberDescriptorType, + METHOD_DESCRIPTOR_WRAPPER: types.MethodDescriptorType, + METHOD_WRAPPER: types.MethodWrapperType, + WRAPPER_DESCRIPTOR: types.WrapperDescriptorType, + staticmethod(defs.SpamOkay.okay): None, + classmethod(defs.SpamOkay.okay): None, + property(defs.SpamOkay.okay): None, +} +BUILTIN_FUNCTIONS = [ + # types.BuiltinFunctionType + len, + sys.is_finalizing, + sys.exit, + _testinternalcapi.get_crossinterp_data, +] +assert 'emptymod' not in sys.modules +with import_helper.ready_to_import('emptymod', ''): + import emptymod as EMPTYMOD +MODULES = [ + sys, + defs, + unittest, + EMPTYMOD, +] +OBJECT = object() +EXCEPTION = Exception() +LAMBDA = (lambda: None) +BUILTIN_SIMPLE = [ + OBJECT, + # singletons + None, + True, + False, + Ellipsis, + NotImplemented, + # bytes + *(i.to_bytes(2, 'little', signed=True) + for i in range(-1, 258)), + # str + 'hello world', + '你好世界', + '', + # int + sys.maxsize + 1, + sys.maxsize, + -sys.maxsize - 1, + -sys.maxsize - 2, + *range(-1, 258), + 2**1000, + # float + 0.0, + 1.1, + -1.0, + 0.12345678, + -0.12345678, +] +TUPLE_EXCEPTION = (0, 1.0, EXCEPTION) +TUPLE_OBJECT = (0, 1.0, OBJECT) +TUPLE_NESTED_EXCEPTION = (0, 1.0, (EXCEPTION,)) +TUPLE_NESTED_OBJECT = (0, 1.0, (OBJECT,)) +MEMORYVIEW_EMPTY = memoryview(b'') +MEMORYVIEW_NOT_EMPTY = memoryview(b'spam'*42) +MAPPING_PROXY_EMPTY = types.MappingProxyType({}) +BUILTIN_CONTAINERS = [ + # tuple (flat) + (), + (1,), + ("hello", "world", ), + (1, True, "hello"), + TUPLE_EXCEPTION, + TUPLE_OBJECT, + # tuple (nested) + ((1,),), + ((1, 2), (3, 4)), + ((1, 2), (3, 4), (5, 6)), + TUPLE_NESTED_EXCEPTION, + TUPLE_NESTED_OBJECT, + # buffer + MEMORYVIEW_EMPTY, + MEMORYVIEW_NOT_EMPTY, + # list + [], + [1, 2, 3], + [[1], (2,), {3: 4}], + # dict + {}, + {1: 7, 2: 8, 3: 9}, + {1: [1], 2: (2,), 3: {3: 4}}, + # set + set(), + {1, 2, 3}, + {frozenset({1}), (2,)}, + # frozenset + frozenset([]), + frozenset({frozenset({1}), (2,)}), + # bytearray + bytearray(b''), + # other + MAPPING_PROXY_EMPTY, + types.SimpleNamespace(), +] +ns = {} +exec(""" +try: + raise Exception +except Exception as exc: + TRACEBACK = exc.__traceback__ + FRAME = TRACEBACK.tb_frame +""", ns, ns) +BUILTIN_OTHER = [ + # types.CellType + types.CellType(), + # types.FrameType + ns['FRAME'], + # types.TracebackType + ns['TRACEBACK'], +] +del ns + +# user-defined objects + +USER_TOP_INSTANCES = [c(*a) for c, a in defs.TOP_CLASSES.items()] +USER_NESTED_INSTANCES = [c(*a) for c, a in defs.NESTED_CLASSES.items()] +USER_INSTANCES = [ + *USER_TOP_INSTANCES, + *USER_NESTED_INSTANCES, +] +USER_EXCEPTIONS = [ + defs.MimimalError('error!'), +] + +# shareable objects + +TUPLES_WITHOUT_EQUALITY = [ + TUPLE_EXCEPTION, + TUPLE_OBJECT, + TUPLE_NESTED_EXCEPTION, + TUPLE_NESTED_OBJECT, +] +_UNSHAREABLE_SIMPLE = [ + Ellipsis, + NotImplemented, + OBJECT, + sys.maxsize + 1, + -sys.maxsize - 2, + 2**1000, +] +with ignore_byteswarning(): + _SHAREABLE_SIMPLE = [o for o in BUILTIN_SIMPLE + if o not in _UNSHAREABLE_SIMPLE] + _SHAREABLE_CONTAINERS = [ + *(o for o in BUILTIN_CONTAINERS if type(o) is memoryview), + *(o for o in BUILTIN_CONTAINERS + if type(o) is tuple and o not in TUPLES_WITHOUT_EQUALITY), + ] + _UNSHAREABLE_CONTAINERS = [o for o in BUILTIN_CONTAINERS + if o not in _SHAREABLE_CONTAINERS] +SHAREABLE = [ + *_SHAREABLE_SIMPLE, + *_SHAREABLE_CONTAINERS, +] +NOT_SHAREABLE = [ + *_UNSHAREABLE_SIMPLE, + *_UNSHAREABLE_CONTAINERS, + *BUILTIN_TYPES, + *BUILTIN_WRAPPERS, + *BUILTIN_EXCEPTIONS, + *BUILTIN_FUNCTIONS, + *MODULES, + *BUILTIN_OTHER, + # types.CodeType + *(f.__code__ for f in defs.FUNCTIONS), + *(f.__code__ for f in defs.FUNCTION_LIKE), + # types.FunctionType + *defs.FUNCTIONS, + defs.SpamOkay.okay, + LAMBDA, + *defs.FUNCTION_LIKE, + # coroutines and generators + *defs.FUNCTION_LIKE_APPLIED, + # user classes + *defs.CLASSES, + *USER_INSTANCES, + # user exceptions + *USER_EXCEPTIONS, +] + +# pickleable objects + +PICKLEABLE = [ + *BUILTIN_SIMPLE, + *(o for o in BUILTIN_CONTAINERS if o not in [ + MEMORYVIEW_EMPTY, + MEMORYVIEW_NOT_EMPTY, + MAPPING_PROXY_EMPTY, + ] or type(o) is dict), + *BUILTINS_TYPES, + *BUILTIN_EXCEPTIONS, + *BUILTIN_FUNCTIONS, + *defs.TOP_FUNCTIONS, + defs.SpamOkay.okay, + *defs.FUNCTION_LIKE, + *defs.TOP_CLASSES, + *USER_TOP_INSTANCES, + *USER_EXCEPTIONS, + # from OTHER_TYPES + types.NoneType, + types.EllipsisType, + types.NotImplementedType, + types.GenericAlias, + types.UnionType, + types.SimpleNamespace, + # from BUILTIN_WRAPPERS + METHOD, + BUILTIN_METHOD, + METHOD_DESCRIPTOR_WRAPPER, + METHOD_WRAPPER, + WRAPPER_DESCRIPTOR, +] +assert not any(isinstance(o, types.MappingProxyType) for o in PICKLEABLE) + + +# helpers DEFS = defs with open(code_defs.__file__) as infile: @@ -111,6 +382,77 @@ class _GetXIDataTests(unittest.TestCase): MODE = None + def assert_functions_equal(self, func1, func2): + assert type(func1) is types.FunctionType, repr(func1) + assert type(func2) is types.FunctionType, repr(func2) + self.assertEqual(func1.__name__, func2.__name__) + self.assertEqual(func1.__code__, func2.__code__) + self.assertEqual(func1.__defaults__, func2.__defaults__) + self.assertEqual(func1.__kwdefaults__, func2.__kwdefaults__) + # We don't worry about __globals__ for now. + + def assert_exc_args_equal(self, exc1, exc2): + args1 = exc1.args + args2 = exc2.args + if isinstance(exc1, ExceptionGroup): + self.assertIs(type(args1), type(args2)) + self.assertEqual(len(args1), 2) + self.assertEqual(len(args1), len(args2)) + self.assertEqual(args1[0], args2[0]) + group1 = args1[1] + group2 = args2[1] + self.assertEqual(len(group1), len(group2)) + for grouped1, grouped2 in zip(group1, group2): + # Currently the "extra" attrs are not preserved + # (via __reduce__). + self.assertIs(type(exc1), type(exc2)) + self.assert_exc_equal(grouped1, grouped2) + else: + self.assertEqual(args1, args2) + + def assert_exc_equal(self, exc1, exc2): + self.assertIs(type(exc1), type(exc2)) + + if type(exc1).__eq__ is not object.__eq__: + self.assertEqual(exc1, exc2) + + self.assert_exc_args_equal(exc1, exc2) + # XXX For now we do not preserve tracebacks. + if exc1.__traceback__ is not None: + self.assertEqual(exc1.__traceback__, exc2.__traceback__) + self.assertEqual( + getattr(exc1, '__notes__', None), + getattr(exc2, '__notes__', None), + ) + # We assume there are no cycles. + if exc1.__cause__ is None: + self.assertIs(exc1.__cause__, exc2.__cause__) + else: + self.assert_exc_equal(exc1.__cause__, exc2.__cause__) + if exc1.__context__ is None: + self.assertIs(exc1.__context__, exc2.__context__) + else: + self.assert_exc_equal(exc1.__context__, exc2.__context__) + + def assert_equal_or_equalish(self, obj, expected): + cls = type(expected) + if cls.__eq__ is not object.__eq__: + self.assertEqual(obj, expected) + elif cls is types.FunctionType: + self.assert_functions_equal(obj, expected) + elif isinstance(expected, BaseException): + self.assert_exc_equal(obj, expected) + elif cls is types.MethodType: + raise NotImplementedError(cls) + elif cls is types.BuiltinMethodType: + raise NotImplementedError(cls) + elif cls is types.MethodWrapperType: + raise NotImplementedError(cls) + elif cls.__bases__ == (object,): + self.assertEqual(obj.__dict__, expected.__dict__) + else: + raise NotImplementedError(cls) + def get_xidata(self, obj, *, mode=None): mode = self._resolve_mode(mode) return _testinternalcapi.get_crossinterp_data(obj, mode) @@ -126,35 +468,37 @@ def _get_roundtrip(self, obj, mode): def assert_roundtrip_identical(self, values, *, mode=None): mode = self._resolve_mode(mode) for obj in values: - with self.subTest(obj): + with self.subTest(repr(obj)): got = self._get_roundtrip(obj, mode) self.assertIs(got, obj) def assert_roundtrip_equal(self, values, *, mode=None, expecttype=None): mode = self._resolve_mode(mode) for obj in values: - with self.subTest(obj): + with self.subTest(repr(obj)): got = self._get_roundtrip(obj, mode) - self.assertEqual(got, obj) + if got is obj: + continue self.assertIs(type(got), type(obj) if expecttype is None else expecttype) + self.assert_equal_or_equalish(got, obj) def assert_roundtrip_equal_not_identical(self, values, *, mode=None, expecttype=None): mode = self._resolve_mode(mode) for obj in values: - with self.subTest(obj): + with self.subTest(repr(obj)): got = self._get_roundtrip(obj, mode) self.assertIsNot(got, obj) self.assertIs(type(got), type(obj) if expecttype is None else expecttype) - self.assertEqual(got, obj) + self.assert_equal_or_equalish(got, obj) def assert_roundtrip_not_equal(self, values, *, mode=None, expecttype=None): mode = self._resolve_mode(mode) for obj in values: - with self.subTest(obj): + with self.subTest(repr(obj)): got = self._get_roundtrip(obj, mode) self.assertIsNot(got, obj) self.assertIs(type(got), @@ -164,7 +508,7 @@ def assert_roundtrip_not_equal(self, values, *, def assert_not_shareable(self, values, exctype=None, *, mode=None): mode = self._resolve_mode(mode) for obj in values: - with self.subTest(obj): + with self.subTest(repr(obj)): with self.assertRaises(NotShareableError) as cm: _testinternalcapi.get_crossinterp_data(obj, mode) if exctype is not None: @@ -182,49 +526,26 @@ class PickleTests(_GetXIDataTests): MODE = 'pickle' def test_shareable(self): - self.assert_roundtrip_equal([ - # singletons - None, - True, - False, - # bytes - *(i.to_bytes(2, 'little', signed=True) - for i in range(-1, 258)), - # str - 'hello world', - '你好世界', - '', - # int - sys.maxsize, - -sys.maxsize - 1, - *range(-1, 258), - # float - 0.0, - 1.1, - -1.0, - 0.12345678, - -0.12345678, - # tuple - (), - (1,), - ("hello", "world", ), - (1, True, "hello"), - ((1,),), - ((1, 2), (3, 4)), - ((1, 2), (3, 4), (5, 6)), - ]) - # not shareable using xidata - self.assert_roundtrip_equal([ - # int - sys.maxsize + 1, - -sys.maxsize - 2, - 2**1000, - # tuple - (0, 1.0, []), - (0, 1.0, {}), - (0, 1.0, ([],)), - (0, 1.0, ({},)), - ]) + with ignore_byteswarning(): + for obj in SHAREABLE: + if obj in PICKLEABLE: + self.assert_roundtrip_equal([obj]) + else: + self.assert_not_shareable([obj]) + + def test_not_shareable(self): + with ignore_byteswarning(): + for obj in NOT_SHAREABLE: + if type(obj) is types.MappingProxyType: + self.assert_not_shareable([obj]) + elif obj in PICKLEABLE: + with self.subTest(repr(obj)): + # We don't worry about checking the actual value. + # The other tests should cover that well enough. + got = self.get_roundtrip(obj) + self.assertIs(type(got), type(obj)) + else: + self.assert_not_shareable([obj]) def test_list(self): self.assert_roundtrip_equal_not_identical([ @@ -266,7 +587,7 @@ def assert_class_defs_same(self, defs): if cls not in defs.CLASSES_WITHOUT_EQUALITY: continue instances.append(cls(*args)) - self.assert_roundtrip_not_equal(instances) + self.assert_roundtrip_equal(instances) def assert_class_defs_other_pickle(self, defs, mod): # Pickle relative to a different module than the original. @@ -286,7 +607,7 @@ def assert_class_defs_other_unpickle(self, defs, mod, *, fail=False): instances = [] for cls, args in defs.TOP_CLASSES.items(): - with self.subTest(cls): + with self.subTest(repr(cls)): setattr(mod, cls.__name__, cls) xid = self.get_xidata(cls) inst = cls(*args) @@ -295,7 +616,7 @@ def assert_class_defs_other_unpickle(self, defs, mod, *, fail=False): (cls, xid, inst, instxid)) for cls, xid, inst, instxid in instances: - with self.subTest(cls): + with self.subTest(repr(cls)): delattr(mod, cls.__name__) if fail: with self.assertRaises(NotShareableError): @@ -403,13 +724,13 @@ def assert_func_defs_same(self, defs): def assert_func_defs_other_pickle(self, defs, mod): # Pickle relative to a different module than the original. for func in defs.TOP_FUNCTIONS: - assert not hasattr(mod, func.__name__), (cls, getattr(mod, func.__name__)) + assert not hasattr(mod, func.__name__), (getattr(mod, func.__name__),) self.assert_not_shareable(defs.TOP_FUNCTIONS) def assert_func_defs_other_unpickle(self, defs, mod, *, fail=False): # Unpickle relative to a different module than the original. for func in defs.TOP_FUNCTIONS: - assert not hasattr(mod, func.__name__), (cls, getattr(mod, func.__name__)) + assert not hasattr(mod, func.__name__), (getattr(mod, func.__name__),) captured = [] for func in defs.TOP_FUNCTIONS: @@ -434,7 +755,7 @@ def assert_func_defs_not_shareable(self, defs): self.assert_not_shareable(defs.TOP_FUNCTIONS) def test_user_function_normal(self): -# self.assert_roundtrip_equal(defs.TOP_FUNCTIONS) + self.assert_roundtrip_equal(defs.TOP_FUNCTIONS) self.assert_func_defs_same(defs) def test_user_func_in___main__(self): @@ -505,7 +826,7 @@ def test_nested_function(self): # exceptions def test_user_exception_normal(self): - self.assert_roundtrip_not_equal([ + self.assert_roundtrip_equal([ defs.MimimalError('error!'), ]) self.assert_roundtrip_equal_not_identical([ @@ -521,7 +842,7 @@ def test_builtin_exception(self): special = { BaseExceptionGroup: (msg, [caught]), ExceptionGroup: (msg, [caught]), -# UnicodeError: (None, msg, None, None, None), + UnicodeError: (None, msg, None, None, None), UnicodeEncodeError: ('utf-8', '', 1, 3, msg), UnicodeDecodeError: ('utf-8', b'', 1, 3, msg), UnicodeTranslateError: ('', 1, 3, msg), @@ -531,7 +852,7 @@ def test_builtin_exception(self): args = special.get(cls) or (msg,) exceptions.append(cls(*args)) - self.assert_roundtrip_not_equal(exceptions) + self.assert_roundtrip_equal(exceptions) class MarshalTests(_GetXIDataTests): @@ -576,7 +897,7 @@ def test_simple_builtin_objects(self): '', ]) self.assert_not_shareable([ - object(), + OBJECT, types.SimpleNamespace(), ]) @@ -647,10 +968,7 @@ def test_builtin_type(self): shareable = [ StopIteration, ] - types = [ - *BUILTIN_TYPES, - *OTHER_TYPES, - ] + types = BUILTIN_TYPES self.assert_not_shareable(cls for cls in types if cls not in shareable) self.assert_roundtrip_identical(cls for cls in types @@ -758,10 +1076,203 @@ def test_other_objects(self): ]) +class ShareableFuncTests(_GetXIDataTests): + + MODE = 'func' + + def test_stateless(self): + self.assert_roundtrip_equal([ + *defs.STATELESS_FUNCTIONS, + # Generators can be stateless too. + *defs.FUNCTION_LIKE, + ]) + + def test_not_stateless(self): + self.assert_not_shareable([ + *(f for f in defs.FUNCTIONS + if f not in defs.STATELESS_FUNCTIONS), + ]) + + def test_other_objects(self): + self.assert_not_shareable([ + None, + True, + False, + Ellipsis, + NotImplemented, + 9999, + 'spam', + b'spam', + (), + [], + {}, + object(), + ]) + + +class PureShareableScriptTests(_GetXIDataTests): + + MODE = 'script-pure' + + VALID_SCRIPTS = [ + '', + 'spam', + '# a comment', + 'print("spam")', + 'raise Exception("spam")', + """if True: + do_something() + """, + """if True: + def spam(x): + return x + class Spam: + def eggs(self): + return 42 + x = Spam().eggs() + raise ValueError(spam(x)) + """, + ] + INVALID_SCRIPTS = [ + ' pass', # IndentationError + '----', # SyntaxError + """if True: + def spam(): + # no body + spam() + """, # IndentationError + ] + + def test_valid_str(self): + self.assert_roundtrip_not_equal([ + *self.VALID_SCRIPTS, + ], expecttype=types.CodeType) + + def test_invalid_str(self): + self.assert_not_shareable([ + *self.INVALID_SCRIPTS, + ]) + + def test_valid_bytes(self): + self.assert_roundtrip_not_equal([ + *(s.encode('utf8') for s in self.VALID_SCRIPTS), + ], expecttype=types.CodeType) + + def test_invalid_bytes(self): + self.assert_not_shareable([ + *(s.encode('utf8') for s in self.INVALID_SCRIPTS), + ]) + + def test_pure_script_code(self): + self.assert_roundtrip_equal_not_identical([ + *(f.__code__ for f in defs.PURE_SCRIPT_FUNCTIONS), + ]) + + def test_impure_script_code(self): + self.assert_not_shareable([ + *(f.__code__ for f in defs.SCRIPT_FUNCTIONS + if f not in defs.PURE_SCRIPT_FUNCTIONS), + ]) + + def test_other_code(self): + self.assert_not_shareable([ + *(f.__code__ for f in defs.FUNCTIONS + if f not in defs.SCRIPT_FUNCTIONS), + *(f.__code__ for f in defs.FUNCTION_LIKE), + ]) + + def test_pure_script_function(self): + self.assert_roundtrip_not_equal([ + *defs.PURE_SCRIPT_FUNCTIONS, + ], expecttype=types.CodeType) + + def test_impure_script_function(self): + self.assert_not_shareable([ + *(f for f in defs.SCRIPT_FUNCTIONS + if f not in defs.PURE_SCRIPT_FUNCTIONS), + ]) + + def test_other_function(self): + self.assert_not_shareable([ + *(f for f in defs.FUNCTIONS + if f not in defs.SCRIPT_FUNCTIONS), + *defs.FUNCTION_LIKE, + ]) + + def test_other_objects(self): + self.assert_not_shareable([ + None, + True, + False, + Ellipsis, + NotImplemented, + (), + [], + {}, + object(), + ]) + + +class ShareableScriptTests(PureShareableScriptTests): + + MODE = 'script' + + def test_impure_script_code(self): + self.assert_roundtrip_equal_not_identical([ + *(f.__code__ for f in defs.SCRIPT_FUNCTIONS + if f not in defs.PURE_SCRIPT_FUNCTIONS), + ]) + + def test_impure_script_function(self): + self.assert_roundtrip_not_equal([ + *(f for f in defs.SCRIPT_FUNCTIONS + if f not in defs.PURE_SCRIPT_FUNCTIONS), + ], expecttype=types.CodeType) + + +class ShareableFallbackTests(_GetXIDataTests): + + MODE = 'fallback' + + def test_shareable(self): + self.assert_roundtrip_equal(SHAREABLE) + + def test_not_shareable(self): + okay = [ + *PICKLEABLE, + *defs.STATELESS_FUNCTIONS, + LAMBDA, + ] + ignored = [ + *TUPLES_WITHOUT_EQUALITY, + OBJECT, + METHOD, + BUILTIN_METHOD, + METHOD_WRAPPER, + ] + with ignore_byteswarning(): + self.assert_roundtrip_equal([ + *(o for o in NOT_SHAREABLE + if o in okay and o not in ignored + and o is not MAPPING_PROXY_EMPTY), + ]) + self.assert_roundtrip_not_equal([ + *(o for o in NOT_SHAREABLE + if o in ignored and o is not MAPPING_PROXY_EMPTY), + ]) + self.assert_not_shareable([ + *(o for o in NOT_SHAREABLE if o not in okay), + MAPPING_PROXY_EMPTY, + ]) + + class ShareableTypeTests(_GetXIDataTests): MODE = 'xidata' + def test_shareable(self): + self.assert_roundtrip_equal(SHAREABLE) + def test_singletons(self): self.assert_roundtrip_identical([ None, @@ -829,8 +1340,8 @@ def test_tuple(self): def test_tuples_containing_non_shareable_types(self): non_shareables = [ - Exception(), - object(), + EXCEPTION, + OBJECT, ] for s in non_shareables: value = tuple([0, 1.0, s]) @@ -845,6 +1356,9 @@ def test_tuples_containing_non_shareable_types(self): # The rest are not shareable. + def test_not_shareable(self): + self.assert_not_shareable(NOT_SHAREABLE) + def test_object(self): self.assert_not_shareable([ object(), @@ -861,12 +1375,12 @@ def test_function_object(self): for func in defs.FUNCTIONS: assert type(func) is types.FunctionType, func assert type(defs.SpamOkay.okay) is types.FunctionType, func - assert type(lambda: None) is types.LambdaType + assert type(LAMBDA) is types.LambdaType self.assert_not_shareable([ *defs.FUNCTIONS, defs.SpamOkay.okay, - (lambda: None), + LAMBDA, ]) def test_builtin_function(self): @@ -931,10 +1445,7 @@ def test_class(self): self.assert_not_shareable(instances) def test_builtin_type(self): - self.assert_not_shareable([ - *BUILTIN_TYPES, - *OTHER_TYPES, - ]) + self.assert_not_shareable(BUILTIN_TYPES) def test_exception(self): self.assert_not_shareable([ @@ -973,7 +1484,7 @@ def test_builtin_objects(self): """, ns, ns) self.assert_not_shareable([ - types.MappingProxyType({}), + MAPPING_PROXY_EMPTY, types.SimpleNamespace(), # types.CellType types.CellType(), @@ -984,5 +1495,54 @@ def test_builtin_objects(self): ]) +class CaptureExceptionTests(unittest.TestCase): + + # Prevent crashes with incompatible TracebackException.format(). + # Regression test for https://github.com/python/cpython/issues/143377. + + def capture_with_formatter(self, exc, formatter): + with support.swap_attr(traceback.TracebackException, "format", formatter): + return _interpreters.capture_exception(exc) + + def test_capture_exception(self): + captured = _interpreters.capture_exception(ValueError("hello")) + + self.assertEqual(captured.type.__name__, "ValueError") + self.assertEqual(captured.type.__qualname__, "ValueError") + self.assertEqual(captured.type.__module__, "builtins") + + self.assertEqual(captured.msg, "hello") + self.assertEqual(captured.formatted, "ValueError: hello") + + def test_capture_exception_custom_format(self): + exc = ValueError("good bye!") + formatter = lambda self: ["hello\n", "world\n"] + captured = self.capture_with_formatter(exc, formatter) + self.assertEqual(captured.msg, "good bye!") + self.assertEqual(captured.formatted, "ValueError: good bye!") + self.assertEqual(captured.errdisplay, "hello\nworld") + + @support.subTests("exc_lines", ([], ["x-no-nl"], ["x-no-nl", "y-no-nl"])) + def test_capture_exception_invalid_format(self, exc_lines): + formatter = lambda self: exc_lines + captured = self.capture_with_formatter(ValueError(), formatter) + self.assertEqual(captured.msg, "") + self.assertEqual(captured.formatted, "ValueError: ") + self.assertEqual(captured.errdisplay, "".join(exc_lines)) + + @unittest.skipUnless( + support.Py_DEBUG, + "printing subinterpreter unraisable exceptions requires DEBUG build", + ) + def test_capture_exception_unraisable_exception(self): + formatter = lambda self: 1 + with support.catch_unraisable_exception() as cm: + captured = self.capture_with_formatter(ValueError(), formatter) + self.assertNotHasAttr(captured, "errdisplay") + self.assertEqual(cm.unraisable.exc_type, TypeError) + self.assertEqual(str(cm.unraisable.exc_value), + "can only join an iterable") + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 9aace57633b0c64..0f5235d5b2beecd 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -553,6 +553,33 @@ def test_roundtrip_escaped_unquoted_newlines(self): self.assertEqual(row, rows[i]) + def test_reader_reentrant_iterator(self): + # gh-145105: re-entering the reader from the iterator must not crash. + class ReentrantIter: + def __init__(self): + self.reader = None + self.n = 0 + def __iter__(self): + return self + def __next__(self): + self.n += 1 + if self.n == 1: + try: + next(self.reader) + except StopIteration: + pass + return "a,b" + if self.n == 2: + return "x" + raise StopIteration + + it = ReentrantIter() + reader = csv.reader(it) + it.reader = reader + with self.assertRaises(csv.Error): + next(reader) + + class TestDialectRegistry(unittest.TestCase): def test_registry_badargs(self): self.assertRaises(TypeError, csv.list_dialects, None) @@ -918,6 +945,14 @@ def test_dict_reader_fieldnames_accepts_list(self): reader = csv.DictReader(f, fieldnames) self.assertEqual(reader.fieldnames, fieldnames) + def test_dict_reader_set_fieldnames(self): + fieldnames = ["a", "b", "c"] + f = StringIO() + reader = csv.DictReader(f) + self.assertIsNone(reader.fieldnames) + reader.fieldnames = fieldnames + self.assertEqual(reader.fieldnames, fieldnames) + def test_dict_writer_fieldnames_rejects_iter(self): fieldnames = ["a", "b", "c"] f = StringIO() @@ -933,6 +968,7 @@ def test_dict_writer_fieldnames_accepts_list(self): def test_dict_reader_fieldnames_is_optional(self): f = StringIO() reader = csv.DictReader(f, fieldnames=None) + self.assertIsNone(reader.fieldnames) def test_read_dict_fields(self): with TemporaryFile("w+", encoding="utf-8") as fileobj: @@ -1122,19 +1158,22 @@ class mydialect(csv.Dialect): with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"quotechar" must be a 1-character string') + '"quotechar" must be a unicode character or None, ' + 'not a string of length 0') mydialect.quotechar = "''" with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"quotechar" must be a 1-character string') + '"quotechar" must be a unicode character or None, ' + 'not a string of length 2') mydialect.quotechar = 4 with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"quotechar" must be string or None, not int') + '"quotechar" must be a unicode character or None, ' + 'not int') def test_delimiter(self): class mydialect(csv.Dialect): @@ -1151,31 +1190,32 @@ class mydialect(csv.Dialect): with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"delimiter" must be a 1-character string') + '"delimiter" must be a unicode character, ' + 'not a string of length 3') mydialect.delimiter = "" with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"delimiter" must be a 1-character string') + '"delimiter" must be a unicode character, not a string of length 0') mydialect.delimiter = b"," with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"delimiter" must be string, not bytes') + '"delimiter" must be a unicode character, not bytes') mydialect.delimiter = 4 with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"delimiter" must be string, not int') + '"delimiter" must be a unicode character, not int') mydialect.delimiter = None with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"delimiter" must be string, not NoneType') + '"delimiter" must be a unicode character, not NoneType') def test_escapechar(self): class mydialect(csv.Dialect): @@ -1189,20 +1229,32 @@ class mydialect(csv.Dialect): self.assertEqual(d.escapechar, "\\") mydialect.escapechar = "" - with self.assertRaisesRegex(csv.Error, '"escapechar" must be a 1-character string'): + with self.assertRaises(csv.Error) as cm: mydialect() + self.assertEqual(str(cm.exception), + '"escapechar" must be a unicode character or None, ' + 'not a string of length 0') mydialect.escapechar = "**" - with self.assertRaisesRegex(csv.Error, '"escapechar" must be a 1-character string'): + with self.assertRaises(csv.Error) as cm: mydialect() + self.assertEqual(str(cm.exception), + '"escapechar" must be a unicode character or None, ' + 'not a string of length 2') mydialect.escapechar = b"*" - with self.assertRaisesRegex(csv.Error, '"escapechar" must be string or None, not bytes'): + with self.assertRaises(csv.Error) as cm: mydialect() + self.assertEqual(str(cm.exception), + '"escapechar" must be a unicode character or None, ' + 'not bytes') mydialect.escapechar = 4 - with self.assertRaisesRegex(csv.Error, '"escapechar" must be string or None, not int'): + with self.assertRaises(csv.Error) as cm: mydialect() + self.assertEqual(str(cm.exception), + '"escapechar" must be a unicode character or None, ' + 'not int') def test_lineterminator(self): class mydialect(csv.Dialect): @@ -1223,7 +1275,13 @@ class mydialect(csv.Dialect): with self.assertRaises(csv.Error) as cm: mydialect() self.assertEqual(str(cm.exception), - '"lineterminator" must be a string') + '"lineterminator" must be a string, not int') + + mydialect.lineterminator = None + with self.assertRaises(csv.Error) as cm: + mydialect() + self.assertEqual(str(cm.exception), + '"lineterminator" must be a string, not NoneType') def test_invalid_chars(self): def create_invalid(field_name, value, **kwargs): @@ -1331,6 +1389,22 @@ class TestSniffer(unittest.TestCase): ghi\0jkl """ + sample15 = "\n\n\n" + sample16 = "abc\ndef\nghi" + + sample17 = ["letter,offset"] + sample17.extend(f"{chr(ord('a') + i)},{i}" for i in range(20)) + sample17.append("v,twenty_one") # 'u' was skipped + sample17 = '\n'.join(sample17) + + sample18 = ["letter,offset"] + sample18.extend(f"{chr(ord('a') + i)},{i}" for i in range(21)) + sample18.append("v,twenty_one") # 'u' was not skipped + sample18 = '\n'.join(sample18) + + sample19 = ('time,title\r\n' + '2020-10-01,"Pocket - Save news, videos, stories and more"\r\n') + def test_issue43625(self): sniffer = csv.Sniffer() self.assertTrue(sniffer.has_header(self.sample12)) @@ -1352,6 +1426,11 @@ def test_has_header_regex_special_delimiter(self): self.assertIs(sniffer.has_header(self.sample8), False) self.assertIs(sniffer.has_header(self.header2 + self.sample8), True) + def test_has_header_checks_20_rows(self): + sniffer = csv.Sniffer() + self.assertFalse(sniffer.has_header(self.sample17)) + self.assertTrue(sniffer.has_header(self.sample18)) + def test_guess_quote_and_delimiter(self): sniffer = csv.Sniffer() for header in (";'123;4';", "'123;4';", ";'123;4'", "'123;4'"): @@ -1401,6 +1480,13 @@ def test_delimiters(self): self.assertEqual(dialect.quotechar, "'") dialect = sniffer.sniff(self.sample14) self.assertEqual(dialect.delimiter, '\0') + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, self.sample15) + self.assertRaisesRegex(csv.Error, "Could not determine delimiter", + sniffer.sniff, self.sample16) + dialect = sniffer.sniff(self.sample19) + self.assertEqual(dialect.delimiter, ',') + self.assertEqual(dialect.quotechar, '"') def test_doublequote(self): sniffer = csv.Sniffer() diff --git a/Lib/test/test_ctypes/test_as_parameter.py b/Lib/test/test_ctypes/test_as_parameter.py index 2da1acfcf2989ee..abbfb6efdcea55b 100644 --- a/Lib/test/test_ctypes/test_as_parameter.py +++ b/Lib/test/test_ctypes/test_as_parameter.py @@ -5,7 +5,8 @@ c_short, c_int, c_long, c_longlong, c_byte, c_wchar, c_float, c_double, ArgumentError) -from test.support import import_helper, skip_if_sanitizer +from test.support import (import_helper, skip_if_sanitizer, + skip_emscripten_stack_overflow, skip_if_huge_c_stack) _ctypes_test = import_helper.import_module("_ctypes_test") @@ -193,6 +194,8 @@ class S8I(Structure): (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) @skip_if_sanitizer('requires deep stack', thread=True) + @skip_if_huge_c_stack() + @skip_emscripten_stack_overflow() def test_recursive_as_param(self): class A: pass diff --git a/Lib/test/test_ctypes/test_byteswap.py b/Lib/test/test_ctypes/test_byteswap.py index ea5951603f93245..6a1bae14773d277 100644 --- a/Lib/test/test_ctypes/test_byteswap.py +++ b/Lib/test/test_ctypes/test_byteswap.py @@ -166,6 +166,48 @@ def test_endian_double(self): self.assertEqual(s.value, math.pi) self.assertEqual(bin(struct.pack(">d", math.pi)), bin(s)) + @unittest.skipUnless(hasattr(ctypes, 'c_float_complex'), "No complex types") + def test_endian_float_complex(self): + c_float_complex = ctypes.c_float_complex + if sys.byteorder == "little": + self.assertIs(c_float_complex.__ctype_le__, c_float_complex) + self.assertIs(c_float_complex.__ctype_be__.__ctype_le__, + c_float_complex) + else: + self.assertIs(c_float_complex.__ctype_be__, c_float_complex) + self.assertIs(c_float_complex.__ctype_le__.__ctype_be__, + c_float_complex) + s = c_float_complex(math.pi+1j) + self.assertEqual(bin(struct.pack("F", math.pi+1j)), bin(s)) + self.assertAlmostEqual(s.value, math.pi+1j, places=6) + s = c_float_complex.__ctype_le__(math.pi+1j) + self.assertAlmostEqual(s.value, math.pi+1j, places=6) + self.assertEqual(bin(struct.pack("F", math.pi+1j)), bin(s)) + + @unittest.skipUnless(hasattr(ctypes, 'c_double_complex'), "No complex types") + def test_endian_double_complex(self): + c_double_complex = ctypes.c_double_complex + if sys.byteorder == "little": + self.assertIs(c_double_complex.__ctype_le__, c_double_complex) + self.assertIs(c_double_complex.__ctype_be__.__ctype_le__, + c_double_complex) + else: + self.assertIs(c_double_complex.__ctype_be__, c_double_complex) + self.assertIs(c_double_complex.__ctype_le__.__ctype_be__, + c_double_complex) + s = c_double_complex(math.pi+1j) + self.assertEqual(bin(struct.pack("D", math.pi+1j)), bin(s)) + self.assertAlmostEqual(s.value, math.pi+1j, places=6) + s = c_double_complex.__ctype_le__(math.pi+1j) + self.assertAlmostEqual(s.value, math.pi+1j, places=6) + self.assertEqual(bin(struct.pack("D", math.pi+1j)), bin(s)) + def test_endian_other(self): self.assertIs(c_byte.__ctype_le__, c_byte) self.assertIs(c_byte.__ctype_be__, c_byte) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 8af34e62b94faa3..5658234f9ec66bf 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -32,6 +32,7 @@ @unittest.skipUnless(sys.platform.startswith('linux'), 'test requires GNU IFUNC support') +@unittest.skipIf(test.support.linked_to_musl(), "Requires glibc") class TestNullDlsym(unittest.TestCase): """GH-126554: Ensure that we catch NULL dlsym return values diff --git a/Lib/test/test_ctypes/test_find.py b/Lib/test/test_ctypes/test_find.py index 3bd41a0e435d915..8bc84c3d2ef9f85 100644 --- a/Lib/test/test_ctypes/test_find.py +++ b/Lib/test/test_ctypes/test_find.py @@ -153,5 +153,73 @@ def test_find(self): self.assertIsNone(find_library(name)) +@unittest.skipUnless(test.support.is_emscripten, + 'Test only valid for Emscripten') +class FindLibraryEmscripten(unittest.TestCase): + @classmethod + def setUpClass(cls): + import tempfile + + # A very simple wasm module + # In WAT format: (module) + cls.wasm_module = b'\x00asm\x01\x00\x00\x00\x00\x08\x04name\x02\x01\x00' + + cls.non_wasm_content = b'This is not a WASM file' + + cls.temp_dir = tempfile.mkdtemp() + cls.libdummy_so_path = os.path.join(cls.temp_dir, 'libdummy.so') + with open(cls.libdummy_so_path, 'wb') as f: + f.write(cls.wasm_module) + + cls.libother_wasm_path = os.path.join(cls.temp_dir, 'libother.wasm') + with open(cls.libother_wasm_path, 'wb') as f: + f.write(cls.wasm_module) + + cls.libnowasm_so_path = os.path.join(cls.temp_dir, 'libnowasm.so') + with open(cls.libnowasm_so_path, 'wb') as f: + f.write(cls.non_wasm_content) + + @classmethod + def tearDownClass(cls): + import shutil + shutil.rmtree(cls.temp_dir) + + def test_find_wasm_file_with_so_extension(self): + with os_helper.EnvironmentVarGuard() as env: + env.set('LD_LIBRARY_PATH', self.temp_dir) + result = find_library('dummy') + self.assertEqual(result, self.libdummy_so_path) + def test_find_wasm_file_with_wasm_extension(self): + with os_helper.EnvironmentVarGuard() as env: + env.set('LD_LIBRARY_PATH', self.temp_dir) + result = find_library('other') + self.assertEqual(result, self.libother_wasm_path) + + def test_ignore_non_wasm_file(self): + with os_helper.EnvironmentVarGuard() as env: + env.set('LD_LIBRARY_PATH', self.temp_dir) + result = find_library('nowasm') + self.assertIsNone(result) + + def test_find_nothing_without_ld_library_path(self): + with os_helper.EnvironmentVarGuard() as env: + if 'LD_LIBRARY_PATH' in env: + del env['LD_LIBRARY_PATH'] + result = find_library('dummy') + self.assertIsNone(result) + result = find_library('other') + self.assertIsNone(result) + + def test_find_nothing_with_wrong_ld_library_path(self): + import tempfile + with tempfile.TemporaryDirectory() as empty_dir: + with os_helper.EnvironmentVarGuard() as env: + env.set('LD_LIBRARY_PATH', empty_dir) + result = find_library('dummy') + self.assertIsNone(result) + result = find_library('other') + self.assertIsNone(result) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_loading.py b/Lib/test/test_ctypes/test_loading.py index 13ed813ad98c311..343f6a07c0a32c4 100644 --- a/Lib/test/test_ctypes/test_loading.py +++ b/Lib/test/test_ctypes/test_loading.py @@ -100,6 +100,20 @@ def test_load_ordinal_functions(self): self.assertRaises(AttributeError, dll.__getitem__, 1234) + @unittest.skipUnless(os.name == "nt", 'Windows-specific test') + def test_load_without_name_and_with_handle(self): + handle = ctypes.windll.kernel32._handle + lib = ctypes.WinDLL(name=None, handle=handle) + self.assertIs(handle, lib._handle) + + @unittest.skipIf(os.name == "nt", 'POSIX-specific test') + @unittest.skipIf(libc_name is None, 'could not find libc') + def test_load_without_name_and_with_handle_posix(self): + lib1 = CDLL(libc_name) + handle = lib1._handle + lib2 = CDLL(name=None, handle=handle) + self.assertIs(lib2._handle, handle) + @unittest.skipUnless(os.name == "nt", 'Windows-specific test') def test_1703286_A(self): # On winXP 64-bit, advapi32 loads at an address that does diff --git a/Lib/test/test_ctypes/test_parameters.py b/Lib/test/test_ctypes/test_parameters.py index f89521cf8b3a67c..46f8ff93efa9152 100644 --- a/Lib/test/test_ctypes/test_parameters.py +++ b/Lib/test/test_ctypes/test_parameters.py @@ -1,3 +1,4 @@ +import sys import unittest import test.support from ctypes import (CDLL, PyDLL, ArgumentError, @@ -240,7 +241,8 @@ def test_parameter_repr(self): self.assertRegex(repr(c_ulonglong.from_param(20000)), r"^$") self.assertEqual(repr(c_float.from_param(1.5)), "") self.assertEqual(repr(c_double.from_param(1.5)), "") - self.assertEqual(repr(c_double.from_param(1e300)), "") + if sys.float_repr_style == 'short': + self.assertEqual(repr(c_double.from_param(1e300)), "") self.assertRegex(repr(c_longdouble.from_param(1.5)), r"^$") self.assertRegex(repr(c_char_p.from_param(b'hihi')), r"^$") self.assertRegex(repr(c_wchar_p.from_param('hihi')), r"^$") diff --git a/Lib/test/test_ctypes/test_pointers.py b/Lib/test/test_ctypes/test_pointers.py index a8d243a45de0f40..771cc8fbe0ec934 100644 --- a/Lib/test/test_ctypes/test_pointers.py +++ b/Lib/test/test_ctypes/test_pointers.py @@ -403,6 +403,16 @@ class Cls(Structure): self.assertEqual(len(ws_typ), 0, ws_typ) self.assertEqual(len(ws_ptr), 0, ws_ptr) + def test_pointer_proto_missing_argtypes_error(self): + class BadType(ctypes._Pointer): + # _type_ is intentionally missing + pass + + func = ctypes.pythonapi.Py_GetVersion + func.argtypes = (BadType,) + + with self.assertRaises(ctypes.ArgumentError): + func(object()) class PointerTypeCacheTestCase(unittest.TestCase): # dummy tests to check warnings and base behavior diff --git a/Lib/test/test_ctypes/test_prototypes.py b/Lib/test/test_ctypes/test_prototypes.py index 63ae799ea86ab2d..d976e8da0e2d30a 100644 --- a/Lib/test/test_ctypes/test_prototypes.py +++ b/Lib/test/test_ctypes/test_prototypes.py @@ -72,6 +72,32 @@ def test_paramflags(self): self.assertEqual(func(None), None) self.assertEqual(func(input=None), None) + def test_invalid_paramflags(self): + proto = CFUNCTYPE(c_int, c_char_p) + with self.assertRaises(ValueError): + func = proto(("myprintf", testdll), ((1, "fmt"), (1, "arg1"))) + + def test_invalid_setattr_argtypes(self): + proto = CFUNCTYPE(c_int, c_char_p) + func = proto(("myprintf", testdll), ((1, "fmt"),)) + + with self.assertRaisesRegex(TypeError, "_argtypes_ must be a sequence of types"): + func.argtypes = 123 + self.assertEqual(func.argtypes, (c_char_p,)) + + with self.assertRaisesRegex(ValueError, "paramflags must have the same length as argtypes"): + func.argtypes = (c_char_p, c_int) + self.assertEqual(func.argtypes, (c_char_p,)) + + def test_paramflags_outarg(self): + proto = CFUNCTYPE(c_int, c_char_p, c_int) + with self.assertRaisesRegex(TypeError, "must be a pointer type"): + func = proto(("myprintf", testdll), ((1, "fmt"), (2, "out"))) + + proto = CFUNCTYPE(c_int, c_char_p, c_void_p) + func = proto(("myprintf", testdll), ((1, "fmt"), (2, "out"))) + with self.assertRaisesRegex(TypeError, "must be a pointer type"): + func.argtypes = (c_char_p, c_int) def test_int_pointer_arg(self): func = testdll._testfunc_p_p diff --git a/Lib/test/test_ctypes/test_struct_fields.py b/Lib/test/test_ctypes/test_struct_fields.py index b50bbcbb65c4233..dc26e26d8a9fb11 100644 --- a/Lib/test/test_ctypes/test_struct_fields.py +++ b/Lib/test/test_ctypes/test_struct_fields.py @@ -130,6 +130,21 @@ class S(Structure): self.check_struct(S) self.assertEqual(S.largeField.bit_size, size * 8) + def test_bitfield_overflow_error_message(self): + with self.assertRaisesRegex( + ValueError, + r"bit field 'x' overflows its type \(2 \+ 7 > 8\)", + ): + CField( + name="x", + type=c_byte, + byte_size=1, + byte_offset=0, + index=0, + _internal_use=True, + bit_size=7, + bit_offset=2, + ) # __set__ and __get__ should raise a TypeError in case their self # argument is not a ctype instance. diff --git a/Lib/test/test_ctypes/test_structures.py b/Lib/test/test_ctypes/test_structures.py index 92d4851d739d47b..65ccc12625f72be 100644 --- a/Lib/test/test_ctypes/test_structures.py +++ b/Lib/test/test_ctypes/test_structures.py @@ -299,9 +299,16 @@ class X(Structure): self.assertEqual(s.first, got.first) self.assertEqual(s.second, got.second) + @unittest.skipIf(support.is_wasm32, "wasm ABI is incompatible with test expectations") def _test_issue18060(self, Vector): # Regression tests for gh-62260 + # This test passes a struct of two doubles by value to atan2(), whose C + # signature is atan2(double, double), so it only works on platforms + # where the abi of a function that takes a struct with two doubles + # matches the abi of a function that takes two doubles. The wasm32 ABI + # does not satisfy this condition and the test breaks. + # The call to atan2() should succeed if the # class fields were correctly cloned in the # subclasses. Otherwise, it will segfault. diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index c307258e5657e2e..a75789cdda7b465 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -249,6 +249,33 @@ def test_refresh_control(self): self.assertIs(win.is_wintouched(), syncok) self.assertIs(stdscr.is_wintouched(), syncok) + # Many tests below use a common set of non-ASCII cases, each applied only + # when the window encoding can represent it -- so the whole suite is meant to + # be run under several locales (e.g. ISO-8859-1, ISO-8859-15, KOI8-U): + # 'A'/'a' ASCII + # 'é' common to the Latin encodings + # '¤'/'€'/'є' byte 0xA4 in ISO-8859-1 / ISO-8859-15 / KOI8-U + # Precomposed characters are used so a round-trip does not depend on the form. + + def _encodable(self, s): + # Wide characters are only supported in a locale that can encode them. + try: + s.encode(self.stdscr.encoding) + except UnicodeEncodeError: + return False + return True + + def _read_char(self, y, x): + # The character written to a cell, read back for output checks. inch() + # is unusable here: on a wide build it returns the low 8 bits of the + # character's code point rather than its locale-encoded byte, mangling + # anything outside Latin-1. in_wch() reads the wide cell directly; + # without it, instr() re-encodes the cell to the window encoding. + stdscr = self.stdscr + if hasattr(stdscr, 'in_wch'): + return str(stdscr.in_wch(y, x)) + return stdscr.instr(y, x, 1).decode(stdscr.encoding) + def test_output_character(self): stdscr = self.stdscr encoding = stdscr.encoding @@ -258,32 +285,95 @@ def test_output_character(self): stdscr.addch('A') stdscr.addch(b'A') stdscr.addch(65) - c = '\u20ac' - try: - stdscr.addch(c) - except UnicodeEncodeError: - self.assertRaises(UnicodeEncodeError, c.encode, encoding) - except OverflowError: - encoded = c.encode(encoding) - self.assertNotEqual(len(encoded), 1, repr(encoded)) + # See _encodable for the character set. Each is either written (mapped + # to a single byte), or raises UnicodeEncodeError (not in the encoding) + # or OverflowError (a multibyte sequence, e.g. in UTF-8). + for c in ('A', '\u00e9', '\u00a4', '\u20ac', '\u0454'): + try: + stdscr.addch(c) + except UnicodeEncodeError: + self.assertRaises(UnicodeEncodeError, c.encode, encoding) + except OverflowError: + encoded = c.encode(encoding) + self.assertNotEqual(len(encoded), 1, repr(encoded)) stdscr.addch('A', curses.A_BOLD) stdscr.addch(1, 2, 'A') stdscr.addch(2, 3, 'A', curses.A_BOLD) self.assertIs(stdscr.is_wintouched(), True) + # The same characters supplied as an int chtype (a byte > 127). The + # cell is read back with _read_char(), not inch(): on a wide build the + # int is stored through the locale as a wide character that inch() + # cannot represent for a character outside Latin-1. + for c in ('é', '¤', '€', 'є'): + try: + b = c.encode(encoding) + except UnicodeEncodeError: + continue + if len(b) != 1: + continue + # A wide build stores a character outside Latin-1 as a wide cell, + # not as its encoded byte, so it cannot round-trip here. + if ord(c) > 0xff and hasattr(stdscr, 'get_wch'): + continue + v = b[0] + with self.subTest(c=c): + stdscr.addch(0, 0, v) + self.assertEqual(self._read_char(0, 0), c) + stdscr.addch(0, 1, v, curses.A_BOLD) + self.assertEqual(self._read_char(0, 1), c) + self.assertTrue(stdscr.inch(0, 1) & curses.A_BOLD) + stdscr.move(2, 0) + stdscr.echochar(v) + self.assertEqual(self._read_char(2, 0), c) + # insch() decodes the byte through the locale like addch(), so + # it round-trips the same character. + stdscr.insch(1, 0, v) + self.assertEqual(self._read_char(1, 0), c) + + # The same characters supplied as a str. Unlike the int path above, a + # str is stored as a wide-character cell on a wide build, so every + # encodable character round-trips, insch() included. A multibyte + # character does not fit a cell on a narrow build and is skipped. + wide = hasattr(stdscr, 'in_wch') + for c in ('é', '¤', '€', 'є'): + if not self._encodable(c): + continue + if not wide and len(c.encode(encoding)) != 1: + continue + # A wide build stores a character outside Latin-1 as a wide cell, + # not as its encoded byte, so it cannot round-trip here. + if ord(c) > 0xff and hasattr(stdscr, 'get_wch'): + continue + with self.subTest(c=c): + stdscr.addch(0, 0, c) + self.assertEqual(self._read_char(0, 0), c) + stdscr.addch(0, 1, c, curses.A_BOLD) + self.assertEqual(self._read_char(0, 1), c) + self.assertTrue(stdscr.inch(0, 1) & curses.A_BOLD) + stdscr.insch(1, 0, c) + self.assertEqual(self._read_char(1, 0), c) + stdscr.move(2, 0) + stdscr.echochar(c) + self.assertEqual(self._read_char(2, 0), c) + # echochar() stdscr.refresh() stdscr.move(0, 0) stdscr.echochar('A') stdscr.echochar(b'A') stdscr.echochar(65) - with self.assertRaises((UnicodeEncodeError, OverflowError)): - # Unicode is not fully supported yet, but at least it does - # not crash. - # It is supposed to fail because either the character is - # not encodable with the current encoding, or it is encoded to - # a multibyte sequence. - stdscr.echochar('\u0114') + # See _encodable for the character set; as in the addch() loop above. + for c in ('A', '\u00e9', '\u00a4', '\u20ac', '\u0454'): + try: + stdscr.echochar(c) + except UnicodeEncodeError: + # The character is not encodable with the current encoding. + self.assertRaises(UnicodeEncodeError, c.encode, encoding) + except OverflowError: + # The character is encoded to a multibyte sequence. + encoded = c.encode(encoding) + self.assertNotEqual(len(encoded), 1, repr(encoded)) stdscr.echochar('A', curses.A_BOLD) self.assertIs(stdscr.is_wintouched(), False) @@ -293,14 +383,18 @@ def test_output_string(self): # addstr()/insstr() for func in [stdscr.addstr, stdscr.insstr]: with self.subTest(func.__qualname__): - stdscr.move(0, 0) func('abcd') func(b'abcd') - s = 'àßçđ' - try: - func(s) - except UnicodeEncodeError: - self.assertRaises(UnicodeEncodeError, s.encode, encoding) + # Common and encoding-distinctive strings (see _encodable for the + # 0xA4 set); 'àßçđ' is UTF-8-only. Each is written if the + # encoding allows, else raises UnicodeEncodeError. + for s in ('soupçon', 'àßçđ', 'soupçon ¤', 'soupçon €', 'дякую'): + stdscr.move(0, 0) + try: + func(s) + except UnicodeEncodeError: + self.assertRaises(UnicodeEncodeError, s.encode, encoding) + stdscr.move(0, 0) func('abcd', curses.A_BOLD) func(1, 2, 'abcd') func(2, 3, 'abcd', curses.A_BOLD) @@ -311,11 +405,14 @@ def test_output_string(self): stdscr.move(0, 0) func('1234', 3) func(b'1234', 3) - s = '\u0661\u0662\u0663\u0664' - try: - func(s, 3) - except UnicodeEncodeError: - self.assertRaises(UnicodeEncodeError, s.encode, encoding) + # As above (see _encodable); Arabic-Indic digits are UTF-8-only. + for s in ('caf\u00e9', '\u0661\u0662\u0663\u0664', 'caf\u00e9 \u00a4', 'caf\u00e9 \u20ac', '\u0434\u044f\u043a\u0443\u044e'): + stdscr.move(0, 0) + try: + func(s, 3) + except UnicodeEncodeError: + self.assertRaises(UnicodeEncodeError, s.encode, encoding) + stdscr.move(0, 0) func('1234', 5) func('1234', 3, curses.A_BOLD) func(1, 2, '1234', 3) @@ -331,6 +428,65 @@ def test_output_string_embedded_null_chars(self): self.assertRaises(ValueError, stdscr.insstr, arg) self.assertRaises(ValueError, stdscr.insnstr, arg, 1) + def test_add_string_behavior(self): + # addstr() advances the cursor past the written text; addnstr() + # writes at most n characters. + win = curses.newwin(1, 10, 0, 0) + win.addstr(0, 0, 'abc') + self.assertEqual(win.getyx(), (0, 3)) + win.erase() + win.addnstr(0, 0, 'abcdef', 3) + self.assertEqual(win.instr(0, 0), b'abc ') + + def test_insert_string_behavior(self): + # insstr()/insnstr() insert at the cursor, shift the rest of the + # line right (losing characters off the edge), and leave the cursor + # where it was. + win = curses.newwin(1, 10, 0, 0) + win.addstr(0, 0, 'abcde') + win.move(0, 1) + win.insstr('XY') + self.assertEqual(win.getyx(), (0, 1)) # cursor did not advance + self.assertEqual(win.instr(0, 0), b'aXYbcde ') + + win.erase() + win.addstr(0, 0, 'ZZZZZ') + win.move(0, 0) + win.insnstr('abcdef', 3) # at most 3 characters + self.assertEqual(win.instr(0, 0), b'abcZZZZZ ') + + def test_insch(self): + # insch() inserts a single character at the cursor (or at y, x), + # shifting the rest of the line right. + win = curses.newwin(2, 10, 0, 0) + win.addstr(0, 0, 'abc') + win.move(0, 1) + win.insch(ord('X')) + self.assertEqual(win.instr(0, 0), b'aXbc ') + win.insch(1, 0, 'Y', curses.A_BOLD) + self.assertEqual(win.inch(1, 0), b'Y'[0] | curses.A_BOLD) + + def test_pad(self): + pad = curses.newpad(10, 20) + pad.addstr(0, 0, 'PADTEXT') + self.assertEqual(pad.instr(0, 0, 7), b'PADTEXT') + + # subpad() creates a pad within the parent pad. Cell sharing with + # the parent is implementation-defined, so write to the subpad itself. + sub = pad.subpad(3, 5, 0, 0) + self.assertEqual(sub.getmaxyx(), (3, 5)) + sub.addstr(1, 0, 'sub') + self.assertEqual(sub.instr(1, 0, 3), b'sub') + + # A pad is refreshed onto an explicit screen rectangle; the + # 6-argument form is required (and rejected for ordinary windows). + pad.refresh(0, 0, 0, 0, 4, 10) + pad.noutrefresh(0, 0, 0, 0, 4, 10) + curses.doupdate() + self.assertRaises(curses.error, pad.refresh) + win = curses.newwin(5, 5, 0, 0) + self.assertRaises(TypeError, win.refresh, 0, 0, 0, 0, 4, 4) + def test_read_from_window(self): stdscr = self.stdscr stdscr.addstr(0, 1, 'ABCD', curses.A_BOLD) @@ -346,6 +502,42 @@ def test_read_from_window(self): self.assertEqual(stdscr.instr(0, 2, 4), b'BCD ') self.assertRaises(ValueError, stdscr.instr, -2) self.assertRaises(ValueError, stdscr.instr, 0, 2, -2) + # A non-ASCII character of an 8-bit locale reads back as its encoded + # byte (see _encodable for the set). Both instr() and inch() return the + # locale byte for any character that fits the locale's single-byte + # encoding. + encoding = stdscr.encoding + for ch in ('A', 'é', '¤', '€', 'є'): + try: + b = ch.encode(encoding) + except UnicodeEncodeError: + continue + if len(b) != 1: + continue + with self.subTest(ch=ch): + stdscr.addstr(2, 0, ch) + self.assertEqual(stdscr.instr(2, 0, 1), b) + self.assertEqual(stdscr.inch(2, 0) & curses.A_CHARTEXT, b[0]) + + def test_coordinate_errors(self): + # Addressing a cell outside the window raises curses.error. + win = curses.newwin(5, 10, 0, 0) + self.assertRaises(curses.error, win.move, 100, 100) + self.assertRaises(curses.error, win.move, -1, -1) + self.assertRaises(curses.error, win.addch, 100, 100, ord('x')) + if hasattr(win, 'chgat'): # chgat() requires wchgat() + self.assertRaises(curses.error, win.chgat, 100, 0, curses.A_BOLD) + + def test_argument_errors(self): + win = curses.newwin(5, 10, 0, 0) + # A character argument must be an int, a byte or a one-element string. + self.assertRaises(TypeError, win.addch, []) + self.assertRaises(OverflowError, win.addch, 2**64) + # A string method rejects a non-string, non-bytes argument. + self.assertRaises(TypeError, win.addstr, 5) + self.assertRaises(TypeError, win.addstr) + # Wrong number of positional arguments. + self.assertRaises(TypeError, win.instr, 0, 0, 0, 0) def test_getch(self): win = curses.newwin(5, 12, 5, 2) @@ -362,6 +554,10 @@ def test_getch(self): self.assertEqual(win.getch(), b'm'[0]) self.assertEqual(win.getch(), b'\n'[0]) + # A key value > 127 is delivered unchanged (it is not locale text). + curses.ungetch(0xE9) + self.assertEqual(win.getch(), 0xE9) + def test_getstr(self): win = curses.newwin(5, 12, 5, 2) curses.echo() @@ -534,6 +730,33 @@ def test_background(self): self.assertEqual(win.inch(0, 0), b'L'[0] | curses.A_REVERSE) self.assertEqual(win.inch(0, 5), b'#'[0] | curses.A_REVERSE) + # A non-ASCII background character of an 8-bit locale reads back as its + # encoded byte. See _encodable for the character set. + win.bkgd(' ') + encoding = win.encoding + for ch in ('é', '¤', '€', 'є'): + try: + b = ch.encode(encoding) + except UnicodeEncodeError: + continue + if len(b) != 1: + continue + # A wide build stores a character outside Latin-1 as a wide cell, + # not as its encoded byte, so it cannot round-trip here. + if ord(ch) > 0xff and hasattr(win, 'get_wch'): + continue + with self.subTest(ch=ch): + win.bkgd(ch) + self.assertEqual(win.getbkgd(), b[0]) + if ord(ch) < 0x100: + # The same byte given as an int. A wide build stores it + # through the locale, so only a Latin-1 byte round-trips. + win.bkgd(' ') + win.bkgdset(b[0]) + self.assertEqual(win.getbkgd(), b[0]) + win.bkgd(b[0]) + self.assertEqual(win.getbkgd(), b[0]) + def test_overlay(self): srcwin = curses.newwin(5, 18, 3, 4) lorem_ipsum(srcwin) @@ -626,6 +849,16 @@ def test_borders_and_lines(self): win.border(65, 66) win.border(65) win.border() + # With no arguments, border() fills the edges with ACS line and corner + # characters. + chartext = curses.A_CHARTEXT + maxy, maxx = win.getmaxyx() + self.assertEqual(win.inch(0, 0) & chartext, curses.ACS_ULCORNER & chartext) + self.assertEqual(win.inch(0, maxx-1) & chartext, curses.ACS_URCORNER & chartext) + self.assertEqual(win.inch(maxy-1, 0) & chartext, curses.ACS_LLCORNER & chartext) + self.assertEqual(win.inch(maxy-1, maxx-1) & chartext, curses.ACS_LRCORNER & chartext) + self.assertEqual(win.inch(0, 1) & chartext, curses.ACS_HLINE & chartext) + self.assertEqual(win.inch(1, 0) & chartext, curses.ACS_VLINE & chartext) win.box(':', '~') self.assertEqual(win.instr(0, 1, 8), b'~~~~~~~~') @@ -636,6 +869,11 @@ def test_borders_and_lines(self): self.assertRaises(TypeError, win.box, 65, 66, 67) self.assertRaises(TypeError, win.box, 65) win.box() + # With no arguments, box() likewise draws ACS corners and lines. + self.assertEqual(win.inch(0, 0) & chartext, curses.ACS_ULCORNER & chartext) + self.assertEqual(win.inch(0, maxx-1) & chartext, curses.ACS_URCORNER & chartext) + self.assertEqual(win.inch(0, 1) & chartext, curses.ACS_HLINE & chartext) + self.assertEqual(win.inch(1, 0) & chartext, curses.ACS_VLINE & chartext) win.move(1, 2) win.hline('-', 5) @@ -657,6 +895,43 @@ def test_borders_and_lines(self): self.assertEqual(win.inch(2, 1), b';'[0] | curses.A_STANDOUT) self.assertEqual(win.inch(3, 1), b'a'[0]) + # A border or line character of an 8-bit locale round-trips as its + # encoded byte. See _encodable for the character set. + encoding = win.encoding + for ch in ('é', '¤', '€', 'є'): + try: + b = ch.encode(encoding) + except UnicodeEncodeError: + continue + if len(b) != 1: + continue + # A wide build stores a character outside Latin-1 as a wide cell, + # not as its encoded byte, so it cannot round-trip here. + if ord(ch) > 0xff and hasattr(win, 'get_wch'): + continue + with self.subTest(ch=ch): + win.erase() + win.hline(2, 0, ch, 5) + self.assertEqual(win.instr(2, 0, 5), b * 5) + win.vline(0, 0, ch, 3) + self.assertEqual(win.instr(0, 0, 1), b) + self.assertEqual(win.instr(1, 0, 1), b) + win.border(ch, ch, ch, ch, ch, ch, ch, ch) + self.assertEqual(win.instr(0, 0), b * maxx) + if ord(ch) < 0x100: + # The same byte given as an int. A wide build stores it + # through the locale, so only a Latin-1 byte round-trips. + v = b[0] + win.erase() + win.hline(2, 0, v, 5) + self.assertEqual(win.instr(2, 0, 5), b * 5) + win.vline(0, 0, v, 3) + self.assertEqual(win.instr(1, 0, 1), b) + win.border(v, v, v, v, v, v, v, v) + self.assertEqual(win.instr(0, 0), b * maxx) + win.box(v, v) + self.assertEqual(win.instr(0, 1, 1), b) + def test_unctrl(self): # TODO: wunctrl() self.assertEqual(curses.unctrl(b'A'), b'A') @@ -665,6 +940,19 @@ def test_unctrl(self): self.assertEqual(curses.unctrl(b'\n'), b'^J') self.assertEqual(curses.unctrl('\n'), b'^J') self.assertEqual(curses.unctrl(10), b'^J') + # A printable non-ASCII byte of an 8-bit locale is returned unchanged. + # See _encodable for the character set. + encoding = self.stdscr.encoding + for ch in ('é', '¤', '€', 'є'): + try: + b = ch.encode(encoding) + except UnicodeEncodeError: + continue + if len(b) != 1: + continue + with self.subTest(ch=ch): + self.assertEqual(curses.unctrl(ch), b) + self.assertEqual(curses.unctrl(b[0]), b) # the byte as an int self.assertRaises(TypeError, curses.unctrl, b'') self.assertRaises(TypeError, curses.unctrl, b'AB') self.assertRaises(TypeError, curses.unctrl, '') @@ -816,6 +1104,10 @@ def test_prog_mode(self): self.skipTest('requires terminal') curses.def_prog_mode() curses.reset_prog_mode() + # def_shell_mode()/reset_shell_mode() are intentionally not exercised + # here: they capture and restore curses' "shell mode" terminal state, + # which is only meaningful before initscr(). Calling them mid-suite + # corrupts the modes that endwin() restores and breaks later tests. def test_beep(self): if (curses.tigetstr("bel") is not None @@ -1028,7 +1320,8 @@ def test_keyname(self): @requires_curses_func('has_key') def test_has_key(self): - curses.has_key(13) + self.assertIsInstance(curses.has_key(13), bool) + self.assertIsInstance(curses.has_key(curses.KEY_LEFT), bool) @requires_curses_func('getmouse') def test_getmouse(self): @@ -1080,6 +1373,291 @@ def test_disallow_instantiation(self): panel = curses.panel.new_panel(w) check_disallow_instantiation(self, type(panel)) + @requires_curses_func('panel') + def test_panel_stack(self): + panel = curses.panel + # new_panel() puts the panel on top of the stack, so the three + # panels end up ordered bottom -> top as p1, p2, p3. + p1 = panel.new_panel(curses.newwin(3, 6, 0, 0)) + p2 = panel.new_panel(curses.newwin(3, 6, 1, 1)) + p3 = panel.new_panel(curses.newwin(3, 6, 2, 2)) + self.addCleanup(self._delete_panels, p1, p2, p3) + + # The most recently created panel is on top. + self.assertIs(panel.top_panel(), p3) + # window() returns the wrapped window. + self.assertEqual(p2.window().getbegyx(), (1, 1)) + + # above()/below() walk the stack one step at a time. + self.assertIs(p1.above(), p2) + self.assertIs(p2.above(), p3) + self.assertIsNone(p3.above()) # nothing above the top panel + self.assertIs(p3.below(), p2) + self.assertIs(p2.below(), p1) + + # top() raises a panel to the top, bottom() lowers it to the bottom. + p1.top() + self.assertIs(panel.top_panel(), p1) + self.assertIsNone(p1.above()) + p1.bottom() + self.assertIs(panel.bottom_panel(), p1) + self.assertIsNone(p1.below()) + + # update_panels() refreshes the virtual screen from the stack. + panel.update_panels() + + @requires_curses_func('panel') + def test_panel_hide_show(self): + p = curses.panel.new_panel(curses.newwin(3, 6, 0, 0)) + self.addCleanup(self._delete_panels, p) + self.assertIs(p.hidden(), False) + p.hide() + self.assertIs(p.hidden(), True) + p.show() + self.assertIs(p.hidden(), False) + + @requires_curses_func('panel') + def test_panel_move(self): + win = curses.newwin(3, 6, 1, 2) + p = curses.panel.new_panel(win) + self.addCleanup(self._delete_panels, p) + self.assertEqual(win.getbegyx(), (1, 2)) + p.move(4, 5) + self.assertEqual(win.getbegyx(), (4, 5)) + + @requires_curses_func('panel') + def test_panel_replace(self): + win1 = curses.newwin(3, 6, 0, 0) + win2 = curses.newwin(4, 8, 1, 1) + p = curses.panel.new_panel(win1) + self.addCleanup(self._delete_panels, p) + self.assertIs(p.window(), win1) + p.replace(win2) + self.assertIs(p.window(), win2) + + @requires_curses_func('panel') + def test_panel_userptr(self): + p = curses.panel.new_panel(curses.newwin(3, 6, 0, 0)) + self.addCleanup(self._delete_panels, p) + obj = ['userptr'] + p.set_userptr(obj) + self.assertIs(p.userptr(), obj) + + def _delete_panels(self, *panels): + # Drop the panels from the global stack so they do not leak into + # later tests that inspect top_panel()/bottom_panel(). + for p in panels: + try: + p.bottom() + except curses.panel.error: + pass + del panels + gc_collect() + + def _make_textbox(self, nlines, ncols, *, insert_mode=False, stripspaces=1): + win = curses.newwin(nlines, ncols, 0, 0) + box = curses.textpad.Textbox(win, insert_mode=insert_mode) + box.stripspaces = stripspaces + return box, win + + def _type(self, box, text): + for ch in text: + box.do_command(ch if isinstance(ch, int) else ord(ch)) + + def test_textbox_gather(self): + # Typed text is read back by gather(). With stripspaces on (the + # default) gather() keeps a single trailing blank on a line and + # drops trailing empty lines. + box, win = self._make_textbox(3, 10) + self._type(box, 'Hello') + self.assertEqual(box.gather(), 'Hello \n') + + def test_textbox_gather_multiline(self): + box, win = self._make_textbox(3, 10) + self._type(box, 'ab') + box.do_command(curses.ascii.NL) # ^j -> start of next line + self._type(box, 'cd') + self.assertEqual(box.gather(), 'ab \ncd \n') + + def test_textbox_stripspaces(self): + box, win = self._make_textbox(1, 8, stripspaces=1) + self._type(box, 'hi') + self.assertEqual(box.gather(), 'hi ') + + box, win = self._make_textbox(1, 8, stripspaces=0) + self._type(box, 'hi') + self.assertEqual(box.gather(), 'hi ') + + def test_textbox_insert_mode(self): + # In insert mode a typed character shifts the rest of the line right. + box, win = self._make_textbox(1, 10, insert_mode=True) + self._type(box, 'aXc') + win.move(0, 1) + self._type(box, 'b') + self.assertEqual(box.gather(), 'abXc ') + + def test_textbox_fill_last_cell(self): + # The lower-right cell can be written, even though addch() there + # cannot advance the cursor past the end of the window. + box, win = self._make_textbox(1, 4, stripspaces=0) + self._type(box, 'abcd') + self.assertEqual(box.gather(), 'abcd') + + def test_textbox_fill_last_cell_multiline(self): + box, win = self._make_textbox(2, 3, stripspaces=0) + self._type(box, 'abc') + box.do_command(curses.ascii.NL) # ^j -> start of next line + self._type(box, 'def') # 'f' lands in the lower-right cell + self.assertEqual(box.gather(), 'abc\ndef\n') + + def test_textbox_fill_last_cell_insert_mode(self): + box, win = self._make_textbox(1, 4, insert_mode=True, stripspaces=0) + self._type(box, 'abcd') + self.assertEqual(box.gather(), 'abcd') + + def test_textbox_fill_last_cell_scrollok(self): + # Writing the lower-right cell must not scroll the window even if it + # has scrolling enabled. + box, win = self._make_textbox(2, 3, stripspaces=0) + win.scrollok(True) + self._type(box, 'abc') + box.do_command(curses.ascii.NL) + self._type(box, 'def') + self.assertEqual(box.gather(), 'abc\ndef\n') + + def test_textbox_8bit(self): + # A character of an 8-bit locale encoding is entered and read back + # through the byte API. The byte path also runs on a wide build, so the + # test is not skipped there. Run the suite under an 8-bit locale + # (ISO-8859-1, ISO-8859-15 or KOI8-U) to reach the non-ASCII cases; each + # string is used only if the encoding maps it to single bytes. 'abc' is + # ASCII, 'café' is common to the Latin encodings, and the rest are + # distinctive (byte 0xA4 is '¤'/'€'/'є' in ISO-8859-1/-15/KOI8-U). + encoding = self.stdscr.encoding + for text in ['abc', 'café', 'naïve ¤¦', 'café €Šž', 'дякую єі']: + try: + data = text.encode(encoding) + except UnicodeEncodeError: + continue + if len(data) != len(text): + continue # a multibyte encoding is not the 8-bit byte path + with self.subTest(text=text): + box, win = self._make_textbox(1, 16) + for byte in data: + box.do_command(byte) + self.assertEqual(box.gather(), text + ' ') + + def test_textbox_8bit_insert(self): + # Insert mode shifts the rest of the line right by reading each cell back + # and rewriting it; a non-ASCII 8-bit-locale character must survive the + # shift, even on a wide build where inch() mangles it. See + # test_textbox_8bit for the character choices. + encoding = self.stdscr.encoding + for ch in ['é', '¤', '€', 'є']: + try: + data = ch.encode(encoding) + except UnicodeEncodeError: + continue + if len(data) != 1: + continue + with self.subTest(ch=ch): + box, win = self._make_textbox(1, 10, insert_mode=True) + for byte in ('a' + ch + 'c').encode(encoding): + box.do_command(byte) + win.move(0, 1) + box.do_command(ord('b')) # insert 'b', shifting ch and 'c' right + self.assertEqual(box.gather(), 'ab' + ch + 'c ') + + def test_textbox_8bit_fill_last_cell(self): + # A non-ASCII 8-bit-locale character must survive being written to the + # lower-right cell, which uses insch() rather than addch(). See + # test_textbox_8bit for the character choices. + encoding = self.stdscr.encoding + for ch in ['é', '¤', '€', 'є']: + try: + data = ch.encode(encoding) + except UnicodeEncodeError: + continue + if len(data) != 1: + continue + with self.subTest(ch=ch): + text = 'ab' + ch # the last character fills the corner + box, win = self._make_textbox(1, len(text), stripspaces=0) + for byte in text.encode(encoding): + box.do_command(byte) + self.assertEqual(box.gather(), text) + + def test_textbox_movement(self): + box, win = self._make_textbox(3, 10) + self._type(box, 'abc') + box.do_command(curses.ascii.SOH) # ^a -> left edge + self.assertEqual(win.getyx(), (0, 0)) + box.do_command(curses.ascii.ENQ) # ^e -> end of line + self.assertEqual(win.getyx(), (0, 3)) + + def test_textbox_kill_to_eol(self): + box, win = self._make_textbox(1, 10) + self._type(box, 'abcdef') + win.move(0, 3) + box.do_command(curses.ascii.VT) # ^k -> clear to end of line + self.assertEqual(box.gather(), 'abc ') + + def test_textbox_backspace(self): + box, win = self._make_textbox(1, 10) + self._type(box, 'abc') + box.do_command(curses.ascii.BS) # ^h -> delete backward + self.assertEqual(box.gather(), 'ab ') + + def test_textbox_edit(self): + # edit() reads characters until Ctrl-G and returns the contents. + box, win = self._make_textbox(1, 10) + for ch in reversed('Hi' + chr(curses.ascii.BEL)): + curses.ungetch(ch) + self.assertEqual(box.edit(), 'Hi ') + + def test_textbox_edit_validate(self): + # The validate hook can rewrite an incoming keystroke. + box, win = self._make_textbox(1, 10) + for ch in reversed('abc' + chr(curses.ascii.BEL)): + curses.ungetch(ch) + box.edit(lambda ch: ord('X') if ch == ord('b') else ch) + self.assertEqual(box.gather(), 'aXc ') + + def test_textpad_rectangle(self): + # rectangle() draws a box with ACS line/corner characters. + win = curses.newwin(6, 12, 0, 0) + curses.textpad.rectangle(win, 0, 0, 4, 8) + chartext = curses.A_CHARTEXT + self.assertEqual(win.inch(0, 0) & chartext, + curses.ACS_ULCORNER & chartext) + self.assertEqual(win.inch(0, 8) & chartext, + curses.ACS_URCORNER & chartext) + self.assertEqual(win.inch(4, 0) & chartext, + curses.ACS_LLCORNER & chartext) + self.assertEqual(win.inch(4, 8) & chartext, + curses.ACS_LRCORNER & chartext) + self.assertEqual(win.inch(0, 1) & chartext, + curses.ACS_HLINE & chartext) + self.assertEqual(win.inch(1, 0) & chartext, + curses.ACS_VLINE & chartext) + + def test_wrapper(self): + # wrapper() sets up curses, passes the screen to the callable along + # with extra arguments, returns its result and restores the terminal. + if not self.isatty: + self.skipTest('requires terminal') + + def body(stdscr, a, b): + self.assertIsInstance(stdscr, type(self.stdscr)) + self.assertIs(curses.isendwin(), False) + return a + b + + self.assertEqual(curses.wrapper(body, 2, 3), 5) + self.assertIs(curses.isendwin(), True) + # wrapper() left the screen ended; revive it so the per-test + # endwin() cleanup does not fail with ERR. + curses.doupdate() + @requires_curses_func('is_term_resized') def test_is_term_resized(self): lines, cols = curses.LINES, curses.COLS @@ -1148,7 +1726,8 @@ def test_issue6243(self): def test_unget_wch(self): stdscr = self.stdscr encoding = stdscr.encoding - for ch in ('a', '\xe9', '\u20ac', '\U0010FFFF'): + # See _encodable for the character set, plus a non-BMP character. + for ch in ('a', '\xe9', '\xa4', '\u20ac', '\u0454', '\U0010FFFF'): try: ch.encode(encoding) except UnicodeEncodeError: @@ -1257,7 +1836,7 @@ class TestAscii(unittest.TestCase): def test_controlnames(self): for name in curses.ascii.controlnames: - self.assertTrue(hasattr(curses.ascii, name), name) + self.assertHasAttr(curses.ascii, name) def test_ctypes(self): def check(func, expected): @@ -1363,6 +1942,11 @@ def setUp(self): self.mock_win = MagicMock(spec=curses.window) self.mock_win.getyx.return_value = (1, 1) self.mock_win.getmaxyx.return_value = (10, 20) + self.mock_win.encoding = 'utf-8' + # A non-blank cell so that _end_of_line() reports a full line: instr() + # backs the text reads, inch() the insert-mode shift. + self.mock_win.instr.return_value = b'x' + self.mock_win.inch.return_value = ord('x') self.textbox = curses.textpad.Textbox(self.mock_win) def test_init(self): diff --git a/Lib/test/test_dataclasses/__init__.py b/Lib/test/test_dataclasses/__init__.py index ac78f8327b808ee..acbbf60ef3745b5 100644 --- a/Lib/test/test_dataclasses/__init__.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -120,7 +120,7 @@ class Some: pass for param in inspect.signature(dataclass).parameters: if param == 'cls': continue - self.assertTrue(hasattr(Some.__dataclass_params__, param), msg=param) + self.assertHasAttr(Some.__dataclass_params__, param) def test_named_init_params(self): @dataclass @@ -671,7 +671,7 @@ class C: self.assertEqual(the_fields[0].name, 'x') self.assertEqual(the_fields[0].type, int) - self.assertFalse(hasattr(C, 'x')) + self.assertNotHasAttr(C, 'x') self.assertTrue (the_fields[0].init) self.assertTrue (the_fields[0].repr) self.assertEqual(the_fields[1].name, 'y') @@ -681,7 +681,7 @@ class C: self.assertTrue (the_fields[1].repr) self.assertEqual(the_fields[2].name, 'z') self.assertEqual(the_fields[2].type, str) - self.assertFalse(hasattr(C, 'z')) + self.assertNotHasAttr(C, 'z') self.assertTrue (the_fields[2].init) self.assertFalse(the_fields[2].repr) @@ -732,8 +732,8 @@ class C: z: object = default t: int = field(default=100) - self.assertFalse(hasattr(C, 'x')) - self.assertFalse(hasattr(C, 'y')) + self.assertNotHasAttr(C, 'x') + self.assertNotHasAttr(C, 'y') self.assertIs (C.z, default) self.assertEqual(C.t, 100) @@ -927,6 +927,20 @@ class C: validate_class(C) + def test_incomplete_annotations(self): + # gh-142214 + @dataclass + class C: + "doc" # needed because otherwise we fetch the annotations at the wrong time + x: int + + C.__annotate__ = lambda _: {} + + self.assertEqual( + annotationlib.get_annotations(C.__init__), + {"return": None} + ) + def test_missing_default(self): # Test that MISSING works the same as a default not being # specified. @@ -2471,6 +2485,149 @@ def __init__(self, a): self.assertEqual(D(5).a, 10) +class TestInitAnnotate(unittest.TestCase): + # Tests for the generated __annotate__ function for __init__ + # See: https://github.com/python/cpython/issues/137530 + + def test_annotate_function(self): + # No forward references + @dataclass + class A: + a: int + + value_annos = annotationlib.get_annotations(A.__init__, format=annotationlib.Format.VALUE) + forwardref_annos = annotationlib.get_annotations(A.__init__, format=annotationlib.Format.FORWARDREF) + string_annos = annotationlib.get_annotations(A.__init__, format=annotationlib.Format.STRING) + + self.assertEqual(value_annos, {'a': int, 'return': None}) + self.assertEqual(forwardref_annos, {'a': int, 'return': None}) + self.assertEqual(string_annos, {'a': 'int', 'return': 'None'}) + + self.assertTrue(getattr(A.__init__.__annotate__, "__generated_by_dataclasses__")) + + def test_annotate_function_forwardref(self): + # With forward references + @dataclass + class B: + b: undefined + + # VALUE annotations should raise while unresolvable + with self.assertRaises(NameError): + _ = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.VALUE) + + forwardref_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.FORWARDREF) + string_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.STRING) + + self.assertEqual(forwardref_annos, {'b': support.EqualToForwardRef('undefined', owner=B, is_class=True), 'return': None}) + self.assertEqual(string_annos, {'b': 'undefined', 'return': 'None'}) + + # Now VALUE and FORWARDREF should resolve, STRING should be unchanged + undefined = int + + value_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.VALUE) + forwardref_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.FORWARDREF) + string_annos = annotationlib.get_annotations(B.__init__, format=annotationlib.Format.STRING) + + self.assertEqual(value_annos, {'b': int, 'return': None}) + self.assertEqual(forwardref_annos, {'b': int, 'return': None}) + self.assertEqual(string_annos, {'b': 'undefined', 'return': 'None'}) + + def test_annotate_function_init_false(self): + # Check `init=False` attributes don't get into the annotations of the __init__ function + @dataclass + class C: + c: str = field(init=False) + + self.assertEqual(annotationlib.get_annotations(C.__init__), {'return': None}) + + def test_annotate_function_contains_forwardref(self): + # Check string annotations on objects containing a ForwardRef + @dataclass + class D: + d: list[undefined] + + with self.assertRaises(NameError): + annotationlib.get_annotations(D.__init__) + + self.assertEqual( + annotationlib.get_annotations(D.__init__, format=annotationlib.Format.FORWARDREF), + {"d": list[support.EqualToForwardRef("undefined", is_class=True, owner=D)], "return": None} + ) + + self.assertEqual( + annotationlib.get_annotations(D.__init__, format=annotationlib.Format.STRING), + {"d": "list[undefined]", "return": "None"} + ) + + # Now test when it is defined + undefined = str + + # VALUE should now resolve + self.assertEqual( + annotationlib.get_annotations(D.__init__), + {"d": list[str], "return": None} + ) + + self.assertEqual( + annotationlib.get_annotations(D.__init__, format=annotationlib.Format.FORWARDREF), + {"d": list[str], "return": None} + ) + + self.assertEqual( + annotationlib.get_annotations(D.__init__, format=annotationlib.Format.STRING), + {"d": "list[undefined]", "return": "None"} + ) + + def test_annotate_function_not_replaced(self): + # Check that __annotate__ is not replaced on non-generated __init__ functions + @dataclass(slots=True) + class E: + x: str + def __init__(self, x: int) -> None: + self.x = x + + self.assertEqual( + annotationlib.get_annotations(E.__init__), {"x": int, "return": None} + ) + + self.assertFalse(hasattr(E.__init__.__annotate__, "__generated_by_dataclasses__")) + + def test_slots_true_init_false(self): + # Test that slots=True and init=False work together and + # that __annotate__ is not added to __init__. + + @dataclass(slots=True, init=False) + class F: + x: int + + f = F() + f.x = 10 + self.assertEqual(f.x, 10) + + self.assertFalse(hasattr(F.__init__, "__annotate__")) + + def test_init_false_forwardref(self): + # Test forward references in fields not required for __init__ annotations. + + # At the moment this raises a NameError for VALUE annotations even though the + # undefined annotation is not required for the __init__ annotations. + # Ideally this will be fixed but currently there is no good way to resolve this + + @dataclass + class F: + not_in_init: list[undefined] = field(init=False, default=None) + in_init: int + + annos = annotationlib.get_annotations(F.__init__, format=annotationlib.Format.FORWARDREF) + self.assertEqual( + annos, + {"in_init": int, "return": None}, + ) + + with self.assertRaises(NameError): + annos = annotationlib.get_annotations(F.__init__) # NameError on not_in_init + + class TestRepr(unittest.TestCase): def test_repr(self): @dataclass @@ -2597,6 +2754,55 @@ def __eq__(self, other): self.assertEqual(C(1), 5) self.assertNotEqual(C(1), 1) + def test_eq_field_by_field(self): + @dataclasses.dataclass + class Point: + x: int + y: int + + p1 = Point(1, 2) + p2 = Point(1, 2) + p3 = Point(2, 1) + self.assertEqual(p1, p2) + self.assertNotEqual(p1, p3) + + def test_eq_type_check(self): + @dataclasses.dataclass + class A: + x: int + + @dataclasses.dataclass + class B: + x: int + + a = A(1) + b = B(1) + self.assertNotEqual(a, b) + + def test_eq_custom_field(self): + class AlwaysEqual(int): + def __eq__(self, other): + return True + + @dataclasses.dataclass + class Foo: + x: AlwaysEqual + y: int + + f1 = Foo(AlwaysEqual(1), 2) + f2 = Foo(AlwaysEqual(2), 2) + self.assertEqual(f1, f2) + + def test_eq_nan_field(self): + @dataclasses.dataclass + class D: + x: float + + nan = float('nan') + d1 = D(nan) + d2 = D(nan) + self.assertNotEqual(d1, d2) + class TestOrdering(unittest.TestCase): def test_functools_total_ordering(self): @@ -2895,29 +3101,41 @@ class C(base): class TestFrozen(unittest.TestCase): + # Some tests have a subtest with a slotted dataclass. + # See https://github.com/python/cpython/issues/105936 for the reasons. + def test_frozen(self): - @dataclass(frozen=True) - class C: - i: int + for slots in (False, True): + with self.subTest(slots=slots): - c = C(10) - self.assertEqual(c.i, 10) - with self.assertRaises(FrozenInstanceError): - c.i = 5 - self.assertEqual(c.i, 10) + @dataclass(frozen=True, slots=slots) + class C: + i: int + + c = C(10) + self.assertEqual(c.i, 10) + with self.assertRaises(FrozenInstanceError): + c.i = 5 + self.assertEqual(c.i, 10) + with self.assertRaises(FrozenInstanceError): + del c.i + self.assertEqual(c.i, 10) def test_frozen_empty(self): - @dataclass(frozen=True) - class C: - pass + for slots in (False, True): + with self.subTest(slots=slots): - c = C() - self.assertFalse(hasattr(c, 'i')) - with self.assertRaises(FrozenInstanceError): - c.i = 5 - self.assertFalse(hasattr(c, 'i')) - with self.assertRaises(FrozenInstanceError): - del c.i + @dataclass(frozen=True, slots=slots) + class C: + pass + + c = C() + self.assertNotHasAttr(c, 'i') + with self.assertRaises(FrozenInstanceError): + c.i = 5 + self.assertNotHasAttr(c, 'i') + with self.assertRaises(FrozenInstanceError): + del c.i def test_inherit(self): @dataclass(frozen=True) @@ -3113,41 +3331,123 @@ class D(I): d.i = 5 def test_non_frozen_normal_derived(self): - # See bpo-32953. - - @dataclass(frozen=True) - class D: - x: int - y: int = 10 - - class S(D): - pass - - s = S(3) - self.assertEqual(s.x, 3) - self.assertEqual(s.y, 10) - s.cached = True - - # But can't change the frozen attributes. - with self.assertRaises(FrozenInstanceError): - s.x = 5 - with self.assertRaises(FrozenInstanceError): - s.y = 5 - self.assertEqual(s.x, 3) - self.assertEqual(s.y, 10) - self.assertEqual(s.cached, True) + # See bpo-32953 and https://github.com/python/cpython/issues/105936 + for slots in (False, True): + with self.subTest(slots=slots): + + @dataclass(frozen=True, slots=slots) + class D: + x: int + y: int = 10 + z: int = 1 + + @property + def readonly(self) -> int: + return self.x + + @property + def prop(self) -> int: + return self.z + + @prop.setter + def prop(self, val: int) -> None: + object.__setattr__(self, 'z', val) + + @prop.deleter + def prop(self) -> None: + object.__setattr__(self, 'z', 0) + + d = D(5) + self.assertEqual(d.x, 5) + self.assertEqual(d.y, 10) + self.assertEqual(d.z, 1) + self.assertEqual(d.readonly, 5) + self.assertEqual(d.prop, 1) + + with self.assertRaises(FrozenInstanceError): + d.x = 5 + with self.assertRaises(FrozenInstanceError): + d.readonly = 5 + with self.assertRaises(FrozenInstanceError): + d.z = 5 + with self.assertRaises(FrozenInstanceError): + d.prop = 5 + with self.assertRaises(FrozenInstanceError): + del d.prop + + self.assertEqual(d.x, 5) + self.assertEqual(d.y, 10) + self.assertEqual(d.z, 1) + self.assertEqual(d.readonly, 5) + self.assertEqual(d.prop, 1) + + class S(D): + pass - with self.assertRaises(FrozenInstanceError): - del s.x - self.assertEqual(s.x, 3) - with self.assertRaises(FrozenInstanceError): - del s.y - self.assertEqual(s.y, 10) - del s.cached - self.assertFalse(hasattr(s, 'cached')) - with self.assertRaises(AttributeError) as cm: - del s.cached - self.assertNotIsInstance(cm.exception, FrozenInstanceError) + s = S(3) + self.assertEqual(s.x, 3) + self.assertEqual(s.y, 10) + self.assertEqual(s.z, 1) + self.assertEqual(s.readonly, 3) + self.assertEqual(s.prop, 1) + # Can set new attrs: + s.cached = True + self.assertTrue(s.cached) + # Can mutate them: + s.cached = False + self.assertFalse(s.cached) + + # Can also change writable properties: + with self.assertRaisesRegex( + AttributeError, + 'object has no setter', + ) as cm: + s.readonly = 5 + self.assertNotIsInstance(cm.exception, FrozenInstanceError) + s.prop = 2 + self.assertEqual(s.x, 3) + self.assertEqual(s.readonly, 3) + self.assertEqual(s.prop, 2) + self.assertEqual(s.z, 2) + + # But can't change the frozen attributes. + with self.assertRaises(FrozenInstanceError): + s.x = 5 + with self.assertRaises(FrozenInstanceError): + s.y = 5 + with self.assertRaises(FrozenInstanceError): + s.z = 5 + self.assertEqual(s.x, 3) + self.assertEqual(s.y, 10) + self.assertEqual(s.z, 2) + self.assertIs(s.cached, False) + + with self.assertRaises(FrozenInstanceError): + del s.x + self.assertEqual(s.x, 3) + with self.assertRaises(FrozenInstanceError): + del s.y + self.assertEqual(s.y, 10) + with self.assertRaisesRegex( + AttributeError, + 'object has no deleter', + ) as cm: + del s.readonly + self.assertNotIsInstance(cm.exception, FrozenInstanceError) + self.assertEqual(s.x, 3) + self.assertEqual(s.readonly, 3) + del s.cached + self.assertNotHasAttr(s, 'cached') + with self.assertRaisesRegex( + AttributeError, + "object has no attribute 'cached'", + ) as cm: + del s.cached + self.assertNotIsInstance(cm.exception, FrozenInstanceError) + del s.prop + self.assertEqual(s.z, 0) + self.assertEqual(s.prop, 0) + del s.prop def test_non_frozen_normal_derived_from_empty_frozen(self): @dataclass(frozen=True) @@ -3158,12 +3458,12 @@ class S(D): pass s = S() - self.assertFalse(hasattr(s, 'x')) + self.assertNotHasAttr(s, 'x') s.x = 5 self.assertEqual(s.x, 5) del s.x - self.assertFalse(hasattr(s, 'x')) + self.assertNotHasAttr(s, 'x') with self.assertRaises(AttributeError) as cm: del s.x self.assertNotIsInstance(cm.exception, FrozenInstanceError) @@ -3393,8 +3693,8 @@ class A: B = dataclass(A, slots=True) self.assertIsNot(A, B) - self.assertFalse(hasattr(A, "__slots__")) - self.assertTrue(hasattr(B, "__slots__")) + self.assertNotHasAttr(A, "__slots__") + self.assertHasAttr(B, "__slots__") # Can't be local to test_frozen_pickle. @dataclass(frozen=True, slots=True) @@ -3804,6 +4104,57 @@ class WithCorrectSuper(CorrectSuper): # that we create internally. self.assertEqual(CorrectSuper.args, ["default", "default"]) + def test_original_class_is_gced(self): + # gh-135228: Make sure when we replace the class with slots=True, the original class + # gets garbage collected. + def make_simple(): + @dataclass(slots=True) + class SlotsTest: + pass + + return SlotsTest + + # See https://github.com/python/cpython/issues/135228#issuecomment-3755979059 + def make_frozen(): + @dataclass(frozen=True, slots=True) + class SlotsTest: + pass + + return SlotsTest + + def make_with_annotations(): + @dataclass(slots=True) + class SlotsTest: + x: int + + return SlotsTest + + def make_with_annotations_and_method(): + @dataclass(slots=True) + class SlotsTest: + x: int + + def method(self) -> int: + return self.x + + return SlotsTest + + def make_with_forwardref(): + @dataclass(slots=True) + class SlotsTest: + x: undefined + y: list[undefined] + + return SlotsTest + + for make in (make_simple, make_frozen, make_with_annotations, make_with_annotations_and_method, make_with_forwardref): + with self.subTest(make=make): + C = make() + support.gc_collect() + candidates = [cls for cls in object.__subclasses__() if cls.__name__ == 'SlotsTest' + and cls.__firstlineno__ == make.__code__.co_firstlineno + 1] + self.assertEqual(candidates, [C]) + class TestDescriptors(unittest.TestCase): def test_set_name(self): @@ -5139,5 +5490,51 @@ def cls(self): # one will be keeping a reference to the underlying class A. self.assertIs(A().cls(), B) + def test_empty_class_cell(self): + # gh-148947: Make sure that we explicitly handle the empty class cell. + def maker(): + if False: + __class__ = 42 + + def method(self): + return __class__ + return method + + from dataclasses import dataclass + + @dataclass(slots=True) + class X: + a: int + + meth = maker() + + with self.assertRaisesRegex(NameError, '__class__'): + X(1).meth() + + def test_class_cell_from_other_class(self): + # This test fails without the "is oldcls" check in + # _update_func_cell_for__class__. + class Base: + def meth(self): + return "Base" + + class Child(Base): + def meth(self): + return super().meth() + " Child" + + @dataclass(slots=True) + class DC(Child): + a: int + + meth = Child.meth + + closure = DC.meth.__closure__ + self.assertEqual(len(closure), 1) + self.assertIs(closure[0].cell_contents, Child) + + self.assertEqual(DC(1).meth(), "Base Child") + + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_datetime.py b/Lib/test/test_datetime.py index 005187f13e665f4..137c8d2686c224b 100644 --- a/Lib/test/test_datetime.py +++ b/Lib/test/test_datetime.py @@ -18,16 +18,19 @@ def load_tests(loader, tests, pattern): finally: # XXX: import_fresh_module() is supposed to leave sys.module cache untouched, # XXX: but it does not, so we have to cleanup ourselves. - for modname in ['datetime', '_datetime', '_strptime']: + for modname in ['datetime', '_datetime', '_pydatetime', '_strptime']: sys.modules.pop(modname, None) test_modules = [pure_tests, fast_tests] test_suffixes = ["_Pure", "_Fast"] + # XXX(gb) First run all the _Pure tests, then all the _Fast tests. You might # not believe this, but in spite of all the sys.modules trickery running a _Pure # test last will leave a mix of pure and native datetime stuff lying around. for module, suffix in zip(test_modules, test_suffixes): test_classes = [] + if module is None: + continue for name, cls in module.__dict__.items(): if not isinstance(cls, type): continue @@ -48,8 +51,8 @@ def setUpClass(cls_, module=module): cls_._save_sys_modules = sys.modules.copy() sys.modules[TESTS] = module sys.modules['datetime'] = module.datetime_module - if hasattr(module, '_pydatetime'): - sys.modules['_pydatetime'] = module._pydatetime + sys.modules['_pydatetime'] = module._pydatetime + sys.modules['_datetime'] = module._datetime sys.modules['_strptime'] = module._strptime super().setUpClass() diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index 4be7c5649da68aa..014c190285c4024 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -66,7 +66,7 @@ def keys_helper(self, f): return keys def test_error(self): - self.assertTrue(issubclass(self.module.error, OSError)) + self.assertIsSubclass(self.module.error, OSError) def test_anydbm_not_existing(self): self.assertRaises(dbm.error, dbm.open, _fname) @@ -213,7 +213,8 @@ def test_whichdb(self): @unittest.skipUnless(ndbm, reason='Test requires ndbm') def test_whichdb_ndbm(self): # Issue 17198: check that ndbm which is referenced in whichdb is defined - with open(_fname + '.db', 'wb'): pass + with open(_fname + '.db', 'wb') as f: + f.write(b'spam') _bytes_fname = os.fsencode(_fname) fnames = [_fname, os_helper.FakePath(_fname), _bytes_fname, os_helper.FakePath(_bytes_fname)] diff --git a/Lib/test/test_dbm_sqlite3.py b/Lib/test/test_dbm_sqlite3.py index 2e1f2d32924badb..f367a98865d4aad 100644 --- a/Lib/test/test_dbm_sqlite3.py +++ b/Lib/test/test_dbm_sqlite3.py @@ -1,3 +1,5 @@ +import os +import stat import sys import unittest from contextlib import closing @@ -14,6 +16,11 @@ from dbm.sqlite3 import _normalize_uri +root_in_posix = False +if hasattr(os, 'geteuid'): + root_in_posix = (os.geteuid() == 0) + + class _SQLiteDbmTests(unittest.TestCase): def setUp(self): @@ -36,7 +43,7 @@ def test_uri_substitutions(self): ) for path, normalized in dataset: with self.subTest(path=path, normalized=normalized): - self.assertTrue(_normalize_uri(path).endswith(normalized)) + self.assertEndsWith(_normalize_uri(path), normalized) @unittest.skipUnless(sys.platform == "win32", "requires Windows") def test_uri_windows(self): @@ -55,7 +62,7 @@ def test_uri_windows(self): with self.subTest(path=path, normalized=normalized): if not Path(path).is_absolute(): self.skipTest(f"skipping relative path: {path!r}") - self.assertTrue(_normalize_uri(path).endswith(normalized)) + self.assertEndsWith(_normalize_uri(path), normalized) class ReadOnly(_SQLiteDbmTests): @@ -90,6 +97,50 @@ def test_readonly_iter(self): self.assertEqual([k for k in self.db], [b"key1", b"key2"]) +@unittest.skipIf(root_in_posix, "test is meanless with root privilege") +class ReadOnlyFilesystem(unittest.TestCase): + + def setUp(self): + self.test_dir = os_helper.TESTFN + self.addCleanup(os_helper.rmtree, self.test_dir) + os.mkdir(self.test_dir) + self.db_path = os.path.join(self.test_dir, "test.db") + + db = dbm_sqlite3.open(self.db_path, "c") + db[b"key"] = b"value" + db.close() + + def test_readonly_file_read(self): + os.chmod(self.db_path, stat.S_IREAD) + with dbm_sqlite3.open(self.db_path, "r") as db: + self.assertEqual(db[b"key"], b"value") + + def test_readonly_file_write(self): + os.chmod(self.db_path, stat.S_IREAD) + with dbm_sqlite3.open(self.db_path, "w") as db: + with self.assertRaises(dbm_sqlite3.error): + db[b"newkey"] = b"newvalue" + + def test_readonly_dir_read(self): + os.chmod(self.test_dir, stat.S_IREAD | stat.S_IEXEC) + with dbm_sqlite3.open(self.db_path, "r") as db: + self.assertEqual(db[b"key"], b"value") + + def test_readonly_dir_write(self): + os.chmod(self.test_dir, stat.S_IREAD | stat.S_IEXEC) + with dbm_sqlite3.open(self.db_path, "w") as db: + try: + db[b"newkey"] = b"newvalue" + modified = True # on Windows and macOS + except dbm_sqlite3.error: + modified = False + with dbm_sqlite3.open(self.db_path, "r") as db: + if modified: + self.assertEqual(db[b"newkey"], b"newvalue") + else: + self.assertNotIn(b"newkey", db) + + class ReadWrite(_SQLiteDbmTests): def setUp(self): diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 9e298401dc3dcc1..7e3896ed34e7bae 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -982,6 +982,7 @@ def test_formatting(self): ('.0f', '0e-2', '0'), ('.0f', '3.14159265', '3'), ('.1f', '3.14159265', '3.1'), + ('.01f', '3.14159265', '3.1'), # leading zero in precision ('.4f', '3.14159265', '3.1416'), ('.6f', '3.14159265', '3.141593'), ('.7f', '3.14159265', '3.1415926'), # round-half-even! @@ -1067,6 +1068,7 @@ def test_formatting(self): ('8,', '123456', ' 123,456'), ('08,', '123456', '0,123,456'), # special case: extra 0 needed ('+08,', '123456', '+123,456'), # but not if there's a sign + ('008,', '123456', '0,123,456'), # leading zero in width (' 08,', '123456', ' 123,456'), ('08,', '-123456', '-123,456'), ('+09,', '123456', '+0,123,456'), @@ -1088,6 +1090,15 @@ def test_formatting(self): ('07_', '1234.56', '1_234.56'), ('_', '1.23456789', '1.23456789'), ('_%', '123.456789', '12_345.6789%'), + # and now for something completely different... + ('.,', '1.23456789', '1.234,567,89'), + ('._', '1.23456789', '1.234_567_89'), + ('.6_f', '12345.23456789', '12345.234_568'), + (',._%', '123.456789', '12,345.678_9%'), + (',._e', '123456', '1.234_56e+5'), + (',.4_e', '123456', '1.234_6e+5'), + (',.3_e', '123456', '1.235e+5'), + (',._E', '123456', '1.234_56E+5'), # negative zero: default behavior ('.1f', '-0', '-0.0'), @@ -1161,6 +1172,10 @@ def test_formatting(self): # bytes format argument self.assertRaises(TypeError, Decimal(1).__format__, b'-020') + # precision or fractional part separator should follow after dot + self.assertRaises(ValueError, format, Decimal(1), '.f') + self.assertRaises(ValueError, format, Decimal(1), '._6f') + def test_negative_zero_format_directed_rounding(self): with self.decimal.localcontext() as ctx: ctx.rounding = ROUND_CEILING @@ -4127,6 +4142,15 @@ def test_float_operation_default(self): @requires_cdecimal class CContextFlags(ContextFlags, unittest.TestCase): decimal = C + + def test_signaldict_repr(self): + Context = self.decimal.Context + ctx = Context(prec=7) + mapping = ctx.flags + del ctx + with self.assertRaisesRegex(ValueError, 'invalid signal dict'): + repr(mapping) + class PyContextFlags(ContextFlags, unittest.TestCase): decimal = P diff --git a/Lib/test/test_defaultdict.py b/Lib/test/test_defaultdict.py index bdbe9b81e8fb3f5..a193eb10f16d178 100644 --- a/Lib/test/test_defaultdict.py +++ b/Lib/test/test_defaultdict.py @@ -186,5 +186,38 @@ def test_union(self): with self.assertRaises(TypeError): i |= None + def test_factory_conflict_with_set_value(self): + key = "conflict_test" + count = 0 + + def default_factory(): + nonlocal count + count += 1 + local_count = count + if count == 1: + test_dict[key] + return local_count + + test_dict = defaultdict(default_factory) + + self.assertEqual(count, 0) + self.assertEqual(test_dict[key], 2) + self.assertEqual(count, 2) + + def test_repr_recursive_factory(self): + # gh-145492: defaultdict.__repr__ should not cause infinite recursion + # when the factory's __repr__ calls repr() on the defaultdict. + class ProblematicFactory: + def __call__(self): + return {} + def __repr__(self): + repr(dd) + return f"ProblematicFactory for {dd}" + + dd = defaultdict(ProblematicFactory()) + # Should not raise RecursionError + r = repr(dd) + self.assertIn("ProblematicFactory for", r) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index 4679f297fd7f4a1..3c45032cda91387 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -287,6 +287,22 @@ def test_index(self): else: self.assertEqual(d.index(element, start, stop), target) + # Test stop argument + for elem in d: + index = d.index(elem) + self.assertEqual( + index, + d.index(elem, 0), + ) + self.assertEqual( + index, + d.index(elem, 0, len(d)), + ) + self.assertEqual( + index, + d.index(elem, 0, len(d) + 100), + ) + # Test large start argument d = deque(range(0, 10000, 10)) for step in range(100): @@ -838,7 +854,7 @@ def test_copy_pickle(self): self.assertEqual(list(d), list(e)) self.assertEqual(e.x, d.x) self.assertEqual(e.z, d.z) - self.assertFalse(hasattr(e, 'y')) + self.assertNotHasAttr(e, 'y') def test_pickle_recursive(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 76937432a43037e..00e5386b1d56b98 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -409,7 +409,7 @@ class ClassPropertiesAndMethods(unittest.TestCase): def test_python_dicts(self): # Testing Python subclass of dict... - self.assertTrue(issubclass(dict, dict)) + self.assertIsSubclass(dict, dict) self.assertIsInstance({}, dict) d = dict() self.assertEqual(d, {}) @@ -433,7 +433,7 @@ def setstate(self, state): self.state = state def getstate(self): return self.state - self.assertTrue(issubclass(C, dict)) + self.assertIsSubclass(C, dict) a1 = C(12) self.assertEqual(a1.state, 12) a2 = C(foo=1, bar=2) @@ -815,6 +815,15 @@ class X(C, int()): class X(int(), C): pass + @unittest.skipIf(_testcapi is None, 'need the _testcapi module') + def test_type_with_null_new_metaclass(self): + metaclass = _testcapi.HeapCTypeMetaclassNullNew + base = _testcapi.pytype_fromspec_meta(metaclass) + + # Exercise type_new's metaclass selection path, not a direct call. + with self.assertRaisesRegex(TypeError, r"cannot create '.*' instances"): + type("Derived", (base,), {}) + def test_module_subclasses(self): # Testing Python subclass of module... log = [] @@ -1048,15 +1057,15 @@ class SubType(types.ModuleType): m = types.ModuleType("m") self.assertTrue(m.__class__ is types.ModuleType) - self.assertFalse(hasattr(m, "a")) + self.assertNotHasAttr(m, "a") m.__class__ = SubType self.assertTrue(m.__class__ is SubType) - self.assertTrue(hasattr(m, "a")) + self.assertHasAttr(m, "a") m.__class__ = types.ModuleType self.assertTrue(m.__class__ is types.ModuleType) - self.assertFalse(hasattr(m, "a")) + self.assertNotHasAttr(m, "a") # Make sure that builtin immutable objects don't support __class__ # assignment, because the object instances may be interned. @@ -1712,6 +1721,28 @@ class SubSpam(spam.spamlist): pass spam_cm.__get__(None, list) self.assertEqual(str(cm.exception), expected_errmsg) + @support.cpython_only + def test_method_get_meth_method_invalid_type(self): + # gh-146615: method_get() for METH_METHOD descriptors used to pass + # Py_TYPE(type)->tp_name as the %V fallback instead of the separate + # %s argument, causing a missing argument for %s and a crash. + # Verify the error message is correct when __get__() is called with a + # non-type as the second argument. + # + # METH_METHOD|METH_FASTCALL|METH_KEYWORDS is the only flag combination + # that enters the affected branch in method_get(). + import io + + obj = io.StringIO() + descr = io.TextIOBase.read + + with self.assertRaises(TypeError) as cm: + descr.__get__(obj, "not_a_type") + self.assertEqual( + str(cm.exception), + "descriptor 'read' needs a type, not 'str', as arg 2", + ) + def test_staticmethods(self): # Testing static methods... class C(object): @@ -1780,7 +1811,7 @@ class D(C): class E: # *not* subclassing from C foo = C.foo self.assertEqual(E().foo.__func__, C.foo) # i.e., unbound - self.assertTrue(repr(C.foo.__get__(C())).startswith("') + self.assertIsNone(sm.__func__) + self.assertIsNone(sm.__wrapped__) + + def test_classmethod_new(self): + class MyClassMethod(classmethod): + def __init__(self, func): + pass + def func(): pass + cm = MyClassMethod(func) + self.assertEqual(repr(cm), '') + self.assertIsNone(cm.__func__) + self.assertIsNone(cm.__wrapped__) + class DictProxyTests(unittest.TestCase): def setUp(self): @@ -5194,8 +5250,8 @@ def test_repr(self): # We can't blindly compare with the repr of another dict as ordering # of keys and values is arbitrary and may differ. r = repr(self.C.__dict__) - self.assertTrue(r.startswith('mappingproxy('), r) - self.assertTrue(r.endswith(')'), r) + self.assertStartsWith(r, 'mappingproxy(') + self.assertEndsWith(r, ')') for k, v in self.C.__dict__.items(): self.assertIn('{!r}: {!r}'.format(k, v), r) diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 3104cbc66cb1150..7cc6dc7715e2fdf 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -12,6 +12,15 @@ from test.support import import_helper +class CustomHash: + def __init__(self, hash): + self.hash = hash + def __hash__(self): + return self.hash + def __repr__(self): + return f'' + + class DictTest(unittest.TestCase): def test_invalid_keyword_arguments(self): @@ -290,6 +299,38 @@ def badgen(): ['Cannot convert dictionary update sequence element #0 to a sequence'], ) + def test_update_shared_keys(self): + class MyClass: pass + + # Subclass str to enable us to create an object during the + # dict.update() call. + class MyStr(str): + def __hash__(self): + return super().__hash__() + + def __eq__(self, other): + # Create an object that shares the same PyDictKeysObject as + # obj.__dict__. + obj2 = MyClass() + obj2.a = "a" + obj2.b = "b" + obj2.c = "c" + return super().__eq__(other) + + obj = MyClass() + obj.a = "a" + obj.b = "b" + + x = {} + x[MyStr("a")] = MyStr("a") + + # gh-132617: this previously raised "dict mutated during update" error + x.update(obj.__dict__) + + self.assertEqual(x, { + MyStr("a"): "a", + "b": "b", + }) def test_fromkeys(self): self.assertEqual(dict.fromkeys('abc'), {'a':None, 'b':None, 'c':None}) @@ -338,17 +379,34 @@ def __setitem__(self, key, value): self.assertRaises(Exc, baddict2.fromkeys, [1]) # test fast path for dictionary inputs + res = dict(zip(range(6), [0]*6)) d = dict(zip(range(6), range(6))) - self.assertEqual(dict.fromkeys(d, 0), dict(zip(range(6), [0]*6))) - + self.assertEqual(dict.fromkeys(d, 0), res) + # test fast path for set inputs + d = set(range(6)) + self.assertEqual(dict.fromkeys(d, 0), res) + # test slow path for other iterable inputs + d = list(range(6)) + self.assertEqual(dict.fromkeys(d, 0), res) + + # test fast path when object's constructor returns large non-empty dict class baddict3(dict): def __new__(cls): return d - d = {i : i for i in range(10)} + d = {i : i for i in range(1000)} res = d.copy() res.update(a=None, b=None, c=None) self.assertEqual(baddict3.fromkeys({"a", "b", "c"}), res) + # test slow path when object is a proper subclass of dict + class baddict4(dict): + def __init__(self): + dict.__init__(self, d) + d = {i : i for i in range(1000)} + res = d.copy() + res.update(a=None, b=None, c=None) + self.assertEqual(baddict4.fromkeys({"a", "b", "c"}), res) + def test_copy(self): d = {1: 1, 2: 2, 3: 3} self.assertIsNot(d.copy(), d) @@ -620,6 +678,7 @@ def __repr__(self): d = {1: BadRepr()} self.assertRaises(Exc, repr, d) + @support.skip_if_huge_c_stack() @support.skip_wasi_stack_overflow() @support.skip_emscripten_stack_overflow() def test_repr_deep(self): @@ -770,8 +829,8 @@ def test_dictview_mixed_set_operations(self): def test_missing(self): # Make sure dict doesn't have a __missing__ method - self.assertFalse(hasattr(dict, "__missing__")) - self.assertFalse(hasattr({}, "__missing__")) + self.assertNotHasAttr(dict, "__missing__") + self.assertNotHasAttr({}, "__missing__") # Test several cases: # (D) subclass defines __missing__ method returning a value # (E) subclass defines __missing__ method raising RuntimeError @@ -1022,10 +1081,8 @@ class C: a = C() a.x = 1 d = a.__dict__ - before_resize = sys.getsizeof(d) d[2] = 2 # split table is resized to a generic combined table - self.assertGreater(sys.getsizeof(d), before_resize) self.assertEqual(list(d), ['x', 2]) def test_iterator_pickling(self): @@ -1513,6 +1570,26 @@ def make_pairs(): self.assertEqual(d.get(key3_3), 44) self.assertGreaterEqual(eq_count, 1) + def test_overwrite_managed_dict(self): + # GH-130327: Overwriting an object's managed dictionary with another object's + # skipped traversal in favor of inline values, causing the GC to believe that + # the __dict__ wasn't reachable. + import gc + + class Shenanigans: + pass + + to_be_deleted = Shenanigans() + to_be_deleted.attr = "whatever" + holds_reference = Shenanigans() + holds_reference.__dict__ = to_be_deleted.__dict__ + holds_reference.ref = {"circular": to_be_deleted, "data": 42} + + del to_be_deleted + gc.collect() + self.assertEqual(holds_reference.ref['data'], 42) + self.assertEqual(holds_reference.attr, "whatever") + def test_unhashable_key(self): d = {'a': 1} key = [1, 2, 3] @@ -1554,6 +1631,139 @@ def __hash__(self): with self.assertRaises(KeyError): d.get(key2) + def test_clear_at_lookup(self): + # gh-140551 dict crash if clear is called at lookup stage + class X: + def __hash__(self): + return 1 + def __eq__(self, other): + nonlocal d + d.clear() + + d = {} + for _ in range(10): + d[X()] = None + + self.assertEqual(len(d), 1) + + d = {} + for _ in range(10): + d.setdefault(X(), None) + + self.assertEqual(len(d), 1) + + def test_split_table_update_with_str_subclass(self): + # gh-142218: inserting into a split table dictionary with a non str + # key that matches an existing key. + class MyStr(str): pass + class MyClass: pass + obj = MyClass() + obj.attr = 1 + obj.__dict__[MyStr('attr')] = 2 + self.assertEqual(obj.attr, 2) + + def test_split_table_insert_with_str_subclass(self): + # gh-143189: inserting into split table dictionary with a non str + # key that matches an existing key in the shared table but not in + # the dict yet. + + class MyStr(str): pass + class MyClass: pass + + obj = MyClass() + obj.attr1 = 1 + + obj2 = MyClass() + d = obj2.__dict__ + d[MyStr("attr1")] = 2 + self.assertIsInstance(list(d)[0], MyStr) + + def test_hash_collision_remove_add(self): + self.maxDiff = None + # There should be enough space, so all elements with unique hash + # will be placed in corresponding cells without collision. + n = 64 + items = [(CustomHash(h), h) for h in range(n)] + # Keys with hash collision. + a = CustomHash(n) + b = CustomHash(n) + items += [(a, 'a'), (b, 'b')] + d = dict(items) + self.assertEqual(len(d), len(items), d) + del d[a] + # "a" has been replaced with a dummy. + del items[n] + self.assertEqual(len(d), len(items), d) + self.assertEqual(d, dict(items)) + d[b] = 'c' + # "b" should not replace the dummy. + items[n] = (b, 'c') + self.assertEqual(len(d), len(items), d) + self.assertEqual(d, dict(items)) + + def test_clear_reentrant_embedded(self): + # gh-130555: dict.clear() must be safe when values are embedded + # in an object and a destructor mutates the dict. + class MyObj: pass + class ClearOnDelete: + def __del__(self): + nonlocal x + del x + + x = MyObj() + x.a = ClearOnDelete() + + d = x.__dict__ + d.clear() + + def test_clear_reentrant_cycle(self): + # gh-130555: dict.clear() must be safe for embedded dicts when the + # object is part of a reference cycle and the last reference to the + # dict is via the cycle. + class MyObj: pass + obj = MyObj() + obj.f = obj + obj.attr = "attr" + + d = obj.__dict__ + del obj + + d.clear() + + def test_clear_reentrant_force_combined(self): + # gh-130555: dict.clear() must be safe when a destructor forces the + # dict from embedded/split to combined (setting ma_values to NULL). + class MyObj: pass + class ForceConvert: + def __del__(self): + d[1] = "trigger" + + x = MyObj() + x.a = ForceConvert() + x.b = "other" + + d = x.__dict__ + d.clear() + + def test_clear_reentrant_delete(self): + # gh-130555: dict.clear() must be safe when a destructor deletes + # a key from the same embedded dict. + class MyObj: pass + class DelKey: + def __del__(self): + try: + del d['b'] + except KeyError: + pass + + x = MyObj() + x.a = DelKey() + x.b = "value_b" + x.c = "value_c" + + d = x.__dict__ + d.clear() + class CAPITest(unittest.TestCase): diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index 691fc5dfb5e8ab9..9816ae6c033ec74 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -2,7 +2,9 @@ import copy import pickle import unittest -from test.support import skip_emscripten_stack_overflow, skip_wasi_stack_overflow, exceeds_recursion_limit +from test.support import (skip_emscripten_stack_overflow, + skip_wasi_stack_overflow, skip_if_huge_c_stack, + exceeds_recursion_limit) class DictSetTest(unittest.TestCase): @@ -277,6 +279,7 @@ def test_recursive_repr(self): # Again. self.assertIsInstance(r, str) + @skip_if_huge_c_stack() @skip_wasi_stack_overflow() @skip_emscripten_stack_overflow() def test_deeply_nested_repr(self): diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 9e217249be7332d..518b8c5278e4c47 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -29,6 +29,16 @@ def test_one_delete(self): ('delete', 40, 41, 40, 40), ('equal', 41, 81, 40, 80)]) + def test_opcode_caching(self): + sm = difflib.SequenceMatcher(None, 'b' * 100, 'a' + 'b' * 100) + opcode = sm.get_opcodes() + self.assertEqual(opcode, + [ ('insert', 0, 0, 0, 1), + ('equal', 0, 100, 1, 101)]) + # Implementation detail: opcodes are cached; + # `get_opcodes()` returns the same object + self.assertIs(opcode, sm.get_opcodes()) + def test_bjunk(self): sm = difflib.SequenceMatcher(isjunk=lambda x: x == ' ', a='a' * 40 + 'b' * 40, b='a' * 44 + 'b' * 40) @@ -272,6 +282,29 @@ def test_make_file_usascii_charset_with_nonascii_input(self): self.assertIn('content="text/html; charset=us-ascii"', output) self.assertIn('ımplıcıt', output) + def test_strip_trailing_newlines_before_diff(self): + # characterization test for the current buggy behavior + # see: gh-71896 + html_diff = difflib.HtmlDiff() + from_lines = [ + "Line 1: no newline after", + "Line 2: one newline after\n", + "Line 3: several newlines after\n\n\n\n\n", + ] + to_lines = [ + "Line 1: no newline after", + "Line 2: one newline after", # actually no \n + "Line 3: several newlines after", # actually no \n + ] + output = html_diff.make_table(from_lines, to_lines) + # we (currently) expect no line change, so all equal + self.assertNotIn('class="diff_add"', output) + self.assertNotIn('class="diff_chg"', output) + self.assertNotIn('class="diff_sub"', output) + self.assertEqual(output.count('>Line 1: no newline after<'), 2) + self.assertEqual(output.count('>Line 2: one newline after<'), 2) + self.assertEqual(output.count('>Line 3: several newlines after<'), 2) + class TestDiffer(unittest.TestCase): def test_close_matches_aligned(self): # Of the 4 closely matching pairs, we want 1 to match with 3, @@ -293,6 +326,15 @@ def test_close_matches_aligned(self): '+ kitten\n', '+ puppy\n']) + def test_one_insert(self): + m = difflib.Differ().compare('b' * 2, 'a' + 'b' * 2) + self.assertEqual(list(m), ['+ a', ' b', ' b']) + + def test_one_delete(self): + m = difflib.Differ().compare('a' + 'b' * 2, 'b' * 2) + self.assertEqual(list(m), ['- a', ' b', ' b']) + + class TestOutputFormat(unittest.TestCase): def test_tab_delimiter(self): args = [['one'], ['two'], 'Original', 'Current', @@ -585,6 +627,26 @@ def test_longest_match_with_popular_chars(self): self.assertFalse(self.longer_match_exists(a, b, match.size)) +class TestCloseMatches(unittest.TestCase): + # Happy paths are tested in the doctests of `difflib.get_close_matches`. + + def test_invalid_inputs(self): + self.assertRaises(ValueError, difflib.get_close_matches, "spam", ['egg'], n=0) + self.assertRaises(ValueError, difflib.get_close_matches, "spam", ['egg'], n=-1) + self.assertRaises(ValueError, difflib.get_close_matches, "spam", ['egg'], cutoff=1.1) + self.assertRaises(ValueError, difflib.get_close_matches, "spam", ['egg'], cutoff=-0.1) + + +class TestRestore(unittest.TestCase): + # Happy paths are tested in the doctests of `difflib.restore`. + + def test_invalid_input(self): + with self.assertRaises(ValueError): + ''.join(difflib.restore([], 0)) + with self.assertRaises(ValueError): + ''.join(difflib.restore([], 3)) + + def setUpModule(): difflib.HtmlDiff._default_prefix = 0 diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index ae68c1dd75c6415..48df7e530de2c1e 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -204,6 +204,7 @@ def bug1333982(x=[]): LOAD_CONST 1 ( at 0x..., file "%s", line %d>) MAKE_FUNCTION LOAD_FAST_BORROW 0 (x) + GET_ITER CALL 0 %3d LOAD_SMALL_INT 1 @@ -606,7 +607,7 @@ async def _asyncwith(c): POP_TOP L1: RESUME 0 -%4d LOAD_FAST_BORROW 0 (c) +%4d LOAD_FAST 0 (c) COPY 1 LOAD_SPECIAL 3 (__aexit__) SWAP 2 @@ -832,6 +833,7 @@ def foo(x): MAKE_FUNCTION SET_FUNCTION_ATTRIBUTE 8 (closure) LOAD_DEREF 1 (y) + GET_ITER CALL 0 CALL 1 RETURN_VALUE @@ -851,8 +853,7 @@ def foo(x): %4d RETURN_GENERATOR POP_TOP L1: RESUME 0 - LOAD_FAST_BORROW 0 (.0) - GET_ITER + LOAD_FAST 0 (.0) L2: FOR_ITER 14 (to L3) STORE_FAST 1 (z) LOAD_DEREF 2 (x) diff --git a/Lib/test/test_doctest/doctest_lineno.py b/Lib/test/test_doctest/doctest_lineno.py index 0dbcd9a11eaba2f..0bd402e98288d01 100644 --- a/Lib/test/test_doctest/doctest_lineno.py +++ b/Lib/test/test_doctest/doctest_lineno.py @@ -76,3 +76,32 @@ def property_with_doctest(self): @decorator def func_with_docstring_wrapped(): """Some unrelated info.""" + + +# https://github.com/python/cpython/issues/136914 +import functools + + +@functools.cache +def cached_func_with_doctest(value): + """ + >>> cached_func_with_doctest(1) + -1 + """ + return -value + + +@functools.cache +def cached_func_without_docstring(value): + return value + 1 + + +class ClassWithACachedProperty: + + @functools.cached_property + def cached(self): + """ + >>> X().cached + -1 + """ + return 0 diff --git a/Lib/test/test_doctest/sample_doctest_errors.py b/Lib/test/test_doctest/sample_doctest_errors.py new file mode 100644 index 000000000000000..4a6f07af2d4e5a0 --- /dev/null +++ b/Lib/test/test_doctest/sample_doctest_errors.py @@ -0,0 +1,46 @@ +"""This is a sample module used for testing doctest. + +This module includes various scenarios involving errors. + +>>> 2 + 2 +5 +>>> 1/0 +1 +""" + +def g(): + [][0] # line 12 + +def errors(): + """ + >>> 2 + 2 + 5 + >>> 1/0 + 1 + >>> def f(): + ... 2 + '2' + ... + >>> f() + 1 + >>> g() + 1 + """ + +def syntax_error(): + """ + >>> 2+*3 + 5 + """ + +__test__ = { + 'bad': """ + >>> 2 + 2 + 5 + >>> 1/0 + 1 + """, +} + +def test_suite(): + import doctest + return doctest.DocTestSuite() diff --git a/Lib/test/test_doctest/test_doctest.py b/Lib/test/test_doctest/test_doctest.py index a4a49298bab3be7..00e6126b61b86f1 100644 --- a/Lib/test/test_doctest/test_doctest.py +++ b/Lib/test/test_doctest/test_doctest.py @@ -678,6 +678,8 @@ def basics(): r""" >>> for t in tests: ... print('%5s %s' % (t.lineno, t.name)) None test.test_doctest.doctest_lineno + None test.test_doctest.doctest_lineno.ClassWithACachedProperty + 102 test.test_doctest.doctest_lineno.ClassWithACachedProperty.cached 22 test.test_doctest.doctest_lineno.ClassWithDocstring 30 test.test_doctest.doctest_lineno.ClassWithDoctest None test.test_doctest.doctest_lineno.ClassWithoutDocstring @@ -687,6 +689,8 @@ def basics(): r""" 45 test.test_doctest.doctest_lineno.MethodWrapper.method_with_doctest None test.test_doctest.doctest_lineno.MethodWrapper.method_without_docstring 61 test.test_doctest.doctest_lineno.MethodWrapper.property_with_doctest + 86 test.test_doctest.doctest_lineno.cached_func_with_doctest + None test.test_doctest.doctest_lineno.cached_func_without_docstring 4 test.test_doctest.doctest_lineno.func_with_docstring 77 test.test_doctest.doctest_lineno.func_with_docstring_wrapped 12 test.test_doctest.doctest_lineno.func_with_doctest @@ -2267,13 +2271,21 @@ def test_DocTestSuite(): >>> import unittest >>> import test.test_doctest.sample_doctest >>> suite = doctest.DocTestSuite(test.test_doctest.sample_doctest) - >>> suite.run(unittest.TestResult()) + >>> result = suite.run(unittest.TestResult()) + >>> result + >>> for tst, _ in result.failures: + ... print(tst) + bad (test.test_doctest.sample_doctest.__test__) + foo (test.test_doctest.sample_doctest) + test_silly_setup (test.test_doctest.sample_doctest) + y_is_one (test.test_doctest.sample_doctest) We can also supply the module by name: >>> suite = doctest.DocTestSuite('test.test_doctest.sample_doctest') - >>> suite.run(unittest.TestResult()) + >>> result = suite.run(unittest.TestResult()) + >>> result The module need not contain any doctest examples: @@ -2297,6 +2309,14 @@ def test_DocTestSuite(): >>> len(result.skipped) 2 + >>> for tst, _ in result.skipped: + ... print(tst) + double_skip (test.test_doctest.sample_doctest_skip) + single_skip (test.test_doctest.sample_doctest_skip) + >>> for tst, _ in result.failures: + ... print(tst) + no_skip_fail (test.test_doctest.sample_doctest_skip) + partial_skip_fail (test.test_doctest.sample_doctest_skip) We can use the current module: @@ -2383,7 +2403,127 @@ def test_DocTestSuite(): modified the test globals, which are a copy of the sample_doctest module dictionary. The test globals are automatically cleared for us after a test. - """ + """ + +def test_DocTestSuite_errors(): + """Tests for error reporting in DocTestSuite. + + >>> import unittest + >>> import test.test_doctest.sample_doctest_errors as mod + >>> suite = doctest.DocTestSuite(mod) + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS + AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors + File "...sample_doctest_errors.py", line 0, in sample_doctest_errors + + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line 5, in test.test_doctest.sample_doctest_errors + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line 7, in test.test_doctest.sample_doctest_errors + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + + >>> print(result.failures[1][1]) # doctest: +ELLIPSIS + AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors.__test__.bad + File "...sample_doctest_errors.py", line unknown line number, in bad + + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + + >>> print(result.failures[2][1]) # doctest: +ELLIPSIS + AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors.errors + File "...sample_doctest_errors.py", line 14, in errors + + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line 16, in test.test_doctest.sample_doctest_errors.errors + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line 18, in test.test_doctest.sample_doctest_errors.errors + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line 23, in test.test_doctest.sample_doctest_errors.errors + Failed example: + f() + Exception raised: + Traceback (most recent call last): + File "", line 1, in + f() + ~^^ + File "", line 2, in f + 2 + '2' + ~~^~~~~ + TypeError: ... + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line 25, in test.test_doctest.sample_doctest_errors.errors + Failed example: + g() + Exception raised: + Traceback (most recent call last): + File "", line 1, in + g() + ~^^ + File "...sample_doctest_errors.py", line 12, in g + [][0] # line 12 + ~~^^^ + IndexError: list index out of range + + >>> print(result.failures[3][1]) # doctest: +ELLIPSIS + AssertionError: Failed doctest test for test.test_doctest.sample_doctest_errors.syntax_error + File "...sample_doctest_errors.py", line 29, in syntax_error + + ---------------------------------------------------------------------- + File "...sample_doctest_errors.py", line 31, in test.test_doctest.sample_doctest_errors.syntax_error + Failed example: + 2+*3 + Exception raised: + File "", line 1 + 2+*3 + ^ + SyntaxError: invalid syntax + + """ def test_DocFileSuite(): """We can test tests found in text files using a DocFileSuite. @@ -2455,12 +2595,16 @@ def test_DocFileSuite(): >>> suite = doctest.DocFileSuite('test_doctest.txt', ... 'test_doctest4.txt', - ... 'test_doctest_skip.txt') + ... 'test_doctest_skip.txt', + ... 'test_doctest_skip2.txt') >>> result = suite.run(unittest.TestResult()) >>> result - - >>> len(result.skipped) - 1 + + >>> len(result.skipped) + 1 + >>> for tst, _ in result.skipped: # doctest: +ELLIPSIS + ... print('=', tst) + = ...test_doctest_skip.txt You can specify initial global variables: @@ -2542,8 +2686,62 @@ def test_DocFileSuite(): ... encoding='utf-8') >>> suite.run(unittest.TestResult()) + """ - """ +def test_DocFileSuite_errors(): + """Tests for error reporting in DocTestSuite. + + >>> import unittest + >>> suite = doctest.DocFileSuite('test_doctest_errors.txt') + >>> result = suite.run(unittest.TestResult()) + >>> result + + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS + AssertionError: Failed doctest test for test_doctest_errors.txt + File "...test_doctest_errors.txt", line 0 + + ---------------------------------------------------------------------- + File "...test_doctest_errors.txt", line 4, in test_doctest_errors.txt + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ---------------------------------------------------------------------- + File "...test_doctest_errors.txt", line 6, in test_doctest_errors.txt + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ---------------------------------------------------------------------- + File "...test_doctest_errors.txt", line 11, in test_doctest_errors.txt + Failed example: + f() + Exception raised: + Traceback (most recent call last): + File "", line 1, in + f() + ~^^ + File "", line 2, in f + 2 + '2' + ~~^~~~~ + TypeError: ... + ---------------------------------------------------------------------- + File "...test_doctest_errors.txt", line 13, in test_doctest_errors.txt + Failed example: + 2+*3 + Exception raised: + File "", line 1 + 2+*3 + ^ + SyntaxError: invalid syntax + + """ def test_trailing_space_in_test(): """ @@ -2612,8 +2810,11 @@ def test_unittest_reportflags(): ... optionflags=doctest.DONT_ACCEPT_BLANKLINE) >>> import unittest >>> result = suite.run(unittest.TestResult()) + >>> result + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS - Traceback ... + AssertionError: Failed doctest test for test_doctest.txt + ... Failed example: favorite_color ... @@ -2629,15 +2830,17 @@ def test_unittest_reportflags(): Now, when we run the test: >>> result = suite.run(unittest.TestResult()) + >>> result + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS - Traceback ... + AssertionError: Failed doctest test for test_doctest.txt + ... Failed example: favorite_color Exception raised: ... NameError: name 'favorite_color' is not defined - We get only the first failure. @@ -2650,12 +2853,15 @@ def test_unittest_reportflags(): Then the default eporting options are ignored: >>> result = suite.run(unittest.TestResult()) + >>> result + *NOTE*: These doctest are intentionally not placed in raw string to depict the trailing whitespace using `\x20` in the diff below. >>> print(result.failures[0][1]) # doctest: +ELLIPSIS - Traceback ... + AssertionError: Failed doctest test for test_doctest.txt + ... Failed example: favorite_color ... @@ -2670,7 +2876,6 @@ def test_unittest_reportflags(): +\x20 b - Test runners can restore the formatting flags after they run: @@ -2860,6 +3065,57 @@ def test_testfile(): r""" >>> _colorize.COLORIZE = save_colorize """ +def test_testfile_errors(): r""" +Tests for error reporting in the testfile() function. + + >>> doctest.testfile('test_doctest_errors.txt', verbose=False) # doctest: +ELLIPSIS + ********************************************************************** + File "...test_doctest_errors.txt", line 4, in test_doctest_errors.txt + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ********************************************************************** + File "...test_doctest_errors.txt", line 6, in test_doctest_errors.txt + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ********************************************************************** + File "...test_doctest_errors.txt", line 11, in test_doctest_errors.txt + Failed example: + f() + Exception raised: + Traceback (most recent call last): + File "", line 1, in + f() + ~^^ + File "", line 2, in f + 2 + '2' + ~~^~~~~ + TypeError: ... + ********************************************************************** + File "...test_doctest_errors.txt", line 13, in test_doctest_errors.txt + Failed example: + 2+*3 + Exception raised: + File "", line 1 + 2+*3 + ^ + SyntaxError: invalid syntax + ********************************************************************** + 1 item had failures: + 4 of 5 in test_doctest_errors.txt + ***Test Failed*** 4 failures. + TestResults(failed=4, attempted=5) +""" + class TestImporter(importlib.abc.MetaPathFinder): def find_spec(self, fullname, path, target=None): @@ -2990,6 +3246,110 @@ def test_testmod(): r""" TestResults(failed=0, attempted=0) """ +def test_testmod_errors(): r""" +Tests for error reporting in the testmod() function. + + >>> import test.test_doctest.sample_doctest_errors as mod + >>> doctest.testmod(mod, verbose=False) # doctest: +ELLIPSIS + ********************************************************************** + File "...sample_doctest_errors.py", line 5, in test.test_doctest.sample_doctest_errors + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ********************************************************************** + File "...sample_doctest_errors.py", line 7, in test.test_doctest.sample_doctest_errors + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ********************************************************************** + File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ********************************************************************** + File "...sample_doctest_errors.py", line ?, in test.test_doctest.sample_doctest_errors.__test__.bad + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ********************************************************************** + File "...sample_doctest_errors.py", line 16, in test.test_doctest.sample_doctest_errors.errors + Failed example: + 2 + 2 + Expected: + 5 + Got: + 4 + ********************************************************************** + File "...sample_doctest_errors.py", line 18, in test.test_doctest.sample_doctest_errors.errors + Failed example: + 1/0 + Exception raised: + Traceback (most recent call last): + File "", line 1, in + 1/0 + ~^~ + ZeroDivisionError: division by zero + ********************************************************************** + File "...sample_doctest_errors.py", line 23, in test.test_doctest.sample_doctest_errors.errors + Failed example: + f() + Exception raised: + Traceback (most recent call last): + File "", line 1, in + f() + ~^^ + File "", line 2, in f + 2 + '2' + ~~^~~~~ + TypeError: ... + ********************************************************************** + File "...sample_doctest_errors.py", line 25, in test.test_doctest.sample_doctest_errors.errors + Failed example: + g() + Exception raised: + Traceback (most recent call last): + File "", line 1, in + g() + ~^^ + File "...sample_doctest_errors.py", line 12, in g + [][0] # line 12 + ~~^^^ + IndexError: list index out of range + ********************************************************************** + File "...sample_doctest_errors.py", line 31, in test.test_doctest.sample_doctest_errors.syntax_error + Failed example: + 2+*3 + Exception raised: + File "", line 1 + 2+*3 + ^ + SyntaxError: invalid syntax + ********************************************************************** + 4 items had failures: + 2 of 2 in test.test_doctest.sample_doctest_errors + 2 of 2 in test.test_doctest.sample_doctest_errors.__test__.bad + 4 of 5 in test.test_doctest.sample_doctest_errors.errors + 1 of 1 in test.test_doctest.sample_doctest_errors.syntax_error + ***Test Failed*** 9 failures. + TestResults(failed=9, attempted=10) +""" + try: os.fsencode("foo-bär@baz.py") supports_unicode = True @@ -3021,11 +3381,6 @@ def test_unicode(): """ raise Exception('clé') Exception raised: Traceback (most recent call last): - File ... - exec(compile(example.source, filename, "single", - ~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - compileflags, True), test.globs) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "", line 1, in raise Exception('clé') Exception: clé diff --git a/Lib/test/test_doctest/test_doctest_errors.txt b/Lib/test/test_doctest/test_doctest_errors.txt new file mode 100644 index 000000000000000..93c3c106e60b32e --- /dev/null +++ b/Lib/test/test_doctest/test_doctest_errors.txt @@ -0,0 +1,14 @@ +This is a sample doctest in a text file, in which all examples fail +or raise an exception. + + >>> 2 + 2 + 5 + >>> 1/0 + 1 + >>> def f(): + ... 2 + '2' + ... + >>> f() + 1 + >>> 2+*3 + 5 diff --git a/Lib/test/test_doctest/test_doctest_skip.txt b/Lib/test/test_doctest/test_doctest_skip.txt index f340e2b81412539..06c23d06e606a38 100644 --- a/Lib/test/test_doctest/test_doctest_skip.txt +++ b/Lib/test/test_doctest/test_doctest_skip.txt @@ -2,3 +2,5 @@ This is a sample doctest in a text file, in which all examples are skipped. >>> 2 + 2 # doctest: +SKIP 5 + >>> 2 + 2 # doctest: +SKIP + 4 diff --git a/Lib/test/test_doctest/test_doctest_skip2.txt b/Lib/test/test_doctest/test_doctest_skip2.txt new file mode 100644 index 000000000000000..85e4938c346a093 --- /dev/null +++ b/Lib/test/test_doctest/test_doctest_skip2.txt @@ -0,0 +1,6 @@ +This is a sample doctest in a text file, in which some examples are skipped. + + >>> 2 + 2 # doctest: +SKIP + 5 + >>> 2 + 2 + 4 diff --git a/Lib/test/test_dtrace.py b/Lib/test/test_dtrace.py index e1adf8e97485067..d2ff10c57a0982c 100644 --- a/Lib/test/test_dtrace.py +++ b/Lib/test/test_dtrace.py @@ -8,11 +8,15 @@ import unittest from test import support -from test.support import findfile +from test.support import findfile, MS_WINDOWS if not support.has_subprocess_support: raise unittest.SkipTest("test module requires subprocess") +if not sysconfig.get_config_var('WITH_DTRACE'): + raise unittest.SkipTest( + "CPython must be configured with the --with-dtrace option." + ) def abspath(filename): @@ -103,6 +107,7 @@ class SystemTapBackend(TraceBackend): COMMAND = ["stap", "-g"] +@unittest.skipIf(MS_WINDOWS, "Tests not compliant with trace on Windows.") class TraceTests: # unittest.TestCase options maxDiff = None @@ -177,12 +182,9 @@ class SystemTapOptimizedTests(TraceTests, unittest.TestCase): class CheckDtraceProbes(unittest.TestCase): @classmethod def setUpClass(cls): - if sysconfig.get_config_var('WITH_DTRACE'): - readelf_major_version, readelf_minor_version = cls.get_readelf_version() - if support.verbose: - print(f"readelf version: {readelf_major_version}.{readelf_minor_version}") - else: - raise unittest.SkipTest("CPython must be configured with the --with-dtrace option.") + readelf_major_version, readelf_minor_version = cls.get_readelf_version() + if support.verbose: + print(f"readelf version: {readelf_major_version}.{readelf_minor_version}") @staticmethod diff --git a/Lib/test/test_dynamicclassattribute.py b/Lib/test/test_dynamicclassattribute.py index 9f694d9eb46771f..b19be33c72f94b0 100644 --- a/Lib/test/test_dynamicclassattribute.py +++ b/Lib/test/test_dynamicclassattribute.py @@ -104,8 +104,8 @@ def test_property_decorator_baseclass(self): self.assertEqual(base.spam, 10) self.assertEqual(base._spam, 10) delattr(base, "spam") - self.assertTrue(not hasattr(base, "spam")) - self.assertTrue(not hasattr(base, "_spam")) + self.assertNotHasAttr(base, "spam") + self.assertNotHasAttr(base, "_spam") base.spam = 20 self.assertEqual(base.spam, 20) self.assertEqual(base._spam, 20) diff --git a/Lib/test/test_email/data/msg_35.txt b/Lib/test/test_email/data/msg_35.txt index be7d5a2f7b9d383..0e2bbcaf71816a3 100644 --- a/Lib/test/test_email/data/msg_35.txt +++ b/Lib/test/test_email/data/msg_35.txt @@ -1,4 +1,4 @@ From: aperson@dom.ain To: bperson@dom.ain Subject: here's something interesting -counter to RFC 2822, there's no separating newline here +counter to RFC 5322, there's no separating newline here diff --git a/Lib/test/test_email/test__header_value_parser.py b/Lib/test/test_email/test__header_value_parser.py index ac12c3b2306f7d7..8c9601288da8f8b 100644 --- a/Lib/test/test_email/test__header_value_parser.py +++ b/Lib/test/test_email/test__header_value_parser.py @@ -463,6 +463,19 @@ def test_get_qp_ctext_non_printables(self): [errors.NonPrintableDefect], ')') self.assertEqual(ptext.defects[0].non_printables[0], '\x00') + def test_get_qp_ctext_close_paren_only(self): + self._test_get_x(parser.get_qp_ctext, + ')', '', ' ', [], ')') + + def test_get_qp_ctext_open_paren_only(self): + self._test_get_x(parser.get_qp_ctext, + '(', '', ' ', [], '(') + + def test_get_qp_ctext_no_end_char(self): + self._test_get_x(parser.get_qp_ctext, + '', '', ' ', [], '') + + # get_qcontent def test_get_qcontent_only(self): @@ -503,6 +516,14 @@ def test_get_qcontent_non_printables(self): [errors.NonPrintableDefect], '"') self.assertEqual(ptext.defects[0].non_printables[0], '\x00') + def test_get_qcontent_empty(self): + self._test_get_x(parser.get_qcontent, + '"', '', '', [], '"') + + def test_get_qcontent_no_end_char(self): + self._test_get_x(parser.get_qcontent, + '', '', '', [], '') + # get_atext def test_get_atext_only(self): @@ -1039,6 +1060,78 @@ def get_phrase_cfws_only_raises(self): with self.assertRaises(errors.HeaderParseError): parser.get_phrase(' (foo) ') + def test_get_phrase_adjacent_ew(self): + # "'linear-white-space' that separates a pair of adjacent + # 'encoded-word's is ignored" (rfc2047 section 6.2) + self._test_get_x(parser.get_phrase, '=?ascii?q?Joi?= \t =?ascii?q?ned?=', 'Joined', 'Joined', [], '') + + def test_get_phrase_adjacent_ew_different_encodings(self): + self._test_get_x( + parser.get_phrase, + '=?utf-8?q?B=C3=A9r?= =?iso-8859-1?q?=E9nice?=', 'Bérénice', 'Bérénice', [], '' + ) + + def test_get_phrase_adjacent_ew_encoded_spaces(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Encoded?= =?ascii?q?_spaces_?= =?ascii?q?preserved?=', + 'Encoded spaces preserved', + 'Encoded spaces preserved', + [], + '' + ) + + def test_get_phrase_adjacent_ew_comment_is_not_linear_white_space(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Comment?= (is not) =?ascii?q?linear-white-space?=', + 'Comment (is not) linear-white-space', + 'Comment linear-white-space', + [], + '', + comments=['is not'], + ) + + def test_get_phrase_adjacent_ew_no_error_on_defects(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Def?= =?ascii?q?ect still joins?=', + 'Defect still joins', + 'Defect still joins', + [errors.InvalidHeaderDefect], # whitespace inside encoded word + '' + ) + + def test_get_phrase_adjacent_ew_ignore_non_ew(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?No?= =?join?= for non-ew', + 'No =?join?= for non-ew', + 'No =?join?= for non-ew', + [], + '' + ) + + def test_get_phrase_adjacent_ew_ignore_invalid_ew(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?No?= =?ascii?rot13?wbva= for invalid ew', + 'No =?ascii?rot13?wbva= for invalid ew', + 'No =?ascii?rot13?wbva= for invalid ew', + [], + '' + ) + + def test_get_phrase_adjacent_ew_missing_space(self): + self._test_get_x( + parser.get_phrase, + '=?ascii?q?Joi?==?ascii?q?ned?=', + 'Joined', + 'Joined', + [errors.InvalidHeaderDefect], # missing trailing whitespace + '' + ) + # get_local_part def test_get_local_part_simple(self): @@ -1283,6 +1376,18 @@ def test_get_dtext_open_bracket_mid_word(self): self._test_get_x(parser.get_dtext, 'foo[bar', 'foo', 'foo', [], '[bar') + def test_get_dtext_open_bracket_only(self): + self._test_get_x(parser.get_dtext, + '[', '', '', [], '[') + + def test_get_dtext_close_bracket_only(self): + self._test_get_x(parser.get_dtext, + ']', '', '', [], ']') + + def test_get_dtext_empty(self): + self._test_get_x(parser.get_dtext, + '', '', '', [], '') + # get_domain_literal def test_get_domain_literal_only(self): @@ -2365,6 +2470,22 @@ def test_get_address_rfc2047_display_name(self): self.assertEqual(address[0].token_type, 'mailbox') + def test_get_address_rfc2047_display_name_adjacent_ews(self): + address = self._test_get_x(parser.get_address, + '=?utf-8?q?B=C3=A9r?= =?utf-8?q?=C3=A9nice?= ', + 'Bérénice ', + 'Bérénice ', + [], + '') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 1) + self.assertEqual(address.mailboxes, + address.all_mailboxes) + self.assertEqual(address.mailboxes[0].display_name, + 'Bérénice') + self.assertEqual(address[0].token_type, + 'mailbox') + def test_get_address_empty_group(self): address = self._test_get_x(parser.get_address, 'Monty Python:;', @@ -2458,6 +2579,38 @@ def test_get_address_quoted_strings_in_atom_list(self): self.assertEqual(address.all_mailboxes[0].domain, 'example.com') self.assertEqual(address.all_mailboxes[0].addr_spec, '"example example"@example.com') + def test_get_address_with_invalid_domain(self): + address = self._test_get_x(parser.get_address, + '', + '', + [errors.InvalidHeaderDefect, # missing trailing '>' on angle-addr + errors.InvalidHeaderDefect, # end of input inside domain-literal + ], + '') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 0) + self.assertEqual(len(address.all_mailboxes), 1) + self.assertEqual(address.all_mailboxes[0].domain, '[]') + self.assertEqual(address.all_mailboxes[0].local_part, 'T') + self.assertEqual(address.all_mailboxes[0].token_type, 'invalid-mailbox') + self.assertEqual(address[0].token_type, 'invalid-mailbox') + + address = self._test_get_x(parser.get_address, + '!an??:=m==fr2@[C', + '!an??:=m==fr2@[C];', + '!an??:=m==fr2@[C];', + [errors.InvalidHeaderDefect, # end of header in group + errors.InvalidHeaderDefect, # end of input inside domain-literal + ], + '') + self.assertEqual(address.token_type, 'address') + self.assertEqual(len(address.mailboxes), 0) + self.assertEqual(len(address.all_mailboxes), 1) + self.assertEqual(address.all_mailboxes[0].domain, '[C]') + self.assertEqual(address.all_mailboxes[0].local_part, '=m==fr2') + self.assertEqual(address.all_mailboxes[0].token_type, 'invalid-mailbox') + self.assertEqual(address[0].token_type, 'group') # get_address_list @@ -2552,7 +2705,7 @@ def test_get_address_list_mailboxes_invalid_addresses(self): '') self.assertEqual(address_list.token_type, 'address-list') self.assertEqual(len(address_list.mailboxes), 1) - self.assertEqual(len(address_list.all_mailboxes), 3) + self.assertEqual(len(address_list.all_mailboxes), 4) self.assertEqual([str(x) for x in address_list.all_mailboxes], [str(x) for x in address_list.addresses]) self.assertEqual(address_list.mailboxes[0].domain, 'example.com') @@ -2561,11 +2714,13 @@ def test_get_address_list_mailboxes_invalid_addresses(self): self.assertEqual(address_list.addresses[1].token_type, 'address') self.assertEqual(len(address_list.addresses[0].mailboxes), 1) self.assertEqual(len(address_list.addresses[1].mailboxes), 0) - self.assertEqual(len(address_list.addresses[1].mailboxes), 0) + self.assertEqual(len(address_list.addresses[2].mailboxes), 0) + self.assertEqual(len(address_list.addresses[3].mailboxes), 0) self.assertEqual( address_list.addresses[1].all_mailboxes[0].local_part, 'Foo x') + self.assertEqual(address_list.addresses[2].all_mailboxes[0].value, '[]') self.assertEqual( - address_list.addresses[2].all_mailboxes[0].display_name, + address_list.addresses[3].all_mailboxes[0].display_name, "Nobody Is. Special") def test_get_address_list_group_empty(self): @@ -2630,6 +2785,14 @@ def test_get_address_list_group_and_mailboxes(self): self.assertEqual(str(address_list.addresses[1]), str(address_list.mailboxes[2])) + def test_get_address_list_trailing_garbage(self): + address_list = self._test_get_x(parser.get_address_list, + 'unlisted-recipients:; (no To-header on input)', + 'unlisted-recipients:; (no To-header on input)', + 'unlisted-recipients:; ', + [errors.InvalidHeaderDefect]*2 + [errors.ObsoleteHeaderDefect], + '') + def test_invalid_content_disposition(self): content_disp = self._test_parse_x( parser.parse_content_disposition_header, @@ -2732,6 +2895,19 @@ def test_parse_valid_message_id(self): ) self.assertEqual(message_id.token_type, 'message-id') + def test_parse_message_id_with_invalid_domain(self): + message_id = self._test_parse_x( + parser.parse_message_id, + "", + "", + [errors.ObsoleteHeaderDefect] + [errors.InvalidHeaderDefect] * 2, + [], + ) + self.assertEqual(message_id.token_type, 'message-id') + self.assertEqual(str(message_id.all_defects[-1]), + "end of input inside domain-literal") + def test_parse_message_id_with_remaining(self): message_id = self._test_parse_x( parser.parse_message_id, @@ -2789,6 +2965,81 @@ def test_get_msg_id_ws_only_local(self): ) self.assertEqual(msg_id.token_type, 'msg-id') + def test_parse_message_ids_valid(self): + message_ids = self._test_parse_x( + parser.parse_message_ids, + " ", + " ", + " ", + [], + ) + self.assertEqual(message_ids.token_type, 'message-id-list') + + def test_parse_message_ids_empty(self): + message_ids = self._test_parse_x( + parser.parse_message_ids, + " ", + " ", + " ", + [errors.InvalidHeaderDefect], + ) + self.assertEqual(message_ids.token_type, 'message-id-list') + + def test_parse_message_ids_comment(self): + message_ids = self._test_parse_x( + parser.parse_message_ids, + " (foo's message from \"bar\")", + " (foo's message from \"bar\")", + " ", + [], + ) + self.assertEqual(message_ids.message_ids[0].value, ' ') + self.assertEqual(message_ids.token_type, 'message-id-list') + + def test_parse_message_ids_no_sep(self): + message_ids = self._test_parse_x( + parser.parse_message_ids, + "", + "", + "", + [], + ) + self.assertEqual(message_ids.message_ids[0].value, '') + self.assertEqual(message_ids.message_ids[1].value, '') + self.assertEqual(message_ids.token_type, 'message-id-list') + + def test_parse_message_ids_comma_sep(self): + message_ids = self._test_parse_x( + parser.parse_message_ids, + ",", + " ", + " ", + [errors.InvalidHeaderDefect], + ) + self.assertEqual(message_ids.message_ids[0].value, '') + self.assertEqual(message_ids.message_ids[1].value, '') + self.assertEqual(message_ids.token_type, 'message-id-list') + + def test_parse_message_ids_invalid_id(self): + message_ids = self._test_parse_x( + parser.parse_message_ids, + "", + "", + "", + [errors.InvalidHeaderDefect]*2, + ) + self.assertEqual(message_ids.token_type, 'message-id-list') + + def test_parse_message_ids_broken_ang(self): + message_ids = self._test_parse_x( + parser.parse_message_ids, + " >bar@foo", + " >bar@foo", + " >bar@foo", + [errors.InvalidHeaderDefect]*1, + ) + self.assertEqual(message_ids.token_type, 'message-id-list') + @parameterize @@ -3141,6 +3392,29 @@ def test_address_list_with_specials_in_long_quoted_string(self): with self.subTest(to=to): self._test(parser.get_address_list(to)[0], folded, policy=policy) + def test_address_list_with_long_unwrapable_comment(self): + policy = self.policy.clone(max_line_length=40) + cases = [ + # (to, folded) + ('(loremipsumdolorsitametconsecteturadipi)', + '(loremipsumdolorsitametconsecteturadipi)\n'), + ('(loremipsumdolorsitametconsecteturadipi)', + '(loremipsumdolorsitametconsecteturadipi)\n'), + ('(loremipsum dolorsitametconsecteturadipi)', + '(loremipsum dolorsitametconsecteturadipi)\n'), + ('(loremipsum dolorsitametconsecteturadipi)', + '(loremipsum\n dolorsitametconsecteturadipi)\n'), + ('(Escaped \\( \\) chars \\\\ in comments stay escaped)', + '(Escaped \\( \\) chars \\\\ in comments stay\n escaped)\n'), + ('((loremipsum)(loremipsum)(loremipsum)(loremipsum))', + '((loremipsum)(loremipsum)(loremipsum)(loremipsum))\n'), + ('((loremipsum)(loremipsum)(loremipsum) (loremipsum))', + '((loremipsum)(loremipsum)(loremipsum)\n (loremipsum))\n'), + ] + for (to, folded) in cases: + with self.subTest(to=to): + self._test(parser.get_address_list(to)[0], folded, policy=policy) + # XXX Need tests with comments on various sides of a unicode token, # and with unicode tokens in the comments. Spaces inside the quotes # currently don't do the right thing. @@ -3177,5 +3451,23 @@ def test_long_filename_attachment(self): " filename*1*=_TEST_TES.txt\n", ) + def test_fold_unfoldable_element_stealing_whitespace(self): + # gh-142006: When an element is too long to fit on the current line + # the previous line's trailing whitespace should not trigger a double newline. + policy = self.policy.clone(max_line_length=10) + # The non-whitespace text needs to exactly fill the max_line_length (10). + text = ("a" * 9) + ", " + ("b" * 20) + expected = ("a" * 9) + ",\n " + ("b" * 20) + "\n" + token = parser.get_address_list(text)[0] + self._test(token, expected, policy=policy) + + def test_encoded_word_with_undecodable_bytes(self): + self._test(parser.get_address_list( + ' =?utf-8?Q?=E5=AE=A2=E6=88=B6=E6=AD=A3=E8=A6=8F=E4=BA=A4=E7?=' + )[0], + ' =?unknown-8bit?b?5a6i5oi25q2j6KaP5Lqk5w==?=\n', + ) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_email/test_asian_codecs.py b/Lib/test/test_email/test_asian_codecs.py index ca44f54c69b39bc..85979ffd8169a75 100644 --- a/Lib/test/test_email/test_asian_codecs.py +++ b/Lib/test/test_email/test_asian_codecs.py @@ -58,6 +58,62 @@ def test_japanese_codecs(self): # TK: full decode comparison eq(str(h).encode(jcode), subject_bytes) + h = Header("Japanese") + s = '\u65e5\u672c\u8a9e' # 日本語 + h.append(s, Charset('euc-jp')) + h.append(s, Charset('iso-2022-jp')) + h.append(s, Charset('shift_jis')) + eq(h.encode(), """\ +Japanese =?iso-2022-jp?b?GyRCRnxLXDhsGyhC?= =?iso-2022-jp?b?GyRCRnxLXDhsGyhC?= + =?iso-2022-jp?b?GyRCRnxLXDhsGyhC?=""") + eq(decode_header(h.encode()), + [(b'Japanese ', None), + (b'\x1b$BF|K\\8l\x1b(B\x1b$BF|K\\8l\x1b(B\x1b$BF|K\\8l\x1b(B', 'iso-2022-jp'), + ]) + + def test_chinese_codecs(self): + eq = self.ndiffAssertEqual + h = Header("Chinese") + s = '\u4e2d\u6587' # 中文 + h.append(s, Charset('gb2312')) + h.append(s, Charset('gbk')) + h.append(s, Charset('gb18030')) + h.append(s, Charset('hz')) + h.append(s, Charset('big5')) + h.append(s, Charset('big5hkscs')) + eq(h.encode(), """\ +Chinese =?gb2312?b?1tDOxA==?= =?gbk?b?1tDOxA==?= =?gb18030?b?1tDOxA==?= + =?hz?b?fntWUE5Efn0=?= =?big5?b?pKSk5Q==?= =?big5hkscs?b?pKSk5Q==?=""") + eq(decode_header(h.encode()), + [(b'Chinese ', None), + (b'\xd6\xd0\xce\xc4', 'gb2312'), + (b'\xd6\xd0\xce\xc4', 'gbk'), + (b'\xd6\xd0\xce\xc4', 'gb18030'), + (b'~{VPND~}', 'hz'), + (b'\xa4\xa4\xa4\xe5', 'big5'), + (b'\xa4\xa4\xa4\xe5', 'big5hkscs'), + ]) + + def test_korean_codecs(self): + eq = self.ndiffAssertEqual + h = Header("Korean") + s = '\ud55c\uad6d\uc5b4' # 한국어 + h.append(s, Charset('euc-kr')) + h.append(s, Charset('ks_c_5601-1987')) + h.append(s, Charset('cp949')) + h.append(s, Charset('iso-2022-kr')) + h.append(s, Charset('johab')) + eq(h.encode(), """\ +Korean =?euc-kr?b?x9Gxub7u?= =?ks_c_5601-1987?b?x9Gxub7uIMfRsbm+7g==?= + =?iso-2022-kr?b?GyQpQw5HUTE5Pm4P?= =?johab?b?0GWKgrTh?=""") + eq(decode_header(h.encode()), + [(b'Korean ', None), + (b'\xc7\xd1\xb1\xb9\xbe\xee', 'euc-kr'), + (b'\xc7\xd1\xb1\xb9\xbe\xee \xc7\xd1\xb1\xb9\xbe\xee', 'ks_c_5601-1987'), + (b'\x1b$)C\x0eGQ19>n\x0f', 'iso-2022-kr'), + (b'\xd0e\x8a\x82\xb4\xe1', 'johab'), + ]) + def test_payload_encoding_utf8(self): jhello = str(b'\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc' b'\xa5\xeb\xa5\xc9\xa1\xaa', 'euc-jp') diff --git a/Lib/test/test_email/test_defect_handling.py b/Lib/test/test_email/test_defect_handling.py index 44e76c8ce5e03a8..acc4accccac7566 100644 --- a/Lib/test/test_email/test_defect_handling.py +++ b/Lib/test/test_email/test_defect_handling.py @@ -126,12 +126,10 @@ def test_multipart_invalid_cte(self): errors.InvalidMultipartContentTransferEncodingDefect) def test_multipart_no_cte_no_defect(self): - if self.raise_expected: return msg = self._str_msg(self.multipart_msg.format('')) self.assertEqual(len(self.get_defects(msg)), 0) def test_multipart_valid_cte_no_defect(self): - if self.raise_expected: return for cte in ('7bit', '8bit', 'BINary'): msg = self._str_msg( self.multipart_msg.format("\nContent-Transfer-Encoding: "+cte)) @@ -300,6 +298,47 @@ def test_missing_ending_boundary(self): self.assertDefectsEqual(self.get_defects(msg), [errors.CloseBoundaryNotFoundDefect]) + def test_line_beginning_colon(self): + string = ( + "Subject: Dummy subject\r\n: faulty header line\r\n\r\nbody\r\n" + ) + + with self._raise_point(errors.InvalidHeaderDefect): + msg = self._str_msg(string) + self.assertEqual(len(self.get_defects(msg)), 1) + self.assertDefectsEqual( + self.get_defects(msg), [errors.InvalidHeaderDefect] + ) + + if msg: + self.assertEqual(msg.items(), [("Subject", "Dummy subject")]) + self.assertEqual(msg.get_payload(), "body\r\n") + + def test_misplaced_envelope(self): + string = ( + "Subject: Dummy subject\r\nFrom wtf\r\nTo: abc\r\n\r\nbody\r\n" + ) + with self._raise_point(errors.MisplacedEnvelopeHeaderDefect): + msg = self._str_msg(string) + self.assertEqual(len(self.get_defects(msg)), 1) + self.assertDefectsEqual( + self.get_defects(msg), [errors.MisplacedEnvelopeHeaderDefect] + ) + + if msg: + headers = [("Subject", "Dummy subject"), ("To", "abc")] + self.assertEqual(msg.items(), headers) + self.assertEqual(msg.get_payload(), "body\r\n") + + + +class TestCompat32(TestDefectsBase, TestEmailBase): + + policy = policy.compat32 + + def get_defects(self, obj): + return obj.defects + class TestDefectDetection(TestDefectsBase, TestEmailBase): @@ -332,6 +371,9 @@ def _raise_point(self, defect): with self.assertRaises(defect): yield + def get_defects(self, obj): + return obj.defects + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py index 7b14305f997e5d9..3d5d2015ee01aa6 100644 --- a/Lib/test/test_email/test_email.py +++ b/Lib/test/test_email/test_email.py @@ -389,6 +389,24 @@ def test_bad_param(self): msg = email.message_from_string("Content-Type: blarg; baz; boo\n") self.assertEqual(msg.get_param('baz'), '') + def test_continuation_sorting_part_order(self): + msg = email.message_from_string( + "Content-Disposition: attachment; " + "filename*=\"ignored\"; " + "filename*0*=\"utf-8''foo%20\"; " + "filename*1*=\"bar.txt\"\n" + ) + filename = msg.get_filename() + self.assertEqual(filename, 'foo bar.txt') + + def test_sorting_no_continuations(self): + msg = email.message_from_string( + "Content-Disposition: attachment; " + "filename*=\"bar.txt\"; " + ) + filename = msg.get_filename() + self.assertEqual(filename, 'bar.txt') + def test_missing_filename(self): msg = email.message_from_string("From: foo\n") self.assertEqual(msg.get_filename(), None) @@ -463,6 +481,27 @@ def test_get_param_with_quotes(self): "Content-Type: foo; bar*0=\"baz\\\"foobar\"; bar*1=\"\\\"baz\"") self.assertEqual(msg.get_param('bar'), 'baz"foobar"baz') + def test_get_param_linear_complexity(self): + # Ensure that email.message._parseparam() is fast. + # See https://github.com/python/cpython/issues/136063. + N = 100_000 + for s, r in [ + ("", ""), + ("foo=bar", "foo=bar"), + (" FOO = bar ", "foo=bar"), + ]: + with self.subTest(s=s, r=r, N=N): + src = f'{s};' * (N - 1) + s + res = email.message._parseparam(src) + self.assertEqual(len(res), N) + self.assertEqual(len(set(res)), 1) + self.assertEqual(res[0], r) + + # This will be considered as a single parameter. + malformed = 's="' + ';' * (N - 1) + res = email.message._parseparam(malformed) + self.assertEqual(res, [malformed]) + def test_field_containment(self): msg = email.message_from_string('Header: exists') self.assertIn('header', msg) @@ -2223,70 +2262,6 @@ def test_parse_missing_minor_type(self): eq(msg.get_content_maintype(), 'text') eq(msg.get_content_subtype(), 'plain') - # test_defect_handling - def test_same_boundary_inner_outer(self): - msg = self._msgobj('msg_15.txt') - # XXX We can probably eventually do better - inner = msg.get_payload(0) - self.assertHasAttr(inner, 'defects') - self.assertEqual(len(inner.defects), 1) - self.assertIsInstance(inner.defects[0], - errors.StartBoundaryNotFoundDefect) - - # test_defect_handling - def test_multipart_no_boundary(self): - msg = self._msgobj('msg_25.txt') - self.assertIsInstance(msg.get_payload(), str) - self.assertEqual(len(msg.defects), 2) - self.assertIsInstance(msg.defects[0], - errors.NoBoundaryInMultipartDefect) - self.assertIsInstance(msg.defects[1], - errors.MultipartInvariantViolationDefect) - - multipart_msg = textwrap.dedent("""\ - Date: Wed, 14 Nov 2007 12:56:23 GMT - From: foo@bar.invalid - To: foo@bar.invalid - Subject: Content-Transfer-Encoding: base64 and multipart - MIME-Version: 1.0 - Content-Type: multipart/mixed; - boundary="===============3344438784458119861=="{} - - --===============3344438784458119861== - Content-Type: text/plain - - Test message - - --===============3344438784458119861== - Content-Type: application/octet-stream - Content-Transfer-Encoding: base64 - - YWJj - - --===============3344438784458119861==-- - """) - - # test_defect_handling - def test_multipart_invalid_cte(self): - msg = self._str_msg( - self.multipart_msg.format("\nContent-Transfer-Encoding: base64")) - self.assertEqual(len(msg.defects), 1) - self.assertIsInstance(msg.defects[0], - errors.InvalidMultipartContentTransferEncodingDefect) - - # test_defect_handling - def test_multipart_no_cte_no_defect(self): - msg = self._str_msg(self.multipart_msg.format('')) - self.assertEqual(len(msg.defects), 0) - - # test_defect_handling - def test_multipart_valid_cte_no_defect(self): - for cte in ('7bit', '8bit', 'BINary'): - msg = self._str_msg( - self.multipart_msg.format( - "\nContent-Transfer-Encoding: {}".format(cte))) - self.assertEqual(len(msg.defects), 0) - # test_headerregistry.TestContentTypeHeader invalid_1 and invalid_2. def test_invalid_content_type(self): eq = self.assertEqual @@ -2334,7 +2309,7 @@ def test_no_separating_blank_line(self): To: bperson@dom.ain Subject: here's something interesting -counter to RFC 2822, there's no separating newline here +counter to RFC 5322, there's no separating newline here """) # test_defect_handling @@ -2363,30 +2338,6 @@ def test_missing_start_boundary(self): self.assertIsInstance(bad.defects[0], errors.StartBoundaryNotFoundDefect) - # test_defect_handling - def test_first_line_is_continuation_header(self): - eq = self.assertEqual - m = ' Line 1\nSubject: test\n\nbody' - msg = email.message_from_string(m) - eq(msg.keys(), ['Subject']) - eq(msg.get_payload(), 'body') - eq(len(msg.defects), 1) - self.assertDefectsEqual(msg.defects, - [errors.FirstHeaderLineIsContinuationDefect]) - eq(msg.defects[0].line, ' Line 1\n') - - # test_defect_handling - def test_missing_header_body_separator(self): - # Our heuristic if we see a line that doesn't look like a header (no - # leading whitespace but no ':') is to assume that the blank line that - # separates the header from the body is missing, and to stop parsing - # headers and start parsing the body. - msg = self._str_msg('Subject: test\nnot a header\nTo: abc\n\nb\n') - self.assertEqual(msg.keys(), ['Subject']) - self.assertEqual(msg.get_payload(), 'not a header\nTo: abc\n\nb\n') - self.assertDefectsEqual(msg.defects, - [errors.MissingHeaderBodySeparatorDefect]) - def test_string_payload_with_extra_space_after_cte(self): # https://github.com/python/cpython/issues/98188 cte = "base64 " @@ -2490,49 +2441,49 @@ def test_rfc2047_Q_invalid_digits(self): [(b'andr\xe9=zz', 'iso-8859-1')]) def test_rfc2047_rfc2047_1(self): - # 1st testcase at end of rfc2047 + # 1st testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'a', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_2(self): - # 2nd testcase at end of rfc2047 + # 2nd testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a?= b)' self.assertEqual(decode_header(s), [(b'(', None), (b'a', 'iso-8859-1'), (b' b)', None)]) def test_rfc2047_rfc2047_3(self): - # 3rd testcase at end of rfc2047 + # 3rd testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a?= =?ISO-8859-1?Q?b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_4(self): - # 4th testcase at end of rfc2047 + # 4th testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a?= =?ISO-8859-1?Q?b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_5a(self): - # 5th testcase at end of rfc2047 newline is \r\n + # 5th testcase at end of RFC 2047 newline is \r\n s = '(=?ISO-8859-1?Q?a?=\r\n =?ISO-8859-1?Q?b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_5b(self): - # 5th testcase at end of rfc2047 newline is \n + # 5th testcase at end of RFC 2047 newline is \n s = '(=?ISO-8859-1?Q?a?=\n =?ISO-8859-1?Q?b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'ab', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_6(self): - # 6th testcase at end of rfc2047 + # 6th testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a_b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'a b', 'iso-8859-1'), (b')', None)]) def test_rfc2047_rfc2047_7(self): - # 7th testcase at end of rfc2047 + # 7th testcase at end of RFC 2047 s = '(=?ISO-8859-1?Q?a?= =?ISO-8859-2?Q?_b?=)' self.assertEqual(decode_header(s), [(b'(', None), (b'a', 'iso-8859-1'), (b' b', 'iso-8859-2'), @@ -2550,6 +2501,18 @@ def test_multiline_header(self): self.assertEqual(str(make_header(decode_header(s))), '"Müller T" ') + def test_unencoded_ascii(self): + # bpo-22833/gh-67022: returns [(str, None)] rather than [(bytes, None)] + s = 'header without encoded words' + self.assertEqual(decode_header(s), + [('header without encoded words', None)]) + + def test_unencoded_utf8(self): + # bpo-22833/gh-67022: returns [(str, None)] rather than [(bytes, None)] + s = 'header with unexpected non ASCII caract\xe8res' + self.assertEqual(decode_header(s), + [('header with unexpected non ASCII caract\xe8res', None)]) + # Test the MIMEMessage class class TestMIMEMessage(TestEmailBase): @@ -3222,8 +3185,8 @@ def test_parsedate_y2k(self): """Test for parsing a date with a two-digit year. Parsing a date with a two-digit year should return the correct - four-digit year. RFC822 allows two-digit years, but RFC2822 (which - obsoletes RFC822) requires four-digit years. + four-digit year. RFC 822 allows two-digit years, but RFC 5322 (which + obsoletes RFC 2822, which obsoletes RFC 822) requires four-digit years. """ self.assertEqual(utils.parsedate_tz('25 Feb 03 13:47:26 -0800'), @@ -3274,7 +3237,7 @@ def test_escape_backslashes(self): self.assertEqual(utils.parseaddr(utils.formataddr((a, b))), (a, b)) def test_quotes_unicode_names(self): - # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + # issue 1690608. email.utils.formataddr() should be RFC 2047 aware. name = "H\u00e4ns W\u00fcrst" addr = 'person@dom.ain' utf8_base64 = "=?utf-8?b?SMOkbnMgV8O8cnN0?= " @@ -3284,7 +3247,7 @@ def test_quotes_unicode_names(self): latin1_quopri) def test_accepts_any_charset_like_object(self): - # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + # issue 1690608. email.utils.formataddr() should be RFC 2047 aware. name = "H\u00e4ns W\u00fcrst" addr = 'person@dom.ain' utf8_base64 = "=?utf-8?b?SMOkbnMgV8O8cnN0?= " @@ -3299,7 +3262,7 @@ def header_encode(self, string): utf8_base64) def test_invalid_charset_like_object_raises_error(self): - # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + # issue 1690608. email.utils.formataddr() should be RFC 2047 aware. name = "H\u00e4ns W\u00fcrst" addr = 'person@dom.ain' # An object without a header_encode method: @@ -3308,7 +3271,7 @@ def test_invalid_charset_like_object_raises_error(self): bad_charset) def test_unicode_address_raises_error(self): - # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + # issue 1690608. email.utils.formataddr() should be RFC 2047 aware. addr = 'pers\u00f6n@dom.in' self.assertRaises(UnicodeError, utils.formataddr, (None, addr)) self.assertRaises(UnicodeError, utils.formataddr, ("Name", addr)) @@ -3329,7 +3292,7 @@ def test_parseaddr_preserves_quoted_pairs_in_addresses(self): # string containing a quoted backslash, followed by 'example' and two # backslashes, followed by another quoted string containing a space and # the word 'example'. parseaddr copies those two backslashes - # literally. Per rfc5322 this is not technically correct since a \ may + # literally. Per RFC 5322 this is not technically correct since a \ may # not appear in an address outside of a quoted string. It is probably # a sensible Postel interpretation, though. eq = self.assertEqual @@ -3341,12 +3304,12 @@ def test_parseaddr_preserves_quoted_pairs_in_addresses(self): ('', '"\\\\"example\\\\" example"@example.com')) def test_parseaddr_preserves_spaces_in_local_part(self): - # issue 9286. A normal RFC5322 local part should not contain any + # issue 9286. A normal RFC 5322 local part should not contain any # folding white space, but legacy local parts can (they are a sequence # of atoms, not dotatoms). On the other hand we strip whitespace from # before the @ and around dots, on the assumption that the whitespace # around the punctuation is a mistake in what would otherwise be - # an RFC5322 local part. Leading whitespace is, usual, stripped as well. + # an RFC 5322 local part. Leading whitespace is, usual, stripped as well. self.assertEqual(('', "merwok wok@xample.com"), utils.parseaddr("merwok wok@xample.com")) self.assertEqual(('', "merwok wok@xample.com"), @@ -4864,6 +4827,15 @@ def test_decode_soft_line_break(self): def test_decode_false_quoting(self): self._test_decode('A=1,B=A ==> A+B==2', 'A=1,B=A ==> A+B==2') + def test_decode_crlf_eol_no_trailing_newline(self): + self._test_decode('abc', 'abc', eol='\r\n') + + def test_decode_crlf_eol_multiline_no_trailing_newline(self): + self._test_decode('a\r\nb', 'a\r\nb', eol='\r\n') + + def test_decode_crlf_eol_with_trailing_newline(self): + self._test_decode('abc\r\n', 'abc\r\n', eol='\r\n') + def _test_encode(self, body, expected_encoded_body, maxlinelen=None, eol=None): kwargs = {} if maxlinelen is None: @@ -5022,15 +4994,8 @@ def test_body_encode(self): # Try the convert argument, where input codec != output codec c = Charset('euc-jp') # With apologies to Tokio Kikuchi ;) - # XXX FIXME -## try: -## eq('\x1b$B5FCO;~IW\x1b(B', -## c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7')) -## eq('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', -## c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', False)) -## except LookupError: -## # We probably don't have the Japanese codecs installed -## pass + eq('\x1b$B5FCO;~IW\x1b(B', + c.body_encode('\u83ca\u5730\u6642\u592b')) # Testing SF bug #625509, which we have to fake, since there are no # built-in encodings where the header encoding is QP but the body # encoding is not. diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py index c75a842c33578e9..3c9a86f3e8cf291 100644 --- a/Lib/test/test_email/test_generator.py +++ b/Lib/test/test_email/test_generator.py @@ -1,13 +1,20 @@ import io import textwrap import unittest +import random +import sys from email import message_from_string, message_from_bytes from email.message import EmailMessage +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText from email.generator import Generator, BytesGenerator +import email.generator from email.headerregistry import Address from email import policy import email.errors from test.test_email import TestEmailBase, parameterize +import test.support + @parameterize @@ -288,6 +295,36 @@ def test_keep_long_encoded_newlines(self): g.flatten(msg) self.assertEqual(s.getvalue(), self.typ(expected)) + def _test_boundary_detection(self, linesep): + # Generate a boundary token in the same way as _make_boundary + token = random.randrange(sys.maxsize) + + def _patch_random_randrange(*args, **kwargs): + return token + + with test.support.swap_attr( + random, "randrange", _patch_random_randrange + ): + boundary = self.genclass._make_boundary(text=None) + boundary_in_part = ( + "this goes before the boundary\n--" + + boundary + + "\nthis goes after\n" + ) + msg = MIMEMultipart() + msg.attach(MIMEText(boundary_in_part)) + self.genclass(self.ioclass()).flatten(msg, linesep=linesep) + # Generator checks the message content for the string it is about + # to use as a boundary ('token' in this test) and when it finds it + # in our attachment appends .0 to make the boundary it uses unique. + self.assertEqual(msg.get_boundary(), boundary + ".0") + + def test_lf_boundary_detection(self): + self._test_boundary_detection("\n") + + def test_crlf_boundary_detection(self): + self._test_boundary_detection("\r\n") + class TestGenerator(TestGeneratorBase, TestEmailBase): @@ -313,7 +350,7 @@ def test_flatten_unicode_linesep(self): self.assertEqual(s.getvalue(), self.typ(expected)) def test_verify_generated_headers(self): - """gh-121650: by default the generator prevents header injection""" + # gh-121650: by default the generator prevents header injection class LiteralHeader(str): name = 'Header' def fold(self, **kwargs): @@ -334,6 +371,8 @@ def fold(self, **kwargs): with self.assertRaises(email.errors.HeaderWriteError): message.as_string() + with self.assertRaises(email.errors.HeaderWriteError): + message.as_bytes() class TestBytesGenerator(TestGeneratorBase, TestEmailBase): @@ -391,6 +430,50 @@ def test_defaults_handle_spaces_at_start_of_continuation_line(self): g.flatten(msg) self.assertEqual(s.getvalue(), expected) + # gh-144156: fold between non-encoded and encoded words don't need to encoded + # the separating space + def test_defaults_handle_spaces_at_start_of_continuation_line_2(self): + source = ("Re: [SOS-1495488] Commande et livraison - Demande de retour - " + "bibijolie - 251210-AABBCC - Abo actualités digitales 20 semaines " + "d’abonnement à 24 heures, Bilan, Tribune de Genève et tous les titres Tamedia") + expected = ( + b"Subject: " + b"Re: [SOS-1495488] Commande et livraison - Demande de retour -\n" + b" bibijolie - 251210-AABBCC - Abo =?utf-8?q?actualit=C3=A9s?= digitales 20\n" + b" semaines =?utf-8?q?d=E2=80=99abonnement_=C3=A0?= 24 heures, Bilan, Tribune de\n" + b" =?utf-8?q?Gen=C3=A8ve?= et tous les titres Tamedia\n\n" + ) + msg = EmailMessage() + msg['Subject'] = source + s = io.BytesIO() + g = BytesGenerator(s) + g.flatten(msg) + self.assertEqual(s.getvalue(), expected) + + def test_ew_folding_round_trip_1(self): + print() + source = "aaaaaaaaa фффффффф " + msg = EmailMessage() + msg['Subject'] = source + s = io.BytesIO() + g = BytesGenerator(s, maxheaderlen=30) + g.flatten(msg) + flat = s.getvalue() + reparsed = message_from_bytes(flat, policy=policy.default)['Subject'] + self.assertMultiLineEqual(reparsed, source) + + def test_ew_folding_round_trip_2(self): + print() + source = "aaa aaaaaaa aaa ффф фффф " + msg = EmailMessage() + msg['Subject'] = source + s = io.BytesIO() + g = BytesGenerator(s, maxheaderlen=30) + g.flatten(msg) + flat = s.getvalue() + reparsed = message_from_bytes(flat, policy=policy.default)['Subject'] + self.assertMultiLineEqual(reparsed, source) + def test_cte_type_7bit_handles_unknown_8bit(self): source = ("Subject: Maintenant je vous présente mon " "collègue\n\n").encode('utf-8') diff --git a/Lib/test/test_email/test_headerregistry.py b/Lib/test/test_email/test_headerregistry.py index ff7a6da644d5721..ddf0fc64b0c676a 100644 --- a/Lib/test/test_email/test_headerregistry.py +++ b/Lib/test/test_email/test_headerregistry.py @@ -220,6 +220,14 @@ def test_invalid_date_value(self): self.assertEqual(len(h.defects), 1) self.assertIsInstance(h.defects[0], errors.InvalidDateDefect) + def test_out_of_range_date_value(self): + s = 'Mon, 20 Nov 9999999999 12:00:00 +0000' + h = self.make_header('date', s) + self.assertEqual(h, s) + self.assertIsNone(h.datetime) + self.assertEqual(len(h.defects), 1) + self.assertIsInstance(h.defects[0], errors.InvalidDateDefect) + def test_datetime_read_only(self): h = self.make_header('date', self.datestring) with self.assertRaises(AttributeError): @@ -1262,12 +1270,12 @@ class TestAddressHeader(TestHeaderBase): 'example.com', None), - } - # XXX: Need many more examples, and in particular some with names in # trailing comments, which aren't currently handled. comments in # general are not handled yet. + } + def example_as_address(self, source, defects, decoded, display_name, addr_spec, username, domain, comment): h = self.make_header('sender', source) @@ -1285,6 +1293,43 @@ def example_as_address(self, source, defects, decoded, display_name, # XXX: we have no comment support yet. #self.assertEqual(a.comment, comment) + example_broken_header_params = { + + 'just_dquote': + ('"', + [errors.InvalidHeaderDefect]*2, + '<>', + '', + '<>', + '', + '', + ), + + } + + def example_broken_header_as_address( + self, + source, + defects, + decoded, + display_name, + addr_spec, + username, + domain, + ): + h = self.make_header('sender', source) + self.assertEqual(h, decoded) + self.assertDefectsEqual(h.defects, defects) + a = h.address + self.assertEqual(str(a), decoded) + self.assertEqual(len(h.groups), 1) + self.assertEqual([a], list(h.groups[0].addresses)) + self.assertEqual([a], list(h.addresses)) + self.assertEqual(a.display_name, display_name) + self.assertEqual(a.addr_spec, addr_spec) + self.assertEqual(a.username, username) + self.assertEqual(a.domain, domain) + def example_as_group(self, source, defects, decoded, display_name, addr_spec, username, domain, comment): source = 'foo: {};'.format(source) @@ -1702,7 +1747,7 @@ def test_fold_unstructured_with_overlong_word(self): 'singlewordthatwontfit') self.assertEqual( h.fold(policy=policy.default.clone(max_line_length=20)), - 'Subject: \n' + 'Subject:\n' ' =?utf-8?q?thisisa?=\n' ' =?utf-8?q?verylon?=\n' ' =?utf-8?q?glineco?=\n' @@ -1718,7 +1763,7 @@ def test_fold_unstructured_with_two_overlong_words(self): 'singlewordthatwontfit plusanotherverylongwordthatwontfit') self.assertEqual( h.fold(policy=policy.default.clone(max_line_length=20)), - 'Subject: \n' + 'Subject:\n' ' =?utf-8?q?thisisa?=\n' ' =?utf-8?q?verylon?=\n' ' =?utf-8?q?glineco?=\n' @@ -1812,5 +1857,18 @@ def test_message_id_header_is_not_folded(self): h.fold(policy=policy.default.clone(max_line_length=20)), 'Message-ID:\n <ईमेलfromMessage@wők.com>\n') + def test_fold_references(self): + h = self.make_header( + 'References', + ' ' + '' + ) + self.assertEqual( + h.fold(policy=policy.default.clone(max_line_length=20)), + 'References: ' + '\n' + ' \n') + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_email/test_message.py b/Lib/test/test_email/test_message.py index 23c39775a8b2e5e..56ad446694d1f88 100644 --- a/Lib/test/test_email/test_message.py +++ b/Lib/test/test_email/test_message.py @@ -1004,6 +1004,32 @@ def test_folding_with_long_nospace_http_policy_1(self): parsed_msg = message_from_bytes(m.as_bytes(), policy=policy.default) self.assertEqual(parsed_msg['Message-ID'], m['Message-ID']) + def test_no_wrapping_max_line_length(self): + # Test that falsey 'max_line_length' are converted to sys.maxsize. + for n in [0, None]: + with self.subTest(max_line_length=n): + self.do_test_no_wrapping_max_line_length(n) + + def do_test_no_wrapping_max_line_length(self, falsey): + self.assertFalse(falsey) + pol = policy.default.clone(max_line_length=falsey) + subj = "S" * 100 + body = "B" * 100 + msg = EmailMessage(policy=pol) + msg["From"] = "a@ex.com" + msg["To"] = "b@ex.com" + msg["Subject"] = subj + msg.set_content(body) + + raw = msg.as_bytes() + self.assertNotIn(b"=\n", raw, + "Found fold indicator; wrapping not disabled") + + parsed = message_from_bytes(raw, policy=policy.default) + self.assertEqual(parsed["Subject"], subj) + parsed_body = parsed.get_body().get_content().rstrip('\n') + self.assertEqual(parsed_body, body) + def test_invalid_header_names(self): invalid_headers = [ ('Invalid Header', 'contains space'), @@ -1055,6 +1081,15 @@ def test_get_body_malformed(self): # AttributeError: 'str' object has no attribute 'is_attachment' m.get_body() + def test_get_bytes_payload_with_quoted_printable_encoding(self): + # We use a memoryview to avoid directly changing the private payload + # and to prevent using the dedicated paths for string or bytes objects. + payload = memoryview(b'Some payload') + m = self._make_message() + m.add_header('Content-Transfer-Encoding', 'quoted-printable') + m.set_payload(payload) + self.assertEqual(m.get_payload(decode=True), payload) + class TestMIMEPart(TestEmailMessageBase, TestEmailBase): # Doing the full test run here may seem a bit redundant, since the two diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py index baa35fd68e49c51..90e8e5580295f9b 100644 --- a/Lib/test/test_email/test_policy.py +++ b/Lib/test/test_email/test_policy.py @@ -273,7 +273,7 @@ def test_non_ascii_chars_do_not_cause_inf_loop(self): actual = policy.fold('Subject', 'ą' * 12) self.assertEqual( actual, - 'Subject: \n' + + 'Subject:\n' + 12 * ' =?utf-8?q?=C4=85?=\n') def test_short_maxlen_error(self): @@ -296,7 +296,7 @@ def test_short_maxlen_error(self): policy.fold("Subject", subject) def test_verify_generated_headers(self): - """Turning protection off allows header injection""" + # Turning protection off allows header injection policy = email.policy.default.clone(verify_generated_headers=False) for text in ( 'Header: Value\r\nBad: Injection\r\n', @@ -319,6 +319,10 @@ def fold(self, **kwargs): message.as_string(), f"{text}\nBody", ) + self.assertEqual( + message.as_bytes(), + f"{text}\nBody".encode(), + ) # XXX: Need subclassing tests. # For adding subclassed objects, make sure the usual rules apply (subclass diff --git a/Lib/test/test_email/test_utils.py b/Lib/test/test_email/test_utils.py index c9d09098b502f9d..86d1bfff891a7e6 100644 --- a/Lib/test/test_email/test_utils.py +++ b/Lib/test/test_email/test_utils.py @@ -77,6 +77,15 @@ def test_parsedate_to_datetime_with_invalid_raises_valueerror(self): with self.subTest(dtstr=dtstr): self.assertRaises(ValueError, utils.parsedate_to_datetime, dtstr) + def test_parsedate_to_datetime_out_of_range_raises_valueerror(self): + out_of_range_dates = [ + 'Mon, 20 Nov 9999999999 12:00:00 +0000', + 'Mon, 20 Nov 2017 12:00:00 +24000000000000', + ] + for dtstr in out_of_range_dates: + with self.subTest(dtstr=dtstr): + self.assertRaises(ValueError, utils.parsedate_to_datetime, dtstr) + class LocaltimeTests(unittest.TestCase): def test_localtime_is_tz_aware_daylight_true(self): diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 95b2d80464c349e..ec5767cd918a003 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -68,6 +68,9 @@ if not os.path.isfile(os.path.join(STDLIB_INSTALL, 'os.py')): STDLIB_INSTALL = None +CODE_EXITCODE_123 = 'raise SystemExit(123)' + + def debug_build(program): program = os.path.basename(program) name = os.path.splitext(program)[0] @@ -117,12 +120,16 @@ def run_embedded_interpreter(self, *args, env=None, env = env.copy() env['SYSTEMROOT'] = os.environ['SYSTEMROOT'] - p = subprocess.Popen(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - env=env, - cwd=cwd) + kwargs = dict( + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + env=env, + cwd=cwd, + ) + if input is not None: + kwargs['stdin'] = subprocess.PIPE + p = subprocess.Popen(cmd, **kwargs) try: (out, err) = p.communicate(input=input, timeout=timeout) except: @@ -239,6 +246,26 @@ def test_repeated_init_and_inittab(self): lines = "\n".join(lines) + "\n" self.assertEqual(out, lines) + def test_inittab_submodule_multiphase(self): + out, err = self.run_embedded_interpreter("test_inittab_submodule_multiphase") + self.assertEqual(err, "") + self.assertEqual(out, + "\n" + "\n" + "Hello from sub-module\n" + "mp_pkg.mp_submod.mp_submod_exec_slot_ran='yes'\n" + "mp_pkg.mp_pkg_exec_slot_ran='yes'\n" + ) + + def test_inittab_submodule_singlephase(self): + out, err = self.run_embedded_interpreter("test_inittab_submodule_singlephase") + self.assertEqual(self._nogil_filtered_err(err, "sp_pkg"), "") + self.assertEqual(out, + "\n" + "\n" + "Hello from sub-module\n" + ) + def test_forced_io_encoding(self): # Checks forced configuration of embedded interpreter IO streams env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape") @@ -516,6 +543,73 @@ def test_getargs_reset_static_parser(self): out, err = self.run_embedded_interpreter("test_repeated_init_exec", code) self.assertEqual(out, '1\n2\n3\n' * INIT_LOOPS) + @staticmethod + def _nogil_filtered_err(err: str, mod_name: str) -> str: + if not support.Py_GIL_DISABLED: + return err + + # the test imports a singlephase init extension, so it emits a warning + # under the free-threaded build + expected_runtime_warning = ( + "RuntimeWarning: The global interpreter lock (GIL)" + f" has been enabled to load module '{mod_name}'" + ) + filtered_err_lines = [ + line + for line in err.strip().splitlines() + if expected_runtime_warning not in line + ] + return "\n".join(filtered_err_lines) + + def check_program_exitcode(self, *args, check_stderr=True, **kwargs): + out, err = self.run_embedded_interpreter(*args, **kwargs) + self.assertEqual(out.rstrip(), 'ok! Py_RunMain() returned 123') + if check_stderr: + self.assertEqual(err, '') + + def test_init_run_main_code_exitcode(self): + code = CODE_EXITCODE_123 + self.check_program_exitcode("test_init_run_main_code_exitcode", code) + + def test_init_run_main_script_exitcode(self): + with tempfile.TemporaryDirectory() as tmpdir: + filename = os.path.join(tmpdir, 'script.py') + with open(filename, 'w') as fp: + fp.write(CODE_EXITCODE_123) + + self.check_program_exitcode("test_init_run_main_script_exitcode", + filename) + + def test_init_run_main_interactive_exitcode(self): + code = CODE_EXITCODE_123 + self.check_program_exitcode("test_init_run_main_interactive_exitcode", + input=code, + check_stderr=False) + + def test_init_run_main_startup_exitcode(self): + with tempfile.TemporaryDirectory() as tmpdir: + filename = os.path.join(tmpdir, 'startup.py') + with open(filename, 'x') as fp: + fp.write(CODE_EXITCODE_123) + + env = dict(os.environ) + env['PYTHONSTARTUP'] = filename + self.check_program_exitcode("test_init_run_main_interactive_exitcode", + env=env, + check_stderr=False) + + def test_init_run_main_module_exitcode(self): + with tempfile.TemporaryDirectory() as tmpdir: + modname = '_testembed_testmodule' + filename = os.path.join(tmpdir, modname + '.py') + with open(filename, 'x', encoding='utf8') as fp: + fp.write(CODE_EXITCODE_123) + + env = dict(os.environ) + env['PYTHONPATH'] = tmpdir + self.check_program_exitcode("test_init_run_main_module_exitcode", + modname, env=env) + def config_dev_mode(preconfig, config): preconfig['allocator'] = PYMEM_ALLOCATOR_DEBUG @@ -1379,7 +1473,7 @@ def module_search_paths(self, prefix=None, exec_prefix=None): if prefix is None: prefix = config['config']['prefix'] if exec_prefix is None: - exec_prefix = config['config']['prefix'] + exec_prefix = config['config']['exec_prefix'] if MS_WINDOWS: return config['config']['module_search_paths'] else: @@ -1513,8 +1607,10 @@ def test_init_is_python_build_with_home(self): expected_paths[1 if MS_WINDOWS else 2] = os.path.normpath( os.path.join(exedir, f'{f.read()}\n$'.splitlines()[0])) if not MS_WINDOWS: - # PREFIX (default) is set when running in build directory - prefix = exec_prefix = sys.prefix + # PREFIX and EXEC_PREFIX (defaults) are set when running in the + # build directory and may differ with --exec-prefix (gh-151096). + prefix = sys.prefix + exec_prefix = sys.exec_prefix # stdlib calculation (/Lib) is not yet supported expected_paths[0] = self.module_search_paths(prefix=prefix)[0] config.update(prefix=prefix, base_prefix=prefix, diff --git a/Lib/test/test_ensurepip.py b/Lib/test/test_ensurepip.py index 6d3c91b0b6d9f98..a8bfbcd4abce994 100644 --- a/Lib/test/test_ensurepip.py +++ b/Lib/test/test_ensurepip.py @@ -7,6 +7,7 @@ import unittest import unittest.mock from pathlib import Path +from test.support import import_helper import ensurepip import ensurepip._uninstall @@ -30,6 +31,15 @@ def test_version_no_dir(self): # when the bundled pip wheel is used, we get _PIP_VERSION self.assertEqual(ensurepip._PIP_VERSION, ensurepip.version()) + def test_wheel_pkg_dir_none(self): + # gh-146310: empty or None WHEEL_PKG_DIR should not search CWD + for value in ('', None): + with unittest.mock.patch('sysconfig.get_config_var', + return_value=value) as get_config_var: + module = import_helper.import_fresh_module('ensurepip') + self.assertIsNone(module._WHEEL_PKG_DIR) + get_config_var.assert_called_once_with('WHEEL_PKG_DIR') + def test_selected_wheel_path_no_dir(self): pip_filename = f'pip-{ensurepip._PIP_VERSION}-py3-none-any.whl' with unittest.mock.patch.object(ensurepip, '_WHEEL_PKG_DIR', None): diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py index d8cb5261244939c..44ce1f8ee2161d9 100644 --- a/Lib/test/test_enum.py +++ b/Lib/test/test_enum.py @@ -36,7 +36,7 @@ def load_tests(loader, tests, ignore): optionflags=doctest.ELLIPSIS|doctest.NORMALIZE_WHITESPACE, )) howto_tests = os.path.join(REPO_ROOT, 'Doc/howto/enum.rst') - if os.path.exists(howto_tests): + if os.path.exists(howto_tests) and sys.float_repr_style == 'short': tests.addTests(doctest.DocFileSuite( howto_tests, module_relative=False, @@ -434,9 +434,9 @@ class Season(self.enum_type): def spam(cls): pass # - self.assertTrue(hasattr(Season, 'spam')) + self.assertHasAttr(Season, 'spam') del Season.spam - self.assertFalse(hasattr(Season, 'spam')) + self.assertNotHasAttr(Season, 'spam') # with self.assertRaises(AttributeError): del Season.SPRING @@ -2652,12 +2652,12 @@ def __new__(cls, value, period): OneDay = day_1 OneWeek = week_1 OneMonth = month_1 - self.assertFalse(hasattr(Period, '_ignore_')) - self.assertFalse(hasattr(Period, 'Period')) - self.assertFalse(hasattr(Period, 'i')) - self.assertTrue(isinstance(Period.day_1, timedelta)) - self.assertTrue(Period.month_1 is Period.day_30) - self.assertTrue(Period.week_4 is Period.day_28) + self.assertNotHasAttr(Period, '_ignore_') + self.assertNotHasAttr(Period, 'Period') + self.assertNotHasAttr(Period, 'i') + self.assertIsInstance(Period.day_1, timedelta) + self.assertIs(Period.month_1, Period.day_30) + self.assertIs(Period.week_4, Period.day_28) def test_nonhash_value(self): class AutoNumberInAList(Enum): @@ -2877,7 +2877,7 @@ class ReformedColor(StrMixin, IntEnum, SomeEnum, AnotherEnum): self.assertEqual(str(ReformedColor.BLUE), 'blue') self.assertEqual(ReformedColor.RED.behavior(), 'booyah') self.assertEqual(ConfusedColor.RED.social(), "what's up?") - self.assertTrue(issubclass(ReformedColor, int)) + self.assertIsSubclass(ReformedColor, int) def test_multiple_inherited_mixin(self): @unique @@ -4984,8 +4984,8 @@ class Color(enum.Enum) | __members__ | Returns a mapping of member name->value. | - | This mapping lists all enum members, including aliases. Note that this - | is a read-only view of the internal mapping.""" + | This mapping lists all enum members, including aliases. Note that + | this is a read-only view of the internal mapping.""" expected_help_output_without_docs = """\ Help on class Color in module %s: diff --git a/Lib/test/test_enumerate.py b/Lib/test/test_enumerate.py index 5cb54cff9b76fdd..c8b85fe86921781 100644 --- a/Lib/test/test_enumerate.py +++ b/Lib/test/test_enumerate.py @@ -3,8 +3,11 @@ import sys import pickle import gc +import threading + from test import support +from test.support import threading_helper class G: 'Sequence using __getitem__' @@ -292,5 +295,28 @@ def enum(self, iterable, start=sys.maxsize + 1): (sys.maxsize+3,'c')] +@threading_helper.requires_working_threading() +class TestThreadSafety(EnumerateStartTestCase): + def test_thread_safety_while_iterating(self): + # gh-153932: calling reduce while iterating should pass with TSAN + + en = enumerate(range(10_000)) + stop = threading.Event() + + def advance(): + for _ in en: + pass + stop.set() + + def read(): + while not stop.is_set(): + en.__reduce__() + + threads = [threading.Thread(target=advance), threading.Thread(target=read)] + + with threading_helper.start_threads(threads): + pass + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_epoll.py b/Lib/test/test_epoll.py index c94946a6ae6b7c3..8195f437843e517 100644 --- a/Lib/test/test_epoll.py +++ b/Lib/test/test_epoll.py @@ -25,6 +25,7 @@ import os import select import socket +import sys import time import unittest from test import support @@ -52,12 +53,11 @@ def tearDown(self): def _connected_pair(self): client = socket.socket() client.setblocking(False) + # The connection is either established immediately or in progress. try: client.connect(('127.0.0.1', self.serverSocket.getsockname()[1])) except OSError as e: self.assertEqual(e.args[0], errno.EINPROGRESS) - else: - raise AssertionError("Connect should have raised EINPROGRESS") server, addr = self.serverSocket.accept() self.connections.extend((client, server)) @@ -218,6 +218,9 @@ def test_errors(self): self.assertRaises(ValueError, select.epoll().register, -1, select.EPOLLIN) + @unittest.skipIf(sys.platform.startswith('sunos'), + 'unregistering a closed file descriptor succeeds ' + 'on Solaris') def test_unregister_closed(self): server, client = self._connected_pair() fd = server.fileno() diff --git a/Lib/test/test_errno.py b/Lib/test/test_errno.py index 5c437e9ccea767c..e7f185c6b1a1812 100644 --- a/Lib/test/test_errno.py +++ b/Lib/test/test_errno.py @@ -12,14 +12,12 @@ class ErrnoAttributeTests(unittest.TestCase): def test_for_improper_attributes(self): # No unexpected attributes should be on the module. for error_code in std_c_errors: - self.assertTrue(hasattr(errno, error_code), - "errno is missing %s" % error_code) + self.assertHasAttr(errno, error_code) def test_using_errorcode(self): # Every key value in errno.errorcode should be on the module. for value in errno.errorcode.values(): - self.assertTrue(hasattr(errno, value), - 'no %s attr in errno' % value) + self.assertHasAttr(errno, value) class ErrorcodeTests(unittest.TestCase): diff --git a/Lib/test/test_exception_group.py b/Lib/test/test_exception_group.py index 92bbf7917642b0f..7d690ca59738ebb 100644 --- a/Lib/test/test_exception_group.py +++ b/Lib/test/test_exception_group.py @@ -1,13 +1,15 @@ -import collections.abc +import collections import types import unittest -from test.support import skip_emscripten_stack_overflow, exceeds_recursion_limit +from test.support import (skip_emscripten_stack_overflow, + skip_wasi_stack_overflow, skip_if_huge_c_stack, + exceeds_recursion_limit) class TestExceptionGroupTypeHierarchy(unittest.TestCase): def test_exception_group_types(self): - self.assertTrue(issubclass(ExceptionGroup, Exception)) - self.assertTrue(issubclass(ExceptionGroup, BaseExceptionGroup)) - self.assertTrue(issubclass(BaseExceptionGroup, BaseException)) + self.assertIsSubclass(ExceptionGroup, Exception) + self.assertIsSubclass(ExceptionGroup, BaseExceptionGroup) + self.assertIsSubclass(BaseExceptionGroup, BaseException) def test_exception_is_not_generic_type(self): with self.assertRaisesRegex(TypeError, 'Exception'): @@ -193,6 +195,77 @@ class MyEG(ExceptionGroup): "MyEG('flat', [ValueError(1), TypeError(2)]), " "TypeError(2)])")) + def test_exceptions_mutation(self): + class MyEG(ExceptionGroup): + pass + + excs = [ValueError(1), TypeError(2)] + eg = MyEG('test', excs) + + self.assertEqual(repr(eg), "MyEG('test', [ValueError(1), TypeError(2)])") + excs.clear() + + # Ensure that clearing the exceptions sequence doesn't change the repr. + self.assertEqual(repr(eg), "MyEG('test', [ValueError(1), TypeError(2)])") + + # Ensure that the args are still as passed. + self.assertEqual(eg.args, ('test', [])) + + excs = (ValueError(1), KeyboardInterrupt(2)) + eg = BaseExceptionGroup('test', excs) + + # Ensure that immutable sequences still work fine. + self.assertEqual( + repr(eg), + "BaseExceptionGroup('test', (ValueError(1), KeyboardInterrupt(2)))" + ) + + # Test non-standard custom sequences. + excs = collections.deque([ValueError(1), TypeError(2)]) + eg = ExceptionGroup('test', excs) + + self.assertEqual( + repr(eg), + "ExceptionGroup('test', deque([ValueError(1), TypeError(2)]))" + ) + excs.clear() + + # Ensure that clearing the exceptions sequence doesn't change the repr. + self.assertEqual( + repr(eg), + "ExceptionGroup('test', deque([ValueError(1), TypeError(2)]))" + ) + + def test_repr_raises(self): + class MySeq(collections.abc.Sequence): + def __init__(self, raises): + self.raises = raises + + def __len__(self): + return 1 + + def __getitem__(self, index): + if index == 0: + return ValueError(1) + raise IndexError + + def __repr__(self): + if self.raises: + raise self.raises + return None + + seq = MySeq(None) + with self.assertRaisesRegex( + TypeError, + r"__repr__ returned non-string \(type NoneType\)" + ): + ExceptionGroup("test", seq) + + seq = MySeq(ValueError) + with self.assertRaises(ValueError): + BaseExceptionGroup("test", seq) + + def create_simple_eg(): excs = [] @@ -464,13 +537,17 @@ def make_deep_eg(self): e = ExceptionGroup('eg', [e]) return e + @skip_if_huge_c_stack() @skip_emscripten_stack_overflow() + @skip_wasi_stack_overflow() def test_deep_split(self): e = self.make_deep_eg() with self.assertRaises(RecursionError): e.split(TypeError) + @skip_if_huge_c_stack() @skip_emscripten_stack_overflow() + @skip_wasi_stack_overflow() def test_deep_subgroup(self): e = self.make_deep_eg() with self.assertRaises(RecursionError): @@ -812,8 +889,8 @@ def test_split_does_not_copy_non_sequence_notes(self): eg = ExceptionGroup("eg", [ValueError(1), TypeError(2)]) eg.__notes__ = 123 match, rest = eg.split(TypeError) - self.assertFalse(hasattr(match, '__notes__')) - self.assertFalse(hasattr(rest, '__notes__')) + self.assertNotHasAttr(match, '__notes__') + self.assertNotHasAttr(rest, '__notes__') def test_drive_invalid_return_value(self): class MyEg(ExceptionGroup): diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index d177e3dc0f5007a..e79e52911f66aba 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -224,6 +224,8 @@ def check(self, src, lineno, offset, end_lineno=None, end_offset=None, encoding= if not isinstance(src, str): src = src.decode(encoding, 'replace') line = src.split('\n')[lineno-1] + if lineno == 1: + line = line.removeprefix('\ufeff') self.assertIn(line, cm.exception.text) def test_error_offset_continuation_characters(self): @@ -239,7 +241,9 @@ def testSyntaxErrorOffset(self): check('Python = "\u1e54\xfd\u0163\u0125\xf2\xf1" +', 1, 20) check(b'# -*- coding: cp1251 -*-\nPython = "\xcf\xb3\xf2\xee\xed" +', 2, 19, encoding='cp1251') - check(b'Python = "\xcf\xb3\xf2\xee\xed" +', 1, 10) + check(b'Python = "\xcf\xb3\xf2\xee\xed" +', 1, 12) + check(b'\n\n\nPython = "\xcf\xb3\xf2\xee\xed" +', 4, 12) + check(b'\xef\xbb\xbfPython = "\xcf\xb3\xf2\xee\xed" +', 1, 12) check('x = "a', 1, 5) check('lambda x: x = 2', 1, 1) check('f{a + b + c}', 1, 2) @@ -287,7 +291,7 @@ def baz(): check("pass\npass\npass\n(1+)\npass\npass\npass", 4, 4) check("(1+)", 1, 4) check("[interesting\nfoo()\n", 1, 1) - check(b"\xef\xbb\xbf#coding: utf8\nprint('\xe6\x88\x91')\n", 0, -1) + check(b"\xef\xbb\xbf#coding: utf8\nprint('\xe6\x88\x91')\n", 1, 0) check("""f''' { (123_a) @@ -357,7 +361,7 @@ def test_capi1(): except TypeError as err: co = err.__traceback__.tb_frame.f_code self.assertEqual(co.co_name, "test_capi1") - self.assertTrue(co.co_filename.endswith('test_exceptions.py')) + self.assertEndsWith(co.co_filename, 'test_exceptions.py') else: self.fail("Expected exception") @@ -369,7 +373,7 @@ def test_capi2(): tb = err.__traceback__.tb_next co = tb.tb_frame.f_code self.assertEqual(co.co_name, "__init__") - self.assertTrue(co.co_filename.endswith('test_exceptions.py')) + self.assertEndsWith(co.co_filename, 'test_exceptions.py') co2 = tb.tb_frame.f_back.f_code self.assertEqual(co2.co_name, "test_capi2") else: @@ -598,7 +602,7 @@ def test_invalid_setstate(self): def test_notes(self): for e in [BaseException(1), Exception(2), ValueError(3)]: with self.subTest(e=e): - self.assertFalse(hasattr(e, '__notes__')) + self.assertNotHasAttr(e, '__notes__') e.add_note("My Note") self.assertEqual(e.__notes__, ["My Note"]) @@ -610,7 +614,7 @@ def test_notes(self): self.assertEqual(e.__notes__, ["My Note", "Your Note"]) del e.__notes__ - self.assertFalse(hasattr(e, '__notes__')) + self.assertNotHasAttr(e, '__notes__') e.add_note("Our Note") self.assertEqual(e.__notes__, ["Our Note"]) @@ -1429,6 +1433,7 @@ def g(): self.assertIn("maximum recursion depth exceeded", str(exc)) @support.skip_wasi_stack_overflow() + @support.skip_emscripten_stack_overflow() @cpython_only @support.requires_resource('cpu') def test_trashcan_recursion(self): @@ -1444,6 +1449,7 @@ def foo(): foo() support.gc_collect() + @support.skip_emscripten_stack_overflow() @cpython_only def test_recursion_normalizing_exception(self): import_module("_testinternalcapi") @@ -1503,6 +1509,7 @@ def gen(): @cpython_only @unittest.skipIf(_testcapi is None, "requires _testcapi") @force_not_colorized + @support.skip_if_huge_c_stack() def test_recursion_normalizing_infinite_exception(self): # Issue #30697. Test that a RecursionError is raised when # maximum recursion depth has been exceeded when creating @@ -1521,6 +1528,7 @@ def test_recursion_normalizing_infinite_exception(self): self.assertIn(b'Done.', out) + @support.skip_emscripten_stack_overflow() def test_recursion_in_except_handler(self): def set_relative_recursion_limit(n): @@ -1568,11 +1576,7 @@ def recurse_in_body_and_except(): sys.setrecursionlimit(recursionlimit) - @cpython_only - # Python built with Py_TRACE_REFS fail with a fatal error in - # _PyRefchain_Trace() on memory allocation error. - @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') - @unittest.skipIf(_testcapi is None, "requires _testcapi") + @support.nomemtest def test_recursion_normalizing_with_no_memory(self): # Issue #30697. Test that in the abort that occurs when there is no # memory left and the size of the Python frames stack is greater than @@ -1626,7 +1630,7 @@ def test_exception_with_doc(self): # test basic usage of PyErr_NewException error1 = _testcapi.make_exception_with_doc("_testcapi.error1") self.assertIs(type(error1), type) - self.assertTrue(issubclass(error1, Exception)) + self.assertIsSubclass(error1, Exception) self.assertIsNone(error1.__doc__) # test with given docstring @@ -1636,21 +1640,21 @@ def test_exception_with_doc(self): # test with explicit base (without docstring) error3 = _testcapi.make_exception_with_doc("_testcapi.error3", base=error2) - self.assertTrue(issubclass(error3, error2)) + self.assertIsSubclass(error3, error2) # test with explicit base tuple class C(object): pass error4 = _testcapi.make_exception_with_doc("_testcapi.error4", doc4, (error3, C)) - self.assertTrue(issubclass(error4, error3)) - self.assertTrue(issubclass(error4, C)) + self.assertIsSubclass(error4, error3) + self.assertIsSubclass(error4, C) self.assertEqual(error4.__doc__, doc4) # test with explicit dictionary error5 = _testcapi.make_exception_with_doc("_testcapi.error5", "", error4, {'a': 1}) - self.assertTrue(issubclass(error5, error4)) + self.assertIsSubclass(error5, error4) self.assertEqual(error5.a, 1) self.assertEqual(error5.__doc__, "") @@ -1699,6 +1703,20 @@ def inner(): gc_collect() # For PyPy or other GCs. self.assertEqual(wr(), None) + def test_oserror_reinit_leak(self): + # gh-150988: Check for memory leak when re-initializing OSError. + # Previously, setting OSError attributes in a subclass + # before calling super().__init__() leaked memory. + class LeakingOSError(OSError): + def __init__(self, code, message, filename, filename2): + self.strerror = message + self.filename = filename + self.filename2 = filename2 + super().__init__(code, message, filename, None, filename2) + + exc = LeakingOSError(1, "some message", "filename.py", "filename2.py") + exc.__init__(2, "another message", "filename3.py", "filename4.py") + def test_errno_ENOTDIR(self): # Issue #12802: "not a directory" errors are ENOTDIR even on Windows with self.assertRaises(OSError) as cm: @@ -1743,13 +1761,9 @@ def test_unhandled(self): self.assertIn("", report) else: self.assertIn("test message", report) - self.assertTrue(report.endswith("\n")) + self.assertEndsWith(report, "\n") - @cpython_only - # Python built with Py_TRACE_REFS fail with a fatal error in - # _PyRefchain_Trace() on memory allocation error. - @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') - @unittest.skipIf(_testcapi is None, "requires _testcapi") + @support.nomemtest def test_memory_error_in_PyErr_PrintEx(self): code = """if 1: import _testcapi @@ -1907,6 +1921,35 @@ def test_keyerror_context(self): exc2 = None + @support.nomemtest + def test_exec_set_nomemory_hang(self): + # gh-134163: A MemoryError inside code that was wrapped by a try/except + # block would lead to an infinite loop. + + # The frame_lasti needs to be greater than 257 to prevent + # PyLong_FromLong() from returning cached integers, which + # don't require a memory allocation. Prepend some dummy code + # to artificially increase the instruction index. + warmup_code = "a = list(range(0, 1))\n" * 60 + user_input = warmup_code + dedent(""" + try: + import _testcapi + _testcapi.set_nomemory(0) + b = list(range(1000, 2000)) + except Exception as e: + import traceback + traceback.print_exc() + """) + with SuppressCrashReport(): + with script_helper.spawn_python('-c', user_input) as p: + p.wait() + output = p.stdout.read() + + self.assertIn(p.returncode, (0, 1)) + self.assertGreater(len(output), 0) # At minimum, should not hang + self.assertIn(b"MemoryError", output) + + class NameErrorTests(unittest.TestCase): def test_name_error_has_name(self): try: @@ -2468,6 +2511,30 @@ def test_incorrect_constructor(self): args = ("bad.py", 1, 2, "abcdefg", 1) self.assertRaises(TypeError, SyntaxError, "bad bad", args) + def test_syntax_error_memory_leak(self): + # gh-146250: memory leak with re-initialization of SyntaxError + e = SyntaxError("msg", ("file.py", 1, 2, "txt", 2, 3)) + e.__init__("new_msg", ("new_file.py", 2, 3, "new_txt", 3, 4)) + self.assertEqual(e.msg, "new_msg") + self.assertEqual(e.args, ("new_msg", ("new_file.py", 2, 3, "new_txt", 3, 4))) + self.assertEqual(e.filename, "new_file.py") + self.assertEqual(e.lineno, 2) + self.assertEqual(e.offset, 3) + self.assertEqual(e.text, "new_txt") + self.assertEqual(e.end_lineno, 3) + self.assertEqual(e.end_offset, 4) + + e = SyntaxError("msg", ("file.py", 1, 2, "txt", 2, 3)) + e.__init__("new_msg", ("new_file.py", 2, 3, "new_txt")) + self.assertEqual(e.msg, "new_msg") + self.assertEqual(e.args, ("new_msg", ("new_file.py", 2, 3, "new_txt"))) + self.assertEqual(e.filename, "new_file.py") + self.assertEqual(e.lineno, 2) + self.assertEqual(e.offset, 3) + self.assertEqual(e.text, "new_txt") + self.assertIsNone(e.end_lineno) + self.assertIsNone(e.end_offset) + class TestInvalidExceptionMatcher(unittest.TestCase): def test_except_star_invalid_exception_type(self): diff --git a/Lib/test/test_external_inspection.py b/Lib/test/test_external_inspection.py index ad3f669a03043eb..08779bdb0081397 100644 --- a/Lib/test/test_external_inspection.py +++ b/Lib/test/test_external_inspection.py @@ -4,9 +4,17 @@ import importlib import sys import socket -from asyncio import staggered, taskgroups +import threading +import time +from asyncio import staggered, taskgroups, base_events, tasks from unittest.mock import ANY -from test.support import os_helper, SHORT_TIMEOUT, busy_retry +from test.support import ( + os_helper, + SHORT_TIMEOUT, + busy_retry, + requires_gil_enabled, +) +from test.support.import_helper import import_module from test.support.script_helper import make_script from test.support.socket_helper import find_unused_port @@ -16,11 +24,13 @@ try: from _remote_debugging import PROCESS_VM_READV_SUPPORTED - from _remote_debugging import get_stack_trace - from _remote_debugging import get_async_stack_trace - from _remote_debugging import get_all_awaited_by + from _remote_debugging import RemoteUnwinder + from _remote_debugging import FrameInfo, CoroInfo, TaskInfo except ImportError: - raise unittest.SkipTest("Test only runs when _remote_debugging is available") + raise unittest.SkipTest( + "Test only runs when _remote_debugging is available" + ) + def _make_test_script(script_dir, script_basename, source): to_return = make_script(script_dir, script_basename, source) @@ -29,12 +39,32 @@ def _make_test_script(script_dir, script_basename, source): skip_if_not_supported = unittest.skipIf( - (sys.platform != "darwin" and sys.platform != "linux" and sys.platform != "win32"), + ( + sys.platform != "darwin" + and sys.platform != "linux" + and sys.platform != "win32" + ), "Test only runs on Linux, Windows and MacOS", ) +def get_stack_trace(pid): + unwinder = RemoteUnwinder(pid, all_threads=True, debug=True) + return unwinder.get_stack_trace() + + +def get_async_stack_trace(pid): + unwinder = RemoteUnwinder(pid, debug=True) + return unwinder.get_async_stack_trace() + + +def get_all_awaited_by(pid): + unwinder = RemoteUnwinder(pid, debug=True) + return unwinder.get_all_awaited_by() + + class TestGetStackTrace(unittest.TestCase): + maxDiff = None @skip_if_not_supported @unittest.skipIf( @@ -46,7 +76,7 @@ def test_remote_stack_trace(self): port = find_unused_port() script = textwrap.dedent( f"""\ - import time, sys, socket + import time, sys, socket, threading # Connect to the test process sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.connect(('localhost', {port})) @@ -55,13 +85,16 @@ def bar(): for x in range(100): if x == 50: baz() + def baz(): foo() def foo(): - sock.sendall(b"ready"); time.sleep(10_000) # same line number + sock.sendall(b"ready:thread\\n"); time.sleep(10_000) # same line number - bar() + t = threading.Thread(target=bar) + t.start() + sock.sendall(b"ready:main\\n"); t.join() # same line number """ ) stack_trace = None @@ -82,11 +115,17 @@ def foo(): p = subprocess.Popen([sys.executable, script_name]) client_socket, _ = server_socket.accept() server_socket.close() - response = client_socket.recv(1024) - self.assertEqual(response, b"ready") + response = b"" + while ( + b"ready:main" not in response + or b"ready:thread" not in response + ): + response += client_socket.recv(1024) stack_trace = get_stack_trace(p.pid) except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") + self.skipTest( + "Insufficient permissions to read the stack trace" + ) finally: if client_socket is not None: client_socket.close() @@ -94,13 +133,65 @@ def foo(): p.terminate() p.wait(timeout=SHORT_TIMEOUT) - expected_stack_trace = [ - ("foo", script_name, 14), - ("baz", script_name, 11), - ("bar", script_name, 9), - ("", script_name, 16), + thread_expected_stack_trace = [ + FrameInfo([script_name, 15, "foo"]), + FrameInfo([script_name, 12, "baz"]), + FrameInfo([script_name, 9, "bar"]), + FrameInfo([threading.__file__, ANY, "Thread.run"]), ] - self.assertEqual(stack_trace, expected_stack_trace) + # Is possible that there are more threads, so we check that the + # expected stack traces are in the result (looking at you Windows!) + self.assertIn((ANY, thread_expected_stack_trace), stack_trace) + + # Check that the main thread stack trace is in the result + frame = FrameInfo([script_name, 19, ""]) + for _, stack in stack_trace: + if frame in stack: + break + else: + self.fail("Main thread stack trace not found in result") + + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + def test_self_trace_after_ctypes_import(self): + """Test that RemoteUnwinder works on the same process after _ctypes import. + + When _ctypes is imported, it may call dlopen on the libpython shared + library, creating a duplicate mapping in the process address space. + The remote debugging code must skip these uninitialized duplicate + mappings and find the real PyRuntime. See gh-144563. + """ + + # Skip the test if the _ctypes module is missing. + import_module("_ctypes") + + # Run the test in a subprocess to avoid side effects + script = textwrap.dedent("""\ + import os + import _remote_debugging + + # Should work before _ctypes import + unwinder = _remote_debugging.RemoteUnwinder(os.getpid()) + + import _ctypes + + # Should still work after _ctypes import (gh-144563) + unwinder = _remote_debugging.RemoteUnwinder(os.getpid()) + """) + + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + text=True, + timeout=SHORT_TIMEOUT, + ) + self.assertEqual( + result.returncode, 0, + f"stdout: {result.stdout}\nstderr: {result.stderr}" + ) @skip_if_not_supported @unittest.skipIf( @@ -160,8 +251,12 @@ def new_eager_loop(): ): script_dir = os.path.join(work_dir, "script_pkg") os.mkdir(script_dir) - server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket = socket.socket( + socket.AF_INET, socket.SOCK_STREAM + ) + server_socket.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEADDR, 1 + ) server_socket.bind(("localhost", port)) server_socket.settimeout(SHORT_TIMEOUT) server_socket.listen(1) @@ -179,7 +274,9 @@ def new_eager_loop(): self.assertEqual(response, b"ready") stack_trace = get_async_stack_trace(p.pid) except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") + self.skipTest( + "Insufficient permissions to read the stack trace" + ) finally: if client_socket is not None: client_socket.close() @@ -187,85 +284,162 @@ def new_eager_loop(): p.terminate() p.wait(timeout=SHORT_TIMEOUT) - # sets are unordered, so we want to sort "awaited_by"s - stack_trace[2].sort(key=lambda x: x[1]) + # First check all the tasks are present + tasks_names = [ + task.task_name for task in stack_trace[0].awaited_by + ] + for task_name in ["c2_root", "sub_main_1", "sub_main_2"]: + self.assertIn(task_name, tasks_names) - root_task = "Task-1" - expected_stack_trace = [ - [ - ("c5", script_name, 10), - ("c4", script_name, 14), - ("c3", script_name, 17), - ("c2", script_name, 20), - ], - "c2_root", - [ - [ - [ - ( - "TaskGroup._aexit", - taskgroups.__file__, - ANY, + # Now ensure that the awaited_by_relationships are correct + id_to_task = { + task.task_id: task for task in stack_trace[0].awaited_by + } + task_name_to_awaited_by = { + task.task_name: set( + id_to_task[awaited.task_name].task_name + for awaited in task.awaited_by + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + task_name_to_awaited_by, + { + "c2_root": {"Task-1", "sub_main_1", "sub_main_2"}, + "Task-1": set(), + "sub_main_1": {"Task-1"}, + "sub_main_2": {"Task-1"}, + }, + ) + + # Now ensure that the coroutine stacks are correct + coroutine_stacks = { + task.task_name: sorted( + tuple(tuple(frame) for frame in coro.call_stack) + for coro in task.coroutine_stack + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + coroutine_stacks, + { + "Task-1": [ + ( + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] ), - ( - "TaskGroup.__aexit__", - taskgroups.__file__, - ANY, + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] ), - ("main", script_name, 26), - ], - "Task-1", - [], + tuple([script_name, 26, "main"]), + ) ], - [ - [("c1", script_name, 23)], - "sub_main_1", - [ - [ - [ - ( + "c2_root": [ + ( + tuple([script_name, 10, "c5"]), + tuple([script_name, 14, "c4"]), + tuple([script_name, 17, "c3"]), + tuple([script_name, 20, "c2"]), + ) + ], + "sub_main_1": [(tuple([script_name, 23, "c1"]),)], + "sub_main_2": [(tuple([script_name, 23, "c1"]),)], + }, + ) + + # Now ensure the coroutine stacks for the awaited_by relationships are correct. + awaited_by_coroutine_stacks = { + task.task_name: sorted( + ( + id_to_task[coro.task_name].task_name, + tuple(tuple(frame) for frame in coro.call_stack), + ) + for coro in task.awaited_by + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + awaited_by_coroutine_stacks, + { + "Task-1": [], + "c2_root": [ + ( + "Task-1", + ( + tuple( + [ + taskgroups.__file__, + ANY, "TaskGroup._aexit", + ] + ), + tuple( + [ taskgroups.__file__, ANY, - ), - ( "TaskGroup.__aexit__", + ] + ), + tuple([script_name, 26, "main"]), + ), + ), + ("sub_main_1", (tuple([script_name, 23, "c1"]),)), + ("sub_main_2", (tuple([script_name, 23, "c1"]),)), + ], + "sub_main_1": [ + ( + "Task-1", + ( + tuple( + [ taskgroups.__file__, ANY, - ), - ("main", script_name, 26), - ], - "Task-1", - [], - ] - ], - ], - [ - [("c1", script_name, 23)], - "sub_main_2", - [ - [ - [ - ( "TaskGroup._aexit", + ] + ), + tuple( + [ taskgroups.__file__, ANY, - ), - ( "TaskGroup.__aexit__", + ] + ), + tuple([script_name, 26, "main"]), + ), + ) + ], + "sub_main_2": [ + ( + "Task-1", + ( + tuple( + [ taskgroups.__file__, ANY, - ), - ("main", script_name, 26), - ], - "Task-1", - [], - ] - ], + "TaskGroup._aexit", + ] + ), + tuple( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + tuple([script_name, 26, "main"]), + ), + ) ], - ], - ] - self.assertEqual(stack_trace, expected_stack_trace) + }, + ) @skip_if_not_supported @unittest.skipIf( @@ -321,7 +495,9 @@ async def main(): self.assertEqual(response, b"ready") stack_trace = get_async_stack_trace(p.pid) except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") + self.skipTest( + "Insufficient permissions to read the stack trace" + ) finally: if client_socket is not None: client_socket.close() @@ -329,19 +505,29 @@ async def main(): p.terminate() p.wait(timeout=SHORT_TIMEOUT) - # sets are unordered, so we want to sort "awaited_by"s - stack_trace[2].sort(key=lambda x: x[1]) + # For this simple asyncgen test, we only expect one task with the full coroutine stack + self.assertEqual(len(stack_trace[0].awaited_by), 1) + task = stack_trace[0].awaited_by[0] + self.assertEqual(task.task_name, "Task-1") - expected_stack_trace = [ + # Check the coroutine stack - based on actual output, only shows main + coroutine_stack = sorted( + tuple(tuple(frame) for frame in coro.call_stack) + for coro in task.coroutine_stack + ) + self.assertEqual( + coroutine_stack, [ - ("gen_nested_call", script_name, 10), - ("gen", script_name, 16), - ("main", script_name, 19), + ( + tuple([script_name, 10, "gen_nested_call"]), + tuple([script_name, 16, "gen"]), + tuple([script_name, 19, "main"]), + ) ], - "Task-1", - [], - ] - self.assertEqual(stack_trace, expected_stack_trace) + ) + + # No awaited_by relationships expected for this simple case + self.assertEqual(task.awaited_by, []) @skip_if_not_supported @unittest.skipIf( @@ -398,7 +584,9 @@ async def main(): self.assertEqual(response, b"ready") stack_trace = get_async_stack_trace(p.pid) except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") + self.skipTest( + "Insufficient permissions to read the stack trace" + ) finally: if client_socket is not None: client_socket.close() @@ -406,15 +594,73 @@ async def main(): p.terminate() p.wait(timeout=SHORT_TIMEOUT) - # sets are unordered, so we want to sort "awaited_by"s - stack_trace[2].sort(key=lambda x: x[1]) - - expected_stack_trace = [ - [("deep", script_name, 11), ("c1", script_name, 15)], - "Task-2", - [[[("main", script_name, 21)], "Task-1", []]], + # First check all the tasks are present + tasks_names = [ + task.task_name for task in stack_trace[0].awaited_by ] - self.assertEqual(stack_trace, expected_stack_trace) + for task_name in ["Task-1", "Task-2"]: + self.assertIn(task_name, tasks_names) + + # Now ensure that the awaited_by_relationships are correct + id_to_task = { + task.task_id: task for task in stack_trace[0].awaited_by + } + task_name_to_awaited_by = { + task.task_name: set( + id_to_task[awaited.task_name].task_name + for awaited in task.awaited_by + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + task_name_to_awaited_by, + { + "Task-1": set(), + "Task-2": {"Task-1"}, + }, + ) + + # Now ensure that the coroutine stacks are correct + coroutine_stacks = { + task.task_name: sorted( + tuple(tuple(frame) for frame in coro.call_stack) + for coro in task.coroutine_stack + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + coroutine_stacks, + { + "Task-1": [(tuple([script_name, 21, "main"]),)], + "Task-2": [ + ( + tuple([script_name, 11, "deep"]), + tuple([script_name, 15, "c1"]), + ) + ], + }, + ) + + # Now ensure the coroutine stacks for the awaited_by relationships are correct. + awaited_by_coroutine_stacks = { + task.task_name: sorted( + ( + id_to_task[coro.task_name].task_name, + tuple(tuple(frame) for frame in coro.call_stack), + ) + for coro in task.awaited_by + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + awaited_by_coroutine_stacks, + { + "Task-1": [], + "Task-2": [ + ("Task-1", (tuple([script_name, 21, "main"]),)) + ], + }, + ) @skip_if_not_supported @unittest.skipIf( @@ -474,7 +720,9 @@ async def main(): self.assertEqual(response, b"ready") stack_trace = get_async_stack_trace(p.pid) except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") + self.skipTest( + "Insufficient permissions to read the stack trace" + ) finally: if client_socket is not None: client_socket.close() @@ -482,27 +730,93 @@ async def main(): p.terminate() p.wait(timeout=SHORT_TIMEOUT) - # sets are unordered, so we want to sort "awaited_by"s - stack_trace[2].sort(key=lambda x: x[1]) - expected_stack_trace = [ - [ - ("deep", script_name, 11), - ("c1", script_name, 15), - ("staggered_race..run_one_coro", staggered.__file__, ANY), - ], - "Task-2", - [ - [ - [ - ("staggered_race", staggered.__file__, ANY), - ("main", script_name, 21), - ], - "Task-1", - [], - ] - ], + # First check all the tasks are present + tasks_names = [ + task.task_name for task in stack_trace[0].awaited_by ] - self.assertEqual(stack_trace, expected_stack_trace) + for task_name in ["Task-1", "Task-2"]: + self.assertIn(task_name, tasks_names) + + # Now ensure that the awaited_by_relationships are correct + id_to_task = { + task.task_id: task for task in stack_trace[0].awaited_by + } + task_name_to_awaited_by = { + task.task_name: set( + id_to_task[awaited.task_name].task_name + for awaited in task.awaited_by + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + task_name_to_awaited_by, + { + "Task-1": set(), + "Task-2": {"Task-1"}, + }, + ) + + # Now ensure that the coroutine stacks are correct + coroutine_stacks = { + task.task_name: sorted( + tuple(tuple(frame) for frame in coro.call_stack) + for coro in task.coroutine_stack + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + coroutine_stacks, + { + "Task-1": [ + ( + tuple([staggered.__file__, ANY, "staggered_race"]), + tuple([script_name, 21, "main"]), + ) + ], + "Task-2": [ + ( + tuple([script_name, 11, "deep"]), + tuple([script_name, 15, "c1"]), + tuple( + [ + staggered.__file__, + ANY, + "staggered_race..run_one_coro", + ] + ), + ) + ], + }, + ) + + # Now ensure the coroutine stacks for the awaited_by relationships are correct. + awaited_by_coroutine_stacks = { + task.task_name: sorted( + ( + id_to_task[coro.task_name].task_name, + tuple(tuple(frame) for frame in coro.call_stack), + ) + for coro in task.awaited_by + ) + for task in stack_trace[0].awaited_by + } + self.assertEqual( + awaited_by_coroutine_stacks, + { + "Task-1": [], + "Task-2": [ + ( + "Task-1", + ( + tuple( + [staggered.__file__, ANY, "staggered_race"] + ), + tuple([script_name, 21, "main"]), + ), + ) + ], + }, + ) @skip_if_not_supported @unittest.skipIf( @@ -630,62 +944,174 @@ async def main(): # expected: at least 1000 pending tasks self.assertGreaterEqual(len(entries), 1000) # the first three tasks stem from the code structure - self.assertIn((ANY, "Task-1", []), entries) main_stack = [ - ( - "TaskGroup._aexit", - taskgroups.__file__, - ANY, - ), - ( - "TaskGroup.__aexit__", - taskgroups.__file__, - ANY, + FrameInfo([taskgroups.__file__, ANY, "TaskGroup._aexit"]), + FrameInfo( + [taskgroups.__file__, ANY, "TaskGroup.__aexit__"] ), - ("main", script_name, 60), + FrameInfo([script_name, 60, "main"]), ] self.assertIn( - (ANY, "server task", [[main_stack, ANY]]), + TaskInfo( + [ANY, "Task-1", [CoroInfo([main_stack, ANY])], []] + ), entries, ) self.assertIn( - (ANY, "echo client spam", [[main_stack, ANY]]), + TaskInfo( + [ + ANY, + "server task", + [ + CoroInfo( + [ + [ + FrameInfo( + [ + base_events.__file__, + ANY, + "Server.serve_forever", + ] + ) + ], + ANY, + ] + ) + ], + [ + CoroInfo( + [ + [ + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + FrameInfo( + [script_name, ANY, "main"] + ), + ], + ANY, + ] + ) + ], + ] + ), + entries, + ) + self.assertIn( + TaskInfo( + [ + ANY, + "Task-4", + [ + CoroInfo( + [ + [ + FrameInfo( + [tasks.__file__, ANY, "sleep"] + ), + FrameInfo( + [ + script_name, + 38, + "echo_client", + ] + ), + ], + ANY, + ] + ) + ], + [ + CoroInfo( + [ + [ + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + FrameInfo( + [ + script_name, + 41, + "echo_client_spam", + ] + ), + ], + ANY, + ] + ) + ], + ] + ), entries, ) - expected_stack = [ - [ + expected_awaited_by = [ + CoroInfo( [ - ( - "TaskGroup._aexit", - taskgroups.__file__, - ANY, - ), - ( - "TaskGroup.__aexit__", - taskgroups.__file__, - ANY, - ), - ("echo_client_spam", script_name, 41), - ], - ANY, - ] + [ + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup._aexit", + ] + ), + FrameInfo( + [ + taskgroups.__file__, + ANY, + "TaskGroup.__aexit__", + ] + ), + FrameInfo( + [script_name, 41, "echo_client_spam"] + ), + ], + ANY, + ] + ) ] - tasks_with_stack = [ - task for task in entries if task[2] == expected_stack + tasks_with_awaited = [ + task + for task in entries + if task.awaited_by == expected_awaited_by ] - self.assertGreaterEqual(len(tasks_with_stack), 1000) + self.assertGreaterEqual(len(tasks_with_awaited), 1000) # the final task will have some random number, but it should for # sure be one of the echo client spam horde (In windows this is not true # for some reason) if sys.platform != "win32": self.assertEqual( - expected_stack, - entries[-1][2], + tasks_with_awaited[-1].awaited_by, + entries[-1].awaited_by, ) except PermissionError: - self.skipTest("Insufficient permissions to read the stack trace") + self.skipTest( + "Insufficient permissions to read the stack trace" + ) finally: if client_socket is not None: client_socket.close() @@ -700,15 +1126,179 @@ async def main(): ) def test_self_trace(self): stack_trace = get_stack_trace(os.getpid()) + # Is possible that there are more threads, so we check that the + # expected stack traces are in the result (looking at you Windows!) + this_tread_stack = None + for thread_id, stack in stack_trace: + if thread_id == threading.get_native_id(): + this_tread_stack = stack + break + self.assertIsNotNone(this_tread_stack) self.assertEqual( - stack_trace[0], - ( - "TestGetStackTrace.test_self_trace", - __file__, - self.test_self_trace.__code__.co_firstlineno + 6, - ), + stack[:2], + [ + FrameInfo( + [ + __file__, + get_stack_trace.__code__.co_firstlineno + 2, + "get_stack_trace", + ] + ), + FrameInfo( + [ + __file__, + self.test_self_trace.__code__.co_firstlineno + 6, + "TestGetStackTrace.test_self_trace", + ] + ), + ], ) + @skip_if_not_supported + @unittest.skipIf( + sys.platform == "linux" and not PROCESS_VM_READV_SUPPORTED, + "Test only runs on Linux with process_vm_readv support", + ) + @requires_gil_enabled("Free threaded builds don't have an 'active thread'") + def test_only_active_thread(self): + # Test that only_active_thread parameter works correctly + port = find_unused_port() + script = textwrap.dedent( + f"""\ + import time, sys, socket, threading + + # Connect to the test process + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.connect(('localhost', {port})) + + def worker_thread(name, barrier, ready_event): + barrier.wait() # Synchronize thread start + ready_event.wait() # Wait for main thread signal + # Sleep to keep thread alive + time.sleep(10_000) + + def main_work(): + # Do busy work to hold the GIL + sock.sendall(b"working\\n") + count = 0 + while count < 100000000: + count += 1 + if count % 10000000 == 0: + pass # Keep main thread busy + sock.sendall(b"done\\n") + + # Create synchronization primitives + num_threads = 3 + barrier = threading.Barrier(num_threads + 1) # +1 for main thread + ready_event = threading.Event() + + # Start worker threads + threads = [] + for i in range(num_threads): + t = threading.Thread(target=worker_thread, args=(f"Worker-{{i}}", barrier, ready_event)) + t.start() + threads.append(t) + + # Wait for all threads to be ready + barrier.wait() + + # Signal ready to parent process + sock.sendall(b"ready\\n") + + # Signal threads to start waiting + ready_event.set() + + # Now do busy work to hold the GIL + main_work() + """ + ) + + with os_helper.temp_dir() as work_dir: + script_dir = os.path.join(work_dir, "script_pkg") + os.mkdir(script_dir) + + # Create a socket server to communicate with the target process + server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server_socket.bind(("localhost", port)) + server_socket.settimeout(SHORT_TIMEOUT) + server_socket.listen(1) + + script_name = _make_test_script(script_dir, "script", script) + client_socket = None + try: + p = subprocess.Popen([sys.executable, script_name]) + client_socket, _ = server_socket.accept() + server_socket.close() + + # Wait for ready signal + response = b"" + while b"ready" not in response: + response += client_socket.recv(1024) + + # Wait for the main thread to start its busy work + while b"working" not in response: + response += client_socket.recv(1024) + + # Get stack trace with all threads + unwinder_all = RemoteUnwinder(p.pid, all_threads=True) + for _ in range(10): + # Wait for the main thread to start its busy work + all_traces = unwinder_all.get_stack_trace() + found = False + for thread_id, stack in all_traces: + if not stack: + continue + current_frame = stack[0] + if ( + current_frame.funcname == "main_work" + and current_frame.lineno > 15 + ): + found = True + + if found: + break + # Give a bit of time to take the next sample + time.sleep(0.1) + else: + self.fail( + "Main thread did not start its busy work on time" + ) + + # Get stack trace with only GIL holder + unwinder_gil = RemoteUnwinder(p.pid, only_active_thread=True) + gil_traces = unwinder_gil.get_stack_trace() + + except PermissionError: + self.skipTest( + "Insufficient permissions to read the stack trace" + ) + finally: + if client_socket is not None: + client_socket.close() + p.kill() + p.terminate() + p.wait(timeout=SHORT_TIMEOUT) + + # Verify we got multiple threads in all_traces + self.assertGreater( + len(all_traces), 1, "Should have multiple threads" + ) + + # Verify we got exactly one thread in gil_traces + self.assertEqual( + len(gil_traces), 1, "Should have exactly one GIL holder" + ) + + # The GIL holder should be in the all_traces list + gil_thread_id = gil_traces[0][0] + all_thread_ids = [trace[0] for trace in all_traces] + self.assertIn( + gil_thread_id, + all_thread_ids, + "GIL holder should be among all threads", + ) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index 371c63adce94123..9f4dbe7f8663475 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -33,6 +33,11 @@ CURRENT_THREAD_HEADER = fr'{CURRENT_THREAD_ID} \(most recent call first\):' +def skip_if_sanitizer_signal(signame): + return support.skip_if_sanitizer(f"TSAN/UBSan itercepts {signame}", + thread=True, ub=True) + + def expected_traceback(lineno1, lineno2, header, min_count=1): regex = header regex += ' File "", line %s in func\n' % lineno1 @@ -247,7 +252,7 @@ def test_fatal_error_c_thread(self): func='faulthandler_fatal_error_thread', py_fatal_error=True) - @support.skip_if_sanitizer("TSAN itercepts SIGABRT", thread=True) + @skip_if_sanitizer_signal("SIGABRT") def test_sigabrt(self): self.check_fatal_error(""" import faulthandler @@ -259,7 +264,7 @@ def test_sigabrt(self): @unittest.skipIf(sys.platform == 'win32', "SIGFPE cannot be caught on Windows") - @support.skip_if_sanitizer("TSAN itercepts SIGFPE", thread=True) + @skip_if_sanitizer_signal("SIGFPE") def test_sigfpe(self): self.check_fatal_error(""" import faulthandler @@ -271,7 +276,7 @@ def test_sigfpe(self): @unittest.skipIf(_testcapi is None, 'need _testcapi') @unittest.skipUnless(hasattr(signal, 'SIGBUS'), 'need signal.SIGBUS') - @support.skip_if_sanitizer("TSAN itercepts SIGBUS", thread=True) + @skip_if_sanitizer_signal("SIGBUS") @skip_segfault_on_android def test_sigbus(self): self.check_fatal_error(""" @@ -286,7 +291,7 @@ def test_sigbus(self): @unittest.skipIf(_testcapi is None, 'need _testcapi') @unittest.skipUnless(hasattr(signal, 'SIGILL'), 'need signal.SIGILL') - @support.skip_if_sanitizer("TSAN itercepts SIGILL", thread=True) + @skip_if_sanitizer_signal("SIGILL") @skip_segfault_on_android def test_sigill(self): self.check_fatal_error(""" diff --git a/Lib/test/test_fcntl.py b/Lib/test/test_fcntl.py index b84c98ef3a2972d..86c5ca4058e7d36 100644 --- a/Lib/test/test_fcntl.py +++ b/Lib/test/test_fcntl.py @@ -8,10 +8,10 @@ import sys import unittest from test.support import ( - cpython_only, get_pagesize, is_apple, requires_subprocess, verbose + cpython_only, get_pagesize, is_apple, requires_subprocess, verbose, is_emscripten ) from test.support.import_helper import import_module -from test.support.os_helper import TESTFN, unlink +from test.support.os_helper import TESTFN, unlink, make_bad_fd # Skip test if no fcntl module. @@ -151,6 +151,7 @@ def test_fcntl_64_bit(self): except OSError as exc: if exc.errno == errno.EINVAL: self.skipTest("F_NOTIFY not available by this environment") + raise fcntl.fcntl(fd, cmd, flags) finally: os.close(fd) @@ -211,6 +212,7 @@ def test_fcntl_f_getpath(self): @unittest.skipUnless( hasattr(fcntl, "F_SETPIPE_SZ") and hasattr(fcntl, "F_GETPIPE_SZ"), "F_SETPIPE_SZ and F_GETPIPE_SZ are not available on all platforms.") + @unittest.skipIf(is_emscripten, "Emscripten pipefs doesn't support these") def test_fcntl_f_pipesize(self): test_pipe_r, test_pipe_w = os.pipe() try: @@ -228,6 +230,15 @@ def test_fcntl_f_pipesize(self): os.close(test_pipe_r) os.close(test_pipe_w) + @unittest.skipUnless(hasattr(fcntl, 'F_DUPFD'), 'need fcntl.F_DUPFD') + def test_bad_fd(self): + # gh-134744: Test error handling + fd = make_bad_fd() + with self.assertRaises(OSError): + fcntl.fcntl(fd, fcntl.F_DUPFD, 0) + with self.assertRaises(OSError): + fcntl.fcntl(fd, fcntl.F_DUPFD, b'\0' * 1024) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index b340ef7ed1621c3..6524baabe7f96ff 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -245,7 +245,7 @@ def test_detached_stdin_binary_mode(self): orig_stdin = sys.stdin try: sys.stdin = BytesIO(b'spam, bacon, sausage, and spam') - self.assertFalse(hasattr(sys.stdin, 'buffer')) + self.assertNotHasAttr(sys.stdin, 'buffer') fi = FileInput(files=['-'], mode='rb') lines = list(fi) self.assertEqual(lines, [b'spam, bacon, sausage, and spam']) diff --git a/Lib/test/test_fileio.py b/Lib/test/test_fileio.py index 5a0f033ebb82d2f..e3d54f6315aade6 100644 --- a/Lib/test/test_fileio.py +++ b/Lib/test/test_fileio.py @@ -591,7 +591,7 @@ def testBytesOpen(self): try: f.write(b"abc") f.close() - with open(TESTFN_ASCII, "rb") as f: + with self.open(TESTFN_ASCII, "rb") as f: self.assertEqual(f.read(), b"abc") finally: os.unlink(TESTFN_ASCII) @@ -608,7 +608,7 @@ def testUtf8BytesOpen(self): try: f.write(b"abc") f.close() - with open(TESTFN_UNICODE, "rb") as f: + with self.open(TESTFN_UNICODE, "rb") as f: self.assertEqual(f.read(), b"abc") finally: os.unlink(TESTFN_UNICODE) @@ -692,13 +692,13 @@ def bug801631(): def testAppend(self): try: - f = open(TESTFN, 'wb') + f = self.FileIO(TESTFN, 'wb') f.write(b'spam') f.close() - f = open(TESTFN, 'ab') + f = self.FileIO(TESTFN, 'ab') f.write(b'eggs') f.close() - f = open(TESTFN, 'rb') + f = self.FileIO(TESTFN, 'rb') d = f.read() f.close() self.assertEqual(d, b'spameggs') @@ -734,6 +734,7 @@ def __setattr__(self, name, value): class COtherFileTests(OtherFileTests, unittest.TestCase): FileIO = _io.FileIO modulename = '_io' + open = _io.open @cpython_only def testInvalidFd_overflow(self): @@ -755,6 +756,7 @@ def test_open_code(self): class PyOtherFileTests(OtherFileTests, unittest.TestCase): FileIO = _pyio.FileIO modulename = '_pyio' + open = _pyio.open def test_open_code(self): # Check that the default behaviour of open_code matches diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index 237d7b5d35edd77..c03b0a09f718899 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -651,6 +651,24 @@ class F(float, H): value = F('nan') self.assertEqual(hash(value), object.__hash__(value)) + def test_issue_gh143006(self): + # When comparing negative non-integer float and int with the + # same number of bits in the integer part, __neg__() in the + # int subclass returning not an int caused an assertion error. + class EvilInt(int): + def __neg__(self): + return "" + + i = -1 << 50 + f = float(i) - 0.5 + i = EvilInt(i) + self.assertFalse(f == i) + self.assertTrue(f != i) + self.assertTrue(f < i) + self.assertTrue(f <= i) + self.assertFalse(f > i) + self.assertFalse(f >= i) + @unittest.skipUnless(hasattr(float, "__getformat__"), "requires __getformat__") class FormatFunctionsTestCase(unittest.TestCase): @@ -795,6 +813,8 @@ def test_format(self): self.assertRaises(ValueError, format, x, '.6,n') @support.requires_IEEE_754 + @unittest.skipUnless(sys.float_repr_style == 'short', + "applies only when using short float repr style") def test_format_testfile(self): with open(format_testfile, encoding="utf-8") as testfile: for line in testfile: diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index c7cc32e09490b28..1f626d87fa6c7ad 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -346,12 +346,12 @@ def __bytes__(self): testcommon(b"%s", memoryview(b"abc"), b"abc") # %a will give the equivalent of # repr(some_obj).encode('ascii', 'backslashreplace') - testcommon(b"%a", 3.14, b"3.14") + testcommon(b"%a", 3.25, b"3.25") testcommon(b"%a", b"ghi", b"b'ghi'") testcommon(b"%a", "jkl", b"'jkl'") testcommon(b"%a", "\u0544", b"'\\u0544'") # %r is an alias for %a - testcommon(b"%r", 3.14, b"3.14") + testcommon(b"%r", 3.25, b"3.25") testcommon(b"%r", b"ghi", b"b'ghi'") testcommon(b"%r", "jkl", b"'jkl'") testcommon(b"%r", "\u0544", b"'\\u0544'") @@ -407,19 +407,19 @@ def test_non_ascii(self): self.assertEqual(format("abc", "\u2007<5"), "abc\u2007\u2007") self.assertEqual(format(123, "\u2007<5"), "123\u2007\u2007") - self.assertEqual(format(12.3, "\u2007<6"), "12.3\u2007\u2007") + self.assertEqual(format(12.5, "\u2007<6"), "12.5\u2007\u2007") self.assertEqual(format(0j, "\u2007<4"), "0j\u2007\u2007") self.assertEqual(format(1+2j, "\u2007<8"), "(1+2j)\u2007\u2007") self.assertEqual(format("abc", "\u2007>5"), "\u2007\u2007abc") self.assertEqual(format(123, "\u2007>5"), "\u2007\u2007123") - self.assertEqual(format(12.3, "\u2007>6"), "\u2007\u200712.3") + self.assertEqual(format(12.5, "\u2007>6"), "\u2007\u200712.5") self.assertEqual(format(1+2j, "\u2007>8"), "\u2007\u2007(1+2j)") self.assertEqual(format(0j, "\u2007>4"), "\u2007\u20070j") self.assertEqual(format("abc", "\u2007^5"), "\u2007abc\u2007") self.assertEqual(format(123, "\u2007^5"), "\u2007123\u2007") - self.assertEqual(format(12.3, "\u2007^6"), "\u200712.3\u2007") + self.assertEqual(format(12.5, "\u2007^6"), "\u200712.5\u2007") self.assertEqual(format(1+2j, "\u2007^8"), "\u2007(1+2j)\u2007") self.assertEqual(format(0j, "\u2007^4"), "\u20070j\u2007") diff --git a/Lib/test/test_fractions.py b/Lib/test/test_fractions.py index 84faa63606439e1..cf42b86358dbcad 100644 --- a/Lib/test/test_fractions.py +++ b/Lib/test/test_fractions.py @@ -1,7 +1,7 @@ """Tests for Lib/fractions.py.""" from decimal import Decimal -from test.support import requires_IEEE_754 +from test.support import requires_IEEE_754, adjust_int_max_str_digits import math import numbers import operator @@ -395,12 +395,14 @@ class B(metaclass=M): def testFromString(self): self.assertEqual((5, 1), _components(F("5"))) + self.assertEqual((5, 1), _components(F("005"))) self.assertEqual((3, 2), _components(F("3/2"))) self.assertEqual((3, 2), _components(F("3 / 2"))) self.assertEqual((3, 2), _components(F(" \n +3/2"))) self.assertEqual((-3, 2), _components(F("-3/2 "))) - self.assertEqual((13, 2), _components(F(" 013/02 \n "))) + self.assertEqual((13, 2), _components(F(" 0013/002 \n "))) self.assertEqual((16, 5), _components(F(" 3.2 "))) + self.assertEqual((16, 5), _components(F("003.2"))) self.assertEqual((-16, 5), _components(F(" -3.2 "))) self.assertEqual((-3, 1), _components(F(" -3. "))) self.assertEqual((3, 5), _components(F(" .6 "))) @@ -419,116 +421,102 @@ def testFromString(self): self.assertRaisesMessage( ZeroDivisionError, "Fraction(3, 0)", F, "3/0") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '3/'", - F, "3/") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '/2'", - F, "/2") - self.assertRaisesMessage( - # Denominators don't need a sign. - ValueError, "Invalid literal for Fraction: '3/+2'", - F, "3/+2") - self.assertRaisesMessage( - # Imitate float's parsing. - ValueError, "Invalid literal for Fraction: '+ 3/2'", - F, "+ 3/2") - self.assertRaisesMessage( - # Avoid treating '.' as a regex special character. - ValueError, "Invalid literal for Fraction: '3a2'", - F, "3a2") - self.assertRaisesMessage( - # Don't accept combinations of decimals and rationals. - ValueError, "Invalid literal for Fraction: '3/7.2'", - F, "3/7.2") - self.assertRaisesMessage( - # Don't accept combinations of decimals and rationals. - ValueError, "Invalid literal for Fraction: '3.2/7'", - F, "3.2/7") - self.assertRaisesMessage( - # Allow 3. and .3, but not . - ValueError, "Invalid literal for Fraction: '.'", - F, ".") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '_'", - F, "_") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '_1'", - F, "_1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1__2'", - F, "1__2") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '/_'", - F, "/_") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1_/'", - F, "1_/") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '_1/'", - F, "_1/") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1__2/'", - F, "1__2/") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/_'", - F, "1/_") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/_1'", - F, "1/_1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/1__2'", - F, "1/1__2") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1._111'", - F, "1._111") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1.1__1'", - F, "1.1__1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1.1e+_1'", - F, "1.1e+_1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1.1e+1__1'", - F, "1.1e+1__1") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '123.dd'", - F, "123.dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '123.5_dd'", - F, "123.5_dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: 'dd.5'", - F, "dd.5") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '7_dd'", - F, "7_dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/dd'", - F, "1/dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/123_dd'", - F, "1/123_dd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '789edd'", - F, "789edd") - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '789e2_dd'", - F, "789e2_dd") + + def check_invalid(s): + msg = "Invalid literal for Fraction: " + repr(s) + self.assertRaisesMessage(ValueError, msg, F, s) + + check_invalid("3/") + check_invalid("/2") + # Denominators don't need a sign. + check_invalid("3/+2") + check_invalid("3/-2") + # Imitate float's parsing. + check_invalid("+ 3/2") + check_invalid("- 3/2") + # Avoid treating '.' as a regex special character. + check_invalid("3a2") + # Don't accept combinations of decimals and rationals. + check_invalid("3/7.2") + check_invalid("3.2/7") + # No space around dot. + check_invalid("3 .2") + check_invalid("3. 2") + # No space around e. + check_invalid("3.2 e1") + check_invalid("3.2e 1") + # Fractional part don't need a sign. + check_invalid("3.+2") + check_invalid("3.-2") + # Only accept base 10. + check_invalid("0x10") + check_invalid("0x10/1") + check_invalid("1/0x10") + check_invalid("0x10.") + check_invalid("0x10.1") + check_invalid("1.0x10") + check_invalid("1.0e0x10") + # Only accept decimal digits. + check_invalid("³") + check_invalid("³/2") + check_invalid("3/²") + check_invalid("³.2") + check_invalid("3.²") + check_invalid("3.2e²") + check_invalid("¼") + # Allow 3. and .3, but not . + check_invalid(".") + check_invalid("_") + check_invalid("_1") + check_invalid("1__2") + check_invalid("/_") + check_invalid("1_/") + check_invalid("_1/") + check_invalid("1__2/") + check_invalid("1/_") + check_invalid("1/_1") + check_invalid("1/1__2") + check_invalid("1._111") + check_invalid("1.1__1") + check_invalid("1.1e+_1") + check_invalid("1.1e+1__1") + check_invalid("123.dd") + check_invalid("123.5_dd") + check_invalid("dd.5") + check_invalid("7_dd") + check_invalid("1/dd") + check_invalid("1/123_dd") + check_invalid("789edd") + check_invalid("789e2_dd") # Test catastrophic backtracking. val = "9"*50 + "_" - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '" + val + "'", - F, val) - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1/" + val + "'", - F, "1/" + val) - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1." + val + "'", - F, "1." + val) - self.assertRaisesMessage( - ValueError, "Invalid literal for Fraction: '1.1+e" + val + "'", - F, "1.1+e" + val) + check_invalid(val) + check_invalid("1/" + val) + check_invalid("1." + val) + check_invalid("." + val) + check_invalid("1.1+e" + val) + check_invalid("1.1e" + val) + + def test_limit_int(self): + maxdigits = 5000 + with adjust_int_max_str_digits(maxdigits): + msg = 'Exceeds the limit' + val = '1' * maxdigits + num = (10**maxdigits - 1)//9 + self.assertEqual((num, 1), _components(F(val))) + self.assertRaisesRegex(ValueError, msg, F, val + '1') + self.assertEqual((num, 2), _components(F(val + '/2'))) + self.assertRaisesRegex(ValueError, msg, F, val + '1/2') + self.assertEqual((1, num), _components(F('1/' + val))) + self.assertRaisesRegex(ValueError, msg, F, '1/1' + val) + self.assertEqual(((10**(maxdigits+1) - 1)//9, 10**maxdigits), + _components(F('1.' + val))) + self.assertRaisesRegex(ValueError, msg, F, '1.1' + val) + self.assertEqual((num, 10**maxdigits), _components(F('.' + val))) + self.assertRaisesRegex(ValueError, msg, F, '.1' + val) + self.assertRaisesRegex(ValueError, msg, F, '1.1e1' + val) + self.assertEqual((11, 10), _components(F('1.1e' + '0' * maxdigits))) + self.assertRaisesRegex(ValueError, msg, F, '1.1e' + '0' * (maxdigits+1)) def testImmutable(self): r = F(7, 3) @@ -1334,6 +1322,8 @@ def test_format_e_presentation_type(self): # Thousands separators (F('1234567.123456'), ',.5e', '1.23457e+06'), (F('123.123456'), '012_.2e', '0_001.23e+02'), + # Thousands separators for fractional part (or for integral too) + (F('1234567.123456'), '.5_e', '1.234_57e+06'), # z flag is legal, but never makes a difference to the output (F(-1, 7**100), 'z.6e', '-3.091690e-85'), ] @@ -1459,6 +1449,12 @@ def test_format_f_presentation_type(self): (F('1234567'), ',.2f', '1,234,567.00'), (F('12345678'), ',.2f', '12,345,678.00'), (F('12345678'), ',f', '12,345,678.000000'), + # Thousands separators for fractional part (or for integral too) + (F('123456.789123123'), '._f', '123456.789_123'), + (F('123456.789123123'), '.7_f', '123456.789_123_1'), + (F('123456.789123123'), '.9_f', '123456.789_123_123'), + (F('123456.789123123'), '.,f', '123456.789,123'), + (F('123456.789123123'), '_.,f', '123_456.789,123'), # Underscore as thousands separator (F(2, 3), '_.2f', '0.67'), (F(2, 3), '_.7f', '0.6666667'), @@ -1492,11 +1488,8 @@ def test_format_f_presentation_type(self): (F('-1234.5678'), '08,.0f', '-001,235'), (F('-1234.5678'), '09,.0f', '-0,001,235'), # Corner-case - zero-padding specified through fill and align - # instead of the zero-pad character - in this case, treat '0' as a - # regular fill character and don't attempt to insert commas into - # the filled portion. This differs from the int and float - # behaviour. - (F('1234.5678'), '0=12,.2f', '00001,234.57'), + # instead of the zero-pad character. + (F('1234.5678'), '0=12,.2f', '0,001,234.57'), # Corner case where it's not clear whether the '0' indicates zero # padding or gives the minimum width, but there's still an obvious # answer to give. We want this to work in case the minimum width @@ -1530,6 +1523,8 @@ def test_format_f_presentation_type(self): (F(51, 1000), '.1f', '0.1'), (F(149, 1000), '.1f', '0.1'), (F(151, 1000), '.1f', '0.2'), + (F(22, 7), '.02f', '3.14'), # issue gh-130662 + (F(22, 7), '005.02f', '03.14'), ] for fraction, spec, expected in testcases: with self.subTest(fraction=fraction, spec=spec): @@ -1628,17 +1623,16 @@ def test_invalid_formats(self): '=010%', '>00.2f', '>00f', - # Too many zeros - minimum width should not have leading zeros - '006f', - # Leading zeros in precision - '.010f', - '.02f', - '.000f', # Missing precision '.e', '.f', '.g', '.%', + # Thousands separators before precision + '._6e', + '._6f', + '._6g', + '._6%', # Z instead of z for negative zero suppression 'Z.2f' # z flag not supported for general formatting diff --git a/Lib/test/test_free_threading/test_bisect.py b/Lib/test/test_free_threading/test_bisect.py new file mode 100644 index 000000000000000..bd7732da0352570 --- /dev/null +++ b/Lib/test/test_free_threading/test_bisect.py @@ -0,0 +1,56 @@ +import unittest +from test.support import import_helper, threading_helper +import random + +py_bisect = import_helper.import_fresh_module('bisect', blocked=['_bisect']) +c_bisect = import_helper.import_fresh_module('bisect', fresh=['_bisect']) + + +NTHREADS = 4 +OBJECT_COUNT = 500 + + +class TestBase: + def do_racing_insort(self, insert_method): + def insert(data): + for _ in range(OBJECT_COUNT): + x = random.randint(-OBJECT_COUNT, OBJECT_COUNT) + insert_method(data, x) + + data = list(range(OBJECT_COUNT)) + threading_helper.run_concurrently( + worker_func=insert, args=(data,), nthreads=NTHREADS + ) + if False: + # These functions are not thread-safe and so the list can become + # unsorted. However, we don't want Python to crash if these + # functions are used concurrently on the same sequence. This + # should also not produce any TSAN warnings. + self.assertTrue(self.is_sorted_ascending(data)) + + def test_racing_insert_right(self): + self.do_racing_insort(self.mod.insort_right) + + def test_racing_insert_left(self): + self.do_racing_insort(self.mod.insort_left) + + @staticmethod + def is_sorted_ascending(lst): + """ + Check if the list is sorted in ascending order (non-decreasing). + """ + return all(lst[i - 1] <= lst[i] for i in range(1, len(lst))) + + +@threading_helper.requires_working_threading() +class TestPyBisect(unittest.TestCase, TestBase): + mod = py_bisect + + +@threading_helper.requires_working_threading() +class TestCBisect(unittest.TestCase, TestBase): + mod = c_bisect + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_capi.py b/Lib/test/test_free_threading/test_capi.py new file mode 100644 index 000000000000000..146d7cfc97adb7d --- /dev/null +++ b/Lib/test/test_free_threading/test_capi.py @@ -0,0 +1,47 @@ +import ctypes +import sys +import unittest + +from test.support import threading_helper +from test.support.threading_helper import run_concurrently + + +_PyImport_AddModuleRef = ctypes.pythonapi.PyImport_AddModuleRef +_PyImport_AddModuleRef.argtypes = (ctypes.c_char_p,) +_PyImport_AddModuleRef.restype = ctypes.py_object + + +@threading_helper.requires_working_threading() +class TestImportCAPI(unittest.TestCase): + def test_pyimport_addmoduleref_thread_safe(self): + # gh-137422: Concurrent calls to PyImport_AddModuleRef with the same + # module name must return the same module object. + + NUM_ITERS = 10 + NTHREADS = 4 + + module_name = f"test_free_threading_addmoduleref_{id(self)}" + module_name_bytes = module_name.encode() + sys.modules.pop(module_name, None) + results = [] + + def worker(): + module = _PyImport_AddModuleRef(module_name_bytes) + results.append(module) + + for _ in range(NUM_ITERS): + try: + run_concurrently(worker_func=worker, nthreads=NTHREADS) + self.assertEqual(len(results), NTHREADS) + reference = results[0] + for module in results[1:]: + self.assertIs(module, reference) + self.assertIn(module_name, sys.modules) + self.assertIs(sys.modules[module_name], reference) + finally: + results.clear() + sys.modules.pop(module_name, None) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_code.py b/Lib/test/test_free_threading/test_code.py index a5136a3ba4edc75..2fc5eea3773c399 100644 --- a/Lib/test/test_free_threading/test_code.py +++ b/Lib/test/test_free_threading/test_code.py @@ -1,9 +1,41 @@ import unittest +try: + import ctypes +except ImportError: + ctypes = None + from threading import Thread from unittest import TestCase from test.support import threading_helper +from test.support.threading_helper import run_concurrently + +if ctypes is not None: + capi = ctypes.pythonapi + + freefunc = ctypes.CFUNCTYPE(None, ctypes.c_voidp) + + RequestCodeExtraIndex = capi.PyUnstable_Eval_RequestCodeExtraIndex + RequestCodeExtraIndex.argtypes = (freefunc,) + RequestCodeExtraIndex.restype = ctypes.c_ssize_t + + SetExtra = capi.PyUnstable_Code_SetExtra + SetExtra.argtypes = (ctypes.py_object, ctypes.c_ssize_t, ctypes.c_voidp) + SetExtra.restype = ctypes.c_int + + GetExtra = capi.PyUnstable_Code_GetExtra + GetExtra.argtypes = ( + ctypes.py_object, + ctypes.c_ssize_t, + ctypes.POINTER(ctypes.c_voidp), + ) + GetExtra.restype = ctypes.c_int + +# Note: each call to RequestCodeExtraIndex permanently allocates a slot +# (the counter is monotonically increasing), up to MAX_CO_EXTRA_USERS (255). +NTHREADS = 20 + @threading_helper.requires_working_threading() class TestCode(TestCase): @@ -25,6 +57,83 @@ def run_in_thread(): for thread in threads: thread.join() + @unittest.skipUnless(ctypes, "ctypes is required") + def test_request_code_extra_index_concurrent(self): + """Test concurrent calls to RequestCodeExtraIndex""" + results = [] + + def worker(): + idx = RequestCodeExtraIndex(freefunc(0)) + self.assertGreaterEqual(idx, 0) + results.append(idx) + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + + # Every thread must get a unique index. + self.assertEqual(len(results), NTHREADS) + self.assertEqual(len(set(results)), NTHREADS) + + @unittest.skipUnless(ctypes, "ctypes is required") + def test_code_extra_all_ops_concurrent(self): + """Test concurrent RequestCodeExtraIndex + SetExtra + GetExtra""" + LOOP = 100 + + def f(): + pass + + code = f.__code__ + + def worker(): + idx = RequestCodeExtraIndex(freefunc(0)) + self.assertGreaterEqual(idx, 0) + + for i in range(LOOP): + ret = SetExtra(code, idx, ctypes.c_voidp(i + 1)) + self.assertEqual(ret, 0) + + for _ in range(LOOP): + extra = ctypes.c_voidp() + ret = GetExtra(code, idx, extra) + self.assertEqual(ret, 0) + # The slot was set by this thread, so the value must + # be the last one written. + self.assertEqual(extra.value, LOOP) + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + + @unittest.skipUnless(ctypes, "ctypes is required") + def test_code_extra_set_get_concurrent(self): + """Test concurrent SetExtra + GetExtra on a shared index""" + LOOP = 100 + + def f(): + pass + + code = f.__code__ + + idx = RequestCodeExtraIndex(freefunc(0)) + self.assertGreaterEqual(idx, 0) + + def worker(): + for i in range(LOOP): + ret = SetExtra(code, idx, ctypes.c_voidp(i + 1)) + self.assertEqual(ret, 0) + + for _ in range(LOOP): + extra = ctypes.c_voidp() + ret = GetExtra(code, idx, extra) + self.assertEqual(ret, 0) + # Value is set by any writer thread. + self.assertTrue(1 <= extra.value <= LOOP) + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + + # Every thread's last write is LOOP, so the final value must be LOOP. + extra = ctypes.c_voidp() + ret = GetExtra(code, idx, extra) + self.assertEqual(ret, 0) + self.assertEqual(extra.value, LOOP) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_free_threading/test_collections.py b/Lib/test/test_free_threading/test_collections.py new file mode 100644 index 000000000000000..849b0480e232fc2 --- /dev/null +++ b/Lib/test/test_free_threading/test_collections.py @@ -0,0 +1,53 @@ +import unittest +from collections import deque +from copy import copy +from test.support import threading_helper + +threading_helper.requires_working_threading(module=True) + + +class TestDeque(unittest.TestCase): + def test_copy_race(self): + # gh-144809: Test that deque copy is thread safe. It previously + # could raise a "deque mutated during iteration" error. + d = deque(range(100)) + + def mutate(): + for i in range(1000): + d.append(i) + if len(d) > 200: + d.popleft() + + def copy_loop(): + for _ in range(1000): + copy(d) + + threading_helper.run_concurrently([mutate, copy_loop]) + + def test_index_race_in_ac(self): + # gh-150750: There was a c_default specified as `Py_SIZE(self)`, + # it was used without a critical section. + + d = deque(range(100)) + + def index(): + for _ in range(10000): + try: + d.index(50) + except ValueError: + pass + + def mutate(): + for _ in range(10000): + d.append(0) + d.clear() + d.extend(range(100)) + d.appendleft(-1) + + threading_helper.run_concurrently( + [index, *[mutate for _ in range(3)]], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_cprofile.py b/Lib/test/test_free_threading/test_cprofile.py new file mode 100644 index 000000000000000..361b800d6b90295 --- /dev/null +++ b/Lib/test/test_free_threading/test_cprofile.py @@ -0,0 +1,43 @@ +import unittest + +from test.support import threading_helper + +import cProfile +import pstats + + +NTHREADS = 10 +INSERT_PER_THREAD = 1000 + + +@threading_helper.requires_working_threading() +class TestCProfile(unittest.TestCase): + def test_cprofile_racing_list_insert(self): + def list_insert(lst): + for i in range(INSERT_PER_THREAD): + lst.insert(0, i) + + lst = [] + + with cProfile.Profile() as pr: + threading_helper.run_concurrently( + worker_func=list_insert, nthreads=NTHREADS, args=(lst,) + ) + pr.create_stats() + ps = pstats.Stats(pr) + stats_profile = ps.get_stats_profile() + list_insert_profile = stats_profile.func_profiles[ + "" + ] + # Even though there is no explicit recursive call to insert, + # cProfile may record some calls as recursive due to limitations + # in its handling of multithreaded programs. This issue is not + # directly related to FT Python itself; however, it tends to be + # more noticeable when using FT Python. Therefore, consider only + # the calls section and disregard the recursive part. + list_insert_ncalls = list_insert_profile.ncalls.split("/")[0] + self.assertEqual( + int(list_insert_ncalls), NTHREADS * INSERT_PER_THREAD + ) + + self.assertEqual(len(lst), NTHREADS * INSERT_PER_THREAD) diff --git a/Lib/test/test_free_threading/test_csv.py b/Lib/test/test_free_threading/test_csv.py new file mode 100644 index 000000000000000..beb4510a1281b82 --- /dev/null +++ b/Lib/test/test_free_threading/test_csv.py @@ -0,0 +1,50 @@ +import csv +import io +import unittest + +from test.support import threading_helper +from test.support.threading_helper import run_concurrently + + +NTHREADS = 10 + + +@threading_helper.requires_working_threading() +class TestCSV(unittest.TestCase): + def test_concurrent_reader_next(self): + input_rows = [f"{i},{i},{i}" for i in range(50)] + input_stream = io.StringIO("\n".join(input_rows)) + reader = csv.reader(input_stream) + output_rows = [] + + def read_row(): + for row in reader: + self.assertEqual(len(row), 3) + output_rows.append(",".join(row)) + + run_concurrently(worker_func=read_row, nthreads=NTHREADS) + self.assertSetEqual(set(input_rows), set(output_rows)) + + def test_concurrent_writer_writerow(self): + output_stream = io.StringIO() + writer = csv.writer(output_stream) + row_per_thread = 10 + expected_rows = [] + + def write_row(): + for i in range(row_per_thread): + writer.writerow([i, i, i]) + expected_rows.append(f"{i},{i},{i}") + + run_concurrently(worker_func=write_row, nthreads=NTHREADS) + + # Rewind to the start of the stream and parse the rows + output_stream.seek(0) + output_rows = [line.strip() for line in output_stream.readlines()] + + self.assertEqual(len(output_rows), NTHREADS * row_per_thread) + self.assertListEqual(sorted(output_rows), sorted(expected_rows)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_dbm_gnu.py b/Lib/test/test_free_threading/test_dbm_gnu.py new file mode 100644 index 000000000000000..d2d7b78be712f7b --- /dev/null +++ b/Lib/test/test_free_threading/test_dbm_gnu.py @@ -0,0 +1,79 @@ +import unittest + +from test.support import import_helper, os_helper, threading_helper +from test.support.threading_helper import run_concurrently + +import threading + +gdbm = import_helper.import_module("dbm.gnu") + +NTHREADS = 10 +KEY_PER_THREAD = 1000 + +gdbm_filename = "test_gdbm_file" + + +@threading_helper.requires_working_threading() +class TestGdbm(unittest.TestCase): + def test_racing_dbm_gnu(self): + def gdbm_multi_op_worker(db): + # Each thread sets, gets, and iterates + tid = threading.get_ident() + + # Insert keys + for i in range(KEY_PER_THREAD): + db[f"key_{tid}_{i}"] = f"value_{tid}_{i}" + + for i in range(KEY_PER_THREAD): + # Keys and values are stored as bytes; encode values for + # comparison + key = f"key_{tid}_{i}" + value = f"value_{tid}_{i}".encode() + self.assertIn(key, db) + self.assertEqual(db[key], value) + self.assertEqual(db.get(key), value) + self.assertIsNone(db.get("not_exist")) + with self.assertRaises(KeyError): + db["not_exist"] + + # Iterate over the database keys and verify only those belonging + # to this thread. Other threads may concurrently delete their keys. + key_prefix = f"key_{tid}".encode() + key = db.firstkey() + key_count = 0 + while key: + if key.startswith(key_prefix): + self.assertIn(key, db) + key_count += 1 + key = db.nextkey(key) + + # Can't assert key_count == KEY_PER_THREAD because concurrent + # threads may insert or delete keys during iteration. This can + # cause keys to be skipped or counted multiple times, making the + # count unreliable. + # See: https://www.gnu.org.ua/software/gdbm/manual/Sequential.html + # self.assertEqual(key_count, KEY_PER_THREAD) + + # Delete this thread's keys + for i in range(KEY_PER_THREAD): + key = f"key_{tid}_{i}" + del db[key] + self.assertNotIn(key, db) + with self.assertRaises(KeyError): + del db["not_exist"] + + # Re-insert keys + for i in range(KEY_PER_THREAD): + db[f"key_{tid}_{i}"] = f"value_{tid}_{i}" + + with os_helper.temp_dir() as tmpdirname: + db = gdbm.open(f"{tmpdirname}/{gdbm_filename}", "c") + run_concurrently( + worker_func=gdbm_multi_op_worker, nthreads=NTHREADS, args=(db,) + ) + self.assertEqual(len(db), NTHREADS * KEY_PER_THREAD) + db.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_dict.py b/Lib/test/test_free_threading/test_dict.py index 476cc3178d843f6..55272a00c3ad501 100644 --- a/Lib/test/test_free_threading/test_dict.py +++ b/Lib/test/test_free_threading/test_dict.py @@ -228,6 +228,67 @@ def reader_func(): self.assertEqual(count, 0) + def test_racing_object_get_set_dict(self): + e = Exception() + + def writer(): + for i in range(10000): + e.__dict__ = {1:2} + + def reader(): + for i in range(10000): + e.__dict__ + + t1 = Thread(target=writer) + t2 = Thread(target=reader) + + with threading_helper.start_threads([t1, t2]): + pass + + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_racing_watch_unwatch_dict(self): + # gh-148393: race between PyDict_Watch / PyDict_Unwatch + # and concurrent dict mutation reading _ma_watcher_tag. + wid = _testcapi.add_dict_watcher(0) + try: + d = {} + ITERS = 1000 + + def writer(): + for i in range(ITERS): + d[i] = i + del d[i] + + def watcher(): + for _ in range(ITERS): + _testcapi.watch_dict(wid, d) + _testcapi.unwatch_dict(wid, d) + + threading_helper.run_concurrently([writer, watcher]) + finally: + _testcapi.clear_dict_watcher(wid) + + def test_racing_dict_update_and_method_lookup(self): + # gh-144295: test race between dict modifications and method lookups. + # Uses BytesIO because the race requires a type without Py_TPFLAGS_INLINE_VALUES + # for the _PyDict_GetMethodStackRef code path. + import io + obj = io.BytesIO() + + def writer(): + for _ in range(10000): + obj.x = 1 + del obj.x + + def reader(): + for _ in range(10000): + obj.getvalue() + + t1 = Thread(target=writer) + t2 = Thread(target=reader) + + with threading_helper.start_threads([t1, t2]): + pass if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_free_threading/test_dict_watcher.py b/Lib/test/test_free_threading/test_dict_watcher.py new file mode 100644 index 000000000000000..6a6843f9344f640 --- /dev/null +++ b/Lib/test/test_free_threading/test_dict_watcher.py @@ -0,0 +1,89 @@ +import unittest + +from test.support import import_helper, threading_helper + +_testcapi = import_helper.import_module("_testcapi") + +ITERS = 100 +NTHREADS = 20 + + +@threading_helper.requires_working_threading() +class TestDictWatcherThreadSafety(unittest.TestCase): + # Watcher kinds from _testcapi + EVENTS = 0 # appends dict events as strings to global event list + + def test_concurrent_add_clear_watchers(self): + """Race AddWatcher and ClearWatcher from multiple threads. + + Uses more threads than available watcher slots (5 user slots out + of DICT_MAX_WATCHERS=8). + """ + results = [] + + def worker(): + for _ in range(ITERS): + try: + wid = _testcapi.add_dict_watcher(self.EVENTS) + except RuntimeError: + continue # All slots taken + self.assertGreaterEqual(wid, 0) + results.append(wid) + _testcapi.clear_dict_watcher(wid) + + threading_helper.run_concurrently(worker, NTHREADS) + + # Verify at least some watchers were successfully added + self.assertGreater(len(results), 0) + + def test_concurrent_watch_unwatch(self): + """Race Watch and Unwatch on the same dict from multiple threads.""" + wid = _testcapi.add_dict_watcher(self.EVENTS) + dicts = [{} for _ in range(10)] + + def worker(): + for _ in range(ITERS): + for d in dicts: + _testcapi.watch_dict(wid, d) + for d in dicts: + _testcapi.unwatch_dict(wid, d) + + try: + threading_helper.run_concurrently(worker, NTHREADS) + + # Verify watching still works after concurrent watch/unwatch + _testcapi.watch_dict(wid, dicts[0]) + dicts[0]["key"] = "value" + events = _testcapi.get_dict_watcher_events() + self.assertIn("new:key:value", events) + finally: + _testcapi.clear_dict_watcher(wid) + + def test_concurrent_modify_watched_dict(self): + """Race dict mutations (triggering callbacks) with watch/unwatch.""" + wid = _testcapi.add_dict_watcher(self.EVENTS) + d = {} + _testcapi.watch_dict(wid, d) + + def mutator(): + for i in range(ITERS): + d[f"key_{i}"] = i + d.pop(f"key_{i}", None) + + def toggler(): + for i in range(ITERS): + _testcapi.watch_dict(wid, d) + d[f"toggler_{i}"] = i + _testcapi.unwatch_dict(wid, d) + + workers = [mutator, toggler] * (NTHREADS // 2) + try: + threading_helper.run_concurrently(workers) + events = _testcapi.get_dict_watcher_events() + self.assertGreater(len(events), 0) + finally: + _testcapi.clear_dict_watcher(wid) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_exceptions.py b/Lib/test/test_free_threading/test_exceptions.py new file mode 100644 index 000000000000000..61146f29c7868e5 --- /dev/null +++ b/Lib/test/test_free_threading/test_exceptions.py @@ -0,0 +1,16 @@ +import unittest +import copy + +from test.support import threading_helper + +threading_helper.requires_working_threading(module=True) +class ExceptionTests(unittest.TestCase): + def test_setstate_data_race(self): + E = Exception() + + def func(): + for i in range(100): + setattr(E, 'x', i) + copy.copy(E) + + threading_helper.run_concurrently(func, nthreads=4) diff --git a/Lib/test/test_free_threading/test_frame.py b/Lib/test/test_free_threading/test_frame.py new file mode 100644 index 000000000000000..bea49df557aa2c5 --- /dev/null +++ b/Lib/test/test_free_threading/test_frame.py @@ -0,0 +1,151 @@ +import functools +import sys +import threading +import unittest + +from test.support import threading_helper + +threading_helper.requires_working_threading(module=True) + + +def run_with_frame(funcs, runner=None, iters=10): + """Run funcs with a frame from another thread that is currently executing. + + Args: + funcs: A function or list of functions that take a frame argument + runner: Optional function to run in the executor thread. If provided, + it will be called and should return eventually. The frame + passed to funcs will be the runner's frame. + iters: Number of iterations each func should run + """ + if not isinstance(funcs, list): + funcs = [funcs] + + frame_var = None + e = threading.Event() + b = threading.Barrier(len(funcs) + 1) + + if runner is None: + def runner(): + j = 0 + for i in range(100): + j += i + + def executor(): + nonlocal frame_var + frame_var = sys._getframe() + e.set() + b.wait() + runner() + + def func_wrapper(func): + e.wait() + frame = frame_var + b.wait() + for _ in range(iters): + func(frame) + + test_funcs = [functools.partial(func_wrapper, f) for f in funcs] + threading_helper.run_concurrently([executor] + test_funcs) + + +class TestFrameRaces(unittest.TestCase): + def test_concurrent_f_lasti(self): + run_with_frame(lambda frame: frame.f_lasti) + + def test_concurrent_f_lineno(self): + run_with_frame(lambda frame: frame.f_lineno) + + def test_concurrent_f_code(self): + run_with_frame(lambda frame: frame.f_code) + + def test_concurrent_f_back(self): + run_with_frame(lambda frame: frame.f_back) + + def test_concurrent_f_globals(self): + run_with_frame(lambda frame: frame.f_globals) + + def test_concurrent_f_builtins(self): + run_with_frame(lambda frame: frame.f_builtins) + + def test_concurrent_f_locals(self): + run_with_frame(lambda frame: frame.f_locals) + + def test_concurrent_f_trace_read(self): + run_with_frame(lambda frame: frame.f_trace) + + def test_concurrent_f_trace_opcodes_read(self): + run_with_frame(lambda frame: frame.f_trace_opcodes) + + def test_concurrent_repr(self): + run_with_frame(lambda frame: repr(frame)) + + def test_concurrent_f_trace_write(self): + def trace_func(frame, event, arg): + return trace_func + + def writer(frame): + frame.f_trace = trace_func + frame.f_trace = None + + run_with_frame(writer) + + def test_concurrent_f_trace_read_write(self): + # Test concurrent reads and writes of f_trace on a live frame. + def trace_func(frame, event, arg): + return trace_func + + def reader(frame): + _ = frame.f_trace + + def writer(frame): + frame.f_trace = trace_func + frame.f_trace = None + + run_with_frame([reader, writer, reader, writer]) + + def test_concurrent_f_trace_opcodes_write(self): + def writer(frame): + frame.f_trace_opcodes = True + frame.f_trace_opcodes = False + + run_with_frame(writer) + + def test_concurrent_f_trace_opcodes_read_write(self): + # Test concurrent reads and writes of f_trace_opcodes on a live frame. + def reader(frame): + _ = frame.f_trace_opcodes + + def writer(frame): + frame.f_trace_opcodes = True + frame.f_trace_opcodes = False + + run_with_frame([reader, writer, reader, writer]) + + def test_concurrent_frame_clear(self): + # Test race between frame.clear() and attribute reads. + def create_frame(): + x = 1 + y = 2 + return sys._getframe() + + frame = create_frame() + + def reader(): + for _ in range(10): + try: + _ = frame.f_locals + _ = frame.f_code + _ = frame.f_lineno + except ValueError: + # Frame may be cleared + pass + + def clearer(): + frame.clear() + + threading_helper.run_concurrently([reader, reader, clearer]) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_functools.py b/Lib/test/test_free_threading/test_functools.py new file mode 100644 index 000000000000000..a442fe056cef159 --- /dev/null +++ b/Lib/test/test_free_threading/test_functools.py @@ -0,0 +1,75 @@ +import random +import unittest + +from functools import lru_cache +from threading import Barrier, Thread + +from test.support import threading_helper + +@threading_helper.requires_working_threading() +class TestLRUCache(unittest.TestCase): + + def _test_concurrent_operations(self, maxsize): + num_threads = 10 + b = Barrier(num_threads) + @lru_cache(maxsize=maxsize) + def func(arg=0): + return object() + + + def thread_func(): + b.wait() + for i in range(1000): + r = random.randint(0, 1000) + if i < 800: + func(i) + elif i < 900: + func.cache_info() + else: + func.cache_clear() + + threads = [] + for i in range(num_threads): + t = Thread(target=thread_func) + threads.append(t) + + with threading_helper.start_threads(threads): + pass + + def test_concurrent_operations_unbounded(self): + self._test_concurrent_operations(maxsize=None) + + def test_concurrent_operations_bounded(self): + self._test_concurrent_operations(maxsize=128) + + def _test_reentrant_cache_clear(self, maxsize): + num_threads = 10 + b = Barrier(num_threads) + @lru_cache(maxsize=maxsize) + def func(arg=0): + func.cache_clear() + return object() + + + def thread_func(): + b.wait() + for i in range(1000): + func(random.randint(0, 10000)) + + threads = [] + for i in range(num_threads): + t = Thread(target=thread_func) + threads.append(t) + + with threading_helper.start_threads(threads): + pass + + def test_reentrant_cache_clear_unbounded(self): + self._test_reentrant_cache_clear(maxsize=None) + + def test_reentrant_cache_clear_bounded(self): + self._test_reentrant_cache_clear(maxsize=128) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_gc.py b/Lib/test/test_free_threading/test_gc.py index 3b83e0431efa6b6..399010234509408 100644 --- a/Lib/test/test_free_threading/test_gc.py +++ b/Lib/test/test_free_threading/test_gc.py @@ -62,6 +62,97 @@ def mutator_thread(): with threading_helper.start_threads(gcs + mutators): pass + def test_freeze_object_in_brc_queue(self): + # GH-142975: Freezing objects in the BRC queue could result in some + # objects having a zero refcount without being deallocated. + + class Weird: + # We need a destructor to trigger the check for object resurrection + def __del__(self): + pass + + # This is owned by the main thread, so the subthread will have to increment + # this object's reference count. + weird = Weird() + + def evil(): + gc.freeze() + + # Decrement the reference count from this thread, which will trigger the + # slow path during resurrection and add our weird object to the BRC queue. + nonlocal weird + del weird + + # Collection will merge the object's reference count and make it zero. + gc.collect() + + # Unfreeze the object, making it visible to the GC. + gc.unfreeze() + gc.collect() + + thread = Thread(target=evil) + thread.start() + thread.join() + + def test_set_threshold(self): + # GH-148613: Setting the GC threshold from another thread could cause a + # race between the `gc_should_collect` and `gc_set_threshold` functions. + NUM_THREADS = 8 + NUM_ITERS = 100_000 + barrier = threading.Barrier(NUM_THREADS) + + class CyclicReference: + def __init__(self): + self.r = self + + def allocator(): + barrier.wait() + for _ in range(NUM_ITERS): + CyclicReference() + + def setter(): + barrier.wait() + for i in range(NUM_ITERS): + gc.set_threshold(100 + (i % 100), 10 + (i % 10), 10 + (i % 10)) + + current_threshold = gc.get_threshold() + try: + threads = [Thread(target=allocator) for _ in range(NUM_THREADS - 1)] + threads.append(Thread(target=setter)) + with threading_helper.start_threads(threads): + pass + finally: + gc.set_threshold(*current_threshold) + + def test_get_count(self): + class CyclicReference: + def __init__(self): + self.ref = self + + NUM_ALLOCATORS = 7 + NUM_READERS = 1 + NUM_THREADS = NUM_ALLOCATORS + NUM_READERS + NUM_ITERS = 1000 + + barrier = threading.Barrier(NUM_THREADS) + + def allocator(): + barrier.wait() + for _ in range(NUM_ITERS): + CyclicReference() + + + def reader(): + barrier.wait() + for _ in range(NUM_ITERS): + gc.get_count() + + threads = [Thread(target=allocator) for _ in range(NUM_ALLOCATORS)] + threads.extend(Thread(target=reader) for _ in range(NUM_READERS)) + + with threading_helper.start_threads(threads): + pass + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_free_threading/test_grp.py b/Lib/test/test_free_threading/test_grp.py new file mode 100644 index 000000000000000..1a47a9757702be8 --- /dev/null +++ b/Lib/test/test_free_threading/test_grp.py @@ -0,0 +1,35 @@ +import unittest + +from test.support import import_helper, threading_helper +from test.support.threading_helper import run_concurrently + +grp = import_helper.import_module("grp") + +from test import test_grp + + +NTHREADS = 10 + + +@threading_helper.requires_working_threading() +class TestGrp(unittest.TestCase): + def setUp(self): + self.test_grp = test_grp.GroupDatabaseTestCase() + + def test_racing_test_values(self): + # test_grp.test_values() calls grp.getgrall() and checks the entries + run_concurrently( + worker_func=self.test_grp.test_values, nthreads=NTHREADS + ) + + def test_racing_test_values_extended(self): + # test_grp.test_values_extended() calls grp.getgrall(), grp.getgrgid(), + # grp.getgrnam() and checks the entries + run_concurrently( + worker_func=self.test_grp.test_values_extended, + nthreads=NTHREADS, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_heapq.py b/Lib/test/test_free_threading/test_heapq.py new file mode 100644 index 000000000000000..d771333ffcc9e0f --- /dev/null +++ b/Lib/test/test_free_threading/test_heapq.py @@ -0,0 +1,247 @@ +import unittest + +import heapq + +from enum import Enum +from threading import Barrier, Lock +from random import shuffle, randint + +from test.support import threading_helper +from test.support.threading_helper import run_concurrently +from test import test_heapq + + +NTHREADS = 10 +OBJECT_COUNT = 5_000 + + +class Heap(Enum): + MIN = 1 + MAX = 2 + + +@threading_helper.requires_working_threading() +class TestHeapq(unittest.TestCase): + def setUp(self): + self.test_heapq = test_heapq.TestHeapPython() + + def test_racing_heapify(self): + heap = list(range(OBJECT_COUNT)) + shuffle(heap) + + run_concurrently( + worker_func=heapq.heapify, nthreads=NTHREADS, args=(heap,) + ) + self.test_heapq.check_invariant(heap) + + def test_racing_heappush(self): + heap = [] + + def heappush_func(heap): + for item in reversed(range(OBJECT_COUNT)): + heapq.heappush(heap, item) + + run_concurrently( + worker_func=heappush_func, nthreads=NTHREADS, args=(heap,) + ) + self.test_heapq.check_invariant(heap) + + def test_racing_heappop(self): + heap = self.create_heap(OBJECT_COUNT, Heap.MIN) + + # Each thread pops (OBJECT_COUNT / NTHREADS) items + self.assertEqual(OBJECT_COUNT % NTHREADS, 0) + per_thread_pop_count = OBJECT_COUNT // NTHREADS + + def heappop_func(heap, pop_count): + local_list = [] + for _ in range(pop_count): + item = heapq.heappop(heap) + local_list.append(item) + + # Each local list should be sorted + self.assertTrue(self.is_sorted_ascending(local_list)) + + run_concurrently( + worker_func=heappop_func, + nthreads=NTHREADS, + args=(heap, per_thread_pop_count), + ) + self.assertEqual(len(heap), 0) + + def test_racing_heappushpop(self): + heap = self.create_heap(OBJECT_COUNT, Heap.MIN) + pushpop_items = self.create_random_list(-5_000, 10_000, OBJECT_COUNT) + + def heappushpop_func(heap, pushpop_items): + for item in pushpop_items: + popped_item = heapq.heappushpop(heap, item) + self.assertTrue(popped_item <= item) + + run_concurrently( + worker_func=heappushpop_func, + nthreads=NTHREADS, + args=(heap, pushpop_items), + ) + self.assertEqual(len(heap), OBJECT_COUNT) + self.test_heapq.check_invariant(heap) + + def test_racing_heapreplace(self): + heap = self.create_heap(OBJECT_COUNT, Heap.MIN) + replace_items = self.create_random_list(-5_000, 10_000, OBJECT_COUNT) + + def heapreplace_func(heap, replace_items): + for item in replace_items: + heapq.heapreplace(heap, item) + + run_concurrently( + worker_func=heapreplace_func, + nthreads=NTHREADS, + args=(heap, replace_items), + ) + self.assertEqual(len(heap), OBJECT_COUNT) + self.test_heapq.check_invariant(heap) + + def test_racing_heapify_max(self): + max_heap = list(range(OBJECT_COUNT)) + shuffle(max_heap) + + run_concurrently( + worker_func=heapq.heapify_max, nthreads=NTHREADS, args=(max_heap,) + ) + self.test_heapq.check_max_invariant(max_heap) + + def test_racing_heappush_max(self): + max_heap = [] + + def heappush_max_func(max_heap): + for item in range(OBJECT_COUNT): + heapq.heappush_max(max_heap, item) + + run_concurrently( + worker_func=heappush_max_func, nthreads=NTHREADS, args=(max_heap,) + ) + self.test_heapq.check_max_invariant(max_heap) + + def test_racing_heappop_max(self): + max_heap = self.create_heap(OBJECT_COUNT, Heap.MAX) + + # Each thread pops (OBJECT_COUNT / NTHREADS) items + self.assertEqual(OBJECT_COUNT % NTHREADS, 0) + per_thread_pop_count = OBJECT_COUNT // NTHREADS + + def heappop_max_func(max_heap, pop_count): + local_list = [] + for _ in range(pop_count): + item = heapq.heappop_max(max_heap) + local_list.append(item) + + # Each local list should be sorted + self.assertTrue(self.is_sorted_descending(local_list)) + + run_concurrently( + worker_func=heappop_max_func, + nthreads=NTHREADS, + args=(max_heap, per_thread_pop_count), + ) + self.assertEqual(len(max_heap), 0) + + def test_racing_heappushpop_max(self): + max_heap = self.create_heap(OBJECT_COUNT, Heap.MAX) + pushpop_items = self.create_random_list(-5_000, 10_000, OBJECT_COUNT) + + def heappushpop_max_func(max_heap, pushpop_items): + for item in pushpop_items: + popped_item = heapq.heappushpop_max(max_heap, item) + self.assertTrue(popped_item >= item) + + run_concurrently( + worker_func=heappushpop_max_func, + nthreads=NTHREADS, + args=(max_heap, pushpop_items), + ) + self.assertEqual(len(max_heap), OBJECT_COUNT) + self.test_heapq.check_max_invariant(max_heap) + + def test_racing_heapreplace_max(self): + max_heap = self.create_heap(OBJECT_COUNT, Heap.MAX) + replace_items = self.create_random_list(-5_000, 10_000, OBJECT_COUNT) + + def heapreplace_max_func(max_heap, replace_items): + for item in replace_items: + heapq.heapreplace_max(max_heap, item) + + run_concurrently( + worker_func=heapreplace_max_func, + nthreads=NTHREADS, + args=(max_heap, replace_items), + ) + self.assertEqual(len(max_heap), OBJECT_COUNT) + self.test_heapq.check_max_invariant(max_heap) + + def test_lock_free_list_read(self): + n, n_threads = 1_000, 10 + l = [] + barrier = Barrier(n_threads * 2) + + count = 0 + lock = Lock() + + def worker(): + with lock: + nonlocal count + x = count + count += 1 + + barrier.wait() + for i in range(n): + if x % 2: + heapq.heappush(l, 1) + heapq.heappop(l) + else: + try: + l[0] + except IndexError: + pass + + run_concurrently(worker, n_threads * 2) + + @staticmethod + def is_sorted_ascending(lst): + """ + Check if the list is sorted in ascending order (non-decreasing). + """ + return all(lst[i - 1] <= lst[i] for i in range(1, len(lst))) + + @staticmethod + def is_sorted_descending(lst): + """ + Check if the list is sorted in descending order (non-increasing). + """ + return all(lst[i - 1] >= lst[i] for i in range(1, len(lst))) + + @staticmethod + def create_heap(size, heap_kind): + """ + Create a min/max heap where elements are in the range (0, size - 1) and + shuffled before heapify. + """ + heap = list(range(OBJECT_COUNT)) + shuffle(heap) + if heap_kind == Heap.MIN: + heapq.heapify(heap) + else: + heapq.heapify_max(heap) + + return heap + + @staticmethod + def create_random_list(a, b, size): + """ + Create a list of random numbers between a and b (inclusive). + """ + return [randint(-a, b) for _ in range(size)] + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_io.py b/Lib/test/test_free_threading/test_io.py new file mode 100644 index 000000000000000..78ad33fe900a78b --- /dev/null +++ b/Lib/test/test_free_threading/test_io.py @@ -0,0 +1,193 @@ +import codecs +import io +import threading +from unittest import TestCase +from test.support import threading_helper +from test.support.threading_helper import run_concurrently +from random import randint +from io import BytesIO +from sys import getsizeof + +threading_helper.requires_working_threading(module=True) + + +class TestBytesIO(TestCase): + # Test pretty much everything that can break under free-threading. + # Non-deterministic, but at least one of these things will fail if + # BytesIO object is not free-thread safe. + + def check(self, funcs, *args): + barrier = threading.Barrier(len(funcs)) + threads = [] + + for func in funcs: + thread = threading.Thread(target=func, args=(barrier, *args)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + @threading_helper.requires_working_threading() + @threading_helper.reap_threads + def test_free_threading(self): + """Test for segfaults and aborts.""" + + def write(barrier, b, *ignore): + barrier.wait() + try: b.write(b'0' * randint(100, 1000)) + except ValueError: pass # ignore write fail to closed file + + def writelines(barrier, b, *ignore): + barrier.wait() + b.write(b'0\n' * randint(100, 1000)) + + def truncate(barrier, b, *ignore): + barrier.wait() + try: b.truncate(0) + except BufferError: pass # ignore exported buffer + + def read(barrier, b, *ignore): + barrier.wait() + b.read() + + def read1(barrier, b, *ignore): + barrier.wait() + b.read1() + + def readline(barrier, b, *ignore): + barrier.wait() + b.readline() + + def readlines(barrier, b, *ignore): + barrier.wait() + b.readlines() + + def readinto(barrier, b, into, *ignore): + barrier.wait() + b.readinto(into) + + def close(barrier, b, *ignore): + barrier.wait() + b.close() + + def getvalue(barrier, b, *ignore): + barrier.wait() + b.getvalue() + + def getbuffer(barrier, b, *ignore): + barrier.wait() + b.getbuffer() + + def iter(barrier, b, *ignore): + barrier.wait() + list(b) + + def getstate(barrier, b, *ignore): + barrier.wait() + b.__getstate__() + + def setstate(barrier, b, st, *ignore): + barrier.wait() + b.__setstate__(st) + + def sizeof(barrier, b, *ignore): + barrier.wait() + getsizeof(b) + + self.check([write] * 10, BytesIO()) + self.check([writelines] * 10, BytesIO()) + self.check([write] * 10 + [truncate] * 10, BytesIO()) + self.check([truncate] + [read] * 10, BytesIO(b'0\n'*204800)) + self.check([truncate] + [read1] * 10, BytesIO(b'0\n'*204800)) + self.check([truncate] + [readline] * 10, BytesIO(b'0\n'*20480)) + self.check([truncate] + [readlines] * 10, BytesIO(b'0\n'*20480)) + self.check([truncate] + [readinto] * 10, BytesIO(b'0\n'*204800), bytearray(b'0\n'*204800)) + self.check([close] + [write] * 10, BytesIO()) + self.check([truncate] + [getvalue] * 10, BytesIO(b'0\n'*204800)) + self.check([truncate] + [getbuffer] * 10, BytesIO(b'0\n'*204800)) + self.check([truncate] + [iter] * 10, BytesIO(b'0\n'*20480)) + self.check([truncate] + [getstate] * 10, BytesIO(b'0\n'*204800)) + self.check([truncate] + [setstate] * 10, BytesIO(b'0\n'*204800), (b'123', 0, None)) + self.check([truncate] + [sizeof] * 10, BytesIO(b'0\n'*204800)) + + # no tests for seek or tell because they don't break anything + + @threading_helper.requires_working_threading() + @threading_helper.reap_threads + def test_concurrent_whole_buffer_read_and_resize(self): + shared = BytesIO(b"x" * 64) + writers = 2 + readers = 8 + loops = 2000 + barrier = threading.Barrier(writers + readers) + + def writer(): + barrier.wait() + for i in range(loops): + shared.seek(0) + shared.write(b"a" * (64 + (i & 63))) + + def reader(): + barrier.wait() + for _ in range(loops): + shared.seek(0) + shared.read() + shared.seek(0) + shared.getvalue() + + threads = [threading.Thread(target=writer) for _ in range(writers)] + threads += [threading.Thread(target=reader) for _ in range(readers)] + with threading_helper.start_threads(threads): + pass + + +class IncrementalNewlineDecoderTest(TestCase): + def make_decoder(self): + utf8_decoder = codecs.getincrementaldecoder('utf-8')() + return io.IncrementalNewlineDecoder(utf8_decoder, translate=True) + + def test_concurrent_reset(self): + decoder = self.make_decoder() + + def worker(): + for _ in range(100): + decoder.reset() + + run_concurrently(worker_func=worker, nthreads=2) + + def test_concurrent_decode(self): + decoder = self.make_decoder() + + def worker(): + for _ in range(100): + decoder.decode(b"line\r\n", final=False) + + run_concurrently(worker_func=worker, nthreads=2) + + def test_concurrent_getstate_setstate(self): + decoder = self.make_decoder() + state = decoder.getstate() + + def getstate_worker(): + for _ in range(100): + decoder.getstate() + + def setstate_worker(): + for _ in range(100): + decoder.setstate(state) + + run_concurrently([getstate_worker] * 2 + [setstate_worker] * 2) + + def test_concurrent_decode_and_reset(self): + decoder = self.make_decoder() + + def decode_worker(): + for _ in range(100): + decoder.decode(b"line\r\n", final=False) + + def reset_worker(): + for _ in range(100): + decoder.reset() + + run_concurrently([decode_worker] * 2 + [reset_worker] * 2) diff --git a/Lib/test/test_free_threading/test_iteration.py b/Lib/test/test_free_threading/test_iteration.py index a51ad0cf83a0065..44d3e9ccfdd14e0 100644 --- a/Lib/test/test_free_threading/test_iteration.py +++ b/Lib/test/test_free_threading/test_iteration.py @@ -12,7 +12,7 @@ NUMITEMS = 1000 NUMTHREADS = 2 else: - NUMITEMS = 100000 + NUMITEMS = 5000 NUMTHREADS = 5 NUMMUTATORS = 2 diff --git a/Lib/test/test_free_threading/test_itertools.py b/Lib/test/test_free_threading/test_itertools.py new file mode 100644 index 000000000000000..0703c7e5e2916e8 --- /dev/null +++ b/Lib/test/test_free_threading/test_itertools.py @@ -0,0 +1,163 @@ +import unittest +from threading import Thread, Barrier +from itertools import ( + accumulate, + chain, + combinations_with_replacement, + cycle, + permutations, + tee, + zip_longest, +) +from test.support import threading_helper + + +threading_helper.requires_working_threading(module=True) + + +def work_iterator(it): + while True: + try: + next(it) + except StopIteration: + break + + +class ItertoolsThreading(unittest.TestCase): + + @threading_helper.reap_threads + def test_cycle(self): + number_of_threads = 6 + number_of_iterations = 10 + number_of_cycles = 400 + + barrier = Barrier(number_of_threads) + def work(it): + barrier.wait() + for _ in range(number_of_cycles): + _ = next(it) + + data = (1, 2, 3, 4) + for it in range(number_of_iterations): + cycle_iterator = cycle(data) + worker_threads = [] + for ii in range(number_of_threads): + worker_threads.append( + Thread(target=work, args=[cycle_iterator])) + + with threading_helper.start_threads(worker_threads): + pass + + barrier.reset() + + @threading_helper.reap_threads + def test_chain(self): + number_of_threads = 6 + number_of_iterations = 20 + + barrier = Barrier(number_of_threads) + def work(it): + barrier.wait() + while True: + try: + next(it) + except StopIteration: + break + + data = [(1, )] * 200 + for it in range(number_of_iterations): + chain_iterator = chain(*data) + worker_threads = [] + for ii in range(number_of_threads): + worker_threads.append( + Thread(target=work, args=[chain_iterator])) + + with threading_helper.start_threads(worker_threads): + pass + + barrier.reset() + + @threading_helper.reap_threads + def test_combinations_with_replacement(self): + number_of_iterations = 6 + for _ in range(number_of_iterations): + it = combinations_with_replacement(tuple(range(2)), 2) + threading_helper.run_concurrently(work_iterator, nthreads=6, args=[it]) + + @threading_helper.reap_threads + def test_permutations(self): + number_of_iterations = 6 + for _ in range(number_of_iterations): + it = permutations(tuple(range(2)), 2) + threading_helper.run_concurrently(work_iterator, nthreads=6, args=[it]) + + @threading_helper.reap_threads + def test_accumulate(self): + number_of_iterations = 10 + for _ in range(number_of_iterations): + it = accumulate(tuple(range(40))) + threading_helper.run_concurrently(work_iterator, nthreads=10, args=[it]) + + @threading_helper.reap_threads + def test_zip_longest(self): + number_of_iterations = 10 + for _ in range(number_of_iterations): + it = zip_longest(list(range(4)), list(range(8)), fillvalue=0) + threading_helper.run_concurrently(work_iterator, nthreads=10, args=[it]) + + + +class TestTeeConcurrent(unittest.TestCase): + # itertools.tee branches share a linked list of internal data cells. + # Concurrent iteration must not corrupt that shared state or crash the + # free-threaded build. A crash shows up as the interpreter dying (not as a + # caught exception); tee is documented as not thread-safe, so a + # ``RuntimeError`` from the re-entrancy guard is an allowed outcome and is + # tolerated here. + + def test_same_branch(self): + # Many threads consume the same tee branch. + errors = [] + + def consume(it): + try: + for _ in it: + pass + except RuntimeError: + pass + except Exception as e: + errors.append(e) + + for _ in range(100): + a, _ = tee(iter(range(2000)), 2) + threading_helper.run_concurrently(consume, nthreads=8, args=(a,)) + + self.assertEqual(errors, [], msg=f"unexpected errors: {errors}") + + def test_sibling_branches(self): + # Each thread consumes a different sibling branch of the same tee. + errors = [] + + def make_worker(it): + def consume(): + try: + for _ in it: + pass + except RuntimeError: + pass + except Exception as e: + errors.append(e) + + return consume + + for _ in range(100): + branches = tee(iter(range(4000)), 8) + threading_helper.run_concurrently( + [make_worker(it) for it in branches] + ) + + self.assertEqual(errors, [], msg=f"unexpected errors: {errors}") + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_itertools_combinatoric.py b/Lib/test/test_free_threading/test_itertools_combinatoric.py new file mode 100644 index 000000000000000..5b3b88deedd1210 --- /dev/null +++ b/Lib/test/test_free_threading/test_itertools_combinatoric.py @@ -0,0 +1,51 @@ +import unittest +from threading import Thread, Barrier +from itertools import combinations, product +from test.support import threading_helper + + +threading_helper.requires_working_threading(module=True) + +def test_concurrent_iteration(iterator, number_of_threads): + barrier = Barrier(number_of_threads) + def iterator_worker(it): + barrier.wait() + while True: + try: + _ = next(it) + except StopIteration: + return + + worker_threads = [] + for ii in range(number_of_threads): + worker_threads.append( + Thread(target=iterator_worker, args=[iterator])) + + with threading_helper.start_threads(worker_threads): + pass + + barrier.reset() + +class ItertoolsThreading(unittest.TestCase): + + @threading_helper.reap_threads + def test_combinations(self): + number_of_threads = 10 + number_of_iterations = 24 + + for it in range(number_of_iterations): + iterator = combinations((1, 2, 3, 4, 5), 2) + test_concurrent_iteration(iterator, number_of_threads) + + @threading_helper.reap_threads + def test_product(self): + number_of_threads = 10 + number_of_iterations = 24 + + for it in range(number_of_iterations): + iterator = product((1, 2, 3, 4, 5), (10, 20, 30)) + test_concurrent_iteration(iterator, number_of_threads) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_list.py b/Lib/test/test_free_threading/test_list.py index 44c0ac74e02aa35..4e68dde494118ef 100644 --- a/Lib/test/test_free_threading/test_list.py +++ b/Lib/test/test_free_threading/test_list.py @@ -91,6 +91,27 @@ def copy_back_and_forth(b, l): with threading_helper.start_threads(threads): pass + # gh-145036: race condition with list.__sizeof__() + def test_list_sizeof_free_threaded_build(self): + L = [] + + def mutate_function(): + for _ in range(100): + L.append(1) + L.pop() + + def size_function(): + for _ in range(100): + L.__sizeof__() + + threads = [] + for _ in range(4): + threads.append(Thread(target=mutate_function)) + threads.append(Thread(target=size_function)) + + with threading_helper.start_threads(threads): + pass + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_free_threading/test_mmap.py b/Lib/test/test_free_threading/test_mmap.py new file mode 100644 index 000000000000000..e23a0ee5da45b1d --- /dev/null +++ b/Lib/test/test_free_threading/test_mmap.py @@ -0,0 +1,270 @@ +import unittest + +from test.support import import_helper, threading_helper +from test.support.threading_helper import run_concurrently + +import os +import string +import tempfile +import threading + +from collections import Counter + +mmap = import_helper.import_module("mmap") + +NTHREADS = 10 +ANONYMOUS_MEM = -1 + + +@threading_helper.requires_working_threading() +class MmapTests(unittest.TestCase): + def test_read_and_read_byte(self): + ascii_uppercase = string.ascii_uppercase.encode() + # Choose a total mmap size that evenly divides across threads and the + # read pattern (3 bytes per loop). + mmap_size = 3 * NTHREADS * len(ascii_uppercase) + num_bytes_to_read_per_thread = mmap_size // NTHREADS + bytes_read_from_mmap = [] + + def read(mm_obj): + nread = 0 + while nread < num_bytes_to_read_per_thread: + b = mm_obj.read_byte() + bytes_read_from_mmap.append(b) + b = mm_obj.read(2) + bytes_read_from_mmap.extend(b) + nread += 3 + + with mmap.mmap(ANONYMOUS_MEM, mmap_size) as mm_obj: + for i in range(mmap_size // len(ascii_uppercase)): + mm_obj.write(ascii_uppercase) + + mm_obj.seek(0) + run_concurrently( + worker_func=read, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + self.assertEqual(len(bytes_read_from_mmap), mmap_size) + # Count each letter/byte to verify read correctness + counter = Counter(bytes_read_from_mmap) + self.assertEqual(len(counter), len(ascii_uppercase)) + # Each letter/byte should be read (3 * NTHREADS) times + for letter in ascii_uppercase: + self.assertEqual(counter[letter], 3 * NTHREADS) + + def test_readline(self): + num_lines = 1000 + lines_read_from_mmap = [] + expected_lines = [] + + def readline(mm_obj): + for i in range(num_lines // NTHREADS): + line = mm_obj.readline() + lines_read_from_mmap.append(line) + + # Allocate mmap enough for num_lines (max line 5 bytes including NL) + with mmap.mmap(ANONYMOUS_MEM, num_lines * 5) as mm_obj: + for i in range(num_lines): + line = b"%d\n" % i + mm_obj.write(line) + expected_lines.append(line) + + mm_obj.seek(0) + run_concurrently( + worker_func=readline, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + self.assertEqual(len(lines_read_from_mmap), num_lines) + # Every line should be read once by threads; order is non-deterministic + # Sort numerically by integer value + lines_read_from_mmap.sort(key=lambda x: int(x)) + self.assertEqual(lines_read_from_mmap, expected_lines) + + def test_write_and_write_byte(self): + thread_letters = list(string.ascii_uppercase) + self.assertLessEqual(NTHREADS, len(thread_letters)) + per_thread_write_loop = 100 + + def write(mm_obj): + # Each thread picks a unique letter to write + thread_letter = thread_letters.pop(0) + thread_bytes = (thread_letter * 2).encode() + for _ in range(per_thread_write_loop): + mm_obj.write_byte(thread_bytes[0]) + mm_obj.write(thread_bytes) + + with mmap.mmap( + ANONYMOUS_MEM, per_thread_write_loop * 3 * NTHREADS + ) as mm_obj: + run_concurrently( + worker_func=write, + args=(mm_obj,), + nthreads=NTHREADS, + ) + mm_obj.seek(0) + data = mm_obj.read() + self.assertEqual(len(data), NTHREADS * per_thread_write_loop * 3) + counter = Counter(data) + self.assertEqual(len(counter), NTHREADS) + # Each thread letter should be written `per_thread_write_loop` * 3 + for letter in counter: + self.assertEqual(counter[letter], per_thread_write_loop * 3) + + def test_move(self): + ascii_uppercase = string.ascii_uppercase.encode() + num_letters = len(ascii_uppercase) + + def move(mm_obj): + for i in range(num_letters): + # Move 1 byte from the first half to the second half + mm_obj.move(0 + i, num_letters + i, 1) + + with mmap.mmap(ANONYMOUS_MEM, 2 * num_letters) as mm_obj: + mm_obj.write(ascii_uppercase) + run_concurrently( + worker_func=move, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + def test_seek_and_tell(self): + seek_per_thread = 10 + + def seek(mm_obj): + self.assertTrue(mm_obj.seekable()) + for _ in range(seek_per_thread): + before_seek = mm_obj.tell() + mm_obj.seek(1, os.SEEK_CUR) + self.assertLess(before_seek, mm_obj.tell()) + + with mmap.mmap(ANONYMOUS_MEM, 1024) as mm_obj: + run_concurrently( + worker_func=seek, + args=(mm_obj,), + nthreads=NTHREADS, + ) + # Each thread seeks from current position, the end position should + # be the sum of all seeks from all threads. + self.assertEqual(mm_obj.tell(), NTHREADS * seek_per_thread) + + def test_slice_update_and_slice_read(self): + thread_letters = list(string.ascii_uppercase) + self.assertLessEqual(NTHREADS, len(thread_letters)) + + def slice_update_and_slice_read(mm_obj): + # Each thread picks a unique letter to write + thread_letter = thread_letters.pop(0) + thread_bytes = (thread_letter * 1024).encode() + for _ in range(100): + mm_obj[:] = thread_bytes + read_bytes = mm_obj[:] + # Read bytes should be all the same letter, showing no + # interleaving + self.assertTrue(all_same(read_bytes)) + + with mmap.mmap(ANONYMOUS_MEM, 1024) as mm_obj: + run_concurrently( + worker_func=slice_update_and_slice_read, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + def test_item_update_and_item_read(self): + thread_indexes = [i for i in range(NTHREADS)] + + def item_update_and_item_read(mm_obj): + # Each thread picks a unique index to write + thread_index = thread_indexes.pop() + for i in range(100): + mm_obj[thread_index] = i + self.assertEqual(mm_obj[thread_index], i) + + # Read values set by other threads, all values + # should be less than '100' + for val in mm_obj: + self.assertLess(int.from_bytes(val), 100) + + with mmap.mmap(ANONYMOUS_MEM, NTHREADS + 1) as mm_obj: + run_concurrently( + worker_func=item_update_and_item_read, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + def test_close_and_closed(self): + def close_mmap(mm_obj): + mm_obj.close() + self.assertTrue(mm_obj.closed) + + with mmap.mmap(ANONYMOUS_MEM, 1) as mm_obj: + run_concurrently( + worker_func=close_mmap, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + def test_find_and_rfind(self): + per_thread_loop = 10 + + def find_and_rfind(mm_obj): + pattern = b'Thread-Ident:"%d"' % threading.get_ident() + mm_obj.write(pattern) + for _ in range(per_thread_loop): + found_at = mm_obj.find(pattern, 0) + self.assertNotEqual(found_at, -1) + # Should not find it after the `found_at` + self.assertEqual(mm_obj.find(pattern, found_at + 1), -1) + found_at_rev = mm_obj.rfind(pattern, 0) + self.assertEqual(found_at, found_at_rev) + # Should not find it after the `found_at` + self.assertEqual(mm_obj.rfind(pattern, found_at + 1), -1) + + with mmap.mmap(ANONYMOUS_MEM, 1024) as mm_obj: + run_concurrently( + worker_func=find_and_rfind, + args=(mm_obj,), + nthreads=NTHREADS, + ) + + def test_mmap_export_as_memoryview(self): + """ + Each thread creates a memoryview and updates the internal state of the + mmap object. + """ + buffer_size = 42 + + def create_memoryview_from_mmap(mm_obj): + memoryviews = [] + for _ in range(100): + mv = memoryview(mm_obj) + memoryviews.append(mv) + self.assertEqual(len(mv), buffer_size) + self.assertEqual(mv[:7], b"CPython") + + # Cannot close the mmap while it is exported as buffers + with self.assertRaisesRegex( + BufferError, "cannot close exported pointers exist" + ): + mm_obj.close() + + with mmap.mmap(ANONYMOUS_MEM, 42) as mm_obj: + mm_obj.write(b"CPython") + run_concurrently( + worker_func=create_memoryview_from_mmap, + args=(mm_obj,), + nthreads=NTHREADS, + ) + # Implicit mm_obj.close() verifies all exports (memoryviews) are + # properly freed. + + +def all_same(lst): + return all(item == lst[0] for item in lst) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_monitoring.py b/Lib/test/test_free_threading/test_monitoring.py index a480e398722c33f..2cd6e7b035ecb41 100644 --- a/Lib/test/test_free_threading/test_monitoring.py +++ b/Lib/test/test_free_threading/test_monitoring.py @@ -2,10 +2,12 @@ environment to verify things are thread-safe in a free-threaded build""" import sys +import threading import time import unittest import weakref +from contextlib import contextmanager from sys import monitoring from test.support import threading_helper from threading import Thread, _PyRLock, Barrier @@ -33,10 +35,10 @@ def work(self, n, funcs): return n return self.work(n - 1, funcs) + self.work(n - 2, funcs) - def start_work(self, n, funcs): + def start_work(self, n, funcs, barrier): # With the GIL builds we need to make sure that the hooks have # a chance to run as it's possible to run w/o releasing the GIL. - time.sleep(0.1) + barrier.wait() self.work(n, funcs) def after_test(self): @@ -51,14 +53,16 @@ def test_instrumentation(self): exec("def f(): pass", x) funcs.append(x["f"]) + barrier = Barrier(self.thread_count + 1) threads = [] for i in range(self.thread_count): # Each thread gets a copy of the func list to avoid contention - t = Thread(target=self.start_work, args=(self.fib, list(funcs))) + t = Thread(target=self.start_work, args=(self.fib, list(funcs), barrier)) t.start() threads.append(t) self.after_threads() + barrier.wait() while True: any_alive = False @@ -71,6 +75,9 @@ def test_instrumentation(self): break self.during_threads() + # Sleep to avoid setting monitoring events too rapidly and + # overflowing the global version counter + time.sleep(0.0001) self.after_test() @@ -115,7 +122,6 @@ class MonitoringMultiThreaded( def setUp(self): super().setUp() self.set = False - self.called = False monitoring.register_callback( self.tool_id, monitoring.events.LINE, self.callback ) @@ -125,10 +131,7 @@ def tearDown(self): super().tearDown() def callback(self, *args): - self.called = True - - def after_test(self): - self.assertTrue(self.called) + pass def during_threads(self): if self.set: @@ -146,16 +149,11 @@ class SetTraceMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): def setUp(self): self.set = False - self.called = False - - def after_test(self): - self.assertTrue(self.called) def tearDown(self): sys.settrace(None) def trace_func(self, frame, event, arg): - self.called = True return self.trace_func def during_threads(self): @@ -172,16 +170,11 @@ class SetProfileMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): def setUp(self): self.set = False - self.called = False - - def after_test(self): - self.assertTrue(self.called) def tearDown(self): sys.setprofile(None) def trace_func(self, frame, event, arg): - self.called = True return self.trace_func def during_threads(self): @@ -192,6 +185,70 @@ def during_threads(self): self.set = not self.set +@threading_helper.requires_working_threading() +class SetProfileAllThreadsMultiThreaded(InstrumentationMultiThreadedMixin, TestCase): + """Uses threading.setprofile_all_threads and repeatedly toggles instrumentation on and off""" + + def setUp(self): + self.set = False + + def tearDown(self): + threading.setprofile_all_threads(None) + + def trace_func(self, frame, event, arg): + return self.trace_func + + def during_threads(self): + if self.set: + threading.setprofile_all_threads(self.trace_func) + else: + threading.setprofile_all_threads(None) + self.set = not self.set + + +class SetProfileAllMultiThreaded(TestCase): + def test_profile_all_threads(self): + done = threading.Event() + + def func(): + pass + + def bg_thread(): + while not done.is_set(): + func() + func() + func() + func() + func() + + def my_profile(frame, event, arg): + return None + + bg_threads = [] + for i in range(10): + t = threading.Thread(target=bg_thread) + t.start() + bg_threads.append(t) + + for i in range(100): + threading.setprofile_all_threads(my_profile) + threading.setprofile_all_threads(None) + + done.set() + for t in bg_threads: + t.join() + + +class TraceBuf: + def __init__(self): + self.traces = [] + self.traces_lock = threading.Lock() + + def append(self, trace): + with self.traces_lock: + self.traces.append(trace) + + @threading_helper.requires_working_threading() class MonitoringMisc(MonitoringTestMixin, TestCase): def register_callback(self, barrier): @@ -246,6 +303,167 @@ def f(): finally: sys.settrace(None) + def test_toggle_setprofile_no_new_events(self): + # gh-136396: Make sure that profile functions are called for newly + # created threads when profiling is toggled but the set of monitoring + # events doesn't change + traces = [] + + def profiler(frame, event, arg): + traces.append((frame.f_code.co_name, event, arg)) + + def a(x, y): + return b(x, y) + + def b(x, y): + return max(x, y) + + sys.setprofile(profiler) + try: + a(1, 2) + finally: + sys.setprofile(None) + traces.clear() + + def thread_main(x, y): + sys.setprofile(profiler) + try: + a(x, y) + finally: + sys.setprofile(None) + t = Thread(target=thread_main, args=(100, 200)) + t.start() + t.join() + + expected = [ + ("a", "call", None), + ("b", "call", None), + ("b", "c_call", max), + ("b", "c_return", max), + ("b", "return", 200), + ("a", "return", 200), + ("thread_main", "c_call", sys.setprofile), + ] + self.assertEqual(traces, expected) + + def observe_threads(self, observer, buf): + def in_child(ident): + return ident + + def child(ident): + with observer(): + in_child(ident) + + def in_parent(ident): + return ident + + def parent(barrier, ident): + barrier.wait() + with observer(): + t = Thread(target=child, args=(ident,)) + t.start() + t.join() + in_parent(ident) + + num_threads = 5 + barrier = Barrier(num_threads) + threads = [] + for i in range(num_threads): + t = Thread(target=parent, args=(barrier, i)) + t.start() + threads.append(t) + for t in threads: + t.join() + + for i in range(num_threads): + self.assertIn(("in_parent", "return", i), buf.traces) + self.assertIn(("in_child", "return", i), buf.traces) + + def test_profile_threads(self): + buf = TraceBuf() + + def profiler(frame, event, arg): + buf.append((frame.f_code.co_name, event, arg)) + + @contextmanager + def profile(): + sys.setprofile(profiler) + try: + yield + finally: + sys.setprofile(None) + + self.observe_threads(profile, buf) + + def test_trace_threads(self): + buf = TraceBuf() + + def tracer(frame, event, arg): + buf.append((frame.f_code.co_name, event, arg)) + return tracer + + @contextmanager + def trace(): + sys.settrace(tracer) + try: + yield + finally: + sys.settrace(None) + + self.observe_threads(trace, buf) + + def test_monitor_threads(self): + buf = TraceBuf() + + def monitor_py_return(code, off, retval): + buf.append((code.co_name, "return", retval)) + + monitoring.register_callback( + self.tool_id, monitoring.events.PY_RETURN, monitor_py_return + ) + + monitoring.set_events( + self.tool_id, monitoring.events.PY_RETURN + ) + + @contextmanager + def noop(): + yield + + self.observe_threads(noop, buf) + + def test_trace_concurrent(self): + # Test calling a function concurrently from a tracing and a non-tracing + # thread + b = threading.Barrier(2) + + def func(): + for _ in range(100): + pass + + def noop(): + pass + + def bg_thread(): + b.wait() + func() # this may instrument `func` + + def tracefunc(frame, event, arg): + # These calls run under tracing can race with the background thread + for _ in range(10): + func() + return tracefunc + + t = Thread(target=bg_thread) + t.start() + try: + sys.settrace(tracefunc) + b.wait() + noop() + finally: + sys.settrace(None) + t.join() + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_free_threading/test_pickle.py b/Lib/test/test_free_threading/test_pickle.py new file mode 100644 index 000000000000000..45ea1bf5f26465a --- /dev/null +++ b/Lib/test/test_free_threading/test_pickle.py @@ -0,0 +1,78 @@ +import pickle +import threading +import unittest + +from test.support import threading_helper + + +@threading_helper.requires_working_threading() +class TestPickleFreeThreading(unittest.TestCase): + + def test_pickle_dumps_with_concurrent_dict_mutation(self): + # gh-146452: Pickling a dict while another thread mutates it + # used to segfault. batch_dict_exact() iterated dict items via + # PyDict_Next() which returns borrowed references, and a + # concurrent pop/replace could free the value before Py_INCREF + # got to it. + shared = {str(i): list(range(20)) for i in range(50)} + + def dumper(): + for _ in range(1000): + try: + pickle.dumps(shared) + except RuntimeError: + # "dictionary changed size during iteration" is expected + pass + + def mutator(): + for j in range(1000): + key = str(j % 50) + shared[key] = list(range(j % 20)) + if j % 10 == 0: + shared.pop(key, None) + shared[key] = [j] + + threads = [] + for _ in range(10): + threads.append(threading.Thread(target=dumper)) + threads.append(threading.Thread(target=mutator)) + + with threading_helper.start_threads(threads): + pass + + def test_pickle_dumps_with_concurrent_list_mutations(self): + # gh-149816: Pickling a list while another thread mutates it + # used to be a UAF in free-threaded mode. batch_list_exact() + # used PyList_GET_ITEM (borrowed) followed by Py_INCREF, and a + # concurrent replace/pop could free the item between those two + # operations. + shared = [list(range(20)) for _ in range(50)] + + def dumper(): + for _ in range(1000): + try: + pickle.dumps(shared) + except (RuntimeError, IndexError): + pass + + def mutator(): + for i in range(1000): + idx = i % 50 + shared[idx] = list(range(i % 20)) + if i % 10 == 0: + try: + shared.pop() + except IndexError: + pass + shared.append([i]) + + threads = [] + for _ in range(10): + threads.append(threading.Thread(target=dumper)) + threads.append(threading.Thread(target=mutator)) + + with threading_helper.start_threads(threads): + pass + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_re.py b/Lib/test/test_free_threading/test_re.py new file mode 100644 index 000000000000000..56f25045d1bf8ea --- /dev/null +++ b/Lib/test/test_free_threading/test_re.py @@ -0,0 +1,62 @@ +import re +import unittest + +from test.support import threading_helper +from test.support.threading_helper import run_concurrently + + +NTHREADS = 10 + + +@threading_helper.requires_working_threading() +class TestRe(unittest.TestCase): + def test_pattern_sub(self): + """Pattern substitution should work across threads""" + pattern = re.compile(r"\w+@\w+\.\w+") + text = "e-mail: test@python.org or user@pycon.org. " * 5 + results = [] + + def worker(): + substituted = pattern.sub("(redacted)", text) + results.append(substituted.count("(redacted)")) + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + self.assertEqual(results, [2 * 5] * NTHREADS) + + def test_pattern_search(self): + """Pattern search should work across threads.""" + emails = ["alice@python.org", "bob@pycon.org"] * 10 + pattern = re.compile(r"\w+@\w+\.\w+") + results = [] + + def worker(): + matches = [pattern.search(e).group() for e in emails] + results.append(len(matches)) + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + self.assertEqual(results, [2 * 10] * NTHREADS) + + def test_scanner_concurrent_access(self): + """Shared scanner should reject concurrent access.""" + pattern = re.compile(r"\w+") + scanner = pattern.scanner("word " * 10) + + def worker(): + for _ in range(100): + try: + scanner.search() + except ValueError as e: + if "already executing" in str(e): + pass + else: + raise + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + # This test has no assertions. Its purpose is to catch crashes and + # enable thread sanitizer to detect race conditions. While "already + # executing" errors are very likely, they're not guaranteed due to + # non-deterministic thread scheduling, so we can't assert errors > 0. + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_resource.py b/Lib/test/test_free_threading/test_resource.py new file mode 100644 index 000000000000000..ecd0e535c442b44 --- /dev/null +++ b/Lib/test/test_free_threading/test_resource.py @@ -0,0 +1,41 @@ +import unittest +from test.support import import_helper, threading_helper + +resource = import_helper.import_module("resource") + + +NTHREADS = 10 +LOOP_PER_THREAD = 1000 + + +@threading_helper.requires_working_threading() +class ResourceTest(unittest.TestCase): + @unittest.skipUnless(hasattr(resource, "getrusage"), "needs getrusage") + @unittest.skipUnless( + hasattr(resource, "RUSAGE_THREAD"), "needs RUSAGE_THREAD" + ) + def test_getrusage(self): + ru_utime_lst = [] + + def dummy_work(ru_utime_lst): + for _ in range(LOOP_PER_THREAD): + pass + + usage_process = resource.getrusage(resource.RUSAGE_SELF) + usage_thread = resource.getrusage(resource.RUSAGE_THREAD) + # Process user time should be greater than thread user time + self.assertGreater(usage_process.ru_utime, usage_thread.ru_utime) + ru_utime_lst.append(usage_thread.ru_utime) + + threading_helper.run_concurrently( + worker_func=dummy_work, args=(ru_utime_lst,), nthreads=NTHREADS + ) + + usage_process = resource.getrusage(resource.RUSAGE_SELF) + self.assertEqual(len(ru_utime_lst), NTHREADS) + # Process user time should be greater than sum of all thread user times + self.assertGreater(usage_process.ru_utime, sum(ru_utime_lst)) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_slots.py b/Lib/test/test_free_threading/test_slots.py index a3b9f4b0175ae76..a73525e1bebfb49 100644 --- a/Lib/test/test_free_threading/test_slots.py +++ b/Lib/test/test_free_threading/test_slots.py @@ -16,18 +16,19 @@ def run_in_threads(targets): thread.join() +class Spam: + __slots__ = [ + "eggs", + ] + + def __init__(self, initial_value): + self.eggs = initial_value + + @threading_helper.requires_working_threading() class TestSlots(TestCase): def test_object(self): - class Spam: - __slots__ = [ - "eggs", - ] - - def __init__(self, initial_value): - self.eggs = initial_value - spam = Spam(0) iters = 20_000 @@ -43,6 +44,24 @@ def reader(): run_in_threads([writer, reader, reader, reader]) + def test_del_object_is_atomic(self): + # Testing whether the implementation of `del slots_object.attribute` + # removes the attribute atomically, thus avoiding non-sequentially- + # consistent behaviors. + # https://github.com/python/cpython/issues/146270 + def deleter(spam, successes): + try: + del spam.eggs + successes.append(True) + except AttributeError: + successes.append(False) + + for _ in range(10): + spam = Spam(0) + successes = [] + threading_helper.run_concurrently(deleter, nthreads=4, args=(spam, successes)) + self.assertEqual(sum(successes), 1) + def test_T_BOOL(self): spam_old = _testcapi._test_structmembersType_OldAPI() spam_new = _testcapi._test_structmembersType_NewAPI() diff --git a/Lib/test/test_free_threading/test_str.py b/Lib/test/test_free_threading/test_str.py index 72044e979b0f481..11e04009956db1d 100644 --- a/Lib/test/test_free_threading/test_str.py +++ b/Lib/test/test_free_threading/test_str.py @@ -1,7 +1,9 @@ +import sys +import threading import unittest from itertools import cycle -from threading import Event, Thread +from threading import Barrier, Event, Thread from unittest import TestCase from test.support import threading_helper @@ -69,6 +71,40 @@ def reader_func(): for reader in readers: reader.join() + def test_intern_unowned_string(self): + # Test interning strings owned by various threads. + strings = [f"intern_race_owner_{i}" for i in range(50)] + + NUM_THREADS = 5 + b = Barrier(NUM_THREADS) + + def interner(): + tid = threading.get_ident() + for i in range(20): + strings.append(f"intern_{tid}_{i}") + b.wait() + for s in strings: + r = sys.intern(s) + self.assertTrue(sys._is_interned(r)) + + threading_helper.run_concurrently(interner, nthreads=NUM_THREADS) + + def test_maketrans_dict_concurrent_modification(self): + for _ in range(5): + d = {2000: 'a'} + + def work(dct): + for i in range(100): + str.maketrans(dct) + dct[2000 + i] = chr(i % 16) + dct.pop(2000 + i, None) + + threading_helper.run_concurrently( + work, + nthreads=5, + args=(d,), + ) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_free_threading/test_string_template_race.py b/Lib/test/test_free_threading/test_string_template_race.py new file mode 100644 index 000000000000000..dc4b4e9f18d2863 --- /dev/null +++ b/Lib/test/test_free_threading/test_string_template_race.py @@ -0,0 +1,35 @@ +import string +import unittest +from string import Template + +from test.support import threading_helper + + +@threading_helper.requires_working_threading() +class TestTemplateCompileRace(unittest.TestCase): + def test_concurrent_first_use(self): + # Racing the lazy pattern compile must not raise a spurious ValueError + # from recompiling an already-compiled pattern. A throwaway subclass, + # re-armed to the sentinel each round, keeps string.Template unmutated + # (subclasses precompile eagerly in __init_subclass__). + uncompiled = string._TemplatePattern + errors = [] + + def use_template(cls): + try: + cls("$x and ${y}").substitute(x=1, y=2) + except Exception as e: + errors.append(e) + + for _ in range(20): + class T(Template): + pass + T.pattern = uncompiled + T.flags = None + threading_helper.run_concurrently(use_template, nthreads=10, args=(T,)) + + self.assertEqual(errors, [], msg=f"unexpected errors: {errors}") + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_sys.py b/Lib/test/test_free_threading/test_sys.py new file mode 100644 index 000000000000000..271fdd13c62b668 --- /dev/null +++ b/Lib/test/test_free_threading/test_sys.py @@ -0,0 +1,49 @@ +import sys +import unittest +from test.support import threading_helper + + +class SysModuleTest(unittest.TestCase): + @unittest.skipUnless(hasattr(sys, "setdlopenflags"), + "test needs sys.setdlopenflags()") + def test_dlopenflags_concurrent(self): + # gh-151644: getdlopenflags() and setdlopenflags() must be safe to + # call concurrently in free-threaded builds. + original = sys.getdlopenflags() + self.addCleanup(sys.setdlopenflags, original) + + # Use a small set of known-valid flag values to avoid integer overflow. + flag_values = [1, 2, 256, 257] + + def worker(worker_id): + for i in range(20_000): + if worker_id % 2 == 0: + sys.getdlopenflags() + else: + sys.setdlopenflags(flag_values[worker_id % len(flag_values)]) + + workers = [lambda i=i: worker(i) for i in range(6)] + threading_helper.run_concurrently(workers) + + def test_int_max_str_digits_thread(self): + # gh-151218: Check that it's safe to call get_int_max_str_digits() and + # set_int_max_str_digits() in parallel. Previously, this test triggered + # warnings in TSan on a free threaded build. + + old_limit = sys.get_int_max_str_digits() + self.addCleanup(sys.set_int_max_str_digits, old_limit) + + def worker(worker_id): + if not worker_id: + for i in range (20_000): + sys.get_int_max_str_digits() + else: + for i in range (20_000): + sys.set_int_max_str_digits(4300 + (i & 7)) + + workers = [lambda: worker(i) for i in range(5)] + threading_helper.run_concurrently(workers) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_threading.py b/Lib/test/test_free_threading/test_threading.py new file mode 100644 index 000000000000000..b5a5ca272b9405a --- /dev/null +++ b/Lib/test/test_free_threading/test_threading.py @@ -0,0 +1,26 @@ +import unittest +from test.support import threading_helper + +threading_helper.requires_working_threading(module=True) + + +class TestRlock(unittest.TestCase): + def test_repr_race(self): + # gh-153292 + import _thread + r = _thread.RLock() + + def repr_thread(): + for _ in range(2000): + repr(r) + + def mutate_thread(): + for _ in range(2000): + r.acquire() + r.release() + + threading_helper.run_concurrently([repr_thread, mutate_thread]) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_type.py b/Lib/test/test_free_threading/test_type.py index ae996e7db3c7fd7..2d995751005d710 100644 --- a/Lib/test/test_free_threading/test_type.py +++ b/Lib/test/test_free_threading/test_type.py @@ -127,6 +127,20 @@ class ClassB(Base): obj.__class__ = ClassB + def test_name_change(self): + class Foo: + pass + + def writer(): + for _ in range(1000): + Foo.__name__ = 'Bar' + + def reader(): + for _ in range(1000): + Foo.__name__ + + self.run_one(writer, reader) + def run_one(self, writer_func, reader_func): barrier = threading.Barrier(NTHREADS) diff --git a/Lib/test/test_free_threading/test_types.py b/Lib/test/test_free_threading/test_types.py new file mode 100644 index 000000000000000..76fcf1590122f54 --- /dev/null +++ b/Lib/test/test_free_threading/test_types.py @@ -0,0 +1,33 @@ +import unittest +from typing import TypeVar +from test.support import threading_helper + +threading_helper.requires_working_threading(module=True) + + +class TestGenericAlias(unittest.TestCase): + def test_parameters_race(self): + # gh-153298 + + T = TypeVar('T') + slot = [list[T]] + + def access(): + for _ in range(2000): + try: + _ = slot[0].__parameters__ + except Exception: + pass + + def refresh(): + for _ in range(2000): + slot[0] = list[T] + + threading_helper.run_concurrently([ + *[access for _ in range(6)], + *[refresh for _ in range(2)], + ]) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_free_threading/test_uuid.py b/Lib/test/test_free_threading/test_uuid.py new file mode 100755 index 000000000000000..d794afc552a6529 --- /dev/null +++ b/Lib/test/test_free_threading/test_uuid.py @@ -0,0 +1,60 @@ +import os +import unittest + +from test.support import import_helper, threading_helper +from test.support.threading_helper import run_concurrently +from uuid import SafeUUID + +c_uuid = import_helper.import_module("_uuid") + +NTHREADS = 10 +UUID_PER_THREAD = 1000 + + +@threading_helper.requires_working_threading() +class UUIDTests(unittest.TestCase): + @unittest.skipUnless(os.name == "posix", "POSIX only") + def test_generate_time_safe(self): + uuids = [] + + def worker(): + local_uuids = [] + for _ in range(UUID_PER_THREAD): + uuid, is_safe = c_uuid.generate_time_safe() + self.assertIs(type(uuid), bytes) + self.assertEqual(len(uuid), 16) + # Collect the UUID only if it is safe. If not, we cannot ensure + # UUID uniqueness. According to uuid_generate_time_safe() man + # page, it is theoretically possible for two concurrently + # running processes to generate the same UUID(s) if the return + # value is not 0. + if is_safe == SafeUUID.safe: + local_uuids.append(uuid) + + # Merge all safe uuids + uuids.extend(local_uuids) + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + self.assertEqual(len(uuids), len(set(uuids))) + + @unittest.skipUnless(os.name == "nt", "Windows only") + def test_UuidCreate(self): + uuids = [] + + def worker(): + local_uuids = [] + for _ in range(UUID_PER_THREAD): + uuid = c_uuid.UuidCreate() + self.assertIs(type(uuid), bytes) + self.assertEqual(len(uuid), 16) + local_uuids.append(uuid) + + # Merge all uuids + uuids.extend(local_uuids) + + run_concurrently(worker_func=worker, nthreads=NTHREADS) + self.assertEqual(len(uuids), len(set(uuids))) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_fstring.py b/Lib/test/test_fstring.py index dd58e032a8befe0..5cc02c30ec2ba33 100644 --- a/Lib/test/test_fstring.py +++ b/Lib/test/test_fstring.py @@ -593,6 +593,17 @@ def test_compile_time_concat_errors(self): r"""b'' f''""", ]) + def test_concat_decode_failure_does_not_crash(self): + script = r''' +import builtins +builtins.__import__ = builtins # Breaks warning machinery so _get_resized_exprs returns NULL +try: + compile('"x"f"\]"b""', '', 'exec') +except Exception: + pass +''' + assert_python_ok('-c', script) + def test_literal(self): self.assertEqual(f'', '') self.assertEqual(f'a', 'a') @@ -1336,9 +1347,9 @@ def test_equal_equal(self): def test_conversions(self): self.assertEqual(f'{3.14:10.10}', ' 3.14') - self.assertEqual(f'{3.14!s:10.10}', '3.14 ') - self.assertEqual(f'{3.14!r:10.10}', '3.14 ') - self.assertEqual(f'{3.14!a:10.10}', '3.14 ') + self.assertEqual(f'{1.25!s:10.10}', '1.25 ') + self.assertEqual(f'{1.25!r:10.10}', '1.25 ') + self.assertEqual(f'{1.25!a:10.10}', '1.25 ') self.assertEqual(f'{"a"}', 'a') self.assertEqual(f'{"a"!r}', "'a'") @@ -1347,7 +1358,7 @@ def test_conversions(self): # Conversions can have trailing whitespace after them since it # does not provide any significance self.assertEqual(f"{3!s }", "3") - self.assertEqual(f'{3.14!s :10.10}', '3.14 ') + self.assertEqual(f'{1.25!s :10.10}', '1.25 ') # Not a conversion. self.assertEqual(f'{"a!r"}', "a!r") @@ -1380,7 +1391,7 @@ def test_conversions(self): for conv in ' s', ' s ': self.assertAllRaise(SyntaxError, "f-string: conversion type must come right after the" - " exclamanation mark", + " exclamation mark", ["f'{3!" + conv + "}'"]) self.assertAllRaise(SyntaxError, @@ -1651,6 +1662,18 @@ def __repr__(self): self.assertEqual(f"{1+2 = # my comment }", '1+2 = \n 3') + self.assertEqual(f'{""" # booo + """=}', '""" # booo\n """=\' # booo\\n \'') + + self.assertEqual(f'{" # nooo "=}', '" # nooo "=\' # nooo \'') + self.assertEqual(f'{" \" # nooo \" "=}', '" \\" # nooo \\" "=\' " # nooo " \'') + + self.assertEqual(f'{ # some comment goes here + """hello"""=}', ' \n """hello"""=\'hello\'') + self.assertEqual(f'{"""# this is not a comment + a""" # this is a comment + }', '# this is not a comment\n a') + # These next lines contains tabs. Backslash escapes don't # work in f-strings. # patchcheck doesn't like these tabs. So the only way to test @@ -1819,6 +1842,41 @@ def test_newlines_in_format_specifiers(self): for case in valid_cases: compile(case, "", "exec") + def test_raw_fstring_format_spec(self): + # Test raw f-string format spec behavior (Issue #137314). + # + # Raw f-strings should preserve literal backslashes in format specifications, + # not interpret them as escape sequences. + class UnchangedFormat: + """Test helper that returns the format spec unchanged.""" + def __format__(self, format): + return format + + # Test basic escape sequences + self.assertEqual(f"{UnchangedFormat():\xFF}", 'ÿ') + self.assertEqual(rf"{UnchangedFormat():\xFF}", '\\xFF') + + # Test nested expressions with raw/non-raw combinations + self.assertEqual(rf"{UnchangedFormat():{'\xFF'}}", 'ÿ') + self.assertEqual(f"{UnchangedFormat():{r'\xFF'}}", '\\xFF') + self.assertEqual(rf"{UnchangedFormat():{r'\xFF'}}", '\\xFF') + + # Test continuation character in format specs + self.assertEqual(f"""{UnchangedFormat():{'a'\ + 'b'}}""", 'ab') + self.assertEqual(rf"""{UnchangedFormat():{'a'\ + 'b'}}""", 'ab') + + # Test multiple format specs in same raw f-string + self.assertEqual(rf"{UnchangedFormat():\xFF} {UnchangedFormat():\n}", '\\xFF \\n') + + def test_gh139516(self): + with temp_cwd(): + script = 'script.py' + with open(script, 'wb') as f: + f.write('''def f(a): pass\nf"{f(a=lambda: 'à'\n)}"'''.encode()) + assert_python_ok(script) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index c864d401f9ed67e..f1eff9430f7351c 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -16,7 +16,7 @@ except ImportError: ssl = None -from unittest import TestCase, skipUnless +from unittest import mock, TestCase, skipUnless from test import support from test.support import requires_subprocess from test.support import threading_helper @@ -1145,6 +1145,40 @@ def testTimeoutDirectAccess(self): ftp.close() +class TestFtpcpSecurity(TestCase): + """ftpcp() must not trust the host a source server advertises in PASV. + + A malicious source server can otherwise redirect the target server's + data connection to an arbitrary host:port (SSRF), so ftpcp() uses the + source server's actual peer address instead, the same as FTP.makepasv(). + """ + + def _make_pair(self, *, advertised_host, real_host, trust=False): + source = mock.Mock(spec=ftplib.FTP) + source.trust_server_pasv_ipv4_address = trust + source.sock.getpeername.return_value = (real_host, 21) + # PASV replies give the host as comma-separated octets, not dotted. + advertised = advertised_host.replace('.', ',') + source.sendcmd.side_effect = lambda cmd: ( + f'227 Entering Passive Mode ({advertised},1,2).' + if cmd == 'PASV' else '150 ok') + target = mock.Mock(spec=ftplib.FTP) + target.sendcmd.return_value = '150 ok' + return source, target + + def test_ftpcp_ignores_untrusted_pasv_host(self): + source, target = self._make_pair(advertised_host='10.0.0.5', + real_host='198.51.100.7') + ftplib.ftpcp(source, 'a', target, 'b') + target.sendport.assert_called_once_with('198.51.100.7', 258) + + def test_ftpcp_trust_server_pasv_ipv4_address(self): + source, target = self._make_pair(advertised_host='10.0.0.5', + real_host='198.51.100.7', trust=True) + ftplib.ftpcp(source, 'a', target, 'b') + target.sendport.assert_called_once_with('10.0.0.5', 258) + + class MiscTestCase(TestCase): def test__all__(self): not_exported = { diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 2e794b0fc95a22d..6531bd2bca33c70 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -21,6 +21,7 @@ import contextlib from inspect import Signature +from test.support import ALWAYS_EQ from test.support import import_helper from test.support import threading_helper from test.support import cpython_only @@ -244,6 +245,13 @@ def test_placeholders(self): actual_args, actual_kwds = p('x', 'y') self.assertEqual(actual_args, ('x', 0, 'y', 1)) self.assertEqual(actual_kwds, {}) + # Checks via `is` and not `eq` + # thus ALWAYS_EQ isn't treated as Placeholder + p = self.partial(capture, ALWAYS_EQ) + actual_args, actual_kwds = p() + self.assertEqual(len(actual_args), 1) + self.assertIs(actual_args[0], ALWAYS_EQ) + self.assertEqual(actual_kwds, {}) def test_placeholders_optimization(self): PH = self.module.Placeholder @@ -260,6 +268,17 @@ def test_placeholders_optimization(self): self.assertEqual(p2.args, (PH, 0)) self.assertEqual(p2(1), ((1, 0), {})) + def test_placeholders_kw_restriction(self): + PH = self.module.Placeholder + with self.assertRaisesRegex(TypeError, "Placeholder"): + self.partial(capture, a=PH) + # Passes, as checks via `is` and not `eq` + p = self.partial(capture, a=ALWAYS_EQ) + actual_args, actual_kwds = p() + self.assertEqual(actual_args, ()) + self.assertEqual(len(actual_kwds), 1) + self.assertIs(actual_kwds['a'], ALWAYS_EQ) + def test_construct_placeholder_singleton(self): PH = self.module.Placeholder tp = type(PH) @@ -387,6 +406,7 @@ def test_setstate(self): def test_setstate_errors(self): f = self.partial(signature) + self.assertRaises(TypeError, f.__setstate__, (capture, (), {})) self.assertRaises(TypeError, f.__setstate__, (capture, (), {}, {}, None)) self.assertRaises(TypeError, f.__setstate__, [capture, (), {}, None]) @@ -394,6 +414,8 @@ def test_setstate_errors(self): self.assertRaises(TypeError, f.__setstate__, (capture, None, {}, None)) self.assertRaises(TypeError, f.__setstate__, (capture, [], {}, None)) self.assertRaises(TypeError, f.__setstate__, (capture, (), [], None)) + self.assertRaises(TypeError, f.__setstate__, (capture, (), {}, ())) + self.assertRaises(TypeError, f.__setstate__, (capture, (), {}, 'test')) def test_setstate_subclasses(self): f = self.partial(signature) @@ -416,6 +438,7 @@ def test_setstate_subclasses(self): self.assertIs(type(r[0]), tuple) @support.skip_if_sanitizer("thread sanitizer crashes in __tsan::FuncEntry", thread=True) + @support.skip_if_huge_c_stack() @support.skip_emscripten_stack_overflow() def test_recursive_pickle(self): with replaced_module('functools', self.module): @@ -491,6 +514,58 @@ def test_partial_genericalias(self): self.assertEqual(alias.__args__, (int,)) self.assertEqual(alias.__parameters__, ()) + # GH-144475: Tests that the partial object does not change until repr finishes + def test_repr_safety_against_reentrant_mutation(self): + g_partial = None + + class Function: + def __init__(self, name): + self.name = name + + def __call__(self): + return None + + def __repr__(self): + return f"Function({self.name})" + + class EvilObject: + def __init__(self): + self.triggered = False + + def __repr__(self): + if not self.triggered and g_partial is not None: + self.triggered = True + new_args_tuple = (None,) + new_keywords_dict = {"keyword": None} + new_tuple_state = (Function("new_function"), new_args_tuple, new_keywords_dict, None) + g_partial.__setstate__(new_tuple_state) + gc.collect() + return f"EvilObject" + + trigger = EvilObject() + func = Function("old_function") + + g_partial = functools.partial(func, None, trigger=trigger) + self.assertEqual(repr(g_partial),"functools.partial(Function(old_function), None, trigger=EvilObject)") + + trigger.triggered = False + g_partial = functools.partial(func, trigger, arg=None) + self.assertEqual(repr(g_partial),"functools.partial(Function(old_function), EvilObject, arg=None)") + + + trigger.triggered = False + g_partial = functools.partial(func, trigger, None) + self.assertEqual(repr(g_partial),"functools.partial(Function(old_function), EvilObject, None)") + + trigger.triggered = False + g_partial = functools.partial(func, trigger=trigger, arg=None) + self.assertEqual(repr(g_partial),"functools.partial(Function(old_function), trigger=EvilObject, arg=None)") + + trigger.triggered = False + g_partial = functools.partial(func, trigger, None, None, None, None, arg=None) + self.assertEqual(repr(g_partial),"functools.partial(Function(old_function), EvilObject, None, None, None, None, arg=None)") + + @unittest.skipUnless(c_functools, 'requires the C _functools module') class TestPartialC(TestPartial, unittest.TestCase): @@ -2117,6 +2192,7 @@ def orig(a: int) -> nonexistent: ... @support.skip_on_s390x @unittest.skipIf(support.is_wasi, "WASI has limited C stack") @support.skip_if_sanitizer("requires deep stack", ub=True, thread=True) + @support.skip_if_unlimited_stack_size @support.skip_emscripten_stack_overflow() def test_lru_recursion(self): @@ -3424,16 +3500,11 @@ def _(item: int, arg: bytes) -> str: def test_method_signatures(self): class A: - def m(self, item, arg: int) -> str: - return str(item) - @classmethod - def cm(cls, item, arg: int) -> str: - return str(item) @functools.singledispatchmethod def func(self, item, arg: int) -> str: return str(item) @func.register - def _(self, item, arg: bytes) -> str: + def _(self, item: int, arg: bytes) -> str: return str(item) @functools.singledispatchmethod @@ -3442,7 +3513,7 @@ def cls_func(cls, item, arg: int) -> str: return str(arg) @func.register @classmethod - def _(cls, item, arg: bytes) -> str: + def _(cls, item: int, arg: bytes) -> str: return str(item) @functools.singledispatchmethod @@ -3451,7 +3522,7 @@ def static_func(item, arg: int) -> str: return str(arg) @func.register @staticmethod - def _(item, arg: bytes) -> str: + def _(item: int, arg: bytes) -> str: return str(item) self.assertEqual(str(Signature.from_callable(A.func)), diff --git a/Lib/test/test_future_stmt/test_future.py b/Lib/test/test_future_stmt/test_future.py index 42c6cb3fefac33d..71f1e616116d81b 100644 --- a/Lib/test/test_future_stmt/test_future.py +++ b/Lib/test/test_future_stmt/test_future.py @@ -422,6 +422,11 @@ def test_annotations(self): eq('(((a)))', 'a') eq('(((a, b)))', '(a, b)') eq("1 + 2 + 3") + eq("t''") + eq("t'{a + b}'") + eq("t'{a!s}'") + eq("t'{a:b}'") + eq("t'{a:b=}'") def test_fstring_debug_annotations(self): # f-strings with '=' don't round trip very well, so set the expected diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 8fae12c478cb3ad..0fe63332d15c9cd 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -262,9 +262,11 @@ class Cyclic(tuple): # finalizer. def __del__(self): - # 5. Create a weakref to `func` now. If we had created - # it earlier, it would have been cleared by the - # garbage collector before calling the finalizers. + # 5. Create a weakref to `func` now. In previous + # versions of Python, this would avoid having it + # cleared by the garbage collector before calling + # the finalizers. Now, weakrefs get cleared after + # calling finalizers. self[1].ref = weakref.ref(self[0]) # 6. Drop the global reference to `latefin`. The only @@ -293,14 +295,18 @@ def func(): # which will find `cyc` and `func` as garbage. gc.collect() - # 9. Previously, this would crash because `func_qualname` - # had been NULL-ed out by func_clear(). + # 9. Previously, this would crash because the weakref + # created in the finalizer revealed the function after + # `tp_clear` was called and `func_qualname` + # had been NULL-ed out by func_clear(). Now, we clear + # weakrefs to unreachable objects before calling `tp_clear` + # but after calling finalizers. print(f"{func=}") """ - # We're mostly just checking that this doesn't crash. rc, stdout, stderr = assert_python_ok("-c", code) self.assertEqual(rc, 0) - self.assertRegex(stdout, rb"""\A\s*func=\s*\z""") + # The `func` global is None because the weakref was cleared. + self.assertRegex(stdout, rb"""\A\s*func=None""") self.assertFalse(stderr) @refcount_test @@ -393,11 +399,19 @@ def test_collect_generations(self): # each call to collect(N) x = [] gc.collect(0) - # x is now in the old gen + # x is now in gen 1 a, b, c = gc.get_count() - # We don't check a since its exact values depends on + gc.collect(1) + # x is now in gen 2 + d, e, f = gc.get_count() + gc.collect(2) + # x is now in gen 3 + g, h, i = gc.get_count() + # We don't check a, d, g since their exact values depends on # internal implementation details of the interpreter. self.assertEqual((b, c), (1, 0)) + self.assertEqual((e, f), (0, 1)) + self.assertEqual((h, i), (0, 0)) def test_trashcan(self): class Ouch: @@ -726,6 +740,9 @@ def run_command(code): self.assertIn(b"ResourceWarning: gc: 2 uncollectable objects at " b"shutdown; use", stderr) self.assertNotIn(b"", stderr) + one_line_re = b"gc: uncollectable " + expected_re = one_line_re + b"\r?\n" + one_line_re + self.assertNotRegex(stderr, expected_re) # With DEBUG_UNCOLLECTABLE, the garbage list gets printed stderr = run_command(code % "gc.DEBUG_UNCOLLECTABLE") self.assertIn(b"ResourceWarning: gc: 2 uncollectable objects at " @@ -733,6 +750,8 @@ def run_command(code): self.assertTrue( (b"[, ]" in stderr) or (b"[, ]" in stderr), stderr) + # we expect two lines with uncollectable objects + self.assertRegex(stderr, expected_re) # With DEBUG_SAVEALL, no additional message should get printed # (because gc.garbage also contains normally reclaimable cyclic # references, and its elements get printed at runtime anyway). @@ -771,6 +790,32 @@ def __del__(self): rc, out, err = assert_python_ok('-c', code) self.assertEqual(out.strip(), b'__del__ called') + @unittest.skipIf(Py_GIL_DISABLED, "requires GC generations or increments") + def test_gc_debug_stats(self): + # Checks that debug information is printed to stderr + # when DEBUG_STATS is set. + code = """if 1: + import gc + gc.set_debug(%s) + gc.collect() + """ + _, _, err = assert_python_ok("-c", code % "gc.DEBUG_STATS") + self.assertRegex(err, b"gc: collecting generation [0-9]+") + self.assertRegex( + err, + b"gc: objects in each generation: [0-9]+ [0-9]+ [0-9]+", + ) + self.assertRegex( + err, b"gc: objects in permanent generation: [0-9]+" + ) + self.assertRegex( + err, + b"gc: done, .* unreachable, .* uncollectable, .* elapsed", + ) + + _, _, err = assert_python_ok("-c", code % "0") + self.assertNotIn(b"elapsed", err) + def test_global_del_SystemExit(self): code = """if 1: class ClassWithDel: @@ -833,10 +878,42 @@ def test_get_objects_generations(self): self.assertTrue( any(l is element for element in gc.get_objects(generation=0)) ) - gc.collect() + self.assertFalse( + any(l is element for element in gc.get_objects(generation=1)) + ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=2)) + ) + gc.collect(generation=0) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=0)) + ) + self.assertTrue( + any(l is element for element in gc.get_objects(generation=1)) + ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=2)) + ) + gc.collect(generation=1) self.assertFalse( any(l is element for element in gc.get_objects(generation=0)) ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=1)) + ) + self.assertTrue( + any(l is element for element in gc.get_objects(generation=2)) + ) + gc.collect(generation=2) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=0)) + ) + self.assertFalse( + any(l is element for element in gc.get_objects(generation=1)) + ) + self.assertTrue( + any(l is element for element in gc.get_objects(generation=2)) + ) del l gc.collect() @@ -914,7 +991,7 @@ def __del__(self): gc.collect() self.assertEqual(len(Lazarus.resurrected_instances), 1) instance = Lazarus.resurrected_instances.pop() - self.assertTrue(hasattr(instance, "cargo")) + self.assertHasAttr(instance, "cargo") self.assertEqual(id(instance.cargo), cargo_id) gc.collect() @@ -1126,65 +1203,31 @@ def test_something(self): assert_python_ok("-c", source) -class IncrementalGCTests(unittest.TestCase): - - def setUp(self): - # Reenable GC as it is disabled module-wide - gc.enable() - - def tearDown(self): - gc.disable() + @unittest.skipUnless(Py_GIL_DISABLED, "requires free-threaded GC") + @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") + def test_tuple_untrack_counts(self): + # This ensures that the free-threaded GC is counting untracked tuples + # in the "long_lived_total" count. This is required to avoid + # performance issues from running the GC too frequently. See + # GH-142531 as an example. + gc.collect() + count = _testinternalcapi.get_long_lived_total() + n = 20_000 + tuples = [(x,) for x in range(n)] + gc.collect() + new_count = _testinternalcapi.get_long_lived_total() + self.assertFalse(gc.is_tracked(tuples[0])) + # Use n // 2 just in case some other objects were collected. + self.assertTrue(new_count - count > (n // 2)) + @requires_gil_enabled('need generational GC') @unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") - @requires_gil_enabled("Free threading does not support incremental GC") - # Use small increments to emulate longer running process in a shorter time - @gc_threshold(200, 10) - def test_incremental_gc_handles_fast_cycle_creation(self): - - class LinkedList: - - #Use slots to reduce number of implicit objects - __slots__ = "next", "prev", "surprise" - - def __init__(self, next=None, prev=None): - self.next = next - if next is not None: - next.prev = self - self.prev = prev - if prev is not None: - prev.next = self - - def make_ll(depth): - head = LinkedList() - for i in range(depth): - head = LinkedList(head, head.prev) - return head - - head = make_ll(1000) - count = 1000 - - # There will be some objects we aren't counting, - # e.g. the gc stats dicts. This test checks - # that the counts don't grow, so we try to - # correct for the uncounted objects - # This is just an estimate. - CORRECTION = 20 - - enabled = gc.isenabled() - gc.enable() - olds = [] - initial_heap_size = _testinternalcapi.get_tracked_heap_size() - for i in range(20_000): - newhead = make_ll(20) - count += 20 - newhead.surprise = head - olds.append(newhead) - if len(olds) == 20: - new_objects = _testinternalcapi.get_tracked_heap_size() - initial_heap_size - self.assertLess(new_objects, 27_000, f"Heap growing. Reached limit after {i} iterations") - del olds[:] - if not enabled: - gc.disable() + def test_heap_size(self): + count = _testinternalcapi.get_tracked_heap_size() + l = [] + self.assertEqual(count + 1, _testinternalcapi.get_tracked_heap_size()) + del l + self.assertEqual(count, _testinternalcapi.get_tracked_heap_size()) class GCCallbackTests(unittest.TestCase): @@ -1374,6 +1417,7 @@ def setUp(self): def tearDown(self): gc.disable() + @unittest.skipIf(Py_GIL_DISABLED, "requires GC generations or increments") def test_bug1055820c(self): # Corresponds to temp2c.py in the bug report. This is pretty # elaborate. @@ -1435,6 +1479,7 @@ def callback(ignored): # The free-threaded build doesn't have multiple generations, so # just trigger a GC manually. gc.collect() + assert not detector.gc_happened while not detector.gc_happened: i += 1 if i > 10000: @@ -1546,6 +1591,20 @@ def test_indirect_calls_with_gc_disabled(self): finally: gc.enable() + # Ensure that setting *threshold0* to zero disables collection. + @gc_threshold(0) + def test_threshold_zero(self): + junk = [] + i = 0 + detector = GC_Detector() + while not detector.gc_happened: + i += 1 + if i > 50000: + break + junk.append([]) # this may eventually trigger gc (if it is enabled) + + self.assertEqual(i, 50001) + class PythonFinalizationTests(unittest.TestCase): def test_ast_fini(self): @@ -1567,6 +1626,19 @@ def test_ast_fini(self): """) assert_python_ok("-c", code) + def test_warnings_fini(self): + # See https://github.com/python/cpython/issues/137384 + code = textwrap.dedent(''' + import asyncio + from contextvars import ContextVar + + context_loop = ContextVar("context_loop", default=None) + loop = asyncio.new_event_loop() + context_loop.set(loop) + ''') + + assert_python_ok("-c", code) + def setUpModule(): global enabled, debug diff --git a/Lib/test/test_gdb/test_pretty_print.py b/Lib/test/test_gdb/test_pretty_print.py index dfc77d65ab16a40..a258d7c0e60c7e1 100644 --- a/Lib/test/test_gdb/test_pretty_print.py +++ b/Lib/test/test_gdb/test_pretty_print.py @@ -107,29 +107,28 @@ def test_bytes(self): @support.requires_resource('cpu') def test_strings(self): 'Verify the pretty-printing of unicode strings' - # We cannot simply call locale.getpreferredencoding() here, - # as GDB might have been linked against a different version - # of Python with a different encoding and coercion policy - # with respect to PEP 538 and PEP 540. + # gdb emits its output in the host charset, which is not necessarily the + # getpreferredencoding() of the (possibly differently coerced) embedded + # Python. stdout, stderr = run_gdb( '--eval-command', - 'python import locale; print(locale.getpreferredencoding())') + 'python import gdb; print(gdb.host_charset())') - encoding = stdout + encoding = stdout.strip() if stderr or not encoding: raise RuntimeError( - f'unable to determine the Python locale preferred encoding ' - f'of embedded Python in GDB\n' + f'unable to determine the host charset of gdb\n' f'stdout={stdout!r}\n' f'stderr={stderr!r}') def check_repr(text): try: text.encode(encoding) - except UnicodeEncodeError: + # LookupError or ValueError if the host charset is unknown or invalid. + except (UnicodeEncodeError, LookupError, ValueError): self.assertGdbRepr(text, ascii(text)) else: - self.assertGdbRepr(text) + self.assertGdbRepr(text, repr(text).encode(encoding).decode('ascii', 'surrogateescape')) self.assertGdbRepr('') self.assertGdbRepr('And now for something hopefully the same') diff --git a/Lib/test/test_gdb/util.py b/Lib/test/test_gdb/util.py index 8097fd52ababe6c..9d9e454ec6de2b5 100644 --- a/Lib/test/test_gdb/util.py +++ b/Lib/test/test_gdb/util.py @@ -57,7 +57,7 @@ def run_gdb(*args, exitcode=0, check=True, **env_vars): stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding="utf8", errors="backslashreplace", + encoding="ascii", errors="surrogateescape", env=env) stdout = proc.stdout diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index d481cb07f757d83..fc34ac2fdc98ce4 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -1601,7 +1601,7 @@ def test_instruction_size_macro(self): frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(OP); - frame->return_offset = 1 ; + frame->return_offset = 1u ; DISPATCH(); } """ diff --git a/Lib/test/test_generators.py b/Lib/test/test_generators.py index 3e41c7b9663491a..97f14314459a3c7 100644 --- a/Lib/test/test_generators.py +++ b/Lib/test/test_generators.py @@ -134,6 +134,18 @@ def gen(): self.assertEqual(len(resurrected), 1) self.assertIsInstance(resurrected[0].gi_code, types.CodeType) + def test_exhausted_generator_frame_cycle(self): + def g(): + yield + + generator = g() + frame = generator.gi_frame + self.assertIsNone(frame.f_back) + next(generator) + self.assertIsNone(frame.f_back) + next(generator, None) + self.assertIsNone(frame.f_back) + class GeneratorTest(unittest.TestCase): @@ -288,8 +300,35 @@ class C: def __iter__(self): return Iterator() - self.assertEqual([1,2], list(i for i in C())) + self.assertEqual([1, 2], list(i for i in C())) + + def test_close_clears_frame(self): + # gh-142766: Test that closing a generator clears its frame + class DetectDelete: + def __init__(self): + DetectDelete.deleted = False + + def __del__(self): + DetectDelete.deleted = True + + def generator(arg): + yield + + # Test a freshly created generator (not suspended) + g = generator(DetectDelete()) + g.close() + self.assertTrue(DetectDelete.deleted) + + # Test a suspended generator + g = generator(DetectDelete()) + next(g) + g.close() + self.assertTrue(DetectDelete.deleted) + # Clear via gi_frame.clear() + g = generator(DetectDelete()) + g.gi_frame.clear() + self.assertTrue(DetectDelete.deleted) class ModifyUnderlyingIterableTest(unittest.TestCase): iterables = [ @@ -318,21 +357,26 @@ def gen(it): yield x return gen(range(10)) - def process_tests(self, get_generator): + def process_tests(self, get_generator, is_expr): + err_iterator = "'.*' object is not an iterator" + err_iterable = "'.*' object is not iterable" for obj in self.iterables: g_obj = get_generator(obj) with self.subTest(g_obj=g_obj, obj=obj): - self.assertListEqual(list(g_obj), list(obj)) + if is_expr: + self.assertRaisesRegex(TypeError, err_iterator, list, g_obj) + else: + self.assertListEqual(list(g_obj), list(obj)) g_iter = get_generator(iter(obj)) with self.subTest(g_iter=g_iter, obj=obj): self.assertListEqual(list(g_iter), list(obj)) - err_regex = "'.*' object is not iterable" for obj in self.non_iterables: g_obj = get_generator(obj) with self.subTest(g_obj=g_obj): - self.assertRaisesRegex(TypeError, err_regex, list, g_obj) + err = err_iterator if is_expr else err_iterable + self.assertRaisesRegex(TypeError, err, list, g_obj) def test_modify_f_locals(self): def modify_f_locals(g, local, obj): @@ -345,8 +389,8 @@ def get_generator_genexpr(obj): def get_generator_genfunc(obj): return modify_f_locals(self.genfunc(), 'it', obj) - self.process_tests(get_generator_genexpr) - self.process_tests(get_generator_genfunc) + self.process_tests(get_generator_genexpr, True) + self.process_tests(get_generator_genfunc, False) def test_new_gen_from_gi_code(self): def new_gen_from_gi_code(g, obj): @@ -359,8 +403,8 @@ def get_generator_genexpr(obj): def get_generator_genfunc(obj): return new_gen_from_gi_code(self.genfunc(), obj) - self.process_tests(get_generator_genexpr) - self.process_tests(get_generator_genfunc) + self.process_tests(get_generator_genexpr, True) + self.process_tests(get_generator_genfunc, False) class ExceptionTest(unittest.TestCase): diff --git a/Lib/test/test_genericalias.py b/Lib/test/test_genericalias.py index 8d21ded45014ba4..121d169dd07285a 100644 --- a/Lib/test/test_genericalias.py +++ b/Lib/test/test_genericalias.py @@ -55,14 +55,14 @@ from unittest.case import _AssertRaisesContext from queue import Queue, SimpleQueue from weakref import WeakSet, ReferenceType, ref -import typing -from typing import Unpack try: from tkinter import Event except ImportError: Event = None +from string.templatelib import Template, Interpolation -from typing import TypeVar +import typing +from typing import TypeVar, Unpack T = TypeVar('T') K = TypeVar('K') V = TypeVar('V') @@ -139,7 +139,10 @@ class BaseTest(unittest.TestCase): DictReader, DictWriter, array, staticmethod, - classmethod] + classmethod, + Template, + Interpolation, + ] if ctypes is not None: generic_types.extend((ctypes.Array, ctypes.LibraryLoader, ctypes.py_object)) if ValueProxy is not None: @@ -232,13 +235,63 @@ class MyGeneric: self.assertEqual(repr(x2), 'tuple[*tuple[int, str]]') x3 = tuple[*tuple[int, ...]] self.assertEqual(repr(x3), 'tuple[*tuple[int, ...]]') - self.assertTrue(repr(MyList[int]).endswith('.BaseTest.test_repr..MyList[int]')) + self.assertEndsWith(repr(MyList[int]), '.BaseTest.test_repr..MyList[int]') self.assertEqual(repr(list[str]()), '[]') # instances should keep their normal repr # gh-105488 - self.assertTrue(repr(MyGeneric[int]).endswith('MyGeneric[int]')) - self.assertTrue(repr(MyGeneric[[]]).endswith('MyGeneric[[]]')) - self.assertTrue(repr(MyGeneric[[int, str]]).endswith('MyGeneric[[int, str]]')) + self.assertEndsWith(repr(MyGeneric[int]), 'MyGeneric[int]') + self.assertEndsWith(repr(MyGeneric[[]]), 'MyGeneric[[]]') + self.assertEndsWith(repr(MyGeneric[[int, str]]), 'MyGeneric[[int, str]]') + + def test_evil_repr1(self): + # gh-143635 + class Zap: + def __init__(self, container): + self.container = container + def __getattr__(self, name): + if name == "__origin__": + self.container.clear() + return None + if name == "__args__": + return () + raise AttributeError + + params = [] + params.append(Zap(params)) + alias = GenericAlias(list, (params,)) + repr_str = repr(alias) + self.assertTrue(repr_str.startswith("list[["), repr_str) + + def test_evil_repr2(self): + class Zap: + def __init__(self, container): + self.container = container + def __getattr__(self, name): + if name == "__qualname__": + self.container.clear() + return "abcd" + if name == "__module__": + return None + raise AttributeError + + params = [] + params.append(Zap(params)) + alias = GenericAlias(list, (params,)) + repr_str = repr(alias) + self.assertTrue(repr_str.startswith("list[["), repr_str) + + def test_evil_repr3(self): + # gh-143823 + lst = [] + class X: + def __repr__(self): + lst.clear() + return "x" + + lst += [X(), 1] + ga = GenericAlias(int, lst) + with self.assertRaises(IndexError): + repr(ga) def test_exposed_type(self): import types @@ -358,7 +411,7 @@ def test_isinstance(self): def test_issubclass(self): class L(list): ... - self.assertTrue(issubclass(L, list)) + self.assertIsSubclass(L, list) with self.assertRaises(TypeError): issubclass(L, list[str]) @@ -398,7 +451,10 @@ def __deepcopy__(self, memo): aliases = [ GenericAlias(list, T), GenericAlias(deque, T), - GenericAlias(X, T) + GenericAlias(X, T), + X[T], + list[T], + deque[T], ] + _UNPACKED_TUPLES for alias in aliases: with self.subTest(alias=alias): @@ -428,10 +484,26 @@ def test_union_generic(self): self.assertEqual(a.__parameters__, (T,)) def test_dir(self): - dir_of_gen_alias = set(dir(list[int])) + ga = list[int] + dir_of_gen_alias = set(dir(ga)) self.assertTrue(dir_of_gen_alias.issuperset(dir(list))) - for generic_alias_property in ("__origin__", "__args__", "__parameters__"): - self.assertIn(generic_alias_property, dir_of_gen_alias) + for generic_alias_property in ( + "__origin__", "__args__", "__parameters__", + "__unpacked__", + ): + with self.subTest(generic_alias_property=generic_alias_property): + self.assertIn(generic_alias_property, dir_of_gen_alias) + for blocked in ( + "__bases__", + "__copy__", + "__deepcopy__", + ): + with self.subTest(blocked=blocked): + self.assertNotIn(blocked, dir_of_gen_alias) + + for entry in dir_of_gen_alias: + with self.subTest(entry=entry): + getattr(ga, entry) # must not raise `AttributeError` def test_weakref(self): for t in self.generic_types: @@ -546,6 +618,14 @@ def test_nested_paramspec_specialization(self): self.assertEqual(deeply_nested_specialized.__args__, ([str, [float], int], float)) self.assertEqual(deeply_nested_specialized.__parameters__, ()) + def test_gh150146(self): + # It used to crash: + for container in [memoryview, list, tuple]: + with self.subTest(container=container): + x = container[TypeVar("")] + with self.assertRaises(TypeError): + x[*typing.Mapping[..., ...]] + class TypeIterationTests(unittest.TestCase): _UNITERABLE_TYPES = (list, tuple) diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index 6c3abe602f557c2..1a44cedcd360b11 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -7,9 +7,9 @@ import sys import unittest import warnings -from test.support import ( - is_apple, is_emscripten, os_helper, warnings_helper -) +from test import support +from test.support import os_helper +from test.support import warnings_helper from test.support.script_helper import assert_python_ok from test.support.os_helper import FakePath @@ -92,8 +92,8 @@ def test_commonprefix(self): for s1 in testlist: for s2 in testlist: p = commonprefix([s1, s2]) - self.assertTrue(s1.startswith(p)) - self.assertTrue(s2.startswith(p)) + self.assertStartsWith(s1, p) + self.assertStartsWith(s2, p) if s1 != s2: n = len(p) self.assertNotEqual(s1[n:n+1], s2[n:n+1]) @@ -445,6 +445,19 @@ def check(value, expected): os.fsencode('$bar%s bar' % nonascii)) check(b'$spam}bar', os.fsencode('%s}bar' % nonascii)) + @support.requires_resource('cpu') + def test_expandvars_large(self): + expandvars = self.pathmodule.expandvars + with os_helper.EnvironmentVarGuard() as env: + env.clear() + env["A"] = "B" + n = 100_000 + self.assertEqual(expandvars('$A'*n), 'B'*n) + self.assertEqual(expandvars('${A}'*n), 'B'*n) + self.assertEqual(expandvars('$A!'*n), 'B!'*n) + self.assertEqual(expandvars('${A}A'*n), 'BA'*n) + self.assertEqual(expandvars('${'*10*n), '${'*10*n) + def test_abspath(self): self.assertIn("foo", self.pathmodule.abspath("foo")) with warnings.catch_warnings(): @@ -502,7 +515,7 @@ def test_nonascii_abspath(self): # directory (when the bytes name is used). and sys.platform not in { "win32", "emscripten", "wasi" - } and not is_apple + } and not support.is_apple ): name = os_helper.TESTFN_UNDECODABLE elif os_helper.TESTFN_NONASCII: diff --git a/Lib/test/test_genexps.py b/Lib/test/test_genexps.py index fe5f18fa3f88a01..d90097dabea0c01 100644 --- a/Lib/test/test_genexps.py +++ b/Lib/test/test_genexps.py @@ -131,6 +131,14 @@ >>> list(g) [1, 9, 25, 49, 81] +Verify that the outermost for-expression makes an immediate check +for iterability + >>> (i for i in 6) + Traceback (most recent call last): + File "", line 1, in -toplevel- + (i for i in 6) + TypeError: 'int' object is not iterable + Verify late binding for the innermost for-expression >>> g = ((i,j) for i in range(3) for j in range(x)) diff --git a/Lib/test/test_getopt.py b/Lib/test/test_getopt.py index 8d0d5084abbb597..fac2ca3db81d703 100644 --- a/Lib/test/test_getopt.py +++ b/Lib/test/test_getopt.py @@ -149,6 +149,18 @@ def test_getopt(self): ('-a', ''), ('--alpha', '')]) self.assertEqual(args, ['arg1', 'arg2']) + # Allow string for single long argument + opts, args = getopt.getopt(cmdline, 'a::', 'alpha=?') + self.assertEqual(opts, [('-a', '1'), ('--alpha', '2'), ('--alpha', ''), + ('-a', ''), ('--alpha', '')]) + self.assertEqual(args, ['arg1', 'arg2']) + + # Pass everything after -- as args + cmdline = ['-a1', '--alpha=2', '--', '-b', '--beta=5'] + opts, args = getopt.getopt(cmdline, 'a:b', ['alpha=', 'beta']) + self.assertEqual(opts, [('-a', '1'), ('--alpha', '2')]) + self.assertEqual(args, ['-b', '--beta=5']) + self.assertError(getopt.getopt, cmdline, 'a:b', ['alpha', 'beta']) def test_gnu_getopt(self): @@ -191,6 +203,25 @@ def test_gnu_getopt(self): self.assertEqual(args, ['arg1', '-b', '1', '--alpha', '--beta=2', '--beta', '3', 'arg2']) + # Allow string for single long argument + opts, args = getopt.gnu_getopt(cmdline, 'ab:', 'alpha') + self.assertEqual(opts, [('-a', '')]) + self.assertEqual(args, ['arg1', '-b', '1', '--alpha', '--beta=2', + '--beta', '3', 'arg2']) + + # Pass everything after -- as args + cmdline = ['-a1', '--alpha=2', '--', '-b', '--beta=5'] + opts, args = getopt.gnu_getopt(cmdline, 'a:b', ['alpha=', 'beta']) + self.assertEqual(opts, [('-a', '1'), ('--alpha', '2')]) + self.assertEqual(args, ['-b', '--beta=5']) + + # In order arguments + cmdline = ["gamma", "--alpha=3"] + opts, args = getopt.gnu_getopt(cmdline, '-', ["alpha="]) + self.assertEqual(opts, [(None, ['gamma']), ('--alpha', '3')]) + self.assertEqual(args, []) + + def test_issue4629(self): longopts, shortopts = getopt.getopt(['--help='], '', ['help=']) self.assertEqual(longopts, [('--help', '')]) @@ -198,6 +229,10 @@ def test_issue4629(self): self.assertEqual(longopts, [('--help', 'x')]) self.assertRaises(getopt.GetoptError, getopt.getopt, ['--help='], '', ['help']) + def test_getopt_error_str(self): + error = getopt.GetoptError('option -a not recognized', 'a') + self.assertEqual(str(error), 'option -a not recognized') + def test_libref_examples(): """ Examples from the Library Reference: Doc/lib/libgetopt.tex diff --git a/Lib/test/test_getpass.py b/Lib/test/test_getpass.py index ab36535a1cfa8a9..9c3def2c3be59b1 100644 --- a/Lib/test/test_getpass.py +++ b/Lib/test/test_getpass.py @@ -201,5 +201,41 @@ def test_control_chars_with_echo_char(self): self.assertEqual('Password: *******\x08 \x08', mock_output.getvalue()) +class GetpassEchoCharTest(unittest.TestCase): + + def test_accept_none(self): + getpass._check_echo_char(None) + + @support.subTests('echo_char', ["*", "A", " "]) + def test_accept_single_printable_ascii(self, echo_char): + getpass._check_echo_char(echo_char) + + def test_reject_empty_string(self): + self.assertRaises(ValueError, getpass.getpass, echo_char="") + + @support.subTests('echo_char', ["***", "AA", "aA*!"]) + def test_reject_multi_character_strings(self, echo_char): + self.assertRaises(ValueError, getpass.getpass, echo_char=echo_char) + + @support.subTests('echo_char', [ + '\N{LATIN CAPITAL LETTER AE}', # non-ASCII single character + '\N{HEAVY BLACK HEART}', # non-ASCII multibyte character + ]) + def test_reject_non_ascii(self, echo_char): + self.assertRaises(ValueError, getpass.getpass, echo_char=echo_char) + + @support.subTests('echo_char', [ + ch for ch in map(chr, range(0, 128)) + if not ch.isprintable() + ]) + def test_reject_non_printable_characters(self, echo_char): + self.assertRaises(ValueError, getpass.getpass, echo_char=echo_char) + + # TypeError Rejection + @support.subTests('echo_char', [b"*", 0, 0.0, [], {}]) + def test_reject_non_string(self, echo_char): + self.assertRaises(TypeError, getpass.getpass, echo_char=echo_char) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_getpath.py b/Lib/test/test_getpath.py index f86df9d0d03485e..83f09f3495547ac 100644 --- a/Lib/test/test_getpath.py +++ b/Lib/test/test_getpath.py @@ -354,6 +354,27 @@ def test_venv_posix(self): actual = getpath(ns, expected) self.assertEqual(expected, actual) + def test_venv_posix_without_home_key(self): + ns = MockPosixNamespace( + argv0="/venv/bin/python3", + PREFIX="/usr", + ENV_PATH="/usr/bin", + ) + # Setup the bare minimum venv + ns.add_known_xfile("/usr/bin/python3") + ns.add_known_xfile("/venv/bin/python3") + ns.add_known_link("/venv/bin/python3", "/usr/bin/python3") + ns.add_known_file("/venv/pyvenv.cfg", [ + # home = key intentionally omitted + ]) + expected = dict( + executable="/venv/bin/python3", + prefix="/venv", + base_prefix="/usr", + ) + actual = getpath(ns, expected) + self.assertEqual(expected, actual) + def test_venv_changed_name_posix(self): "Test a venv layout on *nix." ns = MockPosixNamespace( diff --git a/Lib/test/test_gettext.py b/Lib/test/test_gettext.py index 33b7d75e3ff203d..9ad37909a8ec4e0 100644 --- a/Lib/test/test_gettext.py +++ b/Lib/test/test_gettext.py @@ -937,6 +937,13 @@ def test_lazy_import(self): ensure_lazy_imports("gettext", {"re", "warnings", "locale"}) +class TranslationFallbackTestCase(unittest.TestCase): + def test_translation_fallback(self): + with os_helper.temp_cwd() as tempdir: + t = gettext.translation('gettext', localedir=tempdir, fallback=True) + self.assertIsInstance(t, gettext.NullTranslations) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index c39565144bf7f40..cfb24a5c4578208 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -1,7 +1,7 @@ # Python test set -- part 1, grammar. # This just tests whether the parser accepts them all. -from test.support import check_syntax_error +from test.support import check_syntax_error, skip_wasi_stack_overflow from test.support import import_helper import annotationlib import inspect @@ -249,6 +249,18 @@ def test_eof_error(self): compile(s, "", "exec") self.assertIn("was never closed", str(cm.exception)) + @skip_wasi_stack_overflow() + def test_max_level(self): + # Macro defined in Parser/lexer/state.h + MAXLEVEL = 200 + + result = eval("(" * MAXLEVEL + ")" * MAXLEVEL) + self.assertEqual(result, ()) + + with self.assertRaises(SyntaxError) as cm: + eval("(" * (MAXLEVEL + 1) + ")" * (MAXLEVEL + 1)) + self.assertStartsWith(str(cm.exception), 'too many nested parentheses') + var_annot_global: int # a global annotated is necessary for test_var_annot @@ -1542,6 +1554,8 @@ def check(test): check('[None [i, j]]') check('[True [i, j]]') check('[... [i, j]]') + check('[t"{x}" [i, j]]') + check('[t"x={x}" [i, j]]') msg=r'indices must be integers or slices, not tuple; perhaps you missed a comma\?' check('[(1, 2) [i, j]]') @@ -1552,8 +1566,6 @@ def check(test): check('[f"x={x}" [i, j]]') check('["abc" [i, j]]') check('[b"abc" [i, j]]') - check('[t"{x}" [i, j]]') - check('[t"x={x}" [i, j]]') msg=r'indices must be integers or slices, not tuple;' check('[[1, 2] [3, 4]]') @@ -1574,6 +1586,7 @@ def check(test): check('[[1, 2] [f"{x}"]]') check('[[1, 2] [f"x={x}"]]') check('[[1, 2] ["abc"]]') + msg=r'indices must be integers or slices, not string.templatelib.Template;' check('[[1, 2] [t"{x}"]]') check('[[1, 2] [t"x={x}"]]') msg=r'indices must be integers or slices, not' diff --git a/Lib/test/test_grp.py b/Lib/test/test_grp.py index e52e17b8dc73667..ed86802f069e0f8 100644 --- a/Lib/test/test_grp.py +++ b/Lib/test/test_grp.py @@ -1,5 +1,7 @@ """Test script for the grp module.""" +import random +import string import unittest from test.support import import_helper @@ -50,61 +52,51 @@ def test_values_extended(self): def test_errors(self): self.assertRaises(TypeError, grp.getgrgid) self.assertRaises(TypeError, grp.getgrgid, 3.14) + self.assertRaises(TypeError, grp.getgrgid, 0.0) + self.assertRaises(TypeError, grp.getgrgid, 0, 0) + # should be out of gid_t range + self.assertRaises(OverflowError, grp.getgrgid, 2**128) + self.assertRaises(OverflowError, grp.getgrgid, -2**128) self.assertRaises(TypeError, grp.getgrnam) self.assertRaises(TypeError, grp.getgrnam, 42) - self.assertRaises(TypeError, grp.getgrall, 42) + self.assertRaises(TypeError, grp.getgrnam, b'root') + self.assertRaises(TypeError, grp.getgrnam, 'root', 0) # embedded null character self.assertRaisesRegex(ValueError, 'null', grp.getgrnam, 'a\x00b') + self.assertRaisesRegex(ValueError, 'null', grp.getgrnam, 'root\x00') + self.assertRaises(UnicodeEncodeError, grp.getgrnam, 'roo\udc74') + self.assertRaises(KeyError, grp.getgrnam, '') + self.assertRaises(TypeError, grp.getgrall, 42) - # try to get some errors - bynames = {} - bygids = {} - for (n, p, g, mem) in grp.getgrall(): - if not n or n == '+': - continue # skip NIS entries etc. - bynames[n] = g - bygids[g] = n - - allnames = list(bynames.keys()) - namei = 0 - fakename = allnames[namei] - while fakename in bynames: - chars = list(fakename) - for i in range(len(chars)): - if chars[i] == 'z': - chars[i] = 'A' - break - elif chars[i] == 'Z': - continue + # Find a non-existent group name. + # getgrall() will not necessarily report all existing groups + # (typical for LDAP based directories in big organizations). + for _ in range(30): + fakename = ''.join(random.choices(string.ascii_lowercase, k=6)) + try: + grp.getgrnam(fakename) + except KeyError: + break + else: + self.fail('Cannot find non-existent group name') + + # Find a non-existent gid. + maxgid = 2**31 + for _ in range(30): + fakegid = random.randrange(maxgid) + try: + grp.getgrgid(fakegid) + except KeyError: + break + except OverflowError: + if maxgid == 2**31: + maxgid = 2**16-1 + elif maxgid == 2**16-1: + maxgid = 2**15 else: - chars[i] = chr(ord(chars[i]) + 1) - break - else: - namei = namei + 1 - try: - fakename = allnames[namei] - except IndexError: - # should never happen... if so, just forget it - break - fakename = ''.join(chars) - - self.assertRaises(KeyError, grp.getgrnam, fakename) - - # Choose a non-existent gid. - fakegid = 4127 - while fakegid in bygids: - fakegid = (fakegid * 3) % 0x10000 - - self.assertRaises(KeyError, grp.getgrgid, fakegid) - - def test_noninteger_gid(self): - entries = grp.getgrall() - if not entries: - self.skipTest('no groups') - # Choose an existent gid. - gid = entries[0][2] - self.assertRaises(TypeError, grp.getgrgid, float(gid)) - self.assertRaises(TypeError, grp.getgrgid, str(gid)) + raise + else: + self.fail('Cannot find non-existent gid') if __name__ == "__main__": diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index fa5de7c190e6a36..ccbacc7c19b6e6e 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -331,13 +331,13 @@ def test_mode(self): def test_1647484(self): for mode in ('wb', 'rb'): with gzip.GzipFile(self.filename, mode) as f: - self.assertTrue(hasattr(f, "name")) + self.assertHasAttr(f, "name") self.assertEqual(f.name, self.filename) def test_paddedfile_getattr(self): self.test_write() with gzip.GzipFile(self.filename, 'rb') as f: - self.assertTrue(hasattr(f.fileobj, "name")) + self.assertHasAttr(f.fileobj, "name") self.assertEqual(f.fileobj.name, self.filename) def test_mtime(self): @@ -345,7 +345,7 @@ def test_mtime(self): with gzip.GzipFile(self.filename, 'w', mtime = mtime) as fWrite: fWrite.write(data1) with gzip.GzipFile(self.filename) as fRead: - self.assertTrue(hasattr(fRead, 'mtime')) + self.assertHasAttr(fRead, 'mtime') self.assertIsNone(fRead.mtime) dataRead = fRead.read() self.assertEqual(dataRead, data1) @@ -460,7 +460,7 @@ def test_zero_padded_file(self): self.assertEqual(d, data1 * 50, "Incorrect data in file") def test_gzip_BadGzipFile_exception(self): - self.assertTrue(issubclass(gzip.BadGzipFile, OSError)) + self.assertIsSubclass(gzip.BadGzipFile, OSError) def test_bad_gzip_file(self): with open(self.filename, 'wb') as file: diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 5e3356a02f31b66..c7d7b801ac732a3 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -12,6 +12,7 @@ import itertools import logging import os +import re import sys import sysconfig import tempfile @@ -27,32 +28,28 @@ from http.client import HTTPException -default_builtin_hashes = {'md5', 'sha1', 'sha256', 'sha512', 'sha3', 'blake2'} +default_builtin_hashes = {'md5', 'sha1', 'sha2', 'sha3', 'blake2'} # --with-builtin-hashlib-hashes override builtin_hashes = sysconfig.get_config_var("PY_BUILTIN_HASHLIB_HASHES") if builtin_hashes is None: builtin_hashes = default_builtin_hashes else: - builtin_hashes = { - m.strip() for m in builtin_hashes.strip('"').lower().split(",") - } + builtin_hash_names = builtin_hashes.strip('"').lower().split(",") + builtin_hashes = set(map(str.strip, builtin_hash_names)) -# hashlib with and without OpenSSL backend for PBKDF2 -# only import builtin_hashlib when all builtin hashes are available. -# Otherwise import prints noise on stderr +# Public 'hashlib' module with OpenSSL backend for PBKDF2. openssl_hashlib = import_fresh_module('hashlib', fresh=['_hashlib']) -if builtin_hashes == default_builtin_hashes: - builtin_hashlib = import_fresh_module('hashlib', blocked=['_hashlib']) -else: - builtin_hashlib = None try: - from _hashlib import HASH, HASHXOF, openssl_md_meth_names, get_fips_mode + import _hashlib except ImportError: - HASH = None - HASHXOF = None - openssl_md_meth_names = frozenset() - + _hashlib = None +# The extension module may exist but only define some of these. gh-141907 +HASH = getattr(_hashlib, 'HASH', None) +HASHXOF = getattr(_hashlib, 'HASHXOF', None) +openssl_md_meth_names = getattr(_hashlib, 'openssl_md_meth_names', frozenset()) +get_fips_mode = getattr(_hashlib, 'get_fips_mode', None) +if not get_fips_mode: def get_fips_mode(): return 0 @@ -130,8 +127,11 @@ def __init__(self, *args, **kwargs): algorithms.add(algorithm.lower()) _blake2 = self._conditional_import_module('_blake2') + blake2_hashes = {'blake2b', 'blake2s'} if _blake2: - algorithms.update({'blake2b', 'blake2s'}) + algorithms.update(blake2_hashes) + else: + algorithms.difference_update(blake2_hashes) self.constructors_to_test = {} for algorithm in algorithms: @@ -141,19 +141,18 @@ def __init__(self, *args, **kwargs): # of hashlib.new given the algorithm name. for algorithm, constructors in self.constructors_to_test.items(): constructors.add(getattr(hashlib, algorithm)) - def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs): - if data is None: - return hashlib.new(_alg, **kwargs) - return hashlib.new(_alg, data, **kwargs) - constructors.add(_test_algorithm_via_hashlib_new) + def c(*args, __algorithm_name=algorithm, **kwargs): + return hashlib.new(__algorithm_name, *args, **kwargs) + c.__name__ = f'do_test_algorithm_via_hashlib_new_{algorithm}' + constructors.add(c) _hashlib = self._conditional_import_module('_hashlib') self._hashlib = _hashlib if _hashlib: # These algorithms should always be present when this module # is compiled. If not, something was compiled wrong. - self.assertTrue(hasattr(_hashlib, 'openssl_md5')) - self.assertTrue(hasattr(_hashlib, 'openssl_sha1')) + self.assertHasAttr(_hashlib, 'openssl_md5') + self.assertHasAttr(_hashlib, 'openssl_sha1') for algorithm, constructors in self.constructors_to_test.items(): constructor = getattr(_hashlib, 'openssl_'+algorithm, None) if constructor: @@ -224,7 +223,12 @@ def test_algorithms_available(self): # all available algorithms must be loadable, bpo-47101 self.assertNotIn("undefined", hashlib.algorithms_available) for name in hashlib.algorithms_available: - digest = hashlib.new(name, usedforsecurity=False) + with self.subTest(name): + try: + _ = hashlib.new(name, usedforsecurity=False) + except ValueError as exc: + self.skip_if_blake2_not_builtin(name, exc) + raise def test_usedforsecurity_true(self): hashlib.new("sha256", usedforsecurity=True) @@ -250,6 +254,79 @@ def test_usedforsecurity_false(self): self._hashlib.new("md5", usedforsecurity=False) self._hashlib.openssl_md5(usedforsecurity=False) + @unittest.skipIf(get_fips_mode(), "skip in FIPS mode") + def test_clinic_signature(self): + for constructor in self.hash_constructors: + with self.subTest(constructor.__name__): + constructor(b'') + constructor(data=b'') + constructor(string=b'') # should be deprecated in the future + + digest_name = constructor(b'').name + with self.subTest(digest_name): + hashlib.new(digest_name, b'') + hashlib.new(digest_name, data=b'') + hashlib.new(digest_name, string=b'') + # Make sure that _hashlib contains the constructor + # to test when using a combination of libcrypto and + # interned hash implementations. + if self._hashlib and digest_name in self._hashlib._constructors: + self._hashlib.new(digest_name, b'') + self._hashlib.new(digest_name, data=b'') + self._hashlib.new(digest_name, string=b'') + + @unittest.skipIf(get_fips_mode(), "skip in FIPS mode") + def test_clinic_signature_errors(self): + nomsg = b'' + mymsg = b'msg' + conflicting_call = re.escape( + "'data' and 'string' are mutually exclusive " + "and support for 'string' keyword parameter " + "is slated for removal in a future version." + ) + duplicated_param = re.escape("given by name ('data') and position") + unexpected_param = re.escape("got an unexpected keyword argument '_'") + for args, kwds, errmsg in [ + # Reject duplicated arguments before unknown keyword arguments. + ((nomsg,), dict(data=nomsg, _=nomsg), duplicated_param), + ((mymsg,), dict(data=nomsg, _=nomsg), duplicated_param), + # Reject duplicated arguments before conflicting ones. + *itertools.product( + [[nomsg], [mymsg]], + [dict(data=nomsg), dict(data=nomsg, string=nomsg)], + [duplicated_param] + ), + # Reject unknown keyword arguments before conflicting ones. + *itertools.product( + [()], + [ + dict(_=None), + dict(data=nomsg, _=None), + dict(string=nomsg, _=None), + dict(string=nomsg, data=nomsg, _=None), + ], + [unexpected_param] + ), + ((nomsg,), dict(_=None), unexpected_param), + ((mymsg,), dict(_=None), unexpected_param), + # Reject conflicting arguments. + [(nomsg,), dict(string=nomsg), conflicting_call], + [(mymsg,), dict(string=nomsg), conflicting_call], + [(), dict(data=nomsg, string=nomsg), conflicting_call], + ]: + for constructor in self.hash_constructors: + digest_name = constructor(b'').name + with self.subTest(constructor.__name__, args=args, kwds=kwds): + with self.assertRaisesRegex(TypeError, errmsg): + constructor(*args, **kwds) + with self.subTest(digest_name, args=args, kwds=kwds): + with self.assertRaisesRegex(TypeError, errmsg): + hashlib.new(digest_name, *args, **kwds) + if (self._hashlib and + digest_name in self._hashlib._constructors): + with self.assertRaisesRegex(TypeError, errmsg): + self._hashlib.new(digest_name, *args, **kwds) + def test_unknown_hash(self): self.assertRaises(ValueError, hashlib.new, 'spam spam spam spam spam') self.assertRaises(TypeError, hashlib.new, 1) @@ -374,6 +451,7 @@ def test_sha3_256_update_over_4gb(self): self.assertEqual(h.hexdigest(), "e2d4535e3b613135c14f2fe4e026d7ad8d569db44901740beffa30d430acb038") @requires_resource('cpu') + @requires_blake2 def test_blake2_update_over_4gb(self): # blake2s or blake2b doesn't matter based on how our C code is structured, this tests the # common loop macro logic. @@ -500,9 +578,14 @@ def check_sha3(self, name, capacity, rate, suffix): constructors = self.constructors_to_test[name] for hash_object_constructor in constructors: m = hash_object_constructor() - if HASH is not None and isinstance(m, HASH): - # _hashopenssl's variant does not have extra SHA3 attributes - continue + if name.startswith('shake_'): + if HASHXOF is not None and isinstance(m, HASHXOF): + # _hashopenssl's variant does not have extra SHA3 attributes + continue + else: + if HASH is not None and isinstance(m, HASH): + # _hashopenssl's variant does not have extra SHA3 attributes + continue self.assertEqual(capacity + rate, 1600) self.assertEqual(m._capacity_bits, capacity) self.assertEqual(m._rate_bits, rate) @@ -659,6 +742,12 @@ def test_case_sha512_3(self): "e718483d0ce769644e2e42c7bc15b4638e1f98b13b2044285632a803afa973eb"+ "de0ff244877ea60a4cb0432ce577c31beb009c5c2c49aa2e4eadb217ad8cc09b") + def skip_if_blake2_not_builtin(self, name, skip_reason): + # blake2 builtins may be absent if python built with + # a subset of --with-builtin-hashlib-hashes or none. + if "blake2" in name and "blake2" not in builtin_hashes: + self.skipTest(skip_reason) + def check_blake2(self, constructor, salt_size, person_size, key_size, digest_size, max_offset): self.assertEqual(constructor.SALT_SIZE, salt_size) @@ -719,8 +808,6 @@ def check_blake2(self, constructor, salt_size, person_size, key_size, self.assertRaises(ValueError, constructor, node_offset=-1) self.assertRaises(OverflowError, constructor, node_offset=max_offset+1) - self.assertRaises(TypeError, constructor, data=b'') - self.assertRaises(TypeError, constructor, string=b'') self.assertRaises(TypeError, constructor, '') constructor( @@ -1006,7 +1093,8 @@ def test_disallow_instantiation(self): def test_hash_disallow_instantiation(self): # internal types like _hashlib.HASH are not constructable support.check_disallow_instantiation(self, HASH) - support.check_disallow_instantiation(self, HASHXOF) + if HASHXOF is not None: + support.check_disallow_instantiation(self, HASHXOF) def test_readonly_types(self): for algorithm, constructors in self.constructors_to_test.items(): diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index 70c7943772249e5..1d270b2a43c1845 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -1,16 +1,41 @@ +"""Test suite for HMAC. + +Python provides three different implementations of HMAC: + +- OpenSSL HMAC using OpenSSL hash functions. +- HACL* HMAC using HACL* hash functions. +- Generic Python HMAC using user-defined hash functions. + +The generic Python HMAC implementation is able to use OpenSSL +callables or names, HACL* named hash functions or arbitrary +objects implementing PEP 247 interface. + +In the two first cases, Python HMAC wraps a C HMAC object (either OpenSSL +or HACL*-based). As a last resort, HMAC is re-implemented in pure Python. +It is however interesting to test the pure Python implementation against +the OpenSSL and HACL* hash functions. +""" + import binascii import functools import hmac import hashlib import random -import test.support.hashlib_helper as hashlib_helper import types import unittest -import unittest.mock as mock import warnings from _operator import _compare_digest as operator_compare_digest +from test.support import _4G, bigmemtest from test.support import check_disallow_instantiation -from test.support.import_helper import import_fresh_module, import_module +from test.support import hashlib_helper, import_helper +from test.support.hashlib_helper import ( + BuiltinHashFunctionsTrait, + HashFunctionsTrait, + NamedHashFunctionsTrait, + OpenSSLHashFunctionsTrait, +) +from test.support.import_helper import import_fresh_module +from unittest.mock import patch try: import _hashlib @@ -382,50 +407,7 @@ class BuiltinAssertersMixin(ThroughBuiltinAPIMixin, AssertersMixin): pass -class HashFunctionsTrait: - """Trait class for 'hashfunc' in hmac_new() and hmac_digest().""" - - ALGORITHMS = [ - 'md5', 'sha1', - 'sha224', 'sha256', 'sha384', 'sha512', - 'sha3_224', 'sha3_256', 'sha3_384', 'sha3_512', - ] - - # By default, a missing algorithm skips the test that uses it. - _ = property(lambda self: self.skipTest("missing hash function")) - md5 = sha1 = _ - sha224 = sha256 = sha384 = sha512 = _ - sha3_224 = sha3_256 = sha3_384 = sha3_512 = _ - del _ - - -class WithOpenSSLHashFunctions(HashFunctionsTrait): - """Test a HMAC implementation with an OpenSSL-based callable 'hashfunc'.""" - - @classmethod - def setUpClass(cls): - super().setUpClass() - - for name in cls.ALGORITHMS: - @property - @hashlib_helper.requires_openssl_hashdigest(name) - def func(self, *, __name=name): # __name needed to bind 'name' - return getattr(_hashlib, f'openssl_{__name}') - setattr(cls, name, func) - - -class WithNamedHashFunctions(HashFunctionsTrait): - """Test a HMAC implementation with a named 'hashfunc'.""" - - @classmethod - def setUpClass(cls): - super().setUpClass() - - for name in cls.ALGORITHMS: - setattr(cls, name, name) - - -class RFCTestCaseMixin(AssertersMixin, HashFunctionsTrait): +class RFCTestCaseMixin(HashFunctionsTrait, AssertersMixin): """Test HMAC implementations against RFC 2202/4231 and NIST test vectors. - Test vectors for MD5 and SHA-1 are taken from RFC 2202. @@ -739,26 +721,83 @@ def test_sha3_512_nist(self): ) -class PyRFCTestCase(ThroughObjectMixin, PyAssertersMixin, - WithOpenSSLHashFunctions, RFCTestCaseMixin, - unittest.TestCase): +class PurePythonInitHMAC(PyModuleMixin, HashFunctionsTrait): + + @classmethod + def setUpClass(cls): + super().setUpClass() + for meth in ['_init_openssl_hmac', '_init_builtin_hmac']: + fn = getattr(cls.hmac.HMAC, meth) + cm = patch.object(cls.hmac.HMAC, meth, autospec=True, wraps=fn) + cls.enterClassContext(cm) + + @classmethod + def tearDownClass(cls): + cls.hmac.HMAC._init_openssl_hmac.assert_not_called() + cls.hmac.HMAC._init_builtin_hmac.assert_not_called() + # Do not assert that HMAC._init_old() has been called as it's tricky + # to determine whether a test for a specific hash function has been + # executed or not. On regular builds, it will be called but if a + # hash function is not available, it's hard to detect for which + # test we should checj HMAC._init_old() or not. + super().tearDownClass() + + +class PyRFCOpenSSLTestCase(ThroughObjectMixin, + PyAssertersMixin, + OpenSSLHashFunctionsTrait, + RFCTestCaseMixin, + PurePythonInitHMAC, + unittest.TestCase): """Python implementation of HMAC using hmac.HMAC(). - The underlying hash functions are OpenSSL-based. + The underlying hash functions are OpenSSL-based but + _init_old() is used instead of _init_openssl_hmac(). """ -class PyDotNewRFCTestCase(ThroughModuleAPIMixin, PyAssertersMixin, - WithOpenSSLHashFunctions, RFCTestCaseMixin, - unittest.TestCase): +class PyRFCBuiltinTestCase(ThroughObjectMixin, + PyAssertersMixin, + BuiltinHashFunctionsTrait, + RFCTestCaseMixin, + PurePythonInitHMAC, + unittest.TestCase): + """Python implementation of HMAC using hmac.HMAC(). + + The underlying hash functions are HACL*-based but + _init_old() is used instead of _init_builtin_hmac(). + """ + + +class PyDotNewOpenSSLRFCTestCase(ThroughModuleAPIMixin, + PyAssertersMixin, + OpenSSLHashFunctionsTrait, + RFCTestCaseMixin, + PurePythonInitHMAC, + unittest.TestCase): """Python implementation of HMAC using hmac.new(). - The underlying hash functions are OpenSSL-based. + The underlying hash functions are OpenSSL-based but + _init_old() is used instead of _init_openssl_hmac(). + """ + + +class PyDotNewBuiltinRFCTestCase(ThroughModuleAPIMixin, + PyAssertersMixin, + BuiltinHashFunctionsTrait, + RFCTestCaseMixin, + PurePythonInitHMAC, + unittest.TestCase): + """Python implementation of HMAC using hmac.new(). + + The underlying hash functions are HACL-based but + _init_old() is used instead of _init_openssl_hmac(). """ class OpenSSLRFCTestCase(OpenSSLAssertersMixin, - WithOpenSSLHashFunctions, RFCTestCaseMixin, + OpenSSLHashFunctionsTrait, + RFCTestCaseMixin, unittest.TestCase): """OpenSSL implementation of HMAC. @@ -767,7 +806,8 @@ class OpenSSLRFCTestCase(OpenSSLAssertersMixin, class BuiltinRFCTestCase(BuiltinAssertersMixin, - WithNamedHashFunctions, RFCTestCaseMixin, + NamedHashFunctionsTrait, + RFCTestCaseMixin, unittest.TestCase): """Built-in HACL* implementation of HMAC. @@ -784,12 +824,6 @@ def assert_hmac_extra_cases( self.check_hmac_hexdigest(key, msg, hexdigest, digest_size, func) -# TODO(picnixz): once we have a HACL* HMAC, we should also test the Python -# implementation of HMAC with a HACL*-based hash function. For now, we only -# test it partially via the '_sha2' module, but for completeness we could -# also test the RFC test vectors against all possible implementations. - - class DigestModTestCaseMixin(CreatorMixin, DigestMixin): """Tests for the 'digestmod' parameter for hmac_new() and hmac_digest().""" @@ -916,7 +950,11 @@ class PyConstructorTestCase(ThroughObjectMixin, PyConstructorBaseMixin, class PyModuleConstructorTestCase(ThroughModuleAPIMixin, PyConstructorBaseMixin, unittest.TestCase): - """Test the hmac.new() and hmac.digest() functions.""" + """Test the hmac.new() and hmac.digest() functions. + + Note that "self.hmac" is imported by blocking "_hashlib" and "_hmac". + For testing functions in "hmac", extend PyMiscellaneousTests instead. + """ def test_hmac_digest_digestmod_parameter(self): func = self.hmac_digest @@ -1038,6 +1076,15 @@ def test_properties(self): self.assertEqual(h.digest_size, self.digest_size) self.assertEqual(h.block_size, self.block_size) + def test_copy(self): + # Test a generic copy() and the attributes it exposes. + # See https://github.com/python/cpython/issues/142451. + h1 = self.hmac_new(b"my secret key", digestmod=self.digestname) + h2 = h1.copy() + self.assertEqual(h1.name, h2.name) + self.assertEqual(h1.digest_size, h2.digest_size) + self.assertEqual(h1.block_size, h2.block_size) + def test_repr(self): # HMAC object representation may differ across implementations raise NotImplementedError @@ -1412,9 +1459,8 @@ def test_hmac_constructor_uses_builtin(self): hmac = import_fresh_module("hmac", blocked=["_hashlib"]) def watch_method(cls, name): - return mock.patch.object( - cls, name, autospec=True, wraps=getattr(cls, name) - ) + wraps = getattr(cls, name) + return patch.object(cls, name, autospec=True, wraps=wraps) with ( watch_method(hmac.HMAC, '_init_openssl_hmac') as f, @@ -1466,6 +1512,48 @@ def test_with_fallback(self): finally: cache.pop('foo') + @hashlib_helper.requires_openssl_hashdigest("md5") + @bigmemtest(size=_4G + 5, memuse=2, dry_run=False) + def test_hmac_digest_overflow_error_openssl_only(self, size): + hmac = import_fresh_module("hmac", blocked=["_hmac"]) + self.do_test_hmac_digest_overflow_error_switch_to_slow(hmac, size) + + @hashlib_helper.requires_builtin_hashdigest("_md5", "md5") + @bigmemtest(size=_4G + 5, memuse=2, dry_run=False) + def test_hmac_digest_overflow_error_builtin_only(self, size): + hmac = import_fresh_module("hmac", blocked=["_hashlib"]) + self.do_test_hmac_digest_overflow_error_switch_to_slow(hmac, size) + + def do_test_hmac_digest_overflow_error_switch_to_slow(self, hmac, size): + """Check that hmac.digest() falls back to pure Python. + + The *hmac* argument implements the HMAC module interface. + The *size* argument is a large key size or message size that would + trigger an OverflowError in the C implementation(s) of hmac.digest(). + """ + + bigkey = b'K' * size + bigmsg = b'M' * size + + with patch.object(hmac, "_compute_digest_fallback") as slow: + hmac.digest(bigkey, b'm', "md5") + slow.assert_called_once() + + with patch.object(hmac, "_compute_digest_fallback") as slow: + hmac.digest(b'k', bigmsg, "md5") + slow.assert_called_once() + + @hashlib_helper.requires_hashdigest("md5", openssl=True) + @bigmemtest(size=_4G + 5, memuse=2, dry_run=False) + def test_hmac_digest_no_overflow_error_in_fallback(self, size): + hmac = import_fresh_module("hmac", blocked=["_hashlib", "_hmac"]) + + for key, msg in [(b'K' * size, b'm'), (b'k', b'M' * size)]: + with self.subTest(keysize=len(key), msgsize=len(msg)): + with patch.object(hmac, "_compute_digest_fallback") as slow: + hmac.digest(key, msg, "md5") + slow.assert_called_once() + class BuiltinMiscellaneousTests(BuiltinModuleMixin, unittest.TestCase): """HMAC-BLAKE2 is not standardized as BLAKE2 is a keyed hash function. @@ -1478,7 +1566,7 @@ class BuiltinMiscellaneousTests(BuiltinModuleMixin, unittest.TestCase): @classmethod def setUpClass(cls): super().setUpClass() - cls.blake2 = import_module("_blake2") + cls.blake2 = import_helper.import_module("_blake2") cls.blake2b = cls.blake2.blake2b cls.blake2s = cls.blake2.blake2s diff --git a/Lib/test/test_htmlparser.py b/Lib/test/test_htmlparser.py index b42a611c62c0aa1..3fdaed4ff46b9d0 100644 --- a/Lib/test/test_htmlparser.py +++ b/Lib/test/test_htmlparser.py @@ -5,14 +5,30 @@ import unittest from unittest.mock import patch +from test import support + + +SAMPLE_RCDATA = ( + '' + "" + '' + '' + '' + '\u2603' +) + +SAMPLE_RAWTEXT = SAMPLE_RCDATA + '&☺' class EventCollector(html.parser.HTMLParser): - def __init__(self, *args, **kw): + def __init__(self, *args, autocdata=False, **kw): + self.autocdata = autocdata self.events = [] self.append = self.events.append html.parser.HTMLParser.__init__(self, *args, **kw) + if autocdata: + self._set_support_cdata(False) def get_events(self): # Normalize the list of events so that buffer artefacts don't @@ -33,12 +49,16 @@ def get_events(self): def handle_starttag(self, tag, attrs): self.append(("starttag", tag, attrs)) + if self.autocdata and tag == 'svg': + self._set_support_cdata(True) def handle_startendtag(self, tag, attrs): self.append(("startendtag", tag, attrs)) def handle_endtag(self, tag): self.append(("endtag", tag)) + if self.autocdata and tag == 'svg': + self._set_support_cdata(False) # all other markup @@ -80,14 +100,27 @@ def handle_entityref(self, data): self.fail('This should never be called with convert_charrefs=True') +# The normal event collector normalizes the events in get_events, +# so we override it to return the original list of events. +class EventCollectorNoNormalize(EventCollector): + def get_events(self): + return self.events + + class TestCaseBase(unittest.TestCase): - def get_collector(self): - return EventCollector(convert_charrefs=False) + def get_collector(self, convert_charrefs=False): + return EventCollector(convert_charrefs=convert_charrefs) - def _run_check(self, source, expected_events, collector=None): + def _run_check(self, source, expected_events, + *, collector=None, convert_charrefs=False): if collector is None: - collector = self.get_collector() + collector = self.get_collector(convert_charrefs=convert_charrefs) + if isinstance(source, str): + # Also feed the whole string at once, not just character by + # character (below), to exercise different input buffering. + self._run_check([source], expected_events, + convert_charrefs=convert_charrefs) parser = collector for s in source: parser.feed(s) @@ -101,7 +134,7 @@ def _run_check(self, source, expected_events, collector=None): def _run_check_extra(self, source, events): self._run_check(source, events, - EventCollectorExtra(convert_charrefs=False)) + collector=EventCollectorExtra(convert_charrefs=False)) class HTMLParserTestCase(TestCaseBase): @@ -160,10 +193,87 @@ def test_malformatted_charref(self): ]) def test_unclosed_entityref(self): - self._run_check("&entityref foo", [ - ("entityref", "entityref"), - ("data", " foo"), - ]) + self._run_check('> <', [('entityref', 'gt'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('> <', [('data', '> <')], convert_charrefs=True) + + self._run_check('&undefined <', + [('entityref', 'undefined'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('&undefined <', [('data', '&undefined <')], + convert_charrefs=True) + + self._run_check('>undefined <', + [('entityref', 'gtundefined'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('>undefined <', [('data', '>undefined <')], + convert_charrefs=True) + + self._run_check('& <', [('data', '& '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('& <', [('data', '& <')], convert_charrefs=True) + + def test_eof_in_entityref(self): + self._run_check('>', [('entityref', 'gt')], convert_charrefs=False) + self._run_check('>', [('data', '>')], convert_charrefs=True) + + self._run_check('&g', [('entityref', 'g')], convert_charrefs=False) + self._run_check('&g', [('data', '&g')], convert_charrefs=True) + + self._run_check('&undefined', [('entityref', 'undefined')], + convert_charrefs=False) + self._run_check('&undefined', [('data', '&undefined')], + convert_charrefs=True) + + self._run_check('>undefined', [('entityref', 'gtundefined')], + convert_charrefs=False) + self._run_check('>undefined', [('data', '>undefined')], + convert_charrefs=True) + + self._run_check('&', [('data', '&')], convert_charrefs=False) + self._run_check('&', [('data', '&')], convert_charrefs=True) + + def test_unclosed_charref(self): + self._run_check('{ <', [('charref', '123'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('{ <', [('data', '{ <')], convert_charrefs=True) + self._run_check('« <', [('charref', 'xab'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('« <', [('data', '\xab <')], convert_charrefs=True) + + self._run_check('� <', + [('charref', '123456789'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('� <', [('data', '\ufffd <')], + convert_charrefs=True) + self._run_check('� <', + [('charref', 'x123456789'), ('data', ' '), ('entityref', 'lt')], + convert_charrefs=False) + self._run_check('� <', [('data', '\ufffd <')], + convert_charrefs=True) + + self._run_check('&# <', [('data', '&# '), ('entityref', 'lt')], convert_charrefs=False) + self._run_check('&# <', [('data', '&# <')], convert_charrefs=True) + self._run_check('&#x <', [('data', '&#x '), ('entityref', 'lt')], convert_charrefs=False) + self._run_check('&#x <', [('data', '&#x <')], convert_charrefs=True) + + def test_eof_in_charref(self): + self._run_check('{', [('charref', '123')], convert_charrefs=False) + self._run_check('{', [('data', '{')], convert_charrefs=True) + self._run_check('«', [('charref', 'xab')], convert_charrefs=False) + self._run_check('«', [('data', '\xab')], convert_charrefs=True) + + self._run_check('�', [('charref', '123456789')], + convert_charrefs=False) + self._run_check('�', [('data', '\ufffd')], convert_charrefs=True) + self._run_check('�', [('charref', 'x123456789')], + convert_charrefs=False) + self._run_check('�', [('data', '\ufffd')], convert_charrefs=True) + + self._run_check('&#', [('data', '&#')], convert_charrefs=False) + self._run_check('&#', [('data', '&#')], convert_charrefs=True) + self._run_check('&#x', [('data', '&#x')], convert_charrefs=False) + self._run_check('&#x', [('data', '&#x')], convert_charrefs=True) def test_bad_nesting(self): # Strangely, this *is* supposed to test that overlapping @@ -264,8 +374,7 @@ def test_get_starttag_text(self): ("starttag", "foo:bar", [("one", "1"), ("two", "2")]), ("starttag_text", s)]) - def test_cdata_content(self): - contents = [ + @support.subTests('content', [ ' ¬-an-entity-ref;', "", '

    ', @@ -278,60 +387,243 @@ def test_cdata_content(self): 'src="http://www.example.org/r=\'+new ' 'Date().getTime()+\'"><\\/s\'+\'cript>\');\n//]]>'), '\n\n', - 'foo = "";', '', - # these two should be invalid according to the HTML 5 spec, - # section 8.1.2.2 - #'foo = ', - #'foo = ', - ] - elements = ['script', 'style', 'SCRIPT', 'STYLE', 'Script', 'Style'] - for content in contents: - for element in elements: - element_lower = element.lower() - s = '<{element}>{content}'.format(element=element, - content=content) - self._run_check(s, [("starttag", element_lower, []), - ("data", content), - ("endtag", element_lower)]) - - def test_cdata_with_closing_tags(self): + ]) + def test_script_content(self, content): + s = f'' + self._run_check(s, [ + ("starttag", "script", []), + ("data", content), + ("endtag", "script"), + ]) + + @support.subTests('content', [ + 'a::before { content: ""; }', + 'a::before { content: "¬-an-entity-ref;"; }', + 'a::before { content: ""; }', + 'a::before { content: "\u2603"; }', + ]) + def test_style_content(self, content): + s = f'' + self._run_check(s, [("starttag", "style", []), + ("data", content), + ("endtag", "style")]) + + @support.subTests('tag', ['title', 'textarea']) + def test_rcdata_content(self, tag): + source = f"<{tag}>{SAMPLE_RCDATA}" + self._run_check(source, [ + ("starttag", tag, []), + ("data", SAMPLE_RCDATA), + ("endtag", tag), + ]) + source = f"<{tag}>&" + self._run_check(source, [ + ("starttag", tag, []), + ('entityref', 'amp'), + ("endtag", tag), + ]) + + @support.subTests('tag', + ['style', 'xmp', 'iframe', 'noembed', 'noframes', 'script']) + def test_rawtext_content(self, tag): + source = f"<{tag}>{SAMPLE_RAWTEXT}" + self._run_check(source, [ + ("starttag", tag, []), + ("data", SAMPLE_RAWTEXT), + ("endtag", tag), + ]) + + def test_noscript_content(self): + source = f"" + # scripting=False -- normal mode + self._run_check(source, [ + ('starttag', 'noscript', []), + ('comment', ' not a comment '), + ('starttag', 'not', [('a', 'start tag')]), + ('unknown decl', 'CDATA[not a cdata'), + ('comment', 'not a bogus comment'), + ('endtag', 'not'), + ('data', '☃'), + ('entityref', 'amp'), + ('charref', '9786'), + ('endtag', 'noscript'), + ]) + # scripting=True -- RAWTEXT mode + self._run_check(source, [ + ("starttag", "noscript", []), + ("data", SAMPLE_RAWTEXT), + ("endtag", "noscript"), + ], collector=EventCollector(scripting=True)) + + def test_plaintext_content(self): + content = SAMPLE_RAWTEXT + '' # not closing + source = f"

    {content}" + self._run_check(source, [ + ("starttag", "plaintext", []), + ("data", content), + ]) + + @support.subTests('endtag', ['script', 'SCRIPT', 'script ', 'script\n', + 'script/', 'script foo=bar', 'script foo=">"']) + def test_script_closing_tag(self, endtag): # see issue #13358 # make sure that HTMLParser calls handle_data only once for each CDATA. - # The normal event collector normalizes the events in get_events, - # so we override it to return the original list of events. - class Collector(EventCollector): - def get_events(self): - return self.events - content = """<!-- not a comment --> &not-an-entity-ref; <a href="" /> </p><p> <span></span></style> '</script' + '>'""" - for element in [' script', 'script ', ' script ', - '\nscript', 'script\n', '\nscript\n']: - element_lower = element.lower().strip() - s = '<script>{content}</{element}>'.format(element=element, - content=content) - self._run_check(s, [("starttag", element_lower, []), - ("data", content), - ("endtag", element_lower)], - collector=Collector(convert_charrefs=False)) + s = f'<ScrIPt>{content}</{endtag}>' + self._run_check(s, [("starttag", "script", []), + ("data", content), + ("endtag", "script")], + collector=EventCollectorNoNormalize(convert_charrefs=False)) + + @support.subTests('tag', [ + 'script', 'style', 'xmp', 'iframe', 'noembed', 'noframes', + 'textarea', 'title', 'noscript', + ]) + def test_closing_tag(self, tag): + for endtag in [tag, tag.upper(), f'{tag} ', f'{tag}\n', + f'{tag}/', f'{tag} foo=bar', f'{tag} foo=">"']: + content = "<!-- not a comment --><i>Spam</i>" + s = f'<{tag.upper()}>{content}</{endtag}>' + self._run_check(s, [ + ("starttag", tag, []), + ('data', content), + ("endtag", tag), + ], collector=EventCollectorNoNormalize(convert_charrefs=False, scripting=True)) + + @support.subTests('tag', [ + 'script', 'style', 'xmp', 'iframe', 'noembed', 'noframes', + 'textarea', 'title', 'noscript', + ]) + def test_invalid_closing_tag(self, tag): + content = ( + f'< /{tag}>' + f'</ {tag}>' + f'</{tag}x>' + f'</{tag}\v>' + f'</{tag}\xa0>' + ) + source = f"<{tag}>{content}</{tag}>" + self._run_check(source, [ + ("starttag", tag, []), + ("data", content), + ("endtag", tag), + ], collector=EventCollector(convert_charrefs=False, scripting=True)) + + @support.subTests('tag,endtag', [ + ('title', 'tıtle'), + ('style', 'ſtyle'), + ('style', 'ſtyle'), + ('style', 'style'), + ('iframe', 'ıframe'), + ('noframes', 'noframeſ'), + ('noscript', 'noſcript'), + ('noscript', 'noscrıpt'), + ('script', 'ſcript'), + ('script', 'scrıpt'), + ]) + def test_invalid_nonascii_closing_tag(self, tag, endtag): + content = f"<br></{endtag}>" + source = f"<{tag}>{content}" + self._run_check(source, [ + ("starttag", tag, []), + ("data", content), + ], collector=EventCollector(convert_charrefs=False, scripting=True)) + source = f"<{tag}>{content}</{tag}>" + self._run_check(source, [ + ("starttag", tag, []), + ("data", content), + ("endtag", tag), + ], collector=EventCollector(convert_charrefs=False, scripting=True)) + + @support.subTests('tail,end', [ + ('', False), + ('<', False), + ('</', False), + ('</s', False), + ('</script', False), + ('</script ', True), + ('</script foo=bar', True), + ('</script foo=">', True), + ]) + def test_eof_in_script(self, tail, end): + content = "a = 123" + s = f'<ScrIPt>{content}{tail}' + self._run_check(s, [("starttag", "script", []), + ("data", content if end else content + tail)], + collector=EventCollectorNoNormalize(convert_charrefs=False)) + + @support.subTests('tail,end', [ + ('', False), + ('<', False), + ('</', False), + ('</t', False), + ('</title', False), + ('</title ', True), + ('</title foo=bar', True), + ('</title foo=">', True), + ]) + def test_eof_in_title(self, tail, end): + s = f'<TitLe>Egg &amp; Spam{tail}' + self._run_check(s, [("starttag", "title", []), + ("data", "Egg & Spam" + ('' if end else tail))], + collector=EventCollectorNoNormalize(convert_charrefs=True)) + self._run_check(s, [("starttag", "title", []), + ('data', 'Egg '), + ('entityref', 'amp'), + ('data', ' Spam' + ('' if end else tail))], + collector=EventCollectorNoNormalize(convert_charrefs=False)) def test_comments(self): html = ("<!-- I'm a valid comment -->" '<!--me too!-->' '<!------>' + '<!----->' '<!---->' + # abrupt-closing-of-empty-comment + '<!--->' + '<!-->' '<!----I have many hyphens---->' '<!-- I have a > in the middle -->' - '<!-- and I have -- in the middle! -->') + '<!-- and I have -- in the middle! -->' + '<!--incorrectly-closed-comment--!>' + '<!----!>' + '<!----!-->' + '<!---- >-->' + '<!---!>-->' + '<!--!>-->' + # nested-comment + '<!-- <!-- nested --> -->' + '<!--<!-->' + '<!--<!--!>' + # abruptly closed empty comment must not swallow later text + '<!-->x-->' + '<!--->y-->' + ) expected = [('comment', " I'm a valid comment "), ('comment', 'me too!'), ('comment', '--'), + ('comment', '-'), + ('comment', ''), + ('comment', ''), ('comment', ''), ('comment', '--I have many hyphens--'), ('comment', ' I have a > in the middle '), - ('comment', ' and I have -- in the middle! ')] + ('comment', ' and I have -- in the middle! '), + ('comment', 'incorrectly-closed-comment'), + ('comment', ''), + ('comment', '--!'), + ('comment', '-- >'), + ('comment', '-!>'), + ('comment', '!>'), + ('comment', ' <!-- nested '), ('data', ' -->'), + ('comment', '<!'), + ('comment', '<!'), + ('comment', ''), ('data', 'x-->'), + ('comment', ''), ('data', 'y-->'), + ] self._run_check(html, expected) def test_condcoms(self): @@ -348,18 +640,16 @@ def test_convert_charrefs(self): collector = lambda: EventCollectorCharrefs() self.assertTrue(collector().convert_charrefs) charrefs = ['&quot;', '&#34;', '&#x22;', '&quot', '&#34', '&#x22'] - # check charrefs in the middle of the text/attributes - expected = [('starttag', 'a', [('href', 'foo"zar')]), - ('data', 'a"z'), ('endtag', 'a')] + # check charrefs in the middle of the text + expected = [('starttag', 'a', []), ('data', 'a"z'), ('endtag', 'a')] for charref in charrefs: - self._run_check('<a href="foo{0}zar">a{0}z</a>'.format(charref), + self._run_check('<a>a{0}z</a>'.format(charref), expected, collector=collector()) - # check charrefs at the beginning/end of the text/attributes - expected = [('data', '"'), - ('starttag', 'a', [('x', '"'), ('y', '"X'), ('z', 'X"')]), + # check charrefs at the beginning/end of the text + expected = [('data', '"'), ('starttag', 'a', []), ('data', '"'), ('endtag', 'a'), ('data', '"')] for charref in charrefs: - self._run_check('{0}<a x="{0}" y="{0}X" z="X{0}">' + self._run_check('{0}<a>' '{0}</a>{0}'.format(charref), expected, collector=collector()) # check charrefs in <script>/<style> elements @@ -382,6 +672,35 @@ def test_convert_charrefs(self): self._run_check('no charrefs here', [('data', 'no charrefs here')], collector=collector()) + def test_convert_charrefs_in_attribute_values(self): + # default value for convert_charrefs is now True + collector = lambda: EventCollectorCharrefs() + self.assertTrue(collector().convert_charrefs) + + # always unescape terminated entity refs, numeric and hex char refs: + # - regardless whether they are at start, middle, end of attribute + # - or followed by alphanumeric, non-alphanumeric, or equals char + charrefs = ['&cent;', '&#xa2;', '&#xa2', '&#162;', '&#162'] + expected = [('starttag', 'a', + [('x', '¢'), ('x', 'z¢'), ('x', '¢z'), + ('x', 'z¢z'), ('x', '¢ z'), ('x', '¢=z')]), + ('endtag', 'a')] + for charref in charrefs: + self._run_check('<a x="{0}" x="z{0}" x="{0}z" ' + ' x="z{0}z" x="{0} z" x="{0}=z"></a>' + .format(charref), expected, collector=collector()) + + # only unescape unterminated entity matches if they are not followed by + # an alphanumeric or an equals sign + charref = '&cent' + expected = [('starttag', 'a', + [('x', '¢'), ('x', 'z¢'), ('x', '&centz'), + ('x', 'z&centz'), ('x', '¢ z'), ('x', '&cent=z')]), + ('endtag', 'a')] + self._run_check('<a x="{0}" x="z{0}" x="{0}z" ' + ' x="z{0}z" x="{0} z" x="{0}=z"></a>' + .format(charref), expected, collector=collector()) + # the remaining tests were for the "tolerant" parser (which is now # the default), and check various kind of broken markup def test_tolerant_parsing(self): @@ -393,28 +712,34 @@ def test_tolerant_parsing(self): ('data', '<'), ('starttag', 'bc<', [('a', None)]), ('endtag', 'html'), - ('data', '\n<img src="URL>'), - ('comment', '/img'), - ('endtag', 'html<')]) + ('data', '\n')]) def test_starttag_junk_chars(self): + self._run_check("<", [('data', '<')]) + self._run_check("<>", [('data', '<>')]) + self._run_check("< >", [('data', '< >')]) + self._run_check("< ", [('data', '< ')]) self._run_check("</>", []) + self._run_check("<$>", [('data', '<$>')]) self._run_check("</$>", [('comment', '$')]) self._run_check("</", [('data', '</')]) - self._run_check("</a", [('data', '</a')]) + self._run_check("</a", []) + self._run_check("</ a>", [('comment', ' a')]) + self._run_check("</ a", [('comment', ' a')]) self._run_check("<a<a>", [('starttag', 'a<a', [])]) self._run_check("</a<a>", [('endtag', 'a<a')]) - self._run_check("<!", [('data', '<!')]) - self._run_check("<a", [('data', '<a')]) - self._run_check("<a foo='bar'", [('data', "<a foo='bar'")]) - self._run_check("<a foo='bar", [('data', "<a foo='bar")]) - self._run_check("<a foo='>'", [('data', "<a foo='>'")]) - self._run_check("<a foo='>", [('data', "<a foo='>")]) + self._run_check("<!", [('comment', '')]) + self._run_check("<a", []) + self._run_check("<a foo='bar'", []) + self._run_check("<a foo='bar", []) + self._run_check("<a foo='>'", []) + self._run_check("<a foo='>", []) self._run_check("<a$>", [('starttag', 'a$', [])]) self._run_check("<a$b>", [('starttag', 'a$b', [])]) self._run_check("<a$b/>", [('startendtag', 'a$b', [])]) self._run_check("<a$b >", [('starttag', 'a$b', [])]) self._run_check("<a$b />", [('startendtag', 'a$b', [])]) + self._run_check("</a$b>", [('endtag', 'a$b')]) def test_slashes_in_starttag(self): self._run_check('<a foo="var"/>', [('startendtag', 'a', [('foo', 'var')])]) @@ -447,6 +772,10 @@ def test_slashes_in_starttag(self): ] self._run_check(html, expected) + def test_slashes_in_endtag(self): + self._run_check('</a/>', [('endtag', 'a')]) + self._run_check('</a foo="var"/>', [('endtag', 'a')]) + def test_declaration_junk_chars(self): self._run_check("<!DOCTYPE foo $ >", [('decl', 'DOCTYPE foo $ ')]) @@ -481,15 +810,11 @@ def test_invalid_end_tags(self): self._run_check(html, expected) def test_broken_invalid_end_tag(self): - # This is technically wrong (the "> shouldn't be included in the 'data') - # but is probably not worth fixing it (in addition to all the cases of - # the previous test, it would require a full attribute parsing). - # see #13993 html = '<b>This</b attr=">"> confuses the parser' expected = [('starttag', 'b', []), ('data', 'This'), ('endtag', 'b'), - ('data', '"> confuses the parser')] + ('data', ' confuses the parser')] self._run_check(html, expected) def test_correct_detection_of_start_tags(self): @@ -525,69 +850,165 @@ def test_correct_detection_of_start_tags(self): ] self._run_check(html, expected) - def test_EOF_in_charref(self): - # see #17802 - # This test checks that the UnboundLocalError reported in the issue - # is not raised, however I'm not sure the returned values are correct. - # Maybe HTMLParser should use self.unescape for these + def test_eof_in_comments(self): + data = [ + ('<!--', [('comment', '')]), + ('<!---', [('comment', '')]), + ('<!----', [('comment', '')]), + ('<!-----', [('comment', '-')]), + ('<!------', [('comment', '--')]), + ('<!----!', [('comment', '')]), + ('<!---!', [('comment', '-!')]), + ('<!---!>', [('comment', '-!>')]), + ('<!--foo', [('comment', 'foo')]), + ('<!--foo-', [('comment', 'foo')]), + ('<!--foo--', [('comment', 'foo')]), + ('<!--foo--!', [('comment', 'foo')]), + ('<!--<!--', [('comment', '<!')]), + ('<!--<!--!', [('comment', '<!')]), + ] + for html, expected in data: + self._run_check(html, expected) + + def test_eof_in_declarations(self): data = [ - ('a&', [('data', 'a&')]), - ('a&b', [('data', 'ab')]), - ('a&b ', [('data', 'a'), ('entityref', 'b'), ('data', ' ')]), - ('a&b;', [('data', 'a'), ('entityref', 'b')]), + ('<!', [('comment', '')]), + ('<!-', [('comment', '-')]), + ('<![', [('comment', '[')]), + ('<!DOCTYPE', [('decl', 'DOCTYPE')]), + ('<!DOCTYPE ', [('decl', 'DOCTYPE ')]), + ('<!DOCTYPE html', [('decl', 'DOCTYPE html')]), + ('<!DOCTYPE html ', [('decl', 'DOCTYPE html ')]), + ('<!DOCTYPE html PUBLIC', [('decl', 'DOCTYPE html PUBLIC')]), + ('<!DOCTYPE html PUBLIC "foo', [('decl', 'DOCTYPE html PUBLIC "foo')]), + ('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "foo', + [('decl', 'DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "foo')]), ] for html, expected in data: self._run_check(html, expected) - def test_broken_comments(self): - html = ('<! not really a comment >' + @support.subTests('content', ['', 'x', 'x]', 'x]]']) + def test_eof_in_cdata(self, content): + self._run_check('<![CDATA[' + content, + [('unknown decl', 'CDATA[' + content)]) + self._run_check('<![CDATA[' + content, + [('comment', '[CDATA[' + content)], + collector=EventCollector(autocdata=True)) + self._run_check('<svg><text y="100"><![CDATA[' + content, + [('starttag', 'svg', []), + ('starttag', 'text', [('y', '100')]), + ('unknown decl', 'CDATA[' + content)]) + + def test_bogus_comments(self): + html = ('<!ELEMENT br EMPTY>' + '<! not really a comment >' '<! not a comment either -->' '<! -- close enough -->' '<!><!<-- this was an empty comment>' - '<!!! another bogus comment !!!>') + '<!!! another bogus comment !!!>' + # see #32876 + '<![with square brackets]!>' + '<![\nmultiline\nbogusness\n]!>' + '<![more brackets]-[and a hyphen]!>' + '<![cdata[should be uppercase]]>' + '<![CDATA [whitespaces are not ignored]]>' + '<![CDATA]]>' # required '[' after CDATA + ) expected = [ + ('comment', 'ELEMENT br EMPTY'), ('comment', ' not really a comment '), ('comment', ' not a comment either --'), ('comment', ' -- close enough --'), ('comment', ''), ('comment', '<-- this was an empty comment'), ('comment', '!! another bogus comment !!!'), + ('comment', '[with square brackets]!'), + ('comment', '[\nmultiline\nbogusness\n]!'), + ('comment', '[more brackets]-[and a hyphen]!'), + ('comment', '[cdata[should be uppercase]]'), + ('comment', '[CDATA [whitespaces are not ignored]]'), + ('comment', '[CDATA]]'), ] self._run_check(html, expected) def test_broken_condcoms(self): # these condcoms are missing the '--' after '<!' and before the '>' + # and they are considered bogus comments according to + # "8.2.4.42. Markup declaration open state" html = ('<![if !(IE)]>broken condcom<![endif]>' '<![if ! IE]><link href="favicon.tiff"/><![endif]>' '<![if !IE 6]><img src="firefox.png" /><![endif]>' '<![if !ie 6]><b>foo</b><![endif]>' '<![if (!IE)|(lt IE 9)]><img src="mammoth.bmp" /><![endif]>') - # According to the HTML5 specs sections "8.2.4.44 Bogus comment state" - # and "8.2.4.45 Markup declaration open state", comment tokens should - # be emitted instead of 'unknown decl', but calling unknown_decl - # provides more flexibility. - # See also Lib/_markupbase.py:parse_declaration expected = [ - ('unknown decl', 'if !(IE)'), + ('comment', '[if !(IE)]'), ('data', 'broken condcom'), - ('unknown decl', 'endif'), - ('unknown decl', 'if ! IE'), + ('comment', '[endif]'), + ('comment', '[if ! IE]'), ('startendtag', 'link', [('href', 'favicon.tiff')]), - ('unknown decl', 'endif'), - ('unknown decl', 'if !IE 6'), + ('comment', '[endif]'), + ('comment', '[if !IE 6]'), ('startendtag', 'img', [('src', 'firefox.png')]), - ('unknown decl', 'endif'), - ('unknown decl', 'if !ie 6'), + ('comment', '[endif]'), + ('comment', '[if !ie 6]'), ('starttag', 'b', []), ('data', 'foo'), ('endtag', 'b'), - ('unknown decl', 'endif'), - ('unknown decl', 'if (!IE)|(lt IE 9)'), + ('comment', '[endif]'), + ('comment', '[if (!IE)|(lt IE 9)]'), ('startendtag', 'img', [('src', 'mammoth.bmp')]), - ('unknown decl', 'endif') + ('comment', '[endif]') ] self._run_check(html, expected) + @support.subTests('content', [ + 'just some plain text', + '<!-- not a comment -->', + '&not-an-entity-ref;', + "<not a='start tag'>", + '', + '[[I have many brackets]]', + 'I have a > in the middle', + 'I have a ]] in the middle', + '] ]>', + ']] >', + ('\n' + ' if (a < b && a > b) {\n' + ' printf("[<marquee>How?</marquee>]");\n' + ' }\n'), + ]) + def test_cdata_section_content(self, content): + # See "13.2.5.42 Markup declaration open state", + # "13.2.5.69 CDATA section state", and issue bpo-32876. + html = f'<svg><text y="100"><![CDATA[{content}]]></text></svg>' + expected = [ + ('starttag', 'svg', []), + ('starttag', 'text', [('y', '100')]), + ('unknown decl', 'CDATA[' + content), + ('endtag', 'text'), + ('endtag', 'svg'), + ] + self._run_check(html, expected) + self._run_check(html, expected, collector=EventCollector(autocdata=True)) + + def test_cdata_section(self): + # See "13.2.5.42 Markup declaration open state". + html = ('<![CDATA[foo<br>bar]]>' + '<svg><text y="100"><![CDATA[foo<br>bar]]></text></svg>' + '<![CDATA[foo<br>bar]]>') + expected = [ + ('comment', '[CDATA[foo<br'), + ('data', 'bar]]>'), + ('starttag', 'svg', []), + ('starttag', 'text', [('y', '100')]), + ('unknown decl', 'CDATA[foo<br>bar'), + ('endtag', 'text'), + ('endtag', 'svg'), + ('comment', '[CDATA[foo<br'), + ('data', 'bar]]>'), + ] + self._run_check(html, expected, collector=EventCollector(autocdata=True)) + def test_convert_charrefs_dropped_text(self): # #23144: make sure that all the events are triggered when # convert_charrefs is True, even if we don't call .close() @@ -600,6 +1021,46 @@ def test_convert_charrefs_dropped_text(self): ('endtag', 'a'), ('data', ' bar & baz')] ) + @support.requires_resource('cpu') + def test_eof_no_quadratic_complexity(self): + # Each of these examples used to take about an hour. + # Now they take a fraction of a second. + def check(source): + parser = html.parser.HTMLParser() + parser.feed(source) + parser.close() + n = 120_000 + check("<a " * n) + check("<a a=" * n) + check("</a " * 14 * n) + check("</a a=" * 11 * n) + check("<!--" * 4 * n) + check("<!" * 60 * n) + check("<?" * 19 * n) + check("</$" * 15 * n) + check("<![CDATA[" * 9 * n) + check("<!doctype" * 35 * n) + + @support.requires_resource('cpu') + def test_incremental_no_quadratic_complexity(self): + # An unterminated construct fed in many small chunks used to take + # quadratic time, both to rescan and to concatenate the buffer. + # Now it takes a fraction of a second. + def check(prefix, chunk, suffix): + parser = html.parser.HTMLParser() + parser.feed(prefix) + for _ in range(200_000): + parser.feed(chunk) + parser.feed(suffix) + parser.close() + chunk = "a" * 64 + check("<!--", chunk, "-->") # comment + check("<?", chunk, ">") # processing instruction + check("<!doctype ", chunk, ">") # doctype + check("<![CDATA[", chunk, "]]>") # CDATA section + check("<a href='", chunk, "'>") # start tag + check("<script>", chunk, "</script>") # RAWTEXT element + class AttributesTestCase(TestCaseBase): @@ -608,9 +1069,15 @@ def test_attr_syntax(self): ("starttag", "a", [("b", "v"), ("c", "v"), ("d", "v"), ("e", None)]) ] self._run_check("""<a b='v' c="v" d=v e>""", output) - self._run_check("""<a b = 'v' c = "v" d = v e>""", output) - self._run_check("""<a\nb\n=\n'v'\nc\n=\n"v"\nd\n=\nv\ne>""", output) - self._run_check("""<a\tb\t=\t'v'\tc\t=\t"v"\td\t=\tv\te>""", output) + self._run_check("<a foo==bar>", [('starttag', 'a', [('foo', '=bar')])]) + self._run_check("<a foo =bar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo\t=bar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo\v=bar>", [('starttag', 'a', [('foo\v', 'bar')])]) + self._run_check("<a foo\xa0=bar>", [('starttag', 'a', [('foo\xa0', 'bar')])]) + self._run_check("<a foo= bar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo=\tbar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo=\vbar>", [('starttag', 'a', [('foo', '\vbar')])]) + self._run_check("<a foo=\xa0bar>", [('starttag', 'a', [('foo', '\xa0bar')])]) def test_attr_values(self): self._run_check("""<a b='xxx\n\txxx' c="yyy\t\nyyy" d='\txyz\n'>""", @@ -619,6 +1086,10 @@ def test_attr_values(self): ("d", "\txyz\n")])]) self._run_check("""<a b='' c="">""", [("starttag", "a", [("b", ""), ("c", "")])]) + self._run_check("<a b=\tx c=\ny>", + [('starttag', 'a', [('b', 'x'), ('c', 'y')])]) + self._run_check("<a b=\v c=\xa0>", + [("starttag", "a", [("b", "\v"), ("c", "\xa0")])]) # Regression test for SF patch #669683. self._run_check("<e a=rgb(1,2,3)>", [("starttag", "e", [("a", "rgb(1,2,3)")])]) @@ -685,13 +1156,17 @@ def test_malformed_attributes(self): ) expected = [ ('starttag', 'a', [('href', "test'style='color:red;bad1'")]), - ('data', 'test - bad1'), ('endtag', 'a'), + ('data', 'test - bad1'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'+style='color:red;ba2'")]), - ('data', 'test - bad2'), ('endtag', 'a'), + ('data', 'test - bad2'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'\xa0style='color:red;bad3'")]), - ('data', 'test - bad3'), ('endtag', 'a'), + ('data', 'test - bad3'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'\xa0style='color:red;bad4'")]), - ('data', 'test - bad4'), ('endtag', 'a') + ('data', 'test - bad4'), + ('endtag', 'a'), ] self._run_check(html, expected) diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py index 6bc33b15ec32e9c..04cb440cd4ccf66 100644 --- a/Lib/test/test_http_cookiejar.py +++ b/Lib/test/test_http_cookiejar.py @@ -4,6 +4,7 @@ import stat import sys import re +from test import support from test.support import os_helper from test.support import warnings_helper import time @@ -105,8 +106,7 @@ def test_http2time_formats(self): self.assertEqual(http2time(s.lower()), test_t, s.lower()) self.assertEqual(http2time(s.upper()), test_t, s.upper()) - def test_http2time_garbage(self): - for test in [ + @support.subTests('test', [ '', 'Garbage', 'Mandag 16. September 1996', @@ -121,10 +121,9 @@ def test_http2time_garbage(self): '08-01-3697739', '09 Feb 19942632 22:23:32 GMT', 'Wed, 09 Feb 1994834 22:23:32 GMT', - ]: - self.assertIsNone(http2time(test), - "http2time(%s) is not None\n" - "http2time(test) %s" % (test, http2time(test))) + ]) + def test_http2time_garbage(self, test): + self.assertIsNone(http2time(test)) def test_http2time_redos_regression_actually_completes(self): # LOOSE_HTTP_DATE_RE was vulnerable to malicious input which caused catastrophic backtracking (REDoS). @@ -149,9 +148,7 @@ def parse_date(text): self.assertEqual(parse_date("1994-02-03 19:45:29 +0530"), (1994, 2, 3, 14, 15, 29)) - def test_iso2time_formats(self): - # test iso2time for supported dates. - tests = [ + @support.subTests('s', [ '1994-02-03 00:00:00 -0000', # ISO 8601 format '1994-02-03 00:00:00 +0000', # ISO 8601 format '1994-02-03 00:00:00', # zone is optional @@ -164,16 +161,15 @@ def test_iso2time_formats(self): # A few tests with extra space at various places ' 1994-02-03 ', ' 1994-02-03T00:00:00 ', - ] - + ]) + def test_iso2time_formats(self, s): + # test iso2time for supported dates. test_t = 760233600 # assume broken POSIX counting of seconds - for s in tests: - self.assertEqual(iso2time(s), test_t, s) - self.assertEqual(iso2time(s.lower()), test_t, s.lower()) - self.assertEqual(iso2time(s.upper()), test_t, s.upper()) + self.assertEqual(iso2time(s), test_t, s) + self.assertEqual(iso2time(s.lower()), test_t, s.lower()) + self.assertEqual(iso2time(s.upper()), test_t, s.upper()) - def test_iso2time_garbage(self): - for test in [ + @support.subTests('test', [ '', 'Garbage', 'Thursday, 03-Feb-94 00:00:00 GMT', @@ -186,9 +182,9 @@ def test_iso2time_garbage(self): '01-01-1980 00:00:62', '01-01-1980T00:00:62', '19800101T250000Z', - ]: - self.assertIsNone(iso2time(test), - "iso2time(%r)" % test) + ]) + def test_iso2time_garbage(self, test): + self.assertIsNone(iso2time(test)) def test_iso2time_performance_regression(self): # If ISO_DATE_RE regresses to quadratic complexity, this test will take a very long time to succeed. @@ -199,24 +195,23 @@ def test_iso2time_performance_regression(self): class HeaderTests(unittest.TestCase): - def test_parse_ns_headers(self): - # quotes should be stripped - expected = [[('foo', 'bar'), ('expires', 2209069412), ('version', '0')]] - for hdr in [ + @support.subTests('hdr', [ 'foo=bar; expires=01 Jan 2040 22:23:32 GMT', 'foo=bar; expires="01 Jan 2040 22:23:32 GMT"', - ]: - self.assertEqual(parse_ns_headers([hdr]), expected) - - def test_parse_ns_headers_version(self): - + ]) + def test_parse_ns_headers(self, hdr): # quotes should be stripped - expected = [[('foo', 'bar'), ('version', '1')]] - for hdr in [ + expected = [[('foo', 'bar'), ('expires', 2209069412), ('version', '0')]] + self.assertEqual(parse_ns_headers([hdr]), expected) + + @support.subTests('hdr', [ 'foo=bar; version="1"', 'foo=bar; Version="1"', - ]: - self.assertEqual(parse_ns_headers([hdr]), expected) + ]) + def test_parse_ns_headers_version(self, hdr): + # quotes should be stripped + expected = [[('foo', 'bar'), ('version', '1')]] + self.assertEqual(parse_ns_headers([hdr]), expected) def test_parse_ns_headers_special_names(self): # names such as 'expires' are not special in first name=value pair @@ -226,8 +221,7 @@ def test_parse_ns_headers_special_names(self): expected = [[("expires", "01 Jan 2040 22:23:32 GMT"), ("version", "0")]] self.assertEqual(parse_ns_headers([hdr]), expected) - def test_join_header_words(self): - for src, expected in [ + @support.subTests('src,expected', [ ([[("foo", None), ("bar", "baz")]], "foo; bar=baz"), (([]), ""), (([[]]), ""), @@ -237,12 +231,11 @@ def test_join_header_words(self): 'n; foo="foo;_", bar=foo_bar'), ([[("n", "m"), ("foo", None)], [("bar", "foo_bar")]], 'n=m; foo, bar=foo_bar'), - ]: - with self.subTest(src=src): - self.assertEqual(join_header_words(src), expected) + ]) + def test_join_header_words(self, src, expected): + self.assertEqual(join_header_words(src), expected) - def test_split_header_words(self): - tests = [ + @support.subTests('arg,expect', [ ("foo", [[("foo", None)]]), ("foo=bar", [[("foo", "bar")]]), (" foo ", [[("foo", None)]]), @@ -259,24 +252,22 @@ def test_split_header_words(self): (r'foo; bar=baz, spam=, foo="\,\;\"", bar= ', [[("foo", None), ("bar", "baz")], [("spam", "")], [("foo", ',;"')], [("bar", "")]]), - ] - - for arg, expect in tests: - try: - result = split_header_words([arg]) - except: - import traceback, io - f = io.StringIO() - traceback.print_exc(None, f) - result = "(error -- traceback follows)\n\n%s" % f.getvalue() - self.assertEqual(result, expect, """ + ]) + def test_split_header_words(self, arg, expect): + try: + result = split_header_words([arg]) + except: + import traceback, io + f = io.StringIO() + traceback.print_exc(None, f) + result = "(error -- traceback follows)\n\n%s" % f.getvalue() + self.assertEqual(result, expect, """ When parsing: '%s' Expected: '%s' Got: '%s' """ % (arg, expect, result)) - def test_roundtrip(self): - tests = [ + @support.subTests('arg,expect', [ ("foo", "foo"), ("foo=bar", "foo=bar"), (" foo ", "foo"), @@ -309,12 +300,11 @@ def test_roundtrip(self): ('n; foo="foo;_", bar="foo,_"', 'n; foo="foo;_", bar="foo,_"'), - ] - - for arg, expect in tests: - input = split_header_words([arg]) - res = join_header_words(input) - self.assertEqual(res, expect, """ + ]) + def test_roundtrip(self, arg, expect): + input = split_header_words([arg]) + res = join_header_words(input) + self.assertEqual(res, expect, """ When parsing: '%s' Expected: '%s' Got: '%s' @@ -516,14 +506,7 @@ class CookieTests(unittest.TestCase): ## just the 7 special TLD's listed in their spec. And folks rely on ## that... - def test_domain_return_ok(self): - # test optimization: .domain_return_ok() should filter out most - # domains in the CookieJar before we try to access them (because that - # may require disk access -- in particular, with MSIECookieJar) - # This is only a rough check for performance reasons, so it's not too - # critical as long as it's sufficiently liberal. - pol = DefaultCookiePolicy() - for url, domain, ok in [ + @support.subTests('url,domain,ok', [ ("http://foo.bar.com/", "blah.com", False), ("http://foo.bar.com/", "rhubarb.blah.com", False), ("http://foo.bar.com/", "rhubarb.foo.bar.com", False), @@ -543,11 +526,18 @@ def test_domain_return_ok(self): ("http://foo/", ".local", True), ("http://barfoo.com", ".foo.com", False), ("http://barfoo.com", "foo.com", False), - ]: - request = urllib.request.Request(url) - r = pol.domain_return_ok(domain, request) - if ok: self.assertTrue(r) - else: self.assertFalse(r) + ]) + def test_domain_return_ok(self, url, domain, ok): + # test optimization: .domain_return_ok() should filter out most + # domains in the CookieJar before we try to access them (because that + # may require disk access -- in particular, with MSIECookieJar) + # This is only a rough check for performance reasons, so it's not too + # critical as long as it's sufficiently liberal. + pol = DefaultCookiePolicy() + request = urllib.request.Request(url) + r = pol.domain_return_ok(domain, request) + if ok: self.assertTrue(r) + else: self.assertFalse(r) def test_missing_value(self): # missing = sign in Cookie: header is regarded by Mozilla as a missing @@ -581,10 +571,7 @@ def test_missing_value(self): self.assertEqual(interact_netscape(c, "http://www.acme.com/foo/"), '"spam"; eggs') - def test_rfc2109_handling(self): - # RFC 2109 cookies are handled as RFC 2965 or Netscape cookies, - # dependent on policy settings - for rfc2109_as_netscape, rfc2965, version in [ + @support.subTests('rfc2109_as_netscape,rfc2965,version', [ # default according to rfc2965 if not explicitly specified (None, False, 0), (None, True, 1), @@ -593,24 +580,27 @@ def test_rfc2109_handling(self): (False, True, 1), (True, False, 0), (True, True, 0), - ]: - policy = DefaultCookiePolicy( - rfc2109_as_netscape=rfc2109_as_netscape, - rfc2965=rfc2965) - c = CookieJar(policy) - interact_netscape(c, "http://www.example.com/", "ni=ni; Version=1") - try: - cookie = c._cookies["www.example.com"]["/"]["ni"] - except KeyError: - self.assertIsNone(version) # didn't expect a stored cookie - else: - self.assertEqual(cookie.version, version) - # 2965 cookies are unaffected - interact_2965(c, "http://www.example.com/", - "foo=bar; Version=1") - if rfc2965: - cookie2965 = c._cookies["www.example.com"]["/"]["foo"] - self.assertEqual(cookie2965.version, 1) + ]) + def test_rfc2109_handling(self, rfc2109_as_netscape, rfc2965, version): + # RFC 2109 cookies are handled as RFC 2965 or Netscape cookies, + # dependent on policy settings + policy = DefaultCookiePolicy( + rfc2109_as_netscape=rfc2109_as_netscape, + rfc2965=rfc2965) + c = CookieJar(policy) + interact_netscape(c, "http://www.example.com/", "ni=ni; Version=1") + try: + cookie = c._cookies["www.example.com"]["/"]["ni"] + except KeyError: + self.assertIsNone(version) # didn't expect a stored cookie + else: + self.assertEqual(cookie.version, version) + # 2965 cookies are unaffected + interact_2965(c, "http://www.example.com/", + "foo=bar; Version=1") + if rfc2965: + cookie2965 = c._cookies["www.example.com"]["/"]["foo"] + self.assertEqual(cookie2965.version, 1) def test_ns_parser(self): c = CookieJar() @@ -778,8 +768,7 @@ def test_default_path_with_query(self): # Cookie is sent back to the same URI. self.assertEqual(interact_netscape(cj, uri), value) - def test_escape_path(self): - cases = [ + @support.subTests('arg,result', [ # quoted safe ("/foo%2f/bar", "/foo%2F/bar"), ("/foo%2F/bar", "/foo%2F/bar"), @@ -799,9 +788,9 @@ def test_escape_path(self): ("/foo/bar\u00fc", "/foo/bar%C3%BC"), # UTF-8 encoded # unicode ("/foo/bar\uabcd", "/foo/bar%EA%AF%8D"), # UTF-8 encoded - ] - for arg, result in cases: - self.assertEqual(escape_path(arg), result) + ]) + def test_escape_path(self, arg, result): + self.assertEqual(escape_path(arg), result) def test_request_path(self): # with parameters diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index 2fbc142de2fd34b..3ace949afd403eb 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -1,11 +1,11 @@ # Simple test suite for http/cookies.py - import copy import unittest import doctest from http import cookies import pickle from test import support +import urllib.parse class CookieTests(unittest.TestCase): @@ -17,10 +17,10 @@ def test_basic(self): 'repr': "<SimpleCookie: chips='ahoy' vienna='finger'>", 'output': 'Set-Cookie: chips=ahoy\nSet-Cookie: vienna=finger'}, - {'data': 'keebler="E=mc2; L=\\"Loves\\"; fudge=\\012;"', - 'dict': {'keebler' : 'E=mc2; L="Loves"; fudge=\012;'}, - 'repr': '''<SimpleCookie: keebler='E=mc2; L="Loves"; fudge=\\n;'>''', - 'output': 'Set-Cookie: keebler="E=mc2; L=\\"Loves\\"; fudge=\\012;"'}, + {'data': 'keebler="E=mc2; L=\\"Loves\\"; fudge=;"', + 'dict': {'keebler' : 'E=mc2; L="Loves"; fudge=;'}, + 'repr': '''<SimpleCookie: keebler='E=mc2; L="Loves"; fudge=;'>''', + 'output': 'Set-Cookie: keebler="E=mc2; L=\\"Loves\\"; fudge=;"'}, # Check illegal cookies that have an '=' char in an unquoted value {'data': 'keebler=E=mc2', @@ -152,17 +152,19 @@ def test_load(self): self.assertEqual(C.output(['path']), 'Set-Cookie: Customer="WILE_E_COYOTE"; Path=/acme') - self.assertEqual(C.js_output(), r""" + cookie_encoded = urllib.parse.quote('Customer="WILE_E_COYOTE"; Path=/acme; Version=1', safe='', encoding='utf-8') + self.assertEqual(C.js_output(), fr""" <script type="text/javascript"> <!-- begin hiding - document.cookie = "Customer=\"WILE_E_COYOTE\"; Path=/acme; Version=1"; + document.cookie = decodeURIComponent("{cookie_encoded}"); // end hiding --> </script> """) - self.assertEqual(C.js_output(['path']), r""" + cookie_encoded = urllib.parse.quote('Customer="WILE_E_COYOTE"; Path=/acme', safe='', encoding='utf-8') + self.assertEqual(C.js_output(['path']), fr""" <script type="text/javascript"> <!-- begin hiding - document.cookie = "Customer=\"WILE_E_COYOTE\"; Path=/acme"; + document.cookie = decodeURIComponent("{cookie_encoded}"); // end hiding --> </script> """) @@ -267,17 +269,19 @@ def test_quoted_meta(self): self.assertEqual(C.output(['path']), 'Set-Cookie: Customer="WILE_E_COYOTE"; Path=/acme') - self.assertEqual(C.js_output(), r""" + expected_encoded_cookie = urllib.parse.quote('Customer=\"WILE_E_COYOTE\"; Path=/acme; Version=1', safe='', encoding='utf-8') + self.assertEqual(C.js_output(), fr""" <script type="text/javascript"> <!-- begin hiding - document.cookie = "Customer=\"WILE_E_COYOTE\"; Path=/acme; Version=1"; + document.cookie = decodeURIComponent("{expected_encoded_cookie}"); // end hiding --> </script> """) - self.assertEqual(C.js_output(['path']), r""" + expected_encoded_cookie = urllib.parse.quote('Customer=\"WILE_E_COYOTE\"; Path=/acme', safe='', encoding='utf-8') + self.assertEqual(C.js_output(['path']), fr""" <script type="text/javascript"> <!-- begin hiding - document.cookie = "Customer=\"WILE_E_COYOTE\"; Path=/acme"; + document.cookie = decodeURIComponent("{expected_encoded_cookie}"); // end hiding --> </script> """) @@ -368,13 +372,17 @@ def test_setter(self): self.assertEqual( M.output(), "Set-Cookie: %s=%s; Path=/foo" % (i, "%s_coded_val" % i)) + expected_encoded_cookie = urllib.parse.quote( + "%s=%s; Path=/foo" % (i, "%s_coded_val" % i), + safe='', encoding='utf-8', + ) expected_js_output = """ <script type="text/javascript"> <!-- begin hiding - document.cookie = "%s=%s; Path=/foo"; + document.cookie = decodeURIComponent("%s"); // end hiding --> </script> - """ % (i, "%s_coded_val" % i) + """ % (expected_encoded_cookie,) self.assertEqual(M.js_output(), expected_js_output) for i in ["foo bar", "foo@bar"]: # Try some illegal characters @@ -571,6 +579,88 @@ def test_repr(self): r'Set-Cookie: key=coded_val; ' r'expires=\w+, \d+ \w+ \d+ \d+:\d+:\d+ \w+') + def test_control_characters(self): + for c0 in support.control_characters_c0(): + morsel = cookies.Morsel() + + # .__setitem__() + with self.assertRaises(cookies.CookieError): + morsel[c0] = "val" + with self.assertRaises(cookies.CookieError): + morsel["path"] = c0 + + # .__setstate__() + with self.assertRaises(cookies.CookieError): + morsel.__setstate__({'key': c0, 'value': 'val', 'coded_value': 'coded'}) + with self.assertRaises(cookies.CookieError): + morsel.__setstate__({'key': 'key', 'value': c0, 'coded_value': 'coded'}) + with self.assertRaises(cookies.CookieError): + morsel.__setstate__({'key': 'key', 'value': 'val', 'coded_value': c0}) + + # .setdefault() + with self.assertRaises(cookies.CookieError): + morsel.setdefault("path", c0) + with self.assertRaises(cookies.CookieError): + morsel.setdefault(c0, "val") + + # .set() + with self.assertRaises(cookies.CookieError): + morsel.set(c0, "val", "coded-value") + with self.assertRaises(cookies.CookieError): + morsel.set("path", c0, "coded-value") + with self.assertRaises(cookies.CookieError): + morsel.set("path", "val", c0) + + # .update() + with self.assertRaises(cookies.CookieError): + morsel.update({"path": c0}) + with self.assertRaises(cookies.CookieError): + morsel.update({c0: "val"}) + + # .__ior__() + with self.assertRaises(cookies.CookieError): + morsel |= {"path": c0} + with self.assertRaises(cookies.CookieError): + morsel |= {c0: "val"} + + def test_control_characters_output(self): + # Tests that even if the internals of Morsel are modified + # that a call to .output() has control character safeguards. + for c0 in support.control_characters_c0(): + morsel = cookies.Morsel() + morsel.set("key", "value", "coded-value") + morsel._key = c0 # Override private variable. + cookie = cookies.SimpleCookie() + cookie["cookie"] = morsel + with self.assertRaises(cookies.CookieError): + cookie.output() + + morsel = cookies.Morsel() + morsel.set("key", "value", "coded-value") + morsel._coded_value = c0 # Override private variable. + cookie = cookies.SimpleCookie() + cookie["cookie"] = morsel + with self.assertRaises(cookies.CookieError): + cookie.output() + + # Tests that .js_output() also has control character safeguards. + for c0 in support.control_characters_c0(): + morsel = cookies.Morsel() + morsel.set("key", "value", "coded-value") + morsel._key = c0 # Override private variable. + cookie = cookies.SimpleCookie() + cookie["cookie"] = morsel + with self.assertRaises(cookies.CookieError): + cookie.js_output() + + morsel = cookies.Morsel() + morsel.set("key", "value", "coded-value") + morsel._coded_value = c0 # Override private variable. + cookie = cookies.SimpleCookie() + cookie["cookie"] = morsel + with self.assertRaises(cookies.CookieError): + cookie.js_output() + def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite(cookies)) diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 38429ad480ff1c2..0aaf09b26315ed0 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -369,6 +369,51 @@ def test_invalid_headers(self): with self.assertRaisesRegex(ValueError, 'Invalid header'): conn.putheader(name, value) + def test_invalid_tunnel_headers(self): + cases = ( + ('Invalid\r\nName', 'ValidValue'), + ('Invalid\rName', 'ValidValue'), + ('Invalid\nName', 'ValidValue'), + ('\r\nInvalidName', 'ValidValue'), + ('\rInvalidName', 'ValidValue'), + ('\nInvalidName', 'ValidValue'), + (' InvalidName', 'ValidValue'), + ('\tInvalidName', 'ValidValue'), + ('Invalid:Name', 'ValidValue'), + (':InvalidName', 'ValidValue'), + ('ValidName', 'Invalid\r\nValue'), + ('ValidName', 'Invalid\rValue'), + ('ValidName', 'Invalid\nValue'), + ('ValidName', 'InvalidValue\r\n'), + ('ValidName', 'InvalidValue\r'), + ('ValidName', 'InvalidValue\n'), + ) + for name, value in cases: + with self.subTest((name, value)): + conn = client.HTTPConnection('example.com') + conn.set_tunnel('tunnel', headers={ + name: value + }) + conn.sock = FakeSocket('') + with self.assertRaisesRegex(ValueError, 'Invalid header'): + conn._tunnel() # Called in .connect() + + def test_invalid_tunnel_host(self): + cases = ( + 'invalid\r.host', + '\ninvalid.host', + 'invalid.host\r\n', + 'invalid.host\x00', + 'invalid host', + ) + for tunnel_host in cases: + with self.subTest(tunnel_host): + conn = client.HTTPConnection('example.com') + conn.set_tunnel(tunnel_host) + conn.sock = FakeSocket('') + with self.assertRaisesRegex(ValueError, 'Tunnel host can\'t contain control characters'): + conn._tunnel() # Called in .connect() + def test_headers_debuglevel(self): body = ( b'HTTP/1.1 200 OK\r\n' @@ -1287,6 +1332,35 @@ def test_overflowing_header_limit_after_100(self): self.assertIn('got more than ', str(cm.exception)) self.assertIn('headers', str(cm.exception)) + def test_too_many_interim_responses(self): + # A server streaming "100 Continue" responses forever must not + # hang getresponse(). + body = ( + 'HTTP/1.1 100 Continue\r\n\r\n' + * (client._MAXINTERIMRESPONSES + 1) + ) + resp = client.HTTPResponse(FakeSocket(body)) + with self.assertRaises(client.HTTPException) as cm: + resp.begin() + self.assertIn('got more than ', str(cm.exception)) + self.assertIn('interim responses', str(cm.exception)) + + def test_multiple_interim_responses(self): + # A reasonable number of interim responses before the final + # response is skipped as before. + body = ( + 'HTTP/1.1 100 Continue\r\n\r\n' * 3 + + 'HTTP/1.1 200 OK\r\n' + 'Content-Length: 5\r\n' + '\r\n' + 'hello' + ) + resp = client.HTTPResponse(FakeSocket(body), method="GET") + resp.begin() + self.assertEqual(resp.status, 200) + self.assertEqual(resp.read(), b'hello') + resp.close() + def test_overflowing_chunked_line(self): body = ( 'HTTP/1.1 200 OK\r\n' @@ -1358,6 +1432,35 @@ def test_chunked_trailers(self): self.assertEqual(sock.file.read(), b"") #we read to the end resp.close() + def test_chunked_too_many_trailers(self): + """A response streaming endless trailer lines must raise, not hang""" + too_many_trailers = "".join( + f"X-Trailer{i}: {i}\r\n" for i in range(client._MAXHEADERS + 1) + ) + # An unbounded read() reaches the trailers via the final 0 chunk. + sock = FakeSocket( + chunked_start + last_chunk + too_many_trailers + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + with self.assertRaisesRegex( + client.HTTPException, + f"got more than {client._MAXHEADERS} trailers", + ): + resp.read() + resp.close() + + # A bounded read(amt) larger than the body hits the same limit. + sock = FakeSocket( + chunked_start + last_chunk + too_many_trailers + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + with self.assertRaisesRegex( + client.HTTPException, + f"got more than {client._MAXHEADERS} trailers", + ): + resp.read(len(chunked_expected) + 1) + resp.close() + def test_chunked_sync(self): """Check that we don't read past the end of the chunked-encoding stream""" expected = chunked_expected @@ -1465,6 +1568,72 @@ def run_server(): thread.join() self.assertEqual(result, b"proxied data\n") + def test_large_content_length(self): + serv = socket.create_server((HOST, 0)) + self.addCleanup(serv.close) + + def run_server(): + [conn, address] = serv.accept() + with conn: + while conn.recv(1024): + conn.sendall( + b"HTTP/1.1 200 Ok\r\n" + b"Content-Length: %d\r\n" + b"\r\n" % size) + conn.sendall(b'A' * (size//3)) + conn.sendall(b'B' * (size - size//3)) + + thread = threading.Thread(target=run_server) + thread.start() + self.addCleanup(thread.join, 1.0) + + conn = client.HTTPConnection(*serv.getsockname()) + try: + for w in range(15, 27): + size = 1 << w + conn.request("GET", "/") + with conn.getresponse() as response: + self.assertEqual(len(response.read()), size) + finally: + conn.close() + thread.join(1.0) + + def test_large_content_length_truncated(self): + serv = socket.create_server((HOST, 0)) + self.addCleanup(serv.close) + + def run_server(): + while True: + [conn, address] = serv.accept() + with conn: + conn.recv(1024) + if not size: + break + conn.sendall( + b"HTTP/1.1 200 Ok\r\n" + b"Content-Length: %d\r\n" + b"\r\n" + b"Text" % size) + + thread = threading.Thread(target=run_server) + thread.start() + self.addCleanup(thread.join, 1.0) + + conn = client.HTTPConnection(*serv.getsockname()) + try: + for w in range(18, 65): + size = 1 << w + conn.request("GET", "/") + with conn.getresponse() as response: + self.assertRaises(client.IncompleteRead, response.read) + conn.close() + finally: + conn.close() + size = 0 + conn.request("GET", "/") + conn.close() + thread.join(1.0) + def test_putrequest_override_domain_validation(self): """ It should be possible to override the default validation diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 2cafa4e45a13139..9ba5494a65221bf 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -8,6 +8,7 @@ SimpleHTTPRequestHandler, CGIHTTPRequestHandler from http import server, HTTPStatus +import contextlib import os import socket import sys @@ -359,6 +360,46 @@ def test_head_via_send_error(self): self.assertEqual(b'', data) +class HTTP09ServerTestCase(BaseTestCase): + + class request_handler(NoLogRequestHandler, BaseHTTPRequestHandler): + """Request handler for HTTP/0.9 server.""" + + def do_GET(self): + self.wfile.write(f'OK: here is {self.path}\r\n'.encode()) + + def setUp(self): + super().setUp() + self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.sock = self.enterContext(self.sock) + self.sock.connect((self.HOST, self.PORT)) + + def test_simple_get(self): + self.sock.send(b'GET /index.html\r\n') + res = self.sock.recv(1024) + self.assertEqual(res, b"OK: here is /index.html\r\n") + + def test_invalid_request(self): + self.sock.send(b'POST /index.html\r\n') + res = self.sock.recv(1024) + # The error response is not sent in the bare HTTP/0.9 style. + self.assertStartsWith(res, b'HTTP/1.0 400 ') + self.assertIn(b"Bad HTTP/0.9 request type ('POST')", res) + + def test_single_request(self): + self.sock.send(b'GET /foo.html\r\n') + res = self.sock.recv(1024) + self.assertEqual(res, b"OK: here is /foo.html\r\n") + + # Ignore errors if the connection is already closed, + # as this is the expected behavior of HTTP/0.9. + with contextlib.suppress(OSError): + self.sock.send(b'GET /bar.html\r\n') + res = self.sock.recv(1024) + # The server should not process our request. + self.assertEqual(res, b'') + + def certdata_file(*path): return os.path.join(os.path.dirname(__file__), "certdata", *path) @@ -522,42 +563,120 @@ def close_conn(): reader.close() return body + def check_list_dir_dirname(self, dirname, quotedname=None): + fullpath = os.path.join(self.tempdir, dirname) + try: + os.mkdir(os.path.join(self.tempdir, dirname)) + except (OSError, UnicodeEncodeError): + self.skipTest(f'Can not create directory {dirname!a} ' + f'on current file system') + + if quotedname is None: + quotedname = urllib.parse.quote(dirname, errors='surrogatepass') + response = self.request(self.base_url + '/' + quotedname + '/') + body = self.check_status_and_reason(response, HTTPStatus.OK) + displaypath = html.escape(f'{self.base_url}/{dirname}/', quote=False) + enc = sys.getfilesystemencoding() + prefix = f'listing for {displaypath}</'.encode(enc, 'surrogateescape') + self.assertIn(prefix + b'title>', body) + self.assertIn(prefix + b'h1>', body) + + def check_list_dir_filename(self, filename): + fullpath = os.path.join(self.tempdir, filename) + content = ascii(fullpath).encode() + (os_helper.TESTFN_UNDECODABLE or b'\xff') + try: + with open(fullpath, 'wb') as f: + f.write(content) + except OSError: + self.skipTest(f'Can not create file {filename!a} ' + f'on current file system') + + response = self.request(self.base_url + '/') + body = self.check_status_and_reason(response, HTTPStatus.OK) + quotedname = urllib.parse.quote(filename, errors='surrogatepass') + enc = response.headers.get_content_charset() + self.assertIsNotNone(enc) + self.assertIn((f'href="{quotedname}"').encode('ascii'), body) + displayname = html.escape(filename, quote=False) + self.assertIn(f'>{displayname}<'.encode(enc, 'surrogateescape'), body) + + response = self.request(self.base_url + '/' + quotedname) + self.check_status_and_reason(response, HTTPStatus.OK, data=content) + + @unittest.skipUnless(os_helper.TESTFN_NONASCII, + 'need os_helper.TESTFN_NONASCII') + def test_list_dir_nonascii_dirname(self): + dirname = os_helper.TESTFN_NONASCII + '.dir' + self.check_list_dir_dirname(dirname) + + @unittest.skipUnless(os_helper.TESTFN_NONASCII, + 'need os_helper.TESTFN_NONASCII') + def test_list_dir_nonascii_filename(self): + filename = os_helper.TESTFN_NONASCII + '.txt' + self.check_list_dir_filename(filename) + @unittest.skipIf(is_apple, 'undecodable name cannot always be decoded on Apple platforms') @unittest.skipIf(sys.platform == 'win32', 'undecodable name cannot be decoded on win32') @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, 'need os_helper.TESTFN_UNDECODABLE') - def test_undecodable_filename(self): - enc = sys.getfilesystemencoding() - filename = os.fsdecode(os_helper.TESTFN_UNDECODABLE) + '.txt' - with open(os.path.join(self.tempdir, filename), 'wb') as f: - f.write(os_helper.TESTFN_UNDECODABLE) - response = self.request(self.base_url + '/') - if is_apple: - # On Apple platforms the HFS+ filesystem replaces bytes that - # aren't valid UTF-8 into a percent-encoded value. - for name in os.listdir(self.tempdir): - if name != 'test': # Ignore a filename created in setUp(). - filename = name - break - body = self.check_status_and_reason(response, HTTPStatus.OK) - quotedname = urllib.parse.quote(filename, errors='surrogatepass') - self.assertIn(('href="%s"' % quotedname) - .encode(enc, 'surrogateescape'), body) - self.assertIn(('>%s<' % html.escape(filename, quote=False)) - .encode(enc, 'surrogateescape'), body) - response = self.request(self.base_url + '/' + quotedname) - self.check_status_and_reason(response, HTTPStatus.OK, - data=os_helper.TESTFN_UNDECODABLE) + def test_list_dir_undecodable_dirname(self): + dirname = os.fsdecode(os_helper.TESTFN_UNDECODABLE) + '.dir' + self.check_list_dir_dirname(dirname) - def test_undecodable_parameter(self): - # sanity check using a valid parameter + @unittest.skipIf(is_apple, + 'undecodable name cannot always be decoded on Apple platforms') + @unittest.skipIf(sys.platform == 'win32', + 'undecodable name cannot be decoded on win32') + @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, + 'need os_helper.TESTFN_UNDECODABLE') + def test_list_dir_undecodable_filename(self): + filename = os.fsdecode(os_helper.TESTFN_UNDECODABLE) + '.txt' + self.check_list_dir_filename(filename) + + def test_list_dir_undecodable_dirname2(self): + dirname = '\ufffd.dir' + self.check_list_dir_dirname(dirname, quotedname='%ff.dir') + + @unittest.skipUnless(os_helper.TESTFN_UNENCODABLE, + 'need os_helper.TESTFN_UNENCODABLE') + def test_list_dir_unencodable_dirname(self): + dirname = os_helper.TESTFN_UNENCODABLE + '.dir' + self.check_list_dir_dirname(dirname) + + @unittest.skipUnless(os_helper.TESTFN_UNENCODABLE, + 'need os_helper.TESTFN_UNENCODABLE') + def test_list_dir_unencodable_filename(self): + filename = os_helper.TESTFN_UNENCODABLE + '.txt' + self.check_list_dir_filename(filename) + + def test_list_dir_escape_dirname(self): + # Characters that need special treating in URL or HTML. + for name in ('q?', 'f#', '&amp;', '&amp', '<i>', '"dq"', "'sq'", + '%A4', '%E2%82%AC'): + with self.subTest(name=name): + dirname = name + '.dir' + self.check_list_dir_dirname(dirname, + quotedname=urllib.parse.quote(dirname, safe='&<>\'"')) + + def test_list_dir_escape_filename(self): + # Characters that need special treating in URL or HTML. + for name in ('q?', 'f#', '&amp;', '&amp', '<i>', '"dq"', "'sq'", + '%A4', '%E2%82%AC'): + with self.subTest(name=name): + filename = name + '.txt' + self.check_list_dir_filename(filename) + os_helper.unlink(os.path.join(self.tempdir, filename)) + + def test_list_dir_with_query_and_fragment(self): + prefix = f'listing for {self.base_url}/</'.encode('latin1') + response = self.request(self.base_url + '/#123').read() + self.assertIn(prefix + b'title>', response) + self.assertIn(prefix + b'h1>', response) response = self.request(self.base_url + '/?x=123').read() - self.assertRegex(response, rf'listing for {self.base_url}/\?x=123'.encode('latin1')) - # now the bogus encoding - response = self.request(self.base_url + '/?x=%bb').read() - self.assertRegex(response, rf'listing for {self.base_url}/\?x=\xef\xbf\xbd'.encode('latin1')) + self.assertIn(prefix + b'title>', response) + self.assertIn(prefix + b'h1>', response) def test_get_dir_redirect_location_domain_injection_bug(self): """Ensure //evil.co/..%2f../../X does not put //evil.co/ in Location. @@ -615,10 +734,19 @@ def test_get(self): # check for trailing "/" which should return 404. See Issue17324 response = self.request(self.base_url + '/test/') self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + response = self.request(self.base_url + '/test%2f') + self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) + response = self.request(self.base_url + '/test%2F') + self.check_status_and_reason(response, HTTPStatus.NOT_FOUND) response = self.request(self.base_url + '/') self.check_status_and_reason(response, HTTPStatus.OK) + response = self.request(self.base_url + '%2f') + self.check_status_and_reason(response, HTTPStatus.OK) + response = self.request(self.base_url + '%2F') + self.check_status_and_reason(response, HTTPStatus.OK) response = self.request(self.base_url) self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + self.assertEqual(response.getheader("Location"), self.base_url + "/") self.assertEqual(response.getheader("Content-Length"), "0") response = self.request(self.base_url + '/?hi=2') self.check_status_and_reason(response, HTTPStatus.OK) @@ -724,6 +852,8 @@ def test_path_without_leading_slash(self): self.check_status_and_reason(response, HTTPStatus.OK) response = self.request(self.tempdir_name) self.check_status_and_reason(response, HTTPStatus.MOVED_PERMANENTLY) + self.assertEqual(response.getheader("Location"), + self.tempdir_name + "/") response = self.request(self.tempdir_name + '/?hi=2') self.check_status_and_reason(response, HTTPStatus.OK) response = self.request(self.tempdir_name + '?hi=1') @@ -731,27 +861,6 @@ def test_path_without_leading_slash(self): self.assertEqual(response.getheader("Location"), self.tempdir_name + "/?hi=1") - def test_html_escape_filename(self): - filename = '<test&>.txt' - fullpath = os.path.join(self.tempdir, filename) - - try: - open(fullpath, 'wb').close() - except OSError: - raise unittest.SkipTest('Can not create file %s on current file ' - 'system' % filename) - - try: - response = self.request(self.base_url + '/') - body = self.check_status_and_reason(response, HTTPStatus.OK) - enc = response.headers.get_content_charset() - finally: - os.unlink(fullpath) # avoid affecting test_undecodable_filename - - self.assertIsNotNone(enc) - html_text = '>%s<' % html.escape(filename, quote=False) - self.assertIn(html_text.encode(enc), body) - cgi_file1 = """\ #!%s @@ -806,6 +915,20 @@ def test_html_escape_filename(self): print("</pre>") """ +cgi_file7 = """\ +#!%s +import os +import sys + +print("Content-type: text/plain") +print() + +content_length = int(os.environ["CONTENT_LENGTH"]) +body = sys.stdin.buffer.read(content_length) + +print(f"{content_length} {len(body)}") +""" + @unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, "This test can't be run reliably as root (issue #13308).") @@ -845,6 +968,8 @@ def setUp(self): self.file3_path = None self.file4_path = None self.file5_path = None + self.file6_path = None + self.file7_path = None # The shebang line should be pure ASCII: use symlink if possible. # See issue #7668. @@ -899,6 +1024,11 @@ def setUp(self): file6.write(cgi_file6 % self.pythonexe) os.chmod(self.file6_path, 0o777) + self.file7_path = os.path.join(self.cgi_dir, 'file7.py') + with open(self.file7_path, 'w', encoding='utf-8') as file7: + file7.write(cgi_file7 % self.pythonexe) + os.chmod(self.file7_path, 0o777) + os.chdir(self.parent_dir) def tearDown(self): @@ -921,6 +1051,8 @@ def tearDown(self): os.remove(self.file5_path) if self.file6_path: os.remove(self.file6_path) + if self.file7_path: + os.remove(self.file7_path) os.rmdir(self.cgi_child_dir) os.rmdir(self.cgi_dir) os.rmdir(self.cgi_dir_in_sub_dir) @@ -993,6 +1125,22 @@ def test_post(self): self.assertEqual(res.read(), b'1, python, 123456' + self.linesep) + def test_large_content_length(self): + for w in range(15, 25): + size = 1 << w + body = b'X' * size + headers = {'Content-Length' : str(size)} + res = self.request('/cgi-bin/file7.py', 'POST', body, headers) + self.assertEqual(res.read(), b'%d %d' % (size, size) + self.linesep) + + def test_large_content_length_truncated(self): + with support.swap_attr(self.request_handler, 'timeout', 0.001): + for w in range(18, 65): + size = 1 << w + headers = {'Content-Length' : str(size)} + res = self.request('/cgi-bin/file1.py', 'POST', b'x', headers) + self.assertEqual(res.read(), b'Hello World' + self.linesep) + def test_invaliduri(self): res = self.request('/cgi-bin/invalid') res.read() @@ -1199,6 +1347,19 @@ def test_http_0_9(self): self.assertEqual(result[0], b'<html><body>Data</body></html>\r\n') self.verify_get_called() + @support.subTests('request,code', [ + (b'GET / FUBAR\r\n\r\n', 400), # bad version + (b'GET / HTTP/2.0\r\n\r\n', 505), # unsupported version + (b'GET\r\n', 400), # bad syntax + (b'POST /\r\n', 400), # bad HTTP/0.9 request type + ]) + def test_request_line_error_has_status_line(self, request, code): + self.handler = SocketlessRequestHandler() + result = self.send_typical_request(request) + self.assertStartsWith(result[0], b'HTTP/1.1 %d ' % code) + self.verify_expected_headers(result[1:result.index(b'\r\n')]) + self.assertFalse(self.handler.get_called) + def test_extra_space(self): result = self.send_typical_request( b'GET /spaced out HTTP/1.1\r\n' diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index a13ee58d650e1be..a91de7230b6fab1 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -9,6 +9,7 @@ import calendar import threading import re +import select import socket from test.support import verbose, run_with_tz, run_with_locale, cpython_only @@ -27,6 +28,80 @@ CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir, "certdata", "keycert3.pem") CAFILE = os.path.join(os.path.dirname(__file__) or os.curdir, "certdata", "pycacert.pem") +_quoted_string = re.compile(r'"(?:[^"\\]|\\.)*"') + + +def splitargs(line): + # Split a command line into IMAP arguments: quoted strings, balanced + # (possibly nested) parenthesized lists, and bare atoms. + args = [] + i, n = 0, len(line) + while i < n: + if line[i] == ' ': + i += 1 + continue + start = i + depth = 0 + while i < n and (depth or line[i] != ' '): + c = line[i] + if c == '"': + i = _quoted_string.match(line, i).end() + continue + elif c == '(': + depth += 1 + elif c == ')': + depth -= 1 + i += 1 + args.append(line[start:i]) + return args + + +def parse_seq_number(s, maxmsg): + if s == '*': + return maxmsg + return int(s) + + +def parse_sequence_set(arg, maxmsg): + for s in arg.split(','): + if ':' in s: + lo, hi = s.split(':') + lo = parse_seq_number(lo, maxmsg) + hi = parse_seq_number(hi, maxmsg) + yield from range(min(lo, hi), max(lo, hi) + 1) + else: + yield parse_seq_number(s, maxmsg) + + +def make_simple_handler(command, untagged_response=(), + completed=None): + if completed is None: + completed = f'{command} completed' + def cmd(self, tag, args): + for msg in untagged_response: + self._send_textline(msg) + self._send_tagged(tag, 'OK', completed) + cmd.__name__ = 'cmd_' + command + class Handler(SimpleIMAPHandler): + pass + Handler.__name__ = command.title() + 'Handler' + setattr(Handler, cmd.__name__, cmd) + Handler.__qualname__ = Handler.__name__ + cmd.__qualname__ = Handler.__qualname__ + '.' + cmd.__name__ + return Handler + + +def _read_literal(handler, marker): + # Read one literal, a raw octet sequence, by its count from the marker + # ('{N}', or '(~{N}' in UTF8 mode). + size = int(re.search(r'\{(\d+)\}', marker).group(1)) + # The client must wait for the continuation, so nothing should be readable. + if select.select([handler.connection], [], [], 0)[0]: + raise AssertionError('client sent the literal before the ' + 'continuation request') + handler._send_textline('+') + return handler.rfile.read(size) + class TestImaplib(unittest.TestCase): @@ -68,6 +143,15 @@ def test_Time2Internaldate(self): internal = imaplib.Time2Internaldate(t) self.assertEqual(internal, expected) + @run_with_tz('STD-1DST,M3.2.0,M11.1.0') + def test_Time2Internaldate_datetime_timetuple(self): + date_time = datetime.fromtimestamp(2000000000).timetuple() + self.assertIsNone(date_time.tm_gmtoff) + self.assertEqual( + imaplib.Time2Internaldate(date_time), + '"18-May-2033 05:33:20 +0200"', + ) + def test_that_Time2Internaldate_returns_a_result(self): # Without tzset, we can check only that it successfully # produces a result, not the correctness of the result itself, @@ -92,6 +176,65 @@ def test_imap4_host_default_value(self): imaplib.IMAP4() self.assertIn(cm.exception.errno, expected_errnos) + def test_astring(self): + m = imaplib.IMAP4.__new__(imaplib.IMAP4) + m._encoding = 'ascii' + # Plain atoms are left unquoted. + self.assertEqual(m._astring('INBOX'), b'INBOX') + self.assertEqual(m._astring(b'INBOX'), b'INBOX') + # Names with protocol-sensitive characters are quoted. + self.assertEqual(m._astring('New folder'), b'"New folder"') + self.assertEqual(m._astring('a"b'), b'"a\\"b"') + self.assertEqual(m._astring('a\\b'), b'"a\\\\b"') + self.assertEqual(m._astring(''), b'""') + self.assertEqual(m._astring('*'), b'"*"') + # A well-formed quoted string is passed through unchanged. + self.assertEqual(m._astring('"New folder"'), b'"New folder"') + self.assertEqual(m._astring('""'), b'""') + # Including a lenient (non-RFC) backslash escape, which the server + # may accept. + self.assertEqual(m._astring('"a\\b"'), b'"a\\b"') + # A string that only looks quoted but is not a single token is + # quoted as data, closing the argument injection vector. + self.assertEqual(m._astring('"a" SELECT evil "'), + b'"\\"a\\" SELECT evil \\""') + self.assertEqual(m._astring('"'), b'"\\""') + # Non-ASCII names are only allowed in a quoted string or a + # literal, never in an atom (RFC 6855). + m._encoding = 'utf-8' + self.assertEqual(m._astring('Entwürfe'), '"Entwürfe"'.encode()) + self.assertEqual(m._astring(b'Entw\xc3\xbcrfe'), b'"Entw\xc3\xbcrfe"') + + def test_astring_idempotent(self): + # Quoting an already quoted argument should not change it, so that + # quoting twice gives the same result as quoting once. + m = imaplib.IMAP4.__new__(imaplib.IMAP4) + m._encoding = 'ascii' + for arg in ['INBOX', 'New folder', 'a"b', 'a\\b', '', '*', '%', + '"New folder"', '""', '"a\\b"', '"a" SELECT evil "', + '"', 'a\tb', 'a\rb', '\x7f', '(a)', b'Entw\xc3\xbcrfe']: + with self.subTest(arg=arg): + once = m._astring(arg) + self.assertEqual(m._astring(once), once) + twice = m._list_mailbox(arg) + self.assertEqual(m._list_mailbox(twice), twice) + + def test_list_mailbox(self): + m = imaplib.IMAP4.__new__(imaplib.IMAP4) + m._encoding = 'ascii' + # Wildcards are not quoted in a list pattern. + self.assertEqual(m._list_mailbox('*'), b'*') + self.assertEqual(m._list_mailbox('%'), b'%') + self.assertEqual(m._list_mailbox('foo/%'), b'foo/%') + # But spaces still require quoting. + self.assertEqual(m._list_mailbox('New folder'), b'"New folder"') + self.assertEqual(m._list_mailbox('"New folder"'), b'"New folder"') + # As do non-ASCII names; wildcards keep their meaning inside a + # quoted string. + m._encoding = 'utf-8' + self.assertEqual(m._list_mailbox('Entwürfe/%'), + '"Entwürfe/%"'.encode()) + if ssl: class SecureTCPServer(socketserver.TCPServer): @@ -120,7 +263,7 @@ class SimpleIMAPHandler(socketserver.StreamRequestHandler): def setup(self): super().setup() - self.server.is_selected = False + self.server.is_selected = None self.server.logged = None def _send(self, message): @@ -137,9 +280,11 @@ def _send_textline(self, message): def _send_tagged(self, tag, code, message): self._send_textline(' '.join((tag, code, message))) + welcome = '* OK IMAP4rev1' + def handle(self): # Send a welcome message. - self._send_textline('* OK IMAP4rev1') + self._send_textline(self.welcome) while 1: # Gather up input until we receive a line terminator or we timeout. # Accumulate read(1) because it's simpler to handle the differences @@ -166,10 +311,11 @@ def handle(self): except StopIteration: self.continuation = None continue - splitline = line.decode('ASCII').split() + splitline = splitargs(line.decode().removesuffix('\r\n')) tag = splitline[0] cmd = splitline[1] args = splitline[2:] + self.server.args = args if hasattr(self, 'cmd_' + cmd): continuation = getattr(self, 'cmd_' + cmd)(tag, args) @@ -192,17 +338,22 @@ def cmd_LOGOUT(self, tag, args): self._send_tagged(tag, 'OK', 'LOGOUT completed') def cmd_LOGIN(self, tag, args): - self.server.logged = args[0] + self.server.logged = args self._send_tagged(tag, 'OK', 'LOGIN completed') def cmd_SELECT(self, tag, args): - self.server.is_selected = True + self.server.is_selected = args self._send_line(b'* 2 EXISTS') self._send_tagged(tag, 'OK', '[READ-WRITE] SELECT completed.') + def cmd_EXAMINE(self, tag, args): + self.server.is_selected = args + self._send_line(b'* 2 EXISTS') + self._send_tagged(tag, 'OK', '[READ-ONLY] EXAMINE completed.') + def cmd_UNSELECT(self, tag, args): - if self.server.is_selected: - self.server.is_selected = False + if self.server.is_selected is not None: + self.server.is_selected = None self._send_tagged(tag, 'OK', 'Returned to authenticated state. (Success)') else: self._send_tagged(tag, 'BAD', 'No mailbox selected') @@ -256,7 +407,20 @@ def cmd_IDLE(self, tag, args): self._send_tagged(tag, 'BAD', 'Expected DONE') -class NewIMAPTestsMixin(): +class AuthHandler_CRAM_MD5(SimpleIMAPHandler): + capabilities = 'LOGINDISABLED AUTH=CRAM-MD5' + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+ PDE4OTYuNjk3MTcwOTUyQHBvc3RvZmZpY2Uucm' + 'VzdG9uLm1jaS5uZXQ=') + r = yield + if (r == b'dGltIGYxY2E2YmU0NjRiOWVmYT' + b'FjY2E2ZmZkNmNmMmQ5ZjMy\r\n'): + self._send_tagged(tag, 'OK', 'CRAM-MD5 successful') + else: + self._send_tagged(tag, 'NO', 'No access') + + +class NewIMAPTestsMixin: client = None def _setup(self, imap_handler, connect=True): @@ -317,6 +481,39 @@ def handle(self): self.assertRaises(imaplib.IMAP4.abort, self.imap_class, *server.server_address) + def test_invalid_greeting(self): + # An invalid greeting, e.g. from a POP3 server on the IMAP port, + # must not fail with "error: None" but report the server's line + # (gh-108280). + class Pop3Handler(socketserver.StreamRequestHandler): + def handle(self): + self.wfile.write(b'+OK POP3 server ready\r\n') + _, server = self._setup(Pop3Handler, connect=False) + with self.assertRaisesRegex(imaplib.IMAP4.error, + r'invalid greeting: \+OK POP3 server ready'): + self.imap_class(*server.server_address) + + def test_invalid_greeting_untagged(self): + # An untagged greeting that is neither OK nor PREAUTH (e.g. BYE) + # is reported as is (gh-108280). + class ByeHandler(socketserver.StreamRequestHandler): + def handle(self): + self.wfile.write(b'* BYE Server unavailable\r\n') + _, server = self._setup(ByeHandler, connect=False) + with self.assertRaisesRegex(imaplib.IMAP4.error, + r'invalid greeting: \* BYE Server unavailable'): + self.imap_class(*server.server_address) + + def test_invalid_greeting_bare_continuation(self): + # A bare continuation greeting is still reported (gh-108280). + class BareHandler(socketserver.StreamRequestHandler): + def handle(self): + self.wfile.write(b'+\r\n') + _, server = self._setup(BareHandler, connect=False) + with self.assertRaisesRegex(imaplib.IMAP4.error, + r'invalid greeting: \+'): + self.imap_class(*server.server_address) + def test_line_termination(self): class BadNewlineHandler(SimpleIMAPHandler): def cmd_CAPABILITY(self, tag, args): @@ -359,8 +556,10 @@ def cmd_AUTHENTICATE(self, tag, args): self.server.response = yield self._send_tagged(tag, 'OK', 'FAKEAUTH successful') def cmd_APPEND(self, tag, args): - self._send_textline('+') - self.server.response = yield + self.server.response = args + self.server.response.append(_read_literal(self, args[-1])) + literal = yield + self.server.response.append(literal) self._send_tagged(tag, 'OK', 'okay') client, server = self._setup(UTF8AppendServer) self.assertEqual(client._encoding, 'ascii') @@ -370,11 +569,15 @@ def cmd_APPEND(self, tag, args): code, _ = client.enable('UTF8=ACCEPT') self.assertEqual(code, 'OK') self.assertEqual(client._encoding, 'utf-8') + self.assertEqual(server.args, ['UTF8=ACCEPT']) msg_string = 'Subject: üñí©öðé' - typ, data = client.append(None, None, None, msg_string.encode('utf-8')) + typ, data = client.append( + None, None, None, (msg_string + '\n').encode('utf-8')) self.assertEqual(typ, 'OK') self.assertEqual(server.response, - ('UTF8 (%s)\r\n' % msg_string).encode('utf-8')) + ['INBOX', 'UTF8', + '(~{25}', ('%s\r\n' % msg_string).encode('utf-8'), + b')\r\n' ]) def test_search_disallows_charset_in_utf8_mode(self): class UTF8Server(SimpleIMAPHandler): @@ -394,6 +597,26 @@ def cmd_AUTHENTICATE(self, tag, args): with self.assertRaisesRegex(imaplib.IMAP4.error, 'charset.*UTF8'): client.search('foo', 'bar') + def test_utf8_mailbox_name(self): + class UTF8Server(SimpleIMAPHandler): + capabilities = 'AUTH ENABLE UTF8=ACCEPT' + def cmd_ENABLE(self, tag, args): + self._send_tagged(tag, 'OK', 'ENABLE successful') + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.server.response = yield + self._send_tagged(tag, 'OK', 'FAKEAUTH successful') + client, server = self._setup(UTF8Server) + typ, _ = client.authenticate('MYAUTH', lambda x: b'fake') + self.assertEqual(typ, 'OK') + typ, _ = client.enable('UTF8=ACCEPT') + self.assertEqual(typ, 'OK') + # A non-ASCII mailbox name is only allowed in a quoted string + # or a literal, never in an atom (RFC 6855). + typ, _ = client.select('Entwürfe') + self.assertEqual(typ, 'OK') + self.assertEqual(server.is_selected, ['"Entwürfe"']) + def test_bad_auth_name(self): class MyServer(SimpleIMAPHandler): def cmd_AUTHENTICATE(self, tag, args): @@ -415,6 +638,16 @@ def cmd_AUTHENTICATE(self, tag, args): r'\[AUTHENTICATIONFAILED\] invalid'): client.authenticate('MYAUTH', lambda x: b'fake') + def test_invalid_login(self): + class MyServer(SimpleIMAPHandler): + def cmd_LOGIN(self, tag, args): + self.server.logged = args[0] + self._send_tagged(tag, 'NO', '[LOGIN] failed') + client, _ = self._setup(MyServer) + with self.assertRaisesRegex(imaplib.IMAP4.error, + r'\[LOGIN\] failed'): + client.login('user', 'wrongpass') + def test_valid_authentication_bytes(self): class MyServer(SimpleIMAPHandler): def cmd_AUTHENTICATE(self, tag, args): @@ -439,40 +672,31 @@ def cmd_AUTHENTICATE(self, tag, args): @hashlib_helper.requires_hashdigest('md5', openssl=True) def test_login_cram_md5_bytes(self): - class AuthHandler(SimpleIMAPHandler): - capabilities = 'LOGINDISABLED AUTH=CRAM-MD5' - def cmd_AUTHENTICATE(self, tag, args): - self._send_textline('+ PDE4OTYuNjk3MTcwOTUyQHBvc3RvZmZpY2Uucm' - 'VzdG9uLm1jaS5uZXQ=') - r = yield - if (r == b'dGltIGYxY2E2YmU0NjRiOWVmYT' - b'FjY2E2ZmZkNmNmMmQ5ZjMy\r\n'): - self._send_tagged(tag, 'OK', 'CRAM-MD5 successful') - else: - self._send_tagged(tag, 'NO', 'No access') - client, _ = self._setup(AuthHandler) - self.assertTrue('AUTH=CRAM-MD5' in client.capabilities) + client, _ = self._setup(AuthHandler_CRAM_MD5) + self.assertIn('AUTH=CRAM-MD5', client.capabilities) ret, _ = client.login_cram_md5("tim", b"tanstaaftanstaaf") self.assertEqual(ret, "OK") @hashlib_helper.requires_hashdigest('md5', openssl=True) def test_login_cram_md5_plain_text(self): - class AuthHandler(SimpleIMAPHandler): - capabilities = 'LOGINDISABLED AUTH=CRAM-MD5' - def cmd_AUTHENTICATE(self, tag, args): - self._send_textline('+ PDE4OTYuNjk3MTcwOTUyQHBvc3RvZmZpY2Uucm' - 'VzdG9uLm1jaS5uZXQ=') - r = yield - if (r == b'dGltIGYxY2E2YmU0NjRiOWVmYT' - b'FjY2E2ZmZkNmNmMmQ5ZjMy\r\n'): - self._send_tagged(tag, 'OK', 'CRAM-MD5 successful') - else: - self._send_tagged(tag, 'NO', 'No access') - client, _ = self._setup(AuthHandler) - self.assertTrue('AUTH=CRAM-MD5' in client.capabilities) + client, _ = self._setup(AuthHandler_CRAM_MD5) + self.assertIn('AUTH=CRAM-MD5', client.capabilities) ret, _ = client.login_cram_md5("tim", "tanstaaftanstaaf") self.assertEqual(ret, "OK") + def test_login_cram_md5_blocked(self): + def side_effect(*a, **kw): + raise ValueError + + client, _ = self._setup(AuthHandler_CRAM_MD5) + self.assertIn('AUTH=CRAM-MD5', client.capabilities) + msg = re.escape("CRAM-MD5 authentication is not supported") + with ( + mock.patch("hmac.HMAC", side_effect=side_effect), + self.assertRaisesRegex(imaplib.IMAP4.error, msg) + ): + client.login_cram_md5("tim", b"tanstaaftanstaaf") + def test_aborted_authentication(self): class MyServer(SimpleIMAPHandler): def cmd_AUTHENTICATE(self, tag, args): @@ -530,7 +754,7 @@ def test_with_statement(self): _, server = self._setup(SimpleIMAPHandler, connect=False) with self.imap_class(*server.server_address) as imap: imap.login('user', 'pass') - self.assertEqual(server.logged, 'user') + self.assertEqual(server.logged, ['user', '"pass"']) self.assertIsNone(server.logged) def test_with_statement_logout(self): @@ -538,7 +762,7 @@ def test_with_statement_logout(self): _, server = self._setup(SimpleIMAPHandler, connect=False) with self.imap_class(*server.server_address) as imap: imap.login('user', 'pass') - self.assertEqual(server.logged, 'user') + self.assertEqual(server.logged, ['user', '"pass"']) imap.logout() self.assertIsNone(server.logged) self.assertIsNone(server.logged) @@ -613,11 +837,142 @@ def test_idle_delayed_packet(self): self.fail('multi-packet response was corrupted by idle timeout') def test_login(self): - client, _ = self._setup(SimpleIMAPHandler) + client, server = self._setup(SimpleIMAPHandler) typ, data = client.login('user', 'pass') self.assertEqual(typ, 'OK') self.assertEqual(data[0], b'LOGIN completed') self.assertEqual(client.state, 'AUTH') + # The user name is quoted only when necessary, but the password + # is always quoted. + self.assertEqual(server.logged, ['user', '"pass"']) + self.assertRaises(imaplib.IMAP4.error, client.login, 'user', 'pass') + + def test_login_quoted(self): + client, server = self._setup(SimpleIMAPHandler) + typ, data = client.login('us*r', 'p%ss') + self.assertEqual(typ, 'OK') + self.assertEqual(data[0], b'LOGIN completed') + self.assertEqual(client.state, 'AUTH') + self.assertEqual(server.logged, ['"us*r"', '"p%ss"']) + + def test_login_quoted2(self): + # An already quoted user name is passed through unchanged, rather + # than being quoted a second time; the password is always quoted. + client, server = self._setup(SimpleIMAPHandler) + typ, data = client.login('"user"', '"pass"') + self.assertEqual(typ, 'OK') + self.assertEqual(data[0], b'LOGIN completed') + self.assertEqual(client.state, 'AUTH') + self.assertEqual(server.logged, ['"user"', r'"\"pass\""']) + + def test_append_line_endings(self): + # append() normalizes bare CR and LF in the message to CRLF. + class AppendHandler(SimpleIMAPHandler): + def cmd_APPEND(self, tag, args): + self.server.response = _read_literal(self, args[-1]) + yield # read the trailer line + self._send_tagged(tag, 'OK', 'APPEND completed') + client, server = self._setup(AppendHandler) + client.login('user', 'pass') + message = b'a\rb\nc\r\nd' + client.append('INBOX', None, None, message) + self.assertEqual(server.response, b'a\r\nb\r\nc\r\nd') + + # The mailbox is quoted and the flags are wrapped in parentheses + # when necessary. + client.append('New folder', r'\Seen', None, b'data') + self.assertEqual(server.args, ['"New folder"', r'(\Seen)', '{4}']) + + def test_login_capabilities(self): + # A server may advertise new capabilities after login (as an + # untagged CAPABILITY response); imaplib must refresh its cached + # capability list (gh-63121, gh-103451). + class CapabilityLoginHandler(SimpleIMAPHandler): + def cmd_LOGIN(self, tag, args): + self.server.logged = args[0] + self._send_textline('* CAPABILITY IMAP4rev1 ENABLE UTF8=ACCEPT') + self._send_tagged(tag, 'OK', 'LOGIN completed') + def cmd_ENABLE(self, tag, args): + self._send_tagged(tag, 'OK', 'ENABLE completed') + + client, _ = self._setup(CapabilityLoginHandler) + self.assertNotIn('ENABLE', client.capabilities) + client.login('user', 'pass') + self.assertIn('ENABLE', client.capabilities) + self.assertIn('UTF8=ACCEPT', client.capabilities) + typ, _ = client.enable('UTF8=ACCEPT') + self.assertEqual(typ, 'OK') + + def test_authenticate_capabilities(self): + # Capabilities are also refreshed after AUTHENTICATE, here from a + # CAPABILITY response code in the tagged OK response. + class CapabilityAuthHandler(SimpleIMAPHandler): + def cmd_AUTHENTICATE(self, tag, args): + self._send_textline('+') + self.server.response = yield + self._send_tagged( + tag, 'OK', + '[CAPABILITY IMAP4rev1 ENABLE] AUTHENTICATE completed') + + client, _ = self._setup(CapabilityAuthHandler) + self.assertNotIn('ENABLE', client.capabilities) + client.authenticate('MYAUTH', lambda x: b'fake') + self.assertIn('ENABLE', client.capabilities) + + def test_greeting_capabilities(self): + # Capabilities advertised in the greeting are used directly, + # without sending a separate CAPABILITY command. + class GreetingHandler(SimpleIMAPHandler): + welcome = '* OK [CAPABILITY IMAP4rev1 ENABLE] Server ready' + def cmd_CAPABILITY(self, tag, args): + self.server.capability_queried = True + super().cmd_CAPABILITY(tag, args) + + client, server = self._setup(GreetingHandler) + self.assertEqual(client.capabilities, ('IMAP4REV1', 'ENABLE')) + self.assertFalse(getattr(server, 'capability_queried', False)) + + def test_login_requery_capabilities(self): + # If the server does not advertise capabilities after login, + # imaplib re-queries them (as it does after STARTTLS), so a + # capability that becomes available only after authentication is + # still recognized (gh-63121). + class RequeryHandler(SimpleIMAPHandler): + def cmd_CAPABILITY(self, tag, args): + caps = 'IMAP4rev1 ENABLE' if self.server.logged else 'IMAP4rev1' + self._send_textline('* CAPABILITY ' + caps) + self._send_tagged(tag, 'OK', 'CAPABILITY completed') + + client, _ = self._setup(RequeryHandler) + self.assertNotIn('ENABLE', client.capabilities) + client.login('user', 'pass') + self.assertIn('ENABLE', client.capabilities) + + def test_readonly_error_reports_mailbox(self): + # The read-only error reports the mailbox via repr(), which also + # avoids BytesWarning for a bytes mailbox under -bb. + class ReadOnlyHandler(SimpleIMAPHandler): + def cmd_SELECT(self, tag, args): + self._send_line(b'* 2 EXISTS') + self._send_tagged(tag, 'OK', '[READ-ONLY] SELECT completed.') + client, _ = self._setup(ReadOnlyHandler) + client.login('user', 'pass') + for mailbox, expected in [('INBOX', "'INBOX'"), (b'INBOX', r"b'INBOX'")]: + with self.subTest(mailbox=mailbox): + with self.assertRaisesRegex(imaplib.IMAP4.readonly, + r"%s is not writable" % expected): + client.select(mailbox) + + def test_uid_unknown_command_reports_command(self): + # The unknown-UID-command error reports the command via repr(), which + # also avoids BytesWarning for a bytes command under -bb. + client, _ = self._setup(SimpleIMAPHandler) + for command, expected in [('BOGUS', "'BOGUS'"), (b'BOGUS', r"b'BOGUS'")]: + with self.subTest(command=command): + with self.assertRaisesRegex( + imaplib.IMAP4.error, + r"Unknown IMAP4 UID command: %s" % expected): + client.uid(command, '1') def test_logout(self): client, _ = self._setup(SimpleIMAPHandler) @@ -630,18 +985,88 @@ def test_logout(self): self.assertEqual(client.state, 'LOGOUT') def test_lsub(self): - class LsubCmd(SimpleIMAPHandler): - def cmd_LSUB(self, tag, args): - self._send_textline('* LSUB () "." directoryA') - return self._send_tagged(tag, 'OK', 'LSUB completed') - client, _ = self._setup(LsubCmd) + client, server = self._setup(make_simple_handler('LSUB', + ['* LSUB () "." directoryA', '* LSUB () "." directoryB'])) client.login('user', 'pass') typ, data = client.lsub() self.assertEqual(typ, 'OK') - self.assertEqual(data[0], b'() "." directoryA') + self.assertEqual(data, [b'() "." directoryA', b'() "." directoryB']) + self.assertEqual(server.args, ['""', '*']) + + typ, data = client.lsub('~/Mail/', '%') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['~/Mail/', '%']) + + # The directory is quoted when necessary; wildcards in the pattern + # are preserved. + typ, data = client.lsub('New folder', '%') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['"New folder"', '%']) + + def test_extra_blank_line_after_literal(self): + # Some buggy servers send an extra blank line after the counted + # literal data. imaplib should skip it instead of failing. + class BlankLineHandler(SimpleIMAPHandler): + def cmd_FETCH(self, tag, args): + self._send(b'* 1 FETCH (BODY[HEADER] {13}\r\n') + self._send(b'Subject: test') # 13-byte literal + self._send(b'\r\n)\r\n') # stray blank line, then ')' + self._send_tagged(tag, 'OK', 'FETCH completed') + client, _ = self._setup(BlankLineHandler) + client.login('user', 'pass') + client.select() + typ, data = client.fetch('1', '(BODY[HEADER])') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [(b'1 (BODY[HEADER] {13}', b'Subject: test'), + b')']) + + def test_literal_terminating_response(self): + # A literal ending a response (a LIST mailbox name sent as a literal) + # has an empty trailer that must not be swallowed. Conforming case: + # no spurious blank lines. + names = [b'My (box)"', b'Another', b'Third'] + class Handler(SimpleIMAPHandler): + def cmd_LIST(self, tag, args): + for name in names: + self._send(b'* LIST (\\HasNoChildren) "/" {%d}\r\n' + % len(name)) + self._send(name) + self._send(b'\r\n') # ends the response, no blank + self._send_tagged(tag, 'OK', 'LIST completed') + client, _ = self._setup(Handler) + client.login('user', 'pass') + typ, data = client.list() + self.assertEqual(typ, 'OK') + self.assertEqual(data, [ + (b'(\\HasNoChildren) "/" {9}', b'My (box)"'), b'', + (b'(\\HasNoChildren) "/" {7}', b'Another'), b'', + (b'(\\HasNoChildren) "/" {5}', b'Third'), b'', + ]) + + def test_spurious_blank_lines_between_responses(self): + # A spurious blank line after each terminating literal falls between the + # untagged responses and must be skipped, even several in a row. + names = [b'My (box)"', b'Another', b'Third'] + class Handler(SimpleIMAPHandler): + def cmd_LIST(self, tag, args): + for name in names: + self._send(b'* LIST (\\HasNoChildren) "/" {%d}\r\n' + % len(name)) + self._send(name) + self._send(b'\r\n\r\n') # ends the response, then a blank + self._send_tagged(tag, 'OK', 'LIST completed') + client, _ = self._setup(Handler) + client.login('user', 'pass') + typ, data = client.list() + self.assertEqual(typ, 'OK') + self.assertEqual(data, [ + (b'(\\HasNoChildren) "/" {9}', b'My (box)"'), b'', + (b'(\\HasNoChildren) "/" {7}', b'Another'), b'', + (b'(\\HasNoChildren) "/" {5}', b'Third'), b'', + ]) def test_unselect(self): - client, _ = self._setup(SimpleIMAPHandler) + client, server = self._setup(SimpleIMAPHandler) client.login('user', 'pass') typ, data = client.select() self.assertEqual(typ, 'OK') @@ -651,6 +1076,757 @@ def test_unselect(self): self.assertEqual(typ, 'OK') self.assertEqual(data[0], b'Returned to authenticated state. (Success)') self.assertEqual(client.state, 'AUTH') + self.assertIsNone(server.is_selected) + + def test_enable(self): + class EnableHandler(SimpleIMAPHandler): + capabilities = 'IMAP4rev1 ID LITERAL+ ENABLE X-GOOD-IDEA' + def cmd_ENABLE(self, tag, args): + capabilities = self.capabilities.split() + for arg in args: + if arg in capabilities: + self._send_textline('* ENABLED ' + arg) + self._send_tagged(tag, 'OK', 'foo') + + client, server = self._setup(EnableHandler) + client.login('user', 'pass') + code, data = client.enable('CONDSTORE X-GOOD-IDEA') + self.assertEqual(code, 'OK') + self.assertEqual(data, [b'foo']) + self.assertEqual(server.args, ['CONDSTORE', 'X-GOOD-IDEA']) + + def test_select(self): + client, server = self._setup(SimpleIMAPHandler) + client.login('user', 'pass') + typ, data = client.select() + self.assertEqual(typ, 'OK') + self.assertEqual(data[0], b'2') + self.assertEqual(server.is_selected, ['INBOX']) + + typ, data = client.select('Archive') + self.assertEqual(typ, 'OK') + self.assertEqual(server.is_selected, ['Archive']) + + # readonly=True issues EXAMINE instead of SELECT + # (there is no separate examine() method). + typ, data = client.select(readonly=True) + self.assertEqual(typ, 'OK') + self.assertEqual(data[0], b'2') + self.assertEqual(server.is_selected, ['INBOX']) + self.assertTrue(client.is_readonly) + + typ, data = client.select('New folder') + self.assertEqual(typ, 'OK') + self.assertEqual(server.is_selected, ['"New folder"']) + + def test_expunge(self): + client, server = self._setup(make_simple_handler('EXPUNGE', + ['* 3 EXPUNGE', '* 3 EXPUNGE', '* 5 EXPUNGE', '* 8 EXPUNGE'])) + client.login('user', 'pass') + client.select() + typ, data = client.expunge() + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'3', b'3', b'5', b'8']) + + def test_close(self): + client, server = self._setup(make_simple_handler('CLOSE')) + client.login('user', 'pass') + client.select() + typ, data = client.close() + self.assertEqual(typ, 'OK') + self.assertEqual(client.state, 'AUTH') + + def test_check(self): + client, server = self._setup(make_simple_handler('CHECK')) + client.login('user', 'pass') + client.select() + typ, data = client.check() + self.assertEqual(typ, 'OK') + + def test_noop(self): + client, server = self._setup(make_simple_handler('NOOP', + ['* 4 EXISTS'])) + client.login('user', 'pass') + typ, data = client.noop() + self.assertEqual(typ, 'OK') + # NOOP is used to pick up server-pushed untagged responses. + self.assertEqual(client.untagged_responses['EXISTS'], [b'4']) + + def test_namespace(self): + client, server = self._setup(make_simple_handler('NAMESPACE', + ['* NAMESPACE (("" "/")) NIL NIL'])) + client.login('user', 'pass') + typ, data = client.namespace() + self.assertEqual(typ, 'OK') + self.assertEqual(data[0], b'(("" "/")) NIL NIL') + + def test_capability(self): + client, server = self._setup(make_simple_handler('CAPABILITY', + ['* CAPABILITY IMAP4rev1 IDLE NAMESPACE'])) + client.login('user', 'pass') + typ, data = client.capability() + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'IMAP4rev1 IDLE NAMESPACE']) + + def test_recent(self): + # recent() prods the server with NOOP if no RECENT was seen yet. + client, server = self._setup(make_simple_handler('NOOP', + ['* 5 RECENT'])) + client.login('user', 'pass') + typ, data = client.recent() + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'5']) + + def test_response(self): + client, server = self._setup(SimpleIMAPHandler) + client.login('user', 'pass') + client.select() # the handler answers with '* 2 EXISTS' + typ, data = client.response('EXISTS') + self.assertEqual(typ, 'EXISTS') + self.assertEqual(data, [b'2']) + # The value is cleared once read. + typ, data = client.response('EXISTS') + self.assertEqual(data, [None]) + + def test_create(self): + client, server = self._setup(make_simple_handler('CREATE')) + client.login('user', 'pass') + typ, data = client.create('owatagusiam/') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'CREATE completed']) + self.assertEqual(server.args, ['owatagusiam/']) + + typ, data = client.create('owatagusiam/blurdybloop') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'CREATE completed']) + self.assertEqual(server.args, ['owatagusiam/blurdybloop']) + + typ, data = client.create('New folder') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'CREATE completed']) + self.assertEqual(server.args, ['"New folder"']) + + def test_copy(self): + client, server = self._setup(make_simple_handler('COPY')) + client.login('user', 'pass') + client.select() + typ, data = client.copy('2:4', 'MEETING') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'COPY completed']) + self.assertEqual(server.args, ['2:4', 'MEETING']) + + typ, data = client.copy('2:4', 'New folder') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'COPY completed']) + self.assertEqual(server.args, ['2:4', '"New folder"']) + + def test_uid_copy(self): + client, server = self._setup(make_simple_handler('UID', + completed='UID COPY completed')) + client.login('user', 'pass') + client.select() + typ, data = client.uid('copy', '4827313:4828442', 'MEETING') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [None]) + self.assertEqual(server.args, ['COPY', '4827313:4828442', 'MEETING']) + + typ, data = client.uid('copy', '4827313:4828442', 'New folder') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [None]) + self.assertEqual(server.args, ['COPY', '4827313:4828442', '"New folder"']) + + def test_store(self): + client, server = self._setup(make_simple_handler('STORE', [ + r'* 2 FETCH (FLAGS (\Deleted \Seen))', + r'* 3 FETCH (FLAGS (\Deleted))', + r'* 4 FETCH (FLAGS (\Deleted \Flagged \Seen))', + ])) + client.login('user', 'pass') + client.select() + typ, data = client.store('2:4', '+FLAGS', r'(\Deleted)') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [ + br'2 (FLAGS (\Deleted \Seen))', + br'3 (FLAGS (\Deleted))', + br'4 (FLAGS (\Deleted \Flagged \Seen))', + ]) + self.assertEqual(server.args, ['2:4', '+FLAGS', r'(\Deleted)']) + + typ, data = client.store('2:4', '+FLAGS', r'\Deleted') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['2:4', '+FLAGS', r'(\Deleted)']) + + def test_uid_store(self): + client, server = self._setup(make_simple_handler('UID', [ + r'* 23 FETCH (FLAGS (\Deleted \Seen) UID 4827313)', + r'* 24 FETCH (FLAGS (\Deleted) UID 4827943)', + r'* 25 FETCH (FLAGS (\Deleted \Flagged \Seen) UID 4828442)', + ], 'UID STORE completed')) + client.login('user', 'pass') + client.select() + typ, data = client.uid('store', '4827313:4828442', '+FLAGS', r'(\Deleted)') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [ + br'23 (FLAGS (\Deleted \Seen) UID 4827313)', + br'24 (FLAGS (\Deleted) UID 4827943)', + br'25 (FLAGS (\Deleted \Flagged \Seen) UID 4828442)', + ]) + self.assertEqual(server.args, ['STORE', '4827313:4828442', '+FLAGS', r'(\Deleted)']) + + typ, data = client.uid('store', '4827313:4828442', '+FLAGS', r'\Deleted') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['STORE', '4827313:4828442', '+FLAGS', r'(\Deleted)']) + + def test_fetch(self): + # The handler expands the requested sequence set and answers for + # exactly those messages, so the test exercises the round trip of + # the message set, not just a canned reply. + class FetchHandler(SimpleIMAPHandler): + messages = 4 + def cmd_SELECT(self, tag, args): + self.server.is_selected = args + self._send_line(b'* %d EXISTS' % self.messages) + self._send_tagged(tag, 'OK', '[READ-WRITE] SELECT completed.') + def cmd_FETCH(self, tag, args): + for n in parse_sequence_set(args[0], self.messages): + self._send_textline(r'* %d FETCH (FLAGS (\Seen))' % n) + self._send_tagged(tag, 'OK', 'FETCH completed') + + client, server = self._setup(FetchHandler) + client.login('user', 'pass') + client.select() + typ, data = client.fetch('2:4', '(FLAGS)') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [ + br'2 (FLAGS (\Seen))', + br'3 (FLAGS (\Seen))', + br'4 (FLAGS (\Seen))', + ]) + self.assertEqual(server.args, ['2:4', '(FLAGS)']) + + # A comma-separated set with an open range up to '*'. + typ, data = client.fetch('1,3:*', '(FLAGS)') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [ + br'1 (FLAGS (\Seen))', + br'3 (FLAGS (\Seen))', + br'4 (FLAGS (\Seen))', + ]) + self.assertEqual(server.args, ['1,3:*', '(FLAGS)']) + + # An item with nested parentheses is sent (and parsed) as a + # single argument. + typ, data = client.fetch('1', '(BODY[HEADER.FIELDS (DATE FROM)])') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [br'1 (FLAGS (\Seen))']) + self.assertEqual(server.args, ['1', '(BODY[HEADER.FIELDS (DATE FROM)])']) + + # message_parts is wrapped in parentheses if it is not already. + typ, data = client.fetch('2:4', 'FLAGS') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['2:4', '(FLAGS)']) + + # But the macros are not, as they are not data item names. + typ, data = client.fetch('2:4', 'ALL') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['2:4', 'ALL']) + typ, data = client.fetch('2:4', 'fast') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['2:4', 'fast']) + + def test_uid_fetch(self): + client, server = self._setup(make_simple_handler('UID', [ + r'* 23 FETCH (FLAGS (\Seen) UID 4827313)', + r'* 24 FETCH (FLAGS (\Seen) UID 4827943)', + r'* 25 FETCH (FLAGS (\Seen) UID 4828442)', + ], 'UID FETCH completed')) + client.login('user', 'pass') + client.select() + typ, data = client.uid('fetch', '4827313:4828442', '(FLAGS)') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [ + br'23 (FLAGS (\Seen) UID 4827313)', + br'24 (FLAGS (\Seen) UID 4827943)', + br'25 (FLAGS (\Seen) UID 4828442)', + ]) + self.assertEqual(server.args, ['FETCH', '4827313:4828442', '(FLAGS)']) + + typ, data = client.uid('fetch', '4827313:4828442', 'FLAGS') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['FETCH', '4827313:4828442', '(FLAGS)']) + + typ, data = client.uid('fetch', '4827313:4828442', 'ALL') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['FETCH', '4827313:4828442', 'ALL']) + + def test_partial(self): + client, server = self._setup(make_simple_handler('PARTIAL', + ['* 1 FETCH (RFC822.TEXT<0.10> "0123456789")'])) + client.login('user', 'pass') + client.select() + typ, data = client.partial('1', 'RFC822.TEXT', '0', '10') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'1 (RFC822.TEXT<0.10> "0123456789")']) + self.assertEqual(server.args, ['1', 'RFC822.TEXT', '0', '10']) + + def test_search(self): + response = [] + client, server = self._setup(make_simple_handler('SEARCH', response)) + client.login('user', 'pass') + client.select() + response[:] = ['* SEARCH 2 84 882'] + typ, data = client.search(None, 'FLAGGED', 'SINCE', '1-Feb-1994', 'NOT', 'FROM', '"Smith"') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'2 84 882']) + self.assertEqual(server.args, ['FLAGGED', 'SINCE', '1-Feb-1994', 'NOT', 'FROM', '"Smith"']) + + response[:] = ['* SEARCH'] + typ, data = client.search(None, 'TEXT', '"string not in mailbox"') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'']) + self.assertEqual(server.args, ['TEXT', '"string not in mailbox"']) + + response[:] = ['* SEARCH 43'] + typ, data = client.search('UTF-8', 'TEXT', 'XXXXXX') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'43']) + self.assertEqual(server.args, ['CHARSET', 'UTF-8', 'TEXT', 'XXXXXX']) + + typ, data = client.search('NF_Z_62-010_(1973)', 'TEXT', 'XXXXXX') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['CHARSET', '"NF_Z_62-010_(1973)"', 'TEXT', 'XXXXXX']) + + def test_uid_search(self): + response = [] + client, server = self._setup(make_simple_handler('UID', response, + 'UID SEARCH completed')) + client.login('user', 'pass') + client.select() + response[:] = ['* SEARCH 2 84 882'] + typ, data = client.uid('SEARCH', 'FLAGGED', 'SINCE', '1-Feb-1994', 'NOT', 'FROM', '"Smith"') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'2 84 882']) + self.assertEqual(server.args, ['SEARCH', 'FLAGGED', 'SINCE', '1-Feb-1994', 'NOT', 'FROM', '"Smith"']) + + response[:] = ['* SEARCH'] + typ, data = client.uid('SEARCH', 'TEXT', '"string not in mailbox"') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'']) + self.assertEqual(server.args, ['SEARCH', 'TEXT', '"string not in mailbox"']) + + response[:] = ['* SEARCH 43'] + typ, data = client.uid('SEARCH', 'CHARSET', 'UTF-8', 'TEXT', 'XXXXXX') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'43']) + self.assertEqual(server.args, ['SEARCH', 'CHARSET', 'UTF-8', 'TEXT', 'XXXXXX']) + + typ, data = client.uid('SEARCH', 'CHARSET', '"NF_Z_62-010_(1973)"', 'TEXT', 'XXXXXX') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['SEARCH', 'CHARSET', '"NF_Z_62-010_(1973)"', 'TEXT', 'XXXXXX']) + + def test_sort(self): + response = [] + client, server = self._setup(make_simple_handler('SORT', response)) + client.login('user', 'pass') + client.select() + response[:] = ['* SORT 2 84 882'] + typ, data = client.sort('(SUBJECT)', 'UTF-8', 'SINCE', '1-Feb-1994') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [br'2 84 882']) + self.assertEqual(server.args, ['(SUBJECT)', 'UTF-8', 'SINCE', '1-Feb-1994']) + + response[:] = ['* SORT 5 3 4 1 2'] + typ, data = client.sort('(SUBJECT REVERSE DATE)', 'UTF-8', 'ALL') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [br'5 3 4 1 2']) + self.assertEqual(server.args, ['(SUBJECT REVERSE DATE)', 'UTF-8', 'ALL']) + + response[:] = ['* SORT'] + typ, data = client.sort('(SUBJECT)', 'US-ASCII', 'TEXT', '"not in mailbox"') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [br'']) + self.assertEqual(server.args, ['(SUBJECT)', 'US-ASCII', 'TEXT', '"not in mailbox"']) + + typ, data = client.sort('SUBJECT', 'NF_Z_62-010_(1973)', 'TEXT', '"not in mailbox"') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['(SUBJECT)', '"NF_Z_62-010_(1973)"', 'TEXT', '"not in mailbox"']) + + def test_uid_sort(self): + response = [] + client, server = self._setup(make_simple_handler('UID', response, + 'UID SORT completed')) + client.login('user', 'pass') + client.select() + response[:] = ['* SORT 2 84 882'] + typ, data = client.uid('sort', '(SUBJECT)', 'UTF-8', 'SINCE', '1-Feb-1994') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [br'2 84 882']) + self.assertEqual(server.args, ['SORT', '(SUBJECT)', 'UTF-8', 'SINCE', '1-Feb-1994']) + + response[:] = ['* SORT 5 3 4 1 2'] + typ, data = client.uid('sort', '(SUBJECT REVERSE DATE)', 'UTF-8', 'ALL') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [br'5 3 4 1 2']) + self.assertEqual(server.args, ['SORT', '(SUBJECT REVERSE DATE)', 'UTF-8', 'ALL']) + + response[:] = ['* SORT'] + typ, data = client.uid('sort', '(SUBJECT)', 'US-ASCII', 'TEXT', '"not in mailbox"') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [br'']) + self.assertEqual(server.args, ['SORT', '(SUBJECT)', 'US-ASCII', 'TEXT', '"not in mailbox"']) + + typ, data = client.uid('sort', 'SUBJECT', 'NF_Z_62-010_(1973)', 'TEXT', '"not in mailbox"') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['SORT', '(SUBJECT)', '"NF_Z_62-010_(1973)"', 'TEXT', '"not in mailbox"']) + + def test_thread(self): + response = [] + client, server = self._setup(make_simple_handler('THREAD', response)) + client.login('user', 'pass') + client.select() + response[:] = [ + '* THREAD (166)(167)(168)(169)(172)(170)(171)' + '(173)(174 (175)(176)(178)(181)(180))(179)(177 ' + '(183)(182)(188)(184)(185)(186)(187)(189))(190)' + '(191)(192)(193)(194 195)(196 (197)(198))(199)' + '(200 202)(201)(203)(204)(205)(206 207)(208)'] + typ, data = client.thread('ORDEREDSUBJECT', 'UTF-8', 'SINCE', '5-MAR-2000') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [ + b'(166)(167)(168)(169)(172)(170)(171)' + b'(173)(174 (175)(176)(178)(181)(180))(179)(177 ' + b'(183)(182)(188)(184)(185)(186)(187)(189))(190)' + b'(191)(192)(193)(194 195)(196 (197)(198))(199)' + b'(200 202)(201)(203)(204)(205)(206 207)(208)']) + self.assertEqual(server.args, ['ORDEREDSUBJECT', 'UTF-8', 'SINCE', '5-MAR-2000']) + + response[:] = [ + '* THREAD (166)(167)(168)(169)(172)((170)(179))' + '(171)(173)((174)(175)(176)(178)(181)(180))' + '((177)(183)(182)(188 (184)(189))(185 186)(187))' + '(190)(191)(192)(193)((194)(195 196))(197 198)' + '(199)(200 202)(201)(203)(204)(205 206 207)(208)'] + typ, data = client.thread('ORDEREDSUBJECT', 'US-ASCII', 'TEXT', '"gewp"') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [ + b'(166)(167)(168)(169)(172)((170)(179))' + b'(171)(173)((174)(175)(176)(178)(181)(180))' + b'((177)(183)(182)(188 (184)(189))(185 186)(187))' + b'(190)(191)(192)(193)((194)(195 196))(197 198)' + b'(199)(200 202)(201)(203)(204)(205 206 207)(208)']) + self.assertEqual(server.args, ['ORDEREDSUBJECT', 'US-ASCII', 'TEXT', '"gewp"']) + + typ, data = client.thread('ORDEREDSUBJECT', 'NF_Z_62-010_(1973)', 'TEXT', '"gewp"') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['ORDEREDSUBJECT', '"NF_Z_62-010_(1973)"', 'TEXT', '"gewp"']) + + def test_uid_thread(self): + response = [] + client, server = self._setup(make_simple_handler('UID', response, + 'UID THREAD completed')) + client.login('user', 'pass') + client.select() + response[:] = [ + '* THREAD (166)(167)(168)(169)(172)(170)(171)' + '(173)(174 (175)(176)(178)(181)(180))(179)(177 ' + '(183)(182)(188)(184)(185)(186)(187)(189))(190)' + '(191)(192)(193)(194 195)(196 (197)(198))(199)' + '(200 202)(201)(203)(204)(205)(206 207)(208)'] + typ, data = client.uid('THREAD', 'ORDEREDSUBJECT', 'UTF-8', 'SINCE', '5-MAR-2000') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [ + b'(166)(167)(168)(169)(172)(170)(171)' + b'(173)(174 (175)(176)(178)(181)(180))(179)(177 ' + b'(183)(182)(188)(184)(185)(186)(187)(189))(190)' + b'(191)(192)(193)(194 195)(196 (197)(198))(199)' + b'(200 202)(201)(203)(204)(205)(206 207)(208)']) + self.assertEqual(server.args, ['THREAD', 'ORDEREDSUBJECT', 'UTF-8', 'SINCE', '5-MAR-2000']) + + response[:] = [ + '* THREAD (166)(167)(168)(169)(172)((170)(179))' + '(171)(173)((174)(175)(176)(178)(181)(180))' + '((177)(183)(182)(188 (184)(189))(185 186)(187))' + '(190)(191)(192)(193)((194)(195 196))(197 198)' + '(199)(200 202)(201)(203)(204)(205 206 207)(208)'] + typ, data = client.uid('THREAD', 'ORDEREDSUBJECT', 'US-ASCII', 'TEXT', '"gewp"') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [ + b'(166)(167)(168)(169)(172)((170)(179))' + b'(171)(173)((174)(175)(176)(178)(181)(180))' + b'((177)(183)(182)(188 (184)(189))(185 186)(187))' + b'(190)(191)(192)(193)((194)(195 196))(197 198)' + b'(199)(200 202)(201)(203)(204)(205 206 207)(208)']) + self.assertEqual(server.args, ['THREAD', 'ORDEREDSUBJECT', 'US-ASCII', 'TEXT', '"gewp"']) + + typ, data = client.uid('THREAD', 'ORDEREDSUBJECT', 'NF_Z_62-010_(1973)', 'TEXT', '"gewp"') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['THREAD', 'ORDEREDSUBJECT', '"NF_Z_62-010_(1973)"', 'TEXT', '"gewp"']) + + def test_delete(self): + client, server = self._setup(make_simple_handler('DELETE')) + client.login('user', 'pass') + typ, data = client.delete('blurdybloop') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'DELETE completed']) + self.assertEqual(server.args, ['blurdybloop']) + + typ, data = client.delete('foo/bar') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['foo/bar']) + + typ, data = client.delete('New folder') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['"New folder"']) + + def test_rename(self): + client, server = self._setup(make_simple_handler('RENAME')) + client.login('user', 'pass') + typ, data = client.rename('blurdybloop', 'sarasoop') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'RENAME completed']) + self.assertEqual(server.args, ['blurdybloop', 'sarasoop']) + + typ, data = client.rename('Old folder', 'New folder') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['"Old folder"', '"New folder"']) + + def test_subscribe(self): + client, server = self._setup(make_simple_handler('SUBSCRIBE')) + client.login('user', 'pass') + typ, data = client.subscribe('#news.comp.mail.mime') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'SUBSCRIBE completed']) + self.assertEqual(server.args, ['#news.comp.mail.mime']) + + typ, data = client.subscribe('New folder') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['"New folder"']) + + def test_unsubscribe(self): + client, server = self._setup(make_simple_handler('UNSUBSCRIBE')) + client.login('user', 'pass') + typ, data = client.unsubscribe('#news.comp.mail.mime') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'UNSUBSCRIBE completed']) + self.assertEqual(server.args, ['#news.comp.mail.mime']) + + typ, data = client.unsubscribe('New folder') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['"New folder"']) + + def test_list(self): + client, server = self._setup(make_simple_handler('LIST', + [r'* LIST (\Noselect) "/" ""', + r'* LIST (\Unmarked) "/" "~/Mail/foo"'])) + client.login('user', 'pass') + typ, data = client.list() + self.assertEqual(typ, 'OK') + self.assertEqual(data, [br'(\Noselect) "/" ""', + br'(\Unmarked) "/" "~/Mail/foo"']) + self.assertEqual(server.args, ['""', '*']) + + typ, data = client.list('~/Mail/', '%') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['~/Mail/', '%']) + + typ, data = client.list('New folder', '*') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['"New folder"', '*']) + + # A pattern without wildcards is quoted when necessary. + typ, data = client.list('~/Mail/', 'My Folder') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['~/Mail/', '"My Folder"']) + + def test_status(self): + client, server = self._setup(make_simple_handler('STATUS', + ['* STATUS blurdybloop (MESSAGES 231 UIDNEXT 44292)'])) + client.login('user', 'pass') + typ, data = client.status('blurdybloop', '(UIDNEXT MESSAGES)') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'blurdybloop (MESSAGES 231 UIDNEXT 44292)']) + self.assertEqual(server.args, ['blurdybloop', '(UIDNEXT MESSAGES)']) + + # The names argument is wrapped in parentheses if it is not already. + typ, data = client.status('New folder', 'UIDNEXT MESSAGES') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['"New folder"', '(UIDNEXT MESSAGES)']) + + def test_getacl(self): + client, server = self._setup(make_simple_handler('GETACL', + ['* ACL INBOX Fred rwipslxetad'])) + client.login('user', 'pass') + typ, data = client.getacl('INBOX') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'INBOX Fred rwipslxetad']) + self.assertEqual(server.args, ['INBOX']) + + typ, data = client.getacl('New folder') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['"New folder"']) + + def test_setacl(self): + client, server = self._setup(make_simple_handler('SETACL')) + client.login('user', 'pass') + typ, data = client.setacl('INBOX', 'Fred', 'rwipslxetad') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'SETACL completed']) + self.assertEqual(server.args, ['INBOX', 'Fred', 'rwipslxetad']) + + typ, data = client.setacl('New folder', 'Fred', '+lr') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['"New folder"', 'Fred', '+lr']) + + # The identifier and the rights are quoted when necessary too. + typ, data = client.setacl('INBOX', 'John Doe', 'a b') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['INBOX', '"John Doe"', '"a b"']) + + def test_deleteacl(self): + client, server = self._setup(make_simple_handler('DELETEACL')) + client.login('user', 'pass') + typ, data = client.deleteacl('INBOX', 'Fred') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'DELETEACL completed']) + self.assertEqual(server.args, ['INBOX', 'Fred']) + + # The identifier is quoted when necessary too. + typ, data = client.deleteacl('New folder', 'John Doe') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['"New folder"', '"John Doe"']) + + def test_myrights(self): + client, server = self._setup(make_simple_handler('MYRIGHTS', + ['* MYRIGHTS INBOX rwiptsldaex'])) + client.login('user', 'pass') + typ, data = client.myrights('INBOX') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'INBOX rwiptsldaex']) + self.assertEqual(server.args, ['INBOX']) + + typ, data = client.myrights('New folder') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['"New folder"']) + + def test_getquota(self): + client, server = self._setup(make_simple_handler('GETQUOTA', + ['* QUOTA "" (STORAGE 10 512)'])) + client.login('user', 'pass') + typ, data = client.getquota('#news') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'"" (STORAGE 10 512)']) + self.assertEqual(server.args, ['#news']) + + typ, data = client.getquota('') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['""']) + + def test_getquotaroot(self): + client, server = self._setup(make_simple_handler('GETQUOTAROOT', + ['* QUOTAROOT INBOX ""', '* QUOTA "" (STORAGE 10 512)'])) + client.login('user', 'pass') + typ, data = client.getquotaroot('INBOX') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [[b'INBOX ""'], [b'"" (STORAGE 10 512)']]) + self.assertEqual(server.args, ['INBOX']) + + typ, data = client.getquotaroot('New folder') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['"New folder"']) + + def test_setquota(self): + client, server = self._setup(make_simple_handler('SETQUOTA', + ['* QUOTA "" (STORAGE 512)'])) + client.login('user', 'pass') + typ, data = client.setquota('#news', '(STORAGE 512)') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'"" (STORAGE 512)']) + self.assertEqual(server.args, ['#news', '(STORAGE 512)']) + + typ, data = client.setquota('', '(STORAGE 512)') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['""', '(STORAGE 512)']) + + # The limits argument is wrapped in parentheses if it is not already. + typ, data = client.setquota('', 'STORAGE 512') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['""', '(STORAGE 512)']) + + def test_getannotation(self): + client, server = self._setup(make_simple_handler('GETANNOTATION', + ['* ANNOTATION INBOX "/comment" ("value.shared" "Hello")'])) + client.login('user', 'pass') + typ, data = client.getannotation('INBOX', '/comment', 'value.shared') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'INBOX "/comment" ("value.shared" "Hello")']) + self.assertEqual(server.args, ['INBOX', '/comment', 'value.shared']) + + typ, data = client.getannotation('New folder', '/comment', 'value.shared') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['"New folder"', '/comment', 'value.shared']) + + def test_setannotation(self): + client, server = self._setup(make_simple_handler('SETANNOTATION')) + client.login('user', 'pass') + typ, data = client.setannotation('INBOX', '/comment', + '("value.shared" "My comment")') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, + ['INBOX', '/comment', '("value.shared" "My comment")']) + + typ, data = client.setannotation('New folder', '/comment', + '("value.shared" "My comment")') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, + ['"New folder"', '/comment', '("value.shared" "My comment")']) + + def test_proxyauth(self): + client, server = self._setup(make_simple_handler('PROXYAUTH')) + client.login('user', 'pass') + typ, data = client.proxyauth('user') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'PROXYAUTH completed']) + self.assertEqual(server.args, ['user']) + + typ, data = client.proxyauth('us er') + self.assertEqual(typ, 'OK') + self.assertEqual(server.args, ['"us er"']) + + def test_xatom(self): + client, server = self._setup(make_simple_handler('MYCOMMAND', + completed='MYCOMMAND completed')) + client.login('user', 'pass') + self.addCleanup(imaplib.Commands.pop, 'MYCOMMAND', None) + typ, data = client.xatom('MYCOMMAND', 'arg1', 'arg2') + self.assertEqual(typ, 'OK') + self.assertEqual(data, [b'MYCOMMAND completed']) + self.assertEqual(server.args, ['arg1', 'arg2']) + + def test_uppercase_command_names(self): + client, server = self._setup(SimpleIMAPHandler) + client.login('user', 'pass') + self.assertEqual(client.CAPABILITY, client.capability) + self.assertEqual(client.SELECT, client.select) + typ, data = client.CAPABILITY() + self.assertEqual(typ, 'OK') + with self.assertRaises(AttributeError): + client.NONEXISTENT + + def test_control_characters(self): + client, server = self._setup(SimpleIMAPHandler) + client.login('user', 'pass') + for c in '\0\r\n': + with self.assertRaises(ValueError): + client.select(f'a{c}b') + # Other control characters are valid in a quoted string and can + # occur in mailbox names returned by the server, so the client + # must be able to send them back. + for c in support.control_characters_c0(): + if c in '\0\r\n': + continue + typ, _ = client.select(f'a{c}b') + self.assertEqual(typ, 'OK') + self.assertEqual(server.is_selected, [f'"a{c}b"']) # property tests @@ -769,8 +1945,8 @@ def test_connect(self): @threading_helper.reap_threads def test_bracket_flags(self): - # This violates RFC 3501, which disallows ']' characters in tag names, - # but imaplib has allowed producing such tags forever, other programs + # This violates RFC 3501, which disallows ']' characters in flags, + # but imaplib has allowed producing such flags forever, other programs # also produce them (eg: OtherInbox's Organizer app as of 20140716), # and Gmail, for example, accepts them and produces them. So we # support them. See issue #21815. @@ -882,8 +2058,10 @@ def test_enable_UTF8_True_append(self): class UTF8AppendServer(self.UTF8Server): def cmd_APPEND(self, tag, args): - self._send_textline('+') - self.server.response = yield + self.server.response = args + self.server.response.append(_read_literal(self, args[-1])) + literal = yield + self.server.response.append(literal) self._send_tagged(tag, 'OK', 'okay') with self.reaped_pair(UTF8AppendServer) as (server, client): @@ -895,14 +2073,15 @@ def cmd_APPEND(self, tag, args): code, _ = client.enable('UTF8=ACCEPT') self.assertEqual(code, 'OK') self.assertEqual(client._encoding, 'utf-8') + self.assertEqual(server.args, ['UTF8=ACCEPT']) msg_string = 'Subject: üñí©öðé' typ, data = client.append( - None, None, None, msg_string.encode('utf-8')) + None, None, None, (msg_string + '\n').encode('utf-8')) self.assertEqual(typ, 'OK') - self.assertEqual( - server.response, - ('UTF8 (%s)\r\n' % msg_string).encode('utf-8') - ) + self.assertEqual(server.response, + ['INBOX', 'UTF8', + '(~{25}', ('%s\r\n' % msg_string).encode('utf-8'), + b')\r\n' ]) # XXX also need a test that makes sure that the Literal and Untagged_status # regexes uses unicode in UTF8 mode instead of the default ASCII. @@ -1050,7 +2229,7 @@ def test_with_statement(self): with self.reaped_server(SimpleIMAPHandler) as server: with self.imap_class(*server.server_address) as imap: imap.login('user', 'pass') - self.assertEqual(server.logged, 'user') + self.assertEqual(server.logged, ['user', '"pass"']) self.assertIsNone(server.logged) @threading_helper.reap_threads @@ -1059,7 +2238,7 @@ def test_with_statement_logout(self): with self.reaped_server(SimpleIMAPHandler) as server: with self.imap_class(*server.server_address) as imap: imap.login('user', 'pass') - self.assertEqual(server.logged, 'user') + self.assertEqual(server.logged, ['user', '"pass"']) imap.logout() self.assertIsNone(server.logged) self.assertIsNone(server.logged) diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 6e34094c5aa4223..b06096700d1332a 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -35,7 +35,7 @@ cpython_only, is_apple_mobile, is_emscripten, - is_wasi, + is_wasm32, run_in_subinterp, run_in_subinterp_with_config, Py_TRACE_REFS, @@ -43,6 +43,7 @@ Py_GIL_DISABLED, no_rerun, force_not_colorized_test_class, + catch_unraisable_exception ) from test.support.import_helper import ( forget, make_legacy_pyc, unlink, unload, ready_to_import, @@ -359,6 +360,15 @@ def test_import_raises_ModuleNotFoundError(self): with self.assertRaises(ModuleNotFoundError): import something_that_should_not_exist_anywhere + def test_import_null_byte_in_name_raises_ModuleNotFoundError(self): + # gh-150633: module names containing null bytes should not + # lead to duplicates in sys.modules + before = set(sys.modules.keys()) + with self.assertRaises(ModuleNotFoundError): + __import__('zipimport\x00junk') + + self.assertEqual(set(sys.modules.keys()), before) + def test_from_import_missing_module_raises_ModuleNotFoundError(self): with self.assertRaises(ModuleNotFoundError): from something_that_should_not_exist_anywhere import blah @@ -1187,6 +1197,7 @@ class substr(str): @unittest.skipIf(sys.platform == 'win32', 'Cannot delete cwd on Windows') @unittest.skipIf(sys.platform == 'sunos5', 'Cannot delete cwd on Solaris/Illumos') + @unittest.skipIf(sys.platform.startswith('aix'), 'Cannot delete cwd on AIX') def test_script_shadowing_stdlib_cwd_failure(self): with os_helper.temp_dir() as tmp: subtmp = os.path.join(tmp, "subtmp") @@ -1257,7 +1268,7 @@ class FilePermissionTests(unittest.TestCase): @unittest.skipUnless(os.name == 'posix', "test meaningful only on posix systems") @unittest.skipIf( - is_emscripten or is_wasi, + is_wasm32, "Emscripten's/WASI's umask is a stub." ) def test_creation_mode(self): @@ -1609,6 +1620,11 @@ def _clean(self): unlink(self.source) def setUp(self): + # These tests assume bytecode is written next to the source in a + # local __pycache__ directory, so neutralize any pycache prefix (e.g. + # when the test suite is run with PYTHONPYCACHEPREFIX set). + self._orig_pycache_prefix = sys.pycache_prefix + sys.pycache_prefix = None self.source = TESTFN + '.py' self._clean() with open(self.source, 'w', encoding='utf-8') as fp: @@ -1620,6 +1636,7 @@ def tearDown(self): assert sys.path[0] == os.curdir, 'Unexpected sys.path[0]' del sys.path[0] self._clean() + sys.pycache_prefix = self._orig_pycache_prefix @skip_if_dont_write_bytecode def test_import_pyc_path(self): @@ -2539,6 +2556,32 @@ def test_disallowed_reimport(self): excsnap = _interpreters.run_string(interpid, script) self.assertIsNot(excsnap, None) + @requires_subinterpreters + def test_pyinit_function_raises_exception(self): + # gh-144601: PyInit functions that raised exceptions would cause a + # crash when imported from a subinterpreter. + import _testsinglephase + filename = _testsinglephase.__file__ + script = f"""if True: + from test.test_import import import_extension_from_file + + import_extension_from_file('_testsinglephase_raise_exception', {filename!r})""" + + interp = _interpreters.create() + try: + with catch_unraisable_exception() as cm: + exception = _interpreters.run_string(interp, script) + unraisable = cm.unraisable + finally: + _interpreters.destroy(interp) + + self.assertIsNotNone(exception) + self.assertIsNotNone(exception.type.__name__, "ImportError") + self.assertIsNotNone(exception.msg, "failed to import from subinterpreter due to exception") + self.assertIsNotNone(unraisable) + self.assertIs(unraisable.exc_type, RuntimeError) + self.assertEqual(str(unraisable.exc_value), "evil") + class TestSinglePhaseSnapshot(ModuleSnapshot): """A representation of a single-phase init module for testing. @@ -3159,6 +3202,7 @@ def test_check_state_first(self): # Also, we test with a single-phase module that has global state, # which is shared by all interpreters. + @no_rerun(reason="module state is not cleared (see gh-140657)") @requires_subinterpreters def test_basic_multiple_interpreters_main_no_reset(self): # without resetting; already loaded in main interpreter diff --git a/Lib/test/test_importlib/import_/test_relative_imports.py b/Lib/test/test_importlib/import_/test_relative_imports.py index e535d1197631484..1549cbe96ce2d1a 100644 --- a/Lib/test/test_importlib/import_/test_relative_imports.py +++ b/Lib/test/test_importlib/import_/test_relative_imports.py @@ -223,6 +223,21 @@ def test_relative_import_no_package_exists_absolute(self): self.__import__('sys', {'__package__': '', '__spec__': None}, level=1) + def test_malicious_relative_import(self): + # https://github.com/python/cpython/issues/134100 + # Test to make sure UAF bug with error msg doesn't come back to life + import sys + loooong = "".ljust(0x23000, "b") + name = f"a.{loooong}.c" + + with util.uncache(name): + sys.modules[name] = {} + with self.assertRaisesRegex( + KeyError, + r"'a\.b+' not in sys\.modules as expected" + ): + __import__(f"{loooong}.c", {"__package__": "a"}, level=1) + (Frozen_RelativeImports, Source_RelativeImports diff --git a/Lib/test/test_importlib/test_abc.py b/Lib/test/test_importlib/test_abc.py index 070920d0da7e190..dd943210ffca3cf 100644 --- a/Lib/test/test_importlib/test_abc.py +++ b/Lib/test/test_importlib/test_abc.py @@ -224,15 +224,7 @@ class ResourceLoaderDefaultsTests(ABCTestHarness): SPLIT = make_abc_subclasses(ResourceLoader) def test_get_data(self): - with ( - self.assertRaises(IOError), - self.assertWarnsRegex( - DeprecationWarning, - r"importlib\.abc\.ResourceLoader is deprecated in favour of " - r"supporting resource loading through importlib\.resources" - r"\.abc\.TraversableResources.", - ), - ): + with self.assertRaises(IOError): self.ins.get_data('/some/path') @@ -936,13 +928,8 @@ def get_filename(self, fullname): def path_stats(self, path): return {'mtime': 1} - with self.assertWarnsRegex( - DeprecationWarning, - r"importlib\.abc\.ResourceLoader is deprecated in favour of " - r"supporting resource loading through importlib\.resources" - r"\.abc\.TraversableResources.", - ): - loader = DummySourceLoader() + + loader = DummySourceLoader() with self.assertWarnsRegex( DeprecationWarning, @@ -952,17 +939,5 @@ def path_stats(self, path): loader.path_mtime('foo.py') -class ResourceLoaderDeprecationWarningsTests(unittest.TestCase): - """Tests ResourceLoader deprecation warnings.""" - - def test_deprecated_resource_loader(self): - from importlib.abc import ResourceLoader - class DummyLoader(ResourceLoader): - def get_data(self, path): - return b'' - - with self.assertWarns(DeprecationWarning): - DummyLoader() - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_locks.py b/Lib/test/test_importlib/test_locks.py index befac5d62b0abf1..655e5881a1530b3 100644 --- a/Lib/test/test_importlib/test_locks.py +++ b/Lib/test/test_importlib/test_locks.py @@ -34,6 +34,7 @@ class ModuleLockAsRLockTests: # lock status in repr unsupported test_repr = None test_locked_repr = None + test_repr_count = None def tearDown(self): for splitinit in init.values(): diff --git a/Lib/test/test_importlib/test_threaded_import.py b/Lib/test/test_importlib/test_threaded_import.py index 9af1e4d505c66e0..8b793ebf29bcae4 100644 --- a/Lib/test/test_importlib/test_threaded_import.py +++ b/Lib/test/test_importlib/test_threaded_import.py @@ -135,10 +135,12 @@ def check_parallel_module_init(self, mock_os): if verbose: print("OK.") - def test_parallel_module_init(self): + @support.bigmemtest(size=50, memuse=76*2**20, dry_run=False) + def test_parallel_module_init(self, size): self.check_parallel_module_init() - def test_parallel_meta_path(self): + @support.bigmemtest(size=50, memuse=76*2**20, dry_run=False) + def test_parallel_meta_path(self, size): finder = Finder() sys.meta_path.insert(0, finder) try: @@ -148,7 +150,8 @@ def test_parallel_meta_path(self): finally: sys.meta_path.remove(finder) - def test_parallel_path_hooks(self): + @support.bigmemtest(size=50, memuse=76*2**20, dry_run=False) + def test_parallel_path_hooks(self, size): # Here the Finder instance is only used to check concurrent calls # to path_hook(). finder = Finder() @@ -242,18 +245,85 @@ def target(): __import__(TESTFN) del sys.modules[TESTFN] - def test_concurrent_futures_circular_import(self): + @support.bigmemtest(size=1, memuse=1.8*2**30, dry_run=False) + def test_concurrent_futures_circular_import(self, size): # Regression test for bpo-43515 fn = os.path.join(os.path.dirname(__file__), 'partial', 'cfimport.py') script_helper.assert_python_ok(fn) - def test_multiprocessing_pool_circular_import(self): + @support.bigmemtest(size=1, memuse=1.8*2**30, dry_run=False) + def test_multiprocessing_pool_circular_import(self, size): # Regression test for bpo-41567 fn = os.path.join(os.path.dirname(__file__), 'partial', 'pool_in_threads.py') script_helper.assert_python_ok(fn) + def test_import_failure_race_condition(self): + # Regression test for race condition where a thread could receive + # a partially-initialized module when another thread's import fails. + # The race occurs when: + # 1. Thread 1 starts importing, adds module to sys.modules + # 2. Thread 2 sees the module in sys.modules + # 3. Thread 1's import fails, removes module from sys.modules + # 4. Thread 2 should NOT return the stale module reference + os.mkdir(TESTFN) + self.addCleanup(shutil.rmtree, TESTFN) + sys.path.insert(0, TESTFN) + self.addCleanup(sys.path.remove, TESTFN) + + # Create a module that partially initializes then fails + modname = 'failing_import_module' + with open(os.path.join(TESTFN, modname + '.py'), 'w') as f: + f.write(''' +import time +PARTIAL_ATTR = 'initialized' +time.sleep(0.05) # Widen race window +raise RuntimeError("Intentional import failure") +''') + self.addCleanup(forget, modname) + importlib.invalidate_caches() + + errors = [] + results = [] + + def do_import(delay=0): + time.sleep(delay) + try: + mod = __import__(modname) + # If we got a module, verify it's in sys.modules + if modname not in sys.modules: + errors.append( + f"Got module {mod!r} but {modname!r} not in sys.modules" + ) + elif sys.modules[modname] is not mod: + errors.append( + f"Got different module than sys.modules[{modname!r}]" + ) + else: + results.append(('success', mod)) + except RuntimeError: + results.append(('RuntimeError',)) + except Exception as e: + errors.append(f"Unexpected exception: {e}") + + # Run multiple iterations to increase chance of hitting the race + for _ in range(10): + errors.clear() + results.clear() + if modname in sys.modules: + del sys.modules[modname] + + t1 = threading.Thread(target=do_import, args=(0,)) + t2 = threading.Thread(target=do_import, args=(0.01,)) + t1.start() + t2.start() + t1.join() + t2.join() + + # Neither thread should have errors about stale modules + self.assertEqual(errors, [], f"Race condition detected: {errors}") + def setUpModule(): thread_info = threading_helper.threading_setup() diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 5de89714eb50c72..f7df5c3251edb49 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -334,6 +334,17 @@ class PEP3147Tests: tag = sys.implementation.cache_tag + def setUp(self): + # Most of these tests assume the default (unset) pycache prefix, so + # clear it for the duration of the test (e.g. when the test suite is + # run with PYTHONPYCACHEPREFIX set). Tests that need a specific prefix + # set their own via util.temporary_pycache_prefix(). + self._orig_pycache_prefix = sys.pycache_prefix + sys.pycache_prefix = None + + def tearDown(self): + sys.pycache_prefix = self._orig_pycache_prefix + @unittest.skipIf(sys.implementation.cache_tag is None, 'requires sys.implementation.cache_tag not be None') def test_cache_from_source(self): @@ -580,6 +591,18 @@ def test_cache_from_source_respects_pycache_prefix_relative(self): self.util.cache_from_source(path, optimization=''), os.path.normpath(expect)) + @unittest.skipIf(sys.implementation.cache_tag is None, + 'requires sys.implementation.cache_tag to not be None') + def test_cache_from_source_in_root_with_pycache_prefix(self): + # Regression test for gh-82916 + pycache_prefix = os.path.join(os.path.sep, 'tmp', 'bytecode') + path = 'qux.py' + expect = os.path.join(os.path.sep, 'tmp', 'bytecode', + f'qux.{self.tag}.pyc') + with util.temporary_pycache_prefix(pycache_prefix): + with os_helper.change_cwd('/'): + self.assertEqual(self.util.cache_from_source(path), expect) + @unittest.skipIf(sys.implementation.cache_tag is None, 'requires sys.implementation.cache_tag to not be None') def test_source_from_cache_inside_pycache_prefix(self): @@ -635,7 +658,7 @@ def test_magic_number(self): # stakeholders such as OS package maintainers must be notified # in advance. Such exceptional releases will then require an # adjustment to this test case. - EXPECTED_MAGIC_NUMBER = 3495 + EXPECTED_MAGIC_NUMBER = 3627 actual = int.from_bytes(importlib.util.MAGIC_NUMBER[:2], 'little') msg = ( @@ -776,31 +799,70 @@ def test_complete_multi_phase_init_module(self): self.run_with_own_gil(script) -class MiscTests(unittest.TestCase): - def test_atomic_write_should_notice_incomplete_writes(self): +class PatchAtomicWrites: + def __init__(self, truncate_at_length, never_complete=False): + self.truncate_at_length = truncate_at_length + self.never_complete = never_complete + self.seen_write = False + self._children = [] + + def __enter__(self): import _pyio oldwrite = os.write - seen_write = False - - truncate_at_length = 100 # Emulate an os.write that only writes partial data. def write(fd, data): - nonlocal seen_write - seen_write = True - return oldwrite(fd, data[:truncate_at_length]) + if self.seen_write and self.never_complete: + return None + self.seen_write = True + return oldwrite(fd, data[:self.truncate_at_length]) # Need to patch _io to be _pyio, so that io.FileIO is affected by the # os.write patch. - with (support.swap_attr(_bootstrap_external, '_io', _pyio), - support.swap_attr(os, 'write', write)): - with self.assertRaises(OSError): - # Make sure we write something longer than the point where we - # truncate. - content = b'x' * (truncate_at_length * 2) - _bootstrap_external._write_atomic(os_helper.TESTFN, content) - assert seen_write + self.children = [ + support.swap_attr(_bootstrap_external, '_io', _pyio), + support.swap_attr(os, 'write', write) + ] + for child in self.children: + child.__enter__() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + for child in self.children: + child.__exit__(exc_type, exc_val, exc_tb) + + +class MiscTests(unittest.TestCase): + + def test_atomic_write_retries_incomplete_writes(self): + truncate_at_length = 100 + length = truncate_at_length * 2 + + with PatchAtomicWrites(truncate_at_length=truncate_at_length) as cm: + # Make sure we write something longer than the point where we + # truncate. + content = b'x' * length + _bootstrap_external._write_atomic(os_helper.TESTFN, content) + self.assertTrue(cm.seen_write) + + self.assertEqual(os.stat(support.os_helper.TESTFN).st_size, length) + os.unlink(support.os_helper.TESTFN) + + def test_atomic_write_errors_if_unable_to_complete(self): + truncate_at_length = 100 + + with ( + PatchAtomicWrites( + truncate_at_length=truncate_at_length, never_complete=True, + ) as cm, + self.assertRaises(OSError) + ): + # Make sure we write something longer than the point where we + # truncate. + content = b'x' * (truncate_at_length * 2) + _bootstrap_external._write_atomic(os_helper.TESTFN, content) + self.assertTrue(cm.seen_write) with self.assertRaises(OSError): os.stat(support.os_helper.TESTFN) # Check that the file did not get written. diff --git a/Lib/test/test_importlib/util.py b/Lib/test/test_importlib/util.py index edbe78545a2536a..bd64b03b75f40aa 100644 --- a/Lib/test/test_importlib/util.py +++ b/Lib/test/test_importlib/util.py @@ -15,7 +15,7 @@ import tempfile import types -_testsinglephase = import_helper.import_module("_testsinglephase") +import_helper.import_module("_testmultiphase") BUILTINS = types.SimpleNamespace() diff --git a/Lib/test/test_inspect/inspect_fodder2.py b/Lib/test/test_inspect/inspect_fodder2.py index 43fda6622537fc7..157e12167b5d270 100644 --- a/Lib/test/test_inspect/inspect_fodder2.py +++ b/Lib/test/test_inspect/inspect_fodder2.py @@ -369,3 +369,35 @@ class dc364: # line 369 dc370 = dataclasses.make_dataclass('dc370', (('x', int), ('y', int))) dc371 = dataclasses.make_dataclass('dc370', (('x', int), ('y', int)), module=__name__) + +import inspect +import itertools + +# line 376 +ge377 = ( + inspect.currentframe() + for i in itertools.count() +) + +# line 382 +def func383(): + # line 384 + ge385 = ( + inspect.currentframe() + for i in itertools.count() + ) + return ge385 + +# line 391 +@decorator +# comment +def func394(): + return 395 + +# line 397 +@decorator + +def func400(): + return 401 + +pass # end of file diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index c9b37fcd8f63277..f3b04285b0d1d7c 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -7,6 +7,7 @@ import functools import gc import importlib +import importlib.util import inspect import io import linecache @@ -148,6 +149,29 @@ def meth_self_o(self, object, /): pass def meth_type_noargs(type, /): pass def meth_type_o(type, object, /): pass +# Decorator decorator that returns a simple wrapped function +def identity_wrapper(func): + @functools.wraps(func) + def wrapped(*args, **kwargs): + return func(*args, **kwargs) + return wrapped + +# Original signature of the simple wrapped function returned by +# identity_wrapper(). +varargs_signature = ( + (('args', ..., ..., 'var_positional'), + ('kwargs', ..., ..., 'var_keyword')), + ..., +) + +# Decorator decorator that returns a simple descriptor +class custom_descriptor: + def __init__(self, func): + self.func = func + + def __get__(self, instance, owner): + return self.func.__get__(instance, owner) + class TestPredicates(IsTestBase): @@ -786,12 +810,12 @@ def test_getfile(self): def test_getfile_builtin_module(self): with self.assertRaises(TypeError) as e: inspect.getfile(sys) - self.assertTrue(str(e.exception).startswith('<module')) + self.assertStartsWith(str(e.exception), '<module') def test_getfile_builtin_class(self): with self.assertRaises(TypeError) as e: inspect.getfile(int) - self.assertTrue(str(e.exception).startswith('<class')) + self.assertStartsWith(str(e.exception), '<class') def test_getfile_builtin_function_or_method(self): with self.assertRaises(TypeError) as e_abs: @@ -1189,12 +1213,22 @@ def test_nested_class_definition_inside_async_function(self): self.assertSourceEqual(run(mod2.func225), 226, 227) self.assertSourceEqual(mod2.cls226, 231, 235) + self.assertSourceEqual(mod2.cls226.func232, 232, 235) self.assertSourceEqual(run(mod2.cls226().func232), 233, 234) def test_class_definition_same_name_diff_methods(self): self.assertSourceEqual(mod2.cls296, 296, 298) self.assertSourceEqual(mod2.cls310, 310, 312) + def test_generator_expression(self): + self.assertSourceEqual(next(mod2.ge377), 377, 380) + self.assertSourceEqual(next(mod2.func383()), 385, 388) + + def test_comment_or_empty_line_after_decorator(self): + self.assertSourceEqual(mod2.func394, 392, 395) + self.assertSourceEqual(mod2.func400, 398, 401) + + class TestNoEOL(GetSourceBase): def setUp(self): self.tempdir = TESTFN + '_dir' @@ -1749,14 +1783,55 @@ class C(metaclass=M): class TestFormatAnnotation(unittest.TestCase): def test_typing_replacement(self): - from test.typinganndata.ann_module9 import ann, ann1 + from test.typinganndata.ann_module9 import A, ann, ann1 self.assertEqual(inspect.formatannotation(ann), 'List[str] | int') self.assertEqual(inspect.formatannotation(ann1), 'List[testModule.typing.A] | int') + self.assertEqual(inspect.formatannotation(A, 'testModule.typing'), 'A') + self.assertEqual(inspect.formatannotation(A, 'other'), 'testModule.typing.A') + self.assertEqual( + inspect.formatannotation(ann1, 'testModule.typing'), + 'List[testModule.typing.A] | int', + ) + def test_forwardref(self): fwdref = ForwardRef('fwdref') self.assertEqual(inspect.formatannotation(fwdref), 'fwdref') + def test_formatannotationrelativeto(self): + from test.typinganndata.ann_module9 import A, ann1 + + # Builtin types: + self.assertEqual( + inspect.formatannotationrelativeto(object)(type), + 'type', + ) + + # Custom types: + self.assertEqual( + inspect.formatannotationrelativeto(None)(A), + 'testModule.typing.A', + ) + + class B: ... + B.__module__ = 'testModule.typing' + + self.assertEqual( + inspect.formatannotationrelativeto(B)(A), + 'A', + ) + + self.assertEqual( + inspect.formatannotationrelativeto(object)(A), + 'testModule.typing.A', + ) + + # Not an instance of "type": + self.assertEqual( + inspect.formatannotationrelativeto(A)(ann1), + 'List[testModule.typing.A] | int', + ) + class TestIsMethodDescriptor(unittest.TestCase): @@ -2683,6 +2758,30 @@ def running_check_generator(): # Running after the first yield next(self.generator) + def test_types_coroutine_wrapper_state(self): + def gen(): + yield 1 + yield 2 + + @types.coroutine + def wrapped_generator_coro(): + # return a generator iterator so types.coroutine + # wraps it into types._GeneratorWrapper. + return gen() + + g = wrapped_generator_coro() + self.addCleanup(g.close) + self.assertIs(type(g), types._GeneratorWrapper) + + # _GeneratorWrapper must provide gi_suspended/cr_suspended + # so inspect.get*state() doesn't raise AttributeError. + self.assertEqual(inspect.getgeneratorstate(g), inspect.GEN_CREATED) + self.assertEqual(inspect.getcoroutinestate(g), inspect.CORO_CREATED) + + next(g) + self.assertEqual(inspect.getgeneratorstate(g), inspect.GEN_SUSPENDED) + self.assertEqual(inspect.getcoroutinestate(g), inspect.CORO_SUSPENDED) + def test_easy_debugging(self): # repr() and str() of a generator state should contain the state name names = 'GEN_CREATED GEN_RUNNING GEN_SUSPENDED GEN_CLOSED'.split() @@ -2820,7 +2919,7 @@ async def asyncTearDown(self): @classmethod def tearDownClass(cls): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) def _asyncgenstate(self): return inspect.getasyncgenstate(self.asyncgen) @@ -2949,7 +3048,7 @@ def test(po, /, pk, pkd=100, *args, ko, kod=10, **kwargs): pass sig = inspect.signature(test) - self.assertTrue(repr(sig).startswith('<Signature')) + self.assertStartsWith(repr(sig), '<Signature') self.assertTrue('(po, /, pk' in repr(sig)) # We need two functions, because it is impossible to represent @@ -2958,7 +3057,7 @@ def test2(pod=42, /): pass sig2 = inspect.signature(test2) - self.assertTrue(repr(sig2).startswith('<Signature')) + self.assertStartsWith(repr(sig2), '<Signature') self.assertTrue('(pod=42, /)' in repr(sig2)) po = sig.parameters['po'] @@ -4021,44 +4120,272 @@ def __init__(self, b): ('bar', 2, ..., "keyword_only")), ...)) - def test_signature_on_class_with_decorated_new(self): - def identity(func): - @functools.wraps(func) - def wrapped(*args, **kwargs): - return func(*args, **kwargs) - return wrapped - - class Foo: - @identity - def __new__(cls, a, b): + def test_signature_on_class_with_wrapped_metaclass_call(self): + class CM(type): + @identity_wrapper + def __call__(cls, a): + pass + class C(metaclass=CM): + def __init__(self, b): pass - self.assertEqual(self.signature(Foo), - ((('a', ..., ..., "positional_or_keyword"), - ('b', ..., ..., "positional_or_keyword")), + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), ...)) - self.assertEqual(self.signature(Foo.__new__), - ((('cls', ..., ..., "positional_or_keyword"), - ('a', ..., ..., "positional_or_keyword"), - ('b', ..., ..., "positional_or_keyword")), - ...)) + with self.subTest('classmethod'): + class CM(type): + @classmethod + @identity_wrapper + def __call__(cls, a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass - class Bar: - __new__ = identity(object.__new__) + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) - varargs_signature = ( - (('args', ..., ..., 'var_positional'), - ('kwargs', ..., ..., 'var_keyword')), - ..., - ) + with self.subTest('staticmethod'): + class CM(type): + @staticmethod + @identity_wrapper + def __call__(a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + @identity_wrapper + def call(self, a): + return a + class CM(type): + __call__ = A().call + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('descriptor'): + class CM(type): + @custom_descriptor + @identity_wrapper + def __call__(self, a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + self.assertEqual(self.signature(C.__call__), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + self.assertEqual(self.signature(C, follow_wrapped=False), + varargs_signature) + self.assertEqual(self.signature(C.__call__, follow_wrapped=False), + varargs_signature) + + def test_signature_on_class_with_wrapped_init(self): + class C: + @identity_wrapper + def __init__(self, b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + @identity_wrapper + def __init__(cls, b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + @identity_wrapper + def __init__(b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + @identity_wrapper + def call(self, a): + pass + + class C: + __init__ = A().call + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __init__ = functools.partial(identity_wrapper(lambda x, a, b: None), 2) + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + @identity_wrapper + def _init(self, x, a): + self.a = (x, a) + __init__ = functools.partialmethod(_init, 2) + + self.assertEqual(C(1).a, (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('descriptor'): + class C: + @custom_descriptor + @identity_wrapper + def __init__(self, a): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + self.assertEqual(self.signature(C.__init__), + ((('self', ..., ..., "positional_or_keyword"), + ('a', ..., ..., "positional_or_keyword")), + ...)) + + self.assertEqual(self.signature(C, follow_wrapped=False), + varargs_signature) + if support.MISSING_C_DOCSTRINGS: + self.assertRaisesRegex( + ValueError, "no signature found", + self.signature, C.__new__, follow_wrapped=False, + ) + else: + self.assertEqual(self.signature(C.__new__, follow_wrapped=False), + varargs_signature) + + def test_signature_on_class_with_wrapped_new(self): + with self.subTest('FunctionType'): + class C: + @identity_wrapper + def __new__(cls, a): + return a - self.assertEqual(self.signature(Bar), ((), ...)) - self.assertEqual(self.signature(Bar.__new__), varargs_signature) - self.assertEqual(self.signature(Bar, follow_wrapped=False), - varargs_signature) - self.assertEqual(self.signature(Bar.__new__, follow_wrapped=False), - varargs_signature) + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + @identity_wrapper + def __new__(cls, cls2, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + @identity_wrapper + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + @identity_wrapper + def call(self, cls, a): + return a + class C: + __new__ = A().call + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __new__ = functools.partial(identity_wrapper(lambda x, cls, a: (x, a)), 2) + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + __new__ = functools.partialmethod(identity_wrapper(lambda cls, x, a: (x, a)), 2) + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('descriptor'): + class C: + @custom_descriptor + @identity_wrapper + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + self.assertEqual(self.signature(C.__new__), + ((('cls', ..., ..., "positional_or_keyword"), + ('a', ..., ..., "positional_or_keyword")), + ...)) + + self.assertEqual(self.signature(C, follow_wrapped=False), + varargs_signature) + self.assertEqual(self.signature(C.__new__, follow_wrapped=False), + varargs_signature) def test_signature_on_class_with_init(self): class C: @@ -4997,6 +5324,37 @@ def test_signature_annotation_format(self): with self.assertRaisesRegex(NameError, "undefined"): signature_func(ida.f) + def test_signature_deferred_annotations(self): + def f(x: undef): + pass + + class C: + x: undef + + def __init__(self, x: undef): + self.x = x + + sig = inspect.signature(f, annotation_format=Format.FORWARDREF) + self.assertEqual(list(sig.parameters), ['x']) + sig = inspect.signature(C, annotation_format=Format.FORWARDREF) + self.assertEqual(list(sig.parameters), ['x']) + + class CallableWrapper: + def __init__(self, func): + self.func = func + self.__annotate__ = func.__annotate__ + + def __call__(self, *args, **kwargs): + return self.func(*args, **kwargs) + + @property + def __annotations__(self): + return self.__annotate__(Format.VALUE) + + cw = CallableWrapper(f) + sig = inspect.signature(cw, annotation_format=Format.FORWARDREF) + self.assertEqual(list(sig.parameters), ['args', 'kwargs']) + def test_signature_none_annotation(self): class funclike: # Has to be callable, and have correct @@ -5102,7 +5460,7 @@ def test_signature_parameter_object(self): with self.assertRaisesRegex(ValueError, 'cannot have default values'): p.replace(kind=inspect.Parameter.VAR_POSITIONAL) - self.assertTrue(repr(p).startswith('<Parameter')) + self.assertStartsWith(repr(p), '<Parameter') self.assertTrue('"a=42"' in repr(p)) def test_signature_parameter_hashable(self): @@ -5844,10 +6202,9 @@ def test_operator_module_has_signatures(self): self._test_module_has_signatures(operator) def test_os_module_has_signatures(self): - unsupported_signature = {'chmod', 'link', 'utime'} + unsupported_signature = {'chmod', 'utime'} unsupported_signature |= {name for name in - ['get_terminal_size', 'posix_spawn', 'posix_spawnp', - 'register_at_fork', 'startfile'] + ['get_terminal_size', 'link', 'register_at_fork', 'startfile'] if hasattr(os, name)} self._test_module_has_signatures(os, unsupported_signature=unsupported_signature) @@ -5885,6 +6242,7 @@ def test_sysconfig_module_has_signatures(self): def test_threading_module_has_signatures(self): import threading self._test_module_has_signatures(threading) + self.assertIsNotNone(inspect.signature(threading.__excepthook__)) def test_thread_module_has_signatures(self): import _thread @@ -6058,6 +6416,19 @@ def test_wrapped_descriptor(self): class TestMain(unittest.TestCase): + @staticmethod + def _expected_cached(module): + # assert_python_ok() runs the subprocess in isolated mode (-I), which + # ignores PYTHONPYCACHEPREFIX, so compute the expected cached path the + # same way (i.e. without any pycache prefix) to stay independent of the + # environment the test suite is run in. Modules without a cached path + # (e.g. frozen modules such as ntpath/importlib.machinery on Windows) + # report None, so preserve that. + if module.__spec__.cached is None: + return None + with support.swap_attr(sys, 'pycache_prefix', None): + return importlib.util.cache_from_source(module.__spec__.origin) + def test_only_source(self): module = importlib.import_module('unittest') rc, out, err = assert_python_ok('-m', 'inspect', @@ -6097,13 +6468,13 @@ def test_details(self): rc, out, err = assert_python_ok(*args, '-m', 'inspect', 'unittest', '--details') output = out.decode() + cached = self._expected_cached(module) # Just a quick sanity check on the output self.assertIn(module.__spec__.name, output) self.assertIn(module.__name__, output) self.assertIn(module.__spec__.origin, output) self.assertIn(module.__file__, output) - self.assertIn(module.__spec__.cached, output) - self.assertIn(module.__cached__, output) + self.assertIn(cached, output) self.assertEqual(err, b'') diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 245528ce57a1469..fa675f626cbe109 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -778,6 +778,18 @@ def test_pylong_int_divmod(self): a, b = divmod(n*3 + 1, n) assert a == 3 and b == 1 + @support.cpython_only # tests implementation details of CPython. + @unittest.skipUnless(_pylong, "_pylong module required") + def test_pylong_int_divmod_crash(self): + # Regression test for https://github.com/python/cpython/issues/142554. + bad_int_divmod = lambda a, b: (1,) + # 'k' chosen such that divmod(2**(2*k), 2**k) uses _pylong.int_divmod() + k = 10_000 + a, b = (1 << (2 * k)), (1 << k) + with mock.patch.object(_pylong, "int_divmod", wraps=bad_int_divmod): + msg = r"tuple of length 2 is required from int_divmod\(\)" + self.assertRaisesRegex(ValueError, msg, divmod, a, b) + def test_pylong_str_to_int(self): v1 = 1 << 100_000 s = str(v1) @@ -836,7 +848,7 @@ def test_pylong_roundtrip(self): n = hibit | getrandbits(bits - 1) assert n.bit_length() == bits sn = str(n) - self.assertFalse(sn.startswith('0')) + self.assertNotStartsWith(sn, '0') self.assertEqual(n, int(sn)) bits <<= 1 diff --git a/Lib/test/test_interpreters/test_api.py b/Lib/test/test_interpreters/test_api.py index 66c7afce88f0d80..e88fd399dfba382 100644 --- a/Lib/test/test_interpreters/test_api.py +++ b/Lib/test/test_interpreters/test_api.py @@ -1,18 +1,24 @@ +import contextlib import os import pickle +import sys from textwrap import dedent import threading import types import unittest from test import support +from test.support import os_helper +from test.support import script_helper from test.support import import_helper +from test.support.script_helper import assert_python_ok # Raise SkipTest if subinterpreters not supported. _interpreters = import_helper.import_module('_interpreters') +from concurrent import interpreters from test.support import Py_GIL_DISABLED -from test.support import interpreters from test.support import force_not_colorized -from test.support.interpreters import ( +import test._crossinterp_definitions as defs +from concurrent.interpreters import ( InterpreterError, InterpreterNotFoundError, ExecutionFailed, ) from .utils import ( @@ -29,6 +35,59 @@ WHENCE_STR_STDLIB = '_interpreters module' +def is_pickleable(obj): + try: + pickle.dumps(obj) + except Exception: + return False + return True + + +@contextlib.contextmanager +def defined_in___main__(name, script, *, remove=False): + import __main__ as mainmod + mainns = vars(mainmod) + assert name not in mainns + exec(script, mainns, mainns) + if remove: + yield mainns.pop(name) + else: + try: + yield mainns[name] + finally: + mainns.pop(name, None) + + +def build_excinfo(exctype, msg=None, formatted=None, errdisplay=None): + if isinstance(exctype, type): + assert issubclass(exctype, BaseException), exctype + exctype = types.SimpleNamespace( + __name__=exctype.__name__, + __qualname__=exctype.__qualname__, + __module__=exctype.__module__, + ) + elif isinstance(exctype, str): + module, _, name = exctype.rpartition(exctype) + if not module and name in __builtins__: + module = 'builtins' + exctype = types.SimpleNamespace( + __name__=name, + __qualname__=exctype, + __module__=module or None, + ) + else: + assert isinstance(exctype, types.SimpleNamespace) + assert msg is None or isinstance(msg, str), msg + assert formatted is None or isinstance(formatted, str), formatted + assert errdisplay is None or isinstance(errdisplay, str), errdisplay + return types.SimpleNamespace( + type=exctype, + msg=msg, + formatted=formatted, + errdisplay=errdisplay, + ) + + class ModuleTests(TestBase): def test_queue_aliases(self): @@ -75,7 +134,7 @@ def test_in_subinterpreter(self): main, = interpreters.list_all() interp = interpreters.create() out = _run_output(interp, dedent(""" - from test.support import interpreters + from concurrent import interpreters interp = interpreters.create() print(interp.id) """)) @@ -138,7 +197,7 @@ def test_subinterpreter(self): main = interpreters.get_main() interp = interpreters.create() out = _run_output(interp, dedent(""" - from test.support import interpreters + from concurrent import interpreters cur = interpreters.get_current() print(cur.id) """)) @@ -155,7 +214,7 @@ def test_idempotent(self): with self.subTest('subinterpreter'): interp = interpreters.create() out = _run_output(interp, dedent(""" - from test.support import interpreters + from concurrent import interpreters cur = interpreters.get_current() print(id(cur)) cur = interpreters.get_current() @@ -167,7 +226,7 @@ def test_idempotent(self): with self.subTest('per-interpreter'): interp = interpreters.create() out = _run_output(interp, dedent(""" - from test.support import interpreters + from concurrent import interpreters cur = interpreters.get_current() print(id(cur)) """)) @@ -354,9 +413,27 @@ def test_equality(self): def test_pickle(self): interp = interpreters.create() - data = pickle.dumps(interp) - unpickled = pickle.loads(data) - self.assertEqual(unpickled, interp) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol): + data = pickle.dumps(interp, protocol) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, interp) + + @support.requires_subprocess() + @force_not_colorized + def test_cleanup_in_repl(self): + # GH-135729: Using a subinterpreter in the REPL would lead to an unraisable + # exception during finalization + repl = script_helper.spawn_python("-i") + script = b"""if True: + from concurrent import interpreters + interpreters.create() + exit()""" + stdout, stderr = repl.communicate(script) + self.assertIsNone(stderr) + self.assertIn(b"Interpreter.close()", stdout) + self.assertNotIn(b"Traceback", stdout) + class TestInterpreterIsRunning(TestBase): @@ -524,7 +601,7 @@ def test_from_current(self): main, = interpreters.list_all() interp = interpreters.create() out = _run_output(interp, dedent(f""" - from test.support import interpreters + from concurrent import interpreters interp = interpreters.Interpreter({interp.id}) try: interp.close() @@ -541,7 +618,7 @@ def test_from_sibling(self): self.assertEqual(set(interpreters.list_all()), {main, interp1, interp2}) interp1.exec(dedent(f""" - from test.support import interpreters + from concurrent import interpreters interp2 = interpreters.Interpreter({interp2.id}) interp2.close() interp3 = interpreters.create() @@ -647,6 +724,68 @@ def test_created_with_capi(self): self.interp_exists(interpid)) + def test_remaining_threads(self): + r_interp, w_interp = self.pipe() + + FINISHED = b'F' + + # It's unlikely, but technically speaking, it's possible + # that the thread could've finished before interp.close() is + # reached, so this test might not properly exercise the case. + # However, it's quite unlikely and probably not worth bothering about. + interp = interpreters.create() + interp.exec(f"""if True: + import os + import threading + import time + + def task(): + time.sleep(1) + os.write({w_interp}, {FINISHED!r}) + + threads = (threading.Thread(target=task) for _ in range(3)) + for t in threads: + t.start() + """) + interp.close() + + self.assertEqual(os.read(r_interp, 1), FINISHED) + + def test_remaining_daemon_threads(self): + # Daemon threads leak reference by nature, because they hang threads + # without allowing them to do cleanup (i.e., release refs). + # To prevent that from messing up the refleak hunter and whatnot, we + # run this in a subprocess. + code = '''if True: + import _interpreters + import types + interp = _interpreters.create( + types.SimpleNamespace( + use_main_obmalloc=False, + allow_fork=False, + allow_exec=False, + allow_threads=True, + allow_daemon_threads=True, + check_multi_interp_extensions=True, + gil='own', + ) + ) + _interpreters.exec(interp, f"""if True: + import threading + import time + + def task(): + time.sleep(3) + + threads = (threading.Thread(target=task, daemon=True) for _ in range(3)) + for t in threads: + t.start() + """) + _interpreters.destroy(interp) + ''' + assert_python_ok('-c', code) + + class TestInterpreterPrepareMain(TestBase): def test_empty(self): @@ -748,14 +887,17 @@ def eggs(): ham() """) scriptfile = self.make_script('script.py', tempdir, text=""" - from test.support import interpreters + from concurrent import interpreters def script(): import spam spam.eggs() interp = interpreters.create() - interp.exec(script) + try: + interp.exec(script) + finally: + interp.close() """) stdout, stderr = self.assert_python_failure(scriptfile) @@ -764,12 +906,12 @@ def script(): # File "{interpreters.__file__}", line 179, in exec self.assertEqual(stderr, dedent(f"""\ Traceback (most recent call last): - File "{scriptfile}", line 9, in <module> + File "{scriptfile}", line 10, in <module> interp.exec(script) ~~~~~~~~~~~^^^^^^^^ {interpmod_line.strip()} raise ExecutionFailed(excinfo) - test.support.interpreters.ExecutionFailed: RuntimeError: uh-oh! + concurrent.interpreters.ExecutionFailed: RuntimeError: uh-oh! Uncaught in the interpreter: @@ -839,9 +981,16 @@ def test_bad_script(self): interp.exec(10) def test_bytes_for_script(self): + r, w = self.pipe() + RAN = b'R' + DONE = b'D' interp = interpreters.create() - with self.assertRaises(TypeError): - interp.exec(b'print("spam")') + interp.exec(f"""if True: + import os + os.write({w}, {RAN!r}) + """) + os.write(w, DONE) + self.assertEqual(os.read(r, 1), RAN) def test_with_background_threads_still_running(self): r_interp, w_interp = self.pipe() @@ -879,28 +1028,46 @@ def test_created_with_capi(self): with self.assertRaisesRegex(InterpreterError, 'unrecognized'): interp.exec('raise Exception("it worked!")') + def test_list_comprehension(self): + # gh-135450: List comprehensions caused an assertion failure + # in _PyCode_CheckNoExternalState() + import string + r_interp, w_interp = self.pipe() + + interp = interpreters.create() + interp.exec(f"""if True: + import os + comp = [str(i) for i in range(10)] + os.write({w_interp}, ''.join(comp).encode()) + """) + self.assertEqual(os.read(r_interp, 10).decode(), string.digits) + interp.close() + + # test__interpreters covers the remaining # Interpreter.exec() behavior. -def call_func_noop(): - pass +call_func_noop = defs.spam_minimal +call_func_ident = defs.spam_returns_arg +call_func_failure = defs.spam_raises def call_func_return_shareable(): return (1, None) -def call_func_return_not_shareable(): - return [1, 2, 3] +def call_func_return_stateless_func(): + return (lambda x: x) -def call_func_failure(): - raise Exception('spam!') +def call_func_return_pickleable(): + return [1, 2, 3] -def call_func_ident(value): - return value +def call_func_return_unpickleable(): + x = 42 + return (lambda: x) def get_call_func_closure(value): @@ -909,6 +1076,11 @@ def call_func_closure(): return call_func_closure +def call_func_exec_wrapper(script, ns): + res = exec(script, ns, ns) + return res, ns, id(ns) + + class Spam: @staticmethod @@ -1005,86 +1177,663 @@ class TestInterpreterCall(TestBase): # - preserves info (e.g. SyntaxError) # - matching error display - def test_call(self): + @contextlib.contextmanager + def assert_fails(self, expected): + with self.assertRaises(ExecutionFailed) as cm: + yield cm + uncaught = cm.exception.excinfo + self.assertEqual(uncaught.type.__name__, expected.__name__) + + def assert_fails_not_shareable(self): + return self.assert_fails(interpreters.NotShareableError) + + def assert_code_equal(self, code1, code2): + if code1 == code2: + return + self.assertEqual(code1.co_name, code2.co_name) + self.assertEqual(code1.co_flags, code2.co_flags) + self.assertEqual(code1.co_consts, code2.co_consts) + self.assertEqual(code1.co_varnames, code2.co_varnames) + self.assertEqual(code1.co_cellvars, code2.co_cellvars) + self.assertEqual(code1.co_freevars, code2.co_freevars) + self.assertEqual(code1.co_names, code2.co_names) + self.assertEqual( + _testinternalcapi.get_code_var_counts(code1), + _testinternalcapi.get_code_var_counts(code2), + ) + self.assertEqual(code1.co_code, code2.co_code) + + def assert_funcs_equal(self, func1, func2): + if func1 == func2: + return + self.assertIs(type(func1), type(func2)) + self.assertEqual(func1.__name__, func2.__name__) + self.assertEqual(func1.__defaults__, func2.__defaults__) + self.assertEqual(func1.__kwdefaults__, func2.__kwdefaults__) + self.assertEqual(func1.__closure__, func2.__closure__) + self.assert_code_equal(func1.__code__, func2.__code__) + self.assertEqual( + _testinternalcapi.get_code_var_counts(func1), + _testinternalcapi.get_code_var_counts(func2), + ) + + def assert_exceptions_equal(self, exc1, exc2): + assert isinstance(exc1, Exception) + assert isinstance(exc2, Exception) + if exc1 == exc2: + return + self.assertIs(type(exc1), type(exc2)) + self.assertEqual(exc1.args, exc2.args) + + def test_stateless_funcs(self): interp = interpreters.create() - for i, (callable, args, kwargs) in enumerate([ - (call_func_noop, (), {}), - (call_func_return_shareable, (), {}), - (call_func_return_not_shareable, (), {}), - (Spam.noop, (), {}), + func = call_func_noop + with self.subTest('no args, no return'): + res = interp.call(func) + self.assertIsNone(res) + + func = call_func_return_shareable + with self.subTest('no args, returns shareable'): + res = interp.call(func) + self.assertEqual(res, (1, None)) + + func = call_func_return_stateless_func + expected = (lambda x: x) + with self.subTest('no args, returns stateless func'): + res = interp.call(func) + self.assert_funcs_equal(res, expected) + + func = call_func_return_pickleable + with self.subTest('no args, returns pickleable'): + res = interp.call(func) + self.assertEqual(res, [1, 2, 3]) + + func = call_func_return_unpickleable + with self.subTest('no args, returns unpickleable'): + with self.assertRaises(interpreters.NotShareableError): + interp.call(func) + + def test_stateless_func_returns_arg(self): + interp = interpreters.create() + + for arg in [ + None, + 10, + 'spam!', + b'spam!', + (1, 2, 'spam!'), + memoryview(b'spam!'), + ]: + with self.subTest(f'shareable {arg!r}'): + assert _interpreters.is_shareable(arg) + res = interp.call(defs.spam_returns_arg, arg) + self.assertEqual(res, arg) + + for arg in defs.STATELESS_FUNCTIONS: + with self.subTest(f'stateless func {arg!r}'): + res = interp.call(defs.spam_returns_arg, arg) + self.assert_funcs_equal(res, arg) + + for arg in defs.TOP_FUNCTIONS: + if arg in defs.STATELESS_FUNCTIONS: + continue + with self.subTest(f'stateful func {arg!r}'): + res = interp.call(defs.spam_returns_arg, arg) + self.assert_funcs_equal(res, arg) + assert is_pickleable(arg) + + for arg in [ + Ellipsis, + NotImplemented, + object(), + 2**1000, + [1, 2, 3], + {'a': 1, 'b': 2}, + types.SimpleNamespace(x=42), + # builtin types + object, + type, + Exception, + ModuleNotFoundError, + # builtin exceptions + Exception('uh-oh!'), + ModuleNotFoundError('mymodule'), + # builtin fnctions + len, + sys.exit, + # user classes + *defs.TOP_CLASSES, + *(c(*a) for c, a in defs.TOP_CLASSES.items() + if c not in defs.CLASSES_WITHOUT_EQUALITY), + ]: + with self.subTest(f'pickleable {arg!r}'): + res = interp.call(defs.spam_returns_arg, arg) + if type(arg) is object: + self.assertIs(type(res), object) + elif isinstance(arg, BaseException): + self.assert_exceptions_equal(res, arg) + else: + self.assertEqual(res, arg) + assert is_pickleable(arg) + + for arg in [ + types.MappingProxyType({}), + *(f for f in defs.NESTED_FUNCTIONS + if f not in defs.STATELESS_FUNCTIONS), + ]: + with self.subTest(f'unpickleable {arg!r}'): + assert not _interpreters.is_shareable(arg) + assert not is_pickleable(arg) + with self.assertRaises(interpreters.NotShareableError): + interp.call(defs.spam_returns_arg, arg) + + def test_full_args(self): + interp = interpreters.create() + expected = (1, 2, 3, 4, 5, 6, ('?',), {'g': 7, 'h': 8}) + func = defs.spam_full_args + res = interp.call(func, 1, 2, 3, 4, '?', e=5, f=6, g=7, h=8) + self.assertEqual(res, expected) + + def test_full_defaults(self): + # pickleable, but not stateless + interp = interpreters.create() + expected = (-1, -2, -3, -4, -5, -6, (), {'g': 8, 'h': 9}) + res = interp.call(defs.spam_full_args_with_defaults, g=8, h=9) + self.assertEqual(res, expected) + + def test_modified_arg(self): + interp = interpreters.create() + script = dedent(""" + a = 7 + b = 2 + c = a ** b + """) + ns = {} + expected = {'a': 7, 'b': 2, 'c': 49} + res = interp.call(call_func_exec_wrapper, script, ns) + obj, resns, resid = res + del resns['__builtins__'] + self.assertIsNone(obj) + self.assertEqual(ns, {}) + self.assertEqual(resns, expected) + self.assertNotEqual(resid, id(ns)) + self.assertNotEqual(resid, id(resns)) + + def test_func_in___main___valid(self): + # pickleable, already there' + + with os_helper.temp_dir() as tempdir: + def new_mod(name, text): + script_helper.make_script(tempdir, name, dedent(text)) + + def run(text): + name = 'myscript' + text = dedent(f""" + import sys + sys.path.insert(0, {tempdir!r}) + + """) + dedent(text) + filename = script_helper.make_script(tempdir, name, text) + res = script_helper.assert_python_ok(filename) + return res.out.decode('utf-8').strip() + + # no module indirection + with self.subTest('no indirection'): + text = run(f""" + from concurrent import interpreters + + def spam(): + # This a global var... + return __name__ + + if __name__ == '__main__': + interp = interpreters.create() + res = interp.call(spam) + print(res) + """) + self.assertEqual(text, '<fake __main__>') + + # indirect as func, direct interp + new_mod('mymod', f""" + def run(interp, func): + return interp.call(func) + """) + with self.subTest('indirect as func, direct interp'): + text = run(f""" + from concurrent import interpreters + import mymod + + def spam(): + # This a global var... + return __name__ + + if __name__ == '__main__': + interp = interpreters.create() + res = mymod.run(interp, spam) + print(res) + """) + self.assertEqual(text, '<fake __main__>') + + # indirect as func, indirect interp + new_mod('mymod', f""" + from concurrent import interpreters + def run(func): + interp = interpreters.create() + return interp.call(func) + """) + with self.subTest('indirect as func, indirect interp'): + text = run(f""" + import mymod + + def spam(): + # This a global var... + return __name__ + + if __name__ == '__main__': + res = mymod.run(spam) + print(res) + """) + self.assertEqual(text, '<fake __main__>') + + def test_func_in___main___invalid(self): + interp = interpreters.create() + + funcname = f'{__name__.replace(".", "_")}_spam_okay' + script = dedent(f""" + def {funcname}(): + # This a global var... + return __name__ + """) + + with self.subTest('pickleable, added dynamically'): + with defined_in___main__(funcname, script) as arg: + with self.assertRaises(interpreters.NotShareableError): + interp.call(defs.spam_returns_arg, arg) + + with self.subTest('lying about __main__'): + with defined_in___main__(funcname, script, remove=True) as arg: + with self.assertRaises(interpreters.NotShareableError): + interp.call(defs.spam_returns_arg, arg) + + def test_func_in___main___hidden(self): + # When a top-level function that uses global variables is called + # through Interpreter.call(), it will be pickled, sent over, + # and unpickled. That requires that it be found in the other + # interpreter's __main__ module. However, the original script + # that defined the function is only run in the main interpreter, + # so pickle.loads() would normally fail. + # + # We work around this by running the script in the other + # interpreter. However, this is a one-off solution for the sake + # of unpickling, so we avoid modifying that interpreter's + # __main__ module by running the script in a hidden module. + # + # In this test we verify that the function runs with the hidden + # module as its __globals__ when called in the other interpreter, + # and that the interpreter's __main__ module is unaffected. + text = dedent(""" + eggs = True + + def spam(*, explicit=False): + if explicit: + import __main__ + ns = __main__.__dict__ + else: + # For now we have to have a LOAD_GLOBAL in the + # function in order for globals() to actually return + # spam.__globals__. Maybe it doesn't go through pickle? + # XXX We will fix this later. + spam + ns = globals() + + func = ns.get('spam') + return [ + id(ns), + ns.get('__name__'), + ns.get('__file__'), + id(func), + None if func is None else repr(func), + ns.get('eggs'), + ns.get('ham'), + ] + + if __name__ == "__main__": + from concurrent import interpreters + interp = interpreters.create() + + ham = True + print([ + [ + spam(explicit=True), + spam(), + ], + [ + interp.call(spam, explicit=True), + interp.call(spam), + ], + ]) + """) + with os_helper.temp_dir() as tempdir: + filename = script_helper.make_script(tempdir, 'my-script', text) + res = script_helper.assert_python_ok(filename) + stdout = res.out.decode('utf-8').strip() + local, remote = eval(stdout) + + # In the main interpreter. + main, unpickled = local + nsid, _, _, funcid, func, _, _ = main + self.assertEqual(main, [ + nsid, + '__main__', + filename, + funcid, + func, + True, + True, + ]) + self.assertIsNot(func, None) + self.assertRegex(func, '^<function spam at 0x.*>$') + self.assertEqual(unpickled, main) + + # In the subinterpreter. + main, unpickled = remote + nsid1, _, _, funcid1, _, _, _ = main + self.assertEqual(main, [ + nsid1, + '__main__', + None, + funcid1, + None, + None, + None, + ]) + nsid2, _, _, funcid2, func, _, _ = unpickled + self.assertEqual(unpickled, [ + nsid2, + '<fake __main__>', + filename, + funcid2, + func, + True, + None, + ]) + self.assertIsNot(func, None) + self.assertRegex(func, '^<function spam at 0x.*>$') + self.assertNotEqual(nsid2, nsid1) + self.assertNotEqual(funcid2, funcid1) + + def test_func_in___main___uses_globals(self): + # See the note in test_func_in___main___hidden about pickle + # and the __main__ module. + # + # Additionally, the solution to that problem must provide + # for global variables on which a pickled function might rely. + # + # To check that, we run a script that has two global functions + # and a global variable in the __main__ module. One of the + # functions sets the global variable and the other returns + # the value. + # + # The script calls those functions multiple times in another + # interpreter, to verify the following: + # + # * the global variable is properly initialized + # * the global variable retains state between calls + # * the setter modifies that persistent variable + # * the getter uses the variable + # * the calls in the other interpreter do not modify + # the main interpreter + # * those calls don't modify the interpreter's __main__ module + # * the functions and variable do not actually show up in the + # other interpreter's __main__ module + text = dedent(""" + count = 0 + + def inc(x=1): + global count + count += x + + def get_count(): + return count + + if __name__ == "__main__": + counts = [] + results = [count, counts] + + from concurrent import interpreters + interp = interpreters.create() + + val = interp.call(get_count) + counts.append(val) + + interp.call(inc) + val = interp.call(get_count) + counts.append(val) + + interp.call(inc, 3) + val = interp.call(get_count) + counts.append(val) + + results.append(count) + + modified = {name: interp.call(eval, f'{name!r} in vars()') + for name in ('count', 'inc', 'get_count')} + results.append(modified) + + print(results) + """) + with os_helper.temp_dir() as tempdir: + filename = script_helper.make_script(tempdir, 'my-script', text) + res = script_helper.assert_python_ok(filename) + stdout = res.out.decode('utf-8').strip() + before, counts, after, modified = eval(stdout) + self.assertEqual(modified, { + 'count': False, + 'inc': False, + 'get_count': False, + }) + self.assertEqual(before, 0) + self.assertEqual(after, 0) + self.assertEqual(counts, [0, 1, 4]) + + def test_raises(self): + interp = interpreters.create() + with self.assertRaises(ExecutionFailed): + interp.call(call_func_failure) + + with self.assert_fails(ValueError): + interp.call(call_func_complex, '???', exc=ValueError('spam')) + + def test_call_valid(self): + interp = interpreters.create() + + for i, (callable, args, kwargs, expected) in enumerate([ + (call_func_noop, (), {}, None), + (call_func_ident, ('spamspamspam',), {}, 'spamspamspam'), + (call_func_return_shareable, (), {}, (1, None)), + (call_func_return_pickleable, (), {}, [1, 2, 3]), + (Spam.noop, (), {}, None), + (Spam.from_values, (), {}, Spam(())), + (Spam.from_values, (1, 2, 3), {}, Spam((1, 2, 3))), + (Spam, ('???',), {}, Spam('???')), + (Spam(101), (), {}, (101, (), {})), + (Spam(10101).run, (), {}, (10101, (), {})), + (call_func_complex, ('ident', 'spam'), {}, 'spam'), + (call_func_complex, ('full-ident', 'spam'), {}, ('spam', (), {})), + (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}, + ('spam', ('ham',), {'eggs': '!!!'})), + (call_func_complex, ('globals',), {}, __name__), + (call_func_complex, ('interpid',), {}, interp.id), + (call_func_complex, ('custom', 'spam!'), {}, Spam('spam!')), ]): with self.subTest(f'success case #{i+1}'): - res = interp.call(callable) - self.assertIs(res, None) + res = interp.call(callable, *args, **kwargs) + self.assertEqual(res, expected) + + def test_call_invalid(self): + interp = interpreters.create() + + func = get_call_func_closure + with self.subTest(func): + with self.assertRaises(interpreters.NotShareableError): + interp.call(func, 42) + + func = get_call_func_closure(42) + with self.subTest(func): + with self.assertRaises(interpreters.NotShareableError): + interp.call(func) + + func = call_func_complex + op = 'closure' + with self.subTest(f'{func} ({op})'): + with self.assertRaises(interpreters.NotShareableError): + interp.call(func, op, value='~~~') + + op = 'custom-inner' + with self.subTest(f'{func} ({op})'): + with self.assertRaises(interpreters.NotShareableError): + interp.call(func, op, 'eggs!') + + def test_callable_requires_frame(self): + # There are various functions that require a current frame. + interp = interpreters.create() + for call, expected in [ + ((eval, '[1, 2, 3]'), + [1, 2, 3]), + ((eval, 'sum([1, 2, 3])'), + 6), + ((exec, '...'), + None), + ]: + with self.subTest(str(call)): + res = interp.call(*call) + self.assertEqual(res, expected) + + result_not_pickleable = [ + globals, + locals, + vars, + ] + for func, expectedtype in { + globals: dict, + locals: dict, + vars: dict, + dir: list, + }.items(): + with self.subTest(str(func)): + if func in result_not_pickleable: + with self.assertRaises(interpreters.NotShareableError): + interp.call(func) + else: + res = interp.call(func) + self.assertIsInstance(res, expectedtype) + self.assertIn('__builtins__', res) + + def test_globals_from_builtins(self): + # The builtins exec(), eval(), globals(), locals(), vars(), + # and dir() each runs relative to the target interpreter's + # __main__ module, when called directly. However, + # globals(), locals(), and vars() don't work when called + # directly so we don't check them. + from _frozen_importlib import BuiltinImporter + interp = interpreters.create() + + names = interp.call(dir) + self.assertEqual(names, [ + '__builtins__', + '__doc__', + '__loader__', + '__name__', + '__package__', + '__spec__', + ]) + + values = {name: interp.call(eval, name) + for name in names if name != '__builtins__'} + self.assertEqual(values, { + '__name__': '__main__', + '__doc__': None, + '__spec__': None, # It wasn't imported, so no module spec? + '__package__': None, + '__loader__': BuiltinImporter, + }) + with self.assertRaises(ExecutionFailed): + interp.call(eval, 'spam'), + + interp.call(exec, f'assert dir() == {names}') + + # Update the interpreter's __main__. + interp.prepare_main(spam=42) + expected = names + ['spam'] + + names = interp.call(dir) + self.assertEqual(names, expected) + + value = interp.call(eval, 'spam') + self.assertEqual(value, 42) + + interp.call(exec, f'assert dir() == {expected}, dir()') + + def test_globals_from_stateless_func(self): + # A stateless func, which doesn't depend on any globals, + # doesn't go through pickle, so it runs in __main__. + def set_global(name, value): + globals()[name] = value + + def get_global(name): + return globals().get(name) + + interp = interpreters.create() + + modname = interp.call(get_global, '__name__') + self.assertEqual(modname, '__main__') + + res = interp.call(get_global, 'spam') + self.assertIsNone(res) + + interp.exec('spam = True') + res = interp.call(get_global, 'spam') + self.assertTrue(res) + + interp.call(set_global, 'spam', 42) + res = interp.call(get_global, 'spam') + self.assertEqual(res, 42) + + interp.exec('assert spam == 42, repr(spam)') + + def test_call_in_thread(self): + interp = interpreters.create() for i, (callable, args, kwargs) in enumerate([ - (call_func_ident, ('spamspamspam',), {}), - (get_call_func_closure, (42,), {}), - (get_call_func_closure(42), (), {}), + (call_func_noop, (), {}), + (call_func_return_shareable, (), {}), + (call_func_return_pickleable, (), {}), (Spam.from_values, (), {}), (Spam.from_values, (1, 2, 3), {}), - (Spam, ('???'), {}), (Spam(101), (), {}), (Spam(10101).run, (), {}), + (Spam.noop, (), {}), (call_func_complex, ('ident', 'spam'), {}), (call_func_complex, ('full-ident', 'spam'), {}), (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}), (call_func_complex, ('globals',), {}), (call_func_complex, ('interpid',), {}), - (call_func_complex, ('closure',), {'value': '~~~'}), (call_func_complex, ('custom', 'spam!'), {}), - (call_func_complex, ('custom-inner', 'eggs!'), {}), - (call_func_complex, ('???',), {'exc': ValueError('spam')}), - ]): - with self.subTest(f'invalid case #{i+1}'): - with self.assertRaises(Exception): - if args or kwargs: - raise Exception((args, kwargs)) - interp.call(callable) - - with self.assertRaises(ExecutionFailed): - interp.call(call_func_failure) - - def test_call_in_thread(self): - interp = interpreters.create() - - for i, (callable, args, kwargs) in enumerate([ - (call_func_noop, (), {}), - (call_func_return_shareable, (), {}), - (call_func_return_not_shareable, (), {}), - (Spam.noop, (), {}), ]): with self.subTest(f'success case #{i+1}'): with self.captured_thread_exception() as ctx: - t = interp.call_in_thread(callable) + t = interp.call_in_thread(callable, *args, **kwargs) t.join() self.assertIsNone(ctx.caught) for i, (callable, args, kwargs) in enumerate([ - (call_func_ident, ('spamspamspam',), {}), (get_call_func_closure, (42,), {}), (get_call_func_closure(42), (), {}), - (Spam.from_values, (), {}), - (Spam.from_values, (1, 2, 3), {}), - (Spam, ('???'), {}), - (Spam(101), (), {}), - (Spam(10101).run, (), {}), - (call_func_complex, ('ident', 'spam'), {}), - (call_func_complex, ('full-ident', 'spam'), {}), - (call_func_complex, ('full-ident', 'spam', 'ham'), {'eggs': '!!!'}), - (call_func_complex, ('globals',), {}), - (call_func_complex, ('interpid',), {}), - (call_func_complex, ('closure',), {'value': '~~~'}), - (call_func_complex, ('custom', 'spam!'), {}), - (call_func_complex, ('custom-inner', 'eggs!'), {}), - (call_func_complex, ('???',), {'exc': ValueError('spam')}), ]): with self.subTest(f'invalid case #{i+1}'): - if args or kwargs: - continue with self.captured_thread_exception() as ctx: - t = interp.call_in_thread(callable) + t = interp.call_in_thread(callable, *args, **kwargs) t.join() self.assertIsNotNone(ctx.caught) @@ -1452,6 +2201,14 @@ def test_destroy(self): self.assertFalse( self.interp_exists(interpid)) + with self.subTest('basic C-API'): + interpid = _testinternalcapi.create_interpreter() + self.assertTrue( + self.interp_exists(interpid)) + _testinternalcapi.destroy_interpreter(interpid, basic=True) + self.assertFalse( + self.interp_exists(interpid)) + def test_get_config(self): # This test overlaps with # test.test_capi.test_misc.InterpreterConfigTests. @@ -1529,6 +2286,16 @@ def test_whence(self): whence = eval(text) self.assertEqual(whence, _interpreters.WHENCE_LEGACY_CAPI) + def test_contextvars_missing(self): + script = f""" + import contextvars + print(getattr(contextvars.Token, "MISSING", "'doesn't exist'")) + """ + + orig = _interpreters.create() + text = self.run_and_capture(orig, script) + self.assertEqual(text.strip(), "<Token.MISSING>") + def test_is_running(self): def check(interpid, expected): with self.assertRaisesRegex(InterpreterError, 'unrecognized'): @@ -1585,18 +2352,14 @@ def test_exec(self): with results: exc = _interpreters.exec(interpid, script) out = results.stdout() - self.assertEqual(out, '') - self.assert_ns_equal(exc, types.SimpleNamespace( - type=types.SimpleNamespace( - __name__='Exception', - __qualname__='Exception', - __module__='builtins', - ), - msg='uh-oh!', + expected = build_excinfo( + Exception, 'uh-oh!', # We check these in other tests. formatted=exc.formatted, errdisplay=exc.errdisplay, - )) + ) + self.assertEqual(out, '') + self.assert_ns_equal(exc, expected) with self.subTest('from C-API'): with self.interpreter_from_capi() as interpid: @@ -1608,25 +2371,50 @@ def test_exec(self): self.assertEqual(exc.msg, 'it worked!') def test_call(self): - with self.subTest('no args'): - interpid = _interpreters.create() - exc = _interpreters.call(interpid, call_func_return_shareable) - self.assertIs(exc, None) + interpid = _interpreters.create() + + # Here we focus on basic args and return values. + # See TestInterpreterCall for full operational coverage, + # including supported callables. + + with self.subTest('no args, return None'): + func = defs.spam_minimal + res, exc = _interpreters.call(interpid, func) + self.assertIsNone(exc) + self.assertIsNone(res) + + with self.subTest('empty args, return None'): + func = defs.spam_minimal + res, exc = _interpreters.call(interpid, func, (), {}) + self.assertIsNone(exc) + self.assertIsNone(res) + + with self.subTest('no args, return non-None'): + func = defs.script_with_return + res, exc = _interpreters.call(interpid, func) + self.assertIsNone(exc) + self.assertIs(res, True) + + with self.subTest('full args, return non-None'): + expected = (1, 2, 3, 4, 5, 6, (7, 8), {'g': 9, 'h': 0}) + func = defs.spam_full_args + args = (1, 2, 3, 4, 7, 8) + kwargs = dict(e=5, f=6, g=9, h=0) + res, exc = _interpreters.call(interpid, func, args, kwargs) + self.assertIsNone(exc) + self.assertEqual(res, expected) with self.subTest('uncaught exception'): - interpid = _interpreters.create() - exc = _interpreters.call(interpid, call_func_failure) - self.assertEqual(exc, types.SimpleNamespace( - type=types.SimpleNamespace( - __name__='Exception', - __qualname__='Exception', - __module__='builtins', - ), - msg='spam!', + func = defs.spam_raises + res, exc = _interpreters.call(interpid, func) + expected = build_excinfo( + Exception, 'spam!', # We check these in other tests. formatted=exc.formatted, errdisplay=exc.errdisplay, - )) + ) + self.assertIsNone(res) + self.assertEqual(exc, expected) @requires_test_modules def test_set___main___attrs(self): diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py index eada18f99d04db0..5437792b5a7014a 100644 --- a/Lib/test/test_interpreters/test_channels.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -8,8 +8,8 @@ from test.support import import_helper # Raise SkipTest if subinterpreters not supported. _channels = import_helper.import_module('_interpchannels') -from test.support import interpreters -from test.support.interpreters import channels +from concurrent import interpreters +from test.support import channels from .utils import _run_output, TestBase @@ -47,6 +47,12 @@ def test_list_all(self): after = set(channels.list_all()) self.assertEqual(after, created) + def test_list_all_closed(self): + created = [channels.create() for _ in range(3)] + rch, sch = created.pop(1) + rch.close() + self.assertEqual(set(channels.list_all()), set(created)) + def test_shareable(self): interp = interpreters.create() rch, sch = channels.create() @@ -121,9 +127,11 @@ def test_equality(self): def test_pickle(self): ch, _ = channels.create() - data = pickle.dumps(ch) - unpickled = pickle.loads(data) - self.assertEqual(unpickled, ch) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol): + data = pickle.dumps(ch, protocol) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, ch) class TestSendChannelAttrs(TestBase): @@ -152,9 +160,11 @@ def test_equality(self): def test_pickle(self): _, ch = channels.create() - data = pickle.dumps(ch) - unpickled = pickle.loads(data) - self.assertEqual(unpickled, ch) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol): + data = pickle.dumps(ch, protocol) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, ch) class TestSendRecv(TestBase): @@ -171,7 +181,7 @@ def test_send_recv_main(self): def test_send_recv_same_interpreter(self): interp = interpreters.create() interp.exec(dedent(""" - from test.support.interpreters import channels + from test.support import channels r, s = channels.create() orig = b'spam' s.send_nowait(orig) @@ -244,7 +254,7 @@ def test_send_recv_nowait_main_with_default(self): def test_send_recv_nowait_same_interpreter(self): interp = interpreters.create() interp.exec(dedent(""" - from test.support.interpreters import channels + from test.support import channels r, s = channels.create() orig = b'spam' s.send_nowait(orig) @@ -377,17 +387,17 @@ def common(rch, sch, unbound=None, presize=0): if not unbound: extraargs = '' elif unbound is channels.UNBOUND: - extraargs = ', unbound=channels.UNBOUND' + extraargs = ', unbounditems=channels.UNBOUND' elif unbound is channels.UNBOUND_ERROR: - extraargs = ', unbound=channels.UNBOUND_ERROR' + extraargs = ', unbounditems=channels.UNBOUND_ERROR' elif unbound is channels.UNBOUND_REMOVE: - extraargs = ', unbound=channels.UNBOUND_REMOVE' + extraargs = ', unbounditems=channels.UNBOUND_REMOVE' else: raise NotImplementedError(repr(unbound)) interp = interpreters.create() _run_output(interp, dedent(f""" - from test.support.interpreters import channels + from test.support import channels sch = channels.SendChannel({sch.id}) obj1 = b'spam' obj2 = b'eggs' @@ -454,11 +464,11 @@ def common(rch, sch, unbound=None, presize=0): with self.assertRaises(channels.ChannelEmptyError): rch.recv_nowait() - sch.send_nowait(b'ham', unbound=channels.UNBOUND_REMOVE) + sch.send_nowait(b'ham', unbounditems=channels.UNBOUND_REMOVE) self.assertEqual(_channels.get_count(rch.id), 1) interp = common(rch, sch, channels.UNBOUND_REMOVE, 1) self.assertEqual(_channels.get_count(rch.id), 3) - sch.send_nowait(42, unbound=channels.UNBOUND_REMOVE) + sch.send_nowait(42, unbounditems=channels.UNBOUND_REMOVE) self.assertEqual(_channels.get_count(rch.id), 4) del interp self.assertEqual(_channels.get_count(rch.id), 2) @@ -482,13 +492,13 @@ def test_send_cleared_with_subinterpreter_mixed(self): self.assertEqual(_channels.get_count(rch.id), 0) _run_output(interp, dedent(f""" - from test.support.interpreters import channels + from test.support import channels sch = channels.SendChannel({sch.id}) - sch.send_nowait(1, unbound=channels.UNBOUND) - sch.send_nowait(2, unbound=channels.UNBOUND_ERROR) + sch.send_nowait(1, unbounditems=channels.UNBOUND) + sch.send_nowait(2, unbounditems=channels.UNBOUND_ERROR) sch.send_nowait(3) - sch.send_nowait(4, unbound=channels.UNBOUND_REMOVE) - sch.send_nowait(5, unbound=channels.UNBOUND) + sch.send_nowait(4, unbounditems=channels.UNBOUND_REMOVE) + sch.send_nowait(5, unbounditems=channels.UNBOUND) """)) self.assertEqual(_channels.get_count(rch.id), 5) @@ -518,15 +528,15 @@ def test_send_cleared_with_subinterpreter_multiple(self): sch.send_nowait(1) _run_output(interp1, dedent(f""" - from test.support.interpreters import channels + from test.support import channels rch = channels.RecvChannel({rch.id}) sch = channels.SendChannel({sch.id}) obj1 = rch.recv() - sch.send_nowait(2, unbound=channels.UNBOUND) - sch.send_nowait(obj1, unbound=channels.UNBOUND_REMOVE) + sch.send_nowait(2, unbounditems=channels.UNBOUND) + sch.send_nowait(obj1, unbounditems=channels.UNBOUND_REMOVE) """)) _run_output(interp2, dedent(f""" - from test.support.interpreters import channels + from test.support import channels rch = channels.RecvChannel({rch.id}) sch = channels.SendChannel({sch.id}) obj2 = rch.recv() @@ -535,21 +545,21 @@ def test_send_cleared_with_subinterpreter_multiple(self): self.assertEqual(_channels.get_count(rch.id), 0) sch.send_nowait(3) _run_output(interp1, dedent(""" - sch.send_nowait(4, unbound=channels.UNBOUND) + sch.send_nowait(4, unbounditems=channels.UNBOUND) # interp closed here - sch.send_nowait(5, unbound=channels.UNBOUND_REMOVE) - sch.send_nowait(6, unbound=channels.UNBOUND) + sch.send_nowait(5, unbounditems=channels.UNBOUND_REMOVE) + sch.send_nowait(6, unbounditems=channels.UNBOUND) """)) _run_output(interp2, dedent(""" - sch.send_nowait(7, unbound=channels.UNBOUND_ERROR) + sch.send_nowait(7, unbounditems=channels.UNBOUND_ERROR) # interp closed here - sch.send_nowait(obj1, unbound=channels.UNBOUND_ERROR) - sch.send_nowait(obj2, unbound=channels.UNBOUND_REMOVE) - sch.send_nowait(8, unbound=channels.UNBOUND) + sch.send_nowait(obj1, unbounditems=channels.UNBOUND_ERROR) + sch.send_nowait(obj2, unbounditems=channels.UNBOUND_REMOVE) + sch.send_nowait(8, unbounditems=channels.UNBOUND) """)) _run_output(interp1, dedent(""" - sch.send_nowait(9, unbound=channels.UNBOUND_REMOVE) - sch.send_nowait(10, unbound=channels.UNBOUND) + sch.send_nowait(9, unbounditems=channels.UNBOUND_REMOVE) + sch.send_nowait(10, unbounditems=channels.UNBOUND) """)) self.assertEqual(_channels.get_count(rch.id), 10) diff --git a/Lib/test/test_interpreters/test_lifecycle.py b/Lib/test/test_interpreters/test_lifecycle.py index ac24f6568acd952..39c1441b67c133f 100644 --- a/Lib/test/test_interpreters/test_lifecycle.py +++ b/Lib/test/test_interpreters/test_lifecycle.py @@ -119,7 +119,7 @@ def test_sys_path_0(self): # The main interpreter's sys.path[0] should be used by subinterpreters. script = ''' import sys - from test.support import interpreters + from concurrent import interpreters orig = sys.path[0] @@ -132,6 +132,7 @@ def test_sys_path_0(self): 'sub': sys.path[0], }}, indent=4), flush=True) """) + interp.close() ''' # <tmp>/ # pkg/ @@ -170,9 +171,12 @@ def test_gh_109793(self): # is reported, even when subinterpreters get cleaned up at the end. import subprocess argv = [sys.executable, '-c', '''if True: - from test.support import interpreters + from concurrent import interpreters interp = interpreters.create() - raise Exception + try: + raise Exception + finally: + interp.close() '''] proc = subprocess.run(argv, capture_output=True, text=True) self.assertIn('Traceback', proc.stderr) diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 18f83d097eb3604..8c6a0efbb920d82 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -7,11 +7,12 @@ from test.support import import_helper, Py_DEBUG # Raise SkipTest if subinterpreters not supported. _queues = import_helper.import_module('_interpqueues') -from test.support import interpreters -from test.support.interpreters import queues, _crossinterp +from concurrent import interpreters +from concurrent.interpreters import _queues as queues, _crossinterp +import test._crossinterp_definitions as defs from .utils import _run_output, TestBase as _TestBase - +HUGE_TIMEOUT = 3600 REPLACE = _crossinterp._UNBOUND_CONSTANT_TO_FLAG[_crossinterp.UNBOUND] @@ -42,7 +43,7 @@ def test_highlevel_reloaded(self): importlib.reload(queues) def test_create_destroy(self): - qid = _queues.create(2, 0, REPLACE) + qid = _queues.create(2, REPLACE, -1) _queues.destroy(qid) self.assertEqual(get_num_queues(), 0) with self.assertRaises(queues.QueueNotFoundError): @@ -56,7 +57,7 @@ def test_not_destroyed(self): '-c', dedent(f""" import {_queues.__name__} as _queues - _queues.create(2, 0, {REPLACE}) + _queues.create(2, {REPLACE}, -1) """), ) self.assertEqual(stdout, '') @@ -67,13 +68,13 @@ def test_not_destroyed(self): def test_bind_release(self): with self.subTest('typical'): - qid = _queues.create(2, 0, REPLACE) + qid = _queues.create(2, REPLACE, -1) _queues.bind(qid) _queues.release(qid) self.assertEqual(get_num_queues(), 0) with self.subTest('bind too much'): - qid = _queues.create(2, 0, REPLACE) + qid = _queues.create(2, REPLACE, -1) _queues.bind(qid) _queues.bind(qid) _queues.release(qid) @@ -81,7 +82,7 @@ def test_bind_release(self): self.assertEqual(get_num_queues(), 0) with self.subTest('nested'): - qid = _queues.create(2, 0, REPLACE) + qid = _queues.create(2, REPLACE, -1) _queues.bind(qid) _queues.bind(qid) _queues.release(qid) @@ -89,7 +90,7 @@ def test_bind_release(self): self.assertEqual(get_num_queues(), 0) with self.subTest('release without binding'): - qid = _queues.create(2, 0, REPLACE) + qid = _queues.create(2, REPLACE, -1) with self.assertRaises(queues.QueueError): _queues.release(qid) @@ -126,19 +127,19 @@ def test_shareable(self): interp = interpreters.create() interp.exec(dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue1 = queues.Queue({queue1.id}) """)); with self.subTest('same interpreter'): queue2 = queues.create() - queue1.put(queue2, syncobj=True) + queue1.put(queue2) queue3 = queue1.get() self.assertIs(queue3, queue2) with self.subTest('from current interpreter'): queue4 = queues.create() - queue1.put(queue4, syncobj=True) + queue1.put(queue4) out = _run_output(interp, dedent(""" queue4 = queue1.get() print(queue4.id) @@ -149,7 +150,7 @@ def test_shareable(self): with self.subTest('from subinterpreter'): out = _run_output(interp, dedent(""" queue5 = queues.create() - queue1.put(queue5, syncobj=True) + queue1.put(queue5) print(queue5.id) """)) qid = int(out) @@ -188,9 +189,11 @@ def test_equality(self): def test_pickle(self): queue = queues.create() - data = pickle.dumps(queue) - unpickled = pickle.loads(data) - self.assertEqual(unpickled, queue) + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=protocol): + data = pickle.dumps(queue, protocol) + unpickled = pickle.loads(data) + self.assertEqual(unpickled, queue) class TestQueueOps(TestBase): @@ -198,7 +201,7 @@ class TestQueueOps(TestBase): def test_empty(self): queue = queues.create() before = queue.empty() - queue.put(None, syncobj=True) + queue.put(None) during = queue.empty() queue.get() after = queue.empty() @@ -208,18 +211,64 @@ def test_empty(self): self.assertIs(after, True) def test_full(self): - expected = [False, False, False, True, False, False, False] - actual = [] - queue = queues.create(3) - for _ in range(3): - actual.append(queue.full()) - queue.put(None, syncobj=True) - actual.append(queue.full()) - for _ in range(3): - queue.get() - actual.append(queue.full()) + for maxsize in [1, 3, 11]: + with self.subTest(f'maxsize={maxsize}'): + num_to_add = maxsize + expected = [False] * (num_to_add * 2 + 3) + expected[maxsize] = True + expected[maxsize + 1] = True + + queue = queues.create(maxsize) + actual = [] + empty = [queue.empty()] + + for _ in range(num_to_add): + actual.append(queue.full()) + queue.put_nowait(None) + actual.append(queue.full()) + with self.assertRaises(queues.QueueFull): + queue.put_nowait(None) + empty.append(queue.empty()) + + for _ in range(num_to_add): + actual.append(queue.full()) + queue.get_nowait() + actual.append(queue.full()) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + actual.append(queue.full()) + empty.append(queue.empty()) - self.assertEqual(actual, expected) + self.assertEqual(actual, expected) + self.assertEqual(empty, [True, False, True]) + + # no max size + for args in [(), (0,), (-1,), (-10,)]: + with self.subTest(f'maxsize={args[0]}' if args else '<default>'): + num_to_add = 13 + expected = [False] * (num_to_add * 2 + 3) + + queue = queues.create(*args) + actual = [] + empty = [queue.empty()] + + for _ in range(num_to_add): + actual.append(queue.full()) + queue.put_nowait(None) + actual.append(queue.full()) + empty.append(queue.empty()) + + for _ in range(num_to_add): + actual.append(queue.full()) + queue.get_nowait() + actual.append(queue.full()) + with self.assertRaises(queues.QueueEmpty): + queue.get_nowait() + actual.append(queue.full()) + empty.append(queue.empty()) + + self.assertEqual(actual, expected) + self.assertEqual(empty, [True, False, True]) def test_qsize(self): expected = [0, 1, 2, 3, 2, 3, 2, 1, 0, 1, 0] @@ -227,16 +276,16 @@ def test_qsize(self): queue = queues.create() for _ in range(3): actual.append(queue.qsize()) - queue.put(None, syncobj=True) + queue.put(None) actual.append(queue.qsize()) queue.get() actual.append(queue.qsize()) - queue.put(None, syncobj=True) + queue.put(None) actual.append(queue.qsize()) for _ in range(3): queue.get() actual.append(queue.qsize()) - queue.put(None, syncobj=True) + queue.put(None) actual.append(queue.qsize()) queue.get() actual.append(queue.qsize()) @@ -245,70 +294,38 @@ def test_qsize(self): def test_put_get_main(self): expected = list(range(20)) - for syncobj in (True, False): - kwds = dict(syncobj=syncobj) - with self.subTest(f'syncobj={syncobj}'): - queue = queues.create() - for i in range(20): - queue.put(i, **kwds) - actual = [queue.get() for _ in range(20)] + queue = queues.create() + for i in range(20): + queue.put(i) + actual = [queue.get() for _ in range(20)] - self.assertEqual(actual, expected) + self.assertEqual(actual, expected) def test_put_timeout(self): - for syncobj in (True, False): - kwds = dict(syncobj=syncobj) - with self.subTest(f'syncobj={syncobj}'): - queue = queues.create(2) - queue.put(None, **kwds) - queue.put(None, **kwds) - with self.assertRaises(queues.QueueFull): - queue.put(None, timeout=0.1, **kwds) - queue.get() - queue.put(None, **kwds) + queue = queues.create(2) + queue.put(None) + queue.put(None) + with self.assertRaises(queues.QueueFull): + queue.put(None, timeout=0.1) + with self.assertRaises(queues.QueueFull): + queue.put(None, HUGE_TIMEOUT, 0.1) + queue.get() + queue.put(None) def test_put_nowait(self): - for syncobj in (True, False): - kwds = dict(syncobj=syncobj) - with self.subTest(f'syncobj={syncobj}'): - queue = queues.create(2) - queue.put_nowait(None, **kwds) - queue.put_nowait(None, **kwds) - with self.assertRaises(queues.QueueFull): - queue.put_nowait(None, **kwds) - queue.get() - queue.put_nowait(None, **kwds) - - def test_put_syncobj(self): - for obj in [ - None, - True, - 10, - 'spam', - b'spam', - (0, 'a'), - ]: - with self.subTest(repr(obj)): - queue = queues.create() - - queue.put(obj, syncobj=True) - obj2 = queue.get() - self.assertEqual(obj2, obj) - - queue.put(obj, syncobj=True) - obj2 = queue.get_nowait() - self.assertEqual(obj2, obj) - - for obj in [ - [1, 2, 3], - {'a': 13, 'b': 17}, - ]: - with self.subTest(repr(obj)): - queue = queues.create() - with self.assertRaises(interpreters.NotShareableError): - queue.put(obj, syncobj=True) + queue = queues.create(2) + queue.put_nowait(None) + queue.put_nowait(None) + with self.assertRaises(queues.QueueFull): + queue.put_nowait(None) + with self.assertRaises(queues.QueueFull): + queue.put(None, False) + with self.assertRaises(queues.QueueFull): + queue.put(None, False, timeout=HUGE_TIMEOUT) + queue.get() + queue.put_nowait(None) - def test_put_not_syncobj(self): + def test_put_full_fallback(self): for obj in [ None, True, @@ -323,11 +340,11 @@ def test_put_not_syncobj(self): with self.subTest(repr(obj)): queue = queues.create() - queue.put(obj, syncobj=False) + queue.put(obj) obj2 = queue.get() self.assertEqual(obj2, obj) - queue.put(obj, syncobj=False) + queue.put(obj) obj2 = queue.get_nowait() self.assertEqual(obj2, obj) @@ -335,30 +352,21 @@ def test_get_timeout(self): queue = queues.create() with self.assertRaises(queues.QueueEmpty): queue.get(timeout=0.1) + with self.assertRaises(queues.QueueEmpty): + queue.get(HUGE_TIMEOUT, 0.1) def test_get_nowait(self): queue = queues.create() with self.assertRaises(queues.QueueEmpty): queue.get_nowait() + with self.assertRaises(queues.QueueEmpty): + queue.get(False) + with self.assertRaises(queues.QueueEmpty): + queue.get(False, timeout=HUGE_TIMEOUT) - def test_put_get_default_syncobj(self): - expected = list(range(20)) - queue = queues.create(syncobj=True) - for methname in ('get', 'get_nowait'): - with self.subTest(f'{methname}()'): - get = getattr(queue, methname) - for i in range(20): - queue.put(i) - actual = [get() for _ in range(20)] - self.assertEqual(actual, expected) - - obj = [1, 2, 3] # lists are not shareable - with self.assertRaises(interpreters.NotShareableError): - queue.put(obj) - - def test_put_get_default_not_syncobj(self): + def test_put_get_full_fallback(self): expected = list(range(20)) - queue = queues.create(syncobj=False) + queue = queues.create() for methname in ('get', 'get_nowait'): with self.subTest(f'{methname}()'): get = getattr(queue, methname) @@ -377,14 +385,14 @@ def test_put_get_default_not_syncobj(self): def test_put_get_same_interpreter(self): interp = interpreters.create() interp.exec(dedent(""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue = queues.create() """)) for methname in ('get', 'get_nowait'): with self.subTest(f'{methname}()'): interp.exec(dedent(f""" orig = b'spam' - queue.put(orig, syncobj=True) + queue.put(orig) obj = queue.{methname}() assert obj == orig, 'expected: obj == orig' assert obj is not orig, 'expected: obj is not orig' @@ -399,12 +407,12 @@ def test_put_get_different_interpreters(self): for methname in ('get', 'get_nowait'): with self.subTest(f'{methname}()'): obj1 = b'spam' - queue1.put(obj1, syncobj=True) + queue1.put(obj1) out = _run_output( interp, dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue1 = queues.Queue({queue1.id}) queue2 = queues.Queue({queue2.id}) assert queue1.qsize() == 1, 'expected: queue1.qsize() == 1' @@ -416,7 +424,7 @@ def test_put_get_different_interpreters(self): obj2 = b'eggs' print(id(obj2)) assert queue2.qsize() == 0, 'expected: queue2.qsize() == 0' - queue2.put(obj2, syncobj=True) + queue2.put(obj2) assert queue2.qsize() == 1, 'expected: queue2.qsize() == 1' """)) self.assertEqual(len(queues.list_all()), 2) @@ -433,22 +441,22 @@ def common(queue, unbound=None, presize=0): if not unbound: extraargs = '' elif unbound is queues.UNBOUND: - extraargs = ', unbound=queues.UNBOUND' + extraargs = ', unbounditems=queues.UNBOUND' elif unbound is queues.UNBOUND_ERROR: - extraargs = ', unbound=queues.UNBOUND_ERROR' + extraargs = ', unbounditems=queues.UNBOUND_ERROR' elif unbound is queues.UNBOUND_REMOVE: - extraargs = ', unbound=queues.UNBOUND_REMOVE' + extraargs = ', unbounditems=queues.UNBOUND_REMOVE' else: raise NotImplementedError(repr(unbound)) interp = interpreters.create() _run_output(interp, dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue = queues.Queue({queue.id}) obj1 = b'spam' obj2 = b'eggs' - queue.put(obj1, syncobj=True{extraargs}) - queue.put(obj2, syncobj=True{extraargs}) + queue.put(obj1{extraargs}) + queue.put(obj2{extraargs}) """)) self.assertEqual(queue.qsize(), presize + 2) @@ -501,11 +509,11 @@ def common(queue, unbound=None, presize=0): with self.assertRaises(queues.QueueEmpty): queue.get_nowait() - queue.put(b'ham', unbound=queues.UNBOUND_REMOVE) + queue.put(b'ham', unbounditems=queues.UNBOUND_REMOVE) self.assertEqual(queue.qsize(), 1) interp = common(queue, queues.UNBOUND_REMOVE, 1) self.assertEqual(queue.qsize(), 3) - queue.put(42, unbound=queues.UNBOUND_REMOVE) + queue.put(42, unbounditems=queues.UNBOUND_REMOVE) self.assertEqual(queue.qsize(), 4) del interp self.assertEqual(queue.qsize(), 2) @@ -521,13 +529,13 @@ def test_put_cleared_with_subinterpreter_mixed(self): queue = queues.create() interp = interpreters.create() _run_output(interp, dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue = queues.Queue({queue.id}) - queue.put(1, syncobj=True, unbound=queues.UNBOUND) - queue.put(2, syncobj=True, unbound=queues.UNBOUND_ERROR) - queue.put(3, syncobj=True) - queue.put(4, syncobj=True, unbound=queues.UNBOUND_REMOVE) - queue.put(5, syncobj=True, unbound=queues.UNBOUND) + queue.put(1, unbounditems=queues.UNBOUND) + queue.put(2, unbounditems=queues.UNBOUND_ERROR) + queue.put(3) + queue.put(4, unbounditems=queues.UNBOUND_REMOVE) + queue.put(5, unbounditems=queues.UNBOUND) """)) self.assertEqual(queue.qsize(), 5) @@ -555,16 +563,16 @@ def test_put_cleared_with_subinterpreter_multiple(self): interp1 = interpreters.create() interp2 = interpreters.create() - queue.put(1, syncobj=True) + queue.put(1) _run_output(interp1, dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue = queues.Queue({queue.id}) obj1 = queue.get() - queue.put(2, syncobj=True, unbound=queues.UNBOUND) - queue.put(obj1, syncobj=True, unbound=queues.UNBOUND_REMOVE) + queue.put(2, unbounditems=queues.UNBOUND) + queue.put(obj1, unbounditems=queues.UNBOUND_REMOVE) """)) _run_output(interp2, dedent(f""" - from test.support.interpreters import queues + from concurrent.interpreters import _queues as queues queue = queues.Queue({queue.id}) obj2 = queue.get() obj1 = queue.get() @@ -572,21 +580,21 @@ def test_put_cleared_with_subinterpreter_multiple(self): self.assertEqual(queue.qsize(), 0) queue.put(3) _run_output(interp1, dedent(""" - queue.put(4, syncobj=True, unbound=queues.UNBOUND) + queue.put(4, unbounditems=queues.UNBOUND) # interp closed here - queue.put(5, syncobj=True, unbound=queues.UNBOUND_REMOVE) - queue.put(6, syncobj=True, unbound=queues.UNBOUND) + queue.put(5, unbounditems=queues.UNBOUND_REMOVE) + queue.put(6, unbounditems=queues.UNBOUND) """)) _run_output(interp2, dedent(""" - queue.put(7, syncobj=True, unbound=queues.UNBOUND_ERROR) + queue.put(7, unbounditems=queues.UNBOUND_ERROR) # interp closed here - queue.put(obj1, syncobj=True, unbound=queues.UNBOUND_ERROR) - queue.put(obj2, syncobj=True, unbound=queues.UNBOUND_REMOVE) - queue.put(8, syncobj=True, unbound=queues.UNBOUND) + queue.put(obj1, unbounditems=queues.UNBOUND_ERROR) + queue.put(obj2, unbounditems=queues.UNBOUND_REMOVE) + queue.put(8, unbounditems=queues.UNBOUND) """)) _run_output(interp1, dedent(""" - queue.put(9, syncobj=True, unbound=queues.UNBOUND_REMOVE) - queue.put(10, syncobj=True, unbound=queues.UNBOUND) + queue.put(9, unbounditems=queues.UNBOUND_REMOVE) + queue.put(10, unbounditems=queues.UNBOUND) """)) self.assertEqual(queue.qsize(), 10) @@ -642,12 +650,12 @@ def f(): break except queues.QueueEmpty: continue - queue2.put(obj, syncobj=True) + queue2.put(obj) t = threading.Thread(target=f) t.start() orig = b'spam' - queue1.put(orig, syncobj=True) + queue1.put(orig) obj = queue2.get() t.join() diff --git a/Lib/test/test_interpreters/test_stress.py b/Lib/test/test_interpreters/test_stress.py index fae2f38cb5534bc..50d2444a4c72d31 100644 --- a/Lib/test/test_interpreters/test_stress.py +++ b/Lib/test/test_interpreters/test_stress.py @@ -5,8 +5,9 @@ from test.support import import_helper from test.support import threading_helper # Raise SkipTest if subinterpreters not supported. -import_helper.import_module('_interpreters') -from test.support import interpreters +_interpreters = import_helper.import_module('_interpreters') +from concurrent import interpreters +from concurrent.interpreters import InterpreterError from .utils import TestBase @@ -24,6 +25,7 @@ def test_create_many_sequential(self): del alive support.gc_collect() + @threading_helper.requires_working_threading() @support.bigmemtest(size=200, memuse=32*2**20, dry_run=False) def test_create_many_threaded(self, size): alive = [] @@ -74,6 +76,18 @@ def run(): start.set() support.gc_collect() + @support.nomemtest + def test_create_interpreter_no_memory(self): + import _testcapi + + assertion = self.assertRaises(InterpreterError) + try: + _testcapi.set_nomemory(0, 1) + with assertion: + _interpreters.create() + finally: + _testcapi.remove_mem_hooks() + if __name__ == '__main__': # Test needs to be a package, so we can do relative imports. diff --git a/Lib/test/test_interpreters/utils.py b/Lib/test/test_interpreters/utils.py index fc4ad662e03b66b..784c7ebba9191b7 100644 --- a/Lib/test/test_interpreters/utils.py +++ b/Lib/test/test_interpreters/utils.py @@ -1,5 +1,6 @@ from collections import namedtuple import contextlib +import errno import json import logging import os @@ -22,7 +23,7 @@ import _interpreters except ImportError as exc: raise unittest.SkipTest(str(exc)) -from test.support import interpreters +from concurrent import interpreters try: @@ -52,7 +53,7 @@ def _close_file(file): else: os.close(file) except OSError as exc: - if exc.errno != 9: + if exc.errno != errno.EBADF: raise # re-raise # It was closed already. diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 5a8f1949baaa983..57b42fbb65fe107 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -572,7 +572,7 @@ def do_test(test, obj, abilities): for [test, abilities] in tests: with self.subTest(test): if test == pipe_writer and not threading_helper.can_start_thread: - skipTest() + self.skipTest("Need threads") with test() as obj: do_test(test, obj, abilities) @@ -893,6 +893,22 @@ def test_RawIOBase_read(self): self.assertEqual(rawio.read(2), None) self.assertEqual(rawio.read(2), b"") + def test_RawIOBase_read_bounds_checking(self): + # Make sure a `.readinto` call which returns a value outside + # (0, len(buffer)) raises. + class Misbehaved(self.RawIOBase): + def __init__(self, readinto_return) -> None: + self._readinto_return = readinto_return + def readinto(self, b): + return self._readinto_return + + with self.assertRaises(ValueError) as cm: + Misbehaved(2).read(1) + self.assertEqual(str(cm.exception), "readinto returned 2 outside buffer size 1") + for bad_size in (2147483647, sys.maxsize, -1, -1000): + with self.assertRaises(ValueError): + Misbehaved(bad_size).read() + def test_types_have_dict(self): test = ( self.IOBase(), @@ -902,7 +918,7 @@ def test_types_have_dict(self): self.BytesIO() ) for obj in test: - self.assertTrue(hasattr(obj, "__dict__")) + self.assertHasAttr(obj, "__dict__") def test_opener(self): with self.open(os_helper.TESTFN, "w", encoding="utf-8") as f: @@ -918,7 +934,7 @@ def test_bad_opener_negative_1(self): def badopener(fname, flags): return -1 with self.assertRaises(ValueError) as cm: - open('non-existent', 'r', opener=badopener) + self.open('non-existent', 'r', opener=badopener) self.assertEqual(str(cm.exception), 'opener returned -1') def test_bad_opener_other_negative(self): @@ -926,7 +942,7 @@ def test_bad_opener_other_negative(self): def badopener(fname, flags): return -2 with self.assertRaises(ValueError) as cm: - open('non-existent', 'r', opener=badopener) + self.open('non-existent', 'r', opener=badopener) self.assertEqual(str(cm.exception), 'opener returned -2') def test_opener_invalid_fd(self): @@ -1062,6 +1078,37 @@ def flush(self): # Silence destructor error R.flush = lambda self: None + @threading_helper.requires_working_threading() + def test_write_readline_races(self): + # gh-134908: Concurrent iteration over a file caused races + thread_count = 2 + write_count = 100 + read_count = 100 + + def writer(file, barrier): + barrier.wait() + for _ in range(write_count): + file.write("x") + + def reader(file, barrier): + barrier.wait() + for _ in range(read_count): + for line in file: + self.assertEqual(line, "") + + with self.open(os_helper.TESTFN, "w+") as f: + barrier = threading.Barrier(thread_count + 1) + reader = threading.Thread(target=reader, args=(f, barrier)) + writers = [threading.Thread(target=writer, args=(f, barrier)) + for _ in range(thread_count)] + with threading_helper.catch_threading_exception() as cm: + with threading_helper.start_threads(writers + [reader]): + pass + self.assertIsNone(cm.exc_type) + + self.assertEqual(os.stat(os_helper.TESTFN).st_size, + write_count * thread_count) + class CIOTest(IOTest): @@ -1117,7 +1164,7 @@ def test_class_hierarchy(self): def check_subs(types, base): for tp in types: with self.subTest(tp=tp, base=base): - self.assertTrue(issubclass(tp, base)) + self.assertIsSubclass(tp, base) def recursive_check(d): for k, v in d.items(): @@ -1870,7 +1917,7 @@ def test_write_overflow(self): flushed = b"".join(writer._write_stack) # At least (total - 8) bytes were implicitly flushed, perhaps more # depending on the implementation. - self.assertTrue(flushed.startswith(contents[:-8]), flushed) + self.assertStartsWith(flushed, contents[:-8]) def check_writes(self, intermediate_func): # Lots of writes, test the flushed output is as expected. @@ -1940,7 +1987,7 @@ def test_write_non_blocking(self): self.assertEqual(bufio.write(b"ABCDEFGHI"), 9) s = raw.pop_written() # Previously buffered bytes were flushed - self.assertTrue(s.startswith(b"01234567A"), s) + self.assertStartsWith(s, b"01234567A") def test_write_and_rewind(self): raw = self.BytesIO() @@ -2236,7 +2283,7 @@ def test_write(self): def test_peek(self): pair = self.tp(self.BytesIO(b"abcdef"), self.MockRawIO()) - self.assertTrue(pair.peek(3).startswith(b"abc")) + self.assertStartsWith(pair.peek(3), b"abc") self.assertEqual(pair.read(3), b"abc") def test_readable(self): @@ -3290,6 +3337,24 @@ def test_multibyte_seek_and_tell(self): self.assertEqual(f.tell(), p1) f.close() + def test_tell_after_readline_with_cr(self): + # Test for gh-141314: TextIOWrapper.tell() assertion failure + # when dealing with standalone carriage returns + data = b'line1\r' + with self.open(os_helper.TESTFN, "wb") as f: + f.write(data) + + with self.open(os_helper.TESTFN, "r") as f: + # Read line that ends with \r + line = f.readline() + self.assertEqual(line, "line1\n") + # This should not cause an assertion failure + pos = f.tell() + # Verify we can seek back to this position + f.seek(pos) + remaining = f.read() + self.assertEqual(remaining, "") + def test_seek_with_encoder_state(self): f = self.open(os_helper.TESTFN, "w", encoding="euc_jis_2004") f.write("\u00e6\u0300") @@ -4129,6 +4194,22 @@ def write(self, data): self.assertEqual([b"abcdef", b"middle", b"g"*chunk_size], buf._write_stack) + def test_issue142594(self): + wrapper = None + detached = False + class ReentrantRawIO(self.RawIOBase): + @property + def closed(self): + nonlocal detached + if wrapper is not None and not detached: + detached = True + wrapper.detach() + return False + + raw = ReentrantRawIO() + wrapper = self.TextIOWrapper(raw) + wrapper.close() # should not crash + class PyTextIOWrapperTest(TextIOWrapperTest): io = pyio @@ -4417,7 +4498,7 @@ def test_abc_inheritance_official(self): self._check_abc_inheritance(io) def _check_warn_on_dealloc(self, *args, **kwargs): - f = open(*args, **kwargs) + f = self.open(*args, **kwargs) r = repr(f) with self.assertWarns(ResourceWarning) as cm: f = None @@ -4446,7 +4527,7 @@ def cleanup_fds(): r, w = os.pipe() fds += r, w with warnings_helper.check_no_resource_warning(self): - open(r, *args, closefd=False, **kwargs) + self.open(r, *args, closefd=False, **kwargs) @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_warn_on_dealloc_fd(self): @@ -4618,10 +4699,8 @@ def test_check_encoding_warning(self): proc = assert_python_ok('-X', 'warn_default_encoding', '-c', code) warnings = proc.err.splitlines() self.assertEqual(len(warnings), 2) - self.assertTrue( - warnings[0].startswith(b"<string>:5: EncodingWarning: ")) - self.assertTrue( - warnings[1].startswith(b"<string>:8: EncodingWarning: ")) + self.assertStartsWith(warnings[0], b"<string>:5: EncodingWarning: ") + self.assertStartsWith(warnings[1], b"<string>:8: EncodingWarning: ") def test_text_encoding(self): # PEP 597, bpo-47000. io.text_encoding() returns "locale" or "utf-8" @@ -4834,7 +4913,7 @@ def on_alarm(*args): os.read(r, len(data) * 100) exc = cm.exception if isinstance(exc, RuntimeError): - self.assertTrue(str(exc).startswith("reentrant call"), str(exc)) + self.assertStartsWith(str(exc), "reentrant call") finally: signal.alarm(0) wio.close() @@ -4985,12 +5064,12 @@ def write(self, b: bytes): pass def test_reader_subclass(self): - self.assertIsSubclass(MyReader, io.Reader[bytes]) - self.assertNotIsSubclass(str, io.Reader[bytes]) + self.assertIsSubclass(self.MyReader, io.Reader) + self.assertNotIsSubclass(str, io.Reader) def test_writer_subclass(self): - self.assertIsSubclass(MyWriter, io.Writer[bytes]) - self.assertNotIsSubclass(str, io.Writer[bytes]) + self.assertIsSubclass(self.MyWriter, io.Writer) + self.assertNotIsSubclass(str, io.Writer) def load_tests(loader, tests, pattern): @@ -5004,6 +5083,7 @@ def load_tests(loader, tests, pattern): CTextIOWrapperTest, PyTextIOWrapperTest, CMiscIOTest, PyMiscIOTest, CSignalsTest, PySignalsTest, TestIOCTypes, + ProtocolsTest, ) # Put the namespaces of the IO module we are testing and some useful mock diff --git a/Lib/test/test_ioctl.py b/Lib/test/test_ioctl.py index 7a986048bdac2ac..01e1bfb42dd9864 100644 --- a/Lib/test/test_ioctl.py +++ b/Lib/test/test_ioctl.py @@ -5,7 +5,7 @@ import threading import unittest from test import support -from test.support import threading_helper +from test.support import os_helper, threading_helper from test.support.import_helper import import_module fcntl = import_module('fcntl') termios = import_module('termios') @@ -202,6 +202,15 @@ def test_ioctl_set_window_size(self): new_winsz = struct.unpack("HHHH", result) self.assertEqual(new_winsz[:2], (20, 40)) + @unittest.skipUnless(hasattr(fcntl, 'FICLONE'), 'need fcntl.FICLONE') + def test_bad_fd(self): + # gh-134744: Test error handling + fd = os_helper.make_bad_fd() + with self.assertRaises(OSError): + fcntl.ioctl(fd, fcntl.FICLONE, fd) + with self.assertRaises(OSError): + fcntl.ioctl(fd, fcntl.FICLONE, b'\0' * 1024) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py index d04012d1afd5408..8af91e857d80ed0 100644 --- a/Lib/test/test_ipaddress.py +++ b/Lib/test/test_ipaddress.py @@ -12,6 +12,7 @@ import pickle import ipaddress import weakref +from collections.abc import Iterator from test.support import LARGEST, SMALLEST @@ -397,6 +398,19 @@ def assertBadSplit(addr): # A trailing IPv4 address is two parts assertBadSplit("10:9:8:7:6:5:4:3:42.42.42.42%scope") + def test_bad_address_split_v6_too_long(self): + def assertBadSplit(addr): + msg = r"At most 45 characters expected in '%s" + with self.assertAddressError(msg, re.escape(addr[:45])): + ipaddress.IPv6Address(addr) + + # Long IPv6 address + long_addr = ("0:" * 10000) + "0" + assertBadSplit(long_addr) + assertBadSplit(long_addr + "%zoneid") + assertBadSplit(long_addr + ":255.255.255.255") + assertBadSplit(long_addr + ":ffff:255.255.255.255") + def test_bad_address_split_v6_too_many_parts(self): def assertBadSplit(addr): msg = "Exactly 8 parts expected without '::' in %r" @@ -1459,18 +1473,27 @@ def testGetSupernet4(self): self.ipv6_scoped_network.supernet(new_prefix=62)) def testHosts(self): + hosts = self.ipv4_network.hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(ipaddress.IPv4Address('1.2.3.1'), next(hosts)) hosts = list(self.ipv4_network.hosts()) self.assertEqual(254, len(hosts)) self.assertEqual(ipaddress.IPv4Address('1.2.3.1'), hosts[0]) self.assertEqual(ipaddress.IPv4Address('1.2.3.254'), hosts[-1]) ipv6_network = ipaddress.IPv6Network('2001:658:22a:cafe::/120') + hosts = ipv6_network.hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), next(hosts)) hosts = list(ipv6_network.hosts()) self.assertEqual(255, len(hosts)) self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), hosts[0]) self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::ff'), hosts[-1]) ipv6_scoped_network = ipaddress.IPv6Network('2001:658:22a:cafe::%scope/120') + hosts = ipv6_scoped_network.hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual((ipaddress.IPv6Address('2001:658:22a:cafe::1')), next(hosts)) hosts = list(ipv6_scoped_network.hosts()) self.assertEqual(255, len(hosts)) self.assertEqual(ipaddress.IPv6Address('2001:658:22a:cafe::1'), hosts[0]) @@ -1481,6 +1504,12 @@ def testHosts(self): ipaddress.IPv4Address('2.0.0.1')] str_args = '2.0.0.0/31' tpl_args = ('2.0.0.0', 31) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1490,6 +1519,12 @@ def testHosts(self): addrs = [ipaddress.IPv4Address('1.2.3.4')] str_args = '1.2.3.4/32' tpl_args = ('1.2.3.4', 32) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1499,6 +1534,12 @@ def testHosts(self): ipaddress.IPv6Address('2001:658:22a:cafe::1')] str_args = '2001:658:22a:cafe::/127' tpl_args = ('2001:658:22a:cafe::', 127) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -1507,6 +1548,12 @@ def testHosts(self): addrs = [ipaddress.IPv6Address('2001:658:22a:cafe::1'), ] str_args = '2001:658:22a:cafe::1/128' tpl_args = ('2001:658:22a:cafe::1', 128) + hosts = ipaddress.ip_network(str_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) + hosts = ipaddress.ip_network(tpl_args).hosts() + self.assertIsInstance(hosts, Iterator) + self.assertEqual(next(hosts), addrs[0]) self.assertEqual(addrs, list(ipaddress.ip_network(str_args).hosts())) self.assertEqual(addrs, list(ipaddress.ip_network(tpl_args).hosts())) self.assertEqual(list(ipaddress.ip_network(str_args).hosts()), @@ -2178,6 +2225,11 @@ def testIPv6AddressTooLarge(self): self.assertEqual(ipaddress.ip_address('FFFF::192.0.2.1'), ipaddress.ip_address('FFFF::c000:201')) + self.assertEqual(ipaddress.ip_address('0000:0000:0000:0000:0000:FFFF:192.168.255.255'), + ipaddress.ip_address('::ffff:c0a8:ffff')) + self.assertEqual(ipaddress.ip_address('FFFF:0000:0000:0000:0000:0000:192.168.255.255'), + ipaddress.ip_address('ffff::c0a8:ffff')) + self.assertEqual(ipaddress.ip_address('::FFFF:192.0.2.1%scope'), ipaddress.ip_address('::FFFF:c000:201%scope')) self.assertEqual(ipaddress.ip_address('FFFF::192.0.2.1%scope'), @@ -2190,6 +2242,10 @@ def testIPv6AddressTooLarge(self): ipaddress.ip_address('::FFFF:c000:201%scope')) self.assertNotEqual(ipaddress.ip_address('FFFF::192.0.2.1'), ipaddress.ip_address('FFFF::c000:201%scope')) + self.assertEqual(ipaddress.ip_address('0000:0000:0000:0000:0000:FFFF:192.168.255.255%scope'), + ipaddress.ip_address('::ffff:c0a8:ffff%scope')) + self.assertEqual(ipaddress.ip_address('FFFF:0000:0000:0000:0000:0000:192.168.255.255%scope'), + ipaddress.ip_address('ffff::c0a8:ffff%scope')) def testIPVersion(self): self.assertEqual(ipaddress.IPv4Address.version, 4) @@ -2599,6 +2655,10 @@ def testCompressIPv6Address(self): '::7:6:5:4:3:2:0': '0:7:6:5:4:3:2:0/128', '7:6:5:4:3:2:1::': '7:6:5:4:3:2:1:0/128', '0:6:5:4:3:2:1::': '0:6:5:4:3:2:1:0/128', + '0000:0000:0000:0000:0000:0000:255.255.255.255': '::ffff:ffff/128', + '0000:0000:0000:0000:0000:ffff:255.255.255.255': '::ffff:255.255.255.255/128', + 'ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255': + 'ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff/128', } for uncompressed, compressed in list(test_addresses.items()): self.assertEqual(compressed, str(ipaddress.IPv6Interface( @@ -2762,6 +2822,34 @@ def testV6HashIsNotConstant(self): ipv6_address2 = ipaddress.IPv6Interface("2001:658:22a:cafe:200:0:0:2") self.assertNotEqual(ipv6_address1.__hash__(), ipv6_address2.__hash__()) + # issue 134062 Hash collisions in IPv4Network and IPv6Network + def testNetworkV4HashCollisions(self): + self.assertNotEqual( + ipaddress.IPv4Network("192.168.1.255/32").__hash__(), + ipaddress.IPv4Network("192.168.1.0/24").__hash__() + ) + self.assertNotEqual( + ipaddress.IPv4Network("172.24.255.0/24").__hash__(), + ipaddress.IPv4Network("172.24.0.0/16").__hash__() + ) + self.assertNotEqual( + ipaddress.IPv4Network("192.168.1.87/32").__hash__(), + ipaddress.IPv4Network("192.168.1.86/31").__hash__() + ) + + # issue 134062 Hash collisions in IPv4Network and IPv6Network + def testNetworkV6HashCollisions(self): + self.assertNotEqual( + ipaddress.IPv6Network("fe80::/64").__hash__(), + ipaddress.IPv6Network("fe80::ffff:ffff:ffff:0/112").__hash__() + ) + self.assertNotEqual( + ipaddress.IPv4Network("10.0.0.0/8").__hash__(), + ipaddress.IPv6Network( + "ffff:ffff:ffff:ffff:ffff:ffff:aff:0/112" + ).__hash__() + ) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index daad00e86432d05..16d879a3831bc65 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -263,6 +263,7 @@ def test_subclass_tuple(self): self.assertEqual(True, issubclass(int, (int, (float, int)))) self.assertEqual(True, issubclass(str, (str, (Child, str)))) + @support.skip_if_huge_c_stack() @support.skip_wasi_stack_overflow() @support.skip_emscripten_stack_overflow() def test_subclass_recursion_limit(self): @@ -270,6 +271,7 @@ def test_subclass_recursion_limit(self): # blown self.assertRaises(RecursionError, blowstack, issubclass, str, str) + @support.skip_if_huge_c_stack() @support.skip_wasi_stack_overflow() @support.skip_emscripten_stack_overflow() def test_isinstance_recursion_limit(self): @@ -317,7 +319,9 @@ def __bases__(self): self.assertRaises(RecursionError, issubclass, int, X()) self.assertRaises(RecursionError, isinstance, 1, X()) + @support.skip_if_huge_c_stack() @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_infinite_recursion_via_bases_tuple(self): """Regression test for bpo-30570.""" class Failure(object): @@ -327,7 +331,9 @@ def __getattr__(self, attr): with self.assertRaises(RecursionError): issubclass(Failure(), int) + @support.skip_if_huge_c_stack() @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_infinite_cycle_in_bases(self): """Regression test for bpo-30570.""" class X: diff --git a/Lib/test/test_iter.py b/Lib/test/test_iter.py index 1b9f3cf76240ad4..18e4b676c532368 100644 --- a/Lib/test/test_iter.py +++ b/Lib/test/test_iter.py @@ -1147,7 +1147,7 @@ def test_error_iter(self): def test_exception_locations(self): # The location of an exception raised from __init__ or - # __next__ should should be the iterator expression + # __next__ should be the iterator expression def init_raises(): try: diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 61bea9dba07fec6..cf579d4da4e0dfb 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -733,6 +733,59 @@ def keyfunc(obj): keyfunc.skip = 1 self.assertRaises(ExpectedError, gulp, [None, None], keyfunc) + def test_groupby_reentrant_eq_does_not_crash(self): + # regression test for gh-143543 + class Key: + def __init__(self, do_advance): + self.do_advance = do_advance + + def __eq__(self, other): + if self.do_advance: + self.do_advance = False + next(g) + return NotImplemented + return False + + def keys(): + yield Key(True) + yield Key(False) + + g = itertools.groupby([None, None], keys().send) + next(g) + next(g) # must pass with address sanitizer + + def test_grouper_reentrant_eq_does_not_crash(self): + # regression test for gh-146613 + grouper_iter = None + + class Key: + __hash__ = None + + def __init__(self, do_advance): + self.do_advance = do_advance + + def __eq__(self, other): + nonlocal grouper_iter + if self.do_advance: + self.do_advance = False + if grouper_iter is not None: + try: + next(grouper_iter) + except StopIteration: + pass + return NotImplemented + return True + + def keyfunc(element): + if element == 0: + return Key(do_advance=True) + return Key(do_advance=False) + + g = itertools.groupby(range(4), keyfunc) + key, grouper_iter = next(g) + items = list(grouper_iter) + self.assertEqual(len(items), 1) + def test_filter(self): self.assertEqual(list(filter(isEven, range(6))), [0,2,4]) self.assertEqual(list(filter(None, [0,1,0,2,0])), [1,2]) diff --git a/Lib/test/test_json/json_lines.jsonl b/Lib/test/test_json/json_lines.jsonl new file mode 100644 index 000000000000000..d2f292111956f3a --- /dev/null +++ b/Lib/test/test_json/json_lines.jsonl @@ -0,0 +1,2 @@ +{"ingredients":["frog", "water", "chocolate", "glucose"]} +{"ingredients":["chocolate","steel bolts"]} diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py index 2250af964c022bf..0672413bf43c24f 100644 --- a/Lib/test/test_json/test_decode.py +++ b/Lib/test/test_json/test_decode.py @@ -1,4 +1,5 @@ import decimal +import unittest.mock from io import StringIO from collections import OrderedDict from test.test_json import PyTest, CTest @@ -47,6 +48,26 @@ def test_empty_objects(self): self.assertEqual(self.loads('[]'), []) self.assertEqual(self.loads('""'), "") + def test_object_hook(self): + s = '{"a":{"b":{}}}' + + expected_result = {"a":{"b":{"x":1}, "x":1}, "x":1} + expected_hook_arguments = [ + {}, {"b": {"x":1}}, {"a": {"b": {"x":1}, "x":1}} + ] + + hook_arguments = [] + + def hook(x): + hook_arguments.append(x) + return {**x, "x":1} + + result = self.loads(s, object_hook=hook) + + self.assertEqual(result, expected_result) + self.assertEqual(hook_arguments, expected_hook_arguments) + + def test_object_pairs_hook(self): s = '{"xkd":1, "kcw":2, "art":3, "hxm":4, "qrt":5, "pad":6, "hoy":7}' p = [("xkd", 1), ("kcw", 2), ("art", 3), ("hxm", 4), @@ -123,6 +144,10 @@ def test_negative_index(self): d = self.json.JSONDecoder() self.assertRaises(ValueError, d.raw_decode, 'a'*42, -50000) + def test_unterminated_string(self): + d = self.json.JSONDecoder() + self.assertRaises(self.JSONDecodeError, d.raw_decode, '"\\') + def test_limit_int(self): maxdigits = 5000 with support.adjust_int_max_str_digits(maxdigits): @@ -130,6 +155,15 @@ def test_limit_int(self): with self.assertRaises(ValueError): self.loads('1' * (maxdigits + 1)) + def test_explicit_cls_skips_json_decoder_default(self): + class CustomDecoder: + pass + + with unittest.mock.patch.object( + CustomDecoder, 'decode', create=True) as mock_decode: + self.loads('{}', cls=CustomDecoder) + mock_decode.assert_called_once() + class TestPyDecode(TestDecode, PyTest): pass class TestCDecode(TestDecode, CTest): pass diff --git a/Lib/test/test_json/test_dump.py b/Lib/test/test_json/test_dump.py index 13b40020781bae3..39470754003bb6f 100644 --- a/Lib/test/test_json/test_dump.py +++ b/Lib/test/test_json/test_dump.py @@ -22,6 +22,14 @@ def test_dump_skipkeys(self): self.assertIn('valid_key', o) self.assertNotIn(b'invalid_key', o) + def test_dump_skipkeys_indent_empty(self): + v = {b'invalid_key': False} + self.assertEqual(self.json.dumps(v, skipkeys=True, indent=4), '{}') + + def test_skipkeys_indent(self): + v = {b'invalid_key': False, 'valid_key': True} + self.assertEqual(self.json.dumps(v, skipkeys=True, indent=4), '{\n "valid_key": true\n}') + def test_encode_truefalse(self): self.assertEqual(self.dumps( {True: False, False: True}, sort_keys=True), diff --git a/Lib/test/test_json/test_encode_basestring_ascii.py b/Lib/test/test_json/test_encode_basestring_ascii.py index 6a39b72a09df357..c90d3e968e5ef9c 100644 --- a/Lib/test/test_json/test_encode_basestring_ascii.py +++ b/Lib/test/test_json/test_encode_basestring_ascii.py @@ -8,13 +8,12 @@ ('\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), ('controls', '"controls"'), ('\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), + ('\x00\x1f\x7f', '"\\u0000\\u001f\\u007f"'), ('{"object with 1 member":["array with 1 element"]}', '"{\\"object with 1 member\\":[\\"array with 1 element\\"]}"'), (' s p a c e d ', '" s p a c e d "'), ('\U0001d120', '"\\ud834\\udd20"'), ('\u03b1\u03a9', '"\\u03b1\\u03a9"'), ("`1~!@#$%^&*()_+-={':[,]}|;.</>?", '"`1~!@#$%^&*()_+-={\':[,]}|;.</>?"'), - ('\x08\x0c\n\r\t', '"\\b\\f\\n\\r\\t"'), - ('\u0123\u4567\u89ab\ucdef\uabcd\uef4a', '"\\u0123\\u4567\\u89ab\\ucdef\\uabcd\\uef4a"'), ] class TestEncodeBasestringAscii: diff --git a/Lib/test/test_json/test_fail.py b/Lib/test/test_json/test_fail.py index 7c1696cc66d12b7..7f99a6331d36251 100644 --- a/Lib/test/test_json/test_fail.py +++ b/Lib/test/test_json/test_fail.py @@ -102,7 +102,7 @@ def test_not_serializable(self): with self.assertRaisesRegex(TypeError, 'Object of type module is not JSON serializable') as cm: self.dumps(sys) - self.assertFalse(hasattr(cm.exception, '__notes__')) + self.assertNotHasAttr(cm.exception, '__notes__') with self.assertRaises(TypeError) as cm: self.dumps([1, [2, 3, sys]]) @@ -144,6 +144,13 @@ def test_truncated_input(self): ('"', 'Unterminated string starting at', 0), ('"spam', 'Unterminated string starting at', 0), ] + # A complete \uXXXX escape at end of input leaves it unterminated. + test_cases += [ + (r'"\u0041', 'Unterminated string starting at', 0), + (r'"\ud834', 'Unterminated string starting at', 0), + (r'"\ud834\udd1e', 'Unterminated string starting at', 0), + (r'{"a": "\u0041', 'Unterminated string starting at', 6), + ] for data, msg, idx in test_cases: with self.assertRaises(self.JSONDecodeError) as cm: self.loads(data) diff --git a/Lib/test/test_json/test_pickleable.py b/Lib/test/test_json/test_pickleable.py new file mode 100644 index 000000000000000..517cfcdd41a3516 --- /dev/null +++ b/Lib/test/test_json/test_pickleable.py @@ -0,0 +1,14 @@ +import json +import pickle +import unittest + +class TestPickleable(unittest.TestCase): + def test_json_decode_error_is_pickleable(self): + e = json.JSONDecodeError(msg="abc", doc="def", pos=7) + + pickled = pickle.dumps(e) + unpickled = pickle.loads(pickled) + + self.assertEqual(unpickled.msg, e.msg) + self.assertEqual(unpickled.doc, e.doc) + self.assertEqual(unpickled.pos, e.pos) diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index d82093f38951676..cbae9fbb4d624ba 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -68,9 +68,12 @@ def default(self, o): self.fail("didn't raise ValueError on default recursion") + @support.skip_if_pgo_task # fails during PGO training w/ some stack sizes + @support.skip_if_huge_c_stack(500_000) @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_highly_nested_objects_decoding(self): - very_deep = 200000 + very_deep = 500_000 # test that loading highly-nested objects doesn't segfault when C # accelerations are used. See #12017 with self.assertRaises(RecursionError): @@ -83,12 +86,14 @@ def test_highly_nested_objects_decoding(self): with support.infinite_recursion(): self.loads('[' * very_deep + '1' + ']' * very_deep) + @support.skip_if_unlimited_stack_size @support.skip_wasi_stack_overflow() @support.skip_emscripten_stack_overflow() + @support.requires_resource('cpu') def test_highly_nested_objects_encoding(self): # See #12051 l, d = [], {} - for x in range(200_000): + for x in range(500_000): l, d = [l], {'k':d} with self.assertRaises(RecursionError): with support.infinite_recursion(5000): @@ -97,7 +102,9 @@ def test_highly_nested_objects_encoding(self): with support.infinite_recursion(5000): self.dumps(d) + @support.skip_if_huge_c_stack() @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() def test_endless_recursion(self): # See #12051 class EndlessJSONEncoder(self.json.JSONEncoder): diff --git a/Lib/test/test_json/test_scanstring.py b/Lib/test/test_json/test_scanstring.py index cca556a3b95bab7..a52b01c71130d8c 100644 --- a/Lib/test/test_json/test_scanstring.py +++ b/Lib/test/test_json/test_scanstring.py @@ -137,6 +137,9 @@ def test_bad_escapes(self): '"\\ud834\\u-123"', '"\\ud834\\u+123"', '"\\ud834\\u1_23"', + # Truncated or non-hex \uXXXX escape at end of input. + '"\\u004', + '"\\uXYZW', ] for s in bad_escapes: with self.assertRaises(self.JSONDecodeError, msg=s): @@ -144,7 +147,7 @@ def test_bad_escapes(self): def test_overflow(self): with self.assertRaises(OverflowError): - self.json.decoder.scanstring(b"xxx", sys.maxsize+1) + self.json.decoder.scanstring("xxx", sys.maxsize+1) class TestPyScanstring(TestScanstring, PyTest): pass diff --git a/Lib/test/test_json/test_speedups.py b/Lib/test/test_json/test_speedups.py index 682014cfd5b344e..0b22a0bf4b95387 100644 --- a/Lib/test/test_json/test_speedups.py +++ b/Lib/test/test_json/test_speedups.py @@ -1,4 +1,5 @@ from test.test_json import CTest +from test.support import gc_collect class BadBool: @@ -80,3 +81,94 @@ def test(name): def test_unsortable_keys(self): with self.assertRaises(TypeError): self.json.encoder.JSONEncoder(sort_keys=True).encode({'a': 1, 1: 'a'}) + + def test_current_indent_level(self): + enc = self.json.encoder.c_make_encoder( + markers=None, + default=str, + encoder=self.json.encoder.c_encode_basestring, + indent='\t', + key_separator=': ', + item_separator=', ', + sort_keys=False, + skipkeys=False, + allow_nan=False) + expected = ( + '[\n' + '\t"spam", \n' + '\t{\n' + '\t\t"ham": "eggs"\n' + '\t}\n' + ']') + self.assertEqual(enc(['spam', {'ham': 'eggs'}], 0)[0], expected) + self.assertEqual(enc(['spam', {'ham': 'eggs'}], -3)[0], expected) + expected2 = ( + '[\n' + '\t\t\t\t"spam", \n' + '\t\t\t\t{\n' + '\t\t\t\t\t"ham": "eggs"\n' + '\t\t\t\t}\n' + '\t\t\t]') + self.assertEqual(enc(['spam', {'ham': 'eggs'}], 3)[0], expected2) + self.assertRaises(TypeError, enc, ['spam', {'ham': 'eggs'}], 3.0) + self.assertRaises(TypeError, enc, ['spam', {'ham': 'eggs'}]) + + def test_mutate_dict_items_during_encode(self): + # gh-142831: Clearing the items list via a re-entrant key encoder + # must not cause a use-after-free. BadDict.items() returns a + # mutable list; encode_str clears it while iterating. + items = None + + class BadDict(dict): + def items(self): + nonlocal items + items = [("boom", object())] + return items + + cleared = False + def encode_str(obj): + nonlocal items, cleared + if items is not None: + items.clear() + items = None + cleared = True + gc_collect() + return '"x"' + + encoder = self.json.encoder.c_make_encoder( + None, lambda o: "null", + encode_str, None, + ": ", ", ", False, + False, True + ) + + # Must not crash (use-after-free under ASan before fix) + encoder(BadDict(real=1), 0) + self.assertTrue(cleared) + + def test_mutate_list_during_encode(self): + # gh-142831: Clearing a list mid-iteration via the default + # callback must not cause a use-after-free. + call_count = 0 + lst = [object() for _ in range(10)] + + def default(obj): + nonlocal call_count + call_count += 1 + if call_count == 3: + lst.clear() + gc_collect() + return None + + encoder = self.json.encoder.c_make_encoder( + None, default, + self.json.encoder.c_encode_basestring, None, + ": ", ", ", False, + False, True + ) + + # Must not crash (use-after-free under ASan before fix) + encoder(lst, 0) + # Verify the mutation path was actually hit and the loop + # stopped iterating after the list was cleared. + self.assertEqual(call_count, 3) diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 72cde3f0d6c1bd6..0a96b318b15b1c9 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -1,4 +1,5 @@ import errno +import pathlib import os import sys import textwrap @@ -13,6 +14,7 @@ @support.requires_subprocess() +@support.skip_if_pgo_task class TestMain(unittest.TestCase): data = """ @@ -156,11 +158,19 @@ def test_jsonlines(self): self.assertEqual(process.stdout, self.jsonlines_expect) self.assertEqual(process.stderr, '') + @force_not_colorized + def test_jsonlines_from_file(self): + jsonl = pathlib.Path(__file__).parent / 'json_lines.jsonl' + args = sys.executable, '-m', self.module, '--json-lines', jsonl + process = subprocess.run(args, capture_output=True, text=True, check=True) + self.assertEqual(process.stdout, self.jsonlines_expect) + self.assertEqual(process.stderr, '') + def test_help_flag(self): rc, out, err = assert_python_ok('-m', self.module, '-h', PYTHON_COLORS='0') self.assertEqual(rc, 0) - self.assertTrue(out.startswith(b'usage: ')) + self.assertStartsWith(out, b'usage: ') self.assertEqual(err, b'') def test_sort_keys_flag(self): @@ -270,7 +280,7 @@ def test_colors(self): (r'" \"foo\" "', f'{t.string}" \\"foo\\" "{t.reset}'), ('"α"', f'{t.string}"\\u03b1"{t.reset}'), ('123', f'{t.number}123{t.reset}'), - ('-1.2345e+23', f'{t.number}-1.2345e+23{t.reset}'), + ('-1.25e+23', f'{t.number}-1.25e+23{t.reset}'), (r'{"\\": ""}', f'''\ {ob} @@ -319,6 +329,7 @@ def test_colors(self): @support.requires_subprocess() +@support.skip_if_pgo_task class TestTool(TestMain): module = 'json.tool' diff --git a/Lib/test/test_json/test_unicode.py b/Lib/test/test_json/test_unicode.py index 68629cceeb9be94..1aa9546dc463061 100644 --- a/Lib/test/test_json/test_unicode.py +++ b/Lib/test/test_json/test_unicode.py @@ -32,6 +32,29 @@ def test_encoding7(self): j = self.dumps(u + "\n", ensure_ascii=False) self.assertEqual(j, f'"{u}\\n"') + def test_ascii_non_printable_encode(self): + u = '\b\t\n\f\r\x00\x1f\x7f' + self.assertEqual(self.dumps(u), + '"\\b\\t\\n\\f\\r\\u0000\\u001f\\u007f"') + self.assertEqual(self.dumps(u, ensure_ascii=False), + '"\\b\\t\\n\\f\\r\\u0000\\u001f\x7f"') + + def test_ascii_non_printable_decode(self): + self.assertEqual(self.loads('"\\b\\t\\n\\f\\r"'), + '\b\t\n\f\r') + s = ''.join(map(chr, range(32))) + for c in s: + self.assertRaises(self.JSONDecodeError, self.loads, f'"{c}"') + self.assertEqual(self.loads(f'"{s}"', strict=False), s) + self.assertEqual(self.loads('"\x7f"'), '\x7f') + + def test_escaped_decode(self): + self.assertEqual(self.loads('"\\b\\t\\n\\f\\r"'), '\b\t\n\f\r') + self.assertEqual(self.loads('"\\"\\\\\\/"'), '"\\/') + for c in set(map(chr, range(0x100))) - set('"\\/bfnrt'): + self.assertRaises(self.JSONDecodeError, self.loads, f'"\\{c}"') + self.assertRaises(self.JSONDecodeError, self.loads, f'"\\{c}"', strict=False) + def test_big_unicode_encode(self): u = '\U0001d120' self.assertEqual(self.dumps(u), '"\\ud834\\udd20"') @@ -48,6 +71,18 @@ def test_unicode_decode(self): s = f'"\\u{i:04x}"' self.assertEqual(self.loads(s), u) + def test_single_surrogate_encode(self): + self.assertEqual(self.dumps('\uD83D'), '"\\ud83d"') + self.assertEqual(self.dumps('\uD83D', ensure_ascii=False), '"\ud83d"') + self.assertEqual(self.dumps('\uDC0D'), '"\\udc0d"') + self.assertEqual(self.dumps('\uDC0D', ensure_ascii=False), '"\udc0d"') + + def test_single_surrogate_decode(self): + self.assertEqual(self.loads('"\uD83D"'), '\ud83d') + self.assertEqual(self.loads('"\\uD83D"'), '\ud83d') + self.assertEqual(self.loads('"\udc0d"'), '\udc0d') + self.assertEqual(self.loads('"\\udc0d"'), '\udc0d') + def test_unicode_preservation(self): self.assertEqual(type(self.loads('""')), str) self.assertEqual(type(self.loads('"a"')), str) diff --git a/Lib/test/test_launcher.py b/Lib/test/test_launcher.py index 173fc743cf68ae0..795890bd7e4b473 100644 --- a/Lib/test/test_launcher.py +++ b/Lib/test/test_launcher.py @@ -227,6 +227,8 @@ def run_py(self, args, env=None, allow_fail=False, expect_returncode=0, argv=Non "PYLAUNCHER_LIMIT_TO_COMPANY": "", **{k.upper(): v for k, v in (env or {}).items()}, } + if ini_dir := getattr(self, '_ini_dir', None): + env.setdefault("_PYLAUNCHER_INIDIR", ini_dir) if not argv: argv = [self.py_exe, *args] with subprocess.Popen( @@ -262,11 +264,14 @@ def run_py(self, args, env=None, allow_fail=False, expect_returncode=0, argv=Non return data def py_ini(self, content): - local_appdata = os.environ.get("LOCALAPPDATA") - if not local_appdata: - raise unittest.SkipTest("LOCALAPPDATA environment variable is " - "missing or empty") - return PreservePyIni(Path(local_appdata) / "py.ini", content) + ini_dir = getattr(self, '_ini_dir', None) + if not ini_dir: + local_appdata = os.environ.get("LOCALAPPDATA") + if not local_appdata: + raise unittest.SkipTest("LOCALAPPDATA environment variable is " + "missing or empty") + ini_dir = local_appdata + return PreservePyIni(Path(ini_dir) / "py.ini", content) @contextlib.contextmanager def script(self, content, encoding="utf-8"): @@ -302,6 +307,8 @@ def setUpClass(cls): p = subprocess.check_output("reg query HKCU\\Software\\Python /s") #print(p.decode('mbcs')) + cls._ini_dir = tempfile.mkdtemp() + cls.addClassCleanup(shutil.rmtree, cls._ini_dir, ignore_errors=True) @classmethod def tearDownClass(cls): @@ -443,7 +450,7 @@ def test_search_major_3(self): except subprocess.CalledProcessError: raise unittest.SkipTest("requires at least one Python 3.x install") self.assertEqual("PythonCore", data["env.company"]) - self.assertTrue(data["env.tag"].startswith("3."), data["env.tag"]) + self.assertStartsWith(data["env.tag"], "3.") def test_search_major_3_32(self): try: @@ -453,8 +460,8 @@ def test_search_major_3_32(self): raise unittest.SkipTest("requires at least one 32-bit Python 3.x install") raise self.assertEqual("PythonCore", data["env.company"]) - self.assertTrue(data["env.tag"].startswith("3."), data["env.tag"]) - self.assertTrue(data["env.tag"].endswith("-32"), data["env.tag"]) + self.assertStartsWith(data["env.tag"], "3.") + self.assertEndsWith(data["env.tag"], "-32") def test_search_major_2(self): try: @@ -462,8 +469,9 @@ def test_search_major_2(self): except subprocess.CalledProcessError: if not is_installed("2.7"): raise unittest.SkipTest("requires at least one Python 2.x install") + raise self.assertEqual("PythonCore", data["env.company"]) - self.assertTrue(data["env.tag"].startswith("2."), data["env.tag"]) + self.assertStartsWith(data["env.tag"], "2.") def test_py_default(self): with self.py_ini(TEST_PY_DEFAULTS): diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index e4aa41ebb437626..02f65338428c8fe 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -4,10 +4,12 @@ import unittest import os.path import tempfile +import threading import tokenize from importlib.machinery import ModuleSpec from test import support from test.support import os_helper +from test.support import threading_helper from test.support.script_helper import assert_python_ok @@ -374,5 +376,40 @@ def test_checkcache_with_no_parameter(self): self.assertIn(self.unchanged_file, linecache.cache) +class MultiThreadingTest(unittest.TestCase): + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_read_write_safety(self): + + with tempfile.TemporaryDirectory() as tmpdirname: + filenames = [] + for i in range(10): + name = os.path.join(tmpdirname, f"test_{i}.py") + with open(name, "w") as h: + h.write("import time\n") + h.write("import system\n") + filenames.append(name) + + def linecache_get_line(b): + b.wait() + for _ in range(100): + for name in filenames: + linecache.getline(name, 1) + + def check(funcs): + barrier = threading.Barrier(len(funcs)) + threads = [] + + for func in funcs: + thread = threading.Thread(target=func, args=(barrier,)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + check([linecache_get_line] * 20) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 6894fba2ad1b00c..4d5cb11bddb769d 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -3,7 +3,6 @@ import textwrap from test import list_tests, support from test.support import cpython_only -from test.support.import_helper import import_module from test.support.script_helper import assert_python_failure, assert_python_ok import pickle import unittest @@ -326,10 +325,9 @@ def test_tier2_invalidates_iterator(self): a.append(4) self.assertEqual(list(it), []) - @support.cpython_only + @support.nomemtest def test_no_memory(self): # gh-118331: Make sure we don't crash if list allocation fails - import_module("_testcapi") code = textwrap.dedent(""" import _testcapi, sys # Prime the freelist diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index cffdeeacc5d73bc..b7580867ea782ce 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -171,6 +171,17 @@ def test_references___class__(self): """ self._check_in_scopes(code, raises=NameError) + def test_references___class___nested(self): + code = """ + res = [(lambda: __class__)() for _ in [1]] + """ + self._check_in_scopes(code, raises=NameError) + + def test_references___class___nested_used(self): + class _C: + res = [lambda: __class__ for _ in [1]] + self.assertIs(_C.res[0](), _C) + def test_references___class___defined(self): code = """ __class__ = 2 @@ -180,6 +191,38 @@ def test_references___class___defined(self): code, outputs={"res": [2]}, scopes=["module", "function"]) self._check_in_scopes(code, raises=NameError, scopes=["class"]) + def test_references___class___defined_nested(self): + code = """ + __class__ = 2 + res = [(lambda: __class__)() for x in [1]] + """ + self._check_in_scopes( + code, outputs={"res": [2]}, scopes=["module", "function"]) + self._check_in_scopes(code, raises=NameError, scopes=["class"]) + + def test_references___classdict__(self): + code = """ + class i: [__classdict__ for x in y] + """ + self._check_in_scopes(code, raises=NameError) + + def test_references___classdict___nested(self): + class _C: + res = [(lambda: __classdict__)() for _ in [1]] + self.assertIn("res", _C.res[0]) + + def test_references___conditional_annotations__(self): + code = """ + class i: [__conditional_annotations__ for x in y] + """ + self._check_in_scopes(code, raises=NameError) + + def test_references___conditional_annotations___nested(self): + code = """ + class i: [lambda: __conditional_annotations__ for x in y] + """ + self._check_in_scopes(code, raises=NameError) + def test_references___class___enclosing(self): code = """ __class__ = 2 @@ -716,7 +759,7 @@ def test_multiple_comprehension_name_reuse(self): def test_exception_locations(self): # The location of an exception raised from __init__ or - # __next__ should should be the iterator expression + # __next__ should be the iterator expression def init_raises(): try: @@ -750,6 +793,28 @@ def iter_raises(): self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], expected) + def test_only_calls_dunder_iter_once(self): + + class Iterator: + + def __init__(self): + self.val = 0 + + def __next__(self): + if self.val == 2: + raise StopIteration + self.val += 1 + return self.val + + # No __iter__ method + + class C: + + def __iter__(self): + return Iterator() + + self.assertEqual([1, 2], [i for i in C()]) + __test__ = {'doctests' : doctests} def load_tests(loader, tests, pattern): diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 455d2af37efdc80..f91843562c30fc5 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -1,10 +1,10 @@ from decimal import Decimal from test.support import cpython_only, verbose, is_android, linked_to_musl, os_helper -from test.support.warnings_helper import check_warnings from test.support.import_helper import ensure_lazy_imports, import_fresh_module from unittest import mock import unittest import locale +import os import sys import codecs @@ -349,8 +349,7 @@ def setUp(self): enc = codecs.lookup(locale.getencoding() or 'ascii').name if enc not in ('utf-8', 'iso8859-1', 'cp1252'): raise unittest.SkipTest('encoding not suitable') - if enc != 'iso8859-1' and (sys.platform == 'darwin' or is_android or - sys.platform.startswith('freebsd')): + if enc != 'iso8859-1' and is_android: raise unittest.SkipTest('wcscoll/wcsxfrm have known bugs') BaseLocalizedTest.setUp(self) @@ -387,6 +386,10 @@ def test_c(self): self.check('c', 'C') self.check('posix', 'C') + def test_c_utf8(self): + self.check('c.utf8', 'C.UTF-8') + self.check('C.UTF-8', 'C.UTF-8') + def test_english(self): self.check('en', 'en_US.ISO8859-1') self.check('EN', 'en_US.ISO8859-1') @@ -482,6 +485,54 @@ def test_japanese(self): self.check('jp_jp', 'ja_JP.eucJP') +class TestRealLocales(unittest.TestCase): + def setUp(self): + oldlocale = locale.setlocale(locale.LC_CTYPE) + self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) + + def test_getsetlocale_issue1813(self): + # Issue #1813: setting and getting the locale under a Turkish locale + try: + locale.setlocale(locale.LC_CTYPE, 'tr_TR') + except locale.Error: + # Unsupported locale on this system + self.skipTest('test needs Turkish locale') + loc = locale.getlocale(locale.LC_CTYPE) + if verbose: + print('testing with %a' % (loc,), end=' ', flush=True) + try: + locale.setlocale(locale.LC_CTYPE, loc) + except locale.Error as exc: + # bpo-37945: setlocale(LC_CTYPE) fails with getlocale(LC_CTYPE) + # and the tr_TR locale on Windows. getlocale() builds a locale + # which is not recognize by setlocale(). + self.skipTest(f"setlocale(LC_CTYPE, {loc!r}) failed: {exc!r}") + self.assertEqual(loc, locale.getlocale(locale.LC_CTYPE)) + + @unittest.skipUnless(os.name == 'nt', 'requires Windows') + def test_setlocale_long_encoding(self): + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_CTYPE, 'English.%016d' % 1252) + locale.setlocale(locale.LC_CTYPE, 'English.%015d' % 1252) + loc = locale.setlocale(locale.LC_ALL) + self.assertIn('.1252', loc) + loc2 = loc.replace('.1252', '.%016d' % 1252, 1) + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_ALL, loc2) + loc2 = loc.replace('.1252', '.%015d' % 1252, 1) + locale.setlocale(locale.LC_ALL, loc2) + + # gh-137273: Debug assertion failure on Windows for long encoding. + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_CTYPE, 'en_US.' + 'x'*16) + locale.setlocale(locale.LC_CTYPE, 'en_US.UTF-8') + loc = locale.setlocale(locale.LC_ALL) + self.assertIn('.UTF-8', loc) + loc2 = loc.replace('.UTF-8', '.' + 'x'*16, 1) + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_ALL, loc2) + + class TestMiscellaneous(unittest.TestCase): def test_defaults_UTF8(self): # Issue #18378: on (at least) macOS setting LC_CTYPE to "UTF-8" is @@ -502,8 +553,7 @@ def test_defaults_UTF8(self): env.unset('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE') env.set('LC_CTYPE', 'UTF-8') - with check_warnings(('', DeprecationWarning)): - self.assertEqual(locale.getdefaultlocale(), (None, 'UTF-8')) + self.assertEqual(locale.getdefaultlocale(), (None, 'UTF-8')) finally: if orig_getlocale is not None: _locale._getdefaultlocale = orig_getlocale @@ -548,27 +598,6 @@ def test_setlocale_category(self): # crasher from bug #7419 self.assertRaises(locale.Error, locale.setlocale, 12345) - def test_getsetlocale_issue1813(self): - # Issue #1813: setting and getting the locale under a Turkish locale - oldlocale = locale.setlocale(locale.LC_CTYPE) - self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) - try: - locale.setlocale(locale.LC_CTYPE, 'tr_TR') - except locale.Error: - # Unsupported locale on this system - self.skipTest('test needs Turkish locale') - loc = locale.getlocale(locale.LC_CTYPE) - if verbose: - print('testing with %a' % (loc,), end=' ', flush=True) - try: - locale.setlocale(locale.LC_CTYPE, loc) - except locale.Error as exc: - # bpo-37945: setlocale(LC_CTYPE) fails with getlocale(LC_CTYPE) - # and the tr_TR locale on Windows. getlocale() builds a locale - # which is not recognize by setlocale(). - self.skipTest(f"setlocale(LC_CTYPE, {loc!r}) failed: {exc!r}") - self.assertEqual(loc, locale.getlocale(locale.LC_CTYPE)) - def test_invalid_locale_format_in_localetuple(self): with self.assertRaises(TypeError): locale.setlocale(locale.LC_ALL, b'fi_FI') diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 3f113ec1be47afb..0c27c03ac850610 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -25,6 +25,7 @@ import codecs import configparser +import contextlib import copy import datetime import pathlib @@ -1036,7 +1037,7 @@ class TestTCPServer(ControlMixin, ThreadingTCPServer): """ allow_reuse_address = True - allow_reuse_port = True + allow_reuse_port = False def __init__(self, addr, handler, poll_interval=0.5, bind_and_activate=True): @@ -3633,14 +3634,14 @@ def setup_via_listener(self, text, verify=None): # Ask for a randomly assigned port (by using port 0) t = logging.config.listen(0, verify) t.start() - t.ready.wait() + self.assertTrue(t.ready.wait(support.LONG_TIMEOUT), + msg='the listener did not start') # Now get the port allocated port = t.port t.ready.clear() try: - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sock.settimeout(2.0) - sock.connect(('localhost', port)) + # The server can listen on IPv6, so do not force a family. + sock = socket.create_connection(('localhost', port), timeout=2.0) slen = struct.pack('>L', len(text)) s = slen + text @@ -3755,6 +3756,18 @@ def verify_reverse(stuff): ('ERROR', '2'), ], pat=r"^[\w.]+ -> (\w+): (\d+)$") + @support.requires_working_socket() + def test_listen_server_error(self): + # The "ready" event should be set even if the server fails to start. + t = logging.config.listen(-1) + t.daemon = True + with threading_helper.catch_threading_exception() as cm: + t.start() + self.assertTrue(t.ready.wait(support.SHORT_TIMEOUT), + msg='the listener did not report the failure') + threading_helper.join_thread(t) + self.assertIs(cm.exc_type, OverflowError) + def test_bad_format(self): self.assertRaises(ValueError, self.apply_config, self.bad_format) @@ -5421,7 +5434,7 @@ def test_taskName_with_asyncio_imported(self): logging.logAsyncioTasks = False runner.run(make_record(self.assertIsNone)) finally: - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) @support.requires_working_socket() def test_taskName_without_asyncio_imported(self): @@ -5433,7 +5446,7 @@ def test_taskName_without_asyncio_imported(self): logging.logAsyncioTasks = False runner.run(make_record(self.assertIsNone)) finally: - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class BasicConfigTest(unittest.TestCase): @@ -5737,7 +5750,7 @@ async def log_record(): data = f.read().strip() self.assertRegex(data, r'Task-\d+ - hello world') finally: - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) if handler: handler.close() @@ -5800,7 +5813,7 @@ def cleanup(): self.addCleanup(cleanup) self.addCleanup(logging.shutdown) - self.adapter = logging.LoggerAdapter(logger=self.logger, extra=None) + self.adapter = logging.LoggerAdapter(logger=self.logger) def test_exception(self): msg = 'testing exception: %r' @@ -5971,6 +5984,18 @@ def test_extra_merged(self): self.assertEqual(record.foo, '1') self.assertEqual(record.bar, '2') + self.adapter.critical('no extra') # should not fail + self.assertEqual(len(self.recording.records), 2) + record = self.recording.records[-1] + self.assertEqual(record.foo, '1') + self.assertNotHasAttr(record, 'bar') + + self.adapter.critical('none extra', extra=None) # should not fail + self.assertEqual(len(self.recording.records), 3) + record = self.recording.records[-1] + self.assertEqual(record.foo, '1') + self.assertNotHasAttr(record, 'bar') + def test_extra_merged_log_call_has_precedence(self): self.adapter = logging.LoggerAdapter(logger=self.logger, extra={'foo': '1'}, @@ -5982,6 +6007,25 @@ def test_extra_merged_log_call_has_precedence(self): self.assertHasAttr(record, 'foo') self.assertEqual(record.foo, '2') + def test_extra_merged_without_extra(self): + self.adapter = logging.LoggerAdapter(logger=self.logger, + merge_extra=True) + + self.adapter.critical('foo should be here', extra={'foo': '1'}) + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[-1] + self.assertEqual(record.foo, '1') + + self.adapter.critical('no extra') # should not fail + self.assertEqual(len(self.recording.records), 2) + record = self.recording.records[-1] + self.assertNotHasAttr(record, 'foo') + + self.adapter.critical('none extra', extra=None) # should not fail + self.assertEqual(len(self.recording.records), 3) + record = self.recording.records[-1] + self.assertNotHasAttr(record, 'foo') + class PrefixAdapter(logging.LoggerAdapter): prefix = 'Adapter' @@ -6312,6 +6356,32 @@ def test_should_not_rollover_non_file(self): self.assertFalse(rh.shouldRollover(self.next_rec())) rh.close() + @unittest.skipUnless(hasattr(os, "mkfifo"), 'requires os.mkfifo()') + def test_should_not_rollover_named_pipe(self): + # gh-143237 - test with non-seekable special file (named pipe) + filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, filename) + try: + os.mkfifo(filename) + except PermissionError as e: + self.skipTest('os.mkfifo(): %s' % e) + + data = 'not read' + def other_side(): + nonlocal data + with open(filename, 'rb') as f: + data = f.read() + + thread = threading.Thread(target=other_side) + with threading_helper.start_threads([thread]): + rh = logging.handlers.RotatingFileHandler( + filename, encoding="utf-8", maxBytes=1) + with contextlib.closing(rh): + m = self.next_rec() + self.assertFalse(rh.shouldRollover(m)) + rh.emit(m) + self.assertEqual(data.decode(), m.msg + os.linesep) + def test_should_rollover(self): with open(self.fn, 'wb') as f: f.write(b'\n') diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index f336d49fa4f0086..b48a8812a1a2d13 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -1374,17 +1374,22 @@ def equivalent_python(n, length, byteorder, signed=False): check(tests4, 'little', signed=False) self.assertRaises(OverflowError, (256).to_bytes, 1, 'big', signed=False) - self.assertRaises(OverflowError, (256).to_bytes, 1, 'big', signed=True) self.assertRaises(OverflowError, (256).to_bytes, 1, 'little', signed=False) - self.assertRaises(OverflowError, (256).to_bytes, 1, 'little', signed=True) + self.assertRaises(OverflowError, (128).to_bytes, 1, 'big', signed=True) + self.assertRaises(OverflowError, (128).to_bytes, 1, 'little', signed=True) + self.assertRaises(OverflowError, (-129).to_bytes, 1, 'big', signed=True) + self.assertRaises(OverflowError, (-129).to_bytes, 1, 'little', signed=True) self.assertRaises(OverflowError, (-1).to_bytes, 2, 'big', signed=False) self.assertRaises(OverflowError, (-1).to_bytes, 2, 'little', signed=False) self.assertEqual((0).to_bytes(0, 'big'), b'') + self.assertEqual((0).to_bytes(0, 'big', signed=True), b'') self.assertEqual((1).to_bytes(5, 'big'), b'\x00\x00\x00\x00\x01') self.assertEqual((0).to_bytes(5, 'big'), b'\x00\x00\x00\x00\x00') self.assertEqual((-1).to_bytes(5, 'big', signed=True), b'\xff\xff\xff\xff\xff') self.assertRaises(OverflowError, (1).to_bytes, 0, 'big') + self.assertRaises(OverflowError, (-1).to_bytes, 0, 'big', signed=True) + self.assertRaises(OverflowError, (-1).to_bytes, 0, 'little', signed=True) # gh-98783 class SubStr(str): @@ -1693,5 +1698,21 @@ class MyInt(int): # GH-117195 -- This shouldn't crash object.__sizeof__(1) + def test_hash(self): + # gh-136599 + self.assertEqual(hash(-1), -2) + self.assertEqual(hash(0), 0) + self.assertEqual(hash(10), 10) + + self.assertEqual(hash(sys.hash_info.modulus - 2), sys.hash_info.modulus - 2) + self.assertEqual(hash(sys.hash_info.modulus - 1), sys.hash_info.modulus - 1) + self.assertEqual(hash(sys.hash_info.modulus), 0) + self.assertEqual(hash(sys.hash_info.modulus + 1), 1) + + self.assertEqual(hash(-sys.hash_info.modulus - 2), -2) + self.assertEqual(hash(-sys.hash_info.modulus - 1), -2) + self.assertEqual(hash(-sys.hash_info.modulus), 0) + self.assertEqual(hash(-sys.hash_info.modulus + 1), -sys.hash_info.modulus + 1) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_lzma.py b/Lib/test/test_lzma.py index 9ffb93e797dd80f..e93c3c37354e27f 100644 --- a/Lib/test/test_lzma.py +++ b/Lib/test/test_lzma.py @@ -1025,12 +1025,12 @@ def test_peek(self): with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: result = f.peek() self.assertGreater(len(result), 0) - self.assertTrue(INPUT.startswith(result)) + self.assertStartsWith(INPUT, result) self.assertEqual(f.read(), INPUT) with LZMAFile(BytesIO(COMPRESSED_XZ)) as f: result = f.peek(10) self.assertGreater(len(result), 0) - self.assertTrue(INPUT.startswith(result)) + self.assertStartsWith(INPUT, result) self.assertEqual(f.read(), INPUT) def test_peek_bad_args(self): diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index 8b1fb0eba1f8b60..b67044e66aa331c 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -43,6 +43,11 @@ def test_ints(self): for expected in (-n, n): self.helper(expected) n = n >> 1 + n = 1 << 100 + while n: + for expected in (-n, -n+1, n-1, n): + self.helper(expected) + n = n >> 1 def test_int64(self): # Simulate int marshaling with TYPE_INT64. @@ -312,6 +317,128 @@ def test_recursion_limit(self): last.append([0]) self.assertRaises(ValueError, marshal.dumps, head) + def test_reference_loop_list(self): + a = [] + a.append(a) + for v in range(3): + self.assertRaises(ValueError, marshal.dumps, a, v) + for v in range(3, marshal.version + 1): + d = marshal.dumps(a, v) + b = marshal.loads(d) + self.assertIsInstance(b, list) + self.assertIs(b[0], b) + + def test_reference_loop_dict(self): + a = {} + a[None] = a + for v in range(3): + self.assertRaises(ValueError, marshal.dumps, a, v) + for v in range(3, marshal.version + 1): + d = marshal.dumps(a, v) + b = marshal.loads(d) + self.assertIsInstance(b, dict) + self.assertIs(b[None], b) + + def test_reference_loop_tuple(self): + a = ([],) + a[0].append(a) + for v in range(3): + self.assertRaises(ValueError, marshal.dumps, a, v) + for v in range(3, marshal.version + 1): + d = marshal.dumps(a, v) + b = marshal.loads(d) + self.assertIsInstance(b, tuple) + self.assertIsInstance(b[0], list) + self.assertIs(b[0][0], b) + + def test_reference_loop_code(self): + def f(): + return 1234.5 + code = f.__code__ + a = [] + code = code.replace(co_consts=code.co_consts + (a,)) + # This test creates a reference loop which leads to reference leaks, + # so we need to break the loop manually. See gh-148722. + self.addCleanup(a.clear) + a.append(code) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, code, v) + + def test_reference_loop_slice(self): + a = slice([], None) + a.start.append(a) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, a, v) + + a = slice(None, []) + a.stop.append(a) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, a, v) + + a = slice(None, None, []) + a.step.append(a) + for v in range(marshal.version + 1): + self.assertRaises(ValueError, marshal.dumps, a, v) + + def test_loads_reference_loop_list(self): + data = b'\xdb\x01\x00\x00\x00r\x00\x00\x00\x00' # [<R>] + a = marshal.loads(data) + self.assertIsInstance(a, list) + self.assertIs(a[0], a) + + def test_loads_reference_loop_dict(self): + data = b'\xfbNr\x00\x00\x00\x000' # {None: <R>} + a = marshal.loads(data) + self.assertIsInstance(a, dict) + self.assertIs(a[None], a) + + def test_loads_abnormal_reference_loops(self): + # Indirect self-references of tuples. + data = b'\xa8\x01\x00\x00\x00[\x01\x00\x00\x00r\x00\x00\x00\x00' # ([<R>],) + a = marshal.loads(data) + self.assertIsInstance(a, tuple) + self.assertIsInstance(a[0], list) + self.assertIs(a[0][0], a) + + data = b'\xa8\x01\x00\x00\x00{Nr\x00\x00\x00\x000' # ({None: <R>},) + a = marshal.loads(data) + self.assertIsInstance(a, tuple) + self.assertIsInstance(a[0], dict) + self.assertIs(a[0][None], a) + + # Direct self-reference which cannot be created in Python. + # This creates a reference loop which cannot be collected. + if False: + data = b'\xa8\x01\x00\x00\x00r\x00\x00\x00\x00' # (<R>,) + a = marshal.loads(data) + self.assertIsInstance(a, tuple) + self.assertIs(a[0], a) + + # Direct self-references which cannot be created in Python + # because of unhashability. + data = b'\xfbr\x00\x00\x00\x00N0' # {<R>: None} + self.assertRaises(TypeError, marshal.loads, data) + data = b'\xbc\x01\x00\x00\x00r\x00\x00\x00\x00' # {<R>} + self.assertRaises(TypeError, marshal.loads, data) + + for data in [ + # Indirect self-references of immutable objects. + b'\xba[\x01\x00\x00\x00r\x00\x00\x00\x00NN', # slice([<R>], None) + b'\xbaN[\x01\x00\x00\x00r\x00\x00\x00\x00N', # slice(None, [<R>]) + b'\xbaNN[\x01\x00\x00\x00r\x00\x00\x00\x00', # slice(None, None, [<R>]) + b'\xba{Nr\x00\x00\x00\x000NN', # slice({None: <R>}, None) + b'\xbaN{Nr\x00\x00\x00\x000N', # slice(None, {None: <R>}) + b'\xbaNN{Nr\x00\x00\x00\x000', # slice(None, None, {None: <R>}) + + # Direct self-references which cannot be created in Python. + b'\xbe\x01\x00\x00\x00r\x00\x00\x00\x00', # frozenset({<R>}) + b'\xbar\x00\x00\x00\x00NN', # slice(<R>, None) + b'\xbaNr\x00\x00\x00\x00N', # slice(None, <R>) + b'\xbaNNr\x00\x00\x00\x00', # slice(None, None, <R>) + ]: + with self.subTest(data=data): + self.assertRaises(ValueError, marshal.loads, data) + def test_exact_type_match(self): # Former bug: # >>> class Int(int): pass @@ -408,6 +535,26 @@ def test_deterministic_sets(self): _, dump_1, _ = assert_python_ok(*args, PYTHONHASHSEED="1") self.assertEqual(dump_0, dump_1) + def test_unmarshallable(self): + # Check no crash after encountering unmarshallable objects. + # See https://github.com/python/cpython/issues/106287. + fset = frozenset([int]) + code = compile("a = 1", "<string>", "exec") + code = code.replace(co_consts=(1, fset, None)) + cases = (('tuple', (fset,)), + ('list', [fset]), + ('set', fset), + ('dict key', {fset: 'x'}), + ('dict value', {'x': fset}), + ('dict key & value', {fset: fset}), + ('slice', slice(fset, fset)), + ('code', code)) + for name, arg in cases: + with self.subTest(name, arg=arg): + with self.assertRaisesRegex(ValueError, "unmarshallable object"): + marshal.dumps((arg, memoryview(b''))) + + LARGE_SIZE = 2**31 pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4 diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 913a60bf9e04e33..fd7d1a1c61cfd76 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -238,6 +238,19 @@ def __init__(self, value): def __index__(self): return self.value +# int subclass with broken arithmetic operators; implementations must +# convert their arguments to exact ints instead of using these. +class BadIntSubclass(int): + def _binop(self, other='ignored', mod=None): + return 42 + __add__ = __radd__ = __sub__ = __rsub__ = _binop + __mul__ = __rmul__ = __mod__ = __rmod__ = _binop + __divmod__ = __rdivmod__ = __pow__ = __rpow__ = _binop + __floordiv__ = __rfloordiv__ = _binop + __lshift__ = __rlshift__ = __rshift__ = __rrshift__ = _binop + __and__ = __rand__ = __or__ = __ror__ = __xor__ = __rxor__ = _binop + __lt__ = __le__ = __gt__ = __ge__ = _binop + class BadDescr: def __get__(self, obj, objtype=None): raise ValueError @@ -324,6 +337,8 @@ def testAtanh(self): self.assertRaises(ValueError, math.atanh, NINF) self.assertTrue(math.isnan(math.atanh(NAN))) + @unittest.skipIf(sys.platform.startswith("sunos"), + "skipping, see gh-138573") def testAtan2(self): self.assertRaises(TypeError, math.atan2) self.ftest('atan2(-1, 0)', math.atan2(-1, 0), -math.pi/2) @@ -1115,6 +1130,16 @@ def __index__(self): self.assertIs(type(s), int) self.assertEqual(s, 41) + # Overridden operators of an int subclass must not affect the + # result. + s = math.isqrt(BadIntSubclass(10**20)) + self.assertIs(type(s), int) + self.assertEqual(s, 10**10) + + s = math.isqrt(BadIntSubclass(10**20 - 1)) + self.assertIs(type(s), int) + self.assertEqual(s, 10**10 - 1) + with self.assertRaises(ValueError): math.isqrt(IntegerLike(-3)) @@ -1214,6 +1239,12 @@ def testLdexp(self): self.assertEqual(math.ldexp(NINF, n), NINF) self.assertTrue(math.isnan(math.ldexp(NAN, n))) + @requires_IEEE_754 + def testLdexp_denormal(self): + # Denormal output incorrectly rounded (truncated) + # on some Windows. + self.assertEqual(math.ldexp(6993274598585239, -1126), 1e-323) + def testLog(self): self.assertRaises(TypeError, math.log) self.assertRaises(TypeError, math.log, 1, 2, 3) diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index 95629ed862d6eb8..464fbf67a84b83d 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -54,6 +54,12 @@ def testSeek(self): self.assertEqual(buf[3:], bytesIo.read()) self.assertRaises(TypeError, bytesIo.seek, 0.0) + self.assertEqual(sys.maxsize, bytesIo.seek(sys.maxsize)) + self.assertEqual(self.EOF, bytesIo.read(4)) + + self.assertEqual(sys.maxsize - 2, bytesIo.seek(sys.maxsize - 2)) + self.assertEqual(self.EOF, bytesIo.read(4)) + def testTell(self): buf = self.buftype("1234567890") bytesIo = self.ioclass(buf) @@ -265,8 +271,8 @@ def test_iterator(self): memio = self.ioclass(buf * 10) self.assertEqual(iter(memio), memio) - self.assertTrue(hasattr(memio, '__iter__')) - self.assertTrue(hasattr(memio, '__next__')) + self.assertHasAttr(memio, '__iter__') + self.assertHasAttr(memio, '__next__') i = 0 for line in memio: self.assertEqual(line, buf) @@ -552,6 +558,14 @@ def test_relative_seek(self): memio.seek(1, 1) self.assertEqual(memio.read(), buf[1:]) + def test_issue141311(self): + memio = self.ioclass() + # Seek allows PY_SSIZE_T_MAX, read should handle that. + # Past end of buffer read should always return 0 (EOF). + self.assertEqual(sys.maxsize, memio.seek(sys.maxsize)) + buf = bytearray(2) + self.assertEqual(0, memio.readinto(buf)) + def test_unicode(self): memio = self.ioclass() @@ -573,6 +587,70 @@ def test_issue5449(self): self.ioclass(initial_bytes=buf) self.assertRaises(TypeError, self.ioclass, buf, foo=None) + def test_write_concurrent_close(self): + class B: + def __buffer__(self, flags): + memio.close() + return memoryview(b"A") + + memio = self.ioclass() + self.assertRaises(ValueError, memio.write, B()) + + # Prevent crashes when memio.write() or memio.writelines() + # concurrently mutates (e.g., closes or exports) 'memio'. + # See: https://github.com/python/cpython/issues/143378. + + def test_writelines_concurrent_close(self): + class B: + def __buffer__(self, flags): + memio.close() + return memoryview(b"A") + + memio = self.ioclass() + self.assertRaises(ValueError, memio.writelines, [B()]) + + def test_write_concurrent_export(self): + class B: + buf = None + def __buffer__(self, flags): + self.buf = memio.getbuffer() + return memoryview(b"A") + + memio = self.ioclass() + self.assertRaises(BufferError, memio.write, B()) + + def test_writelines_concurrent_export(self): + class B: + buf = None + def __buffer__(self, flags): + self.buf = memio.getbuffer() + return memoryview(b"A") + + memio = self.ioclass() + self.assertRaises(BufferError, memio.writelines, [B()]) + + def test_write_mutating_buffer(self): + # Test that buffer is exported only once during write(). + # See: https://github.com/python/cpython/issues/143602. + class B: + count = 0 + def __buffer__(self, flags): + self.count += 1 + if self.count == 1: + return memoryview(b"AAA") + else: + return memoryview(b"BBBBBBBBB") + + memio = self.ioclass(b'0123456789') + memio.seek(2) + b = B() + n = memio.write(b) + + self.assertEqual(b.count, 1) + self.assertEqual(n, 3) + self.assertEqual(memio.getvalue(), b"01AAA56789") + self.assertEqual(memio.tell(), 5) + class TextIOTestMixin: @@ -790,7 +868,10 @@ def test_setstate(self): @support.cpython_only def test_sizeof(self): - basesize = support.calcobjsize('P2n2Pn') + if support.Py_GIL_DISABLED: + basesize = support.calcobjsize('P2n2Pni') + else: + basesize = support.calcobjsize('P2n2Pn') check = self.check_sizeof self.assertEqual(object.__sizeof__(io.BytesIO()), basesize) check(io.BytesIO(), basesize ) @@ -889,6 +970,25 @@ def test_setstate(self): memio.close() self.assertRaises(ValueError, memio.__setstate__, ("closed", "", 0, None)) + def test_write_str_subclass(self): + # Writing a str subclass should use the subclass's unicode data + # directly, not call __str__ on it (which may return a different + # value). gh-149047 + class MyStr(str): + def __str__(self): + return "WRONG" + + s = MyStr("correct") + memio = self.ioclass() + memio.write(s) + self.assertEqual(memio.getvalue(), "correct") + + # Also test the fast path where pos == string_size (STATE_ACCUMULATING) + memio2 = self.ioclass() + memio2.write(MyStr("hello ")) + memio2.write(MyStr("world")) + self.assertEqual(memio2.getvalue(), "hello world") + class CStringIOPickleTest(PyStringIOPickleTest): UnsupportedOperation = io.UnsupportedOperation diff --git a/Lib/test/test_memoryview.py b/Lib/test/test_memoryview.py index 61b068c630c7ce9..7b04a4ddd837350 100644 --- a/Lib/test/test_memoryview.py +++ b/Lib/test/test_memoryview.py @@ -387,6 +387,20 @@ def test_hash_writable(self): m = self._view(b) self.assertRaises(ValueError, hash, m) + def test_hash_use_after_free(self): + # Prevent crash in memoryview(v).__hash__ with re-entrant v.__hash__. + # Regression test for https://github.com/python/cpython/issues/142664. + class E(array.array): + def __hash__(self): + mv.release() + self.clear() + return 123 + + v = E('B', b'A' * 4096) + mv = memoryview(v).toreadonly() # must be read-only for hash() + self.assertRaises(BufferError, hash, mv) + self.assertRaises(BufferError, mv.__hash__) + def test_weakref(self): # Check memoryviews are weakrefable for tp in self._types: @@ -442,6 +456,20 @@ def test_issue22668(self): self.assertEqual(c.format, "H") self.assertEqual(d.format, "H") + def test_hex_use_after_free(self): + # Prevent UAF in memoryview.hex(sep) with re-entrant sep.__len__. + # Regression test for https://github.com/python/cpython/issues/143195. + ba = bytearray(b'A' * 1024) + mv = memoryview(ba) + + class S(bytes): + def __len__(self): + mv.release() + ba.clear() + return 1 + + self.assertRaises(BufferError, mv.hex, S(b':')) + # Variations on source objects for the buffer: bytes-like objects, then arrays # with itemsize > 1. @@ -547,6 +575,28 @@ def test_array_assign(self): m[:] = new_a self.assertEqual(a, new_a) + def test_boolean_format(self): + # Test '?' format (keep all the checks below for UBSan) + # See github.com/python/cpython/issues/148390. + + # m1a and m1b are equivalent to [False, True, False] + m1a = memoryview(b'\0\2\0').cast('?') + self.assertEqual(m1a.tolist(), [False, True, False]) + m1b = memoryview(b'\0\4\0').cast('?') + self.assertEqual(m1b.tolist(), [False, True, False]) + self.assertEqual(m1a, m1b) + + # m2a and m2b are equivalent to [True, True, True] + m2a = memoryview(b'\1\3\5').cast('?') + self.assertEqual(m2a.tolist(), [True, True, True]) + m2b = memoryview(b'\2\4\6').cast('?') + self.assertEqual(m2b.tolist(), [True, True, True]) + self.assertEqual(m2a, m2b) + + allbytes = bytes(range(256)) + allbytes = memoryview(allbytes).cast('?') + self.assertEqual(allbytes.tolist(), [False] + [True] * 255) + class BytesMemorySliceTest(unittest.TestCase, BaseMemorySliceTests, BaseBytesMemoryTests): @@ -600,6 +650,25 @@ def test_memoryview_hex(self): m2 = m1[::-1] self.assertEqual(m2.hex(), '30' * 200000) + def test_memoryview_hex_separator(self): + x = bytes(range(97, 102)) + m1 = memoryview(x) + m2 = m1[::-1] + self.assertEqual(m2.hex(':'), '65:64:63:62:61') + self.assertEqual(m2.hex(':', 2), '65:6463:6261') + self.assertEqual(m2.hex(':', -2), '6564:6362:61') + self.assertEqual(m2.hex(sep=':', bytes_per_sep=2), '65:6463:6261') + self.assertEqual(m2.hex(sep=':', bytes_per_sep=-2), '6564:6362:61') + for bytes_per_sep in 5, -5, 2**31-1, -(2**31-1): + with self.subTest(bytes_per_sep=bytes_per_sep): + self.assertEqual(m2.hex(':', bytes_per_sep), '6564636261') + for bytes_per_sep in 2**31, -2**31, 2**1000, -2**1000: + with self.subTest(bytes_per_sep=bytes_per_sep): + try: + self.assertEqual(m2.hex(':', bytes_per_sep), '6564636261') + except OverflowError: + pass + def test_copy(self): m = memoryview(b'abc') with self.assertRaises(TypeError): @@ -743,19 +812,21 @@ def test_racing_getbuf_and_releasebuf(self): from multiprocessing.managers import SharedMemoryManager except ImportError: self.skipTest("Test requires multiprocessing") - from threading import Thread + from threading import Thread, Event - n = 100 + start = Event() with SharedMemoryManager() as smm: obj = smm.ShareableList(range(100)) - threads = [] - for _ in range(n): - # Issue gh-127085, the `ShareableList.count` is just a convenient way to mess the `exports` - # counter of `memoryview`, this issue has no direct relation with `ShareableList`. - threads.append(Thread(target=obj.count, args=(1,))) - + def test(): + # Issue gh-127085, the `ShareableList.count` is just a + # convenient way to mess the `exports` counter of `memoryview`, + # this issue has no direct relation with `ShareableList`. + start.wait(support.SHORT_TIMEOUT) + for i in range(10): + obj.count(1) + threads = [Thread(target=test) for _ in range(10)] with threading_helper.start_threads(threads): - pass + start.set() del obj diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index fb57d5e5544c125..a815ef6a24c1a5f 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -6,8 +6,9 @@ import unittest.mock from platform import win32_edition from test import support -from test.support import cpython_only, force_not_colorized, os_helper +from test.support import cpython_only, force_not_colorized, os_helper, requires_subprocess from test.support.import_helper import ensure_lazy_imports +from test.support.script_helper import assert_python_ok, assert_python_failure try: import _winapi @@ -30,50 +31,11 @@ def tearDownModule(): mimetypes.knownfiles = knownfiles -class MimeTypesTestCase(unittest.TestCase): +class MimeTypesModuleTestCase(unittest.TestCase): def setUp(self): - self.db = mimetypes.MimeTypes() - - def test_case_sensitivity(self): - eq = self.assertEqual - eq(self.db.guess_file_type("foobar.html"), ("text/html", None)) - eq(self.db.guess_type("scheme:foobar.html"), ("text/html", None)) - eq(self.db.guess_file_type("foobar.HTML"), ("text/html", None)) - eq(self.db.guess_type("scheme:foobar.HTML"), ("text/html", None)) - eq(self.db.guess_file_type("foobar.tgz"), ("application/x-tar", "gzip")) - eq(self.db.guess_type("scheme:foobar.tgz"), ("application/x-tar", "gzip")) - eq(self.db.guess_file_type("foobar.TGZ"), ("application/x-tar", "gzip")) - eq(self.db.guess_type("scheme:foobar.TGZ"), ("application/x-tar", "gzip")) - eq(self.db.guess_file_type("foobar.tar.Z"), ("application/x-tar", "compress")) - eq(self.db.guess_type("scheme:foobar.tar.Z"), ("application/x-tar", "compress")) - eq(self.db.guess_file_type("foobar.tar.z"), (None, None)) - eq(self.db.guess_type("scheme:foobar.tar.z"), (None, None)) - - def test_default_data(self): - eq = self.assertEqual - eq(self.db.guess_file_type("foo.html"), ("text/html", None)) - eq(self.db.guess_file_type("foo.HTML"), ("text/html", None)) - eq(self.db.guess_file_type("foo.tgz"), ("application/x-tar", "gzip")) - eq(self.db.guess_file_type("foo.tar.gz"), ("application/x-tar", "gzip")) - eq(self.db.guess_file_type("foo.tar.Z"), ("application/x-tar", "compress")) - eq(self.db.guess_file_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) - eq(self.db.guess_file_type("foo.tar.xz"), ("application/x-tar", "xz")) - - def test_data_urls(self): - eq = self.assertEqual - guess_type = self.db.guess_type - eq(guess_type("data:invalidDataWithoutComma"), (None, None)) - eq(guess_type("data:,thisIsTextPlain"), ("text/plain", None)) - eq(guess_type("data:;base64,thisIsTextPlain"), ("text/plain", None)) - eq(guess_type("data:text/x-foo,thisIsTextXFoo"), ("text/x-foo", None)) - - def test_file_parsing(self): - eq = self.assertEqual - sio = io.StringIO("x-application/x-unittest pyunit\n") - self.db.readfp(sio) - eq(self.db.guess_file_type("foo.pyunit"), - ("x-application/x-unittest", None)) - eq(self.db.guess_extension("x-application/x-unittest"), ".pyunit") + mimetypes.inited = False + mimetypes._default_mime_types() + mimetypes._db = None def test_read_mime_types(self): eq = self.assertEqual @@ -108,101 +70,6 @@ def test_read_mime_types(self): mock_open.assert_called_with(filename, encoding='utf-8') eq(mime_dict[".Français"], "application/no-mans-land") - def test_non_standard_types(self): - eq = self.assertEqual - # First try strict - eq(self.db.guess_file_type('foo.xul', strict=True), (None, None)) - eq(self.db.guess_extension('image/jpg', strict=True), None) - # And then non-strict - eq(self.db.guess_file_type('foo.xul', strict=False), ('text/xul', None)) - eq(self.db.guess_file_type('foo.XUL', strict=False), ('text/xul', None)) - eq(self.db.guess_file_type('foo.invalid', strict=False), (None, None)) - eq(self.db.guess_extension('image/jpg', strict=False), '.jpg') - eq(self.db.guess_extension('image/JPG', strict=False), '.jpg') - - def test_filename_with_url_delimiters(self): - # bpo-38449: URL delimiters cases should be handled also. - # They would have different mime types if interpreted as URL as - # compared to when interpreted as filename because of the semicolon. - eq = self.assertEqual - gzip_expected = ('application/x-tar', 'gzip') - for name in ( - ';1.tar.gz', - '?1.tar.gz', - '#1.tar.gz', - '#1#.tar.gz', - ';1#.tar.gz', - ';&1=123;?.tar.gz', - '?k1=v1&k2=v2.tar.gz', - ): - for prefix in ('', '/', '\\', - 'c:', 'c:/', 'c:\\', 'c:/d/', 'c:\\d\\', - '//share/server/', '\\\\share\\server\\'): - path = prefix + name - with self.subTest(path=path): - eq(self.db.guess_file_type(path), gzip_expected) - eq(self.db.guess_type(path), gzip_expected) - expected = (None, None) if os.name == 'nt' else gzip_expected - for prefix in ('//', '\\\\', '//share/', '\\\\share\\'): - path = prefix + name - with self.subTest(path=path): - eq(self.db.guess_file_type(path), expected) - eq(self.db.guess_type(path), expected) - eq(self.db.guess_file_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) - eq(self.db.guess_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) - - eq(self.db.guess_file_type(r'foo/.tar.gz'), (None, 'gzip')) - eq(self.db.guess_type(r'foo/.tar.gz'), (None, 'gzip')) - expected = (None, 'gzip') if os.name == 'nt' else gzip_expected - eq(self.db.guess_file_type(r'foo\.tar.gz'), expected) - eq(self.db.guess_type(r'foo\.tar.gz'), expected) - eq(self.db.guess_type(r'scheme:foo\.tar.gz'), gzip_expected) - - def test_url(self): - result = self.db.guess_type('http://example.com/host.html') - result = self.db.guess_type('http://host.html') - msg = 'URL only has a host name, not a file' - self.assertSequenceEqual(result, (None, None), msg) - result = self.db.guess_type('http://example.com/host.html') - msg = 'Should be text/html' - self.assertSequenceEqual(result, ('text/html', None), msg) - result = self.db.guess_type('http://example.com/host.html#x.tar') - self.assertSequenceEqual(result, ('text/html', None)) - result = self.db.guess_type('http://example.com/host.html?q=x.tar') - self.assertSequenceEqual(result, ('text/html', None)) - - def test_guess_all_types(self): - # First try strict. Use a set here for testing the results because if - # test_urllib2 is run before test_mimetypes, global state is modified - # such that the 'all' set will have more items in it. - all = self.db.guess_all_extensions('text/plain', strict=True) - self.assertTrue(set(all) >= {'.bat', '.c', '.h', '.ksh', '.pl', '.txt'}) - self.assertEqual(len(set(all)), len(all)) # no duplicates - # And now non-strict - all = self.db.guess_all_extensions('image/jpg', strict=False) - self.assertEqual(all, ['.jpg']) - # And now for no hits - all = self.db.guess_all_extensions('image/jpg', strict=True) - self.assertEqual(all, []) - # And now for type existing in both strict and non-strict mappings. - self.db.add_type('test-type', '.strict-ext') - self.db.add_type('test-type', '.non-strict-ext', strict=False) - all = self.db.guess_all_extensions('test-type', strict=False) - self.assertEqual(all, ['.strict-ext', '.non-strict-ext']) - all = self.db.guess_all_extensions('test-type') - self.assertEqual(all, ['.strict-ext']) - # Test that changing the result list does not affect the global state - all.append('.no-such-ext') - all = self.db.guess_all_extensions('test-type') - self.assertNotIn('.no-such-ext', all) - - def test_encoding(self): - filename = support.findfile("mime.types") - mimes = mimetypes.MimeTypes([filename]) - exts = mimes.guess_all_extensions('application/vnd.geocube+xml', - strict=True) - self.assertEqual(exts, ['.g3', '.g\xb3']) - def test_init_reinitializes(self): # Issue 4936: make sure an init starts clean # First, put some poison into the types table @@ -335,6 +202,203 @@ def test_init_stability(self): self.assertEqual(types_map, mimetypes.types_map) self.assertEqual(common_types, mimetypes.common_types) + def test_init_files(self): + guess_file_type = mimetypes.guess_file_type + self.assertEqual(guess_file_type("file.test2")[0], None) + + filename = support.findfile("mime.types2") + mimetypes.init([filename]) + self.assertEqual(guess_file_type("file.test2")[0], "testing/test2") + + mimetypes.init() + self.assertEqual(guess_file_type("file.test2")[0], None) + + def test_init_knownfiles(self): + guess_file_type = mimetypes.guess_file_type + self.assertEqual(guess_file_type("file.test2")[0], None) + + filename = support.findfile("mime.types2") + mimetypes.knownfiles = [filename, "non-existent"] + self.addCleanup(mimetypes.knownfiles.clear) + + mimetypes.init() + self.assertEqual(guess_file_type("file.test2")[0], "testing/test2") + + def test_added_types_are_used(self): + mimetypes.add_type('testing/default-type', '') + mime_type, _ = mimetypes.guess_type('') + self.assertEqual(mime_type, 'testing/default-type') + + mime_type, _ = mimetypes.guess_type('test.myext') + self.assertEqual(mime_type, None) + + mimetypes.add_type('testing/type', '.myext') + mime_type, _ = mimetypes.guess_type('test.myext') + self.assertEqual(mime_type, 'testing/type') + + def test_add_type_with_undotted_extension_deprecated(self): + with self.assertWarns(DeprecationWarning): + mimetypes.add_type("testing/type", "undotted") + + +class MimeTypesClassTestCase(unittest.TestCase): + def setUp(self): + self.db = mimetypes.MimeTypes() + + def test_init_files(self): + guess_file_type = self.db.guess_file_type + self.assertEqual(guess_file_type("file.test2")[0], None) + + filename = support.findfile("mime.types2") + db = mimetypes.MimeTypes([filename]) + guess_file_type = db.guess_file_type + self.assertEqual(guess_file_type("file.test2")[0], "testing/test2") + + def test_init_knownfiles(self): + filename = support.findfile("mime.types2") + mimetypes.knownfiles = [filename, "non-existent"] + self.addCleanup(mimetypes.knownfiles.clear) + + db = mimetypes.MimeTypes() + guess_file_type = db.guess_file_type + self.assertEqual(guess_file_type("file.test2")[0], None) + + def test_case_sensitivity(self): + eq = self.assertEqual + eq(self.db.guess_file_type("foobar.html"), ("text/html", None)) + eq(self.db.guess_type("scheme:foobar.html"), ("text/html", None)) + eq(self.db.guess_file_type("foobar.HTML"), ("text/html", None)) + eq(self.db.guess_type("scheme:foobar.HTML"), ("text/html", None)) + eq(self.db.guess_file_type("foobar.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_type("scheme:foobar.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foobar.TGZ"), ("application/x-tar", "gzip")) + eq(self.db.guess_type("scheme:foobar.TGZ"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foobar.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_type("scheme:foobar.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_file_type("foobar.tar.z"), (None, None)) + eq(self.db.guess_type("scheme:foobar.tar.z"), (None, None)) + + def test_default_data(self): + eq = self.assertEqual + eq(self.db.guess_file_type("foo.html"), ("text/html", None)) + eq(self.db.guess_file_type("foo.HTML"), ("text/html", None)) + eq(self.db.guess_file_type("foo.tgz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foo.tar.gz"), ("application/x-tar", "gzip")) + eq(self.db.guess_file_type("foo.tar.Z"), ("application/x-tar", "compress")) + eq(self.db.guess_file_type("foo.tar.bz2"), ("application/x-tar", "bzip2")) + eq(self.db.guess_file_type("foo.tar.xz"), ("application/x-tar", "xz")) + + def test_data_urls(self): + eq = self.assertEqual + guess_type = self.db.guess_type + eq(guess_type("data:invalidDataWithoutComma"), (None, None)) + eq(guess_type("data:,thisIsTextPlain"), ("text/plain", None)) + eq(guess_type("data:;base64,thisIsTextPlain"), ("text/plain", None)) + eq(guess_type("data:text/x-foo,thisIsTextXFoo"), ("text/x-foo", None)) + + def test_file_parsing(self): + eq = self.assertEqual + sio = io.StringIO("x-application/x-unittest pyunit\n") + self.db.readfp(sio) + eq(self.db.guess_file_type("foo.pyunit"), + ("x-application/x-unittest", None)) + eq(self.db.guess_extension("x-application/x-unittest"), ".pyunit") + + def test_non_standard_types(self): + eq = self.assertEqual + # First try strict + eq(self.db.guess_file_type('foo.xul', strict=True), (None, None)) + eq(self.db.guess_extension('image/jpg', strict=True), None) + # And then non-strict + eq(self.db.guess_file_type('foo.xul', strict=False), ('text/xul', None)) + eq(self.db.guess_file_type('foo.XUL', strict=False), ('text/xul', None)) + eq(self.db.guess_file_type('foo.invalid', strict=False), (None, None)) + eq(self.db.guess_extension('image/jpg', strict=False), '.jpg') + eq(self.db.guess_extension('image/JPG', strict=False), '.jpg') + + def test_filename_with_url_delimiters(self): + # bpo-38449: URL delimiters cases should be handled also. + # They would have different mime types if interpreted as URL as + # compared to when interpreted as filename because of the semicolon. + eq = self.assertEqual + gzip_expected = ('application/x-tar', 'gzip') + for name in ( + ';1.tar.gz', + '?1.tar.gz', + '#1.tar.gz', + '#1#.tar.gz', + ';1#.tar.gz', + ';&1=123;?.tar.gz', + '?k1=v1&k2=v2.tar.gz', + ): + for prefix in ('', '/', '\\', + 'c:', 'c:/', 'c:\\', 'c:/d/', 'c:\\d\\', + '//share/server/', '\\\\share\\server\\'): + path = prefix + name + with self.subTest(path=path): + eq(self.db.guess_file_type(path), gzip_expected) + eq(self.db.guess_type(path), gzip_expected) + expected = (None, None) if os.name == 'nt' else gzip_expected + for prefix in ('//', '\\\\', '//share/', '\\\\share\\'): + path = prefix + name + with self.subTest(path=path): + eq(self.db.guess_file_type(path), expected) + eq(self.db.guess_type(path), expected) + eq(self.db.guess_file_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) + eq(self.db.guess_type(r" \"\`;b&b&c |.tar.gz"), gzip_expected) + + eq(self.db.guess_file_type(r'foo/.tar.gz'), (None, 'gzip')) + eq(self.db.guess_type(r'foo/.tar.gz'), (None, 'gzip')) + expected = (None, 'gzip') if os.name == 'nt' else gzip_expected + eq(self.db.guess_file_type(r'foo\.tar.gz'), expected) + eq(self.db.guess_type(r'foo\.tar.gz'), expected) + eq(self.db.guess_type(r'scheme:foo\.tar.gz'), gzip_expected) + + def test_url(self): + result = self.db.guess_type('http://example.com/host.html') + result = self.db.guess_type('http://host.html') + msg = 'URL only has a host name, not a file' + self.assertSequenceEqual(result, (None, None), msg) + result = self.db.guess_type('http://example.com/host.html') + msg = 'Should be text/html' + self.assertSequenceEqual(result, ('text/html', None), msg) + result = self.db.guess_type('http://example.com/host.html#x.tar') + self.assertSequenceEqual(result, ('text/html', None)) + result = self.db.guess_type('http://example.com/host.html?q=x.tar') + self.assertSequenceEqual(result, ('text/html', None)) + + def test_guess_all_types(self): + # First try strict. Use a set here for testing the results because if + # test_urllib2 is run before test_mimetypes, global state is modified + # such that the 'all' set will have more items in it. + all = self.db.guess_all_extensions('text/plain', strict=True) + self.assertTrue(set(all) >= {'.bat', '.c', '.h', '.ksh', '.pl', '.txt'}) + self.assertEqual(len(set(all)), len(all)) # no duplicates + # And now non-strict + all = self.db.guess_all_extensions('image/jpg', strict=False) + self.assertEqual(all, ['.jpg']) + # And now for no hits + all = self.db.guess_all_extensions('image/jpg', strict=True) + self.assertEqual(all, []) + # And now for type existing in both strict and non-strict mappings. + self.db.add_type('test-type', '.strict-ext') + self.db.add_type('test-type', '.non-strict-ext', strict=False) + all = self.db.guess_all_extensions('test-type', strict=False) + self.assertEqual(all, ['.strict-ext', '.non-strict-ext']) + all = self.db.guess_all_extensions('test-type') + self.assertEqual(all, ['.strict-ext']) + # Test that changing the result list does not affect the global state + all.append('.no-such-ext') + all = self.db.guess_all_extensions('test-type') + self.assertNotIn('.no-such-ext', all) + + def test_encoding(self): + filename = support.findfile("mime.types") + mimes = mimetypes.MimeTypes([filename]) + exts = mimes.guess_all_extensions('application/vnd.geocube+xml', + strict=True) + self.assertEqual(exts, ['.g3', '.g\xb3']) + def test_path_like_ob(self): filename = "LICENSE.txt" filepath = os_helper.FakePath(filename) @@ -374,22 +438,6 @@ def test_keywords_args_api(self): self.assertEqual(self.db.guess_extension( type='image/jpg', strict=False), '.jpg') - def test_added_types_are_used(self): - mimetypes.add_type('testing/default-type', '') - mime_type, _ = mimetypes.guess_type('') - self.assertEqual(mime_type, 'testing/default-type') - - mime_type, _ = mimetypes.guess_type('test.myext') - self.assertEqual(mime_type, None) - - mimetypes.add_type('testing/type', '.myext') - mime_type, _ = mimetypes.guess_type('test.myext') - self.assertEqual(mime_type, 'testing/type') - - def test_add_type_with_undotted_extension_deprecated(self): - with self.assertWarns(DeprecationWarning): - mimetypes.add_type("testing/type", "undotted") - @unittest.skipUnless(sys.platform.startswith("win"), "Windows only") class Win32MimeTypesTestCase(unittest.TestCase): @@ -470,13 +518,31 @@ def test_parse_args(self): self.assertFalse(args.lenient) self.assertEqual(args.type, ["foo.pic"]) + def test_multiple_inputs(self): + result = "\n".join(mimetypes._main(shlex.split("foo.pdf foo.png"))) + self.assertEqual( + result, + "type: application/pdf encoding: None\n" + "type: image/png encoding: None" + ) + + def test_multiple_inputs_error(self): + result = "\n".join(mimetypes._main(shlex.split("foo.pdf foo.bar_ext"))) + self.assertEqual( + result, + "type: application/pdf encoding: None\n" + "error: media type unknown for foo.bar_ext" + ) + + def test_invocation(self): for command, expected in [ ("-l -e image/jpg", ".jpg"), ("-e image/jpeg", ".jpg"), ("-l foo.webp", "type: image/webp encoding: None"), ]: - self.assertEqual(mimetypes._main(shlex.split(command)), expected) + result = "\n".join(mimetypes._main(shlex.split(command))) + self.assertEqual(result, expected) def test_invocation_error(self): for command, expected in [ @@ -484,8 +550,62 @@ def test_invocation_error(self): ("foo.bar_ext", "error: media type unknown for foo.bar_ext"), ]: with self.subTest(command=command): - with self.assertRaisesRegex(SystemExit, expected): - mimetypes._main(shlex.split(command)) + result = "\n".join(mimetypes._main(shlex.split(command))) + self.assertEqual(result, expected) + + +@requires_subprocess() +class CommandLineSubprocessTest(unittest.TestCase): + def test_help(self): + rc, stdout, stderr = assert_python_ok('-m', 'mimetypes', '--help') + self.assertIn(b'mimetypes', stdout) + self.assertIn(b'--extension', stdout) + self.assertIn(b'--lenient', stdout) + + def test_type_lookup(self): + rc, stdout, stderr = assert_python_ok('-m', 'mimetypes', 'foo.pdf') + self.assertEqual(stdout.strip(), b'type: application/pdf encoding: None') + self.assertEqual(stderr, b'') + + def test_type_lookup_unknown(self): + rc, stdout, stderr = assert_python_failure('-m', 'mimetypes', 'foo.unknownext12345') + self.assertEqual(stdout.strip(), b'error: media type unknown for foo.unknownext12345') + self.assertEqual(stderr, b'') + + def test_extension_flag(self): + rc, stdout, stderr = assert_python_ok('-m', 'mimetypes', '-e', 'image/jpeg') + self.assertEqual(stdout.strip(), b'.jpg') + self.assertEqual(stderr, b'') + + def test_extension_flag_unknown(self): + rc, stdout, stderr = assert_python_failure('-m', 'mimetypes', '-e', 'image/unknowntype12345') + self.assertEqual(stdout.strip(), b'error: unknown type image/unknowntype12345') + self.assertEqual(stderr, b'') + + def test_lenient_flag(self): + rc, stdout, stderr = assert_python_ok('-m', 'mimetypes', '-e', '--lenient', 'text/xul') + self.assertIn(b'.xul', stdout) + self.assertEqual(stderr, b'') + + def test_multiple_inputs(self): + rc, stdout, stderr = assert_python_ok('-m', 'mimetypes', 'foo.pdf', 'foo.png') + self.assertIn(b'type: application/pdf encoding: None', stdout) + self.assertIn(b'type: image/png encoding: None', stdout) + self.assertEqual(stderr, b'') + + def test_multiple_inputs_with_error(self): + rc, stdout, stderr = assert_python_failure( + '-m', 'mimetypes', 'foo.pdf', 'foo.unknownext12345' + ) + self.assertIn(b'type: application/pdf encoding: None', stdout) + self.assertIn(b'error: media type unknown for foo.unknownext12345', stdout) + self.assertEqual(stderr, b'') + + @force_not_colorized + def test_unknown_flag(self): + rc, stdout, stderr = assert_python_failure('-m', 'mimetypes', '--unknown-flag', 'foo.pdf') + self.assertEqual(stdout, b'') + self.assertIn(b'error: unrecognized arguments: --unknown-flag', stderr) if __name__ == "__main__": diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py index 4f25e9c2a03cb4d..46249e5138aed52 100644 --- a/Lib/test/test_minidom.py +++ b/Lib/test/test_minidom.py @@ -8,7 +8,7 @@ import xml.dom.minidom -from xml.dom.minidom import parse, Attr, Node, Document, parseString +from xml.dom.minidom import parse, Attr, Node, Document, Element, parseString from xml.dom.minidom import getDOMImplementation from xml.parsers.expat import ExpatError @@ -173,6 +173,49 @@ def testAppendChild(self): self.assertEqual(dom.documentElement.childNodes[-1].data, "Hello") dom.unlink() + @support.requires_resource('cpu') + def testAppendChildNoQuadraticComplexity(self): + impl = getDOMImplementation() + + def work(n): + doc = impl.createDocument(None, "some_tag", None) + element = doc.documentElement + total_calls = 0 + + # Count attribute accesses as a proxy for work done + def getattribute_counter(self, attr): + nonlocal total_calls + total_calls += 1 + return object.__getattribute__(self, attr) + + with support.swap_attr(Element, "__getattribute__", getattribute_counter): + for _ in range(n): + child = doc.createElement("child") + element.appendChild(child) + element = child + return total_calls + + # Doubling N should not ~quadruple the work. + w1 = work(1024) + w2 = work(2048) + w3 = work(4096) + + self.assertGreater(w1, 0) + r1 = w2 / w1 + r2 = w3 / w2 + self.assertLess( + max(r1, r2), 3.2, + msg=f"Possible quadratic behavior: work={w1,w2,w3} ratios={r1,r2}" + ) + + def testSetAttributeNodeWithoutOwnerDocument(self): + # regression test for gh-142754 + elem = Element("test") + attr = Attr("id") + attr.value = "test-id" + elem.setAttributeNode(attr) + self.assertEqual(elem.getAttribute("id"), "test-id") + def testAppendChildFragment(self): dom, orig, c1, c2, c3, frag = self._create_fragment_test_nodes() dom.documentElement.appendChild(frag) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index fd4197b7086976b..93be6c2830357ed 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -732,7 +732,7 @@ def test_tagname(self): m2.close() m1.close() - with self.assertRaisesRegex(TypeError, 'must be str or None'): + with self.assertRaisesRegex(TypeError, 'tagname'): mmap.mmap(-1, 8, tagname=1) @cpython_only @@ -887,6 +887,10 @@ def test_madvise(self): size = 2 * PAGESIZE m = mmap.mmap(-1, size) + class Number: + def __index__(self): + return 2 + with self.assertRaisesRegex(ValueError, "madvise start out of bounds"): m.madvise(mmap.MADV_NORMAL, size) with self.assertRaisesRegex(ValueError, "madvise start out of bounds"): @@ -895,41 +899,80 @@ def test_madvise(self): m.madvise(mmap.MADV_NORMAL, 0, -1) with self.assertRaisesRegex(OverflowError, "madvise length too large"): m.madvise(mmap.MADV_NORMAL, PAGESIZE, sys.maxsize) + with self.assertRaisesRegex( + TypeError, "'str' object cannot be interpreted as an integer"): + m.madvise(mmap.MADV_NORMAL, PAGESIZE, "Not a Number") self.assertEqual(m.madvise(mmap.MADV_NORMAL), None) self.assertEqual(m.madvise(mmap.MADV_NORMAL, PAGESIZE), None) self.assertEqual(m.madvise(mmap.MADV_NORMAL, PAGESIZE, size), None) self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, 2), None) + self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, Number()), None) self.assertEqual(m.madvise(mmap.MADV_NORMAL, 0, size), None) - @unittest.skipUnless(os.name == 'nt', 'requires Windows') - def test_resize_up_when_mapped_to_pagefile(self): + def test_resize_up_anonymous_mapping(self): """If the mmap is backed by the pagefile ensure a resize up can happen and that the original data is still in place """ start_size = PAGESIZE new_size = 2 * start_size - data = bytes(random.getrandbits(8) for _ in range(start_size)) + data = random.randbytes(start_size) + + with mmap.mmap(-1, start_size) as m: + m[:] = data + if sys.platform.startswith(('linux', 'android', 'netbsd')): + # Can't expand a shared anonymous mapping on Linux + # (see https://bugzilla.kernel.org/show_bug.cgi?id=8691) + # or NetBSD. + with self.assertRaises(ValueError): + m.resize(new_size) + else: + try: + m.resize(new_size) + except SystemError: + pass + else: + self.assertEqual(len(m), new_size) + self.assertEqual(m[:start_size], data) + self.assertEqual(m[start_size:], b'\0' * (new_size - start_size)) - m = mmap.mmap(-1, start_size) - m[:] = data - m.resize(new_size) - self.assertEqual(len(m), new_size) - self.assertEqual(m[:start_size], data[:start_size]) + @unittest.skipUnless(os.name == 'posix', 'requires Posix') + def test_resize_up_private_anonymous_mapping(self): + start_size = PAGESIZE + new_size = 2 * start_size + data = random.randbytes(start_size) - @unittest.skipUnless(os.name == 'nt', 'requires Windows') - def test_resize_down_when_mapped_to_pagefile(self): + with mmap.mmap(-1, start_size, flags=mmap.MAP_PRIVATE) as m: + m[:] = data + try: + m.resize(new_size) + except SystemError: + pass + else: + self.assertEqual(len(m), new_size) + self.assertEqual(m[:start_size], data) + self.assertEqual(m[start_size:], b'\0' * (new_size - start_size)) + + def test_resize_down_anonymous_mapping(self): """If the mmap is backed by the pagefile ensure a resize down up can happen and that a truncated form of the original data is still in place """ - start_size = PAGESIZE + start_size = 2 * PAGESIZE new_size = start_size // 2 - data = bytes(random.getrandbits(8) for _ in range(start_size)) + data = random.randbytes(start_size) - m = mmap.mmap(-1, start_size) - m[:] = data - m.resize(new_size) - self.assertEqual(len(m), new_size) - self.assertEqual(m[:new_size], data[:new_size]) + with mmap.mmap(-1, start_size) as m: + m[:] = data + try: + m.resize(new_size) + except SystemError: + pass + else: + self.assertEqual(len(m), new_size) + self.assertEqual(m[:], data[:new_size]) + if sys.platform.startswith(('linux', 'android')): + # Can't expand to its original size. + with self.assertRaises(ValueError): + m.resize(start_size) @unittest.skipUnless(os.name == 'nt', 'requires Windows') def test_resize_fails_if_mapping_held_elsewhere(self): diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 263e4e6f3941557..ffa1fd957eb4561 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -3,6 +3,7 @@ import collections import dis import functools +import inspect import math import operator import sys @@ -11,10 +12,7 @@ import unittest import test.support -from test.support import requires_specialization_ft, script_helper - -_testcapi = test.support.import_helper.import_module("_testcapi") -_testinternalcapi = test.support.import_helper.import_module("_testinternalcapi") +from test.support import import_helper, requires_specialization_ft, script_helper PAIR = (0,1) @@ -881,6 +879,8 @@ def test_implicit_stop_iteration(self): This test checks that both paths record an equivalent event. """ + _testinternalcapi = import_helper.import_module("_testinternalcapi") + def gen(): yield 1 return 2 @@ -1048,6 +1048,8 @@ def func(): @requires_specialization_ft def test_no_unwind_for_shim_frame(self): + _testinternalcapi = import_helper.import_module("_testinternalcapi") + class ValueErrorRaiser: def __init__(self): raise ValueError() @@ -1078,6 +1080,25 @@ def f(): self.assertEqual(events, expected) + # gh-140373 + def test_gen_unwind(self): + def gen(): + yield 1 + + def f(): + g = gen() + next(g) + g.close() + + recorders = ( + UnwindRecorder, + ) + events = self.get_events(f, TEST_TOOL, recorders) + expected = [ + ("unwind", GeneratorExit, "gen"), + ] + self.assertEqual(events, expected) + class LineRecorder: event_type = E.LINE @@ -1709,6 +1730,27 @@ def func(v=1): ('branch right', 'func', 6, 8), ('branch right', 'func', 2, 10)]) + def test_callback_set_frame_lineno(self): + def func(s: str) -> int: + if s.startswith("t"): + return 1 + else: + return 0 + + def callback(code, from_, to): + # try set frame.f_lineno + frame = inspect.currentframe() + while frame and frame.f_code is not code: + frame = frame.f_back + + self.assertIsNotNone(frame) + frame.f_lineno = frame.f_lineno + 1 # run next instruction + + sys.monitoring.set_local_events(TEST_TOOL, func.__code__, E.BRANCH_LEFT) + sys.monitoring.register_callback(TEST_TOOL, E.BRANCH_LEFT, callback) + + self.assertEqual(func("true"), 1) + class TestBranchConsistency(MonitoringTestBase, unittest.TestCase): @@ -2100,6 +2142,7 @@ def run(): sys.monitoring.set_events(TEST_TOOL, 0) def test_108390(self): + _testinternalcapi = import_helper.import_module("_testinternalcapi") class Foo: def __init__(self, set_event): @@ -2176,6 +2219,8 @@ def test_func(x): class TestTier2Optimizer(CheckEvents): def test_monitoring_already_opimized_loop(self): + _testinternalcapi = import_helper.import_module("_testinternalcapi") + def test_func(recorder): set_events = sys.monitoring.set_events line = E.LINE @@ -2212,18 +2257,20 @@ class TestCApiEventGeneration(MonitoringTestBase, unittest.TestCase): class Scope: def __init__(self, *args): + self._testcapi = import_helper.import_module("_testcapi") self.args = args def __enter__(self): - _testcapi.monitoring_enter_scope(*self.args) + self._testcapi.monitoring_enter_scope(*self.args) def __exit__(self, *args): - _testcapi.monitoring_exit_scope() + self._testcapi.monitoring_exit_scope() def setUp(self): super(TestCApiEventGeneration, self).setUp() - capi = _testcapi + self._testcapi = import_helper.import_module("_testcapi") + capi = self._testcapi self.codelike = capi.CodeLike(2) @@ -2292,7 +2339,7 @@ def __call__(self, *args): def test_fire_event(self): for expected, event, function, *args in self.cases: offset = 0 - self.codelike = _testcapi.CodeLike(1) + self.codelike = self._testcapi.CodeLike(1) with self.subTest(function.__name__): args_ = (self.codelike, offset) + tuple(args) self.check_event_count(event, function, args_, expected) @@ -2303,7 +2350,7 @@ def test_missing_exception(self): continue assert args and isinstance(args[-1], BaseException) offset = 0 - self.codelike = _testcapi.CodeLike(1) + self.codelike = self._testcapi.CodeLike(1) with self.subTest(function.__name__): args_ = (self.codelike, offset) + tuple(args[:-1]) + (None,) evt = int(math.log2(event)) @@ -2313,7 +2360,7 @@ def test_missing_exception(self): def test_fire_event_failing_callback(self): for expected, event, function, *args in self.cases: offset = 0 - self.codelike = _testcapi.CodeLike(1) + self.codelike = self._testcapi.CodeLike(1) with self.subTest(function.__name__): args_ = (self.codelike, offset) + tuple(args) exc = OSError(42) @@ -2363,12 +2410,14 @@ def check_disable(self, event, func, args, expected): def test_disable_event(self): for expected, event, function, *args in self.cases: offset = 0 - self.codelike = _testcapi.CodeLike(2) + self.codelike = self._testcapi.CodeLike(2) with self.subTest(function.__name__): args_ = (self.codelike, 0) + tuple(args) self.check_disable(event, function, args_, expected) def test_enter_scope_two_events(self): + _testcapi = self._testcapi + try: yield_counter = CounterWithDisable() unwind_counter = CounterWithDisable() diff --git a/Lib/test/test_msvcrt.py b/Lib/test/test_msvcrt.py index 1c6905bd1ee5864..fef86ce323e54d5 100644 --- a/Lib/test/test_msvcrt.py +++ b/Lib/test/test_msvcrt.py @@ -4,6 +4,7 @@ import unittest from textwrap import dedent +from test import support from test.support import os_helper, requires_resource from test.support.os_helper import TESTFN, TESTFN_ASCII @@ -67,8 +68,12 @@ def run_in_separated_process(self, code): # Run test in a separated process to avoid stdin conflicts. # See: gh-110147 cmd = [sys.executable, '-c', code] - subprocess.run(cmd, check=True, capture_output=True, - creationflags=subprocess.CREATE_NEW_CONSOLE) + try: + subprocess.run(cmd, check=True, capture_output=True, + creationflags=subprocess.CREATE_NEW_CONSOLE) + except subprocess.CalledProcessError as exc: + support.skip_on_low_desktop_heap_memory_subprocess(exc.returncode) + raise def test_kbhit(self): code = dedent(''' diff --git a/Lib/test/test_multiprocessing_forkserver/__init__.py b/Lib/test/test_multiprocessing_forkserver/__init__.py index d91715a344dfa7f..6bac9b7f9a97134 100644 --- a/Lib/test/test_multiprocessing_forkserver/__init__.py +++ b/Lib/test/test_multiprocessing_forkserver/__init__.py @@ -1,3 +1,4 @@ +import multiprocessing import os.path import sys import unittest @@ -9,5 +10,10 @@ if sys.platform == "win32": raise unittest.SkipTest("forkserver is not available on Windows") +# The forkserver start method requires passing file descriptors over a Unix +# socket, which is not available on every platform (e.g. Solaris/illumos). +if "forkserver" not in multiprocessing.get_all_start_methods(): + raise unittest.SkipTest("forkserver start method is not available") + def load_tests(*args): return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_netrc.py b/Lib/test/test_netrc.py index 81e11a293cc4c8b..9d720f627102e34 100644 --- a/Lib/test/test_netrc.py +++ b/Lib/test/test_netrc.py @@ -1,11 +1,7 @@ import netrc, os, unittest, sys, textwrap +from test import support from test.support import os_helper -try: - import pwd -except ImportError: - pwd = None - temp_filename = os_helper.TESTFN class NetrcTestCase(unittest.TestCase): @@ -269,9 +265,14 @@ def test_comment_at_end_of_machine_line_pass_has_hash(self): machine bar.domain.com login foo password pass """, '#pass') + @unittest.skipUnless(support.is_wasi, 'WASI only test') + def test_security_on_WASI(self): + self.assertFalse(netrc._can_security_check()) + self.assertEqual(netrc._getpwuid(0), 'uid 0') + self.assertEqual(netrc._getpwuid(123456), 'uid 123456') @unittest.skipUnless(os.name == 'posix', 'POSIX only test') - @unittest.skipIf(pwd is None, 'security check requires pwd module') + @unittest.skipUnless(hasattr(os, 'getuid'), "os.getuid is required") @os_helper.skip_unless_working_chmod def test_security(self): # This test is incomplete since we are normally not run as root and diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index c10387b58e3f9c7..9270f3257068d6f 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -6,8 +6,9 @@ import sys import unittest import warnings -from test.support import cpython_only, os_helper -from test.support import TestFailed, is_emscripten +from test import support +from test.support import os_helper +from ntpath import ALLOW_MISSING from test.support.os_helper import FakePath from test import test_genericpath from tempfile import TemporaryFile @@ -57,7 +58,7 @@ def tester(fn, wantResult): fn = fn.replace("\\", "\\\\") gotResult = eval(fn) if wantResult != gotResult and _norm(wantResult) != _norm(gotResult): - raise TestFailed("%s should return: %s but returned: %s" \ + raise support.TestFailed("%s should return: %s but returned: %s" \ %(str(fn), str(wantResult), str(gotResult))) # then with bytes @@ -73,10 +74,14 @@ def tester(fn, wantResult): warnings.simplefilter("ignore", DeprecationWarning) gotResult = eval(fn) if _norm(wantResult) != _norm(gotResult): - raise TestFailed("%s should return: %s but returned: %s" \ + raise support.TestFailed("%s should return: %s but returned: %s" \ %(str(fn), str(wantResult), repr(gotResult))) +def _parameterize(*parameters): + return support.subTests('kwargs', parameters, _do_cleanups=True) + + class NtpathTestCase(unittest.TestCase): def assertPathEqual(self, path1, path2): if path1 == path2 or _norm(path1) == _norm(path2): @@ -124,6 +129,22 @@ def test_splitdrive(self): tester('ntpath.splitdrive("//?/UNC/server/share/dir")', ("//?/UNC/server/share", "/dir")) + def test_splitdrive_invalid_paths(self): + splitdrive = ntpath.splitdrive + self.assertEqual(splitdrive('\\\\ser\x00ver\\sha\x00re\\di\x00r'), + ('\\\\ser\x00ver\\sha\x00re', '\\di\x00r')) + self.assertEqual(splitdrive(b'\\\\ser\x00ver\\sha\x00re\\di\x00r'), + (b'\\\\ser\x00ver\\sha\x00re', b'\\di\x00r')) + self.assertEqual(splitdrive("\\\\\udfff\\\udffe\\\udffd"), + ('\\\\\udfff\\\udffe', '\\\udffd')) + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, splitdrive, b'\\\\\xff\\share\\dir') + self.assertRaises(UnicodeDecodeError, splitdrive, b'\\\\server\\\xff\\dir') + self.assertRaises(UnicodeDecodeError, splitdrive, b'\\\\server\\share\\\xff') + else: + self.assertEqual(splitdrive(b'\\\\\xff\\\xfe\\\xfd'), + (b'\\\\\xff\\\xfe', b'\\\xfd')) + def test_splitroot(self): tester("ntpath.splitroot('')", ('', '', '')) tester("ntpath.splitroot('foo')", ('', '', 'foo')) @@ -214,6 +235,22 @@ def test_splitroot(self): tester('ntpath.splitroot(" :/foo")', (" :", "/", "foo")) tester('ntpath.splitroot("/:/foo")', ("", "/", ":/foo")) + def test_splitroot_invalid_paths(self): + splitroot = ntpath.splitroot + self.assertEqual(splitroot('\\\\ser\x00ver\\sha\x00re\\di\x00r'), + ('\\\\ser\x00ver\\sha\x00re', '\\', 'di\x00r')) + self.assertEqual(splitroot(b'\\\\ser\x00ver\\sha\x00re\\di\x00r'), + (b'\\\\ser\x00ver\\sha\x00re', b'\\', b'di\x00r')) + self.assertEqual(splitroot("\\\\\udfff\\\udffe\\\udffd"), + ('\\\\\udfff\\\udffe', '\\', '\udffd')) + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, splitroot, b'\\\\\xff\\share\\dir') + self.assertRaises(UnicodeDecodeError, splitroot, b'\\\\server\\\xff\\dir') + self.assertRaises(UnicodeDecodeError, splitroot, b'\\\\server\\share\\\xff') + else: + self.assertEqual(splitroot(b'\\\\\xff\\\xfe\\\xfd'), + (b'\\\\\xff\\\xfe', b'\\', b'\xfd')) + def test_split(self): tester('ntpath.split("c:\\foo\\bar")', ('c:\\foo', 'bar')) tester('ntpath.split("\\\\conky\\mountpoint\\foo\\bar")', @@ -226,6 +263,21 @@ def test_split(self): tester('ntpath.split("c:/")', ('c:/', '')) tester('ntpath.split("//conky/mountpoint/")', ('//conky/mountpoint/', '')) + def test_split_invalid_paths(self): + split = ntpath.split + self.assertEqual(split('c:\\fo\x00o\\ba\x00r'), + ('c:\\fo\x00o', 'ba\x00r')) + self.assertEqual(split(b'c:\\fo\x00o\\ba\x00r'), + (b'c:\\fo\x00o', b'ba\x00r')) + self.assertEqual(split('c:\\\udfff\\\udffe'), + ('c:\\\udfff', '\udffe')) + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, split, b'c:\\\xff\\bar') + self.assertRaises(UnicodeDecodeError, split, b'c:\\foo\\\xff') + else: + self.assertEqual(split(b'c:\\\xff\\\xfe'), + (b'c:\\\xff', b'\xfe')) + def test_isabs(self): tester('ntpath.isabs("foo\\bar")', 0) tester('ntpath.isabs("foo/bar")', 0) @@ -333,6 +385,30 @@ def test_join(self): tester("ntpath.join('D:a', './c:b')", 'D:a\\.\\c:b') tester("ntpath.join('D:/a', './c:b')", 'D:\\a\\.\\c:b') + def test_normcase(self): + normcase = ntpath.normcase + self.assertEqual(normcase(''), '') + self.assertEqual(normcase(b''), b'') + self.assertEqual(normcase('ABC'), 'abc') + self.assertEqual(normcase(b'ABC'), b'abc') + self.assertEqual(normcase('\xc4\u0141\u03a8'), '\xe4\u0142\u03c8') + expected = '\u03c9\u2126' if sys.platform == 'win32' else '\u03c9\u03c9' + self.assertEqual(normcase('\u03a9\u2126'), expected) + if sys.platform == 'win32' or sys.getfilesystemencoding() == 'utf-8': + self.assertEqual(normcase('\xc4\u0141\u03a8'.encode()), + '\xe4\u0142\u03c8'.encode()) + self.assertEqual(normcase('\u03a9\u2126'.encode()), + expected.encode()) + + def test_normcase_invalid_paths(self): + normcase = ntpath.normcase + self.assertEqual(normcase('abc\x00def'), 'abc\x00def') + self.assertEqual(normcase(b'abc\x00def'), b'abc\x00def') + self.assertEqual(normcase('\udfff'), '\udfff') + if sys.platform == 'win32': + path = b'ABC' + bytes(range(128, 256)) + self.assertEqual(normcase(path), path.lower()) + def test_normpath(self): tester("ntpath.normpath('A//////././//.//B')", r'A\B') tester("ntpath.normpath('A/./B')", r'A\B') @@ -381,6 +457,21 @@ def test_normpath(self): tester("ntpath.normpath('\\\\')", '\\\\') tester("ntpath.normpath('//?/UNC/server/share/..')", '\\\\?\\UNC\\server\\share\\') + def test_normpath_invalid_paths(self): + normpath = ntpath.normpath + self.assertEqual(normpath('fo\x00o'), 'fo\x00o') + self.assertEqual(normpath(b'fo\x00o'), b'fo\x00o') + self.assertEqual(normpath('fo\x00o\\..\\bar'), 'bar') + self.assertEqual(normpath(b'fo\x00o\\..\\bar'), b'bar') + self.assertEqual(normpath('\udfff'), '\udfff') + self.assertEqual(normpath('\udfff\\..\\foo'), 'foo') + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, normpath, b'\xff') + self.assertRaises(UnicodeDecodeError, normpath, b'\xff\\..\\foo') + else: + self.assertEqual(normpath(b'\xff'), b'\xff') + self.assertEqual(normpath(b'\xff\\..\\foo'), b'foo') + def test_realpath_curdir(self): expected = ntpath.normpath(os.getcwd()) tester("ntpath.realpath('.')", expected) @@ -389,6 +480,27 @@ def test_realpath_curdir(self): tester("ntpath.realpath('.\\.')", expected) tester("ntpath.realpath('\\'.join(['.'] * 100))", expected) + def test_realpath_curdir_strict(self): + expected = ntpath.normpath(os.getcwd()) + tester("ntpath.realpath('.', strict=True)", expected) + tester("ntpath.realpath('./.', strict=True)", expected) + tester("ntpath.realpath('/'.join(['.'] * 100), strict=True)", expected) + tester("ntpath.realpath('.\\.', strict=True)", expected) + tester("ntpath.realpath('\\'.join(['.'] * 100), strict=True)", expected) + + def test_realpath_curdir_missing_ok(self): + expected = ntpath.normpath(os.getcwd()) + tester("ntpath.realpath('.', strict=ALLOW_MISSING)", + expected) + tester("ntpath.realpath('./.', strict=ALLOW_MISSING)", + expected) + tester("ntpath.realpath('/'.join(['.'] * 100), strict=ALLOW_MISSING)", + expected) + tester("ntpath.realpath('.\\.', strict=ALLOW_MISSING)", + expected) + tester("ntpath.realpath('\\'.join(['.'] * 100), strict=ALLOW_MISSING)", + expected) + def test_realpath_pardir(self): expected = ntpath.normpath(os.getcwd()) tester("ntpath.realpath('..')", ntpath.dirname(expected)) @@ -401,28 +513,59 @@ def test_realpath_pardir(self): tester("ntpath.realpath('\\'.join(['..'] * 50))", ntpath.splitdrive(expected)[0] + '\\') + def test_realpath_pardir_strict(self): + expected = ntpath.normpath(os.getcwd()) + tester("ntpath.realpath('..', strict=True)", ntpath.dirname(expected)) + tester("ntpath.realpath('../..', strict=True)", + ntpath.dirname(ntpath.dirname(expected))) + tester("ntpath.realpath('/'.join(['..'] * 50), strict=True)", + ntpath.splitdrive(expected)[0] + '\\') + tester("ntpath.realpath('..\\..', strict=True)", + ntpath.dirname(ntpath.dirname(expected))) + tester("ntpath.realpath('\\'.join(['..'] * 50), strict=True)", + ntpath.splitdrive(expected)[0] + '\\') + + def test_realpath_pardir_missing_ok(self): + expected = ntpath.normpath(os.getcwd()) + tester("ntpath.realpath('..', strict=ALLOW_MISSING)", + ntpath.dirname(expected)) + tester("ntpath.realpath('../..', strict=ALLOW_MISSING)", + ntpath.dirname(ntpath.dirname(expected))) + tester("ntpath.realpath('/'.join(['..'] * 50), strict=ALLOW_MISSING)", + ntpath.splitdrive(expected)[0] + '\\') + tester("ntpath.realpath('..\\..', strict=ALLOW_MISSING)", + ntpath.dirname(ntpath.dirname(expected))) + tester("ntpath.realpath('\\'.join(['..'] * 50), strict=ALLOW_MISSING)", + ntpath.splitdrive(expected)[0] + '\\') + @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') - def test_realpath_basic(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_basic(self, kwargs): ABSTFN = ntpath.abspath(os_helper.TESTFN) open(ABSTFN, "wb").close() self.addCleanup(os_helper.unlink, ABSTFN) self.addCleanup(os_helper.unlink, ABSTFN + "1") os.symlink(ABSTFN, ABSTFN + "1") - self.assertPathEqual(ntpath.realpath(ABSTFN + "1"), ABSTFN) - self.assertPathEqual(ntpath.realpath(os.fsencode(ABSTFN + "1")), + self.assertPathEqual(ntpath.realpath(ABSTFN + "1", **kwargs), ABSTFN) + self.assertPathEqual(ntpath.realpath(os.fsencode(ABSTFN + "1"), **kwargs), os.fsencode(ABSTFN)) # gh-88013: call ntpath.realpath with binary drive name may raise a # TypeError. The drive should not exist to reproduce the bug. drives = {f"{c}:\\" for c in string.ascii_uppercase} - set(os.listdrives()) d = drives.pop().encode() - self.assertEqual(ntpath.realpath(d), d) + self.assertEqual(ntpath.realpath(d, strict=False), d) # gh-106242: Embedded nulls and non-strict fallback to abspath - self.assertEqual(ABSTFN + "\0spam", - ntpath.realpath(os_helper.TESTFN + "\0spam", strict=False)) + if kwargs: + with self.assertRaises(OSError): + ntpath.realpath(os_helper.TESTFN + "\0spam", + **kwargs) + else: + self.assertEqual(ABSTFN + "\0spam", + ntpath.realpath(os_helper.TESTFN + "\0spam", **kwargs)) @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') @@ -434,19 +577,77 @@ def test_realpath_strict(self): self.addCleanup(os_helper.unlink, ABSTFN) self.assertRaises(FileNotFoundError, ntpath.realpath, ABSTFN, strict=True) self.assertRaises(FileNotFoundError, ntpath.realpath, ABSTFN + "2", strict=True) + + @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') + def test_realpath_invalid_paths(self): + realpath = ntpath.realpath + ABSTFN = ntpath.abspath(os_helper.TESTFN) + ABSTFNb = os.fsencode(ABSTFN) + path = ABSTFN + '\x00' + # gh-106242: Embedded nulls and non-strict fallback to abspath + self.assertEqual(realpath(path, strict=False), path) # gh-106242: Embedded nulls should raise OSError (not ValueError) - self.assertRaises(OSError, ntpath.realpath, ABSTFN + "\0spam", strict=True) + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + path = ABSTFNb + b'\x00' + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + path = ABSTFN + '\\nonexistent\\x\x00' + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + path = ABSTFNb + b'\\nonexistent\\x\x00' + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + path = ABSTFN + '\x00\\..' + self.assertEqual(realpath(path, strict=False), os.getcwd()) + self.assertEqual(realpath(path, strict=True), os.getcwd()) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), os.getcwd()) + path = ABSTFNb + b'\x00\\..' + self.assertEqual(realpath(path, strict=False), os.getcwdb()) + self.assertEqual(realpath(path, strict=True), os.getcwdb()) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), os.getcwdb()) + path = ABSTFN + '\\nonexistent\\x\x00\\..' + self.assertEqual(realpath(path, strict=False), ABSTFN + '\\nonexistent') + self.assertRaises(OSError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), ABSTFN + '\\nonexistent') + path = ABSTFNb + b'\\nonexistent\\x\x00\\..' + self.assertEqual(realpath(path, strict=False), ABSTFNb + b'\\nonexistent') + self.assertRaises(OSError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), ABSTFNb + b'\\nonexistent') + + @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_invalid_unicode_paths(self, kwargs): + realpath = ntpath.realpath + ABSTFN = ntpath.abspath(os_helper.TESTFN) + ABSTFNb = os.fsencode(ABSTFN) + path = ABSTFNb + b'\xff' + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + path = ABSTFNb + b'\\nonexistent\\\xff' + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + path = ABSTFNb + b'\xff\\..' + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + path = ABSTFNb + b'\\nonexistent\\\xff\\..' + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') - def test_realpath_relative(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_relative(self, kwargs): ABSTFN = ntpath.abspath(os_helper.TESTFN) open(ABSTFN, "wb").close() self.addCleanup(os_helper.unlink, ABSTFN) self.addCleanup(os_helper.unlink, ABSTFN + "1") os.symlink(ABSTFN, ntpath.relpath(ABSTFN + "1")) - self.assertPathEqual(ntpath.realpath(ABSTFN + "1"), ABSTFN) + self.assertPathEqual(ntpath.realpath(ABSTFN + "1", **kwargs), ABSTFN) @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') @@ -598,7 +799,62 @@ def test_realpath_symlink_loops_strict(self): @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') - def test_realpath_symlink_prefix(self): + def test_realpath_symlink_loops_raise(self): + # Symlink loops raise OSError in ALLOW_MISSING mode + ABSTFN = ntpath.abspath(os_helper.TESTFN) + self.addCleanup(os_helper.unlink, ABSTFN) + self.addCleanup(os_helper.unlink, ABSTFN + "1") + self.addCleanup(os_helper.unlink, ABSTFN + "2") + self.addCleanup(os_helper.unlink, ABSTFN + "y") + self.addCleanup(os_helper.unlink, ABSTFN + "c") + self.addCleanup(os_helper.unlink, ABSTFN + "a") + self.addCleanup(os_helper.unlink, ABSTFN + "x") + + os.symlink(ABSTFN, ABSTFN) + self.assertRaises(OSError, ntpath.realpath, ABSTFN, strict=ALLOW_MISSING) + + os.symlink(ABSTFN + "1", ABSTFN + "2") + os.symlink(ABSTFN + "2", ABSTFN + "1") + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1", + strict=ALLOW_MISSING) + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "2", + strict=ALLOW_MISSING) + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1\\x", + strict=ALLOW_MISSING) + + # Windows eliminates '..' components before resolving links; + # realpath is not expected to raise if this removes the loop. + self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\.."), + ntpath.dirname(ABSTFN)) + self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..\\x"), + ntpath.dirname(ABSTFN) + "\\x") + + os.symlink(ABSTFN + "x", ABSTFN + "y") + self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..\\" + + ntpath.basename(ABSTFN) + "y"), + ABSTFN + "x") + self.assertRaises( + OSError, ntpath.realpath, + ABSTFN + "1\\..\\" + ntpath.basename(ABSTFN) + "1", + strict=ALLOW_MISSING) + + os.symlink(ntpath.basename(ABSTFN) + "a\\b", ABSTFN + "a") + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "a", + strict=ALLOW_MISSING) + + os.symlink("..\\" + ntpath.basename(ntpath.dirname(ABSTFN)) + + "\\" + ntpath.basename(ABSTFN) + "c", ABSTFN + "c") + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "c", + strict=ALLOW_MISSING) + + # Test using relative path as well. + self.assertRaises(OSError, ntpath.realpath, ntpath.basename(ABSTFN), + strict=ALLOW_MISSING) + + @os_helper.skip_unless_symlink + @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_symlink_prefix(self, kwargs): ABSTFN = ntpath.abspath(os_helper.TESTFN) self.addCleanup(os_helper.unlink, ABSTFN + "3") self.addCleanup(os_helper.unlink, "\\\\?\\" + ABSTFN + "3.") @@ -613,9 +869,9 @@ def test_realpath_symlink_prefix(self): f.write(b'1') os.symlink("\\\\?\\" + ABSTFN + "3.", ABSTFN + "3.link") - self.assertPathEqual(ntpath.realpath(ABSTFN + "3link"), + self.assertPathEqual(ntpath.realpath(ABSTFN + "3link", **kwargs), ABSTFN + "3") - self.assertPathEqual(ntpath.realpath(ABSTFN + "3.link"), + self.assertPathEqual(ntpath.realpath(ABSTFN + "3.link", **kwargs), "\\\\?\\" + ABSTFN + "3.") # Resolved paths should be usable to open target files @@ -625,14 +881,17 @@ def test_realpath_symlink_prefix(self): self.assertEqual(f.read(), b'1') # When the prefix is included, it is not stripped - self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3link"), + self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3link", **kwargs), "\\\\?\\" + ABSTFN + "3") - self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3.link"), + self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3.link", **kwargs), "\\\\?\\" + ABSTFN + "3.") @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') def test_realpath_nul(self): tester("ntpath.realpath('NUL')", r'\\.\NUL') + tester("ntpath.realpath('NUL', strict=False)", r'\\.\NUL') + tester("ntpath.realpath('NUL', strict=True)", r'\\.\NUL') + tester("ntpath.realpath('NUL', strict=ALLOW_MISSING)", r'\\.\NUL') @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') @unittest.skipUnless(HAVE_GETSHORTPATHNAME, 'need _getshortpathname') @@ -656,12 +915,20 @@ def test_realpath_cwd(self): self.assertPathEqual(test_file_long, ntpath.realpath(test_file_short)) - with os_helper.change_cwd(test_dir_long): - self.assertPathEqual(test_file_long, ntpath.realpath("file.txt")) - with os_helper.change_cwd(test_dir_long.lower()): - self.assertPathEqual(test_file_long, ntpath.realpath("file.txt")) - with os_helper.change_cwd(test_dir_short): - self.assertPathEqual(test_file_long, ntpath.realpath("file.txt")) + for kwargs in {}, {'strict': True}, {'strict': ALLOW_MISSING}: + with self.subTest(**kwargs): + with os_helper.change_cwd(test_dir_long): + self.assertPathEqual( + test_file_long, + ntpath.realpath("file.txt", **kwargs)) + with os_helper.change_cwd(test_dir_long.lower()): + self.assertPathEqual( + test_file_long, + ntpath.realpath("file.txt", **kwargs)) + with os_helper.change_cwd(test_dir_short): + self.assertPathEqual( + test_file_long, + ntpath.realpath("file.txt", **kwargs)) @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') def test_realpath_permission(self): @@ -682,12 +949,15 @@ def test_realpath_permission(self): # Automatic generation of short names may be disabled on # NTFS volumes for the sake of performance. # They're not supported at all on ReFS and exFAT. - subprocess.run( + p = subprocess.run( # Try to set the short name manually. ['fsutil.exe', 'file', 'setShortName', test_file, 'LONGFI~1.TXT'], creationflags=subprocess.DETACHED_PROCESS ) + if p.returncode: + raise unittest.SkipTest('failed to set short name') + try: self.assertPathEqual(test_file, ntpath.realpath(test_file_short)) except AssertionError: @@ -751,6 +1021,19 @@ def check(value, expected): check('%spam%bar', '%sbar' % nonascii) check('%{}%bar'.format(nonascii), 'ham%sbar' % nonascii) + @support.requires_resource('cpu') + def test_expandvars_large(self): + expandvars = ntpath.expandvars + with os_helper.EnvironmentVarGuard() as env: + env.clear() + env["A"] = "B" + n = 100_000 + self.assertEqual(expandvars('%A%'*n), 'B'*n) + self.assertEqual(expandvars('%A%A'*n), 'BA'*n) + self.assertEqual(expandvars("''"*n + '%%'), "''"*n + '%') + self.assertEqual(expandvars("%%"*n), "%"*n) + self.assertEqual(expandvars("$$"*n), "$"*n) + def test_expanduser(self): tester('ntpath.expanduser("test")', 'test') @@ -812,8 +1095,6 @@ def test_abspath(self): tester('ntpath.abspath("C:/nul")', "\\\\.\\nul") tester('ntpath.abspath("C:\\nul")', "\\\\.\\nul") self.assertTrue(ntpath.isabs(ntpath.abspath("C:spam"))) - self.assertEqual(ntpath.abspath("C:\x00"), ntpath.join(ntpath.abspath("C:"), "\x00")) - self.assertEqual(ntpath.abspath("\x00:spam"), "\x00:\\spam") tester('ntpath.abspath("//..")', "\\\\") tester('ntpath.abspath("//../")', "\\\\..\\") tester('ntpath.abspath("//../..")', "\\\\..\\") @@ -847,6 +1128,26 @@ def test_abspath(self): drive, _ = ntpath.splitdrive(cwd_dir) tester('ntpath.abspath("/abc/")', drive + "\\abc") + def test_abspath_invalid_paths(self): + abspath = ntpath.abspath + if sys.platform == 'win32': + self.assertEqual(abspath("C:\x00"), ntpath.join(abspath("C:"), "\x00")) + self.assertEqual(abspath(b"C:\x00"), ntpath.join(abspath(b"C:"), b"\x00")) + self.assertEqual(abspath("\x00:spam"), "\x00:\\spam") + self.assertEqual(abspath(b"\x00:spam"), b"\x00:\\spam") + self.assertEqual(abspath('c:\\fo\x00o'), 'c:\\fo\x00o') + self.assertEqual(abspath(b'c:\\fo\x00o'), b'c:\\fo\x00o') + self.assertEqual(abspath('c:\\fo\x00o\\..\\bar'), 'c:\\bar') + self.assertEqual(abspath(b'c:\\fo\x00o\\..\\bar'), b'c:\\bar') + self.assertEqual(abspath('c:\\\udfff'), 'c:\\\udfff') + self.assertEqual(abspath('c:\\\udfff\\..\\foo'), 'c:\\foo') + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, abspath, b'c:\\\xff') + self.assertRaises(UnicodeDecodeError, abspath, b'c:\\\xff\\..\\foo') + else: + self.assertEqual(abspath(b'c:\\\xff'), b'c:\\\xff') + self.assertEqual(abspath(b'c:\\\xff\\..\\foo'), b'c:\\foo') + def test_relpath(self): tester('ntpath.relpath("a")', 'a') tester('ntpath.relpath(ntpath.abspath("a"))', 'a') @@ -989,6 +1290,18 @@ def test_ismount(self): self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$")) self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$\\")) + def test_ismount_invalid_paths(self): + ismount = ntpath.ismount + self.assertFalse(ismount("c:\\\udfff")) + if sys.platform == 'win32': + self.assertRaises(ValueError, ismount, "c:\\\x00") + self.assertRaises(ValueError, ismount, b"c:\\\x00") + self.assertRaises(UnicodeDecodeError, ismount, b"c:\\\xff") + else: + self.assertFalse(ismount("c:\\\x00")) + self.assertFalse(ismount(b"c:\\\x00")) + self.assertFalse(ismount(b"c:\\\xff")) + def test_isreserved(self): self.assertFalse(ntpath.isreserved('')) self.assertFalse(ntpath.isreserved('.')) @@ -1095,6 +1408,13 @@ def test_isjunction(self): self.assertFalse(ntpath.isjunction('tmpdir')) self.assertPathEqual(ntpath.realpath('testjunc'), ntpath.realpath('tmpdir')) + def test_isfile_invalid_paths(self): + isfile = ntpath.isfile + self.assertIs(isfile('/tmp\udfffabcds'), False) + self.assertIs(isfile(b'/tmp\xffabcds'), False) + self.assertIs(isfile('/tmp\x00abcds'), False) + self.assertIs(isfile(b'/tmp\x00abcds'), False) + @unittest.skipIf(sys.platform != 'win32', "drive letters are a windows concept") def test_isfile_driveletter(self): drive = os.environ.get('SystemDrive') @@ -1131,7 +1451,7 @@ def test_con_device(self): self.assertTrue(os.path.exists(r"\\.\CON")) @unittest.skipIf(sys.platform != 'win32', "Fast paths are only for win32") - @cpython_only + @support.cpython_only def test_fast_paths_in_use(self): # There are fast paths of these functions implemented in posixmodule.c. # Confirm that they are being used, and not the Python fallbacks in @@ -1195,9 +1515,6 @@ def _check_function(self, func): def test_path_normcase(self): self._check_function(self.path.normcase) - if sys.platform == 'win32': - self.assertEqual(ntpath.normcase('\u03a9\u2126'), 'ωΩ') - self.assertEqual(ntpath.normcase('abc\x00def'), 'abc\x00def') def test_path_isabs(self): self._check_function(self.path.isabs) diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index a45aafc63fa6973..2e50c1c62b328c1 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -567,6 +567,14 @@ def test(default=None): with self.assertRaises(RecursionError): test() + def test_dont_specialize_custom_vectorcall(self): + def f(): + raise Exception("no way") + + _testinternalcapi.set_vectorcall_nop(f) + for _ in range(_testinternalcapi.SPECIALIZATION_THRESHOLD): + f() + def make_deferred_ref_count_obj(): """Create an object that uses deferred reference counting. diff --git a/Lib/test/test_optparse.py b/Lib/test/test_optparse.py index e6ffd2b0ffeb0e9..e476e4727803e5e 100644 --- a/Lib/test/test_optparse.py +++ b/Lib/test/test_optparse.py @@ -615,9 +615,9 @@ def test_float_default(self): self.parser.add_option( "-p", "--prob", help="blow up with probability PROB [default: %default]") - self.parser.set_defaults(prob=0.43) + self.parser.set_defaults(prob=0.25) expected_help = self.help_prefix + \ - " -p PROB, --prob=PROB blow up with probability PROB [default: 0.43]\n" + " -p PROB, --prob=PROB blow up with probability PROB [default: 0.25]\n" self.assertHelp(self.parser, expected_help) def test_alt_expand(self): diff --git a/Lib/test/test_ordered_dict.py b/Lib/test/test_ordered_dict.py index 9f131a9110dccbd..04942effa1fe2cf 100644 --- a/Lib/test/test_ordered_dict.py +++ b/Lib/test/test_ordered_dict.py @@ -147,7 +147,7 @@ def test_fromkeys(self): def test_abc(self): OrderedDict = self.OrderedDict self.assertIsInstance(OrderedDict(), MutableMapping) - self.assertTrue(issubclass(OrderedDict, MutableMapping)) + self.assertIsSubclass(OrderedDict, MutableMapping) def test_clear(self): OrderedDict = self.OrderedDict @@ -314,14 +314,14 @@ def check(dup): check(dup) self.assertIs(dup.x, od.x) self.assertIs(dup.z, od.z) - self.assertFalse(hasattr(dup, 'y')) + self.assertNotHasAttr(dup, 'y') dup = copy.deepcopy(od) check(dup) self.assertEqual(dup.x, od.x) self.assertIsNot(dup.x, od.x) self.assertEqual(dup.z, od.z) self.assertIsNot(dup.z, od.z) - self.assertFalse(hasattr(dup, 'y')) + self.assertNotHasAttr(dup, 'y') # pickle directly pulls the module, so we have to fake it with replaced_module('collections', self.module): for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -330,7 +330,7 @@ def check(dup): check(dup) self.assertEqual(dup.x, od.x) self.assertEqual(dup.z, od.z) - self.assertFalse(hasattr(dup, 'y')) + self.assertNotHasAttr(dup, 'y') check(eval(repr(od))) update_test = OrderedDict() update_test.update(od) @@ -873,6 +873,39 @@ def side_effect(self): self.assertDictEqual(dict1, dict.fromkeys((0, 4.2))) self.assertDictEqual(dict2, dict.fromkeys((0, Key(), 4.2))) + def test_issue148660_copy_clear_in_key_eq(self): + # gh-148660: od.copy() must not crash when a key's __eq__ clears od + # while copy() is inserting into the new dict. + armed = False + calls = 0 + class Key: + def __hash__(self): + return 1 + def __eq__(self, other): + nonlocal calls + if armed: + calls += 1 + if calls == 2: + od.clear() + return self is other + od = self.OrderedDict() + od[Key()] = "v1" + od[Key()] = "v2" + armed = True + msg = "OrderedDict mutated during iteration" + self.assertRaisesRegex(RuntimeError, msg, od.copy) + + def test_issue148660_copy_clear_in_subclass_getitem(self): + # gh-148660: od.copy() must not crash when a subclass __getitem__ + # clears od. + class OD(self.OrderedDict): + def __getitem__(self, key): + od.clear() + return "v" + od = OD([(1, "v1"), (2, "v2")]) + msg = "OrderedDict mutated during iteration" + self.assertRaisesRegex(RuntimeError, msg, od.copy) + @unittest.skipUnless(c_coll, 'requires the C version of the collections module') class CPythonOrderedDictTests(OrderedDictTests, diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 333179a71e3cdcb..8c41d9799b5fa9f 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -13,7 +13,6 @@ import locale import os import pickle -import platform import select import selectors import shutil @@ -99,13 +98,22 @@ def create_file(filename, content=b'content'): fp.write(content) +# On platforms without a native spawnv(), os.py provides a Python fallback +# built on fork()+exec*() that reports argument conversion failures from the +# child as exit status 127 instead of raising, so tests of the C +# implementation's error paths cannot run against it. +requires_native_spawnv = unittest.skipUnless( + isinstance(getattr(os, 'spawnv', None), types.BuiltinFunctionType), + 'requires the native C os.spawnv') + + # bpo-41625: On AIX, splice() only works with a socket, not with a pipe. requires_splice_pipe = unittest.skipIf(sys.platform.startswith("aix"), 'on AIX, splice() only accepts sockets') def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class MiscTests(unittest.TestCase): @@ -160,7 +168,8 @@ def test_getcwd_long_path(self): # ("The filename or extension is too long") break except OSError as exc: - if exc.errno == errno.ENAMETOOLONG: + # DragonFly BSD raises EFAULT for a too long path. + if exc.errno in (errno.ENAMETOOLONG, errno.EFAULT): break else: raise @@ -818,7 +827,7 @@ def test_15261(self): self.assertEqual(ctx.exception.errno, errno.EBADF) def check_file_attributes(self, result): - self.assertTrue(hasattr(result, 'st_file_attributes')) + self.assertHasAttr(result, 'st_file_attributes') self.assertTrue(isinstance(result.st_file_attributes, int)) self.assertTrue(0 <= result.st_file_attributes <= 0xFFFFFFFF) @@ -891,11 +900,18 @@ def support_subsecond(self, filename): or (st.st_mtime != st[8]) or (st.st_ctime != st[9])) + def support_atime(self, filename): + # Heuristic to check if the filesystem stores the access time. + # Use whole seconds, to not depend on the timestamp resolution. + os.utime(filename, (1, 2)) + return os.stat(filename).st_atime == 1 + def _test_utime(self, set_time, filename=None): if not filename: filename = self.fname support_subsecond = self.support_subsecond(filename) + support_atime = self.support_atime(filename) if support_subsecond: # Timestamp with a resolution of 1 microsecond (10^-6). # @@ -922,18 +938,22 @@ def _test_utime(self, set_time, filename=None): # digits worth of sub-second precision. # Some day it would be good to fix this upstream. delta=1e-5 - self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-5) + if support_atime: + self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-5) + self.assertAlmostEqual(st.st_atime_ns, atime_ns, delta=1e9 * 1e-5) self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-5) - self.assertAlmostEqual(st.st_atime_ns, atime_ns, delta=1e9 * 1e-5) self.assertAlmostEqual(st.st_mtime_ns, mtime_ns, delta=1e9 * 1e-5) else: if support_subsecond: - self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-6) + if support_atime: + self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-6) self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-6) else: - self.assertEqual(st.st_atime, atime_ns * 1e-9) + if support_atime: + self.assertEqual(st.st_atime, atime_ns * 1e-9) self.assertEqual(st.st_mtime, mtime_ns * 1e-9) - self.assertEqual(st.st_atime_ns, atime_ns) + if support_atime: + self.assertEqual(st.st_atime_ns, atime_ns) self.assertEqual(st.st_mtime_ns, mtime_ns) def test_utime(self): @@ -1920,6 +1940,8 @@ def test_makedir(self): "WASI's umask is a stub." ) def test_mode(self): + # Note: in some cases, the umask might already be 2 in which case this + # will pass even if os.umask is actually broken. with os_helper.temp_umask(0o002): base = os_helper.TESTFN parent = os.path.join(base, 'dir1') @@ -2181,7 +2203,7 @@ def test_getrandom0(self): self.assertEqual(empty, b'') def test_getrandom_random(self): - self.assertTrue(hasattr(os, 'GRND_RANDOM')) + self.assertHasAttr(os, 'GRND_RANDOM') # Don't test os.getrandom(1, os.GRND_RANDOM) to not consume the rare # resource /dev/random @@ -2416,6 +2438,45 @@ def test_execve_invalid_env(self): with self.assertRaises(ValueError): os.execve(args[0], args, newenv) + # See https://github.com/python/cpython/issues/137934 and the other + # related issues for the reason why we cannot test this on Windows. + @unittest.skipIf(os.name == "nt", "POSIX-specific test") + @unittest.skipUnless(unix_shell and os.path.exists(unix_shell), + "requires a shell") + def test_execve_env_concurrent_mutation_with_fspath_posix(self): + # Prevent crash when mutating environment during parsing. + # Regression test for https://github.com/python/cpython/issues/143309. + + message = "hello from execve" + code = """ + import os, sys + + class MyPath: + def __fspath__(self): + mutated.clear() + return b"pwn" + + mutated = KEYS = VALUES = [MyPath()] + + class MyEnv: + def __getitem__(self): raise RuntimeError("must not be called") + def __len__(self): return 1 + def keys(self): return KEYS + def values(self): return VALUES + + args = [{unix_shell!r}, '-c', 'echo \"{message!s}\"'] + os.execve(args[0], args, MyEnv()) + """.format(unix_shell=unix_shell, message=message) + + # Make sure to forward "LD_*" variables so that assert_python_ok() + # can run correctly. + minimal = {k: v for k, v in os.environ.items() if k.startswith("LD_")} + with os_helper.EnvironmentVarGuard() as env: + env.clear() + env.update(minimal) + _, out, _ = assert_python_ok('-c', code, **env) + self.assertIn(bytes(message, "ascii"), out) + @unittest.skipUnless(sys.platform == "win32", "Win32-specific test") def test_execve_with_empty_path(self): # bpo-32890: Check GetLastError() misuse @@ -2567,6 +2628,16 @@ def test_fpathconf_bad_fd(self): self.check(os.pathconf, "PC_NAME_MAX") self.check(os.fpathconf, "PC_NAME_MAX") + @unittest.skipUnless(hasattr(os, 'pathconf'), 'test needs os.pathconf()') + @unittest.skipIf( + support.linked_to_musl(), + 'musl fpathconf ignores the file descriptor and returns a constant', + ) + def test_pathconf_negative_fd_uses_fd_semantics(self): + with self.assertRaises(OSError) as ctx: + os.pathconf(-1, 1) + self.assertEqual(ctx.exception.errno, errno.EBADF) + @unittest.skipUnless(hasattr(os, 'ftruncate'), 'test needs os.ftruncate()') def test_ftruncate(self): self.check(os.truncate, 0) @@ -3712,6 +3783,25 @@ def test_spawnve_noargs(self): self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, program, ('',), {}) self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, program, [''], {}) + @requires_native_spawnv + def test_spawnv_arg_conversion_errors(self): + # A non-path argv item gets a TypeError naming the argument... + with self.assertRaisesRegex(TypeError, 'must contain only strings'): + os.spawnv(os.P_NOWAIT, sys.executable, [sys.executable, 123]) + # ...but other conversion errors must not be masked as TypeError + # (gh-151416). + with self.assertRaises(ValueError): + os.spawnv(os.P_NOWAIT, sys.executable, + [sys.executable, 'embedded\0null']) + + class RaisingPath: + def __fspath__(self): + raise RuntimeError('gotcha') + + with self.assertRaisesRegex(RuntimeError, 'gotcha'): + os.spawnv(os.P_NOWAIT, sys.executable, + [sys.executable, RaisingPath()]) + def _test_invalid_env(self, spawn): program = sys.executable args = self.quote_args([program, '-c', 'pass']) @@ -3992,7 +4082,6 @@ async def test_trailers(self): @requires_headers_trailers @requires_32b async def test_headers_overflow_32bits(self): - self.server.handler_instance.accumulate = False with self.assertRaises(OSError) as cm: await self.async_sendfile(self.sockno, self.fileno, 0, 0, headers=[b"x" * 2**16] * 2**15) @@ -4001,7 +4090,6 @@ async def test_headers_overflow_32bits(self): @requires_headers_trailers @requires_32b async def test_trailers_overflow_32bits(self): - self.server.handler_instance.accumulate = False with self.assertRaises(OSError) as cm: await self.async_sendfile(self.sockno, self.fileno, 0, 0, trailers=[b"x" * 2**16] * 2**15) @@ -4291,13 +4379,8 @@ def test_eventfd_select(self): @unittest.skipIf(sys.platform == "android", "gh-124873: Test is flaky on Android") @support.requires_linux_version(2, 6, 30) class TimerfdTests(unittest.TestCase): - # 1 ms accuracy is reliably achievable on every platform except Android - # emulators, where we allow 10 ms (gh-108277). - if sys.platform == "android" and platform.android_ver().is_emulator: - CLOCK_RES_PLACES = 2 - else: - CLOCK_RES_PLACES = 3 - + # gh-126112: Use 10 ms to tolerate slow buildbots + CLOCK_RES_PLACES = 2 # 10 ms CLOCK_RES = 10 ** -CLOCK_RES_PLACES CLOCK_RES_NS = 10 ** (9 - CLOCK_RES_PLACES) @@ -4883,6 +4966,28 @@ def test_posix_pty_functions(self): son_path = os.ptsname(mother_fd) son_fd = os.open(son_path, os.O_RDWR|os.O_NOCTTY) self.addCleanup(os.close, son_fd) + if sys.platform.startswith('sunos'): + import fcntl + I_PUSH = 0x5302 + TIOCNOTTY = 0x7471 + # Pushing "ptem" makes the slave a terminal, which a session + # leader without a controlling terminal then acquires as one + # despite O_NOCTTY. Note whether we already had one. + try: + os.close(os.open('/dev/tty', os.O_RDONLY|os.O_NOCTTY)) + had_ctty = True + except OSError: + had_ctty = False + fcntl.ioctl(son_fd, I_PUSH, b'ptem\0') + fcntl.ioctl(son_fd, I_PUSH, b'ldterm\0') + if not had_ctty and os.getsid(0) == os.getpid(): + # Disown it, otherwise closing the file descriptors sends + # SIGHUP to the session. TIOCNOTTY sends it too. + old_handler = signal.signal(signal.SIGHUP, signal.SIG_IGN) + try: + fcntl.ioctl(son_fd, TIOCNOTTY) + finally: + signal.signal(signal.SIGHUP, old_handler) self.assertEqual(os.ptsname(mother_fd), os.ttyname(son_fd)) @unittest.skipUnless(hasattr(os, 'spawnl'), "need os.spawnl()") @@ -5213,6 +5318,30 @@ def test_current_directory(self): finally: os.chdir(old_dir) + @unittest.skipIf(sys.platform != 'win32', "Win32 specific test") + def test_windows_trailing_space_path(self): + import pathlib + + filename = self.create_file("file.txt") + path = self.path + " " + + self.assertTrue(os.path.exists(path)) + os.stat(path) + with open(filename + " ", "rb") as file: + self.assertEqual(file.read(), b"python") + + self.assertEqual(os.listdir(path), ["file.txt"]) + with os.scandir(path) as entries: + self.assertEqual([entry.name for entry in entries], ["file.txt"]) + pathlib_entries = list(pathlib.Path(path).iterdir()) + self.assertEqual([entry.name for entry in pathlib_entries], ["file.txt"]) + del pathlib_entries + + extended_path = "\\\\?\\" + path + self.assertFalse(os.path.exists(extended_path)) + self.assertRaises(FileNotFoundError, os.listdir, extended_path) + self.assertRaises(FileNotFoundError, os.scandir, extended_path) + def test_repr(self): entry = self.create_file_entry() self.assertEqual(repr(entry), "<DirEntry 'file.txt'>") @@ -5431,8 +5560,8 @@ def test_fsencode_fsdecode(self): def test_pathlike(self): self.assertEqual('#feelthegil', self.fspath(FakePath('#feelthegil'))) - self.assertTrue(issubclass(FakePath, os.PathLike)) - self.assertTrue(isinstance(FakePath('x'), os.PathLike)) + self.assertIsSubclass(FakePath, os.PathLike) + self.assertIsInstance(FakePath('x'), os.PathLike) def test_garbage_in_exception_out(self): vapor = type('blah', (), {}) @@ -5458,8 +5587,8 @@ def test_pathlike_subclasshook(self): # true on abstract implementation. class A(os.PathLike): pass - self.assertFalse(issubclass(FakePath, A)) - self.assertTrue(issubclass(FakePath, os.PathLike)) + self.assertNotIsSubclass(FakePath, A) + self.assertIsSubclass(FakePath, os.PathLike) def test_pathlike_class_getitem(self): self.assertIsInstance(os.PathLike[bytes], types.GenericAlias) @@ -5469,7 +5598,7 @@ class A(os.PathLike): __slots__ = () def __fspath__(self): return '' - self.assertFalse(hasattr(A(), '__dict__')) + self.assertNotHasAttr(A(), '__dict__') def test_fspath_set_to_None(self): class Foo: diff --git a/Lib/test/test_pathlib/test_join.py b/Lib/test/test_pathlib/test_join.py index f1a24204b4c30aa..6b51a09e5ac1684 100644 --- a/Lib/test/test_pathlib/test_join.py +++ b/Lib/test/test_pathlib/test_join.py @@ -3,6 +3,8 @@ """ import unittest +import threading +from test.support import threading_helper from .support import is_pypi from .support.lexical_path import LexicalPath @@ -158,6 +160,26 @@ def test_parts(self): parts = p.parts self.assertEqual(parts, (sep, 'a', 'b')) + @threading_helper.requires_working_threading() + def test_parts_multithreaded(self): + P = self.cls + + NUM_THREADS = 10 + NUM_ITERS = 10 + + for _ in range(NUM_ITERS): + b = threading.Barrier(NUM_THREADS) + path = P('a') / 'b' / 'c' / 'd' / 'e' + expected = ('a', 'b', 'c', 'd', 'e') + + def check_parts(): + b.wait() + self.assertEqual(path.parts, expected) + + threads = [threading.Thread(target=check_parts) for _ in range(NUM_THREADS)] + with threading_helper.start_threads(threads): + pass + def test_parent(self): # Relative P = self.cls diff --git a/Lib/test/test_pathlib/test_pathlib.py b/Lib/test/test_pathlib/test_pathlib.py index 8a313cc42925748..a7fa9865b613248 100644 --- a/Lib/test/test_pathlib/test_pathlib.py +++ b/Lib/test/test_pathlib/test_pathlib.py @@ -17,10 +17,10 @@ from test.support import import_helper from test.support import cpython_only -from test.support import is_emscripten, is_wasi +from test.support import is_emscripten, is_wasi, is_wasm32 from test.support import infinite_recursion from test.support import os_helper -from test.support.os_helper import TESTFN, FakePath +from test.support.os_helper import TESTFN, FS_NONASCII, FakePath try: import fcntl except ImportError: @@ -77,8 +77,8 @@ def needs_symlinks(fn): class UnsupportedOperationTest(unittest.TestCase): def test_is_notimplemented(self): - self.assertTrue(issubclass(pathlib.UnsupportedOperation, NotImplementedError)) - self.assertTrue(isinstance(pathlib.UnsupportedOperation(), NotImplementedError)) + self.assertIsSubclass(pathlib.UnsupportedOperation, NotImplementedError) + self.assertIsInstance(pathlib.UnsupportedOperation(), NotImplementedError) class LazyImportTest(unittest.TestCase): @@ -293,6 +293,12 @@ def test_pickling_common(self): self.assertEqual(hash(pp), hash(p)) self.assertEqual(str(pp), str(p)) + def test_unpicking_3_13(self): + data = (b"\x80\x04\x95'\x00\x00\x00\x00\x00\x00\x00\x8c\x0e" + b"pathlib._local\x94\x8c\rPurePosixPath\x94\x93\x94)R\x94.") + p = pickle.loads(data) + self.assertIsInstance(p, pathlib.PurePosixPath) + def test_repr_common(self): for pathstr in ('a', 'a/b', 'a/b/c', '/', '/a/b', '/a/b/c'): with self.subTest(pathstr=pathstr): @@ -300,8 +306,8 @@ def test_repr_common(self): clsname = p.__class__.__name__ r = repr(p) # The repr() is in the form ClassName("forward-slashes path"). - self.assertTrue(r.startswith(clsname + '('), r) - self.assertTrue(r.endswith(')'), r) + self.assertStartsWith(r, clsname + '(') + self.assertEndsWith(r, ')') inner = r[len(clsname) + 1 : -1] self.assertEqual(eval(inner), p.as_posix()) @@ -770,12 +776,16 @@ def test_as_uri_windows(self): self.assertEqual(self.make_uri(P('c:/')), 'file:///c:/') self.assertEqual(self.make_uri(P('c:/a/b.c')), 'file:///c:/a/b.c') self.assertEqual(self.make_uri(P('c:/a/b%#c')), 'file:///c:/a/b%25%23c') - self.assertEqual(self.make_uri(P('c:/a/b\xe9')), 'file:///c:/a/b%C3%A9') self.assertEqual(self.make_uri(P('//some/share/')), 'file://some/share/') self.assertEqual(self.make_uri(P('//some/share/a/b.c')), 'file://some/share/a/b.c') - self.assertEqual(self.make_uri(P('//some/share/a/b%#c\xe9')), - 'file://some/share/a/b%25%23c%C3%A9') + + from urllib.parse import quote_from_bytes + QUOTED_FS_NONASCII = quote_from_bytes(os.fsencode(FS_NONASCII)) + self.assertEqual(self.make_uri(P('c:/a/b' + FS_NONASCII)), + 'file:///c:/a/b' + QUOTED_FS_NONASCII) + self.assertEqual(self.make_uri(P('//some/share/a/b%#c' + FS_NONASCII)), + 'file://some/share/a/b%25%23c' + QUOTED_FS_NONASCII) @needs_windows def test_ordering_windows(self): @@ -2759,6 +2769,7 @@ def test_is_socket_true(self): if (isinstance(e, PermissionError) or "AF_UNIX path too long" in str(e)): self.skipTest("cannot bind Unix socket: " + str(e)) + raise self.assertTrue(P.is_socket()) self.assertFalse(P.is_fifo()) self.assertFalse(P.is_file()) @@ -2950,7 +2961,13 @@ def test_glob_dotdot(self): else: # ".." segments are normalized first on Windows, so this path is stat()able. self.assertEqual(set(p.glob("xyzzy/..")), { P(self.base, "xyzzy", "..") }) - self.assertEqual(set(p.glob("/".join([".."] * 50))), { P(self.base, *[".."] * 50)}) + if sys.platform == "emscripten": + # Emscripten will return ELOOP if there are 49 or more ..'s. + # Can remove when https://github.com/emscripten-core/emscripten/pull/24591 is merged. + NDOTDOTS = 48 + else: + NDOTDOTS = 50 + self.assertEqual(set(p.glob("/".join([".."] * NDOTDOTS))), { P(self.base, *[".."] * NDOTDOTS)}) def test_glob_inaccessible(self): P = self.cls @@ -3154,7 +3171,7 @@ def test_absolute_posix(self): self.assertEqual(str(P('//a/b').absolute()), '//a/b') @unittest.skipIf( - is_emscripten or is_wasi, + is_wasm32, "umask is not implemented on Emscripten/WASI." ) @needs_posix @@ -3185,7 +3202,7 @@ def test_resolve_root(self): os.chdir(current_directory) @unittest.skipIf( - is_emscripten or is_wasi, + is_wasm32, "umask is not implemented on Emscripten/WASI." ) @needs_posix diff --git a/Lib/test/test_pathlib/test_write.py b/Lib/test/test_pathlib/test_write.py index b958490d0a834f6..15054e804ec9fdf 100644 --- a/Lib/test/test_pathlib/test_write.py +++ b/Lib/test/test_pathlib/test_write.py @@ -59,15 +59,17 @@ def test_open_wb(self): def test_write_bytes(self): p = self.root / 'fileA' - p.write_bytes(b'abcdefg') - self.assertEqual(self.ground.readbytes(p), b'abcdefg') + data = b'abcdefg' + self.assertEqual(len(data), p.write_bytes(data)) + self.assertEqual(self.ground.readbytes(p), data) # Check that trying to write str does not truncate the file. self.assertRaises(TypeError, p.write_bytes, 'somestr') - self.assertEqual(self.ground.readbytes(p), b'abcdefg') + self.assertEqual(self.ground.readbytes(p), data) def test_write_text(self): p = self.root / 'fileA' - p.write_text('äbcdefg', encoding='latin-1') + data = 'äbcdefg' + self.assertEqual(len(data), p.write_text(data, encoding='latin-1')) self.assertEqual(self.ground.readbytes(p), b'\xe4bcdefg') # Check that trying to write bytes does not truncate the file. self.assertRaises(TypeError, p.write_text, b'somebytes', encoding='utf-8') diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index 54797d7898ff33e..e51488f8e03b8e7 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -3232,6 +3232,37 @@ def test_pdb_issue_gh_127321(): """ +def test_pdb_issue_gh_136057(): + """See GH-136057 + "step" and "next" commands should be able to get over list comprehensions + >>> def test_function(): + ... import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + ... lst = [i for i in range(10)] + ... for i in lst: pass + + >>> with PdbTestInput([ # doctest: +NORMALIZE_WHITESPACE + ... 'next', + ... 'next', + ... 'step', + ... 'continue', + ... ]): + ... test_function() + > <doctest test.test_pdb.test_pdb_issue_gh_136057[0]>(2)test_function() + -> import pdb; pdb.Pdb(nosigint=True, readrc=False).set_trace() + (Pdb) next + > <doctest test.test_pdb.test_pdb_issue_gh_136057[0]>(3)test_function() + -> lst = [i for i in range(10)] + (Pdb) next + > <doctest test.test_pdb.test_pdb_issue_gh_136057[0]>(4)test_function() + -> for i in lst: pass + (Pdb) step + --Return-- + > <doctest test.test_pdb.test_pdb_issue_gh_136057[0]>(4)test_function()->None + -> for i in lst: pass + (Pdb) continue + """ + + def test_pdb_issue_gh_80731(): """See GH-80731 @@ -3530,6 +3561,35 @@ def _assert_find_function(self, file_content, func_name, expected): self.assertEqual( expected, pdb.find_function(func_name, os_helper.TESTFN)) + def _fd_dir_for_pipe_targets(self): + """Return a directory exposing live file descriptors, if any.""" + return self._proc_fd_dir() or self._dev_fd_dir() + + def _proc_fd_dir(self): + """Return /proc-backed fd dir when it can be used for pipes.""" + # GH-142836: Opening /proc/self/fd entries for pipes raises EACCES on + # Solaris, so prefer other mechanisms there. + if sys.platform.startswith("sunos"): + return None + + proc_fd = "/proc/self/fd" + if os.path.isdir(proc_fd) and os.path.exists(os.path.join(proc_fd, '0')): + return proc_fd + return None + + def _dev_fd_dir(self): + """Return /dev-backed fd dir when usable.""" + dev_fd = "/dev/fd" + if os.path.isdir(dev_fd) and os.path.exists(os.path.join(dev_fd, '0')): + if sys.platform.startswith("freebsd"): + try: + if os.stat("/dev").st_dev == os.stat(dev_fd).st_dev: + return None + except FileNotFoundError: + return None + return dev_fd + return None + def test_find_function_empty_file(self): self._assert_find_function(b'', 'foo', None) @@ -3588,6 +3648,47 @@ def test_spec(self): stdout, _ = self.run_pdb_script(script, commands) self.assertIn('None', stdout) + def test_script_target_anonymous_pipe(self): + """ + _ScriptTarget doesn't fail on an anonymous pipe. + + GH-142315 + """ + fd_dir = self._fd_dir_for_pipe_targets() + if fd_dir is None: + self.skipTest('anonymous pipe targets require /proc/self/fd or /dev/fd') + + read_fd, write_fd = os.pipe() + + def safe_close(fd): + try: + os.close(fd) + except OSError: + pass + + self.addCleanup(safe_close, read_fd) + self.addCleanup(safe_close, write_fd) + + pipe_path = os.path.join(fd_dir, str(read_fd)) + if not os.path.exists(pipe_path): + self.skipTest('fd directory does not expose anonymous pipes') + + script_source = 'marker = "via_pipe"\n' + os.write(write_fd, script_source.encode('utf-8')) + os.close(write_fd) + + original_path0 = sys.path[0] + self.addCleanup(sys.path.__setitem__, 0, original_path0) + + target = pdb._ScriptTarget(pipe_path) + code_text = target.code + namespace = target.namespace + exec(code_text, namespace) + + self.assertEqual(namespace['marker'], 'via_pipe') + self.assertEqual(namespace['__file__'], target.filename) + self.assertIsNone(namespace['__spec__']) + def test_find_function_first_executable_line(self): code = textwrap.dedent("""\ def foo(): pass @@ -3974,7 +4075,10 @@ def test_run_module_with_args(self): commands = """ continue """ - self._run_pdb(["calendar", "-m"], commands, expected_returncode=2) + self._run_pdb(["calendar", "-m"], commands, expected_returncode=1) + + _, stderr = self._run_pdb(["-m", "calendar", "-p", "1"], commands) + self.assertIn("unrecognized arguments: -p", stderr) stdout, _ = self._run_pdb(["-m", "calendar", "1"], commands) self.assertIn("December", stdout) @@ -4539,6 +4643,43 @@ def bar(): ])) self.assertIn('break in bar', stdout) + def test_end_of_options_separator(self): + # gh-148615: Test parsing when '--' separator is used + script = "import sys; print(f'ARGS: {sys.argv[1:]}')" + with open(os_helper.TESTFN, 'w', encoding='utf-8') as f: + f.write(script) + stdout, _ = self._run_pdb(['--', os_helper.TESTFN, '-foo'], 'c\nq') + self.assertIn("ARGS: ['-foo']", stdout) + stdout, _ = self._run_pdb(['-c', 'continue', '--', os_helper.TESTFN, '-c', 'foo'], 'q') + self.assertIn("ARGS: ['-c', 'foo']", stdout) + stdout, stderr = self._run_pdb(['--'], 'q', expected_returncode=2) + self.assertIn("missing script or module to run", stderr) + stdout, stderr = self._run_pdb(['-x', '--', os_helper.TESTFN], 'q', expected_returncode=2) + self.assertIn("unrecognized arguments: -x", stderr) + stdout, _ = self._run_pdb([os_helper.TESTFN, '--', 'arg'], 'c\nq') + self.assertIn("ARGS: ['--', 'arg']", stdout) + with os_helper.temp_cwd(): + with open('mymod.py', 'w', encoding='utf-8') as f: + f.write(script) + stdout, _ = self._run_pdb(['-m', 'mymod', '--', 'arg'], 'c\nq') + self.assertIn("ARGS: ['--', 'arg']", stdout) + + def test_issue_59000(self): + script = """ + def foo(): + pass + + class C: + def foo(self): + pass + """ + commands = """ + break C.foo + quit + """ + stdout, stderr = self.run_pdb_script(script, commands) + self.assertIn("The specified object 'C.foo' is not a function", stdout) + class ChecklineTests(unittest.TestCase): def setUp(self): @@ -4688,6 +4829,28 @@ def foo(): stdout, _ = self._run_script(script, commands) self.assertIn("42", stdout) + def test_readline_not_imported(self): + """GH-138860 + Directly or indirectly importing readline might deadlock a subprocess + if it's launched with process_group=0 or preexec_fn=setpgrp + + It's also a pattern that readline is never imported with just import pdb. + + This test is to ensure that readline is not imported for import pdb. + It's possible that we have a good reason to do that in the future. + """ + + script = textwrap.dedent(""" + import sys + import pdb + if "readline" in sys.modules: + print("readline imported") + """) + commands = "" + stdout, stderr = self._run_script(script, commands) + self.assertNotIn("readline imported", stdout) + self.assertEqual(stderr, "") + @support.force_colorized_test_class class PdbTestColorize(unittest.TestCase): @@ -4749,7 +4912,9 @@ def test_return_from_inline_mode_to_REPL(self): @support.force_not_colorized_test_class @support.requires_subprocess() class PdbTestReadline(unittest.TestCase): - def setUpClass(): + + @classmethod + def setUpClass(cls): # Ensure that the readline module is loaded # If this fails, the test is skipped because SkipTest will be raised readline = import_module('readline') @@ -4848,6 +5013,9 @@ def f(): self.assertIn(b'I love Python', output) + @unittest.skipIf(sys.platform.startswith(('freebsd', 'dragonfly', 'sunos')), + '\\x08 does not remove the auto-indentation with ' + 'GNU readline on this platform') def test_multiline_auto_indent(self): script = textwrap.dedent(""" import pdb; pdb.Pdb().set_trace() @@ -4886,6 +5054,9 @@ def test_multiline_completion(self): self.assertIn(b'42', output) + @unittest.skipIf(sys.platform.startswith(('freebsd', 'dragonfly', 'sunos')), + '\\x08 does not remove the auto-indentation with ' + 'GNU readline on this platform') def test_multiline_indent_completion(self): script = textwrap.dedent(""" import pdb; pdb.Pdb().set_trace() diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 47f51f1979faabc..d06b9a4b6efb5fd 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -1,3 +1,4 @@ +import ast import dis import gc from itertools import combinations, product @@ -292,6 +293,7 @@ def test_constant_folding_unaryop(self): ('---x', 'UNARY_NEGATIVE', None, False, None, None), ('~~~x', 'UNARY_INVERT', None, False, None, None), ('+++x', 'CALL_INTRINSIC_1', intrinsic_positive, False, None, None), + ('~True', 'UNARY_INVERT', None, False, None, None), ] for ( @@ -316,7 +318,7 @@ def negzero(): return -(1.0-1.0) for instr in dis.get_instructions(negzero): - self.assertFalse(instr.opname.startswith('UNARY_')) + self.assertNotStartsWith(instr.opname, 'UNARY_') self.check_lnotab(negzero) def test_constant_folding_binop(self): @@ -718,9 +720,9 @@ def format(fmt, *values): self.assertEqual(format('x = %d!', 1234), 'x = 1234!') self.assertEqual(format('x = %x!', 1234), 'x = 4d2!') self.assertEqual(format('x = %f!', 1234), 'x = 1234.000000!') - self.assertEqual(format('x = %s!', 1234.5678901), 'x = 1234.5678901!') - self.assertEqual(format('x = %f!', 1234.5678901), 'x = 1234.567890!') - self.assertEqual(format('x = %d!', 1234.5678901), 'x = 1234!') + self.assertEqual(format('x = %s!', 1234.0000625), 'x = 1234.0000625!') + self.assertEqual(format('x = %f!', 1234.0000625), 'x = 1234.000063!') + self.assertEqual(format('x = %d!', 1234.0000625), 'x = 1234!') self.assertEqual(format('x = %s%% %%%%', 1234), 'x = 1234% %%') self.assertEqual(format('x = %s!', '%% %s'), 'x = %% %s!') self.assertEqual(format('x = %s, y = %d', 12, 34), 'x = 12, y = 34') @@ -1118,6 +1120,53 @@ def trace(frame, event, arg): class DirectCfgOptimizerTests(CfgOptimizationTestCase): + def test_optimize_cfg_const_index_out_of_range(self): + insts = [ + ('LOAD_CONST', 2, 0), + ('RETURN_VALUE', None, 0), + ] + seq = self.seq_from_insts(insts) + with self.assertRaisesRegex(ValueError, "out of range"): + _testinternalcapi.optimize_cfg(seq, [0, 1], 0) + + def test_optimize_cfg_consts_must_be_list(self): + insts = [ + ('LOAD_CONST', 0, 0), + ('RETURN_VALUE', None, 0), + ] + seq = self.seq_from_insts(insts) + with self.assertRaisesRegex(TypeError, "consts must be a list"): + _testinternalcapi.optimize_cfg(seq, (0,), 0) + + def test_compiler_codegen_metadata_consts_roundtrips_optimize_cfg(self): + tree = ast.parse("x = (1, 2)", mode="exec", optimize=1) + insts, meta = _testinternalcapi.compiler_codegen(tree, "<s>", 0) + consts = meta["consts"] + self.assertIsInstance(consts, list) + _testinternalcapi.optimize_cfg(insts, consts, 0) + + def test_compiler_codegen_consts_include_none_required_for_implicit_return(self): + # Module "pass" only needs the const table entry for None once + # _PyCodegen_AddReturnAtEnd runs. If metadata["consts"] were taken + # before that, the list would not match LOAD_CONST opargs (here: 0 + # for None), and optimize_cfg would read out of range. + tree = ast.parse("pass", mode="exec", optimize=1) + insts, meta = _testinternalcapi.compiler_codegen(tree, "<s>", 0) + consts = meta["consts"] + self.assertEqual(consts, [None]) + + load_const = opcode.opmap["LOAD_CONST"] + self.assertEqual( + [t[1] for t in insts.get_instructions() if t[0] == load_const], + [0], + ) + + # As if consts were snapshotted before AddReturnAtEnd: still LOAD_CONST 0, no row. + with self.assertRaisesRegex(ValueError, "out of range"): + _testinternalcapi.optimize_cfg(insts, [], 0) + + _testinternalcapi.optimize_cfg(insts, list(consts), 0) + def cfg_optimization_test(self, insts, expected_insts, consts=None, expected_consts=None, nlocals=0): @@ -2614,6 +2663,90 @@ def test_send(self): ] self.cfg_optimization_test(insts, expected, consts=[None]) + def test_format_simple(self): + # FORMAT_SIMPLE will leave its operand on the stack if it's a unicode + # object. We treat it conservatively and assume that it always leaves + # its operand on the stack. + insts = [ + ("LOAD_FAST", 0, 1), + ("FORMAT_SIMPLE", None, 2), + ("STORE_FAST", 1, 3), + ] + self.check(insts, insts) + + insts = [ + ("LOAD_FAST", 0, 1), + ("FORMAT_SIMPLE", None, 2), + ("POP_TOP", None, 3), + ] + expected = [ + ("LOAD_FAST_BORROW", 0, 1), + ("FORMAT_SIMPLE", None, 2), + ("POP_TOP", None, 3), + ] + self.check(insts, expected) + + def test_set_function_attribute(self): + # SET_FUNCTION_ATTRIBUTE leaves the function on the stack + insts = [ + ("LOAD_CONST", 0, 1), + ("LOAD_FAST", 0, 2), + ("SET_FUNCTION_ATTRIBUTE", 2, 3), + ("STORE_FAST", 1, 4), + ("LOAD_CONST", 0, 5), + ("RETURN_VALUE", None, 6) + ] + self.cfg_optimization_test(insts, insts, consts=[None]) + + insts = [ + ("LOAD_CONST", 0, 1), + ("LOAD_FAST", 0, 2), + ("SET_FUNCTION_ATTRIBUTE", 2, 3), + ("RETURN_VALUE", None, 4) + ] + expected = [ + ("LOAD_CONST", 0, 1), + ("LOAD_FAST_BORROW", 0, 2), + ("SET_FUNCTION_ATTRIBUTE", 2, 3), + ("RETURN_VALUE", None, 4) + ] + self.cfg_optimization_test(insts, expected, consts=[None]) + + def test_get_yield_from_iter(self): + # GET_YIELD_FROM_ITER may leave its operand on the stack + insts = [ + ("LOAD_FAST", 0, 1), + ("GET_YIELD_FROM_ITER", None, 2), + ("LOAD_CONST", 0, 3), + send := self.Label(), + ("SEND", end := self.Label(), 5), + ("YIELD_VALUE", 1, 6), + ("RESUME", 2, 7), + ("JUMP", send, 8), + end, + ("END_SEND", None, 9), + ("LOAD_CONST", 0, 10), + ("RETURN_VALUE", None, 11), + ] + self.cfg_optimization_test(insts, insts, consts=[None]) + + def test_push_exc_info(self): + insts = [ + ("LOAD_FAST", 0, 1), + ("PUSH_EXC_INFO", None, 2), + ] + self.check(insts, insts) + + def test_load_special(self): + # LOAD_SPECIAL may leave self on the stack + insts = [ + ("LOAD_FAST", 0, 1), + ("LOAD_SPECIAL", 0, 2), + ("STORE_FAST", 1, 3), + ] + self.check(insts, insts) + + def test_del_in_finally(self): # This loads `obj` onto the stack, executes `del obj`, then returns the # `obj` from the stack. See gh-133371 for more details. @@ -2630,6 +2763,14 @@ def create_obj(): gc.collect() self.assertEqual(obj, [42]) + def test_format_simple_unicode(self): + # Repro from gh-134889 + def f(): + var = f"{1}" + var = f"{var}" + return var + self.assertEqual(f(), "1") + if __name__ == "__main__": diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index 1095e7303c188f6..be091c01498c476 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -100,10 +100,16 @@ def setUpClass(cls): with contextlib.ExitStack() as stack: python_exe = stack.enter_context(support.setup_venv_with_pip_setuptools("venv")) - sitepackages = subprocess.check_output( - [python_exe, "-c", "import sysconfig; print(sysconfig.get_path('platlib'))"], - text=True, - ).strip() + + def get_sysconfig_path(name): + # Force UTF-8 to emit the non-ASCII venv path in any locale. + return subprocess.check_output( + [python_exe, "-X", "utf8", "-c", + f"import sysconfig; print(sysconfig.get_path({name!r}))"], + encoding="utf-8", + ).strip() + + sitepackages = get_sysconfig_path("platlib") stack.enter_context(import_helper.DirsOnSysPath(sitepackages)) cls.addClassCleanup(stack.pop_all().close) @@ -387,10 +393,10 @@ def test_with_stmt_with_paren(self) -> None: test_source = """ stmt = "with (\\n a as b,\\n c as d\\n): pass" the_ast = parse.parse_string(stmt, mode=1) - self.assertTrue(ast_dump(the_ast).startswith( + self.assertStartsWith(ast_dump(the_ast), "Module(body=[With(items=[withitem(context_expr=Name(id='a', ctx=Load()), optional_vars=Name(id='b', ctx=Store())), " "withitem(context_expr=Name(id='c', ctx=Load()), optional_vars=Name(id='d', ctx=Store()))]" - )) + ) """ self.run_test(grammar_source, test_source) diff --git a/Lib/test/test_peg_generator/test_grammar_validator.py b/Lib/test/test_peg_generator/test_grammar_validator.py index c7f20e1de802ce2..857aced8ae5dcf6 100644 --- a/Lib/test/test_peg_generator/test_grammar_validator.py +++ b/Lib/test/test_peg_generator/test_grammar_validator.py @@ -4,7 +4,8 @@ test_tools.skip_if_missing("peg_generator") with test_tools.imports_under_tool("peg_generator"): from pegen.grammar_parser import GeneratedParser as GrammarParser - from pegen.validator import SubRuleValidator, ValidationError, RaiseRuleValidator + from pegen.validator import SubRuleValidator, ValidationError + from pegen.validator import RaiseRuleValidator, CutValidator from pegen.testutil import parse_string from pegen.grammar import Grammar @@ -59,3 +60,18 @@ def test_raising_valid_rule(self) -> None: with self.assertRaises(ValidationError): for rule_name, rule in grammar.rules.items(): validator.validate_rule(rule_name, rule) + + def test_cut_validator(self) -> None: + grammar_source = """ + star: (OP ~ OP)* + plus: (OP ~ OP)+ + bracket: [OP ~ OP] + gather: OP.(OP ~ OP)+ + nested: [OP | NAME ~ OP] + """ + grammar: Grammar = parse_string(grammar_source, GrammarParser) + validator = CutValidator(grammar) + for rule_name, rule in grammar.rules.items(): + with self.subTest(rule_name): + with self.assertRaises(ValidationError): + validator.validate_rule(rule_name, rule) diff --git a/Lib/test/test_peg_generator/test_pegen.py b/Lib/test/test_peg_generator/test_pegen.py index d86065213453904..58ce558c5482795 100644 --- a/Lib/test/test_peg_generator/test_pegen.py +++ b/Lib/test/test_peg_generator/test_pegen.py @@ -91,10 +91,8 @@ def test_gather(self) -> None: """ rules = parse_string(grammar, GrammarParser).rules self.assertEqual(str(rules["start"]), "start: ','.thing+ NEWLINE") - self.assertTrue( - repr(rules["start"]).startswith( - "Rule('start', None, Rhs([Alt([NamedItem(None, Gather(StringLeaf(\"','\"), NameLeaf('thing'" - ) + self.assertStartsWith(repr(rules["start"]), + "Rule('start', None, Rhs([Alt([NamedItem(None, Gather(StringLeaf(\"','\"), NameLeaf('thing'" ) self.assertEqual(str(rules["thing"]), "thing: NUMBER") parser_class = make_parser(grammar) @@ -757,6 +755,30 @@ def test_cut(self) -> None: ], ) + def test_cut_is_local_in_rule(self) -> None: + grammar = """ + start: + | inner + | 'x' { "ok" } + inner: + | 'x' ~ 'y' + | 'x' + """ + parser_class = make_parser(grammar) + node = parse_string("x", parser_class) + self.assertEqual(node, 'ok') + + def test_cut_is_local_in_parens(self) -> None: + # we currently don't guarantee this behavior, see gh-143054 + grammar = """ + start: + | ('x' ~ 'y' | 'x') + | 'x' { "ok" } + """ + parser_class = make_parser(grammar) + node = parse_string("x", parser_class) + self.assertEqual(node, 'ok') + def test_dangling_reference(self) -> None: grammar = """ start: foo ENDMARKER diff --git a/Lib/test/test_perf_profiler.py b/Lib/test/test_perf_profiler.py index c176e505155b905..f46331ced2abb0b 100644 --- a/Lib/test/test_perf_profiler.py +++ b/Lib/test/test_perf_profiler.py @@ -34,6 +34,21 @@ def supports_trampoline_profiling(): raise unittest.SkipTest("perf trampoline profiling not supported") +def _perf_env(**env_vars): + env = os.environ.copy() + # Keep perf's output stable regardless of the builder's perf config. + env.update( + { + "DEBUGINFOD_URLS": "", + "PERF_CONFIG": os.devnull, + } + ) + if env_vars: + env.update(env_vars) + env["PYTHON_JIT"] = "0" + return env + + class TestPerfTrampoline(unittest.TestCase): def setUp(self): super().setUp() @@ -63,13 +78,12 @@ def baz(): """ with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) - env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, "-Xperf", script], text=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - env=env, + env=_perf_env(), ) as process: stdout, stderr = process.communicate() @@ -93,9 +107,7 @@ def baz(): perf_line, f"Could not find {expected_symbol} in perf file" ) perf_addr = perf_line.split(" ")[0] - self.assertFalse( - perf_addr.startswith("0x"), "Address should not be prefixed with 0x" - ) + self.assertNotStartsWith(perf_addr, "0x") self.assertTrue( set(perf_addr).issubset(string.hexdigits), "Address should contain only hex characters", @@ -134,13 +146,12 @@ def baz(): """ with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) - env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, "-Xperf", script], text=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - env=env, + env=_perf_env(), ) as process: stdout, stderr = process.communicate() @@ -162,6 +173,56 @@ def baz(): self.assertIn(f"py::bar_fork:{script}", child_perf_file_contents) self.assertIn(f"py::baz_fork:{script}", child_perf_file_contents) + # The parent's map should not contain the child's symbols. + self.assertNotIn(f"py::foo_fork:{script}", perf_file_contents) + self.assertNotIn(f"py::bar_fork:{script}", perf_file_contents) + self.assertNotIn(f"py::baz_fork:{script}", perf_file_contents) + + # The child's map should not contain the parent's symbols. + self.assertNotIn(f"py::foo:{script}", child_perf_file_contents) + self.assertNotIn(f"py::bar:{script}", child_perf_file_contents) + self.assertNotIn(f"py::baz:{script}", child_perf_file_contents) + + @unittest.skipIf(support.check_bolt_optimized(), "fails on BOLT instrumented binaries") + def test_trampoline_works_after_fork_with_many_code_objects(self): + code = """if 1: + import gc, os, sys, signal + + # Create many code objects so trampoline_refcount > 1 + for i in range(50): + exec(compile(f"def _dummy_{i}(): pass", f"<test{i}>", "exec")) + + pid = os.fork() + if pid == 0: + # Child: create and destroy new code objects, + # then collect garbage. If the old code watcher + # survived the fork, the double-decrement of + # trampoline_refcount will cause a SIGSEGV. + for i in range(50): + exec(compile(f"def _child_{i}(): pass", f"<child{i}>", "exec")) + gc.collect() + os._exit(0) + else: + _, status = os.waitpid(pid, 0) + if os.WIFSIGNALED(status): + print(f"FAIL: child killed by signal {os.WTERMSIG(status)}", file=sys.stderr) + sys.exit(1) + sys.exit(os.WEXITSTATUS(status)) + """ + with temp_dir() as script_dir: + script = make_script(script_dir, "perftest", code) + with subprocess.Popen( + [sys.executable, "-Xperf", script], + text=True, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + env=_perf_env(), + ) as process: + stdout, stderr = process.communicate() + + self.assertEqual(process.returncode, 0, stderr) + self.assertEqual(stderr, "") + @unittest.skipIf(support.check_bolt_optimized(), "fails on BOLT instrumented binaries") def test_sys_api(self): code = """if 1: @@ -186,13 +247,12 @@ def baz(): """ with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) - env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, script], text=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - env=env, + env=_perf_env(), ) as process: stdout, stderr = process.communicate() @@ -233,6 +293,24 @@ def test_sys_api_get_status(self): """ assert_python_ok("-c", code, PYTHON_JIT="0") + def test_sys_api_perf_jit_backend(self): + code = """if 1: + import sys + sys.activate_stack_trampoline("perf_jit") + assert sys.is_stack_trampoline_active() is True + sys.deactivate_stack_trampoline() + assert sys.is_stack_trampoline_active() is False + """ + assert_python_ok("-c", code, PYTHON_JIT="0") + + def test_sys_api_with_existing_perf_jit_trampoline(self): + code = """if 1: + import sys + sys.activate_stack_trampoline("perf_jit") + sys.activate_stack_trampoline("perf_jit") + """ + assert_python_ok("-c", code, PYTHON_JIT="0") + def is_unwinding_reliable_with_frame_pointers(): cflags = sysconfig.get_config_var("PY_CORE_CFLAGS") @@ -271,9 +349,12 @@ def perf_command_works(): "-c", 'print("hello")', ) - env = {**os.environ, "PYTHON_JIT": "0"} stdout = subprocess.check_output( - cmd, cwd=script_dir, text=True, stderr=subprocess.STDOUT, env=env + cmd, + cwd=script_dir, + text=True, + stderr=subprocess.STDOUT, + env=_perf_env(), ) except (subprocess.SubprocessError, OSError): return False @@ -285,42 +366,49 @@ def perf_command_works(): def run_perf(cwd, *args, use_jit=False, **env_vars): - env = os.environ.copy() - if env_vars: - env.update(env_vars) - env["PYTHON_JIT"] = "0" + env = _perf_env(**env_vars) output_file = cwd + "/perf_output.perf" - if not use_jit: - base_cmd = ( - "perf", - "record", - "--no-buildid", - "--no-buildid-cache", - "-g", - "--call-graph=fp", - "-o", output_file, - "--" - ) + base_cmd = [ + "perf", + "record", + "--no-buildid", + "--no-buildid-cache", + "-g", + "--call-graph=dwarf,65528" if use_jit else "--call-graph=fp", + ] + if use_jit: + perf_commands = [] + # Some builders have low perf_event_mlock_kb limits. + mmap_sizes = ("4M", "2M", "1M", "512K", "256K", "128K", None) + for mmap_size in mmap_sizes: + command = base_cmd.copy() + if mmap_size is not None: + command += ["-F99", "-k1", "-m", mmap_size] + else: + command += ["-F99", "-k1"] + command += ["-o", output_file, "--"] + perf_commands.append(command) else: - base_cmd = ( - "perf", - "record", - "--no-buildid", - "--no-buildid-cache", - "-g", - "--call-graph=dwarf,65528", - "-F99", - "-k1", - "-o", - output_file, - "--", + perf_commands = [base_cmd + ["-o", output_file, "--"]] + + mmap_pages_error = "try again with a smaller value of -m/--mmap_pages" + for index, base_cmd in enumerate(perf_commands): + proc = subprocess.run( + base_cmd + list(args), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + text=True, ) - proc = subprocess.run( - base_cmd + args, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env, - ) + if ( + proc.returncode + and use_jit + and index != len(perf_commands) - 1 + and mmap_pages_error in proc.stderr + ): + continue + break + if proc.returncode: print(proc.stderr, file=sys.stderr) raise ValueError(f"Perf failed with return code {proc.returncode}") @@ -329,10 +417,10 @@ def run_perf(cwd, *args, use_jit=False, **env_vars): jit_output_file = cwd + "/jit_output.dump" command = ("perf", "inject", "-j", "-i", output_file, "-o", jit_output_file) proc = subprocess.run( - command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env + command, stderr=subprocess.PIPE, stdout=subprocess.PIPE, env=env, text=True ) if proc.returncode: - print(proc.stderr) + print(proc.stderr, file=sys.stderr) raise ValueError(f"Perf failed with return code {proc.returncode}") # Copy the jit_output_file to the output_file os.rename(jit_output_file, output_file) @@ -344,23 +432,40 @@ def run_perf(cwd, *args, use_jit=False, **env_vars): stderr=subprocess.PIPE, env=env, check=True, + text=True, ) - return proc.stdout.decode("utf-8", "replace"), proc.stderr.decode( - "utf-8", "replace" - ) + return proc.stdout, proc.stderr class TestPerfProfilerMixin: - def run_perf(self, script_dir, perf_mode, script): + PERF_CAPTURE_ATTEMPTS = 3 + + def run_perf(self, script_dir, script, activate_trampoline=True): raise NotImplementedError() + def run_perf_with_retries( + self, script_dir, script, expected_symbols=(), activate_trampoline=True + ): + stdout = stderr = "" + for _ in range(self.PERF_CAPTURE_ATTEMPTS): + stdout, stderr = self.run_perf( + script_dir, script, activate_trampoline=activate_trampoline + ) + if activate_trampoline and any( + symbol not in stdout for symbol in expected_symbols + ): + continue + break + return stdout, stderr + def test_python_calls_appear_in_the_stack_if_perf_activated(self): with temp_dir() as script_dir: code = """if 1: + from itertools import repeat + def foo(n): - x = 0 - for i in range(n): - x += i + for _ in repeat(None, n): + pass def bar(n): foo(n) @@ -368,23 +473,29 @@ def bar(n): def baz(n): bar(n) - baz(10000000) + baz(40000000) """ script = make_script(script_dir, "perftest", code) - stdout, stderr = self.run_perf(script_dir, script) - self.assertEqual(stderr, "") + expected_symbols = [ + f"py::foo:{script}", + f"py::bar:{script}", + f"py::baz:{script}", + ] + stdout, _ = self.run_perf_with_retries( + script_dir, script, expected_symbols + ) - self.assertIn(f"py::foo:{script}", stdout) - self.assertIn(f"py::bar:{script}", stdout) - self.assertIn(f"py::baz:{script}", stdout) + for expected_symbol in expected_symbols: + self.assertIn(expected_symbol, stdout) def test_python_calls_do_not_appear_in_the_stack_if_perf_deactivated(self): with temp_dir() as script_dir: code = """if 1: + from itertools import repeat + def foo(n): - x = 0 - for i in range(n): - x += i + for _ in repeat(None, n): + pass def bar(n): foo(n) @@ -392,13 +503,12 @@ def bar(n): def baz(n): bar(n) - baz(10000000) + baz(40000000) """ script = make_script(script_dir, "perftest", code) - stdout, stderr = self.run_perf( + stdout, _ = self.run_perf_with_retries( script_dir, script, activate_trampoline=False ) - self.assertEqual(stderr, "") self.assertNotIn(f"py::foo:{script}", stdout) self.assertNotIn(f"py::bar:{script}", stdout) @@ -468,13 +578,12 @@ def compile_trampolines_for_all_functions(): with temp_dir() as script_dir: script = make_script(script_dir, "perftest", code) - env = {**os.environ, "PYTHON_JIT": "0"} with subprocess.Popen( [sys.executable, "-Xperf", script], universal_newlines=True, stderr=subprocess.PIPE, stdout=subprocess.PIPE, - env=env, + env=_perf_env(), ) as process: stdout, stderr = process.communicate() @@ -508,9 +617,12 @@ def _is_perf_version_at_least(major, minor): # The output of perf --version looks like "perf version 6.7-3" but # it can also be perf version "perf version 5.15.143", or even include # a commit hash in the version string, like "6.12.9.g242e6068fd5c" + # + # PermissionError is raised if perf does not exist on the Windows Subsystem + # for Linux, see #134987 try: output = subprocess.check_output(["perf", "--version"], text=True) - except (subprocess.CalledProcessError, FileNotFoundError): + except (subprocess.CalledProcessError, FileNotFoundError, PermissionError): return False version = output.split()[2] version = version.split("-")[0] diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index 742ca8de1bea8cf..7e26a6ccfa2c9bf 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -413,6 +413,46 @@ def test_issue18339(self): unpickler.memo = {-1: None} unpickler.memo = {1: None} + def test_concurrent_pickler_dump(self): + f = io.BytesIO() + pickler = self.pickler_class(f) + class X: + def __reduce__(slf): + self.assertRaises(RuntimeError, pickler.dump, 42) + return list, () + pickler.dump(X()) # should not crash + self.assertEqual(pickle.loads(f.getvalue()), []) + + def test_concurrent_pickler_dump_and_init(self): + f = io.BytesIO() + pickler = self.pickler_class(f) + class X: + def __reduce__(slf): + self.assertRaises(RuntimeError, pickler.__init__, f) + return list, () + pickler.dump([X()]) # should not fail + self.assertEqual(pickle.loads(f.getvalue()), [[]]) + + def test_concurrent_unpickler_load(self): + global reducer + def reducer(): + self.assertRaises(RuntimeError, unpickler.load) + return 42 + f = io.BytesIO(b'(c%b\nreducer\n(tRl.' % (__name__.encode(),)) + unpickler = self.unpickler_class(f) + unpickled = unpickler.load() # should not fail + self.assertEqual(unpickled, [42]) + + def test_concurrent_unpickler_load_and_init(self): + global reducer + def reducer(): + self.assertRaises(RuntimeError, unpickler.__init__, f) + return 42 + f = io.BytesIO(b'(c%b\nreducer\n(tRl.' % (__name__.encode(),)) + unpickler = self.unpickler_class(f) + unpickled = unpickler.load() # should not crash + self.assertEqual(unpickled, [42]) + class CDispatchTableTests(AbstractDispatchTableTests, unittest.TestCase): pickler_class = pickle.Pickler def get_dispatch_table(self): @@ -461,7 +501,7 @@ class SizeofTests(unittest.TestCase): check_sizeof = support.check_sizeof def test_pickler(self): - basesize = support.calcobjsize('7P2n3i2n3i2P') + basesize = support.calcobjsize('7P2n3i2n4i2P') p = _pickle.Pickler(io.BytesIO()) self.assertEqual(object.__sizeof__(p), basesize) MT_size = struct.calcsize('3nP0n') @@ -478,7 +518,7 @@ def test_pickler(self): 0) # Write buffer is cleared after every dump(). def test_unpickler(self): - basesize = support.calcobjsize('2P2n2P 2P2n2i5P 2P3n8P2n2i') + basesize = support.calcobjsize('2P2n2P 2P2n2i5P 2P3n8P2n3i') unpickler = _pickle.Unpickler P = struct.calcsize('P') # Size of memo table entry. n = struct.calcsize('n') # Size of mark table entry. @@ -611,10 +651,10 @@ def test_name_mapping(self): with self.subTest(((module3, name3), (module2, name2))): if (module2, name2) == ('exceptions', 'OSError'): attr = getattribute(module3, name3) - self.assertTrue(issubclass(attr, OSError)) + self.assertIsSubclass(attr, OSError) elif (module2, name2) == ('exceptions', 'ImportError'): attr = getattribute(module3, name3) - self.assertTrue(issubclass(attr, ImportError)) + self.assertIsSubclass(attr, ImportError) else: module, name = mapping(module2, name2) if module3[:1] != '_': diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index a178d3353eecdf5..cf990874621eae4 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -384,13 +384,13 @@ def test_string_without_quotes(self): self.check_dis_error(b'Sabc"\n.', '', "no string quotes around b'abc\"'") self.check_dis_error(b"S'abc\n.", '', - '''strinq quote b"'" not found at both ends of b"'abc"''') + '''string quote b"'" not found at both ends of b"'abc"''') self.check_dis_error(b'S"abc\n.', '', - r"""strinq quote b'"' not found at both ends of b'"abc'""") + r"""string quote b'"' not found at both ends of b'"abc'""") self.check_dis_error(b"S'abc\"\n.", '', - r"""strinq quote b"'" not found at both ends of b'\\'abc"'""") + r"""string quote b"'" not found at both ends of b'\\'abc"'""") self.check_dis_error(b"S\"abc'\n.", '', - r"""strinq quote b'"' not found at both ends of b'"abc\\''""") + r"""string quote b'"' not found at both ends of b'"abc\\''""") def test_binstring(self): self.check_dis(b"T\x03\x00\x00\x00abc.", '''\ diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 818e807dd3a6fb6..e879e48571f3f70 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -410,7 +410,7 @@ def test_win32_ver(self): for v in version.split('.'): int(v) # should not fail if csd: - self.assertTrue(csd.startswith('SP'), msg=csd) + self.assertStartsWith(csd, 'SP') if ptype: if os.cpu_count() > 1: self.assertIn('Multiprocessor', ptype) @@ -525,8 +525,10 @@ def test_ios_ver(self): self.assertEqual(override.model, "Whiz") self.assertTrue(override.is_simulator) - @unittest.skipIf(support.is_emscripten, "Does not apply to Emscripten") def test_libc_ver(self): + if support.is_emscripten: + assert platform.libc_ver() == ("emscripten", "4.0.12") + return # check that libc_ver(executable) doesn't raise an exception if os.path.isdir(sys.executable) and \ os.path.exists(sys.executable+'.exe'): @@ -558,6 +560,10 @@ def test_libc_ver(self): # musl uses semver, but we accept some variations anyway: (b'/aports/main/musl/src/musl-12.5', ('musl', '12.5')), (b'/aports/main/musl/src/musl-1.2.5.7', ('musl', '1.2.5.7')), + (b'libc.musl.so.1', ('musl', '1')), + (b'libc.musl-x86_64.so.1.2.5', ('musl', '1.2.5')), + (b'ld-musl.so.1', ('musl', '1')), + (b'ld-musl-x86_64.so.1.2.5', ('musl', '1.2.5')), (b'', ('', '')), ): with open(filename, 'wb') as fp: @@ -576,6 +582,14 @@ def test_libc_ver(self): (b'GLIBC_1.23.4\0GLIBC_1.9\0GLIBC_1.21\0', ('glibc', '1.23.4')), (b'libc.so.2.4\0libc.so.9\0libc.so.23.1\0', ('libc', '23.1')), (b'musl-1.4.1\0musl-2.1.1\0musl-2.0.1\0', ('musl', '2.1.1')), + ( + b'libc.musl-x86_64.so.1.4.1\0libc.musl-x86_64.so.2.1.1\0libc.musl-x86_64.so.2.0.1', + ('musl', '2.1.1'), + ), + ( + b'ld-musl-x86_64.so.1.4.1\0ld-musl-x86_64.so.2.1.1\0ld-musl-x86_64.so.2.0.1', + ('musl', '2.1.1'), + ), (b'no match here, so defaults are used', ('test', '100.1.0')), ): with open(filename, 'wb') as f: diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index a0c76e5dec5ebe0..d9216be4d956588 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -509,6 +509,69 @@ def test_bytes(self): data2 = plistlib.dumps(pl2) self.assertEqual(data, data2) + def test_bytes_indent(self): + header = ( + b'<?xml version="1.0" encoding="UTF-8"?>\n' + b'<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">\n' + b'<plist version="1.0">\n') + data = [{'bytes': bytes(range(50))}] + pl = plistlib.dumps(data) + self.assertEqual(pl, header + + b'<array>\n' + b'\t<dict>\n' + b'\t\t<key>bytes</key>\n' + b'\t\t<data>\n' + b'\t\tAAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKiss\n' + b'\t\tLS4vMDE=\n' + b'\t\t</data>\n' + b'\t</dict>\n' + b'</array>\n' + b'</plist>\n') + + def dumps_with_indent(data, indent): + fp = BytesIO() + writer = plistlib._PlistWriter(fp, indent=indent) + writer.write(data) + return fp.getvalue() + + pl = dumps_with_indent(data, b' ') + self.assertEqual(pl, header + + b'<array>\n' + b' <dict>\n' + b' <key>bytes</key>\n' + b' <data>\n' + b' AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDE=\n' + b' </data>\n' + b' </dict>\n' + b'</array>\n' + b'</plist>\n') + + pl = dumps_with_indent(data, b' \t') + self.assertEqual(pl, header + + b'<array>\n' + b' \t<dict>\n' + b' \t \t<key>bytes</key>\n' + b' \t \t<data>\n' + b' \t \tAAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKiss\n' + b' \t \tLS4vMDE=\n' + b' \t \t</data>\n' + b' \t</dict>\n' + b'</array>\n' + b'</plist>\n') + + pl = dumps_with_indent(data, b'\t ') + self.assertEqual(pl, header + + b'<array>\n' + b'\t <dict>\n' + b'\t \t <key>bytes</key>\n' + b'\t \t <data>\n' + b'\t \t AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygp\n' + b'\t \t KissLS4vMDE=\n' + b'\t \t </data>\n' + b'\t </dict>\n' + b'</array>\n' + b'</plist>\n') + def test_loads_str_with_xml_fmt(self): pl = self._create() b = plistlib.dumps(pl) @@ -581,7 +644,6 @@ def test_appleformatting(self): self.assertEqual(data, TESTDATA[fmt], "generated data was not identical to Apple's output") - def test_appleformattingfromliteral(self): self.maxDiff = None for fmt in ALL_FORMATS: @@ -903,8 +965,7 @@ def test_dump_naive_datetime_with_aware_datetime_option(self): class TestBinaryPlistlib(unittest.TestCase): - @staticmethod - def decode(*objects, offset_size=1, ref_size=1): + def build(self, *objects, offset_size=1, ref_size=1): data = [b'bplist00'] offset = 8 offsets = [] @@ -916,7 +977,11 @@ def decode(*objects, offset_size=1, ref_size=1): len(objects), 0, offset) data.extend(offsets) data.append(tail) - return plistlib.loads(b''.join(data), fmt=plistlib.FMT_BINARY) + return b''.join(data) + + def decode(self, *objects, offset_size=1, ref_size=1): + data = self.build(*objects, offset_size=offset_size, ref_size=ref_size) + return plistlib.loads(data, fmt=plistlib.FMT_BINARY) def test_nonstandard_refs_size(self): # Issue #21538: Refs and offsets are 24-bit integers @@ -1024,6 +1089,34 @@ def test_invalid_binary(self): with self.assertRaises(plistlib.InvalidFileException): plistlib.loads(b'bplist00' + data, fmt=plistlib.FMT_BINARY) + def test_truncated_large_data(self): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + def check(data): + with open(os_helper.TESTFN, 'wb') as f: + f.write(data) + # buffered file + with open(os_helper.TESTFN, 'rb') as f: + with self.assertRaises(plistlib.InvalidFileException): + plistlib.load(f, fmt=plistlib.FMT_BINARY) + # unbuffered file + with open(os_helper.TESTFN, 'rb', buffering=0) as f: + with self.assertRaises(plistlib.InvalidFileException): + plistlib.load(f, fmt=plistlib.FMT_BINARY) + for w in range(20, 64): + s = 1 << w + # data + check(self.build(b'\x4f\x13' + s.to_bytes(8, 'big'))) + # ascii string + check(self.build(b'\x5f\x13' + s.to_bytes(8, 'big'))) + # unicode string + check(self.build(b'\x6f\x13' + s.to_bytes(8, 'big'))) + # array + check(self.build(b'\xaf\x13' + s.to_bytes(8, 'big'))) + # dict + check(self.build(b'\xdf\x13' + s.to_bytes(8, 'big'))) + # number of objects + check(b'bplist00' + struct.pack('>6xBBQQQ', 1, 1, s, 0, 8)) + def test_load_aware_datetime(self): data = (b'bplist003B\x04>\xd0d\x00\x00\x00\x08\x00\x00\x00\x00\x00\x00' b'\x01\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00' diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 0817d0a87a38b19..cdc80b1dc0d345b 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -46,7 +46,8 @@ def _supports_sched(): try: posix.sched_getscheduler(0) except OSError as e: - if e.errno == errno.ENOSYS: + # DragonFly BSD requires privileges to use the scheduler API. + if e.errno in (errno.ENOSYS, errno.EPERM): return False return True @@ -415,8 +416,10 @@ def test_posix_fallocate(self): if inst.errno == errno.EINVAL and sys.platform.startswith( ('sunos', 'freebsd', 'openbsd', 'gnukfreebsd')): raise unittest.SkipTest("test may fail on ZFS filesystems") - elif inst.errno == errno.EOPNOTSUPP and sys.platform.startswith("netbsd"): - raise unittest.SkipTest("test may fail on FFS filesystems") + elif inst.errno == errno.EOPNOTSUPP: + # ZFS on FreeBSD, FFS on NetBSD, etc. + raise unittest.SkipTest( + "the file system does not support posix_fallocate()") else: raise finally: @@ -668,6 +671,18 @@ def test_fstat(self): finally: fp.close() + @unittest.skipUnless(hasattr(posix, 'stat'), + 'test needs posix.stat()') + @unittest.skipUnless(os.stat in os.supports_follow_symlinks, + 'test needs follow_symlinks support in os.stat()') + def test_stat_fd_zero_follow_symlinks(self): + with self.assertRaisesRegex(ValueError, + 'cannot use fd and follow_symlinks together'): + posix.stat(0, follow_symlinks=False) + with self.assertRaisesRegex(ValueError, + 'cannot use fd and follow_symlinks together'): + posix.stat(1, follow_symlinks=False) + def test_stat(self): self.assertTrue(posix.stat(os_helper.TESTFN)) self.assertTrue(posix.stat(os.fsencode(os_helper.TESTFN))) @@ -757,7 +772,7 @@ def test_makedev(self): self.assertRaises((ValueError, OverflowError), posix.makedev, x, minor) self.assertRaises((ValueError, OverflowError), posix.makedev, major, x) - if sys.platform == 'linux': + if sys.platform == 'linux' and not support.linked_to_musl(): NODEV = -1 self.assertEqual(posix.major(NODEV), NODEV) self.assertEqual(posix.minor(NODEV), NODEV) @@ -826,7 +841,9 @@ def check_stat(uid, gid): self.assertRaises(OSError, chown_func, first_param, 0, -1) check_stat(uid, gid) if hasattr(os, 'getgroups'): - if 0 not in os.getgroups(): + # Also check the effective gid, which the kernel + # accepts for chown even if not in getgroups(). + if 0 not in os.getgroups() and os.getegid() != 0: self.assertRaises(OSError, chown_func, first_param, -1, 0) check_stat(uid, gid) # test illegal types @@ -1107,7 +1124,7 @@ def test_lchmod_dir_symlink(self): def _test_chflags_regular_file(self, chflags_func, target_file, **kwargs): st = os.stat(target_file) - self.assertTrue(hasattr(st, 'st_flags')) + self.assertHasAttr(st, 'st_flags') # ZFS returns EOPNOTSUPP when attempting to set flag UF_IMMUTABLE. flags = st.st_flags | stat.UF_IMMUTABLE @@ -1143,7 +1160,7 @@ def test_lchflags_regular_file(self): def test_lchflags_symlink(self): testfn_st = os.stat(os_helper.TESTFN) - self.assertTrue(hasattr(testfn_st, 'st_flags')) + self.assertHasAttr(testfn_st, 'st_flags') self.addCleanup(os_helper.unlink, _DUMMY_SYMLINK) os.symlink(os_helper.TESTFN, _DUMMY_SYMLINK) @@ -1366,6 +1383,15 @@ def test_sched_param(self): self.assertNotEqual(newparam, param) self.assertEqual(newparam.sched_priority, 0) + @requires_sched + def test_bug_140634(self): + sched_priority = float('inf') # any new reference + param = posix.sched_param(sched_priority) + param.__reduce__() + del sched_priority, param # should not crash + support.gc_collect() # just to be sure + + @requires_sched @unittest.skipUnless(hasattr(posix, "sched_rr_get_interval"), "no function") def test_sched_rr_get_interval(self): try: @@ -1913,6 +1939,11 @@ def test_setpgroup(self): ) support.wait_process(pid, exitcode=0) + def test_setpgroup_allow_none(self): + path, args = self.NOOP_PROGRAM[0], self.NOOP_PROGRAM + pid = self.spawn_func(path, args, os.environ, setpgroup=None) + support.wait_process(pid, exitcode=0) + def test_setpgroup_wrong_type(self): with self.assertRaises(TypeError): self.spawn_func(sys.executable, @@ -2013,6 +2044,20 @@ def test_setsigdef_wrong_type(self): [sys.executable, "-c", "pass"], os.environ, setsigdef=[signal.NSIG, signal.NSIG+1]) + def test_scheduler_allow_none(self): + path, args = self.NOOP_PROGRAM[0], self.NOOP_PROGRAM + pid = self.spawn_func(path, args, os.environ, scheduler=None) + support.wait_process(pid, exitcode=0) + + @support.subTests("scheduler", [object(), 1, [1, 2]]) + def test_scheduler_wrong_type(self, scheduler): + path, args = self.NOOP_PROGRAM[0], self.NOOP_PROGRAM + with self.assertRaisesRegex( + TypeError, + "scheduler must be a tuple or None", + ): + self.spawn_func(path, args, os.environ, scheduler=scheduler) + @requires_sched @unittest.skipIf(sys.platform.startswith(('freebsd', 'netbsd')), "bpo-34685: test can fail on BSD") @@ -2036,6 +2081,12 @@ def test_setscheduler_only_param(self): @requires_sched @unittest.skipIf(sys.platform.startswith(('freebsd', 'netbsd')), "bpo-34685: test can fail on BSD") + @unittest.skipIf(platform.libc_ver()[0] == 'glibc' and + os.sched_getscheduler(0) in [ + os.SCHED_BATCH, + os.SCHED_IDLE, + os.SCHED_DEADLINE], + "Skip test due to glibc posix_spawn policy") def test_setscheduler_with_policy(self): policy = os.sched_getscheduler(0) priority = os.sched_get_priority_min(policy) @@ -2218,12 +2269,12 @@ def _verify_available(self, name): def test_pwritev(self): self._verify_available("HAVE_PWRITEV") if self.mac_ver >= (10, 16): - self.assertTrue(hasattr(os, "pwritev"), "os.pwritev is not available") - self.assertTrue(hasattr(os, "preadv"), "os.readv is not available") + self.assertHasAttr(os, "pwritev") + self.assertHasAttr(os, "preadv") else: - self.assertFalse(hasattr(os, "pwritev"), "os.pwritev is available") - self.assertFalse(hasattr(os, "preadv"), "os.readv is available") + self.assertNotHasAttr(os, "pwritev") + self.assertNotHasAttr(os, "preadv") def test_stat(self): self._verify_available("HAVE_FSTATAT") diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index fa19d549c26cfad..21f06712548d888 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -4,12 +4,13 @@ import random import sys import unittest -from posixpath import realpath, abspath, dirname, basename +from functools import partial +from posixpath import realpath, abspath, dirname, basename, ALLOW_MISSING from test import support from test import test_genericpath from test.support import import_helper from test.support import os_helper -from test.support.os_helper import FakePath +from test.support.os_helper import FakePath, TESTFN from unittest import mock try: @@ -21,7 +22,7 @@ # An absolute path to a temporary filename for testing. We can't rely on TESTFN # being an absolute path, so we need this. -ABSTFN = abspath(os_helper.TESTFN) +ABSTFN = abspath(TESTFN) def skip_if_ABSTFN_contains_backslash(test): """ @@ -33,21 +34,16 @@ def skip_if_ABSTFN_contains_backslash(test): msg = "ABSTFN is not a posix path - tests fail" return [test, unittest.skip(msg)(test)][found_backslash] -def safe_rmdir(dirname): - try: - os.rmdir(dirname) - except OSError: - pass + +def _parameterize(*parameters): + return support.subTests('kwargs', parameters) + class PosixPathTest(unittest.TestCase): def setUp(self): - self.tearDown() - - def tearDown(self): for suffix in ["", "1", "2"]: - os_helper.unlink(os_helper.TESTFN + suffix) - safe_rmdir(os_helper.TESTFN + suffix) + self.assertFalse(posixpath.lexists(ABSTFN + suffix)) def test_join(self): fn = posixpath.join @@ -194,25 +190,28 @@ def test_dirname(self): self.assertEqual(posixpath.dirname(b"//foo//bar"), b"//foo") def test_islink(self): - self.assertIs(posixpath.islink(os_helper.TESTFN + "1"), False) - self.assertIs(posixpath.lexists(os_helper.TESTFN + "2"), False) + self.assertIs(posixpath.islink(TESTFN + "1"), False) + self.assertIs(posixpath.lexists(TESTFN + "2"), False) - with open(os_helper.TESTFN + "1", "wb") as f: + self.addCleanup(os_helper.unlink, TESTFN + "1") + with open(TESTFN + "1", "wb") as f: f.write(b"foo") - self.assertIs(posixpath.islink(os_helper.TESTFN + "1"), False) + self.assertIs(posixpath.islink(TESTFN + "1"), False) if os_helper.can_symlink(): - os.symlink(os_helper.TESTFN + "1", os_helper.TESTFN + "2") - self.assertIs(posixpath.islink(os_helper.TESTFN + "2"), True) - os.remove(os_helper.TESTFN + "1") - self.assertIs(posixpath.islink(os_helper.TESTFN + "2"), True) - self.assertIs(posixpath.exists(os_helper.TESTFN + "2"), False) - self.assertIs(posixpath.lexists(os_helper.TESTFN + "2"), True) - - self.assertIs(posixpath.islink(os_helper.TESTFN + "\udfff"), False) - self.assertIs(posixpath.islink(os.fsencode(os_helper.TESTFN) + b"\xff"), False) - self.assertIs(posixpath.islink(os_helper.TESTFN + "\x00"), False) - self.assertIs(posixpath.islink(os.fsencode(os_helper.TESTFN) + b"\x00"), False) + self.addCleanup(os_helper.unlink, TESTFN + "2") + os.symlink(TESTFN + "1", TESTFN + "2") + self.assertIs(posixpath.islink(TESTFN + "2"), True) + os.remove(TESTFN + "1") + self.assertIs(posixpath.islink(TESTFN + "2"), True) + self.assertIs(posixpath.exists(TESTFN + "2"), False) + self.assertIs(posixpath.lexists(TESTFN + "2"), True) + + def test_islink_invalid_paths(self): + self.assertIs(posixpath.islink(TESTFN + "\udfff"), False) + self.assertIs(posixpath.islink(os.fsencode(TESTFN) + b"\xff"), False) + self.assertIs(posixpath.islink(TESTFN + "\x00"), False) + self.assertIs(posixpath.islink(os.fsencode(TESTFN) + b"\x00"), False) def test_ismount(self): self.assertIs(posixpath.ismount("/"), True) @@ -227,8 +226,9 @@ def test_ismount_non_existent(self): os.mkdir(ABSTFN) self.assertIs(posixpath.ismount(ABSTFN), False) finally: - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN) + def test_ismount_invalid_paths(self): self.assertIs(posixpath.ismount('/\udfff'), False) self.assertIs(posixpath.ismount(b'/\xff'), False) self.assertIs(posixpath.ismount('/\x00'), False) @@ -241,7 +241,7 @@ def test_ismount_symlinks(self): os.symlink("/", ABSTFN) self.assertIs(posixpath.ismount(ABSTFN), False) finally: - os.unlink(ABSTFN) + os_helper.unlink(ABSTFN) @unittest.skipIf(posix is None, "Test requires posix module") def test_ismount_different_device(self): @@ -448,32 +448,35 @@ def test_normpath(self): self.assertEqual(result, expected) @skip_if_ABSTFN_contains_backslash - def test_realpath_curdir(self): - self.assertEqual(realpath('.'), os.getcwd()) - self.assertEqual(realpath('./.'), os.getcwd()) - self.assertEqual(realpath('/'.join(['.'] * 100)), os.getcwd()) + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_curdir(self, kwargs): + self.assertEqual(realpath('.', **kwargs), os.getcwd()) + self.assertEqual(realpath('./.', **kwargs), os.getcwd()) + self.assertEqual(realpath('/'.join(['.'] * 100), **kwargs), os.getcwd()) - self.assertEqual(realpath(b'.'), os.getcwdb()) - self.assertEqual(realpath(b'./.'), os.getcwdb()) - self.assertEqual(realpath(b'/'.join([b'.'] * 100)), os.getcwdb()) + self.assertEqual(realpath(b'.', **kwargs), os.getcwdb()) + self.assertEqual(realpath(b'./.', **kwargs), os.getcwdb()) + self.assertEqual(realpath(b'/'.join([b'.'] * 100), **kwargs), os.getcwdb()) @skip_if_ABSTFN_contains_backslash - def test_realpath_pardir(self): - self.assertEqual(realpath('..'), dirname(os.getcwd())) - self.assertEqual(realpath('../..'), dirname(dirname(os.getcwd()))) - self.assertEqual(realpath('/'.join(['..'] * 100)), '/') + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_pardir(self, kwargs): + self.assertEqual(realpath('..', **kwargs), dirname(os.getcwd())) + self.assertEqual(realpath('../..', **kwargs), dirname(dirname(os.getcwd()))) + self.assertEqual(realpath('/'.join(['..'] * 100), **kwargs), '/') - self.assertEqual(realpath(b'..'), dirname(os.getcwdb())) - self.assertEqual(realpath(b'../..'), dirname(dirname(os.getcwdb()))) - self.assertEqual(realpath(b'/'.join([b'..'] * 100)), b'/') + self.assertEqual(realpath(b'..', **kwargs), dirname(os.getcwdb())) + self.assertEqual(realpath(b'../..', **kwargs), dirname(dirname(os.getcwdb()))) + self.assertEqual(realpath(b'/'.join([b'..'] * 100), **kwargs), b'/') @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_basic(self): + @_parameterize({}, {'strict': ALLOW_MISSING}) + def test_realpath_basic(self, kwargs): # Basic operation. try: os.symlink(ABSTFN+"1", ABSTFN) - self.assertEqual(realpath(ABSTFN), ABSTFN+"1") + self.assertEqual(realpath(ABSTFN, **kwargs), ABSTFN+"1") finally: os_helper.unlink(ABSTFN) @@ -489,23 +492,121 @@ def test_realpath_strict(self): finally: os_helper.unlink(ABSTFN) + def test_realpath_invalid_paths(self): + path = '/\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = b'/\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = '/nonexistent/x\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = b'/nonexistent/x\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = '/\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = b'/\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + + path = '/nonexistent/x\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = b'/nonexistent/x\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + + path = '/\udfff' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), path) + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=True) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) + path = '/nonexistent/\udfff' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), path) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), path) + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + path = '/\udfff/..' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), '/') + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), '/') + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=True) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) + path = '/nonexistent/\udfff/..' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), '/nonexistent') + self.assertEqual(realpath(path, strict=ALLOW_MISSING), '/nonexistent') + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + + path = b'/\xff' + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=True) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=ALLOW_MISSING) + else: + self.assertEqual(realpath(path, strict=False), path) + if support.is_wasi: + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + else: + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), path) + path = b'/nonexistent/\xff' + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=ALLOW_MISSING) + else: + self.assertEqual(realpath(path, strict=False), path) + if support.is_wasi: + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + else: + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_relative(self): + @_parameterize({}, {'strict': ALLOW_MISSING}) + def test_realpath_relative(self, kwargs): try: os.symlink(posixpath.relpath(ABSTFN+"1"), ABSTFN) - self.assertEqual(realpath(ABSTFN), ABSTFN+"1") + self.assertEqual(realpath(ABSTFN, **kwargs), ABSTFN+"1") finally: os_helper.unlink(ABSTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_missing_pardir(self): + @_parameterize({}, {'strict': ALLOW_MISSING}) + def test_realpath_missing_pardir(self, kwargs): try: - os.symlink(os_helper.TESTFN + "1", os_helper.TESTFN) - self.assertEqual(realpath("nonexistent/../" + os_helper.TESTFN), ABSTFN + "1") + os.symlink(TESTFN + "1", TESTFN) + self.assertEqual( + realpath("nonexistent/../" + TESTFN, **kwargs), ABSTFN + "1") finally: - os_helper.unlink(os_helper.TESTFN) + os_helper.unlink(TESTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash @@ -550,37 +651,38 @@ def test_realpath_symlink_loops(self): @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_symlink_loops_strict(self): + @_parameterize({'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_symlink_loops_strict(self, kwargs): # Bug #43757, raise OSError if we get into an infinite symlink loop in - # strict mode. + # the strict modes. try: os.symlink(ABSTFN, ABSTFN) - self.assertRaises(OSError, realpath, ABSTFN, strict=True) + self.assertRaises(OSError, realpath, ABSTFN, **kwargs) os.symlink(ABSTFN+"1", ABSTFN+"2") os.symlink(ABSTFN+"2", ABSTFN+"1") - self.assertRaises(OSError, realpath, ABSTFN+"1", strict=True) - self.assertRaises(OSError, realpath, ABSTFN+"2", strict=True) + self.assertRaises(OSError, realpath, ABSTFN+"1", **kwargs) + self.assertRaises(OSError, realpath, ABSTFN+"2", **kwargs) - self.assertRaises(OSError, realpath, ABSTFN+"1/x", strict=True) - self.assertRaises(OSError, realpath, ABSTFN+"1/..", strict=True) - self.assertRaises(OSError, realpath, ABSTFN+"1/../x", strict=True) + self.assertRaises(OSError, realpath, ABSTFN+"1/x", **kwargs) + self.assertRaises(OSError, realpath, ABSTFN+"1/..", **kwargs) + self.assertRaises(OSError, realpath, ABSTFN+"1/../x", **kwargs) os.symlink(ABSTFN+"x", ABSTFN+"y") self.assertRaises(OSError, realpath, - ABSTFN+"1/../" + basename(ABSTFN) + "y", strict=True) + ABSTFN+"1/../" + basename(ABSTFN) + "y", **kwargs) self.assertRaises(OSError, realpath, - ABSTFN+"1/../" + basename(ABSTFN) + "1", strict=True) + ABSTFN+"1/../" + basename(ABSTFN) + "1", **kwargs) os.symlink(basename(ABSTFN) + "a/b", ABSTFN+"a") - self.assertRaises(OSError, realpath, ABSTFN+"a", strict=True) + self.assertRaises(OSError, realpath, ABSTFN+"a", **kwargs) os.symlink("../" + basename(dirname(ABSTFN)) + "/" + basename(ABSTFN) + "c", ABSTFN+"c") - self.assertRaises(OSError, realpath, ABSTFN+"c", strict=True) + self.assertRaises(OSError, realpath, ABSTFN+"c", **kwargs) # Test using relative path as well. with os_helper.change_cwd(dirname(ABSTFN)): - self.assertRaises(OSError, realpath, basename(ABSTFN), strict=True) + self.assertRaises(OSError, realpath, basename(ABSTFN), **kwargs) finally: os_helper.unlink(ABSTFN) os_helper.unlink(ABSTFN+"1") @@ -591,28 +693,30 @@ def test_realpath_symlink_loops_strict(self): @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_repeated_indirect_symlinks(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_repeated_indirect_symlinks(self, kwargs): # Issue #6975. try: os.mkdir(ABSTFN) os.symlink('../' + basename(ABSTFN), ABSTFN + '/self') os.symlink('self/self/self', ABSTFN + '/link') - self.assertEqual(realpath(ABSTFN + '/link'), ABSTFN) + self.assertEqual(realpath(ABSTFN + '/link', **kwargs), ABSTFN) finally: os_helper.unlink(ABSTFN + '/self') os_helper.unlink(ABSTFN + '/link') - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_deep_recursion(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_deep_recursion(self, kwargs): depth = 10 try: os.mkdir(ABSTFN) for i in range(depth): os.symlink('/'.join(['%d' % i] * 10), ABSTFN + '/%d' % (i + 1)) os.symlink('.', ABSTFN + '/0') - self.assertEqual(realpath(ABSTFN + '/%d' % depth), ABSTFN) + self.assertEqual(realpath(ABSTFN + '/%d' % depth, **kwargs), ABSTFN) # Test using relative path as well. with os_helper.change_cwd(ABSTFN): @@ -620,11 +724,12 @@ def test_realpath_deep_recursion(self): finally: for i in range(depth + 1): os_helper.unlink(ABSTFN + '/%d' % i) - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_resolve_parents(self): + @_parameterize({}, {'strict': ALLOW_MISSING}) + def test_realpath_resolve_parents(self, kwargs): # We also need to resolve any symlinks in the parents of a relative # path passed to realpath. E.g.: current working directory is # /usr/doc with 'doc' being a symlink to /usr/share/doc. We call @@ -635,15 +740,17 @@ def test_realpath_resolve_parents(self): os.symlink(ABSTFN + "/y", ABSTFN + "/k") with os_helper.change_cwd(ABSTFN + "/k"): - self.assertEqual(realpath("a"), ABSTFN + "/y/a") + self.assertEqual(realpath("a", **kwargs), + ABSTFN + "/y/a") finally: os_helper.unlink(ABSTFN + "/k") - safe_rmdir(ABSTFN + "/y") - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN + "/y") + os_helper.rmdir(ABSTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_resolve_before_normalizing(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_resolve_before_normalizing(self, kwargs): # Bug #990669: Symbolic links should be resolved before we # normalize the path. E.g.: if we have directories 'a', 'k' and 'y' # in the following hierarchy: @@ -658,20 +765,21 @@ def test_realpath_resolve_before_normalizing(self): os.symlink(ABSTFN + "/k/y", ABSTFN + "/link-y") # Absolute path. - self.assertEqual(realpath(ABSTFN + "/link-y/.."), ABSTFN + "/k") + self.assertEqual(realpath(ABSTFN + "/link-y/..", **kwargs), ABSTFN + "/k") # Relative path. with os_helper.change_cwd(dirname(ABSTFN)): - self.assertEqual(realpath(basename(ABSTFN) + "/link-y/.."), + self.assertEqual(realpath(basename(ABSTFN) + "/link-y/..", **kwargs), ABSTFN + "/k") finally: os_helper.unlink(ABSTFN + "/link-y") - safe_rmdir(ABSTFN + "/k/y") - safe_rmdir(ABSTFN + "/k") - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN + "/k/y") + os_helper.rmdir(ABSTFN + "/k") + os_helper.rmdir(ABSTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_resolve_first(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_resolve_first(self, kwargs): # Bug #1213894: The first component of the path, if not absolute, # must be resolved too. @@ -681,12 +789,12 @@ def test_realpath_resolve_first(self): os.symlink(ABSTFN, ABSTFN + "link") with os_helper.change_cwd(dirname(ABSTFN)): base = basename(ABSTFN) - self.assertEqual(realpath(base + "link"), ABSTFN) - self.assertEqual(realpath(base + "link/k"), ABSTFN + "/k") + self.assertEqual(realpath(base + "link", **kwargs), ABSTFN) + self.assertEqual(realpath(base + "link/k", **kwargs), ABSTFN + "/k") finally: os_helper.unlink(ABSTFN + "link") - safe_rmdir(ABSTFN + "/k") - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN + "/k") + os_helper.rmdir(ABSTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash @@ -700,12 +808,67 @@ def test_realpath_unreadable_symlink(self): self.assertEqual(realpath(ABSTFN + '/foo'), ABSTFN + '/foo') self.assertEqual(realpath(ABSTFN + '/../foo'), dirname(ABSTFN) + '/foo') self.assertEqual(realpath(ABSTFN + '/foo/..'), ABSTFN) + finally: + os.chmod(ABSTFN, 0o755, follow_symlinks=False) + os_helper.unlink(ABSTFN) + + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + @unittest.skipIf(os.chmod not in os.supports_follow_symlinks, "Can't set symlink permissions") + @unittest.skipIf(sys.platform != "darwin", "only macOS requires read permission to readlink()") + @_parameterize({'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_unreadable_symlink_strict(self, kwargs): + try: + os.symlink(ABSTFN+"1", ABSTFN) + os.chmod(ABSTFN, 0o000, follow_symlinks=False) + with self.assertRaises(PermissionError): + realpath(ABSTFN, **kwargs) + with self.assertRaises(PermissionError): + realpath(ABSTFN + '/foo', **kwargs), with self.assertRaises(PermissionError): - realpath(ABSTFN, strict=True) + realpath(ABSTFN + '/../foo', **kwargs) + with self.assertRaises(PermissionError): + realpath(ABSTFN + '/foo/..', **kwargs) finally: os.chmod(ABSTFN, 0o755, follow_symlinks=False) os.unlink(ABSTFN) + @skip_if_ABSTFN_contains_backslash + @os_helper.skip_unless_symlink + def test_realpath_unreadable_directory(self): + try: + os.mkdir(ABSTFN) + os.mkdir(ABSTFN + '/k') + os.chmod(ABSTFN, 0o000) + self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=ALLOW_MISSING), ABSTFN) + + try: + os.stat(ABSTFN) + except PermissionError: + pass + else: + self.skipTest('Cannot block permissions') + + self.assertEqual(realpath(ABSTFN + '/k', strict=False), + ABSTFN + '/k') + self.assertRaises(PermissionError, realpath, ABSTFN + '/k', + strict=True) + self.assertRaises(PermissionError, realpath, ABSTFN + '/k', + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + '/missing', strict=False), + ABSTFN + '/missing') + self.assertRaises(PermissionError, realpath, ABSTFN + '/missing', + strict=True) + self.assertRaises(PermissionError, realpath, ABSTFN + '/missing', + strict=ALLOW_MISSING) + finally: + os.chmod(ABSTFN, 0o755) + os_helper.rmdir(ABSTFN + '/k') + os_helper.rmdir(ABSTFN) + @skip_if_ABSTFN_contains_backslash def test_realpath_nonterminal_file(self): try: @@ -713,14 +876,27 @@ def test_realpath_nonterminal_file(self): f.write('test_posixpath wuz ere') self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN) self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=ALLOW_MISSING), ABSTFN) + self.assertEqual(realpath(ABSTFN + "/", strict=False), ABSTFN) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/.", strict=False), ABSTFN) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/..", strict=False), dirname(ABSTFN)) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/subdir", strict=False), ABSTFN + "/subdir") self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", + strict=ALLOW_MISSING) finally: os_helper.unlink(ABSTFN) @@ -733,16 +909,30 @@ def test_realpath_nonterminal_symlink_to_file(self): os.symlink(ABSTFN + "1", ABSTFN) self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN + "1") self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN + "1") + self.assertEqual(realpath(ABSTFN, strict=ALLOW_MISSING), ABSTFN + "1") + self.assertEqual(realpath(ABSTFN + "/", strict=False), ABSTFN + "1") self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/.", strict=False), ABSTFN + "1") self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/..", strict=False), dirname(ABSTFN)) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/subdir", strict=False), ABSTFN + "1/subdir") self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", + strict=ALLOW_MISSING) finally: os_helper.unlink(ABSTFN) + os_helper.unlink(ABSTFN + "1") @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash @@ -754,16 +944,31 @@ def test_realpath_nonterminal_symlink_to_symlinks_to_file(self): os.symlink(ABSTFN + "1", ABSTFN) self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN + "2") self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN + "2") + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN + "2") + self.assertEqual(realpath(ABSTFN + "/", strict=False), ABSTFN + "2") self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/.", strict=False), ABSTFN + "2") self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/..", strict=False), dirname(ABSTFN)) self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", + strict=ALLOW_MISSING) + self.assertEqual(realpath(ABSTFN + "/subdir", strict=False), ABSTFN + "2/subdir") self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", + strict=ALLOW_MISSING) finally: os_helper.unlink(ABSTFN) + os_helper.unlink(ABSTFN + "1") + os_helper.unlink(ABSTFN + "2") def test_relpath(self): (real_getcwd, os.getcwd) = (os.getcwd, lambda: r"/home/user/bar") @@ -889,8 +1094,8 @@ class PathLikeTests(unittest.TestCase): path = posixpath def setUp(self): - self.file_name = os_helper.TESTFN - self.file_path = FakePath(os_helper.TESTFN) + self.file_name = TESTFN + self.file_path = FakePath(TESTFN) self.addCleanup(os_helper.unlink, self.file_name) with open(self.file_name, 'xb', 0) as file: file.write(b"test_posixpath.PathLikeTests") @@ -947,9 +1152,12 @@ def test_path_normpath(self): def test_path_abspath(self): self.assertPathEqual(self.path.abspath) - def test_path_realpath(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_path_realpath(self, kwargs): self.assertPathEqual(self.path.realpath) + self.assertPathEqual(partial(self.path.realpath, **kwargs)) + def test_path_relpath(self): self.assertPathEqual(self.path.relpath) diff --git a/Lib/test/test_pprint.py b/Lib/test/test_pprint.py index f68996f72b15b54..403d2e9008489e0 100644 --- a/Lib/test/test_pprint.py +++ b/Lib/test/test_pprint.py @@ -380,7 +380,7 @@ def __new__(cls, celsius_degrees): return super().__new__(Temperature, celsius_degrees) def __repr__(self): kelvin_degrees = self + 273.15 - return f"{kelvin_degrees}°K" + return f"{kelvin_degrees:.2f}°K" self.assertEqual(pprint.pformat(Temperature(1000)), '1273.15°K') def test_sorted_dict(self): diff --git a/Lib/test/test_property.py b/Lib/test/test_property.py index cea241b0f200d0f..26aefdbf0421dda 100644 --- a/Lib/test/test_property.py +++ b/Lib/test/test_property.py @@ -87,8 +87,8 @@ def test_property_decorator_baseclass(self): self.assertEqual(base.spam, 10) self.assertEqual(base._spam, 10) delattr(base, "spam") - self.assertTrue(not hasattr(base, "spam")) - self.assertTrue(not hasattr(base, "_spam")) + self.assertNotHasAttr(base, "spam") + self.assertNotHasAttr(base, "_spam") base.spam = 20 self.assertEqual(base.spam, 20) self.assertEqual(base._spam, 20) diff --git a/Lib/test/test_pty.py b/Lib/test/test_pty.py index c1728f5019d0426..b5407642f823e51 100644 --- a/Lib/test/test_pty.py +++ b/Lib/test/test_pty.py @@ -1,14 +1,13 @@ import unittest from test.support import ( - is_android, is_apple_mobile, is_emscripten, is_wasi, reap_children, verbose + is_android, is_apple_mobile, is_wasm32, reap_children, verbose ) from test.support.import_helper import import_module -from test.support.os_helper import TESTFN, unlink # Skip these tests if termios is not available import_module('termios') -if is_android or is_apple_mobile or is_emscripten or is_wasi: +if is_android or is_apple_mobile or is_wasm32: raise unittest.SkipTest("pty is not available on this platform") import errno @@ -297,26 +296,27 @@ def test_master_read(self): self.assertEqual(data, b"") def test_spawn_doesnt_hang(self): - self.addCleanup(unlink, TESTFN) - with open(TESTFN, 'wb') as f: - STDOUT_FILENO = 1 - dup_stdout = os.dup(STDOUT_FILENO) - os.dup2(f.fileno(), STDOUT_FILENO) - buf = b'' - def master_read(fd): - nonlocal buf - data = os.read(fd, 1024) - buf += data - return data + # gh-140482: Do the test in a pty.fork() child to avoid messing + # with the interactive test runner's terminal settings. + pid, fd = pty.fork() + if pid == pty.CHILD: + pty.spawn([sys.executable, '-c', 'print("hi there")']) + os._exit(0) + + try: + buf = bytearray() try: - pty.spawn([sys.executable, '-c', 'print("hi there")'], - master_read) - finally: - os.dup2(dup_stdout, STDOUT_FILENO) - os.close(dup_stdout) - self.assertEqual(buf, b'hi there\r\n') - with open(TESTFN, 'rb') as f: - self.assertEqual(f.read(), b'hi there\r\n') + while (data := os.read(fd, 1024)) != b'': + buf.extend(data) + except OSError as e: + if e.errno != errno.EIO: + raise + + (pid, status) = os.waitpid(pid, 0) + self.assertEqual(status, 0) + self.assertEqual(bytes(buf), b"hi there\r\n") + finally: + os.close(fd) class SmallPtyTests(unittest.TestCase): """These tests don't spawn children or hang.""" diff --git a/Lib/test/test_pulldom.py b/Lib/test/test_pulldom.py index 6dc51e4371d0f63..3c8ed251acaa4d8 100644 --- a/Lib/test/test_pulldom.py +++ b/Lib/test/test_pulldom.py @@ -46,7 +46,7 @@ def test_parse_semantics(self): items = pulldom.parseString(SMALL_SAMPLE) evt, node = next(items) # Just check the node is a Document: - self.assertTrue(hasattr(node, "createElement")) + self.assertHasAttr(node, "createElement") self.assertEqual(pulldom.START_DOCUMENT, evt) evt, node = next(items) self.assertEqual(pulldom.START_ELEMENT, evt) @@ -192,7 +192,7 @@ def _test_thorough(self, pd, before_root=True): evt, node = next(pd) self.assertEqual(pulldom.START_DOCUMENT, evt) # Just check the node is a Document: - self.assertTrue(hasattr(node, "createElement")) + self.assertHasAttr(node, "createElement") if before_root: evt, node = next(pd) diff --git a/Lib/test/test_pwd.py b/Lib/test/test_pwd.py index aa090b464a72222..bdf57776c82be13 100644 --- a/Lib/test/test_pwd.py +++ b/Lib/test/test_pwd.py @@ -1,3 +1,5 @@ +import random +import string import sys import unittest from test.support import import_helper @@ -56,59 +58,57 @@ def test_values_extended(self): def test_errors(self): self.assertRaises(TypeError, pwd.getpwuid) self.assertRaises(TypeError, pwd.getpwuid, 3.14) + self.assertRaises(TypeError, pwd.getpwuid, 0.0) + self.assertRaises(TypeError, pwd.getpwuid, 0, 0) + # should be out of uid_t range + self.assertRaises(KeyError, pwd.getpwuid, 2**128) + self.assertRaises(KeyError, pwd.getpwuid, -2**128) self.assertRaises(TypeError, pwd.getpwnam) self.assertRaises(TypeError, pwd.getpwnam, 42) - self.assertRaises(TypeError, pwd.getpwall, 42) + self.assertRaises(TypeError, pwd.getpwnam, b'root') + self.assertRaises(TypeError, pwd.getpwnam, 'root', 0) # embedded null character self.assertRaisesRegex(ValueError, 'null', pwd.getpwnam, 'a\x00b') + self.assertRaisesRegex(ValueError, 'null', pwd.getpwnam, 'root\x00') + self.assertRaises(UnicodeEncodeError, pwd.getpwnam, 'roo\udc74') + self.assertRaises(KeyError, pwd.getpwnam, '') + self.assertRaises(TypeError, pwd.getpwall, 42) - # try to get some errors - bynames = {} - byuids = {} - for (n, p, u, g, gecos, d, s) in pwd.getpwall(): - bynames[n] = u - byuids[u] = n - - allnames = list(bynames.keys()) - namei = 0 - fakename = allnames[namei] if allnames else "invaliduser" - while fakename in bynames: - chars = list(fakename) - for i in range(len(chars)): - if chars[i] == 'z': - chars[i] = 'A' - break - elif chars[i] == 'Z': - continue - else: - chars[i] = chr(ord(chars[i]) + 1) - break - else: - namei = namei + 1 - try: - fakename = allnames[namei] - except IndexError: - # should never happen... if so, just forget it - break - fakename = ''.join(chars) - - self.assertRaises(KeyError, pwd.getpwnam, fakename) - - # In some cases, byuids isn't a complete list of all users in the - # system, so if we try to pick a value not in byuids (via a perturbing - # loop, say), pwd.getpwuid() might still be able to find data for that - # uid. Using sys.maxint may provoke the same problems, but hopefully - # it will be a more repeatable failure. - fakeuid = sys.maxsize - self.assertNotIn(fakeuid, byuids) - self.assertRaises(KeyError, pwd.getpwuid, fakeuid) + # Find a non-existent user name. + # getpwall() will not necessarily report all existing users + # (typical for LDAP based directories in big organizations). + for _ in range(30): + fakename = ''.join(random.choices(string.ascii_lowercase, k=6)) + try: + pwd.getpwnam(fakename) + except KeyError: + break + else: + self.fail('Cannot find non-existent user name') + + # Find a non-existent uid. + maxuid = max(e.pw_uid for e in pwd.getpwall()) + if maxuid < 2**15: + maxuid = 2**15 + elif maxuid < 2**16: + maxuid = 2**16-1 + else: + maxuid = 2**31 + for _ in range(30): + fakeuid = random.randrange(maxuid) + try: + pwd.getpwuid(fakeuid) + except KeyError: + break + else: + self.fail('Cannot find non-existent uid') + + # On Cygwin, getpwuid(-1) returns 'Unknown+User' user + if sys.platform != 'cygwin': + # -1 shouldn't be a valid uid because it has a special meaning in many + # uid-related functions + self.assertRaises(KeyError, pwd.getpwuid, -1) - # -1 shouldn't be a valid uid because it has a special meaning in many - # uid-related functions - self.assertRaises(KeyError, pwd.getpwuid, -1) - # should be out of uid_t range - self.assertRaises(KeyError, pwd.getpwuid, 2**128) - self.assertRaises(KeyError, pwd.getpwuid, -2**128) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py index 64387296e846213..28141be31cb7340 100644 --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -158,21 +158,24 @@ def test_source_date_epoch(self): def test_double_dot_no_clobber(self): # http://bugs.python.org/issue22966 # py_compile foo.bar.py -> __pycache__/foo.cpython-34.pyc - weird_path = os.path.join(self.directory, 'foo.bar.py') - cache_path = importlib.util.cache_from_source(weird_path) - pyc_path = weird_path + 'c' - head, tail = os.path.split(cache_path) - penultimate_tail = os.path.basename(head) - self.assertEqual( - os.path.join(penultimate_tail, tail), - os.path.join( - '__pycache__', - 'foo.bar.{}.pyc'.format(sys.implementation.cache_tag))) - with open(weird_path, 'w') as file: - file.write('x = 123\n') - py_compile.compile(weird_path) - self.assertTrue(os.path.exists(cache_path)) - self.assertFalse(os.path.exists(pyc_path)) + # This test asserts the default __pycache__ layout, so neutralize any + # pycache prefix (e.g. when run with PYTHONPYCACHEPREFIX set). + with support.swap_attr(sys, 'pycache_prefix', None): + weird_path = os.path.join(self.directory, 'foo.bar.py') + cache_path = importlib.util.cache_from_source(weird_path) + pyc_path = weird_path + 'c' + head, tail = os.path.split(cache_path) + penultimate_tail = os.path.basename(head) + self.assertEqual( + os.path.join(penultimate_tail, tail), + os.path.join( + '__pycache__', + 'foo.bar.{}.pyc'.format(sys.implementation.cache_tag))) + with open(weird_path, 'w') as file: + file.write('x = 123\n') + py_compile.compile(weird_path) + self.assertTrue(os.path.exists(cache_path)) + self.assertFalse(os.path.exists(pyc_path)) def test_optimization_path(self): # Specifying optimized bytecode should lead to a path reflecting that. @@ -207,6 +210,14 @@ def test_quiet(self): with self.assertRaises(py_compile.PyCompileError): py_compile.compile(bad_coding, doraise=True, quiet=1) + def test_utf7_decoded_cr_compiles(self): + with open(self.source_path, 'wb') as file: + file.write(b"#coding=U7+AA0''\n") + + pyc_path = py_compile.compile(self.source_path, self.pyc_path, doraise=True) + self.assertEqual(pyc_path, self.pyc_path) + self.assertTrue(os.path.exists(self.pyc_path)) + class PyCompileTestsWithSourceEpoch(PyCompileTestsBase, unittest.TestCase, @@ -263,7 +274,13 @@ def test_with_files(self): self.assertEqual(rc, 0) self.assertEqual(stdout, b'') self.assertEqual(stderr, b'') - self.assertTrue(os.path.exists(self.cache_path)) + # pycompilecmd() runs the interpreter in isolated mode (-I), which + # ignores PYTHONPYCACHEPREFIX, so the bytecode is written next to the + # source. Compute the expected cache path the same way. + with support.swap_attr(sys, 'pycache_prefix', None): + cache_path = importlib.util.cache_from_source( + self.source_path, optimization='' if __debug__ else 1) + self.assertTrue(os.path.exists(cache_path)) def test_bad_syntax(self): bad_syntax = os.path.join(os.path.dirname(__file__), diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py index df05cd07d7e2493..3062f4eb9be5571 100644 --- a/Lib/test/test_pyclbr.py +++ b/Lib/test/test_pyclbr.py @@ -103,7 +103,7 @@ def ismethod(oclass, obj, name): for name, value in dict.items(): if name in ignore: continue - self.assertHasAttr(module, name, ignore) + self.assertHasAttr(module, name) py_item = getattr(module, name) if isinstance(value, pyclbr.Function): self.assertIsInstance(py_item, (FunctionType, BuiltinFunctionType)) @@ -254,7 +254,7 @@ def test_others(self): 'pdb', # pyclbr does not handle elegantly `typing` or properties ignore=('Union', '_ModuleTarget', '_ScriptTarget', '_ZipTarget', 'curframe_locals', - '_InteractState'), + '_InteractState', 'rlcompleter'), ) cm('pydoc', ignore=('input', 'output',)) # properties diff --git a/Lib/test/test_pydoc/pydocfodder.py b/Lib/test/test_pydoc/pydocfodder.py index 3cc2d5bd57fe5b3..412aa3743e430b3 100644 --- a/Lib/test/test_pydoc/pydocfodder.py +++ b/Lib/test/test_pydoc/pydocfodder.py @@ -87,6 +87,8 @@ def B_classmethod(cls, x): object_repr = object.__repr__ get = {}.get # same name dict_get = {}.get + from math import sin + B.B_classmethod_ref = B.B_classmethod @@ -186,3 +188,4 @@ def __call__(self, inst): object_repr = object.__repr__ get = {}.get # same name dict_get = {}.get +from math import sin # noqa: F401 diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index ac88b3c6f13d5ea..f7c1e72b1454e4c 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -11,7 +11,6 @@ import _pickle import pkgutil import re -import stat import tempfile import test.support import time @@ -33,7 +32,7 @@ assert_python_failure, spawn_python) from test.support import threading_helper from test.support import (reap_children, captured_stdout, - captured_stderr, is_emscripten, is_wasi, + captured_stderr, is_wasm32, requires_docstrings, MISSING_C_DOCSTRINGS) from test.support.os_helper import (TESTFN, rmtree, unlink) from test.test_pydoc import pydoc_mod @@ -474,6 +473,32 @@ def test_issue8225(self): result, doc_loc = get_pydoc_text(xml.etree) self.assertEqual(doc_loc, "", "MODULE DOCS incorrectly includes a link") + def test_online_docs_link(self): + import encodings.idna + import importlib._bootstrap + + module_docs = { + 'encodings': 'codecs#module-encodings', + 'encodings.idna': 'codecs#module-encodings.idna', + } + + with unittest.mock.patch('pydoc_data.module_docs.module_docs', module_docs): + doc = pydoc.TextDoc() + + basedir = os.path.dirname(encodings.__file__) + doc_link = doc.getdocloc(encodings, basedir=basedir) + self.assertIsNotNone(doc_link) + self.assertIn('codecs#module-encodings', doc_link) + self.assertNotIn('encodings.html', doc_link) + + doc_link = doc.getdocloc(encodings.idna, basedir=basedir) + self.assertIsNotNone(doc_link) + self.assertIn('codecs#module-encodings.idna', doc_link) + self.assertNotIn('encodings.idna.html', doc_link) + + doc_link = doc.getdocloc(importlib._bootstrap, basedir=basedir) + self.assertIsNone(doc_link) + def test_getpager_with_stdin_none(self): previous_stdin = sys.stdin try: @@ -1300,7 +1325,6 @@ def test_apropos_with_unreadable_dir(self): self.assertEqual(out.getvalue(), '') self.assertEqual(err.getvalue(), '') - @os_helper.skip_unless_working_chmod def test_apropos_empty_doc(self): pkgdir = os.path.join(TESTFN, 'walkpkg') os.mkdir(pkgdir) @@ -1308,14 +1332,9 @@ def test_apropos_empty_doc(self): init_path = os.path.join(pkgdir, '__init__.py') with open(init_path, 'w') as fobj: fobj.write("foo = 1") - current_mode = stat.S_IMODE(os.stat(pkgdir).st_mode) - try: - os.chmod(pkgdir, current_mode & ~stat.S_IEXEC) - with self.restrict_walk_packages(path=[TESTFN]), captured_stdout() as stdout: - pydoc.apropos('') - self.assertIn('walkpkg', stdout.getvalue()) - finally: - os.chmod(pkgdir, current_mode) + with self.restrict_walk_packages(path=[TESTFN]), captured_stdout() as stdout: + pydoc.apropos('') + self.assertIn('walkpkg', stdout.getvalue()) def test_url_search_package_error(self): # URL handler search should cope with packages that raise exceptions @@ -1380,7 +1399,7 @@ def test_modules_search_builtin(self): helper('modules garbage') result = help_io.getvalue() - self.assertTrue(result.startswith(expected)) + self.assertStartsWith(result, expected) def test_importfile(self): try: @@ -1946,9 +1965,11 @@ def test_text_doc_routines_in_class(self, cls=pydocfodder.B): if not support.MISSING_C_DOCSTRINGS: self.assertIn(' | get(key, default=None, /) method of builtins.dict instance', lines) self.assertIn(' | dict_get = get(key, default=None, /) method of builtins.dict instance', lines) + self.assertIn(' | sin(x, /)', lines) else: self.assertIn(' | get(...) method of builtins.dict instance', lines) self.assertIn(' | dict_get = get(...) method of builtins.dict instance', lines) + self.assertIn(' | sin(object, /)', lines) lines = self.getsection(result, f' | Class methods {where}:', ' | ' + '-'*70) self.assertIn(' | B_classmethod(x)', lines) @@ -2034,6 +2055,11 @@ def test_text_doc_routines_in_module(self): self.assertIn(' __repr__(...) unbound builtins.object method', lines) self.assertIn(' object_repr = __repr__(...) unbound builtins.object method', lines) + # builtin functions + if not support.MISSING_C_DOCSTRINGS: + self.assertIn(' sin(x, /)', lines) + else: + self.assertIn(' sin(object, /)', lines) def test_html_doc_routines_in_module(self): doc = pydoc.HTMLDoc() @@ -2074,9 +2100,15 @@ def test_html_doc_routines_in_module(self): self.assertIn(' __repr__(...) unbound builtins.object method', lines) self.assertIn(' object_repr = __repr__(...) unbound builtins.object method', lines) + # builtin functions + if not support.MISSING_C_DOCSTRINGS: + self.assertIn(' sin(x, /)', lines) + else: + self.assertIn(' sin(object, /)', lines) + @unittest.skipIf( - is_emscripten or is_wasi, + is_wasm32, "Socket server not available on Emscripten/WASI." ) class PydocServerTest(unittest.TestCase): @@ -2156,9 +2188,46 @@ def test_url_requests(self): class TestHelper(unittest.TestCase): + def mock_interactive_session(self, inputs): + """ + Given a list of inputs, run an interactive help session. Returns a string + of what would be shown on screen. + """ + input_iter = iter(inputs) + + def mock_getline(prompt): + output.write(prompt) + next_input = next(input_iter) + output.write(next_input + os.linesep) + return next_input + + with captured_stdout() as output: + helper = pydoc.Helper(output=output) + with unittest.mock.patch.object(helper, "getline", mock_getline): + helper.interact() + + # handle different line endings across platforms consistently + return output.getvalue().strip().splitlines(keepends=False) + def test_keywords(self): self.assertEqual(sorted(pydoc.Helper.keywords), - sorted(keyword.kwlist)) + sorted(keyword.kwlist + ['case', 'match'])) + + def test_interact_empty_line_continues(self): + # gh-138568: test pressing Enter without input should continue in help session + self.assertEqual( + self.mock_interactive_session(["", " ", "quit"]), + ["help> ", "help> ", "help> quit"], + ) + + def test_interact_quit_commands_exit(self): + quit_commands = ["quit", "q", "exit"] + for quit_cmd in quit_commands: + with self.subTest(quit_command=quit_cmd): + self.assertEqual( + self.mock_interactive_session([quit_cmd]), + [f"help> {quit_cmd}"], + ) class PydocWithMetaClasses(unittest.TestCase): diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 1d56ccd71cf962b..10e2e5f79908b0b 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -1,14 +1,18 @@ # XXX TypeErrors on calling handlers, or on bad return values from a # handler, are obscure and unhelpful. +import abc +import functools import os +import re import sys import sysconfig +import textwrap import unittest import traceback from io import BytesIO from test import support -from test.support import os_helper +from test.support import import_helper, os_helper from xml.parsers import expat from xml.parsers.expat import errors @@ -224,8 +228,7 @@ def _verify_parse_output(self, operations): "Character data: '\xb5'", "End element: 'root'", ] - for operation, expected_operation in zip(operations, expected_operations): - self.assertEqual(operation, expected_operation) + self.assertEqual(operations, expected_operations) def test_parse_bytes(self): out = self.Outputter() @@ -273,6 +276,119 @@ def test_parse_again(self): self.assertEqual(expat.ErrorString(cm.exception.code), expat.errors.XML_ERROR_FINISHED) + @support.subTests('encoding', [ + 'utf-8', 'utf-16', 'utf-16be', 'utf-16le', + 'iso8859-1', 'iso8859-2', 'iso8859-3', 'iso8859-4', 'iso8859-5', + 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', + 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', + 'cp437', 'cp720', 'cp737', 'cp775', 'cp850', 'cp852', + 'cp855', 'cp856', 'cp857', 'cp858', 'cp860', 'cp861', 'cp862', + 'cp863', 'cp865', 'cp866', 'cp869', 'cp874', 'cp1006', 'cp1125', + 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255', + 'cp1256', 'cp1257', 'cp1258', + 'mac-cyrillic', 'mac-greek', 'mac-iceland', 'mac-latin2', + 'mac-roman', 'mac-turkish', + 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'ptcp154', + ]) + def test_supported_encodings(self, encoding): + out = self.Outputter() + parser = expat.ParserCreate() + self._hookup_callbacks(parser, out) + c = 'éπя\u05d0\u060c€'.encode(encoding, 'ignore').decode(encoding)[0] + data = (f'<?xml version="1.0" encoding="{encoding}"?>\n' + f'<root>{c}</root>').encode(encoding) + parser.Parse(data, True) + self.assertEqual(out.out, [ + ('XML declaration', ('1.0', encoding, -1)), + "Start element: 'root' {}", + f'Character data: {c!r}', + "End element: 'root'", + ]) + + @support.subTests('encoding', [ + 'UTF-8', 'utf-8', 'utf-16', 'utf-16le', 'utf-16be', + 'koi8-u', 'cp1125', 'cp1251', 'iso8859-5', 'mac-cyrillic', + ]) + def test_supported_encodings2(self, encoding): + out = self.Outputter() + parser = expat.ParserCreate() + self._hookup_callbacks(parser, out) + data = (f'<?xml version="1.0" encoding="{encoding}"?>\n' + '<!-- коментар -->' + '<корінь атрибут="значення">зміст</корінь>').encode(encoding) + parser.Parse(data, True) + self.assertEqual(out.out, [ + ('XML declaration', ('1.0', encoding, -1)), + "Comment: ' коментар '", + "Start element: 'корінь' {'атрибут': 'значення'}", + "Character data: 'зміст'", + "End element: 'корінь'", + ]) + + @support.subTests('encoding', [ + 'UTF-7', + "Big5-HKSCS", "Big5", + "cp932", "cp949", "cp950", + "EUC_JIS-2004", "EUC_JISX0213", "EUC-JP", "EUC-KR", + "GB18030", "GB2312", "GBK", + "ISO-2022-KR", + "johab", + "Shift_JIS", "Shift_JIS-2004", "Shift_JISX0213", + ]) + def test_unsupported_encodings(self, encoding): + parser = expat.ParserCreate() + data = (f'<?xml version="1.0" encoding="{encoding}"?>\n' + '<root></root>').encode(encoding) + with self.assertRaises(ValueError): + parser.Parse(data, True) + + parser = expat.ParserCreate() + data = (f'<?xml version="1.0" encoding="{encoding}"?>\n' + '<root></root>').encode() + with self.assertRaises(ValueError): + parser.Parse(data, True) + + @support.subTests('encoding', [ + 'cp037', 'cp273', 'cp424', 'cp500', 'cp864', 'cp875', + 'cp1026', 'cp1140', + 'mac_arabic', 'mac_farsi', + ]) + def test_incompatible_encodings(self, encoding): + parser = expat.ParserCreate() + data = (f'<?xml version="1.0" encoding="{encoding}"?>\n' + '<root></root>').encode(encoding) + with self.assertRaises(expat.ExpatError): + parser.Parse(data, True) + + parser = expat.ParserCreate() + data = (f'<?xml version="1.0" encoding="{encoding}"?>\n' + '<root></root>').encode() + with self.assertRaisesRegex(expat.ExpatError, 'unknown encoding'): + parser.Parse(data, True) + + @support.subTests('encoding', [ + 'hex_codec', 'rot_13', + ]) + def test_non_text_encodings(self, encoding): + parser = expat.ParserCreate() + data = (f'<?xml version="1.0" encoding="{encoding}"?>\n' + '<root></root>').encode() + with self.assertRaises(LookupError): + parser.Parse(data, True) + + def test_undefined_encoding(self): + parser = expat.ParserCreate() + data = b'<?xml version="1.0" encoding="undefined"?>\n<root></root>' + with self.assertRaises(UnicodeError): + parser.Parse(data, True) + + def test_unknown_encoding(self): + parser = expat.ParserCreate() + data = b'<?xml version="1.0" encoding="xyz"?>\n<root></root>' + with self.assertRaises(LookupError): + parser.Parse(data, True) + + class NamespaceSeparatorTest(unittest.TestCase): def test_legal(self): # Tests that make sure we get errors when the namespace_separator value @@ -668,6 +784,56 @@ def test_change_size_2(self): parser.Parse(xml2, True) self.assertEqual(self.n, 4) + @support.requires_resource('cpu') + @support.requires_resource('walltime') + @support.bigmemtest(size=2**31, memuse=4, dry_run=False) + def test_large_character_data_does_not_crash(self, size): + # See https://github.com/python/cpython/issues/148441 + parser = expat.ParserCreate() + parser.buffer_text = True + parser.buffer_size = 2**31 - 1 # INT_MAX + N = 2049 * (1 << 20) - 3 # Character data greater than INT_MAX + self.assertGreater(N, parser.buffer_size) + parser.CharacterDataHandler = lambda text: None + xml_data = b"<r>" + b"A" * N + b"</r>" + self.assertEqual(parser.Parse(xml_data, True), 1) + +class ElementDeclHandlerTest(unittest.TestCase): + def test_trigger_leak(self): + # Unfixed, this test would leak the memory of the so-called + # "content model" in function ``my_ElementDeclHandler`` of pyexpat. + # See https://github.com/python/cpython/issues/140593. + data = textwrap.dedent('''\ + <!DOCTYPE quotations SYSTEM "quotations.dtd" [ + <!ELEMENT root ANY> + ]> + <root/> + ''').encode('UTF-8') + + parser = expat.ParserCreate() + parser.NotStandaloneHandler = lambda: 1.234 # arbitrary float + parser.ElementDeclHandler = lambda _1, _2: None + self.assertRaises(TypeError, parser.Parse, data, True) + + @support.skip_if_huge_c_stack(800_000) + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() + def test_deeply_nested_content_model(self): + # This should raise a RecursionError and not crash. + # See https://github.com/python/cpython/issues/145986. + N = 500_000 + data = ( + b'<!DOCTYPE root [\n<!ELEMENT root ' + + b'(a, ' * N + b'a' + b')' * N + + b'>\n]>\n<root/>\n' + ) + + parser = expat.ParserCreate() + parser.ElementDeclHandler = lambda _1, _2: None + with support.infinite_recursion(): + with self.assertRaises(RecursionError): + parser.Parse(data) + class MalformedInputTest(unittest.TestCase): def test1(self): xml = b"\0\r\n" @@ -755,6 +921,80 @@ def resolve_entity(context, base, system_id, public_id): self.assertEqual(handler_call_args, [("bar", "baz")]) +class ParentParserLifetimeTest(unittest.TestCase): + """ + Subparsers make use of their parent XML_Parser inside of Expat. + As a result, parent parsers need to outlive subparsers. + + See https://github.com/python/cpython/issues/139400. + """ + + def test_parent_parser_outlives_its_subparsers__single(self): + parser = expat.ParserCreate() + subparser = parser.ExternalEntityParserCreate(None) + + # Now try to cause garbage collection of the parent parser + # while it's still being referenced by a related subparser. + del parser + + def test_parent_parser_outlives_its_subparsers__multiple(self): + parser = expat.ParserCreate() + subparser_one = parser.ExternalEntityParserCreate(None) + subparser_two = parser.ExternalEntityParserCreate(None) + + # Now try to cause garbage collection of the parent parser + # while it's still being referenced by a related subparser. + del parser + + def test_parent_parser_outlives_its_subparsers__chain(self): + parser = expat.ParserCreate() + subparser = parser.ExternalEntityParserCreate(None) + subsubparser = subparser.ExternalEntityParserCreate(None) + + # Now try to cause garbage collection of the parent parsers + # while they are still being referenced by a related subparser. + del parser + del subparser + + +class ExternalEntityParserCreateErrorTest(unittest.TestCase): + """ExternalEntityParserCreate error paths should not crash or leak + refcounts on the parent parser. + + See https://github.com/python/cpython/issues/144984. + """ + + @classmethod + def setUpClass(cls): + cls.testcapi = import_helper.import_module('_testcapi') + + @support.nomemtest + def test_error_path_no_crash(self): + # When an allocation inside ExternalEntityParserCreate fails, + # the partially-initialized subparser is deallocated. This + # must not dereference NULL handlers or double-decrement the + # parent parser's refcount. + parser = expat.ParserCreate() + parser.buffer_text = True + rc_before = sys.getrefcount(parser) + + # We avoid self.assertRaises(MemoryError) here because the + # context manager itself needs memory allocations that fail + # while the nomemory hook is active. + self.testcapi.set_nomemory(1, 10) + raised = False + try: + parser.ExternalEntityParserCreate(None) + except MemoryError: + raised = True + finally: + self.testcapi.remove_mem_hooks() + self.assertTrue(raised, "MemoryError not raised") + + rc_after = sys.getrefcount(parser) + self.assertEqual(rc_after, rc_before) + + class ReparseDeferralTest(unittest.TestCase): def test_getter_setter_round_trip(self): parser = expat.ParserCreate() @@ -809,5 +1049,259 @@ def start_element(name, _): self.assertEqual(started, ['doc']) +class AttackProtectionTestBase(abc.ABC): + """ + Base class for testing protections against XML payloads with + disproportionate amplification. + + The protections being tested should detect and prevent attacks + that leverage disproportionate amplification from small inputs. + """ + + @staticmethod + def exponential_expansion_payload(*, nrows, ncols, text='.'): + """Create a billion laughs attack payload. + + Be careful: the number of total items is pow(n, k), thereby + requiring at least pow(ncols, nrows) * sizeof(text) memory! + """ + template = textwrap.dedent(f"""\ + <?xml version="1.0"?> + <!DOCTYPE doc [ + <!ENTITY row0 "{text}"> + <!ELEMENT doc (#PCDATA)> + {{body}} + ]> + <doc>&row{nrows};</doc> + """).rstrip() + + body = '\n'.join( + f'<!ENTITY row{i + 1} "{f"&row{i};" * ncols}">' + for i in range(nrows) + ) + body = textwrap.indent(body, ' ' * 4) + return template.format(body=body) + + def test_payload_generation(self): + # self-test for exponential_expansion_payload() + payload = self.exponential_expansion_payload(nrows=2, ncols=3) + self.assertEqual(payload, textwrap.dedent("""\ + <?xml version="1.0"?> + <!DOCTYPE doc [ + <!ENTITY row0 "."> + <!ELEMENT doc (#PCDATA)> + <!ENTITY row1 "&row0;&row0;&row0;"> + <!ENTITY row2 "&row1;&row1;&row1;"> + ]> + <doc>&row2;</doc> + """).rstrip()) + + def assert_root_parser_failure(self, func, /, *args, **kwargs): + """Check that func(*args, **kwargs) is invalid for a sub-parser.""" + msg = "parser must be a root parser" + self.assertRaisesRegex(expat.ExpatError, msg, func, *args, **kwargs) + + @abc.abstractmethod + def assert_rejected(self, func, /, *args, **kwargs): + """Assert that func(*args, **kwargs) triggers the attack protection. + + Note: this method must ensure that the attack protection being tested + is the one that is actually triggered at runtime, e.g., by matching + the exact error message. + """ + + @abc.abstractmethod + def set_activation_threshold(self, parser, threshold): + """Set the activation threshold for the tested protection.""" + + @abc.abstractmethod + def set_maximum_amplification(self, parser, max_factor): + """Set the maximum amplification factor for the tested protection.""" + + @abc.abstractmethod + def test_set_activation_threshold__threshold_reached(self): + """Test when the activation threshold is exceeded.""" + + @abc.abstractmethod + def test_set_activation_threshold__threshold_not_reached(self): + """Test when the activation threshold is not exceeded.""" + + def test_set_activation_threshold__invalid_threshold_type(self): + parser = expat.ParserCreate() + setter = functools.partial(self.set_activation_threshold, parser) + + self.assertRaises(TypeError, setter, 1.0) + self.assertRaises(TypeError, setter, -1.5) + self.assertRaises(ValueError, setter, -5) + + def test_set_activation_threshold__invalid_threshold_range(self): + _testcapi = import_helper.import_module("_testcapi") + parser = expat.ParserCreate() + setter = functools.partial(self.set_activation_threshold, parser) + + self.assertRaises(OverflowError, setter, _testcapi.ULLONG_MAX + 1) + + def test_set_activation_threshold__fail_for_subparser(self): + parser = expat.ParserCreate() + subparser = parser.ExternalEntityParserCreate(None) + setter = functools.partial(self.set_activation_threshold, subparser) + self.assert_root_parser_failure(setter, 12345) + + @abc.abstractmethod + def test_set_maximum_amplification__amplification_exceeded(self): + """Test when the amplification factor is exceeded.""" + + @abc.abstractmethod + def test_set_maximum_amplification__amplification_not_exceeded(self): + """Test when the amplification factor is not exceeded.""" + + def test_set_maximum_amplification__infinity(self): + inf = float('inf') # an 'inf' threshold is allowed by Expat + parser = expat.ParserCreate() + self.assertIsNone(self.set_maximum_amplification(parser, inf)) + + def test_set_maximum_amplification__invalid_max_factor_type(self): + parser = expat.ParserCreate() + setter = functools.partial(self.set_maximum_amplification, parser) + + self.assertRaises(TypeError, setter, None) + self.assertRaises(TypeError, setter, 'abc') + + def test_set_maximum_amplification__invalid_max_factor_range(self): + parser = expat.ParserCreate() + setter = functools.partial(self.set_maximum_amplification, parser) + + msg = re.escape("'max_factor' must be at least 1.0") + self.assertRaisesRegex(expat.ExpatError, msg, setter, float('nan')) + self.assertRaisesRegex(expat.ExpatError, msg, setter, 0.99) + + def test_set_maximum_amplification__fail_for_subparser(self): + parser = expat.ParserCreate() + subparser = parser.ExternalEntityParserCreate(None) + setter = functools.partial(self.set_maximum_amplification, subparser) + self.assert_root_parser_failure(setter, 123.45) + + +@unittest.skipIf(expat.version_info < (2, 4, 0), "requires Expat >= 2.4.0") +class ExpansionProtectionTest(AttackProtectionTestBase, unittest.TestCase): + + def assert_rejected(self, func, /, *args, **kwargs): + """Check that func(*args, **kwargs) hits the allocation limit.""" + msg = ( + r"limit on input amplification factor \(from DTD and entities\) " + r"breached: line \d+, column \d+" + ) + self.assertRaisesRegex(expat.ExpatError, msg, func, *args, **kwargs) + + def set_activation_threshold(self, parser, threshold): + return parser.SetBillionLaughsAttackProtectionActivationThreshold(threshold) + + def set_maximum_amplification(self, parser, max_factor): + return parser.SetBillionLaughsAttackProtectionMaximumAmplification(max_factor) + + def test_set_activation_threshold__threshold_reached(self): + parser = expat.ParserCreate() + # Choose a threshold expected to be always reached. + self.set_activation_threshold(parser, 3) + # Check that the threshold is reached by choosing a small factor + # and a payload whose peak amplification factor exceeds it. + self.assertIsNone(self.set_maximum_amplification(parser, 1.0)) + payload = self.exponential_expansion_payload(ncols=10, nrows=4) + self.assert_rejected(parser.Parse, payload, True) + + def test_set_activation_threshold__threshold_not_reached(self): + parser = expat.ParserCreate() + # Choose a threshold expected to be never reached. + self.set_activation_threshold(parser, pow(10, 5)) + # Check that the threshold is reached by choosing a small factor + # and a payload whose peak amplification factor exceeds it. + self.assertIsNone(self.set_maximum_amplification(parser, 1.0)) + payload = self.exponential_expansion_payload(ncols=10, nrows=4) + self.assertIsNotNone(parser.Parse(payload, True)) + + def test_set_maximum_amplification__amplification_exceeded(self): + parser = expat.ParserCreate() + # Unconditionally enable maximum activation factor. + self.set_activation_threshold(parser, 0) + # Choose a max amplification factor expected to always be exceeded. + self.assertIsNone(self.set_maximum_amplification(parser, 1.0)) + # Craft a payload for which the peak amplification factor is > 1.0. + payload = self.exponential_expansion_payload(ncols=1, nrows=2) + self.assert_rejected(parser.Parse, payload, True) + + def test_set_maximum_amplification__amplification_not_exceeded(self): + parser = expat.ParserCreate() + # Unconditionally enable maximum activation factor. + self.set_activation_threshold(parser, 0) + # Choose a max amplification factor expected to never be exceeded. + self.assertIsNone(self.set_maximum_amplification(parser, 1e4)) + # Craft a payload for which the peak amplification factor is < 1e4. + payload = self.exponential_expansion_payload(ncols=1, nrows=2) + self.assertIsNotNone(parser.Parse(payload, True)) + + +@unittest.skipIf(not hasattr(expat.XMLParserType, + "SetAllocTrackerMaximumAmplification"), + "requires Python compiled with Expat >= 2.7.2") +class MemoryProtectionTest(AttackProtectionTestBase, unittest.TestCase): + + # NOTE: with the default Expat configuration, the billion laughs protection + # may hit before the allocation limiter if exponential_expansion_payload() + # is not carefully parametrized. As such, the payloads should be chosen so + # that either the allocation limiter is hit before other protections are + # triggered or no protection at all is triggered. + + def assert_rejected(self, func, /, *args, **kwargs): + """Check that func(*args, **kwargs) hits the allocation limit.""" + msg = r"out of memory: line \d+, column \d+" + self.assertRaisesRegex(expat.ExpatError, msg, func, *args, **kwargs) + + def set_activation_threshold(self, parser, threshold): + return parser.SetAllocTrackerActivationThreshold(threshold) + + def set_maximum_amplification(self, parser, max_factor): + return parser.SetAllocTrackerMaximumAmplification(max_factor) + + def test_set_activation_threshold__threshold_reached(self): + parser = expat.ParserCreate() + # Choose a threshold expected to be always reached. + self.set_activation_threshold(parser, 3) + # Check that the threshold is reached by choosing a small factor + # and a payload whose peak amplification factor exceeds it. + self.assertIsNone(self.set_maximum_amplification(parser, 1.0)) + payload = self.exponential_expansion_payload(ncols=10, nrows=4) + self.assert_rejected(parser.Parse, payload, True) + + def test_set_activation_threshold__threshold_not_reached(self): + parser = expat.ParserCreate() + # Choose a threshold expected to be never reached. + self.set_activation_threshold(parser, pow(10, 5)) + # Check that the threshold is reached by choosing a small factor + # and a payload whose peak amplification factor exceeds it. + self.assertIsNone(self.set_maximum_amplification(parser, 1.0)) + payload = self.exponential_expansion_payload(ncols=10, nrows=4) + self.assertIsNotNone(parser.Parse(payload, True)) + + def test_set_maximum_amplification__amplification_exceeded(self): + parser = expat.ParserCreate() + # Unconditionally enable maximum activation factor. + self.set_activation_threshold(parser, 0) + # Choose a max amplification factor expected to always be exceeded. + self.assertIsNone(self.set_maximum_amplification(parser, 1.0)) + # Craft a payload for which the peak amplification factor is > 1.0. + payload = self.exponential_expansion_payload(ncols=1, nrows=2) + self.assert_rejected(parser.Parse, payload, True) + + def test_set_maximum_amplification__amplification_not_exceeded(self): + parser = expat.ParserCreate() + # Unconditionally enable maximum activation factor. + self.set_activation_threshold(parser, 0) + # Choose a max amplification factor expected to never be exceeded. + self.assertIsNone(self.set_maximum_amplification(parser, 1e4)) + # Craft a payload for which the peak amplification factor is < 1e4. + payload = self.exponential_expansion_payload(ncols=1, nrows=2) + self.assertIsNotNone(parser.Parse(payload, True)) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_pyrepl/__init__.py b/Lib/test/test_pyrepl/__init__.py index 8359d9844623c21..2f37bff6df8b4a4 100644 --- a/Lib/test/test_pyrepl/__init__.py +++ b/Lib/test/test_pyrepl/__init__.py @@ -1,14 +1,10 @@ import os import sys -from test.support import requires, load_package_tests -from test.support.import_helper import import_module +from test.support import import_helper, load_package_tests + if sys.platform != "win32": - # On non-Windows platforms, testing pyrepl currently requires that the - # 'curses' resource be given on the regrtest command line using the -u - # option. Additionally, we need to attempt to import curses and readline. - requires("curses") - curses = import_module("curses") + import_helper.import_module("termios") def load_tests(*args): diff --git a/Lib/test/test_pyrepl/support.py b/Lib/test/test_pyrepl/support.py index 4f7f9d77933336f..307bf4505550d61 100644 --- a/Lib/test/test_pyrepl/support.py +++ b/Lib/test/test_pyrepl/support.py @@ -88,6 +88,8 @@ def prepare_console(events: Iterable[Event], **kwargs) -> MagicMock | Console: console.get_event.side_effect = events console.height = 100 console.width = 80 + console.getheightwidth = MagicMock(side_effect=lambda: (console.height, console.width)) + for key, val in kwargs.items(): setattr(console, key, val) return console diff --git a/Lib/test/test_pyrepl/test_eventqueue.py b/Lib/test/test_pyrepl/test_eventqueue.py index edfe6ac4748f337..69d9612b70dc776 100644 --- a/Lib/test/test_pyrepl/test_eventqueue.py +++ b/Lib/test/test_pyrepl/test_eventqueue.py @@ -3,6 +3,8 @@ from unittest.mock import patch from test import support +from _pyrepl import terminfo + try: from _pyrepl.console import Event from _pyrepl import base_eventqueue @@ -172,17 +174,22 @@ def _push(keys): self.assertEqual(eq.get(), _event("key", "a")) +class EmptyTermInfo(terminfo.TermInfo): + def get(self, cap: str) -> bytes: + return b"" + + @unittest.skipIf(support.MS_WINDOWS, "No Unix event queue on Windows") class TestUnixEventQueue(EventQueueTestBase, unittest.TestCase): def setUp(self): - self.enterContext(patch("_pyrepl.curses.tigetstr", lambda x: b"")) self.file = tempfile.TemporaryFile() def tearDown(self) -> None: self.file.close() def make_eventqueue(self) -> base_eventqueue.BaseEventQueue: - return unix_eventqueue.EventQueue(self.file.fileno(), "utf-8") + ti = EmptyTermInfo("ansi") + return unix_eventqueue.EventQueue(self.file.fileno(), "utf-8", ti) @unittest.skipUnless(support.MS_WINDOWS, "No Windows event queue on Unix") diff --git a/Lib/test/test_pyrepl/test_interact.py b/Lib/test/test_pyrepl/test_interact.py index a20719033fc9b7b..f0837ee94e9beb4 100644 --- a/Lib/test/test_pyrepl/test_interact.py +++ b/Lib/test/test_pyrepl/test_interact.py @@ -1,7 +1,7 @@ import contextlib import io -import unittest import warnings +import unittest from unittest.mock import patch from textwrap import dedent @@ -293,7 +293,7 @@ def f(): """) with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("default") + warnings.simplefilter("always") console.runsource(code) count = sum("'return' in a 'finally' block" in str(w.message) diff --git a/Lib/test/test_pyrepl/test_pyrepl.py b/Lib/test/test_pyrepl/test_pyrepl.py index 93029ab6e080ba0..a10e908e513db90 100644 --- a/Lib/test/test_pyrepl/test_pyrepl.py +++ b/Lib/test/test_pyrepl/test_pyrepl.py @@ -1,17 +1,20 @@ +import importlib import io import itertools import os import pathlib +import pkgutil import re import rlcompleter import select import subprocess import sys import tempfile -from unittest import TestCase, skipUnless, skipIf -from unittest.mock import patch -from test.support import force_not_colorized, make_clean_env -from test.support import SHORT_TIMEOUT, STDLIB_DIR +from pkgutil import ModuleInfo +from unittest import TestCase, skipUnless, SkipTest +from unittest.mock import Mock, patch +from test.support import force_not_colorized, make_clean_env, Py_DEBUG +from test.support import has_subprocess_support, SHORT_TIMEOUT, STDLIB_DIR from test.support.import_helper import import_module from test.support.os_helper import EnvironmentVarGuard, unlink @@ -24,19 +27,35 @@ multiline_input, code_to_events, ) +from _pyrepl import terminfo from _pyrepl.console import Event -from _pyrepl._module_completer import ImportParser, ModuleCompleter -from _pyrepl.readline import (ReadlineAlikeReader, ReadlineConfig, - _ReadlineWrapper) +from _pyrepl._module_completer import ( + ImportParser, + ModuleCompleter, + HARDCODED_SUBMODULES, +) +from _pyrepl.readline import ( + ReadlineAlikeReader, + ReadlineConfig, + _ReadlineWrapper, +) from _pyrepl.readline import multiline_input as readline_multiline_input try: import pty except ImportError: pty = None +try: + import readline as readline_module +except ImportError: + readline_module = None class ReplTestCase(TestCase): + def setUp(self): + if not has_subprocess_support: + raise SkipTest("test module requires subprocess") + def run_repl( self, repl_input: str | list[str], @@ -46,6 +65,7 @@ def run_repl( cwd: str | None = None, skip: bool = False, timeout: float = SHORT_TIMEOUT, + exit_on_output: str | None = None, ) -> tuple[str, int]: temp_dir = None if cwd is None: @@ -59,6 +79,7 @@ def run_repl( cwd=cwd, skip=skip, timeout=timeout, + exit_on_output=exit_on_output, ) finally: if temp_dir is not None: @@ -73,6 +94,7 @@ def _run_repl( cwd: str, skip: bool, timeout: float, + exit_on_output: str | None, ) -> tuple[str, int]: assert pty master_fd, slave_fd = pty.openpty() @@ -118,6 +140,11 @@ def _run_repl( except OSError: break output.append(data) + if exit_on_output is not None: + output = ["".join(output)] + if exit_on_output in output[0]: + process.kill() + break else: os.close(master_fd) process.kill() @@ -452,6 +479,11 @@ def test_auto_indent_default(self): ) # fmt: on + events = code_to_events(input_code) + reader = self.prepare_reader(events) + output = multiline_input(reader) + self.assertEqual(output, output_code) + def test_auto_indent_continuation(self): # auto indenting according to previous user indentation # fmt: off @@ -912,7 +944,13 @@ def test_func(self): class TestPyReplModuleCompleter(TestCase): def setUp(self): + # Make iter_modules() search only the standard library. + # This makes the test more reliable in case there are + # other user packages/scripts on PYTHONPATH which can + # interfere with the completions. + lib_path = os.path.dirname(importlib.__path__[0]) self._saved_sys_path = sys.path + sys.path = [lib_path] def tearDown(self): sys.path = self._saved_sys_path @@ -920,19 +958,12 @@ def tearDown(self): def prepare_reader(self, events, namespace): console = FakeConsole(events) config = ReadlineConfig() + config.module_completer = ModuleCompleter(namespace) config.readline_completer = rlcompleter.Completer(namespace).complete reader = ReadlineAlikeReader(console=console, config=config) return reader def test_import_completions(self): - import importlib - # Make iter_modules() search only the standard library. - # This makes the test more reliable in case there are - # other user packages/scripts on PYTHONPATH which can - # intefere with the completions. - lib_path = os.path.dirname(importlib.__path__[0]) - sys.path = [lib_path] - cases = ( ("import path\t\n", "import pathlib"), ("import importlib.\t\tres\t\n", "import importlib.resources"), @@ -946,6 +977,7 @@ def test_import_completions(self): ("from importlib import mac\t\n", "from importlib import machinery"), ("from importlib import res\t\n", "from importlib import resources"), ("from importlib.res\t import a\t\n", "from importlib.resources import abc"), + ("from __phello__ import s\t\n", "from __phello__ import spam"), # frozen module ) for code, expected in cases: with self.subTest(code=code): @@ -954,10 +986,17 @@ def test_import_completions(self): output = reader.readline() self.assertEqual(output, expected) - def test_relative_import_completions(self): + @patch("pkgutil.iter_modules", lambda: [ModuleInfo(None, "public", True), + ModuleInfo(None, "_private", True)]) + @patch("sys.builtin_module_names", ()) + def test_private_completions(self): cases = ( - ("from .readl\t\n", "from .readline"), - ("from . import readl\t\n", "from . import readline"), + # Return public methods by default + ("import \t\n", "import public"), + ("from \t\n", "from public"), + # Return private methods if explicitly specified + ("import _\t\n", "import _private"), + ("from _\t\n", "from _private"), ) for code, expected in cases: with self.subTest(code=code): @@ -966,8 +1005,57 @@ def test_relative_import_completions(self): output = reader.readline() self.assertEqual(output, expected) - @patch("pkgutil.iter_modules", lambda: [(None, 'valid_name', None), - (None, 'invalid-name', None)]) + @patch( + "_pyrepl._module_completer.ModuleCompleter.iter_submodules", + lambda *_: [ + ModuleInfo(None, "public", True), + ModuleInfo(None, "_private", True), + ], + ) + def test_sub_module_private_completions(self): + cases = ( + # Return public methods by default + ("from foo import \t\n", "from foo import public"), + # Return private methods if explicitly specified + ("from foo import _\t\n", "from foo import _private"), + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + + def test_builtin_completion_top_level(self): + cases = ( + ("import bui\t\n", "import builtins"), + ("from bui\t\n", "from builtins"), + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + + def test_relative_import_completions(self): + cases = ( + (None, "from .readl\t\n", "from .readl"), + (None, "from . import readl\t\n", "from . import readl"), + ("_pyrepl", "from .readl\t\n", "from .readline"), + ("_pyrepl", "from . import readl\t\n", "from . import readline"), + ("_pyrepl", "from .. import toodeep\t\n", "from .. import toodeep"), + ("concurrent", "from .futures.i\t\n", "from .futures.interpreter"), + ) + for package, code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={"__package__": package}) + output = reader.readline() + self.assertEqual(output, expected) + + @patch("pkgutil.iter_modules", lambda: [ModuleInfo(None, "valid_name", True), + ModuleInfo(None, "invalid-name", True)]) def test_invalid_identifiers(self): # Make sure modules which are not valid identifiers # are not suggested as those cannot be imported via 'import'. @@ -983,6 +1071,147 @@ def test_invalid_identifiers(self): output = reader.readline() self.assertEqual(output, expected) + def test_no_fallback_on_regular_completion(self): + cases = ( + ("import pri\t\n", "import pri"), + ("from pri\t\n", "from pri"), + ("from typing import Na\t\n", "from typing import Na"), + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + + def test_global_cache(self): + with (tempfile.TemporaryDirectory() as _dir1, + patch.object(sys, "path", [_dir1, *sys.path])): + dir1 = pathlib.Path(_dir1) + (dir1 / "mod_aa.py").mkdir() + (dir1 / "mod_bb.py").mkdir() + events = code_to_events("import mod_a\t\nimport mod_b\t\n") + reader = self.prepare_reader(events, namespace={}) + output_1, output_2 = reader.readline(), reader.readline() + self.assertEqual(output_1, "import mod_aa") + self.assertEqual(output_2, "import mod_bb") + + def test_hardcoded_stdlib_submodules(self): + cases = ( + ("import collections.\t\n", "import collections.abc"), + ("from os import \t\n", "from os import path"), + ("import xml.parsers.expat.\t\te\t\n\n", "import xml.parsers.expat.errors"), + ("from xml.parsers.expat import \t\tm\t\n\n", "from xml.parsers.expat import model"), + ) + for code, expected in cases: + with self.subTest(code=code): + events = code_to_events(code) + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, expected) + + def test_hardcoded_stdlib_submodules_not_proposed_if_local_import(self): + with (tempfile.TemporaryDirectory() as _dir, + patch.object(sys, "modules", {})): # hide imported module + dir = pathlib.Path(_dir) + (dir / "collections").mkdir() + (dir / "collections" / "__init__.py").touch() + (dir / "collections" / "foo.py").touch() + with patch.object(sys, "path", [_dir, *sys.path]): + events = code_to_events("import collections.\t\n") + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, "import collections.foo") + + def test_already_imported_stdlib_module_no_other_suggestions(self): + with (tempfile.TemporaryDirectory() as _dir, + patch.object(sys, "path", [_dir, *sys.path])): + dir = pathlib.Path(_dir) + (dir / "collections").mkdir() + (dir / "collections" / "__init__.py").touch() + (dir / "collections" / "foo.py").touch() + + # collections found in dir, but was already imported + # from stdlib at startup -> suggest stdlib submodules only + events = code_to_events("import collections.\t\n") + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, "import collections.abc") + + def test_already_imported_custom_module_no_suggestions(self): + with (tempfile.TemporaryDirectory() as _dir1, + tempfile.TemporaryDirectory() as _dir2, + patch.object(sys, "path", [_dir2, _dir1, *sys.path])): + dir1 = pathlib.Path(_dir1) + (dir1 / "mymodule").mkdir() + (dir1 / "mymodule" / "__init__.py").touch() + (dir1 / "mymodule" / "foo.py").touch() + importlib.import_module("mymodule") + + dir2 = pathlib.Path(_dir2) + (dir2 / "mymodule").mkdir() + (dir2 / "mymodule" / "__init__.py").touch() + (dir2 / "mymodule" / "bar.py").touch() + # Purge FileFinder cache after adding files + pkgutil.get_importer(_dir2).invalidate_caches() + # mymodule found in dir2 before dir1, but it was already imported + # from dir1 -> do not suggest dir2 submodules + events = code_to_events("import mymodule.\t\n") + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, "import mymodule.") + + del sys.modules["mymodule"] + # mymodule not imported anymore -> suggest dir2 submodules + events = code_to_events("import mymodule.\t\n") + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, "import mymodule.bar") + + def test_already_imported_custom_file_no_suggestions(self): + # Same as before, but mymodule from dir1 has no submodules + # -> propose nothing + with (tempfile.TemporaryDirectory() as _dir1, + tempfile.TemporaryDirectory() as _dir2, + patch.object(sys, "path", [_dir2, _dir1, *sys.path])): + dir1 = pathlib.Path(_dir1) + (dir1 / "mymodule").mkdir() + (dir1 / "mymodule.py").touch() + importlib.import_module("mymodule") + + dir2 = pathlib.Path(_dir2) + (dir2 / "mymodule").mkdir() + (dir2 / "mymodule" / "__init__.py").touch() + (dir2 / "mymodule" / "bar.py").touch() + events = code_to_events("import mymodule.\t\n") + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, "import mymodule.") + del sys.modules["mymodule"] + + def test_already_imported_module_without_origin_or_spec(self): + with (tempfile.TemporaryDirectory() as _dir1, + patch.object(sys, "path", [_dir1, *sys.path])): + dir1 = pathlib.Path(_dir1) + for mod in ("no_origin", "not_has_location", "no_spec"): + (dir1 / mod).mkdir() + (dir1 / mod / "__init__.py").touch() + (dir1 / mod / "foo.py").touch() + pkgutil.get_importer(_dir1).invalidate_caches() + module = importlib.import_module(mod) + assert module.__spec__ + if mod == "no_origin": + module.__spec__.origin = None + elif mod == "not_has_location": + module.__spec__.has_location = False + else: + module.__spec__ = None + events = code_to_events(f"import {mod}.\t\n") + reader = self.prepare_reader(events, namespace={}) + output = reader.readline() + self.assertEqual(output, f"import {mod}.") + del sys.modules[mod] + def test_get_path_and_prefix(self): cases = ( ('', ('', '')), @@ -1050,11 +1279,15 @@ def test_parse(self): self.assertEqual(actual, parsed) # The parser should not get tripped up by any # other preceding statements - code = f'import xyz\n{code}' - with self.subTest(code=code): + _code = f'import xyz\n{code}' + parser = ImportParser(_code) + actual = parser.parse() + with self.subTest(code=_code): self.assertEqual(actual, parsed) - code = f'import xyz;{code}' - with self.subTest(code=code): + _code = f'import xyz;{code}' + parser = ImportParser(_code) + actual = parser.parse() + with self.subTest(code=_code): self.assertEqual(actual, parsed) def test_parse_error(self): @@ -1081,6 +1314,7 @@ def test_parse_error(self): 'import ..foo', 'import .foo.bar', 'import foo; x = 1', + 'import foo; 1,', 'import a.; x = 1', 'import a.b; x = 1', 'import a.b.; x = 1', @@ -1100,6 +1334,8 @@ def test_parse_error(self): 'from foo import import', 'from foo import from', 'from foo import as', + 'from \\x', # _tokenize SyntaxError -> tokenize TokenError + 'if 1:\n pass\n\tpass', # _tokenize TabError -> tokenize TabError ) for code in cases: parser = ImportParser(code) @@ -1107,6 +1343,19 @@ def test_parse_error(self): with self.subTest(code=code): self.assertEqual(actual, None) + +class TestHardcodedSubmodules(TestCase): + def test_hardcoded_stdlib_submodules_are_importable(self): + for parent_path, submodules in HARDCODED_SUBMODULES.items(): + for module_name in submodules: + path = f"{parent_path}.{module_name}" + with self.subTest(path=path): + # We can't use importlib.util.find_spec here, + # since some hardcoded submodules parents are + # not proper packages + importlib.import_module(path) + + class TestPasteEvent(TestCase): def prepare_reader(self, events): console = FakeConsole(events) @@ -1271,6 +1520,9 @@ class TestDumbTerminal(ReplTestCase): def test_dumb_terminal_exits_cleanly(self): env = os.environ.copy() env.pop('PYTHON_BASIC_REPL', None) + # Ignore PYTHONSTARTUP to not pollute the output + # with an unrelated traceback. See GH-137568. + env.pop('PYTHONSTARTUP', None) env.update({"TERM": "dumb"}) output, exit_code = self.run_repl("exit()\n", env=env) self.assertEqual(exit_code, 0) @@ -1279,13 +1531,26 @@ def test_dumb_terminal_exits_cleanly(self): self.assertNotIn("Traceback", output) +def supports_pyrepl(): + # pyrepl falls back to the basic REPL if the terminal lacks any of the + # capabilities which UnixConsole requires. This covers an unset or + # "dumb" TERM as well, they are resolved to the "dumb" capabilities. + try: + info = terminfo.TermInfo(None) + except Exception: + return False + return all(info.get(cap) is not None + for cap in ("bel", "clear", "cup", "el")) + + @skipUnless(pty, "requires pty") -@skipIf((os.environ.get("TERM") or "dumb") == "dumb", "can't use pyrepl in dumb terminal") +@skipUnless(supports_pyrepl(), "can't use pyrepl in this terminal") class TestMain(ReplTestCase): def setUp(self): # Cleanup from PYTHON* variables to isolate from local # user settings, see #121359. Such variables should be # added later in test methods to patched os.environ. + super().setUp() patcher = patch('os.environ', new=make_clean_env()) self.addCleanup(patcher.stop) patcher.start() @@ -1327,7 +1592,7 @@ def _assertMatchOK( ) @force_not_colorized - def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False): + def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False, pythonstartup=False): clean_env = make_clean_env() clean_env["NO_COLOR"] = "1" # force_not_colorized doesn't touch subprocesses @@ -1336,9 +1601,13 @@ def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False blue.mkdir() mod = blue / "calx.py" mod.write_text("FOO = 42", encoding="utf-8") + startup = blue / "startup.py" + startup.write_text("BAR = 64", encoding="utf-8") commands = [ "print(f'^{" + var + "=}')" for var in expectations ] + ["exit()"] + if pythonstartup: + clean_env["PYTHONSTARTUP"] = str(startup) if as_file and as_module: self.fail("as_file and as_module are mutually exclusive") elif as_file: @@ -1357,7 +1626,13 @@ def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False skip=True, ) else: - self.fail("Choose one of as_file or as_module") + output, exit_code = self.run_repl( + commands, + cmdline_args=[], + env=clean_env, + cwd=td, + skip=True, + ) self.assertEqual(exit_code, 0) for var, expected in expectations.items(): @@ -1370,6 +1645,23 @@ def _run_repl_globals_test(self, expectations, *, as_file=False, as_module=False self.assertNotIn("Exception", output) self.assertNotIn("Traceback", output) + def test_globals_initialized_as_default(self): + expectations = { + "__name__": "'__main__'", + "__package__": "None", + # "__file__" is missing in -i, like in the basic REPL + } + self._run_repl_globals_test(expectations) + + def test_globals_initialized_from_pythonstartup(self): + expectations = { + "BAR": "64", + "__name__": "'__main__'", + "__package__": "None", + # "__file__" is missing in -i, like in the basic REPL + } + self._run_repl_globals_test(expectations, pythonstartup=True) + def test_inspect_keeps_globals_from_inspected_file(self): expectations = { "FOO": "42", @@ -1379,6 +1671,16 @@ def test_inspect_keeps_globals_from_inspected_file(self): } self._run_repl_globals_test(expectations, as_file=True) + def test_inspect_keeps_globals_from_inspected_file_with_pythonstartup(self): + expectations = { + "FOO": "42", + "BAR": "64", + "__name__": "'__main__'", + "__package__": "None", + # "__file__" is missing in -i, like in the basic REPL + } + self._run_repl_globals_test(expectations, as_file=True, pythonstartup=True) + def test_inspect_keeps_globals_from_inspected_module(self): expectations = { "FOO": "42", @@ -1388,26 +1690,32 @@ def test_inspect_keeps_globals_from_inspected_module(self): } self._run_repl_globals_test(expectations, as_module=True) + def test_inspect_keeps_globals_from_inspected_module_with_pythonstartup(self): + expectations = { + "FOO": "42", + "BAR": "64", + "__name__": "'__main__'", + "__package__": "'blue'", + "__file__": re.compile(r"^'.*calx.py'$"), + } + self._run_repl_globals_test(expectations, as_module=True, pythonstartup=True) + @force_not_colorized def test_python_basic_repl(self): env = os.environ.copy() - commands = ("from test.support import initialized_with_pyrepl\n" - "initialized_with_pyrepl()\n" - "exit()\n") - + pyrepl_commands = "clear\nexit()\n" env.pop("PYTHON_BASIC_REPL", None) - output, exit_code = self.run_repl(commands, env=env, skip=True) + output, exit_code = self.run_repl(pyrepl_commands, env=env, skip=True) self.assertEqual(exit_code, 0) - self.assertIn("True", output) - self.assertNotIn("False", output) self.assertNotIn("Exception", output) + self.assertNotIn("NameError", output) self.assertNotIn("Traceback", output) + basic_commands = "help\nexit()\n" env["PYTHON_BASIC_REPL"] = "1" - output, exit_code = self.run_repl(commands, env=env) + output, exit_code = self.run_repl(basic_commands, env=env) self.assertEqual(exit_code, 0) - self.assertIn("False", output) - self.assertNotIn("True", output) + self.assertIn("Type help() for interactive help", output) self.assertNotIn("Exception", output) self.assertNotIn("Traceback", output) @@ -1544,6 +1852,17 @@ def test_null_byte(self): self.assertEqual(exit_code, 0) self.assertNotIn("TypeError", output) + @force_not_colorized + def test_non_string_suggestion_candidates(self): + commands = ("import runpy\n" + "runpy._run_module_code('blech', {0: '', 'bluch': ''}, '')\n" + "exit()\n") + + output, exit_code = self.run_repl(commands) + self.assertEqual(exit_code, 0) + self.assertNotIn("all elements in 'candidates' must be strings", output) + self.assertIn("bluch", output) + def test_readline_history_file(self): # skip, if readline module is not available readline = import_module('readline') @@ -1574,18 +1893,24 @@ def test_history_survive_crash(self): commands = "1\n2\n3\nexit()\n" output, exit_code = self.run_repl(commands, env=env, skip=True) + self.assertEqual(exit_code, 0) - commands = "spam\nimport time\ntime.sleep(1000)\nquit\n" - try: - self.run_repl(commands, env=env, timeout=3) - except AssertionError: - pass + # Run until "0xcafe" is printed (as "51966") and then kill the + # process to simulate a crash. Note that the output also includes + # the echoed input commands. + commands = "spam\nimport time\n0xcafe\ntime.sleep(1000)\nquit\n" + output, exit_code = self.run_repl(commands, env=env, + exit_on_output="51966") + self.assertNotEqual(exit_code, 0) history = pathlib.Path(hfile.name).read_text() self.assertIn("2", history) self.assertIn("exit()", history) self.assertIn("spam", history) self.assertIn("import time", history) + # History is written after each command's output is printed to the + # console, so depending on how quickly the process is killed, + # the last command may or may not be written to the history file. self.assertNotIn("sleep", history) self.assertNotIn("quit", history) @@ -1605,3 +1930,232 @@ def test_prompt_after_help(self): # Extra stuff (newline and `exit` rewrites) are necessary # because of how run_repl works. self.assertNotIn(">>> \n>>> >>>", cleaned_output) + + @skipUnless(Py_DEBUG, '-X showrefcount requires a Python debug build') + def test_showrefcount(self): + env = os.environ.copy() + env.pop("PYTHON_BASIC_REPL", "") + output, _ = self.run_repl("1\n1+2\nexit()\n", cmdline_args=['-Xshowrefcount'], env=env) + matches = re.findall(r'\[-?\d+ refs, \d+ blocks\]', output) + self.assertEqual(len(matches), 3) + + env["PYTHON_BASIC_REPL"] = "1" + output, _ = self.run_repl("1\n1+2\nexit()\n", cmdline_args=['-Xshowrefcount'], env=env) + matches = re.findall(r'\[-?\d+ refs, \d+ blocks\]', output) + self.assertEqual(len(matches), 3) + + + @force_not_colorized + def test_no_newline(self): + env = os.environ.copy() + env.pop("PYTHON_BASIC_REPL", "") + env["PYTHON_BASIC_REPL"] = "1" + + commands = "print('Something pretty long', end='')\nexit()\n" + expected_output_sequence = "Something pretty long>>> exit()" + + # gh-143394: The basic REPL needs the readline module to turn off + # ECHO terminal attribute. + if readline_module is not None: + basic_output, basic_exit_code = self.run_repl(commands, env=env) + self.assertEqual(basic_exit_code, 0) + self.assertIn(expected_output_sequence, basic_output) + + output, exit_code = self.run_repl(commands) + self.assertEqual(exit_code, 0) + + # Build patterns for escape sequences that don't affect cursor position + # or visual output. Use terminfo to get platform-specific sequences, + # falling back to hard-coded patterns for capabilities not in terminfo. + from _pyrepl.terminfo import TermInfo + ti = TermInfo(os.environ.get("TERM", "")) + + safe_patterns = [] + + # smkx/rmkx - application cursor keys and keypad mode + smkx = ti.get("smkx") + rmkx = ti.get("rmkx") + if smkx: + safe_patterns.append(re.escape(smkx.decode("ascii"))) + if rmkx: + safe_patterns.append(re.escape(rmkx.decode("ascii"))) + if not smkx and not rmkx: + safe_patterns.append(r'\x1b\[\?1[hl]') # application cursor keys + safe_patterns.append(r'\x1b[=>]') # application keypad mode + + # ich1 - insert character (only safe form that inserts exactly 1 char) + ich1 = ti.get("ich1") + if ich1: + safe_patterns.append(re.escape(ich1.decode("ascii")) + r'(?=[ -~])') + else: + safe_patterns.append(r'\x1b\[(?:1)?@(?=[ -~])') + + # civis/cnorm - cursor visibility (may include cursor blinking control) + civis = ti.get("civis") + cnorm = ti.get("cnorm") + if civis: + safe_patterns.append(re.escape(civis.decode("ascii"))) + if cnorm: + safe_patterns.append(re.escape(cnorm.decode("ascii"))) + if not civis and not cnorm: + safe_patterns.append(r'\x1b\[\?25[hl]') # cursor visibility + safe_patterns.append(r'\x1b\[\?12[hl]') # cursor blinking + + # rmam / smam - automatic margins + rmam = ti.get("rmam") + smam = ti.get("smam") + if rmam: + safe_patterns.append(re.escape(rmam.decode("ascii"))) + if smam: + safe_patterns.append(re.escape(smam.decode("ascii"))) + if not rmam and not smam: + safe_patterns.append(r'\x1b\[\?7l') # turn off automatic margins + safe_patterns.append(r'\x1b\[\?7h') # turn on automatic margins + + # Modern extensions not in standard terminfo - always use patterns + safe_patterns.append(r'\x1b\[\?2004[hl]') # bracketed paste mode + safe_patterns.append(r'\x1b\[\?12[hl]') # cursor blinking (may be separate) + safe_patterns.append(r'\x1b\[\?[01]c') # device attributes + + safe_escapes = re.compile('|'.join(safe_patterns)) + cleaned_output = safe_escapes.sub('', output) + self.assertIn(expected_output_sequence, cleaned_output) + + +@skipUnless(sys.platform == "darwin", "macOS only") +class TestMainAppleTerminal(TestMain): + """Test the REPL with Apple Terminal's TERM_PROGRAM set.""" + + def run_repl(self, repl_input, env=None, **kwargs): + if env is None: + env = os.environ.copy() + env["TERM_PROGRAM"] = "Apple_Terminal" + return super().run_repl(repl_input, env=env, **kwargs) + + +class TestPyReplCtrlD(TestCase): + """Test Ctrl+D behavior in _pyrepl to match old pre-3.13 REPL behavior. + + Ctrl+D should: + - Exit on empty buffer (raises EOFError) + - Delete character when cursor is in middle of line + - Perform no operation when cursor is at end of line without newline + - Exit multiline mode when cursor is at end with trailing newline + - Run code up to that point when pressed on blank line with preceding lines + """ + def prepare_reader(self, events): + console = FakeConsole(events) + config = ReadlineConfig(readline_completer=None) + reader = ReadlineAlikeReader(console=console, config=config) + return reader + + def test_ctrl_d_empty_line(self): + """Test that pressing Ctrl+D on empty line exits the program""" + events = [ + Event(evt="key", data="\x04", raw=bytearray(b"\x04")), # Ctrl+D + ] + reader = self.prepare_reader(events) + with self.assertRaises(EOFError): + multiline_input(reader) + + def test_ctrl_d_multiline_with_new_line(self): + """Test that pressing Ctrl+D in multiline mode with trailing newline exits multiline mode""" + events = itertools.chain( + code_to_events("def f():\n pass\n"), # Enter multiline mode with trailing newline + [ + Event(evt="key", data="\x04", raw=bytearray(b"\x04")), # Ctrl+D + ], + ) + reader, _ = handle_all_events(events) + self.assertTrue(reader.finished) + self.assertEqual("def f():\n pass\n", "".join(reader.buffer)) + + def test_ctrl_d_multiline_middle_of_line(self): + """Test that pressing Ctrl+D in multiline mode with cursor in middle deletes character""" + events = itertools.chain( + code_to_events("def f():\n hello world"), # Enter multiline mode + [ + Event(evt="key", data="left", raw=bytearray(b"\x1bOD")) + ] * 5, # move cursor to 'w' in "world" + [ + Event(evt="key", data="\x04", raw=bytearray(b"\x04")) + ], # Ctrl+D should delete 'w' + ) + reader, _ = handle_all_events(events) + self.assertFalse(reader.finished) + self.assertEqual("def f():\n hello orld", "".join(reader.buffer)) + + def test_ctrl_d_multiline_end_of_line_no_newline(self): + """Test that pressing Ctrl+D at end of line without newline performs no operation""" + events = itertools.chain( + code_to_events("def f():\n hello"), # Enter multiline mode, no trailing newline + [ + Event(evt="key", data="\x04", raw=bytearray(b"\x04")) + ], # Ctrl+D should be no-op + ) + reader, _ = handle_all_events(events) + self.assertFalse(reader.finished) + self.assertEqual("def f():\n hello", "".join(reader.buffer)) + + def test_ctrl_d_single_line_middle_of_line(self): + """Test that pressing Ctrl+D in single line mode deletes current character""" + events = itertools.chain( + code_to_events("hello"), + [Event(evt="key", data="left", raw=bytearray(b"\x1bOD"))], # move left + [Event(evt="key", data="\x04", raw=bytearray(b"\x04"))], # Ctrl+D + ) + reader, _ = handle_all_events(events) + self.assertEqual("hell", "".join(reader.buffer)) + + def test_ctrl_d_single_line_end_no_newline(self): + """Test that pressing Ctrl+D at end of single line without newline does nothing""" + events = itertools.chain( + code_to_events("hello"), # cursor at end of line + [Event(evt="key", data="\x04", raw=bytearray(b"\x04"))], # Ctrl+D + ) + reader, _ = handle_all_events(events) + self.assertEqual("hello", "".join(reader.buffer)) + + +@skipUnless(sys.platform == "win32", "windows console only") +class TestWindowsConsoleEolWrap(TestCase): + def _make_mock_console(self, width=80): + from _pyrepl import windows_console as wc + + console = object.__new__(wc.WindowsConsole) + + console.width = width + console.posxy = (0, 0) + console.screen = [""] + + console._hide_cursor = Mock() + console._show_cursor = Mock() + console._erase_to_end = Mock() + console._move_relative = Mock() + console.move_cursor = Mock() + console._WindowsConsole__write = Mock() + + return console, wc + + def test_short_line_sets_posxy_normally(self): + width = 10 + y = 3 + console, wc = self._make_mock_console(width=width) + old_line = "" + new_line = "a" * 3 + wc.WindowsConsole._WindowsConsole__write_changed_line( + console, y, old_line, new_line, 0 + ) + self.assertEqual(console.posxy, (3, y)) + + def test_exact_width_line_does_not_wrap(self): + width = 10 + y = 3 + console, wc = self._make_mock_console(width=width) + old_line = "" + new_line = "a" * width + + wc.WindowsConsole._WindowsConsole__write_changed_line( + console, y, old_line, new_line, 0 + ) + self.assertEqual(console.posxy, (width - 1, y)) diff --git a/Lib/test/test_pyrepl/test_reader.py b/Lib/test/test_pyrepl/test_reader.py index 4ee320a5a4dabb6..fbf557115f8a254 100644 --- a/Lib/test/test_pyrepl/test_reader.py +++ b/Lib/test/test_pyrepl/test_reader.py @@ -228,6 +228,7 @@ def _prepare_console(events): console.get_event.side_effect = events console.height = 100 console.width = 80 + console.getheightwidth = MagicMock(side_effect=lambda: (console.height, console.width)) console.input_hook = input_hook return console @@ -375,8 +376,10 @@ def funct(case: str = sys.platform) -> None: ) match case: case "emscripten": print("on the web") - case "ios" | "android": print("on the phone") + case "ios" | "android": + print("on the phone") case _: print('arms around', match.group(1)) + type type = type[type] """ ) expected = dedent( @@ -393,8 +396,10 @@ def funct(case: str = sys.platform) -> None: {o}){z} {K}match{z} case{o}:{z} {K}case{z} {s}"emscripten"{z}{o}:{z} {b}print{z}{o}({z}{s}"on the web"{z}{o}){z} - {K}case{z} {s}"ios"{z} {o}|{z} {s}"android"{z}{o}:{z} {b}print{z}{o}({z}{s}"on the phone"{z}{o}){z} + {K}case{z} {s}"ios"{z} {o}|{z} {s}"android"{z}{o}:{z} + {b}print{z}{o}({z}{s}"on the phone"{z}{o}){z} {K}case{z} {K}_{z}{o}:{z} {b}print{z}{o}({z}{s}'arms around'{z}{o},{z} match{o}.{z}group{o}({z}{n}1{z}{o}){z}{o}){z} + {K}type{z} {b}type{z} {o}={z} {b}type{z}{o}[{z}{b}type{z}{o}]{z} """ ) expected_sync = expected.format(a="", **colors) @@ -402,14 +407,14 @@ def funct(case: str = sys.platform) -> None: reader, _ = handle_all_events(events) self.assert_screen_equal(reader, code, clean=True) self.assert_screen_equal(reader, expected_sync) - self.assertEqual(reader.pos, 2**7 + 2**8) - self.assertEqual(reader.cxy, (0, 14)) + self.assertEqual(reader.pos, 419) + self.assertEqual(reader.cxy, (0, 16)) async_msg = "{k}async{z} ".format(**colors) expected_async = expected.format(a=async_msg, **colors) more_events = itertools.chain( code_to_events(code), - [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))] * 13, + [Event(evt="key", data="up", raw=bytearray(b"\x1bOA"))] * 15, code_to_events("async "), ) reader, _ = handle_all_events(more_events) @@ -497,6 +502,57 @@ def unfinished_function(): self.assert_screen_equal(reader, code, clean=True) self.assert_screen_equal(reader, expected) + def test_syntax_highlighting_indentation_error(self): + code = dedent( + """\ + def unfinished_function(): + var = 1 + oops + """ + ) + expected = dedent( + """\ + {k}def{z} {d}unfinished_function{z}{o}({z}{o}){z}{o}:{z} + var {o}={z} {n}1{z} + oops + """ + ).format(**colors) + events = code_to_events(code) + reader, _ = handle_all_events(events) + self.assert_screen_equal(reader, code, clean=True) + self.assert_screen_equal(reader, expected) + + def test_syntax_highlighting_literal_brace_in_fstring_or_tstring(self): + code = dedent( + """\ + f"{{" + f"}}" + f"a{{b" + f"a}}b" + f"a{{b}}c" + t"a{{b}}c" + f"{{{0}}}" + f"{ {0} }" + """ + ) + expected = dedent( + """\ + {s}f"{z}{s}<<{z}{s}"{z} + {s}f"{z}{s}>>{z}{s}"{z} + {s}f"{z}{s}a<<{z}{s}b{z}{s}"{z} + {s}f"{z}{s}a>>{z}{s}b{z}{s}"{z} + {s}f"{z}{s}a<<{z}{s}b>>{z}{s}c{z}{s}"{z} + {s}t"{z}{s}a<<{z}{s}b>>{z}{s}c{z}{s}"{z} + {s}f"{z}{s}<<{z}{o}<{z}{n}0{z}{o}>{z}{s}>>{z}{s}"{z} + {s}f"{z}{o}<{z} {o}<{z}{n}0{z}{o}>{z} {o}>{z}{s}"{z} + """ + ).format(**colors).replace("<", "{").replace(">", "}") + events = code_to_events(code) + reader, _ = handle_all_events(events) + self.assert_screen_equal(reader, code, clean=True) + self.maxDiff=None + self.assert_screen_equal(reader, expected) + def test_control_characters(self): code = 'flag = "🏳️‍🌈"' events = code_to_events(code) diff --git a/Lib/test/test_pyrepl/test_terminfo.py b/Lib/test/test_pyrepl/test_terminfo.py new file mode 100644 index 000000000000000..562cf5c905bd676 --- /dev/null +++ b/Lib/test/test_pyrepl/test_terminfo.py @@ -0,0 +1,651 @@ +"""Tests comparing PyREPL's pure Python curses implementation with the standard curses module.""" + +import json +import os +import subprocess +import sys +import unittest +from test.support import requires, has_subprocess_support +from textwrap import dedent + +# Only run these tests if curses is available +requires("curses") + +try: + import _curses +except ImportError: + try: + import curses as _curses + except ImportError: + _curses = None + +from _pyrepl import terminfo + + +ABSENT_STRING = terminfo.ABSENT_STRING +CANCELLED_STRING = terminfo.CANCELLED_STRING + + +class TestCursesCompatibility(unittest.TestCase): + """Test that PyREPL's curses implementation matches the standard curses behavior. + + Python's `curses` doesn't allow calling `setupterm()` again with a different + $TERM in the same process, so we subprocess all `curses` tests to get correctly + set up terminfo.""" + + @classmethod + def setUpClass(cls): + if _curses is None: + raise unittest.SkipTest( + "`curses` capability provided to regrtest but `_curses` not importable" + ) + + if not has_subprocess_support: + raise unittest.SkipTest("test module requires subprocess") + + # we need to ensure there's a terminfo database on the system and that + # `infocmp` works + cls.infocmp("dumb") + + def setUp(self): + self.original_term = os.environ.get("TERM", None) + + def tearDown(self): + if self.original_term is not None: + os.environ["TERM"] = self.original_term + elif "TERM" in os.environ: + del os.environ["TERM"] + + @classmethod + def infocmp(cls, term) -> list[str]: + all_caps = [] + try: + result = subprocess.run( + ["infocmp", "-l1", term], + capture_output=True, + text=True, + check=True, + ) + except Exception: + raise unittest.SkipTest("calling `infocmp` failed on the system") + + for line in result.stdout.splitlines(): + line = line.strip() + if line.startswith("#"): + if "terminfo" not in line and "termcap" in line: + # PyREPL terminfo doesn't parse termcap databases + raise unittest.SkipTest( + "curses using termcap.db: no terminfo database on" + " the system" + ) + elif "=" in line: + cap_name = line.split("=")[0] + all_caps.append(cap_name) + + return all_caps + + def test_setupterm_basic(self): + """Test basic setupterm functionality.""" + # Test with explicit terminal type + test_terms = ["xterm", "xterm-256color", "vt100", "ansi"] + + for term in test_terms: + with self.subTest(term=term): + ncurses_code = dedent( + f""" + import _curses + import json + try: + _curses.setupterm({repr(term)}, 1) + print(json.dumps({{"success": True}})) + except Exception as e: + print(json.dumps({{"success": False, "error": str(e)}})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + std_success = ncurses_data["success"] + + # Set up with PyREPL curses + try: + terminfo.TermInfo(term, fallback=False) + pyrepl_success = True + except Exception as e: + pyrepl_success = False + pyrepl_error = e + + # Both should succeed or both should fail + if std_success: + self.assertTrue( + pyrepl_success, + f"Standard curses succeeded but PyREPL failed for {term}", + ) + else: + # If standard curses failed, PyREPL might still succeed with fallback + # This is acceptable as PyREPL has hardcoded fallbacks + pass + + def test_setupterm_none(self): + """Test setupterm with None (uses TERM from environment).""" + # Test with current TERM + ncurses_code = dedent( + """ + import _curses + import json + try: + _curses.setupterm(None, 1) + print(json.dumps({"success": True})) + except Exception as e: + print(json.dumps({"success": False, "error": str(e)})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + std_success = ncurses_data["success"] + + try: + terminfo.TermInfo(None, fallback=False) + pyrepl_success = True + except Exception: + pyrepl_success = False + + # Both should have same result + if std_success: + self.assertTrue( + pyrepl_success, + "Standard curses succeeded but PyREPL failed for None", + ) + + def test_tigetstr_common_capabilities(self): + """Test tigetstr for common terminal capabilities.""" + # Test with a known terminal type + term = "xterm" + + # Get ALL capabilities from infocmp + all_caps = self.infocmp(term) + + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + results = {{}} + for cap in {repr(all_caps)}: + try: + val = _curses.tigetstr(cap) + if val is None: + results[cap] = None + elif val == -1: + results[cap] = -1 + else: + results[cap] = list(val) + except BaseException: + results[cap] = "error" + print(json.dumps(results)) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + self.assertEqual( + result.returncode, 0, f"Failed to run ncurses: {result.stderr}" + ) + + ncurses_data = json.loads(result.stdout) + + ti = terminfo.TermInfo(term, fallback=False) + + # Test every single capability + for cap in all_caps: + if cap not in ncurses_data or ncurses_data[cap] == "error": + continue + + with self.subTest(capability=cap): + ncurses_val = ncurses_data[cap] + if isinstance(ncurses_val, list): + ncurses_val = bytes(ncurses_val) + + pyrepl_val = ti.get(cap) + + self.assertEqual( + pyrepl_val, + ncurses_val, + f"Capability {cap}: ncurses={repr(ncurses_val)}, " + f"pyrepl={repr(pyrepl_val)}", + ) + + def test_tigetstr_input_types(self): + """Test tigetstr with different input types.""" + term = "xterm" + cap = "cup" + + # Test standard curses behavior with string in subprocess + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + + # Test with string input + try: + std_str_result = _curses.tigetstr({repr(cap)}) + std_accepts_str = True + if std_str_result is None: + std_str_val = None + elif std_str_result == -1: + std_str_val = -1 + else: + std_str_val = list(std_str_result) + except TypeError: + std_accepts_str = False + std_str_val = None + + print(json.dumps({{ + "accepts_str": std_accepts_str, + "str_result": std_str_val + }})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + + # PyREPL setup + ti = terminfo.TermInfo(term, fallback=False) + + # PyREPL behavior with string + try: + pyrepl_str_result = ti.get(cap) + pyrepl_accepts_str = True + except TypeError: + pyrepl_accepts_str = False + + # PyREPL should also only accept strings for compatibility + with self.assertRaises(TypeError): + ti.get(cap.encode("ascii")) + + # Both should accept string input + self.assertEqual( + pyrepl_accepts_str, + ncurses_data["accepts_str"], + "PyREPL and standard curses should have same string handling", + ) + self.assertTrue( + pyrepl_accepts_str, "PyREPL should accept string input" + ) + + def test_tparm_basic(self): + """Test basic tparm functionality.""" + term = "xterm" + ti = terminfo.TermInfo(term, fallback=False) + + # Test cursor positioning (cup) + cup = ti.get("cup") + if cup and cup not in {ABSENT_STRING, CANCELLED_STRING}: + # Test various parameter combinations + test_cases = [ + (0, 0), # Top-left + (5, 10), # Arbitrary position + (23, 79), # Bottom-right of standard terminal + (999, 999), # Large values + ] + + # Get ncurses results in subprocess + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + + # Get cup capability + cup = _curses.tigetstr('cup') + results = {{}} + + for row, col in {repr(test_cases)}: + try: + result = _curses.tparm(cup, row, col) + results[f"{{row}},{{col}}"] = list(result) + except Exception as e: + results[f"{{row}},{{col}}"] = {{"error": str(e)}} + + print(json.dumps(results)) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + self.assertEqual( + result.returncode, 0, f"Failed to run ncurses: {result.stderr}" + ) + ncurses_data = json.loads(result.stdout) + + for row, col in test_cases: + with self.subTest(row=row, col=col): + # Standard curses tparm from subprocess + key = f"{row},{col}" + if ( + isinstance(ncurses_data[key], dict) + and "error" in ncurses_data[key] + ): + self.fail( + f"ncurses tparm failed: {ncurses_data[key]['error']}" + ) + std_result = bytes(ncurses_data[key]) + + # PyREPL curses tparm + pyrepl_result = terminfo.tparm(cup, row, col) + + # Results should be identical + self.assertEqual( + pyrepl_result, + std_result, + f"tparm(cup, {row}, {col}): " + f"std={repr(std_result)}, pyrepl={repr(pyrepl_result)}", + ) + else: + raise unittest.SkipTest( + "test_tparm_basic() requires the `cup` capability" + ) + + def test_tparm_multiple_params(self): + """Test tparm with capabilities using multiple parameters.""" + term = "xterm" + ti = terminfo.TermInfo(term, fallback=False) + + # Test capabilities that take parameters + param_caps = { + "cub": 1, # cursor_left with count + "cuf": 1, # cursor_right with count + "cuu": 1, # cursor_up with count + "cud": 1, # cursor_down with count + "dch": 1, # delete_character with count + "ich": 1, # insert_character with count + } + + # Get all capabilities from PyREPL first + pyrepl_caps = {} + for cap in param_caps: + cap_value = ti.get(cap) + if cap_value and cap_value not in { + ABSENT_STRING, + CANCELLED_STRING, + }: + pyrepl_caps[cap] = cap_value + + if not pyrepl_caps: + self.skipTest("No parametrized capabilities found") + + # Get ncurses results in subprocess + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + + param_caps = {repr(param_caps)} + test_values = [1, 5, 10, 99] + results = {{}} + + for cap in param_caps: + cap_value = _curses.tigetstr(cap) + if cap_value and cap_value != -1: + for value in test_values: + try: + result = _curses.tparm(cap_value, value) + results[f"{{cap}},{{value}}"] = list(result) + except Exception as e: + results[f"{{cap}},{{value}}"] = {{"error": str(e)}} + + print(json.dumps(results)) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + self.assertEqual( + result.returncode, 0, f"Failed to run ncurses: {result.stderr}" + ) + ncurses_data = json.loads(result.stdout) + + for cap, cap_value in pyrepl_caps.items(): + with self.subTest(capability=cap): + # Test with different parameter values + for value in [1, 5, 10, 99]: + key = f"{cap},{value}" + if key in ncurses_data: + if ( + isinstance(ncurses_data[key], dict) + and "error" in ncurses_data[key] + ): + self.fail( + f"ncurses tparm failed: {ncurses_data[key]['error']}" + ) + std_result = bytes(ncurses_data[key]) + + pyrepl_result = terminfo.tparm(cap_value, value) + self.assertEqual( + pyrepl_result, + std_result, + f"tparm({cap}, {value}): " + f"std={repr(std_result)}, pyrepl={repr(pyrepl_result)}", + ) + + def test_tparm_null_handling(self): + """Test tparm with None/null input.""" + term = "xterm" + + ncurses_code = dedent( + f""" + import _curses + import json + _curses.setupterm({repr(term)}, 1) + + # Test with None + try: + _curses.tparm(None) + raises_typeerror = False + except TypeError: + raises_typeerror = True + except Exception as e: + raises_typeerror = False + error_type = type(e).__name__ + + print(json.dumps({{"raises_typeerror": raises_typeerror}})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + + # PyREPL setup + ti = terminfo.TermInfo(term, fallback=False) + + # Test with None - both should raise TypeError + if ncurses_data["raises_typeerror"]: + with self.assertRaises(TypeError): + terminfo.tparm(None) + else: + # If ncurses doesn't raise TypeError, PyREPL shouldn't either + try: + terminfo.tparm(None) + except TypeError: + self.fail("PyREPL raised TypeError but ncurses did not") + + def test_special_terminals(self): + """Test with special terminal types.""" + special_terms = [ + "dumb", # Minimal terminal + "unknown", # Should fall back to defaults + "linux", # Linux console + "screen", # GNU Screen + "tmux", # tmux + ] + + # Get all string capabilities from ncurses + for term in special_terms: + with self.subTest(term=term): + all_caps = self.infocmp(term) + ncurses_code = dedent( + f""" + import _curses + import json + import sys + + try: + _curses.setupterm({repr(term)}, 1) + results = {{}} + for cap in {repr(all_caps)}: + try: + val = _curses.tigetstr(cap) + if val is None: + results[cap] = None + elif val == -1: + results[cap] = -1 + else: + # Convert bytes to list of ints for JSON + results[cap] = list(val) + except BaseException: + results[cap] = "error" + print(json.dumps(results)) + except Exception as e: + print(json.dumps({{"error": str(e)}})) + """ + ) + + # Get ncurses results + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + if result.returncode != 0: + self.fail( + f"Failed to get ncurses data for {term}: {result.stderr}" + ) + + try: + ncurses_data = json.loads(result.stdout) + except json.JSONDecodeError: + self.fail( + f"Failed to parse ncurses output for {term}: {result.stdout}" + ) + + if "error" in ncurses_data and len(ncurses_data) == 1: + # ncurses failed to setup this terminal + # PyREPL should still work with fallback + ti = terminfo.TermInfo(term, fallback=True) + continue + + ti = terminfo.TermInfo(term, fallback=False) + + # Compare all capabilities + for cap in all_caps: + if cap not in ncurses_data: + continue + + with self.subTest(term=term, capability=cap): + ncurses_val = ncurses_data[cap] + if isinstance(ncurses_val, list): + # Convert back to bytes + ncurses_val = bytes(ncurses_val) + + pyrepl_val = ti.get(cap) + + # Both should return the same value + self.assertEqual( + pyrepl_val, + ncurses_val, + f"Capability {cap} for {term}: " + f"ncurses={repr(ncurses_val)}, " + f"pyrepl={repr(pyrepl_val)}", + ) + + def test_terminfo_fallback(self): + """Test that PyREPL falls back gracefully when terminfo is not found.""" + # Use a non-existent terminal type + fake_term = "nonexistent-terminal-type-12345" + + # Check if standard curses can setup this terminal in subprocess + ncurses_code = dedent( + f""" + import _curses + import json + try: + _curses.setupterm({repr(fake_term)}, 1) + print(json.dumps({{"success": True}})) + except _curses.error: + print(json.dumps({{"success": False, "error": "curses.error"}})) + except Exception as e: + print(json.dumps({{"success": False, "error": str(e)}})) + """ + ) + + result = subprocess.run( + [sys.executable, "-c", ncurses_code], + capture_output=True, + text=True, + ) + ncurses_data = json.loads(result.stdout) + + if ncurses_data["success"]: + # If it succeeded, skip this test as we can't test fallback + self.skipTest( + f"System unexpectedly has terminfo for '{fake_term}'" + ) + + # PyREPL should succeed with fallback + try: + ti = terminfo.TermInfo(fake_term, fallback=True) + pyrepl_ok = True + except Exception: + pyrepl_ok = False + + self.assertTrue( + pyrepl_ok, "PyREPL should fall back for unknown terminals" + ) + + # Should still be able to get basic capabilities + bel = ti.get("bel") + self.assertIsNotNone( + bel, "PyREPL should provide basic capabilities after fallback" + ) + + def test_invalid_terminal_names(self): + cases = [ + (42, TypeError), + ("", ValueError), + ("w\x00t", ValueError), + (f"..{os.sep}name", ValueError), + ] + + for term, exc in cases: + with self.subTest(term=term): + with self.assertRaises(exc): + terminfo._validate_terminal_name_or_raise(term) diff --git a/Lib/test/test_pyrepl/test_unix_console.py b/Lib/test/test_pyrepl/test_unix_console.py index c447b310c49a062..8198d489188f1e1 100644 --- a/Lib/test/test_pyrepl/test_unix_console.py +++ b/Lib/test/test_pyrepl/test_unix_console.py @@ -1,12 +1,16 @@ +import errno import itertools import os +import signal import sys +import threading import unittest from functools import partial from test.support import os_helper, force_not_colorized_test_class +from test.support import threading_helper from unittest import TestCase -from unittest.mock import MagicMock, call, patch, ANY +from unittest.mock import MagicMock, call, patch, ANY, Mock from .support import handle_all_events, code_to_events @@ -16,10 +20,15 @@ except ImportError: pass +from _pyrepl.terminfo import _TERMINAL_CAPABILITIES + +TERM_CAPABILITIES = _TERMINAL_CAPABILITIES["ansi"] + def unix_console(events, **kwargs): - console = UnixConsole() + console = UnixConsole(term="xterm") console.get_event = MagicMock(side_effect=events) + console.getpending = MagicMock(return_value=Event("key", "")) height = kwargs.get("height", 25) width = kwargs.get("width", 80) @@ -49,41 +58,11 @@ def unix_console(events, **kwargs): ) -TERM_CAPABILITIES = { - "bel": b"\x07", - "civis": b"\x1b[?25l", - "clear": b"\x1b[H\x1b[2J", - "cnorm": b"\x1b[?12l\x1b[?25h", - "cub": b"\x1b[%p1%dD", - "cub1": b"\x08", - "cud": b"\x1b[%p1%dB", - "cud1": b"\n", - "cuf": b"\x1b[%p1%dC", - "cuf1": b"\x1b[C", - "cup": b"\x1b[%i%p1%d;%p2%dH", - "cuu": b"\x1b[%p1%dA", - "cuu1": b"\x1b[A", - "dch1": b"\x1b[P", - "dch": b"\x1b[%p1%dP", - "el": b"\x1b[K", - "hpa": b"\x1b[%i%p1%dG", - "ich": b"\x1b[%p1%d@", - "ich1": None, - "ind": b"\n", - "pad": None, - "ri": b"\x1bM", - "rmkx": b"\x1b[?1l\x1b>", - "smkx": b"\x1b[?1h\x1b=", -} - - @unittest.skipIf(sys.platform == "win32", "No Unix event queue on Windows") -@patch("_pyrepl.curses.tigetstr", lambda s: TERM_CAPABILITIES.get(s)) @patch( - "_pyrepl.curses.tparm", + "_pyrepl.terminfo.tparm", lambda s, *args: s + b":" + b",".join(str(i).encode() for i in args), ) -@patch("_pyrepl.curses.setupterm", lambda a, b: None) @patch( "termios.tcgetattr", lambda _: [ @@ -121,6 +100,20 @@ def unix_console(events, **kwargs): @patch("os.write") @force_not_colorized_test_class class TestConsole(TestCase): + def test_no_newline(self, _os_write): + code = "1" + events = code_to_events(code) + _, con = handle_events_unix_console(events) + self.assertNotIn(call(ANY, b'\n'), _os_write.mock_calls) + con.restore() + + def test_newline(self, _os_write): + code = "\n" + events = code_to_events(code) + _, con = handle_events_unix_console(events) + _os_write.assert_any_call(ANY, b"\n") + con.restore() + def test_simple_addition(self, _os_write): code = "12+34" events = code_to_events(code) @@ -257,8 +250,7 @@ def test_resize_bigger_on_multiline_function(self, _os_write): events = itertools.chain(code_to_events(code)) reader, console = handle_events_short_unix_console(events) - console.height = 2 - console.getheightwidth = MagicMock(lambda _: (2, 80)) + console.getheightwidth = MagicMock(side_effect=lambda: (2, 80)) def same_reader(_): return reader @@ -293,8 +285,7 @@ def test_resize_smaller_on_multiline_function(self, _os_write): events = itertools.chain(code_to_events(code)) reader, console = handle_events_unix_console_height_3(events) - console.height = 1 - console.getheightwidth = MagicMock(lambda _: (1, 80)) + console.getheightwidth = MagicMock(side_effect=lambda: (1, 80)) def same_reader(_): return reader @@ -320,7 +311,7 @@ def same_console(events): def test_getheightwidth_with_invalid_environ(self, _os_write): # gh-128636 - console = UnixConsole() + console = UnixConsole(term="xterm") with os_helper.EnvironmentVarGuard() as env: env["LINES"] = "" self.assertIsInstance(console.getheightwidth(), tuple) @@ -328,3 +319,49 @@ def test_getheightwidth_with_invalid_environ(self, _os_write): self.assertIsInstance(console.getheightwidth(), tuple) os.environ = [] self.assertIsInstance(console.getheightwidth(), tuple) + + @unittest.skipUnless(sys.platform == "darwin", "requires macOS") + def test_restore_with_invalid_environ_on_macos(self, _os_write): + # gh-128636 for macOS + console = UnixConsole(term="xterm") + with os_helper.EnvironmentVarGuard(): + os.environ = [] + console.prepare() # needed to call restore() + console.restore() # this should succeed + + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_restore_in_thread(self, _os_write): + # gh-139391: ensure that console.restore() silently suppresses + # exceptions when calling signal.signal() from a non-main thread. + console = unix_console([]) + console.old_sigwinch = signal.SIG_DFL + thread = threading.Thread(target=console.restore) + thread.start() + thread.join() # this should not raise + + +@unittest.skipIf(sys.platform == "win32", "No Unix console on Windows") +class TestUnixConsoleEIOHandling(TestCase): + + @patch('_pyrepl.unix_console.tcsetattr') + @patch('_pyrepl.unix_console.tcgetattr') + def test_eio_error_handling_in_restore(self, mock_tcgetattr, mock_tcsetattr): + + import termios + mock_termios = Mock() + mock_termios.iflag = 0 + mock_termios.oflag = 0 + mock_termios.cflag = 0 + mock_termios.lflag = 0 + mock_termios.cc = [0] * 32 + mock_termios.copy.return_value = mock_termios + mock_tcgetattr.return_value = mock_termios + + console = UnixConsole(term="xterm") + console.prepare() + + mock_tcsetattr.side_effect = termios.error(errno.EIO, "Input/output error") + + # EIO error should be handled gracefully in restore() + console.restore() diff --git a/Lib/test/test_pyrepl/test_utils.py b/Lib/test/test_pyrepl/test_utils.py index 8ce1e5371386f07..b621071f1249c30 100644 --- a/Lib/test/test_pyrepl/test_utils.py +++ b/Lib/test/test_pyrepl/test_utils.py @@ -1,14 +1,33 @@ from unittest import TestCase -from _pyrepl.utils import str_width, wlen, prev_next_window +from _pyrepl.utils import str_width, wlen, prev_next_window, gen_colors class TestUtils(TestCase): def test_str_width(self): - characters = ['a', '1', '_', '!', '\x1a', '\u263A', '\uffb9'] + characters = [ + 'a', + '1', + '_', + '!', + '\x1a', + '\u263A', + '\uffb9', + '\N{LATIN SMALL LETTER E WITH ACUTE}', # é + '\N{LATIN SMALL LETTER E WITH CEDILLA}', # ȩ + '\u00ad', + ] for c in characters: self.assertEqual(str_width(c), 1) + zero_width_characters = [ + '\N{COMBINING ACUTE ACCENT}', + '\N{ZERO WIDTH JOINER}', + ] + for c in zero_width_characters: + with self.subTest(character=c): + self.assertEqual(str_width(c), 0) + characters = [chr(99989), chr(99999)] for c in characters: self.assertEqual(str_width(c), 2) @@ -25,6 +44,8 @@ def test_wlen(self): self.assertEqual(wlen('hello'), 5) self.assertEqual(wlen('hello' + '\x1a'), 7) + self.assertEqual(wlen('e\N{COMBINING ACUTE ACCENT}'), 1) + self.assertEqual(wlen('a\N{ZERO WIDTH JOINER}b'), 2) def test_prev_next_window(self): def gen_normal(): @@ -60,3 +81,41 @@ def gen_raise(): self.assertEqual(next(pnw), (3, 4, None)) with self.assertRaises(ZeroDivisionError): next(pnw) + + def test_gen_colors_keyword_highlighting(self): + cases = [ + # no highlights + ("a.set", [(".", "op")]), + ("obj.list", [(".", "op")]), + ("obj.match", [(".", "op")]), + ("b. \\\n format", [(".", "op")]), + # highlights + ("set", [("set", "builtin")]), + ("list", [("list", "builtin")]), + (" \n dict", [("dict", "builtin")]), + ( + "match +1", + [ + ("match", "soft_keyword"), + ("+", "op"), + ("1", "number"), + ], + ), + ( + "match -1", + [ + ("match", "soft_keyword"), + ("-", "op"), + ("1", "number"), + ], + ), + ] + for code, expected_highlights in cases: + with self.subTest(code=code): + colors = list(gen_colors(code)) + # Extract (text, tag) pairs for comparison + actual_highlights = [] + for color in colors: + span_text = code[color.span.start:color.span.end + 1] + actual_highlights.append((span_text, color.tag)) + self.assertEqual(actual_highlights, expected_highlights) diff --git a/Lib/test/test_pyrepl/test_windows_console.py b/Lib/test/test_pyrepl/test_windows_console.py index e7bab226b31ddf1..518ddee11253970 100644 --- a/Lib/test/test_pyrepl/test_windows_console.py +++ b/Lib/test/test_pyrepl/test_windows_console.py @@ -35,6 +35,7 @@ class WindowsConsoleTests(TestCase): def console(self, events, **kwargs) -> Console: console = WindowsConsole() console.get_event = MagicMock(side_effect=events) + console.getpending = MagicMock(return_value=Event("key", "")) console.wait = MagicMock() console._scroll = MagicMock() console._hide_cursor = MagicMock() @@ -71,6 +72,20 @@ def handle_events_short(self, events, **kwargs): def handle_events_height_3(self, events): return self.handle_events(events, height=3) + def test_no_newline(self): + code = "1" + events = code_to_events(code) + _, con = self.handle_events(events) + self.assertNotIn(call(b'\n'), con.out.write.mock_calls) + con.restore() + + def test_newline(self): + code = "\n" + events = code_to_events(code) + _, con = self.handle_events(events) + con.out.write.assert_any_call(b"\n") + con.restore() + def test_simple_addition(self): code = "12+34" events = code_to_events(code) @@ -100,9 +115,7 @@ def test_resize_wider(self): events = code_to_events(code) reader, console = self.handle_events_narrow(events) - console.height = 20 - console.width = 80 - console.getheightwidth = MagicMock(lambda _: (20, 80)) + console.getheightwidth = MagicMock(side_effect=lambda: (20, 80)) def same_reader(_): return reader @@ -128,9 +141,7 @@ def test_resize_narrower(self): events = code_to_events(code) reader, console = self.handle_events(events) - console.height = 20 - console.width = 4 - console.getheightwidth = MagicMock(lambda _: (20, 4)) + console.getheightwidth = MagicMock(side_effect=lambda: (20, 4)) def same_reader(_): return reader @@ -263,8 +274,7 @@ def test_resize_bigger_on_multiline_function(self): events = itertools.chain(code_to_events(code)) reader, console = self.handle_events_short(events) - console.height = 2 - console.getheightwidth = MagicMock(lambda _: (2, 80)) + console.getheightwidth = MagicMock(side_effect=lambda: (2, 80)) def same_reader(_): return reader @@ -301,8 +311,7 @@ def test_resize_smaller_on_multiline_function(self): events = itertools.chain(code_to_events(code)) reader, console = self.handle_events_height_3(events) - console.height = 1 - console.getheightwidth = MagicMock(lambda _: (1, 80)) + console.getheightwidth = MagicMock(side_effect=lambda: (1, 80)) def same_reader(_): return reader @@ -385,6 +394,7 @@ def get_event(self, input_records, **kwargs) -> Console: self.console._read_input = self.mock self.console._WindowsConsole__vt_support = kwargs.get("vt_support", False) + self.console.wait = MagicMock(return_value=True) event = self.console.get_event(block=False) return event @@ -574,6 +584,32 @@ def test_up_vt(self): Event(evt='key', data='up', raw=bytearray(b'\x1b[A'))) self.assertEqual(self.mock.call_count, 3) + # All tests above assume that there is always keyboard data to read, + # because for simplicity we just use + # self.console.wait = MagicMock(return_value=True) + def test_wait_empty(self): + console = WindowsConsole(encoding='utf-8') + console.wait_for_event = MagicMock(return_value=True) + self.assertTrue(console.event_queue.empty()) + timeout = 2.0 + self.assertTrue(console.wait(timeout)) + self.assertEqual(console.wait_for_event.call_count, 1) + self.assertEqual(console.wait_for_event.mock_calls[0], call(timeout)) + + timeout = 1.1 + console.wait_for_event = MagicMock(return_value=False) + self.assertFalse(console.wait(timeout)) + self.assertEqual(console.wait_for_event.call_count, 1) + self.assertEqual(console.wait_for_event.mock_calls[0], call(timeout)) + + def test_wait_not_empty(self): + console = WindowsConsole(encoding='utf-8') + console.wait_for_event = MagicMock(return_value=True) + console.event_queue.push(b"a") + self.assertFalse(console.event_queue.empty()) + self.assertTrue(console.wait(0.0)) + self.assertEqual(console.wait_for_event.call_count, 0) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_queue.py b/Lib/test/test_queue.py index 7f4fe357034b713..c855fb8fe2b05ae 100644 --- a/Lib/test/test_queue.py +++ b/Lib/test/test_queue.py @@ -6,7 +6,7 @@ import time import unittest import weakref -from test.support import gc_collect +from test.support import gc_collect, bigmemtest from test.support import import_helper from test.support import threading_helper @@ -963,33 +963,33 @@ def test_order(self): # One producer, one consumer => results appended in well-defined order self.assertEqual(results, inputs) - def test_many_threads(self): + @bigmemtest(size=50, memuse=100*2**20, dry_run=False) + def test_many_threads(self, size): # Test multiple concurrent put() and get() - N = 50 q = self.q inputs = list(range(10000)) - results = self.run_threads(N, q, inputs, self.feed, self.consume) + results = self.run_threads(size, q, inputs, self.feed, self.consume) # Multiple consumers without synchronization append the # results in random order self.assertEqual(sorted(results), inputs) - def test_many_threads_nonblock(self): + @bigmemtest(size=50, memuse=100*2**20, dry_run=False) + def test_many_threads_nonblock(self, size): # Test multiple concurrent put() and get(block=False) - N = 50 q = self.q inputs = list(range(10000)) - results = self.run_threads(N, q, inputs, + results = self.run_threads(size, q, inputs, self.feed, self.consume_nonblock) self.assertEqual(sorted(results), inputs) - def test_many_threads_timeout(self): + @bigmemtest(size=50, memuse=100*2**20, dry_run=False) + def test_many_threads_timeout(self, size): # Test multiple concurrent put() and get(timeout=...) - N = 50 q = self.q inputs = list(range(1000)) - results = self.run_threads(N, q, inputs, + results = self.run_threads(size, q, inputs, self.feed, self.consume_timeout) self.assertEqual(sorted(results), inputs) diff --git a/Lib/test/test_raise.py b/Lib/test/test_raise.py index dcf0753bc828f3f..645ef291a584993 100644 --- a/Lib/test/test_raise.py +++ b/Lib/test/test_raise.py @@ -186,18 +186,14 @@ def test_class_cause(self): self.fail("No exception raised") def test_class_cause_nonexception_result(self): - class ConstructsNone(BaseException): - @classmethod + # See https://github.com/python/cpython/issues/140530. + class ConstructMortal(BaseException): def __new__(*args, **kwargs): - return None - try: - raise IndexError from ConstructsNone - except TypeError as e: - self.assertIn("should have returned an instance of BaseException", str(e)) - except IndexError: - self.fail("Wrong kind of exception raised") - else: - self.fail("No exception raised") + return ["mortal value"] + + msg = ".*should have returned an instance of BaseException.*" + with self.assertRaisesRegex(TypeError, msg): + raise IndexError from ConstructMortal def test_instance_cause(self): cause = KeyError() diff --git a/Lib/test/test_random.py b/Lib/test/test_random.py index 43957f525f10c03..3739fe0ac393b3c 100644 --- a/Lib/test/test_random.py +++ b/Lib/test/test_random.py @@ -14,6 +14,15 @@ from fractions import Fraction from collections import abc, Counter + +class MyIndex: + def __init__(self, value): + self.value = value + + def __index__(self): + return self.value + + class TestBasicOps: # Superclass with tests common to all generators. # Subclasses must arrange for self.gen to retrieve the Random instance @@ -142,6 +151,7 @@ def test_sample(self): # Exception raised if size of sample exceeds that of population self.assertRaises(ValueError, self.gen.sample, population, N+1) self.assertRaises(ValueError, self.gen.sample, [], -1) + self.assertRaises(TypeError, self.gen.sample, population, 1.0) def test_sample_distribution(self): # For the entire allowable range of 0 <= k <= N, validate that @@ -259,6 +269,7 @@ def test_choices(self): choices(data, range(4), k=5), choices(k=5, population=data, weights=range(4)), choices(k=5, population=data, cum_weights=range(4)), + choices(data, k=MyIndex(5)), ]: self.assertEqual(len(sample), 5) self.assertEqual(type(sample), list) @@ -369,118 +380,40 @@ def test_gauss(self): self.assertEqual(x1, x2) self.assertEqual(y1, y2) + @support.requires_IEEE_754 + def test_53_bits_per_float(self): + span = 2 ** 53 + cum = 0 + for i in range(100): + cum |= int(self.gen.random() * span) + self.assertEqual(cum, span-1) + def test_getrandbits(self): + getrandbits = self.gen.getrandbits # Verify ranges for k in range(1, 1000): - self.assertTrue(0 <= self.gen.getrandbits(k) < 2**k) - self.assertEqual(self.gen.getrandbits(0), 0) + self.assertTrue(0 <= getrandbits(k) < 2**k) + self.assertEqual(getrandbits(0), 0) # Verify all bits active - getbits = self.gen.getrandbits for span in [1, 2, 3, 4, 31, 32, 32, 52, 53, 54, 119, 127, 128, 129]: all_bits = 2**span-1 cum = 0 cpl_cum = 0 for i in range(100): - v = getbits(span) + v = getrandbits(span) cum |= v cpl_cum |= all_bits ^ v self.assertEqual(cum, all_bits) self.assertEqual(cpl_cum, all_bits) # Verify argument checking - self.assertRaises(TypeError, self.gen.getrandbits) - self.assertRaises(TypeError, self.gen.getrandbits, 1, 2) - self.assertRaises(ValueError, self.gen.getrandbits, -1) - self.assertRaises(TypeError, self.gen.getrandbits, 10.1) - - def test_pickling(self): - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - state = pickle.dumps(self.gen, proto) - origseq = [self.gen.random() for i in range(10)] - newgen = pickle.loads(state) - restoredseq = [newgen.random() for i in range(10)] - self.assertEqual(origseq, restoredseq) - - def test_bug_1727780(self): - # verify that version-2-pickles can be loaded - # fine, whether they are created on 32-bit or 64-bit - # platforms, and that version-3-pickles load fine. - files = [("randv2_32.pck", 780), - ("randv2_64.pck", 866), - ("randv3.pck", 343)] - for file, value in files: - with open(support.findfile(file),"rb") as f: - r = pickle.load(f) - self.assertEqual(int(r.random()*1000), value) - - def test_bug_9025(self): - # Had problem with an uneven distribution in int(n*random()) - # Verify the fix by checking that distributions fall within expectations. - n = 100000 - randrange = self.gen.randrange - k = sum(randrange(6755399441055744) % 3 == 2 for i in range(n)) - self.assertTrue(0.30 < k/n < .37, (k/n)) - - def test_randbytes(self): - # Verify ranges - for n in range(1, 10): - data = self.gen.randbytes(n) - self.assertEqual(type(data), bytes) - self.assertEqual(len(data), n) - - self.assertEqual(self.gen.randbytes(0), b'') - - # Verify argument checking - self.assertRaises(TypeError, self.gen.randbytes) - self.assertRaises(TypeError, self.gen.randbytes, 1, 2) - self.assertRaises(ValueError, self.gen.randbytes, -1) - self.assertRaises(TypeError, self.gen.randbytes, 1.0) - - def test_mu_sigma_default_args(self): - self.assertIsInstance(self.gen.normalvariate(), float) - self.assertIsInstance(self.gen.gauss(), float) - - -try: - random.SystemRandom().random() -except NotImplementedError: - SystemRandom_available = False -else: - SystemRandom_available = True - -@unittest.skipUnless(SystemRandom_available, "random.SystemRandom not available") -class SystemRandom_TestBasicOps(TestBasicOps, unittest.TestCase): - gen = random.SystemRandom() - - def test_autoseed(self): - # Doesn't need to do anything except not fail - self.gen.seed() - - def test_saverestore(self): - self.assertRaises(NotImplementedError, self.gen.getstate) - self.assertRaises(NotImplementedError, self.gen.setstate, None) - - def test_seedargs(self): - # Doesn't need to do anything except not fail - self.gen.seed(100) - - def test_gauss(self): - self.gen.gauss_next = None - self.gen.seed(100) - self.assertEqual(self.gen.gauss_next, None) - - def test_pickling(self): - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - self.assertRaises(NotImplementedError, pickle.dumps, self.gen, proto) - - def test_53_bits_per_float(self): - # This should pass whenever a C double has 53 bit precision. - span = 2 ** 53 - cum = 0 - for i in range(100): - cum |= int(self.gen.random() * span) - self.assertEqual(cum, span-1) + self.assertRaises(TypeError, getrandbits) + self.assertRaises(TypeError, getrandbits, 1, 2) + self.assertRaises(ValueError, getrandbits, -1) + self.assertRaises(OverflowError, getrandbits, 1<<1000) + self.assertRaises(ValueError, getrandbits, -1<<1000) + self.assertRaises(TypeError, getrandbits, 10.1) def test_bigrand(self): # The randrange routine should build-up the required number of bits @@ -559,6 +492,10 @@ def test_randrange_step(self): randrange(1000, step=100) with self.assertRaises(TypeError): randrange(1000, None, step=100) + with self.assertRaises(TypeError): + randrange(1000, step=MyIndex(1)) + with self.assertRaises(TypeError): + randrange(1000, None, step=MyIndex(1)) def test_randbelow_logic(self, _log=log, int=int): # check bitcount transition points: 2**i and 2**(i+1)-1 @@ -581,6 +518,116 @@ def test_randbelow_logic(self, _log=log, int=int): self.assertEqual(k, numbits) # note the stronger assertion self.assertTrue(2**k > n > 2**(k-1)) # note the stronger assertion + def test_randrange_index(self): + randrange = self.gen.randrange + self.assertIn(randrange(MyIndex(5)), range(5)) + self.assertIn(randrange(MyIndex(2), MyIndex(7)), range(2, 7)) + self.assertIn(randrange(MyIndex(5), MyIndex(15), MyIndex(2)), range(5, 15, 2)) + + def test_randint(self): + randint = self.gen.randint + self.assertIn(randint(2, 5), (2, 3, 4, 5)) + self.assertEqual(randint(2, 2), 2) + self.assertIn(randint(MyIndex(2), MyIndex(5)), (2, 3, 4, 5)) + self.assertEqual(randint(MyIndex(2), MyIndex(2)), 2) + + self.assertRaises(ValueError, randint, 5, 2) + self.assertRaises(TypeError, randint) + self.assertRaises(TypeError, randint, 2) + self.assertRaises(TypeError, randint, 2, 5, 1) + self.assertRaises(TypeError, randint, 2.0, 5) + self.assertRaises(TypeError, randint, 2, 5.0) + + def test_pickling(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + state = pickle.dumps(self.gen, proto) + origseq = [self.gen.random() for i in range(10)] + newgen = pickle.loads(state) + restoredseq = [newgen.random() for i in range(10)] + self.assertEqual(origseq, restoredseq) + + def test_bug_1727780(self): + # verify that version-2-pickles can be loaded + # fine, whether they are created on 32-bit or 64-bit + # platforms, and that version-3-pickles load fine. + files = [("randv2_32.pck", 780), + ("randv2_64.pck", 866), + ("randv3.pck", 343)] + for file, value in files: + with open(support.findfile(file),"rb") as f: + r = pickle.load(f) + self.assertEqual(int(r.random()*1000), value) + + def test_bug_9025(self): + # Had problem with an uneven distribution in int(n*random()) + # Verify the fix by checking that distributions fall within expectations. + n = 100000 + randrange = self.gen.randrange + k = sum(randrange(6755399441055744) % 3 == 2 for i in range(n)) + self.assertTrue(0.30 < k/n < .37, (k/n)) + + def test_randrange_bug_1590891(self): + start = 1000000000000 + stop = -100000000000000000000 + step = -200 + x = self.gen.randrange(start, stop, step) + self.assertTrue(stop < x <= start) + self.assertEqual((x+stop)%step, 0) + + def test_randbytes(self): + # Verify ranges + for n in range(1, 10): + data = self.gen.randbytes(n) + self.assertEqual(type(data), bytes) + self.assertEqual(len(data), n) + + self.assertEqual(self.gen.randbytes(0), b'') + + # Verify argument checking + self.assertRaises(TypeError, self.gen.randbytes) + self.assertRaises(TypeError, self.gen.randbytes, 1, 2) + self.assertRaises(ValueError, self.gen.randbytes, -1) + self.assertRaises(OverflowError, self.gen.randbytes, 1<<1000) + self.assertRaises((ValueError, OverflowError), self.gen.randbytes, -1<<1000) + self.assertRaises(TypeError, self.gen.randbytes, 1.0) + + def test_mu_sigma_default_args(self): + self.assertIsInstance(self.gen.normalvariate(), float) + self.assertIsInstance(self.gen.gauss(), float) + + +try: + random.SystemRandom().random() +except NotImplementedError: + SystemRandom_available = False +else: + SystemRandom_available = True + +@unittest.skipUnless(SystemRandom_available, "random.SystemRandom not available") +class SystemRandom_TestBasicOps(TestBasicOps, unittest.TestCase): + gen = random.SystemRandom() + + def test_autoseed(self): + # Doesn't need to do anything except not fail + self.gen.seed() + + def test_saverestore(self): + self.assertRaises(NotImplementedError, self.gen.getstate) + self.assertRaises(NotImplementedError, self.gen.setstate, None) + + def test_seedargs(self): + # Doesn't need to do anything except not fail + self.gen.seed(100) + + def test_gauss(self): + self.gen.gauss_next = None + self.gen.seed(100) + self.assertEqual(self.gen.gauss_next, None) + + def test_pickling(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + self.assertRaises(NotImplementedError, pickle.dumps, self.gen, proto) + class TestRawMersenneTwister(unittest.TestCase): @test.support.cpython_only @@ -766,38 +813,6 @@ def test_long_seed(self): seed = (1 << (10000 * 8)) - 1 # about 10K bytes self.gen.seed(seed) - def test_53_bits_per_float(self): - # This should pass whenever a C double has 53 bit precision. - span = 2 ** 53 - cum = 0 - for i in range(100): - cum |= int(self.gen.random() * span) - self.assertEqual(cum, span-1) - - def test_bigrand(self): - # The randrange routine should build-up the required number of bits - # in stages so that all bit positions are active. - span = 2 ** 500 - cum = 0 - for i in range(100): - r = self.gen.randrange(span) - self.assertTrue(0 <= r < span) - cum |= r - self.assertEqual(cum, span-1) - - def test_bigrand_ranges(self): - for i in [40,80, 160, 200, 211, 250, 375, 512, 550]: - start = self.gen.randrange(2 ** (i-2)) - stop = self.gen.randrange(2 ** i) - if stop <= start: - continue - self.assertTrue(start <= self.gen.randrange(start, stop) < stop) - - def test_rangelimits(self): - for start, stop in [(-2,0), (-(2**60)-2,-(2**60)), (2**60,2**60+2)]: - self.assertEqual(set(range(start,stop)), - set([self.gen.randrange(start,stop) for i in range(100)])) - def test_getrandbits(self): super().test_getrandbits() @@ -805,6 +820,25 @@ def test_getrandbits(self): self.gen.seed(1234567) self.assertEqual(self.gen.getrandbits(100), 97904845777343510404718956115) + self.gen.seed(1234567) + self.assertEqual(self.gen.getrandbits(MyIndex(100)), + 97904845777343510404718956115) + + def test_getrandbits_2G_bits(self): + size = 2**31 + self.gen.seed(1234567) + x = self.gen.getrandbits(size) + self.assertEqual(x.bit_length(), size) + self.assertEqual(x & (2**100-1), 890186470919986886340158459475) + self.assertEqual(x >> (size-100), 1226514312032729439655761284440) + + @support.bigmemtest(size=2**32, memuse=1/8+2/15, dry_run=False) + def test_getrandbits_4G_bits(self, size): + self.gen.seed(1234568) + x = self.gen.getrandbits(size) + self.assertEqual(x.bit_length(), size) + self.assertEqual(x & (2**100-1), 287241425661104632871036099814) + self.assertEqual(x >> (size-100), 739728759900339699429794460738) def test_randrange_uses_getrandbits(self): # Verify use of getrandbits by randrange @@ -816,27 +850,6 @@ def test_randrange_uses_getrandbits(self): self.assertEqual(self.gen.randrange(2**99), 97904845777343510404718956115) - def test_randbelow_logic(self, _log=log, int=int): - # check bitcount transition points: 2**i and 2**(i+1)-1 - # show that: k = int(1.001 + _log(n, 2)) - # is equal to or one greater than the number of bits in n - for i in range(1, 1000): - n = 1 << i # check an exact power of two - numbits = i+1 - k = int(1.00001 + _log(n, 2)) - self.assertEqual(k, numbits) - self.assertEqual(n, 2**(k-1)) - - n += n - 1 # check 1 below the next power of two - k = int(1.00001 + _log(n, 2)) - self.assertIn(k, [numbits, numbits+1]) - self.assertTrue(2**k > n > 2**(k-2)) - - n -= n >> 15 # check a little farther below the next power of two - k = int(1.00001 + _log(n, 2)) - self.assertEqual(k, numbits) # note the stronger assertion - self.assertTrue(2**k > n > 2**(k-1)) # note the stronger assertion - def test_randbelow_without_getrandbits(self): # Random._randbelow() can only use random() when the built-in one # has been overridden but no new getrandbits() method was supplied. @@ -871,14 +884,6 @@ def test_randbelow_without_getrandbits(self): self.gen._randbelow_without_getrandbits(n, maxsize=maxsize) self.assertEqual(random_mock.call_count, 2) - def test_randrange_bug_1590891(self): - start = 1000000000000 - stop = -100000000000000000000 - step = -200 - x = self.gen.randrange(start, stop, step) - self.assertTrue(stop < x <= start) - self.assertEqual((x+stop)%step, 0) - def test_choices_algorithms(self): # The various ways of specifying weights should produce the same results choices = self.gen.choices @@ -962,6 +967,14 @@ def test_randbytes_getrandbits(self): self.assertEqual(self.gen.randbytes(n), gen2.getrandbits(n * 8).to_bytes(n, 'little')) + @support.bigmemtest(size=2**29, memuse=1+16/15, dry_run=False) + def test_randbytes_256M(self, size): + self.gen.seed(2849427419) + x = self.gen.randbytes(size) + self.assertEqual(len(x), size) + self.assertEqual(x[:12].hex(), 'f6fd9ae63855ab91ea238b4f') + self.assertEqual(x[-12:].hex(), '0e7af69a84ee99bf4a11becc') + def test_sample_counts_equivalence(self): # Test the documented strong equivalence to a sample with repeated elements. # We run this test on random.Random() which makes deterministic selections @@ -1061,6 +1074,14 @@ def test_avg_std(self): self.assertAlmostEqual(s2/(N-1), sigmasqrd, places=2, msg='%s%r' % (variate.__name__, args)) + def test_binomialvariate_btrs_random_zero(self): + for p, expected in ((0.25, 25), (0.75, 75)): + with self.subTest(p=p): + g = random.Random() + with unittest.mock.patch.object( + g, 'random', side_effect=(0.0, 0.5, 0.5)): + self.assertEqual(g.binomialvariate(100, p), expected) + def test_constant(self): g = random.Random() N = 100 @@ -1415,27 +1436,27 @@ class CommandLineTest(unittest.TestCase): def test_parse_args(self): args, help_text = random._parse_args(shlex.split("--choice a b c")) self.assertEqual(args.choice, ["a", "b", "c"]) - self.assertTrue(help_text.startswith("usage: ")) + self.assertStartsWith(help_text, "usage: ") args, help_text = random._parse_args(shlex.split("--integer 5")) self.assertEqual(args.integer, 5) - self.assertTrue(help_text.startswith("usage: ")) + self.assertStartsWith(help_text, "usage: ") args, help_text = random._parse_args(shlex.split("--float 2.5")) self.assertEqual(args.float, 2.5) - self.assertTrue(help_text.startswith("usage: ")) + self.assertStartsWith(help_text, "usage: ") args, help_text = random._parse_args(shlex.split("a b c")) self.assertEqual(args.input, ["a", "b", "c"]) - self.assertTrue(help_text.startswith("usage: ")) + self.assertStartsWith(help_text, "usage: ") args, help_text = random._parse_args(shlex.split("5")) self.assertEqual(args.input, ["5"]) - self.assertTrue(help_text.startswith("usage: ")) + self.assertStartsWith(help_text, "usage: ") args, help_text = random._parse_args(shlex.split("2.5")) self.assertEqual(args.input, ["2.5"]) - self.assertTrue(help_text.startswith("usage: ")) + self.assertStartsWith(help_text, "usage: ") def test_main(self): for command, expected in [ diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index 3870b153688b25f..2c9c290e8906b73 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -470,6 +470,16 @@ def test_iterator_setstate(self): it.__setstate__(2**64 - 7) self.assertEqual(list(it), [12, 10]) + def test_iterator_invalid_setstate(self): + for invalid_value in (1.0, ""): + ranges = (('rangeiter', range(10, 100, 2)), + ('longrangeiter', range(10, 2**65, 2))) + for rng_name, rng in ranges: + with self.subTest(invalid_value=invalid_value, range=rng_name): + it = iter(rng) + with self.assertRaises(TypeError): + it.__setstate__(invalid_value) + def test_odd_bug(self): # This used to raise a "SystemError: NULL result without error" # because the range validation step was eating the exception diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index f79a61490789960..f6e797b3785dbeb 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -2178,6 +2178,8 @@ def test_bug_20998(self): self.assertEqual(re.fullmatch('[a-c]+', 'ABC', re.I).span(), (0, 3)) @unittest.skipIf(linked_to_musl(), "musl libc issue, bpo-46390") + @unittest.skipIf(sys.platform.startswith("sunos"), + "test doesn't work on Solaris, gh-91214") def test_locale_caching(self): # Issue #22410 oldlocale = locale.setlocale(locale.LC_CTYPE) @@ -2215,6 +2217,8 @@ def check_en_US_utf8(self): self.assertIsNone(re.match(b'(?Li)\xe5', b'\xc5')) @unittest.skipIf(linked_to_musl(), "musl libc issue, bpo-46390") + @unittest.skipIf(sys.platform.startswith("sunos"), + "test doesn't work on Solaris, gh-91214") def test_locale_compiled(self): oldlocale = locale.setlocale(locale.LC_CTYPE) self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) @@ -2868,11 +2872,11 @@ def test_long_pattern(self): pattern = 'Very %spattern' % ('long ' * 1000) r = repr(re.compile(pattern)) self.assertLess(len(r), 300) - self.assertEqual(r[:30], "re.compile('Very long long lon") + self.assertStartsWith(r, "re.compile('Very long long lon") r = repr(re.compile(pattern, re.I)) self.assertLess(len(r), 300) - self.assertEqual(r[:30], "re.compile('Very long long lon") - self.assertEqual(r[-16:], ", re.IGNORECASE)") + self.assertStartsWith(r, "re.compile('Very long long lon") + self.assertEndsWith(r, ", re.IGNORECASE)") def test_flags_repr(self): self.assertEqual(repr(re.I), "re.IGNORECASE") @@ -2951,7 +2955,7 @@ def test_deprecated_modules(self): self.assertEqual(mod.__name__, name) self.assertEqual(mod.__package__, '') for attr in deprecated[name]: - self.assertTrue(hasattr(mod, attr)) + self.assertHasAttr(mod, attr) del sys.modules[name] @cpython_only diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py index b9d082b3597f131..45192fe508270dd 100644 --- a/Lib/test/test_readline.py +++ b/Lib/test/test_readline.py @@ -1,6 +1,7 @@ """ Very minimal unittests for parts of the readline module. """ +import codecs import locale import os import sys @@ -231,6 +232,13 @@ def test_nonascii(self): # writing and reading non-ASCII bytes into/from a TTY works, but # readline or ncurses ignores non-ASCII bytes on read. self.skipTest(f"the LC_CTYPE locale is {loc!r}") + if sys.flags.utf8_mode: + encoding = locale.getencoding() + encoding = codecs.lookup(encoding).name # normalize the name + if encoding != "utf-8": + # gh-133711: The Python UTF-8 Mode ignores the LC_CTYPE locale + # and always use the UTF-8 encoding. + self.skipTest(f"the LC_CTYPE encoding is {encoding!r}") try: readline.add_history("\xEB\xEF") diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index 7e317d5ab943ff5..42cc1c2f7c7eaa9 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -182,6 +182,22 @@ def test_randomize(self): self.assertTrue(regrtest.randomize) self.assertIsInstance(regrtest.random_seed, int) + def test_no_randomize(self): + ns = self.parse_args([]) + self.assertIs(ns.randomize, False) + + ns = self.parse_args(["--randomize"]) + self.assertIs(ns.randomize, True) + + ns = self.parse_args(["--no-randomize"]) + self.assertIs(ns.randomize, False) + + ns = self.parse_args(["--randomize", "--no-randomize"]) + self.assertIs(ns.randomize, False) + + ns = self.parse_args(["--no-randomize", "--randomize"]) + self.assertIs(ns.randomize, False) + def test_randseed(self): ns = self.parse_args(['--randseed', '12345']) self.assertEqual(ns.random_seed, 12345) @@ -189,6 +205,10 @@ def test_randseed(self): self.checkError(['--randseed'], 'expected one argument') self.checkError(['--randseed', 'foo'], 'invalid int value') + ns = self.parse_args(['--randseed', '12345', '--no-randomize']) + self.assertEqual(ns.random_seed, 12345) + self.assertFalse(ns.randomize) + def test_fromfile(self): for opt in '-f', '--fromfile': with self.subTest(opt=opt): @@ -259,26 +279,56 @@ def test_use(self): for opt in '-u', '--use': with self.subTest(opt=opt): ns = self.parse_args([opt, 'gui,network']) - self.assertEqual(ns.use_resources, ['gui', 'network']) + self.assertEqual(ns.use_resources, {'gui': None, 'network': None}) + ns = self.parse_args([opt, 'gui', opt, 'network']) + self.assertEqual(ns.use_resources, {'gui': None, 'network': None}) ns = self.parse_args([opt, 'gui,none,network']) - self.assertEqual(ns.use_resources, ['network']) + self.assertEqual(ns.use_resources, {'network': None}) + ns = self.parse_args([opt, 'gui', opt, 'none', opt, 'network']) + self.assertEqual(ns.use_resources, {'network': None}) - expected = list(cmdline.ALL_RESOURCES) - expected.remove('gui') + expected = dict.fromkeys(cmdline.ALL_RESOURCES) + del expected['gui'] ns = self.parse_args([opt, 'all,-gui']) self.assertEqual(ns.use_resources, expected) + self.checkError([opt], 'expected one argument') self.checkError([opt, 'foo'], 'invalid resource') # all + a resource not part of "all" + expected = dict.fromkeys(cmdline.ALL_RESOURCES) + expected['tzdata'] = None ns = self.parse_args([opt, 'all,tzdata']) - self.assertEqual(ns.use_resources, - list(cmdline.ALL_RESOURCES) + ['tzdata']) + self.assertEqual(ns.use_resources, expected) + ns = self.parse_args([opt, 'all', opt, 'tzdata']) + self.assertEqual(ns.use_resources, expected) # test another resource which is not part of "all" ns = self.parse_args([opt, 'extralargefile']) - self.assertEqual(ns.use_resources, ['extralargefile']) + self.assertEqual(ns.use_resources, {'extralargefile': None}) + + # test resource with value + ns = self.parse_args([opt, 'xpickle=2.7']) + self.assertEqual(ns.use_resources, {'xpickle': '2.7'}) + ns = self.parse_args([opt, 'xpickle=2.7,xpickle=3.3']) + self.assertEqual(ns.use_resources, {'xpickle': '3.3'}) + ns = self.parse_args([opt, 'xpickle=2.7,none']) + self.assertEqual(ns.use_resources, {}) + ns = self.parse_args([opt, 'xpickle=2.7,-xpickle']) + self.assertEqual(ns.use_resources, {}) + + expected = dict.fromkeys(cmdline.ALL_RESOURCES) + expected['xpickle'] = '2.7' + ns = self.parse_args([opt, 'all,xpickle=2.7']) + self.assertEqual(ns.use_resources, expected) + ns = self.parse_args([opt, 'all', opt, 'xpickle=2.7']) + self.assertEqual(ns.use_resources, expected) + + # test invalid resources with value + self.checkError([opt, 'all=0'], 'invalid resource: all=0') + self.checkError([opt, 'none=0'], 'invalid resource: none=0') + self.checkError([opt, 'all,-gui=0'], 'invalid resource: -gui=0') def test_memlimit(self): for opt in '-M', '--memlimit': @@ -428,29 +478,31 @@ def create_regrtest(self, args): return regrtest - def check_ci_mode(self, args, use_resources, rerun=True): + def check_ci_mode(self, args, use_resources, + *, rerun=True, randomize=True, output_on_failure=True): regrtest = self.create_regrtest(args) self.assertEqual(regrtest.num_workers, -1) self.assertEqual(regrtest.want_rerun, rerun) - self.assertTrue(regrtest.randomize) + self.assertEqual(regrtest.fail_rerun, False) + self.assertEqual(regrtest.randomize, randomize) self.assertIsInstance(regrtest.random_seed, int) self.assertTrue(regrtest.fail_env_changed) self.assertTrue(regrtest.print_slowest) - self.assertTrue(regrtest.output_on_failure) - self.assertEqual(sorted(regrtest.use_resources), sorted(use_resources)) + self.assertEqual(regrtest.output_on_failure, output_on_failure) + self.assertEqual(regrtest.use_resources, use_resources) return regrtest def test_fast_ci(self): args = ['--fast-ci'] - use_resources = sorted(cmdline.ALL_RESOURCES) - use_resources.remove('cpu') + use_resources = dict.fromkeys(cmdline.ALL_RESOURCES) + del use_resources['cpu'] regrtest = self.check_ci_mode(args, use_resources) self.assertEqual(regrtest.timeout, 10 * 60) def test_fast_ci_python_cmd(self): args = ['--fast-ci', '--python', 'python -X dev'] - use_resources = sorted(cmdline.ALL_RESOURCES) - use_resources.remove('cpu') + use_resources = dict.fromkeys(cmdline.ALL_RESOURCES) + del use_resources['cpu'] regrtest = self.check_ci_mode(args, use_resources, rerun=False) self.assertEqual(regrtest.timeout, 10 * 60) self.assertEqual(regrtest.python_cmd, ('python', '-X', 'dev')) @@ -458,17 +510,35 @@ def test_fast_ci_python_cmd(self): def test_fast_ci_resource(self): # it should be possible to override resources individually args = ['--fast-ci', '-u-network'] - use_resources = sorted(cmdline.ALL_RESOURCES) - use_resources.remove('cpu') - use_resources.remove('network') + use_resources = dict.fromkeys(cmdline.ALL_RESOURCES) + del use_resources['cpu'] + del use_resources['network'] self.check_ci_mode(args, use_resources) + def test_fast_ci_verbose(self): + args = ['--fast-ci', '--verbose'] + use_resources = dict.fromkeys(cmdline.ALL_RESOURCES) + del use_resources['cpu'] + regrtest = self.check_ci_mode(args, use_resources, + output_on_failure=False) + self.assertEqual(regrtest.verbose, True) + def test_slow_ci(self): args = ['--slow-ci'] - use_resources = sorted(cmdline.ALL_RESOURCES) + use_resources = dict.fromkeys(cmdline.ALL_RESOURCES) regrtest = self.check_ci_mode(args, use_resources) self.assertEqual(regrtest.timeout, 20 * 60) + def test_ci_no_randomize(self): + use_resources = dict.fromkeys(cmdline.ALL_RESOURCES) + self.check_ci_mode( + ["--slow-ci", "--no-randomize"], use_resources, randomize=False + ) + del use_resources['cpu'] + self.check_ci_mode( + ["--fast-ci", "--no-randomize"], use_resources, randomize=False + ) + def test_dont_add_python_opts(self): args = ['--dont-add-python-opts'] ns = cmdline._parse_args(args) @@ -768,13 +838,16 @@ def run_command(self, args, input=None, exitcode=0, **kw): self.fail(msg) return proc - def run_python(self, args, **kw): + def run_python(self, args, isolated=True, **kw): extraargs = [] if 'uops' in sys._xoptions: # Pass -X uops along extraargs.extend(['-X', 'uops']) - args = [sys.executable, *extraargs, '-X', 'faulthandler', '-I', *args] - proc = self.run_command(args, **kw) + cmd = [sys.executable, *extraargs, '-X', 'faulthandler'] + if isolated: + cmd.append('-I') + cmd.extend(args) + proc = self.run_command(cmd, **kw) return proc.stdout @@ -831,8 +904,8 @@ def check_output(self, output): self.check_executed_tests(output, self.tests, randomize=True, stats=len(self.tests)) - def run_tests(self, args, env=None): - output = self.run_python(args, env=env) + def run_tests(self, args, env=None, isolated=True): + output = self.run_python(args, env=env, isolated=isolated) self.check_output(output) def test_script_regrtest(self): @@ -874,7 +947,10 @@ def test_script_autotest(self): self.run_tests(args) def run_batch(self, *args): - proc = self.run_command(args) + proc = self.run_command(args, + # gh-133711: cmd.exe uses the OEM code page + # to display the non-ASCII current directory + errors="backslashreplace") self.check_output(proc.stdout) @unittest.skipUnless(sysconfig.is_python_build(), @@ -2147,10 +2223,7 @@ def test_unload_tests(self): self.check_executed_tests(output, tests, stats=3) def check_add_python_opts(self, option): - # --fast-ci and --slow-ci add "-u -W default -bb -E" options to Python - - # Skip test if _testinternalcapi is missing - import_helper.import_module('_testinternalcapi') + # --fast-ci and --slow-ci add "-u -W error -bb -E" options to Python code = textwrap.dedent(r""" import sys @@ -2165,25 +2238,26 @@ def check_add_python_opts(self, option): use_environment = (support.is_emscripten or support.is_wasi) class WorkerTests(unittest.TestCase): - @unittest.skipUnless(config_get is None, 'need config_get()') + @unittest.skipIf(config_get is None, 'need config_get()') def test_config(self): - config = config_get() # -u option self.assertEqual(config_get('buffered_stdio'), 0) - # -W default option - self.assertTrue(config_get('warnoptions'), ['default']) + # -W error option + self.assertEqual(config_get('warnoptions'), + ['error', 'error::BytesWarning']) # -bb option - self.assertTrue(config_get('bytes_warning'), 2) + self.assertEqual(config_get('bytes_warning'), 2) # -E option - self.assertTrue(config_get('use_environment'), use_environment) + self.assertEqual(config_get('use_environment'), use_environment) def test_python_opts(self): # -u option self.assertTrue(sys.__stdout__.write_through) self.assertTrue(sys.__stderr__.write_through) - # -W default option - self.assertTrue(sys.warnoptions, ['default']) + # -W error option + self.assertEqual(sys.warnoptions, + ['error', 'error::BytesWarning']) # -bb option self.assertEqual(sys.flags.bytes_warning, 2) @@ -2202,7 +2276,8 @@ def test_python_opts(self): proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - text=True) + text=True, + env=support.make_clean_env()) self.assertEqual(proc.returncode, 0, proc) def test_add_python_opts(self): @@ -2273,7 +2348,6 @@ def test_pass(self): def test_xml(self): code = textwrap.dedent(r""" import unittest - from test import support class VerboseTests(unittest.TestCase): def test_failed(self): @@ -2308,6 +2382,50 @@ def test_failed(self): for out in testcase.iter('system-out'): self.assertEqual(out.text, r"abc \x1b def") + def test_nonascii(self): + code = textwrap.dedent(r""" + import unittest + + class NonASCIITests(unittest.TestCase): + def test_docstring(self): + '''docstring:\u20ac''' + + def test_subtest(self): + with self.subTest(param='subtest:\u20ac'): + pass + + def test_skip(self): + self.skipTest('skipped:\u20ac') + """) + testname = self.create_test(code=code) + + env = dict(os.environ) + env['PYTHONIOENCODING'] = 'ascii' + + def check(output): + self.check_executed_tests(output, testname, stats=TestStats(3, 0, 1)) + self.assertIn(r'docstring:\u20ac', output) + self.assertIn(r'skipped:\u20ac', output) + + # Run sequentially + output = self.run_tests('-v', testname, env=env, isolated=False) + check(output) + + # Run in parallel + output = self.run_tests('-j1', '-v', testname, env=env, isolated=False) + check(output) + + def test_pgo_exclude(self): + # Get PGO tests + output = self.run_tests('--pgo', '--list-tests') + pgo_tests = output.strip().split() + + # Exclude test_re + output = self.run_tests('--pgo', '--list-tests', '-x', 'test_re') + tests = output.strip().split() + self.assertNotIn('test_re', tests) + self.assertEqual(len(tests), len(pgo_tests) - 1) + class TestUtils(unittest.TestCase): def test_format_duration(self): @@ -2347,20 +2465,20 @@ def test_format_resources(self): format_resources = utils.format_resources ALL_RESOURCES = utils.ALL_RESOURCES self.assertEqual( - format_resources(("network",)), + format_resources({"network": None}), 'resources (1): network') self.assertEqual( - format_resources(("audio", "decimal", "network")), + format_resources(dict.fromkeys(("audio", "decimal", "network"))), 'resources (3): audio,decimal,network') self.assertEqual( - format_resources(ALL_RESOURCES), + format_resources(dict.fromkeys(ALL_RESOURCES)), 'resources: all') self.assertEqual( - format_resources(tuple(name for name in ALL_RESOURCES - if name != "cpu")), + format_resources({name: None for name in ALL_RESOURCES + if name != "cpu"}), 'resources: all,-cpu') self.assertEqual( - format_resources((*ALL_RESOURCES, "tzdata")), + format_resources({**dict.fromkeys(ALL_RESOURCES), "tzdata": None}), 'resources: all,tzdata') def test_match_test(self): diff --git a/Lib/test/test_remote_pdb.py b/Lib/test/test_remote_pdb.py index aef8a6b01290922..a6130e2ff387bdc 100644 --- a/Lib/test/test_remote_pdb.py +++ b/Lib/test/test_remote_pdb.py @@ -1,5 +1,4 @@ import io -import time import itertools import json import os @@ -8,16 +7,13 @@ import socket import subprocess import sys -import tempfile import textwrap -import threading import unittest import unittest.mock from contextlib import closing, contextmanager, redirect_stdout, redirect_stderr, ExitStack -from pathlib import Path -from test.support import is_wasi, cpython_only, force_color, requires_subprocess, SHORT_TIMEOUT -from test.support.os_helper import temp_dir, TESTFN, unlink -from typing import Dict, List, Optional, Tuple, Union, Any +from test.support import is_wasi, cpython_only, force_color, requires_subprocess, SHORT_TIMEOUT, subTests +from test.support.os_helper import TESTFN, unlink +from typing import List import pdb from pdb import _PdbServer, _PdbClient @@ -283,37 +279,50 @@ def test_handling_other_message(self): expected_stdout="Some message.\n", ) - def test_handling_help_for_command(self): - """Test handling a request to display help for a command.""" + @unittest.skipIf(sys.flags.optimize >= 2, "Help not available for -OO") + @subTests( + "help_request,expected_substring", + [ + # a request to display help for a command + ({"help": "ll"}, "Usage: ll | longlist"), + # a request to display a help overview + ({"help": ""}, "type help <topic>"), + # a request to display the full PDB manual + ({"help": "pdb"}, ">>> import pdb"), + ], + ) + def test_handling_help_when_available(self, help_request, expected_substring): + """Test handling help requests when help is available.""" incoming = [ - ("server", {"help": "ll"}), + ("server", help_request), ] self.do_test( incoming=incoming, expected_outgoing=[], - expected_stdout_substring="Usage: ll | longlist", + expected_stdout_substring=expected_substring, ) - def test_handling_help_without_a_specific_topic(self): - """Test handling a request to display a help overview.""" + @unittest.skipIf(sys.flags.optimize < 2, "Needs -OO") + @subTests( + "help_request,expected_substring", + [ + # a request to display help for a command + ({"help": "ll"}, "No help for 'll'"), + # a request to display a help overview + ({"help": ""}, "Undocumented commands"), + # a request to display the full PDB manual + ({"help": "pdb"}, "No help for 'pdb'"), + ], + ) + def test_handling_help_when_not_available(self, help_request, expected_substring): + """Test handling help requests when help is not available.""" incoming = [ - ("server", {"help": ""}), + ("server", help_request), ] self.do_test( incoming=incoming, expected_outgoing=[], - expected_stdout_substring="type help <topic>", - ) - - def test_handling_help_pdb(self): - """Test handling a request to display the full PDB manual.""" - incoming = [ - ("server", {"help": "pdb"}), - ] - self.do_test( - incoming=incoming, - expected_outgoing=[], - expected_stdout_substring=">>> import pdb", + expected_stdout_substring=expected_substring, ) def test_handling_pdb_prompts(self): @@ -1432,9 +1441,36 @@ def test_multi_line_commands(self): self.assertIn("Function returned: 42", stdout) self.assertEqual(process.returncode, 0) + def test_exec_in_closure_result_uses_pdb_stdout(self): + """ + Expression results executed via _exec_in_closure() should be written + to the debugger output stream (pdb stdout), not to sys.stdout. + """ + self._create_script() + process, client_file = self._connect_and_get_client_file() + + with kill_on_error(process): + self._read_until_prompt(client_file) + + self._send_command(client_file, "(lambda: 123)()") + messages = self._read_until_prompt(client_file) + result_msg = "".join(msg.get("message", "") for msg in messages) + self.assertIn("123", result_msg) + + self._send_command(client_file, "sum(i for i in (1, 2, 3))") + messages = self._read_until_prompt(client_file) + result_msg = "".join(msg.get("message", "") for msg in messages) + self.assertIn("6", result_msg) + + self._send_command(client_file, "c") + stdout, _ = process.communicate(timeout=SHORT_TIMEOUT) + + self.assertNotIn("\n123\n", stdout) + self.assertNotIn("\n6\n", stdout) + self.assertEqual(process.returncode, 0) + def _supports_remote_attaching(): - from contextlib import suppress PROCESS_VM_READV_SUPPORTED = False try: @@ -1531,6 +1567,9 @@ def do_integration_test(self, client_stdin): redirect_stdout(client_stdout), redirect_stderr(client_stderr), unittest.mock.patch("sys.argv", ["pdb", "-p", str(process.pid)]), + unittest.mock.patch( + "pdb.exit_with_permission_help_text", side_effect=PermissionError + ), ): try: pdb.main() diff --git a/Lib/test/test_repl.py b/Lib/test/test_repl.py index 228b326699e75fd..9115e8cebabaca8 100644 --- a/Lib/test/test_repl.py +++ b/Lib/test/test_repl.py @@ -5,6 +5,8 @@ import subprocess import sys import unittest +from contextlib import contextmanager +from functools import partial from textwrap import dedent from test import support from test.support import ( @@ -15,7 +17,6 @@ SHORT_TIMEOUT, ) from test.support.script_helper import kill_python -from test.support.import_helper import import_module try: import pty @@ -27,7 +28,7 @@ raise unittest.SkipTest("test module requires subprocess") -def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw): +def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, custom=False, isolated=True, **kw): """Run the Python REPL with the given arguments. kw is extra keyword args to pass to subprocess.Popen. Returns a Popen @@ -41,7 +42,14 @@ def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw): # path may be used by Py_GetPath() to build the default module search # path. stdin_fname = os.path.join(os.path.dirname(sys.executable), "<stdin>") - cmd_line = [stdin_fname, '-I', '-i'] + cmd_line = [stdin_fname] + # Isolated mode implies -EPs and ignores PYTHON* variables. + if isolated: + cmd_line.append('-I') + # Don't re-run the built-in REPL from interactive mode + # if we're testing a custom REPL (such as the asyncio REPL). + if not custom: + cmd_line.append('-i') cmd_line.extend(args) # Set TERM=vt100, for the rationale see the comments in spawn_python() of @@ -55,6 +63,23 @@ def spawn_repl(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw): stdout=stdout, stderr=stderr, **kw) + +spawn_asyncio_repl = partial(spawn_repl, "-m", "asyncio", custom=True) + + +@contextmanager +def temp_pythonstartup(*, source: str, histfile: str = ".pythonhist"): + """Create environment variables for a PYTHONSTARTUP script in a temporary directory.""" + with os_helper.temp_dir() as tmpdir: + filename = os.path.join(tmpdir, "pythonstartup.py") + with open(filename, "w") as f: + f.write(source) + yield { + "PYTHONSTARTUP": filename, + "PYTHON_HISTORY": os.path.join(tmpdir, histfile) + } + + def run_on_interactive_mode(source): """Spawn a new Python interpreter, pass the given input source code from the stdin and return the @@ -73,12 +98,8 @@ def run_on_interactive_mode(source): @support.force_not_colorized_test_class class TestInteractiveInterpreter(unittest.TestCase): - @cpython_only - # Python built with Py_TRACE_REFS fail with a fatal error in - # _PyRefchain_Trace() on memory allocation error. - @unittest.skipIf(support.Py_TRACE_REFS, 'cannot test Py_TRACE_REFS build') + @support.nomemtest def test_no_memory(self): - import_module("_testcapi") # Issue #30696: Fix the interactive interpreter looping endlessly when # no memory. Check also that the fix does not break the interactive # loop when an exception is raised. @@ -131,6 +152,22 @@ def test_multiline_string_parsing(self): output = kill_python(p) self.assertEqual(p.returncode, 0) + @cpython_only + def test_lexer_buffer_realloc_with_null_start(self): + # gh-144759: NULL pointer arithmetic in the lexer when start and + # multi_line_start are NULL (uninitialized in tok_mode_stack[0]) + # and the lexer buffer is reallocated while parsing long input. + long_value = "a" * 2000 + user_input = dedent(f"""\ + x = f'{{{long_value!r}}}' + print(x) + """) + p = spawn_repl() + p.stdin.write(user_input) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + self.assertIn(long_value, output) + def test_close_stdin(self): user_input = dedent(''' import os @@ -188,6 +225,66 @@ def foo(x): ] self.assertEqual(traceback_lines, expected_lines) + def test_pythonstartup_error_reporting(self): + # errors based on https://github.com/python/cpython/issues/137576 + + def make_repl(env): + return subprocess.Popen( + [os.path.join(os.path.dirname(sys.executable), '<stdin>'), "-i"], + executable=sys.executable, + text=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env, + ) + + # case 1: error in user input, but PYTHONSTARTUP is fine + with os_helper.temp_dir() as tmpdir: + script = os.path.join(tmpdir, "pythonstartup.py") + with open(script, "w") as f: + f.write("print('from pythonstartup')\n") + + env = os.environ.copy() + env['PYTHONSTARTUP'] = script + env["PYTHON_HISTORY"] = os.path.join(tmpdir, ".pythonhist") + p = make_repl(env) + p.stdin.write("1/0") + output = kill_python(p) + expected = dedent(""" + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + 1/0 + ~^~ + ZeroDivisionError: division by zero + """) + self.assertIn("from pythonstartup", output) + self.assertIn(expected, output) + + # case 2: error in PYTHONSTARTUP triggered by user input + with os_helper.temp_dir() as tmpdir: + script = os.path.join(tmpdir, "pythonstartup.py") + with open(script, "w") as f: + f.write("def foo():\n 1/0\n") + + env = os.environ.copy() + env['PYTHONSTARTUP'] = script + env["PYTHON_HISTORY"] = os.path.join(tmpdir, ".pythonhist") + p = make_repl(env) + p.stdin.write('foo()') + output = kill_python(p) + expected = dedent(""" + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + foo() + ~~~^^ + File "%s", line 2, in foo + 1/0 + ~^~ + ZeroDivisionError: division by zero + """) % script + self.assertIn(expected, output) + def test_runsource_show_syntax_error_location(self): user_input = dedent("""def f(x, x): ... """) @@ -225,19 +322,27 @@ def test_asyncio_repl_reaches_python_startup_script(self): with os_helper.temp_dir() as tmpdir: script = os.path.join(tmpdir, "pythonstartup.py") with open(script, "w") as f: - f.write("print('pythonstartup done!')" + os.linesep) - f.write("exit(0)" + os.linesep) + f.write("print('pythonstartup done!')\n") + env = os.environ.copy() + env["PYTHON_HISTORY"] = os.path.join(tmpdir, ".asyncio_history") + env["PYTHONSTARTUP"] = script + p = spawn_asyncio_repl(isolated=False, env=env) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + self.assertIn("pythonstartup done!", output) + def test_asyncio_repl_respects_isolated_mode(self): + with os_helper.temp_dir() as tmpdir: + script = os.path.join(tmpdir, "pythonstartup.py") + with open(script, "w") as f: + f.write("print('should not print')\n") env = os.environ.copy() env["PYTHON_HISTORY"] = os.path.join(tmpdir, ".asyncio_history") env["PYTHONSTARTUP"] = script - subprocess.check_call( - [sys.executable, "-m", "asyncio"], - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env, - timeout=SHORT_TIMEOUT, - ) + p = spawn_asyncio_repl(isolated=True, env=env) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + self.assertNotIn("should not print", output) @unittest.skipUnless(pty, "requires pty") def test_asyncio_repl_is_ok(self): @@ -297,7 +402,7 @@ def f(): class TestAsyncioREPL(unittest.TestCase): def test_multiple_statements_fail_early(self): user_input = "1 / 0; print(f'afterwards: {1+1}')" - p = spawn_repl("-m", "asyncio") + p = spawn_asyncio_repl() p.stdin.write(user_input) output = kill_python(p) self.assertIn("ZeroDivisionError", output) @@ -309,7 +414,7 @@ def test_toplevel_contextvars_sync(self): var = ContextVar("var", default="failed") var.set("ok") """) - p = spawn_repl("-m", "asyncio") + p = spawn_asyncio_repl() p.stdin.write(user_input) user_input2 = dedent(""" print(f"toplevel contextvar test: {var.get()}") @@ -325,7 +430,7 @@ def test_toplevel_contextvars_async(self): from contextvars import ContextVar var = ContextVar('var', default='failed') """) - p = spawn_repl("-m", "asyncio") + p = spawn_asyncio_repl() p.stdin.write(user_input) user_input2 = "async def set_var(): var.set('ok')\n" p.stdin.write(user_input2) @@ -338,6 +443,39 @@ def test_toplevel_contextvars_async(self): expected = "toplevel contextvar test: ok" self.assertIn(expected, output, expected) + def test_quiet_mode(self): + p = spawn_repl("-q", "-m", "asyncio", custom=True) + output = kill_python(p) + self.assertEqual(p.returncode, 0) + self.assertEqual(output[:3], ">>>") + + @support.force_not_colorized + @support.subTests( + ("startup_code", "expected_error"), + [ + ("some invalid syntax\n", "SyntaxError: invalid syntax"), + ("1/0\n", "ZeroDivisionError: division by zero"), + ], + ) + def test_pythonstartup_failure(self, startup_code, expected_error): + startup_env = self.enterContext( + temp_pythonstartup(source=startup_code, histfile=".asyncio_history")) + + p = spawn_repl( + "-qm", "asyncio", + env=os.environ | startup_env, + isolated=False, + custom=True) + p.stdin.write("print('user code', 'executed')\n") + output = kill_python(p) + self.assertEqual(p.returncode, 0) + + tb_hint = f'File "{startup_env["PYTHONSTARTUP"]}", line 1' + self.assertIn(tb_hint, output) + self.assertIn(expected_error, output) + + self.assertIn("user code executed", output) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_reprlib.py b/Lib/test/test_reprlib.py index ffad35092f9916a..22a55b57c076ebd 100644 --- a/Lib/test/test_reprlib.py +++ b/Lib/test/test_reprlib.py @@ -151,14 +151,38 @@ def test_frozenset(self): eq(r(frozenset({1, 2, 3, 4, 5, 6, 7})), "frozenset({1, 2, 3, 4, 5, 6, ...})") def test_numbers(self): - eq = self.assertEqual - eq(r(123), repr(123)) - eq(r(123), repr(123)) - eq(r(1.0/3), repr(1.0/3)) - - n = 10**100 - expected = repr(n)[:18] + "..." + repr(n)[-19:] - eq(r(n), expected) + for x in [123, 1.0 / 3]: + self.assertEqual(r(x), repr(x)) + + max_digits = sys.get_int_max_str_digits() + for k in [100, max_digits - 1]: + with self.subTest(f'10 ** {k}', k=k): + n = 10 ** k + expected = repr(n)[:18] + "..." + repr(n)[-19:] + self.assertEqual(r(n), expected) + + def re_msg(n, d): + return (rf'<{n.__class__.__name__} instance with roughly {d} ' + rf'digits \(limit at {max_digits}\) at 0x[a-f0-9]+>') + + k = max_digits + with self.subTest(f'10 ** {k}', k=k): + n = 10 ** k + self.assertRaises(ValueError, repr, n) + self.assertRegex(r(n), re_msg(n, k + 1)) + + for k in [max_digits + 1, 2 * max_digits]: + self.assertGreater(k, 100) + with self.subTest(f'10 ** {k}', k=k): + n = 10 ** k + self.assertRaises(ValueError, repr, n) + self.assertRegex(r(n), re_msg(n, k + 1)) + with self.subTest(f'10 ** {k} - 1', k=k): + n = 10 ** k - 1 + # Here, since math.log10(n) == math.log10(n-1), + # the number of digits of n - 1 is overestimated. + self.assertRaises(ValueError, repr, n) + self.assertRegex(r(n), re_msg(n, k + 1)) def test_instance(self): eq = self.assertEqual @@ -173,13 +197,13 @@ def test_instance(self): eq(r(i3), ("<ClassWithFailingRepr instance at %#x>"%id(i3))) s = r(ClassWithFailingRepr) - self.assertTrue(s.startswith("<class ")) - self.assertTrue(s.endswith(">")) + self.assertStartsWith(s, "<class ") + self.assertEndsWith(s, ">") self.assertIn(s.find("..."), [12, 13]) def test_lambda(self): r = repr(lambda x: x) - self.assertTrue(r.startswith("<function ReprTests.test_lambda.<locals>.<lambda"), r) + self.assertStartsWith(r, "<function ReprTests.test_lambda.<locals>.<lambda") # XXX anonymous functions? see func_repr def test_builtin_function(self): @@ -187,8 +211,8 @@ def test_builtin_function(self): # Functions eq(repr(hash), '<built-in function hash>') # Methods - self.assertTrue(repr(''.split).startswith( - '<built-in method split of str object at 0x')) + self.assertStartsWith(repr(''.split), + '<built-in method split of str object at 0x') def test_range(self): eq = self.assertEqual @@ -373,20 +397,20 @@ def test_valid_indent(self): 'object': { 1: 'two', b'three': [ - (4.5, 6.7), + (4.5, 6.25), [set((8, 9)), frozenset((10, 11))], ], }, 'tests': ( (dict(indent=None), '''\ - {1: 'two', b'three': [(4.5, 6.7), [{8, 9}, frozenset({10, 11})]]}'''), + {1: 'two', b'three': [(4.5, 6.25), [{8, 9}, frozenset({10, 11})]]}'''), (dict(indent=False), '''\ { 1: 'two', b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -406,7 +430,7 @@ def test_valid_indent(self): b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -426,7 +450,7 @@ def test_valid_indent(self): b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -446,7 +470,7 @@ def test_valid_indent(self): b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -466,7 +490,7 @@ def test_valid_indent(self): b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -494,7 +518,7 @@ def test_valid_indent(self): b'three': [ ( 4.5, - 6.7, + 6.25, ), [ { @@ -514,7 +538,7 @@ def test_valid_indent(self): -->b'three': [ -->-->( -->-->-->4.5, - -->-->-->6.7, + -->-->-->6.25, -->-->), -->-->[ -->-->-->{ @@ -534,7 +558,7 @@ def test_valid_indent(self): ....b'three': [ ........( ............4.5, - ............6.7, + ............6.25, ........), ........[ ............{ @@ -730,8 +754,8 @@ class baz: importlib.invalidate_caches() from areallylongpackageandmodulenametotestreprtruncation.areallylongpackageandmodulenametotestreprtruncation import baz ibaz = baz.baz() - self.assertTrue(repr(ibaz).startswith( - "<%s.baz object at 0x" % baz.__name__)) + self.assertStartsWith(repr(ibaz), + "<%s.baz object at 0x" % baz.__name__) def test_method(self): self._check_path_limitations('qux') @@ -744,13 +768,13 @@ def amethod(self): pass from areallylongpackageandmodulenametotestreprtruncation.areallylongpackageandmodulenametotestreprtruncation import qux # Unbound methods first r = repr(qux.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.amethod) - self.assertTrue(r.startswith('<function aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.amethod'), r) + self.assertStartsWith(r, '<function aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.amethod') # Bound method next iqux = qux.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa() r = repr(iqux.amethod) - self.assertTrue(r.startswith( + self.assertStartsWith(r, '<bound method aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.amethod of <%s.aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa object at 0x' \ - % (qux.__name__,) ), r) + % (qux.__name__,) ) @unittest.skip('needs a built-in function with a really long name') def test_builtin_function(self): diff --git a/Lib/test/test_resource.py b/Lib/test/test_resource.py index d23d3623235f389..fe05224828bd278 100644 --- a/Lib/test/test_resource.py +++ b/Lib/test/test_resource.py @@ -14,89 +14,154 @@ class ResourceTest(unittest.TestCase): def test_args(self): self.assertRaises(TypeError, resource.getrlimit) - self.assertRaises(TypeError, resource.getrlimit, 42, 42) + self.assertRaises(TypeError, resource.getrlimit, 0, 42) + self.assertRaises(OverflowError, resource.getrlimit, 2**1000) + self.assertRaises(OverflowError, resource.getrlimit, -2**1000) + self.assertRaises(TypeError, resource.getrlimit, '0') self.assertRaises(TypeError, resource.setrlimit) - self.assertRaises(TypeError, resource.setrlimit, 42, 42, 42) + self.assertRaises(TypeError, resource.setrlimit, 0) + self.assertRaises(TypeError, resource.setrlimit, 0, 42) + self.assertRaises(TypeError, resource.setrlimit, 0, 42, 42) + self.assertRaises(OverflowError, resource.setrlimit, 2**1000, (42, 42)) + self.assertRaises(OverflowError, resource.setrlimit, -2**1000, (42, 42)) + self.assertRaises(ValueError, resource.setrlimit, 0, (42,)) + self.assertRaises(ValueError, resource.setrlimit, 0, (42, 42, 42)) + self.assertRaises(TypeError, resource.setrlimit, '0', (42, 42)) + self.assertRaises(TypeError, resource.setrlimit, 0, ('42', 42)) + self.assertRaises(TypeError, resource.setrlimit, 0, (42, '42')) @unittest.skipIf(sys.platform == "vxworks", "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') def test_fsize_ismax(self): - try: - (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) - except AttributeError: - pass - else: - # RLIMIT_FSIZE should be RLIM_INFINITY, which will be a really big - # number on a platform with large file support. On these platforms, - # we need to test that the get/setrlimit functions properly convert - # the number to a C long long and that the conversion doesn't raise - # an error. - self.assertEqual(resource.RLIM_INFINITY, max) - resource.setrlimit(resource.RLIMIT_FSIZE, (cur, max)) + (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) + # RLIMIT_FSIZE should be RLIM_INFINITY, which will be a really big + # number on a platform with large file support. On these platforms, + # we need to test that the get/setrlimit functions properly convert + # the number to a C long long and that the conversion doesn't raise + # an error. + self.assertEqual(resource.RLIM_INFINITY, max) + resource.setrlimit(resource.RLIMIT_FSIZE, (cur, max)) + @unittest.skipIf(sys.platform == "vxworks", + "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') def test_fsize_enforced(self): + (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) + # Check to see what happens when the RLIMIT_FSIZE is small. Some + # versions of Python were terminated by an uncaught SIGXFSZ, but + # pythonrun.c has been fixed to ignore that exception. If so, the + # write() should return EFBIG when the limit is exceeded. + + # At least one platform has an unlimited RLIMIT_FSIZE and attempts + # to change it raise ValueError instead. try: - (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) - except AttributeError: - pass - else: - # Check to see what happens when the RLIMIT_FSIZE is small. Some - # versions of Python were terminated by an uncaught SIGXFSZ, but - # pythonrun.c has been fixed to ignore that exception. If so, the - # write() should return EFBIG when the limit is exceeded. - - # At least one platform has an unlimited RLIMIT_FSIZE and attempts - # to change it raise ValueError instead. try: + resource.setrlimit(resource.RLIMIT_FSIZE, (1024, max)) + limit_set = True + except ValueError: + limit_set = False + f = open(os_helper.TESTFN, "wb") + try: + f.write(b"X" * 1024) try: - resource.setrlimit(resource.RLIMIT_FSIZE, (1024, max)) - limit_set = True - except ValueError: - limit_set = False - f = open(os_helper.TESTFN, "wb") - try: - f.write(b"X" * 1024) - try: - f.write(b"Y") + f.write(b"Y") + f.flush() + # On some systems (e.g., Ubuntu on hppa) the flush() + # doesn't always cause the exception, but the close() + # does eventually. Try flushing several times in + # an attempt to ensure the file is really synced and + # the exception raised. + for i in range(5): + time.sleep(.1) f.flush() - # On some systems (e.g., Ubuntu on hppa) the flush() - # doesn't always cause the exception, but the close() - # does eventually. Try flushing several times in - # an attempt to ensure the file is really synced and - # the exception raised. - for i in range(5): - time.sleep(.1) - f.flush() - except OSError: - if not limit_set: - raise - if limit_set: - # Close will attempt to flush the byte we wrote - # Restore limit first to avoid getting a spurious error - resource.setrlimit(resource.RLIMIT_FSIZE, (cur, max)) - finally: - f.close() - finally: + except OSError: + if not limit_set: + raise if limit_set: + # Close will attempt to flush the byte we wrote + # Restore limit first to avoid getting a spurious error resource.setrlimit(resource.RLIMIT_FSIZE, (cur, max)) - os_helper.unlink(os_helper.TESTFN) + finally: + f.close() + finally: + if limit_set: + resource.setrlimit(resource.RLIMIT_FSIZE, (cur, max)) + os_helper.unlink(os_helper.TESTFN) - def test_fsize_toobig(self): + @unittest.skipIf(sys.platform == "vxworks", + "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') + def test_fsize_too_big(self): # Be sure that setrlimit is checking for really large values too_big = 10**50 + (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) + try: + resource.setrlimit(resource.RLIMIT_FSIZE, (too_big, max)) + except (OverflowError, ValueError): + pass try: - (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) - except AttributeError: + resource.setrlimit(resource.RLIMIT_FSIZE, (max, too_big)) + except (OverflowError, ValueError): pass + + @unittest.skipIf(sys.platform == "vxworks", + "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') + def test_fsize_not_too_big(self): + (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) + self.addCleanup(resource.setrlimit, resource.RLIMIT_FSIZE, (cur, max)) + + def expected(cur): + if resource.RLIM_INFINITY < 0: + return [(cur, max), (resource.RLIM_INFINITY, max)] + elif resource.RLIM_INFINITY < cur: + return [(resource.RLIM_INFINITY, max)] + else: + return [(cur, max)] + + resource.setrlimit(resource.RLIMIT_FSIZE, (2**31-5, max)) + self.assertEqual(resource.getrlimit(resource.RLIMIT_FSIZE), (2**31-5, max)) + + try: + resource.setrlimit(resource.RLIMIT_FSIZE, (2**32, max)) + except OverflowError: + resource.setrlimit(resource.RLIMIT_FSIZE, (2**31, max)) + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**31)) + resource.setrlimit(resource.RLIMIT_FSIZE, (2**32-5, max)) + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**32-5)) else: + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**32)) + resource.setrlimit(resource.RLIMIT_FSIZE, (2**31, max)) + self.assertEqual(resource.getrlimit(resource.RLIMIT_FSIZE), (2**31, max)) + resource.setrlimit(resource.RLIMIT_FSIZE, (2**32-5, max)) + self.assertEqual(resource.getrlimit(resource.RLIMIT_FSIZE), (2**32-5, max)) + + resource.setrlimit(resource.RLIMIT_FSIZE, (2**63-5, max)) + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**63-5)) try: - resource.setrlimit(resource.RLIMIT_FSIZE, (too_big, max)) - except (OverflowError, ValueError): - pass - try: - resource.setrlimit(resource.RLIMIT_FSIZE, (max, too_big)) - except (OverflowError, ValueError): + resource.setrlimit(resource.RLIMIT_FSIZE, (2**63, max)) + except ValueError: + # There is a hard limit on macOS. pass + else: + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**63)) + resource.setrlimit(resource.RLIMIT_FSIZE, (2**64-5, max)) + self.assertIn(resource.getrlimit(resource.RLIMIT_FSIZE), expected(2**64-5)) + + @unittest.skipIf(sys.platform == "vxworks", + "setting RLIMIT_FSIZE is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_FSIZE'), 'requires resource.RLIMIT_FSIZE') + def test_fsize_negative(self): + (cur, max) = resource.getrlimit(resource.RLIMIT_FSIZE) + for value in -5, -2**31, -2**32-5, -2**63, -2**64-5, -2**1000: + with self.subTest(value=value): + # This test assumes that the values don't map to RLIM_INFINITY, + # though Posix doesn't guarantee it. + self.assertNotEqual(value, resource.RLIM_INFINITY) + + self.assertRaises(ValueError, resource.setrlimit, resource.RLIMIT_FSIZE, (value, max)) + self.assertRaises(ValueError, resource.setrlimit, resource.RLIMIT_FSIZE, (cur, value)) @unittest.skipUnless(hasattr(resource, "getrusage"), "needs getrusage") def test_getrusage(self): @@ -117,21 +182,18 @@ def test_getrusage(self): # Issue 6083: Reference counting bug @unittest.skipIf(sys.platform == "vxworks", "setting RLIMIT_CPU is not supported on VxWorks") + @unittest.skipUnless(hasattr(resource, 'RLIMIT_CPU'), 'requires resource.RLIMIT_CPU') def test_setrusage_refcount(self): - try: - limits = resource.getrlimit(resource.RLIMIT_CPU) - except AttributeError: - pass - else: - class BadSequence: - def __len__(self): - return 2 - def __getitem__(self, key): - if key in (0, 1): - return len(tuple(range(1000000))) - raise IndexError + limits = resource.getrlimit(resource.RLIMIT_CPU) + class BadSequence: + def __len__(self): + return 2 + def __getitem__(self, key): + if key in (0, 1): + return len(tuple(range(1000000))) + raise IndexError - resource.setrlimit(resource.RLIMIT_CPU, BadSequence()) + resource.setrlimit(resource.RLIMIT_CPU, BadSequence()) def test_pagesize(self): pagesize = resource.getpagesize() @@ -168,7 +230,8 @@ class BadSeq: def __len__(self): return 2 def __getitem__(self, key): - return limits[key] - 1 # new reference + lim = limits[key] + return lim - 1 if lim > 0 else lim + sys.maxsize*2 # new reference limits = resource.getrlimit(resource.RLIMIT_AS) self.assertEqual(resource.prlimit(0, resource.RLIMIT_AS, BadSeq()), diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index d403a0fe96be3dc..e6d727d417b2985 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -1,6 +1,7 @@ import unittest from unittest.mock import patch import builtins +import types import rlcompleter from test.support import MISSING_C_DOCSTRINGS @@ -88,7 +89,7 @@ def create_expected_for_none(): ['CompleteMe._ham']) matches = self.completer.attr_matches('CompleteMe.__') for x in matches: - self.assertTrue(x.startswith('CompleteMe.__'), x) + self.assertStartsWith(x, 'CompleteMe.__') self.assertIn('CompleteMe.__name__', matches) self.assertIn('CompleteMe.__new__(', matches) @@ -135,6 +136,57 @@ def bar(self): self.assertEqual(completer.complete('f.b', 0), 'f.bar') self.assertFalse(f.property_called) + def test_released_memoryview_completion_works(self): + mv = memoryview(b"abc") + mv.release() + + self.assertIsInstance(type(mv).shape, types.GetSetDescriptorType) + self.assertIsInstance(type(mv).strides, types.GetSetDescriptorType) + + completer = rlcompleter.Completer(dict(mv=mv)) + matches = completer.attr_matches('mv.') + + # These are getset descriptors on memoryview and should be completed + # without evaluating the released-memoryview getters. + self.assertIn('mv.shape', matches) + self.assertIn('mv.strides', matches) + + def test_member_descriptor_not_evaluated(self): + class Foo: + __slots__ = ("boom",) + boom_accesses = 0 + + def __getattribute__(self, name): + if name == "boom": + type(self).boom_accesses += 1 + raise RuntimeError("boom access should be skipped") + return super().__getattribute__(name) + + self.assertIsInstance(Foo.boom, types.MemberDescriptorType) + + completer = rlcompleter.Completer(dict(f=Foo())) + matches = completer.attr_matches('f.') + self.assertIn('f.boom', matches) + self.assertEqual(Foo.boom_accesses, 0) + + def test_raising_descriptor_completion_works(self): + class ExplodingDescriptor: + def __init__(self): + self.instance_get_calls = 0 + + def __get__(self, obj, owner): + if obj is None: + return self + self.instance_get_calls += 1 + raise RuntimeError("descriptor getter exploded") + + class Foo: + boom = ExplodingDescriptor() + + completer = rlcompleter.Completer(dict(f=Foo())) + matches = completer.attr_matches('f.') + self.assertIn('f.boom', matches) + self.assertEqual(Foo.boom.instance_get_calls, 0) def test_uncreated_attr(self): # Attributes like properties and slots should be completed even when diff --git a/Lib/test/test_robotparser.py b/Lib/test/test_robotparser.py index 8d89e2a82244525..1ec64da064d52b7 100644 --- a/Lib/test/test_robotparser.py +++ b/Lib/test/test_robotparser.py @@ -15,6 +15,18 @@ class BaseRobotTest: good = [] bad = [] site_maps = None + expected_output = None + + def __init_subclass__(cls): + super().__init_subclass__() + # Remove tests that do nothing. + if issubclass(cls, unittest.TestCase): + if not cls.good: + cls.test_good_urls = None + if not cls.bad: + cls.test_bad_urls = None + if cls.expected_output is None: + cls.test_string_formatting = None def setUp(self): lines = io.StringIO(self.robots_txt).readlines() @@ -42,6 +54,8 @@ def test_bad_urls(self): def test_site_maps(self): self.assertEqual(self.parser.site_maps(), self.site_maps) + def test_string_formatting(self): + self.assertEqual(str(self.parser), self.expected_output) class UserAgentWildcardTest(BaseRobotTest, unittest.TestCase): robots_txt = """\ @@ -53,6 +67,56 @@ class UserAgentWildcardTest(BaseRobotTest, unittest.TestCase): good = ['/', '/test.html'] bad = ['/cyberworld/map/index.html', '/tmp/xxx', '/foo.html'] +class SimpleExampleTest(BaseRobotTest, unittest.TestCase): + # Example from RFC 9309, section 5.1. + robots_txt = """\ +User-Agent: * +Disallow: *.gif$ +Disallow: /example/ +Allow: /publications/ + +User-Agent: foobot +Disallow:/ +Allow:/example/page.html +Allow:/example/allowed.gif + +User-Agent: barbot +User-Agent: bazbot +Disallow: /example/page.html + +User-Agent: quxbot + """ + good = [ + '/', '/publications/', + ('foobot', '/example/page.html'), ('foobot', '/example/allowed.gif'), + ('barbot', '/'), ('barbot', '/example/'), + ('barbot', '/example/allowed.gif'), + ('barbot', '/example/disallowed.gif'), + ('barbot', '/publications/'), + ('barbot', '/publications/allowed.gif'), + ('bazbot', '/'), ('bazbot', '/example/'), + ('bazbot', '/example/allowed.gif'), + ('bazbot', '/example/disallowed.gif'), + ('bazbot', '/publications/'), + ('bazbot', '/publications/allowed.gif'), + ('quxbot', '/'), ('quxbot', '/example/'), + ('quxbot', '/example/page.html'), ('quxbot', '/example/allowed.gif'), + ('quxbot', '/example/disallowed.gif'), + ('quxbot', '/publications/'), + ('quxbot', '/publications/allowed.gif'), + ] + bad = [ + '/example/', '/example/page.html', '/example/allowed.gif', + '/example/disallowed.gif', + '/publications/allowed.gif', + ('foobot', '/'), ('foobot', '/example/'), + ('foobot', '/example/disallowed.gif'), + ('foobot', '/publications/'), + ('foobot', '/publications/allowed.gif'), + ('barbot', '/example/page.html'), + ('bazbot', '/example/page.html'), + ] + class CrawlDelayAndCustomAgentTest(BaseRobotTest, unittest.TestCase): robots_txt = """\ @@ -94,7 +158,7 @@ class RejectAllRobotsTest(BaseRobotTest, unittest.TestCase): User-agent: * Disallow: / """ - good = [] + good = ['/robots.txt'] bad = ['/cyberworld/map/index.html', '/', '/tmp/'] @@ -124,11 +188,14 @@ def test_request_rate(self): parsed_request_rate.seconds, self.request_rate.seconds ) + else: + self.assertIsNone(parsed_request_rate) class EmptyFileTest(BaseRequestRateTest, unittest.TestCase): robots_txt = '' good = ['/foo'] + expected_output = '' class CrawlDelayAndRequestRateTest(BaseRequestRateTest, unittest.TestCase): @@ -181,6 +248,32 @@ class InvalidCrawlDelayTest(BaseRobotTest, unittest.TestCase): bad = [] +class NonDecimalDigitsTest(BaseRequestRateTest, unittest.TestCase): + # Non-decimal Unicode digits pass str.isdigit() but int() rejects + # them, so the directive must be silently ignored, not raise. + robots_txt = """\ +User-Agent: * +Disallow: /tmp/ +Crawl-delay: ² +Request-rate: ²/5 + """ + good = ['/foo.html'] + bad = ['/tmp/'] + crawl_delay = None + request_rate = None + + +class NonDecimalDenominatorTest(BaseRequestRateTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /tmp/ +Request-rate: 5/² + """ + good = ['/foo.html'] + request_rate = None + bad = ['/tmp/'] + + class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase): # also test that Allow and Diasallow works well with each other robots_txt = """\ @@ -195,69 +288,302 @@ class AnotherInvalidRequestRateTest(BaseRobotTest, unittest.TestCase): class UserAgentOrderingTest(BaseRobotTest, unittest.TestCase): - # the order of User-agent should be correct. note - # that this file is incorrect because "Googlebot" is a - # substring of "Googlebot-Mobile" + # the order of User-agent should not matter robots_txt = """\ User-agent: Googlebot Disallow: / +Allow: /folder1/ User-agent: Googlebot-Mobile Allow: / +Disallow: /folder1/ """ agent = 'Googlebot' bad = ['/something.jpg'] + good = ['/folder1/myfile.html'] class UserAgentGoogleMobileTest(UserAgentOrderingTest): - agent = 'Googlebot-Mobile' + agent = 'Googlebot-mobile' + bad = ['/folder1/myfile.html'] + good = ['/something.jpg'] -class GoogleURLOrderingTest(BaseRobotTest, unittest.TestCase): - # Google also got the order wrong. You need - # to specify the URLs from more specific to more general +class LongestMatchTest(BaseRobotTest, unittest.TestCase): + # Based on example from RFC 9309, section 5.2. robots_txt = """\ -User-agent: Googlebot -Allow: /folder1/myfile.html -Disallow: /folder1/ +User-agent: * +Allow: /example/page/ +Disallow: /example/page/disallowed.gif +Allow: /example/ """ - agent = 'googlebot' - good = ['/folder1/myfile.html'] - bad = ['/folder1/anotherfile.html'] + good = ['/example/', '/example/page/'] + bad = ['/example/page/disallowed.gif'] -class DisallowQueryStringTest(BaseRobotTest, unittest.TestCase): - # see issue #6325 for details +class LongestMatchWildcardTest(BaseRobotTest, unittest.TestCase): robots_txt = """\ User-agent: * -Disallow: /some/path?name=value +Allow: /example/page/ +Disallow: *.gif +Allow: /example/ + """ + good = ['/example/', '/example/page/'] + bad = ['/example/page/disallowed.gif', '/x.gif'] + + +class AllowWinsEqualMatchTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /spam +Allow: /spam +Disallow: /spam """ - good = ['/some/path'] - bad = ['/some/path?name=value'] + good = ['/spam', '/spam/'] -class UseFirstUserAgentWildcardTest(BaseRobotTest, unittest.TestCase): - # obey first * entry (#4108) +class AllowWinsEqualFullMatchTest(BaseRobotTest, unittest.TestCase): robots_txt = """\ User-agent: * +Disallow: /spam +Allow: /spam$ +Disallow: /spam +Disallow: /eggs$ +Allow: /eggs +Disallow: /eggs$ + """ + good = ['/spam', '/eggs', '/eggs/'] + bad = ['/spam/'] + + +class AllowWinsEqualMatchWildcardTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /spam +Allow: *am +Disallow: /spam +Disallow: *gs +Allow: /eggs +Disallow: *gs + """ + good = ['/spam', '/eggs', '/spam/', '/eggs/'] + + +class MergeGroupsTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: spambot +Disallow: /some/path + +User-agent: spambot +Disallow: /another/path + """ + agent = 'spambot' + bad = ['/some/path', '/another/path'] + + +class UserAgentStartsGroupTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: spambot +Disallow: /some/path +User-agent: eggsbot +Disallow: /another/path + """ + good = [('spambot', '/'), ('spambot', '/another/path'), + ('eggsbot', '/'), ('eggsbot', '/some/path')] + bad = [('spambot', '/some/path'), ('eggsbot', '/another/path')] + expected_output = """\ +User-agent: spambot +Disallow: /some/path + +User-agent: eggsbot +Disallow: /another/path\ +""" + +class IgnoreEmptyLinesTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: spambot + +User-agent: eggsbot +Disallow: /some/path + +Disallow: /another/path + """ + good = [('spambot', '/'), ('eggsbot', '/')] + bad = [ + ('spambot', '/some/path'), ('spambot', '/another/path'), + ('eggsbot', '/some/path'), ('eggsbot', '/another/path'), + ] + expected_output = """\ +User-agent: spambot +User-agent: eggsbot +Disallow: /some/path +Disallow: /another/path\ +""" + + +class IgnoreRulesWithoutUserAgentTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ Disallow: /some/path User-agent: * Disallow: /another/path """ - good = ['/another/path'] - bad = ['/some/path'] + good = ['/', '/some/path'] + bad = ['/another/path'] + expected_output = """\ +User-agent: * +Disallow: /another/path\ +""" + + +class EmptyGroupTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /some/path + +User-agent: spambot + """ + agent = 'spambot' + good = ['/', '/some/path'] + expected_output = """\ +User-agent: * +Disallow: /some/path + +User-agent: spambot +Allow:\ +""" + + +class WeirdPathTest(BaseRobotTest, unittest.TestCase): + robots_txt = f"""\ +User-agent: * +Disallow: /a$$$ +Disallow: /b$z +Disallow: /c*** +Disallow: /d***z +Disallow: /e*$**$$ +Disallow: /f*$**$$z +Disallow: /g$*$$** +Disallow: /h$*$$**z + """ + good = ['/ax', '/a$$', '/b', '/bz', '/b$z', '/d', '/f', '/fz', + '/f$$$z', '/fx$y$$z', '/gx', '/g$$$', '/g$x$$y', '/h', '/hz', + '/h$$$z', '/h$x$$yz'] + bad = ['/a', '/c', '/cxy', '/dz', '/dxyz', '/dxzy', '/e', '/exy', + '/e$$', '/ex$y$', '/g'] + expected_output = """\ +User-agent: * +Disallow: /a$ +Disallow: /c* +Disallow: /d*z +Disallow: /e*$ +Disallow: /g$\ +""" -class EmptyQueryStringTest(BaseRobotTest, unittest.TestCase): - # normalize the URL first (#17403) +class PathWithManyWildcardsTest(BaseRobotTest, unittest.TestCase): + # This test would take many years if use naive translation to regular + # expression (* -> .*). + N = 50 + robots_txt = f"""\ +User-agent: * +Disallow: /{'*a'*N}*b + """ + good = ['/' + 'a'*N + 'a'] + bad = ['/' + 'a'*N + 'b'] + + +class DisallowQueryStringTest(BaseRobotTest, unittest.TestCase): + # see issue #6325 for details robots_txt = """\ User-agent: * -Allow: /some/path? +Disallow: /some/path?name=value Disallow: /another/path? +Disallow: /yet/one/path?name=value&more """ - good = ['/some/path?'] - bad = ['/another/path?'] + good = ['/some/path', '/some/path?', + '/some/path%3Fname=value', '/some/path?name%3Dvalue', + '/another/path', '/another/path%3F', + '/yet/one/path?name=value%26more', + '/some/pathxname=value'] + bad = ['/some/path?name=value' + '/another/path?', '/another/path?name=value', + '/yet/one/path?name=value&more'] + + +class PercentEncodingTest(BaseRobotTest, unittest.TestCase): + robots_txt = """\ +User-agent: * +Disallow: /a1/Z-._~ # unreserved characters +Disallow: /a2/%5A%2D%2E%5F%7E # percent-encoded unreserved characters +Disallow: /u1/%F0%9F%90%8D # percent-encoded ASCII Unicode character +Disallow: /u2/%f0%9f%90%8d +Disallow: /u3/\U0001f40d # raw non-ASCII Unicode character +Disallow: /v1/%F0 # percent-encoded non-ASCII octet +Disallow: /v2/%f0 +Disallow: /v3/\udcf0 # raw non-ASCII octet +Disallow: /p1%xy # raw percent +Disallow: /p2% +Disallow: /p3%25xy # percent-encoded percent +Disallow: /p4%2525xy # double percent-encoded percent +Disallow: /john%20smith # space +Disallow: /john doe +Disallow: /trailingspace%20 +Disallow: /question%3Fq=v # not query +Disallow: /hash%23f # not fragment +Disallow: /dollar%24 +Disallow: /asterisk%2A +Disallow: /sub/dir +Disallow: /slash%2F +Disallow: /query/question?q=%3F +Disallow: /query/raw/question?q=? +Disallow: /query/eq?q%3Dv +Disallow: /query/amp?q=v%26a +""" + good = [ + '/u1/%F0', '/u1/%f0', + '/u2/%F0', '/u2/%f0', + '/u3/%F0', '/u3/%f0', + '/p1%2525xy', '/p2%f0', '/p3%2525xy', '/p4%xy', '/p4%25xy', + '/question?q=v', + '/dollar', '/asterisk', + '/query/eq?q=v', + '/query/amp?q=v&a', + ] + bad = [ + '/a1/Z-._~', '/a1/%5A%2D%2E%5F%7E', + '/a2/Z-._~', '/a2/%5A%2D%2E%5F%7E', + '/u1/%F0%9F%90%8D', '/u1/%f0%9f%90%8d', '/u1/\U0001f40d', + '/u2/%F0%9F%90%8D', '/u2/%f0%9f%90%8d', '/u2/\U0001f40d', + '/u3/%F0%9F%90%8D', '/u3/%f0%9f%90%8d', '/u3/\U0001f40d', + '/v1/%F0', '/v1/%f0', '/v1/\udcf0', '/v1/\U0001f40d', + '/v2/%F0', '/v2/%f0', '/v2/\udcf0', '/v2/\U0001f40d', + '/v3/%F0', '/v3/%f0', '/v3/\udcf0', '/v3/\U0001f40d', + '/p1%xy', '/p1%25xy', + '/p2%', '/p2%25', '/p2%2525', '/p2%xy', + '/p3%xy', '/p3%25xy', + '/p4%2525xy', + '/john%20smith', '/john smith', + '/john%20doe', '/john doe', + '/trailingspace%20', '/trailingspace ', + '/question%3Fq=v', + '/hash#f', '/hash%23f', + '/dollar$', '/dollar%24', + '/asterisk*', '/asterisk%2A', + '/sub/dir', '/sub%2Fdir', + '/slash%2F', '/slash/', + '/query/question?q=?', '/query/question?q=%3F', + '/query/raw/question?q=?', '/query/raw/question?q=%3F', + '/query/eq?q%3Dv', + '/query/amp?q=v%26a', + ] + # other reserved characters + for c in ":/#[]@!$&'()*+,;=": + robots_txt += f'Disallow: /raw{c}\nDisallow: /pc%{ord(c):02X}\n' + bad.append(f'/raw{c}') + bad.append(f'/raw%{ord(c):02X}') + bad.append(f'/pc{c}') + bad.append(f'/pc%{ord(c):02X}') class DefaultEntryTest(BaseRequestRateTest, unittest.TestCase): @@ -286,64 +612,193 @@ class StringFormattingTest(BaseRobotTest, unittest.TestCase): """ expected_output = """\ -User-agent: cybermapper -Disallow: /some/path - User-agent: * Crawl-delay: 1 Request-rate: 3/15 -Disallow: /cyberworld/map/\ -""" - - def test_string_formatting(self): - self.assertEqual(str(self.parser), self.expected_output) +Disallow: /cyberworld/map/ +User-agent: cybermapper +Disallow: /some/path\ +""" -class RobotHandler(BaseHTTPRequestHandler): - def do_GET(self): - self.send_error(403, "Forbidden access") +class ConstructedStringFormattingTest(unittest.TestCase): + def test_empty(self): + parser = urllib.robotparser.RobotFileParser() + self.assertEqual(str(parser), '') - def log_message(self, format, *args): - pass + def test_group_without_rules(self): + parser = urllib.robotparser.RobotFileParser() + entry = urllib.robotparser.Entry() + entry.useragents = ['spambot'] + parser._add_entry(entry) + entry = urllib.robotparser.Entry() + entry.useragents = ['hambot'] + entry.rulelines = [urllib.robotparser.RuleLine('/ham', False)] + parser._add_entry(entry) + entry = urllib.robotparser.Entry() + entry.useragents = ['eggsbot'] + parser._add_entry(entry) + self.assertEqual(str(parser), """\ +User-agent: spambot +Allow: + +User-agent: hambot +Disallow: /ham + +User-agent: eggsbot +Allow:\ +""") + + def test_group_without_user_agent(self): + parser = urllib.robotparser.RobotFileParser() + entry = urllib.robotparser.Entry() + entry.rulelines = [urllib.robotparser.RuleLine('/ham', False)] + parser._add_entry(entry) + entry = urllib.robotparser.Entry() + entry.useragents = ['spambot'] + entry.rulelines = [urllib.robotparser.RuleLine('/spam', False)] + parser._add_entry(entry) + entry = urllib.robotparser.Entry() + entry.rulelines = [urllib.robotparser.RuleLine('/eggs', False)] + parser._add_entry(entry) + self.assertEqual(str(parser), """\ +User-agent: spambot +Disallow: /spam\ +""") @unittest.skipUnless( support.has_socket_support, "Socket server requires working socket." ) -class PasswordProtectedSiteTestCase(unittest.TestCase): +class BaseLocalNetworkTestCase: - def setUp(self): + @classmethod + def setUpClass(cls): # clear _opener global variable - self.addCleanup(urllib.request.urlcleanup) + cls.addClassCleanup(urllib.request.urlcleanup) - self.server = HTTPServer((socket_helper.HOST, 0), RobotHandler) + cls.server = HTTPServer((socket_helper.HOST, 0), cls.RobotHandler) + cls.addClassCleanup(cls.server.server_close) - self.t = threading.Thread( + t = threading.Thread( name='HTTPServer serving', - target=self.server.serve_forever, + target=cls.server.serve_forever, # Short poll interval to make the test finish quickly. # Time between requests is short enough that we won't wake # up spuriously too many times. kwargs={'poll_interval':0.01}) - self.t.daemon = True # In case this function raises. - self.t.start() + cls.enterClassContext(threading_helper.start_threads([t])) + cls.addClassCleanup(cls.server.shutdown) + + +SAMPLE_ROBOTS_TXT = b'''\ +User-agent: test_robotparser +Disallow: /utf8/\xf0\x9f\x90\x8d +Disallow: /non-utf8/\xf0 +Disallow: //[spam]/path +''' + + +class LocalNetworkTestCase(BaseLocalNetworkTestCase, unittest.TestCase): + class RobotHandler(BaseHTTPRequestHandler): + + def do_GET(self): + self.send_response(200) + self.end_headers() + self.wfile.write(SAMPLE_ROBOTS_TXT) + + def log_message(self, format, *args): + pass + + def testRead(self): + # Test that reading a weird robots.txt doesn't fail. + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + '/robots.txt' + parser = urllib.robotparser.RobotFileParser() + parser.set_url(robots_url) + parser.read() + # And it can even interpret the weird paths in some reasonable way. + agent = 'test_robotparser' + self.assertTrue(parser.can_fetch(agent, robots_url)) + self.assertTrue(parser.can_fetch(agent, url + '/utf8/')) + self.assertFalse(parser.can_fetch(agent, url + '/utf8/\U0001f40d')) + self.assertFalse(parser.can_fetch(agent, url + '/utf8/%F0%9F%90%8D')) + self.assertTrue(parser.can_fetch(agent, url + '/non-utf8/')) + self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/%F0')) + self.assertFalse(parser.can_fetch(agent, url + '/non-utf8/\U0001f40d')) + self.assertFalse(parser.can_fetch(agent, url + '/%2F[spam]/path')) + + +class HttpErrorsTestCase(BaseLocalNetworkTestCase, unittest.TestCase): + class RobotHandler(BaseHTTPRequestHandler): + + def do_GET(self): + self.send_error(self.server.return_code) + + def log_message(self, format, *args): + pass + + def setUp(self): + # Make sure that a valid code is set in the test. + self.server.return_code = None + + def testUnauthorized(self): + self.server.return_code = 401 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertFalse(parser.can_fetch("*", robots_url)) + self.assertFalse(parser.can_fetch("*", url + '/some/file.html')) - def tearDown(self): - self.server.shutdown() - self.t.join() - self.server.server_close() + def testForbidden(self): + self.server.return_code = 403 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertFalse(parser.can_fetch("*", robots_url)) + self.assertFalse(parser.can_fetch("*", url + '/some/file.html')) + + def testNotFound(self): + self.server.return_code = 404 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertTrue(parser.can_fetch("*", robots_url)) + self.assertTrue(parser.can_fetch("*", url + '/path/file.html')) + + def testTeapot(self): + self.server.return_code = 418 + addr = self.server.server_address + url = f'http://{socket_helper.HOST}:{addr[1]}' + robots_url = url + "/robots.txt" + parser = urllib.robotparser.RobotFileParser() + parser.set_url(url) + parser.read() + self.assertTrue(parser.can_fetch("*", robots_url)) + self.assertTrue(parser.can_fetch("*", url + '/pot-1?milk-type=Cream')) - @threading_helper.reap_threads - def testPasswordProtectedSite(self): + def testServiceUnavailable(self): + self.server.return_code = 503 addr = self.server.server_address - url = 'http://' + socket_helper.HOST + ':' + str(addr[1]) + url = f'http://{socket_helper.HOST}:{addr[1]}' robots_url = url + "/robots.txt" parser = urllib.robotparser.RobotFileParser() parser.set_url(url) parser.read() self.assertFalse(parser.can_fetch("*", robots_url)) + self.assertFalse(parser.can_fetch("*", url + '/path/file.html')) @support.requires_working_socket() @@ -355,6 +810,7 @@ class NetworkTestCase(unittest.TestCase): @classmethod def setUpClass(cls): support.requires('network') + cls.addClassCleanup(urllib.request.urlcleanup) with socket_helper.transient_internet(cls.base_url): cls.parser = urllib.robotparser.RobotFileParser(cls.robots_txt) cls.parser.read() @@ -374,7 +830,7 @@ def test_basic(self): def test_can_fetch(self): self.assertTrue(self.parser.can_fetch('*', self.url('elsewhere'))) self.assertFalse(self.parser.can_fetch('Nutch', self.base_url)) - self.assertFalse(self.parser.can_fetch('Nutch', self.url('brian'))) + self.assertTrue(self.parser.can_fetch('Nutch', self.url('brian'))) self.assertFalse(self.parser.can_fetch('Nutch', self.url('webstats'))) self.assertFalse(self.parser.can_fetch('*', self.url('webstats'))) self.assertTrue(self.parser.can_fetch('*', self.base_url)) diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index ada78ec8e6b0c75..7eb4cee36107ab0 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -216,6 +216,25 @@ def test_invalid_names(self): # Package without __main__.py self.expect_import_error("multiprocessing") + def test_invalid_names_set_name_attribute(self): + cases = [ + # (mod_name, expected_name) -- comment indicates raise site + ("nonexistent_runpy_test_module", + "nonexistent_runpy_test_module"), # spec is None + ("sys.imp.eric", "sys.imp.eric"), # find_spec error + (".relative_name", ".relative_name"), # relative name rejected + ("sys", "sys"), # builtin: no code object + ("multiprocessing", "multiprocessing"), # package without __main__ + ] + for mod_name, expected_name in cases: + with self.subTest(mod_name=mod_name): + try: + run_module(mod_name) + except ImportError as exc: + self.assertEqual(exc.name, expected_name) + else: + self.fail("Expected ImportError for %r" % mod_name) + def test_library_module(self): self.assertEqual(run_module("runpy")["__name__"], "runpy") @@ -714,6 +733,17 @@ def test_directory_error(self): msg = "can't find '__main__' module in %r" % script_dir self._check_import_error(script_dir, msg) + def test_directory_error_sets_name_attribute(self): + with temp_dir() as script_dir: + self._make_test_script(script_dir, 'not_main') + try: + run_path(script_dir) + except ImportError as exc: + self.assertEqual(exc.name, '__main__') + else: + self.fail("Expected ImportError for directory without " + "__main__.py") + def test_zipfile(self): with temp_dir() as script_dir: mod_name = '__main__' @@ -796,7 +826,7 @@ def assertSigInt(self, cmd, *args, **kwargs): # Use -E to ignore PYTHONSAFEPATH cmd = [sys.executable, '-E', *cmd] proc = subprocess.run(cmd, *args, **kwargs, text=True, stderr=subprocess.PIPE) - self.assertTrue(proc.stderr.endswith("\nKeyboardInterrupt\n"), proc.stderr) + self.assertEndsWith(proc.stderr, "\nKeyboardInterrupt\n") self.assertEqual(proc.returncode, self.EXPECTED_CODE) def test_pymain_run_file(self): diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py index 24a366efc6ca05d..520fbc1b66237bf 100644 --- a/Lib/test/test_scope.py +++ b/Lib/test/test_scope.py @@ -778,7 +778,7 @@ class X: class X: locals()["x"] = 43 del x - self.assertFalse(hasattr(X, "x")) + self.assertNotHasAttr(X, "x") self.assertEqual(x, 42) @cpython_only diff --git a/Lib/test/test_script_helper.py b/Lib/test/test_script_helper.py index f7871fd3b77c02d..eeea6c4842b4881 100644 --- a/Lib/test/test_script_helper.py +++ b/Lib/test/test_script_helper.py @@ -74,8 +74,7 @@ class TestScriptHelperEnvironment(unittest.TestCase): """Code coverage for interpreter_requires_environment().""" def setUp(self): - self.assertTrue( - hasattr(script_helper, '__cached_interp_requires_environment')) + self.assertHasAttr(script_helper, '__cached_interp_requires_environment') # Reset the private cached state. script_helper.__dict__['__cached_interp_requires_environment'] = None diff --git a/Lib/test/test_set.py b/Lib/test/test_set.py index c01e323553d768b..34da01c34151287 100644 --- a/Lib/test/test_set.py +++ b/Lib/test/test_set.py @@ -20,6 +20,14 @@ def check_pass_thru(): raise PassThru yield 1 +class CustomHash: + def __init__(self, hash): + self.hash = hash + def __hash__(self): + return self.hash + def __repr__(self): + return f'<CustomHash {self.hash} at {id(self):#x}>' + class BadCmp: def __hash__(self): return 1 @@ -237,7 +245,7 @@ def test_pickling(self): if type(self.s) not in (set, frozenset): self.assertEqual(self.s.x, dup.x) self.assertEqual(self.s.z, dup.z) - self.assertFalse(hasattr(self.s, 'y')) + self.assertNotHasAttr(self.s, 'y') del self.s.x, self.s.z def test_iterator_pickling(self): @@ -675,6 +683,28 @@ def __hash__(self): with self.assertRaises(KeyError): myset.discard(elem2) + def test_hash_collision_remove_add(self): + self.maxDiff = None + # There should be enough space, so all elements with unique hash + # will be placed in corresponding cells without collision. + n = 64 + elems = [CustomHash(h) for h in range(n)] + # Elements with hash collision. + a = CustomHash(n) + b = CustomHash(n) + elems += [a, b] + s = self.thetype(elems) + self.assertEqual(len(s), len(elems), s) + s.remove(a) + # "a" has been replaced with a dummy. + del elems[n] + self.assertEqual(len(s), len(elems), s) + self.assertEqual(s, set(elems)) + s.add(b) + # "b" should not replace the dummy. + self.assertEqual(len(s), len(elems), s) + self.assertEqual(s, set(elems)) + class SetSubclass(set): pass @@ -876,8 +906,8 @@ def test_repr(self): def check_repr_against_values(self): text = repr(self.set) - self.assertTrue(text.startswith('{')) - self.assertTrue(text.endswith('}')) + self.assertStartsWith(text, '{') + self.assertEndsWith(text, '}') result = text[1:-1].split(', ') result.sort() @@ -1853,6 +1883,7 @@ def test_iter_and_mutate(self): list(si) def test_merge_and_mutate(self): + # gh-141805 class X: def __hash__(self): return hash(0) @@ -1865,6 +1896,33 @@ def __eq__(self, o): s = {0} s.update(other) + def test_hash_collision_concurrent_add(self): + class X: + def __hash__(self): + return 0 + class Y: + flag = False + def __hash__(self): + return 0 + def __eq__(self, other): + if not self.flag: + self.flag = True + s.add(X()) + return self is other + + a = X() + s = set() + s.add(a) + s.add(X()) + s.remove(a) + # Now the set contains a dummy entry followed by an entry + # for an object with hash 0. + s.add(Y()) + # The following operations should not crash. + repr(s) + list(s) + set() | s + class TestOperationsMutating: """Regression test for bpo-46615""" diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index a13ddcb76b7bcb8..a88dc286e4ee631 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -1,8 +1,11 @@ import io import itertools +import os import shlex import string +import tempfile import unittest +from unittest.mock import patch from test.support import cpython_only from test.support import import_helper @@ -330,6 +333,7 @@ def testQuote(self): unsafe = '"`$\\!' + unicode_sample self.assertEqual(shlex.quote(''), "''") + self.assertEqual(shlex.quote(None), "''") self.assertEqual(shlex.quote(safeunquoted), safeunquoted) self.assertEqual(shlex.quote('test file name'), "'test file name'") for u in unsafe: @@ -338,6 +342,8 @@ def testQuote(self): for u in unsafe: self.assertEqual(shlex.quote("test%s'name'" % u), "'test%s'\"'\"'name'\"'\"''" % u) + self.assertRaises(TypeError, shlex.quote, 42) + self.assertRaises(TypeError, shlex.quote, b"abc") def testJoin(self): for split_command, command in [ @@ -365,6 +371,203 @@ def testPunctuationCharsReadOnly(self): with self.assertRaises(AttributeError): shlex_instance.punctuation_chars = False + def testLinenoAfterNewLine(self): + s = shlex.shlex("line 1\nline 2") + self.assertEqual(s.lineno, 1) # before consumption + list(s) + self.assertEqual(s.lineno, 2) + + def testLinenoAfterComment(self): + """Comment handler increments lineno even without a trailing newline.""" + s = shlex.shlex("line 1 # line 2") + list(s) + self.assertEqual(s.lineno, 2) + + def testPushToken(self): + s = shlex.shlex("b c") + s.push_token("a") + self.assertListEqual(list(s), ["a", "b", "c"]) + + def testPushTokenLifo(self): + s = shlex.shlex("") + s.push_token("first") + s.push_token("last") + self.assertListEqual(list(s), ["last", "first"]) + + def testPushTokenDebug(self): + s = shlex.shlex("") + s.debug = 1 + tok = "a" + with patch("builtins.print") as mock_print: + s.push_token(tok) + mock_print.assert_called_once_with(f"shlex: pushing token {tok!r}") + + def testPushSourceString(self): + s = shlex.shlex("world") + s.push_source("hello") + self.assertListEqual(list(s), ["hello", "world"]) + + def testPushSourceStream(self): + s = shlex.shlex("world") + s.push_source(io.StringIO("hello")) + self.assertListEqual(list(s), ["hello", "world"]) + + def testPushSourceStreamDebug(self): + s = shlex.shlex("") + stream = io.StringIO("hello") + s.debug = 1 + with patch("builtins.print") as mock_print: + s.push_source(stream) + mock_print.assert_called_once_with(f"shlex: pushing to stream {stream}") + + def testPushSourceNewfile(self): + """shlex.push_source sets infile to newfile; pop_source restores the original on exhaustion.""" + original_file = "original.sh" + new_file = "new.sh" + s = shlex.shlex("b", infile=original_file) + s.debug = 1 + with patch("builtins.print") as mock_print: + s.push_source("a", newfile=new_file) + mock_print.assert_called_once_with(f"shlex: pushing to file {new_file}") + self.assertEqual(s.infile, new_file) + s.debug = 0 + list(s) + self.assertEqual(s.infile, original_file) + + def testPopSourceDebug(self): + """pop_source emits debug output when debug is set.""" + s = shlex.shlex("b") + original_stream = s.instream + s.push_source("a") + s.debug = 1 + with patch("builtins.print") as mock_print: + list(s) # exhausts pushed source and triggers pop_source internally + mock_print.assert_any_call(f"shlex: popping to {original_stream}, line 1") + + def testErrorLeaderTracksPosition(self): + infile_label = "test.sh" + s = shlex.shlex("line 1\nline 2", infile=infile_label) + list(s) + result = s.error_leader() + self.assertEqual(result, f'"{infile_label}", line 2: ') + + def testErrorLeaderOverrides(self): + s = shlex.shlex("foo", infile="original.sh") + infile_label_override = "override.sh" + lineno_override = 42 + result = s.error_leader(infile=infile_label_override, lineno=lineno_override) + self.assertEqual(result, f'"{infile_label_override}", line {lineno_override}: ') + + def testNoClosingQuotation(self): + s = shlex.shlex('"foo') + with self.assertRaisesRegex(ValueError, "No closing quotation"): + list(s) + + def testNoEscapedCharacter(self): + s = shlex.shlex("\\", posix=True) + with self.assertRaisesRegex(ValueError, "No escaped character"): + list(s) + + def testSourcehookStripsQuotes(self): + with tempfile.NamedTemporaryFile(mode="w", suffix=".sh", delete_on_close=False) as f: + f.write("hello") + f.close() + s = shlex.shlex("") + newfile, stream = s.sourcehook(f'"{f.name}"') + stream.close() + self.assertEqual(newfile, f.name) + + def testSourcehookAbsolutePath(self): + with tempfile.NamedTemporaryFile(mode="w", delete_on_close=False) as f: + f.close() + s = shlex.shlex("", infile="/some/dir/main.sh") + newfile, stream = s.sourcehook(f.name) + stream.close() + self.assertEqual(newfile, f.name) + + def testSourcehookRelativePath(self): + with tempfile.TemporaryDirectory() as d: + fpath = os.path.join(d, "included.sh") + with open(fpath, "w"): + pass + s = shlex.shlex("", infile=os.path.join(d, "main.sh")) + newfile, stream = s.sourcehook("included.sh") + stream.close() + self.assertEqual(newfile, fpath) + + def testSourceInclusion(self): + """shlex.source sets a trigger keyword: when the lexer reads a token equal + to it, the next token is consumed as a filename and passed to + sourcehook, which returns a stream to push onto the input stack. + Tokens flow from that stream first, then resume from the original. + """ + s = shlex.shlex("trigger filename remaining") + s.source = "trigger" + s.sourcehook = lambda f: (f, io.StringIO("included")) + self.assertEqual(list(s), ["included", "remaining"]) + + def testGetTokenPopsPushbackDebug(self): + s = shlex.shlex("") + s.push_token("hello") + s.debug = 1 # set after push_token to isolate the pop-token branch + with patch("builtins.print") as mock_print: + tok = s.get_token() + self.assertEqual(tok, "hello") + mock_print.assert_called_once_with("shlex: popping token 'hello'") + + def testDebugWhitespaceInWhitespaceState(self): + s = shlex.shlex(" a") + s.debug = 2 + with patch("builtins.print") as mock_print: + list(s) + mock_print.assert_any_call("shlex: I see whitespace in whitespace state") + + def testDebugWhitespaceInWordState(self): + s = shlex.shlex("a b") + s.debug = 2 + with patch("builtins.print") as mock_print: + list(s) + mock_print.assert_any_call("shlex: I see whitespace in word state") + + def testDebugPunctuationInWordState(self): + s = shlex.shlex("a(") + s.debug = 2 + with patch("builtins.print") as mock_print: + list(s) + mock_print.assert_any_call("shlex: I see punctuation in word state") + + def testDebugRawToken(self): + s = shlex.shlex("hello") + s.debug = 2 + with patch("builtins.print") as mock_print: + list(s) + mock_print.assert_any_call("shlex: raw token='hello'") + + def testDebugEOFInQuote(self): + s = shlex.shlex('"oops', posix=True) + s.debug = 2 + with patch('builtins.print') as mock_print: + with self.assertRaises(ValueError): + list(s) + msgs = [call.args[0] for call in mock_print.call_args_list] + self.assertTrue(any("EOF in quotes" in m for m in msgs)) + + def testDebugEOFInEscape(self): + s = shlex.shlex("oops\\", posix=True) + s.debug = 2 + with patch("builtins.print") as mock_print: + with self.assertRaises(ValueError): + list(s) + msgs = [call.args[0] for call in mock_print.call_args_list] + self.assertTrue(any("EOF in escape" in m for m in msgs)) + + def testDebugStateTrace(self): + s = shlex.shlex("a") + s.debug = 3 + with patch("builtins.print") as mock_print: + list(s) + mock_print.assert_any_call("shlex: in state ' ' I see character: 'a'") + @cpython_only def test_lazy_imports(self): import_helper.ensure_lazy_imports('shlex', {'collections', 're', 'os'}) diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index 87991fbda4c7df0..8813cc46cb602ef 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -427,12 +427,12 @@ def check_args_to_onerror(self, func, arg, exc): else: self.assertIs(func, os.listdir) self.assertIn(arg, [TESTFN, self.child_dir_path]) - self.assertTrue(issubclass(exc[0], OSError)) + self.assertIsSubclass(exc[0], OSError) self.errorState += 1 else: self.assertEqual(func, os.rmdir) self.assertEqual(arg, TESTFN) - self.assertTrue(issubclass(exc[0], OSError)) + self.assertIsSubclass(exc[0], OSError) self.errorState = 3 @unittest.skipIf(sys.platform[:6] == 'cygwin', @@ -2110,8 +2110,6 @@ def test_make_zipfile_rootdir_nodir(self): def check_unpack_archive(self, format, **kwargs): self.check_unpack_archive_with_converter( format, lambda path: path, **kwargs) - self.check_unpack_archive_with_converter( - format, FakePath, **kwargs) self.check_unpack_archive_with_converter(format, FakePath, **kwargs) def check_unpack_archive_with_converter(self, format, converter, **kwargs): @@ -2168,6 +2166,71 @@ def test_unpack_archive_zip(self): with self.assertRaises(TypeError): self.check_unpack_archive('zip', filter='data') + def test_unpack_archive_zip_badpaths(self): + srcdir = self.mkdtemp() + zipname = os.path.join(srcdir, 'test.zip') + abspath = os.path.join(srcdir, 'abspath') + with zipfile.ZipFile(zipname, 'w') as zf: + zf.writestr(abspath, 'badfile') + zf.writestr(os.sep + abspath, 'badfile') + zf.writestr('/abspath', 'badfile') + zf.writestr('C:/abspath', 'badfile') + zf.writestr('D:\\abspath', 'badfile') + zf.writestr('E:abspath', 'badfile') + zf.writestr('F:/G:/abspath', 'badfile') + zf.writestr('//server/share/abspath', 'badfile') + zf.writestr('\\\\server2\\share\\abspath', 'badfile') + zf.writestr('../relpath', 'badfile') + zf.writestr(os.pardir + os.sep + 'relpath2', 'badfile') + zf.writestr('good/file', 'goodfile') + zf.writestr('good..file', 'goodfile') + + dstdir = os.path.join(self.mkdtemp(), 'dst') + unpack_archive(zipname, dstdir) + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'good', 'file'))) + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'good..file'))) + self.assertFalse(os.path.exists(abspath)) + self.assertFalse(os.path.exists(os.path.join(dstdir, 'abspath'))) + self.assertFalse(os.path.exists(os.path.join(dstdir, 'G_'))) + self.assertFalse(os.path.exists(os.path.join(dstdir, 'server'))) + if os.name != 'nt': + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'C:', 'abspath'))) + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'D:\\abspath'))) + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'E:abspath'))) + self.assertTrue(os.path.isfile(os.path.join(dstdir, 'F:', 'G:', 'abspath'))) + self.assertTrue(os.path.isfile(os.path.join(dstdir, '\\\\server2\\share\\abspath'))) + if os.pardir == '..': + self.assertFalse(os.path.exists(os.path.join(dstdir, '..', 'relpath'))) + self.assertFalse(os.path.exists(os.path.join(dstdir, 'relpath'))) + else: + self.assertTrue(os.path.isfile(os.path.join(dstdir, '..', 'relpath'))) + self.assertFalse(os.path.exists(os.path.join(dstdir, os.pardir, 'relpath2'))) + self.assertFalse(os.path.exists(os.path.join(dstdir, 'relpath2'))) + + dstdir2 = os.path.join(self.mkdtemp(), 'dst') + os.mkdir(dstdir2) + with os_helper.change_cwd(dstdir2): + unpack_archive(zipname, '') + self.assertTrue(os.path.isfile(os.path.join('good', 'file'))) + self.assertTrue(os.path.isfile('good..file')) + self.assertFalse(os.path.exists(abspath)) + self.assertFalse(os.path.exists('abspath')) + self.assertFalse(os.path.exists('C_')) + self.assertFalse(os.path.exists('server')) + if os.name != 'nt': + self.assertTrue(os.path.isfile(os.path.join('C:', 'abspath'))) + self.assertTrue(os.path.isfile('D:\\abspath')) + self.assertTrue(os.path.isfile('E:abspath')) + self.assertTrue(os.path.isfile(os.path.join('F:', 'G:', 'abspath'))) + self.assertTrue(os.path.isfile('\\\\server2\\share\\abspath')) + if os.pardir == '..': + self.assertFalse(os.path.exists(os.path.join('..', 'relpath'))) + self.assertFalse(os.path.exists('relpath')) + else: + self.assertTrue(os.path.isfile(os.path.join('..', 'relpath'))) + self.assertFalse(os.path.exists(os.path.join(os.pardir, 'relpath2'))) + self.assertFalse(os.path.exists('relpath2')) + def test_unpack_registry(self): formats = get_unpack_formats() @@ -2825,6 +2888,23 @@ def test_destinsrc_false_positive(self): finally: os_helper.rmtree(TESTFN) + @os_helper.skip_unless_symlink + def test_destinsrc_symlink_bypass(self): + tmp = self.mkdtemp() + src = os.path.join(tmp, 'src') + os.makedirs(src) + # tmp/link -> tmp (one level up) + link = os.path.join(tmp, 'link') + os.symlink(tmp, link) + # raw path: tmp/link/src/sub - no src prefix in string space + # real path: tmp/src/sub - physically inside src + dst = os.path.join(link, 'src', 'sub') + self.assertTrue( + shutil._destinsrc(src, dst), + msg='_destinsrc failed to detect dst inside src via symlink ' + '(dst=%s, src=%s)' % (dst, src), + ) + @os_helper.skip_unless_symlink @mock_rename def test_move_file_symlink(self): @@ -3479,7 +3559,7 @@ class PublicAPITests(unittest.TestCase): """Ensures that the correct values are exposed in the public API.""" def test_module_all_attribute(self): - self.assertTrue(hasattr(shutil, '__all__')) + self.assertHasAttr(shutil, '__all__') target_api = ['copyfileobj', 'copyfile', 'copymode', 'copystat', 'copy', 'copy2', 'copytree', 'move', 'rmtree', 'Error', 'SpecialFileError', 'make_archive', diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index a7e9241f44d243f..f8c6f37c607cfbf 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -13,6 +13,7 @@ from test.support import socket_helper from test.support import captured_stderr from test.support.os_helper import TESTFN, EnvironmentVarGuard +from test.support.script_helper import spawn_python, kill_python import ast import builtins import glob @@ -25,6 +26,7 @@ import sys import sysconfig import tempfile +from textwrap import dedent import urllib.error import urllib.request from unittest import mock @@ -307,8 +309,7 @@ def test_getuserbase(self): with EnvironmentVarGuard() as environ: environ['PYTHONUSERBASE'] = 'xoxo' - self.assertTrue(site.getuserbase().startswith('xoxo'), - site.getuserbase()) + self.assertStartsWith(site.getuserbase(), 'xoxo') @unittest.skipUnless(HAS_USER_SITE, 'need user site') def test_getusersitepackages(self): @@ -318,7 +319,7 @@ def test_getusersitepackages(self): # the call sets USER_BASE *and* USER_SITE self.assertEqual(site.USER_SITE, user_site) - self.assertTrue(user_site.startswith(site.USER_BASE), user_site) + self.assertStartsWith(user_site, site.USER_BASE) self.assertEqual(site.USER_BASE, site.getuserbase()) def test_getsitepackages(self): @@ -359,11 +360,10 @@ def test_no_home_directory(self): environ.unset('PYTHONUSERBASE', 'APPDATA') user_base = site.getuserbase() - self.assertTrue(user_base.startswith('~' + os.sep), - user_base) + self.assertStartsWith(user_base, '~' + os.sep) user_site = site.getusersitepackages() - self.assertTrue(user_site.startswith(user_base), user_site) + self.assertStartsWith(user_site, user_base) with mock.patch('os.path.isdir', return_value=False) as mock_isdir, \ mock.patch.object(site, 'addsitedir') as mock_addsitedir, \ @@ -495,18 +495,18 @@ def test_add_build_dir(self): def test_setting_quit(self): # 'quit' and 'exit' should be injected into builtins - self.assertTrue(hasattr(builtins, "quit")) - self.assertTrue(hasattr(builtins, "exit")) + self.assertHasAttr(builtins, "quit") + self.assertHasAttr(builtins, "exit") def test_setting_copyright(self): # 'copyright', 'credits', and 'license' should be in builtins - self.assertTrue(hasattr(builtins, "copyright")) - self.assertTrue(hasattr(builtins, "credits")) - self.assertTrue(hasattr(builtins, "license")) + self.assertHasAttr(builtins, "copyright") + self.assertHasAttr(builtins, "credits") + self.assertHasAttr(builtins, "license") def test_setting_help(self): # 'help' should be set in builtins - self.assertTrue(hasattr(builtins, "help")) + self.assertHasAttr(builtins, "help") def test_sitecustomize_executed(self): # If sitecustomize is available, it should have been imported. @@ -805,5 +805,110 @@ def test_underpth_dll_file(self): self.assertTrue(rc, "sys.path is incorrect") +class CommandLineTests(unittest.TestCase): + def exists(self, path): + if path is not None and os.path.isdir(path): + return "exists" + else: + return "doesn't exist" + + def get_excepted_output(self, *args): + if len(args) == 0: + user_base = site.getuserbase() + user_site = site.getusersitepackages() + output = io.StringIO() + output.write("sys.path = [\n") + for dir in sys.path: + output.write(" %r,\n" % (dir,)) + output.write("]\n") + output.write(f"USER_BASE: {user_base} ({self.exists(user_base)})\n") + output.write(f"USER_SITE: {user_site} ({self.exists(user_site)})\n") + output.write(f"ENABLE_USER_SITE: {site.ENABLE_USER_SITE}\n") + return 0, dedent(output.getvalue()).strip() + + buffer = [] + if '--user-base' in args: + buffer.append(site.getuserbase()) + if '--user-site' in args: + buffer.append(site.getusersitepackages()) + + if buffer: + return_code = 3 + if site.ENABLE_USER_SITE: + return_code = 0 + elif site.ENABLE_USER_SITE is False: + return_code = 1 + elif site.ENABLE_USER_SITE is None: + return_code = 2 + output = os.pathsep.join(buffer) + return return_code, os.path.normpath(dedent(output).strip()) + else: + return 10, None + + def invoke_command_line(self, *args): + cmd_args = [] + if sys.flags.no_user_site: + cmd_args.append("-s") + cmd_args.extend(["-m", "site", *args]) + + with EnvironmentVarGuard() as env: + env["PYTHONUTF8"] = "1" + env["PYTHONIOENCODING"] = "utf-8" + proc = spawn_python(*cmd_args, text=True, env=env, + encoding='utf-8', errors='replace') + + output = kill_python(proc) + return_code = proc.returncode + return return_code, os.path.normpath(dedent(output).strip()) + + @support.requires_subprocess() + def test_no_args(self): + return_code, output = self.invoke_command_line() + excepted_return_code, _ = self.get_excepted_output() + self.assertEqual(return_code, excepted_return_code) + lines = output.splitlines() + self.assertEqual(lines[0], "sys.path = [") + self.assertEqual(lines[-4], "]") + excepted_base = f"USER_BASE: '{site.getuserbase()}'" +\ + f" ({self.exists(site.getuserbase())})" + self.assertEqual(lines[-3], excepted_base) + excepted_site = f"USER_SITE: '{site.getusersitepackages()}'" +\ + f" ({self.exists(site.getusersitepackages())})" + self.assertEqual(lines[-2], excepted_site) + self.assertEqual(lines[-1], f"ENABLE_USER_SITE: {site.ENABLE_USER_SITE}") + + @support.requires_subprocess() + def test_unknown_args(self): + return_code, output = self.invoke_command_line("--unknown-arg") + excepted_return_code, _ = self.get_excepted_output("--unknown-arg") + self.assertEqual(return_code, excepted_return_code) + self.assertIn('[--user-base] [--user-site]', output) + + @support.requires_subprocess() + def test_base_arg(self): + return_code, output = self.invoke_command_line("--user-base") + excepted = self.get_excepted_output("--user-base") + excepted_return_code, excepted_output = excepted + self.assertEqual(return_code, excepted_return_code) + self.assertEqual(output, excepted_output) + + @support.requires_subprocess() + def test_site_arg(self): + return_code, output = self.invoke_command_line("--user-site") + excepted = self.get_excepted_output("--user-site") + excepted_return_code, excepted_output = excepted + self.assertEqual(return_code, excepted_return_code) + self.assertEqual(output, excepted_output) + + @support.requires_subprocess() + def test_both_args(self): + return_code, output = self.invoke_command_line("--user-base", + "--user-site") + excepted = self.get_excepted_output("--user-base", "--user-site") + excepted_return_code, excepted_output = excepted + self.assertEqual(return_code, excepted_return_code) + self.assertEqual(output, excepted_output) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index 4c9fc14bd43f548..fb3ea34d766e50e 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -17,6 +17,7 @@ import threading import unittest +import unittest.mock as mock from test import support, mock_socket from test.support import hashlib_helper from test.support import socket_helper @@ -926,11 +927,14 @@ def _auth_cram_md5(self, arg=None): except ValueError as e: self.push('535 Splitting response {!r} into user and password ' 'failed: {}'.format(logpass, e)) - return False - valid_hashed_pass = hmac.HMAC( - sim_auth[1].encode('ascii'), - self._decode_base64(sim_cram_md5_challenge).encode('ascii'), - 'md5').hexdigest() + return + pwd = sim_auth[1].encode('ascii') + msg = self._decode_base64(sim_cram_md5_challenge).encode('ascii') + try: + valid_hashed_pass = hmac.HMAC(pwd, msg, 'md5').hexdigest() + except ValueError: + self.push('504 CRAM-MD5 is not supported') + return self._authenticated(user, hashed_pass == valid_hashed_pass) # end AUTH related stuff. @@ -1181,6 +1185,39 @@ def testAUTH_CRAM_MD5(self): self.assertEqual(resp, (235, b'Authentication Succeeded')) smtp.close() + @mock.patch("hmac.HMAC") + @mock.patch("smtplib._have_cram_md5_support", False) + def testAUTH_CRAM_MD5_blocked(self, hmac_constructor): + # CRAM-MD5 is the only "known" method by the server, + # but it is not supported by the client. In particular, + # no challenge will ever be sent. + self.serv.add_feature("AUTH CRAM-MD5") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + msg = re.escape("No suitable authentication method found.") + with self.assertRaisesRegex(smtplib.SMTPException, msg): + smtp.login(sim_auth[0], sim_auth[1]) + hmac_constructor.assert_not_called() # call has been bypassed + + @mock.patch("smtplib._have_cram_md5_support", False) + def testAUTH_CRAM_MD5_blocked_and_fallback(self): + # Test that PLAIN is tried after CRAM-MD5 failed + self.serv.add_feature("AUTH CRAM-MD5 PLAIN") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + with ( + mock.patch.object(smtp, "auth_cram_md5") as smtp_auth_cram_md5, + mock.patch.object( + smtp, "auth_plain", wraps=smtp.auth_plain + ) as smtp_auth_plain + ): + resp = smtp.login(sim_auth[0], sim_auth[1]) + smtp_auth_plain.assert_called_once() + smtp_auth_cram_md5.assert_not_called() # no call to HMAC constructor + self.assertEqual(resp, (235, b'Authentication Succeeded')) + @hashlib_helper.requires_hashdigest('md5', openssl=True) def testAUTH_multiple(self): # Test that multiple authentication methods are tried. diff --git a/Lib/test/test_smtpnet.py b/Lib/test/test_smtpnet.py index d765746987bc4b0..861e7e6a725c407 100644 --- a/Lib/test/test_smtpnet.py +++ b/Lib/test/test_smtpnet.py @@ -45,6 +45,59 @@ def test_connect_starttls(self): server.ehlo() server.quit() + def test_connect_host_port_starttls(self): + support.get_attribute(smtplib, 'SMTP_SSL') + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + with socket_helper.transient_internet(self.testServer): + server = smtplib.SMTP(f'{self.testServer}:{self.remotePort}') + try: + server.starttls(context=context) + except smtplib.SMTPException as e: + if e.args[0] == 'STARTTLS extension not supported by server.': + unittest.skip(e.args[0]) + else: + raise + server.ehlo() + server.quit() + + def test_explicit_connect_starttls(self): + support.get_attribute(smtplib, 'SMTP_SSL') + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + with socket_helper.transient_internet(self.testServer): + server = smtplib.SMTP() + server.connect(self.testServer, self.remotePort) + try: + server.starttls(context=context) + except smtplib.SMTPException as e: + if e.args[0] == 'STARTTLS extension not supported by server.': + unittest.skip(e.args[0]) + else: + raise + server.ehlo() + server.quit() + + def test_explicit_connect_host_port_starttls(self): + support.get_attribute(smtplib, 'SMTP_SSL') + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + with socket_helper.transient_internet(self.testServer): + server = smtplib.SMTP() + server.connect(f'{self.testServer}:{self.remotePort}') + try: + server.starttls(context=context) + except smtplib.SMTPException as e: + if e.args[0] == 'STARTTLS extension not supported by server.': + unittest.skip(e.args[0]) + else: + raise + server.ehlo() + server.quit() + class SmtpSSLTest(unittest.TestCase): testServer = SMTP_TEST_SERVER diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index 03c54151a2218ff..263fdafbad86d78 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -52,6 +52,14 @@ VSOCKPORT = 1234 AIX = platform.system() == "AIX" +SOLARIS = sys.platform.startswith("sunos") +# NetBSD, OpenBSD and DragonFly deliver the file descriptors from only one +# SCM_RIGHTS control message when several are sent in a single sendmsg(). +BSD_COMBINES_SCM_RIGHTS = sys.platform.startswith( + ("netbsd", "openbsd", "dragonfly")) +# OpenBSD and DragonFly fail recvmsg() with EMSGSIZE, instead of setting +# MSG_CTRUNC, when the ancillary data buffer is too small for a cmsghdr. +CMSG_TRUNC_RAISES_EMSGSIZE = sys.platform.startswith(("openbsd", "dragonfly")) WSL = "microsoft-standard-WSL" in platform.release() try: @@ -202,6 +210,25 @@ def _have_socket_hyperv(): return True +def _have_udp_lite(): + if not hasattr(socket, "IPPROTO_UDPLITE"): + return False + # Older Android versions block UDPLITE with SELinux. + if support.is_android and platform.android_ver().api_level < 29: + return False + + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDPLITE) + except OSError as exc: + # Linux 7.1 removed UDP Lite support + if exc.errno == errno.EPROTONOSUPPORT: + return False + raise + sock.close() + + return True + + @contextlib.contextmanager def socket_setdefaulttimeout(timeout): old_timeout = socket.getdefaulttimeout() @@ -244,10 +271,7 @@ def downgrade_malformed_data_warning(): HAVE_SOCKET_VSOCK = _have_socket_vsock() -# Older Android versions block UDPLITE with SELinux. -HAVE_SOCKET_UDPLITE = ( - hasattr(socket, "IPPROTO_UDPLITE") - and not (support.is_android and platform.android_ver().api_level < 29)) +HAVE_SOCKET_UDPLITE = _have_udp_lite() HAVE_SOCKET_BLUETOOTH = _have_socket_bluetooth() @@ -560,8 +584,8 @@ def clientTearDown(self): @unittest.skipIf(WSL, 'VSOCK does not work on Microsoft WSL') @unittest.skipUnless(HAVE_SOCKET_VSOCK, 'VSOCK sockets required for this test.') -@unittest.skipUnless(get_cid() != 2, # VMADDR_CID_HOST - "This test can only be run on a virtual guest.") +@unittest.skipIf(get_cid() == getattr(socket, 'VMADDR_CID_HOST', 2), + "This test can only be run on a virtual guest.") class ThreadedVSOCKSocketStreamTest(unittest.TestCase, ThreadableTest): def __init__(self, methodName='runTest'): @@ -571,7 +595,16 @@ def __init__(self, methodName='runTest'): def setUp(self): self.serv = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM) self.addCleanup(self.serv.close) - self.serv.bind((socket.VMADDR_CID_ANY, VSOCKPORT)) + cid = get_cid() + if cid in (socket.VMADDR_CID_HOST, socket.VMADDR_CID_ANY): + cid = socket.VMADDR_CID_LOCAL + try: + self.serv.bind((cid, VSOCKPORT)) + except OSError as exc: + if exc.errno == errno.EADDRNOTAVAIL: + self.skipTest(f"bind() failed with {exc!r}") + else: + raise self.serv.listen() self.serverExplicitReady() self.serv.settimeout(support.LOOPBACK_TIMEOUT) @@ -1085,9 +1118,7 @@ def test3542SocketOptions(self): 'IPV6_USE_MIN_MTU', } for opt in opts: - self.assertTrue( - hasattr(socket, opt), f"Missing RFC3542 socket option '{opt}'" - ) + self.assertHasAttr(socket, opt) def testHostnameRes(self): # Testing hostname resolution mechanisms @@ -1175,7 +1206,10 @@ def testInterfaceNameIndex(self): 'socket.if_indextoname() not available.') @support.skip_android_selinux('if_indextoname') def testInvalidInterfaceIndexToName(self): - self.assertRaises(OSError, socket.if_indextoname, 0) + with self.assertRaises(OSError) as cm: + socket.if_indextoname(0) + self.assertIsNotNone(cm.exception.errno) + self.assertRaises(ValueError, socket.if_indextoname, -1) self.assertRaises(OverflowError, socket.if_indextoname, 2**1000) self.assertRaises(TypeError, socket.if_indextoname, '_DEADBEEF') @@ -1195,8 +1229,11 @@ def testInvalidInterfaceIndexToName(self): 'socket.if_nametoindex() not available.') @support.skip_android_selinux('if_nametoindex') def testInvalidInterfaceNameToIndex(self): + with self.assertRaises(OSError) as cm: + socket.if_nametoindex("_DEADBEEF") + self.assertIsNotNone(cm.exception.errno) + self.assertRaises(TypeError, socket.if_nametoindex, 0) - self.assertRaises(OSError, socket.if_nametoindex, '_DEADBEEF') @unittest.skipUnless(hasattr(sys, 'getrefcount'), 'test needs sys.getrefcount()') @@ -1263,7 +1300,8 @@ def testGetServBy(self): # protocol, at least for modern Linuxes. if ( sys.platform.startswith( - ('linux', 'android', 'freebsd', 'netbsd', 'gnukfreebsd')) + ('linux', 'android', 'freebsd', 'dragonfly', 'netbsd', + 'gnukfreebsd')) or is_apple ): # avoid the 'echo' service on this platform, as there is an @@ -1385,6 +1423,7 @@ def testIPv6toString(self): except OSError as e: if e.winerror == 10022: self.skipTest('IPv6 might not be supported') + raise f = lambda a: inet_pton(AF_INET6, a) assertInvalid = lambda a: self.assertRaises( @@ -1475,6 +1514,7 @@ def testStringToIPv6(self): except OSError as e: if e.winerror == 10022: self.skipTest('IPv6 might not be supported') + raise f = lambda a: inet_ntop(AF_INET6, a) assertInvalid = lambda a: self.assertRaises( @@ -1593,11 +1633,11 @@ def test_getsockaddrarg(self): @unittest.skipUnless(os.name == "nt", "Windows specific") def test_sock_ioctl(self): - self.assertTrue(hasattr(socket.socket, 'ioctl')) - self.assertTrue(hasattr(socket, 'SIO_RCVALL')) - self.assertTrue(hasattr(socket, 'RCVALL_ON')) - self.assertTrue(hasattr(socket, 'RCVALL_OFF')) - self.assertTrue(hasattr(socket, 'SIO_KEEPALIVE_VALS')) + self.assertHasAttr(socket.socket, 'ioctl') + self.assertHasAttr(socket, 'SIO_RCVALL') + self.assertHasAttr(socket, 'RCVALL_ON') + self.assertHasAttr(socket, 'RCVALL_OFF') + self.assertHasAttr(socket, 'SIO_KEEPALIVE_VALS') s = socket.socket() self.addCleanup(s.close) self.assertRaises(ValueError, s.ioctl, -1, None) @@ -2171,6 +2211,24 @@ def test_addressinfo_enum(self): source=_socket) enum._test_simple_enum(CheckedAddressInfo, socket.AddressInfo) + @unittest.skipUnless(hasattr(socket.socket, "sendmsg"),"sendmsg not supported") + def test_sendmsg_reentrant_ancillary_mutation(self): + + class Mut: + def __index__(self): + seq.clear() + return socket.SCM_RIGHTS + + seq = [ + (socket.SOL_SOCKET, Mut(), b'xxxx'), + (socket.SOL_SOCKET, socket.SCM_RIGHTS, b'xxxx'), + ] + + left, right = socket.socketpair() + self.addCleanup(left.close) + self.addCleanup(right.close) + self.assertRaises(OSError, left.sendmsg, [b'x'], seq) + @unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.') class BasicCANTest(unittest.TestCase): @@ -3890,6 +3948,10 @@ def testCMSG_SPACE(self): # Test CMSG_SPACE() with various valid and invalid values, # checking the assumptions used by sendmsg(). toobig = self.socklen_t_limit - socket.CMSG_SPACE(1) + 1 + if SOLARIS and platform.processor() == "sparc": + # On Solaris SPARC, number of bytes returned by socket.CMSG_SPACE + # increases at different lengths; see gh-91214. + toobig -= 3 values = list(range(257)) + list(range(toobig - 257, toobig)) last = socket.CMSG_SPACE(0) @@ -4036,7 +4098,9 @@ def _testFDPassCMSG_LEN(self): self.createAndSendFDs(1) @unittest.skipIf(is_apple, "skipping, see issue #12958") + @unittest.skipIf(SOLARIS, "skipping, see gh-91214") @unittest.skipIf(AIX, "skipping, see issue #22397") + @unittest.skipIf(BSD_COMBINES_SCM_RIGHTS, "skipping, see gh-125860") @requireAttrs(socket, "CMSG_SPACE") def testFDPassSeparate(self): # Pass two FDs in two separate arrays. Arrays may be combined @@ -4047,7 +4111,9 @@ def testFDPassSeparate(self): @testFDPassSeparate.client_skip @unittest.skipIf(is_apple, "skipping, see issue #12958") + @unittest.skipIf(SOLARIS, "skipping, see gh-91214") @unittest.skipIf(AIX, "skipping, see issue #22397") + @unittest.skipIf(BSD_COMBINES_SCM_RIGHTS, "skipping, see gh-125860") def _testFDPassSeparate(self): fd0, fd1 = self.newFDs(2) self.assertEqual( @@ -4060,7 +4126,9 @@ def _testFDPassSeparate(self): len(MSG)) @unittest.skipIf(is_apple, "skipping, see issue #12958") + @unittest.skipIf(SOLARIS, "skipping, see gh-91214") @unittest.skipIf(AIX, "skipping, see issue #22397") + @unittest.skipIf(BSD_COMBINES_SCM_RIGHTS, "skipping, see gh-125860") @requireAttrs(socket, "CMSG_SPACE") def testFDPassSeparateMinSpace(self): # Pass two FDs in two separate arrays, receiving them into the @@ -4074,7 +4142,9 @@ def testFDPassSeparateMinSpace(self): @testFDPassSeparateMinSpace.client_skip @unittest.skipIf(is_apple, "skipping, see issue #12958") + @unittest.skipIf(SOLARIS, "skipping, see gh-91214") @unittest.skipIf(AIX, "skipping, see issue #22397") + @unittest.skipIf(BSD_COMBINES_SCM_RIGHTS, "skipping, see gh-125860") def _testFDPassSeparateMinSpace(self): fd0, fd1 = self.newFDs(2) self.assertEqual( @@ -4199,6 +4269,7 @@ def _testCmsgTrunc0(self): # (but still too small) buffer sizes. @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testCmsgTrunc1(self): self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 1)) @@ -4207,6 +4278,7 @@ def _testCmsgTrunc1(self): self.createAndSendFDs(1) @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testCmsgTrunc2Int(self): # The cmsghdr structure has at least three members, two of # which are ints, so we still shouldn't see any ancillary @@ -4219,6 +4291,7 @@ def _testCmsgTrunc2Int(self): self.createAndSendFDs(1) @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testCmsgTruncLen0Minus1(self): self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), socket.CMSG_LEN(0) - 1)) @@ -4256,6 +4329,7 @@ def checkTruncatedArray(self, ancbuf, maxdata, mindata=0): self.checkFDs(fds) @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testCmsgTruncLen0(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0), maxdata=0) @@ -4264,6 +4338,7 @@ def _testCmsgTruncLen0(self): self.createAndSendFDs(1) @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testCmsgTruncLen0Plus1(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0) + 1, maxdata=1) @@ -4272,6 +4347,7 @@ def _testCmsgTruncLen0Plus1(self): self.createAndSendFDs(2) @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testCmsgTruncLen1(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(SIZEOF_INT), maxdata=SIZEOF_INT) @@ -4282,6 +4358,7 @@ def _testCmsgTruncLen1(self): @skipForRefleakHuntinIf(sys.platform == "darwin", "#80931") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testCmsgTruncLen2Minus1(self): self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(2 * SIZEOF_INT) - 1, maxdata=(2 * SIZEOF_INT) - 1) @@ -4554,6 +4631,7 @@ def _testSingleCmsgTrunc0(self): # (but still too small) buffer sizes. @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testSingleCmsgTrunc1(self): self.checkHopLimitTruncatedHeader(ancbufsize=1) @@ -4563,6 +4641,7 @@ def _testSingleCmsgTrunc1(self): self.sendToServer(MSG) @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testSingleCmsgTrunc2Int(self): self.checkHopLimitTruncatedHeader(ancbufsize=2 * SIZEOF_INT) @@ -4572,6 +4651,7 @@ def _testSingleCmsgTrunc2Int(self): self.sendToServer(MSG) @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testSingleCmsgTruncLen0Minus1(self): self.checkHopLimitTruncatedHeader(ancbufsize=socket.CMSG_LEN(0) - 1) @@ -4581,6 +4661,7 @@ def _testSingleCmsgTruncLen0Minus1(self): self.sendToServer(MSG) @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testSingleCmsgTruncInData(self): # Test truncation of a control message inside its associated # data. The message may be returned with its data truncated, @@ -4653,6 +4734,7 @@ def _testSecondCmsgTrunc0(self): @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", "IPV6_RECVTCLASS", "IPV6_TCLASS") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testSecondCmsgTrunc1(self): self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + 1) @@ -4663,6 +4745,7 @@ def _testSecondCmsgTrunc1(self): @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", "IPV6_RECVTCLASS", "IPV6_TCLASS") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testSecondCmsgTrunc2Int(self): self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + 2 * SIZEOF_INT) @@ -4674,6 +4757,7 @@ def _testSecondCmsgTrunc2Int(self): @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", "IPV6_RECVTCLASS", "IPV6_TCLASS") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testSecondCmsgTruncLen0Minus1(self): self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + socket.CMSG_LEN(0) - 1) @@ -4685,6 +4769,7 @@ def _testSecondCmsgTruncLen0Minus1(self): @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", "IPV6_RECVTCLASS", "IPV6_TCLASS") + @unittest.skipIf(CMSG_TRUNC_RAISES_EMSGSIZE, "skipping, see gh-154240") def testSecondCmsgTruncInData(self): # Test truncation of the second of two control messages inside # its associated data. @@ -6082,10 +6167,10 @@ def testTimeoutZero(self): class TestExceptions(unittest.TestCase): def testExceptionTree(self): - self.assertTrue(issubclass(OSError, Exception)) - self.assertTrue(issubclass(socket.herror, OSError)) - self.assertTrue(issubclass(socket.gaierror, OSError)) - self.assertTrue(issubclass(socket.timeout, OSError)) + self.assertIsSubclass(OSError, Exception) + self.assertIsSubclass(socket.herror, OSError) + self.assertIsSubclass(socket.gaierror, OSError) + self.assertIsSubclass(socket.timeout, OSError) self.assertIs(socket.error, OSError) self.assertIs(socket.timeout, TimeoutError) @@ -7047,8 +7132,14 @@ def test_aes_cbc(self): self.assertEqual(len(dec), msglen * multiplier) self.assertEqual(dec, msg * multiplier) - @support.requires_linux_version(4, 9) # see issue29324 + @support.requires_linux_version(4, 9) # see gh-73510 def test_aead_aes_gcm(self): + kernel_version = support._get_kernel_version("Linux") + if kernel_version is not None: + if kernel_version >= (6, 16) and kernel_version < (6, 18): + # See https://github.com/python/cpython/issues/139310. + self.skipTest("upstream Linux kernel issue") + key = bytes.fromhex('c939cc13397c1d37de6ae0e1cb7c423c') iv = bytes.fromhex('b3d8cc017cbb89b39e0f67e2') plain = bytes.fromhex('c3b3c41f113a31b73d9a5cd432103069') @@ -7387,6 +7478,62 @@ def detach(): pass +class ReentrantMutationTests(unittest.TestCase): + """Regression tests for re-entrant mutation in sendmsg/recvmsg_into. + + These tests verify that mutating sequences during argument parsing + via __buffer__ protocol does not cause crashes. + + See: https://github.com/python/cpython/issues/143988 + """ + + @unittest.skipUnless(hasattr(socket.socket, "sendmsg"), + "sendmsg not supported") + def test_sendmsg_reentrant_data_mutation(self): + seq = [] + + class MutBuffer: + def __init__(self): + self.tripped = False + + def __buffer__(self, flags): + if not self.tripped: + self.tripped = True + seq.clear() + return memoryview(b'Hello') + + seq = [MutBuffer(), b'World', b'Test'] + + left, right = socket.socketpair() + with left, right: + left.sendmsg(seq) + self.assertEqual(right.recv(1024), b'HelloWorldTest') + + @unittest.skipUnless(hasattr(socket.socket, "recvmsg_into"), + "recvmsg_into not supported") + def test_recvmsg_into_reentrant_buffer_mutation(self): + seq = [] + buf1 = bytearray(100) + + class MutBuffer: + def __init__(self): + self.tripped = False + + def __buffer__(self, flags): + if not self.tripped: + self.tripped = True + seq.clear() + return memoryview(buf1) + + seq = [MutBuffer(), bytearray(100), bytearray(100)] + + left, right = socket.socketpair() + with left, right: + left.send(b'Hello World!') + right.recvmsg_into(seq) + self.assertEqual(buf1, b'Hello World!'.ljust(100, b'\x00')) + + def setUpModule(): thread_info = threading_helper.threading_setup() unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info) diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 0f62f9eb200e42d..2ca356606b260cb 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -218,12 +218,16 @@ def test_ForkingUDPServer(self): self.dgram_examine) @requires_unix_sockets + @unittest.skipIf(test.support.is_apple_mobile and test.support.on_github_actions, + "gh-140702: Test fails regularly on iOS simulator on GitHub Actions") def test_UnixDatagramServer(self): self.run_server(socketserver.UnixDatagramServer, socketserver.DatagramRequestHandler, self.dgram_examine) @requires_unix_sockets + @unittest.skipIf(test.support.is_apple_mobile and test.support.on_github_actions, + "gh-140702: Test fails regularly on iOS simulator on GitHub Actions") def test_ThreadingUnixDatagramServer(self): self.run_server(socketserver.ThreadingUnixDatagramServer, socketserver.DatagramRequestHandler, diff --git a/Lib/test/test_source_encoding.py b/Lib/test/test_source_encoding.py index 61b00778f8361c6..3481bc5af798502 100644 --- a/Lib/test/test_source_encoding.py +++ b/Lib/test/test_source_encoding.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- import unittest -from test.support import script_helper, captured_stdout, requires_subprocess, requires_resource +from test import support +from test.support import script_helper from test.support.os_helper import TESTFN, unlink, rmtree from test.support.import_helper import unload import importlib @@ -64,7 +65,24 @@ def test_issue7820(self): # two bytes in common with the UTF-8 BOM self.assertRaises(SyntaxError, eval, b'\xef\xbb\x20') - @requires_subprocess() + def test_truncated_utf8_at_eof(self): + # Regression test for https://issues.oss-fuzz.com/issues/451112368 + # Truncated multi-byte UTF-8 sequences at end of input caused an + # out-of-bounds read in Parser/tokenizer/helpers.c:valid_utf8(). + truncated = [ + b'\xc2', # 2-byte lead, missing 1 continuation + b'\xdf', # 2-byte lead, missing 1 continuation + b'\xe0', # 3-byte lead, missing 2 continuations + b'\xe0\xa0', # 3-byte lead, missing 1 continuation + b'\xf0\x90', # 4-byte lead, missing 2 continuations + b'\xf0\x90\x80', # 4-byte lead, missing 1 continuation + b'\xf3', # 4-byte lead, missing 3 (the oss-fuzz reproducer) + ] + for seq in truncated: + with self.subTest(seq=seq): + self.assertRaises(SyntaxError, compile, seq, '<test>', 'exec') + + @support.requires_subprocess() def test_20731(self): sub = subprocess.Popen([sys.executable, os.path.join(os.path.dirname(__file__), @@ -145,8 +163,7 @@ def test_error_from_string(self): compile(input, "<string>", "exec") expected = "'ascii' codec can't decode byte 0xe2 in position 16: " \ "ordinal not in range(128)" - self.assertTrue(c.exception.args[0].startswith(expected), - msg=c.exception.args[0]) + self.assertStartsWith(c.exception.args[0], expected) def test_file_parse_error_multiline(self): # gh96611: @@ -173,6 +190,8 @@ def test_tokenizer_fstring_warning_in_first_line(self): os.unlink(TESTFN) +BUFSIZ = 2**13 + class AbstractSourceEncodingTest: def test_default_coding(self): @@ -185,14 +204,20 @@ def test_first_coding_line(self): self.check_script_output(src, br"'\xc3\u20ac'") def test_second_coding_line(self): - src = (b'#\n' + src = (b'#!/usr/bin/python\n' + b'#coding:iso8859-15\n' + b'print(ascii("\xc3\xa4"))\n') + self.check_script_output(src, br"'\xc3\u20ac'") + + def test_second_coding_line_empty_first_line(self): + src = (b'\n' b'#coding:iso8859-15\n' b'print(ascii("\xc3\xa4"))\n') self.check_script_output(src, br"'\xc3\u20ac'") def test_third_coding_line(self): # Only first two lines are tested for a magic comment. - src = (b'#\n' + src = (b'#!/usr/bin/python\n' b'#\n' b'#coding:iso8859-15\n' b'print(ascii("\xc3\xa4"))\n') @@ -210,48 +235,196 @@ def test_double_coding_same_line(self): b'print(ascii("\xc3\xa4"))\n') self.check_script_output(src, br"'\xc3\u20ac'") + def test_double_coding_utf8(self): + src = (b'#coding:utf-8\n' + b'#coding:latin1\n' + b'print(ascii("\xc3\xa4"))\n') + self.check_script_output(src, br"'\xe4'") + + def test_long_first_coding_line(self): + src = (b'#' + b' '*BUFSIZ + b'coding:iso8859-15\n' + b'print(ascii("\xc3\xa4"))\n') + self.check_script_output(src, br"'\xc3\u20ac'") + + def test_long_second_coding_line(self): + src = (b'#!/usr/bin/python\n' + b'#' + b' '*BUFSIZ + b'coding:iso8859-15\n' + b'print(ascii("\xc3\xa4"))\n') + self.check_script_output(src, br"'\xc3\u20ac'") + + def test_long_coding_line(self): + src = (b'#coding:iso-8859-15' + b' '*BUFSIZ + b'\n' + b'print(ascii("\xc3\xa4"))\n') + self.check_script_output(src, br"'\xc3\u20ac'") + + def test_long_coding_name(self): + src = (b'#coding:iso-8859-1-' + b'x'*BUFSIZ + b'\n' + b'print(ascii("\xc3\xa4"))\n') + self.check_script_output(src, br"'\xc3\xa4'") + + def test_long_first_utf8_line(self): + src = b'#' + b'\xc3\xa4'*(BUFSIZ//2) + b'\n' + self.check_script_output(src, b'') + src = b'# ' + b'\xc3\xa4'*(BUFSIZ//2) + b'\n' + self.check_script_output(src, b'') + + def test_long_second_utf8_line(self): + src = b'\n#' + b'\xc3\xa4'*(BUFSIZ//2) + b'\n' + self.check_script_output(src, b'') + src = b'\n# ' + b'\xc3\xa4'*(BUFSIZ//2) + b'\n' + self.check_script_output(src, b'') + def test_first_non_utf8_coding_line(self): src = (b'#coding:iso-8859-15 \xa4\n' b'print(ascii("\xc3\xa4"))\n') self.check_script_output(src, br"'\xc3\u20ac'") def test_second_non_utf8_coding_line(self): - src = (b'\n' + src = (b'#!/usr/bin/python\n' b'#coding:iso-8859-15 \xa4\n' b'print(ascii("\xc3\xa4"))\n') self.check_script_output(src, br"'\xc3\u20ac'") + def test_first_utf8_coding_line_error(self): + src = (b'#coding:ascii \xc3\xa4\n' + b'raise RuntimeError\n') + self.check_script_error(src, br"(\(unicode error\) )?'ascii' codec can't decode byte") + + def test_second_utf8_coding_line_error(self): + src = (b'#!/usr/bin/python\n' + b'#coding:ascii \xc3\xa4\n' + b'raise RuntimeError\n') + self.check_script_error(src, br"(\(unicode error\) )?'ascii' codec can't decode byte") + def test_utf8_bom(self): src = (b'\xef\xbb\xbfprint(ascii("\xc3\xa4"))\n') self.check_script_output(src, br"'\xe4'") + def test_utf8_bom_utf8_comments(self): + src = (b'\xef\xbb\xbf#\xc3\xa4\n' + b'#\xc3\xa4\n' + b'print(ascii("\xc3\xa4"))\n') + self.check_script_output(src, br"'\xe4'") + def test_utf8_bom_and_utf8_coding_line(self): src = (b'\xef\xbb\xbf#coding:utf-8\n' b'print(ascii("\xc3\xa4"))\n') self.check_script_output(src, br"'\xe4'") + def test_utf8_bom_and_non_utf8_first_coding_line(self): + src = (b'\xef\xbb\xbf#coding:iso-8859-15\n' + b'raise RuntimeError\n') + self.check_script_error(src, + br"encoding problem: iso-8859-15 with BOM", + lineno=1) + + def test_utf8_bom_and_non_utf8_second_coding_line(self): + src = (b'\xef\xbb\xbf#first\n' + b'#coding:iso-8859-15\n' + b'raise RuntimeError\n') + self.check_script_error(src, + br"encoding problem: iso-8859-15 with BOM", + lineno=2) + + def test_non_utf8_shebang(self): + src = (b'#!/home/\xa4/bin/python\n' + b'#coding:iso-8859-15\n' + b'print(ascii("\xc3\xa4"))\n') + self.check_script_output(src, br"'\xc3\u20ac'") + + def test_utf8_shebang_error(self): + src = (b'#!/home/\xc3\xa4/bin/python\n' + b'#coding:ascii\n' + b'raise RuntimeError\n') + self.check_script_error(src, br"(\(unicode error\) )?'ascii' codec can't decode byte") + + def test_non_utf8_shebang_error(self): + src = (b'#!/home/\xa4/bin/python\n' + b'raise RuntimeError\n') + self.check_script_error(src, br"Non-UTF-8 code starting with .* on line 1", + lineno=1) + + def test_non_utf8_second_line_error(self): + src = (b'#first\n' + b'#second\xa4\n' + b'raise RuntimeError\n') + self.check_script_error(src, + br"Non-UTF-8 code starting with .* on line 2", + lineno=2) + + def test_non_utf8_third_line_error(self): + src = (b'#first\n' + b'#second\n' + b'#third\xa4\n' + b'raise RuntimeError\n') + self.check_script_error(src, + br"Non-UTF-8 code starting with .* on line 3", + lineno=3) + + def test_utf8_bom_non_utf8_third_line_error(self): + src = (b'\xef\xbb\xbf#first\n' + b'#second\n' + b'#third\xa4\n' + b'raise RuntimeError\n') + self.check_script_error(src, + br"Non-UTF-8 code starting with .* on line 3|" + br"'utf-8' codec can't decode byte", + lineno=3) + + def test_utf_8_non_utf8_third_line_error(self): + src = (b'#coding: utf-8\n' + b'#second\n' + b'#third\xa4\n' + b'raise RuntimeError\n') + self.check_script_error(src, + br"Non-UTF-8 code starting with .* on line 3|" + br"'utf-8' codec can't decode byte", + lineno=3) + + def test_utf8_non_utf8_third_line_error(self): + src = (b'#coding: utf8\n' + b'#second\n' + b'#third\xa4\n' + b'raise RuntimeError\n') + self.check_script_error(src, + br"'utf-8' codec can't decode byte") + def test_crlf(self): src = (b'print(ascii("""\r\n"""))\n') - out = self.check_script_output(src, br"'\n'") + self.check_script_output(src, br"'\n'") def test_crcrlf(self): src = (b'print(ascii("""\r\r\n"""))\n') - out = self.check_script_output(src, br"'\n\n'") + self.check_script_output(src, br"'\n\n'") def test_crcrcrlf(self): src = (b'print(ascii("""\r\r\r\n"""))\n') - out = self.check_script_output(src, br"'\n\n\n'") + self.check_script_output(src, br"'\n\n\n'") def test_crcrcrlf2(self): src = (b'#coding:iso-8859-1\n' b'print(ascii("""\r\r\r\n"""))\n') - out = self.check_script_output(src, br"'\n\n\n'") + self.check_script_output(src, br"'\n\n\n'") + + def test_nul_in_first_coding_line(self): + src = (b'#coding:iso8859-15\x00\n' + b'\n' + b'\n' + b'raise RuntimeError\n') + self.check_script_error(src, br"source code (string )?cannot contain null bytes") + + def test_nul_in_second_coding_line(self): + src = (b'#!/usr/bin/python\n' + b'#coding:iso8859-15\x00\n' + b'\n' + b'raise RuntimeError\n') + self.check_script_error(src, br"source code (string )?cannot contain null bytes") class UTF8ValidatorTest(unittest.TestCase): @unittest.skipIf(not sys.platform.startswith("linux"), "Too slow to run on non-Linux platforms") - @requires_resource('cpu') + @support.requires_resource('cpu') def test_invalid_utf8(self): # This is a port of test_utf8_decode_invalid_sequences in # test_unicode.py to exercise the separate utf8 validator in @@ -317,15 +490,29 @@ def check(content): check(b'\xF4'+cb+b'\xBF\xBF') +@support.force_not_colorized_test_class class BytesSourceEncodingTest(AbstractSourceEncodingTest, unittest.TestCase): def check_script_output(self, src, expected): - with captured_stdout() as stdout: + with support.captured_stdout() as stdout: exec(src) out = stdout.getvalue().encode('latin1') self.assertEqual(out.rstrip(), expected) + def check_script_error(self, src, expected, lineno=...): + with self.assertRaises(SyntaxError) as cm: + exec(src) + exc = cm.exception + self.assertRegex(str(exc), expected.decode()) + if lineno is not ...: + self.assertEqual(exc.lineno, lineno) + line = src.splitlines()[lineno-1].decode(errors='replace') + if lineno == 1: + line = line.removeprefix('\ufeff') + self.assertEqual(line, exc.text) + +@support.force_not_colorized_test_class class FileSourceEncodingTest(AbstractSourceEncodingTest, unittest.TestCase): def check_script_output(self, src, expected): @@ -336,6 +523,38 @@ def check_script_output(self, src, expected): res = script_helper.assert_python_ok(fn) self.assertEqual(res.out.rstrip(), expected) + def check_script_error(self, src, expected, lineno=...): + with tempfile.TemporaryDirectory() as tmpd: + fn = os.path.join(tmpd, 'test.py') + with open(fn, 'wb') as fp: + fp.write(src) + res = script_helper.assert_python_failure(fn) + err = res.err.rstrip() + self.assertRegex(err.splitlines()[-1], b'SyntaxError: ' + expected) + if lineno is not ...: + self.assertIn(f', line {lineno}\n'.encode(), + err.replace(os.linesep.encode(), b'\n')) + line = src.splitlines()[lineno-1].decode(errors='replace') + if lineno == 1: + line = line.removeprefix('\ufeff') + line = line.encode(sys.__stderr__.encoding, sys.__stderr__.errors) + self.assertIn(line, err) + + def test_coding_spec_unknown_encoding(self): + src = (b'# coding: c1252\n' + b'print("Hi!")\n') + self.check_script_error(src, br"unknown encoding: c1252") + + def test_coding_spec_decode_error(self): + src = (b'# coding: shift-jis\n' + b'print("\xc4\x85")\n') + self.check_script_error(src, br"'shift_jis' codec can't decode byte") + + def test_coding_spec_non_text_encoding(self): + src = (b'# coding: hex_codec\n' + b'print("eggs")\n') + self.check_script_error(src, br"'hex_codec' is not a text encoding") + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_sqlite3/__init__.py b/Lib/test/test_sqlite3/__init__.py index d777fca82da4b0d..145f3b800248290 100644 --- a/Lib/test/test_sqlite3/__init__.py +++ b/Lib/test/test_sqlite3/__init__.py @@ -6,10 +6,14 @@ import os import sqlite3 +# make sure only print once +_printed_version = False + # Implement the unittest "load tests" protocol. -def load_tests(*args): +def load_tests(loader, tests, pattern): + global _printed_version + if verbose and not _printed_version: + print(f"test_sqlite3: testing with SQLite version {sqlite3.sqlite_version}") + _printed_version = True pkg_dir = os.path.dirname(__file__) - return load_package_tests(pkg_dir, *args) - -if verbose: - print(f"test_sqlite3: testing with SQLite version {sqlite3.sqlite_version}") + return load_package_tests(pkg_dir, loader, tests, pattern) diff --git a/Lib/test/test_sqlite3/test_cli.py b/Lib/test/test_sqlite3/test_cli.py index ad0dcb3cccb5dab..a03d7cbe16ba847 100644 --- a/Lib/test/test_sqlite3/test_cli.py +++ b/Lib/test/test_sqlite3/test_cli.py @@ -116,6 +116,38 @@ def test_interact_version(self): self.assertEqual(out.count(self.PS2), 0) self.assertIn(sqlite3.sqlite_version, out) + def test_interact_empty_source(self): + out, err = self.run_cli(commands=("", " ")) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 3) + self.assertEqual(out.count(self.PS2), 0) + + def test_interact_dot_commands_unknown(self): + out, err = self.run_cli(commands=(".unknown_command", )) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 2) + self.assertEqual(out.count(self.PS2), 0) + self.assertIn("Error", err) + # test "unknown_command" is pointed out in the error message + self.assertIn("unknown_command", err) + + def test_interact_dot_commands_empty(self): + out, err = self.run_cli(commands=(".")) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 2) + self.assertEqual(out.count(self.PS2), 0) + + def test_interact_dot_commands_with_whitespaces(self): + out, err = self.run_cli(commands=(".version ", ". version")) + self.assertIn(self.MEMORY_DB_MSG, err) + self.assertEqual(out.count(sqlite3.sqlite_version + "\n"), 2) + self.assertEndsWith(out, self.PS1) + self.assertEqual(out.count(self.PS1), 3) + self.assertEqual(out.count(self.PS2), 0) + def test_interact_valid_sql(self): out, err = self.run_cli(commands=("SELECT 1;",)) self.assertIn(self.MEMORY_DB_MSG, err) diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index c3aa3bf2d7bd9f0..7165729cd524f01 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -21,6 +21,7 @@ # 3. This notice may not be removed or altered from any source distribution. import contextlib +import functools import os import sqlite3 as sqlite import subprocess @@ -31,8 +32,7 @@ import warnings from test.support import ( - SHORT_TIMEOUT, check_disallow_instantiation, requires_subprocess, - is_apple, is_emscripten, is_wasi + SHORT_TIMEOUT, check_disallow_instantiation, requires_subprocess ) from test.support import gc_collect from test.support import threading_helper, import_helper @@ -639,6 +639,14 @@ def test_deserialize_corrupt_database(self): class OpenTests(unittest.TestCase): _sql = "create table test(id integer)" + def test_open_with_bytes_path(self): + path = os.fsencode(TESTFN) + self.addCleanup(unlink, path) + self.assertFalse(os.path.exists(path)) + with contextlib.closing(sqlite.connect(path)) as cx: + self.assertTrue(os.path.exists(path)) + cx.execute(self._sql) + def test_open_with_path_like_object(self): """ Checks that we can successfully connect to a database using an object that is PathLike, i.e. has __fspath__(). """ @@ -649,14 +657,21 @@ def test_open_with_path_like_object(self): self.assertTrue(os.path.exists(path)) cx.execute(self._sql) + def get_undecodable_path(self): + path = TESTFN_UNDECODABLE + if not path: + self.skipTest("only works if there are undecodable paths") + try: + open(path, 'wb').close() + except OSError: + self.skipTest(f"can't create file with undecodable path {path!r}") + unlink(path) + return path + @unittest.skipIf(sys.platform == "win32", "skipped on Windows") - @unittest.skipIf(is_apple, "skipped on Apple platforms") - @unittest.skipIf(is_emscripten or is_wasi, "not supported on Emscripten/WASI") - @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") def test_open_with_undecodable_path(self): - path = TESTFN_UNDECODABLE + path = self.get_undecodable_path() self.addCleanup(unlink, path) - self.assertFalse(os.path.exists(path)) with contextlib.closing(sqlite.connect(path)) as cx: self.assertTrue(os.path.exists(path)) cx.execute(self._sql) @@ -696,14 +711,10 @@ def test_open_uri_readonly(self): cx.execute(self._sql) @unittest.skipIf(sys.platform == "win32", "skipped on Windows") - @unittest.skipIf(is_apple, "skipped on Apple platforms") - @unittest.skipIf(is_emscripten or is_wasi, "not supported on Emscripten/WASI") - @unittest.skipUnless(TESTFN_UNDECODABLE, "only works if there are undecodable paths") def test_open_undecodable_uri(self): - path = TESTFN_UNDECODABLE + path = self.get_undecodable_path() self.addCleanup(unlink, path) uri = "file:" + urllib.parse.quote(path) - self.assertFalse(os.path.exists(path)) with contextlib.closing(sqlite.connect(uri, uri=True)) as cx: self.assertTrue(os.path.exists(path)) cx.execute(self._sql) @@ -1058,7 +1069,7 @@ def test_array_size(self): # now set to 2 self.cu.arraysize = 2 - # now make the query return 3 rows + # now make the query return 2 rows from a table of 3 rows self.cu.execute("delete from test") self.cu.execute("insert into test(name) values ('A')") self.cu.execute("insert into test(name) values ('B')") @@ -1068,13 +1079,50 @@ def test_array_size(self): self.assertEqual(len(res), 2) + def test_invalid_array_size(self): + UINT32_MAX = (1 << 32) - 1 + setter = functools.partial(setattr, self.cu, 'arraysize') + + self.assertRaises(TypeError, setter, 1.0) + self.assertRaises(ValueError, setter, -3) + self.assertRaises(OverflowError, setter, UINT32_MAX + 1) + def test_fetchmany(self): + # no active SQL statement + res = self.cu.fetchmany() + self.assertEqual(res, []) + res = self.cu.fetchmany(1000) + self.assertEqual(res, []) + + # test default parameter self.cu.execute("select name from test") + res = self.cu.fetchmany() + self.assertEqual(len(res), 1) + + # test when the number of requested rows exceeds the actual count + self.cu.execute("select name from test") + res = self.cu.fetchmany(100) + self.assertEqual(len(res), 1) + res = self.cu.fetchmany(100) + self.assertEqual(res, []) + + # test when size = 0 + self.cu.execute("select name from test") + res = self.cu.fetchmany(0) + self.assertEqual(res, []) res = self.cu.fetchmany(100) self.assertEqual(len(res), 1) res = self.cu.fetchmany(100) self.assertEqual(res, []) + def test_invalid_fetchmany(self): + UINT32_MAX = (1 << 32) - 1 + fetchmany = self.cu.fetchmany + + self.assertRaises(TypeError, fetchmany, 1.0) + self.assertRaises(ValueError, fetchmany, -3) + self.assertRaises(OverflowError, fetchmany, UINT32_MAX + 1) + def test_fetchmany_kw_arg(self): """Checks if fetchmany works with keyword arguments""" self.cu.execute("select name from test") @@ -1339,6 +1387,11 @@ def test_blob_get_slice(self): def test_blob_get_empty_slice(self): self.assertEqual(self.blob[5:5], b"") + def test_blob_get_empty_slice_oob_indices(self): + self.cx.execute("insert into test(b) values (?)", (b"abc",)) + with self.cx.blobopen("test", "b", 2) as blob: + self.assertEqual(blob[5:-5], b"") + def test_blob_get_slice_negative_index(self): self.assertEqual(self.blob[5:-5], self.data[5:-5]) @@ -1355,6 +1408,18 @@ def test_blob_set_empty_slice(self): self.blob[0:0] = b"" self.assertEqual(self.blob[:], self.data) + def test_blob_set_empty_slice_wrong_type(self): + with self.assertRaises(TypeError): + self.blob[5:5] = None + + def test_blob_set_empty_slice_wrong_size(self): + with self.assertRaisesRegex(IndexError, "wrong size"): + self.blob[5:5] = b"123" + + def test_blob_set_empty_slice_correct(self): + self.blob[5:5] = b"" + self.assertEqual(self.blob[:], self.data) + def test_blob_set_slice_with_skip(self): self.blob[0:10:2] = b"12345" actual = self.cx.execute("select b from test").fetchone()[0] diff --git a/Lib/test/test_sqlite3/test_dump.py b/Lib/test/test_sqlite3/test_dump.py index e18a207e9f64949..7841b610cd222a7 100644 --- a/Lib/test/test_sqlite3/test_dump.py +++ b/Lib/test/test_sqlite3/test_dump.py @@ -56,6 +56,24 @@ def test_table_dump(self): [self.assertEqual(expected_sqls[i], actual_sqls[i]) for i in range(len(expected_sqls))] + def test_dump_single_quote_in_identifier(self): + # A single quote in a table or column name must not break the dump. + self.cu.execute("""CREATE TABLE "a'b" ("c'd" text);""") + self.cu.execute("""INSERT INTO "a'b" VALUES('x''y');""") + expected = [ + "BEGIN TRANSACTION;", + """CREATE TABLE "a'b" ("c'd" text);""", + """INSERT INTO "a'b" VALUES('x''y');""", + "COMMIT;", + ] + actual = list(self.cx.iterdump()) + self.assertEqual(expected, actual) + # The dump restores into a fresh database. + with memory_database() as cx2: + cx2.executescript("".join(actual)) + row = cx2.execute("""SELECT "c'd" FROM "a'b";""").fetchone() + self.assertEqual(row[0], "x'y") + def test_table_dump_filter(self): all_table_sqls = [ """CREATE TABLE "some_table_2" ("id_1" INTEGER);""", diff --git a/Lib/test/test_sqlite3/test_factory.py b/Lib/test/test_sqlite3/test_factory.py index cc9f1ec5c4bec5e..e62891635e2fad4 100644 --- a/Lib/test/test_sqlite3/test_factory.py +++ b/Lib/test/test_sqlite3/test_factory.py @@ -155,6 +155,16 @@ def test_sqlite_row_index(self): with self.assertRaises(IndexError): row[complex()] # index must be int or string + def test_delete_connection_row_factory(self): + # gh-149738: deleting row_factory should raise an exception + with self.assertRaises(AttributeError): + del self.con.row_factory + + def test_delete_connection_text_factory(self): + # gh-149738: deleting text_factory should raise an exception + with self.assertRaises(AttributeError): + del self.con.text_factory + def test_sqlite_row_index_unicode(self): row = self.con.execute("select 1 as \xff").fetchone() self.assertEqual(row["\xff"], 1) @@ -238,6 +248,17 @@ def test_sqlite_row_hash_cmp(self): self.assertEqual(hash(row_1), hash(row_2)) + def test_sqlite_row_hash_unhashable(self): + # An unhashable value must raise TypeError, not SystemError. + sqlite.register_converter("LST", lambda b: [1, 2, 3]) + self.addCleanup(sqlite.converters.pop, "LST", None) + with memory_database(detect_types=sqlite.PARSE_DECLTYPES) as con: + con.row_factory = sqlite.Row + con.execute("create table t(x LST)") + con.execute("insert into t values(?)", (b"x",)) + row = con.execute("select x from t").fetchone() + self.assertRaisesRegex(TypeError, "unhashable", hash, row) + def test_sqlite_row_as_sequence(self): # Checks if the row object can act like a sequence. row = self.con.execute("select 1 as a, 2 as b").fetchone() diff --git a/Lib/test/test_sqlite3/test_hooks.py b/Lib/test/test_sqlite3/test_hooks.py index 53b8a39bf29a75f..36a1cb9761d1d5a 100644 --- a/Lib/test/test_sqlite3/test_hooks.py +++ b/Lib/test/test_sqlite3/test_hooks.py @@ -24,11 +24,15 @@ import sqlite3 as sqlite import unittest +from test.support import import_helper from test.support.os_helper import TESTFN, unlink from .util import memory_database, cx_limit, with_tracebacks from .util import MemoryDatabaseMixin +# TODO(picnixz): increase test coverage for other callbacks +# such as 'func', 'step', 'finalize', and 'collation'. + class CollationTests(MemoryDatabaseMixin, unittest.TestCase): @@ -116,6 +120,21 @@ def test_collation_register_twice(self): self.assertEqual(result[0][0], 'b') self.assertEqual(result[1][0], 'a') + def test_collation_register_when_busy(self): + # See https://github.com/python/cpython/issues/146090. + con = self.con + con.create_collation("mycoll", lambda x, y: (x > y) - (x < y)) + con.execute("CREATE TABLE t(x TEXT)") + con.execute("INSERT INTO t VALUES (?)", ("a",)) + con.execute("INSERT INTO t VALUES (?)", ("b",)) + con.commit() + + cursor = self.con.execute("SELECT x FROM t ORDER BY x COLLATE mycoll") + next(cursor) + # Replace the collation while the statement is active -> SQLITE_BUSY. + with self.assertRaises(sqlite.OperationalError) as cm: + self.con.create_collation("mycoll", lambda a, b: 0) + def test_deregister_collation(self): """ Register a collation, then deregister it. Make sure an error is raised if we try @@ -129,8 +148,55 @@ def test_deregister_collation(self): self.assertEqual(str(cm.exception), 'no such collation sequence: mycoll') +class AuthorizerTests(MemoryDatabaseMixin, unittest.TestCase): + + def assert_not_authorized(self, func, /, *args, **kwargs): + with self.assertRaisesRegex(sqlite.DatabaseError, "not authorized"): + func(*args, **kwargs) + + # When a handler has an invalid signature, the exception raised is + # the same that would be raised if the handler "negatively" replied. + + def test_authorizer_invalid_signature(self): + self.cx.execute("create table if not exists test(a number)") + self.cx.set_authorizer(lambda: None) + self.assert_not_authorized(self.cx.execute, "select * from test") + + # Tests for checking that callback context mutations do not crash. + # Regression tests for https://github.com/python/cpython/issues/142830. + + @with_tracebacks(ZeroDivisionError, regex="hello world") + def test_authorizer_concurrent_mutation_in_call(self): + self.cx.execute("create table if not exists test(a number)") + + def handler(*a, **kw): + self.cx.set_authorizer(None) + raise ZeroDivisionError("hello world") + + self.cx.set_authorizer(handler) + self.assert_not_authorized(self.cx.execute, "select * from test") + + @with_tracebacks(OverflowError) + def test_authorizer_concurrent_mutation_with_overflown_value(self): + _testcapi = import_helper.import_module("_testcapi") + self.cx.execute("create table if not exists test(a number)") + + def handler(*a, **kw): + self.cx.set_authorizer(None) + # We expect 'int' at the C level, so this one will raise + # when converting via PyLong_Int(). + return _testcapi.INT_MAX + 1 + + self.cx.set_authorizer(handler) + self.assert_not_authorized(self.cx.execute, "select * from test") + + class ProgressTests(MemoryDatabaseMixin, unittest.TestCase): + def assert_interrupted(self, func, /, *args, **kwargs): + with self.assertRaisesRegex(sqlite.OperationalError, "interrupted"): + func(*args, **kwargs) + def test_progress_handler_used(self): """ Test that the progress handler is invoked once it is set. @@ -219,7 +285,7 @@ def bad_progress(): create table foo(a, b) """) - def test_progress_handler_keyword_args(self): + def test_set_progress_handler_keyword_args(self): regex = ( r"Passing keyword argument 'progress_handler' to " r"_sqlite3.Connection.set_progress_handler\(\) is deprecated. " @@ -231,6 +297,43 @@ def test_progress_handler_keyword_args(self): self.con.set_progress_handler(progress_handler=lambda: None, n=1) self.assertEqual(cm.filename, __file__) + # When a handler has an invalid signature, the exception raised is + # the same that would be raised if the handler "negatively" replied. + + def test_progress_handler_invalid_signature(self): + self.cx.execute("create table if not exists test(a number)") + self.cx.set_progress_handler(lambda x: None, 1) + self.assert_interrupted(self.cx.execute, "select * from test") + + # Tests for checking that callback context mutations do not crash. + # Regression tests for https://github.com/python/cpython/issues/142830. + + @with_tracebacks(ZeroDivisionError, regex="hello world") + def test_progress_handler_concurrent_mutation_in_call(self): + self.cx.execute("create table if not exists test(a number)") + + def handler(*a, **kw): + self.cx.set_progress_handler(None, 1) + raise ZeroDivisionError("hello world") + + self.cx.set_progress_handler(handler, 1) + self.assert_interrupted(self.cx.execute, "select * from test") + + def test_progress_handler_concurrent_mutation_in_conversion(self): + self.cx.execute("create table if not exists test(a number)") + + class Handler: + def __bool__(_): + # clear the progress handler + self.cx.set_progress_handler(None, 1) + raise ValueError # force PyObject_True() to fail + + self.cx.set_progress_handler(Handler.__init__, 1) + self.assert_interrupted(self.cx.execute, "select * from test") + + # Running with tracebacks makes the second execution of this + # function raise another exception because of a database change. + class TraceCallbackTests(MemoryDatabaseMixin, unittest.TestCase): @@ -352,7 +455,7 @@ def test_trace_bad_handler(self): cx.set_trace_callback(lambda stmt: 5/0) cx.execute("select 1") - def test_trace_keyword_args(self): + def test_set_trace_callback_keyword_args(self): regex = ( r"Passing keyword argument 'trace_callback' to " r"_sqlite3.Connection.set_trace_callback\(\) is deprecated. " @@ -364,6 +467,35 @@ def test_trace_keyword_args(self): self.con.set_trace_callback(trace_callback=lambda: None) self.assertEqual(cm.filename, __file__) + # When a handler has an invalid signature, the exception raised is + # the same that would be raised if the handler "negatively" replied, + # but for the trace handler, exceptions are never re-raised (only + # printed when needed). + + @with_tracebacks( + TypeError, + regex=r".*<lambda>\(\) missing 6 required positional arguments", + ) + def test_trace_handler_invalid_signature(self): + self.cx.execute("create table if not exists test(a number)") + self.cx.set_trace_callback(lambda x, y, z, t, a, b, c: None) + self.cx.execute("select * from test") + + # Tests for checking that callback context mutations do not crash. + # Regression tests for https://github.com/python/cpython/issues/142830. + + @with_tracebacks(ZeroDivisionError, regex="hello world") + def test_trace_callback_concurrent_mutation_in_call(self): + self.cx.execute("create table if not exists test(a number)") + + def handler(statement): + # clear the progress handler + self.cx.set_trace_callback(None) + raise ZeroDivisionError("hello world") + + self.cx.set_trace_callback(handler) + self.cx.execute("select * from test") + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 395b2ef88ab6220..91bfb408bb9965d 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1,5 +1,6 @@ # Test the support for SSL and sockets +import contextlib import sys import unittest import unittest.mock @@ -31,6 +32,7 @@ import platform import sysconfig import functools +from contextlib import nullcontext try: import ctypes except ImportError: @@ -46,9 +48,20 @@ PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) HOST = socket_helper.HOST +IS_AWS_LC = "AWS-LC" in ssl.OPENSSL_VERSION IS_OPENSSL_3_0_0 = ssl.OPENSSL_VERSION_INFO >= (3, 0, 0) PY_SSL_DEFAULT_CIPHERS = sysconfig.get_config_var('PY_SSL_DEFAULT_CIPHERS') +HAS_KEYLOG = hasattr(ssl.SSLContext, 'keylog_filename') +requires_keylog = unittest.skipUnless( + HAS_KEYLOG, 'test requires OpenSSL 1.1.1 with keylog callback') +CAN_SET_KEYLOG = HAS_KEYLOG and os.name != "nt" +requires_keylog_setter = unittest.skipUnless( + CAN_SET_KEYLOG, + "cannot set 'keylog_filename' on Windows" +) + + PROTOCOL_TO_TLS_VERSION = {} for proto, ver in ( ("PROTOCOL_SSLv3", "SSLv3"), @@ -257,26 +270,67 @@ def utc_offset(): #NOTE: ignore issues like #1647654 ) -def test_wrap_socket(sock, *, - cert_reqs=ssl.CERT_NONE, ca_certs=None, - ciphers=None, certfile=None, keyfile=None, - **kwargs): - if not kwargs.get("server_side"): - kwargs["server_hostname"] = SIGNED_CERTFILE_HOSTNAME - context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - else: +def make_test_context( + *, + server_side=False, + check_hostname=None, + cert_reqs=ssl.CERT_NONE, + ca_certs=None, certfile=None, keyfile=None, + ciphers=None, + min_version=None, max_version=None, +): + if server_side: context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - if cert_reqs is not None: + else: + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + + if check_hostname is None: if cert_reqs == ssl.CERT_NONE: context.check_hostname = False + else: + context.check_hostname = check_hostname + + if cert_reqs is not None: context.verify_mode = cert_reqs + if ca_certs is not None: context.load_verify_locations(ca_certs) if certfile is not None or keyfile is not None: context.load_cert_chain(certfile, keyfile) + if ciphers is not None: context.set_ciphers(ciphers) - return context.wrap_socket(sock, **kwargs) + + if min_version is not None: + context.minimum_version = min_version + if max_version is not None: + context.maximum_version = max_version + + return context + + +def test_wrap_socket( + sock, + *, + server_side=False, + check_hostname=None, + cert_reqs=ssl.CERT_NONE, + ca_certs=None, certfile=None, keyfile=None, + ciphers=None, + min_version=None, max_version=None, + **kwargs, +): + context = make_test_context( + server_side=server_side, + check_hostname=check_hostname, + cert_reqs=cert_reqs, + ca_certs=ca_certs, certfile=certfile, keyfile=keyfile, + ciphers=ciphers, + min_version=min_version, max_version=max_version, + ) + if not server_side: + kwargs.setdefault("server_hostname", SIGNED_CERTFILE_HOSTNAME) + return context.wrap_socket(sock, server_side=server_side, **kwargs) USE_SAME_TEST_CONTEXT = False @@ -316,6 +370,20 @@ def testing_context(server_cert=SIGNED_CERTFILE, *, server_chain=True): return client_context, server_context, hostname +def do_ssl_object_handshake(sslobject, outgoing, max_retry=25): + """Call do_handshake() on the sslobject and return the sent data. + + If do_handshake() fails more than *max_retry* times, return None. + """ + data, attempt = None, 0 + while not data and attempt < max_retry: + with contextlib.suppress(ssl.SSLWantReadError): + sslobject.do_handshake() + data = outgoing.read() + attempt += 1 + return data + + class BasicSocketTests(unittest.TestCase): def test_constants(self): @@ -539,9 +607,9 @@ def test_openssl_version(self): openssl_ver = f"OpenSSL {major:d}.{minor:d}.{patch:d}" else: openssl_ver = f"OpenSSL {major:d}.{minor:d}.{fix:d}" - self.assertTrue( - s.startswith((openssl_ver, libressl_ver, "AWS-LC")), - (s, t, hex(n)) + self.assertStartsWith( + s, (openssl_ver, libressl_ver, "AWS-LC"), + (t, hex(n)) ) @support.cpython_only @@ -1084,7 +1152,12 @@ def test_min_max_version(self): ctx.maximum_version = ssl.TLSVersion.MINIMUM_SUPPORTED self.assertIn( ctx.maximum_version, - {ssl.TLSVersion.TLSv1, ssl.TLSVersion.TLSv1_1, ssl.TLSVersion.SSLv3} + { + ssl.TLSVersion.TLSv1, + ssl.TLSVersion.TLSv1_1, + ssl.TLSVersion.TLSv1_2, + ssl.TLSVersion.SSLv3, + } ) ctx.minimum_version = ssl.TLSVersion.MAXIMUM_SUPPORTED @@ -1238,6 +1311,29 @@ def getpass(self): # Make sure the password function isn't called if it isn't needed ctx.load_cert_chain(CERTFILE, password=getpass_exception) + @support.skip_if_sanitizer("gh-143756: data race in " + "SSLContext.load_cert_chain when called " + "concurrently on the same context", + thread=True) + @threading_helper.requires_working_threading() + def test_load_cert_chain_thread_safety(self): + # gh-134698: _ssl detaches the thread state (and as such, + # releases the GIL and critical sections) around expensive + # OpenSSL calls. Unfortunately, OpenSSL structures aren't + # thread-safe, so executing these calls concurrently led + # to crashes. + ctx = ssl.create_default_context() + + def race(): + ctx.load_cert_chain(CERTFILE) + + threads = [threading.Thread(target=race) for _ in range(8)] + with threading_helper.catch_threading_exception() as cm: + with threading_helper.start_threads(threads): + pass + + self.assertIsNone(cm.exc_value) + def test_load_verify_locations(self): ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) ctx.load_verify_locations(CERTFILE) @@ -1386,6 +1482,49 @@ def dummycallback(sock, servername, ctx): ctx.set_servername_callback(None) ctx.set_servername_callback(dummycallback) + def test_sni_callback_on_dead_references(self): + # See https://github.com/python/cpython/issues/146080. + c_ctx = make_test_context() + c_inc, c_out = ssl.MemoryBIO(), ssl.MemoryBIO() + client = c_ctx.wrap_bio(c_inc, c_out, server_hostname=SIGNED_CERTFILE_HOSTNAME) + + def sni_callback(sock, servername, ctx): pass + sni_callback = unittest.mock.Mock(wraps=sni_callback) + s_ctx = make_test_context(server_side=True, certfile=SIGNED_CERTFILE) + s_ctx.set_servername_callback(sni_callback) + + s_inc, s_out = ssl.MemoryBIO(), ssl.MemoryBIO() + server = s_ctx.wrap_bio(s_inc, s_out, server_side=True) + server_impl = server._sslobj + + # Perform the handshake on the client side first. + data = do_ssl_object_handshake(client, c_out) + sni_callback.assert_not_called() + if data is None: + self.skipTest("cannot establish a handshake from the client") + s_inc.write(data) + sni_callback.assert_not_called() + # Delete the server object before it starts doing its handshake + # and ensure that we did not call the SNI callback yet. + del server + gc.collect() + # Try to continue the server's handshake by directly using + # the internal SSL object. The latter is a weak reference + # stored in the server context and has now a dead owner. + with self.assertRaises(ssl.SSLError) as cm: + server_impl.do_handshake() + # The SNI C callback raised an exception before calling our callback. + sni_callback.assert_not_called() + + # In AWS-LC, any handshake failures reports SSL_R_PARSE_TLSEXT, + # while OpenSSL uses SSL_R_CALLBACK_FAILED on SNI callback failures. + if IS_AWS_LC: + libssl_error_reason = "PARSE_TLSEXT" + else: + libssl_error_reason = "callback failed" + self.assertIn(libssl_error_reason, str(cm.exception)) + self.assertEqual(cm.exception.errno, ssl.SSL_ERROR_SSL) + def test_sni_callback_refcycle(self): # Reference cycles through the servername callback are detected # and cleared. @@ -1398,6 +1537,59 @@ def dummycallback(sock, servername, ctx, cycle=ctx): gc.collect() self.assertIs(wr(), None) + @support.skip_if_sanitizer("gh-150191: OpenSSL has an internal data race " + "when the SNI callback is replaced during a " + "handshake", thread=True) + @threading_helper.requires_working_threading() + def test_sni_callback_race(self): + # Replacing sni_callback while a handshake is in-flight must not + # crash (use-after-free on the callback in free-threaded builds). + # + # Use a single handshake thread: OpenSSL has internal data races + # on shared SSL_CTX state when multiple handshakes run + # concurrently against the same context (gh-150191). Concurrency + # on the *setter* is what exercises the fix from gh-149816, so + # multiple toggler threads race against each other and against + # the one handshake worker. + client_ctx, server_ctx, hostname = testing_context() + + server_ctx.sni_callback = lambda *a: None + deadline = time.monotonic() + 0.1 + + def do_handshakes(): + while time.monotonic() < deadline: + c_in = ssl.MemoryBIO() + c_out = ssl.MemoryBIO() + s_in = ssl.MemoryBIO() + s_out = ssl.MemoryBIO() + client = client_ctx.wrap_bio( + c_in, c_out, server_hostname=hostname) + server = server_ctx.wrap_bio(s_in, s_out, server_side=True) + for _ in range(50): + try: + client.do_handshake() + except ssl.SSLWantReadError: + pass + except ssl.SSLError: + break + if c_out.pending: + s_in.write(c_out.read()) + try: + server.do_handshake() + except ssl.SSLWantReadError: + pass + except ssl.SSLError: + break + if s_out.pending: + c_in.write(s_out.read()) + + def toggle_callback(): + while time.monotonic() < deadline: + server_ctx.sni_callback = lambda *a: None + server_ctx.sni_callback = None + + threading_helper.run_concurrently([do_handshakes] + 4 * [toggle_callback]) + def test_cert_store_stats(self): ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) self.assertEqual(ctx.cert_store_stats(), @@ -1640,6 +1832,39 @@ def test_num_tickest(self): with self.assertRaises(ValueError): ctx.num_tickets = 1 + @support.cpython_only + def test_refcycle_msg_callback(self): + # See https://github.com/python/cpython/issues/142516. + ctx = make_test_context() + def msg_callback(*args, _=ctx, **kwargs): ... + ctx._msg_callback = msg_callback + + @support.cpython_only + @requires_keylog_setter + def test_refcycle_keylog_filename(self): + # See https://github.com/python/cpython/issues/142516. + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + ctx = make_test_context() + class KeylogFilename(str): ... + ctx.keylog_filename = KeylogFilename(os_helper.TESTFN) + ctx.keylog_filename._ = ctx + + @support.cpython_only + @unittest.skipUnless(ssl.HAS_PSK, 'requires TLS-PSK') + def test_refcycle_psk_client_callback(self): + # See https://github.com/python/cpython/issues/142516. + ctx = make_test_context() + def psk_client_callback(*args, _=ctx, **kwargs): ... + ctx.set_psk_client_callback(psk_client_callback) + + @support.cpython_only + @unittest.skipUnless(ssl.HAS_PSK, 'requires TLS-PSK') + def test_refcycle_psk_server_callback(self): + # See https://github.com/python/cpython/issues/142516. + ctx = make_test_context(server_side=True) + def psk_server_callback(*args, _=ctx, **kwargs): ... + ctx.set_psk_server_callback(psk_server_callback) + class SSLErrorTests(unittest.TestCase): @@ -1668,7 +1893,7 @@ def test_lib_reason(self): regex = "(NO_START_LINE|UNSUPPORTED_PUBLIC_KEY_TYPE)" self.assertRegex(cm.exception.reason, regex) s = str(cm.exception) - self.assertTrue("NO_START_LINE" in s, s) + self.assertIn("NO_START_LINE", s) def test_subclass(self): # Check that the appropriate SSLError subclass is raised @@ -1683,7 +1908,7 @@ def test_subclass(self): with self.assertRaises(ssl.SSLWantReadError) as cm: c.do_handshake() s = str(cm.exception) - self.assertTrue(s.startswith("The operation did not complete (read)"), s) + self.assertStartsWith(s, "The operation did not complete (read)") # For compatibility self.assertEqual(cm.exception.errno, ssl.SSL_ERROR_WANT_READ) @@ -2838,11 +3063,16 @@ def test_echo(self): 'Cannot create a client socket with a PROTOCOL_TLS_SERVER context', str(e.exception)) + @support.skip_if_sanitizer("gh-143756: data race in " + "SSLContext.load_cert_chain when called " + "concurrently on the same context", + thread=True) @unittest.skipUnless(support.Py_GIL_DISABLED, "test is only useful if the GIL is disabled") def test_ssl_in_multiple_threads(self): # See GH-124984: OpenSSL is not thread safe. threads = [] + warnings_filters = sys.flags.context_aware_warnings global USE_SAME_TEST_CONTEXT USE_SAME_TEST_CONTEXT = True try: @@ -2851,7 +3081,10 @@ def test_ssl_in_multiple_threads(self): self.test_alpn_protocols, self.test_getpeercert, self.test_crl_check, - self.test_check_hostname_idn, + functools.partial( + self.test_check_hostname_idn, + warnings_filters=warnings_filters, + ), self.test_wrong_cert_tls12, self.test_wrong_cert_tls13, ): @@ -3097,7 +3330,7 @@ def test_dual_rsa_ecc(self): cipher = s.cipher()[0].split('-') self.assertTrue(cipher[:2], ('ECDHE', 'ECDSA')) - def test_check_hostname_idn(self): + def test_check_hostname_idn(self, warnings_filters=True): if support.verbose: sys.stdout.write("\n") @@ -3152,16 +3385,30 @@ def test_check_hostname_idn(self): server_hostname="python.example.org") as s: with self.assertRaises(ssl.CertificateError): s.connect((HOST, server.port)) - with ThreadedEchoServer(context=server_context, chatty=True) as server: - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(UnicodeError): - context.wrap_socket(socket.socket(), - server_hostname='.pythontest.net') - with ThreadedEchoServer(context=server_context, chatty=True) as server: - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(UnicodeDecodeError): - context.wrap_socket(socket.socket(), - server_hostname=b'k\xf6nig.idn.pythontest.net') + with ( + ThreadedEchoServer(context=server_context, chatty=True) as server, + ( + warnings_helper.check_no_resource_warning(self) + if warnings_filters + else nullcontext() + ), + self.assertRaises(UnicodeError), + ): + context.wrap_socket(socket.socket(), server_hostname='.pythontest.net') + + with ( + ThreadedEchoServer(context=server_context, chatty=True) as server, + ( + warnings_helper.check_no_resource_warning(self) + if warnings_filters + else nullcontext() + ), + self.assertRaises(UnicodeDecodeError), + ): + context.wrap_socket( + socket.socket(), + server_hostname=b'k\xf6nig.idn.pythontest.net', + ) def test_wrong_cert_tls12(self): """Connecting when the server rejects the client's certificate @@ -4519,6 +4766,45 @@ def server_callback(identity): with client_context.wrap_socket(socket.socket()) as s: s.connect((HOST, server.port)) + @support.skip_if_sanitizer("gh-150191: OpenSSL races on SSL->rwstate and " + "the socket BIO flags with concurrent read " + "and write", thread=True) + def test_thread_recv_while_main_thread_sends(self): + # GH-137583: Locking was added to calls to send() and recv() on SSL + # socket objects. This seemed fine at the surface level because those + # calls weren't re-entrant, but recv() calls would implicitly mimick + # holding a lock by blocking until it received data. This means that + # if a thread started to infinitely block until data was received, calls + # to send() would deadlock, because it would wait forever on the lock + # that the recv() call held. + data = b"1" * 1024 + event = threading.Event() + def background(sock): + event.set() + received = sock.recv(len(data)) + self.assertEqual(received, data) + + client_context, server_context, hostname = testing_context() + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as sock: + sock.connect((HOST, server.port)) + sock.settimeout(1) + sock.setblocking(1) + # Ensure that the server is ready to accept requests + sock.sendall(b"123") + self.assertEqual(sock.recv(3), b"123") + with threading_helper.catch_threading_exception() as cm: + thread = threading.Thread(target=background, + args=(sock,), daemon=True) + thread.start() + event.wait() + sock.sendall(data) + thread.join() + if cm.exc_value is not None: + raise cm.exc_value + @unittest.skipUnless(has_tls_version('TLSv1_3') and ssl.HAS_PHA, "Test needs TLS 1.3 PHA") @@ -4835,10 +5121,6 @@ def test_internal_chain_server(self): self.assertEqual(res, b'\x02\n') -HAS_KEYLOG = hasattr(ssl.SSLContext, 'keylog_filename') -requires_keylog = unittest.skipUnless( - HAS_KEYLOG, 'test requires OpenSSL 1.1.1 with keylog callback') - class TestSSLDebug(unittest.TestCase): def keylog_lines(self, fname=os_helper.TESTFN): @@ -5076,15 +5358,27 @@ def non_linux_skip_if_other_okay_error(self, err): return # Expect the full test setup to always work on Linux. if (isinstance(err, ConnectionResetError) or (isinstance(err, OSError) and err.errno == errno.EINVAL) or - re.search('wrong.version.number', str(getattr(err, "reason", "")), re.I)): + re.search( + # Matches the following error messages: + # '[SSL: WRONG_VERSION_NUMBER] wrong version number (_ssl.c:1123)' + # '[SSL: RECORD_LAYER_FAILURE] record layer failure (_ssl.c:1109)' + # '[SSL: HTTP_REQUEST] http request (_ssl.c:1143)' + r'wrong.version.number|record.layer.failure|http.request', + str(getattr(err, "reason", "")), + re.IGNORECASE, + ) + ): # On Windows the TCP RST leads to a ConnectionResetError # (ECONNRESET) which Linux doesn't appear to surface to userspace. # If wrap_socket() winds up on the "if connected:" path and doing - # the actual wrapping... we get an SSLError from OpenSSL. Typically - # WRONG_VERSION_NUMBER. While appropriate, neither is the scenario - # we're specifically trying to test. The way this test is written - # is known to work on Linux. We'll skip it anywhere else that it - # does not present as doing so. + # the actual wrapping... we get an SSLError from OpenSSL. This is + # typically WRONG_VERSION_NUMBER. The same happens on iOS, but + # RECORD_LAYER_FAILURE or HTTP_REQUEST is the error. + # + # While appropriate, these scenarios aren't what we're specifically + # trying to test. The way this test is written is known to work on + # Linux. We'll skip it anywhere else that it does not present as + # doing so. try: self.skipTest(f"Could not recreate conditions on {sys.platform}:" f" {err=}") @@ -5285,7 +5579,7 @@ def test_tlsalerttype(self): class Checked_TLSAlertType(enum.IntEnum): """Alert types for TLSContentType.ALERT messages - See RFC 8466, section B.2 + See RFC 8446, section B.2 """ CLOSE_NOTIFY = 0 UNEXPECTED_MESSAGE = 10 diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py index 49013a4bcd8af6b..a83f7d076f027ef 100644 --- a/Lib/test/test_stat.py +++ b/Lib/test/test_stat.py @@ -157,12 +157,17 @@ def test_mode(self): os.chmod(TESTFN, 0o700) st_mode, modestr = self.get_mode() - self.assertEqual(modestr[:3], '-rw') + self.assertStartsWith(modestr, '-rw') self.assertS_IS("REG", st_mode) self.assertEqual(self.statmod.S_IFMT(st_mode), self.statmod.S_IFREG) self.assertEqual(self.statmod.S_IMODE(st_mode), 0o666) + def test_filemode_does_not_misclassify_random_bits(self): + # gh-144050 regression test + self.assertEqual(self.statmod.filemode(0o77777)[0], "?") + self.assertEqual(self.statmod.filemode(0o177777)[0], "?") + @os_helper.skip_unless_working_chmod def test_directory(self): os.mkdir(TESTFN) @@ -256,7 +261,7 @@ def test_flags_consistent(self): "FILE_ATTRIBUTE_* constants are Win32 specific") def test_file_attribute_constants(self): for key, value in sorted(self.file_attributes.items()): - self.assertTrue(hasattr(self.statmod, key), key) + self.assertHasAttr(self.statmod, key) modvalue = getattr(self.statmod, key) self.assertEqual(value, modvalue, key) @@ -314,7 +319,7 @@ def test_macosx_attribute_values(self): self.assertEqual(self.statmod.S_ISGID, 0o002000) self.assertEqual(self.statmod.S_ISVTX, 0o001000) - self.assertFalse(hasattr(self.statmod, "S_ISTXT")) + self.assertNotHasAttr(self.statmod, "S_ISTXT") self.assertEqual(self.statmod.S_IREAD, self.statmod.S_IRUSR) self.assertEqual(self.statmod.S_IWRITE, self.statmod.S_IWUSR) self.assertEqual(self.statmod.S_IEXEC, self.statmod.S_IXUSR) diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index c69baa4bf4d1b11..677a87b51b91925 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -645,7 +645,7 @@ def do_test(self, args): def test_numerictestcase_is_testcase(self): # Ensure that NumericTestCase actually is a TestCase. - self.assertTrue(issubclass(NumericTestCase, unittest.TestCase)) + self.assertIsSubclass(NumericTestCase, unittest.TestCase) def test_error_msg_numeric(self): # Test the error message generated for numeric comparisons. @@ -683,32 +683,23 @@ class GlobalsTest(unittest.TestCase): def test_meta(self): # Test for the existence of metadata. for meta in self.expected_metadata: - self.assertTrue(hasattr(self.module, meta), - "%s not present" % meta) + self.assertHasAttr(self.module, meta) def test_check_all(self): # Check everything in __all__ exists and is public. module = self.module for name in module.__all__: # No private names in __all__: - self.assertFalse(name.startswith("_"), + self.assertNotStartsWith(name, "_", 'private name "%s" in __all__' % name) # And anything in __all__ must exist: - self.assertTrue(hasattr(module, name), - 'missing name "%s" in __all__' % name) + self.assertHasAttr(module, name) class StatisticsErrorTest(unittest.TestCase): def test_has_exception(self): - errmsg = ( - "Expected StatisticsError to be a ValueError, but got a" - " subclass of %r instead." - ) - self.assertTrue(hasattr(statistics, 'StatisticsError')) - self.assertTrue( - issubclass(statistics.StatisticsError, ValueError), - errmsg % statistics.StatisticsError.__base__ - ) + self.assertHasAttr(statistics, 'StatisticsError') + self.assertIsSubclass(statistics.StatisticsError, ValueError) # === Tests for private utility functions === @@ -2014,7 +2005,6 @@ def test_iter_list_same(self): expected = self.func(data) self.assertEqual(self.func(iter(data)), expected) - class TestPVariance(VarianceStdevMixin, NumericTestCase, UnivariateTypeMixin): # Tests for population variance. def setUp(self): @@ -2122,6 +2112,14 @@ def test_center_not_at_mean(self): self.assertEqual(self.func(data), 2.5) self.assertEqual(self.func(data, mu=0.5), 6.5) + def test_gh_140938(self): + # Inputs with inf/nan should raise a ValueError + with self.assertRaises(ValueError): + self.func([1.0, math.inf]) + with self.assertRaises(ValueError): + self.func([1.0, math.nan]) + + class TestSqrtHelpers(unittest.TestCase): def test_integer_sqrt_of_frac_rto(self): @@ -2355,6 +2353,7 @@ def test_mixed_int_and_float(self): class TestKDE(unittest.TestCase): + @support.requires_resource('cpu') def test_kde(self): kde = statistics.kde StatisticsError = statistics.StatisticsError @@ -3327,7 +3326,8 @@ def tearDown(self): def load_tests(loader, tests, ignore): """Used for doctest/unittest integration.""" tests.addTests(doctest.DocTestSuite()) - tests.addTests(doctest.DocTestSuite(statistics)) + if sys.float_repr_style == 'short': + tests.addTests(doctest.DocTestSuite(statistics)) return tests diff --git a/Lib/test/test_str.py b/Lib/test/test_str.py index d6a7bd0da599109..444841ddd470aa7 100644 --- a/Lib/test/test_str.py +++ b/Lib/test/test_str.py @@ -600,6 +600,21 @@ def test_replace_id(self): text = 'abc def' self.assertIs(text.replace(pattern, pattern), text) + @support.nomemtest + def test_replace_oom(self): + # https://github.com/python/cpython/issues/152228 + s1 = "轘" * 4 + s2 = "&" + s3 = "&amp;" + assertion = self.assertRaises(MemoryError) + _testcapi.set_nomemory(0, 0) + try: + # No allocations made in the test itself: + with assertion: + s1.replace(s2, s3) # this line used to crash before + finally: + _testcapi.remove_mem_hooks() + def test_repeat_id_preserving(self): a = '123abc1@' b = '456zyx-+' @@ -1231,10 +1246,10 @@ def __repr__(self): self.assertEqual('{0:\x00^6}'.format(3), '\x00\x003\x00\x00\x00') self.assertEqual('{0:<6}'.format(3), '3 ') - self.assertEqual('{0:\x00<6}'.format(3.14), '3.14\x00\x00') - self.assertEqual('{0:\x01<6}'.format(3.14), '3.14\x01\x01') - self.assertEqual('{0:\x00^6}'.format(3.14), '\x003.14\x00') - self.assertEqual('{0:^6}'.format(3.14), ' 3.14 ') + self.assertEqual('{0:\x00<6}'.format(3.25), '3.25\x00\x00') + self.assertEqual('{0:\x01<6}'.format(3.25), '3.25\x01\x01') + self.assertEqual('{0:\x00^6}'.format(3.25), '\x003.25\x00') + self.assertEqual('{0:^6}'.format(3.25), ' 3.25 ') self.assertEqual('{0:\x00<12}'.format(3+2.0j), '(3+2j)\x00\x00\x00\x00\x00\x00') self.assertEqual('{0:\x01<12}'.format(3+2.0j), '(3+2j)\x01\x01\x01\x01\x01\x01') diff --git a/Lib/test/test_strftime.py b/Lib/test/test_strftime.py index 752e31359cf2062..375f6aaedd89349 100644 --- a/Lib/test/test_strftime.py +++ b/Lib/test/test_strftime.py @@ -39,7 +39,21 @@ def _update_variables(self, now): if now[3] < 12: self.ampm='(AM|am)' else: self.ampm='(PM|pm)' - self.jan1 = time.localtime(time.mktime((now[0], 1, 1, 0, 0, 0, 0, 1, 0))) + jan1 = time.struct_time( + ( + now.tm_year, # Year + 1, # Month (January) + 1, # Day (1st) + 0, # Hour (0) + 0, # Minute (0) + 0, # Second (0) + -1, # tm_wday (will be determined) + 1, # tm_yday (day 1 of the year) + -1, # tm_isdst (let the system determine) + ) + ) + # use mktime to get the correct tm_wday and tm_isdst values + self.jan1 = time.localtime(time.mktime(jan1)) try: if now[8]: self.tz = time.tzname[1] diff --git a/Lib/test/test_string/_support.py b/Lib/test/test_string/_support.py index eaa3354a5592463..cfead782b7d4c13 100644 --- a/Lib/test/test_string/_support.py +++ b/Lib/test/test_string/_support.py @@ -3,33 +3,45 @@ class TStringBaseCase: + def assertInterpolationEqual(self, i, exp): + """Test Interpolation equality. + + The *i* argument must be an Interpolation instance. + + The *exp* argument must be a tuple of the form + (value, expression, conversion, format_spec) where the final three + items may be omitted and are assumed to be '', None and '' respectively. + """ + if len(exp) == 4: + actual = (i.value, i.expression, i.conversion, i.format_spec) + self.assertEqual(actual, exp) + elif len(exp) == 3: + self.assertEqual((i.value, i.expression, i.conversion), exp) + self.assertEqual(i.format_spec, "") + elif len(exp) == 2: + self.assertEqual((i.value, i.expression), exp) + self.assertEqual(i.conversion, None) + self.assertEqual(i.format_spec, "") + elif len(exp) == 1: + self.assertEqual((i.value,), exp) + self.assertEqual(i.expression, "") + self.assertEqual(i.conversion, None) + self.assertEqual(i.format_spec, "") + def assertTStringEqual(self, t, strings, interpolations): """Test template string literal equality. The *strings* argument must be a tuple of strings equal to *t.strings*. The *interpolations* argument must be a sequence of tuples which are - compared against *t.interpolations*. Each tuple consists of - (value, expression, conversion, format_spec), though the final two - items may be omitted, and are assumed to be None and '' respectively. + compared against *t.interpolations*. Each tuple must match the form + described in the `assertInterpolationEqual` method. """ self.assertEqual(t.strings, strings) self.assertEqual(len(t.interpolations), len(interpolations)) for i, exp in zip(t.interpolations, interpolations, strict=True): - if len(exp) == 4: - actual = (i.value, i.expression, i.conversion, i.format_spec) - self.assertEqual(actual, exp) - continue - - if len(exp) == 3: - self.assertEqual((i.value, i.expression, i.conversion), exp) - self.assertEqual(i.format_spec, '') - continue - - self.assertEqual((i.value, i.expression), exp) - self.assertEqual(i.format_spec, '') - self.assertIsNone(i.conversion) + self.assertInterpolationEqual(i, exp) def convert(value, conversion): diff --git a/Lib/test/test_string/test_string.py b/Lib/test/test_string/test_string.py index 5394fe4e12cd41c..350784a9f00817b 100644 --- a/Lib/test/test_string/test_string.py +++ b/Lib/test/test_string/test_string.py @@ -299,6 +299,20 @@ def test_SafeTemplate(self): eq(s.safe_substitute(dict(who='tim', what='ham', meal='dinner')), 'tim likes ham for dinner') + def test_precompiled_pattern(self): + # A subclass may supply an already-compiled pattern; it must be reused, + # not recompiled (re.compile() rejects flags on a compiled pattern). + import re + compiled = re.compile( + r'\$(?:(?P<escaped>\$)|(?P<named>[a-z]+)|' + r'\{(?P<braced>[a-z]+)\}|(?P<invalid>))') + class MyTemplate(Template): + pattern = compiled + self.assertIs(MyTemplate.pattern, compiled) + self.assertEqual( + MyTemplate('$who likes $what').substitute(who='tim', what='ham'), + 'tim likes ham') + def test_invalid_placeholders(self): raises = self.assertRaises s = Template('$who likes $') diff --git a/Lib/test/test_string/test_templatelib.py b/Lib/test/test_string/test_templatelib.py index 5b9490c2be6de07..1c86717155fd5ab 100644 --- a/Lib/test/test_string/test_templatelib.py +++ b/Lib/test/test_string/test_templatelib.py @@ -1,7 +1,7 @@ import pickle import unittest from collections.abc import Iterator, Iterable -from string.templatelib import Template, Interpolation +from string.templatelib import Template, Interpolation, convert from test.test_string._support import TStringBaseCase, fstring @@ -45,6 +45,19 @@ def test_basic_creation(self): self.assertEqual(len(t.interpolations), 0) self.assertEqual(fstring(t), 'Hello,\nworld') + def test_interpolation_creation(self): + i = Interpolation('Maria', 'name', 'a', 'fmt') + self.assertInterpolationEqual(i, ('Maria', 'name', 'a', 'fmt')) + + i = Interpolation('Maria', 'name', 'a') + self.assertInterpolationEqual(i, ('Maria', 'name', 'a')) + + i = Interpolation('Maria', 'name') + self.assertInterpolationEqual(i, ('Maria', 'name')) + + i = Interpolation('Maria') + self.assertInterpolationEqual(i, ('Maria',)) + def test_creation_interleaving(self): # Should add strings on either side t = Template(Interpolation('Maria', 'name', None, '')) @@ -148,6 +161,33 @@ def test_iter(self): self.assertEqual(res[1].format_spec, '') self.assertEqual(res[2], ' yz') + def test_exhausted(self): + # See https://github.com/python/cpython/issues/134119. + template_iter = iter(t"{1}") + self.assertIsInstance(next(template_iter), Interpolation) + self.assertRaises(StopIteration, next, template_iter) + self.assertRaises(StopIteration, next, template_iter) + + +class TestFunctions(unittest.TestCase): + def test_convert(self): + from fractions import Fraction + + for obj in ('Café', None, 3.14, Fraction(1, 2)): + with self.subTest(f'{obj=}'): + self.assertEqual(convert(obj, None), obj) + self.assertEqual(convert(obj, 's'), str(obj)) + self.assertEqual(convert(obj, 'r'), repr(obj)) + self.assertEqual(convert(obj, 'a'), ascii(obj)) + + # Invalid conversion specifier + with self.assertRaises(ValueError): + convert(obj, 'z') + with self.assertRaises(ValueError): + convert(obj, 1) + with self.assertRaises(ValueError): + convert(obj, object()) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_strptime.py b/Lib/test/test_strptime.py index 268230f6da78f8a..e277e7f4ef7b1a9 100644 --- a/Lib/test/test_strptime.py +++ b/Lib/test/test_strptime.py @@ -221,14 +221,16 @@ def test_ValueError(self): self.assertRaises(ValueError, _strptime._strptime_time, data_string="%d", format="%A") for bad_format in ("%", "% ", "%\n"): - with self.assertRaisesRegex(ValueError, "stray % in format "): + with (self.subTest(format=bad_format), + self.assertRaisesRegex(ValueError, "stray % in format ")): _strptime._strptime_time("2005", bad_format) - for bad_format in ("%e", "%Oe", "%O", "%O ", "%Ee", "%E", "%E ", - "%.", "%+", "%_", "%~", "%\\", + for bad_format in ("%i", "%Oi", "%O", "%O ", "%Ee", "%E", "%E ", + "%.", "%+", "%~", "%\\", "%O.", "%O+", "%O_", "%O~", "%O\\"): directive = bad_format[1:].rstrip() - with self.assertRaisesRegex(ValueError, - f"'{re.escape(directive)}' is a bad directive in format "): + with (self.subTest(format=bad_format), + self.assertRaisesRegex(ValueError, + f"'{re.escape(directive)}' is a bad directive in format ")): _strptime._strptime_time("2005", bad_format) msg_week_no_year_or_weekday = r"ISO week directive '%V' must be used with " \ @@ -335,6 +337,15 @@ def test_month_locale(self): self.roundtrip('%B', 1, (1900, m, 1, 0, 0, 0, 0, 1, 0)) self.roundtrip('%b', 1, (1900, m, 1, 0, 0, 0, 0, 1, 0)) + @run_with_locales('LC_TIME', 'az_AZ', 'ber_DZ', 'ber_MA', 'crh_UA') + def test_month_locale2(self): + # Test for month directives + # Month name contains 'İ' ('\u0130') + self.roundtrip('%B', 1, (2025, 6, 1, 0, 0, 0, 6, 152, 0)) + self.roundtrip('%b', 1, (2025, 6, 1, 0, 0, 0, 6, 152, 0)) + self.roundtrip('%B', 1, (2025, 7, 1, 0, 0, 0, 1, 182, 0)) + self.roundtrip('%b', 1, (2025, 7, 1, 0, 0, 0, 1, 182, 0)) + def test_day(self): # Test for day directives self.roundtrip('%d %Y', 2) @@ -480,13 +491,11 @@ def test_bad_timezone(self): # * Year is not included: ha_NG. # * Use non-Gregorian calendar: lo_LA, thai, th_TH. # On Windows: ar_IN, ar_SA, fa_IR, ps_AF. - # - # BUG: Generates regexp that does not match the current date and time - # for lzh_TW. @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', 'he_IL', 'eu_ES', 'ar_AE', 'mfe_MU', 'yo_NG', 'csb_PL', 'br_FR', 'gez_ET', 'brx_IN', - 'my_MM', 'or_IN', 'shn_MM', 'az_IR') + 'my_MM', 'or_IN', 'shn_MM', 'az_IR', + 'byn_ER', 'wal_ET', 'lzh_TW') def test_date_time_locale(self): # Test %c directive loc = locale.getlocale(locale.LC_TIME)[0] @@ -525,11 +534,9 @@ def test_date_time_locale2(self): # NB: Does not roundtrip because use non-Gregorian calendar: # lo_LA, thai, th_TH. On Windows: ar_IN, ar_SA, fa_IR, ps_AF. - # BUG: Generates regexp that does not match the current date - # for lzh_TW. @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', 'he_IL', 'eu_ES', 'ar_AE', - 'az_IR', 'my_MM', 'or_IN', 'shn_MM') + 'az_IR', 'my_MM', 'or_IN', 'shn_MM', 'lzh_TW') def test_date_locale(self): # Test %x directive now = time.time() @@ -546,11 +553,11 @@ def test_date_locale(self): # NB: Dates before 1969 do not roundtrip on many locales, including C. @unittest.skipIf(support.linked_to_musl(), "musl libc issue, bpo-46390") @run_with_locales('LC_TIME', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', - 'eu_ES', 'ar_AE', 'my_MM', 'shn_MM') + 'eu_ES', 'ar_AE', 'my_MM', 'shn_MM', 'lzh_TW') def test_date_locale2(self): # Test %x directive loc = locale.getlocale(locale.LC_TIME)[0] - if sys.platform.startswith('sunos'): + if sys.platform.startswith(('sunos', 'aix')): if loc in ('en_US', 'de_DE', 'ar_AE'): self.skipTest(f'locale {loc!r} may not work on this platform') self.roundtrip('%x', slice(0, 3), (1900, 1, 1, 0, 0, 0, 0, 1, 0)) @@ -562,11 +569,11 @@ def test_date_locale2(self): # norwegian, nynorsk. # * Hours are in 12-hour notation without AM/PM indication: hy_AM, # ms_MY, sm_WS. - # BUG: Generates regexp that does not match the current time for lzh_TW. @run_with_locales('LC_TIME', 'C', 'en_US', 'fr_FR', 'de_DE', 'ja_JP', 'aa_ET', 'am_ET', 'az_IR', 'byn_ER', 'fa_IR', 'gez_ET', 'my_MM', 'om_ET', 'or_IN', 'shn_MM', 'sid_ET', 'so_SO', - 'ti_ET', 'tig_ER', 'wal_ET') + 'ti_ET', 'tig_ER', 'wal_ET', 'lzh_TW', + 'ar_SA', 'bg_BG') def test_time_locale(self): # Test %X directive loc = locale.getlocale(locale.LC_TIME)[0] diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 7df01f28f09118e..e18bad9c9908262 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -181,6 +181,16 @@ def test_calcsize(self): self.assertGreaterEqual(struct.calcsize('n'), struct.calcsize('i')) self.assertGreaterEqual(struct.calcsize('n'), struct.calcsize('P')) + def test_cache_bytes_vs_str_bb(self): + # Mixing str and bytes formats must not raise BytesWarning under -bb. + code = ( + 'import struct\n' + 'struct.calcsize(b"!d"); struct.calcsize("!d")\n' + 'struct.calcsize(">d"); struct.calcsize(b">d")\n' + 'struct.Struct(b"i"); struct.Struct("i")\n' + ) + assert_python_ok('-bb', '-c', code) + def test_integers(self): # Integer tests (bBhHiIlLqQnN). import binascii @@ -397,6 +407,21 @@ def test_705836(self): big = (1 << 25) - 1 big = math.ldexp(big, 127 - 24) self.assertRaises(OverflowError, struct.pack, ">f", big) + self.assertRaises(OverflowError, struct.pack, "<f", big) + # same for native format, see gh-145633 + self.assertRaises(OverflowError, struct.pack, "f", big) + + # And for half-floats + big = (1 << 11) - 1 + big = math.ldexp(big, 15 - 10) + packed = struct.pack(">e", big) + unpacked = struct.unpack(">e", packed)[0] + self.assertEqual(big, unpacked) + big = (1 << 12) - 1 + big = math.ldexp(big, 15 - 11) + self.assertRaises(OverflowError, struct.pack, ">e", big) + self.assertRaises(OverflowError, struct.pack, "<e", big) + self.assertRaises(OverflowError, struct.pack, "e", big) def test_1530559(self): for code, byteorder in iter_integer_formats(): @@ -433,56 +458,63 @@ def test_unpack_from(self): self.assertEqual(s.unpack_from(buffer=test_string, offset=2), (b'cd01',)) - def test_pack_into(self): + def _test_pack_into(self, pack_into): test_string = b'Reykjavik rocks, eow!' - writable_buf = array.array('b', b' '*100) - fmt = '21s' - s = struct.Struct(fmt) + writable_buf = memoryview(array.array('b', b' '*100)) # Test without offset - s.pack_into(writable_buf, 0, test_string) + pack_into(writable_buf, 0, test_string) from_buf = writable_buf.tobytes()[:len(test_string)] self.assertEqual(from_buf, test_string) # Test with offset. - s.pack_into(writable_buf, 10, test_string) + pack_into(writable_buf, 10, test_string) from_buf = writable_buf.tobytes()[:len(test_string)+10] self.assertEqual(from_buf, test_string[:10] + test_string) + # Test with negative offset. + pack_into(writable_buf, -30, test_string) + from_buf = writable_buf.tobytes()[-30:-30+len(test_string)] + self.assertEqual(from_buf, test_string) + # Go beyond boundaries. small_buf = array.array('b', b' '*10) - self.assertRaises((ValueError, struct.error), s.pack_into, small_buf, 0, - test_string) - self.assertRaises((ValueError, struct.error), s.pack_into, small_buf, 2, - test_string) + with self.assertRaises((ValueError, struct.error)): + pack_into(small_buf, 0, test_string) + with self.assertRaises((ValueError, struct.error)): + pack_into(writable_buf, 90, test_string) + with self.assertRaises((ValueError, struct.error)): + pack_into(writable_buf, -10, test_string) + with self.assertRaises((ValueError, struct.error)): + pack_into(writable_buf, 150, test_string) + with self.assertRaises((ValueError, struct.error)): + pack_into(writable_buf, -150, test_string) + + # Test invalid buffer. + self.assertRaises(TypeError, pack_into, b' '*100, 0, test_string) + self.assertRaises(TypeError, pack_into, ' '*100, 0, test_string) + self.assertRaises(TypeError, pack_into, [0]*100, 0, test_string) + self.assertRaises(TypeError, pack_into, None, 0, test_string) + self.assertRaises(TypeError, pack_into, writable_buf[::2], 0, test_string) + self.assertRaises(TypeError, pack_into, writable_buf[::-1], 0, test_string) + + # Test bogus offset (issue bpo-3694) + with self.assertRaises(TypeError): + pack_into(writable_buf, None, test_string) + with self.assertRaises(TypeError): + pack_into(writable_buf, 0.0, test_string) + with self.assertRaises((IndexError, OverflowError)): + pack_into(writable_buf, 2**1000, test_string) + with self.assertRaises((IndexError, OverflowError)): + pack_into(writable_buf, -2**1000, test_string) - # Test bogus offset (issue 3694) - sb = small_buf - self.assertRaises((TypeError, struct.error), struct.pack_into, b'', sb, - None) + def test_pack_into(self): + s = struct.Struct('21s') + self._test_pack_into(s.pack_into) def test_pack_into_fn(self): - test_string = b'Reykjavik rocks, eow!' - writable_buf = array.array('b', b' '*100) - fmt = '21s' - pack_into = lambda *args: struct.pack_into(fmt, *args) - - # Test without offset. - pack_into(writable_buf, 0, test_string) - from_buf = writable_buf.tobytes()[:len(test_string)] - self.assertEqual(from_buf, test_string) - - # Test with offset. - pack_into(writable_buf, 10, test_string) - from_buf = writable_buf.tobytes()[:len(test_string)+10] - self.assertEqual(from_buf, test_string[:10] + test_string) - - # Go beyond boundaries. - small_buf = array.array('b', b' '*10) - self.assertRaises((ValueError, struct.error), pack_into, small_buf, 0, - test_string) - self.assertRaises((ValueError, struct.error), pack_into, small_buf, 2, - test_string) + pack_into = lambda *args: struct.pack_into('21s', *args) + self._test_pack_into(pack_into) def test_unpack_with_buffer(self): # SF bug 1563759: struct.unpack doesn't support buffer protocol objects @@ -545,6 +577,15 @@ def test_count_overflow(self): hugecount2 = '{}b{}H'.format(sys.maxsize//2, sys.maxsize//2) self.assertRaises(struct.error, struct.calcsize, hugecount2) + hugecount3 = '{}i{}q'.format(sys.maxsize // 4, sys.maxsize // 8) + self.assertRaises(struct.error, struct.calcsize, hugecount3) + + hugecount4 = '{}?s'.format(sys.maxsize) + self.assertRaises(struct.error, struct.calcsize, hugecount4) + + hugecount5 = '{}?p'.format(sys.maxsize) + self.assertRaises(struct.error, struct.calcsize, hugecount5) + def test_trailing_counter(self): store = array.array('b', b' '*100) @@ -574,8 +615,24 @@ def test_Struct_reinitialization(self): # Issue 9422: there was a memory leak when reinitializing a # Struct instance. This test can be used to detect the leak # when running with regrtest -L. - s = struct.Struct('i') - s.__init__('ii') + s = struct.Struct('>h') + s.__init__('>hh') + self.assertEqual(s.format, '>hh') + packed = b'\x00\x01\x00\x02' + self.assertEqual(s.pack(1, 2), packed) + self.assertEqual(s.unpack(packed), (1, 2)) + + with self.assertRaises(UnicodeEncodeError): + s.__init__('\udc00') + self.assertEqual(s.format, '>hh') + self.assertEqual(s.pack(1, 2), packed) + self.assertEqual(s.unpack(packed), (1, 2)) + + with self.assertRaises(struct.error): + s.__init__('$') + self.assertEqual(s.format, '>hh') + self.assertEqual(s.pack(1, 2), packed) + self.assertEqual(s.unpack(packed), (1, 2)) def check_sizeof(self, format_str, number_of_codes): # The size of 'PyStructObject' @@ -799,6 +856,55 @@ def test_c_complex_round_trip(self): round_trip = struct.unpack(f, struct.pack(f, z))[0] self.assertComplexesAreIdentical(z, round_trip) + @unittest.skipIf( + support.is_android or support.is_apple_mobile, + "Subinterpreters are not supported on Android and iOS" + ) + def test_endian_table_init_subinterpreters(self): + # Verify that the _struct extension module can be initialized + # concurrently in subinterpreters (gh-140260). + try: + from concurrent.futures import InterpreterPoolExecutor + except ImportError: + raise unittest.SkipTest("InterpreterPoolExecutor not available") + + code = "import struct" + with InterpreterPoolExecutor(max_workers=5) as executor: + results = executor.map(exec, [code] * 5) + self.assertListEqual(list(results), [None] * 5) + + def test_operations_on_half_initialized_Struct(self): + S = struct.Struct.__new__(struct.Struct) + + spam = array.array('b', b' ') + self.assertRaises(RuntimeError, S.iter_unpack, spam) + self.assertRaises(RuntimeError, S.pack, 1) + self.assertRaises(RuntimeError, S.pack_into, spam, 1) + self.assertRaises(RuntimeError, S.unpack, spam) + self.assertRaises(RuntimeError, S.unpack_from, spam) + self.assertRaises(RuntimeError, getattr, S, 'format') + self.assertRaises(RuntimeError, S.__sizeof__) + self.assertRaises(RuntimeError, repr, S) + self.assertEqual(S.size, -1) + + def test_float_round_trip(self): + for format in ( + "f", "<f", ">f", + "d", "<d", ">d", + "e", "<e", ">e", + ): + with self.subTest(format=format): + f = struct.unpack(format, struct.pack(format, 1.5))[0] + self.assertEqual(f, 1.5) + f = struct.unpack(format, struct.pack(format, NAN))[0] + self.assertTrue(math.isnan(f), f) + f = struct.unpack(format, struct.pack(format, INF))[0] + self.assertTrue(math.isinf(f), f) + self.assertEqual(math.copysign(1.0, f), 1.0) + f = struct.unpack(format, struct.pack(format, -INF))[0] + self.assertTrue(math.isinf(f), f) + self.assertEqual(math.copysign(1.0, f), -1.0) + class UnpackIteratorTest(unittest.TestCase): """ @@ -873,6 +979,7 @@ def test_module_func(self): self.assertRaises(StopIteration, next, it) def test_half_float(self): + _testcapi = import_helper.import_module('_testcapi') # Little-endian examples from: # http://en.wikipedia.org/wiki/Half_precision_floating-point_format format_bits_float__cleanRoundtrip_list = [ @@ -917,10 +1024,17 @@ def test_half_float(self): # Check that packing produces a bit pattern representing a quiet NaN: # all exponent bits and the msb of the fraction should all be 1. + if _testcapi.nan_msb_is_signaling: + # HP PA RISC and some MIPS CPUs use 0 for quiet, see: + # https://en.wikipedia.org/wiki/NaN#Encoding + expected = 0x7c + else: + expected = 0x7e + packed = struct.pack('<e', math.nan) - self.assertEqual(packed[1] & 0x7e, 0x7e) + self.assertEqual(packed[1] & 0x7e, expected) packed = struct.pack('<e', -math.nan) - self.assertEqual(packed[1] & 0x7e, 0x7e) + self.assertEqual(packed[1] & 0x7e, expected) # Checks for round-to-even behavior format_bits_float__rounding_list = [ diff --git a/Lib/test/test_structseq.py b/Lib/test/test_structseq.py index d0bc0bd7b615200..74506fc54de50e7 100644 --- a/Lib/test/test_structseq.py +++ b/Lib/test/test_structseq.py @@ -1,4 +1,5 @@ import copy +import gc import os import pickle import re @@ -42,7 +43,7 @@ def test_repr(self): # os.stat() gives a complicated struct sequence. st = os.stat(__file__) rep = repr(st) - self.assertTrue(rep.startswith("os.stat_result")) + self.assertStartsWith(rep, "os.stat_result") self.assertIn("st_mode=", rep) self.assertIn("st_ino=", rep) self.assertIn("st_dev=", rep) @@ -307,7 +308,7 @@ def test_copy_replace_with_invisible_fields(self): self.assertEqual(t5.tm_mon, 2) # named invisible fields - self.assertTrue(hasattr(t, 'tm_zone'), f"{t} has no attribute 'tm_zone'") + self.assertHasAttr(t, 'tm_zone') with self.assertRaisesRegex(AttributeError, 'readonly attribute'): t.tm_zone = 'some other zone' self.assertEqual(t2.tm_zone, t.tm_zone) @@ -355,6 +356,14 @@ def test_reference_cycle(self): type(t).refcyle = t """)) + def test_replace_gc_tracked(self): + # Verify that __replace__ results are properly GC-tracked + time_struct = time.gmtime(0) + lst = [] + replaced_struct = time_struct.__replace__(tm_year=lst) + lst.append(replaced_struct) + + self.assertTrue(gc.is_tracked(replaced_struct)) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index ca35804fb360767..70e58c965ab955b 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -22,6 +22,7 @@ import sysconfig import select import shutil +import socket import threading import gc import textwrap @@ -957,6 +958,48 @@ def test_communicate(self): self.assertEqual(stdout, b"banana") self.assertEqual(stderr, b"pineapple") + def test_communicate_memoryview_input(self): + # Test memoryview input with byte elements + test_data = b"Hello, memoryview!" + mv = memoryview(test_data) + p = subprocess.Popen([sys.executable, "-c", + 'import sys; sys.stdout.write(sys.stdin.read())'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stdin.close) + (stdout, stderr) = p.communicate(mv) + self.assertEqual(stdout, test_data) + self.assertIsNone(stderr) + + def test_communicate_memoryview_input_nonbyte(self): + # Test memoryview input with non-byte elements (e.g., int32) + # This tests the fix for gh-134453 where non-byte memoryviews + # had incorrect length tracking on POSIX + import array + # Create an array of 32-bit integers that's large enough to trigger + # the chunked writing behavior (> PIPE_BUF) + pipe_buf = getattr(select, 'PIPE_BUF', 512) + # Each 'i' element is 4 bytes, so we need more than pipe_buf/4 elements + # Add some extra to ensure we exceed the buffer size + num_elements = pipe_buf + 1 + test_array = array.array('i', [0x64306f66 for _ in range(num_elements)]) + expected_bytes = test_array.tobytes() + mv = memoryview(test_array) + + p = subprocess.Popen([sys.executable, "-c", + 'import sys; ' + 'data = sys.stdin.buffer.read(); ' + 'sys.stdout.buffer.write(data)'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + self.addCleanup(p.stdout.close) + self.addCleanup(p.stdin.close) + (stdout, stderr) = p.communicate(mv) + self.assertEqual(stdout, expected_bytes, + msg=f"{len(stdout)=} =? {len(expected_bytes)=}") + self.assertIsNone(stderr) + def test_communicate_timeout(self): p = subprocess.Popen([sys.executable, "-c", 'import sys,os,time;' @@ -992,6 +1035,101 @@ def test_communicate_timeout_large_output(self): (stdout, _) = p.communicate() self.assertEqual(len(stdout), 4 * 64 * 1024) + def test_communicate_timeout_large_input(self): + # Test that timeout is enforced when writing large input to a + # slow-to-read subprocess, and that partial input is preserved + # for continuation after timeout (gh-141473). + # + # This is a regression test for Windows matching POSIX behavior. + # On POSIX, select() is used to multiplex I/O with timeout checking. + # On Windows, stdin writing must also honor the timeout rather than + # blocking indefinitely when the pipe buffer fills. + + input_data = b"x" * (128 * 1024) # > typical pipe buffer + + # Cross-platform wake mechanism: the slow reader connects to a + # loopback TCP socket and blocks in select() on it (capped at 9s + # as a safety net we don't expect to hit). After phase 1 raises + # TimeoutExpired, the parent sends a byte to release the child so + # it drains stdin. A socket (rather than a raw pipe) is required + # because Windows select() only supports sockets, not arbitrary + # file descriptors. + server = socket.create_server(('127.0.0.1', 0), backlog=1) + server.settimeout(10) # bound the accept() if the child fails to start + port = server.getsockname()[1] + # The child sends one byte (low byte of its PID) first so the parent + # can detect the rare case of an unrelated process on the same host + # connecting to our ephemeral port before our child does. A single + # byte gives 1/256 collision odds, which is plenty for flake-prevention. + slow_reader = ( + "import os, socket, sys, select; " + f"s = socket.create_connection(('127.0.0.1', {port}), timeout=9); " + "s.sendall(bytes([os.getpid() & 0xff])); " + "select.select([s], [], [], 9); " + "sys.stdout.buffer.write(sys.stdin.buffer.read())" + ) + + p = subprocess.Popen( + [sys.executable, "-c", slow_reader], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + conn = None + try: + conn, _ = server.accept() + server.close() + server = None + + conn.settimeout(5) + peer_byte = conn.recv(1) + conn.settimeout(None) + self.assertEqual(peer_byte, bytes([p.pid & 0xff]), + f"loopback handshake byte {peer_byte!r} != " + f"low byte of child PID {p.pid} ({p.pid & 0xff:#x})") + + timeout = 0.2 + start = time.monotonic() + try: + p.communicate(input_data, timeout=timeout) + # If we get here without TimeoutExpired, the timeout was ignored + elapsed = time.monotonic() - start + self.fail( + f"TimeoutExpired not raised. communicate() completed in " + f"{elapsed:.2f}s, but slow reader stalls for up to 9s. " + "Stdin writing blocked without enforcing timeout.") + except subprocess.TimeoutExpired: + elapsed = time.monotonic() - start + + # Timeout should occur close to the specified timeout value, + # not after waiting for the subprocess to finish sleeping. + # Allow generous margin for slow CI, but must be well under + # the slow-reader's stall cap. + self.assertLess(elapsed, 5.0, + f"TimeoutExpired raised after {elapsed:.2f}s; expected ~{timeout}s. " + "Stdin writing blocked without checking timeout.") + + # Release the slow reader so it stops blocking and drains stdin. + conn.sendall(b'go') + conn.close() + conn = None + + # After timeout, continue communication. The remaining input + # should be sent and we should receive all data back. + stdout, stderr = p.communicate() + + # Verify all input was eventually received by the subprocess + self.assertEqual(len(stdout), len(input_data), + f"Expected {len(input_data)} bytes output but got {len(stdout)}") + self.assertEqual(stdout, input_data) + finally: + if conn is not None: + conn.close() + if server is not None: + server.close() + p.kill() + p.wait() + # Test for the fd leak reported in http://bugs.python.org/issue2791. def test_communicate_pipe_fd_leak(self): for stdin_pipe in (False, True): @@ -1062,6 +1200,19 @@ def test_writes_before_communicate(self): self.assertEqual(stdout, b"bananasplit") self.assertEqual(stderr, b"") + def test_communicate_stdin_closed_before_call(self): + # gh-70560, gh-74389: stdin.close() before communicate() + # should not raise ValueError from stdin.flush() + with subprocess.Popen([sys.executable, "-c", + 'import sys; sys.exit(0)'], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) as p: + p.stdin.close() # Close stdin before communicate + # This should not raise ValueError + (stdout, stderr) = p.communicate() + self.assertEqual(p.returncode, 0) + def test_universal_newlines_and_text(self): args = [ sys.executable, "-c", @@ -1178,7 +1329,7 @@ def test_universal_newlines_communicate_stdin_stdout_stderr(self): self.assertEqual("line1\nline2\nline3\nline4\nline5\n", stdout) # Python debug build push something like "[42442 refs]\n" # to stderr at exit of subprocess. - self.assertTrue(stderr.startswith("eline2\neline6\neline7\n")) + self.assertStartsWith(stderr, "eline2\neline6\neline7\n") def test_universal_newlines_communicate_encodings(self): # Check that universal newlines mode works for various encodings, @@ -1510,7 +1661,7 @@ def test_issue8780(self): "[sys.executable, '-c', 'print(\"Hello World!\")'])", 'assert retcode == 0')) output = subprocess.check_output([sys.executable, '-c', code]) - self.assertTrue(output.startswith(b'Hello World!'), ascii(output)) + self.assertStartsWith(output, b'Hello World!') def test_handles_closed_on_exception(self): # If CreateProcess exits with an error, ensure the @@ -1643,6 +1794,40 @@ def test_wait_negative_timeout(self): self.assertEqual(proc.wait(), 0) + def test_post_timeout_communicate_sends_input(self): + """GH-141473 regression test; the stdin pipe must close""" + with subprocess.Popen( + [sys.executable, "-uc", """\ +import sys +while c := sys.stdin.read(512): + sys.stdout.write(c) +print() +"""], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) as proc: + try: + data = f"spam{'#'*4096}beans" + proc.communicate( + input=data, + timeout=0, + ) + except subprocess.TimeoutExpired as exc: + pass + # Prior to the bugfix, this would hang as the stdin + # pipe to the child had not been closed. + try: + stdout, stderr = proc.communicate(timeout=15) + except subprocess.TimeoutExpired as exc: + self.fail("communicate() hung waiting on child process that should have seen its stdin pipe close and exit") + self.assertEqual( + proc.returncode, 0, + msg=f"STDERR:\n{stderr}\nSTDOUT:\n{stdout}") + self.assertStartsWith(stdout, "spam") + self.assertIn("beans", stdout) + class RunFuncTestCase(BaseTestCase): def run_python(self, code, **kwargs): @@ -1835,8 +2020,8 @@ def test_encoding_warning(self): capture_output=True) lines = cp.stderr.splitlines() self.assertEqual(len(lines), 2, lines) - self.assertTrue(lines[0].startswith(b"<string>:2: EncodingWarning: ")) - self.assertTrue(lines[1].startswith(b"<string>:3: EncodingWarning: ")) + self.assertStartsWith(lines[0], b"<string>:2: EncodingWarning: ") + self.assertStartsWith(lines[1], b"<string>:3: EncodingWarning: ") def _get_test_grp_name(): @@ -3545,13 +3730,17 @@ def test_startupinfo_copy(self): self.assertEqual(startupinfo.wShowWindow, subprocess.SW_HIDE) self.assertEqual(startupinfo.lpAttributeList, {"handle_list": []}) + # CREATE_NEW_CONSOLE creates a "popup" window. + @support.requires_resource('gui') def test_creationflags(self): # creationflags argument CREATE_NEW_CONSOLE = 16 sys.stderr.write(" a DOS box should flash briefly ...\n") - subprocess.call(sys.executable + - ' -c "import time; time.sleep(0.25)"', - creationflags=CREATE_NEW_CONSOLE) + rc = subprocess.call(sys.executable + + ' -c "import time; time.sleep(0.25)"', + creationflags=CREATE_NEW_CONSOLE) + support.skip_on_low_desktop_heap_memory_subprocess(rc) + self.assertEqual(rc, 0) def test_invalid_args(self): # invalid arguments should raise ValueError diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index 5cef612a340be95..193c8b7d7f3e131 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -547,11 +547,11 @@ def test_special_methods(self): self.assertEqual(s.__reduce__, e.__reduce__) self.assertEqual(s.__reduce_ex__, e.__reduce_ex__) self.assertEqual(s.__getstate__, e.__getstate__) - self.assertFalse(hasattr(s, '__getnewargs__')) - self.assertFalse(hasattr(s, '__getnewargs_ex__')) - self.assertFalse(hasattr(s, '__setstate__')) - self.assertFalse(hasattr(s, '__copy__')) - self.assertFalse(hasattr(s, '__deepcopy__')) + self.assertNotHasAttr(s, '__getnewargs__') + self.assertNotHasAttr(s, '__getnewargs_ex__') + self.assertNotHasAttr(s, '__setstate__') + self.assertNotHasAttr(s, '__copy__') + self.assertNotHasAttr(s, '__deepcopy__') def test_pickling(self): e = E() diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 8446da03e3645b9..d786aac3aede9fd 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -407,10 +407,10 @@ class Obj: with support.swap_attr(obj, "y", 5) as y: self.assertEqual(obj.y, 5) self.assertIsNone(y) - self.assertFalse(hasattr(obj, 'y')) + self.assertNotHasAttr(obj, 'y') with support.swap_attr(obj, "y", 5): del obj.y - self.assertFalse(hasattr(obj, 'y')) + self.assertNotHasAttr(obj, 'y') def test_swap_item(self): D = {"x":1} @@ -559,16 +559,28 @@ def test_args_from_interpreter_flags(self): # -X options ['-X', 'dev'], ['-Wignore', '-X', 'dev'], + ['-X', 'cpu_count=4'], + ['-X', 'disable-remote-debug'], ['-X', 'faulthandler'], ['-X', 'importtime'], ['-X', 'importtime=2'], + ['-X', 'int_max_str_digits=1000'], + ['-X', 'lazy_imports=all'], + ['-X', 'no_debug_ranges'], ['-X', 'showrefcount'], ['-X', 'tracemalloc'], ['-X', 'tracemalloc=3'], + ['-X', 'warn_default_encoding'], ): with self.subTest(opts=opts): self.check_options(opts, 'args_from_interpreter_flags') + with os_helper.temp_dir() as temp_path: + prefix = os.path.join(temp_path, 'pycache') + opts = ['-X', f'pycache_prefix={prefix}'] + with self.subTest(opts=opts): + self.check_options(opts, 'args_from_interpreter_flags') + self.check_options(['-I', '-E', '-s', '-P'], 'args_from_interpreter_flags', ['-I']) @@ -662,6 +674,7 @@ def test_recursive(depth, limit): """) script_helper.assert_python_ok("-c", code) + @support.skip_if_unlimited_stack_size def test_recursion(self): # Test infinite_recursion() and get_recursion_available() functions. def recursive_function(depth): @@ -777,6 +790,7 @@ def test_get_signal_name(self): (128 + int(signal.SIGABRT), 'SIGABRT'), (3221225477, "STATUS_ACCESS_VIOLATION"), (0xC00000FD, "STATUS_STACK_OVERFLOW"), + (0xC0000906, "0xC0000906"), ): self.assertEqual(support.get_signal_name(exitcode), expected, exitcode) @@ -784,14 +798,14 @@ def test_get_signal_name(self): def test_linked_to_musl(self): linked = support.linked_to_musl() self.assertIsNotNone(linked) - if support.is_wasi or support.is_emscripten: + if support.is_wasm32: self.assertTrue(linked) # The value is cached, so make sure it returns the same value again. self.assertIs(linked, support.linked_to_musl()) - # The unlike libc, the musl version is a triple. + # The musl version is either triple or just a major version number. if linked: self.assertIsInstance(linked, tuple) - self.assertEqual(3, len(linked)) + self.assertIn(len(linked), (1, 3)) for v in linked: self.assertIsInstance(v, int) diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index 24d89b09d946ad3..9537de3756fd32f 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -527,6 +527,13 @@ def test_symtable_entry_repr(self): expected = f"<symtable entry top({self.top.get_id()}), line {self.top.get_lineno()}>" self.assertEqual(repr(self.top._table), expected) + def test__symtable_refleak(self): + # Regression test for reference leak in PyUnicode_FSDecoder. + # See https://github.com/python/cpython/issues/139748. + mortal_str = 'this is a mortal string' + # check error path when 'compile_type' AC conversion failed + self.assertRaises(TypeError, symtable.symtable, '', mortal_str, 1) + class ComprehensionTests(unittest.TestCase): def get_identifiers_recursive(self, st, res): diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 0ee17849e281217..de8c94fe8d2d18c 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -382,6 +382,13 @@ Traceback (most recent call last): SyntaxError: invalid syntax +# But prefixes of soft keywords should +# still raise specialized errors + +>>> (mat x) +Traceback (most recent call last): +SyntaxError: invalid syntax. Perhaps you forgot a comma? + From compiler_complex_args(): >>> def f(None=1): @@ -1431,6 +1438,23 @@ Traceback (most recent call last): SyntaxError: cannot use except* statement with literal +Regression tests for gh-133999: + + >>> try: pass + ... except TypeError as name: raise from None + Traceback (most recent call last): + SyntaxError: invalid syntax + + >>> try: pass + ... except* TypeError as name: raise from None + Traceback (most recent call last): + SyntaxError: invalid syntax + + >>> match 1: + ... case 1 | 2 as abc: raise from None + Traceback (most recent call last): + SyntaxError: invalid syntax + Ensure that early = are not matched by the parser as invalid comparisons >>> f(2, 4, x=34); 1 $ 2 Traceback (most recent call last): @@ -2087,6 +2111,25 @@ Traceback (most recent call last): SyntaxError: cannot use subscript as import target +# Check that we don't raise a "cannot use name as import target" error +# if there is an error in an unrelated statement after ';' + +>>> import a as b; None = 1 +Traceback (most recent call last): +SyntaxError: cannot assign to None + +>>> import a, b as c; d = 1; None = 1 +Traceback (most recent call last): +SyntaxError: cannot assign to None + +>>> from a import b as c; None = 1 +Traceback (most recent call last): +SyntaxError: cannot assign to None + +>>> from a import b, c as d; e = 1; None = 1 +Traceback (most recent call last): +SyntaxError: cannot assign to None + # Check that we dont raise the "trailing comma" error if there is more # input to the left of the valid part that we parsed. diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 59ef5c993099f06..e31020b6f8549e1 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -1,12 +1,12 @@ import builtins import codecs -import _datetime import gc import io import locale import operator import os import random +import shutil import socket import struct import subprocess @@ -24,7 +24,7 @@ from test.support import force_not_colorized from test.support import SHORT_TIMEOUT try: - from test.support import interpreters + from concurrent import interpreters except ImportError: interpreters = None import textwrap @@ -57,7 +57,7 @@ def test_original_displayhook(self): dh(None) self.assertEqual(out.getvalue(), "") - self.assertTrue(not hasattr(builtins, "_")) + self.assertNotHasAttr(builtins, "_") # sys.displayhook() requires arguments self.assertRaises(TypeError, dh) @@ -172,7 +172,7 @@ def test_original_excepthook(self): with support.captured_stderr() as err: sys.__excepthook__(*sys.exc_info()) - self.assertTrue(err.getvalue().endswith("ValueError: 42\n")) + self.assertEndsWith(err.getvalue(), "ValueError: 42\n") self.assertRaises(TypeError, sys.__excepthook__) @@ -192,7 +192,7 @@ def test_excepthook_bytes_filename(self): err = err.getvalue() self.assertIn(""" File "b'bytes_filename'", line 123\n""", err) self.assertIn(""" text\n""", err) - self.assertTrue(err.endswith("SyntaxError: msg\n")) + self.assertEndsWith(err, "SyntaxError: msg\n") def test_excepthook(self): with test.support.captured_output("stderr") as stderr: @@ -269,8 +269,7 @@ def check_exit_message(code, expected, **env_vars): rc, out, err = assert_python_failure('-c', code, **env_vars) self.assertEqual(rc, 1) self.assertEqual(out, b'') - self.assertTrue(err.startswith(expected), - "%s doesn't start with %s" % (ascii(err), ascii(expected))) + self.assertStartsWith(err, expected) # test that stderr buffer is flushed before the exit message is written # into stderr @@ -437,7 +436,7 @@ def test_call_tracing(self): @unittest.skipUnless(hasattr(sys, "setdlopenflags"), 'test needs sys.setdlopenflags()') def test_dlopenflags(self): - self.assertTrue(hasattr(sys, "getdlopenflags")) + self.assertHasAttr(sys, "getdlopenflags") self.assertRaises(TypeError, sys.getdlopenflags, 42) oldflags = sys.getdlopenflags() self.assertRaises(TypeError, sys.setdlopenflags) @@ -623,8 +622,7 @@ def g456(): # And the next record must be for g456(). filename, lineno, funcname, sourceline = stack[i+1] self.assertEqual(funcname, "g456") - self.assertTrue((sourceline.startswith("if leave_g.wait(") or - sourceline.startswith("g_raised.set()"))) + self.assertStartsWith(sourceline, ("if leave_g.wait(", "g_raised.set()")) finally: # Reap the spawned thread. leave_g.set() @@ -860,7 +858,7 @@ def test_sys_flags(self): "hash_randomization", "isolated", "dev_mode", "utf8_mode", "warn_default_encoding", "safe_path", "int_max_str_digits") for attr in attrs: - self.assertTrue(hasattr(sys.flags, attr), attr) + self.assertHasAttr(sys.flags, attr) attr_type = bool if attr in ("dev_mode", "safe_path") else int self.assertEqual(type(getattr(sys.flags, attr)), attr_type, attr) self.assertTrue(repr(sys.flags)) @@ -871,12 +869,7 @@ def test_sys_flags(self): def assert_raise_on_new_sys_type(self, sys_attr): # Users are intentionally prevented from creating new instances of # sys.flags, sys.version_info, and sys.getwindowsversion. - arg = sys_attr - attr_type = type(sys_attr) - with self.assertRaises(TypeError): - attr_type(arg) - with self.assertRaises(TypeError): - attr_type.__new__(attr_type, arg) + support.check_disallow_instantiation(self, type(sys_attr), sys_attr) def test_sys_flags_no_instantiation(self): self.assert_raise_on_new_sys_type(sys.flags) @@ -1072,10 +1065,11 @@ def test_implementation(self): levels = {'alpha': 0xA, 'beta': 0xB, 'candidate': 0xC, 'final': 0xF} - self.assertTrue(hasattr(sys.implementation, 'name')) - self.assertTrue(hasattr(sys.implementation, 'version')) - self.assertTrue(hasattr(sys.implementation, 'hexversion')) - self.assertTrue(hasattr(sys.implementation, 'cache_tag')) + self.assertHasAttr(sys.implementation, 'name') + self.assertHasAttr(sys.implementation, 'version') + self.assertHasAttr(sys.implementation, 'hexversion') + self.assertHasAttr(sys.implementation, 'cache_tag') + self.assertHasAttr(sys.implementation, 'supports_isolated_interpreters') version = sys.implementation.version self.assertEqual(version[:2], (version.major, version.minor)) @@ -1089,6 +1083,15 @@ def test_implementation(self): self.assertEqual(sys.implementation.name, sys.implementation.name.lower()) + # https://peps.python.org/pep-0734 + sii = sys.implementation.supports_isolated_interpreters + self.assertIsInstance(sii, bool) + if test.support.check_impl_detail(cpython=True): + if test.support.is_emscripten or test.support.is_wasi: + self.assertFalse(sii) + else: + self.assertTrue(sii) + @test.support.cpython_only def test_debugmallocstats(self): # Test sys._debugmallocstats() @@ -1137,23 +1140,12 @@ def test_getallocatedblocks(self): b = sys.getallocatedblocks() self.assertLessEqual(b, a) try: - # While we could imagine a Python session where the number of - # multiple buffer objects would exceed the sharing of references, - # it is unlikely to happen in a normal test run. - # - # In free-threaded builds each code object owns an array of - # pointers to copies of the bytecode. When the number of - # code objects is a large fraction of the total number of - # references, this can cause the total number of allocated - # blocks to exceed the total number of references. - # - # For some reason, iOS seems to trigger the "unlikely to happen" - # case reliably under CI conditions. It's not clear why; but as - # this test is checking the behavior of getallocatedblock() - # under garbage collection, we can skip this pre-condition check - # for now. See GH-130384. - if not support.Py_GIL_DISABLED and not support.is_apple_mobile: - self.assertLess(a, sys.gettotalrefcount()) + # The reported blocks will include immortalized strings, but the + # total ref count will not. This will sanity check that among all + # other objects (those eligible for garbage collection) there + # are more references being tracked than allocated blocks. + interned_immortal = sys.getunicodeinternedsize(_only_immortal=True) + self.assertLess(a - interned_immortal, sys.gettotalrefcount()) except AttributeError: # gettotalrefcount() not available pass @@ -1345,6 +1337,26 @@ def test_pystats(self): def test_disable_gil_abi(self): self.assertEqual('t' in sys.abiflags, support.Py_GIL_DISABLED) + def test_int_max_str_digits(self): + old_limit = sys.get_int_max_str_digits() + self.assertIsInstance(old_limit, int) + self.assertGreaterEqual(old_limit, 0) + self.addCleanup(sys.set_int_max_str_digits, old_limit) + + sys.set_int_max_str_digits(0) + self.assertEqual(sys.get_int_max_str_digits(), 0) + + sys.set_int_max_str_digits(2_048) + self.assertEqual(sys.get_int_max_str_digits(), 2_048) + + with self.assertRaises(ValueError): + # the minimum is 640 digits + sys.set_int_max_str_digits(5) + with self.assertRaises(ValueError): + sys.set_int_max_str_digits(-2) + with self.assertRaises(TypeError): + sys.set_int_max_str_digits(2_048.0) + @test.support.cpython_only class UnraisableHookTest(unittest.TestCase): @@ -1419,7 +1431,7 @@ def __del__(self): else: self.assertIn("ValueError", report) self.assertIn("del is broken", report) - self.assertTrue(report.endswith("\n")) + self.assertEndsWith(report, "\n") def test_original_unraisablehook_exception_qualname(self): # See bpo-41031, bpo-45083. @@ -1488,6 +1500,7 @@ def hook_func(args): def test_custom_unraisablehook_fail(self): _testcapi = import_helper.import_module('_testcapi') from _testcapi import err_writeunraisable + def hook_func(*args): raise Exception("hook_func failed") @@ -1736,7 +1749,12 @@ def delx(self): del self.__x x = property(getx, setx, delx, "") check(x, size('5Pi')) # PyCapsule - check(_datetime.datetime_CAPI, size('6P')) + try: + import _datetime + except ModuleNotFoundError: + pass + else: + check(_datetime.datetime_CAPI, size('6P')) # rangeiterator check(iter(range(1)), size('3l')) check(iter(range(2**65)), size('3P')) @@ -1955,33 +1973,20 @@ def write(self, s): self.assertEqual(out, b"") self.assertEqual(err, b"") - -def _supports_remote_attaching(): - PROCESS_VM_READV_SUPPORTED = False - - try: - from _remote_debugging import PROCESS_VM_READV_SUPPORTED - except ImportError: - pass - - return PROCESS_VM_READV_SUPPORTED - -@unittest.skipIf(not sys.is_remote_debug_enabled(), "Remote debugging is not enabled") -@unittest.skipIf(sys.platform != "darwin" and sys.platform != "linux" and sys.platform != "win32", - "Test only runs on Linux, Windows and MacOS") -@unittest.skipIf(sys.platform == "linux" and not _supports_remote_attaching(), - "Test only runs on Linux with process_vm_readv support") +@test.support.support_remote_exec_only @test.support.cpython_only class TestRemoteExec(unittest.TestCase): def tearDown(self): test.support.reap_children() - def _run_remote_exec_test(self, script_code, python_args=None, env=None, prologue=''): + def _run_remote_exec_test(self, script_code, python_args=None, env=None, + python_executable=None, prologue='', + after_ready=None, + script_path=os_helper.TESTFN + '_remote.py'): # Create the script that will be remotely executed - script = os_helper.TESTFN + '_remote.py' - self.addCleanup(os_helper.unlink, script) + self.addCleanup(os_helper.unlink, script_path) - with open(script, 'w') as f: + with open(script_path, 'w') as f: f.write(script_code) # Create and run the target process @@ -2024,7 +2029,10 @@ def _run_remote_exec_test(self, script_code, python_args=None, env=None, prologu ''') # Start the target process and capture its output - cmd = [sys.executable] + if python_executable is None: + python_executable = sys.executable + + cmd = [python_executable] if python_args: cmd.extend(python_args) cmd.append(target) @@ -2049,8 +2057,11 @@ def _run_remote_exec_test(self, script_code, python_args=None, env=None, prologu response = client_socket.recv(1024) self.assertEqual(response, b"ready") + if after_ready is not None: + after_ready(proc) + # Try remote exec on the target process - sys.remote_exec(proc.pid, script) + sys.remote_exec(proc.pid, script_path) # Signal script to continue client_socket.sendall(b"continue") @@ -2071,16 +2082,47 @@ def _run_remote_exec_test(self, script_code, python_args=None, env=None, prologu proc.terminate() proc.wait(timeout=SHORT_TIMEOUT) + def _run_remote_exec_with_deleted_mapping(self, deleted_path, **kwargs): + def delete_loaded_mapping(proc): + os_helper.unlink(deleted_path) + with open(f'/proc/{proc.pid}/maps', encoding='utf-8') as maps: + self.assertIn(f'{deleted_path} (deleted)', maps.read()) + + script = 'print("Remote script executed successfully!")' + returncode, stdout, stderr = self._run_remote_exec_test( + script, after_ready=delete_loaded_mapping, **kwargs) + self.assertEqual(returncode, 0) + self.assertIn(b"Remote script executed successfully!", stdout) + self.assertEqual(stderr, b"") + def test_remote_exec(self): """Test basic remote exec functionality""" - script = ''' -print("Remote script executed successfully!") -''' + script = 'print("Remote script executed successfully!")' returncode, stdout, stderr = self._run_remote_exec_test(script) # self.assertEqual(returncode, 0) self.assertIn(b"Remote script executed successfully!", stdout) self.assertEqual(stderr, b"") + def test_remote_exec_bytes(self): + script = 'print("Remote script executed successfully!")' + script_path = os.fsencode(os_helper.TESTFN) + b'_bytes_remote.py' + returncode, stdout, stderr = self._run_remote_exec_test(script, + script_path=script_path) + self.assertIn(b"Remote script executed successfully!", stdout) + self.assertEqual(stderr, b"") + + @unittest.skipUnless(os_helper.TESTFN_UNDECODABLE, 'requires undecodable path') + @unittest.skipIf(sys.platform == 'darwin', + 'undecodable paths are not supported on macOS') + def test_remote_exec_undecodable(self): + script = 'print("Remote script executed successfully!")' + script_path = os_helper.TESTFN_UNDECODABLE + b'_undecodable_remote.py' + for script_path in [script_path, os.fsdecode(script_path)]: + returncode, stdout, stderr = self._run_remote_exec_test(script, + script_path=script_path) + self.assertIn(b"Remote script executed successfully!", stdout) + self.assertEqual(stderr, b"") + def test_remote_exec_with_self_process(self): """Test remote exec with the target process being the same as the test process""" @@ -2110,7 +2152,7 @@ def audit_hook(event, arg): returncode, stdout, stderr = self._run_remote_exec_test(script, prologue=prologue) self.assertEqual(returncode, 0) self.assertIn(b"Remote script executed successfully!", stdout) - self.assertIn(b"Audit event: remote_debugger_script, arg: ", stdout) + self.assertIn(b"Audit event: cpython.remote_debugger_script, arg: ", stdout) self.assertEqual(stderr, b"") def test_remote_exec_with_exception(self): @@ -2157,6 +2199,13 @@ def test_remote_exec_invalid_pid(self): with self.assertRaises(OSError): sys.remote_exec(99999, "print('should not run')") + def test_remote_exec_invalid_script(self): + """Test remote exec with invalid script type""" + with self.assertRaises(TypeError): + sys.remote_exec(0, None) + with self.assertRaises(TypeError): + sys.remote_exec(0, 123) + def test_remote_exec_syntax_error(self): """Test remote exec with syntax error in script""" script = ''' @@ -2172,6 +2221,74 @@ def test_remote_exec_invalid_script_path(self): with self.assertRaises(OSError): sys.remote_exec(os.getpid(), "invalid_script_path") + @unittest.skipUnless(sys.platform == 'linux', 'Linux-only regression test') + @unittest.skipUnless( + sysconfig.get_config_var('Py_ENABLE_SHARED') == 1, + 'requires a shared libpython build') + def test_remote_exec_deleted_libpython(self): + """Test remote exec when the target libpython was deleted.""" + build_dir = sysconfig.get_config_var('abs_builddir') + ldlibrary = sysconfig.get_config_var('LDLIBRARY') + instsoname = sysconfig.get_config_var('INSTSONAME') + if not build_dir or not ldlibrary or not instsoname: + self.skipTest('cannot determine shared libpython location') + + source_libpython = os.path.join(build_dir, instsoname) + if not os.path.exists(source_libpython): + self.skipTest(f'{source_libpython!r} does not exist') + + with os_helper.temp_dir() as lib_dir: + copied_libpython = os.path.join(lib_dir, instsoname) + shutil.copy2(source_libpython, copied_libpython) + if ldlibrary != instsoname: + os.symlink(instsoname, os.path.join(lib_dir, ldlibrary)) + + env = os.environ.copy() + ld_library_path = env.get('LD_LIBRARY_PATH') + env['LD_LIBRARY_PATH'] = lib_dir if not ld_library_path else ( + lib_dir + os.pathsep + ld_library_path) + + self._run_remote_exec_with_deleted_mapping(copied_libpython, + env=env) + + @unittest.skipUnless(sys.platform == 'linux', 'Linux-only regression test') + @unittest.skipUnless( + sysconfig.get_config_var('Py_ENABLE_SHARED') == 0, + 'requires a static Python build') + def test_remote_exec_deleted_static_executable(self): + """Test remote exec when the target static executable was deleted.""" + build_dir = sysconfig.get_config_var('abs_builddir') + stdlib_dir = os.path.dirname(os.path.abspath(os.__file__)) + if not build_dir or not os.path.isdir(stdlib_dir): + self.skipTest('cannot determine build-tree locations') + + pybuilddir_txt = os.path.join(build_dir, 'pybuilddir.txt') + if not os.path.exists(pybuilddir_txt): + self.skipTest(f'{pybuilddir_txt!r} does not exist') + + with open(pybuilddir_txt, encoding='utf-8') as pybuilddir_file: + pybuilddir = pybuilddir_file.read().strip() + source_ext_dir = os.path.join(build_dir, pybuilddir) + if not os.path.isdir(source_ext_dir): + self.skipTest(f'{source_ext_dir!r} does not exist') + + with os_helper.temp_dir() as copied_root: + copied_build_dir = os.path.join(copied_root, 'build') + copied_pybuilddir = os.path.join(copied_build_dir, pybuilddir) + os.makedirs(os.path.dirname(copied_pybuilddir)) + os.symlink(stdlib_dir, os.path.join(copied_root, 'Lib')) + os.symlink(source_ext_dir, copied_pybuilddir) + shutil.copy2(pybuilddir_txt, + os.path.join(copied_build_dir, 'pybuilddir.txt')) + + copied_python = os.path.join(copied_build_dir, + os.path.basename(sys.executable)) + shutil.copy2(sys.executable, copied_python) + + self._run_remote_exec_with_deleted_mapping( + copied_python, python_args=['-S'], + python_executable=copied_python) + def test_remote_exec_in_process_without_debug_fails_envvar(self): """Test remote exec in a process without remote debugging enabled""" script = os_helper.TESTFN + '_remote.py' diff --git a/Lib/test/test_sys_setprofile.py b/Lib/test/test_sys_setprofile.py index 345c022bd2374c7..a22b728b5694196 100644 --- a/Lib/test/test_sys_setprofile.py +++ b/Lib/test/test_sys_setprofile.py @@ -272,6 +272,8 @@ def g(p): self.check_events(g, [(1, 'call', g_ident, None), (2, 'call', f_ident, None), (2, 'return', f_ident, 0), + (2, 'call', f_ident, None), + (2, 'return', f_ident, None), (1, 'return', g_ident, None), ], check_args=True) diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 53e55383bf9c72f..1fe4b6849fc9194 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -186,7 +186,7 @@ def test_posix_venv_scheme(self): # The include directory on POSIX isn't exactly the same as before, # but it is "within" sysconfig_includedir = sysconfig.get_path('include', scheme='posix_venv', vars=vars) - self.assertTrue(sysconfig_includedir.startswith(incpath + os.sep)) + self.assertStartsWith(sysconfig_includedir, incpath + os.sep) def test_nt_venv_scheme(self): # The following directories were hardcoded in the venv module @@ -353,6 +353,13 @@ def test_get_platform(self): self.assertEqual(get_platform(), 'macosx-10.4-%s' % arch) + for macver in range(11, 16): + _osx_support._remove_original_values(get_config_vars()) + get_config_vars()['CFLAGS'] = ('-fno-strict-overflow -Wsign-compare -Wunreachable-code' + '-arch arm64 -fno-common -dynamic -DNDEBUG -g -O3 -Wall') + get_config_vars()['MACOSX_DEPLOYMENT_TARGET'] = f"{macver}.0" + self.assertEqual(get_platform(), 'macosx-%d.0-arm64' % macver) + # linux debian sarge os.name = 'posix' sys.version = ('2.3.5 (#1, Jul 4 2007, 17:28:59) ' @@ -368,10 +375,12 @@ def test_get_platform(self): sys.platform = 'android' get_config_vars()['ANDROID_API_LEVEL'] = 9 for machine, abi in { - 'x86_64': 'x86_64', - 'i686': 'x86', 'aarch64': 'arm64_v8a', + 'arm': 'armeabi_v7a', 'armv7l': 'armeabi_v7a', + 'armv8l': 'armeabi_v7a', + 'i686': 'x86', + 'x86_64': 'x86_64', }.items(): with self.subTest(machine): self._set_uname(('Linux', 'localhost', '3.18.91+', @@ -531,13 +540,10 @@ def test_srcdir(self): Python_h = os.path.join(srcdir, 'Include', 'Python.h') self.assertTrue(os.path.exists(Python_h), Python_h) # <srcdir>/PC/pyconfig.h.in always exists even if unused - pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h.in') - self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) pyconfig_h_in = os.path.join(srcdir, 'pyconfig.h.in') self.assertTrue(os.path.exists(pyconfig_h_in), pyconfig_h_in) if os.name == 'nt': - # <executable dir>/pyconfig.h exists on Windows in a build tree - pyconfig_h = os.path.join(sys.executable, '..', 'pyconfig.h') + pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h') self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) elif os.name == 'posix': makefile_dir = os.path.dirname(sysconfig.get_makefile_filename()) @@ -572,26 +578,26 @@ def test_linux_ext_suffix(self): expected_suffixes = 'i386-linux-gnu.so', 'x86_64-linux-gnux32.so', 'i386-linux-musl.so' else: # 8 byte pointer size expected_suffixes = 'x86_64-linux-gnu.so', 'x86_64-linux-musl.so' - self.assertTrue(suffix.endswith(expected_suffixes), - f'unexpected suffix {suffix!r}') + self.assertEndsWith(suffix, expected_suffixes) @unittest.skipUnless(sys.platform == 'android', 'Android-specific test') def test_android_ext_suffix(self): machine = platform.machine() suffix = sysconfig.get_config_var('EXT_SUFFIX') expected_triplet = { - "x86_64": "x86_64-linux-android", - "i686": "i686-linux-android", "aarch64": "aarch64-linux-android", + "arm": "arm-linux-androideabi", "armv7l": "arm-linux-androideabi", + "armv8l": "arm-linux-androideabi", + "i686": "i686-linux-android", + "x86_64": "x86_64-linux-android", }[machine] - self.assertTrue(suffix.endswith(f"-{expected_triplet}.so"), - f"{machine=}, {suffix=}") + self.assertEndsWith(suffix, f"-{expected_triplet}.so") @unittest.skipUnless(sys.platform == 'darwin', 'OS X-specific test') def test_osx_ext_suffix(self): suffix = sysconfig.get_config_var('EXT_SUFFIX') - self.assertTrue(suffix.endswith('-darwin.so'), suffix) + self.assertEndsWith(suffix, '-darwin.so') def test_always_set_py_debug(self): self.assertIn('Py_DEBUG', sysconfig.get_config_vars()) @@ -714,11 +720,11 @@ def test_sysconfigdata_json(self): ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase'} # Keys dependent on Python being run from the prefix targetted when building (different on relocatable installs) if sysconfig._installation_is_relocated(): - ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase', 'installed_base', 'installed_platbase'} + ignore_keys |= {'prefix', 'exec_prefix', 'base', 'platbase', 'installed_base', 'installed_platbase', 'srcdir'} for key in ignore_keys: - json_config_vars.pop(key) - system_config_vars.pop(key) + json_config_vars.pop(key, None) + system_config_vars.pop(key, None) self.assertEqual(system_config_vars, json_config_vars) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 2d9649237a9382b..f3b61d9fbad6a61 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -55,6 +55,7 @@ def sha256sum(data): zstname = os.path.join(TEMPDIR, "testtar.tar.zst") tmpname = os.path.join(TEMPDIR, "tmp.tar") dotlessname = os.path.join(TEMPDIR, "testtar") +SPACE = b" " sha256_regtype = ( "e09e4bc8b3c9d9177e77256353b36c159f5f040531bbd4b024a8f9b9196c71ce" @@ -548,6 +549,53 @@ def test_extractfile_attrs(self): self.assertIs(fobj.seekable(), True) +class ReadSizeRecorder(io.BytesIO): + # Records the largest size ever passed to read(), so a test can check + # that tarfile does not request far more data than the archive holds + # (which on a real file would pre-allocate it). + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.max_read_size = 0 + + def read(self, size=-1): + if size is not None and size >= 0: + self.max_read_size = max(self.max_read_size, size) + return super().read(size) + + +@support.cpython_only +class ExtendedHeaderMemoryTest(unittest.TestCase): + # gh-151497: the size of a GNU long name/link or a pax extended header is + # read from the archive and is untrusted. A crafted header can claim a + # size far larger than the file actually contains; opening such an archive + # must not try to read (and so pre-allocate) the claimed size in one go. + + def crafted_archive(self, hdrtype): + tarinfo = tarfile.TarInfo("A") + tarinfo.type = hdrtype + tarinfo.size = 0xFFFFFFFF # ~4 GiB claimed in a 512-byte header + return tarinfo.tobuf(format=tarfile.GNU_FORMAT) + + def check(self, hdrtype): + fobj = ReadSizeRecorder(self.crafted_archive(hdrtype)) + try: + with tarfile.open(fileobj=fobj, mode="r:") as tar: + tar.getmembers() + except tarfile.ReadError: + pass # a truncated header is fine; we only check the allocation + # The bogus ~4 GiB size must never reach a single read() call. + self.assertLessEqual(fobj.max_read_size, tarfile._EXTHEADER_READ_CHUNK) + + def test_gnu_longname_oversized_size(self): + self.check(tarfile.GNUTYPE_LONGNAME) + + def test_gnu_longlink_oversized_size(self): + self.check(tarfile.GNUTYPE_LONGLINK) + + def test_pax_header_oversized_size(self): + self.check(tarfile.XHDTYPE) + + class MiscReadTestBase(CommonReadTest): is_stream = False @@ -840,10 +888,90 @@ def test_next_on_empty_tarfile(self): with tarfile.open(fileobj=fd, mode="r") as tf: self.assertEqual(tf.next(), None) + def _setup_symlink_to_target(self, temp_dirpath): + target_filepath = os.path.join(temp_dirpath, "target") + ustar_dirpath = os.path.join(temp_dirpath, "ustar") + hardlink_filepath = os.path.join(ustar_dirpath, "lnktype") + with open(target_filepath, "wb") as f: + f.write(b"target") + os.makedirs(ustar_dirpath) + os.symlink(target_filepath, hardlink_filepath) + return target_filepath, hardlink_filepath + + def _assert_on_file_content(self, filepath, digest): + with open(filepath, "rb") as f: + data = f.read() + self.assertEqual(sha256sum(data), digest) + + @unittest.skipUnless( + hasattr(os, "link"), "Missing hardlink implementation" + ) + @os_helper.skip_unless_symlink + def test_extract_hardlink_on_symlink(self): + """ + This test verifies that extracting a hardlink will not follow an + existing symlink after a FileExistsError on os.link. + """ + with os_helper.temp_dir() as DIR: + target_filepath, hardlink_filepath = self._setup_symlink_to_target(DIR) + with tarfile.open(tarname, encoding="iso8859-1") as tar: + tar.extract("ustar/regtype", DIR, filter="data") + tar.extract("ustar/lnktype", DIR, filter="data") + self._assert_on_file_content(target_filepath, sha256sum(b"target")) + self._assert_on_file_content(hardlink_filepath, sha256_regtype) + + @unittest.skipUnless( + hasattr(os, "link"), "Missing hardlink implementation" + ) + @os_helper.skip_unless_symlink + def test_extractall_hardlink_on_symlink(self): + """ + This test verifies that extracting a hardlink will not follow an + existing symlink after a FileExistsError on os.link. + """ + with os_helper.temp_dir() as DIR: + target_filepath, hardlink_filepath = self._setup_symlink_to_target(DIR) + with tarfile.open(tarname, encoding="iso8859-1") as tar: + tar.extractall( + DIR, members=["ustar/regtype", "ustar/lnktype"], filter="data", + ) + self._assert_on_file_content(target_filepath, sha256sum(b"target")) + self._assert_on_file_content(hardlink_filepath, sha256_regtype) + + +class GzipReadTestBase: + + def test_read_with_extra_field(self): + with open(self.tarname, 'rb') as f: + data = bytearray(f.read()) + flags = data[3] + self.assertEqual(flags, 8) + data[3] = flags | 4 + data[10:10] = b'\x05\x00extra' + with open(tmpname, 'wb') as f: + f.write(data) + print(self.mode) + with tarfile.open(tmpname, mode=self.mode): + pass + + def test_read_with_file_comment(self): + with open(self.tarname, 'rb') as f: + data = bytearray(f.read()) + flags = data[3] + self.assertEqual(flags, 8) + data[3] = flags | 16 + i = data.index(0, 10) + 1 + data[i:i] = b'comment\x00' + with open(tmpname, 'wb') as f: + f.write(data) + with tarfile.open(tmpname, mode=self.mode): + pass + + class MiscReadTest(MiscReadTestBase, unittest.TestCase): test_fail_comp = None -class GzipMiscReadTest(GzipTest, MiscReadTestBase, unittest.TestCase): +class GzipMiscReadTest(GzipTest, GzipReadTestBase, MiscReadTestBase, unittest.TestCase): pass class Bz2MiscReadTest(Bz2Test, MiscReadTestBase, unittest.TestCase): @@ -917,7 +1045,7 @@ def test_compare_members(self): finally: tar1.close() -class GzipStreamReadTest(GzipTest, StreamReadTest): +class GzipStreamReadTest(GzipTest, GzipReadTestBase, StreamReadTest): pass class Bz2StreamReadTest(Bz2Test, StreamReadTest): @@ -1182,6 +1310,25 @@ def test_longname_directory(self): self.assertIsNotNone(tar.getmember(longdir)) self.assertIsNotNone(tar.getmember(longdir.removesuffix('/'))) + def test_longname_file_not_directory(self): + # Test reading a longname file and ensure it is not handled as a directory + # Issue #141707 + buf = io.BytesIO() + with tarfile.open(mode='w', fileobj=buf, format=self.format) as tar: + ti = tarfile.TarInfo() + ti.type = tarfile.AREGTYPE + ti.name = ('a' * 99) + '/' + ('b' * 3) + tar.addfile(ti) + + expected = {t.name: t.type for t in tar.getmembers()} + + buf.seek(0) + with tarfile.open(mode='r', fileobj=buf) as tar: + actual = {t.name: t.type for t in tar.getmembers()} + + self.assertEqual(expected, actual) + + class GNUReadTest(LongnameTest, ReadTest, unittest.TestCase): subdir = "gnu" @@ -1650,7 +1797,7 @@ def test_cwd(self): try: for t in tar: if t.name != ".": - self.assertTrue(t.name.startswith("./"), t.name) + self.assertStartsWith(t.name, "./") finally: tar.close() @@ -1736,6 +1883,16 @@ def test_file_mode(self): finally: os.umask(original_umask) + def test_pathlike_name(self): + expected_name = os.path.abspath(tmpname) + tarpath = os_helper.FakePath(tmpname) + + for func in (tarfile.open, tarfile.TarFile.open): + with self.subTest(): + with func(tarpath, self.mode) as tar: + self.assertEqual(tar.name, expected_name) + os_helper.unlink(tmpname) + class GzipStreamWriteTest(GzipTest, StreamWriteTest): def test_source_directory_not_leaked(self): @@ -2715,6 +2872,31 @@ def test_useful_error_message_when_modules_missing(self): str(excinfo.exception), ) + @unittest.skipUnless(os_helper.can_symlink(), 'requires symlink support') + @unittest.skipUnless(hasattr(os, 'chmod'), "missing os.chmod") + @unittest.mock.patch('os.chmod') + def test_deferred_directory_attributes_update(self, mock_chmod): + # Regression test for gh-127987: setting attributes on arbitrary files + tempdir = os.path.join(TEMPDIR, 'test127987') + def mock_chmod_side_effect(path, mode, **kwargs): + target_path = os.path.realpath(path) + if os.path.commonpath([target_path, tempdir]) != tempdir: + raise Exception("should not try to chmod anything outside the destination", target_path) + mock_chmod.side_effect = mock_chmod_side_effect + + outside_tree_dir = os.path.join(TEMPDIR, 'outside_tree_dir') + with ArchiveMaker() as arc: + arc.add('x', symlink_to='.') + arc.add('x', type=tarfile.DIRTYPE, mode='?rwsrwsrwt') + arc.add('x', symlink_to=outside_tree_dir) + + os.makedirs(outside_tree_dir) + try: + arc.open().extractall(path=tempdir, filter='tar') + finally: + os_helper.rmtree(outside_tree_dir) + os_helper.rmtree(tempdir) + class CommandLineTest(unittest.TestCase): @@ -3275,6 +3457,10 @@ def check_files_present(self, directory): got_paths = set( p.relative_to(directory) for p in pathlib.Path(directory).glob('**/*')) + if self.extraction_filter in (None, 'data'): + # The 'data' filter is expected to reject special files + for path in 'ustar/fifotype', 'ustar/blktype', 'ustar/chrtype': + got_paths.discard(pathlib.Path(path)) self.assertEqual(self.control_paths, got_paths) @contextmanager @@ -3490,11 +3676,12 @@ class ArchiveMaker: with t.open() as tar: ... # `tar` is now a TarFile with 'filename' in it! """ - def __init__(self): + def __init__(self, **kwargs): self.bio = io.BytesIO() + self.tar_kwargs = dict(kwargs) def __enter__(self): - self.tar_w = tarfile.TarFile(mode='w', fileobj=self.bio) + self.tar_w = tarfile.TarFile(mode='w', fileobj=self.bio, **self.tar_kwargs) return self def __exit__(self, *exc): @@ -3503,12 +3690,28 @@ def __exit__(self, *exc): self.bio = None def add(self, name, *, type=None, symlink_to=None, hardlink_to=None, - mode=None, size=None, **kwargs): - """Add a member to the test archive. Call within `with`.""" + mode=None, size=None, content=None, **kwargs): + """Add a member to the test archive. Call within `with`. + + Provides many shortcuts: + - default `type` is based on symlink_to, hardlink_to, and trailing `/` + in name (which is stripped) + - size & content defaults are based on each other + - content can be str or bytes + - mode should be textual ('-rwxrwxrwx') + + (add more! this is unstable internal test-only API) + """ name = str(name) tarinfo = tarfile.TarInfo(name).replace(**kwargs) + if content is not None: + if isinstance(content, str): + content = content.encode() + size = len(content) if size is not None: tarinfo.size = size + if content is None: + content = bytes(tarinfo.size) if mode: tarinfo.mode = _filemode_to_int(mode) if symlink_to is not None: @@ -3522,7 +3725,7 @@ def add(self, name, *, type=None, symlink_to=None, hardlink_to=None, if type is not None: tarinfo.type = type if tarinfo.isreg(): - fileobj = io.BytesIO(bytes(tarinfo.size)) + fileobj = io.BytesIO(content) else: fileobj = None self.tar_w.addfile(tarinfo, fileobj) @@ -3555,8 +3758,41 @@ class TestExtractionFilters(unittest.TestCase): # The destination for the extraction, within `outerdir` destdir = outerdir / 'dest' + @classmethod + def setUpClass(cls): + # Posix and Windows have different pathname resolution: + # either symlink or a '..' component resolve first. + # Let's see which we are on. + if os_helper.can_symlink(): + testpath = os.path.join(TEMPDIR, 'resolution_test') + os.mkdir(testpath) + + # testpath/current links to `.` which is all of: + # - `testpath` + # - `testpath/current` + # - `testpath/current/current` + # - etc. + os.symlink('.', os.path.join(testpath, 'current')) + + # we'll test where `testpath/current/../file` ends up + with open(os.path.join(testpath, 'current', '..', 'file'), 'w'): + pass + + if os.path.exists(os.path.join(testpath, 'file')): + # Windows collapses 'current\..' to '.' first, leaving + # 'testpath\file' + cls.dotdot_resolves_early = True + elif os.path.exists(os.path.join(testpath, '..', 'file')): + # Posix resolves 'current' to '.' first, leaving + # 'testpath/../file' + cls.dotdot_resolves_early = False + else: + raise AssertionError('Could not determine link resolution') + else: + cls.dotdot_resolves_early = False + @contextmanager - def check_context(self, tar, filter): + def check_context(self, tar, filter, *, check_flag=True): """Extracts `tar` to `self.destdir` and allows checking the result If an error occurs, it must be checked using `expect_exception` @@ -3565,27 +3801,40 @@ def check_context(self, tar, filter): except the destination directory itself and parent directories of other files. When checking directories, do so before their contents. + + A file called 'flag' is made in outerdir (i.e. outside destdir) + before extraction; it should not be altered nor should its contents + be read/copied. """ with os_helper.temp_dir(self.outerdir): + flag_path = self.outerdir / 'flag' + flag_path.write_text('capture me') try: tar.extractall(self.destdir, filter=filter) except Exception as exc: self.raised_exception = exc + self.reraise_exception = True self.expected_paths = set() else: self.raised_exception = None + self.reraise_exception = False self.expected_paths = set(self.outerdir.glob('**/*')) self.expected_paths.discard(self.destdir) + self.expected_paths.discard(flag_path) try: - yield + yield self finally: tar.close() - if self.raised_exception: + if self.reraise_exception: raise self.raised_exception self.assertEqual(self.expected_paths, set()) + if check_flag: + self.assertEqual(flag_path.read_text(), 'capture me') + else: + assert filter == 'fully_trusted' def expect_file(self, name, type=None, symlink_to=None, mode=None, - size=None): + size=None, content=None): """Check a single file. See check_context.""" if self.raised_exception: raise self.raised_exception @@ -3604,26 +3853,45 @@ def expect_file(self, name, type=None, symlink_to=None, mode=None, # The symlink might be the same (textually) as what we expect, # but some systems change the link to an equivalent path, so # we fall back to samefile(). - if expected != got: - self.assertTrue(got.samefile(expected)) + try: + if expected != got: + self.assertTrue(got.samefile(expected)) + except Exception as e: + # attach a note, so it's shown even if `samefile` fails + e.add_note(f'{expected=}, {got=}') + raise elif type == tarfile.REGTYPE or type is None: self.assertTrue(path.is_file()) elif type == tarfile.DIRTYPE: self.assertTrue(path.is_dir()) elif type == tarfile.FIFOTYPE: self.assertTrue(path.is_fifo()) + elif type == tarfile.SYMTYPE: + self.assertTrue(path.is_symlink()) else: raise NotImplementedError(type) if size is not None: self.assertEqual(path.stat().st_size, size) + if content is not None: + self.assertEqual(path.read_text(), content) for parent in path.parents: self.expected_paths.discard(parent) + def expect_any_tree(self, name): + """Check a directory; forget about its contents.""" + tree_path = (self.destdir / name).resolve() + self.expect_file(tree_path, type=tarfile.DIRTYPE) + self.expected_paths = { + p for p in self.expected_paths + if tree_path not in p.parents + } + def expect_exception(self, exc_type, message_re='.'): with self.assertRaisesRegex(exc_type, message_re): if self.raised_exception is not None: raise self.raised_exception - self.raised_exception = None + self.reraise_exception = False + return self.raised_exception def test_benign_file(self): with ArchiveMaker() as arc: @@ -3694,10 +3962,19 @@ def test_parent_symlink(self): + "which is outside the destination") with self.check_context(arc.open(), 'data'): - self.expect_exception( - tarfile.LinkOutsideDestinationError, - """'parent' would link to ['"].*outerdir['"], """ - + "which is outside the destination") + if self.dotdot_resolves_early: + # 'current/../..' normalises to '..', which is rejected. + self.expect_exception( + tarfile.LinkOutsideDestinationError, + """'parent' would link to ['"].*outerdir['"], """ + + "which is outside the destination") + else: + # 'current/..' normalises to '.'; the rewritten link is + # created and 'parent/evil' lands harmlessly inside the + # destination. + self.expect_file('current', symlink_to='.') + self.expect_file('parent', symlink_to='.') + self.expect_file('evil') else: # No symlink support. The symlinks are ignored. @@ -3709,38 +3986,83 @@ def test_parent_symlink(self): self.expect_file('parent/evil') @symlink_test - def test_parent_symlink2(self): - # Test interplaying symlinks - # Inspired by 'dirsymlink2b' in jwilk/traversal-archives - - # Posix and Windows have different pathname resolution: - # either symlink or a '..' component resolve first. - # Let's see which we are on. - if os_helper.can_symlink(): - testpath = os.path.join(TEMPDIR, 'resolution_test') - os.mkdir(testpath) + @os_helper.skip_unless_symlink + def test_realpath_limit_attack(self): + # (CVE-2025-4517) - # testpath/current links to `.` which is all of: - # - `testpath` - # - `testpath/current` - # - `testpath/current/current` - # - etc. - os.symlink('.', os.path.join(testpath, 'current')) + with ArchiveMaker() as arc: + # populate the symlinks and dirs that expand in os.path.realpath() + # The component length is chosen so that in common cases, the unexpanded + # path fits in PATH_MAX, but it overflows when the final symlink + # is expanded + steps = "abcdefghijklmnop" + if sys.platform == 'win32': + component = 'd' * 25 + elif 'PC_PATH_MAX' in os.pathconf_names: + max_path_len = os.pathconf(self.outerdir.parent, "PC_PATH_MAX") + path_sep_len = 1 + dest_len = len(str(self.destdir)) + path_sep_len + component_len = (max_path_len - dest_len) // (len(steps) + path_sep_len) + component = 'd' * component_len + else: + raise NotImplementedError("Need to guess component length for {sys.platform}") + path = "" + step_path = "" + for i in steps: + arc.add(os.path.join(path, component), type=tarfile.DIRTYPE, + mode='drwxrwxrwx') + arc.add(os.path.join(path, i), symlink_to=component) + path = os.path.join(path, component) + step_path = os.path.join(step_path, i) + # create the final symlink that exceeds PATH_MAX and simply points + # to the top dir. + # this link will never be expanded by + # os.path.realpath(strict=False), nor anything after it. + linkpath = os.path.join(*steps, "l"*254) + parent_segments = [".."] * len(steps) + arc.add(linkpath, symlink_to=os.path.join(*parent_segments)) + # make a symlink outside to keep the tar command happy + arc.add("escape", symlink_to=os.path.join(linkpath, "..")) + # use the symlinks above, that are not checked, to create a hardlink + # to a file outside of the destination path + arc.add("flaglink", hardlink_to=os.path.join("escape", "flag")) + # now that we have the hardlink we can overwrite the file + arc.add("flaglink", content='overwrite') + # we can also create new files as well! + arc.add("escape/newfile", content='new') + + with (self.subTest('fully_trusted'), + self.check_context(arc.open(), filter='fully_trusted', + check_flag=False)): + if sys.platform == 'win32': + self.expect_exception((FileNotFoundError, FileExistsError)) + elif self.raised_exception: + # Cannot symlink/hardlink: tarfile falls back to getmember() + self.expect_exception(KeyError) + # Otherwise, this block should never enter. + else: + self.expect_any_tree(component) + self.expect_file('flaglink', content='overwrite') + self.expect_file('../newfile', content='new') + self.expect_file('escape', type=tarfile.SYMTYPE) + self.expect_file('a', symlink_to=component) - # we'll test where `testpath/current/../file` ends up - with open(os.path.join(testpath, 'current', '..', 'file'), 'w'): - pass + for filter in 'tar', 'data': + with self.subTest(filter), self.check_context(arc.open(), filter=filter): + exc = self.expect_exception((OSError, KeyError)) + if isinstance(exc, OSError): + if sys.platform == 'win32': + # 3: ERROR_PATH_NOT_FOUND + # 5: ERROR_ACCESS_DENIED + # 206: ERROR_FILENAME_EXCED_RANGE + self.assertIn(exc.winerror, (3, 5, 206)) + else: + self.assertEqual(exc.errno, errno.ENAMETOOLONG) - if os.path.exists(os.path.join(testpath, 'file')): - # Windows collapses 'current\..' to '.' first, leaving - # 'testpath\file' - dotdot_resolves_early = True - elif os.path.exists(os.path.join(testpath, '..', 'file')): - # Posix resolves 'current' to '.' first, leaving - # 'testpath/../file' - dotdot_resolves_early = False - else: - raise AssertionError('Could not determine link resolution') + @symlink_test + def test_parent_symlink2(self): + # Test interplaying symlinks + # Inspired by 'dirsymlink2b' in jwilk/traversal-archives with ArchiveMaker() as arc: @@ -3777,7 +4099,7 @@ def test_parent_symlink2(self): with self.check_context(arc.open(), 'data'): if os_helper.can_symlink(): - if dotdot_resolves_early: + if self.dotdot_resolves_early: # Fail when extracting a file outside destination self.expect_exception( tarfile.OutsideDestinationError, @@ -3897,6 +4219,76 @@ def test_sly_relative2(self): + """['"].*moo['"], which is outside the """ + "destination") + @symlink_test + @os_helper.skip_unless_symlink + def test_normpath_realpath_mismatch(self): + # The link-target check must validate the value that will actually + # be written to disk (the normalised linkname), not the original. + # Here 'a' is a symlink to a deep nonexistent path, so realpath() + # of 'a/../../...' stays inside the destination while normpath() + # collapses 'a/..' lexically and escapes. + depth = len(self.destdir.parts) + 5 + deep = '/'.join(f'p{i}' for i in range(depth)) + sneaky = 'a/' + '../' * depth + 'flag' + for kind in 'symlink_to', 'hardlink_to': + with self.subTest(kind): + with ArchiveMaker() as arc: + arc.add('a', symlink_to=deep) + arc.add('escape', **{kind: sneaky}) + with self.check_context(arc.open(), 'data'): + self.expect_exception( + tarfile.LinkOutsideDestinationError) + + @symlink_test + @os_helper.skip_unless_symlink + def test_symlink_trailing_slash(self): + # A trailing slash on a symlink member's name must not cause the + # link target to be resolved relative to the wrong directory. + with ArchiveMaker() as arc: + t = tarfile.TarInfo('x/') + t.type = tarfile.SYMTYPE + t.linkname = '..' + arc.tar_w.addfile(t) + arc.add('x/escaped', content='hi') + + with self.check_context(arc.open(), 'data'): + self.expect_exception(tarfile.LinkOutsideDestinationError) + + @symlink_test + @os_helper.skip_unless_symlink + def test_link_at_destination(self): + # A link member whose name resolves to the destination directory + # itself must be rejected: otherwise the destination is replaced + # by a symlink and later members can be redirected through it. + for name in '', '.', './': + with ArchiveMaker() as arc: + t = tarfile.TarInfo(name) + t.type = tarfile.SYMTYPE + t.linkname = '.' + arc.tar_w.addfile(t) + + with self.check_context(arc.open(), 'data'): + self.expect_exception(tarfile.OutsideDestinationError) + + @symlink_test + @os_helper.skip_unless_symlink + def test_empty_name_symlink_chain(self): + # Regression test for a chain of empty-named symlinks that + # incrementally redirects the destination outwards. + with ArchiveMaker() as arc: + for name, target in [('', ''), ('a/', '..'), + ('', 'dummy'), ('', 'a'), + ('b/', '..'), + ('', 'dummy'), ('', 'a/b')]: + t = tarfile.TarInfo(name) + t.type = tarfile.SYMTYPE + t.linkname = target + arc.tar_w.addfile(t) + arc.add('escaped', content='hi') + + with self.check_context(arc.open(), 'data'): + self.expect_exception(tarfile.FilterError) + @symlink_test def test_deep_symlink(self): # Test that symlinks and hardlinks inside a directory @@ -3930,8 +4322,8 @@ def test_chains(self): arc.add('symlink2', symlink_to=os.path.join( 'linkdir', 'hardlink2')) arc.add('targetdir/target', size=3) - arc.add('linkdir/hardlink', hardlink_to='targetdir/target') - arc.add('linkdir/hardlink2', hardlink_to='linkdir/symlink') + arc.add('linkdir/hardlink', hardlink_to=os.path.join('targetdir', 'target')) + arc.add('linkdir/hardlink2', hardlink_to=os.path.join('linkdir', 'symlink')) for filter in 'tar', 'data', 'fully_trusted': with self.check_context(arc.open(), filter): @@ -3947,6 +4339,245 @@ def test_chains(self): self.expect_file('linkdir/symlink', size=3) self.expect_file('symlink2', size=3) + @symlink_test + def test_sneaky_hardlink_fallback(self): + # (CVE-2025-4330) + # Test that when hardlink extraction falls back to extracting members + # from the archive, the extracted member is (re-)filtered. + with ArchiveMaker() as arc: + # Create a directory structure so the c/escape symlink stays + # inside the path + arc.add("a/t/dummy") + # Create b/ directory + arc.add("b/") + # Point "c" to the bottom of the tree in "a" + arc.add("c", symlink_to=os.path.join("a", "t")) + # link to non-existant location under "a" + arc.add("c/escape", symlink_to=os.path.join("..", "..", + "link_here")) + # Move "c" to point to "b" ("c/escape" no longer exists) + arc.add("c", symlink_to="b") + # Attempt to create a hard link to "c/escape". Since it doesn't + # exist it will attempt to extract "cescape" but at "boom". + arc.add("boom", hardlink_to=os.path.join("c", "escape")) + + with self.check_context(arc.open(), 'data'): + if not os_helper.can_symlink(): + # When 'c/escape' is extracted, 'c' is a regular + # directory, and 'c/escape' *would* point outside + # the destination if symlinks were allowed. + self.expect_exception( + tarfile.LinkOutsideDestinationError) + elif sys.platform == "win32": + # On Windows, 'c/escape' points outside the destination + self.expect_exception(tarfile.LinkOutsideDestinationError) + else: + e = self.expect_exception( + tarfile.LinkFallbackError, + "link 'boom' would be extracted as a copy of " + + "'c/escape', which was rejected") + self.assertIsInstance(e.__cause__, + tarfile.LinkOutsideDestinationError) + for filter in 'tar', 'fully_trusted': + with self.subTest(filter), self.check_context(arc.open(), filter): + if not os_helper.can_symlink(): + self.expect_file("a/t/dummy") + self.expect_file("b/") + self.expect_file("c/") + else: + self.expect_file("a/t/dummy") + self.expect_file("b/") + self.expect_file("a/t/escape", symlink_to='../../link_here') + self.expect_file("boom", symlink_to='../../link_here') + self.expect_file("c", symlink_to='b') + + @symlink_test + def test_sneaky_hardlink_fallback_deep(self): + # (CVE-2026-11940) + with ArchiveMaker() as arc: + arc.add("a/b/s", symlink_to=os.path.join("..", "escape")) + arc.add("s", hardlink_to=os.path.join("a", "b", "s")) + + with self.check_context(arc.open(), 'data'): + e = self.expect_exception( + tarfile.LinkFallbackError, + "link 's' would be extracted as a copy of " + + "'a/b/s', which was rejected") + self.assertIsInstance(e.__cause__, + tarfile.LinkOutsideDestinationError) + + for filter in 'tar', 'fully_trusted': + with self.subTest(filter), self.check_context(arc.open(), filter): + if not os_helper.can_symlink(): + self.expect_file("a/") + self.expect_file("a/b/") + else: + self.expect_file("a/b/s", symlink_to=os.path.join('..', 'escape')) + self.expect_file("s", symlink_to=os.path.join('..', 'escape')) + + @symlink_test + def test_exfiltration_via_symlink(self): + # (CVE-2025-4138) + # Test changing symlinks that result in a symlink pointing outside + # the extraction directory, unless prevented by 'data' filter's + # normalization. + with ArchiveMaker() as arc: + arc.add("escape", symlink_to=os.path.join('link', 'link', '..', '..', 'link-here')) + arc.add("link", symlink_to='./') + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter): + if os_helper.can_symlink(): + self.expect_file("link", symlink_to='./') + if filter == 'data': + self.expect_file("escape", symlink_to='link-here') + else: + self.expect_file("escape", + symlink_to='link/link/../../link-here') + else: + # Nothing is extracted. + pass + + @symlink_test + def test_chmod_outside_dir(self): + # (CVE-2024-12718) + # Test that members used for delayed updates of directory metadata + # are (re-)filtered. + with ArchiveMaker() as arc: + # "pwn" is a veeeery innocent symlink: + arc.add("a/pwn", symlink_to='.') + # But now "pwn" is also a directory, so it's scheduled to have its + # metadata updated later: + arc.add("a/pwn/", mode='drwxrwxrwx') + # Oops, "pwn" is not so innocent any more: + arc.add("a/pwn", symlink_to='x/../') + # Newly created symlink points to the dest dir, + # so it's OK for the "data" filter. + arc.add('a/x', symlink_to=('../')) + # But now "pwn" points outside the dest dir + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter) as cc: + if not os_helper.can_symlink(): + self.expect_file("a/pwn/") + elif filter == 'data': + self.expect_file("a/x", symlink_to='../') + self.expect_file("a/pwn", symlink_to='.') + else: + self.expect_file("a/x", symlink_to='../') + self.expect_file("a/pwn", symlink_to='x/../') + if sys.platform != "win32": + st_mode = cc.outerdir.stat().st_mode + self.assertNotEqual(st_mode & 0o777, 0o777) + + @symlink_test + @unittest.skipUnless(hasattr(os, 'chown'), "missing os.chown") + @unittest.skipUnless(hasattr(os, 'lchown'), "missing os.lchown") + @unittest.skipUnless(hasattr(os, 'geteuid'), "missing os.geteuid") + @support.subTests('link_type', (tarfile.SYMTYPE, tarfile.LNKTYPE)) + def test_chown_links_on_extract(self, link_type): + with ArchiveMaker() as arc: + arc.add("test.txt", + uid=1337, gid=1337, uname="", gname="", mode='-rwxr-xr-x') + arc.add("link", + type=link_type, + linkname='test.txt', + uid=1337, gid=1337, uname="", gname="", mode='-rwxr-xr-x') + + with ( + os_helper.temp_dir() as tmpdir, + arc.open() as tar, + unittest.mock.patch("os.chown") as mock_chown, + unittest.mock.patch("os.lchown") as mock_lchown, + unittest.mock.patch("os.geteuid") as mock_geteuid, + ): + # Set UID to 0 so chown() is attempted. + mock_geteuid.return_value = 0 + tar.extract("link", path=tmpdir, filter='data') + extract_path = os.path.join(tmpdir, "link") + + if link_type == tarfile.SYMTYPE: + mock_chown.assert_not_called() + mock_lchown.assert_called_once_with(extract_path, -1, -1) + else: + mock_chown.assert_has_calls([ + unittest.mock.call(extract_path, -1, -1), + unittest.mock.call(extract_path, -1, -1) + ]) + mock_lchown.assert_not_called() + + @symlink_test + @unittest.skipUnless(hasattr(os, 'chown'), "missing os.chown") + @unittest.skipUnless(hasattr(os, 'lchown'), "missing os.lchown") + @unittest.skipUnless(hasattr(os, 'geteuid'), "missing os.geteuid") + @support.subTests('link_type', (tarfile.SYMTYPE, tarfile.LNKTYPE)) + def test_chown_links_on_extractall(self, link_type): + with ArchiveMaker() as arc: + arc.add("test.txt", + uid=1337, gid=1337, uname="", gname="", mode='-rwxr-xr-x') + arc.add("link", + type=link_type, + linkname='test.txt', + uid=1337, gid=1337, uname="", gname="", mode='-rwxr-xr-x') + + with ( + os_helper.temp_dir() as tmpdir, + arc.open() as tar, + unittest.mock.patch("os.chown") as mock_chown, + unittest.mock.patch("os.lchown") as mock_lchown, + unittest.mock.patch("os.geteuid") as mock_geteuid, + ): + # Set UID to 0 so chown() is attempted. + mock_geteuid.return_value = 0 + tar.extractall(path=tmpdir, filter='data') + extract_link_path = os.path.join(tmpdir, "link") + extract_file_path = os.path.join(tmpdir, "test.txt") + + if link_type == tarfile.SYMTYPE: + mock_chown.assert_called_once_with(extract_file_path, -1, -1) + mock_lchown.assert_called_once_with(extract_link_path, -1, -1) + else: + mock_chown.assert_has_calls([ + unittest.mock.call(extract_file_path, -1, -1), + unittest.mock.call(extract_link_path, -1, -1) + ]) + mock_lchown.assert_not_called() + + def test_extract_filters_target(self): + # Test that when extract() falls back to extracting (rather than + # linking) a hardlink target, it filters the target. + with ArchiveMaker() as arc: + arc.add("target") + arc.add("link", hardlink_to="target") + def testing_filter(member, path): + if member.name == 'target': + # target: set read-only + return member.replace(mode=stat.S_IRUSR) + # link: don't overwrite the mode + return member.replace(mode=None) + tempdir = pathlib.Path(TEMPDIR) / 'extract' + with os_helper.temp_dir(tempdir), arc.open() as tar: + tar.extract("link", path=tempdir, filter=testing_filter) + path = tempdir / 'link' + if os_helper.can_chmod(): + self.assertFalse(path.stat().st_mode & stat.S_IWUSR) + + def test_link_fallback_normalizes(self): + # Make sure hardlink fallbacks work for non-normalized paths for all + # filters + with ArchiveMaker() as arc: + arc.add("dir/") + arc.add("dir/../afile") + arc.add("link1", hardlink_to='dir/../afile') + arc.add("link2", hardlink_to='dir/../dir/../afile') + + for filter in 'tar', 'data', 'fully_trusted': + with self.check_context(arc.open(), filter) as cc: + self.expect_file("dir/") + self.expect_file("afile") + self.expect_file("link1") + self.expect_file("link2") + def test_modes(self): # Test how file modes are extracted # (Note that the modes are ignored on platforms without working chmod) @@ -4071,24 +4702,64 @@ def test_tar_filter(self): # The 'tar' filter returns TarInfo objects with the same name/type. # (It can also fail for particularly "evil" input, but we don't have # that in the test archive.) - with tarfile.TarFile.open(tarname) as tar: + with tarfile.TarFile.open(tarname, encoding="iso8859-1") as tar: for tarinfo in tar.getmembers(): - filtered = tarfile.tar_filter(tarinfo, '') + try: + filtered = tarfile.tar_filter(tarinfo, '') + except UnicodeEncodeError: + continue self.assertIs(filtered.name, tarinfo.name) self.assertIs(filtered.type, tarinfo.type) def test_data_filter(self): # The 'data' filter either raises, or returns TarInfo with the same # name/type. - with tarfile.TarFile.open(tarname) as tar: + with tarfile.TarFile.open(tarname, encoding="iso8859-1") as tar: for tarinfo in tar.getmembers(): try: filtered = tarfile.data_filter(tarinfo, '') - except tarfile.FilterError: + except (tarfile.FilterError, UnicodeEncodeError): continue self.assertIs(filtered.name, tarinfo.name) self.assertIs(filtered.type, tarinfo.type) + @unittest.skipIf(sys.platform == 'win32', 'requires native bytes paths') + def test_filter_unencodable(self): + # Sanity check using a valid path. + tarinfo = tarfile.TarInfo(os_helper.TESTFN) + filtered = tarfile.tar_filter(tarinfo, '') + self.assertIs(filtered.name, tarinfo.name) + filtered = tarfile.data_filter(tarinfo, '') + self.assertIs(filtered.name, tarinfo.name) + + tarinfo = tarfile.TarInfo('test\x00') + self.assertRaises(ValueError, tarfile.tar_filter, tarinfo, '') + self.assertRaises(ValueError, tarfile.data_filter, tarinfo, '') + tarinfo = tarfile.TarInfo('\ud800') + self.assertRaises(UnicodeEncodeError, tarfile.tar_filter, tarinfo, '') + self.assertRaises(UnicodeEncodeError, tarfile.data_filter, tarinfo, '') + + @unittest.skipIf(sys.platform == 'win32', 'requires native bytes paths') + def test_extract_unencodable(self): + # Create a member with name \xed\xa0\x80 which is UTF-8 encoded + # lone surrogate \ud800. + with ArchiveMaker(encoding='ascii', errors='surrogateescape') as arc: + arc.add('\udced\udca0\udc80') + with os_helper.temp_cwd() as tmp: + tar = arc.open(encoding='utf-8', errors='surrogatepass', + errorlevel=1) + self.assertEqual(tar.getnames(), ['\ud800']) + with self.assertRaises(UnicodeEncodeError): + tar.extractall() + self.assertEqual(os.listdir(), []) + + tar = arc.open(encoding='utf-8', errors='surrogatepass', + errorlevel=0, debug=1) + with support.captured_stderr() as stderr: + tar.extractall() + self.assertEqual(os.listdir(), []) + self.assertIn('tarfile: UnicodeEncodeError ', stderr.getvalue()) + def test_change_default_filter_on_instance(self): tar = tarfile.TarFile(tarname, 'r') def strict_filter(tarinfo, path): @@ -4201,13 +4872,13 @@ def valueerror_filter(tarinfo, path): # If errorlevel is 0, errors affected by errorlevel are ignored with self.check_context(arc.open(errorlevel=0), extracterror_filter): - self.expect_file('file') + pass with self.check_context(arc.open(errorlevel=0), filtererror_filter): - self.expect_file('file') + pass with self.check_context(arc.open(errorlevel=0), oserror_filter): - self.expect_file('file') + pass with self.check_context(arc.open(errorlevel=0), tarerror_filter): self.expect_exception(tarfile.TarError) @@ -4218,7 +4889,7 @@ def valueerror_filter(tarinfo, path): # If 1, all fatal errors are raised with self.check_context(arc.open(errorlevel=1), extracterror_filter): - self.expect_file('file') + pass with self.check_context(arc.open(errorlevel=1), filtererror_filter): self.expect_exception(tarfile.FilterError) @@ -4254,6 +4925,22 @@ def valueerror_filter(tarinfo, path): with self.check_context(arc.open(errorlevel='boo!'), filtererror_filter): self.expect_exception(TypeError) # errorlevel is not int + @support.subTests('format', [tarfile.GNU_FORMAT, tarfile.PAX_FORMAT]) + def test_getmembers_big_size(self, format): + # gh-151981: A loop in seek() for streaming files tried to read the + # declared number of blocks even at EOF + tinfo = tarfile.TarInfo("huge-file") + tinfo.size = 1 << 64 + bio = io.BytesIO() + # Write header without data + bio.write(tinfo.tobuf(format)) + + # Reset & try to get contents + bio.seek(0) + with tarfile.open(fileobj=bio, mode="r|") as tar: + with self.assertRaises(tarfile.ReadError): + tar.getmembers() + class OverwriteTests(archiver_tests.OverwriteTests, unittest.TestCase): testdir = os.path.join(TEMPDIR, "testoverwrite") @@ -4287,6 +4974,161 @@ def extractall(self, ar): ar.extractall(self.testdir, filter='fully_trusted') +class OffsetValidationTests(unittest.TestCase): + tarname = tmpname + invalid_posix_header = ( + # name: 100 bytes + tarfile.NUL * tarfile.LENGTH_NAME + # mode, space, null terminator: 8 bytes + + b"000755" + SPACE + tarfile.NUL + # uid, space, null terminator: 8 bytes + + b"000001" + SPACE + tarfile.NUL + # gid, space, null terminator: 8 bytes + + b"000001" + SPACE + tarfile.NUL + # size, space: 12 bytes + + b"\xff" * 11 + SPACE + # mtime, space: 12 bytes + + tarfile.NUL * 11 + SPACE + # chksum: 8 bytes + + b"0011407" + tarfile.NUL + # type: 1 byte + + tarfile.REGTYPE + # linkname: 100 bytes + + tarfile.NUL * tarfile.LENGTH_LINK + # magic: 6 bytes, version: 2 bytes + + tarfile.POSIX_MAGIC + # uname: 32 bytes + + tarfile.NUL * 32 + # gname: 32 bytes + + tarfile.NUL * 32 + # devmajor, space, null terminator: 8 bytes + + tarfile.NUL * 6 + SPACE + tarfile.NUL + # devminor, space, null terminator: 8 bytes + + tarfile.NUL * 6 + SPACE + tarfile.NUL + # prefix: 155 bytes + + tarfile.NUL * tarfile.LENGTH_PREFIX + # padding: 12 bytes + + tarfile.NUL * 12 + ) + invalid_gnu_header = ( + # name: 100 bytes + tarfile.NUL * tarfile.LENGTH_NAME + # mode, null terminator: 8 bytes + + b"0000755" + tarfile.NUL + # uid, null terminator: 8 bytes + + b"0000001" + tarfile.NUL + # gid, space, null terminator: 8 bytes + + b"0000001" + tarfile.NUL + # size, space: 12 bytes + + b"\xff" * 11 + SPACE + # mtime, space: 12 bytes + + tarfile.NUL * 11 + SPACE + # chksum: 8 bytes + + b"0011327" + tarfile.NUL + # type: 1 byte + + tarfile.REGTYPE + # linkname: 100 bytes + + tarfile.NUL * tarfile.LENGTH_LINK + # magic: 8 bytes + + tarfile.GNU_MAGIC + # uname: 32 bytes + + tarfile.NUL * 32 + # gname: 32 bytes + + tarfile.NUL * 32 + # devmajor, null terminator: 8 bytes + + tarfile.NUL * 8 + # devminor, null terminator: 8 bytes + + tarfile.NUL * 8 + # padding: 167 bytes + + tarfile.NUL * 167 + ) + invalid_v7_header = ( + # name: 100 bytes + tarfile.NUL * tarfile.LENGTH_NAME + # mode, space, null terminator: 8 bytes + + b"000755" + SPACE + tarfile.NUL + # uid, space, null terminator: 8 bytes + + b"000001" + SPACE + tarfile.NUL + # gid, space, null terminator: 8 bytes + + b"000001" + SPACE + tarfile.NUL + # size, space: 12 bytes + + b"\xff" * 11 + SPACE + # mtime, space: 12 bytes + + tarfile.NUL * 11 + SPACE + # chksum: 8 bytes + + b"0010070" + tarfile.NUL + # type: 1 byte + + tarfile.REGTYPE + # linkname: 100 bytes + + tarfile.NUL * tarfile.LENGTH_LINK + # padding: 255 bytes + + tarfile.NUL * 255 + ) + valid_gnu_header = tarfile.TarInfo("filename").tobuf(tarfile.GNU_FORMAT) + data_block = b"\xff" * tarfile.BLOCKSIZE + + def _write_buffer(self, buffer): + with open(self.tarname, "wb") as f: + f.write(buffer) + + def _get_members(self, ignore_zeros=None): + with open(self.tarname, "rb") as f: + with tarfile.open( + mode="r", fileobj=f, ignore_zeros=ignore_zeros + ) as tar: + return tar.getmembers() + + def _assert_raises_read_error_exception(self): + with self.assertRaisesRegex( + tarfile.ReadError, "file could not be opened successfully" + ): + self._get_members() + + def test_invalid_offset_header_validations(self): + for tar_format, invalid_header in ( + ("posix", self.invalid_posix_header), + ("gnu", self.invalid_gnu_header), + ("v7", self.invalid_v7_header), + ): + with self.subTest(format=tar_format): + self._write_buffer(invalid_header) + self._assert_raises_read_error_exception() + + def test_early_stop_at_invalid_offset_header(self): + buffer = self.valid_gnu_header + self.invalid_gnu_header + self.valid_gnu_header + self._write_buffer(buffer) + members = self._get_members() + self.assertEqual(len(members), 1) + self.assertEqual(members[0].name, "filename") + self.assertEqual(members[0].offset, 0) + + def test_ignore_invalid_archive(self): + # 3 invalid headers with their respective data + buffer = (self.invalid_gnu_header + self.data_block) * 3 + self._write_buffer(buffer) + members = self._get_members(ignore_zeros=True) + self.assertEqual(len(members), 0) + + def test_ignore_invalid_offset_headers(self): + for first_block, second_block, expected_offset in ( + ( + (self.valid_gnu_header), + (self.invalid_gnu_header + self.data_block), + 0, + ), + ( + (self.invalid_gnu_header + self.data_block), + (self.valid_gnu_header), + 1024, + ), + ): + self._write_buffer(first_block + second_block) + members = self._get_members(ignore_zeros=True) + self.assertEqual(len(members), 1) + self.assertEqual(members[0].name, "filename") + self.assertEqual(members[0].offset, expected_offset) + + def setUpModule(): os_helper.unlink(TEMPDIR) os.makedirs(TEMPDIR) diff --git a/Lib/test/test_tcl.py b/Lib/test/test_tcl.py index d479f7d7515d9b0..47450d3fd5976fa 100644 --- a/Lib/test/test_tcl.py +++ b/Lib/test/test_tcl.py @@ -40,6 +40,9 @@ def setUp(self): self.interp = Tcl() self.wantobjects = self.interp.tk.wantobjects() + def passValue(self, value): + return self.interp.call('set', '_', value) + def testEval(self): tcl = self.interp tcl.eval('set a 1') @@ -490,8 +493,7 @@ def test_expr_bignum(self): self.assertIsInstance(result, str) def test_passing_values(self): - def passValue(value): - return self.interp.call('set', '_', value) + passValue = self.passValue self.assertEqual(passValue(True), True if self.wantobjects else '1') self.assertEqual(passValue(False), False if self.wantobjects else '0') @@ -537,6 +539,24 @@ def passValue(value): self.assertEqual(passValue(['a', ['b', 'c']]), ('a', ('b', 'c')) if self.wantobjects else 'a {b c}') + def test_set_object_concurrent_mutation_in_sequence_conversion(self): + # Prevent SIGSEV when the object to convert is concurrently mutated. + # See: https://github.com/python/cpython/issues/143310. + + string = "value" + + class Value: + def __str__(self): + values.clear() + return string + + class List(list): + pass + + expect = (string, "pad") if self.wantobjects else f"{string} pad" + self.assertEqual(self.passValue(values := [Value(), "pad"]), expect) + self.assertEqual(self.passValue(values := List([Value(), "pad"])), expect) + def test_user_command(self): result = None def testfunc(arg): @@ -797,6 +817,10 @@ def test_huge_string_builtins2(self, size): def setUpModule(): + wantobjects = support.get_resource_value('wantobjects') + if wantobjects is not None: + unittest.enterModuleContext( + support.swap_attr(tkinter, 'wantobjects', int(wantobjects))) if support.verbose: tcl = Tcl() print('patchlevel =', tcl.call('info', 'patchlevel'), flush=True) diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index d46d3c0f0406019..70425ce49e9da05 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -330,17 +330,40 @@ def _mock_candidate_names(*names): class TestBadTempdir: def test_read_only_directory(self): with _inside_empty_temp_dir(): - oldmode = mode = os.stat(tempfile.tempdir).st_mode - mode &= ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) - os.chmod(tempfile.tempdir, mode) + probe = os.path.join(tempfile.tempdir, 'probe') + if os.name == 'nt': + # Use security identifier *S-1-1-0 instead + # of localized "Everyone" to not depend on the locale. + cmd = ['icacls', tempfile.tempdir, '/deny', '*S-1-1-0:(W)'] + stdout = None if support.verbose > 1 else subprocess.DEVNULL + subprocess.run(cmd, check=True, stdout=stdout) + else: + oldmode = mode = os.stat(tempfile.tempdir).st_mode + mode &= ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) + mode = stat.S_IREAD + os.chmod(tempfile.tempdir, mode) try: - if os.access(tempfile.tempdir, os.W_OK): + # Check that the directory is read-only. + try: + os.mkdir(probe) + except PermissionError: + pass + else: + os.rmdir(probe) self.skipTest("can't set the directory read-only") + # gh-66305: Now it takes a split second, but previously + # it took about 10 days on Windows. with self.assertRaises(PermissionError): self.make_temp() - self.assertEqual(os.listdir(tempfile.tempdir), []) finally: - os.chmod(tempfile.tempdir, oldmode) + if os.name == 'nt': + # Use security identifier *S-1-1-0 instead + # of localized "Everyone" to not depend on the locale. + cmd = ['icacls', tempfile.tempdir, '/grant:r', '*S-1-1-0:(M)'] + subprocess.run(cmd, check=True, stdout=stdout) + else: + os.chmod(tempfile.tempdir, oldmode) + self.assertEqual(os.listdir(tempfile.tempdir), []) def test_nonexisting_directory(self): with _inside_empty_temp_dir(): @@ -516,11 +539,11 @@ def test_collision_with_existing_file(self): _mock_candidate_names('aaa', 'aaa', 'bbb'): (fd1, name1) = self.make_temp() os.close(fd1) - self.assertTrue(name1.endswith('aaa')) + self.assertEndsWith(name1, 'aaa') (fd2, name2) = self.make_temp() os.close(fd2) - self.assertTrue(name2.endswith('bbb')) + self.assertEndsWith(name2, 'bbb') def test_collision_with_existing_directory(self): # _mkstemp_inner tries another name when a directory with @@ -528,11 +551,11 @@ def test_collision_with_existing_directory(self): with _inside_empty_temp_dir(), \ _mock_candidate_names('aaa', 'aaa', 'bbb'): dir = tempfile.mkdtemp() - self.assertTrue(dir.endswith('aaa')) + self.assertEndsWith(dir, 'aaa') (fd, name) = self.make_temp() os.close(fd) - self.assertTrue(name.endswith('bbb')) + self.assertEndsWith(name, 'bbb') class TestGetTempPrefix(BaseTestCase): @@ -828,9 +851,9 @@ def test_collision_with_existing_file(self): _mock_candidate_names('aaa', 'aaa', 'bbb'): file = tempfile.NamedTemporaryFile(delete=False) file.close() - self.assertTrue(file.name.endswith('aaa')) + self.assertEndsWith(file.name, 'aaa') dir = tempfile.mkdtemp() - self.assertTrue(dir.endswith('bbb')) + self.assertEndsWith(dir, 'bbb') def test_collision_with_existing_directory(self): # mkdtemp tries another name when a directory with @@ -838,9 +861,9 @@ def test_collision_with_existing_directory(self): with _inside_empty_temp_dir(), \ _mock_candidate_names('aaa', 'aaa', 'bbb'): dir1 = tempfile.mkdtemp() - self.assertTrue(dir1.endswith('aaa')) + self.assertEndsWith(dir1, 'aaa') dir2 = tempfile.mkdtemp() - self.assertTrue(dir2.endswith('bbb')) + self.assertEndsWith(dir2, 'bbb') def test_for_tempdir_is_bytes_issue40701_api_warts(self): orig_tempdir = tempfile.tempdir diff --git a/Lib/test/test_termios.py b/Lib/test/test_termios.py index e5d11cf84d2a66e..aa8927c1ba7ea23 100644 --- a/Lib/test/test_termios.py +++ b/Lib/test/test_termios.py @@ -104,7 +104,7 @@ def test_tcsendbreak(self): try: termios.tcsendbreak(self.fd, 1) except termios.error as exc: - if exc.args[0] == errno.ENOTTY and sys.platform.startswith(('freebsd', "netbsd")): + if exc.args[0] == errno.ENOTTY and sys.platform.startswith(('freebsd', 'netbsd', 'openbsd', 'dragonfly')): self.skipTest('termios.tcsendbreak() is not supported ' 'with pseudo-terminals (?) on this platform') raise @@ -180,6 +180,9 @@ def test_tcflow(self): termios.tcflow(self.fd, termios.TCOON) termios.tcflow(self.fd, termios.TCIOFF) termios.tcflow(self.fd, termios.TCION) + # Discard the transmitted STOP and START characters, + # otherwise closing the pseudo-terminal can block. + termios.tcflush(self.fd, termios.TCOFLUSH) @support.skip_android_selinux('tcflow') def test_tcflow_errors(self): @@ -290,8 +293,8 @@ def test_ioctl_constants(self): self.assertGreaterEqual(value, 0) def test_exception(self): - self.assertTrue(issubclass(termios.error, Exception)) - self.assertFalse(issubclass(termios.error, OSError)) + self.assertIsSubclass(termios.error, Exception) + self.assertNotIsSubclass(termios.error, OSError) if __name__ == '__main__': diff --git a/Lib/test/test_textwrap.py b/Lib/test/test_textwrap.py index cbd383ea4e26561..aca1f427656bb50 100644 --- a/Lib/test/test_textwrap.py +++ b/Lib/test/test_textwrap.py @@ -605,7 +605,7 @@ def test_break_long(self): # bug 1146. Prevent a long word to be wrongly wrapped when the # preceding word is exactly one character shorter than the width self.check_wrap(self.text, 12, - ['Did you say ', + ['Did you say', '"supercalifr', 'agilisticexp', 'ialidocious?', @@ -633,7 +633,7 @@ def test_nobreak_long(self): def test_max_lines_long(self): self.check_wrap(self.text, 12, - ['Did you say ', + ['Did you say', '"supercalifr', 'agilisticexp', '[...]'], diff --git a/Lib/test/test_thread.py b/Lib/test/test_thread.py index d94e04250c93076..ac924728febc991 100644 --- a/Lib/test/test_thread.py +++ b/Lib/test/test_thread.py @@ -76,6 +76,14 @@ def test_stack_size(self): thread.stack_size(0) self.assertEqual(thread.stack_size(), 0, "stack_size not reset to default") + with self.assertRaises(ValueError): + # 123 bytes is too small + thread.stack_size(123) + + with self.assertRaises(ValueError): + # size must be positive + thread.stack_size(-4096) + @unittest.skipIf(os.name not in ("nt", "posix"), 'test meant for nt and posix') def test_nt_and_posix_stack_size(self): try: diff --git a/Lib/test/test_threadedtempfile.py b/Lib/test/test_threadedtempfile.py index 420fc6ec8be3d8d..acb427b0c78ae94 100644 --- a/Lib/test/test_threadedtempfile.py +++ b/Lib/test/test_threadedtempfile.py @@ -15,6 +15,7 @@ import tempfile +from test import support from test.support import threading_helper import unittest import io @@ -49,7 +50,8 @@ def run(self): class ThreadedTempFileTest(unittest.TestCase): - def test_main(self): + @support.bigmemtest(size=NUM_THREADS, memuse=60*2**20, dry_run=False) + def test_main(self, size): threads = [TempFileGreedy() for i in range(NUM_THREADS)] with threading_helper.start_threads(threads, startEvent.set): pass diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 4ab38c2598b50a4..bb51ddd38e2cade 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -28,7 +28,7 @@ from test import support try: - from test.support import interpreters + from concurrent import interpreters except ImportError: interpreters = None @@ -530,7 +530,8 @@ def test_enumerate_after_join(self): finally: sys.setswitchinterval(old_interval) - def test_join_from_multiple_threads(self): + @support.bigmemtest(size=20, memuse=72*2**20, dry_run=False) + def test_join_from_multiple_threads(self, size): # Thread.join() should be thread-safe errors = [] @@ -1351,6 +1352,62 @@ def do_flush(*args, **kwargs): ''') assert_python_ok("-c", script) + @skip_unless_reliable_fork + @unittest.skipUnless(hasattr(threading, 'get_native_id'), "test needs threading.get_native_id()") + def test_native_id_after_fork(self): + script = """if True: + import threading + import os + from test import support + + parent_thread_native_id = threading.current_thread().native_id + print(parent_thread_native_id, flush=True) + assert parent_thread_native_id == threading.get_native_id() + childpid = os.fork() + if childpid == 0: + print(threading.current_thread().native_id, flush=True) + assert threading.current_thread().native_id == threading.get_native_id() + else: + try: + assert parent_thread_native_id == threading.current_thread().native_id + assert parent_thread_native_id == threading.get_native_id() + finally: + support.wait_process(childpid, exitcode=0) + """ + rc, out, err = assert_python_ok('-c', script) + self.assertEqual(rc, 0) + self.assertEqual(err, b"") + native_ids = out.strip().splitlines() + self.assertEqual(len(native_ids), 2) + self.assertNotEqual(native_ids[0], native_ids[1]) + + def test_stop_the_world_during_finalization(self): + # gh-137433: Test functions that trigger a stop-the-world in the free + # threading build concurrent with interpreter finalization. + script = """if True: + import gc + import sys + import threading + NUM_THREADS = 5 + b = threading.Barrier(NUM_THREADS + 1) + def run_in_bg(): + b.wait() + while True: + sys.setprofile(None) + gc.collect() + + for _ in range(NUM_THREADS): + t = threading.Thread(target=run_in_bg, daemon=True) + t.start() + + b.wait() + print("Exiting...") + """ + rc, out, err = assert_python_ok('-c', script) + self.assertEqual(rc, 0) + self.assertEqual(err, b"") + self.assertEqual(out.strip(), b"Exiting...") + class ThreadJoinOnShutdown(BaseTestCase): def _run_and_join(self, script): @@ -1431,7 +1488,8 @@ def worker(): self._run_and_join(script) @unittest.skipIf(sys.platform in platforms_to_skip, "due to known OS bug") - def test_4_daemon_threads(self): + @support.bigmemtest(size=40, memuse=70*2**20, dry_run=False) + def test_4_daemon_threads(self, size): # Check that a daemon thread cannot crash the interpreter on shutdown # by manipulating internal structures that are being disposed of in # the main thread. @@ -1669,6 +1727,7 @@ def task(): self.assertEqual(os.read(r_interp, 1), DONE) @cpython_only + @support.skip_if_sanitizer(thread=True, memory=True) def test_daemon_threads_fatal_error(self): import_module("_testcapi") subinterp_code = f"""if 1: @@ -1687,10 +1746,7 @@ def f(): _testcapi.run_in_subinterp(%r) """ % (subinterp_code,) - with test.support.SuppressCrashReport(): - rc, out, err = assert_python_failure("-c", script) - self.assertIn("Fatal Python error: Py_EndInterpreter: " - "not the last thread", err.decode()) + assert_python_ok("-c", script) def _check_allowed(self, before_start='', *, allowed=True, @@ -1962,6 +2018,23 @@ def modify_file(): t.start() t.join() + def test_dummy_thread_on_interpreter_shutdown(self): + # GH-130522: When `threading` held a reference to itself and then a + # _DummyThread() object was created, destruction of the dummy thread + # would emit an unraisable exception at shutdown, due to a lock being + # destroyed. + code = """if True: + import sys + import threading + + threading.x = sys.modules[__name__] + x = threading._DummyThread() + """ + rc, out, err = assert_python_ok("-c", code) + self.assertEqual(rc, 0) + self.assertEqual(out, b"") + self.assertEqual(err, b"") + class ThreadRunFail(threading.Thread): def run(self): @@ -2182,6 +2255,9 @@ def test__all__(self): @unittest.skipUnless(hasattr(_thread, 'set_name'), "missing _thread.set_name") @unittest.skipUnless(hasattr(_thread, '_get_name'), "missing _thread._get_name") def test_set_name(self): + # Ensure main thread name is restored after test + self.addCleanup(_thread.set_name, _thread._get_name()) + # set_name() limit in bytes truncate = getattr(_thread, "_NAME_MAXLEN", None) limit = truncate or 100 @@ -2221,7 +2297,8 @@ def test_set_name(self): tests.append(os_helper.TESTFN_UNENCODABLE) if sys.platform.startswith("sunos"): - encoding = "utf-8" + # Use ASCII encoding on Solaris/Illumos/OpenIndiana + encoding = "ascii" else: encoding = sys.getfilesystemencoding() @@ -2237,7 +2314,7 @@ def work(): if truncate is not None: encoded = encoded[:truncate] if sys.platform.startswith("sunos"): - expected = encoded.decode("utf-8", "surrogateescape") + expected = encoded.decode("ascii", "surrogateescape") else: expected = os.fsdecode(encoded) else: @@ -2256,7 +2333,11 @@ def work(): if '\0' in expected: expected = expected.split('\0', 1)[0] - with self.subTest(name=name, expected=expected): + with self.subTest(name=name, expected=expected, thread="main"): + _thread.set_name(name) + self.assertEqual(_thread._get_name(), expected) + + with self.subTest(name=name, expected=expected, thread="worker"): work_name = None thread = threading.Thread(target=work, name=name) thread.start() diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index d06f65270efe79f..ff7d982fb3621c0 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -577,11 +577,10 @@ def test_thread_time(self): # thread_time() should not include time spend during a sleep start = time.thread_time() - time.sleep(0.100) + time.sleep(0.200) stop = time.thread_time() - # use 20 ms because thread_time() has usually a resolution of 15 ms - # on Windows - self.assertLess(stop - start, 0.020) + # gh-143528: use 100 ms to support slow CI + self.assertLess(stop - start, 0.100) info = time.get_clock_info('thread_time') self.assertTrue(info.monotonic) @@ -761,17 +760,17 @@ def test_localtime_timezone(self): # Get the localtime and examine it for the offset and zone. lt = time.localtime() - self.assertTrue(hasattr(lt, "tm_gmtoff")) - self.assertTrue(hasattr(lt, "tm_zone")) + self.assertHasAttr(lt, "tm_gmtoff") + self.assertHasAttr(lt, "tm_zone") # See if the offset and zone are similar to the module # attributes. if lt.tm_gmtoff is None: - self.assertTrue(not hasattr(time, "timezone")) + self.assertNotHasAttr(time, "timezone") else: self.assertEqual(lt.tm_gmtoff, -[time.timezone, time.altzone][lt.tm_isdst]) if lt.tm_zone is None: - self.assertTrue(not hasattr(time, "tzname")) + self.assertNotHasAttr(time, "tzname") else: self.assertEqual(lt.tm_zone, time.tzname[lt.tm_isdst]) @@ -978,6 +977,17 @@ def test_FromSecondsObject(self): with self.assertRaises(ValueError): _PyTime_FromSecondsObject(float('nan'), time_rnd) + def test_FromSecondsObject_float_overflow_message(self): + # Float path must report a PyTime_t overflow, like the integer path. + from _testinternalcapi import _PyTime_FromSecondsObject + for value in (PyTime_MAX, PyTime_MIN): + for time_rnd, _ in ROUNDING_MODES: + with self.subTest(value=value, time_rnd=time_rnd): + with self.assertRaisesRegex( + OverflowError, + "timestamp out of range for C PyTime_t"): + _PyTime_FromSecondsObject(value, time_rnd) + def test_AsSecondsDouble(self): from _testcapi import PyTime_AsSecondsDouble @@ -1184,11 +1194,11 @@ def test_clock_functions(self): if mac_ver >= (10, 12): for name in clock_names: - self.assertTrue(hasattr(time, name), f"time.{name} is not available") + self.assertHasAttr(time, name) else: for name in clock_names: - self.assertFalse(hasattr(time, name), f"time.{name} is available") + self.assertNotHasAttr(time, name) if __name__ == "__main__": diff --git a/Lib/test/test_timeit.py b/Lib/test/test_timeit.py index f5ae0a84eb3506c..2b0745e947cee9d 100644 --- a/Lib/test/test_timeit.py +++ b/Lib/test/test_timeit.py @@ -222,8 +222,8 @@ def test_repeat_function_zero_iters(self): def assert_exc_string(self, exc_string, expected_exc_name): exc_lines = exc_string.splitlines() self.assertGreater(len(exc_lines), 2) - self.assertTrue(exc_lines[0].startswith('Traceback')) - self.assertTrue(exc_lines[-1].startswith(expected_exc_name)) + self.assertStartsWith(exc_lines[0], 'Traceback') + self.assertStartsWith(exc_lines[-1], expected_exc_name) def test_print_exc(self): s = io.StringIO() @@ -302,7 +302,7 @@ def test_main_help(self): def test_main_verbose(self): s = self.run_main(switches=['-v']) self.assertEqual(s, dedent("""\ - 1 loop -> 1 secs + 1 loop -> 1 sec raw times: 1 sec, 1 sec, 1 sec, 1 sec, 1 sec @@ -312,19 +312,19 @@ def test_main_verbose(self): def test_main_very_verbose(self): s = self.run_main(seconds_per_increment=0.000_030, switches=['-vv']) self.assertEqual(s, dedent("""\ - 1 loop -> 3e-05 secs - 2 loops -> 6e-05 secs - 5 loops -> 0.00015 secs - 10 loops -> 0.0003 secs - 20 loops -> 0.0006 secs - 50 loops -> 0.0015 secs - 100 loops -> 0.003 secs - 200 loops -> 0.006 secs - 500 loops -> 0.015 secs - 1000 loops -> 0.03 secs - 2000 loops -> 0.06 secs - 5000 loops -> 0.15 secs - 10000 loops -> 0.3 secs + 1 loop -> 3e-05 sec + 2 loops -> 6e-05 sec + 5 loops -> 0.00015 sec + 10 loops -> 0.0003 sec + 20 loops -> 0.0006 sec + 50 loops -> 0.0015 sec + 100 loops -> 0.003 sec + 200 loops -> 0.006 sec + 500 loops -> 0.015 sec + 1000 loops -> 0.03 sec + 2000 loops -> 0.06 sec + 5000 loops -> 0.15 sec + 10000 loops -> 0.3 sec raw times: 300 msec, 300 msec, 300 msec, 300 msec, 300 msec diff --git a/Lib/test/test_tkinter/__init__.py b/Lib/test/test_tkinter/__init__.py index aa196c12c804ac0..62890c705a6ca66 100644 --- a/Lib/test/test_tkinter/__init__.py +++ b/Lib/test/test_tkinter/__init__.py @@ -22,3 +22,9 @@ def load_tests(*args): return load_package_tests(os.path.dirname(__file__), *args) + +def setUpModule(): + wantobjects = support.get_resource_value('wantobjects') + if wantobjects is not None: + unittest.enterModuleContext( + support.swap_attr(tkinter, 'wantobjects', int(wantobjects))) diff --git a/Lib/test/test_tkinter/support.py b/Lib/test/test_tkinter/support.py index ebb9e00ff91bf0e..31feee2b8a40ee6 100644 --- a/Lib/test/test_tkinter/support.py +++ b/Lib/test/test_tkinter/support.py @@ -1,5 +1,15 @@ import functools +import time import tkinter +import unittest +from test import support + + +def setUpModule(): + wantobjects = support.get_resource_value('wantobjects') + if wantobjects is not None: + unittest.enterModuleContext( + support.swap_attr(tkinter, 'wantobjects', int(wantobjects))) class AbstractTkTest: @@ -10,6 +20,8 @@ def setUpClass(cls): tkinter.NoDefaultRoot() cls.root = tkinter.Tk() cls.wantobjects = cls.root.wantobjects() + if support.is_resource_enabled('wantobjects'): + assert cls.wantobjects == int(support.get_resource_value('wantobjects')) # De-maximize main window. # Some window managers can maximize new windows. cls.root.wm_state('normal') @@ -34,6 +46,20 @@ def tearDown(self): w.destroy() self.root.withdraw() + def require_mapped(self, widget, timeout=None): + """Realize *widget*, or skip the test if the window manager will + not map it (e.g. a tiling WM or a headless/contended display). + + Use this instead of a bare update() before querying realized + geometry (winfo_width(), identify(), coords(), place_info(), ...). + See gh-69134, gh-74941 and bpo-40722. + """ + if timeout is None: + timeout = support.LOOPBACK_TIMEOUT + if not wait_until_mapped(widget, timeout): + self.skipTest('widget was not mapped by the window manager ' + f'(timed out after {timeout:g}s)') + class AbstractDefaultRootTest: @@ -58,7 +84,7 @@ def _test_widget(self, constructor): destroy_default_root() tkinter.NoDefaultRoot() self.assertRaises(RuntimeError, constructor) - self.assertFalse(hasattr(tkinter, '_default_root')) + self.assertNotHasAttr(tkinter, '_default_root') def destroy_default_root(): @@ -67,6 +93,41 @@ def destroy_default_root(): tkinter._default_root.destroy() tkinter._default_root = None +def wait_until_mapped(widget, timeout=None, *, full_size=False): + """Wait until *widget* is actually mapped and laid out by the window + manager, so that realized-geometry queries (winfo_width(), identify(), + coords(), ...) return meaningful values. + + Return True once the widget is mapped with a non-trivial size, or False + if that has not happened within *timeout* seconds (default: + ``support.LOOPBACK_TIMEOUT``). Unlike Misc.wait_visibility(), this + never blocks indefinitely, so it is safe under a window manager that + never maps the window (see gh-69134, gh-74941, bpo-40722). + + If *full_size* is true, also wait until the realized size reaches the + requested size, so that per-pixel queries near an edge (e.g. identify()) + are reliable even under load. + """ + if timeout is None: + timeout = support.LOOPBACK_TIMEOUT + deadline = time.monotonic() + timeout + widget.update_idletasks() + while True: + widget.update() # drain pending Map/Configure events + if widget.winfo_ismapped(): + if full_size: + w_ok = widget.winfo_width() >= widget.winfo_reqwidth() > 1 + h_ok = widget.winfo_height() >= widget.winfo_reqheight() > 1 + else: + w_ok = widget.winfo_width() > 1 + h_ok = widget.winfo_height() > 1 + if w_ok and h_ok: + return True + if time.monotonic() >= deadline: + return False + time.sleep(0.01) + + def simulate_mouse_click(widget, x, y): """Generate proper events to click at the x, y position (tries to act like an X server).""" diff --git a/Lib/test/test_tkinter/test_colorchooser.py b/Lib/test/test_tkinter/test_colorchooser.py index 9bba21392d8d145..8a7e97f207a41fa 100644 --- a/Lib/test/test_tkinter/test_colorchooser.py +++ b/Lib/test/test_tkinter/test_colorchooser.py @@ -1,6 +1,7 @@ import unittest import tkinter from test.support import requires, swap_attr +from test.test_tkinter.support import setUpModule # noqa: F401 from test.test_tkinter.support import AbstractDefaultRootTest, AbstractTkTest from tkinter import colorchooser from tkinter.colorchooser import askcolor diff --git a/Lib/test/test_tkinter/test_dnd.py b/Lib/test/test_tkinter/test_dnd.py new file mode 100644 index 000000000000000..fe3685aefe48be9 --- /dev/null +++ b/Lib/test/test_tkinter/test_dnd.py @@ -0,0 +1,171 @@ +import unittest +import tkinter +from tkinter import dnd +from test.support import requires +from test.test_tkinter.support import setUpModule # noqa: F401 +from test.test_tkinter.support import AbstractTkTest + +requires('gui') + + +class Target: + def __init__(self, widget, log): + self.widget = widget + self.log = log + widget.dnd_accept = self.dnd_accept + + def dnd_accept(self, source, event): + self.log.append('accept') + return self + + def dnd_enter(self, source, event): + self.log.append('enter') + + def dnd_motion(self, source, event): + self.log.append('motion') + + def dnd_leave(self, source, event): + self.log.append('leave') + + def dnd_commit(self, source, event): + self.log.append('commit') + + +class Source: + def __init__(self, log): + self.log = log + + def dnd_end(self, target, event): + self.log.append('end') + + +class FakeEvent: + def __init__(self, widget, num=1): + self.num = num + self.widget = widget + self.x = self.y = self.x_root = self.y_root = 0 + + +class DndTest(AbstractTkTest, unittest.TestCase): + + def setUp(self): + super().setUp() + self.canvas = tkinter.Canvas(self.root) + self.canvas.pack() + # on_motion() locates the target with winfo_containing(). Bypass that + # real screen lookup, which depends on the window being visible and + # unobscured, so the test does not hinge on the window manager. + self.canvas.winfo_containing = lambda x, y: self.canvas + self.log = [] + self.source = Source(self.log) + self.target = Target(self.canvas, self.log) + + def tearDown(self): + # Make sure no drag-and-drop is left active between tests: the + # recursion guard is a name-mangled attribute on the root. + try: + del self.root._DndHandler__dnd + except AttributeError: + pass + super().tearDown() + + def test_drag_and_drop(self): + handler = dnd.dnd_start(self.source, FakeEvent(self.canvas)) + self.assertIsNotNone(handler) + handler.on_motion(FakeEvent(self.canvas)) # Enter the target. + handler.on_motion(FakeEvent(self.canvas)) # Move within the target. + handler.on_release(FakeEvent(self.canvas)) # Drop on the target. + self.assertEqual(self.log, + ['accept', 'enter', 'accept', 'motion', 'commit', 'end']) + + def test_cancel(self): + handler = dnd.dnd_start(self.source, FakeEvent(self.canvas)) + handler.on_motion(FakeEvent(self.canvas)) # Enter the target. + handler.cancel() # Leaves the target without committing. + self.assertEqual(self.log, ['accept', 'enter', 'leave', 'end']) + + def test_no_target(self): + # Nothing under the pointer accepts the drag. + self.canvas.winfo_containing = lambda x, y: None + handler = dnd.dnd_start(self.source, FakeEvent(self.canvas)) + handler.on_motion(FakeEvent(self.canvas)) + handler.on_release(FakeEvent(self.canvas)) + self.assertEqual(self.log, ['end']) + + def test_no_recursive_start(self): + handler = dnd.dnd_start(self.source, FakeEvent(self.canvas)) + self.assertIsNotNone(handler) + # A drag is already in progress, so a second start is ignored. + self.assertIsNone(dnd.dnd_start(self.source, FakeEvent(self.canvas))) + handler.cancel() + + def test_high_button_number_ignored(self): + self.assertIsNone(dnd.dnd_start(self.source, FakeEvent(self.canvas, num=6))) + + def test_restart_after_finish(self): + handler = dnd.dnd_start(self.source, FakeEvent(self.canvas)) + handler.cancel() + # Once a drag has finished a new one can start. + handler = dnd.dnd_start(self.source, FakeEvent(self.canvas)) + self.assertIsNotNone(handler) + handler.cancel() + + def test_drag_cursor(self): + self.canvas['cursor'] = 'watch' + handler = dnd.dnd_start(self.source, FakeEvent(self.canvas)) + # The drag cursor is shown while dragging, the original restored after. + self.assertEqual(handler.save_cursor, 'watch') + self.assertEqual(str(self.canvas['cursor']), 'hand2') + handler.cancel() + self.assertEqual(str(self.canvas['cursor']), 'watch') + + def test_bindings_added_and_removed(self): + handler = dnd.dnd_start(self.source, FakeEvent(self.canvas)) + self.assertIn('<Motion>', self.canvas.bind()) + self.assertIn('<B1-ButtonRelease-1>', self.canvas.bind()) + handler.cancel() + self.assertNotIn('<Motion>', self.canvas.bind()) + self.assertNotIn('<B1-ButtonRelease-1>', self.canvas.bind()) + + def test_switch_target(self): + log1, log2 = [], [] + w1, w2 = tkinter.Frame(self.root), tkinter.Frame(self.root) + target1, target2 = Target(w1, log1), Target(w2, log2) + handler = dnd.dnd_start(self.source, FakeEvent(self.canvas)) + self.canvas.winfo_containing = lambda x, y: w1 + handler.on_motion(FakeEvent(self.canvas)) # Enter target1. + self.canvas.winfo_containing = lambda x, y: w2 + handler.on_motion(FakeEvent(self.canvas)) # Leave target1, enter target2. + self.assertIs(handler.target, target2) + self.assertEqual(log1, ['accept', 'enter', 'leave']) + self.assertEqual(log2, ['accept', 'enter']) + handler.cancel() + + def test_target_in_ancestor(self): + # The widget under the pointer has no dnd_accept, but an ancestor + # does: the search walks up the master chain to find it. + parent = tkinter.Frame(self.root) + target = Target(parent, self.log) + child = tkinter.Frame(parent) + self.canvas.winfo_containing = lambda x, y: child + handler = dnd.dnd_start(self.source, FakeEvent(self.canvas)) + handler.on_motion(FakeEvent(self.canvas)) + self.assertIs(handler.target, target) + self.assertEqual(self.log, ['accept', 'enter']) + handler.cancel() + + def test_accept_returning_none_continues(self): + # dnd_accept() returning None means "not me, keep looking up". + parent = tkinter.Frame(self.root) + target = Target(parent, self.log) + child = tkinter.Frame(parent) + child.dnd_accept = lambda source, event: None + self.canvas.winfo_containing = lambda x, y: child + handler = dnd.dnd_start(self.source, FakeEvent(self.canvas)) + handler.on_motion(FakeEvent(self.canvas)) + self.assertIs(handler.target, target) + handler.cancel() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_tkinter/test_filedialog.py b/Lib/test/test_tkinter/test_filedialog.py new file mode 100644 index 000000000000000..054e719a0f883d9 --- /dev/null +++ b/Lib/test/test_tkinter/test_filedialog.py @@ -0,0 +1,73 @@ +import os +import unittest +from tkinter import filedialog +from tkinter.commondialog import Dialog +from test.support import requires, swap_attr +from test.test_tkinter.support import setUpModule # noqa: F401 +from test.test_tkinter.support import AbstractTkTest + +requires('gui') + + +class NativeDialogTest(AbstractTkTest, unittest.TestCase): + # Open, SaveAs and Directory wrap modal native dialogs. The _test_callback + # seam is called by show() just before the dialog would open; replacing it + # with a function that raises exercises show() without blocking on the + # dialog. + + def check(self, dialog_class, command): + self.assertEqual(dialog_class.command, command) + master = None + def test_callback(dialog, parent): + nonlocal master + master = parent + raise ZeroDivisionError + with swap_attr(Dialog, '_test_callback', test_callback): + d = dialog_class(self.root, title='Test') + self.assertRaises(ZeroDivisionError, d.show) + self.assertIs(master, self.root) + + def test_open(self): + self.check(filedialog.Open, 'tk_getOpenFile') + + def test_saveas(self): + self.check(filedialog.SaveAs, 'tk_getSaveFile') + + def test_directory(self): + self.check(filedialog.Directory, 'tk_chooseDirectory') + + +class FileDialogTest(AbstractTkTest, unittest.TestCase): + # The pure-Python FileDialog runs its own modal loop in go(); its logic is + # exercised here without entering the loop. + + def test_selection(self): + d = filedialog.FileDialog(self.root) + d.directory = os.path.abspath(os.sep) + d.set_selection('spam.txt') + self.assertEqual(os.path.basename(d.get_selection()), 'spam.txt') + + def test_filter(self): + d = filedialog.FileDialog(self.root) + d.set_filter(os.getcwd(), '*.py') + self.assertEqual(d.get_filter(), (os.getcwd(), '*.py')) + + def test_ok_cancel(self): + d = filedialog.FileDialog(self.root) + d.directory = os.getcwd() + d.set_selection('spam.txt') + d.ok_command() # Accepts the current selection. + self.assertEqual(os.path.basename(d.how), 'spam.txt') + d.cancel_command() # Returns no selection. + self.assertIsNone(d.how) + + def test_subclasses(self): + for cls in filedialog.LoadFileDialog, filedialog.SaveFileDialog: + with self.subTest(cls=cls.__name__): + d = cls(self.root) + self.assertIsInstance(d, filedialog.FileDialog) + self.assertEqual(d.top.title(), cls.title) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_tkinter/test_font.py b/Lib/test/test_tkinter/test_font.py index 563707ddd2fa9b9..00150ce83d07c83 100644 --- a/Lib/test/test_tkinter/test_font.py +++ b/Lib/test/test_tkinter/test_font.py @@ -2,6 +2,7 @@ import tkinter from tkinter import font from test.support import requires, gc_collect, ALWAYS_EQ +from test.test_tkinter.support import setUpModule # noqa: F401 from test.test_tkinter.support import AbstractTkTest, AbstractDefaultRootTest requires('gui') @@ -19,6 +20,7 @@ def setUpClass(cls): cls.font = font.Font(root=cls.root, name=fontname, exists=False) def test_configure(self): + self.assertEqual(self.font.config, self.font.configure) options = self.font.configure() self.assertGreaterEqual(set(options), {'family', 'size', 'weight', 'slant', 'underline', 'overstrike'}) @@ -34,6 +36,58 @@ def test_configure(self): self.assertIsInstance(options[key], sizetype) self.assertIsInstance(self.font.cget(key), sizetype) self.assertIsInstance(self.font[key], sizetype) + self.assertRaisesRegex(tkinter.TclError, 'bad option "-spam"', + self.font.cget, 'spam') + self.assertRaisesRegex(tkinter.TclError, 'bad option "-spam"', + self.font.configure, spam='x') + self.assertRaises(TypeError, self.font.cget) + self.assertRaises(TypeError, self.font.cget, 'size', 'weight') + + def test_create_from_named_font(self): + # gh-143990: a font created from a named font copies its configured + # options, preserving a size specified in pixels (a negative size). + sizetype = int if self.wantobjects else str + named = font.Font(root=self.root, name='my named font', # name with spaces + family='Times', size=-20, weight='bold') + # The source is the name of a named font or a Font representing one. + for source in ['my named font', named]: + with self.subTest(source=source): + f = font.Font(root=self.root, font=source) + self.assertEqual(f.cget('size'), sizetype(-20)) + self.assertEqual(f.actual('family'), named.actual('family')) + self.assertEqual(f.actual('weight'), 'bold') + + def test_create_from_description(self): + # gh-143990: a font created from a font description is resolved via + # "font actual", so a size in pixels (negative) becomes a size in points. + descriptions = [ + ('Times', -20), # tuple + ('Times', -20, 'bold'), # tuple with a style + 'Times -20', # string + 'Times -20 bold', # string with a style + '{Times New Roman} -20', # string, family with spaces + ] + for desc in descriptions: + with self.subTest(font=desc): + f = font.Font(root=self.root, font=desc) + self.assertGreater(int(f.cget('size')), 0) # pixels -> points + + def test_copy(self): + # size=-20 (pixels): copy() copies the configured options, so the + # size is preserved rather than resolved (gh-143990). + f = font.Font(root=self.root, family='Times', size=-20, weight='bold') + copied = f.copy() + self.assertIsInstance(copied, font.Font) + self.assertIsNot(copied, f) + self.assertNotEqual(copied.name, f.name) + self.assertEqual(copied.actual(), f.actual()) + sizetype = int if self.wantobjects else str + self.assertEqual(copied.cget('size'), sizetype(-20)) + # The copy is independent of the original. + copied.configure(size=20) + self.assertEqual(f.cget('size'), sizetype(-20)) + self.assertEqual(copied.cget('size'), sizetype(20)) + self.assertRaises(TypeError, f.copy, 'x') def test_unicode_family(self): family = 'MS \u30b4\u30b7\u30c3\u30af' @@ -58,6 +112,9 @@ def test_actual(self): for key in 'size', 'underline', 'overstrike': self.assertIsInstance(options[key], sizetype) self.assertIsInstance(self.font.actual(key), sizetype) + self.assertRaisesRegex(tkinter.TclError, 'bad option "-spam"', + self.font.actual, 'spam') + self.assertRaises(TypeError, self.font.actual, 'size', 'weight', 'slant') def test_name(self): self.assertEqual(self.font.name, fontname) @@ -81,6 +138,11 @@ def test_equality(self): def test_measure(self): self.assertIsInstance(self.font.measure('abc'), int) + self.assertEqual(self.font.measure(''), 0) + self.assertIsInstance( + self.font.measure('abc', displayof=self.root), int) + self.assertRaises(TypeError, self.font.measure) + self.assertRaises(TypeError, self.font.measure, 'a', 'b', 'c') def test_metrics(self): metrics = self.font.metrics() @@ -88,8 +150,12 @@ def test_metrics(self): {'ascent', 'descent', 'linespace', 'fixed'}) for key in metrics: self.assertEqual(self.font.metrics(key), metrics[key]) + self.assertEqual(self.font.metrics(key, displayof=self.root), + metrics[key]) self.assertIsInstance(metrics[key], int) self.assertIsInstance(self.font.metrics(key), int) + self.assertRaisesRegex(tkinter.TclError, 'bad metric "-spam"', + self.font.metrics, 'spam') def test_families(self): families = font.families(self.root) diff --git a/Lib/test/test_tkinter/test_geometry_managers.py b/Lib/test/test_tkinter/test_geometry_managers.py index d71a634a7673105..5c38c7044e5cfb1 100644 --- a/Lib/test/test_tkinter/test_geometry_managers.py +++ b/Lib/test/test_tkinter/test_geometry_managers.py @@ -4,6 +4,7 @@ from tkinter import TclError from test.support import requires +from test.test_tkinter.support import setUpModule # noqa: F401 from test.test_tkinter.support import pixels_conv from test.test_tkinter.widget_tests import AbstractWidgetTest @@ -18,6 +19,7 @@ class PackTest(AbstractWidgetTest, unittest.TestCase): test_keys = None + test_options_in_docstring = None def create2(self): pack = tkinter.Toplevel(self.root, name='pack') @@ -280,10 +282,28 @@ def test_pack_slaves(self): b.pack_configure() self.assertEqual(pack.pack_slaves(), [a, b]) + def test_pack_short_aliases(self): + # slaves, content and propagate are aliases of the pack_* methods + # (Misc precedes Pack, Place and Grid in the method resolution order). + pack, a, b, c, d = self.create2() + self.assertEqual(pack.slaves, pack.pack_slaves) + self.assertEqual(pack.propagate, pack.pack_propagate) + + self.assertEqual(pack.slaves(), []) + a.pack_configure() + self.assertEqual(pack.slaves(), [a]) + + pack.configure(width=300, height=200) + pack.propagate(False) + self.root.update() + self.assertEqual(pack.winfo_reqwidth(), 300) + self.assertEqual(pack.winfo_reqheight(), 200) + class PlaceTest(AbstractWidgetTest, unittest.TestCase): test_keys = None + test_options_in_docstring = None def create2(self): t = tkinter.Toplevel(self.root, width=300, height=200, bd=0) @@ -485,10 +505,22 @@ def test_place_slaves(self): with self.assertRaises(TypeError): foo.place_slaves(0) + def test_place_method_aliases(self): + # The Place manager defines configure, info, forget, slaves and + # content as aliases of its place_* methods. On a real widget the + # short names are provided by Misc and Pack (earlier in the method + # resolution order), so the aliases are checked on the class itself. + self.assertIs(tkinter.Place.configure, tkinter.Place.place_configure) + self.assertIs(tkinter.Place.config, tkinter.Place.place_configure) + self.assertIs(tkinter.Place.info, tkinter.Place.place_info) + self.assertIs(tkinter.Place.forget, tkinter.Place.place_forget) + self.assertIs(tkinter.Place.slaves, tkinter.Misc.place_slaves) + class GridTest(AbstractWidgetTest, unittest.TestCase): test_keys = None + test_options_in_docstring = None def tearDown(self): cols, rows = self.root.grid_size() @@ -903,6 +935,30 @@ def test_grid_slaves(self): self.assertEqual(self.root.grid_slaves(column=1), [d, c, a]) self.assertEqual(self.root.grid_slaves(row=1, column=1), [d, c]) + def test_grid_short_aliases(self): + # columnconfigure, rowconfigure, size, anchor and bbox are aliases of + # the corresponding grid_* methods (Misc precedes Pack, Place and Grid + # in the method resolution order). + root = self.root + self.assertEqual(root.columnconfigure, root.grid_columnconfigure) + self.assertEqual(root.rowconfigure, root.grid_rowconfigure) + self.assertEqual(root.size, root.grid_size) + self.assertEqual(root.anchor, root.grid_anchor) + self.assertEqual(root.bbox, root.grid_bbox) + + self.assertEqual(root.size(), (0, 0)) + b = tkinter.Button(root) + b.grid_configure(column=2, row=3) + self.assertEqual(root.size(), (3, 4)) + + root.columnconfigure(0, weight=2) + self.assertEqual(root.grid_columnconfigure(0, 'weight'), 2) + root.rowconfigure(0, weight=3) + self.assertEqual(root.grid_rowconfigure(0, 'weight'), 3) + + root.anchor('se') + self.assertEqual(root.tk.call('grid', 'anchor', root), 'se') + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_tkinter/test_images.py b/Lib/test/test_tkinter/test_images.py index 38371fe00d6eb5b..eb5794d391e7589 100644 --- a/Lib/test/test_tkinter/test_images.py +++ b/Lib/test/test_tkinter/test_images.py @@ -2,6 +2,7 @@ import tkinter from test import support from test.support import os_helper +from test.test_tkinter.support import setUpModule # noqa: F401 from test.test_tkinter.support import AbstractTkTest, AbstractDefaultRootTest, requires_tk support.requires('gui') @@ -271,6 +272,11 @@ def test_configure_width_height(self): image.configure(height=10) self.assertEqual(image['width'], '20') self.assertEqual(image['height'], '10') + self.assertEqual(image.cget('width'), image['width']) + self.assertEqual(image.cget('height'), image['height']) + self.assertRaises(TypeError, image.cget) + self.assertRaises(TypeError, image.cget, 'width', 'height') + self.assertEqual(image.config, image.configure) self.assertEqual(image.width(), 20) self.assertEqual(image.height(), 10) @@ -641,12 +647,20 @@ def test_data(self): def test_transparency(self): image = self.create() - self.assertEqual(image.transparency_get(0, 0), True) - self.assertEqual(image.transparency_get(4, 6), False) + self.assertIs(image.transparency_get(0, 0), True) + self.assertIs(image.transparency_get(4, 6), False) image.transparency_set(4, 6, True) - self.assertEqual(image.transparency_get(4, 6), True) + self.assertIs(image.transparency_get(4, 6), True) image.transparency_set(4, 6, False) - self.assertEqual(image.transparency_get(4, 6), False) + self.assertIs(image.transparency_get(4, 6), False) + self.assertRaises(tkinter.TclError, image.transparency_get, -1, 0) + self.assertRaises(tkinter.TclError, image.transparency_get, 16, 0) + self.assertRaises(tkinter.TclError, image.transparency_set, -1, 0, True) + self.assertRaises(tkinter.TclError, image.transparency_set, 16, 0, True) + self.assertRaises(TypeError, image.transparency_get, 0) + self.assertRaises(TypeError, image.transparency_get, 0, 0, 0) + self.assertRaises(TypeError, image.transparency_set, 0, 0) + self.assertRaises(TypeError, image.transparency_set, 0, 0, True, 0) if __name__ == "__main__": diff --git a/Lib/test/test_tkinter/test_loadtk.py b/Lib/test/test_tkinter/test_loadtk.py index 61b0eda2fc750ae..7cff654621eb350 100644 --- a/Lib/test/test_tkinter/test_loadtk.py +++ b/Lib/test/test_tkinter/test_loadtk.py @@ -3,6 +3,7 @@ import unittest import test.support as test_support from test.support import os_helper +from test.test_tkinter.support import setUpModule # noqa: F401 from tkinter import Tcl, TclError test_support.requires('gui') diff --git a/Lib/test/test_tkinter/test_messagebox.py b/Lib/test/test_tkinter/test_messagebox.py index f41bdc98286283a..f29f0c7ba590861 100644 --- a/Lib/test/test_tkinter/test_messagebox.py +++ b/Lib/test/test_tkinter/test_messagebox.py @@ -1,6 +1,7 @@ import unittest import tkinter from test.support import requires, swap_attr +from test.test_tkinter.support import setUpModule # noqa: F401 from test.test_tkinter.support import AbstractDefaultRootTest from tkinter.commondialog import Dialog from tkinter.messagebox import showinfo diff --git a/Lib/test/test_tkinter/test_misc.py b/Lib/test/test_tkinter/test_misc.py index 96ea3f0117ca036..ee13cfd13a703b7 100644 --- a/Lib/test/test_tkinter/test_misc.py +++ b/Lib/test/test_tkinter/test_misc.py @@ -1,11 +1,19 @@ import functools +import platform +import sys +import textwrap import unittest +import weakref import tkinter from tkinter import TclError import enum from test import support +from test.support import os_helper +from test.support.script_helper import assert_python_ok +from test.test_tkinter.support import setUpModule # noqa: F401 from test.test_tkinter.support import (AbstractTkTest, AbstractDefaultRootTest, - requires_tk, get_tk_patchlevel) + requires_tk, get_tk_patchlevel, + tcl_version) support.requires('gui') @@ -46,6 +54,33 @@ class Button2(tkinter.Button): b4 = Button2(f2) self.assertEqual(len({str(b), str(b2), str(b3), str(b4)}), 4) + def test_dealloc_in_wrong_thread(self): + # gh-83274: deallocating the interpreter in the wrong thread must not + # crash. + script = textwrap.dedent(""" + import threading + import tkinter + root = tkinter.Tk() + root.destroy() + # Let another thread drop the last reference. + ready = threading.Event() + t = threading.Thread(target=lambda obj: ready.wait(), args=(root,)) + t.start() + del root + ready.set() + t.join() + print('ok') + """) + rc, out, err = assert_python_ok('-c', script) + self.assertEqual(out.strip(), b'ok') + if not support.Py_GIL_DISABLED: + # On the free-threaded build the interpreter may instead be + # deallocated in its own thread (deferred reference counting), so + # the warning is not necessarily emitted. The crucial guarantee -- + # no crash -- is already checked by assert_python_ok() above. + self.assertIn(b'RuntimeWarning', err) + self.assertIn(b'gh-83274', err) + @requires_tk(8, 6, 6) def test_tk_busy(self): root = self.root @@ -317,44 +352,241 @@ def test_clipboard_astral(self): with self.assertRaises(tkinter.TclError): root.clipboard_get() - def test_winfo_rgb(self): + def test_getint(self): + self.assertEqual(self.root.getint('42'), 42) + self.assertEqual(self.root.getint(42), 42) + self.assertEqual(self.root.getint('-5'), -5) + self.assertRaises(ValueError, self.root.getint, 'spam') - def assertApprox(col1, col2): - # A small amount of flexibility is required (bpo-45496) - # 33 is ~0.05% of 65535, which is a reasonable margin - for col1_channel, col2_channel in zip(col1, col2): - self.assertAlmostEqual(col1_channel, col2_channel, delta=33) + def test_getdouble(self): + self.assertEqual(self.root.getdouble('3.5'), 3.5) + self.assertEqual(self.root.getdouble(3), 3.0) + self.assertRaises(ValueError, self.root.getdouble, 'spam') - root = self.root - rgb = root.winfo_rgb + def test_getvar(self): + self.root.setvar('test_var', 'hello') + self.assertEqual(self.root.getvar('test_var'), 'hello') - # Color name. - self.assertEqual(rgb('red'), (65535, 0, 0)) - self.assertEqual(rgb('dark slate blue'), (18504, 15677, 35723)) - # #RGB - extends each 4-bit hex value to be 16-bit. - self.assertEqual(rgb('#F0F'), (0xFFFF, 0x0000, 0xFFFF)) - # #RRGGBB - extends each 8-bit hex value to be 16-bit. - assertApprox(rgb('#4a3c8c'), (0x4a4a, 0x3c3c, 0x8c8c)) - # #RRRRGGGGBBBB - assertApprox(rgb('#dede14143939'), (0xdede, 0x1414, 0x3939)) - # Invalid string. - with self.assertRaises(tkinter.TclError): - rgb('#123456789a') - # RGB triplet is invalid input. - with self.assertRaises(tkinter.TclError): - rgb((111, 78, 55)) + def test_register(self): + result = [] + def callback(): + result.append(1) + return 'spam' + name = self.root.register(callback) + self.assertIsInstance(name, str) + self.assertEqual(self.root.tk.call(name), 'spam') + self.assertEqual(result, [1]) + self.root.deletecommand(name) + self.assertRaises(TclError, self.root.tk.call, name) + + def test_createcommand_no_leak(self): + # gh-80937: dropping the interpreter must release a command's callback, + # even without an explicit deletecommand(). + interp = tkinter.Tcl() + callback = lambda: '' + ref = weakref.ref(callback) + interp.tk.createcommand('cb', callback) + del callback, interp + support.gc_collect() + self.assertIsNone(ref()) + + def test_option(self): + self.addCleanup(self.root.option_clear) + self.root.option_add('*Button.background', 'red') + b = tkinter.Button(self.root) + self.assertEqual(b.option_get('background', 'Background'), 'red') + self.assertEqual(b.option_get('foreground', 'Foreground'), '') + self.root.option_clear() + self.assertEqual(b.option_get('background', 'Background'), '') + + def test_option_readfile(self): + self.addCleanup(self.root.option_clear) + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with open(os_helper.TESTFN, 'w') as f: + f.write('*Button.background: red\n') + self.root.option_readfile(os_helper.TESTFN) + b = tkinter.Button(self.root) + self.assertEqual(b.option_get('background', 'Background'), 'red') + self.assertRaises(TclError, self.root.option_readfile, + os_helper.TESTFN + '.nonexistent') + self.assertRaises(TypeError, self.root.option_readfile) + self.assertRaises(TypeError, self.root.option_readfile, 'a', 'b', 'c') + + def test_nametowidget(self): + b = tkinter.Button(self.root, name='btn') + self.assertIs(self.root.nametowidget('btn'), b) + self.assertIs(self.root.nametowidget(str(b)), b) + self.assertRaises(KeyError, self.root.nametowidget, '.nonexistent') + + def test_nametowidget_menu_clone(self): + # A menu used as a menubar or cascade is cloned by Tk under an + # auto-generated name (each path component is the original name + # prefixed with one or more '#' clone markers). nametowidget() + # maps such a name back to the original widget (gh-38464). + menubar = tkinter.Menu(self.root) + filemenu = tkinter.Menu(menubar, tearoff=0) + menubar.add_cascade(label='File', menu=filemenu) + submenu = tkinter.Menu(filemenu, tearoff=0) + filemenu.add_cascade(label='More', menu=submenu) + self.root['menu'] = menubar + self.root.update_idletasks() - def test_winfo_pathname(self): - t = tkinter.Toplevel(self.root) - w = tkinter.Button(t) - wid = w.winfo_id() - self.assertIsInstance(wid, int) - self.assertEqual(self.root.winfo_pathname(hex(wid)), str(w)) - self.assertEqual(self.root.winfo_pathname(hex(wid), displayof=None), str(w)) - self.assertEqual(self.root.winfo_pathname(hex(wid), displayof=t), str(w)) - self.assertEqual(self.root.winfo_pathname(wid), str(w)) - self.assertEqual(self.root.winfo_pathname(wid, displayof=None), str(w)) - self.assertEqual(self.root.winfo_pathname(wid, displayof=t), str(w)) + originals = {menubar, filemenu, submenu} + clones = [] + def collect(parent): + for name in self.root.tk.splitlist( + self.root.tk.call('winfo', 'children', parent)): + clones.append(name) + collect(name) + collect('.') + # Every menu (originals and clones) resolves to an original widget. + self.assertTrue(any('#' in name for name in clones)) + for name in clones: + self.assertIn(self.root.nametowidget(name), originals) + + def test_focus_methods(self): + f = tkinter.Frame(self.root, width=150, height=100) + f.pack() + self.root.wait_visibility() # needed on Windows + self.root.update_idletasks() + f.focus_force() + self.root.update() + self.assertIs(self.root.focus_get(), f) + self.assertIs(self.root.focus_displayof(), f) + self.assertIs(f.focus_lastfor(), f) + b = tkinter.Button(f) + b.pack() + self.root.update() + b.focus_set() + self.root.update() + self.assertIs(self.root.focus_get(), b) + + def test_focus_methods_unresolvable(self): + # The focus may be on a widget that tkinter did not create and so + # cannot map to an instance (e.g. a torn-off menu). The focus + # methods return None instead of raising KeyError (gh-88758). + menu = tkinter.Menu(self.root, tearoff=1) + menu.add_command(label='Hello') + tearoff = self.root.tk.call('tk::TearOffMenu', str(menu), 0, 0) + self.addCleanup(self.root.tk.call, 'destroy', tearoff) + self.root.update() + self.assertRaises(KeyError, self.root.nametowidget, tearoff) + + self.root.tk.call('focus', '-force', tearoff) + self.root.update() + self.assertIsNone(self.root.focus_get()) + self.assertIsNone(self.root.focus_displayof()) + + def test_grab(self): + f = tkinter.Frame(self.root) + f.pack() + self.root.wait_visibility() + self.assertIsNone(self.root.grab_current()) + self.assertIsNone(f.grab_status()) + f.grab_set() + self.assertEqual(f.grab_status(), 'local') + self.assertIs(self.root.grab_current(), f) + f.grab_release() + self.assertIsNone(f.grab_status()) + self.assertIsNone(self.root.grab_current()) + + def test_selection_own(self): + self.root.selection_own() + self.assertIs(self.root.selection_own_get(), self.root) + f = tkinter.Frame(self.root) + f.selection_own() + self.assertIs(self.root.selection_own_get(), f) + + def test_event_add_delete_info(self): + self.addCleanup(self.root.event_delete, '<<TestEvent>>') + self.root.event_add('<<TestEvent>>', '<Control-z>', '<Control-y>') + self.assertEqual(self.root.event_info('<<TestEvent>>'), + ('<Control-Key-z>', '<Control-Key-y>')) + self.assertIn('<<TestEvent>>', self.root.event_info()) + self.root.event_delete('<<TestEvent>>', '<Control-y>') + self.assertEqual(self.root.event_info('<<TestEvent>>'), + ('<Control-Key-z>',)) + self.root.event_delete('<<TestEvent>>') + self.assertEqual(self.root.event_info('<<TestEvent>>'), ()) + self.assertRaises(TypeError, self.root.event_delete) + + def test_bell(self): + self.root.bell() # No exception. + self.root.bell(displayof=self.root) + + def test_tk_focusNext_focusPrev(self): + f = tkinter.Frame(self.root) + f.pack() + entries = [tkinter.Entry(f) for _ in range(3)] + for entry in entries: + entry.pack() + # tk_focusNext skips widgets that are not viewable. + entries[-1].wait_visibility() + self.assertIs(entries[0].tk_focusNext(), entries[1]) + self.assertIs(entries[1].tk_focusNext(), entries[2]) + self.assertIs(entries[2].tk_focusPrev(), entries[1]) + self.assertIs(entries[1].tk_focusPrev(), entries[0]) + self.assertRaises(TypeError, entries[0].tk_focusNext, 'x') + self.assertRaises(TypeError, entries[0].tk_focusPrev, 'x') + + def test_tk_strictMotif(self): + self.addCleanup(self.root.tk_strictMotif, False) + self.assertIs(self.root.tk_strictMotif(), False) + self.assertIs(self.root.tk_strictMotif(True), True) + self.assertIs(self.root.tk_strictMotif(), True) + self.assertIs(self.root.tk_strictMotif(False), False) + self.assertRaises(TypeError, self.root.tk_strictMotif, 1, 2) + + def test_tk_bisque(self): + # tk_bisque resets the color palette; use a separate root so that + # the shared one is not affected. + root = tkinter.Tk() + self.addCleanup(root.destroy) + root.tk_bisque() + self.assertEqual(root['background'], '#ffe4c4') + self.assertRaises(TypeError, root.tk_bisque, 'x') + + def test_wait_variable(self): + var = tkinter.StringVar(self.root) + self.assertEqual(self.root.waitvar, self.root.wait_variable) + self.root.after(1, var.set, 'done') + self.root.wait_variable(var) # Returns once the variable is set. + self.assertEqual(var.get(), 'done') + + def test_wait_window(self): + top = tkinter.Toplevel(self.root) + self.root.after(1, top.destroy) + self.root.wait_window(top) # Returns once the window is destroyed. + self.assertFalse(top.winfo_exists()) + + def test_tk_focusFollowsMouse(self): + self.root.tk_focusFollowsMouse() # No exception. + + def test_selection_handle(self): + f = tkinter.Frame(self.root) + def handler(offset, length): + return 'PAYLOAD'[int(offset):int(offset) + int(length)] + f.selection_handle(handler) + f.selection_own() + self.assertEqual(f.selection_get(), 'PAYLOAD') + + def test_grab_set_global(self): + # A successful global grab directs all events on the display to this + # application, so only the error paths are tested here. + self.assertRaises(TypeError, self.root.grab_set_global, 'extra') + with self.subTest('non-viewable window'): + if self.root._windowingsystem != 'x11': + # Grabbing a non-viewable window fails only on X11; elsewhere + # it would actually grab the whole display. + self.skipTest('only X11 fails the grab') + f = tkinter.Frame(self.root) # not yet viewable + self.assertRaisesRegex(TclError, 'grab failed', f.grab_set_global) + + def test_send(self): + if self.root._windowingsystem != 'x11': + self.skipTest('send is only supported on X11') + self.assertRaisesRegex(TclError, 'no application named', + self.root.send, 'no_such_interp_xyzzy', 'set x 1') def test_event_repr_defaults(self): e = tkinter.Event() @@ -497,7 +729,7 @@ def test_info_patchlevel(self): self.assertEqual(vi.serial, 0) else: self.assertEqual(vi.micro, 0) - self.assertTrue(str(vi).startswith(f'{vi.major}.{vi.minor}')) + self.assertStartsWith(str(vi), f'{vi.major}.{vi.minor}') def test_embedded_null(self): widget = tkinter.Entry(self.root) @@ -509,6 +741,174 @@ def test_embedded_null(self): self.assertEqual(widget.selection_get(), '\u20ac\0abc\x00def') +class TkTest(AbstractTkTest, unittest.TestCase): + + def test_className(self): + # The className argument sets the class of the root window. Tk + # title-cases it: the first letter is upper-cased, the rest lower-cased. + cases = [ + ('fooBAR', 'Foobar'), + ('éÉ', 'Éé'), # small and capital E WITH ACUTE + ] + if tcl_version >= (9, 0): + # small and capital DESERET LETTER LONG I (a non-BMP script) + cases.append(('\U00010428\U00010400', '\U00010400\U00010428')) + for className, klass in cases: + root = tkinter.Tk(className=className) + try: + self.assertEqual(root.winfo_class(), klass) + finally: + root.destroy() + if tcl_version < (9, 0): + # gh-126219: title-casing a non-BMP first letter crashed Tcl 8.x; + # such a class name is now rejected. + self.assertRaises(ValueError, tkinter.Tk, className='\U00010428') + + +class WinfoTest(AbstractTkTest, unittest.TestCase): + + def test_winfo_rgb(self): + + def assertApprox(col1, col2): + # A small amount of flexibility is required (bpo-45496) + # 33 is ~0.05% of 65535, which is a reasonable margin + for col1_channel, col2_channel in zip(col1, col2): + self.assertAlmostEqual(col1_channel, col2_channel, delta=33) + + root = self.root + rgb = root.winfo_rgb + + # Color name. + self.assertEqual(rgb('red'), (65535, 0, 0)) + self.assertEqual(rgb('dark slate blue'), (18504, 15677, 35723)) + # #RGB - extends each 4-bit hex value to be 16-bit. + self.assertEqual(rgb('#F0F'), (0xFFFF, 0x0000, 0xFFFF)) + # #RRGGBB - extends each 8-bit hex value to be 16-bit. + assertApprox(rgb('#4a3c8c'), (0x4a4a, 0x3c3c, 0x8c8c)) + # #RRRRGGGGBBBB + assertApprox(rgb('#dede14143939'), (0xdede, 0x1414, 0x3939)) + # Invalid string. + with self.assertRaises(tkinter.TclError): + rgb('#123456789a') + # RGB triplet is invalid input. + with self.assertRaises(tkinter.TclError): + rgb((111, 78, 55)) + + def test_winfo_pathname(self): + t = tkinter.Toplevel(self.root) + w = tkinter.Button(t) + wid = w.winfo_id() + self.assertIsInstance(wid, int) + self.assertEqual(self.root.winfo_pathname(hex(wid)), str(w)) + self.assertEqual(self.root.winfo_pathname(hex(wid), displayof=None), str(w)) + self.assertEqual(self.root.winfo_pathname(hex(wid), displayof=t), str(w)) + self.assertEqual(self.root.winfo_pathname(wid), str(w)) + self.assertEqual(self.root.winfo_pathname(wid, displayof=None), str(w)) + self.assertEqual(self.root.winfo_pathname(wid, displayof=t), str(w)) + + def test_winfo_class_name_parent(self): + f = tkinter.Frame(self.root) + b = tkinter.Button(f) + self.assertEqual(f.winfo_class(), 'Frame') + self.assertEqual(b.winfo_class(), 'Button') + self.assertEqual(b.winfo_name(), str(b).rsplit('.', 1)[-1]) + self.assertEqual(b.winfo_parent(), str(f)) + self.assertEqual(f.winfo_parent(), str(self.root)) + self.assertIs(f.winfo_toplevel(), self.root) + t = tkinter.Toplevel(self.root) + self.assertIs(tkinter.Button(t).winfo_toplevel(), t) + self.assertEqual(self.root.nametowidget(b.winfo_parent()), f) + + def test_winfo_children(self): + self.assertEqual(self.root.winfo_children(), []) + f = tkinter.Frame(self.root) + b = tkinter.Button(f) + self.assertEqual(self.root.winfo_children(), [f]) + self.assertEqual(f.winfo_children(), [b]) + + def test_winfo_visual_info(self): + f = tkinter.Frame(self.root) + self.assertIsInstance(f.winfo_depth(), int) + self.assertIsInstance(f.winfo_cells(), int) + self.assertIsInstance(f.winfo_visual(), str) + self.assertIsInstance(f.winfo_visualid(), str) + self.assertIsInstance(f.winfo_colormapfull(), bool) + visuals = self.root.winfo_visualsavailable() + self.assertIsInstance(visuals, list) + for name, depth in visuals: + self.assertIsInstance(name, str) + self.assertIsInstance(depth, int) + + def test_winfo_viewable(self): + f = tkinter.Frame(self.root) + self.assertFalse(f.winfo_viewable()) + f.pack() + f.wait_visibility() + self.root.update() + self.assertTrue(f.winfo_viewable()) + + def test_winfo_atom(self): + atom = self.root.winfo_atom('PRIMARY') + self.assertIsInstance(atom, int) + self.assertEqual(self.root.winfo_atomname(atom), 'PRIMARY') + self.assertEqual( + self.root.winfo_atomname(atom, displayof=self.root), 'PRIMARY') + self.assertEqual( + self.root.winfo_atom('PRIMARY', displayof=self.root), atom) + self.assertRaisesRegex(TclError, 'no atom exists', + self.root.winfo_atomname, 10 ** 9) + + def test_winfo_pointer(self): + self.assertIsInstance(self.root.winfo_pointerx(), int) + self.assertIsInstance(self.root.winfo_pointery(), int) + xy = self.root.winfo_pointerxy() + self.assertIsInstance(xy, tuple) + self.assertEqual(len(xy), 2) + self.assertTrue(all(isinstance(v, int) for v in xy)) + + def test_winfo_containing(self): + self.root.update() + # No window contains a point far off the screen. + self.assertIsNone(self.root.winfo_containing(-10000, -10000)) + self.assertIsNone( + self.root.winfo_containing(-10000, -10000, displayof=self.root)) + + def test_winfo_fpixels(self): + self.assertIsInstance(self.root.winfo_fpixels('1i'), float) + self.assertAlmostEqual(self.root.winfo_fpixels('1i'), + self.root.winfo_fpixels('72p')) + # Tk < 9 reports 'bad screen distance "spam"', Tk 9 reports + # 'expected screen distance ... but got "spam"'. + self.assertRaisesRegex(TclError, + r'(bad|expected) screen distance.*"spam"', + self.root.winfo_fpixels, 'spam') + + def test_winfo_screen(self): + for name in ('winfo_screenwidth', 'winfo_screenheight', + 'winfo_screenmmwidth', 'winfo_screenmmheight', + 'winfo_screencells', 'winfo_screendepth'): + value = getattr(self.root, name)() + self.assertIsInstance(value, int) + self.assertGreater(value, 0) + self.assertIsInstance(self.root.winfo_screenvisual(), str) + self.assertIsInstance(self.root.winfo_screen(), str) + self.assertIsInstance(self.root.winfo_server(), str) + + def test_winfo_vroot(self): + for name in ('winfo_vrootwidth', 'winfo_vrootheight', + 'winfo_vrootx', 'winfo_vrooty'): + self.assertIsInstance(getattr(self.root, name)(), int) + + def test_winfo_interps(self): + interps = self.root.winfo_interps() + self.assertIsInstance(interps, tuple) + # The registry of interpreters is only populated where "send" is + # supported (i.e. X11), so do not require this interpreter's name. + if self.root._windowingsystem == 'x11': + self.assertIn(self.root.tk.call('tk', 'appname'), interps) + self.assertEqual(self.root.winfo_interps(displayof=self.root), interps) + + class WmTest(AbstractTkTest, unittest.TestCase): def test_wm_attribute(self): @@ -565,12 +965,24 @@ def test_wm_attribute(self): def test_wm_iconbitmap(self): t = tkinter.Toplevel(self.root) + patchlevel = get_tk_patchlevel(t) + + if ( + t._windowingsystem == 'aqua' + and sys.platform == 'darwin' + and platform.machine() == 'x86_64' + and platform.mac_ver()[0].startswith('26.') + ): + # https://github.com/python/cpython/issues/146531 + # Tk bug 4a2070f0d3a99aa412bc582d386d575ca2f37323 + # Not fixed as of Tk 8.6.18 and 9.0.4. + self.skipTest('wm iconbitmap hangs on macOS 26 Intel') + self.assertEqual(t.wm_iconbitmap(), '') t.wm_iconbitmap('hourglass') bug = False if t._windowingsystem == 'aqua': # Tk bug 13ac26b35dc55f7c37f70b39d59d7ef3e63017c8. - patchlevel = get_tk_patchlevel(t) if patchlevel < (8, 6, 17) or (9, 0) <= patchlevel < (9, 0, 2): bug = True if not bug: @@ -588,6 +1000,156 @@ def test_wm_iconbitmap(self): t.destroy() + def test_wm_iconphoto(self): + t = tkinter.Toplevel(self.root) + img = tkinter.PhotoImage(master=t, width=16, height=16) + t.wm_iconphoto(False, img) # No exception. + t.wm_iconphoto(True, img) + self.assertRaises(tkinter.TclError, t.wm_iconphoto, False, 'spam') + + def test_wm_title(self): + t = tkinter.Toplevel(self.root) + t.title('Hello') + self.assertEqual(t.title(), 'Hello') + self.assertEqual(t.wm_title(), 'Hello') + t.wm_title('Spam') + self.assertEqual(t.title(), 'Spam') + + def test_wm_geometry(self): + t = tkinter.Toplevel(self.root) + t.geometry('200x100+10+20') + t.update() + self.assertRegex(t.geometry(), r'^200x100\+-?\d+\+-?\d+$') + self.assertEqual(t.wm_geometry(), t.geometry()) + + def test_wm_minsize_maxsize(self): + t = tkinter.Toplevel(self.root) + # Use a width above the minimum enforced by some platforms (72 on Aqua). + t.minsize(150, 100) + self.assertEqual(t.minsize(), (150, 100)) + t.maxsize(500, 600) + self.assertEqual(t.maxsize(), (500, 600)) + + def test_wm_resizable(self): + t = tkinter.Toplevel(self.root) + t.resizable(False, True) + self.assertEqual(t.resizable(), (0, 1)) + self.assertRaisesRegex(TclError, 'expected boolean value', + t.resizable, 'spam', True) + + def test_wm_aspect(self): + t = tkinter.Toplevel(self.root) + self.assertEqual(t.aspect(), None) + t.aspect(1, 2, 3, 4) + self.assertEqual(t.aspect(), (1, 2, 3, 4)) + + def test_wm_grid(self): + t = tkinter.Toplevel(self.root) + t.wm_grid(10, 10, 5, 5) + self.assertEqual(t.wm_grid(), (10, 10, 5, 5)) + + def test_wm_positionfrom_sizefrom(self): + # These set X11 size hints and may be no-ops on other platforms. + t = tkinter.Toplevel(self.root) + t.positionfrom('user') + self.assertIn(t.positionfrom(), ('user', '')) + t.sizefrom('program') + self.assertIn(t.sizefrom(), ('program', '')) + + def test_wm_focusmodel(self): + t = tkinter.Toplevel(self.root) + self.assertEqual(t.focusmodel(), 'passive') + t.focusmodel('active') + self.assertEqual(t.focusmodel(), 'active') + self.assertRaises(TclError, t.focusmodel, 'spam') + + def test_wm_iconname(self): + # WM_ICON_NAME is an X11 property and may be a no-op elsewhere. + t = tkinter.Toplevel(self.root) + t.iconname('Icon') + self.assertIn(t.iconname(), ('Icon', '')) + + def test_wm_iconposition(self): + t = tkinter.Toplevel(self.root) + t.wm_iconposition(3, 4) # An X11 hint; may be a no-op elsewhere. + if t._windowingsystem == 'x11': + self.assertEqual(t.wm_iconposition(), (3, 4)) + + def test_wm_iconmask_iconwindow(self): + if self.root._windowingsystem != 'x11': + self.skipTest('iconmask and iconwindow are X11-specific') + t = tkinter.Toplevel(self.root) + t.wm_iconmask('gray50') # No exception. + icon = tkinter.Toplevel(self.root) + t.wm_iconwindow(icon) + self.assertEqual(str(t.wm_iconwindow()), str(icon)) + + def test_wm_colormapwindows(self): + if self.root._windowingsystem != 'x11': + self.skipTest('colormapwindows is X11-specific') + t = tkinter.Toplevel(self.root) + self.assertEqual(t.wm_colormapwindows(), []) + f = tkinter.Frame(t) + t.wm_colormapwindows(f) + self.assertEqual([str(w) for w in t.wm_colormapwindows()], [str(f)]) + + def test_wm_manage_forget(self): + f = tkinter.Frame(self.root) + self.root.wm_manage(f) # Make the frame a top-level window. + self.root.wm_forget(f) # Revert it; no exception either way. + + def test_wm_client_command(self): + t = tkinter.Toplevel(self.root) + t.client('myhost') + t.wm_command('myapp -x') + # WM_CLIENT_MACHINE and WM_COMMAND are X11 properties; elsewhere the + # setters may be no-ops and wm_command may return a split list. + if t._windowingsystem == 'x11': + self.assertEqual(t.client(), 'myhost') + self.assertEqual(t.wm_command(), 'myapp -x') + + def test_wm_overrideredirect(self): + t = tkinter.Toplevel(self.root) + self.assertFalse(t.overrideredirect()) + t.overrideredirect(True) + self.assertTrue(t.overrideredirect()) + + def test_wm_state(self): + t = tkinter.Toplevel(self.root) + t.update() + self.assertEqual(t.state(), 'normal') + t.withdraw() + self.assertEqual(t.state(), 'withdrawn') + t.deiconify() + t.update() + self.assertEqual(t.state(), 'normal') + self.assertRaises(TclError, t.state, 'spam') + + def test_wm_frame(self): + t = tkinter.Toplevel(self.root) + t.update() + self.assertIsInstance(t.frame(), str) + + def test_wm_group(self): + # The window group is an X11 concept and may be a no-op elsewhere. + t = tkinter.Toplevel(self.root) + t.group(self.root) + self.assertIn(t.group(), (str(self.root), '')) + + def test_wm_protocol(self): + t = tkinter.Toplevel(self.root) + self.assertIsInstance(t.protocol(), tuple) + t.protocol('WM_SAVE_YOURSELF', lambda: None) + self.assertIn('WM_SAVE_YOURSELF', t.protocol()) + # Querying a single protocol returns the bound command name. + self.assertTrue(t.protocol('WM_SAVE_YOURSELF')) + + def test_wm_transient(self): + t = tkinter.Toplevel(self.root) + self.assertEqual(t.transient(), '') + t.transient(self.root) + self.assertEqual(str(t.transient()), str(self.root)) + class EventTest(AbstractTkTest, unittest.TestCase): @@ -609,7 +1171,7 @@ def test_focus(self): self.assertIsInstance(e.serial, int) self.assertEqual(e.time, '??') self.assertIs(e.send_event, False) - self.assertFalse(hasattr(e, 'focus')) + self.assertNotHasAttr(e, 'focus') self.assertEqual(e.num, '??') self.assertEqual(e.state, '??') self.assertEqual(e.char, '??') @@ -642,7 +1204,7 @@ def test_configure(self): self.assertIsInstance(e.serial, int) self.assertEqual(e.time, '??') self.assertIs(e.send_event, False) - self.assertFalse(hasattr(e, 'focus')) + self.assertNotHasAttr(e, 'focus') self.assertEqual(e.num, '??') self.assertEqual(e.state, '??') self.assertEqual(e.char, '??') @@ -676,7 +1238,7 @@ def test_event_generate_key_press(self): self.assertIsInstance(e.serial, int) self.assertEqual(e.time, 0) self.assertIs(e.send_event, False) - self.assertFalse(hasattr(e, 'focus')) + self.assertNotHasAttr(e, 'focus') self.assertEqual(e.num, '??') self.assertIsInstance(e.state, int) self.assertNotEqual(e.state, 0) @@ -747,7 +1309,7 @@ def test_event_generate_button_press(self): self.assertIsInstance(e.serial, int) self.assertEqual(e.time, 0) self.assertIs(e.send_event, False) - self.assertFalse(hasattr(e, 'focus')) + self.assertNotHasAttr(e, 'focus') self.assertEqual(e.num, 1) self.assertEqual(e.state, 0) self.assertEqual(e.char, '??') @@ -781,7 +1343,7 @@ def test_event_generate_motion(self): self.assertIsInstance(e.serial, int) self.assertEqual(e.time, 0) self.assertIs(e.send_event, False) - self.assertFalse(hasattr(e, 'focus')) + self.assertNotHasAttr(e, 'focus') self.assertEqual(e.num, '??') self.assertEqual(e.state, 0x100) self.assertEqual(e.char, '??') @@ -814,7 +1376,7 @@ def test_event_generate_mouse_wheel(self): self.assertIs(e.widget, f) self.assertIsInstance(e.serial, int) self.assertIs(e.send_event, False) - self.assertFalse(hasattr(e, 'focus')) + self.assertNotHasAttr(e, 'focus') self.assertEqual(e.time, 0) self.assertEqual(e.num, '??') self.assertEqual(e.state, 0) @@ -849,7 +1411,7 @@ def test_generate_event_virtual_event(self): self.assertIsInstance(e.serial, int) self.assertEqual(e.time, 0) self.assertIs(e.send_event, False) - self.assertFalse(hasattr(e, 'focus')) + self.assertNotHasAttr(e, 'focus') self.assertEqual(e.num, '??') self.assertEqual(e.state, 0) self.assertEqual(e.char, '??') @@ -1308,17 +1870,17 @@ def test_no_default_root(self): self.assertIs(tkinter._default_root, root) tkinter.NoDefaultRoot() self.assertIs(tkinter._support_default_root, False) - self.assertFalse(hasattr(tkinter, '_default_root')) + self.assertNotHasAttr(tkinter, '_default_root') # repeated call is no-op tkinter.NoDefaultRoot() self.assertIs(tkinter._support_default_root, False) - self.assertFalse(hasattr(tkinter, '_default_root')) + self.assertNotHasAttr(tkinter, '_default_root') root.destroy() self.assertIs(tkinter._support_default_root, False) - self.assertFalse(hasattr(tkinter, '_default_root')) + self.assertNotHasAttr(tkinter, '_default_root') root = tkinter.Tk() self.assertIs(tkinter._support_default_root, False) - self.assertFalse(hasattr(tkinter, '_default_root')) + self.assertNotHasAttr(tkinter, '_default_root') root.destroy() def test_getboolean(self): diff --git a/Lib/test/test_tkinter/test_scrolledtext.py b/Lib/test/test_tkinter/test_scrolledtext.py new file mode 100644 index 000000000000000..913e7d8aab6c46a --- /dev/null +++ b/Lib/test/test_tkinter/test_scrolledtext.py @@ -0,0 +1,65 @@ +import unittest +import tkinter +from tkinter.scrolledtext import ScrolledText +from test.support import requires +from test.test_tkinter.support import setUpModule # noqa: F401 +from test.test_tkinter.support import AbstractTkTest + +requires('gui') + + +class ScrolledTextTest(AbstractTkTest, unittest.TestCase): + + def create(self, **kwargs): + st = ScrolledText(self.root, **kwargs) + self.addCleanup(st.destroy) + return st + + def test_create(self): + st = self.create(background='red', height=5) + # It is a Text widget held in a Frame together with a Scrollbar. + self.assertIsInstance(st, tkinter.Text) + self.assertIsInstance(st.frame, tkinter.Frame) + self.assertIsInstance(st.vbar, tkinter.Scrollbar) + self.assertEqual(st.winfo_parent(), str(st.frame)) + # str() returns the frame, so that geometry managers manage it. + self.assertEqual(str(st), str(st.frame)) + # Keyword options configure the Text. + self.assertEqual(str(st['background']), 'red') + self.assertEqual(st['height'], 5 if self.wantobjects else '5') + + def test_text_methods(self): + st = self.create() + st.insert('1.0', 'hello\nworld') + self.assertEqual(st.get('1.0', 'end-1c'), 'hello\nworld') + self.assertEqual(st.index('end-1c'), '2.5') + st.delete('1.0', 'end') + self.assertEqual(st.get('1.0', 'end-1c'), '') + + def test_geometry_methods(self): + st = self.create() + # configure is not redirected; it configures the Text. + st.configure(height=8) + self.assertEqual(st['height'], 8 if self.wantobjects else '8') + # Pack, Grid and Place methods are redirected to the frame. + st.pack() + self.root.update() + self.assertEqual(st.frame.winfo_manager(), 'pack') + self.assertEqual(st.pack_info(), st.frame.pack_info()) + st.pack_forget() + self.assertEqual(st.frame.winfo_manager(), '') + + def test_scrollbar(self): + st = self.create(height=5) + st.pack() + st.insert('1.0', '\n'.join(map(str, range(100)))) + self.root.update() + # The scrollbar tracks the text view. + self.assertEqual(st.vbar.get(), st.yview()) + st.yview_moveto(1.0) + self.root.update() + self.assertEqual(st.vbar.get()[1], 1.0) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_tkinter/test_simpledialog.py b/Lib/test/test_tkinter/test_simpledialog.py index 502f7f7098a322d..64edbb645bc80fa 100644 --- a/Lib/test/test_tkinter/test_simpledialog.py +++ b/Lib/test/test_tkinter/test_simpledialog.py @@ -1,12 +1,148 @@ import unittest import tkinter +from tkinter import messagebox from test.support import requires, swap_attr -from test.test_tkinter.support import AbstractDefaultRootTest -from tkinter.simpledialog import Dialog, askinteger +from test.test_tkinter.support import setUpModule # noqa: F401 +from test.test_tkinter.support import AbstractDefaultRootTest, AbstractTkTest +from tkinter.simpledialog import (Dialog, SimpleDialog, + askinteger, askfloat, askstring, + _QueryInteger, _QueryFloat, _QueryString) requires('gui') +class SimpleDialogTest(AbstractTkTest, unittest.TestCase): + # SimpleDialog's modal loop is in go(); its bindings are exercised here by + # generating events on the constructed dialog, without entering the loop. + + def create(self, **kw): + kw.setdefault('text', 'Question?') + kw.setdefault('buttons', ['Yes', 'No']) + kw.setdefault('default', 0) + kw.setdefault('cancel', 1) + d = SimpleDialog(self.root, **kw) + self.addCleanup(lambda: d.root.winfo_exists() and d.root.destroy()) + return d + + def test_message(self): + # The text is shown in a message widget. + d = self.create(text='Hello?') + self.assertEqual(d.message.winfo_class(), 'Message') + self.assertEqual(str(d.message.cget('text')), 'Hello?') + + def test_class_name(self): + # class_ sets the Tk class of the dialog window. + d = self.create(class_='MyDialog') + self.assertEqual(d.root.winfo_class(), 'MyDialog') + + def test_button(self): + # Pressing a button records its index. + d = self.create(buttons=['Yes', 'No']) + d.frame.winfo_children()[1].invoke() # "No" + self.assertEqual(d.num, 1) + + def test_default_button(self): + # The default button is drawn with a raised border. + d = self.create(buttons=['Yes', 'No'], default=0) + self.assertEqual(str(d.frame.winfo_children()[0].cget('relief')), 'ridge') + + def test_return_activates_default(self): + # <Return> invokes the default button. + d = self.create() # default 0 + self.require_mapped(d.root) + d.root.focus_force() + d.root.update() + d.root.event_generate('<Return>') + d.root.update() + self.assertEqual(d.num, 0) + + def test_return_no_default(self): + # With no default button, <Return> rings the bell and leaves the dialog + # open instead of activating a button. + d = self.create(default=None) + self.require_mapped(d.root) + d.root.focus_force() + d.root.update() + bells = [] + with swap_attr(d.root, 'bell', lambda *a, **k: bells.append(True)): + d.root.event_generate('<Return>') + d.root.update() + self.assertTrue(bells) # rang the bell + self.assertIsNone(d.num) + self.assertTrue(d.root.winfo_exists()) + + def test_wm_delete_cancels(self): + # Closing the window through the window manager records the cancel index. + d = self.create() # cancel 1 + d.wm_delete_window() + self.assertEqual(d.num, 1) + + def test_wm_delete_no_cancel(self): + # With no cancel index, closing the window through the window manager + # rings the bell and leaves the dialog open instead of recording an + # index. + d = self.create(default=None, cancel=None) + d.root.update() + bells = [] + with swap_attr(d.root, 'bell', lambda *a, **k: bells.append(True)): + d.wm_delete_window() + d.root.update() + self.assertTrue(bells) # rang the bell + self.assertIsNone(d.num) + self.assertTrue(d.root.winfo_exists()) + + def test_go(self): + # go() runs the modal loop and returns the chosen button's index. + d = self.create() + d.root.after(1, lambda: d.done(0)) + self.assertEqual(d.go(), 0) + + +class DialogTest(AbstractTkTest, unittest.TestCase): + # Dialog is a base class for custom dialogs; exercise it via _QueryInteger. + + def open(self, **kw): + with swap_attr(Dialog, 'wait_window', staticmethod(lambda w: None)): + d = _QueryInteger('Title', 'Prompt', parent=self.root, **kw) + self.addCleanup(lambda: d.winfo_exists() and d.destroy()) + return d + + def buttons(self, d): + # Map the button box's buttons by their label. + return {str(b.cget('text')): b + for frame in d.winfo_children() + for b in frame.winfo_children() + if b.winfo_class() == 'Button'} + + def test_background(self): + # The classic dialog keeps the default Toplevel background. + d = self.open() + ref = tkinter.Toplevel(self.root) + self.addCleanup(ref.destroy) + self.assertEqual(str(d.cget('background')), str(ref.cget('background'))) + + def test_buttons(self): + # The button box has OK (the default) and Cancel buttons. + buttons = self.buttons(self.open()) + self.assertEqual(set(buttons), {'OK', 'Cancel'}) + self.assertEqual(str(buttons['OK'].cget('default')), 'active') + + def test_ok(self): + # The OK button validates the entry and stores the result. + d = self.open() + d.entry.insert(0, '42') + self.buttons(d)['OK'].invoke() + self.assertEqual(d.result, 42) + self.assertFalse(d.winfo_exists()) # The dialog closed. + + def test_cancel(self): + # The Cancel button closes the dialog without a result. + d = self.open() + self.buttons(d)['Cancel'].invoke() + self.assertIsNone(d.result) + self.assertFalse(d.winfo_exists()) + + class DefaultRootTest(AbstractDefaultRootTest, unittest.TestCase): def test_askinteger(self): @@ -31,5 +167,109 @@ def mock_wait_window(w): self.assertRaises(RuntimeError, askinteger, "Go To Line", "Line number") +class QueryDialogTest(AbstractTkTest, unittest.TestCase): + # The query dialogs are modal: their __init__ blocks in wait_window(). + # Mock that out so the dialog stays alive and can be driven with generated + # events, exercising the <Return>/<Escape> bindings and the validation. + + def open(self, query, **kw): + with swap_attr(Dialog, 'wait_window', staticmethod(lambda w: None)): + d = query("Title", "Prompt", parent=self.root, **kw) + self.addCleanup(lambda: d.winfo_exists() and d.destroy()) + d.focus_force() + d.update() + return d + + def enter(self, d, value, key='<Return>'): + d.entry.delete(0, 'end') + d.entry.insert(0, value) + d.event_generate(key) + d.update() + + def test_initialvalue(self): + # The entry is pre-filled with the initial value, which is accepted. + d = self.open(_QueryInteger, initialvalue=42) + self.assertEqual(d.entry.get(), '42') + d.event_generate('<Return>') + d.update() + self.assertEqual(d.result, 42) + + def test_show(self): + # _QueryString hides the entered text when show is given. + d = self.open(_QueryString, show='*') + self.assertEqual(str(d.entry.cget('show')), '*') + + def test_return_accepts(self): + for query, value, expected in [ + (_QueryInteger, '42', 42), + (_QueryFloat, '1.5', 1.5), + (_QueryString, 'spam', 'spam'), + ]: + with self.subTest(query=query.__name__): + d = self.open(query) + self.enter(d, value) + self.assertEqual(d.result, expected) + self.assertFalse(d.winfo_exists()) # The dialog closed. + + def test_escape_cancels(self): + d = self.open(_QueryString) + self.enter(d, 'spam', '<Escape>') + self.assertIsNone(d.result) + self.assertFalse(d.winfo_exists()) + + def test_invalid_value(self): + warnings = [] + d = self.open(_QueryInteger) + with swap_attr(messagebox, 'showwarning', + lambda *a, **k: warnings.append(a)): + self.enter(d, 'not a number') + self.assertIsNone(d.result) + self.assertTrue(d.winfo_exists()) # The dialog stays open. + self.assertTrue(warnings) + + def test_out_of_range(self): + warnings = [] + d = self.open(_QueryInteger, minvalue=10, maxvalue=20) + with swap_attr(messagebox, 'showwarning', + lambda *a, **k: warnings.append(a)): + self.enter(d, '5') # Below the minimum. + self.assertIsNone(d.result) + self.enter(d, '25') # Above the maximum. + self.assertIsNone(d.result) + self.assertTrue(d.winfo_exists()) + self.assertEqual(len(warnings), 2) + + def test_boundary_values_accepted(self): + # The min/max checks are inclusive: a value equal to a bound passes. + d = self.open(_QueryInteger, minvalue=10, maxvalue=20) + self.enter(d, '10') # Exactly the minimum. + self.assertEqual(d.result, 10) + self.assertFalse(d.winfo_exists()) + + d = self.open(_QueryInteger, minvalue=10, maxvalue=20) + self.enter(d, '20') # Exactly the maximum. + self.assertEqual(d.result, 20) + self.assertFalse(d.winfo_exists()) + + def run_ask(self, ask, value, **kw): + # Drive a modal ask* function: enter a value and accept it. + def accept(d): + d.entry.delete(0, 'end') + d.entry.insert(0, value) + d.ok() + with swap_attr(Dialog, 'wait_window', staticmethod(accept)): + return ask('Title', 'Prompt', parent=self.root, **kw) + + def test_ask_functions(self): + self.assertEqual(self.run_ask(askinteger, '42'), 42) + self.assertEqual(self.run_ask(askfloat, '1.5'), 1.5) + self.assertEqual(self.run_ask(askstring, 'spam'), 'spam') + + def test_ask_cancelled(self): + # A cancelled ask* returns None. + with swap_attr(Dialog, 'wait_window', staticmethod(lambda d: d.cancel())): + self.assertIsNone(askstring('Title', 'Prompt', parent=self.root)) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tkinter/test_text.py b/Lib/test/test_tkinter/test_text.py index b26956930d34028..acbee1acad633e4 100644 --- a/Lib/test/test_tkinter/test_text.py +++ b/Lib/test/test_tkinter/test_text.py @@ -1,7 +1,9 @@ import unittest import tkinter +from tkinter import TclError from test.support import requires -from test.test_tkinter.support import AbstractTkTest +from test.test_tkinter.support import setUpModule # noqa: F401 +from test.test_tkinter.support import AbstractTkTest, requires_tk requires('gui') @@ -16,30 +18,605 @@ def test_debug(self): text = self.text olddebug = text.debug() try: - text.debug(0) - self.assertEqual(text.debug(), 0) - text.debug(1) - self.assertEqual(text.debug(), 1) + text.debug(False) + self.assertIs(text.debug(), False) + text.debug(True) + self.assertIs(text.debug(), True) finally: text.debug(olddebug) self.assertEqual(text.debug(), olddebug) + def test_index(self): + text = self.text + text.insert('1.0', 'Lorem ipsum\ndolor sit amet') + self.assertEqual(text.index('1.0'), '1.0') + self.assertEqual(text.index('1.end'), '1.11') + self.assertEqual(text.index('end'), '3.0') + self.assertEqual(text.index('2.5'), '2.5') + # Index past the end of a line is clamped to its end. + self.assertEqual(text.index('1.100'), '1.11') + self.assertRaises(TclError, text.index, 'invalid') + self.assertRaises(TypeError, text.index) + self.assertRaises(TypeError, text.index, '1.0', '2.5') + + def test_compare(self): + text = self.text + text.insert('1.0', 'Lorem ipsum\ndolor sit amet') + self.assertIs(text.compare('1.0', '<', '2.0'), True) + self.assertIs(text.compare('2.0', '<', '1.0'), False) + self.assertIs(text.compare('1.5', '==', '1.5'), True) + self.assertIs(text.compare('1.5', '!=', '1.5'), False) + self.assertIs(text.compare('2.0', '>=', '2.0'), True) + self.assertIs(text.compare('1.0', '<=', 'end'), True) + self.assertRaises(TclError, text.compare, '1.0', 'invalid', '2.0') + self.assertRaises(TypeError, text.compare, '1.0', '<') + self.assertRaises(TypeError, text.compare, '1.0', '<', '2.0', '3.0') + + def test_insert_get_delete(self): + text = self.text + text.insert('1.0', 'Lorem ipsum') + self.assertEqual(text.get('1.0', 'end'), 'Lorem ipsum\n') + self.assertEqual(text.get('1.0', '1.5'), 'Lorem') + self.assertEqual(text.get('1.6'), 'i') # single character + + # insert before an existing index + text.insert('1.0', '*** ') + self.assertEqual(text.get('1.0', 'end'), '*** Lorem ipsum\n') + + text.delete('1.0', '1.4') + self.assertEqual(text.get('1.0', 'end'), 'Lorem ipsum\n') + text.delete('1.5') # delete a single character + self.assertEqual(text.get('1.0', 'end'), 'Loremipsum\n') + self.assertRaises(TypeError, text.get) + self.assertRaises(TypeError, text.get, '1.0', '1.5', 'end') + self.assertRaises(TypeError, text.delete, '1.0', '1.5', 'end') + + def test_insert_with_tags(self): + text = self.text + text.insert('1.0', 'hello', 'a', ' ', ('a', 'b'), 'world', 'b') + self.assertEqual(text.get('1.0', 'end'), 'hello world\n') + self.assertEqual([str(i) for i in text.tag_ranges('a')], + ['1.0', '1.6']) + self.assertEqual([str(i) for i in text.tag_ranges('b')], + ['1.5', '1.11']) + + def test_replace(self): + text = self.text + text.insert('1.0', 'Lorem ipsum') + text.replace('1.0', '1.5', 'Hello') + self.assertEqual(text.get('1.0', 'end'), 'Hello ipsum\n') + text.replace('1.6', 'end - 1 char', 'world') + self.assertEqual(text.get('1.0', 'end'), 'Hello world\n') + self.assertRaises(TclError, text.replace, 'invalid', '1.5', 'x') + # The first index must not be after the second. + self.assertRaises(TclError, text.replace, '1.5', '1.0', 'x') + self.assertRaises(TypeError, text.replace, '1.0', '1.5') + + def test_mark(self): + text = self.text + text.insert('1.0', 'Lorem ipsum\ndolor sit amet') + # 'insert' and 'current' marks always exist. + self.assertIn('insert', text.mark_names()) + self.assertIn('current', text.mark_names()) + + text.mark_set('here', '1.5') + self.assertIn('here', text.mark_names()) + self.assertEqual(text.index('here'), '1.5') + text.mark_set('here', '2.3') + self.assertEqual(text.index('here'), '2.3') + + text.mark_unset('here') + self.assertNotIn('here', text.mark_names()) + + # Unsetting a non-existent mark is not an error. + text.mark_unset('nonexistent') + self.assertRaises(TclError, text.mark_set, 'm', 'invalid') + self.assertRaises(TypeError, text.mark_set, 'm') + self.assertRaises(TypeError, text.mark_set, 'm', '1.0', '2.0') + + def test_mark_gravity(self): + text = self.text + text.insert('1.0', 'Lorem ipsum') + text.mark_set('here', '1.5') + # The default gravity is right. + self.assertEqual(text.mark_gravity('here'), 'right') + text.mark_gravity('here', 'left') + self.assertEqual(text.mark_gravity('here'), 'left') + # With left gravity the mark stays before inserted text. + text.insert('here', 'XXX') + self.assertEqual(text.index('here'), '1.5') + # With right gravity the mark moves after inserted text. + text.mark_gravity('here', 'right') + text.insert('here', 'YYY') + self.assertEqual(text.index('here'), '1.8') + + self.assertRaises(TclError, text.mark_gravity, 'nonexistent') + self.assertRaises(TclError, text.mark_gravity, 'here', 'invalid') + self.assertRaises(TypeError, text.mark_gravity) + self.assertRaises(TypeError, text.mark_gravity, 'here', 'left', 'extra') + + def test_mark_next_previous(self): + text = self.text + text.insert('1.0', 'Lorem ipsum') + # Keep the builtin 'insert' and 'current' marks at 1.0 so they do + # not interfere with the queries below. + text.mark_set('insert', '1.0') + text.mark_set('current', '1.0') + text.mark_set('m1', '1.3') + text.mark_set('m2', '1.7') + # mark_next finds a mark at a position at or after the index, + # mark_previous finds one at a position strictly before it. + self.assertEqual(text.mark_next('1.1'), 'm1') + self.assertEqual(text.mark_next('1.3'), 'm1') + self.assertEqual(text.mark_next('1.4'), 'm2') + self.assertIsNone(text.mark_next('1.8')) + self.assertEqual(text.mark_previous('1.4'), 'm1') + self.assertEqual(text.mark_previous('1.7'), 'm1') + self.assertEqual(text.mark_previous('end'), 'm2') + self.assertIsNone(text.mark_previous('1.0')) + + self.assertRaises(TclError, text.mark_next, 'invalid') + self.assertRaises(TclError, text.mark_previous, 'invalid') + self.assertRaises(TypeError, text.mark_next) + self.assertRaises(TypeError, text.mark_previous) + self.assertRaises(TypeError, text.mark_next, '1.0', '2.0') + self.assertRaises(TypeError, text.mark_previous, '1.0', '2.0') + + def test_tag_add_remove_ranges(self): + text = self.text + text.insert('1.0', 'Lorem ipsum\ndolor sit amet') + self.assertEqual(text.tag_ranges('sel'), ()) + + text.tag_add('big', '1.0', '1.5') + self.assertEqual([str(i) for i in text.tag_ranges('big')], + ['1.0', '1.5']) + # Add a second, disjoint range. + text.tag_add('big', '2.0', '2.5') + self.assertEqual([str(i) for i in text.tag_ranges('big')], + ['1.0', '1.5', '2.0', '2.5']) + + text.tag_remove('big', '1.0', '1.3') + self.assertEqual([str(i) for i in text.tag_ranges('big')], + ['1.3', '1.5', '2.0', '2.5']) + + # tag_ranges of an undefined tag is empty, not an error. + self.assertEqual(text.tag_ranges('nonexistent'), ()) + self.assertRaises(TclError, text.tag_add, 'big', 'invalid') + self.assertRaises(TypeError, text.tag_add, 'big') + self.assertRaises(TclError, text.tag_remove, 'big', 'invalid') + self.assertRaises(TypeError, text.tag_remove, 'big', '1.0', '2.0', '3.0') + self.assertRaises(TypeError, text.tag_ranges, 'big', 'extra') + + def test_tag_names(self): + text = self.text + text.insert('1.0', 'Lorem ipsum') + text.tag_add('a', '1.0', '1.5') + text.tag_add('b', '1.3', '1.8') + # 'sel' always exists; order reflects priority (creation order). + self.assertEqual(set(text.tag_names()), {'sel', 'a', 'b'}) + # Tags applied to the character at the given index. + self.assertEqual(set(text.tag_names('1.4')), {'a', 'b'}) + self.assertEqual(set(text.tag_names('1.0')), {'a'}) + self.assertEqual(set(text.tag_names('1.10')), set()) + self.assertRaises(TclError, text.tag_names, 'invalid') + self.assertRaises(TypeError, text.tag_names, '1.0', '2.0') + + def test_tag_nextrange_prevrange(self): + text = self.text + text.insert('1.0', 'Lorem ipsum dolor') + text.tag_add('a', '1.0', '1.5') + text.tag_add('a', '1.12', '1.17') + + self.assertEqual([str(i) for i in text.tag_nextrange('a', '1.0')], + ['1.0', '1.5']) + self.assertEqual([str(i) for i in text.tag_nextrange('a', '1.5')], + ['1.12', '1.17']) + self.assertEqual(text.tag_nextrange('a', '1.17'), ()) + + self.assertEqual([str(i) for i in text.tag_prevrange('a', 'end')], + ['1.12', '1.17']) + self.assertEqual([str(i) for i in text.tag_prevrange('a', '1.12')], + ['1.0', '1.5']) + self.assertEqual(text.tag_prevrange('a', '1.0'), ()) + + # An undefined tag has no ranges, but the index must be valid. + self.assertEqual(text.tag_nextrange('nonexistent', '1.0'), ()) + self.assertRaises(TclError, text.tag_nextrange, 'a', 'invalid') + self.assertRaises(TclError, text.tag_prevrange, 'a', 'invalid') + self.assertRaises(TypeError, text.tag_nextrange, 'a') + self.assertRaises(TypeError, text.tag_prevrange, 'a') + self.assertRaises(TypeError, text.tag_nextrange, 'a', '1.0', '2.0', '3.0') + self.assertRaises(TypeError, text.tag_prevrange, 'a', '1.0', '2.0', '3.0') + + def test_tag_delete(self): + text = self.text + text.insert('1.0', 'Lorem ipsum') + text.tag_add('a', '1.0', '1.5') + text.tag_add('b', '1.6', '1.11') + self.assertEqual(set(text.tag_names()), {'sel', 'a', 'b'}) + text.tag_delete('a', 'b') + self.assertEqual(set(text.tag_names()), {'sel'}) + self.assertEqual(text.tag_ranges('a'), ()) + + def test_tag_raise_lower(self): + text = self.text + text.tag_configure('a', foreground='red') + text.tag_configure('b', foreground='green') + text.tag_configure('c', foreground='blue') + # Creation order is lowest-to-highest priority; tag_names lists + # them in increasing priority order. + self.assertEqual(text.tag_names(), ('sel', 'a', 'b', 'c')) + text.tag_raise('a') + self.assertEqual(text.tag_names(), ('sel', 'b', 'c', 'a')) + text.tag_lower('a') + self.assertEqual(text.tag_names(), ('a', 'sel', 'b', 'c')) + text.tag_raise('a', 'b') + self.assertEqual(text.tag_names(), ('sel', 'b', 'a', 'c')) + text.tag_lower('c', 'b') + self.assertEqual(text.tag_names(), ('sel', 'c', 'b', 'a')) + + self.assertRaises(TclError, text.tag_raise, 'nonexistent') + self.assertRaises(TclError, text.tag_lower, 'nonexistent') + self.assertRaises(TclError, text.tag_raise, 'a', 'nonexistent') + self.assertRaises(TypeError, text.tag_raise) + self.assertRaises(TypeError, text.tag_raise, 'a', 'b', 'c') + self.assertRaises(TypeError, text.tag_lower, 'a', 'b', 'c') + + def test_tag_cget_configure(self): + text = self.text + text.tag_configure('a', foreground='red', underline=True) + self.assertEqual(str(text.tag_cget('a', 'foreground')), 'red') + self.assertIn(text.tag_cget('a', 'underline'), (1, '1', True)) + # tag_cget normalizes the option name (no leading hyphen needed). + self.assertEqual(str(text.tag_cget('a', '-foreground')), 'red') + # configure() query returns the full option spec. + self.assertEqual(text.tag_configure('a', 'foreground')[-1], 'red') + text.tag_configure('a', foreground='blue') + self.assertEqual(str(text.tag_cget('a', 'foreground')), 'blue') + + self.assertRaises(TclError, text.tag_cget, 'nonexistent', + 'foreground') + self.assertRaises(TypeError, text.tag_cget, 'a') + self.assertRaises(TypeError, text.tag_cget, 'a', 'foreground', 'extra') + + def test_edit_modified(self): + text = self.text + self.assertEqual(text.edit_modified(), 0) + text.insert('1.0', 'spam') + self.assertEqual(text.edit_modified(), 1) + text.edit_modified(False) + self.assertEqual(text.edit_modified(), 0) + text.edit_modified(True) + self.assertEqual(text.edit_modified(), 1) + + def test_edit_undo_redo(self): + text = self.text + text.configure(undo=True) + text.insert('1.0', 'spam') + text.edit_separator() + text.insert('end', ' eggs') + self.assertEqual(text.get('1.0', 'end'), 'spam eggs\n') + + text.edit_undo() + self.assertEqual(text.get('1.0', 'end'), 'spam\n') + text.edit_redo() + self.assertEqual(text.get('1.0', 'end'), 'spam eggs\n') + + # An empty undo stack raises. + text.edit_reset() + self.assertRaises(TclError, text.edit_undo) + + def test_dump(self): + text = self.text + text.insert('1.0', 'hello') + text.tag_add('a', '1.0', '1.3') + text.mark_set('here', '1.2') + + dump = text.dump('1.0', 'end') + self.assertIsInstance(dump, list) + for item in dump: + self.assertIsInstance(item, tuple) + self.assertEqual(len(item), 3) + kinds = {item[0] for item in dump} + self.assertIn('text', kinds) + self.assertIn('tagon', kinds) + self.assertIn('mark', kinds) + + # Filtering by kind. + text_only = text.dump('1.0', 'end', text=True) + self.assertEqual({item[0] for item in text_only}, {'text'}) + self.assertEqual(''.join(item[1] for item in text_only), 'hello\n') + + # The command callback receives each triple and suppresses the result. + collected = [] + result = text.dump('1.0', 'end', text=True, + command=lambda *a: collected.append(a)) + self.assertIsNone(result) + self.assertEqual(''.join(key_value_index[1] + for key_value_index in collected), 'hello\n') + + self.assertRaises(TclError, text.dump, 'invalid') + self.assertRaises(TypeError, text.dump) + self.assertRaises(TypeError, text.dump, '1.0', 'end', None, 'extra') + + def test_image(self): + text = self.text + image = tkinter.PhotoImage(master=self.root, width=10, height=10) + text.insert('1.0', 'AB') + name = text.image_create('1.1', image=image) + self.assertIsInstance(name, str) + self.assertIn(name, self.root.splitlist(text.image_names())) + self.assertEqual(str(text.image_cget(name, 'image')), str(image)) + + text.image_configure(name, align='top') + self.assertEqual(str(text.image_cget(name, 'align')), 'top') + + # An embedded image occupies a single index position. + self.assertEqual(text.index('end - 1 char'), '1.3') + + # Either a name or an image is required, and the index must be valid. + self.assertRaises(TclError, text.image_create, '1.0') + self.assertRaises(TclError, text.image_create, 'invalid', + image=image) + self.assertRaises(TypeError, text.image_cget, name) + self.assertRaises(TypeError, text.image_cget, name, 'image', 'extra') + + def test_window(self): + text = self.text + button = tkinter.Button(text, text='ok') + text.insert('1.0', 'AB') + text.window_create('1.1', window=button) + self.assertEqual(self.root.splitlist(text.window_names()), + (str(button),)) + self.assertEqual(text.window_cget('1.1', 'window'), str(button)) + + text.window_configure('1.1', padx=5) + self.assertEqual(text.window_cget('1.1', 'padx'), 5) + + # No embedded window at this index, and the index must be valid. + self.assertRaises(TclError, text.window_cget, '1.0', 'window') + self.assertRaises(TclError, text.window_cget, 'invalid', 'window') + self.assertRaises(TypeError, text.window_cget, '1.1') + self.assertRaises(TypeError, text.window_cget, '1.1', 'window', 'extra') + button.destroy() + + def test_tag_configure(self): + text = self.text + tag = 'a' + getint = text.tk.getint + getboolean = text.tk.getboolean + + # Color options. + for opt in ('foreground', 'background'): + text.tag_configure(tag, **{opt: '#ff0000'}) + self.assertEqual(str(text.tag_cget(tag, opt)), '#ff0000') + + # Stipple (bitmap) options. + for opt in ('bgstipple', 'fgstipple'): + text.tag_configure(tag, **{opt: 'gray50'}) + self.assertEqual(str(text.tag_cget(tag, opt)), 'gray50') + + # Enumerated options. + for opt, value in (('justify', 'center'), ('wrap', 'word'), + ('relief', 'raised'), ('tabstyle', 'wordprocessor'), + ('offset', '5')): + text.tag_configure(tag, **{opt: value}) + self.assertEqual(str(text.tag_cget(tag, 'justify')), 'center') + self.assertEqual(str(text.tag_cget(tag, 'wrap')), 'word') + self.assertEqual(str(text.tag_cget(tag, 'relief')), 'raised') + self.assertEqual(str(text.tag_cget(tag, 'tabstyle')), 'wordprocessor') + + # Boolean options. + for opt in ('underline', 'overstrike', 'elide'): + text.tag_configure(tag, **{opt: True}) + self.assertIs(getboolean(text.tag_cget(tag, opt)), True) + text.tag_configure(tag, **{opt: False}) + self.assertIs(getboolean(text.tag_cget(tag, opt)), False) + + # Screen-distance (pixel) options. + for opt in ('borderwidth', 'lmargin1', 'lmargin2', 'rmargin', + 'spacing1', 'spacing2', 'spacing3'): + text.tag_configure(tag, **{opt: 7}) + self.assertEqual(getint(text.tag_cget(tag, opt)), 7) + + # Other options. + text.tag_configure(tag, font='Helvetica 12') + self.assertEqual(str(text.tag_cget(tag, 'font')), 'Helvetica 12') + text.tag_configure(tag, tabs=(10.2, '1i')) + self.assertEqual([str(x) for x in text.tag_ranges('sel')], []) + + self.assertRaises(TclError, text.tag_cget, tag, 'spam') + + @requires_tk(8, 6, 6) + def test_tag_configure_colors(self): + # Tag color options added in Tk 8.6.6. + text = self.text + tag = 'a' + for opt in ('selectforeground', 'selectbackground', + 'lmargincolor', 'rmargincolor', + 'underlinefg', 'overstrikefg'): + text.tag_configure(tag, **{opt: '#00ff00'}) + self.assertEqual(str(text.tag_cget(tag, opt)), '#00ff00') + + def test_tag_configure_query(self): + text = self.text + tag = 'a' + # Querying all options returns a dict keyed by option name. + cnf = text.tag_configure(tag) + self.assertIsInstance(cnf, dict) + self.assertIn('foreground', cnf) + # The value is the full 5-tuple option specification. + self.assertEqual(len(cnf['foreground']), 5) + + # Querying a single option returns its specification. + spec = text.tag_configure(tag, 'foreground') + self.assertEqual(spec[0], 'foreground') + self.assertEqual(spec[-1], '') # unset by default + + # Setting via keyword arguments and via a dict are equivalent. + text.tag_configure(tag, foreground='red') + self.assertEqual(str(text.tag_configure(tag, 'foreground')[-1]), 'red') + text.tag_configure(tag, {'foreground': 'blue'}) + self.assertEqual(str(text.tag_cget(tag, 'foreground')), 'blue') + + # tag_config is an alias of tag_configure. + self.assertEqual(text.tag_config, text.tag_configure) + + def test_image_configure(self): + text = self.text + image = tkinter.PhotoImage(master=self.root, width=10, height=10) + text.insert('1.0', 'AB') + name = text.image_create('1.1', image=image, name='img') + self.assertEqual(name, 'img') + + self.assertEqual(str(text.image_cget(name, 'name')), 'img') + self.assertEqual(str(text.image_cget(name, 'image')), str(image)) + for value in ('top', 'center', 'bottom', 'baseline'): + text.image_configure(name, align=value) + self.assertEqual(str(text.image_cget(name, 'align')), value) + text.image_configure(name, padx=3, pady=4) + self.assertEqual(text.tk.getint(text.image_cget(name, 'padx')), 3) + self.assertEqual(text.tk.getint(text.image_cget(name, 'pady')), 4) + + # Querying returns the full option set. + cnf = text.image_configure(name) + self.assertIsInstance(cnf, dict) + self.assertIn('align', cnf) + self.assertRaises(TclError, text.image_cget, name, 'spam') + + def test_window_configure(self): + text = self.text + button = tkinter.Button(text, text='ok') + text.insert('1.0', 'AB') + text.window_create('1.1', window=button) + + self.assertEqual(text.window_cget('1.1', 'window'), str(button)) + for value in ('top', 'center', 'bottom', 'baseline'): + text.window_configure('1.1', align=value) + self.assertEqual(str(text.window_cget('1.1', 'align')), value) + text.window_configure('1.1', padx=3, pady=4) + self.assertEqual(text.tk.getint(text.window_cget('1.1', 'padx')), 3) + self.assertEqual(text.tk.getint(text.window_cget('1.1', 'pady')), 4) + text.window_configure('1.1', stretch=True) + self.assertIs(text.tk.getboolean(text.window_cget('1.1', 'stretch')), + True) + + cnf = text.window_configure('1.1') + self.assertIsInstance(cnf, dict) + self.assertIn('stretch', cnf) + self.assertRaises(TclError, text.window_cget, '1.1', 'spam') + self.assertEqual(text.window_config, text.window_configure) + button.destroy() + + def test_peer(self): + text = self.text + text.insert('1.0', 'Lorem ipsum') + self.assertEqual(text.peer_names(), ()) + + text.peer_create('.peer1') + names = self.root.splitlist(text.tk.call('winfo', 'children', '.')) + self.assertIn('.peer1', [str(n) for n in names]) + self.assertEqual([str(p) for p in text.peer_names()], ['.peer1']) + # Peers share content. + self.assertEqual(text.tk.call('.peer1', 'get', '1.0', 'end'), + 'Lorem ipsum\n') + text.tk.call('destroy', '.peer1') + + def test_bbox(self): + text = self.text + text.insert('1.0', 'hello') + text.update() + bbox = text.bbox('1.0') + self.assertIsInstance(bbox, tuple) + self.assertEqual(len(bbox), 4) + for v in bbox: + self.assertIsInstance(v, int) + # A character that is not displayed has no bounding box. + self.assertIsNone(text.bbox('end')) + self.assertRaises(TclError, text.bbox, 'invalid') + self.assertRaises(TypeError, text.bbox) + self.assertRaises(TypeError, text.bbox, '1.0', '2.0') + + def test_dlineinfo(self): + text = self.text + text.insert('1.0', 'hello\nworld') + text.update() + info = text.dlineinfo('1.0') + self.assertIsInstance(info, tuple) + self.assertEqual(len(info), 5) + for v in info: + self.assertIsInstance(v, int) + self.assertRaises(TclError, text.dlineinfo, 'invalid') + self.assertRaises(TypeError, text.dlineinfo) + self.assertRaises(TypeError, text.dlineinfo, '1.0', '2.0') + + def test_see(self): + text = self.text + text.insert('1.0', '\n'.join('line %d' % i for i in range(200))) + text.update() + # Initially the last line is not visible. + self.assertIsNone(text.bbox('200.0')) + text.see('200.0') + text.update() + self.assertIsNotNone(text.bbox('200.0')) + self.assertRaises(TclError, text.see, 'invalid') + self.assertRaises(TypeError, text.see) + self.assertRaises(TypeError, text.see, '1.0', '2.0') + + # yview_pickplace is a deprecated way to make an index visible. + text.yview_pickplace('1.0') + text.update() + self.assertIsNotNone(text.bbox('1.0')) + def test_search(self): text = self.text # pattern and index are obligatory arguments. - self.assertRaises(tkinter.TclError, text.search, None, '1.0') - self.assertRaises(tkinter.TclError, text.search, 'a', None) - self.assertRaises(tkinter.TclError, text.search, None, None) + self.assertRaises(TclError, text.search, None, '1.0') + self.assertRaises(TclError, text.search, 'a', None) + self.assertRaises(TclError, text.search, None, None) # Invalid text index. - self.assertRaises(tkinter.TclError, text.search, '', 0) + self.assertRaises(TclError, text.search, '', 0) + self.assertRaises(TclError, text.search, '', '') + self.assertRaises(TclError, text.search, '', 'invalid') + self.assertRaises(TclError, text.search, '', '1.0', 'invalid') + + text.insert('1.0', + 'This is a test. This is only a test.\n' + 'Another line.\n' + 'Yet another line.\n' + '64-bit') + + self.assertEqual(text.search('test', '1.0'), '1.10') + self.assertEqual(text.search('test', '1.0', 'end'), '1.10') + self.assertEqual(text.search('test', '1.0', '1.10'), '') + self.assertEqual(text.search('test', '1.11'), '1.31') + self.assertEqual(text.search('test', '1.32', 'end'), '') + self.assertEqual(text.search('test', '1.32'), '1.10') + + self.assertEqual(text.search('', '1.0'), '1.0') # empty pattern + self.assertEqual(text.search('nonexistent', '1.0'), '') + self.assertEqual(text.search('-bit', '1.0'), '4.2') # starts with a hyphen + + self.assertEqual(text.search('line', '3.0'), '3.12') + self.assertEqual(text.search('line', '3.0', forwards=True), '3.12') + self.assertEqual(text.search('line', '3.0', backwards=True), '2.8') + self.assertEqual(text.search('line', '3.0', forwards=True, backwards=True), '2.8') + + self.assertEqual(text.search('t.', '1.0'), '1.13') + self.assertEqual(text.search('t.', '1.0', exact=True), '1.13') + self.assertEqual(text.search('t.', '1.0', regexp=True), '1.10') + self.assertEqual(text.search('t.', '1.0', exact=True, regexp=True), '1.10') + + self.assertEqual(text.search('TEST', '1.0'), '') + self.assertEqual(text.search('TEST', '1.0', nocase=True), '1.10') + + var = tkinter.Variable(self.root) + self.assertEqual(text.search('test', '1.0', count=var), '1.10') + self.assertEqual(var.get(), 4 if self.wantobjects else '4') - # Check if we are getting the indices as strings -- you are likely - # to get Tcl_Obj under Tk 8.5 if Tkinter doesn't convert it. - text.insert('1.0', 'hi-test') - self.assertEqual(text.search('-test', '1.0', 'end'), '1.2') - self.assertEqual(text.search('test', '1.0', 'end'), '1.3') + # TODO: Add test for elide=True def test_count(self): text = self.text @@ -76,8 +653,8 @@ def test_count(self): self.assertEqual(text.count('1.0', 'end'), (124,)) self.assertEqual(text.count('1.0', 'end', 'indices', return_ints=True), 124) self.assertEqual(text.count('1.0', 'end', 'indices'), (124,)) - self.assertRaises(tkinter.TclError, text.count, '1.0', 'end', 'spam') - self.assertRaises(tkinter.TclError, text.count, '1.0', 'end', '-lines') + self.assertRaises(TclError, text.count, '1.0', 'end', 'spam') + self.assertRaises(TclError, text.count, '1.0', 'end', '-lines') self.assertIsInstance(text.count('1.3', '1.5', 'ypixels', return_ints=True), int) self.assertIsInstance(text.count('1.3', '1.5', 'ypixels'), tuple) diff --git a/Lib/test/test_tkinter/test_tkinter_pipe.py b/Lib/test/test_tkinter/test_tkinter_pipe.py new file mode 100644 index 000000000000000..775d0ff5e19b0ce --- /dev/null +++ b/Lib/test/test_tkinter/test_tkinter_pipe.py @@ -0,0 +1,55 @@ +# test_tkinter_pipe.py +import unittest +import subprocess +import sys +from test import support + + +@unittest.skipUnless(support.has_subprocess_support, "test requires subprocess") +class TkinterPipeTest(unittest.TestCase): + + def test_tkinter_pipe_buffered(self): + args = [sys.executable, "-i"] + proc = subprocess.Popen(args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc.stdin.write(b"import tkinter\n") + proc.stdin.write(b"interpreter = tkinter.Tcl()\n") + proc.stdin.write(b"print('hello')\n") + proc.stdin.write(b"print('goodbye')\n") + proc.stdin.write(b"quit()\n") + stdout, stderr = proc.communicate(timeout=support.SHORT_TIMEOUT) + stdout = stdout.decode() + self.assertEqual(stdout.split(), ['hello', 'goodbye']) + + def test_tkinter_pipe_unbuffered(self): + args = [sys.executable, "-i", "-u"] + proc = subprocess.Popen(args, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + proc.stdin.write(b"import tkinter\n") + proc.stdin.write(b"interpreter = tkinter.Tcl()\n") + + proc.stdin.write(b"print('hello')\n") + proc.stdin.flush() + stdout = proc.stdout.readline() + stdout = stdout.decode() + self.assertEqual(stdout.strip(), 'hello') + + proc.stdin.write(b"print('hello again')\n") + proc.stdin.flush() + stdout = proc.stdout.readline() + stdout = stdout.decode() + self.assertEqual(stdout.strip(), 'hello again') + + proc.stdin.write(b"print('goodbye')\n") + proc.stdin.write(b"quit()\n") + stdout, stderr = proc.communicate(timeout=support.SHORT_TIMEOUT) + stdout = stdout.decode() + self.assertEqual(stdout.strip(), 'goodbye') + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_tkinter/test_variables.py b/Lib/test/test_tkinter/test_variables.py index 75b3a6934fc0e31..5ee4e9467dd0cba 100644 --- a/Lib/test/test_tkinter/test_variables.py +++ b/Lib/test/test_tkinter/test_variables.py @@ -6,6 +6,7 @@ from tkinter import (Variable, StringVar, IntVar, DoubleVar, BooleanVar, Tcl, TclError) from test.support import ALWAYS_EQ +from test.test_tkinter.support import setUpModule # noqa: F401 from test.test_tkinter.support import AbstractDefaultRootTest, tcl_version @@ -110,6 +111,7 @@ def test_initialize(self): self.assertFalse(v.side_effect) v.set("value") self.assertTrue(v.side_effect) + self.assertEqual(Variable.initialize, Variable.set) def test_trace_old(self): if tcl_version >= (9, 0): @@ -117,6 +119,7 @@ def test_trace_old(self): # Old interface v = Variable(self.root) vname = str(v) + self.assertEqual(v.trace, v.trace_variable) trace = [] def read_tracer(*args): trace.append(('read',) + args) @@ -327,6 +330,7 @@ def test_set(self): self.assertEqual(self.root.globalgetvar("name"), false) v.set("on") self.assertEqual(self.root.globalgetvar("name"), true) + self.assertEqual(BooleanVar.initialize, BooleanVar.set) def test_invalid_value_domain(self): false = 0 if self.root.wantobjects() else "0" diff --git a/Lib/test/test_tkinter/test_widgets.py b/Lib/test/test_tkinter/test_widgets.py index ff3f92e9b5ef831..14c06ee913dc8a5 100644 --- a/Lib/test/test_tkinter/test_widgets.py +++ b/Lib/test/test_tkinter/test_widgets.py @@ -4,8 +4,10 @@ import os from test.support import requires +from test.test_tkinter.support import setUpModule # noqa: F401 from test.test_tkinter.support import (requires_tk, tk_version, get_tk_patchlevel, widget_eq, + wait_until_mapped, AbstractDefaultRootTest) from test.test_tkinter.widget_tests import ( @@ -25,12 +27,8 @@ def float_round(x): return float(round(x)) class AbstractToplevelTest(AbstractWidgetTest, PixelSizeTests): - if tk_version < (9, 0): - _no_round = {'padx', 'pady'} - else: - _no_round = {'borderwidth', 'height', 'highlightthickness', 'padx', - 'pady', 'width'} - if tk_version < (9, 0): + _no_round = {'padx', 'pady'} + if tk_version < (8, 7): _clipped = {'highlightthickness'} else: _clipped = {'borderwidth', 'height', 'highlightthickness', 'padx', @@ -121,11 +119,6 @@ class FrameTest(AbstractToplevelTest, unittest.TestCase): 'highlightbackground', 'highlightcolor', 'highlightthickness', 'padx', 'pady', 'relief', 'takefocus', 'tile', 'visual', 'width', ) - if tk_version < (9, 0): - _no_round = {'padx', 'pady'} - else: - _no_round = {'borderwidth', 'height', 'highlightthickness', 'padx', - 'pady', 'width'} def create(self, **kwargs): return tkinter.Frame(self.root, **kwargs) @@ -141,11 +134,6 @@ class LabelFrameTest(AbstractToplevelTest, unittest.TestCase): 'labelanchor', 'labelwidget', 'padx', 'pady', 'relief', 'takefocus', 'text', 'visual', 'width', ) - if tk_version < (9, 0): - _no_round = {'padx', 'pady'} - else: - _no_round = {'borderwidth', 'height', 'highlightthickness', 'padx', - 'pady', 'width'} def create(self, **kwargs): return tkinter.LabelFrame(self.root, **kwargs) @@ -166,11 +154,19 @@ def test_configure_labelwidget(self): # Label, Button, Checkbutton, Radiobutton, MenuButton class AbstractLabelTest(AbstractWidgetTest, IntegerSizeTests): _rounds_pixels = False - if tk_version < (9, 0): + if tk_version < (8, 7): _clipped = {} + elif tk_version < (9, 0): + _clipped = {'borderwidth', 'height', 'highlightthickness', 'padx', 'pady', 'width'} else: - _clipped = {'borderwidth', 'insertborderwidth', 'highlightthickness', - 'padx', 'pady'} + _clipped = {'borderwidth', 'height', 'highlightthickness', + 'insertborderwidth', 'padx', 'pady', 'width'} + + def setUp(self): + super().setUp() + if tk_version[:2] == (9, 0) and get_tk_patchlevel(self.root) < (9, 0, 2): + self._clipped = self._clipped - {'height', 'width'} + @add_configure_tests(StandardOptionsTests) class LabelTest(AbstractLabelTest, unittest.TestCase): @@ -200,6 +196,11 @@ class ButtonTest(AbstractLabelTest, unittest.TestCase): 'repeatdelay', 'repeatinterval', 'state', 'takefocus', 'text', 'textvariable', 'underline', 'width', 'wraplength') + if tk_version < (8, 7): + _clipped = {} + else: + _clipped = {'borderwidth', 'height', 'highlightthickness', + 'padx', 'pady', 'width'} def create(self, **kwargs): return tkinter.Button(self.root, **kwargs) @@ -208,6 +209,23 @@ def test_configure_default(self): widget = self.create() self.checkEnumParam(widget, 'default', 'active', 'disabled', 'normal') + def test_invoke(self): + success = [] + widget = self.create(command=lambda: success.append(1)) + widget.pack() + widget.invoke() + self.assertEqual(success, [1]) + # invoke does nothing for a disabled button. + widget.configure(state='disabled') + widget.invoke() + self.assertEqual(success, [1]) + + def test_flash(self): + widget = self.create() + widget.pack() + widget.update_idletasks() + widget.flash() # No exception. + @add_configure_tests(StandardOptionsTests) class CheckbuttonTest(AbstractLabelTest, unittest.TestCase): @@ -263,6 +281,39 @@ def test_same_name(self): b2.deselect() self.assertEqual(v.get(), 0) + def test_invoke(self): + success = [] + v = tkinter.IntVar(self.root) + widget = self.create(variable=v, onvalue=1, offvalue=0, + command=lambda: success.append(v.get())) + widget.pack() + widget.invoke() + self.assertEqual(v.get(), 1) + self.assertEqual(success, [1]) + widget.invoke() + self.assertEqual(v.get(), 0) + self.assertEqual(success, [1, 0]) + # A disabled checkbutton is not toggled and its command is not called. + widget.configure(state='disabled') + widget.invoke() + self.assertEqual(v.get(), 0) + self.assertEqual(success, [1, 0]) + + def test_toggle(self): + v = tkinter.IntVar(self.root) + widget = self.create(variable=v, onvalue=1, offvalue=0) + self.assertEqual(v.get(), 0) + widget.toggle() + self.assertEqual(v.get(), 1) + widget.toggle() + self.assertEqual(v.get(), 0) + + def test_flash(self): + widget = self.create() + widget.pack() + widget.update_idletasks() + widget.flash() # No exception. + @add_configure_tests(StandardOptionsTests) class RadiobuttonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( @@ -285,6 +336,28 @@ def test_configure_value(self): widget = self.create() self.checkParams(widget, 'value', 1, 2.3, '', 'any string') + def test_invoke(self): + success = [] + v = tkinter.StringVar(self.root) + widget = self.create(variable=v, value='on', + command=lambda: success.append(v.get())) + widget.pack() + widget.invoke() + self.assertEqual(v.get(), 'on') + self.assertEqual(success, ['on']) + # invoke does nothing for a disabled radiobutton. + v.set('') + widget.configure(state='disabled') + widget.invoke() + self.assertEqual(v.get(), '') + self.assertEqual(success, ['on']) + + def test_flash(self): + widget = self.create() + widget.pack() + widget.update_idletasks() + widget.flash() # No exception. + @add_configure_tests(StandardOptionsTests) class MenubuttonTest(AbstractLabelTest, unittest.TestCase): @@ -300,10 +373,17 @@ class MenubuttonTest(AbstractLabelTest, unittest.TestCase): 'underline', 'width', 'wraplength', ) _rounds_pixels = (tk_version < (9, 0)) - if tk_version < (9, 0): + if tk_version < (8, 7): _clipped = {'highlightthickness', 'padx', 'pady'} + elif tk_version < (9, 0): + _clipped = {'borderwidth', 'highlightthickness', 'padx', 'pady'} else: - _clipped ={ 'insertborderwidth', 'highlightthickness', 'padx', 'pady'} + _clipped = {'borderwidth', 'highlightthickness', 'insertborderwidth', 'padx', 'pady'} + + def setUp(self): + super().setUp() + if tk_version[:2] == (9, 0) and get_tk_patchlevel(self.root) < (9, 0, 1): + self._clipped = self._clipped - {'borderwidth'} def create(self, **kwargs): return tkinter.Menubutton(self.root, **kwargs) @@ -315,13 +395,17 @@ def test_configure_direction(self): def test_configure_height(self): widget = self.create() - self.checkIntegerParam(widget, 'height', 100, -100, 0, conv=str) + if tk_version < (8, 7) or (tk_version[:2] == (9, 0) and get_tk_patchlevel(self.root) < (9, 0, 1)): + conv = str + else: + conv = False + self.checkIntegerParam(widget, 'height', 100, -100, 0, conv=conv) def test_configure_image(self): widget = self.create() image = tkinter.PhotoImage(master=self.root, name='image1') self.checkParam(widget, 'image', image, conv=str) - if tk_version < (9, 0): + if tk_version < (8, 7): errmsg = 'image "spam" doesn\'t exist' else: errmsg = 'image "spam" does not exist' @@ -342,7 +426,11 @@ def test_configure_menu(self): def test_configure_width(self): widget = self.create() - self.checkIntegerParam(widget, 'width', 402, -402, 0, conv=str) + if tk_version < (8, 7) or (tk_version[:2] == (9, 0) and get_tk_patchlevel(self.root) < (9, 0, 1)): + conv = str + else: + conv = False + self.checkIntegerParam(widget, 'width', 402, -402, 0, conv=conv) class OptionMenuTest(MenubuttonTest, unittest.TestCase): @@ -361,12 +449,11 @@ def test_specify_name(self): @add_configure_tests(IntegerSizeTests, StandardOptionsTests) class EntryTest(AbstractWidgetTest, unittest.TestCase): - _rounds_pixels = (tk_version < (9, 0)) - if tk_version < (9, 0): + if tk_version < (8, 7): _clipped = {'highlightthickness'} else: - _clipped = {'highlightthickness', 'borderwidth', 'insertborderwidth', - 'selectborderwidth'} + _clipped = {'borderwidth', 'highlightthickness', 'insertborderwidth', + 'insertwidth', 'selectborderwidth'} OPTIONS = ( 'background', 'borderwidth', 'cursor', @@ -392,23 +479,20 @@ def test_configure_disabledbackground(self): def test_configure_insertborderwidth(self): widget = self.create(insertwidth=100) self.checkPixelsParam(widget, 'insertborderwidth', - 0, 1.3, 2.6, 6, '10p') - self.checkParam(widget, 'insertborderwidth', -2) + 0, 1.3, 2.6, 6, -2, '10p') # insertborderwidth is bounded above by a half of insertwidth. - expected = 100 // 2 if tk_version < (9, 0) else 60 + expected = 100 // 2 if tk_version < (8, 7) else 60 self.checkParam(widget, 'insertborderwidth', 60, expected=expected) def test_configure_insertwidth(self): widget = self.create() - self.checkPixelsParam(widget, 'insertwidth', 1.3, 3.6, '10p') - if tk_version < (9, 0): + self.checkPixelsParam(widget, 'insertwidth', 1.3, 3.6, 0.9, '10p') + if tk_version < (8, 7): + self.checkParam(widget, 'insertwidth', 0, expected=2) self.checkParam(widget, 'insertwidth', 0.1, expected=2) self.checkParam(widget, 'insertwidth', -2, expected=2) - self.checkParam(widget, 'insertwidth', 0.9, expected=1) else: - self.checkParam(widget, 'insertwidth', 0.1) - self.checkParam(widget, 'insertwidth', -2, expected=0) - self.checkParam(widget, 'insertwidth', 0.9) + self.checkPixelsParam(widget, 'insertwidth', 0, 0.1, -2) def test_configure_invalidcommand(self): widget = self.create() @@ -465,6 +549,47 @@ def test_selection_methods(self): self.assertEqual(widget.selection_get(), '12345') widget.selection_adjust(0) + def test_delete(self): + widget = self.create() + widget.insert(0, 'abcdef') + widget.delete(1, 3) + self.assertEqual(widget.get(), 'adef') + widget.delete(1) + self.assertEqual(widget.get(), 'aef') + widget.delete(0, 'end') + self.assertEqual(widget.get(), '') + self.assertRaisesRegex(TclError, r'bad (entry|spinbox) index "xyz"', + widget.delete, 'xyz') + self.assertRaises(TypeError, widget.delete) + + def test_icursor(self): + widget = self.create() + widget.insert(0, 'abcdef') + widget.icursor(3) + widget.insert('insert', 'XYZ') + self.assertEqual(widget.get(), 'abcXYZdef') + self.assertRaisesRegex(TclError, r'bad (entry|spinbox) index "xyz"', + widget.icursor, 'xyz') + self.assertRaises(TypeError, widget.icursor) + + def test_select_aliases(self): + # The select_* methods are aliases of the selection_* methods. + widget = self.create() + widget.insert(0, '12345') + self.assertFalse(widget.select_present()) + widget.select_range(0, 'end') + self.assertTrue(widget.select_present()) + self.assertEqual(widget.selection_get(), '12345') + widget.select_from(1) + widget.select_to(3) + self.assertEqual(widget.selection_get(), '23') + widget.select_adjust(4) + self.assertEqual(widget.selection_get(), '234') + widget.select_clear() + self.assertFalse(widget.select_present()) + self.assertRaisesRegex(TclError, 'bad entry index "xyz"', + widget.select_range, 'xyz', 'end') + @add_configure_tests(StandardOptionsTests) class SpinboxTest(EntryTest, unittest.TestCase): @@ -551,11 +676,22 @@ def test_configure_values(self): # XXX widget = self.create() self.assertEqual(widget['values'], '') - self.checkParam(widget, 'values', 'mon tue wed thur') + if tk_version < (8, 7) or (tk_version[:2] == (9, 0) and get_tk_patchlevel(self.root) < (9, 0, 1)): + expected = 'mon tue wed thur' + else: + expected = ('mon', 'tue', 'wed', 'thur') + self.checkParam(widget, 'values', 'mon tue wed thur', + expected=expected) self.checkParam(widget, 'values', ('mon', 'tue', 'wed', 'thur'), - expected='mon tue wed thur') + expected=expected) + + if tk_version < (8, 7) or (tk_version[:2] == (9, 0) and get_tk_patchlevel(self.root) < (9, 0, 1)): + expected = '42 3.14 {} {any string}' + else: + expected = (42, 3.14, '', 'any string') self.checkParam(widget, 'values', (42, 3.14, '', 'any string'), - expected='42 3.14 {} {any string}') + expected=expected) + self.checkParam(widget, 'values', '') def test_configure_wrap(self): @@ -602,6 +738,39 @@ def test_selection_element(self): widget.selection_element("buttondown") self.assertEqual(widget.selection_element(), "buttondown") + # Spinbox has no select_* aliases, unlike Entry. + test_select_aliases = None + + def test_invoke(self): + widget = self.create(from_=0, to=10) + widget.delete(0, 'end') + widget.insert(0, '5') + widget.invoke('buttonup') + self.assertEqual(widget.get(), '6') + widget.invoke('buttondown') + self.assertEqual(widget.get(), '5') + self.assertRaisesRegex(TclError, 'bad element "spam"', + widget.invoke, 'spam') + + def test_identify(self): + widget = self.create() + widget.pack() + # Identifying the element under a point requires the widget to be + # mapped with a real size. + if wait_until_mapped(widget): + self.assertIn(widget.identify(5, 5), + ('entry', 'buttonup', 'buttondown', 'none')) + self.assertRaises(TclError, widget.identify, 'a', 'b') + + def test_scan(self): + widget = self.create() + widget.insert(0, 'a' * 100) + widget.pack() + widget.update_idletasks() + self.assertEqual(widget.scan_mark(10), ()) + self.assertEqual(widget.scan_dragto(0), ()) + self.assertRaises(TypeError, widget.scan_mark) + @add_configure_tests(StandardOptionsTests) class TextTest(AbstractWidgetTest, unittest.TestCase): @@ -618,9 +787,20 @@ class TextTest(AbstractWidgetTest, unittest.TestCase): 'tabs', 'tabstyle', 'takefocus', 'undo', 'width', 'wrap', 'xscrollcommand', 'yscrollcommand', ) - _rounds_pixels = (tk_version < (9, 0)) _no_round = {'selectborderwidth'} - _clipped = {'highlightthickness'} + if tk_version < (9, 0): + _clipped = {'highlightthickness', 'spacing1', 'spacing2', 'spacing3'} + else: + _clipped = {'borderwidth', 'height', 'highlightthickness', + 'insertborderwidth', 'insertwidth', 'padx', 'pady', + 'selectborderwidth', 'spacing1', 'spacing2', 'spacing3'} + + def setUp(self): + super().setUp() + if tk_version[:2] == (9, 0) and get_tk_patchlevel(self.root) < (9, 0, 2): + self._clipped = self._clipped - {'borderwidth', 'height', 'padx', 'pady'} + if tk_version[:2] == (9, 0) and get_tk_patchlevel(self.root) < (9, 0, 1): + self._clipped = self._clipped - {'insertborderwidth', 'insertwidth', 'selectborderwidth'} def create(self, **kwargs): return tkinter.Text(self.root, **kwargs) @@ -649,10 +829,11 @@ def test_configure_endline(self): def test_configure_height(self): widget = self.create() self.checkPixelsParam(widget, 'height', 100, 101.2, 102.6, '3c') - self.checkParam(widget, 'height', -100, - expected=1 if tk_version < (9, 0) else -100) - self.checkParam(widget, 'height', 0, - expected=1 if tk_version < (9, 0) else 0 ) + if tk_version < (9, 0): + self.checkParam(widget, 'height', 0, expected=1) + self.checkParam(widget, 'height', -100, expected=1) + else: + self.checkPixelsParam(widget, 'height', 0, -100) def test_configure_maxundo(self): widget = self.create() @@ -668,25 +849,17 @@ def test_configure_insertunfocussed(self): self.checkEnumParam(widget, 'insertunfocussed', 'hollow', 'none', 'solid') - def test_configure_selectborderwidth(self): - widget = self.create() - self.checkPixelsParam(widget, 'selectborderwidth', - 1.3, 2.6, -2, '10p', conv=False) - def test_configure_spacing1(self): widget = self.create() - self.checkPixelsParam(widget, 'spacing1', 20, 21.4, 22.6, '0.5c') - self.checkParam(widget, 'spacing1', -5, expected=0) + self.checkPixelsParam(widget, 'spacing1', 20, 21.4, 22.6, -5, '0.5c') def test_configure_spacing2(self): widget = self.create() - self.checkPixelsParam(widget, 'spacing2', 5, 6.4, 7.6, '0.1c') - self.checkParam(widget, 'spacing2', -1, expected=0) + self.checkPixelsParam(widget, 'spacing2', 5, 6.4, 7.6, -1, '0.1c') def test_configure_spacing3(self): widget = self.create() - self.checkPixelsParam(widget, 'spacing3', 20, 21.4, 22.6, '0.5c') - self.checkParam(widget, 'spacing3', -10, expected=0) + self.checkPixelsParam(widget, 'spacing3', 20, 21.4, 22.6, -10, '0.5c') def test_configure_startline(self): widget = self.create() @@ -759,17 +932,22 @@ class CanvasTest(AbstractWidgetTest, unittest.TestCase): 'xscrollcommand', 'xscrollincrement', 'yscrollcommand', 'yscrollincrement', 'width', ) - _rounds_pixels = True - if tk_version < (9, 0): - _noround = {} + if tk_version < (8, 7): _clipped = {'highlightthickness'} else: - _no_round = {'borderwidth', 'height', 'highlightthickness', 'width', - 'xscrollincrement', 'yscrollincrement'} - _clipped = {'borderwidth', 'height', 'highlightthickness', 'width', - 'xscrollincrement', 'yscrollincrement'} + _clipped = {'borderwidth', 'height', 'highlightthickness', + 'insertborderwidth', 'insertwidth', 'selectborderwidth', + 'width', 'xscrollincrement', 'yscrollincrement'} _stringify = True + def setUp(self): + super().setUp() + if tk_version[:2] == (9, 0) and get_tk_patchlevel(self.root) < (9, 0, 1): + self._rounds_pixels = True + self._no_round = {'borderwidth', 'height', 'highlightthickness', + 'width', 'xscrollincrement', 'yscrollincrement'} + self._clipped = self._clipped - {'insertborderwidth', 'insertwidth', 'selectborderwidth'} + def create(self, **kwargs): return tkinter.Canvas(self.root, **kwargs) @@ -916,7 +1094,6 @@ def test_create_line(self): def test_create_polygon(self): c = self.create() - tk87 = tk_version >= (8, 7) # In Tk < 8.7 polygons are filled, but has no outline by default. # This affects its size, so always explicitly specify outline. i1 = c.create_polygon(20, 30, 40, 50, 60, 10, outline='red') @@ -948,6 +1125,122 @@ def test_create_polygon(self): self._test_option_smooth(c, lambda **kwargs: c.create_polygon(20, 30, 40, 50, 60, 10, **kwargs)) + def test_create_arc(self): + c = self.create() + i = c.create_arc(10, 20, 30, 40) + self.assertEqual(c.coords(i), [10.0, 20.0, 30.0, 40.0]) + self.assertEqual(c.itemcget(i, 'style'), 'pieslice') + self.assertEqual(float(c.itemcget(i, 'start')), 0.0) + self.assertEqual(float(c.itemcget(i, 'extent')), 90.0) + + for style in 'pieslice', 'chord', 'arc': + i = c.create_arc(10, 20, 30, 40, style=style) + self.assertEqual(c.itemcget(i, 'style'), style) + self.assertRaises(TclError, c.create_arc, 10, 20, 30, 40, style='spam') + + i = c.create_arc(10, 20, 30, 40, start=45, extent=120, + outline='red', fill='blue', width=3) + self.assertEqual(float(c.itemcget(i, 'start')), 45.0) + self.assertEqual(float(c.itemcget(i, 'extent')), 120.0) + self.assertEqual(str(c.itemcget(i, 'outline')), 'red') + self.assertEqual(str(c.itemcget(i, 'fill')), 'blue') + self.assertEqual(float(c.itemcget(i, 'width')), 3.0) + self.assertRaises(TclError, c.create_arc, 10, 20, 30, 40, extent='spam') + + def test_create_oval(self): + c = self.create() + i = c.create_oval(10, 20, 30, 40) + self.assertEqual(c.coords(i), [10.0, 20.0, 30.0, 40.0]) + self.assertEqual(c.bbox(i), (9, 19, 31, 41)) + self.assertEqual(c.itemcget(i, 'stipple'), '') + + i = c.create_oval(10, 20, 30, 40, fill='red', outline='blue', width=2) + self.assertEqual(str(c.itemcget(i, 'fill')), 'red') + self.assertEqual(str(c.itemcget(i, 'outline')), 'blue') + self.assertEqual(float(c.itemcget(i, 'width')), 2.0) + self.assertRaises(TclError, c.create_oval, 10, 20, 30, 40, width='spam') + + def test_create_bitmap(self): + c = self.create() + i = c.create_bitmap(10, 20, bitmap='gray50') + self.assertEqual(c.coords(i), [10.0, 20.0]) + self.assertEqual(c.itemcget(i, 'bitmap'), 'gray50') + self.assertEqual(c.itemcget(i, 'anchor'), 'center') + + for anchor in 'n', 'ne', 'e', 'se', 's', 'sw', 'w', 'nw', 'center': + i = c.create_bitmap(10, 20, bitmap='gray50', anchor=anchor) + self.assertEqual(c.itemcget(i, 'anchor'), anchor) + self.assertRaises(TclError, c.create_bitmap, 10, 20, + bitmap='gray50', anchor='spam') + + i = c.create_bitmap(10, 20, bitmap='gray50', + foreground='red', background='blue') + self.assertEqual(str(c.itemcget(i, 'foreground')), 'red') + self.assertEqual(str(c.itemcget(i, 'background')), 'blue') + if c._windowingsystem != 'aqua': + # Aqua resolves bitmaps lazily and does not report a bad name here. + self.assertRaises(TclError, c.create_bitmap, 10, 20, bitmap='spam') + + def test_create_image(self): + c = self.create() + image = tkinter.PhotoImage(master=self.root, width=10, height=10) + i = c.create_image(10, 20, image=image) + self.assertEqual(c.coords(i), [10.0, 20.0]) + self.assertEqual(str(c.itemcget(i, 'image')), str(image)) + self.assertEqual(c.itemcget(i, 'anchor'), 'center') + + for anchor in 'n', 'ne', 'e', 'se', 's', 'sw', 'w', 'nw', 'center': + i = c.create_image(10, 20, image=image, anchor=anchor) + self.assertEqual(c.itemcget(i, 'anchor'), anchor) + self.assertRaises(TclError, c.create_image, 10, 20, + image=image, anchor='spam') + + def test_create_text(self): + c = self.create() + i = c.create_text(10, 20, text='Hello') + self.assertEqual(c.coords(i), [10.0, 20.0]) + self.assertEqual(c.itemcget(i, 'text'), 'Hello') + self.assertEqual(c.itemcget(i, 'anchor'), 'center') + self.assertEqual(c.itemcget(i, 'justify'), 'left') + + for justify in 'left', 'right', 'center': + i = c.create_text(10, 20, text='Hello', justify=justify) + self.assertEqual(c.itemcget(i, 'justify'), justify) + self.assertRaises(TclError, c.create_text, 10, 20, justify='spam') + + for anchor in 'n', 'ne', 'e', 'se', 's', 'sw', 'w', 'nw', 'center': + i = c.create_text(10, 20, text='Hello', anchor=anchor) + self.assertEqual(c.itemcget(i, 'anchor'), anchor) + self.assertRaises(TclError, c.create_text, 10, 20, anchor='spam') + + i = c.create_text(10, 20, text='Hello', fill='red', angle=45, + font='TkFixedFont') + self.assertEqual(str(c.itemcget(i, 'fill')), 'red') + self.assertEqual(float(c.itemcget(i, 'angle')), 45.0) + self.assertEqual(str(c.itemcget(i, 'font')), 'TkFixedFont') + self.assertRaises(TclError, c.create_text, 10, 20, angle='spam') + + def test_create_window(self): + c = self.create() + button = tkinter.Button(c, text='ok') + i = c.create_window(10, 20, window=button) + self.assertEqual(c.coords(i), [10.0, 20.0]) + self.assertEqual(c.itemcget(i, 'window'), str(button)) + self.assertEqual(c.itemcget(i, 'anchor'), 'center') + + for anchor in 'n', 'ne', 'e', 'se', 's', 'sw', 'w', 'nw', 'center': + i = c.create_window(10, 20, window=tkinter.Button(c), anchor=anchor) + self.assertEqual(c.itemcget(i, 'anchor'), anchor) + self.assertRaises(TclError, c.create_window, 10, 20, + window=button, anchor='spam') + + i = c.create_window(10, 20, window=tkinter.Button(c), + width=30, height=40) + self.assertEqual(int(c.itemcget(i, 'width')), 30) + self.assertEqual(int(c.itemcget(i, 'height')), 40) + self.assertRaises(TclError, c.create_window, 10, 20, + window=button, width='spam') + def test_coords(self): c = self.create() i = c.create_line(20, 30, 40, 50, 60, 10, tags='x') @@ -1008,6 +1301,391 @@ def test_moveto(self): self.assertEqual(x2_2 - x1_2, x2_3 - x1_3) self.assertEqual(y2_2 - y1_2, y2_3 - y1_3) + def test_create_items(self): + c = self.create() + image = tkinter.PhotoImage(master=self.root, width=10, height=10) + button = tkinter.Button(c, text='ok') + items = { + 'arc': c.create_arc(10, 20, 30, 40), + 'bitmap': c.create_bitmap(10, 20, bitmap='gray50'), + 'image': c.create_image(10, 20, image=image), + 'line': c.create_line(10, 20, 30, 40), + 'oval': c.create_oval(10, 20, 30, 40), + 'polygon': c.create_polygon(10, 20, 30, 40, 50, 20), + 'rectangle': c.create_rectangle(10, 20, 30, 40), + 'text': c.create_text(10, 20, text='hello'), + 'window': c.create_window(10, 20, window=button), + } + for itemtype, item in items.items(): + self.assertIsInstance(item, int) + self.assertEqual(c.type(item), itemtype) + # All items are reported by find_all in creation (stacking) order. + self.assertEqual(c.find_all(), tuple(sorted(items.values()))) + + # No coordinates at all is an IndexError; a bad number is a TclError. + self.assertRaises(IndexError, c.create_arc) + self.assertRaises(TclError, c.create_arc, 1, 2, 3) + self.assertRaises(TclError, c.create_oval, 1, 2) + + def test_type(self): + c = self.create() + rect = c.create_rectangle(10, 20, 30, 40) + self.assertEqual(c.type(rect), 'rectangle') + # An unmatched tag or id is not an error. + self.assertIsNone(c.type('nonexistent')) + self.assertIsNone(c.type(9999)) + self.assertRaises(TypeError, c.type) + self.assertRaises(TypeError, c.type, rect, 'extra') + + def test_bbox(self): + c = self.create() + rect = c.create_rectangle(10, 10, 30, 30) + bbox = c.bbox(rect) + self.assertIsInstance(bbox, tuple) + self.assertEqual(len(bbox), 4) + for v in bbox: + self.assertIsInstance(v, int) + # The bounding box encloses the item (with a small margin). + self.assertEqual(bbox, (9, 9, 31, 31)) + # bbox over several items is their union. + rect2 = c.create_rectangle(50, 50, 70, 70) + self.assertEqual(c.bbox(rect, rect2), (9, 9, 71, 71)) + # An unmatched tag has no bounding box. + self.assertIsNone(c.bbox('nonexistent')) + # At least one tag or id is required. + self.assertRaises(TclError, c.bbox) + + def test_canvasx_canvasy(self): + c = self.create(borderwidth=0, highlightthickness=0) + self.assertEqual(c.canvasx(0), 0.0) + self.assertEqual(c.canvasx(10), 10.0) + self.assertIsInstance(c.canvasx(10), float) + # gridspacing rounds to the nearest multiple. + self.assertEqual(c.canvasx(13, 5), 15.0) + self.assertEqual(c.canvasy(0), 0.0) + self.assertEqual(c.canvasy(10), 10.0) + self.assertRaises(TypeError, c.canvasx) + self.assertRaises(TypeError, c.canvasx, 0, 5, 1) + self.assertRaises(TypeError, c.canvasy) + self.assertRaises(TypeError, c.canvasy, 0, 5, 1) + self.assertRaises(TclError, c.canvasx, 'spam') + self.assertRaises(TclError, c.canvasy, 'spam') + + def test_move(self): + c = self.create() + rect = c.create_rectangle(10, 10, 30, 30) + c.move(rect, 5, 7) + self.assertEqual(c.coords(rect), [15.0, 17.0, 35.0, 37.0]) + c.move(rect, -5, -7) + self.assertEqual(c.coords(rect), [10.0, 10.0, 30.0, 30.0]) + # move() takes variable arguments; bad or missing values reach Tcl. + self.assertRaises(TclError, c.move, rect, 'spam', 0) + self.assertRaises(TclError, c.move, rect) + + def test_scale(self): + c = self.create() + rect = c.create_rectangle(10, 10, 30, 30) + c.scale(rect, 0, 0, 2, 2) + self.assertEqual(c.coords(rect), [20.0, 20.0, 60.0, 60.0]) + c.scale(rect, 0, 0, 0.5, 0.5) + self.assertEqual(c.coords(rect), [10.0, 10.0, 30.0, 30.0]) + self.assertRaises(TclError, c.scale, rect, 0, 0, 'spam', 2) + self.assertRaises(TclError, c.scale, rect, 0, 0) # missing factors + + def test_delete(self): + c = self.create() + r1 = c.create_rectangle(10, 10, 30, 30) + r2 = c.create_rectangle(50, 50, 70, 70) + r3 = c.create_rectangle(90, 90, 110, 110) + self.assertEqual(c.find_all(), (r1, r2, r3)) + c.delete(r2) + self.assertEqual(c.find_all(), (r1, r3)) + # Deleting a non-existent item is not an error. + c.delete(9999) + c.delete('all') + self.assertEqual(c.find_all(), ()) + + def test_find(self): + c = self.create() + r1 = c.create_rectangle(10, 10, 30, 30) + r2 = c.create_rectangle(50, 50, 70, 70) + r3 = c.create_rectangle(100, 100, 120, 120) + + self.assertEqual(c.find_all(), (r1, r2, r3)) + # find_above/find_below return the single adjacent item. + self.assertEqual(c.find_above(r1), (r2,)) + self.assertEqual(c.find_below(r3), (r2,)) + self.assertEqual(c.find_above(r3), ()) # nothing above the top item + self.assertEqual(c.find_withtag(r2), (r2,)) + self.assertEqual(c.find_closest(60, 60), (r2,)) + self.assertEqual(c.find_enclosed(0, 0, 80, 80), (r1, r2)) + self.assertEqual(c.find_overlapping(0, 0, 20, 20), (r1,)) + # An unmatched query returns an empty tuple. + self.assertEqual(c.find_withtag('nonexistent'), ()) + for result in (c.find_all(), c.find_withtag(r1)): + self.assertIsInstance(result, tuple) + + self.assertRaises(TclError, c.find_closest, 'spam', 0) + self.assertRaises(TclError, c.find_enclosed, 0, 0, 'spam', 0) + self.assertRaises(TclError, c.find_overlapping, 0, 0, 'spam', 0) + self.assertRaises(TypeError, c.find_withtag) + self.assertRaises(TypeError, c.find_withtag, r1, 'extra') + self.assertRaises(TypeError, c.find_above) + self.assertRaises(TypeError, c.find_above, r1, 'extra') + self.assertRaises(TypeError, c.find_below) + self.assertRaises(TypeError, c.find_closest) + self.assertRaises(TypeError, c.find_closest, 0, 0, 1, 2, 3) + self.assertRaises(TypeError, c.find_enclosed, 0, 0, 1) + self.assertRaises(TypeError, c.find_enclosed, 0, 0, 1, 2, 3) + + def test_addtag_gettags_dtag(self): + c = self.create() + r1 = c.create_rectangle(10, 10, 30, 30) + r2 = c.create_rectangle(50, 50, 70, 70) + + self.assertEqual(c.gettags(r1), ()) + c.addtag_withtag('spam', r1) + self.assertEqual(c.gettags(r1), ('spam',)) + self.assertEqual(c.find_withtag('spam'), (r1,)) + + c.addtag_all('all') + self.assertIn('all', c.gettags(r1)) + self.assertIn('all', c.gettags(r2)) + + c.addtag_above('above1', r1) + self.assertIn('above1', c.gettags(r2)) + c.addtag_below('below2', r2) + self.assertIn('below2', c.gettags(r1)) + + c.addtag_enclosed('enc', 0, 0, 40, 40) + self.assertEqual(c.find_withtag('enc'), (r1,)) + c.addtag_overlapping('ov', 0, 0, 20, 20) + self.assertEqual(c.find_withtag('ov'), (r1,)) + c.addtag_closest('close', 60, 60) + self.assertEqual(c.find_withtag('close'), (r2,)) + + # gettags of an unmatched tag is empty. + self.assertEqual(c.gettags('nonexistent'), ()) + + # dtag removes a tag from an item. + c.dtag(r1, 'spam') + self.assertNotIn('spam', c.gettags(r1)) + + self.assertRaises(TypeError, c.addtag_withtag, 'tag') + self.assertRaises(TypeError, c.addtag_withtag, 'tag', r1, 'extra') + self.assertRaises(TypeError, c.addtag_all) + self.assertRaises(TypeError, c.addtag_enclosed, 'tag', 0, 0, 1) + self.assertRaises(TypeError, c.addtag_enclosed, 'tag', 0, 0, 1, 2, 3) + self.assertRaises(TclError, c.addtag_closest, 'tag', 'spam', 0) + self.assertRaises(TclError, c.addtag_enclosed, 'tag', 0, 0, 'spam', 0) + self.assertRaises(TclError, c.gettags) # needs an item + + def test_itemconfigure(self): + c = self.create() + rect = c.create_rectangle(10, 10, 30, 30) + c.itemconfigure(rect, fill='red', width=3) + self.assertEqual(str(c.itemcget(rect, 'fill')), 'red') + self.assertEqual(float(c.itemcget(rect, 'width')), 3.0) + + # Querying all options returns a dict; a single one returns its spec. + cnf = c.itemconfigure(rect) + self.assertIsInstance(cnf, dict) + self.assertIn('fill', cnf) + self.assertEqual(c.itemconfigure(rect, 'fill')[-1], 'red') + + # itemconfig is an alias of itemconfigure. + self.assertEqual(c.itemconfig, c.itemconfigure) + + self.assertRaises(TclError, c.itemcget, rect, 'badoption') + self.assertRaises(TclError, c.itemconfigure, rect, badoption='x') + self.assertRaises(TypeError, c.itemcget, rect) + self.assertRaises(TypeError, c.itemcget, rect, 'fill', 'extra') + + def test_tag_raise_lower(self): + c = self.create() + r1 = c.create_rectangle(10, 10, 30, 30) + r2 = c.create_rectangle(50, 50, 70, 70) + r3 = c.create_rectangle(90, 90, 110, 110) + self.assertEqual(c.find_all(), (r1, r2, r3)) + + c.tag_raise(r1) + self.assertEqual(c.find_all(), (r2, r3, r1)) + c.tag_lower(r1) + self.assertEqual(c.find_all(), (r1, r2, r3)) + # Raise above / lower below a specific item. + c.tag_raise(r1, r2) + self.assertEqual(c.find_all(), (r2, r1, r3)) + c.tag_lower(r3, r2) + self.assertEqual(c.find_all(), (r3, r2, r1)) + + # lower/lift are aliases of tag_lower/tag_raise. + self.assertEqual(c.lower, c.tag_lower) + self.assertEqual(c.lift, c.tag_raise) + + # raise/lower need at least an item; an unmatched tag is not an error. + self.assertRaises(TclError, c.tag_raise) + self.assertRaises(TclError, c.tag_lower) + self.assertIsNone(c.tag_raise('nonexistent')) + + def test_text_item(self): + c = self.create() + item = c.create_text(10, 10, text='Hello') + self.assertEqual(c.index(item, 'end'), 5) + self.assertIsInstance(c.index(item, 'end'), int) + + c.insert(item, 'end', ' world') + self.assertEqual(c.itemcget(item, 'text'), 'Hello world') + self.assertEqual(c.index(item, 'end'), 11) + c.insert(item, 0, '>> ') + self.assertEqual(c.itemcget(item, 'text'), '>> Hello world') + + c.dchars(item, 0, 2) + self.assertEqual(c.itemcget(item, 'text'), 'Hello world') + c.icursor(item, 3) + + # index requires an indexable (text) item and a valid index. + self.assertRaises(TclError, c.index, item, 'badspec') + self.assertRaises(TclError, c.index, item) # missing index + self.assertRaises(TclError, c.dchars, item, 'badspec', 'end') + rect = c.create_rectangle(10, 10, 30, 30) + self.assertRaises(TclError, c.index, rect, 'end') + + def test_select(self): + c = self.create() + item = c.create_text(10, 10, text='Hello world') + self.assertIsNone(c.select_item()) + + c.select_from(item, 0) + c.select_to(item, 4) + self.assertEqual(int(c.select_item()), item) + c.select_adjust(item, 6) + + c.select_clear() + self.assertIsNone(c.select_item()) + self.assertRaises(TypeError, c.select_from, item) + self.assertRaises(TypeError, c.select_from, item, 0, 'extra') + # A bad index reaches Tcl; selection applies only to text items. + self.assertRaises(TclError, c.select_from, item, 'badspec') + rect = c.create_rectangle(10, 10, 30, 30) + self.assertRaises(TclError, c.select_from, rect, 0) + + def test_focus(self): + c = self.create() + item = c.create_text(10, 10, text='Hello') + # Only text items can take the focus. + c.focus(item) + self.assertEqual(int(c.focus()), item) + c.focus('') + self.assertIn(c.focus(), ('', None)) + + def test_scan(self): + c = self.create() + c.create_rectangle(10, 10, 300, 300) + c.scan_mark(0, 0) + c.scan_dragto(5, 5) # default gain=10 + c.scan_dragto(5, 5, 1) + self.assertRaises(TypeError, c.scan_mark) + self.assertRaises(TypeError, c.scan_mark, 0, 0, 0) + self.assertRaises(TclError, c.scan_mark, 'spam', 0) + + def test_postscript(self): + c = self.create() + c.create_rectangle(10, 10, 30, 30, fill='black') + ps = c.postscript() + self.assertIsInstance(ps, str) + self.assertStartsWith(ps, '%!PS-Adobe') + self.assertRaises(TclError, c.postscript, badoption='spam') + + def assertCommandExist(self, widget, funcid): + self.assertEqual( + widget.tk.splitlist(widget.tk.call('info', 'commands', funcid)), + (funcid,)) + + def assertCommandNotExist(self, widget, funcid): + self.assertEqual( + widget.tk.splitlist(widget.tk.call('info', 'commands', funcid)), + ()) + + def test_tag_bind(self): + c = self.create() + c.pack() + item = c.create_rectangle(20, 20, 100, 100, fill='red') + self.assertEqual(c.tag_bind(item), ()) + self.assertEqual(c.tag_bind(item, '<Button-1>'), '') + + events = [] + def test1(e): events.append('a') + def test2(e): events.append('b') + + funcid = c.tag_bind(item, '<Button-1>', test1) + self.assertEqual(c.tag_bind(item), ('<Button-1>',)) + script = c.tag_bind(item, '<Button-1>') + self.assertIn(funcid, script) + self.assertCommandExist(c, funcid) + + funcid2 = c.tag_bind(item, '<Button-1>', test2, add=True) + script = c.tag_bind(item, '<Button-1>') + self.assertIn(funcid, script) + self.assertIn(funcid2, script) + self.assertCommandExist(c, funcid) + self.assertCommandExist(c, funcid2) + + c.wait_visibility() + c.focus_force() + c.update() + c.event_generate('<Button-1>', x=50, y=50) + c.update() + self.assertEqual(events, ['a', 'b']) + + # Binding to a tag applies to all items carrying it. + c.addtag_withtag('spam', item) + events.clear() + c.tag_bind('spam', '<Button-3>', test1) + c.event_generate('<Button-3>', x=50, y=50) + c.update() + self.assertEqual(events, ['a']) + + def test_tag_unbind(self): + c = self.create() + c.pack() + item = c.create_rectangle(20, 20, 100, 100, fill='red') + + events = [] + def test1(e): events.append('a') + def test2(e): events.append('b') + + funcid = c.tag_bind(item, '<Button-1>', test1) + funcid2 = c.tag_bind(item, '<Button-1>', test2, add=True) + c.wait_visibility() + c.focus_force() + c.update() + c.event_generate('<Button-1>', x=50, y=50) + c.update() + self.assertEqual(events, ['a', 'b']) + + # Removing one function leaves the other in place. + c.tag_unbind(item, '<Button-1>', funcid) + script = c.tag_bind(item, '<Button-1>') + self.assertNotIn(funcid, script) + self.assertIn(funcid2, script) + self.assertCommandNotExist(c, funcid) + self.assertCommandExist(c, funcid2) + events.clear() + c.event_generate('<Button-1>', x=50, y=50) + c.update() + self.assertEqual(events, ['b']) + + # Without a funcid all bindings for the sequence are removed. + c.tag_unbind(item, '<Button-1>') + self.assertEqual(c.tag_bind(item, '<Button-1>'), '') + self.assertEqual(c.tag_bind(item), ()) + events.clear() + c.event_generate('<Button-1>', x=50, y=50) + c.update() + self.assertEqual(events, []) + + self.assertRaises(TypeError, c.tag_unbind, item) + @add_configure_tests(IntegerSizeTests, StandardOptionsTests) class ListboxTest(AbstractWidgetTest, unittest.TestCase): @@ -1021,11 +1699,10 @@ class ListboxTest(AbstractWidgetTest, unittest.TestCase): 'selectmode', 'setgrid', 'state', 'takefocus', 'width', 'xscrollcommand', 'yscrollcommand', ) - _rounds_pixels = (tk_version < (9, 0)) - if tk_version < (9, 0): + if tk_version < (8, 7): _clipped = {'highlightthickness'} else: - _clipped = { 'borderwidth', 'highlightthickness', 'selectborderwidth'} + _clipped = {'borderwidth', 'highlightthickness', 'selectborderwidth'} def create(self, **kwargs): return tkinter.Listbox(self.root, **kwargs) @@ -1151,6 +1828,123 @@ def test_get(self): self.assertRaises(TypeError, lb.get, 1, 2, 3) self.assertRaises(TclError, lb.get, 2.4) + def test_size(self): + lb = self.create() + self.assertEqual(lb.size(), 0) + lb.insert(0, *('el%d' % i for i in range(8))) + self.assertEqual(lb.size(), 8) + lb.delete(0, 2) + self.assertEqual(lb.size(), 5) + self.assertRaises(TypeError, lb.size, 0) + + def test_delete(self): + lb = self.create() + lb.insert(0, *('el%d' % i for i in range(8))) + lb.delete(0) + self.assertEqual(lb.get(0, 'end'), + ('el1', 'el2', 'el3', 'el4', 'el5', 'el6', 'el7')) + lb.delete(2, 4) + self.assertEqual(lb.get(0, 'end'), ('el1', 'el2', 'el6', 'el7')) + lb.delete(0, 'end') + self.assertEqual(lb.size(), 0) + self.assertRaises(TclError, lb.delete, 'noindex') + self.assertRaises(TypeError, lb.delete) + + def test_index(self): + lb = self.create() + lb.insert(0, *('el%d' % i for i in range(8))) + self.assertEqual(lb.index(3), 3) + self.assertEqual(lb.index('end'), 8) # the number of elements + lb.activate(4) + self.assertEqual(lb.index('active'), 4) + lb.selection_anchor(2) + self.assertEqual(lb.index('anchor'), 2) + self.assertRaisesRegex(TclError, 'bad listbox index "spam"', + lb.index, 'spam') + + def test_nearest(self): + lb = self.create() + lb.insert(0, *('el%d' % i for i in range(8))) + lb.pack() + lb.wait_visibility() + lb.update() + # Derive the line height from the first item, which is always + # displayed (bbox() returns None for items that are not). + x, y, w, h = lb.bbox(0) + self.assertEqual(lb.nearest(y + h // 2), 0) + self.assertEqual(lb.nearest(y + 3 * h + h // 2), 3) + self.assertRaises(TclError, lb.nearest, 'spam') + self.assertRaises(TypeError, lb.nearest) + + def test_see(self): + lb = self.create(height=5) + lb.insert(0, *('el%d' % i for i in range(20))) + lb.pack() + lb.update_idletasks() + lb.see('end') + lb.update_idletasks() + self.assertEqual(lb.yview()[1], 1.0) + lb.see(0) + lb.update_idletasks() + self.assertEqual(lb.yview()[0], 0.0) + self.assertRaises(TclError, lb.see, 'spam') + + def test_activate(self): + lb = self.create() + lb.insert(0, *('el%d' % i for i in range(8))) + lb.activate(3) + self.assertEqual(lb.index('active'), 3) + lb.activate('end') + self.assertEqual(lb.index('active'), 7) + self.assertRaises(TclError, lb.activate, 'spam') + self.assertRaises(TypeError, lb.activate) + + def test_selection(self): + lb = self.create() + lb.insert(0, *('el%d' % i for i in range(8))) + self.assertEqual(lb.curselection(), ()) + self.assertFalse(lb.selection_includes(0)) + + lb.selection_set(2, 4) + lb.selection_set(6) + self.assertEqual(lb.curselection(), (2, 3, 4, 6)) + self.assertTrue(lb.selection_includes(3)) + self.assertFalse(lb.selection_includes(5)) + + lb.selection_clear(3, 4) + self.assertEqual(lb.curselection(), (2, 6)) + + lb.selection_anchor(5) + self.assertEqual(lb.index('anchor'), 5) + + # select_* are aliases of the selection_* methods. + lb.select_clear(0, 'end') + self.assertEqual(lb.curselection(), ()) + lb.select_set(1) + self.assertTrue(lb.select_includes(1)) + lb.select_anchor(1) + self.assertEqual(lb.index('anchor'), 1) + + self.assertRaisesRegex(TclError, 'bad listbox index "spam"', + lb.selection_includes, 'spam') + + def test_selection_event(self): + # Keyboard navigation changes the selection and fires the + # <<ListboxSelect>> virtual event. + lb = self.create(selectmode='browse', exportselection=False) + lb.insert(0, *('el%d' % i for i in range(5))) + lb.pack() + self.require_mapped(lb) + events = [] + lb.bind('<<ListboxSelect>>', lambda e: events.append(lb.curselection())) + lb.focus_force() + lb.activate(0) + lb.event_generate('<Down>') + lb.event_generate('<Down>') + lb.update() + self.assertEqual(events, [(1,), (2,)]) + self.assertEqual(lb.curselection(), (2,)) + @add_configure_tests(PixelSizeTests, StandardOptionsTests) class ScaleTest(AbstractWidgetTest, unittest.TestCase): @@ -1163,7 +1957,6 @@ class ScaleTest(AbstractWidgetTest, unittest.TestCase): 'resolution', 'showvalue', 'sliderlength', 'sliderrelief', 'state', 'takefocus', 'tickinterval', 'to', 'troughcolor', 'variable', 'width', ) - _rounds_pixels = (tk_version < (9, 0)) _clipped = {'highlightthickness'} default_orient = 'vertical' @@ -1221,6 +2014,18 @@ def test_configure_to(self): self.checkFloatParam(widget, 'to', 300, 14.9, 15.1, -10, conv=float_round) + def test_identify(self): + widget = self.create() + widget.pack() + # Probe a point on the trough centreline (Scale.coords()) rather than + # a fixed pixel: (5, 5) lies outside the trough and always identifies + # as '', so it would not actually exercise identify(). + if wait_until_mapped(widget): + x, y = widget.coords() + self.assertIn(widget.identify(int(x), int(y)), + ('slider', 'trough1', 'trough2')) + self.assertRaises(TclError, widget.identify, 'a', 'b') + @add_configure_tests(PixelSizeTests, StandardOptionsTests) class ScrollbarTest(AbstractWidgetTest, unittest.TestCase): @@ -1233,14 +2038,13 @@ class ScrollbarTest(AbstractWidgetTest, unittest.TestCase): 'repeatdelay', 'repeatinterval', 'takefocus', 'troughcolor', 'width', ) - _rounds_pixels = True - if tk_version >= (9, 0): - _no_round = {'borderwidth', 'elementborderwidth', 'highlightthickness', - 'width'} - if tk_version < (9, 0): + if tk_version < (8, 7): _clipped = {'highlightthickness'} + elif tk_version < (9, 0): + _clipped = {'borderwidth', 'elementborderwidth', 'highlightthickness'} else: - _clipped = {'borderwidth', 'highlightthickness', 'width'} + _clipped = {'borderwidth', 'elementborderwidth', 'highlightthickness', 'width'} + _clipped_to_default = {'elementborderwidth'} _stringify = True default_orient = 'vertical' @@ -1249,9 +2053,7 @@ def create(self, **kwargs): def test_configure_elementborderwidth(self): widget = self.create() - self.checkPixelsParam(widget, 'elementborderwidth', 4.3, 5.6, '1m') - expected = self._default_pixels if tk_version >= (8, 7) else -2 - self.checkParam(widget, 'elementborderwidth', -2, expected=expected) + self.checkPixelsParam(widget, 'elementborderwidth', 4.3, 5.6, -2, '1m') def test_configure_orient(self): widget = self.create() @@ -1277,8 +2079,38 @@ def test_set(self): self.assertRaises(TypeError, sb.set, 0.6) self.assertRaises(TypeError, sb.set, 0.6, 0.7, 0.8) + def test_fraction(self): + sb = self.create() + sb.pack(fill='y', expand=True) + sb.update_idletasks() + self.assertIsInstance(sb.fraction(0, 0), float) + f = sb.fraction(0, 1000) + self.assertIsInstance(f, float) + self.assertGreaterEqual(f, 0.0) + self.assertLessEqual(f, 1.0) + self.assertRaises(TclError, sb.fraction, 'a', 'b') + self.assertRaises(TypeError, sb.fraction, 0) + + def test_delta(self): + sb = self.create() + sb.pack(fill='y', expand=True) + sb.update_idletasks() + self.assertIsInstance(sb.delta(0, 10), float) + self.assertRaises(TclError, sb.delta, 'a', 'b') + self.assertRaises(TypeError, sb.delta, 0) + + def test_identify(self): + sb = self.create() + sb.pack(fill='y', expand=True) + # Identifying the element under a point requires the widget to be + # mapped with a real size. + if wait_until_mapped(sb): + self.assertIn(sb.identify(5, 5), + ('arrow1', 'arrow2', 'slider', 'trough1', 'trough2')) + self.assertRaises(TclError, sb.identify, 'a', 'b') -@add_configure_tests(StandardOptionsTests) + +@add_configure_tests(PixelSizeTests, StandardOptionsTests) class PanedWindowTest(AbstractWidgetTest, unittest.TestCase): OPTIONS = ( 'background', 'borderwidth', 'cursor', @@ -1289,14 +2121,8 @@ class PanedWindowTest(AbstractWidgetTest, unittest.TestCase): 'sashcursor', 'sashpad', 'sashrelief', 'sashwidth', 'showhandle', 'width', ) - _rounds_pixels = True - if tk_version < (9, 0): - _no_round = {'handlesize', 'height', 'proxyborderwidth', 'sashwidth', - 'selectborderwidth', 'width'} - else: - _no_round = {'borderwidth', 'handlepad', 'handlesize', 'height', - 'proxyborderwidth', 'sashpad', 'sashwidth', - 'selectborderwidth', 'width'} + _no_round = {'handlesize', 'height', 'proxyborderwidth', 'sashwidth', + 'selectborderwidth', 'width'} _clipped = {} default_orient = 'horizontal' @@ -1309,13 +2135,7 @@ def test_configure_handlepad(self): def test_configure_handlesize(self): widget = self.create() - self.checkPixelsParam(widget, 'handlesize', 8, 9.4, 10.6, -3, '2m', - conv=False) - - def test_configure_height(self): - widget = self.create() - self.checkPixelsParam(widget, 'height', 100, 101.2, 102.6, -100, 0, '1i', - conv=False) + self.checkPixelsParam(widget, 'handlesize', 8, 9.4, 10.6, -3, '2m') def test_configure_opaqueresize(self): widget = self.create() @@ -1330,8 +2150,7 @@ def test_configure_proxybackground(self): def test_configure_proxyborderwidth(self): widget = self.create() self.checkPixelsParam(widget, 'proxyborderwidth', - 0, 1.3, 2.9, 6, -2, '10p', - conv=False) + 0, 1.3, 2.9, 6, -2, '10p') @requires_tk(8, 6, 5) def test_configure_proxyrelief(self): @@ -1353,18 +2172,12 @@ def test_configure_sashrelief(self): def test_configure_sashwidth(self): widget = self.create() - self.checkPixelsParam(widget, 'sashwidth', 10, 11.1, 15.6, -3, '1m', - conv=False) + self.checkPixelsParam(widget, 'sashwidth', 10, 11.1, 15.6, -3, '1m') def test_configure_showhandle(self): widget = self.create() self.checkBooleanParam(widget, 'showhandle') - def test_configure_width(self): - widget = self.create() - self.checkPixelsParam(widget, 'width', 402, 403.4, 404.6, -402, 0, '5i', - conv=False) - def create2(self): p = self.create() b = tkinter.Button(p) @@ -1373,6 +2186,77 @@ def create2(self): p.add(c) return p, b, c + def test_panes(self): + p, b, c = self.create2() + self.assertEqual([str(x) for x in p.panes()], [str(b), str(c)]) + + def test_remove(self): + p, b, c = self.create2() + p.remove(b) + self.assertEqual([str(x) for x in p.panes()], [str(c)]) + p.forget(c) # forget is an alias of remove. + self.assertEqual(p.panes(), ()) + + def test_sash(self): + p, b, c = self.create2() + p.configure(width=200, height=50) + p.pack() + p.update() + x, y = p.sash_coord(0) + self.assertIsInstance(x, int) + self.assertIsInstance(y, int) + p.sash_place(0, 120, 0) + p.update() + self.assertEqual(p.sash_coord(0)[0], 120) + p.sash_mark(0) # No exception. + self.assertRaises(TclError, p.sash_coord, 5) + + def test_proxy(self): + p, b, c = self.create2() + p.configure(width=200, height=50) + p.pack() + p.update() + p.proxy_place(100, 10) + p.update() + self.assertEqual(p.proxy_coord()[0], 100) + p.proxy_forget() + p.update() + + def test_identify(self): + p, b, c = self.create2() + p.configure(width=200, height=50) + p.pack() + # Locating the sash requires the widget to be mapped with a real + # size; the rest of the checks do not. + if wait_until_mapped(p): + x, y = p.sash_coord(0) + # A point over the sash reports the sash. + self.assertIn('sash', p.identify(x + 1, y + 5)) + # A point over a pane reports nothing. + self.assertFalse(p.identify(2, 2)) + self.assertRaises(TclError, p.identify, 'a', 'b') + + def test_add_options(self): + p = self.create() + b = tkinter.Button(p) + p.add(b, minsize=40, padx=3, sticky='ns') + self.assertEqual(p.panecget(b, 'minsize'), + 40 if self.wantobjects else '40') + self.assertEqual(p.panecget(b, 'padx'), + 3 if self.wantobjects else '3') + self.assertEqual(p.panecget(b, 'sticky'), 'ns') + self.assertRaisesRegex(TclError, 'unknown option "-spam"', + p.add, tkinter.Button(p), spam='x') + self.assertRaisesRegex(TclError, 'bad window path name "spam"', + p.add, 'spam') + + def test_paneconfigure_errors(self): + p, b, c = self.create2() + self.assertRaisesRegex(TclError, 'unknown option "-spam"', + p.paneconfigure, b, spam='x') + self.assertRaises(TclError, p.panecget, b, 'spam') + self.assertRaises(TclError, p.paneconfigure, 'spam') + def test_paneconfigure(self): p, b, c = self.create2() self.assertRaises(TypeError, p.paneconfigure) @@ -1536,6 +2420,192 @@ def test_entryconfigure_variable(self): m1.entryconfigure(1, variable=v2) self.assertEqual(str(m1.entrycget(1, 'variable')), str(v2)) + def test_add(self): + m = self.create(tearoff=False) + m.add_command(label='Command') + m.add_checkbutton(label='Checkbutton') + m.add_radiobutton(label='Radiobutton') + m.add_separator() + m.add_cascade(label='Cascade', menu=tkinter.Menu(m, tearoff=False)) + self.assertEqual(m.index('end'), 4) + self.assertEqual([m.type(i) for i in range(5)], + ['command', 'checkbutton', 'radiobutton', + 'separator', 'cascade']) + self.assertEqual(m.entrycget(0, 'label'), 'Command') + self.assertRaisesRegex(TclError, 'bad menu entry type "spam"', + m.add, 'spam') + + def test_insert(self): + m = self.create(tearoff=False) + m.add_command(label='A') + m.add_command(label='C') + m.insert_command(1, label='B') + m.insert_separator(0) + m.insert_checkbutton('end', label='D') + m.insert_radiobutton(0, label='top') + m.insert_cascade(2, label='sub', + menu=tkinter.Menu(m, tearoff=False)) + self.assertEqual( + [m.type(i) for i in range(m.index('end') + 1)], + ['radiobutton', 'separator', 'cascade', 'command', + 'command', 'command', 'checkbutton']) + self.assertEqual( + [m.entrycget(i, 'label') for i in (3, 4, 5)], + ['A', 'B', 'C']) + self.assertRaisesRegex(TclError, 'bad menu entry type "spam"', + m.insert, 0, 'spam') + self.assertRaisesRegex(TclError, 'bad menu entry index "spam"', + m.insert_command, 'spam', label='z') + + def test_delete(self): + m = self.create(tearoff=False) + commands = [] + for label in 'ABCDE': + m.add_command(label=label, + command=lambda label=label: commands.append(label)) + # The Tcl command for a deleted item is cleaned up. + funcid = str(m.entrycget(2, 'command')) + self.assertEqual( + m.tk.splitlist(m.tk.call('info', 'commands', funcid)), (funcid,)) + + m.delete(2) # Delete a single item ('C'). + self.assertEqual([m.entrycget(i, 'label') for i in range(4)], + ['A', 'B', 'D', 'E']) + self.assertEqual( + m.tk.splitlist(m.tk.call('info', 'commands', funcid)), ()) + + m.delete(1, 2) # Delete a range ('B' and 'D'). + self.assertEqual([m.entrycget(i, 'label') for i in range(2)], + ['A', 'E']) + self.assertRaises(TypeError, m.delete) + + def test_index(self): + m = self.create(tearoff=False) + self.assertIsNone(m.index('end')) + m.add_command(label='First') + m.add_command(label='Second') + self.assertEqual(m.index('end'), 1) + self.assertEqual(m.index('last'), 1) + self.assertEqual(m.index('Second'), 1) + self.assertEqual(m.index(0), 0) + # 'active' and 'none' map to None when no item is active. + self.assertIsNone(m.index('active')) + self.assertIsNone(m.index('none')) + self.assertRaisesRegex(TclError, 'bad menu entry index "spam"', + m.index, 'spam') + + def test_invoke(self): + m = self.create(tearoff=False) + commands = [] + m.add_command(label='Command', + command=lambda: commands.append('invoked')) + var = tkinter.IntVar(self.root) + m.add_checkbutton(label='Check', variable=var, + onvalue=1, offvalue=0) + rvar = tkinter.StringVar(self.root) + m.add_radiobutton(label='Radio', variable=rvar, value='on') + + m.invoke(0) + self.assertEqual(commands, ['invoked']) + m.invoke(1) + self.assertEqual(var.get(), 1) + m.invoke(1) + self.assertEqual(var.get(), 0) + m.invoke(2) + self.assertEqual(rvar.get(), 'on') + self.assertRaisesRegex(TclError, 'bad menu entry index "spam"', + m.invoke, 'spam') + + def test_xposition_yposition(self): + m = self.create(tearoff=False) + m.add_command(label='First') + m.add_command(label='Second') + m.update_idletasks() + self.assertIsInstance(m.xposition(0), int) + y0 = m.yposition(0) + y1 = m.yposition(1) + self.assertIsInstance(y0, int) + self.assertLess(y0, y1) + # An out-of-range index gives the position past the last item. + self.assertEqual(m.xposition('end'), m.xposition(1)) + self.assertRaisesRegex(TclError, 'bad menu entry index "spam"', + m.xposition, 'spam') + self.assertRaisesRegex(TclError, 'bad menu entry index "spam"', + m.yposition, 'spam') + + def test_post_unpost(self): + m = self.create(tearoff=False) + if m._windowingsystem != 'x11': + # Posting a menu is modal on Windows and uses a native, unmapped + # menu on Aqua, so it cannot be tested synchronously there. + self.skipTest('menu posting is not testable on this platform') + m.add_command(label='First') + m.add_command(label='Second') + self.assertFalse(m.winfo_ismapped()) + + m.post(0, 0) + m.update() + self.assertTrue(m.winfo_ismapped()) + m.unpost() + m.update() + self.assertFalse(m.winfo_ismapped()) + + m.tk_popup(0, 0) + m.update() + self.assertTrue(m.winfo_ismapped()) + m.unpost() + m.update() + self.assertFalse(m.winfo_ismapped()) + + def check_entry_option(self, m, index, option, value, expected=None): + if expected is None: + expected = value + m.entryconfigure(index, **{option: value}) + self.assertEqual(str(m.entrycget(index, option)), str(expected)) + self.assertEqual(str(m.entryconfigure(index, option)[4]), str(expected)) + + def test_entry_options(self): + m = self.create(tearoff=False) + m.add_command(label='Command') + self.check_entry_option(m, 0, 'accelerator', 'Ctrl+O') + self.check_entry_option(m, 0, 'underline', 2) + self.check_entry_option(m, 0, 'state', 'disabled') + self.check_entry_option(m, 0, 'background', 'red') + self.check_entry_option(m, 0, 'foreground', 'blue') + self.check_entry_option(m, 0, 'columnbreak', 1) + self.check_entry_option(m, 0, 'hidemargin', 1) + + m.add_checkbutton(label='Checkbutton') + self.check_entry_option(m, 1, 'onvalue', 'Y') + self.check_entry_option(m, 1, 'offvalue', 'N') + self.check_entry_option(m, 1, 'indicatoron', 0) + + m.add_radiobutton(label='Radiobutton') + self.check_entry_option(m, 2, 'value', 'V') + self.check_entry_option(m, 2, 'selectcolor', 'green') + + def test_entry_options_invalid(self): + m = self.create(tearoff=False) + m.add_command(label='Command') + self.assertRaisesRegex(TclError, 'unknown option "-spam"', + m.entrycget, 0, 'spam') + self.assertRaisesRegex(TclError, 'unknown option "-spam"', + m.entryconfigure, 0, spam='x') + self.assertRaisesRegex(TclError, 'bad state "spam"', + m.entryconfigure, 0, state='spam') + # Tk < 9 reports "expected integer but got ...", while Tk 9, where + # underline accepts an index, reports "bad index ...". + self.assertRaisesRegex(TclError, + r'(expected integer but got|bad index) "spam"', + m.entryconfigure, 0, underline='spam') + self.assertRaisesRegex(TclError, 'unknown color name "spam"', + m.entryconfigure, 0, background='spam') + self.assertRaisesRegex(TclError, 'expected boolean value but got "spam"', + m.entryconfigure, 0, columnbreak='spam') + # onvalue applies only to checkbutton and radiobutton entries. + self.assertRaisesRegex(TclError, 'unknown option "-onvalue"', + m.entrycget, 0, 'onvalue') + @add_configure_tests(PixelSizeTests, StandardOptionsTests) class MessageTest(AbstractWidgetTest, unittest.TestCase): @@ -1546,12 +2616,12 @@ class MessageTest(AbstractWidgetTest, unittest.TestCase): 'justify', 'padx', 'pady', 'relief', 'takefocus', 'text', 'textvariable', 'width', ) - _rounds_pixels = (tk_version < (9, 0)) _no_round = {'padx', 'pady'} - if tk_version < (9, 0): + if tk_version < (8, 7): _clipped = {'highlightthickness'} else: - _clipped = {'borderwidth', 'highlightthickness', 'padx', 'pady'} + _clipped = {'borderwidth', 'highlightthickness', 'padx', 'pady', 'width'} + _clipped_to_default = {'padx', 'pady'} def create(self, **kwargs): return tkinter.Message(self.root, **kwargs) @@ -1560,24 +2630,6 @@ def test_configure_aspect(self): widget = self.create() self.checkIntegerParam(widget, 'aspect', 250, 0, -300) - def test_configure_padx(self): - widget = self.create() - self.checkPixelsParam(widget, 'padx', 3, 4.4, 5.6, '12m') - expected = -2 if tk_version < (9, 0) else self._default_pixels - self.checkParam(widget, 'padx', -2, expected=expected) - - def test_configure_pady(self): - widget = self.create() - self.checkPixelsParam(widget, 'pady', 3, 4.4, 5.6, '12m') - expected = -2 if tk_version < (9, 0) else self._default_pixels - self.checkParam(widget, 'pady', -2, expected=expected) - - def test_configure_width(self): - widget = self.create() - self.checkPixelsParam(widget, 'width', 402, 403.4, 404.6, 0, '5i') - expected = 0 if tk_version >= (8, 7) else -402 - self.checkParam(widget, 'width', -402, expected=expected) - class DefaultRootTest(AbstractDefaultRootTest, unittest.TestCase): diff --git a/Lib/test/test_tkinter/widget_tests.py b/Lib/test/test_tkinter/widget_tests.py index f518925e994e908..014906cba2902c3 100644 --- a/Lib/test/test_tkinter/widget_tests.py +++ b/Lib/test/test_tkinter/widget_tests.py @@ -8,15 +8,26 @@ _sentinel = object() +# Abbreviated option names accepted by Tk in addition to the full names. +_OPTION_ALIASES = { + 'bd': 'borderwidth', + 'bg': 'background', + 'bgimg': 'backgroundimage', + 'fg': 'foreground', + 'invcmd': 'invalidcommand', + 'vcmd': 'validatecommand', +} + # Options which accept all values allowed by Tk_GetPixels # borderwidth = bd class AbstractWidgetTest(AbstractTkTest): - _default_pixels = '' # Value for unset pixel options. - _rounds_pixels = True # True if some pixel options are rounded. - _no_round = {} # Pixel options which are not rounded nonetheless + _default_pixels = '' if tk_version >= (9, 0) else -1 # Value for unset pixel options. + _rounds_pixels = (tk_version < (9, 0)) # True if some pixel options are rounded. + _no_round = set() # Pixel options which are not rounded nonetheless _stringify = False # Whether to convert tuples to strings _allow_empty_justify = False + _clipped_to_default = set() @property def scaling(self): @@ -43,9 +54,12 @@ def checkParam(self, widget, name, value, *, expected=_sentinel, widget[name] = value if expected is _sentinel: expected = value - if name in self._clipped: - if not isinstance(expected, str): - expected = max(expected, 0) + if name in self._clipped: + if not isinstance(expected, str) and expected < 0: + if tk_version >= (8, 7) and name in self._clipped_to_default: + expected = self._default_pixels + else: + expected = 0 if conv: expected = conv(expected) if self._stringify or not self.wantobjects: @@ -143,10 +157,10 @@ def checkEnumParam(self, widget, name, *values, self.checkInvalidParam(widget, name, 'spam', errmsg=errmsg) def checkPixelsParam(self, widget, name, *values, conv=None, **kwargs): - if not self._rounds_pixels or name in self._no_round: - conv = False - elif conv != str: - conv = round + if conv is None: + if self._rounds_pixels and name not in self._no_round: + conv = round + alow_neg = tk_version < (9, 1) for value in values: expected = _sentinel conv1 = conv @@ -156,6 +170,9 @@ def checkPixelsParam(self, widget, name, *values, conv=None, **kwargs): if conv1 and conv1 is not str: expected = pixels_conv(value) * self.scaling conv1 = round + elif not alow_neg and isinstance(value, (int, float)) and value < 0: + self.checkInvalidParam(widget, name, value) + continue self.checkParam(widget, name, value, expected=expected, conv=conv1, **kwargs) errmsg = '(bad|expected) screen distance ((or "" )?but got )?"{}"' @@ -177,7 +194,7 @@ def checkReliefParam(self, widget, name, *, allow_empty=False): def checkImageParam(self, widget, name): image = tkinter.PhotoImage(master=self.root, name='image1') self.checkParam(widget, name, image, conv=str) - if tk_version < (9, 0): + if tk_version < (8, 7): errmsg = 'image "spam" doesn\'t exist' else: errmsg = 'image "spam" does not exist' @@ -207,23 +224,44 @@ def test_keys(self): widget[k] # Test if OPTIONS contains all keys if test.support.verbose: - aliases = { - 'bd': 'borderwidth', - 'bg': 'background', - 'bgimg': 'backgroundimage', - 'fg': 'foreground', - 'invcmd': 'invalidcommand', - 'vcmd': 'validatecommand', - } keys = set(keys) expected = set(self.OPTIONS) for k in sorted(keys - expected): - if not (k in aliases and - aliases[k] in keys and - aliases[k] in expected): + if not (k in _OPTION_ALIASES and + _OPTION_ALIASES[k] in keys and + _OPTION_ALIASES[k] in expected): print('%s.OPTIONS doesn\'t contain "%s"' % (self.__class__.__name__, k)) + def test_options_in_docstring(self): + # Every option in OPTIONS must be listed in the docstring of the + # __init__ method (see gh-78335). Options reported by keys() but + # missing from the docstring are only printed in verbose mode, as + # some of them depend on the Tk version. + widget = self.create() + doc = type(widget).__init__.__doc__ + if doc is None: + self.skipTest('docstrings are not available (run with -OO)') + # Look at the option list only, not the leading description. + start = doc.find('Valid option names') + if start < 0: + start = doc.find('OPTIONS') + if start < 0: + self.skipTest('the __init__ docstring does not list options') + documented = set(re.findall(r'[a-z][a-z0-9]+', doc[start:])) + def is_documented(option): + return (option in documented or + _OPTION_ALIASES.get(option) in documented) + missing = sorted(o for o in self.OPTIONS if not is_documented(o)) + self.assertEqual(missing, [], + '%s options missing from the __init__ docstring: %s' + % (type(widget).__name__, missing)) + if test.support.verbose: + for key in sorted(set(widget.keys())): + if not is_documented(key): + print('%s.__init__ docstring doesn\'t contain "%s"' + % (type(widget).__name__, key)) + class PixelOptionsTests: """Standard options that accept all formats acceptable to Tk_GetPixels. @@ -246,8 +284,8 @@ def test_configure_activeborderwidth(self): def test_configure_borderwidth(self): widget = self.create() self.checkPixelsParam(widget, 'borderwidth', - 0, 1.3, 2.6, 6, '10p') - self.checkParam(widget, 'borderwidth', -2) + 0, 1.3, 2.6, 6, -2, '10p') + if 'bd' in self.OPTIONS: self.checkPixelsParam(widget, 'bd', 0, 1.3, 2.6, 6, '10p') self.checkParam(widget, 'bd', -2, expected=expected) @@ -255,14 +293,11 @@ def test_configure_borderwidth(self): def test_configure_highlightthickness(self): widget = self.create() self.checkPixelsParam(widget, 'highlightthickness', - 0, 1.3, 2.6, 6, '10p') - self.checkParam(widget, 'highlightthickness', -2) + 0, 1.3, 2.6, 6, -2, '10p') def test_configure_insertborderwidth(self): widget = self.create() - self.checkPixelsParam(widget, 'insertborderwidth', - 0, 1.3, 2.6, 6, '10p') - self.checkParam(widget, 'insertborderwidth', -2) + self.checkPixelsParam(widget, 'insertborderwidth', 0, 1.3, 2.6, 6, -2, '10p') def test_configure_insertwidth(self): widget = self.create() @@ -270,18 +305,17 @@ def test_configure_insertwidth(self): def test_configure_padx(self): widget = self.create() - self.checkPixelsParam(widget, 'padx', 3, 4.4, 5.6, '12m') - self.checkParam(widget, 'padx', -2) + self.checkPixelsParam(widget, 'padx', 3, 4.4, 5.6, -2, '12m') def test_configure_pady(self): widget = self.create() - self.checkPixelsParam(widget, 'pady', 3, 4.4, 5.6, '12m') - self.checkParam(widget, 'pady', -2) + self.checkPixelsParam(widget, 'pady', 3, 4.4, 5.6, -2, '12m') def test_configure_selectborderwidth(self): widget = self.create() self.checkPixelsParam(widget, 'selectborderwidth', 1.3, 2.6, -2, '10p') + class StandardOptionsTests(PixelOptionsTests): STANDARD_OPTIONS = ( 'activebackground', 'activeforeground', diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 2d41a5e5ac06975..ca67e381958757d 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -1,6 +1,8 @@ import contextlib +import itertools import os import re +import string import tempfile import token import tokenize @@ -1214,6 +1216,23 @@ def test_multiline_non_ascii_fstring_with_expr(self): FSTRING_END "\'\'\'" (3, 1) (3, 4) """) + # gh-139516, the '\n' is explicit to ensure no trailing whitespace which would invalidate the test + self.check_tokenize('''f"{f(a=lambda: 'à'\n)}"''', """\ + FSTRING_START \'f"\' (1, 0) (1, 2) + OP '{' (1, 2) (1, 3) + NAME 'f' (1, 3) (1, 4) + OP '(' (1, 4) (1, 5) + NAME 'a' (1, 5) (1, 6) + OP '=' (1, 6) (1, 7) + NAME 'lambda' (1, 7) (1, 13) + OP ':' (1, 13) (1, 14) + STRING "\'à\'" (1, 15) (1, 18) + NL '\\n' (1, 18) (1, 19) + OP ')' (2, 0) (2, 1) + OP '}' (2, 1) (2, 2) + FSTRING_END \'"\' (2, 2) (2, 3) + """) + class GenerateTokensTest(TokenizeTest): def check_tokenize(self, s, expected): # Format the tokens in s in a table format. @@ -1344,7 +1363,8 @@ def readline(): def test_no_bom_no_encoding_cookie(self): lines = ( - b'# something\n', + b'#!/home/\xc3\xa4/bin/python\n', + b'# something \xe2\x82\xac\n', b'print(something)\n', b'do_something(else)\n' ) @@ -1352,16 +1372,54 @@ def test_no_bom_no_encoding_cookie(self): self.assertEqual(encoding, 'utf-8') self.assertEqual(consumed_lines, list(lines[:2])) + def test_no_bom_no_encoding_cookie_first_line_error(self): + lines = ( + b'#!/home/\xa4/bin/python\n\n', + b'print(something)\n', + b'do_something(else)\n' + ) + with self.assertRaises(SyntaxError): + tokenize.detect_encoding(self.get_readline(lines)) + + def test_no_bom_no_encoding_cookie_second_line_error(self): + lines = ( + b'#!/usr/bin/python\n', + b'# something \xe2\n', + b'print(something)\n', + b'do_something(else)\n' + ) + with self.assertRaises(SyntaxError): + tokenize.detect_encoding(self.get_readline(lines)) + def test_bom_no_cookie(self): lines = ( - b'\xef\xbb\xbf# something\n', + b'\xef\xbb\xbf#!/home/\xc3\xa4/bin/python\n', b'print(something)\n', b'do_something(else)\n' ) encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) self.assertEqual(encoding, 'utf-8-sig') self.assertEqual(consumed_lines, - [b'# something\n', b'print(something)\n']) + [b'#!/home/\xc3\xa4/bin/python\n', b'print(something)\n']) + + def test_bom_no_cookie_first_line_error(self): + lines = ( + b'\xef\xbb\xbf#!/home/\xa4/bin/python\n', + b'print(something)\n', + b'do_something(else)\n' + ) + with self.assertRaises(SyntaxError): + tokenize.detect_encoding(self.get_readline(lines)) + + def test_bom_no_cookie_second_line_error(self): + lines = ( + b'\xef\xbb\xbf#!/usr/bin/python\n', + b'# something \xe2\n', + b'print(something)\n', + b'do_something(else)\n' + ) + with self.assertRaises(SyntaxError): + tokenize.detect_encoding(self.get_readline(lines)) def test_cookie_first_line_no_bom(self): lines = ( @@ -1437,16 +1495,60 @@ def test_cookie_second_line_noncommented_first_line(self): expected = [b"print('\xc2\xa3')\n"] self.assertEqual(consumed_lines, expected) - def test_cookie_second_line_commented_first_line(self): + def test_first_non_utf8_coding_line(self): lines = ( - b"#print('\xc2\xa3')\n", - b'# vim: set fileencoding=iso8859-15 :\n', - b"print('\xe2\x82\xac')\n" + b'#coding:iso-8859-15 \xa4\n', + b'print(something)\n' ) encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) - self.assertEqual(encoding, 'iso8859-15') - expected = [b"#print('\xc2\xa3')\n", b'# vim: set fileencoding=iso8859-15 :\n'] - self.assertEqual(consumed_lines, expected) + self.assertEqual(encoding, 'iso-8859-15') + self.assertEqual(consumed_lines, list(lines[:1])) + + def test_first_utf8_coding_line_error(self): + lines = ( + b'#coding:ascii \xc3\xa4\n', + b'print(something)\n' + ) + with self.assertRaises(SyntaxError): + tokenize.detect_encoding(self.get_readline(lines)) + + def test_second_non_utf8_coding_line(self): + lines = ( + b'#!/usr/bin/python\n', + b'#coding:iso-8859-15 \xa4\n', + b'print(something)\n' + ) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) + self.assertEqual(encoding, 'iso-8859-15') + self.assertEqual(consumed_lines, list(lines[:2])) + + def test_second_utf8_coding_line_error(self): + lines = ( + b'#!/usr/bin/python\n', + b'#coding:ascii \xc3\xa4\n', + b'print(something)\n' + ) + with self.assertRaises(SyntaxError): + tokenize.detect_encoding(self.get_readline(lines)) + + def test_non_utf8_shebang(self): + lines = ( + b'#!/home/\xa4/bin/python\n', + b'#coding:iso-8859-15\n', + b'print(something)\n' + ) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) + self.assertEqual(encoding, 'iso-8859-15') + self.assertEqual(consumed_lines, list(lines[:2])) + + def test_utf8_shebang_error(self): + lines = ( + b'#!/home/\xc3\xa4/bin/python\n', + b'#coding:ascii\n', + b'print(something)\n' + ) + with self.assertRaises(SyntaxError): + tokenize.detect_encoding(self.get_readline(lines)) def test_cookie_second_line_empty_first_line(self): lines = ( @@ -1459,6 +1561,70 @@ def test_cookie_second_line_empty_first_line(self): expected = [b'\n', b'# vim: set fileencoding=iso8859-15 :\n'] self.assertEqual(consumed_lines, expected) + def test_cookie_third_line(self): + lines = ( + b'#!/home/\xc3\xa4/bin/python\n', + b'# something\n', + b'# vim: set fileencoding=ascii :\n', + b'print(something)\n', + b'do_something(else)\n' + ) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) + self.assertEqual(encoding, 'utf-8') + self.assertEqual(consumed_lines, list(lines[:2])) + + def test_double_coding_line(self): + # If the first line matches the second line is ignored. + lines = ( + b'#coding:iso8859-15\n', + b'#coding:latin1\n', + b'print(something)\n' + ) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) + self.assertEqual(encoding, 'iso8859-15') + self.assertEqual(consumed_lines, list(lines[:1])) + + def test_double_coding_same_line(self): + lines = ( + b'#coding:iso8859-15 coding:latin1\n', + b'print(something)\n' + ) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) + self.assertEqual(encoding, 'iso8859-15') + self.assertEqual(consumed_lines, list(lines[:1])) + + def test_double_coding_utf8(self): + lines = ( + b'#coding:utf-8\n', + b'#coding:latin1\n', + b'print(something)\n' + ) + encoding, consumed_lines = tokenize.detect_encoding(self.get_readline(lines)) + self.assertEqual(encoding, 'utf-8') + self.assertEqual(consumed_lines, list(lines[:1])) + + def test_nul_in_first_coding_line(self): + lines = ( + b'#coding:iso8859-15\x00\n', + b'\n', + b'\n', + b'print(something)\n' + ) + with self.assertRaisesRegex(SyntaxError, + "source code cannot contain null bytes"): + tokenize.detect_encoding(self.get_readline(lines)) + + def test_nul_in_second_coding_line(self): + lines = ( + b'#!/usr/bin/python\n', + b'#coding:iso8859-15\x00\n', + b'\n', + b'print(something)\n' + ) + with self.assertRaisesRegex(SyntaxError, + "source code cannot contain null bytes"): + tokenize.detect_encoding(self.get_readline(lines)) + def test_latin1_normalization(self): # See get_normal_name() in Parser/tokenizer/helpers.c. encodings = ("latin-1", "iso-8859-1", "iso-latin-1", "latin-1-unix", @@ -1483,7 +1649,6 @@ def test_syntaxerror_latin1(self): readline = self.get_readline(lines) self.assertRaises(SyntaxError, tokenize.detect_encoding, readline) - def test_utf8_normalization(self): # See get_normal_name() in Parser/tokenizer/helpers.c. encodings = ("utf-8", "utf-8-mac", "utf-8-unix") @@ -1975,6 +2140,10 @@ def test_roundtrip(self): for case in cases: self.check_roundtrip(case) + self.check_roundtrip(r"t'{ {}}'") + self.check_roundtrip(r"t'{f'{ {}}'}{ {}}'") + self.check_roundtrip(r"f'{t'{ {}}'}{ {}}'") + def test_continuation(self): # Balancing continuation @@ -3014,6 +3183,7 @@ def get_tokens(string): f'__{ x:d }__'""", + " a\n\x00", ]: with self.subTest(case=case): self.assertRaises(tokenize.TokenError, get_tokens, case) @@ -3234,5 +3404,77 @@ def test_exact_flag(self): self.check_output(source, expect, flag) +class StringPrefixTest(unittest.TestCase): + @staticmethod + def determine_valid_prefixes(): + # Try all lengths until we find a length that has zero valid + # prefixes. This will miss the case where for example there + # are no valid 3 character prefixes, but there are valid 4 + # character prefixes. That seems unlikely. + + single_char_valid_prefixes = set() + + # Find all of the single character string prefixes. Just get + # the lowercase version, we'll deal with combinations of upper + # and lower case later. I'm using this logic just in case + # some uppercase-only prefix is added. + for letter in itertools.chain(string.ascii_lowercase, string.ascii_uppercase): + try: + eval(f'{letter}""') + single_char_valid_prefixes.add(letter.lower()) + except SyntaxError: + pass + + # This logic assumes that all combinations of valid prefixes only use + # the characters that are valid single character prefixes. That seems + # like a valid assumption, but if it ever changes this will need + # adjusting. + valid_prefixes = set() + for length in itertools.count(): + num_at_this_length = 0 + for prefix in ( + "".join(l) + for l in itertools.combinations(single_char_valid_prefixes, length) + ): + for t in itertools.permutations(prefix): + for u in itertools.product(*[(c, c.upper()) for c in t]): + p = "".join(u) + if p == "not": + # 'not' can never be a string prefix, + # because it's a valid expression: not "" + continue + try: + eval(f'{p}""') + + # No syntax error, so p is a valid string + # prefix. + + valid_prefixes.add(p) + num_at_this_length += 1 + except SyntaxError: + pass + if num_at_this_length == 0: + return valid_prefixes + + + def test_prefixes(self): + # Get the list of defined string prefixes. I don't see an + # obvious documented way of doing this, but probably the best + # thing is to split apart tokenize.StringPrefix. + + # Make sure StringPrefix begins and ends in parens. We're + # assuming it's of the form "(a|b|ab)", if a, b, and cd are + # valid string prefixes. + self.assertEqual(tokenize.StringPrefix[0], '(') + self.assertEqual(tokenize.StringPrefix[-1], ')') + + # Then split apart everything else by '|'. + defined_prefixes = set(tokenize.StringPrefix[1:-1].split('|')) + + # Now compute the actual allowed string prefixes and compare + # to what is defined in the tokenize module. + self.assertEqual(defined_prefixes, self.determine_valid_prefixes()) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tomllib/test_misc.py b/Lib/test/test_tomllib/test_misc.py index 59116afa1f36ad1..85526ed80155423 100644 --- a/Lib/test/test_tomllib/test_misc.py +++ b/Lib/test/test_tomllib/test_misc.py @@ -93,6 +93,7 @@ def test_deepcopy(self): } self.assertEqual(obj_copy, expected_obj) + @support.skip_if_unlimited_stack_size def test_inline_array_recursion_limit(self): with support.infinite_recursion(max_depth=100): available = support.get_recursion_available() @@ -104,6 +105,7 @@ def test_inline_array_recursion_limit(self): recursive_array_toml = "arr = " + nest_count * "[" + nest_count * "]" tomllib.loads(recursive_array_toml) + @support.skip_if_unlimited_stack_size def test_inline_table_recursion_limit(self): with support.infinite_recursion(max_depth=100): available = support.get_recursion_available() @@ -115,6 +117,19 @@ def test_inline_table_recursion_limit(self): recursive_table_toml = nest_count * "key = {" + nest_count * "}" tomllib.loads(recursive_table_toml) + def test_key_recursion_limit(self): + nest_count = tomllib._parser.MAX_KEY_PARTS - 2 + nested_key_toml = "a." * nest_count + "a = 1" + tomllib.loads(nested_key_toml) + + nest_count = tomllib._parser.MAX_KEY_PARTS + 2 + nested_key_toml = "a." * nest_count + "a = 1" + with self.assertRaisesRegex( + RecursionError, + r"TOML key has more than the allowed [0-9]+ parts", + ): + tomllib.loads(nested_key_toml) + def test_types_import(self): """Test that `_types` module runs. diff --git a/Lib/test/test_tools/test_compute_changes.py b/Lib/test/test_tools/test_compute_changes.py new file mode 100644 index 000000000000000..b20ff975fc28346 --- /dev/null +++ b/Lib/test/test_tools/test_compute_changes.py @@ -0,0 +1,144 @@ +"""Tests to cover the Tools/build/compute-changes.py script.""" + +import importlib +import os +import unittest +from pathlib import Path +from unittest.mock import patch + +from test.test_tools import skip_if_missing, imports_under_tool + +skip_if_missing("build") + +with patch.dict(os.environ, {"GITHUB_DEFAULT_BRANCH": "main"}): + with imports_under_tool("build"): + compute_changes = importlib.import_module("compute-changes") + +process_changed_files = compute_changes.process_changed_files +Outputs = compute_changes.Outputs +ANDROID_DIRS = compute_changes.ANDROID_DIRS +IOS_DIRS = compute_changes.IOS_DIRS +MACOS_DIRS = compute_changes.MACOS_DIRS +WASI_DIRS = compute_changes.WASI_DIRS +RUN_TESTS_IGNORE = compute_changes.RUN_TESTS_IGNORE +UNIX_BUILD_SYSTEM_FILE_NAMES = compute_changes.UNIX_BUILD_SYSTEM_FILE_NAMES +LIBRARY_FUZZER_PATHS = compute_changes.LIBRARY_FUZZER_PATHS + + +class TestProcessChangedFiles(unittest.TestCase): + + def test_windows(self): + f = {Path(".github/workflows/reusable-windows.yml")} + result = process_changed_files(f) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_windows_tests) + + def test_docs(self): + for f in ( + ".github/workflows/reusable-docs.yml", + "Doc/library/datetime.rst", + "Doc/Makefile", + ): + with self.subTest(f=f): + result = process_changed_files({Path(f)}) + self.assertTrue(result.run_docs) + self.assertFalse(result.run_tests) + + def test_ci_fuzz_stdlib(self): + for p in LIBRARY_FUZZER_PATHS: + with self.subTest(p=p): + if p.is_dir(): + f = p / "file" + elif p.is_file(): + f = p + else: + continue + result = process_changed_files({f}) + self.assertTrue(result.run_ci_fuzz_stdlib) + + def test_android(self): + for d in ANDROID_DIRS: + with self.subTest(d=d): + result = process_changed_files({Path(d) / "file"}) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_android) + self.assertFalse(result.run_windows_tests) + + def test_ios(self): + for d in IOS_DIRS: + with self.subTest(d=d): + result = process_changed_files({Path(d) / "file"}) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_ios) + self.assertFalse(result.run_windows_tests) + + def test_macos(self): + f = {Path(".github/workflows/reusable-macos.yml")} + result = process_changed_files(f) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_macos) + + for d in MACOS_DIRS: + with self.subTest(d=d): + result = process_changed_files({Path(d) / "file"}) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_macos) + self.assertFalse(result.run_windows_tests) + + def test_wasi(self): + f = {Path(".github/workflows/reusable-wasi.yml")} + result = process_changed_files(f) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_wasi) + + for d in WASI_DIRS: + with self.subTest(d=d): + result = process_changed_files({d / "file"}) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_wasi) + self.assertFalse(result.run_windows_tests) + + def test_unix(self): + for f in UNIX_BUILD_SYSTEM_FILE_NAMES: + with self.subTest(f=f): + result = process_changed_files({f}) + self.assertTrue(result.run_tests) + self.assertFalse(result.run_windows_tests) + + def test_msi(self): + for f in ( + ".github/workflows/reusable-windows-msi.yml", + "Tools/msi/build.bat", + ): + with self.subTest(f=f): + result = process_changed_files({Path(f)}) + self.assertTrue(result.run_windows_msi) + + def test_all_run(self): + for f in ( + ".github/workflows/some-new-workflow.yml", + ".github/workflows/build.yml", + ): + with self.subTest(f=f): + result = process_changed_files({Path(f)}) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_android) + self.assertTrue(result.run_ios) + self.assertTrue(result.run_macos) + self.assertTrue(result.run_ubuntu) + self.assertTrue(result.run_wasi) + + def test_all_ignored(self): + for f in RUN_TESTS_IGNORE: + with self.subTest(f=f): + self.assertEqual(process_changed_files({Path(f)}), Outputs()) + + def test_wasi_and_android(self): + f = {Path(".github/workflows/reusable-wasi.yml"), Path("Android/file")} + result = process_changed_files(f) + self.assertTrue(result.run_tests) + self.assertTrue(result.run_wasi) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_tools/test_makefile.py b/Lib/test/test_tools/test_makefile.py index 4c7588d4d93fc38..31a516067394e1e 100644 --- a/Lib/test/test_tools/test_makefile.py +++ b/Lib/test/test_tools/test_makefile.py @@ -48,15 +48,18 @@ def test_makefile_test_folders(self): if dirname == '__pycache__' or dirname.startswith('.'): dirs.clear() # do not process subfolders continue - # Skip empty dirs: + + # Skip empty dirs (ignoring hidden files and __pycache__): + files = [ + filename for filename in files + if not filename.startswith('.') + ] + dirs = [ + dirname for dirname in dirs + if not dirname.startswith('.') and dirname != "__pycache__" + ] if not dirs and not files: continue - # Skip dirs with hidden-only files: - if files and all( - filename.startswith('.') or filename == '__pycache__' - for filename in files - ): - continue relpath = os.path.relpath(dirpath, support.STDLIB_DIR) with self.subTest(relpath=relpath): diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index b9be87f357ffdd5..db6063b8650f421 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -18,8 +18,8 @@ from test.support import (Error, captured_output, cpython_only, ALWAYS_EQ, requires_debug_ranges, has_no_debug_ranges, requires_subprocess) -from test.support.os_helper import TESTFN, unlink -from test.support.script_helper import assert_python_ok, assert_python_failure +from test.support.os_helper import TESTFN, temp_dir, unlink +from test.support.script_helper import assert_python_ok, assert_python_failure, make_script from test.support.import_helper import forget from test.support import force_not_colorized, force_not_colorized_test_class @@ -504,6 +504,33 @@ def __del__(self): b'ZeroDivisionError: division by zero'] self.assertEqual(stderr.splitlines(), expected) + @cpython_only + def test_lost_io_open(self): + # GH-142737: Display the traceback even if io.open is lost + crasher = textwrap.dedent("""\ + import io + import traceback + # Trigger fallback mode + traceback._print_exception_bltin = None + del io.open + raise RuntimeError("should not crash") + """) + + # Create a temporary script to exercise _Py_FindSourceFile + with temp_dir() as script_dir: + script = make_script( + script_dir=script_dir, + script_basename='tb_test_no_io_open', + source=crasher) + rc, stdout, stderr = assert_python_failure(script) + + self.assertEqual(rc, 1) # Make sure it's not a crash + + expected = [b'Traceback (most recent call last):', + f' File "{script}", line 6, in <module>'.encode(), + b'RuntimeError: should not crash'] + self.assertEqual(stderr.splitlines(), expected) + def test_print_exception(self): output = StringIO() traceback.print_exception( @@ -1740,6 +1767,49 @@ def f(): ] self.assertEqual(result_lines, expected) +class TestKeywordTypoSuggestions(unittest.TestCase): + TYPO_CASES = [ + ("with block ad something:\n pass", "and"), + ("fur a in b:\n pass", "for"), + ("for a in b:\n pass\nelso:\n pass", "else"), + ("whille True:\n pass", "while"), + ("iff x > 5:\n pass", "if"), + ("if x:\n pass\nelseif y:\n pass", "elif"), + ("tyo:\n pass\nexcept y:\n pass", "try"), + ("classe MyClass:\n pass", "class"), + ("impor math", "import"), + ("form x import y", "from"), + ("defn calculate_sum(a, b):\n return a + b", "def"), + ("def foo():\n returm result", "return"), + ("lamda x: x ** 2", "lambda"), + ("def foo():\n yeld i", "yield"), + ("def foo():\n globel counter", "global"), + ("frum math import sqrt", "from"), + ("asynch def fetch_data():\n pass", "async"), + ("async def foo():\n awaid fetch_data()", "await"), + ('raisee ValueError("Error")', "raise"), + ("[x for x\nin range(3)\nof x]", "if"), + ("[123 fur x\nin range(3)\nif x]", "for"), + ("for x im n:\n pass", "in"), + ] + + def test_keyword_suggestions_from_file(self): + with tempfile.TemporaryDirectory() as script_dir: + for i, (code, expected_kw) in enumerate(self.TYPO_CASES): + with self.subTest(typo=expected_kw): + source = textwrap.dedent(code).strip() + script_name = make_script(script_dir, f"script_{i}", source) + rc, stdout, stderr = assert_python_failure(script_name) + stderr_text = stderr.decode('utf-8') + self.assertIn(f"Did you mean '{expected_kw}'", stderr_text) + + def test_keyword_suggestions_from_command_string(self): + for code, expected_kw in self.TYPO_CASES: + with self.subTest(typo=expected_kw): + source = textwrap.dedent(code).strip() + rc, stdout, stderr = assert_python_failure('-c', source) + stderr_text = stderr.decode('utf-8') + self.assertIn(f"Did you mean '{expected_kw}'", stderr_text) @requires_debug_ranges() @force_not_colorized_test_class @@ -4098,6 +4168,27 @@ def method(self, name): self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, '_luch'))) self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, 'bluch'))) + def test_getattr_suggestions_with_custom___dir__(self): + class M(type): + def __dir__(cls): + return [None, "fox"] + + class C0: + def __dir__(self): + return [..., "bluch"] + + class C1(C0, metaclass=M): + pass + + self.assertNotIn("'bluch'", self.get_suggestion(C0, "blach")) + self.assertIn("'bluch'", self.get_suggestion(C0(), "blach")) + + self.assertIn("'fox'", self.get_suggestion(C1, "foo")) + self.assertNotIn("'fox'", self.get_suggestion(C1(), "foo")) + + self.assertNotIn("'bluch'", self.get_suggestion(C1, "blach")) + self.assertIn("'bluch'", self.get_suggestion(C1(), "blach")) + def test_getattr_suggestions_do_not_trigger_for_long_attributes(self): class A: blech = None @@ -4188,6 +4279,15 @@ def __dir__(self): self.assertNotIn("blech", actual) self.assertNotIn("oh no!", actual) + def test_attribute_error_with_non_string_candidates(self): + class T: + bluch = 1 + + instance = T() + instance.__dict__[0] = 1 + actual = self.get_suggestion(instance, 'blich') + self.assertIn("bluch", actual) + def test_attribute_error_with_bad_name(self): def raise_attribute_error_with_bad_name(): raise AttributeError(name=12, obj=23) @@ -4223,8 +4323,8 @@ def make_module(self, code): return mod_name - def get_import_from_suggestion(self, mod_dict, name): - modname = self.make_module(mod_dict) + def get_import_from_suggestion(self, code, name): + modname = self.make_module(code) def callable(): try: @@ -4301,6 +4401,13 @@ def test_import_from_suggestions_underscored(self): self.assertIn("'_bluch'", self.get_import_from_suggestion(code, '_luch')) self.assertNotIn("'_bluch'", self.get_import_from_suggestion(code, 'bluch')) + def test_import_from_suggestions_non_string(self): + modWithNonStringAttr = textwrap.dedent("""\ + globals()[0] = 1 + bluch = 1 + """) + self.assertIn("'bluch'", self.get_import_from_suggestion(modWithNonStringAttr, 'blech')) + def test_import_from_suggestions_do_not_trigger_for_long_attributes(self): code = "blech = None" @@ -4397,6 +4504,15 @@ def func(): actual = self.get_suggestion(func) self.assertIn("'ZeroDivisionError'?", actual) + def test_name_error_suggestions_with_non_string_candidates(self): + def func(): + abc = 1 + custom_globals = globals().copy() + custom_globals[0] = 1 + print(eval("abv", custom_globals, locals())) + actual = self.get_suggestion(func) + self.assertIn("abc", actual) + def test_name_error_suggestions_do_not_trigger_for_long_names(self): def func(): somethingverywronghehehehehehe = None diff --git a/Lib/test/test_tstring.py b/Lib/test/test_tstring.py index e72a1ea54176d55..74653c77c55de17 100644 --- a/Lib/test/test_tstring.py +++ b/Lib/test/test_tstring.py @@ -150,7 +150,6 @@ def test_raw_tstrings(self): t = tr"{path}\Documents" self.assertTStringEqual(t, ("", r"\Documents"), [(path, "path")]) - def test_template_concatenation(self): # Test template + template t1 = t"Hello, " @@ -161,9 +160,10 @@ def test_template_concatenation(self): # Test template + string t1 = t"Hello" - combined = t1 + ", world" - self.assertTStringEqual(combined, ("Hello, world",), ()) - self.assertEqual(fstring(combined), "Hello, world") + expected_msg = 'can only concatenate string.templatelib.Template ' \ + '\\(not "str"\\) to string.templatelib.Template' + with self.assertRaisesRegex(TypeError, expected_msg): + t1 + ", world" # Test template + template with interpolation name = "Python" @@ -174,9 +174,10 @@ def test_template_concatenation(self): self.assertEqual(fstring(combined), "Hello, Python") # Test string + template - t = "Hello, " + t"{name}" - self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")]) - self.assertEqual(fstring(t), "Hello, Python") + expected_msg = 'can only concatenate str ' \ + '\\(not "string.templatelib.Template"\\) to str' + with self.assertRaisesRegex(TypeError, expected_msg): + "Hello, " + t"{name}" def test_nested_templates(self): # Test a template inside another template expression @@ -219,6 +220,7 @@ def test_syntax_errors(self): ("t'{lambda:1}'", "t-string: lambda expressions are not allowed " "without parentheses"), ("t'{x:{;}}'", "t-string: expecting a valid expression after '{'"), + ("t'{1:d\n}'", "t-string: newlines are not allowed in format specifiers") ): with self.subTest(case), self.assertRaisesRegex(SyntaxError, err): eval(case) @@ -240,52 +242,28 @@ def test_literal_concatenation(self): self.assertTStringEqual(t, ("Hello, ", ""), [(name, "name")]) self.assertEqual(fstring(t), "Hello, Python") - # Test concatenation with string literal - name = "Python" - t = t"Hello, {name}" "and welcome!" - self.assertTStringEqual( - t, ("Hello, ", "and welcome!"), [(name, "name")] - ) - self.assertEqual(fstring(t), "Hello, Pythonand welcome!") - - # Test concatenation with Unicode literal - name = "Python" - t = t"Hello, {name}" u"and welcome!" - self.assertTStringEqual( - t, ("Hello, ", "and welcome!"), [(name, "name")] - ) - self.assertEqual(fstring(t), "Hello, Pythonand welcome!") - - # Test concatenation with f-string literal - tab = '\t' - t = t"Tab: {tab}. " f"f-tab: {tab}." - self.assertTStringEqual(t, ("Tab: ", ". f-tab: \t."), [(tab, "tab")]) - self.assertEqual(fstring(t), "Tab: \t. f-tab: \t.") - - # Test concatenation with raw string literal - tab = '\t' - t = t"Tab: {tab}. " r"Raw tab: \t." - self.assertTStringEqual( - t, ("Tab: ", r". Raw tab: \t."), [(tab, "tab")] - ) - self.assertEqual(fstring(t), "Tab: \t. Raw tab: \\t.") - - # Test concatenation with raw f-string literal - tab = '\t' - t = t"Tab: {tab}. " rf"f-tab: {tab}. Raw tab: \t." - self.assertTStringEqual( - t, ("Tab: ", ". f-tab: \t. Raw tab: \\t."), [(tab, "tab")] - ) - self.assertEqual(fstring(t), "Tab: \t. f-tab: \t. Raw tab: \\t.") - + # Test disallowed mix of t-string and string/f-string (incl. bytes) what = 't' - expected_msg = 'cannot mix bytes and nonbytes literals' + expected_msg = 'cannot mix t-string literals with string or bytes literals' for case in ( + "t'{what}-string literal' 'str literal'", + "t'{what}-string literal' u'unicode literal'", + "t'{what}-string literal' f'f-string literal'", + "t'{what}-string literal' r'raw string literal'", + "t'{what}-string literal' rf'raw f-string literal'", "t'{what}-string literal' b'bytes literal'", "t'{what}-string literal' br'raw bytes literal'", + "'str literal' t'{what}-string literal'", + "u'unicode literal' t'{what}-string literal'", + "f'f-string literal' t'{what}-string literal'", + "r'raw string literal' t'{what}-string literal'", + "rf'raw f-string literal' t'{what}-string literal'", + "b'bytes literal' t'{what}-string literal'", + "br'raw bytes literal' t'{what}-string literal'", ): - with self.assertRaisesRegex(SyntaxError, expected_msg): - eval(case) + with self.subTest(case): + with self.assertRaisesRegex(SyntaxError, expected_msg): + eval(case) def test_triple_quoted(self): # Test triple-quoted t-strings diff --git a/Lib/test/test_ttk/__init__.py b/Lib/test/test_ttk/__init__.py index 7ee7ffbd6d74086..971a5d24f06eb48 100644 --- a/Lib/test/test_ttk/__init__.py +++ b/Lib/test/test_ttk/__init__.py @@ -20,6 +20,10 @@ def setUpModule(): + wantobjects = support.get_resource_value('wantobjects') + if wantobjects is not None: + unittest.enterModuleContext( + support.swap_attr(tkinter, 'wantobjects', int(wantobjects))) root = None try: root = tkinter.Tk() diff --git a/Lib/test/test_ttk/test_extensions.py b/Lib/test/test_ttk/test_extensions.py index 05bca59e703936c..f6ca5e976109fc2 100644 --- a/Lib/test/test_ttk/test_extensions.py +++ b/Lib/test/test_ttk/test_extensions.py @@ -3,10 +3,15 @@ import tkinter from tkinter import ttk from test.support import requires, gc_collect -from test.test_tkinter.support import AbstractTkTest, AbstractDefaultRootTest +from test.test_tkinter.support import setUpModule # noqa: F401 +from test.test_tkinter.support import (AbstractTkTest, AbstractDefaultRootTest, + get_tk_patchlevel, widget_eq) requires('gui') +from test.test_ttk import test_widgets + + class LabeledScaleTest(AbstractTkTest, unittest.TestCase): def tearDown(self): @@ -104,7 +109,7 @@ def check_positions(scale, scale_pos, label, label_pos): def test_horizontal_range(self): lscale = ttk.LabeledScale(self.root, from_=0, to=10) lscale.pack() - lscale.update() + self.require_mapped(lscale) linfo_1 = lscale.label.place_info() prev_xcoord = lscale.scale.coords()[0] @@ -133,7 +138,7 @@ def test_horizontal_range(self): def test_variable_change(self): x = ttk.LabeledScale(self.root) x.pack() - x.update() + self.require_mapped(x) curr_xcoord = x.scale.coords()[0] newval = x.value + 1 @@ -176,7 +181,7 @@ def test_resize(self): x = ttk.LabeledScale(self.root) x.pack(expand=True, fill='both') gc_collect() # For PyPy or other GCs. - x.update() + self.require_mapped(x) width, height = x.master.winfo_width(), x.master.winfo_height() width_new, height_new = width * 2, height * 2 @@ -192,7 +197,11 @@ def test_resize(self): x.destroy() -class OptionMenuTest(AbstractTkTest, unittest.TestCase): +class OptionMenuTest(test_widgets.MenubuttonTest, unittest.TestCase): + + # OptionMenu documents only its own options, not the inherited + # Menubutton options (like the classic tkinter.OptionMenu). + test_options_in_docstring = None def setUp(self): super().setUp() @@ -202,6 +211,47 @@ def tearDown(self): del self.textvar super().tearDown() + def create(self, default='b', values=('a', 'b', 'c'), **kwargs): + return ttk.OptionMenu(self.root, self.textvar, default, *values, **kwargs) + + def test_bad_kwarg(self): + with self.assertRaisesRegex(tkinter.TclError, r"^unknown option -image$"): + ttk.OptionMenu(self.root, self.textvar, 'b', image='') + + def test_configure_class(self): + # Unlike a plain Menubutton, OptionMenu does not accept a class at + # construction, so only the read-only nature of the option is tested. + widget = self.create() + self.assertEqual(widget['class'], '') + errmsg = 'attempt to change read-only option' + if get_tk_patchlevel(self.root) < (8, 6, 0, 'beta', 3): + errmsg = 'Attempt to change read-only option' + self.checkInvalidParam(widget, 'class', 'Foo', errmsg=errmsg) + + def test_configure_style(self): + # Like Menubutton, but OptionMenu does not accept a class at + # construction, so the custom-class part of the standard test is omitted. + widget = self.create() + self.assertEqual(widget['style'], '') + self.checkInvalidParam(widget, 'style', 'Foo', + errmsg='Layout Foo not found') + style = ttk.Style(self.root) + style.configure('Custom.TMenubutton') + self.checkParam(widget, 'style', 'Custom.TMenubutton') + + def test_configure_menu(self): + # OptionMenu manages its own menu; ['menu'] returns that Menu widget. + widget = self.create() + self.assertIsInstance(widget['menu'], tkinter.Menu) + self.checkParam(widget, 'menu', widget['menu'], eq=widget_eq) + + def test_configure_text(self): + # The displayed text is governed by the textvariable. + widget = self.create() + self.textvar.set('a') + self.assertEqual(widget['text'], 'a') + self.textvar.set('c') + self.assertEqual(widget['text'], 'c') def test_widget_destroy(self): var = tkinter.StringVar(self.root) @@ -280,6 +330,24 @@ def cb_test(item): optmenu.destroy() + def test_set_menu(self): + optmenu = ttk.OptionMenu(self.root, self.textvar, 'a', 'a', 'b', 'c') + menu = optmenu['menu'] + self.assertEqual(menu.index('end'), 2) + + # set_menu rebuilds the menu with new values and an optional default. + optmenu.set_menu('y', 'x', 'y', 'z') + self.assertEqual(self.textvar.get(), 'y') + self.assertEqual([menu.entrycget(i, 'label') for i in range(3)], + ['x', 'y', 'z']) + + # Without a default the variable is left unchanged. + optmenu.set_menu(None, 'p', 'q') + self.assertEqual(self.textvar.get(), 'y') + self.assertEqual([menu.entrycget(i, 'label') for i in range(2)], + ['p', 'q']) + optmenu.destroy() + def test_unique_radiobuttons(self): # check that radiobuttons are unique across instances (bpo25684) items = ('a', 'b', 'c') diff --git a/Lib/test/test_ttk/test_style.py b/Lib/test/test_ttk/test_style.py index 19918772514ad4e..56d8a5931fdfa5c 100644 --- a/Lib/test/test_ttk/test_style.py +++ b/Lib/test/test_ttk/test_style.py @@ -5,6 +5,7 @@ from tkinter import TclError from test import support from test.support import requires +from test.test_tkinter.support import setUpModule # noqa: F401 from test.test_tkinter.support import AbstractTkTest, get_tk_patchlevel requires('gui') @@ -123,6 +124,22 @@ def test_theme_use(self): self.style.theme_use(curr_theme) + def test_theme_settings(self): + style = self.style + theme = style.theme_use() + style.theme_settings(theme, { + 'Test.TLabel': { + 'configure': {'foreground': 'red', 'background': 'blue'}, + 'map': {'foreground': [('active', 'green')]}, + }, + }) + self.assertEqual(style.lookup('Test.TLabel', 'foreground'), 'red') + self.assertEqual(style.lookup('Test.TLabel', 'background'), 'blue') + self.assertEqual(style.map('Test.TLabel', 'foreground'), + [('active', 'green')]) + self.assertRaises(tkinter.TclError, style.theme_settings, + 'nonexistingname', {}) + def test_configure_custom_copy(self): style = self.style @@ -403,7 +420,8 @@ def test_theme_create_image(self): b = ttk.Label(self.root, style='TestWidget') b.pack(expand=True, fill='both') - self.assertEqual(b.winfo_reqwidth(), 134) + # The exact width varies with the Tk version and display scaling. + self.assertGreater(b.winfo_reqwidth(), 130) self.assertEqual(b.winfo_reqheight(), 100) style.theme_use(curr_theme) diff --git a/Lib/test/test_ttk/test_widgets.py b/Lib/test/test_ttk/test_widgets.py index f33da2a8848738b..53bcd5ed2aab1b0 100644 --- a/Lib/test/test_ttk/test_widgets.py +++ b/Lib/test/test_ttk/test_widgets.py @@ -5,9 +5,10 @@ import sys from test.test_ttk_textonly import MockTclObj +from test.test_tkinter.support import setUpModule # noqa: F401 from test.test_tkinter.support import ( AbstractTkTest, requires_tk, tk_version, get_tk_patchlevel, - simulate_mouse_click, AbstractDefaultRootTest) + simulate_mouse_click, wait_until_mapped, AbstractDefaultRootTest) from test.test_tkinter.widget_tests import (add_configure_tests, AbstractWidgetTest, StandardOptionsTests, IntegerSizeTests, PixelSizeTests) @@ -77,11 +78,13 @@ def setUp(self): self.widget.pack() def test_identify(self): - self.widget.update() - self.assertEqual(self.widget.identify( - int(self.widget.winfo_width() / 2), - int(self.widget.winfo_height() / 2) - ), "label") + # Identifying the element under a point requires the widget to be + # mapped with a real size; the rest of the checks do not. + if wait_until_mapped(self.widget): + self.assertEqual(self.widget.identify( + int(self.widget.winfo_width() / 2), + int(self.widget.winfo_height() / 2) + ), "label") self.assertEqual(self.widget.identify(-1, -1), "") self.assertRaises(tkinter.TclError, self.widget.identify, None, 5) @@ -182,7 +185,7 @@ def checkImageParam(self, widget, name): expected=('image1', 'active', 'image2')) self.checkParam(widget, name, 'image1 active image2', expected=('image1', 'active', 'image2')) - if tk_version < (9, 0): + if tk_version < (8, 7): errmsg = 'image "spam" doesn\'t exist' else: errmsg = 'image "spam" does not exist' @@ -354,6 +357,19 @@ def setUp(self): def create(self, **kwargs): return ttk.Entry(self.root, **kwargs) + def _arrow_x(self, widget, y, arrow): + # Return an x coordinate on the up/down *arrow* element in row y, + # found by scanning inward from the right edge of the widget. A + # fixed inset such as width - 5 lands in the arrow element's + # right-hand border, which scales with the display scaling, so at a + # high DPI identify() there returns 'field' instead of the arrow. + self.assertTrue(wait_until_mapped(widget, full_size=True)) + width = widget.winfo_width() + for x in range(width - 1, width // 2, -1): + if widget.identify(x, y).endswith(arrow): + return x + self.fail(f'no {arrow} found in row {y}') + def test_configure_invalidcommand(self): widget = self.create() self.checkCommandParam(widget, 'invalidcommand') @@ -384,9 +400,16 @@ def test_identify(self): self.skipTest('Test does not work on macOS Tk 9.') # https://core.tcl-lang.org/tk/tktview/8b49e9cfa6 self.entry.pack() - self.root.update() - self.assertIn(self.entry.identify(5, 5), self.IDENTIFY_AS) + # Identifying the element under a point requires the widget to be + # mapped with a real size; the rest of the checks do not. + if wait_until_mapped(self.entry): + # Probe the centre of the widget: a fixed pixel such as (5, 5) + # lands in the field at a normal scaling but in the surrounding + # padding at a high DPI. + x = self.entry.winfo_width() // 2 + y = self.entry.winfo_height() // 2 + self.assertIn(self.entry.identify(x, y), self.IDENTIFY_AS) self.assertEqual(self.entry.identify(-1, -1), "") self.assertRaises(tkinter.TclError, self.entry.identify, None, 5) @@ -487,10 +510,11 @@ def test_configure_height(self): self.checkParams(widget, 'height', 100, 101.2, 102.6, -100, 0, '1i') def _show_drop_down_listbox(self): - width = self.combo.winfo_width() - x, y = width - 5, 5 + y = 5 if sys.platform != 'darwin': # there's no down arrow on macOS - self.assertRegex(self.combo.identify(x, y), r'.*downarrow\z') + x = self._arrow_x(self.combo, y, 'downarrow') + else: + x = self.combo.winfo_width() - 5 self.combo.event_generate('<Button-1>', x=x, y=y) self.combo.event_generate('<ButtonRelease-1>', x=x, y=y) @@ -505,7 +529,7 @@ def test_virtual_event(self): self.combo.bind('<<ComboboxSelected>>', lambda evt: success.append(True)) self.combo.pack() - self.combo.update() + self.require_mapped(self.combo) height = self.combo.winfo_height() self._show_drop_down_listbox() @@ -524,7 +548,7 @@ def test_configure_postcommand(self): self.combo['postcommand'] = lambda: success.append(True) self.combo.pack() - self.combo.update() + self.require_mapped(self.combo) self._show_drop_down_listbox() self.assertTrue(success) @@ -788,6 +812,7 @@ def cb_test(): self.assertEqual(str(cbtn['variable']), str(cbtn2['variable'])) +@add_configure_tests(StandardTtkOptionsTests) class MenubuttonTest(AbstractLabelTest, unittest.TestCase): OPTIONS = ( 'class', 'compound', 'cursor', 'direction', @@ -873,8 +898,10 @@ def test_get(self): else: conv = float - scale_width = self.scale.winfo_width() - self.assertEqual(self.scale.get(scale_width, 0), self.scale['to']) + # Reading the value at the far edge needs the realized width. + if wait_until_mapped(self.scale): + scale_width = self.scale.winfo_width() + self.assertEqual(self.scale.get(scale_width, 0), self.scale['to']) self.assertEqual(conv(self.scale.get(0, 0)), conv(self.scale['from'])) self.assertEqual(self.scale.get(), self.scale['value']) @@ -916,7 +943,10 @@ def test_set(self): # nevertheless, note that the max/min values we can get specifying # x, y coords are the ones according to the current range self.assertEqual(conv(self.scale.get(0, 0)), min) - self.assertEqual(conv(self.scale.get(self.scale.winfo_width(), 0)), max) + # Reading the value at the far edge needs the realized width. + if wait_until_mapped(self.scale): + self.assertEqual( + conv(self.scale.get(self.scale.winfo_width(), 0)), max) self.assertRaises(tkinter.TclError, self.scale.set, None) @@ -972,6 +1002,27 @@ def test_configure_value(self): test_configure_wraplength = requires_tk(8, 7)(StandardOptionsTests.test_configure_wraplength) + def test_step(self): + widget = self.create(maximum=100, mode='determinate') + self.assertEqual(float(widget['value']), 0.0) + widget.step() # The default increment is 1.0. + self.assertEqual(float(widget['value']), 1.0) + widget.step(5) + self.assertEqual(float(widget['value']), 6.0) + widget.step(-2) + self.assertEqual(float(widget['value']), 4.0) + + def test_start_stop(self): + widget = self.create(maximum=100, mode='determinate') + widget.pack() + widget.start() # Schedule autoincrement; no exception. + widget.update() + widget.stop() # Cancel it. + # After stopping, the value no longer changes. + value = float(widget['value']) + widget.update() + self.assertEqual(float(widget['value']), value) + @unittest.skipIf(sys.platform == 'darwin', 'ttk.Scrollbar is special on MacOSX') @@ -1036,21 +1087,15 @@ def test_tab_identifiers(self): self.nb.pack() self.nb.update() - if sys.platform == 'darwin': - tb_idx = "@20,5" - else: - tb_idx = "@5,5" - self.assertEqual(self.nb.tab(tb_idx), self.nb.tab('current')) + # Address a tab by a point taken from Tk's own geometry rather than a + # fixed pixel, which can land on a different tab or miss the tabs + # entirely (e.g. on Aqua the tabs are inset) as the display scaling + # changes. + x, y = self._tab_point(self.nb.index('current')) + self.assertEqual(self.nb.tab(f'@{x},{y}'), self.nb.tab('current')) - for i in range(5, 100, 5): - try: - if self.nb.tab('@%d, 5' % i, text=None) == 'a': - break - except tkinter.TclError: - pass - - else: - self.fail("Tab with text 'a' not found") + x, y = self._tab_point(self.nb.index(self.child1)) + self.assertEqual(self.nb.tab(f'@{x},{y}', text=None), 'a') def test_add_and_hidden(self): self.assertRaises(tkinter.TclError, self.nb.hide, -1) @@ -1180,20 +1225,43 @@ def test_configure_tabs(self): self.assertEqual(self.nb.tabs(), ()) + def _tab_point(self, index): + # Return a window coordinate that really lies on tab *index*, taken + # from Tk's own geometry (ttk::notebook index @x,y), so that it is + # correct regardless of the theme, platform and display scaling. A + # fixed pixel such as (5, 5) can land on a different tab element + # depending on the DPI, or miss the tabs entirely (e.g. on Aqua the + # tabs are inset), which would make a click there test nothing. + self.assertTrue(wait_until_mapped(self.nb, full_size=True)) + w, h = self.nb.winfo_width(), self.nb.winfo_height() + # The selected pane fills the widget except for the tab strip, so skip + # that large tab-free region instead of querying every pixel in it. + pane = self.nb.nametowidget(self.nb.select()) + px, py = pane.winfo_x(), pane.winfo_y() + pw, ph = pane.winfo_width(), pane.winfo_height() + for y in range(h): + inside_y = py <= y < py + ph + for x in range(w): + if inside_y and px <= x < px + pw: + continue + try: + if self.nb.index(f'@{x},{y}') == index: + return x, y + except TclError: + continue + self.fail(f'no point found on tab {index}') + def test_traversal(self): self.nb.pack() self.nb.update() - self.nb.select(0) + # A mouse click selects the tab it lands on. + self.nb.select(1) + self.assertEqual(self.nb.select(), str(self.child2)) + simulate_mouse_click(self.nb, *self._tab_point(0)) + self.assertEqual(self.nb.select(), str(self.child1)) - if sys.platform == 'darwin': - focus_identify_as = '' - elif sys.platform == 'win32': - focus_identify_as = 'focus' - else: - focus_identify_as = 'focus' if tk_version < (9,0) else 'padding' - self.assertEqual(self.nb.identify(5, 5), focus_identify_as) - simulate_mouse_click(self.nb, 5, 5) + # Control-Tab and Shift-Control-Tab traverse the tabs. self.nb.focus_force() self.nb.event_generate('<Control-Tab>') self.assertEqual(self.nb.select(), str(self.child2)) @@ -1208,20 +1276,25 @@ def test_traversal(self): self.nb.tab(self.child2, text='e', underline=0) self.nb.enable_traversal() self.nb.focus_force() - self.assertEqual(self.nb.identify(5, 5), focus_identify_as) - simulate_mouse_click(self.nb, 5, 5) - # on macOS Emacs-style keyboard shortcuts are region-dependent; - # let's use the regular arrow keys instead + + # A click still selects the tab it lands on after enable_traversal(). + self.nb.select(1) + self.assertEqual(self.nb.select(), str(self.child2)) + simulate_mouse_click(self.nb, *self._tab_point(0)) + self.assertEqual(self.nb.select(), str(self.child1)) + + # Mnemonics traverse the tabs (macOS uses region-dependent + # Emacs-style shortcuts, so use the regular arrow keys there). if sys.platform == 'darwin': begin = '<Left>' end = '<Right>' else: begin = '<Alt-a>' end = '<Alt-e>' - self.nb.event_generate(begin) - self.assertEqual(self.nb.select(), str(self.child1)) self.nb.event_generate(end) self.assertEqual(self.nb.select(), str(self.child2)) + self.nb.event_generate(begin) + self.assertEqual(self.nb.select(), str(self.child1)) @add_configure_tests(IntegerSizeTests, StandardTtkOptionsTests) @@ -1246,21 +1319,17 @@ def create(self, **kwargs): return ttk.Spinbox(self.root, **kwargs) def _click_increment_arrow(self): - width = self.spin.winfo_width() - height = self.spin.winfo_height() - x = width - 5 - y = height//2 - 5 - self.assertRegex(self.spin.identify(x, y), r'.*uparrow\z') + self.require_mapped(self.spin) + y = self.spin.winfo_height()//2 - 5 + x = self._arrow_x(self.spin, y, 'uparrow') self.spin.event_generate('<ButtonPress-1>', x=x, y=y) self.spin.event_generate('<ButtonRelease-1>', x=x, y=y) self.spin.update_idletasks() def _click_decrement_arrow(self): - width = self.spin.winfo_width() - height = self.spin.winfo_height() - x = width - 5 - y = height//2 + 4 - self.assertRegex(self.spin.identify(x, y), r'.*downarrow\z') + self.require_mapped(self.spin) + y = self.spin.winfo_height()//2 + 4 + x = self._arrow_x(self.spin, y, 'downarrow') self.spin.event_generate('<ButtonPress-1>', x=x, y=y) self.spin.event_generate('<ButtonRelease-1>', x=x, y=y) self.spin.update_idletasks() @@ -1285,6 +1354,19 @@ def test_configure_command(self): self.spin.update() self.assertEqual(len(success), 2) + def test_increment_decrement_events(self): + # Clicking the arrows fires the <<Increment>> and <<Decrement>> + # virtual events. + events = [] + self.spin.bind('<<Increment>>', lambda e: events.append('increment')) + self.spin.bind('<<Decrement>>', lambda e: events.append('decrement')) + self.spin.update() + self._click_increment_arrow() + self.spin.update() + self._click_decrement_arrow() + self.spin.update() + self.assertEqual(events, ['increment', 'decrement']) + def test_configure_to(self): self.spin['from'] = 0 self.spin['to'] = 5 @@ -1453,8 +1535,13 @@ def test_configure_height(self): def test_configure_selectmode(self): widget = self.create() - self.checkEnumParam(widget, 'selectmode', - 'none', 'browse', 'extended') + if tk_version >= (9, 1): + self.checkEnumParam(widget, 'selectmode', + 'none', 'single', 'browse', 'extended', + 'multiple') + else: + self.checkEnumParam(widget, 'selectmode', + 'none', 'browse', 'extended') @requires_tk(8, 7) def test_configure_selecttype(self): @@ -1629,15 +1716,59 @@ def test_detach_reattach(self): self.assertEqual(self.tv.get_children(item_id), ()) def test_exists(self): - self.assertEqual(self.tv.exists('something'), False) - self.assertEqual(self.tv.exists(''), True) - self.assertEqual(self.tv.exists({}), False) + self.assertIs(self.tv.exists('something'), False) + self.assertIs(self.tv.exists(''), True) + self.assertIs(self.tv.exists({}), False) # the following will make a tk.call equivalent to # tk.call(treeview, "exists") which should result in an error # in the tcl interpreter since tk requires an item. self.assertRaises(tkinter.TclError, self.tv.exists, None) + def test_parent(self): + a = self.tv.insert('', 'end') + b = self.tv.insert(a, 'end') + self.assertEqual(self.tv.parent(b), a) + self.assertEqual(self.tv.parent(a), '') + self.assertRaises(tkinter.TclError, self.tv.parent, 'nonexistent') + + def test_next_prev(self): + a = self.tv.insert('', 'end') + b = self.tv.insert('', 'end') + c = self.tv.insert('', 'end') + self.assertEqual(self.tv.next(a), b) + self.assertEqual(self.tv.next(b), c) + self.assertEqual(self.tv.next(c), '') + self.assertEqual(self.tv.prev(c), b) + self.assertEqual(self.tv.prev(b), a) + self.assertEqual(self.tv.prev(a), '') + self.assertRaises(tkinter.TclError, self.tv.next, 'nonexistent') + self.assertRaises(tkinter.TclError, self.tv.prev, 'nonexistent') + + def test_see(self): + a = self.tv.insert('', 'end') + b = self.tv.insert(a, 'end') + # see() opens all of the item's ancestors. + self.assertFalse(self.tv.tk.getboolean(self.tv.item(a, 'open'))) + self.tv.see(b) + self.assertTrue(self.tv.tk.getboolean(self.tv.item(a, 'open'))) + self.assertRaises(tkinter.TclError, self.tv.see, 'nonexistent') + + def test_identify_element(self): + self.tv.pack() + self.tv.wait_visibility() + parent = self.tv.insert('', 'end', text='parent') + self.tv.insert(parent, 'end', text='child') + self.tv.update() + x, y, w, h = self.tv.bbox(parent) + # The Treeitem.indicator element is packed at the left of the row in + # the Item layout on every platform and theme. + element = self.tv.identify_element(x + 8, y + h // 2) + self.assertRegex(element, r'.*indicator\z') + # The empty string is returned outside the widget. + self.assertEqual(self.tv.identify_element(-1, -1), '') + self.assertRaises(tkinter.TclError, self.tv.identify_element, None, 5) + def test_focus(self): # nothing is focused right now self.assertEqual(self.tv.focus(), '') @@ -1693,6 +1824,9 @@ def simulate_heading_click(x, y): commands = self.tv.master._tclCommands self.tv.heading('#0', command=str(self.tv.heading('#0', command=None))) self.assertEqual(commands, self.tv.master._tclCommands) + # Click elsewhere first, so the second heading click is not reported + # as a double click (which does not invoke the command). + simulate_mouse_click(self.tv, 5, 50) simulate_heading_click(5, 5) if not success: self.fail("The command associated to the treeview heading wasn't " @@ -1801,10 +1935,11 @@ def test_insert_item(self): value) # test for values which are not None + keep_type = self.wantobjects and tk_version >= (9, 1) itemid = self.tv.insert('', 'end', 0) - self.assertEqual(itemid, '0') + self.assertEqual(itemid, 0 if keep_type else '0') itemid = self.tv.insert('', 'end', 0.0) - self.assertEqual(itemid, '0.0') + self.assertEqual(itemid, 0.0 if keep_type else '0.0') # this is because False resolves to 0 and element with 0 iid is already present self.assertRaises(tkinter.TclError, self.tv.insert, '', 'end', False) self.assertRaises(tkinter.TclError, self.tv.insert, '', 'end', '') @@ -1864,7 +1999,10 @@ def test_selection(self): self.tv.insert('', 'end', id=b'bytes\xe2\x82\xac') self.tv.selection_set(b'bytes\xe2\x82\xac') - self.assertEqual(self.tv.selection(), ('bytes\xe2\x82\xac',)) + self.assertEqual(self.tv.selection(), + (b'bytes\xe2\x82\xac',) + if self.wantobjects and tk_version >= (9, 1) + else ('bytes\xe2\x82\xac',)) self.tv.selection_set() self.assertEqual(self.tv.selection(), ()) @@ -1879,17 +2017,53 @@ def test_selection(self): self.tv.selection_toggle((c1, c3)) self.assertEqual(self.tv.selection(), (c3, item2)) + def test_virtual_events(self): + # Keyboard navigation fires the <<TreeviewSelect>>, <<TreeviewOpen>> + # and <<TreeviewClose>> virtual events. + parent = self.tv.insert('', 'end') + self.tv.insert(parent, 'end') + item2 = self.tv.insert('', 'end') + self.tv.pack() + self.require_mapped(self.tv) + selects, opens, closes = [], [], [] + self.tv.bind('<<TreeviewSelect>>', + lambda e: selects.append(self.tv.selection())) + self.tv.bind('<<TreeviewOpen>>', lambda e: opens.append(self.tv.focus())) + self.tv.bind('<<TreeviewClose>>', lambda e: closes.append(self.tv.focus())) + self.tv.focus_force() + self.tv.focus(parent) + self.tv.selection_set(parent) + self.tv.update() + + self.tv.event_generate('<Right>') # Open the focused parent. + self.tv.update() + self.assertEqual(opens, [parent]) + + self.tv.event_generate('<Left>') # Close it again. + self.tv.update() + self.assertEqual(closes, [parent]) + + self.tv.event_generate('<Down>') # Move the selection. + self.tv.update() + self.assertEqual(self.tv.selection(), (item2,)) + self.assertIn((item2,), selects) + def test_set(self): self.tv['columns'] = ['A', 'B'] item = self.tv.insert('', 'end', values=['a', 'b']) - self.assertEqual(self.tv.set(item), {'A': 'a', 'B': 'b'}) + values = self.tv.set(item) + if tk_version >= (9, 1): + self.assertEqual(values.pop('#0'), '') + self.assertEqual(values, {'A': 'a', 'B': 'b'}) self.tv.set(item, 'B', 'a') self.assertEqual(self.tv.item(item, values=None), ('a', 'a') if self.wantobjects else 'a a') self.tv['columns'] = ['B'] - self.assertEqual(self.tv.set(item), {'B': 'a'}) + values = self.tv.set(item) + values.pop('#0', None) + self.assertEqual(values, {'B': 'a'}) self.tv.set(item, 'B', 'b') self.assertEqual(self.tv.set(item, column='B'), 'b') @@ -1901,7 +2075,9 @@ def test_set(self): 123 if self.wantobjects else '123') self.assertEqual(self.tv.item(item, values=None), (123, 'a') if self.wantobjects else '123 a') - self.assertEqual(self.tv.set(item), + values = self.tv.set(item) + values.pop('#0', None) + self.assertEqual(values, {'B': 123} if self.wantobjects else {'B': '123'}) # inexistent column @@ -1923,14 +2099,17 @@ def test_tag_bind(self): self.tv.pack() self.tv.update() + # Find the y coordinate of each item. Scan the whole height of the + # widget rather than a fixed pixel range, since the row height grows + # with the display scaling. pos_y = set() found = set() - for i in range(0, 100, 10): + for y in range(self.tv.winfo_height()): if len(found) == 2: # item1 and item2 already found break - item_id = self.tv.identify_row(i) + item_id = self.tv.identify_row(y) if item_id and item_id not in found: - pos_y.add(i) + pos_y.add(y) found.add(item_id) self.assertEqual(len(pos_y), 2) # item1 and item2 y pos diff --git a/Lib/test/test_tuple.py b/Lib/test/test_tuple.py index 9ce80c5e8ea0093..e533392b8cae94a 100644 --- a/Lib/test/test_tuple.py +++ b/Lib/test/test_tuple.py @@ -290,12 +290,18 @@ def test_repr(self): self.assertEqual(repr(a0), "()") self.assertEqual(repr(a2), "(0, 1, 2)") + # Checks that t is not tracked without any GC collections. + def _not_tracked_instantly(self, t): + self.assertFalse(gc.is_tracked(t), t) + + # Checks that t is not tracked after GC collection. def _not_tracked(self, t): # Nested tuples can take several collections to untrack gc.collect() gc.collect() self.assertFalse(gc.is_tracked(t), t) + # Checks that t continues to be tracked even after GC collection. def _tracked(self, t): self.assertTrue(gc.is_tracked(t), t) gc.collect() @@ -307,13 +313,19 @@ def test_track_literals(self): # Test GC-optimization of tuple literals x, y, z = 1.5, "a", [] - self._not_tracked(()) - self._not_tracked((1,)) - self._not_tracked((1, 2)) - self._not_tracked((1, 2, "a")) - self._not_tracked((1, 2, (None, True, False, ()), int)) - self._not_tracked((object(),)) + # We check that those objects aren't tracked at all. + # It's essential for the GC performance, see gh-139951. + self._not_tracked_instantly(()) + self._not_tracked_instantly((1,)) + self._not_tracked_instantly((1, 2)) + self._not_tracked_instantly((1, 2, "a")) + self._not_tracked_instantly((1, 2) * 5) + self._not_tracked_instantly((12, 10**10, 'a_' * 100)) + self._not_tracked_instantly((object(),)) + self._not_tracked(((1, x), y, (2, 3))) + self._not_tracked((1, 2, (None, True, False, ()), int)) + self._not_tracked((object(), ())) # Tuples with mutable elements are always tracked, even if those # elements are not tracked right now. @@ -343,6 +355,12 @@ def check_track_dynamic(self, tp, always_track): self._tracked(tp(tuple([obj]) for obj in [x, y, z])) self._tracked(tuple(tp([obj]) for obj in [x, y, z])) + t = tp([1, x, y, z]) + self.assertEqual(type(t), tp) + self._tracked(t) + self.assertEqual(type(t[:]), tuple) + self._tracked(t[:]) + @support.cpython_only def test_track_dynamic(self): # Test GC-optimization of dynamically constructed tuples. diff --git a/Lib/test/test_turtle.py b/Lib/test/test_turtle.py index d02cac284a909ac..c49ce9cdb6f6d95 100644 --- a/Lib/test/test_turtle.py +++ b/Lib/test/test_turtle.py @@ -60,12 +60,25 @@ def patch_screen(): We must patch the _Screen class itself instead of the _Screen instance because instantiating it requires a display. """ + # Create a mock screen that delegates color validation to the real TurtleScreen methods + mock_screen = unittest.mock.MagicMock() + mock_screen.__class__ = turtle._Screen + mock_screen.mode.return_value = "standard" + mock_screen._colormode = 1.0 + + def mock_iscolorstring(color): + valid_colors = {'red', 'green', 'blue', 'black', 'white', 'yellow', + 'orange', 'purple', 'pink', 'brown', 'gray', 'grey', + 'cyan', 'magenta'} + + return color in valid_colors or (isinstance(color, str) and color.startswith('#')) + + mock_screen._iscolorstring = mock_iscolorstring + mock_screen._colorstr = turtle._Screen._colorstr.__get__(mock_screen) + return unittest.mock.patch( "turtle._Screen.__new__", - **{ - "return_value.__class__": turtle._Screen, - "return_value.mode.return_value": "standard", - }, + return_value=mock_screen ) @@ -564,6 +577,28 @@ def test_no_animation_calls_update_at_exit(self): s.update.assert_not_called() s.update.assert_called_once() + def test_update_is_not_reentrant(self): + # ondrag(goto) reenters _update() while cv.update() processes events; + # without a guard this recurses without bound (gh-50966). + s = turtle.TurtleScreen(cv=unittest.mock.MagicMock()) + depth = max_depth = 0 + + def reenter(): + nonlocal depth, max_depth + depth += 1 + max_depth = max(max_depth, depth) + if depth < 50: + s._update() # as an event handler would + depth -= 1 + + s.cv.update.reset_mock() # ignore calls made during construction + s.cv.update.side_effect = reenter + s._update() + # cv.update() runs once; reentrant calls only flush idle tasks. + self.assertEqual(s.cv.update.call_count, 1) + self.assertEqual(max_depth, 1) + self.assertTrue(s.cv.update_idletasks.called) + class TestTurtle(unittest.TestCase): def setUp(self): @@ -635,6 +670,28 @@ def test_poly_context_when_creating_poly(self): self.assertTrue(self.turtle._creatingPoly) self.assertFalse(self.turtle._creatingPoly) + def test_dot_signature(self): + self.turtle.dot() + self.turtle.dot(10) + self.turtle.dot(size=10) + self.turtle.dot((0, 0, 0)) + self.turtle.dot(size=(0, 0, 0)) + self.turtle.dot("blue") + self.turtle.dot("") + self.turtle.dot(size="blue") + self.turtle.dot(20, "blue") + self.turtle.dot(20, "blue") + self.turtle.dot(20, (0, 0, 0)) + self.turtle.dot(20, 0, 0, 0) + with self.assertRaises(TypeError): + self.turtle.dot(color="blue") + self.assertRaises(turtle.TurtleGraphicsError, self.turtle.dot, "_not_a_color_") + self.assertRaises(turtle.TurtleGraphicsError, self.turtle.dot, 0, (0, 0, 0, 0)) + self.assertRaises(turtle.TurtleGraphicsError, self.turtle.dot, 0, 0, 0, 0, 0) + self.assertRaises(turtle.TurtleGraphicsError, self.turtle.dot, 0, (-1, 0, 0)) + self.assertRaises(turtle.TurtleGraphicsError, self.turtle.dot, 0, -1, 0, 0) + self.assertRaises(turtle.TurtleGraphicsError, self.turtle.dot, 0, (0, 257, 0)) + self.assertRaises(turtle.TurtleGraphicsError, self.turtle.dot, 0, 0, 257, 0) class TestModuleLevel(unittest.TestCase): def test_all_signatures(self): diff --git a/Lib/test/test_type_annotations.py b/Lib/test/test_type_annotations.py index 2c886bb6d362fa8..c98b99e98e9c8e6 100644 --- a/Lib/test/test_type_annotations.py +++ b/Lib/test/test_type_annotations.py @@ -485,6 +485,13 @@ def test_comprehension_in_annotation(self): ns = run_code("x: [y for y in range(10)]") self.assertEqual(ns["__annotate__"](1), {"x": list(range(10))}) + def test_class_annotation_dunder_classdict(self): + ns = run_code(""" + class C: + __classdict__: int + """) + self.assertEqual(ns["C"].__annotations__, {"__classdict__": int}) + def test_future_annotations(self): code = """ from __future__ import annotations @@ -498,6 +505,28 @@ def f(x: int) -> int: pass self.assertEqual(f.__annotate__(annotationlib.Format.VALUE), annos) self.assertEqual(f.__annotations__, annos) + def test_set_annotations(self): + function_code = textwrap.dedent(""" + def f(x: int): + pass + """) + class_code = textwrap.dedent(""" + class f: + x: int + """) + for future in (False, True): + for label, code in (("function", function_code), ("class", class_code)): + with self.subTest(future=future, label=label): + if future: + code = "from __future__ import annotations\n" + code + ns = run_code(code) + f = ns["f"] + anno = "int" if future else int + self.assertEqual(f.__annotations__, {"x": anno}) + + f.__annotations__ = {"x": str} + self.assertEqual(f.__annotations__, {"x": str}) + def test_name_clash_with_format(self): # this test would fail if __annotate__'s parameter was called "format" # during symbol table construction @@ -813,3 +842,57 @@ def test_complex_comprehension_inlining_exec(self): genexp = annos["unique_name_2"][0] lamb = list(genexp)[0] self.assertEqual(lamb(), 42) + + def test_annotate_qualname(self): + code = """ + def f() -> None: + def nested() -> None: pass + return nested + class Outer: + x: int + def method(self, x: int): + pass + """ + ns = run_code(code) + method = ns["Outer"].method + self.assertEqual(ns["f"].__annotate__.__qualname__, "f.__annotate__") + self.assertEqual(ns["f"]().__annotate__.__qualname__, "f.<locals>.nested.__annotate__") + self.assertEqual(method.__annotate__.__qualname__, "Outer.method.__annotate__") + self.assertEqual(ns["Outer"].__annotate__.__qualname__, "Outer.__annotate__") + + # gh-138349 + def test_module_level_annotation_plus_listcomp(self): + cases = [ + """ + def report_error(): + pass + try: + [0 for name_2 in unique_name_0 if (lambda: name_2)] + except: + pass + annotated_name: 0 + """, + """ + class Generic: + pass + try: + [0 for name_2 in unique_name_0 if (0 for unique_name_1 in unique_name_2 for unique_name_3 in name_2)] + except: + pass + annotated_name: 0 + """, + """ + class Generic: + pass + annotated_name: 0 + try: + [0 for name_2 in [[0]] for unique_name_1 in unique_name_2 if (lambda: name_2)] + except: + pass + """, + ] + for code in cases: + with self.subTest(code=code): + mod = build_module(code) + annos = mod.__annotations__ + self.assertEqual(annos, {"annotated_name": 0}) diff --git a/Lib/test/test_type_cache.py b/Lib/test/test_type_cache.py index 7469a1047f81d70..0390e6496109ec0 100644 --- a/Lib/test/test_type_cache.py +++ b/Lib/test/test_type_cache.py @@ -1,4 +1,5 @@ """ Tests for the internal type cache in CPython. """ +import collections.abc import dis import unittest import warnings @@ -114,6 +115,25 @@ class HolderSub(Holder): Holder.set_value() HolderSub.value + def test_abc_register_invalidates_subclass_versions(self): + class Parent: + pass + + class Child(Parent): + pass + + type_assign_version(Parent) + type_assign_version(Child) + parent_version = type_get_version(Parent) + child_version = type_get_version(Child) + if parent_version == 0 or child_version == 0: + self.skipTest("Could not assign valid type versions") + + collections.abc.Mapping.register(Parent) + + self.assertEqual(type_get_version(Parent), 0) + self.assertEqual(type_get_version(Child), 0) + @support.cpython_only class TypeCacheWithSpecializationTests(unittest.TestCase): def tearDown(self): diff --git a/Lib/test/test_type_comments.py b/Lib/test/test_type_comments.py index ee8939f62d082cd..d827ac271085bdd 100644 --- a/Lib/test/test_type_comments.py +++ b/Lib/test/test_type_comments.py @@ -1,6 +1,7 @@ import ast import sys import unittest +from test.support import import_helper funcdef = """\ @@ -344,7 +345,7 @@ def test_longargs(self): todo = set(t.name[1:]) self.assertEqual(len(t.args.args) + len(t.args.posonlyargs), len(todo) - bool(t.args.vararg) - bool(t.args.kwarg)) - self.assertTrue(t.name.startswith('f'), t.name) + self.assertStartsWith(t.name, 'f') for index, c in enumerate(t.name[1:]): todo.remove(c) if c == 'v': @@ -391,6 +392,16 @@ def check_both_ways(source): check_both_ways("pass # type: ignorewhatever\n") check_both_ways("pass # type: ignoreé\n") + def test_non_utf8_type_comment_with_ignore_cookie(self): + _testcapi = import_helper.import_module('_testcapi') + flags = 0x0800 | 0x1000 # PyCF_IGNORE_COOKIE | PyCF_TYPE_COMMENTS + with self.assertRaises(UnicodeDecodeError): + _testcapi.Py_CompileStringExFlags( + b"a=1 # type: \x80", "<test>", 256, flags) + with self.assertRaises(UnicodeDecodeError): + _testcapi.Py_CompileStringExFlags( + b"def a(f=8, #type: \x80\n\x80", "<test>", 256, flags) + def test_func_type_input(self): def parse_func_type_input(source): diff --git a/Lib/test/test_type_params.py b/Lib/test/test_type_params.py index 0f393def8272713..27660379ec43a37 100644 --- a/Lib/test/test_type_params.py +++ b/Lib/test/test_type_params.py @@ -1,4 +1,5 @@ import annotationlib +import inspect import textwrap import types import unittest @@ -152,6 +153,13 @@ def test_incorrect_mro_explicit_object(self): with self.assertRaisesRegex(TypeError, r"\(MRO\) for bases object, Generic"): class My[X](object): ... + def test_compile_error_in_type_param_bound(self): + # This should not crash, see gh-145187 + check_syntax_error( + self, + "if True:\n class h[l:{7for*()in 0}]:2" + ) + class TypeParamsNonlocalTest(unittest.TestCase): def test_nonlocal_disallowed_01(self): @@ -1439,6 +1447,30 @@ def f[T: int = int, **P = int, *Ts = int](): pass self.assertIs(annotationlib.call_evaluate_function(case, annotationlib.Format.FORWARDREF), int) self.assertEqual(annotationlib.call_evaluate_function(case, annotationlib.Format.STRING), 'int') + def test_signature(self): + # gh-151665: the ".format" parameter of compiler-generated evaluators + # used to break inspect.signature(). It should show up as "format". + type Alias = int + def f[T: int = int, **P = int, *Ts = int](): pass + T, P, Ts = f.__type_params__ + def g[T: (int, str)](): pass + T3, = g.__type_params__ + cases = [ + Alias.evaluate_value, + T.evaluate_bound, + T.evaluate_default, + P.evaluate_default, + Ts.evaluate_default, + T3.evaluate_constraints, + ] + for case in cases: + with self.subTest(case=case): + sig = inspect.signature(case) + self.assertEqual(str(sig), '(format=1, /)') + param, = sig.parameters.values() + self.assertEqual(param.name, 'format') + self.assertIs(param.kind, inspect.Parameter.POSITIONAL_ONLY) + def test_constraints(self): def f[T: (int, str)](): pass T, = f.__type_params__ diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 3552b6b4ef846c9..bf10dc2e31701f7 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -2,7 +2,7 @@ from test.support import ( run_with_locale, cpython_only, no_rerun, - MISSING_C_DOCSTRINGS, EqualToForwardRef, + MISSING_C_DOCSTRINGS, EqualToForwardRef, check_disallow_instantiation, ) from test.support.script_helper import assert_python_ok from test.support.import_helper import import_fresh_module @@ -517,8 +517,8 @@ def test(f, format_spec, result): # and a number after the decimal. This is tricky, because # a totally empty format specifier means something else. # So, just use a sign flag - test(1e200, '+g', '+1e+200') - test(1e200, '+', '+1e+200') + test(1.25e200, '+g', '+1.25e+200') + test(1.25e200, '+', '+1.25e+200') test(1.1e200, '+g', '+1.1e+200') test(1.1e200, '+', '+1.1e+200') @@ -827,15 +827,15 @@ def test_instancecheck_and_subclasscheck(self): self.assertIsInstance(True, x) self.assertIsInstance('a', x) self.assertNotIsInstance(None, x) - self.assertTrue(issubclass(int, x)) - self.assertTrue(issubclass(bool, x)) - self.assertTrue(issubclass(str, x)) - self.assertFalse(issubclass(type(None), x)) + self.assertIsSubclass(int, x) + self.assertIsSubclass(bool, x) + self.assertIsSubclass(str, x) + self.assertNotIsSubclass(type(None), x) for x in (int | None, typing.Union[int, None]): with self.subTest(x=x): self.assertIsInstance(None, x) - self.assertTrue(issubclass(type(None), x)) + self.assertIsSubclass(type(None), x) for x in ( int | collections.abc.Mapping, @@ -844,8 +844,8 @@ def test_instancecheck_and_subclasscheck(self): with self.subTest(x=x): self.assertIsInstance({}, x) self.assertNotIsInstance((), x) - self.assertTrue(issubclass(dict, x)) - self.assertFalse(issubclass(list, x)) + self.assertIsSubclass(dict, x) + self.assertNotIsSubclass(list, x) def test_instancecheck_and_subclasscheck_order(self): T = typing.TypeVar('T') @@ -857,7 +857,7 @@ def test_instancecheck_and_subclasscheck_order(self): for x in will_resolve: with self.subTest(x=x): self.assertIsInstance(1, x) - self.assertTrue(issubclass(int, x)) + self.assertIsSubclass(int, x) wont_resolve = ( T | int, @@ -890,7 +890,7 @@ class BadMeta(type): def __subclasscheck__(cls, sub): 1/0 x = int | BadMeta('A', (), {}) - self.assertTrue(issubclass(int, x)) + self.assertIsSubclass(int, x) self.assertRaises(ZeroDivisionError, issubclass, list, x) def test_or_type_operator_with_TypeVar(self): @@ -1148,8 +1148,7 @@ def test_or_type_operator_reference_cycle(self): msg='Check for union reference leak.') def test_instantiation(self): - with self.assertRaises(TypeError): - types.UnionType() + check_disallow_instantiation(self, types.UnionType) self.assertIs(int, types.UnionType[int]) self.assertIs(int, types.UnionType[int, int]) self.assertEqual(int | str, types.UnionType[int, str]) @@ -1399,7 +1398,7 @@ def test_new_class_basics(self): def test_new_class_subclass(self): C = types.new_class("C", (int,)) - self.assertTrue(issubclass(C, int)) + self.assertIsSubclass(C, int) def test_new_class_meta(self): Meta = self.Meta @@ -1444,7 +1443,7 @@ def func(ns): bases=(int,), kwds=dict(metaclass=Meta, z=2), exec_body=func) - self.assertTrue(issubclass(C, int)) + self.assertIsSubclass(C, int) self.assertIsInstance(C, Meta) self.assertEqual(C.x, 0) self.assertEqual(C.y, 1) @@ -2114,6 +2113,21 @@ class Spam(types.SimpleNamespace): self.assertIs(type(spam2), Spam) self.assertEqual(vars(spam2), {'ham': 5, 'eggs': 9}) + def test_replace_invalid_subtype(self): + # See https://github.com/python/cpython/issues/143636. + class MyNS(types.SimpleNamespace): + def __new__(cls, *args, **kwargs): + if created: + return 12345 + return super().__new__(cls) + + created = False + ns = MyNS() + created = True + err = (r"^expect types\.SimpleNamespace type, " + r"but .+\.MyNS\(\) returned 'int' object") + self.assertRaisesRegex(TypeError, err, copy.replace, ns) + def test_fake_namespace_compare(self): # Issue #24257: Incorrect use of PyObject_IsInstance() caused # SystemError. @@ -2241,8 +2255,8 @@ def foo(): return gen self.assertIs(wrapper.__name__, gen.__name__) # Test AttributeErrors - for name in {'gi_running', 'gi_frame', 'gi_code', 'gi_yieldfrom', - 'cr_running', 'cr_frame', 'cr_code', 'cr_await'}: + for name in {'gi_running', 'gi_frame', 'gi_code', 'gi_yieldfrom', 'gi_suspended', + 'cr_running', 'cr_frame', 'cr_code', 'cr_await', 'cr_suspended'}: with self.assertRaises(AttributeError): getattr(wrapper, name) @@ -2251,14 +2265,17 @@ def foo(): return gen gen.gi_frame = object() gen.gi_code = object() gen.gi_yieldfrom = object() + gen.gi_suspended = object() self.assertIs(wrapper.gi_running, gen.gi_running) self.assertIs(wrapper.gi_frame, gen.gi_frame) self.assertIs(wrapper.gi_code, gen.gi_code) self.assertIs(wrapper.gi_yieldfrom, gen.gi_yieldfrom) + self.assertIs(wrapper.gi_suspended, gen.gi_suspended) self.assertIs(wrapper.cr_running, gen.gi_running) self.assertIs(wrapper.cr_frame, gen.gi_frame) self.assertIs(wrapper.cr_code, gen.gi_code) self.assertIs(wrapper.cr_await, gen.gi_yieldfrom) + self.assertIs(wrapper.cr_suspended, gen.gi_suspended) wrapper.close() gen.close.assert_called_once_with() @@ -2377,7 +2394,7 @@ def foo(): return gen self.assertIs(wrapper.__await__(), gen) for name in ('__name__', '__qualname__', 'gi_code', - 'gi_running', 'gi_frame'): + 'gi_running', 'gi_frame', 'gi_suspended'): self.assertIs(getattr(foo(), name), getattr(gen, name)) self.assertIs(foo().cr_code, gen.gi_code) @@ -2440,8 +2457,8 @@ def coro(): self.assertEqual(repr(wrapper), str(wrapper)) self.assertTrue(set(dir(wrapper)).issuperset({ '__await__', '__iter__', '__next__', 'cr_code', 'cr_running', - 'cr_frame', 'gi_code', 'gi_frame', 'gi_running', 'send', - 'close', 'throw'})) + 'cr_frame', 'cr_suspended', 'gi_code', 'gi_frame', 'gi_running', + 'gi_suspended', 'send', 'close', 'throw'})) class FunctionTests(unittest.TestCase): @@ -2513,15 +2530,16 @@ class SubinterpreterTests(unittest.TestCase): def setUpClass(cls): global interpreters try: - from test.support import interpreters + from concurrent import interpreters except ModuleNotFoundError: raise unittest.SkipTest('subinterpreters required') - import test.support.interpreters.channels + from test.support import channels # noqa: F401 + cls.create_channel = staticmethod(channels.create) @cpython_only @no_rerun('channels (and queues) might have a refleak; see gh-122199') def test_static_types_inherited_slots(self): - rch, sch = interpreters.channels.create() + rch, sch = self.create_channel() script = textwrap.dedent(""" import test.support @@ -2547,7 +2565,7 @@ def collate_results(raw): main_results = collate_results(raw) interp = interpreters.create() - interp.exec('from test.support import interpreters') + interp.exec('from concurrent import interpreters') interp.prepare_main(sch=sch) interp.exec(script) raw = rch.recv_nowait() diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index 8c55ba4623e7191..c8b11b3cfdc6d89 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -13,7 +13,7 @@ import pickle import re import sys -from unittest import TestCase, main, skip +from unittest import TestCase, main from unittest.mock import patch from copy import copy, deepcopy @@ -52,6 +52,8 @@ from test.support import ( captured_stderr, cpython_only, infinite_recursion, requires_docstrings, import_helper, run_code, EqualToForwardRef, + exceeds_recursion_limit, skip_if_huge_c_stack, skip_wasi_stack_overflow, + skip_emscripten_stack_overflow, ) from test.typinganndata import ( ann_module695, mod_generics_cache, _typed_dict_helper, @@ -762,6 +764,16 @@ class A(Generic[T, P, U]): ... self.assertEqual(A[float, [range]].__args__, (float, (range,), float)) self.assertEqual(A[float, [range], int].__args__, (float, (range,), int)) + def test_paramspec_and_typevar_specialization_2(self): + T = TypeVar("T") + P = ParamSpec('P', default=...) + U = TypeVar("U", default=float) + self.assertEqual(P.__default__, ...) + class A(Generic[T, P, U]): ... + self.assertEqual(A[float].__args__, (float, ..., float)) + self.assertEqual(A[float, [range]].__args__, (float, (range,), float)) + self.assertEqual(A[float, [range], int].__args__, (float, (range,), int)) + def test_typevartuple_none(self): U = TypeVarTuple('U') U_None = TypeVarTuple('U_None', default=None) @@ -2267,6 +2279,15 @@ class Ints(enum.IntEnum): self.assertEqual(Union[Literal[1], Literal[Ints.B], Literal[True]].__args__, (Literal[1], Literal[Ints.B], Literal[True])) + def test_allow_non_types_in_or(self): + # gh-140348: Test that using | with a Union object allows things that are + # not allowed by is_unionable(). + U1 = Union[int, str] + self.assertEqual(U1 | float, Union[int, str, float]) + self.assertEqual(U1 | "float", Union[int, str, "float"]) + self.assertEqual(float | U1, Union[float, int, str]) + self.assertEqual("float" | U1, Union["float", int, str]) + class TupleTests(BaseTestCase): @@ -2720,8 +2741,14 @@ def test_args(self): self.assertEqual(Literal[1, 2, 3].__args__, (1, 2, 3)) self.assertEqual(Literal[1, 2, 3, 3].__args__, (1, 2, 3)) self.assertEqual(Literal[1, Literal[2], Literal[3, 4]].__args__, (1, 2, 3, 4)) - # Mutable arguments will not be deduplicated - self.assertEqual(Literal[[], []].__args__, ([], [])) + # Unhashable arguments will be deduplicated too + self.assertEqual(Literal[[], []].__args__, ([],)) + self.assertEqual(Literal[{"a": 1}, {"a": 1}].__args__, ({"a": 1},)) + self.assertEqual( + Literal[1, {'a': 'b'}, 2, {'a': 'b'}, 3].__args__, + (1, {'a': 'b'}, 2, 3), + ) + self.assertEqual(Literal[{1}, {1}, {2}, {2}].__args__, ({1}, {2})) def test_flatten(self): l1 = Literal[Literal[1], Literal[2], Literal[3]] @@ -4674,6 +4701,34 @@ class D(Generic[T]): pass with self.assertRaises(TypeError): D[()] + def test_generic_init_subclass_not_called_error(self): + notes = ["Note: this exception may have been caused by " + r"'GenericTests.test_generic_init_subclass_not_called_error.<locals>.Base.__init_subclass__' " + "(or the '__init_subclass__' method on a superclass) not calling 'super().__init_subclass__()'"] + + class Base: + def __init_subclass__(cls) -> None: + # Oops, I forgot super().__init_subclass__()! + pass + + with self.subTest(): + class Sub(Base, Generic[T]): + pass + + with self.assertRaises(AttributeError) as cm: + Sub[int] + + self.assertEqual(cm.exception.__notes__, notes) + + with self.subTest(): + class Sub[U](Base): + pass + + with self.assertRaises(AttributeError) as cm: + Sub[int] + + self.assertEqual(cm.exception.__notes__, notes) + def test_generic_subclass_checks(self): for typ in [list[int], List[int], tuple[int, str], Tuple[int, str], @@ -4873,6 +4928,17 @@ class MM2(collections.abc.MutableMapping, MutableMapping[str, str]): pass self.assertEqual(MM2.__bases__, (collections.abc.MutableMapping, Generic)) + @cpython_only + @skip_if_huge_c_stack() + @skip_wasi_stack_overflow() + @skip_emscripten_stack_overflow() + def test_parameters_deep_recursion(self): + x = [0] + for _ in range(exceeds_recursion_limit()): + x = [x] + with self.assertRaisesRegex(RecursionError, "in __parameter__ calculation"): + list[x].__parameters__ + def test_orig_bases(self): T = TypeVar('T') class C(typing.Dict[str, T]): ... @@ -5638,6 +5704,27 @@ def foo(x: T): foo(42) + def test_genericalias_instance_isclass(self): + # test against user-defined generic classes + T = TypeVar('T') + + class Node(Generic[T]): + def __init__(self, label: T, + left: 'Node[T] | None' = None, + right: 'Node[T] | None' = None): + self.label = label + self.left = left + self.right = right + + self.assertTrue(inspect.isclass(Node)) + self.assertFalse(inspect.isclass(Node[int])) + self.assertFalse(inspect.isclass(Node[str])) + + # test against standard generic classes + self.assertFalse(inspect.isclass(set[int])) + self.assertFalse(inspect.isclass(list[bytes])) + self.assertFalse(inspect.isclass(dict[str, str])) + def test_implicit_any(self): T = TypeVar('T') @@ -5797,6 +5884,23 @@ class A: with self.assertRaises(TypeError): a[int] + def test_return_non_tuple_while_unpacking(self): + # GH-138497: GenericAlias objects didn't ensure that __typing_subst__ actually + # returned a tuple + class EvilTypeVar: + __typing_is_unpacked_typevartuple__ = True + def __typing_prepare_subst__(*_): + return None # any value + def __typing_subst__(*_): + return 42 # not tuple + + evil = EvilTypeVar() + # Create a dummy TypeAlias that will be given the evil generic from + # above. + type type_alias[*_] = 0 + with self.assertRaisesRegex(TypeError, ".+__typing_subst__.+tuple.+int.*"): + type_alias[evil][0] + class ClassVarTests(BaseTestCase): @@ -6622,11 +6726,7 @@ def test_get_type_hints_modules(self): self.assertEqual(gth(ann_module2), {}) self.assertEqual(gth(ann_module3), {}) - @skip("known bug") def test_get_type_hints_modules_forwardref(self): - # FIXME: This currently exposes a bug in typing. Cached forward references - # don't account for the case where there are multiple types of the same - # name coming from different modules in the same program. mgc_hints = {'default_a': Optional[mod_generics_cache.A], 'default_b': Optional[mod_generics_cache.B]} self.assertEqual(gth(mod_generics_cache), mgc_hints) @@ -6714,6 +6814,24 @@ def test_get_type_hints_wrapped_decoratored_func(self): self.assertEqual(gth(ForRefExample.func), expects) self.assertEqual(gth(ForRefExample.nested), expects) + def test_get_type_hints_wrapped_cycle_self(self): + # gh-146553: __wrapped__ self-reference must raise ValueError, + # not loop forever. + def f(x: int) -> str: ... + f.__wrapped__ = f + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + get_type_hints(f) + + def test_get_type_hints_wrapped_cycle_mutual(self): + # gh-146553: mutual __wrapped__ cycle (a -> b -> a) must raise + # ValueError, not loop forever. + def a(): ... + def b(): ... + a.__wrapped__ = b + b.__wrapped__ = a + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + get_type_hints(a) + def test_get_type_hints_annotated(self): def foobar(x: List['X']): ... X = Annotated[int, (1, 10)] @@ -6859,12 +6977,10 @@ def test_forward_ref_and_final(self): self.assertEqual(hints, {'value': Final}) def test_top_level_class_var(self): - # https://bugs.python.org/issue45166 - with self.assertRaisesRegex( - TypeError, - r'typing.ClassVar\[int\] is not valid as type argument', - ): - get_type_hints(ann_module6) + # This is not meaningful but we don't raise for it. + # https://github.com/python/cpython/issues/133959 + hints = get_type_hints(ann_module6) + self.assertEqual(hints, {'wrong': ClassVar[int]}) def test_get_type_hints_typeddict(self): self.assertEqual(get_type_hints(TotalMovie), {'title': str, 'year': int}) @@ -6967,6 +7083,11 @@ def foo(a: 'Callable[..., T]'): self.assertEqual(get_type_hints(foo, globals(), locals()), {'a': Callable[..., T]}) + def test_special_forms_no_forward(self): + def f(x: ClassVar[int]): + pass + self.assertEqual(get_type_hints(f), {'x': ClassVar[int]}) + def test_special_forms_forward(self): class C: @@ -6982,8 +7103,9 @@ class CF: self.assertEqual(get_type_hints(C, globals())['b'], Final[int]) self.assertEqual(get_type_hints(C, globals())['x'], ClassVar) self.assertEqual(get_type_hints(C, globals())['y'], Final) - with self.assertRaises(TypeError): - get_type_hints(CF, globals()), + lfi = get_type_hints(CF, globals())['b'] + self.assertIs(get_origin(lfi), list) + self.assertEqual(get_args(lfi), (Final[int],)) def test_union_forward_recursion(self): ValueList = List['Value'] @@ -7111,6 +7233,19 @@ def add_right(self, node: 'Node[T]' = None): right_hints = get_type_hints(t.add_right, globals(), locals()) self.assertEqual(right_hints['node'], Node[T]) + def test_stringified_typeddict(self): + ns = run_code( + """ + from __future__ import annotations + from typing import TypedDict + class TD[UniqueT](TypedDict): + a: UniqueT + """ + ) + TD = ns['TD'] + self.assertEqual(TD.__annotations__, {'a': EqualToForwardRef('UniqueT', owner=TD, module=TD.__module__)}) + self.assertEqual(get_type_hints(TD), {'a': TD.__type_params__[0]}) + class GetUtilitiesTestCase(TestCase): def test_get_origin(self): @@ -7216,34 +7351,132 @@ class C(Generic[T]): pass class EvaluateForwardRefTests(BaseTestCase): def test_evaluate_forward_ref(self): int_ref = ForwardRef('int') - missing = ForwardRef('missing') + self.assertIs(typing.evaluate_forward_ref(int_ref), int) self.assertIs( typing.evaluate_forward_ref(int_ref, type_params=()), int, ) + self.assertIs( + typing.evaluate_forward_ref(int_ref, format=annotationlib.Format.VALUE), + int, + ) self.assertIs( typing.evaluate_forward_ref( - int_ref, type_params=(), format=annotationlib.Format.FORWARDREF, + int_ref, format=annotationlib.Format.FORWARDREF, ), int, ) + self.assertEqual( + typing.evaluate_forward_ref( + int_ref, format=annotationlib.Format.STRING, + ), + 'int', + ) + + def test_evaluate_forward_ref_undefined(self): + missing = ForwardRef('missing') + with self.assertRaises(NameError): + typing.evaluate_forward_ref(missing) self.assertIs( typing.evaluate_forward_ref( - missing, type_params=(), format=annotationlib.Format.FORWARDREF, + missing, format=annotationlib.Format.FORWARDREF, ), missing, ) self.assertEqual( typing.evaluate_forward_ref( - int_ref, type_params=(), format=annotationlib.Format.STRING, + missing, format=annotationlib.Format.STRING, ), - 'int', + "missing", + ) + + def test_evaluate_forward_ref_nested(self): + ref = ForwardRef("int | list['str']") + self.assertEqual( + typing.evaluate_forward_ref(ref), + int | list[str], + ) + self.assertEqual( + typing.evaluate_forward_ref(ref, format=annotationlib.Format.FORWARDREF), + int | list[str], + ) + self.assertEqual( + typing.evaluate_forward_ref(ref, format=annotationlib.Format.STRING), + "int | list['str']", + ) + + why = ForwardRef('"\'str\'"') + self.assertIs(typing.evaluate_forward_ref(why), str) + + def test_evaluate_forward_ref_none(self): + none_ref = ForwardRef('None') + self.assertIs(typing.evaluate_forward_ref(none_ref), None) + + def test_globals(self): + A = "str" + ref = ForwardRef('list[A]') + with self.assertRaises(NameError): + typing.evaluate_forward_ref(ref) + self.assertEqual( + typing.evaluate_forward_ref(ref, globals={'A': A}), + list[str], + ) + + def test_owner(self): + ref = ForwardRef("A") + + with self.assertRaises(NameError): + typing.evaluate_forward_ref(ref) + + # We default to the globals of `owner`, + # so it no longer raises `NameError` + self.assertIs( + typing.evaluate_forward_ref(ref, owner=Loop), A ) - def test_evaluate_forward_ref_no_type_params(self): - ref = ForwardRef('int') - self.assertIs(typing.evaluate_forward_ref(ref), int) + def test_inherited_owner(self): + # owner passed to evaluate_forward_ref + ref = ForwardRef("list['A']") + self.assertEqual( + typing.evaluate_forward_ref(ref, owner=Loop), + list[A], + ) + # owner set on the ForwardRef + ref = ForwardRef("list['A']", owner=Loop) + self.assertEqual( + typing.evaluate_forward_ref(ref), + list[A], + ) + + def test_partial_evaluation(self): + ref = ForwardRef("list[A]") + with self.assertRaises(NameError): + typing.evaluate_forward_ref(ref) + + self.assertEqual( + typing.evaluate_forward_ref(ref, format=annotationlib.Format.FORWARDREF), + list[EqualToForwardRef('A')], + ) + + def test_with_module(self): + from test.typinganndata import fwdref_module + + typing.evaluate_forward_ref( + fwdref_module.fw,) + + def test_evaluate_forward_ref_string_format(self): + # Test evaluating forward references in STRING format + # does not 'leak' internal names + # See https://github.com/python/cpython/issues/150641 + + def f(arg: unknown | str | int | list[str] | tuple[int, ...]): ... + + ref = annotationlib.get_annotations(f, format=annotationlib.Format.FORWARDREF)['arg'] + self.assertEqual( + typing.evaluate_forward_ref(ref, format=annotationlib.Format.STRING), + "unknown | str | int | list[str] | tuple[int, ...]", + ) class CollectionsAbcTests(BaseTestCase): @@ -7339,6 +7572,16 @@ def test_mutablesequence(self): self.assertIsInstance([], typing.MutableSequence) self.assertNotIsInstance((), typing.MutableSequence) + def test_bytestring(self): + with self.assertWarns(DeprecationWarning): + self.assertIsInstance(b'', typing.ByteString) + with self.assertWarns(DeprecationWarning): + self.assertIsInstance(bytearray(b''), typing.ByteString) + with self.assertWarns(DeprecationWarning): + class Foo(typing.ByteString): ... + with self.assertWarns(DeprecationWarning): + class Bar(typing.ByteString, typing.Awaitable): ... + def test_list(self): self.assertIsSubclass(list, typing.List) @@ -8538,6 +8781,36 @@ class Child(Base1, Base2): self.assertEqual(Child.__required_keys__, frozenset(['a'])) self.assertEqual(Child.__optional_keys__, frozenset()) + def test_inheritance_pep563(self): + def _make_td(future, class_name, annos, base, extra_names=None): + lines = [] + if future: + lines.append('from __future__ import annotations') + lines.append('from typing import TypedDict') + lines.append(f'class {class_name}({base}):') + for name, anno in annos.items(): + lines.append(f' {name}: {anno}') + code = '\n'.join(lines) + ns = run_code(code, extra_names) + return ns[class_name] + + for base_future in (True, False): + for child_future in (True, False): + with self.subTest(base_future=base_future, child_future=child_future): + base = _make_td( + base_future, "Base", {"base": "int"}, "TypedDict" + ) + self.assertIsNotNone(base.__annotate__) + child = _make_td( + child_future, "Child", {"child": "int"}, "Base", {"Base": base} + ) + base_anno = ForwardRef("int", module="builtins", owner=base) if base_future else int + child_anno = ForwardRef("int", module="builtins", owner=child) if child_future else int + self.assertEqual(base.__annotations__, {'base': base_anno}) + self.assertEqual( + child.__annotations__, {'child': child_anno, 'base': base_anno} + ) + def test_required_notrequired_keys(self): self.assertEqual(NontotalMovie.__required_keys__, frozenset({"title"})) @@ -9691,6 +9964,19 @@ class B(str): ... self.assertIs(type(field_c2.__metadata__[0]), float) self.assertIs(type(field_c3.__metadata__[0]), bool) + def test_forwardref_partial_evaluation(self): + # Test that Annotated partially evaluates if it contains a ForwardRef + # See: https://github.com/python/cpython/issues/137706 + def f(x: Annotated[undefined, '']): pass + + ann = annotationlib.get_annotations(f, format=annotationlib.Format.FORWARDREF) + + # Test that the attributes are retrievable from the partially evaluated annotation + x_ann = ann['x'] + self.assertIs(get_origin(x_ann), Annotated) + self.assertEqual(x_ann.__origin__, EqualToForwardRef('undefined', owner=f)) + self.assertEqual(x_ann.__metadata__, ('',)) + class TypeAliasTests(BaseTestCase): def test_canonical_usage_with_variable_annotation(self): @@ -10313,6 +10599,7 @@ def test_special_attrs(self): typing.AsyncIterable: 'AsyncIterable', typing.AsyncIterator: 'AsyncIterator', typing.Awaitable: 'Awaitable', + typing.ByteString: 'ByteString', typing.Callable: 'Callable', typing.ChainMap: 'ChainMap', typing.Collection: 'Collection', @@ -10624,6 +10911,10 @@ def test_no_attributes(self): with self.assertRaises(AttributeError): type(NoDefault).foo + def test_no_subclassing(self): + with self.assertRaises(TypeError): + class Test(type(NoDefault)): ... + class AllTests(BaseTestCase): """Tests for __all__.""" @@ -10731,6 +11022,9 @@ def test_eq(self): with self.assertWarns(DeprecationWarning): self.assertNotEqual(int, typing._UnionGenericAlias) + def test_hashable(self): + self.assertEqual(hash(typing._UnionGenericAlias), hash(Union)) + def load_tests(loader, tests, pattern): import doctest diff --git a/Lib/test/test_ucn.py b/Lib/test/test_ucn.py index 0e2c25aaff2fe9d..fb8e98af25bb957 100644 --- a/Lib/test/test_ucn.py +++ b/Lib/test/test_ucn.py @@ -88,6 +88,9 @@ def test_hangul_syllables(self): self.checkletter("HANGUL SYLLABLE HWEOK", "\ud6f8") self.checkletter("HANGUL SYLLABLE HIH", "\ud7a3") + self.checkletter("haNGul SYllABle WAe", '\uc65c') + self.checkletter("HAngUL syLLabLE waE", '\uc65c') + self.assertRaises(ValueError, unicodedata.name, "\ud7a4") def test_cjk_unified_ideographs(self): @@ -103,6 +106,35 @@ def test_cjk_unified_ideographs(self): self.checkletter("CJK UNIFIED IDEOGRAPH-2B81D", "\U0002B81D") self.checkletter("CJK UNIFIED IDEOGRAPH-3134A", "\U0003134A") + self.checkletter("cjK UniFIeD idEogRAph-3aBc", "\u3abc") + self.checkletter("CJk uNIfiEd IDeOGraPH-3AbC", "\u3abc") + self.checkletter("cjK UniFIeD idEogRAph-2aBcD", "\U0002abcd") + self.checkletter("CJk uNIfiEd IDeOGraPH-2AbCd", "\U0002abcd") + + def test_tangut_ideographs(self): + self.checkletter("TANGUT IDEOGRAPH-17000", "\U00017000") + self.checkletter("TANGUT IDEOGRAPH-187F7", "\U000187f7") + self.checkletter("TANGUT IDEOGRAPH-18D00", "\U00018D00") + self.checkletter("TANGUT IDEOGRAPH-18D08", "\U00018d08") + self.checkletter("tangut ideograph-18d08", "\U00018d08") + + def test_egyptian_hieroglyphs(self): + self.checkletter("EGYPTIAN HIEROGLYPH-13460", "\U00013460") + self.checkletter("EGYPTIAN HIEROGLYPH-143FA", "\U000143fa") + self.checkletter("egyptian hieroglyph-143fa", "\U000143fa") + + def test_khitan_small_script_characters(self): + self.checkletter("KHITAN SMALL SCRIPT CHARACTER-18B00", "\U00018b00") + self.checkletter("KHITAN SMALL SCRIPT CHARACTER-18CD5", "\U00018cd5") + self.checkletter("KHITAN SMALL SCRIPT CHARACTER-18CFF", "\U00018cff") + self.checkletter("KHITAN SMALL SCRIPT CHARACTER-18CFF", "\U00018cff") + self.checkletter("khitan small script character-18cff", "\U00018cff") + + def test_nushu_characters(self): + self.checkletter("NUSHU CHARACTER-1B170", "\U0001b170") + self.checkletter("NUSHU CHARACTER-1B2FB", "\U0001b2fb") + self.checkletter("nushu character-1b2fb", "\U0001b2fb") + def test_bmp_characters(self): for code in range(0x10000): char = chr(code) diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 8e3fef6b6fe4a06..21a7cb5da41792f 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -6,6 +6,7 @@ """ +from functools import partial import hashlib from http.client import HTTPException import sys @@ -18,20 +19,31 @@ cpython_only, check_disallow_instantiation, force_not_colorized, + is_resource_enabled, + findfile, ) +quicktest = not is_resource_enabled('cpu') + +def iterallchars(): + maxunicode = 0xffff if quicktest else sys.maxunicode + return map(chr, range(maxunicode + 1)) + class UnicodeMethodsTest(unittest.TestCase): # update this, if the database changes - expectedchecksum = '9e43ee3929471739680c0e705482b4ae1c4122e4' + expectedchecksum = ('486bf97d506d0ccf0e463fd1f40c51029805af5a' + if quicktest else + '9e43ee3929471739680c0e705482b4ae1c4122e4') - @requires_resource('cpu') def test_method_checksum(self): h = hashlib.sha1() - for i in range(sys.maxunicode + 1): - char = chr(i) - data = [ + for char in iterallchars(): + s1 = char + 'abc' + s2 = char + 'ABC' + s3 = char + '123' + data = ( # Predicates (single char) "01"[char.isalnum()], "01"[char.isalpha()], @@ -44,15 +56,15 @@ def test_method_checksum(self): "01"[char.isupper()], # Predicates (multiple chars) - "01"[(char + 'abc').isalnum()], - "01"[(char + 'abc').isalpha()], - "01"[(char + '123').isdecimal()], - "01"[(char + '123').isdigit()], - "01"[(char + 'abc').islower()], - "01"[(char + '123').isnumeric()], + "01"[s1.isalnum()], + "01"[s1.isalpha()], + "01"[s3.isdecimal()], + "01"[s3.isdigit()], + "01"[s1.islower()], + "01"[s3.isnumeric()], "01"[(char + ' \t').isspace()], - "01"[(char + 'abc').istitle()], - "01"[(char + 'ABC').isupper()], + "01"[s1.istitle()], + "01"[s2.isupper()], # Mappings (single char) char.lower(), @@ -60,54 +72,104 @@ def test_method_checksum(self): char.title(), # Mappings (multiple chars) - (char + 'abc').lower(), - (char + 'ABC').upper(), - (char + 'abc').title(), - (char + 'ABC').title(), + s1.lower(), + s2.upper(), + s1.title(), + s2.title(), - ] + ) h.update(''.join(data).encode('utf-8', 'surrogatepass')) result = h.hexdigest() self.assertEqual(result, self.expectedchecksum) -class UnicodeDatabaseTest(unittest.TestCase): - db = unicodedata -class UnicodeFunctionsTest(UnicodeDatabaseTest): +class UnicodeFunctionsTest(unittest.TestCase): + db = unicodedata + old = False # Update this if the database changes. Make sure to do a full rebuild # (e.g. 'make distclean && make') to get the correct checksum. - expectedchecksum = '23ab09ed4abdf93db23b97359108ed630dd8311d' + expectedchecksum = ('1ba453ec456896f1190d849b6e9b7c2e1a4128e0' + if quicktest else + '46ca89d9fe34881d0be3a4a4b29f5aa8c019640c') - @requires_resource('cpu') def test_function_checksum(self): + db = self.db data = [] h = hashlib.sha1() - for i in range(sys.maxunicode + 1): - char = chr(i) - data = [ + for char in iterallchars(): + data = "%.12g%.12g%.12g%s%s%s%s%s%s%s" % ( # Properties - format(self.db.digit(char, -1), '.12g'), - format(self.db.numeric(char, -1), '.12g'), - format(self.db.decimal(char, -1), '.12g'), - self.db.category(char), - self.db.bidirectional(char), - self.db.decomposition(char), - str(self.db.mirrored(char)), - str(self.db.combining(char)), - unicodedata.east_asian_width(char), - self.db.name(char, ""), - ] - h.update(''.join(data).encode("ascii")) + db.digit(char, -1), + db.numeric(char, -1), + db.decimal(char, -1), + db.category(char), + db.bidirectional(char), + db.decomposition(char), + db.mirrored(char), + db.combining(char), + db.east_asian_width(char), + db.name(char, ""), + ) + h.update(data.encode("ascii")) result = h.hexdigest() self.assertEqual(result, self.expectedchecksum) + def test_name(self): + name = self.db.name + self.assertRaises(ValueError, name, '\0') + self.assertRaises(ValueError, name, '\n') + self.assertRaises(ValueError, name, '\x1F') + self.assertRaises(ValueError, name, '\x7F') + self.assertRaises(ValueError, name, '\x9F') + self.assertRaises(ValueError, name, '\uFFFE') + self.assertRaises(ValueError, name, '\uFFFF') + self.assertRaises(ValueError, name, '\U0010FFFF') + self.assertEqual(name('\U0010FFFF', 42), 42) + + self.assertEqual(name(' '), 'SPACE') + self.assertEqual(name('1'), 'DIGIT ONE') + self.assertEqual(name('A'), 'LATIN CAPITAL LETTER A') + self.assertEqual(name('\xA0'), 'NO-BREAK SPACE') + self.assertEqual(name('\u0221', None), None if self.old else + 'LATIN SMALL LETTER D WITH CURL') + self.assertEqual(name('\u3400'), 'CJK UNIFIED IDEOGRAPH-3400') + self.assertEqual(name('\u9FA5'), 'CJK UNIFIED IDEOGRAPH-9FA5') + self.assertEqual(name('\uAC00'), 'HANGUL SYLLABLE GA') + self.assertEqual(name('\uD7A3'), 'HANGUL SYLLABLE HIH') + self.assertEqual(name('\uF900'), 'CJK COMPATIBILITY IDEOGRAPH-F900') + self.assertEqual(name('\uFA6A'), 'CJK COMPATIBILITY IDEOGRAPH-FA6A') + self.assertEqual(name('\uFBF9'), + 'ARABIC LIGATURE UIGHUR KIRGHIZ YEH WITH HAMZA ' + 'ABOVE WITH ALEF MAKSURA ISOLATED FORM') + self.assertEqual(name('\U00013460', None), None if self.old else + 'EGYPTIAN HIEROGLYPH-13460') + self.assertEqual(name('\U000143FA', None), None if self.old else + 'EGYPTIAN HIEROGLYPH-143FA') + self.assertEqual(name('\U00018B00', None), None if self.old else + 'KHITAN SMALL SCRIPT CHARACTER-18B00') + self.assertEqual(name('\U00018CD5', None), None if self.old else + 'KHITAN SMALL SCRIPT CHARACTER-18CD5') + self.assertEqual(name('\U00018CFF', None), None if self.old else + 'KHITAN SMALL SCRIPT CHARACTER-18CFF') + self.assertEqual(name('\U0001B170', None), None if self.old else + 'NUSHU CHARACTER-1B170') + self.assertEqual(name('\U0001B2FB', None), None if self.old else + 'NUSHU CHARACTER-1B2FB') + self.assertEqual(name('\U0001FBA8', None), None if self.old else + 'BOX DRAWINGS LIGHT DIAGONAL UPPER CENTRE TO ' + 'MIDDLE LEFT AND MIDDLE RIGHT TO LOWER CENTRE') + self.assertEqual(name('\U0002A6D6'), 'CJK UNIFIED IDEOGRAPH-2A6D6') + self.assertEqual(name('\U0002FA1D'), 'CJK COMPATIBILITY IDEOGRAPH-2FA1D') + self.assertEqual(name('\U000323AF', None), None if self.old else + 'CJK UNIFIED IDEOGRAPH-323AF') + @requires_resource('cpu') def test_name_inverse_lookup(self): - for i in range(sys.maxunicode + 1): - char = chr(i) - if looked_name := self.db.name(char, None): + for char in iterallchars(): + looked_name = self.db.name(char, None) + if looked_name is not None: self.assertEqual(self.db.lookup(looked_name), char) def test_no_names_in_pua(self): @@ -127,6 +189,17 @@ def test_lookup_nonexistant(self): "HANDBUG", "MODIFIER LETTER CYRILLIC SMALL QUESTION MARK", "???", + "CJK UNIFIED IDEOGRAPH-03400", + "CJK UNIFIED IDEOGRAPH-020000", + "CJK UNIFIED IDEOGRAPH-33FF", + "CJK UNIFIED IDEOGRAPH-F900", + "CJK UNIFIED IDEOGRAPH-13460", + "CJK UNIFIED IDEOGRAPH-17000", + "CJK UNIFIED IDEOGRAPH-18B00", + "CJK UNIFIED IDEOGRAPH-1B170", + "CJK COMPATIBILITY IDEOGRAPH-3400", + "TANGUT IDEOGRAPH-3400", + "HANGUL SYLLABLE AC00", ]: self.assertRaises(KeyError, self.db.lookup, nonexistent) @@ -138,6 +211,13 @@ def test_digit(self): self.assertEqual(self.db.digit('\U00020000', None), None) self.assertEqual(self.db.digit('\U0001D7FD'), 7) + # New in 13.0.0 + self.assertEqual(self.db.digit('\U0001fbf9', None), 9) + # New in 14.0.0 + self.assertEqual(self.db.digit('\U00016ac9', None), 9) + # New in 15.0.0 + self.assertEqual(self.db.digit('\U0001e4f9', None), 9) + self.assertRaises(TypeError, self.db.digit) self.assertRaises(TypeError, self.db.digit, 'xx') self.assertRaises(ValueError, self.db.digit, 'x') @@ -147,9 +227,28 @@ def test_numeric(self): self.assertEqual(self.db.numeric('9'), 9) self.assertEqual(self.db.numeric('\u215b'), 0.125) self.assertEqual(self.db.numeric('\u2468'), 9.0) - self.assertEqual(self.db.numeric('\ua627'), 7.0) self.assertEqual(self.db.numeric('\U00020000', None), None) - self.assertEqual(self.db.numeric('\U0001012A'), 9000) + + # New in 4.1.0 + self.assertEqual(self.db.numeric('\U0001012A', None), None if self.old else 9000) + # Changed in 4.1.0 + self.assertEqual(self.db.numeric('\u5793', None), 1e20 if self.old else None) + # New in 5.0.0 + self.assertEqual(self.db.numeric('\u07c0', None), None if self.old else 0.0) + # New in 5.1.0 + self.assertEqual(self.db.numeric('\ua627', None), None if self.old else 7.0) + # Changed in 5.2.0 + self.assertEqual(self.db.numeric('\u09f6'), 3.0 if self.old else 3/16) + # New in 6.0.0 + self.assertEqual(self.db.numeric('\u0b72', None), None if self.old else 0.25) + # New in 12.0.0 + self.assertEqual(self.db.numeric('\U0001ed3c', None), None if self.old else 0.5) + # New in 13.0.0 + self.assertEqual(self.db.numeric('\U0001fbf9', None), None if self.old else 9) + # New in 14.0.0 + self.assertEqual(self.db.numeric('\U00016ac9', None), None if self.old else 9) + # New in 15.0.0 + self.assertEqual(self.db.numeric('\U0001e4f9', None), None if self.old else 9) self.assertRaises(TypeError, self.db.numeric) self.assertRaises(TypeError, self.db.numeric, 'xx') @@ -163,6 +262,18 @@ def test_decimal(self): self.assertEqual(self.db.decimal('\U00020000', None), None) self.assertEqual(self.db.decimal('\U0001D7FD'), 7) + # New in 4.1.0 + self.assertEqual(self.db.decimal('\xb2', None), 2 if self.old else None) + self.assertEqual(self.db.decimal('\u1369', None), 1 if self.old else None) + # New in 5.0.0 + self.assertEqual(self.db.decimal('\u07c0', None), None if self.old else 0) + # New in 13.0.0 + self.assertEqual(self.db.decimal('\U0001fbf9', None), None if self.old else 9) + # New in 14.0.0 + self.assertEqual(self.db.decimal('\U00016ac9', None), None if self.old else 9) + # New in 15.0.0 + self.assertEqual(self.db.decimal('\U0001e4f9', None), None if self.old else 9) + self.assertRaises(TypeError, self.db.decimal) self.assertRaises(TypeError, self.db.decimal, 'xx') self.assertRaises(ValueError, self.db.decimal, 'x') @@ -172,7 +283,21 @@ def test_category(self): self.assertEqual(self.db.category('a'), 'Ll') self.assertEqual(self.db.category('A'), 'Lu') self.assertEqual(self.db.category('\U00020000'), 'Lo') - self.assertEqual(self.db.category('\U0001012A'), 'No') + + # New in 4.1.0 + self.assertEqual(self.db.category('\U0001012A'), 'Cn' if self.old else 'No') + self.assertEqual(self.db.category('\U000e01ef'), 'Cn' if self.old else 'Mn') + # New in 5.1.0 + self.assertEqual(self.db.category('\u0374'), 'Sk' if self.old else 'Lm') + # Changed in 13.0.0 + self.assertEqual(self.db.category('\u0b55'), 'Cn' if self.old else 'Mn') + self.assertEqual(self.db.category('\U0003134a'), 'Cn' if self.old else 'Lo') + # Changed in 14.0.0 + self.assertEqual(self.db.category('\u061d'), 'Cn' if self.old else 'Po') + self.assertEqual(self.db.category('\U0002b738'), 'Cn' if self.old else 'Lo') + # Changed in 15.0.0 + self.assertEqual(self.db.category('\u0cf3'), 'Cn' if self.old else 'Mc') + self.assertEqual(self.db.category('\U000323af'), 'Cn' if self.old else 'Lo') self.assertRaises(TypeError, self.db.category) self.assertRaises(TypeError, self.db.category, 'xx') @@ -183,6 +308,23 @@ def test_bidirectional(self): self.assertEqual(self.db.bidirectional('A'), 'L') self.assertEqual(self.db.bidirectional('\U00020000'), 'L') + # New in 4.1.0 + self.assertEqual(self.db.bidirectional('+'), 'ET' if self.old else 'ES') + self.assertEqual(self.db.bidirectional('\u0221'), '' if self.old else 'L') + self.assertEqual(self.db.bidirectional('\U000e01ef'), '' if self.old else 'NSM') + # New in 13.0.0 + self.assertEqual(self.db.bidirectional('\u0b55'), '' if self.old else 'NSM') + self.assertEqual(self.db.bidirectional('\U0003134a'), '' if self.old else 'L') + # New in 14.0.0 + self.assertEqual(self.db.bidirectional('\u061d'), '' if self.old else 'AL') + self.assertEqual(self.db.bidirectional('\U0002b738'), '' if self.old else 'L') + # New in 15.0.0 + self.assertEqual(self.db.bidirectional('\u0cf3'), '' if self.old else 'L') + self.assertEqual(self.db.bidirectional('\U000323af'), '' if self.old else 'L') + # New in 16.0.0 + self.assertEqual(self.db.bidirectional('\u0897'), '' if self.old else 'NSM') + self.assertEqual(self.db.bidirectional('\U0001fbef'), '' if self.old else 'ON') + self.assertRaises(TypeError, self.db.bidirectional) self.assertRaises(TypeError, self.db.bidirectional, 'xx') @@ -190,6 +332,26 @@ def test_decomposition(self): self.assertEqual(self.db.decomposition('\uFFFE'),'') self.assertEqual(self.db.decomposition('\u00bc'), '<fraction> 0031 2044 0034') + # New in 4.1.0 + self.assertEqual(self.db.decomposition('\u03f9'), '' if self.old else '<compat> 03A3') + # New in 13.0.0 + self.assertEqual(self.db.decomposition('\uab69'), '' if self.old else '<super> 028D') + self.assertEqual(self.db.decomposition('\U00011938'), '' if self.old else '11935 11930') + self.assertEqual(self.db.decomposition('\U0001fbf9'), '' if self.old else '<font> 0039') + # New in 14.0.0 + self.assertEqual(self.db.decomposition('\ua7f2'), '' if self.old else '<super> 0043') + self.assertEqual(self.db.decomposition('\U000107ba'), '' if self.old else '<super> 1DF1E') + # New in 15.0.0 + self.assertEqual(self.db.decomposition('\U0001e06d'), '' if self.old else '<super> 04B1') + # New in 16.0.0 + self.assertEqual(self.db.decomposition('\U0001CCD6'), '' if self.old else '<font> 0041') + + # Hangul characters + self.assertEqual(self.db.decomposition('\uAC00'), '1100 1161') + self.assertEqual(self.db.decomposition('\uD4DB'), '1111 1171 11B6') + self.assertEqual(self.db.decomposition('\uC2F8'), '110A 1161') + self.assertEqual(self.db.decomposition('\uD7A3'), '1112 1175 11C2') + self.assertRaises(TypeError, self.db.decomposition) self.assertRaises(TypeError, self.db.decomposition, 'xx') @@ -199,6 +361,16 @@ def test_mirrored(self): self.assertEqual(self.db.mirrored('\u2201'), 1) self.assertEqual(self.db.mirrored('\U00020000'), 0) + # New in 5.0.0 + self.assertEqual(self.db.mirrored('\u0f3a'), 0 if self.old else 1) + self.assertEqual(self.db.mirrored('\U0001d7c3'), 0 if self.old else 1) + # New in 11.0.0 + self.assertEqual(self.db.mirrored('\u29a1'), 1 if self.old else 0) + # New in 14.0.0 + self.assertEqual(self.db.mirrored('\u2e5c'), 0 if self.old else 1) + # New in 16.0.0 + self.assertEqual(self.db.mirrored('\u226D'), 0 if self.old else 1) + self.assertRaises(TypeError, self.db.mirrored) self.assertRaises(TypeError, self.db.mirrored, 'xx') @@ -208,9 +380,187 @@ def test_combining(self): self.assertEqual(self.db.combining('\u20e1'), 230) self.assertEqual(self.db.combining('\U00020000'), 0) + # New in 4.1.0 + self.assertEqual(self.db.combining('\u0350'), 0 if self.old else 230) + # New in 9.0.0 + self.assertEqual(self.db.combining('\U0001e94a'), 0 if self.old else 7) + # New in 13.0.0 + self.assertEqual(self.db.combining('\u1abf'), 0 if self.old else 220) + self.assertEqual(self.db.combining('\U00016ff1'), 0 if self.old else 6) + # New in 14.0.0 + self.assertEqual(self.db.combining('\u0c3c'), 0 if self.old else 7) + self.assertEqual(self.db.combining('\U0001e2ae'), 0 if self.old else 230) + # New in 15.0.0 + self.assertEqual(self.db.combining('\U00010efd'), 0 if self.old else 220) + # New in 16.0.0 + self.assertEqual(self.db.combining('\u0897'), 0 if self.old else 230) + self.assertRaises(TypeError, self.db.combining) self.assertRaises(TypeError, self.db.combining, 'xx') + def test_normalization(self): + # Test normalize() and is_normalized() + def check(ch, expected): + if isinstance(expected, str): + expected = [expected]*4 + forms = ('NFC', 'NFD', 'NFKC', 'NFKD') + result = [self.db.normalize(form, ch) for form in forms] + self.assertEqual(ascii(result), ascii(list(expected))) + self.assertEqual([self.db.is_normalized(form, ch) for form in forms], + [ch == y for x, y in zip(result, expected)]) + + check('', '') + check('A', 'A') + check(' ', ' ') + check('\U0010ffff', '\U0010ffff') + check('abc', 'abc') + # Broken in 4.0.0 + check('\u0340', '\u0300') + check('\u0300', '\u0300') + check('\U0002fa1d', '\U0002a600') + check('\U0002a600', '\U0002a600') + check('\u0344', '\u0308\u0301') + check('\u0308\u0301', '\u0308\u0301') + # Broken in 4.0.0 and 4.0.1 + check('\U0001d1bc', '\U0001d1ba\U0001d165') + check('\U0001d1ba\U0001d165', '\U0001d1ba\U0001d165') + check('\ufb2c', '\u05e9\u05bc\u05c1') + check('\u05e9\u05bc\u05c1', '\u05e9\u05bc\u05c1') + check('\U0001d1c0', '\U0001d1ba\U0001d165\U0001d16f') + check('\U0001d1ba\U0001d165\U0001d16f', '\U0001d1ba\U0001d165\U0001d16f') + + # Broken in 4.0.0 + check('\xa0', ['\xa0', '\xa0', ' ', ' ']) + check('\u2003', ['\u2003', '\u2003', ' ', ' ']) + check('\U0001d7ff', ['\U0001d7ff', '\U0001d7ff', '9', '9']) + + check('\xa8', ['\xa8', '\xa8', ' \u0308', ' \u0308']) + check(' \u0308', ' \u0308') + + check('\xc0', ['\xc0', 'A\u0300']*2) + check('A\u0300', ['\xc0', 'A\u0300']*2) + + check('\ud7a3', ['\ud7a3', '\u1112\u1175\u11c2']*2) + check('\u1112\u1175\u11c2', ['\ud7a3', '\u1112\u1175\u11c2']*2) + + check('\xb4', ['\xb4', '\xb4', ' \u0301', ' \u0301']) + check('\u1ffd', ['\xb4', '\xb4', ' \u0301', ' \u0301']) + check(' \u0301', ' \u0301') + + check('\xc5', ['\xc5', 'A\u030a']*2) + check('\u212b', ['\xc5', 'A\u030a']*2) + check('A\u030a', ['\xc5', 'A\u030a']*2) + + check('\u1f71', ['\u03ac', '\u03b1\u0301']*2) + check('\u03ac', ['\u03ac', '\u03b1\u0301']*2) + check('\u03b1\u0301', ['\u03ac', '\u03b1\u0301']*2) + + check('\u01c4', ['\u01c4', '\u01c4', 'D\u017d', 'DZ\u030c']) + check('D\u017d', ['D\u017d', 'DZ\u030c']*2) + check('DZ\u030c', ['D\u017d', 'DZ\u030c']*2) + + check('\u1fed', ['\u1fed', '\xa8\u0300', ' \u0308\u0300', ' \u0308\u0300']) + check('\xa8\u0300', ['\u1fed', '\xa8\u0300', ' \u0308\u0300', ' \u0308\u0300']) + check(' \u0308\u0300', ' \u0308\u0300') + + check('\u326e', ['\u326e', '\u326e', '\uac00', '\u1100\u1161']) + check('\u320e', ['\u320e', '\u320e', '(\uac00)', '(\u1100\u1161)']) + check('(\uac00)', ['(\uac00)', '(\u1100\u1161)']*2) + check('(\u1100\u1161)', ['(\uac00)', '(\u1100\u1161)']*2) + + check('\u0385', ['\u0385', '\xa8\u0301', ' \u0308\u0301', ' \u0308\u0301']) + check('\u1fee', ['\u0385', '\xa8\u0301', ' \u0308\u0301', ' \u0308\u0301']) + check('\xa8\u0301', ['\u0385', '\xa8\u0301', ' \u0308\u0301', ' \u0308\u0301']) + check(' \u0308\u0301', ' \u0308\u0301') + + check('\u1fdf', ['\u1fdf', '\u1ffe\u0342', ' \u0314\u0342', ' \u0314\u0342']) + check('\u1ffe\u0342', ['\u1fdf', '\u1ffe\u0342', ' \u0314\u0342', ' \u0314\u0342']) + check('\u1ffe', ['\u1ffe', '\u1ffe', ' \u0314', ' \u0314']) + check(' \u0314\u0342', ' \u0314\u0342') + + check('\u03d3', ['\u03d3', '\u03d2\u0301', '\u038e', '\u03a5\u0301']) + check('\u03d2\u0301', ['\u03d3', '\u03d2\u0301', '\u038e', '\u03a5\u0301']) + check('\u038e', ['\u038e', '\u03a5\u0301']*2) + check('\u1feb', ['\u038e', '\u03a5\u0301']*2) + check('\u03a5\u0301', ['\u038e', '\u03a5\u0301']*2) + + check('\u0626', ['\u0626', '\u064a\u0654']*2) + check('\u064a\u0654', ['\u0626', '\u064a\u0654']*2) + check('\ufe89', ['\ufe89', '\ufe89', '\u0626', '\u064a\u0654']) + check('\ufe8a', ['\ufe8a', '\ufe8a', '\u0626', '\u064a\u0654']) + check('\ufe8b', ['\ufe8b', '\ufe8b', '\u0626', '\u064a\u0654']) + check('\ufe8c', ['\ufe8c', '\ufe8c', '\u0626', '\u064a\u0654']) + + check('\ufef9', ['\ufef9', '\ufef9', '\u0644\u0625', '\u0644\u0627\u0655']) + check('\ufefa', ['\ufefa', '\ufefa', '\u0644\u0625', '\u0644\u0627\u0655']) + check('\ufefb', ['\ufefb', '\ufefb', '\u0644\u0627', '\u0644\u0627']) + check('\ufefc', ['\ufefc', '\ufefc', '\u0644\u0627', '\u0644\u0627']) + check('\u0644\u0625', ['\u0644\u0625', '\u0644\u0627\u0655']*2) + check('\u0644\u0627\u0655', ['\u0644\u0625', '\u0644\u0627\u0655']*2) + check('\u0644\u0627', '\u0644\u0627') + + # Broken in 4.0.0 + check('\u327c', '\u327c' if self.old else + ['\u327c', '\u327c', '\ucc38\uace0', '\u110e\u1161\u11b7\u1100\u1169']) + check('\ucc38\uace0', ['\ucc38\uace0', '\u110e\u1161\u11b7\u1100\u1169']*2) + check('\ucc38', ['\ucc38', '\u110e\u1161\u11b7']*2) + check('\u110e\u1161\u11b7\u1100\u1169', + ['\ucc38\uace0', '\u110e\u1161\u11b7\u1100\u1169']*2) + check('\u110e\u1161\u11b7\u1100', + ['\ucc38\u1100', '\u110e\u1161\u11b7\u1100']*2) + check('\u110e\u1161\u11b7', + ['\ucc38', '\u110e\u1161\u11b7']*2) + check('\u110e\u1161', + ['\ucc28', '\u110e\u1161']*2) + check('\u110e', '\u110e') + # Broken in 4.0.0-12.0.0 + check('\U00011938', '\U00011938' if self.old else + ['\U00011938', '\U00011935\U00011930']*2) + check('\U00011935\U00011930', ['\U00011938', '\U00011935\U00011930']*2) + # New in 4.0.1 + check('\u321d', '\u321d' if self.old else + ['\u321d', '\u321d', '(\uc624\uc804)', '(\u110b\u1169\u110c\u1165\u11ab)']) + check('(\uc624\uc804)', + ['(\uc624\uc804)', '(\u110b\u1169\u110c\u1165\u11ab)']*2) + check('(\u110b\u1169\u110c\u1165\u11ab)', + ['(\uc624\uc804)', '(\u110b\u1169\u110c\u1165\u11ab)']*2) + check('\u4d57', '\u4d57') + check('\u45d7', '\u45d7' if self.old else '\u45d7') + check('\U0002f9bf', '\u4d57' if self.old else '\u45d7') + # New in 4.1.0 + check('\u03a3', '\u03a3') + check('\u03f9', '\u03f9' if self.old else + ['\u03f9', '\u03f9', '\u03a3', '\u03a3']) + # New in 5.0.0 + check('\u1b06', '\u1b06' if self.old else ['\u1b06', '\u1b05\u1b35']*2) + # New in 5.2.0 + check('\U0001f213', '\U0001f213' if self.old else + ['\U0001f213', '\U0001f213', '\u30c7', '\u30c6\u3099']) + # New in 6.1.0 + check('\ufa2e', '\ufa2e' if self.old else '\u90de') + # New in 13.0.0 + check('\U00011938', '\U00011938' if self.old else + ['\U00011938', '\U00011935\U00011930', '\U00011938', '\U00011935\U00011930']) + check('\U0001fbf9', '\U0001fbf9' if self.old else + ['\U0001fbf9', '\U0001fbf9', '9', '9']) + # New in 14.0.0 + check('\U000107ba', '\U000107ba' if self.old else + ['\U000107ba', '\U000107ba', '\U0001df1e', '\U0001df1e']) + # New in 15.0.0 + check('\U0001e06d', '\U0001e06d' if self.old else + ['\U0001e06d', '\U0001e06d', '\u04b1', '\u04b1']) + # New in 16.0.0 + check('\U0001ccd6', '\U0001ccd6' if self.old else + ['\U0001ccd6', '\U0001ccd6', 'A', 'A']) + + self.assertRaises(TypeError, self.db.normalize) + self.assertRaises(TypeError, self.db.normalize, 'NFC') + self.assertRaises(ValueError, self.db.normalize, 'SPAM', 'A') + + self.assertRaises(TypeError, self.db.is_normalized) + self.assertRaises(TypeError, self.db.is_normalized, 'NFC') + self.assertRaises(ValueError, self.db.is_normalized, 'SPAM', 'A') + def test_pr29(self): # https://www.unicode.org/review/pr-29.html # See issues #1054943 and #10254. @@ -225,10 +575,39 @@ def test_pr29(self): def test_issue10254(self): # Crash reported in #10254 + # New in 4.1.0 a = 'C\u0338' * 20 + 'C\u0327' b = 'C\u0338' * 20 + '\xC7' self.assertEqual(self.db.normalize('NFC', a), b) + def test_long_combining_mark_run(self): + # gh-149079: avoid quadratic canonical ordering. + payload = "a" + ("\u0300\u0327" * 32) + nfd = "a" + ("\u0327" * 32) + ("\u0300" * 32) + nfc = "\u00e0" + ("\u0327" * 32) + ("\u0300" * 31) + + self.assertEqual(self.db.normalize("NFD", payload), nfd) + self.assertEqual(self.db.normalize("NFKD", payload), nfd) + self.assertEqual(self.db.normalize("NFC", payload), nfc) + self.assertEqual(self.db.normalize("NFKC", payload), nfc) + + def test_combining_mark_run_fast_paths(self): + # gh-149079: cover short runs and already-sorted long runs. + short_payload = "a" + ("\u0300\u0327" * 9) + "\u0300" + short_nfd = "a" + ("\u0327" * 9) + ("\u0300" * 10) + short_nfc = "\u00e0" + ("\u0327" * 9) + ("\u0300" * 9) + long_sorted = "a" + ("\u0327" * 30) + ("\u0300" * 30) + long_sorted_nfc = "\u00e0" + ("\u0327" * 30) + ("\u0300" * 29) + + self.assertEqual(self.db.normalize("NFD", short_payload), short_nfd) + self.assertEqual(self.db.normalize("NFKD", short_payload), short_nfd) + self.assertEqual(self.db.normalize("NFC", short_payload), short_nfc) + self.assertEqual(self.db.normalize("NFKC", short_payload), short_nfc) + self.assertEqual(self.db.normalize("NFD", long_sorted), long_sorted) + self.assertEqual(self.db.normalize("NFKD", long_sorted), long_sorted) + self.assertEqual(self.db.normalize("NFC", long_sorted), long_sorted_nfc) + self.assertEqual(self.db.normalize("NFKC", long_sorted), long_sorted_nfc) + def test_issue29456(self): # Fix #29456 u1176_str_a = '\u1100\u1176\u11a8' @@ -238,6 +617,7 @@ def test_issue29456(self): u11c3_str_a = '\u1100\u1175\u11c3' u11c3_str_b = '\uae30\u11c3' self.assertEqual(self.db.normalize('NFC', u1176_str_a), u1176_str_b) + # New in 4.1.0 self.assertEqual(self.db.normalize('NFC', u11a7_str_a), u11a7_str_b) self.assertEqual(self.db.normalize('NFC', u11c3_str_a), u11c3_str_b) @@ -255,6 +635,32 @@ def test_east_asian_width(self): self.assertEqual(eaw('\u2010'), 'A') self.assertEqual(eaw('\U00020000'), 'W') + # New in 4.1.0 + self.assertEqual(eaw('\u0350'), 'N' if self.old else 'A') + self.assertEqual(eaw('\U000e01ef'), 'N' if self.old else 'A') + # New in 5.2.0 + self.assertEqual(eaw('\u115a'), 'N' if self.old else 'W') + # New in 9.0.0 + self.assertEqual(eaw('\u231a'), 'N' if self.old else 'W') + self.assertEqual(eaw('\u2614'), 'N' if self.old else 'W') + self.assertEqual(eaw('\U0001f19a'), 'N' if self.old else 'W') + self.assertEqual(eaw('\U0001f991'), 'N' if self.old else 'W') + self.assertEqual(eaw('\U0001f9c0'), 'N' if self.old else 'W') + # New in 12.0.0 + self.assertEqual(eaw('\u32ff'), 'N' if self.old else 'W') + self.assertEqual(eaw('\U0001fa95'), 'N' if self.old else 'W') + # New in 13.0.0 + self.assertEqual(eaw('\u31bb'), 'N' if self.old else 'W') + self.assertEqual(eaw('\U0003134a'), 'N' if self.old else 'W') + # New in 14.0.0 + self.assertEqual(eaw('\u9ffd'), 'N' if self.old else 'W') + self.assertEqual(eaw('\U0002b738'), 'N' if self.old else 'W') + # New in 15.0.0 + self.assertEqual(eaw('\U000323af'), 'N' if self.old else 'W') + # New in 16.0.0 + self.assertEqual(eaw('\u2630'), 'N' if self.old else 'W') + self.assertEqual(eaw('\U0001FAE9'), 'N' if self.old else 'W') + def test_east_asian_width_unassigned(self): eaw = self.db.east_asian_width # unassigned @@ -263,7 +669,8 @@ def test_east_asian_width_unassigned(self): self.assertIs(self.db.name(char, None), None) # unassigned but reserved for CJK - for char in '\uFA6E\uFADA\U0002A6E0\U0002FA20\U0003134B\U0003FFFD': + for char in ('\U0002A6E0\U0002FA20\U0003134B\U0003FFFD' + '\uFA6E\uFADA'): # New in 5.2.0 self.assertEqual(eaw(char), 'W') self.assertIs(self.db.name(char, None), None) @@ -272,11 +679,17 @@ def test_east_asian_width_unassigned(self): self.assertEqual(eaw(char), 'A') self.assertIs(self.db.name(char, None), None) - def test_east_asian_width_9_0_changes(self): - self.assertEqual(self.db.ucd_3_2_0.east_asian_width('\u231a'), 'N') - self.assertEqual(self.db.east_asian_width('\u231a'), 'W') -class UnicodeMiscTest(UnicodeDatabaseTest): +class Unicode_3_2_0_FunctionsTest(UnicodeFunctionsTest): + db = unicodedata.ucd_3_2_0 + old = True + expectedchecksum = ('883824cb6c0ccf994e4451ebf281e2d6d479af47' + if quicktest else + 'caf1a7f2f380f927461837f1901ef20683f98683') + + +class UnicodeMiscTest(unittest.TestCase): + db = unicodedata @cpython_only def test_disallow_instantiation(self): @@ -300,37 +713,60 @@ def test_failed_import_during_compiling(self): "(can't load unicodedata module)" self.assertIn(error, result.err.decode("ascii")) + def test_unicodedata_unload_reload(self): + # gh-149449: dropping unicodedata and running gc must not leave the + # cached _ucnhash_CAPI pointer dangling. + code = ( + "import gc, sys\n" + "assert '\\N{GRINNING FACE}'.encode(" + " 'ascii', errors='namereplace') == b'\\\\N{GRINNING FACE}'\n" + "compile(r\"x = '\\\\N{LATIN CAPITAL LETTER A}'\", '<x>', 'exec')\n" + "del sys.modules['unicodedata']\n" + "gc.collect()\n" + "assert '\\N{WINKING FACE}'.encode(" + " 'ascii', errors='namereplace') == b'\\\\N{WINKING FACE}'\n" + "compile(r\"x = '\\\\N{LATIN CAPITAL LETTER B}'\", '<x>', 'exec')\n" + ) + script_helper.assert_python_ok("-c", code) + def test_decimal_numeric_consistent(self): # Test that decimal and numeric are consistent, # i.e. if a character has a decimal value, # its numeric value should be the same. count = 0 - for i in range(0x10000): - c = chr(i) + for c in iterallchars(): dec = self.db.decimal(c, -1) if dec != -1: self.assertEqual(dec, self.db.numeric(c)) count += 1 - self.assertTrue(count >= 10) # should have tested at least the ASCII digits + self.assertTrue(count >= 10, count) # should have tested at least the ASCII digits def test_digit_numeric_consistent(self): # Test that digit and numeric are consistent, # i.e. if a character has a digit value, # its numeric value should be the same. count = 0 - for i in range(0x10000): - c = chr(i) + for c in iterallchars(): dec = self.db.digit(c, -1) if dec != -1: self.assertEqual(dec, self.db.numeric(c)) count += 1 - self.assertTrue(count >= 10) # should have tested at least the ASCII digits + self.assertTrue(count >= 10, count) # should have tested at least the ASCII digits + + def test_normalize_consistent(self): + allchars = list(iterallchars()) + for form in ('NFC', 'NFD', 'NFKC', 'NFKD'): + for c in allchars: + norm = self.db.normalize(form, c) + self.assertEqual(self.db.is_normalized(form, c), norm == c) + if norm != c: + self.assertEqual(self.db.normalize(form, norm), norm) + self.assertTrue(self.db.is_normalized(form, norm)) def test_bug_1704793(self): self.assertEqual(self.db.lookup("GOTHIC LETTER FAIHU"), '\U00010346') def test_ucd_510(self): - import unicodedata # In UCD 5.1.0, a mirrored property changed wrt. UCD 3.2.0 self.assertTrue(unicodedata.mirrored("\u0f3a")) self.assertTrue(not unicodedata.ucd_3_2_0.mirrored("\u0f3a")) @@ -346,10 +782,10 @@ def test_bug_5828(self): # Only U+0000 should have U+0000 as its upper/lower/titlecase variant self.assertEqual( [ - c for c in range(sys.maxunicode+1) - if "\x00" in chr(c).lower()+chr(c).upper()+chr(c).title() + c for c in iterallchars() + if "\x00" in (c.lower(), c.upper(), c.title()) ], - [0] + ["\x00"] ) def test_bug_4971(self): @@ -359,15 +795,16 @@ def test_bug_4971(self): self.assertEqual("\u01c6".title(), "\u01c5") def test_linebreak_7643(self): - for i in range(0x10000): - lines = (chr(i) + 'A').splitlines() - if i in (0x0a, 0x0b, 0x0c, 0x0d, 0x85, - 0x1c, 0x1d, 0x1e, 0x2028, 0x2029): + for c in iterallchars(): + lines = (c + 'A').splitlines() + if c in ('\x0a', '\x0b', '\x0c', '\x0d', '\x85', + '\x1c', '\x1d', '\x1e', '\u2028', '\u2029'): self.assertEqual(len(lines), 2, - r"\u%.4x should be a linebreak" % i) + r"%a should be a linebreak" % c) else: self.assertEqual(len(lines), 1, - r"\u%.4x should not be a linebreak" % i) + r"%a should not be a linebreak" % c) + class NormalizationTest(unittest.TestCase): @staticmethod @@ -397,23 +834,23 @@ def test_normalization(self): self.skipTest(f"Failed to download {TESTDATAURL}: {exc}") with testdata: - self.run_normalization_tests(testdata) - - def run_normalization_tests(self, testdata): - part = None - part1_data = {} - - def NFC(str): - return unicodedata.normalize("NFC", str) + self.run_normalization_tests(testdata, unicodedata) - def NFKC(str): - return unicodedata.normalize("NFKC", str) + @requires_resource('cpu') + def test_normalization_3_2_0(self): + testdatafile = findfile('NormalizationTest-3.2.0.txt') + with open(testdatafile, encoding='utf-8') as testdata: + self.run_normalization_tests(testdata, unicodedata.ucd_3_2_0) - def NFD(str): - return unicodedata.normalize("NFD", str) + def run_normalization_tests(self, testdata, ucd): + part = None + part1_data = set() - def NFKD(str): - return unicodedata.normalize("NFKD", str) + NFC = partial(ucd.normalize, "NFC") + NFKC = partial(ucd.normalize, "NFKC") + NFD = partial(ucd.normalize, "NFD") + NFKD = partial(ucd.normalize, "NFKD") + is_normalized = ucd.is_normalized for line in testdata: if '#' in line: @@ -438,25 +875,24 @@ def NFKD(str): NFKD(c3) == NFKD(c4) == NFKD(c5), line) - self.assertTrue(unicodedata.is_normalized("NFC", c2)) - self.assertTrue(unicodedata.is_normalized("NFC", c4)) + self.assertTrue(is_normalized("NFC", c2)) + self.assertTrue(is_normalized("NFC", c4)) - self.assertTrue(unicodedata.is_normalized("NFD", c3)) - self.assertTrue(unicodedata.is_normalized("NFD", c5)) + self.assertTrue(is_normalized("NFD", c3)) + self.assertTrue(is_normalized("NFD", c5)) - self.assertTrue(unicodedata.is_normalized("NFKC", c4)) - self.assertTrue(unicodedata.is_normalized("NFKD", c5)) + self.assertTrue(is_normalized("NFKC", c4)) + self.assertTrue(is_normalized("NFKD", c5)) # Record part 1 data if part == "@Part1": - part1_data[c1] = 1 + part1_data.add(c1) # Perform tests for all other data - for c in range(sys.maxunicode+1): - X = chr(c) + for X in iterallchars(): if X in part1_data: continue - self.assertTrue(X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), c) + self.assertTrue(X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X), ord(X)) def test_edge_cases(self): self.assertRaises(TypeError, unicodedata.normalize) diff --git a/Lib/test/test_unittest/test_async_case.py b/Lib/test/test_unittest/test_async_case.py index 993e6bf013cfbf5..91d45283eb3b1bb 100644 --- a/Lib/test/test_unittest/test_async_case.py +++ b/Lib/test/test_unittest/test_async_case.py @@ -12,7 +12,7 @@ class MyException(Exception): def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TestCM: @@ -480,7 +480,7 @@ def test_setup_get_event_loop(self): class TestCase1(unittest.IsolatedAsyncioTestCase): def setUp(self): - asyncio._get_event_loop_policy().get_event_loop() + asyncio.events._get_event_loop_policy().get_event_loop() async def test_demo1(self): pass @@ -490,7 +490,7 @@ async def test_demo1(self): self.assertTrue(result.wasSuccessful()) def test_loop_factory(self): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class TestCase1(unittest.IsolatedAsyncioTestCase): loop_factory = asyncio.EventLoop diff --git a/Lib/test/test_unittest/test_case.py b/Lib/test/test_unittest/test_case.py index a04af55f3fc0aed..d66cab146af2468 100644 --- a/Lib/test/test_unittest/test_case.py +++ b/Lib/test/test_unittest/test_case.py @@ -1989,7 +1989,7 @@ def testAssertNoLogsYieldsNone(self): pass self.assertIsNone(value) - def testAssertStartswith(self): + def testAssertStartsWith(self): self.assertStartsWith('ababahalamaha', 'ababa') self.assertStartsWith('ababahalamaha', ('x', 'ababa', 'y')) self.assertStartsWith(UserString('ababahalamaha'), 'ababa') @@ -2034,7 +2034,7 @@ def testAssertStartswith(self): self.assertStartsWith('ababahalamaha', 'amaha', msg='abracadabra') self.assertIn('ababahalamaha', str(cm.exception)) - def testAssertNotStartswith(self): + def testAssertNotStartsWith(self): self.assertNotStartsWith('ababahalamaha', 'amaha') self.assertNotStartsWith('ababahalamaha', ('x', 'amaha', 'y')) self.assertNotStartsWith(UserString('ababahalamaha'), 'amaha') @@ -2079,7 +2079,7 @@ def testAssertNotStartswith(self): self.assertNotStartsWith('ababahalamaha', 'ababa', msg='abracadabra') self.assertIn('ababahalamaha', str(cm.exception)) - def testAssertEndswith(self): + def testAssertEndsWith(self): self.assertEndsWith('ababahalamaha', 'amaha') self.assertEndsWith('ababahalamaha', ('x', 'amaha', 'y')) self.assertEndsWith(UserString('ababahalamaha'), 'amaha') @@ -2124,7 +2124,7 @@ def testAssertEndswith(self): self.assertEndsWith('ababahalamaha', 'ababa', msg='abracadabra') self.assertIn('ababahalamaha', str(cm.exception)) - def testAssertNotEndswith(self): + def testAssertNotEndsWith(self): self.assertNotEndsWith('ababahalamaha', 'ababa') self.assertNotEndsWith('ababahalamaha', ('x', 'ababa', 'y')) self.assertNotEndsWith(UserString('ababahalamaha'), 'ababa') diff --git a/Lib/test/test_unittest/test_loader.py b/Lib/test/test_unittest/test_loader.py index cdff6d1a20c8dfd..f4e50a3dccb2bc2 100644 --- a/Lib/test/test_unittest/test_loader.py +++ b/Lib/test/test_unittest/test_loader.py @@ -1,3 +1,4 @@ +import abc import functools import sys import types @@ -98,6 +99,22 @@ def test_loadTestsFromTestCase__from_FunctionTestCase(self): self.assertIsInstance(suite, loader.suiteClass) self.assertEqual(list(suite), []) + # "Do not load any tests from a TestCase-derived class that is an abstract + # base class." + def test_loadTestsFromTestCase__from_abc_TestCase(self): + class FooBase(unittest.TestCase, metaclass=abc.ABCMeta): + @abc.abstractmethod + def test(self): ... + class Foo(FooBase): + def test(self): pass + + empty_suite = unittest.TestSuite() + + loader = unittest.TestLoader() + suite = loader.loadTestsFromTestCase(Foo) + self.assertEqual(loader.loadTestsFromTestCase(FooBase), empty_suite) + self.assertEqual(list(suite), [Foo('test')]) + ################################################################ ### /Tests for TestLoader.loadTestsFromTestCase @@ -252,6 +269,24 @@ def load_tests(loader, tests, pattern): self.assertRaisesRegex(TypeError, "some failure", test.m) + # Check that loadTestsFromModule skips abstract base classes derived from + # TestCase, which can't be instantiated. + def test_loadTestsFromModule__skip_abc_TestCase(self): + m = types.ModuleType('m') + class MyTestCaseBase(unittest.TestCase, metaclass=abc.ABCMeta): + @abc.abstractmethod + def test(self): + ... + class MyTestCase(MyTestCaseBase): + def test(self): + pass + m.testcase_1 = MyTestCaseBase + m.testcase_2 = MyTestCase + loader = unittest.TestLoader() + suite = loader.loadTestsFromModule(m) + expected = [loader.suiteClass([MyTestCase('test')])] + self.assertEqual(list(suite), expected) + ################################################################ ### /Tests for TestLoader.loadTestsFromModule() @@ -1052,6 +1087,22 @@ def test_loadTestsFromNames__module_not_loaded(self): if module_name in sys.modules: del sys.modules[module_name] + # "The specifier should not refer to a test method in a TestCase-derived + # subclass that is an abstract base class" + def test_loadTestsFromNames__testmethod_in_abc_TestCase(self): + m = types.ModuleType('m') + class Foo(unittest.TestCase, metaclass=abc.ABCMeta): + @abc.abstractmethod + def test_1(self): ... + def test_2(self): pass + m.Foo = Foo + + loader = unittest.TestLoader() + for name in 'Foo.test_1', 'Foo.test_2': + with self.subTest(name=name), self.assertRaisesRegex(TypeError, + "Cannot instantiate abstract test case Foo"): + loader.loadTestsFromNames([name], m) + ################################################################ ### /Tests for TestLoader.loadTestsFromNames() diff --git a/Lib/test/test_unittest/test_program.py b/Lib/test/test_unittest/test_program.py index 6092ed292d8f60f..8ed92373e5e984a 100644 --- a/Lib/test/test_unittest/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -75,6 +75,14 @@ def testUnexpectedSuccess(self): class Empty(unittest.TestCase): pass + class SetUpClassFailure(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + raise Exception + def testPass(self): + pass + class TestLoader(unittest.TestLoader): """Test loader that returns a suite containing the supplied testcase.""" @@ -191,6 +199,18 @@ def test_ExitEmptySuite(self): out = stream.getvalue() self.assertIn('\nNO TESTS RAN\n', out) + def test_ExitSetUpClassFailureSuite(self): + stream = BufferedWriter() + with self.assertRaises(SystemExit) as cm: + unittest.main( + argv=["setup_class_failure"], + testRunner=unittest.TextTestRunner(stream=stream), + testLoader=self.TestLoader(self.SetUpClassFailure)) + self.assertEqual(cm.exception.code, 1) + out = stream.getvalue() + self.assertIn("ERROR: setUpClass", out) + self.assertIn("SetUpClassFailure", out) + class InitialisableProgram(unittest.TestProgram): exit = False diff --git a/Lib/test/test_unittest/testmock/testasync.py b/Lib/test/test_unittest/testmock/testasync.py index 0791675b5401ca1..dc36ceeb6502e84 100644 --- a/Lib/test/test_unittest/testmock/testasync.py +++ b/Lib/test/test_unittest/testmock/testasync.py @@ -15,7 +15,7 @@ def tearDownModule(): - asyncio._set_event_loop_policy(None) + asyncio.events._set_event_loop_policy(None) class AsyncClass: diff --git a/Lib/test/test_unittest/testmock/testhelpers.py b/Lib/test/test_unittest/testmock/testhelpers.py index d1e48bde982040e..0e82c723ec3eaa2 100644 --- a/Lib/test/test_unittest/testmock/testhelpers.py +++ b/Lib/test/test_unittest/testmock/testhelpers.py @@ -1050,6 +1050,7 @@ def __post_init__(self): create_autospec(WithPostInit()), ]: with self.subTest(mock=mock): + self.assertIsInstance(mock, WithPostInit) self.assertIsInstance(mock.a, int) self.assertIsInstance(mock.b, int) @@ -1072,6 +1073,7 @@ class WithDefault: create_autospec(WithDefault(1)), ]: with self.subTest(mock=mock): + self.assertIsInstance(mock, WithDefault) self.assertIsInstance(mock.a, int) self.assertIsInstance(mock.b, int) @@ -1087,6 +1089,7 @@ def b(self) -> int: create_autospec(WithMethod(1)), ]: with self.subTest(mock=mock): + self.assertIsInstance(mock, WithMethod) self.assertIsInstance(mock.a, int) mock.b.assert_not_called() @@ -1102,11 +1105,29 @@ class WithNonFields: create_autospec(WithNonFields(1)), ]: with self.subTest(mock=mock): + self.assertIsInstance(mock, WithNonFields) with self.assertRaisesRegex(AttributeError, msg): mock.a with self.assertRaisesRegex(AttributeError, msg): mock.b + def test_dataclass_special_attrs(self): + @dataclass + class Description: + name: str + + for mock in [ + create_autospec(Description, instance=True), + create_autospec(Description(1)), + ]: + with self.subTest(mock=mock): + self.assertIsInstance(mock, Description) + self.assertIs(mock.__class__, Description) + self.assertIsInstance(mock.__dataclass_fields__, MagicMock) + self.assertIsInstance(mock.__dataclass_params__, MagicMock) + self.assertIsInstance(mock.__match_args__, MagicMock) + self.assertIsInstance(mock.__hash__, MagicMock) + class TestCallList(unittest.TestCase): def test_args_list_contains_call_list(self): diff --git a/Lib/test/test_unittest/testmock/testmock.py b/Lib/test/test_unittest/testmock/testmock.py index 386d53bf5a5c63a..ef69f060886d0a4 100644 --- a/Lib/test/test_unittest/testmock/testmock.py +++ b/Lib/test/test_unittest/testmock/testmock.py @@ -1743,6 +1743,13 @@ def static_method(): pass mock_method.assert_called_once_with() self.assertRaises(TypeError, mock_method, 'extra_arg') + # gh-145754 + def test_create_autospec_type_hints_typechecking(self): + def foo(x: Tuple[int, ...]) -> None: + pass + + mock.create_autospec(foo) + #Issue21238 def test_mock_unsafe(self): m = Mock() @@ -2101,6 +2108,15 @@ def test_mock_open_after_eof(self): self.assertEqual([], h.readlines()) self.assertEqual([], h.readlines()) + def test_mock_open_exit_with_contextlib_exit_stack(self): + # gh-150484: mock_open's __exit__ should work when called from + # contextlib.ExitStack, which passes (exctype, excinst, exctb). + from contextlib import ExitStack + with mock.patch('builtins.open', mock.mock_open()) as m: + with ExitStack() as exit_stack: + with exit_stack.enter_context(open('/tmp/test.txt', 'w')): + pass + def test_mock_parents(self): for Klass in Mock, MagicMock: m = Klass() diff --git a/Lib/test/test_unittest/testmock/testthreadingmock.py b/Lib/test/test_unittest/testmock/testthreadingmock.py index a02b532ed447cd5..dda4916434ec1b4 100644 --- a/Lib/test/test_unittest/testmock/testthreadingmock.py +++ b/Lib/test/test_unittest/testmock/testthreadingmock.py @@ -1,8 +1,10 @@ +import sys import time import unittest +import threading import concurrent.futures -from test.support import threading_helper +from test.support import setswitchinterval, threading_helper from unittest.mock import patch, ThreadingMock @@ -196,6 +198,102 @@ def test_reset_mock_resets_wait(self): m.wait_until_any_call_with() m.assert_called_once() + def test_call_count_thread_safe(self): + # See https://github.com/python/cpython/issues/142651. + m = ThreadingMock() + LOOPS = 100 + THREADS = 10 + def test_function(): + for _ in range(LOOPS): + m() + + oldswitchinterval = sys.getswitchinterval() + setswitchinterval(1e-6) + try: + threads = [threading.Thread(target=test_function) for _ in range(THREADS)] + with threading_helper.start_threads(threads): + pass + finally: + sys.setswitchinterval(oldswitchinterval) + + self.assertEqual(m.call_count, LOOPS * THREADS) + + + def test_call_args_thread_safe(self): + m = ThreadingMock() + LOOPS = 100 + THREADS = 10 + def test_function(thread_id): + for i in range(LOOPS): + m(thread_id, i) + + oldswitchinterval = sys.getswitchinterval() + setswitchinterval(1e-6) + try: + threads = [ + threading.Thread(target=test_function, args=(thread_id,)) + for thread_id in range(THREADS) + ] + with threading_helper.start_threads(threads): + pass + finally: + sys.setswitchinterval(oldswitchinterval) + expected_calls = { + (thread_id, i) + for thread_id in range(THREADS) + for i in range(LOOPS) + } + self.assertSetEqual({call.args for call in m.call_args_list}, expected_calls) + + def test_method_calls_thread_safe(self): + m = ThreadingMock() + LOOPS = 100 + THREADS = 10 + def test_function(thread_id): + for i in range(LOOPS): + getattr(m, f"method_{thread_id}")(i) + + oldswitchinterval = sys.getswitchinterval() + setswitchinterval(1e-6) + try: + threads = [ + threading.Thread(target=test_function, args=(thread_id,)) + for thread_id in range(THREADS) + ] + with threading_helper.start_threads(threads): + pass + finally: + sys.setswitchinterval(oldswitchinterval) + for thread_id in range(THREADS): + self.assertEqual(getattr(m, f"method_{thread_id}").call_count, LOOPS) + self.assertEqual({call.args for call in getattr(m, f"method_{thread_id}").call_args_list}, + {(i,) for i in range(LOOPS)}) + + def test_mock_calls_thread_safe(self): + m = ThreadingMock() + LOOPS = 100 + THREADS = 10 + def test_function(thread_id): + for i in range(LOOPS): + m(thread_id, i) + + oldswitchinterval = sys.getswitchinterval() + setswitchinterval(1e-6) + try: + threads = [ + threading.Thread(target=test_function, args=(thread_id,)) + for thread_id in range(THREADS) + ] + with threading_helper.start_threads(threads): + pass + finally: + sys.setswitchinterval(oldswitchinterval) + expected_calls = { + (thread_id, i) + for thread_id in range(THREADS) + for i in range(LOOPS) + } + self.assertSetEqual({call.args for call in m.mock_calls}, expected_calls) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_unpack_ex.py b/Lib/test/test_unpack_ex.py index 9e2d54bd3a8c4ea..c948d51452dbd92 100644 --- a/Lib/test/test_unpack_ex.py +++ b/Lib/test/test_unpack_ex.py @@ -383,13 +383,13 @@ Some size constraints (all fail.) - >>> s = ", ".join("a%d" % i for i in range(1<<8)) + ", *rest = range(1<<8 + 1)" + >>> s = ", ".join("a%d" % i for i in range(1<<8)) + ", *rest = range((1<<8) + 1)" >>> compile(s, 'test', 'exec') # doctest:+ELLIPSIS Traceback (most recent call last): ... SyntaxError: too many expressions in star-unpacking assignment - >>> s = ", ".join("a%d" % i for i in range(1<<8 + 1)) + ", *rest = range(1<<8 + 2)" + >>> s = ", ".join("a%d" % i for i in range((1<<8) + 1)) + ", *rest = range((1<<8) + 2)" >>> compile(s, 'test', 'exec') # doctest:+ELLIPSIS Traceback (most recent call last): ... diff --git a/Lib/test/test_unparse.py b/Lib/test/test_unparse.py index d3af7a8489e650e..35e4652a87b423c 100644 --- a/Lib/test/test_unparse.py +++ b/Lib/test/test_unparse.py @@ -206,10 +206,97 @@ def test_tstrings(self): self.check_ast_roundtrip("t'foo'") self.check_ast_roundtrip("t'foo {bar}'") self.check_ast_roundtrip("t'foo {bar!s:.2f}'") - self.check_ast_roundtrip("t'foo {bar}' f'{bar}'") - self.check_ast_roundtrip("f'{bar}' t'foo {bar}'") - self.check_ast_roundtrip("t'foo {bar}' fr'\\hello {bar}'") - self.check_ast_roundtrip("t'foo {bar}' u'bar'") + self.check_ast_roundtrip("t'{a + b}'") + self.check_ast_roundtrip("t'{a + b:x}'") + self.check_ast_roundtrip("t'{a + b!s}'") + self.check_ast_roundtrip("t'{ {a}}'") + self.check_ast_roundtrip("t'{ {a}=}'") + self.check_ast_roundtrip("t'{{a}}'") + self.check_ast_roundtrip("t''") + self.check_ast_roundtrip('t""') + self.check_ast_roundtrip("t'{(lambda x: x)}'") + self.check_ast_roundtrip("t'{t'{x}'}'") + + def test_tstring_with_nonsensical_str_field(self): + # `value` suggests that the original code is `t'{test1}`, but `str` suggests otherwise + self.assertEqual( + ast.unparse( + ast.TemplateStr( + values=[ + ast.Interpolation( + value=ast.Name(id="test1", ctx=ast.Load()), str="test2", conversion=-1 + ) + ] + ) + ), + "t'{test2}'", + ) + + def test_tstring_with_none_str_field(self): + self.assertEqual( + ast.unparse( + ast.TemplateStr( + [ast.Interpolation(value=ast.Name(id="test1"), str=None, conversion=-1)] + ) + ), + "t'{test1}'", + ) + self.assertEqual( + ast.unparse( + ast.TemplateStr( + [ + ast.Interpolation( + value=ast.Lambda( + args=ast.arguments(args=[ast.arg(arg="x")]), + body=ast.Name(id="x"), + ), + str=None, + conversion=-1, + ) + ] + ) + ), + "t'{(lambda x: x)}'", + ) + self.assertEqual( + ast.unparse( + ast.TemplateStr( + values=[ + ast.Interpolation( + value=ast.TemplateStr( + # `str` field kept here + [ast.Interpolation(value=ast.Name(id="x"), str="y", conversion=-1)] + ), + str=None, + conversion=-1, + ) + ] + ) + ), + '''t"{t'{y}'}"''', + ) + self.assertEqual( + ast.unparse( + ast.TemplateStr( + values=[ + ast.Interpolation( + value=ast.TemplateStr( + [ast.Interpolation(value=ast.Name(id="x"), str=None, conversion=-1)] + ), + str=None, + conversion=-1, + ) + ] + ) + ), + '''t"{t'{x}'}"''', + ) + self.assertEqual( + ast.unparse(ast.TemplateStr( + [ast.Interpolation(value=ast.Constant(value="foo"), str=None, conversion=114)] + )), + '''t"{'foo'!r}"''', + ) def test_strings(self): self.check_ast_roundtrip("u'foo'") diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index c965860fbb10efa..1e5f79998e7cab2 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -10,6 +10,7 @@ from test import support from test.support import os_helper from test.support import socket_helper +from test.support import control_characters_c0 import os import socket try: @@ -109,7 +110,7 @@ def setUp(self): finally: f.close() self.pathname = os_helper.TESTFN - self.quoted_pathname = urllib.parse.quote(self.pathname) + self.quoted_pathname = urllib.parse.quote(os.fsencode(self.pathname)) self.returned_obj = urllib.request.urlopen("file:%s" % self.quoted_pathname) def tearDown(self): @@ -466,6 +467,25 @@ def test_redirect_limit_independent(self): finally: self.unfakehttp() + def test_http_error_attribute_values(self): + hdrs = { + "Authorization": "Bearer foobar", + "Accept": "application/json" + } + err = urllib.error.HTTPError("http://something", 404, "foo", hdrs, None) + self.assertEqual(err.filename, "http://something") + self.assertEqual(err.code, 404) + self.assertEqual(err.msg, "foo") + self.assertEqual(err.reason, "foo") + self.assertEqual(err.hdrs, hdrs) + self.assertEqual(err.headers, hdrs) + err.close() + + def test_http_error_default_fp(self): + err = urllib.error.HTTPError("http://something", 404, "foo", {}, None) + self.assertIsInstance(err.fp, io.BytesIO) + err.close() + def test_empty_socket(self): # urlopen() raises OSError if the underlying socket does not send any # data. (#1680230) @@ -512,6 +532,11 @@ def test_ftp_nonexisting(self): self.assertFalse(e.exception.filename) self.assertTrue(e.exception.reason) + def test_url_error_stringified(self): + reason = 'sixseven' + err = urllib.error.URLError(reason) + self.assertEqual(str(err), f'<urlopen error {reason}>') + class urlopen_DataTests(unittest.TestCase): """Test urlopen() opening a data URL.""" @@ -590,6 +615,13 @@ def test_invalid_base64_data(self): # missing padding character self.assertRaises(ValueError,urllib.request.urlopen,'data:;base64,Cg=') + def test_invalid_mediatype(self): + for c0 in control_characters_c0(): + self.assertRaises(ValueError,urllib.request.urlopen, + f'data:text/html;{c0},data') + for c0 in control_characters_c0(): + self.assertRaises(ValueError,urllib.request.urlopen, + f'data:text/html{c0};base64,ZGF0YQ==') class urlretrieve_FileTests(unittest.TestCase): """Test urllib.urlretrieve() on local files""" @@ -1526,6 +1558,14 @@ def test_url2pathname(self): self.assertEqual(fn('////foo/bar'), f'{sep}{sep}foo{sep}bar') self.assertEqual(fn('data:blah'), 'data:blah') self.assertEqual(fn('data://blah'), f'data:{sep}{sep}blah') + self.assertEqual(fn('foo?bar'), 'foo') + self.assertEqual(fn('foo#bar'), 'foo') + self.assertEqual(fn('foo?bar=baz'), 'foo') + self.assertEqual(fn('foo?bar#baz'), 'foo') + self.assertEqual(fn('foo%3Fbar'), 'foo?bar') + self.assertEqual(fn('foo%23bar'), 'foo#bar') + self.assertEqual(fn('foo%3Fbar%3Dbaz'), 'foo?bar=baz') + self.assertEqual(fn('foo%3Fbar%23baz'), 'foo?bar#baz') def test_url2pathname_require_scheme(self): sep = os.path.sep @@ -1569,6 +1609,7 @@ def test_url2pathname_require_scheme_errors(self): urllib.request.url2pathname, url, require_scheme=True) + @unittest.skipIf(support.is_emscripten, "Fixed by https://github.com/emscripten-core/emscripten/pull/24593") def test_url2pathname_resolve_host(self): fn = urllib.request.url2pathname sep = os.path.sep @@ -1581,6 +1622,10 @@ def test_url2pathname_resolve_host(self): def test_url2pathname_win(self): fn = urllib.request.url2pathname self.assertEqual(fn('/C:/'), 'C:\\') + self.assertEqual(fn('//C:'), 'C:') + self.assertEqual(fn('//C:/'), 'C:\\') + self.assertEqual(fn('//C:\\'), 'C:\\') + self.assertEqual(fn('//C:80/'), 'C:80\\') self.assertEqual(fn("///C|"), 'C:') self.assertEqual(fn("///C:"), 'C:') self.assertEqual(fn('///C:/'), 'C:\\') diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 7d7f2fa00d35b62..eeea9cda2f420fd 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -963,6 +963,35 @@ def test_http(self): self.assertEqual(req.unredirected_hdrs["Host"], "baz") self.assertEqual(req.unredirected_hdrs["Spam"], "foo") + def test_http_header_priority(self): + # gh-47005: regular headers set via add_header() must override + # unredirected headers with the same name in do_open(), consistent + # with get_header() and header_items(). + cases = [ + ("Content-Type", "application/json", "application/x-www-form-urlencoded"), + ("Content-Length", "99", "0"), + ("Host", "override.example.com", "internal.example.com"), + ("Authorization", "Bearer user-token", "Basic stale="), + ("Cookie", "a=1", "b=2"), + ("User-Agent", "MyApp/1.0", "Python-urllib/test"), + ] + h = urllib.request.AbstractHTTPHandler() + h.parent = MockOpener() + + for key, regular, unredirected in cases: + req = Request("http://example.com/", headers={key: regular}) + req.timeout = None + req.add_unredirected_header(key, unredirected) + + http = MockHTTPClass() + h.do_open(http, req) + + sent_headers = dict(http.req_headers) + self.assertEqual(sent_headers[key], regular) + # key is capitalized by add_header() and add_unredirected_header() calls + self.assertEqual(req.get_header(key.capitalize()), regular) + self.assertEqual(dict(req.header_items())[key.capitalize()], regular) + def test_http_body_file(self): # A regular file - chunked encoding is used unless Content Length is # already set. diff --git a/Lib/test/test_urllib2net.py b/Lib/test/test_urllib2net.py index e6a184769084953..d015267cefdd64b 100644 --- a/Lib/test/test_urllib2net.py +++ b/Lib/test/test_urllib2net.py @@ -1,9 +1,13 @@ +import contextlib import errno +import sysconfig import unittest +from unittest import mock from test import support from test.support import os_helper from test.support import socket_helper from test.support import ResourceDenied +from test.support.warnings_helper import check_no_resource_warning import os import socket @@ -143,6 +147,43 @@ def test_ftp(self): ] self._test_urls(urls, self._extra_handlers()) + @support.requires_resource('walltime') + @unittest.skipIf(sysconfig.get_platform() == 'linux-ppc64le', + 'leaks on PPC64LE (gh-140691)') + def test_ftp_no_leak(self): + # gh-140691: When the data connection (but not control connection) + # cannot be made established, we shouldn't leave an open socket object. + + class MockError(OSError): + pass + + orig_create_connection = socket.create_connection + def patched_create_connection(address, *args, **kwargs): + """Simulate REJECTing connections to ports other than 21""" + host, port = address + if port != 21: + raise MockError() + return orig_create_connection(address, *args, **kwargs) + + url = 'ftp://www.pythontest.net/README' + entry = url, None, urllib.error.URLError + no_cache_handlers = [urllib.request.FTPHandler()] + cache_handlers = self._extra_handlers() + with mock.patch('socket.create_connection', patched_create_connection): + with check_no_resource_warning(self): + # Try without CacheFTPHandler + self._test_urls([entry], handlers=no_cache_handlers, + retry=False) + with check_no_resource_warning(self): + # Try with CacheFTPHandler (uncached) + self._test_urls([entry], cache_handlers, retry=False) + with check_no_resource_warning(self): + # Try with CacheFTPHandler (cached) + self._test_urls([entry], cache_handlers, retry=False) + # Try without the mock: the handler should not use a closed connection + with check_no_resource_warning(self): + self._test_urls([url], cache_handlers, retry=False) + def test_file(self): TESTFN = os_helper.TESTFN f = open(TESTFN, 'w') @@ -255,18 +296,16 @@ def _test_urls(self, urls, handlers, retry=True): else: req = expected_err = None + if expected_err: + context = self.assertRaises(expected_err) + else: + context = contextlib.nullcontext() + with socket_helper.transient_internet(url): - try: + f = None + with context: f = urlopen(url, req, support.INTERNET_TIMEOUT) - # urllib.error.URLError is a subclass of OSError - except OSError as err: - if expected_err: - msg = ("Didn't get expected error(s) %s for %s %s, got %s: %s" % - (expected_err, url, req, type(err), err)) - self.assertIsInstance(err, expected_err, msg) - else: - raise - else: + if f is not None: try: with time_out, \ socket_peer_reset, \ diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index aabc360289a0d00..98f0b190ee759b2 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -2,6 +2,7 @@ import unicodedata import unittest import urllib.parse +from test import support RFC1808_BASE = "http://a/b/c/d;p?q#f" RFC2396_BASE = "http://a/b/c/d;p?q" @@ -156,27 +157,25 @@ def checkRoundtrips(self, url, parsed, split, url2=None): self.assertEqual(result3.hostname, result.hostname) self.assertEqual(result3.port, result.port) - def test_qsl(self): - for orig, expect in parse_qsl_test_cases: - result = urllib.parse.parse_qsl(orig, keep_blank_values=True) - self.assertEqual(result, expect, "Error parsing %r" % orig) - expect_without_blanks = [v for v in expect if len(v[1])] - result = urllib.parse.parse_qsl(orig, keep_blank_values=False) - self.assertEqual(result, expect_without_blanks, - "Error parsing %r" % orig) - - def test_qs(self): - for orig, expect in parse_qs_test_cases: - result = urllib.parse.parse_qs(orig, keep_blank_values=True) - self.assertEqual(result, expect, "Error parsing %r" % orig) - expect_without_blanks = {v: expect[v] - for v in expect if len(expect[v][0])} - result = urllib.parse.parse_qs(orig, keep_blank_values=False) - self.assertEqual(result, expect_without_blanks, - "Error parsing %r" % orig) - - def test_roundtrips(self): - str_cases = [ + @support.subTests('orig,expect', parse_qsl_test_cases) + def test_qsl(self, orig, expect): + result = urllib.parse.parse_qsl(orig, keep_blank_values=True) + self.assertEqual(result, expect) + expect_without_blanks = [v for v in expect if len(v[1])] + result = urllib.parse.parse_qsl(orig, keep_blank_values=False) + self.assertEqual(result, expect_without_blanks) + + @support.subTests('orig,expect', parse_qs_test_cases) + def test_qs(self, orig, expect): + result = urllib.parse.parse_qs(orig, keep_blank_values=True) + self.assertEqual(result, expect) + expect_without_blanks = {v: expect[v] + for v in expect if len(expect[v][0])} + result = urllib.parse.parse_qs(orig, keep_blank_values=False) + self.assertEqual(result, expect_without_blanks) + + @support.subTests('bytes', (False, True)) + @support.subTests('url,parsed,split', [ ('path/to/file', ('', '', 'path/to/file', '', '', ''), ('', '', 'path/to/file', '', '')), @@ -263,23 +262,21 @@ def test_roundtrips(self): ('sch_me:path/to/file', ('', '', 'sch_me:path/to/file', '', '', ''), ('', '', 'sch_me:path/to/file', '', '')), - ] - def _encode(t): - return (t[0].encode('ascii'), - tuple(x.encode('ascii') for x in t[1]), - tuple(x.encode('ascii') for x in t[2])) - bytes_cases = [_encode(x) for x in str_cases] - str_cases += [ ('schème:path/to/file', ('', '', 'schème:path/to/file', '', '', ''), ('', '', 'schème:path/to/file', '', '')), - ] - for url, parsed, split in str_cases + bytes_cases: - with self.subTest(url): - self.checkRoundtrips(url, parsed, split) - - def test_roundtrips_normalization(self): - str_cases = [ + ]) + def test_roundtrips(self, bytes, url, parsed, split): + if bytes: + if not url.isascii(): + self.skipTest('non-ASCII bytes') + url = str_encode(url) + parsed = tuple_encode(parsed) + split = tuple_encode(split) + self.checkRoundtrips(url, parsed, split) + + @support.subTests('bytes', (False, True)) + @support.subTests('url,url2,parsed,split', [ ('///path/to/file', '/path/to/file', ('', '', '/path/to/file', '', '', ''), @@ -300,22 +297,18 @@ def test_roundtrips_normalization(self): 'https:///tmp/junk.txt', ('https', '', '/tmp/junk.txt', '', '', ''), ('https', '', '/tmp/junk.txt', '', '')), - ] - def _encode(t): - return (t[0].encode('ascii'), - t[1].encode('ascii'), - tuple(x.encode('ascii') for x in t[2]), - tuple(x.encode('ascii') for x in t[3])) - bytes_cases = [_encode(x) for x in str_cases] - for url, url2, parsed, split in str_cases + bytes_cases: - with self.subTest(url): - self.checkRoundtrips(url, parsed, split, url2) - - def test_http_roundtrips(self): - # urllib.parse.urlsplit treats 'http:' as an optimized special case, - # so we test both 'http:' and 'https:' in all the following. - # Three cheers for white box knowledge! - str_cases = [ + ]) + def test_roundtrips_normalization(self, bytes, url, url2, parsed, split): + if bytes: + url = str_encode(url) + url2 = str_encode(url2) + parsed = tuple_encode(parsed) + split = tuple_encode(split) + self.checkRoundtrips(url, parsed, split, url2) + + @support.subTests('bytes', (False, True)) + @support.subTests('scheme', ('http', 'https')) + @support.subTests('url,parsed,split', [ ('://www.python.org', ('www.python.org', '', '', '', ''), ('www.python.org', '', '', '')), @@ -331,23 +324,20 @@ def test_http_roundtrips(self): ('://a/b/c/d;p?q#f', ('a', '/b/c/d', 'p', 'q', 'f'), ('a', '/b/c/d;p', 'q', 'f')), - ] - def _encode(t): - return (t[0].encode('ascii'), - tuple(x.encode('ascii') for x in t[1]), - tuple(x.encode('ascii') for x in t[2])) - bytes_cases = [_encode(x) for x in str_cases] - str_schemes = ('http', 'https') - bytes_schemes = (b'http', b'https') - str_tests = str_schemes, str_cases - bytes_tests = bytes_schemes, bytes_cases - for schemes, test_cases in (str_tests, bytes_tests): - for scheme in schemes: - for url, parsed, split in test_cases: - url = scheme + url - parsed = (scheme,) + parsed - split = (scheme,) + split - self.checkRoundtrips(url, parsed, split) + ]) + def test_http_roundtrips(self, bytes, scheme, url, parsed, split): + # urllib.parse.urlsplit treats 'http:' as an optimized special case, + # so we test both 'http:' and 'https:' in all the following. + # Three cheers for white box knowledge! + if bytes: + scheme = str_encode(scheme) + url = str_encode(url) + parsed = tuple_encode(parsed) + split = tuple_encode(split) + url = scheme + url + parsed = (scheme,) + parsed + split = (scheme,) + split + self.checkRoundtrips(url, parsed, split) def checkJoin(self, base, relurl, expected, *, relroundtrip=True): with self.subTest(base=base, relurl=relurl): @@ -363,12 +353,13 @@ def checkJoin(self, base, relurl, expected, *, relroundtrip=True): relurlb = urllib.parse.urlunsplit(urllib.parse.urlsplit(relurlb)) self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) - def test_unparse_parse(self): - str_cases = ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',] - bytes_cases = [x.encode('ascii') for x in str_cases] - for u in str_cases + bytes_cases: - self.assertEqual(urllib.parse.urlunsplit(urllib.parse.urlsplit(u)), u) - self.assertEqual(urllib.parse.urlunparse(urllib.parse.urlparse(u)), u) + @support.subTests('bytes', (False, True)) + @support.subTests('u', ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',]) + def test_unparse_parse(self, bytes, u): + if bytes: + u = str_encode(u) + self.assertEqual(urllib.parse.urlunsplit(urllib.parse.urlsplit(u)), u) + self.assertEqual(urllib.parse.urlunparse(urllib.parse.urlparse(u)), u) def test_RFC1808(self): # "normal" cases from RFC 1808: @@ -695,8 +686,8 @@ def test_urljoins_relative_base(self): self.checkJoin('///b/c', '///w', '///w') self.checkJoin('///b/c', 'w', '///b/w') - def test_RFC2732(self): - str_cases = [ + @support.subTests('bytes', (False, True)) + @support.subTests('url,hostname,port', [ ('http://Test.python.org:5432/foo/', 'test.python.org', 5432), ('http://12.34.56.78:5432/foo/', '12.34.56.78', 5432), ('http://[::1]:5432/foo/', '::1', 5432), @@ -727,26 +718,28 @@ def test_RFC2732(self): ('http://[::12.34.56.78]:/foo/', '::12.34.56.78', None), ('http://[::ffff:12.34.56.78]:/foo/', '::ffff:12.34.56.78', None), - ] - def _encode(t): - return t[0].encode('ascii'), t[1].encode('ascii'), t[2] - bytes_cases = [_encode(x) for x in str_cases] - for url, hostname, port in str_cases + bytes_cases: - urlparsed = urllib.parse.urlparse(url) - self.assertEqual((urlparsed.hostname, urlparsed.port) , (hostname, port)) - - str_cases = [ + ]) + def test_RFC2732(self, bytes, url, hostname, port): + if bytes: + url = str_encode(url) + hostname = str_encode(hostname) + urlparsed = urllib.parse.urlparse(url) + self.assertEqual((urlparsed.hostname, urlparsed.port), (hostname, port)) + + @support.subTests('bytes', (False, True)) + @support.subTests('invalid_url', [ 'http://::12.34.56.78]/', 'http://[::1/foo/', 'ftp://[::1/foo/bad]/bad', 'http://[::1/foo/bad]/bad', - 'http://[::ffff:12.34.56.78'] - bytes_cases = [x.encode('ascii') for x in str_cases] - for invalid_url in str_cases + bytes_cases: - self.assertRaises(ValueError, urllib.parse.urlparse, invalid_url) - - def test_urldefrag(self): - str_cases = [ + 'http://[::ffff:12.34.56.78']) + def test_RFC2732_invalid(self, bytes, invalid_url): + if bytes: + invalid_url = str_encode(invalid_url) + self.assertRaises(ValueError, urllib.parse.urlparse, invalid_url) + + @support.subTests('bytes', (False, True)) + @support.subTests('url,defrag,frag', [ ('http://python.org#frag', 'http://python.org', 'frag'), ('http://python.org', 'http://python.org', ''), ('http://python.org/#frag', 'http://python.org/', 'frag'), @@ -770,18 +763,18 @@ def test_urldefrag(self): ('http:?q#f', 'http:?q', 'f'), ('//a/b/c;p?q#f', '//a/b/c;p?q', 'f'), ('://a/b/c;p?q#f', '://a/b/c;p?q', 'f'), - ] - def _encode(t): - return type(t)(x.encode('ascii') for x in t) - bytes_cases = [_encode(x) for x in str_cases] - for url, defrag, frag in str_cases + bytes_cases: - with self.subTest(url): - result = urllib.parse.urldefrag(url) - hash = '#' if isinstance(url, str) else b'#' - self.assertEqual(result.geturl(), url.rstrip(hash)) - self.assertEqual(result, (defrag, frag)) - self.assertEqual(result.url, defrag) - self.assertEqual(result.fragment, frag) + ]) + def test_urldefrag(self, bytes, url, defrag, frag): + if bytes: + url = str_encode(url) + defrag = str_encode(defrag) + frag = str_encode(frag) + result = urllib.parse.urldefrag(url) + hash = '#' if isinstance(url, str) else b'#' + self.assertEqual(result.geturl(), url.rstrip(hash)) + self.assertEqual(result, (defrag, frag)) + self.assertEqual(result.url, defrag) + self.assertEqual(result.fragment, frag) def test_urlsplit_scoped_IPv6(self): p = urllib.parse.urlsplit('http://[FE80::822a:a8ff:fe49:470c%tESt]:1234') @@ -981,42 +974,35 @@ def test_urlsplit_strip_url(self): self.assertEqual(p.scheme, "https") self.assertEqual(p.geturl(), "https://www.python.org/") - def test_attributes_bad_port(self): + @support.subTests('bytes', (False, True)) + @support.subTests('parse', (urllib.parse.urlsplit, urllib.parse.urlparse)) + @support.subTests('port', ("foo", "1.5", "-1", "0x10", "-0", "1_1", " 1", "1 ", "६")) + def test_attributes_bad_port(self, bytes, parse, port): """Check handling of invalid ports.""" - for bytes in (False, True): - for parse in (urllib.parse.urlsplit, urllib.parse.urlparse): - for port in ("foo", "1.5", "-1", "0x10", "-0", "1_1", " 1", "1 ", "६"): - with self.subTest(bytes=bytes, parse=parse, port=port): - netloc = "www.example.net:" + port - url = "http://" + netloc + "/" - if bytes: - if netloc.isascii() and port.isascii(): - netloc = netloc.encode("ascii") - url = url.encode("ascii") - else: - continue - p = parse(url) - self.assertEqual(p.netloc, netloc) - with self.assertRaises(ValueError): - p.port + netloc = "www.example.net:" + port + url = "http://" + netloc + "/" + if bytes: + if not (netloc.isascii() and port.isascii()): + self.skipTest('non-ASCII bytes') + netloc = str_encode(netloc) + url = str_encode(url) + p = parse(url) + self.assertEqual(p.netloc, netloc) + with self.assertRaises(ValueError): + p.port - def test_attributes_bad_scheme(self): + @support.subTests('bytes', (False, True)) + @support.subTests('parse', (urllib.parse.urlsplit, urllib.parse.urlparse)) + @support.subTests('scheme', (".", "+", "-", "0", "http&", "६http")) + def test_attributes_bad_scheme(self, bytes, parse, scheme): """Check handling of invalid schemes.""" - for bytes in (False, True): - for parse in (urllib.parse.urlsplit, urllib.parse.urlparse): - for scheme in (".", "+", "-", "0", "http&", "६http"): - with self.subTest(bytes=bytes, parse=parse, scheme=scheme): - url = scheme + "://www.example.net" - if bytes: - if url.isascii(): - url = url.encode("ascii") - else: - continue - p = parse(url) - if bytes: - self.assertEqual(p.scheme, b"") - else: - self.assertEqual(p.scheme, "") + url = scheme + "://www.example.net" + if bytes: + if not url.isascii(): + self.skipTest('non-ASCII bytes') + url = url.encode("ascii") + p = parse(url) + self.assertEqual(p.scheme, b"" if bytes else "") def test_attributes_without_netloc(self): # This example is straight from RFC 3261. It looks like it @@ -1128,24 +1114,21 @@ def test_anyscheme(self): self.assertEqual(urllib.parse.urlparse(b"x-newscheme://foo.com/stuff?query"), (b'x-newscheme', b'foo.com', b'/stuff', b'', b'query', b'')) - def test_default_scheme(self): + @support.subTests('func', (urllib.parse.urlparse, urllib.parse.urlsplit)) + def test_default_scheme(self, func): # Exercise the scheme parameter of urlparse() and urlsplit() - for func in (urllib.parse.urlparse, urllib.parse.urlsplit): - with self.subTest(function=func): - result = func("http://example.net/", "ftp") - self.assertEqual(result.scheme, "http") - result = func(b"http://example.net/", b"ftp") - self.assertEqual(result.scheme, b"http") - self.assertEqual(func("path", "ftp").scheme, "ftp") - self.assertEqual(func("path", scheme="ftp").scheme, "ftp") - self.assertEqual(func(b"path", scheme=b"ftp").scheme, b"ftp") - self.assertEqual(func("path").scheme, "") - self.assertEqual(func(b"path").scheme, b"") - self.assertEqual(func(b"path", "").scheme, b"") - - def test_parse_fragments(self): - # Exercise the allow_fragments parameter of urlparse() and urlsplit() - tests = ( + result = func("http://example.net/", "ftp") + self.assertEqual(result.scheme, "http") + result = func(b"http://example.net/", b"ftp") + self.assertEqual(result.scheme, b"http") + self.assertEqual(func("path", "ftp").scheme, "ftp") + self.assertEqual(func("path", scheme="ftp").scheme, "ftp") + self.assertEqual(func(b"path", scheme=b"ftp").scheme, b"ftp") + self.assertEqual(func("path").scheme, "") + self.assertEqual(func(b"path").scheme, b"") + self.assertEqual(func(b"path", "").scheme, b"") + + @support.subTests('url,attr,expected_frag', ( ("http:#frag", "path", "frag"), ("//example.net#frag", "path", "frag"), ("index.html#frag", "path", "frag"), @@ -1156,24 +1139,24 @@ def test_parse_fragments(self): ("//abc#@frag", "path", "@frag"), ("//abc:80#@frag", "path", "@frag"), ("//abc#@frag:80", "path", "@frag:80"), - ) - for url, attr, expected_frag in tests: - for func in (urllib.parse.urlparse, urllib.parse.urlsplit): - if attr == "params" and func is urllib.parse.urlsplit: - attr = "path" - with self.subTest(url=url, function=func): - result = func(url, allow_fragments=False) - self.assertEqual(result.fragment, "") - self.assertEndsWith(getattr(result, attr), - "#" + expected_frag) - self.assertEqual(func(url, "", False).fragment, "") - - result = func(url, allow_fragments=True) - self.assertEqual(result.fragment, expected_frag) - self.assertNotEndsWith(getattr(result, attr), expected_frag) - self.assertEqual(func(url, "", True).fragment, - expected_frag) - self.assertEqual(func(url).fragment, expected_frag) + )) + @support.subTests('func', (urllib.parse.urlparse, urllib.parse.urlsplit)) + def test_parse_fragments(self, url, attr, expected_frag, func): + # Exercise the allow_fragments parameter of urlparse() and urlsplit() + if attr == "params" and func is urllib.parse.urlsplit: + attr = "path" + result = func(url, allow_fragments=False) + self.assertEqual(result.fragment, "") + self.assertEndsWith(getattr(result, attr), + "#" + expected_frag) + self.assertEqual(func(url, "", False).fragment, "") + + result = func(url, allow_fragments=True) + self.assertEqual(result.fragment, expected_frag) + self.assertNotEndsWith(getattr(result, attr), expected_frag) + self.assertEqual(func(url, "", True).fragment, + expected_frag) + self.assertEqual(func(url).fragment, expected_frag) def test_mixed_types_rejected(self): # Several functions that process either strings or ASCII encoded bytes @@ -1199,7 +1182,14 @@ def test_mixed_types_rejected(self): with self.assertRaisesRegex(TypeError, "Cannot mix str"): urllib.parse.urljoin(b"http://python.org", "http://python.org") - def _check_result_type(self, str_type): + @support.subTests('result_type', [ + urllib.parse.DefragResult, + urllib.parse.SplitResult, + urllib.parse.ParseResult, + ]) + def test_result_pairs(self, result_type): + # Check encoding and decoding between result pairs + str_type = result_type num_args = len(str_type._fields) bytes_type = str_type._encoded_counterpart self.assertIs(bytes_type._decoded_counterpart, str_type) @@ -1224,16 +1214,6 @@ def _check_result_type(self, str_type): self.assertEqual(str_result.encode(encoding, errors), bytes_args) self.assertEqual(str_result.encode(encoding, errors), bytes_result) - def test_result_pairs(self): - # Check encoding and decoding between result pairs - result_types = [ - urllib.parse.DefragResult, - urllib.parse.SplitResult, - urllib.parse.ParseResult, - ] - for result_type in result_types: - self._check_result_type(result_type) - def test_parse_qs_encoding(self): result = urllib.parse.parse_qs("key=\u0141%E9", encoding="latin-1") self.assertEqual(result, {'key': ['\u0141\xE9']}) @@ -1265,8 +1245,7 @@ def test_parse_qsl_max_num_fields(self): urllib.parse.parse_qsl('&'.join(['a=a']*11), max_num_fields=10) urllib.parse.parse_qsl('&'.join(['a=a']*10), max_num_fields=10) - def test_parse_qs_separator(self): - parse_qs_semicolon_cases = [ + @support.subTests('orig,expect', [ (";", {}), (";;", {}), (";a=b", {'a': ['b']}), @@ -1277,17 +1256,14 @@ def test_parse_qs_separator(self): (b";a=b", {b'a': [b'b']}), (b"a=a+b;b=b+c", {b'a': [b'a b'], b'b': [b'b c']}), (b"a=1;a=2", {b'a': [b'1', b'2']}), - ] - for orig, expect in parse_qs_semicolon_cases: - with self.subTest(f"Original: {orig!r}, Expected: {expect!r}"): - result = urllib.parse.parse_qs(orig, separator=';') - self.assertEqual(result, expect, "Error parsing %r" % orig) - result_bytes = urllib.parse.parse_qs(orig, separator=b';') - self.assertEqual(result_bytes, expect, "Error parsing %r" % orig) - - - def test_parse_qsl_separator(self): - parse_qsl_semicolon_cases = [ + ]) + def test_parse_qs_separator(self, orig, expect): + result = urllib.parse.parse_qs(orig, separator=';') + self.assertEqual(result, expect) + result_bytes = urllib.parse.parse_qs(orig, separator=b';') + self.assertEqual(result_bytes, expect) + + @support.subTests('orig,expect', [ (";", []), (";;", []), (";a=b", [('a', 'b')]), @@ -1298,13 +1274,12 @@ def test_parse_qsl_separator(self): (b";a=b", [(b'a', b'b')]), (b"a=a+b;b=b+c", [(b'a', b'a b'), (b'b', b'b c')]), (b"a=1;a=2", [(b'a', b'1'), (b'a', b'2')]), - ] - for orig, expect in parse_qsl_semicolon_cases: - with self.subTest(f"Original: {orig!r}, Expected: {expect!r}"): - result = urllib.parse.parse_qsl(orig, separator=';') - self.assertEqual(result, expect, "Error parsing %r" % orig) - result_bytes = urllib.parse.parse_qsl(orig, separator=b';') - self.assertEqual(result_bytes, expect, "Error parsing %r" % orig) + ]) + def test_parse_qsl_separator(self, orig, expect): + result = urllib.parse.parse_qsl(orig, separator=';') + self.assertEqual(result, expect) + result_bytes = urllib.parse.parse_qsl(orig, separator=b';') + self.assertEqual(result_bytes, expect) def test_parse_qsl_bytes(self): self.assertEqual(urllib.parse.parse_qsl(b'a=b'), [(b'a', b'b')]) @@ -1451,15 +1426,20 @@ def test_splitting_bracketed_hosts(self): self.assertEqual(p1.username, 'user') self.assertEqual(p1.path, '/path') self.assertEqual(p1.port, 1234) - p2 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7%test]/path?query') - self.assertEqual(p2.hostname, '0439:23af:2309::fae7%test') + p2 = urllib.parse.urlsplit('scheme://user@[V6a.ip]:1234/path?query') + self.assertEqual(p2.hostname, 'v6a.ip') self.assertEqual(p2.username, 'user') self.assertEqual(p2.path, '/path') - self.assertIs(p2.port, None) - p3 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7:1234:192.0.2.146%test]/path?query') - self.assertEqual(p3.hostname, '0439:23af:2309::fae7:1234:192.0.2.146%test') + self.assertEqual(p2.port, 1234) + p3 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7%test]/path?query') + self.assertEqual(p3.hostname, '0439:23af:2309::fae7%test') self.assertEqual(p3.username, 'user') self.assertEqual(p3.path, '/path') + self.assertIs(p3.port, None) + p4 = urllib.parse.urlsplit('scheme://user@[0439:23af:2309::fae7:1234:192.0.2.146%test]/path?query') + self.assertEqual(p4.hostname, '0439:23af:2309::fae7:1234:192.0.2.146%test') + self.assertEqual(p4.username, 'user') + self.assertEqual(p4.path, '/path') def test_port_casting_failure_message(self): message = "Port could not be cast to integer value as 'oracle'" @@ -1695,11 +1675,12 @@ def test_to_bytes(self): self.assertRaises(UnicodeError, urllib.parse._to_bytes, 'http://www.python.org/medi\u00e6val') - def test_unwrap(self): - for wrapped_url in ('<URL:scheme://host/path>', '<scheme://host/path>', - 'URL:scheme://host/path', 'scheme://host/path'): - url = urllib.parse.unwrap(wrapped_url) - self.assertEqual(url, 'scheme://host/path') + @support.subTests('wrapped_url', + ('<URL:scheme://host/path>', '<scheme://host/path>', + 'URL:scheme://host/path', 'scheme://host/path')) + def test_unwrap(self, wrapped_url): + url = urllib.parse.unwrap(wrapped_url) + self.assertEqual(url, 'scheme://host/path') class DeprecationTest(unittest.TestCase): @@ -1780,5 +1761,11 @@ def test_to_bytes_deprecation(self): 'urllib.parse.to_bytes() is deprecated as of 3.8') +def str_encode(s): + return s.encode('ascii') + +def tuple_encode(t): + return tuple(str_encode(x) for x in t) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_userdict.py b/Lib/test/test_userdict.py index ace84ef564df72f..13285c9b2a3b7fe 100644 --- a/Lib/test/test_userdict.py +++ b/Lib/test/test_userdict.py @@ -1,8 +1,18 @@ # Check every path through every method of UserDict +from collections import UserDict from test import mapping_tests import unittest import collections +import types + + +class UserDictSubclass(UserDict): + pass + +class UserDictSubclass2(UserDict): + pass + d0 = {} d1 = {"one": 1} @@ -155,6 +165,25 @@ def test_init(self): self.assertRaises(TypeError, collections.UserDict, (), ()) self.assertRaises(TypeError, collections.UserDict.__init__) + def test_data(self): + u = UserDict() + self.assertEqual(u.data, {}) + self.assertIs(type(u.data), dict) + d = {'a': 1, 'b': 2} + u = UserDict(d) + self.assertEqual(u.data, d) + self.assertIsNot(u.data, d) + self.assertIs(type(u.data), dict) + u = UserDict(u) + self.assertEqual(u.data, d) + self.assertIs(type(u.data), dict) + u = UserDict([('a', 1), ('b', 2)]) + self.assertEqual(u.data, d) + self.assertIs(type(u.data), dict) + u = UserDict(a=1, b=2) + self.assertEqual(u.data, d) + self.assertIs(type(u.data), dict) + def test_update(self): for kw in 'self', 'dict', 'other', 'iterable': d = collections.UserDict() @@ -166,7 +195,7 @@ def test_update(self): def test_missing(self): # Make sure UserDict doesn't have a __missing__ method - self.assertEqual(hasattr(collections.UserDict, "__missing__"), False) + self.assertNotHasAttr(collections.UserDict, "__missing__") # Test several cases: # (D) subclass defines __missing__ method returning a value # (E) subclass defines __missing__ method raising RuntimeError @@ -215,6 +244,69 @@ class G(collections.UserDict): test_repr_deep = mapping_tests.TestHashMappingProtocol.test_repr_deep + def test_mixed_or(self): + for t in UserDict, dict, types.MappingProxyType: + with self.subTest(t.__name__): + u = UserDict({0: 'a', 1: 'b'}) | t({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDict) + + u = t({0: 'a', 1: 'b'}) | UserDict({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDict) + + u = UserDict({0: 'a', 1: 'b'}) | UserDictSubclass({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDict) + + u = UserDictSubclass({0: 'a', 1: 'b'}) | UserDict({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDictSubclass) + + u = UserDictSubclass({0: 'a', 1: 'b'}) | UserDictSubclass2({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDictSubclass) + + u = UserDict({1: 'c', 2: 'd'}).__ror__(UserDict({0: 'a', 1: 'b'})) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDict) + + u = UserDictSubclass({1: 'c', 2: 'd'}).__ror__(UserDictSubclass2({0: 'a', 1: 'b'})) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDictSubclass) + + def test_mixed_ior(self): + for t in UserDict, dict, types.MappingProxyType: + with self.subTest(t.__name__): + u = u2 = UserDict({0: 'a', 1: 'b'}) + u |= t({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDict) + self.assertIs(u, u2) + + u = dict({0: 'a', 1: 'b'}) + u |= UserDict({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), dict) + + u = u2 = UserDict({0: 'a', 1: 'b'}) + u |= UserDictSubclass({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDict) + self.assertIs(u, u2) + + u = u2 = UserDictSubclass({0: 'a', 1: 'b'}) + u |= UserDict({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDictSubclass) + self.assertIs(u, u2) + + u = u2 = UserDictSubclass({0: 'a', 1: 'b'}) + u |= UserDictSubclass2({1: 'c', 2: 'd'}) + self.assertEqual(u, {0: 'a', 1: 'c', 2: 'd'}) + self.assertIs(type(u), UserDictSubclass) + self.assertIs(u, u2) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_userlist.py b/Lib/test/test_userlist.py index d3d9f4cff8da3ad..3e5c5ff19458eb1 100644 --- a/Lib/test/test_userlist.py +++ b/Lib/test/test_userlist.py @@ -2,12 +2,36 @@ from collections import UserList from test import list_tests +from test import support import unittest +class UserListSubclass(UserList): + pass + +class UserListSubclass2(UserList): + pass + + class UserListTest(list_tests.CommonTest): type2test = UserList + def test_data(self): + u = UserList() + self.assertEqual(u.data, []) + self.assertIs(type(u.data), list) + a = [1, 2] + u = UserList(a) + self.assertEqual(u.data, a) + self.assertIsNot(u.data, a) + self.assertIs(type(u.data), list) + u = UserList(u) + self.assertEqual(u.data, a) + self.assertIs(type(u.data), list) + u = UserList("spam") + self.assertEqual(u.data, list("spam")) + self.assertIs(type(u.data), list) + def test_getslice(self): super().test_getslice() l = [0, 1, 2, 3, 4] @@ -24,34 +48,74 @@ def test_slice_type(self): self.assertIsInstance(u[:], u.__class__) self.assertEqual(u[:],u) - def test_add_specials(self): - u = UserList("spam") - u2 = u + "eggs" - self.assertEqual(u2, list("spameggs")) + def test_mixed_add(self): + for t in UserList, list, str, tuple, iter: + with self.subTest(t.__name__): + u = UserList("spam") + t("eggs") + self.assertEqual(u, list("spameggs")) + self.assertIs(type(u), UserList) + + u = t("spam") + UserList("eggs") + self.assertEqual(u, list("spameggs")) + self.assertIs(type(u), UserList) + + u = UserList("spam") + UserListSubclass("eggs") + self.assertEqual(u, list("spameggs")) + self.assertIs(type(u), UserList) - def test_radd_specials(self): - u = UserList("eggs") - u2 = "spam" + u + u = UserListSubclass("spam") + UserList("eggs") + self.assertEqual(u, list("spameggs")) + self.assertIs(type(u), UserListSubclass) + + u = UserListSubclass("spam") + UserListSubclass2("eggs") + self.assertEqual(u, list("spameggs")) + self.assertIs(type(u), UserListSubclass) + + u2 = UserList("eggs").__radd__(UserList("spam")) self.assertEqual(u2, list("spameggs")) - u2 = u.__radd__(UserList("spam")) + self.assertIs(type(u), UserListSubclass) + + u2 = UserListSubclass("eggs").__radd__(UserListSubclass2("spam")) self.assertEqual(u2, list("spameggs")) + self.assertIs(type(u), UserListSubclass) - def test_iadd(self): - super().test_iadd() - u = [0, 1] - u += UserList([0, 1]) - self.assertEqual(u, [0, 1, 0, 1]) + def test_mixed_iadd(self): + for t in UserList, list, str, tuple, iter: + with self.subTest(t.__name__): + u = u2 = UserList("spam") + u += t("eggs") + self.assertEqual(u, list("spameggs")) + self.assertIs(type(u), UserList) + self.assertIs(u, u2) - def test_mixedcmp(self): - u = self.type2test([0, 1]) - self.assertEqual(u, [0, 1]) - self.assertNotEqual(u, [0]) - self.assertNotEqual(u, [0, 2]) + u = t("spam") + u += UserList("eggs") + self.assertEqual(u, list("spameggs")) + self.assertIs(type(u), UserList) - def test_mixedadd(self): + u = u2 = UserList("spam") + u += UserListSubclass("eggs") + self.assertEqual(u, list("spameggs")) + self.assertIs(type(u), UserList) + self.assertIs(u, u2) + + u = u2 = UserListSubclass("spam") + u += UserList("eggs") + self.assertEqual(u, list("spameggs")) + self.assertIs(type(u), UserListSubclass) + self.assertIs(u, u2) + + u = u2 = UserListSubclass("spam") + u += UserListSubclass2("eggs") + self.assertEqual(u, list("spameggs")) + self.assertIs(type(u), UserListSubclass) + self.assertIs(u, u2) + + def test_mixed_cmp(self): u = self.type2test([0, 1]) - self.assertEqual(u + [], u) - self.assertEqual(u + [2], [0, 1, 2]) + self._assert_cmp(u, [0, 1], 0) + self._assert_cmp(u, [0], 1) + self._assert_cmp(u, [0, 2], -1) def test_getitemoverwriteiter(self): # Verify that __getitem__ overrides *are* recognized by __iter__ @@ -60,6 +124,43 @@ def __getitem__(self, key): return str(key) + '!!!' self.assertEqual(next(iter(T((1,2)))), "0!!!") + def test_implementation(self): + u = UserList([1]) + with (support.swap_attr(UserList, '__len__', None), + support.swap_attr(UserList, 'insert', None)): + u.append(2) + self.assertEqual(u, [1, 2]) + with support.swap_attr(UserList, 'append', None): + u.extend([3, 4]) + self.assertEqual(u, [1, 2, 3, 4]) + with support.swap_attr(UserList, 'append', None): + u.extend(UserList([3, 4])) + self.assertEqual(u, [1, 2, 3, 4, 3, 4]) + with support.swap_attr(UserList, '__iter__', None): + c = u.count(3) + self.assertEqual(c, 2) + with (support.swap_attr(UserList, '__iter__', None), + support.swap_attr(UserList, '__getitem__', None)): + i = u.index(4) + self.assertEqual(i, 3) + with (support.swap_attr(UserList, 'index', None), + support.swap_attr(UserList, '__getitem__', None)): + u.remove(3) + self.assertEqual(u, [1, 2, 4, 3, 4]) + with (support.swap_attr(UserList, '__getitem__', None), + support.swap_attr(UserList, '__delitem__', None)): + u.pop() + self.assertEqual(u, [1, 2, 4, 3]) + with (support.swap_attr(UserList, '__len__', None), + support.swap_attr(UserList, '__getitem__', None), + support.swap_attr(UserList, '__setitem__', None)): + u.reverse() + self.assertEqual(u, [3, 4, 2, 1]) + with (support.swap_attr(UserList, '__len__', None), + support.swap_attr(UserList, 'pop', None)): + u.clear() + self.assertEqual(u, []) + def test_userlist_copy(self): u = self.type2test([6, 8, 1, 9, 1]) v = u.copy() diff --git a/Lib/test/test_userstring.py b/Lib/test/test_userstring.py index 74df52f5412af01..cc85c06bf933633 100644 --- a/Lib/test/test_userstring.py +++ b/Lib/test/test_userstring.py @@ -3,9 +3,18 @@ import unittest from test import string_tests +from test import support from collections import UserString + +class UserStringSubclass(UserString): + pass + +class UserStringSubclass2(UserString): + pass + + class UserStringTest( string_tests.StringLikeTest, unittest.TestCase @@ -40,6 +49,78 @@ def checkcall(self, object, methodname, *args): # we don't fix the arguments, because UserString can't cope with it getattr(object, methodname)(*args) + def test_data(self): + u = UserString("spam") + self.assertEqual(u.data, "spam") + self.assertIs(type(u.data), str) + u = UserString(u) + self.assertEqual(u.data, "spam") + self.assertIs(type(u.data), str) + u = UserString(42) + self.assertEqual(u.data, "42") + self.assertIs(type(u.data), str) + + def test_mixed_add(self): + u = UserString("spam") + "eggs" + self.assertEqual(u, "spameggs") + self.assertIs(type(u), UserString) + + u = "spam" + UserString("eggs") + self.assertEqual(u, "spameggs") + self.assertIs(type(u), UserString) + + u = UserString("spam") + UserStringSubclass("eggs") + self.assertEqual(u, "spameggs") + self.assertIs(type(u), UserString) + + u = UserStringSubclass("spam") + UserString("eggs") + self.assertEqual(u, "spameggs") + self.assertIs(type(u), UserStringSubclass) + + u = UserStringSubclass("spam") + UserStringSubclass2("eggs") + self.assertEqual(u, "spameggs") + self.assertIs(type(u), UserStringSubclass) + + u2 = UserString("eggs").__radd__(UserString("spam")) + self.assertEqual(u2, "spameggs") + self.assertIs(type(u), UserStringSubclass) + + u2 = UserStringSubclass("eggs").__radd__(UserStringSubclass2("spam")) + self.assertEqual(u2, "spameggs") + self.assertIs(type(u), UserStringSubclass) + + def test_mixed_iadd(self): + u = UserString("spam") + u += "eggs" + self.assertEqual(u, "spameggs") + self.assertIs(type(u), UserString) + + u = "spam" + u += UserString("eggs") + self.assertEqual(u, "spameggs") + self.assertIs(type(u), UserString) + + u = UserString("spam") + u += UserStringSubclass("eggs") + self.assertEqual(u, "spameggs") + self.assertIs(type(u), UserString) + + u = UserStringSubclass("spam") + u += UserString("eggs") + self.assertEqual(u, "spameggs") + self.assertIs(type(u), UserStringSubclass) + + u = UserStringSubclass("spam") + u += UserStringSubclass2("eggs") + self.assertEqual(u, "spameggs") + self.assertIs(type(u), UserStringSubclass) + + def test_mixed_cmp(self): + a = self.fixtype('ab') + self._assert_cmp(a, 'ab', 0) + self._assert_cmp(a, 'a', 1) + self._assert_cmp(a, 'ac', -1) + def test_rmod(self): class ustr2(UserString): pass @@ -66,6 +147,20 @@ def test_encode_explicit_none_args(self): # Check that errors defaults to 'strict' self.checkraises(UnicodeError, '\ud800', 'encode', None, None) + def test_implementation(self): + s = UserString('ababahalamaha') + with support.swap_attr(UserString, '__iter__', None): + c = s.count('a') + c2 = s.count(UserString('a')) + self.assertEqual(c, 7) + self.assertEqual(c2, 7) + with (support.swap_attr(UserString, '__iter__', None), + support.swap_attr(UserString, '__getitem__', None)): + i = s.index('h') + i2 = s.index(UserString('h')) + self.assertEqual(i, 5) + self.assertEqual(i2, 5) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 958be5408ce90a7..c8400da3ae987d2 100755 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -14,6 +14,7 @@ from test import support from test.support import import_helper +from test.support.script_helper import assert_python_ok py_uuid = import_helper.import_fresh_module('uuid', blocked=['_uuid']) c_uuid = import_helper.import_fresh_module('uuid', fresh=['_uuid']) @@ -1139,6 +1140,23 @@ def test_uuid_weakref(self): weak = weakref.ref(strong) self.assertIs(strong, weak()) + +class CommandLineTestCases: + uuid = None # to be defined in subclasses + + def do_test_standalone_uuid(self, version): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.uuid.main() + output = stdout.getvalue().strip() + u = self.uuid.UUID(output) + self.assertEqual(output, str(u)) + self.assertEqual(u.version, version) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid1"]) + def test_cli_uuid1(self): + self.do_test_standalone_uuid(1) + @mock.patch.object(sys, "argv", ["", "-u", "uuid3", "-n", "@dns"]) @mock.patch('sys.stderr', new_callable=io.StringIO) def test_cli_namespace_required_for_uuid3(self, mock_err): @@ -1158,6 +1176,47 @@ def test_cli_name_required_for_uuid3(self, mock_err): self.assertEqual(cm.exception.code, 2) self.assertIn("error: Incorrect number of arguments", mock_err.getvalue()) + @mock.patch.object(sys, "argv", + ["", "-u", "uuid3", "-n", "@dns", "-N", "python.org"]) + def test_cli_uuid3_outputted_with_valid_namespace_and_name(self): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.uuid.main() + + output = stdout.getvalue().strip() + uuid_output = self.uuid.UUID(output) + + # Output should be in the form of uuid3 + self.assertEqual(output, str(uuid_output)) + self.assertEqual(uuid_output.version, 3) + + @mock.patch.object(sys, "argv", + ["", "-u", "uuid3", "-n", + "0d6a16cc-34a7-47d8-b660-214d0ae184d2", + "-N", "some.user"]) + def test_cli_uuid3_outputted_with_custom_namespace_and_name(self): + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + self.uuid.main() + + output = stdout.getvalue().strip() + uuid_output = self.uuid.UUID(output) + + # Output should be in the form of uuid3 + self.assertEqual(output, str(uuid_output)) + self.assertEqual(uuid_output.version, 3) + + @mock.patch.object(sys, "argv", + ["", "-u", "uuid3", "-n", "any UUID", "-N", "python.org"]) + @mock.patch('sys.stderr', new_callable=io.StringIO) + def test_cli_uuid3_with_invalid_namespace(self, mock_err): + with self.assertRaises(SystemExit) as cm: + self.uuid.main() + # Check that exception code is the same as argparse.ArgumentParser.error + self.assertEqual(cm.exception.code, 2) + self.assertIn("error: badly formed hexadecimal UUID string", + mock_err.getvalue()) + @mock.patch.object(sys, "argv", [""]) def test_cli_uuid4_outputted_with_no_args(self): stdout = io.StringIO() @@ -1186,8 +1245,8 @@ def test_cli_uuid4_outputted_with_count(self): self.assertEqual(uuid_output.version, 4) @mock.patch.object(sys, "argv", - ["", "-u", "uuid3", "-n", "@dns", "-N", "python.org"]) - def test_cli_uuid3_ouputted_with_valid_namespace_and_name(self): + ["", "-u", "uuid5", "-n", "@dns", "-N", "python.org"]) + def test_cli_uuid5_outputted_with_valid_namespace_and_name(self): stdout = io.StringIO() with contextlib.redirect_stdout(stdout): self.uuid.main() @@ -1197,11 +1256,13 @@ def test_cli_uuid3_ouputted_with_valid_namespace_and_name(self): # Output should be in the form of uuid5 self.assertEqual(output, str(uuid_output)) - self.assertEqual(uuid_output.version, 3) + self.assertEqual(uuid_output.version, 5) @mock.patch.object(sys, "argv", - ["", "-u", "uuid5", "-n", "@dns", "-N", "python.org"]) - def test_cli_uuid5_ouputted_with_valid_namespace_and_name(self): + ["", "-u", "uuid5", "-n", + "0d6a16cc-34a7-47d8-b660-214d0ae184d2", + "-N", "some.user"]) + def test_cli_uuid5_ouputted_with_custom_namespace_and_name(self): stdout = io.StringIO() with contextlib.redirect_stdout(stdout): self.uuid.main() @@ -1213,14 +1274,64 @@ def test_cli_uuid5_ouputted_with_valid_namespace_and_name(self): self.assertEqual(output, str(uuid_output)) self.assertEqual(uuid_output.version, 5) + @mock.patch.object(sys, "argv", + ["", "-u", "uuid5", "-n", "any UUID", "-N", "python.org"]) + @mock.patch('sys.stderr', new_callable=io.StringIO) + def test_cli_uuid5_with_invalid_namespace(self, mock_err): + with self.assertRaises(SystemExit) as cm: + self.uuid.main() + # Check that exception code is the same as argparse.ArgumentParser.error + self.assertEqual(cm.exception.code, 2) + self.assertIn("error: badly formed hexadecimal UUID string", + mock_err.getvalue()) -class TestUUIDWithoutExtModule(BaseTestUUID, unittest.TestCase): + @mock.patch.object(sys, "argv", ["", "-u", "uuid6"]) + def test_cli_uuid6(self): + self.do_test_standalone_uuid(6) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid7"]) + def test_cli_uuid7(self): + self.do_test_standalone_uuid(7) + + @mock.patch.object(sys, "argv", ["", "-u", "uuid8"]) + def test_cli_uuid8(self): + self.do_test_standalone_uuid(8) + + +class TestUUIDWithoutExtModule(CommandLineTestCases, BaseTestUUID, unittest.TestCase): uuid = py_uuid + @unittest.skipUnless(c_uuid, 'requires the C _uuid module') -class TestUUIDWithExtModule(BaseTestUUID, unittest.TestCase): +class TestUUIDWithExtModule(CommandLineTestCases, BaseTestUUID, unittest.TestCase): uuid = c_uuid + def check_has_stable_libuuid_extractable_node(self): + if not self.uuid._has_stable_extractable_node: + self.skipTest("libuuid cannot deduce MAC address") + + @unittest.skipUnless(os.name == 'posix', 'POSIX only') + def test_unix_getnode_from_libuuid(self): + self.check_has_stable_libuuid_extractable_node() + script = 'import uuid; print(uuid._unix_getnode())' + _, n_a, _ = assert_python_ok('-c', script) + _, n_b, _ = assert_python_ok('-c', script) + n_a, n_b = n_a.decode().strip(), n_b.decode().strip() + self.assertTrue(n_a.isdigit()) + self.assertTrue(n_b.isdigit()) + self.assertEqual(n_a, n_b) + + @unittest.skipUnless(os.name == 'nt', 'Windows only') + def test_windows_getnode_from_libuuid(self): + self.check_has_stable_libuuid_extractable_node() + script = 'import uuid; print(uuid._windll_getnode())' + _, n_a, _ = assert_python_ok('-c', script) + _, n_b, _ = assert_python_ok('-c', script) + n_a, n_b = n_a.decode().strip(), n_b.decode().strip() + self.assertTrue(n_a.isdigit()) + self.assertTrue(n_b.isdigit()) + self.assertEqual(n_a, n_b) + class BaseTestInternals: _uuid = py_uuid diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index adc86a49b0668d7..9937acd06a7a016 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -11,17 +11,17 @@ import os.path import pathlib import re +import shlex import shutil import struct import subprocess import sys import sysconfig import tempfile -import shlex from test.support import (captured_stdout, captured_stderr, skip_if_broken_multiprocessing_synchronize, verbose, requires_subprocess, is_android, is_apple_mobile, - is_emscripten, is_wasi, + is_wasm32, requires_venv_with_pip, TEST_HOME_DIR, requires_resource, copy_python_src_ignore) from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree, @@ -42,7 +42,7 @@ or sys._base_executable != sys.executable, 'cannot run venv.create from within a venv on this platform') -if is_android or is_apple_mobile or is_emscripten or is_wasi: +if is_android or is_apple_mobile or is_wasm32: raise unittest.SkipTest("venv is not available on this platform") @requires_subprocess() @@ -379,6 +379,16 @@ def create_contents(self, paths, filename): with open(fn, 'wb') as f: f.write(b'Still here?') + @unittest.skipUnless(hasattr(os, 'listxattr'), 'test requires os.listxattr') + def test_install_scripts_selinux(self): + """ + gh-145417: Test that install_scripts does not copy SELinux context + when copying scripts. + """ + with patch('os.listxattr') as listxattr_mock: + venv.create(self.env_dir) + listxattr_mock.assert_not_called() + def test_overwrite_existing(self): """ Test creating environment in an existing directory. @@ -522,6 +532,8 @@ def test_special_chars_bash(self): # gh-124651: test quoted strings @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows') + @unittest.skipIf(sys.platform.startswith('netbsd'), + "NetBSD csh fails with quoted special chars; see gh-139308") def test_special_chars_csh(self): """ Test that the template strings are quoted properly (csh) @@ -586,6 +598,51 @@ def test_unicode_in_batch_file(self): ) self.assertEqual(out.strip(), '0') + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') + def test_activate_bat_respects_disable_prompt(self): + rmtree(self.env_dir) + env_dir = os.path.join(os.path.realpath(self.env_dir), 'venv') + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate.bat') + test_batch = os.path.join(self.env_dir, 'test_disable_prompt.bat') + with open(test_batch, "w") as f: + f.write('@echo off\n' + 'set "PROMPT=base$G"\n' + 'set "VIRTUAL_ENV_DISABLE_PROMPT=1"\n' + f'call "{activate}"\n' + 'echo ACTIVE_PROMPT:%PROMPT%\n' + 'echo VIRTUAL_ENV:%VIRTUAL_ENV%\n' + 'set "PROMPT=changed$G"\n' + 'call deactivate\n' + 'echo FINAL_PROMPT:%PROMPT%\n') + out, err = check_output([test_batch]) + lines = out.splitlines() + self.assertEqual(lines[0], b'ACTIVE_PROMPT:base$G') + self.assertEndsWith(lines[1], os.fsencode(env_dir)) + self.assertEqual(lines[2], b'FINAL_PROMPT:changed$G') + + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') + def test_activate_bat_prefixes_prompt_by_default(self): + rmtree(self.env_dir) + env_dir = os.path.join(os.path.realpath(self.env_dir), 'venv') + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate.bat') + test_batch = os.path.join(self.env_dir, 'test_enable_prompt.bat') + with open(test_batch, "w") as f: + f.write('@echo off\n' + 'set "PROMPT=base) $G"\n' + 'set "VIRTUAL_ENV_DISABLE_PROMPT="\n' + f'call "{activate}"\n' + 'echo ACTIVE_PROMPT:%PROMPT%\n' + 'call deactivate\n' + 'echo FINAL_PROMPT:%PROMPT%\n') + out, err = check_output([test_batch]) + lines = out.splitlines() + self.assertEqual(lines[0], b'ACTIVE_PROMPT:(venv) base) $G') + self.assertEqual(lines[1], b'FINAL_PROMPT:base) $G') + @unittest.skipUnless(os.name == 'nt' and can_symlink(), 'symlinks on Windows') def test_failed_symlink(self): @@ -650,6 +707,26 @@ def test_deactivate_with_strict_bash_opts(self): self.assertEqual(out, "".encode()) self.assertEqual(err, "".encode()) + # gh-149701: Test exit code is zero even when hashing is disabled + @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') + def test_deactivate_with_strict_bash_opts_and_hashing_disabled(self): + bash = shutil.which("bash") + if bash is None: + self.skipTest("bash required for this test") + rmtree(self.env_dir) + builder = venv.EnvBuilder(clear=True) + builder.create(self.env_dir) + activate = os.path.join(self.env_dir, self.bindir, "activate") + test_script = os.path.join(self.env_dir, "test_hash_disabled.sh") + with open(test_script, "w") as f: + f.write("set -euo pipefail\n" + "set +h\n" # disable hashing + f"source {activate}\n" + "deactivate") + out, err = check_output([bash, test_script]) + self.assertEqual(out, "".encode()) + self.assertEqual(err, "".encode()) + @unittest.skipUnless(sys.platform == 'darwin', 'only relevant on macOS') def test_macos_env(self): @@ -774,7 +851,7 @@ def test_activate_shell_script_has_no_dos_newlines(self): with open(script_path, 'rb') as script: for i, line in enumerate(script, 1): error_message = f"CR LF found in line {i}" - self.assertFalse(line.endswith(b'\r\n'), error_message) + self.assertNotEndsWith(line, b'\r\n', error_message) @requireVenvCreate def test_scm_ignore_files_git(self): @@ -978,7 +1055,7 @@ def do_test_with_pip(self, system_site_packages): self.assertEqual(err, "") out = out.decode("latin-1") # Force to text, prevent decoding errors expected_version = "pip {}".format(ensurepip.version()) - self.assertEqual(out[:len(expected_version)], expected_version) + self.assertStartsWith(out, expected_version) env_dir = os.fsencode(self.env_dir).decode("latin-1") self.assertIn(env_dir, out) diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py index 05710c469348c46..799ea4939ddab59 100644 --- a/Lib/test/test_warnings/__init__.py +++ b/Lib/test/test_warnings/__init__.py @@ -1,5 +1,6 @@ from contextlib import contextmanager import linecache +import logging import os import importlib import inspect @@ -102,7 +103,7 @@ class PublicAPITests(BaseTest): """ def test_module_all_attribute(self): - self.assertTrue(hasattr(self.module, '__all__')) + self.assertHasAttr(self.module, '__all__') target_api = ["warn", "warn_explicit", "showwarning", "formatwarning", "filterwarnings", "simplefilter", "resetwarnings", "catch_warnings", "deprecated"] @@ -241,6 +242,85 @@ def test_once(self): 42) self.assertEqual(len(w), 0) + def test_filter_module(self): + MS_WINDOWS = (sys.platform == 'win32') + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module=r'package\.module\z') + self.module.warn_explicit('msg', UserWarning, 'filename', 42, + module='package.module') + self.assertEqual(len(w), 1) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module.py', 42) + + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module='package') + self.module.warn_explicit('msg', UserWarning, 'filename', 42, + module='package.module') + self.assertEqual(len(w), 1) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, 'filename', 42, + module='other.package.module') + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, '/path/to/otherpackage/module.py', 42) + + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module=r'/path/to/package/module\z') + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module', 42) + self.assertEqual(len(w), 1) + self.module.warn_explicit('msg', UserWarning, '/path/to/package/module.py', 42) + self.assertEqual(len(w), 2) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, '/PATH/TO/PACKAGE/MODULE', 42) + if MS_WINDOWS: + if self.module is py_warnings: + self.module.warn_explicit('msg', UserWarning, r'/path/to/package/module.PY', 42) + self.assertEqual(len(w), 3) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'/path/to/package/module/__init__.py', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'/path/to/package/module.pyw', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'\path\to\package\module', 42) + + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module=r'/path/to/package/__init__\z') + self.module.warn_explicit('msg', UserWarning, '/path/to/package/__init__.py', 42) + self.assertEqual(len(w), 1) + self.module.warn_explicit('msg', UserWarning, '/path/to/package/__init__', 42) + self.assertEqual(len(w), 2) + + if MS_WINDOWS: + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module=r'C:\\path\\to\\package\\module\z') + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module', 42) + self.assertEqual(len(w), 1) + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module.py', 42) + self.assertEqual(len(w), 2) + if self.module is py_warnings: + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module.PY', 42) + self.assertEqual(len(w), 3) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module.pyw', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'C:\PATH\TO\PACKAGE\MODULE', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'C:/path/to/package/module', 42) + with self.assertRaises(UserWarning): + self.module.warn_explicit('msg', UserWarning, r'C:\path\to\package\module\__init__.py', 42) + + with self.module.catch_warnings(record=True) as w: + self.module.simplefilter('error') + self.module.filterwarnings('always', module=r'<unknown>\z') + self.module.warn_explicit('msg', UserWarning, '', 42) + self.assertEqual(len(w), 1) + def test_module_globals(self): with self.module.catch_warnings(record=True) as w: self.module.simplefilter("always", UserWarning) @@ -419,6 +499,47 @@ def test_catchwarnings_with_simplefilter_error(self): stderr = stderr.getvalue() self.assertIn(error_msg, stderr) + def test_catchwarnings_with_showwarning(self): + # gh-146358: catch_warnings must override warnings.showwarning() + # if it's not the default implementation. + + warns = [] + def custom_showwarning(message, category, filename, lineno, + file=None, line=None): + warns.append(message) + + with self.module.catch_warnings(): + self.module.resetwarnings() + + with support.swap_attr(self.module, 'showwarning', + custom_showwarning): + with self.module.catch_warnings(record=True) as recorded: + self.module.warn("recorded") + self.assertEqual(len(recorded), 1) + self.assertEqual(str(recorded[0].message), 'recorded') + self.assertIs(self.module.showwarning, custom_showwarning) + + self.module.warn("custom") + + self.assertEqual(len(warns), 1) + self.assertEqual(str(warns[0]), "custom") + + def test_catchwarnings_logging(self): + # gh-146358: catch_warnings(record=True) must replace the + # showwarning() function set by logging.captureWarnings(True). + + with self.module.catch_warnings(): + self.module.resetwarnings() + logging.captureWarnings(True) + + with self.module.catch_warnings(record=True) as recorded: + self.module.warn("recorded") + self.assertEqual(len(recorded), 1) + self.assertEqual(str(recorded[0].message), 'recorded') + + logging.captureWarnings(False) + + class CFilterTests(FilterTests, unittest.TestCase): module = c_warnings @@ -555,13 +676,7 @@ def test_warn_explicit_non_ascii_filename(self): with self.module.catch_warnings(record=True) as w: self.module.resetwarnings() self.module.filterwarnings("always", category=UserWarning) - filenames = ["nonascii\xe9\u20ac"] - if not support.is_emscripten: - # JavaScript does not like surrogates. - # Invalid UTF-8 leading byte 0x80 encountered when - # deserializing a UTF-8 string in wasm memory to a JS - # string! - filenames.append("surrogate\udc80") + filenames = ["nonascii\xe9\u20ac", "surrogate\udc80"] for filename in filenames: try: os.fsencode(filename) @@ -735,7 +850,7 @@ class CWarnTests(WarnTests, unittest.TestCase): # test.import_helper.import_fresh_module utility function def test_accelerated(self): self.assertIsNot(original_warnings, self.module) - self.assertFalse(hasattr(self.module.warn, '__code__')) + self.assertNotHasAttr(self.module.warn, '__code__') class PyWarnTests(WarnTests, unittest.TestCase): module = py_warnings @@ -744,7 +859,7 @@ class PyWarnTests(WarnTests, unittest.TestCase): # test.import_helper.import_fresh_module utility function def test_pure_python(self): self.assertIsNot(original_warnings, self.module) - self.assertTrue(hasattr(self.module.warn, '__code__')) + self.assertHasAttr(self.module.warn, '__code__') class WCmdLineTests(BaseTest): @@ -1528,12 +1643,12 @@ def test_late_resource_warning(self): # (_warnings will try to import it) code = "f = open(%a)" % __file__ rc, out, err = assert_python_ok("-Wd", "-c", code) - self.assertTrue(err.startswith(expected), ascii(err)) + self.assertStartsWith(err, expected) # import the warnings module code = "import warnings; f = open(%a)" % __file__ rc, out, err = assert_python_ok("-Wd", "-c", code) - self.assertTrue(err.startswith(expected), ascii(err)) + self.assertStartsWith(err, expected) class AsyncTests(BaseTest): @@ -1875,6 +1990,25 @@ class D(C, x=3): self.assertEqual(D.inited, 3) + def test_existing_init_subclass_in_sibling_base(self): + @deprecated("A will go away soon") + class A: + pass + class B: + def __init_subclass__(cls, x): + super().__init_subclass__() + cls.inited = x + + with self.assertWarnsRegex(DeprecationWarning, "A will go away soon"): + class C(A, B, x=42): + pass + self.assertEqual(C.inited, 42) + + with self.assertWarnsRegex(DeprecationWarning, "A will go away soon"): + class D(B, A, x=42): + pass + self.assertEqual(D.inited, 42) + def test_init_subclass_has_correct_cls(self): init_subclass_saw = None diff --git a/Lib/test/test_wave.py b/Lib/test/test_wave.py index 5e771c8de969ec6..346a343761a7c1b 100644 --- a/Lib/test/test_wave.py +++ b/Lib/test/test_wave.py @@ -2,6 +2,7 @@ from test import audiotests from test import support import io +import os import struct import sys import wave @@ -222,6 +223,14 @@ def test_read_wrong_sample_width(self): with self.assertRaisesRegex(wave.Error, 'bad sample width'): wave.open(io.BytesIO(b)) + def test_open_in_write_raises(self): + # gh-136523: Wave_write.__del__ should not throw + with support.catch_unraisable_exception() as cm: + with self.assertRaises(OSError): + wave.open(os.curdir, "wb") + support.gc_collect() + self.assertIsNone(cm.unraisable) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 4faad6629fe23c2..d776a311ead2e99 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -432,7 +432,7 @@ def check_proxy(self, o, proxy): self.assertEqual(proxy.foo, 2, "proxy does not reflect attribute modification") del o.foo - self.assertFalse(hasattr(proxy, 'foo'), + self.assertNotHasAttr(proxy, 'foo', "proxy does not reflect attribute removal") proxy.foo = 1 @@ -442,7 +442,7 @@ def check_proxy(self, o, proxy): self.assertEqual(o.foo, 2, "object does not reflect attribute modification via proxy") del proxy.foo - self.assertFalse(hasattr(o, 'foo'), + self.assertNotHasAttr(o, 'foo', "object does not reflect attribute removal via proxy") def test_proxy_deletion(self): @@ -1019,7 +1019,7 @@ def __del__(self): pass del x support.gc_collect() - @support.cpython_only + @support.nomemtest def test_no_memory_when_clearing(self): # gh-118331: Make sure we do not raise an exception from the destructor # when clearing weakrefs if allocating the intermediate tuple fails. @@ -1108,7 +1108,7 @@ def meth(self): self.assertEqual(r.slot1, "abc") self.assertEqual(r.slot2, "def") self.assertEqual(r.meth(), "abcdef") - self.assertFalse(hasattr(r, "__dict__")) + self.assertNotHasAttr(r, "__dict__") def test_subclass_refs_with_cycle(self): """Confirm https://bugs.python.org/issue3100 is fixed.""" diff --git a/Lib/test/test_weakset.py b/Lib/test/test_weakset.py index 76e8e5c8ab7d3c5..c1e4f9c8366e58a 100644 --- a/Lib/test/test_weakset.py +++ b/Lib/test/test_weakset.py @@ -466,7 +466,7 @@ def test_copying(self): self.assertIsNot(dup, s) self.assertIs(dup.x, s.x) self.assertIs(dup.z, s.z) - self.assertFalse(hasattr(dup, 'y')) + self.assertNotHasAttr(dup, 'y') dup = copy.deepcopy(s) self.assertIsInstance(dup, cls) @@ -476,7 +476,7 @@ def test_copying(self): self.assertIsNot(dup.x, s.x) self.assertEqual(dup.z, s.z) self.assertIsNot(dup.z, s.z) - self.assertFalse(hasattr(dup, 'y')) + self.assertNotHasAttr(dup, 'y') if __name__ == "__main__": diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index 4c3ea1cd8df13ec..bfbcf112b0b0850 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -57,6 +57,14 @@ def _test(self, meth, *, args=[URL], kw={}, options, arguments): popen_args.pop(popen_args.index(option)) self.assertEqual(popen_args, arguments) + def test_reject_dash_prefixes(self): + browser = self.browser_class(name=CMD_NAME) + with self.assertRaisesRegex( + ValueError, + r"^Invalid URL \(leading dash disallowed\): '--key=val http.*'$" + ): + browser.open(f"--key=val {URL}") + class GenericBrowserCommandTest(CommandTestMixin, unittest.TestCase): @@ -111,6 +119,15 @@ def test_open_bad_new_parameter(self): arguments=[URL], kw=dict(new=999)) + def test_reject_action_dash_prefixes(self): + browser = self.browser_class(name=CMD_NAME) + with self.assertRaises(ValueError): + browser.open('%action--incognito') + # new=1: action is "--new-window", so "%action" itself expands to + # a dash-prefixed flag even with no dash in the original URL. + with self.assertRaises(ValueError): + browser.open('%action', new=1) + class EdgeCommandTest(CommandTestMixin, unittest.TestCase): @@ -321,7 +338,6 @@ def close(self): @unittest.skipUnless(sys.platform == "darwin", "macOS specific test") @requires_subprocess() class MacOSXOSAScriptTest(unittest.TestCase): - def setUp(self): # Ensure that 'BROWSER' is not set to 'open' or something else. # See: https://github.com/python/cpython/issues/131254. @@ -344,7 +360,7 @@ def test_default_open(self): url = "https://python.org" self.browser.open(url) self.assertTrue(self.popen_pipe._closed) - self.assertEqual(self.popen_pipe.cmd, "osascript") + self.assertEqual(self.popen_pipe.cmd, "/usr/bin/osascript") script = self.popen_pipe.pipe.getvalue() self.assertEqual(script.strip(), f'open location "{url}"') @@ -371,6 +387,13 @@ def test_explicit_browser(self): self.assertIn('tell application "safari"', script) self.assertIn('open location "https://python.org"', script) + def test_reject_dash_prefixes(self): + with self.assertRaisesRegex( + ValueError, + r"^Invalid URL \(leading dash disallowed\): '--key=val http.*'$" + ): + self.browser.open(f"--key=val {URL}") + class BrowserRegistrationTest(unittest.TestCase): diff --git a/Lib/test/test_winapi.py b/Lib/test/test_winapi.py index e64208330ad2f9e..b4a2d92c08a236a 100644 --- a/Lib/test/test_winapi.py +++ b/Lib/test/test_winapi.py @@ -11,9 +11,16 @@ MAXIMUM_WAIT_OBJECTS = 64 MAXIMUM_BATCHED_WAIT_OBJECTS = (MAXIMUM_WAIT_OBJECTS - 1) ** 2 + +def close_events(events): + for handle in events: + _winapi.CloseHandle(handle) + + class WinAPIBatchedWaitForMultipleObjectsTests(unittest.TestCase): def _events_waitall_test(self, n): evts = [_winapi.CreateEventW(0, False, False, None) for _ in range(n)] + self.addCleanup(close_events, evts) with self.assertRaises(TimeoutError): _winapi.BatchedWaitForMultipleObjects(evts, True, 100) @@ -41,6 +48,7 @@ def _events_waitall_test(self, n): def _events_waitany_test(self, n): evts = [_winapi.CreateEventW(0, False, False, None) for _ in range(n)] + self.addCleanup(close_events, evts) with self.assertRaises(TimeoutError): _winapi.BatchedWaitForMultipleObjects(evts, False, 100) diff --git a/Lib/test/test_winconsoleio.py b/Lib/test/test_winconsoleio.py index d9076e77c158a26..1bae884ed9ae3ea 100644 --- a/Lib/test/test_winconsoleio.py +++ b/Lib/test/test_winconsoleio.py @@ -17,9 +17,9 @@ class WindowsConsoleIOTests(unittest.TestCase): def test_abc(self): - self.assertTrue(issubclass(ConIO, io.RawIOBase)) - self.assertFalse(issubclass(ConIO, io.BufferedIOBase)) - self.assertFalse(issubclass(ConIO, io.TextIOBase)) + self.assertIsSubclass(ConIO, io.RawIOBase) + self.assertNotIsSubclass(ConIO, io.BufferedIOBase) + self.assertNotIsSubclass(ConIO, io.TextIOBase) def test_open_fd(self): self.assertRaisesRegex(ValueError, diff --git a/Lib/test/test_winreg.py b/Lib/test/test_winreg.py index 924a962781a75bf..1bc830c02c39ce5 100644 --- a/Lib/test/test_winreg.py +++ b/Lib/test/test_winreg.py @@ -3,6 +3,7 @@ import gc import os, sys, errno +import itertools import threading import unittest from platform import machine, win32_edition @@ -291,6 +292,37 @@ def run(self): DeleteKey(HKEY_CURRENT_USER, test_key_name+'\\changing_value') DeleteKey(HKEY_CURRENT_USER, test_key_name) + def test_queryvalueex_race_condition(self): + # gh-142282: QueryValueEx could read garbage buffer under race + # condition when another thread changes the value size + done = False + ready = threading.Event() + values = [b'ham', b'spam'] + + class WriterThread(threading.Thread): + def run(self): + with CreateKey(HKEY_CURRENT_USER, test_key_name) as key: + values_iter = itertools.cycle(values) + while not done: + val = next(values_iter) + SetValueEx(key, 'test_value', 0, REG_BINARY, val) + ready.set() + + thread = WriterThread() + thread.start() + try: + ready.wait() + with CreateKey(HKEY_CURRENT_USER, test_key_name) as key: + for _ in range(1000): + result, typ = QueryValueEx(key, 'test_value') + # The result must be one of the written values, + # not garbage data from uninitialized buffer + self.assertIn(result, values) + finally: + done = True + thread.join() + DeleteKey(HKEY_CURRENT_USER, test_key_name) + def test_long_key(self): # Issue2810, in 2.6 and 3.1 when the key name was exactly 256 # characters, EnumKey raised "WindowsError: More data is diff --git a/Lib/test/test_with.py b/Lib/test/test_with.py index fd7abd1782ec4d2..f16611b29a2658c 100644 --- a/Lib/test/test_with.py +++ b/Lib/test/test_with.py @@ -679,7 +679,7 @@ def testSingleComplexTarget(self): class C: pass blah = C() with mock_contextmanager_generator() as blah.foo: - self.assertEqual(hasattr(blah, "foo"), True) + self.assertHasAttr(blah, "foo") def testMultipleComplexTargets(self): class C: diff --git a/Lib/test/test_wmi.py b/Lib/test/test_wmi.py index ac7c9cb3a5a493a..90eb40439d4b4a2 100644 --- a/Lib/test/test_wmi.py +++ b/Lib/test/test_wmi.py @@ -70,8 +70,8 @@ def test_wmi_query_overflow(self): def test_wmi_query_multiple_rows(self): # Multiple instances should have an extra null separator r = wmi_exec_query("SELECT ProcessId FROM Win32_Process WHERE ProcessId < 1000") - self.assertFalse(r.startswith("\0"), r) - self.assertFalse(r.endswith("\0"), r) + self.assertNotStartsWith(r, "\0") + self.assertNotEndsWith(r, "\0") it = iter(r.split("\0")) try: while True: diff --git a/Lib/test/test_wsgiref.py b/Lib/test/test_wsgiref.py index b047f7b06f85d38..0b33db9378000d9 100644 --- a/Lib/test/test_wsgiref.py +++ b/Lib/test/test_wsgiref.py @@ -1,6 +1,6 @@ from unittest import mock from test import support -from test.support import socket_helper +from test.support import socket_helper, control_characters_c0 from test.test_httpservers import NoLogRequestHandler from unittest import TestCase from wsgiref.util import setup_testing_defaults @@ -149,9 +149,9 @@ def bad_app(environ,start_response): start_response("200 OK", ('Content-Type','text/plain')) return ["Hello, world!"] out, err = run_amock(validator(bad_app)) - self.assertTrue(out.endswith( + self.assertEndsWith(out, b"A server error occurred. Please contact the administrator." - )) + ) self.assertEqual( err.splitlines()[-2], "AssertionError: Headers (('Content-Type', 'text/plain')) must" @@ -174,9 +174,9 @@ def bad_app(environ, start_response): for status, exc_message in tests: with self.subTest(status=status): out, err = run_amock(create_bad_app(status)) - self.assertTrue(out.endswith( + self.assertEndsWith(out, b"A server error occurred. Please contact the administrator." - )) + ) self.assertEqual(err.splitlines()[-2], exc_message) def test_wsgi_input(self): @@ -185,9 +185,9 @@ def bad_app(e,s): s("200 OK", [("Content-Type", "text/plain; charset=utf-8")]) return [b"data"] out, err = run_amock(validator(bad_app)) - self.assertTrue(out.endswith( + self.assertEndsWith(out, b"A server error occurred. Please contact the administrator." - )) + ) self.assertEqual( err.splitlines()[-2], "AssertionError" ) @@ -200,7 +200,7 @@ def app(e, s): ]) return [b"data"] out, err = run_amock(validator(app)) - self.assertTrue(err.endswith('"GET / HTTP/1.0" 200 4\n')) + self.assertEndsWith(err, '"GET / HTTP/1.0" 200 4\n') ver = sys.version.split()[0].encode('ascii') py = python_implementation().encode('ascii') pyver = py + b"/" + ver @@ -503,6 +503,22 @@ def testExtras(self): '\r\n' ) + def testRaisesControlCharacters(self): + for c0 in control_characters_c0(): + with self.subTest(c0): + headers = Headers() + self.assertRaises(ValueError, headers.__setitem__, f"key{c0}", "val") + self.assertRaises(ValueError, headers.add_header, f"key{c0}", "val", param="param") + # HTAB (\x09) is allowed in values, not names. + if c0 == "\t": + headers["key"] = f"val{c0}" + headers.add_header("key", f"val{c0}") + headers.setdefault(f"key", f"val{c0}") + else: + self.assertRaises(ValueError, headers.__setitem__, "key", f"val{c0}") + self.assertRaises(ValueError, headers.add_header, "key", f"val{c0}", param="param") + self.assertRaises(ValueError, headers.add_header, "key", "val", param=f"param{c0}") + class ErrorHandler(BaseCGIHandler): """Simple handler subclass for testing BaseHandler""" @@ -839,6 +855,25 @@ def write(self, b): self.assertIsNotNone(h.status) self.assertIsNotNone(h.environ) + def testRaisesControlCharacters(self): + for c0 in control_characters_c0(): + with self.subTest(c0): + base = BaseHandler() + with self.assertRaises(ValueError): + base.start_response(c0, [('x', 'y')]) + + base = BaseHandler() + with self.assertRaises(ValueError): + base.start_response('200 OK', [(c0, 'y')]) + + # HTAB (\x09) is allowed in header values, but not in names. + base = BaseHandler() + if c0 != "\t": + with self.assertRaises(ValueError): + base.start_response('200 OK', [('x', c0)]) + else: + base.start_response('200 OK', [('x', c0)]) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 5fe9d6884106adc..e268ed7f1ebef3d 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -218,6 +218,33 @@ class ElementTreeTest(unittest.TestCase): def serialize_check(self, elem, expected): self.assertEqual(serialize(elem), expected) + def test_constructor(self): + # Test constructor behavior. + + with self.assertRaises(TypeError): + tree = ET.ElementTree("") + with self.assertRaises(TypeError): + tree = ET.ElementTree(ET.ElementTree()) + + def test_setroot(self): + # Test _setroot behavior. + + tree = ET.ElementTree() + element = ET.Element("tag") + tree._setroot(element) + self.assertEqual(tree.getroot().tag, "tag") + self.assertEqual(tree.getroot(), element) + + # Test behavior with an invalid root element + + tree = ET.ElementTree() + with self.assertRaises(TypeError): + tree._setroot("") + with self.assertRaises(TypeError): + tree._setroot(ET.ElementTree()) + with self.assertRaises(TypeError): + tree._setroot(None) + def test_interface(self): # Test element tree interface. @@ -225,8 +252,7 @@ def check_element(element): self.assertTrue(ET.iselement(element), msg="not an element") direlem = dir(element) for attr in 'tag', 'attrib', 'text', 'tail': - self.assertTrue(hasattr(element, attr), - msg='no %s member' % attr) + self.assertHasAttr(element, attr) self.assertIn(attr, direlem, msg='no %s visible by dir' % attr) @@ -251,7 +277,7 @@ def check_element(element): # Make sure all standard element methods exist. def check_method(method): - self.assertTrue(hasattr(method, '__call__'), + self.assertHasAttr(method, '__call__', msg="%s not callable" % method) check_method(element.append) @@ -548,208 +574,6 @@ def test_parseliteral(self): self.assertEqual(len(ids), 1) self.assertEqual(ids["body"].tag, 'body') - def test_iterparse(self): - # Test iterparse interface. - - iterparse = ET.iterparse - - context = iterparse(SIMPLE_XMLFILE) - self.assertIsNone(context.root) - action, elem = next(context) - self.assertIsNone(context.root) - self.assertEqual((action, elem.tag), ('end', 'element')) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('end', 'element'), - ('end', 'empty-element'), - ('end', 'root'), - ]) - self.assertEqual(context.root.tag, 'root') - - context = iterparse(SIMPLE_NS_XMLFILE) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('end', '{namespace}element'), - ('end', '{namespace}element'), - ('end', '{namespace}empty-element'), - ('end', '{namespace}root'), - ]) - - with open(SIMPLE_XMLFILE, 'rb') as source: - context = iterparse(source) - action, elem = next(context) - self.assertEqual((action, elem.tag), ('end', 'element')) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('end', 'element'), - ('end', 'empty-element'), - ('end', 'root'), - ]) - self.assertEqual(context.root.tag, 'root') - - events = () - context = iterparse(SIMPLE_XMLFILE, events) - self.assertEqual([(action, elem.tag) for action, elem in context], []) - - events = () - context = iterparse(SIMPLE_XMLFILE, events=events) - self.assertEqual([(action, elem.tag) for action, elem in context], []) - - events = ("start", "end") - context = iterparse(SIMPLE_XMLFILE, events) - self.assertEqual([(action, elem.tag) for action, elem in context], [ - ('start', 'root'), - ('start', 'element'), - ('end', 'element'), - ('start', 'element'), - ('end', 'element'), - ('start', 'empty-element'), - ('end', 'empty-element'), - ('end', 'root'), - ]) - - events = ("start", "end", "start-ns", "end-ns") - context = iterparse(SIMPLE_NS_XMLFILE, events) - self.assertEqual([(action, elem.tag) if action in ("start", "end") - else (action, elem) - for action, elem in context], [ - ('start-ns', ('', 'namespace')), - ('start', '{namespace}root'), - ('start', '{namespace}element'), - ('end', '{namespace}element'), - ('start', '{namespace}element'), - ('end', '{namespace}element'), - ('start', '{namespace}empty-element'), - ('end', '{namespace}empty-element'), - ('end', '{namespace}root'), - ('end-ns', None), - ]) - - events = ('start-ns', 'end-ns') - context = iterparse(io.StringIO(r"<root xmlns=''/>"), events) - res = [action for action, elem in context] - self.assertEqual(res, ['start-ns', 'end-ns']) - - events = ("start", "end", "bogus") - with open(SIMPLE_XMLFILE, "rb") as f: - with self.assertRaises(ValueError) as cm: - iterparse(f, events) - self.assertFalse(f.closed) - self.assertEqual(str(cm.exception), "unknown event 'bogus'") - - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(ValueError) as cm: - iterparse(SIMPLE_XMLFILE, events) - self.assertEqual(str(cm.exception), "unknown event 'bogus'") - del cm - - source = io.BytesIO( - b"<?xml version='1.0' encoding='iso-8859-1'?>\n" - b"<body xmlns='http://&#233;ffbot.org/ns'\n" - b" xmlns:cl\xe9='http://effbot.org/ns'>text</body>\n") - events = ("start-ns",) - context = iterparse(source, events) - self.assertEqual([(action, elem) for action, elem in context], [ - ('start-ns', ('', 'http://\xe9ffbot.org/ns')), - ('start-ns', ('cl\xe9', 'http://effbot.org/ns')), - ]) - - source = io.StringIO("<document />junk") - it = iterparse(source) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'document')) - with self.assertRaises(ET.ParseError) as cm: - next(it) - self.assertEqual(str(cm.exception), - 'junk after document element: line 1, column 12') - - self.addCleanup(os_helper.unlink, TESTFN) - with open(TESTFN, "wb") as f: - f.write(b"<document />junk") - it = iterparse(TESTFN) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'document')) - with warnings_helper.check_no_resource_warning(self): - with self.assertRaises(ET.ParseError) as cm: - next(it) - self.assertEqual(str(cm.exception), - 'junk after document element: line 1, column 12') - del cm, it - - # Not exhausting the iterator still closes the resource (bpo-43292) - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - del it - - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - it.close() - del it - - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'element')) - del it, elem - - with warnings_helper.check_no_resource_warning(self): - it = iterparse(SIMPLE_XMLFILE) - action, elem = next(it) - it.close() - self.assertEqual((action, elem.tag), ('end', 'element')) - del it, elem - - with self.assertRaises(FileNotFoundError): - iterparse("nonexistent") - - def test_iterparse_close(self): - iterparse = ET.iterparse - - it = iterparse(SIMPLE_XMLFILE) - it.close() - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - with open(SIMPLE_XMLFILE, 'rb') as source: - it = iterparse(source) - it.close() - self.assertFalse(source.closed) - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - it = iterparse(SIMPLE_XMLFILE) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'element')) - it.close() - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - with open(SIMPLE_XMLFILE, 'rb') as source: - it = iterparse(source) - action, elem = next(it) - self.assertEqual((action, elem.tag), ('end', 'element')) - it.close() - self.assertFalse(source.closed) - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - it = iterparse(SIMPLE_XMLFILE) - list(it) - it.close() - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - - with open(SIMPLE_XMLFILE, 'rb') as source: - it = iterparse(source) - list(it) - it.close() - self.assertFalse(source.closed) - with self.assertRaises(StopIteration): - next(it) - it.close() # idempotent - def test_writefile(self): elem = ET.Element("tag") elem.text = "text" @@ -1151,12 +975,12 @@ def check(encoding, body=''): check("cp437", '\u221a') check("mac-roman", '\u02da') - def xml(encoding): - return "<?xml version='1.0' encoding='%s'?><xml />" % encoding - def bxml(encoding): - return xml(encoding).encode(encoding) + def xml(encoding, body=''): + return "<?xml version='1.0' encoding='%s'?><xml>%s</xml>" % (encoding, body) + def bxml(encoding, body=''): + return xml(encoding, body).encode(encoding) supported_encodings = [ - 'ascii', 'utf-8', 'utf-8-sig', 'utf-16', 'utf-16be', 'utf-16le', + 'utf-8', 'utf-16', 'utf-16be', 'utf-16le', 'iso8859-1', 'iso8859-2', 'iso8859-3', 'iso8859-4', 'iso8859-5', 'iso8859-6', 'iso8859-7', 'iso8859-8', 'iso8859-9', 'iso8859-10', 'iso8859-13', 'iso8859-14', 'iso8859-15', 'iso8859-16', @@ -1167,13 +991,14 @@ def bxml(encoding): 'cp1256', 'cp1257', 'cp1258', 'mac-cyrillic', 'mac-greek', 'mac-iceland', 'mac-latin2', 'mac-roman', 'mac-turkish', - 'iso2022-jp', 'iso2022-jp-1', 'iso2022-jp-2', 'iso2022-jp-2004', - 'iso2022-jp-3', 'iso2022-jp-ext', - 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', - 'hz', 'ptcp154', + 'koi8-r', 'koi8-t', 'koi8-u', 'kz1048', 'ptcp154', ] for encoding in supported_encodings: - self.assertEqual(ET.tostring(ET.XML(bxml(encoding))), b'<xml />') + with self.subTest(encoding=encoding): + self.assertEqual(ET.tostring(ET.XML(bxml(encoding))), b'<xml />') + c = 'éπя\u05d0\u060c€'.encode(encoding, 'ignore').decode(encoding)[0] + self.assertEqual(ET.tostring(ET.XML(bxml(encoding, c))), + ('<xml>&#%d;</xml>' % ord(c)).encode()) unsupported_ascii_compatible_encodings = [ 'big5', 'big5hkscs', @@ -1185,14 +1010,16 @@ def bxml(encoding): 'utf-7', ] for encoding in unsupported_ascii_compatible_encodings: - self.assertRaises(ValueError, ET.XML, bxml(encoding)) + with self.subTest(encoding=encoding): + self.assertRaises(ValueError, ET.XML, bxml(encoding)) unsupported_ascii_incompatible_encodings = [ 'cp037', 'cp424', 'cp500', 'cp864', 'cp875', 'cp1026', 'cp1140', 'utf_32', 'utf_32_be', 'utf_32_le', ] for encoding in unsupported_ascii_incompatible_encodings: - self.assertRaises(ET.ParseError, ET.XML, bxml(encoding)) + with self.subTest(encoding=encoding): + self.assertRaises(ET.ParseError, ET.XML, bxml(encoding)) self.assertRaises(ValueError, ET.XML, xml('undefined').encode('ascii')) self.assertRaises(LookupError, ET.XML, xml('xxx').encode('ascii')) @@ -1420,7 +1247,12 @@ def check(p, expected, namespaces=None): {'': 'http://www.w3.org/2001/XMLSchema', 'ns': 'http://www.w3.org/2001/XMLSchema'}) - def test_processinginstruction(self): + def test_comment_serialization(self): + comm = ET.Comment('<spam> & ham') + # comments are not escaped + self.assertEqual(ET.tostring(comm), b'<!--<spam> & ham-->') + + def test_processinginstruction_serialization(self): # Test ProcessingInstruction directly self.assertEqual(ET.tostring(ET.ProcessingInstruction('test', 'instruction')), @@ -1429,13 +1261,22 @@ def test_processinginstruction(self): b'<?test instruction?>') # Issue #2746 - + # processing instructions are not escaped self.assertEqual(ET.tostring(ET.PI('test', '<testing&>')), b'<?test <testing&>?>') self.assertEqual(ET.tostring(ET.PI('test', '<testing&>\xe3'), 'latin-1'), b"<?xml version='1.0' encoding='latin-1'?>\n" b"<?test <testing&>\xe3?>") + @support.subTests('tag', ("script", "style", "xmp", "iframe", "noembed", "noframes")) + def test_html_cdata_elems_serialization(self, tag): + # content of raw text elements is not escaped in html + tag = tag.title() + elem = ET.Element(tag) + elem.text = '<spam>&ham' + self.assertEqual(ET.tostring(elem, method='html'), + ('<%s><spam>&ham</%s>' % (tag, tag)).encode()) + def test_html_empty_elems_serialization(self): # issue 15970 # from http://www.w3.org/TR/html401/index/elements.html @@ -1450,6 +1291,14 @@ def test_html_empty_elems_serialization(self): method='html') self.assertEqual(serialized, expected) + def test_html_plaintext_serialization(self): + # content of plaintext is not escaped in html + # no end tag for plaintext + elem = ET.Element('PlainText') + elem.text = '<spam>&ham' + self.assertEqual(ET.tostring(elem, method='html'), + b'<PlainText><spam>&ham') + def test_dump_attribute_order(self): # See BPO 34160 e = ET.Element('cirriculum', status='public', company='example') @@ -1473,6 +1322,234 @@ def test_attlist_default(self): {'{http://www.w3.org/XML/1998/namespace}lang': 'eng'}) +class IterparseTest(unittest.TestCase): + # Test iterparse interface. + + def test_basic(self): + iterparse = ET.iterparse + + it = iterparse(SIMPLE_XMLFILE) + self.assertIsNone(it.root) + action, elem = next(it) + self.assertIsNone(it.root) + self.assertEqual((action, elem.tag), ('end', 'element')) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('end', 'element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + self.assertEqual(it.root.tag, 'root') + it.close() + + it = iterparse(SIMPLE_NS_XMLFILE) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('end', '{namespace}element'), + ('end', '{namespace}element'), + ('end', '{namespace}empty-element'), + ('end', '{namespace}root'), + ]) + it.close() + + def test_external_file(self): + with open(SIMPLE_XMLFILE, 'rb') as source: + it = ET.iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('end', 'element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + self.assertEqual(it.root.tag, 'root') + + def test_events(self): + iterparse = ET.iterparse + + events = () + it = iterparse(SIMPLE_XMLFILE, events) + self.assertEqual([(action, elem.tag) for action, elem in it], []) + it.close() + + events = () + it = iterparse(SIMPLE_XMLFILE, events=events) + self.assertEqual([(action, elem.tag) for action, elem in it], []) + it.close() + + events = ("start", "end") + it = iterparse(SIMPLE_XMLFILE, events) + self.assertEqual([(action, elem.tag) for action, elem in it], [ + ('start', 'root'), + ('start', 'element'), + ('end', 'element'), + ('start', 'element'), + ('end', 'element'), + ('start', 'empty-element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + it.close() + + def test_namespace_events(self): + iterparse = ET.iterparse + + events = ("start", "end", "start-ns", "end-ns") + it = iterparse(SIMPLE_NS_XMLFILE, events) + self.assertEqual([(action, elem.tag) if action in ("start", "end") + else (action, elem) + for action, elem in it], [ + ('start-ns', ('', 'namespace')), + ('start', '{namespace}root'), + ('start', '{namespace}element'), + ('end', '{namespace}element'), + ('start', '{namespace}element'), + ('end', '{namespace}element'), + ('start', '{namespace}empty-element'), + ('end', '{namespace}empty-element'), + ('end', '{namespace}root'), + ('end-ns', None), + ]) + it.close() + + events = ('start-ns', 'end-ns') + it = iterparse(io.BytesIO(br"<root xmlns=''/>"), events) + res = [action for action, elem in it] + self.assertEqual(res, ['start-ns', 'end-ns']) + it.close() + + def test_unknown_events(self): + iterparse = ET.iterparse + + events = ("start", "end", "bogus") + with open(SIMPLE_XMLFILE, "rb") as f: + with self.assertRaises(ValueError) as cm: + iterparse(f, events) + self.assertFalse(f.closed) + self.assertEqual(str(cm.exception), "unknown event 'bogus'") + + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(ValueError) as cm: + iterparse(SIMPLE_XMLFILE, events) + self.assertEqual(str(cm.exception), "unknown event 'bogus'") + del cm + gc_collect() + + def test_non_utf8(self): + source = io.BytesIO( + b"<?xml version='1.0' encoding='iso-8859-1'?>\n" + b"<body xmlns='http://&#233;ffbot.org/ns'\n" + b" xmlns:cl\xe9='http://effbot.org/ns'>text</body>\n") + events = ("start-ns",) + it = ET.iterparse(source, events) + self.assertEqual([(action, elem) for action, elem in it], [ + ('start-ns', ('', 'http://\xe9ffbot.org/ns')), + ('start-ns', ('cl\xe9', 'http://effbot.org/ns')), + ]) + + def test_parsing_error(self): + source = io.BytesIO(b"<document />junk") + it = ET.iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'document')) + with self.assertRaises(ET.ParseError) as cm: + next(it) + self.assertEqual(str(cm.exception), + 'junk after document element: line 1, column 12') + + def test_nonexistent_file(self): + with self.assertRaises(FileNotFoundError): + ET.iterparse("nonexistent") + + def test_resource_warnings_not_exhausted(self): + # Not exhausting the iterator still closes the underlying file (bpo-43292) + it = ET.iterparse(SIMPLE_XMLFILE) + with warnings_helper.check_no_resource_warning(self): + del it + gc_collect() + + it = ET.iterparse(SIMPLE_XMLFILE) + with warnings_helper.check_no_resource_warning(self): + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + del it, elem + gc_collect() + + def test_resource_warnings_failed_iteration(self): + self.addCleanup(os_helper.unlink, TESTFN) + with open(TESTFN, "wb") as f: + f.write(b"<document />junk") + + it = ET.iterparse(TESTFN) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'document')) + with warnings_helper.check_no_resource_warning(self): + with self.assertRaises(ET.ParseError) as cm: + next(it) + self.assertEqual(str(cm.exception), + 'junk after document element: line 1, column 12') + del cm, it + gc_collect() + + def test_resource_warnings_exhausted(self): + it = ET.iterparse(SIMPLE_XMLFILE) + with warnings_helper.check_no_resource_warning(self): + list(it) + del it + gc_collect() + + def test_close_not_exhausted(self): + iterparse = ET.iterparse + + it = iterparse(SIMPLE_XMLFILE) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + def test_close_exhausted(self): + iterparse = ET.iterparse + it = iterparse(SIMPLE_XMLFILE) + list(it) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + list(it) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + class XMLPullParserTest(unittest.TestCase): def _feed(self, parser, data, chunk_size=None, flush=False): @@ -1723,6 +1800,8 @@ def __next__(self): def test_unknown_event(self): with self.assertRaises(ValueError): ET.XMLPullParser(events=('start', 'end', 'bogus')) + with self.assertRaisesRegex(ValueError, "unknown event 'bogus'"): + ET.XMLPullParser(events=(x.decode() for x in (b'start', b'end', b'bogus'))) @unittest.skipIf(pyexpat.version_info < (2, 6, 0), f'Expat {pyexpat.version_info} does not ' @@ -2909,32 +2988,72 @@ def __del__(self): elem = b.close() self.assertEqual(elem[0].tail, 'ABCDEFGHIJKL') - def test_subscr(self): - # Issue #27863 + def test_subscr_with_clear(self): + # See https://github.com/python/cpython/issues/143200. + self.do_test_subscr_with_mutating_slice(use_clear_method=True) + + def test_subscr_with_delete(self): + # See https://github.com/python/cpython/issues/72050. + self.do_test_subscr_with_mutating_slice(use_clear_method=False) + + def do_test_subscr_with_mutating_slice(self, *, use_clear_method): class X: + def __init__(self, i=0): + self.i = i def __index__(self): - del e[:] - return 1 + if use_clear_method: + e.clear() + else: + del e[:] + return self.i - e = ET.Element('elem') - e.append(ET.Element('child')) - e[:X()] # shouldn't crash + for s in self.get_mutating_slices(X, 10): + with self.subTest(s): + e = ET.Element('elem') + e.extend([ET.Element(f'c{i}') for i in range(10)]) + e[s] # shouldn't crash - e.append(ET.Element('child')) - e[0:10:X()] # shouldn't crash + def test_ass_subscr_with_mutating_slice(self): + # See https://github.com/python/cpython/issues/72050 + # and https://github.com/python/cpython/issues/143200. - def test_ass_subscr(self): - # Issue #27863 class X: + def __init__(self, i=0): + self.i = i def __index__(self): e[:] = [] - return 1 + return self.i + + for s in self.get_mutating_slices(X, 10): + with self.subTest(s): + e = ET.Element('elem') + e.extend([ET.Element(f'c{i}') for i in range(10)]) + e[s] = [] # shouldn't crash + + def get_mutating_slices(self, index_class, n_children): + self.assertGreaterEqual(n_children, 10) + return [ + slice(index_class(), None, None), + slice(index_class(2), None, None), + slice(None, index_class(), None), + slice(None, index_class(2), None), + slice(0, 2, index_class(1)), + slice(0, 2, index_class(2)), + slice(0, n_children, index_class(1)), + slice(0, n_children, index_class(2)), + slice(0, 2 * n_children, index_class(1)), + slice(0, 2 * n_children, index_class(2)), + ] - e = ET.Element('elem') - for _ in range(10): - e.insert(0, ET.Element('child')) + def test_ass_subscr_with_mutating_iterable_value(self): + class V: + def __iter__(self): + e.clear() + return iter([ET.Element('a'), ET.Element('b')]) - e[0:10:X()] = [] # shouldn't crash + e = ET.Element('elem') + e.extend([ET.Element(f'c{i}') for i in range(10)]) + e[:] = V() def test_treebuilder_start(self): # Issue #27863 @@ -2960,6 +3079,63 @@ def element_factory(x, y): del b gc_collect() + def test_deepcopy_clear(self): + # Prevent crashes when __deepcopy__() clears the children list. + # See https://github.com/python/cpython/issues/133009. + class X(ET.Element): + def __deepcopy__(self, memo): + root.clear() + return self + + root = ET.Element('a') + evil = X('x') + root.extend([evil, ET.Element('y')]) + if is_python_implementation(): + # Mutating a list over which we iterate raises an error. + self.assertRaises(RuntimeError, copy.deepcopy, root) + else: + c = copy.deepcopy(root) + # In the C implementation, we can still copy the evil element. + self.assertListEqual(list(c), [evil]) + + def test_deepcopy_grow(self): + # Prevent crashes when __deepcopy__() mutates the children list. + # See https://github.com/python/cpython/issues/133009. + a = ET.Element('a') + b = ET.Element('b') + c = ET.Element('c') + + class X(ET.Element): + def __deepcopy__(self, memo): + root.append(a) + root.append(b) + return self + + root = ET.Element('top') + evil1, evil2 = X('1'), X('2') + root.extend([evil1, c, evil2]) + children = list(copy.deepcopy(root)) + # mock deep copies + self.assertIs(children[0], evil1) + self.assertIs(children[2], evil2) + # true deep copies + self.assertEqual(children[1].tag, c.tag) + self.assertEqual([c.tag for c in children[3:]], + [a.tag, b.tag, a.tag, b.tag]) + + @support.skip_if_huge_c_stack(500_000) + @support.skip_emscripten_stack_overflow() + @support.skip_wasi_stack_overflow() + def test_deeply_nested_deepcopy(self): + # This should raise a RecursionError and not crash. + # See https://github.com/python/cpython/issues/148801. + root = cur = ET.Element('s') + for _ in range(500_000): + cur = ET.SubElement(cur, 'u') + with support.infinite_recursion(): + with self.assertRaises(RecursionError): + copy.deepcopy(root) + class MutationDeleteElementPath(str): def __new__(cls, elem, *args): @@ -3028,6 +3204,16 @@ def test_findtext_with_mutating(self): e.extend([ET.Element('bar')]) e.findtext(cls(e, 'x')) + def test_findtext_with_mutating_non_none_text(self): + for cls in [MutationDeleteElementPath, MutationClearElementPath]: + with self.subTest(cls): + e = ET.Element('foo') + child = ET.Element('bar') + child.text = str(object()) + e.append(child) + del child + repr(e.findtext(cls(e, 'x'))) + def test_findtext_with_error(self): e = ET.Element('foo') e.extend([ET.Element('bar')]) @@ -3401,6 +3587,32 @@ def test_basic(self): doc = ET.XML("<root>a&amp;<sub>b&amp;</sub>c&amp;</root>") self.assertEqual(''.join(doc.itertext()), 'a&b&c&') + def test_comment(self): + e = ET.Element('root') + e.text = 'before' + comment = ET.Comment('comment') + self.assertEqual(comment.text, 'comment') + comment.tail = 'after' + e.append(comment) + self.assertEqual(''.join(e.itertext()), 'beforeafter') + self.assertEqual(list(e.iter()), [e, comment]) + self.assertEqual(list(e.iter('root')), [e]) + self.assertEqual(''.join(comment.itertext()), '') + self.assertEqual(list(comment.iter()), [comment]) + + def test_processinginstruction(self): + e = ET.Element('root') + e.text = 'before' + pi = ET.PI('test', 'instruction') + self.assertEqual(pi.text, 'test instruction') + pi.tail = 'after' + e.append(pi) + self.assertEqual(''.join(e.itertext()), 'beforeafter') + self.assertEqual(list(e.iter()), [e, pi]) + self.assertEqual(list(e.iter('root')), [e]) + self.assertEqual(''.join(pi.itertext()), '') + self.assertEqual(list(pi.iter()), [pi]) + def test_corners(self): # single root, no subelements a = ET.Element('a') diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py index 9ed0f4096a45e3c..270b9d6da8e7b9e 100644 --- a/Lib/test/test_xml_etree_c.py +++ b/Lib/test/test_xml_etree_c.py @@ -58,7 +58,7 @@ def test_del_attribute(self): self.assertEqual(element.attrib, {'A': 'B', 'C': 'D'}) @support.skip_wasi_stack_overflow() - @unittest.skipIf(support.is_emscripten, "segfaults") + @support.skip_emscripten_stack_overflow() def test_trashcan(self): # If this test fails, it will most likely die via segfault. e = root = cET.Element('root') diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index 2803c6d45c27bfa..ee0e24f6e86ae33 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -208,6 +208,17 @@ def test_dump_encoding(self): self.assertEqual(xmlrpclib.loads(strg)[0][0], value) self.assertEqual(xmlrpclib.loads(strg)[1], methodname) + def test_dump_escape_methodname(self): + payload = 'foo</methodName><injected attr="evil"/><methodName>bar' + s = xmlrpclib.dumps((), methodname=payload) + self.assertIn( + '<methodName>foo&lt;/methodName&gt;&lt;injected attr="evil"/&gt;' + '&lt;methodName&gt;bar</methodName>', s + ) + self.assertNotIn('<injected attr="evil"/>', s) + load, m = xmlrpclib.loads(s) + self.assertEqual(m, payload) + def test_dump_bytes(self): sample = b"my dog has fleas" self.assertEqual(sample, xmlrpclib.Binary(sample)) diff --git a/Lib/test/test_xpickle.py b/Lib/test/test_xpickle.py new file mode 100644 index 000000000000000..d87c671d4f5394f --- /dev/null +++ b/Lib/test/test_xpickle.py @@ -0,0 +1,281 @@ +# This test covers backwards compatibility with previous versions of Python +# by bouncing pickled objects through Python versions by running xpickle_worker.py. +import io +import os +import pickle +import struct +import subprocess +import sys +import unittest + + +from test import support +from test import pickletester + +try: + import _pickle + has_c_implementation = True +except ModuleNotFoundError: + has_c_implementation = False + +support.requires('xpickle') + +is_windows = sys.platform.startswith('win') + +# Map python version to a tuple containing the name of a corresponding valid +# Python binary to execute and its arguments. +py_executable_map = {} + +protocols_map = { + 3: (3, 0), + 4: (3, 4), + 5: (3, 8), +} + +def highest_proto_for_py_version(py_version): + """Finds the highest supported pickle protocol for a given Python version. + Args: + py_version: a 2-tuple of the major, minor version. Eg. Python 3.7 would + be (3, 7) + Returns: + int for the highest supported pickle protocol + """ + proto = 2 + for p, v in protocols_map.items(): + if py_version < v: + break + proto = p + return proto + +def have_python_version(py_version): + """Check whether a Python binary exists for the given py_version and has + support. This respects your PATH. + For Windows, it will first try to use the py launcher specified in PEP 397. + Otherwise (and for all other platforms), it will attempt to check for + python<py_version[0]>.<py_version[1]>. + + Eg. given a *py_version* of (3, 7), the function will attempt to try + 'py -3.7' (for Windows) first, then 'python3.7', and return + ['py', '-3.7'] (on Windows) or ['python3.7'] on other platforms. + + Args: + py_version: a 2-tuple of the major, minor version. Eg. python 3.7 would + be (3, 7) + Returns: + List/Tuple containing the Python binary name and its required arguments, + or None if no valid binary names found. + """ + python_str = ".".join(map(str, py_version)) + targets = [('py', f'-{python_str}'), (f'python{python_str}',)] + if py_version not in py_executable_map: + for target in targets[0 if is_windows else 1:]: + try: + worker = subprocess.Popen([*target, '-c', 'pass'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + shell=is_windows) + worker.communicate() + if worker.returncode == 0: + py_executable_map[py_version] = target + break + except FileNotFoundError: + pass + + return py_executable_map.get(py_version, None) + + +def read_exact(f, n): + buf = b'' + while len(buf) < n: + chunk = f.read(n - len(buf)) + if not chunk: + raise EOFError + buf += chunk + return buf + + +class AbstractCompatTests(pickletester.AbstractPickleTests): + py_version = None + worker = None + + @classmethod + def setUpClass(cls): + assert cls.py_version is not None, 'Needs a python version tuple' + if not have_python_version(cls.py_version): + py_version_str = ".".join(map(str, cls.py_version)) + raise unittest.SkipTest(f'Python {py_version_str} not available') + cls.addClassCleanup(cls.finish_worker) + # Override the default pickle protocol to match what xpickle worker + # will be running. + highest_protocol = highest_proto_for_py_version(cls.py_version) + cls.enterClassContext(support.swap_attr(pickletester, 'protocols', + range(highest_protocol + 1))) + cls.enterClassContext(support.swap_attr(pickle, 'HIGHEST_PROTOCOL', + highest_protocol)) + + @classmethod + def start_worker(cls, python): + target = os.path.join(os.path.dirname(__file__), 'xpickle_worker.py') + worker = subprocess.Popen([*python, target], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + # For windows bpo-17023. + shell=is_windows) + cls.worker = worker + return worker + + @classmethod + def finish_worker(cls): + worker = cls.worker + if worker is None: + return + cls.worker = None + worker.stdin.close() + worker.stdout.close() + worker.stderr.close() + worker.terminate() + worker.wait() + + @classmethod + def send_to_worker(cls, python, data): + """Bounce a pickled object through another version of Python. + This will send data to a child process where it will + be unpickled, then repickled and sent back to the parent process. + Args: + python: list containing the python binary to start and its arguments + data: bytes object to send to the child process + Returns: + The pickled data received from the child process. + """ + worker = cls.worker + if worker is None: + worker = cls.start_worker(python) + + try: + worker.stdin.write(struct.pack('!i', len(data)) + data) + worker.stdin.flush() + + size, = struct.unpack('!i', read_exact(worker.stdout, 4)) + if size > 0: + return read_exact(worker.stdout, size) + # if the worker fails, it will write the exception to stdout + if size < 0: + stdout = read_exact(worker.stdout, -size) + try: + exception = pickle.loads(stdout) + except (pickle.UnpicklingError, EOFError): + pass + else: + if isinstance(exception, Exception): + # To allow for tests which test for errors. + raise exception + _, stderr = worker.communicate() + raise RuntimeError(stderr) + except: + cls.finish_worker() + raise + + def dumps(self, arg, proto=0, **kwargs): + # Skip tests that require buffer_callback arguments since + # there isn't a reliable way to marshal/pickle the callback and ensure + # it works in a different Python version. + if 'buffer_callback' in kwargs: + self.skipTest('Test does not support "buffer_callback" argument.') + f = io.BytesIO() + p = self.pickler(f, proto, **kwargs) + p.dump(arg) + data = struct.pack('!i', proto) + f.getvalue() + python = py_executable_map[self.py_version] + return self.send_to_worker(python, data) + + def loads(self, buf, **kwds): + f = io.BytesIO(buf) + u = self.unpickler(f, **kwds) + return u.load() + + # A scaled-down version of test_bytes from pickletester, to reduce + # the number of calls to self.dumps() and hence reduce the number of + # child python processes forked. This allows the test to complete + # much faster (the one from pickletester takes 3-4 minutes when running + # under text_xpickle). + def test_bytes(self): + if self.py_version < (3, 0): + self.skipTest('not supported in Python < 3.0') + for proto in pickletester.protocols: + for s in b'', b'xyz', b'xyz'*100: + p = self.dumps(s, proto) + self.assert_is_copy(s, self.loads(p)) + s = bytes(range(256)) + p = self.dumps(s, proto) + self.assert_is_copy(s, self.loads(p)) + s = bytes([i for i in range(256) for _ in range(2)]) + p = self.dumps(s, proto) + self.assert_is_copy(s, self.loads(p)) + + # These tests are disabled because they require some special setup + # on the worker that's hard to keep in sync. + test_global_ext1 = None + test_global_ext2 = None + test_global_ext4 = None + + # These tests fail because they require classes from pickletester + # which cannot be properly imported by the xpickle worker. + test_recursive_nested_names = None + test_recursive_nested_names2 = None + + # Attribute lookup problems are expected, disable the test + test_dynamic_class = None + test_evil_class_mutating_dict = None + + # Expected exception is raised during unpickling in a subprocess. + test_pickle_setstate_None = None + + # Other Python version may not have NumPy. + test_buffers_numpy = None + + # Skip tests that require buffer_callback arguments since + # there isn't a reliable way to marshal/pickle the callback and ensure + # it works in a different Python version. + test_in_band_buffers = None + test_buffers_error = None + test_oob_buffers = None + test_oob_buffers_writable_to_readonly = None + +class PyPicklePythonCompat(AbstractCompatTests): + pickler = pickle._Pickler + unpickler = pickle._Unpickler + +if has_c_implementation: + class CPicklePythonCompat(AbstractCompatTests): + pickler = _pickle.Pickler + unpickler = _pickle.Unpickler + + +def make_test(py_version, base): + class_dict = {'py_version': py_version} + name = base.__name__.replace('Python', 'Python%d%d' % py_version) + return type(name, (base, unittest.TestCase), class_dict) + +def load_tests(loader, tests, pattern): + def add_tests(py_version): + test_class = make_test(py_version, PyPicklePythonCompat) + tests.addTest(loader.loadTestsFromTestCase(test_class)) + if has_c_implementation: + test_class = make_test(py_version, CPicklePythonCompat) + tests.addTest(loader.loadTestsFromTestCase(test_class)) + + value = support.get_resource_value('xpickle') + if value is None: + major = sys.version_info.major + assert major == 3 + add_tests((2, 7)) + for minor in range(2, sys.version_info.minor): + add_tests((major, minor)) + else: + add_tests(tuple(map(int, value.split('.')))) + return tests + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_xxlimited.py b/Lib/test/test_xxlimited.py index 6dbfb3f439393c6..b52e78bc4fb7e05 100644 --- a/Lib/test/test_xxlimited.py +++ b/Lib/test/test_xxlimited.py @@ -31,7 +31,7 @@ def test_foo(self): self.assertEqual(self.module.foo(1, 2), 3) def test_str(self): - self.assertTrue(issubclass(self.module.Str, str)) + self.assertIsSubclass(self.module.Str, str) self.assertIsNot(self.module.Str, str) custom_string = self.module.Str("abcd") diff --git a/Lib/test/test_zipapp.py b/Lib/test/test_zipapp.py index d4766c59a102db4..8fb0a68deba535c 100644 --- a/Lib/test/test_zipapp.py +++ b/Lib/test/test_zipapp.py @@ -259,7 +259,7 @@ def test_pack_to_fileobj(self): (source / '__main__.py').touch() target = io.BytesIO() zipapp.create_archive(str(source), target, interpreter='python') - self.assertTrue(target.getvalue().startswith(b'#!python\n')) + self.assertStartsWith(target.getvalue(), b'#!python\n') def test_read_shebang(self): # Test that we can read the shebang line correctly. @@ -300,7 +300,7 @@ def test_write_shebang_to_fileobj(self): zipapp.create_archive(str(source), str(target), interpreter='python') new_target = io.BytesIO() zipapp.create_archive(str(target), new_target, interpreter='python2.7') - self.assertTrue(new_target.getvalue().startswith(b'#!python2.7\n')) + self.assertStartsWith(new_target.getvalue(), b'#!python2.7\n') def test_read_from_pathlike_obj(self): # Test that we can copy an archive using a path-like object @@ -326,7 +326,7 @@ def test_read_from_fileobj(self): new_target = io.BytesIO() temp_archive.seek(0) zipapp.create_archive(temp_archive, new_target, interpreter='python2.7') - self.assertTrue(new_target.getvalue().startswith(b'#!python2.7\n')) + self.assertStartsWith(new_target.getvalue(), b'#!python2.7\n') def test_remove_shebang(self): # Test that we can remove the shebang from a file. diff --git a/Lib/test/test_zipfile/_path/_test_params.py b/Lib/test/test_zipfile/_path/_test_params.py index bc95b4ebf4a168c..00a9eaf2f99c1ae 100644 --- a/Lib/test/test_zipfile/_path/_test_params.py +++ b/Lib/test/test_zipfile/_path/_test_params.py @@ -1,5 +1,5 @@ -import types import functools +import types from ._itertools import always_iterable diff --git a/Lib/test/test_zipfile/_path/test_complexity.py b/Lib/test/test_zipfile/_path/test_complexity.py index b505dd7c3764623..7c108fc6ab8191d 100644 --- a/Lib/test/test_zipfile/_path/test_complexity.py +++ b/Lib/test/test_zipfile/_path/test_complexity.py @@ -8,10 +8,8 @@ from ._functools import compose from ._itertools import consume - from ._support import import_or_skip - big_o = import_or_skip('big_o') pytest = import_or_skip('pytest') diff --git a/Lib/test/test_zipfile/_path/test_path.py b/Lib/test/test_zipfile/_path/test_path.py index 0afabc0c6683c4c..351d9eefeb09a51 100644 --- a/Lib/test/test_zipfile/_path/test_path.py +++ b/Lib/test/test_zipfile/_path/test_path.py @@ -1,6 +1,6 @@ +import contextlib import io import itertools -import contextlib import pathlib import pickle import stat @@ -9,12 +9,11 @@ import zipfile import zipfile._path -from test.support.os_helper import temp_dir, FakePath +from test.support.os_helper import FakePath, temp_dir from ._functools import compose from ._itertools import Counter - -from ._test_params import parameterize, Invoked +from ._test_params import Invoked, parameterize class jaraco: @@ -193,10 +192,10 @@ def test_encoding_warnings(self, alpharep): """EncodingWarning must blame the read_text and open calls.""" assert sys.flags.warn_default_encoding root = zipfile.Path(alpharep) - with self.assertWarns(EncodingWarning) as wc: + with self.assertWarns(EncodingWarning) as wc: # noqa: F821 (astral-sh/ruff#13296) root.joinpath("a.txt").read_text() assert __file__ == wc.filename - with self.assertWarns(EncodingWarning) as wc: + with self.assertWarns(EncodingWarning) as wc: # noqa: F821 (astral-sh/ruff#13296) root.joinpath("a.txt").open("r").close() assert __file__ == wc.filename @@ -275,7 +274,8 @@ def test_pathlike_construction(self, alpharep): """ zipfile_ondisk = self.zipfile_ondisk(alpharep) pathlike = FakePath(str(zipfile_ondisk)) - zipfile.Path(pathlike) + root = zipfile.Path(pathlike) + root.root.close() @pass_alpharep def test_traverse_pathlike(self, alpharep): @@ -364,6 +364,18 @@ def test_root_name(self, alpharep): root = zipfile.Path(alpharep) assert root.name == 'alpharep.zip' == root.filename.name + @pass_alpharep + def test_root_on_disk(self, alpharep): + """ + The name/stem of the root should match the zipfile on disk. + + This condition must hold across platforms. + """ + root = zipfile.Path(self.zipfile_ondisk(alpharep)) + assert root.name == 'alpharep.zip' == root.filename.name + assert root.stem == 'alpharep' == root.filename.stem + root.root.close() + @pass_alpharep def test_suffix(self, alpharep): """ @@ -564,11 +576,13 @@ def test_inheritance(self, alpharep): ) def test_pickle(self, alpharep, path_type, subpath): zipfile_ondisk = path_type(str(self.zipfile_ondisk(alpharep))) - - saved_1 = pickle.dumps(zipfile.Path(zipfile_ondisk, at=subpath)) + root = zipfile.Path(zipfile_ondisk, at=subpath) + saved_1 = pickle.dumps(root) + root.root.close() restored_1 = pickle.loads(saved_1) first, *rest = restored_1.iterdir() assert first.read_text(encoding='utf-8').startswith('content of ') + restored_1.root.close() @pass_alpharep def test_extract_orig_with_implied_dirs(self, alpharep): @@ -580,6 +594,7 @@ def test_extract_orig_with_implied_dirs(self, alpharep): # wrap the zipfile for its side effect zipfile.Path(zf) zf.extractall(source_path.parent) + zf.close() @pass_alpharep def test_getinfo_missing(self, alpharep): diff --git a/Lib/test/test_zipfile/_path/write-alpharep.py b/Lib/test/test_zipfile/_path/write-alpharep.py index 48c09b537179fd2..7418391abadde5e 100644 --- a/Lib/test/test_zipfile/_path/write-alpharep.py +++ b/Lib/test/test_zipfile/_path/write-alpharep.py @@ -1,4 +1,3 @@ from . import test_path - __name__ == '__main__' and test_path.build_alpharep_fixture().extractall('alpharep') diff --git a/Lib/test/test_zipfile/test_core.py b/Lib/test/test_zipfile/test_core.py index ae898150658565c..85bc010fe007841 100644 --- a/Lib/test/test_zipfile/test_core.py +++ b/Lib/test/test_zipfile/test_core.py @@ -1,6 +1,7 @@ import _pyio import array import contextlib +import errno import importlib.util import io import itertools @@ -312,26 +313,26 @@ def test_low_compression(self): self.assertEqual(openobj.read(1), b'2') def test_writestr_compression(self): - zipfp = zipfile.ZipFile(TESTFN2, "w") - zipfp.writestr("b.txt", "hello world", compress_type=self.compression) - info = zipfp.getinfo('b.txt') - self.assertEqual(info.compress_type, self.compression) + with zipfile.ZipFile(TESTFN2, "w") as zipfp: + zipfp.writestr("b.txt", "hello world", compress_type=self.compression) + info = zipfp.getinfo('b.txt') + self.assertEqual(info.compress_type, self.compression) def test_writestr_compresslevel(self): - zipfp = zipfile.ZipFile(TESTFN2, "w", compresslevel=1) - zipfp.writestr("a.txt", "hello world", compress_type=self.compression) - zipfp.writestr("b.txt", "hello world", compress_type=self.compression, - compresslevel=2) + with zipfile.ZipFile(TESTFN2, "w", compresslevel=1) as zipfp: + zipfp.writestr("a.txt", "hello world", compress_type=self.compression) + zipfp.writestr("b.txt", "hello world", compress_type=self.compression, + compresslevel=2) - # Compression level follows the constructor. - a_info = zipfp.getinfo('a.txt') - self.assertEqual(a_info.compress_type, self.compression) - self.assertEqual(a_info.compress_level, 1) + # Compression level follows the constructor. + a_info = zipfp.getinfo('a.txt') + self.assertEqual(a_info.compress_type, self.compression) + self.assertEqual(a_info.compress_level, 1) - # Compression level is overridden. - b_info = zipfp.getinfo('b.txt') - self.assertEqual(b_info.compress_type, self.compression) - self.assertEqual(b_info._compresslevel, 2) + # Compression level is overridden. + b_info = zipfp.getinfo('b.txt') + self.assertEqual(b_info.compress_type, self.compression) + self.assertEqual(b_info._compresslevel, 2) def test_read_return_size(self): # Issue #9837: ZipExtFile.read() shouldn't return more bytes @@ -654,6 +655,12 @@ def test_add_file_after_2107(self): os.utime(TESTFN, (ts, ts)) except OverflowError: self.skipTest('Host fs cannot set timestamp to required value.') + except OSError as exc: + # Some file systems (e.g. UFS and ZFS on illumos) do not + # support timestamps that do not fit in 32 bits. + if exc.errno != errno.EOVERFLOW: + raise + self.skipTest('Host fs cannot set timestamp to required value.') mtime_ns = os.stat(TESTFN).st_mtime_ns if mtime_ns != (4386268800 * 10**9): @@ -898,6 +905,8 @@ def make_zip64_file( self, file_size_64_set=False, file_size_extra=False, compress_size_64_set=False, compress_size_extra=False, header_offset_64_set=False, header_offset_extra=False, + extensible_data=b'', + end_of_central_dir_size=None, offset_to_end_of_central_dir=None, ): """Generate bytes sequence for a zip with (incomplete) zip64 data. @@ -951,6 +960,12 @@ def make_zip64_file( central_dir_size = struct.pack('<Q', 58 + 8 * len(central_zip64_fields)) offset_to_central_dir = struct.pack('<Q', 50 + 8 * len(local_zip64_fields)) + if end_of_central_dir_size is None: + end_of_central_dir_size = 44 + len(extensible_data) + if offset_to_end_of_central_dir is None: + offset_to_end_of_central_dir = (108 + + 8 * len(local_zip64_fields) + + 8 * len(central_zip64_fields)) local_extra_length = struct.pack("<H", 4 + 8 * len(local_zip64_fields)) central_extra_length = struct.pack("<H", 4 + 8 * len(central_zip64_fields)) @@ -979,14 +994,17 @@ def make_zip64_file( + filename + central_extra # Zip64 end of central directory - + b"PK\x06\x06,\x00\x00\x00\x00\x00\x00\x00-\x00-" - + b"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00" + + b"PK\x06\x06" + + struct.pack('<Q', end_of_central_dir_size) + + b"-\x00-\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00" + b"\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00" + central_dir_size + offset_to_central_dir + + extensible_data # Zip64 end of central directory locator - + b"PK\x06\x07\x00\x00\x00\x00l\x00\x00\x00\x00\x00\x00\x00\x01" - + b"\x00\x00\x00" + + b"PK\x06\x07\x00\x00\x00\x00" + + struct.pack('<Q', offset_to_end_of_central_dir) + + b"\x01\x00\x00\x00" # end of central directory + b"PK\x05\x06\x00\x00\x00\x00\x01\x00\x01\x00:\x00\x00\x002\x00" + b"\x00\x00\x00\x00" @@ -1017,6 +1035,7 @@ def test_bad_zip64_extra(self): with self.assertRaises(zipfile.BadZipFile) as e: zipfile.ZipFile(io.BytesIO(missing_file_size_extra)) self.assertIn('file size', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_file_size_extra))) # zip64 file size present, zip64 compress size present, one field in # extra, expecting two, equals missing compress size. @@ -1028,6 +1047,7 @@ def test_bad_zip64_extra(self): with self.assertRaises(zipfile.BadZipFile) as e: zipfile.ZipFile(io.BytesIO(missing_compress_size_extra)) self.assertIn('compress size', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_compress_size_extra))) # zip64 compress size present, no fields in extra, expecting one, # equals missing compress size. @@ -1037,6 +1057,7 @@ def test_bad_zip64_extra(self): with self.assertRaises(zipfile.BadZipFile) as e: zipfile.ZipFile(io.BytesIO(missing_compress_size_extra)) self.assertIn('compress size', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_compress_size_extra))) # zip64 file size present, zip64 compress size present, zip64 header # offset present, two fields in extra, expecting three, equals missing @@ -1051,6 +1072,7 @@ def test_bad_zip64_extra(self): with self.assertRaises(zipfile.BadZipFile) as e: zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) self.assertIn('header offset', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_header_offset_extra))) # zip64 compress size present, zip64 header offset present, one field # in extra, expecting two, equals missing header offset @@ -1063,6 +1085,7 @@ def test_bad_zip64_extra(self): with self.assertRaises(zipfile.BadZipFile) as e: zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) self.assertIn('header offset', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_header_offset_extra))) # zip64 file size present, zip64 header offset present, one field in # extra, expecting two, equals missing header offset @@ -1075,6 +1098,7 @@ def test_bad_zip64_extra(self): with self.assertRaises(zipfile.BadZipFile) as e: zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) self.assertIn('header offset', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_header_offset_extra))) # zip64 header offset present, no fields in extra, expecting one, # equals missing header offset @@ -1086,6 +1110,63 @@ def test_bad_zip64_extra(self): with self.assertRaises(zipfile.BadZipFile) as e: zipfile.ZipFile(io.BytesIO(missing_header_offset_extra)) self.assertIn('header offset', str(e.exception).lower()) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(missing_header_offset_extra))) + + def test_bad_zip64_end_of_central_dir(self): + zipdata = self.make_zip64_file(end_of_central_dir_size=0) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Corrupt.*record'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + zipdata = self.make_zip64_file(end_of_central_dir_size=100) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Corrupt.*record'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + zipdata = self.make_zip64_file(offset_to_end_of_central_dir=0) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Corrupt.*record'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + zipdata = self.make_zip64_file(offset_to_end_of_central_dir=1000) + with self.assertRaisesRegex(zipfile.BadZipFile, 'Corrupt.*locator'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + def test_zip64_end_of_central_dir_record_not_found(self): + zipdata = self.make_zip64_file() + zipdata = zipdata.replace(b"PK\x06\x06", b'\x00'*4) + with self.assertRaisesRegex(zipfile.BadZipFile, 'record not found'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + zipdata = self.make_zip64_file( + extensible_data=b'\xca\xfe\x04\x00\x00\x00data') + zipdata = zipdata.replace(b"PK\x06\x06", b'\x00'*4) + with self.assertRaisesRegex(zipfile.BadZipFile, 'record not found'): + zipfile.ZipFile(io.BytesIO(zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(zipdata))) + + def test_zip64_extensible_data(self): + # These values are what is set in the make_zip64_file method. + expected_file_size = 8 + expected_compress_size = 8 + expected_header_offset = 0 + expected_content = b"test1234" + + zipdata = self.make_zip64_file( + extensible_data=b'\xca\xfe\x04\x00\x00\x00data') + with zipfile.ZipFile(io.BytesIO(zipdata)) as zf: + zinfo = zf.infolist()[0] + self.assertEqual(zinfo.file_size, expected_file_size) + self.assertEqual(zinfo.compress_size, expected_compress_size) + self.assertEqual(zinfo.header_offset, expected_header_offset) + self.assertEqual(zf.read(zinfo), expected_content) + self.assertTrue(zipfile.is_zipfile(io.BytesIO(zipdata))) + + with self.assertRaisesRegex(zipfile.BadZipFile, 'record not found'): + zipfile.ZipFile(io.BytesIO(b'prepended' + zipdata)) + self.assertFalse(zipfile.is_zipfile(io.BytesIO(b'prepended' + zipdata))) def test_generated_valid_zip64_extra(self): # These values are what is set in the make_zip64_file method. @@ -1812,11 +1893,8 @@ def test_write_with_source_date_epoch(self): with zipfile.ZipFile(TESTFN, "r") as zf: zip_info = zf.getinfo("test_source_date_epoch.txt") - get_time = time.localtime(int(os.environ['SOURCE_DATE_EPOCH']))[:6] - # Compare each element of the date_time tuple - # Allow for a 1-second difference - for z_time, g_time in zip(zip_info.date_time, get_time): - self.assertAlmostEqual(z_time, g_time, delta=1) + expected_utc = (2025, 1, 1, 7, 19, 58) + self.assertEqual(zip_info.date_time, expected_utc) def test_write_without_source_date_epoch(self): with os_helper.EnvironmentVarGuard() as env: @@ -1827,9 +1905,13 @@ def test_write_without_source_date_epoch(self): with zipfile.ZipFile(TESTFN, "r") as zf: zip_info = zf.getinfo("test_no_source_date_epoch.txt") - current_time = time.localtime()[:6] - for z_time, c_time in zip(zip_info.date_time, current_time): - self.assertAlmostEqual(z_time, c_time, delta=1) + self.assertTimestampAlmostEqual(time.localtime(), zip_info.date_time, tolerance=2) + + def assertTimestampAlmostEqual(self, time1, time2, tolerance): + import datetime + dt1 = datetime.datetime(*time1[:6]) + dt2 = datetime.datetime(*time2[:6]) + self.assertLessEqual((dt1 - dt2).total_seconds(), tolerance) def test_close(self): """Check that the zipfile is closed after the 'with' block.""" @@ -1991,6 +2073,25 @@ def test_is_zip_erroneous_file(self): self.assertFalse(zipfile.is_zipfile(fp)) fp.seek(0, 0) self.assertFalse(zipfile.is_zipfile(fp)) + # - passing non-zipfile with ZIP header elements + # data created using pyPNG like so: + # d = [(ord('P'), ord('K'), 5, 6), (ord('P'), ord('K'), 6, 6)] + # w = png.Writer(1,2,alpha=True,compression=0) + # f = open('onepix.png', 'wb') + # w.write(f, d) + # w.close() + data = (b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00" + b"\x00\x02\x08\x06\x00\x00\x00\x99\x81\xb6'\x00\x00\x00\x15I" + b"DATx\x01\x01\n\x00\xf5\xff\x00PK\x05\x06\x00PK\x06\x06\x07" + b"\xac\x01N\xc6|a\r\x00\x00\x00\x00IEND\xaeB`\x82") + # - passing a filename + with open(TESTFN, "wb") as fp: + fp.write(data) + self.assertFalse(zipfile.is_zipfile(TESTFN)) + # - passing a file-like object + fp = io.BytesIO() + fp.write(data) + self.assertFalse(zipfile.is_zipfile(fp)) def test_damaged_zipfile(self): """Check that zipfiles with missing bytes at the end raise BadZipFile.""" @@ -2237,6 +2338,7 @@ def test_empty_zipfile(self): zipf = zipfile.ZipFile(TESTFN, mode="r") except zipfile.BadZipFile: self.fail("Unable to create empty ZIP file in 'w' mode") + zipf.close() zipf = zipfile.ZipFile(TESTFN, mode="a") zipf.close() @@ -2244,6 +2346,7 @@ def test_empty_zipfile(self): zipf = zipfile.ZipFile(TESTFN, mode="r") except: self.fail("Unable to create empty ZIP file in 'a' mode") + zipf.close() def test_open_empty_file(self): # Issue 1710703: Check that opening a file with less than 22 bytes @@ -2436,6 +2539,10 @@ def test_decompress_without_3rd_party_library(self): @requires_zlib() def test_full_overlap_different_names(self): + # The ZIP file contains two central directory entries with + # different names which refer to the same local header. + # The name of the local header matches the name of the first + # central directory entry. data = ( b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00b\xed' @@ -2465,6 +2572,10 @@ def test_full_overlap_different_names(self): @requires_zlib() def test_full_overlap_different_names2(self): + # The ZIP file contains two central directory entries with + # different names which refer to the same local header. + # The name of the local header matches the name of the second + # central directory entry. data = ( b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00a\xed' @@ -2496,6 +2607,8 @@ def test_full_overlap_different_names2(self): @requires_zlib() def test_full_overlap_same_name(self): + # The ZIP file contains two central directory entries with + # the same name which refer to the same local header. data = ( b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05\xe2\x1e' b'8\xbb\x10\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00a\xed' @@ -2528,6 +2641,8 @@ def test_full_overlap_same_name(self): @requires_zlib() def test_quoted_overlap(self): + # The ZIP file contains two files. The second local header + # is contained in the range of the first file. data = ( b'PK\x03\x04\x14\x00\x00\x00\x08\x00\xa0lH\x05Y\xfc' b'8\x044\x00\x00\x00(\x04\x00\x00\x01\x00\x00\x00a\x00' @@ -2559,6 +2674,7 @@ def test_quoted_overlap(self): @requires_zlib() def test_overlap_with_central_dir(self): + # The local header offset is equal to the central directory offset. data = ( b'PK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00G_|Z' b'\xe2\x1e8\xbb\x0b\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00' @@ -2573,11 +2689,15 @@ def test_overlap_with_central_dir(self): self.assertEqual(zi.header_offset, 0) self.assertEqual(zi.compress_size, 11) self.assertEqual(zi.file_size, 1033) + # Found central directory signature PK\x01\x02 instead of + # local header signature PK\x03\x04. with self.assertRaisesRegex(zipfile.BadZipFile, 'Bad magic number'): zipf.read('a') @requires_zlib() def test_overlap_with_archive_comment(self): + # The local header is written after the central directory, + # in the archive comment. data = ( b'PK\x01\x02\x14\x03\x14\x00\x00\x00\x08\x00G_|Z' b'\xe2\x1e8\xbb\x0b\x00\x00\x00\t\x04\x00\x00\x01\x00\x00\x00' @@ -3179,7 +3299,7 @@ def test_write_dir(self): with zipfile.ZipFile(TESTFN, "w") as zipf: zipf.write(dirpath) zinfo = zipf.filelist[0] - self.assertTrue(zinfo.filename.endswith("/x/")) + self.assertEndsWith(zinfo.filename, "/x/") self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) zipf.write(dirpath, "y") zinfo = zipf.filelist[1] @@ -3187,7 +3307,7 @@ def test_write_dir(self): self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) with zipfile.ZipFile(TESTFN, "r") as zipf: zinfo = zipf.filelist[0] - self.assertTrue(zinfo.filename.endswith("/x/")) + self.assertEndsWith(zinfo.filename, "/x/") self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10) zinfo = zipf.filelist[1] self.assertTrue(zinfo.filename, "y/") @@ -3207,7 +3327,7 @@ def test_writestr_dir(self): self.assertEqual(zinfo.external_attr, (0o40775 << 16) | 0x10) with zipfile.ZipFile(TESTFN, "r") as zipf: zinfo = zipf.filelist[0] - self.assertTrue(zinfo.filename.endswith("x/")) + self.assertEndsWith(zinfo.filename, "x/") self.assertEqual(zinfo.external_attr, (0o40775 << 16) | 0x10) target = os.path.join(TESTFN2, "target") os.mkdir(target) @@ -3451,60 +3571,6 @@ def test_execute_zip64(self): self.assertIn(b'number in executable: 5', output) -class TestDataOffsetPrependedZip(unittest.TestCase): - """Test .data_offset on reading zip files with an executable prepended.""" - - def setUp(self): - self.exe_zip = findfile('exe_with_zip', subdir='archivetestdata') - self.exe_zip64 = findfile('exe_with_z64', subdir='archivetestdata') - - def _test_data_offset(self, name): - with zipfile.ZipFile(name) as zipfp: - self.assertEqual(zipfp.data_offset, 713) - - def test_data_offset_with_exe_prepended(self): - self._test_data_offset(self.exe_zip) - - def test_data_offset_with_exe_prepended_zip64(self): - self._test_data_offset(self.exe_zip64) - -class TestDataOffsetZipWrite(unittest.TestCase): - """Test .data_offset for ZipFile opened in write mode.""" - - def setUp(self): - os.mkdir(TESTFNDIR) - self.addCleanup(rmtree, TESTFNDIR) - self.test_path = os.path.join(TESTFNDIR, 'testoffset.zip') - - def test_data_offset_write_no_prefix(self): - with io.BytesIO() as fp: - with zipfile.ZipFile(fp, "w") as zipfp: - self.assertEqual(zipfp.data_offset, 0) - - def test_data_offset_write_with_prefix(self): - with io.BytesIO() as fp: - fp.write(b"this is a prefix") - with zipfile.ZipFile(fp, "w") as zipfp: - self.assertEqual(zipfp.data_offset, 16) - - def test_data_offset_append_with_bad_zip(self): - with io.BytesIO() as fp: - fp.write(b"this is a prefix") - with zipfile.ZipFile(fp, "a") as zipfp: - self.assertEqual(zipfp.data_offset, 16) - - def test_data_offset_write_no_tell(self): - # The initializer in ZipFile checks if tell raises AttributeError or - # OSError when creating a file in write mode when deducing the offset - # of the beginning of zip data - class NoTellBytesIO(io.BytesIO): - def tell(self): - raise OSError("Unimplemented!") - with NoTellBytesIO() as fp: - with zipfile.ZipFile(fp, "w") as zipfp: - self.assertIsNone(zipfp.data_offset) - - class EncodedMetadataTests(unittest.TestCase): file_names = ['\u4e00', '\u4e8c', '\u4e09'] # Han 'one', 'two', 'three' file_content = [ @@ -3581,29 +3647,23 @@ def test_read_with_unsuitable_metadata_encoding(self): def test_read_after_append(self): newname = '\u56db' # Han 'four' - expected_names = [name.encode('shift_jis').decode('cp437') - for name in self.file_names[:2]] + self.file_names[2:] - expected_names.append(newname) - expected_content = (*self.file_content, b"newcontent") + newname2 = 'fünf' # representable in cp437, but still stored as UTF-8 + expected_names = [*self.file_names, newname, newname2] + mojibake_expected_names = [name.encode('shift_jis').decode('cp437') + if i < 2 else name + for i, name in enumerate(expected_names)] + expected_content = (*self.file_content, b"newcontent", b"newcontent2") with zipfile.ZipFile(TESTFN, "a") as zipfp: zipfp.writestr(newname, "newcontent") - self.assertEqual(sorted(zipfp.namelist()), sorted(expected_names)) + zipfp.writestr(newname2, "newcontent2") + self.assertEqual(sorted(zipfp.namelist()), sorted(mojibake_expected_names)) with zipfile.ZipFile(TESTFN, "r") as zipfp: - self._test_read(zipfp, expected_names, expected_content) + self._test_read(zipfp, mojibake_expected_names, expected_content) with zipfile.ZipFile(TESTFN, "r", metadata_encoding='shift_jis') as zipfp: - self.assertEqual(sorted(zipfp.namelist()), sorted(expected_names)) - for i, (name, content) in enumerate(zip(expected_names, expected_content)): - info = zipfp.getinfo(name) - self.assertEqual(info.filename, name) - self.assertEqual(info.file_size, len(content)) - if i < 2: - with self.assertRaises(zipfile.BadZipFile): - zipfp.read(name) - else: - self.assertEqual(zipfp.read(name), content) + self._test_read(zipfp, expected_names, expected_content) def test_write_with_metadata_encoding(self): ZF = zipfile.ZipFile @@ -3612,6 +3672,20 @@ def test_write_with_metadata_encoding(self): "^metadata_encoding is only"): ZF("nonesuch.zip", mode, metadata_encoding="shift_jis") + def test_add_comment(self): + with zipfile.ZipFile(TESTFN, "r") as zipfp: + mojibake_expected_names = zipfp.namelist() + + with zipfile.ZipFile(TESTFN, "a") as zipfp: + zipfp.comment = b'comment' + self.assertEqual(zipfp.namelist(), mojibake_expected_names) + + with zipfile.ZipFile(TESTFN, "r") as zipfp: + self._test_read(zipfp, mojibake_expected_names, self.file_content) + + with zipfile.ZipFile(TESTFN, "r", metadata_encoding='shift_jis') as zipfp: + self._test_read(zipfp, self.file_names, self.file_content) + def test_cli_with_metadata_encoding(self): errmsg = "Non-conforming encodings not supported with -c." args = ["--metadata-encoding=shift_jis", "-c", "nonesuch", "nonesuch"] @@ -3642,7 +3716,7 @@ def test_cli_with_metadata_encoding_extract(self): except OSError: pass except UnicodeEncodeError: - self.skipTest(f'cannot encode file name {fn!r}') + self.skipTest(f'cannot encode file name {fn!a}') zipfile.main(["--metadata-encoding=shift_jis", "-e", TESTFN, TESTFN2]) listing = os.listdir(TESTFN2) diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index 1f288c8b45d5899..b5b4acf5f850bee 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -835,11 +835,11 @@ def doTraceback(self, module): s = io.StringIO() print_tb(tb, 1, s) - self.assertTrue(s.getvalue().endswith( + self.assertEndsWith(s.getvalue(), ' def do_raise(): raise TypeError\n' '' if support.has_no_debug_ranges() else ' ^^^^^^^^^^^^^^^\n' - )) + ) else: raise AssertionError("This ought to be impossible") diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index d2845495c7f8b69..0fc1445e6b0bf55 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -18,10 +18,11 @@ from functools import cached_property from test.support import MISSING_C_DOCSTRINGS -from test.support.os_helper import EnvironmentVarGuard +from test.support.os_helper import EnvironmentVarGuard, FakePath from test.test_zoneinfo import _support as test_support from test.test_zoneinfo._support import TZPATH_TEST_LOCK, ZoneInfoTestBase from test.support.import_helper import import_module, CleanImport +from test.support.script_helper import assert_python_ok lzma = import_module('lzma') py_zoneinfo, c_zoneinfo = test_support.get_modules() @@ -58,6 +59,10 @@ def tearDownModule(): shutil.rmtree(TEMP_DIR) +class CustomError(Exception): + pass + + class TzPathUserMixin: """ Adds a setUp() and tearDown() to make TZPATH manipulations thread-safe. @@ -247,6 +252,8 @@ def test_bad_zones(self): bad_zones = [ b"", # Empty file b"AAAA3" + b" " * 15, # Bad magic + # Truncated V2 file (should not loop indefinitely) + b"TZif2" + (b"\x00" * 39) + b"TZif2" + (b"\x00" * 39) + b"\n" + b"Part", ] for bad_zone in bad_zones: @@ -404,6 +411,25 @@ def test_time_fixed_offset(self): self.assertEqual(t.utcoffset(), offset.utcoffset) self.assertEqual(t.dst(), offset.dst) + def test_cache_exception(self): + class Incomparable(str): + eq_called = False + def __eq__(self, other): + self.eq_called = True + raise CustomError + __hash__ = str.__hash__ + + key = "America/Los_Angeles" + tz1 = self.klass(key) + key = Incomparable(key) + try: + tz2 = self.klass(key) + except CustomError: + self.assertTrue(key.eq_called) + else: + self.assertFalse(key.eq_called) + self.assertIs(tz2, tz1) + class CZoneInfoTest(ZoneInfoTest): module = c_zoneinfo @@ -715,6 +741,38 @@ def test_empty_zone(self): with self.assertRaises(ValueError): self.klass.from_file(zf) + def test_invalid_transition_index(self): + STD = ZoneOffset("STD", ZERO) + DST = ZoneOffset("DST", ONE_H, ONE_H) + + zf = self.construct_zone([ + ZoneTransition(datetime(2026, 3, 1, 2), STD, DST), + ZoneTransition(datetime(2026, 11, 1, 2), DST, STD), + ], after="", version=1) + + data = bytearray(zf.read()) + timecnt = struct.unpack_from(">l", data, 32)[0] + idx_offset = 44 + timecnt * 4 + data[idx_offset + 1] = 2 # typecnt is 2, so index 2 is OOB + f = io.BytesIO(bytes(data)) + + with self.assertRaises(ValueError): + self.klass.from_file(f) + + def test_transition_lookahead_out_of_bounds(self): + STD = ZoneOffset("STD", ZERO) + DST = ZoneOffset("DST", ONE_H, ONE_H) + EXT = ZoneOffset("EXT", ONE_H) + + zf = self.construct_zone([ + ZoneTransition(datetime(2026, 3, 1), STD, DST), + ZoneTransition(datetime(2026, 6, 1), DST, EXT), + ZoneTransition(datetime(2026, 9, 1), EXT, DST), + ], after="") + + zi = self.klass.from_file(zf) + self.assertIsNotNone(zi) + def test_zone_very_large_timestamp(self): """Test when a transition is in the far past or future. @@ -951,14 +1009,14 @@ def populate_tzstr_header(cls): cls._tzif_header = bytes(out) - def zone_from_tzstr(self, tzstr): + def zone_from_tzstr(self, tzstr, encoding="ascii"): """Creates a zoneinfo file following a POSIX rule.""" zonefile = io.BytesIO(self._tzif_header) zonefile.seek(0, 2) # Write the footer zonefile.write(b"\x0A") - zonefile.write(tzstr.encode("ascii")) + zonefile.write(tzstr.encode(encoding)) zonefile.write(b"\x0A") zonefile.seek(0) @@ -1053,6 +1111,9 @@ def test_extreme_tzstr(self): "AAA4BBB,J1/2,J1/14", "AAA4BBB,J20/2,J365/2", "AAA4BBB,J365/2,J365/14", + # Leading-zero day-of-year + "AAA4BBB,J001/2,J065/2", + "AAA4BBB,001/2,065/2", # Extreme transition hour "AAA4BBB,J60/167,J300/2", "AAA4BBB,J60/+167,J300/2", @@ -1084,9 +1145,21 @@ def test_extreme_tzstr(self): def test_invalid_tzstr(self): invalid_tzstrs = [ "PST8PDT", # DST but no transition specified + # gh-152212: the std offset is required (POSIX TZ grammar) + "AAA", + "A", + "AA", + "B", "+11", # Unquoted alphanumeric "GMT,M3.2.0/2,M11.1.0/3", # Transition rule but no DST "GMT0+11,M3.2.0/2,M11.1.0/3", # Unquoted alphanumeric in DST + # Unquoted abbreviation with embedded or leading whitespace + "AB C3", + " A B 3", + "AAA4BB B,J60/2,J300/2", # Embedded whitespace in DST + # Empty quoted abbreviation + "<>5", + "AAA4<>,M3.2.0/2,M11.1.0/3", "PST8PDT,M3.2.0/2", # Only one transition rule # Invalid offset hours "AAA168", @@ -1126,6 +1199,11 @@ def test_invalid_tzstr(self): # Invalid weekday "AAA4BBB,M1.1.7/2,M2.1.1/2", "AAA4BBB,M1.1.1/2,M2.1.7/2", + # Invalid Mm.w.d separator + "AAA4BBB,M3.2X0,M11.1.0", + "AAA4BBB,M3.2.0,M11.1X0", + "AAA4BBB,M3.2-0,M11.1.0/3", + "AAA4BBB,M3.2.0/2,M11.1:0", # Invalid numeric offset "AAA4BBB,-1/2,20/2", "AAA4BBB,1/2,-1/2", @@ -1134,6 +1212,15 @@ def test_invalid_tzstr(self): # Invalid julian offset "AAA4BBB,J0/2,J20/2", "AAA4BBB,J20/2,J366/2", + # gh-152847: non-digit day-of-year + "AAA4BBB,J1_0,J300/2", + "AAA4BBB,J60/2,J30_0/2", + "AAA4BBB,1_0,J300/2", + "AAA4BBB,J+1,J300/2", + "AAA4BBB,J 1,J300/2", + "AAA4BBB, 1,J300/2", + "AAA4BBB,J0001,J300/2", + "AAA4BBB,0001,J300/2", # Invalid transition time "AAA4BBB,J60/2/3,J300/2", "AAA4BBB,J60/2,J300/2/3", @@ -1164,6 +1251,24 @@ def test_invalid_tzstr(self): with self.assertRaisesRegex(ValueError, tzstr_regex): self.zone_from_tzstr(invalid_tzstr) + def test_invalid_tzstr_non_ascii_abbr(self): + tzstr = "ABÀC3" + if self.module is py_zoneinfo: + expected = re.escape(tzstr) + else: + expected = re.escape(repr(tzstr.encode("utf-8"))) + with self.assertRaisesRegex(ValueError, expected): + self.zone_from_tzstr(tzstr, encoding="utf-8") + + def test_invalid_tzstr_non_ascii_dst_date(self): + tzstr = "AAA4BBB,J١,J300/2" + if self.module is py_zoneinfo: + expected = re.escape(tzstr) + else: + expected = re.escape(repr(tzstr.encode("utf-8"))) + with self.assertRaisesRegex(ValueError, expected): + self.zone_from_tzstr(tzstr, encoding="utf-8") + @classmethod def _populate_test_cases(cls): # This method uses a somewhat unusual style in that it populates the @@ -1507,10 +1612,100 @@ def test_clear_cache_two_keys(self): self.assertIsNot(dub0, dub1) self.assertIs(tok0, tok1) + def test_clear_cache_refleak(self): + class Stringy(str): + allow_comparisons = True + def __eq__(self, other): + if not self.allow_comparisons: + raise CustomError + return super().__eq__(other) + __hash__ = str.__hash__ + + key = Stringy("America/Los_Angeles") + self.klass(key) + key.allow_comparisons = False + try: + # Note: This is try/except rather than assertRaises because + # there is no guarantee that the key is even still in the cache, + # or that the key for the cache is the original `key` object. + self.klass.clear_cache(only_keys="America/Los_Angeles") + except CustomError: + pass + + def test_weak_cache_descriptor_use_after_free(self): + class BombDescriptor: + def __get__(self, obj, owner): + return {} + + class EvilZoneInfo(self.klass): + pass + + # Must be set after the class creation. + EvilZoneInfo._weak_cache = BombDescriptor() + + key = "America/Los_Angeles" + zone1 = EvilZoneInfo(key) + self.assertEqual(str(zone1), key) + + EvilZoneInfo.clear_cache() + zone2 = EvilZoneInfo(key) + self.assertEqual(str(zone2), key) + self.assertIsNot(zone2, zone1) + class CZoneInfoCacheTest(ZoneInfoCacheTest): module = c_zoneinfo + def test_inconsistent_weak_cache_get(self): + class Cache: + def get(self, key, default=None): + return 1337 + + class ZI(self.klass): + pass + # Class attribute must be set after class creation + # to override zoneinfo.ZoneInfo.__init_subclass__. + ZI._weak_cache = Cache() + + with self.assertRaises(RuntimeError) as te: + ZI("America/Los_Angeles") + self.assertEqual( + str(te.exception), + "Unexpected instance of int in ZI weak cache for key 'America/Los_Angeles'" + ) + + def test_deleted_weak_cache(self): + class ZI(self.klass): + pass + delattr(ZI, '_weak_cache') + + # These should not segfault + with self.assertRaises(AttributeError): + ZI("UTC") + + with self.assertRaises(AttributeError): + ZI.clear_cache() + + def test_inconsistent_weak_cache_setdefault(self): + class Cache: + def get(self, key, default=None): + return default + def setdefault(self, key, value): + return 1337 + + class ZI(self.klass): + pass + # Class attribute must be set after class creation + # to override zoneinfo.ZoneInfo.__init_subclass__. + ZI._weak_cache = Cache() + + with self.assertRaises(RuntimeError) as te: + ZI("America/Los_Angeles") + self.assertEqual( + str(te.exception), + "Unexpected instance of int in ZI weak cache for key 'America/Los_Angeles'" + ) + class ZoneInfoPickleTest(TzPathUserMixin, ZoneInfoTestBase): module = py_zoneinfo @@ -1740,6 +1935,7 @@ def test_reset_tzpath_relative_paths(self): ("/usr/share/zoneinfo", "../relative/path",), ("path/to/somewhere", "../relative/path",), ("/usr/share/zoneinfo", "path/to/somewhere", "../relative/path",), + (FakePath("path/to/somewhere"),) ] for input_paths in bad_values: with self.subTest(input_paths=input_paths): @@ -1751,6 +1947,9 @@ def test_tzpath_type_error(self): "/etc/zoneinfo:/usr/share/zoneinfo", b"/etc/zoneinfo:/usr/share/zoneinfo", 0, + (b"/bytes/path", "/valid/path"), + (FakePath(b"/bytes/path"),), + (0,), ] for bad_value in bad_values: @@ -1761,6 +1960,7 @@ def test_tzpath_type_error(self): def test_tzpath_attribute(self): tzpath_0 = [f"{DRIVE}/one", f"{DRIVE}/two"] tzpath_1 = [f"{DRIVE}/three"] + tzpath_pathlike = (FakePath(f"{DRIVE}/usr/share/zoneinfo"),) with self.tzpath_context(tzpath_0): query_0 = self.module.TZPATH @@ -1768,8 +1968,12 @@ def test_tzpath_attribute(self): with self.tzpath_context(tzpath_1): query_1 = self.module.TZPATH + with self.tzpath_context(tzpath_pathlike): + query_pathlike = self.module.TZPATH + self.assertSequenceEqual(tzpath_0, query_0) self.assertSequenceEqual(tzpath_1, query_1) + self.assertSequenceEqual(tuple([os.fspath(p) for p in tzpath_pathlike]), query_pathlike) class CTzPathTest(TzPathTest): @@ -1903,6 +2107,26 @@ class CTestModule(TestModule): module = c_zoneinfo +class MiscTests(unittest.TestCase): + def test_pydatetime(self): + # Test that zoneinfo works if the C implementation of datetime + # is not available and the Python implementation of datetime is used. + # The Python implementation of zoneinfo should be used in thet case. + # + # Run the test in a subprocess, as importing _zoneinfo with + # _datettime disabled causes crash in the previously imported + # _zoneinfo. + assert_python_ok('-c', '''if 1: + import sys + sys.modules['_datetime'] = None + import datetime + import zoneinfo + tzinfo = zoneinfo.ZoneInfo('Europe/London') + datetime.datetime(2025, 10, 26, 2, 0, tzinfo=tzinfo) + ''', + PYTHONTZPATH=str(ZONEINFO_DATA.tzpath)) + + class ExtensionBuiltTest(unittest.TestCase): """Smoke test to ensure that the C and Python extensions are both tested. @@ -1915,8 +2139,8 @@ class ExtensionBuiltTest(unittest.TestCase): def test_cache_location(self): # The pure Python version stores caches on attributes, but the C # extension stores them in C globals (at least for now) - self.assertFalse(hasattr(c_zoneinfo.ZoneInfo, "_weak_cache")) - self.assertTrue(hasattr(py_zoneinfo.ZoneInfo, "_weak_cache")) + self.assertNotHasAttr(c_zoneinfo.ZoneInfo, "_weak_cache") + self.assertHasAttr(py_zoneinfo.ZoneInfo, "_weak_cache") def test_gc_tracked(self): import gc diff --git a/Lib/test/test_zoneinfo/test_zoneinfo_property.py b/Lib/test/test_zoneinfo/test_zoneinfo_property.py index feaa77f3e7f0b92..c00815e2fd4c364 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo_property.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo_property.py @@ -146,20 +146,22 @@ def setUp(self): @add_key_examples def test_pickle_unpickle_cache(self, key): zi = self.klass(key) - pkl_str = pickle.dumps(zi) - zi_rt = pickle.loads(pkl_str) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + pkl_str = pickle.dumps(zi, proto) + zi_rt = pickle.loads(pkl_str) - self.assertIs(zi, zi_rt) + self.assertIs(zi, zi_rt) @hypothesis.given(key=valid_keys()) @add_key_examples def test_pickle_unpickle_no_cache(self, key): zi = self.klass.no_cache(key) - pkl_str = pickle.dumps(zi) - zi_rt = pickle.loads(pkl_str) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + pkl_str = pickle.dumps(zi, proto) + zi_rt = pickle.loads(pkl_str) - self.assertIsNot(zi, zi_rt) - self.assertEqual(str(zi), str(zi_rt)) + self.assertIsNot(zi, zi_rt) + self.assertEqual(str(zi), str(zi_rt)) @hypothesis.given(key=valid_keys()) @add_key_examples diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index f4a25376e5234a6..6de63d840898ac7 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -63,11 +63,18 @@ TRAINED_DICT = None -SUPPORT_MULTITHREADING = False +# Cannot be deferred to setup as it is used to check whether or not to skip +# tests +try: + SUPPORT_MULTITHREADING = CompressionParameter.nb_workers.bounds() != (0, 0) +except Exception: + SUPPORT_MULTITHREADING = False + +C_INT_MIN = -(2**31) +C_INT_MAX = (2**31) - 1 + def setUpModule(): - global SUPPORT_MULTITHREADING - SUPPORT_MULTITHREADING = CompressionParameter.nb_workers.bounds() != (0, 0) # uncompressed size 130KB, more than a zstd block. # with a frame epilogue, 4 bytes checksum. global DAT_130K_D @@ -196,14 +203,21 @@ def test_simple_compress_bad_args(self): self.assertRaises(TypeError, ZstdCompressor, zstd_dict=b"abcd1234") self.assertRaises(TypeError, ZstdCompressor, zstd_dict={1: 2, 3: 4}) - with self.assertRaises(ValueError): - ZstdCompressor(2**31) - with self.assertRaises(ValueError): - ZstdCompressor(options={2**31: 100}) + # valid range for compression level is [-(1<<17), 22] + msg = r'illegal compression level {}; the valid range is \[-?\d+, -?\d+\]' + with self.assertRaisesRegex(ValueError, msg.format(C_INT_MAX)): + ZstdCompressor(C_INT_MAX) + with self.assertRaisesRegex(ValueError, msg.format(C_INT_MIN)): + ZstdCompressor(C_INT_MIN) + msg = r'illegal compression level; the valid range is \[-?\d+, -?\d+\]' + with self.assertRaisesRegex(ValueError, msg): + ZstdCompressor(level=-(2**1000)) + with self.assertRaisesRegex(ValueError, msg): + ZstdCompressor(level=2**1000) - with self.assertRaises(ZstdError): + with self.assertRaises(ValueError): ZstdCompressor(options={CompressionParameter.window_log: 100}) - with self.assertRaises(ZstdError): + with self.assertRaises(ValueError): ZstdCompressor(options={3333: 100}) # Method bad arguments @@ -254,43 +268,57 @@ def test_compress_parameters(self): } ZstdCompressor(options=d) - # larger than signed int, ValueError d1 = d.copy() - d1[CompressionParameter.ldm_bucket_size_log] = 2**31 - self.assertRaises(ValueError, ZstdCompressor, options=d1) + # larger than signed int + d1[CompressionParameter.ldm_bucket_size_log] = C_INT_MAX + with self.assertRaises(ValueError): + ZstdCompressor(options=d1) + # smaller than signed int + d1[CompressionParameter.ldm_bucket_size_log] = C_INT_MIN + with self.assertRaises(ValueError): + ZstdCompressor(options=d1) - # clamp compressionLevel + # out of bounds compression level level_min, level_max = CompressionParameter.compression_level.bounds() - compress(b'', level_max+1) - compress(b'', level_min-1) - - compress(b'', options={CompressionParameter.compression_level:level_max+1}) - compress(b'', options={CompressionParameter.compression_level:level_min-1}) + with self.assertRaises(ValueError): + compress(b'', level_max+1) + with self.assertRaises(ValueError): + compress(b'', level_min-1) + with self.assertRaises(ValueError): + compress(b'', 2**1000) + with self.assertRaises(ValueError): + compress(b'', -(2**1000)) + with self.assertRaises(ValueError): + compress(b'', options={ + CompressionParameter.compression_level: level_max+1}) + with self.assertRaises(ValueError): + compress(b'', options={ + CompressionParameter.compression_level: level_min-1}) # zstd lib doesn't support MT compression if not SUPPORT_MULTITHREADING: - with self.assertRaises(ZstdError): + with self.assertRaises(ValueError): ZstdCompressor(options={CompressionParameter.nb_workers:4}) - with self.assertRaises(ZstdError): + with self.assertRaises(ValueError): ZstdCompressor(options={CompressionParameter.job_size:4}) - with self.assertRaises(ZstdError): + with self.assertRaises(ValueError): ZstdCompressor(options={CompressionParameter.overlap_log:4}) # out of bounds error msg option = {CompressionParameter.window_log:100} - with self.assertRaisesRegex(ZstdError, - (r'Error when setting zstd compression parameter "window_log", ' - r'it should \d+ <= value <= \d+, provided value is 100\. ' - r'\(zstd v\d\.\d\.\d, (?:32|64)-bit build\)')): + with self.assertRaisesRegex( + ValueError, + "compression parameter 'window_log' received an illegal value 100; " + r'the valid range is \[-?\d+, -?\d+\]', + ): compress(b'', options=option) def test_unknown_compression_parameter(self): KEY = 100001234 option = {CompressionParameter.compression_level: 10, KEY: 200000000} - pattern = r'Zstd compression parameter.*?"unknown parameter \(key %d\)"' \ - % KEY - with self.assertRaisesRegex(ZstdError, pattern): + pattern = rf"invalid compression parameter 'unknown parameter \(key {KEY}\)'" + with self.assertRaisesRegex(ValueError, pattern): ZstdCompressor(options=option) @unittest.skipIf(not SUPPORT_MULTITHREADING, @@ -371,6 +399,115 @@ def test_compress_empty(self): c = ZstdCompressor() self.assertNotEqual(c.compress(b'', c.FLUSH_FRAME), b'') + def test_set_pledged_input_size(self): + DAT = DECOMPRESSED_100_PLUS_32KB + CHUNK_SIZE = len(DAT) // 3 + + # wrong value + c = ZstdCompressor() + with self.assertRaisesRegex(ValueError, + r'should be a positive int less than \d+'): + c.set_pledged_input_size(-300) + # overflow + with self.assertRaisesRegex(ValueError, + r'should be a positive int less than \d+'): + c.set_pledged_input_size(2**64) + # ZSTD_CONTENTSIZE_ERROR is invalid + with self.assertRaisesRegex(ValueError, + r'should be a positive int less than \d+'): + c.set_pledged_input_size(2**64-2) + # ZSTD_CONTENTSIZE_UNKNOWN should use None + with self.assertRaisesRegex(ValueError, + r'should be a positive int less than \d+'): + c.set_pledged_input_size(2**64-1) + + # check valid values are settable + c.set_pledged_input_size(2**63) + c.set_pledged_input_size(2**64-3) + + # check that zero means empty frame + c = ZstdCompressor(level=1) + c.set_pledged_input_size(0) + c.compress(b'') + dat = c.flush() + ret = get_frame_info(dat) + self.assertEqual(ret.decompressed_size, 0) + + + # wrong mode + c = ZstdCompressor(level=1) + c.compress(b'123456') + self.assertEqual(c.last_mode, c.CONTINUE) + with self.assertRaisesRegex(ValueError, + r'last_mode == FLUSH_FRAME'): + c.set_pledged_input_size(300) + + # None value + c = ZstdCompressor(level=1) + c.set_pledged_input_size(None) + dat = c.compress(DAT) + c.flush() + + ret = get_frame_info(dat) + self.assertEqual(ret.decompressed_size, None) + + # correct value + c = ZstdCompressor(level=1) + c.set_pledged_input_size(len(DAT)) + + chunks = [] + posi = 0 + while posi < len(DAT): + dat = c.compress(DAT[posi:posi+CHUNK_SIZE]) + posi += CHUNK_SIZE + chunks.append(dat) + + dat = c.flush() + chunks.append(dat) + chunks = b''.join(chunks) + + ret = get_frame_info(chunks) + self.assertEqual(ret.decompressed_size, len(DAT)) + self.assertEqual(decompress(chunks), DAT) + + c.set_pledged_input_size(len(DAT)) # the second frame + dat = c.compress(DAT) + c.flush() + + ret = get_frame_info(dat) + self.assertEqual(ret.decompressed_size, len(DAT)) + self.assertEqual(decompress(dat), DAT) + + # not enough data + c = ZstdCompressor(level=1) + c.set_pledged_input_size(len(DAT)+1) + + for start in range(0, len(DAT), CHUNK_SIZE): + end = min(start+CHUNK_SIZE, len(DAT)) + _dat = c.compress(DAT[start:end]) + + with self.assertRaises(ZstdError): + c.flush() + + # too much data + c = ZstdCompressor(level=1) + c.set_pledged_input_size(len(DAT)) + + for start in range(0, len(DAT), CHUNK_SIZE): + end = min(start+CHUNK_SIZE, len(DAT)) + _dat = c.compress(DAT[start:end]) + + with self.assertRaises(ZstdError): + c.compress(b'extra', ZstdCompressor.FLUSH_FRAME) + + # content size not set if content_size_flag == 0 + c = ZstdCompressor(options={CompressionParameter.content_size_flag: 0}) + c.set_pledged_input_size(10) + dat1 = c.compress(b"hello") + dat2 = c.compress(b"world") + dat3 = c.flush() + frame_data = get_frame_info(dat1 + dat2 + dat3) + self.assertIsNone(frame_data.decompressed_size) + + class DecompressorTestCase(unittest.TestCase): def test_simple_decompress_bad_args(self): @@ -385,12 +522,22 @@ def test_simple_decompress_bad_args(self): self.assertRaises(TypeError, ZstdDecompressor, options=b'abc') with self.assertRaises(ValueError): - ZstdDecompressor(options={2**31 : 100}) + ZstdDecompressor(options={C_INT_MAX: 100}) + with self.assertRaises(ValueError): + ZstdDecompressor(options={C_INT_MIN: 100}) + with self.assertRaises(ValueError): + ZstdDecompressor(options={0: C_INT_MAX}) + with self.assertRaises(OverflowError): + ZstdDecompressor(options={2**1000: 100}) + with self.assertRaises(OverflowError): + ZstdDecompressor(options={-(2**1000): 100}) + with self.assertRaises(OverflowError): + ZstdDecompressor(options={0: -(2**1000)}) - with self.assertRaises(ZstdError): - ZstdDecompressor(options={DecompressionParameter.window_log_max:100}) - with self.assertRaises(ZstdError): - ZstdDecompressor(options={3333 : 100}) + with self.assertRaises(ValueError): + ZstdDecompressor(options={DecompressionParameter.window_log_max: 100}) + with self.assertRaises(ValueError): + ZstdDecompressor(options={3333: 100}) empty = compress(b'') lzd = ZstdDecompressor() @@ -403,26 +550,52 @@ def test_decompress_parameters(self): d = {DecompressionParameter.window_log_max : 15} ZstdDecompressor(options=d) - # larger than signed int, ValueError d1 = d.copy() - d1[DecompressionParameter.window_log_max] = 2**31 - self.assertRaises(ValueError, ZstdDecompressor, None, d1) + # larger than signed int + d1[DecompressionParameter.window_log_max] = 2**1000 + with self.assertRaises(OverflowError): + ZstdDecompressor(None, d1) + # smaller than signed int + d1[DecompressionParameter.window_log_max] = -(2**1000) + with self.assertRaises(OverflowError): + ZstdDecompressor(None, d1) + + d1[DecompressionParameter.window_log_max] = C_INT_MAX + with self.assertRaises(ValueError): + ZstdDecompressor(None, d1) + d1[DecompressionParameter.window_log_max] = C_INT_MIN + with self.assertRaises(ValueError): + ZstdDecompressor(None, d1) # out of bounds error msg options = {DecompressionParameter.window_log_max:100} - with self.assertRaisesRegex(ZstdError, - (r'Error when setting zstd decompression parameter "window_log_max", ' - r'it should \d+ <= value <= \d+, provided value is 100\. ' - r'\(zstd v\d\.\d\.\d, (?:32|64)-bit build\)')): + with self.assertRaisesRegex( + ValueError, + "decompression parameter 'window_log_max' received an illegal value 100; " + r'the valid range is \[-?\d+, -?\d+\]', + ): + decompress(b'', options=options) + + # out of bounds deecompression parameter + options[DecompressionParameter.window_log_max] = C_INT_MAX + with self.assertRaises(ValueError): + decompress(b'', options=options) + options[DecompressionParameter.window_log_max] = C_INT_MIN + with self.assertRaises(ValueError): + decompress(b'', options=options) + options[DecompressionParameter.window_log_max] = 2**1000 + with self.assertRaises(OverflowError): + decompress(b'', options=options) + options[DecompressionParameter.window_log_max] = -(2**1000) + with self.assertRaises(OverflowError): decompress(b'', options=options) def test_unknown_decompression_parameter(self): KEY = 100001234 options = {DecompressionParameter.window_log_max: DecompressionParameter.window_log_max.bounds()[1], KEY: 200000000} - pattern = r'Zstd decompression parameter.*?"unknown parameter \(key %d\)"' \ - % KEY - with self.assertRaisesRegex(ZstdError, pattern): + pattern = rf"invalid decompression parameter 'unknown parameter \(key {KEY}\)'" + with self.assertRaisesRegex(ValueError, pattern): ZstdDecompressor(options=options) def test_decompress_epilogue_flags(self): @@ -507,7 +680,7 @@ def test_decompress_epilogue_flags(self): self.assertFalse(d.needs_input) def test_decompressor_arg(self): - zd = ZstdDict(b'12345678', True) + zd = ZstdDict(b'12345678', is_raw=True) with self.assertRaises(TypeError): d = ZstdDecompressor(zstd_dict={}) @@ -1021,6 +1194,10 @@ def test_decompressor_skippable(self): class ZstdDictTestCase(unittest.TestCase): def test_is_raw(self): + # must be passed as a keyword argument + with self.assertRaises(TypeError): + ZstdDict(bytes(8), True) + # content < 8 b = b'1234567' with self.assertRaises(ValueError): @@ -1068,35 +1245,49 @@ def test_invalid_dict(self): # corrupted zd = ZstdDict(dict_content, is_raw=False) - with self.assertRaisesRegex(ZstdError, r'ZSTD_CDict.*?corrupted'): + with self.assertRaisesRegex(ZstdError, r'ZSTD_CDict.*?content\.$'): ZstdCompressor(zstd_dict=zd.as_digested_dict) - with self.assertRaisesRegex(ZstdError, r'ZSTD_DDict.*?corrupted'): + with self.assertRaisesRegex(ZstdError, r'ZSTD_DDict.*?content\.$'): ZstdDecompressor(zd) # wrong type - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): - ZstdCompressor(zstd_dict=(zd, b'123')) - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdCompressor(zstd_dict=[zd, 1]) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdCompressor(zstd_dict=(zd, 1.0)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdCompressor(zstd_dict=(zd,)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): ZstdCompressor(zstd_dict=(zd, 1, 2)) - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): ZstdCompressor(zstd_dict=(zd, -1)) - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): ZstdCompressor(zstd_dict=(zd, 3)) - - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): - ZstdDecompressor(zstd_dict=(zd, b'123')) - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): + with self.assertRaises(OverflowError): + ZstdCompressor(zstd_dict=(zd, 2**1000)) + with self.assertRaises(OverflowError): + ZstdCompressor(zstd_dict=(zd, -2**1000)) + + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdDecompressor(zstd_dict=[zd, 1]) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdDecompressor(zstd_dict=(zd, 1.0)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): + ZstdDecompressor((zd,)) + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): ZstdDecompressor((zd, 1, 2)) - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): ZstdDecompressor((zd, -1)) - with self.assertRaisesRegex(TypeError, r'should be ZstdDict object'): + with self.assertRaisesRegex(TypeError, r'should be a ZstdDict object'): ZstdDecompressor((zd, 3)) + with self.assertRaises(OverflowError): + ZstdDecompressor((zd, 2**1000)) + with self.assertRaises(OverflowError): + ZstdDecompressor((zd, -2**1000)) def test_train_dict(self): - - TRAINED_DICT = train_dict(SAMPLES, DICT_SIZE1) - ZstdDict(TRAINED_DICT.dict_content, False) + ZstdDict(TRAINED_DICT.dict_content, is_raw=False) self.assertNotEqual(TRAINED_DICT.dict_id, 0) self.assertGreater(len(TRAINED_DICT.dict_content), 0) @@ -1174,43 +1365,91 @@ def test_finalize_dict_arguments(self): def test_train_dict_c(self): # argument wrong type with self.assertRaises(TypeError): - _zstd._train_dict({}, (), 100) + _zstd.train_dict({}, (), 100) + with self.assertRaises(TypeError): + _zstd.train_dict(bytearray(), (), 100) + with self.assertRaises(TypeError): + _zstd.train_dict(b'', 99, 100) + with self.assertRaises(TypeError): + _zstd.train_dict(b'', [], 100) with self.assertRaises(TypeError): - _zstd._train_dict(b'', 99, 100) + _zstd.train_dict(b'', (), 100.1) with self.assertRaises(TypeError): - _zstd._train_dict(b'', (), 100.1) + _zstd.train_dict(b'', (99.1,), 100) + with self.assertRaises(ValueError): + _zstd.train_dict(b'abc', (4, -1), 100) + with self.assertRaises(ValueError): + _zstd.train_dict(b'abc', (2,), 100) + with self.assertRaises(ValueError): + _zstd.train_dict(b'', (99,), 100) # size > size_t with self.assertRaises(ValueError): - _zstd._train_dict(b'', (2**64+1,), 100) + _zstd.train_dict(b'', (2**1000,), 100) + with self.assertRaises(ValueError): + _zstd.train_dict(b'', (-2**1000,), 100) # dict_size <= 0 with self.assertRaises(ValueError): - _zstd._train_dict(b'', (), 0) + _zstd.train_dict(b'', (), 0) + with self.assertRaises(ValueError): + _zstd.train_dict(b'', (), -1) + + with self.assertRaises(ZstdError): + _zstd.train_dict(b'', (), 1) def test_finalize_dict_c(self): with self.assertRaises(TypeError): - _zstd._finalize_dict(1, 2, 3, 4, 5) + _zstd.finalize_dict(1, 2, 3, 4, 5) # argument wrong type with self.assertRaises(TypeError): - _zstd._finalize_dict({}, b'', (), 100, 5) + _zstd.finalize_dict({}, b'', (), 100, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(bytearray(TRAINED_DICT.dict_content), b'', (), 100, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, {}, (), 100, 5) with self.assertRaises(TypeError): - _zstd._finalize_dict(TRAINED_DICT.dict_content, {}, (), 100, 5) + _zstd.finalize_dict(TRAINED_DICT.dict_content, bytearray(), (), 100, 5) with self.assertRaises(TypeError): - _zstd._finalize_dict(TRAINED_DICT.dict_content, b'', 99, 100, 5) + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', 99, 100, 5) with self.assertRaises(TypeError): - _zstd._finalize_dict(TRAINED_DICT.dict_content, b'', (), 100.1, 5) + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', [], 100, 5) with self.assertRaises(TypeError): - _zstd._finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 5.1) + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100.1, 5) + with self.assertRaises(TypeError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 5.1) + + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'abc', (4, -1), 100, 5) + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'abc', (2,), 100, 5) + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (99,), 100, 5) # size > size_t with self.assertRaises(ValueError): - _zstd._finalize_dict(TRAINED_DICT.dict_content, b'', (2**64+1,), 100, 5) + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (2**1000,), 100, 5) + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (-2**1000,), 100, 5) # dict_size <= 0 with self.assertRaises(ValueError): - _zstd._finalize_dict(TRAINED_DICT.dict_content, b'', (), 0, 5) + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 0, 5) + with self.assertRaises(ValueError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), -1, 5) + with self.assertRaises(OverflowError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 2**1000, 5) + with self.assertRaises(OverflowError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), -2**1000, 5) + + with self.assertRaises(OverflowError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 2**1000) + with self.assertRaises(OverflowError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, -2**1000) + + with self.assertRaises(ZstdError): + _zstd.finalize_dict(TRAINED_DICT.dict_content, b'', (), 100, 5) def test_train_buffer_protocol_samples(self): def _nbytes(dat): @@ -1232,25 +1471,25 @@ def _nbytes(dat): # wrong size list with self.assertRaisesRegex(ValueError, "The samples size tuple doesn't match the concatenation's size"): - _zstd._train_dict(concatenation, tuple(wrong_size_lst), 100*_1K) + _zstd.train_dict(concatenation, tuple(wrong_size_lst), 100*_1K) # correct size list - _zstd._train_dict(concatenation, tuple(correct_size_lst), 3*_1K) + _zstd.train_dict(concatenation, tuple(correct_size_lst), 3*_1K) # wrong size list with self.assertRaisesRegex(ValueError, "The samples size tuple doesn't match the concatenation's size"): - _zstd._finalize_dict(TRAINED_DICT.dict_content, + _zstd.finalize_dict(TRAINED_DICT.dict_content, concatenation, tuple(wrong_size_lst), 300*_1K, 5) # correct size list - _zstd._finalize_dict(TRAINED_DICT.dict_content, + _zstd.finalize_dict(TRAINED_DICT.dict_content, concatenation, tuple(correct_size_lst), 300*_1K, 5) def test_as_prefix(self): # V1 V1 = THIS_FILE_BYTES - zd = ZstdDict(V1, True) + zd = ZstdDict(V1, is_raw=True) # V2 mid = len(V1) // 2 @@ -1266,7 +1505,7 @@ def test_as_prefix(self): self.assertEqual(decompress(dat, zd.as_prefix), V2) # use wrong prefix - zd2 = ZstdDict(SAMPLES[0], True) + zd2 = ZstdDict(SAMPLES[0], is_raw=True) try: decompressed = decompress(dat, zd2.as_prefix) except ZstdError: # expected @@ -1420,11 +1659,12 @@ def test_init_bad_mode(self): with self.assertRaises(ValueError): ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "rw") - with self.assertRaisesRegex(TypeError, r"NOT be CompressionParameter"): + with self.assertRaisesRegex(TypeError, + r"not be a CompressionParameter"): ZstdFile(io.BytesIO(), 'rb', options={CompressionParameter.compression_level:5}) with self.assertRaisesRegex(TypeError, - r"NOT be DecompressionParameter"): + r"not be a DecompressionParameter"): ZstdFile(io.BytesIO(), 'wb', options={DecompressionParameter.window_log_max:21}) @@ -1435,19 +1675,19 @@ def test_init_bad_check(self): with self.assertRaises(TypeError): ZstdFile(io.BytesIO(), "w", level='asd') # CHECK_UNKNOWN and anything above CHECK_ID_MAX should be invalid. - with self.assertRaises(ZstdError): + with self.assertRaises(ValueError): ZstdFile(io.BytesIO(), "w", options={999:9999}) - with self.assertRaises(ZstdError): + with self.assertRaises(ValueError): ZstdFile(io.BytesIO(), "w", options={CompressionParameter.window_log:99}) with self.assertRaises(TypeError): ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), "r", options=33) - with self.assertRaises(ValueError): + with self.assertRaises(OverflowError): ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), options={DecompressionParameter.window_log_max:2**31}) - with self.assertRaises(ZstdError): + with self.assertRaises(ValueError): ZstdFile(io.BytesIO(COMPRESSED_100_PLUS_32KB), options={444:333}) @@ -1463,7 +1703,7 @@ def test_init_close_fp(self): tmp_f.write(DAT_130K_C) filename = tmp_f.name - with self.assertRaises(ValueError): + with self.assertRaises(TypeError): ZstdFile(filename, options={'a':'b'}) # for PyPy @@ -1682,10 +1922,10 @@ def test_read_incomplete(self): # Trailing data isn't a valid compressed stream with ZstdFile(io.BytesIO(self.FRAME_42 + b'12345')) as f: - self.assertEqual(f.read(), self.DECOMPRESSED_42) + self.assertRaises(ZstdError, f.read) with ZstdFile(io.BytesIO(SKIPPABLE_FRAME + b'12345')) as f: - self.assertEqual(f.read(), b'') + self.assertRaises(ZstdError, f.read) def test_read_truncated(self): # Drop stream epilogue: 4 bytes checksum @@ -2134,6 +2374,8 @@ def read(self, size): f.read(100) with self.assertRaises(io.UnsupportedOperation): f.seek(100) + with self.assertRaises(io.UnsupportedOperation): + next(iter(f)) self.assertEqual(f.closed, True) with self.assertRaises(ValueError): f.readable() @@ -2428,15 +2670,18 @@ def test_buffer_protocol(self): class FreeThreadingMethodTests(unittest.TestCase): - @unittest.skipUnless(Py_GIL_DISABLED, 'this test can only possibly fail with GIL disabled') @threading_helper.reap_threads @threading_helper.requires_working_threading() def test_compress_locking(self): input = b'a'* (16*_1K) num_threads = 8 + # gh-136394: the first output of .compress() includes the frame header + # we run the first .compress() call outside of the threaded portion + # to make the test order-independent + comp = ZstdCompressor() - parts = [] + parts = [comp.compress(input, ZstdCompressor.FLUSH_BLOCK)] for _ in range(num_threads): res = comp.compress(input, ZstdCompressor.FLUSH_BLOCK) if res: @@ -2445,7 +2690,7 @@ def test_compress_locking(self): expected = b''.join(parts) + rest1 comp = ZstdCompressor() - output = [] + output = [comp.compress(input, ZstdCompressor.FLUSH_BLOCK)] def run_method(method, input_data, output_data): res = method(input_data, ZstdCompressor.FLUSH_BLOCK) if res: @@ -2465,7 +2710,6 @@ def run_method(method, input_data, output_data): actual = b''.join(output) + rest2 self.assertEqual(expected, actual) - @unittest.skipUnless(Py_GIL_DISABLED, 'this test can only possibly fail with GIL disabled') @threading_helper.reap_threads @threading_helper.requires_working_threading() def test_decompress_locking(self): @@ -2501,6 +2745,59 @@ def run_method(method, input_data, output_data): actual = b''.join(output) self.assertEqual(expected, actual) + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_compress_shared_dict(self): + num_threads = 8 + + def run_method(b): + level = threading.get_ident() % 4 + # sync threads to increase chance of contention on + # capsule storing dictionary levels + b.wait() + ZstdCompressor(level=level, + zstd_dict=TRAINED_DICT.as_digested_dict) + b.wait() + ZstdCompressor(level=level, + zstd_dict=TRAINED_DICT.as_undigested_dict) + b.wait() + ZstdCompressor(level=level, + zstd_dict=TRAINED_DICT.as_prefix) + threads = [] + + b = threading.Barrier(num_threads) + for i in range(num_threads): + thread = threading.Thread(target=run_method, args=(b,)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass + + @threading_helper.reap_threads + @threading_helper.requires_working_threading() + def test_decompress_shared_dict(self): + num_threads = 8 + + def run_method(b): + # sync threads to increase chance of contention on + # decompression dictionary + b.wait() + ZstdDecompressor(zstd_dict=TRAINED_DICT.as_digested_dict) + b.wait() + ZstdDecompressor(zstd_dict=TRAINED_DICT.as_undigested_dict) + b.wait() + ZstdDecompressor(zstd_dict=TRAINED_DICT.as_prefix) + threads = [] + + b = threading.Barrier(num_threads) + for i in range(num_threads): + thread = threading.Thread(target=run_method, args=(b,)) + + threads.append(thread) + + with threading_helper.start_threads(threads): + pass if __name__ == "__main__": diff --git a/Lib/test/typinganndata/fwdref_module.py b/Lib/test/typinganndata/fwdref_module.py new file mode 100644 index 000000000000000..7347a7a42455c22 --- /dev/null +++ b/Lib/test/typinganndata/fwdref_module.py @@ -0,0 +1,6 @@ +from typing import ForwardRef + +MyList = list[int] +MyDict = dict[str, 'MyList'] + +fw = ForwardRef('MyDict', module=__name__) diff --git a/Lib/test/xpickle_worker.py b/Lib/test/xpickle_worker.py new file mode 100644 index 000000000000000..1b49515123c6abd --- /dev/null +++ b/Lib/test/xpickle_worker.py @@ -0,0 +1,62 @@ +# This script is called by test_xpickle as a subprocess to load and dump +# pickles in a different Python version. +import os +import pickle +import struct +import sys + + +# This allows the xpickle worker to import picklecommon.py, which it needs +# since some of the pickle objects hold references to picklecommon.py. +test_mod_path = os.path.abspath(os.path.join(os.path.dirname(__file__), + 'picklecommon.py')) +if sys.version_info >= (3, 5): + import importlib.util + spec = importlib.util.spec_from_file_location('test.picklecommon', test_mod_path) + sys.modules['test'] = type(sys)('test') + test_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(test_module) + sys.modules['test.picklecommon'] = test_module +else: + test_module = type(sys)('test.picklecommon') + sys.modules['test.picklecommon'] = test_module + sys.modules['test'] = type(sys)('test') + with open(test_mod_path, 'rb') as f: + sources = f.read() + exec(sources, vars(test_module)) + +def read_exact(f, n): + buf = b'' + while len(buf) < n: + chunk = f.read(n - len(buf)) + if not chunk: + raise EOFError + buf += chunk + return buf + +in_stream = getattr(sys.stdin, 'buffer', sys.stdin) +out_stream = getattr(sys.stdout, 'buffer', sys.stdout) + +try: + while True: + size, = struct.unpack('!i', read_exact(in_stream, 4)) + if not size: + break + data = read_exact(in_stream, size) + protocol, = struct.unpack('!i', data[:4]) + obj = pickle.loads(data[4:]) + data = pickle.dumps(obj, protocol) + out_stream.write(struct.pack('!i', len(data)) + data) + out_stream.flush() +except Exception as exc: + # dump the exception to stdout and write to stderr, then exit + try: + data = pickle.dumps(exc) + out_stream.write(struct.pack('!i', -len(data)) + data) + out_stream.flush() + except Exception: + out_stream.write(struct.pack('!i', 0)) + out_stream.flush() + sys.stderr.write(repr(exc)) + sys.stderr.flush() + sys.exit(1) diff --git a/Lib/textwrap.py b/Lib/textwrap.py index 5ae439f5cd3b784..41366fbf443a4fc 100644 --- a/Lib/textwrap.py +++ b/Lib/textwrap.py @@ -211,7 +211,7 @@ def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width): # If we're allowed to break long words, then do so: put as much # of the next chunk onto the current line as will fit. - if self.break_long_words: + if self.break_long_words and space_left > 0: end = space_left chunk = reversed_chunks[-1] if self.break_on_hyphens and len(chunk) > space_left: diff --git a/Lib/threading.py b/Lib/threading.py index 39a1a7f4cdfda0c..c03b0b5370c9336 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -165,7 +165,7 @@ def __repr__(self): except KeyError: pass return "<%s %s.%s object owner=%r count=%d at %s>" % ( - "locked" if self._block.locked() else "unlocked", + "locked" if self.locked() else "unlocked", self.__class__.__module__, self.__class__.__qualname__, owner, @@ -244,7 +244,7 @@ def __exit__(self, t, v, tb): def locked(self): """Return whether this object is locked.""" - return self._count > 0 + return self._block.locked() # Internal methods used by condition variables @@ -660,7 +660,8 @@ def wait(self, timeout=None): (or fractions thereof). This method returns the internal flag on exit, so it will always return - True except if a timeout is given and the operation times out. + ``True`` except if a timeout is given and the operation times out, when + it will return ``False``. """ with self._cond: @@ -951,6 +952,8 @@ def _after_fork(self, new_ident=None): # This thread is alive. self._ident = new_ident assert self._os_thread_handle.ident == new_ident + if _HAVE_THREAD_NATIVE_ID: + self._set_native_id() else: # Otherwise, the thread is dead, Jim. _PyThread_AfterFork() # already marked our handle done. @@ -1419,7 +1422,7 @@ def __init__(self, dummy_thread): # the related _DummyThread will be kept forever! _thread_local_info._track_dummy_thread_ref = self - def __del__(self): + def __del__(self, _active_limbo_lock=_active_limbo_lock, _active=_active): with _active_limbo_lock: if _active.get(self._tident) is self._dummy_thread: _active.pop(self._tident, None) diff --git a/Lib/timeit.py b/Lib/timeit.py index e767f0187826df6..0451f66bdf1ca78 100644 --- a/Lib/timeit.py +++ b/Lib/timeit.py @@ -319,7 +319,7 @@ def main(args=None, *, _wrap_timer=None): callback = None if verbose: def callback(number, time_taken): - msg = "{num} loop{s} -> {secs:.{prec}g} secs" + msg = "{num} loop{s} -> {secs:.{prec}g} sec" plural = (number != 1) print(msg.format(num=number, s='s' if plural else '', secs=time_taken, prec=precision)) diff --git a/Lib/tkinter/__init__.py b/Lib/tkinter/__init__.py index a693b04870b9953..1c2afadd3cb234d 100644 --- a/Lib/tkinter/__init__.py +++ b/Lib/tkinter/__init__.py @@ -3,29 +3,29 @@ Tkinter provides classes which allow the display, positioning and control of widgets. Toplevel widgets are Tk and Toplevel. Other widgets are Frame, Label, Entry, Text, Canvas, Button, Radiobutton, -Checkbutton, Scale, Listbox, Scrollbar, OptionMenu, Spinbox +Checkbutton, Scale, Listbox, Scrollbar, OptionMenu, Spinbox, LabelFrame and PanedWindow. Properties of the widgets are specified with keyword arguments. -Keyword arguments have the same name as the corresponding resource +Keyword arguments have the same name as the corresponding options under Tk. Widgets are positioned with one of the geometry managers Place, Pack or Grid. These managers can be called with methods place, pack, grid available in every Widget. -Actions are bound to events by resources (e.g. keyword argument -command) or with the method bind. +Actions are bound to events by options (e.g. the command +keyword argument) or with the bind() method. Example (Hello, World): import tkinter from tkinter.constants import * tk = tkinter.Tk() frame = tkinter.Frame(tk, relief=RIDGE, borderwidth=2) -frame.pack(fill=BOTH,expand=1) +frame.pack(fill=BOTH, expand=1) label = tkinter.Label(frame, text="Hello, World") label.pack(fill=X, expand=1) -button = tkinter.Button(frame,text="Exit",command=tk.destroy) +button = tkinter.Button(frame, text="Exit", command=tk.destroy) button.pack(side=BOTTOM) tk.mainloop() """ @@ -817,7 +817,10 @@ def focus_get(self): the focus.""" name = self.tk.call('focus') if name == 'none' or not name: return None - return self._nametowidget(name) + try: + return self._nametowidget(name) + except KeyError: + return None def focus_displayof(self): """Return the widget which has currently the focus on the @@ -826,14 +829,20 @@ def focus_displayof(self): Return None if the application does not have the focus.""" name = self.tk.call('focus', '-displayof', self._w) if name == 'none' or not name: return None - return self._nametowidget(name) + try: + return self._nametowidget(name) + except KeyError: + return None def focus_lastfor(self): """Return the widget which would have the focus if top level for this widget gets the focus from the window manager.""" name = self.tk.call('focus', '-lastfor', self._w) if name == 'none' or not name: return None - return self._nametowidget(name) + try: + return self._nametowidget(name) + except KeyError: + return None def tk_focusFollowsMouse(self): """The widget under mouse will get automatically focus. Can not @@ -847,7 +856,7 @@ def tk_focusNext(self): The focus order first goes to the next child, then to the children of the child recursively and then to the next sibling which is higher in the stacking order. A - widget is omitted if it has the takefocus resource set + widget is omitted if it has the takefocus option set to 0.""" name = self.tk.call('tk_focusNext', self._w) if not name: return None @@ -1236,7 +1245,10 @@ def winfo_containing(self, rootX, rootY, displayof=0): + self._displayof(displayof) + (rootX, rootY) name = self.tk.call(args) if not name: return None - return self._nametowidget(name) + try: + return self._nametowidget(name) + except KeyError: + return None def winfo_depth(self): """Return the number of bits per pixel.""" @@ -1688,7 +1700,16 @@ def nametowidget(self, name): for n in name: if not n: break - w = w.children[n] + try: + w = w.children[n] + except KeyError: + # Menu clones (a menu used as a menubar or a cascade) get + # auto-generated names where each path component is the + # original name prefixed with one or more '#' clone markers. + # Map such a name back to the original widget. + if not n.startswith('#'): + raise + w = w.children[n.rsplit('#', 1)[-1]] return w @@ -1827,18 +1848,24 @@ def _configure(self, cmd, cnf, kw): # These used to be defined in Widget: def configure(self, cnf=None, **kw): - """Configure resources of a widget. + """Query or modify the configuration options of the widget. + + If no arguments are specified, return a dictionary describing + all of the available options for the widget. - The values for resources are specified as keyword - arguments. To get an overview about - the allowed keyword arguments call the method keys. + If an option name is specified, then return a tuple describing + the one named option. + + If one or more keyword arguments are specified or a dictionary + is specified, then modify the widget option(s) to have the given + value(s). """ return self._configure('configure', cnf, kw) config = configure def cget(self, key): - """Return the resource value for a KEY given as string.""" + """Return the current value of the configuration option.""" return self.tk.call(self._w, 'cget', '-' + key) __getitem__ = cget @@ -1847,7 +1874,7 @@ def __setitem__(self, key, value): self.configure({key: value}) def keys(self): - """Return a list of all resource names of this widget.""" + """Return a list of all option names of this widget.""" splitlist = self.tk.splitlist return [splitlist(x)[0][1:] for x in splitlist(self.tk.call(self._w, 'configure'))] @@ -1860,15 +1887,15 @@ def __repr__(self): return '<%s.%s object %s>' % ( self.__class__.__module__, self.__class__.__qualname__, self._w) - # Pack methods that apply to the master + # Pack methods that apply to the container widget _noarg_ = ['_noarg_'] def pack_propagate(self, flag=_noarg_): """Set or get the status for propagation of geometry information. - A boolean argument specifies whether the geometry information - of the slaves will determine the size of this widget. If no argument - is given the current setting will be returned. + A boolean argument specifies whether the size of this container will + be determined by the geometry information of its content. + If no argument is given the current setting will be returned. """ if flag is Misc._noarg_: return self._getboolean(self.tk.call( @@ -1879,28 +1906,28 @@ def pack_propagate(self, flag=_noarg_): propagate = pack_propagate def pack_slaves(self): - """Return a list of all slaves of this widget - in its packing order.""" + """Returns a list of all of the content widgets in the packing order + for this container.""" return [self._nametowidget(x) for x in self.tk.splitlist( self.tk.call('pack', 'slaves', self._w))] slaves = pack_slaves - # Place method that applies to the master + # Place method that applies to the container widget def place_slaves(self): - """Return a list of all slaves of this widget - in its packing order.""" + """Returns a list of all the content widgets for which this widget is + the container.""" return [self._nametowidget(x) for x in self.tk.splitlist( self.tk.call( 'place', 'slaves', self._w))] - # Grid methods that apply to the master + # Grid methods that apply to the container widget def grid_anchor(self, anchor=None): # new in Tk 8.5 """The anchor value controls how to place the grid within the - master when no row/column has any weight. + container widget when no row/column has any weight. The default anchor is nw.""" self.tk.call('grid', 'anchor', self._w, anchor) @@ -1917,7 +1944,7 @@ def grid_bbox(self, column=None, row=None, col2=None, row2=None): starts at that cell. The returned integers specify the offset of the upper left - corner in the master widget and the width and height. + corner in the container widget and the width and height. """ args = ('grid', 'bbox', self._w) if column is not None and row is not None: @@ -1966,7 +1993,7 @@ def _grid_configure(self, command, index, cnf, kw): def grid_columnconfigure(self, index, cnf={}, **kw): """Configure column INDEX of a grid. - Valid resources are minsize (minimum size of the column), + Valid options are minsize (minimum size of the column), weight (how much does additional space propagate to this column) and pad (how much space to let additionally).""" return self._grid_configure('columnconfigure', index, cnf, kw) @@ -1975,7 +2002,7 @@ def grid_columnconfigure(self, index, cnf={}, **kw): def grid_location(self, x, y): """Return a tuple of column and row which identify the cell - at which the pixel at position X and Y inside the master + at which the pixel at position X and Y inside the container widget is located.""" return self._getints( self.tk.call( @@ -1984,9 +2011,9 @@ def grid_location(self, x, y): def grid_propagate(self, flag=_noarg_): """Set or get the status for propagation of geometry information. - A boolean argument specifies whether the geometry information - of the slaves will determine the size of this widget. If no argument - is given, the current setting will be returned. + A boolean argument specifies whether the size of this container will + be determined by the geometry information of its content. + If no argument is given the current setting will be returned. """ if flag is Misc._noarg_: return self._getboolean(self.tk.call( @@ -1997,7 +2024,7 @@ def grid_propagate(self, flag=_noarg_): def grid_rowconfigure(self, index, cnf={}, **kw): """Configure row INDEX of a grid. - Valid resources are minsize (minimum size of the row), + Valid options are minsize (minimum size of the row), weight (how much does additional space propagate to this row) and pad (how much space to let additionally).""" return self._grid_configure('rowconfigure', index, cnf, kw) @@ -2012,8 +2039,13 @@ def grid_size(self): size = grid_size def grid_slaves(self, row=None, column=None): - """Return a list of all slaves of this widget - in its packing order.""" + """Returns a list of the content widgets. + + If no arguments are supplied, a list of all of the content in this + container widget is returned, most recently managed first. + If ROW or COLUMN is specified, only the content in the row or + column is returned. + """ args = () if row is not None: args = args + ('-row', row) @@ -2333,7 +2365,7 @@ def wm_iconphoto(self, default=False, *args): # new in Tk 8.5 def wm_iconposition(self, x=None, y=None): """Set the position of the icon of this widget to X and Y. Return - a tuple of the current values of X and X if None is given.""" + a tuple of the current values of X and Y if None is given.""" return self._getints(self.tk.call( 'wm', 'iconposition', self._w, x, y)) @@ -2599,8 +2631,8 @@ def pack_configure(self, cnf={}, **kw): before=widget - pack it before you will pack widget expand=bool - expand widget if parent size grows fill=NONE or X or Y or BOTH - fill widget if widget grows - in=master - use master to contain this widget - in_=master - see 'in' option description + in=container - use the container widget to contain this widget + in_=container - see 'in' option description ipadx=amount - add internal padding in x direction ipady=amount - add internal padding in y direction padx=amount - add padding in x direction @@ -2639,25 +2671,31 @@ class Place: def place_configure(self, cnf={}, **kw): """Place a widget in the parent widget. Use as options: - in=master - master relative to which the widget is placed - in_=master - see 'in' option description - x=amount - locate anchor of this widget at position x of master - y=amount - locate anchor of this widget at position y of master + in=container - the container widget relative to which this widget is + placed + in_=container - see 'in' option description + x=amount - locate anchor of this widget at position x of the + container widget + y=amount - locate anchor of this widget at position y of the + container widget relx=amount - locate anchor of this widget between 0.0 and 1.0 - relative to width of master (1.0 is right edge) + relative to width of the container widget (1.0 is + right edge) rely=amount - locate anchor of this widget between 0.0 and 1.0 - relative to height of master (1.0 is bottom edge) - anchor=NSEW (or subset) - position anchor according to given direction + relative to height of the container widget (1.0 is + bottom edge) + anchor=NSEW (or subset) - position anchor according to given + direction width=amount - width of this widget in pixel height=amount - height of this widget in pixel relwidth=amount - width of this widget between 0.0 and 1.0 - relative to width of master (1.0 is the same width - as the master) + relative to width of the container widget (1.0 is + the same width as the container widget) relheight=amount - height of this widget between 0.0 and 1.0 - relative to height of master (1.0 is the same - height as the master) + relative to height of the container widget (1.0 + is the same height as the container widget) bordermode="inside" or "outside" - whether to take border width of - master widget into account + the container widget into account """ self.tk.call( ('place', 'configure', self._w) @@ -2693,8 +2731,8 @@ def grid_configure(self, cnf={}, **kw): """Position a widget in the parent widget in a grid. Use as options: column=number - use cell identified with given column (starting with 0) columnspan=number - this widget will span several columns - in=master - use master to contain this widget - in_=master - see 'in' option description + in=container - use the container widget to contain this widget + in_=container - see 'in' option description ipadx=amount - add internal padding in x direction ipady=amount - add internal padding in y direction padx=amount - add padding in x direction @@ -2817,10 +2855,11 @@ class Toplevel(BaseWidget, Wm): def __init__(self, master=None, cnf={}, **kw): """Construct a toplevel widget with the parent MASTER. - Valid resource names: background, bd, bg, borderwidth, class, - colormap, container, cursor, height, highlightbackground, - highlightcolor, highlightthickness, menu, relief, screen, takefocus, - use, visual, width.""" + Valid option names: background, backgroundimage (Tk 9.0+), bd, bg, + bgimg (Tk 9.0+), borderwidth, class, colormap, container, + cursor, height, highlightbackground, highlightcolor, + highlightthickness, menu, padx, pady, relief, screen, + takefocus, tile (Tk 9.0+), use, visual, width.""" if kw: cnf = _cnfmerge((cnf, kw)) extra = () @@ -2894,7 +2933,7 @@ class Canvas(Widget, XView, YView): def __init__(self, master=None, cnf={}, **kw): """Construct a canvas widget with the parent MASTER. - Valid resource names: background, bd, bg, borderwidth, closeenough, + Valid option names: background, bd, bg, borderwidth, closeenough, confine, cursor, height, highlightbackground, highlightcolor, highlightthickness, insertbackground, insertborderwidth, insertofftime, insertontime, insertwidth, offset, relief, @@ -2940,7 +2979,7 @@ def addtag_withtag(self, newtag, tagOrId): """Add tag NEWTAG to all items with TAGORID.""" self.addtag(newtag, 'withtag', tagOrId) - def bbox(self, *args): + def bbox(self, *args): # overrides Misc.bbox """Return a tuple of X1,Y1,X2,Y2 coordinates for a rectangle which encloses all items with tags specified as arguments.""" return self._getints( @@ -3079,7 +3118,7 @@ def find_withtag(self, tagOrId): """Return all items with TAGORID.""" return self.find('withtag', tagOrId) - def focus(self, *args): + def focus(self, *args): # overrides Misc.focus """Set focus to the first item specified in ARGS.""" return self.tk.call((self._w, 'focus') + args) @@ -3103,16 +3142,14 @@ def insert(self, *args): self.tk.call((self._w, 'insert') + args) def itemcget(self, tagOrId, option): - """Return the resource value for an OPTION for item TAGORID.""" + """Return the value of OPTION for item TAGORID.""" return self.tk.call( (self._w, 'itemcget') + (tagOrId, '-'+option)) def itemconfigure(self, tagOrId, cnf=None, **kw): - """Configure resources of an item TAGORID. + """Query or modify the configuration options of an item TAGORID. - The values for resources are specified as keyword - arguments. To get an overview about - the allowed keyword arguments call the method without arguments. + Similar to configure() except that it applies to the specified item. """ return self._configure(('itemconfigure', tagOrId), cnf, kw) @@ -3127,7 +3164,7 @@ def tag_lower(self, *args): (optional below another item).""" self.tk.call((self._w, 'lower') + args) - lower = tag_lower + lower = tag_lower # overrides Misc.lower def move(self, *args): """Move an item TAGORID given in ARGS.""" @@ -3155,7 +3192,7 @@ def tag_raise(self, *args): (optional above another item).""" self.tk.call((self._w, 'raise') + args) - lift = tkraise = tag_raise + lift = tkraise = tag_raise # overrides Misc.tkraise def scale(self, *args): """Scale item TAGORID with XORIGIN, YORIGIN, XSCALE, YSCALE.""" @@ -3204,13 +3241,14 @@ class Checkbutton(Widget): def __init__(self, master=None, cnf={}, **kw): """Construct a checkbutton widget with the parent MASTER. - Valid resource names: activebackground, activeforeground, anchor, - background, bd, bg, bitmap, borderwidth, command, cursor, - disabledforeground, fg, font, foreground, height, - highlightbackground, highlightcolor, highlightthickness, image, - indicatoron, justify, offvalue, onvalue, padx, pady, relief, - selectcolor, selectimage, state, takefocus, text, textvariable, - underline, variable, width, wraplength.""" + Valid option names: activebackground, activeforeground, anchor, + background, bd, bg, bitmap, borderwidth, command, compound, + cursor, disabledforeground, fg, font, foreground, height, + highlightbackground, highlightcolor, highlightthickness, + image, indicatoron, justify, offrelief, offvalue, onvalue, + overrelief, padx, pady, relief, selectcolor, selectimage, + state, takefocus, text, textvariable, tristateimage, + tristatevalue, underline, variable, width, wraplength.""" Widget.__init__(self, master, 'checkbutton', cnf, kw) def _setup(self, master, cnf): @@ -3235,7 +3273,7 @@ def flash(self): self.tk.call(self._w, 'flash') def invoke(self): - """Toggle the button and invoke a command if given as resource.""" + """Toggle the button and invoke a command if given as option.""" return self.tk.call(self._w, 'invoke') def select(self): @@ -3253,14 +3291,16 @@ class Entry(Widget, XView): def __init__(self, master=None, cnf={}, **kw): """Construct an entry widget with the parent MASTER. - Valid resource names: background, bd, bg, borderwidth, cursor, - exportselection, fg, font, foreground, highlightbackground, - highlightcolor, highlightthickness, insertbackground, - insertborderwidth, insertofftime, insertontime, insertwidth, - invalidcommand, invcmd, justify, relief, selectbackground, - selectborderwidth, selectforeground, show, state, takefocus, - textvariable, validate, validatecommand, vcmd, width, - xscrollcommand.""" + Valid option names: background, bd, bg, borderwidth, cursor, + disabledbackground, disabledforeground, exportselection, fg, + font, foreground, highlightbackground, highlightcolor, + highlightthickness, insertbackground, insertborderwidth, + insertofftime, insertontime, insertwidth, invalidcommand, + invcmd, justify, locale (Tk 9.1+), placeholder (Tk 9.0+), + placeholderforeground (Tk 9.0+), readonlybackground, relief, + selectbackground, selectborderwidth, selectforeground, show, + state, takefocus, textvariable, validate, validatecommand, + vcmd, width, xscrollcommand.""" Widget.__init__(self, master, 'entry', cnf, kw) def delete(self, first, last=None): @@ -3300,7 +3340,7 @@ def selection_adjust(self, index): select_adjust = selection_adjust - def selection_clear(self): + def selection_clear(self): # overrides Misc.selection_clear """Clear the selection if it is in this widget.""" self.tk.call(self._w, 'selection', 'clear') @@ -3339,9 +3379,11 @@ class Frame(Widget): def __init__(self, master=None, cnf={}, **kw): """Construct a frame widget with the parent MASTER. - Valid resource names: background, bd, bg, borderwidth, class, - colormap, container, cursor, height, highlightbackground, - highlightcolor, highlightthickness, relief, takefocus, visual, width.""" + Valid option names: background, backgroundimage (Tk 9.0+), bd, bg, + bgimg (Tk 9.0+), borderwidth, class, colormap, container, + cursor, height, highlightbackground, highlightcolor, + highlightthickness, padx, pady, relief, takefocus, tile (Tk + 9.0+), visual, width.""" cnf = _cnfmerge((cnf, kw)) extra = () if 'class_' in cnf: @@ -3371,7 +3413,8 @@ def __init__(self, master=None, cnf={}, **kw): WIDGET-SPECIFIC OPTIONS - height, state, width + compound, height, state, + textangle (Tk 9.1+), width """ Widget.__init__(self, master, 'label', cnf, kw) @@ -3383,18 +3426,21 @@ class Listbox(Widget, XView, YView): def __init__(self, master=None, cnf={}, **kw): """Construct a listbox widget with the parent MASTER. - Valid resource names: background, bd, bg, borderwidth, cursor, - exportselection, fg, font, foreground, height, highlightbackground, - highlightcolor, highlightthickness, relief, selectbackground, - selectborderwidth, selectforeground, selectmode, setgrid, takefocus, - width, xscrollcommand, yscrollcommand, listvariable.""" + Valid option names: activestyle, background, bd, bg, borderwidth, + cursor, disabledforeground, exportselection, fg, font, + foreground, height, highlightbackground, highlightcolor, + highlightthickness, inactiveselectbackground (Tk 9.1+), + inactiveselectforeground (Tk 9.1+), justify, listvariable, + relief, selectbackground, selectborderwidth, selectforeground, + selectmode, setgrid, state, takefocus, width, xscrollcommand, + yscrollcommand.""" Widget.__init__(self, master, 'listbox', cnf, kw) def activate(self, index): """Activate item identified by INDEX.""" self.tk.call(self._w, 'activate', index) - def bbox(self, index): + def bbox(self, index): # overrides Misc.bbox """Return a tuple of X1,Y1,X2,Y2 coordinates for a rectangle which encloses the item identified by the given index.""" return self._getints(self.tk.call(self._w, 'bbox', index)) or None @@ -3450,7 +3496,7 @@ def selection_anchor(self, index): select_anchor = selection_anchor - def selection_clear(self, first, last=None): + def selection_clear(self, first, last=None): # overrides Misc.selection_clear """Clear the selection from FIRST to LAST (included).""" self.tk.call(self._w, 'selection', 'clear', first, last) @@ -3471,23 +3517,20 @@ def selection_set(self, first, last=None): select_set = selection_set - def size(self): + def size(self): # overrides Misc.size """Return the number of elements in the listbox.""" return self.tk.getint(self.tk.call(self._w, 'size')) def itemcget(self, index, option): - """Return the resource value for an ITEM and an OPTION.""" + """Return the value of OPTION for item at INDEX.""" return self.tk.call( (self._w, 'itemcget') + (index, '-'+option)) def itemconfigure(self, index, cnf=None, **kw): - """Configure resources of an ITEM. + """Query or modify the configuration options of an item at INDEX. - The values for resources are specified as keyword arguments. - To get an overview about the allowed keyword arguments - call the method without arguments. - Valid resource names: background, bg, foreground, fg, - selectbackground, selectforeground.""" + Similar to configure() except that it applies to the specified item. + """ return self._configure(('itemconfigure', index), cnf, kw) itemconfig = itemconfigure @@ -3499,8 +3542,9 @@ class Menu(Widget): def __init__(self, master=None, cnf={}, **kw): """Construct menu widget with the parent MASTER. - Valid resource names: activebackground, activeborderwidth, - activeforeground, background, bd, bg, borderwidth, cursor, + Valid option names: activebackground, activeborderwidth, + activeforeground, activerelief (Tk 9.0+), background, bd, bg, + borderwidth, cursor, disabledforeground, fg, font, foreground, postcommand, relief, selectcolor, takefocus, tearoff, tearoffcommand, title, type.""" Widget.__init__(self, master, 'menu', cnf, kw) @@ -3580,11 +3624,15 @@ def delete(self, index1, index2=None): self.tk.call(self._w, 'delete', index1, index2) def entrycget(self, index, option): - """Return the resource value of a menu item for OPTION at INDEX.""" + """Return the value of OPTION for a menu item at INDEX.""" return self.tk.call(self._w, 'entrycget', index, '-' + option) def entryconfigure(self, index, cnf=None, **kw): - """Configure a menu item at INDEX.""" + """Query or modify the configuration options of a menu item at INDEX. + + Similar to configure() except that it applies to the specified + menu item. + """ return self._configure(('entryconfigure', index), cnf, kw) entryconfig = entryconfigure @@ -3626,6 +3674,14 @@ class Menubutton(Widget): """Menubutton widget, obsolete since Tk8.0.""" def __init__(self, master=None, cnf={}, **kw): + """Construct a menubutton widget with the parent MASTER. + + Valid option names: activebackground, activeforeground, anchor, + background, bd, bg, bitmap, borderwidth, compound, cursor, + direction, disabledforeground, fg, font, foreground, height, + highlightbackground, highlightcolor, highlightthickness, + image, indicatoron, justify, menu, padx, pady, relief, state, + takefocus, text, textvariable, underline, width, wraplength.""" Widget.__init__(self, master, 'menubutton', cnf, kw) @@ -3633,6 +3689,12 @@ class Message(Widget): """Message widget to display multiline text. Obsolete since Label does it too.""" def __init__(self, master=None, cnf={}, **kw): + """Construct a message widget with the parent MASTER. + + Valid option names: anchor, aspect, background, bd, bg, borderwidth, + cursor, fg, font, foreground, highlightbackground, + highlightcolor, highlightthickness, justify, padx, pady, + relief, takefocus, text, textvariable, width.""" Widget.__init__(self, master, 'message', cnf, kw) @@ -3642,13 +3704,14 @@ class Radiobutton(Widget): def __init__(self, master=None, cnf={}, **kw): """Construct a radiobutton widget with the parent MASTER. - Valid resource names: activebackground, activeforeground, anchor, - background, bd, bg, bitmap, borderwidth, command, cursor, - disabledforeground, fg, font, foreground, height, - highlightbackground, highlightcolor, highlightthickness, image, - indicatoron, justify, padx, pady, relief, selectcolor, selectimage, - state, takefocus, text, textvariable, underline, value, variable, - width, wraplength.""" + Valid option names: activebackground, activeforeground, anchor, + background, bd, bg, bitmap, borderwidth, command, compound, + cursor, disabledforeground, fg, font, foreground, height, + highlightbackground, highlightcolor, highlightthickness, + image, indicatoron, justify, offrelief, overrelief, padx, + pady, relief, selectcolor, selectimage, state, takefocus, + text, textvariable, tristateimage, tristatevalue, underline, + value, variable, width, wraplength.""" Widget.__init__(self, master, 'radiobutton', cnf, kw) def deselect(self): @@ -3661,7 +3724,7 @@ def flash(self): self.tk.call(self._w, 'flash') def invoke(self): - """Toggle the button and invoke a command if given as resource.""" + """Toggle the button and invoke a command if given as option.""" return self.tk.call(self._w, 'invoke') def select(self): @@ -3675,7 +3738,7 @@ class Scale(Widget): def __init__(self, master=None, cnf={}, **kw): """Construct a scale widget with the parent MASTER. - Valid resource names: activebackground, background, bigincrement, bd, + Valid option names: activebackground, background, bigincrement, bd, bg, borderwidth, command, cursor, digits, fg, font, foreground, from, highlightbackground, highlightcolor, highlightthickness, label, length, orient, relief, repeatdelay, repeatinterval, resolution, @@ -3714,7 +3777,7 @@ class Scrollbar(Widget): def __init__(self, master=None, cnf={}, **kw): """Construct a scrollbar widget with the parent MASTER. - Valid resource names: activebackground, activerelief, + Valid option names: activebackground, activerelief, background, bd, bg, borderwidth, command, cursor, elementborderwidth, highlightbackground, highlightcolor, highlightthickness, jump, orient, @@ -3779,14 +3842,16 @@ def __init__(self, master=None, cnf={}, **kw): WIDGET-SPECIFIC OPTIONS - autoseparators, height, maxundo, - spacing1, spacing2, spacing3, - state, tabs, undo, width, wrap, + autoseparators, blockcursor, endline, + height, inactiveselectbackground, + insertunfocussed, locale (Tk 9.1+), maxundo, + spacing1, spacing2, spacing3, startline, + state, tabs, tabstyle, undo, width, wrap, """ Widget.__init__(self, master, 'text', cnf, kw) - def bbox(self, index): + def bbox(self, index): # overrides Misc.bbox """Return a tuple of (x,y,width,height) which gives the bounding box of the visible part of the character at the given index.""" return self._getints( @@ -3958,7 +4023,11 @@ def image_cget(self, index, option): return self.tk.call(self._w, "image", "cget", index, option) def image_configure(self, index, cnf=None, **kw): - """Configure an embedded image at INDEX.""" + """Query or modify the configuration options of an embedded image at INDEX. + + Similar to configure() except that it applies to the specified + embedded image. + """ return self._configure(('image', 'configure', index), cnf, kw) def image_create(self, index, cnf={}, **kw): @@ -3967,7 +4036,7 @@ def image_create(self, index, cnf={}, **kw): self._w, "image", "create", index, *self._options(cnf, kw)) - def image_names(self): + def image_names(self): # overrides Misc.image_names """Return all names of embedded images in this widget.""" return self.tk.call(self._w, "image", "names") @@ -4096,7 +4165,10 @@ def tag_cget(self, tagName, option): return self.tk.call(self._w, 'tag', 'cget', tagName, option) def tag_configure(self, tagName, cnf=None, **kw): - """Configure a tag TAGNAME.""" + """Query or modify the configuration options of a tag TAGNAME. + + Similar to configure() except that it applies to the specified tag. + """ return self._configure(('tag', 'configure', tagName), cnf, kw) tag_config = tag_configure @@ -4154,7 +4226,11 @@ def window_cget(self, index, option): return self.tk.call(self._w, 'window', 'cget', index, option) def window_configure(self, index, cnf=None, **kw): - """Configure an embedded window at INDEX.""" + """Query or modify the configuration options of an embedded window at INDEX. + + Similar to configure() except that it applies to the specified + embedded window. + """ return self._configure(('window', 'configure', index), cnf, kw) window_config = window_configure @@ -4194,7 +4270,7 @@ class OptionMenu(Menubutton): def __init__(self, master, variable, value, *values, **kwargs): """Construct an optionmenu widget with the parent MASTER, with - the resource textvariable set to VARIABLE, the initially selected + the option textvariable set to VARIABLE, the initially selected value VALUE, the other menu values VALUES and an additional keyword argument command.""" kw = {"borderwidth": 2, "textvariable": variable, @@ -4296,7 +4372,7 @@ class PhotoImage(Image): def __init__(self, name=None, cnf={}, master=None, **kw): """Create an image with NAME. - Valid resource names: data, format, file, gamma, height, palette, + Valid option names: data, format, file, gamma, height, palette, width.""" Image.__init__(self, 'photo', name, cnf, master, **kw) @@ -4559,7 +4635,7 @@ class BitmapImage(Image): def __init__(self, name=None, cnf={}, master=None, **kw): """Create a bitmap with NAME. - Valid resource names: background, data, file, foreground, maskdata, maskfile.""" + Valid option names: background, data, file, foreground, maskdata, maskfile.""" Image.__init__(self, 'bitmap', name, cnf, master, **kw) @@ -4598,14 +4674,17 @@ def __init__(self, master=None, cnf={}, **kw): buttondownrelief, buttonuprelief, command, disabledbackground, disabledforeground, format, from, - invalidcommand, increment, + invalidcommand, invcmd, increment, + locale (Tk 9.1+), + placeholder (Tk 9.0+), + placeholderforeground (Tk 9.0+), readonlybackground, state, to, - validate, validatecommand values, + validate, validatecommand, vcmd, values, width, wrap, """ Widget.__init__(self, master, 'spinbox', cnf, kw) - def bbox(self, index): + def bbox(self, index): # overrides Misc.bbox """Return a tuple of X1,Y1,X2,Y2 coordinates for a rectangle which encloses the character given by index. @@ -4714,7 +4793,7 @@ def selection_adjust(self, index): """ return self.selection("adjust", index) - def selection_clear(self): + def selection_clear(self): # overrides Misc.selection_clear """Clear the selection If the selection isn't in this widget then the @@ -4789,8 +4868,9 @@ def __init__(self, master=None, cnf={}, **kw): WIDGET-SPECIFIC OPTIONS handlepad, handlesize, opaqueresize, - sashcursor, sashpad, sashrelief, - sashwidth, showhandle, + proxybackground, proxyborderwidth, + proxyrelief, sashcursor, sashpad, + sashrelief, sashwidth, showhandle, """ Widget.__init__(self, master, 'panedwindow', cnf, kw) @@ -4811,7 +4891,7 @@ def remove(self, child): """ self.tk.call(self._w, 'forget', child) - forget = remove + forget = remove # overrides Pack.forget def identify(self, x, y): """Identify the panedwindow component at point x, y @@ -4877,26 +4957,17 @@ def sash_place(self, index, x, y): return self.sash("place", index, x, y) def panecget(self, child, option): - """Query a management option for window. - - Option may be any value allowed by the paneconfigure subcommand - """ + """Return the value of option for a child window.""" return self.tk.call( (self._w, 'panecget') + (child, '-'+option)) def paneconfigure(self, tagOrId, cnf=None, **kw): - """Query or modify the management options for window. - - If no option is specified, returns a list describing all - of the available options for pathName. If option is - specified with no value, then the command returns a list - describing the one named option (this list will be identical - to the corresponding sublist of the value returned if no - option is specified). If one or more option-value pairs are - specified, then the command modifies the given widget - option(s) to have the given value(s); in this case the - command returns an empty string. The following options - are supported: + """Query or modify the configuration options for a child window. + + Similar to configure() except that it applies to the specified + window. + + The following options are supported: after window Insert the window after the window specified. window diff --git a/Lib/tkinter/colorchooser.py b/Lib/tkinter/colorchooser.py index e2fb69dba927633..4085bde1637a79f 100644 --- a/Lib/tkinter/colorchooser.py +++ b/Lib/tkinter/colorchooser.py @@ -1,3 +1,5 @@ +"""Interface to the native Tk color selection dialog.""" + # tk common color chooser dialogue # # this module provides an interface to the native color dialogue diff --git a/Lib/tkinter/commondialog.py b/Lib/tkinter/commondialog.py index 86f5387e001bf28..408cedc345def63 100644 --- a/Lib/tkinter/commondialog.py +++ b/Lib/tkinter/commondialog.py @@ -1,3 +1,5 @@ +"""Base class for the Tk common dialogs.""" + # base class for tk common dialogues # # this module provides a base class for accessing the common diff --git a/Lib/tkinter/dialog.py b/Lib/tkinter/dialog.py index 36ae6c277cb66ac..ef005af0d5a19c0 100644 --- a/Lib/tkinter/dialog.py +++ b/Lib/tkinter/dialog.py @@ -1,5 +1,7 @@ # dialog.py -- Tkinter interface to the tk_dialog script. +"""Classic Tk dialog box, wrapping the tk_dialog script.""" + from tkinter import _cnfmerge, Widget, TclError, Button, Pack __all__ = ["Dialog"] @@ -8,6 +10,8 @@ class Dialog(Widget): + """A modal dialog box built from the classic (non-themed) Tk widgets.""" + def __init__(self, master=None, cnf={}, **kw): cnf = _cnfmerge((cnf, kw)) self.widgetName = '__dialog__' @@ -21,7 +25,8 @@ def __init__(self, master=None, cnf={}, **kw): try: Widget.destroy(self) except TclError: pass - def destroy(self): pass + def destroy(self): + """Do nothing; the dialog window is already destroyed.""" def _test(): diff --git a/Lib/tkinter/font.py b/Lib/tkinter/font.py index 3e24e28ef58cde9..569c31222136e1b 100644 --- a/Lib/tkinter/font.py +++ b/Lib/tkinter/font.py @@ -1,3 +1,5 @@ +"""Utilities to help work with fonts in Tkinter.""" + # Tkinter font wrapper # # written by Fredrik Lundh, February 1998 @@ -72,8 +74,15 @@ def __init__(self, root=None, font=None, name=None, exists=False, root = tkinter._get_default_root('use font') tk = getattr(root, 'tk', root) if font: - # get actual settings corresponding to the given font - font = tk.splitlist(tk.call("font", "actual", font)) + # start from the settings of the given font + try: + # a named font: copy its options, preserving the size, + # which can be negative (specified in pixels) + font = tk.splitlist(tk.call("font", "configure", font)) + except tkinter.TclError: + # a font description: resolve it ("font configure" only + # accepts a font name); this loses a size in pixels + font = tk.splitlist(tk.call("font", "actual", font)) else: font = self._set(options) if not name: @@ -124,7 +133,7 @@ def __del__(self): def copy(self): "Return a distinct copy of the current font" - return Font(self._tk, **self.actual()) + return Font(self._tk, self.name) def actual(self, option=None, displayof=None): "Return actual font attributes" diff --git a/Lib/tkinter/messagebox.py b/Lib/tkinter/messagebox.py index 5f0343b660c68c5..f6848142a9aff62 100644 --- a/Lib/tkinter/messagebox.py +++ b/Lib/tkinter/messagebox.py @@ -1,3 +1,5 @@ +"""Interface to the standard Tk message boxes.""" + # tk common message boxes # # this module provides an interface to the native message boxes @@ -99,24 +101,24 @@ def showerror(title=None, message=None, **options): def askquestion(title=None, message=None, **options): - "Ask a question" + "Ask a question; return the symbolic name of the selected button" return _show(title, message, QUESTION, YESNO, **options) def askokcancel(title=None, message=None, **options): - "Ask if operation should proceed; return true if the answer is ok" + "Ask if operation should proceed; return True if the answer is ok" s = _show(title, message, QUESTION, OKCANCEL, **options) return s == OK def askyesno(title=None, message=None, **options): - "Ask a question; return true if the answer is yes" + "Ask a question; return True if the answer is yes" s = _show(title, message, QUESTION, YESNO, **options) return s == YES def askyesnocancel(title=None, message=None, **options): - "Ask a question; return true if the answer is yes, None if cancelled." + "Ask a question; return True if the answer is yes, None if cancelled" s = _show(title, message, QUESTION, YESNOCANCEL, **options) # s might be a Tcl index object, so convert it to a string s = str(s) @@ -126,7 +128,7 @@ def askyesnocancel(title=None, message=None, **options): def askretrycancel(title=None, message=None, **options): - "Ask if operation should be retried; return true if the answer is yes" + "Ask if operation should be retried; return True if the answer is retry" s = _show(title, message, WARNING, RETRYCANCEL, **options) return s == RETRY diff --git a/Lib/tkinter/scrolledtext.py b/Lib/tkinter/scrolledtext.py index 4f9a8815b6184b2..8dcead5e31930ea 100644 --- a/Lib/tkinter/scrolledtext.py +++ b/Lib/tkinter/scrolledtext.py @@ -23,7 +23,7 @@ def __init__(self, master=None, **kw): self.vbar = Scrollbar(self.frame) self.vbar.pack(side=RIGHT, fill=Y) - kw.update({'yscrollcommand': self.vbar.set}) + kw['yscrollcommand'] = self.vbar.set Text.__init__(self, self.frame, **kw) self.pack(side=LEFT, fill=BOTH, expand=True) self.vbar['command'] = self.yview diff --git a/Lib/tkinter/simpledialog.py b/Lib/tkinter/simpledialog.py index 6e5b025a9f9d7df..81edd77e0bd2446 100644 --- a/Lib/tkinter/simpledialog.py +++ b/Lib/tkinter/simpledialog.py @@ -8,7 +8,7 @@ # fredrik@pythonware.com # http://www.pythonware.com # -"""This modules handles dialog boxes. +"""This module handles dialog boxes. It contains the following public symbols: diff --git a/Lib/tkinter/ttk.py b/Lib/tkinter/ttk.py index c0cf1e787fa9adf..e002ac8cfd0b641 100644 --- a/Lib/tkinter/ttk.py +++ b/Lib/tkinter/ttk.py @@ -581,8 +581,8 @@ def __init__(self, master=None, **kw): STANDARD OPTIONS - class, compound, cursor, image, state, style, takefocus, - text, textvariable, underline, width + class, compound, cursor, image, justify (Tk 9.0+), padding, + state, style, takefocus, text, textvariable, underline, width WIDGET-SPECIFIC OPTIONS @@ -604,8 +604,8 @@ def __init__(self, master=None, **kw): STANDARD OPTIONS - class, compound, cursor, image, state, style, takefocus, - text, textvariable, underline, width + class, compound, cursor, image, justify (Tk 9.0+), padding, + state, style, takefocus, text, textvariable, underline, width WIDGET-SPECIFIC OPTIONS @@ -638,8 +638,10 @@ def __init__(self, master=None, widget=None, **kw): WIDGET-SPECIFIC OPTIONS - exportselection, invalidcommand, justify, show, state, - textvariable, validate, validatecommand, width + background, exportselection, font, foreground, invalidcommand, + justify, locale (Tk 9.1+), placeholder (Tk 9.0+), + placeholderforeground (Tk 9.0+), show, state, textvariable, + validate, validatecommand, width VALIDATION MODES @@ -648,7 +650,7 @@ def __init__(self, master=None, widget=None, **kw): Widget.__init__(self, master, widget or "ttk::entry", kw) - def bbox(self, index): + def bbox(self, index): # overrides Misc.bbox """Return a tuple of (x, y, width, height) which describes the bounding box of the character given by index.""" return self._getints(self.tk.call(self._w, "bbox", index)) @@ -676,12 +678,14 @@ def __init__(self, master=None, **kw): STANDARD OPTIONS - class, cursor, style, takefocus + class, cursor, style, takefocus, xscrollcommand WIDGET-SPECIFIC OPTIONS - exportselection, justify, height, postcommand, state, - textvariable, values, width + background, exportselection, font, foreground, height, + invalidcommand, justify, locale (Tk 9.1+), placeholder (Tk 9.0+), + placeholderforeground (Tk 9.0+), postcommand, show, state, + textvariable, validate, validatecommand, values, width """ Entry.__init__(self, master, "ttk::combobox", **kw) @@ -730,13 +734,13 @@ def __init__(self, master=None, **kw): STANDARD OPTIONS - class, compound, cursor, image, style, takefocus, text, - textvariable, underline, width + class, compound, cursor, image, state, style, takefocus, + text, textvariable, underline, width WIDGET-SPECIFIC OPTIONS - anchor, background, font, foreground, justify, padding, - relief, text, wraplength + anchor, background, borderwidth, font, foreground, justify, + padding, relief, text, textangle (Tk 9.1+), wraplength """ Widget.__init__(self, master, "ttk::label", kw) @@ -754,8 +758,9 @@ def __init__(self, master=None, **kw): class, cursor, style, takefocus WIDGET-SPECIFIC OPTIONS - labelanchor, text, underline, padding, labelwidget, width, - height + + borderwidth, height, labelanchor, labelwidget, padding, + relief, text, underline, width """ Widget.__init__(self, master, "ttk::labelframe", kw) @@ -771,8 +776,8 @@ def __init__(self, master=None, **kw): STANDARD OPTIONS - class, compound, cursor, image, state, style, takefocus, - text, textvariable, underline, width + class, compound, cursor, image, justify (Tk 9.0+), padding, + state, style, takefocus, text, textvariable, underline, width WIDGET-SPECIFIC OPTIONS @@ -826,7 +831,7 @@ def add(self, child, **kw): self.tk.call(self._w, "add", child, *(_format_optdict(kw))) - def forget(self, tab_id): + def forget(self, tab_id): # overrides Pack.forget """Removes the tab specified by tab_id, unmaps and unmanages the associated window.""" self.tk.call(self._w, "forget", tab_id) @@ -985,7 +990,9 @@ def __init__(self, master=None, **kw): STANDARD OPTIONS - class, cursor, style, takefocus + anchor (Tk 9.0+), class, cursor, font (Tk 9.0+), + foreground (Tk 9.0+), justify (Tk 9.0+), style, takefocus, + text (Tk 9.0+), wraplength (Tk 9.0+) WIDGET-SPECIFIC OPTIONS @@ -1024,8 +1031,8 @@ def __init__(self, master=None, **kw): STANDARD OPTIONS - class, compound, cursor, image, state, style, takefocus, - text, textvariable, underline, width + class, compound, cursor, image, justify (Tk 9.0+), padding, + state, style, takefocus, text, textvariable, underline, width WIDGET-SPECIFIC OPTIONS @@ -1052,7 +1059,7 @@ def __init__(self, master=None, **kw): STANDARD OPTIONS - class, cursor, style, takefocus + class, cursor, state, style, takefocus WIDGET-SPECIFIC OPTIONS @@ -1149,7 +1156,10 @@ def __init__(self, master=None, **kw): WIDGET-SPECIFIC OPTIONS - to, from_, increment, values, wrap, format, command + background, command, exportselection, font, foreground, + format, from_, increment, justify, locale (Tk 9.1+), + placeholder (Tk 9.0+), placeholderforeground (Tk 9.0+), show, + state, textvariable, to, values, width, wrap """ Entry.__init__(self, master, "ttk::spinbox", **kw) @@ -1176,7 +1186,10 @@ def __init__(self, master=None, **kw): WIDGET-SPECIFIC OPTIONS - columns, displaycolumns, height, padding, selectmode, show + columns, displaycolumns, headingheight (Tk 9.1+), height, + padding, rowheight (Tk 9.1+), selectmode, selecttype (Tk 9.0+), + show, striped (Tk 9.0+), titlecolumns (Tk 9.0+), + titleitems (Tk 9.0+) ITEM OPTIONS @@ -1189,7 +1202,7 @@ def __init__(self, master=None, **kw): Widget.__init__(self, master, "ttk::treeview", kw) - def bbox(self, item, column=None): + def bbox(self, item, column=None): # overrides Misc.bbox """Returns the bounding box (relative to the treeview widget's window) of the specified item in the form x y width height. @@ -1248,7 +1261,7 @@ def exists(self, item): return self.tk.getboolean(self.tk.call(self._w, "exists", item)) - def focus(self, item=None): + def focus(self, item=None): # overrides Misc.focus """If item is specified, sets the focus item to item. Otherwise, returns the current focus item, or '' if there is none.""" return self.tk.call(self._w, "focus", item) @@ -1472,8 +1485,8 @@ def tag_configure(self, tagname, option=None, **kw): def tag_has(self, tagname, item=None): - """If item is specified, returns 1 or 0 depending on whether the - specified item has the given tagname. Otherwise, returns a list of + """If item is specified, returns True if the specified item has the + given tagname, False otherwise. Otherwise, returns a list of all items which have the specified tag. * Availability: Tk 8.6""" @@ -1589,7 +1602,7 @@ class OptionMenu(Menubutton): def __init__(self, master, variable, default=None, *values, **kwargs): """Construct a themed OptionMenu widget with master as the parent, - the resource textvariable set to variable, the initially selected + the option textvariable set to variable, the initially selected value specified by the default parameter, the menu values given by *values and additional keywords. diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 8d01fd7bce41b04..1f31258ce361c94 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -36,7 +36,7 @@ from token import EXACT_TOKEN_TYPES import _tokenize -cookie_re = re.compile(r'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) +cookie_re = re.compile(br'^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)', re.ASCII) blank_re = re.compile(br'^[ \t\f]*(?:[#\r\n]|$)', re.ASCII) import token @@ -86,7 +86,7 @@ def _all_string_prefixes(): # The valid string prefixes. Only contain the lower case versions, # and don't contain any permutations (include 'fr', but not # 'rf'). The various permutations will be generated. - _valid_string_prefixes = ['b', 'r', 'u', 'f', 'br', 'fr'] + _valid_string_prefixes = ['b', 'r', 'u', 'f', 't', 'br', 'fr', 'tr'] # if we add binary f-strings, add: ['fb', 'fbr'] result = {''} for prefix in _valid_string_prefixes: @@ -274,7 +274,7 @@ def compat(self, token, iterable): toks_append = self.tokens.append startline = token[0] in (NEWLINE, NL) prevstring = False - in_fstring = 0 + in_fstring_or_tstring = 0 for tok in _itertools.chain([token], iterable): toknum, tokval = tok[:2] @@ -293,10 +293,10 @@ def compat(self, token, iterable): else: prevstring = False - if toknum == FSTRING_START: - in_fstring += 1 - elif toknum == FSTRING_END: - in_fstring -= 1 + if toknum in {FSTRING_START, TSTRING_START}: + in_fstring_or_tstring += 1 + elif toknum in {FSTRING_END, TSTRING_END}: + in_fstring_or_tstring -= 1 if toknum == INDENT: indents.append(tokval) continue @@ -311,8 +311,8 @@ def compat(self, token, iterable): elif toknum in {FSTRING_MIDDLE, TSTRING_MIDDLE}: tokval = self.escape_brackets(tokval) - # Insert a space between two consecutive brackets if we are in an f-string - if tokval in {"{", "}"} and self.tokens and self.tokens[-1] == tokval and in_fstring: + # Insert a space between two consecutive brackets if we are in an f-string or t-string + if tokval in {"{", "}"} and self.tokens and self.tokens[-1] == tokval and in_fstring_or_tstring: tokval = ' ' + tokval # Insert a space between two consecutive f-strings @@ -385,22 +385,23 @@ def read_or_stop(): except StopIteration: return b'' - def find_cookie(line): + def check(line, encoding): + # Check if the line matches the encoding. + if 0 in line: + raise SyntaxError("source code cannot contain null bytes") try: - # Decode as UTF-8. Either the line is an encoding declaration, - # in which case it should be pure ASCII, or it must be UTF-8 - # per default encoding. - line_string = line.decode('utf-8') + line.decode(encoding) except UnicodeDecodeError: msg = "invalid or missing encoding declaration" if filename is not None: msg = '{} for {!r}'.format(msg, filename) raise SyntaxError(msg) - match = cookie_re.match(line_string) + def find_cookie(line): + match = cookie_re.match(line) if not match: return None - encoding = _get_normal_name(match.group(1)) + encoding = _get_normal_name(match.group(1).decode()) try: codec = lookup(encoding) except LookupError: @@ -433,18 +434,23 @@ def find_cookie(line): encoding = find_cookie(first) if encoding: + check(first, encoding) return encoding, [first] if not blank_re.match(first): + check(first, default) return default, [first] second = read_or_stop() if not second: + check(first, default) return default, [first] encoding = find_cookie(second) if encoding: + check(first + second, encoding) return encoding, [first, second] + check(first + second, default) return default, [first, second] diff --git a/Lib/tomllib/_parser.py b/Lib/tomllib/_parser.py index 3ee47aa9e0afbad..84e70495dd0d3a2 100644 --- a/Lib/tomllib/_parser.py +++ b/Lib/tomllib/_parser.py @@ -4,6 +4,7 @@ from __future__ import annotations +import sys from types import MappingProxyType from ._re import ( @@ -18,10 +19,17 @@ TYPE_CHECKING = False if TYPE_CHECKING: from collections.abc import Iterable - from typing import IO, Any + from typing import IO, Any, Final from ._types import Key, ParseFloat, Pos +# Pathologically excessive number of parts in a key runs into quadratic +# behavior (e.g. in Flags.is_). +# Even if keys aren't currently parsed using recursion, they name a +# recursive structure, so it makes sense to limit it using getrecursionlimit() +# and RecursionError. +MAX_KEY_PARTS: Final = sys.getrecursionlimit() + ASCII_CTRL = frozenset(chr(i) for i in range(32)) | frozenset(chr(127)) # Neither of these sets include quotation mark or backslash. They are @@ -462,6 +470,10 @@ def parse_key(src: str, pos: Pos) -> tuple[Pos, Key]: pos = skip_chars(src, pos, TOML_WS) pos, key_part = parse_key_part(src, pos) key += (key_part,) + if len(key) > MAX_KEY_PARTS: + raise RecursionError( + f"TOML key has more than the allowed {MAX_KEY_PARTS} parts" + ) pos = skip_chars(src, pos, TOML_WS) diff --git a/Lib/tomllib/mypy.ini b/Lib/tomllib/mypy.ini index 1761dce45562a60..f7eeffd575c1c76 100644 --- a/Lib/tomllib/mypy.ini +++ b/Lib/tomllib/mypy.ini @@ -12,6 +12,4 @@ pretty = True # Enable most stricter settings enable_error_code = ignore-without-code strict = True -strict_bytes = True -local_partial_types = True warn_unreachable = True diff --git a/Lib/traceback.py b/Lib/traceback.py index 17b082eced6f05b..227ca8efa172543 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -74,10 +74,10 @@ def extract_tb(tb, limit=None): This is useful for alternate formatting of stack traces. If 'limit' is omitted or None, all entries are extracted. A pre-processed stack trace entry is a FrameSummary object - containing attributes filename, lineno, name, and line - representing the information that is usually printed for a stack - trace. The line is a string with leading and trailing - whitespace stripped; if the source is not available it is None. + representing the information that is usually printed for a + stack trace. The line attribute is a string with + leading and trailing whitespace stripped; if the source is not + available the corresponding attribute is None. """ return StackSummary._extract_from_extended_frame_gen( _walk_tb_with_full_positions(tb), limit=limit) @@ -253,9 +253,8 @@ def extract_stack(f=None, limit=None): The return value has the same format as for extract_tb(). The optional 'f' and 'limit' arguments have the same meaning as for - print_stack(). Each item in the list is a quadruple (filename, - line number, function name, text), and the entries are in order - from oldest to newest stack frame. + print_stack(). Each item in the list is a FrameSummary object, + and the entries are in order from oldest to newest stack frame. """ if f is None: f = sys._getframe().f_back @@ -283,7 +282,7 @@ class FrameSummary: active when the frame was captured. - :attr:`name` The name of the function or method that was executing when the frame was captured. - - :attr:`line` The text from the linecache module for the + - :attr:`line` The text from the linecache module for the line of code that was running when the frame was captured. - :attr:`locals` Either None if locals were not supplied, or a dict mapping the name to the repr() of the variable. @@ -534,7 +533,7 @@ def format_frame_summary(self, frame_summary, **kwargs): colorize = kwargs.get("colorize", False) row = [] filename = frame_summary.filename - if frame_summary.filename.startswith("<stdin>-"): + if frame_summary.filename.startswith("<stdin-") and frame_summary.filename.endswith('>'): filename = "<stdin>" if colorize: theme = _colorize.get_theme(force_color=True).traceback @@ -958,7 +957,7 @@ def setup_positions(expr, force_valid=True): _WIDE_CHAR_SPECIFIERS = "WF" def _display_width(line, offset=None): - """Calculate the extra amount of width space the given source + """Calculate the amount of width space the given source code segment might take if it were to be displayed on a fixed width output device. Supports wide unicode characters and emojis.""" @@ -1044,7 +1043,7 @@ class TracebackException: def __init__(self, exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, max_group_width=15, max_group_depth=10, save_exc_type=True, _seen=None): - # NB: we need to accept exc_traceback, exc_value, exc_traceback to + # NB: we need to accept exc_type, exc_value, exc_traceback to # permit backwards compat with the existing API, otherwise we # need stub thunk objects just to glue it together. # Handle loops in __cause__ or __context__. @@ -1310,7 +1309,6 @@ def _find_keyword_typos(self): lines = source.splitlines() error_code = lines[line -1 if line > 0 else 0:end_line] - error_code[0] = error_code[0][offset:] error_code = textwrap.dedent('\n'.join(error_code)) # Do not continue if the source is too large @@ -1326,7 +1324,8 @@ def _find_keyword_typos(self): if token.type != tokenize.NAME: continue # Only consider NAME tokens on the same line as the error - if from_filename and token.start[0]+line != end_line+1: + the_end = end_line if line == 0 else end_line + 1 + if from_filename and token.start[0]+line != the_end: continue wrong_name = token.string if wrong_name in keyword.kwlist: @@ -1589,13 +1588,23 @@ def _substitution_cost(ch_a, ch_b): return _MOVE_COST +def _get_safe___dir__(obj): + # Use obj.__dir__() to avoid a TypeError when calling dir(obj). + # See gh-131001 and gh-139933. + try: + d = obj.__dir__() + except TypeError: # when obj is a class + d = type(obj).__dir__(obj) + return sorted(x for x in d if isinstance(x, str)) + + def _compute_suggestion_error(exc_value, tb, wrong_name): if wrong_name is None or not isinstance(wrong_name, str): return None if isinstance(exc_value, AttributeError): obj = exc_value.obj try: - d = dir(obj) + d = _get_safe___dir__(obj) hide_underscored = (wrong_name[:1] != '_') if hide_underscored and tb is not None: while tb.tb_next is not None: @@ -1610,7 +1619,7 @@ def _compute_suggestion_error(exc_value, tb, wrong_name): elif isinstance(exc_value, ImportError): try: mod = __import__(exc_value.name) - d = dir(mod) + d = _get_safe___dir__(mod) if wrong_name[:1] != '_': d = [x for x in d if x[:1] != '_'] except Exception: @@ -1628,6 +1637,7 @@ def _compute_suggestion_error(exc_value, tb, wrong_name): + list(frame.f_globals) + list(frame.f_builtins) ) + d = [x for x in d if isinstance(x, str)] # Check first if we are in a method and the instance # has the wrong name as attribute diff --git a/Lib/turtle.py b/Lib/turtle.py index e88981d298ad52e..db38a5a23581a7f 100644 --- a/Lib/turtle.py +++ b/Lib/turtle.py @@ -486,6 +486,7 @@ def __init__(self, cv): self.canvwidth = w self.canvheight = h self.xscale = self.yscale = 1.0 + self._updating = False def _createpoly(self): """Create an invisible polygon item on canvas self.cv) @@ -555,7 +556,16 @@ def _delete(self, item): def _update(self): """Redraw graphics items on canvas """ - self.cv.update() + if self._updating: + # Reentrant call (e.g. a drag handler moving the turtle, + # gh-50966): flush drawing without reprocessing input. + self.cv.update_idletasks() + return + self._updating = True + try: + self.cv.update() + finally: + self._updating = False def _delay(self, delay): """Delay subsequent canvas actions for delay ms.""" @@ -989,8 +999,8 @@ def __init__(self, cv, mode=_CFG["mode"], # the Turtle window will show behind the Terminal window when you # start the demo from the command line. rootwindow = cv.winfo_toplevel() - rootwindow.call('wm', 'attributes', '.', '-topmost', '1') - rootwindow.call('wm', 'attributes', '.', '-topmost', '0') + rootwindow.wm_attributes(topmost=True) + rootwindow.wm_attributes(topmost=False) def clear(self): """Delete all drawings and all turtles from the TurtleScreen. @@ -1214,16 +1224,32 @@ def turtles(self): def bgcolor(self, *args): """Set or return backgroundcolor of the TurtleScreen. - Arguments (if given): a color string or three numbers - in the range 0..colormode or a 3-tuple of such numbers. + Four input formats are allowed: + - bgcolor() + Return the current background color as color specification + string or as a tuple (see example). May be used as input + to another color/pencolor/fillcolor/bgcolor call. + - bgcolor(colorstring) + Set the background color to colorstring, which is a Tk color + specification string, such as "red", "yellow", or "#33cc8c". + - bgcolor((r, g, b)) + Set the background color to the RGB color represented by + the tuple of r, g, and b. Each of r, g, and b must be in + the range 0..colormode, where colormode is either 1.0 or 255 + (see colormode()). + - bgcolor(r, g, b) + Set the background color to the RGB color represented by + r, g, and b. Each of r, g, and b must be in the range + 0..colormode. Example (for a TurtleScreen instance named screen): >>> screen.bgcolor("orange") >>> screen.bgcolor() 'orange' - >>> screen.bgcolor(0.5,0,0.5) + >>> colormode(255) + >>> screen.bgcolor('#800080') >>> screen.bgcolor() - '#800080' + (128.0, 0.0, 128.0) """ if args: color = self._colorstr(args) @@ -1678,7 +1704,7 @@ def forward(self, distance): Example (for a Turtle instance named turtle): >>> turtle.position() - (0.00, 0.00) + (0.00,0.00) >>> turtle.forward(25) >>> turtle.position() (25.00,0.00) @@ -1701,10 +1727,10 @@ def back(self, distance): Example (for a Turtle instance named turtle): >>> turtle.position() - (0.00, 0.00) + (0.00,0.00) >>> turtle.backward(30) >>> turtle.position() - (-30.00, 0.00) + (-30.00,0.00) """ self._go(-distance) @@ -1811,7 +1837,7 @@ def goto(self, x, y=None): Example (for a Turtle instance named turtle): >>> tp = turtle.pos() >>> tp - (0.00, 0.00) + (0.00,0.00) >>> turtle.setpos(60,30) >>> turtle.pos() (60.00,30.00) @@ -1891,7 +1917,7 @@ def distance(self, x, y=None): Example (for a Turtle instance named turtle): >>> turtle.pos() - (0.00, 0.00) + (0.00,0.00) >>> turtle.distance(30,40) 50.0 >>> pen = Turtle() @@ -2230,19 +2256,17 @@ def color(self, *args): Arguments: Several input formats are allowed. - They use 0, 1, 2, or 3 arguments as follows: - - color() - Return the current pencolor and the current fillcolor - as a pair of color specification strings as are returned - by pencolor and fillcolor. - color(colorstring), color((r,g,b)), color(r,g,b) - inputs as in pencolor, set both, fillcolor and pencolor, + They use 0 to 3 arguments as follows: + - color() + Return the current pencolor and the current fillcolor as + a pair of color specification strings or tuples as returned + by pencolor() and fillcolor(). + - color(colorstring), color((r,g,b)), color(r,g,b) + Inputs as in pencolor(), set both, fillcolor and pencolor, to the given value. - color(colorstring1, colorstring2), - color((r1,g1,b1), (r2,g2,b2)) - equivalent to pencolor(colorstring1) and fillcolor(colorstring2) - and analogously, if the other input format is used. + - color(colorstring1, colorstring2), color((r1,g1,b1), (r2,g2,b2)) + Equivalent to pencolor(colorstring1) and fillcolor(colorstring2) + and analogously if the other input format is used. If turtleshape is a polygon, outline and interior of that polygon is drawn with the newly set colors. @@ -2253,9 +2277,9 @@ def color(self, *args): >>> turtle.color() ('red', 'green') >>> colormode(255) - >>> color((40, 80, 120), (160, 200, 240)) + >>> color(('#285078', '#a0c8f0')) >>> color() - ('#285078', '#a0c8f0') + ((40.0, 80.0, 120.0), (160.0, 200.0, 240.0)) """ if args: l = len(args) @@ -2277,28 +2301,32 @@ def pencolor(self, *args): Arguments: Four input formats are allowed: - pencolor() - Return the current pencolor as color specification string, - possibly in hex-number format (see example). - May be used as input to another color/pencolor/fillcolor call. + Return the current pencolor as color specification string or + as a tuple (see example). May be used as input to another + color/pencolor/fillcolor/bgcolor call. - pencolor(colorstring) - s is a Tk color specification string, such as "red" or "yellow" + Set pencolor to colorstring, which is a Tk color + specification string, such as "red", "yellow", or "#33cc8c". - pencolor((r, g, b)) - *a tuple* of r, g, and b, which represent, an RGB color, - and each of r, g, and b are in the range 0..colormode, - where colormode is either 1.0 or 255 + Set pencolor to the RGB color represented by the tuple of + r, g, and b. Each of r, g, and b must be in the range + 0..colormode, where colormode is either 1.0 or 255 (see + colormode()). - pencolor(r, g, b) - r, g, and b represent an RGB color, and each of r, g, and b - are in the range 0..colormode + Set pencolor to the RGB color represented by r, g, and b. + Each of r, g, and b must be in the range 0..colormode. If turtleshape is a polygon, the outline of that polygon is drawn with the newly set pencolor. Example (for a Turtle instance named turtle): >>> turtle.pencolor('brown') - >>> tup = (0.2, 0.8, 0.55) - >>> turtle.pencolor(tup) >>> turtle.pencolor() - '#33cc8c' + 'brown' + >>> colormode(255) + >>> turtle.pencolor('#32c18f') + >>> turtle.pencolor() + (50.0, 193.0, 143.0) """ if args: color = self._colorstr(args) @@ -2315,26 +2343,31 @@ def fillcolor(self, *args): Four input formats are allowed: - fillcolor() Return the current fillcolor as color specification string, - possibly in hex-number format (see example). - May be used as input to another color/pencolor/fillcolor call. + possibly in tuple format (see example). May be used as + input to another color/pencolor/fillcolor/bgcolor call. - fillcolor(colorstring) - s is a Tk color specification string, such as "red" or "yellow" + Set fillcolor to colorstring, which is a Tk color + specification string, such as "red", "yellow", or "#33cc8c". - fillcolor((r, g, b)) - *a tuple* of r, g, and b, which represent, an RGB color, - and each of r, g, and b are in the range 0..colormode, - where colormode is either 1.0 or 255 + Set fillcolor to the RGB color represented by the tuple of + r, g, and b. Each of r, g, and b must be in the range + 0..colormode, where colormode is either 1.0 or 255 (see + colormode()). - fillcolor(r, g, b) - r, g, and b represent an RGB color, and each of r, g, and b - are in the range 0..colormode + Set fillcolor to the RGB color represented by r, g, and b. + Each of r, g, and b must be in the range 0..colormode. If turtleshape is a polygon, the interior of that polygon is drawn with the newly set fillcolor. Example (for a Turtle instance named turtle): >>> turtle.fillcolor('violet') - >>> col = turtle.pencolor() - >>> turtle.fillcolor(col) - >>> turtle.fillcolor(0, .5, 0) + >>> turtle.fillcolor() + 'violet' + >>> colormode(255) + >>> turtle.fillcolor('#ffffff') + >>> turtle.fillcolor() + (255.0, 255.0, 255.0) """ if args: color = self._colorstr(args) diff --git a/Lib/turtledemo/__main__.py b/Lib/turtledemo/__main__.py index b49c0beab3ccf7b..7c2d753f4c31113 100644 --- a/Lib/turtledemo/__main__.py +++ b/Lib/turtledemo/__main__.py @@ -136,7 +136,7 @@ def __init__(self, filename=None): # so that our menu bar appears. subprocess.run( [ - 'osascript', + '/usr/bin/osascript', '-e', 'tell application "System Events"', '-e', 'set frontmost of the first process whose ' 'unix id is {} to true'.format(os.getpid()), diff --git a/Lib/types.py b/Lib/types.py index 6efac3394345a54..60a4b22c200a802 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -189,18 +189,19 @@ class Baz(list[str]): ... class DynamicClassAttribute: """Route attribute access on a class to __getattr__. - This is a descriptor, used to define attributes that act differently when - accessed through an instance and through a class. Instance access remains - normal, but access to an attribute through a class will be routed to the - class's __getattr__ method; this is done by raising AttributeError. + This is a descriptor, used to define attributes that act differently + when accessed through an instance and through a class. Instance access + remains normal, but access to an attribute through a class will be + routed to the class's __getattr__ method; this is done by raising + AttributeError. - This allows one to have properties active on an instance, and have virtual - attributes on the class with the same name. (Enum used this between Python - versions 3.4 - 3.9 .) + This allows one to have properties active on an instance, and have + virtual attributes on the class with the same name. (Enum used this + between Python versions 3.4 - 3.9 .) - Subclass from this to use a different method of accessing virtual attributes - and still be treated properly by the inspect module. (Enum uses this since - Python 3.10 .) + Subclass from this to use a different method of accessing virtual + attributes and still be treated properly by the inspect module. (Enum + uses this since Python 3.10 .) """ def __init__(self, fget=None, fset=None, fdel=None, doc=None): @@ -274,10 +275,14 @@ def gi_running(self): @property def gi_yieldfrom(self): return self.__wrapped.gi_yieldfrom + @property + def gi_suspended(self): + return self.__wrapped.gi_suspended cr_code = gi_code cr_frame = gi_frame cr_running = gi_running cr_await = gi_yieldfrom + cr_suspended = gi_suspended def __next__(self): return next(self.__wrapped) def __iter__(self): diff --git a/Lib/typing.py b/Lib/typing.py index 2baf655256d1eb7..2b4389b2967d128 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -5,7 +5,7 @@ * Generic, Protocol, and internal machinery to support generic aliases. All subscripted types like X[int], Union[int, str] are generic aliases. * Various "special forms" that have unique meanings in type annotations: - NoReturn, Never, ClassVar, Self, Concatenate, Unpack, and others. + Any, Never, ClassVar, Self, Concatenate, Unpack, and others. * Classes whose instances can be type arguments to generic classes and functions: TypeVar, ParamSpec, TypeVarTuple. * Public helper functions: get_type_hints, overload, cast, final, and others. @@ -65,6 +65,7 @@ # ABCs (from collections.abc). 'AbstractSet', # collections.abc.Set. + 'ByteString', 'Container', 'ContextManager', 'Hashable', @@ -171,16 +172,16 @@ def __getattr__(self, attr): _lazy_annotationlib = _LazyAnnotationLib() -def _type_convert(arg, module=None, *, allow_special_forms=False): +def _type_convert(arg, module=None, *, allow_special_forms=False, owner=None): """For converting None to type(None), and strings to ForwardRef.""" if arg is None: return type(None) if isinstance(arg, str): - return _make_forward_ref(arg, module=module, is_class=allow_special_forms) + return _make_forward_ref(arg, module=module, is_class=allow_special_forms, owner=owner) return arg -def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms=False): +def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms=False, owner=None): """Check that the argument is a type, and return it (internal helper). As a special case, accept None and return type(None) instead. Also wrap strings @@ -198,7 +199,7 @@ def _type_check(arg, msg, is_argument=True, module=None, *, allow_special_forms= if is_argument: invalid_generic_forms += (Final,) - arg = _type_convert(arg, module=module, allow_special_forms=allow_special_forms) + arg = _type_convert(arg, module=module, allow_special_forms=allow_special_forms, owner=owner) if (isinstance(arg, _GenericAlias) and arg.__origin__ in invalid_generic_forms): raise TypeError(f"{arg} is not valid as type argument") @@ -430,7 +431,7 @@ def __repr__(self): def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=frozenset(), - format=None, owner=None): + format=None, owner=None, parent_fwdref=None, prefer_fwd_module=False): """Evaluate all forward references in the given type t. For use of globalns and localns see the docstring for get_type_hints(). @@ -443,15 +444,27 @@ def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=f if isinstance(t, _lazy_annotationlib.ForwardRef): # If the forward_ref has __forward_module__ set, evaluate() infers the globals # from the module, and it will probably pick better than the globals we have here. - if t.__forward_module__ is not None: + # We do this only for calls from get_type_hints() (which opts in through the + # prefer_fwd_module flag), so that the default behavior remains more straightforward. + if prefer_fwd_module and t.__forward_module__ is not None: globalns = None + # If there are type params on the owner, we need to add them back, because + # annotationlib won't. + if owner_type_params := getattr(owner, "__type_params__", None): + globalns = getattr( + sys.modules.get(t.__forward_module__, None), "__dict__", None + ) + if globalns is not None: + globalns = dict(globalns) + for type_param in owner_type_params: + globalns[type_param.__name__] = type_param return evaluate_forward_ref(t, globals=globalns, locals=localns, type_params=type_params, owner=owner, _recursive_guard=recursive_guard, format=format) if isinstance(t, (_GenericAlias, GenericAlias, Union)): if isinstance(t, GenericAlias): args = tuple( - _make_forward_ref(arg) if isinstance(arg, str) else arg + _make_forward_ref(arg, parent_fwdref=parent_fwdref) if isinstance(arg, str) else arg for arg in t.__args__ ) is_unpacked = t.__unpacked__ @@ -465,7 +478,7 @@ def _eval_type(t, globalns, localns, type_params=_sentinel, *, recursive_guard=f ev_args = tuple( _eval_type( a, globalns, localns, type_params, recursive_guard=recursive_guard, - format=format, owner=owner, + format=format, owner=owner, prefer_fwd_module=prefer_fwd_module, ) for a in t.__args__ ) @@ -575,12 +588,12 @@ def __repr__(self): class Any(metaclass=_AnyMeta): """Special type indicating an unconstrained type. - - Any is compatible with every type. - - Any assumed to have all methods. - - All values assumed to be instances of Any. + - Any is assignable to every type. + - Any assumed to have all methods and attributes. + - All values are assignable to Any. Note that all the above statements are true from the point of view of - static type checkers. At runtime, Any should not be used with instance + static type checkers. At runtime, Any cannot be used with instance checks. """ @@ -699,7 +712,7 @@ class Starship: ClassVar accepts only types and cannot be further subscribed. - Note that ClassVar is not a class itself, and should not + Note that ClassVar is not a class itself, and cannot be used with isinstance() or issubclass(). """ item = _type_check(parameters, f'{self} accepts only single type.', allow_special_forms=True) @@ -729,7 +742,7 @@ class FastConnector(Connection): @_SpecialForm def Optional(self, parameters): - """Optional[X] is equivalent to Union[X, None].""" + """Optional[X] is equivalent to X | None.""" arg = _type_check(parameters, f"{self} requires a single type.") return Union[arg, type(None)] @@ -759,20 +772,23 @@ def open_helper(file: str, mode: MODE) -> str: # There is no '_type_check' call because arguments to Literal[...] are # values, not types. parameters = _flatten_literal_params(parameters) + value_and_type_parameters = list(_value_and_type_iter(parameters)) + deduplicated_parameters = tuple( + p + for p, _ in _deduplicate( + value_and_type_parameters, + unhashable_fallback=True, + ) + ) - try: - parameters = tuple(p for p, _ in _deduplicate(list(_value_and_type_iter(parameters)))) - except TypeError: # unhashable parameters - pass - - return _LiteralGenericAlias(self, parameters) + return _LiteralGenericAlias(self, deduplicated_parameters) @_SpecialForm def TypeAlias(self, parameters): """Special form for marking type aliases. - Use TypeAlias to indicate that an assignment should + TypeAlias can be used to indicate that an assignment should be recognized as a proper type alias definition by type checkers. @@ -936,7 +952,12 @@ def run(arg: Child | Unrelated): return _GenericAlias(self, (item,)) -def _make_forward_ref(code, **kwargs): +def _make_forward_ref(code, *, parent_fwdref=None, **kwargs): + if parent_fwdref is not None: + if parent_fwdref.__forward_module__ is not None: + kwargs['module'] = parent_fwdref.__forward_module__ + if parent_fwdref.__owner__ is not None: + kwargs['owner'] = parent_fwdref.__owner__ forward_ref = _lazy_annotationlib.ForwardRef(code, **kwargs) # For compatibility, eagerly compile the forwardref's code. forward_ref.__forward_code__ @@ -956,12 +977,8 @@ def evaluate_forward_ref( """Evaluate a forward reference as a type hint. This is similar to calling the ForwardRef.evaluate() method, - but unlike that method, evaluate_forward_ref() also: - - * Recursively evaluates forward references nested within the type hint. - * Rejects certain objects that are not valid type hints. - * Replaces type hints that evaluate to None with types.NoneType. - * Supports the *FORWARDREF* and *STRING* formats. + but unlike that method, evaluate_forward_ref() also + recursively evaluates forward references nested within the type hint. *forward_ref* must be an instance of ForwardRef. *owner*, if given, should be the object that holds the annotations that the forward reference @@ -977,39 +994,43 @@ def evaluate_forward_ref( """ if format == _lazy_annotationlib.Format.STRING: - return forward_ref.__forward_arg__ + return forward_ref.__resolved_str__ if forward_ref.__forward_arg__ in _recursive_guard: return forward_ref - try: - value = forward_ref.evaluate(globals=globals, locals=locals, - type_params=type_params, owner=owner) - except NameError: - if format == _lazy_annotationlib.Format.FORWARDREF: - return forward_ref - else: - raise - - type_ = _type_check( - value, - "Forward references must evaluate to types.", - is_argument=forward_ref.__forward_is_argument__, - allow_special_forms=forward_ref.__forward_is_class__, - ) + if format is None: + format = _lazy_annotationlib.Format.VALUE + value = forward_ref.evaluate(globals=globals, locals=locals, + type_params=type_params, owner=owner, format=format) + + if (isinstance(value, _lazy_annotationlib.ForwardRef) + and format == _lazy_annotationlib.Format.FORWARDREF): + return value + + if isinstance(value, str): + value = _make_forward_ref(value, module=forward_ref.__forward_module__, + owner=owner or forward_ref.__owner__, + is_argument=forward_ref.__forward_is_argument__, + is_class=forward_ref.__forward_is_class__) + if owner is None: + owner = forward_ref.__owner__ return _eval_type( - type_, + value, globals, locals, type_params, recursive_guard=_recursive_guard | {forward_ref.__forward_arg__}, format=format, owner=owner, + parent_fwdref=forward_ref, ) def _is_unpacked_typevartuple(x: Any) -> bool: + # Need to check 'is True' here + # See: https://github.com/python/cpython/issues/137706 return ((not isinstance(x, type)) and - getattr(x, '__typing_is_unpacked_typevartuple__', False)) + getattr(x, '__typing_is_unpacked_typevartuple__', False) is True) def _is_typevar_like(x: Any) -> bool: @@ -1079,7 +1100,7 @@ def _paramspec_prepare_subst(self, alias, args): params = alias.__parameters__ i = params.index(self) if i == len(args) and self.has_default(): - args = [*args, self.__default__] + args = (*args, self.__default__) if i >= len(args): raise TypeError(f"Too few arguments for {alias}") # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612. @@ -1124,14 +1145,26 @@ def _generic_class_getitem(cls, args): f"Parameters to {cls.__name__}[...] must all be unique") else: # Subscripting a regular Generic subclass. - for param in cls.__parameters__: + try: + parameters = cls.__parameters__ + except AttributeError as e: + init_subclass = getattr(cls, '__init_subclass__', None) + if init_subclass not in {None, Generic.__init_subclass__}: + e.add_note( + f"Note: this exception may have been caused by " + f"{init_subclass.__qualname__!r} (or the " + f"'__init_subclass__' method on a superclass) not " + f"calling 'super().__init_subclass__()'" + ) + raise + for param in parameters: prepare = getattr(param, '__typing_prepare_subst__', None) if prepare is not None: args = prepare(cls, args) _check_generic_specialization(cls, args) new_args = [] - for param, new_arg in zip(cls.__parameters__, args): + for param, new_arg in zip(parameters, args): if isinstance(param, TypeVarTuple): new_args.extend(new_arg) else: @@ -1283,31 +1316,35 @@ def __dir__(self): class _GenericAlias(_BaseGenericAlias, _root=True): - # The type of parameterized generics. - # - # That is, for example, `type(List[int])` is `_GenericAlias`. - # - # Objects which are instances of this class include: - # * Parameterized container types, e.g. `Tuple[int]`, `List[int]`. - # * Note that native container types, e.g. `tuple`, `list`, use - # `types.GenericAlias` instead. - # * Parameterized classes: - # class C[T]: pass - # # C[int] is a _GenericAlias - # * `Callable` aliases, generic `Callable` aliases, and - # parameterized `Callable` aliases: - # T = TypeVar('T') - # # _CallableGenericAlias inherits from _GenericAlias. - # A = Callable[[], None] # _CallableGenericAlias - # B = Callable[[T], None] # _CallableGenericAlias - # C = B[int] # _CallableGenericAlias - # * Parameterized `Final`, `ClassVar`, `TypeGuard`, and `TypeIs`: - # # All _GenericAlias - # Final[int] - # ClassVar[float] - # TypeGuard[bool] - # TypeIs[range] - + """The type of parameterized generics. + + That is, for example, `type(List[int])` is `_GenericAlias`. + + Objects which are instances of this class include: + * Parameterized container types, e.g. `Tuple[int]`, `List[int]`. + * Note that native container types, e.g. `tuple`, `list`, use + `types.GenericAlias` instead. + * Parameterized classes: + class C[T]: pass + # C[int] is a _GenericAlias + * `Callable` aliases, generic `Callable` aliases, and + parameterized `Callable` aliases: + T = TypeVar('T') + # _CallableGenericAlias inherits from _GenericAlias. + A = Callable[[], None] # _CallableGenericAlias + B = Callable[[T], None] # _CallableGenericAlias + C = B[int] # _CallableGenericAlias + * Parameterized `Final`, `ClassVar`, `TypeForm`, `TypeGuard`, and `TypeIs`: + # All _GenericAlias + Final[int] + ClassVar[float] + TypeForm[bytearray] + TypeGuard[bool] + TypeIs[range] + + Note that instances of this class are not classes (e.g by `inspect.isclass`), + even though they behave like them. + """ def __init__(self, origin, args, *, inst=True, name=None): super().__init__(origin, inst=inst, name=name) if not isinstance(args, tuple): @@ -1339,20 +1376,21 @@ def __ror__(self, left): @_tp_cache def __getitem__(self, args): - # Parameterizes an already-parameterized object. - # - # For example, we arrive here doing something like: - # T1 = TypeVar('T1') - # T2 = TypeVar('T2') - # T3 = TypeVar('T3') - # class A(Generic[T1]): pass - # B = A[T2] # B is a _GenericAlias - # C = B[T3] # Invokes _GenericAlias.__getitem__ - # - # We also arrive here when parameterizing a generic `Callable` alias: - # T = TypeVar('T') - # C = Callable[[T], None] - # C[int] # Invokes _GenericAlias.__getitem__ + """Parameterizes an already-parameterized object. + + For example, we arrive here doing something like: + T1 = TypeVar('T1') + T2 = TypeVar('T2') + T3 = TypeVar('T3') + class A(Generic[T1]): pass + B = A[T2] # B is a _GenericAlias + C = B[T3] # Invokes _GenericAlias.__getitem__ + + We also arrive here when parameterizing a generic `Callable` alias: + T = TypeVar('T') + C = Callable[[T], None] + C[int] # Invokes _GenericAlias.__getitem__ + """ if self.__origin__ in (Generic, Protocol): # Can't subscript Generic[...] or Protocol[...]. @@ -1369,20 +1407,20 @@ def __getitem__(self, args): return r def _determine_new_args(self, args): - # Determines new __args__ for __getitem__. - # - # For example, suppose we had: - # T1 = TypeVar('T1') - # T2 = TypeVar('T2') - # class A(Generic[T1, T2]): pass - # T3 = TypeVar('T3') - # B = A[int, T3] - # C = B[str] - # `B.__args__` is `(int, T3)`, so `C.__args__` should be `(int, str)`. - # Unfortunately, this is harder than it looks, because if `T3` is - # anything more exotic than a plain `TypeVar`, we need to consider - # edge cases. - + """Determines new __args__ for __getitem__. + + For example, suppose we had: + T1 = TypeVar('T1') + T2 = TypeVar('T2') + class A(Generic[T1, T2]): pass + T3 = TypeVar('T3') + B = A[int, T3] + C = B[str] + `B.__args__` is `(int, T3)`, so `C.__args__` should be `(int, str)`. + Unfortunately, this is harder than it looks, because if `T3` is + anything more exotic than a plain `TypeVar`, we need to consider + edge cases. + """ params = self.__parameters__ # In the example above, this would be {T3: str} for param in params: @@ -1515,9 +1553,9 @@ def __init__(self, origin, nparams, *, inst=True, name=None, defaults=()): self._nparams = nparams self._defaults = defaults if origin.__module__ == 'builtins': - self.__doc__ = f'A generic version of {origin.__qualname__}.' + self.__doc__ = f'Deprecated alias to {origin.__qualname__}.' else: - self.__doc__ = f'A generic version of {origin.__module__}.{origin.__qualname__}.' + self.__doc__ = f'Deprecated alias to {origin.__module__}.{origin.__qualname__}.' @_tp_cache def __getitem__(self, params): @@ -1567,6 +1605,21 @@ def __ror__(self, left): return Union[left, self] +class _DeprecatedGenericAlias(_SpecialGenericAlias, _root=True): + def __init__( + self, origin, nparams, *, removal_version, inst=True, name=None + ): + super().__init__(origin, nparams, inst=inst, name=name) + self._removal_version = removal_version + + def __instancecheck__(self, inst): + import warnings + warnings._deprecated( + f"{self.__module__}.{self._name}", remove=self._removal_version + ) + return super().__instancecheck__(inst) + + class _CallableGenericAlias(_NotIterable, _GenericAlias, _root=True): def __repr__(self): assert self._name == 'Callable' @@ -1649,6 +1702,9 @@ def __eq__(self, other): return True return NotImplemented + def __hash__(self): + return hash(Union) + class _UnionGenericAlias(metaclass=_UnionGenericAliasMeta): """Compatibility hack. @@ -1732,7 +1788,7 @@ class Movie(TypedDict): def foo(**kwargs: Unpack[Movie]): ... Note that there is only some runtime checking of this operator. Not - everything the runtime allows may be accepted by static type checkers. + everything the runtime allows is accepted by static type checkers. For more information, see PEPs 646 and 692. """ @@ -2213,7 +2269,7 @@ def runtime_checkable(cls): Such protocol can be used with isinstance() and issubclass(). Raise TypeError if applied to a non-protocol class. This allows a simple-minded structural check very similar to - one trick ponies in collections.abc such as Iterable. + one-trick ponies in collections.abc such as Iterable. For example:: @@ -2280,8 +2336,8 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, *, format=None): """Return type hints for an object. - This is often the same as obj.__annotations__, but it handles - forward references encoded as string literals and recursively replaces all + This is often the same as annotationlib.get_annotations(obj) or obj.__annotations__, + but it handles forward references encoded as string literals and recursively replaces all 'Annotated[T, ...]' with 'T' (unless 'include_extras=True'). The argument may be a module, class, method, or function. The annotations @@ -2334,13 +2390,16 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, # *base_globals* first rather than *base_locals*. # This only affects ForwardRefs. base_globals, base_locals = base_locals, base_globals + type_params = base.__type_params__ + base_globals, base_locals = _add_type_params_to_scope( + type_params, base_globals, base_locals, True) for name, value in ann.items(): - if value is None: - value = type(None) if isinstance(value, str): value = _make_forward_ref(value, is_argument=False, is_class=True) - value = _eval_type(value, base_globals, base_locals, base.__type_params__, - format=format, owner=obj) + value = _eval_type(value, base_globals, base_locals, (), + format=format, owner=obj, prefer_fwd_module=True) + if value is None: + value = type(None) hints[name] = value if include_extras or format == Format.STRING: return hints @@ -2365,17 +2424,20 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, else: nsobj = obj # Find globalns for the unwrapped object. + seen = {id(nsobj)} while hasattr(nsobj, '__wrapped__'): nsobj = nsobj.__wrapped__ + if id(nsobj) in seen: + raise ValueError(f'wrapper loop when unwrapping {obj!r}') + seen.add(id(nsobj)) globalns = getattr(nsobj, '__globals__', {}) if localns is None: localns = globalns elif localns is None: localns = globalns type_params = getattr(obj, "__type_params__", ()) + globalns, localns = _add_type_params_to_scope(type_params, globalns, localns, False) for name, value in hints.items(): - if value is None: - value = type(None) if isinstance(value, str): # class-level forward refs were handled above, this must be either # a module-level annotation or a function argument annotation @@ -2384,10 +2446,27 @@ def get_type_hints(obj, globalns=None, localns=None, include_extras=False, is_argument=not isinstance(obj, types.ModuleType), is_class=False, ) - hints[name] = _eval_type(value, globalns, localns, type_params, format=format, owner=obj) + value = _eval_type(value, globalns, localns, (), format=format, owner=obj, prefer_fwd_module=True) + if value is None: + value = type(None) + hints[name] = value return hints if include_extras else {k: _strip_annotations(t) for k, t in hints.items()} +# Add type parameters to the globals and locals scope. This is needed for +# compatibility. +def _add_type_params_to_scope(type_params, globalns, localns, is_class): + if not type_params: + return globalns, localns + globalns = dict(globalns) + localns = dict(localns) + for param in type_params: + if not is_class or param.__name__ not in globalns: + globalns[param.__name__] = param + localns.pop(param.__name__, None) + return globalns, localns + + def _strip_annotations(t): """Strip the annotations from a given type.""" if isinstance(t, _AnnotatedAlias): @@ -2470,7 +2549,7 @@ def get_args(tp): def is_typeddict(tp): - """Check if an annotation is a TypedDict class. + """Check if an object is a TypedDict class. For example:: @@ -2584,10 +2663,10 @@ def _overload_dummy(*args, **kwds): def overload(func): """Decorator for overloaded functions/methods. - In a stub file, place two or more stub definitions for the same - function in a row, each decorated with @overload. - - For example:: + In a non-stub file, place two or more stub definitions for the same + function in a row, each decorated with @overload, followed + by an implementation. The implementation should *not* + be decorated with @overload:: @overload def utf8(value: None) -> None: ... @@ -2595,10 +2674,11 @@ def utf8(value: None) -> None: ... def utf8(value: bytes) -> bytes: ... @overload def utf8(value: str) -> bytes: ... + def utf8(value): + ... # implementation goes here - In a non-stub file (i.e. a regular .py file), do the same but - follow it with an implementation. The implementation should *not* - be decorated with @overload:: + In a stub file or in an abstract method (for example, in a Protocol definition), + the implementation may be omitted:: @overload def utf8(value: None) -> None: ... @@ -2606,8 +2686,6 @@ def utf8(value: None) -> None: ... def utf8(value: bytes) -> bytes: ... @overload def utf8(value: str) -> bytes: ... - def utf8(value): - ... # implementation goes here The overloads for a function can be retrieved at runtime using the get_overloads() function. @@ -2643,7 +2721,7 @@ def final(f): """Decorator to indicate final methods and final classes. Use this decorator to indicate to type checkers that the decorated - method cannot be overridden, and decorated class cannot be subclassed. + method cannot be overridden, and the decorated class cannot be subclassed. For example:: @@ -2685,7 +2763,7 @@ class Other(Leaf): # Error reported by type checker V_co = TypeVar('V_co', covariant=True) # Any type covariant containers. VT_co = TypeVar('VT_co', covariant=True) # Value type covariant containers. T_contra = TypeVar('T_contra', contravariant=True) # Ditto contravariant. -# Internal type variable used for Type[]. +# Internal type bound to class object types. CT_co = TypeVar('CT_co', covariant=True, bound=type) @@ -2730,6 +2808,9 @@ class Other(Leaf): # Error reported by type checker MutableMapping = _alias(collections.abc.MutableMapping, 2) Sequence = _alias(collections.abc.Sequence, 1) MutableSequence = _alias(collections.abc.MutableSequence, 1) +ByteString = _DeprecatedGenericAlias( + collections.abc.ByteString, 0, removal_version=(3, 17) # Not generic. +) # Tuple accepts variable number of parameters. Tuple = _TupleType(tuple, -1, inst=False, name='Tuple') Tuple.__doc__ = \ @@ -2773,7 +2854,7 @@ class TeamUser(User): ... And a function that takes a class argument that's a subclass of User and returns an instance of the corresponding class:: - def new_user[U](user_class: Type[U]) -> U: + def new_user[U](user_class: type[U]) -> U: user = user_class() # (Here we could write the user object to a database) return user @@ -2786,7 +2867,7 @@ def new_user[U](user_class: Type[U]) -> U: @runtime_checkable class SupportsInt(Protocol): - """An ABC with one abstract method __int__.""" + """A protocol with one abstract method __int__.""" __slots__ = () @@ -2797,7 +2878,7 @@ def __int__(self) -> int: @runtime_checkable class SupportsFloat(Protocol): - """An ABC with one abstract method __float__.""" + """A protocol with one abstract method __float__.""" __slots__ = () @@ -2808,7 +2889,7 @@ def __float__(self) -> float: @runtime_checkable class SupportsComplex(Protocol): - """An ABC with one abstract method __complex__.""" + """A protocol with one abstract method __complex__.""" __slots__ = () @@ -2819,7 +2900,7 @@ def __complex__(self) -> complex: @runtime_checkable class SupportsBytes(Protocol): - """An ABC with one abstract method __bytes__.""" + """A protocol with one abstract method __bytes__.""" __slots__ = () @@ -2830,7 +2911,7 @@ def __bytes__(self) -> bytes: @runtime_checkable class SupportsIndex(Protocol): - """An ABC with one abstract method __index__.""" + """A protocol with one abstract method __index__.""" __slots__ = () @@ -2841,7 +2922,7 @@ def __index__(self) -> int: @runtime_checkable class SupportsAbs[T](Protocol): - """An ABC with one abstract method __abs__ that is covariant in its return type.""" + """A protocol with one abstract method __abs__ that is covariant in its return type.""" __slots__ = () @@ -2852,7 +2933,7 @@ def __abs__(self) -> T: @runtime_checkable class SupportsRound[T](Protocol): - """An ABC with one abstract method __round__ that is covariant in its return type.""" + """A protocol with one abstract method __round__ that is covariant in its return type.""" __slots__ = () @@ -2969,7 +3050,7 @@ def annotate(format): def NamedTuple(typename, fields=_sentinel, /, **kwargs): - """Typed version of namedtuple. + """Typed version of collections.namedtuple. Usage:: @@ -2981,8 +3062,8 @@ class Employee(NamedTuple): Employee = collections.namedtuple('Employee', ['name', 'id']) - The resulting class has an extra __annotations__ attribute, giving a - dict that maps field names to types. (The field names are also in + The types for each field name can be retrieved by calling + annotationlib.get_annotations(Employee). (The field names are also in the _fields attribute, which is part of the namedtuple API.) An alternative equivalent functional syntax is also accepted:: @@ -3071,7 +3152,7 @@ def __new__(cls, name, bases, ns, total=True): This method is called when TypedDict is subclassed, or when TypedDict is instantiated. This way - TypedDict supports all three syntax forms described in its docstring. + TypedDict classes can be created through both class-based and functional syntax. Subclasses and instances of TypedDict return actual dictionaries. """ for base in bases: @@ -3084,14 +3165,16 @@ def __new__(cls, name, bases, ns, total=True): else: generic_base = () + ns_annotations = ns.pop('__annotations__', None) + tp_dict = type.__new__(_TypedDictMeta, name, (*generic_base, dict), ns) if not hasattr(tp_dict, '__orig_bases__'): tp_dict.__orig_bases__ = bases - if "__annotations__" in ns: + if ns_annotations is not None: own_annotate = None - own_annotations = ns["__annotations__"] + own_annotations = ns_annotations elif (own_annotate := _lazy_annotationlib.get_annotate_from_class_namespace(ns)) is not None: own_annotations = _lazy_annotationlib.call_annotate_function( own_annotate, _lazy_annotationlib.Format.FORWARDREF, owner=tp_dict @@ -3101,7 +3184,7 @@ def __new__(cls, name, bases, ns, total=True): own_annotations = {} msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type" own_checked_annotations = { - n: _type_check(tp, msg, module=tp_dict.__module__) + n: _type_check(tp, msg, owner=tp_dict, module=tp_dict.__module__) for n, tp in own_annotations.items() } required_keys = set() @@ -3162,7 +3245,7 @@ def __annotate__(format): if base_annotate is None: continue base_annos = _lazy_annotationlib.call_annotate_function( - base.__annotate__, format, owner=base) + base_annotate, format, owner=base) annos.update(base_annos) if own_annotate is not None: own = _lazy_annotationlib.call_annotate_function( @@ -3218,14 +3301,22 @@ def TypedDict(typename, fields=_sentinel, /, *, total=True): >>> Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') True - The type info can be accessed via the Point2D.__annotations__ dict, and - the Point2D.__required_keys__ and Point2D.__optional_keys__ frozensets. + The type info can be accessed by calling annotationlib.get_annotations(Point2D), and + via the Point2D.__required_keys__ and Point2D.__optional_keys__ frozensets. TypedDict supports an additional equivalent form:: Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str}) By default, all keys must be present in a TypedDict. It is possible - to override this by specifying totality:: + to override this by using the NotRequired and Required special forms:: + + class Point2D(TypedDict): + x: int # the "x" key must always be present (Required is the default) + y: NotRequired[int] # the "y" key can be omitted + + This means that a Point2D TypedDict can have the "y" key omitted, but the "x" key must be present. + Items are required by default, so the Required special form is not necessary in this example. + In addition, the total argument to the TypedDict function can be used to make all items not required:: class Point2D(TypedDict, total=False): x: int @@ -3234,16 +3325,8 @@ class Point2D(TypedDict, total=False): This means that a Point2D TypedDict can have any of the keys omitted. A type checker is only expected to support a literal False or True as the value of the total argument. True is the default, and makes all items defined in the - class body be required. - - The Required and NotRequired special forms can also be used to mark - individual keys as being required or not required:: - - class Point2D(TypedDict): - x: int # the "x" key must always be present (Required is the default) - y: NotRequired[int] # the "y" key can be omitted - - See PEP 655 for more details on Required and NotRequired. + class body be required. The Required special form can be used to mark individual + keys as required in a total=False TypedDict. The ReadOnly special form can be used to mark individual keys as immutable for type checkers:: @@ -3252,6 +3335,7 @@ class DatabaseUser(TypedDict): id: ReadOnly[int] # the "id" key must not be modified username: str # the "username" key can be changed + See PEPs 589, 655, and 705 for more information. """ if fields is _sentinel or fields is None: import warnings @@ -3298,7 +3382,7 @@ class Movie(TypedDict, total=False): year: int m = Movie( - title='The Matrix', # typechecker error if key is omitted + title='The Matrix', # type checker error if key is omitted year=1999, ) @@ -3320,7 +3404,7 @@ class Movie(TypedDict): year: NotRequired[int] m = Movie( - title='The Matrix', # typechecker error if key is omitted + title='The Matrix', # type checker error if key is omitted year=1999, ) """ @@ -3340,7 +3424,7 @@ class Movie(TypedDict): def mutate_movie(m: Movie) -> None: m["year"] = 1992 # allowed - m["title"] = "The Matrix" # typechecker error + m["title"] = "The Matrix" # type checker error There is no runtime checking for this property. """ @@ -3427,8 +3511,8 @@ class IO(Generic[AnyStr]): classes (text vs. binary, read vs. write vs. read/write, append-only, unbuffered). The TextIO and BinaryIO subclasses below capture the distinctions between text vs. binary, which is - pervasive in the interface; however we currently do not offer a - way to track the other distinctions in the type system. + pervasive in the interface. For more precise types, define a custom + Protocol. """ __slots__ = () @@ -3518,7 +3602,7 @@ def __exit__(self, type, value, traceback) -> None: class BinaryIO(IO[bytes]): - """Typed version of the return of open() in binary mode.""" + """Typed approximation of the return of open() in binary mode.""" __slots__ = () @@ -3532,7 +3616,7 @@ def __enter__(self) -> BinaryIO: class TextIO(IO[str]): - """Typed version of the return of open() in text mode.""" + """Typed approximation of the return of open() in text mode.""" __slots__ = () @@ -3599,7 +3683,7 @@ def dataclass_transform( field_specifiers: tuple[type[Any] | Callable[..., Any], ...] = (), **kwargs: Any, ) -> _IdentityCallable: - """Decorator to mark an object as providing dataclass-like behaviour. + """Decorator to mark an object as providing dataclass-like behavior. The decorator can be applied to a function, class, or metaclass. diff --git a/Lib/unittest/loader.py b/Lib/unittest/loader.py index a52950dad224ee7..697520246f0e3c6 100644 --- a/Lib/unittest/loader.py +++ b/Lib/unittest/loader.py @@ -1,5 +1,6 @@ """Loading unittests.""" +import inspect import os import re import sys @@ -84,8 +85,10 @@ def loadTestsFromTestCase(self, testCaseClass): raise TypeError("Test cases should not be derived from " "TestSuite. Maybe you meant to derive from " "TestCase?") - if testCaseClass in (case.TestCase, case.FunctionTestCase): - # We don't load any tests from base types that should not be loaded. + if (testCaseClass in (case.TestCase, case.FunctionTestCase) or + inspect.isabstract(testCaseClass)): + # We don't load any tests from base types that should not be loaded, + # and abstract base classes that can't be instantiated testCaseNames = [] else: testCaseNames = self.getTestCaseNames(testCaseClass) @@ -103,6 +106,7 @@ def loadTestsFromModule(self, module, *, pattern=None): isinstance(obj, type) and issubclass(obj, case.TestCase) and obj not in (case.TestCase, case.FunctionTestCase) + and not inspect.isabstract(obj) ): tests.append(self.loadTestsFromTestCase(obj)) @@ -181,6 +185,9 @@ def loadTestsFromName(self, name, module=None): elif (isinstance(obj, types.FunctionType) and isinstance(parent, type) and issubclass(parent, case.TestCase)): + if inspect.isabstract(parent): + raise TypeError( + "Cannot instantiate abstract test case %s" % parent.__name__) name = parts[-1] inst = parent(name) # static methods follow a different path diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index 6fd949581f31467..be99d93c78cca6c 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -269,12 +269,12 @@ def runTests(self): testRunner = self.testRunner self.result = testRunner.run(self.test) if self.exit: - if self.result.testsRun == 0 and len(self.result.skipped) == 0: + if not self.result.wasSuccessful(): + sys.exit(1) + elif self.result.testsRun == 0 and len(self.result.skipped) == 0: sys.exit(_NO_TESTS_EXITCODE) - elif self.result.wasSuccessful(): - sys.exit(0) else: - sys.exit(1) + sys.exit(0) main = TestProgram diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index 55cb4b1f6aff901..f940235a6e50dcb 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -34,6 +34,7 @@ import pkgutil from inspect import iscoroutinefunction import threading +from annotationlib import Format from dataclasses import fields, is_dataclass from types import CodeType, ModuleType, MethodType from unittest.util import safe_repr @@ -119,7 +120,7 @@ def _get_signature_object(func, as_instance, eat_self): else: sig_func = func try: - return func, inspect.signature(sig_func) + return func, inspect.signature(sig_func, annotation_format=Format.FORWARDREF) except ValueError: # Certain callable types are not supported by inspect.signature() return None @@ -569,6 +570,11 @@ def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False, __dict__['_mock_methods'] = spec __dict__['_spec_asyncs'] = _spec_asyncs + def _mock_extend_spec_methods(self, spec_methods): + methods = self.__dict__.get('_mock_methods') or [] + methods.extend(spec_methods) + self.__dict__['_mock_methods'] = methods + def __get_return_value(self): ret = self._mock_return_value if self._mock_delegate is not None: @@ -1175,14 +1181,20 @@ def _mock_call(self, /, *args, **kwargs): def _increment_mock_call(self, /, *args, **kwargs): self.called = True - self.call_count += 1 # handle call_args # needs to be set here so assertions on call arguments pass before # execution in the case of awaited calls - _call = _Call((args, kwargs), two=True) - self.call_args = _call - self.call_args_list.append(_call) + with NonCallableMock._lock: + # Lock is used here so that call_args_list and call_count are + # set atomically otherwise it is possible that by the time call_count + # is set another thread may have appended to call_args_list. + # The rest of this function relies on list.append being atomic and + # skips locking. + _call = _Call((args, kwargs), two=True) + self.call_args = _call + self.call_args_list.append(_call) + self.call_count = len(self.call_args_list) # initial stuff for method_calls: do_method_calls = self._mock_parent is not None @@ -2766,14 +2778,16 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, raise InvalidSpecError(f'Cannot autospec a Mock object. ' f'[object={spec!r}]') is_async_func = _is_async_func(spec) + _kwargs = {'spec': spec} entries = [(entry, _missing) for entry in dir(spec)] if is_type and instance and is_dataclass(spec): + is_dataclass_spec = True dataclass_fields = fields(spec) entries.extend((f.name, f.type) for f in dataclass_fields) - _kwargs = {'spec': [f.name for f in dataclass_fields]} + dataclass_spec_list = [f.name for f in dataclass_fields] else: - _kwargs = {'spec': spec} + is_dataclass_spec = False if spec_set: _kwargs = {'spec_set': spec} @@ -2810,6 +2824,8 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, mock = Klass(parent=_parent, _new_parent=_parent, _new_name=_new_name, name=_name, **_kwargs) + if is_dataclass_spec: + mock._mock_extend_spec_methods(dataclass_spec_list) if isinstance(spec, FunctionTypes): # should only happen at the top level because we don't @@ -2994,7 +3010,7 @@ def _next_side_effect(): return handle.readline.return_value return next(_state[0]) - def _exit_side_effect(exctype, excinst, exctb): + def _exit_side_effect(*args): handle.close() global file_spec @@ -3105,6 +3121,10 @@ def _mock_call(self, *args, **kwargs): return ret_value + def _increment_mock_call(self, /, *args, **kwargs): + with self._mock_calls_events_lock: + super()._increment_mock_call(*args, **kwargs) + def wait_until_called(self, *, timeout=_timeout_unset): """Wait until the mock object is called. diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index 67d9bbea0d31503..c9f8a33e72c39df 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -1,6 +1,6 @@ """Parse (absolute and relative) URLs. -urlparse module is based upon the following RFC specifications. +urllib.parse module is based upon the following RFC specifications. RFC 3986 (STD66): "Uniform Resource Identifiers" by T. Berners-Lee, R. Fielding and L. Masinter, January 2005. @@ -20,7 +20,7 @@ McCahill, December 1994 RFC 3986 is considered the current standard and any future changes to -urlparse module should conform with it. The urlparse module is +urllib.parse module should conform with it. The urllib.parse module is currently not entirely compliant with this RFC due to defacto scenarios for parsing, and for backward compatibility purposes, some parsing quirks from older RFCs are retained. The testcases in @@ -390,6 +390,8 @@ def urlparse(url, scheme='', allow_fragments=True): path or query. Note that % escapes are not expanded. + + urlsplit() should generally be used instead of urlparse(). """ url, scheme, _coerce_result = _coerce_args(url, scheme) scheme, netloc, url, params, query, fragment = _urlparse(url, scheme, allow_fragments) @@ -459,8 +461,8 @@ def _check_bracketed_netloc(netloc): # Valid bracketed hosts are defined in # https://www.rfc-editor.org/rfc/rfc3986#page-49 and https://url.spec.whatwg.org/ def _check_bracketed_host(hostname): - if hostname.startswith('v'): - if not re.match(r"\Av[a-fA-F0-9]+\..+\z", hostname): + if hostname.startswith(('v', 'V')): + if not re.match(r"\A[vV][a-fA-F0-9]+\..+\z", hostname): raise ValueError(f"IPvFuture address is invalid") else: ip = ipaddress.ip_address(hostname) # Throws Value Error if not IPv6 or IPv4 diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index 41dc5d7b35dedb4..a8f40100b7b440c 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -1291,8 +1291,7 @@ def do_open(self, http_class, req, **http_conn_args): h.set_debuglevel(self._debuglevel) headers = dict(req.unredirected_hdrs) - headers.update({k: v for k, v in req.headers.items() - if k not in headers}) + headers.update(req.headers) # TODO(jhylton): Should this be redesigned to handle # persistent connections? @@ -1535,6 +1534,7 @@ def ftp_open(self, req): dirs, file = dirs[:-1], dirs[-1] if dirs and not dirs[0]: dirs = dirs[1:] + fw = None try: fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout) type = file and 'I' or 'D' @@ -1552,8 +1552,12 @@ def ftp_open(self, req): headers += "Content-length: %d\n" % retrlen headers = email.message_from_string(headers) return addinfourl(fp, headers, req.full_url) - except ftplib.all_errors as exp: - raise URLError(f"ftp error: {exp}") from exp + except Exception as exp: + if fw is not None and not fw.keepalive: + fw.close() + if isinstance(exp, ftplib.all_errors): + raise URLError(f"ftp error: {exp}") from exp + raise def connect_ftp(self, user, passwd, host, port, dirs, timeout): return ftpwrapper(user, passwd, host, port, dirs, timeout, @@ -1577,14 +1581,15 @@ def setMaxConns(self, m): def connect_ftp(self, user, passwd, host, port, dirs, timeout): key = user, host, port, '/'.join(dirs), timeout - if key in self.cache: - self.timeout[key] = time.time() + self.delay - else: - self.cache[key] = ftpwrapper(user, passwd, host, port, - dirs, timeout) - self.timeout[key] = time.time() + self.delay + conn = self.cache.get(key) + if conn is None or not conn.keepalive: + if conn is not None: + conn.close() + conn = self.cache[key] = ftpwrapper(user, passwd, host, port, + dirs, timeout) + self.timeout[key] = time.time() + self.delay self.check_cache() - return self.cache[key] + return conn def check_cache(self): # first check for old ones @@ -1628,6 +1633,11 @@ def data_open(self, req): scheme, data = url.split(":",1) mediatype, data = data.split(",",1) + # Disallow control characters within mediatype. + if re.search(r"[\x00-\x1F\x7F]", mediatype): + raise ValueError( + "Control characters not allowed in data: mediatype") + # even base64 encoded data URLs might be quoted so unquote in any case: data = unquote_to_bytes(data) if mediatype.endswith(";base64"): @@ -1654,13 +1664,16 @@ def url2pathname(url, *, require_scheme=False, resolve_host=False): The URL authority may be resolved with gethostbyname() if *resolve_host* is set to true. """ - if require_scheme: - scheme, url = _splittype(url) - if scheme != 'file': - raise URLError("URL is missing a 'file:' scheme") - authority, url = _splithost(url) + if not require_scheme: + url = 'file:' + url + scheme, authority, url = urlsplit(url)[:3] # Discard query and fragment. + if scheme != 'file': + raise URLError("URL is missing a 'file:' scheme") if os.name == 'nt': - if not _is_local_authority(authority, resolve_host): + if authority[1:2] == ':': + # e.g. file://c:/file.txt + url = authority + url + elif not _is_local_authority(authority, resolve_host): # e.g. file://server/share/file.txt url = '//' + authority + url elif url[:3] == '///': diff --git a/Lib/urllib/robotparser.py b/Lib/urllib/robotparser.py index 409f2b2e48de6ee..d267ed00345c490 100644 --- a/Lib/urllib/robotparser.py +++ b/Lib/urllib/robotparser.py @@ -7,10 +7,11 @@ 2) PSF license for Python 2.2 The robots.txt Exclusion Protocol is implemented as specified in - http://www.robotstxt.org/norobots-rfc.txt + RFC 9309 """ import collections +import re import urllib.error import urllib.parse import urllib.request @@ -28,6 +29,7 @@ class RobotFileParser: def __init__(self, url=''): self.entries = [] + self.groups = {} self.sitemaps = [] self.default_entry = None self.disallow_all = False @@ -55,7 +57,7 @@ def modified(self): def set_url(self, url): """Sets the URL referring to a robots.txt file.""" self.url = url - self.host, self.path = urllib.parse.urlparse(url)[1:3] + self.host, self.path = urllib.parse.urlsplit(url)[1:3] def read(self): """Reads the robots.txt URL and feeds it to the parser.""" @@ -63,22 +65,30 @@ def read(self): f = urllib.request.urlopen(self.url) except urllib.error.HTTPError as err: if err.code in (401, 403): + # If access to robot.txt has the status Unauthorized/Forbidden, + # then most likely this applies to the entire site. self.disallow_all = True - elif err.code >= 400 and err.code < 500: + elif 400 <= err.code < 500: + # RFC 9309, Section 2.3.1.3: the crawler MAY access any + # resources on the server. self.allow_all = True + elif 500 <= err.code < 600: + # RFC 9309, Section 2.3.1.4: the crawler MUST assume + # complete disallow. + self.disallow_all = True err.close() else: raw = f.read() - self.parse(raw.decode("utf-8").splitlines()) + self.parse(raw.decode("utf-8", "surrogateescape").splitlines()) def _add_entry(self, entry): - if "*" in entry.useragents: - # the default entry is considered last - if self.default_entry is None: - # the first default entry wins - self.default_entry = entry - else: - self.entries.append(entry) + self.entries.append(entry) + for agent in entry.useragents: + agent = agent.lower() + if agent not in self.groups: + self.groups[agent] = entry + else: + self.groups[agent] = merge_entries(self.groups[agent], entry) def parse(self, lines): """Parse the input lines from a robots.txt file. @@ -86,6 +96,7 @@ def parse(self, lines): We allow that a user-agent: line is not preceded by one or more blank lines. """ + entries = [] # states: # 0: start state # 1: saw user-agent line @@ -95,14 +106,6 @@ def parse(self, lines): self.modified() for line in lines: - if not line: - if state == 1: - entry = Entry() - state = 0 - elif state == 2: - self._add_entry(entry) - entry = Entry() - state = 0 # remove optional comment and strip line i = line.find('#') if i >= 0: @@ -113,35 +116,42 @@ def parse(self, lines): line = line.split(':', 1) if len(line) == 2: line[0] = line[0].strip().lower() - line[1] = urllib.parse.unquote(line[1].strip()) + line[1] = line[1].strip() if line[0] == "user-agent": if state == 2: self._add_entry(entry) entry = Entry() - entry.useragents.append(line[1]) + product_token = line[1] + entry.useragents.append(product_token) state = 1 elif line[0] == "disallow": if state != 0: - entry.rulelines.append(RuleLine(line[1], False)) state = 2 + try: + entry.rulelines.append(RuleLine(line[1], False)) + except ValueError: + pass elif line[0] == "allow": if state != 0: - entry.rulelines.append(RuleLine(line[1], True)) state = 2 + try: + entry.rulelines.append(RuleLine(line[1], True)) + except ValueError: + pass elif line[0] == "crawl-delay": if state != 0: # before trying to convert to int we need to make # sure that robots.txt has valid syntax otherwise # it will crash - if line[1].strip().isdigit(): + if line[1].strip().isdecimal(): entry.delay = int(line[1]) state = 2 elif line[0] == "request-rate": if state != 0: numbers = line[1].split('/') # check if all values are sane - if (len(numbers) == 2 and numbers[0].strip().isdigit() - and numbers[1].strip().isdigit()): + if (len(numbers) == 2 and numbers[0].strip().isdecimal() + and numbers[1].strip().isdecimal()): entry.req_rate = RequestRate(int(numbers[0]), int(numbers[1])) state = 2 elif line[0] == "sitemap": @@ -150,9 +160,18 @@ def parse(self, lines): # so it doesn't matter where you place it in your file." # Therefore we do not change the state of the parser. self.sitemaps.append(line[1]) - if state == 2: + if state != 0: self._add_entry(entry) + def _find_entry(self, useragent): + entry = self.groups.get(useragent.lower()) + if entry is not None: + return entry + for entry in self.groups.values(): + if entry.applies_to(useragent): + return entry + return self.groups.get('*') + def can_fetch(self, useragent, url): """using the parsed robots.txt decide if useragent can fetch url""" if self.disallow_all: @@ -165,42 +184,36 @@ def can_fetch(self, useragent, url): # calls can_fetch() before calling read(). if not self.last_checked: return False - # search for given user agent matches - # the first match counts - parsed_url = urllib.parse.urlparse(urllib.parse.unquote(url)) - url = urllib.parse.urlunparse(('','',parsed_url.path, - parsed_url.params,parsed_url.query, parsed_url.fragment)) - url = urllib.parse.quote(url) + # TODO: The private API is used in order to preserve an empty query. + # This is temporary until the public API starts supporting this feature. + parsed_url = urllib.parse._urlsplit(url, '') + url = urllib.parse._urlunsplit(None, None, *parsed_url[2:]) + url = normalize_uri(url) if not url: url = "/" - for entry in self.entries: - if entry.applies_to(useragent): - return entry.allowance(url) - # try the default entry last - if self.default_entry: - return self.default_entry.allowance(url) - # agent not found ==> access granted - return True + if url == '/robots.txt': + # The /robots.txt URI is implicitly allowed. + return True + entry = self._find_entry(useragent) + if entry is None: + return True + return entry.allowance(url) def crawl_delay(self, useragent): if not self.mtime(): return None - for entry in self.entries: - if entry.applies_to(useragent): - return entry.delay - if self.default_entry: - return self.default_entry.delay - return None + entry = self._find_entry(useragent) + if entry is None: + return None + return entry.delay def request_rate(self, useragent): if not self.mtime(): return None - for entry in self.entries: - if entry.applies_to(useragent): - return entry.req_rate - if self.default_entry: - return self.default_entry.req_rate - return None + entry = self._find_entry(useragent) + if entry is None: + return None + return entry.req_rate def site_maps(self): if not self.sitemaps: @@ -211,8 +224,7 @@ def __str__(self): entries = self.entries if self.default_entry is not None: entries = entries + [self.default_entry] - return '\n\n'.join(map(str, entries)) - + return '\n\n'.join(filter(None, map(str, entries))) class RuleLine: """A rule line is a single "Allow:" (allowance==True) or "Disallow:" @@ -221,15 +233,42 @@ def __init__(self, path, allowance): if path == '' and not allowance: # an empty value means allow all allowance = True - path = urllib.parse.urlunparse(urllib.parse.urlparse(path)) - self.path = urllib.parse.quote(path) + path = re.sub(r'[*]{2,}', '*', path) + path = re.sub(r'[$][$*]+', '$', path) + path = normalize_pattern(path) + self.fullmatch = path.endswith('$') + path = path.rstrip('$') + if '$' in path: + raise ValueError('$ not at the end of path') + self.matcher = None + if '*' in path: + pattern = re.compile(translate_pattern(path), re.DOTALL) + if self.fullmatch: + self.matcher = pattern.fullmatch + else: + self.matcher = pattern.match + self.path = path self.allowance = allowance def applies_to(self, filename): - return self.path == "*" or filename.startswith(self.path) + # If the filename matches the rule, return the matching length plus 1. + # If it does not match, return 0. + if self.matcher is not None: + m = self.matcher(filename) + if m: + return m.end() + 1 + else: + if self.fullmatch: + if filename == self.path: + return len(self.path) + 1 + else: + if filename.startswith(self.path): + return len(self.path) + 1 + return 0 def __str__(self): - return ("Allow" if self.allowance else "Disallow") + ": " + self.path + return (("Allow" if self.allowance else "Disallow") + ": " + self.path + + ('$' if self.fullmatch else '')) class Entry: @@ -241,6 +280,8 @@ def __init__(self): self.req_rate = None def __str__(self): + if not self.useragents: + return '' ret = [] for agent in self.useragents: ret.append(f"User-agent: {agent}") @@ -249,27 +290,74 @@ def __str__(self): if self.req_rate is not None: rate = self.req_rate ret.append(f"Request-rate: {rate.requests}/{rate.seconds}") - ret.extend(map(str, self.rulelines)) + if self.rulelines: + ret.extend(map(str, self.rulelines)) + else: + ret.append("Allow:") return '\n'.join(ret) def applies_to(self, useragent): """check if this entry applies to the specified agent""" + if useragent is None: + return '*' in self.useragents # split the name token and make it lower case useragent = useragent.split("/")[0].lower() for agent in self.useragents: - if agent == '*': - # we have the catch-all agent - return True - agent = agent.lower() - if agent in useragent: - return True + if agent != '*': + agent = agent.lower() + if agent in useragent: + return True return False def allowance(self, filename): """Preconditions: - our agent applies to this entry - - filename is URL decoded""" + - filename is URL encoded + """ + best_match = -1 + allowance = True for line in self.rulelines: - if line.applies_to(filename): - return line.allowance - return True + m = line.applies_to(filename) + if m: + if m > best_match: + best_match = m + allowance = line.allowance + elif m == best_match and not allowance: + allowance = line.allowance + return allowance + + +def normalize(path): + unquoted = urllib.parse.unquote(path, errors='surrogateescape') + return urllib.parse.quote(unquoted, errors='surrogateescape') + +def normalize_uri(path): + path, sep, query = path.partition('?') + path = normalize(path) + if sep: + query = re.sub(r'[^=&]+', lambda m: normalize(m[0]), query) + path += '?' + query + return path + +def normalize_pattern(path): + path, sep, query = path.partition('?') + path = re.sub(r'[^*$]+', lambda m: normalize(m[0]), path) + if sep: + query = re.sub(r'[^=&*$]+', lambda m: normalize(m[0]), query) + path += '?' + query + return path + +def translate_pattern(path): + parts = list(map(re.escape, path.split('*'))) + for i in range(1, len(parts)-1): + parts[i] = f'(?>.*?{parts[i]})' + parts[-1] = f'.*{parts[-1]}' + return ''.join(parts) + +def merge_entries(e1, e2): + entry = Entry() + entry.useragents = list(filter(set(e2.useragents).__contains__, e1.useragents)) + entry.rulelines = e1.rulelines + e2.rulelines + entry.delay = e1.delay if e2.delay is None else e2.delay + entry.req_rate = e1.req_rate if e2.req_rate is None else e2.req_rate + return entry diff --git a/Lib/uuid.py b/Lib/uuid.py index 036ffebf67a0be6..043ad7bcbde8493 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -633,39 +633,43 @@ def _netstat_getnode(): try: import _uuid _generate_time_safe = getattr(_uuid, "generate_time_safe", None) + _has_stable_extractable_node = _uuid.has_stable_extractable_node _UuidCreate = getattr(_uuid, "UuidCreate", None) except ImportError: _uuid = None _generate_time_safe = None + _has_stable_extractable_node = False _UuidCreate = None def _unix_getnode(): """Get the hardware address on Unix using the _uuid extension module.""" - if _generate_time_safe: + if _generate_time_safe and _has_stable_extractable_node: uuid_time, _ = _generate_time_safe() return UUID(bytes=uuid_time).node def _windll_getnode(): """Get the hardware address on Windows using the _uuid extension module.""" - if _UuidCreate: + if _UuidCreate and _has_stable_extractable_node: uuid_bytes = _UuidCreate() return UUID(bytes_le=uuid_bytes).node def _random_getnode(): """Get a random node ID.""" - # RFC 4122, $4.1.6 says "For systems with no IEEE address, a randomly or - # pseudo-randomly generated value may be used; see Section 4.5. The - # multicast bit must be set in such addresses, in order that they will - # never conflict with addresses obtained from network cards." + # RFC 9562, §6.10-3 says that + # + # Implementations MAY elect to obtain a 48-bit cryptographic-quality + # random number as per Section 6.9 to use as the Node ID. [...] [and] + # implementations MUST set the least significant bit of the first octet + # of the Node ID to 1. This bit is the unicast or multicast bit, which + # will never be set in IEEE 802 addresses obtained from network cards. # # The "multicast bit" of a MAC address is defined to be "the least # significant bit of the first octet". This works out to be the 41st bit # counting from 1 being the least significant bit, or 1<<40. # # See https://en.wikipedia.org/w/index.php?title=MAC_address&oldid=1128764812#Universal_vs._local_(U/L_bit) - import random - return random.getrandbits(48) | (1 << 40) + return int.from_bytes(os.urandom(6)) | (1 << 40) # _OS_GETTERS, when known, are targeted for a specific OS or platform. @@ -957,7 +961,7 @@ def main(): default="uuid4", help="function to generate the UUID") parser.add_argument("-n", "--namespace", - choices=["any UUID", *namespaces.keys()], + metavar=f"{{any UUID,{','.join(namespaces)}}}", help="uuid3/uuid5 only: " "a UUID, or a well-known predefined UUID addressed " "by namespace name") @@ -979,7 +983,13 @@ def main(): f"{args.uuid} requires a namespace and a name. " "Run 'python -m uuid -h' for more information." ) - namespace = namespaces[namespace] if namespace in namespaces else UUID(namespace) + if namespace in namespaces: + namespace = namespaces[namespace] + else: + try: + namespace = UUID(namespace) + except ValueError as exc: + parser.error(f"{exc}: {args.namespace!r}") for _ in range(args.count): print(uuid_func(namespace, name)) else: diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 15e15b7a5184c27..88f3340af418342 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -315,7 +315,7 @@ def setup_python(self, context): os.chmod(path, 0o755) suffixes = ['python', 'python3', f'python3.{sys.version_info[1]}'] - if sys.version_info[:2] == (3, 14): + if sys.version_info[:2] == (3, 14) and sys.getfilesystemencoding() == 'utf-8': suffixes.append('𝜋thon') for suffix in suffixes: path = os.path.join(binpath, suffix) @@ -588,7 +588,7 @@ def skip_file(f): 'may be binary: %s', srcfile, e) continue if new_data == data: - shutil.copy2(srcfile, dstfile) + shutil.copy(srcfile, dstfile) else: with open(dstfile, 'wb') as f: f.write(new_data) diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index 70673a265d41f80..241a8650bda33aa 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -17,7 +17,7 @@ deactivate () { # Call hash to forget past locations. Without forgetting # past locations the $PATH changes we made may not be respected. # See "man bash" for more details. hash is usually a builtin of your shell - hash -r 2> /dev/null + hash -r 2> /dev/null || true if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then PS1="${_OLD_VIRTUAL_PS1:-}" @@ -73,4 +73,4 @@ fi # Call hash to forget past commands. Without forgetting # past commands the $PATH changes we made may not be respected -hash -r 2> /dev/null +hash -r 2> /dev/null || true diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat index 35533e4b551155a..5624a3284d42114 100644 --- a/Lib/venv/scripts/nt/activate.bat +++ b/Lib/venv/scripts/nt/activate.bat @@ -15,8 +15,8 @@ if not defined PROMPT set PROMPT=$P$G if defined _OLD_VIRTUAL_PROMPT set PROMPT=%_OLD_VIRTUAL_PROMPT% if defined _OLD_VIRTUAL_PYTHONHOME set PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME% -set "_OLD_VIRTUAL_PROMPT=%PROMPT%" -set "PROMPT=(__VENV_PROMPT__) %PROMPT%" +if not defined VIRTUAL_ENV_DISABLE_PROMPT set "_OLD_VIRTUAL_PROMPT=%PROMPT%" +if not defined VIRTUAL_ENV_DISABLE_PROMPT set "PROMPT=(__VENV_PROMPT__) %PROMPT%" if defined PYTHONHOME set _OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME% set PYTHONHOME= diff --git a/Lib/venv/scripts/posix/activate.csh b/Lib/venv/scripts/posix/activate.csh index 2a3fa835476ab9b..7226034df0f3941 100644 --- a/Lib/venv/scripts/posix/activate.csh +++ b/Lib/venv/scripts/posix/activate.csh @@ -16,10 +16,12 @@ setenv PATH "$VIRTUAL_ENV/"__VENV_BIN_NAME__":$PATH" setenv VIRTUAL_ENV_PROMPT __VENV_PROMPT__ -set _OLD_VIRTUAL_PROMPT="$prompt" +if ($?prompt) then + set _OLD_VIRTUAL_PROMPT="$prompt" -if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then - set prompt = "("__VENV_PROMPT__") $prompt:q" + if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then + set prompt = "("__VENV_PROMPT__") $prompt:q" + endif endif alias pydoc python -m pydoc diff --git a/Lib/wave.py b/Lib/wave.py index a34af244c3e2244..b8476e264868fc7 100644 --- a/Lib/wave.py +++ b/Lib/wave.py @@ -441,6 +441,8 @@ class Wave_write: _datawritten -- the size of the audio samples actually written """ + _file = None + def __init__(self, f): self._i_opened_the_file = None if isinstance(f, str): diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index f2e2394089d5a16..97aad6eea509eb5 100644 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -163,6 +163,12 @@ def open_new(self, url): def open_new_tab(self, url): return self.open(url, 2) + @staticmethod + def _check_url(url): + """Ensures that the URL is safe to pass to subprocesses as a parameter""" + if url and url.lstrip().startswith("-"): + raise ValueError(f"Invalid URL (leading dash disallowed): {url!r}") + class GenericBrowser(BaseBrowser): """Class for all browsers started with a command @@ -180,6 +186,7 @@ def __init__(self, name): def open(self, url, new=0, autoraise=True): sys.audit("webbrowser.open", url) + self._check_url(url) cmdline = [self.name] + [arg.replace("%s", url) for arg in self.args] try: @@ -200,6 +207,7 @@ def open(self, url, new=0, autoraise=True): cmdline = [self.name] + [arg.replace("%s", url) for arg in self.args] sys.audit("webbrowser.open", url) + self._check_url(url) try: if sys.platform[:3] == 'win': p = subprocess.Popen(cmdline) @@ -279,7 +287,9 @@ def open(self, url, new=0, autoraise=True): raise Error("Bad 'new' parameter to open(); " f"expected 0, 1, or 2, got {new}") - args = [arg.replace("%s", url).replace("%action", action) + self._check_url(url.replace("%action", action)) + + args = [arg.replace("%action", action).replace("%s", url) for arg in self.remote_args] args = [arg for arg in args if arg] success = self._invoke(args, True, autoraise, url) @@ -357,6 +367,7 @@ class Konqueror(BaseBrowser): def open(self, url, new=0, autoraise=True): sys.audit("webbrowser.open", url) + self._check_url(url) # XXX Currently I know no way to prevent KFM from opening a new win. if new == 2: action = "newTab" @@ -588,6 +599,7 @@ def register_standard_browsers(): class WindowsDefault(BaseBrowser): def open(self, url, new=0, autoraise=True): sys.audit("webbrowser.open", url) + self._check_url(url) try: os.startfile(url) except OSError: @@ -608,6 +620,7 @@ def __init__(self, name='default'): def open(self, url, new=0, autoraise=True): sys.audit("webbrowser.open", url) + self._check_url(url) url = url.replace('"', '%22') if self.name == 'default': proto, _sep, _rest = url.partition(":") @@ -644,7 +657,7 @@ def open(self, url, new=0, autoraise=True): end ''' - osapipe = os.popen("osascript", "w") + osapipe = os.popen("/usr/bin/osascript", "w") if osapipe is None: return False @@ -664,6 +677,7 @@ def open(self, url, new=0, autoraise=True): class IOSBrowser(BaseBrowser): def open(self, url, new=0, autoraise=True): sys.audit("webbrowser.open", url) + self._check_url(url) # If ctypes isn't available, we can't open a browser if objc is None: return False diff --git a/Lib/wsgiref/handlers.py b/Lib/wsgiref/handlers.py index cafe872c7aae9b0..f8fe89f6e13436b 100644 --- a/Lib/wsgiref/handlers.py +++ b/Lib/wsgiref/handlers.py @@ -1,7 +1,7 @@ """Base classes for server/gateway implementations""" from .util import FileWrapper, guess_scheme, is_hop_by_hop -from .headers import Headers +from .headers import Headers, _name_disallowed_re import sys, os, time @@ -249,6 +249,8 @@ def start_response(self, status, headers,exc_info=None): return self.write def _validate_status(self, status): + if _name_disallowed_re.search(status): + raise ValueError("Control characters are not allowed in status") if len(status) < 4: raise AssertionError("Status must be at least 4 characters") if not status[:3].isdigit(): diff --git a/Lib/wsgiref/headers.py b/Lib/wsgiref/headers.py index c78879f80c7df24..eb6ea6a412dcc90 100644 --- a/Lib/wsgiref/headers.py +++ b/Lib/wsgiref/headers.py @@ -9,6 +9,11 @@ # existence of which force quoting of the parameter value. import re tspecials = re.compile(r'[ \(\)<>@,;:\\"/\[\]\?=]') +# Disallowed characters for headers and values. +# HTAB (\x09) is allowed in header values, but +# not in header names. (RFC 9110 Section 5.5) +_name_disallowed_re = re.compile(r'[\x00-\x1F\x7F]') +_value_disallowed_re = re.compile(r'[\x00-\x08\x0A-\x1F\x7F]') def _formatparam(param, value=None, quote=1): """Convenience function to format and return a key=value pair. @@ -35,12 +40,15 @@ def __init__(self, headers=None): self._headers = headers if __debug__: for k, v in headers: - self._convert_string_type(k) - self._convert_string_type(v) + self._convert_string_type(k, name=True) + self._convert_string_type(v, name=False) - def _convert_string_type(self, value): + def _convert_string_type(self, value, *, name): """Convert/check value type.""" if type(value) is str: + regex = (_name_disallowed_re if name else _value_disallowed_re) + if regex.search(value): + raise ValueError("Control characters not allowed in headers") return value raise AssertionError("Header names/values must be" " of type str (got {0})".format(repr(value))) @@ -53,14 +61,14 @@ def __setitem__(self, name, val): """Set the value of a header.""" del self[name] self._headers.append( - (self._convert_string_type(name), self._convert_string_type(val))) + (self._convert_string_type(name, name=True), self._convert_string_type(val, name=False))) def __delitem__(self,name): """Delete all occurrences of a header, if present. Does *not* raise an exception if the header is missing. """ - name = self._convert_string_type(name.lower()) + name = self._convert_string_type(name.lower(), name=True) self._headers[:] = [kv for kv in self._headers if kv[0].lower() != name] def __getitem__(self,name): @@ -87,13 +95,13 @@ def get_all(self, name): fields deleted and re-inserted are always appended to the header list. If no fields exist with the given name, returns an empty list. """ - name = self._convert_string_type(name.lower()) + name = self._convert_string_type(name.lower(), name=True) return [kv[1] for kv in self._headers if kv[0].lower()==name] def get(self,name,default=None): """Get the first header value for 'name', or return 'default'""" - name = self._convert_string_type(name.lower()) + name = self._convert_string_type(name.lower(), name=True) for k,v in self._headers: if k.lower()==name: return v @@ -148,8 +156,8 @@ def setdefault(self,name,value): and value 'value'.""" result = self.get(name) if result is None: - self._headers.append((self._convert_string_type(name), - self._convert_string_type(value))) + self._headers.append((self._convert_string_type(name, name=True), + self._convert_string_type(value, name=False))) return value else: return result @@ -172,13 +180,13 @@ def add_header(self, _name, _value, **_params): """ parts = [] if _value is not None: - _value = self._convert_string_type(_value) + _value = self._convert_string_type(_value, name=False) parts.append(_value) for k, v in _params.items(): - k = self._convert_string_type(k) + k = self._convert_string_type(k, name=True) if v is None: parts.append(k.replace('_', '-')) else: - v = self._convert_string_type(v) + v = self._convert_string_type(v, name=False) parts.append(_formatparam(k.replace('_', '-'), v)) - self._headers.append((self._convert_string_type(_name), "; ".join(parts))) + self._headers.append((self._convert_string_type(_name, name=True), "; ".join(parts))) diff --git a/Lib/xml/dom/minidom.py b/Lib/xml/dom/minidom.py index db51f350ea0153d..16b33b90184dc59 100644 --- a/Lib/xml/dom/minidom.py +++ b/Lib/xml/dom/minidom.py @@ -292,13 +292,6 @@ def _append_child(self, node): childNodes.append(node) node.parentNode = self -def _in_document(node): - # return True iff node is part of a document tree - while node is not None: - if node.nodeType == Node.DOCUMENT_NODE: - return True - node = node.parentNode - return False def _write_data(writer, text, attr): "Writes datachars to writer." @@ -371,6 +364,7 @@ class Attr(Node): def __init__(self, qName, namespaceURI=EMPTY_NAMESPACE, localName=None, prefix=None): self.ownerElement = None + self.ownerDocument = None self._name = qName self.namespaceURI = namespaceURI self._prefix = prefix @@ -696,6 +690,7 @@ class Element(Node): def __init__(self, tagName, namespaceURI=EMPTY_NAMESPACE, prefix=None, localName=None): + self.ownerDocument = None self.parentNode = None self.tagName = self.nodeName = tagName self.prefix = prefix @@ -1555,7 +1550,7 @@ def _clear_id_cache(node): if node.nodeType == Node.DOCUMENT_NODE: node._id_cache.clear() node._id_search_stack = None - elif _in_document(node): + elif node.ownerDocument: node.ownerDocument._id_cache.clear() node.ownerDocument._id_search_stack= None diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 44ab5d18624e733..0a4203d372ce991 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -8,8 +8,8 @@ 2. Element represents a single node in this tree. Interactions with the whole document (reading and writing to/from files) are - usually done on the ElementTree level. Interactions with a single XML element - and its sub-elements are done on the Element level. + usually done on the ElementTree level. Interactions with a single XML + element and its sub-elements are done on the Element level. Element is a flexible container object designed to store hierarchical data structures in memory. It can be described as a cross between a list and a @@ -277,7 +277,8 @@ def find(self, path, namespaces=None): """Find first matching element by tag name or path. *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return the first matching element, or None if no element was found. @@ -289,7 +290,8 @@ def findtext(self, path, default=None, namespaces=None): *path* is a string having either an element tag or an XPath, *default* is the value to return if the element was not found, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return text content of first matching element, or default value if none was found. Note that if an element is found having no text @@ -302,7 +304,8 @@ def findall(self, path, namespaces=None): """Find all matching subelements by tag name or path. *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Returns list containing all matching elements in document order. @@ -313,7 +316,8 @@ def iterfind(self, path, namespaces=None): """Find all matching subelements by tag name or path. *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return an iterable yielding all matching elements in document order. @@ -527,7 +531,9 @@ class ElementTree: """ def __init__(self, element=None, file=None): - # assert element is None or iselement(element) + if element is not None and not iselement(element): + raise TypeError('expected an Element, not %s' % + type(element).__name__) self._root = element # first node if file: self.parse(file) @@ -543,14 +549,16 @@ def _setroot(self, element): with the given element. Use with care! """ - # assert iselement(element) + if not iselement(element): + raise TypeError('expected an Element, not %s' + % type(element).__name__) self._root = element def parse(self, source, parser=None): """Load external XML document into element tree. - *source* is a file name or file object, *parser* is an optional parser - instance that defaults to XMLParser. + *source* is a file name or file object, *parser* is an optional + parser instance that defaults to XMLParser. ParseError is raised if the parser fails to parse the document. @@ -583,7 +591,8 @@ def parse(self, source, parser=None): def iter(self, tag=None): """Create and return tree iterator for the root element. - The iterator loops over all elements in this tree, in document order. + The iterator loops over all elements in this tree, in document + order. *tag* is a string with the tag name to iterate over (default is to return all elements). @@ -598,7 +607,8 @@ def find(self, path, namespaces=None): Same as getroot().find(path), which is Element.find() *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return the first matching element, or None if no element was found. @@ -620,7 +630,8 @@ def findtext(self, path, default=None, namespaces=None): Same as getroot().findtext(path), which is Element.findtext() *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return the first matching element, or None if no element was found. @@ -642,7 +653,8 @@ def findall(self, path, namespaces=None): Same as getroot().findall(path), which is Element.findall(). *path* is a string having either an element tag or an XPath, - *namespaces* is an optional mapping from namespace prefix to full name. + *namespaces* is an optional mapping from namespace prefix to full + name. Return list containing all matching elements in document order. @@ -689,26 +701,30 @@ def write(self, file_or_filename, """Write element tree to a file as XML. Arguments: - *file_or_filename* -- file name or a file object opened for writing + *file_or_filename* -- file name or a file object opened for + writing *encoding* -- the output encoding (default: US-ASCII) - *xml_declaration* -- bool indicating if an XML declaration should be - added to the output. If None, an XML declaration - is added if encoding IS NOT either of: - US-ASCII, UTF-8, or Unicode + *xml_declaration* -- bool indicating if an XML declaration should + be added to the output. If None, an XML + declaration is added if encoding IS NOT + either of: US-ASCII, UTF-8, or Unicode - *default_namespace* -- sets the default XML namespace (for "xmlns") + *default_namespace* -- sets the default XML namespace (for + "xmlns") *method* -- either "xml" (default), "html, "text", or "c14n" *short_empty_elements* -- controls the formatting of elements - that contain no content. If True (default) - they are emitted as a single self-closed - tag, otherwise they are emitted as a pair - of start/end tags + that contain no content. If True + (default) they are emitted as a single + self-closed tag, otherwise they are + emitted as a pair of start/end tags """ + if self._root is None: + raise TypeError('ElementTree not initialized') if not method: method = "xml" elif method not in _serialize: @@ -901,9 +917,12 @@ def _serialize_xml(write, elem, qnames, namespaces, if elem.tail: write(_escape_cdata(elem.tail)) +_CDATA_CONTENT_ELEMENTS = {"script", "style", "xmp", "iframe", "noembed", + "noframes", "plaintext"} + HTML_EMPTY = {"area", "base", "basefont", "br", "col", "embed", "frame", "hr", "img", "input", "isindex", "link", "meta", "param", "source", - "track", "wbr"} + "track", "wbr", "plaintext"} def _serialize_html(write, elem, qnames, namespaces, **kwargs): tag = elem.tag @@ -944,7 +963,7 @@ def _serialize_html(write, elem, qnames, namespaces, **kwargs): write(">") ltag = tag.lower() if text: - if ltag == "script" or ltag == "style": + if ltag in _CDATA_CONTENT_ELEMENTS: write(text) else: write(_escape_cdata(text)) @@ -1077,9 +1096,9 @@ def tostring(element, encoding=None, method=None, *, is returned. Otherwise a bytestring is returned. *element* is an Element instance, *encoding* is an optional output - encoding defaulting to US-ASCII, *method* is an optional output which can - be one of "xml" (default), "html", "text" or "c14n", *default_namespace* - sets the default XML namespace (for "xmlns"). + encoding defaulting to US-ASCII, *method* is an optional output which + can be one of "xml" (default), "html", "text" or "c14n", + *default_namespace* sets the default XML namespace (for "xmlns"). Returns an (optionally) encoded string containing the XML data. @@ -1219,7 +1238,8 @@ def iterparse(source, events=None, parser=None): "end" events are reported. *source* is a filename or file object containing XML data, *events* is - a list of events to report back, *parser* is an optional parser instance. + a list of events to report back, *parser* is an optional parser + instance. Returns an iterator providing (event, elem) pairs. @@ -1751,10 +1771,11 @@ def flush(self): def canonicalize(xml_data=None, *, out=None, from_file=None, **options): """Convert XML to its C14N 2.0 serialised form. - If *out* is provided, it must be a file or file-like object that receives - the serialised canonical XML output (text, not bytes) through its ``.write()`` - method. To write to a file, open it in text mode with encoding "utf-8". - If *out* is not provided, this function returns the output as text string. + If *out* is provided, it must be a file or file-like object that + receives the serialised canonical XML output (text, not bytes) through + its ``.write()`` method. To write to a file, open it in text mode with + encoding "utf-8". If *out* is not provided, this function returns the + output as text string. Either *xml_data* (an XML string) or *from_file* (a file path or file-like object) must be provided as input. @@ -1788,19 +1809,22 @@ class C14NWriterTarget: Serialises parse events to XML C14N 2.0. The *write* function is used for writing out the resulting data stream - as text (not bytes). To write to a file, open it in text mode with encoding - "utf-8" and pass its ``.write`` method. + as text (not bytes). To write to a file, open it in text mode with + encoding "utf-8" and pass its ``.write`` method. Configuration options: - *with_comments*: set to true to include comments - - *strip_text*: set to true to strip whitespace before and after text content - - *rewrite_prefixes*: set to true to replace namespace prefixes by "n{number}" + - *strip_text*: set to true to strip whitespace before and after text + content + - *rewrite_prefixes*: set to true to replace namespace prefixes by + "n{number}" - *qname_aware_tags*: a set of qname aware tag names in which prefixes should be replaced in text content - - *qname_aware_attrs*: a set of qname aware attribute names in which prefixes - should be replaced in text content - - *exclude_attrs*: a set of attribute names that should not be serialised + - *qname_aware_attrs*: a set of qname aware attribute names in which + prefixes should be replaced in text content + - *exclude_attrs*: a set of attribute names that should not be + serialised - *exclude_tags*: a set of tag names that should not be serialised """ def __init__(self, write, *, diff --git a/Lib/xmlrpc/client.py b/Lib/xmlrpc/client.py index f441376d09c4aa2..84e4e4d11a7319e 100644 --- a/Lib/xmlrpc/client.py +++ b/Lib/xmlrpc/client.py @@ -965,7 +965,7 @@ def dumps(params, methodname=None, methodresponse=None, encoding=None, data = ( xmlheader, "<methodCall>\n" - "<methodName>", methodname, "</methodName>\n", + "<methodName>", escape(methodname), "</methodName>\n", data, "</methodCall>\n" ) diff --git a/Lib/xmlrpc/server.py b/Lib/xmlrpc/server.py index 90a356fbb8eae45..3e6871157d09299 100644 --- a/Lib/xmlrpc/server.py +++ b/Lib/xmlrpc/server.py @@ -239,7 +239,7 @@ def register_multicall_functions(self): see http://www.xmlrpc.com/discuss/msgReader$1208""" - self.funcs.update({'system.multicall' : self.system_multicall}) + self.funcs['system.multicall'] = self.system_multicall def _marshaled_dispatch(self, data, dispatch_method = None, path = None): """Dispatches an XML-RPC method from marshalled (XML) data. @@ -578,7 +578,7 @@ class SimpleXMLRPCServer(socketserver.TCPServer, """ allow_reuse_address = True - allow_reuse_port = True + allow_reuse_port = False # Warning: this is for debugging purposes only! Never set this to True in # production code, as will be sending out sensitive information (exception diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index 88356abe8cbaebb..2b6e6163b96592d 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -38,8 +38,8 @@ __all__ = ["BadZipFile", "BadZipfile", "error", "ZIP_STORED", "ZIP_DEFLATED", "ZIP_BZIP2", "ZIP_LZMA", - "is_zipfile", "ZipInfo", "ZipFile", "PyZipFile", "LargeZipFile", - "Path"] + "ZIP_ZSTANDARD", "is_zipfile", "ZipInfo", "ZipFile", "PyZipFile", + "LargeZipFile", "Path"] class BadZipFile(Exception): pass @@ -234,8 +234,19 @@ def strip(cls, data, xids): def _check_zipfile(fp): try: - if _EndRecData(fp): - return True # file has correct magic number + endrec = _EndRecData(fp) + if endrec: + if endrec[_ECD_ENTRIES_TOTAL] == 0 and endrec[_ECD_SIZE] == 0 and endrec[_ECD_OFFSET] == 0: + return True # Empty zipfiles are still zipfiles + elif endrec[_ECD_DISK_NUMBER] == endrec[_ECD_DISK_START]: + # Central directory is on the same disk + fp.seek(sum(_handle_prepended_data(endrec))) + if endrec[_ECD_SIZE] >= sizeCentralDir: + data = fp.read(sizeCentralDir) # CD is where we expect it to be + if len(data) == sizeCentralDir: + centdir = struct.unpack(structCentralDir, data) # CD is the right size + if centdir[_CD_SIGNATURE] == stringCentralDir: + return True # First central directory entry has correct magic number except OSError: pass return False @@ -254,24 +265,36 @@ def is_zipfile(filename): else: with open(filename, "rb") as fp: result = _check_zipfile(fp) - except OSError: + except (OSError, BadZipFile): pass return result +def _handle_prepended_data(endrec, debug=0): + size_cd = endrec[_ECD_SIZE] # bytes in central directory + offset_cd = endrec[_ECD_OFFSET] # offset of central directory + + # "concat" is zero, unless zip was concatenated to another file + concat = endrec[_ECD_LOCATION] - size_cd - offset_cd + + if debug > 2: + inferred = concat + offset_cd + print("given, inferred, offset", offset_cd, inferred, concat) + + return offset_cd, concat + def _EndRecData64(fpin, offset, endrec): """ Read the ZIP64 end-of-archive records and use that to update endrec """ - try: - fpin.seek(offset - sizeEndCentDir64Locator, 2) - except OSError: - # If the seek fails, the file is not large enough to contain a ZIP64 + offset -= sizeEndCentDir64Locator + if offset < 0: + # The file is not large enough to contain a ZIP64 # end-of-archive record, so just return the end record we were given. return endrec - + fpin.seek(offset) data = fpin.read(sizeEndCentDir64Locator) if len(data) != sizeEndCentDir64Locator: - return endrec + raise OSError("Unknown I/O error") sig, diskno, reloff, disks = struct.unpack(structEndArchive64Locator, data) if sig != stringEndArchive64Locator: return endrec @@ -279,16 +302,33 @@ def _EndRecData64(fpin, offset, endrec): if diskno != 0 or disks > 1: raise BadZipFile("zipfiles that span multiple disks are not supported") - # Assume no 'zip64 extensible data' - fpin.seek(offset - sizeEndCentDir64Locator - sizeEndCentDir64, 2) + offset -= sizeEndCentDir64 + if reloff > offset: + raise BadZipFile("Corrupt zip64 end of central directory locator") + # First, check the assumption that there is no prepended data. + fpin.seek(reloff) + extrasz = offset - reloff data = fpin.read(sizeEndCentDir64) if len(data) != sizeEndCentDir64: - return endrec + raise OSError("Unknown I/O error") + if not data.startswith(stringEndArchive64) and reloff != offset: + # Since we already have seen the Zip64 EOCD Locator, it's + # possible we got here because there is prepended data. + # Assume no 'zip64 extensible data' + fpin.seek(offset) + extrasz = 0 + data = fpin.read(sizeEndCentDir64) + if len(data) != sizeEndCentDir64: + raise OSError("Unknown I/O error") + if not data.startswith(stringEndArchive64): + raise BadZipFile("Zip64 end of central directory record not found") + sig, sz, create_version, read_version, disk_num, disk_dir, \ dircount, dircount2, dirsize, diroffset = \ struct.unpack(structEndArchive64, data) - if sig != stringEndArchive64: - return endrec + if (diroffset + dirsize != reloff or + sz + 12 != sizeEndCentDir64 + extrasz): + raise BadZipFile("Corrupt zip64 end of central directory record") # Update the original endrec using data from the ZIP64 record endrec[_ECD_SIGNATURE] = sig @@ -298,6 +338,7 @@ def _EndRecData64(fpin, offset, endrec): endrec[_ECD_ENTRIES_TOTAL] = dircount2 endrec[_ECD_SIZE] = dirsize endrec[_ECD_OFFSET] = diroffset + endrec[_ECD_LOCATION] = offset - extrasz return endrec @@ -331,7 +372,7 @@ def _EndRecData(fpin): endrec.append(filesize - sizeEndCentDir) # Try to read the "Zip64 end of central directory" structure - return _EndRecData64(fpin, -sizeEndCentDir, endrec) + return _EndRecData64(fpin, filesize - sizeEndCentDir, endrec) # Either this is not a ZIP file, or it is a ZIP file with an archive # comment. Search the end of the file for the "end of central directory" @@ -355,8 +396,7 @@ def _EndRecData(fpin): endrec.append(maxCommentStart + start) # Try to read the "Zip64 end of central directory" structure - return _EndRecData64(fpin, maxCommentStart + start - filesize, - endrec) + return _EndRecData64(fpin, maxCommentStart + start, endrec) # Unable to find a valid end of central directory structure return None @@ -526,8 +566,12 @@ def FileHeader(self, zip64=None): return header + filename + extra def _encodeFilenameFlags(self): + if self.flag_bits & _MASK_UTF_FILENAME: + encoding = 'ascii' + else: + encoding = 'cp437' try: - return self.filename.encode('ascii'), self.flag_bits + return self.filename.encode(encoding), self.flag_bits & ~_MASK_UTF_FILENAME except UnicodeEncodeError: return self.filename.encode('utf-8'), self.flag_bits | _MASK_UTF_FILENAME @@ -580,11 +624,12 @@ def _decodeExtra(self, filename_crc): def from_file(cls, filename, arcname=None, *, strict_timestamps=True): """Construct an appropriate ZipInfo for a file on the filesystem. - filename should be the path to a file or directory on the filesystem. + filename should be the path to a file or directory on the + filesystem. - arcname is the name which it will have within the archive (by default, - this will be the same as filename, but without a drive letter and with - leading path separators removed). + arcname is the name which it will have within the archive (by + default, this will be the same as filename, but without a drive + letter and with leading path separators removed). """ if isinstance(filename, os.PathLike): filename = os.fspath(filename) @@ -623,9 +668,12 @@ def _for_archive(self, archive): Return self. """ # gh-91279: Set the SOURCE_DATE_EPOCH to a specific timestamp - epoch = os.environ.get('SOURCE_DATE_EPOCH') - get_time = int(epoch) if epoch else time.time() - self.date_time = time.localtime(get_time)[:6] + source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH') + + if source_date_epoch: + self.date_time = time.gmtime(int(source_date_epoch))[:6] + else: + self.date_time = time.localtime(time.time())[:6] self.compress_type = archive.compression self.compress_level = archive.compresslevel @@ -812,11 +860,11 @@ def _get_compressor(compress_type, compresslevel=None): if compresslevel is not None: return bz2.BZ2Compressor(compresslevel) return bz2.BZ2Compressor() - # compresslevel is ignored for ZIP_LZMA and ZIP_ZSTANDARD + # compresslevel is ignored for ZIP_LZMA elif compress_type == ZIP_LZMA: return LZMACompressor() elif compress_type == ZIP_ZSTANDARD: - return zstd.ZstdCompressor() + return zstd.ZstdCompressor(level=compresslevel) else: return None @@ -910,7 +958,7 @@ class ZipExtFile(io.BufferedIOBase): """ # Max size supported by decompressor. - MAX_N = 1 << 31 - 1 + MAX_N = (1 << 31) - 1 # Read from compressed files in 4k blocks. MIN_READ_SIZE = 4096 @@ -1352,25 +1400,30 @@ class ZipFile: mode: The mode can be either read 'r', write 'w', exclusive create 'x', or append 'a'. compression: ZIP_STORED (no compression), ZIP_DEFLATED (requires zlib), - ZIP_BZIP2 (requires bz2) or ZIP_LZMA (requires lzma). - allowZip64: if True ZipFile will create files with ZIP64 extensions when - needed, otherwise it will raise an exception when this would - be necessary. - compresslevel: None (default for the given compression type) or an integer - specifying the level to pass to the compressor. - When using ZIP_STORED or ZIP_LZMA this keyword has no effect. - When using ZIP_DEFLATED integers 0 through 9 are accepted. - When using ZIP_BZIP2 integers 1 through 9 are accepted. + ZIP_BZIP2 (requires bz2), ZIP_LZMA (requires lzma), or + ZIP_ZSTANDARD (requires compression.zstd). + allowZip64: if True ZipFile will create files with ZIP64 extensions + when needed, otherwise it will raise an exception when this + would be necessary. + compresslevel: None (default for the given compression type) or + an integer specifying the level to pass to the compressor. + When using ZIP_STORED or ZIP_LZMA this keyword has no effect. + When using ZIP_DEFLATED integers 0 through 9 are accepted. + When using ZIP_BZIP2 integers 1 through 9 are accepted. + When using ZIP_ZSTANDARD integers -7 though 22 are common, + see the CompressionParameter enum in compression.zstd for + details. """ fp = None # Set here since __del__ checks it _windows_illegal_name_trans_table = None + _ignore_invalid_names = False def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, compresslevel=None, *, strict_timestamps=True, metadata_encoding=None): - """Open the ZIP file with mode read 'r', write 'w', exclusive create 'x', - or append 'a'.""" + """Open the ZIP file with mode read 'r', write 'w', exclusive create + 'x', or append 'a'.""" if mode not in ('r', 'w', 'x', 'a'): raise ValueError("ZipFile requires mode 'r', 'w', 'x', or 'a'") @@ -1421,7 +1474,6 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, self._lock = threading.RLock() self._seekable = True self._writing = False - self._data_offset = None try: if mode == 'r': @@ -1432,7 +1484,6 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, self._didModify = True try: self.start_dir = self.fp.tell() - self._data_offset = self.start_dir except (AttributeError, OSError): self.fp = _Tellable(self.fp) self.start_dir = 0 @@ -1457,7 +1508,6 @@ def __init__(self, file, mode="r", compression=ZIP_STORED, allowZip64=True, # even if no files are added to the archive self._didModify = True self.start_dir = self.fp.tell() - self._data_offset = self.start_dir else: raise ValueError("Mode must be 'r', 'w', 'x', or 'a'") except: @@ -1497,28 +1547,17 @@ def _RealGetContents(self): raise BadZipFile("File is not a zip file") if self.debug > 1: print(endrec) - size_cd = endrec[_ECD_SIZE] # bytes in central directory - offset_cd = endrec[_ECD_OFFSET] # offset of central directory self._comment = endrec[_ECD_COMMENT] # archive comment - # "concat" is zero, unless zip was concatenated to another file - concat = endrec[_ECD_LOCATION] - size_cd - offset_cd - if endrec[_ECD_SIGNATURE] == stringEndArchive64: - # If Zip64 extension structures are present, account for them - concat -= (sizeEndCentDir64 + sizeEndCentDir64Locator) + offset_cd, concat = _handle_prepended_data(endrec, self.debug) - # store the offset to the beginning of data for the - # .data_offset property - self._data_offset = concat - - if self.debug > 2: - inferred = concat + offset_cd - print("given, inferred, offset", offset_cd, inferred, concat) # self.start_dir: Position of start of central directory self.start_dir = offset_cd + concat + if self.start_dir < 0: raise BadZipFile("Bad offset for central directory") fp.seek(self.start_dir, 0) + size_cd = endrec[_ECD_SIZE] data = fp.read(size_cd) fp = io.BytesIO(data) total = 0 @@ -1575,12 +1614,6 @@ def _RealGetContents(self): zinfo._end_offset = end_offset end_offset = zinfo.header_offset - @property - def data_offset(self): - """The offset to the start of zip data in the file or None if - unavailable.""" - return self._data_offset - def namelist(self): """Return a list of file names in the archive.""" return [data.filename for data in self.filelist] @@ -1668,10 +1701,10 @@ def open(self, name, mode="r", pwd=None, *, force_zip64=False): pwd is the password to decrypt files (only used for reading). - When writing, if the file size is not known in advance but may exceed - 2 GiB, pass force_zip64 to use the ZIP64 format, which can handle large - files. If the size is known in advance, it is best to pass a ZipInfo - instance for name, with zinfo.file_size set. + When writing, if the file size is not known in advance but may + exceed 2 GiB, pass force_zip64 to use the ZIP64 format, which can + handle large files. If the size is known in advance, it is best to + pass a ZipInfo instance for name, with zinfo.file_size set. """ if mode not in {"r", "w"}: raise ValueError('open() requires mode "r" or "w"') @@ -1783,7 +1816,7 @@ def _open_to_write(self, zinfo, force_zip64=False): zinfo.compress_size = 0 zinfo.CRC = 0 - zinfo.flag_bits = 0x00 + zinfo.flag_bits = _MASK_UTF_FILENAME if zinfo.compress_type == ZIP_LZMA: # Compressed data includes an end-of-stream (EOS) marker zinfo.flag_bits |= _MASK_COMPRESS_OPTION_1 @@ -1866,21 +1899,31 @@ def _extract_member(self, member, targetpath, pwd): # build the destination pathname, replacing # forward slashes to platform specific separators. - arcname = member.filename.replace('/', os.path.sep) - - if os.path.altsep: + arcname = member.filename + if os.path.sep != '/': + arcname = arcname.replace('/', os.path.sep) + if os.path.altsep and os.path.altsep != '/': arcname = arcname.replace(os.path.altsep, os.path.sep) # interpret absolute pathname as relative, remove drive letter or # UNC path, redundant separators, "." and ".." components. - arcname = os.path.splitdrive(arcname)[1] + drive, root, arcname = os.path.splitroot(arcname) + if self._ignore_invalid_names and (drive or root): + return None + if self._ignore_invalid_names and os.path.pardir in arcname.split(os.path.sep): + return None invalid_path_parts = ('', os.path.curdir, os.path.pardir) arcname = os.path.sep.join(x for x in arcname.split(os.path.sep) if x not in invalid_path_parts) if os.path.sep == '\\': # filter illegal characters on Windows - arcname = self._sanitize_windows_name(arcname, os.path.sep) + arcname2 = self._sanitize_windows_name(arcname, os.path.sep) + if self._ignore_invalid_names and arcname2 != arcname: + return None + arcname = arcname2 if not arcname and not member.is_dir(): + if self._ignore_invalid_names: + return None raise ValueError("Empty filename.") targetpath = os.path.join(targetpath, arcname) @@ -2093,6 +2136,8 @@ def _write_end_record(self): min_version = max(BZIP2_VERSION, min_version) elif zinfo.compress_type == ZIP_LZMA: min_version = max(LZMA_VERSION, min_version) + elif zinfo.compress_type == ZIP_ZSTANDARD: + min_version = max(ZSTANDARD_VERSION, min_version) extract_version = max(min_version, zinfo.extract_version) create_version = max(min_version, zinfo.create_version) @@ -2129,7 +2174,7 @@ def _write_end_record(self): " would require ZIP64 extensions") zip64endrec = struct.pack( structEndArchive64, stringEndArchive64, - 44, 45, 45, 0, 0, centDirCount, centDirCount, + sizeEndCentDir64 - 12, 45, 45, 0, 0, centDirCount, centDirCount, centDirSize, centDirOffset) self.fp.write(zip64endrec) diff --git a/Lib/zipfile/_path/__init__.py b/Lib/zipfile/_path/__init__.py index 5ae16ec970dda42..80f5d607731fd78 100644 --- a/Lib/zipfile/_path/__init__.py +++ b/Lib/zipfile/_path/__init__.py @@ -7,19 +7,18 @@ for more detail. """ +import contextlib import io -import posixpath -import zipfile import itertools -import contextlib import pathlib +import posixpath import re import stat import sys +import zipfile from .glob import Translator - __all__ = ['Path'] @@ -192,11 +191,13 @@ def _name_set(self): self.__lookup = super()._name_set() return self.__lookup - def _extract_text_encoding(encoding=None, *args, **kwargs): # compute stack level so that the caller of the caller sees any warning. is_pypy = sys.implementation.name == 'pypy' - stack_level = 3 + is_pypy + # PyPy no longer special cased after 7.3.19 (or maybe 7.3.18) + # See jaraco/zipp#143 + is_old_pypi = is_pypy and sys.pypy_version_info < (7, 3, 19) + stack_level = 3 + is_old_pypi return io.text_encoding(encoding, stack_level), args, kwargs @@ -351,7 +352,7 @@ def open(self, mode='r', *args, pwd=None, **kwargs): return io.TextIOWrapper(stream, encoding, *args, **kwargs) def _base(self): - return pathlib.PurePosixPath(self.at or self.root.filename) + return pathlib.PurePosixPath(self.at) if self.at else self.filename @property def name(self): diff --git a/Lib/zipfile/_path/glob.py b/Lib/zipfile/_path/glob.py index d7fe45a494717ac..bd2839304b7db2a 100644 --- a/Lib/zipfile/_path/glob.py +++ b/Lib/zipfile/_path/glob.py @@ -1,7 +1,6 @@ import os import re - _default_seps = os.sep + str(os.altsep) * bool(os.altsep) diff --git a/Lib/zoneinfo/__init__.py b/Lib/zoneinfo/__init__.py index f5510ee04975134..df2ae909f53cf29 100644 --- a/Lib/zoneinfo/__init__.py +++ b/Lib/zoneinfo/__init__.py @@ -12,7 +12,10 @@ try: from _zoneinfo import ZoneInfo -except ImportError: # pragma: nocover +except (ImportError, AttributeError): # pragma: nocover + # AttributeError: module 'datetime' has no attribute 'datetime_CAPI'. + # This happens when the '_datetime' module is not available and the + # pure Python implementation is used instead. from ._zoneinfo import ZoneInfo reset_tzpath = _tzpath.reset_tzpath diff --git a/Lib/zoneinfo/_common.py b/Lib/zoneinfo/_common.py index 6e05abc32394bca..98668c15d8bf94b 100644 --- a/Lib/zoneinfo/_common.py +++ b/Lib/zoneinfo/_common.py @@ -9,9 +9,13 @@ def load_tzdata(key): resource_name = components[-1] try: - return resources.files(package_name).joinpath(resource_name).open("rb") + path = resources.files(package_name).joinpath(resource_name) + # gh-85702: Prevent PermissionError on Windows + if path.is_dir(): + raise IsADirectoryError + return path.open("rb") except (ImportError, FileNotFoundError, UnicodeEncodeError, IsADirectoryError): - # There are three types of exception that can be raised that all amount + # There are four types of exception that can be raised that all amount # to "we cannot find this key": # # ImportError: If package_name doesn't exist (e.g. if tzdata is not @@ -63,6 +67,10 @@ def load_data(fobj): f">{timecnt}{time_type}", fobj.read(timecnt * time_size) ) trans_idx = struct.unpack(f">{timecnt}B", fobj.read(timecnt)) + + if max(trans_idx) >= typecnt: + raise ValueError("Invalid transition index found while reading TZif: " + f"{max(trans_idx)}") else: trans_list_utc = () trans_idx = () @@ -114,11 +122,10 @@ def get_abbr(idx): c = fobj.read(1) # Should be \n assert c == b"\n", c - tz_bytes = b"" - while (c := fobj.read(1)) != b"\n": - tz_bytes += c - - tz_str = tz_bytes + line = fobj.readline() + if not line.endswith(b"\n"): + raise ValueError("Invalid TZif file: unexpected end of file") + tz_str = line.rstrip(b"\n") else: tz_str = None diff --git a/Lib/zoneinfo/_tzpath.py b/Lib/zoneinfo/_tzpath.py index 5db17bea045d8c8..177d32c35eff296 100644 --- a/Lib/zoneinfo/_tzpath.py +++ b/Lib/zoneinfo/_tzpath.py @@ -13,6 +13,13 @@ def _reset_tzpath(to=None, stacklevel=4): + f"not {type(tzpaths)}: {tzpaths!r}" ) + tzpaths = [os.fspath(p) for p in tzpaths] + if not all(isinstance(p, str) for p in tzpaths): + raise TypeError( + "All elements of a tzpath sequence must be strings or " + "os.PathLike objects which convert to strings." + ) + if not all(map(os.path.isabs, tzpaths)): raise ValueError(_get_invalid_paths_message(tzpaths)) base_tzpath = tzpaths @@ -124,7 +131,8 @@ def available_timezones(): # Start with loading from the tzdata package if it exists: this has a # pre-assembled list of zones that only requires opening one file. try: - with resources.files("tzdata").joinpath("zones").open("r") as f: + zones_file = resources.files("tzdata").joinpath("zones") + with zones_file.open("r", encoding="utf-8") as f: for zone in f: zone = zone.strip() if zone: diff --git a/Lib/zoneinfo/_zoneinfo.py b/Lib/zoneinfo/_zoneinfo.py index b77dc0ed391bb36..48154775568b6dc 100644 --- a/Lib/zoneinfo/_zoneinfo.py +++ b/Lib/zoneinfo/_zoneinfo.py @@ -47,7 +47,11 @@ def __new__(cls, key): cls._strong_cache[key] = cls._strong_cache.pop(key, instance) if len(cls._strong_cache) > cls._strong_cache_size: - cls._strong_cache.popitem(last=False) + try: + cls._strong_cache.popitem(last=False) + except KeyError: + # another thread may have already emptied the cache + pass return instance @@ -75,12 +79,12 @@ def _new_instance(cls, key): return obj @classmethod - def from_file(cls, fobj, /, key=None): + def from_file(cls, file_obj, /, key=None): obj = super().__new__(cls) obj._key = key obj._file_path = None - obj._load_file(fobj) - obj._file_repr = repr(fobj) + obj._load_file(file_obj) + obj._file_repr = repr(file_obj) # Disable pickling for objects created from files obj.__reduce__ = obj._file_reduce @@ -334,7 +338,7 @@ def _utcoff_to_dstoff(trans_idx, utcoffsets, isdsts): if not isdsts[comp_idx]: dstoff = utcoff - utcoffsets[comp_idx] - if not dstoff and idx < (typecnt - 1): + if not dstoff and idx < (typecnt - 1) and i + 1 < len(trans_idx): comp_idx = trans_idx[i + 1] # If the following transition is also DST and we couldn't @@ -636,11 +640,11 @@ def _parse_tz_str(tz_str): parser_re = re.compile( r""" - (?P<std>[^<0-9:.+-]+|<[a-zA-Z0-9+-]+>) + (?P<std>[a-zA-Z]+|<[a-zA-Z0-9+-]+>) (?: (?P<stdoff>[+-]?\d{1,3}(?::\d{2}(?::\d{2})?)?) (?: - (?P<dst>[^0-9:.+-]+|<[a-zA-Z0-9+-]+>) + (?P<dst>[a-zA-Z]+|<[a-zA-Z0-9+-]+>) (?P<dstoff>[+-]?\d{1,3}(?::\d{2}(?::\d{2})?)?)? )? # dst )? # stdoff @@ -668,7 +672,8 @@ def _parse_tz_str(tz_str): except ValueError as e: raise ValueError(f"Invalid STD offset in {tz_str}") from e else: - std_offset = 0 + # The STD offset is required + raise ValueError(f"Invalid STD offset in {tz_str}") if dst_abbr is not None: if dst_offset := m.group("dstoff"): @@ -703,7 +708,7 @@ def _parse_dst_start_end(dststr): type = date[:1] if type == "M": n_is_julian = False - m = re.fullmatch(r"M(\d{1,2})\.(\d).(\d)", date, re.ASCII) + m = re.fullmatch(r"M(\d{1,2})\.(\d)\.(\d)", date, re.ASCII) if m is None: raise ValueError(f"Invalid dst start/end date: {dststr}") date_offset = tuple(map(int, m.groups())) @@ -715,6 +720,8 @@ def _parse_dst_start_end(dststr): else: n_is_julian = False + if re.fullmatch(r"\d{1,3}", date, re.ASCII) is None: + raise ValueError(f"Invalid dst start/end date: {dststr}") doy = int(date) offset = _DayOffset(doy, n_is_julian) diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index b31cb766a468f4e..5112dd2416c27dd 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -246,9 +246,9 @@ def library_recipes(): result.extend([ dict( - name="OpenSSL 3.0.16", - url="https://github.com/openssl/openssl/releases/download/openssl-3.0.16/openssl-3.0.16.tar.gz", - checksum='57e03c50feab5d31b152af2b764f10379aecd8ee92f16c985983ce4a99f7ef86', + name="OpenSSL 3.5.7", + url="https://github.com/openssl/openssl/releases/download/openssl-3.5.7/openssl-3.5.7.tar.gz", + checksum="a8c0d28a529ca480f9f36cf5792e2cd21984552a3c8e4aa11a24aa31aeac98e8", buildrecipe=build_universal_openssl, configure=None, install=None, @@ -264,10 +264,10 @@ def library_recipes(): tk_patches = ['backport_gh71383_fix.patch', 'tk868_on_10_8_10_9.patch', 'backport_gh110950_fix.patch'] else: - tcl_tk_ver='8.6.16' - tcl_checksum='91cb8fa61771c63c262efb553059b7c7ad6757afa5857af6265e4b0bdc2a14a5' + tcl_tk_ver='9.0.4' + tcl_checksum='d0aed49230bc02a65c1e0229e65f34590a4b037ec40d546f32573b467f7551ea' - tk_checksum='be9f94d3575d4b3099d84bc3c10de8994df2d7aa405208173c709cc404a7e5fe' + tk_checksum='d7a146d2917eb8b5cc95276dbf0e3d03c7464d2b19c1675357857c989301dbb4' tk_patches = [] @@ -359,9 +359,9 @@ def library_recipes(): ), ), dict( - name="SQLite 3.49.1", - url="https://sqlite.org/2025/sqlite-autoconf-3490100.tar.gz", - checksum="106642d8ccb36c5f7323b64e4152e9b719f7c0215acf5bfeac3d5e7f97b59254", + name="SQLite 3.50.4", + url="https://www.sqlite.org/2025/sqlite-autoconf-3500400.tar.gz", + checksum="a3db587a1b92ee5ddac2f66b3edb41b26f9c867275782d46c3a088977d6a5b18", extra_cflags=('-Os ' '-DSQLITE_ENABLE_FTS5 ' '-DSQLITE_ENABLE_FTS4 ' @@ -378,9 +378,9 @@ def library_recipes(): install=f"make && ranlib libsqlite3.a && make install DESTDIR={shellQuote(os.path.join(WORKDIR, 'libraries'))}", ), dict( - name="libmpdec 4.0.0", - url="https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-4.0.0.tar.gz", - checksum="942445c3245b22730fd41a67a7c5c231d11cb1b9936b9c0f76334fb7d0b4468c", + name="libmpdec 4.0.1", + url="https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-4.0.1.tar.gz", + checksum="96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8", configure_pre=[ "--disable-cxx", "MACHINE=universal", @@ -1747,7 +1747,7 @@ def main(): fn = os.path.join(folder, "ReadMe.rtf") patchFile("resources/ReadMe.rtf", fn) fn = os.path.join(folder, "Update Shell Profile.command") - patchScript("scripts/postflight.patch-profile", fn) + patchScript("resources/update_shell_profile.command", fn) fn = os.path.join(folder, "Install Certificates.command") patchScript("resources/install_certificates.command", fn) os.chmod(folder, STAT_0o755) diff --git a/Mac/BuildScript/resources/install_certificates.command b/Mac/BuildScript/resources/install_certificates.command index 19b4adac07bb1d6..700eb462b68c187 100755 --- a/Mac/BuildScript/resources/install_certificates.command +++ b/Mac/BuildScript/resources/install_certificates.command @@ -25,7 +25,8 @@ def main(): print(" -- pip install --upgrade certifi") subprocess.check_call([sys.executable, - "-E", "-s", "-m", "pip", "install", "--upgrade", "certifi"]) + "-E", "-s", "-m", "pip", "install", "--upgrade", "certifi", + "--disable-pip-version-check"]) import certifi diff --git a/Mac/BuildScript/resources/update_shell_profile.command b/Mac/BuildScript/resources/update_shell_profile.command new file mode 100755 index 000000000000000..3cf4d74de9f09a5 --- /dev/null +++ b/Mac/BuildScript/resources/update_shell_profile.command @@ -0,0 +1,116 @@ +#!/bin/sh + +echo "This script will update your shell profile when the 'bin' directory" +echo "of python is not early enough of the PATH of your shell." +echo "These changes will be effective only in shell windows that you open" +echo "after running this script." + +PYVER=@PYVER@ +PYTHON_ROOT="/Library/Frameworks/Python.framework/Versions/@PYVER@" + +if [ `id -ur` = 0 ]; then + # Run from the installer, do some trickery to fetch the information + # we need. + theShell="`finger $USER | grep Shell: | head -1 | awk '{ print $NF }'`" + +else + theShell="${SHELL}" +fi + +# Make sure the directory ${PYTHON_ROOT}/bin is on the users PATH. +BSH="`basename "${theShell}"`" +case "${BSH}" in +bash|ksh|sh|*csh|zsh|fish) + if [ `id -ur` = 0 ]; then + P=`su - ${USER} -c 'echo A-X-4-X@@$PATH@@X-4-X-A' | grep 'A-X-4-X@@.*@@X-4-X-A' | sed -e 's/^A-X-4-X@@//g' -e 's/@@X-4-X-A$//g'` + else + P="`(exec -l ${theShell} -c 'echo $PATH')`" + fi + ;; +*) + echo "Sorry, I don't know how to patch $BSH shells" + exit 0 + ;; +esac + +# Now ensure that our bin directory is on $P and before /usr/bin at that +for elem in `echo $P | tr ':' ' '` +do + if [ "${elem}" = "${PYTHON_ROOT}/bin" ]; then + echo "All right, you're a python lover already" + exit 0 + elif [ "${elem}" = "/usr/bin" ]; then + break + fi +done + +echo "${PYTHON_ROOT}/bin is not on your PATH or at least not early enough" +case "${BSH}" in +*csh) + if [ -f "${HOME}/.tcshrc" ]; then + RC="${HOME}/.tcshrc" + else + RC="${HOME}/.cshrc" + fi + # Create backup copy before patching + if [ -f "${RC}" ]; then + cp -fp "${RC}" "${RC}.pysave" + fi + echo "" >> "${RC}" + echo "# Setting PATH for Python ${PYVER}" >> "${RC}" + echo "# The original version is saved in .cshrc.pysave" >> "${RC}" + echo "set path=(${PYTHON_ROOT}/bin "'$path'")" >> "${RC}" + if [ `id -ur` = 0 ]; then + chown -h "${USER}" "${RC}" + fi + exit 0 + ;; +bash) + if [ -e "${HOME}/.bash_profile" ]; then + PR="${HOME}/.bash_profile" + elif [ -e "${HOME}/.bash_login" ]; then + PR="${HOME}/.bash_login" + elif [ -e "${HOME}/.profile" ]; then + PR="${HOME}/.profile" + else + PR="${HOME}/.bash_profile" + fi + ;; +fish) + CONFIG_DIR="${HOME}/.config/fish/conf.d/" + RC="${CONFIG_DIR}/python-${PYVER}.fish" + mkdir -p "$CONFIG_DIR" + if [ -f "${RC}" ]; then + cp -fp "${RC}" "${RC}.pysave" + fi + echo "# Setting PATH for Python ${PYVER}" > "${RC}" + if [ -f "${RC}.pysave" ]; then + echo "# The original version is saved in ${RC}.pysave" >> "${RC}" + fi + echo "fish_add_path -g \"${PYTHON_ROOT}/bin\"" >> "${RC}" + if [ `id -ur` = 0 ]; then + chown -h "${USER}" "${RC}" + fi + exit 0 + ;; +zsh) + PR="${HOME}/.zprofile" + ;; +*sh) + PR="${HOME}/.profile" + ;; +esac + +# Create backup copy before patching +if [ -f "${PR}" ]; then + cp -fp "${PR}" "${PR}.pysave" +fi +echo "" >> "${PR}" +echo "# Setting PATH for Python ${PYVER}" >> "${PR}" +echo "# The original version is saved in `basename ${PR}`.pysave" >> "${PR}" +echo 'PATH="'"${PYTHON_ROOT}/bin"':${PATH}"' >> "${PR}" +echo 'export PATH' >> "${PR}" +if [ `id -ur` = 0 ]; then + chown -h "${USER}" "${PR}" +fi +exit 0 diff --git a/Mac/BuildScript/scripts/postflight.patch-profile b/Mac/BuildScript/scripts/postflight.patch-profile index 9caf62211ddd16c..ce8720f895d1b5c 100755 --- a/Mac/BuildScript/scripts/postflight.patch-profile +++ b/Mac/BuildScript/scripts/postflight.patch-profile @@ -1,116 +1,104 @@ #!/bin/sh -echo "This script will update your shell profile when the 'bin' directory" -echo "of python is not early enough of the PATH of your shell." -echo "These changes will be effective only in shell windows that you open" -echo "after running this script." - PYVER=@PYVER@ PYTHON_ROOT="/Library/Frameworks/Python.framework/Versions/@PYVER@" -if [ `id -ur` = 0 ]; then - # Run from the installer, do some trickery to fetch the information - # we need. - theShell="`finger $USER | grep Shell: | head -1 | awk '{ print $NF }'`" -else - theShell="${SHELL}" -fi +# Run from the installer, do some trickery to fetch the information +# we need. +theShell="`finger $USER | grep Shell: | head -1 | awk '{ print $NF }'`" # Make sure the directory ${PYTHON_ROOT}/bin is on the users PATH. BSH="`basename "${theShell}"`" case "${BSH}" in bash|ksh|sh|*csh|zsh|fish) - if [ `id -ur` = 0 ]; then - P=`su - ${USER} -c 'echo A-X-4-X@@$PATH@@X-4-X-A' | grep 'A-X-4-X@@.*@@X-4-X-A' | sed -e 's/^A-X-4-X@@//g' -e 's/@@X-4-X-A$//g'` - else - P="`(exec -l ${theShell} -c 'echo $PATH')`" - fi - ;; + true + ;; *) - echo "Sorry, I don't know how to patch $BSH shells" - exit 0 - ;; + exit 0 + ;; esac -# Now ensure that our bin directory is on $P and before /usr/bin at that -for elem in `echo $P | tr ':' ' '` -do - if [ "${elem}" = "${PYTHON_ROOT}/bin" ]; then - echo "All right, you're a python lover already" - exit 0 - elif [ "${elem}" = "/usr/bin" ]; then - break - fi -done - -echo "${PYTHON_ROOT}/bin is not on your PATH or at least not early enough" case "${BSH}" in *csh) - if [ -f "${HOME}/.tcshrc" ]; then - RC="${HOME}/.tcshrc" - else - RC="${HOME}/.cshrc" - fi - # Create backup copy before patching - if [ -f "${RC}" ]; then - cp -fp "${RC}" "${RC}.pysave" - fi - echo "" >> "${RC}" - echo "# Setting PATH for Python ${PYVER}" >> "${RC}" - echo "# The original version is saved in .cshrc.pysave" >> "${RC}" - echo "set path=(${PYTHON_ROOT}/bin "'$path'")" >> "${RC}" - if [ `id -ur` = 0 ]; then - chown "${USER}" "${RC}" - fi - exit 0 - ;; + if [ -f "${HOME}/.tcshrc" ]; then + RC="${HOME}/.tcshrc" + else + RC="${HOME}/.cshrc" + fi + + # Drop privileges while writing files. + su -m ${USER} <<EOFC + # Create backup copy before patching + if [ -f "${RC}" ]; then + cp -fp "${RC}" "${RC}.pysave" + fi + echo "" >> "${RC}" + echo "# Setting PATH for Python ${PYVER}" >> "${RC}" + echo "# The original version is saved in .cshrc.pysave" >> "${RC}" + echo "set path=(${PYTHON_ROOT}/bin "'\$path'")" >> "${RC}" +EOFC + + if [ `id -ur` = 0 ]; then + chown -h "${USER}" "${RC}" + fi + exit 0 + ;; bash) - if [ -e "${HOME}/.bash_profile" ]; then - PR="${HOME}/.bash_profile" - elif [ -e "${HOME}/.bash_login" ]; then - PR="${HOME}/.bash_login" - elif [ -e "${HOME}/.profile" ]; then - PR="${HOME}/.profile" - else - PR="${HOME}/.bash_profile" - fi - ;; + if [ -e "${HOME}/.bash_profile" ]; then + PR="${HOME}/.bash_profile" + elif [ -e "${HOME}/.bash_login" ]; then + PR="${HOME}/.bash_login" + elif [ -e "${HOME}/.profile" ]; then + PR="${HOME}/.profile" + else + PR="${HOME}/.bash_profile" + fi + ;; fish) - CONFIG_DIR="${HOME}/.config/fish/conf.d/" - RC="${CONFIG_DIR}/python-${PYVER}.fish" - mkdir -p "$CONFIG_DIR" - if [ -f "${RC}" ]; then - cp -fp "${RC}" "${RC}.pysave" - fi - echo "# Setting PATH for Python ${PYVER}" > "${RC}" - if [ -f "${RC}.pysave" ]; then - echo "# The original version is saved in ${RC}.pysave" >> "${RC}" - fi - echo "fish_add_path -g \"${PYTHON_ROOT}/bin\"" >> "${RC}" - if [ `id -ur` = 0 ]; then - chown "${USER}" "${RC}" - fi - exit 0 - ;; + CONFIG_DIR="${HOME}/.config/fish/conf.d/" + RC="${CONFIG_DIR}/python-${PYVER}.fish" + + # Drop privileges while writing files. + su -m ${USER} <<EOFF + mkdir -p "$CONFIG_DIR" + if [ -f "${RC}" ]; then + cp -fp "${RC}" "${RC}.pysave" + fi + echo "# Setting PATH for Python ${PYVER}" > "${RC}" + if [ -f "${RC}.pysave" ]; then + echo "# The original version is saved in ${RC}.pysave" >> "${RC}" + fi + echo "fish_add_path -g \"${PYTHON_ROOT}/bin\"" >> "${RC}" +EOFF + + if [ `id -ur` = 0 ]; then + chown -h "${USER}" "${RC}" + fi + exit 0 + ;; zsh) - PR="${HOME}/.zprofile" - ;; + PR="${HOME}/.zprofile" + ;; *sh) - PR="${HOME}/.profile" - ;; + PR="${HOME}/.profile" + ;; esac +# Drop privileges while writing files. +su -m ${USER} <<EOFS # Create backup copy before patching if [ -f "${PR}" ]; then - cp -fp "${PR}" "${PR}.pysave" + cp -fp "${PR}" "${PR}.pysave" fi echo "" >> "${PR}" echo "# Setting PATH for Python ${PYVER}" >> "${PR}" echo "# The original version is saved in `basename ${PR}`.pysave" >> "${PR}" -echo 'PATH="'"${PYTHON_ROOT}/bin"':${PATH}"' >> "${PR}" +echo 'PATH="'"${PYTHON_ROOT}/bin"':\${PATH}"' >> "${PR}" echo 'export PATH' >> "${PR}" +EOFS + if [ `id -ur` = 0 ]; then - chown "${USER}" "${PR}" + chown -h "${USER}" "${PR}" fi exit 0 diff --git a/Mac/IDLE/IDLE.app/Contents/Info.plist b/Mac/IDLE/IDLE.app/Contents/Info.plist index 8549e405e2a65a0..696625e64cdf326 100644 --- a/Mac/IDLE/IDLE.app/Contents/Info.plist +++ b/Mac/IDLE/IDLE.app/Contents/Info.plist @@ -1,9 +1,23 @@ <?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <plist version="1.0"> <dict> - <key>CFBundleDevelopmentRegion</key> - <string>English</string> + <key>CFBundleName</key> + <string>IDLE</string> + <key>CFBundleIdentifier</key> + <string>org.python.IDLE</string> + <key>CFBundleVersion</key> + <string>%version%</string> + <key>CFBundlePackageType</key> + <string>APPL</string> + <key>CFBundleSignature</key> + <string>????</string> + <key>CFBundleExecutable</key> + <string>IDLE</string> + <key>CFBundleIconFile</key> + <string>IDLE.icns</string> + <key>CFBundleInfoDictionaryVersion</key> + <string>6.0</string> <key>CFBundleDocumentTypes</key> <array> <dict> @@ -34,26 +48,16 @@ <string>Editor</string> </dict> </array> - <key>CFBundleExecutable</key> - <string>IDLE</string> - <key>CFBundleGetInfoString</key> - <string>%version%, © 2001-2024 Python Software Foundation</string> - <key>CFBundleIconFile</key> - <string>IDLE.icns</string> - <key>CFBundleIdentifier</key> - <string>org.python.IDLE</string> - <key>CFBundleInfoDictionaryVersion</key> - <string>6.0</string> - <key>CFBundleName</key> - <string>IDLE</string> - <key>CFBundlePackageType</key> - <string>APPL</string> <key>CFBundleShortVersionString</key> <string>%version%</string> - <key>CFBundleSignature</key> - <string>????</string> - <key>CFBundleVersion</key> - <string>%version%</string> + <key>CFBundleSupportedPlatforms</key> + <array> + <string>MacOSX</string> + </array> + <key>NSHumanReadableCopyright</key> + <string>Copyright © 2001 Python Software Foundation. All rights reserved.</string> + <key>CFBundleDevelopmentRegion</key> + <string>English</string> <key>NSHighResolutionCapable</key> <true/> <key>CFBundleAllowMixedLocalizations</key> diff --git a/Mac/PythonLauncher/Info.plist.in b/Mac/PythonLauncher/Info.plist.in index ce8f27cd7d4de70..dd63187ab836b38 100644 --- a/Mac/PythonLauncher/Info.plist.in +++ b/Mac/PythonLauncher/Info.plist.in @@ -2,8 +2,22 @@ <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <plist version="1.0"> <dict> - <key>CFBundleDevelopmentRegion</key> - <string>en</string> + <key>CFBundleName</key> + <string>Python Launcher</string> + <key>CFBundleIdentifier</key> + <string>org.python.PythonLauncher</string> + <key>CFBundleVersion</key> + <string>%VERSION%</string> + <key>CFBundlePackageType</key> + <string>APPL</string> + <key>CFBundleSignature</key> + <string>PytL</string> + <key>CFBundleExecutable</key> + <string>Python Launcher</string> + <key>CFBundleIconFile</key> + <string>PythonLauncher.icns</string> + <key>CFBundleInfoDictionaryVersion</key> + <string>6.0</string> <key>CFBundleDocumentTypes</key> <array> <dict> @@ -37,28 +51,16 @@ <string>MyDocument</string> </dict> </array> - <key>CFBundleExecutable</key> - <string>Python Launcher</string> - <key>NSHumanReadableCopyright</key> - <string>Copyright © 2001 Python Software Foundation</string> - <key>CFBundleGetInfoString</key> - <string>%VERSION%, © 2001 Python Software Foundation</string> - <key>CFBundleIconFile</key> - <string>PythonLauncher.icns</string> - <key>CFBundleIdentifier</key> - <string>org.python.PythonLauncher</string> - <key>CFBundleInfoDictionaryVersion</key> - <string>6.0</string> - <key>CFBundleName</key> - <string>Python Launcher</string> - <key>CFBundlePackageType</key> - <string>APPL</string> <key>CFBundleShortVersionString</key> <string>%VERSION%</string> - <key>CFBundleSignature</key> - <string>PytL</string> - <key>CFBundleVersion</key> - <string>%VERSION%</string> + <key>CFBundleSupportedPlatforms</key> + <array> + <string>MacOSX</string> + </array> + <key>NSHumanReadableCopyright</key> + <string>Copyright © 2001 Python Software Foundation. All rights reserved.</string> + <key>CFBundleDevelopmentRegion</key> + <string>English</string> <key>NSMainNibFile</key> <string>MainMenu</string> <key>NSPrincipalClass</key> diff --git a/Mac/Resources/app/Info.plist.in b/Mac/Resources/app/Info.plist.in index a1fc1511c40e96d..07dc351398d7b2a 100644 --- a/Mac/Resources/app/Info.plist.in +++ b/Mac/Resources/app/Info.plist.in @@ -1,9 +1,23 @@ <?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <plist version="1.0"> <dict> - <key>CFBundleDevelopmentRegion</key> - <string>English</string> + <key>CFBundleName</key> + <string>Python</string> + <key>CFBundleIdentifier</key> + <string>%bundleid%</string> + <key>CFBundleVersion</key> + <string>%version%</string> + <key>CFBundlePackageType</key> + <string>APPL</string> + <key>CFBundleSignature</key> + <string>PytX</string> + <key>CFBundleExecutable</key> + <string>Python</string> + <key>CFBundleIconFile</key> + <string>PythonInterpreter.icns</string> + <key>CFBundleInfoDictionaryVersion</key> + <string>6.0</string> <key>CFBundleDocumentTypes</key> <array> <dict> @@ -17,48 +31,33 @@ <string>Viewer</string> </dict> </array> - <key>CFBundleExecutable</key> - <string>Python</string> - <key>CFBundleGetInfoString</key> - <string>%version%, (c) 2001-2024 Python Software Foundation.</string> + <key>CFBundleShortVersionString</key> + <string>%version%</string> + <key>CFBundleSupportedPlatforms</key> + <array> + <string>MacOSX</string> + </array> + <key>NSHumanReadableCopyright</key> + <string>Copyright © 2001 Python Software Foundation. All rights reserved.</string> + <key>CFBundleDevelopmentRegion</key> + <string>English</string> + <key>NSHighResolutionCapable</key> + <true/> + <key>NSAppleScriptEnabled</key> + <true/> + <key>CFBundleAllowMixedLocalizations</key> + <true/> <key>CFBundleHelpBookFolder</key> <array> <string>Documentation</string> - <string>PythonDocumentation</string> </array> <key>CFBundleHelpBookName</key> - <string>MacPython Help</string> + <string>Python Help</string> <key>CFBundleHelpTOCFile</key> <string>index.html</string> - <key>CFBundleIconFile</key> - <string>PythonInterpreter.icns</string> - <key>CFBundleIdentifier</key> - <string>%bundleid%</string> - <key>CFBundleInfoDictionaryVersion</key> - <string>6.0</string> - <key>CFBundleLongVersionString</key> - <string>%version%, (c) 2001-2024 Python Software Foundation.</string> - <key>CFBundleName</key> - <string>Python</string> - <key>CFBundlePackageType</key> - <string>APPL</string> - <key>CFBundleShortVersionString</key> - <string>%version%</string> - <key>CFBundleSignature</key> - <string>PytX</string> - <key>CFBundleVersion</key> - <string>%version%</string> <key>CSResourcesFileMapped</key> <true/> <key>LSRequiresCarbon</key> <true/> - <key>NSAppleScriptEnabled</key> - <true/> - <key>NSHumanReadableCopyright</key> - <string>(c) 2001-2024 Python Software Foundation.</string> - <key>NSHighResolutionCapable</key> - <true/> - <key>CFBundleAllowMixedLocalizations</key> - <true/> </dict> </plist> diff --git a/Mac/Resources/framework/Info.plist.in b/Mac/Resources/framework/Info.plist.in index 4c42971ed90ee4c..41400c91a5f1b0e 100644 --- a/Mac/Resources/framework/Info.plist.in +++ b/Mac/Resources/framework/Info.plist.in @@ -1,29 +1,31 @@ <?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE plist SYSTEM "file://localhost/System/Library/DTDs/PropertyList.dtd"> -<plist version="0.9"> +<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> <dict> - <key>CFBundleDevelopmentRegion</key> - <string>English</string> - <key>CFBundleExecutable</key> + <key>CFBundleName</key> <string>Python</string> - <key>CFBundleGetInfoString</key> - <string>Python Runtime and Library</string> <key>CFBundleIdentifier</key> <string>@PYTHONFRAMEWORKIDENTIFIER@</string> - <key>CFBundleInfoDictionaryVersion</key> - <string>6.0</string> - <key>CFBundleName</key> - <string>Python</string> + <key>CFBundleVersion</key> + <string>%VERSION%</string> <key>CFBundlePackageType</key> <string>FMWK</string> - <key>CFBundleShortVersionString</key> - <string>%VERSION%, (c) 2001-2024 Python Software Foundation.</string> - <key>CFBundleLongVersionString</key> - <string>%VERSION%, (c) 2001-2024 Python Software Foundation.</string> <key>CFBundleSignature</key> <string>????</string> - <key>CFBundleVersion</key> + <key>CFBundleExecutable</key> + <string>Python</string> + <key>CFBundleInfoDictionaryVersion</key> + <string>6.0</string> + <key>CFBundleShortVersionString</key> <string>%VERSION%</string> + <key>CFBundleSupportedPlatforms</key> + <array> + <string>MacOSX</string> + </array> + <key>NSHumanReadableCopyright</key> + <string>Copyright © 2001 Python Software Foundation. All rights reserved.</string> + <key>CFBundleDevelopmentRegion</key> + <string>English</string> <key>CFBundleAllowMixedLocalizations</key> <true/> </dict> diff --git a/Makefile.pre.in b/Makefile.pre.in index 17e0c9904cc3aa2..66844d7b338cdb0 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -126,6 +126,8 @@ PY_CORE_CFLAGS= $(PY_STDMODULE_CFLAGS) -DPy_BUILD_CORE PY_CORE_LDFLAGS=$(PY_LDFLAGS) $(PY_LDFLAGS_NODIST) # Strict or non-strict aliasing flags used to compile dtoa.c, see above CFLAGS_ALIASING=@CFLAGS_ALIASING@ +# Compilation flags only for ceval.c. +CFLAGS_CEVAL=@CFLAGS_CEVAL@ # Machine-dependent subdirectories @@ -642,13 +644,16 @@ LIBEXPAT_HEADERS= \ Modules/expat/expat.h \ Modules/expat/expat_config.h \ Modules/expat/expat_external.h \ + Modules/expat/fallthrough.h \ Modules/expat/iasciitab.h \ Modules/expat/internal.h \ Modules/expat/latin1tab.h \ + Modules/expat/memory_sanitizer.h \ Modules/expat/nametab.h \ Modules/expat/pyexpatns.h \ Modules/expat/siphash.h \ Modules/expat/utf8tab.h \ + Modules/expat/xcsinc.c \ Modules/expat/xmlrole.h \ Modules/expat/xmltok.h \ Modules/expat/xmltok_impl.h \ @@ -801,10 +806,10 @@ build_all: check-clean-src check-app-store-compliance $(BUILDPYTHON) platform sh .PHONY: build_wasm build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \ - python-config checksharedmods + python-config checksharedmods build-details.json .PHONY: build_emscripten -build_emscripten: build_wasm web_example +build_emscripten: build_wasm web_example web_example_pyrepl_jspi # Check that the source is clean when building out of source. .PHONY: check-clean-src @@ -818,7 +823,7 @@ check-clean-src: echo "Building Python out of the source tree (in $(abs_builddir)) requires a clean source tree ($(abs_srcdir))" ; \ echo "Build artifacts such as .o files, executables, and Python/frozen_modules/*.h must not exist within $(srcdir)." ; \ echo "Try to run:" ; \ - echo " (cd \"$(srcdir)\" && make clean || git clean -fdx -e Doc/venv)" ; \ + echo " (cd \"$(srcdir)\" && make distclean || git clean -fdx -e Doc/venv)" ; \ exit 1; \ fi @@ -834,7 +839,7 @@ check-app-store-compliance: # Profile generation build must start from a clean tree. profile-clean-stamp: - $(MAKE) clean + $(MAKE) clean-profile touch $@ # Compile with profile generation enabled. @@ -1012,7 +1017,12 @@ $(LIBRARY): $(LIBRARY_OBJS) $(AR) $(ARFLAGS) $@ $(LIBRARY_OBJS) libpython$(LDVERSION).so: $(LIBRARY_OBJS) $(DTRACE_OBJS) - $(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM) + # AIX Linker don't support "-h" option + if test "$(MACHDEP)" != "aix"; then \ + $(BLDSHARED) -Wl,-h$(INSTSONAME) -o $(INSTSONAME) $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ + else \ + $(BLDSHARED) -o $@ $(LIBRARY_OBJS) $(MODLIBS) $(SHLIBS) $(LIBC) $(LIBM); \ + fi if test $(INSTSONAME) != $@; then \ $(LN) -f $(INSTSONAME) $@; \ fi @@ -1052,7 +1062,7 @@ $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION)/$(PYTHONFRAMEWORK): \ $(INSTALL) -d -m $(DIRMODE) $(PYTHONFRAMEWORKDIR)/Versions/$(VERSION) $(CC) -o $(LDLIBRARY) $(PY_CORE_LDFLAGS) -dynamiclib \ -all_load $(LIBRARY) \ - -install_name $(DESTDIR)$(PYTHONFRAMEWORKINSTALLNAMEPREFIX)/$(PYTHONFRAMEWORK) \ + -install_name $(PYTHONFRAMEWORKINSTALLNAMEPREFIX)/$(PYTHONFRAMEWORK) \ -compatibility_version $(VERSION) \ -current_version $(VERSION) \ -framework CoreFoundation $(LIBS); @@ -1090,8 +1100,17 @@ $(DLLLIBRARY) libpython$(LDVERSION).dll.a: $(LIBRARY_OBJS) # wasm32-emscripten browser web example -WEBEX_DIR=$(srcdir)/Tools/wasm/emscripten/web_example/ -web_example/python.html: $(WEBEX_DIR)/python.html +EMSCRIPTEN_DIR=$(srcdir)/Platforms/emscripten +WEBEX_DIR=$(EMSCRIPTEN_DIR)/web_example/ + +ZIP_STDLIB=python$(VERSION)$(ABI_THREAD).zip +$(ZIP_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ + $(EMSCRIPTEN_DIR)/wasm_assets.py \ + Makefile pybuilddir.txt Modules/Setup.local + $(PYTHON_FOR_BUILD) $(EMSCRIPTEN_DIR)/wasm_assets.py \ + --buildroot . --prefix $(prefix) -o $@ + +web_example/index.html: $(WEBEX_DIR)/index.html @mkdir -p web_example @cp $< $@ @@ -1103,12 +1122,9 @@ web_example/server.py: $(WEBEX_DIR)/server.py @mkdir -p web_example @cp $< $@ -WEB_STDLIB=web_example/python$(VERSION)$(ABI_THREAD).zip -$(WEB_STDLIB): $(srcdir)/Lib/*.py $(srcdir)/Lib/*/*.py \ - $(WEBEX_DIR)/wasm_assets.py \ - Makefile pybuilddir.txt Modules/Setup.local - $(PYTHON_FOR_BUILD) $(WEBEX_DIR)/wasm_assets.py \ - --buildroot . --prefix $(prefix) -o $@ +web_example/$(ZIP_STDLIB): $(ZIP_STDLIB) + @mkdir -p web_example + @cp $< $@ web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON) @if test $(HOST_GNU_TYPE) != 'wasm32-unknown-emscripten' ; then \ @@ -1119,7 +1135,35 @@ web_example/python.mjs web_example/python.wasm: $(BUILDPYTHON) cp python.wasm web_example/python.wasm .PHONY: web_example -web_example: web_example/python.mjs web_example/python.worker.mjs web_example/python.html web_example/server.py $(WEB_STDLIB) +web_example: web_example/python.mjs web_example/python.worker.mjs web_example/index.html web_example/server.py web_example/$(ZIP_STDLIB) + +WEBEX2=web_example_pyrepl_jspi +WEBEX2_DIR=$(EMSCRIPTEN_DIR)/$(WEBEX2)/ + +$(WEBEX2)/python.mjs $(WEBEX2)/python.wasm: $(BUILDPYTHON) + @if test $(HOST_GNU_TYPE) != 'wasm32-unknown-emscripten' ; then \ + echo "Can only build web_example when target is Emscripten" ;\ + exit 1 ;\ + fi + @mkdir -p $(WEBEX2) + @cp python.mjs $(WEBEX2)/python.mjs + @cp python.wasm $(WEBEX2)/python.wasm + +$(WEBEX2)/index.html: $(WEBEX2_DIR)/index.html + @mkdir -p $(WEBEX2) + @cp $< $@ + +$(WEBEX2)/src.mjs: $(WEBEX2_DIR)/src.mjs + @mkdir -p $(WEBEX2) + @cp $< $@ + +$(WEBEX2)/$(ZIP_STDLIB): $(ZIP_STDLIB) + @mkdir -p $(WEBEX2) + @cp $< $@ + +.PHONY: web_example_pyrepl_jspi +web_example_pyrepl_jspi: $(WEBEX2)/python.mjs $(WEBEX2)/index.html $(WEBEX2)/src.mjs $(WEBEX2)/$(ZIP_STDLIB) + ############################################################################ # Header files @@ -1201,6 +1245,7 @@ PYTHON_HEADERS= \ $(srcdir)/Include/unicodeobject.h \ $(srcdir)/Include/warnings.h \ $(srcdir)/Include/weakrefobject.h \ + $(srcdir)/Python/remote_debug.h \ \ pyconfig.h \ $(PARSER_HEADERS) \ @@ -1637,6 +1682,8 @@ Programs/_bootstrap_python.o: Programs/_bootstrap_python.c $(BOOTSTRAP_HEADERS) _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modules/getpath.o Modules/Setup.local $(LINKCC) $(PY_LDFLAGS_NOLTO) -o $@ $(LIBRARY_OBJS_OMIT_FROZEN) \ Programs/_bootstrap_python.o Modules/getpath.o $(LIBS) $(MODLIBS) $(SYSLIBS) + # Dummy pybuilddir.txt is needed for _bootstrap_python to be runnable + @echo "none" > ./pybuilddir.txt ############################################################################ @@ -2190,7 +2237,7 @@ Python/frozen.o: $(FROZEN_FILES_OUT) # an include guard, so we can't use a pipeline to transform its output. Include/pydtrace_probes.h: $(srcdir)/Include/pydtrace.d $(MKDIR_P) Include - CC="$(CC)" CFLAGS="$(CFLAGS)" $(DTRACE) $(DFLAGS) -o $@ -h -s $< + CC="$(CC)" CFLAGS="$(CFLAGS)" $(DTRACE) $(DFLAGS) -o $@ -h -s $(srcdir)/Include/pydtrace.d : sed in-place edit with POSIX-only tools sed 's/PYTHON_/PyDTrace_/' $@ > $@.tmp mv $@.tmp $@ @@ -2200,7 +2247,7 @@ Python/gc.o: $(srcdir)/Include/pydtrace.h Python/import.o: $(srcdir)/Include/pydtrace.h Python/pydtrace.o: $(srcdir)/Include/pydtrace.d $(DTRACE_DEPS) - CC="$(CC)" CFLAGS="$(CFLAGS)" $(DTRACE) $(DFLAGS) -o $@ -G -s $< $(DTRACE_DEPS) + CC="$(CC)" CFLAGS="$(CFLAGS)" $(DTRACE) $(DFLAGS) -o $@ -G -s $(srcdir)/Include/pydtrace.d $(DTRACE_DEPS) Objects/typeobject.o: Objects/typeslots.inc @@ -2279,7 +2326,7 @@ testios: fi # Clone the testbed project into the XCFOLDER - $(PYTHON_FOR_BUILD) $(srcdir)/iOS/testbed clone --framework $(PYTHONFRAMEWORKPREFIX) "$(XCFOLDER)" + $(PYTHON_FOR_BUILD) $(srcdir)/Apple/testbed clone --framework $(PYTHONFRAMEWORKPREFIX) "$(XCFOLDER)" # Run the testbed project $(PYTHON_FOR_BUILD) "$(XCFOLDER)" run --verbose -- test -uall --single-process --rerun -W @@ -2507,9 +2554,8 @@ maninstall: altmaninstall XMLLIBSUBDIRS= xml xml/dom xml/etree xml/parsers xml/sax LIBSUBDIRS= asyncio \ collections \ - compression compression/bz2 compression/gzip compression/zstd \ - compression/lzma compression/zlib compression/_common \ - concurrent concurrent/futures \ + compression compression/_common compression/zstd \ + concurrent concurrent/futures concurrent/interpreters \ csv \ ctypes ctypes/macholib \ curses \ @@ -2568,7 +2614,6 @@ TESTSUBDIRS= idlelib/idle_test \ test/subprocessdata \ test/support \ test/support/_hypothesis_stubs \ - test/support/interpreters \ test/test_asyncio \ test/test_capi \ test/test_cext \ @@ -2995,6 +3040,9 @@ frameworkinstallunversionedstructure: $(LDLIBRARY) $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR) sed 's/%VERSION%/'"`$(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import platform; print(platform.python_version())'`"'/g' < $(RESSRCDIR)/Info.plist > $(DESTDIR)$(PYTHONFRAMEWORKINSTALLDIR)/Info.plist $(INSTALL_SHARED) $(LDLIBRARY) $(DESTDIR)$(PYTHONFRAMEWORKPREFIX)/$(LDLIBRARY) + $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(LIBDIR) + $(LN) -fs "../$(LDLIBRARY)" "$(DESTDIR)$(prefix)/lib/libpython$(LDVERSION).dylib" + $(LN) -fs "../$(LDLIBRARY)" "$(DESTDIR)$(prefix)/lib/libpython$(VERSION).dylib" $(INSTALL) -d -m $(DIRMODE) $(DESTDIR)$(BINDIR) for file in $(srcdir)/$(RESSRCDIR)/bin/* ; do \ $(INSTALL) -m $(EXEMODE) $$file $(DESTDIR)$(BINDIR); \ @@ -3067,6 +3115,12 @@ config.status: $(srcdir)/configure Python/asm_trampoline.o: $(srcdir)/Python/asm_trampoline.S $(CC) -c $(PY_CORE_CFLAGS) -o $@ $< +Python/emscripten_trampoline_inner.wasm: $(srcdir)/Python/emscripten_trampoline_inner.c + # emcc has a path that ends with emsdk/upstream/emscripten/emcc, we're looking for emsdk/upstream/bin/clang. + $$(em-config LLVM_ROOT)/clang -o $@ $< -mgc -O2 -Wl,--no-entry -Wl,--import-table -Wl,--import-memory -target wasm32-unknown-unknown -nostdlib + +Python/emscripten_trampoline_wasm.c: Python/emscripten_trampoline_inner.wasm + $(PYTHON_FOR_REGEN) $(srcdir)/Platforms/emscripten/prepare_external_wasm.py $< $@ getWasmTrampolineModule JIT_DEPS = \ $(srcdir)/Tools/jit/*.c \ @@ -3095,6 +3149,9 @@ regen-jit: Python/dtoa.o: Python/dtoa.c $(CC) -c $(PY_CORE_CFLAGS) $(CFLAGS_ALIASING) -o $@ $< +Python/ceval.o: Python/ceval.c + $(CC) -c $(PY_CORE_CFLAGS) $(CFLAGS_CEVAL) -o $@ $< + # Run reindent on the library .PHONY: reindent reindent: @@ -3175,14 +3232,13 @@ clean-retain-profile: pycremoval -rm -rf Python/deepfreeze -rm -f Python/frozen_modules/*.h -rm -f Python/frozen_modules/MANIFEST - -rm -f jit_stencils.h -find build -type f -a ! -name '*.gc??' -exec rm -f {} ';' -rm -f Include/pydtrace_probes.h -rm -f profile-gen-stamp - -rm -rf iOS/testbed/Python.xcframework/ios-*/bin - -rm -rf iOS/testbed/Python.xcframework/ios-*/lib - -rm -rf iOS/testbed/Python.xcframework/ios-*/include - -rm -rf iOS/testbed/Python.xcframework/ios-*/Python.framework + -rm -rf Apple/iOS/testbed/Python.xcframework/ios-*/bin + -rm -rf Apple/iOS/testbed/Python.xcframework/ios-*/lib + -rm -rf Apple/iOS/testbed/Python.xcframework/ios-*/include + -rm -rf Apple/iOS/testbed/Python.xcframework/ios-*/Python.framework .PHONY: profile-removal profile-removal: @@ -3194,13 +3250,21 @@ profile-removal: rm -f profile-run-stamp rm -f profile-bolt-stamp -.PHONY: clean -clean: clean-retain-profile clean-bolt +.PHONY: clean-profile +clean-profile: clean-retain-profile clean-bolt @if test @DEF_MAKE_ALL_RULE@ = profile-opt -o @DEF_MAKE_ALL_RULE@ = bolt-opt; then \ rm -f profile-gen-stamp profile-clean-stamp; \ $(MAKE) profile-removal; \ fi +# gh-141808: The JIT stencils are deliberately kept in clean-profile +.PHONY: clean-jit-stencils +clean-jit-stencils: + -rm -f jit_stencils*.h + +.PHONY: clean +clean: clean-profile clean-jit-stencils + .PHONY: clobber clobber: clean -rm -f $(BUILDPYTHON) $(LIBRARY) $(LDLIBRARY) $(DLLLIBRARY) \ @@ -3208,7 +3272,7 @@ clobber: clean config.cache config.log pyconfig.h Modules/config.c -rm -rf build platform -rm -rf $(PYTHONFRAMEWORKDIR) - -rm -rf iOS/Frameworks + -rm -rf Apple/iOS/Frameworks -rm -rf iOSTestbed.* -rm -f python-config.py python-config -rm -rf cross-build @@ -3248,6 +3312,11 @@ check-c-globals: --format summary \ --traceback +# Check for undocumented C APIs. +.PHONY: check-c-api-docs +check-c-api-docs: + $(PYTHON_FOR_REGEN) $(srcdir)/Tools/check-c-api-docs/main.py + # Find files with funny names .PHONY: funny funny: @@ -3341,7 +3410,7 @@ MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/parts.h $(srcdir)/Modules/_tes MODULE__TESTLIMITEDCAPI_DEPS=$(srcdir)/Modules/_testlimitedcapi/testcapi_long.h $(srcdir)/Modules/_testlimitedcapi/parts.h $(srcdir)/Modules/_testlimitedcapi/util.h MODULE__TESTINTERNALCAPI_DEPS=$(srcdir)/Modules/_testinternalcapi/parts.h MODULE__SQLITE3_DEPS=$(srcdir)/Modules/_sqlite/connection.h $(srcdir)/Modules/_sqlite/cursor.h $(srcdir)/Modules/_sqlite/microprotocols.h $(srcdir)/Modules/_sqlite/module.h $(srcdir)/Modules/_sqlite/prepare_protocol.h $(srcdir)/Modules/_sqlite/row.h $(srcdir)/Modules/_sqlite/util.h -MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h +MODULE__ZSTD_DEPS=$(srcdir)/Modules/_zstd/_zstdmodule.h $(srcdir)/Modules/_zstd/buffer.h $(srcdir)/Modules/_zstd/zstddict.h CODECS_COMMON_HEADERS=$(srcdir)/Modules/cjkcodecs/multibytecodec.h $(srcdir)/Modules/cjkcodecs/cjkcodecs.h MODULE__CODECS_CN_DEPS=$(srcdir)/Modules/cjkcodecs/mappings_cn.h $(CODECS_COMMON_HEADERS) diff --git a/Misc/ACKS b/Misc/ACKS index 610dcf9f4238dee..1abeac0de245247 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -478,9 +478,11 @@ Dean Draayer Fred L. Drake, Jr. Mehdi Drissi Derk Drukker +Weilin Du John DuBois Paul Dubois Jacques Ducasse +Jadon Duff Andrei Dorian Duma Graham Dumpleton Quinn Dunkan @@ -658,6 +660,7 @@ Michael Goderbauer Karan Goel Jeroen Van Goey Christoph Gohlke +Daniel Golding Tim Golden Yonatan Goldschmidt Mark Gollahon @@ -763,6 +766,7 @@ Chris Herborth Ivan Herman Jürgen Hermann Joshua Jay Herman +Kevin Hernandez Gary Herron Ernie Hershey Thomas Herve @@ -900,6 +904,8 @@ Jim Jewett Pedro Diaz Jimenez Orjan Johansen Fredrik Johansson +Benjamin Johnson +Benjamin K. Johnson Gregory K. Johnson Kent Johnson Michael Johnson @@ -940,6 +946,7 @@ Anton Kasyanov Lou Kates Makoto Kato Irit Katriel +Kattni Hiroaki Kawai Dmitry Kazakov Brian Kearns @@ -1050,6 +1057,7 @@ Alexander Lakeev David Lam Thomas Lamb Valerie Lambert +Kliment Lamonov Peter Lamut Jean-Baptiste "Jiba" Lamy Ronan Lamy @@ -1063,6 +1071,7 @@ Wolfgang Langner Detlef Lannert Rémi Lapeyre Soren Larsen +Seth Michael Larson Amos Latteier Keenan Lau Piers Lauder @@ -1336,6 +1345,7 @@ Gustavo Niemeyer Oscar Nierstrasz Lysandros Nikolaou Hrvoje Nikšić +Jan-Eric Nitschke Gregory Nofi Jesse Noller Bill Noon @@ -1525,6 +1535,7 @@ Ashwin Ramaswami Jeff Ramnani Grant Ramsay Bayard Randel +Eashwar Ranganathan Varpu Rantala Brodie Rao Rémi Rampin @@ -1716,6 +1727,7 @@ Ha Shao Richard Shapiro Soumya Sharma Varun Sharma +Eoin Shaughnessy Daniel Shaulov Vlad Shcherbina Justin Sheehy @@ -1724,6 +1736,7 @@ Charlie Shepherd Bruce Sherwood Gregory Shevchenko Hai Shi +Daniel Shields Alexander Shigin Pete Shinners Michael Shiplett @@ -2096,6 +2109,8 @@ Xiang Zhang Robert Xiao Florent Xicluna Yanbo, Xie +Ivy Xu +Kaisheng Xu Xinhang Xu Arnon Yaari Alakshendra Yadav diff --git a/Misc/Brewfile b/Misc/Brewfile new file mode 100644 index 000000000000000..37e3acde4a057a6 --- /dev/null +++ b/Misc/Brewfile @@ -0,0 +1,14 @@ +brew "gdbm" +brew "mpdecimal" +brew "openssl@3.0" +brew "pkg-config" +brew "tcl-tk@8" +brew "xz" +brew "zstd" + +brew "bzip2" if OS.linux? +brew "libedit" if OS.linux? +brew "libffi" if OS.linux? +brew "ncurses" if OS.linux? +brew "unzip" if OS.linux? +brew "zlib-ng-compat" if OS.linux? diff --git a/Misc/NEWS.d/3.10.0a1.rst b/Misc/NEWS.d/3.10.0a1.rst index f09842f1e77dea9..349a7855e41c009 100644 --- a/Misc/NEWS.d/3.10.0a1.rst +++ b/Misc/NEWS.d/3.10.0a1.rst @@ -95,7 +95,7 @@ convention. Patch by Donghee Na. .. bpo: 1635741 .. date: 2020-09-26-14-43-30 .. nonce: aJS9B3 -.. section: Core and Builtins +.. section: Library Port the :mod:`!_bisect` module to the multi-phase initialization API (:pep:`489`). @@ -126,7 +126,7 @@ Taskaya. .. bpo: 1635741 .. date: 2020-09-12-18-34-34 .. nonce: lh335O -.. section: Core and Builtins +.. section: Library Port the :mod:`!_lsprof` extension module to multi-phase initialization (:pep:`489`). @@ -136,7 +136,7 @@ Port the :mod:`!_lsprof` extension module to multi-phase initialization .. bpo: 1635741 .. date: 2020-09-08-21-58-47 .. nonce: vdjSLH -.. section: Core and Builtins +.. section: Library Port the :mod:`cmath` extension module to multi-phase initialization (:pep:`489`). @@ -146,7 +146,7 @@ Port the :mod:`cmath` extension module to multi-phase initialization .. bpo: 1635741 .. date: 2020-09-08-20-39-43 .. nonce: jiXmyT -.. section: Core and Builtins +.. section: Library Port the :mod:`!_scproxy` extension module to multi-phase initialization (:pep:`489`). @@ -156,7 +156,7 @@ Port the :mod:`!_scproxy` extension module to multi-phase initialization .. bpo: 1635741 .. date: 2020-09-07-11-35-02 .. nonce: rvIexb -.. section: Core and Builtins +.. section: Library Port the :mod:`termios` extension module to multi-phase initialization (:pep:`489`). @@ -166,7 +166,7 @@ Port the :mod:`termios` extension module to multi-phase initialization .. bpo: 1635741 .. date: 2020-09-07-09-45-47 .. nonce: QuDIut -.. section: Core and Builtins +.. section: Library Convert the :mod:`!_sha256` extension module types to heap types. @@ -185,7 +185,7 @@ classes with a huge amount of arguments. Patch by Pablo Galindo. .. bpo: 1635741 .. date: 2020-09-01-17-22-35 .. nonce: CnRME3 -.. section: Core and Builtins +.. section: Library Port the :mod:`!_overlapped` extension module to multi-phase initialization (:pep:`489`). @@ -195,7 +195,7 @@ Port the :mod:`!_overlapped` extension module to multi-phase initialization .. bpo: 1635741 .. date: 2020-09-01-17-08-07 .. nonce: X9CZgo -.. section: Core and Builtins +.. section: Library Port the :mod:`!_curses_panel` extension module to multi-phase initialization (:pep:`489`). @@ -205,7 +205,7 @@ Port the :mod:`!_curses_panel` extension module to multi-phase initialization .. bpo: 1635741 .. date: 2020-09-01-17-06-02 .. nonce: 5jZymK -.. section: Core and Builtins +.. section: Library Port the :mod:`!_opcode` extension module to multi-phase initialization (:pep:`489`). @@ -225,7 +225,7 @@ format string in f-string and :meth:`str.format`. .. bpo: 41675 .. date: 2020-08-31-14-53-17 .. nonce: VSoqWU -.. section: Core and Builtins +.. section: Library The implementation of :func:`signal.siginterrupt` now uses :c:func:`!sigaction` (if it is available in the system) instead of the @@ -257,7 +257,7 @@ Fix a crash that occurred when destroying subclasses of .. bpo: 1635741 .. date: 2020-08-28-20-54-04 .. nonce: 7ijlcI -.. section: Core and Builtins +.. section: Library Port the :mod:`zlib` extension module to multi-phase initialization (:pep:`489`). @@ -280,7 +280,7 @@ initialized ``_ast`` module. .. bpo: 40077 .. date: 2020-08-25-22-43-33 .. nonce: vcxSUa -.. section: Core and Builtins +.. section: Library Convert :mod:`!_operator` to use :c:func:`PyType_FromSpec`. @@ -298,7 +298,7 @@ Port :mod:`!_sha3` to multi-phase init. Convert static types to heap types. .. bpo: 1635741 .. date: 2020-08-13-07-18-05 .. nonce: FC13e7 -.. section: Core and Builtins +.. section: Library Port the :mod:`!_blake2` extension module to the multi-phase initialization API (:pep:`489`). @@ -337,7 +337,7 @@ The output of ``python --help`` contains now only ASCII characters. .. bpo: 1635741 .. date: 2020-08-10-16-11-32 .. nonce: O0d3ym -.. section: Core and Builtins +.. section: Library Port the :mod:`!_sha1`, :mod:`!_sha512`, and :mod:`!_md5` extension modules to multi-phase initialization API (:pep:`489`). @@ -454,7 +454,7 @@ Port :mod:`multiprocessing` to multi-phase initialization .. bpo: 1635741 .. date: 2020-07-06-20-43-19 .. nonce: LYhsni -.. section: Core and Builtins +.. section: Library Port :mod:`winapi` to multiphase initialization @@ -486,7 +486,7 @@ will tagged as so. .. bpo: 1635741 .. date: 2020-07-03-23-10-02 .. nonce: F5coWe -.. section: Core and Builtins +.. section: Library Port :mod:`faulthandler` to multiphase initialization. @@ -495,7 +495,7 @@ Port :mod:`faulthandler` to multiphase initialization. .. bpo: 1635741 .. date: 2020-07-01-20-17-38 .. nonce: -AtPYu -.. section: Core and Builtins +.. section: Library Port :mod:`sha256` to multiphase initialization @@ -634,7 +634,7 @@ Remove the remaining files from the old parser and the :mod:`symbol` module. .. bpo: 40077 .. date: 2020-06-18-19-04-30 .. nonce: _yI-ax -.. section: Core and Builtins +.. section: Library Convert :mod:`!_bz2` to use :c:func:`PyType_FromSpec`. @@ -712,7 +712,7 @@ Fix refleak in _Py_fopen_obj() when PySys_Audit() fails .. bpo: 40950 .. date: 2020-06-12-00-12-28 .. nonce: tzMy7m -.. section: Core and Builtins +.. section: Library Add a state to the :mod:`!nis` module (:pep:`3121`) and apply the multiphase initialization. Patch by Donghee Na. @@ -999,7 +999,7 @@ Improve performance of generators by not raising internal StopIteration. .. bpo: 1635741 .. date: 2020-04-10-23-54-57 .. nonce: ZURqoN -.. section: Core and Builtins +.. section: Library Port :mod:`mmap` to multiphase initialization. @@ -1008,7 +1008,7 @@ Port :mod:`mmap` to multiphase initialization. .. bpo: 1635741 .. date: 2020-04-05-02-35-08 .. nonce: Kfe9fT -.. section: Core and Builtins +.. section: Library Port :mod:`!_lzma` to multiphase initialization. @@ -1040,7 +1040,7 @@ representation of an integer. Patch by Niklas Fiekas. .. bpo: 36982 .. date: 2019-05-25-05-27-39 .. nonce: 0UHgfB -.. section: Core and Builtins +.. section: Library Use ncurses extended color functions when available to support terminals with 256 colors, and add the new function @@ -3275,8 +3275,8 @@ Types created with :c:func:`PyType_FromSpec` now make any signature in their .. nonce: u6Xfr2 .. section: C API -Fix bug in PyOS_mystrnicmp and PyOS_mystricmp that incremented pointers -beyond the end of a string. +Fix bug in :c:func:`PyOS_mystrnicmp` and :c:func:`PyOS_mystricmp` that +incremented pointers beyond the end of a string. .. diff --git a/Misc/NEWS.d/3.10.0a2.rst b/Misc/NEWS.d/3.10.0a2.rst index 3e82de9ef266d66..bae62d93759e2dd 100644 --- a/Misc/NEWS.d/3.10.0a2.rst +++ b/Misc/NEWS.d/3.10.0a2.rst @@ -212,7 +212,7 @@ Micro optimization for range.index if step is 1. Patch by Donghee Na. .. bpo: 41435 .. date: 2020-08-07-13-42-48 .. nonce: qPWjJA -.. section: Core and Builtins +.. section: Library Add ``sys._current_exceptions()`` function to retrieve a dictionary mapping each thread's identifier to the topmost exception currently active in that diff --git a/Misc/NEWS.d/3.10.0a3.rst b/Misc/NEWS.d/3.10.0a3.rst index 33c3e14b7a4bcfd..45c4d8761408216 100644 --- a/Misc/NEWS.d/3.10.0a3.rst +++ b/Misc/NEWS.d/3.10.0a3.rst @@ -89,7 +89,7 @@ non-buffer object. .. bpo: 1635741 .. date: 2020-11-18-23-46-31 .. nonce: GVOQ-m -.. section: Core and Builtins +.. section: Library Port the ``_warnings`` extension module to the multi-phase initialization API (:pep:`489`). Patch by Victor Stinner. @@ -204,7 +204,7 @@ Pablo Galindo. .. bpo: 40077 .. date: 2020-11-03-21-58-27 .. nonce: a9qM1j -.. section: Core and Builtins +.. section: Library Convert :mod:`queue` to use heap types. @@ -223,7 +223,7 @@ objects. See PEP 626 for details. .. bpo: 40077 .. date: 2020-11-02-14-39-48 .. nonce: grY9TG -.. section: Core and Builtins +.. section: Library Convert :mod:`mmap` to use heap types. diff --git a/Misc/NEWS.d/3.10.0a4.rst b/Misc/NEWS.d/3.10.0a4.rst index 19f0db9a6be5e98..16eca7a55db746e 100644 --- a/Misc/NEWS.d/3.10.0a4.rst +++ b/Misc/NEWS.d/3.10.0a4.rst @@ -102,7 +102,7 @@ blocks .. bpo: 42639 .. date: 2020-12-09-01-55-10 .. nonce: 5pI5HG -.. section: Core and Builtins +.. section: Library Make the :mod:`atexit` module state per-interpreter. It is now safe have more than one :mod:`atexit` module instance. Patch by Donghee Na and Victor @@ -124,7 +124,7 @@ the filename. .. bpo: 42195 .. date: 2020-11-20-00-57-47 .. nonce: HeqcpS -.. section: Core and Builtins +.. section: Library The ``__args__`` of the parameterized generics for :data:`typing.Callable` and :class:`collections.abc.Callable` are now consistent. The ``__args__`` @@ -143,7 +143,7 @@ Ken Jin. .. bpo: 40137 .. date: 2020-11-19-23-12-57 .. nonce: bihl9O -.. section: Core and Builtins +.. section: Library Convert functools module to use :c:func:`PyType_FromModuleAndSpec`. @@ -152,7 +152,7 @@ Convert functools module to use :c:func:`PyType_FromModuleAndSpec`. .. bpo: 40077 .. date: 2020-11-03-13-46-10 .. nonce: NfAIdj -.. section: Core and Builtins +.. section: Library Convert :mod:`array` to use heap types, and establish module state for these. @@ -162,7 +162,7 @@ these. .. bpo: 42008 .. date: 2020-10-12-14-51-59 .. nonce: ijWw2I -.. section: Core and Builtins +.. section: Library Fix _random.Random() seeding. @@ -171,7 +171,7 @@ Fix _random.Random() seeding. .. bpo: 1635741 .. date: 2020-09-12-19-21-52 .. nonce: F2kDrU -.. section: Core and Builtins +.. section: Library Port the :mod:`pyexpat` extension module to multi-phase initialization (:pep:`489`). @@ -622,7 +622,7 @@ Harmonized :func:`random.randrange` argument handling to match :func:`range`. .. nonce: O4VcCY .. section: Library -Restore compatibility for ``uname_result`` around deepcopy and _replace. +Restore compatibility for :class:`os.uname_result` around deepcopy and _replace. .. diff --git a/Misc/NEWS.d/3.10.0a5.rst b/Misc/NEWS.d/3.10.0a5.rst index a85ea1ff1c28178..6ac0acc0b800a59 100644 --- a/Misc/NEWS.d/3.10.0a5.rst +++ b/Misc/NEWS.d/3.10.0a5.rst @@ -136,7 +136,7 @@ frame.f_lineno is correct even if frame.f_trace is set to True .. bpo: 37324 .. date: 2020-12-12-20-09-12 .. nonce: jB-9_U -.. section: Core and Builtins +.. section: Library Remove deprecated aliases to :ref:`collections-abstract-base-classes` from the :mod:`collections` module. diff --git a/Misc/NEWS.d/3.10.0a6.rst b/Misc/NEWS.d/3.10.0a6.rst index 31b7df2c61158e4..ec5b868708ba0ba 100644 --- a/Misc/NEWS.d/3.10.0a6.rst +++ b/Misc/NEWS.d/3.10.0a6.rst @@ -88,7 +88,7 @@ Patch by Pablo Galindo. .. bpo: 42819 .. date: 2021-01-04-23-54-34 .. nonce: 4KO6wU -.. section: Core and Builtins +.. section: Library :mod:`readline`: Explicitly disable bracketed paste in the interactive interpreter, even if it's set in the inputrc, is enabled by default (eg GNU @@ -144,7 +144,7 @@ Implement :pep:`634` (structural pattern matching). Patch by Brandt Bucher. .. bpo: 40692 .. date: 2020-05-19-22-10-05 .. nonce: ajEhrR -.. section: Core and Builtins +.. section: Library In the :class:`concurrent.futures.ProcessPoolExecutor`, validate that :func:`multiprocess.synchronize` is available on a given platform and rely diff --git a/Misc/NEWS.d/3.10.0a7.rst b/Misc/NEWS.d/3.10.0a7.rst index d866e805fd3a7ed..33e5e073202efc3 100644 --- a/Misc/NEWS.d/3.10.0a7.rst +++ b/Misc/NEWS.d/3.10.0a7.rst @@ -81,7 +81,7 @@ instruction dispatch a bit. .. bpo: 40645 .. date: 2021-03-29-11-55-06 .. nonce: PhaT-B -.. section: Core and Builtins +.. section: Library Fix reference leak in the :mod:`!_hashopenssl` extension. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/3.10.0b1.rst b/Misc/NEWS.d/3.10.0b1.rst index 406a5d7853edc0f..adae971d4002d1b 100644 --- a/Misc/NEWS.d/3.10.0b1.rst +++ b/Misc/NEWS.d/3.10.0b1.rst @@ -202,7 +202,7 @@ line number tables more robust in some circumstances. .. bpo: 43908 .. date: 2021-04-26-21-20-41 .. nonce: 2L51nO -.. section: Core and Builtins +.. section: Library Make :mod:`re` types immutable. Patch by Erlend E. Aasland. @@ -211,7 +211,7 @@ Make :mod:`re` types immutable. Patch by Erlend E. Aasland. .. bpo: 43908 .. date: 2021-04-26-20-59-17 .. nonce: -COW4- -.. section: Core and Builtins +.. section: Library Make the :class:`array.array` type immutable. Patch by Erlend E. Aasland. @@ -402,8 +402,8 @@ the heap. Should speed up dispatch in the interpreter. .. nonce: eUn4p5 .. section: Core and Builtins -Static methods (:func:`@staticmethod <staticmethod>`) and class methods -(:func:`@classmethod <classmethod>`) now inherit the method attributes +Static methods (:deco:`staticmethod`) and class methods +(:deco:`classmethod`) now inherit the method attributes (``__module__``, ``__name__``, ``__qualname__``, ``__doc__``, ``__annotations__``) and have a new ``__wrapped__`` attribute. Patch by Victor Stinner. @@ -442,7 +442,7 @@ coroutine. .. bpo: 43105 .. date: 2021-03-31-20-35-11 .. nonce: PBVmHm -.. section: Core and Builtins +.. section: Library Importlib now resolves relative paths when creating module spec objects from file locations. @@ -454,7 +454,7 @@ file locations. .. nonce: VSF3vg .. section: Core and Builtins -Static methods (:func:`@staticmethod <staticmethod>`) are now callable as +Static methods (:deco:`staticmethod`) are now callable as regular functions. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.11.0a1.rst b/Misc/NEWS.d/3.11.0a1.rst index 0b49c2a78771d26..af77a79823ea2cc 100644 --- a/Misc/NEWS.d/3.11.0a1.rst +++ b/Misc/NEWS.d/3.11.0a1.rst @@ -127,7 +127,7 @@ default value of ``1`` for the ``length`` argument. .. bpo: 44219 .. date: 2021-09-09-10-32-33 .. nonce: WiYyjz -.. section: Core and Builtins +.. section: Library Release the GIL while performing ``isatty`` system calls on arbitrary file descriptors. In particular, this affects :func:`os.isatty`, @@ -149,7 +149,7 @@ Added fallback to extension modules with '.sl' suffix on HP-UX .. bpo: 45121 .. date: 2021-09-07-17-10-16 .. nonce: iG-Hsf -.. section: Core and Builtins +.. section: Library Fix issue where ``Protocol.__init__`` raises ``RecursionError`` when it's called directly or via ``super()``. Patch provided by Yurii Karabas. @@ -269,7 +269,7 @@ it's running out of the source tree). .. bpo: 45012 .. date: 2021-08-31-11-09-52 .. nonce: ueeOcx -.. section: Core and Builtins +.. section: Library In :mod:`posix`, release GIL during ``stat()``, ``lstat()``, and ``fstatat()`` syscalls made by :func:`os.DirEntry.stat`. Patch by Stanisław @@ -309,7 +309,7 @@ objects. Patch by Pablo Galindo. .. bpo: 44962 .. date: 2021-08-23-19-55-08 .. nonce: J00ftt -.. section: Core and Builtins +.. section: Library Fix a race in WeakKeyDictionary, WeakValueDictionary and WeakSet when two threads attempt to commit the last pending removal. This fixes @@ -379,7 +379,7 @@ new instructions: .. bpo: 44929 .. date: 2021-08-16-23-16-17 .. nonce: qpMEky -.. section: Core and Builtins +.. section: Library Fix some edge cases of ``enum.Flag`` string representation in the REPL. Patch by Pablo Galindo. @@ -728,7 +728,7 @@ Collapse union of equal types. E.g. the result of ``int | int`` is now .. bpo: 44611 .. date: 2021-07-16-01-01-11 .. nonce: LcfHN- -.. section: Core and Builtins +.. section: Library On Windows, :func:`os.urandom`: uses BCryptGenRandom API instead of CryptGenRandom API which is deprecated from Microsoft Windows API. Patch by @@ -840,7 +840,7 @@ from dynload_shlib.c. .. bpo: 44490 .. date: 2021-07-06-22-22-15 .. nonce: BJxPbZ -.. section: Core and Builtins +.. section: Library :mod:`typing` now searches for type parameters in ``types.Union`` objects. ``get_type_hints`` will also properly resolve annotations with nested @@ -875,7 +875,7 @@ Remove uses of :c:func:`PyObject_GC_Del` in error path when initializing .. bpo: 41486 .. date: 2021-07-04-17-41-47 .. nonce: DiM24a -.. section: Core and Builtins +.. section: Library Fix a memory consumption and copying performance regression in earlier 3.10 beta releases if someone used an output buffer larger than 4GiB with @@ -1116,7 +1116,7 @@ Improve tokenizer error with improved locations. Patch by Pablo Galindo. .. bpo: 44304 .. date: 2021-06-05-02-34-57 .. nonce: _MAoPc -.. section: Core and Builtins +.. section: Library Fix a crash in the :mod:`sqlite3` module that happened when the garbage collector clears :class:`sqlite.Statement` objects. Patch by Pablo Galindo @@ -2953,7 +2953,7 @@ support for Metadata 2.2. .. nonce: xTUyyX .. section: Library -Remove the :func:`@asyncio.coroutine <asyncio.coroutine>` :term:`decorator` +Remove the :deco:`asyncio.coroutine` :term:`decorator` enabling legacy generator-based coroutines to be compatible with async/await code; remove :class:`asyncio.coroutines.CoroWrapper` used for wrapping legacy coroutine objects in the debug mode. The decorator has been diff --git a/Misc/NEWS.d/3.11.0a3.rst b/Misc/NEWS.d/3.11.0a3.rst index 6a0ae20d1fb5edf..c051074f50a1481 100644 --- a/Misc/NEWS.d/3.11.0a3.rst +++ b/Misc/NEWS.d/3.11.0a3.rst @@ -133,7 +133,7 @@ additional allocation when the frame object outlives the frame activation. .. bpo: 45614 .. date: 2021-11-23-12-06-41 .. nonce: fIekgI -.. section: Core and Builtins +.. section: Library Fix :mod:`traceback` display for exceptions with invalid module name. diff --git a/Misc/NEWS.d/3.11.0a4.rst b/Misc/NEWS.d/3.11.0a4.rst index a2d362020458874..df8b3cdda8eeec7 100644 --- a/Misc/NEWS.d/3.11.0a4.rst +++ b/Misc/NEWS.d/3.11.0a4.rst @@ -119,7 +119,7 @@ perform tracing and optimizer checks. .. bpo: 46208 .. date: 2022-01-04-01-53-35 .. nonce: i00Vz5 -.. section: Core and Builtins +.. section: Library Fix the regression of os.path.normpath("A/../../B") not returning expected "../B" but "B". @@ -178,7 +178,7 @@ sequence of other opcodes. .. bpo: 46085 .. date: 2021-12-30-00-23-41 .. nonce: bDuJqu -.. section: Core and Builtins +.. section: Library Fix iterator cache mechanism of :class:`OrderedDict`. @@ -344,7 +344,7 @@ themselves before raising exceptions. Patch by Pablo Galindo. .. bpo: 46000 .. date: 2021-12-07-11-42-44 .. nonce: v_ru3k -.. section: Core and Builtins +.. section: Library Improve compatibility of the :mod:`curses` module with NetBSD curses. diff --git a/Misc/NEWS.d/3.11.0a5.rst b/Misc/NEWS.d/3.11.0a5.rst index 5418d5d59dd5839..de03458062b95d8 100644 --- a/Misc/NEWS.d/3.11.0a5.rst +++ b/Misc/NEWS.d/3.11.0a5.rst @@ -204,7 +204,7 @@ platform when dividing an int by a value smaller than ``2**30``. .. bpo: 46383 .. date: 2022-01-14-20-55-34 .. nonce: v8MTl4 -.. section: Core and Builtins +.. section: Library Fix invalid signature of ``_zoneinfo``'s ``module_free`` function to resolve a crash on wasm32-emscripten platform. @@ -255,7 +255,7 @@ that are ended by line continuation characters. Patch by Pablo Galindo .. bpo: 30512 .. date: 2021-12-12-00-49-19 .. nonce: nU9E9V -.. section: Core and Builtins +.. section: Library Add CAN Socket support for NetBSD. @@ -264,7 +264,7 @@ Add CAN Socket support for NetBSD. .. bpo: 46045 .. date: 2021-12-11-11-36-48 .. nonce: sfThay -.. section: Core and Builtins +.. section: Build Do not use POSIX semaphores on NetBSD diff --git a/Misc/NEWS.d/3.11.0a6.rst b/Misc/NEWS.d/3.11.0a6.rst index e88142e641f0400..7fdfa4ebdf2e374 100644 --- a/Misc/NEWS.d/3.11.0a6.rst +++ b/Misc/NEWS.d/3.11.0a6.rst @@ -255,7 +255,7 @@ Fix specialization stats gathering for :opcode:`!PRECALL` instructions. .. bpo: 46794 .. date: 2022-02-22-12-07-53 .. nonce: 6WvJ9o -.. section: Core and Builtins +.. section: Library Bump up the libexpat version into 2.4.6 @@ -379,7 +379,7 @@ involving lots of brackets. Patch by Pablo Galindo. .. bpo: 46323 .. date: 2022-02-10-02-29-12 .. nonce: HK_cs0 -.. section: Core and Builtins +.. section: Library :mod:`ctypes` now allocates memory on the stack instead of on the heap to pass arguments while calling a Python callback function. Patch by Donghee @@ -429,7 +429,7 @@ now 254. .. bpo: 40479 .. date: 2022-02-06-23-08-30 .. nonce: zED3Zu -.. section: Core and Builtins +.. section: Library Add a missing call to ``va_end()`` in ``Modules/_hashopenssl.c``. @@ -438,7 +438,7 @@ Add a missing call to ``va_end()`` in ``Modules/_hashopenssl.c``. .. bpo: 46323 .. date: 2022-02-05-14-46-21 .. nonce: FC1OJg -.. section: Core and Builtins +.. section: Library Use :c:func:`PyObject_Vectorcall` while calling ctypes callback function. Patch by Donghee Na. diff --git a/Misc/NEWS.d/3.11.0a7.rst b/Misc/NEWS.d/3.11.0a7.rst index eff2ea2dac13f8c..fddbb63218d1b93 100644 --- a/Misc/NEWS.d/3.11.0a7.rst +++ b/Misc/NEWS.d/3.11.0a7.rst @@ -181,7 +181,7 @@ to simplify clearing and deallocing frames and generators. .. bpo: 46968 .. date: 2022-03-17-14-22-23 .. nonce: 4gz4NA -.. section: Core and Builtins +.. section: Library Check for the existence of the "sys/auxv.h" header in :mod:`faulthandler` to avoid compilation problems in systems where this header doesn't exist. Patch @@ -243,7 +243,7 @@ reducing the number of invocations of ``memcpy``. .. bpo: 46829 .. date: 2022-03-12-21-07-21 .. nonce: cpGoPV -.. section: Core and Builtins +.. section: Library Deprecate passing a message into :meth:`asyncio.Future.cancel` and :meth:`asyncio.Task.cancel` diff --git a/Misc/NEWS.d/3.11.0b1.rst b/Misc/NEWS.d/3.11.0b1.rst index c3a1942b881ad46..b7ea1f018838a2c 100644 --- a/Misc/NEWS.d/3.11.0b1.rst +++ b/Misc/NEWS.d/3.11.0b1.rst @@ -388,7 +388,7 @@ Add internal documentation explaining design of new (for 3.11) frame stack. .. bpo: 47197 .. date: 2022-04-03-17-21-04 .. nonce: Ji_c30 -.. section: Core and Builtins +.. section: Library ctypes used to mishandle ``void`` return types, so that for instance a function declared like ``ctypes.CFUNCTYPE(None, ctypes.c_int)`` would be @@ -500,7 +500,7 @@ at runtime where types are known at C compile time. .. bpo: 46045 .. date: 2021-12-11-11-36-48 .. nonce: sfThay -.. section: Core and Builtins +.. section: Build Do not use POSIX semaphores on NetBSD @@ -664,7 +664,7 @@ for :func:`os.fcopyfile` available in macOs. .. nonce: l1p7CJ .. section: Library -For :func:`@dataclass <dataclasses.dataclass>`, add *weakref_slot*. +For :deco:`~dataclasses.dataclass`, add *weakref_slot*. The new parameter defaults to ``False``. If true, and if ``slots=True``, add a slot named ``"__weakref__"``, which will allow instances to be weakref'd. Contributed by Eric V. Smith diff --git a/Misc/NEWS.d/3.12.0a1.rst b/Misc/NEWS.d/3.12.0a1.rst index f2668e99a6299b6..c977022d87fc958 100644 --- a/Misc/NEWS.d/3.12.0a1.rst +++ b/Misc/NEWS.d/3.12.0a1.rst @@ -160,7 +160,7 @@ to calculate those doing pointer arithmetic. .. date: 2022-10-06-15-45-57 .. gh-issue: 96078 .. nonce: fS-6mU -.. section: Core and Builtins +.. section: Library :func:`os.sched_yield` now release the GIL while calling sched_yield(2). Patch by Donghee Na. @@ -170,7 +170,7 @@ Patch by Donghee Na. .. date: 2022-10-06-14-14-28 .. gh-issue: 97955 .. nonce: Nq5VXD -.. section: Core and Builtins +.. section: Library Migrate :mod:`zoneinfo` to Argument Clinic. @@ -361,7 +361,7 @@ branching conditions. .. date: 2022-09-19-03-35-01 .. gh-issue: 96821 .. nonce: izK6JA -.. section: Core and Builtins +.. section: Library Fix undefined behaviour in ``audioop.c``. @@ -481,7 +481,7 @@ Fix case of undefined behavior in ceval.c .. date: 2022-09-08-20-58-10 .. gh-issue: 64373 .. nonce: AfCi36 -.. section: Core and Builtins +.. section: Library Convert :mod:`!_functools` to argument clinic. @@ -490,7 +490,7 @@ Convert :mod:`!_functools` to argument clinic. .. date: 2022-09-07-13-38-37 .. gh-issue: 96641 .. nonce: wky0Fc -.. section: Core and Builtins +.. section: Library Do not expose ``KeyWrapper`` in :mod:`!_functools`. @@ -990,7 +990,7 @@ bytecode compiler. .. date: 2022-07-20-09-04-55 .. gh-issue: 95023 .. nonce: bs-xd7 -.. section: Core and Builtins +.. section: Library Implement :func:`os.setns` and :func:`os.unshare` for Linux. Patch by Noam Cohen. @@ -1021,7 +1021,7 @@ Previously it could cause SystemError or other undesired behavior. .. date: 2022-07-19-04-34-56 .. gh-issue: 94996 .. nonce: dV564A -.. section: Core and Builtins +.. section: Library :func:`ast.parse` will no longer parse function definitions with positional-only params when passed ``feature_version`` less than ``(3, 8)``. @@ -1041,7 +1041,7 @@ Allow jumping within, out of, and across exception handlers in the debugger. .. date: 2022-07-18-05-10-29 .. gh-issue: 94949 .. nonce: OsZ7_s -.. section: Core and Builtins +.. section: Library :func:`ast.parse` will no longer parse parenthesized context managers when passed ``feature_version`` less than ``(3, 9)``. Patch by Shantanu Jain. @@ -1051,7 +1051,7 @@ passed ``feature_version`` less than ``(3, 9)``. Patch by Shantanu Jain. .. date: 2022-07-18-04-48-34 .. gh-issue: 94947 .. nonce: df9gUw -.. section: Core and Builtins +.. section: Library :func:`ast.parse` will no longer parse assignment expressions when passed ``feature_version`` less than ``(3, 8)``. Patch by Shantanu Jain. @@ -1394,7 +1394,7 @@ calls. Previously, the end column could precede the column offset. .. date: 2022-06-09-19-19-02 .. gh-issue: 93461 .. nonce: 5DqP1e -.. section: Core and Builtins +.. section: Library :func:`importlib.invalidate_caches` now drops entries from :data:`sys.path_importer_cache` with a relative path as name. This solves a @@ -1729,7 +1729,7 @@ tracing functions implemented in C. .. date: 2022-05-11-09-16-54 .. gh-issue: 91102 .. nonce: lenv9h -.. section: Core and Builtins +.. section: Library :meth:`!_warnings.warn_explicit` is ported to Argument Clinic. @@ -1759,7 +1759,7 @@ no longer does anything. .. date: 2022-05-03-20-12-18 .. gh-issue: 92261 .. nonce: aigLnb -.. section: Core and Builtins +.. section: Library Fix hang when trying to iterate over a ``typing.Union``. @@ -4330,7 +4330,7 @@ and ``sendfile`` inside ``IocpProactor``. .. nonce: GsBL9- .. section: Library -Fixed :meth:`collections.UserDict.get` to not call :meth:`__missing__` when +Fixed :meth:`collections.UserDict.get` to not call :meth:`~object.__missing__` when a value is not found. This matches the behavior of :class:`dict`. Patch by Bar Harel. diff --git a/Misc/NEWS.d/3.12.0a2.rst b/Misc/NEWS.d/3.12.0a2.rst index bc028f30636bf7e..9c73ae95d10a4c6 100644 --- a/Misc/NEWS.d/3.12.0a2.rst +++ b/Misc/NEWS.d/3.12.0a2.rst @@ -35,11 +35,11 @@ Update bundled libexpat to 2.5.0 .. nonce: ik4iOv .. section: Core and Builtins -The docs clearly say that ``PyImport_Inittab``, +The docs clearly say that :c:data:`PyImport_Inittab`, :c:func:`PyImport_AppendInittab`, and :c:func:`PyImport_ExtendInittab` should not be used after :c:func:`Py_Initialize` has been called. We now enforce this for the two functions. Additionally, the runtime now uses an -internal copy of ``PyImport_Inittab``, to guard against modification. +internal copy of :c:data:`PyImport_Inittab`, to guard against modification. .. @@ -333,7 +333,7 @@ aware of this shim frame and the changes to the semantics of .. date: 2022-10-19-01-01-08 .. gh-issue: 98415 .. nonce: ZS2eWh -.. section: Core and Builtins +.. section: Build Fix detection of MAC addresses for :mod:`uuid` on certain OSs. Patch by Chaim Sanders @@ -405,7 +405,7 @@ Expose :const:`~socket.ETH_P_ALL` and some of the :ref:`ETHERTYPE_* constants .. date: 2022-06-10-16-37-44 .. gh-issue: 93696 .. nonce: 65BI2R -.. section: Core and Builtins +.. section: Library Allow :mod:`pdb` to locate source for frozen modules in the standard library. diff --git a/Misc/NEWS.d/3.12.0a3.rst b/Misc/NEWS.d/3.12.0a3.rst index 04a2bf9fb916b70..c71a66757c65662 100644 --- a/Misc/NEWS.d/3.12.0a3.rst +++ b/Misc/NEWS.d/3.12.0a3.rst @@ -100,7 +100,7 @@ Fix bug where an :exc:`ExceptionGroup` subclass can wrap a .. date: 2022-11-16-21-35-30 .. gh-issue: 99547 .. nonce: p_c_bp -.. section: Core and Builtins +.. section: Library Add a function to os.path to check if a path is a junction: isjunction. Add similar functionality to pathlib.Path as is_junction. @@ -110,7 +110,7 @@ similar functionality to pathlib.Path as is_junction. .. date: 2022-11-12-01-39-57 .. gh-issue: 99370 .. nonce: _cu32j -.. section: Core and Builtins +.. section: Library Fix zip path for venv created from a non-installed python on POSIX platforms. @@ -454,8 +454,8 @@ event loop but the current event loop was set. .. nonce: humlhz .. section: Library -On ``uname_result``, restored expectation that ``_fields`` and ``_asdict`` -would include all six properties including ``processor``. +On :class:`os.uname_result`, restored expectation that ``_fields`` and +``_asdict`` would include all six properties including ``processor``. .. diff --git a/Misc/NEWS.d/3.12.0a4.rst b/Misc/NEWS.d/3.12.0a4.rst index 57fb2052764b6f2..1fdebf54da9cdc9 100644 --- a/Misc/NEWS.d/3.12.0a4.rst +++ b/Misc/NEWS.d/3.12.0a4.rst @@ -125,7 +125,7 @@ Improve the accuracy of ``sum()`` with compensated summation. .. date: 2022-12-20-16-14-19 .. gh-issue: 100374 .. nonce: YRrVHT -.. section: Core and Builtins +.. section: Library Fix incorrect result and delay in :func:`socket.getfqdn`. Patch by Dominic Socular. @@ -315,7 +315,7 @@ Improve performance of ``list.pop`` for small lists. .. date: 2022-06-17-08-00-34 .. gh-issue: 89051 .. nonce: yP4Na0 -.. section: Core and Builtins +.. section: Library Add :const:`ssl.OP_LEGACY_SERVER_CONNECT` diff --git a/Misc/NEWS.d/3.12.0a5.rst b/Misc/NEWS.d/3.12.0a5.rst index 5dc443bb55b617b..fb0f4b22c6b0915 100644 --- a/Misc/NEWS.d/3.12.0a5.rst +++ b/Misc/NEWS.d/3.12.0a5.rst @@ -181,7 +181,7 @@ regen-all``. .. bpo: 32780 .. date: 2018-02-05-21-54-46 .. nonce: Dtiz8z -.. section: Core and Builtins +.. section: Library Inter-field padding is now inserted into the PEP3118 format strings obtained from :class:`ctypes.Structure` objects, reflecting their true representation diff --git a/Misc/NEWS.d/3.12.0a7.rst b/Misc/NEWS.d/3.12.0a7.rst index f48b9ce0550440f..e0c079c5def9712 100644 --- a/Misc/NEWS.d/3.12.0a7.rst +++ b/Misc/NEWS.d/3.12.0a7.rst @@ -102,7 +102,7 @@ Shrink the number of inline :opcode:`CACHE` entries used by .. date: 2023-03-08-08-37-36 .. gh-issue: 102491 .. nonce: SFvvsC -.. section: Core and Builtins +.. section: Library Improve import time of ``platform`` by removing IronPython version parsing. The IronPython version parsing was not functional (see @@ -187,7 +187,7 @@ Improve build support for the Xbox. Patch by Max Bachmann. .. date: 2023-02-21-23-42-39 .. gh-issue: 102027 .. nonce: fQARG0 -.. section: Core and Builtins +.. section: Build Fix SSE2 and SSE3 detection in ``_blake2`` internal module. Patch by Max Bachmann. @@ -207,7 +207,7 @@ Deprecate ``co_lnotab`` in code objects, schedule it for removal in Python .. bpo: 1635741 .. date: 2020-07-04-09-04-41 .. nonce: ZsP31Y -.. section: Core and Builtins +.. section: Library Adapt :mod:`!_pickle` to :pep:`687`. Patch by Mohamed Koubaa and Erlend Aasland. diff --git a/Misc/NEWS.d/3.12.0b1.rst b/Misc/NEWS.d/3.12.0b1.rst index 3a3870ac9fe6210..aa4a32b4e779384 100644 --- a/Misc/NEWS.d/3.12.0b1.rst +++ b/Misc/NEWS.d/3.12.0b1.rst @@ -44,7 +44,7 @@ response to :cve:`2023-24329`. Patch by Illia Volochii. .. date: 2023-05-20-23-08-48 .. gh-issue: 102856 .. nonce: Knv9WT -.. section: Core and Builtins +.. section: Library Implement PEP 701 changes in the :mod:`tokenize` module. Patch by Marta Gómez Macías and Pablo Galindo Salgado @@ -312,7 +312,7 @@ Patch by Eric Traut, Larry Hastings, and Jelle Zijlstra. .. date: 2023-04-24-21-47-38 .. gh-issue: 103801 .. nonce: WaBanq -.. section: Core and Builtins +.. section: Library Adds three minor linting fixes to the wasm module caught that were caught by ruff. @@ -322,7 +322,7 @@ ruff. .. date: 2023-04-24-14-38-16 .. gh-issue: 103793 .. nonce: kqoH6Q -.. section: Core and Builtins +.. section: Library Optimized asyncio Task creation by deferring expensive string formatting (task name generation) from Task creation to the first time ``get_name`` is @@ -573,7 +573,7 @@ raising an :exc:`IndexError`. .. bpo: 31821 .. date: 2019-12-01-12-58-31 .. nonce: 1FNmwk -.. section: Core and Builtins +.. section: Library Fix :func:`!pause_reading` to work when called from :func:`!connection_made` in :mod:`asyncio`. diff --git a/Misc/NEWS.d/3.13.0a5.rst b/Misc/NEWS.d/3.13.0a5.rst index d56b1542b018230..6f8c82e5af3d75d 100644 --- a/Misc/NEWS.d/3.13.0a5.rst +++ b/Misc/NEWS.d/3.13.0a5.rst @@ -136,7 +136,7 @@ threads to be interrupted. .. date: 2024-02-08-16-01-18 .. gh-issue: 115154 .. nonce: ji96FV -.. section: Core and Builtins +.. section: Library Fix a bug that was causing the :func:`tokenize.untokenize` function to handle unicode named literals incorrectly. Patch by Pablo Galindo @@ -156,7 +156,7 @@ Add ability to force alignment of :mod:`ctypes.Structure` by way of the new .. date: 2023-07-16-15-02-47 .. gh-issue: 104090 .. nonce: oMjNa9 -.. section: Core and Builtins +.. section: Library The multiprocessing resource tracker now exits with non-zero status code if a resource leak was detected. It still exits with status code 0 otherwise. @@ -742,8 +742,8 @@ Add ``windows_31j`` to aliases for ``cp932`` codec .. nonce: fv35wU .. section: Library -:func:`functools.partial`s of :func:`repr` has been improved to include the -:term:`module` name. Patched by Furkan Onder and Anilyka Barry. +Always include the :term:`module` name in the :func:`repr` of +:func:`functools.partial` objects. Patch by Furkan Onder and Anilyka Barry. .. diff --git a/Misc/NEWS.d/3.13.0a6.rst b/Misc/NEWS.d/3.13.0a6.rst index 2740b4f0d967bab..b97973fad020af7 100644 --- a/Misc/NEWS.d/3.13.0a6.rst +++ b/Misc/NEWS.d/3.13.0a6.rst @@ -224,7 +224,7 @@ When the collecting space becomes empty, the two spaces are swapped. .. date: 2023-10-14-00-05-17 .. gh-issue: 109870 .. nonce: oKpJ3P -.. section: Core and Builtins +.. section: Library Dataclasses now calls :func:`exec` once per dataclass, instead of once per method being added. This can speed up dataclass creation by up to 20%. @@ -234,7 +234,7 @@ method being added. This can speed up dataclass creation by up to 20%. .. date: 2022-10-05-09-33-48 .. gh-issue: 97901 .. nonce: BOLluU -.. section: Core and Builtins +.. section: Library Mime type ``text/rtf`` is now supported by :mod:`mimetypes`. @@ -264,7 +264,8 @@ Improve performance of :func:`os.path.join` and :func:`os.path.expanduser`. .. nonce: hqk9Hn .. section: Library -Raise :exc:`TypeError` for non-paths in :func:`posixpath.relpath`. +Raise :exc:`TypeError` for non-paths in :func:`posixpath.relpath +<os.path.relpath>`. .. @@ -273,7 +274,8 @@ Raise :exc:`TypeError` for non-paths in :func:`posixpath.relpath`. .. nonce: l6rWlj .. section: Library -Preserve mailbox ownership when rewriting in :func:`mailbox.mbox.flush`. +Preserve mailbox ownership when rewriting in :func:`mailbox.mbox.flush +<mailbox.Mailbox.flush>`. Patch by Tony Mountifield. .. diff --git a/Misc/NEWS.d/3.13.0b1.rst b/Misc/NEWS.d/3.13.0b1.rst index 97731276679ba61..d0eef2d8c11641a 100644 --- a/Misc/NEWS.d/3.13.0b1.rst +++ b/Misc/NEWS.d/3.13.0b1.rst @@ -60,7 +60,7 @@ for speed. .. date: 2024-05-03-18-01-26 .. gh-issue: 95382 .. nonce: 73FSEv -.. section: Core and Builtins +.. section: Library Improve performance of :func:`json.dumps` and :func:`json.dump` when using the argument *indent*. Depending on the data the encoding using diff --git a/Misc/NEWS.d/3.14.0.rst b/Misc/NEWS.d/3.14.0.rst new file mode 100644 index 000000000000000..438905c8fe9a825 --- /dev/null +++ b/Misc/NEWS.d/3.14.0.rst @@ -0,0 +1,78 @@ +.. date: 2025-10-06-23-56-36 +.. gh-issue: 124111 +.. nonce: KOlBvs +.. release date: 2025-10-07 +.. section: macOS + +Update macOS installer to use Tcl/Tk 8.6.17. + +.. + +.. date: 2025-10-04-12-29-31 +.. gh-issue: 139573 +.. nonce: vVpHaP +.. section: macOS + +Updated bundled version of OpenSSL to 3.0.18. + +.. + +.. date: 2025-10-04-12-18-45 +.. gh-issue: 139573 +.. nonce: EO9kVB +.. section: Windows + +Updated bundled version of OpenSSL to 3.0.18. + +.. + +.. date: 2025-09-25-10-31-02 +.. gh-issue: 139330 +.. nonce: 5WWkY0 +.. section: Tools/Demos + +SBOM generation tool didn't cross-check the version and checksum values +against the ``Modules/expat/refresh.sh`` script, leading to the values +becoming out-of-date during routine updates. + +.. + +.. date: 2025-08-28-06-22-26 +.. gh-issue: 132006 +.. nonce: eZQmc6 +.. section: Tools/Demos + +XCframeworks now include privacy manifests to satisfy Apple App Store +submission requirements. + +.. + +.. date: 2025-08-27-11-14-53 +.. gh-issue: 138171 +.. nonce: Suz8ob +.. section: Tools/Demos + +A script for building an iOS XCframework was added. As part of this change, +the top level ``iOS`` folder has been moved to be a subdirectory of the +``Apple`` folder. + +.. + +.. date: 2025-09-29-00-01-28 +.. gh-issue: 139400 +.. nonce: X2T-jO +.. section: Security + +:mod:`xml.parsers.expat`: Make sure that parent Expat parsers are only +garbage-collected once they are no longer referenced by subparsers created +by :meth:`~xml.parsers.expat.xmlparser.ExternalEntityParserCreate`. Patch by +Sebastian Pipping. + +.. + +.. date: 2025-09-25-07-33-43 +.. gh-issue: 139312 +.. nonce: ygE8AC +.. section: Library + +Upgrade bundled libexpat to 2.7.3 diff --git a/Misc/NEWS.d/3.14.0a1.rst b/Misc/NEWS.d/3.14.0a1.rst index 98639f0d3505d58..825cfe535c8edfb 100644 --- a/Misc/NEWS.d/3.14.0a1.rst +++ b/Misc/NEWS.d/3.14.0a1.rst @@ -1033,7 +1033,7 @@ retrieve the spec information. .. nonce: s3vKql .. section: Library -:mod:`argparse` vim supports abbreviated single-dash long options separated +:mod:`argparse` supports abbreviated single-dash long options separated by ``=`` from its value. .. @@ -1999,7 +1999,7 @@ with an escape character. .. nonce: vi2bP- .. section: Library -:func:`@warnings.deprecated <warnings.deprecated>` now copies the coroutine +:deco:`warnings.deprecated` now copies the coroutine status of functions and methods so that :func:`inspect.iscoroutinefunction` returns the correct result. @@ -4016,7 +4016,7 @@ Make ``this_instr`` and ``prev_instr`` const in cases generator. .. date: 2024-10-05-23-53-06 .. gh-issue: 125008 .. nonce: ETANpd -.. section: Core and Builtins +.. section: Library Fix :func:`tokenize.untokenize` producing invalid syntax for double braces preceded by certain escape characters. @@ -4275,7 +4275,7 @@ devdanzin .. date: 2024-09-02-20-39-10 .. gh-issue: 123614 .. nonce: 26TMHp -.. section: Core and Builtins +.. section: Library Add :func:`turtle.save` to easily save Turtle drawings as PostScript files. Patch by Marie Roald and Yngve Mardal Moe. @@ -4761,7 +4761,7 @@ enabled (yet). .. date: 2024-07-18-21-19-04 .. gh-issue: 121999 .. nonce: 8IBbTK -.. section: Core and Builtins +.. section: Library The default extraction filter for the :mod:`tarfile` module is now set to :func:`'data' <tarfile.data_filter>`. @@ -4913,11 +4913,11 @@ Allow tuples of length 20 in the freelist to be reused. .. nonce: lYKYYP .. section: Core and Builtins -:exc:`ValueError` messages for :meth:`!list.index`, :meth:`!range.index`, +:exc:`ValueError` messages for :meth:`list.index`, :meth:`range.index`, :meth:`!deque.index`, :meth:`!deque.remove` and :meth:`!ShareableList.index` no longer contain the repr of the searched value (which can be arbitrary -large) and are consistent with error messages for other :meth:`!index` and -:meth:`!remove` methods. +large) and are consistent with error messages for other :meth:`~sequence.index` and +:meth:`~sequence.remove` methods. .. @@ -4955,7 +4955,7 @@ Galindo .. date: 2024-06-28-23-17-22 .. gh-issue: 121381 .. nonce: i2xL7P -.. section: Core and Builtins +.. section: Library Remove ``subprocess._USE_VFORK`` escape hatch code and documentation. It was added just in case, and doesn't have any known cases that require it. @@ -5115,7 +5115,7 @@ and identities of :class:`str` objects. .. date: 2024-06-14-07-52-00 .. gh-issue: 120485 .. nonce: yy4K4b -.. section: Core and Builtins +.. section: Library Add an override of ``allow_reuse_port`` on classes subclassing ``socketserver.TCPServer`` where ``allow_reuse_address`` is also overridden. @@ -5147,7 +5147,7 @@ after exception handlers are moved to the end of the code. .. date: 2024-06-12-18-23-15 .. gh-issue: 120380 .. nonce: edtqjq -.. section: Core and Builtins +.. section: Library Fix Python implementation of :class:`pickle.Pickler` for :class:`bytes` and :class:`bytearray` objects when using protocol version 5. Patch by Bénédikt @@ -5604,7 +5604,7 @@ Using :data:`NotImplemented` in a boolean context now raises .. nonce: wNMKVd .. section: Core and Builtins -Fix race condition in free-threaded build where :meth:`!list.extend` could +Fix race condition in free-threaded build where :meth:`list.extend` could expose uninitialised memory to concurrent readers. .. @@ -5625,7 +5625,7 @@ in the future. .. date: 2024-04-27-18-36-46 .. gh-issue: 115801 .. nonce: SVeHSy -.. section: Core and Builtins +.. section: Library Raise ``TypeError`` when passing a string to :func:`difflib.unified_diff` and :func:`difflib.context_diff`. @@ -6092,7 +6092,7 @@ Patch by Victor Stinner. .. nonce: qOr9GF .. section: C API -Soft deprecate the :c:macro:`!Py_MEMCPY` macro: use directly ``memcpy()`` +Soft deprecate the :c:macro:`Py_MEMCPY` macro: use directly ``memcpy()`` instead. Patch by Victor Stinner. .. diff --git a/Misc/NEWS.d/3.14.0a2.rst b/Misc/NEWS.d/3.14.0a2.rst index 7405a1344a9fa61..85009034ad316c3 100644 --- a/Misc/NEWS.d/3.14.0a2.rst +++ b/Misc/NEWS.d/3.14.0a2.rst @@ -1253,7 +1253,7 @@ including SerenityOS. .. date: 2024-11-09-16-10-22 .. gh-issue: 126066 .. nonce: 9zs4m4 -.. section: Core and Builtins +.. section: Library Fix :mod:`importlib` to not write an incomplete .pyc files when a ulimit or some other operating system mechanism is preventing the write to go through @@ -1285,7 +1285,7 @@ its ``__iter__``. .. date: 2024-11-02-18-01-31 .. gh-issue: 126209 .. nonce: 2ZIhrS -.. section: Core and Builtins +.. section: Library Fix an issue with ``skip_file_prefixes`` parameter which resulted in an inconsistent behaviour between the C and Python implementations of @@ -1567,7 +1567,7 @@ Wannes Boeykens. .. date: 2024-05-12-03-10-36 .. gh-issue: 118950 .. nonce: 5Wc4vp -.. section: Core and Builtins +.. section: Library Fix bug where SSLProtocol.connection_lost wasn't getting called when OSError was thrown on writing to socket. @@ -1577,7 +1577,7 @@ was thrown on writing to socket. .. date: 2023-12-30-00-21-45 .. gh-issue: 113570 .. nonce: _XQgsW -.. section: Core and Builtins +.. section: Library Fixed a bug in ``reprlib.repr`` where it incorrectly called the repr method on shadowed Python built-in types. diff --git a/Misc/NEWS.d/3.14.0a3.rst b/Misc/NEWS.d/3.14.0a3.rst index 8393be8909ff8fb..b4264335a2d8c36 100644 --- a/Misc/NEWS.d/3.14.0a3.rst +++ b/Misc/NEWS.d/3.14.0a3.rst @@ -820,7 +820,7 @@ Fix possible undefined behavior division by zero in :class:`complex`'s .. date: 2024-11-23-04-54-42 .. gh-issue: 127133 .. nonce: WMoJjF -.. section: Core and Builtins +.. section: Library Calling :meth:`argparse.ArgumentParser.add_argument_group` on an argument group, and calling :meth:`argparse.ArgumentParser.add_argument_group` or diff --git a/Misc/NEWS.d/3.14.0a4.rst b/Misc/NEWS.d/3.14.0a4.rst index 176ba72da65e4bf..943505560934336 100644 --- a/Misc/NEWS.d/3.14.0a4.rst +++ b/Misc/NEWS.d/3.14.0a4.rst @@ -548,7 +548,7 @@ atomic operation. Patch by Donghee Na. .. date: 2024-12-23-11-14-07 .. gh-issue: 128192 .. nonce: 02mEhD -.. section: Core and Builtins +.. section: Library Upgrade HTTP digest authentication algorithm for :mod:`urllib.request` by supporting SHA-256 digest authentication as specified in :rfc:`7616`. @@ -613,7 +613,7 @@ object when importing a non-existent symbol from a non-module object. .. date: 2024-12-17-18-20-37 .. gh-issue: 128035 .. nonce: JwqHdB -.. section: Core and Builtins +.. section: Library Indicate through :data:`ssl.HAS_PHA` whether the :mod:`ssl` module supports TLSv1.3 post-handshake client authentication (PHA). Patch by Will diff --git a/Misc/NEWS.d/3.14.0a5.rst b/Misc/NEWS.d/3.14.0a5.rst index a3548d0a7b0357f..6242fce823c569a 100644 --- a/Misc/NEWS.d/3.14.0a5.rst +++ b/Misc/NEWS.d/3.14.0a5.rst @@ -944,7 +944,7 @@ It is always ``'freebsd'``, instead of ``'freebsd13'`` or ``'freebsd14'``. .. date: 2025-01-28-06-23-59 .. gh-issue: 129345 .. nonce: uOjkML -.. section: Core and Builtins +.. section: Library Fix null pointer dereference in :func:`syslog.openlog` when an audit hook raises an exception. @@ -1197,7 +1197,7 @@ generator. Patch by Mikhail Efimov. .. date: 2024-11-03-06-05-16 .. gh-issue: 126349 .. nonce: 7YwWsI -.. section: Core and Builtins +.. section: Library Add :func:`turtle.fill`, :func:`turtle.poly` and :func:`turtle.no_animation` context managers. Patch by Marie Roald and Yngve Mardal Moe. @@ -1218,7 +1218,7 @@ Willmer. .. date: 2023-12-04-15-53-25 .. gh-issue: 112713 .. nonce: Zrhv77 -.. section: Core and Builtins +.. section: Library Added support for the ``Partitioned`` cookie flag in :mod:`http.cookies`. diff --git a/Misc/NEWS.d/3.14.0a6.rst b/Misc/NEWS.d/3.14.0a6.rst index bafd8845de69735..6ffd0c9d31a45de 100644 --- a/Misc/NEWS.d/3.14.0a6.rst +++ b/Misc/NEWS.d/3.14.0a6.rst @@ -1187,7 +1187,7 @@ Improve the experimental JIT's handling of returns to unknown callers. .. date: 2025-02-11-20-38-37 .. gh-issue: 129983 .. nonce: _1Fujo -.. section: Core and Builtins +.. section: Library Fix data race in compile_template in :file:`sre.c`. @@ -1325,7 +1325,7 @@ variable. .. nonce: d75n8U .. section: Core and Builtins -Adapt :func:`reversed` for use in the free-theading build. The +Adapt :func:`reversed` for use in the free-threading build. The :func:`reversed` is still not thread-safe in the sense that concurrent iterations may see the same object, but they will not corrupt the interpreter state. @@ -1335,7 +1335,7 @@ interpreter state. .. date: 2022-12-21-14-28-01 .. gh-issue: 100388 .. nonce: vne8ky -.. section: Core and Builtins +.. section: Library Fix the ``platform._sys_version()`` method when ``__DATE__`` is undefined at buildtime by changing default buildtime datetime string to the UNIX epoch. diff --git a/Misc/NEWS.d/3.14.0a7.rst b/Misc/NEWS.d/3.14.0a7.rst index 35b96d33da4175d..752d8be8dc8f44d 100644 --- a/Misc/NEWS.d/3.14.0a7.rst +++ b/Misc/NEWS.d/3.14.0a7.rst @@ -192,7 +192,7 @@ The :class:`ctypes.py_object` type now supports subscription, making it a .. nonce: cX4yTn .. section: Library -Add the :attr:`zipfile.ZipFile.data_offset` attribute, which stores the +Add the :attr:`!zipfile.ZipFile.data_offset` attribute, which stores the offset to the beginning of ZIP data in a file when available. When the :class:`zipfile.ZipFile` is opened in either mode ``'w'`` or ``'x'`` and the underlying file does not support ``tell()``, the value will be ``None`` @@ -671,7 +671,7 @@ Allow the JIT to remove an extra ``_TO_BOOL_BOOL`` instruction after .. nonce: dNh64H .. section: Core and Builtins -Fix crash when calling :meth:`!list.append` as an unbound method. +Fix crash when calling :meth:`list.append` as an unbound method. .. @@ -880,7 +880,7 @@ Fix an issue with thread identifiers being sign-extended on some platforms. .. date: 2025-02-15-14-36-32 .. gh-issue: 99108 .. nonce: u6CfmK -.. section: Core and Builtins +.. section: Library Add support for built-in implementation of HMAC (:rfc:`2104`) based on HACL*. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/3.14.0b1.rst b/Misc/NEWS.d/3.14.0b1.rst index 5847dea7d5e8ee6..66457bb04725c60 100644 --- a/Misc/NEWS.d/3.14.0b1.rst +++ b/Misc/NEWS.d/3.14.0b1.rst @@ -1325,7 +1325,7 @@ Add new utilities of observing JIT compilation: .. date: 2025-04-30-13-09-20 .. gh-issue: 133194 .. nonce: 25_G5c -.. section: Core and Builtins +.. section: Library :func:`ast.parse` will no longer parse new :pep:`758` syntax with older *feature_version* passed. @@ -1498,7 +1498,7 @@ helpful fix suggestion for the typo. Contributed by Pablo Galindo Salgado. .. date: 2025-04-19-18-07-34 .. gh-issue: 132737 .. nonce: 9mW1il -.. section: Core and Builtins +.. section: Library Support profiling code that requires ``__main__``, such as :mod:`pickle`. @@ -1756,7 +1756,7 @@ Add support for macOS multi-arch builds with the JIT enabled .. nonce: q9fvyM .. section: Core and Builtins -PyREPL now supports syntax highlighing. Contributed by Łukasz Langa. +PyREPL now supports syntax highlighting. Contributed by Łukasz Langa. .. @@ -1797,7 +1797,7 @@ non-``None`` ``closure``. Patch by Bartosz Sławecki. .. nonce: Uj7lyY .. section: Core and Builtins -Fix a bug that was allowing newlines inconsitently in format specifiers for +Fix a bug that was allowing newlines inconsistently in format specifiers for single-quoted f-strings. Patch by Pablo Galindo. .. @@ -1891,7 +1891,7 @@ Steven Sun) .. date: 2022-12-29-19-10-36 .. gh-issue: 89562 .. nonce: g8m8RC -.. section: Core and Builtins +.. section: Library Remove ``hostflags`` member from ``PySSLContext`` struct. @@ -2012,7 +2012,7 @@ interpreter. .. nonce: ofI5Fl .. section: C API -Add support of nullable arguments in :c:func:`PyArg_Parse` and similar +[Reverted in :gh:`136991`] Add support of nullable arguments in :c:func:`PyArg_Parse` and similar functions. Adding ``?`` after any format unit makes ``None`` be accepted as a value. diff --git a/Misc/NEWS.d/3.14.0b2.rst b/Misc/NEWS.d/3.14.0b2.rst new file mode 100644 index 000000000000000..95570ee4a638731 --- /dev/null +++ b/Misc/NEWS.d/3.14.0b2.rst @@ -0,0 +1,766 @@ +.. date: 2025-05-20-21-43-20 +.. gh-issue: 130727 +.. nonce: -69t4D +.. release date: 2025-05-26 +.. section: Windows + +Fix a race in internal calls into WMI that can result in an "invalid handle" +exception under high load. Patch by Chris Eibl. + +.. + +.. date: 2025-05-19-03-02-04 +.. gh-issue: 76023 +.. nonce: vHOf6M +.. section: Windows + +Make :func:`os.path.realpath` ignore Windows error 1005 when in non-strict +mode. + +.. + +.. date: 2025-05-13-13-25-27 +.. gh-issue: 133779 +.. nonce: -YcTBz +.. section: Windows + +Reverts the change to generate different :file:`pyconfig.h` files based on +compiler settings, as it was frequently causing extension builds to break. +In particular, the ``Py_GIL_DISABLED`` preprocessor variable must now always +be defined explicitly when compiling for the experimental free-threaded +runtime. The :func:`sysconfig.get_config_var` function can be used to +determine whether the current runtime was compiled with that flag or not. + +.. + +.. date: 2025-05-08-19-07-26 +.. gh-issue: 133626 +.. nonce: yFTKYK +.. section: Windows + +Ensures packages are not accidentally bundled into the traditional +installer. + +.. + +.. date: 2025-05-19-14-57-46 +.. gh-issue: 134215 +.. nonce: sbdDK6 +.. section: Tools/Demos + +:term:`REPL` import autocomplete only suggests private modules when +explicitly specified. + +.. + +.. date: 2025-05-09-14-54-48 +.. gh-issue: 133744 +.. nonce: LCquu0 +.. section: Tests + +Fix multiprocessing interrupt test. Add an event to synchronize the parent +process with the child process: wait until the child process starts +sleeping. Patch by Victor Stinner. + +.. + +.. date: 2025-05-09-04-11-06 +.. gh-issue: 133682 +.. nonce: -_lwo3 +.. section: Tests + +Fixed test case ``test.test_annotationlib.TestStringFormat.test_displays`` +which ensures proper handling of complex data structures (lists, sets, +dictionaries, and tuples) in string annotations. + +.. + +.. date: 2025-05-08-15-06-01 +.. gh-issue: 133639 +.. nonce: 50-kbV +.. section: Tests + +Fix ``TestPyReplAutoindent.test_auto_indent_default()`` doesn't run +``input_code``. + +.. + +.. date: 2025-05-09-20-22-54 +.. gh-issue: 133767 +.. nonce: kN2i3Q +.. section: Security + +Fix use-after-free in the "unicode-escape" decoder with a non-"strict" error +handler. + +.. + +.. date: 2025-01-14-11-19-07 +.. gh-issue: 128840 +.. nonce: M1doZW +.. section: Security + +Short-circuit the processing of long IPv6 addresses early in +:mod:`ipaddress` to prevent excessive memory consumption and a minor +denial-of-service. + +.. + +.. date: 2025-05-26-12-31-08 +.. gh-issue: 132710 +.. nonce: ApU3TZ +.. section: Library + +If possible, ensure that :func:`uuid.getnode` returns the same result even +across different processes. Previously, the result was constant only within +the same process. Patch by Bénédikt Tran. + +.. + +.. date: 2025-05-24-03-10-36 +.. gh-issue: 80334 +.. nonce: z21cMa +.. section: Library + +:func:`multiprocessing.freeze_support` now checks for work on any "spawn" +start method platform rather than only on Windows. + +.. + +.. date: 2025-05-23-23-43-39 +.. gh-issue: 134582 +.. nonce: 9POq3l +.. section: Library + +Fix tokenize.untokenize() round-trip errors related to t-strings braces +escaping + +.. + +.. date: 2025-05-22-18-14-13 +.. gh-issue: 134546 +.. nonce: fjLVzK +.. section: Library + +Ensure :mod:`pdb` remote debugging script is readable by remote Python +process. + +.. + +.. date: 2025-05-22-14-12-53 +.. gh-issue: 134451 +.. nonce: M1rD-j +.. section: Library + +Converted ``asyncio.tools.CycleFoundException`` from dataclass to a regular +exception type. + +.. + +.. date: 2025-05-22-13-10-32 +.. gh-issue: 114177 +.. nonce: 3TYUJ3 +.. section: Library + +Fix :mod:`asyncio` to not close subprocess pipes which would otherwise error +out when the event loop is already closed. + +.. + +.. date: 2025-05-20-21-45-58 +.. gh-issue: 90871 +.. nonce: Gkvtp6 +.. section: Library + +Fixed an off by one error concerning the backlog parameter in +:meth:`~asyncio.loop.create_unix_server`. Contributed by Christian Harries. + +.. + +.. date: 2025-05-20-19-16-30 +.. gh-issue: 134323 +.. nonce: ZQZGvw +.. section: Library + +Fix the :meth:`threading.RLock.locked` method. + +.. + +.. date: 2025-05-20-15-13-43 +.. gh-issue: 86802 +.. nonce: trF7TM +.. section: Library + +Fixed asyncio memory leak in cancelled shield tasks. For shielded tasks +where the shield was cancelled, log potential exceptions through the +exception handler. Contributed by Christian Harries. + +.. + +.. date: 2025-05-19-20-59-06 +.. gh-issue: 134209 +.. nonce: anhTcF +.. section: Library + +:mod:`curses`: The :meth:`curses.window.instr` and +:meth:`curses.window.getstr` methods now allocate their internal buffer on +the heap instead of the stack; in addition, the max buffer size is increased +from 1023 to 2047. + +.. + +.. date: 2025-05-19-15-05-24 +.. gh-issue: 134235 +.. nonce: pz9PwV +.. section: Library + +Updated tab completion on REPL to include builtin modules. Contributed by +Tom Wang, Hunter Young + +.. + +.. date: 2025-05-19-10-32-11 +.. gh-issue: 134152 +.. nonce: INJC2j +.. section: Library + +Fixed :exc:`UnboundLocalError` that could occur during :mod:`email` header +parsing if an expected trailing delimiter is missing in some contexts. + +.. + +.. date: 2025-05-18-13-23-29 +.. gh-issue: 134168 +.. nonce: hgx3Xg +.. section: Library + +:mod:`http.server`: Fix IPv6 address binding and :option:`--directory +<http.server --directory>` handling when using HTTPS. + +.. + +.. date: 2025-05-18-12-48-39 +.. gh-issue: 62184 +.. nonce: y11l10 +.. section: Library + +Remove import of C implementation of :class:`io.FileIO` from Python +implementation which has its own implementation + +.. + +.. date: 2025-05-17-20-23-57 +.. gh-issue: 133982 +.. nonce: smS7au +.. section: Library + +Emit :exc:`RuntimeWarning` in the Python implementation of :mod:`io` when +the :term:`file-like object <file object>` is not closed explicitly in the +presence of multiple I/O layers. + +.. + +.. date: 2025-05-17-18-08-35 +.. gh-issue: 133890 +.. nonce: onn9_X +.. section: Library + +The :mod:`tarfile` module now handles :exc:`UnicodeEncodeError` in the same +way as :exc:`OSError` when cannot extract a member. + +.. + +.. date: 2025-05-17-13-46-20 +.. gh-issue: 134097 +.. nonce: fgkjE1 +.. section: Library + +Fix interaction of the new :term:`REPL` and :option:`-X showrefcount <-X>` +command line option. + +.. + +.. date: 2025-05-17-12-40-12 +.. gh-issue: 133889 +.. nonce: Eh-zO4 +.. section: Library + +The generated directory listing page in +:class:`http.server.SimpleHTTPRequestHandler` now only shows the decoded +path component of the requested URL, and not the query and fragment. + +.. + +.. date: 2025-05-16-20-10-25 +.. gh-issue: 134098 +.. nonce: YyTkKr +.. section: Library + +Fix handling paths that end with a percent-encoded slash (``%2f`` or +``%2F``) in :class:`http.server.SimpleHTTPRequestHandler`. + +.. + +.. date: 2025-05-16-12-40-37 +.. gh-issue: 132124 +.. nonce: T_5Odx +.. section: Library + +On POSIX-compliant systems, :func:`!multiprocessing.util.get_temp_dir` now +ignores :envvar:`TMPDIR` (and similar environment variables) if the path +length of ``AF_UNIX`` socket files exceeds the platform-specific maximum +length when using the :ref:`forkserver +<multiprocessing-start-method-forkserver>` start method. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-05-15-14-27-01 +.. gh-issue: 134062 +.. nonce: fRbJet +.. section: Library + +:mod:`ipaddress`: fix collisions in :meth:`~object.__hash__` for +:class:`~ipaddress.IPv4Network` and :class:`~ipaddress.IPv6Network` objects. + +.. + +.. date: 2025-05-13-18-54-56 +.. gh-issue: 133970 +.. nonce: 6G-Oi6 +.. section: Library + +Make :class:`!string.templatelib.Template` and +:class:`!string.templatelib.Interpolation` generic. + +.. + +.. date: 2025-05-13-18-21-59 +.. gh-issue: 71253 +.. nonce: -3Sf_K +.. section: Library + +Raise :exc:`ValueError` in :func:`open` if *opener* returns a negative +file-descriptor in the Python implementation of :mod:`io` to match the C +implementation. + +.. + +.. date: 2025-05-12-20-38-57 +.. gh-issue: 133960 +.. nonce: Aee79f +.. section: Library + +Simplify and improve :func:`typing.evaluate_forward_ref`. It now no longer +raises errors on certain invalid types. In several situations, it is now +able to evaluate forward references that were previously unsupported. + +.. + +.. date: 2025-05-12-06-52-10 +.. gh-issue: 133925 +.. nonce: elInBY +.. section: Library + +Make the private class ``typing._UnionGenericAlias`` hashable. + +.. + +.. date: 2025-05-10-12-06-55 +.. gh-issue: 133653 +.. nonce: Gb2aG4 +.. section: Library + +Fix :class:`argparse.ArgumentParser` with the *formatter_class* argument. +Fix TypeError when *formatter_class* is a custom subclass of +:class:`!HelpFormatter`. Fix TypeError when *formatter_class* is not a +subclass of :class:`!HelpFormatter` and non-standard *prefix_char* is used. +Fix support of colorizing when *formatter_class* is not a subclass of +:class:`!HelpFormatter`. + +.. + +.. date: 2025-05-09-20-59-24 +.. gh-issue: 132641 +.. nonce: 3qTw44 +.. section: Library + +Fixed a race in :func:`functools.lru_cache` under free-threading. + +.. + +.. date: 2025-05-09-19-05-24 +.. gh-issue: 133783 +.. nonce: 1voCnR +.. section: Library + +Fix bug with applying :func:`copy.replace` to :mod:`ast` objects. Attributes +that default to ``None`` were incorrectly treated as required for manually +created AST nodes. + +.. + +.. date: 2025-05-09-18-29-25 +.. gh-issue: 133684 +.. nonce: Y1DFSt +.. section: Library + +Fix bug where :func:`annotationlib.get_annotations` would return the wrong +result for certain classes that are part of a class hierarchy where ``from +__future__ import annotations`` is used. + +.. + +.. date: 2025-05-09-15-50-00 +.. gh-issue: 77057 +.. nonce: fV8SU- +.. section: Library + +Fix handling of invalid markup declarations in +:class:`html.parser.HTMLParser`. + +.. + +.. date: 2025-05-09-09-10-34 +.. gh-issue: 130328 +.. nonce: s9h4By +.. section: Library + +Speedup pasting in ``PyREPL`` on Windows in a legacy console. Patch by Chris +Eibl. + +.. + +.. date: 2025-05-09-08-49-03 +.. gh-issue: 133701 +.. nonce: KI8tGz +.. section: Library + +Fix bug where :class:`typing.TypedDict` classes defined under ``from +__future__ import annotations`` and inheriting from another ``TypedDict`` +had an incorrect ``__annotations__`` attribute. + +.. + +.. date: 2025-05-07-19-16-41 +.. gh-issue: 133581 +.. nonce: kERUCJ +.. section: Library + +Improve unparsing of t-strings in :func:`ast.unparse` and ``from __future__ +import annotations``. Empty t-strings now round-trip correctly and +formatting in interpolations is preserved. Patch by Jelle Zijlstra. + +.. + +.. date: 2025-05-06-22-54-37 +.. gh-issue: 133551 +.. nonce: rfy1tJ +.. section: Library + +Support t-strings (:pep:`750`) in :mod:`annotationlib`. Patch by Jelle +Zijlstra. + +.. + +.. date: 2025-05-05-22-11-24 +.. gh-issue: 133439 +.. nonce: LpmyFz +.. section: Library + +Fix dot commands with trailing spaces are mistaken for multi-line SQL +statements in the sqlite3 command-line interface. + +.. + +.. date: 2025-05-04-17-04-55 +.. gh-issue: 132493 +.. nonce: huirKi +.. section: Library + +Avoid accessing ``__annotations__`` unnecessarily in +:func:`inspect.signature`. + +.. + +.. date: 2025-04-29-11-48-46 +.. gh-issue: 132876 +.. nonce: lyTQGZ +.. section: Library + +``ldexp()`` on Windows doesn't round subnormal results before Windows 11, +but should. Python's :func:`math.ldexp` wrapper now does round them, so +results may change slightly, in rare cases of very small results, on Windows +versions before 11. + +.. + +.. date: 2025-04-26-15-50-12 +.. gh-issue: 133009 +.. nonce: etBuz5 +.. section: Library + +:mod:`xml.etree.ElementTree`: Fix a crash in :meth:`Element.__deepcopy__ +<object.__deepcopy__>` when the element is concurrently mutated. Patch by +Bénédikt Tran. + +.. + +.. date: 2025-03-30-16-42-38 +.. gh-issue: 91555 +.. nonce: ShVtwW +.. section: Library + +Ignore log messages generated during handling of log messages, to avoid +deadlock or infinite recursion. [NOTE: This change has since been reverted.] + +.. + +.. date: 2024-10-28-06-54-22 +.. gh-issue: 125028 +.. nonce: GEY8Ws +.. section: Library + +:data:`functools.Placeholder` cannot be passed to :func:`functools.partial` +as a keyword argument. + +.. + +.. date: 2023-02-13-21-56-38 +.. gh-issue: 62824 +.. nonce: CBZzX3 +.. section: Library + +Fix aliases for ``iso8859_8`` encoding. Patch by Dave Goncalves. + +.. + +.. date: 2023-02-13-21-41-34 +.. gh-issue: 86155 +.. nonce: ppIGSC +.. section: Library + +:meth:`html.parser.HTMLParser.close` no longer loses data when the +``<script>`` tag is not closed. Patch by Waylan Limberg. + +.. + +.. date: 2022-07-24-20-56-32 +.. gh-issue: 69426 +.. nonce: unccw7 +.. section: Library + +Fix :class:`html.parser.HTMLParser` to not unescape character entities in +attribute values if they are followed by an ASCII alphanumeric or an equals +sign. + +.. + +.. bpo: 28494 +.. date: 2017-12-30-18-21-00 +.. nonce: Dt_Wks +.. section: Library + +Improve Zip file validation false positive rate in +:func:`zipfile.is_zipfile`. + +.. + +.. date: 2025-05-22-14-48-19 +.. gh-issue: 134381 +.. nonce: 2BXhth +.. section: Library + +Fix :exc:`RuntimeError` when using a not-started :class:`threading.Thread` +after calling :func:`os.fork` + +.. + +.. date: 2025-05-21-18-02-56 +.. gh-issue: 127960 +.. nonce: W3J_2X +.. section: Core and Builtins + +PyREPL interactive shell no longer starts with ``__package__`` and +``__file__`` global names set to ``_pyrepl`` package internals. Contributed +by Yuichiro Tachibana. + +.. + +.. date: 2025-05-21-15-14-32 +.. gh-issue: 130397 +.. nonce: aG6EON +.. section: Core and Builtins + +Remove special-casing for C stack depth limits for WASI. Due to +WebAssembly's built-in stack protection this does not pose a security +concern. + +.. + +.. date: 2025-05-20-14-41-50 +.. gh-issue: 128066 +.. nonce: qzzGfv +.. section: Core and Builtins + +Fixes an edge case where PyREPL improperly threw an error when Python is +invoked on a read only filesystem while trying to write history file +entries. + +.. + +.. date: 2025-05-18-14-33-23 +.. gh-issue: 69605 +.. nonce: ZMO49F +.. section: Core and Builtins + +When auto-completing an import in the :term:`REPL`, finding no candidates +now issues no suggestion, rather than suggestions from the current +namespace. + +.. + +.. date: 2025-05-17-20-44-51 +.. gh-issue: 134158 +.. nonce: ewLNLp +.. section: Core and Builtins + +Fix coloring of double braces in f-strings and t-strings in the +:term:`REPL`. + +.. + +.. date: 2025-05-16-20-59-12 +.. gh-issue: 134119 +.. nonce: w8expI +.. section: Core and Builtins + +Fix crash when calling :func:`next` on an exhausted template string +iterator. Patch by Jelle Zijlstra. + +.. + +.. date: 2025-05-16-17-25-52 +.. gh-issue: 134100 +.. nonce: 5-FbLK +.. section: Core and Builtins + +Fix a use-after-free bug that occurs when an imported module isn't in +:data:`sys.modules` after its initial import. Patch by Nico-Posada. + +.. + +.. date: 2025-05-15-11-38-16 +.. gh-issue: 133999 +.. nonce: uBZ8uS +.. section: Core and Builtins + +Fix :exc:`SyntaxError` regression in :keyword:`except` parsing after +:gh:`123440`. + +.. + +.. date: 2025-05-11-13-40-42 +.. gh-issue: 133886 +.. nonce: ryBAyo +.. section: Core and Builtins + +Fix :func:`sys.remote_exec` for non-ASCII paths in non-UTF-8 locales and +non-UTF-8 paths in UTF-8 locales. + +.. + +.. date: 2025-05-10-17-12-27 +.. gh-issue: 133703 +.. nonce: bVM-re +.. section: Core and Builtins + +Fix hashtable in dict can be bigger than intended in some situations. + +.. + +.. date: 2025-05-09-18-11-21 +.. gh-issue: 133778 +.. nonce: pWEV3t +.. section: Core and Builtins + +Fix bug where assigning to the :attr:`~type.__annotations__` attributes of +classes defined under ``from __future__ import annotations`` had no effect. + +.. + +.. date: 2025-05-08-13-48-02 +.. gh-issue: 132762 +.. nonce: tKbygC +.. section: Core and Builtins + +:meth:`~dict.fromkeys` no longer loops forever when adding a small set of +keys to a large base dict. Patch by Angela Liss. + +.. + +.. date: 2025-05-07-23-26-53 +.. gh-issue: 133541 +.. nonce: bHIC55 +.. section: Core and Builtins + +Inconsistent indentation in user input crashed the new REPL when syntax +highlighting was active. This is now fixed. + +.. + +.. date: 2025-05-06-15-01-41 +.. gh-issue: 133516 +.. nonce: RqWVf2 +.. section: Core and Builtins + +Raise :exc:`ValueError` when constants ``True``, ``False`` or ``None`` are +used as an identifier after NFKC normalization. + +.. + +.. date: 2025-04-19-17-16-46 +.. gh-issue: 132542 +.. nonce: 7T_TY_ +.. section: Core and Builtins + +Update :attr:`Thread.native_id <threading.Thread.native_id>` after +:manpage:`fork(2)` to ensure accuracy. Patch by Noam Cohen. + +.. + +.. date: 2025-05-17-14-41-21 +.. gh-issue: 134144 +.. nonce: xVpZik +.. section: C API + +Fix crash when calling :c:func:`Py_EndInterpreter` with a :term:`thread +state` that isn't the initial thread for the interpreter. + +.. + +.. date: 2025-05-21-19-46-28 +.. gh-issue: 134455 +.. nonce: vdwlrq +.. section: Build + +Fixed ``build-details.json`` generation to use the correct ``c_api.headers`` +as defined in :pep:`739`, instead of ``c_api.include``. + +.. + +.. date: 2025-04-30-10-22-08 +.. gh-issue: 131769 +.. nonce: H0oy5x +.. section: Build + +Fix detecting when the build Python in a cross-build is a pydebug build. + +.. + +.. date: 2025-04-16-09-38-48 +.. gh-issue: 117088 +.. nonce: EFt_5c +.. section: Build + +AIX linker don't support -h option, so avoid it through platform check diff --git a/Misc/NEWS.d/3.14.0b3.rst b/Misc/NEWS.d/3.14.0b3.rst new file mode 100644 index 000000000000000..479f72276ca1673 --- /dev/null +++ b/Misc/NEWS.d/3.14.0b3.rst @@ -0,0 +1,571 @@ +.. date: 2025-06-03-18-26-54 +.. gh-issue: 135099 +.. nonce: Q9usKm +.. release date: 2025-06-17 +.. section: Windows + +Fix a crash that could occur on Windows when a background thread waits on a +:c:type:`PyMutex` while the main thread is shutting down the interpreter. + +.. + +.. date: 2025-06-17-08-48-08 +.. gh-issue: 132815 +.. nonce: CY1Esu +.. section: Tests + +Fix test__opcode: add ``JUMP_BACKWARD`` to specialization stats. + +.. + +.. date: 2025-06-14-13-20-17 +.. gh-issue: 135489 +.. nonce: Uh0yVO +.. section: Tests + +Show verbose output for failing tests during PGO profiling step with +--enable-optimizations. + +.. + +.. date: 2025-06-04-13-07-44 +.. gh-issue: 135120 +.. nonce: NapnZT +.. section: Tests + +Add :func:`!test.support.subTests`. + +.. + +.. date: 2025-06-13-15-55-22 +.. gh-issue: 135462 +.. nonce: KBeJpc +.. section: Security + +Fix quadratic complexity in processing specially crafted input in +:class:`html.parser.HTMLParser`. End-of-file errors are now handled +according to the HTML5 specs -- comments and declarations are automatically +closed, tags are ignored. + +.. + +.. date: 2025-06-02-11-32-23 +.. gh-issue: 135034 +.. nonce: RLGjbp +.. section: Security + +Fixes multiple issues that allowed ``tarfile`` extraction filters +(``filter="data"`` and ``filter="tar"``) to be bypassed using crafted +symlinks and hard links. + +Addresses :cve:`2024-12718`, :cve:`2025-4138`, :cve:`2025-4330`, and +:cve:`2025-4517`. + +.. + +.. date: 2025-06-15-03-03-22 +.. gh-issue: 65697 +.. nonce: COdwZd +.. section: Library + +:class:`configparser`'s error message when attempting to write an invalid +key is now more helpful. + +.. + +.. date: 2025-06-14-14-19-13 +.. gh-issue: 135497 +.. nonce: 1pzwdA +.. section: Library + +Fix :func:`os.getlogin` failing for longer usernames on BSD-based platforms. + +.. + +.. date: 2025-06-12-18-15-31 +.. gh-issue: 135429 +.. nonce: mch75_ +.. section: Library + +Fix the argument mismatch in ``_lsprof`` for ``PY_THROW`` event. + +.. + +.. date: 2025-06-12-10-45-02 +.. gh-issue: 135368 +.. nonce: OjWVHL +.. section: Library + +Fix :class:`unittest.mock.Mock` generation on :func:`dataclasses.dataclass` +objects. Now all special attributes are set as it was before :gh:`124429`. + +.. + +.. date: 2025-06-10-16-11-00 +.. gh-issue: 133967 +.. nonce: P0c24q +.. section: Library + +Do not normalize :mod:`locale` name 'C.UTF-8' to 'en_US.UTF-8'. + +.. + +.. date: 2025-06-10-00-42-30 +.. gh-issue: 135321 +.. nonce: UHh9jT +.. section: Library + +Raise a correct exception for values greater than 0x7fffffff for the +``BINSTRING`` opcode in the C implementation of :mod:`pickle`. + +.. + +.. date: 2025-06-08-14-50-34 +.. gh-issue: 135276 +.. nonce: ZLUhV1 +.. section: Library + +Backported bugfixes in zipfile.Path from zipp 3.23. Fixed ``.name``, +``.stem`` and other basename-based properties on Windows when working with a +zipfile on disk. + +.. + +.. date: 2025-06-08-10-22-22 +.. gh-issue: 135244 +.. nonce: Y2SOTJ +.. section: Library + +:mod:`uuid`: when the MAC address cannot be determined, the 48-bit node ID +is now generated with a cryptographically-secure pseudo-random number +generator (CSPRNG) as per :rfc:`RFC 9562, §6.10.3 <9562#section-6.10-3>`. +This affects :func:`~uuid.uuid1` and :func:`~uuid.uuid6`. + +.. + +.. date: 2025-05-31-12-08-12 +.. gh-issue: 134970 +.. nonce: lgSaxq +.. section: Library + +Fix the "unknown action" exception in +:meth:`argparse.ArgumentParser.add_argument_group` to correctly replace the +action class. + +.. + +.. date: 2025-05-30-13-07-29 +.. gh-issue: 134718 +.. nonce: 9Qvhxn +.. section: Library + +:func:`ast.dump` now only omits ``None`` and ``[]`` values if they are +default values. + +.. + +.. date: 2025-05-30-09-46-21 +.. gh-issue: 134939 +.. nonce: Pu3nnm +.. section: Library + +Add the :mod:`concurrent.interpreters` module. See :pep:`734`. + +.. + +.. date: 2025-05-29-06-53-40 +.. gh-issue: 134885 +.. nonce: -_L22o +.. section: Library + +Fix possible crash in the :mod:`compression.zstd` module related to setting +parameter types. Patch by Jelle Zijlstra. + +.. + +.. date: 2025-05-28-20-49-29 +.. gh-issue: 134857 +.. nonce: dVYXVO +.. section: Library + +Improve error report for :mod:`doctest`\ s run with :mod:`unittest`. Remove +:mod:`!doctest` module frames from tracebacks and redundant newline +character from a failure message. + +.. + +.. date: 2025-05-28-15-53-27 +.. gh-issue: 128840 +.. nonce: Nur2pB +.. section: Library + +Fix parsing long IPv6 addresses with embedded IPv4 address. + +.. + +.. date: 2025-05-26-17-06-40 +.. gh-issue: 134637 +.. nonce: 9-3zRL +.. section: Library + +Fix performance regression in calling a :mod:`ctypes` function pointer in +:term:`free threading`. + +.. + +.. date: 2025-05-26-14-04-39 +.. gh-issue: 134696 +.. nonce: P04xUa +.. section: Library + +Built-in HACL* and OpenSSL implementations of hash function constructors now +correctly accept the same *documented* named arguments. For instance, +:func:`~hashlib.md5` could be previously invoked as ``md5(data=data)`` or +``md5(string=string)`` depending on the underlying implementation but these +calls were not compatible. Patch by Bénédikt Tran. + +.. + +.. date: 2025-05-25-23-23-05 +.. gh-issue: 134151 +.. nonce: 13Wwsb +.. section: Library + +:mod:`email`: Fix :exc:`TypeError` in :func:`email.utils.decode_params` when +sorting :rfc:`2231` continuations that contain an unnumbered section. + +.. + +.. date: 2025-05-24-13-10-35 +.. gh-issue: 134210 +.. nonce: 0IuMY2 +.. section: Library + +:func:`curses.window.getch` now correctly handles signals. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-05-18-23-46-21 +.. gh-issue: 134152 +.. nonce: 30HwbX +.. section: Library + +:mod:`email`: Fix parsing of email message ID with invalid domain. + +.. + +.. date: 2025-05-08-13-43-19 +.. gh-issue: 133489 +.. nonce: 9eGS1Z +.. section: Library + +:func:`random.getrandbits` can now generate more that 2\ :sup:`31` bits. +:func:`random.randbytes` can now generate more that 256 MiB. + +.. + +.. date: 2025-05-01-10-56-44 +.. gh-issue: 132813 +.. nonce: rKurvp +.. section: Library + +Improve error messages for incorrect types and values of +:class:`csv.Dialect` attributes. + +.. + +.. date: 2025-04-30-19-32-18 +.. gh-issue: 132969 +.. nonce: EagQ3G +.. section: Library + +Prevent the :class:`~concurrent.futures.ProcessPoolExecutor` executor +thread, which remains running when :meth:`shutdown(wait=False) +<concurrent.futures.Executor.shutdown>`, from attempting to adjust the +pool's worker processes after the object state has already been reset during +shutdown. A combination of conditions, including a worker process having +terminated abormally, resulted in an exception and a potential hang when the +still-running executor thread attempted to replace dead workers within the +pool. + +.. + +.. date: 2025-04-21-01-03-15 +.. gh-issue: 127081 +.. nonce: WXRliX +.. section: Library + +Fix libc thread safety issues with :mod:`os` by replacing ``getlogin`` with +``getlogin_r`` re-entrant version. + +.. + +.. date: 2025-04-07-06-41-54 +.. gh-issue: 131884 +.. nonce: ym9BJN +.. section: Library + +Fix formatting issues in :func:`json.dump` when both *indent* and *skipkeys* +are used. + +.. + +.. date: 2025-03-09-03-13-41 +.. gh-issue: 130999 +.. nonce: tBRBVB +.. section: Library + +Avoid exiting the new REPL and offer suggestions even if there are +non-string candidates when errors occur. + +.. + +.. date: 2025-06-10-17-02-06 +.. gh-issue: 135171 +.. nonce: quHvts +.. section: Documentation + +Document that the :term:`iterator` for the leftmost :keyword:`!for` clause +in the generator expression is created immediately. + +.. + +.. bpo: 45210 +.. date: 2021-09-15-13-07-25 +.. nonce: RtGk7i +.. section: Documentation + +Document that error indicator may be set in tp_dealloc, and how to avoid +clobbering it. + +.. + +.. date: 2025-06-14-01-01-14 +.. gh-issue: 135496 +.. nonce: ER0Me3 +.. section: Core and Builtins + +Fix typo in the f-string conversion type error ("exclamanation" -> +"exclamation"). + +.. + +.. date: 2025-06-12-18-12-42 +.. gh-issue: 135371 +.. nonce: R_YUtR +.. section: Core and Builtins + +Fixed :mod:`asyncio` debugging tools to properly display internal coroutine +call stacks alongside external task dependencies. The ``python -m asyncio +ps`` and ``python -m asyncio pstree`` commands now show complete execution +context. Patch by Pablo Galindo. + +.. + +.. date: 2025-06-11-15-08-10 +.. gh-issue: 127319 +.. nonce: OVGFSZ +.. section: Library + +Set the ``allow_reuse_port`` class variable to ``False`` on the XMLRPC, +logging, and HTTP servers. This matches the behavior in prior Python +releases, which is to not allow port reuse. + +.. + +.. date: 2025-06-10-17-37-11 +.. gh-issue: 135171 +.. nonce: P9UDfS +.. section: Core and Builtins + +Reverts the behavior of async generator expressions when created with object +w/o __aiter__ method to the pre-3.13 behavior of raising a TypeError. + +.. + +.. date: 2025-06-09-23-57-37 +.. gh-issue: 130077 +.. nonce: MHknDB +.. section: Core and Builtins + +Properly raise custom syntax errors when incorrect syntax containing names +that are prefixes of soft keywords is encountered. Patch by Pablo Galindo. + +.. + +.. date: 2025-06-06-18-57-30 +.. gh-issue: 135171 +.. nonce: 0YtLq6 +.. section: Core and Builtins + +Reverts the behavior of generator expressions when created with a +non-iterable to the pre-3.13 behavior of raising a TypeError. It is no +longer possible to cause a crash in the debugger by altering the generator +expression's local variables. This is achieved by moving the ``GET_ITER`` +instruction back to the creation of the generator expression and adding an +additional check to ``FOR_ITER``. + +.. + +.. date: 2025-06-02-13-57-40 +.. gh-issue: 116738 +.. nonce: ycJsL8 +.. section: Library + +Make methods in :mod:`heapq` thread-safe on the :term:`free threaded <free +threading>` build. + +.. + +.. date: 2025-05-31-10-26-46 +.. gh-issue: 134876 +.. nonce: 8mBGJI +.. section: Core and Builtins + +Add support to :pep:`768` remote debugging for Linux kernels which don't +have CONFIG_CROSS_MEMORY_ATTACH configured. + +.. + +.. date: 2025-05-30-18-09-54 +.. gh-issue: 134889 +.. nonce: Ic9UM- +.. section: Core and Builtins + +Fix handling of a few opcodes that leave operands on the stack when +optimizing ``LOAD_FAST``. + +.. + +.. date: 2025-05-30-15-56-19 +.. gh-issue: 134908 +.. nonce: 3a7PxM +.. section: Library + +Fix crash when iterating over lines in a text file on the :term:`free +threaded <free threading>` build. + +.. + +.. date: 2025-05-27-20-29-00 +.. gh-issue: 132617 +.. nonce: EmUfQQ +.. section: Core and Builtins + +Fix :meth:`dict.update` modification check that could incorrectly raise a +"dict mutated during update" error when a different dictionary was modified +that happens to share the same underlying keys object. + +.. + +.. date: 2025-05-27-18-59-54 +.. gh-issue: 134679 +.. nonce: FWPBu6 +.. section: Core and Builtins + +Fix crash in the :term:`free threading` build's QSBR code that could occur +when changing an object's ``__dict__`` attribute. + +.. + +.. date: 2025-05-27-09-19-21 +.. gh-issue: 127682 +.. nonce: 9WwFrM +.. section: Core and Builtins + +No longer call ``__iter__`` twice in list comprehensions. This brings the +behavior of list comprehensions in line with other forms of iteration + +.. + +.. date: 2025-05-26-15-55-50 +.. gh-issue: 133912 +.. nonce: -xAguL +.. section: Core and Builtins + +Fix the C API function ``PyObject_GenericSetDict`` to handle extension +classes with inline values. + +.. + +.. date: 2025-06-05-11-06-07 +.. gh-issue: 134989 +.. nonce: 74p4ud +.. section: C API + +Fix ``Py_RETURN_NONE``, ``Py_RETURN_TRUE`` and ``Py_RETURN_FALSE`` macros in +the limited C API 3.11 and older: don't treat ``Py_None``, ``Py_True`` and +``Py_False`` as immortal. Patch by Victor Stinner. + +.. + +.. date: 2025-06-02-13-19-22 +.. gh-issue: 134989 +.. nonce: sDDyBN +.. section: C API + +Implement :c:func:`PyObject_DelAttr` and :c:func:`PyObject_DelAttrString` as +macros in the limited C API 3.12 and older. Patch by Victor Stinner. + +.. + +.. date: 2025-05-13-16-06-46 +.. gh-issue: 133968 +.. nonce: 6alWst +.. section: C API + +Add :c:func:`PyUnicodeWriter_WriteASCII` function to write an ASCII string +into a :c:type:`PyUnicodeWriter`. The function is faster than +:c:func:`PyUnicodeWriter_WriteUTF8`, but has an undefined behavior if the +input string contains non-ASCII characters. Patch by Victor Stinner. + +.. + +.. date: 2025-06-16-07-20-28 +.. gh-issue: 119132 +.. nonce: fcI8s7 +.. section: Build + +Remove "experimental" tag from the CPython free-threading build. + +.. + +.. date: 2025-06-14-10-32-11 +.. gh-issue: 135497 +.. nonce: ajlV4F +.. section: Build + +Fix the detection of ``MAXLOGNAME`` in the ``configure.ac`` script. + +.. + +.. date: 2025-05-30-11-02-30 +.. gh-issue: 134923 +.. nonce: gBkRg4 +.. section: Build + +Windows builds with profile-guided optimization enabled now use +``/GENPROFILE`` and ``/USEPROFILE`` instead of deprecated ``/LTCG:`` +options. + +.. + +.. date: 2025-05-27-17-04-20 +.. gh-issue: 134774 +.. nonce: CusyjW +.. section: Build + +Fix :c:macro:`Py_DEBUG` macro redefinition warnings on Windows debug builds. +Patch by Chris Eibl. + +.. + +.. date: 2025-05-24-16-59-20 +.. gh-issue: 134632 +.. nonce: i0W2hc +.. section: Build + +Fixed ``build-details.json`` generation to use ``INCLUDEPY``, in order to +reference the ``pythonX.Y`` subdirectory of the include directory, as +required in :pep:`739`, instead of the top-level include directory. diff --git a/Misc/NEWS.d/3.14.0b4.rst b/Misc/NEWS.d/3.14.0b4.rst new file mode 100644 index 000000000000000..349023ec75865d1 --- /dev/null +++ b/Misc/NEWS.d/3.14.0b4.rst @@ -0,0 +1,484 @@ +.. date: 2025-06-26-15-58-13 +.. gh-issue: 135968 +.. nonce: C4v_-W +.. release date: 2025-07-08 +.. section: Tools/Demos + +Stubs for ``strip`` are now provided as part of an iOS install. + +.. + +.. date: 2025-06-25-10-36-22 +.. gh-issue: 133600 +.. nonce: bkdgHC +.. section: Tools/Demos + +Backport file reorganization for Tools/wasm/wasi. + +This should make backporting future code changes easier. It also simplifies +instructions around how to do WASI builds in the devguide. + +.. + +.. date: 2025-06-26-15-15-35 +.. gh-issue: 135966 +.. nonce: EBpF8Y +.. section: Tests + +The iOS testbed now handles the ``app_packages`` folder as a site directory. + +.. + +.. date: 2025-06-19-15-29-38 +.. gh-issue: 135494 +.. nonce: FVl9a0 +.. section: Tests + +Fix regrtest to support excluding tests from ``--pgo`` tests. Patch by +Victor Stinner. + +.. + +.. date: 2025-06-27-21-23-19 +.. gh-issue: 136053 +.. nonce: QZxcee +.. section: Security + +:mod:`marshal`: fix a possible crash when deserializing :class:`slice` +objects. + +.. + +.. date: 2025-06-25-14-13-39 +.. gh-issue: 135661 +.. nonce: idjQ0B +.. section: Security + +Fix parsing start and end tags in :class:`html.parser.HTMLParser` according +to the HTML5 standard. + +* Whitespaces no longer accepted between ``</`` and the tag name. + E.g. ``</ script>`` does not end the script section. + +* Vertical tabulation (``\v``) and non-ASCII whitespaces no longer recognized + as whitespaces. The only whitespaces are ``\t\n\r\f`` and space. + +* Null character (U+0000) no longer ends the tag name. + +* Attributes and slashes after the tag name in end tags are now ignored, + instead of terminating after the first ``>`` in quoted attribute value. + E.g. ``</script/foo=">"/>``. + +* Multiple slashes and whitespaces between the last attribute and closing ``>`` + are now ignored in both start and end tags. E.g. ``<a foo=bar/ //>``. + +* Multiple ``=`` between attribute name and value are no longer collapsed. + E.g. ``<a foo==bar>`` produces attribute "foo" with value "=bar". + +* [Reverted in :gh:`136927`] Whitespaces between the ``=`` separator and attribute name or value are no + longer ignored. E.g. ``<a foo =bar>`` produces two attributes "foo" and + "=bar", both with value None; ``<a foo= bar>`` produces two attributes: + "foo" with value "" and "bar" with value None. + +.. + +.. date: 2025-06-18-13-28-08 +.. gh-issue: 102555 +.. nonce: nADrzJ +.. section: Security + +Fix comment parsing in :class:`html.parser.HTMLParser` according to the +HTML5 standard. ``--!>`` now ends the comment. ``-- >`` no longer ends the +comment. Support abnormally ended empty comments ``<-->`` and ``<--->``. + +.. + +.. date: 2025-07-05-09-45-04 +.. gh-issue: 136286 +.. nonce: N67Amr +.. section: Library + +Fix pickling failures for protocols 0 and 1 for many objects realted to +subinterpreters. + +.. + +.. date: 2025-07-05-06-56-16 +.. gh-issue: 136316 +.. nonce: 3zj_Do +.. section: Library + +Improve support for evaluating nested forward references in +:func:`typing.evaluate_forward_ref`. + +.. + +.. date: 2025-06-30-11-12-24 +.. gh-issue: 85702 +.. nonce: 0Lrbwu +.. section: Library + +If ``zoneinfo._common.load_tzdata`` is given a package without a resource a +:exc:`zoneinfo.ZoneInfoNotFoundError` is raised rather than a +:exc:`PermissionError`. Patch by Victor Stinner. + +.. + +.. date: 2025-06-27-13-34-28 +.. gh-issue: 136028 +.. nonce: RY727g +.. section: Library + +Fix parsing month names containing "İ" (U+0130, LATIN CAPITAL LETTER I WITH +DOT ABOVE) in :func:`time.strptime`. This affects locales az_AZ, ber_DZ, +ber_MA and crh_UA. + +.. + +.. date: 2025-06-26-17-28-49 +.. gh-issue: 135995 +.. nonce: pPrDCt +.. section: Library + +In the palmos encoding, make byte ``0x9b`` decode to ``›`` (U+203A - SINGLE +RIGHT-POINTING ANGLE QUOTATION MARK). + +.. + +.. date: 2025-06-26-11-52-40 +.. gh-issue: 53203 +.. nonce: TMigBr +.. section: Library + +Fix :func:`time.strptime` for ``%c`` and ``%x`` formats on locales byn_ER, +wal_ET and lzh_TW, and for ``%X`` format on locales ar_SA, bg_BG and lzh_TW. + +.. + +.. date: 2025-06-25-17-25-53 +.. gh-issue: 91555 +.. nonce: xUpTLD +.. section: Library + +An earlier change, which was introduced in 3.14.0b2, has been reverted. It +disabled logging for a logger during handling of log messages for that +logger. Since the reversion, the behaviour should be as it was before +3.14.0b2. + +.. + +.. date: 2025-06-24-14-43-24 +.. gh-issue: 135878 +.. nonce: Db4roX +.. section: Library + +Fixes a crash of :class:`types.SimpleNamespace` on :term:`free threading` +builds, when several threads were calling its :meth:`~object.__repr__` +method at the same time. + +.. + +.. date: 2025-06-24-10-52-35 +.. gh-issue: 135836 +.. nonce: s37351 +.. section: Library + +Fix :exc:`IndexError` in :meth:`asyncio.loop.create_connection` that could +occur when non-\ :exc:`OSError` exception is raised during connection and +socket's ``close()`` raises :exc:`!OSError`. + +.. + +.. date: 2025-06-23-11-04-25 +.. gh-issue: 135836 +.. nonce: -C-c4v +.. section: Library + +Fix :exc:`IndexError` in :meth:`asyncio.loop.create_connection` that could +occur when the Happy Eyeballs algorithm resulted in an empty exceptions list +during connection attempts. + +.. + +.. date: 2025-06-23-10-19-11 +.. gh-issue: 135855 +.. nonce: -J0AGF +.. section: Library + +Raise :exc:`TypeError` instead of :exc:`SystemError` when +:func:`!_interpreters.set___main___attrs` is passed a non-dict object. Patch +by Brian Schubert. + +.. + +.. date: 2025-06-22-16-23-44 +.. gh-issue: 135815 +.. nonce: 0DandH +.. section: Library + +:mod:`netrc`: skip security checks if :func:`os.getuid` is missing. Patch by +Bénédikt Tran. + +.. + +.. date: 2025-06-22-02-16-17 +.. gh-issue: 135640 +.. nonce: FXyFL6 +.. section: Library + +Address bug where it was possible to call +:func:`xml.etree.ElementTree.ElementTree.write` on an ElementTree object +with an invalid root element. This behavior blanked the file passed to +``write`` if it already existed. + +.. + +.. date: 2025-06-18-13-58-13 +.. gh-issue: 135645 +.. nonce: 109nff +.. section: Library + +Added ``supports_isolated_interpreters`` field to +:data:`sys.implementation`. + +.. + +.. date: 2025-06-18-11-43-17 +.. gh-issue: 135646 +.. nonce: r7ekEn +.. section: Library + +Raise consistent :exc:`NameError` exceptions in +:func:`annotationlib.ForwardRef.evaluate` + +.. + +.. date: 2025-06-17-23-13-56 +.. gh-issue: 135557 +.. nonce: Bfcy4v +.. section: Library + +Fix races on :mod:`heapq` updates and :class:`list` reads on the :term:`free +threaded <free threading>` build. + +.. + +.. date: 2025-06-17-22-44-19 +.. gh-issue: 119180 +.. nonce: Ogv8Nj +.. section: Library + +Only fetch globals and locals if necessary in +:func:`annotationlib.get_annotations` + +.. + +.. date: 2025-06-16-15-03-03 +.. gh-issue: 135561 +.. nonce: mJCN8D +.. section: Library + +Fix a crash on DEBUG builds when an HACL* HMAC routine fails. Patch by +Bénédikt Tran. + +.. + +.. date: 2025-06-14-12-06-55 +.. gh-issue: 135487 +.. nonce: KdVFff +.. section: Library + +Fix :meth:`!reprlib.Repr.repr_int` when given integers with more than +:func:`sys.get_int_max_str_digits` digits. Patch by Bénédikt Tran. + +.. + +.. date: 2025-06-10-21-42-04 +.. gh-issue: 135335 +.. nonce: WnUqb_ +.. section: Library + +:mod:`multiprocessing`: Flush ``stdout`` and ``stderr`` after preloading +modules in the ``forkserver``. + +.. + +.. date: 2025-06-03-12-59-17 +.. gh-issue: 135069 +.. nonce: xop30V +.. section: Library + +Fix the "Invalid error handling" exception in +:class:`!encodings.idna.IncrementalDecoder` to correctly replace the +'errors' parameter. + +.. + +.. date: 2025-06-02-14-36-28 +.. gh-issue: 130662 +.. nonce: Gpr2GB +.. section: Library + ++Accept leading zeros in precision and width fields for ++:class:`~decimal.Decimal` formatting, for example ``format(Decimal(1.25), +'.016f')``. + +.. + +.. date: 2025-06-02-14-28-30 +.. gh-issue: 130662 +.. nonce: EIgIR8 +.. section: Library + +Accept leading zeros in precision and width fields for +:class:`~fractions.Fraction` formatting, for example ``format(Fraction(1, +3), '.016f')``. + +.. + +.. date: 2025-04-07-10-20-16 +.. gh-issue: 87790 +.. nonce: X2SjJe +.. section: Library + +Support underscore and comma as thousands separators in the fractional part +for :class:`~fractions.Fraction`'s formatting. Patch by Sergey B Kirpichev. + +.. + +.. date: 2025-04-07-09-53-54 +.. gh-issue: 87790 +.. nonce: 6nj3zQ +.. section: Library + +Support underscore and comma as thousands separators in the fractional part +for :class:`~decimal.Decimal`'s formatting. Patch by Sergey B Kirpichev. + +.. + +.. date: 2025-03-11-05-24-14 +.. gh-issue: 130664 +.. nonce: g0yNMm +.. section: Library + +Handle corner-case for :class:`~fractions.Fraction`'s formatting: treat +zero-padding (preceding the width field by a zero (``'0'``) character) as an +equivalent to a fill character of ``'0'`` with an alignment type of ``'='``, +just as in case of :class:`float`'s. + +.. + +.. date: 2025-07-01-21-04-47 +.. gh-issue: 136155 +.. nonce: ufmH4Q +.. section: Documentation + +EPUB builds are fixed by excluding non-XHTML-compatible tags. + +.. + +.. date: 2025-07-06-14-53-19 +.. gh-issue: 109700 +.. nonce: KVNQQi +.. section: Core and Builtins + +Fix memory error handling in :c:func:`PyDict_SetDefault`. + +.. + +.. date: 2025-06-26-15-25-51 +.. gh-issue: 78465 +.. nonce: MbDN8X +.. section: Core and Builtins + +Fix error message for ``cls.__new__(cls, ...)`` where ``cls`` is not +instantiable builtin or extension type (with ``tp_new`` set to ``NULL``). + +.. + +.. date: 2025-06-24-06-41-47 +.. gh-issue: 129958 +.. nonce: EaJuS0 +.. section: Core and Builtins + +Differentiate between t-strings and f-strings in syntax error for newlines +in format specifiers of single-quoted interpolated strings. + +.. + +.. date: 2025-06-23-18-08-32 +.. gh-issue: 135871 +.. nonce: 50C528 +.. section: Core and Builtins + +Non-blocking mutex lock attempts now return immediately when the lock is +busy instead of briefly spinning in the :term:`free threading` build. + +.. + +.. date: 2025-06-18-16-45-36 +.. gh-issue: 135106 +.. nonce: cpl6Aq +.. section: Core and Builtins + +Restrict the trashcan mechanism to GC'ed objects and untrack them while in +the trashcan to prevent the GC and trashcan mechanisms conflicting. + +.. + +.. date: 2025-06-17-22-34-58 +.. gh-issue: 135607 +.. nonce: ucsLVu +.. section: Core and Builtins + +Fix potential :mod:`weakref` races in an object's destructor on the +:term:`free threaded <free threading>` build. + +.. + +.. date: 2025-06-17-12-50-48 +.. gh-issue: 135608 +.. nonce: PnHckD +.. section: Core and Builtins + +Fix a crash in the JIT involving attributes of modules. + +.. + +.. date: 2025-06-16-02-31-42 +.. gh-issue: 135543 +.. nonce: 6b0HOF +.. section: Core and Builtins + +Emit ``sys.remote_exec`` audit event when :func:`sys.remote_exec` is called +and migrate ``remote_debugger_script`` to +``cpython.remote_debugger_script``. + +.. + +.. date: 2025-05-31-19-24-54 +.. gh-issue: 134280 +.. nonce: NDVbzY +.. section: Core and Builtins + +Disable constant folding for ``~`` with a boolean argument. This moves the +deprecation warning from compile time to runtime. + +.. + +.. date: 2025-06-25-01-03-10 +.. gh-issue: 135906 +.. nonce: UBrCWq +.. section: C API + +Fix compilation errors when compiling the internal headers with a C++ +compiler. + +.. + +.. date: 2025-05-19-18-09-20 +.. gh-issue: 134273 +.. nonce: ZAliyy +.. section: Build + +Add support for configuring compiler flags for the JIT with ``CFLAGS_JIT`` diff --git a/Misc/NEWS.d/3.14.0rc1.rst b/Misc/NEWS.d/3.14.0rc1.rst new file mode 100644 index 000000000000000..01c4fc92a70e21a --- /dev/null +++ b/Misc/NEWS.d/3.14.0rc1.rst @@ -0,0 +1,316 @@ +.. date: 2025-07-05-15-10-42 +.. gh-issue: 136251 +.. nonce: GRM6o8 +.. release date: 2025-07-22 +.. section: Tools/Demos + +Fixes and usability improvements for ``Tools/wasm/emscripten/web_example`` + +.. + +.. date: 2025-07-21-14-15-25 +.. gh-issue: 135661 +.. nonce: nAxXw5 +.. section: Security + +Fix parsing attributes with whitespaces around the ``=`` separator in +:class:`html.parser.HTMLParser` according to the HTML5 standard. + +.. + +.. date: 2025-06-09-20-38-25 +.. gh-issue: 118350 +.. nonce: KgWCcP +.. section: Security + +Fix support of escapable raw text mode (elements "textarea" and "title") in +:class:`html.parser.HTMLParser`. + +.. + +.. date: 2025-07-21-22-35-50 +.. gh-issue: 136170 +.. nonce: QUlc78 +.. section: Library + +Removed the unreleased ``zipfile.ZipFile.data_offset`` property added in +3.14.0a7 as it wasn't fully clear which behavior it should have in some +situations so the result was not always what a user might expect. + +.. + +.. date: 2025-07-21-16-10-24 +.. gh-issue: 124621 +.. nonce: wyoWc1 +.. section: Library + +pyrepl now works in Emscripten. + +.. + +.. date: 2025-07-20-16-02-00 +.. gh-issue: 136874 +.. nonce: cLC3o1 +.. section: Library + +Discard URL query and fragment in :func:`urllib.request.url2pathname`. + +.. + +.. date: 2025-07-19-16-20-54 +.. gh-issue: 130645 +.. nonce: O-dYcN +.. section: Library + +Enable color help by default in :mod:`argparse`. + +.. + +.. date: 2025-07-11-23-04-39 +.. gh-issue: 136549 +.. nonce: oAi8u4 +.. section: Library + +Fix signature of :func:`threading.excepthook`. + +.. + +.. date: 2025-07-11-03-39-15 +.. gh-issue: 136523 +.. nonce: s7caKL +.. section: Library + +Fix :class:`wave.Wave_write` emitting an unraisable when open raises. + +.. + +.. date: 2025-07-10-10-18-19 +.. gh-issue: 52876 +.. nonce: 9Vjrd8 +.. section: Library + +Add missing ``keepends`` (default ``True``) parameter to +:meth:`!codecs.StreamReaderWriter.readline` and +:meth:`!codecs.StreamReaderWriter.readlines`. + +.. + +.. date: 2025-07-10-00-47-37 +.. gh-issue: 136470 +.. nonce: KlUEUG +.. section: Library + +Correct :class:`concurrent.futures.InterpreterPoolExecutor`'s default thread +name. + +.. + +.. date: 2025-07-09-20-29-30 +.. gh-issue: 136476 +.. nonce: HyLLzh +.. section: Library + +Fix a bug that was causing the ``get_async_stack_trace`` function to miss +some frames in the stack trace. + +.. + +.. date: 2025-07-08-20-58-01 +.. gh-issue: 136434 +.. nonce: uuJsjS +.. section: Library + +Fix docs generation of ``UnboundItem`` in :mod:`concurrent.interpreters` +when running with :option:`-OO`. + +.. + +.. date: 2025-07-07-22-12-32 +.. gh-issue: 136380 +.. nonce: 1b_nXl +.. section: Library + +Raises :exc:`AttributeError` when accessing +:class:`concurrent.futures.InterpreterPoolExecutor` and subinterpreters are +not available. + +.. + +.. date: 2025-06-28-11-32-57 +.. gh-issue: 134759 +.. nonce: AjjKcG +.. section: Library + +Fix :exc:`UnboundLocalError` in :func:`email.message.Message.get_payload` +when the payload to decode is a :class:`bytes` object. Patch by Kliment +Lamonov. + +.. + +.. date: 2025-05-25-11-02-05 +.. gh-issue: 134657 +.. nonce: 3YFhR9 +.. section: Library + +:mod:`asyncio`: Remove some private names from ``asyncio.__all__``. + +.. + +.. date: 2025-07-19-12-37-05 +.. gh-issue: 136801 +.. nonce: XU_tF2 +.. section: Core and Builtins + +Fix PyREPL syntax highlighting on match cases after multi-line case. +Contributed by Olga Matoula. + +.. + +.. date: 2025-07-12-09-59-14 +.. gh-issue: 136421 +.. nonce: ZD1rNj +.. section: Library + +Fix crash when initializing :mod:`datetime` concurrently. + +.. + +.. date: 2025-07-11-13-45-48 +.. gh-issue: 136541 +.. nonce: uZ_-Ju +.. section: Core and Builtins + +Fix some issues with the perf trampolines on x86-64 and aarch64. The +trampolines were not being generated correctly for some cases, which could +lead to the perf integration not working correctly. Patch by Pablo Galindo. + +.. + +.. date: 2025-07-10-23-23-50 +.. gh-issue: 136517 +.. nonce: _NHJyv +.. section: Core and Builtins + +Fixed a typo that prevented printing of uncollectable objects when the +:const:`gc.DEBUG_UNCOLLECTABLE` mode was set. + +.. + +.. date: 2025-07-10-15-53-16 +.. gh-issue: 136525 +.. nonce: xAko0e +.. section: Core and Builtins + +Fix issue where per-thread bytecode was not instrumented for newly created +threads. + +.. + +.. date: 2025-07-08-23-53-51 +.. gh-issue: 132661 +.. nonce: B84iYt +.. section: Core and Builtins + +``Interpolation.expression`` now has a default, the empty string. + +.. + +.. date: 2025-07-08-23-22-08 +.. gh-issue: 132661 +.. nonce: 34ftJl +.. section: Core and Builtins + +Reflect recent :pep:`750` change. + +Disallow concatenation of ``string.templatelib.Template`` and :class:`str`. +Also, disallow implicit concatenation of t-string literals with string or +f-string literals. + +.. + +.. date: 2025-06-12-00-03-34 +.. gh-issue: 116738 +.. nonce: iBBAdo +.. section: Library + +Make functions in :mod:`grp` thread-safe on the :term:`free threaded <free +threading>` build. + +.. + +.. date: 2025-06-06-02-24-42 +.. gh-issue: 135148 +.. nonce: r-t2sC +.. section: Core and Builtins + +Fixed a bug where f-string debug expressions (using =) would incorrectly +strip out parts of strings containing escaped quotes and # characters. Patch +by Pablo Galindo. + +.. + +.. date: 2025-06-03-21-06-22 +.. gh-issue: 133136 +.. nonce: Usnvri +.. section: Core and Builtins + +Limit excess memory usage in the :term:`free threading` build when a large +dictionary or list is resized and accessed by multiple threads. + +.. + +.. date: 2025-05-17-20-56-05 +.. gh-issue: 91153 +.. nonce: afgtG2 +.. section: Core and Builtins + +Fix a crash when a :class:`bytearray` is concurrently mutated during item +assignment. + +.. + +.. date: 2025-04-16-12-01-13 +.. gh-issue: 127971 +.. nonce: pMDOQ0 +.. section: Core and Builtins + +Fix off-by-one read beyond the end of a string in string search. + +.. + +.. date: 2025-07-22-15-18-08 +.. gh-issue: 112068 +.. nonce: 4WvT-8 +.. section: C API + +Revert support of nullable arguments in :c:func:`PyArg_Parse`. + +.. + +.. date: 2025-06-24-11-10-01 +.. gh-issue: 133296 +.. nonce: lIEuVJ +.. section: C API + +New variants for the critical section API that accept one or two +:c:type:`PyMutex` pointers rather than :c:type:`PyObject` instances are now +public in the non-limited C API. + +.. + +.. date: 2025-05-20-17-13-51 +.. gh-issue: 134009 +.. nonce: CpCmry +.. section: C API + +Expose :c:func:`PyMutex_IsLocked` as part of the public C API. + +.. + +.. date: 2025-07-18-17-15-00 +.. gh-issue: 135621 +.. nonce: 9cyCNb +.. section: Build + +PyREPL no longer depends on the :mod:`curses` standard library. Contributed +by Łukasz Langa. diff --git a/Misc/NEWS.d/3.14.0rc2.rst b/Misc/NEWS.d/3.14.0rc2.rst new file mode 100644 index 000000000000000..ffa043eebbd7a0e --- /dev/null +++ b/Misc/NEWS.d/3.14.0rc2.rst @@ -0,0 +1,216 @@ +.. date: 2025-08-06-06-29-12 +.. gh-issue: 137450 +.. nonce: JZypb7 +.. release date: 2025-08-14 +.. section: macOS + +macOS installer shell path management improvements: separate the installer +``Shell profile updater`` postinstall script from the ``Update Shell +Profile.command`` to enable more robust error handling. + +.. + +.. date: 2025-07-27-02-17-40 +.. gh-issue: 137134 +.. nonce: pjgITs +.. section: macOS + +Update macOS installer to ship with SQLite version 3.50.4. + +.. + +.. date: 2025-07-27-02-16-53 +.. gh-issue: 137134 +.. nonce: W0WpDF +.. section: Windows + +Update Windows installer to ship with SQLite 3.50.4. + +.. + +.. date: 2025-08-08-15-00-38 +.. gh-issue: 137426 +.. nonce: lW-Rk2 +.. section: Library + +Remove the code deprecation of ``importlib.abc.ResourceLoader``. It is +documented as deprecated, but left for backwards compatibility with other +classes in ``importlib.abc``. + +.. + +.. date: 2025-07-31-10-31-56 +.. gh-issue: 137282 +.. nonce: GOCwIC +.. section: Library + +Fix tab completion and :func:`dir` on :mod:`concurrent.futures`. + +.. + +.. date: 2025-07-30-18-07-33 +.. gh-issue: 137257 +.. nonce: XBtzf2 +.. section: Library + +Bump the version of pip bundled in ensurepip to version 25.2 + +.. + +.. date: 2025-07-29-21-18-31 +.. gh-issue: 137226 +.. nonce: B_4lpu +.. section: Library + +Fix behavior of :meth:`annotationlib.ForwardRef.evaluate` when the +*type_params* parameter is passed and the name of a type param is also +present in an enclosing scope. + +.. + +.. date: 2025-07-25-09-21-56 +.. gh-issue: 130522 +.. nonce: Crwq68 +.. section: Library + +Fix unraisable :exc:`TypeError` raised during :term:`interpreter shutdown` +in the :mod:`threading` module. + +.. + +.. date: 2025-07-24-00-38-07 +.. gh-issue: 137059 +.. nonce: fr64oW +.. section: Library + +Fix handling of file URLs with a Windows drive letter in the URL authority +by :func:`urllib.request.url2pathname`. This fixes a regression in earlier +pre-releases of Python 3.14. + +.. + +.. date: 2025-07-23-00-35-29 +.. gh-issue: 130577 +.. nonce: c7EITy +.. section: Library + +:mod:`tarfile` now validates archives to ensure member offsets are +non-negative. (Contributed by Alexander Enrique Urieles Nieto in +:gh:`130577`.) + +.. + +.. date: 2025-07-20-16-56-55 +.. gh-issue: 135228 +.. nonce: n_XIao +.. section: Library + +When :mod:`dataclasses` replaces a class with a slotted dataclass, the +original class can now be garbage collected again. Earlier changes in Python +3.14 caused this class to always remain in existence together with the +replacement class synthesized by :mod:`dataclasses`. + +.. + +.. date: 2025-07-01-23-00-58 +.. gh-issue: 136155 +.. nonce: 4siQQO +.. section: Documentation + +We are now checking for fatal errors in EPUB builds in CI. + +.. + +.. date: 2025-08-06-15-39-54 +.. gh-issue: 137400 +.. nonce: xIw0zs +.. section: Core and Builtins + +Fix a crash in the :term:`free threading` build when disabling profiling or +tracing across all threads with :c:func:`PyEval_SetProfileAllThreads` or +:c:func:`PyEval_SetTraceAllThreads` or their Python equivalents +:func:`threading.settrace_all_threads` and +:func:`threading.setprofile_all_threads`. + +.. + +.. date: 2025-08-02-23-04-57 +.. gh-issue: 137314 +.. nonce: wjEdzD +.. section: Core and Builtins + +Fixed a regression where raw f-strings incorrectly interpreted escape +sequences in format specifications. Raw f-strings now properly preserve +literal backslashes in format specs, matching the behavior from Python 3.11. +For example, ``rf"{obj:\xFF}"`` now correctly produces ``'\\xFF'`` instead +of ``'ÿ'``. Patch by Pablo Galindo. + +.. + +.. date: 2025-08-02-10-27-53 +.. gh-issue: 137308 +.. nonce: at05p_ +.. section: Core and Builtins + +A standalone docstring in a node body is optimized as a :keyword:`pass` +statement to ensure that the node's body is never empty. There was a +:exc:`ValueError` in :func:`compile` otherwise. + +.. + +.. date: 2025-08-01-18-54-31 +.. gh-issue: 137288 +.. nonce: FhE7ku +.. section: Core and Builtins + +Fix bug where some bytecode instructions of a boolean expression are not +associated with the correct exception handler. + +.. + +.. date: 2025-07-28-19-11-34 +.. gh-issue: 134291 +.. nonce: IiB9Id +.. section: Core and Builtins + +Remove some newer macOS API usage from the JIT compiler in order to restore +compatibility with older OSX 10.15 deployment targets. + +.. + +.. date: 2025-07-25-22-31-52 +.. gh-issue: 131338 +.. nonce: zJDCMp +.. section: Core and Builtins + +Disable computed stack limit checks on non-glibc linux platforms to fix +crashes on deep recursion. + +.. + +.. date: 2025-07-24-17-30-58 +.. gh-issue: 136870 +.. nonce: ncx82J +.. section: Core and Builtins + +Fix data races while de-instrumenting bytecode of code objects running +concurrently in threads. + +.. + +.. date: 2025-08-13-13-41-04 +.. gh-issue: 137573 +.. nonce: r6uwRf +.. section: C API + +Mark ``_PyOptimizer_Optimize`` as :c:macro:`Py_NO_INLINE` to prevent stack +overflow crashes on macOS. + +.. + +.. date: 2025-08-13-12-10-12 +.. gh-issue: 132339 +.. nonce: 3Czz5y +.. section: Build + +Add support for OpenSSL 3.5. diff --git a/Misc/NEWS.d/3.14.0rc3.rst b/Misc/NEWS.d/3.14.0rc3.rst new file mode 100644 index 000000000000000..339f0ab3b1c7c9a --- /dev/null +++ b/Misc/NEWS.d/3.14.0rc3.rst @@ -0,0 +1,295 @@ +.. date: 2025-09-15-15-34-29 +.. gh-issue: 138896 +.. nonce: lkiF_7 +.. release date: 2025-09-18 +.. section: Windows + +Fix error installing C runtime on non-updated Windows machines + +.. + +.. date: 2025-08-21-14-04-50 +.. gh-issue: 137873 +.. nonce: qxffLt +.. section: Tools/Demos + +The iOS test runner has been simplified, resolving some issues that have +been observed using the runner in GitHub Actions and Azure Pipelines test +environments. + +.. + +.. date: 2025-06-18-13-34-55 +.. gh-issue: 135661 +.. nonce: NZlpWf +.. section: Security + +Fix CDATA section parsing in :class:`html.parser.HTMLParser` according to +the HTML5 standard: ``] ]>`` and ``]] >`` no longer end the CDATA section. +Add private method ``_set_support_cdata()`` which can be used to specify how +to parse ``<[CDATA[`` --- as a CDATA section in foreign content (SVG or +MathML) or as a bogus comment in the HTML namespace. + +.. + +.. date: 2025-09-16-19-05-29 +.. gh-issue: 138998 +.. nonce: URl0Y_ +.. section: Library + +Update bundled libexpat to 2.7.2 + +.. + +.. date: 2025-09-16-15-56-29 +.. gh-issue: 118803 +.. nonce: aOPtmL +.. section: Library + +Add back :class:`collections.abc.ByteString` and :class:`typing.ByteString`. +Both had been removed in prior alpha, beta and release candidates for Python +3.14, but their removal has now been postponed to Python 3.17. + +.. + +.. date: 2025-09-15-13-09-19 +.. gh-issue: 137226 +.. nonce: HH3_ik +.. section: Library + +Fix :func:`typing.get_type_hints` calls on generic :class:`typing.TypedDict` +classes defined with string annotations. + +.. + +.. date: 2025-09-12-01-01-05 +.. gh-issue: 138804 +.. nonce: 46ZukT +.. section: Library + +Raise :exc:`TypeError` instead of :exc:`AttributeError` when an argument of +incorrect type is passed to :func:`shlex.quote`. This restores the behavior +of the function prior to 3.14. + +.. + +.. date: 2025-09-10-10-02-59 +.. gh-issue: 128636 +.. nonce: ldRKGZ +.. section: Library + +Fix crash in PyREPL when os.environ is overwritten with an invalid value for +mac + +.. + +.. date: 2025-09-06-11-26-21 +.. gh-issue: 138514 +.. nonce: 66ltOb +.. section: Library + +Raise :exc:`ValueError` when a multi-character string is passed to the +*echo_char* parameter of :func:`getpass.getpass`. Patch by Benjamin Johnson. + +.. + +.. date: 2025-09-05-07-50-18 +.. gh-issue: 138515 +.. nonce: E3M-pu +.. section: Library + +:mod:`email` is added to Emscripten build. + +.. + +.. date: 2025-09-05-05-53-43 +.. gh-issue: 99948 +.. nonce: KMSlG6 +.. section: Library + +:func:`ctypes.util.find_library` now works in Emscripten build. + +.. + +.. date: 2025-08-30-10-58-15 +.. gh-issue: 138253 +.. nonce: 9Ehj-N +.. section: Library + +Add the *block* parameter in the :meth:`!put` and :meth:`!get` methods of +the :mod:`concurrent.interpreters` queues for compatibility with the +:class:`queue.Queue` interface. + +.. + +.. date: 2025-08-25-18-06-04 +.. gh-issue: 138133 +.. nonce: Zh9rGo +.. section: Library + +Prevent infinite traceback loop when sending CTRL^C to Python through +``strace``. + +.. + +.. date: 2025-08-18-16-02-51 +.. gh-issue: 134869 +.. nonce: GnAjnU +.. section: Library + +Fix an issue where pressing Ctrl+C during tab completion in the REPL would +leave the autocompletion menu in a corrupted state. + +.. + +.. date: 2025-08-16-18-11-41 +.. gh-issue: 90548 +.. nonce: q3aJUK +.. section: Library + +Fix ``musl`` detection for :func:`platform.libc_ver` on Alpine Linux if +compiled with --strip-all. + +.. + +.. date: 2025-07-13-13-31-22 +.. gh-issue: 136134 +.. nonce: mh6VjS +.. section: Library + +:meth:`!SMTP.auth_cram_md5` now raises an :exc:`~smtplib.SMTPException` +instead of a :exc:`ValueError` if Python has been built without MD5 support. +In particular, :class:`~smtplib.SMTP` clients will not attempt to use this +method even if the remote server is assumed to support it. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-07-13-11-20-05 +.. gh-issue: 136134 +.. nonce: xhh0Kq +.. section: Library + +:meth:`IMAP4.login_cram_md5 <imaplib.IMAP4.login_cram_md5>` now raises an +:exc:`IMAP4.error <imaplib.IMAP4.error>` if CRAM-MD5 authentication is not +supported. Patch by Bénédikt Tran. + +.. + +.. date: 2025-06-01-11-14-00 +.. gh-issue: 134953 +.. nonce: ashdfs +.. section: Library + +Expand ``_colorize`` theme with ``keyword_constant`` and implement in +:term:`repl`. + +.. + +.. date: 2025-09-10-14-53-59 +.. gh-issue: 71810 +.. nonce: ppf0J- +.. section: Core and Builtins + +Raise :exc:`OverflowError` for ``(-1).to_bytes()`` for signed conversions +when bytes count is zero. Patch by Sergey B Kirpichev. + +.. + +.. date: 2025-09-05-01-19-04 +.. gh-issue: 138192 +.. nonce: erluq5 +.. section: Core and Builtins + +Fix :mod:`contextvars` initialization so that all subinterpreters are +assigned the :attr:`~contextvars.Token.MISSING` value. + +.. + +.. date: 2025-09-03-17-00-30 +.. gh-issue: 138479 +.. nonce: qUxgWs +.. section: Core and Builtins + +Fix a crash when a generic object's ``__typing_subst__`` returns an object +that isn't a :class:`tuple`. + +.. + +.. date: 2025-09-02-09-10-06 +.. gh-issue: 138372 +.. nonce: h1Xk4- +.. section: Core and Builtins + +Fix :exc:`SyntaxWarning` emitted for erroneous subscript expressions +involving :ref:`template string literals <t-strings>`. Patch by Brian +Schubert. + +.. + +.. date: 2025-09-01-16-09-02 +.. gh-issue: 138318 +.. nonce: t-WEN5 +.. section: Core and Builtins + +The default REPL now avoids highlighting built-in names (for instance +:class:`set` or :func:`format`) when they are used as attribute names (for +instance in ``value.set`` or ``text.format``). + +.. + +.. date: 2025-09-01-13-54-43 +.. gh-issue: 138349 +.. nonce: 0fGmAi +.. section: Core and Builtins + +Fix crash in certain cases where a module contains both a module-level +annotation and a comprehension. + +.. + +.. date: 2025-08-22-11-39-40 +.. gh-issue: 137384 +.. nonce: j4b_in +.. section: Core and Builtins + +Fix a crash when using the :mod:`warnings` module in a finalizer at +shutdown. Patch by Kumar Aditya. + +.. + +.. date: 2025-08-17-13-36-53 +.. gh-issue: 137883 +.. nonce: 55VDCN +.. section: Core and Builtins + +Fix runaway recursion when calling a function with keyword arguments. + +.. + +.. date: 2025-08-15-15-45-26 +.. gh-issue: 137079 +.. nonce: YEow69 +.. section: Core and Builtins + +Fix keyword typo recognition when parsing files. Patch by Pablo Galindo. + +.. + +.. date: 2025-08-14-14-18-29 +.. gh-issue: 137728 +.. nonce: HdYS9R +.. section: Core and Builtins + +Fix the JIT's handling of many local variables. This previously caused a +segfault. + +.. + +.. date: 2025-08-10-21-34-12 +.. gh-issue: 137576 +.. nonce: 0ZicS- +.. section: Core and Builtins + +Fix for incorrect source code being shown in tracebacks from the Basic REPL +when :envvar:`PYTHONSTARTUP` is given. Patch by Adam Hartz. diff --git a/Misc/NEWS.d/3.14.1.rst b/Misc/NEWS.d/3.14.1.rst new file mode 100644 index 000000000000000..eb26d1c40c18128 --- /dev/null +++ b/Misc/NEWS.d/3.14.1.rst @@ -0,0 +1,2105 @@ +.. date: 2025-10-08-22-54-38 +.. gh-issue: 139810 +.. nonce: LAaemi +.. release date: 2025-12-02 +.. section: Windows + +Installing with ``py install 3[.x]-dev`` will now select final versions as +well as prereleases. + +.. + +.. date: 2025-11-18-13-55-47 +.. gh-issue: 141692 +.. nonce: tud9if +.. section: Tools/Demos + +Each slice of an iOS XCframework now contains a ``lib`` folder that contains +a symlink to the libpython dylib. This allows binary modules to be compiled +for iOS using dynamic libreary linking, rather than Framework linking. + +.. + +.. date: 2025-11-12-12-54-28 +.. gh-issue: 141442 +.. nonce: 50dS3P +.. section: Tools/Demos + +The iOS testbed now correctly handles test arguments that contain spaces. + +.. + +.. date: 2025-10-29-15-20-19 +.. gh-issue: 140702 +.. nonce: ZXtW8h +.. section: Tools/Demos + +The iOS testbed app will now expose the ``GITHUB_ACTIONS`` environment +variable to iOS apps being tested. + +.. + +.. date: 2025-08-06-11-54-55 +.. gh-issue: 137484 +.. nonce: 8iFAQs +.. section: Tools/Demos + +Have ``Tools/wasm/wasi`` put the build Python into a directory named after +the build triple instead of "build". + +.. + +.. date: 2025-07-30-11-15-47 +.. gh-issue: 137248 +.. nonce: 8IxwY3 +.. section: Tools/Demos + +Add a ``--logdir`` option to ``Tools/wasm/wasi`` for specifying where to +write log files. + +.. + +.. date: 2025-07-30-10-28-35 +.. gh-issue: 137243 +.. nonce: NkdUqH +.. section: Tools/Demos + +Have Tools/wasm/wasi detect a WASI SDK install in /opt when it was directly +extracted from a release tarball. + +.. + +.. date: 2025-10-23-16-39-49 +.. gh-issue: 140482 +.. nonce: ZMtyeD +.. section: Tests + +Preserve and restore the state of ``stty echo`` as part of the test +environment. + +.. + +.. date: 2025-10-15-00-52-12 +.. gh-issue: 140082 +.. nonce: fpET50 +.. section: Tests + +Update ``python -m test`` to set ``FORCE_COLOR=1`` when being run with color +enabled so that :mod:`unittest` which is run by it with redirected output +will output in color. + +.. + +.. date: 2025-09-22-15-40-09 +.. gh-issue: 139208 +.. nonce: Tc13dl +.. section: Tests + +Fix regrtest ``--fast-ci --verbose``: don't ignore the ``--verbose`` option +anymore. Patch by Victor Stinner. + +.. + +.. date: 2025-07-09-21-45-51 +.. gh-issue: 136442 +.. nonce: jlbklP +.. section: Tests + +Use exitcode ``1`` instead of ``5`` if :func:`unittest.TestCase.setUpClass` +raises an exception + +.. + +.. date: 2025-10-07-19-31-34 +.. gh-issue: 139700 +.. nonce: vNHU1O +.. section: Security + +Check consistency of the zip64 end of central directory record. Support +records with "zip64 extensible data" if there are no bytes prepended to the +ZIP file. + +.. + +.. date: 2025-09-24-13-39-56 +.. gh-issue: 139283 +.. nonce: jODz_q +.. section: Security + +:mod:`sqlite3`: correctly handle maximum number of rows to fetch in +:meth:`Cursor.fetchmany <sqlite3.Cursor.fetchmany>` and reject negative +values for :attr:`Cursor.arraysize <sqlite3.Cursor.arraysize>`. Patch by +Bénédikt Tran. + +.. + +.. date: 2025-08-15-23-08-44 +.. gh-issue: 137836 +.. nonce: b55rhh +.. section: Security + +Add support of the "plaintext" element, RAWTEXT elements "xmp", "iframe", +"noembed" and "noframes", and optionally RAWTEXT element "noscript" in +:class:`html.parser.HTMLParser`. + +.. + +.. date: 2025-06-28-13-23-53 +.. gh-issue: 136063 +.. nonce: aGk0Jv +.. section: Security + +:mod:`email.message`: ensure linear complexity for legacy HTTP parameters +parsing. Patch by Bénédikt Tran. + +.. + +.. date: 2025-05-30-22-33-27 +.. gh-issue: 136065 +.. nonce: bu337o +.. section: Security + +Fix quadratic complexity in :func:`os.path.expandvars`. + +.. + +.. date: 2024-05-23-11-47-48 +.. gh-issue: 119451 +.. nonce: qkJe9- +.. section: Security + +Fix a potential memory denial of service in the :mod:`http.client` module. +When connecting to a malicious server, it could cause an arbitrary amount of +memory to be allocated. This could have led to symptoms including a +:exc:`MemoryError`, swapping, out of memory (OOM) killed processes or +containers, or even system crashes. + +.. + +.. date: 2024-05-21-22-11-31 +.. gh-issue: 119342 +.. nonce: BTFj4Z +.. section: Security + +Fix a potential memory denial of service in the :mod:`plistlib` module. When +reading a Plist file received from untrusted source, it could cause an +arbitrary amount of memory to be allocated. This could have led to symptoms +including a :exc:`MemoryError`, swapping, out of memory (OOM) killed +processes or containers, or even system crashes. + +.. + +.. date: 2025-11-29-04-20-44 +.. gh-issue: 74389 +.. nonce: pW3URj +.. section: Library + +When the stdin being used by a :class:`subprocess.Popen` instance is closed, +this is now ignored in :meth:`subprocess.Popen.communicate` instead of +leaving the class in an inconsistent state. + +.. + +.. date: 2025-11-29-03-02-45 +.. gh-issue: 87512 +.. nonce: bn4xbm +.. section: Library + +Fix :func:`subprocess.Popen.communicate` timeout handling on Windows when +writing large input. Previously, the timeout was ignored during stdin +writing, causing the method to block indefinitely if the child process did +not consume input quickly. The stdin write is now performed in a background +thread, allowing the timeout to be properly enforced. + +.. + +.. date: 2025-11-27-20-16-38 +.. gh-issue: 141473 +.. nonce: Wq4xVN +.. section: Library + +When :meth:`subprocess.Popen.communicate` was called with *input* and a +*timeout* and is called for a second time after a +:exc:`~subprocess.TimeoutExpired` exception before the process has died, it +should no longer hang. + +.. + +.. date: 2025-11-25-16-00-29 +.. gh-issue: 59000 +.. nonce: YtOyJy +.. section: Library + +Fix :mod:`pdb` breakpoint resolution for class methods when the module +defining the class is not imported. + +.. + +.. date: 2025-11-18-14-39-31 +.. gh-issue: 141570 +.. nonce: q3n984 +.. section: Library + +Support :term:`file-like object` raising :exc:`OSError` from +:meth:`~io.IOBase.fileno` in color detection (``_colorize.can_colorize()``). +This can occur when ``sys.stdout`` is redirected. + +.. + +.. date: 2025-11-17-08-16-30 +.. gh-issue: 141659 +.. nonce: QNi9Aj +.. section: Library + +Fix bad file descriptor errors from ``_posixsubprocess`` on AIX. + +.. + +.. date: 2025-11-15-14-58-12 +.. gh-issue: 141600 +.. nonce: XY2BXg +.. section: Library + +Fix musl version detection on Void Linux. + +.. + +.. date: 2025-11-14-16-24-20 +.. gh-issue: 141497 +.. nonce: L_CxDJ +.. section: Library + +:mod:`ipaddress`: ensure that the methods :meth:`IPv4Network.hosts() +<ipaddress.IPv4Network.hosts>` and :meth:`IPv6Network.hosts() +<ipaddress.IPv6Network.hosts>` always return an iterator. + +.. + +.. date: 2025-11-13-14-51-30 +.. gh-issue: 140938 +.. nonce: kXsHHv +.. section: Library + +The :func:`statistics.stdev` and :func:`statistics.pstdev` functions now +raise a :exc:`ValueError` when the input contains an infinity or a NaN. + +.. + +.. date: 2025-11-12-15-42-47 +.. gh-issue: 124111 +.. nonce: hTw4OE +.. section: Library + +Updated Tcl threading configuration in :mod:`_tkinter` to assume that +threads are always available in Tcl 9 and later. + +.. + +.. date: 2025-11-12-01-49-03 +.. gh-issue: 137109 +.. nonce: D6sq2B +.. section: Library + +The :mod:`os.fork` and related forking APIs will no longer warn in the +common case where Linux or macOS platform APIs return the number of threads +in a process and find the answer to be 1 even when a +:func:`os.register_at_fork` ``after_in_parent=`` callback (re)starts a +thread. + +.. + +.. date: 2025-11-10-01-47-18 +.. gh-issue: 141314 +.. nonce: baaa28 +.. section: Library + +Fix assertion failure in :meth:`io.TextIOWrapper.tell` when reading files +with standalone carriage return (``\r``) line endings. + +.. + +.. date: 2025-11-09-18-55-13 +.. gh-issue: 141311 +.. nonce: qZ3swc +.. section: Library + +Fix assertion failure in :func:`!io.BytesIO.readinto` and undefined behavior +arising when read position is above capcity in :class:`io.BytesIO`. + +.. + +.. date: 2025-11-06-15-11-50 +.. gh-issue: 141141 +.. nonce: tgIfgH +.. section: Library + +Fix a thread safety issue with :func:`base64.b85decode`. Contributed by +Benel Tayar. + +.. + +.. date: 2025-11-04-15-40-35 +.. gh-issue: 137969 +.. nonce: 9VZQVt +.. section: Library + +Fix :meth:`annotationlib.ForwardRef.evaluate` returning +:class:`~annotationlib.ForwardRef` objects which don't update with new +globals. + +.. + +.. date: 2025-11-03-17-13-00 +.. gh-issue: 140911 +.. nonce: 7KFvSQ +.. section: Library + +:mod:`collections`: Ensure that the methods ``UserString.rindex()`` and +``UserString.index()`` accept :class:`collections.UserString` instances as +the sub argument. + +.. + +.. date: 2025-11-03-16-23-54 +.. gh-issue: 140797 +.. nonce: DuFEeR +.. section: Library + +The undocumented :class:`!re.Scanner` class now forbids regular expressions +containing capturing groups in its lexicon patterns. Patterns using +capturing groups could previously lead to crashes with segmentation fault. +Use non-capturing groups (?:...) instead. + +.. + +.. date: 2025-11-03-05-38-31 +.. gh-issue: 125115 +.. nonce: jGS8MN +.. section: Library + +Refactor the :mod:`pdb` parsing issue so positional arguments can pass +through intuitively. + +.. + +.. date: 2025-11-02-19-23-32 +.. gh-issue: 140815 +.. nonce: McEG-T +.. section: Library + +:mod:`faulthandler` now detects if a frame or a code object is invalid or +freed. Patch by Victor Stinner. + +.. + +.. date: 2025-11-02-11-46-00 +.. gh-issue: 100218 +.. nonce: 9Ezfdq +.. section: Library + +Correctly set :attr:`~OSError.errno` when :func:`socket.if_nametoindex` or +:func:`socket.if_indextoname` raise an :exc:`OSError`. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-11-02-10-44-23 +.. gh-issue: 140875 +.. nonce: wt6B37 +.. section: Library + +Fix handling of unclosed character references (named and numerical) followed +by the end of file in :class:`html.parser.HTMLParser` with +``convert_charrefs=False``. + +.. + +.. date: 2025-11-02-09-37-22 +.. gh-issue: 140734 +.. nonce: f8gST9 +.. section: Library + +:mod:`multiprocessing`: fix off-by-one error when checking the length of a +temporary socket file path. Patch by Bénédikt Tran. + +.. + +.. date: 2025-11-01-00-36-14 +.. gh-issue: 140874 +.. nonce: eAWt3K +.. section: Library + +Bump the version of pip bundled in ensurepip to version 25.3 + +.. + +.. date: 2025-10-31-15-06-26 +.. gh-issue: 140691 +.. nonce: JzHGtg +.. section: Library + +In :mod:`urllib.request`, when opening a FTP URL fails because a data +connection cannot be made, the control connection's socket is now closed to +avoid a :exc:`ResourceWarning`. + +.. + +.. date: 2025-10-31-13-57-55 +.. gh-issue: 103847 +.. nonce: VM7TnW +.. section: Library + +Fix hang when cancelling process created by +:func:`asyncio.create_subprocess_exec` or +:func:`asyncio.create_subprocess_shell`. Patch by Kumar Aditya. + +.. + +.. date: 2025-10-29-16-12-41 +.. gh-issue: 120057 +.. nonce: qGj5Dl +.. section: Library + +Add :func:`os.reload_environ` to ``os.__all__``. + +.. + +.. date: 2025-10-28-17-43-51 +.. gh-issue: 140228 +.. nonce: 8kfHhO +.. section: Library + +Avoid making unnecessary filesystem calls for frozen modules in +:mod:`linecache` when the global module cache is not present. + +.. + +.. date: 2025-10-27-18-29-42 +.. gh-issue: 140590 +.. nonce: LT9HHn +.. section: Library + +Fix arguments checking for the :meth:`!functools.partial.__setstate__` that +may lead to internal state corruption and crash. Patch by Sergey Miryanov. + +.. + +.. date: 2025-10-27-16-01-41 +.. gh-issue: 125434 +.. nonce: qy0uRA +.. section: Library + +Display thread name in :mod:`faulthandler` on Windows. Patch by Victor +Stinner. + +.. + +.. date: 2025-10-27-13-49-31 +.. gh-issue: 140634 +.. nonce: ULng9G +.. section: Library + +Fix a reference counting bug in :meth:`!os.sched_param.__reduce__`. + +.. + +.. date: 2025-10-26-16-24-12 +.. gh-issue: 140633 +.. nonce: ioayC1 +.. section: Library + +Ignore :exc:`AttributeError` when setting a module's ``__file__`` attribute +when loading an extension module packaged as Apple Framework. + +.. + +.. date: 2025-10-25-21-26-16 +.. gh-issue: 140593 +.. nonce: OxlLc9 +.. section: Library + +:mod:`xml.parsers.expat`: Fix a memory leak that could affect users with +:meth:`~xml.parsers.expat.xmlparser.ElementDeclHandler` set to a custom +element declaration handler. Patch by Sebastian Pipping. + +.. + +.. date: 2025-10-25-21-04-00 +.. gh-issue: 140607 +.. nonce: oOZGxS +.. section: Library + +Inside :meth:`io.RawIOBase.read`, validate that the count of bytes returned +by :meth:`io.RawIOBase.readinto` is valid (inside the provided buffer). + +.. + +.. date: 2025-10-23-19-39-16 +.. gh-issue: 138162 +.. nonce: Znw5DN +.. section: Library + +Fix :class:`logging.LoggerAdapter` with ``merge_extra=True`` and without the +*extra* argument. + +.. + +.. date: 2025-10-23-12-12-22 +.. gh-issue: 138774 +.. nonce: mnh2gU +.. section: Library + +:func:`ast.unparse` now generates full source code when handling +:class:`ast.Interpolation` nodes that do not have a specified source. + +.. + +.. date: 2025-10-22-20-52-13 +.. gh-issue: 140474 +.. nonce: xIWlip +.. section: Library + +Fix memory leak in :class:`array.array` when creating arrays from an empty +:class:`str` and the ``u`` type code. + +.. + +.. date: 2025-10-21-15-54-13 +.. gh-issue: 137530 +.. nonce: ZyIVUH +.. section: Library + +:mod:`dataclasses` Fix annotations for generated ``__init__`` methods by +replacing the annotations that were in-line in the generated source code +with ``__annotate__`` functions attached to the methods. + +.. + +.. date: 2025-10-20-12-33-49 +.. gh-issue: 140348 +.. nonce: SAKnQZ +.. section: Library + +Fix regression in Python 3.14.0 where using the ``|`` operator on a +:class:`typing.Union` object combined with an object that is not a type +would raise an error. + +.. + +.. date: 2025-10-17-23-58-11 +.. gh-issue: 140272 +.. nonce: lhY8uS +.. section: Library + +Fix memory leak in the :meth:`!clear` method of the :mod:`dbm.gnu` database. + +.. + +.. date: 2025-10-15-21-42-13 +.. gh-issue: 140041 +.. nonce: _Fka2j +.. section: Library + +Fix import of :mod:`ctypes` on Android and Cygwin when ABI flags are +present. + +.. + +.. date: 2025-10-15-20-47-04 +.. gh-issue: 140120 +.. nonce: 3gffZq +.. section: Library + +Fixed a memory leak in :mod:`hmac` when it was using the hacl-star backend. +Discovered by ``@ashm-dev`` using AddressSanitizer. + +.. + +.. date: 2025-10-11-10-02-56 +.. gh-issue: 139905 +.. nonce: UyJIR_ +.. section: Library + +Add suggestion to error message for :class:`typing.Generic` subclasses when +``cls.__parameters__`` is missing due to a parent class failing to call +:meth:`super().__init_subclass__() <object.__init_subclass__>` in its +``__init_subclass__``. + +.. + +.. date: 2025-10-10-11-22-50 +.. gh-issue: 139894 +.. nonce: ECAXqj +.. section: Library + +Fix incorrect sharing of current task with the child process while forking +in :mod:`asyncio`. Patch by Kumar Aditya. + +.. + +.. date: 2025-10-09-21-37-20 +.. gh-issue: 139845 +.. nonce: dzx5UP +.. section: Library + +Fix to not print KeyboardInterrupt twice in default asyncio REPL. + +.. + +.. date: 2025-10-09-13-48-28 +.. gh-issue: 139783 +.. nonce: __NUgo +.. section: Library + +Fix :func:`inspect.getsourcelines` for the case when a decorator is followed +by a comment or an empty line. + +.. + +.. date: 2025-10-09-03-06-19 +.. gh-issue: 139809 +.. nonce: lzHJNu +.. section: Library + +Prevent premature colorization of subparser ``prog`` in +:meth:`argparse.ArgumentParser.add_subparsers` to respect color environment +variable changes after parser creation. + +.. + +.. date: 2025-10-08-00-06-30 +.. gh-issue: 139736 +.. nonce: baPeBd +.. section: Library + +Fix excessive indentation in the default :mod:`argparse` +:class:`!HelpFormatter`. Patch by Alexander Edland. + +.. + +.. date: 2025-10-02-17-40-10 +.. gh-issue: 70765 +.. nonce: zVlLZn +.. section: Library + +:mod:`http.server`: fix default handling of HTTP/0.9 requests in +:class:`~http.server.BaseHTTPRequestHandler`. Previously, +:meth:`!BaseHTTPRequestHandler.parse_request` incorrectly waited for headers +in the request although those are not supported in HTTP/0.9. Patch by +Bénédikt Tran. + +.. + +.. date: 2025-09-30-12-52-54 +.. gh-issue: 63161 +.. nonce: mECM1A +.. section: Library + +Fix :func:`tokenize.detect_encoding`. Support non-UTF-8 shebang and comments +if non-UTF-8 encoding is specified. Detect decoding error for non-UTF-8 +encoding. Detect null bytes in source code. + +.. + +.. date: 2025-09-28-16-34-11 +.. gh-issue: 139391 +.. nonce: nRFnmx +.. section: Library + +Fix an issue when, on non-Windows platforms, it was not possible to +gracefully exit a ``python -m asyncio`` process suspended by Ctrl+Z and +later resumed by :manpage:`fg` other than with :manpage:`kill`. + +.. + +.. date: 2025-09-25-20-16-10 +.. gh-issue: 101828 +.. nonce: yTxJlJ +.. section: Library + +Fix ``'shift_jisx0213'``, ``'shift_jis_2004'``, ``'euc_jisx0213'`` and +``'euc_jis_2004'`` codecs truncating null chars as they were treated as part +of multi-character sequences. + +.. + +.. date: 2025-09-24-14-17-34 +.. gh-issue: 139289 +.. nonce: Vmk25k +.. section: Library + +Do a real lazy-import on :mod:`rlcompleter` in :mod:`pdb` and restore the +existing completer after importing :mod:`rlcompleter`. + +.. + +.. date: 2025-09-23-09-46-46 +.. gh-issue: 139246 +.. nonce: pzfM-w +.. section: Library + +fix: paste zero-width in default repl width is wrong. + +.. + +.. date: 2025-09-22-14-40-11 +.. gh-issue: 90949 +.. nonce: UM35nb +.. section: Library + +Add :meth:`~xml.parsers.expat.xmlparser.SetAllocTrackerActivationThreshold` +and :meth:`~xml.parsers.expat.xmlparser.SetAllocTrackerMaximumAmplification` +to :ref:`xmlparser <xmlparser-objects>` objects to tune protections against +disproportional amounts of dynamic memory usage from within an Expat parser. +Patch by Bénédikt Tran. + +.. + +.. date: 2025-09-21-15-58-57 +.. gh-issue: 139210 +.. nonce: HGbMvz +.. section: Library + +Fix use-after-free when reporting unknown event in +:func:`xml.etree.ElementTree.iterparse`. Patch by Ken Jin. + +.. + +.. date: 2025-09-20-17-50-31 +.. gh-issue: 138860 +.. nonce: Y9JXap +.. section: Library + +Lazy import :mod:`rlcompleter` in :mod:`pdb` to avoid deadlock in +subprocess. + +.. + +.. date: 2025-09-19-09-36-42 +.. gh-issue: 112729 +.. nonce: mmty0_ +.. section: Library + +Fix crash when calling :func:`concurrent.interpreters.create` when the +process is out of memory. + +.. + +.. date: 2025-09-18-05-32-18 +.. gh-issue: 135729 +.. nonce: 8AmMza +.. section: Library + +Fix unraisable exception during finalization when using +:mod:`concurrent.interpreters` in the REPL. + +.. + +.. date: 2025-09-17-21-54-53 +.. gh-issue: 139076 +.. nonce: 2eX9lG +.. section: Library + +Fix a bug in the :mod:`pydoc` module that was hiding functions in a Python +module if they were implemented in an extension module and the module did +not have ``__all__``. + +.. + +.. date: 2025-09-17-19-08-34 +.. gh-issue: 139065 +.. nonce: Hu8fM5 +.. section: Library + +Fix trailing space before a wrapped long word if the line length is exactly +*width* in :mod:`textwrap`. + +.. + +.. date: 2025-09-17-12-07-21 +.. gh-issue: 139001 +.. nonce: O6tseN +.. section: Library + +Fix race condition in :class:`pathlib.Path` on the internal ``_raw_paths`` +field. + +.. + +.. date: 2025-09-17-08-32-43 +.. gh-issue: 138813 +.. nonce: LHkHjX +.. section: Library + +:class:`!multiprocessing.BaseProcess` defaults ``kwargs`` to ``None`` +instead of a shared dictionary. + +.. + +.. date: 2025-09-16-16-46-58 +.. gh-issue: 138993 +.. nonce: -8s8_T +.. section: Library + +Dedent :data:`credits` text. + +.. + +.. date: 2025-09-15-21-03-11 +.. gh-issue: 138891 +.. nonce: oZFdtR +.. section: Library + +Fix ``SyntaxError`` when ``inspect.get_annotations(f, eval_str=True)`` is +called on a function annotated with a :pep:`646` ``star_expression`` + +.. + +.. date: 2025-09-15-19-29-12 +.. gh-issue: 130567 +.. nonce: shDEnT +.. section: Library + +Fix possible crash in :func:`locale.strxfrm` due to a platform bug on macOS. + +.. + +.. date: 2025-09-13-12-19-17 +.. gh-issue: 138859 +.. nonce: PxjIoN +.. section: Library + +Fix generic type parameterization raising a :exc:`TypeError` when omitting a +:class:`ParamSpec` that has a default which is not a list of types. + +.. + +.. date: 2025-09-12-09-34-37 +.. gh-issue: 138764 +.. nonce: mokHoY +.. section: Library + +Prevent :func:`annotationlib.call_annotate_function` from calling +``__annotate__`` functions that don't support ``VALUE_WITH_FAKE_GLOBALS`` in +a fake globals namespace with empty globals. + +Make ``FORWARDREF`` and ``STRING`` annotations fall back to using ``VALUE`` +annotations in the case that neither their own format, nor +``VALUE_WITH_FAKE_GLOBALS`` are supported. + +.. + +.. date: 2025-09-11-15-03-37 +.. gh-issue: 138775 +.. nonce: w7rnSx +.. section: Library + +Use of ``python -m`` with :mod:`base64` has been fixed to detect input from +a terminal so that it properly notices EOF. + +.. + +.. date: 2025-09-11-11-09-28 +.. gh-issue: 138779 +.. nonce: TNZnLr +.. section: Library + +Support device numbers larger than ``2**63-1`` for the +:attr:`~os.stat_result.st_rdev` field of the :class:`os.stat_result` +structure. + +.. + +.. date: 2025-09-05-21-10-24 +.. gh-issue: 137706 +.. nonce: 0EztiJ +.. section: Library + +Fix the partial evaluation of annotations that use ``typing.Annotated[T, +x]`` where ``T`` is a forward reference. + +.. + +.. date: 2025-09-05-15-35-59 +.. gh-issue: 88375 +.. nonce: dC491a +.. section: Library + +Fix normalization of the ``robots.txt`` rules and URLs in the +:mod:`urllib.robotparser` module. No longer ignore trailing ``?``. +Distinguish raw special characters ``?``, ``=`` and ``&`` from the +percent-encoded ones. + +.. + +.. date: 2025-09-04-15-18-11 +.. gh-issue: 111788 +.. nonce: tuTEM5 +.. section: Library + +Fix parsing errors in the :mod:`urllib.robotparser` module. Don't fail +trying to parse weird paths. Don't fail trying to decode non-UTF-8 +``robots.txt`` files. + +.. + +.. date: 2025-09-03-20-18-39 +.. gh-issue: 98896 +.. nonce: tjez89 +.. section: Library + +Fix a failure in multiprocessing resource_tracker when SharedMemory names +contain colons. Patch by Rani Pinchuk. + +.. + +.. date: 2025-09-03-18-26-07 +.. gh-issue: 138425 +.. nonce: cVE9Ho +.. section: Library + +Fix partial evaluation of :class:`annotationlib.ForwardRef` objects which +rely on names defined as globals. + +.. + +.. date: 2025-09-03-15-20-10 +.. gh-issue: 138432 +.. nonce: RMc7UX +.. section: Library + +:meth:`zoneinfo.reset_tzpath` will now convert any :class:`os.PathLike` +objects it receives into strings before adding them to ``TZPATH``. It will +raise ``TypeError`` if anything other than a string is found after this +conversion. If given an :class:`os.PathLike` object that represents a +relative path, it will now raise ``ValueError`` instead of ``TypeError``, +and present a more informative error message. + +.. + +.. date: 2025-08-31-09-06-49 +.. gh-issue: 138008 +.. nonce: heOvsU +.. section: Library + +Fix segmentation faults in the :mod:`ctypes` module due to invalid +:attr:`~ctypes._CFuncPtr.argtypes`. Patch by Dung Nguyen. + +.. + +.. date: 2025-08-30-10-04-28 +.. gh-issue: 60462 +.. nonce: yh_vDc +.. section: Library + +Fix :func:`locale.strxfrm` on Solaris (and possibly other platforms). + +.. + +.. date: 2025-08-29-12-56-55 +.. gh-issue: 138239 +.. nonce: uthZFI +.. section: Library + +The REPL now highlights :keyword:`type` as a soft keyword in :ref:`type +statements <type>`. + +.. + +.. date: 2025-08-28-13-20-09 +.. gh-issue: 138204 +.. nonce: 8oLOud +.. section: Library + +Forbid expansion of shared anonymous :mod:`memory maps <mmap>` on Linux, +which caused a bus error. + +.. + +.. date: 2025-08-27-17-05-36 +.. gh-issue: 138010 +.. nonce: ZZJmPL +.. section: Library + +Fix an issue where defining a class with an :func:`@warnings.deprecated +<warnings.deprecated>`-decorated base class may not invoke the correct +:meth:`~object.__init_subclass__` method in cases involving multiple +inheritance. Patch by Brian Schubert. + +.. + +.. date: 2025-08-26-08-17-56 +.. gh-issue: 138151 +.. nonce: I6CdAk +.. section: Library + +In :mod:`annotationlib`, improve evaluation of forward references to +nonlocal variables that are not yet defined when the annotations are +initially evaluated. + +.. + +.. date: 2025-08-16-16-04-15 +.. gh-issue: 137317 +.. nonce: Dl13B5 +.. section: Library + +:func:`inspect.signature` now correctly handles classes that use a +descriptor on a wrapped :meth:`!__init__` or :meth:`!__new__` method. +Contributed by Yongyu Yan. + +.. + +.. date: 2025-08-16-09-02-11 +.. gh-issue: 137754 +.. nonce: mCev1Y +.. section: Library + +Fix import of the :mod:`zoneinfo` module if the C implementation of the +:mod:`datetime` module is not available. + +.. + +.. date: 2025-08-07-17-18-57 +.. gh-issue: 137490 +.. nonce: s89ieZ +.. section: Library + +Handle :data:`~errno.ECANCELED` in the same way as :data:`~errno.EINTR` in +:func:`signal.sigwaitinfo` on NetBSD. + +.. + +.. date: 2025-08-06-23-16-42 +.. gh-issue: 137477 +.. nonce: bk6BDV +.. section: Library + +Fix :func:`!inspect.getblock`, :func:`inspect.getsourcelines` and +:func:`inspect.getsource` for generator expressions. + +.. + +.. date: 2025-08-03-13-16-39 +.. gh-issue: 137044 +.. nonce: 0hPVL_ +.. section: Library + +Return large limit values as positive integers instead of negative integers +in :func:`resource.getrlimit`. Accept large values and reject negative +values (except :data:`~resource.RLIM_INFINITY`) for limits in +:func:`resource.setrlimit`. + +.. + +.. date: 2025-08-01-23-52-49 +.. gh-issue: 75989 +.. nonce: 5aYXNJ +.. section: Library + +:func:`tarfile.TarFile.extractall` and :func:`tarfile.TarFile.extract` now +overwrite symlinks when extracting hardlinks. (Contributed by Alexander +Enrique Urieles Nieto in :gh:`75989`.) + +.. + +.. date: 2025-08-01-23-11-25 +.. gh-issue: 137017 +.. nonce: 0yGcNc +.. section: Library + +Fix :obj:`threading.Thread.is_alive` to remain ``True`` until the underlying +OS thread is fully cleaned up. This avoids false negatives in edge cases +involving thread monitoring or premature :obj:`threading.Thread.is_alive` +calls. + +.. + +.. date: 2025-08-01-15-07-59 +.. gh-issue: 137273 +.. nonce: 4V8Xmv +.. section: Library + +Fix debug assertion failure in :func:`locale.setlocale` on Windows. + +.. + +.. date: 2025-07-30-17-42-36 +.. gh-issue: 137239 +.. nonce: qSpj32 +.. section: Library + +:mod:`heapq`: Update :data:`!heapq.__all__` with ``*_max`` functions. + +.. + +.. date: 2025-07-28-23-11-29 +.. gh-issue: 81325 +.. nonce: jMJFBe +.. section: Library + +:class:`tarfile.TarFile` now accepts a :term:`path-like <path-like object>` +when working on a tar archive. (Contributed by Alexander Enrique Urieles +Nieto in :gh:`81325`.) + +.. + +.. date: 2025-07-28-20-48-32 +.. gh-issue: 137185 +.. nonce: fgI7-B +.. section: Library + +Fix a potential async-signal-safety issue in :mod:`faulthandler` when +printing C stack traces. + +.. + +.. date: 2025-07-21-15-40-00 +.. gh-issue: 136914 +.. nonce: -GNG-d +.. section: Library + +Fix retrieval of :attr:`doctest.DocTest.lineno` for objects decorated with +:func:`functools.cache` or :class:`functools.cached_property`. + +.. + +.. date: 2025-07-21-11-56-47 +.. gh-issue: 136912 +.. nonce: zWosAL +.. section: Library + +:func:`hmac.digest` now properly handles large keys and messages by falling +back to the pure Python implementation when necessary. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-07-21-01-16-32 +.. gh-issue: 83424 +.. nonce: Y3tEV4 +.. section: Library + +Allows creating a :class:`ctypes.CDLL` without name when passing a handle as +an argument. + +.. + +.. date: 2025-07-17-16-12-23 +.. gh-issue: 136234 +.. nonce: VmTxtj +.. section: Library + +Fix :meth:`asyncio.WriteTransport.writelines` to be robust to connection +failure, by using the same behavior as +:meth:`~asyncio.WriteTransport.write`. + +.. + +.. date: 2025-07-10-21-02-43 +.. gh-issue: 136507 +.. nonce: pnEuGS +.. section: Library + +Fix mimetypes CLI to handle multiple file parameters. + +.. + +.. date: 2025-07-01-04-57-57 +.. gh-issue: 136057 +.. nonce: 4-t596 +.. section: Library + +Fixed the bug in :mod:`pdb` and :mod:`bdb` where ``next`` and ``step`` can't +go over the line if a loop exists in the line. + +.. + +.. date: 2025-06-16-15-00-13 +.. gh-issue: 135386 +.. nonce: lNrxLc +.. section: Library + +Fix opening a :mod:`dbm.sqlite3` database for reading from read-only file or +directory. + +.. + +.. date: 2025-06-16-12-37-02 +.. gh-issue: 135444 +.. nonce: An2eeA +.. section: Library + +Fix :meth:`asyncio.DatagramTransport.sendto` to account for datagram header +size when data cannot be sent. + +.. + +.. date: 2025-06-10-21-00-48 +.. gh-issue: 126631 +.. nonce: eITVJd +.. section: Library + +Fix :mod:`multiprocessing` ``forkserver`` bug which prevented ``__main__`` +from being preloaded. + +.. + +.. date: 2025-06-10-18-02-29 +.. gh-issue: 135307 +.. nonce: fXGrcK +.. section: Library + +:mod:`email`: Fix exception in ``set_content()`` when encoding text and +max_line_length is set to ``0`` or ``None`` (unlimited). + +.. + +.. date: 2025-05-30-18-37-44 +.. gh-issue: 134453 +.. nonce: kxkA-o +.. section: Library + +Fixed :func:`subprocess.Popen.communicate` ``input=`` handling of +:class:`memoryview` instances that were non-byte shaped on POSIX platforms. +Those are now properly cast to a byte shaped view instead of truncating the +input. Windows platforms did not have this bug. + +.. + +.. date: 2025-05-26-10-52-27 +.. gh-issue: 134698 +.. nonce: aJ1mZ1 +.. section: Library + +Fix a crash when calling methods of :class:`ssl.SSLContext` or +:class:`ssl.SSLSocket` across multiple threads. + +.. + +.. date: 2025-05-10-17-42-03 +.. gh-issue: 125996 +.. nonce: vaQp0- +.. section: Library + +Fix thread safety of :class:`collections.OrderedDict`. Patch by Kumar +Aditya. + +.. + +.. date: 2025-05-10-15-10-54 +.. gh-issue: 133789 +.. nonce: I-ZlUX +.. section: Library + +Fix unpickling of :mod:`pathlib` objects that were pickled in Python 3.13. + +.. + +.. date: 2025-04-21-01-05-14 +.. gh-issue: 127081 +.. nonce: Egrpq7 +.. section: Library + +Fix libc thread safety issues with :mod:`dbm` by performing stateful +operations in critical sections. + +.. + +.. date: 2025-04-16-21-02-57 +.. gh-issue: 132551 +.. nonce: Psa7pL +.. section: Library + +Make :class:`io.BytesIO` safe in :term:`free-threaded <free threading>` +build. + +.. + +.. date: 2025-03-27-08-13-32 +.. gh-issue: 131788 +.. nonce: 0RWiFc +.. section: Library + +Make ``ResourceTracker.send`` from :mod:`multiprocessing` re-entrant safe + +.. + +.. date: 2024-05-13-09-50-31 +.. gh-issue: 118981 +.. nonce: zgOQPv +.. section: Library + +Fix potential hang in ``multiprocessing.popen_spawn_posix`` that can happen +when the child proc dies early by closing the child fds right away. + +.. + +.. date: 2023-03-21-10-59-40 +.. gh-issue: 102431 +.. nonce: eUDnf4 +.. section: Library + +Clarify constraints for "logical" arguments in methods of +:class:`decimal.Context`. + +.. + +.. date: 2023-02-13-20-34-52 +.. gh-issue: 78319 +.. nonce: V1zzed +.. section: Library + +UTF8 support for the IMAP APPEND command has been made RFC compliant. + +.. + +.. bpo: 38735 +.. date: 2022-01-07-16-56-57 +.. nonce: NFfJX6 +.. section: Library + +Fix failure when importing a module from the root directory on unix-like +platforms with sys.pycache_prefix set. + +.. + +.. bpo: 41839 +.. date: 2020-09-23-11-54-17 +.. nonce: kU5Ywl +.. section: Library + +Allow negative priority values from :func:`os.sched_get_priority_min` and +:func:`os.sched_get_priority_max` functions. + +.. + +.. date: 2025-10-09-12-53-47 +.. gh-issue: 96491 +.. nonce: 4YKxvy +.. section: IDLE + +Deduplicate version number in IDLE shell title bar after saving to a file. + +.. + +.. date: 2025-10-08-08-35-50 +.. gh-issue: 139742 +.. nonce: B3fZLg +.. section: IDLE + +Colorize t-string prefixes for template strings in IDLE, as done for +f-string prefixes. + +.. + +.. date: 2025-11-26-23-30-09 +.. gh-issue: 141994 +.. nonce: arBEG6 +.. section: Documentation + +:mod:`xml.sax.handler`: Make Documentation of +:data:`xml.sax.handler.feature_external_ges` warn of opening up to `external +entity attacks <https://en.wikipedia.org/wiki/XML_external_entity_attack>`_. +Patch by Sebastian Pipping. + +.. + +.. date: 2025-10-27-23-06-01 +.. gh-issue: 140578 +.. nonce: FMBdEn +.. section: Documentation + +Remove outdated sencence in the documentation for :mod:`multiprocessing`, +that implied that :class:`concurrent.futures.ThreadPoolExecutor` did not +exist. + +.. + +.. date: 2025-12-01-20-41-26 +.. gh-issue: 142048 +.. nonce: c2YosX +.. section: Core and Builtins + +Fix quadratically increasing garbage collection delays in free-threaded +build. + +.. + +.. date: 2025-11-25-13-13-34 +.. gh-issue: 116738 +.. nonce: MnZRdV +.. section: Library + +Fix thread safety issue with :mod:`re` scanner objects in free-threaded +builds. + +.. + +.. date: 2025-11-24-21-09-30 +.. gh-issue: 141930 +.. nonce: hIIzSd +.. section: Core and Builtins + +When importing a module, use Python's regular file object to ensure that +writes to ``.pyc`` files are complete or an appropriate error is raised. + +.. + +.. date: 2025-11-22-10-43-26 +.. gh-issue: 120158 +.. nonce: 41_rXd +.. section: Core and Builtins + +Fix inconsistent state when enabling or disabling monitoring events too many +times. + +.. + +.. date: 2025-11-17-14-40-45 +.. gh-issue: 139653 +.. nonce: LzOy1M +.. section: Core and Builtins + +Only raise a ``RecursionError`` or trigger a fatal error if the stack +pointer is both below the limit pointer *and* above the stack base. If +outside of these bounds assume that it is OK. This prevents false positives +when user-space threads swap stacks. + +.. + +.. date: 2025-11-15-23-58-23 +.. gh-issue: 139103 +.. nonce: 9cVYJ0 +.. section: Core and Builtins + +Improve multithreaded scaling of dataclasses on the free-threaded build. + +.. + +.. date: 2025-11-15-01-21-00 +.. gh-issue: 141579 +.. nonce: aB7cD9 +.. section: Core and Builtins + +Fix :func:`sys.activate_stack_trampoline` to properly support the +``perf_jit`` backend. Patch by Pablo Galindo. + +.. + +.. date: 2025-11-14-16-25-15 +.. gh-issue: 114203 +.. nonce: n3tlQO +.. section: Core and Builtins + +Skip locking if object is already locked by two-mutex critical section. + +.. + +.. date: 2025-11-14-00-19-45 +.. gh-issue: 141528 +.. nonce: VWdax1 +.. section: Core and Builtins + +Suggest using :meth:`concurrent.interpreters.Interpreter.close` instead of +the private ``_interpreters.destroy`` function when warning about remaining +subinterpreters. Patch by Sergey Miryanov. + +.. + +.. date: 2025-11-10-23-07-06 +.. gh-issue: 141312 +.. nonce: H-58GB +.. section: Core and Builtins + +Fix the assertion failure in the ``__setstate__`` method of the range +iterator when a non-integer argument is passed. Patch by Sergey Miryanov. + +.. + +.. date: 2025-11-10-00-14-20 +.. gh-issue: 116738 +.. nonce: IxliC_ +.. section: Library + +Make csv module thread-safe on the :term:`free threaded <free threading>` +build. + +.. + +.. date: 2025-11-03-17-21-38 +.. gh-issue: 140939 +.. nonce: FVboAw +.. section: Core and Builtins + +Fix memory leak when :class:`bytearray` or :class:`bytes` is formated with +the ``%*b`` format with a large width that results in a :exc:`MemoryError`. + +.. + +.. date: 2025-11-02-15-28-33 +.. gh-issue: 140260 +.. nonce: JNzlGz +.. section: Library + +Fix :mod:`struct` data race in endian table initialization with +subinterpreters. Patch by Shamil Abdulaev. + +.. + +.. date: 2025-11-02-12-47-38 +.. gh-issue: 140530 +.. nonce: S934bp +.. section: Core and Builtins + +Fix a reference leak when ``raise exc from cause`` fails. Patch by Bénédikt +Tran. + +.. + +.. date: 2025-10-29-20-59-10 +.. gh-issue: 140373 +.. nonce: -uoaPP +.. section: Core and Builtins + +Correctly emit ``PY_UNWIND`` event when generator object is closed. Patch by +Mikhail Efimov. + +.. + +.. date: 2025-10-25-17-36-46 +.. gh-issue: 140576 +.. nonce: kj0SCY +.. section: Core and Builtins + +Fixed crash in :func:`tokenize.generate_tokens` in case of specific +incorrect input. Patch by Mikhail Efimov. + +.. + +.. date: 2025-10-24-20-42-33 +.. gh-issue: 140551 +.. nonce: -9swrl +.. section: Core and Builtins + +Fixed crash in :class:`dict` if :meth:`dict.clear` is called at the lookup +stage. Patch by Mikhail Efimov and Inada Naoki. + +.. + +.. date: 2025-10-24-20-16-42 +.. gh-issue: 140517 +.. nonce: cqun-K +.. section: Core and Builtins + +Fixed a reference leak when iterating over the result of :func:`map` with +``strict=True`` when the input iterables have different lengths. Patch by +Mikhail Efimov. + +.. + +.. date: 2025-10-23-16-05-50 +.. gh-issue: 140471 +.. nonce: Ax_aXn +.. section: Core and Builtins + +Fix potential buffer overflow in :class:`ast.AST` node initialization when +encountering malformed :attr:`~ast.AST._fields` containing non-:class:`str`. + +.. + +.. date: 2025-10-22-17-22-22 +.. gh-issue: 140431 +.. nonce: m8D_A- +.. section: Core and Builtins + +Fix a crash in Python's :term:`garbage collector <garbage collection>` due +to partially initialized :term:`coroutine` objects when coroutine origin +tracking depth is enabled (:func:`sys.set_coroutine_origin_tracking_depth`). + +.. + +.. date: 2025-10-21-09-20-03 +.. gh-issue: 140398 +.. nonce: SoABwJ +.. section: Library + +Fix memory leaks in :mod:`readline` functions +:func:`~readline.read_init_file`, :func:`~readline.read_history_file`, +:func:`~readline.write_history_file`, and +:func:`~readline.append_history_file` when :c:func:`PySys_Audit` fails. + +.. + +.. date: 2025-10-21-06-51-50 +.. gh-issue: 140406 +.. nonce: 0gJs8M +.. section: Core and Builtins + +Fix memory leak when an object's :meth:`~object.__hash__` method returns an +object that isn't an :class:`int`. + +.. + +.. date: 2025-10-20-11-24-36 +.. gh-issue: 140358 +.. nonce: UQuKdV +.. section: Core and Builtins + +Restore elapsed time and unreachable object count in GC debug output. These +were inadvertently removed during a refactor of ``gc.c``. The debug log now +again reports elapsed collection time and the number of unreachable objects. +Contributed by Pål Grønås Drange. + +.. + +.. date: 2025-10-18-21-29-45 +.. gh-issue: 140306 +.. nonce: xS5CcS +.. section: Core and Builtins + +Fix memory leaks in cross-interpreter channel operations and shared +namespace handling. + +.. + +.. date: 2025-10-18-18-08-36 +.. gh-issue: 140301 +.. nonce: m-2HxC +.. section: Core and Builtins + +Fix memory leak of ``PyConfig`` in subinterpreters. + +.. + +.. date: 2025-10-17-20-23-19 +.. gh-issue: 140257 +.. nonce: 8Txmem +.. section: Core and Builtins + +Fix data race between interpreter_clear() and take_gil() on eval_breaker +during finalization with daemon threads. + +.. + +.. date: 2025-10-17-18-03-12 +.. gh-issue: 139951 +.. nonce: IdwM2O +.. section: Core and Builtins + +Fixes a regression in GC performance for a growing heap composed mostly of +small tuples. + +* Counts number of actually tracked objects, instead of trackable objects. + This ensures that untracking tuples has the desired effect of reducing GC overhead. +* Does not track most untrackable tuples during creation. + This prevents large numbers of small tuples causing excessive GCs. + +.. + +.. date: 2025-10-16-21-47-00 +.. gh-issue: 140104 +.. nonce: A8SQIm +.. section: Core and Builtins + +Fix a bug with exception handling in the JIT. Patch by Ken Jin. Bug reported +by Daniel Diniz. + +.. + +.. date: 2025-10-15-00-21-40 +.. gh-issue: 140061 +.. nonce: J0XeDV +.. section: Core and Builtins + +Fixing the checking of whether an object is uniquely referenced to ensure +free-threaded compatibility. Patch by Sergey Miryanov. + +.. + +.. date: 2025-10-14-17-07-37 +.. gh-issue: 140067 +.. nonce: ID2gOm +.. section: Core and Builtins + +Fix memory leak in sub-interpreter creation. + +.. + +.. date: 2025-10-13-17-56-23 +.. gh-issue: 140000 +.. nonce: tLhn3e +.. section: Core and Builtins + +Fix potential memory leak when a reference cycle exists between an instance +of :class:`typing.TypeAliasType`, :class:`typing.TypeVar`, +:class:`typing.ParamSpec`, or :class:`typing.TypeVarTuple` and its +``__name__`` attribute. Patch by Mikhail Efimov. + +.. + +.. date: 2025-10-13-13-54-19 +.. gh-issue: 139914 +.. nonce: M-y_3E +.. section: Core and Builtins + +Restore support for HP PA-RISC, which has an upwards-growing stack. + +.. + +.. date: 2025-10-12-11-00-06 +.. gh-issue: 139988 +.. nonce: 4wi51t +.. section: Core and Builtins + +Fix a memory leak when failing to create a :class:`~typing.Union` type. +Patch by Bénédikt Tran. + +.. + +.. date: 2025-10-08-13-52-00 +.. gh-issue: 139748 +.. nonce: jq0yFJ +.. section: Core and Builtins + +Fix reference leaks in error branches of functions accepting path strings or +bytes such as :func:`compile` and :func:`os.system`. Patch by Bénédikt Tran. + +.. + +.. date: 2025-10-06-13-15-26 +.. gh-issue: 139516 +.. nonce: d9Pkur +.. section: Core and Builtins + +Fix lambda colon erroneously start format spec in f-string in tokenizer. + +.. + +.. date: 2025-10-06-10-03-37 +.. gh-issue: 139640 +.. nonce: gY5oTb2 +.. section: Core and Builtins + +:func:`ast.parse` no longer emits syntax warnings for +``return``/``break``/``continue`` in ``finally`` (see :pep:`765`) -- they +are only emitted during compilation. + +.. + +.. date: 2025-10-06-10-03-37 +.. gh-issue: 139640 +.. nonce: gY5oTb +.. section: Core and Builtins + +Fix swallowing some syntax warnings in different modules if they +accidentally have the same message and are emitted from the same line. Fix +duplicated warnings in the ``finally`` block. + +.. + +.. date: 2025-10-01-18-21-19 +.. gh-issue: 63161 +.. nonce: ef1S6N +.. section: Core and Builtins + +Support non-UTF-8 shebang and comments in Python source files if non-UTF-8 +encoding is specified. Detect decoding error in comments for default (UTF-8) +encoding. Show the line and position of decoding error for default encoding +in a traceback. Show the line containing the coding cookie when it conflicts +with the BOM in a traceback. + +.. + +.. date: 2025-09-21-14-33-17 +.. gh-issue: 116738 +.. nonce: vNaI4h +.. section: Library + +Make :mod:`mmap` thread-safe on the :term:`free threaded <free threading>` +build. + +.. + +.. date: 2025-09-17-17-17-21 +.. gh-issue: 138558 +.. nonce: 0VbzCH +.. section: Core and Builtins + +Fix handling of unusual t-string annotations in annotationlib. Patch by Dave +Peck. + +.. + +.. date: 2025-09-15-14-04-56 +.. gh-issue: 134466 +.. nonce: yR4fYW +.. section: Core and Builtins + +Don't run PyREPL in a degraded environment where setting termios attributes +is not allowed. + +.. + +.. date: 2025-09-15-13-06-11 +.. gh-issue: 138944 +.. nonce: PeCgLb +.. section: Core and Builtins + +Fix :exc:`SyntaxError` message when invalid syntax appears on the same line +as a valid ``import ... as ...`` or ``from ... import ... as ...`` +statement. Patch by Brian Schubert. + +.. + +.. date: 2025-09-06-13-53-33 +.. gh-issue: 105487 +.. nonce: a43YaY +.. section: Core and Builtins + +Remove non-existent :meth:`~object.__copy__`, :meth:`~object.__deepcopy__`, +and :attr:`~type.__bases__` from the :meth:`~object.__dir__` entries of +:class:`types.GenericAlias`. + +.. + +.. date: 2025-08-30-17-15-05 +.. gh-issue: 69605 +.. nonce: KjBk99 +.. section: Core and Builtins + +Fix some standard library submodules missing from the :term:`REPL` +auto-completion of imports. + +.. + +.. date: 2025-08-28-09-29-46 +.. gh-issue: 116738 +.. nonce: yLZJpV +.. section: Library + +Make :mod:`cProfile` thread-safe on the :term:`free threaded <free +threading>` build. + +.. + +.. date: 2025-08-21-06-31-42 +.. gh-issue: 138004 +.. nonce: FH2Hre +.. section: Library + +On Solaris/Illumos platforms, thread names are now encoded as ASCII to avoid +errors on systems (e.g. OpenIndiana) that don't support non-ASCII names. + +.. + +.. date: 2025-08-13-13-39-02 +.. gh-issue: 137433 +.. nonce: g6Atfz +.. section: Core and Builtins + +Fix a potential deadlock in the :term:`free threading` build when daemon +threads enable or disable profiling or tracing while the main thread is +shutting down the interpreter. + +.. + +.. date: 2025-08-07-09-52-19 +.. gh-issue: 137400 +.. nonce: AK1dy- +.. section: Core and Builtins + +Fix a crash in the :term:`free threading` build when disabling profiling or +tracing across all threads with :c:func:`PyEval_SetProfileAllThreads` or +:c:func:`PyEval_SetTraceAllThreads` or their Python equivalents +:func:`threading.settrace_all_threads` and +:func:`threading.setprofile_all_threads`. + +.. + +.. date: 2025-08-05-17-22-24 +.. gh-issue: 58124 +.. nonce: q1__53 +.. section: Core and Builtins + +Fix name of the Python encoding in Unicode errors of the code page codec: +use "cp65000" and "cp65001" instead of "CP_UTF7" and "CP_UTF8" which are not +valid Python code names. Patch by Victor Stinner. + +.. + +.. date: 2025-07-09-21-27-14 +.. gh-issue: 132657 +.. nonce: kSA8R3 +.. section: Core and Builtins + +Improve performance of :class:`frozenset` by removing locks in the +free-threading build. + +.. + +.. date: 2025-05-11-09-40-19 +.. gh-issue: 133400 +.. nonce: zkWla8 +.. section: Core and Builtins + +Fixed Ctrl+D (^D) behavior in _pyrepl module to match old pre-3.13 REPL +behavior. + +.. + +.. date: 2025-01-08-12-52-47 +.. gh-issue: 128640 +.. nonce: 9nbh9z +.. section: Core and Builtins + +Fix a crash when using threads inside of a subinterpreter. + +.. + +.. date: 2025-11-21-10-34-00 +.. gh-issue: 137422 +.. nonce: tzZKLi +.. section: C API + +Fix :term:`free threading` race condition in +:c:func:`PyImport_AddModuleRef`. It was previously possible for two calls to +the function return two different objects, only one of which was stored in +:data:`sys.modules`. + +.. + +.. date: 2025-11-18-04-16-09 +.. gh-issue: 140042 +.. nonce: S1C7id +.. section: C API + +Removed the sqlite3_shutdown call that could cause closing connections for +sqlite when used with multiple sub interpreters. + +.. + +.. date: 2025-11-06-06-28-14 +.. gh-issue: 141042 +.. nonce: brOioJ +.. section: C API + +Make qNaN in :c:func:`PyFloat_Pack2` and :c:func:`PyFloat_Pack4`, if while +conversion to a narrower precision floating-point format --- the remaining +after truncation payload will be zero. Patch by Sergey B Kirpichev. + +.. + +.. date: 2025-10-26-16-45-06 +.. gh-issue: 140487 +.. nonce: fGOqss +.. section: C API + +Fix :c:macro:`Py_RETURN_NOTIMPLEMENTED` in limited C API 3.11 and older: +don't treat ``Py_NotImplemented`` as immortal. Patch by Victor Stinner. + +.. + +.. date: 2025-10-15-15-59-59 +.. gh-issue: 140153 +.. nonce: BO7sH4 +.. section: C API + +Fix :c:func:`Py_REFCNT` definition on limited C API 3.11-3.13. Patch by +Victor Stinner. + +.. + +.. date: 2025-10-06-22-17-47 +.. gh-issue: 139653 +.. nonce: 6-1MOd +.. section: C API + +Add :c:func:`PyUnstable_ThreadState_SetStackProtection` and +:c:func:`PyUnstable_ThreadState_ResetStackProtection` functions to set the +stack protection base address and stack protection size of a Python thread +state. Patch by Victor Stinner. + +.. + +.. date: 2025-11-28-19-49-01 +.. gh-issue: 141808 +.. nonce: cV5K12 +.. section: Build + +Do not generate the jit stencils twice in case of PGO builds on Windows. + +.. + +.. date: 2025-11-20-17-01-05 +.. gh-issue: 141784 +.. nonce: LkYI2n +.. section: Build + +Fix ``_remote_debugging_module.c`` compilation on 32-bit Linux. Include +Python.h before system headers to make sure that +``_remote_debugging_module.c`` uses the same types (ABI) than Python. Patch +by Victor Stinner. + +.. + +.. date: 2025-10-29-12-30-38 +.. gh-issue: 140768 +.. nonce: ITYrzw +.. section: Build + +Warn when the WASI SDK version doesn't match what's supported. + +.. + +.. date: 2025-10-25-08-07-06 +.. gh-issue: 140513 +.. nonce: 6OhLTs +.. section: Build + +Generate a clear compilation error when ``_Py_TAIL_CALL_INTERP`` is enabled +but either ``preserve_none`` or ``musttail`` is not supported. + +.. + +.. date: 2025-10-16-11-30-53 +.. gh-issue: 140189 +.. nonce: YCrUyt +.. section: Build + +iOS builds were added to CI. + +.. + +.. date: 2025-09-24-13-59-26 +.. gh-issue: 138489 +.. nonce: 1AcuZM +.. section: Build + +When cross-compiling for WASI by ``build_wasm`` or ``build_emscripten``, the +``build-details.json`` step is now included in the build process, just like +with native builds. + +This fixes the ``libinstall`` task which requires the ``build-details.json`` +file during the process. + +.. + +.. date: 2025-08-10-22-28-06 +.. gh-issue: 137618 +.. nonce: FdNvIE +.. section: Build + +``PYTHON_FOR_REGEN`` now requires Python 3.10 to Python 3.15. Patch by Adam +Turner. + +.. + +.. date: 2025-01-03-13-02-06 +.. gh-issue: 123681 +.. nonce: gQ67nK +.. section: Build + +Check the ``strftime()`` behavior at runtime instead of at the compile time +to support cross-compiling. Remove the internal macro +``_Py_NORMALIZE_CENTURY``. diff --git a/Misc/NEWS.d/3.14.2.rst b/Misc/NEWS.d/3.14.2.rst new file mode 100644 index 000000000000000..fba5fb96b3d614f --- /dev/null +++ b/Misc/NEWS.d/3.14.2.rst @@ -0,0 +1,85 @@ +.. date: 2025-12-01-09-36-45 +.. gh-issue: 142145 +.. nonce: tcAUhg +.. release date: 2025-12-05 +.. section: Security + +Remove quadratic behavior in ``xml.minidom`` node ID cache clearing. + +.. + +.. date: 2024-05-23-11-44-41 +.. gh-issue: 119452 +.. nonce: PRfsSv +.. section: Security + +Fix a potential memory denial of service in the :mod:`http.server` module. +When a malicious user is connected to the CGI server on Windows, it could +cause an arbitrary amount of memory to be allocated. This could have led to +symptoms including a :exc:`MemoryError`, swapping, out of memory (OOM) +killed processes or containers, or even system crashes. + +.. + +.. date: 2025-12-05-17-58-29 +.. gh-issue: 140797 +.. nonce: YxB27u +.. section: Library + +Revert changes to the undocumented :class:`!re.Scanner` class. Capturing +groups are still allowed for backward compatibility, although using them can +lead to incorrect result. They will be forbidden in future Python versions. + +.. + +.. date: 2025-12-03-09-36-29 +.. gh-issue: 142206 +.. nonce: ilwegH +.. section: Library + +The resource tracker in the :mod:`multiprocessing` module now uses the +original communication protocol, as in Python 3.14.0 and below, by default. +This avoids issues with upgrading Python while it is running. (Note that +such 'in-place' upgrades are not tested.) The tracker remains compatible +with subprocesses that use new protocol (that is, subprocesses using Python +3.13.10, 3.14.1 and 3.15). + +.. + +.. date: 2025-12-03-06-12-39 +.. gh-issue: 142214 +.. nonce: appYNZ +.. section: Library + +Fix two regressions in :mod:`dataclasses` in Python 3.14.1 related to +annotations. + +* An exception is no longer raised if ``slots=True`` is used and the + ``__init__`` method does not have an ``__annotate__`` attribute + (likely because ``init=False`` was used). + +* An exception is no longer raised if annotations are requested on the + ``__init__`` method and one of the fields is not present in the class + annotations. This can occur in certain dynamic scenarios. + +Patch by Jelle Zijlstra. + +.. + +.. date: 2025-12-03-11-03-35 +.. gh-issue: 142218 +.. nonce: 44Fq_J +.. section: Core and Builtins + +Fix crash when inserting into a split table dictionary with a non +:class:`str` key that matches an existing key. + +.. + +.. date: 2025-12-01-10-03-08 +.. gh-issue: 116738 +.. nonce: 972YsG +.. section: Library + +Fix :mod:`cmath` data race when initializing trigonometric tables with +subinterpreters. diff --git a/Misc/NEWS.d/3.14.3.rst b/Misc/NEWS.d/3.14.3.rst new file mode 100644 index 000000000000000..93985abb772092e --- /dev/null +++ b/Misc/NEWS.d/3.14.3.rst @@ -0,0 +1,1238 @@ +.. date: 2025-09-14-13-35-44 +.. gh-issue: 128067 +.. nonce: BGdP_A +.. release date: 2026-02-03 +.. section: Windows + +Fix a bug in PyREPL on Windows where output without a trailing newline was +overwritten by the next prompt. + +.. + +.. date: 2026-01-02-11-44-56 +.. gh-issue: 142095 +.. nonce: 4ssgnM +.. section: Tools/Demos + +Make gdb 'py-bt' command use frame from thread local state when available. +Patch by Sam Gross and Victor Stinner. + +.. + +.. date: 2026-02-03-07-57-24 +.. gh-issue: 144415 +.. nonce: U3L15r +.. section: Tests + +The Android testbed now distinguishes between stdout/stderr messages which +were triggered by a newline, and those triggered by a manual call to +``flush``. This fixes logging of progress indicators and similar content. + +.. + +.. date: 2026-01-09-13-52-10 +.. gh-issue: 143460 +.. nonce: _nW2jt +.. section: Tests + +Skip tests relying on infinite recusion if stack size is unlimited. + +.. + +.. date: 2026-01-08-16-56-59 +.. gh-issue: 65784 +.. nonce: aKNo1U +.. section: Tests + +Add support for parametrized resource ``wantobjects`` in regrtests, which +allows to run Tkinter tests with the specified value of +:data:`!tkinter.wantobjects`, for example ``-u wantobjects=0``. + +.. + +.. date: 2026-01-08-11-50-06 +.. gh-issue: 143553 +.. nonce: KyyNTt +.. section: Tests + +Add support for parametrized resources, such as ``-u xpickle=2.7``. + +.. + +.. date: 2025-12-17-02-02-57 +.. gh-issue: 142836 +.. nonce: mR-fvK +.. section: Tests + +Accommodated Solaris in ``test_pdb.test_script_target_anonymous_pipe``. + +.. + +.. bpo: 31391 +.. date: 2020-09-29-23-14-01 +.. nonce: IZr2P8 +.. section: Tests + +Forward-port test_xpickle from Python 2 to Python 3 and add the resource +back to test's command line. + +.. + +.. date: 2026-01-21-12-34-05 +.. gh-issue: 144125 +.. nonce: TAz5uo +.. section: Security + +:mod:`~email.generator.BytesGenerator` will now refuse to serialize (write) +headers that are unsafely folded or delimited; see +:attr:`~email.policy.Policy.verify_generated_headers`. (Contributed by Bas +Bloemsaat and Petr Viktorin in :gh:`121650`). + +.. + +.. date: 2026-01-16-14-40-31 +.. gh-issue: 143935 +.. nonce: U2YtKl +.. section: Security + +Fixed a bug in the folding of comments when flattening an email message +using a modern email policy. Comments consisting of a very long sequence of +non-foldable characters could trigger a forced line wrap that omitted the +required leading space on the continuation line, causing the remainder of +the comment to be interpreted as a new header field. This enabled header +injection with carefully crafted inputs. + +.. + +.. date: 2026-01-16-11-51-19 +.. gh-issue: 143925 +.. nonce: mrtcHW +.. section: Security + +Reject control characters in ``data:`` URL media types. + +.. + +.. date: 2026-01-16-11-13-15 +.. gh-issue: 143919 +.. nonce: kchwZV +.. section: Security + +Reject control characters in :class:`http.cookies.Morsel` fields and values. + +.. + +.. date: 2026-01-16-11-07-36 +.. gh-issue: 143916 +.. nonce: dpWeOD +.. section: Security + +Reject C0 control characters within wsgiref.headers.Headers fields, values, +and parameters. + +.. + +.. date: 2026-02-01-15-25-00 +.. gh-issue: 144380 +.. nonce: U7py_s +.. section: Library + +Improve performance of :class:`io.BufferedReader` line iteration by ~49%. + +.. + +.. date: 2026-01-23-06-43-21 +.. gh-issue: 144169 +.. nonce: LFy9yi +.. section: Library + +Fix three crashes when non-string keyword arguments are supplied to objects +in the :mod:`ast` module. + +.. + +.. date: 2026-01-21-19-39-07 +.. gh-issue: 144100 +.. nonce: hLMZ8Y +.. section: Library + +Fixed a crash in ctypes when using a deprecated ``POINTER(str)`` type in +``argtypes``. Instead of aborting, ctypes now raises a proper Python +exception when the pointer target type is unresolved. + +.. + +.. date: 2026-01-20-16-35-55 +.. gh-issue: 144050 +.. nonce: 0kKFbF +.. section: Library + +Fix :func:`stat.filemode` in the pure-Python implementation to avoid +misclassifying invalid mode values as block devices. + +.. + +.. date: 2026-01-19-00-57-40 +.. gh-issue: 144023 +.. nonce: 29XUcp +.. section: Library + +Fixed validation of file descriptor 0 in posix functions when used with +follow_symlinks parameter. + +.. + +.. date: 2026-01-18-14-35-37 +.. gh-issue: 143999 +.. nonce: MneN4O +.. section: Library + +Fix an issue where :func:`inspect.getgeneratorstate` and +:func:`inspect.getcoroutinestate` could fail for generators wrapped by +:func:`types.coroutine` in the suspended state. + +.. + +.. date: 2026-01-16-06-22-10 +.. gh-issue: 143831 +.. nonce: VLBTLp +.. section: Library + +:class:`annotationlib.ForwardRef` objects are now hashable when created from +annotation scopes with closures. Previously, hashing such objects would +throw an exception. Patch by Bartosz Sławecki. + +.. + +.. date: 2026-01-15-16-04-39 +.. gh-issue: 143874 +.. nonce: 1qQgvo +.. section: Library + +Fixed a bug in :mod:`pdb` where expression results were not sent back to +remote client. + +.. + +.. date: 2026-01-15-13-03-22 +.. gh-issue: 143880 +.. nonce: sWoLsf +.. section: Library + +Fix data race in :func:`functools.partial` in the :term:`free threading` +build. + +.. + +.. date: 2026-01-12-07-17-38 +.. gh-issue: 143706 +.. nonce: sysArgv +.. section: Library + +Fix :mod:`multiprocessing` forkserver so that :data:`sys.argv` is correctly +set before ``__main__`` is preloaded. Previously, :data:`sys.argv` was empty +during main module import in forkserver child processes. This fixes a +regression introduced in 3.13.8 and 3.14.1. Root caused by Aaron Wieczorek, +test provided by Thomas Watson, thanks! + +.. + +.. date: 2026-01-10-16-42-47 +.. gh-issue: 143638 +.. nonce: du5G7d +.. section: Library + +Forbid reentrant calls of the :class:`pickle.Pickler` and +:class:`pickle.Unpickler` methods for the C implementation. Previously, this +could cause crash or data corruption, now concurrent calls of methods of the +same object raise :exc:`RuntimeError`. + +.. + +.. date: 2026-01-10-10-04-08 +.. gh-issue: 78724 +.. nonce: xkXfxX +.. section: Library + +Raise :exc:`RuntimeError`'s when user attempts to call methods on +half-initialized :class:`~struct.Struct` objects, For example, created by +``Struct.__new__(Struct)``. Patch by Sergey B Kirpichev. + +.. + +.. date: 2026-01-09-17-50-26 +.. gh-issue: 143196 +.. nonce: WxKxzU +.. section: Library + +Fix crash when the internal encoder object returned by undocumented function +:func:`!json.encoder.c_make_encoder` was called with non-zero second +(*_current_indent_level*) argument. + +.. + +.. date: 2026-01-09-13-07-22 +.. gh-issue: 143191 +.. nonce: PPR_vW +.. section: Library + +:func:`_thread.stack_size` now raises :exc:`ValueError` if the stack size is +too small. Patch by Victor Stinner. + +.. + +.. date: 2026-01-09-12-37-19 +.. gh-issue: 143602 +.. nonce: V8vQpj +.. section: Library + +Fix a inconsistency issue in :meth:`~io.RawIOBase.write` that leads to +unexpected buffer overwrite by deduplicating the buffer exports. + +.. + +.. date: 2026-01-08-14-53-46 +.. gh-issue: 143547 +.. nonce: wHBVlr +.. section: Library + +Fix :func:`sys.unraisablehook` when the hook raises an exception and changes +:func:`sys.unraisablehook`: hold a strong reference to the old hook. Patch +by Victor Stinner. + +.. + +.. date: 2026-01-07-15-49-06 +.. gh-issue: 143517 +.. nonce: FP5KgL +.. section: Library + +:func:`annotationlib.get_annotations` no longer raises a :exc:`SyntaxError` +when evaluating a stringified starred annotation that starts with one or +more whitespace characters followed by a ``*``. Patch by Bartosz Sławecki. + +.. + +.. date: 2026-01-03-19-41-36 +.. gh-issue: 143378 +.. nonce: 29AvE7 +.. section: Library + +Fix use-after-free crashes when a :class:`~io.BytesIO` object is +concurrently mutated during :meth:`~io.RawIOBase.write` or +:meth:`~io.IOBase.writelines`. + +.. + +.. date: 2026-01-02-12-55-52 +.. gh-issue: 143346 +.. nonce: iTekce +.. section: Library + +Fix incorrect wrapping of the Base64 data in :class:`!plistlib._PlistWriter` +when the indent contains a mix of tabs and spaces. + +.. + +.. date: 2026-01-01-11-21-57 +.. gh-issue: 143310 +.. nonce: 8rxtH3 +.. section: Library + +:mod:`tkinter`: fix a crash when a Python :class:`list` is mutated during +the conversion to a Tcl object (e.g., when setting a Tcl variable). Patch by +Bénédikt Tran. + +.. + +.. date: 2025-12-31-20-43-02 +.. gh-issue: 143309 +.. nonce: cdFxdH +.. section: Library + +Fix a crash in :func:`os.execve` on non-Windows platforms when given a +custom environment mapping which is then mutated during parsing. Patch by +Bénédikt Tran. + +.. + +.. date: 2025-12-31-17-38-33 +.. gh-issue: 143308 +.. nonce: lY8UCR +.. section: Library + +:mod:`pickle`: fix use-after-free crashes when a +:class:`~pickle.PickleBuffer` is concurrently mutated by a custom buffer +callback during pickling. Patch by Bénédikt Tran and Aaron Wieczorek. + +.. + +.. date: 2025-12-28-20-28-05 +.. gh-issue: 143237 +.. nonce: q1ymuA +.. section: Library + +Fix support of named pipes in the rotating :mod:`logging` handlers. + +.. + +.. date: 2025-12-28-14-41-02 +.. gh-issue: 143249 +.. nonce: K4vEp4 +.. section: Library + +Fix possible buffer leaks in Windows overlapped I/O on error handling. + +.. + +.. date: 2025-12-28-13-49-06 +.. gh-issue: 143241 +.. nonce: 5H4b8d +.. section: Library + +:mod:`zoneinfo`: fix infinite loop in :meth:`ZoneInfo.from_file +<zoneinfo.ZoneInfo.from_file>` when parsing a malformed TZif file. Patch by +Fatih Celik. + +.. + +.. date: 2025-12-28-13-12-40 +.. gh-issue: 142830 +.. nonce: uEyd6r +.. section: Library + +:mod:`sqlite3`: fix use-after-free crashes when the connection's callbacks +are mutated during a callback execution. Patch by Bénédikt Tran. + +.. + +.. date: 2025-12-27-15-41-27 +.. gh-issue: 143200 +.. nonce: 2hEUAl +.. section: Library + +:mod:`xml.etree.ElementTree`: fix use-after-free crashes in +:meth:`~object.__getitem__` and :meth:`~object.__setitem__` methods of +:class:`~xml.etree.ElementTree.Element` when the element is concurrently +mutated. Patch by Bénédikt Tran. + +.. + +.. date: 2025-12-27-00-14-56 +.. gh-issue: 142195 +.. nonce: UgBEo5 +.. section: Library + +Updated timeout evaluation logic in :mod:`subprocess` to be compatible with +deterministic environments like Shadow where time moves exactly as +requested. + +.. + +.. date: 2025-12-25-08-58-55 +.. gh-issue: 142164 +.. nonce: XrFztf +.. section: Library + +Fix the ctypes bitfield overflow error message to report the correct offset +and size calculation. + +.. + +.. date: 2025-12-24-14-18-52 +.. gh-issue: 143145 +.. nonce: eXLw8D +.. section: Library + +Fixed a possible reference leak in ctypes when constructing results with +multiple output parameters on error. + +.. + +.. date: 2025-12-22-22-36-21 +.. gh-issue: 122431 +.. nonce: 9E3085 +.. section: Library + +Corrected the error message in :func:`readline.append_history_file` to state +that ``nelements`` must be non-negative instead of positive. + +.. + +.. date: 2025-12-22-00-00-00 +.. gh-issue: 143004 +.. nonce: uaf-counter +.. section: Library + +Fix a potential use-after-free in :meth:`collections.Counter.update` when +user code mutates the Counter during an update. + +.. + +.. date: 2025-12-21-17-44-28 +.. gh-issue: 143046 +.. nonce: GBa5Ip +.. section: Library + +The :mod:`asyncio` REPL no longer prints copyright and version messages in +the quiet mode (:option:`-q`). Patch by Bartosz Sławecki. + +.. + +.. date: 2025-12-21-17-24-29 +.. gh-issue: 140648 +.. nonce: i8dca6 +.. section: Library + +The :mod:`asyncio` REPL now respects the :option:`-I` flag (isolated mode). +Previously, it would load and execute :envvar:`PYTHONSTARTUP` even if the +flag was set. Contributed by Bartosz Sławecki. + +.. + +.. date: 2025-12-20-10-21-23 +.. gh-issue: 142991 +.. nonce: jYHD9E +.. section: Library + +Fixed socket operations such as recvfrom() and sendto() for FreeBSD +divert(4) socket. + +.. + +.. date: 2025-12-20-01-49-02 +.. gh-issue: 143010 +.. nonce: _-SWX0 +.. section: Library + +Fixed a bug in :mod:`mailbox` where the precise timing of an external event +could result in the library opening an existing file instead of a file it +expected to create. + +.. + +.. date: 2025-12-17-20-18-17 +.. gh-issue: 142881 +.. nonce: 5IizIQ +.. section: Library + +Fix concurrent and reentrant call of :func:`atexit.unregister`. + +.. + +.. date: 2025-12-17-14-41-09 +.. gh-issue: 112127 +.. nonce: 13OHQk +.. section: Library + +Fix possible use-after-free in :func:`atexit.unregister` when the callback +is unregistered during comparison. + +.. + +.. date: 2025-12-16-14-49-19 +.. gh-issue: 142783 +.. nonce: VPV1ig +.. section: Library + +Fix zoneinfo use-after-free with descriptor _weak_cache. a descriptor as +_weak_cache could cause crashes during object creation. The fix ensures +proper reference counting for descriptor-provided objects. + +.. + +.. date: 2025-12-16-11-55-55 +.. gh-issue: 142754 +.. nonce: xuCrt3 +.. section: Library + +Add the *ownerDocument* attribute to :mod:`xml.dom.minidom` elements and +attributes created by directly instantiating the ``Element`` or ``Attr`` +class. Note that this way of creating nodes is not supported; creator +functions like :py:meth:`xml.dom.Document.documentElement` should be used +instead. + +.. + +.. date: 2025-12-16-04-39-27 +.. gh-issue: 142784 +.. nonce: HBGJag +.. section: Library + +The :mod:`asyncio` REPL now properly closes the loop upon the end of +interactive session. Previously, it could cause surprising warnings. +Contributed by Bartosz Sławecki. + +.. + +.. date: 2025-12-15-02-02-45 +.. gh-issue: 142555 +.. nonce: EC9QN_ +.. section: Library + +:mod:`array`: fix a crash in ``a[i] = v`` when converting *i* to an index +via :meth:`i.__index__ <object.__index__>` or :meth:`i.__float__ +<object.__float__>` mutates the array. + +.. + +.. date: 2025-12-14-18-30-48 +.. gh-issue: 142594 +.. nonce: belDmD +.. section: Library + +Fix crash in ``TextIOWrapper.close()`` when the underlying buffer's +``closed`` property calls :meth:`~io.TextIOBase.detach`. + +.. + +.. date: 2025-12-14-10-00-23 +.. gh-issue: 142451 +.. nonce: _rkf2S +.. section: Library + +:mod:`hmac`: Ensure that the :attr:`HMAC.block_size <hmac.HMAC.block_size>` +attribute is correctly copied by :meth:`HMAC.copy <hmac.HMAC.copy>`. Patch +by Bénédikt Tran. + +.. + +.. date: 2025-12-13-23-26-42 +.. gh-issue: 142495 +.. nonce: I88Uv_ +.. section: Library + +:class:`collections.defaultdict` now prioritizes :meth:`~object.__setitem__` +when inserting default values from ``default_factory``. This prevents race +conditions where a default value would overwrite a value set before +``default_factory`` returns. + +.. + +.. date: 2025-12-13-06-17-44 +.. gh-issue: 142651 +.. nonce: ZRtBu4 +.. section: Library + +:mod:`unittest.mock`: fix a thread safety issue where :attr:`Mock.call_count +<unittest.mock.Mock.call_count>` may return inaccurate values when the mock +is called concurrently from multiple threads. + +.. + +.. date: 2025-12-12-02-56-26 +.. gh-issue: 142595 +.. nonce: wHvTqq +.. section: Library + +Added type check during initialization of the :mod:`decimal` module to +prevent a crash in case of broken stdlib. Patch by Sergey B Kirpichev. + +.. + +.. date: 2025-12-11-09-03-07 +.. gh-issue: 142556 +.. nonce: RuiBte +.. section: Library + +Fix crash when a task gets re-registered during finalization in +:mod:`asyncio`. Patch by Kumar Aditya. + +.. + +.. date: 2025-12-10-11-20-05 +.. gh-issue: 123241 +.. nonce: oYg2n7 +.. section: Library + +Avoid reference count operations in garbage collection of :mod:`ctypes` +objects. + +.. + +.. date: 2025-12-10-10-00-06 +.. gh-issue: 142517 +.. nonce: fG4hbe +.. section: Library + +The non-``compat32`` :mod:`email` policies now correctly handle refolding +encoded words that contain bytes that can not be decoded in their specified +character set. Previously this resulted in an encoding exception during +folding. + +.. + +.. date: 2025-12-09-14-40-45 +.. gh-issue: 112527 +.. nonce: Tvf5Zk +.. section: Library + +The help text for required options in :mod:`argparse` no longer extended +with " (default: None)". + +.. + +.. date: 2025-12-07-17-30-05 +.. gh-issue: 142346 +.. nonce: okcAAp +.. section: Library + +Fix usage formatting for mutually exclusive groups in :mod:`argparse` when +they are preceded by positional arguments or followed or intermixed with +other optional arguments. + +.. + +.. date: 2025-12-07-02-36-24 +.. gh-issue: 142315 +.. nonce: 02o5E_ +.. section: Library + +Pdb can now run scripts from anonymous pipes used in process substitution. +Patch by Bartosz Sławecki. + +.. + +.. date: 2025-12-06-13-02-13 +.. gh-issue: 142332 +.. nonce: PNvXCV +.. section: Library + +Fix usage formatting for positional arguments in mutually exclusive groups +in :mod:`argparse`. in :mod:`argparse`. + +.. + +.. date: 2025-12-05-18-26-50 +.. gh-issue: 142282 +.. nonce: g6RQUN +.. section: Library + +Fix :func:`winreg.QueryValueEx` to not accidentally read garbage buffer +under race condition. + +.. + +.. date: 2025-12-05-16-39-17 +.. gh-issue: 75949 +.. nonce: pHxW98 +.. section: Library + +Fix :mod:`argparse` to preserve ``|`` separators in mutually exclusive +groups when the usage line wraps due to length. + +.. + +.. date: 2025-12-04-23-26-12 +.. gh-issue: 142267 +.. nonce: yOM6fP +.. section: Library + +Improve :mod:`argparse` performance by caching the formatter used for +argument validation. + +.. + +.. date: 2025-12-04-09-22-31 +.. gh-issue: 68552 +.. nonce: I_v-xB +.. section: Library + +``MisplacedEnvelopeHeaderDefect`` and ``Missing header name`` defects are +now correctly passed to the ``handle_defect`` method of ``policy`` in +:class:`~email.parser.FeedParser`. + +.. + +.. date: 2025-11-27-10-49-13 +.. gh-issue: 142006 +.. nonce: nzJDG5 +.. section: Library + +Fix a bug in the :mod:`email.policy.default` folding algorithm which +incorrectly resulted in a doubled newline when a line ending at exactly +max_line_length was followed by an unfoldable token. + +.. + +.. date: 2025-11-18-15-48-13 +.. gh-issue: 105836 +.. nonce: sbUw24 +.. section: Library + +Fix :meth:`asyncio.run_coroutine_threadsafe` leaving underlying cancelled +asyncio task running. + +.. + +.. date: 2025-10-12-12-05-52 +.. gh-issue: 139971 +.. nonce: UdoStU +.. section: Library + +:mod:`pydoc`: Ensure that the link to the online documentation of a +:term:`stdlib` module is correct. + +.. + +.. date: 2025-09-23-16-41-21 +.. gh-issue: 139262 +.. nonce: RO0E98 +.. section: Library + +Some keystrokes can be swallowed in the new ``PyREPL`` on Windows, +especially when used together with the ALT key. Fix by Chris Eibl. + +.. + +.. date: 2025-09-14-22-26-50 +.. gh-issue: 138897 +.. nonce: vnUb_L +.. section: Library + +Improved :data:`license`/:data:`copyright`/:data:`credits` display in the +:term:`REPL`: now uses a pager. + +.. + +.. date: 2025-07-29-11-37-22 +.. gh-issue: 79986 +.. nonce: fnJbE_ +.. section: Library + +Add parsing for ``References`` and ``In-Reply-To`` headers to the +:mod:`email` library that parses the header content as lists of message id +tokens. This prevents them from being folded incorrectly. + +.. + +.. date: 2025-07-05-08-30-07 +.. gh-issue: 136282 +.. nonce: K3JKyD +.. section: Library + +Add support for :const:`~configparser.UNNAMED_SECTION` when creating a +section via the mapping protocol access + +.. + +.. date: 2025-06-22-18-57-19 +.. gh-issue: 109263 +.. nonce: f92V95 +.. section: Library + +Starting a process from spawn context in :mod:`multiprocessing` no longer +sets the start method globally. + +.. + +.. date: 2025-05-05-10-41-41 +.. gh-issue: 133253 +.. nonce: J5-xDD +.. section: Library + +Fix thread-safety issues in :mod:`linecache`. + +.. + +.. date: 2025-04-19-17-34-11 +.. gh-issue: 132715 +.. nonce: XXl47F +.. section: Library + +Skip writing objects during marshalling once a failure has occurred. + +.. + +.. date: 2026-01-13-01-21-20 +.. gh-issue: 143774 +.. nonce: rqGwX1 +.. section: IDLE + +Better explain the operation of Format / Format Paragraph. + +.. + +.. date: 2025-10-30-19-28-42 +.. gh-issue: 140806 +.. nonce: RBT9YH +.. section: Documentation + +Add documentation for :func:`enum.bin`. + +.. + +.. date: 2026-01-29-02-18-08 +.. gh-issue: 144307 +.. nonce: CLbm_o +.. section: Core and Builtins + +Prevent a reference leak in module teardown at interpreter finalization. + +.. + +.. date: 2026-01-23-20-20-42 +.. gh-issue: 144194 +.. nonce: IbXfxd +.. section: Core and Builtins + +Fix error handling in perf jitdump initialization on memory allocation +failure. + +.. + +.. date: 2026-01-19-02-33-45 +.. gh-issue: 144012 +.. nonce: wVEEWs +.. section: Core and Builtins + +Check if the result is ``NULL`` in ``BINARY_OP_EXTENT`` opcode. + +.. + +.. date: 2026-01-13-22-26-49 +.. gh-issue: 141805 +.. nonce: QzIKPS +.. section: Core and Builtins + +Fix crash in :class:`set` when objects with the same hash are concurrently +added to the set after removing an element with the same hash while the set +still contains elements with the same hash. + +.. + +.. date: 2026-01-11-20-11-36 +.. gh-issue: 143670 +.. nonce: klnGoD +.. section: Core and Builtins + +Fixes a crash in ``ga_repr_items_list`` function. + +.. + +.. date: 2026-01-04-16-56-17 +.. gh-issue: 143377 +.. nonce: YJqMCa +.. section: Core and Builtins + +Fix a crash in :func:`!_interpreters.capture_exception` when the exception +is incorrectly formatted. Patch by Bénédikt Tran. + +.. + +.. date: 2026-01-03-14-02-11 +.. gh-issue: 136924 +.. nonce: UMgdPn +.. section: Core and Builtins + +The interactive help mode in the :term:`REPL` no longer incorrectly syntax +highlights text input as Python code. Contributed by Olga Matoula. + +.. + +.. date: 2025-12-30-06-48-48 +.. gh-issue: 143189 +.. nonce: in_sv2 +.. section: Core and Builtins + +Fix crash when inserting a non-:class:`str` key into a split table +dictionary when the key matches an existing key in the split table but has +no corresponding value in the dict. + +.. + +.. date: 2025-12-27-23-57-43 +.. gh-issue: 143228 +.. nonce: m3EF9E +.. section: Core and Builtins + +Fix use-after-free in perf trampoline when toggling profiling while threads +are running or during interpreter finalization with daemon threads active. +The fix uses reference counting to ensure trampolines are not freed while +any code object could still reference them. Pach by Pablo Galindo + +.. + +.. date: 2025-12-27-13-18-12 +.. gh-issue: 142664 +.. nonce: peeEDV +.. section: Core and Builtins + +Fix a use-after-free crash in :meth:`memoryview.__hash__ <object.__hash__>` +when the ``__hash__`` method of the referenced object mutates that object or +the view. Patch by Bénédikt Tran. + +.. + +.. date: 2025-12-27-12-25-06 +.. gh-issue: 142557 +.. nonce: KWOc8b +.. section: Core and Builtins + +Fix a use-after-free crash in :ref:`bytearray.__mod__ <bytes-formatting>` +when the :class:`!bytearray` is mutated while formatting the ``%``-style +arguments. Patch by Bénédikt Tran. + +.. + +.. date: 2025-12-27-10-14-26 +.. gh-issue: 143195 +.. nonce: MNldfr +.. section: Core and Builtins + +Fix use-after-free crashes in :meth:`bytearray.hex` and +:meth:`memoryview.hex` when the separator's :meth:`~object.__len__` mutates +the original object. Patch by Bénédikt Tran. + +.. + +.. date: 2025-12-24-13-44-24 +.. gh-issue: 142975 +.. nonce: 8C4vIP +.. section: Core and Builtins + +Fix crash after unfreezing all objects tracked by the garbage collector on +the :term:`free threaded <free threading>` build. + +.. + +.. date: 2025-12-24-11-39-59 +.. gh-issue: 143135 +.. nonce: 3d5ovx +.. section: Core and Builtins + +Set :data:`sys.flags.inspect` to ``1`` when :envvar:`PYTHONINSPECT` is +``0``. Previously, it was set to ``0`` in this case. + +.. + +.. date: 2025-12-23-00-13-02 +.. gh-issue: 143003 +.. nonce: 92g5qW +.. section: Core and Builtins + +Fix an overflow of the shared empty buffer in :meth:`bytearray.extend` when +``__length_hint__()`` returns 0 for non-empty iterator. + +.. + +.. date: 2025-12-22-22-37-53 +.. gh-issue: 143006 +.. nonce: ZBQwbN +.. section: Core and Builtins + +Fix a possible assertion error when comparing negative non-integer ``float`` +and ``int`` with the same number of bits in the integer part. + +.. + +.. date: 2025-12-22-12-03-09 +.. gh-issue: 143057 +.. nonce: Majsre +.. section: Core and Builtins + +Avoid locking in :c:func:`PyTraceMalloc_Track` and +:c:func:`PyTraceMalloc_Untrack` when :mod:`tracemalloc` is not enabled. + +.. + +.. date: 2025-12-18-01-00-14 +.. gh-issue: 142776 +.. nonce: ACaoeP +.. section: Core and Builtins + +Fix a file descriptor leak in import.c + +.. + +.. date: 2025-12-17-19-45-10 +.. gh-issue: 142829 +.. nonce: ICtLXy +.. section: Core and Builtins + +Fix a use-after-free crash in :class:`contextvars.Context` comparison when a +custom ``__eq__`` method modifies the context via +:meth:`~contextvars.ContextVar.set`. + +.. + +.. date: 2025-12-16-11-56-20 +.. gh-issue: 142766 +.. nonce: Uy2HTm +.. section: Core and Builtins + +Clear the frame of a generator when :meth:`generator.close` is called. + +.. + +.. date: 2025-12-15-15-01-21 +.. gh-issue: 142737 +.. nonce: xYXzeB +.. section: Core and Builtins + +Tracebacks will be displayed in fallback mode even if :func:`io.open` is +lost. Previously, this would crash the interpreter. Patch by Bartosz +Sławecki. + +.. + +.. date: 2025-12-13-17-20-38 +.. gh-issue: 142554 +.. nonce: wNtEFF +.. section: Core and Builtins + +Fix a crash in :func:`divmod` when :func:`!_pylong.int_divmod` does not +return a tuple of length two exactly. Patch by Bénédikt Tran. + +.. + +.. date: 2025-12-11-22-59-33 +.. gh-issue: 142560 +.. nonce: GkJrkk +.. section: Core and Builtins + +Fix use-after-free in :class:`bytearray` search-like methods +(:meth:`~bytearray.find`, :meth:`~bytearray.count`, +:meth:`~bytearray.index`, :meth:`~bytearray.rindex`, and +:meth:`~bytearray.rfind`) by marking the storage as exported which causes +reallocation attempts to raise :exc:`BufferError`. For +:func:`~operator.contains`, :meth:`~bytearray.split`, and +:meth:`~bytearray.rsplit` the :ref:`buffer protocol <bufferobjects>` is used +for this. + +.. + +.. date: 2025-12-10-23-03-10 +.. gh-issue: 142531 +.. nonce: NUEa1T +.. section: Core and Builtins + +Fix a free-threaded GC performance regression. If there are many untracked +tuples, the GC will run too often, resulting in poor performance. The fix +is to include untracked tuples in the "long lived" object count. The number +of frozen objects is also now included since the free-threaded GC must scan +those too. + +.. + +.. date: 2025-12-08-17-34-57 +.. gh-issue: 142402 +.. nonce: iV0ON3 +.. section: Core and Builtins + +Fix reference counting when adjacent literal parts are merged while +constructing :class:`string.templatelib.Template`, preventing the displaced +string object from leaking. + +.. + +.. date: 2025-12-08-15-46-06 +.. gh-issue: 133932 +.. nonce: HAxa4p +.. section: Core and Builtins + +Fix crash in the free threading build when clearing frames that hold tagged +integers. + +.. + +.. date: 2025-12-08-13-04-37 +.. gh-issue: 142343 +.. nonce: BTAyML +.. section: Core and Builtins + +Fix SIGILL crash on m68k due to incorrect assembly constraint. + +.. + +.. date: 2025-11-06-05-21-25 +.. gh-issue: 100964 +.. nonce: TxPf1b +.. section: Core and Builtins + +Fix reference cycle in exhausted generator frames. Patch by Savannah +Ostrowski. + +.. + +.. date: 2025-09-30-21-59-56 +.. gh-issue: 69605 +.. nonce: qcmGF3 +.. section: Core and Builtins + +Fix edge-cases around already imported modules in the :term:`REPL` +auto-completion of imports. + +.. + +.. date: 2025-09-06-08-29-08 +.. gh-issue: 138568 +.. nonce: iZlalC +.. section: Core and Builtins + +Adjusted the built-in :func:`help` function so that empty inputs are ignored +in interactive mode. + +.. + +.. date: 2025-07-22-16-20-06 +.. gh-issue: 137007 +.. nonce: 1oPvvK +.. section: Core and Builtins + +Fix a bug during JIT compilation failure which caused garbage collection +debug assertions to fail. + +.. + +.. date: 2025-12-11-13-01-49 +.. gh-issue: 142589 +.. nonce: nNAqgw +.. section: C API + +Fix :c:func:`PyUnstable_Object_IsUniqueReferencedTemporary()` handling of +tagged ints on the interpreter stack. + +.. + +.. date: 2025-12-11-09-06-36 +.. gh-issue: 142571 +.. nonce: Csdxnn +.. section: C API + +:c:func:`!PyUnstable_CopyPerfMapFile` now checks that opening the file +succeeded before flushing. + +.. + +.. date: 2025-12-09-13-33-46 +.. gh-issue: 142454 +.. nonce: cqUxzQ +.. section: Build + +When calculating the digest of the JIT stencils input, sort the hashed files +by filenames before adding their content to the hasher. This ensures +deterministic hash input and hence deterministic hash, independent on +filesystem order. + +.. + +.. date: 2025-11-20-23-15-39 +.. gh-issue: 141808 +.. nonce: NEewZC +.. section: Build + +When running ``make clean-retain-profile``, keep the generated JIT stencils. +That way, the stencils are not generated twice when Profile-guided +optimization (PGO) is used. It also allows distributors to supply their own +pre-built JIT stencils. + +.. + +.. date: 2025-08-27-09-52-45 +.. gh-issue: 138061 +.. nonce: fMVS9w +.. section: Build + +Ensure reproducible builds by making JIT stencil header generation +deterministic. diff --git a/Misc/NEWS.d/3.14.4.rst b/Misc/NEWS.d/3.14.4.rst new file mode 100644 index 000000000000000..3db06840456e2de --- /dev/null +++ b/Misc/NEWS.d/3.14.4.rst @@ -0,0 +1,1339 @@ +.. date: 2026-03-14-17-31-39 +.. gh-issue: 145986 +.. nonce: ifSSr8 +.. release date: 2026-04-07 +.. section: Security + +:mod:`xml.parsers.expat`: Fixed a crash caused by unbounded C recursion when +converting deeply nested XML content models with +:meth:`~xml.parsers.expat.xmlparser.ElementDeclHandler`. This addresses +:cve:`2026-4224`. + +.. + +.. date: 2026-03-06-17-03-38 +.. gh-issue: 145599 +.. nonce: kchwZV +.. section: Security + +Reject control characters in :class:`http.cookies.Morsel` +:meth:`~http.cookies.Morsel.update` and +:meth:`~http.cookies.BaseCookie.js_output`. This addresses :cve:`2026-3644`. + +.. + +.. date: 2026-03-04-18-59-17 +.. gh-issue: 145506 +.. nonce: 6hwvEh +.. section: Security + +Fixes :cve:`2026-2297` by ensuring that ``SourcelessFileLoader`` uses +:func:`io.open_code` when opening ``.pyc`` files. + +.. + +.. date: 2026-01-31-21-56-54 +.. gh-issue: 144370 +.. nonce: fp9m8t +.. section: Security + +Disallow usage of control characters in status in :mod:`wsgiref.handlers` to +prevent HTTP header injections. Patch by Benedikt Johannes. + +.. + +.. date: 2026-01-16-12-04-49 +.. gh-issue: 143930 +.. nonce: zYC5x3 +.. section: Security + +Reject leading dashes in URLs passed to :func:`webbrowser.open`. + +.. + +.. date: 2026-04-06-11-15-46 +.. gh-issue: 148157 +.. nonce: JFnZDn +.. section: Core and Builtins + +Fix an unlikely crash when parsing an invalid type comments for function +parameters. Found by OSS Fuzz in :oss-fuzz:`492782951`. + +.. + +.. date: 2026-04-05-15-20-00 +.. gh-issue: 148144 +.. nonce: f7qA0x +.. section: Core and Builtins + +Initialize ``_PyInterpreterFrame.visited`` when copying interpreter frames +so incremental GC does not read an uninitialized byte from generator and +frame-object copies. + +.. + +.. date: 2026-03-31-01-06-35 +.. gh-issue: 146615 +.. nonce: fix-method-get +.. section: Core and Builtins + +Fix a crash in :meth:`~object.__get__` for :c:expr:`METH_METHOD` descriptors +when an invalid (non-type) object is passed as the second argument. Patch by +Steven Sun. + +.. + +.. date: 2026-03-22-19-30-00 +.. gh-issue: 146308 +.. nonce: AxnRVA +.. section: Core and Builtins + +Fixed several error handling issues in the :mod:`!_remote_debugging` module, +including safer validation of remote ``int`` objects, clearer asyncio task +chain failures, and cache cleanup fixes that avoid leaking or double-freeing +metadata on allocation failure. Patch by Pablo Galindo. + +.. + +.. date: 2026-03-21-15-05-14 +.. gh-issue: 146128 +.. nonce: DG1Hfa +.. section: Core and Builtins + +Fix a bug which could cause constant values to be partially corrupted in +AArch64 JIT code. This issue is theoretical, and hasn't actually been +observed in unmodified Python interpreters. + +.. + +.. date: 2026-03-21-11-55-16 +.. gh-issue: 146250 +.. nonce: ahl3O2 +.. section: Core and Builtins + +Fixed a memory leak in :exc:`SyntaxError` when re-initializing it. + +.. + +.. date: 2026-03-21-08-48-25 +.. gh-issue: 146245 +.. nonce: cqM3_4 +.. section: Core and Builtins + +Fixed reference leaks in :mod:`socket` when audit hooks raise exceptions in +:func:`socket.getaddrinfo` and :meth:`!socket.sendto`. + +.. + +.. date: 2026-03-20-13-55-14 +.. gh-issue: 146196 +.. nonce: Zg70Kb +.. section: Core and Builtins + +Fix potential Undefined Behavior in :c:func:`PyUnicodeWriter_WriteASCII` by +adding a zero-length check. Patch by Shamil Abdulaev. + +.. + +.. date: 2026-03-20-13-07-33 +.. gh-issue: 146227 +.. nonce: MqBPEo +.. section: Core and Builtins + +Fix wrong type in ``_Py_atomic_load_uint16`` in the C11 atomics backend +(``pyatomic_std.h``), which used a 32-bit atomic load instead of 16-bit. +Found by Mohammed Zuhaib. + +.. + +.. date: 2026-03-18-18-52-00 +.. gh-issue: 146056 +.. nonce: r1tVSo +.. section: Core and Builtins + +Fix :func:`repr` for lists and tuples containing ``NULL``\ s. + +.. + +.. date: 2026-03-18-16-57-56 +.. gh-issue: 146092 +.. nonce: wCKFYS +.. section: Core and Builtins + +Handle properly memory allocation failures on str and float opcodes. Patch +by Victor Stinner. + +.. + +.. date: 2026-03-17-00-00-00 +.. gh-issue: 146041 +.. nonce: 7799bb +.. section: Core and Builtins + +Fix free-threading scaling bottleneck in :func:`sys.intern` and +:c:func:`PyObject_SetAttr` by avoiding the interpreter-wide lock when the +string is already interned and immortalized. + +.. + +.. date: 2026-03-15-21-45-35 +.. gh-issue: 145990 +.. nonce: tmXwRB +.. section: Core and Builtins + +``python --help-env`` sections are now sorted by environment variable name. + +.. + +.. date: 2026-03-15-20-47-34 +.. gh-issue: 145990 +.. nonce: 14BUzw +.. section: Core and Builtins + +``python --help-xoptions`` is now sorted by ``-X`` option name. + +.. + +.. date: 2026-03-11-21-27-28 +.. gh-issue: 145376 +.. nonce: LfDvyw +.. section: Core and Builtins + +Fix GC tracking in ``structseq.__replace__()``. + +.. + +.. date: 2026-03-11-19-09-47 +.. gh-issue: 145792 +.. nonce: X5KUhc +.. section: Core and Builtins + +Fix out-of-bounds access when invoking faulthandler on a CPython build +compiled without support for VLAs. + +.. + +.. date: 2026-03-11-00-13-59 +.. gh-issue: 142183 +.. nonce: 2iVhJH +.. section: Core and Builtins + +Avoid a pathological case where repeated calls at a specific stack depth +could be significantly slower. + +.. + +.. date: 2026-03-10-22-38-40 +.. gh-issue: 145779 +.. nonce: 5375381d80 +.. section: Core and Builtins + +Improve scaling of :func:`classmethod` and :func:`staticmethod` calls in the +free-threaded build by avoiding the descriptor ``__get__`` call. + +.. + +.. date: 2026-03-10-19-00-39 +.. gh-issue: 145783 +.. nonce: dS5TM9 +.. section: Core and Builtins + +Fix an unlikely crash in the parser when certain errors were erroneously not +propagated. Found by OSS Fuzz in :oss-fuzz:`491369109`. + +.. + +.. date: 2026-03-10-12-52-06 +.. gh-issue: 145685 +.. nonce: 80B7gK +.. section: Core and Builtins + +Improve scaling of type attribute lookups in the :term:`free-threaded build` +by avoiding contention on the internal type lock. + +.. + +.. date: 2026-03-09-18-52-03 +.. gh-issue: 145701 +.. nonce: 79KQyO +.. section: Core and Builtins + +Fix :exc:`SystemError` when ``__classdict__`` or +``__conditional_annotations__`` is in a class-scope inlined comprehension. +Found by OSS Fuzz in :oss-fuzz:`491105000`. + +.. + +.. date: 2026-03-09-00-00-00 +.. gh-issue: 145713 +.. nonce: KR6azvzI +.. section: Core and Builtins + +Make :meth:`bytearray.resize` thread-safe in the free-threaded build by +using a critical section and calling the lock-held variant of the resize +function. + +.. + +.. date: 2026-03-06-21-05-05 +.. gh-issue: 145615 +.. nonce: NKXXZgDW +.. section: Core and Builtins + +Fixed a memory leak in the :term:`free-threaded build` where mimalloc pages +could become permanently unreclaimable until the owning thread exited. + +.. + +.. date: 2026-03-05-19-10-56 +.. gh-issue: 145566 +.. nonce: H4RupyYN +.. section: Core and Builtins + +In the free threading build, skip the stop-the-world pause when reassigning +``__class__`` on a newly created object. + +.. + +.. date: 2026-03-01-13-37-31 +.. gh-issue: 145335 +.. nonce: e36kPJ +.. section: Core and Builtins + +Fix a crash in :func:`os.pathconf` when called with ``-1`` as the path +argument. + +.. + +.. date: 2026-02-28-18-42-36 +.. gh-issue: 145036 +.. nonce: 70Kbfz +.. section: Core and Builtins + +In free-threaded build, fix race condition when calling :meth:`!__sizeof__` +on a :class:`list` + +.. + +.. date: 2026-02-28-16-46-17 +.. gh-issue: 145376 +.. nonce: lG5u1a +.. section: Core and Builtins + +Fix reference leaks in various unusual error scenarios. + +.. + +.. date: 2026-02-26-21-36-00 +.. gh-issue: 145234 +.. nonce: w0mQ9n +.. section: Core and Builtins + +Fixed a ``SystemError`` in the parser when an encoding cookie (for example, +UTF-7) decodes to carriage returns (``\r``). Newlines are now normalized +after decoding in the string tokenizer. + +Patch by Pablo Galindo. + +.. + +.. date: 2026-02-26-12-00-00 +.. gh-issue: 130555 +.. nonce: TMSOIu +.. section: Core and Builtins + +Fix use-after-free in :meth:`dict.clear` when the dictionary values are +embedded in an object and a destructor causes re-entrant mutation of the +dictionary. + +.. + +.. date: 2026-02-24-18-30-56 +.. gh-issue: 145187 +.. nonce: YjPu1Z +.. section: Core and Builtins + +Fix compiler assertion fail when a type parameter bound contains an invalid +expression in a conditional block. + +.. + +.. date: 2026-02-23-23-18-28 +.. gh-issue: 145142 +.. nonce: T-XbVe +.. section: Core and Builtins + +Fix a crash in the free-threaded build when the dictionary argument to +:meth:`str.maketrans` is concurrently modified. + +.. + +.. date: 2026-02-16-12-28-43 +.. gh-issue: 144872 +.. nonce: k9_Q30 +.. section: Core and Builtins + +Fix heap buffer overflow in the parser found by OSS-Fuzz. + +.. + +.. date: 2026-02-13-18-30-59 +.. gh-issue: 144766 +.. nonce: JGu3x3 +.. section: Core and Builtins + +Fix a crash in fork child process when perf support is enabled. + +.. + +.. date: 2026-02-13-12-00-00 +.. gh-issue: 144759 +.. nonce: d3qYpe +.. section: Core and Builtins + +Fix undefined behavior in the lexer when ``start`` and ``multi_line_start`` +pointers are ``NULL`` in ``_PyLexer_remember_fstring_buffers()`` and +``_PyLexer_restore_fstring_buffers()``. The ``NULL`` pointer arithmetic +(``NULL - valid_pointer``) is now guarded with explicit ``NULL`` checks. + +.. + +.. date: 2026-02-08-18-13-38 +.. gh-issue: 144563 +.. nonce: hb3kpp +.. section: Core and Builtins + +Fix interaction of the Tachyon profiler and :mod:`ctypes` and other modules +that load the Python shared library (if present) in an independent map as +this was causing the mechanism that loads the binary information to be +confused. Patch by Pablo Galindo + +.. + +.. date: 2026-02-08-12-47-27 +.. gh-issue: 144601 +.. nonce: E4Yi9J +.. section: Core and Builtins + +Fix crash when importing a module whose ``PyInit`` function raises an +exception from a subinterpreter. + +.. + +.. date: 2026-02-06-21-45-52 +.. gh-issue: 144438 +.. nonce: GI_uB1LR +.. section: Core and Builtins + +Align the QSBR thread state array to a 64-byte cache line boundary to avoid +false sharing in the :term:`free-threaded build`. + +.. + +.. date: 2026-02-05-13-30-00 +.. gh-issue: 144513 +.. nonce: IjSTd7 +.. section: Core and Builtins + +Fix potential deadlock when using critical sections during stop-the-world +pauses in the free-threaded build. + +.. + +.. date: 2026-02-03-17-08-13 +.. gh-issue: 144446 +.. nonce: db5619 +.. section: Core and Builtins + +Fix data races in the free-threaded build when reading frame object +attributes while another thread is executing the frame. + +.. + +.. date: 2026-01-10-12-59-58 +.. gh-issue: 143636 +.. nonce: dzr26e +.. section: Core and Builtins + +Fix a crash when calling :class:`SimpleNamespace.__replace__() +<types.SimpleNamespace>` on non-namespace instances. Patch by Bénédikt Tran. + +.. + +.. date: 2026-01-10-10-58-36 +.. gh-issue: 143650 +.. nonce: k8mR4x +.. section: Core and Builtins + +Fix race condition in :mod:`importlib` where a thread could receive a stale +module reference when another thread's import fails. + +.. + +.. date: 2025-11-19-16-40-24 +.. gh-issue: 141732 +.. nonce: PTetqp +.. section: Core and Builtins + +Ensure the :meth:`~object.__repr__` for :exc:`ExceptionGroup` and +:exc:`BaseExceptionGroup` does not change when the exception sequence that +was original passed in to its constructor is subsequently mutated. + +.. + +.. date: 2025-11-02-16-23-17 +.. gh-issue: 140594 +.. nonce: YIWUpl +.. section: Core and Builtins + +Fix an out of bounds read when a single NUL character is read from the +standard input. Patch by Shamil Abdulaev. + +.. + +.. date: 2025-07-07-17-26-06 +.. gh-issue: 91636 +.. nonce: GyHU72 +.. section: Core and Builtins + +While performing garbage collection, clear weakrefs to unreachable objects +that are created during running of finalizers. If those weakrefs were are +not cleared, they could reveal unreachable objects. + +.. + +.. date: 2025-02-19-21-06-30 +.. gh-issue: 130327 +.. nonce: z3TaR8 +.. section: Core and Builtins + +Fix erroneous clearing of an object's :attr:`~object.__dict__` if +overwritten at runtime. + +.. + +.. date: 2023-07-26-00-03-00 +.. gh-issue: 80667 +.. nonce: N7Dh8B +.. section: Core and Builtins + +Literals using the ``\N{name}`` escape syntax can now construct CJK +ideographs and Hangul syllables using case-insensitive names. + +.. + +.. date: 2026-04-07-01-04-00 +.. gh-issue: 144503 +.. nonce: argvfs +.. section: Library + +Fix a regression introduced in 3.14.3 and 3.13.12 where the +:mod:`multiprocessing` ``forkserver`` start method would fail with +:exc:`BrokenPipeError` when the parent process had a very large +:data:`sys.argv`. The argv is now passed to the forkserver as separate +command-line arguments rather than being embedded in the ``-c`` command +string, avoiding the operating system's per-argument length limit. + +.. + +.. date: 2026-04-01-11-05-36 +.. gh-issue: 146613 +.. nonce: GzjUFK +.. section: Library + +:mod:`itertools`: Fix a crash in :func:`itertools.groupby` when the grouper +iterator is concurrently mutated. + +.. + +.. date: 2026-03-28-13-19-20 +.. gh-issue: 146080 +.. nonce: srN12a +.. section: Library + +:mod:`ssl`: fix a crash when an SNI callback tries to use an SSL object that +has already been garbage-collected. Patch by Bénédikt Tran. + +.. + +.. date: 2026-03-28-12-20-19 +.. gh-issue: 146556 +.. nonce: Y8Eson +.. section: Library + +Fix :func:`annotationlib.get_annotations` hanging indefinitely when called +with ``eval_str=True`` on a callable that has a circular ``__wrapped__`` +chain (e.g. ``f.__wrapped__ = f``). Cycle detection using an id-based +visited set now stops the traversal and falls back to the globals found so +far, mirroring the approach of :func:`inspect.unwrap`. + +.. + +.. date: 2026-03-28-12-05-34 +.. gh-issue: 146090 +.. nonce: wf9_ef +.. section: Library + +:mod:`sqlite3`: fix a crash when :meth:`sqlite3.Connection.create_collation` +fails with `SQLITE_BUSY <https://sqlite.org/rescode.html#busy>`__. Patch by +Bénédikt Tran. + +.. + +.. date: 2026-03-28-12-01-48 +.. gh-issue: 146090 +.. nonce: wh1qJR +.. section: Library + +:mod:`sqlite3`: properly raise :exc:`MemoryError` instead of +:exc:`SystemError` when a context callback fails to be allocated. Patch by +Bénédikt Tran. + +.. + +.. date: 2026-03-26-11-04-42 +.. gh-issue: 145633 +.. nonce: RWjlaX +.. section: Library + +Fix ``struct.pack('f', float)``: use :c:func:`PyFloat_Pack4` to raise +:exc:`OverflowError`. Patch by Sergey B Kirpichev and Victor Stinner. + +.. + +.. date: 2026-03-24-03-49-50 +.. gh-issue: 146310 +.. nonce: WhlDir +.. section: Library + +The :mod:`ensurepip` module no longer looks for ``pip-*.whl`` wheel packages +in the current directory. + +.. + +.. date: 2026-03-17-20-52-24 +.. gh-issue: 146083 +.. nonce: NxZa_c +.. section: Library + +Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.7.5. + +.. + +.. date: 2026-03-17-20-41-27 +.. gh-issue: 146076 +.. nonce: yoBNnB +.. section: Library + +:mod:`zoneinfo`: fix crashes when deleting ``_weak_cache`` from a +:class:`zoneinfo.ZoneInfo` subclass. + +.. + +.. date: 2026-03-17-11-46-20 +.. gh-issue: 146054 +.. nonce: udYcqn +.. section: Library + +Limit the size of :func:`encodings.search_function` cache. Found by OSS Fuzz +in :oss-fuzz:`493449985`. + +.. + +.. date: 2026-03-16-00-00-00 +.. gh-issue: 146004 +.. nonce: xOptProp +.. section: Library + +All :option:`-X` options from the Python command line are now propagated to +child processes spawned by :mod:`multiprocessing`, not just a hard-coded +subset. This makes the behavior consistent between default "spawn" and +"forkserver" start methods and the old "fork" start method. The options +that were previously not propagated are: ``context_aware_warnings``, +``cpu_count``, ``disable-remote-debug``, ``int_max_str_digits``, +``lazy_imports``, ``no_debug_ranges``, ``pathconfig_warnings``, ``perf``, +``perf_jit``, ``presite``, ``pycache_prefix``, ``thread_inherit_context``, +and ``warn_default_encoding``. + +.. + +.. date: 2026-03-12-21-01-48 +.. gh-issue: 145883 +.. nonce: lUvXcc +.. section: Library + +:mod:`zoneinfo`: Fix heap buffer overflow reads from malformed TZif data. +Found by OSS Fuzz, issues :oss-fuzz:`492245058` and :oss-fuzz:`492230068`. + +.. + +.. date: 2026-03-10-14-57-15 +.. gh-issue: 145754 +.. nonce: YBL5Ko +.. section: Library + +Request signature during mock autospec with ``FORWARDREF`` annotation +format. This prevents runtime errors when an annotation uses a name that is +not defined at runtime. + +.. + +.. date: 2026-03-10-14-13-12 +.. gh-issue: 145750 +.. nonce: iQsTeX +.. section: Library + +Avoid undefined behaviour from signed integer overflow when parsing format +strings in the :mod:`struct` module. Found by OSS Fuzz in +:oss-fuzz:`488466741`. + +.. + +.. date: 2026-03-09-00-00-00 +.. gh-issue: 145492 +.. nonce: 457Afc +.. section: Library + +Fix infinite recursion in :class:`collections.defaultdict` ``__repr__`` when +a ``defaultdict`` contains itself. Based on analysis by KowalskiThomas in +:gh:`145492`. + +.. + +.. date: 2026-03-07-15-00-00 +.. gh-issue: 145623 +.. nonce: 2Y7LzT +.. section: Library + +Fix crash in :mod:`struct` when calling :func:`repr` or ``__sizeof__()`` on +an uninitialized :class:`struct.Struct` object created via +``Struct.__new__()`` without calling ``__init__()``. + +.. + +.. date: 2026-03-07-02-44-52 +.. gh-issue: 145616 +.. nonce: x8Mf23 +.. section: Library + +Detect Android sysconfig ABI correctly on 32-bit ARM Android on 64-bit ARM +kernel + +.. + +.. date: 2026-03-05-19-01-28 +.. gh-issue: 145551 +.. nonce: gItPRl +.. section: Library + +Fix InvalidStateError when cancelling process created by +:func:`asyncio.create_subprocess_exec` or +:func:`asyncio.create_subprocess_shell`. Patch by Daan De Meyer. + +.. + +.. date: 2026-03-03-23-21-40 +.. gh-issue: 145446 +.. nonce: 0c-TJX +.. section: Library + +Now :mod:`functools` is safer in free-threaded build when using keywords in +:func:`functools.partial` + +.. + +.. date: 2026-03-03-11-49-44 +.. gh-issue: 145417 +.. nonce: m_HxIL +.. section: Library + +:mod:`venv`: Prevent incorrect preservation of SELinux context when copying +the ``Activate.ps1`` script. The script inherited the SELinux security +context of the system template directory, rather than the destination +project directory. + +.. + +.. date: 2026-03-02-19-41-39 +.. gh-issue: 145376 +.. nonce: OOzSOh +.. section: Library + +Fix double free and null pointer dereference in unusual error scenarios in +:mod:`hashlib` and :mod:`hmac` modules. + +.. + +.. date: 2026-02-28-00-55-00 +.. gh-issue: 145301 +.. nonce: Lk2bRl +.. section: Library + +:mod:`hmac`: fix a crash when the initialization of the underlying C +extension module fails. + +.. + +.. date: 2026-02-27-19-00-26 +.. gh-issue: 145301 +.. nonce: 2Wih4b +.. section: Library + +:mod:`hashlib`: fix a crash when the initialization of the underlying C +extension module fails. + +.. + +.. date: 2026-02-26-20-13-16 +.. gh-issue: 145264 +.. nonce: 4pggX_ +.. section: Library + +Base64 decoder (see :func:`binascii.a2b_base64`, :func:`base64.b64decode`, +etc) no longer ignores excess data after the first padded quad in non-strict +(default) mode. Instead, in conformance with :rfc:`4648`, section 3.3, it +now ignores the pad character, "=", if it is present before the end of the +encoded data. + +.. + +.. date: 2026-02-23-20-52-55 +.. gh-issue: 145158 +.. nonce: vWJtxI +.. section: Library + +Avoid undefined behaviour from signed integer overflow when parsing format +strings in the :mod:`struct` module. + +.. + +.. date: 2026-02-19-12-00-00 +.. gh-issue: 144984 +.. nonce: b93995c982 +.. section: Library + +Fix crash in :meth:`xml.parsers.expat.xmlparser.ExternalEntityParserCreate` +when an allocation fails. The error paths could dereference NULL +``handlers`` and double-decrement the parent parser's reference count. + +.. + +.. date: 2026-02-19-10-57-40 +.. gh-issue: 88091 +.. nonce: N7qGV- +.. section: Library + +Fix :func:`unicodedata.decomposition` for Hangul characters. + +.. + +.. date: 2026-02-19-00-00-00 +.. gh-issue: 144986 +.. nonce: atexit-leak +.. section: Library + +Fix a memory leak in :func:`atexit.register`. Patch by Shamil Abdulaev. + +.. + +.. date: 2026-02-18-13-45-00 +.. gh-issue: 144777 +.. nonce: R97q0a +.. section: Library + +Fix data races in :class:`io.IncrementalNewlineDecoder` in the +:term:`free-threaded build`. + +.. + +.. date: 2026-02-18-00-00-00 +.. gh-issue: 144809 +.. nonce: nYpEUx +.. section: Library + +Make :class:`collections.deque` copy atomic in the :term:`free-threaded +build`. + +.. + +.. date: 2026-02-15-12-02-20 +.. gh-issue: 144835 +.. nonce: w_oS_J +.. section: Library + +Added missing explanations for some parameters in :func:`glob.glob` and +:func:`glob.iglob`. + +.. + +.. date: 2026-02-15-00-00-00 +.. gh-issue: 144833 +.. nonce: TUelo1 +.. section: Library + +Fixed a use-after-free in :mod:`ssl` when ``SSL_new()`` returns NULL in +``newPySSLSocket()``. The error was reported via a dangling pointer after +the object had already been freed. + +.. + +.. date: 2026-02-13-14-20-10 +.. gh-issue: 144782 +.. nonce: 0Y8TKj +.. section: Library + +Fix :class:`argparse.ArgumentParser` to be :mod:`pickleable <pickle>`. + +.. + +.. date: 2026-02-11-21-01-30 +.. gh-issue: 144259 +.. nonce: OAhOR8 +.. section: Library + +Fix inconsistent display of long multiline pasted content in the REPL. + +.. + +.. date: 2026-02-10-22-05-51 +.. gh-issue: 144156 +.. nonce: UbrC7F +.. section: Library + +Fix the folding of headers by the :mod:`email` library when :rfc:`2047` +encoded words are used. Now whitespace is correctly preserved and also +correctly added between adjacent encoded words. The latter property was +broken by the fix for gh-92081, which mostly fixed previous failures to +preserve whitespace. + +.. + +.. date: 2026-02-10-16-56-05 +.. gh-issue: 66305 +.. nonce: PZ6GN8 +.. section: Library + +Fixed a hang on Windows in the :mod:`tempfile` module when trying to create +a temporary file or subdirectory in a non-writable directory. + +.. + +.. date: 2026-02-08-22-04-06 +.. gh-issue: 140814 +.. nonce: frzSpn +.. section: Library + +:func:`multiprocessing.freeze_support` no longer sets the default start +method as a side effect, which previously caused a subsequent +:func:`multiprocessing.set_start_method` call to raise :exc:`RuntimeError`. + +.. + +.. date: 2026-02-07-16-37-42 +.. gh-issue: 144475 +.. nonce: 8tFEXw +.. section: Library + +Calling :func:`repr` on :func:`functools.partial` is now safer when the +partial object's internal attributes are replaced while the string +representation is being generated. + +.. + +.. date: 2026-02-06-23-58-54 +.. gh-issue: 144538 +.. nonce: 5_OvGv +.. section: Library + +Bump the version of pip bundled in ensurepip to version 26.0.1 + +.. + +.. date: 2026-02-05-13-16-57 +.. gh-issue: 144494 +.. nonce: SmcsR3 +.. section: Library + +Fix performance regression in :func:`asyncio.all_tasks` on +:term:`free-threaded builds <free-threaded build>`. Patch by Kumar Aditya. + +.. + +.. date: 2026-02-03-19-57-41 +.. gh-issue: 144316 +.. nonce: wop870 +.. section: Library + +Fix crash in ``_remote_debugging`` that caused ``test_external_inspection`` +to intermittently fail. Patch by Taegyun Kim. + +.. + +.. date: 2026-01-31-17-15-49 +.. gh-issue: 144363 +.. nonce: X9f0sU +.. section: Library + +Update bundled `libexpat <https://libexpat.github.io/>`_ to 2.7.4 + +.. + +.. date: 2026-01-17-08-44-25 +.. gh-issue: 143637 +.. nonce: qyPqDo +.. section: Library + +Fixed a crash in socket.sendmsg() that could occur if ancillary data is +mutated re-entrantly during argument parsing. + +.. + +.. date: 2026-01-13-10-38-43 +.. gh-issue: 143543 +.. nonce: DeQRCO +.. section: Library + +Fix a crash in itertools.groupby that could occur when a user-defined +:meth:`~object.__eq__` method re-enters the iterator during key comparison. + +.. + +.. date: 2026-01-12-19-39-57 +.. gh-issue: 140652 +.. nonce: HvM9Bl +.. section: Library + +Fix a crash in :func:`!_interpchannels.list_all` after closing a channel. + +.. + +.. date: 2026-01-11-18-35-52 +.. gh-issue: 143698 +.. nonce: gXDzsJ +.. section: Library + +Allow *scheduler* and *setpgroup* arguments to be explicitly :const:`None` +when calling :func:`os.posix_spawn` or :func:`os.posix_spawnp`. Patch by +Bénédikt Tran. + +.. + +.. date: 2026-01-11-16-59-22 +.. gh-issue: 143698 +.. nonce: b-Cpeb +.. section: Library + +Raise :exc:`TypeError` instead of :exc:`SystemError` when the *scheduler* in +:func:`os.posix_spawn` or :func:`os.posix_spawnp` is not a tuple. Patch by +Bénédikt Tran. + +.. + +.. date: 2026-01-11-13-03-32 +.. gh-issue: 142516 +.. nonce: u7An-s +.. section: Library + +:mod:`ssl`: fix reference leaks in :class:`ssl.SSLContext` objects. Patch by +Bénédikt Tran. + +.. + +.. date: 2026-01-01-05-26-00 +.. gh-issue: 143304 +.. nonce: Kv7x9Q +.. section: Library + +Fix :class:`ctypes.CDLL` to honor the ``handle`` parameter on POSIX systems. + +.. + +.. date: 2025-12-18-00-14-16 +.. gh-issue: 142781 +.. nonce: gcOeYF +.. section: Library + +:mod:`zoneinfo`: fix a crash when instantiating :class:`~zoneinfo.ZoneInfo` +objects for which the internal class-level cache is inconsistent. + +.. + +.. date: 2025-12-18-00-00-00 +.. gh-issue: 142763 +.. nonce: AJpZPVG5 +.. section: Library + +Fix a race condition between :class:`zoneinfo.ZoneInfo` creation and +:func:`zoneinfo.ZoneInfo.clear_cache` that could raise :exc:`KeyError`. + +.. + +.. date: 2025-12-16-13-34-48 +.. gh-issue: 142787 +.. nonce: wNitJX +.. section: Library + +Fix assertion failure in :mod:`sqlite3` blob subscript when slicing with +indices that result in an empty slice. + +.. + +.. date: 2025-12-06-16-14-18 +.. gh-issue: 142352 +.. nonce: pW5HLX88 +.. section: Library + +Fix :meth:`asyncio.StreamWriter.start_tls` to transfer buffered data from +:class:`~asyncio.StreamReader` to the SSL layer, preventing data loss when +upgrading a connection to TLS mid-stream (e.g., when implementing PROXY +protocol support). + +.. + +.. date: 2025-11-18-06-35-53 +.. gh-issue: 141707 +.. nonce: DBmQIy +.. section: Library + +Don't change :class:`tarfile.TarInfo` type from ``AREGTYPE`` to ``DIRTYPE`` +when parsing GNU long name or link headers. + +.. + +.. date: 2025-10-11-11-50-59 +.. gh-issue: 139933 +.. nonce: 05MHlx +.. section: Library + +Improve :exc:`AttributeError` suggestions for classes with a custom +:meth:`~object.__dir__` method returning a list of unsortable values. Patch +by Bénédikt Tran. + +.. + +.. date: 2025-08-04-23-20-43 +.. gh-issue: 137335 +.. nonce: IIjDJN +.. section: Library + +Get rid of any possibility of a name conflict for named pipes in +:mod:`multiprocessing` and :mod:`asyncio` on Windows, no matter how small. + +.. + +.. date: 2023-02-05-20-02-30 +.. gh-issue: 80667 +.. nonce: 7LmzeA +.. section: Library + +Support lookup for Tangut Ideographs in :mod:`unicodedata`. + +.. + +.. bpo: 40243 +.. date: 2020-04-10-14-29-53 +.. nonce: 85HRib +.. section: Library + +Fix :meth:`!unicodedata.ucd_3_2_0.numeric` for non-decimal values. + +.. + +.. date: 2026-03-25-00-00-00 +.. gh-issue: 126676 +.. nonce: 052336 +.. section: Documentation + +Expand :mod:`argparse` documentation for ``type=bool`` with a demonstration +of the surprising behavior and pointers to common alternatives. + +.. + +.. date: 2026-03-09-00-00-00 +.. gh-issue: 145649 +.. nonce: 8BcbAB +.. section: Documentation + +Fix text wrapping and formatting of ``-X`` option descriptions in the +:manpage:`python(1)` man page by using proper roff markup. + +.. + +.. date: 2026-03-03-08-18-00 +.. gh-issue: 145450 +.. nonce: VI7GXj +.. section: Documentation + +Document missing public :class:`wave.Wave_write` getter methods. + +.. + +.. date: 2025-08-02-18-59-01 +.. gh-issue: 136246 +.. nonce: RIK7nE +.. section: Documentation + +A new "Improve this page" link is available in the left-hand sidebar of the +docs, offering links to create GitHub issues, discussion forum posts, or +pull requests. + +.. + +.. date: 2026-04-03-21-37-18 +.. gh-issue: 144418 +.. nonce: PusC0S +.. section: Tests + +The Android testbed's emulator RAM has been increased from 2 GB to 4 GB. + +.. + +.. date: 2026-03-24-00-15-58 +.. gh-issue: 146202 +.. nonce: LgH6Bj +.. section: Tests + +Fix a race condition in regrtest: make sure that the temporary directory is +created in the worker process. Previously, temp_cwd() could fail on Windows +if the "build" directory was not created. Patch by Victor Stinner. + +.. + +.. date: 2026-02-12-12-12-00 +.. gh-issue: 144739 +.. nonce: -fx1tN +.. section: Tests + +When Python was compiled with system expat older then 2.7.2 but tests run +with newer expat, still skip +:class:`!test.test_pyexpat.MemoryProtectionTest`. + +.. + +.. date: 2026-03-28-02-48-51 +.. gh-issue: 146541 +.. nonce: k-zlM6 +.. section: Build + +The Android testbed can now be built for 32-bit ARM and x86 targets. + +.. + +.. date: 2026-03-27-06-55-10 +.. gh-issue: 146498 +.. nonce: uOiCab +.. section: Build + +The iOS XCframework build script now ensures libpython isn't included in +installed app content, and is more robust in identifying standard library +binary content that requires processing. + +.. + +.. date: 2026-03-26-14-35-29 +.. gh-issue: 146450 +.. nonce: 9Kmp5Q +.. section: Build + +The Android build script was modified to improve parity with other platform +build scripts. + +.. + +.. date: 2026-03-26-12-48-42 +.. gh-issue: 146446 +.. nonce: 0GyMu4 +.. section: Build + +The clean target for the Apple/iOS XCframework build script is now more +selective when targeting a single architecture. + +.. + +.. date: 2026-03-11-11-58-42 +.. gh-issue: 145801 +.. nonce: iCXa3v +.. section: Build + +When Python build is optimized with GCC using PGO, use +``-fprofile-update=atomic`` option to use atomic operations when updating +profile information. This option reduces the risk of gcov Data Files (.gcda) +corruption which can cause random GCC crashes. Patch by Victor Stinner. + +.. + +.. date: 2026-02-27-10-57-20 +.. gh-issue: 145307 +.. nonce: ueoT7j +.. section: Windows + +Defers loading of the ``psapi.dll`` module until it is used by +:func:`ctypes.util.dllist`. + +.. + +.. date: 2026-02-13-11-07-51 +.. gh-issue: 144551 +.. nonce: ENtMYD +.. section: Windows + +Updated bundled version of OpenSSL to 3.0.19. + +.. + +.. date: 2025-10-19-23-44-46 +.. gh-issue: 140131 +.. nonce: AABF2k +.. section: Windows + +Fix REPL cursor position on Windows when module completion suggestion line +hits console width. + +.. + +.. date: 2026-02-17-00-15-11 +.. gh-issue: 144551 +.. nonce: ydhtXd +.. section: macOS + +Update macOS installer to use OpenSSL 3.0.19. + +.. + +.. date: 2025-10-17-01-07-03 +.. gh-issue: 137586 +.. nonce: kVzxvp +.. section: macOS + +Invoke :program:`osascript` with absolute path in :mod:`webbrowser` and +:mod:`!turtledemo`. + +.. + +.. date: 2026-03-18-20-18-59 +.. gh-issue: 146056 +.. nonce: nnZIgp +.. section: C API + +:c:func:`PyUnicodeWriter_WriteRepr` now supports ``NULL`` argument. + +.. + +.. date: 2026-02-19-18-39-11 +.. gh-issue: 145010 +.. nonce: mKzjci +.. section: C API + +Use GCC dialect alternatives for inline assembly in ``object.h`` so that the +Python headers compile correctly with ``-masm=intel``. + +.. + +.. date: 2026-02-18-15-12-34 +.. gh-issue: 144981 +.. nonce: 4ZdM63 +.. section: C API + +Made :c:func:`PyUnstable_Code_SetExtra`, :c:func:`PyUnstable_Code_GetExtra`, +and :c:func:`PyUnstable_Eval_RequestCodeExtraIndex` thread-safe on the +:term:`free threaded <free threading>` build. diff --git a/Misc/NEWS.d/3.14.5.rst b/Misc/NEWS.d/3.14.5.rst new file mode 100644 index 000000000000000..9dd152f87215172 --- /dev/null +++ b/Misc/NEWS.d/3.14.5.rst @@ -0,0 +1,122 @@ +.. date: 2026-04-06-13-55-00 +.. gh-issue: 148178 +.. nonce: Rs7kLm +.. release date: 2026-05-10 +.. section: Security + +Hardened :mod:`!_remote_debugging` by validating remote debug offset tables +before using them to size memory reads or interpret remote layouts. + +.. + +.. date: 2026-04-20-15-25-55 +.. gh-issue: 146270 +.. nonce: qZYfyc +.. section: Core and Builtins + +Fix a sequential consistency bug in ``structmember.c``. + +.. + +.. date: 2025-08-01-20-31-30 +.. gh-issue: 137293 +.. nonce: 4x3JbV +.. section: Core and Builtins + +Fix :exc:`SystemError` when searching ELF Files in :func:`sys.remote_exec`. + +.. + +.. date: 2026-05-07-21-58-17 +.. gh-issue: 149388 +.. nonce: DDBPeA +.. section: Library + +Make :class:`!asyncio.windows_utils.PipeHandle` closing idempotent. + +.. + +.. date: 2026-05-04-19-28-48 +.. gh-issue: 149377 +.. nonce: WNlc8Y +.. section: Library + +Update bundled pip to 26.1.1 + +.. + +.. date: 2026-04-25-14-11-24 +.. gh-issue: 138907 +.. nonce: u21Wnh +.. section: Library + +Support :rfc:`9309` in :mod:`urllib.robotparser`. + +.. + +.. date: 2026-04-15-16-08-12 +.. gh-issue: 148615 +.. nonce: Uvx50R +.. section: Library + +Fix :mod:`pdb` to accept standard -- end of options separator. Reported by +haampie. Patched by Shrey Naithani. + +.. + +.. date: 2026-02-19-04-40-57 +.. gh-issue: 130750 +.. nonce: 0hW52O +.. section: Library + +Restore quoting of choices in :mod:`argparse` error messages for improved +clarity and consistency with documentation. + +.. + +.. date: 2025-12-06-08-48-26 +.. gh-issue: 141449 +.. nonce: hQvNW_ +.. section: Library + +Improve tests and documentation for non-function callables as +:term:`annotate functions <annotate function>`. + +.. + +.. date: 2026-05-05-18-49-44 +.. gh-issue: 149425 +.. nonce: QnQL8j +.. section: Tests + +Increase time delta in +``test.test_zipfile.test_core.OtherTests.test_write_without_source_date_epoch`` + +.. + +.. date: 2026-05-05-17-08-36 +.. gh-issue: 145736 +.. nonce: JYdLx4 +.. section: Tests + +Fix test_tkinter test_configure_values test case backport miss for Tk 9. + +.. + +.. date: 2026-05-06-18-23-36 +.. gh-issue: 142295 +.. nonce: O9RmZH +.. section: macOS + +For Python macOS framework builds, update Info.plist files to be more +compliant with current Apple guidelines. Original patch contributed by +Martinus Verburg. + +.. + +.. date: 2026-05-05-18-42-59 +.. gh-issue: 124111 +.. nonce: WmQG7S +.. section: macOS + +Update macOS installer to use Tcl/Tk 9.0.3. diff --git a/Misc/NEWS.d/3.14.5rc1.rst b/Misc/NEWS.d/3.14.5rc1.rst new file mode 100644 index 000000000000000..1744c19d9864d2c --- /dev/null +++ b/Misc/NEWS.d/3.14.5rc1.rst @@ -0,0 +1,624 @@ +.. date: 2026-05-02-16-22-20 +.. gh-issue: 149254 +.. nonce: 9ozXB9 +.. release date: 2026-05-04 +.. section: Security + +Update Android and iOS installer to use OpenSSL 3.0.20. + +.. + +.. date: 2026-04-26-17-49-58 +.. gh-issue: 149017 +.. nonce: EiVFPo +.. section: Security + +Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.8.0. + +.. + +.. date: 2026-04-21-13-46-30 +.. gh-issue: 90309 +.. nonce: srvj9q +.. section: Security + +Base64-encode values when embedding cookies to JavaScript using the +:meth:`http.cookies.BaseCookie.js_output` method to avoid injection and +escaping. + +.. + +.. date: 2026-04-20-15-31-37 +.. gh-issue: 148808 +.. nonce: _Z8JL0 +.. section: Security + +Added buffer boundary check when using ``nbytes`` parameter with +:meth:`!asyncio.AbstractEventLoop.sock_recvfrom_into`. Only relevant for +Windows and the :class:`asyncio.ProactorEventLoop`. + +.. + +.. date: 2026-04-10-16-28-21 +.. gh-issue: 148395 +.. nonce: kfzm0G +.. section: Security + +Fix a dangling input pointer in :class:`lzma.LZMADecompressor`, +:class:`bz2.BZ2Decompressor`, and internal :class:`!zlib._ZlibDecompressor` +when memory allocation fails with :exc:`MemoryError`, which could let a +subsequent :meth:`!decompress` call read or write through a stale pointer to +the already-released caller buffer. + +.. + +.. date: 2026-03-31-09-15-51 +.. gh-issue: 148169 +.. nonce: EZJzz2 +.. section: Security + +A bypass in :mod:`webbrowser` allowed URLs prefixed with ``%action`` to pass +the dash-prefix safety check. + +.. + +.. date: 2026-03-29-12-51-33 +.. gh-issue: 146581 +.. nonce: 4vZfB0 +.. section: Security + +Fix vulnerability in :func:`shutil.unpack_archive` for ZIP files on Windows +which allowed to write files outside of the destination tree if the patch in +the archive contains a Windows drive prefix. Now such invalid paths will be +skipped. Files containing ".." in the name (like "foo..bar") are no longer +skipped. + +.. + +.. date: 2026-03-25-00-51-03 +.. gh-issue: 146333 +.. nonce: LqdL__bn +.. section: Security + +Fix quadratic backtracking in :class:`configparser.RawConfigParser` option +parsing regexes (``OPTCRE`` and ``OPTCRE_NV``). A crafted configuration line +with many whitespace characters could cause excessive CPU usage. + +.. + +.. date: 2026-03-20-09-29-42 +.. gh-issue: 146211 +.. nonce: PQVbs7 +.. section: Security + +Reject CR/LF characters in tunnel request headers for the +HTTPConnection.set_tunnel() method. + +.. + +.. date: 2026-04-29-14-06-00 +.. gh-issue: 149122 +.. nonce: P8k2Lm +.. section: Core and Builtins + +Fix a crash in optimized calls to :func:`all`, :func:`any`, :func:`tuple`, +:func:`list`, and :func:`set` with an async generator expression argument +(for example, ``tuple(await x for x in y)``). These calls now correctly +raise ``TypeError`` instead of crashing. + +.. + +.. date: 2026-04-22-14-55-18 +.. gh-issue: 113956 +.. nonce: 0VEXd6 +.. section: Core and Builtins + +Fix a data race in :func:`sys.intern` in the free-threaded build when +interning a string owned by another thread. An interned copy owned by the +current thread is used instead when it is not safe to immortalize the +original. + +.. + +.. date: 2026-04-21-14-36-44 +.. gh-issue: 148820 +.. nonce: XhOGhA +.. section: Core and Builtins + +Fix a race in :c:type:`!_PyRawMutex` on the free-threaded build where a +``Py_PARK_INTR`` return from ``_PySemaphore_Wait`` could let the waiter +destroy its semaphore before the unlocking thread's ``_PySemaphore_Wakeup`` +completed, causing a fatal ``ReleaseSemaphore`` error. + +.. + +.. date: 2026-04-17-20-37-02 +.. gh-issue: 148653 +.. nonce: nbbHMh +.. section: Core and Builtins + +Forbid :mod:`marshalling <marshal>` recursive code objects and +:class:`slice` objects which cannot be correctly unmarshalled. + +.. + +.. date: 2026-04-17-11-30-00 +.. gh-issue: 142516 +.. nonce: GcGen315 +.. section: Core and Builtins + +Forward-port the generational cycle garbage collector to the default 3.14 +build, replacing the incremental collector while leaving the free-threaded +collector unchanged. + +.. + +.. date: 2026-04-12-17-27-28 +.. gh-issue: 148390 +.. nonce: MAhw7F +.. section: Core and Builtins + +Fix an undefined behavior in :class:`memoryview` when using the native +boolean format (``?``) in :meth:`~memoryview.cast`. Previously, on some +common platforms, calling ``memoryview(b).cast("?").tolist()`` incorrectly +returned ``[False]`` instead of ``[True]`` for any even byte *b*. Patch by +Bénédikt Tran. + +.. + +.. date: 2026-04-12-10-40-57 +.. gh-issue: 148418 +.. nonce: ggA1LZ +.. section: Core and Builtins + +Fix a possible reference leak in a corrupted ``TYPE_CODE`` marshal stream. + +.. + +.. date: 2026-04-11-17-28-52 +.. gh-issue: 148393 +.. nonce: lX6gwN +.. section: Core and Builtins + +Fix data races between :c:func:`PyDict_Watch` / :c:func:`PyDict_Unwatch` and +concurrent dict mutation in the :term:`free-threaded build`. + +.. + +.. date: 2026-04-10-14-20-54 +.. gh-issue: 148284 +.. nonce: HKs-S_ +.. section: Core and Builtins + +Fix high stack consumption in Python's interpreter loop on Clang 22 by +setting function limits for inlining when building with computed gotos. + +.. + +.. date: 2026-04-09-14-18-33 +.. gh-issue: 148037 +.. nonce: aP3CSX +.. section: Core and Builtins + +Remove critical section from :c:func:`!PyCode_Addr2Line` in free-threading. + +.. + +.. date: 2026-04-07-20-37-23 +.. gh-issue: 148222 +.. nonce: uF4D4E +.. section: Core and Builtins + +Fix vectorcall support in :class:`types.GenericAlias` when the underlying +type does not support the vectorcall protocol. Fix possible leaks in +:class:`types.GenericAlias` and :class:`types.UnionType` in case of memory +error. + +.. + +.. date: 2026-04-07-20-21-44 +.. gh-issue: 148208 +.. nonce: JAxpDU +.. section: Core and Builtins + +Fix recursion depth leak in :c:func:`PyObject_Print` + +.. + +.. date: 2026-04-07-07-21-30 +.. gh-issue: 137814 +.. nonce: 6yRTeu +.. section: Core and Builtins + +Fix the ``__qualname__`` attribute of ``__annotate__`` functions on +functions. + +.. + +.. date: 2026-04-02-13-25-09 +.. gh-issue: 147998 +.. nonce: wnzkRT +.. section: Core and Builtins + +Fixed a memory leak in interpreter helper calls so cleanup works when an +operation falls across interpreter boundaries. Patch by Maurycy +Pawłowski-Wieroński. + +.. + +.. date: 2026-03-26-08-49-35 +.. gh-issue: 146455 +.. nonce: f54083a9 +.. section: Core and Builtins + +Fix O(N²) compile-time regression in constant folding after it was moved +from AST to CFG optimizer. + +.. + +.. date: 2026-05-02-12-03-48 +.. gh-issue: 149221 +.. nonce: __KOks +.. section: Library + +Catch rare math domain error for :func:`random.binomialvariate`. + +.. + +.. date: 2026-04-29-16-11-27 +.. gh-issue: 149117 +.. nonce: yEeTYd +.. section: Library + +Fix :func:`runpy.run_module` and :func:`runpy.run_path` to set the +:attr:`~ImportError.name` attribute on the :exc:`ImportError` they raise. + +.. + +.. date: 2026-04-29-14-33-42 +.. gh-issue: 149148 +.. nonce: EaiYvk +.. section: Library + +:mod:`ensurepip`: Upgrade bundled pip to 26.1. This version fixes the +:cve:`2026-3219` vulnerability. Patch by Victor Stinner. + +.. + +.. date: 2026-04-27-22-34-09 +.. gh-issue: 148093 +.. nonce: 9pWceM +.. section: Library + +Fix an out-of-bounds read of one byte in :func:`binascii.a2b_uu`. Raise +:exc:`binascii.Error`, instead of reading past the buffer end. + +.. + +.. date: 2026-04-27-17-12-11 +.. gh-issue: 148914 +.. nonce: i5C3kW +.. section: Library + +Fix memoization of in-band :class:`~pickle.PickleBuffer` in the Python +implementation of :mod:`pickle`. Previously, identical +:class:`!PickleBuffer`\ s did not preserve identity, and empty writable +:class:`!PickleBuffer` memoized an empty bytearray object in place of +``b''``, so the following references to ``b''`` were unpickled as an empty +bytearray object. + +.. + +.. date: 2026-04-23-21-47-49 +.. gh-issue: 148947 +.. nonce: W4V2lG +.. section: Library + +Fix crash in :deco:`dataclasses.dataclass` with ``slots=True`` that occurred +when a function found within the class had an empty ``__class__`` cell. + +.. + +.. date: 2026-04-23-07-38-04 +.. gh-issue: 148680 +.. nonce: ___ePl +.. section: Library + +``ForwardRef`` objects that contain internal names to represent known +objects now show the ``type_repr`` of the known object rather than the +internal ``__annotationlib_name_x__`` name when evaluated as strings. + +.. + +.. date: 2026-04-20-18-29-21 +.. gh-issue: 148801 +.. nonce: ROeNqs +.. section: Library + +:mod:`xml.etree.ElementTree`: Fix a crash in :meth:`Element.__deepcopy__ +<object.__deepcopy__>` on deeply nested trees. + +.. + +.. date: 2026-04-18-21-39-15 +.. gh-issue: 148735 +.. nonce: siw6DG +.. section: Library + +:mod:`xml.etree.ElementTree`: Fix a use-after-free in +:meth:`Element.findtext <xml.etree.ElementTree.Element.findtext>` when the +element tree is mutated concurrently during the search. + +.. + +.. date: 2026-04-18-17-37-13 +.. gh-issue: 148740 +.. nonce: sYnFi0 +.. section: Library + +Fix usage for :mod:`uuid` command-line interface to support a custom +namespace be provided for uuid3 and uuid5. + +.. + +.. date: 2026-04-16-13-30-00 +.. gh-issue: 148651 +.. nonce: ZsTdLk +.. section: Library + +Fix reference leak in :class:`compression.zstd.ZstdDecompressor` when an +invalid option key is passed. + +.. + +.. date: 2026-04-15-11-00-39 +.. gh-issue: 146553 +.. nonce: VGOsoP +.. section: Library + +Fix infinite loop in :func:`typing.get_type_hints` when ``__wrapped__`` +forms a cycle. Patch by Shamil Abdulaev. + +.. + +.. date: 2026-04-14-09-04-35 +.. gh-issue: 148508 +.. nonce: -GiXml +.. section: Library + +An intermittent timing error when running SSL tests on iOS has been +resolved. + +.. + +.. date: 2026-04-13-15-59-44 +.. gh-issue: 148518 +.. nonce: RQdvsu +.. section: Library + +If an email containing an address header that ended in an open double quote +was parsed with a non-``compat32`` policy, accessing the ``username`` +attribute of the mailbox accessed through that header object would result in +an ``IndexError``. It now correctly returns an empty string as the result. + +.. + +.. date: 2026-04-13-06-22-27 +.. gh-issue: 148464 +.. nonce: Bj_NZy +.. section: Library + +Add missing ``__ctype_le/be__`` attributes for +:class:`~ctypes.c_float_complex` and :class:`~ctypes.c_double_complex`. +Patch by Sergey B Kirpichev. + +.. + +.. date: 2026-04-12-16-40-11 +.. gh-issue: 148370 +.. nonce: 0Li2EK +.. section: Library + +:mod:`configparser`: prevent quadratic behavior when a +:exc:`~configparser.ParsingError` is raised after a parser fails to parse +multiple lines. Patch by Bénédikt Tran. + +.. + +.. date: 2026-04-09-12-42-42 +.. gh-issue: 148254 +.. nonce: Xt7vKs +.. section: Library + +Use singular "sec" instead of "secs" in :mod:`timeit` verbose output for +consistency with other time units. + +.. + +.. date: 2026-04-07-14-13-40 +.. gh-issue: 148192 +.. nonce: 34AUYQ +.. section: Library + +``email.generator.Generator._make_boundary`` could fail to detect a +duplicate boundary string if linesep was not \n. It now correctly detects +boundary strings when linesep is \r\n as well. + +.. + +.. date: 2026-03-22-23-42-22 +.. gh-issue: 146313 +.. nonce: RtDeAd +.. section: Library + +Fix a deadlock in :mod:`multiprocessing`'s resource tracker where the parent +process could hang indefinitely in :func:`os.waitpid` during interpreter +shutdown if a child created via :func:`os.fork` still held the resource +tracker's pipe open. + +.. + +.. date: 2026-03-11-15-09-52 +.. gh-issue: 145831 +.. nonce: _sW94w +.. section: Library + +Fix :func:`!email.quoprimime.decode` leaving a stray ``\r`` when +``eol='\r\n'`` by stripping the full *eol* string instead of one character. + +.. + +.. date: 2026-02-22-00-00-00 +.. gh-issue: 145105 +.. nonce: csv-reader-reentrant +.. section: Library + +Fix crash in :mod:`csv` reader when iterating with a re-entrant iterator +that calls :func:`next` on the same reader from within ``__next__``. + +.. + +.. date: 2026-01-19-21-23-18 +.. gh-issue: 105936 +.. nonce: dGrzjM +.. section: Library + +Attempting to mutate non-field attributes of :mod:`dataclasses` with both +*frozen* and *slots* being ``True`` now raises +:class:`~dataclasses.FrozenInstanceError` instead of :class:`TypeError`. +Their non-dataclass subclasses can now freely mutate non-field attributes, +and the original non-slotted class can be garbage collected. + +.. + +.. date: 2025-10-18-12-13-39 +.. gh-issue: 140287 +.. nonce: 49iU-4 +.. section: Library + +The :mod:`asyncio` REPL now handles exceptions when executing +:envvar:`PYTHONSTARTUP` scripts. Patch by Bartosz Sławecki. + +.. + +.. date: 2025-04-17-15-26-35 +.. gh-issue: 132631 +.. nonce: IDFZfb +.. section: Library + +Fix "I/O operation on closed file" when parsing JSON Lines file with +:mod:`JSON CLI <json.tool>`. + +.. + +.. date: 2024-02-10-21-25-22 +.. gh-issue: 70039 +.. nonce: 6wvcAP +.. section: Library + +Fixed bug where :meth:`smtplib.SMTP.starttls` could fail if +:meth:`smtplib.SMTP.connect` is called explicitly rather than implicitly. + +.. + +.. date: 2023-09-08-13-10-32 +.. gh-issue: 83281 +.. nonce: 2Plpcj +.. section: Library + +:mod:`email`: improve handling trailing garbage in address lists to avoid +throwing AttributeError in certain edge cases + +.. + +.. date: 2026-04-17-02-28-55 +.. gh-issue: 148663 +.. nonce: MHIbRB +.. section: Documentation + +Document that :class:`calendar.IllegalMonthError` is a subclass of both +:exc:`ValueError` and :exc:`IndexError` since Python 3.12. + +.. + +.. date: 2026-04-02-07-20-00 +.. gh-issue: 146646 +.. nonce: GlobDoc1 +.. section: Documentation + +Document that :func:`glob.glob`, :func:`glob.iglob`, +:meth:`pathlib.Path.glob`, and :meth:`pathlib.Path.rglob` silently suppress +:exc:`OSError` exceptions raised from scanning the filesystem. + +.. + +.. date: 2026-05-04-06-03-50 +.. gh-issue: 149351 +.. nonce: hN4sF0 +.. section: Build + +Avoid possible broken macOS framework install names when DESTDIR is +specified during builds. + +.. + +.. date: 2026-04-30-08-43-47 +.. gh-issue: 146475 +.. nonce: 1cL4hX +.. section: Build + +Block Apple Clang from being used to build the JIT as it ships without +required LLVM tools. + +.. + +.. date: 2026-04-14-15-20-29 +.. gh-issue: 148535 +.. nonce: JjKiaa +.. section: Build + +No longer use the ``gcc -fprofile-update=atomic`` flag on i686. The flag has +been added to fix a random GCC internal error on PGO build (:gh:`145801`) +caused by corruption of profile data (.gcda files). The problem is that it +makes the PGO build way slower (up to 47x slower) on i686. Since the GCC +internal error was not seen on i686 so far, don't use +``-fprofile-update=atomic`` on i686 anymore. Patch by Victor Stinner. + +.. + +.. date: 2026-03-21-18-51-31 +.. gh-issue: 146264 +.. nonce: Q9Ej4m +.. section: Build + +Fix static module builds on non-WASI targets by linking HACL dependencies as +static libraries when ``MODULE_BUILDTYPE=static``, preventing duplicate +``_Py_LibHacl_*`` symbol errors at link time. + +.. + +.. date: 2026-05-03-14-07-51 +.. gh-issue: 149254 +.. nonce: ENtMYD +.. section: Windows + +Updated bundled version of OpenSSL to 3.0.20. + +.. + +.. date: 2026-03-27-22-06-10 +.. gh-issue: 146458 +.. nonce: fYj0UQ +.. section: Windows + +Fix incorrect REPL height and width tracking on console window resize on +Windows. + +.. + +.. date: 2026-05-01-19-38-16 +.. gh-issue: 149254 +.. nonce: enO7uj +.. section: macOS + +Update macOS installer to use OpenSSL 3.0.20. diff --git a/Misc/NEWS.d/3.14.6.rst b/Misc/NEWS.d/3.14.6.rst new file mode 100644 index 000000000000000..4427f4a17f7be33 --- /dev/null +++ b/Misc/NEWS.d/3.14.6.rst @@ -0,0 +1,720 @@ +.. date: 2026-06-09-10-23-57 +.. gh-issue: 151159 +.. nonce: 91GpWQ +.. release date: 2026-06-10 +.. section: Security + +Update Android and iOS installers to use OpenSSL 3.5.7. + +.. + +.. date: 2026-05-30-09-36-20 +.. gh-issue: 150599 +.. nonce: nlHqU- +.. section: Security + +Fix a possible stack buffer overflow in :mod:`bz2` when a +:class:`bz2.BZ2Decompressor` is reused after a decompression error. The +decompressor now becomes unusable after libbz2 reports an error. + +.. + +.. date: 2026-05-18-17-46-00 +.. gh-issue: 149835 +.. nonce: EebFlk +.. section: Security + +:func:`shutil.move` now resolves symlinks via :func:`os.path.realpath` when +checking whether the destination is inside the source directory, preventing +a symlink-based bypass of that guard. + +.. + +.. date: 2026-05-11-21-15-07 +.. gh-issue: 149698 +.. nonce: OudOcW +.. section: Security + +Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.8.1 +for the fix for :cve:`2026-45186`. + +.. + +.. date: 2026-05-10-18-05-32 +.. gh-issue: 87451 +.. nonce: XkKB6M +.. section: Security + +The :mod:`ftplib` module's undocumented ``ftpcp`` function no longer trusts +the IPv4 address value returned from the source server in response to the +``PASV`` command by default, completing the fix for CVE-2021-4189. As with +:class:`ftplib.FTP`, the former behavior can be re-enabled by setting the +``trust_server_pasv_ipv4_address`` attribute on the source +:class:`ftplib.FTP` instance to ``True``. Thanks to Qi Deng at Aurascape AI +for the report. + +.. + +.. date: 2026-05-03-21-00-00 +.. gh-issue: 149486 +.. nonce: tarflt +.. section: Security + +:func:`tarfile.data_filter` now validates link targets using the same +normalised value that is written to disk, strips trailing separators from +the member name when resolving a symlink's directory, and rejects link +members that would replace the destination directory itself. This closes +several path-traversal bypasses of the ``data`` extraction filter. + +.. + +.. date: 2026-04-27-16-36-11 +.. gh-issue: 149079 +.. nonce: vKl-LM +.. section: Security + +Fix a potential denial of service in :func:`unicodedata.normalize`. The +canonical ordering step of Unicode normalization used a quadratic-time +insertion sort for reordering combining characters, which could be exploited +with crafted input containing many combining characters in non-canonical +order. Replaced with a linear-time counting sort for long runs. + +.. + +.. date: 2026-04-26-19-30-45 +.. gh-issue: 149018 +.. nonce: a9SqWb +.. section: Security + +Improved protection against XML hash-flooding attacks in +:mod:`xml.parsers.expat` and :mod:`xml.etree.ElementTree` when Python is +compiled with libExpat 2.8.0 or later. + +.. + +.. date: 2026-06-09-12-24-35 +.. gh-issue: 151112 +.. nonce: 4RKCkD +.. section: Core and Builtins + +Fix a crash in the compiler that could occur when running out of memory. + +.. + +.. date: 2026-06-09-10-28-30 +.. gh-issue: 151126 +.. nonce: DKa6Sl +.. section: Core and Builtins + +Fix a crash, when there's no memory left on a device, which happened in: + +- code compilation - :func:`!_winapi.CreateProcess` + +Now these places raise proper :exc:`MemoryError` errors. + +.. + +.. date: 2026-06-01-19-00-00 +.. gh-issue: 150700 +.. nonce: W8CzVR +.. section: Core and Builtins + +Fix a :exc:`SystemError` when compiling a class-scope comprehension +containing a ``lambda`` that references ``__class__``, ``__classdict__``, or +``__conditional_annotations__``. Patch by Bartosz Sławecki. + +.. + +.. date: 2026-05-30-20-19-35 +.. gh-issue: 150633 +.. nonce: XkNul0 +.. section: Core and Builtins + +Fix the frozen importer accepting module names with embedded null bytes, +which caused it to bypass the :data:`sys.modules` cache and create duplicate +module objects. + +.. + +.. date: 2026-05-24-22-46-49 +.. gh-issue: 148613 +.. nonce: PLpmyd +.. section: Core and Builtins + +Fix a data race in the free-threaded build between :func:`gc.set_threshold` +and garbage collection scheduling during object allocation. + +.. + +.. date: 2026-05-24-14-45-00 +.. gh-issue: 149156 +.. nonce: NP73rB +.. section: Core and Builtins + +Fix an intermittent crash after :func:`os.fork` when perf trampoline +profiling is enabled and the child returns through trampoline frames +inherited from the parent process. + +.. + +.. date: 2026-05-23-22-08-01 +.. gh-issue: 149449 +.. nonce: 2lhQFF +.. section: Core and Builtins + +Fix a use-after-free crash when the :mod:`unicodedata` module was removed +from :data:`sys.modules` and garbage-collected between calls that decode +``\N{...}`` escapes or use the ``namereplace`` codec error handler. + +.. + +.. date: 2026-05-22-21-52-38 +.. gh-issue: 150207 +.. nonce: l2BUtI +.. section: Core and Builtins + +Fix a crash when a memory allocation fails during tokenizer initialization. +A proper :exc:`MemoryError` is now raised instead. + +.. + +.. date: 2026-05-22-17-09-28 +.. gh-issue: 150107 +.. nonce: GD72-D +.. section: Core and Builtins + +:mod:`asyncio`: ``sendfile()`` and ``sock_sendfile()`` event loop methods +now call ``file.seek(offset)`` if *file* has a ``seek()`` method, even if +*offset* is ``0`` (default value). + +.. + +.. date: 2026-05-20-13-06-17 +.. gh-issue: 150146 +.. nonce: i5m_SL +.. section: Core and Builtins + +Fix a crash on a complex type variable substitution. + +``from typing import TypeVar; memoryview[TypeVar("")][*typing.Mapping[..., +...]]`` used to fail due to missing ``NULL`` check on ``_unpack_args`` C +function call. + +.. + +.. date: 2026-05-18-13-47-17 +.. gh-issue: 149590 +.. nonce: IPBeQx +.. section: Core and Builtins + +Fix crash when faulthandler is imported more than once. + +.. + +.. date: 2026-05-16-11-03-54 +.. gh-issue: 149816 +.. nonce: X_gqMT +.. section: Core and Builtins + +Fix a race condition in ``_PyBytes_FromList`` in free-threading mode. + +.. + +.. date: 2026-05-15-11-31-57 +.. gh-issue: 149816 +.. nonce: ugN2rx +.. section: Core and Builtins + +Fix a race condition in :class:`memoryview` with free-threading. + +.. + +.. date: 2026-05-13-21-26-26 +.. gh-issue: 149805 +.. nonce: IG6cza +.. section: Core and Builtins + +Fix a :exc:`SystemError` when compiling a compiling ``__classdict__`` class +annotation. Found by OSS-Fuzz in :oss-fuzz:`512907042`. + +.. + +.. date: 2026-05-13-06-54-41 +.. gh-issue: 149738 +.. nonce: 4BLFoH +.. section: Core and Builtins + +:mod:`sqlite3`: Disallow removing ``row_factory`` and ``text_factory`` +attributes of a connection to prevent a crash on a query. + +.. + +.. date: 2026-05-12-16-47-23 +.. gh-issue: 139808 +.. nonce: iIs7_E +.. section: Core and Builtins + +Add branch protections for AArch64 (BTI/PAC) in assembly code used by +:option:`-X perf_jit <-X>` (Linux perf profiler integration). + +.. + +.. date: 2026-04-15-15-48-04 +.. gh-issue: 148450 +.. nonce: 2MEVqH +.. section: Core and Builtins + +Fix ``abc.register()`` so it invalidates type version tags for registered +classes. + +.. + +.. date: 2026-06-07-17-29-33 +.. gh-issue: 151039 +.. nonce: AZ0qBn +.. section: Library + +Fix a crash when static :mod:`datetime` types outlive the ``_datetime`` +module. + +.. + +.. date: 2026-06-04-21-49-18 +.. gh-issue: 150913 +.. nonce: EmptyBl +.. section: Library + +Fix :class:`sqlite3.Blob` slice assignment to raise :exc:`TypeError` and +:exc:`IndexError` for type and size mismatches respectively, even when the +target slice is empty. + +.. + +.. date: 2026-06-04-18-22-56 +.. gh-issue: 143008 +.. nonce: z5tw-J +.. section: Library + +Fix race conditions when re-initializing a :class:`io.TextIOWrapper` object. + +.. + +.. date: 2026-06-02-14-21-46 +.. gh-issue: 150750 +.. nonce: SVS2o0 +.. section: Library + +Fix a race condition in :meth:`collections.deque.index` with free-threading. + +.. + +.. date: 2026-05-31-17-47-30 +.. gh-issue: 150685 +.. nonce: EBB2mU +.. section: Library + +Update bundled pip to 26.1.2 + +.. + +.. date: 2026-05-25-17-00-00 +.. gh-issue: 150406 +.. nonce: jF3g63 +.. section: Library + +Fix a possible crash occurring during :mod:`socket` module initialization +when the system is out of memory on platforms without a reentrant +``gethostbyname``. + +.. + +.. date: 2026-05-25-07-22-05 +.. gh-issue: 150372 +.. nonce: 9hLqhe +.. section: Library + +:mod:`readline`: Fix a potential crash during tab completion caused by an +out-of-memory error during module initialization. + +.. + +.. date: 2026-05-21-20-47-45 +.. gh-issue: 150157 +.. nonce: ZvmO-bQZ +.. section: Library + +Fix a crash in free-threaded builds that occurs when pickling by name +objects without a ``__module__`` attribute while :data:`sys.modules` is +concurrently being modified. + +.. + +.. date: 2026-05-21-11-25-58 +.. gh-issue: 150175 +.. nonce: 8H4Caz +.. section: Library + +Fix race condition in :class:`unittest.mock.ThreadingMock` where concurrent +calls could lose increments to ``call_count`` and other attributes due to a +missing lock in ``_increment_mock_call``. + +.. + +.. date: 2026-05-19-19-00-49 +.. gh-issue: 84353 +.. nonce: ZU5zaQ +.. section: Library + +Preserve non-UTF-8 encoded filenames when appending to a +:class:`zipfile.ZipFile`. Previously, non-ASCII names stored in a legacy +encoding (without the UTF-8 flag bit set) could be corrupted when the +central directory was rewritten: they were decoded as cp437 and then +re-stored as UTF-8. + +.. + +.. date: 2026-05-18-22-45-54 +.. gh-issue: 149816 +.. nonce: T68vc_ +.. section: Library + +Fix race condition in :attr:`ssl.SSLContext.sni_callback` + +.. + +.. date: 2026-05-18-07-44-46 +.. gh-issue: 149995 +.. nonce: vvtFHn +.. section: Library + +Update various docstrings in :mod:`typing`. + +.. + +.. date: 2026-05-17-22-37-02 +.. gh-issue: 88726 +.. nonce: BAoL6j +.. section: Library + +The :mod:`email` package now uses standard MIME charset names "gb2312" and +"big5" instead of non-standard names "eucgb2312_cn" and "big5_tw". + +.. + +.. date: 2026-05-17-02-25-56 +.. gh-issue: 149571 +.. nonce: LNyuWJ +.. section: Library + +Fix the C implementation of :meth:`xml.etree.ElementTree.Element.itertext`: +it no longer emits text for comments and processing instructions. + +.. + +.. date: 2026-05-16-21-08-33 +.. gh-issue: 149921 +.. nonce: I1yNML +.. section: Library + +Fix reference leaks in error paths of the :mod:`!_interpchannels` and +:mod:`!_interpqueues` extension modules. + +.. + +.. date: 2026-05-14-15-55-28 +.. gh-issue: 149816 +.. nonce: ZaXQ0q +.. section: Library + +Fix a race condition in ``_random.Random.__init__`` method in free-threading +mode. + +.. + +.. date: 2026-05-13-23-18-39 +.. gh-issue: 149801 +.. nonce: S_FfGr +.. section: Library + +Add IANA registered names and aliases with leading zeros before number (like +IBM00858, CP00858, IBM01140, CP01140) for corresponding codecs. + +.. + +.. date: 2026-05-12-06-24-54 +.. gh-issue: 149701 +.. nonce: 8v9RTm +.. section: Library + +Fix bad return code from Lib/venv/bin/activate if hashing is disabled + +.. + +.. date: 2026-05-08-15-08-35 +.. gh-issue: 112821 +.. nonce: t9T1YD +.. section: Library + +In the REPL, autocompletion might run arbitrary code in the getter of a +descriptor. If that getter raised an exception, autocompletion would fail to +present any options for the entire object. Autocompletion now works as +expected for these objects. + +.. + +.. date: 2026-05-07-14-18-47 +.. gh-issue: 149489 +.. nonce: bX9iHe +.. section: Library + +Fix :mod:`~xml.etree.ElementTree` serialization to HTML. The content of +elements "xmp", "iframe", "noembed", "noframes", and "plaintext" is no +longer escaped. The "plaintext" element no longer have the closing tag. + +.. + +.. date: 2026-05-01-16-45-31 +.. gh-issue: 149231 +.. nonce: x2nBEE +.. section: Library + +In :mod:`tomllib`, the number of parts in TOML keys is now limited. + +.. + +.. date: 2026-04-27-11-12-00 +.. gh-issue: 149046 +.. nonce: 74shDd +.. section: Library + +:mod:`io`: Fix :class:`io.StringIO` serialization: no longer call +``str(obj)`` on :class:`str` subclasses. Patch by Thomas Kowalski. + +.. + +.. date: 2026-04-24-19-54-00 +.. gh-issue: 148954 +.. nonce: v1 +.. section: Library + +Fix XML injection vulnerability in :func:`xmlrpc.client.dumps` where the +``methodname`` was not being escaped before interpolation into the XML body. + +.. + +.. date: 2026-04-23-12-50-15 +.. gh-issue: 148441 +.. nonce: zvpCkR +.. section: Library + +:mod:`xml.parsers.expat`: prevent a crash in +:meth:`~xml.parsers.expat.xmlparser.CharacterDataHandler` when the character +data size exceeds the parser's :attr:`buffer size +<xml.parsers.expat.xmlparser.buffer_size>`. + +.. + +.. date: 2026-03-26-09-30-00 +.. gh-issue: 146452 +.. nonce: Y2N6qZ8J +.. section: Library + +Fix segfault in :mod:`pickle` when pickling a dictionary concurrently +mutated by another thread in the free-threaded build. + +.. + +.. date: 2025-12-17-04-10-35 +.. gh-issue: 142831 +.. nonce: ee3t4L +.. section: Library + +Fix a crash in the :mod:`json` module where a use-after-free could occur if +the object being encoded is modified during serialization. + +.. + +.. date: 2025-09-26-18-04-28 +.. gh-issue: 90949 +.. nonce: YHjSzX +.. section: Library + +Add +:meth:`~xml.parsers.expat.xmlparser.SetBillionLaughsAttackProtectionActivationThreshold` +and +:meth:`~xml.parsers.expat.xmlparser.SetBillionLaughsAttackProtectionMaximumAmplification` +to :ref:`xmlparser <xmlparser-objects>` objects to tune protections against +`billion laughs <https://en.wikipedia.org/wiki/Billion_laughs_attack>`_ +attacks. Patch by Bénédikt Tran. + +.. + +.. date: 2025-05-19-21-08-25 +.. gh-issue: 134261 +.. nonce: ravGYm +.. section: Library + +zip: On reproducible builds, ZipFile uses UTC instead of the local time when +writing file datetimes to avoid underflows. + +.. + +.. date: 2025-03-01-13-36-02 +.. gh-issue: 128110 +.. nonce: 9wx_G0 +.. section: Library + +Fix bug in the parsing of :mod:`email` address headers that could result in +extraneous spaces in the decoded text when using a modern email policy. +Space between pairs of adjacent :rfc:`2047` encoded-words is now ignored, +per section 6.2 (and consistent with existing parsing of unstructured +headers like *Subject*). + +.. + +.. date: 2024-11-02-02-02-31 +.. gh-issue: 107398 +.. nonce: uUtA6Q +.. section: Library + +Fix :mod:`tarfile` stream mode exception when process the file with the gzip +extra field. + +.. + +.. date: 2024-09-09-12-48-37 +.. gh-issue: 123853 +.. nonce: e-zFxb +.. section: Library + +Update the table of Windows language code identifiers (LCIDs) used by +:func:`locale.getdefaultlocale` on Windows to protocol version 16.0 +(2024-04-23). + +.. + +.. date: 2023-02-26-14-07-18 +.. gh-issue: 91099 +.. nonce: _QPbEL +.. section: Library + +:meth:`imaplib.IMAP4.login` now raises exceptions with :class:`str` instead +of :class:`bytes`. Patch by Florian Best. + +.. + +.. date: 2026-05-23-17-27-41 +.. gh-issue: 150319 +.. nonce: ol9tWK +.. section: Documentation + +Generic builtin and standard library types now document the meaning of their +type parameters. + +.. + +.. date: 2023-09-16-23-42-27 +.. gh-issue: 109503 +.. nonce: mZ-kdU +.. section: Documentation + +Fix documentation for :func:`shutil.move` on usage of :func:`os.rename` +since nonatomic move might be used even if the files are on the same +filesystem. Patch by Fang Li + +.. + +.. date: 2026-06-09-11-52-52 +.. gh-issue: 151130 +.. nonce: 1vslPH +.. section: Tests + +Add more tests for ``PyWeakref_*`` C API. + +.. + +.. date: 2026-05-13-14-53-23 +.. gh-issue: 149776 +.. nonce: orqgsn +.. section: Tests + +Fix test_socket on Linux kernel 7.1 and newer: skip UDP Lite tests if it's +not supported. Patch by Victor Stinner. + +.. + +.. date: 2026-05-21-15-14-59 +.. gh-issue: 148294 +.. nonce: VtFaW4 +.. section: Build + +Corrected the use of ``AC_PATH_TOOL`` in ``configure.ac`` to allow a C++ +compiler to be found on :envvar:`!PATH`. + +.. + +.. date: 2026-06-09-11-40-48 +.. gh-issue: 151159 +.. nonce: JKVfme +.. section: Windows + +Updated bundled version of OpenSSL to 3.5.7. + +.. + +.. date: 2026-06-09-11-33-51 +.. gh-issue: 151159 +.. nonce: ds-9f8 +.. section: macOS + +Update macOS installer to use OpenSSL 3.5.7. + +.. + +.. date: 2026-05-31-10-40-00 +.. gh-issue: 150644 +.. nonce: zLWyjj +.. section: macOS + +When system logging is enabled (with ``config.use_system_logger``, messages +are now tagged as public. This allows the macOS 26 system logger to view +messages without special configuration. + +.. + +.. date: 2025-10-14-00-17-48 +.. gh-issue: 115119 +.. nonce: 470I1N +.. section: macOS + +Update macOS installer to use libmpdecimal 4.0.1. + +.. + +.. bpo: 6699 +.. date: 2019-12-12-03-18-02 +.. nonce: 1CqJFG +.. section: IDLE + +Warn the user if a file will be overwritten when saving. + +.. + +.. date: 2026-06-04-14-26-17 +.. gh-issue: 150907 +.. nonce: CA91_B +.. section: C API + +Fix ``dynamic_annotations.h`` header file when built with C++ and Valgrind: +add ``extern "C++" scope`` for the C++ template. Patch by Victor Stinner. + +.. + +.. date: 2026-02-25-13-37-10 +.. gh-issue: 145235 +.. nonce: -1ySNR +.. section: C API + +Made :c:func:`PyDict_AddWatcher`, :c:func:`PyDict_ClearWatcher`, +:c:func:`PyDict_Watch`, and :c:func:`PyDict_Unwatch` thread-safe on the +:term:`free threaded <free threading>` build. diff --git a/Misc/NEWS.d/3.7.0a4.rst b/Misc/NEWS.d/3.7.0a4.rst index 2ceb9e78e0421b7..691ac0f4a8cec83 100644 --- a/Misc/NEWS.d/3.7.0a4.rst +++ b/Misc/NEWS.d/3.7.0a4.rst @@ -403,7 +403,7 @@ SOCK_CLOEXEC. .. nonce: zmO8G2 .. section: Library -Add :class:`importlib.abc.ResourceReader` as an ABC for loaders to provide a +Add :class:`!importlib.abc.ResourceReader` as an ABC for loaders to provide a unified API for reading resources contained within packages. Also add :mod:`importlib.resources` as the port of ``importlib_resources``. diff --git a/Misc/NEWS.d/3.7.0b1.rst b/Misc/NEWS.d/3.7.0b1.rst index c9786e55c207390..d785f6d8c4c8004 100644 --- a/Misc/NEWS.d/3.7.0b1.rst +++ b/Misc/NEWS.d/3.7.0b1.rst @@ -598,7 +598,7 @@ Add socket.getblocking() method. .. nonce: zmO8G2 .. section: Library -Add :mod:`importlib.resources` and :class:`importlib.abc.ResourceReader` as +Add :mod:`importlib.resources` and :class:`!importlib.abc.ResourceReader` as the unified API for reading resources contained within packages. Loaders wishing to support resource reading must implement the :meth:`get_resource_reader` method. File-based and zipimport-based diff --git a/Misc/NEWS.d/3.7.0b4.rst b/Misc/NEWS.d/3.7.0b4.rst index 93627f54900ddd6..789d96dffd666f4 100644 --- a/Misc/NEWS.d/3.7.0b4.rst +++ b/Misc/NEWS.d/3.7.0b4.rst @@ -152,7 +152,7 @@ Ensure line-endings are respected when using lib2to3. .. section: Library Have :func:`importlib.resources.contents` and -:meth:`importlib.abc.ResourceReader.contents` return an :term:`iterable` +:meth:`!importlib.abc.ResourceReader.contents` return an :term:`iterable` instead of an :term:`iterator`. .. diff --git a/Misc/NEWS.d/3.8.0a1.rst b/Misc/NEWS.d/3.8.0a1.rst index 93995bc8feaad76..3a6966deb878252 100644 --- a/Misc/NEWS.d/3.8.0a1.rst +++ b/Misc/NEWS.d/3.8.0a1.rst @@ -5130,7 +5130,7 @@ Ensure line-endings are respected when using lib2to3. .. section: Library Have :func:`importlib.resources.contents` and -:meth:`importlib.abc.ResourceReader.contents` return an :term:`iterable` +:meth:`!importlib.abc.ResourceReader.contents` return an :term:`iterable` instead of an :term:`iterator`. .. diff --git a/Misc/NEWS.d/next/Build/2026-05-18-16-00-41.gh-issue-148260.UwFiIX.rst b/Misc/NEWS.d/next/Build/2026-05-18-16-00-41.gh-issue-148260.UwFiIX.rst new file mode 100644 index 000000000000000..8248c24cbd511ac --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-05-18-16-00-41.gh-issue-148260.UwFiIX.rst @@ -0,0 +1,3 @@ +On Linux when Python is linked to the musl C library, use a thread stack +size of at least 1 MiB instead of musl default which is 128 kiB. Patch by +Victor Stinner. diff --git a/Misc/NEWS.d/next/Build/2026-06-28-16-17-55.gh-issue-152502.6SNqMg.rst b/Misc/NEWS.d/next/Build/2026-06-28-16-17-55.gh-issue-152502.6SNqMg.rst new file mode 100644 index 000000000000000..cc0fabed10f1261 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-06-28-16-17-55.gh-issue-152502.6SNqMg.rst @@ -0,0 +1,5 @@ +The :mod:`curses` module now detects ``set_escdelay()``, ``set_tabsize()`` and +the ``ESCDELAY`` and ``TABSIZE`` variables with :program:`configure` capability +probes instead of the ncurses-specific ``NCURSES_EXT_FUNCS`` macro, so they are +exposed when building against other curses implementations such as NetBSD curses +that provide them. diff --git a/Misc/NEWS.d/next/Build/2026-07-09-22-45-00.gh-issue-153438.Qr7N2p.rst b/Misc/NEWS.d/next/Build/2026-07-09-22-45-00.gh-issue-153438.Qr7N2p.rst new file mode 100644 index 000000000000000..edab8e6ba7b2595 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-07-09-22-45-00.gh-issue-153438.Qr7N2p.rst @@ -0,0 +1,2 @@ +Update Windows build and installer tooling and documentation to use the +current download URL for ``nuget.exe``. diff --git a/Misc/NEWS.d/next/Build/2026-07-11-07-26-16.gh-issue-126877.Zt7Kq2.rst b/Misc/NEWS.d/next/Build/2026-07-11-07-26-16.gh-issue-126877.Zt7Kq2.rst new file mode 100644 index 000000000000000..1d7d4fad886c56f --- /dev/null +++ b/Misc/NEWS.d/next/Build/2026-07-11-07-26-16.gh-issue-126877.Zt7Kq2.rst @@ -0,0 +1,2 @@ +Fix the :program:`configure` check for Tcl/Tk which could wrongly succeed +with optimizing compilers when the libraries are missing. diff --git a/Misc/NEWS.d/next/C_API/2026-06-10-16-43-37.gh-issue-123619.dV82r6.rst b/Misc/NEWS.d/next/C_API/2026-06-10-16-43-37.gh-issue-123619.dV82r6.rst new file mode 100644 index 000000000000000..4d4c94563330c06 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-06-10-16-43-37.gh-issue-123619.dV82r6.rst @@ -0,0 +1,3 @@ +:c:func:`PyUnstable_Object_EnableDeferredRefcount` now returns ``0`` if the +object is not tracked by the garbage collector: if :func:`gc.is_tracked` is +false. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2026-07-08-00-38-32.gh-issue-153300.lVoWyR.rst b/Misc/NEWS.d/next/C_API/2026-07-08-00-38-32.gh-issue-153300.lVoWyR.rst new file mode 100644 index 000000000000000..3c19c0e15d2bc65 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-07-08-00-38-32.gh-issue-153300.lVoWyR.rst @@ -0,0 +1,3 @@ +:c:func:`PyConfig_Set()` now also set global configuration variables. For +example, ``PyConfig_Set("inspect", value)`` now also sets +:c:var:`Py_InspectFlag`. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/C_API/2026-07-09-17-33-29.gh-issue-152132.Ma8MJZ.rst b/Misc/NEWS.d/next/C_API/2026-07-09-17-33-29.gh-issue-152132.Ma8MJZ.rst new file mode 100644 index 000000000000000..633c41e40bf2696 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2026-07-09-17-33-29.gh-issue-152132.Ma8MJZ.rst @@ -0,0 +1,3 @@ +Fix :c:func:`Py_RunMain` to return an exit code, rather than calling +:c:func:`Py_Exit`, when running a script, a command, or the REPL. Patch by +Victor Stinner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-29-15-10-59.gh-issue-149162.BPPyrq.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-29-15-10-59.gh-issue-149162.BPPyrq.rst new file mode 100644 index 000000000000000..b4c443ecee07adf --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-29-15-10-59.gh-issue-149162.BPPyrq.rst @@ -0,0 +1,3 @@ +Fix a potential deadlock in :c:func:`PyUnicode_InternFromString` and other +interning functions in the :term:`free-threaded build` when called from C++ +static local initializers. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-11-15-58-23.gh-issue-149689.9Ht49z.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-11-15-58-23.gh-issue-149689.9Ht49z.rst new file mode 100644 index 000000000000000..aa672760730bd43 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-11-15-58-23.gh-issue-149689.9Ht49z.rst @@ -0,0 +1,2 @@ +Fix missing error propagation in parser action helpers when memory allocation +fails. Patch by Thomas Kowalski. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-26-00-06-30.gh-issue-150411.u-d-_5.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-26-00-06-30.gh-issue-150411.u-d-_5.rst new file mode 100644 index 000000000000000..5b19a4fff5ddc73 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-26-00-06-30.gh-issue-150411.u-d-_5.rst @@ -0,0 +1,2 @@ +Fix a data race in the free-threaded build when :func:`gc.get_count` reads +the young generation allocation count while another thread updates it. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-05-29-17-51-11.gh-issue-144774.jPcixe.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-29-17-51-11.gh-issue-144774.jPcixe.rst new file mode 100644 index 000000000000000..3dd117e7d3c3502 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-05-29-17-51-11.gh-issue-144774.jPcixe.rst @@ -0,0 +1 @@ +Fix data race in :class:`BaseException` when an exception is copied while being mutated. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-05-22-52-41.gh-issue-150988.fDKfMJ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-05-22-52-41.gh-issue-150988.fDKfMJ.rst new file mode 100644 index 000000000000000..6fb70a1ce2685c8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-05-22-52-41.gh-issue-150988.fDKfMJ.rst @@ -0,0 +1,2 @@ +Fix a reference leak in :exc:`OSError` when attributes are set before +``super().__init__()``. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-06-20-15-08.gh-issue-151029.A33CKK.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-06-20-15-08.gh-issue-151029.A33CKK.rst new file mode 100644 index 000000000000000..cbfe5952627ad8a --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-06-20-15-08.gh-issue-151029.A33CKK.rst @@ -0,0 +1,2 @@ +On Linux, fix :func:`sys.remote_exec` unable to find remote writable memory +when ``libpython`` replaced on disk. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-08-05-31-22.gh-issue-151065._o_31F.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-08-05-31-22.gh-issue-151065._o_31F.rst new file mode 100644 index 000000000000000..e46c96ef784cc9e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-08-05-31-22.gh-issue-151065._o_31F.rst @@ -0,0 +1 @@ +Fix memory leak when using the :ref:`mimalloc memory allocator <mimalloc>`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-08-13-14-42.gh-issue-150902.-CWZ66.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-08-13-14-42.gh-issue-150902.-CWZ66.rst new file mode 100644 index 000000000000000..e3b7cd387b439fe --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-08-13-14-42.gh-issue-150902.-CWZ66.rst @@ -0,0 +1 @@ +Apply an existing optimization of PyCriticalSection (single mutex) to PyCriticalSection2: avoid acquiring the same locks that the current CS has already acquired. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-09-10-28-30.gh-issue-151126.DKa6Sl.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-09-10-28-30.gh-issue-151126.DKa6Sl.rst new file mode 100644 index 000000000000000..c91939dbe559cd5 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-09-10-28-30.gh-issue-151126.DKa6Sl.rst @@ -0,0 +1,4 @@ +Fix a crash, when there's no memory left on a device, +which happened in :mod:`!_interpchannels` module. + +Now it raises proper :exc:`MemoryError` errors. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-10-15-19-58.gh-issue-151238.C9Wu4x.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-10-15-19-58.gh-issue-151238.C9Wu4x.rst new file mode 100644 index 000000000000000..fe7519fb4878952 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-10-15-19-58.gh-issue-151238.C9Wu4x.rst @@ -0,0 +1,2 @@ +Fix a crash when compiling a concatenated f-string or t-string if an error +occurs when processing one of it's parts. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-10-15-42-46.gh-issue-151253.7MMQ8P.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-10-15-42-46.gh-issue-151253.7MMQ8P.rst new file mode 100644 index 000000000000000..56d2f3b2633bb01 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-10-15-42-46.gh-issue-151253.7MMQ8P.rst @@ -0,0 +1,3 @@ +If ``import encodings`` (first import) fails at Python startup, dump the +Python path configuration to help users debugging their configuration. Patch +by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-12-15-30-25.gh-issue-151218.5M_nv8.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-12-15-30-25.gh-issue-151218.5M_nv8.rst new file mode 100644 index 000000000000000..46539efc373eb0d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-12-15-30-25.gh-issue-151218.5M_nv8.rst @@ -0,0 +1,3 @@ +:c:func:`PyConfig_Set` and :func:`sys.set_int_max_str_digits` now replace +:data:`sys.flags` (create a new object), instead of modifying :data:`sys.flags` +in-place. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-14-05-05-15.gh-issue-151461.5q0s88.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-14-05-05-15.gh-issue-151461.5q0s88.rst new file mode 100644 index 000000000000000..d76a9bc95278bcb --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-14-05-05-15.gh-issue-151461.5q0s88.rst @@ -0,0 +1,3 @@ +Fix direct execution of files with invalid source encodings to report the +underlying codec lookup or decoding error instead of the generic +``SyntaxError: encoding problem`` message. Patch by Bartosz Sławecki. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-16-17-23-37.gh-issue-151546.LhiaZz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-16-17-23-37.gh-issue-151546.LhiaZz.rst new file mode 100644 index 000000000000000..af1c23bd50355f2 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-16-17-23-37.gh-issue-151546.LhiaZz.rst @@ -0,0 +1,3 @@ +Fix the stack limit check if Python is linked to musl (ex: Alpine Linux). +Use the stack size set by the linker to compute the stack limits. Patch by +Victor Stinner. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-17-16-46-07.gh-issue-151126.vhTL0T.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-17-16-46-07.gh-issue-151126.vhTL0T.rst new file mode 100644 index 000000000000000..6f2d230b1dcfc00 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-17-16-46-07.gh-issue-151126.vhTL0T.rst @@ -0,0 +1,2 @@ +Avoid possible crash in ``_winapi.c`` where a device has no memory left. Now +it properly raises a :exc:`MemoryError`. Patch by Ivy Xu. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-18-00-00-00.gh-issue-151644.5cFffN.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-18-00-00-00.gh-issue-151644.5cFffN.rst new file mode 100644 index 000000000000000..ff5058e14b42f15 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-18-00-00-00.gh-issue-151644.5cFffN.rst @@ -0,0 +1,4 @@ +Fix a data race in :func:`sys.setdlopenflags` and :func:`sys.getdlopenflags` +when called concurrently in the free-threaded build. The underlying +``_PyImport_GetDLOpenFlags`` and ``_PyImport_SetDLOpenFlags`` functions now +use atomic load/store operations. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-18-16-00-10.gh-issue-151126.tBqn6I.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-18-16-00-10.gh-issue-151126.tBqn6I.rst new file mode 100644 index 000000000000000..d495df43ede932c --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-18-16-00-10.gh-issue-151126.tBqn6I.rst @@ -0,0 +1,3 @@ +Fix a crash when sharing :class:`memoryview` objects between interpreters +fails due to running out of memory. It now raises a proper +:exc:`MemoryError`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-21-16-00-00.gh-issue-151773.mN4kRt.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-21-16-00-00.gh-issue-151773.mN4kRt.rst new file mode 100644 index 000000000000000..b4193c5e15ffef8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-21-16-00-00.gh-issue-151773.mN4kRt.rst @@ -0,0 +1,2 @@ +Fix a crash in :func:`contextvars.ContextVar.set` when memory allocation +fails. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-06-26-34.gh-issue-151905.FOLMYg.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-06-26-34.gh-issue-151905.FOLMYg.rst new file mode 100644 index 000000000000000..c71122df6b8580f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-06-26-34.gh-issue-151905.FOLMYg.rst @@ -0,0 +1 @@ +Fix OOM error handling in :c:func:`PyFrame_GetBack` to propagate exceptions instead of masking them as None. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-14-48-22.gh-issue-151912.YcxfnU.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-14-48-22.gh-issue-151912.YcxfnU.rst new file mode 100644 index 000000000000000..07b73949b438a33 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-22-14-48-22.gh-issue-151912.YcxfnU.rst @@ -0,0 +1 @@ +Fixed a crash in ``type()`` when selecting a metaclass whose ``tp_new`` slot is ``NULL``. Such metaclasses are now rejected with ``TypeError`` instead of causing a NULL pointer dereference. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-23-12-03-55.gh-issue-151763.K7QfWG.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-23-12-03-55.gh-issue-151763.K7QfWG.rst new file mode 100644 index 000000000000000..64d0146cb091016 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-23-12-03-55.gh-issue-151763.K7QfWG.rst @@ -0,0 +1,3 @@ +Fix a potential crash in :func:`compile`, :func:`exec`, :func:`eval` and +:func:`ast.parse` when an allocation fails: the parser or compiler could +return without setting an exception. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-23-19-50-22.gh-issue-152020.DTKXjR.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-23-19-50-22.gh-issue-152020.DTKXjR.rst new file mode 100644 index 000000000000000..93c716f7a6a1c86 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-23-19-50-22.gh-issue-152020.DTKXjR.rst @@ -0,0 +1,3 @@ +On the free-threaded build, :func:`asyncio.all_tasks` no longer loses +eager-started tasks when called from a thread other than the one running the +event loop. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-23-23-48-54.gh-issue-151763.Eu8pYQ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-23-23-48-54.gh-issue-151763.Eu8pYQ.rst new file mode 100644 index 000000000000000..d4746e992f8779d --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-23-23-48-54.gh-issue-151763.Eu8pYQ.rst @@ -0,0 +1 @@ +Fixes possible crash on :class:`types.CodeType` deallocation. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-25-21-34-15.gh-issue-152228.a6K14K.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-25-21-34-15.gh-issue-152228.a6K14K.rst new file mode 100644 index 000000000000000..8af7ae0d1739130 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-25-21-34-15.gh-issue-152228.a6K14K.rst @@ -0,0 +1,2 @@ +Fix an assertion failure when python is built in a debug mode +that happened in :meth:`str.replace` under a limited memory situation. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-26-05-49-13.gh-issue-152235.YU20T9.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-26-05-49-13.gh-issue-152235.YU20T9.rst new file mode 100644 index 000000000000000..8d386ad458dfff1 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-26-05-49-13.gh-issue-152235.YU20T9.rst @@ -0,0 +1,2 @@ +Defer GC tracking of a :class:`set` or :class:`frozenset` to the end of its +construction from iterable. Patch by Donghee Na. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-26-22-03-16.gh-issue-152235.ZKWiWk.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-26-22-03-16.gh-issue-152235.ZKWiWk.rst new file mode 100644 index 000000000000000..ff89a2445cb4fe8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-26-22-03-16.gh-issue-152235.ZKWiWk.rst @@ -0,0 +1,3 @@ +Defer GC tracking of :meth:`set.intersection`, :meth:`set.difference`, +:meth:`set.symmetric_difference`, :meth:`set.union` and ``set.__sub__``. +Patch by Donghee Na. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-27-10-05-12.gh-issue-152375.L-ZBk6.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-27-10-05-12.gh-issue-152375.L-ZBk6.rst new file mode 100644 index 000000000000000..db6ae3060d8328f --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-27-10-05-12.gh-issue-152375.L-ZBk6.rst @@ -0,0 +1,2 @@ +Fix undefined behaviour when a :mod:`sys.monitoring` callback raised an +exception while the program was following a branch or loop. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-29-23-29-14.gh-issue-152635.O21J0O.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-29-23-29-14.gh-issue-152635.O21J0O.rst new file mode 100644 index 000000000000000..dadb25fc7c44771 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-29-23-29-14.gh-issue-152635.O21J0O.rst @@ -0,0 +1,2 @@ +Fix a crash caused when running out of memory creating a +:mod:`!_interpchannels` channel. Now a :exc:`MemoryError` is correctly raised. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-30-14-00-00.gh-issue-152682.yId7e5.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-30-14-00-00.gh-issue-152682.yId7e5.rst new file mode 100644 index 000000000000000..aba0b594382bcf8 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-30-14-00-00.gh-issue-152682.yId7e5.rst @@ -0,0 +1,3 @@ +Fix NULL pointer dereference in :func:`compile` when a reserved name (e.g. +``__classdict__``) is used as a type parameter name and memory allocation +fails while formatting the error message. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-07-04-16-48.gh-issue-153205.5fE8QZ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-07-04-16-48.gh-issue-153205.5fE8QZ.rst new file mode 100644 index 000000000000000..9ccfbff7ee69778 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-07-04-16-48.gh-issue-153205.5fE8QZ.rst @@ -0,0 +1,2 @@ +Fix a potential :exc:`SystemError` during vector calls when memory allocation fails. +A :exc:`MemoryError` is now raised instead. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-08-13-23-11.gh-issue-153298.wvcXxN.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-08-13-23-11.gh-issue-153298.wvcXxN.rst new file mode 100644 index 000000000000000..38d548834d5e02e --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-08-13-23-11.gh-issue-153298.wvcXxN.rst @@ -0,0 +1,2 @@ +Fixes a data race in :class:`types.GenericAlias` ``__parameters__`` +initialization on free-threading builds. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-19-12-33-27.gh-issue-153932.lKw2bo.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-19-12-33-27.gh-issue-153932.lKw2bo.rst new file mode 100644 index 000000000000000..56b96307ed04cd9 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-19-12-33-27.gh-issue-153932.lKw2bo.rst @@ -0,0 +1,2 @@ +Fix thread safety issue in the ``__reduce__`` method of +:py:class:`enumerate`. diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-07-20-20-16-08.gh-issue-154275.l961cA.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-20-20-16-08.gh-issue-154275.l961cA.rst new file mode 100644 index 000000000000000..998e91c77098e75 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-07-20-20-16-08.gh-issue-154275.l961cA.rst @@ -0,0 +1,2 @@ +Fix a crash when getting deeply nested ``__parameters__`` from a +:class:`types.GenericAlias` objects. diff --git a/Misc/NEWS.d/next/Documentation/2026-06-17-12-00-00.gh-issue-86726.__bOgH.rst b/Misc/NEWS.d/next/Documentation/2026-06-17-12-00-00.gh-issue-86726.__bOgH.rst new file mode 100644 index 000000000000000..787b95dbf236279 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2026-06-17-12-00-00.gh-issue-86726.__bOgH.rst @@ -0,0 +1,4 @@ +Greatly expand the :mod:`tkinter` documentation to cover the full public API +of the package and its submodules. The descriptions are oriented towards +Python rather than Tcl/Tk, with corrected return types and +``versionadded``/``versionchanged`` information. diff --git a/Misc/NEWS.d/next/Documentation/2026-07-17-22-33-43.gh-issue-118150.m7iFdP.rst b/Misc/NEWS.d/next/Documentation/2026-07-17-22-33-43.gh-issue-118150.m7iFdP.rst new file mode 100644 index 000000000000000..afa6007a46e1745 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2026-07-17-22-33-43.gh-issue-118150.m7iFdP.rst @@ -0,0 +1,2 @@ +Clarify in the :mod:`difflib` documentation what *junk* actually does, its +drawbacks, and how to control it. diff --git a/Misc/NEWS.d/next/IDLE/2021-10-03-21-55-34.gh-issue-89520.etEExa.rst b/Misc/NEWS.d/next/IDLE/2021-10-03-21-55-34.gh-issue-89520.etEExa.rst new file mode 100644 index 000000000000000..e8e181cac213589 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2021-10-03-21-55-34.gh-issue-89520.etEExa.rst @@ -0,0 +1,3 @@ +Make IDLE extension configuration look at user config files, allowing +user-installed extensions to have settings and key bindings defined in +~/.idlerc. diff --git a/Misc/NEWS.d/next/IDLE/2025-10-05-19-33-39.gh-issue-139551.TX9BRc.rst b/Misc/NEWS.d/next/IDLE/2025-10-05-19-33-39.gh-issue-139551.TX9BRc.rst new file mode 100644 index 000000000000000..5ea1dfc9b5100d3 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2025-10-05-19-33-39.gh-issue-139551.TX9BRc.rst @@ -0,0 +1 @@ +Support rendering :exc:`BaseExceptionGroup` in IDLE. diff --git a/Misc/NEWS.d/next/IDLE/2026-03-07-20-47-40.gh-issue-94523.dq7m2k.rst b/Misc/NEWS.d/next/IDLE/2026-03-07-20-47-40.gh-issue-94523.dq7m2k.rst new file mode 100644 index 000000000000000..831f6ac628768c7 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-03-07-20-47-40.gh-issue-94523.dq7m2k.rst @@ -0,0 +1,2 @@ +Detect file if modified at local disk and prompt to ask refresh. Patch by +Shixian Li. diff --git a/Misc/NEWS.d/next/IDLE/2026-06-28-06-46-46.gh-issue-85320.Hq2vKn.rst b/Misc/NEWS.d/next/IDLE/2026-06-28-06-46-46.gh-issue-85320.Hq2vKn.rst new file mode 100644 index 000000000000000..ec34bf8ead9eb9f --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-06-28-06-46-46.gh-issue-85320.Hq2vKn.rst @@ -0,0 +1,4 @@ +IDLE now reads and writes its configuration files and the breakpoints file +using UTF-8 instead of the locale encoding. This keeps non-ASCII data (such +as non-ASCII paths) from being corrupted and makes the files portable between +environments. diff --git a/Misc/NEWS.d/next/IDLE/2026-07-01-00-00-00.gh-issue-66331.wMcLaS.rst b/Misc/NEWS.d/next/IDLE/2026-07-01-00-00-00.gh-issue-66331.wMcLaS.rst new file mode 100644 index 000000000000000..e00e7c7d8fdc652 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-07-01-00-00-00.gh-issue-66331.wMcLaS.rst @@ -0,0 +1,3 @@ +Set the ``WM_CLASS`` window property of IDLE's windows to ``Idle`` on X11, +so that window managers group and label them correctly instead of using the +default ``Toplevel``. diff --git a/Misc/NEWS.d/next/IDLE/2026-07-01-00-15-58.gh-issue-152728.yxIhMN.rst b/Misc/NEWS.d/next/IDLE/2026-07-01-00-15-58.gh-issue-152728.yxIhMN.rst new file mode 100644 index 000000000000000..d73512a615c1898 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-07-01-00-15-58.gh-issue-152728.yxIhMN.rst @@ -0,0 +1,2 @@ +Move functions run.fix_scaling, editor.fixwordbreaks (as fix_word_breaks) +and pyshell.fix_x11_paste to idlelib.util. diff --git a/Misc/NEWS.d/next/IDLE/2026-07-01-12-00-00.gh-issue-71956.rPlAlL.rst b/Misc/NEWS.d/next/IDLE/2026-07-01-12-00-00.gh-issue-71956.rPlAlL.rst new file mode 100644 index 000000000000000..1a92ed4cd389cf5 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-07-01-12-00-00.gh-issue-71956.rPlAlL.rst @@ -0,0 +1,3 @@ +Fix Replace All in the IDLE editor's Replace dialog when the search +direction is "Up" and "Wrap around" is off: it now replaces all matches +above the current position instead of only the first one. diff --git a/Misc/NEWS.d/next/IDLE/2026-07-01-14-00-00.gh-issue-89360.mUlTiC.rst b/Misc/NEWS.d/next/IDLE/2026-07-01-14-00-00.gh-issue-89360.mUlTiC.rst new file mode 100644 index 000000000000000..9e3023f8228233f --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-07-01-14-00-00.gh-issue-89360.mUlTiC.rst @@ -0,0 +1,3 @@ +Fix a rare crash in the IDLE editor when the completion window is closed: +deleting a key binding for a sequence that is not bound to the virtual +event is now ignored instead of raising a ``ValueError``. diff --git a/Misc/NEWS.d/next/IDLE/2026-07-01-15-00-00.gh-issue-134300.iDlPaT.rst b/Misc/NEWS.d/next/IDLE/2026-07-01-15-00-00.gh-issue-134300.iDlPaT.rst new file mode 100644 index 000000000000000..791acbbed62aaf3 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-07-01-15-00-00.gh-issue-134300.iDlPaT.rst @@ -0,0 +1,3 @@ +Do not add the ``idlelib`` directory to the path of the IDLE user process. +User code run in IDLE can no longer import ``idlelib`` submodules as +top-level modules, such as ``import help``. diff --git a/Misc/NEWS.d/next/IDLE/2026-07-01-16-00-00.gh-issue-80504.gReP.rst b/Misc/NEWS.d/next/IDLE/2026-07-01-16-00-00.gh-issue-80504.gReP.rst new file mode 100644 index 000000000000000..93adb33202a41c1 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-07-01-16-00-00.gh-issue-80504.gReP.rst @@ -0,0 +1,3 @@ +The "In files:" field of IDLE's Find in Files dialog now always contains a +full directory path, even for an unsaved editor or the Shell. This shows +in the grep output which directory was searched. diff --git a/Misc/NEWS.d/next/IDLE/2026-07-01-17-00-00.gh-issue-65339.oUtTxt.rst b/Misc/NEWS.d/next/IDLE/2026-07-01-17-00-00.gh-issue-65339.oUtTxt.rst new file mode 100644 index 000000000000000..a8b216e86dc8d7e --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-07-01-17-00-00.gh-issue-65339.oUtTxt.rst @@ -0,0 +1,3 @@ +Saving the IDLE Shell or an Output window now defaults to a ``.txt`` +extension and lists text files before Python files, since their content is +not Python source. diff --git a/Misc/NEWS.d/next/IDLE/2026-07-01-18-00-00.gh-issue-83653.cFgInt.rst b/Misc/NEWS.d/next/IDLE/2026-07-01-18-00-00.gh-issue-83653.cFgInt.rst new file mode 100644 index 000000000000000..5812b2dd977f6a9 --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-07-01-18-00-00.gh-issue-83653.cFgInt.rst @@ -0,0 +1,3 @@ +Blanking an integer entry in IDLE's Settings dialog, such as "Auto squeeze +min lines", no longer saves an empty string as an invalid configuration +value. diff --git a/Misc/NEWS.d/next/IDLE/2026-07-01-19-00-00.gh-issue-82183.rNsTrt.rst b/Misc/NEWS.d/next/IDLE/2026-07-01-19-00-00.gh-issue-82183.rNsTrt.rst new file mode 100644 index 000000000000000..2af412f5200d0de --- /dev/null +++ b/Misc/NEWS.d/next/IDLE/2026-07-01-19-00-00.gh-issue-82183.rNsTrt.rst @@ -0,0 +1,3 @@ +When the shell is busy running code, using "Run... Customized" with "Restart +shell" unchecked now reports that the shell is executing instead of +restarting it anyway. diff --git a/Misc/NEWS.d/next/Library/2023-01-23-21-23-50.gh-issue-101267._f-cFH.rst b/Misc/NEWS.d/next/Library/2023-01-23-21-23-50.gh-issue-101267._f-cFH.rst new file mode 100644 index 000000000000000..901a3fb60ab5b9f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-01-23-21-23-50.gh-issue-101267._f-cFH.rst @@ -0,0 +1,7 @@ +When a worker process terminates unexpectedly, +:class:`concurrent.futures.ProcessPoolExecutor` now sets a separate +:exc:`~concurrent.futures.process.BrokenProcessPool` exception on each pending +future instead of sharing a single instance among them all. Sharing one +exception produced malformed tracebacks: each +:meth:`Future.result() <concurrent.futures.Future.result>` call re-raised the +same object, appending another copy of the traceback to it. diff --git a/Misc/NEWS.d/next/Library/2023-04-27-14-50-03.gh-issue-103925.khC-El.rst b/Misc/NEWS.d/next/Library/2023-04-27-14-50-03.gh-issue-103925.khC-El.rst new file mode 100644 index 000000000000000..858ce6e5bd98e70 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-04-27-14-50-03.gh-issue-103925.khC-El.rst @@ -0,0 +1,3 @@ +Fix :meth:`csv.Sniffer.sniff` for a sample with ``\r\n`` line endings in +which a quoted field ends a line: a letter could be detected as the +delimiter. diff --git a/Misc/NEWS.d/next/Library/2023-06-12-21-31-02.gh-issue-105708.5fFwP8.rst b/Misc/NEWS.d/next/Library/2023-06-12-21-31-02.gh-issue-105708.5fFwP8.rst new file mode 100644 index 000000000000000..79289c1f0bd4217 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-06-12-21-31-02.gh-issue-105708.5fFwP8.rst @@ -0,0 +1 @@ +Accept an uppercase V prefix in IPvFuture addresses in :func:`urllib.parse.urlsplit`. diff --git a/Misc/NEWS.d/next/Library/2024-06-18-04-08-37.gh-issue-120665.x7T1hV.rst b/Misc/NEWS.d/next/Library/2024-06-18-04-08-37.gh-issue-120665.x7T1hV.rst new file mode 100644 index 000000000000000..27e93988ed11efb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-06-18-04-08-37.gh-issue-120665.x7T1hV.rst @@ -0,0 +1 @@ +Fixed an issue where ``unittest`` loaders would load and instantiate :class:`unittest.TestCase`-derived subclasses that are also abstract base classes, which can't be instantiated. diff --git a/Misc/NEWS.d/next/Library/2025-03-13-20-48-58.gh-issue-123471.cM4w4f.rst b/Misc/NEWS.d/next/Library/2025-03-13-20-48-58.gh-issue-123471.cM4w4f.rst new file mode 100644 index 000000000000000..cfc783900de70fe --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-03-13-20-48-58.gh-issue-123471.cM4w4f.rst @@ -0,0 +1 @@ +Make concurrent iterations over :class:`itertools.cycle` safe under free-threading. diff --git a/Misc/NEWS.d/next/Library/2025-04-22-21-00-23.gh-issue-123471.asOLA2.rst b/Misc/NEWS.d/next/Library/2025-04-22-21-00-23.gh-issue-123471.asOLA2.rst new file mode 100644 index 000000000000000..a4b4b6d2c23d495 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-04-22-21-00-23.gh-issue-123471.asOLA2.rst @@ -0,0 +1 @@ +Make concurrent iterations over :class:`itertools.combinations` and :class:`itertools.product` safe under free-threading. diff --git a/Misc/NEWS.d/next/Library/2025-06-18-19-25-32.gh-issue-123471.lx1Xbt.rst b/Misc/NEWS.d/next/Library/2025-06-18-19-25-32.gh-issue-123471.lx1Xbt.rst new file mode 100644 index 000000000000000..6f395024a9e1790 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-18-19-25-32.gh-issue-123471.lx1Xbt.rst @@ -0,0 +1 @@ +Make concurrent iterations over :class:`itertools.chain` safe under :term:`free threading`. diff --git a/Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst b/Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst new file mode 100644 index 000000000000000..bd9fff0bc2e31b6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-09-05-20-50-35.gh-issue-79638.Y-JfaH.rst @@ -0,0 +1,2 @@ +Disallow all access in :mod:`urllib.robotparser` if the ``robots.txt`` file +is unreachable due to server or network errors. diff --git a/Misc/NEWS.d/next/Library/2025-11-02-05-28-56.gh-issue-115634.JbcNnF.rst b/Misc/NEWS.d/next/Library/2025-11-02-05-28-56.gh-issue-115634.JbcNnF.rst new file mode 100644 index 000000000000000..f023ce999c60b7c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-11-02-05-28-56.gh-issue-115634.JbcNnF.rst @@ -0,0 +1,5 @@ +Fix a deadlock in :class:`concurrent.futures.ProcessPoolExecutor` when +using ``max_tasks_per_child``, present since the feature was introduced in +Python 3.11. The executor stopped scheduling queued tasks after a worker +process exited upon reaching its task limit. Based on a fix proposed by +Tabrez Mohammed. diff --git a/Misc/NEWS.d/next/Library/2025-12-23-11-43-05.gh-issue-130796.TkzUGx.rst b/Misc/NEWS.d/next/Library/2025-12-23-11-43-05.gh-issue-130796.TkzUGx.rst new file mode 100644 index 000000000000000..a078561a1014faa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-12-23-11-43-05.gh-issue-130796.TkzUGx.rst @@ -0,0 +1,2 @@ +Undeprecate the :func:`locale.getdefaultlocale` function. +Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2026-01-18-06-42-47.gh-issue-143988.MtLtCP.rst b/Misc/NEWS.d/next/Library/2026-01-18-06-42-47.gh-issue-143988.MtLtCP.rst new file mode 100644 index 000000000000000..fcc0cb54934b90e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-01-18-06-42-47.gh-issue-143988.MtLtCP.rst @@ -0,0 +1,2 @@ +Fixed crashes in :meth:`socket.socket.sendmsg` and :meth:`socket.socket.recvmsg_into` +that could occur if buffer sequences are concurrently mutated. diff --git a/Misc/NEWS.d/next/Library/2026-02-03-08-50-58.gh-issue-123471.yF1Gym.rst b/Misc/NEWS.d/next/Library/2026-02-03-08-50-58.gh-issue-123471.yF1Gym.rst new file mode 100644 index 000000000000000..85e9a03426e1fca --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-02-03-08-50-58.gh-issue-123471.yF1Gym.rst @@ -0,0 +1 @@ +Make concurrent iteration over :class:`itertools.combinations_with_replacement` and :class:`itertools.permutations` safe under free-threading. diff --git a/Misc/NEWS.d/next/Library/2026-02-04-20-30-59.gh-issue-123471.1dnPvs.rst b/Misc/NEWS.d/next/Library/2026-02-04-20-30-59.gh-issue-123471.1dnPvs.rst new file mode 100644 index 000000000000000..d650103e28ee686 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-02-04-20-30-59.gh-issue-123471.1dnPvs.rst @@ -0,0 +1 @@ +Make concurrent iteration over :class:`itertools.accumulate` safe under free-threading. diff --git a/Misc/NEWS.d/next/Library/2026-03-17-19-51-05.gh-issue-123471.oY4UR5.rst b/Misc/NEWS.d/next/Library/2026-03-17-19-51-05.gh-issue-123471.oY4UR5.rst new file mode 100644 index 000000000000000..8d2e1b970e81711 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-17-19-51-05.gh-issue-123471.oY4UR5.rst @@ -0,0 +1 @@ +Make concurrent iteration over :class:`itertools.zip_longest` safe under free-threading. diff --git a/Misc/NEWS.d/next/Library/2026-03-26-00-00-00.gh-issue-47005.xxc89c.rst b/Misc/NEWS.d/next/Library/2026-03-26-00-00-00.gh-issue-47005.xxc89c.rst new file mode 100644 index 000000000000000..646367f0fa069d3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-03-26-00-00-00.gh-issue-47005.xxc89c.rst @@ -0,0 +1,4 @@ +Fix :meth:`!urllib.request.AbstractHTTPHandler.do_open` to give regular +headers set via :meth:`~urllib.request.Request.add_header` priority over +unredirected headers, consistent with :meth:`~urllib.request.Request.get_header` +and :meth:`~urllib.request.Request.header_items`. diff --git a/Misc/NEWS.d/next/Library/2026-05-06-23-13-40.gh-issue-149319.PLM93t.rst b/Misc/NEWS.d/next/Library/2026-05-06-23-13-40.gh-issue-149319.PLM93t.rst new file mode 100644 index 000000000000000..4018ec43f5a47d9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-06-23-13-40.gh-issue-149319.PLM93t.rst @@ -0,0 +1 @@ +The :mod:`asyncio` REPL now ignores :envvar:`PYTHONSTARTUP` and :envvar:`PYTHON_BASIC_REPL` when :option:`-E` or :option:`-I` is used. Patch by Jonathan Dung. diff --git a/Misc/NEWS.d/next/Library/2026-05-18-12-42-31.gh-issue-149816.F98iME.rst b/Misc/NEWS.d/next/Library/2026-05-18-12-42-31.gh-issue-149816.F98iME.rst new file mode 100644 index 000000000000000..21e3ae0df57621c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-18-12-42-31.gh-issue-149816.F98iME.rst @@ -0,0 +1,2 @@ +Fix a potential use after free condition in :func:`pickle.dumps` in free-threaded +mode when serializing lists. diff --git a/Misc/NEWS.d/next/Library/2026-05-28-00-00-00.gh-issue-150484.XxYyZz.rst b/Misc/NEWS.d/next/Library/2026-05-28-00-00-00.gh-issue-150484.XxYyZz.rst new file mode 100644 index 000000000000000..522c70058ed29d7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-28-00-00-00.gh-issue-150484.XxYyZz.rst @@ -0,0 +1 @@ +Fix :func:`unittest.mock.mock_open` ``__exit__`` raising ``TypeError`` when used with :class:`contextlib.ExitStack`. diff --git a/Misc/NEWS.d/next/Library/2026-05-31-14-39-25.gh-issue-150641.LLIhd1.rst b/Misc/NEWS.d/next/Library/2026-05-31-14-39-25.gh-issue-150641.LLIhd1.rst new file mode 100644 index 000000000000000..eba899c96fcd26b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-05-31-14-39-25.gh-issue-150641.LLIhd1.rst @@ -0,0 +1,2 @@ +Fix bug where :func:`typing.evaluate_forward_ref` with the ``STRING`` format +could leak internal names used by the annotation machinery. diff --git a/Misc/NEWS.d/next/Library/2026-06-03-19-35-32.gh-issue-150583.dedI24.rst b/Misc/NEWS.d/next/Library/2026-06-03-19-35-32.gh-issue-150583.dedI24.rst new file mode 100644 index 000000000000000..edb551b3c3c8cbd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-03-19-35-32.gh-issue-150583.dedI24.rst @@ -0,0 +1,2 @@ +Correctly set the default compression level in :mod:`compression.zstd` when passing a digested +dictionary during compression. diff --git a/Misc/NEWS.d/next/Library/2026-06-10-00-00-02.gh-issue-109940.Cx1099.rst b/Misc/NEWS.d/next/Library/2026-06-10-00-00-02.gh-issue-109940.Cx1099.rst new file mode 100644 index 000000000000000..130dc780b612864 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-10-00-00-02.gh-issue-109940.Cx1099.rst @@ -0,0 +1,2 @@ +Fix Windows :mod:`venv` activation in ``cmd.exe`` to respect +``VIRTUAL_ENV_DISABLE_PROMPT``. diff --git a/Misc/NEWS.d/next/Library/2026-06-11-00-00-00.gh-issue-151295.NQYUzW.rst b/Misc/NEWS.d/next/Library/2026-06-11-00-00-00.gh-issue-151295.NQYUzW.rst new file mode 100644 index 000000000000000..e9012f023ff7f77 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-11-00-00-00.gh-issue-151295.NQYUzW.rst @@ -0,0 +1,4 @@ +Fixed a crash (use-after-free) in :meth:`bytes.join` and +:meth:`bytearray.join` that could occur if an item's +:meth:`~object.__buffer__` concurrently mutates the sequence being joined. +The mutation is now reported as a :exc:`RuntimeError` instead. diff --git a/Misc/NEWS.d/next/Library/2026-06-11-10-00-00.gh-issue-119710.Qz7Kp2.rst b/Misc/NEWS.d/next/Library/2026-06-11-10-00-00.gh-issue-119710.Qz7Kp2.rst new file mode 100644 index 000000000000000..45f99d72ea29a2f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-11-10-00-00.gh-issue-119710.Qz7Kp2.rst @@ -0,0 +1,4 @@ +Fix :mod:`asyncio` subprocess :meth:`~asyncio.subprocess.Process.wait` +hanging when the process has exited but one of its pipes is kept open by an +inherited child process (so the pipe never reaches EOF). ``wait()`` now +returns as soon as the process exits, regardless of the pipes' state. diff --git a/Misc/NEWS.d/next/Library/2026-06-11-16-07-00.gh-issue-151126.cWw5pb.rst b/Misc/NEWS.d/next/Library/2026-06-11-16-07-00.gh-issue-151126.cWw5pb.rst new file mode 100644 index 000000000000000..f27744d42f48246 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-11-16-07-00.gh-issue-151126.cWw5pb.rst @@ -0,0 +1,4 @@ +Fix a crash, when there's no memory left on a device, which happened in +:mod:`!_interpchannels` module. + +Now it raises proper :exc:`MemoryError` errors. diff --git a/Misc/NEWS.d/next/Library/2026-06-11-16-25-38.gh-issue-151126.bh_Usy.rst b/Misc/NEWS.d/next/Library/2026-06-11-16-25-38.gh-issue-151126.bh_Usy.rst new file mode 100644 index 000000000000000..25149057aa7d092 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-11-16-25-38.gh-issue-151126.bh_Usy.rst @@ -0,0 +1,2 @@ +Fix a crash when :exc:`MemoryError` in :func:`!os._path_splitroot` +was not set properly. diff --git a/Misc/NEWS.d/next/Library/2026-06-11-21-43-24.gh-issue-151337.JSVV18.rst b/Misc/NEWS.d/next/Library/2026-06-11-21-43-24.gh-issue-151337.JSVV18.rst new file mode 100644 index 000000000000000..0344eee9471d292 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-11-21-43-24.gh-issue-151337.JSVV18.rst @@ -0,0 +1 @@ +Avoid possible memory leak in ``tkinter.c`` on Windows. diff --git a/Misc/NEWS.d/next/Library/2026-06-12-00-00-00.gh-issue-151416.spawnUA.rst b/Misc/NEWS.d/next/Library/2026-06-12-00-00-00.gh-issue-151416.spawnUA.rst new file mode 100644 index 000000000000000..fd034d9885a8302 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-12-00-00-00.gh-issue-151416.spawnUA.rst @@ -0,0 +1,5 @@ +Fix a crash in :func:`os.spawnv` and :func:`os.spawnve` when an *argv* +item's :meth:`~os.PathLike.__fspath__` method mutates the *argv* list +during argument conversion. :func:`!os.spawnv` argument conversion errors +other than :exc:`TypeError`, such as the :exc:`ValueError` for an embedded +null, are no longer replaced with a generic :exc:`TypeError`. diff --git a/Misc/NEWS.d/next/Library/2026-06-12-00-04-34.gh-issue-151126.aHaBYq.rst b/Misc/NEWS.d/next/Library/2026-06-12-00-04-34.gh-issue-151126.aHaBYq.rst new file mode 100644 index 000000000000000..20ef69d5de5ac55 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-12-00-04-34.gh-issue-151126.aHaBYq.rst @@ -0,0 +1,2 @@ +Fix crash on unset :exc:`MemoryError` on allocation failure in +:func:`ctypes.get_errno`. diff --git a/Misc/NEWS.d/next/Library/2026-06-12-07-20-08.gh-issue-151390.CmYN9EeJ.rst b/Misc/NEWS.d/next/Library/2026-06-12-07-20-08.gh-issue-151390.CmYN9EeJ.rst new file mode 100644 index 000000000000000..ff8de30599c6ad5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-12-07-20-08.gh-issue-151390.CmYN9EeJ.rst @@ -0,0 +1 @@ +Colorize ``match`` in the :term:`REPL` when followed by a unary ``+`` or ``-`` operator. Patch by Bartosz Sławecki. diff --git a/Misc/NEWS.d/next/Library/2026-06-12-22-46-31.gh-issue-151403.DalZWh.rst b/Misc/NEWS.d/next/Library/2026-06-12-22-46-31.gh-issue-151403.DalZWh.rst new file mode 100644 index 000000000000000..ca779ed684e7616 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-12-22-46-31.gh-issue-151403.DalZWh.rst @@ -0,0 +1,3 @@ +Fixed a crash in :class:`subprocess.Popen` (and ``_posixsubprocess.fork_exec``) +when an ``argv`` item's :meth:`~os.PathLike.__fspath__` concurrently mutates the +``args`` sequence being converted. diff --git a/Misc/NEWS.d/next/Library/2026-06-15-15-32-36.gh-issue-151497.1cfmSV.rst b/Misc/NEWS.d/next/Library/2026-06-15-15-32-36.gh-issue-151497.1cfmSV.rst new file mode 100644 index 000000000000000..a4c03c9d71d7618 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-15-15-32-36.gh-issue-151497.1cfmSV.rst @@ -0,0 +1,4 @@ +Opening a :mod:`tarfile` archive no longer attempts to pre-allocate a huge +buffer when a crafted or truncated member claims an oversized extended header +(a GNU long name/link or a pax header). The extended header is now read in +bounded chunks, so its size field can no longer trigger memory exhaustion. diff --git a/Misc/NEWS.d/next/Library/2026-06-17-00-00-00.gh-issue-148660.odcopy.rst b/Misc/NEWS.d/next/Library/2026-06-17-00-00-00.gh-issue-148660.odcopy.rst new file mode 100644 index 000000000000000..30c2e366ca2ba07 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-17-00-00-00.gh-issue-148660.odcopy.rst @@ -0,0 +1,3 @@ +Fix a crash in :meth:`!collections.OrderedDict.copy` when a key's +``__eq__`` or a subclass method mutates the dict during the copy. Now +raises :exc:`RuntimeError` instead, as iteration does. diff --git a/Misc/NEWS.d/next/Library/2026-06-18-12-48-03.gh-issue-151640.R4c3Fx.rst b/Misc/NEWS.d/next/Library/2026-06-18-12-48-03.gh-issue-151640.R4c3Fx.rst new file mode 100644 index 000000000000000..ef61a7d27dd07dd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-18-12-48-03.gh-issue-151640.R4c3Fx.rst @@ -0,0 +1,3 @@ +Fix a data race in :class:`io.BytesIO` in free-threaded builds when +whole-buffer reads or peeks, or :meth:`~io.BytesIO.getvalue`, share the +internal buffer with concurrent writes. diff --git a/Misc/NEWS.d/next/Library/2026-06-18-23-59-46.gh-issue-151596.5ma144.rst b/Misc/NEWS.d/next/Library/2026-06-18-23-59-46.gh-issue-151596.5ma144.rst new file mode 100644 index 000000000000000..17ec1341142e93d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-18-23-59-46.gh-issue-151596.5ma144.rst @@ -0,0 +1 @@ +Add missing ``size`` positional argument to the pure-Python implementation of :meth:`io.TextIOBase.readline`. diff --git a/Misc/NEWS.d/next/Library/2026-06-19-07-26-20.gh-issue-151695.IBDlkN.rst b/Misc/NEWS.d/next/Library/2026-06-19-07-26-20.gh-issue-151695.IBDlkN.rst new file mode 100644 index 000000000000000..f44cb6b93071656 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-19-07-26-20.gh-issue-151695.IBDlkN.rst @@ -0,0 +1,4 @@ +Fix a use-after-free in the :mod:`curses` module. The encoding of the initial +screen, used by :func:`curses.unctrl` and :func:`curses.ungetch` to encode +non-ASCII characters, is now kept as a private copy instead of a borrowed +pointer to a window object that may be deallocated. diff --git a/Misc/NEWS.d/next/Library/2026-06-20-14-47-55.gh-issue-151665.82fmzx.rst b/Misc/NEWS.d/next/Library/2026-06-20-14-47-55.gh-issue-151665.82fmzx.rst new file mode 100644 index 000000000000000..d08a1220cbe5efc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-20-14-47-55.gh-issue-151665.82fmzx.rst @@ -0,0 +1,2 @@ +:func:`inspect.signature` now works on the lazy evaluators of type aliases +and type parameters instead of raising :exc:`ValueError`. diff --git a/Misc/NEWS.d/next/Library/2026-06-20-15-00-00.gh-issue-151770.dtiso0.rst b/Misc/NEWS.d/next/Library/2026-06-20-15-00-00.gh-issue-151770.dtiso0.rst new file mode 100644 index 000000000000000..10b3db8efa42b0f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-20-15-00-00.gh-issue-151770.dtiso0.rst @@ -0,0 +1,3 @@ +Fix :meth:`datetime.datetime.fromisoformat` raising :exc:`AssertionError` +instead of :exc:`ValueError` for an out-of-range month combined with a +``24:00`` time. diff --git a/Misc/NEWS.d/next/Library/2026-06-20-21-15-13.gh-issue-151814.OIbgsO.rst b/Misc/NEWS.d/next/Library/2026-06-20-21-15-13.gh-issue-151814.OIbgsO.rst new file mode 100644 index 000000000000000..1365fb4d8edb1d2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-20-21-15-13.gh-issue-151814.OIbgsO.rst @@ -0,0 +1,2 @@ +Fix unbounded memory growth in :class:`io.TextIOWrapper` when repeatedly +writing an empty string. diff --git a/Misc/NEWS.d/next/Library/2026-06-21-15-50-24.gh-issue-86165.dyac1G.rst b/Misc/NEWS.d/next/Library/2026-06-21-15-50-24.gh-issue-86165.dyac1G.rst new file mode 100644 index 000000000000000..0b630e9ae346834 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-21-15-50-24.gh-issue-86165.dyac1G.rst @@ -0,0 +1,3 @@ +Fix :func:`imaplib.Time2Internaldate` to use the local timezone offset for +``time.struct_time`` values with ``tm_gmtoff`` set to ``None``, as returned by +``datetime.datetime.timetuple()``. Contributed by Xiao Yuan. diff --git a/Misc/NEWS.d/next/Library/2026-06-23-13-27-14.gh-issue-126219.kOfv2g.rst b/Misc/NEWS.d/next/Library/2026-06-23-13-27-14.gh-issue-126219.kOfv2g.rst new file mode 100644 index 000000000000000..8cc8c64c719471d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-23-13-27-14.gh-issue-126219.kOfv2g.rst @@ -0,0 +1,3 @@ +Fixed a crash in :class:`tkinter.Tk` when *className* contains a non-BMP +character and tkinter is built against Tcl/Tk 8.x. Such a name is now +rejected with a :exc:`ValueError`. diff --git a/Misc/NEWS.d/next/Library/2026-06-24-10-46-35.gh-issue-152060.f2asrt.rst b/Misc/NEWS.d/next/Library/2026-06-24-10-46-35.gh-issue-152060.f2asrt.rst new file mode 100644 index 000000000000000..6088a6b80829bb4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-24-10-46-35.gh-issue-152060.f2asrt.rst @@ -0,0 +1,3 @@ +Fix :meth:`datetime.datetime.fromisoformat` raising :exc:`AssertionError` +instead of :exc:`ValueError` for some malformed strings in the pure-Python +implementation, matching the C implementation. diff --git a/Misc/NEWS.d/next/Library/2026-06-24-12-00-00.gh-issue-152052.yBssDE.rst b/Misc/NEWS.d/next/Library/2026-06-24-12-00-00.gh-issue-152052.yBssDE.rst new file mode 100644 index 000000000000000..f9ffbbd342fc81a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-24-12-00-00.gh-issue-152052.yBssDE.rst @@ -0,0 +1,2 @@ +The :mod:`json` C accelerator now correctly reports an unterminated string for a +``\uXXXX`` escape at the end of the input. diff --git a/Misc/NEWS.d/next/Library/2026-06-24-12-00-00.gh-issue-152079.f1tzus.rst b/Misc/NEWS.d/next/Library/2026-06-24-12-00-00.gh-issue-152079.f1tzus.rst new file mode 100644 index 000000000000000..492d00724f6a46e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-24-12-00-00.gh-issue-152079.f1tzus.rst @@ -0,0 +1,3 @@ +Fix :meth:`datetime.datetime.fromisoformat` in the C implementation dropping +the sub-second part of a UTC offset whose whole-second part is zero, matching +the pure-Python implementation. diff --git a/Misc/NEWS.d/next/Library/2026-06-24-22-16-35.gh-issue-105895.hRkuEw.rst b/Misc/NEWS.d/next/Library/2026-06-24-22-16-35.gh-issue-105895.hRkuEw.rst new file mode 100644 index 000000000000000..c69e6fa7d14f4dc --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-24-22-16-35.gh-issue-105895.hRkuEw.rst @@ -0,0 +1,2 @@ +Add :keyword:`match` and :keyword:`case` to the list of supported topics by +:func:`help`. diff --git a/Misc/NEWS.d/next/Library/2026-06-25-01-00-43.gh-issue-151763.wWeHBe.rst b/Misc/NEWS.d/next/Library/2026-06-25-01-00-43.gh-issue-151763.wWeHBe.rst new file mode 100644 index 000000000000000..2f5e84027ad31bb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-25-01-00-43.gh-issue-151763.wWeHBe.rst @@ -0,0 +1,2 @@ +Fix crash in :func:`!_interpqueues.create` whe :exc:`MemoryError` +happens on queue creation. diff --git a/Misc/NEWS.d/next/Library/2026-06-25-07-08-17.gh-issue-152157.dt5Ef0.rst b/Misc/NEWS.d/next/Library/2026-06-25-07-08-17.gh-issue-152157.dt5Ef0.rst new file mode 100644 index 000000000000000..00e83b4af6be7a9 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-25-07-08-17.gh-issue-152157.dt5Ef0.rst @@ -0,0 +1,3 @@ +The C implementations of :meth:`~datetime.datetime.fromisoformat` and :meth:`~datetime.time.fromisoformat` +now reject a decimal separator that is not followed by any +fractional digit before a timezone designator. diff --git a/Misc/NEWS.d/next/Library/2026-06-25-10-07-54.gh-issue-152156.gscPU9.rst b/Misc/NEWS.d/next/Library/2026-06-25-10-07-54.gh-issue-152156.gscPU9.rst new file mode 100644 index 000000000000000..30e561274dc50c2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-25-10-07-54.gh-issue-152156.gscPU9.rst @@ -0,0 +1,2 @@ +Fix a possible crash in :func:`concurrent.interpreters.create` under limited +memory conditions. diff --git a/Misc/NEWS.d/next/Library/2026-06-25-12-00-00.gh-issue-152212.Zk7Qm2.rst b/Misc/NEWS.d/next/Library/2026-06-25-12-00-00.gh-issue-152212.Zk7Qm2.rst new file mode 100644 index 000000000000000..2b1d272638b11ab --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-25-12-00-00.gh-issue-152212.Zk7Qm2.rst @@ -0,0 +1,3 @@ +Fix the pure-Python :mod:`zoneinfo` parser accepting a POSIX TZ string with a +``std`` abbreviation but no offset. This is invalid per POSIX and now +raises :exc:`ValueError`, matching the C accelerator. Patch by tonghuaroot. diff --git a/Misc/NEWS.d/next/Library/2026-06-25-14-32-42.gh-issue-139145.kT9rmP.rst b/Misc/NEWS.d/next/Library/2026-06-25-14-32-42.gh-issue-139145.kT9rmP.rst new file mode 100644 index 000000000000000..3fa639c5f727baa --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-25-14-32-42.gh-issue-139145.kT9rmP.rst @@ -0,0 +1,5 @@ +Fix a busy loop in :mod:`tkinter` on interactive Python. When a Tcl command +running its own event loop (such as ``vwait`` or :meth:`!wait_variable`) was +active and input arrived on stdin, the event loop kept spinning at 100% CPU. +The stdin file handler is now removed as soon as input is available. Based on +a patch by Michiel de Hoon. diff --git a/Misc/NEWS.d/next/Library/2026-06-25-14-32-43.gh-issue-139816.Lq2vXa.rst b/Misc/NEWS.d/next/Library/2026-06-25-14-32-43.gh-issue-139816.Lq2vXa.rst new file mode 100644 index 000000000000000..b180dee686caf8c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-25-14-32-43.gh-issue-139816.Lq2vXa.rst @@ -0,0 +1,4 @@ +Fix a hang in :mod:`tkinter` on interactive Python built without +:mod:`readline`. An exception raised in a callback no longer causes the +event loop to stop and wait for the user to press Enter; pending callbacks +now keep running until input is actually available on stdin. diff --git a/Misc/NEWS.d/next/Library/2026-06-26-12-50-26.gh-issue-152246.MfXMd1.rst b/Misc/NEWS.d/next/Library/2026-06-26-12-50-26.gh-issue-152246.MfXMd1.rst new file mode 100644 index 000000000000000..0858199a9fa2798 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-26-12-50-26.gh-issue-152246.MfXMd1.rst @@ -0,0 +1,2 @@ +Fix the pure-Python :mod:`zoneinfo` parser accepting an invalid POSIX TZ +transition rule with a non-period separator. Patch by tonghuaroot. diff --git a/Misc/NEWS.d/next/Library/2026-06-26-13-10-00.gh-issue-80937.Hq3mNp.rst b/Misc/NEWS.d/next/Library/2026-06-26-13-10-00.gh-issue-80937.Hq3mNp.rst new file mode 100644 index 000000000000000..4ea0179c6586d4f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-26-13-10-00.gh-issue-80937.Hq3mNp.rst @@ -0,0 +1,4 @@ +Fix a memory leak in :mod:`tkinter` when a Tcl command created with +``createcommand`` was not explicitly removed before the interpreter was +deleted. The command no longer keeps the interpreter alive through a +reference cycle. diff --git a/Misc/NEWS.d/next/Library/2026-06-26-13-39-11.gh-issue-152248.N2Rmaf.rst b/Misc/NEWS.d/next/Library/2026-06-26-13-39-11.gh-issue-152248.N2Rmaf.rst new file mode 100644 index 000000000000000..8921ad17349e975 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-26-13-39-11.gh-issue-152248.N2Rmaf.rst @@ -0,0 +1,3 @@ +Make the C and pure-Python :mod:`zoneinfo` parsers validate POSIX TZ +abbreviations consistently, rejecting unquoted abbreviations with non-letter +characters and empty quoted abbreviations. Patch by tonghuaroot. diff --git a/Misc/NEWS.d/next/Library/2026-06-26-15-00-00.gh-issue-38464.yDPjKH.rst b/Misc/NEWS.d/next/Library/2026-06-26-15-00-00.gh-issue-38464.yDPjKH.rst new file mode 100644 index 000000000000000..c1c272b97d4b0a8 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-26-15-00-00.gh-issue-38464.yDPjKH.rst @@ -0,0 +1,3 @@ +:meth:`!tkinter.Misc.nametowidget` now resolves the auto-generated names of +cloned menus (a menu used as a menubar or a cascade) back to the original +widget. diff --git a/Misc/NEWS.d/next/Library/2026-06-26-15-30-00.gh-issue-88758.Qw7nLp.rst b/Misc/NEWS.d/next/Library/2026-06-26-15-30-00.gh-issue-88758.Qw7nLp.rst new file mode 100644 index 000000000000000..31d567194fb9031 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-26-15-30-00.gh-issue-88758.Qw7nLp.rst @@ -0,0 +1,4 @@ +:meth:`!tkinter.Misc.focus_get`, :meth:`!focus_displayof`, +:meth:`!focus_lastfor` and :meth:`!winfo_containing` now return ``None`` +instead of raising :exc:`KeyError` when the widget was not created by +:mod:`tkinter` (for example a torn-off menu). diff --git a/Misc/NEWS.d/next/Library/2026-06-26-15-41-34.gh-issue-152305.WnbbBc.rst b/Misc/NEWS.d/next/Library/2026-06-26-15-41-34.gh-issue-152305.WnbbBc.rst new file mode 100644 index 000000000000000..4f27e2ed016d694 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-26-15-41-34.gh-issue-152305.WnbbBc.rst @@ -0,0 +1,2 @@ +Fix the pure-Python :meth:`datetime.time.strftime` implementation raising :exc:`AttributeError` for the +year directives. Patch by tonghuaroot. diff --git a/Misc/NEWS.d/next/Library/2026-06-26-16-30-00.gh-issue-83274.Kx9mQv.rst b/Misc/NEWS.d/next/Library/2026-06-26-16-30-00.gh-issue-83274.Kx9mQv.rst new file mode 100644 index 000000000000000..3b722d2176be920 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-26-16-30-00.gh-issue-83274.Kx9mQv.rst @@ -0,0 +1,3 @@ +Deallocating a :mod:`tkinter` application from a thread other than the one it +was created in no longer crashes the interpreter. The underlying Tcl +interpreter is leaked instead, and a :exc:`RuntimeWarning` is reported. diff --git a/Misc/NEWS.d/next/Library/2026-06-26-23-56-40.gh-issue-71880.782D31.rst b/Misc/NEWS.d/next/Library/2026-06-26-23-56-40.gh-issue-71880.782D31.rst new file mode 100644 index 000000000000000..5a12f428f2a8b6c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-26-23-56-40.gh-issue-71880.782D31.rst @@ -0,0 +1,5 @@ +:class:`curses.textpad.Textbox` now lets the lower-right cell of the window be +edited. Writing it with :meth:`~curses.window.addch` would move the cursor +past the end of the window, raising an error and scrolling a scrollable window, +so it is now written with :meth:`~curses.window.insch`, which keeps the cursor +in place. diff --git a/Misc/NEWS.d/next/Library/2026-06-27-12-30-00.gh-issue-133031.Na8Bit.rst b/Misc/NEWS.d/next/Library/2026-06-27-12-30-00.gh-issue-133031.Na8Bit.rst new file mode 100644 index 000000000000000..96e9efe20e42f04 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-27-12-30-00.gh-issue-133031.Na8Bit.rst @@ -0,0 +1,3 @@ +:class:`curses.textpad.Textbox` now enters and reads back the non-ASCII +characters of an 8-bit locale encoding, instead of mangling them with a 7-bit +mask. diff --git a/Misc/NEWS.d/next/Library/2026-06-27-17-19-09.gh-issue-151126.huUyOM.rst b/Misc/NEWS.d/next/Library/2026-06-27-17-19-09.gh-issue-151126.huUyOM.rst new file mode 100644 index 000000000000000..2e51fd45b595486 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-27-17-19-09.gh-issue-151126.huUyOM.rst @@ -0,0 +1,2 @@ +Fix two crashes in :mod:`tkinter` and :mod:`socket` modules initialization +under a memory pressure. Sets missing :exc:`MemoryError`. diff --git a/Misc/NEWS.d/next/Library/2026-06-28-11-00-40.gh-issue-78335.Kp3mWq.rst b/Misc/NEWS.d/next/Library/2026-06-28-11-00-40.gh-issue-78335.Kp3mWq.rst new file mode 100644 index 000000000000000..641889cfa9ef41d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-28-11-00-40.gh-issue-78335.Kp3mWq.rst @@ -0,0 +1,4 @@ +Update the docstrings of :mod:`tkinter` and :mod:`tkinter.ttk` widget classes +to list all supported widget options, including options added in Tk 9.0 and +9.1. ``tkinter.Menubutton`` and ``tkinter.Message`` previously had no option +list at all. diff --git a/Misc/NEWS.d/next/Library/2026-06-29-09-45-00.gh-issue-152569.Kf7Lq2.rst b/Misc/NEWS.d/next/Library/2026-06-29-09-45-00.gh-issue-152569.Kf7Lq2.rst new file mode 100644 index 000000000000000..fb7e32a5260b39a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-29-09-45-00.gh-issue-152569.Kf7Lq2.rst @@ -0,0 +1,3 @@ +Fix :func:`asyncio.wait` leaking waiting tasks via the await-graph when racing a +future that never resolves. The waiting task is now discarded from every future's +``awaited_by`` set once :func:`~asyncio.wait` returns, even for pending futures. diff --git a/Misc/NEWS.d/next/Library/2026-06-29-22-16-57.gh-issue-50966.Tq5mLp.rst b/Misc/NEWS.d/next/Library/2026-06-29-22-16-57.gh-issue-50966.Tq5mLp.rst new file mode 100644 index 000000000000000..96a0938c02d226b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-29-22-16-57.gh-issue-50966.Tq5mLp.rst @@ -0,0 +1,3 @@ +Fix unbounded recursion in :mod:`turtle` when a mouse event handler that moves +the turtle is reentered while the screen is being redrawn, for example with +``screen.ondrag(turtle.goto)``. This could previously crash the interpreter. diff --git a/Misc/NEWS.d/next/Library/2026-06-30-12-00-00.gh-issue-40038.qK7mGv.rst b/Misc/NEWS.d/next/Library/2026-06-30-12-00-00.gh-issue-40038.qK7mGv.rst new file mode 100644 index 000000000000000..1f393d23266bcef --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-30-12-00-00.gh-issue-40038.qK7mGv.rst @@ -0,0 +1,6 @@ +:mod:`imaplib` now again quotes command arguments when necessary, for +example mailbox names containing a space. Such quoting was inadvertently +disabled when the module was ported to Python 3, and the arguments are now +quoted according to the :rfc:`3501` grammar. For backward compatibility, +an argument already enclosed in double quotes is left unchanged, so code +that quotes arguments itself keeps working. diff --git a/Misc/NEWS.d/next/Library/2026-06-30-13-00-00.gh-issue-151842.OOM31g.rst b/Misc/NEWS.d/next/Library/2026-06-30-13-00-00.gh-issue-151842.OOM31g.rst new file mode 100644 index 000000000000000..8a2ecf3c40032f5 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-30-13-00-00.gh-issue-151842.OOM31g.rst @@ -0,0 +1,2 @@ +Fix a crash in :func:`!_interpreters.capture_exception` when +:exc:`MemoryError` happens. Patch by Amrutha Modela. diff --git a/Misc/NEWS.d/next/Library/2026-06-30-21-40-00.gh-issue-152502.Kq3Vn7.rst b/Misc/NEWS.d/next/Library/2026-06-30-21-40-00.gh-issue-152502.Kq3Vn7.rst new file mode 100644 index 000000000000000..815bc47966c40c3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-30-21-40-00.gh-issue-152502.Kq3Vn7.rst @@ -0,0 +1,5 @@ +Detect the :mod:`curses` mouse interface (:func:`~curses.getmouse`, the +``BUTTON*`` constants, and others) with a configure capability probe or library +macros instead of gating it on ncurses-specific macros. It is now also +available with other curses implementations that provide it, such as NetBSD +curses and PDCurses (the latter underpins ``windows-curses``). diff --git a/Misc/NEWS.d/next/Library/2026-07-01-10-00-00.gh-issue-88574.Kz3wQm.rst b/Misc/NEWS.d/next/Library/2026-07-01-10-00-00.gh-issue-88574.Kz3wQm.rst new file mode 100644 index 000000000000000..371f4d08bccd2e2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-01-10-00-00.gh-issue-88574.Kz3wQm.rst @@ -0,0 +1,4 @@ +:mod:`imaplib` no longer fails when a server sends a spurious blank line +after the counted data of a literal, including after a literal that +terminates a response (such as a mailbox name returned by ``LIST``). +Such blank lines are now skipped without swallowing the following line. diff --git a/Misc/NEWS.d/next/Library/2026-07-01-10-30-00.gh-issue-63121.Rm4tZp.rst b/Misc/NEWS.d/next/Library/2026-07-01-10-30-00.gh-issue-63121.Rm4tZp.rst new file mode 100644 index 000000000000000..a63b164ad7a627b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-01-10-30-00.gh-issue-63121.Rm4tZp.rst @@ -0,0 +1,7 @@ +:mod:`imaplib` now refreshes the cached capability list after a successful +:meth:`~imaplib.IMAP4.login` or :meth:`~imaplib.IMAP4.authenticate`, using +the ``CAPABILITY`` response sent by the server or, if none was sent, by +querying it, so that capabilities that become available only after +authentication (such as ``ENABLE`` on Gmail) are recognized. Capabilities +advertised in the server greeting are now also used, avoiding a redundant +``CAPABILITY`` command. diff --git a/Misc/NEWS.d/next/Library/2026-07-01-12-00-00.gh-issue-108280.Nq8vTr.rst b/Misc/NEWS.d/next/Library/2026-07-01-12-00-00.gh-issue-108280.Nq8vTr.rst new file mode 100644 index 000000000000000..487c9da67f1832b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-01-12-00-00.gh-issue-108280.Nq8vTr.rst @@ -0,0 +1,4 @@ +Connecting :mod:`imaplib` to a server that does not send a valid IMAP4 +greeting (for example a POP3 server answering on the IMAP port) now raises +an error reporting the server's response instead of +``imaplib.IMAP4.error: None``. diff --git a/Misc/NEWS.d/next/Library/2026-07-02-12-00-00.gh-issue-152847.PitKqc.rst b/Misc/NEWS.d/next/Library/2026-07-02-12-00-00.gh-issue-152847.PitKqc.rst new file mode 100644 index 000000000000000..3f20d78a201e84c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-02-12-00-00.gh-issue-152847.PitKqc.rst @@ -0,0 +1,3 @@ +Reject a POSIX TZ transition rule with non-digit characters in the +day-of-year field in the pure-Python :mod:`zoneinfo` parser. Patch by +tonghuaroot. diff --git a/Misc/NEWS.d/next/Library/2026-07-02-13-30-00.gh-issue-152849.K9dRvP.rst b/Misc/NEWS.d/next/Library/2026-07-02-13-30-00.gh-issue-152849.K9dRvP.rst new file mode 100644 index 000000000000000..053a5dfea3f3ff4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-02-13-30-00.gh-issue-152849.K9dRvP.rst @@ -0,0 +1,2 @@ +Out-of-range float and integer timestamps now raise :exc:`OverflowError` +with the same message. Patch by tonghuaroot. diff --git a/Misc/NEWS.d/next/Library/2026-07-02-22-19-53.gh-issue-150880.vPZ7jK.rst b/Misc/NEWS.d/next/Library/2026-07-02-22-19-53.gh-issue-150880.vPZ7jK.rst new file mode 100644 index 000000000000000..a7bf210f8e1ca9c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-02-22-19-53.gh-issue-150880.vPZ7jK.rst @@ -0,0 +1,3 @@ +Normalize non-extended Windows paths before appending the wildcard used by +``os.listdir()`` and ``os.scandir()``, making paths with trailing spaces +behave consistently with other filesystem APIs. diff --git a/Misc/NEWS.d/next/Library/2026-07-03-14-54-33.gh-issue-152951.u8tPCI.rst b/Misc/NEWS.d/next/Library/2026-07-03-14-54-33.gh-issue-152951.u8tPCI.rst new file mode 100644 index 000000000000000..c3d56c6d59c91b3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-03-14-54-33.gh-issue-152951.u8tPCI.rst @@ -0,0 +1,2 @@ +:class:`collections.deque` prevent rare crash when calling ``extend`` under +high memory pressure conditions. diff --git a/Misc/NEWS.d/next/Library/2026-07-03-21-10-00.gh-issue-54930.hq09Er.rst b/Misc/NEWS.d/next/Library/2026-07-03-21-10-00.gh-issue-54930.hq09Er.rst new file mode 100644 index 000000000000000..1cfc0212c609a69 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-03-21-10-00.gh-issue-54930.hq09Er.rst @@ -0,0 +1,5 @@ +Error responses of :class:`http.server.BaseHTTPRequestHandler` to malformed +request lines now include a status line and headers instead of being sent in +the bare HTTP/0.9 style. +Only a valid HTTP/0.9 request (a two-word ``GET`` request line) now receives +an HTTP/0.9 style response. diff --git a/Misc/NEWS.d/next/Library/2026-07-04-11-14-06.gh-issue-152851.diYvTf.rst b/Misc/NEWS.d/next/Library/2026-07-04-11-14-06.gh-issue-152851.diYvTf.rst new file mode 100644 index 000000000000000..aab033ebeed6954 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-04-11-14-06.gh-issue-152851.diYvTf.rst @@ -0,0 +1,2 @@ +Prevent a crash when allocation fails while copying a :func:`BLAKE-2s/2b +<hashlib.blake2b>` object. Patch by Bénédikt Tran. diff --git a/Misc/NEWS.d/next/Library/2026-07-04-13-00-00.gh-issue-135661.CIkADG.rst b/Misc/NEWS.d/next/Library/2026-07-04-13-00-00.gh-issue-135661.CIkADG.rst new file mode 100644 index 000000000000000..26a912a6f3a1949 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-04-13-00-00.gh-issue-135661.CIkADG.rst @@ -0,0 +1,3 @@ +Fix :class:`html.parser.HTMLParser`: an abruptly closed empty comment +(``<!-->`` or ``<!--->``) no longer extends up to a later ``-->`` in the same +:meth:`~html.parser.HTMLParser.feed` call. diff --git a/Misc/NEWS.d/next/Library/2026-07-04-20-28-51.gh-issue-153037.XUSp_g.rst b/Misc/NEWS.d/next/Library/2026-07-04-20-28-51.gh-issue-153037.XUSp_g.rst new file mode 100644 index 000000000000000..cbfc62f75184706 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-04-20-28-51.gh-issue-153037.XUSp_g.rst @@ -0,0 +1,3 @@ +Fix :class:`~compression.zstd.ZstdFile` raising :exc:`AttributeError` +instead of :exc:`io.UnsupportedOperation` when iterating over a file that +is not open for reading. diff --git a/Misc/NEWS.d/next/Library/2026-07-05-12-00-00.gh-issue-153056.tMpLat.rst b/Misc/NEWS.d/next/Library/2026-07-05-12-00-00.gh-issue-153056.tMpLat.rst new file mode 100644 index 000000000000000..a2bc89f25a53c19 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-05-12-00-00.gh-issue-153056.tMpLat.rst @@ -0,0 +1,4 @@ +Fix :class:`string.Template` raising a spurious :exc:`ValueError` when the +*pattern* attribute is a compiled regular expression object, which the +documentation allows. On the free-threaded build this also occurred as a data +race on the first concurrent use. diff --git a/Misc/NEWS.d/next/Library/2026-07-05-14-30-00.gh-issue-153062.teeFT1.rst b/Misc/NEWS.d/next/Library/2026-07-05-14-30-00.gh-issue-153062.teeFT1.rst new file mode 100644 index 000000000000000..ed1aaa15d37bd9c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-05-14-30-00.gh-issue-153062.teeFT1.rst @@ -0,0 +1,2 @@ +Fix a crash when concurrently iterating an :func:`itertools.tee` iterator on +the free-threaded build. diff --git a/Misc/NEWS.d/next/Library/2026-07-05-16-09-31.gh-issue-153068.huY9Jh.rst b/Misc/NEWS.d/next/Library/2026-07-05-16-09-31.gh-issue-153068.huY9Jh.rst new file mode 100644 index 000000000000000..accf6e99cd6434d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-05-16-09-31.gh-issue-153068.huY9Jh.rst @@ -0,0 +1,2 @@ +Fix :meth:`!cProfile.Profile.enable` to no longer overwrite errors from +:mod:`sys.monitoring`. diff --git a/Misc/NEWS.d/next/Library/2026-07-06-12-00-00.gh-issue-153200.isqrtLt.rst b/Misc/NEWS.d/next/Library/2026-07-06-12-00-00.gh-issue-153200.isqrtLt.rst new file mode 100644 index 000000000000000..a27ac73fe31a70a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-06-12-00-00.gh-issue-153200.isqrtLt.rst @@ -0,0 +1,3 @@ +Fix :func:`math.isqrt` returning an incorrect result for arguments not +less than 2**64 that are instances of an :class:`int` subclass with an +overridden comparison operator. diff --git a/Misc/NEWS.d/next/Library/2026-07-07-02-26-43.gh-issue-153210.LUXIcm.rst b/Misc/NEWS.d/next/Library/2026-07-07-02-26-43.gh-issue-153210.LUXIcm.rst new file mode 100644 index 000000000000000..d5d4b7c1b8c0ed4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-07-02-26-43.gh-issue-153210.LUXIcm.rst @@ -0,0 +1 @@ +Fix crash on :mod:`array` import under a memory pressure. diff --git a/Misc/NEWS.d/next/Library/2026-07-07-13-31-52.gh-issue-148286.-qu-em.rst b/Misc/NEWS.d/next/Library/2026-07-07-13-31-52.gh-issue-148286.-qu-em.rst new file mode 100644 index 000000000000000..60cd49020c954a4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-07-13-31-52.gh-issue-148286.-qu-em.rst @@ -0,0 +1,3 @@ +Fix undefined behavior in +:attr:`compression.zstd.ZstdDecompressor.unused_data` when a complete frame +was decompressed in a single call. diff --git a/Misc/NEWS.d/next/Library/2026-07-07-17-50-54.gh-issue-143990.FoNtCf.rst b/Misc/NEWS.d/next/Library/2026-07-07-17-50-54.gh-issue-143990.FoNtCf.rst new file mode 100644 index 000000000000000..a78dcccf482ab59 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-07-17-50-54.gh-issue-143990.FoNtCf.rst @@ -0,0 +1,4 @@ +A :class:`tkinter.font.Font` created from a named font, +including by :meth:`~tkinter.font.Font.copy`, +now copies its configured options rather than the options resolved by Tcl's ``font actual``, +preserving a size specified in pixels (a negative size). diff --git a/Misc/NEWS.d/next/Library/2026-07-08-01-03-17.gh-issue-153292.oHDt3l.rst b/Misc/NEWS.d/next/Library/2026-07-08-01-03-17.gh-issue-153292.oHDt3l.rst new file mode 100644 index 000000000000000..dc363e160074302 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-08-01-03-17.gh-issue-153292.oHDt3l.rst @@ -0,0 +1 @@ +Fix data race in repr of :class:`threading.RLock` in free-threading build. diff --git a/Misc/NEWS.d/next/Library/2026-07-08-01-47-15.gh-issue-153083.XbKZCp.rst b/Misc/NEWS.d/next/Library/2026-07-08-01-47-15.gh-issue-153083.XbKZCp.rst new file mode 100644 index 000000000000000..b6d96851e17342f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-08-01-47-15.gh-issue-153083.XbKZCp.rst @@ -0,0 +1,2 @@ +Defer GC tracking of an :class:`array.array` to the end of its construction. +Patch by Donghee Na. diff --git a/Misc/NEWS.d/next/Library/2026-07-09-00-00-00.gh-issue-153406.vyMmB6.rst b/Misc/NEWS.d/next/Library/2026-07-09-00-00-00.gh-issue-153406.vyMmB6.rst new file mode 100644 index 000000000000000..c09575754e33804 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-09-00-00-00.gh-issue-153406.vyMmB6.rst @@ -0,0 +1,3 @@ +:func:`email.utils.parsedate_to_datetime` now raises :exc:`ValueError` +instead of :exc:`OverflowError` when the parsed year or timezone offset is +out of range, matching its documented behavior. diff --git a/Misc/NEWS.d/next/Library/2026-07-09-14-00-00.gh-issue-153417.Bw7Rq2.rst b/Misc/NEWS.d/next/Library/2026-07-09-14-00-00.gh-issue-153417.Bw7Rq2.rst new file mode 100644 index 000000000000000..ebc36fa7c228d08 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-09-14-00-00.gh-issue-153417.Bw7Rq2.rst @@ -0,0 +1,3 @@ +Error messages from :meth:`imaplib.IMAP4.select` and :meth:`imaplib.IMAP4.uid` +no longer raise :exc:`BytesWarning` under :option:`!-bb` +when the mailbox or command argument is :class:`bytes`. diff --git a/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst b/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst new file mode 100644 index 000000000000000..0746d3f12149d52 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-09-16-35-47.gh-issue-153404.daRkes.rst @@ -0,0 +1,4 @@ +:class:`urllib.robotparser.RobotFileParser` now silently ignores a +``Crawl-delay`` or ``Request-rate`` value written with non-decimal digits +(such as ``U+00B2 SUPERSCRIPT TWO``) instead of raising :exc:`ValueError` +and aborting the parse of the whole ``robots.txt`` file. diff --git a/Misc/NEWS.d/next/Library/2026-07-12-00-00-00.gh-issue-85943.8f906e.rst b/Misc/NEWS.d/next/Library/2026-07-12-00-00-00.gh-issue-85943.8f906e.rst new file mode 100644 index 000000000000000..0a93bf2e7b803e3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-12-00-00-00.gh-issue-85943.8f906e.rst @@ -0,0 +1,4 @@ +Fix :mod:`struct` functions raising :exc:`BytesWarning` under the ``-bb`` +command line option when a :class:`str` format is used after an equal +:class:`bytes` format (or vice versa). The internal format cache no longer +mixes :class:`str` and :class:`bytes` keys. diff --git a/Misc/NEWS.d/next/Library/2026-07-13-10-27-08.gh-issue-153658.A6o4n5.rst b/Misc/NEWS.d/next/Library/2026-07-13-10-27-08.gh-issue-153658.A6o4n5.rst new file mode 100644 index 000000000000000..06385bb7754b127 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-13-10-27-08.gh-issue-153658.A6o4n5.rst @@ -0,0 +1,2 @@ +Fix :meth:`sqlite3.Connection.iterdump` raising :exc:`sqlite3.OperationalError` +when a table name contains a single quote. Patch by tonghuaroot. diff --git a/Misc/NEWS.d/next/Library/2026-07-14-15-00-00.gh-issue-127049.pQw7Rk.rst b/Misc/NEWS.d/next/Library/2026-07-14-15-00-00.gh-issue-127049.pQw7Rk.rst new file mode 100644 index 000000000000000..d7557d8f562f7a6 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-14-15-00-00.gh-issue-127049.pQw7Rk.rst @@ -0,0 +1,5 @@ +Fix a race condition in :mod:`asyncio` on Unix where +:meth:`asyncio.subprocess.Process.send_signal`, :meth:`~asyncio.subprocess.Process.terminate` +or :meth:`~asyncio.subprocess.Process.kill` could signal an unrelated process +that was recycled onto the PID of the already-reaped child when ThreadedChildWatcher is used. +Patch by Kumar Aditya. diff --git a/Misc/NEWS.d/next/Library/2026-07-14-19-45-21.gh-issue-153695.UYdZJZ.rst b/Misc/NEWS.d/next/Library/2026-07-14-19-45-21.gh-issue-153695.UYdZJZ.rst new file mode 100644 index 000000000000000..43d7feebf97d6f2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-14-19-45-21.gh-issue-153695.UYdZJZ.rst @@ -0,0 +1,2 @@ +Hashing a :class:`sqlite3.Row` that contains an unhashable value now raises +:exc:`TypeError` instead of :exc:`SystemError`. Patch by tonghuaroot. diff --git a/Misc/NEWS.d/next/Library/2026-07-15-14-24-18.gh-issue-153761.8jqs67.rst b/Misc/NEWS.d/next/Library/2026-07-15-14-24-18.gh-issue-153761.8jqs67.rst new file mode 100644 index 000000000000000..68e007e66e0396c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-15-14-24-18.gh-issue-153761.8jqs67.rst @@ -0,0 +1 @@ +Fix cancelling :meth:`asyncio.loop.sock_accept` dropping a pending connection. diff --git a/Misc/NEWS.d/next/Library/2026-07-15-21-56-40.gh-issue-146011.nWmHif.rst b/Misc/NEWS.d/next/Library/2026-07-15-21-56-40.gh-issue-146011.nWmHif.rst new file mode 100644 index 000000000000000..0cac0257040bd6b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-15-21-56-40.gh-issue-146011.nWmHif.rst @@ -0,0 +1,2 @@ +Fix a heap-use-after-free in the C implementation of :mod:`decimal` +when calling :func:`repr` after deleting the :class:`~decimal.Context`. diff --git a/Misc/NEWS.d/next/Library/2026-07-17-20-31-00.gh-issue-153862.X3kjoY.rst b/Misc/NEWS.d/next/Library/2026-07-17-20-31-00.gh-issue-153862.X3kjoY.rst new file mode 100644 index 000000000000000..aa646dc6c49174b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-17-20-31-00.gh-issue-153862.X3kjoY.rst @@ -0,0 +1,3 @@ +On a wide :mod:`curses` build, :meth:`curses.window.inch` now returns the +locale-encoded byte of a non-ASCII character, matching +:meth:`~curses.window.instr`, instead of the low byte of its code point. diff --git a/Misc/NEWS.d/next/Library/2026-07-17-20-56-32.gh-issue-153864.WmA9By.rst b/Misc/NEWS.d/next/Library/2026-07-17-20-56-32.gh-issue-153864.WmA9By.rst new file mode 100644 index 000000000000000..79857185d7d0c76 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-17-20-56-32.gh-issue-153864.WmA9By.rst @@ -0,0 +1,3 @@ +On a wide :mod:`curses` build, :meth:`curses.window.insch` now inserts a +non-ASCII byte as the character it encodes in the window's encoding, +consistently with :meth:`~curses.window.addch`, instead of its code point. diff --git a/Misc/NEWS.d/next/Library/2026-07-18-10-52-10.gh-issue-153896.87oevp.rst b/Misc/NEWS.d/next/Library/2026-07-18-10-52-10.gh-issue-153896.87oevp.rst new file mode 100644 index 000000000000000..217a3d3d272ed07 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-18-10-52-10.gh-issue-153896.87oevp.rst @@ -0,0 +1 @@ +Deduplicate unhashable args in :data:`typing.Literal`. diff --git a/Misc/NEWS.d/next/Library/2026-07-18-11-16-41.gh-issue-153908.82FiGk.rst b/Misc/NEWS.d/next/Library/2026-07-18-11-16-41.gh-issue-153908.82FiGk.rst new file mode 100644 index 000000000000000..6ee943a5fb66724 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-18-11-16-41.gh-issue-153908.82FiGk.rst @@ -0,0 +1 @@ +Fix data race when calling :func:`repr` on :class:`itertools.count` under the :term:`free-threaded build`. diff --git a/Misc/NEWS.d/next/Library/2026-07-18-17-09-49.gh-issue-154001.3brrUv.rst b/Misc/NEWS.d/next/Library/2026-07-18-17-09-49.gh-issue-154001.3brrUv.rst new file mode 100644 index 000000000000000..ff019aa36188479 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-18-17-09-49.gh-issue-154001.3brrUv.rst @@ -0,0 +1,2 @@ +Fix :func:`random.binomialvariate` raising :exc:`ZeroDivisionError` +when :func:`random.random` returns zero. diff --git a/Misc/NEWS.d/next/Library/2026-07-20-14-00-00.gh-issue-154227.openpt.rst b/Misc/NEWS.d/next/Library/2026-07-20-14-00-00.gh-issue-154227.openpt.rst new file mode 100644 index 000000000000000..e33d090d78c5934 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-20-14-00-00.gh-issue-154227.openpt.rst @@ -0,0 +1,2 @@ +Fix :func:`os.posix_openpt` on OpenBSD, where it rejected the +:data:`~os.O_CLOEXEC` flag. diff --git a/Misc/NEWS.d/next/Library/2026-07-20-14-10-00.gh-issue-154225.openpty.rst b/Misc/NEWS.d/next/Library/2026-07-20-14-10-00.gh-issue-154225.openpty.rst new file mode 100644 index 000000000000000..f15022b66320062 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-20-14-10-00.gh-issue-154225.openpty.rst @@ -0,0 +1,2 @@ +Fix :func:`os.openpty` on Solaris and illumos: it no longer leaves the +pseudo-terminal as the controlling terminal of the calling process. diff --git a/Misc/NEWS.d/next/Library/2026-07-20-18-30-00.gh-issue-154258.mmapresize.rst b/Misc/NEWS.d/next/Library/2026-07-20-18-30-00.gh-issue-154258.mmapresize.rst new file mode 100644 index 000000000000000..4c33336e0bbdcad --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-20-18-30-00.gh-issue-154258.mmapresize.rst @@ -0,0 +1,3 @@ +Fix a crash in :meth:`mmap.mmap.resize` on NetBSD when growing a shared +anonymous mapping. :meth:`!resize` now raises :exc:`ValueError` in this +case, as it already did on Linux. diff --git a/Misc/NEWS.d/next/Library/2026-07-20-21-36-40.gh-issue-154283.Xip7xE.rst b/Misc/NEWS.d/next/Library/2026-07-20-21-36-40.gh-issue-154283.Xip7xE.rst new file mode 100644 index 000000000000000..a278d67e1824d58 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-20-21-36-40.gh-issue-154283.Xip7xE.rst @@ -0,0 +1,2 @@ +On DragonFly BSD, :func:`threading.get_native_id` now returns a value that is +unique across processes, matching the other platforms. diff --git a/Misc/NEWS.d/next/Library/2026-07-20-22-03-44.gh-issue-154291.Mi5HeQ.rst b/Misc/NEWS.d/next/Library/2026-07-20-22-03-44.gh-issue-154291.Mi5HeQ.rst new file mode 100644 index 000000000000000..d962f8987ad491e --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-20-22-03-44.gh-issue-154291.Mi5HeQ.rst @@ -0,0 +1,3 @@ +Fix :func:`socket.has_dualstack_ipv6` to return ``False`` on platforms such as +DragonFly BSD where setting ``IPV6_V6ONLY`` to 0 silently has no +effect. diff --git a/Misc/NEWS.d/next/Library/2026-07-21-01-13-39.gh-issue-154307.UNFKL4.rst b/Misc/NEWS.d/next/Library/2026-07-21-01-13-39.gh-issue-154307.UNFKL4.rst new file mode 100644 index 000000000000000..fdba0c2940f0050 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-21-01-13-39.gh-issue-154307.UNFKL4.rst @@ -0,0 +1,3 @@ +Fix :meth:`tempfile.TemporaryDirectory.cleanup` on DragonFly BSD, where removing +a file with the ``UF_NOUNLINK`` flag failed with ``EISDIR`` instead of +``EPERM``. diff --git a/Misc/NEWS.d/next/Library/2026-07-21-20-34-05.gh-issue-154389.DlTiii.rst b/Misc/NEWS.d/next/Library/2026-07-21-20-34-05.gh-issue-154389.DlTiii.rst new file mode 100644 index 000000000000000..eb2e2edc2d720e7 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-21-20-34-05.gh-issue-154389.DlTiii.rst @@ -0,0 +1,3 @@ +Fix :func:`uuid.uuid1` on OpenBSD: +it returned a version 4 UUID, +because ``uuid_create()`` generates random UUIDs on this platform. diff --git a/Misc/NEWS.d/next/Library/2026-07-21-21-36-59.gh-issue-154399.VSa6oH.rst b/Misc/NEWS.d/next/Library/2026-07-21-21-36-59.gh-issue-154399.VSa6oH.rst new file mode 100644 index 000000000000000..598dd26694c73e4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-21-21-36-59.gh-issue-154399.VSa6oH.rst @@ -0,0 +1,3 @@ +Fix :mod:`venv` activation in a non-interactive csh: +``activate.csh`` no longer fails +when the ``prompt`` variable is not set. diff --git a/Misc/NEWS.d/next/Library/2026-07-22-10-30-08.gh-issue-154435.xCxm0T.rst b/Misc/NEWS.d/next/Library/2026-07-22-10-30-08.gh-issue-154435.xCxm0T.rst new file mode 100644 index 000000000000000..b468e929b1ca18b --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-22-10-30-08.gh-issue-154435.xCxm0T.rst @@ -0,0 +1,3 @@ +Fix :func:`os.posix_fadvise` and :func:`os.posix_fallocate` on DragonFly BSD: +they raised :exc:`OSError` with a meaningless error code, +because these functions return -1 and set ``errno`` there. diff --git a/Misc/NEWS.d/next/Library/2026-07-22-13-00-00.gh-issue-154460.wCsFtM.rst b/Misc/NEWS.d/next/Library/2026-07-22-13-00-00.gh-issue-154460.wCsFtM.rst new file mode 100644 index 000000000000000..b0a7058f0519ff0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-22-13-00-00.gh-issue-154460.wCsFtM.rst @@ -0,0 +1,2 @@ +Fix :func:`time.strftime` and :meth:`datetime.datetime.strftime` returning a +wrong ISO 8601 week number (``%V``) on OpenBSD. diff --git a/Misc/NEWS.d/next/Library/2026-07-22-13-01-24.gh-issue-73458.byxSPi.rst b/Misc/NEWS.d/next/Library/2026-07-22-13-01-24.gh-issue-73458.byxSPi.rst new file mode 100644 index 000000000000000..27b6eeac5b520b2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-22-13-01-24.gh-issue-73458.byxSPi.rst @@ -0,0 +1,5 @@ +Fix :func:`logging.config.listen`: it left the caller waiting for the +``ready`` event forever if the server could not be started, +for example if the port was invalid or already in use. +It now also binds to an IPv6 address if the host has no IPv4 address, +for example if ``localhost`` is only aliased to ``::1``. diff --git a/Misc/NEWS.d/next/Library/2026-07-23-19-45-00.gh-issue-154551.Ct7pL9.rst b/Misc/NEWS.d/next/Library/2026-07-23-19-45-00.gh-issue-154551.Ct7pL9.rst new file mode 100644 index 000000000000000..c4db19d89364e07 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-23-19-45-00.gh-issue-154551.Ct7pL9.rst @@ -0,0 +1 @@ +Fix :func:`ctypes.util.find_library` returning ``None`` in non-UTF-8 locales. diff --git a/Misc/NEWS.d/next/Security/2026-01-16-11-41-06.gh-issue-143921.AeCOor.rst b/Misc/NEWS.d/next/Security/2026-01-16-11-41-06.gh-issue-143921.AeCOor.rst new file mode 100644 index 000000000000000..de62e020a95e6b2 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-01-16-11-41-06.gh-issue-143921.AeCOor.rst @@ -0,0 +1,2 @@ +Reject NUL, CR and LF characters in IMAP commands. Other control +characters are allowed and sent quoted. diff --git a/Misc/NEWS.d/next/Security/2026-01-16-11-58-19.gh-issue-143927.aviFeG.rst b/Misc/NEWS.d/next/Security/2026-01-16-11-58-19.gh-issue-143927.aviFeG.rst new file mode 100644 index 000000000000000..ca554997e5c3963 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-01-16-11-58-19.gh-issue-143927.aviFeG.rst @@ -0,0 +1,2 @@ +Normalize all line endings (CR, CRLF, and LF) to LF+TAB when writing +multi-line configparser values. diff --git a/Misc/NEWS.d/next/Security/2026-05-30-00-00-00.gh-issue-150743.httpdos.rst b/Misc/NEWS.d/next/Security/2026-05-30-00-00-00.gh-issue-150743.httpdos.rst new file mode 100644 index 000000000000000..4cc4e4b092da71c --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-05-30-00-00-00.gh-issue-150743.httpdos.rst @@ -0,0 +1,6 @@ +:mod:`http.client` now limits the number of chunked-response trailer lines +it will read to 100, and the number of interim (1xx) responses it will +skip to 100. A malicious or broken server could previously stream trailer +lines or ``100 Continue`` responses forever, hanging the client even when +a socket timeout was in use. +Reported by ``@YLChen-007`` via GHSA-w4q2-g22w-6fr4. diff --git a/Misc/NEWS.d/next/Security/2026-06-10-13-08-19.gh-issue-151558.mL74i2.rst b/Misc/NEWS.d/next/Security/2026-06-10-13-08-19.gh-issue-151558.mL74i2.rst new file mode 100644 index 000000000000000..74459d5680e21a3 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-06-10-13-08-19.gh-issue-151558.mL74i2.rst @@ -0,0 +1,3 @@ +Fixed an vulnerability in the :mod:`tarfile` ``data`` and ``tar`` extraction +filters where crafted archives could create a symlink pointing outside the +destination directory. This was a bypass of :cve:`2025-4330`. diff --git a/Misc/NEWS.d/next/Security/2026-06-16-14-58-02.gh-issue-151544._bexVy.rst b/Misc/NEWS.d/next/Security/2026-06-16-14-58-02.gh-issue-151544._bexVy.rst new file mode 100644 index 000000000000000..418e3b4b9677943 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-06-16-14-58-02.gh-issue-151544._bexVy.rst @@ -0,0 +1,4 @@ +:file:`Modules/Setup.local` is no longer used as a landmark to discover +whether Python is running in a source tree, as it could potentially affect +actual installs. The :file:`pybuilddir.txt` file is now the sole indicator +of running in a source tree. diff --git a/Misc/NEWS.d/next/Security/2026-06-23-13-28-16.gh-issue-151981.xBHEcU.rst b/Misc/NEWS.d/next/Security/2026-06-23-13-28-16.gh-issue-151981.xBHEcU.rst new file mode 100644 index 000000000000000..2123ab8e081b1d9 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-06-23-13-28-16.gh-issue-151981.xBHEcU.rst @@ -0,0 +1,2 @@ +In :mod:`tarfile`, seeking a stream now stops when end of the stream is +reached. diff --git a/Misc/NEWS.d/next/Security/2026-06-23-14-19-30.gh-issue-151987.8mNIMf.rst b/Misc/NEWS.d/next/Security/2026-06-23-14-19-30.gh-issue-151987.8mNIMf.rst new file mode 100644 index 000000000000000..9eea7b32c4d2b4f --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-06-23-14-19-30.gh-issue-151987.8mNIMf.rst @@ -0,0 +1,2 @@ +The :meth:`tarfile.TarFile.extract` method now applies the given filter when +it extracts a link target from the archive as a fallback. diff --git a/Misc/NEWS.d/next/Security/2026-06-25-20-44-20.gh-issue-152216.VxiK5y.rst b/Misc/NEWS.d/next/Security/2026-06-25-20-44-20.gh-issue-152216.VxiK5y.rst new file mode 100644 index 000000000000000..2769048e001d694 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-06-25-20-44-20.gh-issue-152216.VxiK5y.rst @@ -0,0 +1 @@ +Update bundled `libexpat <https://libexpat.github.io/>`_ to version 2.8.2. diff --git a/Misc/NEWS.d/next/Security/2026-07-04-17-00-00.gh-issue-153030.RovkP6.rst b/Misc/NEWS.d/next/Security/2026-07-04-17-00-00.gh-issue-153030.RovkP6.rst new file mode 100644 index 000000000000000..d1d60593f4ba7d2 --- /dev/null +++ b/Misc/NEWS.d/next/Security/2026-07-04-17-00-00.gh-issue-153030.RovkP6.rst @@ -0,0 +1,3 @@ +Fixed quadratic complexity in incremental parsing of long unterminated +constructs (such as tags or comments) in :class:`html.parser.HTMLParser`, +which could be exploited for a denial of service. diff --git a/Misc/NEWS.d/next/Tests/2026-04-24-01-38-56.gh-issue-148853._uM4_Q.rst b/Misc/NEWS.d/next/Tests/2026-04-24-01-38-56.gh-issue-148853._uM4_Q.rst new file mode 100644 index 000000000000000..9d3fbc2590dc7a2 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-04-24-01-38-56.gh-issue-148853._uM4_Q.rst @@ -0,0 +1,2 @@ +Fix tests failing on FreeBSD in test.support's +in_systemd_nspawn_sync_suppressed() due to unreadable /run directory. diff --git a/Misc/NEWS.d/next/Tests/2026-06-16-05-11-38.gh-issue-151096.Kq3Lp9.rst b/Misc/NEWS.d/next/Tests/2026-06-16-05-11-38.gh-issue-151096.Kq3Lp9.rst new file mode 100644 index 000000000000000..3251c6fb094723b --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-06-16-05-11-38.gh-issue-151096.Kq3Lp9.rst @@ -0,0 +1,2 @@ +Fix ``test_embed`` failing when CPython is configured with a split exec prefix +(``--exec-prefix`` differing from ``--prefix``). diff --git a/Misc/NEWS.d/next/Tests/2026-06-22-19-45-00.gh-issue-151626.K9pZ2x.rst b/Misc/NEWS.d/next/Tests/2026-06-22-19-45-00.gh-issue-151626.K9pZ2x.rst new file mode 100644 index 000000000000000..9858415018ae1ba --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-06-22-19-45-00.gh-issue-151626.K9pZ2x.rst @@ -0,0 +1,5 @@ +Fix several tests in ``test.test_inspect``, ``test.test_import``, +``test.test_importlib``, ``test.test_py_compile`` and +``test.test_compileall`` that failed when the test suite was run with +:envvar:`PYTHONPYCACHEPREFIX` set. These tests now neutralize the pycache +prefix where they assume the default ``__pycache__`` bytecode layout. diff --git a/Misc/NEWS.d/next/Tests/2026-07-19-18-20-00.gh-issue-154144.tcbool.rst b/Misc/NEWS.d/next/Tests/2026-07-19-18-20-00.gh-issue-154144.tcbool.rst new file mode 100644 index 000000000000000..dc1ac8b4035153c --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-07-19-18-20-00.gh-issue-154144.tcbool.rst @@ -0,0 +1 @@ +Fix building the :mod:`!_testcapi` module on NetBSD. diff --git a/Misc/NEWS.d/next/Tests/2026-07-19-20-15-00.gh-issue-154167.sigint.rst b/Misc/NEWS.d/next/Tests/2026-07-19-20-15-00.gh-issue-154167.sigint.rst new file mode 100644 index 000000000000000..b2620f4a0bee54c --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-07-19-20-15-00.gh-issue-154167.sigint.rst @@ -0,0 +1,3 @@ +The test runner (regrtest) now restores the default SIGINT handler if it +was inherited as ignored, so the test suite no longer hangs when run as +a shell background job. diff --git a/Misc/NEWS.d/next/Tests/2026-07-20-11-30-00.gh-issue-154211.hugestack.rst b/Misc/NEWS.d/next/Tests/2026-07-20-11-30-00.gh-issue-154211.hugestack.rst new file mode 100644 index 000000000000000..8030db869c7ba6e --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-07-20-11-30-00.gh-issue-154211.hugestack.rst @@ -0,0 +1,3 @@ +Add ``test.support.skip_if_huge_c_stack()`` and use it to skip tests +that exhaust the C stack if the stack limit is very large +(e.g. on DragonFly BSD or with ``ulimit -s unlimited``). diff --git a/Misc/NEWS.d/next/Tests/2026-07-24-12-00-00.gh-issue-76595.KfF5xR.rst b/Misc/NEWS.d/next/Tests/2026-07-24-12-00-00.gh-issue-76595.KfF5xR.rst new file mode 100644 index 000000000000000..c344ee2f2cb30b7 --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2026-07-24-12-00-00.gh-issue-76595.KfF5xR.rst @@ -0,0 +1 @@ +Add C API tests for :c:func:`PyCapsule_Import`. diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-07-24-09-30-00.gh-issue-154580.Kp3nQ2.rst b/Misc/NEWS.d/next/Tools-Demos/2026-07-24-09-30-00.gh-issue-154580.Kp3nQ2.rst new file mode 100644 index 000000000000000..f6a039245b015ed --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2026-07-24-09-30-00.gh-issue-154580.Kp3nQ2.rst @@ -0,0 +1,3 @@ +Fix ``python-gdb.py`` raising :exc:`UnicodeEncodeError` when pretty-printing a +non-ASCII :class:`str` in a locale whose host charset cannot encode it, such as +any non-ASCII string in the C locale. diff --git a/Misc/NEWS.d/next/Windows/2026-04-21-16-07-11.gh-issue-140146.TAcUHA.rst b/Misc/NEWS.d/next/Windows/2026-04-21-16-07-11.gh-issue-140146.TAcUHA.rst new file mode 100644 index 000000000000000..2d3e42c86fd521f --- /dev/null +++ b/Misc/NEWS.d/next/Windows/2026-04-21-16-07-11.gh-issue-140146.TAcUHA.rst @@ -0,0 +1,3 @@ +Prevent :mod:`tkinter` from hanging on Windows if stdin is redirected to a pipe in an +interactive session. This is helpful for testing interactive usage of +tkinter from a script, for example as part of the cpython test suite. diff --git a/Misc/NEWS.d/next/macOS/2026-07-17-23-35-36.gh-issue-124111.dmM4lk.rst b/Misc/NEWS.d/next/macOS/2026-07-17-23-35-36.gh-issue-124111.dmM4lk.rst new file mode 100644 index 000000000000000..48ab66407831a86 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2026-07-17-23-35-36.gh-issue-124111.dmM4lk.rst @@ -0,0 +1 @@ +Update macOS installer to use Tcl/Tk 9.0.4. diff --git a/Misc/externals.spdx.json b/Misc/externals.spdx.json index 69f3beec82ed342..cfc57981092c2e7 100644 --- a/Misc/externals.spdx.json +++ b/Misc/externals.spdx.json @@ -70,42 +70,42 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "6bb739ecddbd2cfb6d255eb5898437a9b5739277dee931338d3275bac5d96ba2" + "checksumValue": "ca94e7c6c223d9caf77bb51aac5949186379608ea2a0cad3aa8bdf31856912e9" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.0.16.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/openssl-3.5.7.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:openssl:openssl:3.0.16:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:openssl:openssl:3.5.7:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "openssl", "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.0.16" + "versionInfo": "3.5.7" }, { "SPDXID": "SPDXRef-PACKAGE-sqlite", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "e335aeb44fa36cde60ecbb6a9f8be6f5d449d645ce9b0199ee53a7e6728d19d2" + "checksumValue": "fb5ab81f27612b0a7b4861ba655906c76dc85ee969e7a4905d2075aff931e8d0" } ], - "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.49.1.0.tar.gz", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/sqlite-3.50.4.0.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.49.1.0:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:sqlite:sqlite:3.50.4.0:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], "licenseConcluded": "NOASSERTION", "name": "sqlite", "primaryPackagePurpose": "SOURCE", - "versionInfo": "3.49.1.0" + "versionInfo": "3.50.4.0" }, { "SPDXID": "SPDXRef-PACKAGE-tcl-core", diff --git a/Misc/python.man b/Misc/python.man index 15174b62d5fea4a..a65fb98a697b503 100644 --- a/Misc/python.man +++ b/Misc/python.man @@ -320,82 +320,105 @@ a regular expression on the warning message. .TP .BI "\-X " option Set implementation-specific option. The following options are available: - - \fB\-X cpu_count=\fIN\fR: override the return value of \fIos.cpu_count()\fR; - \fB\-X cpu_count=default\fR cancels overriding; also \fBPYTHON_CPU_COUNT\fI - - \fB\-X dev\fR: enable CPython's "development mode", introducing additional - runtime checks which are too expensive to be enabled by default. It - will not be more verbose than the default if the code is correct: new - warnings are only emitted when an issue is detected. Effect of the - developer mode: - * Add default warning filter, as \fB\-W default\fR - * Install debug hooks on memory allocators: see the - PyMem_SetupDebugHooks() C function - * Enable the faulthandler module to dump the Python traceback on a - crash - * Enable asyncio debug mode - * Set the dev_mode attribute of sys.flags to True - * io.IOBase destructor logs close() exceptions - - \fB\-X importtime\fR: show how long each import takes. It shows module name, - cumulative time (including nested imports) and self time (excluding - nested imports). Note that its output may be broken in multi-threaded - application. Typical usage is - \fBpython3 \-X importtime \-c 'import asyncio'\fR - - \fB\-X importtime=2\fR enables additional output that indicates when an - imported module has already been loaded. In such cases, the string - \fBcached\fR will be printed in both time columns. - - \fB\-X faulthandler\fR: enable faulthandler - - \fB\-X frozen_modules=\fR[\fBon\fR|\fBoff\fR]: whether or not frozen modules - should be used. - The default is "on" (or "off" if you are running a local build). - - \fB\-X gil=\fR[\fB0\fR|\fB1\fR]: enable (1) or disable (0) the GIL; also - \fBPYTHON_GIL\fR - Only available in builds configured with \fB\-\-disable\-gil\fR. - - \fB\-X int_max_str_digits=\fInumber\fR: limit the size of int<->str conversions. - This helps avoid denial of service attacks when parsing untrusted data. - The default is sys.int_info.default_max_str_digits. 0 disables. - - \fB\-X no_debug_ranges\fR: disable the inclusion of the tables mapping extra - location information (end line, start column offset and end column - offset) to every instruction in code objects. This is useful when - smaller code objects and pyc files are desired as well as suppressing - the extra visual location indicators when the interpreter displays - tracebacks. - - \fB\-X perf\fR: support the Linux "perf" profiler; also \fBPYTHONPERFSUPPORT=1\fR - - \fB\-X perf_jit\fR: support the Linux "perf" profiler with DWARF support; - also \fBPYTHON_PERF_JIT_SUPPORT=1\fR - - \fB\-X presite=\fIMOD\fR: import this module before site; also \fBPYTHON_PRESITE\fR - This only works on debug builds. - - \fB\-X pycache_prefix=\fIPATH\fR: enable writing .pyc files to a parallel - tree rooted at the given directory instead of to the code tree. - - \fB\-X showrefcount\fR: output the total reference count and number of used - memory blocks when the program finishes or after each statement in the - interactive interpreter. This only works on debug builds - - \fB\-X tracemalloc\fR: start tracing Python memory allocations using the - tracemalloc module. By default, only the most recent frame is stored in a - traceback of a trace. Use \-X tracemalloc=NFRAME to start tracing with a - traceback limit of NFRAME frames - - \fB\-X utf8\fR: enable UTF-8 mode for operating system interfaces, - overriding the default locale-aware mode. \fB\-X utf8=0\fR explicitly - disables UTF-8 mode (even when it would otherwise activate - automatically). See \fBPYTHONUTF8\fR for more details - - \fB\-X warn_default_encoding\fR: enable opt-in EncodingWarning for 'encoding=None' - +.RS +.TP +\fB\-X cpu_count=\fIN\fR +Override the return value of \fIos.cpu_count()\fR. +\fB\-X cpu_count=default\fR cancels overriding. +See also \fBPYTHON_CPU_COUNT\fR. +.TP +\fB\-X dev\fR +Enable CPython's "development mode", introducing additional +runtime checks which are too expensive to be enabled by default. It +will not be more verbose than the default if the code is correct: new +warnings are only emitted when an issue is detected. Effect of the +developer mode: +.RS +.IP \(bu 2 +Add default warning filter, as \fB\-W default\fR. +.IP \(bu 2 +Install debug hooks on memory allocators: see the +PyMem_SetupDebugHooks() C function. +.IP \(bu 2 +Enable the faulthandler module to dump the Python traceback on a crash. +.IP \(bu 2 +Enable asyncio debug mode. +.IP \(bu 2 +Set the dev_mode attribute of sys.flags to True. +.IP \(bu 2 +io.IOBase destructor logs close() exceptions. +.RE +.TP +\fB\-X importtime\fR +Show how long each import takes. It shows module name, +cumulative time (including nested imports) and self time (excluding +nested imports). Note that its output may be broken in multi-threaded +application. Typical usage is +\fBpython3 \-X importtime \-c 'import asyncio'\fR. +.IP +\fB\-X importtime=2\fR enables additional output that indicates when an +imported module has already been loaded. In such cases, the string +\fBcached\fR will be printed in both time columns. +.TP +\fB\-X faulthandler\fR +Enable faulthandler. +.TP +\fB\-X frozen_modules=\fR[\fBon\fR|\fBoff\fR] +Whether or not frozen modules should be used. +The default is "on" (or "off" if you are running a local build). +.TP +\fB\-X gil=\fR[\fB0\fR|\fB1\fR] +Enable (1) or disable (0) the GIL. See also \fBPYTHON_GIL\fR. +Only available in builds configured with \fB\-\-disable\-gil\fR. +.TP +\fB\-X int_max_str_digits=\fInumber\fR +Limit the size of int<->str conversions. +This helps avoid denial of service attacks when parsing untrusted data. +The default is sys.int_info.default_max_str_digits. 0 disables. +.TP +\fB\-X no_debug_ranges\fR +Disable the inclusion of the tables mapping extra +location information (end line, start column offset and end column +offset) to every instruction in code objects. This is useful when +smaller code objects and pyc files are desired as well as suppressing +the extra visual location indicators when the interpreter displays +tracebacks. +.TP +\fB\-X perf\fR +Support the Linux "perf" profiler. See also \fBPYTHONPERFSUPPORT=1\fR. +.TP +\fB\-X perf_jit\fR +Support the Linux "perf" profiler with DWARF support. +See also \fBPYTHON_PERF_JIT_SUPPORT=1\fR. +.TP +\fB\-X presite=\fIMOD\fR +Import this module before site. See also \fBPYTHON_PRESITE\fR. +This only works on debug builds. +.TP +\fB\-X pycache_prefix=\fIPATH\fR +Enable writing .pyc files to a parallel +tree rooted at the given directory instead of to the code tree. +.TP +\fB\-X showrefcount\fR +Output the total reference count and number of used +memory blocks when the program finishes or after each statement in the +interactive interpreter. This only works on debug builds. +.TP +\fB\-X tracemalloc\fR +Start tracing Python memory allocations using the +tracemalloc module. By default, only the most recent frame is stored in a +traceback of a trace. Use \fB\-X tracemalloc=\fINFRAME\fR to start tracing with a +traceback limit of NFRAME frames. +.TP +\fB\-X utf8\fR +Enable UTF-8 mode for operating system interfaces, +overriding the default locale-aware mode. \fB\-X utf8=0\fR explicitly +disables UTF-8 mode (even when it would otherwise activate +automatically). See \fBPYTHONUTF8\fR for more details. +.TP +\fB\-X warn_default_encoding\fR +Enable opt-in EncodingWarning for 'encoding=None'. +.RE .TP .B \-x Skip the first line of the source. This is intended for a DOS @@ -529,11 +552,11 @@ See also the \fB\-X frozen_modules\fR option. If this variable is set to 1, the global interpreter lock (GIL) will be forced on. Setting it to 0 forces the GIL off. Only available in builds configured with \fB\-\-disable\-gil\fP. +.IP +This is equivalent to the \fB\-X gil\fR option. .IP PYTHON_HISTORY This environment variable can be used to set the location of a history file (on Unix, it is \fI~/.python_history\fP by default). -.IP -This is equivalent to the \fB\-X gil\fR option. .IP PYTHONNODEBUGRANGES If this variable is set, it disables the inclusion of the tables mapping extra location information (end line, start column offset and end column diff --git a/Misc/sbom.spdx.json b/Misc/sbom.spdx.json index 4a697d047ca6e40..4b277bcdfca8b9f 100644 --- a/Misc/sbom.spdx.json +++ b/Misc/sbom.spdx.json @@ -6,11 +6,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "39e6f567a10e36b2e77727e98e60bbcb3eb3af0b" + "checksumValue": "f1b1126ed7da8f2068302e7a692b0600e6f94b07" }, { "algorithm": "SHA256", - "checksumValue": "122f2c27000472a201d337b9b31f7eb2b52d091b02857061a8880371612d9534" + "checksumValue": "31b15de82aa19a845156169a17a5488bf597e561b2c318d159ed583139b25e87" } ], "fileName": "Modules/expat/COPYING" @@ -48,11 +48,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "6984055af7b4e01429d8ebc910fe2be900d8ee9c" + "checksumValue": "00c5e72b384f5c305be613729184ad37bb491238" }, { "algorithm": "SHA256", - "checksumValue": "7c16a5cf0eea844ae579db083b8d75f23a71859cac77e3c4cb7a8fa3b7621685" + "checksumValue": "eb43180fbdca40e36d9558060e6e654ef4c451ca656ad679e9e1269eb45456b3" } ], "fileName": "Modules/expat/expat.h" @@ -62,15 +62,29 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "9e615c6e5c3ba00670f674a6b071bb855b0b563d" + "checksumValue": "d8f9211d52ff0384e229e4d4d56adae5db2d7f91" }, { "algorithm": "SHA256", - "checksumValue": "3d90a4b65c40a3f848c36100f4d73b933a015c7b7cd85c28e4331a6b845c1ad0" + "checksumValue": "b77f8192baf90aaa41f7023bc68fd1f22ab2552f98758271a1e090544537def5" } ], "fileName": "Modules/expat/expat_external.h" }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-fallthrough.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "69db5031480c50a1e35a51190bd64fcb816b6caf" + }, + { + "algorithm": "SHA256", + "checksumValue": "3dac2e4fdec819ede1b081ef776f2421c98ab509f69d5647a21d63d651179df2" + } + ], + "fileName": "Modules/expat/fallthrough.h" + }, { "SPDXID": "SPDXRef-FILE-Modules-expat-iasciitab.h", "checksums": [ @@ -90,11 +104,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "60b0ee8b4a93ef0276193ed1051c15ecab73c02e" + "checksumValue": "2555e70b29c1efc0af40879daafd12f8b36aca2c" }, { "algorithm": "SHA256", - "checksumValue": "6af6e8fbf5c83c1431464a2811b10ea2d1ff64c0eabfd9f18b1d4e53bf400c35" + "checksumValue": "4feb1df53898a48ae0ae04b5d0352c90395c8e693e5c2675f8ced41903d6fa94" } ], "fileName": "Modules/expat/internal.h" @@ -113,6 +127,20 @@ ], "fileName": "Modules/expat/latin1tab.h" }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-memory-sanitizer.h", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "abdf1b9e9642176466ecbb77d5f48e502d01d0d8" + }, + { + "algorithm": "SHA256", + "checksumValue": "10c66f1cf9dc28608d57cb19adc8c7c654800e8b5b1b9be176eaed802deca2d9" + } + ], + "fileName": "Modules/expat/memory_sanitizer.h" + }, { "SPDXID": "SPDXRef-FILE-Modules-expat-nametab.h", "checksums": [ @@ -132,11 +160,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "aca27f46d9fd387b63ce7ff2e4f172cad130b39b" + "checksumValue": "6648f9c12a8d6f77a6eab5ef5c7caa976e7549aa" }, { "algorithm": "SHA256", - "checksumValue": "f537add526ecda8389503b7ef45fb52b6217e4dc171dcc3a8dc6903ff6134726" + "checksumValue": "c1be28dd62282e668d4daedfe756edb68b7da165c442202bb06a9e597cd5c289" } ], "fileName": "Modules/expat/siphash.h" @@ -169,16 +197,30 @@ ], "fileName": "Modules/expat/winconfig.h" }, + { + "SPDXID": "SPDXRef-FILE-Modules-expat-xcsinc.c", + "checksums": [ + { + "algorithm": "SHA1", + "checksumValue": "69fa7a25f555164f089fdcf290f3af6a1731621f" + }, + { + "algorithm": "SHA256", + "checksumValue": "071345c5ccfbe2e46a02839331ee60ee20048348b4408964bff48c4355473b9e" + } + ], + "fileName": "Modules/expat/xcsinc.c" + }, { "SPDXID": "SPDXRef-FILE-Modules-expat-xmlparse.c", "checksums": [ { "algorithm": "SHA1", - "checksumValue": "3db0435d69e5eb904c9c88400a5ab073a81049bc" + "checksumValue": "a5a5dd44ee8eeb73f42fa42ca1499cd282ddf6a6" }, { "algorithm": "SHA256", - "checksumValue": "633b272fa893dfbef539edbba35f1b11ecf09a13b89189105b0dfa6c7ecfc3bf" + "checksumValue": "5d2c99b576744edd9545cefe613e0577617f57e16c86387edc13d45710dc97d2" } ], "fileName": "Modules/expat/xmlparse.c" @@ -188,11 +230,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "ef767128d2dda99436712dcf3465dde5dbaab876" + "checksumValue": "c8769fcb93f00272a6e6ca560be633649c817ff7" }, { "algorithm": "SHA256", - "checksumValue": "71fb52aa302cf6f56e41943009965804f49ff2210d9bd15b258f70aaf70db772" + "checksumValue": "5b81f0eb0e144b611dbd1bc9e6037075a16bff94f823d57a81eb2a3e4999e91a" } ], "fileName": "Modules/expat/xmlrole.c" @@ -202,11 +244,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "c961fb1a80f7b0601a63e69fba793fe5f6dff157" + "checksumValue": "ac2964cca107f62dd133bfd4736a9a17defbc401" }, { "algorithm": "SHA256", - "checksumValue": "228470eb9181a9a7575b63137edcb61b817ee4e0923faffdbeba29e07c939713" + "checksumValue": "92e41f373b67f6e0dcd7735faef3c3f1e2c17fe59e007e6b74beef6a2e70fa88" } ], "fileName": "Modules/expat/xmlrole.h" @@ -216,11 +258,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "8394790c0199c8f88108542ad78f23095d28a3fe" + "checksumValue": "08dba893b34338e64b66f1044f542aa6760dbab1" }, { "algorithm": "SHA256", - "checksumValue": "5b16c671ccc42496374762768e4bf48f614aecfd2025a07925b8d94244aec645" + "checksumValue": "0c842b1876503e699517994317805dd2298d9e11c94dcc87c2ecb6c2b8e00bc0" } ], "fileName": "Modules/expat/xmltok.c" @@ -230,11 +272,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "7d2943a0128094455004b1a98007b98734221bae" + "checksumValue": "d126831eaa5158cff187a8c93f4bc1c8118f3b17" }, { "algorithm": "SHA256", - "checksumValue": "6b8919dc951606dc6f2b0175f8955a9ced901ce8bd08db47f291b6c04227ae7f" + "checksumValue": "91bf003a725a675761ea8d92cebc299a76fd28c3a950572f41bc7ce5327ee7b5" } ], "fileName": "Modules/expat/xmltok.h" @@ -244,11 +286,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "7756f7c0d3625ae7dde6cf7d386685ffacb57c7e" + "checksumValue": "3ccb9335589c3ff60aae50ac79b2cdac57999bee" }, { "algorithm": "SHA256", - "checksumValue": "a3fe18ff32b21fbcb7c190895c68158404e1b9fb449db6431bc08b261dc03938" + "checksumValue": "94eeaef9ed46d10f80f748924fbf41f950b4859d6f234f61fdb2c8f1a8e77bd1" } ], "fileName": "Modules/expat/xmltok_impl.c" @@ -272,11 +314,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "2d82d0a1201f78d478b30d108ff8fc27ee3e2672" + "checksumValue": "41b8c8fc275882c76d4210b7d40a18e506b07147" }, { "algorithm": "SHA256", - "checksumValue": "6ce6d03193279078d55280150fe91e7370370b504a6c123a79182f28341f3e90" + "checksumValue": "b2188c7e5fa5b33e355cf6cf342dfb8f6e23859f2a6b1ddf79841d7f84f7b196" } ], "fileName": "Modules/expat/xmltok_ns.c" @@ -300,11 +342,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "de7179fe6970e2b5d281dfed977ed91be635b8d2" + "checksumValue": "a57d53c35aa916ce0cd1567c8f10dcea58270321" }, { "algorithm": "SHA256", - "checksumValue": "c0ba888d87775c7d7f7d8a08dac7b3988fed81e11bb52396d90f762a8e90a7eb" + "checksumValue": "d68209032703137326f9aadd9abac8fe4a5c92d391e2f43b0414fa63087adc6b" } ], "fileName": "Modules/_hacl/Hacl_HMAC.h" @@ -314,11 +356,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "4b6e7696e8d84f322fb24b1fbb08ccb9b0e7d51b" + "checksumValue": "808af7ff8a2cb2b4ef3a9ce3dbfef58d90828c9f" }, { "algorithm": "SHA256", - "checksumValue": "50a65a34a7a7569eedf7fa864a7892eeee5840a7fdf6fa8f1e87d42c65f6c877" + "checksumValue": "6a492aa586f2d10b1b300ce8ce4c72c976ff7548fee667aded2253f99969ac87" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2b.c" @@ -328,11 +370,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "7c66ac004a1dcf3fee0ab9aa62d61972f029de3a" + "checksumValue": "4d767388a34e2a2000e0cbd31f06cf36af1ae10d" }, { "algorithm": "SHA256", - "checksumValue": "9a7239a01a4ee8defbe3ebd9f0d12c873a1dd8e0659070380b2eab3ab0177333" + "checksumValue": "fac493e96af252abcf5705f0ab4eec59c1f3bc8244e105d75a57c43552dd1569" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2b.h" @@ -342,11 +384,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "0f75e44a42775247a46acc2beaa6bae8f199a3d9" + "checksumValue": "1bb072f2be9e5d194274fdcc87825cb094e4b32e" }, { "algorithm": "SHA256", - "checksumValue": "03b612c24193464ed6848aeebbf44f9266b78ec6eed2486056211cde8992c49a" + "checksumValue": "9eb22953ce60dde9dc970fec9dfce9d94235f4b7ccd8f0151cad4707dc835d1d" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c" @@ -356,11 +398,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "7f273d26942233e5dcdfb4c1a16ff2486b15f899" + "checksumValue": "280fd03ee23e13ecf90711d2be8230035d957343" }, { "algorithm": "SHA256", - "checksumValue": "dbc0dacc68ed52dbf1b7d6fba2c87870317998bc046e65f6deaaa150625432f8" + "checksumValue": "817dcd05d06d804587fce7d8f2f3f42a6fcf6818d2419a3551ef0df70cb7125a" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2b_Simd256.h" @@ -384,11 +426,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "65bf44140691b046dcfed3ab1576dbf8bbf96dc5" + "checksumValue": "cdd6e9ca86dbede92d1a47b9224d2af70c599a71" }, { "algorithm": "SHA256", - "checksumValue": "0f98959dafffce039ade9d296f7a05bed151c9c512498f48e4b326a5523a240b" + "checksumValue": "f3204f3e60734d811b6630f879b69ce54eaf367f3fca5889c1026e7a1f3ee1e4" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2s.c" @@ -398,11 +440,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "2120c8c467aeebcc7c8b9678c15e79648433b91a" + "checksumValue": "79f47ab5458d88bce0f210a566aa117ce2125049" }, { "algorithm": "SHA256", - "checksumValue": "45735f7fe2dbbad7656d07854e9ec8176ad26c79f90dcc0fec0b9a59a6311ba7" + "checksumValue": "5cc96313f8c066f055c2819e473c79aeff086ba91a1449d54aa569127cd8601e" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2s.h" @@ -412,11 +454,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "0da9782455923aede8d8dce9dfdc38f4fc1de572" + "checksumValue": "4056bb6e3ed184400d1610bdfd4260b3fd05ccbb" }, { "algorithm": "SHA256", - "checksumValue": "2d17ae768fd3d7d6decddd8b4aaf23ce02a809ee62bb98da32c8a7f54acf92d0" + "checksumValue": "c93746df2f219cbb1634ee6fb0ab1c4cbd381d1f36c637114c68346c9935326d" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c" @@ -426,11 +468,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "9028e24c9876d9d16b2435ec29240c6b57bfe2a0" + "checksumValue": "c98890b6193baa3dbd472cfb67f22aea3dd0d3e1" }, { "algorithm": "SHA256", - "checksumValue": "062e3b856acac4f929c1e04b8264a754cad21ca6580215f7094a3f0a04edb912" + "checksumValue": "fe3f17bf237166f872ba88c8204333c4f64bdc4bb29c265448581c7bfea4b151" } ], "fileName": "Modules/_hacl/Hacl_Hash_Blake2s_Simd128.h" @@ -454,11 +496,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "38e8d96ef1879480780494058a93cec181f8d6d7" + "checksumValue": "61f678cd9234c6eab5d4409ae66f470b068b959b" }, { "algorithm": "SHA256", - "checksumValue": "61e77d2063cf60c96e9ce06af215efe5d42c43026833bffed5732326fe97ed1e" + "checksumValue": "5b91ed0339074e2e546119833398e2cdb7190c33a59c405bf43b2417c789547d" } ], "fileName": "Modules/_hacl/Hacl_Hash_MD5.c" @@ -468,11 +510,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "e67a9bc18358c57afaeff3a174893ddfdb52dfc6" + "checksumValue": "2ed70f612a998ef6c04d62f9487a1b2534e6d76a" }, { "algorithm": "SHA256", - "checksumValue": "16e982081f6c2fd03ea751fcc64f5a835c94652841836e231fe562b9e287f4bc" + "checksumValue": "a34534ef36bc8428ac1169ca5f4f1c17a0817507ebae388e3dd825d1a331f28d" } ], "fileName": "Modules/_hacl/Hacl_Hash_MD5.h" @@ -482,11 +524,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "986dd5ba0b34d15f3e5e5c656979aea1b502e8aa" + "checksumValue": "c7fc5c9721caf37c5a24c9beff27b0ac2ed68cc9" }, { "algorithm": "SHA256", - "checksumValue": "38d5f1f2e67a0eb30789f81fc56c07a6e7246e2b1be6c65485bcca1dcd0e0806" + "checksumValue": "ce08721d491f3b8a9bd4cde6c27bfcc8fc01471512ccca4bd3c0b764cb551d29" } ], "fileName": "Modules/_hacl/Hacl_Hash_SHA1.c" @@ -496,11 +538,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "f1ca21f1ee8b15ad9ccfbda72165b9d86912166c" + "checksumValue": "a3f9bdc9e73f80eb4a586dc180246cfb8267ffc0" }, { "algorithm": "SHA256", - "checksumValue": "4b2ad9ea93fdd9c2fdc521fc4e14e02550666c2717a23b85819db2e07ea555f3" + "checksumValue": "2d5cae94382f5473cf6d2654760375ff1ee9cebdb8d4506b63ba33f488de1559" } ], "fileName": "Modules/_hacl/Hacl_Hash_SHA1.h" @@ -510,11 +552,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "f732a6710fe3e13cd28130f0f20504e347d1c412" + "checksumValue": "fffe8c4f67669ac8ccd87c2e0f95db2427481df1" }, { "algorithm": "SHA256", - "checksumValue": "86cf32e4d1f3ba93a94108271923fdafe2204447792a918acf4a2250f352dbde" + "checksumValue": "f8af382de7e29a73c726f9c70770498ddd99e2c4702489ed6e634f0b68597c95" } ], "fileName": "Modules/_hacl/Hacl_Hash_SHA2.c" @@ -524,11 +566,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "f38cebeeca40a83aeb2cf5dfce578ffefe176d84" + "checksumValue": "68b35d0c573f7301ba4d6919b1680d55675a0d98" }, { "algorithm": "SHA256", - "checksumValue": "ee03bf9368d1a3a3c70cfd4e9391b2485466404db4a60bfc5319630cc314b590" + "checksumValue": "442d8997d2bcda20ddf628665e2e69945400e1ab3019bc14fc7c8e20db20c320" } ], "fileName": "Modules/_hacl/Hacl_Hash_SHA2.h" @@ -538,11 +580,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "50f75337b31f509b5bfcc7ebb3d066b82a0f1b33" + "checksumValue": "77d3d879dfa5949030bca0e8ee75d3d369ec54a7" }, { "algorithm": "SHA256", - "checksumValue": "c9e1442899e5b902fa39f413f1a3131f7ab5c2283d5100dc8ac675a7d5ebbdf1" + "checksumValue": "8744f5b6e054c3e5c44f413a60f9154b76dd7230135dcee26fd063755ec64be1" } ], "fileName": "Modules/_hacl/Hacl_Hash_SHA3.c" @@ -552,11 +594,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "01717207aef77174e328186d48c27517f6644c15" + "checksumValue": "db1008095b64b2f8ee2a4ce72fa7cfc4a622a108" }, { "algorithm": "SHA256", - "checksumValue": "620dded172e94cb3f25f9904b44977d91f2cc9573e41b38f19e929d083ae0308" + "checksumValue": "961a686186b76a1cd6a64bcfd06afdc8657b1f901bac991166f498ed16f9349a" } ], "fileName": "Modules/_hacl/Hacl_Hash_SHA3.h" @@ -566,11 +608,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "8140310f505bb2619a749777a91487d666237bcf" + "checksumValue": "eb224e26bc0503a48c88c837e28bbc7a9878ba5c" }, { "algorithm": "SHA256", - "checksumValue": "9d95e6a651c22185d9b7c38f363d30159f810e6fcdc2208f29492837ed891e82" + "checksumValue": "29fcf948a3715e09cbbd434cebb6105f276236a6e4947d237b7bf06634bf2432" } ], "fileName": "Modules/_hacl/Hacl_Streaming_HMAC.c" @@ -580,11 +622,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "49523144583a15d96ba1646af02dc292e633bf8f" + "checksumValue": "2d6c600024275c780e8313e0a5ab506e025bb4d6" }, { "algorithm": "SHA256", - "checksumValue": "78345519bf6789264f6792b809ee97a9ecf7cb5829c674c61e2d99bfdfdc36fc" + "checksumValue": "6450aef92f507fb0a8a3086b1d2792e7fca07121580c24fdaedd1b32e9ad0a76" } ], "fileName": "Modules/_hacl/Hacl_Streaming_HMAC.h" @@ -594,11 +636,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "372448599774a98e5c5d083e91f301ed1c4b822e" + "checksumValue": "ed283b95ebb772b05bdf802b70dbb301ece84e91" }, { "algorithm": "SHA256", - "checksumValue": "95d8e70ca4bc6aa98f6d2435ceb6410ead299b1f700fae1f5c603ec3f57ea551" + "checksumValue": "efc7bd11460744768425aedf4e004d3ad3397a5489752b80044ae785f30b78d4" } ], "fileName": "Modules/_hacl/Hacl_Streaming_Types.h" @@ -608,11 +650,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "c9651ef21479c4d8a3b04c5baa1902866dbb1cdf" + "checksumValue": "0a0b7f3714167ad45ddf5a6a48d76f525a119c9c" }, { "algorithm": "SHA256", - "checksumValue": "e039c82ba670606ca111573942baad800f75da467abbc74cd7d1fe175ebcdfaf" + "checksumValue": "135d4afb4812468885c963c9c87a55ba5fae9181df4431af5fbad08588dda229" } ], "fileName": "Modules/_hacl/Lib_Memzero0.c" @@ -622,11 +664,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "eaa543c778300238dc23034aafeada0951154af1" + "checksumValue": "ae06c415ae7e0e1e85558863f29d1b9013e46e9b" }, { "algorithm": "SHA256", - "checksumValue": "3fd2552d527a23110d61ad2811c774810efb1eaee008f136c2a0d609daa77f5b" + "checksumValue": "69dc2e78411e9b271100eb0d2a2d8dc39dd2348afc26f8b4cfba14c025102c7f" } ], "fileName": "Modules/_hacl/include/krml/FStar_UInt128_Verified.h" @@ -636,11 +678,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "41ee1e34ede7ef5b24b87d4ca816fd6d9fac8010" + "checksumValue": "edefe48ced707327fd6cdf3f18b3ca42dda9a9f7" }, { "algorithm": "SHA256", - "checksumValue": "d48ed03e504cb87793a310a9552fb3ba2ebd6fe90127b7d642c8740fba1b9748" + "checksumValue": "f01e3f0892935f3f659019875f580445b9ea23482afbe11ffbe7cdbd22dbd4d2" } ], "fileName": "Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h" @@ -678,11 +720,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "e01d7d493fbaceeedc4b1c6451d8240bcb9c903a" + "checksumValue": "7a943fbb8f55729a960f9b426e9ff2794453aca9" }, { "algorithm": "SHA256", - "checksumValue": "c2f0a43884771f24d7cb744b79818b160020d2739b2881b2054cfc97fb2e7b4a" + "checksumValue": "08bb43ef5626a8450f985da0af345b6658ac8bcc4585f77271decd0e41fc02e0" } ], "fileName": "Modules/_hacl/include/krml/internal/target.h" @@ -692,11 +734,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "3f66313d16891f43b21c1a736081c2c6d46bf370" + "checksumValue": "b4e6c5fc5e17864cc2bf452db2688be733d6df72" }, { "algorithm": "SHA256", - "checksumValue": "78e9bff9124968108e1699e1c6388e3d4ec9bd72dd8adff49734a69ab380ee5c" + "checksumValue": "404691cf9b2269ecc754ceca27bb8f8f029f1c8deaa4d967990dcf5ce08cd016" } ], "fileName": "Modules/_hacl/include/krml/internal/types.h" @@ -706,11 +748,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "e18efc9239a5df0f222b5f7b0a65f72509d7e304" + "checksumValue": "8c3e09e00459951e060ad469c1fa31eeb9b6b9cb" }, { "algorithm": "SHA256", - "checksumValue": "47dd5a7d21b5302255f9fff28884f65d3056fc3f54471ed62ec85fa1904f8aa5" + "checksumValue": "6d1a350ec272c83761f1789611d8f60947a4d69fed3d23d8eee38709a0ad2c0a" } ], "fileName": "Modules/_hacl/include/krml/lowstar_endianness.h" @@ -720,11 +762,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "aaa656e25a92ba83655e1398a97efa6981f60fc4" + "checksumValue": "c9517c43046de129f02da8b74e454bade4872c00" }, { "algorithm": "SHA256", - "checksumValue": "a59abc6e9b3019cb18976a15e634f5146bd965fc9babf4ccbf2b531164a34f85" + "checksumValue": "96c2a973bc01b1295f7af67c7de3839facfeee36c8848d7d88c62695de98ab21" } ], "fileName": "Modules/_hacl/internal/Hacl_HMAC.h" @@ -734,11 +776,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "0741cb8497309d648428be1e7b5944b1fc167187" + "checksumValue": "1c5eeb5d1866acb6bc8b4e5a01a107e3a87f5694" }, { "algorithm": "SHA256", - "checksumValue": "f9b923a566d62de047c753637143d439ca1c25221c08352ddc1738ff4a6ac721" + "checksumValue": "f262738390f56e8e9692acadecd1434c688075047d788283e1fb45d920ea6956" } ], "fileName": "Modules/_hacl/internal/Hacl_Hash_Blake2b.h" @@ -748,11 +790,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "878ae284c93824b80b1763e8b3e6be3c410777a8" + "checksumValue": "b600c1b5eafc34b29a778186737d8a51e2342d85" }, { "algorithm": "SHA256", - "checksumValue": "49df6223f6403daf503a1af1a3d2f943d30b5889fe7ed20299c3df24c1e3853d" + "checksumValue": "f22e088ad9c2eb739aefc3685ef3ab1ccaab3c5ef2e5d06cc846ad9f8e3d2669" } ], "fileName": "Modules/_hacl/internal/Hacl_Hash_Blake2b_Simd256.h" @@ -762,11 +804,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "25552d8cbf8aa345907635b38f284eec9075301e" + "checksumValue": "eb618ecc6ca2830066448f5f6d4df84a5c09f0f4" }, { "algorithm": "SHA256", - "checksumValue": "a3424cf4c5518654908086bbbf5d465715ec3b23625ef0cadc29492d1f90366c" + "checksumValue": "a4728e43deb0a9d8213b8ddcbda68a63bc73a12fb99aad54b7d28b314776a0d4" } ], "fileName": "Modules/_hacl/internal/Hacl_Hash_Blake2s.h" @@ -776,11 +818,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "54a712fc3ed5a817288351cbac5b7d9afa7e379f" + "checksumValue": "5441b6a97cc053c332b29477238d70fa011c74ac" }, { "algorithm": "SHA256", - "checksumValue": "c6abae648b8a1e9d5631c0a959620cad1f7e92ce522e07c3416199fe51debef6" + "checksumValue": "dad568d256a2ccbbbcdd419fe0543ea7137d8065a713a5f009aa52521c0f7f6a" } ], "fileName": "Modules/_hacl/internal/Hacl_Hash_Blake2s_Simd128.h" @@ -790,11 +832,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "c15c5f83bbb9f62611c49f0f8f723eaab1a27488" + "checksumValue": "7081b58f28568f600d4624dd5bd6f735191e068b" }, { "algorithm": "SHA256", - "checksumValue": "95cd5d91c4a9217901d0b3395dcd8881e62e2055d723b532ec5176386a636d22" + "checksumValue": "305af1422213ed97e8e5d3d8a9dfee4f21cb2fcd2acf65ee7303ce00b2b4bd2a" } ], "fileName": "Modules/_hacl/internal/Hacl_Hash_MD5.h" @@ -804,11 +846,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "7b8717e3a24e7e16a34b251d0d02da6f68439695" + "checksumValue": "b6fb6219cb40d039e789666e34b43461e3b5c82a" }, { "algorithm": "SHA256", - "checksumValue": "9473d8bc9506fe0053d7d98c225d4873011329863f1c4a8e93e43fc71bd1f314" + "checksumValue": "63c58363ff95e8146d5628dab2da25bc2fd0e8b590fd4823b512c33f843355bb" } ], "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA1.h" @@ -818,11 +860,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "e319c949f5a2dd765be2c8c7ff77bfe52ee6c7da" + "checksumValue": "bcc71e702df1070cb0081cb983aec7683e036719" }, { "algorithm": "SHA256", - "checksumValue": "75261448e51c3eb1ba441e973b193e23570b167f67743942ee2ee57417491c9f" + "checksumValue": "709c6272f77e2368f6cc0bf36527e3b16dd5d5f3dc26a2afdef8238200af8770" } ], "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA2.h" @@ -832,11 +874,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "dbd92415c31606804102b79d5ba3d1752fe03887" + "checksumValue": "e3b5e6add5357760554b032b818992ce9bc211e0" }, { "algorithm": "SHA256", - "checksumValue": "5d74a76a0ac3659a1ae1276c3ca55521f09e83d2f0039f5c519a76f8f3c76a8e" + "checksumValue": "cf49d536a5663379fb4517018b4b3f8a232f8d4c7ddc7edfd60163d13f98a530" } ], "fileName": "Modules/_hacl/internal/Hacl_Hash_SHA3.h" @@ -846,11 +888,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "ad788265f8e1b078c4d1cb6e90b8c031590e6baf" + "checksumValue": "09a0ea364fb073f5f622a763f1351fbf4bac4627" }, { "algorithm": "SHA256", - "checksumValue": "d8354a9b75e2470085fa7e538493130e81fa23a804a6a69d34da8fdcc941c038" + "checksumValue": "916d54d7217517f0360edea920dc21585fbe6d1c2458eac86826182e12c82ec2" } ], "fileName": "Modules/_hacl/internal/Hacl_Impl_Blake2_Constants.h" @@ -860,11 +902,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "2048f3cd61dbda2df862a2982ebaf24b6815ed51" + "checksumValue": "b082645b43f8841db5e9e57c0b9a3825fa7e52f5" }, { "algorithm": "SHA256", - "checksumValue": "b0f5a79c98525b0cb1659238e095641328b7da16a94cb57a0793e635d1da3653" + "checksumValue": "059cf0d31427abf84c626797a97c140630a1a6ead578005162f46fad46663389" } ], "fileName": "Modules/_hacl/internal/Hacl_Streaming_HMAC.h" @@ -874,11 +916,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "4e6b098e89fd447bd03f47b55208208456b20966" + "checksumValue": "1f85ed69e395829b3ac5af9e2d049af7708cb9cb" }, { "algorithm": "SHA256", - "checksumValue": "d54d947968ca125978d61fea844711b990f0a18ab0fbca87e41029004d9d04b6" + "checksumValue": "76997c7069a347ac78b65d26354a0324d54add4ca7a02e7be36d6d5e1c9702e0" } ], "fileName": "Modules/_hacl/internal/Hacl_Streaming_Types.h" @@ -930,11 +972,11 @@ "checksums": [ { "algorithm": "SHA1", - "checksumValue": "4a0bdb9496d49bbfa3ad50bb7854d8f099e84891" + "checksumValue": "682b069d3888f3e8f8cc90f1a49bac9d1a5903d2" }, { "algorithm": "SHA256", - "checksumValue": "b1a45149239ee7af7de769a3e9339950d47c199bb9eaa10edce8a00fde603b12" + "checksumValue": "8551d2c3fde03b92c6fab5febde00347bd9184ea0085077976863c7836e9669d" } ], "fileName": "Modules/_hacl/python_hacl_namespaces.h" @@ -1730,14 +1772,14 @@ "checksums": [ { "algorithm": "SHA256", - "checksumValue": "17aa6cfc5c4c219c09287abfc10bc13f0c06f30bb654b28bfe6f567ca646eb79" + "checksumValue": "ef7d1994f533c9e7343d6c19f31064fc8ebbcbcaa144be3812b4f43052a05f4c" } ], - "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_6_3/expat-2.6.3.tar.gz", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_2/expat-2.8.2.tar.gz", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.6.3:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:libexpat_project:libexpat:2.8.2:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1745,21 +1787,21 @@ "name": "expat", "originator": "Organization: Expat development team", "primaryPackagePurpose": "SOURCE", - "versionInfo": "2.6.3" + "versionInfo": "2.8.2" }, { "SPDXID": "SPDXRef-PACKAGE-hacl-star", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "02dfcf0c79d488b120d7f2c2a0f9206301c7927ed5106545e0b6f2aef88da76a" + "checksumValue": "61e48893f37cb2280d106cefacf6fb5afe84edf625fec39572d0ee94e1018f26" } ], - "downloadLocation": "https://github.com/hacl-star/hacl-star/archive/7720f6d4fc0468a99d5ea6120976bcc271e42727.zip", + "downloadLocation": "https://github.com/hacl-star/hacl-star/archive/8ba599b2f6c9701b3dc961db895b0856a2210f76.zip", "externalRefs": [ { "referenceCategory": "SECURITY", - "referenceLocator": "cpe:2.3:a:hacl-star:hacl-star:7720f6d4fc0468a99d5ea6120976bcc271e42727:*:*:*:*:*:*:*", + "referenceLocator": "cpe:2.3:a:hacl-star:hacl-star:8ba599b2f6c9701b3dc961db895b0856a2210f76:*:*:*:*:*:*:*", "referenceType": "cpe23Type" } ], @@ -1767,7 +1809,7 @@ "name": "hacl-star", "originator": "Organization: HACL* Developers", "primaryPackagePurpose": "SOURCE", - "versionInfo": "7720f6d4fc0468a99d5ea6120976bcc271e42727" + "versionInfo": "8ba599b2f6c9701b3dc961db895b0856a2210f76" }, { "SPDXID": "SPDXRef-PACKAGE-macholib", @@ -1840,6 +1882,11 @@ "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-expat" }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-fallthrough.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, { "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-iasciitab.h", "relationshipType": "CONTAINS", @@ -1855,6 +1902,11 @@ "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-expat" }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-memory-sanitizer.h", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, { "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-nametab.h", "relationshipType": "CONTAINS", @@ -1875,6 +1927,11 @@ "relationshipType": "CONTAINS", "spdxElementId": "SPDXRef-PACKAGE-expat" }, + { + "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xcsinc.c", + "relationshipType": "CONTAINS", + "spdxElementId": "SPDXRef-PACKAGE-expat" + }, { "relatedSpdxElement": "SPDXRef-FILE-Modules-expat-xmlparse.c", "relationshipType": "CONTAINS", diff --git a/Misc/stable_abi.toml b/Misc/stable_abi.toml index d3e1f0db0570239..b91c078b60a5d1a 100644 --- a/Misc/stable_abi.toml +++ b/Misc/stable_abi.toml @@ -1787,7 +1787,6 @@ [const.METH_COEXIST] added = '3.2' # METH_STACKLESS is undocumented -# METH_FASTCALL is not part of limited API. # The following are defined in private headers, but historically # they were exported as part of the stable ABI. @@ -1889,6 +1888,13 @@ added = '3.5' [data.PyModuleDef_Type] added = '3.5' +[const.Py_mod_create] + added = '3.5' +[const.Py_mod_exec] + added = '3.5' +[struct.PyModuleDef_Slot] + added = '3.5' + struct_abi_kind = 'full-abi' # New slots in 3.5: # d51374ed78a3e3145911a16cdf3b9b84b3ba7d15 - Matrix multiplication (PEP 465) @@ -2114,8 +2120,6 @@ # New method flags in 3.7 (PEP 590): -[const.METH_FASTCALL] - added = '3.7' [const.METH_METHOD] added = '3.7' @@ -2265,6 +2269,10 @@ [data.PyStructSequence_UnnamedField] added = '3.11' +# Added in 3.7 but in the Stable ABI from 3.10 +[const.METH_FASTCALL] + added = '3.10' + # Add stable Py_buffer API in Python 3.11 (https://bugs.python.org/issue45459) [struct.Py_buffer] added = '3.11' @@ -2293,6 +2301,10 @@ added = '3.11' [function.PyMemoryView_FromBuffer] added = '3.11' +[const.Py_bf_getbuffer] + added = '3.11' +[const.Py_bf_releasebuffer] + added = '3.11' # Constants for Py_buffer API added to this list in Python 3.11.1 (https://github.com/python/cpython/issues/98680) # (they were available with 3.11.0) @@ -2427,6 +2439,9 @@ added = '3.12' [const.Py_TPFLAGS_ITEMS_AT_END] added = '3.12' +[const.Py_mod_multiple_interpreters] + added = '3.12' + [function.PyImport_AddModuleRef] added = '3.13' [function.PyWeakref_GetRef] @@ -2505,6 +2520,8 @@ added = '3.13' [function.PyEval_GetFrameLocals] added = '3.13' +[const.Py_mod_gil] + added = '3.13' [function.Py_TYPE] added = '3.14' diff --git a/Misc/valgrind-python.supp b/Misc/valgrind-python.supp index 8b2027cd4527679..4df8a118f089b70 100644 --- a/Misc/valgrind-python.supp +++ b/Misc/valgrind-python.supp @@ -317,13 +317,8 @@ { Avoid problems w/readline doing a putenv and leaking on exit Memcheck:Leak - fun:malloc - fun:xmalloc - fun:sh_set_lines_and_columns - fun:_rl_get_screen_size - fun:_rl_init_terminal_io - obj:/lib/libreadline.so.4.3 - fun:rl_initialize + ... + fun:setup_readline } # Valgrind emits "Conditional jump or move depends on uninitialised value(s)" diff --git a/Modules/Setup.bootstrap.in b/Modules/Setup.bootstrap.in index 2b2e8cb3e3cacde..65a1fefe72e92ec 100644 --- a/Modules/Setup.bootstrap.in +++ b/Modules/Setup.bootstrap.in @@ -12,6 +12,8 @@ posix posixmodule.c _signal signalmodule.c _tracemalloc _tracemalloc.c _suggestions _suggestions.c +# needs libm and on some platforms librt +_datetime _datetimemodule.c # modules used by importlib, deepfreeze, freeze, runpy, and sysconfig _codecs _codecsmodule.c diff --git a/Modules/Setup.stdlib.in b/Modules/Setup.stdlib.in index 3a38a60a152e8c5..0f92ef49bbc3e44 100644 --- a/Modules/Setup.stdlib.in +++ b/Modules/Setup.stdlib.in @@ -56,9 +56,6 @@ @MODULE_CMATH_TRUE@cmath cmathmodule.c @MODULE__STATISTICS_TRUE@_statistics _statisticsmodule.c -# needs libm and on some platforms librt -@MODULE__DATETIME_TRUE@_datetime _datetimemodule.c - # _decimal uses libmpdec # either static libmpdec.a from Modules/_decimal/libmpdec or libmpdec.so # with ./configure --with-system-libmpdec @@ -178,8 +175,8 @@ @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c _testinternalcapi/test_lock.c _testinternalcapi/pytime.c _testinternalcapi/set.c _testinternalcapi/test_critical_sections.c _testinternalcapi/complex.c -@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c -@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c +@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/heaptype.c _testcapi/abstract.c _testcapi/unicode.c _testcapi/dict.c _testcapi/set.c _testcapi/list.c _testcapi/tuple.c _testcapi/getargs.c _testcapi/datetime.c _testcapi/docstring.c _testcapi/mem.c _testcapi/watchers.c _testcapi/long.c _testcapi/float.c _testcapi/complex.c _testcapi/numbers.c _testcapi/structmember.c _testcapi/exceptions.c _testcapi/code.c _testcapi/buffer.c _testcapi/pyatomic.c _testcapi/run.c _testcapi/file.c _testcapi/codec.c _testcapi/immortal.c _testcapi/gc.c _testcapi/hash.c _testcapi/time.c _testcapi/bytes.c _testcapi/object.c _testcapi/monitoring.c _testcapi/config.c _testcapi/import.c _testcapi/frame.c _testcapi/type.c _testcapi/function.c _testcapi/weakref.c +@MODULE__TESTLIMITEDCAPI_TRUE@_testlimitedcapi _testlimitedcapi.c _testlimitedcapi/abstract.c _testlimitedcapi/bytearray.c _testlimitedcapi/bytes.c _testlimitedcapi/capsule.c _testlimitedcapi/codec.c _testlimitedcapi/complex.c _testlimitedcapi/dict.c _testlimitedcapi/eval.c _testlimitedcapi/float.c _testlimitedcapi/heaptype_relative.c _testlimitedcapi/import.c _testlimitedcapi/list.c _testlimitedcapi/long.c _testlimitedcapi/object.c _testlimitedcapi/pyos.c _testlimitedcapi/set.c _testlimitedcapi/sys.c _testlimitedcapi/tuple.c _testlimitedcapi/unicode.c _testlimitedcapi/vectorcall_limited.c _testlimitedcapi/version.c _testlimitedcapi/file.c _testlimitedcapi/weakref.c _testlimitedcapi/run.c @MODULE__TESTCLINIC_TRUE@_testclinic _testclinic.c @MODULE__TESTCLINIC_LIMITED_TRUE@_testclinic_limited _testclinic_limited.c diff --git a/Modules/_abc.c b/Modules/_abc.c index d6a953b336025d2..c589cc2baa61397 100644 --- a/Modules/_abc.c +++ b/Modules/_abc.c @@ -911,14 +911,14 @@ _abc.get_cache_token Returns the current ABC cache token. -The token is an opaque object (supporting equality testing) identifying the -current version of the ABC cache for virtual subclasses. The token changes -with every call to register() on any ABC. +The token is an opaque object (supporting equality testing) identifying +the current version of the ABC cache for virtual subclasses. The token +changes with every call to register() on any ABC. [clinic start generated code]*/ static PyObject * _abc_get_cache_token_impl(PyObject *module) -/*[clinic end generated code: output=c7d87841e033dacc input=70413d1c423ad9f9]*/ +/*[clinic end generated code: output=c7d87841e033dacc input=d87acc04492f6bf3]*/ { _abcmodule_state *state = get_abc_state(module); return PyLong_FromUnsignedLongLong(get_invalidation_counter(state)); diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 5f9181395c4828c..46a55f8b91d387e 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -955,8 +955,7 @@ FutureObj_traverse(PyObject *op, visitproc visit, void *arg) Py_VISIT(fut->fut_cancel_msg); Py_VISIT(fut->fut_cancelled_exc); Py_VISIT(fut->fut_awaited_by); - PyObject_VisitManagedDict((PyObject *)fut, visit, arg); - return 0; + return PyObject_VisitManagedDict((PyObject *)fut, visit, arg); } /*[clinic input] @@ -967,12 +966,13 @@ Return the result this future represents. If the future has been cancelled, raises CancelledError. If the future's result isn't yet available, raises InvalidStateError. If -the future is done and has an exception set, this exception is raised. +the future is done and has an exception set, this exception is +raised. [clinic start generated code]*/ static PyObject * _asyncio_Future_result_impl(FutureObj *self) -/*[clinic end generated code: output=f35f940936a4b1e5 input=61d89f48e4c8b670]*/ +/*[clinic end generated code: output=f35f940936a4b1e5 input=ee20e126776cbb04]*/ { asyncio_state *state = get_asyncio_state_by_def((PyObject *)self); PyObject *result; @@ -1107,15 +1107,15 @@ _asyncio.Future.add_done_callback Add a callback to be run when the future becomes done. -The callback is called with a single argument - the future object. If -the future is already done when this is called, the callback is +The callback is called with a single argument - the future object. +If the future is already done when this is called, the callback is scheduled with call_soon. [clinic start generated code]*/ static PyObject * _asyncio_Future_add_done_callback_impl(FutureObj *self, PyTypeObject *cls, PyObject *fn, PyObject *context) -/*[clinic end generated code: output=922e9a4cbd601167 input=37d97f941beb7b3e]*/ +/*[clinic end generated code: output=922e9a4cbd601167 input=f4f6adb074cd3e0f]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); if (context == NULL) { @@ -1264,15 +1264,15 @@ _asyncio.Future.cancel Cancel the future and schedule callbacks. -If the future is already done or cancelled, return False. Otherwise, -change the future's state to cancelled, schedule the callbacks and -return True. +If the future is already done or cancelled, return False. +Otherwise, change the future's state to cancelled, schedule the +callbacks and return True. [clinic start generated code]*/ static PyObject * _asyncio_Future_cancel_impl(FutureObj *self, PyTypeObject *cls, PyObject *msg) -/*[clinic end generated code: output=074956f35904b034 input=44ab4003da839970]*/ +/*[clinic end generated code: output=074956f35904b034 input=0c9157547a964c4c]*/ { asyncio_state *state = get_asyncio_state_by_cls(cls); ENSURE_FUTURE_ALIVE(state, self) @@ -1304,13 +1304,13 @@ _asyncio.Future.done Return True if the future is done. -Done means either that a result / exception are available, or that the -future was cancelled. +Done means either that a result / exception are available, or that +the future was cancelled. [clinic start generated code]*/ static PyObject * _asyncio_Future_done_impl(FutureObj *self) -/*[clinic end generated code: output=244c5ac351145096 input=7204d3cc63bef7f3]*/ +/*[clinic end generated code: output=244c5ac351145096 input=acf2c2347f3c01d8]*/ { if (!future_is_alive(self) || self->fut_state == STATE_PENDING) { Py_RETURN_FALSE; @@ -1755,7 +1755,8 @@ static PyMethodDef FutureType_methods[] = { _ASYNCIO_FUTURE_DONE_METHODDEF _ASYNCIO_FUTURE_GET_LOOP_METHODDEF _ASYNCIO_FUTURE__MAKE_CANCELLED_ERROR_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("Futures are generic over the type of their results")}, {NULL, NULL} /* Sentinel */ }; @@ -2255,7 +2256,7 @@ enter_task(PyObject *loop, PyObject *task) PyExc_RuntimeError, "Cannot enter into task %R while another " \ "task %R is being executed.", - task, ts->asyncio_running_task, NULL); + task, ts->asyncio_running_task); return -1; } @@ -2278,7 +2279,7 @@ leave_task(PyObject *loop, PyObject *task) PyExc_RuntimeError, "Invalid attempt to leave task %R while " \ "task %R is entered.", - task, ts->asyncio_running_task ? ts->asyncio_running_task : Py_None, NULL); + task, ts->asyncio_running_task ? ts->asyncio_running_task : Py_None); return -1; } Py_CLEAR(ts->asyncio_running_task); @@ -2343,7 +2344,7 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, self->task_log_destroy_pending = 0; PyErr_Format(PyExc_TypeError, "a coroutine was expected, got %R", - coro, NULL); + coro); return -1; } @@ -2384,7 +2385,11 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, if (self->task_name == NULL) { return -1; } - +#ifdef Py_GIL_DISABLED + // This is required so that _Py_TryIncref(self) + // works correctly in non-owning threads. + _PyObject_SetMaybeWeakref((PyObject *)self); +#endif if (eager_start) { PyObject *res = PyObject_CallMethodNoArgs(loop, &_Py_ID(is_running)); if (res == NULL) { @@ -2403,11 +2408,6 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop, if (task_call_step_soon(state, self, NULL)) { return -1; } -#ifdef Py_GIL_DISABLED - // This is required so that _Py_TryIncref(self) - // works correctly in non-owning threads. - _PyObject_SetMaybeWeakref((PyObject *)self); -#endif register_task(self); return 0; } @@ -2445,8 +2445,7 @@ TaskObj_traverse(PyObject *op, visitproc visit, void *arg) Py_VISIT(fut->fut_cancel_msg); Py_VISIT(fut->fut_cancelled_exc); Py_VISIT(fut->fut_awaited_by); - PyObject_VisitManagedDict((PyObject *)fut, visit, arg); - return 0; + return PyObject_VisitManagedDict((PyObject *)fut, visit, arg); } /*[clinic input] @@ -2948,7 +2947,8 @@ static PyMethodDef TaskType_methods[] = { _ASYNCIO_TASK_SET_NAME_METHODDEF _ASYNCIO_TASK_GET_CORO_METHODDEF _ASYNCIO_TASK_GET_CONTEXT_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("Tasks are generic over the return type of their wrapped coroutines")}, {NULL, NULL} /* Sentinel */ }; @@ -2990,16 +2990,12 @@ static PyType_Spec Task_spec = { static void TaskObj_dealloc(PyObject *self) { - _PyObject_ResurrectStart(self); - // Unregister the task here so that even if any subclass of Task - // which doesn't end up calling TaskObj_finalize not crashes. - unregister_task((TaskObj *)self); - - PyObject_CallFinalizer(self); - - if (_PyObject_ResurrectEnd(self)) { - return; + if (PyObject_CallFinalizerFromDealloc(self) < 0) { + return; // resurrected } + // unregister the task after finalization so that + // if the task gets resurrected, it remains registered + unregister_task((TaskObj *)self); PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); @@ -4079,30 +4075,44 @@ _asyncio_all_tasks_impl(PyObject *module, PyObject *loop) return NULL; } - PyInterpreterState *interp = PyInterpreterState_Get(); - // Stop the world and traverse the per-thread linked list - // of asyncio tasks for every thread, as well as the - // interpreter's linked list, and add them to `tasks`. - // The interpreter linked list is used for any lingering tasks - // whose thread state has been deallocated while the task was - // still alive. This can happen if a task is referenced by - // a different thread, in which case the task is moved to - // the interpreter's linked list from the thread's linked - // list before deallocation. See PyThreadState_Clear. - // - // The stop-the-world pause is required so that no thread - // modifies its linked list while being iterated here - // in parallel. This design allows for lock-free - // register_task/unregister_task for loops running in parallel - // in different threads (the general case). - _PyEval_StopTheWorld(interp); - int ret = add_tasks_interp(interp, (PyListObject *)tasks); - _PyEval_StartTheWorld(interp); - if (ret < 0) { - // call any escaping calls after starting the world to avoid any deadlocks. - Py_DECREF(tasks); - Py_DECREF(loop); - return NULL; + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)_PyThreadState_GET(); + if (ts->asyncio_running_loop == loop) { + // Fast path for the current running loop of current thread + // no locking or stop the world pause is required + struct llist_node *head = &ts->asyncio_tasks_head; + if (add_tasks_llist(head, (PyListObject *)tasks) < 0) { + Py_DECREF(tasks); + Py_DECREF(loop); + return NULL; + } + } + else { + // Slow path for loop running in different thread + PyInterpreterState *interp = ts->base.interp; + // Stop the world and traverse the per-thread linked list + // of asyncio tasks for every thread, as well as the + // interpreter's linked list, and add them to `tasks`. + // The interpreter linked list is used for any lingering tasks + // whose thread state has been deallocated while the task was + // still alive. This can happen if a task is referenced by + // a different thread, in which case the task is moved to + // the interpreter's linked list from the thread's linked + // list before deallocation. See PyThreadState_Clear. + // + // The stop-the-world pause is required so that no thread + // modifies its linked list while being iterated here + // in parallel. This design allows for lock-free + // register_task/unregister_task for loops running in parallel + // in different threads (the general case). + _PyEval_StopTheWorld(interp); + int ret = add_tasks_interp(interp, (PyListObject *)tasks); + _PyEval_StartTheWorld(interp); + if (ret < 0) { + // call any escaping calls after starting the world to avoid any deadlocks. + Py_DECREF(tasks); + Py_DECREF(loop); + return NULL; + } } // All the tasks are now in the list, now filter the tasks which are done diff --git a/Modules/_bisectmodule.c b/Modules/_bisectmodule.c index 9b146265445d9ab..16fc5b6a9197b76 100644 --- a/Modules/_bisectmodule.c +++ b/Modules/_bisectmodule.c @@ -161,8 +161,8 @@ _bisect.bisect_right -> Py_ssize_t Return the index where to insert item x in list a, assuming a is sorted. The return value i is such that all e in a[:i] have e <= x, and all e in -a[i:] have e > x. So if x already appears in the list, a.insert(i, x) will -insert just after the rightmost x already there. +a[i:] have e > x. So if x already appears in the list, a.insert(i, x) +will insert just after the rightmost x already there. Optional args lo (default 0) and hi (default len(a)) bound the slice of a to be searched. @@ -173,7 +173,7 @@ A custom key function can be supplied to customize the sort order. static Py_ssize_t _bisect_bisect_right_impl(PyObject *module, PyObject *a, PyObject *x, Py_ssize_t lo, Py_ssize_t hi, PyObject *key) -/*[clinic end generated code: output=3a4bc09cc7c8a73d input=43071869772dd53a]*/ +/*[clinic end generated code: output=3a4bc09cc7c8a73d input=b8951a7bb11516e1]*/ { return internal_bisect_right(a, x, lo, hi, key); } @@ -346,8 +346,8 @@ _bisect.bisect_left -> Py_ssize_t Return the index where to insert item x in list a, assuming a is sorted. The return value i is such that all e in a[:i] have e < x, and all e in -a[i:] have e >= x. So if x already appears in the list, a.insert(i, x) will -insert just before the leftmost x already there. +a[i:] have e >= x. So if x already appears in the list, a.insert(i, x) +will insert just before the leftmost x already there. Optional args lo (default 0) and hi (default len(a)) bound the slice of a to be searched. @@ -358,7 +358,7 @@ A custom key function can be supplied to customize the sort order. static Py_ssize_t _bisect_bisect_left_impl(PyObject *module, PyObject *a, PyObject *x, Py_ssize_t lo, Py_ssize_t hi, PyObject *key) -/*[clinic end generated code: output=70749d6e5cae9284 input=f29c4fe7f9b797c7]*/ +/*[clinic end generated code: output=70749d6e5cae9284 input=d24dc2b6439000f7]*/ { return internal_bisect_left(a, x, lo, hi, key); } diff --git a/Modules/_bz2module.c b/Modules/_bz2module.c index 9e85e0de42cd8d7..e6b8198b909138d 100644 --- a/Modules/_bz2module.c +++ b/Modules/_bz2module.c @@ -116,6 +116,7 @@ typedef struct { typedef struct { PyObject_HEAD bz_stream bzs; + int bzerror; char eof; /* Py_T_BOOL expects a char */ PyObject *unused_data; char needs_input; @@ -459,8 +460,11 @@ decompress_buf(BZ2Decompressor *d, Py_ssize_t max_length) d->bzs_avail_in_real += bzs->avail_in; - if (catch_bz2_error(bzret)) + if (catch_bz2_error(bzret)) { + d->bzerror = bzret; + _Py_atomic_store_char_relaxed(&d->needs_input, 0); goto error; + } if (bzret == BZ_STREAM_END) { d->eof = 1; break; @@ -593,6 +597,7 @@ decompress(BZ2Decompressor *d, char *data, size_t len, Py_ssize_t max_length) return result; error: + bzs->next_in = NULL; Py_XDECREF(result); return NULL; } @@ -605,32 +610,40 @@ _bz2.BZ2Decompressor.decompress Decompress *data*, returning uncompressed data as bytes. -If *max_length* is nonnegative, returns at most *max_length* bytes of -decompressed data. If this limit is reached and further output can be -produced, *self.needs_input* will be set to ``False``. In this case, the next -call to *decompress()* may provide *data* as b'' to obtain more of the output. +If *max_length* is nonnegative, returns at most *max_length* bytes +of decompressed data. If this limit is reached and further output +can be produced, *self.needs_input* will be set to ``False``. In +this case, the next call to *decompress()* may provide *data* as b'' +to obtain more of the output. -If all of the input data was decompressed and returned (either because this -was less than *max_length* bytes, or because *max_length* was negative), -*self.needs_input* will be set to True. +If all of the input data was decompressed and returned (either +because this was less than *max_length* bytes, or because +*max_length* was negative), *self.needs_input* will be set to True. -Attempting to decompress data after the end of stream is reached raises an -EOFError. Any data found after the end of the stream is ignored and saved in -the unused_data attribute. +Attempting to decompress data after the end of stream is reached +raises an EOFError. Any data found after the end of the stream is +ignored and saved in the unused_data attribute. [clinic start generated code]*/ static PyObject * _bz2_BZ2Decompressor_decompress_impl(BZ2Decompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=23e41045deb240a3 input=52e1ffc66a8ea624]*/ +/*[clinic end generated code: output=23e41045deb240a3 input=7f68faa9ff7a1b51]*/ { PyObject *result = NULL; ACQUIRE_LOCK(self); - if (self->eof) + if (self->eof) { PyErr_SetString(PyExc_EOFError, "End of stream already reached"); - else + } + else if (self->bzerror) { + // Re-entering BZ2_bzDecompress() after an error can write out of bounds. + PyErr_SetString(PyExc_ValueError, + "Decompressor is unusable after a previous error"); + } + else { result = decompress(self, data->buf, data->len, max_length); + } RELEASE_LOCK(self); return result; } @@ -664,6 +677,7 @@ _bz2_BZ2Decompressor_impl(PyTypeObject *type) return NULL; } + self->bzerror = 0; self->needs_input = 1; self->bzs_avail_in_real = 0; self->input_buffer = NULL; diff --git a/Modules/_codecsmodule.c b/Modules/_codecsmodule.c index 7cf3f152eeecc6d..2cae8942b356937 100644 --- a/Modules/_codecsmodule.c +++ b/Modules/_codecsmodule.c @@ -55,14 +55,15 @@ _codecs.register Register a codec search function. -Search functions are expected to take one argument, the encoding name in -all lower case letters, and either return None, or a tuple of functions -(encoder, decoder, stream_reader, stream_writer) (or a CodecInfo object). +Search functions are expected to take one argument, the encoding +name in all lower case letters, and either return None, or a tuple +of functions (encoder, decoder, stream_reader, stream_writer) (or +a CodecInfo object). [clinic start generated code]*/ static PyObject * _codecs_register(PyObject *module, PyObject *search_function) -/*[clinic end generated code: output=d1bf21e99db7d6d3 input=369578467955cae4]*/ +/*[clinic end generated code: output=d1bf21e99db7d6d3 input=2321d8c8c0420dfc]*/ { if (PyCodec_Register(search_function)) return NULL; @@ -115,16 +116,16 @@ _codecs.encode Encodes obj using the codec registered for encoding. The default encoding is 'utf-8'. errors may be given to set a -different error handling scheme. Default is 'strict' meaning that encoding -errors raise a ValueError. Other possible values are 'ignore', 'replace' -and 'backslashreplace' as well as any other name registered with -codecs.register_error that can handle ValueErrors. +different error handling scheme. Default is 'strict' meaning that +encoding errors raise a ValueError. Other possible values are 'ignore', +'replace' and 'backslashreplace' as well as any other name registered +with codecs.register_error that can handle ValueErrors. [clinic start generated code]*/ static PyObject * _codecs_encode_impl(PyObject *module, PyObject *obj, const char *encoding, const char *errors) -/*[clinic end generated code: output=385148eb9a067c86 input=cd5b685040ff61f0]*/ +/*[clinic end generated code: output=385148eb9a067c86 input=e5271d443e391d7f]*/ { if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); @@ -142,16 +143,16 @@ _codecs.decode Decodes obj using the codec registered for encoding. Default encoding is 'utf-8'. errors may be given to set a -different error handling scheme. Default is 'strict' meaning that encoding -errors raise a ValueError. Other possible values are 'ignore', 'replace' -and 'backslashreplace' as well as any other name registered with -codecs.register_error that can handle ValueErrors. +different error handling scheme. Default is 'strict' meaning that +encoding errors raise a ValueError. Other possible values are 'ignore', +'replace' and 'backslashreplace' as well as any other name registered +with codecs.register_error that can handle ValueErrors. [clinic start generated code]*/ static PyObject * _codecs_decode_impl(PyObject *module, PyObject *obj, const char *encoding, const char *errors) -/*[clinic end generated code: output=679882417dc3a0bd input=7702c0cc2fa1add6]*/ +/*[clinic end generated code: output=679882417dc3a0bd input=3e6254628f9ca538]*/ { if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); @@ -966,14 +967,15 @@ _codecs.register_error Register the specified error handler under the name errors. handler must be a callable object, that will be called with an exception -instance containing information about the location of the encoding/decoding -error and must return a (replacement, new position) tuple. +instance containing information about the location of the +encoding/decoding error and must return a (replacement, new position) +tuple. [clinic start generated code]*/ static PyObject * _codecs_register_error_impl(PyObject *module, const char *errors, PyObject *handler) -/*[clinic end generated code: output=fa2f7d1879b3067d input=5e6709203c2e33fe]*/ +/*[clinic end generated code: output=fa2f7d1879b3067d input=5bea01dfe835d9d8]*/ { if (PyCodec_RegisterError(errors, handler)) return NULL; @@ -1011,13 +1013,13 @@ _codecs.lookup_error lookup_error(errors) -> handler -Return the error handler for the specified error handling name or raise a -LookupError, if no handler exists under this name. +Return the error handler for the specified error handling name or raise +a LookupError, if no handler exists under this name. [clinic start generated code]*/ static PyObject * _codecs_lookup_error_impl(PyObject *module, const char *name) -/*[clinic end generated code: output=087f05dc0c9a98cc input=4775dd65e6235aba]*/ +/*[clinic end generated code: output=087f05dc0c9a98cc input=86cfb6a7a9c67113]*/ { return PyCodec_LookupError(name); } diff --git a/Modules/_collectionsmodule.c b/Modules/_collectionsmodule.c index ad670293ec5b6ab..e48fe39dda6a099 100644 --- a/Modules/_collectionsmodule.c +++ b/Modules/_collectionsmodule.c @@ -5,6 +5,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_pyatomic_ft_wrappers.h" #include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> @@ -507,7 +508,6 @@ deque_extend_impl(dequeobject *deque, PyObject *iterable) iternext = *Py_TYPE(it)->tp_iternext; while ((item = iternext(it)) != NULL) { if (deque_append_lock_held(deque, item, maxlen) == -1) { - Py_DECREF(item); Py_DECREF(it); return NULL; } @@ -604,29 +604,42 @@ deque_copy_impl(dequeobject *deque) collections_state *state = find_module_state_by_def(Py_TYPE(deque)); if (Py_IS_TYPE(deque, state->deque_type)) { dequeobject *new_deque; - PyObject *rv; + Py_ssize_t n = Py_SIZE(deque); new_deque = (dequeobject *)deque_new(state->deque_type, NULL, NULL); if (new_deque == NULL) return NULL; new_deque->maxlen = old_deque->maxlen; - /* Fast path for the deque_repeat() common case where len(deque) == 1 - * - * It's safe to not acquire the per-object lock for new_deque; it's - * invisible to other threads. + + /* Copy elements directly by walking the block structure. + * This is safe because the caller holds the deque lock and + * the new deque is not yet visible to other threads. */ - if (Py_SIZE(deque) == 1) { - PyObject *item = old_deque->leftblock->data[old_deque->leftindex]; - rv = deque_append_impl(new_deque, item); - } else { - rv = deque_extend_impl(new_deque, (PyObject *)deque); - } - if (rv != NULL) { - Py_DECREF(rv); - return (PyObject *)new_deque; + if (n > 0) { + block *b = old_deque->leftblock; + Py_ssize_t index = old_deque->leftindex; + + /* Space saving heuristic. Start filling from the left */ + assert(new_deque->leftblock == new_deque->rightblock); + assert(new_deque->leftindex == new_deque->rightindex + 1); + new_deque->leftindex = 1; + new_deque->rightindex = 0; + + for (Py_ssize_t i = 0; i < n; i++) { + PyObject *item = b->data[index]; + if (deque_append_lock_held(new_deque, Py_NewRef(item), + new_deque->maxlen) < 0) { + Py_DECREF(new_deque); + return NULL; + } + index++; + if (index == BLOCKLEN) { + b = b->rightlink; + index = 0; + } + } } - Py_DECREF(new_deque); - return NULL; + return (PyObject *)new_deque; } if (old_deque->maxlen < 0) result = PyObject_CallOneArg((PyObject *)(Py_TYPE(deque)), @@ -1233,7 +1246,7 @@ _collections.deque.index as deque_index deque: dequeobject value as v: object start: object(converter='_PyEval_SliceIndexNotNone', type='Py_ssize_t', c_default='0') = NULL - stop: object(converter='_PyEval_SliceIndexNotNone', type='Py_ssize_t', c_default='Py_SIZE(deque)') = NULL + stop: object(converter='_PyEval_SliceIndexNotNone', type='Py_ssize_t', c_default='PY_SSIZE_T_MAX') = NULL / Return first index of value. @@ -1244,7 +1257,7 @@ Raises ValueError if the value is not present. static PyObject * deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start, Py_ssize_t stop) -/*[clinic end generated code: output=df45132753175ef9 input=90f48833a91e1743]*/ +/*[clinic end generated code: output=df45132753175ef9 input=1c3b19632cf3484f]*/ { Py_ssize_t i, n; PyObject *item; @@ -1252,22 +1265,23 @@ deque_index_impl(dequeobject *deque, PyObject *v, Py_ssize_t start, Py_ssize_t index = deque->leftindex; size_t start_state = deque->state; int cmp; + Py_ssize_t size = Py_SIZE(deque); if (start < 0) { - start += Py_SIZE(deque); + start += size; if (start < 0) start = 0; } if (stop < 0) { - stop += Py_SIZE(deque); + stop += size; if (stop < 0) stop = 0; } - if (stop > Py_SIZE(deque)) - stop = Py_SIZE(deque); + if (stop > size) + stop = size; if (start > stop) start = stop; - assert(0 <= start && start <= stop && stop <= Py_SIZE(deque)); + assert(0 <= start && start <= stop && stop <= size); for (i=0 ; i < start - BLOCKLEN ; i += BLOCKLEN) { b = b->rightlink; @@ -1532,9 +1546,7 @@ deque_dealloc(PyObject *self) Py_ssize_t i; PyObject_GC_UnTrack(deque); - if (deque->weakreflist != NULL) { - PyObject_ClearWeakRefs(self); - } + FT_CLEAR_WEAKREFS(self, deque->weakreflist); if (deque->leftblock != NULL) { (void)deque_clear(self); assert(deque->leftblock != NULL); @@ -1839,7 +1851,7 @@ static PyMethodDef deque_methods[] = { DEQUE_ROTATE_METHODDEF DEQUE___SIZEOF___METHODDEF {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("deques are generic over the type of their contents")}, {NULL, NULL} /* sentinel */ }; @@ -2232,11 +2244,11 @@ defdict_missing(PyObject *op, PyObject *key) value = _PyObject_CallNoArgs(factory); if (value == NULL) return value; - if (PyObject_SetItem(op, key, value) < 0) { - Py_DECREF(value); - return NULL; - } - return value; + PyObject *result = NULL; + (void)PyDict_SetDefaultRef(op, key, value, &result); + // 'result' is NULL, or a strong reference to 'value' or 'op[key]' + Py_DECREF(value); + return result; } static inline PyObject* @@ -2315,6 +2327,12 @@ defdict_reduce(PyObject *op, PyObject *Py_UNUSED(dummy)) return result; } + +PyDoc_STRVAR(defdict_class_getitem_doc, +"defaultdicts are generic over two types, signifying (respectively) the types \ +of the dictionary's keys and values"); + + static PyMethodDef defdict_methods[] = { {"__missing__", defdict_missing, METH_O, defdict_missing_doc}, @@ -2325,7 +2343,7 @@ static PyMethodDef defdict_methods[] = { {"__reduce__", defdict_reduce, METH_NOARGS, reduce_doc}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, - PyDoc_STR("See PEP 585")}, + defdict_class_getitem_doc}, {NULL} }; @@ -2370,9 +2388,10 @@ defdict_repr(PyObject *op) } defrepr = PyUnicode_FromString("..."); } - else + else { defrepr = PyObject_Repr(dd->default_factory); - Py_ReprLeave(dd->default_factory); + Py_ReprLeave(dd->default_factory); + } } if (defrepr == NULL) { Py_DECREF(baserepr); @@ -2578,7 +2597,12 @@ _collections__count_elements_impl(PyObject *module, PyObject *mapping, if (_PyDict_SetItem_KnownHash(mapping, key, one, hash) < 0) goto done; } else { + /* oldval is a borrowed reference. Keep it alive across + PyNumber_Add(), which can execute arbitrary user code and + mutate (or even clear) the underlying dict. */ + Py_INCREF(oldval); newval = PyNumber_Add(oldval, one); + Py_DECREF(oldval); if (newval == NULL) goto done; if (_PyDict_SetItem_KnownHash(mapping, key, newval, hash) < 0) diff --git a/Modules/_csv.c b/Modules/_csv.c index e5ae853590bf2cb..fd681f81a3b1972 100644 --- a/Modules/_csv.c +++ b/Modules/_csv.c @@ -237,7 +237,7 @@ _set_int(const char *name, int *target, PyObject *src, int dflt) int value; if (!PyLong_CheckExact(src)) { PyErr_Format(PyExc_TypeError, - "\"%s\" must be an integer", name); + "\"%s\" must be an integer, not %T", name, src); return -1; } value = PyLong_AsInt(src); @@ -255,27 +255,29 @@ _set_char_or_none(const char *name, Py_UCS4 *target, PyObject *src, Py_UCS4 dflt if (src == NULL) { *target = dflt; } - else { + else if (src == Py_None) { *target = NOT_SET; - if (src != Py_None) { - if (!PyUnicode_Check(src)) { - PyErr_Format(PyExc_TypeError, - "\"%s\" must be string or None, not %.200s", name, - Py_TYPE(src)->tp_name); - return -1; - } - Py_ssize_t len = PyUnicode_GetLength(src); - if (len < 0) { - return -1; - } - if (len != 1) { - PyErr_Format(PyExc_TypeError, - "\"%s\" must be a 1-character string", - name); - return -1; - } - *target = PyUnicode_READ_CHAR(src, 0); + } + else { + // similar to PyArg_Parse("C?") + if (!PyUnicode_Check(src)) { + PyErr_Format(PyExc_TypeError, + "\"%s\" must be a unicode character or None, not %T", + name, src); + return -1; + } + Py_ssize_t len = PyUnicode_GetLength(src); + if (len < 0) { + return -1; + } + if (len != 1) { + PyErr_Format(PyExc_TypeError, + "\"%s\" must be a unicode character or None, " + "not a string of length %zd", + name, len); + return -1; } + *target = PyUnicode_READ_CHAR(src, 0); } return 0; } @@ -287,11 +289,12 @@ _set_char(const char *name, Py_UCS4 *target, PyObject *src, Py_UCS4 dflt) *target = dflt; } else { + // similar to PyArg_Parse("C") if (!PyUnicode_Check(src)) { PyErr_Format(PyExc_TypeError, - "\"%s\" must be string, not %.200s", name, - Py_TYPE(src)->tp_name); - return -1; + "\"%s\" must be a unicode character, not %T", + name, src); + return -1; } Py_ssize_t len = PyUnicode_GetLength(src); if (len < 0) { @@ -299,8 +302,9 @@ _set_char(const char *name, Py_UCS4 *target, PyObject *src, Py_UCS4 dflt) } if (len != 1) { PyErr_Format(PyExc_TypeError, - "\"%s\" must be a 1-character string", - name); + "\"%s\" must be a unicode character, " + "not a string of length %zd", + name, len); return -1; } *target = PyUnicode_READ_CHAR(src, 0); @@ -311,19 +315,19 @@ _set_char(const char *name, Py_UCS4 *target, PyObject *src, Py_UCS4 dflt) static int _set_str(const char *name, PyObject **target, PyObject *src, const char *dflt) { - if (src == NULL) + if (src == NULL) { *target = PyUnicode_DecodeASCII(dflt, strlen(dflt), NULL); + if (*target == NULL) { + return -1; + } + } else { - if (src == Py_None) - *target = NULL; - else if (!PyUnicode_Check(src)) { + if (!PyUnicode_Check(src)) { PyErr_Format(PyExc_TypeError, - "\"%s\" must be a string", name); + "\"%s\" must be a string, not %T", name, src); return -1; } - else { - Py_XSETREF(*target, Py_NewRef(src)); - } + Py_XSETREF(*target, Py_NewRef(src)); } return 0; } @@ -533,11 +537,6 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) /* validate options */ if (dialect_check_quoting(self->quoting)) goto err; - if (self->delimiter == NOT_SET) { - PyErr_SetString(PyExc_TypeError, - "\"delimiter\" must be a 1-character string"); - goto err; - } if (quotechar == Py_None && quoting == NULL) self->quoting = QUOTE_NONE; if (self->quoting != QUOTE_NONE && self->quotechar == NOT_SET) { @@ -545,10 +544,6 @@ dialect_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) "quotechar must be set if quoting enabled"); goto err; } - if (self->lineterminator == NULL) { - PyErr_SetString(PyExc_TypeError, "lineterminator must be set"); - goto err; - } if (dialect_check_char("delimiter", self->delimiter, self, true) || dialect_check_char("escapechar", self->escapechar, self, !self->skipinitialspace) || @@ -927,7 +922,7 @@ parse_reset(ReaderObj *self) } static PyObject * -Reader_iternext(PyObject *op) +Reader_iternext_lock_held(PyObject *op) { ReaderObj *self = _ReaderObj_CAST(op); @@ -970,6 +965,12 @@ Reader_iternext(PyObject *op) Py_DECREF(lineobj); return NULL; } + if (self->fields == NULL) { + PyErr_SetString(module_state->error_obj, + "iterator has already advanced the reader"); + Py_DECREF(lineobj); + return NULL; + } ++self->line_num; kind = PyUnicode_KIND(lineobj); data = PyUnicode_DATA(lineobj); @@ -994,6 +995,16 @@ Reader_iternext(PyObject *op) return fields; } +static PyObject * +Reader_iternext(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = Reader_iternext_lock_held(op); + Py_END_CRITICAL_SECTION(); + return result; +} + static void Reader_dealloc(PyObject *op) { @@ -1312,14 +1323,8 @@ join_append_lineterminator(WriterObj *self) return 1; } -PyDoc_STRVAR(csv_writerow_doc, -"writerow(iterable)\n" -"\n" -"Construct and write a CSV record from an iterable of fields. Non-string\n" -"elements will be converted to string."); - static PyObject * -csv_writerow(PyObject *op, PyObject *seq) +csv_writerow_lock_held(PyObject *op, PyObject *seq) { WriterObj *self = _WriterObj_CAST(op); DialectObj *dialect = self->dialect; @@ -1422,11 +1427,29 @@ csv_writerow(PyObject *op, PyObject *seq) return result; } +PyDoc_STRVAR(csv_writerow_doc, +"writerow($self, row, /)\n" +"--\n\n" +"Construct and write a CSV record from an iterable of fields.\n" +"\n" +"Non-string elements will be converted to string."); + +static PyObject * +csv_writerow(PyObject *op, PyObject *seq) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = csv_writerow_lock_held(op, seq); + Py_END_CRITICAL_SECTION(); + return result; +} + PyDoc_STRVAR(csv_writerows_doc, -"writerows(iterable of iterables)\n" +"writerows($self, rows, /)\n" +"--\n\n" +"Construct and write a series of iterables to a csv file.\n" "\n" -"Construct and write a series of iterables to a csv file. Non-string\n" -"elements will be converted to string."); +"Non-string elements will be converted to string."); static PyObject * csv_writerows(PyObject *self, PyObject *seqseq) @@ -1583,13 +1606,11 @@ csv_writer(PyObject *module, PyObject *args, PyObject *keyword_args) _csv.list_dialects Return a list of all known dialect names. - - names = csv.list_dialects() [clinic start generated code]*/ static PyObject * _csv_list_dialects_impl(PyObject *module) -/*[clinic end generated code: output=a5b92b215b006a6d input=8953943eb17d98ab]*/ +/*[clinic end generated code: output=a5b92b215b006a6d input=ec58040aafd6a20a]*/ { return PyDict_Keys(get_csv_state(module)->dialects); } @@ -1626,13 +1647,11 @@ _csv.unregister_dialect name: object Delete the name/dialect mapping associated with a string name. - - csv.unregister_dialect(name) [clinic start generated code]*/ static PyObject * _csv_unregister_dialect_impl(PyObject *module, PyObject *name) -/*[clinic end generated code: output=0813ebca6c058df4 input=6b5c1557bf60c7e7]*/ +/*[clinic end generated code: output=0813ebca6c058df4 input=e1cf81bfe3ba0f62]*/ { _csvstate *module_state = get_csv_state(module); int rc = PyDict_Pop(module_state->dialects, name, NULL); @@ -1652,13 +1671,11 @@ _csv.get_dialect name: object Return the dialect instance associated with name. - - dialect = csv.get_dialect(name) [clinic start generated code]*/ static PyObject * _csv_get_dialect_impl(PyObject *module, PyObject *name) -/*[clinic end generated code: output=aa988cd573bebebb input=edf9ddab32e448fb]*/ +/*[clinic end generated code: output=aa988cd573bebebb input=74865c659dcb441f]*/ { return get_dialect_from_registry(name, get_csv_state(module)); } @@ -1670,15 +1687,13 @@ _csv.field_size_limit Sets an upper limit on parsed fields. - csv.field_size_limit([limit]) - Returns old limit. If limit is not given, no new limit is set and the old limit is returned [clinic start generated code]*/ static PyObject * _csv_field_size_limit_impl(PyObject *module, PyObject *new_limit) -/*[clinic end generated code: output=f2799ecd908e250b input=cec70e9226406435]*/ +/*[clinic end generated code: output=f2799ecd908e250b input=77db7485ee3ae90a]*/ { _csvstate *module_state = get_csv_state(module); Py_ssize_t old_limit = FT_ATOMIC_LOAD_SSIZE_RELAXED(module_state->field_limit); @@ -1714,14 +1729,13 @@ PyType_Spec error_spec = { PyDoc_STRVAR(csv_module_doc, "CSV parsing and writing.\n"); PyDoc_STRVAR(csv_reader_doc, -" csv_reader = reader(iterable [, dialect='excel']\n" -" [optional keyword args])\n" -" for row in csv_reader:\n" -" process(row)\n" +"reader($module, iterable, /, dialect='excel', **fmtparams)\n" +"--\n\n" +"Return a reader object that will process lines from the given iterable.\n" "\n" "The \"iterable\" argument can be any object that returns a line\n" "of input for each iteration, such as a file object or a list. The\n" -"optional \"dialect\" parameter is discussed below. The function\n" +"optional \"dialect\" argument defines a CSV dialect. The function\n" "also accepts optional keyword arguments which override settings\n" "provided by the dialect.\n" "\n" @@ -1729,22 +1743,24 @@ PyDoc_STRVAR(csv_reader_doc, "of the CSV file (which can span multiple input lines).\n"); PyDoc_STRVAR(csv_writer_doc, -" csv_writer = csv.writer(fileobj [, dialect='excel']\n" -" [optional keyword args])\n" -" for row in sequence:\n" -" csv_writer.writerow(row)\n" +"writer($module, fileobj, /, dialect='excel', **fmtparams)\n" +"--\n\n" +"Return a writer object that will write user data on the given file object.\n" "\n" -" [or]\n" -"\n" -" csv_writer = csv.writer(fileobj [, dialect='excel']\n" -" [optional keyword args])\n" -" csv_writer.writerows(rows)\n" -"\n" -"The \"fileobj\" argument can be any object that supports the file API.\n"); +"The \"fileobj\" argument can be any object that supports the file API.\n" +"The optional \"dialect\" argument defines a CSV dialect. The function\n" +"also accepts optional keyword arguments which override settings\n" +"provided by the dialect.\n"); PyDoc_STRVAR(csv_register_dialect_doc, -"Create a mapping from a string name to a dialect class.\n" -" dialect = csv.register_dialect(name[, dialect[, **fmtparams]])"); +"register_dialect($module, name, /, dialect='excel', **fmtparams)\n" +"--\n\n" +"Create a mapping from a string name to a CVS dialect.\n" +"\n" +"The optional \"dialect\" argument specifies the base dialect instance\n" +"or the name of the registered dialect. The function also accepts\n" +"optional keyword arguments which override settings provided by the\n" +"dialect.\n"); static struct PyMethodDef csv_methods[] = { { "reader", _PyCFunction_CAST(csv_reader), diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 1bb65e0a64920d0..d152653d121b33c 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -1413,7 +1413,13 @@ PyCPointerType_from_param_impl(PyObject *type, PyTypeObject *cls, /* If we expect POINTER(<type>), but receive a <type> instance, accept it by calling byref(<type>). */ - assert(typeinfo->proto); + if (typeinfo->proto == NULL) { + PyErr_SetString( + PyExc_TypeError, + "cannot convert argument: POINTER _type_ type is not set" + ); + return NULL; + } switch (PyObject_IsInstance(value, typeinfo->proto)) { case 1: Py_INCREF(value); /* _byref steals a refcount */ @@ -2226,6 +2232,36 @@ c_void_p_from_param_impl(PyObject *type, PyTypeObject *cls, PyObject *value) return NULL; } +static int +set_stginfo_ffi_type_pointer(StgInfo *stginfo, struct fielddesc *fmt) +{ +#if defined(_Py_FFI_SUPPORT_C_COMPLEX) + if (!fmt->pffi_type->elements) { + stginfo->ffi_type_pointer = *fmt->pffi_type; + } + else { + /* From primitive types - only complex types have the elements + struct field as non-NULL (two element array). */ + assert(fmt->pffi_type->type == FFI_TYPE_COMPLEX); + const size_t els_size = 2 * sizeof(ffi_type *); + stginfo->ffi_type_pointer.size = fmt->pffi_type->size; + stginfo->ffi_type_pointer.alignment = fmt->pffi_type->alignment; + stginfo->ffi_type_pointer.type = fmt->pffi_type->type; + stginfo->ffi_type_pointer.elements = PyMem_Malloc(els_size); + if (!stginfo->ffi_type_pointer.elements) { + PyErr_NoMemory(); + return -1; + } + memcpy(stginfo->ffi_type_pointer.elements, + fmt->pffi_type->elements, els_size); + } +#else + assert(!fmt->pffi_type->elements); + stginfo->ffi_type_pointer = *fmt->pffi_type; +#endif + return 0; +} + static PyMethodDef c_void_p_methods[] = {C_VOID_P_FROM_PARAM_METHODDEF {0}}; static PyMethodDef c_char_p_methods[] = {C_CHAR_P_FROM_PARAM_METHODDEF {0}}; static PyMethodDef c_wchar_p_methods[] = {C_WCHAR_P_FROM_PARAM_METHODDEF {0}}; @@ -2270,8 +2306,10 @@ static PyObject *CreateSwappedType(ctypes_state *st, PyTypeObject *type, Py_DECREF(result); return NULL; } - - stginfo->ffi_type_pointer = *fmt->pffi_type; + if (set_stginfo_ffi_type_pointer(stginfo, fmt)) { + Py_DECREF(result); + return NULL; + } stginfo->align = fmt->pffi_type->alignment; stginfo->length = 0; stginfo->size = fmt->pffi_type->size; @@ -2366,18 +2404,8 @@ PyCSimpleType_init(PyObject *self, PyObject *args, PyObject *kwds) if (!stginfo) { goto error; } - - if (!fmt->pffi_type->elements) { - stginfo->ffi_type_pointer = *fmt->pffi_type; - } - else { - const size_t els_size = sizeof(fmt->pffi_type->elements); - stginfo->ffi_type_pointer.size = fmt->pffi_type->size; - stginfo->ffi_type_pointer.alignment = fmt->pffi_type->alignment; - stginfo->ffi_type_pointer.type = fmt->pffi_type->type; - stginfo->ffi_type_pointer.elements = PyMem_Malloc(els_size); - memcpy(stginfo->ffi_type_pointer.elements, - fmt->pffi_type->elements, els_size); + if (set_stginfo_ffi_type_pointer(stginfo, fmt)) { + goto error; } stginfo->align = fmt->pffi_type->alignment; stginfo->length = 0; @@ -3591,6 +3619,48 @@ generic_pycdata_new(ctypes_state *st, PyCFuncPtr_Type */ +static inline void +atomic_xsetref(PyObject **field, PyObject *value) +{ +#ifdef Py_GIL_DISABLED + PyObject *old = *field; + _Py_atomic_store_ptr(field, value); + Py_XDECREF(old); +#else + Py_XSETREF(*field, value); +#endif +} +/* + This function atomically loads the reference from *field, and + tries to get a new reference to it. If the incref fails, + it acquires critical section of obj and returns a new reference to the *field. + In the general case, this avoids contention on acquiring the critical section. +*/ +static inline PyObject * +atomic_xgetref(PyObject *obj, PyObject **field) +{ +#ifdef Py_GIL_DISABLED + PyObject *value = _Py_atomic_load_ptr(field); + if (value == NULL) { + return NULL; + } + if (_Py_TryIncrefCompare(field, value)) { + return value; + } + Py_BEGIN_CRITICAL_SECTION(obj); + value = Py_XNewRef(*field); + Py_END_CRITICAL_SECTION(); + return value; +#else + return Py_XNewRef(*field); +#endif +} + +static int +_validate_paramflags(ctypes_state *st, PyTypeObject *type, PyObject *paramflags, PyObject *argtypes); + + + /*[clinic input] @critical_section @setter @@ -3607,7 +3677,7 @@ _ctypes_CFuncPtr_errcheck_set_impl(PyCFuncPtrObject *self, PyObject *value) return -1; } Py_XINCREF(value); - Py_XSETREF(self->errcheck, value); + atomic_xsetref(&self->errcheck, value); return 0; } @@ -3639,12 +3709,10 @@ static int _ctypes_CFuncPtr_restype_set_impl(PyCFuncPtrObject *self, PyObject *value) /*[clinic end generated code: output=0be0a086abbabf18 input=683c3bef4562ccc6]*/ { - PyObject *checker, *oldchecker; + PyObject *checker; if (value == NULL) { - oldchecker = self->checker; - self->checker = NULL; - Py_CLEAR(self->restype); - Py_XDECREF(oldchecker); + atomic_xsetref(&self->restype, NULL); + atomic_xsetref(&self->checker, NULL); return 0; } ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); @@ -3660,11 +3728,9 @@ _ctypes_CFuncPtr_restype_set_impl(PyCFuncPtrObject *self, PyObject *value) if (PyObject_GetOptionalAttr(value, &_Py_ID(_check_retval_), &checker) < 0) { return -1; } - oldchecker = self->checker; - self->checker = checker; Py_INCREF(value); - Py_XSETREF(self->restype, value); - Py_XDECREF(oldchecker); + atomic_xsetref(&self->checker, checker); + atomic_xsetref(&self->restype, value); return 0; } @@ -3706,19 +3772,25 @@ static int _ctypes_CFuncPtr_argtypes_set_impl(PyCFuncPtrObject *self, PyObject *value) /*[clinic end generated code: output=596a36e2ae89d7d1 input=c4627573e980aa8b]*/ { - PyObject *converters; - if (value == NULL || value == Py_None) { - Py_CLEAR(self->converters); - Py_CLEAR(self->argtypes); + atomic_xsetref(&self->argtypes, NULL); + atomic_xsetref(&self->converters, NULL); } else { - ctypes_state *st = get_module_state_by_def(Py_TYPE(Py_TYPE(self))); - converters = converters_from_argtypes(st, value); + PyTypeObject *type = Py_TYPE(self); + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + + PyObject *converters = converters_from_argtypes(st, value); if (!converters) return -1; - Py_XSETREF(self->converters, converters); + + /* Verify paramflags again due to constraints with argtypes */ + if (!_validate_paramflags(st, type, self->paramflags, value)) { + Py_DECREF(converters); + return -1; + } + atomic_xsetref(&self->converters, converters); Py_INCREF(value); - Py_XSETREF(self->argtypes, value); + atomic_xsetref(&self->argtypes, value); } return 0; } @@ -3845,10 +3917,9 @@ _check_outarg_type(ctypes_state *st, PyObject *arg, Py_ssize_t index) /* Returns 1 on success, 0 on error */ static int -_validate_paramflags(ctypes_state *st, PyTypeObject *type, PyObject *paramflags) +_validate_paramflags(ctypes_state *st, PyTypeObject *type, PyObject *paramflags, PyObject *argtypes) { Py_ssize_t i, len; - PyObject *argtypes; StgInfo *info; if (PyStgInfo_FromType(st, (PyObject *)type, &info) < 0) { @@ -3859,10 +3930,13 @@ _validate_paramflags(ctypes_state *st, PyTypeObject *type, PyObject *paramflags) "abstract class"); return 0; } - argtypes = info->argtypes; + if (argtypes == NULL) { + argtypes = info->argtypes; + } - if (paramflags == NULL || info->argtypes == NULL) + if (paramflags == NULL || argtypes == NULL) { return 1; + } if (!PyTuple_Check(paramflags)) { PyErr_SetString(PyExc_TypeError, @@ -3871,7 +3945,7 @@ _validate_paramflags(ctypes_state *st, PyTypeObject *type, PyObject *paramflags) } len = PyTuple_GET_SIZE(paramflags); - if (len != PyTuple_GET_SIZE(info->argtypes)) { + if (len != PyTuple_GET_SIZE(argtypes)) { PyErr_SetString(PyExc_ValueError, "paramflags must have the same length as argtypes"); return 0; @@ -3883,7 +3957,9 @@ _validate_paramflags(ctypes_state *st, PyTypeObject *type, PyObject *paramflags) PyObject *name = Py_None; PyObject *defval; PyObject *typ; - if (!PyArg_ParseTuple(item, "i|U?O", &flag, &name, &defval)) { + if (!PyArg_ParseTuple(item, "i|OO", &flag, &name, &defval) || + !(name == Py_None || PyUnicode_Check(name))) + { PyErr_SetString(PyExc_TypeError, "paramflags must be a sequence of (int [,string [,value]]) tuples"); return 0; @@ -3948,8 +4024,10 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) void *handle; PyObject *paramflags = NULL; - if (!PyArg_ParseTuple(args, "O|O?", &ftuple, &paramflags)) + if (!PyArg_ParseTuple(args, "O|O", &ftuple, &paramflags)) return NULL; + if (paramflags == Py_None) + paramflags = NULL; ftuple = PySequence_Tuple(ftuple); if (!ftuple) @@ -4043,7 +4121,7 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) #endif #undef USE_DLERROR ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); - if (!_validate_paramflags(st, type, paramflags)) { + if (!_validate_paramflags(st, type, paramflags, NULL)) { Py_DECREF(ftuple); return NULL; } @@ -4081,11 +4159,13 @@ PyCFuncPtr_FromVtblIndex(PyTypeObject *type, PyObject *args, PyObject *kwds) GUID *iid = NULL; Py_ssize_t iid_len = 0; - if (!PyArg_ParseTuple(args, "is|O?z#", &index, &name, &paramflags, &iid, &iid_len)) + if (!PyArg_ParseTuple(args, "is|Oz#", &index, &name, &paramflags, &iid, &iid_len)) return NULL; + if (paramflags == Py_None) + paramflags = NULL; ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); - if (!_validate_paramflags(st, type, paramflags)) { + if (!_validate_paramflags(st, type, paramflags, NULL)) { return NULL; } self = (PyCFuncPtrObject *)generic_pycdata_new(st, type, args, kwds); @@ -4499,6 +4579,7 @@ _build_result(PyObject *result, PyObject *callargs, v = PyTuple_GET_ITEM(callargs, i); v = PyObject_CallMethodNoArgs(v, &_Py_ID(__ctypes_from_outparam__)); if (v == NULL || numretvals == 1) { + Py_XDECREF(tup); Py_DECREF(callargs); return v; } @@ -4514,16 +4595,11 @@ _build_result(PyObject *result, PyObject *callargs, } static PyObject * -PyCFuncPtr_call_lock_held(PyObject *op, PyObject *inargs, PyObject *kwds) +PyCFuncPtr_call(PyObject *op, PyObject *inargs, PyObject *kwds) { - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); - PyObject *restype; - PyObject *converters; - PyObject *checker; - PyObject *argtypes; - PyObject *result; - PyObject *callargs; - PyObject *errcheck; + PyObject *result = NULL; + PyObject *callargs = NULL; + PyObject *ret = NULL; #ifdef MS_WIN32 IUnknown *piunk = NULL; #endif @@ -4541,13 +4617,24 @@ PyCFuncPtr_call_lock_held(PyObject *op, PyObject *inargs, PyObject *kwds) } assert(info); /* Cannot be NULL for PyCFuncPtrObject instances */ - restype = self->restype ? self->restype : info->restype; - converters = self->converters ? self->converters : info->converters; - checker = self->checker ? self->checker : info->checker; - argtypes = self->argtypes ? self->argtypes : info->argtypes; -/* later, we probably want to have an errcheck field in stginfo */ - errcheck = self->errcheck /* ? self->errcheck : info->errcheck */; - + PyObject *restype = atomic_xgetref(op, &self->restype); + if (restype == NULL) { + restype = Py_XNewRef(info->restype); + } + PyObject *converters = atomic_xgetref(op, &self->converters); + if (converters == NULL) { + converters = Py_XNewRef(info->converters); + } + PyObject *checker = atomic_xgetref(op, &self->checker); + if (checker == NULL) { + checker = Py_XNewRef(info->checker); + } + PyObject *argtypes = atomic_xgetref(op, &self->argtypes); + if (argtypes == NULL) { + argtypes = Py_XNewRef(info->argtypes); + } + /* later, we probably want to have an errcheck field in stginfo */ + PyObject *errcheck = atomic_xgetref(op, &self->errcheck); pProc = *(void **)self->b_ptr; #ifdef MS_WIN32 @@ -4558,25 +4645,25 @@ PyCFuncPtr_call_lock_held(PyObject *op, PyObject *inargs, PyObject *kwds) if (!this) { PyErr_SetString(PyExc_ValueError, "native com method call without 'this' parameter"); - return NULL; + goto finally; } if (!CDataObject_Check(st, this)) { PyErr_SetString(PyExc_TypeError, "Expected a COM this pointer as first argument"); - return NULL; + goto finally; } /* there should be more checks? No, in Python */ /* First arg is a pointer to an interface instance */ if (!this->b_ptr || *(void **)this->b_ptr == NULL) { PyErr_SetString(PyExc_ValueError, "NULL COM pointer access"); - return NULL; + goto finally; } piunk = *(IUnknown **)this->b_ptr; if (NULL == piunk->lpVtbl) { PyErr_SetString(PyExc_ValueError, "COM method call without VTable"); - return NULL; + goto finally; } pProc = ((void **)piunk->lpVtbl)[self->index - 0x1000]; } @@ -4584,8 +4671,9 @@ PyCFuncPtr_call_lock_held(PyObject *op, PyObject *inargs, PyObject *kwds) callargs = _build_callargs(st, self, argtypes, inargs, kwds, &outmask, &inoutmask, &numretvals); - if (callargs == NULL) - return NULL; + if (callargs == NULL) { + goto finally; + } if (converters) { int required = Py_SAFE_DOWNCAST(PyTuple_GET_SIZE(converters), @@ -4604,7 +4692,7 @@ PyCFuncPtr_call_lock_held(PyObject *op, PyObject *inargs, PyObject *kwds) required, required == 1 ? "" : "s", actual); - return NULL; + goto finally; } } else if (required != actual) { Py_DECREF(callargs); @@ -4613,7 +4701,7 @@ PyCFuncPtr_call_lock_held(PyObject *op, PyObject *inargs, PyObject *kwds) required, required == 1 ? "" : "s", actual); - return NULL; + goto finally; } } @@ -4644,23 +4732,19 @@ PyCFuncPtr_call_lock_held(PyObject *op, PyObject *inargs, PyObject *kwds) if (v == NULL || v != callargs) { Py_DECREF(result); Py_DECREF(callargs); - return v; + ret = v; + goto finally; } Py_DECREF(v); } - - return _build_result(result, callargs, - outmask, inoutmask, numretvals); -} - -static PyObject * -PyCFuncPtr_call(PyObject *op, PyObject *inargs, PyObject *kwds) -{ - PyObject *result; - Py_BEGIN_CRITICAL_SECTION(op); - result = PyCFuncPtr_call_lock_held(op, inargs, kwds); - Py_END_CRITICAL_SECTION(); - return result; + ret = _build_result(result, callargs, outmask, inoutmask, numretvals); +finally: + Py_XDECREF(restype); + Py_XDECREF(converters); + Py_XDECREF(checker); + Py_XDECREF(argtypes); + Py_XDECREF(errcheck); + return ret; } static int @@ -5222,7 +5306,7 @@ Array_length(PyObject *myself) static PyMethodDef Array_methods[] = { {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("Arrays are generic over the type of their elements")}, { NULL, NULL } }; diff --git a/Modules/_ctypes/_ctypes_test.c b/Modules/_ctypes/_ctypes_test.c index d28e5708b449330..991ff0d675c2f1c 100644 --- a/Modules/_ctypes/_ctypes_test.c +++ b/Modules/_ctypes/_ctypes_test.c @@ -23,7 +23,7 @@ # define _Py_thread_local __thread #endif -#if defined(Py_FFI_SUPPORT_C_COMPLEX) +#if defined(_Py_FFI_SUPPORT_C_COMPLEX) # include <complex.h> // csqrt() # undef I // for _ctypes_test_generated.c.h #endif @@ -446,7 +446,7 @@ EXPORT(char *)my_strtok(char *token, const char *delim) return strtok(token, delim); } -EXPORT(char *)my_strchr(const char *s, int c) +EXPORT(const char *) my_strchr(const char *s, int c) { return strchr(s, c); } @@ -457,7 +457,7 @@ EXPORT(double) my_sqrt(double a) return sqrt(a); } -#if defined(Py_FFI_SUPPORT_C_COMPLEX) +#if defined(_Py_FFI_SUPPORT_C_COMPLEX) EXPORT(double complex) my_csqrt(double complex a) { return csqrt(a); @@ -989,13 +989,10 @@ EXPORT(RECT) ReturnRect(int i, RECT ar, RECT* br, POINT cp, RECT dr, { case 0: return ar; - break; case 1: return dr; - break; case 2: return gr; - break; } return ar; diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 856b0376e5eaa00..8f85bde61a4df07 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -176,8 +176,9 @@ _ctypes_get_errobj(ctypes_state *st, int **pspace) } else { void *space = PyMem_Calloc(2, sizeof(int)); - if (space == NULL) - return NULL; + if (space == NULL) { + return PyErr_NoMemory(); + } errobj = PyCapsule_New(space, CTYPES_CAPSULE_NAME_PYMEM, pymem_destructor); if (errobj == NULL) { PyMem_Free(space); diff --git a/Modules/_ctypes/cfield.c b/Modules/_ctypes/cfield.c index 86bcc805360a3fd..0f51eab41c93e16 100644 --- a/Modules/_ctypes/cfield.c +++ b/Modules/_ctypes/cfield.c @@ -160,8 +160,8 @@ PyCField_new_impl(PyTypeObject *type, PyObject *name, PyObject *proto, if ((bitfield_size + bit_offset) > byte_size * 8) { PyErr_Format( PyExc_ValueError, - "bit field %R overflows its type (%zd + %zd >= %zd)", - name, bit_offset, byte_size*8); + "bit field %R overflows its type (%zd + %zd > %zd)", + name, bit_offset, bitfield_size, byte_size * 8); goto error; } } @@ -759,7 +759,7 @@ d_get(void *ptr, Py_ssize_t size) return PyFloat_FromDouble(val); } -#if defined(Py_FFI_SUPPORT_C_COMPLEX) +#if defined(_Py_FFI_SUPPORT_C_COMPLEX) /* We don't use _Complex types here, using arrays instead, as the C11+ standard says: "Each complex type has the same representation and alignment @@ -792,6 +792,44 @@ D_get(void *ptr, Py_ssize_t size) return PyComplex_FromDoubles(x[0], x[1]); } +static PyObject * +D_set_sw(void *ptr, PyObject *value, Py_ssize_t size) +{ + assert(NUM_BITS(size) || (size == 2*sizeof(double))); + Py_complex c = PyComplex_AsCComplex(value); + + if (c.real == -1 && PyErr_Occurred()) { + return NULL; + } +#ifdef WORDS_BIGENDIAN + if (PyFloat_Pack8(c.real, ptr, 1) + || PyFloat_Pack8(c.imag, ptr + sizeof(double), 1)) + { + return NULL; + } +#else + if (PyFloat_Pack8(c.real, ptr, 0) + || PyFloat_Pack8(c.imag, ptr + sizeof(double), 0)) + { + return NULL; + } +#endif + _RET(value); +} + +static PyObject * +D_get_sw(void *ptr, Py_ssize_t size) +{ + assert(NUM_BITS(size) || (size == 2*sizeof(double))); +#ifdef WORDS_BIGENDIAN + return PyComplex_FromDoubles(PyFloat_Unpack8(ptr, 1), + PyFloat_Unpack8(ptr + sizeof(double), 1)); +#else + return PyComplex_FromDoubles(PyFloat_Unpack8(ptr, 0), + PyFloat_Unpack8(ptr + sizeof(double), 0)); +#endif +} + /* F: float complex */ static PyObject * F_set(void *ptr, PyObject *value, Py_ssize_t size) @@ -817,6 +855,44 @@ F_get(void *ptr, Py_ssize_t size) return PyComplex_FromDoubles(x[0], x[1]); } +static PyObject * +F_set_sw(void *ptr, PyObject *value, Py_ssize_t size) +{ + assert(NUM_BITS(size) || (size == 2*sizeof(float))); + Py_complex c = PyComplex_AsCComplex(value); + + if (c.real == -1 && PyErr_Occurred()) { + return NULL; + } +#ifdef WORDS_BIGENDIAN + if (PyFloat_Pack4(c.real, ptr, 1) + || PyFloat_Pack4(c.imag, ptr + sizeof(float), 1)) + { + return NULL; + } +#else + if (PyFloat_Pack4(c.real, ptr, 0) + || PyFloat_Pack4(c.imag, ptr + sizeof(float), 0)) + { + return NULL; + } +#endif + _RET(value); +} + +static PyObject * +F_get_sw(void *ptr, Py_ssize_t size) +{ + assert(NUM_BITS(size) || (size == 2*sizeof(float))); +#ifdef WORDS_BIGENDIAN + return PyComplex_FromDoubles(PyFloat_Unpack4(ptr, 1), + PyFloat_Unpack4(ptr + sizeof(float), 1)); +#else + return PyComplex_FromDoubles(PyFloat_Unpack4(ptr, 0), + PyFloat_Unpack4(ptr + sizeof(float), 0)); +#endif +} + /* G: long double complex */ static PyObject * G_set(void *ptr, PyObject *value, Py_ssize_t size) @@ -1599,10 +1675,12 @@ for base_code, base_c_type in [ /////////////////////////////////////////////////////////////////////////// TABLE_ENTRY_SW(d, &ffi_type_double); -#if defined(Py_FFI_SUPPORT_C_COMPLEX) +#if defined(_Py_FFI_SUPPORT_C_COMPLEX) if (Py_FFI_COMPLEX_AVAILABLE) { TABLE_ENTRY(D, &ffi_type_complex_double); + TABLE_ENTRY_SW(D, &ffi_type_complex_double); TABLE_ENTRY(F, &ffi_type_complex_float); + TABLE_ENTRY_SW(F, &ffi_type_complex_float); TABLE_ENTRY(G, &ffi_type_complex_longdouble); } #endif diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 2d859ed63e1758d..78a33fea0192c2b 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -11,7 +11,7 @@ // Do we support C99 complex types in ffi? // For Apple's libffi, this must be determined at runtime (see gh-128156). -#if defined(Py_FFI_SUPPORT_C_COMPLEX) +#if defined(_Py_FFI_SUPPORT_C_COMPLEX) # if USING_APPLE_OS_LIBFFI && defined(__has_builtin) # if __has_builtin(__builtin_available) # define Py_FFI_COMPLEX_AVAILABLE __builtin_available(macOS 10.15, *) @@ -419,7 +419,7 @@ typedef struct { visible to other threads before the `dict_final` bit is set. */ -#define STGINFO_LOCK(stginfo) Py_BEGIN_CRITICAL_SECTION_MUT(&(stginfo)->mutex) +#define STGINFO_LOCK(stginfo) Py_BEGIN_CRITICAL_SECTION_MUTEX(&(stginfo)->mutex) #define STGINFO_UNLOCK() Py_END_CRITICAL_SECTION() static inline uint8_t @@ -596,7 +596,8 @@ PyStgInfo_FromAny(ctypes_state *state, PyObject *obj, StgInfo **result) return _stginfo_from_type(state, Py_TYPE(obj), result); } -/* A variant of PyStgInfo_FromType that doesn't need the state, +/* A variant of PyStgInfo_FromType that doesn't need the state + * and doesn't modify any refcounts, * so it can be called from finalization functions when the module * state is torn down. */ @@ -604,17 +605,12 @@ static inline StgInfo * _PyStgInfo_FromType_NoState(PyObject *type) { PyTypeObject *PyCType_Type; - if (PyType_GetBaseByToken(Py_TYPE(type), &pyctype_type_spec, &PyCType_Type) < 0) { - return NULL; - } - if (PyCType_Type == NULL) { - PyErr_Format(PyExc_TypeError, "expected a ctypes type, got '%N'", type); + if (_PyType_GetBaseByToken_Borrow(Py_TYPE(type), &pyctype_type_spec, &PyCType_Type) < 0 || + PyCType_Type == NULL) { return NULL; } - StgInfo *info = PyObject_GetTypeData(type, PyCType_Type); - Py_DECREF(PyCType_Type); - return info; + return PyObject_GetTypeData(type, PyCType_Type); } // Initialize StgInfo on a newly created type diff --git a/Modules/_ctypes/stgdict.c b/Modules/_ctypes/stgdict.c index f208d2956e429ec..ab955a0b824a2f3 100644 --- a/Modules/_ctypes/stgdict.c +++ b/Modules/_ctypes/stgdict.c @@ -484,7 +484,7 @@ error:; /* Replace array elements at stginfo->ffi_type_pointer.elements. - Return -1 if error occured. + Return -1 if error occurred. */ int _replace_array_elements(ctypes_state *st, PyObject *layout_fields, diff --git a/Modules/_curses_panel.c b/Modules/_curses_panel.c index eecf7a1c8a1e56b..ce24d567dcd238c 100644 --- a/Modules/_curses_panel.c +++ b/Modules/_curses_panel.c @@ -32,6 +32,8 @@ typedef struct { PyTypeObject *PyCursesPanel_Type; } _curses_panel_state; +typedef struct PyCursesPanelObject PyCursesPanelObject; + static inline _curses_panel_state * get_curses_panel_state(PyObject *module) { @@ -40,6 +42,25 @@ get_curses_panel_state(PyObject *module) return (_curses_panel_state *)state; } +static inline _curses_panel_state * +get_curses_panel_state_by_panel(PyCursesPanelObject *panel) +{ + /* + * Note: 'state' may be NULL if Py_TYPE(panel) is not a heap + * type associated with this module, but the compiler would + * have likely already complained with an "invalid pointer + * type" at compile-time. + * + * To make it more robust, all functions recovering a module's + * state from an object should expect to return NULL with an + * exception set (in contrast to functions recovering a module's + * state from a module itself). + */ + void *state = PyType_GetModuleState(Py_TYPE(panel)); + assert(state != NULL); + return (_curses_panel_state *)state; +} + static int _curses_panel_clear(PyObject *mod) { @@ -95,7 +116,7 @@ PyCursesCheckERR(_curses_panel_state *state, int code, const char *fname) /* Definition of the panel object and panel type */ -typedef struct { +typedef struct PyCursesPanelObject { PyObject_HEAD PANEL *pan; PyCursesWindowObject *wo; /* for reference counts */ @@ -213,12 +234,13 @@ _curses_panel.panel.hide Hide the panel. -This does not delete the object, it just makes the window on screen invisible. +This does not delete the object, it just makes the window on screen +invisible. [clinic start generated code]*/ static PyObject * _curses_panel_panel_hide_impl(PyCursesPanelObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=cc6ab7203cdc1450 input=1bfc741f473e6055]*/ +/*[clinic end generated code: output=cc6ab7203cdc1450 input=805065e45e6fc1cd]*/ { _curses_panel_state *state = PyType_GetModuleState(cls); return PyCursesCheckERR(state, hide_panel(self->pan), "hide"); @@ -230,11 +252,13 @@ _curses_panel.panel.show cls: defining_class Display the panel (which might have been hidden). + +The panel is placed on top of the panel stack. [clinic start generated code]*/ static PyObject * _curses_panel_panel_show_impl(PyCursesPanelObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=dc3421de375f0409 input=8122e80151cb4379]*/ +/*[clinic end generated code: output=dc3421de375f0409 input=59375ba57303320d]*/ { _curses_panel_state *state = PyType_GetModuleState(cls); return PyCursesCheckERR(state, show_panel(self->pan), "show"); @@ -262,8 +286,11 @@ static PyObject * PyCursesPanel_New(_curses_panel_state *state, PANEL *pan, PyCursesWindowObject *wo) { - PyCursesPanelObject *po = PyObject_New(PyCursesPanelObject, - state->PyCursesPanel_Type); + assert(state != NULL); + PyTypeObject *type = state->PyCursesPanel_Type; + assert(type != NULL); + assert(type->tp_alloc != NULL); + PyCursesPanelObject *po = (PyCursesPanelObject *)type->tp_alloc(type, 0); if (po == NULL) { return NULL; } @@ -278,27 +305,57 @@ PyCursesPanel_New(_curses_panel_state *state, PANEL *pan, return (PyObject *)po; } +static int +PyCursesPanel_Clear(PyObject *op) +{ + PyCursesPanelObject *self = _PyCursesPanelObject_CAST(op); + PyObject *extra = (PyObject *)panel_userptr(self->pan); + if (extra != NULL) { + Py_DECREF(extra); + if (set_panel_userptr(self->pan, NULL) == ERR) { + _curses_panel_state *state = get_curses_panel_state_by_panel(self); + PyErr_SetString(state->PyCursesError, + "set_panel_userptr() returned ERR"); + return -1; + } + } + // self->wo should not be cleared because an associated WINDOW may exist + return 0; +} + static void PyCursesPanel_Dealloc(PyObject *self) { - PyObject *tp, *obj; - PyCursesPanelObject *po = _PyCursesPanelObject_CAST(self); + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); - tp = (PyObject *) Py_TYPE(po); - obj = (PyObject *) panel_userptr(po->pan); - if (obj) { - (void)set_panel_userptr(po->pan, NULL); - Py_DECREF(obj); + PyCursesPanelObject *po = _PyCursesPanelObject_CAST(self); + if (PyCursesPanel_Clear(self) < 0) { + PyErr_FormatUnraisable("Exception ignored in PyCursesPanel_Dealloc()"); + } + if (del_panel(po->pan) == ERR && !PyErr_Occurred()) { + _curses_panel_state *state = get_curses_panel_state_by_panel(po); + PyErr_SetString(state->PyCursesError, "del_panel() returned ERR"); + PyErr_FormatUnraisable("Exception ignored in PyCursesPanel_Dealloc()"); } - (void)del_panel(po->pan); if (po->wo != NULL) { Py_DECREF(po->wo); remove_lop(po); } - PyObject_Free(po); + tp->tp_free(po); Py_DECREF(tp); } +static int +PyCursesPanel_Traverse(PyObject *op, visitproc visit, void *arg) +{ + PyCursesPanelObject *self = _PyCursesPanelObject_CAST(op); + Py_VISIT(Py_TYPE(op)); + Py_VISIT(panel_userptr(self->pan)); + Py_VISIT(self->wo); + return 0; +} + /* panel_above(NULL) returns the bottom panel in the stack. To get this behaviour we use curses.panel.bottom_panel(). */ /*[clinic input] @@ -520,7 +577,9 @@ static PyMethodDef PyCursesPanel_Methods[] = { /* -------------------------------------------------------*/ static PyType_Slot PyCursesPanel_Type_slots[] = { + {Py_tp_clear, PyCursesPanel_Clear}, {Py_tp_dealloc, PyCursesPanel_Dealloc}, + {Py_tp_traverse, PyCursesPanel_Traverse}, {Py_tp_methods, PyCursesPanel_Methods}, {0, 0}, }; @@ -528,7 +587,11 @@ static PyType_Slot PyCursesPanel_Type_slots[] = { static PyType_Spec PyCursesPanel_Type_spec = { .name = "_curses_panel.panel", .basicsize = sizeof(PyCursesPanelObject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .flags = ( + Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_DISALLOW_INSTANTIATION + | Py_TPFLAGS_HAVE_GC + ), .slots = PyCursesPanel_Type_slots }; @@ -573,11 +636,13 @@ _curses_panel.new_panel / Return a panel object, associating it with the given window win. + +The new panel is placed on top of the panel stack. [clinic start generated code]*/ static PyObject * _curses_panel_new_panel_impl(PyObject *module, PyCursesWindowObject *win) -/*[clinic end generated code: output=45e948e0176a9bd2 input=74d4754e0ebe4800]*/ +/*[clinic end generated code: output=45e948e0176a9bd2 input=3b6fea647b808fd7]*/ { _curses_panel_state *state = get_curses_panel_state(module); @@ -629,12 +694,13 @@ _curses_panel.update_panels Updates the virtual screen after changes in the panel stack. -This does not call curses.doupdate(), so you'll have to do this yourself. +This does not call curses.doupdate(), so you'll have to do this +yourself. [clinic start generated code]*/ static PyObject * _curses_panel_update_panels_impl(PyObject *module) -/*[clinic end generated code: output=2f3b4c2e03d90ded input=5299624c9a708621]*/ +/*[clinic end generated code: output=2f3b4c2e03d90ded input=0d0db79f05ec3ef4]*/ { PyCursesInitialised; update_panels(); diff --git a/Modules/_cursesmodule.c b/Modules/_cursesmodule.c index cd185bc2b02ea5c..12641b51c1e90d0 100644 --- a/Modules/_cursesmodule.c +++ b/Modules/_cursesmodule.c @@ -209,7 +209,11 @@ static int curses_initscr_called = FALSE; /* Tells whether start_color() has been called to initialise color usage. */ static int curses_start_color_called = FALSE; -static const char *curses_screen_encoding = NULL; +/* Encoding of the initial screen, used by module-level functions that have + no window object to take it from (e.g. unctrl(), ungetch()). This is a + private copy: the window object that initscr() returns may be deallocated + while these functions are still in use. */ +static char *curses_screen_encoding = NULL; /* Utility Checking Procedures */ @@ -1139,12 +1143,12 @@ _curses.window.attron attr: long / -Add attribute attr from the "background" set. +Add attribute attr to the "background" set. [clinic start generated code]*/ static PyObject * _curses_window_attron_impl(PyCursesWindowObject *self, long attr) -/*[clinic end generated code: output=7afea43b237fa870 input=5a88fba7b1524f32]*/ +/*[clinic end generated code: output=7afea43b237fa870 input=b57f824e1bf58326]*/ { return PyCursesCheckERR_ForWin(self, wattron(self->win, (attr_t)attr), "attron"); } @@ -1214,10 +1218,10 @@ _curses.window.border Draw a border around the edges of the window. -Each parameter specifies the character to use for a specific part of the -border. The characters can be specified as integers or as one-character -strings. A 0 value for any parameter will cause the default character to be -used for that parameter. +Each parameter specifies the character to use for a specific part of +the border. The characters can be specified as integers or as +one-character strings. A 0 value for any parameter will cause the +default character to be used for that parameter. [clinic start generated code]*/ static PyObject * @@ -1225,7 +1229,7 @@ _curses_window_border_impl(PyCursesWindowObject *self, PyObject *ls, PyObject *rs, PyObject *ts, PyObject *bs, PyObject *tl, PyObject *tr, PyObject *bl, PyObject *br) -/*[clinic end generated code: output=670ef38d3d7c2aa3 input=e015f735d67a240b]*/ +/*[clinic end generated code: output=670ef38d3d7c2aa3 input=42568c1458221d24]*/ { chtype ch[8]; int i; @@ -1268,14 +1272,15 @@ _curses.window.box Draw a border around the edges of the window. -Similar to border(), but both ls and rs are verch and both ts and bs are -horch. The default corner characters are always used by this function. +Similar to border(), but both ls and rs are verch and both ts and bs +are horch. The default corner characters are always used by this +function. [clinic start generated code]*/ static PyObject * _curses_window_box_impl(PyCursesWindowObject *self, int group_right_1, PyObject *verch, PyObject *horch) -/*[clinic end generated code: output=f3fcb038bb287192 input=f00435f9c8c98f60]*/ +/*[clinic end generated code: output=f3fcb038bb287192 input=e11acb7dbf6790b6]*/ { chtype ch1 = 0, ch2 = 0; if (group_right_1) { @@ -1312,32 +1317,27 @@ int py_mvwdelch(WINDOW *w, int y, int x) /* chgat, added by Fabian Kreutz <fabian.kreutz at gmx.net> */ #ifdef HAVE_CURSES_WCHGAT -/*[-clinic input] -_curses.window.chgat - [ - y: int - Y-coordinate. - x: int - X-coordinate. - ] - - n: int = -1 - Number of characters. - - attr: long - Attributes for characters. - / +PyDoc_STRVAR(_curses_window_chgat__doc__, +"chgat([y, x,] [n=-1,] attr)\n" +"Set the attributes of characters.\n" +"\n" +" y\n" +" Y-coordinate.\n" +" x\n" +" X-coordinate.\n" +" n\n" +" Number of characters.\n" +" attr\n" +" Attributes for characters.\n" +"\n" +"Set the attributes of num characters at the current cursor position, or at\n" +"position (y, x) if supplied. If no value of num is given or num = -1, the\n" +"attribute will be set on all the characters to the end of the line. This\n" +"function does not move the cursor. The changed line will be touched using\n" +"the touchline() method so that the contents will be redisplayed by the next\n" +"window refresh."); -Set the attributes of characters. - -Set the attributes of num characters at the current cursor position, or at -position (y, x) if supplied. If no value of num is given or num = -1, the -attribute will be set on all the characters to the end of the line. This -function does not move the cursor. The changed line will be touched using -the touchline() method so that the contents will be redisplayed by the next -window refresh. -[-clinic start generated code]*/ static PyObject * PyCursesWindow_ChgAt(PyObject *op, PyObject *args) { @@ -1363,19 +1363,20 @@ PyCursesWindow_ChgAt(PyObject *op, PyObject *args) attr = lattr; break; case 3: - if (!PyArg_ParseTuple(args,"iil;int,int,attr", &y, &x, &lattr)) + if (!PyArg_ParseTuple(args,"iil;y,x,attr", &y, &x, &lattr)) return NULL; attr = lattr; use_xy = TRUE; break; case 4: - if (!PyArg_ParseTuple(args,"iiil;int,int,n,attr", &y, &x, &num, &lattr)) + if (!PyArg_ParseTuple(args,"iiil;y,x,n,attr", &y, &x, &num, &lattr)) return NULL; attr = lattr; use_xy = TRUE; break; default: - PyErr_SetString(PyExc_TypeError, "chgat requires 1 to 4 arguments"); + PyErr_SetString(PyExc_TypeError, + "_curses.window.chgat requires 1 to 4 arguments"); return NULL; } @@ -1405,13 +1406,16 @@ _curses.window.delch ] / -Delete any character at (y, x). +Delete the character under the cursor, or at (y, x) if specified. + +All characters to the right on the same line are shifted one +position left. [clinic start generated code]*/ static PyObject * _curses_window_delch_impl(PyCursesWindowObject *self, int group_right_1, int y, int x) -/*[clinic end generated code: output=22e77bb9fa11b461 input=d2f79e630a4fc6d0]*/ +/*[clinic end generated code: output=22e77bb9fa11b461 input=61db3c7f4885e90c]*/ { if (!group_right_1) { return PyCursesCheckERR_ForWin(self, wdelch(self->win), "wdelch"); @@ -1438,15 +1442,15 @@ _curses.window.derwin Create a sub-window (window-relative coordinates). -derwin() is the same as calling subwin(), except that begin_y and begin_x -are relative to the origin of the window, rather than relative to the entire -screen. +derwin() is the same as calling subwin(), except that begin_y and +begin_x are relative to the origin of the window, rather than +relative to the entire screen. [clinic start generated code]*/ static PyObject * _curses_window_derwin_impl(PyCursesWindowObject *self, int group_left_1, int nlines, int ncols, int begin_y, int begin_x) -/*[clinic end generated code: output=7924b112d9f70d6e input=966d9481f7f5022e]*/ +/*[clinic end generated code: output=7924b112d9f70d6e input=6efb50722be444ba]*/ { WINDOW *win; @@ -1498,7 +1502,7 @@ _curses_window_echochar_impl(PyCursesWindowObject *self, PyObject *ch, "echochar"); } -#ifdef NCURSES_MOUSE_VERSION +#if defined(HAVE_CURSES_GETMOUSE) || defined(PDCURSES) /*[clinic input] _curses.window.enclose @@ -1533,7 +1537,7 @@ _curses_window_getbkgd_impl(PyCursesWindowObject *self) } /*[clinic input] -_curses.window.getch -> int +_curses.window.getch [ y: int @@ -1545,15 +1549,16 @@ _curses.window.getch -> int Get a character code from terminal keyboard. -The integer returned does not have to be in ASCII range: function keys, -keypad keys and so on return numbers higher than 256. In no-delay mode, -1 -is returned if there is no input, else getch() waits until a key is pressed. +The integer returned does not have to be in ASCII range: function +keys, keypad keys and so on return numbers higher than 256. In +no-delay mode, -1 is returned if there is no input, else getch() +waits until a key is pressed. [clinic start generated code]*/ -static int +static PyObject * _curses_window_getch_impl(PyCursesWindowObject *self, int group_right_1, int y, int x) -/*[clinic end generated code: output=980aa6af0c0ca387 input=bb24ebfb379f991f]*/ +/*[clinic end generated code: output=e1639e87d545e676 input=0dc5ff40e079787a]*/ { int rtn; @@ -1566,7 +1571,17 @@ _curses_window_getch_impl(PyCursesWindowObject *self, int group_right_1, } Py_END_ALLOW_THREADS - return rtn; + if (rtn == ERR) { + // We suppress ERR returned by wgetch() in nodelay mode + // after we handled possible interruption signals. + if (PyErr_CheckSignals()) { + return NULL; + } + // ERR is an implementation detail, so to be on the safe side, + // we forcibly set the return value to -1 as documented above. + rtn = -1; + } + return PyLong_FromLong(rtn); } /*[clinic input] @@ -1582,15 +1597,16 @@ _curses.window.getkey Get a character (string) from terminal keyboard. -Returning a string instead of an integer, as getch() does. Function keys, -keypad keys and other special keys return a multibyte string containing the -key name. In no-delay mode, an exception is raised if there is no input. +Returning a string instead of an integer, as getch() does. Function +keys, keypad keys and other special keys return a multibyte string +containing the key name. In no-delay mode, an exception is raised +if there is no input. [clinic start generated code]*/ static PyObject * _curses_window_getkey_impl(PyCursesWindowObject *self, int group_right_1, int y, int x) -/*[clinic end generated code: output=8490a182db46b10f input=be2dee34f5cf57f8]*/ +/*[clinic end generated code: output=8490a182db46b10f input=bd24a7da1ed9c73b]*/ { int rtn; @@ -1678,84 +1694,102 @@ _curses_window_get_wch_impl(PyCursesWindowObject *self, int group_right_1, } #endif -/*[-clinic input] -_curses.window.getstr - - [ - y: int - Y-coordinate. - x: int - X-coordinate. - ] - n: int = 1023 - Maximal number of characters. - / +/* + * Helper function for parsing parameters from getstr() and instr(). + * This function is necessary because Argument Clinic does not know + * how to handle nested optional groups with default values inside. + * + * Return 1 on success and 0 on failure, similar to PyArg_ParseTuple(). + */ +static int +curses_clinic_parse_optional_xy_n(PyObject *args, + int *y, int *x, unsigned int *n, int *use_xy, + const char *qualname) +{ + switch (PyTuple_GET_SIZE(args)) { + case 0: { + *use_xy = 0; + return 1; + } + case 1: { + *use_xy = 0; + return PyArg_ParseTuple(args, "O&;n", + _PyLong_UnsignedInt_Converter, n); + } + case 2: { + *use_xy = 1; + return PyArg_ParseTuple(args, "ii;y,x", y, x); + } + case 3: { + *use_xy = 1; + return PyArg_ParseTuple(args, "iiO&;y,x,n", y, x, + _PyLong_UnsignedInt_Converter, n); + } + default: { + *use_xy = 0; + PyErr_Format(PyExc_TypeError, "%s requires 0 to 3 arguments", + qualname); + return 0; + } + } +} -Read a string from the user, with primitive line editing capacity. -[-clinic start generated code]*/ +PyDoc_STRVAR(_curses_window_getstr__doc__, +"getstr([[y, x,] n=2047])\n" +"Read a string from the user, with primitive line editing capacity.\n" +"\n" +" y\n" +" Y-coordinate.\n" +" x\n" +" X-coordinate.\n" +" n\n" +" Maximal number of characters."); static PyObject * -PyCursesWindow_GetStr(PyObject *op, PyObject *args) +PyCursesWindow_getstr(PyObject *op, PyObject *args) { PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); + int rtn, use_xy = 0, y = 0, x = 0; + unsigned int max_buf_size = 2048; + unsigned int n = max_buf_size - 1; + PyObject *res; - int x, y, n; - char rtn[1024]; /* This should be big enough.. I hope */ - int rtn2; + if (!curses_clinic_parse_optional_xy_n(args, &y, &x, &n, &use_xy, + "_curses.window.instr")) + { + return NULL; + } - switch (PyTuple_Size(args)) { - case 0: - Py_BEGIN_ALLOW_THREADS - rtn2 = wgetnstr(self->win,rtn, 1023); - Py_END_ALLOW_THREADS - break; - case 1: - if (!PyArg_ParseTuple(args,"i;n", &n)) - return NULL; - if (n < 0) { - PyErr_SetString(PyExc_ValueError, "'n' must be nonnegative"); - return NULL; - } - Py_BEGIN_ALLOW_THREADS - rtn2 = wgetnstr(self->win, rtn, Py_MIN(n, 1023)); - Py_END_ALLOW_THREADS - break; - case 2: - if (!PyArg_ParseTuple(args,"ii;y,x",&y,&x)) - return NULL; + n = Py_MIN(n, max_buf_size - 1); + res = PyBytes_FromStringAndSize(NULL, n + 1); + if (res == NULL) { + return NULL; + } + char *buf = PyBytes_AS_STRING(res); + + if (use_xy) { Py_BEGIN_ALLOW_THREADS #ifdef STRICT_SYSV_CURSES - rtn2 = wmove(self->win,y,x)==ERR ? ERR : wgetnstr(self->win, rtn, 1023); + rtn = wmove(self->win, y, x) == ERR + ? ERR + : wgetnstr(self->win, buf, n); #else - rtn2 = mvwgetnstr(self->win,y,x,rtn, 1023); + rtn = mvwgetnstr(self->win, y, x, buf, n); #endif Py_END_ALLOW_THREADS - break; - case 3: - if (!PyArg_ParseTuple(args,"iii;y,x,n", &y, &x, &n)) - return NULL; - if (n < 0) { - PyErr_SetString(PyExc_ValueError, "'n' must be nonnegative"); - return NULL; - } -#ifdef STRICT_SYSV_CURSES - Py_BEGIN_ALLOW_THREADS - rtn2 = wmove(self->win,y,x)==ERR ? ERR : - wgetnstr(self->win, rtn, Py_MIN(n, 1023)); - Py_END_ALLOW_THREADS -#else + } + else { Py_BEGIN_ALLOW_THREADS - rtn2 = mvwgetnstr(self->win, y, x, rtn, Py_MIN(n, 1023)); + rtn = wgetnstr(self->win, buf, n); Py_END_ALLOW_THREADS -#endif - break; - default: - PyErr_SetString(PyExc_TypeError, "getstr requires 0 to 3 arguments"); - return NULL; } - if (rtn2 == ERR) - rtn[0] = 0; - return PyBytes_FromString(rtn); + + if (rtn == ERR) { + Py_DECREF(res); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + } + _PyBytes_Resize(&res, strlen(buf)); // 'res' is set to NULL on failure + return res; } /*[clinic input] @@ -1821,15 +1855,15 @@ _curses.window.insch Insert a character before the current or specified position. -All characters to the right of the cursor are shifted one position right, with -the rightmost characters on the line being lost. +All characters to the right of the cursor are shifted one position +right, with the rightmost characters on the line being lost. [clinic start generated code]*/ static PyObject * _curses_window_insch_impl(PyCursesWindowObject *self, int group_left_1, int y, int x, PyObject *ch, int group_right_1, long attr) -/*[clinic end generated code: output=ade8cfe3a3bf3e34 input=336342756ee19812]*/ +/*[clinic end generated code: output=ade8cfe3a3bf3e34 input=d662a0f96f33e15a]*/ { int rtn; chtype ch_ = 0; @@ -1837,6 +1871,27 @@ _curses_window_insch_impl(PyCursesWindowObject *self, int group_left_1, if (!PyCurses_ConvertToChtype(self, ch, &ch_)) return NULL; +#ifdef HAVE_NCURSESW + /* winsch() does not locale-decode a byte above 127 on a wide build, + unlike waddch(), so decode it here and insert it as a wide character. */ + chtype cch = ch_ & A_CHARTEXT; + if (cch > 127) { + wint_t wc = btowc((int)cch); + if (wc != WEOF) { + cchar_t wch; + wchar_t wstr[2] = { (wchar_t)wc, L'\0' }; + attr_t cattr = (attr_t)((ch_ | (attr_t)attr) & ~(chtype)A_CHARTEXT); + setcchar(&wch, wstr, cattr, PAIR_NUMBER(cattr), NULL); + if (!group_left_1) { + rtn = wins_wch(self->win, &wch); + } + else { + rtn = mvwins_wch(self->win, y, x, &wch); + } + return PyCursesCheckERR_ForWin(self, rtn, "insch"); + } + } +#endif if (!group_left_1) { rtn = winsch(self->win, ch_ | (attr_t)attr); } @@ -1847,6 +1902,34 @@ _curses_window_insch_impl(PyCursesWindowObject *self, int group_left_1, return PyCursesCheckERR_ForWin(self, rtn, "insch"); } +#ifdef HAVE_NCURSESW +/* winch() returns the low 8 bits of the character's code point with no locale + conversion, unlike instr(), so recover the locale byte from the wide cell + when the character maps to exactly one byte, keeping the attribute and color + bits in RTN. A character with no single-byte form is left to winch(). */ +static chtype +curses_cell_locale_byte(chtype rtn, const cchar_t *cell) +{ + wchar_t wstr[CCHARW_MAX + 1]; + attr_t attrs; + short pair; + /* getcchar() is not guaranteed to write the text of an empty cell. */ + wstr[0] = L'\0'; + if (getcchar(cell, wstr, &attrs, &pair, NULL) == ERR + || wstr[0] == L'\0' || wstr[1] != L'\0') + { + return rtn; + } + /* wctob() mirrors ncurses' own _nc_to_char(): the single-byte form, or EOF + when the character has none in this locale. */ + int byte = wctob(wstr[0]); + if (byte != EOF) { + rtn = (rtn & ~(chtype)A_CHARTEXT) | (unsigned char)byte; + } + return rtn; +} +#endif + /*[clinic input] _curses.window.inch -> unsigned_long @@ -1860,13 +1943,14 @@ _curses.window.inch -> unsigned_long Return the character at the given position in the window. -The bottom 8 bits are the character proper, and upper bits are the attributes. +The bottom 8 bits are the character proper, and upper bits are the +attributes. [clinic start generated code]*/ static unsigned long _curses_window_inch_impl(PyCursesWindowObject *self, int group_right_1, int y, int x) -/*[clinic end generated code: output=6c4719fe978fe86a input=fac23ee11e3b3a66]*/ +/*[clinic end generated code: output=6c4719fe978fe86a input=0d862382571c19d9]*/ { unsigned long rtn; @@ -1876,73 +1960,70 @@ _curses_window_inch_impl(PyCursesWindowObject *self, int group_right_1, else { rtn = mvwinch(self->win, y, x); } - +#ifdef HAVE_NCURSESW + cchar_t cell = {0}; + if ((group_right_1 ? mvwin_wch(self->win, y, x, &cell) + : win_wch(self->win, &cell)) != ERR) + { + rtn = curses_cell_locale_byte(rtn, &cell); + } +#endif return rtn; } -/*[-clinic input] -_curses.window.instr - - [ - y: int - Y-coordinate. - x: int - X-coordinate. - ] - n: int = 1023 - Maximal number of characters. - / - -Return a string of characters, extracted from the window. +PyDoc_STRVAR(_curses_window_instr__doc__, +"instr([y, x,] n=2047)\n" +"Return a string of characters, extracted from the window.\n" +"\n" +" y\n" +" Y-coordinate.\n" +" x\n" +" X-coordinate.\n" +" n\n" +" Maximal number of characters.\n" +"\n" +"Return a string of characters, extracted from the window starting\n" +"at the current cursor position, or at y, x if specified, and\n" +"stopping at the end of the line. Attributes and color\n" +"information are stripped from the characters. If n is specified,\n" +"instr() returns a string at most n characters long (exclusive of\n" +"the trailing NUL)."); -Return a string of characters, extracted from the window starting at the -current cursor position, or at y, x if specified. Attributes are stripped -from the characters. If n is specified, instr() returns a string at most -n characters long (exclusive of the trailing NUL). -[-clinic start generated code]*/ static PyObject * -PyCursesWindow_InStr(PyObject *op, PyObject *args) +PyCursesWindow_instr(PyObject *op, PyObject *args) { PyCursesWindowObject *self = _PyCursesWindowObject_CAST(op); + int rtn, use_xy = 0, y = 0, x = 0; + unsigned int max_buf_size = 2048; + unsigned int n = max_buf_size - 1; + PyObject *res; - int x, y, n; - char rtn[1024]; /* This should be big enough.. I hope */ - int rtn2; + if (!curses_clinic_parse_optional_xy_n(args, &y, &x, &n, &use_xy, + "_curses.window.instr")) + { + return NULL; + } - switch (PyTuple_Size(args)) { - case 0: - rtn2 = winnstr(self->win,rtn, 1023); - break; - case 1: - if (!PyArg_ParseTuple(args,"i;n", &n)) - return NULL; - if (n < 0) { - PyErr_SetString(PyExc_ValueError, "'n' must be nonnegative"); - return NULL; - } - rtn2 = winnstr(self->win, rtn, Py_MIN(n, 1023)); - break; - case 2: - if (!PyArg_ParseTuple(args,"ii;y,x",&y,&x)) - return NULL; - rtn2 = mvwinnstr(self->win,y,x,rtn,1023); - break; - case 3: - if (!PyArg_ParseTuple(args, "iii;y,x,n", &y, &x, &n)) - return NULL; - if (n < 0) { - PyErr_SetString(PyExc_ValueError, "'n' must be nonnegative"); - return NULL; - } - rtn2 = mvwinnstr(self->win, y, x, rtn, Py_MIN(n,1023)); - break; - default: - PyErr_SetString(PyExc_TypeError, "instr requires 0 or 3 arguments"); + n = Py_MIN(n, max_buf_size - 1); + res = PyBytes_FromStringAndSize(NULL, n + 1); + if (res == NULL) { return NULL; } - if (rtn2 == ERR) - rtn[0] = 0; - return PyBytes_FromString(rtn); + char *buf = PyBytes_AS_STRING(res); + + if (use_xy) { + rtn = mvwinnstr(self->win, y, x, buf, n); + } + else { + rtn = winnstr(self->win, buf, n); + } + + if (rtn == ERR) { + Py_DECREF(res); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + } + _PyBytes_Resize(&res, strlen(buf)); // 'res' is set to NULL on failure + return res; } /*[clinic input] @@ -1966,18 +2047,18 @@ _curses.window.insstr Insert the string before the current or specified position. -Insert a character string (as many characters as will fit on the line) -before the character under the cursor. All characters to the right of -the cursor are shifted right, with the rightmost characters on the line -being lost. The cursor position does not change (after moving to y, x, -if specified). +Insert a character string (as many characters as will fit on the +line) before the character under the cursor. All characters to the +right of the cursor are shifted right, with the rightmost characters +on the line being lost. The cursor position does not change (after +moving to y, x, if specified). [clinic start generated code]*/ static PyObject * _curses_window_insstr_impl(PyCursesWindowObject *self, int group_left_1, int y, int x, PyObject *str, int group_right_1, long attr) -/*[clinic end generated code: output=c259a5265ad0b777 input=6827cddc6340a7f3]*/ +/*[clinic end generated code: output=c259a5265ad0b777 input=dbfbdd3892155ea6]*/ { int rtn; int strtype; @@ -2050,19 +2131,19 @@ _curses.window.insnstr Insert at most n characters of the string. -Insert a character string (as many characters as will fit on the line) -before the character under the cursor, up to n characters. If n is zero -or negative, the entire string is inserted. All characters to the right -of the cursor are shifted right, with the rightmost characters on the line -being lost. The cursor position does not change (after moving to y, x, if -specified). +Insert a character string (as many characters as will fit on the +line) before the character under the cursor, up to n characters. If +n is zero or negative, the entire string is inserted. All +characters to the right of the cursor are shifted right, with the +rightmost characters on the line being lost. The cursor position +does not change (after moving to y, x, if specified). [clinic start generated code]*/ static PyObject * _curses_window_insnstr_impl(PyCursesWindowObject *self, int group_left_1, int y, int x, PyObject *str, int n, int group_right_1, long attr) -/*[clinic end generated code: output=971a32ea6328ec8b input=70fa0cd543901a4c]*/ +/*[clinic end generated code: output=971a32ea6328ec8b input=fd0a9b65b84b385f]*/ { int rtn; int strtype; @@ -2120,12 +2201,13 @@ _curses.window.is_linetouched Return True if the specified line was modified, otherwise return False. -Raise a curses.error exception if line is not valid for the given window. +Raise a curses.error exception if line is not valid for the given +window. [clinic start generated code]*/ static PyObject * _curses_window_is_linetouched_impl(PyCursesWindowObject *self, int line) -/*[clinic end generated code: output=ad4a4edfee2db08c input=a7be0c189f243914]*/ +/*[clinic end generated code: output=ad4a4edfee2db08c input=b5c1149c36e58222]*/ { int erg; erg = is_linetouched(self->win, line); @@ -2153,9 +2235,9 @@ _curses.window.noutrefresh Mark for refresh but wait. -This function updates the data structure representing the desired state of the -window, but does not force an update of the physical screen. To accomplish -that, call doupdate(). +This function updates the data structure representing the desired +state of the window, but does not force an update of the physical +screen. To accomplish that, call doupdate(). [clinic start generated code]*/ static PyObject * @@ -2163,21 +2245,21 @@ _curses_window_noutrefresh_impl(PyCursesWindowObject *self, int group_right_1, int pminrow, int pmincol, int sminrow, int smincol, int smaxrow, int smaxcol) -/*[clinic end generated code: output=809a1f3c6a03e23e input=3e56898388cd739e]*/ +/*[clinic end generated code: output=809a1f3c6a03e23e input=8b4c74bf55008803]*/ #else /*[clinic input] _curses.window.noutrefresh Mark for refresh but wait. -This function updates the data structure representing the desired state of the -window, but does not force an update of the physical screen. To accomplish -that, call doupdate(). +This function updates the data structure representing the desired +state of the window, but does not force an update of the physical +screen. To accomplish that, call doupdate(). [clinic start generated code]*/ static PyObject * _curses_window_noutrefresh_impl(PyCursesWindowObject *self) -/*[clinic end generated code: output=6ef6dec666643fee input=876902e3fa431dbd]*/ +/*[clinic end generated code: output=6ef6dec666643fee input=a7c6306f8af9d0dd]*/ #endif { int rtn; @@ -2226,14 +2308,15 @@ _curses.window.overlay Overlay the window on top of destwin. -The windows need not be the same size, only the overlapping region is copied. -This copy is non-destructive, which means that the current background -character does not overwrite the old contents of destwin. +The windows need not be the same size, only the overlapping region +is copied. This copy is non-destructive, which means that the +current background character does not overwrite the old contents of +destwin. -To get fine-grained control over the copied region, the second form of -overlay() can be used. sminrow and smincol are the upper-left coordinates -of the source window, and the other variables mark a rectangle in the -destination window. +To get fine-grained control over the copied region, the second form +of overlay() can be used. sminrow and smincol are the upper-left +coordinates of the source window, and the other variables mark +a rectangle in the destination window. [clinic start generated code]*/ static PyObject * @@ -2241,7 +2324,7 @@ _curses_window_overlay_impl(PyCursesWindowObject *self, PyCursesWindowObject *destwin, int group_right_1, int sminrow, int smincol, int dminrow, int dmincol, int dmaxrow, int dmaxcol) -/*[clinic end generated code: output=82bb2c4cb443ca58 input=6e4b32a7c627a356]*/ +/*[clinic end generated code: output=82bb2c4cb443ca58 input=da0cec7f7bda1b3f]*/ { int rtn; @@ -2273,14 +2356,15 @@ _curses.window.overwrite Overwrite the window on top of destwin. -The windows need not be the same size, in which case only the overlapping -region is copied. This copy is destructive, which means that the current -background character overwrites the old contents of destwin. +The windows need not be the same size, in which case only the +overlapping region is copied. This copy is destructive, which means +that the current background character overwrites the old contents of +destwin. -To get fine-grained control over the copied region, the second form of -overwrite() can be used. sminrow and smincol are the upper-left coordinates -of the source window, the other variables mark a rectangle in the destination -window. +To get fine-grained control over the copied region, the second form +of overwrite() can be used. sminrow and smincol are the upper-left +coordinates of the source window, the other variables mark +a rectangle in the destination window. [clinic start generated code]*/ static PyObject * @@ -2289,7 +2373,7 @@ _curses_window_overwrite_impl(PyCursesWindowObject *self, int group_right_1, int sminrow, int smincol, int dminrow, int dmincol, int dmaxrow, int dmaxcol) -/*[clinic end generated code: output=12ae007d1681be28 input=d83dd8b24ff2bcc9]*/ +/*[clinic end generated code: output=12ae007d1681be28 input=4244ab8a97087898]*/ { int rtn; @@ -2387,23 +2471,24 @@ _curses.window.refresh Update the display immediately. Synchronize actual screen with previous drawing/deleting methods. -The 6 optional arguments can only be specified when the window is a pad -created with newpad(). The additional parameters are needed to indicate -what part of the pad and screen are involved. pminrow and pmincol specify -the upper left-hand corner of the rectangle to be displayed in the pad. -sminrow, smincol, smaxrow, and smaxcol specify the edges of the rectangle to -be displayed on the screen. The lower right-hand corner of the rectangle to -be displayed in the pad is calculated from the screen coordinates, since the -rectangles must be the same size. Both rectangles must be entirely contained -within their respective structures. Negative values of pminrow, pmincol, -sminrow, or smincol are treated as if they were zero. +The 6 optional arguments can only be specified when the window is +a pad created with newpad(). The additional parameters are needed +to indicate what part of the pad and screen are involved. pminrow +and pmincol specify the upper left-hand corner of the rectangle to +be displayed in the pad. sminrow, smincol, smaxrow, and smaxcol +specify the edges of the rectangle to be displayed on the screen. +The lower right-hand corner of the rectangle to be displayed in the +pad is calculated from the screen coordinates, since the rectangles +must be the same size. Both rectangles must be entirely contained +within their respective structures. Negative values of pminrow, +pmincol, sminrow, or smincol are treated as if they were zero. [clinic start generated code]*/ static PyObject * _curses_window_refresh_impl(PyCursesWindowObject *self, int group_right_1, int pminrow, int pmincol, int sminrow, int smincol, int smaxrow, int smaxcol) -/*[clinic end generated code: output=42199543115e6e63 input=95e01cb5ffc635d0]*/ +/*[clinic end generated code: output=42199543115e6e63 input=ff2e900c6b2696b1]*/ { int rtn; @@ -2472,14 +2557,14 @@ _curses.window.subwin Create a sub-window (screen-relative coordinates). -By default, the sub-window will extend from the specified position to the -lower right corner of the window. +By default, the sub-window will extend from the specified position +to the lower right corner of the window. [clinic start generated code]*/ static PyObject * _curses_window_subwin_impl(PyCursesWindowObject *self, int group_left_1, int nlines, int ncols, int begin_y, int begin_x) -/*[clinic end generated code: output=93e898afc348f59a input=2129fa47fd57721c]*/ +/*[clinic end generated code: output=93e898afc348f59a input=07b5058cb8820595]*/ { WINDOW *win; @@ -2513,13 +2598,14 @@ _curses.window.scroll Scroll the screen or scrolling region. -Scroll upward if the argument is positive and downward if it is negative. +Scroll upward if the argument is positive and downward if it is +negative. [clinic start generated code]*/ static PyObject * _curses_window_scroll_impl(PyCursesWindowObject *self, int group_right_1, int lines) -/*[clinic end generated code: output=4541a8a11852d360 input=c969ca0cfabbdbec]*/ +/*[clinic end generated code: output=4541a8a11852d360 input=d8d81a5b52b9b40f]*/ { if (!group_right_1) { return PyCursesCheckERR_ForWin(self, scroll(self->win), "scroll"); @@ -2541,14 +2627,15 @@ _curses.window.touchline Pretend count lines have been changed, starting with line start. -If changed is supplied, it specifies whether the affected lines are marked -as having been changed (changed=True) or unchanged (changed=False). +If changed is supplied, it specifies whether the affected lines are +marked as having been changed (changed=True) or unchanged +(changed=False). [clinic start generated code]*/ static PyObject * _curses_window_touchline_impl(PyCursesWindowObject *self, int start, int count, int group_right_1, int changed) -/*[clinic end generated code: output=65d05b3f7438c61d input=a98aa4f79b6be845]*/ +/*[clinic end generated code: output=65d05b3f7438c61d input=e0dc62f90d9dea55]*/ { if (!group_right_1) { return PyCursesCheckERR_ForWin(self, touchline(self->win, start, count), "touchline"); @@ -2653,79 +2740,158 @@ static PyMethodDef PyCursesWindow_methods[] = { _CURSES_WINDOW_ATTRSET_METHODDEF _CURSES_WINDOW_BKGD_METHODDEF #ifdef HAVE_CURSES_WCHGAT - {"chgat", PyCursesWindow_ChgAt, METH_VARARGS}, + { + "chgat", PyCursesWindow_ChgAt, METH_VARARGS, + _curses_window_chgat__doc__ + }, #endif _CURSES_WINDOW_BKGDSET_METHODDEF _CURSES_WINDOW_BORDER_METHODDEF _CURSES_WINDOW_BOX_METHODDEF - {"clear", PyCursesWindow_wclear, METH_NOARGS}, - {"clearok", PyCursesWindow_clearok, METH_VARARGS}, - {"clrtobot", PyCursesWindow_wclrtobot, METH_NOARGS}, - {"clrtoeol", PyCursesWindow_wclrtoeol, METH_NOARGS}, - {"cursyncup", PyCursesWindow_wcursyncup, METH_NOARGS}, + {"clear", PyCursesWindow_wclear, METH_NOARGS, + "clear($self, /)\n--\n\n" + "Clear the window and repaint it completely on the next refresh()."}, + {"clearok", PyCursesWindow_clearok, METH_VARARGS, + "clearok($self, flag, /)\n--\n\n" + "Clear the window on the next refresh() if flag is true."}, + {"clrtobot", PyCursesWindow_wclrtobot, METH_NOARGS, + "clrtobot($self, /)\n--\n\n" + "Erase from the cursor to the end of the window."}, + {"clrtoeol", PyCursesWindow_wclrtoeol, METH_NOARGS, + "clrtoeol($self, /)\n--\n\n" + "Erase from the cursor to the end of the line."}, + {"cursyncup", PyCursesWindow_wcursyncup, METH_NOARGS, + "cursyncup($self, /)\n--\n\n" + "Update the cursor position of all ancestor windows to match."}, _CURSES_WINDOW_DELCH_METHODDEF - {"deleteln", PyCursesWindow_wdeleteln, METH_NOARGS}, + {"deleteln", PyCursesWindow_wdeleteln, METH_NOARGS, + "deleteln($self, /)\n--\n\n" + "Delete the line under the cursor; move following lines up by one."}, _CURSES_WINDOW_DERWIN_METHODDEF _CURSES_WINDOW_ECHOCHAR_METHODDEF _CURSES_WINDOW_ENCLOSE_METHODDEF - {"erase", PyCursesWindow_werase, METH_NOARGS}, - {"getbegyx", PyCursesWindow_getbegyx, METH_NOARGS}, + {"erase", PyCursesWindow_werase, METH_NOARGS, + "erase($self, /)\n--\n\n" + "Clear the window."}, + {"getbegyx", PyCursesWindow_getbegyx, METH_NOARGS, + "getbegyx($self, /)\n--\n\n" + "Return a tuple (y, x) of the upper-left corner coordinates."}, _CURSES_WINDOW_GETBKGD_METHODDEF _CURSES_WINDOW_GETCH_METHODDEF _CURSES_WINDOW_GETKEY_METHODDEF _CURSES_WINDOW_GET_WCH_METHODDEF - {"getmaxyx", PyCursesWindow_getmaxyx, METH_NOARGS}, - {"getparyx", PyCursesWindow_getparyx, METH_NOARGS}, - {"getstr", PyCursesWindow_GetStr, METH_VARARGS}, - {"getyx", PyCursesWindow_getyx, METH_NOARGS}, + {"getmaxyx", PyCursesWindow_getmaxyx, METH_NOARGS, + "getmaxyx($self, /)\n--\n\n" + "Return a tuple (y, x) of the window height and width."}, + {"getparyx", PyCursesWindow_getparyx, METH_NOARGS, + "getparyx($self, /)\n--\n\n" + "Return (y, x) relative to the parent window, or (-1, -1) if none."}, + { + "getstr", PyCursesWindow_getstr, METH_VARARGS, + _curses_window_getstr__doc__ + }, + {"getyx", PyCursesWindow_getyx, METH_NOARGS, + "getyx($self, /)\n--\n\n" + "Return a tuple (y, x) of the current cursor position."}, _CURSES_WINDOW_HLINE_METHODDEF - {"idcok", PyCursesWindow_idcok, METH_VARARGS}, - {"idlok", PyCursesWindow_idlok, METH_VARARGS}, + {"idcok", PyCursesWindow_idcok, METH_VARARGS, + "idcok($self, flag, /)\n--\n\n" + "Enable or disable the hardware insert/delete character feature."}, + {"idlok", PyCursesWindow_idlok, METH_VARARGS, + "idlok($self, flag, /)\n--\n\n" + "Enable or disable the hardware insert/delete line feature."}, #ifdef HAVE_CURSES_IMMEDOK - {"immedok", PyCursesWindow_immedok, METH_VARARGS}, + {"immedok", PyCursesWindow_immedok, METH_VARARGS, + "immedok($self, flag, /)\n--\n\n" + "If flag is true, refresh the window on every change to it."}, #endif _CURSES_WINDOW_INCH_METHODDEF _CURSES_WINDOW_INSCH_METHODDEF - {"insdelln", PyCursesWindow_winsdelln, METH_VARARGS}, - {"insertln", PyCursesWindow_winsertln, METH_NOARGS}, + {"insdelln", PyCursesWindow_winsdelln, METH_VARARGS, + "insdelln($self, nlines, /)\n--\n\n" + "Insert (nlines > 0) or delete (nlines < 0) lines above the cursor."}, + {"insertln", PyCursesWindow_winsertln, METH_NOARGS, + "insertln($self, /)\n--\n\n" + "Insert a blank line under the cursor; move following lines down."}, _CURSES_WINDOW_INSNSTR_METHODDEF _CURSES_WINDOW_INSSTR_METHODDEF - {"instr", PyCursesWindow_InStr, METH_VARARGS}, + { + "instr", PyCursesWindow_instr, METH_VARARGS, + _curses_window_instr__doc__ + }, _CURSES_WINDOW_IS_LINETOUCHED_METHODDEF - {"is_wintouched", PyCursesWindow_is_wintouched, METH_NOARGS}, - {"keypad", PyCursesWindow_keypad, METH_VARARGS}, - {"leaveok", PyCursesWindow_leaveok, METH_VARARGS}, - {"move", PyCursesWindow_wmove, METH_VARARGS}, - {"mvderwin", PyCursesWindow_mvderwin, METH_VARARGS}, - {"mvwin", PyCursesWindow_mvwin, METH_VARARGS}, - {"nodelay", PyCursesWindow_nodelay, METH_VARARGS}, - {"notimeout", PyCursesWindow_notimeout, METH_VARARGS}, + {"is_wintouched", PyCursesWindow_is_wintouched, METH_NOARGS, + "is_wintouched($self, /)\n--\n\n" + "Return True if the window changed since the last refresh()."}, + {"keypad", PyCursesWindow_keypad, METH_VARARGS, + "keypad($self, flag, /)\n--\n\n" + "Interpret escape sequences for special keys if flag is true."}, + {"leaveok", PyCursesWindow_leaveok, METH_VARARGS, + "leaveok($self, flag, /)\n--\n\n" + "If flag is true, leave the cursor where the update leaves it."}, + {"move", PyCursesWindow_wmove, METH_VARARGS, + "move($self, new_y, new_x, /)\n--\n\n" + "Move the cursor to (new_y, new_x)."}, + {"mvderwin", PyCursesWindow_mvderwin, METH_VARARGS, + "mvderwin($self, y, x, /)\n--\n\n" + "Move the window inside its parent window."}, + {"mvwin", PyCursesWindow_mvwin, METH_VARARGS, + "mvwin($self, new_y, new_x, /)\n--\n\n" + "Move the window so its upper-left corner is at (new_y, new_x)."}, + {"nodelay", PyCursesWindow_nodelay, METH_VARARGS, + "nodelay($self, flag, /)\n--\n\n" + "If flag is true, getch() becomes non-blocking."}, + {"notimeout", PyCursesWindow_notimeout, METH_VARARGS, + "notimeout($self, flag, /)\n--\n\n" + "If flag is true, do not time out escape sequences."}, _CURSES_WINDOW_NOUTREFRESH_METHODDEF _CURSES_WINDOW_OVERLAY_METHODDEF _CURSES_WINDOW_OVERWRITE_METHODDEF _CURSES_WINDOW_PUTWIN_METHODDEF _CURSES_WINDOW_REDRAWLN_METHODDEF - {"redrawwin", PyCursesWindow_redrawwin, METH_NOARGS}, + {"redrawwin", PyCursesWindow_redrawwin, METH_NOARGS, + "redrawwin($self, /)\n--\n\n" + "Mark the entire window for redraw on the next refresh()."}, _CURSES_WINDOW_REFRESH_METHODDEF #ifndef STRICT_SYSV_CURSES - {"resize", PyCursesWindow_wresize, METH_VARARGS}, + {"resize", PyCursesWindow_wresize, METH_VARARGS, + "resize($self, nlines, ncols, /)\n--\n\n" + "Resize the window to nlines rows and ncols columns."}, #endif _CURSES_WINDOW_SCROLL_METHODDEF - {"scrollok", PyCursesWindow_scrollok, METH_VARARGS}, + {"scrollok", PyCursesWindow_scrollok, METH_VARARGS, + "scrollok($self, flag, /)\n--\n\n" + "Control whether the window scrolls when the cursor moves off it."}, _CURSES_WINDOW_SETSCRREG_METHODDEF - {"standend", PyCursesWindow_wstandend, METH_NOARGS}, - {"standout", PyCursesWindow_wstandout, METH_NOARGS}, + {"standend", PyCursesWindow_wstandend, METH_NOARGS, + "standend($self, /)\n--\n\n" + "Turn off the standout attribute."}, + {"standout", PyCursesWindow_wstandout, METH_NOARGS, + "standout($self, /)\n--\n\n" + "Turn on the A_STANDOUT attribute."}, {"subpad", _curses_window_subwin, METH_VARARGS, _curses_window_subwin__doc__}, _CURSES_WINDOW_SUBWIN_METHODDEF - {"syncdown", PyCursesWindow_wsyncdown, METH_NOARGS}, + {"syncdown", PyCursesWindow_wsyncdown, METH_NOARGS, + "syncdown($self, /)\n--\n\n" + "Touch each location changed in any ancestor of the window."}, #ifdef HAVE_CURSES_SYNCOK - {"syncok", PyCursesWindow_syncok, METH_VARARGS}, + {"syncok", PyCursesWindow_syncok, METH_VARARGS, + "syncok($self, flag, /)\n--\n\n" + "If flag is true, call syncup() on every change to the window."}, #endif - {"syncup", PyCursesWindow_wsyncup, METH_NOARGS}, - {"timeout", PyCursesWindow_wtimeout, METH_VARARGS}, + {"syncup", PyCursesWindow_wsyncup, METH_NOARGS, + "syncup($self, /)\n--\n\n" + "Touch locations in ancestors that changed in this window."}, + {"timeout", PyCursesWindow_wtimeout, METH_VARARGS, + "timeout($self, delay, /)\n--\n\n" + "Set blocking or non-blocking read behavior for the window."}, _CURSES_WINDOW_TOUCHLINE_METHODDEF - {"touchwin", PyCursesWindow_touchwin, METH_NOARGS}, - {"untouchwin", PyCursesWindow_untouchwin, METH_NOARGS}, + {"touchwin", PyCursesWindow_touchwin, METH_NOARGS, + "touchwin($self, /)\n--\n\n" + "Mark the whole window as changed."}, + {"untouchwin", PyCursesWindow_untouchwin, METH_NOARGS, + "untouchwin($self, /)\n--\n\n" + "Mark all lines in the window as unchanged since last refresh()."}, _CURSES_WINDOW_VLINE_METHODDEF {NULL, NULL} /* sentinel */ }; @@ -2813,11 +2979,17 @@ static PyType_Spec PyCursesWindow_Type_spec = { /*[clinic input] _curses.filter +Restrict screen updates to the current line. + +Must be called before initscr(). Afterwards curses confines the cursor +and screen updates to a single line, which is useful for enabling +character-at-a-time line editing without touching the rest of the +screen. [clinic start generated code]*/ static PyObject * _curses_filter_impl(PyObject *module) -/*[clinic end generated code: output=fb5b8a3642eb70b5 input=668c75a6992d3624]*/ +/*[clinic end generated code: output=fb5b8a3642eb70b5 input=e3c64d6ab2106132]*/ { /* not checking for PyCursesInitialised here since filter() must be called before initscr() */ @@ -2868,16 +3040,17 @@ _curses.cbreak Enter cbreak mode. -In cbreak mode (sometimes called "rare" mode) normal tty line buffering is -turned off and characters are available to be read one by one. However, -unlike raw mode, special characters (interrupt, quit, suspend, and flow -control) retain their effects on the tty driver and calling program. -Calling first raw() then cbreak() leaves the terminal in cbreak mode. +In cbreak mode (sometimes called "rare" mode) normal tty line buffering +is turned off and characters are available to be read one by one. +However, unlike raw mode, special characters (interrupt, quit, suspend, +and flow control) retain their effects on the tty driver and calling +program. Calling first raw() then cbreak() leaves the terminal in +cbreak mode. [clinic start generated code]*/ static PyObject * _curses_cbreak_impl(PyObject *module, int flag) -/*[clinic end generated code: output=9f9dee9664769751 input=c7d0bddda93016c1]*/ +/*[clinic end generated code: output=9f9dee9664769751 input=42d81687f11ddbf3]*/ NoArgOrFlagNoReturnFunctionBody(cbreak, flag) /*[clinic input] @@ -2889,13 +3062,14 @@ _curses.color_content Return the red, green, and blue (RGB) components of the specified color. -A 3-tuple is returned, containing the R, G, B values for the given color, -which will be between 0 (no component) and 1000 (maximum amount of component). +A 3-tuple is returned, containing the R, G, B values for the given +color, which will be between 0 (no component) and 1000 (maximum amount +of component). [clinic start generated code]*/ static PyObject * _curses_color_content_impl(PyObject *module, int color_number) -/*[clinic end generated code: output=17b466df7054e0de input=03b5ed0472662aea]*/ +/*[clinic end generated code: output=17b466df7054e0de input=c95fb50093fa0be0]*/ { _CURSES_COLOR_VAL_TYPE r,g,b; @@ -2922,12 +3096,13 @@ _curses.color_pair Return the attribute value for displaying text in the specified color. This attribute value can be combined with A_STANDOUT, A_REVERSE, and the -other A_* attributes. pair_number() is the counterpart to this function. +other A_* attributes. pair_number() is the counterpart to this +function. [clinic start generated code]*/ static PyObject * _curses_color_pair_impl(PyObject *module, int pair_number) -/*[clinic end generated code: output=60718abb10ce9feb input=6034e9146f343802]*/ +/*[clinic end generated code: output=60718abb10ce9feb input=cf74bb81d3cc3370]*/ { PyCursesStatefulInitialised(module); PyCursesStatefulInitialisedColor(module); @@ -2945,14 +3120,14 @@ _curses.curs_set Set the cursor state. If the terminal supports the visibility requested, the previous cursor -state is returned; otherwise, an exception is raised. On many terminals, -the "visible" mode is an underline cursor and the "very visible" mode is -a block cursor. +state is returned; otherwise, an exception is raised. On many +terminals, the "visible" mode is an underline cursor and the "very +visible" mode is a block cursor. [clinic start generated code]*/ static PyObject * _curses_curs_set_impl(PyObject *module, int visibility) -/*[clinic end generated code: output=ee8e62483b1d6cd4 input=81a7924a65d29504]*/ +/*[clinic end generated code: output=ee8e62483b1d6cd4 input=e010767a328f322b]*/ { int erg; @@ -2984,14 +3159,15 @@ _curses.def_shell_mode Save the current terminal mode as the "shell" mode. -The "shell" mode is the mode when the running program is not using curses. +The "shell" mode is the mode when the running program is not using +curses. Subsequent calls to reset_shell_mode() will restore this mode. [clinic start generated code]*/ static PyObject * _curses_def_shell_mode_impl(PyObject *module) -/*[clinic end generated code: output=d6e42f5c768f860f input=5ead21f6f0baa894]*/ +/*[clinic end generated code: output=d6e42f5c768f860f input=3809f85615c0b693]*/ NoArgNoReturnFunctionBody(def_shell_mode) /*[clinic input] @@ -3033,12 +3209,13 @@ _curses.echo Enter echo mode. -In echo mode, each character input is echoed to the screen as it is entered. +In echo mode, each character input is echoed to the screen as it is +entered. [clinic start generated code]*/ static PyObject * _curses_echo_impl(PyObject *module, int flag) -/*[clinic end generated code: output=03acb2ddfa6c8729 input=86cd4d5bb1d569c0]*/ +/*[clinic end generated code: output=03acb2ddfa6c8729 input=b4e9064326da9da4]*/ NoArgOrFlagNoReturnFunctionBody(echo, flag) /*[clinic input] @@ -3076,12 +3253,13 @@ _curses.flash Flash the screen. -That is, change it to reverse-video and then change it back in a short interval. +That is, change it to reverse-video and then change it back in a short +interval. [clinic start generated code]*/ static PyObject * _curses_flash_impl(PyObject *module) -/*[clinic end generated code: output=488b8a0ebd9ea9b8 input=02fdfb06c8fc3171]*/ +/*[clinic end generated code: output=488b8a0ebd9ea9b8 input=90878e305432add9]*/ NoArgNoReturnFunctionBody(flash) /*[clinic input] @@ -3089,13 +3267,13 @@ _curses.flushinp Flush all input buffers. -This throws away any typeahead that has been typed by the user and has not -yet been processed by the program. +This throws away any typeahead that has been typed by the user and has +not yet been processed by the program. [clinic start generated code]*/ static PyObject * _curses_flushinp_impl(PyObject *module) -/*[clinic end generated code: output=7e7a1fc1473960f5 input=59d042e705cef5ec]*/ +/*[clinic end generated code: output=7e7a1fc1473960f5 input=3a63c7213be8043c]*/ NoArgNoReturnVoidFunctionBody(flushinp) #ifdef getsyx @@ -3122,7 +3300,7 @@ _curses_getsyx_impl(PyObject *module) } #endif -#ifdef NCURSES_MOUSE_VERSION +#if defined(HAVE_CURSES_GETMOUSE) || defined(PDCURSES) /*[clinic input] _curses.getmouse @@ -3371,13 +3549,14 @@ _curses.init_pair Change the definition of a color-pair. -If the color-pair was previously initialized, the screen is refreshed and -all occurrences of that color-pair are changed to the new definition. +If the color-pair was previously initialized, the screen is refreshed +and all occurrences of that color-pair are changed to the new +definition. [clinic start generated code]*/ static PyObject * _curses_init_pair_impl(PyObject *module, int pair_number, int fg, int bg) -/*[clinic end generated code: output=a0bba03d2bbc3ee6 input=54b421b44c12c389]*/ +/*[clinic end generated code: output=a0bba03d2bbc3ee6 input=5486c3a105130dae]*/ { PyCursesStatefulInitialised(module); PyCursesStatefulInitialisedColor(module); @@ -3399,6 +3578,21 @@ _curses_init_pair_impl(PyObject *module, int pair_number, int fg, int bg) Py_RETURN_NONE; } +/* Refresh the private copy of the screen encoding from a freshly created + stdscr window object. Returns 0 on success, -1 with an exception set. */ +static int +curses_update_screen_encoding(PyObject *winobj) +{ + char *copy = _PyMem_Strdup(((PyCursesWindowObject *)winobj)->encoding); + if (copy == NULL) { + PyErr_NoMemory(); + return -1; + } + PyMem_Free(curses_screen_encoding); + curses_screen_encoding = copy; + return 0; +} + /*[clinic input] _curses.initscr @@ -3416,7 +3610,15 @@ _curses_initscr_impl(PyObject *module) if (curses_initscr_called) { wrefresh(stdscr); cursesmodule_state *state = get_cursesmodule_state(module); - return PyCursesWindow_New(state, stdscr, NULL, NULL); + PyObject *winobj = PyCursesWindow_New(state, stdscr, NULL, NULL); + if (winobj == NULL) { + return NULL; + } + if (curses_update_screen_encoding(winobj) < 0) { + Py_DECREF(winobj); + return NULL; + } + return winobj; } win = initscr(); @@ -3524,7 +3726,10 @@ _curses_initscr_impl(PyObject *module) if (winobj == NULL) { return NULL; } - curses_screen_encoding = ((PyCursesWindowObject *)winobj)->encoding; + if (curses_update_screen_encoding(winobj) < 0) { + Py_DECREF(winobj); + return NULL; + } return winobj; } @@ -3587,25 +3792,26 @@ _curses_setupterm_impl(PyObject *module, const char *term, int fd) Py_RETURN_NONE; } -#if defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20081102 -// https://invisible-island.net/ncurses/NEWS.html#index-t20080119 - +#ifdef HAVE_CURSES_ESCDELAY /*[clinic input] _curses.get_escdelay Gets the curses ESCDELAY setting. -Gets the number of milliseconds to wait after reading an escape character, -to distinguish between an individual escape character entered on the -keyboard from escape sequences sent by cursor and function keys. +Gets the number of milliseconds to wait after reading an escape +character, to distinguish between an individual escape character entered +on the keyboard from escape sequences sent by cursor and function keys. [clinic start generated code]*/ static PyObject * _curses_get_escdelay_impl(PyObject *module) -/*[clinic end generated code: output=222fa1a822555d60 input=be2d5b3dd974d0a4]*/ +/*[clinic end generated code: output=222fa1a822555d60 input=b39eeae4b8f169ab]*/ { return PyLong_FromLong(ESCDELAY); } +#endif /* HAVE_CURSES_ESCDELAY */ + +#ifdef HAVE_CURSES_SET_ESCDELAY /*[clinic input] _curses.set_escdelay ms: int @@ -3614,14 +3820,14 @@ _curses.set_escdelay Sets the curses ESCDELAY setting. -Sets the number of milliseconds to wait after reading an escape character, -to distinguish between an individual escape character entered on the -keyboard from escape sequences sent by cursor and function keys. +Sets the number of milliseconds to wait after reading an escape +character, to distinguish between an individual escape character entered +on the keyboard from escape sequences sent by cursor and function keys. [clinic start generated code]*/ static PyObject * _curses_set_escdelay_impl(PyObject *module, int ms) -/*[clinic end generated code: output=43818efbf7980ac4 input=7796fe19f111e250]*/ +/*[clinic end generated code: output=43818efbf7980ac4 input=cc2529bcdda3b06c]*/ { if (ms <= 0) { PyErr_SetString(PyExc_ValueError, "ms must be > 0"); @@ -3630,22 +3836,27 @@ _curses_set_escdelay_impl(PyObject *module, int ms) return PyCursesCheckERR(module, set_escdelay(ms), "set_escdelay"); } +#endif /* HAVE_CURSES_SET_ESCDELAY */ +#ifdef HAVE_CURSES_TABSIZE /*[clinic input] _curses.get_tabsize Gets the curses TABSIZE setting. -Gets the number of columns used by the curses library when converting a tab -character to spaces as it adds the tab to a window. +Gets the number of columns used by the curses library when converting +a tab character to spaces as it adds the tab to a window. [clinic start generated code]*/ static PyObject * _curses_get_tabsize_impl(PyObject *module) -/*[clinic end generated code: output=7e9e51fb6126fbdf input=74af86bf6c9f5d7e]*/ +/*[clinic end generated code: output=7e9e51fb6126fbdf input=58bdaacb337c103b]*/ { return PyLong_FromLong(TABSIZE); } +#endif /* HAVE_CURSES_TABSIZE */ + +#ifdef HAVE_CURSES_SET_TABSIZE /*[clinic input] _curses.set_tabsize size: int @@ -3654,13 +3865,13 @@ _curses.set_tabsize Sets the curses TABSIZE setting. -Sets the number of columns used by the curses library when converting a tab -character to spaces as it adds the tab to a window. +Sets the number of columns used by the curses library when converting +a tab character to spaces as it adds the tab to a window. [clinic start generated code]*/ static PyObject * _curses_set_tabsize_impl(PyObject *module, int size) -/*[clinic end generated code: output=c1de5a76c0daab1e input=78cba6a3021ad061]*/ +/*[clinic end generated code: output=c1de5a76c0daab1e input=34c1be9a78cd28a2]*/ { if (size <= 0) { PyErr_SetString(PyExc_ValueError, "size must be > 0"); @@ -3669,7 +3880,7 @@ _curses_set_tabsize_impl(PyObject *module, int size) return PyCursesCheckERR(module, set_tabsize(size), "set_tabsize"); } -#endif +#endif /* HAVE_CURSES_SET_TABSIZE */ /*[clinic input] _curses.intrflush @@ -3677,11 +3888,16 @@ _curses.intrflush flag: bool / +Control flushing of the output buffer when an interrupt key is pressed. + +If flag is true, pressing an interrupt key (interrupt, break, or quit) +flushes all output in the terminal driver queue. If flag is false, no +flushing is done. [clinic start generated code]*/ static PyObject * _curses_intrflush_impl(PyObject *module, int flag) -/*[clinic end generated code: output=c1986df35e999a0f input=c65fe2ef973fe40a]*/ +/*[clinic end generated code: output=c1986df35e999a0f input=66588c2bccc7e8fa]*/ { PyCursesStatefulInitialised(module); @@ -3771,13 +3987,13 @@ _curses.longname Return the terminfo long name field describing the current terminal. -The maximum length of a verbose description is 128 characters. It is defined -only after the call to initscr(). +The maximum length of a verbose description is 128 characters. It is +defined only after the call to initscr(). [clinic start generated code]*/ static PyObject * _curses_longname_impl(PyObject *module) -/*[clinic end generated code: output=fdf30433727ef568 input=84c3f20201b1098e]*/ +/*[clinic end generated code: output=fdf30433727ef568 input=a924fabba0de78a6]*/ NoArgReturnStringFunctionBody(longname) /*[clinic input] @@ -3801,7 +4017,7 @@ _curses_meta_impl(PyObject *module, int yes) return PyCursesCheckERR(module, meta(stdscr, yes), "meta"); } -#ifdef NCURSES_MOUSE_VERSION +#if defined(HAVE_CURSES_GETMOUSE) || defined(PDCURSES) /*[clinic input] _curses.mouseinterval @@ -3812,13 +4028,13 @@ _curses.mouseinterval Set and retrieve the maximum time between press and release in a click. Set the maximum time that can elapse between press and release events in -order for them to be recognized as a click, and return the previous interval -value. +order for them to be recognized as a click, and return the previous +interval value. [clinic start generated code]*/ static PyObject * _curses_mouseinterval_impl(PyObject *module, int interval) -/*[clinic end generated code: output=c4f5ff04354634c5 input=75aaa3f0db10ac4e]*/ +/*[clinic end generated code: output=c4f5ff04354634c5 input=b90249254389c080]*/ { PyCursesStatefulInitialised(module); @@ -3831,17 +4047,17 @@ _curses.mousemask newmask: unsigned_long(bitwise=True) / -Set the mouse events to be reported, and return a tuple (availmask, oldmask). +Set the mouse events to be reported, and return (availmask, oldmask). Return a tuple (availmask, oldmask). availmask indicates which of the -specified mouse events can be reported; on complete failure it returns 0. -oldmask is the previous value of the given window's mouse event mask. -If this function is never called, no mouse events are ever reported. +specified mouse events can be reported; on complete failure it returns +0. oldmask is the previous value of the mouse event mask. If this +function is never called, no mouse events are ever reported. [clinic start generated code]*/ static PyObject * _curses_mousemask_impl(PyObject *module, unsigned long newmask) -/*[clinic end generated code: output=9406cf1b8a36e485 input=bdf76b7568a3c541]*/ +/*[clinic end generated code: output=9406cf1b8a36e485 input=b8a9a4ccbce633f4]*/ { mmask_t oldmask, availmask; @@ -3924,14 +4140,14 @@ _curses.newwin Return a new window. -By default, the window will extend from the specified position to the lower -right corner of the screen. +By default, the window will extend from the specified position to the +lower right corner of the screen. [clinic start generated code]*/ static PyObject * _curses_newwin_impl(PyObject *module, int nlines, int ncols, int group_right_1, int begin_y, int begin_x) -/*[clinic end generated code: output=c1e0a8dc8ac2826c input=29312c15a72a003d]*/ +/*[clinic end generated code: output=c1e0a8dc8ac2826c input=a1517cbfea4ab24b]*/ { WINDOW *win; @@ -3957,13 +4173,14 @@ _curses.nl Enter newline mode. -This mode translates the return key into newline on input, and translates -newline into return and line-feed on output. Newline mode is initially on. +This mode translates the return key into newline on input, and +translates newline into return and line-feed on output. Newline mode +is initially on. [clinic start generated code]*/ static PyObject * _curses_nl_impl(PyObject *module, int flag) -/*[clinic end generated code: output=b39cc0ffc9015003 input=18e3e9c6e8cfcf6f]*/ +/*[clinic end generated code: output=b39cc0ffc9015003 input=3fb21dcf55521ee4]*/ NoArgOrFlagNoReturnFunctionBody(nl, flag) /*[clinic input] @@ -3997,13 +4214,13 @@ _curses.nonl Leave newline mode. -Disable translation of return into newline on input, and disable low-level -translation of newline into newline/return on output. +Disable translation of return into newline on input, and disable +low-level translation of newline into newline/return on output. [clinic start generated code]*/ static PyObject * _curses_nonl_impl(PyObject *module) -/*[clinic end generated code: output=99e917e9715770c6 input=9d37dd122d3022fc]*/ +/*[clinic end generated code: output=99e917e9715770c6 input=75cce08e4b6b3ef1]*/ NoArgNoReturnFunctionBody(nonl) /*[clinic input] @@ -4192,11 +4409,14 @@ update_lines_cols(PyObject *private_module) /*[clinic input] _curses.update_lines_cols +Update the LINES and COLS module variables. + +This is useful for detecting manual screen resize. [clinic start generated code]*/ static PyObject * _curses_update_lines_cols_impl(PyObject *module) -/*[clinic end generated code: output=423f2b1e63ed0f75 input=5f065ab7a28a5d90]*/ +/*[clinic end generated code: output=423f2b1e63ed0f75 input=1d8ea7c356b61a8b]*/ { if (!update_lines_cols(module)) { return NULL; @@ -4216,13 +4436,13 @@ _curses.raw Enter raw mode. In raw mode, normal line buffering and processing of interrupt, quit, -suspend, and flow control keys are turned off; characters are presented to -curses input functions one by one. +suspend, and flow control keys are turned off; characters are presented +to curses input functions one by one. [clinic start generated code]*/ static PyObject * _curses_raw_impl(PyObject *module, int flag) -/*[clinic end generated code: output=a750e4b342be015b input=4b447701389fb4df]*/ +/*[clinic end generated code: output=a750e4b342be015b input=18a7de7eef16987a]*/ NoArgOrFlagNoReturnFunctionBody(raw, flag) /*[clinic input] @@ -4270,13 +4490,13 @@ _curses.resizeterm Resize the standard and current windows to the specified dimensions. -Adjusts other bookkeeping data used by the curses library that record the -window dimensions (in particular the SIGWINCH handler). +Adjusts other bookkeeping data used by the curses library that record +the window dimensions (in particular the SIGWINCH handler). [clinic start generated code]*/ static PyObject * _curses_resizeterm_impl(PyObject *module, short nlines, short ncols) -/*[clinic end generated code: output=4de3abab50c67f02 input=414e92a63e3e9899]*/ +/*[clinic end generated code: output=4de3abab50c67f02 input=7f0f077df2da1cf5]*/ { PyObject *result; @@ -4307,15 +4527,16 @@ _curses.resize_term Backend function used by resizeterm(), performing most of the work. When resizing the windows, resize_term() blank-fills the areas that are -extended. The calling application should fill in these areas with appropriate -data. The resize_term() function attempts to resize all windows. However, -due to the calling convention of pads, it is not possible to resize these -without additional interaction with the application. +extended. The calling application should fill in these areas with +appropriate data. The resize_term() function attempts to resize all +windows. However, due to the calling convention of pads, it is not +possible to resize these without additional interaction with the +application. [clinic start generated code]*/ static PyObject * _curses_resize_term_impl(PyObject *module, short nlines, short ncols) -/*[clinic end generated code: output=46c6d749fa291dbd input=276afa43d8ea7091]*/ +/*[clinic end generated code: output=46c6d749fa291dbd input=ff4baaf2320c8ac9]*/ { PyObject *result; @@ -4375,17 +4596,17 @@ _curses.start_color Initializes eight basic colors and global variables COLORS and COLOR_PAIRS. -Must be called if the programmer wants to use colors, and before any other -color manipulation routine is called. It is good practice to call this -routine right after initscr(). +Must be called if the programmer wants to use colors, and before any +other color manipulation routine is called. It is good practice to call +this routine right after initscr(). -It also restores the colors on the terminal to the values they had when the -terminal was just turned on. +It also restores the colors on the terminal to the values they had when +the terminal was just turned on. [clinic start generated code]*/ static PyObject * _curses_start_color_impl(PyObject *module) -/*[clinic end generated code: output=8b772b41d8090ede input=0ca0ecb2b77e1a12]*/ +/*[clinic end generated code: output=8b772b41d8090ede input=0eed9bc743283a8d]*/ { PyCursesStatefulInitialised(module); @@ -4474,13 +4695,13 @@ _curses.tigetnum Return the value of the numeric capability. -The value -2 is returned if capname is not a numeric capability, or -1 if -it is canceled or absent from the terminal description. +The value -2 is returned if capname is not a numeric capability, or -1 +if it is canceled or absent from the terminal description. [clinic start generated code]*/ static PyObject * _curses_tigetnum_impl(PyObject *module, const char *capname) -/*[clinic end generated code: output=46f8b0a1b5dff42f input=5cdf2f410b109720]*/ +/*[clinic end generated code: output=46f8b0a1b5dff42f input=87a64beec16ae077]*/ { PyCursesStatefulSetupTermCalled(module); @@ -4496,13 +4717,13 @@ _curses.tigetstr Return the value of the string capability. -None is returned if capname is not a string capability, or is canceled or -absent from the terminal description. +None is returned if capname is not a string capability, or is canceled +or absent from the terminal description. [clinic start generated code]*/ static PyObject * _curses_tigetstr_impl(PyObject *module, const char *capname) -/*[clinic end generated code: output=f22b576ad60248f3 input=36644df25c73c0a7]*/ +/*[clinic end generated code: output=f22b576ad60248f3 input=00bf0feda2207724]*/ { PyCursesStatefulSetupTermCalled(module); @@ -4703,19 +4924,19 @@ _curses.use_env Use environment variables LINES and COLUMNS. -If used, this function should be called before initscr() or newterm() are -called. +If used, this function should be called before initscr() or newterm() +are called. -When flag is False, the values of lines and columns specified in the terminfo -database will be used, even if environment variables LINES and COLUMNS (used -by default) are set, or if curses is running in a window (in which case -default behavior would be to use the window size if LINES and COLUMNS are -not set). +When flag is False, the values of lines and columns specified in the +terminfo database will be used, even if environment variables LINES and +COLUMNS (used by default) are set, or if curses is running in a window +(in which case default behavior would be to use the window size if LINES +and COLUMNS are not set). [clinic start generated code]*/ static PyObject * _curses_use_env_impl(PyObject *module, int flag) -/*[clinic end generated code: output=b2c445e435c0b164 input=06ac30948f2d78e4]*/ +/*[clinic end generated code: output=b2c445e435c0b164 input=8e8feed746cf7fc1]*/ { use_env(flag); Py_RETURN_NONE; @@ -4916,10 +5137,8 @@ static PyMethodDef cursesmodule_methods[] = { _CURSES_RESIZETERM_METHODDEF _CURSES_RESIZE_TERM_METHODDEF _CURSES_SAVETTY_METHODDEF -#if defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20081102 _CURSES_GET_ESCDELAY_METHODDEF _CURSES_SET_ESCDELAY_METHODDEF -#endif _CURSES_GET_TABSIZE_METHODDEF _CURSES_SET_TABSIZE_METHODDEF _CURSES_SETSYX_METHODDEF @@ -5061,6 +5280,8 @@ static void cursesmodule_free(void *mod) { (void)cursesmodule_clear((PyObject *)mod); + PyMem_Free(curses_screen_encoding); + curses_screen_encoding = NULL; curses_module_loaded = 0; // allow reloading once garbage-collected } @@ -5220,7 +5441,7 @@ cursesmodule_exec(PyObject *module) SetDictInt("COLOR_CYAN", COLOR_CYAN); SetDictInt("COLOR_WHITE", COLOR_WHITE); -#ifdef NCURSES_MOUSE_VERSION +#if defined(HAVE_CURSES_GETMOUSE) || defined(PDCURSES) /* Mouse-related constants */ SetDictInt("BUTTON1_PRESSED", BUTTON1_PRESSED); SetDictInt("BUTTON1_RELEASED", BUTTON1_RELEASED); @@ -5246,7 +5467,7 @@ cursesmodule_exec(PyObject *module) SetDictInt("BUTTON4_DOUBLE_CLICKED", BUTTON4_DOUBLE_CLICKED); SetDictInt("BUTTON4_TRIPLE_CLICKED", BUTTON4_TRIPLE_CLICKED); -#if NCURSES_MOUSE_VERSION > 1 +#ifdef BUTTON5_PRESSED SetDictInt("BUTTON5_PRESSED", BUTTON5_PRESSED); SetDictInt("BUTTON5_RELEASED", BUTTON5_RELEASED); SetDictInt("BUTTON5_CLICKED", BUTTON5_CLICKED); diff --git a/Modules/_datetimemodule.c b/Modules/_datetimemodule.c index 9bba0e3354b26ba..f81234f39307c67 100644 --- a/Modules/_datetimemodule.c +++ b/Modules/_datetimemodule.c @@ -14,6 +14,8 @@ #include "pycore_object.h" // _PyObject_Init() #include "pycore_time.h" // _PyTime_ObjectToTime_t() #include "pycore_unicodeobject.h" // _PyUnicode_Copy() +#include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_pyatomic_ft_wrappers.h" #include "datetime.h" @@ -123,11 +125,10 @@ get_module_state(PyObject *module) #define INTERP_KEY ((PyObject *)&_Py_ID(cached_datetime_module)) -static PyObject * -get_current_module(PyInterpreterState *interp, int *p_reloading) +static int +get_current_module(PyInterpreterState *interp, PyObject **p_mod) { PyObject *mod = NULL; - int reloading = 0; PyObject *dict = PyInterpreterState_GetDict(interp); if (dict == NULL) { @@ -137,24 +138,24 @@ get_current_module(PyInterpreterState *interp, int *p_reloading) if (PyDict_GetItemRef(dict, INTERP_KEY, &ref) < 0) { goto error; } - if (ref != NULL) { - reloading = 1; - if (ref != Py_None) { - (void)PyWeakref_GetRef(ref, &mod); - if (mod == Py_None) { - Py_CLEAR(mod); - } + if (ref != NULL && ref != Py_None) { + if (PyWeakref_GetRef(ref, &mod) < 0) { Py_DECREF(ref); + goto error; } + if (mod == Py_None) { + Py_CLEAR(mod); + } + Py_DECREF(ref); } - if (p_reloading != NULL) { - *p_reloading = reloading; - } - return mod; + assert(!PyErr_Occurred()); + *p_mod = mod; + return mod != NULL; error: assert(PyErr_Occurred()); - return NULL; + *p_mod = NULL; + return -1; } static PyModuleDef datetimemodule; @@ -163,22 +164,26 @@ static datetime_state * _get_current_state(PyObject **p_mod) { PyInterpreterState *interp = PyInterpreterState_Get(); - PyObject *mod = get_current_module(interp, NULL); + PyObject *mod; + if (get_current_module(interp, &mod) < 0) { + goto error; + } if (mod == NULL) { - assert(!PyErr_Occurred()); - if (PyErr_Occurred()) { - return NULL; - } /* The static types can outlive the module, * so we must re-import the module. */ mod = PyImport_ImportModule("_datetime"); if (mod == NULL) { - return NULL; + goto error; } } datetime_state *st = get_module_state(mod); *p_mod = mod; return st; + +error: + assert(PyErr_Occurred()); + *p_mod = NULL; + return NULL; } #define GET_CURRENT_STATE(MOD_VAR) \ @@ -1026,7 +1031,16 @@ parse_hh_mm_ss_ff(const char *tstr, const char *tstr_end, int *hour, has_separator = (c == ':'); } - if (p >= p_end) { + if (c == '.' || c == ',') { + if (i < 2) { + return -3; // Decimal mark on hour or minute + } + if (p >= p_end) { + return -3; // Decimal mark not followed by any digit + } + break; + } + else if (p >= p_end) { return c != '\0'; } else if (has_separator && (c == ':')) { @@ -1035,14 +1049,10 @@ parse_hh_mm_ss_ff(const char *tstr, const char *tstr_end, int *hour, } continue; } - else if (c == '.' || c == ',') { - if (i < 2) { - return -3; // Decimal mark on hour or minute - } - break; - } else if (!has_separator) { + else if (!has_separator) { --p; - } else { + } + else { return -4; // Malformed time separator } } @@ -1655,8 +1665,8 @@ tzinfo_from_isoformat_results(int rv, int tzoffset, int tz_useconds) { PyObject *tzinfo; if (rv == 1) { - // Create a timezone from offset in seconds (0 returns UTC) - if (tzoffset == 0) { + // Create a timezone from the offset (a zero offset returns UTC) + if (tzoffset == 0 && tz_useconds == 0) { return Py_NewRef(CONST_UTC(NO_STATE)); } @@ -1757,6 +1767,24 @@ format_utcoffset(char *buf, size_t buflen, const char *sep, return 0; } +/* Check whether year with century should be normalized for strftime. */ +inline static int +normalize_century(void) +{ + static int cache = -1; + if (cache < 0) { + char year[5]; + struct tm date = { + .tm_year = -1801, + .tm_mon = 0, + .tm_mday = 1 + }; + cache = (strftime(year, sizeof(year), "%Y", &date) && + strcmp(year, "0099") != 0); + } + return cache; +} + static PyObject * make_somezreplacement(PyObject *object, char *sep, PyObject *tzinfoarg) { @@ -1928,10 +1956,9 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, } replacement = freplacement; } -#ifdef Py_NORMALIZE_CENTURY - else if (ch == 'Y' || ch == 'G' - || ch == 'F' || ch == 'C' - ) { + else if (normalize_century() + && (ch == 'Y' || ch == 'G' || ch == 'F' || ch == 'C')) + { /* 0-pad year with century as necessary */ PyObject *item = PySequence_GetItem(timetuple, 0); if (item == NULL) { @@ -1982,7 +2009,6 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, } continue; } -#endif else { /* percent followed by something else */ continue; @@ -2107,8 +2133,11 @@ delta_to_microseconds(PyDateTime_Delta *self) PyObject *x3 = NULL; PyObject *result = NULL; - PyObject *current_mod = NULL; + PyObject *current_mod; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (st == NULL) { + return NULL; + } x1 = PyLong_FromLong(GET_TD_DAYS(self)); if (x1 == NULL) @@ -2186,8 +2215,11 @@ microseconds_to_delta_ex(PyObject *pyus, PyTypeObject *type) PyObject *num = NULL; PyObject *result = NULL; - PyObject *current_mod = NULL; + PyObject *current_mod; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (st == NULL) { + return NULL; + } tuple = checked_divmod(pyus, CONST_US_PER_SECOND(st)); if (tuple == NULL) { @@ -2521,14 +2553,16 @@ static Py_hash_t delta_hash(PyObject *op) { PyDateTime_Delta *self = PyDelta_CAST(op); - if (self->hashcode == -1) { + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->hashcode); + if (hash == -1) { PyObject *temp = delta_getstate(self); if (temp != NULL) { - self->hashcode = PyObject_Hash(temp); + hash = PyObject_Hash(temp); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); Py_DECREF(temp); } } - return self->hashcode; + return hash; } static PyObject * @@ -2771,8 +2805,11 @@ delta_new(PyTypeObject *type, PyObject *args, PyObject *kw) { PyObject *self = NULL; - PyObject *current_mod = NULL; + PyObject *current_mod; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (st == NULL) { + return NULL; + } /* Argument objects. */ PyObject *day = NULL; @@ -2990,8 +3027,12 @@ delta_total_seconds(PyObject *op, PyObject *Py_UNUSED(dummy)) if (total_microseconds == NULL) return NULL; - PyObject *current_mod = NULL; + PyObject *current_mod; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (st == NULL) { + Py_DECREF(total_microseconds); + return NULL; + } total_seconds = PyNumber_TrueDivide(total_microseconds, CONST_US_PER_SECOND(st)); @@ -3287,13 +3328,13 @@ datetime.date.fromtimestamp Create a date from a POSIX timestamp. -The timestamp is a number, e.g. created via time.time(), that is interpreted -as local time. +The timestamp is a number, e.g. created via time.time(), that is +interpreted as local time. [clinic start generated code]*/ static PyObject * datetime_date_fromtimestamp_impl(PyTypeObject *type, PyObject *timestamp) -/*[clinic end generated code: output=59def4e32c028fb6 input=eabb3fe7f40491fe]*/ +/*[clinic end generated code: output=59def4e32c028fb6 input=15720eef43b169a1]*/ { return date_fromtimestamp((PyObject *) type, timestamp); } @@ -3773,8 +3814,11 @@ date_isocalendar(PyObject *self, PyObject *Py_UNUSED(dummy)) week = 0; } - PyObject *current_mod = NULL; + PyObject *current_mod; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (st == NULL) { + return NULL; + } PyObject *v = iso_calendar_date_new_impl(ISOCALENDAR_DATE_TYPE(st), year, week + 1, day + 1); @@ -3848,12 +3892,14 @@ static Py_hash_t date_hash(PyObject *op) { PyDateTime_Date *self = PyDate_CAST(op); - if (self->hashcode == -1) { - self->hashcode = generic_hash( + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->hashcode); + if (hash == -1) { + hash = generic_hash( (unsigned char *)self->data, _PyDateTime_DATE_DATASIZE); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); } - return self->hashcode; + return hash; } static PyObject * @@ -4476,7 +4522,7 @@ static PyTypeObject PyDateTime_TimeZoneType = { timezone_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ - 0, /* tp_base; filled in PyInit__datetime */ + &PyDateTime_TZInfoType, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ @@ -4932,7 +4978,8 @@ static Py_hash_t time_hash(PyObject *op) { PyDateTime_Time *self = PyTime_CAST(op); - if (self->hashcode == -1) { + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->hashcode); + if (hash == -1) { PyObject *offset, *self0; if (TIME_GET_FOLD(self)) { self0 = new_time_ex2(TIME_GET_HOUR(self), @@ -4954,10 +5001,11 @@ time_hash(PyObject *op) return -1; /* Reduce this to a hash of another object. */ - if (offset == Py_None) - self->hashcode = generic_hash( + if (offset == Py_None) { + hash = generic_hash( (unsigned char *)self->data, _PyDateTime_TIME_DATASIZE); - else { + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); + } else { PyObject *temp1, *temp2; int seconds, microseconds; assert(HASTZINFO(self)); @@ -4976,12 +5024,13 @@ time_hash(PyObject *op) Py_DECREF(offset); return -1; } - self->hashcode = PyObject_Hash(temp2); + hash = PyObject_Hash(temp2); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); Py_DECREF(temp2); } Py_DECREF(offset); } - return self->hashcode; + return hash; } /*[clinic input] @@ -5937,7 +5986,7 @@ datetime_fromisoformat(PyObject *cls, PyObject *dtstr) goto error; } - if ((hour == 24) && (month <= 12)) { + if ((hour == 24) && (month >= 1 && month <= 12)) { int d_in_month = days_in_month(year, month); if (day <= d_in_month) { if (minute == 0 && second == 0 && microsecond == 0) { @@ -6439,7 +6488,8 @@ static Py_hash_t datetime_hash(PyObject *op) { PyDateTime_DateTime *self = PyDateTime_CAST(op); - if (self->hashcode == -1) { + Py_hash_t hash = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->hashcode); + if (hash == -1) { PyObject *offset, *self0; if (DATE_GET_FOLD(self)) { self0 = new_datetime_ex2(GET_YEAR(self), @@ -6464,10 +6514,11 @@ datetime_hash(PyObject *op) return -1; /* Reduce this to a hash of another object. */ - if (offset == Py_None) - self->hashcode = generic_hash( + if (offset == Py_None) { + hash = generic_hash( (unsigned char *)self->data, _PyDateTime_DATETIME_DATASIZE); - else { + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); + } else { PyObject *temp1, *temp2; int days, seconds; @@ -6491,12 +6542,13 @@ datetime_hash(PyObject *op) Py_DECREF(offset); return -1; } - self->hashcode = PyObject_Hash(temp2); + hash = PyObject_Hash(temp2); + FT_ATOMIC_STORE_SSIZE_RELAXED(self->hashcode, hash); Py_DECREF(temp2); } Py_DECREF(offset); } - return self->hashcode; + return hash; } /*[clinic input] @@ -6598,8 +6650,11 @@ local_timezone(PyDateTime_DateTime *utc_time) PyObject *one_second; PyObject *seconds; - PyObject *current_mod = NULL; + PyObject *current_mod; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (st == NULL) { + return NULL; + } delta = datetime_subtract((PyObject *)utc_time, CONST_EPOCH(st)); RELEASE_CURRENT_STATE(st, current_mod); @@ -6842,8 +6897,11 @@ datetime_timestamp(PyObject *op, PyObject *Py_UNUSED(dummy)) PyObject *result; if (HASTZINFO(self) && self->tzinfo != Py_None) { - PyObject *current_mod = NULL; + PyObject *current_mod; datetime_state *st = GET_CURRENT_STATE(current_mod); + if (st == NULL) { + return NULL; + } PyObject *delta; delta = datetime_subtract(op, CONST_EPOCH(st)); @@ -7131,8 +7189,7 @@ static PyTypeObject PyDateTime_DateTimeType = { datetime_methods, /* tp_methods */ 0, /* tp_members */ datetime_getset, /* tp_getset */ - 0, /* tp_base; filled in - PyInit__datetime */ + &PyDateTime_DateType, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ @@ -7313,29 +7370,82 @@ clear_state(datetime_state *st) } -static int -init_static_types(PyInterpreterState *interp, int reloading) +PyStatus +_PyDateTime_InitTypes(PyInterpreterState *interp) { - if (reloading) { - return 0; - } - - // `&...` is not a constant expression according to a strict reading - // of C standards. Fill tp_base at run-time rather than statically. - // See https://bugs.python.org/issue40777 - PyDateTime_TimeZoneType.tp_base = &PyDateTime_TZInfoType; - PyDateTime_DateTimeType.tp_base = &PyDateTime_DateType; - /* Bases classes must be initialized before subclasses, * so capi_types must have the types in the appropriate order. */ for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) { PyTypeObject *type = capi_types[i]; if (_PyStaticType_InitForExtension(interp, type) < 0) { - return -1; + return _PyStatus_ERR("could not initialize static types"); } } - return 0; +#define DATETIME_ADD_MACRO(dict, c, value_expr) \ + do { \ + assert(!PyErr_Occurred()); \ + PyObject *value = (value_expr); \ + if (value == NULL) { \ + goto error; \ + } \ + if (PyDict_SetItemString(dict, c, value) < 0) { \ + Py_DECREF(value); \ + goto error; \ + } \ + Py_DECREF(value); \ + } while(0) + + /* timedelta values */ + PyObject *d = _PyType_GetDict(&PyDateTime_DeltaType); + DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); + DATETIME_ADD_MACRO(d, "min", new_delta(-MAX_DELTA_DAYS, 0, 0, 0)); + DATETIME_ADD_MACRO(d, "max", + new_delta(MAX_DELTA_DAYS, 24*3600-1, 1000000-1, 0)); + + /* date values */ + d = _PyType_GetDict(&PyDateTime_DateType); + DATETIME_ADD_MACRO(d, "min", new_date(1, 1, 1)); + DATETIME_ADD_MACRO(d, "max", new_date(MAXYEAR, 12, 31)); + DATETIME_ADD_MACRO(d, "resolution", new_delta(1, 0, 0, 0)); + + /* time values */ + d = _PyType_GetDict(&PyDateTime_TimeType); + DATETIME_ADD_MACRO(d, "min", new_time(0, 0, 0, 0, Py_None, 0)); + DATETIME_ADD_MACRO(d, "max", new_time(23, 59, 59, 999999, Py_None, 0)); + DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); + + /* datetime values */ + d = _PyType_GetDict(&PyDateTime_DateTimeType); + DATETIME_ADD_MACRO(d, "min", + new_datetime(1, 1, 1, 0, 0, 0, 0, Py_None, 0)); + DATETIME_ADD_MACRO(d, "max", new_datetime(MAXYEAR, 12, 31, 23, 59, 59, + 999999, Py_None, 0)); + DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); + + /* timezone values */ + d = _PyType_GetDict(&PyDateTime_TimeZoneType); + if (PyDict_SetItemString(d, "utc", (PyObject *)&utc_timezone) < 0) { + goto error; + } + + /* bpo-37642: These attributes are rounded to the nearest minute for backwards + * compatibility, even though the constructor will accept a wider range of + * values. This may change in the future.*/ + + /* -23:59 */ + DATETIME_ADD_MACRO(d, "min", create_timezone_from_delta(-1, 60, 0, 1)); + + /* +23:59 */ + DATETIME_ADD_MACRO( + d, "max", create_timezone_from_delta(0, (23 * 60 + 59) * 60, 0, 0)); + +#undef DATETIME_ADD_MACRO + + return _PyStatus_OK(); + +error: + return _PyStatus_NO_MEMORY(); } @@ -7353,20 +7463,14 @@ _datetime_exec(PyObject *module) { int rc = -1; datetime_state *st = get_module_state(module); - int reloading = 0; PyInterpreterState *interp = PyInterpreterState_Get(); - PyObject *old_module = get_current_module(interp, &reloading); - if (PyErr_Occurred()) { - assert(old_module == NULL); + PyObject *old_module; + if (get_current_module(interp, &old_module) < 0) { goto error; } /* We actually set the "current" module right before a successful return. */ - if (init_static_types(interp, reloading) < 0) { - goto error; - } - for (size_t i = 0; i < Py_ARRAY_LENGTH(capi_types); i++) { PyTypeObject *type = capi_types[i]; const char *name = _PyType_Name(type); @@ -7380,68 +7484,6 @@ _datetime_exec(PyObject *module) goto error; } -#define DATETIME_ADD_MACRO(dict, c, value_expr) \ - do { \ - assert(!PyErr_Occurred()); \ - PyObject *value = (value_expr); \ - if (value == NULL) { \ - goto error; \ - } \ - if (PyDict_SetItemString(dict, c, value) < 0) { \ - Py_DECREF(value); \ - goto error; \ - } \ - Py_DECREF(value); \ - } while(0) - - if (!reloading) { - /* timedelta values */ - PyObject *d = _PyType_GetDict(&PyDateTime_DeltaType); - DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - DATETIME_ADD_MACRO(d, "min", new_delta(-MAX_DELTA_DAYS, 0, 0, 0)); - DATETIME_ADD_MACRO(d, "max", - new_delta(MAX_DELTA_DAYS, 24*3600-1, 1000000-1, 0)); - - /* date values */ - d = _PyType_GetDict(&PyDateTime_DateType); - DATETIME_ADD_MACRO(d, "min", new_date(1, 1, 1)); - DATETIME_ADD_MACRO(d, "max", new_date(MAXYEAR, 12, 31)); - DATETIME_ADD_MACRO(d, "resolution", new_delta(1, 0, 0, 0)); - - /* time values */ - d = _PyType_GetDict(&PyDateTime_TimeType); - DATETIME_ADD_MACRO(d, "min", new_time(0, 0, 0, 0, Py_None, 0)); - DATETIME_ADD_MACRO(d, "max", new_time(23, 59, 59, 999999, Py_None, 0)); - DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - - /* datetime values */ - d = _PyType_GetDict(&PyDateTime_DateTimeType); - DATETIME_ADD_MACRO(d, "min", - new_datetime(1, 1, 1, 0, 0, 0, 0, Py_None, 0)); - DATETIME_ADD_MACRO(d, "max", new_datetime(MAXYEAR, 12, 31, 23, 59, 59, - 999999, Py_None, 0)); - DATETIME_ADD_MACRO(d, "resolution", new_delta(0, 0, 1, 0)); - - /* timezone values */ - d = _PyType_GetDict(&PyDateTime_TimeZoneType); - if (PyDict_SetItemString(d, "utc", (PyObject *)&utc_timezone) < 0) { - goto error; - } - - /* bpo-37642: These attributes are rounded to the nearest minute for backwards - * compatibility, even though the constructor will accept a wider range of - * values. This may change in the future.*/ - - /* -23:59 */ - DATETIME_ADD_MACRO(d, "min", create_timezone_from_delta(-1, 60, 0, 1)); - - /* +23:59 */ - DATETIME_ADD_MACRO( - d, "max", create_timezone_from_delta(0, (23 * 60 + 59) * 60, 0, 0)); - } - -#undef DATETIME_ADD_MACRO - /* Add module level attributes */ if (PyModule_AddIntMacro(module, MINYEAR) < 0) { goto error; diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c index cc65cbd98d71dcc..af39ff87fa75174 100644 --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -69,6 +69,7 @@ typedef struct { #include "clinic/_dbmmodule.c.h" #define check_dbmobject_open(v, err) \ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED((v)) \ if ((v)->di_dbm == NULL) { \ PyErr_SetString(err, "DBM object has already been closed"); \ return NULL; \ @@ -116,7 +117,7 @@ dbm_dealloc(PyObject *self) } static Py_ssize_t -dbm_length(PyObject *self) +dbm_length_lock_held(PyObject *self) { dbmobject *dp = dbmobject_CAST(self); _dbm_state *state = PyType_GetModuleState(Py_TYPE(dp)); @@ -138,8 +139,18 @@ dbm_length(PyObject *self) return dp->di_size; } +static Py_ssize_t +dbm_length(PyObject *self) +{ + Py_ssize_t result; + Py_BEGIN_CRITICAL_SECTION(self); + result = dbm_length_lock_held(self); + Py_END_CRITICAL_SECTION(); + return result; +} + static int -dbm_bool(PyObject *self) +dbm_bool_lock_held(PyObject *self) { dbmobject *dp = dbmobject_CAST(self); _dbm_state *state = PyType_GetModuleState(Py_TYPE(dp)); @@ -170,8 +181,18 @@ dbm_bool(PyObject *self) return 1; } +static int +dbm_bool(PyObject *self) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(self); + result = dbm_bool_lock_held(self); + Py_END_CRITICAL_SECTION(); + return result; +} + static PyObject * -dbm_subscript(PyObject *self, PyObject *key) +dbm_subscript_lock_held(PyObject *self, PyObject *key) { datum drec, krec; Py_ssize_t tmp_size; @@ -197,8 +218,18 @@ dbm_subscript(PyObject *self, PyObject *key) return PyBytes_FromStringAndSize(drec.dptr, drec.dsize); } +static PyObject * +dbm_subscript(PyObject *self, PyObject *key) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(self); + result = dbm_subscript_lock_held(self, key); + Py_END_CRITICAL_SECTION(); + return result; +} + static int -dbm_ass_sub(PyObject *self, PyObject *v, PyObject *w) +dbm_ass_sub_lock_held(PyObject *self, PyObject *v, PyObject *w) { datum krec, drec; Py_ssize_t tmp_size; @@ -252,7 +283,18 @@ dbm_ass_sub(PyObject *self, PyObject *v, PyObject *w) return 0; } +static int +dbm_ass_sub(PyObject *self, PyObject *v, PyObject *w) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(self); + result = dbm_ass_sub_lock_held(self, v, w); + Py_END_CRITICAL_SECTION(); + return result; +} + /*[clinic input] +@critical_section _dbm.dbm.close Close the database. @@ -260,7 +302,7 @@ Close the database. static PyObject * _dbm_dbm_close_impl(dbmobject *self) -/*[clinic end generated code: output=c8dc5b6709600b86 input=046db72377d51be8]*/ +/*[clinic end generated code: output=c8dc5b6709600b86 input=4a94f79facbc28ca]*/ { if (self->di_dbm) { dbm_close(self->di_dbm); @@ -270,6 +312,7 @@ _dbm_dbm_close_impl(dbmobject *self) } /*[clinic input] +@critical_section _dbm.dbm.keys cls: defining_class @@ -279,7 +322,7 @@ Return a list of all keys in the database. static PyObject * _dbm_dbm_keys_impl(dbmobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=f2a593b3038e5996 input=d3706a28fc051097]*/ +/*[clinic end generated code: output=f2a593b3038e5996 input=6ddefeadf2a80156]*/ { PyObject *v, *item; datum key; @@ -310,7 +353,7 @@ _dbm_dbm_keys_impl(dbmobject *self, PyTypeObject *cls) } static int -dbm_contains(PyObject *self, PyObject *arg) +dbm_contains_lock_held(PyObject *self, PyObject *arg) { dbmobject *dp = dbmobject_CAST(self); datum key, val; @@ -343,7 +386,18 @@ dbm_contains(PyObject *self, PyObject *arg) return val.dptr != NULL; } +static int +dbm_contains(PyObject *self, PyObject *arg) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(self); + result = dbm_contains_lock_held(self, arg); + Py_END_CRITICAL_SECTION(); + return result; +} + /*[clinic input] +@critical_section _dbm.dbm.get cls: defining_class key: str(accept={str, robuffer}, zeroes=True) @@ -356,7 +410,7 @@ Return the value for key if present, otherwise default. static PyObject * _dbm_dbm_get_impl(dbmobject *self, PyTypeObject *cls, const char *key, Py_ssize_t key_length, PyObject *default_value) -/*[clinic end generated code: output=b4e55f8b6d482bc4 input=66b993b8349fa8c1]*/ +/*[clinic end generated code: output=b4e55f8b6d482bc4 input=1d88a22bb5e55202]*/ { datum dbm_key, val; _dbm_state *state = PyType_GetModuleState(cls); @@ -373,6 +427,7 @@ _dbm_dbm_get_impl(dbmobject *self, PyTypeObject *cls, const char *key, } /*[clinic input] +@critical_section _dbm.dbm.setdefault cls: defining_class key: str(accept={str, robuffer}, zeroes=True) @@ -381,13 +436,14 @@ _dbm.dbm.setdefault Return the value for key if present, otherwise default. -If key is not in the database, it is inserted with default as the value. +If key is not in the database, it is inserted with default as the +value. [clinic start generated code]*/ static PyObject * _dbm_dbm_setdefault_impl(dbmobject *self, PyTypeObject *cls, const char *key, Py_ssize_t key_length, PyObject *default_value) -/*[clinic end generated code: output=9c2f6ea6d0fb576c input=126a3ff15c5f8232]*/ +/*[clinic end generated code: output=9c2f6ea6d0fb576c input=81224965c110f830]*/ { datum dbm_key, val; Py_ssize_t tmp_size; @@ -427,6 +483,7 @@ _dbm_dbm_setdefault_impl(dbmobject *self, PyTypeObject *cls, const char *key, } /*[clinic input] +@critical_section _dbm.dbm.clear cls: defining_class / @@ -436,7 +493,7 @@ Remove all items from the database. static PyObject * _dbm_dbm_clear_impl(dbmobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=8d126b9e1d01a434 input=43aa6ca1acb7f5f5]*/ +/*[clinic end generated code: output=8d126b9e1d01a434 input=a1aa5d99adfb9656]*/ { _dbm_state *state = PyType_GetModuleState(cls); assert(state != NULL); @@ -467,8 +524,12 @@ dbm__enter__(PyObject *self, PyObject *Py_UNUSED(dummy)) static PyObject * dbm__exit__(PyObject *self, PyObject *Py_UNUSED(args)) { + PyObject *result; dbmobject *dp = dbmobject_CAST(self); - return _dbm_dbm_close_impl(dp); + Py_BEGIN_CRITICAL_SECTION(self); + result = _dbm_dbm_close_impl(dp); + Py_END_CRITICAL_SECTION(); + return result; } static PyMethodDef dbm_methods[] = { diff --git a/Modules/_decimal/_decimal.c b/Modules/_decimal/_decimal.c index 602b23cfca89450..bead9b99309bddf 100644 --- a/Modules/_decimal/_decimal.c +++ b/Modules/_decimal/_decimal.c @@ -1452,6 +1452,20 @@ static int context_clear(PyObject *op) { PyDecContextObject *self = _PyDecContextObject_CAST(op); + /* Since traps and flags hold a borrowed reference to the + flags stored in the context object, these references need + to be cleared when the context object is deallocated + because traps and flags can survive. See gh-146011. */ + PyDecSignalDictObject *traps = _PyDecSignalDictObject_CAST(self->traps); + PyDecSignalDictObject *flags = _PyDecSignalDictObject_CAST(self->flags); + + if (traps != NULL) { + traps->flags = NULL; + } + if (flags != NULL) { + flags->flags = NULL; + } + Py_CLEAR(self->traps); Py_CLEAR(self->flags); return 0; @@ -3552,7 +3566,8 @@ dec_format(PyObject *dec, PyObject *args) if (size > 0 && fmt[size-1] == 'N') { if (PyErr_WarnEx(PyExc_DeprecationWarning, - "Format specifier 'N' is deprecated", 1) < 0) { + "Format specifier 'N' is deprecated and " + "slated for removal in Python 3.18", 1) < 0) { return NULL; } } @@ -6040,10 +6055,15 @@ _decimal_exec(PyObject *m) /* DecimalTuple */ ASSIGN_PTR(collections, PyImport_ImportModule("collections")); - ASSIGN_PTR(state->DecimalTuple, (PyTypeObject *)PyObject_CallMethod(collections, - "namedtuple", "(ss)", "DecimalTuple", - "sign digits exponent")); - + ASSIGN_PTR(obj, PyObject_CallMethod(collections, "namedtuple", "(ss)", + "DecimalTuple", + "sign digits exponent")); + if (!PyType_Check(obj)) { + PyErr_SetString(PyExc_TypeError, + "type is expected from namedtuple call"); + goto error; + } + ASSIGN_PTR(state->DecimalTuple, (PyTypeObject *)obj); ASSIGN_PTR(obj, PyUnicode_FromString("decimal")); CHECK_INT(PyDict_SetItemString(state->DecimalTuple->tp_dict, "__module__", obj)); Py_CLEAR(obj); diff --git a/Modules/_decimal/docstrings.h b/Modules/_decimal/docstrings.h index 77017a92252cb84..9c2da5e35f919ab 100644 --- a/Modules/_decimal/docstrings.h +++ b/Modules/_decimal/docstrings.h @@ -292,22 +292,26 @@ an infinity then Decimal('Infinity') is returned.\n\ PyDoc_STRVAR(doc_logical_and, "logical_and($self, /, other, context=None)\n--\n\n\ -Return the digit-wise 'and' of the two (logical) operands.\n\ +Applies an 'and' operation between self and other's digits.\n\n\ +Both self and other must be logical numbers.\n\ \n"); PyDoc_STRVAR(doc_logical_invert, "logical_invert($self, /, context=None)\n--\n\n\ -Return the digit-wise inversion of the (logical) operand.\n\ +Invert all its digits.\n\n\ +The self must be logical number.\n\ \n"); PyDoc_STRVAR(doc_logical_or, "logical_or($self, /, other, context=None)\n--\n\n\ -Return the digit-wise 'or' of the two (logical) operands.\n\ +Applies an 'or' operation between self and other's digits.\n\n\ +Both self and other must be logical numbers. \n\ \n"); PyDoc_STRVAR(doc_logical_xor, "logical_xor($self, /, other, context=None)\n--\n\n\ -Return the digit-wise 'exclusive or' of the two (logical) operands.\n\ +Applies an 'xor' operation between self and other's digits.\n\n\ +Both self and other must be logical numbers.\n\ \n"); PyDoc_STRVAR(doc_max, @@ -712,22 +716,90 @@ Return the exponent of the magnitude of the operand's MSD.\n\ PyDoc_STRVAR(doc_ctx_logical_and, "logical_and($self, x, y, /)\n--\n\n\ -Digit-wise and of x and y.\n\ +Applies the logical operation 'and' between each operand's digits.\n\n\ +The operands must be both logical numbers.\n\n\ + >>> ExtendedContext.logical_and(Decimal('0'), Decimal('0'))\n\ + Decimal('0')\n\ + >>> ExtendedContext.logical_and(Decimal('0'), Decimal('1'))\n\ + Decimal('0')\n\ + >>> ExtendedContext.logical_and(Decimal('1'), Decimal('0'))\n\ + Decimal('0')\n\ + >>> ExtendedContext.logical_and(Decimal('1'), Decimal('1'))\n\ + Decimal('1')\n\ + >>> ExtendedContext.logical_and(Decimal('1100'), Decimal('1010'))\n\ + Decimal('1000')\n\ + >>> ExtendedContext.logical_and(Decimal('1111'), Decimal('10'))\n\ + Decimal('10')\n\ + >>> ExtendedContext.logical_and(110, 1101)\n\ + Decimal('100')\n\ + >>> ExtendedContext.logical_and(Decimal(110), 1101)\n\ + Decimal('100')\n\ + >>> ExtendedContext.logical_and(110, Decimal(1101))\n\ + Decimal('100')\n\ \n"); PyDoc_STRVAR(doc_ctx_logical_invert, "logical_invert($self, x, /)\n--\n\n\ -Invert all digits of x.\n\ +Invert all the digits in the operand.\n\n\ +The operand must be a logical number.\n\n\ + >>> ExtendedContext.logical_invert(Decimal('0'))\n\ + Decimal('111111111')\n\ + >>> ExtendedContext.logical_invert(Decimal('1'))\n\ + Decimal('111111110')\n\ + >>> ExtendedContext.logical_invert(Decimal('111111111'))\n\ + Decimal('0')\n\ + >>> ExtendedContext.logical_invert(Decimal('101010101'))\n\ + Decimal('10101010')\n\ + >>> ExtendedContext.logical_invert(1101)\n\ + Decimal('111110010')\n\ \n"); PyDoc_STRVAR(doc_ctx_logical_or, "logical_or($self, x, y, /)\n--\n\n\ -Digit-wise or of x and y.\n\ +Applies the logical operation 'or' between each operand's digits.\n\n\ +The operands must be both logical numbers.\n\n\ + >>> ExtendedContext.logical_or(Decimal('0'), Decimal('0'))\n\ + Decimal('0')\n\ + >>> ExtendedContext.logical_or(Decimal('0'), Decimal('1'))\n\ + Decimal('1')\n\ + >>> ExtendedContext.logical_or(Decimal('1'), Decimal('0'))\n\ + Decimal('1')\n\ + >>> ExtendedContext.logical_or(Decimal('1'), Decimal('1'))\n\ + Decimal('1')\n\ + >>> ExtendedContext.logical_or(Decimal('1100'), Decimal('1010'))\n\ + Decimal('1110')\n\ + >>> ExtendedContext.logical_or(Decimal('1110'), Decimal('10'))\n\ + Decimal('1110')\n\ + >>> ExtendedContext.logical_or(110, 1101)\n\ + Decimal('1111')\n\ + >>> ExtendedContext.logical_or(Decimal(110), 1101)\n\ + Decimal('1111')\n\ + >>> ExtendedContext.logical_or(110, Decimal(1101))\n\ + Decimal('1111')\n\ \n"); PyDoc_STRVAR(doc_ctx_logical_xor, "logical_xor($self, x, y, /)\n--\n\n\ -Digit-wise xor of x and y.\n\ +Applies the logical operation 'xor' between each operand's digits.\n\n\ +The operands must be both logical numbers.\n\n\ + >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('0'))\n\ + Decimal('0')\n\ + >>> ExtendedContext.logical_xor(Decimal('0'), Decimal('1'))\n\ + Decimal('1')\n\ + >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('0'))\n\ + Decimal('1')\n\ + >>> ExtendedContext.logical_xor(Decimal('1'), Decimal('1'))\n\ + Decimal('0')\n\ + >>> ExtendedContext.logical_xor(Decimal('1100'), Decimal('1010'))\n\ + Decimal('110')\n\ + >>> ExtendedContext.logical_xor(Decimal('1111'), Decimal('10'))\n\ + Decimal('1101')\n\ + >>> ExtendedContext.logical_xor(110, 1101)\n\ + Decimal('1011')\n\ + >>> ExtendedContext.logical_xor(Decimal(110), 1101)\n\ + Decimal('1011')\n\ + >>> ExtendedContext.logical_xor(110, Decimal(1101))\n\ + Decimal('1011')\n\ \n"); PyDoc_STRVAR(doc_ctx_max, diff --git a/Modules/_elementtree.c b/Modules/_elementtree.c index 8c3efa36353e24f..6d867d2632cc9af 100644 --- a/Modules/_elementtree.c +++ b/Modules/_elementtree.c @@ -16,7 +16,9 @@ #endif #include "Python.h" +#include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_pyhash.h" // _Py_HashSecret +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> // offsetof() #include "expat.h" @@ -562,7 +564,7 @@ element_get_attrib(ElementObject* self) LOCAL(PyObject*) element_get_text(ElementObject* self) { - /* return borrowed reference to text attribute */ + /* return new reference to text attribute */ PyObject *res = self->text; @@ -577,13 +579,13 @@ element_get_text(ElementObject* self) } } - return res; + return Py_NewRef(res); } LOCAL(PyObject*) element_get_tail(ElementObject* self) { - /* return borrowed reference to text attribute */ + /* return new reference to tail attribute */ PyObject *res = self->tail; @@ -598,7 +600,7 @@ element_get_tail(ElementObject* self) } } - return res; + return Py_NewRef(res); } static PyObject* @@ -690,8 +692,7 @@ element_dealloc(PyObject *op) /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, self->weakreflist); /* element_gc_clear clears all references and deallocates extra */ @@ -802,24 +803,31 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) /*[clinic end generated code: output=eefc3df50465b642 input=a2d40348c0aade10]*/ { Py_ssize_t i; - ElementObject* element; + ElementObject* element = NULL; PyObject* tag; PyObject* attrib; PyObject* text; PyObject* tail; PyObject* id; + if (_Py_EnterRecursiveCall(" in Element.__deepcopy__")) { + return NULL; + } + PyTypeObject *tp = Py_TYPE(self); elementtreestate *st = get_elementtree_state_by_type(tp); + // The deepcopy() helper takes care of incrementing the refcount + // of the object to copy so to avoid use-after-frees. tag = deepcopy(st, self->tag, memo); - if (!tag) - return NULL; + if (!tag) { + goto error; + } if (self->extra && self->extra->attrib) { attrib = deepcopy(st, self->extra->attrib, memo); if (!attrib) { Py_DECREF(tag); - return NULL; + goto error; } } else { attrib = NULL; @@ -830,8 +838,9 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) Py_DECREF(tag); Py_XDECREF(attrib); - if (!element) - return NULL; + if (!element) { + goto error; + } text = deepcopy(st, JOIN_OBJ(self->text), memo); if (!text) @@ -845,11 +854,13 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) assert(!element->extra || !element->extra->length); if (self->extra) { - if (element_resize(element, self->extra->length) < 0) + Py_ssize_t expected_count = self->extra->length; + if (element_resize(element, expected_count) < 0) { + assert(!element->extra->length); goto error; + } - // TODO(picnixz): check for an evil child's __deepcopy__ on 'self' - for (i = 0; i < self->extra->length; i++) { + for (i = 0; self->extra && i < self->extra->length; i++) { PyObject* child = deepcopy(st, self->extra->children[i], memo); if (!child || !Element_Check(st, child)) { if (child) { @@ -859,11 +870,24 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) element->extra->length = i; goto error; } + if (self->extra && expected_count != self->extra->length) { + // 'self->extra' got mutated and 'element' may not have + // sufficient space to hold the next iteration's item. + expected_count = self->extra->length; + if (element_resize(element, expected_count) < 0) { + Py_DECREF(child); + element->extra->length = i; + goto error; + } + } element->extra->children[i] = child; } assert(!element->extra->length); - element->extra->length = self->extra->length; + // The original 'self->extra' may be gone at this point if deepcopy() + // has side-effects. However, 'i' is the number of copied items that + // we were able to successfully copy. + element->extra->length = i; } /* add object to memo dictionary (so deepcopy won't visit it again) */ @@ -878,10 +902,12 @@ _elementtree_Element___deepcopy___impl(ElementObject *self, PyObject *memo) if (i < 0) goto error; + _Py_LeaveRecursiveCall(); return (PyObject*) element; error: - Py_DECREF(element); + _Py_LeaveRecursiveCall(); + Py_XDECREF(element); return NULL; } @@ -895,7 +921,7 @@ deepcopy(elementtreestate *st, PyObject *object, PyObject *memo) return Py_NewRef(object); } - if (Py_REFCNT(object) == 1) { + if (_PyObject_IsUniquelyReferenced(object)) { if (PyDict_CheckExact(object)) { PyObject *key, *value; Py_ssize_t pos = 0; @@ -906,13 +932,20 @@ deepcopy(elementtreestate *st, PyObject *object, PyObject *memo) break; } } - if (simple) + if (simple) { return PyDict_Copy(object); + } /* Fall through to general case */ } else if (Element_CheckExact(st, object)) { - return _elementtree_Element___deepcopy___impl( + // The __deepcopy__() call may call arbitrary code even if the + // object to copy is a built-in XML element (one of its children + // any of its parents in its own __deepcopy__() implementation). + Py_INCREF(object); + PyObject *res = _elementtree_Element___deepcopy___impl( (ElementObject *)object, memo); + Py_DECREF(object); + return res; } } @@ -923,8 +956,11 @@ deepcopy(elementtreestate *st, PyObject *object, PyObject *memo) return NULL; } + Py_INCREF(object); PyObject *args[2] = {object, memo}; - return PyObject_Vectorcall(st->deepcopy_obj, args, 2, NULL); + PyObject *res = PyObject_Vectorcall(st->deepcopy_obj, args, 2, NULL); + Py_DECREF(object); + return res; } @@ -1314,9 +1350,9 @@ _elementtree_Element_findtext_impl(ElementObject *self, PyTypeObject *cls, PyObject *text = element_get_text((ElementObject *)item); Py_DECREF(item); if (text == Py_None) { + Py_DECREF(text); return Py_GetConstant(Py_CONSTANT_EMPTY_STR); } - Py_XINCREF(text); return text; } Py_DECREF(item); @@ -1779,16 +1815,20 @@ element_subscr(PyObject *op, PyObject *item) return element_getitem(op, i); } else if (PySlice_Check(item)) { + // Note: 'slicelen' is computed once we are sure that 'self->extra' + // cannot be mutated by user-defined code. + // See https://github.com/python/cpython/issues/143200. Py_ssize_t start, stop, step, slicelen, i; size_t cur; PyObject* list; - if (!self->extra) - return PyList_New(0); - if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } + + if (self->extra == NULL) { + return PyList_New(0); + } slicelen = PySlice_AdjustIndices(self->extra->length, &start, &stop, step); @@ -1831,28 +1871,26 @@ element_ass_subscr(PyObject *op, PyObject *item, PyObject *value) return element_setitem(op, i, value); } else if (PySlice_Check(item)) { + // Note: 'slicelen' is computed once we are sure that 'self->extra' + // cannot be mutated by user-defined code. + // See https://github.com/python/cpython/issues/143200. Py_ssize_t start, stop, step, slicelen, newlen, i; size_t cur; PyObject* recycle = NULL; PyObject* seq; - if (!self->extra) { - if (create_extra(self, NULL) < 0) - return -1; - } - if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return -1; } - slicelen = PySlice_AdjustIndices(self->extra->length, &start, &stop, - step); if (value == NULL) { /* Delete slice */ - size_t cur; - Py_ssize_t i; - + if (self->extra == NULL) { + return 0; + } + slicelen = PySlice_AdjustIndices(self->extra->length, &start, &stop, + step); if (slicelen <= 0) return 0; @@ -1921,8 +1959,16 @@ element_ass_subscr(PyObject *op, PyObject *item, PyObject *value) } newlen = PySequence_Fast_GET_SIZE(seq); - if (step != 1 && newlen != slicelen) - { + if (self->extra == NULL) { + if (create_extra(self, NULL) < 0) { + Py_DECREF(seq); + return -1; + } + } + slicelen = PySlice_AdjustIndices(self->extra->length, &start, &stop, + step); + + if (step != 1 && newlen != slicelen) { Py_DECREF(seq); PyErr_Format(PyExc_ValueError, "attempt to assign sequence of size %zd " @@ -2010,16 +2056,14 @@ static PyObject* element_text_getter(PyObject *op, void *closure) { ElementObject *self = _Element_CAST(op); - PyObject *res = element_get_text(self); - return Py_XNewRef(res); + return element_get_text(self); } static PyObject* element_tail_getter(PyObject *op, void *closure) { ElementObject *self = _Element_CAST(op); - PyObject *res = element_get_tail(self); - return Py_XNewRef(res); + return element_get_tail(self); } static PyObject* @@ -2245,6 +2289,10 @@ elementiter_next(PyObject *op) return NULL; } if (it->gettext) { + if (elem->tag != Py_None && !PyUnicode_Check(elem->tag)) { + Py_DECREF(elem); + continue; + } text = element_get_text(elem); goto gettext; } @@ -2262,16 +2310,14 @@ elementiter_next(PyObject *op) continue; gettext: + Py_DECREF(elem); if (!text) { - Py_DECREF(elem); return NULL; } if (text == Py_None) { - Py_DECREF(elem); + Py_DECREF(text); } else { - Py_INCREF(text); - Py_DECREF(elem); rc = PyObject_IsTrue(text); if (rc > 0) return text; @@ -2767,8 +2813,9 @@ treebuilder_handle_data(TreeBuilderObject* self, PyObject* data) self->data = Py_NewRef(data); } else { /* more than one item; use a list to collect items */ - if (PyBytes_CheckExact(self->data) && Py_REFCNT(self->data) == 1 && - PyBytes_CheckExact(data) && PyBytes_GET_SIZE(data) == 1) { + if (PyBytes_CheckExact(self->data) + && _PyObject_IsUniquelyReferenced(self->data) + && PyBytes_CheckExact(data) && PyBytes_GET_SIZE(data) == 1) { /* XXX this code path unused in Python 3? */ /* expat often generates single character data sections; handle the most common case by resizing the existing string... */ @@ -3681,8 +3728,12 @@ _elementtree_XMLParser___init___impl(XMLParserObject *self, PyObject *target, PyErr_NoMemory(); return -1; } - /* expat < 2.1.0 has no XML_SetHashSalt() */ - if (EXPAT(st, SetHashSalt) != NULL) { + // Prefer 16-byte entropy, only expat >= 2.8.0. See gh-149018 + if (EXPAT(st, SetHashSalt16Bytes) != NULL) { + EXPAT(st, SetHashSalt16Bytes)(self->parser, + _Py_HashSecret.expat.hashsalt16); + } + else if (EXPAT(st, SetHashSalt) != NULL) { EXPAT(st, SetHashSalt)(self->parser, (unsigned long)_Py_HashSecret.expat.hashsalt); } @@ -4187,8 +4238,8 @@ _elementtree_XMLParser__setevents_impl(XMLParserObject *self, (XML_ProcessingInstructionHandler) expat_pi_handler ); } else { - Py_DECREF(events_seq); PyErr_Format(PyExc_ValueError, "unknown event '%s'", event_name); + Py_DECREF(events_seq); return NULL; } } diff --git a/Modules/_functoolsmodule.c b/Modules/_functoolsmodule.c index e6c454faf4b16f3..a4cfac825e83e0f 100644 --- a/Modules/_functoolsmodule.c +++ b/Modules/_functoolsmodule.c @@ -7,6 +7,7 @@ #include "pycore_pyatomic_ft_wrappers.h" #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_tuple.h" // _PyTuple_ITEMS() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "clinic/_functoolsmodule.c.h" @@ -196,6 +197,19 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; } + /* keyword Placeholder prohibition */ + if (kw != NULL) { + PyObject *key, *val; + Py_ssize_t pos = 0; + while (PyDict_Next(kw, &pos, &key, &val)) { + if (val == phold) { + PyErr_SetString(PyExc_TypeError, + "Placeholder cannot be passed as a keyword argument"); + return NULL; + } + } + } + /* check wrapped function / object */ pto_args = pto_kw = NULL; int res = PyObject_TypeCheck(func, state->partial_type); @@ -284,7 +298,7 @@ partial_new(PyTypeObject *type, PyObject *args, PyObject *kw) if (kw == NULL) { pto->kw = PyDict_New(); } - else if (Py_REFCNT(kw) == 1) { + else if (_PyObject_IsUniquelyReferenced(kw)) { pto->kw = Py_NewRef(kw); } else { @@ -338,9 +352,7 @@ partial_dealloc(PyObject *self) PyTypeObject *tp = Py_TYPE(self); /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(self); - if (partialobject_CAST(self)->weakreflist != NULL) { - PyObject_ClearWeakRefs(self); - } + FT_CLEAR_WEAKREFS(self, partialobject_CAST(self)->weakreflist); (void)partial_clear(self); tp->tp_free(self); Py_DECREF(tp); @@ -363,7 +375,9 @@ partial_vectorcall_fallback(PyThreadState *tstate, partialobject *pto, PyObject *const *args, size_t nargsf, PyObject *kwnames) { +#ifndef Py_GIL_DISABLED pto->vectorcall = NULL; +#endif Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); return _PyObject_MakeTpCall(tstate, (PyObject *)pto, args, nargs, kwnames); } @@ -597,65 +611,79 @@ partial_repr(PyObject *self) { partialobject *pto = partialobject_CAST(self); PyObject *result = NULL; - PyObject *arglist; - PyObject *mod; - PyObject *name; + PyObject *arglist = NULL; + PyObject *mod = NULL; + PyObject *name = NULL; Py_ssize_t i, n; PyObject *key, *value; int status; status = Py_ReprEnter(self); if (status != 0) { - if (status < 0) + if (status < 0) { return NULL; + } return PyUnicode_FromString("..."); } + /* Reference arguments in case they change */ + PyObject *fn = Py_NewRef(pto->fn); + PyObject *args = Py_NewRef(pto->args); + PyObject *kw = Py_NewRef(pto->kw); + assert(PyTuple_Check(args)); + assert(PyDict_Check(kw)); arglist = Py_GetConstant(Py_CONSTANT_EMPTY_STR); - if (arglist == NULL) + if (arglist == NULL) { goto done; + } /* Pack positional arguments */ - assert(PyTuple_Check(pto->args)); - n = PyTuple_GET_SIZE(pto->args); + n = PyTuple_GET_SIZE(args); for (i = 0; i < n; i++) { Py_SETREF(arglist, PyUnicode_FromFormat("%U, %R", arglist, - PyTuple_GET_ITEM(pto->args, i))); - if (arglist == NULL) + PyTuple_GET_ITEM(args, i))); + if (arglist == NULL) { goto done; + } } /* Pack keyword arguments */ - assert (PyDict_Check(pto->kw)); - for (i = 0; PyDict_Next(pto->kw, &i, &key, &value);) { + int error = 0; + Py_BEGIN_CRITICAL_SECTION(kw); + for (i = 0; PyDict_Next(kw, &i, &key, &value);) { /* Prevent key.__str__ from deleting the value. */ Py_INCREF(value); Py_SETREF(arglist, PyUnicode_FromFormat("%U, %S=%R", arglist, key, value)); Py_DECREF(value); - if (arglist == NULL) - goto done; + if (arglist == NULL) { + error = 1; + break; + } + } + Py_END_CRITICAL_SECTION(); + if (error) { + goto done; } mod = PyType_GetModuleName(Py_TYPE(pto)); if (mod == NULL) { - goto error; + goto done; } + name = PyType_GetQualName(Py_TYPE(pto)); if (name == NULL) { - Py_DECREF(mod); - goto error; + goto done; } - result = PyUnicode_FromFormat("%S.%S(%R%U)", mod, name, pto->fn, arglist); - Py_DECREF(mod); - Py_DECREF(name); - Py_DECREF(arglist); - done: + result = PyUnicode_FromFormat("%S.%S(%R%U)", mod, name, fn, arglist); +done: + Py_XDECREF(name); + Py_XDECREF(mod); + Py_XDECREF(arglist); + Py_DECREF(fn); + Py_DECREF(args); + Py_DECREF(kw); Py_ReprLeave(self); return result; - error: - Py_DECREF(arglist); - Py_ReprLeave(self); - return NULL; } /* Pickle strategy: @@ -687,7 +715,8 @@ partial_setstate(PyObject *self, PyObject *state) if (!PyArg_ParseTuple(state, "OOOO", &fn, &fnargs, &kw, &dict) || !PyCallable_Check(fn) || !PyTuple_Check(fnargs) || - (kw != Py_None && !PyDict_Check(kw))) + (kw != Py_None && !PyDict_Check(kw)) || + (dict != Py_None && !PyDict_Check(dict))) { PyErr_SetString(PyExc_TypeError, "invalid partial state"); return NULL; @@ -742,7 +771,8 @@ static PyMethodDef partial_methods[] = { {"__reduce__", partial_reduce, METH_NOARGS}, {"__setstate__", partial_setstate, METH_O}, {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + PyDoc_STR("partial is generic over the wrapped function's return type")}, {NULL, NULL} /* sentinel */ }; @@ -950,9 +980,9 @@ _functools.reduce Apply a function of two arguments cumulatively to the items of an iterable, from left to right. -This effectively reduces the iterable to a single value. If initial is present, -it is placed before the items of the iterable in the calculation, and serves as -a default when the iterable is empty. +This effectively reduces the iterable to a single value. If initial is +present, it is placed before the items of the iterable in the +calculation, and serves as a default when the iterable is empty. For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates ((((1 + 2) + 3) + 4) + 5). @@ -961,7 +991,7 @@ calculates ((((1 + 2) + 3) + 4) + 5). static PyObject * _functools_reduce_impl(PyObject *module, PyObject *func, PyObject *seq, PyObject *result) -/*[clinic end generated code: output=30d898fe1267c79d input=1511e9a8c38581ac]*/ +/*[clinic end generated code: output=30d898fe1267c79d input=1e2c850f5229ff2a]*/ { PyObject *args, *it; @@ -983,7 +1013,7 @@ _functools_reduce_impl(PyObject *module, PyObject *func, PyObject *seq, for (;;) { PyObject *op2; - if (Py_REFCNT(args) > 1) { + if (!_PyObject_IsUniquelyReferenced(args)) { Py_DECREF(args); if ((args = PyTuple_New(2)) == NULL) goto Fail; @@ -1370,8 +1400,8 @@ bounded_lru_cache_update_lock_held(lru_cache_object *self, this same key, then this setitem call will update the cache dict with this new link, leaving the old link as an orphan (i.e. not having a cache dict entry that refers to it). */ - if (_PyDict_SetItem_KnownHash(self->cache, key, (PyObject *)link, - hash) < 0) { + if (_PyDict_SetItem_KnownHash_LockHeld((PyDictObject *)self->cache, key, + (PyObject *)link, hash) < 0) { Py_DECREF(link); return NULL; } @@ -1440,8 +1470,8 @@ bounded_lru_cache_update_lock_held(lru_cache_object *self, for successful insertion in the cache dict before adding the link to the linked list. Otherwise, the potentially reentrant __eq__ call could cause the then orphan link to be visited. */ - if (_PyDict_SetItem_KnownHash(self->cache, key, (PyObject *)link, - hash) < 0) { + if (_PyDict_SetItem_KnownHash_LockHeld((PyDictObject *)self->cache, key, + (PyObject *)link, hash) < 0) { /* Somehow the cache dict update failed. We no longer can restore the old link. Let the error propagate upward and leave the cache short one link. */ @@ -1608,9 +1638,7 @@ lru_cache_dealloc(PyObject *op) PyTypeObject *tp = Py_TYPE(obj); /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(obj); - if (obj->weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, obj->weakreflist); (void)lru_cache_tp_clear(op); tp->tp_free(obj); @@ -1676,7 +1704,13 @@ _functools__lru_cache_wrapper_cache_clear_impl(PyObject *self) lru_list_elem *list = lru_cache_unlink_list(_self); FT_ATOMIC_STORE_SSIZE_RELAXED(_self->hits, 0); FT_ATOMIC_STORE_SSIZE_RELAXED(_self->misses, 0); - PyDict_Clear(_self->cache); + if (_self->wrapper == bounded_lru_cache_wrapper) { + /* The critical section on the lru cache itself protects the dictionary + for bounded_lru_cache instances. */ + _PyDict_Clear_LockHeld(_self->cache); + } else { + PyDict_Clear(_self->cache); + } lru_cache_clear_list(list); Py_RETURN_NONE; } diff --git a/Modules/_gdbmmodule.c b/Modules/_gdbmmodule.c index ab2ebdba9249bf9..83f588af6275fa3 100644 --- a/Modules/_gdbmmodule.c +++ b/Modules/_gdbmmodule.c @@ -81,6 +81,7 @@ typedef struct { #include "clinic/_gdbmmodule.c.h" #define check_gdbmobject_open(v, err) \ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED((v)) \ if ((v)->di_dbm == NULL) { \ PyErr_SetString(err, "GDBM object has already been closed"); \ return NULL; \ @@ -142,7 +143,7 @@ gdbm_dealloc(PyObject *op) } static Py_ssize_t -gdbm_length(PyObject *op) +gdbm_length_lock_held(PyObject *op) { gdbmobject *dp = _gdbmobject_CAST(op); _gdbm_state *state = PyType_GetModuleState(Py_TYPE(dp)); @@ -188,8 +189,18 @@ gdbm_length(PyObject *op) return dp->di_size; } +static Py_ssize_t +gdbm_length(PyObject *op) +{ + Py_ssize_t result; + Py_BEGIN_CRITICAL_SECTION(op); + result = gdbm_length_lock_held(op); + Py_END_CRITICAL_SECTION(); + return result; +} + static int -gdbm_bool(PyObject *op) +gdbm_bool_lock_held(PyObject *op) { gdbmobject *dp = _gdbmobject_CAST(op); _gdbm_state *state = PyType_GetModuleState(Py_TYPE(dp)); @@ -218,6 +229,16 @@ gdbm_bool(PyObject *op) return 1; } +static int +gdbm_bool(PyObject *op) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(op); + result = gdbm_bool_lock_held(op); + Py_END_CRITICAL_SECTION(); + return result; +} + // Wrapper function for PyArg_Parse(o, "s#", &d.dptr, &d.size). // This function is needed to support PY_SSIZE_T_CLEAN. // Return 1 on success, same to PyArg_Parse(). @@ -240,7 +261,7 @@ parse_datum(PyObject *o, datum *d, const char *failmsg) } static PyObject * -gdbm_subscript(PyObject *op, PyObject *key) +gdbm_subscript_lock_held(PyObject *op, PyObject *key) { PyObject *v; datum drec, krec; @@ -265,6 +286,16 @@ gdbm_subscript(PyObject *op, PyObject *key) return v; } +static PyObject * +gdbm_subscript(PyObject *op, PyObject *key) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = gdbm_subscript_lock_held(op, key); + Py_END_CRITICAL_SECTION(); + return result; +} + /*[clinic input] _gdbm.gdbm.get @@ -290,7 +321,7 @@ _gdbm_gdbm_get_impl(gdbmobject *self, PyObject *key, PyObject *default_value) } static int -gdbm_ass_sub(PyObject *op, PyObject *v, PyObject *w) +gdbm_ass_sub_lock_held(PyObject *op, PyObject *v, PyObject *w) { datum krec, drec; const char *failmsg = "gdbm mappings have bytes or string indices only"; @@ -335,7 +366,18 @@ gdbm_ass_sub(PyObject *op, PyObject *v, PyObject *w) return 0; } +static int +gdbm_ass_sub(PyObject *op, PyObject *v, PyObject *w) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(op); + result = gdbm_ass_sub_lock_held(op, v, w); + Py_END_CRITICAL_SECTION(); + return result; +} + /*[clinic input] +@critical_section _gdbm.gdbm.setdefault key: object @@ -348,7 +390,7 @@ Get value for key, or set it to default and return default if not present. static PyObject * _gdbm_gdbm_setdefault_impl(gdbmobject *self, PyObject *key, PyObject *default_value) -/*[clinic end generated code: output=f3246e880509f142 input=0db46b69e9680171]*/ +/*[clinic end generated code: output=f3246e880509f142 input=854374cd81ab51b6]*/ { PyObject *res; @@ -363,6 +405,7 @@ _gdbm_gdbm_setdefault_impl(gdbmobject *self, PyObject *key, } /*[clinic input] +@critical_section _gdbm.gdbm.close Close the database. @@ -370,7 +413,7 @@ Close the database. static PyObject * _gdbm_gdbm_close_impl(gdbmobject *self) -/*[clinic end generated code: output=f5abb4d6bb9e52d5 input=0a203447379b45fd]*/ +/*[clinic end generated code: output=f5abb4d6bb9e52d5 input=56b604f4e77f533d]*/ { if (self->di_dbm) { gdbm_close(self->di_dbm); @@ -381,6 +424,7 @@ _gdbm_gdbm_close_impl(gdbmobject *self) /* XXX Should return a set or a set view */ /*[clinic input] +@critical_section _gdbm.gdbm.keys cls: defining_class @@ -390,7 +434,7 @@ Get a list of all keys in the database. static PyObject * _gdbm_gdbm_keys_impl(gdbmobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=c24b824e81404755 input=1428b7c79703d7d5]*/ +/*[clinic end generated code: output=c24b824e81404755 input=785988b1ea8f77e0]*/ { PyObject *v, *item; datum key, nextkey; @@ -432,7 +476,7 @@ _gdbm_gdbm_keys_impl(gdbmobject *self, PyTypeObject *cls) } static int -gdbm_contains(PyObject *self, PyObject *arg) +gdbm_contains_lock_held(PyObject *self, PyObject *arg) { gdbmobject *dp = (gdbmobject *)self; datum key; @@ -463,21 +507,32 @@ gdbm_contains(PyObject *self, PyObject *arg) return gdbm_exists(dp->di_dbm, key); } +static int +gdbm_contains(PyObject *self, PyObject *arg) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(self); + result = gdbm_contains_lock_held(self, arg); + Py_END_CRITICAL_SECTION(); + return result; +} + /*[clinic input] +@critical_section _gdbm.gdbm.firstkey cls: defining_class Return the starting key for the traversal. -It's possible to loop over every key in the database using this method -and the nextkey() method. The traversal is ordered by GDBM's internal -hash values, and won't be sorted by the key values. +It's possible to loop over every key in the database using this +method and the nextkey() method. The traversal is ordered by GDBM's +internal hash values, and won't be sorted by the key values. [clinic start generated code]*/ static PyObject * _gdbm_gdbm_firstkey_impl(gdbmobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=139275e9c8b60827 input=ed8782a029a5d299]*/ +/*[clinic end generated code: output=139275e9c8b60827 input=ba40f0d81eae0f35]*/ { PyObject *v; datum key; @@ -497,6 +552,7 @@ _gdbm_gdbm_firstkey_impl(gdbmobject *self, PyTypeObject *cls) } /*[clinic input] +@critical_section _gdbm.gdbm.nextkey cls: defining_class @@ -505,8 +561,8 @@ _gdbm.gdbm.nextkey Returns the key that follows key in the traversal. -The following code prints every key in the database db, without having -to create a list in memory that contains them all: +The following code prints every key in the database db, without +having to create a list in memory that contains them all: k = db.firstkey() while k is not None: @@ -517,7 +573,7 @@ to create a list in memory that contains them all: static PyObject * _gdbm_gdbm_nextkey_impl(gdbmobject *self, PyTypeObject *cls, const char *key, Py_ssize_t key_length) -/*[clinic end generated code: output=c81a69300ef41766 input=365e297bc0b3db48]*/ +/*[clinic end generated code: output=c81a69300ef41766 input=78293a913b02387e]*/ { PyObject *v; datum dbm_key, nextkey; @@ -539,6 +595,7 @@ _gdbm_gdbm_nextkey_impl(gdbmobject *self, PyTypeObject *cls, const char *key, } /*[clinic input] +@critical_section _gdbm.gdbm.reorganize cls: defining_class @@ -547,14 +604,14 @@ Reorganize the database. If you have carried out a lot of deletions and would like to shrink the space used by the GDBM file, this routine will reorganize the -database. GDBM will not shorten the length of a database file except -by using this reorganization; otherwise, deleted file space will be -kept and reused as new (key,value) pairs are added. +database. GDBM will not shorten the length of a database file +except by using this reorganization; otherwise, deleted file space +will be kept and reused as new (key,value) pairs are added. [clinic start generated code]*/ static PyObject * _gdbm_gdbm_reorganize_impl(gdbmobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=d77c69e8e3dd644a input=e1359faeef844e46]*/ +/*[clinic end generated code: output=d77c69e8e3dd644a input=d7fcf03051c6f7cd]*/ { _gdbm_state *state = PyType_GetModuleState(cls); assert(state != NULL); @@ -573,6 +630,7 @@ _gdbm_gdbm_reorganize_impl(gdbmobject *self, PyTypeObject *cls) } /*[clinic input] +@critical_section _gdbm.gdbm.sync cls: defining_class @@ -585,7 +643,7 @@ any unwritten data to be written to the disk. static PyObject * _gdbm_gdbm_sync_impl(gdbmobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=bb680a2035c3f592 input=3d749235f79b6f2a]*/ +/*[clinic end generated code: output=bb680a2035c3f592 input=6054385b071d238a]*/ { _gdbm_state *state = PyType_GetModuleState(cls); assert(state != NULL); @@ -595,6 +653,7 @@ _gdbm_gdbm_sync_impl(gdbmobject *self, PyTypeObject *cls) } /*[clinic input] +@critical_section _gdbm.gdbm.clear cls: defining_class / @@ -604,7 +663,7 @@ Remove all items from the database. static PyObject * _gdbm_gdbm_clear_impl(gdbmobject *self, PyTypeObject *cls) -/*[clinic end generated code: output=673577c573318661 input=34136d52fcdd4210]*/ +/*[clinic end generated code: output=673577c573318661 input=b17467adfe62f23d]*/ { _gdbm_state *state = PyType_GetModuleState(cls); assert(state != NULL); @@ -619,8 +678,10 @@ _gdbm_gdbm_clear_impl(gdbmobject *self, PyTypeObject *cls) } if (gdbm_delete(self->di_dbm, key) < 0) { PyErr_SetString(state->gdbm_error, "cannot delete item from database"); + free(key.dptr); return NULL; } + free(key.dptr); } Py_RETURN_NONE; } @@ -634,7 +695,11 @@ gdbm__enter__(PyObject *self, PyObject *args) static PyObject * gdbm__exit__(PyObject *self, PyObject *args) { - return _gdbm_gdbm_close_impl((gdbmobject *)self); + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(self); + result = _gdbm_gdbm_close_impl((gdbmobject *)self); + Py_END_CRITICAL_SECTION(); + return result; } static PyMethodDef gdbm_methods[] = { diff --git a/Modules/_hacl/Hacl_HMAC.h b/Modules/_hacl/Hacl_HMAC.h index 10ff15183f2834a..7dca53c92545469 100644 --- a/Modules/_hacl/Hacl_HMAC.h +++ b/Modules/_hacl/Hacl_HMAC.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_HMAC_H -#define __Hacl_HMAC_H +#ifndef Hacl_HMAC_H +#define Hacl_HMAC_H #if defined(__cplusplus) extern "C" { @@ -220,5 +220,5 @@ Hacl_HMAC_compute_blake2b_32( } #endif -#define __Hacl_HMAC_H_DEFINED -#endif +#define Hacl_HMAC_H_DEFINED +#endif /* Hacl_HMAC_H */ diff --git a/Modules/_hacl/Hacl_Hash_Blake2b.c b/Modules/_hacl/Hacl_Hash_Blake2b.c index 21ab2b88c799a6d..a5b75d61798949c 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2b.c +++ b/Modules/_hacl/Hacl_Hash_Blake2b.c @@ -544,11 +544,9 @@ void Hacl_Hash_Blake2b_init(uint64_t *hash, uint32_t kk, uint32_t nn) uint64_t x = r; os[i] = x;); tmp[0U] = - (uint64_t)nn1 - ^ - ((uint64_t)kk1 - << 8U - ^ ((uint64_t)p.fanout << 16U ^ ((uint64_t)p.depth << 24U ^ (uint64_t)p.leaf_length << 32U))); + (uint64_t)nn1 ^ + ((uint64_t)kk1 << 8U ^ + ((uint64_t)p.fanout << 16U ^ ((uint64_t)p.depth << 24U ^ (uint64_t)p.leaf_length << 32U))); tmp[1U] = p.node_offset; tmp[2U] = (uint64_t)p.node_depth ^ (uint64_t)p.inner_length << 8U; tmp[3U] = 0ULL; @@ -860,14 +858,10 @@ static Hacl_Hash_Blake2b_state_t uint64_t x = r4; os[i0] = x;); tmp[0U] = - (uint64_t)nn1 - ^ - ((uint64_t)kk2 - << 8U - ^ - ((uint64_t)pv.fanout - << 16U - ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); + (uint64_t)nn1 ^ + ((uint64_t)kk2 << 8U ^ + ((uint64_t)pv.fanout << 16U ^ + ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); tmp[1U] = pv.node_offset; tmp[2U] = (uint64_t)pv.node_depth ^ (uint64_t)pv.inner_length << 8U; tmp[3U] = 0ULL; @@ -1059,11 +1053,9 @@ static void reset_raw(Hacl_Hash_Blake2b_state_t *state, Hacl_Hash_Blake2b_params uint64_t x = r; os[i0] = x;); tmp[0U] = - (uint64_t)nn1 - ^ - ((uint64_t)kk2 - << 8U - ^ ((uint64_t)pv.fanout << 16U ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); + (uint64_t)nn1 ^ + ((uint64_t)kk2 << 8U ^ + ((uint64_t)pv.fanout << 16U ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); tmp[1U] = pv.node_offset; tmp[2U] = (uint64_t)pv.node_depth ^ (uint64_t)pv.inner_length << 8U; tmp[3U] = 0ULL; @@ -1200,8 +1192,7 @@ Hacl_Hash_Blake2b_update(Hacl_Hash_Blake2b_state_t *state, uint8_t *chunk, uint3 uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Hash_Blake2b_state_t){ .block_state = block_state1, @@ -1265,8 +1256,7 @@ Hacl_Hash_Blake2b_update(Hacl_Hash_Blake2b_state_t *state, uint8_t *chunk, uint3 nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2b_state_t){ .block_state = block_state1, @@ -1296,8 +1286,7 @@ Hacl_Hash_Blake2b_update(Hacl_Hash_Blake2b_state_t *state, uint8_t *chunk, uint3 uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Hash_Blake2b_state_t){ .block_state = block_state10, @@ -1359,8 +1348,7 @@ Hacl_Hash_Blake2b_update(Hacl_Hash_Blake2b_state_t *state, uint8_t *chunk, uint3 nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2b_state_t){ .block_state = block_state1, @@ -1690,14 +1678,10 @@ Hacl_Hash_Blake2b_hash_with_key_and_params( uint64_t x = r; os[i] = x;); tmp[0U] = - (uint64_t)nn - ^ - ((uint64_t)kk - << 8U - ^ - ((uint64_t)params.fanout - << 16U - ^ ((uint64_t)params.depth << 24U ^ (uint64_t)params.leaf_length << 32U))); + (uint64_t)nn ^ + ((uint64_t)kk << 8U ^ + ((uint64_t)params.fanout << 16U ^ + ((uint64_t)params.depth << 24U ^ (uint64_t)params.leaf_length << 32U))); tmp[1U] = params.node_offset; tmp[2U] = (uint64_t)params.node_depth ^ (uint64_t)params.inner_length << 8U; tmp[3U] = 0ULL; diff --git a/Modules/_hacl/Hacl_Hash_Blake2b.h b/Modules/_hacl/Hacl_Hash_Blake2b.h index 3a73f358c98cc34..b07893ba339245a 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2b.h +++ b/Modules/_hacl/Hacl_Hash_Blake2b.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_Hash_Blake2b_H -#define __Hacl_Hash_Blake2b_H +#ifndef Hacl_Hash_Blake2b_H +#define Hacl_Hash_Blake2b_H #if defined(__cplusplus) extern "C" { @@ -220,5 +220,5 @@ Hacl_Hash_Blake2b_hash_with_key_and_params( } #endif -#define __Hacl_Hash_Blake2b_H_DEFINED -#endif +#define Hacl_Hash_Blake2b_H_DEFINED +#endif /* Hacl_Hash_Blake2b_H */ diff --git a/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c index c4d9b4a689d28f7..f955f1c6115b1e6 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c +++ b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.c @@ -274,11 +274,9 @@ Hacl_Hash_Blake2b_Simd256_init(Lib_IntVector_Intrinsics_vec256 *hash, uint32_t k uint64_t x = r; os[i] = x;); tmp[0U] = - (uint64_t)nn1 - ^ - ((uint64_t)kk1 - << 8U - ^ ((uint64_t)p.fanout << 16U ^ ((uint64_t)p.depth << 24U ^ (uint64_t)p.leaf_length << 32U))); + (uint64_t)nn1 ^ + ((uint64_t)kk1 << 8U ^ + ((uint64_t)p.fanout << 16U ^ ((uint64_t)p.depth << 24U ^ (uint64_t)p.leaf_length << 32U))); tmp[1U] = p.node_offset; tmp[2U] = (uint64_t)p.node_depth ^ (uint64_t)p.inner_length << 8U; tmp[3U] = 0ULL; @@ -746,14 +744,10 @@ static Hacl_Hash_Blake2b_Simd256_state_t uint64_t x = r4; os[i0] = x;); tmp[0U] = - (uint64_t)nn1 - ^ - ((uint64_t)kk2 - << 8U - ^ - ((uint64_t)pv.fanout - << 16U - ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); + (uint64_t)nn1 ^ + ((uint64_t)kk2 << 8U ^ + ((uint64_t)pv.fanout << 16U ^ + ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); tmp[1U] = pv.node_offset; tmp[2U] = (uint64_t)pv.node_depth ^ (uint64_t)pv.inner_length << 8U; tmp[3U] = 0ULL; @@ -936,11 +930,9 @@ reset_raw(Hacl_Hash_Blake2b_Simd256_state_t *state, Hacl_Hash_Blake2b_params_and uint64_t x = r; os[i0] = x;); tmp[0U] = - (uint64_t)nn1 - ^ - ((uint64_t)kk2 - << 8U - ^ ((uint64_t)pv.fanout << 16U ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); + (uint64_t)nn1 ^ + ((uint64_t)kk2 << 8U ^ + ((uint64_t)pv.fanout << 16U ^ ((uint64_t)pv.depth << 24U ^ (uint64_t)pv.leaf_length << 32U))); tmp[1U] = pv.node_offset; tmp[2U] = (uint64_t)pv.node_depth ^ (uint64_t)pv.inner_length << 8U; tmp[3U] = 0ULL; @@ -1075,8 +1067,7 @@ Hacl_Hash_Blake2b_Simd256_update( uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Hash_Blake2b_Simd256_state_t){ .block_state = block_state1, @@ -1140,8 +1131,7 @@ Hacl_Hash_Blake2b_Simd256_update( nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2b_Simd256_state_t){ .block_state = block_state1, @@ -1171,8 +1161,7 @@ Hacl_Hash_Blake2b_Simd256_update( uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Hash_Blake2b_Simd256_state_t){ .block_state = block_state10, @@ -1234,8 +1223,7 @@ Hacl_Hash_Blake2b_Simd256_update( nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2b_Simd256_state_t){ .block_state = block_state1, @@ -1578,14 +1566,10 @@ Hacl_Hash_Blake2b_Simd256_hash_with_key_and_params( uint64_t x = r; os[i] = x;); tmp[0U] = - (uint64_t)nn - ^ - ((uint64_t)kk - << 8U - ^ - ((uint64_t)params.fanout - << 16U - ^ ((uint64_t)params.depth << 24U ^ (uint64_t)params.leaf_length << 32U))); + (uint64_t)nn ^ + ((uint64_t)kk << 8U ^ + ((uint64_t)params.fanout << 16U ^ + ((uint64_t)params.depth << 24U ^ (uint64_t)params.leaf_length << 32U))); tmp[1U] = params.node_offset; tmp[2U] = (uint64_t)params.node_depth ^ (uint64_t)params.inner_length << 8U; tmp[3U] = 0ULL; diff --git a/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.h b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.h index 0271ab8e024e516..8be8f32db62b151 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.h +++ b/Modules/_hacl/Hacl_Hash_Blake2b_Simd256.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_Hash_Blake2b_Simd256_H -#define __Hacl_Hash_Blake2b_Simd256_H +#ifndef Hacl_Hash_Blake2b_Simd256_H +#define Hacl_Hash_Blake2b_Simd256_H #if defined(__cplusplus) extern "C" { @@ -206,5 +206,5 @@ Hacl_Hash_Blake2b_Simd256_hash_with_key_and_params( } #endif -#define __Hacl_Hash_Blake2b_Simd256_H_DEFINED -#endif +#define Hacl_Hash_Blake2b_Simd256_H_DEFINED +#endif /* Hacl_Hash_Blake2b_Simd256_H */ diff --git a/Modules/_hacl/Hacl_Hash_Blake2s.c b/Modules/_hacl/Hacl_Hash_Blake2s.c index 730ba135afb2fb3..0d4fcc7f3951fce 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2s.c +++ b/Modules/_hacl/Hacl_Hash_Blake2s.c @@ -543,13 +543,13 @@ void Hacl_Hash_Blake2s_init(uint32_t *hash, uint32_t kk, uint32_t nn) uint32_t x = r; os[i] = x;); tmp[0U] = - (uint32_t)(uint8_t)nn - ^ ((uint32_t)(uint8_t)kk << 8U ^ ((uint32_t)p.fanout << 16U ^ (uint32_t)p.depth << 24U)); + (uint32_t)(uint8_t)nn ^ + ((uint32_t)(uint8_t)kk << 8U ^ ((uint32_t)p.fanout << 16U ^ (uint32_t)p.depth << 24U)); tmp[1U] = p.leaf_length; tmp[2U] = (uint32_t)p.node_offset; tmp[3U] = - (uint32_t)(p.node_offset >> 32U) - ^ ((uint32_t)p.node_depth << 16U ^ (uint32_t)p.inner_length << 24U); + (uint32_t)(p.node_offset >> 32U) ^ + ((uint32_t)p.node_depth << 16U ^ (uint32_t)p.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; @@ -846,16 +846,14 @@ static Hacl_Hash_Blake2s_state_t uint32_t x = r4; os[i0] = x;); tmp[0U] = - (uint32_t)pv.digest_length - ^ - ((uint32_t)pv.key_length - << 8U - ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); + (uint32_t)pv.digest_length ^ + ((uint32_t)pv.key_length << 8U ^ + ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); tmp[1U] = pv.leaf_length; tmp[2U] = (uint32_t)pv.node_offset; tmp[3U] = - (uint32_t)(pv.node_offset >> 32U) - ^ ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); + (uint32_t)(pv.node_offset >> 32U) ^ + ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; @@ -1042,13 +1040,13 @@ static void reset_raw(Hacl_Hash_Blake2s_state_t *state, Hacl_Hash_Blake2b_params uint32_t x = r; os[i0] = x;); tmp[0U] = - (uint32_t)pv.digest_length - ^ ((uint32_t)pv.key_length << 8U ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); + (uint32_t)pv.digest_length ^ + ((uint32_t)pv.key_length << 8U ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); tmp[1U] = pv.leaf_length; tmp[2U] = (uint32_t)pv.node_offset; tmp[3U] = - (uint32_t)(pv.node_offset >> 32U) - ^ ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); + (uint32_t)(pv.node_offset >> 32U) ^ + ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; @@ -1182,8 +1180,7 @@ Hacl_Hash_Blake2s_update(Hacl_Hash_Blake2s_state_t *state, uint8_t *chunk, uint3 uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Hash_Blake2s_state_t){ .block_state = block_state1, @@ -1237,8 +1234,7 @@ Hacl_Hash_Blake2s_update(Hacl_Hash_Blake2s_state_t *state, uint8_t *chunk, uint3 Hacl_Hash_Blake2s_update_multi(data1_len, wv, hash, total_len1, data1, nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2s_state_t){ .block_state = block_state1, @@ -1268,8 +1264,7 @@ Hacl_Hash_Blake2s_update(Hacl_Hash_Blake2s_state_t *state, uint8_t *chunk, uint3 uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Hash_Blake2s_state_t){ .block_state = block_state10, @@ -1321,8 +1316,7 @@ Hacl_Hash_Blake2s_update(Hacl_Hash_Blake2s_state_t *state, uint8_t *chunk, uint3 Hacl_Hash_Blake2s_update_multi(data1_len, wv, hash, total_len1, data1, nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2s_state_t){ .block_state = block_state1, @@ -1639,16 +1633,14 @@ Hacl_Hash_Blake2s_hash_with_key_and_params( uint32_t x = r; os[i] = x;); tmp[0U] = - (uint32_t)params.digest_length - ^ - ((uint32_t)params.key_length - << 8U - ^ ((uint32_t)params.fanout << 16U ^ (uint32_t)params.depth << 24U)); + (uint32_t)params.digest_length ^ + ((uint32_t)params.key_length << 8U ^ + ((uint32_t)params.fanout << 16U ^ (uint32_t)params.depth << 24U)); tmp[1U] = params.leaf_length; tmp[2U] = (uint32_t)params.node_offset; tmp[3U] = - (uint32_t)(params.node_offset >> 32U) - ^ ((uint32_t)params.node_depth << 16U ^ (uint32_t)params.inner_length << 24U); + (uint32_t)(params.node_offset >> 32U) ^ + ((uint32_t)params.node_depth << 16U ^ (uint32_t)params.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; diff --git a/Modules/_hacl/Hacl_Hash_Blake2s.h b/Modules/_hacl/Hacl_Hash_Blake2s.h index fbf8cff5cd10732..2582a3d34e34fce 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2s.h +++ b/Modules/_hacl/Hacl_Hash_Blake2s.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_Hash_Blake2s_H -#define __Hacl_Hash_Blake2s_H +#ifndef Hacl_Hash_Blake2s_H +#define Hacl_Hash_Blake2s_H #if defined(__cplusplus) extern "C" { @@ -198,5 +198,5 @@ Hacl_Hash_Blake2s_hash_with_key_and_params( } #endif -#define __Hacl_Hash_Blake2s_H_DEFINED -#endif +#define Hacl_Hash_Blake2s_H_DEFINED +#endif /* Hacl_Hash_Blake2s_H */ diff --git a/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c index 7e9cd79544f8f10..fa46be045f34412 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c +++ b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.c @@ -271,13 +271,13 @@ Hacl_Hash_Blake2s_Simd128_init(Lib_IntVector_Intrinsics_vec128 *hash, uint32_t k uint32_t x = r; os[i] = x;); tmp[0U] = - (uint32_t)(uint8_t)nn - ^ ((uint32_t)(uint8_t)kk << 8U ^ ((uint32_t)p.fanout << 16U ^ (uint32_t)p.depth << 24U)); + (uint32_t)(uint8_t)nn ^ + ((uint32_t)(uint8_t)kk << 8U ^ ((uint32_t)p.fanout << 16U ^ (uint32_t)p.depth << 24U)); tmp[1U] = p.leaf_length; tmp[2U] = (uint32_t)p.node_offset; tmp[3U] = - (uint32_t)(p.node_offset >> 32U) - ^ ((uint32_t)p.node_depth << 16U ^ (uint32_t)p.inner_length << 24U); + (uint32_t)(p.node_offset >> 32U) ^ + ((uint32_t)p.node_depth << 16U ^ (uint32_t)p.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; @@ -736,16 +736,14 @@ static Hacl_Hash_Blake2s_Simd128_state_t uint32_t x = r4; os[i0] = x;); tmp[0U] = - (uint32_t)pv.digest_length - ^ - ((uint32_t)pv.key_length - << 8U - ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); + (uint32_t)pv.digest_length ^ + ((uint32_t)pv.key_length << 8U ^ + ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); tmp[1U] = pv.leaf_length; tmp[2U] = (uint32_t)pv.node_offset; tmp[3U] = - (uint32_t)(pv.node_offset >> 32U) - ^ ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); + (uint32_t)(pv.node_offset >> 32U) ^ + ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; @@ -923,13 +921,13 @@ reset_raw(Hacl_Hash_Blake2s_Simd128_state_t *state, Hacl_Hash_Blake2b_params_and uint32_t x = r; os[i0] = x;); tmp[0U] = - (uint32_t)pv.digest_length - ^ ((uint32_t)pv.key_length << 8U ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); + (uint32_t)pv.digest_length ^ + ((uint32_t)pv.key_length << 8U ^ ((uint32_t)pv.fanout << 16U ^ (uint32_t)pv.depth << 24U)); tmp[1U] = pv.leaf_length; tmp[2U] = (uint32_t)pv.node_offset; tmp[3U] = - (uint32_t)(pv.node_offset >> 32U) - ^ ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); + (uint32_t)(pv.node_offset >> 32U) ^ + ((uint32_t)pv.node_depth << 16U ^ (uint32_t)pv.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; @@ -1061,8 +1059,7 @@ Hacl_Hash_Blake2s_Simd128_update( uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Hash_Blake2s_Simd128_state_t){ .block_state = block_state1, @@ -1116,8 +1113,7 @@ Hacl_Hash_Blake2s_Simd128_update( Hacl_Hash_Blake2s_Simd128_update_multi(data1_len, wv, hash, total_len1, data1, nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2s_Simd128_state_t){ .block_state = block_state1, @@ -1147,8 +1143,7 @@ Hacl_Hash_Blake2s_Simd128_update( uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Hash_Blake2s_Simd128_state_t){ .block_state = block_state10, @@ -1200,8 +1195,7 @@ Hacl_Hash_Blake2s_Simd128_update( Hacl_Hash_Blake2s_Simd128_update_multi(data1_len, wv, hash, total_len1, data1, nb); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_Blake2s_Simd128_state_t){ .block_state = block_state1, @@ -1531,16 +1525,14 @@ Hacl_Hash_Blake2s_Simd128_hash_with_key_and_params( uint32_t x = r; os[i] = x;); tmp[0U] = - (uint32_t)params.digest_length - ^ - ((uint32_t)params.key_length - << 8U - ^ ((uint32_t)params.fanout << 16U ^ (uint32_t)params.depth << 24U)); + (uint32_t)params.digest_length ^ + ((uint32_t)params.key_length << 8U ^ + ((uint32_t)params.fanout << 16U ^ (uint32_t)params.depth << 24U)); tmp[1U] = params.leaf_length; tmp[2U] = (uint32_t)params.node_offset; tmp[3U] = - (uint32_t)(params.node_offset >> 32U) - ^ ((uint32_t)params.node_depth << 16U ^ (uint32_t)params.inner_length << 24U); + (uint32_t)(params.node_offset >> 32U) ^ + ((uint32_t)params.node_depth << 16U ^ (uint32_t)params.inner_length << 24U); uint32_t tmp0 = tmp[0U]; uint32_t tmp1 = tmp[1U]; uint32_t tmp2 = tmp[2U]; diff --git a/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.h b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.h index 56456e6fbb3df89..4b3b4e4fbb56307 100644 --- a/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.h +++ b/Modules/_hacl/Hacl_Hash_Blake2s_Simd128.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_Hash_Blake2s_Simd128_H -#define __Hacl_Hash_Blake2s_Simd128_H +#ifndef Hacl_Hash_Blake2s_Simd128_H +#define Hacl_Hash_Blake2s_Simd128_H #if defined(__cplusplus) extern "C" { @@ -206,5 +206,5 @@ Hacl_Hash_Blake2s_Simd128_hash_with_key_and_params( } #endif -#define __Hacl_Hash_Blake2s_Simd128_H_DEFINED -#endif +#define Hacl_Hash_Blake2s_Simd128_H_DEFINED +#endif /* Hacl_Hash_Blake2s_Simd128_H */ diff --git a/Modules/_hacl/Hacl_Hash_MD5.c b/Modules/_hacl/Hacl_Hash_MD5.c index 75ce8d2926e6e10..305c75483c06ea8 100644 --- a/Modules/_hacl/Hacl_Hash_MD5.c +++ b/Modules/_hacl/Hacl_Hash_MD5.c @@ -66,11 +66,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti0 = _t[0U]; uint32_t v = - vb0 - + - ((va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) - << 7U - | (va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) >> 25U); + vb0 + + ((va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) << 7U | + (va + ((vb0 & vc0) | (~vb0 & vd0)) + xk + ti0) >> 25U); abcd[0U] = v; uint32_t va0 = abcd[3U]; uint32_t vb1 = abcd[0U]; @@ -82,11 +80,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti1 = _t[1U]; uint32_t v0 = - vb1 - + - ((va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) - << 12U - | (va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) >> 20U); + vb1 + + ((va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) << 12U | + (va0 + ((vb1 & vc1) | (~vb1 & vd1)) + xk0 + ti1) >> 20U); abcd[3U] = v0; uint32_t va1 = abcd[2U]; uint32_t vb2 = abcd[3U]; @@ -98,11 +94,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti2 = _t[2U]; uint32_t v1 = - vb2 - + - ((va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) - << 17U - | (va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) >> 15U); + vb2 + + ((va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) << 17U | + (va1 + ((vb2 & vc2) | (~vb2 & vd2)) + xk1 + ti2) >> 15U); abcd[2U] = v1; uint32_t va2 = abcd[1U]; uint32_t vb3 = abcd[2U]; @@ -114,11 +108,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti3 = _t[3U]; uint32_t v2 = - vb3 - + - ((va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) - << 22U - | (va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) >> 10U); + vb3 + + ((va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) << 22U | + (va2 + ((vb3 & vc3) | (~vb3 & vd3)) + xk2 + ti3) >> 10U); abcd[1U] = v2; uint32_t va3 = abcd[0U]; uint32_t vb4 = abcd[1U]; @@ -130,11 +122,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti4 = _t[4U]; uint32_t v3 = - vb4 - + - ((va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) - << 7U - | (va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) >> 25U); + vb4 + + ((va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) << 7U | + (va3 + ((vb4 & vc4) | (~vb4 & vd4)) + xk3 + ti4) >> 25U); abcd[0U] = v3; uint32_t va4 = abcd[3U]; uint32_t vb5 = abcd[0U]; @@ -146,11 +136,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti5 = _t[5U]; uint32_t v4 = - vb5 - + - ((va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) - << 12U - | (va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) >> 20U); + vb5 + + ((va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) << 12U | + (va4 + ((vb5 & vc5) | (~vb5 & vd5)) + xk4 + ti5) >> 20U); abcd[3U] = v4; uint32_t va5 = abcd[2U]; uint32_t vb6 = abcd[3U]; @@ -162,11 +150,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti6 = _t[6U]; uint32_t v5 = - vb6 - + - ((va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) - << 17U - | (va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) >> 15U); + vb6 + + ((va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) << 17U | + (va5 + ((vb6 & vc6) | (~vb6 & vd6)) + xk5 + ti6) >> 15U); abcd[2U] = v5; uint32_t va6 = abcd[1U]; uint32_t vb7 = abcd[2U]; @@ -178,11 +164,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti7 = _t[7U]; uint32_t v6 = - vb7 - + - ((va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) - << 22U - | (va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) >> 10U); + vb7 + + ((va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) << 22U | + (va6 + ((vb7 & vc7) | (~vb7 & vd7)) + xk6 + ti7) >> 10U); abcd[1U] = v6; uint32_t va7 = abcd[0U]; uint32_t vb8 = abcd[1U]; @@ -194,11 +178,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti8 = _t[8U]; uint32_t v7 = - vb8 - + - ((va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) - << 7U - | (va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) >> 25U); + vb8 + + ((va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) << 7U | + (va7 + ((vb8 & vc8) | (~vb8 & vd8)) + xk7 + ti8) >> 25U); abcd[0U] = v7; uint32_t va8 = abcd[3U]; uint32_t vb9 = abcd[0U]; @@ -210,11 +192,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti9 = _t[9U]; uint32_t v8 = - vb9 - + - ((va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) - << 12U - | (va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) >> 20U); + vb9 + + ((va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) << 12U | + (va8 + ((vb9 & vc9) | (~vb9 & vd9)) + xk8 + ti9) >> 20U); abcd[3U] = v8; uint32_t va9 = abcd[2U]; uint32_t vb10 = abcd[3U]; @@ -226,11 +206,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti10 = _t[10U]; uint32_t v9 = - vb10 - + - ((va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) - << 17U - | (va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) >> 15U); + vb10 + + ((va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) << 17U | + (va9 + ((vb10 & vc10) | (~vb10 & vd10)) + xk9 + ti10) >> 15U); abcd[2U] = v9; uint32_t va10 = abcd[1U]; uint32_t vb11 = abcd[2U]; @@ -242,11 +220,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti11 = _t[11U]; uint32_t v10 = - vb11 - + - ((va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) - << 22U - | (va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) >> 10U); + vb11 + + ((va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) << 22U | + (va10 + ((vb11 & vc11) | (~vb11 & vd11)) + xk10 + ti11) >> 10U); abcd[1U] = v10; uint32_t va11 = abcd[0U]; uint32_t vb12 = abcd[1U]; @@ -258,11 +234,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti12 = _t[12U]; uint32_t v11 = - vb12 - + - ((va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) - << 7U - | (va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) >> 25U); + vb12 + + ((va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) << 7U | + (va11 + ((vb12 & vc12) | (~vb12 & vd12)) + xk11 + ti12) >> 25U); abcd[0U] = v11; uint32_t va12 = abcd[3U]; uint32_t vb13 = abcd[0U]; @@ -274,11 +248,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti13 = _t[13U]; uint32_t v12 = - vb13 - + - ((va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) - << 12U - | (va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) >> 20U); + vb13 + + ((va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) << 12U | + (va12 + ((vb13 & vc13) | (~vb13 & vd13)) + xk12 + ti13) >> 20U); abcd[3U] = v12; uint32_t va13 = abcd[2U]; uint32_t vb14 = abcd[3U]; @@ -290,11 +262,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti14 = _t[14U]; uint32_t v13 = - vb14 - + - ((va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) - << 17U - | (va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) >> 15U); + vb14 + + ((va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) << 17U | + (va13 + ((vb14 & vc14) | (~vb14 & vd14)) + xk13 + ti14) >> 15U); abcd[2U] = v13; uint32_t va14 = abcd[1U]; uint32_t vb15 = abcd[2U]; @@ -306,11 +276,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti15 = _t[15U]; uint32_t v14 = - vb15 - + - ((va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) - << 22U - | (va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) >> 10U); + vb15 + + ((va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) << 22U | + (va14 + ((vb15 & vc15) | (~vb15 & vd15)) + xk14 + ti15) >> 10U); abcd[1U] = v14; uint32_t va15 = abcd[0U]; uint32_t vb16 = abcd[1U]; @@ -322,11 +290,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti16 = _t[16U]; uint32_t v15 = - vb16 - + - ((va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) - << 5U - | (va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) >> 27U); + vb16 + + ((va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) << 5U | + (va15 + ((vb16 & vd16) | (vc16 & ~vd16)) + xk15 + ti16) >> 27U); abcd[0U] = v15; uint32_t va16 = abcd[3U]; uint32_t vb17 = abcd[0U]; @@ -338,11 +304,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti17 = _t[17U]; uint32_t v16 = - vb17 - + - ((va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) - << 9U - | (va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) >> 23U); + vb17 + + ((va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) << 9U | + (va16 + ((vb17 & vd17) | (vc17 & ~vd17)) + xk16 + ti17) >> 23U); abcd[3U] = v16; uint32_t va17 = abcd[2U]; uint32_t vb18 = abcd[3U]; @@ -354,11 +318,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti18 = _t[18U]; uint32_t v17 = - vb18 - + - ((va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) - << 14U - | (va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) >> 18U); + vb18 + + ((va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) << 14U | + (va17 + ((vb18 & vd18) | (vc18 & ~vd18)) + xk17 + ti18) >> 18U); abcd[2U] = v17; uint32_t va18 = abcd[1U]; uint32_t vb19 = abcd[2U]; @@ -370,11 +332,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti19 = _t[19U]; uint32_t v18 = - vb19 - + - ((va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) - << 20U - | (va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) >> 12U); + vb19 + + ((va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) << 20U | + (va18 + ((vb19 & vd19) | (vc19 & ~vd19)) + xk18 + ti19) >> 12U); abcd[1U] = v18; uint32_t va19 = abcd[0U]; uint32_t vb20 = abcd[1U]; @@ -386,11 +346,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti20 = _t[20U]; uint32_t v19 = - vb20 - + - ((va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) - << 5U - | (va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) >> 27U); + vb20 + + ((va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) << 5U | + (va19 + ((vb20 & vd20) | (vc20 & ~vd20)) + xk19 + ti20) >> 27U); abcd[0U] = v19; uint32_t va20 = abcd[3U]; uint32_t vb21 = abcd[0U]; @@ -402,11 +360,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti21 = _t[21U]; uint32_t v20 = - vb21 - + - ((va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) - << 9U - | (va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) >> 23U); + vb21 + + ((va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) << 9U | + (va20 + ((vb21 & vd21) | (vc21 & ~vd21)) + xk20 + ti21) >> 23U); abcd[3U] = v20; uint32_t va21 = abcd[2U]; uint32_t vb22 = abcd[3U]; @@ -418,11 +374,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti22 = _t[22U]; uint32_t v21 = - vb22 - + - ((va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) - << 14U - | (va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) >> 18U); + vb22 + + ((va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) << 14U | + (va21 + ((vb22 & vd22) | (vc22 & ~vd22)) + xk21 + ti22) >> 18U); abcd[2U] = v21; uint32_t va22 = abcd[1U]; uint32_t vb23 = abcd[2U]; @@ -434,11 +388,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti23 = _t[23U]; uint32_t v22 = - vb23 - + - ((va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) - << 20U - | (va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) >> 12U); + vb23 + + ((va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) << 20U | + (va22 + ((vb23 & vd23) | (vc23 & ~vd23)) + xk22 + ti23) >> 12U); abcd[1U] = v22; uint32_t va23 = abcd[0U]; uint32_t vb24 = abcd[1U]; @@ -450,11 +402,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti24 = _t[24U]; uint32_t v23 = - vb24 - + - ((va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) - << 5U - | (va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) >> 27U); + vb24 + + ((va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) << 5U | + (va23 + ((vb24 & vd24) | (vc24 & ~vd24)) + xk23 + ti24) >> 27U); abcd[0U] = v23; uint32_t va24 = abcd[3U]; uint32_t vb25 = abcd[0U]; @@ -466,11 +416,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti25 = _t[25U]; uint32_t v24 = - vb25 - + - ((va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) - << 9U - | (va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) >> 23U); + vb25 + + ((va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) << 9U | + (va24 + ((vb25 & vd25) | (vc25 & ~vd25)) + xk24 + ti25) >> 23U); abcd[3U] = v24; uint32_t va25 = abcd[2U]; uint32_t vb26 = abcd[3U]; @@ -482,11 +430,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti26 = _t[26U]; uint32_t v25 = - vb26 - + - ((va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) - << 14U - | (va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) >> 18U); + vb26 + + ((va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) << 14U | + (va25 + ((vb26 & vd26) | (vc26 & ~vd26)) + xk25 + ti26) >> 18U); abcd[2U] = v25; uint32_t va26 = abcd[1U]; uint32_t vb27 = abcd[2U]; @@ -498,11 +444,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti27 = _t[27U]; uint32_t v26 = - vb27 - + - ((va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) - << 20U - | (va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) >> 12U); + vb27 + + ((va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) << 20U | + (va26 + ((vb27 & vd27) | (vc27 & ~vd27)) + xk26 + ti27) >> 12U); abcd[1U] = v26; uint32_t va27 = abcd[0U]; uint32_t vb28 = abcd[1U]; @@ -514,11 +458,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti28 = _t[28U]; uint32_t v27 = - vb28 - + - ((va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) - << 5U - | (va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) >> 27U); + vb28 + + ((va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) << 5U | + (va27 + ((vb28 & vd28) | (vc28 & ~vd28)) + xk27 + ti28) >> 27U); abcd[0U] = v27; uint32_t va28 = abcd[3U]; uint32_t vb29 = abcd[0U]; @@ -530,11 +472,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti29 = _t[29U]; uint32_t v28 = - vb29 - + - ((va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) - << 9U - | (va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) >> 23U); + vb29 + + ((va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) << 9U | + (va28 + ((vb29 & vd29) | (vc29 & ~vd29)) + xk28 + ti29) >> 23U); abcd[3U] = v28; uint32_t va29 = abcd[2U]; uint32_t vb30 = abcd[3U]; @@ -546,11 +486,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti30 = _t[30U]; uint32_t v29 = - vb30 - + - ((va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) - << 14U - | (va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) >> 18U); + vb30 + + ((va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) << 14U | + (va29 + ((vb30 & vd30) | (vc30 & ~vd30)) + xk29 + ti30) >> 18U); abcd[2U] = v29; uint32_t va30 = abcd[1U]; uint32_t vb31 = abcd[2U]; @@ -562,11 +500,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti31 = _t[31U]; uint32_t v30 = - vb31 - + - ((va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) - << 20U - | (va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) >> 12U); + vb31 + + ((va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) << 20U | + (va30 + ((vb31 & vd31) | (vc31 & ~vd31)) + xk30 + ti31) >> 12U); abcd[1U] = v30; uint32_t va31 = abcd[0U]; uint32_t vb32 = abcd[1U]; @@ -578,11 +514,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti32 = _t[32U]; uint32_t v31 = - vb32 - + - ((va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) - << 4U - | (va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) >> 28U); + vb32 + + ((va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) << 4U | + (va31 + (vb32 ^ (vc32 ^ vd32)) + xk31 + ti32) >> 28U); abcd[0U] = v31; uint32_t va32 = abcd[3U]; uint32_t vb33 = abcd[0U]; @@ -594,11 +528,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti33 = _t[33U]; uint32_t v32 = - vb33 - + - ((va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) - << 11U - | (va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) >> 21U); + vb33 + + ((va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) << 11U | + (va32 + (vb33 ^ (vc33 ^ vd33)) + xk32 + ti33) >> 21U); abcd[3U] = v32; uint32_t va33 = abcd[2U]; uint32_t vb34 = abcd[3U]; @@ -610,11 +542,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti34 = _t[34U]; uint32_t v33 = - vb34 - + - ((va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) - << 16U - | (va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) >> 16U); + vb34 + + ((va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) << 16U | + (va33 + (vb34 ^ (vc34 ^ vd34)) + xk33 + ti34) >> 16U); abcd[2U] = v33; uint32_t va34 = abcd[1U]; uint32_t vb35 = abcd[2U]; @@ -626,11 +556,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti35 = _t[35U]; uint32_t v34 = - vb35 - + - ((va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) - << 23U - | (va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) >> 9U); + vb35 + + ((va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) << 23U | + (va34 + (vb35 ^ (vc35 ^ vd35)) + xk34 + ti35) >> 9U); abcd[1U] = v34; uint32_t va35 = abcd[0U]; uint32_t vb36 = abcd[1U]; @@ -642,11 +570,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti36 = _t[36U]; uint32_t v35 = - vb36 - + - ((va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) - << 4U - | (va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) >> 28U); + vb36 + + ((va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) << 4U | + (va35 + (vb36 ^ (vc36 ^ vd36)) + xk35 + ti36) >> 28U); abcd[0U] = v35; uint32_t va36 = abcd[3U]; uint32_t vb37 = abcd[0U]; @@ -658,11 +584,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti37 = _t[37U]; uint32_t v36 = - vb37 - + - ((va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) - << 11U - | (va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) >> 21U); + vb37 + + ((va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) << 11U | + (va36 + (vb37 ^ (vc37 ^ vd37)) + xk36 + ti37) >> 21U); abcd[3U] = v36; uint32_t va37 = abcd[2U]; uint32_t vb38 = abcd[3U]; @@ -674,11 +598,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti38 = _t[38U]; uint32_t v37 = - vb38 - + - ((va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) - << 16U - | (va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) >> 16U); + vb38 + + ((va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) << 16U | + (va37 + (vb38 ^ (vc38 ^ vd38)) + xk37 + ti38) >> 16U); abcd[2U] = v37; uint32_t va38 = abcd[1U]; uint32_t vb39 = abcd[2U]; @@ -690,11 +612,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti39 = _t[39U]; uint32_t v38 = - vb39 - + - ((va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) - << 23U - | (va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) >> 9U); + vb39 + + ((va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) << 23U | + (va38 + (vb39 ^ (vc39 ^ vd39)) + xk38 + ti39) >> 9U); abcd[1U] = v38; uint32_t va39 = abcd[0U]; uint32_t vb40 = abcd[1U]; @@ -706,11 +626,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti40 = _t[40U]; uint32_t v39 = - vb40 - + - ((va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) - << 4U - | (va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) >> 28U); + vb40 + + ((va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) << 4U | + (va39 + (vb40 ^ (vc40 ^ vd40)) + xk39 + ti40) >> 28U); abcd[0U] = v39; uint32_t va40 = abcd[3U]; uint32_t vb41 = abcd[0U]; @@ -722,11 +640,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti41 = _t[41U]; uint32_t v40 = - vb41 - + - ((va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) - << 11U - | (va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) >> 21U); + vb41 + + ((va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) << 11U | + (va40 + (vb41 ^ (vc41 ^ vd41)) + xk40 + ti41) >> 21U); abcd[3U] = v40; uint32_t va41 = abcd[2U]; uint32_t vb42 = abcd[3U]; @@ -738,11 +654,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti42 = _t[42U]; uint32_t v41 = - vb42 - + - ((va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) - << 16U - | (va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) >> 16U); + vb42 + + ((va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) << 16U | + (va41 + (vb42 ^ (vc42 ^ vd42)) + xk41 + ti42) >> 16U); abcd[2U] = v41; uint32_t va42 = abcd[1U]; uint32_t vb43 = abcd[2U]; @@ -754,11 +668,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti43 = _t[43U]; uint32_t v42 = - vb43 - + - ((va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) - << 23U - | (va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) >> 9U); + vb43 + + ((va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) << 23U | + (va42 + (vb43 ^ (vc43 ^ vd43)) + xk42 + ti43) >> 9U); abcd[1U] = v42; uint32_t va43 = abcd[0U]; uint32_t vb44 = abcd[1U]; @@ -770,11 +682,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti44 = _t[44U]; uint32_t v43 = - vb44 - + - ((va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) - << 4U - | (va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) >> 28U); + vb44 + + ((va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) << 4U | + (va43 + (vb44 ^ (vc44 ^ vd44)) + xk43 + ti44) >> 28U); abcd[0U] = v43; uint32_t va44 = abcd[3U]; uint32_t vb45 = abcd[0U]; @@ -786,11 +696,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti45 = _t[45U]; uint32_t v44 = - vb45 - + - ((va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) - << 11U - | (va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) >> 21U); + vb45 + + ((va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) << 11U | + (va44 + (vb45 ^ (vc45 ^ vd45)) + xk44 + ti45) >> 21U); abcd[3U] = v44; uint32_t va45 = abcd[2U]; uint32_t vb46 = abcd[3U]; @@ -802,11 +710,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti46 = _t[46U]; uint32_t v45 = - vb46 - + - ((va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) - << 16U - | (va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) >> 16U); + vb46 + + ((va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) << 16U | + (va45 + (vb46 ^ (vc46 ^ vd46)) + xk45 + ti46) >> 16U); abcd[2U] = v45; uint32_t va46 = abcd[1U]; uint32_t vb47 = abcd[2U]; @@ -818,11 +724,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti47 = _t[47U]; uint32_t v46 = - vb47 - + - ((va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) - << 23U - | (va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) >> 9U); + vb47 + + ((va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) << 23U | + (va46 + (vb47 ^ (vc47 ^ vd47)) + xk46 + ti47) >> 9U); abcd[1U] = v46; uint32_t va47 = abcd[0U]; uint32_t vb48 = abcd[1U]; @@ -834,11 +738,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti48 = _t[48U]; uint32_t v47 = - vb48 - + - ((va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) - << 6U - | (va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) >> 26U); + vb48 + + ((va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) << 6U | + (va47 + (vc48 ^ (vb48 | ~vd48)) + xk47 + ti48) >> 26U); abcd[0U] = v47; uint32_t va48 = abcd[3U]; uint32_t vb49 = abcd[0U]; @@ -850,11 +752,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti49 = _t[49U]; uint32_t v48 = - vb49 - + - ((va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) - << 10U - | (va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) >> 22U); + vb49 + + ((va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) << 10U | + (va48 + (vc49 ^ (vb49 | ~vd49)) + xk48 + ti49) >> 22U); abcd[3U] = v48; uint32_t va49 = abcd[2U]; uint32_t vb50 = abcd[3U]; @@ -866,11 +766,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti50 = _t[50U]; uint32_t v49 = - vb50 - + - ((va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) - << 15U - | (va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) >> 17U); + vb50 + + ((va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) << 15U | + (va49 + (vc50 ^ (vb50 | ~vd50)) + xk49 + ti50) >> 17U); abcd[2U] = v49; uint32_t va50 = abcd[1U]; uint32_t vb51 = abcd[2U]; @@ -882,11 +780,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti51 = _t[51U]; uint32_t v50 = - vb51 - + - ((va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) - << 21U - | (va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) >> 11U); + vb51 + + ((va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) << 21U | + (va50 + (vc51 ^ (vb51 | ~vd51)) + xk50 + ti51) >> 11U); abcd[1U] = v50; uint32_t va51 = abcd[0U]; uint32_t vb52 = abcd[1U]; @@ -898,11 +794,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti52 = _t[52U]; uint32_t v51 = - vb52 - + - ((va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) - << 6U - | (va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) >> 26U); + vb52 + + ((va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) << 6U | + (va51 + (vc52 ^ (vb52 | ~vd52)) + xk51 + ti52) >> 26U); abcd[0U] = v51; uint32_t va52 = abcd[3U]; uint32_t vb53 = abcd[0U]; @@ -914,11 +808,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti53 = _t[53U]; uint32_t v52 = - vb53 - + - ((va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) - << 10U - | (va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) >> 22U); + vb53 + + ((va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) << 10U | + (va52 + (vc53 ^ (vb53 | ~vd53)) + xk52 + ti53) >> 22U); abcd[3U] = v52; uint32_t va53 = abcd[2U]; uint32_t vb54 = abcd[3U]; @@ -930,11 +822,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti54 = _t[54U]; uint32_t v53 = - vb54 - + - ((va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) - << 15U - | (va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) >> 17U); + vb54 + + ((va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) << 15U | + (va53 + (vc54 ^ (vb54 | ~vd54)) + xk53 + ti54) >> 17U); abcd[2U] = v53; uint32_t va54 = abcd[1U]; uint32_t vb55 = abcd[2U]; @@ -946,11 +836,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti55 = _t[55U]; uint32_t v54 = - vb55 - + - ((va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) - << 21U - | (va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) >> 11U); + vb55 + + ((va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) << 21U | + (va54 + (vc55 ^ (vb55 | ~vd55)) + xk54 + ti55) >> 11U); abcd[1U] = v54; uint32_t va55 = abcd[0U]; uint32_t vb56 = abcd[1U]; @@ -962,11 +850,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti56 = _t[56U]; uint32_t v55 = - vb56 - + - ((va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) - << 6U - | (va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) >> 26U); + vb56 + + ((va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) << 6U | + (va55 + (vc56 ^ (vb56 | ~vd56)) + xk55 + ti56) >> 26U); abcd[0U] = v55; uint32_t va56 = abcd[3U]; uint32_t vb57 = abcd[0U]; @@ -978,11 +864,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti57 = _t[57U]; uint32_t v56 = - vb57 - + - ((va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) - << 10U - | (va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) >> 22U); + vb57 + + ((va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) << 10U | + (va56 + (vc57 ^ (vb57 | ~vd57)) + xk56 + ti57) >> 22U); abcd[3U] = v56; uint32_t va57 = abcd[2U]; uint32_t vb58 = abcd[3U]; @@ -994,11 +878,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti58 = _t[58U]; uint32_t v57 = - vb58 - + - ((va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) - << 15U - | (va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) >> 17U); + vb58 + + ((va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) << 15U | + (va57 + (vc58 ^ (vb58 | ~vd58)) + xk57 + ti58) >> 17U); abcd[2U] = v57; uint32_t va58 = abcd[1U]; uint32_t vb59 = abcd[2U]; @@ -1010,11 +892,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti59 = _t[59U]; uint32_t v58 = - vb59 - + - ((va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) - << 21U - | (va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) >> 11U); + vb59 + + ((va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) << 21U | + (va58 + (vc59 ^ (vb59 | ~vd59)) + xk58 + ti59) >> 11U); abcd[1U] = v58; uint32_t va59 = abcd[0U]; uint32_t vb60 = abcd[1U]; @@ -1026,11 +906,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti60 = _t[60U]; uint32_t v59 = - vb60 - + - ((va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) - << 6U - | (va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) >> 26U); + vb60 + + ((va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) << 6U | + (va59 + (vc60 ^ (vb60 | ~vd60)) + xk59 + ti60) >> 26U); abcd[0U] = v59; uint32_t va60 = abcd[3U]; uint32_t vb61 = abcd[0U]; @@ -1042,11 +920,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti61 = _t[61U]; uint32_t v60 = - vb61 - + - ((va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) - << 10U - | (va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) >> 22U); + vb61 + + ((va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) << 10U | + (va60 + (vc61 ^ (vb61 | ~vd61)) + xk60 + ti61) >> 22U); abcd[3U] = v60; uint32_t va61 = abcd[2U]; uint32_t vb62 = abcd[3U]; @@ -1058,11 +934,9 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti62 = _t[62U]; uint32_t v61 = - vb62 - + - ((va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) - << 15U - | (va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) >> 17U); + vb62 + + ((va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) << 15U | + (va61 + (vc62 ^ (vb62 | ~vd62)) + xk61 + ti62) >> 17U); abcd[2U] = v61; uint32_t va62 = abcd[1U]; uint32_t vb = abcd[2U]; @@ -1074,11 +948,8 @@ static void update(uint32_t *abcd, uint8_t *x) uint32_t ti = _t[63U]; uint32_t v62 = - vb - + - ((va62 + (vc ^ (vb | ~vd)) + xk62 + ti) - << 21U - | (va62 + (vc ^ (vb | ~vd)) + xk62 + ti) >> 11U); + vb + + ((va62 + (vc ^ (vb | ~vd)) + xk62 + ti) << 21U | (va62 + (vc ^ (vb | ~vd)) + xk62 + ti) >> 11U); abcd[1U] = v62; uint32_t a = abcd[0U]; uint32_t b = abcd[1U]; @@ -1282,8 +1153,7 @@ Hacl_Hash_MD5_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, @@ -1328,8 +1198,7 @@ Hacl_Hash_MD5_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t Hacl_Hash_MD5_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, @@ -1359,8 +1228,7 @@ Hacl_Hash_MD5_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state10, @@ -1403,8 +1271,7 @@ Hacl_Hash_MD5_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t Hacl_Hash_MD5_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, diff --git a/Modules/_hacl/Hacl_Hash_MD5.h b/Modules/_hacl/Hacl_Hash_MD5.h index 521c2addc502897..b220bbf6890ffad 100644 --- a/Modules/_hacl/Hacl_Hash_MD5.h +++ b/Modules/_hacl/Hacl_Hash_MD5.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_Hash_MD5_H -#define __Hacl_Hash_MD5_H +#ifndef Hacl_Hash_MD5_H +#define Hacl_Hash_MD5_H #if defined(__cplusplus) extern "C" { @@ -62,5 +62,5 @@ void Hacl_Hash_MD5_hash(uint8_t *output, uint8_t *input, uint32_t input_len); } #endif -#define __Hacl_Hash_MD5_H_DEFINED -#endif +#define Hacl_Hash_MD5_H_DEFINED +#endif /* Hacl_Hash_MD5_H */ diff --git a/Modules/_hacl/Hacl_Hash_SHA1.c b/Modules/_hacl/Hacl_Hash_SHA1.c index 508e447bf275daf..97bd4f2204cf246 100644 --- a/Modules/_hacl/Hacl_Hash_SHA1.c +++ b/Modules/_hacl/Hacl_Hash_SHA1.c @@ -315,8 +315,7 @@ Hacl_Hash_SHA1_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_ uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, @@ -361,8 +360,7 @@ Hacl_Hash_SHA1_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_ Hacl_Hash_SHA1_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, @@ -392,8 +390,7 @@ Hacl_Hash_SHA1_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_ uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state10, @@ -436,8 +433,7 @@ Hacl_Hash_SHA1_update(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_ Hacl_Hash_SHA1_update_multi(block_state1, data1, data1_len / 64U); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, diff --git a/Modules/_hacl/Hacl_Hash_SHA1.h b/Modules/_hacl/Hacl_Hash_SHA1.h index 63ac83f9dce0186..7fc8c8cfd205d97 100644 --- a/Modules/_hacl/Hacl_Hash_SHA1.h +++ b/Modules/_hacl/Hacl_Hash_SHA1.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_Hash_SHA1_H -#define __Hacl_Hash_SHA1_H +#ifndef Hacl_Hash_SHA1_H +#define Hacl_Hash_SHA1_H #if defined(__cplusplus) extern "C" { @@ -62,5 +62,5 @@ void Hacl_Hash_SHA1_hash(uint8_t *output, uint8_t *input, uint32_t input_len); } #endif -#define __Hacl_Hash_SHA1_H_DEFINED -#endif +#define Hacl_Hash_SHA1_H_DEFINED +#endif /* Hacl_Hash_SHA1_H */ diff --git a/Modules/_hacl/Hacl_Hash_SHA2.c b/Modules/_hacl/Hacl_Hash_SHA2.c index d612bafa72cdc4a..d2ee0c9ef517210 100644 --- a/Modules/_hacl/Hacl_Hash_SHA2.c +++ b/Modules/_hacl/Hacl_Hash_SHA2.c @@ -100,15 +100,14 @@ static inline void sha256_update(uint8_t *b, uint32_t *hash) uint32_t k_e_t = k_t; uint32_t t1 = - h02 - + ((e0 << 26U | e0 >> 6U) ^ ((e0 << 21U | e0 >> 11U) ^ (e0 << 7U | e0 >> 25U))) - + ((e0 & f0) ^ (~e0 & g0)) + h02 + ((e0 << 26U | e0 >> 6U) ^ ((e0 << 21U | e0 >> 11U) ^ (e0 << 7U | e0 >> 25U))) + + ((e0 & f0) ^ (~e0 & g0)) + k_e_t + ws_t; uint32_t t2 = - ((a0 << 30U | a0 >> 2U) ^ ((a0 << 19U | a0 >> 13U) ^ (a0 << 10U | a0 >> 22U))) - + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); + ((a0 << 30U | a0 >> 2U) ^ ((a0 << 19U | a0 >> 13U) ^ (a0 << 10U | a0 >> 22U))) + + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); uint32_t a1 = t1 + t2; uint32_t b1 = a0; uint32_t c1 = b0; @@ -301,15 +300,14 @@ static inline void sha512_update(uint8_t *b, uint64_t *hash) uint64_t k_e_t = k_t; uint64_t t1 = - h02 - + ((e0 << 50U | e0 >> 14U) ^ ((e0 << 46U | e0 >> 18U) ^ (e0 << 23U | e0 >> 41U))) - + ((e0 & f0) ^ (~e0 & g0)) + h02 + ((e0 << 50U | e0 >> 14U) ^ ((e0 << 46U | e0 >> 18U) ^ (e0 << 23U | e0 >> 41U))) + + ((e0 & f0) ^ (~e0 & g0)) + k_e_t + ws_t; uint64_t t2 = - ((a0 << 36U | a0 >> 28U) ^ ((a0 << 30U | a0 >> 34U) ^ (a0 << 25U | a0 >> 39U))) - + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); + ((a0 << 36U | a0 >> 28U) ^ ((a0 << 30U | a0 >> 34U) ^ (a0 << 25U | a0 >> 39U))) + + ((a0 & b0) ^ ((a0 & c0) ^ (b0 & c0))); uint64_t a1 = t1 + t2; uint64_t b1 = a0; uint64_t c1 = b0; @@ -639,8 +637,7 @@ update_224_256(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, @@ -685,8 +682,7 @@ update_224_256(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk Hacl_Hash_SHA2_sha256_update_nblocks(data1_len / 64U * 64U, data1, block_state1); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, @@ -716,8 +712,7 @@ update_224_256(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state10, @@ -760,8 +755,7 @@ update_224_256(Hacl_Streaming_MD_state_32 *state, uint8_t *chunk, uint32_t chunk Hacl_Hash_SHA2_sha256_update_nblocks(data1_len / 64U * 64U, data1, block_state1); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_32){ .block_state = block_state1, @@ -1205,8 +1199,7 @@ update_384_512(Hacl_Streaming_MD_state_64 *state, uint8_t *chunk, uint32_t chunk uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Streaming_MD_state_64){ .block_state = block_state1, @@ -1251,8 +1244,7 @@ update_384_512(Hacl_Streaming_MD_state_64 *state, uint8_t *chunk, uint32_t chunk Hacl_Hash_SHA2_sha512_update_nblocks(data1_len / 128U * 128U, data1, block_state1); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_64){ .block_state = block_state1, @@ -1282,8 +1274,7 @@ update_384_512(Hacl_Streaming_MD_state_64 *state, uint8_t *chunk, uint32_t chunk uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Streaming_MD_state_64){ .block_state = block_state10, @@ -1326,8 +1317,7 @@ update_384_512(Hacl_Streaming_MD_state_64 *state, uint8_t *chunk, uint32_t chunk Hacl_Hash_SHA2_sha512_update_nblocks(data1_len / 128U * 128U, data1, block_state1); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_MD_state_64){ .block_state = block_state1, diff --git a/Modules/_hacl/Hacl_Hash_SHA2.h b/Modules/_hacl/Hacl_Hash_SHA2.h index a93138fb7ee7a78..dd168b8aa1a424b 100644 --- a/Modules/_hacl/Hacl_Hash_SHA2.h +++ b/Modules/_hacl/Hacl_Hash_SHA2.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_Hash_SHA2_H -#define __Hacl_Hash_SHA2_H +#ifndef Hacl_Hash_SHA2_H +#define Hacl_Hash_SHA2_H #if defined(__cplusplus) extern "C" { @@ -199,5 +199,5 @@ void Hacl_Hash_SHA2_hash_384(uint8_t *output, uint8_t *input, uint32_t input_len } #endif -#define __Hacl_Hash_SHA2_H_DEFINED -#endif +#define Hacl_Hash_SHA2_H_DEFINED +#endif /* Hacl_Hash_SHA2_H */ diff --git a/Modules/_hacl/Hacl_Hash_SHA3.c b/Modules/_hacl/Hacl_Hash_SHA3.c index 87638df9549fbbf..466d2b96c0cdfa1 100644 --- a/Modules/_hacl/Hacl_Hash_SHA3.c +++ b/Modules/_hacl/Hacl_Hash_SHA3.c @@ -866,8 +866,7 @@ Hacl_Hash_SHA3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *chunk, uint32_t ch uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ((Hacl_Hash_SHA3_state_t){ .block_state = block_state1, .buf = buf, .total_len = total_len2 }); } else if (sz == 0U) @@ -910,8 +909,7 @@ Hacl_Hash_SHA3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *chunk, uint32_t ch Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data1, data1_len / block_len(a1)); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_SHA3_state_t){ .block_state = block_state1, @@ -941,8 +939,7 @@ Hacl_Hash_SHA3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *chunk, uint32_t ch uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Hash_SHA3_state_t){ .block_state = block_state10, @@ -972,10 +969,8 @@ Hacl_Hash_SHA3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *chunk, uint32_t ch uint32_t ite; if ( - (uint64_t)(chunk_len - diff) - % (uint64_t)block_len(i) - == 0ULL - && (uint64_t)(chunk_len - diff) > 0ULL + (uint64_t)(chunk_len - diff) % (uint64_t)block_len(i) == 0ULL && + (uint64_t)(chunk_len - diff) > 0ULL ) { ite = block_len(i); @@ -994,8 +989,7 @@ Hacl_Hash_SHA3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *chunk, uint32_t ch Hacl_Hash_SHA3_update_multi_sha3(a1, s2, data1, data1_len / block_len(a1)); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Hash_SHA3_state_t){ .block_state = block_state1, @@ -2422,9 +2416,7 @@ Hacl_Hash_SHA3_shake128_squeeze_nblocks( 5U, 1U, _C[i] = - state[i - + 0U] - ^ (state[i + 5U] ^ (state[i + 10U] ^ (state[i + 15U] ^ state[i + 20U])));); + state[i + 0U] ^ (state[i + 5U] ^ (state[i + 10U] ^ (state[i + 15U] ^ state[i + 20U])));); KRML_MAYBE_FOR5(i2, 0U, 5U, diff --git a/Modules/_hacl/Hacl_Hash_SHA3.h b/Modules/_hacl/Hacl_Hash_SHA3.h index 65ec99ee3a3ac91..df6ebe439e98b11 100644 --- a/Modules/_hacl/Hacl_Hash_SHA3.h +++ b/Modules/_hacl/Hacl_Hash_SHA3.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_Hash_SHA3_H -#define __Hacl_Hash_SHA3_H +#ifndef Hacl_Hash_SHA3_H +#define Hacl_Hash_SHA3_H #if defined(__cplusplus) extern "C" { @@ -155,5 +155,5 @@ Hacl_Hash_SHA3_shake128_squeeze_nblocks( } #endif -#define __Hacl_Hash_SHA3_H_DEFINED -#endif +#define Hacl_Hash_SHA3_H_DEFINED +#endif /* Hacl_Hash_SHA3_H */ diff --git a/Modules/_hacl/Hacl_Streaming_HMAC.c b/Modules/_hacl/Hacl_Streaming_HMAC.c index d28b39792af576a..f89c00ee1ae4162 100644 --- a/Modules/_hacl/Hacl_Streaming_HMAC.c +++ b/Modules/_hacl/Hacl_Streaming_HMAC.c @@ -198,8 +198,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = ((Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_MD5_a, { .case_MD5_a = s1 } }); + st[0U] = ((Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_MD5_a, { .case_MD5_a = s1 } }); } if (st == NULL) { @@ -220,8 +219,8 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = ((Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA1_a, { .case_SHA1_a = s1 } }); + st[0U] = + ((Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA1_a, { .case_SHA1_a = s1 } }); } if (st == NULL) { @@ -242,8 +241,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA2_224_a, @@ -270,8 +268,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA2_256_a, @@ -298,8 +295,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA2_384_a, @@ -326,8 +322,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA2_512_a, @@ -354,8 +349,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA3_224_a, @@ -382,8 +376,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA3_256_a, @@ -410,8 +403,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA3_384_a, @@ -438,8 +430,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_SHA3_512_a, @@ -466,8 +457,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_Blake2S_a, @@ -495,8 +485,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_Blake2S_128_a, @@ -531,8 +520,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_Blake2B_a, @@ -560,8 +548,7 @@ static Hacl_Agile_Hash_state_s *malloc_(Hacl_Agile_Hash_impl a) *st = (Hacl_Agile_Hash_state_s *)KRML_HOST_MALLOC(sizeof (Hacl_Agile_Hash_state_s)); if (st != NULL) { - st[0U] - = + st[0U] = ( (Hacl_Agile_Hash_state_s){ .tag = Hacl_Agile_Hash_Blake2B_256_a, @@ -2059,8 +2046,8 @@ Hacl_Streaming_HMAC_update( Hacl_Streaming_HMAC_Definitions_index i1 = Hacl_Streaming_HMAC_index_of_state(block_state); if ( - (uint64_t)chunk_len - > max_input_len64(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - total_len + (uint64_t)chunk_len > + max_input_len64(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - total_len ) { return Hacl_Streaming_Types_MaximumLengthExceeded; @@ -2068,9 +2055,7 @@ Hacl_Streaming_HMAC_update( uint32_t sz; if ( - total_len - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - == 0ULL + total_len % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == 0ULL && total_len > 0ULL ) { @@ -2079,8 +2064,8 @@ Hacl_Streaming_HMAC_update( else { sz = - (uint32_t)(total_len - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)(total_len % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } if (chunk_len <= block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - sz) { @@ -2091,9 +2076,7 @@ Hacl_Streaming_HMAC_update( uint32_t sz1; if ( - total_len1 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - == 0ULL + total_len1 % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == 0ULL && total_len1 > 0ULL ) { @@ -2102,14 +2085,13 @@ Hacl_Streaming_HMAC_update( else { sz1 = - (uint32_t)(total_len1 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)(total_len1 % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } uint8_t *buf2 = buf + sz1; memcpy(buf2, chunk, chunk_len * sizeof (uint8_t)); uint64_t total_len2 = total_len1 + (uint64_t)chunk_len; - *state - = + *state = ( (Hacl_Streaming_HMAC_agile_state){ .block_state = block_state1, @@ -2127,9 +2109,7 @@ Hacl_Streaming_HMAC_update( uint32_t sz1; if ( - total_len1 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - == 0ULL + total_len1 % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == 0ULL && total_len1 > 0ULL ) { @@ -2138,8 +2118,8 @@ Hacl_Streaming_HMAC_update( else { sz1 = - (uint32_t)(total_len1 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)(total_len1 % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } if (!(sz1 == 0U)) { @@ -2153,8 +2133,8 @@ Hacl_Streaming_HMAC_update( uint32_t ite; if ( - (uint64_t)chunk_len - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) + (uint64_t)chunk_len % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == 0ULL && (uint64_t)chunk_len > 0ULL ) @@ -2164,8 +2144,8 @@ Hacl_Streaming_HMAC_update( else { ite = - (uint32_t)((uint64_t)chunk_len - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)((uint64_t)chunk_len % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } uint32_t n_blocks = (chunk_len - ite) / block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))); @@ -2178,8 +2158,7 @@ Hacl_Streaming_HMAC_update( update_multi(s11, total_len1, data1, data1_len); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_HMAC_agile_state){ .block_state = block_state1, @@ -2200,9 +2179,8 @@ Hacl_Streaming_HMAC_update( uint32_t sz10; if ( - total_len10 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - == 0ULL + total_len10 % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == + 0ULL && total_len10 > 0ULL ) { @@ -2211,14 +2189,13 @@ Hacl_Streaming_HMAC_update( else { sz10 = - (uint32_t)(total_len10 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)(total_len10 % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } uint8_t *buf2 = buf0 + sz10; memcpy(buf2, chunk1, diff * sizeof (uint8_t)); uint64_t total_len2 = total_len10 + (uint64_t)diff; - *state - = + *state = ( (Hacl_Streaming_HMAC_agile_state){ .block_state = block_state10, @@ -2233,9 +2210,7 @@ Hacl_Streaming_HMAC_update( uint32_t sz1; if ( - total_len1 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - == 0ULL + total_len1 % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == 0ULL && total_len1 > 0ULL ) { @@ -2244,8 +2219,8 @@ Hacl_Streaming_HMAC_update( else { sz1 = - (uint32_t)(total_len1 - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)(total_len1 % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } if (!(sz1 == 0U)) { @@ -2259,8 +2234,8 @@ Hacl_Streaming_HMAC_update( uint32_t ite; if ( - (uint64_t)(chunk_len - diff) - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) + (uint64_t)(chunk_len - diff) % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == 0ULL && (uint64_t)(chunk_len - diff) > 0ULL ) @@ -2270,13 +2245,12 @@ Hacl_Streaming_HMAC_update( else { ite = - (uint32_t)((uint64_t)(chunk_len - diff) - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)((uint64_t)(chunk_len - diff) % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } uint32_t n_blocks = - (chunk_len - diff - ite) - / block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))); + (chunk_len - diff - ite) / block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))); uint32_t data1_len = n_blocks * block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))); uint32_t data2_len = chunk_len - diff - data1_len; @@ -2286,8 +2260,7 @@ Hacl_Streaming_HMAC_update( update_multi(s11, total_len1, data1, data1_len); uint8_t *dst = buf; memcpy(dst, data2, data2_len * sizeof (uint8_t)); - *state - = + *state = ( (Hacl_Streaming_HMAC_agile_state){ .block_state = block_state1, @@ -2324,9 +2297,7 @@ Hacl_Streaming_HMAC_digest( uint32_t r; if ( - total_len - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) - == 0ULL + total_len % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1))) == 0ULL && total_len > 0ULL ) { @@ -2335,8 +2306,8 @@ Hacl_Streaming_HMAC_digest( else { r = - (uint32_t)(total_len - % (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); + (uint32_t)(total_len % + (uint64_t)block_len(alg_of_impl(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)))); } uint8_t *buf_1 = buf_; Hacl_Agile_Hash_state_s *s110 = malloc_(dfst__Hacl_Agile_Hash_impl_uint32_t(i1)); @@ -2404,9 +2375,13 @@ Hacl_Streaming_HMAC_digest( Hacl_Agile_Hash_state_s *s112 = tmp_block_state1.snd; update_multi(s112, prev_len, buf_multi, 0U); uint64_t prev_len_last = total_len - (uint64_t)r; - Hacl_Agile_Hash_state_s *s11 = tmp_block_state1.snd; - update_last(s11, prev_len_last, buf_last, r); + Hacl_Agile_Hash_state_s *s113 = tmp_block_state1.snd; + update_last(s113, prev_len_last, buf_last, r); finish0(tmp_block_state1, output); + Hacl_Agile_Hash_state_s *s210 = tmp_block_state1.thd; + Hacl_Agile_Hash_state_s *s11 = tmp_block_state1.snd; + free_(s11); + free_(s210); return Hacl_Streaming_Types_Success; } KRML_HOST_EPRINTF("KaRaMeL abort at %s:%d\n%s\n", diff --git a/Modules/_hacl/Hacl_Streaming_HMAC.h b/Modules/_hacl/Hacl_Streaming_HMAC.h index a0806c02d0bb9fa..6c302b1f92d6a36 100644 --- a/Modules/_hacl/Hacl_Streaming_HMAC.h +++ b/Modules/_hacl/Hacl_Streaming_HMAC.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_Streaming_HMAC_H -#define __Hacl_Streaming_HMAC_H +#ifndef Hacl_Streaming_HMAC_H +#define Hacl_Streaming_HMAC_H #if defined(__cplusplus) extern "C" { @@ -130,5 +130,5 @@ Hacl_Streaming_HMAC_agile_state } #endif -#define __Hacl_Streaming_HMAC_H_DEFINED -#endif +#define Hacl_Streaming_HMAC_H_DEFINED +#endif /* Hacl_Streaming_HMAC_H */ diff --git a/Modules/_hacl/Hacl_Streaming_Types.h b/Modules/_hacl/Hacl_Streaming_Types.h index 5c497750793720c..cce25eb79467502 100644 --- a/Modules/_hacl/Hacl_Streaming_Types.h +++ b/Modules/_hacl/Hacl_Streaming_Types.h @@ -23,8 +23,8 @@ */ -#ifndef __Hacl_Streaming_Types_H -#define __Hacl_Streaming_Types_H +#ifndef Hacl_Streaming_Types_H +#define Hacl_Streaming_Types_H #if defined(__cplusplus) extern "C" { @@ -68,5 +68,5 @@ typedef struct Hacl_Streaming_MD_state_64_s Hacl_Streaming_MD_state_64; } #endif -#define __Hacl_Streaming_Types_H_DEFINED -#endif +#define Hacl_Streaming_Types_H_DEFINED +#endif /* Hacl_Streaming_Types_H */ diff --git a/Modules/_hacl/Lib_Memzero0.c b/Modules/_hacl/Lib_Memzero0.c index 28abd1aa4e2d546..f94e0e2254a9128 100644 --- a/Modules/_hacl/Lib_Memzero0.c +++ b/Modules/_hacl/Lib_Memzero0.c @@ -11,18 +11,18 @@ #if defined(__APPLE__) && defined(__MACH__) #include <AvailabilityMacros.h> // memset_s is available from macOS 10.9, iOS 7, watchOS 2, and on all tvOS and visionOS versions. -# if (defined(MAC_OS_X_VERSION_MIN_REQUIRED) && (MAC_OS_X_VERSION_MIN_REQUIRED >= __MAC_10_9)) -# define APPLE_HAS_MEMSET_S 1 -# elif (defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && (__IPHONE_OS_VERSION_MIN_REQUIRED >= __IPHONE_7_0)) -# define APPLE_HAS_MEMSET_S 1 +# if (defined(MAC_OS_X_VERSION_MIN_REQUIRED) && defined(MAC_OS_X_VERSION_10_9) && (MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_9)) +# define APPLE_HAS_MEMSET_S +# elif (defined(__IPHONE_OS_VERSION_MIN_REQUIRED) && defined(__IPHONE_7_0) && (__IPHONE_OS_VERSION_MIN_REQUIRED >= __IPHONE_7_0)) +# define APPLE_HAS_MEMSET_S # elif (defined(TARGET_OS_TV) && TARGET_OS_TV) -# define APPLE_HAS_MEMSET_S 1 -# elif (defined(__WATCH_OS_VERSION_MIN_REQUIRED) && (__WATCH_OS_VERSION_MIN_REQUIRED >= __WATCHOS_2_0)) -# define APPLE_HAS_MEMSET_S 1 +# define APPLE_HAS_MEMSET_S +# elif (defined(__WATCH_OS_VERSION_MIN_REQUIRED) && defined(__WATCHOS_2_0) && (__WATCH_OS_VERSION_MIN_REQUIRED >= __WATCHOS_2_0)) +# define APPLE_HAS_MEMSET_S # elif (defined(TARGET_OS_VISION) && TARGET_OS_VISION) -# define APPLE_HAS_MEMSET_S 1 +# define APPLE_HAS_MEMSET_S # else -# define APPLE_HAS_MEMSET_S 0 +# undef APPLE_HAS_MEMSET_S # endif #endif @@ -55,7 +55,7 @@ void Lib_Memzero0_memzero0(void *dst, uint64_t len) { #ifdef _WIN32 SecureZeroMemory(dst, len_); - #elif defined(__APPLE__) && defined(__MACH__) && APPLE_HAS_MEMSET_S + #elif defined(__APPLE__) && defined(__MACH__) && defined(APPLE_HAS_MEMSET_S) memset_s(dst, len_, 0, len_); #elif (defined(__linux__) && !defined(LINUX_NO_EXPLICIT_BZERO)) || defined(__FreeBSD__) || defined(__OpenBSD__) explicit_bzero(dst, len_); diff --git a/Modules/_hacl/include/krml/FStar_UInt128_Verified.h b/Modules/_hacl/include/krml/FStar_UInt128_Verified.h index d4a90220beafb5b..7aa2a6d83f9c4b9 100644 --- a/Modules/_hacl/include/krml/FStar_UInt128_Verified.h +++ b/Modules/_hacl/include/krml/FStar_UInt128_Verified.h @@ -4,8 +4,8 @@ */ -#ifndef __FStar_UInt128_Verified_H -#define __FStar_UInt128_Verified_H +#ifndef FStar_UInt128_Verified_H +#define FStar_UInt128_Verified_H #include "FStar_UInt_8_16_32_64.h" #include <inttypes.h> @@ -257,11 +257,11 @@ FStar_UInt128_gte_mask(FStar_UInt128_uint128 a, FStar_UInt128_uint128 b) { FStar_UInt128_uint128 lit; lit.low = - (FStar_UInt64_gte_mask(a.high, b.high) & ~FStar_UInt64_eq_mask(a.high, b.high)) - | (FStar_UInt64_eq_mask(a.high, b.high) & FStar_UInt64_gte_mask(a.low, b.low)); + (FStar_UInt64_gte_mask(a.high, b.high) & ~FStar_UInt64_eq_mask(a.high, b.high)) | + (FStar_UInt64_eq_mask(a.high, b.high) & FStar_UInt64_gte_mask(a.low, b.low)); lit.high = - (FStar_UInt64_gte_mask(a.high, b.high) & ~FStar_UInt64_eq_mask(a.high, b.high)) - | (FStar_UInt64_eq_mask(a.high, b.high) & FStar_UInt64_gte_mask(a.low, b.low)); + (FStar_UInt64_gte_mask(a.high, b.high) & ~FStar_UInt64_eq_mask(a.high, b.high)) | + (FStar_UInt64_eq_mask(a.high, b.high) & FStar_UInt64_gte_mask(a.low, b.low)); return lit; } @@ -294,14 +294,12 @@ static inline FStar_UInt128_uint128 FStar_UInt128_mul32(uint64_t x, uint32_t y) { FStar_UInt128_uint128 lit; lit.low = - FStar_UInt128_u32_combine((x >> FStar_UInt128_u32_32) - * (uint64_t)y - + (FStar_UInt128_u64_mod_32(x) * (uint64_t)y >> FStar_UInt128_u32_32), + FStar_UInt128_u32_combine((x >> FStar_UInt128_u32_32) * (uint64_t)y + + (FStar_UInt128_u64_mod_32(x) * (uint64_t)y >> FStar_UInt128_u32_32), FStar_UInt128_u64_mod_32(FStar_UInt128_u64_mod_32(x) * (uint64_t)y)); lit.high = - ((x >> FStar_UInt128_u32_32) - * (uint64_t)y - + (FStar_UInt128_u64_mod_32(x) * (uint64_t)y >> FStar_UInt128_u32_32)) + ((x >> FStar_UInt128_u32_32) * (uint64_t)y + + (FStar_UInt128_u64_mod_32(x) * (uint64_t)y >> FStar_UInt128_u32_32)) >> FStar_UInt128_u32_32; return lit; } @@ -315,32 +313,23 @@ static inline FStar_UInt128_uint128 FStar_UInt128_mul_wide(uint64_t x, uint64_t { FStar_UInt128_uint128 lit; lit.low = - FStar_UInt128_u32_combine_(FStar_UInt128_u64_mod_32(x) - * (y >> FStar_UInt128_u32_32) - + - FStar_UInt128_u64_mod_32((x >> FStar_UInt128_u32_32) - * FStar_UInt128_u64_mod_32(y) - + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32)), + FStar_UInt128_u32_combine_(FStar_UInt128_u64_mod_32(x) * (y >> FStar_UInt128_u32_32) + + FStar_UInt128_u64_mod_32((x >> FStar_UInt128_u32_32) * FStar_UInt128_u64_mod_32(y) + + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32)), FStar_UInt128_u64_mod_32(FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y))); lit.high = - (x >> FStar_UInt128_u32_32) - * (y >> FStar_UInt128_u32_32) - + - (((x >> FStar_UInt128_u32_32) - * FStar_UInt128_u64_mod_32(y) - + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32)) + (x >> FStar_UInt128_u32_32) * (y >> FStar_UInt128_u32_32) + + (((x >> FStar_UInt128_u32_32) * FStar_UInt128_u64_mod_32(y) + + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32)) >> FStar_UInt128_u32_32) + - ((FStar_UInt128_u64_mod_32(x) - * (y >> FStar_UInt128_u32_32) - + - FStar_UInt128_u64_mod_32((x >> FStar_UInt128_u32_32) - * FStar_UInt128_u64_mod_32(y) - + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32))) + ((FStar_UInt128_u64_mod_32(x) * (y >> FStar_UInt128_u32_32) + + FStar_UInt128_u64_mod_32((x >> FStar_UInt128_u32_32) * FStar_UInt128_u64_mod_32(y) + + (FStar_UInt128_u64_mod_32(x) * FStar_UInt128_u64_mod_32(y) >> FStar_UInt128_u32_32))) >> FStar_UInt128_u32_32); return lit; } -#define __FStar_UInt128_Verified_H_DEFINED -#endif +#define FStar_UInt128_Verified_H_DEFINED +#endif /* FStar_UInt128_Verified_H */ diff --git a/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h b/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h index 00be80836574af3..2720348bbb79e66 100644 --- a/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h +++ b/Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h @@ -4,8 +4,8 @@ */ -#ifndef __FStar_UInt_8_16_32_64_H -#define __FStar_UInt_8_16_32_64_H +#ifndef FStar_UInt_8_16_32_64_H +#define FStar_UInt_8_16_32_64_H #include <inttypes.h> #include <stdbool.h> @@ -30,6 +30,8 @@ extern uint64_t FStar_UInt64_zero; extern uint64_t FStar_UInt64_one; +extern bool FStar_UInt64_ne(uint64_t a, uint64_t b); + extern uint64_t FStar_UInt64_minus(uint64_t a); extern uint32_t FStar_UInt64_n_minus_one; @@ -80,6 +82,8 @@ extern uint32_t FStar_UInt32_zero; extern uint32_t FStar_UInt32_one; +extern bool FStar_UInt32_ne(uint32_t a, uint32_t b); + extern uint32_t FStar_UInt32_minus(uint32_t a); extern uint32_t FStar_UInt32_n_minus_one; @@ -130,6 +134,8 @@ extern uint16_t FStar_UInt16_zero; extern uint16_t FStar_UInt16_one; +extern bool FStar_UInt16_ne(uint16_t a, uint16_t b); + extern uint16_t FStar_UInt16_minus(uint16_t a); extern uint32_t FStar_UInt16_n_minus_one; @@ -180,6 +186,8 @@ extern uint8_t FStar_UInt8_zero; extern uint8_t FStar_UInt8_one; +extern bool FStar_UInt8_ne(uint8_t a, uint8_t b); + extern uint8_t FStar_UInt8_minus(uint8_t a); extern uint32_t FStar_UInt8_n_minus_one; @@ -217,5 +225,5 @@ extern uint8_t FStar_UInt8_of_string(Prims_string uu___); typedef uint8_t FStar_UInt8_byte; -#define __FStar_UInt_8_16_32_64_H_DEFINED -#endif +#define FStar_UInt_8_16_32_64_H_DEFINED +#endif /* FStar_UInt_8_16_32_64_H */ diff --git a/Modules/_hacl/include/krml/internal/target.h b/Modules/_hacl/include/krml/internal/target.h index c592214634a3bd4..73555ab5ca19a43 100644 --- a/Modules/_hacl/include/krml/internal/target.h +++ b/Modules/_hacl/include/krml/internal/target.h @@ -1,8 +1,8 @@ /* Copyright (c) INRIA and Microsoft Corporation. All rights reserved. Licensed under the Apache 2.0 and MIT Licenses. */ -#ifndef __KRML_TARGET_H -#define __KRML_TARGET_H +#ifndef KRML_HEADER_TARGET_H +#define KRML_HEADER_TARGET_H #include <assert.h> #include <inttypes.h> @@ -12,6 +12,9 @@ #include <stdio.h> #include <stdlib.h> +typedef float float32_t; +typedef double float64_t; + /* Since KaRaMeL emits the inline keyword unconditionally, we follow the * guidelines at https://gcc.gnu.org/onlinedocs/gcc/Inline.html and make this * __inline__ to ensure the code compiles with -std=c90 and earlier. */ @@ -425,4 +428,4 @@ inline static int32_t krml_time(void) { #else # define KRML_MAYBE_FOR16(i, z, n, k, x) KRML_ACTUAL_FOR(i, z, n, k, x) #endif -#endif +#endif /* KRML_HEADER_TARGET_H */ diff --git a/Modules/_hacl/include/krml/internal/types.h b/Modules/_hacl/include/krml/internal/types.h index 2280dfad48db1e0..d96ed19fcca5ac7 100644 --- a/Modules/_hacl/include/krml/internal/types.h +++ b/Modules/_hacl/include/krml/internal/types.h @@ -92,7 +92,7 @@ typedef FStar_UInt128_uint128 FStar_UInt128_t, uint128_t; /* Avoid a circular loop: if this header is included via FStar_UInt8_16_32_64, * then don't bring the uint128 definitions into scope. */ -#ifndef __FStar_UInt_8_16_32_64_H +#ifndef FStar_UInt_8_16_32_64_H #if !defined(KRML_VERIFIED_UINT128) && defined(IS_MSVC64) #include "fstar_uint128_msvc.h" diff --git a/Modules/_hacl/include/krml/lowstar_endianness.h b/Modules/_hacl/include/krml/lowstar_endianness.h index af6b882cf259cc5..5f706fa51aad108 100644 --- a/Modules/_hacl/include/krml/lowstar_endianness.h +++ b/Modules/_hacl/include/krml/lowstar_endianness.h @@ -1,8 +1,8 @@ /* Copyright (c) INRIA and Microsoft Corporation. All rights reserved. Licensed under the Apache 2.0 and MIT Licenses. */ -#ifndef __LOWSTAR_ENDIANNESS_H -#define __LOWSTAR_ENDIANNESS_H +#ifndef KRML_HEADER_LOWSTAR_ENDIANNESS_H +#define KRML_HEADER_LOWSTAR_ENDIANNESS_H #include <string.h> #include <inttypes.h> @@ -228,4 +228,4 @@ inline static void store64(uint8_t *b, uint64_t i) { #define load128_be0 load128_be #define store128_be0 store128_be -#endif +#endif /* KRML_HEADER_LOWSTAR_ENDIANNESS_H */ diff --git a/Modules/_hacl/internal/Hacl_HMAC.h b/Modules/_hacl/internal/Hacl_HMAC.h index ad29d50760c3aa8..ef9f25f5d420d34 100644 --- a/Modules/_hacl/internal/Hacl_HMAC.h +++ b/Modules/_hacl/internal/Hacl_HMAC.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_HMAC_H -#define __internal_Hacl_HMAC_H +#ifndef internal_Hacl_HMAC_H +#define internal_Hacl_HMAC_H #if defined(__cplusplus) extern "C" { @@ -48,5 +48,5 @@ K___uint32_t_uint32_t; } #endif -#define __internal_Hacl_HMAC_H_DEFINED -#endif +#define internal_Hacl_HMAC_H_DEFINED +#endif /* internal_Hacl_HMAC_H */ diff --git a/Modules/_hacl/internal/Hacl_Hash_Blake2b.h b/Modules/_hacl/internal/Hacl_Hash_Blake2b.h index e74c320e073f4b0..7ca8e10e34cbbc5 100644 --- a/Modules/_hacl/internal/Hacl_Hash_Blake2b.h +++ b/Modules/_hacl/internal/Hacl_Hash_Blake2b.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Hash_Blake2b_H -#define __internal_Hacl_Hash_Blake2b_H +#ifndef internal_Hacl_Hash_Blake2b_H +#define internal_Hacl_Hash_Blake2b_H #if defined(__cplusplus) extern "C" { @@ -91,5 +91,5 @@ Hacl_Hash_Blake2b_state_t; } #endif -#define __internal_Hacl_Hash_Blake2b_H_DEFINED -#endif +#define internal_Hacl_Hash_Blake2b_H_DEFINED +#endif /* internal_Hacl_Hash_Blake2b_H */ diff --git a/Modules/_hacl/internal/Hacl_Hash_Blake2b_Simd256.h b/Modules/_hacl/internal/Hacl_Hash_Blake2b_Simd256.h index 27633f22b24f0da..6507d287922eeca 100644 --- a/Modules/_hacl/internal/Hacl_Hash_Blake2b_Simd256.h +++ b/Modules/_hacl/internal/Hacl_Hash_Blake2b_Simd256.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Hash_Blake2b_Simd256_H -#define __internal_Hacl_Hash_Blake2b_Simd256_H +#ifndef internal_Hacl_Hash_Blake2b_Simd256_H +#define internal_Hacl_Hash_Blake2b_Simd256_H #if defined(__cplusplus) extern "C" { @@ -134,5 +134,5 @@ Hacl_Hash_Blake2b_Simd256_state_t; } #endif -#define __internal_Hacl_Hash_Blake2b_Simd256_H_DEFINED -#endif +#define internal_Hacl_Hash_Blake2b_Simd256_H_DEFINED +#endif /* internal_Hacl_Hash_Blake2b_Simd256_H */ diff --git a/Modules/_hacl/internal/Hacl_Hash_Blake2s.h b/Modules/_hacl/internal/Hacl_Hash_Blake2s.h index 0c5781df8cea814..5fa5bb34efc0e02 100644 --- a/Modules/_hacl/internal/Hacl_Hash_Blake2s.h +++ b/Modules/_hacl/internal/Hacl_Hash_Blake2s.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Hash_Blake2s_H -#define __internal_Hacl_Hash_Blake2s_H +#ifndef internal_Hacl_Hash_Blake2s_H +#define internal_Hacl_Hash_Blake2s_H #if defined(__cplusplus) extern "C" { @@ -90,5 +90,5 @@ Hacl_Hash_Blake2s_state_t; } #endif -#define __internal_Hacl_Hash_Blake2s_H_DEFINED -#endif +#define internal_Hacl_Hash_Blake2s_H_DEFINED +#endif /* internal_Hacl_Hash_Blake2s_H */ diff --git a/Modules/_hacl/internal/Hacl_Hash_Blake2s_Simd128.h b/Modules/_hacl/internal/Hacl_Hash_Blake2s_Simd128.h index 0f5db552d6cfedc..3220cddac302888 100644 --- a/Modules/_hacl/internal/Hacl_Hash_Blake2s_Simd128.h +++ b/Modules/_hacl/internal/Hacl_Hash_Blake2s_Simd128.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Hash_Blake2s_Simd128_H -#define __internal_Hacl_Hash_Blake2s_Simd128_H +#ifndef internal_Hacl_Hash_Blake2s_Simd128_H +#define internal_Hacl_Hash_Blake2s_Simd128_H #if defined(__cplusplus) extern "C" { @@ -134,5 +134,5 @@ Hacl_Hash_Blake2s_Simd128_state_t; } #endif -#define __internal_Hacl_Hash_Blake2s_Simd128_H_DEFINED -#endif +#define internal_Hacl_Hash_Blake2s_Simd128_H_DEFINED +#endif /* internal_Hacl_Hash_Blake2s_Simd128_H */ diff --git a/Modules/_hacl/internal/Hacl_Hash_MD5.h b/Modules/_hacl/internal/Hacl_Hash_MD5.h index 7fe71a49c6df853..d8761de99f5e2a7 100644 --- a/Modules/_hacl/internal/Hacl_Hash_MD5.h +++ b/Modules/_hacl/internal/Hacl_Hash_MD5.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Hash_MD5_H -#define __internal_Hacl_Hash_MD5_H +#ifndef internal_Hacl_Hash_MD5_H +#define internal_Hacl_Hash_MD5_H #if defined(__cplusplus) extern "C" { @@ -53,5 +53,5 @@ void Hacl_Hash_MD5_hash_oneshot(uint8_t *output, uint8_t *input, uint32_t input_ } #endif -#define __internal_Hacl_Hash_MD5_H_DEFINED -#endif +#define internal_Hacl_Hash_MD5_H_DEFINED +#endif /* internal_Hacl_Hash_MD5_H */ diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA1.h b/Modules/_hacl/internal/Hacl_Hash_SHA1.h index ed53be559fe4659..0e9f1c911f61657 100644 --- a/Modules/_hacl/internal/Hacl_Hash_SHA1.h +++ b/Modules/_hacl/internal/Hacl_Hash_SHA1.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Hash_SHA1_H -#define __internal_Hacl_Hash_SHA1_H +#ifndef internal_Hacl_Hash_SHA1_H +#define internal_Hacl_Hash_SHA1_H #if defined(__cplusplus) extern "C" { @@ -52,5 +52,5 @@ void Hacl_Hash_SHA1_hash_oneshot(uint8_t *output, uint8_t *input, uint32_t input } #endif -#define __internal_Hacl_Hash_SHA1_H_DEFINED -#endif +#define internal_Hacl_Hash_SHA1_H_DEFINED +#endif /* internal_Hacl_Hash_SHA1_H */ diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA2.h b/Modules/_hacl/internal/Hacl_Hash_SHA2.h index 98498ee93769961..e6750207980aef9 100644 --- a/Modules/_hacl/internal/Hacl_Hash_SHA2.h +++ b/Modules/_hacl/internal/Hacl_Hash_SHA2.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Hash_SHA2_H -#define __internal_Hacl_Hash_SHA2_H +#ifndef internal_Hacl_Hash_SHA2_H +#define internal_Hacl_Hash_SHA2_H #if defined(__cplusplus) extern "C" { @@ -161,5 +161,5 @@ void Hacl_Hash_SHA2_sha384_finish(uint64_t *st, uint8_t *h); } #endif -#define __internal_Hacl_Hash_SHA2_H_DEFINED -#endif +#define internal_Hacl_Hash_SHA2_H_DEFINED +#endif /* internal_Hacl_Hash_SHA2_H */ diff --git a/Modules/_hacl/internal/Hacl_Hash_SHA3.h b/Modules/_hacl/internal/Hacl_Hash_SHA3.h index e653c73b1d03f72..c08a4e7858bbabe 100644 --- a/Modules/_hacl/internal/Hacl_Hash_SHA3.h +++ b/Modules/_hacl/internal/Hacl_Hash_SHA3.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Hash_SHA3_H -#define __internal_Hacl_Hash_SHA3_H +#ifndef internal_Hacl_Hash_SHA3_H +#define internal_Hacl_Hash_SHA3_H #if defined(__cplusplus) extern "C" { @@ -81,5 +81,5 @@ Hacl_Hash_SHA3_state_t; } #endif -#define __internal_Hacl_Hash_SHA3_H_DEFINED -#endif +#define internal_Hacl_Hash_SHA3_H_DEFINED +#endif /* internal_Hacl_Hash_SHA3_H */ diff --git a/Modules/_hacl/internal/Hacl_Impl_Blake2_Constants.h b/Modules/_hacl/internal/Hacl_Impl_Blake2_Constants.h index fb3a045cd5c608f..2726cc484780e8c 100644 --- a/Modules/_hacl/internal/Hacl_Impl_Blake2_Constants.h +++ b/Modules/_hacl/internal/Hacl_Impl_Blake2_Constants.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Impl_Blake2_Constants_H -#define __internal_Hacl_Impl_Blake2_Constants_H +#ifndef internal_Hacl_Impl_Blake2_Constants_H +#define internal_Hacl_Impl_Blake2_Constants_H #if defined(__cplusplus) extern "C" { @@ -69,5 +69,5 @@ Hacl_Hash_Blake2b_ivTable_B[8U] = } #endif -#define __internal_Hacl_Impl_Blake2_Constants_H_DEFINED -#endif +#define internal_Hacl_Impl_Blake2_Constants_H_DEFINED +#endif /* internal_Hacl_Impl_Blake2_Constants_H */ diff --git a/Modules/_hacl/internal/Hacl_Streaming_HMAC.h b/Modules/_hacl/internal/Hacl_Streaming_HMAC.h index acc4f3996026ee7..fb44f707463e044 100644 --- a/Modules/_hacl/internal/Hacl_Streaming_HMAC.h +++ b/Modules/_hacl/internal/Hacl_Streaming_HMAC.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Streaming_HMAC_H -#define __internal_Hacl_Streaming_HMAC_H +#ifndef internal_Hacl_Streaming_HMAC_H +#define internal_Hacl_Streaming_HMAC_H #if defined(__cplusplus) extern "C" { @@ -90,5 +90,5 @@ Hacl_Streaming_HMAC_agile_state; } #endif -#define __internal_Hacl_Streaming_HMAC_H_DEFINED -#endif +#define internal_Hacl_Streaming_HMAC_H_DEFINED +#endif /* internal_Hacl_Streaming_HMAC_H */ diff --git a/Modules/_hacl/internal/Hacl_Streaming_Types.h b/Modules/_hacl/internal/Hacl_Streaming_Types.h index fed3cbd425917cf..d6c4fc0d6255f73 100644 --- a/Modules/_hacl/internal/Hacl_Streaming_Types.h +++ b/Modules/_hacl/internal/Hacl_Streaming_Types.h @@ -23,8 +23,8 @@ */ -#ifndef __internal_Hacl_Streaming_Types_H -#define __internal_Hacl_Streaming_Types_H +#ifndef internal_Hacl_Streaming_Types_H +#define internal_Hacl_Streaming_Types_H #if defined(__cplusplus) extern "C" { @@ -83,5 +83,5 @@ Hacl_Streaming_MD_state_64; } #endif -#define __internal_Hacl_Streaming_Types_H_DEFINED -#endif +#define internal_Hacl_Streaming_Types_H_DEFINED +#endif /* internal_Hacl_Streaming_Types_H */ diff --git a/Modules/_hacl/python_hacl_namespaces.h b/Modules/_hacl/python_hacl_namespaces.h index 1c2f7fea5c837a1..d0b4500395e7c3b 100644 --- a/Modules/_hacl/python_hacl_namespaces.h +++ b/Modules/_hacl/python_hacl_namespaces.h @@ -2,95 +2,43 @@ #define _PYTHON_HACL_NAMESPACES_H /* - * C's excuse for namespaces: Use globally unique names to avoid linkage - * conflicts with builds linking or dynamically loading other code potentially - * using HACL* libraries. + * Use globally unique names to avoid linkage conflicts with builds linking + * or dynamically loading other code potentially using HACL* libraries. * - * Something like this to generate new entries for the list: - * - * nm *.o | grep Hacl | cut -c 20- | sort | uniq | grep -v _Py_LibHacl_ | egrep ^_ | sed 's/_\(.*\)/#define \1 _Py_LibHacl_\1/g' - */ + * Assuming that the current working directory is Modules/_hacl, + * use the following command to generate a list of candidates: -#define Lib_Memzero0_memzero0 _Py_LibHacl_Lib_Memzero0_memzero0 + nm -j *.o | grep -i hacl | grep -P '^[a-zA-Z_][a-zA-Z0-9_]+' \ + | sed -e 's/^_Py_LibHacl_//g' \ + | sed 's/\(.*\)/#define \1 _Py_LibHacl_\1/g' \ + | sort -u -#define Hacl_Hash_SHA2_state_sha2_224_s _Py_LibHacl_Hacl_Hash_SHA2_state_sha2_224_s -#define Hacl_Hash_SHA2_state_sha2_224 _Py_LibHacl_Hacl_Hash_SHA2_state_sha2_224 -#define Hacl_Hash_SHA2_state_sha2_256 _Py_LibHacl_Hacl_Hash_SHA2_state_sha2_256 -#define Hacl_Hash_SHA2_state_sha2_384_s _Py_LibHacl_Hacl_Hash_SHA2_state_sha2_384_s -#define Hacl_Hash_SHA2_state_sha2_384 _Py_LibHacl_Hacl_Hash_SHA2_state_sha2_384 -#define Hacl_Hash_SHA2_state_sha2_512 _Py_LibHacl_Hacl_Hash_SHA2_state_sha2_512 -#define Hacl_Hash_SHA2_malloc_256 _Py_LibHacl_Hacl_Hash_SHA2_malloc_256 -#define Hacl_Hash_SHA2_malloc_224 _Py_LibHacl_Hacl_Hash_SHA2_malloc_224 -#define Hacl_Hash_SHA2_malloc_512 _Py_LibHacl_Hacl_Hash_SHA2_malloc_512 -#define Hacl_Hash_SHA2_malloc_384 _Py_LibHacl_Hacl_Hash_SHA2_malloc_384 -#define Hacl_Hash_SHA2_copy_256 _Py_LibHacl_Hacl_Hash_SHA2_copy_256 -#define Hacl_Hash_SHA2_copy_224 _Py_LibHacl_Hacl_Hash_SHA2_copy_224 -#define Hacl_Hash_SHA2_copy_512 _Py_LibHacl_Hacl_Hash_SHA2_copy_512 -#define Hacl_Hash_SHA2_copy_384 _Py_LibHacl_Hacl_Hash_SHA2_copy_384 -#define Hacl_Hash_SHA2_init_256 _Py_LibHacl_Hacl_Hash_SHA2_init_256 -#define Hacl_Hash_SHA2_init_224 _Py_LibHacl_Hacl_Hash_SHA2_init_224 -#define Hacl_Hash_SHA2_init_512 _Py_LibHacl_Hacl_Hash_SHA2_init_512 -#define Hacl_Hash_SHA2_init_384 _Py_LibHacl_Hacl_Hash_SHA2_init_384 -#define Hacl_SHA2_Scalar32_sha512_init _Py_LibHacl_Hacl_SHA2_Scalar32_sha512_init -#define Hacl_Hash_SHA2_update_256 _Py_LibHacl_Hacl_Hash_SHA2_update_256 -#define Hacl_Hash_SHA2_update_224 _Py_LibHacl_Hacl_Hash_SHA2_update_224 -#define Hacl_Hash_SHA2_update_512 _Py_LibHacl_Hacl_Hash_SHA2_update_512 -#define Hacl_Hash_SHA2_update_384 _Py_LibHacl_Hacl_Hash_SHA2_update_384 -#define Hacl_Hash_SHA2_digest_256 _Py_LibHacl_Hacl_Hash_SHA2_digest_256 -#define Hacl_Hash_SHA2_digest_224 _Py_LibHacl_Hacl_Hash_SHA2_digest_224 -#define Hacl_Hash_SHA2_digest_512 _Py_LibHacl_Hacl_Hash_SHA2_digest_512 -#define Hacl_Hash_SHA2_digest_384 _Py_LibHacl_Hacl_Hash_SHA2_digest_384 -#define Hacl_Hash_SHA2_free_256 _Py_LibHacl_Hacl_Hash_SHA2_free_256 -#define Hacl_Hash_SHA2_free_224 _Py_LibHacl_Hacl_Hash_SHA2_free_224 -#define Hacl_Hash_SHA2_free_512 _Py_LibHacl_Hacl_Hash_SHA2_free_512 -#define Hacl_Hash_SHA2_free_384 _Py_LibHacl_Hacl_Hash_SHA2_free_384 -#define Hacl_Hash_SHA2_sha256 _Py_LibHacl_Hacl_Hash_SHA2_sha256 -#define Hacl_Hash_SHA2_sha224 _Py_LibHacl_Hacl_Hash_SHA2_sha224 -#define Hacl_Hash_SHA2_sha512 _Py_LibHacl_Hacl_Hash_SHA2_sha512 -#define Hacl_Hash_SHA2_sha384 _Py_LibHacl_Hacl_Hash_SHA2_sha384 + * Compare the entries to add as follows: -#define Hacl_Hash_MD5_malloc _Py_LibHacl_Hacl_Hash_MD5_malloc -#define Hacl_Hash_MD5_init _Py_LibHacl_Hacl_Hash_MD5_init -#define Hacl_Hash_MD5_update _Py_LibHacl_Hacl_Hash_MD5_update -#define Hacl_Hash_MD5_digest _Py_LibHacl_Hacl_Hash_MD5_digest -#define Hacl_Hash_MD5_free _Py_LibHacl_Hacl_Hash_MD5_free -#define Hacl_Hash_MD5_copy _Py_LibHacl_Hacl_Hash_MD5_copy -#define Hacl_Hash_MD5_hash _Py_LibHacl_Hacl_Hash_MD5_hash - -#define Hacl_Hash_SHA1_malloc _Py_LibHacl_Hacl_Hash_SHA1_malloc -#define Hacl_Hash_SHA1_init _Py_LibHacl_Hacl_Hash_SHA1_init -#define Hacl_Hash_SHA1_update _Py_LibHacl_Hacl_Hash_SHA1_update -#define Hacl_Hash_SHA1_digest _Py_LibHacl_Hacl_Hash_SHA1_digest -#define Hacl_Hash_SHA1_free _Py_LibHacl_Hacl_Hash_SHA1_free -#define Hacl_Hash_SHA1_copy _Py_LibHacl_Hacl_Hash_SHA1_copy -#define Hacl_Hash_SHA1_hash _Py_LibHacl_Hacl_Hash_SHA1_hash - -#define Hacl_Hash_SHA3_update_last_sha3 _Py_LibHacl_Hacl_Hash_SHA3_update_last_sha3 -#define Hacl_Hash_SHA3_update_multi_sha3 _Py_LibHacl_Hacl_Hash_SHA3_update_multi_sha3 -#define Hacl_Impl_SHA3_absorb_inner _Py_LibHacl_Hacl_Impl_SHA3_absorb_inner -#define Hacl_Impl_SHA3_keccak _Py_LibHacl_Hacl_Impl_SHA3_keccak -#define Hacl_Impl_SHA3_loadState _Py_LibHacl_Hacl_Impl_SHA3_loadState -#define Hacl_Impl_SHA3_squeeze _Py_LibHacl_Hacl_Impl_SHA3_squeeze -#define Hacl_Impl_SHA3_state_permute _Py_LibHacl_Hacl_Impl_SHA3_state_permute -#define Hacl_SHA3_sha3_224 _Py_LibHacl_Hacl_SHA3_sha3_224 -#define Hacl_SHA3_sha3_256 _Py_LibHacl_Hacl_SHA3_sha3_256 -#define Hacl_SHA3_sha3_384 _Py_LibHacl_Hacl_SHA3_sha3_384 -#define Hacl_SHA3_sha3_512 _Py_LibHacl_Hacl_SHA3_sha3_512 -#define Hacl_SHA3_shake128_hacl _Py_LibHacl_Hacl_SHA3_shake128_hacl -#define Hacl_SHA3_shake256_hacl _Py_LibHacl_Hacl_SHA3_shake256_hacl -#define Hacl_Hash_SHA3_block_len _Py_LibHacl_Hacl_Hash_SHA3_block_len -#define Hacl_Hash_SHA3_copy _Py_LibHacl_Hacl_Hash_SHA3_copy -#define Hacl_Hash_SHA3_digest _Py_LibHacl_Hacl_Hash_SHA3_digest -#define Hacl_Hash_SHA3_free _Py_LibHacl_Hacl_Hash_SHA3_free -#define Hacl_Hash_SHA3_get_alg _Py_LibHacl_Hacl_Hash_SHA3_get_alg -#define Hacl_Hash_SHA3_hash_len _Py_LibHacl_Hacl_Hash_SHA3_hash_len -#define Hacl_Hash_SHA3_is_shake _Py_LibHacl_Hacl_Hash_SHA3_is_shake -#define Hacl_Hash_SHA3_init_ _Py_LibHacl_Hacl_Hash_SHA3_init_ -#define Hacl_Hash_SHA3_malloc _Py_LibHacl_Hacl_Hash_SHA3_malloc -#define Hacl_Hash_SHA3_reset _Py_LibHacl_Hacl_Hash_SHA3_reset -#define Hacl_Hash_SHA3_update _Py_LibHacl_Hacl_Hash_SHA3_update -#define Hacl_Hash_SHA3_squeeze _Py_LibHacl_Hacl_Hash_SHA3_squeeze + diff -y --suppress-common-lines \ + <(grep -P '^#define (?!_PY.+_H)' python_hacl_namespaces.h | sort -u) \ + <(nm -j *.o | grep -i hacl | grep -P '^[a-zA-Z_][a-zA-Z0-9_]+' \ + | sed -e 's/^_Py_LibHacl_//g' \ + | sed 's/\(.*\)/#define \1 _Py_LibHacl_\1/g' | sort -u) + */ +// --- Utils ------------------------------------------------------------------ +#define Lib_Memzero0_memzero0 _Py_LibHacl_Lib_Memzero0_memzero0 +// --- HASH-BLAKE-2b ---------------------------------------------------------- +#define Hacl_Hash_Blake2b_copy _Py_LibHacl_Hacl_Hash_Blake2b_copy +#define Hacl_Hash_Blake2b_digest _Py_LibHacl_Hacl_Hash_Blake2b_digest +#define Hacl_Hash_Blake2b_finish _Py_LibHacl_Hacl_Hash_Blake2b_finish +#define Hacl_Hash_Blake2b_free _Py_LibHacl_Hacl_Hash_Blake2b_free +#define Hacl_Hash_Blake2b_hash_with_key _Py_LibHacl_Hacl_Hash_Blake2b_hash_with_key +#define Hacl_Hash_Blake2b_hash_with_key_and_params _Py_LibHacl_Hacl_Hash_Blake2b_hash_with_key_and_params +#define Hacl_Hash_Blake2b_info _Py_LibHacl_Hacl_Hash_Blake2b_info +#define Hacl_Hash_Blake2b_init _Py_LibHacl_Hacl_Hash_Blake2b_init +#define Hacl_Hash_Blake2b_malloc _Py_LibHacl_Hacl_Hash_Blake2b_malloc +#define Hacl_Hash_Blake2b_malloc_with_key _Py_LibHacl_Hacl_Hash_Blake2b_malloc_with_key +#define Hacl_Hash_Blake2b_malloc_with_params_and_key _Py_LibHacl_Hacl_Hash_Blake2b_malloc_with_params_and_key +#define Hacl_Hash_Blake2b_reset _Py_LibHacl_Hacl_Hash_Blake2b_reset +#define Hacl_Hash_Blake2b_reset_with_key _Py_LibHacl_Hacl_Hash_Blake2b_reset_with_key +#define Hacl_Hash_Blake2b_reset_with_key_and_params _Py_LibHacl_Hacl_Hash_Blake2b_reset_with_key_and_params #define Hacl_Hash_Blake2b_Simd256_copy _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_copy #define Hacl_Hash_Blake2b_Simd256_copy_internal_state _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_copy_internal_state #define Hacl_Hash_Blake2b_Simd256_digest _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_digest @@ -104,7 +52,6 @@ #define Hacl_Hash_Blake2b_Simd256_malloc _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_malloc #define Hacl_Hash_Blake2b_Simd256_malloc_internal_state_with_key _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_malloc_internal_state_with_key #define Hacl_Hash_Blake2b_Simd256_malloc_with_key _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_malloc_with_key -#define Hacl_Hash_Blake2b_Simd256_malloc_with_key0 _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_malloc_with_key0 #define Hacl_Hash_Blake2b_Simd256_malloc_with_params_and_key _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_malloc_with_params_and_key #define Hacl_Hash_Blake2b_Simd256_reset _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_reset #define Hacl_Hash_Blake2b_Simd256_reset_with_key _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_reset_with_key @@ -115,23 +62,24 @@ #define Hacl_Hash_Blake2b_Simd256_update_last_no_inline _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_update_last_no_inline #define Hacl_Hash_Blake2b_Simd256_update_multi _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_update_multi #define Hacl_Hash_Blake2b_Simd256_update_multi_no_inline _Py_LibHacl_Hacl_Hash_Blake2b_Simd256_update_multi_no_inline -#define Hacl_Hash_Blake2b_copy _Py_LibHacl_Hacl_Hash_Blake2b_copy -#define Hacl_Hash_Blake2b_digest _Py_LibHacl_Hacl_Hash_Blake2b_digest -#define Hacl_Hash_Blake2b_finish _Py_LibHacl_Hacl_Hash_Blake2b_finish -#define Hacl_Hash_Blake2b_free _Py_LibHacl_Hacl_Hash_Blake2b_free -#define Hacl_Hash_Blake2b_hash_with_key _Py_LibHacl_Hacl_Hash_Blake2b_hash_with_key -#define Hacl_Hash_Blake2b_hash_with_key_and_params _Py_LibHacl_Hacl_Hash_Blake2b_hash_with_key_and_params -#define Hacl_Hash_Blake2b_info _Py_LibHacl_Hacl_Hash_Blake2b_info -#define Hacl_Hash_Blake2b_init _Py_LibHacl_Hacl_Hash_Blake2b_init -#define Hacl_Hash_Blake2b_malloc _Py_LibHacl_Hacl_Hash_Blake2b_malloc -#define Hacl_Hash_Blake2b_malloc_with_key _Py_LibHacl_Hacl_Hash_Blake2b_malloc_with_key -#define Hacl_Hash_Blake2b_malloc_with_params_and_key _Py_LibHacl_Hacl_Hash_Blake2b_malloc_with_params_and_key -#define Hacl_Hash_Blake2b_reset _Py_LibHacl_Hacl_Hash_Blake2b_reset -#define Hacl_Hash_Blake2b_reset_with_key _Py_LibHacl_Hacl_Hash_Blake2b_reset_with_key -#define Hacl_Hash_Blake2b_reset_with_key_and_params _Py_LibHacl_Hacl_Hash_Blake2b_reset_with_key_and_params #define Hacl_Hash_Blake2b_update _Py_LibHacl_Hacl_Hash_Blake2b_update #define Hacl_Hash_Blake2b_update_last _Py_LibHacl_Hacl_Hash_Blake2b_update_last #define Hacl_Hash_Blake2b_update_multi _Py_LibHacl_Hacl_Hash_Blake2b_update_multi +// --- HASH-BLAKE-2s ---------------------------------------------------------- +#define Hacl_Hash_Blake2s_copy _Py_LibHacl_Hacl_Hash_Blake2s_copy +#define Hacl_Hash_Blake2s_digest _Py_LibHacl_Hacl_Hash_Blake2s_digest +#define Hacl_Hash_Blake2s_finish _Py_LibHacl_Hacl_Hash_Blake2s_finish +#define Hacl_Hash_Blake2s_free _Py_LibHacl_Hacl_Hash_Blake2s_free +#define Hacl_Hash_Blake2s_hash_with_key _Py_LibHacl_Hacl_Hash_Blake2s_hash_with_key +#define Hacl_Hash_Blake2s_hash_with_key_and_params _Py_LibHacl_Hacl_Hash_Blake2s_hash_with_key_and_params +#define Hacl_Hash_Blake2s_info _Py_LibHacl_Hacl_Hash_Blake2s_info +#define Hacl_Hash_Blake2s_init _Py_LibHacl_Hacl_Hash_Blake2s_init +#define Hacl_Hash_Blake2s_malloc _Py_LibHacl_Hacl_Hash_Blake2s_malloc +#define Hacl_Hash_Blake2s_malloc_with_key _Py_LibHacl_Hacl_Hash_Blake2s_malloc_with_key +#define Hacl_Hash_Blake2s_malloc_with_params_and_key _Py_LibHacl_Hacl_Hash_Blake2s_malloc_with_params_and_key +#define Hacl_Hash_Blake2s_reset _Py_LibHacl_Hacl_Hash_Blake2s_reset +#define Hacl_Hash_Blake2s_reset_with_key _Py_LibHacl_Hacl_Hash_Blake2s_reset_with_key +#define Hacl_Hash_Blake2s_reset_with_key_and_params _Py_LibHacl_Hacl_Hash_Blake2s_reset_with_key_and_params #define Hacl_Hash_Blake2s_Simd128_copy _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_copy #define Hacl_Hash_Blake2s_Simd128_copy_internal_state _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_copy_internal_state #define Hacl_Hash_Blake2s_Simd128_digest _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_digest @@ -145,7 +93,6 @@ #define Hacl_Hash_Blake2s_Simd128_malloc _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_malloc #define Hacl_Hash_Blake2s_Simd128_malloc_internal_state_with_key _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_malloc_internal_state_with_key #define Hacl_Hash_Blake2s_Simd128_malloc_with_key _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_malloc_with_key -#define Hacl_Hash_Blake2s_Simd128_malloc_with_key0 _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_malloc_with_key0 #define Hacl_Hash_Blake2s_Simd128_malloc_with_params_and_key _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_malloc_with_params_and_key #define Hacl_Hash_Blake2s_Simd128_reset _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_reset #define Hacl_Hash_Blake2s_Simd128_reset_with_key _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_reset_with_key @@ -156,37 +103,54 @@ #define Hacl_Hash_Blake2s_Simd128_update_last_no_inline _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_update_last_no_inline #define Hacl_Hash_Blake2s_Simd128_update_multi _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_update_multi #define Hacl_Hash_Blake2s_Simd128_update_multi_no_inline _Py_LibHacl_Hacl_Hash_Blake2s_Simd128_update_multi_no_inline -#define Hacl_Hash_Blake2s_copy _Py_LibHacl_Hacl_Hash_Blake2s_copy -#define Hacl_Hash_Blake2s_digest _Py_LibHacl_Hacl_Hash_Blake2s_digest -#define Hacl_Hash_Blake2s_finish _Py_LibHacl_Hacl_Hash_Blake2s_finish -#define Hacl_Hash_Blake2s_free _Py_LibHacl_Hacl_Hash_Blake2s_free -#define Hacl_Hash_Blake2s_hash_with_key _Py_LibHacl_Hacl_Hash_Blake2s_hash_with_key -#define Hacl_Hash_Blake2s_hash_with_key_and_params _Py_LibHacl_Hacl_Hash_Blake2s_hash_with_key_and_params -#define Hacl_Hash_Blake2s_info _Py_LibHacl_Hacl_Hash_Blake2s_info -#define Hacl_Hash_Blake2s_init _Py_LibHacl_Hacl_Hash_Blake2s_init -#define Hacl_Hash_Blake2s_malloc _Py_LibHacl_Hacl_Hash_Blake2s_malloc -#define Hacl_Hash_Blake2s_malloc_with_key _Py_LibHacl_Hacl_Hash_Blake2s_malloc_with_key -#define Hacl_Hash_Blake2s_malloc_with_params_and_key _Py_LibHacl_Hacl_Hash_Blake2s_malloc_with_params_and_key -#define Hacl_Hash_Blake2s_reset _Py_LibHacl_Hacl_Hash_Blake2s_reset -#define Hacl_Hash_Blake2s_reset_with_key _Py_LibHacl_Hacl_Hash_Blake2s_reset_with_key -#define Hacl_Hash_Blake2s_reset_with_key_and_params _Py_LibHacl_Hacl_Hash_Blake2s_reset_with_key_and_params #define Hacl_Hash_Blake2s_update _Py_LibHacl_Hacl_Hash_Blake2s_update #define Hacl_Hash_Blake2s_update_last _Py_LibHacl_Hacl_Hash_Blake2s_update_last #define Hacl_Hash_Blake2s_update_multi _Py_LibHacl_Hacl_Hash_Blake2s_update_multi +// --- HASH-MD5 --------------------------------------------------------------- +#define Hacl_Hash_MD5_copy _Py_LibHacl_Hacl_Hash_MD5_copy +#define Hacl_Hash_MD5_digest _Py_LibHacl_Hacl_Hash_MD5_digest #define Hacl_Hash_MD5_finish _Py_LibHacl_Hacl_Hash_MD5_finish +#define Hacl_Hash_MD5_free _Py_LibHacl_Hacl_Hash_MD5_free +#define Hacl_Hash_MD5_hash _Py_LibHacl_Hacl_Hash_MD5_hash #define Hacl_Hash_MD5_hash_oneshot _Py_LibHacl_Hacl_Hash_MD5_hash_oneshot +#define Hacl_Hash_MD5_init _Py_LibHacl_Hacl_Hash_MD5_init +#define Hacl_Hash_MD5_malloc _Py_LibHacl_Hacl_Hash_MD5_malloc #define Hacl_Hash_MD5_reset _Py_LibHacl_Hacl_Hash_MD5_reset +#define Hacl_Hash_MD5_update _Py_LibHacl_Hacl_Hash_MD5_update #define Hacl_Hash_MD5_update_last _Py_LibHacl_Hacl_Hash_MD5_update_last #define Hacl_Hash_MD5_update_multi _Py_LibHacl_Hacl_Hash_MD5_update_multi +// --- HASH-SHA-1 ------------------------------------------------------------- +#define Hacl_Hash_SHA1_copy _Py_LibHacl_Hacl_Hash_SHA1_copy +#define Hacl_Hash_SHA1_digest _Py_LibHacl_Hacl_Hash_SHA1_digest #define Hacl_Hash_SHA1_finish _Py_LibHacl_Hacl_Hash_SHA1_finish +#define Hacl_Hash_SHA1_free _Py_LibHacl_Hacl_Hash_SHA1_free +#define Hacl_Hash_SHA1_hash _Py_LibHacl_Hacl_Hash_SHA1_hash #define Hacl_Hash_SHA1_hash_oneshot _Py_LibHacl_Hacl_Hash_SHA1_hash_oneshot +#define Hacl_Hash_SHA1_init _Py_LibHacl_Hacl_Hash_SHA1_init +#define Hacl_Hash_SHA1_malloc _Py_LibHacl_Hacl_Hash_SHA1_malloc #define Hacl_Hash_SHA1_reset _Py_LibHacl_Hacl_Hash_SHA1_reset +#define Hacl_Hash_SHA1_update _Py_LibHacl_Hacl_Hash_SHA1_update #define Hacl_Hash_SHA1_update_last _Py_LibHacl_Hacl_Hash_SHA1_update_last #define Hacl_Hash_SHA1_update_multi _Py_LibHacl_Hacl_Hash_SHA1_update_multi +// --- HASH-SHA-2 ------------------------------------------------------------- +#define Hacl_Hash_SHA2_copy_256 _Py_LibHacl_Hacl_Hash_SHA2_copy_256 +#define Hacl_Hash_SHA2_copy_512 _Py_LibHacl_Hacl_Hash_SHA2_copy_512 +#define Hacl_Hash_SHA2_digest_224 _Py_LibHacl_Hacl_Hash_SHA2_digest_224 +#define Hacl_Hash_SHA2_digest_256 _Py_LibHacl_Hacl_Hash_SHA2_digest_256 +#define Hacl_Hash_SHA2_digest_384 _Py_LibHacl_Hacl_Hash_SHA2_digest_384 +#define Hacl_Hash_SHA2_digest_512 _Py_LibHacl_Hacl_Hash_SHA2_digest_512 +#define Hacl_Hash_SHA2_free_224 _Py_LibHacl_Hacl_Hash_SHA2_free_224 +#define Hacl_Hash_SHA2_free_256 _Py_LibHacl_Hacl_Hash_SHA2_free_256 +#define Hacl_Hash_SHA2_free_384 _Py_LibHacl_Hacl_Hash_SHA2_free_384 +#define Hacl_Hash_SHA2_free_512 _Py_LibHacl_Hacl_Hash_SHA2_free_512 #define Hacl_Hash_SHA2_hash_224 _Py_LibHacl_Hacl_Hash_SHA2_hash_224 #define Hacl_Hash_SHA2_hash_256 _Py_LibHacl_Hacl_Hash_SHA2_hash_256 #define Hacl_Hash_SHA2_hash_384 _Py_LibHacl_Hacl_Hash_SHA2_hash_384 #define Hacl_Hash_SHA2_hash_512 _Py_LibHacl_Hacl_Hash_SHA2_hash_512 +#define Hacl_Hash_SHA2_malloc_224 _Py_LibHacl_Hacl_Hash_SHA2_malloc_224 +#define Hacl_Hash_SHA2_malloc_256 _Py_LibHacl_Hacl_Hash_SHA2_malloc_256 +#define Hacl_Hash_SHA2_malloc_384 _Py_LibHacl_Hacl_Hash_SHA2_malloc_384 +#define Hacl_Hash_SHA2_malloc_512 _Py_LibHacl_Hacl_Hash_SHA2_malloc_512 #define Hacl_Hash_SHA2_reset_224 _Py_LibHacl_Hacl_Hash_SHA2_reset_224 #define Hacl_Hash_SHA2_reset_256 _Py_LibHacl_Hacl_Hash_SHA2_reset_256 #define Hacl_Hash_SHA2_reset_384 _Py_LibHacl_Hacl_Hash_SHA2_reset_384 @@ -207,10 +171,25 @@ #define Hacl_Hash_SHA2_sha512_init _Py_LibHacl_Hacl_Hash_SHA2_sha512_init #define Hacl_Hash_SHA2_sha512_update_last _Py_LibHacl_Hacl_Hash_SHA2_sha512_update_last #define Hacl_Hash_SHA2_sha512_update_nblocks _Py_LibHacl_Hacl_Hash_SHA2_sha512_update_nblocks +#define Hacl_Hash_SHA2_update_224 _Py_LibHacl_Hacl_Hash_SHA2_update_224 +#define Hacl_Hash_SHA2_update_256 _Py_LibHacl_Hacl_Hash_SHA2_update_256 +#define Hacl_Hash_SHA2_update_384 _Py_LibHacl_Hacl_Hash_SHA2_update_384 +#define Hacl_Hash_SHA2_update_512 _Py_LibHacl_Hacl_Hash_SHA2_update_512 +// --- HASH-SHA-3 ------------------------------------------------------------- #define Hacl_Hash_SHA3_absorb_inner_32 _Py_LibHacl_Hacl_Hash_SHA3_absorb_inner_32 +#define Hacl_Hash_SHA3_block_len _Py_LibHacl_Hacl_Hash_SHA3_block_len +#define Hacl_Hash_SHA3_copy _Py_LibHacl_Hacl_Hash_SHA3_copy +#define Hacl_Hash_SHA3_digest _Py_LibHacl_Hacl_Hash_SHA3_digest +#define Hacl_Hash_SHA3_free _Py_LibHacl_Hacl_Hash_SHA3_free +#define Hacl_Hash_SHA3_get_alg _Py_LibHacl_Hacl_Hash_SHA3_get_alg +#define Hacl_Hash_SHA3_hash_len _Py_LibHacl_Hacl_Hash_SHA3_hash_len +#define Hacl_Hash_SHA3_init_ _Py_LibHacl_Hacl_Hash_SHA3_init_ +#define Hacl_Hash_SHA3_is_shake _Py_LibHacl_Hacl_Hash_SHA3_is_shake #define Hacl_Hash_SHA3_keccak_piln _Py_LibHacl_Hacl_Hash_SHA3_keccak_piln #define Hacl_Hash_SHA3_keccak_rndc _Py_LibHacl_Hacl_Hash_SHA3_keccak_rndc #define Hacl_Hash_SHA3_keccak_rotc _Py_LibHacl_Hacl_Hash_SHA3_keccak_rotc +#define Hacl_Hash_SHA3_malloc _Py_LibHacl_Hacl_Hash_SHA3_malloc +#define Hacl_Hash_SHA3_reset _Py_LibHacl_Hacl_Hash_SHA3_reset #define Hacl_Hash_SHA3_sha3_224 _Py_LibHacl_Hacl_Hash_SHA3_sha3_224 #define Hacl_Hash_SHA3_sha3_256 _Py_LibHacl_Hacl_Hash_SHA3_sha3_256 #define Hacl_Hash_SHA3_sha3_384 _Py_LibHacl_Hacl_Hash_SHA3_sha3_384 @@ -220,37 +199,39 @@ #define Hacl_Hash_SHA3_shake128_absorb_nblocks _Py_LibHacl_Hacl_Hash_SHA3_shake128_absorb_nblocks #define Hacl_Hash_SHA3_shake128_squeeze_nblocks _Py_LibHacl_Hacl_Hash_SHA3_shake128_squeeze_nblocks #define Hacl_Hash_SHA3_shake256 _Py_LibHacl_Hacl_Hash_SHA3_shake256 +#define Hacl_Hash_SHA3_squeeze _Py_LibHacl_Hacl_Hash_SHA3_squeeze #define Hacl_Hash_SHA3_state_free _Py_LibHacl_Hacl_Hash_SHA3_state_free #define Hacl_Hash_SHA3_state_malloc _Py_LibHacl_Hacl_Hash_SHA3_state_malloc - -// Streaming HMAC +#define Hacl_Hash_SHA3_update _Py_LibHacl_Hacl_Hash_SHA3_update +#define Hacl_Hash_SHA3_update_last_sha3 _Py_LibHacl_Hacl_Hash_SHA3_update_last_sha3 +#define Hacl_Hash_SHA3_update_multi_sha3 _Py_LibHacl_Hacl_Hash_SHA3_update_multi_sha3 +// --- STREAMING-MAC ---------------------------------------------------------- +#define Hacl_Streaming_HMAC_copy _Py_LibHacl_Hacl_Streaming_HMAC_copy +#define Hacl_Streaming_HMAC_digest _Py_LibHacl_Hacl_Streaming_HMAC_digest +#define Hacl_Streaming_HMAC_free _Py_LibHacl_Hacl_Streaming_HMAC_free +#define Hacl_Streaming_HMAC_get_impl _Py_LibHacl_Hacl_Streaming_HMAC_get_impl #define Hacl_Streaming_HMAC_index_of_state _Py_LibHacl_Hacl_Streaming_HMAC_index_of_state #define Hacl_Streaming_HMAC_malloc_ _Py_LibHacl_Hacl_Streaming_HMAC_malloc_ -#define Hacl_Streaming_HMAC_get_impl _Py_LibHacl_Hacl_Streaming_HMAC_get_impl #define Hacl_Streaming_HMAC_reset _Py_LibHacl_Hacl_Streaming_HMAC_reset -#define Hacl_Streaming_HMAC_update _Py_LibHacl_Hacl_Streaming_HMAC_update -#define Hacl_Streaming_HMAC_digest _Py_LibHacl_Hacl_Streaming_HMAC_digest -#define Hacl_Streaming_HMAC_copy _Py_LibHacl_Hacl_Streaming_HMAC_copy -#define Hacl_Streaming_HMAC_free _Py_LibHacl_Hacl_Streaming_HMAC_free #define Hacl_Streaming_HMAC_s1 _Py_LibHacl_Hacl_Streaming_HMAC_s1 #define Hacl_Streaming_HMAC_s2 _Py_LibHacl_Hacl_Streaming_HMAC_s2 - -// HMAC-MD5 +#define Hacl_Streaming_HMAC_update _Py_LibHacl_Hacl_Streaming_HMAC_update +// --- HMAC-MD5 --------------------------------------------------------------- #define Hacl_HMAC_compute_md5 _Py_LibHacl_Hacl_HMAC_compute_md5 -// HMAC-SHA-1 +// --- HMAC-SHA-1 ------------------------------------------------------------- #define Hacl_HMAC_compute_sha1 _Py_LibHacl_Hacl_HMAC_compute_sha1 -// HMAC-SHA-2 +// --- HMAC-SHA-2 ------------------------------------------------------------- #define Hacl_HMAC_compute_sha2_224 _Py_LibHacl_Hacl_HMAC_compute_sha2_224 #define Hacl_HMAC_compute_sha2_256 _Py_LibHacl_Hacl_HMAC_compute_sha2_256 #define Hacl_HMAC_compute_sha2_384 _Py_LibHacl_Hacl_HMAC_compute_sha2_384 #define Hacl_HMAC_compute_sha2_512 _Py_LibHacl_Hacl_HMAC_compute_sha2_512 -// HMAC-SHA-3 +// --- HMAC-SHA-3 ------------------------------------------------------------- #define Hacl_HMAC_compute_sha3_224 _Py_LibHacl_Hacl_HMAC_compute_sha3_224 #define Hacl_HMAC_compute_sha3_256 _Py_LibHacl_Hacl_HMAC_compute_sha3_256 #define Hacl_HMAC_compute_sha3_384 _Py_LibHacl_Hacl_HMAC_compute_sha3_384 #define Hacl_HMAC_compute_sha3_512 _Py_LibHacl_Hacl_HMAC_compute_sha3_512 -// HMAC-BLAKE -#define Hacl_HMAC_compute_blake2s_32 _Py_LibHacl_Hacl_HMAC_compute_blake2s_32 +// --- HMAC-BLAKE-2 ----------------------------------------------------------- #define Hacl_HMAC_compute_blake2b_32 _Py_LibHacl_Hacl_HMAC_compute_blake2b_32 +#define Hacl_HMAC_compute_blake2s_32 _Py_LibHacl_Hacl_HMAC_compute_blake2s_32 #endif // _PYTHON_HACL_NAMESPACES_H diff --git a/Modules/_hacl/refresh.sh b/Modules/_hacl/refresh.sh index d91650b44bb4e7d..72ceb27b7f70e8e 100755 --- a/Modules/_hacl/refresh.sh +++ b/Modules/_hacl/refresh.sh @@ -22,7 +22,7 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. -expected_hacl_star_rev=7720f6d4fc0468a99d5ea6120976bcc271e42727 +expected_hacl_star_rev=8ba599b2f6c9701b3dc961db895b0856a2210f76 hacl_dir="$(realpath "$1")" cd "$(dirname "$0")" diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index 48eed5eac975edf..0191a958d41daa5 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -238,7 +238,7 @@ py_hashentry_table_new(void) { if (h->py_alias != NULL) { if (_Py_hashtable_set(ht, (const void*)entry->py_alias, (void*)entry) < 0) { - PyMem_Free(entry); + /* entry is already in ht, will be freed by _Py_hashtable_destroy() */ goto error; } entry->refcnt++; @@ -943,9 +943,9 @@ static PyType_Spec EVPXOFtype_spec = { #endif -static PyObject* -py_evp_fromname(PyObject *module, const char *digestname, PyObject *data_obj, - int usedforsecurity) +static PyObject * +_hashlib_HASH(PyObject *module, const char *digestname, PyObject *data_obj, + int usedforsecurity) { Py_buffer view = { 0 }; PY_EVP_MD *digest = NULL; @@ -1017,16 +1017,25 @@ py_evp_fromname(PyObject *module, const char *digestname, PyObject *data_obj, return (PyObject *)self; } +#define CALL_HASHLIB_NEW(MODULE, NAME, DATA, STRING, USEDFORSECURITY) \ + do { \ + PyObject *data_obj; \ + if (_Py_hashlib_data_argument(&data_obj, DATA, STRING) < 0) { \ + return NULL; \ + } \ + return _hashlib_HASH(MODULE, NAME, data_obj, USEDFORSECURITY); \ + } while (0) /* The module-level function: new() */ /*[clinic input] -_hashlib.new as EVP_new +_hashlib.new - name as name_obj: object - string as data_obj: object(c_default="NULL") = b'' + name: str + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Return a new hash object using the named algorithm. @@ -1037,136 +1046,137 @@ The MD5 and SHA1 algorithms are always supported. [clinic start generated code]*/ static PyObject * -EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=ddd5053f92dffe90 input=c24554d0337be1b0]*/ +_hashlib_new_impl(PyObject *module, const char *name, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=c01feb4ad6a6303d input=f5ec9bf1fa749d07]*/ { - char *name; - if (!PyArg_Parse(name_obj, "s", &name)) { - PyErr_SetString(PyExc_TypeError, "name must be a string"); - return NULL; - } - return py_evp_fromname(module, name, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, name, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_md5 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a md5 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_md5_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=87b0186440a44f8c input=990e36d5e689b16e]*/ +_hashlib_openssl_md5_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=ca8cf184d90f7432 input=e7c0adbd6a867db1]*/ { - return py_evp_fromname(module, Py_hash_md5, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_md5, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha1 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha1 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha1_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=6813024cf690670d input=948f2f4b6deabc10]*/ +_hashlib_openssl_sha1_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=1736fb7b310d64be input=f7e5bb1711e952d8]*/ { - return py_evp_fromname(module, Py_hash_sha1, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha1, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha224 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha224 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha224_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=a2dfe7cc4eb14ebb input=f9272821fadca505]*/ +_hashlib_openssl_sha224_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=0d6ff57be5e5c140 input=3820fff7ed3a53b8]*/ { - return py_evp_fromname(module, Py_hash_sha224, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha224, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha256 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha256 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha256_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=1f874a34870f0a68 input=549fad9d2930d4c5]*/ +_hashlib_openssl_sha256_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=412ea7111555b6e7 input=9a2f115cf1f7e0eb]*/ { - return py_evp_fromname(module, Py_hash_sha256, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha256, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha384 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha384 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha384_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=58529eff9ca457b2 input=48601a6e3bf14ad7]*/ +_hashlib_openssl_sha384_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=2e0dc395b59ed726 input=1ea48f6f01e77cfb]*/ { - return py_evp_fromname(module, Py_hash_sha384, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha384, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha512 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha512 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=2c744c9e4a40d5f6 input=c5c46a2a817aa98f]*/ +_hashlib_openssl_sha512_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=4bdd760388dbfc0f input=3cf56903e07d1f5c]*/ { - return py_evp_fromname(module, Py_hash_sha512, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha512, data, string, usedforsecurity); } @@ -1175,77 +1185,81 @@ _hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj, /*[clinic input] _hashlib.openssl_sha3_224 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha3-224 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha3_224_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=144641c1d144b974 input=e3a01b2888916157]*/ +_hashlib_openssl_sha3_224_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=6d8dc2a924f3ba35 input=7f14f16a9f6a3158]*/ { - return py_evp_fromname(module, Py_hash_sha3_224, data_obj, usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha3_224, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha3_256 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha3-256 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha3_256_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=c61f1ab772d06668 input=e2908126c1b6deed]*/ +_hashlib_openssl_sha3_256_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=9e520f537b3a4622 input=7987150939d5e352]*/ { - return py_evp_fromname(module, Py_hash_sha3_256, data_obj , usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha3_256, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha3_384 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha3-384 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha3_384_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=f68e4846858cf0ee input=ec0edf5c792f8252]*/ +_hashlib_openssl_sha3_384_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=d239ba0463fd6138 input=fc943401f67e3b81]*/ { - return py_evp_fromname(module, Py_hash_sha3_384, data_obj , usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha3_384, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_sha3_512 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a sha3-512 hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_sha3_512_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=2eede478c159354a input=64e2cc0c094d56f4]*/ +_hashlib_openssl_sha3_512_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=17662f21038c2278 input=6601ddd2c6c1516d]*/ { - return py_evp_fromname(module, Py_hash_sha3_512, data_obj , usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_sha3_512, data, string, usedforsecurity); } #endif /* PY_OPENSSL_HAS_SHA3 */ @@ -1253,42 +1267,46 @@ _hashlib_openssl_sha3_512_impl(PyObject *module, PyObject *data_obj, /*[clinic input] _hashlib.openssl_shake_128 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a shake-128 variable hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_shake_128_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=bc49cdd8ada1fa97 input=6c9d67440eb33ec8]*/ +_hashlib_openssl_shake_128_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=4e6afed8d18980ad input=373c3f1c93d87b37]*/ { - return py_evp_fromname(module, Py_hash_shake_128, data_obj , usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_shake_128, data, string, usedforsecurity); } /*[clinic input] _hashlib.openssl_shake_256 - string as data_obj: object(py_default="b''") = NULL + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Returns a shake-256 variable hash object; optionally initialized with a string [clinic start generated code]*/ static PyObject * -_hashlib_openssl_shake_256_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity) -/*[clinic end generated code: output=358d213be8852df7 input=479cbe9fefd4a9f8]*/ +_hashlib_openssl_shake_256_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string) +/*[clinic end generated code: output=62481bce4a77d16c input=101c139ea2ddfcbf]*/ { - return py_evp_fromname(module, Py_hash_shake_256, data_obj , usedforsecurity); + CALL_HASHLIB_NEW(module, Py_hash_shake_256, data, string, usedforsecurity); } #endif /* PY_OPENSSL_HAS_SHAKE */ +#undef CALL_HASHLIB_NEW + /*[clinic input] _hashlib.pbkdf2_hmac as pbkdf2_hmac @@ -1816,13 +1834,13 @@ _hashlib.HMAC.hexdigest Return hexadecimal digest of the bytes passed to the update() method so far. -This may be used to exchange the value safely in email or other non-binary -environments. +This may be used to exchange the value safely in email or other +non-binary environments. [clinic start generated code]*/ static PyObject * _hashlib_HMAC_hexdigest_impl(HMACobject *self) -/*[clinic end generated code: output=80d825be1eaae6a7 input=5abc42702874ddcf]*/ +/*[clinic end generated code: output=80d825be1eaae6a7 input=865e6128c7ec0781]*/ { unsigned char digest[EVP_MAX_MD_SIZE]; unsigned int digest_size = _hashlib_hmac_digest_size(self); @@ -1987,8 +2005,8 @@ _hashlib.get_fips_mode -> int Determine the OpenSSL FIPS mode of operation. For OpenSSL 3.0.0 and newer it returns the state of the default provider -in the default OSSL context. It's not quite the same as FIPS_mode() but good -enough for unittests. +in the default OSSL context. It's not quite the same as FIPS_mode() but +good enough for unittests. Effectively any non-zero return value indicates FIPS mode; values other than 1 may have additional significance. @@ -1996,7 +2014,7 @@ values other than 1 may have additional significance. static int _hashlib_get_fips_mode_impl(PyObject *module) -/*[clinic end generated code: output=87eece1bab4d3fa9 input=2db61538c41c6fef]*/ +/*[clinic end generated code: output=87eece1bab4d3fa9 input=a6cdb6901421d122]*/ { #if OPENSSL_VERSION_NUMBER >= 0x30000000L @@ -2134,7 +2152,7 @@ _hashlib_compare_digest_impl(PyObject *module, PyObject *a, PyObject *b) /* List of functions exported by this module */ static struct PyMethodDef EVP_functions[] = { - EVP_NEW_METHODDEF + _HASHLIB_NEW_METHODDEF PBKDF2_HMAC_METHODDEF _HASHLIB_SCRYPT_METHODDEF _HASHLIB_GET_FIPS_MODE_METHODDEF diff --git a/Modules/_heapqmodule.c b/Modules/_heapqmodule.c index 095866eec7d75a4..fbfc887f7f195c3 100644 --- a/Modules/_heapqmodule.c +++ b/Modules/_heapqmodule.c @@ -11,7 +11,8 @@ annotated by François Pinard, and converted to C by Raymond Hettinger. #endif #include "Python.h" -#include "pycore_list.h" // _PyList_ITEMS() +#include "pycore_list.h" // _PyList_ITEMS(), _PyList_AppendTakeRef() +#include "pycore_pyatomic_ft_wrappers.h" #include "clinic/_heapqmodule.c.h" @@ -59,8 +60,8 @@ siftdown(PyListObject *heap, Py_ssize_t startpos, Py_ssize_t pos) arr = _PyList_ITEMS(heap); parent = arr[parentpos]; newitem = arr[pos]; - arr[parentpos] = newitem; - arr[pos] = parent; + FT_ATOMIC_STORE_PTR_RELAXED(arr[parentpos], newitem); + FT_ATOMIC_STORE_PTR_RELAXED(arr[pos], parent); pos = parentpos; } return 0; @@ -108,8 +109,8 @@ siftup(PyListObject *heap, Py_ssize_t pos) /* Move the smaller child up. */ tmp1 = arr[childpos]; tmp2 = arr[pos]; - arr[childpos] = tmp2; - arr[pos] = tmp1; + FT_ATOMIC_STORE_PTR_RELAXED(arr[childpos], tmp2); + FT_ATOMIC_STORE_PTR_RELAXED(arr[pos], tmp1); pos = childpos; } /* Bubble it up to its final resting place (by sifting its parents down). */ @@ -117,6 +118,7 @@ siftup(PyListObject *heap, Py_ssize_t pos) } /*[clinic input] +@critical_section heap _heapq.heappush heap: object(subclass_of='&PyList_Type') @@ -128,13 +130,22 @@ Push item onto heap, maintaining the heap invariant. static PyObject * _heapq_heappush_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=912c094f47663935 input=7c69611f3698aceb]*/ +/*[clinic end generated code: output=912c094f47663935 input=f7a4f03ef8d52e67]*/ { - if (PyList_Append(heap, item)) + if (item == NULL) { + PyErr_BadInternalCall(); return NULL; + } + + // In a free-threaded build, the heap is locked at this point. + // Therefore, calling _PyList_AppendTakeRef() is safe and no overhead. + if (_PyList_AppendTakeRef((PyListObject *)heap, Py_NewRef(item))) { + return NULL; + } - if (siftdown((PyListObject *)heap, 0, PyList_GET_SIZE(heap)-1)) + if (siftdown((PyListObject *)heap, 0, PyList_GET_SIZE(heap)-1)) { return NULL; + } Py_RETURN_NONE; } @@ -162,8 +173,9 @@ heappop_internal(PyObject *heap, int siftup_func(PyListObject *, Py_ssize_t)) if (!n) return lastelt; returnitem = PyList_GET_ITEM(heap, 0); - PyList_SET_ITEM(heap, 0, lastelt); - if (siftup_func((PyListObject *)heap, 0)) { + PyListObject *list = _PyList_CAST(heap); + FT_ATOMIC_STORE_PTR_RELAXED(list->ob_item[0], lastelt); + if (siftup_func(list, 0)) { Py_DECREF(returnitem); return NULL; } @@ -171,6 +183,7 @@ heappop_internal(PyObject *heap, int siftup_func(PyListObject *, Py_ssize_t)) } /*[clinic input] +@critical_section heap _heapq.heappop heap: object(subclass_of='&PyList_Type') @@ -181,7 +194,7 @@ Pop the smallest item off the heap, maintaining the heap invariant. static PyObject * _heapq_heappop_impl(PyObject *module, PyObject *heap) -/*[clinic end generated code: output=96dfe82d37d9af76 input=91487987a583c856]*/ +/*[clinic end generated code: output=96dfe82d37d9af76 input=ed396461b153dd51]*/ { return heappop_internal(heap, siftup); } @@ -197,8 +210,9 @@ heapreplace_internal(PyObject *heap, PyObject *item, int siftup_func(PyListObjec } returnitem = PyList_GET_ITEM(heap, 0); - PyList_SET_ITEM(heap, 0, Py_NewRef(item)); - if (siftup_func((PyListObject *)heap, 0)) { + PyListObject *list = _PyList_CAST(heap); + FT_ATOMIC_STORE_PTR_RELAXED(list->ob_item[0], Py_NewRef(item)); + if (siftup_func(list, 0)) { Py_DECREF(returnitem); return NULL; } @@ -207,6 +221,7 @@ heapreplace_internal(PyObject *heap, PyObject *item, int siftup_func(PyListObjec /*[clinic input] +@critical_section heap _heapq.heapreplace heap: object(subclass_of='&PyList_Type') @@ -226,12 +241,13 @@ this routine unless written as part of a conditional replacement: static PyObject * _heapq_heapreplace_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=82ea55be8fbe24b4 input=719202ac02ba10c8]*/ +/*[clinic end generated code: output=82ea55be8fbe24b4 input=9be1678b817ef1a9]*/ { return heapreplace_internal(heap, item, siftup); } /*[clinic input] +@critical_section heap _heapq.heappushpop heap: object(subclass_of='&PyList_Type') @@ -246,7 +262,7 @@ a separate call to heappop(). static PyObject * _heapq_heappushpop_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=67231dc98ed5774f input=5dc701f1eb4a4aa7]*/ +/*[clinic end generated code: output=67231dc98ed5774f input=db05c81b1dd92c44]*/ { PyObject *returnitem; int cmp; @@ -271,8 +287,9 @@ _heapq_heappushpop_impl(PyObject *module, PyObject *heap, PyObject *item) } returnitem = PyList_GET_ITEM(heap, 0); - PyList_SET_ITEM(heap, 0, Py_NewRef(item)); - if (siftup((PyListObject *)heap, 0)) { + PyListObject *list = _PyList_CAST(heap); + FT_ATOMIC_STORE_PTR_RELAXED(list->ob_item[0], Py_NewRef(item)); + if (siftup(list, 0)) { Py_DECREF(returnitem); return NULL; } @@ -371,6 +388,7 @@ heapify_internal(PyObject *heap, int siftup_func(PyListObject *, Py_ssize_t)) } /*[clinic input] +@critical_section heap _heapq.heapify heap: object(subclass_of='&PyList_Type') @@ -381,7 +399,7 @@ Transform list into a heap, in-place, in O(len(heap)) time. static PyObject * _heapq_heapify_impl(PyObject *module, PyObject *heap) -/*[clinic end generated code: output=e63a636fcf83d6d0 input=53bb7a2166febb73]*/ +/*[clinic end generated code: output=e63a636fcf83d6d0 input=aaaaa028b9b6af08]*/ { return heapify_internal(heap, siftup); } @@ -423,8 +441,8 @@ siftdown_max(PyListObject *heap, Py_ssize_t startpos, Py_ssize_t pos) arr = _PyList_ITEMS(heap); parent = arr[parentpos]; newitem = arr[pos]; - arr[parentpos] = newitem; - arr[pos] = parent; + FT_ATOMIC_STORE_PTR_RELAXED(arr[parentpos], newitem); + FT_ATOMIC_STORE_PTR_RELAXED(arr[pos], parent); pos = parentpos; } return 0; @@ -445,11 +463,11 @@ siftup_max(PyListObject *heap, Py_ssize_t pos) return -1; } - /* Bubble up the smaller child until hitting a leaf. */ + /* Bubble up the larger child until hitting a leaf. */ arr = _PyList_ITEMS(heap); limit = endpos >> 1; /* smallest pos that has no child */ while (pos < limit) { - /* Set childpos to index of smaller child. */ + /* Set childpos to index of larger child. */ childpos = 2*pos + 1; /* leftmost child position */ if (childpos + 1 < endpos) { PyObject* a = arr[childpos + 1]; @@ -469,11 +487,11 @@ siftup_max(PyListObject *heap, Py_ssize_t pos) return -1; } } - /* Move the smaller child up. */ + /* Move the larger child up. */ tmp1 = arr[childpos]; tmp2 = arr[pos]; - arr[childpos] = tmp2; - arr[pos] = tmp1; + FT_ATOMIC_STORE_PTR_RELAXED(arr[childpos], tmp2); + FT_ATOMIC_STORE_PTR_RELAXED(arr[pos], tmp1); pos = childpos; } /* Bubble it up to its final resting place (by sifting its parents down). */ @@ -481,6 +499,7 @@ siftup_max(PyListObject *heap, Py_ssize_t pos) } /*[clinic input] +@critical_section heap _heapq.heappush_max heap: object(subclass_of='&PyList_Type') @@ -492,9 +511,16 @@ Push item onto max heap, maintaining the heap invariant. static PyObject * _heapq_heappush_max_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=c869d5f9deb08277 input=4743d7db137b6e2b]*/ +/*[clinic end generated code: output=c869d5f9deb08277 input=c437e3d1ff8dcb70]*/ { - if (PyList_Append(heap, item)) { + if (item == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + + // In a free-threaded build, the heap is locked at this point. + // Therefore, calling _PyList_AppendTakeRef() is safe and no overhead. + if (_PyList_AppendTakeRef((PyListObject *)heap, Py_NewRef(item))) { return NULL; } @@ -506,6 +532,7 @@ _heapq_heappush_max_impl(PyObject *module, PyObject *heap, PyObject *item) } /*[clinic input] +@critical_section heap _heapq.heappop_max heap: object(subclass_of='&PyList_Type') @@ -516,12 +543,13 @@ Maxheap variant of heappop. static PyObject * _heapq_heappop_max_impl(PyObject *module, PyObject *heap) -/*[clinic end generated code: output=2f051195ab404b77 input=e62b14016a5a26de]*/ +/*[clinic end generated code: output=2f051195ab404b77 input=5d70c997798aec64]*/ { return heappop_internal(heap, siftup_max); } /*[clinic input] +@critical_section heap _heapq.heapreplace_max heap: object(subclass_of='&PyList_Type') @@ -533,12 +561,13 @@ Maxheap variant of heapreplace. static PyObject * _heapq_heapreplace_max_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=8770778b5a9cbe9b input=21a3d28d757c881c]*/ +/*[clinic end generated code: output=8770778b5a9cbe9b input=fe70175356e4a649]*/ { return heapreplace_internal(heap, item, siftup_max); } /*[clinic input] +@critical_section heap _heapq.heapify_max heap: object(subclass_of='&PyList_Type') @@ -549,12 +578,13 @@ Maxheap variant of heapify. static PyObject * _heapq_heapify_max_impl(PyObject *module, PyObject *heap) -/*[clinic end generated code: output=8401af3856529807 input=edda4255728c431e]*/ +/*[clinic end generated code: output=8401af3856529807 input=4eee63231e7d1573]*/ { return heapify_internal(heap, siftup_max); } /*[clinic input] +@critical_section heap _heapq.heappushpop_max heap: object(subclass_of='&PyList_Type') @@ -563,13 +593,13 @@ _heapq.heappushpop_max Maxheap variant of heappushpop. -The combined action runs more efficiently than heappush_max() followed by -a separate call to heappop_max(). +The combined action runs more efficiently than heappush_max() +followed by a separate call to heappop_max(). [clinic start generated code]*/ static PyObject * _heapq_heappushpop_max_impl(PyObject *module, PyObject *heap, PyObject *item) -/*[clinic end generated code: output=ff0019f0941aca0d input=525a843013cbd6c0]*/ +/*[clinic end generated code: output=ff0019f0941aca0d input=52030929667a4c08]*/ { PyObject *returnitem; int cmp; @@ -595,8 +625,9 @@ _heapq_heappushpop_max_impl(PyObject *module, PyObject *heap, PyObject *item) } returnitem = PyList_GET_ITEM(heap, 0); - PyList_SET_ITEM(heap, 0, Py_NewRef(item)); - if (siftup_max((PyListObject *)heap, 0) < 0) { + PyListObject *list = _PyList_CAST(heap); + FT_ATOMIC_STORE_PTR_RELAXED(list->ob_item[0], Py_NewRef(item)); + if (siftup_max(list, 0) < 0) { Py_DECREF(returnitem); return NULL; } diff --git a/Modules/_interpchannelsmodule.c b/Modules/_interpchannelsmodule.c index 172cebcaa4884f2..c38495058692768 100644 --- a/Modules/_interpchannelsmodule.c +++ b/Modules/_interpchannelsmodule.c @@ -20,9 +20,11 @@ #endif #define REGISTERS_HEAP_TYPES +#define HAS_FALLBACK #define HAS_UNBOUND_ITEMS #include "_interpreters_common.h" #undef HAS_UNBOUND_ITEMS +#undef HAS_FALLBACK #undef REGISTERS_HEAP_TYPES @@ -218,6 +220,22 @@ wait_for_lock(PyThread_type_lock mutex, PY_TIMEOUT_T timeout) return 0; } +static int +ensure_highlevel_module_loaded(void) +{ + PyObject *highlevel = + PyImport_ImportModule("concurrent.interpreters._channels"); + if (highlevel == NULL) { + PyErr_Clear(); + highlevel = PyImport_ImportModule("test.support.channels"); + if (highlevel == NULL) { + return -1; + } + } + Py_DECREF(highlevel); + return 0; +} + /* module state *************************************************************/ @@ -252,10 +270,10 @@ _get_current_module_state(void) { PyObject *mod = _get_current_module(); if (mod == NULL) { - // XXX import it? - PyErr_SetString(PyExc_RuntimeError, - MODULE_NAME_STR " module not imported yet"); - return NULL; + mod = PyImport_ImportModule(MODULE_NAME_STR); + if (mod == NULL) { + return NULL; + } } module_state *state = get_module_state(mod); Py_DECREF(mod); @@ -336,7 +354,7 @@ clear_module_state(module_state *state) #define ERR_CHANNEL_INTERP_CLOSED -4 #define ERR_CHANNEL_EMPTY -5 #define ERR_CHANNEL_NOT_EMPTY -6 -#define ERR_CHANNEL_MUTEX_INIT -7 +#define ERR_CHANNEL_MUTEX_INIT -7 // currently unused #define ERR_CHANNELS_MUTEX_INIT -8 #define ERR_NO_NEXT_CHANNEL_ID -9 #define ERR_CHANNEL_CLOSED_WAITING -10 @@ -409,10 +427,6 @@ handle_channel_error(int err, PyObject *mod, int64_t cid) "if not empty (try force=True)", cid); } - else if (err == ERR_CHANNEL_MUTEX_INIT) { - PyErr_SetString(state->ChannelError, - "can't initialize mutex for new channel"); - } else if (err == ERR_CHANNELS_MUTEX_INIT) { PyErr_SetString(state->ChannelError, "can't initialize mutex for channel management"); @@ -493,12 +507,12 @@ _waiting_release(_waiting_t *waiting, int received) assert(!waiting->received); waiting->status = WAITING_RELEASING; - PyThread_release_lock(waiting->mutex); if (waiting->received != received) { assert(received == 1); waiting->received = received; } waiting->status = WAITING_RELEASED; + PyThread_release_lock(waiting->mutex); } static void @@ -523,7 +537,7 @@ typedef struct _channelitem { int64_t interpid; _PyXIData_t *data; _waiting_t *waiting; - int unboundop; + unboundop_t unboundop; struct _channelitem *next; } _channelitem; @@ -536,7 +550,7 @@ _channelitem_ID(_channelitem *item) static void _channelitem_init(_channelitem *item, int64_t interpid, _PyXIData_t *data, - _waiting_t *waiting, int unboundop) + _waiting_t *waiting, unboundop_t unboundop) { if (interpid < 0) { interpid = _get_interpid(data); @@ -562,7 +576,7 @@ _channelitem_clear_data(_channelitem *item, int removed) { if (item->data != NULL) { // It was allocated in channel_send(). - (void)_release_xid_data(item->data, XID_IGNORE_EXC & XID_FREE); + (void)_release_xid_data(item->data, XID_IGNORE_EXC | XID_FREE); item->data = NULL; } @@ -583,7 +597,7 @@ _channelitem_clear(_channelitem *item) static _channelitem * _channelitem_new(int64_t interpid, _PyXIData_t *data, - _waiting_t *waiting, int unboundop) + _waiting_t *waiting, unboundop_t unboundop) { _channelitem *item = GLOBAL_MALLOC(_channelitem); if (item == NULL) { @@ -694,7 +708,7 @@ _channelqueue_free(_channelqueue *queue) static int _channelqueue_put(_channelqueue *queue, int64_t interpid, _PyXIData_t *data, - _waiting_t *waiting, int unboundop) + _waiting_t *waiting, unboundop_t unboundop) { _channelitem *item = _channelitem_new(interpid, data, waiting, unboundop); if (item == NULL) { @@ -798,7 +812,7 @@ _channelqueue_remove(_channelqueue *queue, _channelitem_id_t itemid, } queue->count -= 1; - int unboundop; + unboundop_t unboundop; _channelitem_popped(item, p_data, p_waiting, &unboundop); } @@ -903,7 +917,8 @@ static _channelends * _channelends_new(void) { _channelends *ends = GLOBAL_MALLOC(_channelends); - if (ends== NULL) { + if (ends == NULL) { + PyErr_NoMemory(); return NULL; } ends->numsendopen = 0; @@ -1083,18 +1098,21 @@ typedef struct _channel { PyThread_type_lock mutex; _channelqueue *queue; _channelends *ends; - struct { - int unboundop; + struct _channeldefaults { + unboundop_t unboundop; + xidata_fallback_t fallback; } defaults; int open; struct _channel_closing *closing; } _channel_state; static _channel_state * -_channel_new(PyThread_type_lock mutex, int unboundop) +_channel_new(PyThread_type_lock mutex, struct _channeldefaults defaults) { + assert(check_unbound(defaults.unboundop)); _channel_state *chan = GLOBAL_MALLOC(_channel_state); if (chan == NULL) { + PyErr_NoMemory(); return NULL; } chan->mutex = mutex; @@ -1109,7 +1127,7 @@ _channel_new(PyThread_type_lock mutex, int unboundop) GLOBAL_FREE(chan); return NULL; } - chan->defaults.unboundop = unboundop; + chan->defaults = defaults; chan->open = 1; chan->closing = NULL; return chan; @@ -1130,7 +1148,7 @@ _channel_free(_channel_state *chan) static int _channel_add(_channel_state *chan, int64_t interpid, - _PyXIData_t *data, _waiting_t *waiting, int unboundop) + _PyXIData_t *data, _waiting_t *waiting, unboundop_t unboundop) { int res = -1; PyThread_acquire_lock(chan->mutex, WAIT_LOCK); @@ -1293,6 +1311,7 @@ _channelref_new(int64_t cid, _channel_state *chan) { _channelref *ref = GLOBAL_MALLOC(_channelref); if (ref == NULL) { + PyErr_NoMemory(); return NULL; } ref->cid = cid; @@ -1611,7 +1630,7 @@ _channels_release_cid_object(_channels *channels, int64_t cid) struct channel_id_and_info { int64_t id; - int unboundop; + struct _channeldefaults defaults; }; static struct channel_id_and_info * @@ -1624,14 +1643,16 @@ _channels_list_all(_channels *channels, int64_t *count) if (ids == NULL) { goto done; } - _channelref *ref = channels->head; - for (int64_t i=0; ref != NULL; ref = ref->next, i++) { - ids[i] = (struct channel_id_and_info){ - .id = ref->cid, - .unboundop = ref->chan->defaults.unboundop, - }; + int64_t i = 0; + for (_channelref *ref = channels->head; ref != NULL; ref = ref->next) { + if (ref->chan != NULL) { + ids[i++] = (struct channel_id_and_info){ + .id = ref->cid, + .defaults = ref->chan->defaults, + }; + } } - *count = channels->numopen; + *count = i; cids = ids; done: @@ -1676,6 +1697,7 @@ _channel_set_closing(_channelref *ref, PyThread_type_lock mutex) { } chan->closing = GLOBAL_MALLOC(struct _channel_closing); if (chan->closing == NULL) { + PyErr_NoMemory(); goto done; } chan->closing->ref = ref; @@ -1714,13 +1736,14 @@ _channel_finish_closing(_channel_state *chan) { // Create a new channel. static int64_t -channel_create(_channels *channels, int unboundop) +channel_create(_channels *channels, struct _channeldefaults defaults) { PyThread_type_lock mutex = PyThread_allocate_lock(); if (mutex == NULL) { - return ERR_CHANNEL_MUTEX_INIT; + PyErr_NoMemory(); + return -1; } - _channel_state *chan = _channel_new(mutex, unboundop); + _channel_state *chan = _channel_new(mutex, defaults); if (chan == NULL) { PyThread_free_lock(mutex); return -1; @@ -1752,7 +1775,7 @@ channel_destroy(_channels *channels, int64_t cid) // Optionally request to be notified when it is received. static int channel_send(_channels *channels, int64_t cid, PyObject *obj, - _waiting_t *waiting, int unboundop) + _waiting_t *waiting, unboundop_t unboundop, xidata_fallback_t fallback) { PyThreadState *tstate = _PyThreadState_GET(); PyInterpreterState *interp = tstate->interp; @@ -1779,7 +1802,7 @@ channel_send(_channels *channels, int64_t cid, PyObject *obj, PyThread_release_lock(mutex); return -1; } - if (_PyObject_GetXIData(tstate, obj, data) != 0) { + if (_PyObject_GetXIData(tstate, obj, fallback, data) != 0) { PyThread_release_lock(mutex); GLOBAL_FREE(data); return -1; @@ -1823,7 +1846,8 @@ channel_clear_sent(_channels *channels, int64_t cid, _waiting_t *waiting) // Like channel_send(), but strictly wait for the object to be received. static int channel_send_wait(_channels *channels, int64_t cid, PyObject *obj, - int unboundop, PY_TIMEOUT_T timeout) + unboundop_t unboundop, PY_TIMEOUT_T timeout, + xidata_fallback_t fallback) { // We use a stack variable here, so we must ensure that &waiting // is not held by any channel item at the point this function exits. @@ -1834,7 +1858,7 @@ channel_send_wait(_channels *channels, int64_t cid, PyObject *obj, } /* Queue up the object. */ - int res = channel_send(channels, cid, obj, &waiting, unboundop); + int res = channel_send(channels, cid, obj, &waiting, unboundop, fallback); if (res < 0) { assert(waiting.status == WAITING_NO_STATUS); goto finally; @@ -2005,6 +2029,20 @@ channel_is_associated(_channels *channels, int64_t cid, int64_t interpid, return (end != NULL && end->open); } +static int +channel_get_defaults(_channels *channels, int64_t cid, struct _channeldefaults *defaults) +{ + PyThread_type_lock mutex = NULL; + _channel_state *channel = NULL; + int err = _channels_lookup(channels, cid, &mutex, &channel); + if (err != 0) { + return err; + } + *defaults = channel->defaults; + PyThread_release_lock(mutex); + return 0; +} + static int _channel_get_count(_channels *channels, int64_t cid, Py_ssize_t *p_count) { @@ -2549,6 +2587,7 @@ static PyObject * _channelid_from_xid(_PyXIData_t *data) { struct _channelid_xid *xid = (struct _channelid_xid *)_PyXIData_DATA(data); + PyObject *cidobj = NULL; // It might not be imported yet, so we can't use _get_current_module(). PyObject *mod = PyImport_ImportModule(MODULE_NAME_STR); @@ -2558,11 +2597,10 @@ _channelid_from_xid(_PyXIData_t *data) assert(mod != Py_None); module_state *state = get_module_state(mod); if (state == NULL) { - return NULL; + goto done; } // Note that we do not preserve the "resolve" flag. - PyObject *cidobj = NULL; int err = newchannelid(state->ChannelIDType, xid->cid, xid->end, _global_channels(), 0, 0, (channelid **)&cidobj); @@ -2694,7 +2732,7 @@ add_channelid_type(PyObject *mod) Py_DECREF(cls); return NULL; } - if (ensure_xid_class(cls, _channelid_shared) < 0) { + if (ensure_xid_class(cls, GETDATA(_channelid_shared)) < 0) { Py_DECREF(cls); return NULL; } @@ -2723,15 +2761,9 @@ _get_current_channelend_type(int end) } if (cls == NULL) { // Force the module to be loaded, to register the type. - PyObject *highlevel = PyImport_ImportModule("interpreters.channels"); - if (highlevel == NULL) { - PyErr_Clear(); - highlevel = PyImport_ImportModule("test.support.interpreters.channels"); - if (highlevel == NULL) { - return NULL; - } + if (ensure_highlevel_module_loaded() < 0) { + return NULL; } - Py_DECREF(highlevel); if (end == CHANNEL_SEND) { cls = state->send_channel_type; } @@ -2797,12 +2829,12 @@ set_channelend_types(PyObject *mod, PyTypeObject *send, PyTypeObject *recv) // Add and register the types. state->send_channel_type = (PyTypeObject *)Py_NewRef(send); state->recv_channel_type = (PyTypeObject *)Py_NewRef(recv); - if (ensure_xid_class(send, _channelend_shared) < 0) { + if (ensure_xid_class(send, GETDATA(_channelend_shared)) < 0) { Py_CLEAR(state->send_channel_type); Py_CLEAR(state->recv_channel_type); return -1; } - if (ensure_xid_class(recv, _channelend_shared) < 0) { + if (ensure_xid_class(recv, GETDATA(_channelend_shared)) < 0) { (void)clear_xid_class(state->send_channel_type); Py_CLEAR(state->send_channel_type); Py_CLEAR(state->recv_channel_type); @@ -2881,22 +2913,30 @@ clear_interpreter(void *data) static PyObject * channelsmod_create(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"unboundop", NULL}; - int unboundop; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "i:create", kwlist, - &unboundop)) + static char *kwlist[] = {"unboundop", "fallback", NULL}; + int unboundarg = -1; + int fallbackarg = -1; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|ii:create", kwlist, + &unboundarg, &fallbackarg)) { return NULL; } - if (!check_unbound(unboundop)) { - PyErr_Format(PyExc_ValueError, - "unsupported unboundop %d", unboundop); + struct _channeldefaults defaults = {0}; + if (resolve_unboundop(unboundarg, UNBOUND_REPLACE, + &defaults.unboundop) < 0) + { + return NULL; + } + if (resolve_fallback(fallbackarg, _PyXIDATA_FULL_FALLBACK, + &defaults.fallback) < 0) + { return NULL; } - int64_t cid = channel_create(&_globals.channels, unboundop); + int64_t cid = channel_create(&_globals.channels, defaults); if (cid < 0) { - (void)handle_channel_error(-1, self, cid); + // Negative `cid` can't be too big for a downcast: + (void)handle_channel_error((int)cid, self, cid); return NULL; } module_state *state = get_module_state(self); @@ -2909,10 +2949,8 @@ channelsmod_create(PyObject *self, PyObject *args, PyObject *kwds) &cidobj); if (handle_channel_error(err, self, cid)) { assert(cidobj == NULL); - err = channel_destroy(&_globals.channels, cid); - if (handle_channel_error(err, self, cid)) { - // XXX issue a warning? - } + assert(PyErr_Occurred()); + (void)channel_destroy(&_globals.channels, cid); return NULL; } assert(cidobj != NULL); @@ -2987,7 +3025,9 @@ channelsmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) } assert(cidobj != NULL); - PyObject *item = Py_BuildValue("Oi", cidobj, cur->unboundop); + PyObject *item = Py_BuildValue("Oii", cidobj, + cur->defaults.unboundop, + cur->defaults.fallback); Py_DECREF(cidobj); if (item == NULL) { Py_SETREF(ids, NULL); @@ -3075,40 +3115,54 @@ receive end."); static PyObject * channelsmod_send(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"cid", "obj", "unboundop", "blocking", "timeout", - NULL}; + static char *kwlist[] = {"cid", "obj", "unboundop", "fallback", + "blocking", "timeout", NULL}; struct channel_id_converter_data cid_data = { .module = self, }; PyObject *obj; - int unboundop = UNBOUND_REPLACE; + int unboundarg = -1; + int fallbackarg = -1; int blocking = 1; PyObject *timeout_obj = NULL; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O|i$pO:channel_send", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, + "O&O|ii$pO:channel_send", kwlist, channel_id_converter, &cid_data, &obj, - &unboundop, &blocking, &timeout_obj)) + &unboundarg, &fallbackarg, + &blocking, &timeout_obj)) { return NULL; } - if (!check_unbound(unboundop)) { - PyErr_Format(PyExc_ValueError, - "unsupported unboundop %d", unboundop); - return NULL; - } - int64_t cid = cid_data.cid; PY_TIMEOUT_T timeout; if (PyThread_ParseTimeoutArg(timeout_obj, blocking, &timeout) < 0) { return NULL; } + struct _channeldefaults defaults = {-1, -1}; + if (unboundarg < 0 || fallbackarg < 0) { + int err = channel_get_defaults(&_globals.channels, cid, &defaults); + if (handle_channel_error(err, self, cid)) { + return NULL; + } + } + unboundop_t unboundop; + if (resolve_unboundop(unboundarg, defaults.unboundop, &unboundop) < 0) { + return NULL; + } + xidata_fallback_t fallback; + if (resolve_fallback(fallbackarg, defaults.fallback, &fallback) < 0) { + return NULL; + } /* Queue up the object. */ int err = 0; if (blocking) { - err = channel_send_wait(&_globals.channels, cid, obj, unboundop, timeout); + err = channel_send_wait( + &_globals.channels, cid, obj, unboundop, timeout, fallback); } else { - err = channel_send(&_globals.channels, cid, obj, NULL, unboundop); + err = channel_send( + &_globals.channels, cid, obj, NULL, unboundop, fallback); } if (handle_channel_error(err, self, cid)) { return NULL; @@ -3126,32 +3180,44 @@ By default this waits for the object to be received."); static PyObject * channelsmod_send_buffer(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"cid", "obj", "unboundop", "blocking", "timeout", - NULL}; + static char *kwlist[] = {"cid", "obj", "unboundop", "fallback", + "blocking", "timeout", NULL}; struct channel_id_converter_data cid_data = { .module = self, }; PyObject *obj; - int unboundop = UNBOUND_REPLACE; - int blocking = 1; + int unboundarg = -1; + int fallbackarg = -1; + int blocking = -1; PyObject *timeout_obj = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O&O|i$pO:channel_send_buffer", kwlist, + "O&O|ii$pO:channel_send_buffer", kwlist, channel_id_converter, &cid_data, &obj, - &unboundop, &blocking, &timeout_obj)) { - return NULL; - } - if (!check_unbound(unboundop)) { - PyErr_Format(PyExc_ValueError, - "unsupported unboundop %d", unboundop); + &unboundarg, &fallbackarg, + &blocking, &timeout_obj)) + { return NULL; } - int64_t cid = cid_data.cid; PY_TIMEOUT_T timeout; if (PyThread_ParseTimeoutArg(timeout_obj, blocking, &timeout) < 0) { return NULL; } + struct _channeldefaults defaults = {-1, -1}; + if (unboundarg < 0 || fallbackarg < 0) { + int err = channel_get_defaults(&_globals.channels, cid, &defaults); + if (handle_channel_error(err, self, cid)) { + return NULL; + } + } + unboundop_t unboundop; + if (resolve_unboundop(unboundarg, defaults.unboundop, &unboundop) < 0) { + return NULL; + } + xidata_fallback_t fallback; + if (resolve_fallback(fallbackarg, defaults.fallback, &fallback) < 0) { + return NULL; + } PyObject *tempobj = PyMemoryView_FromObject(obj); if (tempobj == NULL) { @@ -3162,10 +3228,11 @@ channelsmod_send_buffer(PyObject *self, PyObject *args, PyObject *kwds) int err = 0; if (blocking) { err = channel_send_wait( - &_globals.channels, cid, tempobj, unboundop, timeout); + &_globals.channels, cid, tempobj, unboundop, timeout, fallback); } else { - err = channel_send(&_globals.channels, cid, tempobj, NULL, unboundop); + err = channel_send( + &_globals.channels, cid, tempobj, NULL, unboundop, fallback); } Py_DECREF(tempobj); if (handle_channel_error(err, self, cid)) { @@ -3197,7 +3264,7 @@ channelsmod_recv(PyObject *self, PyObject *args, PyObject *kwds) cid = cid_data.cid; PyObject *obj = NULL; - int unboundop = 0; + unboundop_t unboundop = 0; int err = channel_recv(&_globals.channels, cid, &obj, &unboundop); if (err == ERR_CHANNEL_EMPTY && dflt != NULL) { // Use the default. @@ -3388,17 +3455,14 @@ channelsmod_get_channel_defaults(PyObject *self, PyObject *args, PyObject *kwds) } int64_t cid = cid_data.cid; - PyThread_type_lock mutex = NULL; - _channel_state *channel = NULL; - int err = _channels_lookup(&_globals.channels, cid, &mutex, &channel); + struct _channeldefaults defaults = {0}; + int err = channel_get_defaults(&_globals.channels, cid, &defaults); if (handle_channel_error(err, self, cid)) { return NULL; } - int unboundop = channel->defaults.unboundop; - PyThread_release_lock(mutex); - PyObject *defaults = Py_BuildValue("i", unboundop); - return defaults; + PyObject *res = Py_BuildValue("ii", defaults.unboundop, defaults.fallback); + return res; } PyDoc_STRVAR(channelsmod_get_channel_defaults_doc, @@ -3552,8 +3616,7 @@ module_traverse(PyObject *mod, visitproc visit, void *arg) { module_state *state = get_module_state(mod); assert(state != NULL); - (void)traverse_module_state(state, visit, arg); - return 0; + return traverse_module_state(state, visit, arg); } static int @@ -3563,8 +3626,7 @@ module_clear(PyObject *mod) assert(state != NULL); // Now we clear the module state. - (void)clear_module_state(state); - return 0; + return clear_module_state(state); } static void diff --git a/Modules/_interpqueuesmodule.c b/Modules/_interpqueuesmodule.c index 526249a0e1aec3e..0de2de0b3db5826 100644 --- a/Modules/_interpqueuesmodule.c +++ b/Modules/_interpqueuesmodule.c @@ -9,9 +9,11 @@ #include "pycore_crossinterp.h" // _PyXIData_t #define REGISTERS_HEAP_TYPES +#define HAS_FALLBACK #define HAS_UNBOUND_ITEMS #include "_interpreters_common.h" #undef HAS_UNBOUND_ITEMS +#undef HAS_FALLBACK #undef REGISTERS_HEAP_TYPES @@ -134,13 +136,10 @@ idarg_int64_converter(PyObject *arg, void *ptr) static int ensure_highlevel_module_loaded(void) { - PyObject *highlevel = PyImport_ImportModule("interpreters.queues"); + PyObject *highlevel = + PyImport_ImportModule("concurrent.interpreters._queues"); if (highlevel == NULL) { - PyErr_Clear(); - highlevel = PyImport_ImportModule("test.support.interpreters.queues"); - if (highlevel == NULL) { - return -1; - } + return -1; } Py_DECREF(highlevel); return 0; @@ -297,7 +296,7 @@ add_QueueError(PyObject *mod) { module_state *state = get_module_state(mod); -#define PREFIX "test.support.interpreters." +#define PREFIX "concurrent.interpreters." #define ADD_EXCTYPE(NAME, BASE, DOC) \ assert(state->NAME == NULL); \ if (add_exctype(mod, &state->NAME, PREFIX #NAME, DOC, BASE) < 0) { \ @@ -401,14 +400,13 @@ typedef struct _queueitem { meaning the interpreter has been destroyed. */ int64_t interpid; _PyXIData_t *data; - int fmt; - int unboundop; + unboundop_t unboundop; struct _queueitem *next; } _queueitem; static void _queueitem_init(_queueitem *item, - int64_t interpid, _PyXIData_t *data, int fmt, int unboundop) + int64_t interpid, _PyXIData_t *data, unboundop_t unboundop) { if (interpid < 0) { interpid = _get_interpid(data); @@ -422,7 +420,6 @@ _queueitem_init(_queueitem *item, *item = (_queueitem){ .interpid = interpid, .data = data, - .fmt = fmt, .unboundop = unboundop, }; } @@ -434,7 +431,7 @@ _queueitem_clear_data(_queueitem *item) return; } // It was allocated in queue_put(). - (void)_release_xid_data(item->data, XID_IGNORE_EXC & XID_FREE); + (void)_release_xid_data(item->data, XID_IGNORE_EXC | XID_FREE); item->data = NULL; } @@ -446,14 +443,14 @@ _queueitem_clear(_queueitem *item) } static _queueitem * -_queueitem_new(int64_t interpid, _PyXIData_t *data, int fmt, int unboundop) +_queueitem_new(int64_t interpid, _PyXIData_t *data, int unboundop) { _queueitem *item = GLOBAL_MALLOC(_queueitem); if (item == NULL) { PyErr_NoMemory(); return NULL; } - _queueitem_init(item, interpid, data, fmt, unboundop); + _queueitem_init(item, interpid, data, unboundop); return item; } @@ -476,10 +473,9 @@ _queueitem_free_all(_queueitem *item) static void _queueitem_popped(_queueitem *item, - _PyXIData_t **p_data, int *p_fmt, int *p_unboundop) + _PyXIData_t **p_data, unboundop_t *p_unboundop) { *p_data = item->data; - *p_fmt = item->fmt; *p_unboundop = item->unboundop; // We clear them here, so they won't be released in _queueitem_clear(). item->data = NULL; @@ -527,16 +523,16 @@ typedef struct _queue { _queueitem *first; _queueitem *last; } items; - struct { - int fmt; + struct _queuedefaults { + xidata_fallback_t fallback; int unboundop; } defaults; } _queue; static int -_queue_init(_queue *queue, Py_ssize_t maxsize, int fmt, int unboundop) +_queue_init(_queue *queue, Py_ssize_t maxsize, struct _queuedefaults defaults) { - assert(check_unbound(unboundop)); + assert(check_unbound(defaults.unboundop)); PyThread_type_lock mutex = PyThread_allocate_lock(); if (mutex == NULL) { return ERR_QUEUE_ALLOC; @@ -547,10 +543,7 @@ _queue_init(_queue *queue, Py_ssize_t maxsize, int fmt, int unboundop) .items = { .maxsize = maxsize, }, - .defaults = { - .fmt = fmt, - .unboundop = unboundop, - }, + .defaults = defaults, }; return 0; } @@ -631,8 +624,7 @@ _queue_unlock(_queue *queue) } static int -_queue_add(_queue *queue, int64_t interpid, _PyXIData_t *data, - int fmt, int unboundop) +_queue_add(_queue *queue, int64_t interpid, _PyXIData_t *data, int unboundop) { int err = _queue_lock(queue); if (err < 0) { @@ -648,7 +640,7 @@ _queue_add(_queue *queue, int64_t interpid, _PyXIData_t *data, return ERR_QUEUE_FULL; } - _queueitem *item = _queueitem_new(interpid, data, fmt, unboundop); + _queueitem *item = _queueitem_new(interpid, data, unboundop); if (item == NULL) { _queue_unlock(queue); return -1; @@ -668,8 +660,7 @@ _queue_add(_queue *queue, int64_t interpid, _PyXIData_t *data, } static int -_queue_next(_queue *queue, - _PyXIData_t **p_data, int *p_fmt, int *p_unboundop) +_queue_next(_queue *queue, _PyXIData_t **p_data, int *p_unboundop) { int err = _queue_lock(queue); if (err < 0) { @@ -688,7 +679,7 @@ _queue_next(_queue *queue, } queue->items.count -= 1; - _queueitem_popped(item, p_data, p_fmt, p_unboundop); + _queueitem_popped(item, p_data, p_unboundop); _queue_unlock(queue); return 0; @@ -716,8 +707,11 @@ _queue_is_full(_queue *queue, int *p_is_full) return err; } - assert(queue->items.count <= queue->items.maxsize); - *p_is_full = queue->items.count == queue->items.maxsize; + assert(queue->items.maxsize <= 0 + || queue->items.count <= queue->items.maxsize); + *p_is_full = queue->items.maxsize > 0 + ? queue->items.count == queue->items.maxsize + : 0; _queue_unlock(queue); return 0; @@ -1035,8 +1029,7 @@ _queues_decref(_queues *queues, int64_t qid) struct queue_id_and_info { int64_t id; - int fmt; - int unboundop; + struct _queuedefaults defaults; }; static struct queue_id_and_info * @@ -1053,8 +1046,7 @@ _queues_list_all(_queues *queues, int64_t *p_count) for (int64_t i=0; ref != NULL; ref = ref->next, i++) { ids[i].id = ref->qid; assert(ref->queue != NULL); - ids[i].fmt = ref->queue->defaults.fmt; - ids[i].unboundop = ref->queue->defaults.unboundop; + ids[i].defaults = ref->queue->defaults; } *p_count = queues->count; @@ -1090,19 +1082,21 @@ _queue_free(_queue *queue) // Create a new queue. static int64_t -queue_create(_queues *queues, Py_ssize_t maxsize, int fmt, int unboundop) +queue_create(_queues *queues, Py_ssize_t maxsize, + struct _queuedefaults defaults) { _queue *queue = GLOBAL_MALLOC(_queue); if (queue == NULL) { return ERR_QUEUE_ALLOC; } - int err = _queue_init(queue, maxsize, fmt, unboundop); + int err = _queue_init(queue, maxsize, defaults); if (err < 0) { GLOBAL_FREE(queue); return (int64_t)err; } int64_t qid = _queues_add(queues, queue); if (qid < 0) { + queue->alive = 0; _queue_clear(queue); GLOBAL_FREE(queue); } @@ -1125,7 +1119,8 @@ queue_destroy(_queues *queues, int64_t qid) // Push an object onto the queue. static int -queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt, int unboundop) +queue_put(_queues *queues, int64_t qid, PyObject *obj, unboundop_t unboundop, + xidata_fallback_t fallback) { PyThreadState *tstate = PyThreadState_Get(); @@ -1138,27 +1133,27 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt, int unboundop) assert(queue != NULL); // Convert the object to cross-interpreter data. - _PyXIData_t *data = _PyXIData_New(); - if (data == NULL) { + _PyXIData_t *xidata = _PyXIData_New(); + if (xidata == NULL) { _queue_unmark_waiter(queue, queues->mutex); return -1; } - if (_PyObject_GetXIData(tstate, obj, data) != 0) { + if (_PyObject_GetXIData(tstate, obj, fallback, xidata) != 0) { _queue_unmark_waiter(queue, queues->mutex); - GLOBAL_FREE(data); + GLOBAL_FREE(xidata); return -1; } - assert(_PyXIData_INTERPID(data) == + assert(_PyXIData_INTERPID(xidata) == PyInterpreterState_GetID(tstate->interp)); // Add the data to the queue. int64_t interpid = -1; // _queueitem_init() will set it. - int res = _queue_add(queue, interpid, data, fmt, unboundop); + int res = _queue_add(queue, interpid, xidata, unboundop); _queue_unmark_waiter(queue, queues->mutex); if (res != 0) { // We may chain an exception here: - (void)_release_xid_data(data, 0); - GLOBAL_FREE(data); + (void)_release_xid_data(xidata, 0); + GLOBAL_FREE(xidata); return res; } @@ -1169,7 +1164,7 @@ queue_put(_queues *queues, int64_t qid, PyObject *obj, int fmt, int unboundop) // XXX Support a "wait" mutex? static int queue_get(_queues *queues, int64_t qid, - PyObject **res, int *p_fmt, int *p_unboundop) + PyObject **res, int *p_unboundop) { int err; *res = NULL; @@ -1185,7 +1180,7 @@ queue_get(_queues *queues, int64_t qid, // Pop off the next item from the queue. _PyXIData_t *data = NULL; - err = _queue_next(queue, &data, p_fmt, p_unboundop); + err = _queue_next(queue, &data, p_unboundop); _queue_unmark_waiter(queue, queues->mutex); if (err != 0) { return err; @@ -1216,6 +1211,20 @@ queue_get(_queues *queues, int64_t qid, return 0; } +static int +queue_get_defaults(_queues *queues, int64_t qid, + struct _queuedefaults *p_defaults) +{ + _queue *queue = NULL; + int err = _queues_lookup(queues, qid, &queue); + if (err != 0) { + return err; + } + *p_defaults = queue->defaults; + _queue_unmark_waiter(queue, queues->mutex); + return 0; +} + static int queue_get_maxsize(_queues *queues, int64_t qid, Py_ssize_t *p_maxsize) { @@ -1270,7 +1279,7 @@ set_external_queue_type(module_state *state, PyTypeObject *queue_type) } // Add and register the new type. - if (ensure_xid_class(queue_type, _queueobj_shared) < 0) { + if (ensure_xid_class(queue_type, GETDATA(_queueobj_shared)) < 0) { return -1; } state->queue_type = (PyTypeObject *)Py_NewRef(queue_type); @@ -1348,10 +1357,11 @@ _queueobj_from_xid(_PyXIData_t *data) PyObject *mod = _get_current_module(); if (mod == NULL) { - // XXX import it? - PyErr_SetString(PyExc_RuntimeError, - MODULE_NAME_STR " module not imported yet"); - return NULL; + mod = PyImport_ImportModule(MODULE_NAME_STR); + if (mod == NULL) { + Py_DECREF(qidobj); + return NULL; + } } PyTypeObject *cls = get_external_queue_type(mod); @@ -1474,22 +1484,28 @@ qidarg_converter(PyObject *arg, void *ptr) static PyObject * queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"maxsize", "fmt", "unboundop", NULL}; + static char *kwlist[] = {"maxsize", "unboundop", "fallback", NULL}; Py_ssize_t maxsize; - int fmt; - int unboundop; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "nii:create", kwlist, - &maxsize, &fmt, &unboundop)) + int unboundarg = -1; + int fallbackarg = -1; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "n|ii:create", kwlist, + &maxsize, &unboundarg, &fallbackarg)) { return NULL; } - if (!check_unbound(unboundop)) { - PyErr_Format(PyExc_ValueError, - "unsupported unboundop %d", unboundop); + struct _queuedefaults defaults = {0}; + if (resolve_unboundop(unboundarg, UNBOUND_REPLACE, + &defaults.unboundop) < 0) + { + return NULL; + } + if (resolve_fallback(fallbackarg, _PyXIDATA_FULL_FALLBACK, + &defaults.fallback) < 0) + { return NULL; } - int64_t qid = queue_create(&_globals.queues, maxsize, fmt, unboundop); + int64_t qid = queue_create(&_globals.queues, maxsize, defaults); if (qid < 0) { (void)handle_queue_error((int)qid, self, qid); return NULL; @@ -1511,7 +1527,7 @@ queuesmod_create(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(queuesmod_create_doc, -"create(maxsize, fmt, unboundop) -> qid\n\ +"create(maxsize, unboundop, fallback) -> qid\n\ \n\ Create a new cross-interpreter queue and return its unique generated ID.\n\ It is a new reference as though bind() had been called on the queue.\n\ @@ -1560,8 +1576,9 @@ queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) } struct queue_id_and_info *cur = qids; for (int64_t i=0; i < count; cur++, i++) { - PyObject *item = Py_BuildValue("Lii", cur->id, cur->fmt, - cur->unboundop); + PyObject *item = Py_BuildValue("Lii", cur->id, + cur->defaults.unboundop, + cur->defaults.fallback); if (item == NULL) { Py_SETREF(ids, NULL); break; @@ -1575,34 +1592,44 @@ queuesmod_list_all(PyObject *self, PyObject *Py_UNUSED(ignored)) } PyDoc_STRVAR(queuesmod_list_all_doc, -"list_all() -> [(qid, fmt)]\n\ +"list_all() -> [(qid, unboundop, fallback)]\n\ \n\ Return the list of IDs for all queues.\n\ -Each corresponding default format is also included."); +Each corresponding default unbound op and fallback is also included."); static PyObject * queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) { - static char *kwlist[] = {"qid", "obj", "fmt", "unboundop", NULL}; + static char *kwlist[] = {"qid", "obj", "unboundop", "fallback", NULL}; qidarg_converter_data qidarg = {0}; PyObject *obj; - int fmt; - int unboundop; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&Oii:put", kwlist, - qidarg_converter, &qidarg, &obj, &fmt, - &unboundop)) + int unboundarg = -1; + int fallbackarg = -1; + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O&O|ii:put", kwlist, + qidarg_converter, &qidarg, &obj, + &unboundarg, &fallbackarg)) { return NULL; } int64_t qid = qidarg.id; - if (!check_unbound(unboundop)) { - PyErr_Format(PyExc_ValueError, - "unsupported unboundop %d", unboundop); + struct _queuedefaults defaults = {-1, -1}; + if (unboundarg < 0 || fallbackarg < 0) { + int err = queue_get_defaults(&_globals.queues, qid, &defaults); + if (handle_queue_error(err, self, qid)) { + return NULL; + } + } + unboundop_t unboundop; + if (resolve_unboundop(unboundarg, defaults.unboundop, &unboundop) < 0) { + return NULL; + } + xidata_fallback_t fallback; + if (resolve_fallback(fallbackarg, defaults.fallback, &fallback) < 0) { return NULL; } /* Queue up the object. */ - int err = queue_put(&_globals.queues, qid, obj, fmt, unboundop); + int err = queue_put(&_globals.queues, qid, obj, unboundop, fallback); // This is the only place that raises QueueFull. if (handle_queue_error(err, self, qid)) { return NULL; @@ -1612,7 +1639,7 @@ queuesmod_put(PyObject *self, PyObject *args, PyObject *kwds) } PyDoc_STRVAR(queuesmod_put_doc, -"put(qid, obj, fmt)\n\ +"put(qid, obj)\n\ \n\ Add the object's data to the queue."); @@ -1628,27 +1655,26 @@ queuesmod_get(PyObject *self, PyObject *args, PyObject *kwds) int64_t qid = qidarg.id; PyObject *obj = NULL; - int fmt = 0; int unboundop = 0; - int err = queue_get(&_globals.queues, qid, &obj, &fmt, &unboundop); + int err = queue_get(&_globals.queues, qid, &obj, &unboundop); // This is the only place that raises QueueEmpty. if (handle_queue_error(err, self, qid)) { return NULL; } if (obj == NULL) { - return Py_BuildValue("Oii", Py_None, fmt, unboundop); + return Py_BuildValue("Oi", Py_None, unboundop); } - PyObject *res = Py_BuildValue("OiO", obj, fmt, Py_None); + PyObject *res = Py_BuildValue("OO", obj, Py_None); Py_DECREF(obj); return res; } PyDoc_STRVAR(queuesmod_get_doc, -"get(qid) -> (obj, fmt)\n\ +"get(qid) -> (obj, unboundop)\n\ \n\ Return a new object from the data at the front of the queue.\n\ -The object's format is also returned.\n\ +The unbound op is also returned.\n\ \n\ If there is nothing to receive then raise QueueEmpty."); @@ -1748,17 +1774,14 @@ queuesmod_get_queue_defaults(PyObject *self, PyObject *args, PyObject *kwds) } int64_t qid = qidarg.id; - _queue *queue = NULL; - int err = _queues_lookup(&_globals.queues, qid, &queue); + struct _queuedefaults defaults = {0}; + int err = queue_get_defaults(&_globals.queues, qid, &defaults); if (handle_queue_error(err, self, qid)) { return NULL; } - int fmt = queue->defaults.fmt; - int unboundop = queue->defaults.unboundop; - _queue_unmark_waiter(queue, _globals.queues.mutex); - PyObject *defaults = Py_BuildValue("ii", fmt, unboundop); - return defaults; + PyObject *res = Py_BuildValue("ii", defaults.unboundop, defaults.fallback); + return res; } PyDoc_STRVAR(queuesmod_get_queue_defaults_doc, @@ -1931,8 +1954,7 @@ static int module_traverse(PyObject *mod, visitproc visit, void *arg) { module_state *state = get_module_state(mod); - (void)traverse_module_state(state, visit, arg); - return 0; + return traverse_module_state(state, visit, arg); } static int @@ -1941,8 +1963,7 @@ module_clear(PyObject *mod) module_state *state = get_module_state(mod); // Now we clear the module state. - (void)clear_module_state(state); - return 0; + return clear_module_state(state); } static void diff --git a/Modules/_interpreters_common.h b/Modules/_interpreters_common.h index edd65577284a20f..40fd51d752e324b 100644 --- a/Modules/_interpreters_common.h +++ b/Modules/_interpreters_common.h @@ -5,8 +5,10 @@ _RESOLVE_MODINIT_FUNC_NAME(NAME) +#define GETDATA(FUNC) ((_PyXIData_getdata_t){.basic=FUNC}) + static int -ensure_xid_class(PyTypeObject *cls, xidatafunc getdata) +ensure_xid_class(PyTypeObject *cls, _PyXIData_getdata_t getdata) { PyThreadState *tstate = PyThreadState_Get(); return _PyXIData_RegisterClass(tstate, cls, getdata); @@ -37,10 +39,37 @@ _get_interpid(_PyXIData_t *data) } +#ifdef HAS_FALLBACK +static int +resolve_fallback(int arg, xidata_fallback_t dflt, + xidata_fallback_t *p_fallback) +{ + if (arg < 0) { + *p_fallback = dflt; + return 0; + } + xidata_fallback_t fallback; + if (arg == _PyXIDATA_XIDATA_ONLY) { + fallback =_PyXIDATA_XIDATA_ONLY; + } + else if (arg == _PyXIDATA_FULL_FALLBACK) { + fallback = _PyXIDATA_FULL_FALLBACK; + } + else { + PyErr_Format(PyExc_ValueError, "unsupported fallback %d", arg); + return -1; + } + *p_fallback = fallback; + return 0; +} +#endif + + /* unbound items ************************************************************/ #ifdef HAS_UNBOUND_ITEMS +typedef int unboundop_t; #define UNBOUND_REMOVE 1 #define UNBOUND_ERROR 2 #define UNBOUND_REPLACE 3 @@ -51,6 +80,7 @@ _get_interpid(_PyXIData_t *data) // object is released but the underlying data is copied (with the "raw" // allocator) and used when the item is popped off the queue. +#ifndef NDEBUG static int check_unbound(int unboundop) { @@ -63,5 +93,31 @@ check_unbound(int unboundop) return 0; } } +#endif + +static int +resolve_unboundop(int arg, unboundop_t dflt, unboundop_t *p_unboundop) +{ + if (arg < 0) { + *p_unboundop = dflt; + return 0; + } + unboundop_t op; + if (arg == UNBOUND_REMOVE) { + op = UNBOUND_REMOVE; + } + else if (arg == UNBOUND_ERROR) { + op = UNBOUND_ERROR; + } + else if (arg == UNBOUND_REPLACE) { + op = UNBOUND_REPLACE; + } + else { + PyErr_Format(PyExc_ValueError, "unsupported unboundop %d", arg); + return -1; + } + *p_unboundop = op; + return 0; +} #endif diff --git a/Modules/_interpretersmodule.c b/Modules/_interpretersmodule.c index 77678f7c1260053..f6abf1061325b6a 100644 --- a/Modules/_interpretersmodule.c +++ b/Modules/_interpretersmodule.c @@ -8,6 +8,7 @@ #include "Python.h" #include "pycore_code.h" // _PyCode_HAS_EXECUTORS() #include "pycore_crossinterp.h" // _PyXIData_t +#include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_interp.h" // _PyInterpreterState_IDIncref() #include "pycore_modsupport.h" // _PyArg_BadArgument() #include "pycore_namespace.h" // _PyNamespace_New() @@ -71,6 +72,22 @@ is_running_main(PyInterpreterState *interp) } +static inline int +is_notshareable_raised(PyThreadState *tstate) +{ + PyObject *exctype = _PyXIData_GetNotShareableErrorType(tstate); + return _PyErr_ExceptionMatches(tstate, exctype); +} + +static void +unwrap_not_shareable(PyThreadState *tstate, _PyXI_failure *failure) +{ + if (_PyXI_UnwrapNotShareableError(tstate, failure) < 0) { + _PyErr_Clear(tstate); + } +} + + /* Cross-interpreter Buffer Views *******************************************/ /* When a memoryview object is "shared" between interpreters, @@ -120,7 +137,7 @@ xibufferview_from_buffer(PyTypeObject *cls, Py_buffer *view, int64_t interpid) Py_buffer *copied = PyMem_RawMalloc(sizeof(Py_buffer)); if (copied == NULL) { - return NULL; + return PyErr_NoMemory(); } /* This steals the view->obj reference */ *copied = *view; @@ -128,7 +145,7 @@ xibufferview_from_buffer(PyTypeObject *cls, Py_buffer *view, int64_t interpid) xibufferview *self = PyObject_Malloc(sizeof(xibufferview)); if (self == NULL) { PyMem_RawFree(copied); - return NULL; + return PyErr_NoMemory(); } PyObject_Init(&self->base, cls); *self = (xibufferview){ @@ -253,6 +270,7 @@ _pybuffer_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *data) { struct xibuffer *view = PyMem_RawMalloc(sizeof(struct xibuffer)); if (view == NULL) { + PyErr_NoMemory(); return -1; } view->used = 0; @@ -286,7 +304,7 @@ register_memoryview_xid(PyObject *mod, PyTypeObject **p_state) *p_state = cls; // Register XID for the builtin memoryview type. - if (ensure_xid_class(&PyMemoryView_Type, _pybuffer_shared) < 0) { + if (ensure_xid_class(&PyMemoryView_Type, GETDATA(_pybuffer_shared)) < 0) { return -1; } // We don't ever bother un-registering memoryview. @@ -319,10 +337,10 @@ _get_current_module_state(void) { PyObject *mod = _get_current_module(); if (mod == NULL) { - // XXX import it? - PyErr_SetString(PyExc_RuntimeError, - MODULE_NAME_STR " module not imported yet"); - return NULL; + mod = PyImport_ImportModule(MODULE_NAME_STR); + if (mod == NULL) { + return NULL; + } } module_state *state = get_module_state(mod); Py_DECREF(mod); @@ -359,96 +377,6 @@ _get_current_xibufferview_type(void) } -/* Python code **************************************************************/ - -static const char * -check_code_str(PyUnicodeObject *text) -{ - assert(text != NULL); - if (PyUnicode_GET_LENGTH(text) == 0) { - return "too short"; - } - - // XXX Verify that it parses? - - return NULL; -} - -static const char * -check_code_object(PyCodeObject *code) -{ - assert(code != NULL); - if (code->co_argcount > 0 - || code->co_posonlyargcount > 0 - || code->co_kwonlyargcount > 0 - || code->co_flags & (CO_VARARGS | CO_VARKEYWORDS)) - { - return "arguments not supported"; - } - if (code->co_ncellvars > 0) { - return "closures not supported"; - } - // We trust that no code objects under co_consts have unbound cell vars. - - if (_PyCode_HAS_EXECUTORS(code) || _PyCode_HAS_INSTRUMENTATION(code)) { - return "only basic functions are supported"; - } - if (code->_co_monitoring != NULL) { - return "only basic functions are supported"; - } - if (code->co_extra != NULL) { - return "only basic functions are supported"; - } - - return NULL; -} - -#define RUN_TEXT 1 -#define RUN_CODE 2 - -static const char * -get_code_str(PyObject *arg, Py_ssize_t *len_p, PyObject **bytes_p, int *flags_p) -{ - const char *codestr = NULL; - Py_ssize_t len = -1; - PyObject *bytes_obj = NULL; - int flags = 0; - - if (PyUnicode_Check(arg)) { - assert(PyUnicode_Check(arg) - && (check_code_str((PyUnicodeObject *)arg) == NULL)); - codestr = PyUnicode_AsUTF8AndSize(arg, &len); - if (codestr == NULL) { - return NULL; - } - if (strlen(codestr) != (size_t)len) { - PyErr_SetString(PyExc_ValueError, - "source code string cannot contain null bytes"); - return NULL; - } - flags = RUN_TEXT; - } - else { - assert(PyCode_Check(arg) - && (check_code_object((PyCodeObject *)arg) == NULL)); - flags = RUN_CODE; - - // Serialize the code object. - bytes_obj = PyMarshal_WriteObjectToString(arg, Py_MARSHAL_VERSION); - if (bytes_obj == NULL) { - return NULL; - } - codestr = PyBytes_AS_STRING(bytes_obj); - len = PyBytes_GET_SIZE(bytes_obj); - } - - *flags_p = flags; - *bytes_p = bytes_obj; - *len_p = len; - return codestr; -} - - /* interpreter-specific code ************************************************/ static int @@ -511,73 +439,290 @@ config_from_object(PyObject *configobj, PyInterpreterConfig *config) } +struct interp_call { + _PyXIData_t *func; + _PyXIData_t *args; + _PyXIData_t *kwargs; + struct { + _PyXIData_t func; + _PyXIData_t args; + _PyXIData_t kwargs; + } _preallocated; +}; + +static void +_interp_call_clear(struct interp_call *call) +{ + if (call->func != NULL) { + _PyXIData_Clear(NULL, call->func); + } + if (call->args != NULL) { + _PyXIData_Clear(NULL, call->args); + } + if (call->kwargs != NULL) { + _PyXIData_Clear(NULL, call->kwargs); + } + *call = (struct interp_call){0}; +} + static int -_run_script(PyObject *ns, const char *codestr, Py_ssize_t codestrlen, int flags) +_interp_call_pack(PyThreadState *tstate, struct interp_call *call, + PyObject *func, PyObject *args, PyObject *kwargs) { - PyObject *result = NULL; - if (flags & RUN_TEXT) { - result = PyRun_StringFlags(codestr, Py_file_input, ns, ns, NULL); - } - else if (flags & RUN_CODE) { - PyObject *code = PyMarshal_ReadObjectFromString(codestr, codestrlen); - if (code != NULL) { - result = PyEval_EvalCode(code, ns, ns); - Py_DECREF(code); + xidata_fallback_t fallback = _PyXIDATA_FULL_FALLBACK; + assert(call->func == NULL); + assert(call->args == NULL); + assert(call->kwargs == NULL); + // Handle the func. + if (!PyCallable_Check(func)) { + _PyErr_Format(tstate, PyExc_TypeError, + "expected a callable, got %R", func); + return -1; + } + if (_PyFunction_GetXIData(tstate, func, &call->_preallocated.func) < 0) { + PyObject *exc = _PyErr_GetRaisedException(tstate); + if (_PyPickle_GetXIData(tstate, func, &call->_preallocated.func) < 0) { + _PyErr_SetRaisedException(tstate, exc); + return -1; } + Py_DECREF(exc); + } + call->func = &call->_preallocated.func; + // Handle the args. + if (args == NULL || args == Py_None) { + // Leave it empty. } else { - Py_UNREACHABLE(); + assert(PyTuple_Check(args)); + if (PyTuple_GET_SIZE(args) > 0) { + if (_PyObject_GetXIData( + tstate, args, fallback, &call->_preallocated.args) < 0) + { + _interp_call_clear(call); + return -1; + } + call->args = &call->_preallocated.args; + } } - if (result == NULL) { - return -1; + // Handle the kwargs. + if (kwargs == NULL || kwargs == Py_None) { + // Leave it empty. + } + else { + assert(PyDict_Check(kwargs)); + if (PyDict_GET_SIZE(kwargs) > 0) { + if (_PyObject_GetXIData( + tstate, kwargs, fallback, &call->_preallocated.kwargs) < 0) + { + _interp_call_clear(call); + return -1; + } + call->kwargs = &call->_preallocated.kwargs; + } } - Py_DECREF(result); // We throw away the result. return 0; } +static void +wrap_notshareable(PyThreadState *tstate, const char *label) +{ + if (!is_notshareable_raised(tstate)) { + return; + } + assert(label != NULL && strlen(label) > 0); + PyObject *cause = _PyErr_GetRaisedException(tstate); + _PyXIData_FormatNotShareableError(tstate, "%s not shareable", label); + PyObject *exc = _PyErr_GetRaisedException(tstate); + PyException_SetCause(exc, cause); + _PyErr_SetRaisedException(tstate, exc); +} + static int -_run_in_interpreter(PyInterpreterState *interp, - const char *codestr, Py_ssize_t codestrlen, - PyObject *shareables, int flags, - PyObject **p_excinfo) +_interp_call_unpack(struct interp_call *call, + PyObject **p_func, PyObject **p_args, PyObject **p_kwargs) { - assert(!PyErr_Occurred()); - _PyXI_session session = {0}; + PyThreadState *tstate = PyThreadState_Get(); - // Prep and switch interpreters. - if (_PyXI_Enter(&session, interp, shareables) < 0) { - if (PyErr_Occurred()) { - // If an error occured at this step, it means that interp - // was not prepared and switched. + // Unpack the func. + PyObject *func = _PyXIData_NewObject(call->func); + if (func == NULL) { + wrap_notshareable(tstate, "func"); + return -1; + } + // Unpack the args. + PyObject *args; + if (call->args == NULL) { + args = PyTuple_New(0); + if (args == NULL) { + Py_DECREF(func); return -1; } - // Now, apply the error from another interpreter: - PyObject *excinfo = _PyXI_ApplyError(session.error); - if (excinfo != NULL) { - *p_excinfo = excinfo; + } + else { + args = _PyXIData_NewObject(call->args); + if (args == NULL) { + wrap_notshareable(tstate, "args"); + Py_DECREF(func); + return -1; } - assert(PyErr_Occurred()); + assert(PyTuple_Check(args)); + } + // Unpack the kwargs. + PyObject *kwargs = NULL; + if (call->kwargs != NULL) { + kwargs = _PyXIData_NewObject(call->kwargs); + if (kwargs == NULL) { + wrap_notshareable(tstate, "kwargs"); + Py_DECREF(func); + Py_DECREF(args); + return -1; + } + assert(PyDict_Check(kwargs)); + } + *p_func = func; + *p_args = args; + *p_kwargs = kwargs; + return 0; +} + +static int +_make_call(struct interp_call *call, + PyObject **p_result, _PyXI_failure *failure) +{ + assert(call != NULL && call->func != NULL); + PyThreadState *tstate = _PyThreadState_GET(); + + // Get the func and args. + PyObject *func = NULL, *args = NULL, *kwargs = NULL; + if (_interp_call_unpack(call, &func, &args, &kwargs) < 0) { + assert(func == NULL); + assert(args == NULL); + assert(kwargs == NULL); + _PyXI_InitFailure(failure, _PyXI_ERR_OTHER, NULL); + unwrap_not_shareable(tstate, failure); return -1; } + assert(!_PyErr_Occurred(tstate)); - // Run the script. - int res = _run_script(session.main_ns, codestr, codestrlen, flags); + // Make the call. + PyObject *resobj = PyObject_Call(func, args, kwargs); + Py_DECREF(func); + Py_XDECREF(args); + Py_XDECREF(kwargs); + if (resobj == NULL) { + return -1; + } + *p_result = resobj; + return 0; +} - // Clean up and switch back. - _PyXI_Exit(&session); +static int +_run_script(_PyXIData_t *script, PyObject *ns, _PyXI_failure *failure) +{ + PyObject *code = _PyXIData_NewObject(script); + if (code == NULL) { + _PyXI_InitFailure(failure, _PyXI_ERR_NOT_SHAREABLE, NULL); + return -1; + } + PyObject *result = PyEval_EvalCode(code, ns, ns); + Py_DECREF(code); + if (result == NULL) { + _PyXI_InitFailure(failure, _PyXI_ERR_UNCAUGHT_EXCEPTION, NULL); + return -1; + } + assert(result == Py_None); + Py_DECREF(result); // We throw away the result. + return 0; +} - // Propagate any exception out to the caller. - assert(!PyErr_Occurred()); - if (res < 0) { - PyObject *excinfo = _PyXI_ApplyCapturedException(&session); - if (excinfo != NULL) { - *p_excinfo = excinfo; +struct run_result { + PyObject *result; + PyObject *excinfo; +}; + +static void +_run_result_clear(struct run_result *runres) +{ + Py_CLEAR(runres->result); + Py_CLEAR(runres->excinfo); +} + +static int +_run_in_interpreter(PyThreadState *tstate, PyInterpreterState *interp, + _PyXIData_t *script, struct interp_call *call, + PyObject *shareables, struct run_result *runres) +{ + assert(!_PyErr_Occurred(tstate)); + int res = -1; + _PyXI_failure *failure = _PyXI_NewFailure(); + if (failure == NULL) { + return -1; + } + _PyXI_session *session = _PyXI_NewSession(); + if (session == NULL) { + _PyXI_FreeFailure(failure); + return -1; + } + _PyXI_session_result result = {0}; + + // Prep and switch interpreters. + if (_PyXI_Enter(session, interp, shareables, &result) < 0) { + // If an error occurred at this step, it means that interp + // was not prepared and switched. + _PyXI_FreeSession(session); + _PyXI_FreeFailure(failure); + assert(result.excinfo == NULL); + return -1; + } + + // Run in the interpreter. + if (script != NULL) { + assert(call == NULL); + PyObject *mainns = _PyXI_GetMainNamespace(session, failure); + if (mainns == NULL) { + goto finally; } + res = _run_script(script, mainns, failure); } else { - assert(!_PyXI_HasCapturedException(&session)); + assert(call != NULL); + PyObject *resobj; + res = _make_call(call, &resobj, failure); + if (res == 0) { + res = _PyXI_Preserve(session, "resobj", resobj, failure); + Py_DECREF(resobj); + if (res < 0) { + goto finally; + } + } } +finally: + // Clean up and switch back. + (void)res; + int exitres = _PyXI_Exit(session, failure, &result); + assert(res == 0 || exitres != 0); + _PyXI_FreeSession(session); + _PyXI_FreeFailure(failure); + + res = exitres; + if (_PyErr_Occurred(tstate)) { + // It's a directly propagated exception. + assert(res < 0); + } + else if (res < 0) { + assert(result.excinfo != NULL); + runres->excinfo = Py_NewRef(result.excinfo); + res = -1; + } + else { + assert(result.excinfo == NULL); + runres->result = _PyXI_GetPreserved(&result, "resobj"); + if (_PyErr_Occurred(tstate)) { + res = -1; + } + } + _PyXI_ClearResult(&result); return res; } @@ -895,8 +1040,8 @@ interp_set___main___attrs(PyObject *self, PyObject *args, PyObject *kwargs) PyObject *id, *updates; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "OO|$p:" MODULE_NAME_STR ".set___main___attrs", - kwlist, &id, &updates, &restricted)) + "OO!|$p:" MODULE_NAME_STR ".set___main___attrs", + kwlist, &id, &PyDict_Type, &updates, &restricted)) { return NULL; } @@ -910,34 +1055,39 @@ interp_set___main___attrs(PyObject *self, PyObject *args, PyObject *kwargs) } // Check the updates. - if (updates != Py_None) { - Py_ssize_t size = PyObject_Size(updates); - if (size < 0) { - return NULL; - } - if (size == 0) { - PyErr_SetString(PyExc_ValueError, - "arg 2 must be a non-empty mapping"); - return NULL; - } + Py_ssize_t size = PyDict_Size(updates); + if (size < 0) { + return NULL; + } + if (size == 0) { + PyErr_SetString(PyExc_ValueError, + "arg 2 must be a non-empty dict"); + return NULL; } - _PyXI_session session = {0}; + _PyXI_session *session = _PyXI_NewSession(); + if (session == NULL) { + return NULL; + } // Prep and switch interpreters, including apply the updates. - if (_PyXI_Enter(&session, interp, updates) < 0) { - if (!PyErr_Occurred()) { - _PyXI_ApplyCapturedException(&session); - assert(PyErr_Occurred()); - } - else { - assert(!_PyXI_HasCapturedException(&session)); - } + if (_PyXI_Enter(session, interp, updates, NULL) < 0) { + _PyXI_FreeSession(session); return NULL; } // Clean up and switch back. - _PyXI_Exit(&session); + assert(!PyErr_Occurred()); + int res = _PyXI_Exit(session, NULL, NULL); + _PyXI_FreeSession(session); + assert(res == 0); + if (res < 0) { + // unreachable + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_RuntimeError, "unresolved error"); + } + return NULL; + } Py_RETURN_NONE; } @@ -948,122 +1098,31 @@ PyDoc_STRVAR(set___main___attrs_doc, Bind the given attributes in the interpreter's __main__ module."); -static PyUnicodeObject * -convert_script_arg(PyObject *arg, const char *fname, const char *displayname, - const char *expected) -{ - PyUnicodeObject *str = NULL; - if (PyUnicode_CheckExact(arg)) { - str = (PyUnicodeObject *)Py_NewRef(arg); - } - else if (PyUnicode_Check(arg)) { - // XXX str = PyUnicode_FromObject(arg); - str = (PyUnicodeObject *)Py_NewRef(arg); - } - else { - _PyArg_BadArgument(fname, displayname, expected, arg); - return NULL; - } - - const char *err = check_code_str(str); - if (err != NULL) { - Py_DECREF(str); - PyErr_Format(PyExc_ValueError, - "%.200s(): bad script text (%s)", fname, err); - return NULL; - } - - return str; -} - -static PyCodeObject * -convert_code_arg(PyObject *arg, const char *fname, const char *displayname, - const char *expected) +static PyObject * +_handle_script_error(struct run_result *runres) { - const char *kind = NULL; - PyCodeObject *code = NULL; - if (PyFunction_Check(arg)) { - if (PyFunction_GetClosure(arg) != NULL) { - PyErr_Format(PyExc_ValueError, - "%.200s(): closures not supported", fname); - return NULL; - } - code = (PyCodeObject *)PyFunction_GetCode(arg); - if (code == NULL) { - if (PyErr_Occurred()) { - // This chains. - PyErr_Format(PyExc_ValueError, - "%.200s(): bad func", fname); - } - else { - PyErr_Format(PyExc_ValueError, - "%.200s(): func.__code__ missing", fname); - } - return NULL; - } - Py_INCREF(code); - kind = "func"; - } - else if (PyCode_Check(arg)) { - code = (PyCodeObject *)Py_NewRef(arg); - kind = "code object"; - } - else { - _PyArg_BadArgument(fname, displayname, expected, arg); - return NULL; - } - - const char *err = check_code_object(code); - if (err != NULL) { - Py_DECREF(code); - PyErr_Format(PyExc_ValueError, - "%.200s(): bad %s (%s)", fname, kind, err); + assert(runres->result == NULL); + if (runres->excinfo == NULL) { + assert(PyErr_Occurred()); return NULL; } - - return code; -} - -static int -_interp_exec(PyObject *self, PyInterpreterState *interp, - PyObject *code_arg, PyObject *shared_arg, PyObject **p_excinfo) -{ - if (shared_arg != NULL && !PyDict_CheckExact(shared_arg)) { - PyErr_SetString(PyExc_TypeError, "expected 'shared' to be a dict"); - return -1; - } - - // Extract code. - Py_ssize_t codestrlen = -1; - PyObject *bytes_obj = NULL; - int flags = 0; - const char *codestr = get_code_str(code_arg, - &codestrlen, &bytes_obj, &flags); - if (codestr == NULL) { - return -1; - } - - // Run the code in the interpreter. - int res = _run_in_interpreter(interp, codestr, codestrlen, - shared_arg, flags, p_excinfo); - Py_XDECREF(bytes_obj); - if (res < 0) { - return -1; - } - - return 0; + assert(!PyErr_Occurred()); + return runres->excinfo; } static PyObject * interp_exec(PyObject *self, PyObject *args, PyObject *kwds) { +#define FUNCNAME MODULE_NAME_STR ".exec" + PyThreadState *tstate = _PyThreadState_GET(); static char *kwlist[] = {"id", "code", "shared", "restrict", NULL}; PyObject *id, *code; PyObject *shared = NULL; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O$p:" MODULE_NAME_STR ".exec", kwlist, - &id, &code, &shared, &restricted)) + "OO|O!$p:" FUNCNAME, kwlist, + &id, &code, &PyDict_Type, &shared, + &restricted)) { return NULL; } @@ -1075,27 +1134,24 @@ interp_exec(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - const char *expected = "a string, a function, or a code object"; - if (PyUnicode_Check(code)) { - code = (PyObject *)convert_script_arg(code, MODULE_NAME_STR ".exec", - "argument 2", expected); - } - else { - code = (PyObject *)convert_code_arg(code, MODULE_NAME_STR ".exec", - "argument 2", expected); - } - if (code == NULL) { + // We don't need the script to be "pure", which means it can use + // global variables. They will be resolved against __main__. + _PyXIData_t xidata = {0}; + if (_PyCode_GetScriptXIData(tstate, code, &xidata) < 0) { + unwrap_not_shareable(tstate, NULL); return NULL; } - PyObject *excinfo = NULL; - int res = _interp_exec(self, interp, code, shared, &excinfo); - Py_DECREF(code); + struct run_result runres = {0}; + int res = _run_in_interpreter( + tstate, interp, &xidata, NULL, shared, &runres); + _PyXIData_Release(&xidata); if (res < 0) { - assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); - return excinfo; + return _handle_script_error(&runres); } + assert(runres.result == NULL); Py_RETURN_NONE; +#undef FUNCNAME } PyDoc_STRVAR(exec_doc, @@ -1118,13 +1174,16 @@ is ignored, including its __globals__ dict."); static PyObject * interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) { +#define FUNCNAME MODULE_NAME_STR ".run_string" + PyThreadState *tstate = _PyThreadState_GET(); static char *kwlist[] = {"id", "script", "shared", "restrict", NULL}; PyObject *id, *script; PyObject *shared = NULL; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OU|O$p:" MODULE_NAME_STR ".run_string", - kwlist, &id, &script, &shared, &restricted)) + "OU|O!$p:" FUNCNAME, kwlist, + &id, &script, &PyDict_Type, &shared, + &restricted)) { return NULL; } @@ -1136,20 +1195,27 @@ interp_run_string(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - script = (PyObject *)convert_script_arg(script, MODULE_NAME_STR ".run_string", - "argument 2", "a string"); - if (script == NULL) { + if (PyFunction_Check(script) || PyCode_Check(script)) { + _PyArg_BadArgument(FUNCNAME, "argument 2", "a string", script); + return NULL; + } + + _PyXIData_t xidata = {0}; + if (_PyCode_GetScriptXIData(tstate, script, &xidata) < 0) { + unwrap_not_shareable(tstate, NULL); return NULL; } - PyObject *excinfo = NULL; - int res = _interp_exec(self, interp, script, shared, &excinfo); - Py_DECREF(script); + struct run_result runres = {0}; + int res = _run_in_interpreter( + tstate, interp, &xidata, NULL, shared, &runres); + _PyXIData_Release(&xidata); if (res < 0) { - assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); - return excinfo; + return _handle_script_error(&runres); } + assert(runres.result == NULL); Py_RETURN_NONE; +#undef FUNCNAME } PyDoc_STRVAR(run_string_doc, @@ -1162,13 +1228,16 @@ Execute the provided string in the identified interpreter.\n\ static PyObject * interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) { +#define FUNCNAME MODULE_NAME_STR ".run_func" + PyThreadState *tstate = _PyThreadState_GET(); static char *kwlist[] = {"id", "func", "shared", "restrict", NULL}; PyObject *id, *func; PyObject *shared = NULL; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|O$p:" MODULE_NAME_STR ".run_func", - kwlist, &id, &func, &shared, &restricted)) + "OO|O!$p:" FUNCNAME, kwlist, + &id, &func, &PyDict_Type, &shared, + &restricted)) { return NULL; } @@ -1180,21 +1249,36 @@ interp_run_func(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - PyCodeObject *code = convert_code_arg(func, MODULE_NAME_STR ".exec", - "argument 2", - "a function or a code object"); - if (code == NULL) { + // We don't worry about checking globals. They will be resolved + // against __main__. + PyObject *code; + if (PyFunction_Check(func)) { + code = PyFunction_GET_CODE(func); + } + else if (PyCode_Check(func)) { + code = func; + } + else { + _PyArg_BadArgument(FUNCNAME, "argument 2", "a function", func); return NULL; } - PyObject *excinfo = NULL; - int res = _interp_exec(self, interp, (PyObject *)code, shared, &excinfo); - Py_DECREF(code); + _PyXIData_t xidata = {0}; + if (_PyCode_GetScriptXIData(tstate, code, &xidata) < 0) { + unwrap_not_shareable(tstate, NULL); + return NULL; + } + + struct run_result runres = {0}; + int res = _run_in_interpreter( + tstate, interp, &xidata, NULL, shared, &runres); + _PyXIData_Release(&xidata); if (res < 0) { - assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); - return excinfo; + return _handle_script_error(&runres); } + assert(runres.result == NULL); Py_RETURN_NONE; +#undef FUNCNAME } PyDoc_STRVAR(run_func_doc, @@ -1209,16 +1293,21 @@ are not supported. Methods and other callables are not supported either.\n\ static PyObject * interp_call(PyObject *self, PyObject *args, PyObject *kwds) { +#define FUNCNAME MODULE_NAME_STR ".call" + PyThreadState *tstate = _PyThreadState_GET(); static char *kwlist[] = {"id", "callable", "args", "kwargs", - "restrict", NULL}; + "preserve_exc", "restrict", NULL}; PyObject *id, *callable; PyObject *args_obj = NULL; PyObject *kwargs_obj = NULL; + int preserve_exc = 0; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "OO|OO$p:" MODULE_NAME_STR ".call", kwlist, - &id, &callable, &args_obj, &kwargs_obj, - &restricted)) + "OO|O!O!$pp:" FUNCNAME, kwlist, + &id, &callable, + &PyTuple_Type, &args_obj, + &PyDict_Type, &kwargs_obj, + &preserve_exc, &restricted)) { return NULL; } @@ -1230,42 +1319,37 @@ interp_call(PyObject *self, PyObject *args, PyObject *kwds) return NULL; } - if (args_obj != NULL) { - PyErr_SetString(PyExc_ValueError, "got unexpected args"); - return NULL; - } - if (kwargs_obj != NULL) { - PyErr_SetString(PyExc_ValueError, "got unexpected kwargs"); + struct interp_call call = {0}; + if (_interp_call_pack(tstate, &call, callable, args_obj, kwargs_obj) < 0) { return NULL; } - PyObject *code = (PyObject *)convert_code_arg(callable, MODULE_NAME_STR ".call", - "argument 2", "a function"); - if (code == NULL) { - return NULL; + PyObject *res_and_exc = NULL; + struct run_result runres = {0}; + if (_run_in_interpreter(tstate, interp, NULL, &call, NULL, &runres) < 0) { + if (runres.excinfo == NULL) { + assert(_PyErr_Occurred(tstate)); + goto finally; + } + assert(!_PyErr_Occurred(tstate)); } + assert(runres.result == NULL || runres.excinfo == NULL); + res_and_exc = Py_BuildValue("OO", + (runres.result ? runres.result : Py_None), + (runres.excinfo ? runres.excinfo : Py_None)); - PyObject *excinfo = NULL; - int res = _interp_exec(self, interp, code, NULL, &excinfo); - Py_DECREF(code); - if (res < 0) { - assert((excinfo == NULL) != (PyErr_Occurred() == NULL)); - return excinfo; - } - Py_RETURN_NONE; +finally: + _interp_call_clear(&call); + _run_result_clear(&runres); + return res_and_exc; +#undef FUNCNAME } PyDoc_STRVAR(call_doc, "call(id, callable, args=None, kwargs=None, *, restrict=False)\n\ \n\ Call the provided object in the identified interpreter.\n\ -Pass the given args and kwargs, if possible.\n\ -\n\ -\"callable\" may be a plain function with no free vars that takes\n\ -no arguments.\n\ -\n\ -The function's code object is used and all its state\n\ -is ignored, including its __globals__ dict."); +Pass the given args and kwargs, if possible."); static PyObject * @@ -1332,11 +1416,14 @@ interp_get_config(PyObject *self, PyObject *args, PyObject *kwds) PyObject *idobj = NULL; int restricted = 0; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "O?|$p:get_config", kwlist, + "O|$p:get_config", kwlist, &idobj, &restricted)) { return NULL; } + if (idobj == Py_None) { + idobj = NULL; + } int reqready = 0; PyInterpreterState *interp = \ @@ -1453,14 +1540,14 @@ capture_exception(PyObject *self, PyObject *args, PyObject *kwds) static char *kwlist[] = {"exc", NULL}; PyObject *exc_arg = NULL; if (!PyArg_ParseTupleAndKeywords(args, kwds, - "|O?:capture_exception", kwlist, + "|O:capture_exception", kwlist, &exc_arg)) { return NULL; } PyObject *exc = exc_arg; - if (exc == NULL) { + if (exc == NULL || exc == Py_None) { exc = PyErr_GetRaisedException(); if (exc == NULL) { Py_RETURN_NONE; @@ -1472,16 +1559,16 @@ capture_exception(PyObject *self, PyObject *args, PyObject *kwds) } PyObject *captured = NULL; - _PyXI_excinfo info = {0}; - if (_PyXI_InitExcInfo(&info, exc) < 0) { + _PyXI_excinfo *info = _PyXI_NewExcInfo(exc); + if (info == NULL) { goto finally; } - captured = _PyXI_ExcInfoAsObject(&info); + captured = _PyXI_ExcInfoAsObject(info); if (captured == NULL) { goto finally; } - PyObject *formatted = _PyXI_FormatExcInfo(&info); + PyObject *formatted = _PyXI_FormatExcInfo(info); if (formatted == NULL) { Py_CLEAR(captured); goto finally; @@ -1494,7 +1581,9 @@ capture_exception(PyObject *self, PyObject *args, PyObject *kwds) } finally: - _PyXI_ClearExcInfo(&info); + if (info != NULL) { + _PyXI_FreeExcInfo(info); + } if (exc != exc_arg) { if (PyErr_Occurred()) { PyErr_SetRaisedException(exc); @@ -1623,8 +1712,7 @@ module_traverse(PyObject *mod, visitproc visit, void *arg) { module_state *state = get_module_state(mod); assert(state != NULL); - (void)traverse_module_state(state, visit, arg); - return 0; + return traverse_module_state(state, visit, arg); } static int @@ -1632,8 +1720,7 @@ module_clear(PyObject *mod) { module_state *state = get_module_state(mod); assert(state != NULL); - (void)clear_module_state(state); - return 0; + return clear_module_state(state); } static void diff --git a/Modules/_io/_iomodule.c b/Modules/_io/_iomodule.c index 50fe5d50c91a35e..f37e4831ccad158 100644 --- a/Modules/_io/_iomodule.c +++ b/Modules/_io/_iomodule.c @@ -85,112 +85,113 @@ Open file and return a stream. Raise OSError upon failure. file is either a text or byte string giving the name (and the path if the file isn't in the current working directory) of the file to be opened or an integer file descriptor of the file to be -wrapped. (If a file descriptor is given, it is closed when the +wrapped. (If a file descriptor is given, it is closed when the returned I/O object is closed, unless closefd is set to False.) mode is an optional string that specifies the mode in which the file -is opened. It defaults to 'r' which means open for reading in text +is opened. It defaults to 'r' which means open for reading in text mode. Other common values are 'w' for writing (truncating the file if it already exists), 'x' for creating and writing to a new file, and 'a' for appending (which on some Unix systems, means that all writes append to the end of the file regardless of the current seek position). In text mode, if encoding is not specified the encoding used is platform -dependent: locale.getencoding() is called to get the current locale encoding. -(For reading and writing raw bytes use binary mode and leave encoding -unspecified.) The available modes are: +dependent: locale.getencoding() is called to get the current locale +encoding. (For reading and writing raw bytes use binary mode and leave +encoding unspecified.) The available modes are: -========= =============================================================== +========= ========================================================== Character Meaning ---------- --------------------------------------------------------------- +--------- ---------------------------------------------------------- 'r' open for reading (default) 'w' open for writing, truncating the file first 'x' create a new file and open it for writing -'a' open for writing, appending to the end of the file if it exists +'a' open for writing, appending to the end of the file if it + exists 'b' binary mode 't' text mode (default) '+' open a disk file for updating (reading and writing) -========= =============================================================== +========= ========================================================== -The default mode is 'rt' (open for reading text). For binary random +The default mode is 'rt' (open for reading text). For binary random access, the mode 'w+b' opens and truncates the file to 0 bytes, while -'r+b' opens the file without truncation. The 'x' mode implies 'w' and +'r+b' opens the file without truncation. The 'x' mode implies 'w' and raises an `FileExistsError` if the file already exists. Python distinguishes between files opened in binary and text modes, -even when the underlying operating system doesn't. Files opened in +even when the underlying operating system doesn't. Files opened in binary mode (appending 'b' to the mode argument) return contents as -bytes objects without any decoding. In text mode (the default, or when +bytes objects without any decoding. In text mode (the default, or when 't' is appended to the mode argument), the contents of the file are returned as strings, the bytes having been first decoded using a platform-dependent encoding or using the specified encoding if given. buffering is an optional integer used to set the buffering policy. -Pass 0 to switch buffering off (only allowed in binary mode), 1 to select -line buffering (only usable in text mode), and an integer > 1 to indicate -the size of a fixed-size chunk buffer. When no buffering argument is -given, the default buffering policy works as follows: +Pass 0 to switch buffering off (only allowed in binary mode), 1 to +select line buffering (only usable in text mode), and an integer > 1 to +indicate the size of a fixed-size chunk buffer. When no buffering +argument is given, the default buffering policy works as follows: * Binary files are buffered in fixed-size chunks; the size of the buffer - is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) - when the device block size is available. - On most systems, the buffer will typically be 128 kilobytes long. + is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) when the device + block size is available. + On most systems, the buffer will typically be 128 kilobytes long. * "Interactive" text files (files for which isatty() returns True) use line buffering. Other text files use the policy described above for binary files. encoding is the name of the encoding used to decode or encode the -file. This should only be used in text mode. The default encoding is +file. This should only be used in text mode. The default encoding is platform dependent, but any encoding supported by Python can be passed. See the codecs module for the list of supported encodings. errors is an optional string that specifies how encoding errors are to -be handled---this argument should not be used in binary mode. Pass +be handled---this argument should not be used in binary mode. Pass 'strict' to raise a ValueError exception if there is an encoding error (the default of None has the same effect), or pass 'ignore' to ignore -errors. (Note that ignoring encoding errors can lead to data loss.) +errors. (Note that ignoring encoding errors can lead to data loss.) See the documentation for codecs.register or run 'help(codecs.Codec)' for a list of the permitted encoding error strings. newline controls how universal newlines works (it only applies to text -mode). It can be None, '', '\n', '\r', and '\r\n'. It works as +mode). It can be None, '', '\n', '\r', and '\r\n'. It works as follows: -* On input, if newline is None, universal newlines mode is - enabled. Lines in the input can end in '\n', '\r', or '\r\n', and - these are translated into '\n' before being returned to the - caller. If it is '', universal newline mode is enabled, but line - endings are returned to the caller untranslated. If it has any of - the other legal values, input lines are only terminated by the given - string, and the line ending is returned to the caller untranslated. +* On input, if newline is None, universal newlines mode is enabled. + Lines in the input can end in '\n', '\r', or '\r\n', and these are + translated into '\n' before being returned to the caller. If it is + '', universal newline mode is enabled, but line endings are returned + to the caller untranslated. If it has any of the other legal values, + input lines are only terminated by the given string, and the line + ending is returned to the caller untranslated. * On output, if newline is None, any '\n' characters written are - translated to the system default line separator, os.linesep. If - newline is '' or '\n', no translation takes place. If newline is any + translated to the system default line separator, os.linesep. If + newline is '' or '\n', no translation takes place. If newline is any of the other legal values, any '\n' characters written are translated to the given string. If closefd is False, the underlying file descriptor will be kept open -when the file is closed. This does not work when a file name is given +when the file is closed. This does not work when a file name is given and must be True in that case. -A custom opener can be used by passing a callable as *opener*. The +A custom opener can be used by passing a callable as *opener*. The underlying file descriptor for the file object is then obtained by -calling *opener* with (*file*, *flags*). *opener* must return an open +calling *opener* with (*file*, *flags*). *opener* must return an open file descriptor (passing os.open as *opener* results in functionality similar to passing None). open() returns a file object whose type depends on the mode, and through which the standard file operations such as reading and writing -are performed. When open() is used to open a file in a text mode ('w', -'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open +are performed. When open() is used to open a file in a text mode ('w', +'r', 'wt', 'rt', etc.), it returns a TextIOWrapper. When used to open a file in a binary mode, the returned class varies: in read binary mode, it returns a BufferedReader; in write binary and append binary modes, it returns a BufferedWriter, and in read/write mode, it returns a BufferedRandom. It is also possible to use a string or bytearray as a file for both -reading and writing. For strings StringIO can be used like a file +reading and writing. For strings StringIO can be used like a file opened in a text mode, and for bytes a BytesIO can be used like a file opened in a binary mode. [clinic start generated code]*/ @@ -199,7 +200,7 @@ static PyObject * _io_open_impl(PyObject *module, PyObject *file, const char *mode, int buffering, const char *encoding, const char *errors, const char *newline, int closefd, PyObject *opener) -/*[clinic end generated code: output=aefafc4ce2b46dc0 input=28027fdaabb8d744]*/ +/*[clinic end generated code: output=aefafc4ce2b46dc0 input=b3cefa70bef404b3]*/ { size_t i; @@ -504,14 +505,14 @@ _io.open_code Opens the provided file with the intent to import the contents. -This may perform extra validation beyond open(), but is otherwise interchangeable -with calling open(path, 'rb'). +This may perform extra validation beyond open(), but is otherwise +interchangeable with calling open(path, 'rb'). [clinic start generated code]*/ static PyObject * _io_open_code_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=2fe4ecbd6f3d6844 input=f5c18e23f4b2ed9f]*/ +/*[clinic end generated code: output=2fe4ecbd6f3d6844 input=2803c35aeb63c719]*/ { return PyFile_OpenCodeObject(path); } diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 4724e97982f349e..50769e419303fe9 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -13,6 +13,7 @@ #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _Py_FatalErrorFormat() #include "pycore_pylifecycle.h" // _Py_IsInterpreterFinalizing() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "_iomodule.h" @@ -421,8 +422,7 @@ buffered_dealloc(PyObject *op) return; _PyObject_GC_UNTRACK(self); self->ok = 0; - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, self->weakreflist); if (self->buffer) { PyMem_Free(self->buffer); self->buffer = NULL; @@ -1493,11 +1493,13 @@ buffered_iternext(PyObject *op) _PyIO_State *state = find_io_state_by_def(Py_TYPE(self)); tp = Py_TYPE(self); - if (Py_IS_TYPE(tp, state->PyBufferedReader_Type) || - Py_IS_TYPE(tp, state->PyBufferedRandom_Type)) + if (tp == state->PyBufferedReader_Type || + tp == state->PyBufferedRandom_Type) { /* Skip method call overhead for speed */ + Py_BEGIN_CRITICAL_SECTION(self); line = _buffered_readline(self, -1); + Py_END_CRITICAL_SECTION(); } else { line = PyObject_CallMethodNoArgs((PyObject *)self, @@ -2312,8 +2314,7 @@ bufferedrwpair_dealloc(PyObject *op) rwpair *self = rwpair_CAST(op); PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, self->weakreflist); (void)bufferedrwpair_clear(op); tp->tp_free(self); Py_DECREF(tp); diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index e45a2d1a16dcba3..73241471f69d6c9 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -1,8 +1,11 @@ #include "Python.h" +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() #include "pycore_object.h" #include "pycore_sysmodule.h" // _PySys_GetSizeOf() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() +#include "pycore_pyatomic_ft_wrappers.h" -#include <stddef.h> // offsetof() +#include <stddef.h> // offsetof() #include "_iomodule.h" /*[clinic input] @@ -19,6 +22,9 @@ typedef struct { PyObject *dict; PyObject *weakreflist; Py_ssize_t exports; +#ifdef Py_GIL_DISABLED + int buf_shared; +#endif } bytesio; #define bytesio_CAST(op) ((bytesio *)(op)) @@ -50,7 +56,7 @@ check_closed(bytesio *self) static int check_exports(bytesio *self) { - if (self->exports > 0) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) > 0) { PyErr_SetString(PyExc_BufferError, "Existing exports of data: object cannot be re-sized"); return 1; @@ -68,15 +74,55 @@ check_exports(bytesio *self) return NULL; \ } -#define SHARED_BUF(self) (Py_REFCNT((self)->buf) > 1) +#ifdef Py_GIL_DISABLED +#define SHARED_BUF(self) ((self)->buf_shared || !_PyObject_IsUniquelyReferenced((self)->buf)) +#else +#define SHARED_BUF(self) (!_PyObject_IsUniquelyReferenced((self)->buf)) +#endif + +static inline void +set_shared_buf(bytesio *self) +{ +#ifdef Py_GIL_DISABLED + self->buf_shared = 1; +#endif +} + +static inline void +clear_shared_buf(bytesio *self) +{ +#ifdef Py_GIL_DISABLED + self->buf_shared = 0; +#endif +} + +static int +resize_unshared_buffer_lock_held(bytesio *self, Py_ssize_t size) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + +#ifdef Py_GIL_DISABLED + /* If the internal bytes object escaped via a zero-copy getvalue(), read(), + or peek(), resizing it would mutate an object visible to Python code. + Callers must detach first. */ + assert(!self->buf_shared); +#endif + int ret = _PyBytes_Resize(&self->buf, size); + if (ret == 0) { + clear_shared_buf(self); + } + return ret; +} /* Internal routine to get a line from the buffer of a BytesIO object. Returns the length between the current position to the next newline character. */ static Py_ssize_t -scan_eol(bytesio *self, Py_ssize_t len) +scan_eol_lock_held(bytesio *self, Py_ssize_t len) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + const char *start, *n; Py_ssize_t maxlen; @@ -109,11 +155,13 @@ scan_eol(bytesio *self, Py_ssize_t len) The caller should ensure that the 'size' argument is non-negative and not lesser than self->string_size. Returns 0 on success, -1 otherwise. */ static int -unshare_buffer(bytesio *self, size_t size) +unshare_buffer_lock_held(bytesio *self, size_t size) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + PyObject *new_buf; assert(SHARED_BUF(self)); - assert(self->exports == 0); + assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0); assert(size >= (size_t)self->string_size); new_buf = PyBytes_FromStringAndSize(NULL, size); if (new_buf == NULL) @@ -121,6 +169,7 @@ unshare_buffer(bytesio *self, size_t size) memcpy(PyBytes_AS_STRING(new_buf), PyBytes_AS_STRING(self->buf), self->string_size); Py_SETREF(self->buf, new_buf); + clear_shared_buf(self); return 0; } @@ -128,10 +177,12 @@ unshare_buffer(bytesio *self, size_t size) The caller should ensure that the 'size' argument is non-negative. Returns 0 on success, -1 otherwise. */ static int -resize_buffer(bytesio *self, size_t size) +resize_buffer_lock_held(bytesio *self, size_t size) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + assert(self->buf != NULL); - assert(self->exports == 0); + assert(FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0); /* Here, unsigned types are used to avoid dealing with signed integer overflow, which is undefined in C. */ @@ -160,11 +211,11 @@ resize_buffer(bytesio *self, size_t size) } if (SHARED_BUF(self)) { - if (unshare_buffer(self, alloc) < 0) + if (unshare_buffer_lock_held(self, alloc) < 0) return -1; } else { - if (_PyBytes_Resize(&self->buf, alloc) < 0) + if (resize_unshared_buffer_lock_held(self, alloc) < 0) return -1; } @@ -181,20 +232,22 @@ resize_buffer(bytesio *self, size_t size) Inlining is disabled because it's significantly decreases performance of writelines() in PGO build. */ Py_NO_INLINE static Py_ssize_t -write_bytes(bytesio *self, PyObject *b) +write_bytes_lock_held(bytesio *self, PyObject *b) { - if (check_closed(self)) { - return -1; - } - if (check_exports(self)) { - return -1; - } + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); Py_buffer buf; + Py_ssize_t len; if (PyObject_GetBuffer(b, &buf, PyBUF_CONTIG_RO) < 0) { return -1; } - Py_ssize_t len = buf.len; + + if (check_closed(self) || check_exports(self)) { + len = -1; + goto done; + } + + len = buf.len; if (len == 0) { goto done; } @@ -202,13 +255,13 @@ write_bytes(bytesio *self, PyObject *b) assert(self->pos >= 0); size_t endpos = (size_t)self->pos + len; if (endpos > (size_t)PyBytes_GET_SIZE(self->buf)) { - if (resize_buffer(self, endpos) < 0) { + if (resize_buffer_lock_held(self, endpos) < 0) { len = -1; goto done; } } else if (SHARED_BUF(self)) { - if (unshare_buffer(self, Py_MAX(endpos, (size_t)self->string_size)) < 0) { + if (unshare_buffer_lock_held(self, Py_MAX(endpos, (size_t)self->string_size)) < 0) { len = -1; goto done; } @@ -245,16 +298,21 @@ write_bytes(bytesio *self, PyObject *b) static PyObject * bytesio_get_closed(PyObject *op, void *Py_UNUSED(closure)) { + PyObject *ret; bytesio *self = bytesio_CAST(op); + Py_BEGIN_CRITICAL_SECTION(self); if (self->buf == NULL) { - Py_RETURN_TRUE; + ret = Py_True; } else { - Py_RETURN_FALSE; + ret = Py_False; } + Py_END_CRITICAL_SECTION(); + return ret; } /*[clinic input] +@critical_section _io.BytesIO.readable Returns True if the IO object can be read. @@ -262,13 +320,14 @@ Returns True if the IO object can be read. static PyObject * _io_BytesIO_readable_impl(bytesio *self) -/*[clinic end generated code: output=4e93822ad5b62263 input=96c5d0cccfb29f5c]*/ +/*[clinic end generated code: output=4e93822ad5b62263 input=ab7816facef48bfd]*/ { CHECK_CLOSED(self); Py_RETURN_TRUE; } /*[clinic input] +@critical_section _io.BytesIO.writable Returns True if the IO object can be written. @@ -276,13 +335,14 @@ Returns True if the IO object can be written. static PyObject * _io_BytesIO_writable_impl(bytesio *self) -/*[clinic end generated code: output=64ff6a254b1150b8 input=700eed808277560a]*/ +/*[clinic end generated code: output=64ff6a254b1150b8 input=4f35d49d26dab024]*/ { CHECK_CLOSED(self); Py_RETURN_TRUE; } /*[clinic input] +@critical_section _io.BytesIO.seekable Returns True if the IO object can be seeked. @@ -290,13 +350,14 @@ Returns True if the IO object can be seeked. static PyObject * _io_BytesIO_seekable_impl(bytesio *self) -/*[clinic end generated code: output=6b417f46dcc09b56 input=9421f65627a344dd]*/ +/*[clinic end generated code: output=6b417f46dcc09b56 input=9cc78d15aa1deaa3]*/ { CHECK_CLOSED(self); Py_RETURN_TRUE; } /*[clinic input] +@critical_section _io.BytesIO.flush Does nothing. @@ -304,13 +365,14 @@ Does nothing. static PyObject * _io_BytesIO_flush_impl(bytesio *self) -/*[clinic end generated code: output=187e3d781ca134a0 input=561ea490be4581a7]*/ +/*[clinic end generated code: output=187e3d781ca134a0 input=c60842743910b381]*/ { CHECK_CLOSED(self); Py_RETURN_NONE; } /*[clinic input] +@critical_section _io.BytesIO.getbuffer cls: defining_class @@ -321,7 +383,7 @@ Get a read-write view over the contents of the BytesIO object. static PyObject * _io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls) -/*[clinic end generated code: output=045091d7ce87fe4e input=0668fbb48f95dffa]*/ +/*[clinic end generated code: output=045091d7ce87fe4e input=8295764061be77fd]*/ { _PyIO_State *state = get_io_state_by_cls(cls); PyTypeObject *type = state->PyBytesIOBuffer_Type; @@ -340,6 +402,7 @@ _io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls) } /*[clinic input] +@critical_section _io.BytesIO.getvalue Retrieve the entire contents of the BytesIO object. @@ -347,27 +410,29 @@ Retrieve the entire contents of the BytesIO object. static PyObject * _io_BytesIO_getvalue_impl(bytesio *self) -/*[clinic end generated code: output=b3f6a3233c8fd628 input=4b403ac0af3973ed]*/ +/*[clinic end generated code: output=b3f6a3233c8fd628 input=c91bff398df0c352]*/ { CHECK_CLOSED(self); - if (self->string_size <= 1 || self->exports > 0) + if (self->string_size <= 1 || FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) > 0) return PyBytes_FromStringAndSize(PyBytes_AS_STRING(self->buf), self->string_size); if (self->string_size != PyBytes_GET_SIZE(self->buf)) { if (SHARED_BUF(self)) { - if (unshare_buffer(self, self->string_size) < 0) + if (unshare_buffer_lock_held(self, self->string_size) < 0) return NULL; } else { - if (_PyBytes_Resize(&self->buf, self->string_size) < 0) + if (resize_unshared_buffer_lock_held(self, self->string_size) < 0) return NULL; } } + set_shared_buf(self); return Py_NewRef(self->buf); } /*[clinic input] +@critical_section _io.BytesIO.isatty Always returns False. @@ -377,13 +442,14 @@ BytesIO objects are not connected to a TTY-like device. static PyObject * _io_BytesIO_isatty_impl(bytesio *self) -/*[clinic end generated code: output=df67712e669f6c8f input=6f97f0985d13f827]*/ +/*[clinic end generated code: output=df67712e669f6c8f input=50487b74dc5ae8a9]*/ { CHECK_CLOSED(self); Py_RETURN_FALSE; } /*[clinic input] +@critical_section _io.BytesIO.tell Current file position, an integer. @@ -391,32 +457,43 @@ Current file position, an integer. static PyObject * _io_BytesIO_tell_impl(bytesio *self) -/*[clinic end generated code: output=b54b0f93cd0e5e1d input=b106adf099cb3657]*/ +/*[clinic end generated code: output=b54b0f93cd0e5e1d input=2c7b0e8f82e05c4d]*/ { CHECK_CLOSED(self); return PyLong_FromSsize_t(self->pos); } static PyObject * -read_bytes(bytesio *self, Py_ssize_t size) +read_bytes_lock_held(bytesio *self, Py_ssize_t size) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + const char *output; assert(self->buf != NULL); assert(size <= self->string_size); if (size > 1 && self->pos == 0 && size == PyBytes_GET_SIZE(self->buf) && - self->exports == 0) { + FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0) { self->pos += size; + set_shared_buf(self); return Py_NewRef(self->buf); } + /* gh-141311: Avoid undefined behavior when self->pos (limit PY_SSIZE_T_MAX) + is beyond the size of self->buf. Assert above validates size is always in + bounds. When self->pos is out of bounds calling code sets size to 0. */ + if (size == 0) { + return PyBytes_FromStringAndSize(NULL, 0); + } + output = PyBytes_AS_STRING(self->buf) + self->pos; self->pos += size; return PyBytes_FromStringAndSize(output, size); } /*[clinic input] +@critical_section _io.BytesIO.read size: Py_ssize_t(accept={int, NoneType}) = -1 / @@ -429,7 +506,7 @@ Return an empty bytes object at EOF. static PyObject * _io_BytesIO_read_impl(bytesio *self, Py_ssize_t size) -/*[clinic end generated code: output=9cc025f21c75bdd2 input=74344a39f431c3d7]*/ +/*[clinic end generated code: output=9cc025f21c75bdd2 input=9e2f7ff3075fdd39]*/ { Py_ssize_t n; @@ -443,29 +520,31 @@ _io_BytesIO_read_impl(bytesio *self, Py_ssize_t size) size = 0; } - return read_bytes(self, size); + return read_bytes_lock_held(self, size); } /*[clinic input] +@critical_section _io.BytesIO.read1 size: Py_ssize_t(accept={int, NoneType}) = -1 / Read at most size bytes, returned as a bytes object. -If the size argument is negative or omitted, read until EOF is reached. -Return an empty bytes object at EOF. +If the size argument is negative or omitted, read until EOF is +reached. Return an empty bytes object at EOF. [clinic start generated code]*/ static PyObject * _io_BytesIO_read1_impl(bytesio *self, Py_ssize_t size) -/*[clinic end generated code: output=d0f843285aa95f1c input=440a395bf9129ef5]*/ +/*[clinic end generated code: output=d0f843285aa95f1c input=796ff4e0efccc4d9]*/ { return _io_BytesIO_read_impl(self, size); } /*[clinic input] +@critical_section _io.BytesIO.readline size: Py_ssize_t(accept={int, NoneType}) = -1 / @@ -479,18 +558,19 @@ Return an empty bytes object at EOF. static PyObject * _io_BytesIO_readline_impl(bytesio *self, Py_ssize_t size) -/*[clinic end generated code: output=4bff3c251df8ffcd input=e7c3fbd1744e2783]*/ +/*[clinic end generated code: output=4bff3c251df8ffcd input=db09d47e23cf2c9e]*/ { Py_ssize_t n; CHECK_CLOSED(self); - n = scan_eol(self, size); + n = scan_eol_lock_held(self, size); - return read_bytes(self, n); + return read_bytes_lock_held(self, n); } /*[clinic input] +@critical_section _io.BytesIO.readlines size as arg: object = None / @@ -504,7 +584,7 @@ total number of bytes in the lines returned. static PyObject * _io_BytesIO_readlines_impl(bytesio *self, PyObject *arg) -/*[clinic end generated code: output=09b8e34c880808ff input=691aa1314f2c2a87]*/ +/*[clinic end generated code: output=09b8e34c880808ff input=5c57d7d78e409985]*/ { Py_ssize_t maxsize, size, n; PyObject *result, *line; @@ -533,7 +613,7 @@ _io_BytesIO_readlines_impl(bytesio *self, PyObject *arg) return NULL; output = PyBytes_AS_STRING(self->buf) + self->pos; - while ((n = scan_eol(self, -1)) != 0) { + while ((n = scan_eol_lock_held(self, -1)) != 0) { self->pos += n; line = PyBytes_FromStringAndSize(output, n); if (!line) @@ -556,6 +636,7 @@ _io_BytesIO_readlines_impl(bytesio *self, PyObject *arg) } /*[clinic input] +@critical_section _io.BytesIO.readinto buffer: Py_buffer(accept={rwbuffer}) / @@ -568,7 +649,7 @@ is set not to block and has no data to read. static PyObject * _io_BytesIO_readinto_impl(bytesio *self, Py_buffer *buffer) -/*[clinic end generated code: output=a5d407217dcf0639 input=1424d0fdce857919]*/ +/*[clinic end generated code: output=a5d407217dcf0639 input=093a8d330de3fcd1]*/ { Py_ssize_t len, n; @@ -579,21 +660,25 @@ _io_BytesIO_readinto_impl(bytesio *self, Py_buffer *buffer) n = self->string_size - self->pos; if (len > n) { len = n; - if (len < 0) - len = 0; + if (len < 0) { + /* gh-141311: Avoid undefined behavior when self->pos (limit + PY_SSIZE_T_MAX) points beyond the size of self->buf. */ + return PyLong_FromSsize_t(0); + } } - memcpy(buffer->buf, PyBytes_AS_STRING(self->buf) + self->pos, len); - assert(self->pos + len < PY_SSIZE_T_MAX); + assert(self->pos + len <= PY_SSIZE_T_MAX); assert(len >= 0); + memcpy(buffer->buf, PyBytes_AS_STRING(self->buf) + self->pos, len); self->pos += len; return PyLong_FromSsize_t(len); } /*[clinic input] +@critical_section _io.BytesIO.truncate - size: Py_ssize_t(accept={int, NoneType}, c_default="((bytesio *)self)->pos") = None + size: object = None / Truncate the file to at most size bytes. @@ -603,44 +688,68 @@ The current file position is unchanged. Returns the new size. [clinic start generated code]*/ static PyObject * -_io_BytesIO_truncate_impl(bytesio *self, Py_ssize_t size) -/*[clinic end generated code: output=9ad17650c15fa09b input=dae4295e11c1bbb4]*/ +_io_BytesIO_truncate_impl(bytesio *self, PyObject *size) +/*[clinic end generated code: output=ab42491b4824f384 input=b4acb5f80481c053]*/ { CHECK_CLOSED(self); CHECK_EXPORTS(self); - if (size < 0) { - PyErr_Format(PyExc_ValueError, - "negative size value %zd", size); - return NULL; + Py_ssize_t new_size; + + if (size == Py_None) { + new_size = self->pos; + } + else { + new_size = PyLong_AsLong(size); + if (new_size == -1 && PyErr_Occurred()) { + return NULL; + } + if (new_size < 0) { + PyErr_Format(PyExc_ValueError, + "negative size value %zd", new_size); + return NULL; + } } - if (size < self->string_size) { - self->string_size = size; - if (resize_buffer(self, size) < 0) + if (new_size < self->string_size) { + self->string_size = new_size; + if (resize_buffer_lock_held(self, new_size) < 0) return NULL; } - return PyLong_FromSsize_t(size); + return PyLong_FromSsize_t(new_size); } static PyObject * -bytesio_iternext(PyObject *op) +bytesio_iternext_lock_held(PyObject *op) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + Py_ssize_t n; bytesio *self = bytesio_CAST(op); CHECK_CLOSED(self); - n = scan_eol(self, -1); + n = scan_eol_lock_held(self, -1); if (n == 0) return NULL; - return read_bytes(self, n); + return read_bytes_lock_held(self, n); +} + +static PyObject * +bytesio_iternext(PyObject *op) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = bytesio_iternext_lock_held(op); + Py_END_CRITICAL_SECTION(); + return ret; } /*[clinic input] +@critical_section _io.BytesIO.seek pos: Py_ssize_t whence: int = 0 @@ -657,7 +766,7 @@ Returns the new absolute position. static PyObject * _io_BytesIO_seek_impl(bytesio *self, Py_ssize_t pos, int whence) -/*[clinic end generated code: output=c26204a68e9190e4 input=1e875e6ebc652948]*/ +/*[clinic end generated code: output=c26204a68e9190e4 input=20f05ddf659255df]*/ { CHECK_CLOSED(self); @@ -700,6 +809,7 @@ _io_BytesIO_seek_impl(bytesio *self, Py_ssize_t pos, int whence) } /*[clinic input] +@critical_section _io.BytesIO.write b: object / @@ -711,13 +821,14 @@ Return the number of bytes written. static PyObject * _io_BytesIO_write_impl(bytesio *self, PyObject *b) -/*[clinic end generated code: output=d3e46bcec8d9e21c input=f5ec7c8c64ed720a]*/ +/*[clinic end generated code: output=d3e46bcec8d9e21c input=46c0c17eac7474a4]*/ { - Py_ssize_t n = write_bytes(self, b); + Py_ssize_t n = write_bytes_lock_held(self, b); return n >= 0 ? PyLong_FromSsize_t(n) : NULL; } /*[clinic input] +@critical_section _io.BytesIO.writelines lines: object / @@ -725,13 +836,13 @@ _io.BytesIO.writelines Write lines to the file. Note that newlines are not added. lines can be any iterable object -producing bytes-like objects. This is equivalent to calling write() for -each element. +producing bytes-like objects. This is equivalent to calling write() +for each element. [clinic start generated code]*/ static PyObject * _io_BytesIO_writelines_impl(bytesio *self, PyObject *lines) -/*[clinic end generated code: output=03a43a75773bc397 input=e972539176fc8fc1]*/ +/*[clinic end generated code: output=03a43a75773bc397 input=d265f76533b058e7]*/ { PyObject *it, *item; @@ -742,7 +853,7 @@ _io_BytesIO_writelines_impl(bytesio *self, PyObject *lines) return NULL; while ((item = PyIter_Next(it)) != NULL) { - Py_ssize_t ret = write_bytes(self, item); + Py_ssize_t ret = write_bytes_lock_held(self, item); Py_DECREF(item); if (ret < 0) { Py_DECREF(it); @@ -759,6 +870,7 @@ _io_BytesIO_writelines_impl(bytesio *self, PyObject *lines) } /*[clinic input] +@critical_section _io.BytesIO.close Disable all I/O operations. @@ -766,7 +878,7 @@ Disable all I/O operations. static PyObject * _io_BytesIO_close_impl(bytesio *self) -/*[clinic end generated code: output=1471bb9411af84a0 input=37e1f55556e61f60]*/ +/*[clinic end generated code: output=1471bb9411af84a0 input=34ce76d8bd17a23b]*/ { CHECK_EXPORTS(self); Py_CLEAR(self->buf); @@ -788,35 +900,49 @@ _io_BytesIO_close_impl(bytesio *self) function to use the efficient instance representation of PEP 307. */ + static PyObject * + bytesio_getstate_lock_held(PyObject *op) + { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + + bytesio *self = bytesio_CAST(op); + PyObject *initvalue = _io_BytesIO_getvalue_impl(self); + PyObject *dict; + PyObject *state; + + if (initvalue == NULL) + return NULL; + if (self->dict == NULL) { + dict = Py_NewRef(Py_None); + } + else { + dict = PyDict_Copy(self->dict); + if (dict == NULL) { + Py_DECREF(initvalue); + return NULL; + } + } + + state = Py_BuildValue("(OnN)", initvalue, self->pos, dict); + Py_DECREF(initvalue); + return state; +} + static PyObject * bytesio_getstate(PyObject *op, PyObject *Py_UNUSED(dummy)) { - bytesio *self = bytesio_CAST(op); - PyObject *initvalue = _io_BytesIO_getvalue_impl(self); - PyObject *dict; - PyObject *state; - - if (initvalue == NULL) - return NULL; - if (self->dict == NULL) { - dict = Py_NewRef(Py_None); - } - else { - dict = PyDict_Copy(self->dict); - if (dict == NULL) { - Py_DECREF(initvalue); - return NULL; - } - } - - state = Py_BuildValue("(OnN)", initvalue, self->pos, dict); - Py_DECREF(initvalue); - return state; + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = bytesio_getstate_lock_held(op); + Py_END_CRITICAL_SECTION(); + return ret; } static PyObject * -bytesio_setstate(PyObject *op, PyObject *state) +bytesio_setstate_lock_held(PyObject *op, PyObject *state) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + PyObject *result; PyObject *position_obj; PyObject *dict; @@ -890,21 +1016,30 @@ bytesio_setstate(PyObject *op, PyObject *state) Py_RETURN_NONE; } +static PyObject * +bytesio_setstate(PyObject *op, PyObject *state) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = bytesio_setstate_lock_held(op, state); + Py_END_CRITICAL_SECTION(); + return ret; +} + static void bytesio_dealloc(PyObject *op) { bytesio *self = bytesio_CAST(op); PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); - if (self->exports > 0) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) > 0) { PyErr_SetString(PyExc_SystemError, "deallocated BytesIO object has exported buffers"); PyErr_Print(); } Py_CLEAR(self->buf); Py_CLEAR(self->dict); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, self->weakreflist); tp->tp_free(self); Py_DECREF(tp); } @@ -932,6 +1067,7 @@ bytesio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } /*[clinic input] +@critical_section _io.BytesIO.__init__ initial_bytes as initvalue: object(c_default="NULL") = b'' @@ -940,13 +1076,13 @@ Buffered I/O implementation using an in-memory bytes buffer. static int _io_BytesIO___init___impl(bytesio *self, PyObject *initvalue) -/*[clinic end generated code: output=65c0c51e24c5b621 input=aac7f31b67bf0fb6]*/ +/*[clinic end generated code: output=65c0c51e24c5b621 input=3da5a74ee4c4f1ac]*/ { /* In case, __init__ is called multiple times. */ self->string_size = 0; self->pos = 0; - if (self->exports > 0) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) > 0) { PyErr_SetString(PyExc_BufferError, "Existing exports of data: object cannot be re-sized"); return -1; @@ -954,6 +1090,7 @@ _io_BytesIO___init___impl(bytesio *self, PyObject *initvalue) if (initvalue && initvalue != Py_None) { if (PyBytes_CheckExact(initvalue)) { Py_XSETREF(self->buf, Py_NewRef(initvalue)); + clear_shared_buf(self); self->string_size = PyBytes_GET_SIZE(initvalue); } else { @@ -970,8 +1107,10 @@ _io_BytesIO___init___impl(bytesio *self, PyObject *initvalue) } static PyObject * -bytesio_sizeof(PyObject *op, PyObject *Py_UNUSED(dummy)) +bytesio_sizeof_lock_held(PyObject *op) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); + bytesio *self = bytesio_CAST(op); size_t res = _PyObject_SIZE(Py_TYPE(self)); if (self->buf && !SHARED_BUF(self)) { @@ -984,6 +1123,16 @@ bytesio_sizeof(PyObject *op, PyObject *Py_UNUSED(dummy)) return PyLong_FromSize_t(res); } +static PyObject * +bytesio_sizeof(PyObject *op, PyObject *Py_UNUSED(dummy)) +{ + PyObject *ret; + Py_BEGIN_CRITICAL_SECTION(op); + ret = bytesio_sizeof_lock_held(op); + Py_END_CRITICAL_SECTION(); + return ret; +} + static int bytesio_traverse(PyObject *op, visitproc visit, void *arg) { @@ -999,7 +1148,7 @@ bytesio_clear(PyObject *op) { bytesio *self = bytesio_CAST(op); Py_CLEAR(self->dict); - if (self->exports == 0) { + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(self->exports) == 0) { Py_CLEAR(self->buf); } return 0; @@ -1077,18 +1226,15 @@ PyType_Spec bytesio_spec = { */ static int -bytesiobuf_getbuffer(PyObject *op, Py_buffer *view, int flags) +bytesiobuf_getbuffer_lock_held(PyObject *op, Py_buffer *view, int flags) { bytesiobuf *obj = bytesiobuf_CAST(op); bytesio *b = bytesio_CAST(obj->source); - if (view == NULL) { - PyErr_SetString(PyExc_BufferError, - "bytesiobuf_getbuffer: view==NULL argument is obsolete"); - return -1; - } - if (b->exports == 0 && SHARED_BUF(b)) { - if (unshare_buffer(b, b->string_size) < 0) + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(b); + + if (FT_ATOMIC_LOAD_SSIZE_RELAXED(b->exports) == 0 && SHARED_BUF(b)) { + if (unshare_buffer_lock_held(b, b->string_size) < 0) return -1; } @@ -1096,16 +1242,32 @@ bytesiobuf_getbuffer(PyObject *op, Py_buffer *view, int flags) (void)PyBuffer_FillInfo(view, op, PyBytes_AS_STRING(b->buf), b->string_size, 0, flags); - b->exports++; + FT_ATOMIC_ADD_SSIZE(b->exports, 1); return 0; } +static int +bytesiobuf_getbuffer(PyObject *op, Py_buffer *view, int flags) +{ + if (view == NULL) { + PyErr_SetString(PyExc_BufferError, + "bytesiobuf_getbuffer: view==NULL argument is obsolete"); + return -1; + } + + int ret; + Py_BEGIN_CRITICAL_SECTION(bytesiobuf_CAST(op)->source); + ret = bytesiobuf_getbuffer_lock_held(op, view, flags); + Py_END_CRITICAL_SECTION(); + return ret; +} + static void bytesiobuf_releasebuffer(PyObject *op, Py_buffer *Py_UNUSED(view)) { bytesiobuf *obj = bytesiobuf_CAST(op); bytesio *b = bytesio_CAST(obj->source); - b->exports--; + FT_ATOMIC_ADD_SSIZE(b->exports, -1); } static int diff --git a/Modules/_io/clinic/_iomodule.c.h b/Modules/_io/clinic/_iomodule.c.h index 90b80af3018fb0f..f03638064385e28 100644 --- a/Modules/_io/clinic/_iomodule.c.h +++ b/Modules/_io/clinic/_iomodule.c.h @@ -18,112 +18,113 @@ PyDoc_STRVAR(_io_open__doc__, "file is either a text or byte string giving the name (and the path\n" "if the file isn\'t in the current working directory) of the file to\n" "be opened or an integer file descriptor of the file to be\n" -"wrapped. (If a file descriptor is given, it is closed when the\n" +"wrapped. (If a file descriptor is given, it is closed when the\n" "returned I/O object is closed, unless closefd is set to False.)\n" "\n" "mode is an optional string that specifies the mode in which the file\n" -"is opened. It defaults to \'r\' which means open for reading in text\n" +"is opened. It defaults to \'r\' which means open for reading in text\n" "mode. Other common values are \'w\' for writing (truncating the file if\n" "it already exists), \'x\' for creating and writing to a new file, and\n" "\'a\' for appending (which on some Unix systems, means that all writes\n" "append to the end of the file regardless of the current seek position).\n" "In text mode, if encoding is not specified the encoding used is platform\n" -"dependent: locale.getencoding() is called to get the current locale encoding.\n" -"(For reading and writing raw bytes use binary mode and leave encoding\n" -"unspecified.) The available modes are:\n" +"dependent: locale.getencoding() is called to get the current locale\n" +"encoding. (For reading and writing raw bytes use binary mode and leave\n" +"encoding unspecified.) The available modes are:\n" "\n" -"========= ===============================================================\n" +"========= ==========================================================\n" "Character Meaning\n" -"--------- ---------------------------------------------------------------\n" +"--------- ----------------------------------------------------------\n" "\'r\' open for reading (default)\n" "\'w\' open for writing, truncating the file first\n" "\'x\' create a new file and open it for writing\n" -"\'a\' open for writing, appending to the end of the file if it exists\n" +"\'a\' open for writing, appending to the end of the file if it\n" +" exists\n" "\'b\' binary mode\n" "\'t\' text mode (default)\n" "\'+\' open a disk file for updating (reading and writing)\n" -"========= ===============================================================\n" +"========= ==========================================================\n" "\n" -"The default mode is \'rt\' (open for reading text). For binary random\n" +"The default mode is \'rt\' (open for reading text). For binary random\n" "access, the mode \'w+b\' opens and truncates the file to 0 bytes, while\n" -"\'r+b\' opens the file without truncation. The \'x\' mode implies \'w\' and\n" +"\'r+b\' opens the file without truncation. The \'x\' mode implies \'w\' and\n" "raises an `FileExistsError` if the file already exists.\n" "\n" "Python distinguishes between files opened in binary and text modes,\n" -"even when the underlying operating system doesn\'t. Files opened in\n" +"even when the underlying operating system doesn\'t. Files opened in\n" "binary mode (appending \'b\' to the mode argument) return contents as\n" -"bytes objects without any decoding. In text mode (the default, or when\n" +"bytes objects without any decoding. In text mode (the default, or when\n" "\'t\' is appended to the mode argument), the contents of the file are\n" "returned as strings, the bytes having been first decoded using a\n" "platform-dependent encoding or using the specified encoding if given.\n" "\n" "buffering is an optional integer used to set the buffering policy.\n" -"Pass 0 to switch buffering off (only allowed in binary mode), 1 to select\n" -"line buffering (only usable in text mode), and an integer > 1 to indicate\n" -"the size of a fixed-size chunk buffer. When no buffering argument is\n" -"given, the default buffering policy works as follows:\n" +"Pass 0 to switch buffering off (only allowed in binary mode), 1 to\n" +"select line buffering (only usable in text mode), and an integer > 1 to\n" +"indicate the size of a fixed-size chunk buffer. When no buffering\n" +"argument is given, the default buffering policy works as follows:\n" "\n" "* Binary files are buffered in fixed-size chunks; the size of the buffer\n" -" is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE)\n" -" when the device block size is available.\n" -" On most systems, the buffer will typically be 128 kilobytes long.\n" +" is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE) when the device\n" +" block size is available.\n" +" On most systems, the buffer will typically be 128 kilobytes long.\n" "\n" "* \"Interactive\" text files (files for which isatty() returns True)\n" " use line buffering. Other text files use the policy described above\n" " for binary files.\n" "\n" "encoding is the name of the encoding used to decode or encode the\n" -"file. This should only be used in text mode. The default encoding is\n" +"file. This should only be used in text mode. The default encoding is\n" "platform dependent, but any encoding supported by Python can be\n" "passed. See the codecs module for the list of supported encodings.\n" "\n" "errors is an optional string that specifies how encoding errors are to\n" -"be handled---this argument should not be used in binary mode. Pass\n" +"be handled---this argument should not be used in binary mode. Pass\n" "\'strict\' to raise a ValueError exception if there is an encoding error\n" "(the default of None has the same effect), or pass \'ignore\' to ignore\n" -"errors. (Note that ignoring encoding errors can lead to data loss.)\n" +"errors. (Note that ignoring encoding errors can lead to data loss.)\n" "See the documentation for codecs.register or run \'help(codecs.Codec)\'\n" "for a list of the permitted encoding error strings.\n" "\n" "newline controls how universal newlines works (it only applies to text\n" -"mode). It can be None, \'\', \'\\n\', \'\\r\', and \'\\r\\n\'. It works as\n" +"mode). It can be None, \'\', \'\\n\', \'\\r\', and \'\\r\\n\'. It works as\n" "follows:\n" "\n" -"* On input, if newline is None, universal newlines mode is\n" -" enabled. Lines in the input can end in \'\\n\', \'\\r\', or \'\\r\\n\', and\n" -" these are translated into \'\\n\' before being returned to the\n" -" caller. If it is \'\', universal newline mode is enabled, but line\n" -" endings are returned to the caller untranslated. If it has any of\n" -" the other legal values, input lines are only terminated by the given\n" -" string, and the line ending is returned to the caller untranslated.\n" +"* On input, if newline is None, universal newlines mode is enabled.\n" +" Lines in the input can end in \'\\n\', \'\\r\', or \'\\r\\n\', and these are\n" +" translated into \'\\n\' before being returned to the caller. If it is\n" +" \'\', universal newline mode is enabled, but line endings are returned\n" +" to the caller untranslated. If it has any of the other legal values,\n" +" input lines are only terminated by the given string, and the line\n" +" ending is returned to the caller untranslated.\n" "\n" "* On output, if newline is None, any \'\\n\' characters written are\n" -" translated to the system default line separator, os.linesep. If\n" -" newline is \'\' or \'\\n\', no translation takes place. If newline is any\n" +" translated to the system default line separator, os.linesep. If\n" +" newline is \'\' or \'\\n\', no translation takes place. If newline is any\n" " of the other legal values, any \'\\n\' characters written are translated\n" " to the given string.\n" "\n" "If closefd is False, the underlying file descriptor will be kept open\n" -"when the file is closed. This does not work when a file name is given\n" +"when the file is closed. This does not work when a file name is given\n" "and must be True in that case.\n" "\n" -"A custom opener can be used by passing a callable as *opener*. The\n" +"A custom opener can be used by passing a callable as *opener*. The\n" "underlying file descriptor for the file object is then obtained by\n" -"calling *opener* with (*file*, *flags*). *opener* must return an open\n" +"calling *opener* with (*file*, *flags*). *opener* must return an open\n" "file descriptor (passing os.open as *opener* results in functionality\n" "similar to passing None).\n" "\n" "open() returns a file object whose type depends on the mode, and\n" "through which the standard file operations such as reading and writing\n" -"are performed. When open() is used to open a file in a text mode (\'w\',\n" -"\'r\', \'wt\', \'rt\', etc.), it returns a TextIOWrapper. When used to open\n" +"are performed. When open() is used to open a file in a text mode (\'w\',\n" +"\'r\', \'wt\', \'rt\', etc.), it returns a TextIOWrapper. When used to open\n" "a file in a binary mode, the returned class varies: in read binary\n" "mode, it returns a BufferedReader; in write binary and append binary\n" "modes, it returns a BufferedWriter, and in read/write mode, it returns\n" "a BufferedRandom.\n" "\n" "It is also possible to use a string or bytearray as a file for both\n" -"reading and writing. For strings StringIO can be used like a file\n" +"reading and writing. For strings StringIO can be used like a file\n" "opened in a text mode, and for bytes a BytesIO can be used like a file\n" "opened in a binary mode."); @@ -352,8 +353,8 @@ PyDoc_STRVAR(_io_open_code__doc__, "\n" "Opens the provided file with the intent to import the contents.\n" "\n" -"This may perform extra validation beyond open(), but is otherwise interchangeable\n" -"with calling open(path, \'rb\')."); +"This may perform extra validation beyond open(), but is otherwise\n" +"interchangeable with calling open(path, \'rb\')."); #define _IO_OPEN_CODE_METHODDEF \ {"open_code", _PyCFunction_CAST(_io_open_code), METH_FASTCALL|METH_KEYWORDS, _io_open_code__doc__}, @@ -410,4 +411,4 @@ _io_open_code(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=7a8e032c0424bce2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5190d11f0803bfe8 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/bytesio.c.h b/Modules/_io/clinic/bytesio.c.h index aaf4884d1732a94..fad11ea6c9f6cf6 100644 --- a/Modules/_io/clinic/bytesio.c.h +++ b/Modules/_io/clinic/bytesio.c.h @@ -7,6 +7,7 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_io_BytesIO_readable__doc__, @@ -24,7 +25,13 @@ _io_BytesIO_readable_impl(bytesio *self); static PyObject * _io_BytesIO_readable(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_readable_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_readable_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_writable__doc__, @@ -42,7 +49,13 @@ _io_BytesIO_writable_impl(bytesio *self); static PyObject * _io_BytesIO_writable(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_writable_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_writable_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_seekable__doc__, @@ -60,7 +73,13 @@ _io_BytesIO_seekable_impl(bytesio *self); static PyObject * _io_BytesIO_seekable(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_seekable_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_seekable_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_flush__doc__, @@ -78,7 +97,13 @@ _io_BytesIO_flush_impl(bytesio *self); static PyObject * _io_BytesIO_flush(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_flush_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_flush_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_getbuffer__doc__, @@ -96,11 +121,18 @@ _io_BytesIO_getbuffer_impl(bytesio *self, PyTypeObject *cls); static PyObject * _io_BytesIO_getbuffer(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "getbuffer() takes no arguments"); - return NULL; + goto exit; } - return _io_BytesIO_getbuffer_impl((bytesio *)self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_getbuffer_impl((bytesio *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_io_BytesIO_getvalue__doc__, @@ -118,7 +150,13 @@ _io_BytesIO_getvalue_impl(bytesio *self); static PyObject * _io_BytesIO_getvalue(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_getvalue_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_getvalue_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_isatty__doc__, @@ -138,7 +176,13 @@ _io_BytesIO_isatty_impl(bytesio *self); static PyObject * _io_BytesIO_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_isatty_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_isatty_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_tell__doc__, @@ -156,7 +200,13 @@ _io_BytesIO_tell_impl(bytesio *self); static PyObject * _io_BytesIO_tell(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_tell_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_tell_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO_read__doc__, @@ -190,7 +240,9 @@ _io_BytesIO_read(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_read_impl((bytesio *)self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -202,8 +254,8 @@ PyDoc_STRVAR(_io_BytesIO_read1__doc__, "\n" "Read at most size bytes, returned as a bytes object.\n" "\n" -"If the size argument is negative or omitted, read until EOF is reached.\n" -"Return an empty bytes object at EOF."); +"If the size argument is negative or omitted, read until EOF is\n" +"reached. Return an empty bytes object at EOF."); #define _IO_BYTESIO_READ1_METHODDEF \ {"read1", _PyCFunction_CAST(_io_BytesIO_read1), METH_FASTCALL, _io_BytesIO_read1__doc__}, @@ -227,7 +279,9 @@ _io_BytesIO_read1(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_read1_impl((bytesio *)self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -265,7 +319,9 @@ _io_BytesIO_readline(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_readline_impl((bytesio *)self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -301,7 +357,9 @@ _io_BytesIO_readlines(PyObject *self, PyObject *const *args, Py_ssize_t nargs) } arg = args[0]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_readlines_impl((bytesio *)self, arg); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -332,7 +390,9 @@ _io_BytesIO_readinto(PyObject *self, PyObject *arg) _PyArg_BadArgument("readinto", "argument", "read-write bytes-like object", arg); goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_readinto_impl((bytesio *)self, &buffer); + Py_END_CRITICAL_SECTION(); exit: /* Cleanup for buffer */ @@ -356,13 +416,13 @@ PyDoc_STRVAR(_io_BytesIO_truncate__doc__, {"truncate", _PyCFunction_CAST(_io_BytesIO_truncate), METH_FASTCALL, _io_BytesIO_truncate__doc__}, static PyObject * -_io_BytesIO_truncate_impl(bytesio *self, Py_ssize_t size); +_io_BytesIO_truncate_impl(bytesio *self, PyObject *size); static PyObject * _io_BytesIO_truncate(PyObject *self, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_ssize_t size = ((bytesio *)self)->pos; + PyObject *size = Py_None; if (!_PyArg_CheckPositional("truncate", nargs, 0, 1)) { goto exit; @@ -370,11 +430,11 @@ _io_BytesIO_truncate(PyObject *self, PyObject *const *args, Py_ssize_t nargs) if (nargs < 1) { goto skip_optional; } - if (!_Py_convert_optional_to_ssize_t(args[0], &size)) { - goto exit; - } + size = args[0]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_truncate_impl((bytesio *)self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -428,7 +488,9 @@ _io_BytesIO_seek(PyObject *self, PyObject *const *args, Py_ssize_t nargs) goto exit; } skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_seek_impl((bytesio *)self, pos, whence); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -453,7 +515,9 @@ _io_BytesIO_write(PyObject *self, PyObject *b) { PyObject *return_value = NULL; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_write_impl((bytesio *)self, b); + Py_END_CRITICAL_SECTION(); return return_value; } @@ -465,8 +529,8 @@ PyDoc_STRVAR(_io_BytesIO_writelines__doc__, "Write lines to the file.\n" "\n" "Note that newlines are not added. lines can be any iterable object\n" -"producing bytes-like objects. This is equivalent to calling write() for\n" -"each element."); +"producing bytes-like objects. This is equivalent to calling write()\n" +"for each element."); #define _IO_BYTESIO_WRITELINES_METHODDEF \ {"writelines", (PyCFunction)_io_BytesIO_writelines, METH_O, _io_BytesIO_writelines__doc__}, @@ -479,7 +543,9 @@ _io_BytesIO_writelines(PyObject *self, PyObject *lines) { PyObject *return_value = NULL; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO_writelines_impl((bytesio *)self, lines); + Py_END_CRITICAL_SECTION(); return return_value; } @@ -499,7 +565,13 @@ _io_BytesIO_close_impl(bytesio *self); static PyObject * _io_BytesIO_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_BytesIO_close_impl((bytesio *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_BytesIO_close_impl((bytesio *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_BytesIO___init____doc__, @@ -558,9 +630,11 @@ _io_BytesIO___init__(PyObject *self, PyObject *args, PyObject *kwargs) } initvalue = fastargs[0]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_BytesIO___init___impl((bytesio *)self, initvalue); + Py_END_CRITICAL_SECTION(); exit: return return_value; } -/*[clinic end generated code: output=6dbfd82f4e9d4ef3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=eac3911e207aaf45 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/fileio.c.h b/Modules/_io/clinic/fileio.c.h index 04870b1c8903619..1cb5fca49e98d36 100644 --- a/Modules/_io/clinic/fileio.c.h +++ b/Modules/_io/clinic/fileio.c.h @@ -15,8 +15,8 @@ PyDoc_STRVAR(_io_FileIO_close__doc__, "\n" "Close the file.\n" "\n" -"A closed file cannot be used for further I/O operations. close() may be\n" -"called more than once without error."); +"A closed file cannot be used for further I/O operations. close()\n" +"may be called more than once without error."); #define _IO_FILEIO_CLOSE_METHODDEF \ {"close", _PyCFunction_CAST(_io_FileIO_close), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_close__doc__}, @@ -41,16 +41,19 @@ PyDoc_STRVAR(_io_FileIO___init____doc__, "Open a file.\n" "\n" "The mode can be \'r\' (default), \'w\', \'x\' or \'a\' for reading,\n" -"writing, exclusive creation or appending. The file will be created if it\n" -"doesn\'t exist when opened for writing or appending; it will be truncated\n" -"when opened for writing. A FileExistsError will be raised if it already\n" -"exists when opened for creating. Opening a file for creating implies\n" -"writing so this mode behaves in a similar way to \'w\'.Add a \'+\' to the mode\n" -"to allow simultaneous reading and writing. A custom opener can be used by\n" -"passing a callable as *opener*. The underlying file descriptor for the file\n" -"object is then obtained by calling opener with (*name*, *flags*).\n" -"*opener* must return an open file descriptor (passing os.open as *opener*\n" -"results in functionality similar to passing None)."); +"writing, exclusive creation or appending. The file will be created\n" +"if it doesn\'t exist when opened for writing or appending; it will be\n" +"truncated when opened for writing. A FileExistsError will be raised\n" +"if it already exists when opened for creating. Opening a file for\n" +"creating implies writing so this mode behaves in a similar way to\n" +"\'w\'. Add a \'+\' to the mode to allow simultaneous reading and\n" +"writing.\n" +"\n" +"A custom opener can be used by passing a callable as *opener*.\n" +"The underlying file descriptor for the file object is then obtained\n" +"by calling opener with (*name*, *flags*). *opener* must return\n" +"an open file descriptor (passing os.open as *opener* results in\n" +"functionality similar to passing None)."); static int _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, @@ -270,11 +273,13 @@ PyDoc_STRVAR(_io_FileIO_readall__doc__, "\n" "Read all data from the file, returned as bytes.\n" "\n" -"Reads until either there is an error or read() returns size 0 (indicates EOF).\n" -"If the file is already at EOF, returns an empty bytes object.\n" +"Reads until either there is an error or read() returns size 0\n" +"(indicates EOF). If the file is already at EOF, returns an empty\n" +"bytes object.\n" "\n" -"In non-blocking mode, returns as much data as could be read before EAGAIN. If no\n" -"data is available (EAGAIN is returned before bytes are read) returns None."); +"In non-blocking mode, returns as much data as could be read before\n" +"EAGAIN. If no data is available (EAGAIN is returned before bytes\n" +"are read) returns None."); #define _IO_FILEIO_READALL_METHODDEF \ {"readall", (PyCFunction)_io_FileIO_readall, METH_NOARGS, _io_FileIO_readall__doc__}, @@ -294,14 +299,14 @@ PyDoc_STRVAR(_io_FileIO_read__doc__, "\n" "Read at most size bytes, returned as bytes.\n" "\n" -"If size is less than 0, read all bytes in the file making multiple read calls.\n" -"See ``FileIO.readall``.\n" +"If size is less than 0, read all bytes in the file making multiple\n" +"read calls. See ``FileIO.readall``.\n" "\n" -"Attempts to make only one system call, retrying only per PEP 475 (EINTR). This\n" -"means less data may be returned than requested.\n" +"Attempts to make only one system call, retrying only per PEP 475\n" +"(EINTR). This means less data may be returned than requested.\n" "\n" -"In non-blocking mode, returns None if no data is available. Return an empty\n" -"bytes object at EOF."); +"In non-blocking mode, returns None if no data is available. Return\n" +"an empty bytes object at EOF."); #define _IO_FILEIO_READ_METHODDEF \ {"read", _PyCFunction_CAST(_io_FileIO_read), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_read__doc__}, @@ -354,8 +359,8 @@ PyDoc_STRVAR(_io_FileIO_write__doc__, "Write buffer b to file, return number of bytes written.\n" "\n" "Only makes one system call, so not all of the data may be written.\n" -"The number of bytes actually written is returned. In non-blocking mode,\n" -"returns None if the write would block."); +"The number of bytes actually written is returned. In non-blocking\n" +"mode, returns None if the write would block."); #define _IO_FILEIO_WRITE_METHODDEF \ {"write", _PyCFunction_CAST(_io_FileIO_write), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io_FileIO_write__doc__}, @@ -408,11 +413,12 @@ PyDoc_STRVAR(_io_FileIO_seek__doc__, "\n" "Move to new file position and return the file position.\n" "\n" -"Argument offset is a byte count. Optional argument whence defaults to\n" -"SEEK_SET or 0 (offset from start of file, offset should be >= 0); other values\n" -"are SEEK_CUR or 1 (move relative to current position, positive or negative),\n" -"and SEEK_END or 2 (move relative to end of file, usually negative, although\n" -"many platforms allow seeking beyond the end of a file).\n" +"Argument offset is a byte count. Optional argument whence defaults\n" +"to SEEK_SET or 0 (offset from start of file, offset should be >= 0);\n" +"other values are SEEK_CUR or 1 (move relative to current position,\n" +"positive or negative), and SEEK_END or 2 (move relative to end of\n" +"file, usually negative, although many platforms allow seeking beyond\n" +"the end of a file).\n" "\n" "Note that not all file objects are seekable."); @@ -543,4 +549,4 @@ _io_FileIO_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO_FILEIO_TRUNCATE_METHODDEF #define _IO_FILEIO_TRUNCATE_METHODDEF #endif /* !defined(_IO_FILEIO_TRUNCATE_METHODDEF) */ -/*[clinic end generated code: output=1902fac9e39358aa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3be887af000af7ce input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/iobase.c.h b/Modules/_io/clinic/iobase.c.h index 402448545dfc516..e4438c26431aa8e 100644 --- a/Modules/_io/clinic/iobase.c.h +++ b/Modules/_io/clinic/iobase.c.h @@ -19,11 +19,13 @@ PyDoc_STRVAR(_io__IOBase_seek__doc__, " whence\n" " The relative position to seek from.\n" "\n" -"The offset is interpreted relative to the position indicated by whence.\n" -"Values for whence are:\n" +"The offset is interpreted relative to the position indicated by\n" +"whence. Values for whence are:\n" "\n" -"* os.SEEK_SET or 0 -- start of stream (the default); offset should be zero or positive\n" -"* os.SEEK_CUR or 1 -- current stream position; offset may be negative\n" +"* os.SEEK_SET or 0 -- start of stream (the default); offset should\n" +" be zero or positive\n" +"* os.SEEK_CUR or 1 -- current stream position; offset may be\n" +" negative\n" "* os.SEEK_END or 2 -- end of stream; offset is usually negative\n" "\n" "Return the new absolute position."); @@ -103,8 +105,8 @@ PyDoc_STRVAR(_io__IOBase_truncate__doc__, "\n" "Truncate file to size bytes.\n" "\n" -"File pointer is left unchanged. Size defaults to the current IO position\n" -"as reported by tell(). Return the new size."); +"File pointer is left unchanged. Size defaults to the current IO\n" +"position as reported by tell(). Return the new size."); #define _IO__IOBASE_TRUNCATE_METHODDEF \ {"truncate", _PyCFunction_CAST(_io__IOBase_truncate), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__IOBase_truncate__doc__}, @@ -443,4 +445,4 @@ _io__RawIOBase_readall(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _io__RawIOBase_readall_impl(self); } -/*[clinic end generated code: output=9359e74d95534bef input=a9049054013a1b77]*/ +/*[clinic end generated code: output=28c06bb6db32c096 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/stringio.c.h b/Modules/_io/clinic/stringio.c.h index 8e8cd8df9ab8a15..684cde170cede0b 100644 --- a/Modules/_io/clinic/stringio.c.h +++ b/Modules/_io/clinic/stringio.c.h @@ -181,7 +181,8 @@ PyDoc_STRVAR(_io_StringIO_seek__doc__, "\n" "Change stream position.\n" "\n" -"Seek to character offset pos relative to position indicated by whence:\n" +"Seek to character offset pos relative to position indicated by\n" +"whence:\n" " 0 Start of stream (the default). pos should be >= 0;\n" " 1 Current position - pos must be 0;\n" " 2 End of stream - pos must be 0.\n" @@ -552,4 +553,4 @@ _io_StringIO_newlines_get(PyObject *self, void *Py_UNUSED(context)) return return_value; } -/*[clinic end generated code: output=5bfaaab7f41ee6b5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=101f798d9622b724 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/textio.c.h b/Modules/_io/clinic/textio.c.h index 128a5ad1678f26a..8d59bda5f74b386 100644 --- a/Modules/_io/clinic/textio.c.h +++ b/Modules/_io/clinic/textio.c.h @@ -16,7 +16,8 @@ PyDoc_STRVAR(_io__TextIOBase_detach__doc__, "\n" "Separate the underlying buffer from the TextIOBase and return it.\n" "\n" -"After the underlying buffer has been detached, the TextIO is in an unusable state."); +"After the underlying buffer has been detached, the TextIO is in\n" +"an unusable state."); #define _IO__TEXTIOBASE_DETACH_METHODDEF \ {"detach", _PyCFunction_CAST(_io__TextIOBase_detach), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__TextIOBase_detach__doc__}, @@ -40,8 +41,8 @@ PyDoc_STRVAR(_io__TextIOBase_read__doc__, "\n" "Read at most size characters from stream.\n" "\n" -"Read from underlying buffer until we have size characters or we hit EOF.\n" -"If size is negative or omitted, read until EOF."); +"Read from underlying buffer until we have size characters or we hit\n" +"EOF. If size is negative or omitted, read until EOF."); #define _IO__TEXTIOBASE_READ_METHODDEF \ {"read", _PyCFunction_CAST(_io__TextIOBase_read), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _io__TextIOBase_read__doc__}, @@ -430,7 +431,9 @@ _io_IncrementalNewlineDecoder_decode(PyObject *self, PyObject *const *args, Py_s goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_IncrementalNewlineDecoder_decode_impl((nldecoder_object *)self, input, final); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -450,7 +453,13 @@ _io_IncrementalNewlineDecoder_getstate_impl(nldecoder_object *self); static PyObject * _io_IncrementalNewlineDecoder_getstate(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_IncrementalNewlineDecoder_getstate_impl((nldecoder_object *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_IncrementalNewlineDecoder_getstate_impl((nldecoder_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_IncrementalNewlineDecoder_setstate__doc__, @@ -470,7 +479,9 @@ _io_IncrementalNewlineDecoder_setstate(PyObject *self, PyObject *state) { PyObject *return_value = NULL; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_IncrementalNewlineDecoder_setstate_impl((nldecoder_object *)self, state); + Py_END_CRITICAL_SECTION(); return return_value; } @@ -489,7 +500,13 @@ _io_IncrementalNewlineDecoder_reset_impl(nldecoder_object *self); static PyObject * _io_IncrementalNewlineDecoder_reset(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _io_IncrementalNewlineDecoder_reset_impl((nldecoder_object *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _io_IncrementalNewlineDecoder_reset_impl((nldecoder_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_io_TextIOWrapper___init____doc__, @@ -649,7 +666,9 @@ _io_TextIOWrapper___init__(PyObject *self, PyObject *args, PyObject *kwargs) goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _io_TextIOWrapper___init___impl((textio *)self, buffer, encoding, errors, newline, line_buffering, write_through); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -948,8 +967,8 @@ PyDoc_STRVAR(_io_TextIOWrapper_tell__doc__, "\n" "Return the stream position as an opaque number.\n" "\n" -"The return value of tell() can be given as input to seek(), to restore a\n" -"previous stream position."); +"The return value of tell() can be given as input to seek(), to\n" +"restore a previous stream position."); #define _IO_TEXTIOWRAPPER_TELL_METHODDEF \ {"tell", (PyCFunction)_io_TextIOWrapper_tell, METH_NOARGS, _io_TextIOWrapper_tell__doc__}, @@ -1312,4 +1331,4 @@ _io_TextIOWrapper__CHUNK_SIZE_set(PyObject *self, PyObject *value, void *Py_UNUS return return_value; } -/*[clinic end generated code: output=30404271a1151056 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8c571c9dba87d2b1 input=a9049054013a1b77]*/ diff --git a/Modules/_io/clinic/winconsoleio.c.h b/Modules/_io/clinic/winconsoleio.c.h index 7af5923b6c17473..bd8073cd0af3f64 100644 --- a/Modules/_io/clinic/winconsoleio.c.h +++ b/Modules/_io/clinic/winconsoleio.c.h @@ -46,9 +46,9 @@ PyDoc_STRVAR(_io__WindowsConsoleIO___init____doc__, "\n" "Open a console buffer by file descriptor.\n" "\n" -"The mode can be \'rb\' (default), or \'wb\' for reading or writing bytes. All\n" -"other mode characters will be ignored. Mode \'b\' will be assumed if it is\n" -"omitted. The *opener* parameter is always ignored."); +"The mode can be \'rb\' (default), or \'wb\' for reading or writing\n" +"bytes. All other mode characters will be ignored. Mode \'b\' will be\n" +"assumed if it is omitted. The *opener* parameter is always ignored."); static int _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, @@ -463,4 +463,4 @@ _io__WindowsConsoleIO_isatty(PyObject *self, PyObject *Py_UNUSED(ignored)) #ifndef _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #define _IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF #endif /* !defined(_IO__WINDOWSCONSOLEIO_ISATTY_METHODDEF) */ -/*[clinic end generated code: output=ce50bcd905f1f213 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=dfe49dd71f4f4b1d input=a9049054013a1b77]*/ diff --git a/Modules/_io/fileio.c b/Modules/_io/fileio.c index 8fcb27049d6c7c7..c26e21123b0feca 100644 --- a/Modules/_io/fileio.c +++ b/Modules/_io/fileio.c @@ -4,6 +4,7 @@ #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stdbool.h> // bool #ifdef HAVE_UNISTD_H @@ -151,13 +152,13 @@ _io.FileIO.close Close the file. -A closed file cannot be used for further I/O operations. close() may be -called more than once without error. +A closed file cannot be used for further I/O operations. close() +may be called more than once without error. [clinic start generated code]*/ static PyObject * _io_FileIO_close_impl(fileio *self, PyTypeObject *cls) -/*[clinic end generated code: output=c30cbe9d1f23ca58 input=70da49e63db7c64d]*/ +/*[clinic end generated code: output=c30cbe9d1f23ca58 input=b405751dc4163da3]*/ { PyObject *res; int rc; @@ -228,22 +229,25 @@ _io.FileIO.__init__ Open a file. The mode can be 'r' (default), 'w', 'x' or 'a' for reading, -writing, exclusive creation or appending. The file will be created if it -doesn't exist when opened for writing or appending; it will be truncated -when opened for writing. A FileExistsError will be raised if it already -exists when opened for creating. Opening a file for creating implies -writing so this mode behaves in a similar way to 'w'.Add a '+' to the mode -to allow simultaneous reading and writing. A custom opener can be used by -passing a callable as *opener*. The underlying file descriptor for the file -object is then obtained by calling opener with (*name*, *flags*). -*opener* must return an open file descriptor (passing os.open as *opener* -results in functionality similar to passing None). +writing, exclusive creation or appending. The file will be created +if it doesn't exist when opened for writing or appending; it will be +truncated when opened for writing. A FileExistsError will be raised +if it already exists when opened for creating. Opening a file for +creating implies writing so this mode behaves in a similar way to +'w'. Add a '+' to the mode to allow simultaneous reading and +writing. + +A custom opener can be used by passing a callable as *opener*. +The underlying file descriptor for the file object is then obtained +by calling opener with (*name*, *flags*). *opener* must return +an open file descriptor (passing os.open as *opener* results in +functionality similar to passing None). [clinic start generated code]*/ static int _io_FileIO___init___impl(fileio *self, PyObject *nameobj, const char *mode, int closefd, PyObject *opener) -/*[clinic end generated code: output=23413f68e6484bbd input=588aac967e0ba74b]*/ +/*[clinic end generated code: output=23413f68e6484bbd input=bac4efcd8f930bf3]*/ { #ifdef MS_WINDOWS wchar_t *widename = NULL; @@ -570,9 +574,7 @@ fileio_dealloc(PyObject *op) PyMem_Free(self->stat_atopen); self->stat_atopen = NULL; } - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, self->weakreflist); (void)fileio_clear(op); PyTypeObject *tp = Py_TYPE(op); @@ -727,16 +729,18 @@ _io.FileIO.readall Read all data from the file, returned as bytes. -Reads until either there is an error or read() returns size 0 (indicates EOF). -If the file is already at EOF, returns an empty bytes object. +Reads until either there is an error or read() returns size 0 +(indicates EOF). If the file is already at EOF, returns an empty +bytes object. -In non-blocking mode, returns as much data as could be read before EAGAIN. If no -data is available (EAGAIN is returned before bytes are read) returns None. +In non-blocking mode, returns as much data as could be read before +EAGAIN. If no data is available (EAGAIN is returned before bytes +are read) returns None. [clinic start generated code]*/ static PyObject * _io_FileIO_readall_impl(fileio *self) -/*[clinic end generated code: output=faa0292b213b4022 input=1e19849857f5d0a1]*/ +/*[clinic end generated code: output=faa0292b213b4022 input=393194434f380889]*/ { Py_off_t pos, end; PyObject *result; @@ -851,19 +855,19 @@ _io.FileIO.read Read at most size bytes, returned as bytes. -If size is less than 0, read all bytes in the file making multiple read calls. -See ``FileIO.readall``. +If size is less than 0, read all bytes in the file making multiple +read calls. See ``FileIO.readall``. -Attempts to make only one system call, retrying only per PEP 475 (EINTR). This -means less data may be returned than requested. +Attempts to make only one system call, retrying only per PEP 475 +(EINTR). This means less data may be returned than requested. -In non-blocking mode, returns None if no data is available. Return an empty -bytes object at EOF. +In non-blocking mode, returns None if no data is available. Return +an empty bytes object at EOF. [clinic start generated code]*/ static PyObject * _io_FileIO_read_impl(fileio *self, PyTypeObject *cls, Py_ssize_t size) -/*[clinic end generated code: output=bbd749c7c224143e input=cf21fddef7d38ab6]*/ +/*[clinic end generated code: output=bbd749c7c224143e input=c7baa3b440af9337]*/ { char *ptr; Py_ssize_t n; @@ -919,13 +923,13 @@ _io.FileIO.write Write buffer b to file, return number of bytes written. Only makes one system call, so not all of the data may be written. -The number of bytes actually written is returned. In non-blocking mode, -returns None if the write would block. +The number of bytes actually written is returned. In non-blocking +mode, returns None if the write would block. [clinic start generated code]*/ static PyObject * _io_FileIO_write_impl(fileio *self, PyTypeObject *cls, Py_buffer *b) -/*[clinic end generated code: output=927e25be80f3b77b input=2776314f043088f5]*/ +/*[clinic end generated code: output=927e25be80f3b77b input=233f1f70f9e8b09e]*/ { Py_ssize_t n; int err; @@ -1026,18 +1030,19 @@ _io.FileIO.seek Move to new file position and return the file position. -Argument offset is a byte count. Optional argument whence defaults to -SEEK_SET or 0 (offset from start of file, offset should be >= 0); other values -are SEEK_CUR or 1 (move relative to current position, positive or negative), -and SEEK_END or 2 (move relative to end of file, usually negative, although -many platforms allow seeking beyond the end of a file). +Argument offset is a byte count. Optional argument whence defaults +to SEEK_SET or 0 (offset from start of file, offset should be >= 0); +other values are SEEK_CUR or 1 (move relative to current position, +positive or negative), and SEEK_END or 2 (move relative to end of +file, usually negative, although many platforms allow seeking beyond +the end of a file). Note that not all file objects are seekable. [clinic start generated code]*/ static PyObject * _io_FileIO_seek_impl(fileio *self, PyObject *pos, int whence) -/*[clinic end generated code: output=c976acdf054e6655 input=0439194b0774d454]*/ +/*[clinic end generated code: output=c976acdf054e6655 input=f165a1b4f5d494ad]*/ { if (self->fd < 0) return err_closed(); diff --git a/Modules/_io/iobase.c b/Modules/_io/iobase.c index cd4c7e7cead2776..e291d7cfc4710f8 100644 --- a/Modules/_io/iobase.c +++ b/Modules/_io/iobase.c @@ -14,6 +14,7 @@ #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_object.h" // _PyType_HasFeature() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> // offsetof() #include "_iomodule.h" @@ -102,11 +103,13 @@ _io._IOBase.seek Change the stream position to the given byte offset. -The offset is interpreted relative to the position indicated by whence. -Values for whence are: +The offset is interpreted relative to the position indicated by +whence. Values for whence are: -* os.SEEK_SET or 0 -- start of stream (the default); offset should be zero or positive -* os.SEEK_CUR or 1 -- current stream position; offset may be negative +* os.SEEK_SET or 0 -- start of stream (the default); offset should + be zero or positive +* os.SEEK_CUR or 1 -- current stream position; offset may be + negative * os.SEEK_END or 2 -- end of stream; offset is usually negative Return the new absolute position. @@ -115,7 +118,7 @@ Return the new absolute position. static PyObject * _io__IOBase_seek_impl(PyObject *self, PyTypeObject *cls, int Py_UNUSED(offset), int Py_UNUSED(whence)) -/*[clinic end generated code: output=8bd74ea6538ded53 input=74211232b363363e]*/ +/*[clinic end generated code: output=8bd74ea6538ded53 input=22eaf07a7a0ee289]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return iobase_unsupported(state, "seek"); @@ -142,14 +145,14 @@ _io._IOBase.truncate Truncate file to size bytes. -File pointer is left unchanged. Size defaults to the current IO position -as reported by tell(). Return the new size. +File pointer is left unchanged. Size defaults to the current IO +position as reported by tell(). Return the new size. [clinic start generated code]*/ static PyObject * _io__IOBase_truncate_impl(PyObject *self, PyTypeObject *cls, PyObject *Py_UNUSED(size)) -/*[clinic end generated code: output=2013179bff1fe8ef input=660ac20936612c27]*/ +/*[clinic end generated code: output=2013179bff1fe8ef input=5b3b6ab3c7abd806]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return iobase_unsupported(state, "truncate"); @@ -383,8 +386,7 @@ iobase_dealloc(PyObject *op) } PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, self->weakreflist); Py_CLEAR(self->dict); tp->tp_free(self); Py_DECREF(tp); @@ -938,14 +940,21 @@ _io__RawIOBase_read_impl(PyObject *self, Py_ssize_t n) return res; } - n = PyNumber_AsSsize_t(res, PyExc_ValueError); + Py_ssize_t bytes_filled = PyNumber_AsSsize_t(res, PyExc_ValueError); Py_DECREF(res); - if (n == -1 && PyErr_Occurred()) { + if (bytes_filled == -1 && PyErr_Occurred()) { Py_DECREF(b); return NULL; } + if (bytes_filled < 0 || bytes_filled > n) { + Py_DECREF(b); + PyErr_Format(PyExc_ValueError, + "readinto returned %zd outside buffer size %zd", + bytes_filled, n); + return NULL; + } - res = PyBytes_FromStringAndSize(PyByteArray_AsString(b), n); + res = PyBytes_FromStringAndSize(PyByteArray_AsString(b), bytes_filled); Py_DECREF(b); return res; } diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 9d1bfa3ea05ceab..8f7801dab96dd1d 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -1,6 +1,7 @@ #include "Python.h" #include <stddef.h> // offsetof() #include "pycore_object.h" +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "_iomodule.h" /* Implementation note: the buffer is always at least one character longer @@ -224,7 +225,9 @@ write_str(stringio *self, PyObject *obj) if (self->state == STATE_ACCUMULATING) { if (self->string_size == self->pos) { - if (PyUnicodeWriter_WriteStr(self->writer, decoded)) + // gh-149046: Avoid PyUnicodeWriter_WriteStr() which calls str(obj) + // on str subclasses + if (_PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)self->writer, decoded)) goto fail; goto success; } @@ -486,7 +489,8 @@ _io.StringIO.seek Change stream position. -Seek to character offset pos relative to position indicated by whence: +Seek to character offset pos relative to position indicated by +whence: 0 Start of stream (the default). pos should be >= 0; 1 Current position - pos must be 0; 2 End of stream - pos must be 0. @@ -495,7 +499,7 @@ Returns the new absolute position. static PyObject * _io_StringIO_seek_impl(stringio *self, Py_ssize_t pos, int whence) -/*[clinic end generated code: output=e9e0ac9a8ae71c25 input=c75ced09343a00d7]*/ +/*[clinic end generated code: output=e9e0ac9a8ae71c25 input=ffef24668fd71a5d]*/ { CHECK_INITIALIZED(self); CHECK_CLOSED(self); @@ -628,9 +632,7 @@ stringio_dealloc(PyObject *op) } PyUnicodeWriter_Discard(self->writer); (void)stringio_clear(op); - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, self->weakreflist); tp->tp_free(self); Py_DECREF(tp); } diff --git a/Modules/_io/textio.c b/Modules/_io/textio.c index 86328e46a7b1311..74775e035e47a44 100644 --- a/Modules/_io/textio.c +++ b/Modules/_io/textio.c @@ -16,6 +16,7 @@ #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_unicodeobject.h" // _PyUnicode_AsASCIIString() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "_iomodule.h" @@ -58,12 +59,13 @@ _io._TextIOBase.detach Separate the underlying buffer from the TextIOBase and return it. -After the underlying buffer has been detached, the TextIO is in an unusable state. +After the underlying buffer has been detached, the TextIO is in +an unusable state. [clinic start generated code]*/ static PyObject * _io__TextIOBase_detach_impl(PyObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=50915f40c609eaa4 input=987ca3640d0a3776]*/ +/*[clinic end generated code: output=50915f40c609eaa4 input=8099c088abcb87d8]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return _unsupported(state, "detach"); @@ -77,14 +79,14 @@ _io._TextIOBase.read Read at most size characters from stream. -Read from underlying buffer until we have size characters or we hit EOF. -If size is negative or omitted, read until EOF. +Read from underlying buffer until we have size characters or we hit +EOF. If size is negative or omitted, read until EOF. [clinic start generated code]*/ static PyObject * _io__TextIOBase_read_impl(PyObject *self, PyTypeObject *cls, int Py_UNUSED(size)) -/*[clinic end generated code: output=51a5178a309ce647 input=f5e37720f9fc563f]*/ +/*[clinic end generated code: output=51a5178a309ce647 input=c9fd4cc1cf1b4614]*/ { _PyIO_State *state = get_io_state_by_cls(cls); return _unsupported(state, "read"); @@ -517,6 +519,7 @@ _PyIncrementalNewlineDecoder_decode(PyObject *myself, } /*[clinic input] +@critical_section _io.IncrementalNewlineDecoder.decode input: object final: bool = False @@ -525,18 +528,19 @@ _io.IncrementalNewlineDecoder.decode static PyObject * _io_IncrementalNewlineDecoder_decode_impl(nldecoder_object *self, PyObject *input, int final) -/*[clinic end generated code: output=0d486755bb37a66e input=90e223c70322c5cd]*/ +/*[clinic end generated code: output=0d486755bb37a66e input=9475d16a73168504]*/ { return _PyIncrementalNewlineDecoder_decode((PyObject *) self, input, final); } /*[clinic input] +@critical_section _io.IncrementalNewlineDecoder.getstate [clinic start generated code]*/ static PyObject * _io_IncrementalNewlineDecoder_getstate_impl(nldecoder_object *self) -/*[clinic end generated code: output=f0d2c9c136f4e0d0 input=f8ff101825e32e7f]*/ +/*[clinic end generated code: output=f0d2c9c136f4e0d0 input=dc3e1f27aa850f12]*/ { PyObject *buffer; unsigned long long flag; @@ -574,6 +578,7 @@ _io_IncrementalNewlineDecoder_getstate_impl(nldecoder_object *self) } /*[clinic input] +@critical_section _io.IncrementalNewlineDecoder.setstate state: object / @@ -582,7 +587,7 @@ _io.IncrementalNewlineDecoder.setstate static PyObject * _io_IncrementalNewlineDecoder_setstate_impl(nldecoder_object *self, PyObject *state) -/*[clinic end generated code: output=09135cb6e78a1dc8 input=c53fb505a76dbbe2]*/ +/*[clinic end generated code: output=09135cb6e78a1dc8 input=275fd3982d2b08cb]*/ { PyObject *buffer; unsigned long long flag; @@ -612,12 +617,13 @@ _io_IncrementalNewlineDecoder_setstate_impl(nldecoder_object *self, } /*[clinic input] +@critical_section _io.IncrementalNewlineDecoder.reset [clinic start generated code]*/ static PyObject * _io_IncrementalNewlineDecoder_reset_impl(nldecoder_object *self) -/*[clinic end generated code: output=32fa40c7462aa8ff input=728678ddaea776df]*/ +/*[clinic end generated code: output=32fa40c7462aa8ff input=31bd8ae4e36cec83]*/ { CHECK_INITIALIZED_DECODER(self); @@ -1055,6 +1061,7 @@ io_check_errors(PyObject *errors) /*[clinic input] +@critical_section _io.TextIOWrapper.__init__ buffer: object encoding: str(accept={str, NoneType}) = None @@ -1098,7 +1105,7 @@ _io_TextIOWrapper___init___impl(textio *self, PyObject *buffer, const char *encoding, PyObject *errors, const char *newline, int line_buffering, int write_through) -/*[clinic end generated code: output=72267c0c01032ed2 input=e6cfaaaf6059d4f5]*/ +/*[clinic end generated code: output=72267c0c01032ed2 input=0f077220214c40a4]*/ { PyObject *raw, *codec_info = NULL; PyObject *res; @@ -1469,8 +1476,7 @@ textiowrapper_dealloc(PyObject *op) return; self->ok = 0; _PyObject_GC_UNTRACK(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, self->weakreflist); (void)textiowrapper_clear(op); tp->tp_free(self); Py_DECREF(tp); @@ -1578,6 +1584,8 @@ _io_TextIOWrapper_detach_impl(textio *self) static int _textiowrapper_writeflush(textio *self) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + if (self->pending_bytes == NULL) return 0; @@ -1747,32 +1755,38 @@ _io_TextIOWrapper_write_impl(textio *self, PyObject *text) } } - if (self->pending_bytes == NULL) { - assert(self->pending_bytes_count == 0); - self->pending_bytes = b; - } - else if (!PyList_CheckExact(self->pending_bytes)) { - PyObject *list = PyList_New(2); - if (list == NULL) { + if (bytes_len > 0) { + if (self->pending_bytes == NULL) { + assert(self->pending_bytes_count == 0); + self->pending_bytes = b; + } + else if (!PyList_CheckExact(self->pending_bytes)) { + PyObject *list = PyList_New(2); + if (list == NULL) { + Py_DECREF(b); + return NULL; + } + // Since Python 3.12, allocating GC object won't trigger GC and release + // GIL. See https://github.com/python/cpython/issues/97922 + assert(!PyList_CheckExact(self->pending_bytes)); + PyList_SET_ITEM(list, 0, self->pending_bytes); + PyList_SET_ITEM(list, 1, b); + self->pending_bytes = list; + } + else { + if (PyList_Append(self->pending_bytes, b) < 0) { + Py_DECREF(b); + return NULL; + } Py_DECREF(b); - return NULL; } - // Since Python 3.12, allocating GC object won't trigger GC and release - // GIL. See https://github.com/python/cpython/issues/97922 - assert(!PyList_CheckExact(self->pending_bytes)); - PyList_SET_ITEM(list, 0, self->pending_bytes); - PyList_SET_ITEM(list, 1, b); - self->pending_bytes = list; + + self->pending_bytes_count += bytes_len; } else { - if (PyList_Append(self->pending_bytes, b) < 0) { - Py_DECREF(b); - return NULL; - } Py_DECREF(b); } - self->pending_bytes_count += bytes_len; if (self->pending_bytes_count >= self->chunk_size || needflush || text_needflush) { if (_textiowrapper_writeflush(self) < 0) @@ -2720,13 +2734,13 @@ _io.TextIOWrapper.tell Return the stream position as an opaque number. -The return value of tell() can be given as input to seek(), to restore a -previous stream position. +The return value of tell() can be given as input to seek(), to +restore a previous stream position. [clinic start generated code]*/ static PyObject * _io_TextIOWrapper_tell_impl(textio *self) -/*[clinic end generated code: output=4f168c08bf34ad5f input=415d6b4e4f8e6e8c]*/ +/*[clinic end generated code: output=4f168c08bf34ad5f input=aeece020f747fd92]*/ { PyObject *res; PyObject *posobj = NULL; @@ -2842,7 +2856,7 @@ _io_TextIOWrapper_tell_impl(textio *self) current pos */ skip_bytes = (Py_ssize_t) (self->b2cratio * chars_to_skip); skip_back = 1; - assert(skip_back <= PyBytes_GET_SIZE(next_input)); + assert(skip_bytes <= PyBytes_GET_SIZE(next_input)); input = PyBytes_AS_STRING(next_input); while (skip_bytes > 0) { /* Decode up to temptative start point */ @@ -3147,6 +3161,9 @@ _io_TextIOWrapper_close_impl(textio *self) if (r > 0) { Py_RETURN_NONE; /* stream already closed */ } + if (self->detached) { + Py_RETURN_NONE; /* gh-142594 null pointer issue */ + } else { PyObject *exc = NULL; if (self->finalizing) { @@ -3173,8 +3190,9 @@ _io_TextIOWrapper_close_impl(textio *self) } static PyObject * -textiowrapper_iternext(PyObject *op) +textiowrapper_iternext_lock_held(PyObject *op) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); PyObject *line; textio *self = textio_CAST(op); @@ -3210,6 +3228,16 @@ textiowrapper_iternext(PyObject *op) return line; } +static PyObject * +textiowrapper_iternext(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = textiowrapper_iternext_lock_held(op); + Py_END_CRITICAL_SECTION(); + return result; +} + /*[clinic input] @critical_section @getter diff --git a/Modules/_io/winconsoleio.c b/Modules/_io/winconsoleio.c index 3e783b9da456523..598ec9e61789c9a 100644 --- a/Modules/_io/winconsoleio.c +++ b/Modules/_io/winconsoleio.c @@ -10,6 +10,7 @@ #include "pycore_fileutils.h" // _Py_BEGIN_SUPPRESS_IPH #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #ifdef HAVE_WINDOWS_CONSOLE_IO @@ -314,16 +315,16 @@ _io._WindowsConsoleIO.__init__ Open a console buffer by file descriptor. -The mode can be 'rb' (default), or 'wb' for reading or writing bytes. All -other mode characters will be ignored. Mode 'b' will be assumed if it is -omitted. The *opener* parameter is always ignored. +The mode can be 'rb' (default), or 'wb' for reading or writing +bytes. All other mode characters will be ignored. Mode 'b' will be +assumed if it is omitted. The *opener* parameter is always ignored. [clinic start generated code]*/ static int _io__WindowsConsoleIO___init___impl(winconsoleio *self, PyObject *nameobj, const char *mode, int closefd, PyObject *opener) -/*[clinic end generated code: output=3fd9cbcdd8d95429 input=7a3eed6bbe998fd9]*/ +/*[clinic end generated code: output=3fd9cbcdd8d95429 input=f31100e2cd724617]*/ { const char *s; wchar_t *name = NULL; @@ -518,8 +519,7 @@ winconsoleio_dealloc(PyObject *op) if (_PyIOBase_finalize(op) < 0) return; _PyObject_GC_UNTRACK(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, self->weakreflist); Py_CLEAR(self->dict); tp->tp_free(self); Py_DECREF(tp); @@ -673,12 +673,13 @@ read_console_w(HANDLE handle, DWORD maxlen, DWORD *readlen) { maxlen += 1; Py_BLOCK_THREADS newbuf = (wchar_t*)PyMem_Realloc(buf, maxlen * sizeof(wchar_t)); - Py_UNBLOCK_THREADS if (!newbuf) { sig = -1; PyErr_NoMemory(); + Py_UNBLOCK_THREADS break; } + Py_UNBLOCK_THREADS buf = newbuf; /* Only advance by n and not BUFSIZ in this case */ off += n; diff --git a/Modules/_json.c b/Modules/_json.c index 89b0a41dd10acb5..250dd67d2a9687c 100644 --- a/Modules/_json.c +++ b/Modules/_json.c @@ -316,11 +316,12 @@ raise_errmsg(const char *msg, PyObject *s, Py_ssize_t end) PyObject *exc; exc = PyObject_CallFunction(JSONDecodeError, "zOn", msg, s, end); - Py_DECREF(JSONDecodeError); if (exc) { PyErr_SetObject(JSONDecodeError, exc); Py_DECREF(exc); } + + Py_DECREF(JSONDecodeError); } static void @@ -473,7 +474,7 @@ scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict, Py_ssize_t *next c = 0; next++; end = next + 4; - if (end >= len) { + if (end > len) { raise_errmsg("Invalid \\uXXXX escape", pystr, next - 1); goto bail; } @@ -1228,16 +1229,23 @@ encoder_new(PyTypeObject *type, PyObject *args, PyObject *kwds) static char *kwlist[] = {"markers", "default", "encoder", "indent", "key_separator", "item_separator", "sort_keys", "skipkeys", "allow_nan", NULL}; PyEncoderObject *s; - PyObject *markers = Py_None, *defaultfn, *encoder, *indent, *key_separator; + PyObject *markers, *defaultfn, *encoder, *indent, *key_separator; PyObject *item_separator; int sort_keys, skipkeys, allow_nan; - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!?OOOUUppp:make_encoder", kwlist, - &PyDict_Type, &markers, &defaultfn, &encoder, &indent, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "OOOOUUppp:make_encoder", kwlist, + &markers, &defaultfn, &encoder, &indent, &key_separator, &item_separator, &sort_keys, &skipkeys, &allow_nan)) return NULL; + if (markers != Py_None && !PyDict_Check(markers)) { + PyErr_Format(PyExc_TypeError, + "make_encoder() argument 1 must be dict or None, " + "not %.200s", Py_TYPE(markers)->tp_name); + return NULL; + } + s = (PyEncoderObject *)type->tp_alloc(type, 0); if (s == NULL) return NULL; @@ -1375,6 +1383,7 @@ encoder_call(PyObject *op, PyObject *args, PyObject *kwds) return NULL; } } + indent_level = 0; if (encoder_listencode_obj(self, writer, obj, indent_level, indent_cache)) { PyUnicodeWriter_Discard(writer); Py_XDECREF(indent_cache); @@ -1476,13 +1485,13 @@ encoder_listencode_obj(PyEncoderObject *s, PyUnicodeWriter *writer, int rv; if (obj == Py_None) { - return PyUnicodeWriter_WriteUTF8(writer, "null", 4); + return PyUnicodeWriter_WriteASCII(writer, "null", 4); } else if (obj == Py_True) { - return PyUnicodeWriter_WriteUTF8(writer, "true", 4); + return PyUnicodeWriter_WriteASCII(writer, "true", 4); } else if (obj == Py_False) { - return PyUnicodeWriter_WriteUTF8(writer, "false", 5); + return PyUnicodeWriter_WriteASCII(writer, "false", 5); } else if (PyUnicode_Check(obj)) { PyObject *encoded = encoder_encode_string(s, obj); @@ -1609,6 +1618,12 @@ encoder_encode_key_value(PyEncoderObject *s, PyUnicodeWriter *writer, bool *firs if (*first) { *first = false; + if (s->indent != Py_None) { + if (write_newline_indent(writer, indent_level, indent_cache) < 0) { + Py_DECREF(keystr); + return -1; + } + } } else { if (PyUnicodeWriter_WriteStr(writer, item_separator) < 0) { @@ -1649,7 +1664,7 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, if (PyDict_GET_SIZE(dct) == 0) { /* Fast path */ - return PyUnicodeWriter_WriteUTF8(writer, "{}", 2); + return PyUnicodeWriter_WriteASCII(writer, "{}", 2); } if (s->markers != Py_None) { @@ -1676,11 +1691,8 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, if (s->indent != Py_None) { indent_level++; separator = get_item_separator(s, indent_level, indent_cache); - if (separator == NULL || - write_newline_indent(writer, indent_level, indent_cache) < 0) - { + if (separator == NULL) goto bail; - } } if (s->sort_keys || !PyDict_CheckExact(dct)) { @@ -1690,9 +1702,13 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, for (Py_ssize_t i = 0; i < PyList_GET_SIZE(items); i++) { PyObject *item = PyList_GET_ITEM(items, i); + // gh-142831: encoder_encode_key_value() can invoke user code + // that mutates the items list, invalidating this borrowed ref. + Py_INCREF(item); if (!PyTuple_Check(item) || PyTuple_GET_SIZE(item) != 2) { PyErr_SetString(PyExc_ValueError, "items must return 2-tuples"); + Py_DECREF(item); goto bail; } @@ -1700,18 +1716,30 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, value = PyTuple_GET_ITEM(item, 1); if (encoder_encode_key_value(s, writer, &first, dct, key, value, indent_level, indent_cache, - separator) < 0) + separator) < 0) { + Py_DECREF(item); goto bail; + } + Py_DECREF(item); } Py_CLEAR(items); } else { Py_ssize_t pos = 0; while (PyDict_Next(dct, &pos, &key, &value)) { + // gh-142831: encoder_encode_key_value() can invoke user code + // that mutates the dict, invalidating these borrowed refs. + Py_INCREF(key); + Py_INCREF(value); if (encoder_encode_key_value(s, writer, &first, dct, key, value, indent_level, indent_cache, - separator) < 0) + separator) < 0) { + Py_DECREF(key); + Py_DECREF(value); goto bail; + } + Py_DECREF(key); + Py_DECREF(value); } } @@ -1720,7 +1748,7 @@ encoder_listencode_dict(PyEncoderObject *s, PyUnicodeWriter *writer, goto bail; Py_CLEAR(ident); } - if (s->indent != Py_None) { + if (s->indent != Py_None && !first) { indent_level--; if (write_newline_indent(writer, indent_level, indent_cache) < 0) { goto bail; @@ -1753,7 +1781,7 @@ encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer, return -1; if (PySequence_Fast_GET_SIZE(s_fast) == 0) { Py_DECREF(s_fast); - return PyUnicodeWriter_WriteUTF8(writer, "[]", 2); + return PyUnicodeWriter_WriteASCII(writer, "[]", 2); } if (s->markers != Py_None) { @@ -1788,14 +1816,21 @@ encoder_listencode_list(PyEncoderObject *s, PyUnicodeWriter *writer, } for (i = 0; i < PySequence_Fast_GET_SIZE(s_fast); i++) { PyObject *obj = PySequence_Fast_GET_ITEM(s_fast, i); + // gh-142831: encoder_listencode_obj() can invoke user code + // that mutates the sequence, invalidating this borrowed ref. + Py_INCREF(obj); if (i) { - if (PyUnicodeWriter_WriteStr(writer, separator) < 0) + if (PyUnicodeWriter_WriteStr(writer, separator) < 0) { + Py_DECREF(obj); goto bail; + } } if (encoder_listencode_obj(s, writer, obj, indent_level, indent_cache)) { _PyErr_FormatNote("when serializing %T item %zd", seq, i); + Py_DECREF(obj); goto bail; } + Py_DECREF(obj); } if (ident != NULL) { if (PyDict_DelItem(s->markers, ident)) diff --git a/Modules/_localemodule.c b/Modules/_localemodule.c index ad618398d5b824f..b1d9e74db623cb8 100644 --- a/Modules/_localemodule.c +++ b/Modules/_localemodule.c @@ -87,6 +87,41 @@ copy_grouping(const char* s) return result; } +#if defined(MS_WINDOWS) + +// 16 is the number of elements in the szCodePage field +// of the __crt_locale_strings structure. +#define MAX_CP_LEN 15 + +static int +check_locale_name(const char *locale, const char *end) +{ + size_t len = end ? (size_t)(end - locale) : strlen(locale); + const char *dot = memchr(locale, '.', len); + if (dot && locale + len - dot - 1 > MAX_CP_LEN) { + return -1; + } + return 0; +} + +static int +check_locale_name_all(const char *locale) +{ + const char *start = locale; + while (1) { + const char *end = strchr(start, ';'); + if (check_locale_name(start, end) < 0) { + return -1; + } + if (end == NULL) { + break; + } + start = end + 1; + } + return 0; +} +#endif + /*[clinic input] _locale.setlocale @@ -111,6 +146,18 @@ _locale_setlocale_impl(PyObject *module, int category, const char *locale) "invalid locale category"); return NULL; } + if (locale) { + if ((category == LC_ALL + ? check_locale_name_all(locale) + : check_locale_name(locale, NULL)) < 0) + { + /* Debug assertion failure on Windows. + * _Py_BEGIN_SUPPRESS_IPH/_Py_END_SUPPRESS_IPH do not help. */ + PyErr_SetString(get_locale_state(module)->Error, + "unsupported locale setting"); + return NULL; + } + } #endif if (locale) { @@ -410,7 +457,9 @@ _locale_strxfrm_impl(PyObject *module, PyObject *str) /* assume no change in size, first */ n1 = n1 + 1; - buf = PyMem_New(wchar_t, n1); + /* Yet another +1 is needed to work around a platform bug in wcsxfrm() + * on macOS. See gh-130567. */ + buf = PyMem_New(wchar_t, n1+1); if (!buf) { PyErr_NoMemory(); goto exit; @@ -436,7 +485,54 @@ _locale_strxfrm_impl(PyObject *module, PyObject *str) goto exit; } } - result = PyUnicode_FromWideChar(buf, n2); + /* The result is just a sequence of integers, they are not necessary + Unicode code points, so PyUnicode_FromWideChar() cannot be used + here. For example, 0xD83D 0xDC0D should not be larger than 0xFF41. + */ +#if SIZEOF_WCHAR_T == 4 + { + /* Some codes can exceed the range of Unicode code points + (0 - 0x10FFFF), so they cannot be directly used in + PyUnicode_FromKindAndData(). They should be first encoded in + a way that preserves the lexicographical order. + + Codes in the range 0-0xFFFF represent themself. + Codes larger than 0xFFFF are encoded as a pair: + * 0x1xxxx -- the highest 16 bits + * 0x0xxxx -- the lowest 16 bits + */ + size_t n3 = 0; + for (size_t i = 0; i < n2; i++) { + if ((Py_UCS4)buf[i] > 0x10000u) { + n3++; + } + } + if (n3) { + n3 += n2; // no integer overflow + Py_UCS4 *buf2 = PyMem_New(Py_UCS4, n3); + if (buf2 == NULL) { + PyErr_NoMemory(); + goto exit; + } + size_t j = 0; + for (size_t i = 0; i < n2; i++) { + Py_UCS4 c = (Py_UCS4)buf[i]; + if (c > 0x10000u) { + buf2[j++] = (c >> 16) | 0x10000u; + buf2[j++] = c & 0xFFFFu; + } + else { + buf2[j++] = c; + } + } + assert(j == n3); + result = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf2, n3); + PyMem_Free(buf2); + goto exit; + } + } +#endif + result = PyUnicode_FromKindAndData(sizeof(wchar_t), buf, n2); exit: PyMem_Free(buf); PyMem_Free(s); @@ -483,7 +579,6 @@ _locale__getdefaultlocale_impl(PyObject *module) } /* cannot determine the language code (very unlikely) */ - Py_INCREF(Py_None); return Py_BuildValue("Os", Py_None, encoding); } #endif @@ -692,7 +787,17 @@ _locale_nl_langinfo_impl(PyObject *module, int item) result = result != NULL ? result : ""; char *oldloc = NULL; if (langinfo_constants[i].category != LC_CTYPE - && !is_all_ascii(result) + && *result && ( +#ifdef __GLIBC__ + // gh-133740: Always change the locale for ALT_DIGITS and ERA +# ifdef ALT_DIGITS + item == ALT_DIGITS || +# endif +# ifdef ERA + item == ERA || +# endif +#endif + !is_all_ascii(result)) && change_locale(langinfo_constants[i].category, &oldloc) < 0) { return NULL; diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 626c176715bdacd..412909cd9bf39df 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -534,6 +534,7 @@ static int statsForEntry(rotating_node_t *node, void *arg) } /*[clinic input] +@critical_section _lsprof.Profiler.getstats cls: defining_class @@ -565,7 +566,7 @@ profiler_subentry objects: static PyObject * _lsprof_Profiler_getstats_impl(ProfilerObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=1806ef720019ee03 input=445e193ef4522902]*/ +/*[clinic end generated code: output=1806ef720019ee03 input=3dc69eb85ed73d91]*/ { statscollector_t collect; collect.state = _PyType_GetModuleState(cls); @@ -613,6 +614,7 @@ setBuiltins(ProfilerObject *pObj, int nvalue) } /*[clinic input] +@critical_section _lsprof.Profiler._pystart_callback code: object @@ -624,7 +626,7 @@ _lsprof.Profiler._pystart_callback static PyObject * _lsprof_Profiler__pystart_callback_impl(ProfilerObject *self, PyObject *code, PyObject *instruction_offset) -/*[clinic end generated code: output=5fec8b7ad5ed25e8 input=b166e6953c579cda]*/ +/*[clinic end generated code: output=5fec8b7ad5ed25e8 input=b61a0e79cf1f8499]*/ { ptrace_enter_call((PyObject*)self, (void *)code, code); @@ -632,6 +634,29 @@ _lsprof_Profiler__pystart_callback_impl(ProfilerObject *self, PyObject *code, } /*[clinic input] +@critical_section +_lsprof.Profiler._pythrow_callback + + code: object + instruction_offset: object + exception: object + / + +[clinic start generated code]*/ + +static PyObject * +_lsprof_Profiler__pythrow_callback_impl(ProfilerObject *self, PyObject *code, + PyObject *instruction_offset, + PyObject *exception) +/*[clinic end generated code: output=0a32988919dfb94c input=60c7f272206d3758]*/ +{ + ptrace_enter_call((PyObject*)self, (void *)code, code); + + Py_RETURN_NONE; +} + +/*[clinic input] +@critical_section _lsprof.Profiler._pyreturn_callback code: object @@ -646,7 +671,7 @@ _lsprof_Profiler__pyreturn_callback_impl(ProfilerObject *self, PyObject *code, PyObject *instruction_offset, PyObject *retval) -/*[clinic end generated code: output=9e2f6fc1b882c51e input=667ffaeb2fa6fd1f]*/ +/*[clinic end generated code: output=9e2f6fc1b882c51e input=0ddcc1ec53faa928]*/ { ptrace_leave_call((PyObject*)self, (void *)code); @@ -682,6 +707,7 @@ PyObject* get_cfunc_from_callable(PyObject* callable, PyObject* self_arg, PyObje } /*[clinic input] +@critical_section _lsprof.Profiler._ccall_callback code: object @@ -696,7 +722,7 @@ static PyObject * _lsprof_Profiler__ccall_callback_impl(ProfilerObject *self, PyObject *code, PyObject *instruction_offset, PyObject *callable, PyObject *self_arg) -/*[clinic end generated code: output=152db83cabd18cad input=0e66687cfb95c001]*/ +/*[clinic end generated code: output=152db83cabd18cad input=2fc1e0630ee5e32b]*/ { if (self->flags & POF_BUILTINS) { PyObject* cfunc = get_cfunc_from_callable(callable, self_arg, self->missing); @@ -712,6 +738,7 @@ _lsprof_Profiler__ccall_callback_impl(ProfilerObject *self, PyObject *code, } /*[clinic input] +@critical_section _lsprof.Profiler._creturn_callback code: object @@ -727,7 +754,7 @@ _lsprof_Profiler__creturn_callback_impl(ProfilerObject *self, PyObject *code, PyObject *instruction_offset, PyObject *callable, PyObject *self_arg) -/*[clinic end generated code: output=1e886dde8fed8fb0 input=b18afe023746923a]*/ +/*[clinic end generated code: output=1e886dde8fed8fb0 input=bdc246d6b5b8714a]*/ { if (self->flags & POF_BUILTINS) { PyObject* cfunc = get_cfunc_from_callable(callable, self_arg, self->missing); @@ -747,7 +774,7 @@ static const struct { } callback_table[] = { {PY_MONITORING_EVENT_PY_START, "_pystart_callback"}, {PY_MONITORING_EVENT_PY_RESUME, "_pystart_callback"}, - {PY_MONITORING_EVENT_PY_THROW, "_pystart_callback"}, + {PY_MONITORING_EVENT_PY_THROW, "_pythrow_callback"}, {PY_MONITORING_EVENT_PY_RETURN, "_pyreturn_callback"}, {PY_MONITORING_EVENT_PY_YIELD, "_pyreturn_callback"}, {PY_MONITORING_EVENT_PY_UNWIND, "_pyreturn_callback"}, @@ -759,6 +786,7 @@ static const struct { /*[clinic input] +@critical_section _lsprof.Profiler.enable subcalls: bool = True @@ -775,7 +803,7 @@ Start collecting profiling information. static PyObject * _lsprof_Profiler_enable_impl(ProfilerObject *self, int subcalls, int builtins) -/*[clinic end generated code: output=1e747f9dc1edd571 input=9ab81405107ab7f1]*/ +/*[clinic end generated code: output=1e747f9dc1edd571 input=0b88115b1c796173]*/ { int all_events = 0; if (setSubcalls(self, subcalls) < 0 || setBuiltins(self, builtins) < 0) { @@ -791,7 +819,6 @@ _lsprof_Profiler_enable_impl(ProfilerObject *self, int subcalls, "use_tool_id", "is", self->tool_id, "cProfile"); if (check == NULL) { - PyErr_Format(PyExc_ValueError, "Another profiling tool is already active"); goto error; } Py_DECREF(check); @@ -848,6 +875,7 @@ flush_unmatched(ProfilerObject *pObj) /*[clinic input] +@critical_section _lsprof.Profiler.disable Stop collecting profiling information. @@ -855,7 +883,7 @@ Stop collecting profiling information. static PyObject * _lsprof_Profiler_disable_impl(ProfilerObject *self) -/*[clinic end generated code: output=838cffef7f651870 input=05700b3fc68d1f50]*/ +/*[clinic end generated code: output=838cffef7f651870 input=f7e4787cae20f7f6]*/ { if (self->flags & POF_EXT_TIMER) { PyErr_SetString(PyExc_RuntimeError, @@ -907,6 +935,7 @@ _lsprof_Profiler_disable_impl(ProfilerObject *self) } /*[clinic input] +@critical_section _lsprof.Profiler.clear Clear all profiling information collected so far. @@ -914,7 +943,7 @@ Clear all profiling information collected so far. static PyObject * _lsprof_Profiler_clear_impl(ProfilerObject *self) -/*[clinic end generated code: output=dd1c668fb84b1335 input=fbe1f88c28be4f98]*/ +/*[clinic end generated code: output=dd1c668fb84b1335 input=4aab219d5d7a9bec]*/ { if (self->flags & POF_EXT_TIMER) { PyErr_SetString(PyExc_RuntimeError, @@ -1002,6 +1031,7 @@ static PyMethodDef profiler_methods[] = { _LSPROF_PROFILER_DISABLE_METHODDEF _LSPROF_PROFILER_CLEAR_METHODDEF _LSPROF_PROFILER__PYSTART_CALLBACK_METHODDEF + _LSPROF_PROFILER__PYTHROW_CALLBACK_METHODDEF _LSPROF_PROFILER__PYRETURN_CALLBACK_METHODDEF _LSPROF_PROFILER__CCALL_CALLBACK_METHODDEF _LSPROF_PROFILER__CRETURN_CALLBACK_METHODDEF diff --git a/Modules/_lzmamodule.c b/Modules/_lzmamodule.c index f9b4c2a170e9c5b..bad524803d2d7bf 100644 --- a/Modules/_lzmamodule.c +++ b/Modules/_lzmamodule.c @@ -17,6 +17,7 @@ #include <lzma.h> +#include "pycore_long.h" // _PyLong_UInt32_Converter() // Blocks output buffer wrappers #include "pycore_blocks_output_buffer.h" @@ -223,8 +224,6 @@ FUNCNAME(PyObject *obj, void *ptr) \ return 1; \ } -INT_TYPE_CONVERTER_FUNC(uint32_t, uint32_converter) -INT_TYPE_CONVERTER_FUNC(uint64_t, uint64_converter) INT_TYPE_CONVERTER_FUNC(lzma_vli, lzma_vli_converter) INT_TYPE_CONVERTER_FUNC(lzma_mode, lzma_mode_converter) INT_TYPE_CONVERTER_FUNC(lzma_match_finder, lzma_mf_converter) @@ -254,7 +253,7 @@ parse_filter_spec_lzma(_lzma_state *state, PyObject *spec) return NULL; } if (preset_obj != NULL) { - int ok = uint32_converter(preset_obj, &preset); + int ok = _PyLong_UInt32_Converter(preset_obj, &preset); Py_DECREF(preset_obj); if (!ok) { return NULL; @@ -275,14 +274,14 @@ parse_filter_spec_lzma(_lzma_state *state, PyObject *spec) if (!PyArg_ParseTupleAndKeywords(state->empty_tuple, spec, "|OOO&O&O&O&O&O&O&O&", optnames, &id, &preset_obj, - uint32_converter, &options->dict_size, - uint32_converter, &options->lc, - uint32_converter, &options->lp, - uint32_converter, &options->pb, + _PyLong_UInt32_Converter, &options->dict_size, + _PyLong_UInt32_Converter, &options->lc, + _PyLong_UInt32_Converter, &options->lp, + _PyLong_UInt32_Converter, &options->pb, lzma_mode_converter, &options->mode, - uint32_converter, &options->nice_len, + _PyLong_UInt32_Converter, &options->nice_len, lzma_mf_converter, &options->mf, - uint32_converter, &options->depth)) { + _PyLong_UInt32_Converter, &options->depth)) { PyErr_SetString(PyExc_ValueError, "Invalid filter specifier for LZMA filter"); PyMem_Free(options); @@ -301,7 +300,7 @@ parse_filter_spec_delta(_lzma_state *state, PyObject *spec) lzma_options_delta *options; if (!PyArg_ParseTupleAndKeywords(state->empty_tuple, spec, "|OO&", optnames, - &id, uint32_converter, &dist)) { + &id, _PyLong_UInt32_Converter, &dist)) { PyErr_SetString(PyExc_ValueError, "Invalid filter specifier for delta filter"); return NULL; @@ -325,7 +324,7 @@ parse_filter_spec_bcj(_lzma_state *state, PyObject *spec) lzma_options_bcj *options; if (!PyArg_ParseTupleAndKeywords(state->empty_tuple, spec, "|OO&", optnames, - &id, uint32_converter, &start_offset)) { + &id, _PyLong_UInt32_Converter, &start_offset)) { PyErr_SetString(PyExc_ValueError, "Invalid filter specifier for BCJ filter"); return NULL; @@ -806,7 +805,7 @@ Compressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) return NULL; } - if (preset_obj != Py_None && !uint32_converter(preset_obj, &preset)) { + if (preset_obj != Py_None && !_PyLong_UInt32_Converter(preset_obj, &preset)) { return NULL; } @@ -1121,6 +1120,7 @@ decompress(Decompressor *d, uint8_t *data, size_t len, Py_ssize_t max_length) return result; error: + lzs->next_in = NULL; Py_XDECREF(result); return NULL; } @@ -1133,24 +1133,25 @@ _lzma.LZMADecompressor.decompress Decompress *data*, returning uncompressed data as bytes. -If *max_length* is nonnegative, returns at most *max_length* bytes of -decompressed data. If this limit is reached and further output can be -produced, *self.needs_input* will be set to ``False``. In this case, the next -call to *decompress()* may provide *data* as b'' to obtain more of the output. +If *max_length* is nonnegative, returns at most *max_length* bytes +of decompressed data. If this limit is reached and further output +can be produced, *self.needs_input* will be set to ``False``. In +this case, the next call to *decompress()* may provide *data* as b'' +to obtain more of the output. -If all of the input data was decompressed and returned (either because this -was less than *max_length* bytes, or because *max_length* was negative), -*self.needs_input* will be set to True. +If all of the input data was decompressed and returned (either +because this was less than *max_length* bytes, or because +*max_length* was negative), *self.needs_input* will be set to True. -Attempting to decompress data after the end of stream is reached raises an -EOFError. Any data found after the end of the stream is ignored and saved in -the unused_data attribute. +Attempting to decompress data after the end of stream is reached +raises an EOFError. Any data found after the end of the stream is +ignored and saved in the unused_data attribute. [clinic start generated code]*/ static PyObject * _lzma_LZMADecompressor_decompress_impl(Decompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=ef4e20ec7122241d input=60c1f135820e309d]*/ +/*[clinic end generated code: output=ef4e20ec7122241d input=0eb62669c4315dee]*/ { PyObject *result = NULL; @@ -1226,7 +1227,7 @@ _lzma_LZMADecompressor_impl(PyTypeObject *type, int format, "Cannot specify memory limit with FORMAT_RAW"); return NULL; } - if (!uint64_converter(memlimit, &memlimit_)) { + if (!_PyLong_UInt64_Converter(memlimit, &memlimit_)) { return NULL; } } diff --git a/Modules/_multiprocessing/clinic/posixshmem.c.h b/Modules/_multiprocessing/clinic/posixshmem.c.h index a545ff4d80f067d..a4d7273aea718ae 100644 --- a/Modules/_multiprocessing/clinic/posixshmem.c.h +++ b/Modules/_multiprocessing/clinic/posixshmem.c.h @@ -50,9 +50,9 @@ PyDoc_STRVAR(_posixshmem_shm_unlink__doc__, "\n" "Remove a shared memory object (similar to unlink()).\n" "\n" -"Remove a shared memory object name, and, once all processes have unmapped\n" -"the object, de-allocates and destroys the contents of the associated memory\n" -"region."); +"Remove a shared memory object name, and, once all processes have\n" +"unmapped the object, de-allocates and destroys the contents of the\n" +"associated memory region."); #define _POSIXSHMEM_SHM_UNLINK_METHODDEF \ {"shm_unlink", (PyCFunction)_posixshmem_shm_unlink, METH_O, _posixshmem_shm_unlink__doc__}, @@ -86,4 +86,4 @@ _posixshmem_shm_unlink(PyObject *module, PyObject *arg) #ifndef _POSIXSHMEM_SHM_UNLINK_METHODDEF #define _POSIXSHMEM_SHM_UNLINK_METHODDEF #endif /* !defined(_POSIXSHMEM_SHM_UNLINK_METHODDEF) */ -/*[clinic end generated code: output=74588a5abba6e36c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e69afacce7b0595e input=a9049054013a1b77]*/ diff --git a/Modules/_multiprocessing/posixshmem.c b/Modules/_multiprocessing/posixshmem.c index ab45e4136c7d46e..22b4af212662b3c 100644 --- a/Modules/_multiprocessing/posixshmem.c +++ b/Modules/_multiprocessing/posixshmem.c @@ -81,15 +81,15 @@ _posixshmem.shm_unlink Remove a shared memory object (similar to unlink()). -Remove a shared memory object name, and, once all processes have unmapped -the object, de-allocates and destroys the contents of the associated memory -region. +Remove a shared memory object name, and, once all processes have +unmapped the object, de-allocates and destroys the contents of the +associated memory region. [clinic start generated code]*/ static PyObject * _posixshmem_shm_unlink_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=42f8b23d134b9ff5 input=298369d013dcad63]*/ +/*[clinic end generated code: output=42f8b23d134b9ff5 input=cf7a30ec6503cf78]*/ { int rv; int async_err = 0; diff --git a/Modules/_opcode.c b/Modules/_opcode.c index c295f7b3152577f..bf783cda0f7e303 100644 --- a/Modules/_opcode.c +++ b/Modules/_opcode.c @@ -5,7 +5,7 @@ #include "Python.h" #include "compile.h" #include "opcode.h" -#include "pycore_ceval.h" +#include "pycore_ceval.h" // SPECIAL_MAX #include "pycore_code.h" #include "pycore_compile.h" #include "pycore_intrinsics.h" @@ -119,7 +119,6 @@ _opcode_has_const_impl(PyObject *module, int opcode) } /*[clinic input] - _opcode.has_name -> bool opcode: int @@ -129,7 +128,7 @@ Return True if the opcode accesses an attribute by name, False otherwise. static int _opcode_has_name_impl(PyObject *module, int opcode) -/*[clinic end generated code: output=b49a83555c2fa517 input=448aa5e4bcc947ba]*/ +/*[clinic end generated code: output=b49a83555c2fa517 input=3032469628dd0849]*/ { return IS_VALID_OPCODE(opcode) && OPCODE_HAS_NAME(opcode); } diff --git a/Modules/_pickle.c b/Modules/_pickle.c index d260f1a68f8c704..eca966abd253903 100644 --- a/Modules/_pickle.c +++ b/Modules/_pickle.c @@ -645,6 +645,7 @@ typedef struct PicklerObject { int fast_nesting; int fix_imports; /* Indicate whether Pickler should fix the name of globals for Python 2.x. */ + int running; /* True when a method of Pickler is executing. */ PyObject *fast_memo; PyObject *buffer_callback; /* Callback for out-of-band buffers, or NULL */ } PicklerObject; @@ -688,6 +689,8 @@ typedef struct UnpicklerObject { int proto; /* Protocol of the pickle loaded. */ int fix_imports; /* Indicate whether Unpickler should fix the name of globals pickled by Python 2.x. */ + int running; /* True when a method of Unpickler is executing. */ + } UnpicklerObject; typedef struct { @@ -705,6 +708,32 @@ typedef struct { #define PicklerMemoProxyObject_CAST(op) ((PicklerMemoProxyObject *)(op)) #define UnpicklerMemoProxyObject_CAST(op) ((UnpicklerMemoProxyObject *)(op)) +#define BEGIN_USING_PICKLER(SELF, RET) do { \ + if ((SELF)->running) { \ + PyErr_SetString(PyExc_RuntimeError, \ + "Pickler object is already used"); \ + return (RET); \ + } \ + (SELF)->running = 1; \ + } while (0) + +#define END_USING_PICKLER(SELF) do { \ + (SELF)->running = 0; \ + } while (0) + +#define BEGIN_USING_UNPICKLER(SELF, RET) do { \ + if ((SELF)->running) { \ + PyErr_SetString(PyExc_RuntimeError, \ + "Unpickler object is already used"); \ + return (RET); \ + } \ + (SELF)->running = 1; \ + } while (0) + +#define END_USING_UNPICKLER(SELF) do { \ + (SELF)->running = 0; \ + } while (0) + /* Forward declarations */ static int save(PickleState *state, PicklerObject *, PyObject *, int); static int save_reduce(PickleState *, PicklerObject *, PyObject *, PyObject *); @@ -1134,6 +1163,7 @@ _Pickler_New(PickleState *st) self->fast = 0; self->fast_nesting = 0; self->fix_imports = 0; + self->running = 0; self->fast_memo = NULL; self->buffer_callback = NULL; @@ -1637,6 +1667,7 @@ _Unpickler_New(PyObject *module) self->marks_size = 0; self->proto = 0; self->fix_imports = 0; + self->running = 0; PyObject_GC_Track(self); return self; @@ -1916,22 +1947,34 @@ whichmodule(PickleState *st, PyObject *global, PyObject *global_name, PyObject * return NULL; } if (PyDict_CheckExact(modules)) { + PyObject *found_name = NULL; + int error = 0; i = 0; + Py_BEGIN_CRITICAL_SECTION(modules); while (PyDict_Next(modules, &i, &module_name, &module)) { Py_INCREF(module_name); Py_INCREF(module); if (_checkmodule(module_name, module, global, dotted_path) == 0) { Py_DECREF(module); - Py_DECREF(modules); - return module_name; + found_name = module_name; + break; } Py_DECREF(module); Py_DECREF(module_name); if (PyErr_Occurred()) { - Py_DECREF(modules); - return NULL; + error = 1; + break; } } + Py_END_CRITICAL_SECTION(); + if (error) { + Py_DECREF(modules); + return NULL; + } + if (found_name != NULL) { + Py_DECREF(modules); + return found_name; + } } else { PyObject *iterator = PyObject_GetIter(modules); @@ -2561,53 +2604,61 @@ save_picklebuffer(PickleState *st, PicklerObject *self, PyObject *obj) "PickleBuffer can only be pickled with protocol >= 5"); return -1; } - const Py_buffer* view = PyPickleBuffer_GetBuffer(obj); - if (view == NULL) { + Py_buffer view; + if (PyObject_GetBuffer(obj, &view, PyBUF_FULL_RO) != 0) { return -1; } - if (view->suboffsets != NULL || !PyBuffer_IsContiguous(view, 'A')) { + if (view.suboffsets != NULL || !PyBuffer_IsContiguous(&view, 'A')) { PyErr_SetString(st->PicklingError, "PickleBuffer can not be pickled when " "pointing to a non-contiguous buffer"); - return -1; + goto error; } + + int rc = 0; int in_band = 1; if (self->buffer_callback != NULL) { PyObject *ret = PyObject_CallOneArg(self->buffer_callback, obj); if (ret == NULL) { - return -1; + goto error; } in_band = PyObject_IsTrue(ret); Py_DECREF(ret); if (in_band == -1) { - return -1; + goto error; } } if (in_band) { /* Write data in-band */ - if (view->readonly) { - return _save_bytes_data(st, self, obj, (const char *)view->buf, - view->len); + if (view.readonly) { + rc = _save_bytes_data(st, self, obj, (const char *)view.buf, + view.len); } else { - return _save_bytearray_data(st, self, obj, (const char *)view->buf, - view->len); + rc = _save_bytearray_data(st, self, obj, (const char *)view.buf, + view.len); } } else { /* Write data out-of-band */ const char next_buffer_op = NEXT_BUFFER; if (_Pickler_Write(self, &next_buffer_op, 1) < 0) { - return -1; + goto error; } - if (view->readonly) { + if (view.readonly) { const char readonly_buffer_op = READONLY_BUFFER; if (_Pickler_Write(self, &readonly_buffer_op, 1) < 0) { - return -1; + goto error; } } } - return 0; + + PyBuffer_Release(&view); + return rc; + +error: + PyBuffer_Release(&view); + return -1; } /* A copy of PyUnicode_AsRawUnicodeEscapeString() that also translates @@ -3041,7 +3092,7 @@ static int batch_list_exact(PickleState *state, PicklerObject *self, PyObject *obj) { PyObject *item = NULL; - Py_ssize_t this_batch, total; + Py_ssize_t this_batch, total, list_size; const char append_op = APPEND; const char appends_op = APPENDS; @@ -3051,9 +3102,14 @@ batch_list_exact(PickleState *state, PicklerObject *self, PyObject *obj) assert(self->proto > 0); assert(PyList_CheckExact(obj)); - if (PyList_GET_SIZE(obj) == 1) { - item = PyList_GET_ITEM(obj, 0); - Py_INCREF(item); + list_size = PyList_GET_SIZE(obj); + + if (list_size == 1) { + item = PyList_GetItemRef(obj, 0); + if (item == NULL) { + _PyErr_FormatNote("when serializing %T item 0", obj); + return -1; + } int err = save(state, self, item, 0); Py_DECREF(item); if (err < 0) { @@ -3072,8 +3128,11 @@ batch_list_exact(PickleState *state, PicklerObject *self, PyObject *obj) if (_Pickler_Write(self, &mark_op, 1) < 0) return -1; while (total < PyList_GET_SIZE(obj)) { - item = PyList_GET_ITEM(obj, total); - Py_INCREF(item); + item = PyList_GetItemRef(obj, total); + if (item == NULL) { + _PyErr_FormatNote("when serializing %T item %zd", obj, total); + return -1; + } int err = save(state, self, item, 0); Py_DECREF(item); if (err < 0) { @@ -3086,8 +3145,14 @@ batch_list_exact(PickleState *state, PicklerObject *self, PyObject *obj) } if (_Pickler_Write(self, &appends_op, 1) < 0) return -1; + if (PyList_GET_SIZE(obj) != list_size) { + PyErr_Format( + PyExc_RuntimeError, + "list changed size during iteration"); + return -1; + } - } while (total < PyList_GET_SIZE(obj)); + } while (total < list_size); return 0; } @@ -3314,7 +3379,7 @@ batch_dict(PickleState *state, PicklerObject *self, PyObject *iter, PyObject *or * Note that this currently doesn't work for protocol 0. */ static int -batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) +batch_dict_exact_impl(PickleState *state, PicklerObject *self, PyObject *obj) { PyObject *key = NULL, *value = NULL; int i; @@ -3385,6 +3450,18 @@ batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) return -1; } +/* gh-146452: Wrap the dict iteration in a critical section to prevent + concurrent mutation from invalidating PyDict_Next() iteration state. */ +static int +batch_dict_exact(PickleState *state, PicklerObject *self, PyObject *obj) +{ + int ret; + Py_BEGIN_CRITICAL_SECTION(obj); + ret = batch_dict_exact_impl(state, self, obj); + Py_END_CRITICAL_SECTION(); + return ret; +} + static int save_dict(PickleState *state, PicklerObject *self, PyObject *obj) { @@ -3530,16 +3607,13 @@ save_set(PickleState *state, PicklerObject *self, PyObject *obj) } static int -save_frozenset(PickleState *state, PicklerObject *self, PyObject *obj) +save_frozenset_impl(PickleState *state, PicklerObject *self, PyObject *obj) { PyObject *iter; const char mark_op = MARK; const char frozenset_op = FROZENSET; - if (self->fast && !fast_save_enter(self, obj)) - return -1; - if (self->proto < 4) { PyObject *items; PyObject *reduce_value; @@ -3610,6 +3684,19 @@ save_frozenset(PickleState *state, PicklerObject *self, PyObject *obj) return 0; } +static int +save_frozenset(PickleState *state, PicklerObject *self, PyObject *obj) +{ + if (self->fast && !fast_save_enter(self, obj)) { + return -1; + } + int status = save_frozenset_impl(state, self, obj); + if (self->fast && !fast_save_leave(self, obj)) { + return -1; + } + return status; +} + static int fix_imports(PickleState *st, PyObject **module_name, PyObject **global_name) { @@ -4685,17 +4772,23 @@ _pickle_Pickler_dump_impl(PicklerObject *self, PyTypeObject *cls, Py_TYPE(self)->tp_name); return NULL; } + BEGIN_USING_PICKLER(self, NULL); - if (_Pickler_ClearBuffer(self) < 0) - return NULL; - - if (dump(st, self, obj) < 0) - return NULL; - - if (_Pickler_FlushToFile(self) < 0) - return NULL; - + if (_Pickler_ClearBuffer(self) < 0) { + goto error; + } + if (dump(st, self, obj) < 0) { + goto error; + } + if (_Pickler_FlushToFile(self) < 0) { + goto error; + } + END_USING_PICKLER(self); Py_RETURN_NONE; + +error: + END_USING_PICKLER(self); + return NULL; } /*[clinic input] @@ -4836,47 +4929,54 @@ _pickle_Pickler___init___impl(PicklerObject *self, PyObject *file, PyObject *buffer_callback) /*[clinic end generated code: output=0abedc50590d259b input=cddc50f66b770002]*/ { + BEGIN_USING_PICKLER(self, -1); /* In case of multiple __init__() calls, clear previous content. */ if (self->write != NULL) (void)Pickler_clear((PyObject *)self); - if (_Pickler_SetProtocol(self, protocol, fix_imports) < 0) - return -1; - - if (_Pickler_SetOutputStream(self, file) < 0) - return -1; - - if (_Pickler_SetBufferCallback(self, buffer_callback) < 0) - return -1; - + if (_Pickler_SetProtocol(self, protocol, fix_imports) < 0) { + goto error; + } + if (_Pickler_SetOutputStream(self, file) < 0) { + goto error; + } + if (_Pickler_SetBufferCallback(self, buffer_callback) < 0) { + goto error; + } /* memo and output_buffer may have already been created in _Pickler_New */ if (self->memo == NULL) { self->memo = PyMemoTable_New(); - if (self->memo == NULL) - return -1; + if (self->memo == NULL) { + goto error; + } } self->output_len = 0; if (self->output_buffer == NULL) { self->max_output_len = WRITE_BUF_SIZE; self->output_buffer = PyBytes_FromStringAndSize(NULL, self->max_output_len); - if (self->output_buffer == NULL) - return -1; + if (self->output_buffer == NULL) { + goto error; + } } self->fast = 0; self->fast_nesting = 0; self->fast_memo = NULL; - if (self->dispatch_table != NULL) { - return 0; - } - if (PyObject_GetOptionalAttr((PyObject *)self, &_Py_ID(dispatch_table), - &self->dispatch_table) < 0) { - return -1; + if (self->dispatch_table == NULL) { + if (PyObject_GetOptionalAttr((PyObject *)self, &_Py_ID(dispatch_table), + &self->dispatch_table) < 0) { + goto error; + } } + END_USING_PICKLER(self); return 0; + +error: + END_USING_PICKLER(self); + return -1; } @@ -5539,17 +5639,16 @@ static int load_counted_binstring(PickleState *st, UnpicklerObject *self, int nbytes) { PyObject *obj; - Py_ssize_t size; + long size; char *s; if (_Unpickler_Read(self, st, &s, nbytes) < 0) return -1; - size = calc_binsize(s, nbytes); + size = calc_binint(s, nbytes); if (size < 0) { - PyErr_Format(st->UnpicklingError, - "BINSTRING exceeds system's maximum size of %zd bytes", - PY_SSIZE_T_MAX); + PyErr_SetString(st->UnpicklingError, + "BINSTRING pickle has negative byte count"); return -1; } @@ -7066,22 +7165,22 @@ static PyObject * _pickle_Unpickler_load_impl(UnpicklerObject *self, PyTypeObject *cls) /*[clinic end generated code: output=cc88168f608e3007 input=f5d2f87e61d5f07f]*/ { - UnpicklerObject *unpickler = (UnpicklerObject*)self; - PickleState *st = _Pickle_GetStateByClass(cls); /* Check whether the Unpickler was initialized correctly. This prevents segfaulting if a subclass overridden __init__ with a function that does not call Unpickler.__init__(). Here, we simply ensure that self->read is not NULL. */ - if (unpickler->read == NULL) { + if (self->read == NULL) { PyErr_Format(st->UnpicklingError, "Unpickler.__init__() was not called by %s.__init__()", - Py_TYPE(unpickler)->tp_name); + Py_TYPE(self)->tp_name); return NULL; } - - return load(st, unpickler); + BEGIN_USING_UNPICKLER(self, NULL); + PyObject *res = load(st, self); + END_USING_UNPICKLER(self); + return res; } /* The name of find_class() is misleading. In newer pickle protocols, this @@ -7343,35 +7442,41 @@ _pickle_Unpickler___init___impl(UnpicklerObject *self, PyObject *file, const char *errors, PyObject *buffers) /*[clinic end generated code: output=09f0192649ea3f85 input=ca4c1faea9553121]*/ { + BEGIN_USING_UNPICKLER(self, -1); /* In case of multiple __init__() calls, clear previous content. */ if (self->read != NULL) (void)Unpickler_clear((PyObject *)self); - if (_Unpickler_SetInputStream(self, file) < 0) - return -1; - - if (_Unpickler_SetInputEncoding(self, encoding, errors) < 0) - return -1; - - if (_Unpickler_SetBuffers(self, buffers) < 0) - return -1; - + if (_Unpickler_SetInputStream(self, file) < 0) { + goto error; + } + if (_Unpickler_SetInputEncoding(self, encoding, errors) < 0) { + goto error; + } + if (_Unpickler_SetBuffers(self, buffers) < 0) { + goto error; + } self->fix_imports = fix_imports; PyTypeObject *tp = Py_TYPE(self); PickleState *state = _Pickle_FindStateByType(tp); self->stack = (Pdata *)Pdata_New(state); - if (self->stack == NULL) - return -1; - + if (self->stack == NULL) { + goto error; + } self->memo_size = 32; self->memo = _Unpickler_NewMemo(self->memo_size); - if (self->memo == NULL) - return -1; - + if (self->memo == NULL) { + goto error; + } self->proto = 0; + END_USING_UNPICKLER(self); return 0; + +error: + END_USING_UNPICKLER(self); + return -1; } diff --git a/Modules/_posixsubprocess.c b/Modules/_posixsubprocess.c index b542f86b6fe8da2..d25ceca5fcf3ca6 100644 --- a/Modules/_posixsubprocess.c +++ b/Modules/_posixsubprocess.c @@ -514,7 +514,13 @@ _close_open_fds_maybe_unsafe(int start_fd, int *fds_to_keep, proc_fd_dir = NULL; else #endif +#if defined(_AIX) + char fd_path[PATH_MAX]; + snprintf(fd_path, sizeof(fd_path), "/proc/%ld/fd", (long)getpid()); + proc_fd_dir = opendir(fd_path); +#else proc_fd_dir = opendir(FD_DIR); +#endif if (!proc_fd_dir) { /* No way to get a list of open fds. */ _close_range_except(start_fd, -1, fds_to_keep, fds_to_keep_len, @@ -630,7 +636,7 @@ reset_signal_handlers(const sigset_t *child_sigmask) * (v)fork to set things up and call exec(). * * All of the code in this function must only use async-signal-safe functions, - * listed at `man 7 signal` or + * listed at `man 7 signal-safety` or * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html. * * This restriction is documented at @@ -983,15 +989,15 @@ _posixsubprocess.fork_exec as subprocess_fork_exec Spawn a fresh new child process. -Fork a child process, close parent file descriptors as appropriate in the -child and duplicate the few that are needed before calling exec() in the -child process. +Fork a child process, close parent file descriptors as appropriate in +the child and duplicate the few that are needed before calling exec() in +the child process. -If close_fds is True, close file descriptors 3 and higher, except those listed -in the sorted tuple pass_fds. +If close_fds is True, close file descriptors 3 and higher, except those +listed in the sorted tuple pass_fds. -The preexec_fn, if supplied, will be called immediately before closing file -descriptors and exec. +The preexec_fn, if supplied, will be called immediately before closing +file descriptors and exec. WARNING: preexec_fn is NOT SAFE if your application uses threads. It may trigger infrequent, difficult to debug deadlocks. @@ -1016,7 +1022,7 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, PyObject *extra_groups_packed, PyObject *uid_object, int child_umask, PyObject *preexec_fn) -/*[clinic end generated code: output=288464dc56e373c7 input=f311c3bcb5dd55c8]*/ +/*[clinic end generated code: output=288464dc56e373c7 input=5e56eac3e036e349]*/ { PyObject *converted_args = NULL, *fast_args = NULL; PyObject *preexec_fn_args_tuple = NULL; @@ -1084,8 +1090,14 @@ subprocess_fork_exec_impl(PyObject *module, PyObject *process_args, goto cleanup; } borrowed_arg = PySequence_Fast_GET_ITEM(fast_args, arg_num); - if (PyUnicode_FSConverter(borrowed_arg, &converted_arg) == 0) + /* borrowed_arg is only borrowed; its __fspath__() may run Python + that drops fast_args' last reference to it. */ + Py_INCREF(borrowed_arg); + if (PyUnicode_FSConverter(borrowed_arg, &converted_arg) == 0) { + Py_DECREF(borrowed_arg); goto cleanup; + } + Py_DECREF(borrowed_arg); PyTuple_SET_ITEM(converted_args, arg_num, converted_arg); } diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index 3ee14b61b821d66..790909aa4bf5882 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -7,6 +7,7 @@ #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_parking_lot.h" #include "pycore_time.h" // _PyTime_FromSecondsObject() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stdbool.h> #include <stddef.h> // offsetof() @@ -164,6 +165,7 @@ RingBuf_Put(RingBuf *buf, PyObject *item) // Buffer is full, grow it. if (resize_ringbuf(buf, buf->items_cap * 2) < 0) { PyErr_NoMemory(); + Py_DECREF(item); return -1; } } @@ -221,9 +223,7 @@ simplequeue_dealloc(PyObject *op) PyObject_GC_UnTrack(self); (void)simplequeue_clear(op); - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, self->weakreflist); tp->tp_free(self); Py_DECREF(tp); } @@ -297,15 +297,16 @@ _queue.SimpleQueue.put Put the item on the queue. -The optional 'block' and 'timeout' arguments are ignored, as this method -never blocks. They are provided for compatibility with the Queue class. +The optional 'block' and 'timeout' arguments are ignored, as this +method never blocks. They are provided for compatibility with the +Queue class. [clinic start generated code]*/ static PyObject * _queue_SimpleQueue_put_impl(simplequeueobject *self, PyObject *item, int block, PyObject *timeout) -/*[clinic end generated code: output=4333136e88f90d8b input=a16dbb33363c0fa8]*/ +/*[clinic end generated code: output=4333136e88f90d8b input=9f9ff270a74670c3]*/ { HandoffData data = { .handed_off = 0, @@ -365,10 +366,11 @@ _queue.SimpleQueue.get Remove and return an item from the queue. -If optional args 'block' is true and 'timeout' is None (the default), -block if necessary until an item is available. If 'timeout' is -a non-negative number, it blocks at most 'timeout' seconds and raises -the Empty exception if no item was available within that time. +If optional args 'block' is true and 'timeout' is None (the +default), block if necessary until an item is available. If +'timeout' is a non-negative number, it blocks at most 'timeout' +seconds and raises the Empty exception if no item was available +within that time. Otherwise ('block' is false), return an item if one is immediately available, else raise the Empty exception ('timeout' is ignored in that case). @@ -378,7 +380,7 @@ in that case). static PyObject * _queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, int block, PyObject *timeout_obj) -/*[clinic end generated code: output=5c2cca914cd1e55b input=f7836c65e5839c51]*/ +/*[clinic end generated code: output=5c2cca914cd1e55b input=afa0889bbc6b4761]*/ { PyTime_t endtime = 0; @@ -536,7 +538,7 @@ static PyMethodDef simplequeue_methods[] = { _QUEUE_SIMPLEQUEUE_PUT_NOWAIT_METHODDEF _QUEUE_SIMPLEQUEUE_QSIZE_METHODDEF {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("SimpleQueues are generic over the type of their contents")}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c index d5bac2f5b78120d..3ce6f5b54766899 100644 --- a/Modules/_randommodule.c +++ b/Modules/_randommodule.c @@ -123,9 +123,9 @@ typedef struct { /*[clinic input] module _random -class _random.Random "RandomObject *" "_randomstate_type(type)->Random_Type" +class _random.Random "RandomObject *" "(PyTypeObject *)_randomstate_type(Py_TYPE(self))->Random_Type" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=70a2c99619474983]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=f04bcbfba61a322e]*/ /* Random methods */ @@ -497,34 +497,32 @@ _random_Random_setstate_impl(RandomObject *self, PyObject *state) _random.Random.getrandbits self: self(type="RandomObject *") - k: int + k: uint64 / getrandbits(k) -> x. Generates an int with k random bits. [clinic start generated code]*/ static PyObject * -_random_Random_getrandbits_impl(RandomObject *self, int k) -/*[clinic end generated code: output=b402f82a2158887f input=87603cd60f79f730]*/ +_random_Random_getrandbits_impl(RandomObject *self, uint64_t k) +/*[clinic end generated code: output=c30ef8435f3433cf input=64226ac13bb4d2a3]*/ { - int i, words; + Py_ssize_t i, words; uint32_t r; uint32_t *wordarray; PyObject *result; - if (k < 0) { - PyErr_SetString(PyExc_ValueError, - "number of bits must be non-negative"); - return NULL; - } - if (k == 0) return PyLong_FromLong(0); if (k <= 32) /* Fast path */ return PyLong_FromUnsignedLong(genrand_uint32(self) >> (32 - k)); - words = (k - 1) / 32 + 1; + if ((k - 1u) / 32u + 1u > PY_SSIZE_T_MAX / 4u) { + PyErr_NoMemory(); + return NULL; + } + words = (Py_ssize_t)((k - 1u) / 32u + 1u); wordarray = (uint32_t *)PyMem_Malloc(words * 4); if (wordarray == NULL) { PyErr_NoMemory(); @@ -551,27 +549,20 @@ _random_Random_getrandbits_impl(RandomObject *self, int k) return result; } -static int -random_init(PyObject *self, PyObject *args, PyObject *kwds) -{ - PyObject *arg = NULL; - _randomstate *state = _randomstate_type(Py_TYPE(self)); - - if ((Py_IS_TYPE(self, (PyTypeObject *)state->Random_Type) || - Py_TYPE(self)->tp_init == ((PyTypeObject*)state->Random_Type)->tp_init) && - !_PyArg_NoKeywords("Random", kwds)) { - return -1; - } - - if (PyTuple_GET_SIZE(args) > 1) { - PyErr_SetString(PyExc_TypeError, "Random() requires 0 or 1 argument"); - return -1; - } +/*[clinic input] +@critical_section +@text_signature "($self, [seed])" +_random.Random.__init__ as random_init - if (PyTuple_GET_SIZE(args) == 1) - arg = PyTuple_GET_ITEM(args, 0); + seed: object = NULL + / +[clinic start generated code]*/ - return random_seed(RandomObject_CAST(self), arg); +static int +random_init_impl(RandomObject *self, PyObject *seed) +/*[clinic end generated code: output=260734a3739c394f input=e516bf32e8a05e28]*/ +{ + return random_seed(self, seed); } diff --git a/Modules/_remote_debugging_module.c b/Modules/_remote_debugging_module.c index 9314ddd9bed5d7f..0b6b9d7f301a988 100644 --- a/Modules/_remote_debugging_module.c +++ b/Modules/_remote_debugging_module.c @@ -1,12 +1,15 @@ + /****************************************************************************** + * Python Remote Debugging Module + * + * This module provides functionality to debug Python processes remotely by + * reading their memory and reconstructing stack traces and asyncio task states. + ******************************************************************************/ + #define _GNU_SOURCE -#include <errno.h> -#include <fcntl.h> -#include <stddef.h> -#include <stdint.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> +/* ============================================================================ + * HEADERS AND INCLUDES + * ============================================================================ */ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 @@ -17,12 +20,74 @@ #include <internal/pycore_interpframe.h> // FRAME_OWNED_BY_CSTACK #include <internal/pycore_llist.h> // struct llist_node #include <internal/pycore_stackref.h> // Py_TAG_BITS +#include <internal/pycore_tstate.h> // _PyThreadStateImpl #include "../Python/remote_debug.h" +// gh-141784: Python.h header must be included first, before system headers. +// Otherwise, some types such as ino_t can be defined differently, causing ABI +// issues. +#include <errno.h> +#include <fcntl.h> +#include <stddef.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + #ifndef HAVE_PROCESS_VM_READV # define HAVE_PROCESS_VM_READV 0 #endif +/* ============================================================================ + * TYPE DEFINITIONS AND STRUCTURES + * ============================================================================ */ + +#define GET_MEMBER(type, obj, offset) \ + (*(type *)memcpy(&(type){0}, (const char *)(obj) + (offset), sizeof(type))) +#define CLEAR_PTR_TAG(ptr) (((uintptr_t)(ptr) & ~Py_TAG_BITS)) +#define GET_MEMBER_NO_TAG(type, obj, offset) \ + (type)(CLEAR_PTR_TAG(GET_MEMBER(type, obj, offset))) + +/* Size macros for opaque buffers */ +#define SIZEOF_BYTES_OBJ sizeof(PyBytesObject) +#define SIZEOF_CODE_OBJ sizeof(PyCodeObject) +#define SIZEOF_GEN_OBJ sizeof(PyGenObject) +#define SIZEOF_INTERP_FRAME sizeof(_PyInterpreterFrame) +#define SIZEOF_LLIST_NODE sizeof(struct llist_node) +#define SIZEOF_PAGE_CACHE_ENTRY sizeof(page_cache_entry_t) +#define SIZEOF_PYOBJECT sizeof(PyObject) +#define SIZEOF_SET_OBJ sizeof(PySetObject) +#define SIZEOF_TASK_OBJ 4096 +#define SIZEOF_THREAD_STATE sizeof(PyThreadState) +#define SIZEOF_TYPE_OBJ sizeof(PyTypeObject) +#define SIZEOF_UNICODE_OBJ sizeof(PyUnicodeObject) +#define SIZEOF_LONG_OBJ sizeof(PyLongObject) + +// Calculate the minimum buffer size needed to read interpreter state fields +// We need to read code_object_generation and potentially tlbc_generation +#ifndef MAX +#define MAX(a, b) ((a) > (b) ? (a) : (b)) +#endif + +#ifdef Py_GIL_DISABLED +#define INTERP_STATE_MIN_SIZE MAX(MAX(MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ + offsetof(PyInterpreterState, tlbc_indices.tlbc_generation) + sizeof(uint32_t)), \ + offsetof(PyInterpreterState, threads.head) + sizeof(void*)), \ + offsetof(PyInterpreterState, _gil.last_holder) + sizeof(PyThreadState*)) +#else +#define INTERP_STATE_MIN_SIZE MAX(MAX(offsetof(PyInterpreterState, _code_object_generation) + sizeof(uint64_t), \ + offsetof(PyInterpreterState, threads.head) + sizeof(void*)), \ + offsetof(PyInterpreterState, _gil.last_holder) + sizeof(PyThreadState*)) +#endif +#define INTERP_STATE_BUFFER_SIZE MAX(INTERP_STATE_MIN_SIZE, 256) +#define MAX_STACK_CHUNK_SIZE (16 * 1024 * 1024) /* 16 MB max for stack chunks */ +#define MAX_SET_TABLE_SIZE (1 << 20) /* 1 million entries max for set iteration */ +#define MAX_LONG_DIGITS 64 /* Allows values up to ~2^1920 */ + + + +// Copied from Modules/_asynciomodule.c because it's not exported + struct _Py_AsyncioModuleDebugOffsets { struct _asyncio_task_object { uint64_t size; @@ -45,750 +110,2037 @@ struct _Py_AsyncioModuleDebugOffsets { } asyncio_thread_state; }; -// Helper to chain exceptions and avoid repetitions -static void -chain_exceptions(PyObject *exception, const char *string) -{ - PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(exception, string); - _PyErr_ChainExceptions1(exc); -} - -// Get the PyAsyncioDebug section address for any platform -static uintptr_t -_Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle) -{ - uintptr_t address; - -#ifdef MS_WINDOWS - // On Windows, search for asyncio debug in executable or DLL - address = search_windows_map_for_section(handle, "AsyncioD", L"_asyncio"); -#elif defined(__linux__) - // On Linux, search for asyncio debug in executable or DLL - address = search_linux_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython"); -#elif defined(__APPLE__) && TARGET_OS_OSX - // On macOS, try libpython first, then fall back to python - address = search_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython"); - if (address == 0) { - PyErr_Clear(); - address = search_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython"); - } -#else - Py_UNREACHABLE(); -#endif - - return address; -} +/* Treat the remote debug tables as untrusted input and validate every + * size/offset we later dereference against a fixed local buffer or object + * layout before the unwinder starts using them. */ +#define FIELD_SIZE(type, member) sizeof(((type *)0)->member) +#define PY_REMOTE_DEBUG_INVALID_ASYNC_DEBUG_OFFSETS (-2) static inline int -read_ptr(proc_handle_t *handle, uintptr_t address, uintptr_t *ptr_addr) +validate_section_size(const char *section_name, uint64_t size) { - int result = _Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(void*), ptr_addr); - if (result < 0) { + if (size == 0) { + PyErr_Format( + PyExc_RuntimeError, + "Invalid debug offsets: %s.size must be greater than zero", + section_name); return -1; } return 0; } static inline int -read_Py_ssize_t(proc_handle_t *handle, uintptr_t address, Py_ssize_t *size) +validate_read_size(const char *section_name, uint64_t size, size_t buffer_size) { - int result = _Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(Py_ssize_t), size); - if (result < 0) { + if (validate_section_size(section_name, size) < 0) { return -1; } - return 0; -} - -static int -read_py_ptr(proc_handle_t *handle, uintptr_t address, uintptr_t *ptr_addr) -{ - if (read_ptr(handle, address, ptr_addr)) { + if (size > buffer_size) { + PyErr_Format( + PyExc_RuntimeError, + "Invalid debug offsets: %s.size=%llu exceeds local buffer size %zu", + section_name, + (unsigned long long)size, + buffer_size); return -1; } - *ptr_addr &= ~Py_TAG_BITS; return 0; } -static int -read_char(proc_handle_t *handle, uintptr_t address, char *result) +static inline int +validate_span( + const char *field_name, + uint64_t offset, + size_t width, + uint64_t limit, + const char *limit_name) { - int res = _Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(char), result); - if (res < 0) { + uint64_t span = (uint64_t)width; + if (span > limit || offset > limit - span) { + PyErr_Format( + PyExc_RuntimeError, + "Invalid debug offsets: %s=%llu with width %zu exceeds %s %llu", + field_name, + (unsigned long long)offset, + width, + limit_name, + (unsigned long long)limit); return -1; } return 0; } -static int -read_sized_int(proc_handle_t *handle, uintptr_t address, void *result, size_t size) +static inline int +validate_alignment( + const char *field_name, + uint64_t offset, + size_t alignment) { - int res = _Py_RemoteDebug_ReadRemoteMemory(handle, address, size, result); - if (res < 0) { + if (alignment > 1 && offset % alignment != 0) { + PyErr_Format( + PyExc_RuntimeError, + "Invalid debug offsets: %s=%llu is not aligned to %zu bytes", + field_name, + (unsigned long long)offset, + alignment); return -1; } return 0; } -static int -read_unsigned_long(proc_handle_t *handle, uintptr_t address, unsigned long *result) +static inline int +validate_field( + const char *field_name, + uint64_t offset, + uint64_t reported_size, + size_t width, + size_t alignment, + size_t buffer_size) { - int res = _Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(unsigned long), result); - if (res < 0) { + if (validate_alignment(field_name, offset, alignment) < 0) { return -1; } - return 0; + if (validate_span(field_name, offset, width, reported_size, "reported size") < 0) { + return -1; + } + return validate_span(field_name, offset, width, buffer_size, "local buffer size"); } -static int -read_pyobj(proc_handle_t *handle, uintptr_t address, PyObject *ptr_addr) +static inline int +validate_fixed_field( + const char *field_name, + uint64_t offset, + size_t width, + size_t alignment, + size_t buffer_size) { - int res = _Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(PyObject), ptr_addr); - if (res < 0) { + if (validate_alignment(field_name, offset, alignment) < 0) { return -1; } - return 0; + return validate_span(field_name, offset, width, buffer_size, "local buffer size"); } -static PyObject * -read_py_str( - proc_handle_t *handle, - _Py_DebugOffsets* debug_offsets, - uintptr_t address, - Py_ssize_t max_len -) { - PyObject *result = NULL; - char *buf = NULL; +#define PY_REMOTE_DEBUG_VALIDATE_SECTION(section) \ + do { \ + if (validate_section_size(#section, debug_offsets->section.size) < 0) { \ + return -1; \ + } \ + } while (0) + +#define PY_REMOTE_DEBUG_VALIDATE_READ_SECTION(section, buffer_size) \ + do { \ + if (validate_read_size(#section, debug_offsets->section.size, buffer_size) < 0) { \ + return -1; \ + } \ + } while (0) + +#define PY_REMOTE_DEBUG_VALIDATE_FIELD(section, field, field_size, field_alignment, buffer_size) \ + do { \ + if (validate_field( \ + #section "." #field, \ + debug_offsets->section.field, \ + debug_offsets->section.size, \ + field_size, \ + field_alignment, \ + buffer_size) < 0) { \ + return -1; \ + } \ + } while (0) + +#define PY_REMOTE_DEBUG_VALIDATE_FIXED_FIELD(section, field, field_size, field_alignment, buffer_size) \ + do { \ + if (validate_fixed_field( \ + #section "." #field, \ + debug_offsets->section.field, \ + field_size, \ + field_alignment, \ + buffer_size) < 0) { \ + return -1; \ + } \ + } while (0) - Py_ssize_t len; - int res = _Py_RemoteDebug_ReadRemoteMemory( - handle, - address + debug_offsets->unicode_object.length, - sizeof(Py_ssize_t), - &len - ); - if (res < 0) { - goto err; - } +static inline int +validate_debug_offsets_layout(struct _Py_DebugOffsets *debug_offsets) +{ + PY_REMOTE_DEBUG_VALIDATE_SECTION(runtime_state); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + runtime_state, + interpreters_head, + sizeof(uintptr_t), + _Alignof(uintptr_t), + sizeof(_PyRuntimeState)); - buf = (char *)PyMem_RawMalloc(len+1); - if (buf == NULL) { - PyErr_NoMemory(); - return NULL; - } + PY_REMOTE_DEBUG_VALIDATE_SECTION(interpreter_state); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + interpreter_state, + threads_head, + sizeof(uintptr_t), + _Alignof(uintptr_t), + INTERP_STATE_BUFFER_SIZE); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + interpreter_state, + threads_main, + sizeof(uintptr_t), + _Alignof(uintptr_t), + INTERP_STATE_BUFFER_SIZE); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + interpreter_state, + gil_runtime_state_locked, + sizeof(int), + _Alignof(int), + INTERP_STATE_BUFFER_SIZE); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + interpreter_state, + gil_runtime_state_holder, + sizeof(PyThreadState *), + _Alignof(PyThreadState *), + INTERP_STATE_BUFFER_SIZE); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + interpreter_state, + code_object_generation, + sizeof(uint64_t), + _Alignof(uint64_t), + INTERP_STATE_BUFFER_SIZE); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + interpreter_state, + tlbc_generation, + sizeof(uint32_t), + _Alignof(uint32_t), + INTERP_STATE_BUFFER_SIZE); + + PY_REMOTE_DEBUG_VALIDATE_READ_SECTION(thread_state, SIZEOF_THREAD_STATE); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + thread_state, + next, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_THREAD_STATE); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + thread_state, + current_frame, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_THREAD_STATE); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + thread_state, + native_thread_id, + sizeof(unsigned long), + _Alignof(unsigned long), + SIZEOF_THREAD_STATE); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + thread_state, + datastack_chunk, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_THREAD_STATE); - size_t offset = debug_offsets->unicode_object.asciiobject_size; - res = _Py_RemoteDebug_ReadRemoteMemory(handle, address + offset, len, buf); - if (res < 0) { - goto err; - } - buf[len] = '\0'; + PY_REMOTE_DEBUG_VALIDATE_SECTION(interpreter_frame); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + interpreter_frame, + previous, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_INTERP_FRAME); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + interpreter_frame, + executable, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_INTERP_FRAME); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + interpreter_frame, + instr_ptr, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_INTERP_FRAME); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + interpreter_frame, + owner, + sizeof(char), + _Alignof(char), + SIZEOF_INTERP_FRAME); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + interpreter_frame, + stackpointer, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_INTERP_FRAME); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + interpreter_frame, + tlbc_index, + sizeof(int32_t), + _Alignof(int32_t), + SIZEOF_INTERP_FRAME); + + PY_REMOTE_DEBUG_VALIDATE_SECTION(code_object); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + code_object, + qualname, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_CODE_OBJ); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + code_object, + filename, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_CODE_OBJ); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + code_object, + linetable, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_CODE_OBJ); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + code_object, + firstlineno, + sizeof(int), + _Alignof(int), + SIZEOF_CODE_OBJ); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + code_object, + co_code_adaptive, + sizeof(char), + _Alignof(char), + SIZEOF_CODE_OBJ); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + code_object, + co_tlbc, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_CODE_OBJ); - result = PyUnicode_FromStringAndSize(buf, len); - if (result == NULL) { - goto err; - } + PY_REMOTE_DEBUG_VALIDATE_SECTION(pyobject); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + pyobject, + ob_type, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_PYOBJECT); + + PY_REMOTE_DEBUG_VALIDATE_SECTION(type_object); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + type_object, + tp_flags, + sizeof(unsigned long), + _Alignof(unsigned long), + SIZEOF_TYPE_OBJ); + + PY_REMOTE_DEBUG_VALIDATE_SECTION(set_object); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + set_object, + used, + sizeof(Py_ssize_t), + _Alignof(Py_ssize_t), + SIZEOF_SET_OBJ); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + set_object, + mask, + sizeof(Py_ssize_t), + _Alignof(Py_ssize_t), + SIZEOF_SET_OBJ); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + set_object, + table, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_SET_OBJ); - PyMem_RawFree(buf); - assert(result != NULL); - return result; + PY_REMOTE_DEBUG_VALIDATE_READ_SECTION(long_object, SIZEOF_LONG_OBJ); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + long_object, + lv_tag, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_LONG_OBJ); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + long_object, + ob_digit, + sizeof(digit), + _Alignof(digit), + SIZEOF_LONG_OBJ); + + PY_REMOTE_DEBUG_VALIDATE_SECTION(bytes_object); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + bytes_object, + ob_size, + sizeof(Py_ssize_t), + _Alignof(Py_ssize_t), + SIZEOF_BYTES_OBJ); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + bytes_object, + ob_sval, + sizeof(char), + _Alignof(char), + SIZEOF_BYTES_OBJ); + + PY_REMOTE_DEBUG_VALIDATE_SECTION(unicode_object); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + unicode_object, + length, + sizeof(Py_ssize_t), + _Alignof(Py_ssize_t), + SIZEOF_UNICODE_OBJ); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + unicode_object, + asciiobject_size, + sizeof(char), + _Alignof(char), + SIZEOF_UNICODE_OBJ); + + PY_REMOTE_DEBUG_VALIDATE_SECTION(gen_object); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + gen_object, + gi_frame_state, + sizeof(int8_t), + _Alignof(int8_t), + SIZEOF_GEN_OBJ); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + gen_object, + gi_iframe, + FIELD_SIZE(PyGenObject, gi_iframe), + _Alignof(_PyInterpreterFrame), + SIZEOF_GEN_OBJ); + + PY_REMOTE_DEBUG_VALIDATE_FIXED_FIELD( + llist_node, + next, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_LLIST_NODE); -err: - if (buf != NULL) { - PyMem_RawFree(buf); - } - return NULL; + return 0; } -static PyObject * -read_py_bytes( - proc_handle_t *handle, - _Py_DebugOffsets* debug_offsets, - uintptr_t address -) { - PyObject *result = NULL; - char *buf = NULL; +static inline int +validate_async_debug_offsets_layout(struct _Py_AsyncioModuleDebugOffsets *debug_offsets) +{ + PY_REMOTE_DEBUG_VALIDATE_READ_SECTION(asyncio_task_object, SIZEOF_TASK_OBJ); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + asyncio_task_object, + task_name, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_TASK_OBJ); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + asyncio_task_object, + task_awaited_by, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_TASK_OBJ); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + asyncio_task_object, + task_is_task, + sizeof(char), + _Alignof(char), + SIZEOF_TASK_OBJ); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + asyncio_task_object, + task_awaited_by_is_set, + sizeof(char), + _Alignof(char), + SIZEOF_TASK_OBJ); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + asyncio_task_object, + task_coro, + sizeof(uintptr_t), + _Alignof(uintptr_t), + SIZEOF_TASK_OBJ); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + asyncio_task_object, + task_node, + SIZEOF_LLIST_NODE, + _Alignof(struct llist_node), + SIZEOF_TASK_OBJ); + + PY_REMOTE_DEBUG_VALIDATE_SECTION(asyncio_interpreter_state); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + asyncio_interpreter_state, + asyncio_tasks_head, + SIZEOF_LLIST_NODE, + _Alignof(struct llist_node), + sizeof(PyInterpreterState)); + + PY_REMOTE_DEBUG_VALIDATE_SECTION(asyncio_thread_state); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + asyncio_thread_state, + asyncio_running_loop, + sizeof(uintptr_t), + _Alignof(uintptr_t), + sizeof(_PyThreadStateImpl)); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + asyncio_thread_state, + asyncio_running_task, + sizeof(uintptr_t), + _Alignof(uintptr_t), + sizeof(_PyThreadStateImpl)); + PY_REMOTE_DEBUG_VALIDATE_FIELD( + asyncio_thread_state, + asyncio_tasks_head, + SIZEOF_LLIST_NODE, + _Alignof(struct llist_node), + sizeof(_PyThreadStateImpl)); - Py_ssize_t len; - int res = _Py_RemoteDebug_ReadRemoteMemory( - handle, - address + debug_offsets->bytes_object.ob_size, - sizeof(Py_ssize_t), - &len - ); - if (res < 0) { - goto err; - } + return 0; +} - buf = (char *)PyMem_RawMalloc(len+1); - if (buf == NULL) { - PyErr_NoMemory(); - return NULL; - } +#undef PY_REMOTE_DEBUG_VALIDATE_SECTION +#undef PY_REMOTE_DEBUG_VALIDATE_READ_SECTION +#undef PY_REMOTE_DEBUG_VALIDATE_FIELD +#undef PY_REMOTE_DEBUG_VALIDATE_FIXED_FIELD +#undef FIELD_SIZE + +/* ============================================================================ + * STRUCTSEQ TYPE DEFINITIONS + * ============================================================================ */ + +// TaskInfo structseq type - replaces 4-tuple (task_id, task_name, coroutine_stack, awaited_by) +static PyStructSequence_Field TaskInfo_fields[] = { + {"task_id", "Task ID (memory address)"}, + {"task_name", "Task name"}, + {"coroutine_stack", "Coroutine call stack"}, + {"awaited_by", "Tasks awaiting this task"}, + {NULL} +}; - size_t offset = debug_offsets->bytes_object.ob_sval; - res = _Py_RemoteDebug_ReadRemoteMemory(handle, address + offset, len, buf); - if (res < 0) { - goto err; - } - buf[len] = '\0'; +static PyStructSequence_Desc TaskInfo_desc = { + "_remote_debugging.TaskInfo", + "Information about an asyncio task", + TaskInfo_fields, + 4 +}; - result = PyBytes_FromStringAndSize(buf, len); - if (result == NULL) { - goto err; - } +// FrameInfo structseq type - replaces 3-tuple (filename, lineno, funcname) +static PyStructSequence_Field FrameInfo_fields[] = { + {"filename", "Source code filename"}, + {"lineno", "Line number"}, + {"funcname", "Function name"}, + {NULL} +}; - PyMem_RawFree(buf); - assert(result != NULL); - return result; +static PyStructSequence_Desc FrameInfo_desc = { + "_remote_debugging.FrameInfo", + "Information about a frame", + FrameInfo_fields, + 3 +}; -err: - if (buf != NULL) { - PyMem_RawFree(buf); - } - return NULL; -} +// CoroInfo structseq type - replaces 2-tuple (call_stack, task_name) +static PyStructSequence_Field CoroInfo_fields[] = { + {"call_stack", "Coroutine call stack"}, + {"task_name", "Task name"}, + {NULL} +}; +static PyStructSequence_Desc CoroInfo_desc = { + "_remote_debugging.CoroInfo", + "Information about a coroutine", + CoroInfo_fields, + 2 +}; +// ThreadInfo structseq type - replaces 2-tuple (thread_id, frame_info) +static PyStructSequence_Field ThreadInfo_fields[] = { + {"thread_id", "Thread ID"}, + {"frame_info", "Frame information"}, + {NULL} +}; -static long -read_py_long(proc_handle_t *handle, _Py_DebugOffsets* offsets, uintptr_t address) -{ - unsigned int shift = PYLONG_BITS_IN_DIGIT; +static PyStructSequence_Desc ThreadInfo_desc = { + "_remote_debugging.ThreadInfo", + "Information about a thread", + ThreadInfo_fields, + 2 +}; - Py_ssize_t size; - uintptr_t lv_tag; +// AwaitedInfo structseq type - replaces 2-tuple (tid, awaited_by_list) +static PyStructSequence_Field AwaitedInfo_fields[] = { + {"thread_id", "Thread ID"}, + {"awaited_by", "List of tasks awaited by this thread"}, + {NULL} +}; - int bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, address + offsets->long_object.lv_tag, - sizeof(uintptr_t), - &lv_tag); - if (bytes_read < 0) { - return -1; - } +static PyStructSequence_Desc AwaitedInfo_desc = { + "_remote_debugging.AwaitedInfo", + "Information about what a thread is awaiting", + AwaitedInfo_fields, + 2 +}; - int negative = (lv_tag & 3) == 2; - size = lv_tag >> 3; +typedef struct { + PyObject *func_name; + PyObject *file_name; + int first_lineno; + PyObject *linetable; // bytes + uintptr_t addr_code_adaptive; +} CachedCodeMetadata; + +typedef struct { + /* Types */ + PyTypeObject *RemoteDebugging_Type; + PyTypeObject *TaskInfo_Type; + PyTypeObject *FrameInfo_Type; + PyTypeObject *CoroInfo_Type; + PyTypeObject *ThreadInfo_Type; + PyTypeObject *AwaitedInfo_Type; +} RemoteDebuggingState; + +typedef struct { + PyObject_HEAD + proc_handle_t handle; + uintptr_t runtime_start_address; + struct _Py_DebugOffsets debug_offsets; + int async_debug_offsets_available; + struct _Py_AsyncioModuleDebugOffsets async_debug_offsets; + uintptr_t interpreter_addr; + uintptr_t tstate_addr; + uint64_t code_object_generation; + _Py_hashtable_t *code_object_cache; + int debug; + int only_active_thread; + RemoteDebuggingState *cached_state; // Cached module state +#ifdef Py_GIL_DISABLED + // TLBC cache invalidation tracking + uint32_t tlbc_generation; // Track TLBC index pool changes + _Py_hashtable_t *tlbc_cache; // Cache of TLBC arrays by code object address +#endif +} RemoteUnwinderObject; - if (size == 0) { - return 0; - } +#define RemoteUnwinder_CAST(op) ((RemoteUnwinderObject *)(op)) - digit *digits = (digit *)PyMem_RawMalloc(size * sizeof(digit)); - if (!digits) { - PyErr_NoMemory(); - return -1; - } +typedef struct +{ + int lineno; + int end_lineno; + int column; + int end_column; +} LocationInfo; - bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, - address + offsets->long_object.ob_digit, - sizeof(digit) * size, - digits - ); - if (bytes_read < 0) { - goto error; - } +typedef struct { + uintptr_t remote_addr; + size_t size; + void *local_copy; +} StackChunkInfo; - long long value = 0; +typedef struct { + StackChunkInfo *chunks; + size_t count; +} StackChunkList; - // In theory this can overflow, but because of llvm/llvm-project#16778 - // we can't use __builtin_mul_overflow because it fails to link with - // __muloti4 on aarch64. In practice this is fine because all we're - // testing here are task numbers that would fit in a single byte. - for (Py_ssize_t i = 0; i < size; ++i) { - long long factor = digits[i] * (1UL << (Py_ssize_t)(shift * i)); - value += factor; - } - PyMem_RawFree(digits); - if (negative) { - value *= -1; - } - return (long)value; -error: - PyMem_RawFree(digits); - return -1; -} +#include "clinic/_remote_debugging_module.c.h" -static PyObject * -parse_task_name( - proc_handle_t *handle, - _Py_DebugOffsets* offsets, - struct _Py_AsyncioModuleDebugOffsets* async_offsets, - uintptr_t task_address -) { - uintptr_t task_name_addr; - int err = read_py_ptr( - handle, - task_address + async_offsets->asyncio_task_object.task_name, - &task_name_addr); - if (err) { - return NULL; - } +/*[clinic input] +module _remote_debugging +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=5f507d5b2e76a7f7]*/ - // The task name can be a long or a string so we need to check the type - PyObject task_name_obj; - err = read_pyobj( - handle, - task_name_addr, - &task_name_obj); - if (err) { - return NULL; - } +/* ============================================================================ + * FORWARD DECLARATIONS + * ============================================================================ */ - unsigned long flags; - err = read_unsigned_long( - handle, - (uintptr_t)task_name_obj.ob_type + offsets->type_object.tp_flags, - &flags); - if (err) { - return NULL; - } +static inline int +is_frame_valid( + RemoteUnwinderObject *unwinder, + uintptr_t frame_addr, + uintptr_t code_object_addr +); - if ((flags & Py_TPFLAGS_LONG_SUBCLASS)) { - long res = read_py_long(handle, offsets, task_name_addr); - if (res == -1) { - chain_exceptions(PyExc_RuntimeError, "Failed to get task name"); - return NULL; - } - return PyUnicode_FromFormat("Task-%d", res); - } +typedef int (*thread_processor_func)( + RemoteUnwinderObject *unwinder, + uintptr_t thread_state_addr, + unsigned long tid, + void *context +); - if(!(flags & Py_TPFLAGS_UNICODE_SUBCLASS)) { - PyErr_SetString(PyExc_RuntimeError, "Invalid task name object"); - return NULL; - } +typedef int (*set_entry_processor_func)( + RemoteUnwinderObject *unwinder, + uintptr_t key_addr, + void *context +); + + +static int +parse_task( + RemoteUnwinderObject *unwinder, + uintptr_t task_address, + PyObject *render_to +); + +static int +parse_coro_chain( + RemoteUnwinderObject *unwinder, + uintptr_t coro_address, + PyObject *render_to +); + +/* Forward declarations for task parsing functions */ +static int parse_frame_object( + RemoteUnwinderObject *unwinder, + PyObject** result, + uintptr_t address, + uintptr_t* address_of_code_object, + uintptr_t* previous_frame +); + +static int +parse_async_frame_chain( + RemoteUnwinderObject *unwinder, + PyObject *calls, + uintptr_t address_of_thread, + uintptr_t running_task_code_obj +); + +static int read_py_ptr(RemoteUnwinderObject *unwinder, uintptr_t address, uintptr_t *ptr_addr); +static int read_Py_ssize_t(RemoteUnwinderObject *unwinder, uintptr_t address, Py_ssize_t *size); + +static int process_task_and_waiters(RemoteUnwinderObject *unwinder, uintptr_t task_addr, PyObject *result); +static int process_task_awaited_by(RemoteUnwinderObject *unwinder, uintptr_t task_address, set_entry_processor_func processor, void *context); +static int find_running_task_in_thread(RemoteUnwinderObject *unwinder, uintptr_t thread_state_addr, uintptr_t *running_task_addr); +static int get_task_code_object(RemoteUnwinderObject *unwinder, uintptr_t task_addr, uintptr_t *code_obj_addr); +static int append_awaited_by(RemoteUnwinderObject *unwinder, unsigned long tid, uintptr_t head_addr, PyObject *result); + +/* ============================================================================ + * UTILITY FUNCTIONS AND HELPERS + * ============================================================================ */ + +#define set_exception_cause(unwinder, exc_type, message) \ + do { \ + assert(PyErr_Occurred() && "function returned -1 without setting exception"); \ + if (unwinder->debug) { \ + _set_debug_exception_cause(exc_type, message); \ + } \ + } while (0) + +static void +cached_code_metadata_destroy(void *ptr) +{ + CachedCodeMetadata *meta = (CachedCodeMetadata *)ptr; + Py_DECREF(meta->func_name); + Py_DECREF(meta->file_name); + Py_DECREF(meta->linetable); + PyMem_RawFree(meta); +} + +static inline RemoteDebuggingState * +RemoteDebugging_GetState(PyObject *module) +{ + void *state = _PyModule_GetState(module); + assert(state != NULL); + return (RemoteDebuggingState *)state; +} + +static inline RemoteDebuggingState * +RemoteDebugging_GetStateFromType(PyTypeObject *type) +{ + PyObject *module = PyType_GetModule(type); + assert(module != NULL); + return RemoteDebugging_GetState(module); +} + +static inline RemoteDebuggingState * +RemoteDebugging_GetStateFromObject(PyObject *obj) +{ + RemoteUnwinderObject *unwinder = (RemoteUnwinderObject *)obj; + if (unwinder->cached_state == NULL) { + unwinder->cached_state = RemoteDebugging_GetStateFromType(Py_TYPE(obj)); + } + return unwinder->cached_state; +} + +static inline int +RemoteDebugging_InitState(RemoteDebuggingState *st) +{ + return 0; +} + +static int +is_prerelease_version(uint64_t version) +{ + return (version & 0xF0) != 0xF0; +} + +static inline int +validate_debug_offsets(struct _Py_DebugOffsets *debug_offsets) +{ + if (memcmp(debug_offsets->cookie, _Py_Debug_Cookie, sizeof(debug_offsets->cookie)) != 0) { + // The remote is probably running a Python version predating debug offsets. + PyErr_SetString( + PyExc_RuntimeError, + "Can't determine the Python version of the remote process"); + return -1; + } + + // Assume debug offsets could change from one pre-release version to another, + // or one minor version to another, but are stable across patch versions. + if (is_prerelease_version(Py_Version) && Py_Version != debug_offsets->version) { + PyErr_SetString( + PyExc_RuntimeError, + "Can't attach from a pre-release Python interpreter" + " to a process running a different Python version"); + return -1; + } + + if (is_prerelease_version(debug_offsets->version) && Py_Version != debug_offsets->version) { + PyErr_SetString( + PyExc_RuntimeError, + "Can't attach to a pre-release Python interpreter" + " from a process running a different Python version"); + return -1; + } + + unsigned int remote_major = (debug_offsets->version >> 24) & 0xFF; + unsigned int remote_minor = (debug_offsets->version >> 16) & 0xFF; + + if (PY_MAJOR_VERSION != remote_major || PY_MINOR_VERSION != remote_minor) { + PyErr_Format( + PyExc_RuntimeError, + "Can't attach from a Python %d.%d process to a Python %d.%d process", + PY_MAJOR_VERSION, PY_MINOR_VERSION, remote_major, remote_minor); + return -1; + } + + // The debug offsets differ between free threaded and non-free threaded builds. + if (_Py_Debug_Free_Threaded && !debug_offsets->free_threaded) { + PyErr_SetString( + PyExc_RuntimeError, + "Cannot attach from a free-threaded Python process" + " to a process running a non-free-threaded version"); + return -1; + } + + if (!_Py_Debug_Free_Threaded && debug_offsets->free_threaded) { + PyErr_SetString( + PyExc_RuntimeError, + "Cannot attach to a free-threaded Python process" + " from a process running a non-free-threaded version"); + return -1; + } + + return validate_debug_offsets_layout(debug_offsets); +} + +// Generic function to iterate through all threads +static int +iterate_threads( + RemoteUnwinderObject *unwinder, + thread_processor_func processor, + void *context +) { + uintptr_t thread_state_addr; + unsigned long tid = 0; + + if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + unwinder->interpreter_addr + (uintptr_t)unwinder->debug_offsets.interpreter_state.threads_main, + sizeof(void*), + &thread_state_addr)) + { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read main thread state"); + return -1; + } + + while (thread_state_addr != 0) { + if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + thread_state_addr + (uintptr_t)unwinder->debug_offsets.thread_state.native_thread_id, + sizeof(tid), + &tid)) + { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread ID"); + return -1; + } + + // Call the processor function for this thread + if (processor(unwinder, thread_state_addr, tid, context) < 0) { + return -1; + } + + // Move to next thread + if (0 > _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + thread_state_addr + (uintptr_t)unwinder->debug_offsets.thread_state.next, + sizeof(void*), + &thread_state_addr)) + { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read next thread state"); + return -1; + } + } + + return 0; +} + +// Generic function to iterate through set entries +static int +iterate_set_entries( + RemoteUnwinderObject *unwinder, + uintptr_t set_addr, + set_entry_processor_func processor, + void *context +) { + char set_object[SIZEOF_SET_OBJ]; + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, set_addr, + SIZEOF_SET_OBJ, set_object) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read set object"); + return -1; + } + + Py_ssize_t num_els = GET_MEMBER(Py_ssize_t, set_object, unwinder->debug_offsets.set_object.used); + Py_ssize_t mask = GET_MEMBER(Py_ssize_t, set_object, unwinder->debug_offsets.set_object.mask); + uintptr_t table_ptr = GET_MEMBER(uintptr_t, set_object, unwinder->debug_offsets.set_object.table); + if (mask < 0 || mask >= MAX_SET_TABLE_SIZE || num_els < 0 || num_els > mask + 1) { + PyErr_SetString(PyExc_RuntimeError, "Invalid set object (corrupted remote memory)"); + set_exception_cause(unwinder, PyExc_RuntimeError, + "Invalid set object (corrupted remote memory)"); + return -1; + } + Py_ssize_t set_len = mask + 1; + + Py_ssize_t i = 0; + Py_ssize_t els = 0; + while (i < set_len && els < num_els) { + uintptr_t key_addr; + if (read_py_ptr(unwinder, table_ptr, &key_addr) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read set entry key"); + return -1; + } + + if ((void*)key_addr != NULL) { + Py_ssize_t ref_cnt; + if (read_Py_ssize_t(unwinder, table_ptr, &ref_cnt) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read set entry ref count"); + return -1; + } + + if (ref_cnt) { + // Process this valid set entry + if (processor(unwinder, key_addr, context) < 0) { + return -1; + } + els++; + } + } + table_ptr += sizeof(void*) * 2; + i++; + } + + return 0; +} + +// Processor function for task waiters +static int +process_waiter_task( + RemoteUnwinderObject *unwinder, + uintptr_t key_addr, + void *context +) { + PyObject *result = (PyObject *)context; + return process_task_and_waiters(unwinder, key_addr, result); +} + +// Processor function for parsing tasks in sets +static int +process_task_parser( + RemoteUnwinderObject *unwinder, + uintptr_t key_addr, + void *context +) { + PyObject *awaited_by = (PyObject *)context; + return parse_task(unwinder, key_addr, awaited_by); +} + +/* ============================================================================ + * MEMORY READING FUNCTIONS + * ============================================================================ */ + +#define DEFINE_MEMORY_READER(type_name, c_type, error_msg) \ +static inline int \ +read_##type_name(RemoteUnwinderObject *unwinder, uintptr_t address, c_type *result) \ +{ \ + int res = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address, sizeof(c_type), result); \ + if (res < 0) { \ + set_exception_cause(unwinder, PyExc_RuntimeError, error_msg); \ + return -1; \ + } \ + return 0; \ +} + +DEFINE_MEMORY_READER(ptr, uintptr_t, "Failed to read pointer from remote memory") +DEFINE_MEMORY_READER(Py_ssize_t, Py_ssize_t, "Failed to read Py_ssize_t from remote memory") +DEFINE_MEMORY_READER(char, char, "Failed to read char from remote memory") + +static int +read_py_ptr(RemoteUnwinderObject *unwinder, uintptr_t address, uintptr_t *ptr_addr) +{ + if (read_ptr(unwinder, address, ptr_addr)) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read Python pointer"); + return -1; + } + *ptr_addr &= ~Py_TAG_BITS; + return 0; +} + +/* ============================================================================ + * PYTHON OBJECT READING FUNCTIONS + * ============================================================================ */ + +static PyObject * +read_py_str( + RemoteUnwinderObject *unwinder, + uintptr_t address, + Py_ssize_t max_len +) { + PyObject *result = NULL; + char *buf = NULL; + + // Read the entire PyUnicodeObject at once + char unicode_obj[SIZEOF_UNICODE_OBJ]; + int res = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + address, + SIZEOF_UNICODE_OBJ, + unicode_obj + ); + if (res < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read PyUnicodeObject"); + goto err; + } + + Py_ssize_t len = GET_MEMBER(Py_ssize_t, unicode_obj, unwinder->debug_offsets.unicode_object.length); + if (len < 0 || len > max_len) { + PyErr_Format(PyExc_RuntimeError, + "Invalid string length (%zd) at 0x%lx", len, address); + set_exception_cause(unwinder, PyExc_RuntimeError, "Invalid string length in remote Unicode object"); + return NULL; + } + + buf = (char *)PyMem_RawMalloc(len+1); + if (buf == NULL) { + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate buffer for string reading"); + return NULL; + } + + size_t offset = (size_t)unwinder->debug_offsets.unicode_object.asciiobject_size; + res = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address + offset, len, buf); + if (res < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read string data from remote memory"); + goto err; + } + buf[len] = '\0'; + + result = PyUnicode_FromStringAndSize(buf, len); + if (result == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create PyUnicode from remote string data"); + goto err; + } + + PyMem_RawFree(buf); + assert(result != NULL); + return result; + +err: + if (buf != NULL) { + PyMem_RawFree(buf); + } + return NULL; +} + +static PyObject * +read_py_bytes( + RemoteUnwinderObject *unwinder, + uintptr_t address, + Py_ssize_t max_len +) { + PyObject *result = NULL; + char *buf = NULL; + + // Read the entire PyBytesObject at once + char bytes_obj[SIZEOF_BYTES_OBJ]; + int res = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + address, + SIZEOF_BYTES_OBJ, + bytes_obj + ); + if (res < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read PyBytesObject"); + goto err; + } + + Py_ssize_t len = GET_MEMBER(Py_ssize_t, bytes_obj, unwinder->debug_offsets.bytes_object.ob_size); + if (len < 0 || len > max_len) { + PyErr_Format(PyExc_RuntimeError, + "Invalid bytes length (%zd) at 0x%lx", len, address); + set_exception_cause(unwinder, PyExc_RuntimeError, "Invalid bytes length in remote bytes object"); + return NULL; + } + + buf = (char *)PyMem_RawMalloc(len+1); + if (buf == NULL) { + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate buffer for bytes reading"); + return NULL; + } + + size_t offset = (size_t)unwinder->debug_offsets.bytes_object.ob_sval; + res = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, address + offset, len, buf); + if (res < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read bytes data from remote memory"); + goto err; + } + buf[len] = '\0'; + + result = PyBytes_FromStringAndSize(buf, len); + if (result == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create PyBytes from remote bytes data"); + goto err; + } + + PyMem_RawFree(buf); + assert(result != NULL); + return result; + +err: + if (buf != NULL) { + PyMem_RawFree(buf); + } + return NULL; +} + +static long +read_py_long( + RemoteUnwinderObject *unwinder, + uintptr_t address +) +{ + unsigned int shift = PYLONG_BITS_IN_DIGIT; + + // Read the entire PyLongObject at once + char long_obj[SIZEOF_LONG_OBJ]; + int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + address, + (size_t)unwinder->debug_offsets.long_object.size, + long_obj); + if (bytes_read < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read PyLongObject"); + return -1; + } + + uintptr_t lv_tag = GET_MEMBER(uintptr_t, long_obj, unwinder->debug_offsets.long_object.lv_tag); + int negative = (lv_tag & 3) == 2; + Py_ssize_t size = lv_tag >> 3; + + if (size == 0) { + return 0; + } + + if (size < 0 || size > MAX_LONG_DIGITS) { + PyErr_Format(PyExc_RuntimeError, + "Invalid PyLong digit count: %zd (expected 0-%d)", + size, MAX_LONG_DIGITS); + set_exception_cause(unwinder, PyExc_RuntimeError, + "Invalid PyLong size (corrupted remote memory)"); + return -1; + } + + // If the long object has inline digits, use them directly + digit *digits; + if (size <= _PY_NSMALLNEGINTS + _PY_NSMALLPOSINTS) { + // For small integers, digits are inline in the long_value.ob_digit array + digits = (digit *)PyMem_RawMalloc(size * sizeof(digit)); + if (!digits) { + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate digits for small PyLong"); + return -1; + } + memcpy(digits, long_obj + unwinder->debug_offsets.long_object.ob_digit, size * sizeof(digit)); + } else { + // For larger integers, we need to read the digits separately + digits = (digit *)PyMem_RawMalloc(size * sizeof(digit)); + if (!digits) { + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate digits for large PyLong"); + return -1; + } + + bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + address + (uintptr_t)unwinder->debug_offsets.long_object.ob_digit, + sizeof(digit) * size, + digits + ); + if (bytes_read < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read PyLong digits from remote memory"); + goto error; + } + } + + long long value = 0; + + // In theory this can overflow, but because of llvm/llvm-project#16778 + // we can't use __builtin_mul_overflow because it fails to link with + // __muloti4 on aarch64. In practice this is fine because all we're + // testing here are task numbers that would fit in a single byte. + for (Py_ssize_t i = 0; i < size; ++i) { + long long factor = digits[i] * (1UL << (Py_ssize_t)(shift * i)); + value += factor; + } + PyMem_RawFree(digits); + if (negative) { + value *= -1; + } + return (long)value; +error: + PyMem_RawFree(digits); + return -1; +} + +/* ============================================================================ + * ASYNCIO DEBUG FUNCTIONS + * ============================================================================ */ + +// Get the PyAsyncioDebug section address for any platform +static uintptr_t +_Py_RemoteDebug_GetAsyncioDebugAddress(proc_handle_t* handle) +{ + uintptr_t address; + +#ifdef MS_WINDOWS + // On Windows, search for asyncio debug in executable or DLL + address = search_windows_map_for_section(handle, "AsyncioD", L"_asyncio", NULL); + if (address == 0) { + // Error out: 'python' substring covers both executable and DLL + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); + _PyErr_ChainExceptions1(exc); + } +#elif defined(__linux__) + // On Linux, search for asyncio debug in executable or DLL + address = search_linux_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython", NULL); + if (address == 0) { + // Error out: 'python' substring covers both executable and DLL + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); + _PyErr_ChainExceptions1(exc); + } +#elif defined(__APPLE__) && TARGET_OS_OSX + // On macOS, try libpython first, then fall back to python + address = search_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython", NULL); + if (address == 0) { + PyErr_Clear(); + address = search_map_for_section(handle, "AsyncioDebug", "_asyncio.cpython", NULL); + } + if (address == 0) { + // Error out: 'python' substring covers both executable and DLL + PyObject *exc = PyErr_GetRaisedException(); + PyErr_SetString(PyExc_RuntimeError, "Failed to find the AsyncioDebug section in the process."); + _PyErr_ChainExceptions1(exc); + } +#else + Py_UNREACHABLE(); +#endif + + return address; +} + +static int +read_async_debug( + RemoteUnwinderObject *unwinder +) { + uintptr_t async_debug_addr = _Py_RemoteDebug_GetAsyncioDebugAddress(&unwinder->handle); + if (!async_debug_addr) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to get AsyncioDebug address"); + return -1; + } + + size_t size = sizeof(struct _Py_AsyncioModuleDebugOffsets); + int result = _Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, async_debug_addr, size, &unwinder->async_debug_offsets); + if (result < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read AsyncioDebug offsets"); + return result; + } + if (validate_async_debug_offsets_layout(&unwinder->async_debug_offsets) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Invalid AsyncioDebug offsets"); + return PY_REMOTE_DEBUG_INVALID_ASYNC_DEBUG_OFFSETS; + } + return 0; +} + +/* ============================================================================ + * ASYNCIO TASK PARSING FUNCTIONS + * ============================================================================ */ + +static PyObject * +parse_task_name( + RemoteUnwinderObject *unwinder, + uintptr_t task_address +) { + // Read the entire TaskObj at once + char task_obj[SIZEOF_TASK_OBJ]; + int err = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + task_address, + (size_t)unwinder->async_debug_offsets.asyncio_task_object.size, + task_obj); + if (err < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task object"); + return NULL; + } + + uintptr_t task_name_addr = GET_MEMBER_NO_TAG(uintptr_t, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_name); + + // The task name can be a long or a string so we need to check the type + char task_name_obj[SIZEOF_PYOBJECT]; + err = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + task_name_addr, + SIZEOF_PYOBJECT, + task_name_obj); + if (err < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task name object"); + return NULL; + } + + // Now read the type object to get the flags + char type_obj[SIZEOF_TYPE_OBJ]; + err = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + GET_MEMBER(uintptr_t, task_name_obj, unwinder->debug_offsets.pyobject.ob_type), + SIZEOF_TYPE_OBJ, + type_obj); + if (err < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task name type object"); + return NULL; + } + + if ((GET_MEMBER(unsigned long, type_obj, unwinder->debug_offsets.type_object.tp_flags) & Py_TPFLAGS_LONG_SUBCLASS)) { + long res = read_py_long(unwinder, task_name_addr); + if (res == -1) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Task name PyLong parsing failed"); + return NULL; + } + return PyUnicode_FromFormat("Task-%ld", res); + } + + if(!(GET_MEMBER(unsigned long, type_obj, unwinder->debug_offsets.type_object.tp_flags) & Py_TPFLAGS_UNICODE_SUBCLASS)) { + PyErr_SetString(PyExc_RuntimeError, "Invalid task name object"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Task name object is neither long nor unicode"); + return NULL; + } return read_py_str( - handle, - offsets, + unwinder, task_name_addr, 255 ); } -static int -parse_frame_object( - proc_handle_t *handle, - PyObject** result, - struct _Py_DebugOffsets* offsets, - uintptr_t address, - uintptr_t* previous_frame -); +static int parse_task_awaited_by( + RemoteUnwinderObject *unwinder, + uintptr_t task_address, + PyObject *awaited_by +) { + return process_task_awaited_by(unwinder, task_address, process_task_parser, awaited_by); +} + +static int +handle_yield_from_frame( + RemoteUnwinderObject *unwinder, + uintptr_t gi_iframe_addr, + uintptr_t gen_type_addr, + PyObject *render_to +) { + // Read the entire interpreter frame at once + char iframe[SIZEOF_INTERP_FRAME]; + int err = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + gi_iframe_addr, + SIZEOF_INTERP_FRAME, + iframe); + if (err < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read interpreter frame in yield_from handler"); + return -1; + } + + if (GET_MEMBER(char, iframe, unwinder->debug_offsets.interpreter_frame.owner) != FRAME_OWNED_BY_GENERATOR) { + PyErr_SetString( + PyExc_RuntimeError, + "generator doesn't own its frame \\_o_/"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Frame ownership mismatch in yield_from"); + return -1; + } + + uintptr_t stackpointer_addr = GET_MEMBER_NO_TAG(uintptr_t, iframe, unwinder->debug_offsets.interpreter_frame.stackpointer); + + if ((void*)stackpointer_addr != NULL) { + uintptr_t gi_await_addr; + err = read_py_ptr( + unwinder, + stackpointer_addr - sizeof(void*), + &gi_await_addr); + if (err) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read gi_await address"); + return -1; + } + + if ((void*)gi_await_addr != NULL) { + uintptr_t gi_await_addr_type_addr; + err = read_ptr( + unwinder, + gi_await_addr + (uintptr_t)unwinder->debug_offsets.pyobject.ob_type, + &gi_await_addr_type_addr); + if (err) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read gi_await type address"); + return -1; + } + + if (gen_type_addr == gi_await_addr_type_addr) { + /* This needs an explanation. We always start with parsing + native coroutine / generator frames. Ultimately they + are awaiting on something. That something can be + a native coroutine frame or... an iterator. + If it's the latter -- we can't continue building + our chain. So the condition to bail out of this is + to do that when the type of the current coroutine + doesn't match the type of whatever it points to + in its cr_await. + */ + err = parse_coro_chain(unwinder, gi_await_addr, render_to); + if (err) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse coroutine chain in yield_from"); + return -1; + } + } + } + } + + return 0; +} + +static int +parse_coro_chain( + RemoteUnwinderObject *unwinder, + uintptr_t coro_address, + PyObject *render_to +) { + assert((void*)coro_address != NULL); + + // Read the entire generator object at once + char gen_object[SIZEOF_GEN_OBJ]; + int err = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + coro_address, + SIZEOF_GEN_OBJ, + gen_object); + if (err < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read generator object in coro chain"); + return -1; + } + + int8_t frame_state = GET_MEMBER(int8_t, gen_object, unwinder->debug_offsets.gen_object.gi_frame_state); + if (frame_state == FRAME_CLEARED) { + return 0; + } + + uintptr_t gen_type_addr = GET_MEMBER(uintptr_t, gen_object, unwinder->debug_offsets.pyobject.ob_type); + + PyObject* name = NULL; + + // Parse the previous frame using the gi_iframe from local copy + uintptr_t prev_frame; + uintptr_t gi_iframe_addr = coro_address + (uintptr_t)unwinder->debug_offsets.gen_object.gi_iframe; + uintptr_t address_of_code_object = 0; + if (parse_frame_object(unwinder, &name, gi_iframe_addr, &address_of_code_object, &prev_frame) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse frame object in coro chain"); + return -1; + } + + if (!name) { + return 0; + } + + if (PyList_Append(render_to, name)) { + Py_DECREF(name); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append frame to coro chain"); + return -1; + } + Py_DECREF(name); + + if (frame_state == FRAME_SUSPENDED_YIELD_FROM) { + return handle_yield_from_frame(unwinder, gi_iframe_addr, gen_type_addr, render_to); + } + + return 0; +} + +static PyObject* +create_task_result( + RemoteUnwinderObject *unwinder, + uintptr_t task_address +) { + PyObject* result = NULL; + PyObject *call_stack = NULL; + PyObject *tn = NULL; + char task_obj[SIZEOF_TASK_OBJ]; + uintptr_t coro_addr; + + // Create call_stack first since it's the first tuple element + call_stack = PyList_New(0); + if (call_stack == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create call stack list"); + goto error; + } + + // Create task name/address for second tuple element + tn = PyLong_FromUnsignedLongLong(task_address); + if (tn == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create task name/address"); + goto error; + } + + // Parse coroutine chain + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, task_address, + (size_t)unwinder->async_debug_offsets.asyncio_task_object.size, + task_obj) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task object for coro chain"); + goto error; + } + + coro_addr = GET_MEMBER_NO_TAG(uintptr_t, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_coro); + + if ((void*)coro_addr != NULL) { + if (parse_coro_chain(unwinder, coro_addr, call_stack) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse coroutine chain"); + goto error; + } + + if (PyList_Reverse(call_stack)) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to reverse call stack"); + goto error; + } + } + + // Create final CoroInfo result + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); + result = PyStructSequence_New(state->CoroInfo_Type); + if (result == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create CoroInfo"); + goto error; + } + + // PyStructSequence_SetItem steals references, so we don't need to DECREF on success + PyStructSequence_SetItem(result, 0, call_stack); // This steals the reference + PyStructSequence_SetItem(result, 1, tn); // This steals the reference + + return result; + +error: + Py_XDECREF(result); + Py_XDECREF(call_stack); + Py_XDECREF(tn); + return NULL; +} + +static int +parse_task( + RemoteUnwinderObject *unwinder, + uintptr_t task_address, + PyObject *render_to +) { + char is_task; + PyObject* result = NULL; + int err; + + err = read_char( + unwinder, + task_address + (uintptr_t)unwinder->async_debug_offsets.asyncio_task_object.task_is_task, + &is_task); + if (err) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read is_task flag"); + goto error; + } + + if (is_task) { + result = create_task_result(unwinder, task_address); + if (!result) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create task result"); + goto error; + } + } else { + // Create an empty CoroInfo for non-task objects + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); + result = PyStructSequence_New(state->CoroInfo_Type); + if (result == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create empty CoroInfo"); + goto error; + } + PyObject *empty_list = PyList_New(0); + if (empty_list == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create empty list"); + goto error; + } + PyObject *task_name = PyLong_FromUnsignedLongLong(task_address); + if (task_name == NULL) { + Py_DECREF(empty_list); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create task name"); + goto error; + } + PyStructSequence_SetItem(result, 0, empty_list); // This steals the reference + PyStructSequence_SetItem(result, 1, task_name); // This steals the reference + } + if (PyList_Append(render_to, result)) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append task result to render list"); + goto error; + } + + Py_DECREF(result); + return 0; + +error: + Py_XDECREF(result); + return -1; +} + +static int +process_single_task_node( + RemoteUnwinderObject *unwinder, + uintptr_t task_addr, + PyObject **task_info, + PyObject *result +) { + PyObject *tn = NULL; + PyObject *current_awaited_by = NULL; + PyObject *task_id = NULL; + PyObject *result_item = NULL; + PyObject *coroutine_stack = NULL; + + tn = parse_task_name(unwinder, task_addr); + if (tn == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse task name in single task node"); + goto error; + } + + current_awaited_by = PyList_New(0); + if (current_awaited_by == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create awaited_by list in single task node"); + goto error; + } + + // Extract the coroutine stack for this task + coroutine_stack = PyList_New(0); + if (coroutine_stack == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create coroutine stack list in single task node"); + goto error; + } + + if (parse_task(unwinder, task_addr, coroutine_stack) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse task coroutine stack in single task node"); + goto error; + } + + task_id = PyLong_FromUnsignedLongLong(task_addr); + if (task_id == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create task ID in single task node"); + goto error; + } + + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); + result_item = PyStructSequence_New(state->TaskInfo_Type); + if (result_item == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create TaskInfo in single task node"); + goto error; + } + + PyStructSequence_SetItem(result_item, 0, task_id); // steals ref + PyStructSequence_SetItem(result_item, 1, tn); // steals ref + PyStructSequence_SetItem(result_item, 2, coroutine_stack); // steals ref + PyStructSequence_SetItem(result_item, 3, current_awaited_by); // steals ref + + // References transferred to tuple + task_id = NULL; + tn = NULL; + coroutine_stack = NULL; + current_awaited_by = NULL; + + if (PyList_Append(result, result_item)) { + Py_DECREF(result_item); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append result item in single task node"); + return -1; + } + if (task_info != NULL) { + *task_info = result_item; + } + Py_DECREF(result_item); + + // Get back current_awaited_by reference for parse_task_awaited_by + current_awaited_by = PyStructSequence_GetItem(result_item, 3); + if (parse_task_awaited_by(unwinder, task_addr, current_awaited_by) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse awaited_by in single task node"); + // No cleanup needed here since all references were transferred to result_item + // and result_item was already added to result list and decreffed + return -1; + } + + return 0; + +error: + Py_XDECREF(tn); + Py_XDECREF(current_awaited_by); + Py_XDECREF(task_id); + Py_XDECREF(result_item); + Py_XDECREF(coroutine_stack); + return -1; +} + +// Thread processor for get_all_awaited_by +static int +process_thread_for_awaited_by( + RemoteUnwinderObject *unwinder, + uintptr_t thread_state_addr, + unsigned long tid, + void *context +) { + PyObject *result = (PyObject *)context; + uintptr_t head_addr = thread_state_addr + (uintptr_t)unwinder->async_debug_offsets.asyncio_thread_state.asyncio_tasks_head; + return append_awaited_by(unwinder, tid, head_addr, result); +} + +// Generic function to process task awaited_by +static int +process_task_awaited_by( + RemoteUnwinderObject *unwinder, + uintptr_t task_address, + set_entry_processor_func processor, + void *context +) { + // Read the entire TaskObj at once + char task_obj[SIZEOF_TASK_OBJ]; + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, task_address, + (size_t)unwinder->async_debug_offsets.asyncio_task_object.size, + task_obj) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task object"); + return -1; + } + + uintptr_t task_ab_addr = GET_MEMBER_NO_TAG(uintptr_t, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_awaited_by); + if ((void*)task_ab_addr == NULL) { + return 0; // No tasks waiting for this one + } + + char awaited_by_is_a_set = GET_MEMBER(char, task_obj, unwinder->async_debug_offsets.asyncio_task_object.task_awaited_by_is_set); + + if (awaited_by_is_a_set) { + return iterate_set_entries(unwinder, task_ab_addr, processor, context); + } else { + // Single task waiting + return processor(unwinder, task_ab_addr, context); + } +} static int -parse_coro_chain( - proc_handle_t *handle, - struct _Py_DebugOffsets* offsets, - struct _Py_AsyncioModuleDebugOffsets* async_offsets, - uintptr_t coro_address, - PyObject *render_to +process_running_task_chain( + RemoteUnwinderObject *unwinder, + uintptr_t running_task_addr, + uintptr_t thread_state_addr, + PyObject *result ) { - assert((void*)coro_address != NULL); + uintptr_t running_task_code_obj = 0; + if(get_task_code_object(unwinder, running_task_addr, &running_task_code_obj) < 0) { + return -1; + } - uintptr_t gen_type_addr; - int err = read_ptr( - handle, - coro_address + offsets->pyobject.ob_type, - &gen_type_addr); - if (err) { + // First, add this task to the result + PyObject *task_info = NULL; + if (process_single_task_node(unwinder, running_task_addr, &task_info, result) < 0) { return -1; } - PyObject* name = NULL; - uintptr_t prev_frame; - if (parse_frame_object( - handle, - &name, - offsets, - coro_address + offsets->gen_object.gi_iframe, - &prev_frame) - < 0) - { + // Get the chain from the current frame to this task + PyObject *coro_chain = PyStructSequence_GET_ITEM(task_info, 2); + assert(coro_chain != NULL); + if (PyList_GET_SIZE(coro_chain) != 1) { + PyErr_Format(PyExc_RuntimeError, + "Expected single-item coro chain, got %zd items", + PyList_GET_SIZE(coro_chain)); + set_exception_cause(unwinder, PyExc_RuntimeError, "Coro chain is not a single item"); return -1; } + PyObject *coro_info = PyList_GET_ITEM(coro_chain, 0); + assert(coro_info != NULL); + PyObject *frame_chain = PyStructSequence_GET_ITEM(coro_info, 0); + assert(frame_chain != NULL); - if (PyList_Append(render_to, name)) { - Py_DECREF(name); + // Clear the coro_chain + if (PyList_Clear(frame_chain) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to clear coroutine chain"); return -1; } - Py_DECREF(name); - int8_t gi_frame_state; - err = read_sized_int( - handle, - coro_address + offsets->gen_object.gi_frame_state, - &gi_frame_state, - sizeof(int8_t) - ); - if (err) { + // Add the chain from the current frame to this task + if (parse_async_frame_chain(unwinder, frame_chain, thread_state_addr, running_task_code_obj) < 0) { return -1; } - if (gi_frame_state == FRAME_SUSPENDED_YIELD_FROM) { - char owner; - err = read_char( - handle, - coro_address + offsets->gen_object.gi_iframe + - offsets->interpreter_frame.owner, - &owner - ); - if (err) { + // Now find all tasks that are waiting for this task and process them + if (process_task_awaited_by(unwinder, running_task_addr, process_waiter_task, result) < 0) { + return -1; + } + + return 0; +} + +// Thread processor for get_async_stack_trace +static int +process_thread_for_async_stack_trace( + RemoteUnwinderObject *unwinder, + uintptr_t thread_state_addr, + unsigned long tid, + void *context +) { + PyObject *result = (PyObject *)context; + + // Find running task in this thread + uintptr_t running_task_addr; + if (find_running_task_in_thread(unwinder, thread_state_addr, &running_task_addr) < 0) { + return 0; + } + + // If we found a running task, process it and its waiters + if ((void*)running_task_addr != NULL) { + PyObject *task_list = PyList_New(0); + if (task_list == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create task list for thread"); return -1; } - if (owner != FRAME_OWNED_BY_GENERATOR) { - PyErr_SetString( - PyExc_RuntimeError, - "generator doesn't own its frame \\_o_/"); + + if (process_running_task_chain(unwinder, running_task_addr, thread_state_addr, task_list) < 0) { + Py_DECREF(task_list); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process running task chain"); return -1; } - uintptr_t stackpointer_addr; - err = read_py_ptr( - handle, - coro_address + offsets->gen_object.gi_iframe + - offsets->interpreter_frame.stackpointer, - &stackpointer_addr); - if (err) { + // Create AwaitedInfo structure for this thread + PyObject *tid_py = PyLong_FromUnsignedLong(tid); + if (tid_py == NULL) { + Py_DECREF(task_list); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread ID"); return -1; } - if ((void*)stackpointer_addr != NULL) { - uintptr_t gi_await_addr; - err = read_py_ptr( - handle, - stackpointer_addr - sizeof(void*), - &gi_await_addr); - if (err) { - return -1; - } + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); + PyObject *awaited_info = PyStructSequence_New(state->AwaitedInfo_Type); + if (awaited_info == NULL) { + Py_DECREF(tid_py); + Py_DECREF(task_list); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create AwaitedInfo"); + return -1; + } - if ((void*)gi_await_addr != NULL) { - uintptr_t gi_await_addr_type_addr; - int err = read_ptr( - handle, - gi_await_addr + offsets->pyobject.ob_type, - &gi_await_addr_type_addr); - if (err) { - return -1; - } + PyStructSequence_SetItem(awaited_info, 0, tid_py); // steals ref + PyStructSequence_SetItem(awaited_info, 1, task_list); // steals ref - if (gen_type_addr == gi_await_addr_type_addr) { - /* This needs an explanation. We always start with parsing - native coroutine / generator frames. Ultimately they - are awaiting on something. That something can be - a native coroutine frame or... an iterator. - If it's the latter -- we can't continue building - our chain. So the condition to bail out of this is - to do that when the type of the current coroutine - doesn't match the type of whatever it points to - in its cr_await. - */ - err = parse_coro_chain( - handle, - offsets, - async_offsets, - gi_await_addr, - render_to - ); - if (err) { - return -1; - } - } - } + if (PyList_Append(result, awaited_info)) { + Py_DECREF(awaited_info); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append AwaitedInfo to result"); + return -1; } - + Py_DECREF(awaited_info); } return 0; } - -static int -parse_task_awaited_by( - proc_handle_t *handle, - struct _Py_DebugOffsets* offsets, - struct _Py_AsyncioModuleDebugOffsets* async_offsets, - uintptr_t task_address, - PyObject *awaited_by, - int recurse_task -); - - static int -parse_task( - proc_handle_t *handle, - struct _Py_DebugOffsets* offsets, - struct _Py_AsyncioModuleDebugOffsets* async_offsets, - uintptr_t task_address, - PyObject *render_to, - int recurse_task +process_task_and_waiters( + RemoteUnwinderObject *unwinder, + uintptr_t task_addr, + PyObject *result ) { - char is_task; - int err = read_char( - handle, - task_address + async_offsets->asyncio_task_object.task_is_task, - &is_task); - if (err) { - return -1; - } - - PyObject* result = PyList_New(0); - if (result == NULL) { + // First, add this task to the result + if (process_single_task_node(unwinder, task_addr, NULL, result) < 0) { return -1; } - PyObject *call_stack = PyList_New(0); - if (call_stack == NULL) { - goto err; - } - if (PyList_Append(result, call_stack)) { - Py_DECREF(call_stack); - goto err; - } - /* we can operate on a borrowed one to simplify cleanup */ - Py_DECREF(call_stack); - - if (is_task) { - PyObject *tn = NULL; - if (recurse_task) { - tn = parse_task_name( - handle, offsets, async_offsets, task_address); - } else { - tn = PyLong_FromUnsignedLongLong(task_address); - } - if (tn == NULL) { - goto err; - } - if (PyList_Append(result, tn)) { - Py_DECREF(tn); - goto err; - } - Py_DECREF(tn); - - uintptr_t coro_addr; - err = read_py_ptr( - handle, - task_address + async_offsets->asyncio_task_object.task_coro, - &coro_addr); - if (err) { - goto err; - } + // Now find all tasks that are waiting for this task and process them + return process_task_awaited_by(unwinder, task_addr, process_waiter_task, result); +} - if ((void*)coro_addr != NULL) { - err = parse_coro_chain( - handle, - offsets, - async_offsets, - coro_addr, - call_stack - ); - if (err) { - goto err; - } +static int +find_running_task_in_thread( + RemoteUnwinderObject *unwinder, + uintptr_t thread_state_addr, + uintptr_t *running_task_addr +) { + *running_task_addr = (uintptr_t)NULL; - if (PyList_Reverse(call_stack)) { - goto err; - } - } + uintptr_t address_of_running_loop; + int bytes_read = read_py_ptr( + unwinder, + thread_state_addr + (uintptr_t)unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_loop, + &address_of_running_loop); + if (bytes_read == -1) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running loop address"); + return -1; } - if (PyList_Append(render_to, result)) { - goto err; + // no asyncio loop is now running + if ((void*)address_of_running_loop == NULL) { + return 0; } - if (recurse_task) { - PyObject *awaited_by = PyList_New(0); - if (awaited_by == NULL) { - goto err; - } - if (PyList_Append(result, awaited_by)) { - Py_DECREF(awaited_by); - goto err; - } - /* we can operate on a borrowed one to simplify cleanup */ - Py_DECREF(awaited_by); - - if (parse_task_awaited_by(handle, offsets, async_offsets, - task_address, awaited_by, 1) - ) { - goto err; - } + int err = read_ptr( + unwinder, + thread_state_addr + (uintptr_t)unwinder->async_debug_offsets.asyncio_thread_state.asyncio_running_task, + running_task_addr); + if (err) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running task address"); + return -1; } - Py_DECREF(result); return 0; - -err: - Py_DECREF(result); - return -1; } static int -parse_tasks_in_set( - proc_handle_t *handle, - struct _Py_DebugOffsets* offsets, - struct _Py_AsyncioModuleDebugOffsets* async_offsets, - uintptr_t set_addr, - PyObject *awaited_by, - int recurse_task -) { - uintptr_t set_obj; - if (read_py_ptr( - handle, - set_addr, - &set_obj) - ) { +get_task_code_object(RemoteUnwinderObject *unwinder, uintptr_t task_addr, uintptr_t *code_obj_addr) { + uintptr_t running_coro_addr = 0; + + if(read_py_ptr( + unwinder, + task_addr + (uintptr_t)unwinder->async_debug_offsets.asyncio_task_object.task_coro, + &running_coro_addr) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Running task coro read failed"); return -1; } - Py_ssize_t num_els; - if (read_Py_ssize_t( - handle, - set_obj + offsets->set_object.used, - &num_els) - ) { + if (running_coro_addr == 0) { + PyErr_SetString(PyExc_RuntimeError, "Running task coro is NULL"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Running task coro address is NULL"); return -1; } - Py_ssize_t set_len; - if (read_Py_ssize_t( - handle, - set_obj + offsets->set_object.mask, - &set_len) - ) { + // note: genobject's gi_iframe is an embedded struct so the address to + // the offset leads directly to its first field: f_executable + if (read_py_ptr( + unwinder, + running_coro_addr + (uintptr_t)unwinder->debug_offsets.gen_object.gi_iframe, code_obj_addr) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read running task code object"); return -1; } - set_len++; // The set contains the `mask+1` element slots. - uintptr_t table_ptr; - if (read_ptr( - handle, - set_obj + offsets->set_object.table, - &table_ptr) - ) { + if (*code_obj_addr == 0) { + PyErr_SetString(PyExc_RuntimeError, "Running task code object is NULL"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Running task code object address is NULL"); return -1; } - Py_ssize_t i = 0; - Py_ssize_t els = 0; - while (i < set_len) { - uintptr_t key_addr; - if (read_py_ptr(handle, table_ptr, &key_addr)) { - return -1; - } + return 0; +} - if ((void*)key_addr != NULL) { - Py_ssize_t ref_cnt; - if (read_Py_ssize_t(handle, table_ptr, &ref_cnt)) { - return -1; - } +/* ============================================================================ + * TLBC CACHING FUNCTIONS + * ============================================================================ */ - if (ref_cnt) { - // if 'ref_cnt=0' it's a set dummy marker - - if (parse_task( - handle, - offsets, - async_offsets, - key_addr, - awaited_by, - recurse_task - ) - ) { - return -1; - } +#ifdef Py_GIL_DISABLED - if (++els == num_els) { - break; - } - } - } +typedef struct { + void *tlbc_array; // Local copy of the TLBC array + Py_ssize_t tlbc_array_size; // Size of the TLBC array + uint32_t generation; // Generation when this was cached +} TLBCCacheEntry; - table_ptr += sizeof(void*) * 2; - i++; +static void +tlbc_cache_entry_destroy(void *ptr) +{ + TLBCCacheEntry *entry = (TLBCCacheEntry *)ptr; + if (entry->tlbc_array) { + PyMem_RawFree(entry->tlbc_array); } - return 0; + PyMem_RawFree(entry); } +static TLBCCacheEntry * +get_tlbc_cache_entry(RemoteUnwinderObject *self, uintptr_t code_addr, uint32_t current_generation) +{ + void *key = (void *)code_addr; + TLBCCacheEntry *entry = _Py_hashtable_get(self->tlbc_cache, key); + + if (entry && entry->generation != current_generation) { + // Entry is stale, remove it by setting to NULL + _Py_hashtable_set(self->tlbc_cache, key, NULL); + entry = NULL; + } + + return entry; +} static int -parse_task_awaited_by( - proc_handle_t *handle, - struct _Py_DebugOffsets* offsets, - struct _Py_AsyncioModuleDebugOffsets* async_offsets, - uintptr_t task_address, - PyObject *awaited_by, - int recurse_task -) { - uintptr_t task_ab_addr; - int err = read_py_ptr( - handle, - task_address + async_offsets->asyncio_task_object.task_awaited_by, - &task_ab_addr); - if (err) { - return -1; +cache_tlbc_array(RemoteUnwinderObject *unwinder, uintptr_t code_addr, uintptr_t tlbc_array_addr, uint32_t generation) +{ + uintptr_t tlbc_array_ptr; + void *tlbc_array = NULL; + TLBCCacheEntry *entry = NULL; + + // Read the TLBC array pointer + if (read_ptr(unwinder, tlbc_array_addr, &tlbc_array_ptr) != 0 || tlbc_array_ptr == 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read TLBC array pointer"); + return 0; // No TLBC array } - if ((void*)task_ab_addr == NULL) { - return 0; + // Read the TLBC array size + Py_ssize_t tlbc_size; + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, tlbc_array_ptr, sizeof(tlbc_size), &tlbc_size) != 0 || tlbc_size <= 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read TLBC array size"); + return 0; // Invalid size } - char awaited_by_is_a_set; - err = read_char( - handle, - task_address + async_offsets->asyncio_task_object.task_awaited_by_is_set, - &awaited_by_is_a_set); - if (err) { - return -1; + // Allocate and read the entire TLBC array + size_t array_data_size = tlbc_size * sizeof(void*); + tlbc_array = PyMem_RawMalloc(sizeof(Py_ssize_t) + array_data_size); + if (!tlbc_array) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate TLBC array"); + return 0; // Memory error } - if (awaited_by_is_a_set) { - if (parse_tasks_in_set( - handle, - offsets, - async_offsets, - task_address + async_offsets->asyncio_task_object.task_awaited_by, - awaited_by, - recurse_task - ) - ) { - return -1; - } - } else { - uintptr_t sub_task; - if (read_py_ptr( - handle, - task_address + async_offsets->asyncio_task_object.task_awaited_by, - &sub_task) - ) { - return -1; - } + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, tlbc_array_ptr, sizeof(Py_ssize_t) + array_data_size, tlbc_array) != 0) { + PyMem_RawFree(tlbc_array); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read TLBC array data"); + return 0; // Read error + } - if (parse_task( - handle, - offsets, - async_offsets, - sub_task, - awaited_by, - recurse_task - ) - ) { - return -1; - } + // Create cache entry + entry = PyMem_RawMalloc(sizeof(TLBCCacheEntry)); + if (!entry) { + PyMem_RawFree(tlbc_array); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate TLBC cache entry"); + return 0; // Memory error } - return 0; + entry->tlbc_array = tlbc_array; + entry->tlbc_array_size = tlbc_size; + entry->generation = generation; + + // Store in cache + void *key = (void *)code_addr; + if (_Py_hashtable_set(unwinder->tlbc_cache, key, entry) < 0) { + tlbc_cache_entry_destroy(entry); + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to store TLBC entry in cache"); + return 0; // Cache error + } + + return 1; // Success } -typedef struct -{ - int lineno; - int end_lineno; - int column; - int end_column; -} LocationInfo; + + +#endif + +/* ============================================================================ + * LINE TABLE PARSING FUNCTIONS + * ============================================================================ */ static int scan_varint(const uint8_t **ptr) @@ -818,12 +2170,11 @@ scan_signed_varint(const uint8_t **ptr) } } - static bool parse_linetable(const uintptr_t addrq, const char* linetable, int firstlineno, LocationInfo* info) { const uint8_t* ptr = (const uint8_t*)(linetable); - uint64_t addr = 0; + uintptr_t addr = 0; info->lineno = firstlineno; while (*ptr != '\0') { @@ -863,7 +2214,9 @@ parse_linetable(const uintptr_t addrq, const char* linetable, int firstlineno, L } default: { uint8_t second_byte = *(ptr++); - assert((second_byte & 128) == 0); + if ((second_byte & 128) != 0) { + return false; + } info->column = code << 3 | (second_byte >> 4); info->end_column = info->column + (second_byte & 15); break; @@ -877,270 +2230,439 @@ parse_linetable(const uintptr_t addrq, const char* linetable, int firstlineno, L return false; } +/* ============================================================================ + * CODE OBJECT AND FRAME PARSING FUNCTIONS + * ============================================================================ */ + static int -read_remote_pointer(proc_handle_t *handle, uintptr_t address, uintptr_t *out_ptr, const char *error_message) +parse_code_object(RemoteUnwinderObject *unwinder, + PyObject **result, + uintptr_t address, + uintptr_t instruction_pointer, + uintptr_t *previous_frame, + int32_t tlbc_index) { - int bytes_read = _Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(void *), out_ptr); - if (bytes_read < 0) { - return -1; - } + void *key = (void *)address; + CachedCodeMetadata *meta = NULL; + PyObject *func = NULL; + PyObject *file = NULL; + PyObject *linetable = NULL; + PyObject *lineno = NULL; + PyObject *tuple = NULL; - if ((void *)(*out_ptr) == NULL) { - PyErr_SetString(PyExc_RuntimeError, error_message); - return -1; +#ifdef Py_GIL_DISABLED + // In free threading builds, code object addresses might have the low bit set + // as a flag, so we need to mask it off to get the real address + uintptr_t real_address = address & (~1); +#else + uintptr_t real_address = address; +#endif + + if (unwinder && unwinder->code_object_cache != NULL) { + meta = _Py_hashtable_get(unwinder->code_object_cache, key); } - return 0; -} + if (meta == NULL) { + char code_object[SIZEOF_CODE_OBJ]; + if (_Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, real_address, SIZEOF_CODE_OBJ, code_object) < 0) + { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read code object"); + goto error; + } -static int -read_instruction_ptr(proc_handle_t *handle, struct _Py_DebugOffsets *offsets, - uintptr_t current_frame, uintptr_t *instruction_ptr) -{ - return read_remote_pointer( - handle, - current_frame + offsets->interpreter_frame.instr_ptr, - instruction_ptr, - "No instruction ptr found" - ); -} + func = read_py_str(unwinder, + GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.code_object.qualname), 1024); + if (!func) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read function name from code object"); + goto error; + } -static int -parse_code_object(proc_handle_t *handle, - PyObject **result, - struct _Py_DebugOffsets *offsets, - uintptr_t address, - uintptr_t current_frame, - uintptr_t *previous_frame) -{ - uintptr_t addr_func_name, addr_file_name, addr_linetable, instruction_ptr; + file = read_py_str(unwinder, + GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.code_object.filename), 1024); + if (!file) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read filename from code object"); + goto error; + } - if (read_remote_pointer(handle, address + offsets->code_object.qualname, &addr_func_name, "No function name found") < 0 || - read_remote_pointer(handle, address + offsets->code_object.filename, &addr_file_name, "No file name found") < 0 || - read_remote_pointer(handle, address + offsets->code_object.linetable, &addr_linetable, "No linetable found") < 0 || - read_instruction_ptr(handle, offsets, current_frame, &instruction_ptr) < 0) { - return -1; - } + linetable = read_py_bytes(unwinder, + GET_MEMBER(uintptr_t, code_object, unwinder->debug_offsets.code_object.linetable), 4096); + if (!linetable) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read linetable from code object"); + goto error; + } - int firstlineno; - if (_Py_RemoteDebug_ReadRemoteMemory(handle, - address + offsets->code_object.firstlineno, - sizeof(int), - &firstlineno) < 0) { - return -1; + meta = PyMem_RawMalloc(sizeof(CachedCodeMetadata)); + if (!meta) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate cached code metadata"); + goto error; + } + + meta->func_name = func; + meta->file_name = file; + meta->linetable = linetable; + meta->first_lineno = GET_MEMBER(int, code_object, unwinder->debug_offsets.code_object.firstlineno); + meta->addr_code_adaptive = real_address + (uintptr_t)unwinder->debug_offsets.code_object.co_code_adaptive; + + if (unwinder && unwinder->code_object_cache && _Py_hashtable_set(unwinder->code_object_cache, key, meta) < 0) { + func = NULL; + file = NULL; + linetable = NULL; + cached_code_metadata_destroy(meta); + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to cache code metadata"); + goto error; + } + + // Ownership transferred to meta + func = NULL; + file = NULL; + linetable = NULL; } - PyObject *py_linetable = read_py_bytes(handle, offsets, addr_linetable); - if (!py_linetable) { - return -1; + uintptr_t ip = instruction_pointer; + ptrdiff_t addrq; + +#ifdef Py_GIL_DISABLED + // Handle thread-local bytecode (TLBC) in free threading builds + if (tlbc_index == 0 || unwinder->debug_offsets.code_object.co_tlbc == 0 || unwinder == NULL) { + // No TLBC or no unwinder - use main bytecode directly + addrq = (uint16_t *)ip - (uint16_t *)meta->addr_code_adaptive; + goto done_tlbc; } - uintptr_t addr_code_adaptive = address + offsets->code_object.co_code_adaptive; - ptrdiff_t addrq = (uint16_t *)instruction_ptr - (uint16_t *)addr_code_adaptive; + // Try to get TLBC data from cache (we'll get generation from the caller) + TLBCCacheEntry *tlbc_entry = get_tlbc_cache_entry(unwinder, real_address, unwinder->tlbc_generation); - LocationInfo info; - parse_linetable(addrq, PyBytes_AS_STRING(py_linetable), firstlineno, &info); - Py_DECREF(py_linetable); // Done with linetable + if (!tlbc_entry) { + // Cache miss - try to read and cache TLBC array + if (!cache_tlbc_array(unwinder, real_address, real_address + unwinder->debug_offsets.code_object.co_tlbc, unwinder->tlbc_generation)) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to cache TLBC array"); + goto error; + } + tlbc_entry = get_tlbc_cache_entry(unwinder, real_address, unwinder->tlbc_generation); + } - PyObject *py_line = PyLong_FromLong(info.lineno); - if (!py_line) { - return -1; + if (tlbc_entry) { + if (tlbc_index < 0 || tlbc_index >= tlbc_entry->tlbc_array_size) { + PyErr_Format(PyExc_RuntimeError, + "Invalid tlbc_index %d (array size %zd, corrupted remote memory)", + tlbc_index, tlbc_entry->tlbc_array_size); + set_exception_cause(unwinder, PyExc_RuntimeError, + "Invalid tlbc_index (corrupted remote memory)"); + goto error; + } + // Use cached TLBC data + uintptr_t *entries = (uintptr_t *)((char *)tlbc_entry->tlbc_array + sizeof(Py_ssize_t)); + uintptr_t tlbc_bytecode_addr = entries[tlbc_index]; + + if (tlbc_bytecode_addr != 0) { + // Calculate offset from TLBC bytecode + addrq = (uint16_t *)ip - (uint16_t *)tlbc_bytecode_addr; + goto done_tlbc; + } } - PyObject *py_func_name = read_py_str(handle, offsets, addr_func_name, 256); - if (!py_func_name) { - Py_DECREF(py_line); - return -1; + // Fall back to main bytecode + addrq = (uint16_t *)ip - (uint16_t *)meta->addr_code_adaptive; + +done_tlbc: +#else + // Non-free-threaded build, always use the main bytecode + (void)tlbc_index; // Suppress unused parameter warning + (void)unwinder; // Suppress unused parameter warning + addrq = (uint16_t *)ip - (uint16_t *)meta->addr_code_adaptive; +#endif + ; // Empty statement to avoid C23 extension warning + LocationInfo info = {0}; + bool ok = parse_linetable(addrq, PyBytes_AS_STRING(meta->linetable), + meta->first_lineno, &info); + if (!ok) { + info.lineno = -1; } - PyObject *py_file_name = read_py_str(handle, offsets, addr_file_name, 256); - if (!py_file_name) { - Py_DECREF(py_line); - Py_DECREF(py_func_name); - return -1; + lineno = PyLong_FromLong(info.lineno); + if (!lineno) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create line number object"); + goto error; } - PyObject *result_tuple = PyTuple_New(3); - if (!result_tuple) { - Py_DECREF(py_line); - Py_DECREF(py_func_name); - Py_DECREF(py_file_name); - return -1; + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); + tuple = PyStructSequence_New(state->FrameInfo_Type); + if (!tuple) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create FrameInfo for code object"); + goto error; } - PyTuple_SET_ITEM(result_tuple, 0, py_func_name); // steals ref - PyTuple_SET_ITEM(result_tuple, 1, py_file_name); // steals ref - PyTuple_SET_ITEM(result_tuple, 2, py_line); // steals ref + Py_INCREF(meta->func_name); + Py_INCREF(meta->file_name); + PyStructSequence_SetItem(tuple, 0, meta->file_name); + PyStructSequence_SetItem(tuple, 1, lineno); + PyStructSequence_SetItem(tuple, 2, meta->func_name); - *result = result_tuple; + *result = tuple; return 0; + +error: + Py_XDECREF(func); + Py_XDECREF(file); + Py_XDECREF(linetable); + Py_XDECREF(lineno); + Py_XDECREF(tuple); + return -1; +} + +/* ============================================================================ + * STACK CHUNK MANAGEMENT FUNCTIONS + * ============================================================================ */ + +static void +cleanup_stack_chunks(StackChunkList *chunks) +{ + for (size_t i = 0; i < chunks->count; ++i) { + PyMem_RawFree(chunks->chunks[i].local_copy); + } + PyMem_RawFree(chunks->chunks); } static int -parse_frame_object( - proc_handle_t *handle, - PyObject** result, - struct _Py_DebugOffsets* offsets, - uintptr_t address, - uintptr_t* previous_frame +process_single_stack_chunk( + RemoteUnwinderObject *unwinder, + uintptr_t chunk_addr, + StackChunkInfo *chunk_info ) { - int err; + // Start with default size assumption + size_t current_size = _PY_DATA_STACK_CHUNK_SIZE; - Py_ssize_t bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, - address + offsets->interpreter_frame.previous, - sizeof(void*), - previous_frame - ); - if (bytes_read < 0) { + char *this_chunk = PyMem_RawMalloc(current_size); + if (!this_chunk) { + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate stack chunk buffer"); return -1; } - char owner; - if (read_char(handle, address + offsets->interpreter_frame.owner, &owner)) { + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, chunk_addr, current_size, this_chunk) < 0) { + PyMem_RawFree(this_chunk); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read stack chunk"); return -1; } - if (owner >= FRAME_OWNED_BY_INTERPRETER) { - return 0; - } + // Check actual size and reread if necessary + size_t actual_size = GET_MEMBER(size_t, this_chunk, offsetof(_PyStackChunk, size)); + if (actual_size != current_size) { + if (actual_size <= offsetof(_PyStackChunk, data) || actual_size > MAX_STACK_CHUNK_SIZE) { + PyMem_RawFree(this_chunk); + PyErr_Format(PyExc_RuntimeError, + "Invalid stack chunk size %zu (corrupted remote memory)", actual_size); + set_exception_cause(unwinder, PyExc_RuntimeError, + "Invalid stack chunk size (corrupted remote memory)"); + return -1; + } - uintptr_t address_of_code_object; - err = read_py_ptr( - handle, - address + offsets->interpreter_frame.executable, - &address_of_code_object - ); - if (err) { - return -1; - } + char *tmp = PyMem_RawRealloc(this_chunk, actual_size); + if (!tmp) { + PyMem_RawFree(this_chunk); + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to reallocate stack chunk buffer"); + return -1; + } + this_chunk = tmp; - if ((void*)address_of_code_object == NULL) { - return 0; + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, chunk_addr, actual_size, this_chunk) < 0) { + PyMem_RawFree(this_chunk); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to reread stack chunk with correct size"); + return -1; + } + current_size = actual_size; } - return parse_code_object( - handle, result, offsets, address_of_code_object, address, previous_frame); + chunk_info->remote_addr = chunk_addr; + chunk_info->size = current_size; + chunk_info->local_copy = this_chunk; + return 0; } static int -parse_async_frame_object( - proc_handle_t *handle, - PyObject** result, - struct _Py_DebugOffsets* offsets, - uintptr_t address, - uintptr_t* previous_frame, - uintptr_t* code_object -) { - int err; +copy_stack_chunks(RemoteUnwinderObject *unwinder, + uintptr_t tstate_addr, + StackChunkList *out_chunks) +{ + uintptr_t chunk_addr; + StackChunkInfo *chunks = NULL; + size_t count = 0; + size_t max_chunks = 16; - Py_ssize_t bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, - address + offsets->interpreter_frame.previous, - sizeof(void*), - previous_frame - ); - if (bytes_read < 0) { + if (read_ptr(unwinder, tstate_addr + (uintptr_t)unwinder->debug_offsets.thread_state.datastack_chunk, &chunk_addr)) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read initial stack chunk address"); return -1; } - char owner; - bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, address + offsets->interpreter_frame.owner, sizeof(char), &owner); - if (bytes_read < 0) { + chunks = PyMem_RawMalloc(max_chunks * sizeof(StackChunkInfo)); + if (!chunks) { + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to allocate stack chunks array"); return -1; } - if (owner == FRAME_OWNED_BY_CSTACK || owner == FRAME_OWNED_BY_INTERPRETER) { - return 0; // C frame - } + while (chunk_addr != 0) { + // Grow array if needed + if (count >= max_chunks) { + max_chunks *= 2; + StackChunkInfo *new_chunks = PyMem_RawRealloc(chunks, max_chunks * sizeof(StackChunkInfo)); + if (!new_chunks) { + PyErr_NoMemory(); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to grow stack chunks array"); + goto error; + } + chunks = new_chunks; + } - if (owner != FRAME_OWNED_BY_GENERATOR - && owner != FRAME_OWNED_BY_THREAD) { - PyErr_Format(PyExc_RuntimeError, "Unhandled frame owner %d.\n", owner); - return -1; - } + // Process this chunk + if (process_single_stack_chunk(unwinder, chunk_addr, &chunks[count]) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process stack chunk"); + goto error; + } - err = read_py_ptr( - handle, - address + offsets->interpreter_frame.executable, - code_object - ); - if (err) { - return -1; + // Get next chunk address and increment count + chunk_addr = GET_MEMBER(uintptr_t, chunks[count].local_copy, offsetof(_PyStackChunk, previous)); + count++; } - assert(code_object != NULL); - if ((void*)*code_object == NULL) { - return 0; - } + out_chunks->chunks = chunks; + out_chunks->count = count; + return 0; - if (parse_code_object( - handle, result, offsets, *code_object, address, previous_frame)) { - return -1; +error: + for (size_t i = 0; i < count; ++i) { + PyMem_RawFree(chunks[i].local_copy); } + PyMem_RawFree(chunks); + return -1; +} - return 1; +static void * +find_frame_in_chunks(StackChunkList *chunks, uintptr_t remote_ptr) +{ + for (size_t i = 0; i < chunks->count; ++i) { + if (chunks->chunks[i].size <= offsetof(_PyStackChunk, data)) { + continue; + } + uintptr_t base = chunks->chunks[i].remote_addr + offsetof(_PyStackChunk, data); + size_t payload = chunks->chunks[i].size - offsetof(_PyStackChunk, data); + + if (payload >= SIZEOF_INTERP_FRAME && + remote_ptr >= base && + remote_ptr <= base + payload - SIZEOF_INTERP_FRAME) { + return (char *)chunks->chunks[i].local_copy + (remote_ptr - chunks->chunks[i].remote_addr); + } + } + return NULL; } static int -read_async_debug( - proc_handle_t *handle, - struct _Py_AsyncioModuleDebugOffsets* async_debug +parse_frame_from_chunks( + RemoteUnwinderObject *unwinder, + PyObject **result, + uintptr_t address, + uintptr_t *previous_frame, + StackChunkList *chunks ) { - uintptr_t async_debug_addr = _Py_RemoteDebug_GetAsyncioDebugAddress(handle); - if (!async_debug_addr) { + void *frame_ptr = find_frame_in_chunks(chunks, address); + if (!frame_ptr) { + PyErr_Format(PyExc_RuntimeError, "Frame at address 0x%lx not found in stack chunks", address); + set_exception_cause(unwinder, PyExc_RuntimeError, "Frame not found in stack chunks"); return -1; } - size_t size = sizeof(struct _Py_AsyncioModuleDebugOffsets); - int result = _Py_RemoteDebug_ReadRemoteMemory(handle, async_debug_addr, size, async_debug); - return result; + char *frame = (char *)frame_ptr; + *previous_frame = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.previous); + uintptr_t code_object = GET_MEMBER_NO_TAG(uintptr_t, frame_ptr, unwinder->debug_offsets.interpreter_frame.executable); + int frame_valid = is_frame_valid(unwinder, (uintptr_t)frame, code_object); + if (frame_valid != 1) { + return frame_valid; + } + + uintptr_t instruction_pointer = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.instr_ptr); + + // Get tlbc_index for free threading builds + int32_t tlbc_index = 0; +#ifdef Py_GIL_DISABLED + if (unwinder->debug_offsets.interpreter_frame.tlbc_index != 0) { + tlbc_index = GET_MEMBER(int32_t, frame, unwinder->debug_offsets.interpreter_frame.tlbc_index); + } +#endif + + return parse_code_object(unwinder, result, code_object, instruction_pointer, previous_frame, tlbc_index); } +/* ============================================================================ + * INTERPRETER STATE AND THREAD DISCOVERY FUNCTIONS + * ============================================================================ */ + static int -find_running_frame( - proc_handle_t *handle, +populate_initial_state_data( + int all_threads, + RemoteUnwinderObject *unwinder, uintptr_t runtime_start_address, - _Py_DebugOffsets* local_debug_offsets, - uintptr_t *frame + uintptr_t *interpreter_state, + uintptr_t *tstate ) { - uint64_t interpreter_state_list_head = - local_debug_offsets->runtime_state.interpreters_head; + uintptr_t interpreter_state_list_head = + (uintptr_t)unwinder->debug_offsets.runtime_state.interpreters_head; uintptr_t address_of_interpreter_state; - int bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, + int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, runtime_start_address + interpreter_state_list_head, sizeof(void*), &address_of_interpreter_state); if (bytes_read < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read interpreter state address"); return -1; } if (address_of_interpreter_state == 0) { PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Interpreter state is NULL"); return -1; } - uintptr_t address_of_thread; - bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, - address_of_interpreter_state + - local_debug_offsets->interpreter_state.threads_main, + *interpreter_state = address_of_interpreter_state; + + if (all_threads) { + *tstate = 0; + return 0; + } + + uintptr_t address_of_thread = address_of_interpreter_state + + (uintptr_t)unwinder->debug_offsets.interpreter_state.threads_main; + + if (_Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + address_of_thread, sizeof(void*), - &address_of_thread); - if (bytes_read < 0) { + tstate) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read main thread state address"); return -1; } - // No Python frames are available for us (can happen at tear-down). + return 0; +} + + +static int +find_running_frame( + RemoteUnwinderObject *unwinder, + uintptr_t address_of_thread, + uintptr_t *frame +) { if ((void*)address_of_thread != NULL) { int err = read_ptr( - handle, - address_of_thread + local_debug_offsets->thread_state.current_frame, + unwinder, + address_of_thread + (uintptr_t)unwinder->debug_offsets.thread_state.current_frame, frame); if (err) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read current frame pointer"); return -1; } return 0; @@ -1150,651 +2672,995 @@ find_running_frame( return 0; } -static int -find_running_task( - proc_handle_t *handle, - uintptr_t runtime_start_address, - _Py_DebugOffsets *local_debug_offsets, - struct _Py_AsyncioModuleDebugOffsets *async_offsets, - uintptr_t *running_task_addr +/* ============================================================================ + * FRAME PARSING FUNCTIONS + * ============================================================================ */ + +static inline int +is_frame_valid( + RemoteUnwinderObject *unwinder, + uintptr_t frame_addr, + uintptr_t code_object_addr ) { - *running_task_addr = (uintptr_t)NULL; + if ((void*)code_object_addr == NULL) { + return 0; + } - uint64_t interpreter_state_list_head = - local_debug_offsets->runtime_state.interpreters_head; + void* frame = (void*)frame_addr; - uintptr_t address_of_interpreter_state; - int bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, - runtime_start_address + interpreter_state_list_head, - sizeof(void*), - &address_of_interpreter_state); - if (bytes_read < 0) { - return -1; + if (GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) == FRAME_OWNED_BY_CSTACK || + GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) == FRAME_OWNED_BY_INTERPRETER) { + return 0; // C frame } - if (address_of_interpreter_state == 0) { - PyErr_SetString(PyExc_RuntimeError, "No interpreter state found"); + if (GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) != FRAME_OWNED_BY_GENERATOR + && GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner) != FRAME_OWNED_BY_THREAD) { + PyErr_Format(PyExc_RuntimeError, "Unhandled frame owner %d.\n", + GET_MEMBER(char, frame, unwinder->debug_offsets.interpreter_frame.owner)); + set_exception_cause(unwinder, PyExc_RuntimeError, "Unhandled frame owner type in async frame"); return -1; } + return 1; +} - uintptr_t address_of_thread; - bytes_read = _Py_RemoteDebug_ReadRemoteMemory( - handle, - address_of_interpreter_state + - local_debug_offsets->interpreter_state.threads_head, - sizeof(void*), - &address_of_thread); +static int +parse_frame_object( + RemoteUnwinderObject *unwinder, + PyObject** result, + uintptr_t address, + uintptr_t* address_of_code_object, + uintptr_t* previous_frame +) { + char frame[SIZEOF_INTERP_FRAME]; + *address_of_code_object = 0; + + Py_ssize_t bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + address, + SIZEOF_INTERP_FRAME, + frame + ); if (bytes_read < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read interpreter frame"); return -1; } - uintptr_t address_of_running_loop; - // No Python frames are available for us (can happen at tear-down). - if ((void*)address_of_thread == NULL) { - return 0; + *previous_frame = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.previous); + uintptr_t code_object = GET_MEMBER_NO_TAG(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.executable); + int frame_valid = is_frame_valid(unwinder, (uintptr_t)frame, code_object); + if (frame_valid != 1) { + return frame_valid; } - bytes_read = read_py_ptr( - handle, - address_of_thread - + async_offsets->asyncio_thread_state.asyncio_running_loop, - &address_of_running_loop); - if (bytes_read == -1) { - return -1; - } + uintptr_t instruction_pointer = GET_MEMBER(uintptr_t, frame, unwinder->debug_offsets.interpreter_frame.instr_ptr); - // no asyncio loop is now running - if ((void*)address_of_running_loop == NULL) { - return 0; + // Get tlbc_index for free threading builds + int32_t tlbc_index = 0; +#ifdef Py_GIL_DISABLED + if (unwinder->debug_offsets.interpreter_frame.tlbc_index != 0) { + tlbc_index = GET_MEMBER(int32_t, frame, unwinder->debug_offsets.interpreter_frame.tlbc_index); } +#endif - int err = read_ptr( - handle, - address_of_thread - + async_offsets->asyncio_thread_state.asyncio_running_task, - running_task_addr); - if (err) { + *address_of_code_object = code_object; + return parse_code_object(unwinder, result, code_object, instruction_pointer, previous_frame, tlbc_index); +} + +static int + parse_async_frame_chain( + RemoteUnwinderObject *unwinder, + PyObject *calls, + uintptr_t address_of_thread, + uintptr_t running_task_code_obj +) { + uintptr_t address_of_current_frame; + if (find_running_frame(unwinder, address_of_thread, &address_of_current_frame) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Running frame search failed in async chain"); return -1; } + while ((void*)address_of_current_frame != NULL) { + PyObject* frame_info = NULL; + uintptr_t address_of_code_object; + int res = parse_frame_object( + unwinder, + &frame_info, + address_of_current_frame, + &address_of_code_object, + &address_of_current_frame + ); + + if (res < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Async frame object parsing failed in chain"); + return -1; + } + + if (!frame_info) { + continue; + } + + if (PyList_Append(calls, frame_info) == -1) { + Py_DECREF(frame_info); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append frame info to async chain"); + return -1; + } + + Py_DECREF(frame_info); + + if (address_of_code_object == running_task_code_obj) { + break; + } + } + return 0; } +/* ============================================================================ + * AWAITED BY PARSING FUNCTIONS + * ============================================================================ */ + static int append_awaited_by_for_thread( - proc_handle_t *handle, + RemoteUnwinderObject *unwinder, uintptr_t head_addr, - struct _Py_DebugOffsets *debug_offsets, - struct _Py_AsyncioModuleDebugOffsets *async_offsets, PyObject *result ) { - struct llist_node task_node; + char task_node[SIZEOF_LLIST_NODE]; - if (0 > _Py_RemoteDebug_ReadRemoteMemory( - handle, - head_addr, - sizeof(task_node), - &task_node)) - { + if (_Py_RemoteDebug_PagedReadRemoteMemory(&unwinder->handle, head_addr, + sizeof(task_node), task_node) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read task node head"); return -1; } size_t iteration_count = 0; const size_t MAX_ITERATIONS = 2 << 15; // A reasonable upper bound - while ((uintptr_t)task_node.next != head_addr) { + + while (GET_MEMBER(uintptr_t, task_node, unwinder->debug_offsets.llist_node.next) != head_addr) { if (++iteration_count > MAX_ITERATIONS) { PyErr_SetString(PyExc_RuntimeError, "Task list appears corrupted"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Task list iteration limit exceeded"); return -1; } - if (task_node.next == NULL) { - PyErr_SetString( - PyExc_RuntimeError, - "Invalid linked list structure reading remote memory"); + if (GET_MEMBER(uintptr_t, task_node, unwinder->debug_offsets.llist_node.next) == 0) { + PyErr_SetString(PyExc_RuntimeError, + "Invalid linked list structure reading remote memory"); + set_exception_cause(unwinder, PyExc_RuntimeError, "NULL pointer in task linked list"); return -1; } - uintptr_t task_addr = (uintptr_t)task_node.next - - async_offsets->asyncio_task_object.task_node; + uintptr_t task_addr = (uintptr_t)GET_MEMBER(uintptr_t, task_node, unwinder->debug_offsets.llist_node.next) + - (uintptr_t)unwinder->async_debug_offsets.asyncio_task_object.task_node; - PyObject *tn = parse_task_name( - handle, - debug_offsets, - async_offsets, - task_addr); - if (tn == NULL) { + if (process_single_task_node(unwinder, task_addr, NULL, result) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process task node in awaited_by"); return -1; } - PyObject *current_awaited_by = PyList_New(0); - if (current_awaited_by == NULL) { - Py_DECREF(tn); + // Read next node + if (_Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, + (uintptr_t)GET_MEMBER(uintptr_t, task_node, unwinder->debug_offsets.llist_node.next), + sizeof(task_node), + task_node) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read next task node in awaited_by"); return -1; } + } + + return 0; +} + +static int +append_awaited_by( + RemoteUnwinderObject *unwinder, + unsigned long tid, + uintptr_t head_addr, + PyObject *result) +{ + PyObject *tid_py = PyLong_FromUnsignedLong(tid); + if (tid_py == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread ID object"); + return -1; + } + + PyObject* awaited_by_for_thread = PyList_New(0); + if (awaited_by_for_thread == NULL) { + Py_DECREF(tid_py); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create awaited_by thread list"); + return -1; + } + + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); + PyObject *result_item = PyStructSequence_New(state->AwaitedInfo_Type); + if (result_item == NULL) { + Py_DECREF(tid_py); + Py_DECREF(awaited_by_for_thread); + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create AwaitedInfo"); + return -1; + } + + PyStructSequence_SetItem(result_item, 0, tid_py); // steals ref + PyStructSequence_SetItem(result_item, 1, awaited_by_for_thread); // steals ref + if (PyList_Append(result, result_item)) { + Py_DECREF(result_item); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append awaited_by result item"); + return -1; + } + Py_DECREF(result_item); + + if (append_awaited_by_for_thread(unwinder, head_addr, awaited_by_for_thread)) + { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append awaited_by for thread"); + return -1; + } - PyObject* task_id = PyLong_FromUnsignedLongLong(task_addr); - if (task_id == NULL) { - Py_DECREF(tn); - Py_DECREF(current_awaited_by); + return 0; +} + +/* ============================================================================ + * STACK UNWINDING FUNCTIONS + * ============================================================================ */ + +static int +process_frame_chain( + RemoteUnwinderObject *unwinder, + uintptr_t initial_frame_addr, + StackChunkList *chunks, + PyObject *frame_info +) { + uintptr_t frame_addr = initial_frame_addr; + uintptr_t prev_frame_addr = 0; + const size_t MAX_FRAMES = 1024; + size_t frame_count = 0; + + while ((void*)frame_addr != NULL) { + PyObject *frame = NULL; + uintptr_t next_frame_addr = 0; + + if (++frame_count > MAX_FRAMES) { + PyErr_SetString(PyExc_RuntimeError, "Too many stack frames (possible infinite loop)"); + set_exception_cause(unwinder, PyExc_RuntimeError, "Frame chain iteration limit exceeded"); return -1; } - PyObject *result_item = PyTuple_New(3); - if (result_item == NULL) { - Py_DECREF(tn); - Py_DECREF(current_awaited_by); - Py_DECREF(task_id); - return -1; + // Try chunks first, fallback to direct memory read + if (parse_frame_from_chunks(unwinder, &frame, frame_addr, &next_frame_addr, chunks) < 0) { + PyErr_Clear(); + uintptr_t address_of_code_object = 0; + if (parse_frame_object(unwinder, &frame, frame_addr, &address_of_code_object ,&next_frame_addr) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to parse frame object in chain"); + return -1; + } } - PyTuple_SET_ITEM(result_item, 0, task_id); // steals ref - PyTuple_SET_ITEM(result_item, 1, tn); // steals ref - PyTuple_SET_ITEM(result_item, 2, current_awaited_by); // steals ref - if (PyList_Append(result, result_item)) { - Py_DECREF(result_item); - return -1; + if (!frame) { + break; } - Py_DECREF(result_item); - if (parse_task_awaited_by(handle, debug_offsets, async_offsets, - task_addr, current_awaited_by, 0)) - { + if (prev_frame_addr && frame_addr != prev_frame_addr) { + PyErr_Format(PyExc_RuntimeError, + "Broken frame chain: expected frame at 0x%lx, got 0x%lx", + prev_frame_addr, frame_addr); + Py_DECREF(frame); + set_exception_cause(unwinder, PyExc_RuntimeError, "Frame chain consistency check failed"); return -1; } - // onto the next one... - if (0 > _Py_RemoteDebug_ReadRemoteMemory( - handle, - (uintptr_t)task_node.next, - sizeof(task_node), - &task_node)) - { + if (PyList_Append(frame_info, frame) == -1) { + Py_DECREF(frame); + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to append frame to frame info list"); return -1; } + Py_DECREF(frame); + + prev_frame_addr = next_frame_addr; + frame_addr = next_frame_addr; } return 0; } +static PyObject* +unwind_stack_for_thread( + RemoteUnwinderObject *unwinder, + uintptr_t *current_tstate +) { + PyObject *frame_info = NULL; + PyObject *thread_id = NULL; + PyObject *result = NULL; + StackChunkList chunks = {0}; + + char ts[SIZEOF_THREAD_STATE]; + int bytes_read = _Py_RemoteDebug_PagedReadRemoteMemory( + &unwinder->handle, *current_tstate, (size_t)unwinder->debug_offsets.thread_state.size, ts); + if (bytes_read < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to read thread state"); + goto error; + } + + uintptr_t frame_addr = GET_MEMBER(uintptr_t, ts, unwinder->debug_offsets.thread_state.current_frame); + + frame_info = PyList_New(0); + if (!frame_info) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create frame info list"); + goto error; + } + + if (copy_stack_chunks(unwinder, *current_tstate, &chunks) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to copy stack chunks"); + goto error; + } + + if (process_frame_chain(unwinder, frame_addr, &chunks, frame_info) < 0) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to process frame chain"); + goto error; + } + + *current_tstate = GET_MEMBER(uintptr_t, ts, unwinder->debug_offsets.thread_state.next); + + thread_id = PyLong_FromLongLong( + GET_MEMBER(long, ts, unwinder->debug_offsets.thread_state.native_thread_id)); + if (thread_id == NULL) { + set_exception_cause(unwinder, PyExc_RuntimeError, "Failed to create thread ID"); + goto error; + } + + RemoteDebuggingState *state = RemoteDebugging_GetStateFromObject((PyObject*)unwinder); + result = PyStructSequence_New(state->ThreadInfo_Type); + if (result == NULL) { + set_exception_cause(unwinder, PyExc_MemoryError, "Failed to create ThreadInfo"); + goto error; + } + + PyStructSequence_SetItem(result, 0, thread_id); // Steals reference + PyStructSequence_SetItem(result, 1, frame_info); // Steals reference + + cleanup_stack_chunks(&chunks); + return result; + +error: + Py_XDECREF(frame_info); + Py_XDECREF(thread_id); + Py_XDECREF(result); + cleanup_stack_chunks(&chunks); + return NULL; +} + + +/* ============================================================================ + * REMOTEUNWINDER CLASS IMPLEMENTATION + * ============================================================================ */ + +/*[clinic input] +class _remote_debugging.RemoteUnwinder "RemoteUnwinderObject *" "&RemoteUnwinder_Type" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=55f164d8803318be]*/ + +/*[clinic input] +_remote_debugging.RemoteUnwinder.__init__ + pid: int + * + all_threads: bool = False + only_active_thread: bool = False + debug: bool = False + +Initialize a new RemoteUnwinder object for debugging a remote Python process. + +Args: + pid: Process ID of the target Python process to debug + all_threads: If True, initialize state for all threads in the process. + If False, only initialize for the main thread. + only_active_thread: If True, only sample the thread holding the GIL. + Cannot be used together with all_threads=True. + debug: If True, chain exceptions to explain the sequence of events that + lead to the exception. + +The RemoteUnwinder provides functionality to inspect and debug a running Python +process, including examining thread states, stack frames and other runtime data. + +Raises: + PermissionError: If access to the target process is denied + OSError: If unable to attach to the target process or access its memory + RuntimeError: If unable to read debug information from the target process + ValueError: If both all_threads and only_active_thread are True +[clinic start generated code]*/ + static int -append_awaited_by( - proc_handle_t *handle, - unsigned long tid, - uintptr_t head_addr, - struct _Py_DebugOffsets *debug_offsets, - struct _Py_AsyncioModuleDebugOffsets *async_offsets, - PyObject *result) +_remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, + int pid, int all_threads, + int only_active_thread, + int debug) +/*[clinic end generated code: output=13ba77598ecdcbe1 input=8f8f12504e17da04]*/ { - PyObject *tid_py = PyLong_FromUnsignedLong(tid); - if (tid_py == NULL) { + // Validate that all_threads and only_active_thread are not both True + if (all_threads && only_active_thread) { + PyErr_SetString(PyExc_ValueError, + "all_threads and only_active_thread cannot both be True"); + return -1; + } + +#ifdef Py_GIL_DISABLED + if (only_active_thread) { + PyErr_SetString(PyExc_ValueError, + "only_active_thread is not supported when Py_GIL_DISABLED is not defined"); + return -1; + } +#endif + + self->debug = debug; + self->only_active_thread = only_active_thread; + self->cached_state = NULL; + if (_Py_RemoteDebug_InitProcHandle(&self->handle, pid) < 0) { + set_exception_cause(self, PyExc_RuntimeError, "Failed to initialize process handle"); return -1; } - PyObject *result_item = PyTuple_New(2); - if (result_item == NULL) { - Py_DECREF(tid_py); + self->runtime_start_address = _Py_RemoteDebug_GetPyRuntimeAddress(&self->handle); + if (self->runtime_start_address == 0) { + set_exception_cause(self, PyExc_RuntimeError, "Failed to get Python runtime address"); return -1; } - PyObject* awaited_by_for_thread = PyList_New(0); - if (awaited_by_for_thread == NULL) { - Py_DECREF(tid_py); - Py_DECREF(result_item); + if (_Py_RemoteDebug_ReadDebugOffsets(&self->handle, + &self->runtime_start_address, + &self->debug_offsets) < 0) + { + set_exception_cause(self, PyExc_RuntimeError, "Failed to read debug offsets"); return -1; } - PyTuple_SET_ITEM(result_item, 0, tid_py); // steals ref - PyTuple_SET_ITEM(result_item, 1, awaited_by_for_thread); // steals ref - if (PyList_Append(result, result_item)) { - Py_DECREF(result_item); + // Validate that the debug offsets are valid + if(validate_debug_offsets(&self->debug_offsets) == -1) { + set_exception_cause(self, PyExc_RuntimeError, "Invalid debug offsets found"); return -1; } - Py_DECREF(result_item); - if (append_awaited_by_for_thread( - handle, - head_addr, - debug_offsets, - async_offsets, - awaited_by_for_thread)) + // Try to read async debug offsets, but don't fail if they're not available + self->async_debug_offsets_available = 1; + int async_debug_result = read_async_debug(self); + if (async_debug_result == PY_REMOTE_DEBUG_INVALID_ASYNC_DEBUG_OFFSETS) { + return -1; + } + if (async_debug_result < 0) { + PyErr_Clear(); + memset(&self->async_debug_offsets, 0, sizeof(self->async_debug_offsets)); + self->async_debug_offsets_available = 0; + } + + if (populate_initial_state_data(all_threads, self, self->runtime_start_address, + &self->interpreter_addr ,&self->tstate_addr) < 0) { + set_exception_cause(self, PyExc_RuntimeError, "Failed to populate initial state data"); return -1; } + self->code_object_cache = _Py_hashtable_new_full( + _Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct, + NULL, // keys are stable pointers, don't destroy + cached_code_metadata_destroy, + NULL + ); + if (self->code_object_cache == NULL) { + PyErr_NoMemory(); + set_exception_cause(self, PyExc_MemoryError, "Failed to create code object cache"); + return -1; + } + +#ifdef Py_GIL_DISABLED + // Initialize TLBC cache + self->tlbc_generation = 0; + self->tlbc_cache = _Py_hashtable_new_full( + _Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct, + NULL, // keys are stable pointers, don't destroy + tlbc_cache_entry_destroy, + NULL + ); + if (self->tlbc_cache == NULL) { + _Py_hashtable_destroy(self->code_object_cache); + PyErr_NoMemory(); + set_exception_cause(self, PyExc_MemoryError, "Failed to create TLBC cache"); + return -1; + } +#endif + return 0; } -static PyObject* -get_all_awaited_by(PyObject* self, PyObject* args) +/*[clinic input] +@critical_section +_remote_debugging.RemoteUnwinder.get_stack_trace + +Returns a list of stack traces for threads in the target process. + +Each element in the returned list is a tuple of (thread_id, frame_list), where: +- thread_id is the OS thread identifier +- frame_list is a list of tuples (function_name, filename, line_number) representing + the Python stack frames for that thread, ordered from most recent to oldest + +The threads returned depend on the initialization parameters: +- If only_active_thread was True: returns only the thread holding the GIL +- If all_threads was True: returns all threads +- Otherwise: returns only the main thread + +Example: + [ + (1234, [ + ('process_data', 'worker.py', 127), + ('run_worker', 'worker.py', 45), + ('main', 'app.py', 23) + ]), + (1235, [ + ('handle_request', 'server.py', 89), + ('serve_forever', 'server.py', 52) + ]) + ] + +Raises: + RuntimeError: If there is an error copying memory from the target process + OSError: If there is an error accessing the target process + PermissionError: If access to the target process is denied + UnicodeDecodeError: If there is an error decoding strings from the target process + +[clinic start generated code]*/ + +static PyObject * +_remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self) +/*[clinic end generated code: output=666192b90c69d567 input=f756f341206f9116]*/ { -#if (!defined(__linux__) && !defined(__APPLE__)) && !defined(MS_WINDOWS) || \ - (defined(__linux__) && !HAVE_PROCESS_VM_READV) - PyErr_SetString( - PyExc_RuntimeError, - "get_all_awaited_by is not implemented on this platform"); - return NULL; + PyObject* result = NULL; + // Read interpreter state into opaque buffer + char interp_state_buffer[INTERP_STATE_BUFFER_SIZE]; + if (_Py_RemoteDebug_PagedReadRemoteMemory( + &self->handle, + self->interpreter_addr, + INTERP_STATE_BUFFER_SIZE, + interp_state_buffer) < 0) { + set_exception_cause(self, PyExc_RuntimeError, "Failed to read interpreter state buffer"); + goto exit; + } + + // Get code object generation from buffer + uint64_t code_object_generation = GET_MEMBER(uint64_t, interp_state_buffer, + self->debug_offsets.interpreter_state.code_object_generation); + + if (code_object_generation != self->code_object_generation) { + self->code_object_generation = code_object_generation; + _Py_hashtable_clear(self->code_object_cache); + } + + // If only_active_thread is true, we need to determine which thread holds the GIL + PyThreadState* gil_holder = NULL; + if (self->only_active_thread) { + // The GIL state is already in interp_state_buffer, just read from there + // Check if GIL is locked + int gil_locked = GET_MEMBER(int, interp_state_buffer, + self->debug_offsets.interpreter_state.gil_runtime_state_locked); + + if (gil_locked) { + // Get the last holder (current holder when GIL is locked) + gil_holder = GET_MEMBER(PyThreadState*, interp_state_buffer, + self->debug_offsets.interpreter_state.gil_runtime_state_holder); + } else { + // GIL is not locked, return empty list + result = PyList_New(0); + if (!result) { + set_exception_cause(self, PyExc_MemoryError, "Failed to create empty result list"); + } + goto exit; + } + } + +#ifdef Py_GIL_DISABLED + // Check TLBC generation and invalidate cache if needed + uint32_t current_tlbc_generation = GET_MEMBER(uint32_t, interp_state_buffer, + self->debug_offsets.interpreter_state.tlbc_generation); + if (current_tlbc_generation != self->tlbc_generation) { + self->tlbc_generation = current_tlbc_generation; + _Py_hashtable_clear(self->tlbc_cache); + } #endif - int pid; - if (!PyArg_ParseTuple(args, "i", &pid)) { - return NULL; + uintptr_t current_tstate; + if (self->only_active_thread && gil_holder != NULL) { + // We have the GIL holder, process only that thread + current_tstate = (uintptr_t)gil_holder; + } else if (self->tstate_addr == 0) { + // Get threads head from buffer + current_tstate = GET_MEMBER(uintptr_t, interp_state_buffer, + self->debug_offsets.interpreter_state.threads_head); + } else { + current_tstate = self->tstate_addr; } - proc_handle_t the_handle; - proc_handle_t *handle = &the_handle; - if (_Py_RemoteDebug_InitProcHandle(handle, pid) < 0) { - return 0; + result = PyList_New(0); + if (!result) { + set_exception_cause(self, PyExc_MemoryError, "Failed to create stack trace result list"); + goto exit; } - PyObject *result = NULL; + while (current_tstate != 0) { + uintptr_t prev_tstate = current_tstate; + PyObject* frame_info = unwind_stack_for_thread(self, &current_tstate); + if (!frame_info) { + Py_CLEAR(result); + set_exception_cause(self, PyExc_RuntimeError, "Failed to unwind stack for thread"); + goto exit; + } + + if (PyList_Append(result, frame_info) == -1) { + Py_DECREF(frame_info); + Py_CLEAR(result); + set_exception_cause(self, PyExc_RuntimeError, "Failed to append thread frame info"); + goto exit; + } + Py_DECREF(frame_info); - uintptr_t runtime_start_addr = _Py_RemoteDebug_GetPyRuntimeAddress(handle); - if (runtime_start_addr == 0) { - if (!PyErr_Occurred()) { - PyErr_SetString( - PyExc_RuntimeError, "Failed to get .PyRuntime address"); + if (current_tstate == prev_tstate) { + PyErr_Format(PyExc_RuntimeError, + "Thread list cycle detected at address 0x%lx (corrupted remote memory)", + current_tstate); + set_exception_cause(self, PyExc_RuntimeError, + "Thread list cycle detected (corrupted remote memory)"); + Py_CLEAR(result); + goto exit; } - goto result_err; - } - struct _Py_DebugOffsets local_debug_offsets; - if (_Py_RemoteDebug_ReadDebugOffsets(handle, &runtime_start_addr, &local_debug_offsets)) { - chain_exceptions(PyExc_RuntimeError, "Failed to read debug offsets"); - goto result_err; - } + // We are targeting a single tstate, break here + if (self->tstate_addr) { + break; + } - struct _Py_AsyncioModuleDebugOffsets local_async_debug; - if (read_async_debug(handle, &local_async_debug)) { - chain_exceptions(PyExc_RuntimeError, "Failed to read asyncio debug offsets"); - goto result_err; + // If we're only processing the GIL holder, we're done after one iteration + if (self->only_active_thread && gil_holder != NULL) { + break; + } } - result = PyList_New(0); - if (result == NULL) { - goto result_err; - } +exit: + _Py_RemoteDebug_ClearCache(&self->handle); + return result; +} - uint64_t interpreter_state_list_head = - local_debug_offsets.runtime_state.interpreters_head; +/*[clinic input] +@critical_section +_remote_debugging.RemoteUnwinder.get_all_awaited_by + +Get all tasks and their awaited_by relationships from the remote process. + +This provides a tree structure showing which tasks are waiting for other tasks. + +For each task, returns: +1. The call stack frames leading to where the task is currently executing +2. The name of the task +3. A list of tasks that this task is waiting for, with their own frames/names/etc + +Returns a list of [frames, task_name, subtasks] where: +- frames: List of (func_name, filename, lineno) showing the call stack +- task_name: String identifier for the task +- subtasks: List of tasks being awaited by this task, in same format + +Raises: + RuntimeError: If AsyncioDebug section is not available in the remote process + MemoryError: If memory allocation fails + OSError: If reading from the remote process fails + +Example output: +[ + # Task c2_root waiting for two subtasks + [ + # Call stack of c2_root + [("c5", "script.py", 10), ("c4", "script.py", 14)], + "c2_root", + [ + # First subtask (sub_main_2) and what it's waiting for + [ + [("c1", "script.py", 23)], + "sub_main_2", + [...] + ], + # Second subtask and its waiters + [...] + ] + ] +] +[clinic start generated code]*/ - uintptr_t interpreter_state_addr; - if (0 > _Py_RemoteDebug_ReadRemoteMemory( - handle, - runtime_start_addr + interpreter_state_list_head, - sizeof(void*), - &interpreter_state_addr)) - { - goto result_err; +static PyObject * +_remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *self) +/*[clinic end generated code: output=6a49cd345e8aec53 input=a452c652bb00701a]*/ +{ + if (!self->async_debug_offsets_available) { + PyErr_SetString(PyExc_RuntimeError, "AsyncioDebug section not available"); + set_exception_cause(self, PyExc_RuntimeError, "AsyncioDebug section unavailable in get_all_awaited_by"); + return NULL; } - uintptr_t thread_state_addr; - unsigned long tid = 0; - if (0 > _Py_RemoteDebug_ReadRemoteMemory( - handle, - interpreter_state_addr - + local_debug_offsets.interpreter_state.threads_head, - sizeof(void*), - &thread_state_addr)) - { + PyObject *result = PyList_New(0); + if (result == NULL) { + set_exception_cause(self, PyExc_MemoryError, "Failed to create awaited_by result list"); goto result_err; } - uintptr_t head_addr; - while (thread_state_addr != 0) { - if (0 > _Py_RemoteDebug_ReadRemoteMemory( - handle, - thread_state_addr - + local_debug_offsets.thread_state.native_thread_id, - sizeof(tid), - &tid)) - { - goto result_err; - } - - head_addr = thread_state_addr - + local_async_debug.asyncio_thread_state.asyncio_tasks_head; - - if (append_awaited_by(handle, tid, head_addr, &local_debug_offsets, - &local_async_debug, result)) - { - goto result_err; - } - - if (0 > _Py_RemoteDebug_ReadRemoteMemory( - handle, - thread_state_addr + local_debug_offsets.thread_state.next, - sizeof(void*), - &thread_state_addr)) - { - goto result_err; - } + // Process all threads + if (iterate_threads(self, process_thread_for_awaited_by, result) < 0) { + goto result_err; } - head_addr = interpreter_state_addr - + local_async_debug.asyncio_interpreter_state.asyncio_tasks_head; + uintptr_t head_addr = self->interpreter_addr + + (uintptr_t)self->async_debug_offsets.asyncio_interpreter_state.asyncio_tasks_head; // On top of a per-thread task lists used by default by asyncio to avoid // contention, there is also a fallback per-interpreter list of tasks; // any tasks still pending when a thread is destroyed will be moved to the // per-interpreter task list. It's unlikely we'll find anything here, but // interesting for debugging. - if (append_awaited_by(handle, 0, head_addr, &local_debug_offsets, - &local_async_debug, result)) + if (append_awaited_by(self, 0, head_addr, result)) { + set_exception_cause(self, PyExc_RuntimeError, "Failed to append interpreter awaited_by in get_all_awaited_by"); goto result_err; } - _Py_RemoteDebug_CleanupProcHandle(handle); + _Py_RemoteDebug_ClearCache(&self->handle); return result; result_err: + _Py_RemoteDebug_ClearCache(&self->handle); Py_XDECREF(result); - _Py_RemoteDebug_CleanupProcHandle(handle); return NULL; } -static PyObject* -get_stack_trace(PyObject* self, PyObject* args) -{ -#if (!defined(__linux__) && !defined(__APPLE__)) && !defined(MS_WINDOWS) || \ - (defined(__linux__) && !HAVE_PROCESS_VM_READV) - PyErr_SetString( - PyExc_RuntimeError, - "get_stack_trace is not supported on this platform"); - return NULL; -#endif +/*[clinic input] +@critical_section +_remote_debugging.RemoteUnwinder.get_async_stack_trace + +Get the currently running async tasks and their dependency graphs from the remote process. + +This returns information about running tasks and all tasks that are waiting for them, +forming a complete dependency graph for each thread's active task. + +For each thread with a running task, returns the running task plus all tasks that +transitively depend on it (tasks waiting for the running task, tasks waiting for +those tasks, etc.). + +Returns a list of per-thread results, where each thread result contains: +- Thread ID +- List of task information for the running task and all its waiters + +Each task info contains: +- Task ID (memory address) +- Task name +- Call stack frames: List of (func_name, filename, lineno) +- List of tasks waiting for this task (recursive structure) + +Raises: + RuntimeError: If AsyncioDebug section is not available in the target process + MemoryError: If memory allocation fails + OSError: If reading from the remote process fails + +Example output (similar structure to get_all_awaited_by but only for running tasks): +[ + # Thread 140234 results + (140234, [ + # Running task and its complete waiter dependency graph + (4345585712, 'main_task', + [("run_server", "server.py", 127), ("main", "app.py", 23)], + [ + # Tasks waiting for main_task + (4345585800, 'worker_1', [...], [...]), + (4345585900, 'worker_2', [...], [...]) + ]) + ]) +] + +[clinic start generated code]*/ - int pid; - if (!PyArg_ParseTuple(args, "i", &pid)) { +static PyObject * +_remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject *self) +/*[clinic end generated code: output=6433d52b55e87bbe input=8744b47c9ec2220a]*/ +{ + if (!self->async_debug_offsets_available) { + PyErr_SetString(PyExc_RuntimeError, "AsyncioDebug section not available"); + set_exception_cause(self, PyExc_RuntimeError, "AsyncioDebug section unavailable in get_async_stack_trace"); return NULL; } - proc_handle_t the_handle; - proc_handle_t *handle = &the_handle; - if (_Py_RemoteDebug_InitProcHandle(handle, pid) < 0) { - return 0; - } - - PyObject* result = NULL; - - uintptr_t runtime_start_address = _Py_RemoteDebug_GetPyRuntimeAddress(handle); - if (runtime_start_address == 0) { - if (!PyErr_Occurred()) { - PyErr_SetString( - PyExc_RuntimeError, "Failed to get .PyRuntime address"); - } - goto result_err; - } - struct _Py_DebugOffsets local_debug_offsets; - - if (_Py_RemoteDebug_ReadDebugOffsets(handle, &runtime_start_address, &local_debug_offsets)) { - chain_exceptions(PyExc_RuntimeError, "Failed to read debug offsets"); - goto result_err; - } - - uintptr_t address_of_current_frame; - if (find_running_frame( - handle, runtime_start_address, &local_debug_offsets, - &address_of_current_frame) - ) { - goto result_err; - } - - result = PyList_New(0); + PyObject *result = PyList_New(0); if (result == NULL) { - goto result_err; + set_exception_cause(self, PyExc_MemoryError, "Failed to create result list in get_async_stack_trace"); + return NULL; } - while ((void*)address_of_current_frame != NULL) { - PyObject* frame_info = NULL; - if (parse_frame_object( - handle, - &frame_info, - &local_debug_offsets, - address_of_current_frame, - &address_of_current_frame) - < 0) - { - Py_DECREF(result); - goto result_err; - } - - if (!frame_info) { - continue; - } - - if (PyList_Append(result, frame_info) == -1) { - Py_DECREF(result); - goto result_err; - } - - Py_DECREF(frame_info); - frame_info = NULL; - + // Process all threads + if (iterate_threads(self, process_thread_for_async_stack_trace, result) < 0) { + goto result_err; } -result_err: - _Py_RemoteDebug_CleanupProcHandle(handle); + _Py_RemoteDebug_ClearCache(&self->handle); return result; +result_err: + _Py_RemoteDebug_ClearCache(&self->handle); + Py_XDECREF(result); + return NULL; } -static PyObject* -get_async_stack_trace(PyObject* self, PyObject* args) -{ -#if (!defined(__linux__) && !defined(__APPLE__)) && !defined(MS_WINDOWS) || \ - (defined(__linux__) && !HAVE_PROCESS_VM_READV) - PyErr_SetString( - PyExc_RuntimeError, - "get_stack_trace is not supported on this platform"); - return NULL; -#endif - int pid; +static PyMethodDef RemoteUnwinder_methods[] = { + _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_STACK_TRACE_METHODDEF + _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_ALL_AWAITED_BY_METHODDEF + _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_ASYNC_STACK_TRACE_METHODDEF + {NULL, NULL} +}; - if (!PyArg_ParseTuple(args, "i", &pid)) { - return NULL; +static void +RemoteUnwinder_dealloc(PyObject *op) +{ + RemoteUnwinderObject *self = RemoteUnwinder_CAST(op); + PyTypeObject *tp = Py_TYPE(self); + if (self->code_object_cache) { + _Py_hashtable_destroy(self->code_object_cache); } - - proc_handle_t the_handle; - proc_handle_t *handle = &the_handle; - if (_Py_RemoteDebug_InitProcHandle(handle, pid) < 0) { - return 0; +#ifdef Py_GIL_DISABLED + if (self->tlbc_cache) { + _Py_hashtable_destroy(self->tlbc_cache); } - - PyObject *result = NULL; - - uintptr_t runtime_start_address = _Py_RemoteDebug_GetPyRuntimeAddress(handle); - if (runtime_start_address == 0) { - if (!PyErr_Occurred()) { - PyErr_SetString( - PyExc_RuntimeError, "Failed to get .PyRuntime address"); - } - goto result_err; +#endif + if (self->handle.pid != 0) { + _Py_RemoteDebug_ClearCache(&self->handle); + _Py_RemoteDebug_CleanupProcHandle(&self->handle); } - struct _Py_DebugOffsets local_debug_offsets; + PyObject_Del(self); + Py_DECREF(tp); +} - if (_Py_RemoteDebug_ReadDebugOffsets(handle, &runtime_start_address, &local_debug_offsets)) { - chain_exceptions(PyExc_RuntimeError, "Failed to read debug offsets"); - goto result_err; - } +static PyType_Slot RemoteUnwinder_slots[] = { + {Py_tp_doc, (void *)"RemoteUnwinder(pid): Inspect stack of a remote Python process."}, + {Py_tp_methods, RemoteUnwinder_methods}, + {Py_tp_init, _remote_debugging_RemoteUnwinder___init__}, + {Py_tp_dealloc, RemoteUnwinder_dealloc}, + {0, NULL} +}; - struct _Py_AsyncioModuleDebugOffsets local_async_debug; - if (read_async_debug(handle, &local_async_debug)) { - chain_exceptions(PyExc_RuntimeError, "Failed to read asyncio debug offsets"); - goto result_err; - } +static PyType_Spec RemoteUnwinder_spec = { + .name = "_remote_debugging.RemoteUnwinder", + .basicsize = sizeof(RemoteUnwinderObject), + .flags = Py_TPFLAGS_DEFAULT, + .slots = RemoteUnwinder_slots, +}; - result = PyList_New(1); - if (result == NULL) { - goto result_err; - } - PyObject* calls = PyList_New(0); - if (calls == NULL) { - goto result_err; - } - if (PyList_SetItem(result, 0, calls)) { /* steals ref to 'calls' */ - Py_DECREF(calls); - goto result_err; - } +/* ============================================================================ + * MODULE INITIALIZATION + * ============================================================================ */ - uintptr_t running_task_addr = (uintptr_t)NULL; - if (find_running_task( - handle, runtime_start_address, &local_debug_offsets, &local_async_debug, - &running_task_addr) - ) { - chain_exceptions(PyExc_RuntimeError, "Failed to find running task"); - goto result_err; +static int +_remote_debugging_exec(PyObject *m) +{ + RemoteDebuggingState *st = RemoteDebugging_GetState(m); +#define CREATE_TYPE(mod, type, spec) \ + do { \ + type = (PyTypeObject *)PyType_FromMetaclass(NULL, mod, spec, NULL); \ + if (type == NULL) { \ + return -1; \ + } \ + } while (0) + + CREATE_TYPE(m, st->RemoteDebugging_Type, &RemoteUnwinder_spec); + + if (PyModule_AddType(m, st->RemoteDebugging_Type) < 0) { + return -1; } - if ((void*)running_task_addr == NULL) { - PyErr_SetString(PyExc_RuntimeError, "No running task found"); - goto result_err; + // Initialize structseq types + st->TaskInfo_Type = PyStructSequence_NewType(&TaskInfo_desc); + if (st->TaskInfo_Type == NULL) { + return -1; } - - uintptr_t running_coro_addr; - if (read_py_ptr( - handle, - running_task_addr + local_async_debug.asyncio_task_object.task_coro, - &running_coro_addr - )) { - chain_exceptions(PyExc_RuntimeError, "Failed to read running task coro"); - goto result_err; + if (PyModule_AddType(m, st->TaskInfo_Type) < 0) { + return -1; } - if ((void*)running_coro_addr == NULL) { - PyErr_SetString(PyExc_RuntimeError, "Running task coro is NULL"); - goto result_err; + st->FrameInfo_Type = PyStructSequence_NewType(&FrameInfo_desc); + if (st->FrameInfo_Type == NULL) { + return -1; } - - // note: genobject's gi_iframe is an embedded struct so the address to - // the offset leads directly to its first field: f_executable - uintptr_t address_of_running_task_code_obj; - if (read_py_ptr( - handle, - running_coro_addr + local_debug_offsets.gen_object.gi_iframe, - &address_of_running_task_code_obj - )) { - goto result_err; + if (PyModule_AddType(m, st->FrameInfo_Type) < 0) { + return -1; } - if ((void*)address_of_running_task_code_obj == NULL) { - PyErr_SetString(PyExc_RuntimeError, "Running task code object is NULL"); - goto result_err; + st->CoroInfo_Type = PyStructSequence_NewType(&CoroInfo_desc); + if (st->CoroInfo_Type == NULL) { + return -1; } - - uintptr_t address_of_current_frame; - if (find_running_frame( - handle, runtime_start_address, &local_debug_offsets, - &address_of_current_frame) - ) { - chain_exceptions(PyExc_RuntimeError, "Failed to find running frame"); - goto result_err; + if (PyModule_AddType(m, st->CoroInfo_Type) < 0) { + return -1; } - uintptr_t address_of_code_object; - while ((void*)address_of_current_frame != NULL) { - PyObject* frame_info = NULL; - int res = parse_async_frame_object( - handle, - &frame_info, - &local_debug_offsets, - address_of_current_frame, - &address_of_current_frame, - &address_of_code_object - ); - - if (res < 0) { - chain_exceptions(PyExc_RuntimeError, "Failed to parse async frame object"); - goto result_err; - } - - if (!frame_info) { - continue; - } - - if (PyList_Append(calls, frame_info) == -1) { - Py_DECREF(calls); - goto result_err; - } - - Py_DECREF(frame_info); - frame_info = NULL; - - if (address_of_code_object == address_of_running_task_code_obj) { - break; - } + st->ThreadInfo_Type = PyStructSequence_NewType(&ThreadInfo_desc); + if (st->ThreadInfo_Type == NULL) { + return -1; + } + if (PyModule_AddType(m, st->ThreadInfo_Type) < 0) { + return -1; } - PyObject *tn = parse_task_name( - handle, &local_debug_offsets, &local_async_debug, running_task_addr); - if (tn == NULL) { - goto result_err; + st->AwaitedInfo_Type = PyStructSequence_NewType(&AwaitedInfo_desc); + if (st->AwaitedInfo_Type == NULL) { + return -1; } - if (PyList_Append(result, tn)) { - Py_DECREF(tn); - goto result_err; + if (PyModule_AddType(m, st->AwaitedInfo_Type) < 0) { + return -1; } - Py_DECREF(tn); - - PyObject* awaited_by = PyList_New(0); - if (awaited_by == NULL) { - goto result_err; +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif + int rc = PyModule_AddIntConstant(m, "PROCESS_VM_READV_SUPPORTED", HAVE_PROCESS_VM_READV); + if (rc < 0) { + return -1; } - if (PyList_Append(result, awaited_by)) { - Py_DECREF(awaited_by); - goto result_err; + if (RemoteDebugging_InitState(st) < 0) { + return -1; } - Py_DECREF(awaited_by); + return 0; +} - if (parse_task_awaited_by( - handle, &local_debug_offsets, &local_async_debug, - running_task_addr, awaited_by, 1) - ) { - goto result_err; - } +static int +remote_debugging_traverse(PyObject *mod, visitproc visit, void *arg) +{ + RemoteDebuggingState *state = RemoteDebugging_GetState(mod); + Py_VISIT(state->RemoteDebugging_Type); + Py_VISIT(state->TaskInfo_Type); + Py_VISIT(state->FrameInfo_Type); + Py_VISIT(state->CoroInfo_Type); + Py_VISIT(state->ThreadInfo_Type); + Py_VISIT(state->AwaitedInfo_Type); + return 0; +} - _Py_RemoteDebug_CleanupProcHandle(handle); - return result; +static int +remote_debugging_clear(PyObject *mod) +{ + RemoteDebuggingState *state = RemoteDebugging_GetState(mod); + Py_CLEAR(state->RemoteDebugging_Type); + Py_CLEAR(state->TaskInfo_Type); + Py_CLEAR(state->FrameInfo_Type); + Py_CLEAR(state->CoroInfo_Type); + Py_CLEAR(state->ThreadInfo_Type); + Py_CLEAR(state->AwaitedInfo_Type); + return 0; +} -result_err: - _Py_RemoteDebug_CleanupProcHandle(handle); - Py_XDECREF(result); - return NULL; +static void +remote_debugging_free(void *mod) +{ + (void)remote_debugging_clear((PyObject *)mod); } +static PyModuleDef_Slot remote_debugging_slots[] = { + {Py_mod_exec, _remote_debugging_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL}, +}; -static PyMethodDef methods[] = { - {"get_stack_trace", get_stack_trace, METH_VARARGS, - "Get the Python stack from a given pod"}, - {"get_async_stack_trace", get_async_stack_trace, METH_VARARGS, - "Get the asyncio stack from a given pid"}, - {"get_all_awaited_by", get_all_awaited_by, METH_VARARGS, - "Get all tasks and their awaited_by from a given pid"}, +static PyMethodDef remote_debugging_methods[] = { {NULL, NULL, 0, NULL}, }; -static struct PyModuleDef module = { - .m_base = PyModuleDef_HEAD_INIT, +static struct PyModuleDef remote_debugging_module = { + PyModuleDef_HEAD_INIT, .m_name = "_remote_debugging", - .m_size = -1, - .m_methods = methods, + .m_size = sizeof(RemoteDebuggingState), + .m_methods = remote_debugging_methods, + .m_slots = remote_debugging_slots, + .m_traverse = remote_debugging_traverse, + .m_clear = remote_debugging_clear, + .m_free = remote_debugging_free, }; PyMODINIT_FUNC PyInit__remote_debugging(void) { - PyObject* mod = PyModule_Create(&module); - if (mod == NULL) { - return NULL; - } -#ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); -#endif - int rc = PyModule_AddIntConstant( - mod, "PROCESS_VM_READV_SUPPORTED", HAVE_PROCESS_VM_READV); - if (rc < 0) { - Py_DECREF(mod); - return NULL; - } - return mod; + return PyModuleDef_Init(&remote_debugging_module); } diff --git a/Modules/_sqlite/blob.c b/Modules/_sqlite/blob.c index 35d090e3ca2dce3..a99e2e8bf35eb07 100644 --- a/Modules/_sqlite/blob.c +++ b/Modules/_sqlite/blob.c @@ -4,6 +4,7 @@ #include "blob.h" #include "util.h" +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #define clinic_state() (pysqlite_get_state_by_type(Py_TYPE(self))) #include "clinic/blob.c.h" @@ -56,9 +57,7 @@ blob_dealloc(PyObject *op) close_blob(self); - if (self->in_weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, self->in_weakreflist); (void)tp->tp_clear(op); tp->tp_free(self); Py_DECREF(tp); @@ -173,14 +172,14 @@ _sqlite3.Blob.read as blob_read Read data at the current offset position. -If the end of the blob is reached, the data up to end of file will be returned. -When length is not specified, or is negative, Blob.read() will read until the -end of the blob. +If the end of the blob is reached, the data up to end of file will +be returned. When length is not specified, or is negative, +Blob.read() will read until the end of the blob. [clinic start generated code]*/ static PyObject * blob_read_impl(pysqlite_Blob *self, int length) -/*[clinic end generated code: output=1fc99b2541360dde input=f2e4aa4378837250]*/ +/*[clinic end generated code: output=1fc99b2541360dde input=6b745ad37720e556]*/ { if (!check_blob(self)) { return NULL; @@ -240,13 +239,13 @@ _sqlite3.Blob.write as blob_write Write data at the current offset. -This function cannot change the blob length. Writing beyond the end of the -blob will result in an exception being raised. +This function cannot change the blob length. Writing beyond the end +of the blob will result in an exception being raised. [clinic start generated code]*/ static PyObject * blob_write_impl(pysqlite_Blob *self, Py_buffer *data) -/*[clinic end generated code: output=b34cf22601b570b2 input=a84712f24a028e6d]*/ +/*[clinic end generated code: output=b34cf22601b570b2 input=0d372cb0240a5d49]*/ { if (!check_blob(self)) { return NULL; @@ -270,14 +269,15 @@ _sqlite3.Blob.seek as blob_seek Set the current access position to offset. -The origin argument defaults to os.SEEK_SET (absolute blob positioning). -Other values for origin are os.SEEK_CUR (seek relative to the current position) -and os.SEEK_END (seek relative to the blob's end). +The origin argument defaults to os.SEEK_SET (absolute blob +positioning). Other values for origin are os.SEEK_CUR (seek +relative to the current position) and os.SEEK_END (seek relative to +the blob's end). [clinic start generated code]*/ static PyObject * blob_seek_impl(pysqlite_Blob *self, int offset, int origin) -/*[clinic end generated code: output=854c5a0e208547a5 input=5da9a07e55fe6bb6]*/ +/*[clinic end generated code: output=854c5a0e208547a5 input=84aea1b6b48607dd]*/ { if (!check_blob(self)) { return NULL; @@ -435,6 +435,10 @@ subscript_slice(pysqlite_Blob *self, PyObject *item) return NULL; } + if (len == 0) { + return PyBytes_FromStringAndSize(NULL, 0); + } + if (step == 1) { return read_multiple(self, len, start); } @@ -520,21 +524,25 @@ ass_subscript_slice(pysqlite_Blob *self, PyObject *item, PyObject *value) return -1; } - if (len == 0) { - return 0; - } - Py_buffer vbuf; if (PyObject_GetBuffer(value, &vbuf, PyBUF_SIMPLE) < 0) { return -1; } - int rc = -1; if (vbuf.len != len) { PyErr_SetString(PyExc_IndexError, "Blob slice assignment is wrong size"); + PyBuffer_Release(&vbuf); + return -1; } - else if (step == 1) { + + if (len == 0) { + PyBuffer_Release(&vbuf); + return 0; + } + + int rc = -1; + if (step == 1) { rc = inner_write(self, vbuf.buf, len, start); } else { diff --git a/Modules/_sqlite/clinic/blob.c.h b/Modules/_sqlite/clinic/blob.c.h index 921e7cbd7ffcaba..929703257f04bee 100644 --- a/Modules/_sqlite/clinic/blob.c.h +++ b/Modules/_sqlite/clinic/blob.c.h @@ -31,9 +31,9 @@ PyDoc_STRVAR(blob_read__doc__, " length\n" " Read length in bytes.\n" "\n" -"If the end of the blob is reached, the data up to end of file will be returned.\n" -"When length is not specified, or is negative, Blob.read() will read until the\n" -"end of the blob."); +"If the end of the blob is reached, the data up to end of file will\n" +"be returned. When length is not specified, or is negative,\n" +"Blob.read() will read until the end of the blob."); #define BLOB_READ_METHODDEF \ {"read", _PyCFunction_CAST(blob_read), METH_FASTCALL, blob_read__doc__}, @@ -70,8 +70,8 @@ PyDoc_STRVAR(blob_write__doc__, "\n" "Write data at the current offset.\n" "\n" -"This function cannot change the blob length. Writing beyond the end of the\n" -"blob will result in an exception being raised."); +"This function cannot change the blob length. Writing beyond the end\n" +"of the blob will result in an exception being raised."); #define BLOB_WRITE_METHODDEF \ {"write", (PyCFunction)blob_write, METH_O, blob_write__doc__}, @@ -105,9 +105,10 @@ PyDoc_STRVAR(blob_seek__doc__, "\n" "Set the current access position to offset.\n" "\n" -"The origin argument defaults to os.SEEK_SET (absolute blob positioning).\n" -"Other values for origin are os.SEEK_CUR (seek relative to the current position)\n" -"and os.SEEK_END (seek relative to the blob\'s end)."); +"The origin argument defaults to os.SEEK_SET (absolute blob\n" +"positioning). Other values for origin are os.SEEK_CUR (seek\n" +"relative to the current position) and os.SEEK_END (seek relative to\n" +"the blob\'s end)."); #define BLOB_SEEK_METHODDEF \ {"seek", _PyCFunction_CAST(blob_seek), METH_FASTCALL, blob_seek__doc__}, @@ -211,4 +212,4 @@ blob_exit(PyObject *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=f03f4ba622b67ae0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b0e3d38063739b17 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/connection.c.h b/Modules/_sqlite/clinic/connection.c.h index c8e1d0b7a738d3c..25129b63b256a07 100644 --- a/Modules/_sqlite/clinic/connection.c.h +++ b/Modules/_sqlite/clinic/connection.c.h @@ -235,7 +235,7 @@ pysqlite_connection_cursor(PyObject *self, PyObject *const *args, Py_ssize_t nar } PyDoc_STRVAR(blobopen__doc__, -"blobopen($self, table, column, row, /, *, readonly=False, name=\'main\')\n" +"blobopen($self, table, column, rowid, /, *, readonly=False, name=\'main\')\n" "--\n" "\n" "Open and return a BLOB object.\n" @@ -244,8 +244,8 @@ PyDoc_STRVAR(blobopen__doc__, " Table name.\n" " column\n" " Column name.\n" -" row\n" -" Row index.\n" +" rowid\n" +" Row id.\n" " readonly\n" " Open the BLOB without write permissions.\n" " name\n" @@ -818,13 +818,14 @@ PyDoc_STRVAR(pysqlite_connection_set_progress_handler__doc__, "\n" " progress_handler\n" " A callable that takes no arguments.\n" -" If the callable returns non-zero, the current query is terminated,\n" -" and an exception is raised.\n" +" If the callable returns non-zero, the current query is\n" +" terminated, and an exception is raised.\n" " n\n" " The number of SQLite virtual machine instructions that are\n" " executed between invocations of \'progress_handler\'.\n" "\n" -"If \'progress_handler\' is None or \'n\' is 0, the progress handler is disabled.\n" +"If \'progress_handler\' is None or \'n\' is 0, the progress handler is\n" +"disabled.\n" "\n" "Note: Passing keyword argument \'progress_handler\' to\n" "_sqlite3.Connection.set_progress_handler() is deprecated. Parameter\n" @@ -1502,10 +1503,10 @@ PyDoc_STRVAR(serialize__doc__, " name\n" " Which database to serialize.\n" "\n" -"For an ordinary on-disk database file, the serialization is just a copy of the\n" -"disk file. For an in-memory database or a \"temp\" database, the serialization is\n" -"the same sequence of bytes which would be written to disk if that database\n" -"were backed up to disk."); +"For an ordinary on-disk database file, the serialization is just\n" +"a copy of the disk file. For an in-memory database or a \"temp\"\n" +"database, the serialization is the same sequence of bytes which\n" +"would be written to disk if that database were backed up to disk."); #define SERIALIZE_METHODDEF \ {"serialize", _PyCFunction_CAST(serialize), METH_FASTCALL|METH_KEYWORDS, serialize__doc__}, @@ -1591,12 +1592,13 @@ PyDoc_STRVAR(deserialize__doc__, " name\n" " Which database to reopen with the deserialization.\n" "\n" -"The deserialize interface causes the database connection to disconnect from the\n" -"target database, and then reopen it as an in-memory database based on the given\n" -"serialized data.\n" +"The deserialize interface causes the database connection to\n" +"disconnect from the target database, and then reopen it as\n" +"an in-memory database based on the given serialized data.\n" "\n" -"The deserialize interface will fail with SQLITE_BUSY if the database is\n" -"currently in a read transaction or is involved in a backup operation."); +"The deserialize interface will fail with SQLITE_BUSY if the database\n" +"is currently in a read transaction or is involved in a backup\n" +"operation."); #define DESERIALIZE_METHODDEF \ {"deserialize", _PyCFunction_CAST(deserialize), METH_FASTCALL|METH_KEYWORDS, deserialize__doc__}, @@ -1717,7 +1719,8 @@ PyDoc_STRVAR(pysqlite_connection_exit__doc__, "\n" "Called when the connection is used as a context manager.\n" "\n" -"If there was any exception, a rollback takes place; otherwise we commit."); +"If there was any exception, a rollback takes place; otherwise we\n" +"commit."); #define PYSQLITE_CONNECTION_EXIT_METHODDEF \ {"__exit__", _PyCFunction_CAST(pysqlite_connection_exit), METH_FASTCALL, pysqlite_connection_exit__doc__}, @@ -1755,12 +1758,12 @@ PyDoc_STRVAR(setlimit__doc__, " category\n" " The limit category to be set.\n" " limit\n" -" The new limit. If the new limit is a negative number, the limit is\n" -" unchanged.\n" +" The new limit. If the new limit is a negative number, the limit\n" +" is unchanged.\n" "\n" -"Attempts to increase a limit above its hard upper bound are silently truncated\n" -"to the hard upper bound. Regardless of whether or not the limit was changed,\n" -"the prior value of the limit is returned."); +"Attempts to increase a limit above its hard upper bound are silently\n" +"truncated to the hard upper bound. Regardless of whether or not the\n" +"limit was changed, the prior value of the limit is returned."); #define SETLIMIT_METHODDEF \ {"setlimit", _PyCFunction_CAST(setlimit), METH_FASTCALL, setlimit__doc__}, @@ -1921,4 +1924,4 @@ getconfig(PyObject *self, PyObject *arg) #ifndef DESERIALIZE_METHODDEF #define DESERIALIZE_METHODDEF #endif /* !defined(DESERIALIZE_METHODDEF) */ -/*[clinic end generated code: output=2f325c2444b4bb47 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=072e199b0fc4a271 input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/clinic/cursor.c.h b/Modules/_sqlite/clinic/cursor.c.h index 350577f488df4b4..3cad9f3aef5ecd5 100644 --- a/Modules/_sqlite/clinic/cursor.c.h +++ b/Modules/_sqlite/clinic/cursor.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_long.h" // _PyLong_UInt32_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() static int @@ -181,7 +182,7 @@ PyDoc_STRVAR(pysqlite_cursor_fetchmany__doc__, {"fetchmany", _PyCFunction_CAST(pysqlite_cursor_fetchmany), METH_FASTCALL|METH_KEYWORDS, pysqlite_cursor_fetchmany__doc__}, static PyObject * -pysqlite_cursor_fetchmany_impl(pysqlite_Cursor *self, int maxrows); +pysqlite_cursor_fetchmany_impl(pysqlite_Cursor *self, uint32_t maxrows); static PyObject * pysqlite_cursor_fetchmany(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -216,7 +217,7 @@ pysqlite_cursor_fetchmany(PyObject *self, PyObject *const *args, Py_ssize_t narg #undef KWTUPLE PyObject *argsbuf[1]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - int maxrows = ((pysqlite_Cursor *)self)->arraysize; + uint32_t maxrows = ((pysqlite_Cursor *)self)->arraysize; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -226,8 +227,7 @@ pysqlite_cursor_fetchmany(PyObject *self, PyObject *const *args, Py_ssize_t narg if (!noptargs) { goto skip_optional_pos; } - maxrows = PyLong_AsInt(args[0]); - if (maxrows == -1 && PyErr_Occurred()) { + if (!_PyLong_UInt32_Converter(args[0], &maxrows)) { goto exit; } skip_optional_pos: @@ -329,4 +329,46 @@ pysqlite_cursor_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { return pysqlite_cursor_close_impl((pysqlite_Cursor *)self); } -/*[clinic end generated code: output=d05c7cbbc8bcab26 input=a9049054013a1b77]*/ + +#if !defined(_sqlite3_Cursor_arraysize_DOCSTR) +# define _sqlite3_Cursor_arraysize_DOCSTR NULL +#endif +#if defined(_SQLITE3_CURSOR_ARRAYSIZE_GETSETDEF) +# undef _SQLITE3_CURSOR_ARRAYSIZE_GETSETDEF +# define _SQLITE3_CURSOR_ARRAYSIZE_GETSETDEF {"arraysize", (getter)_sqlite3_Cursor_arraysize_get, (setter)_sqlite3_Cursor_arraysize_set, _sqlite3_Cursor_arraysize_DOCSTR}, +#else +# define _SQLITE3_CURSOR_ARRAYSIZE_GETSETDEF {"arraysize", (getter)_sqlite3_Cursor_arraysize_get, NULL, _sqlite3_Cursor_arraysize_DOCSTR}, +#endif + +static PyObject * +_sqlite3_Cursor_arraysize_get_impl(pysqlite_Cursor *self); + +static PyObject * +_sqlite3_Cursor_arraysize_get(PyObject *self, void *Py_UNUSED(context)) +{ + return _sqlite3_Cursor_arraysize_get_impl((pysqlite_Cursor *)self); +} + +#if !defined(_sqlite3_Cursor_arraysize_DOCSTR) +# define _sqlite3_Cursor_arraysize_DOCSTR NULL +#endif +#if defined(_SQLITE3_CURSOR_ARRAYSIZE_GETSETDEF) +# undef _SQLITE3_CURSOR_ARRAYSIZE_GETSETDEF +# define _SQLITE3_CURSOR_ARRAYSIZE_GETSETDEF {"arraysize", (getter)_sqlite3_Cursor_arraysize_get, (setter)_sqlite3_Cursor_arraysize_set, _sqlite3_Cursor_arraysize_DOCSTR}, +#else +# define _SQLITE3_CURSOR_ARRAYSIZE_GETSETDEF {"arraysize", NULL, (setter)_sqlite3_Cursor_arraysize_set, NULL}, +#endif + +static int +_sqlite3_Cursor_arraysize_set_impl(pysqlite_Cursor *self, PyObject *value); + +static int +_sqlite3_Cursor_arraysize_set(PyObject *self, PyObject *value, void *Py_UNUSED(context)) +{ + int return_value; + + return_value = _sqlite3_Cursor_arraysize_set_impl((pysqlite_Cursor *)self, value); + + return return_value; +} +/*[clinic end generated code: output=a0e3ebba9e4d0ece input=a9049054013a1b77]*/ diff --git a/Modules/_sqlite/connection.c b/Modules/_sqlite/connection.c index 2a184f787542ec0..462a35410fd7cbe 100644 --- a/Modules/_sqlite/connection.c +++ b/Modules/_sqlite/connection.c @@ -144,8 +144,9 @@ class _sqlite3.Connection "pysqlite_Connection *" "clinic_state()->ConnectionTyp [clinic start generated code]*/ /*[clinic end generated code: output=da39a3ee5e6b4b0d input=67369db2faf80891]*/ -static void _pysqlite_drop_unused_cursor_references(pysqlite_Connection* self); -static void free_callback_context(callback_context *ctx); +static int _pysqlite_drop_unused_cursor_references(pysqlite_Connection* self); +static void incref_callback_context(callback_context *ctx); +static void decref_callback_context(callback_context *ctx); static void set_callback_context(callback_context **ctx_pp, callback_context *ctx); static int connection_close(pysqlite_Connection *self); @@ -562,7 +563,10 @@ pysqlite_connection_cursor_impl(pysqlite_Connection *self, PyObject *factory) return NULL; } - _pysqlite_drop_unused_cursor_references(self); + if (_pysqlite_drop_unused_cursor_references(self) < 0) { + Py_DECREF(cursor); + return NULL; + } if (cursor && self->row_factory != Py_None) { Py_INCREF(self->row_factory); @@ -572,6 +576,47 @@ pysqlite_connection_cursor_impl(pysqlite_Connection *self, PyObject *factory) return cursor; } +static PyObject * +connection_get_row_factory(PyObject *op, void *closure) +{ + pysqlite_Connection *self = (pysqlite_Connection *)op; + return Py_NewRef(self->row_factory); +} + +static int +connection_set_row_factory(PyObject *op, PyObject *value, void *closure) +{ + pysqlite_Connection *self = (pysqlite_Connection *)op; + if (value == NULL) { + PyErr_SetString(PyExc_AttributeError, + "cannot delete row_factory attribute"); + return -1; + } + Py_XSETREF(self->row_factory, Py_NewRef(value)); + return 0; +} + +static PyObject * +connection_get_text_factory(PyObject *op, void *closure) +{ + pysqlite_Connection *self = (pysqlite_Connection *)op; + return Py_NewRef(self->text_factory); +} + +static int +connection_set_text_factory(PyObject *op, PyObject *value, void *closure) +{ + pysqlite_Connection *self = (pysqlite_Connection *)op; + if (value == NULL) { + PyErr_SetString(PyExc_AttributeError, + "cannot delete text_factory attribute"); + return -1; + } + Py_XSETREF(self->text_factory, Py_NewRef(value)); + return 0; +} + + /*[clinic input] _sqlite3.Connection.blobopen as blobopen @@ -579,8 +624,8 @@ _sqlite3.Connection.blobopen as blobopen Table name. column as col: str Column name. - row: sqlite3_int64 - Row index. + rowid as row: sqlite3_int64 + Row id. / * readonly: bool = False @@ -594,7 +639,7 @@ Open and return a BLOB object. static PyObject * blobopen_impl(pysqlite_Connection *self, const char *table, const char *col, sqlite3_int64 row, int readonly, const char *name) -/*[clinic end generated code: output=6a02d43efb885d1c input=23576bd1108d8774]*/ +/*[clinic end generated code: output=6a02d43efb885d1c input=cc3d4b47dac08401]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -934,8 +979,9 @@ func_callback(sqlite3_context *context, int argc, sqlite3_value **argv) args = _pysqlite_build_py_params(context, argc, argv); if (args) { callback_context *ctx = (callback_context *)sqlite3_user_data(context); - assert(ctx != NULL); + incref_callback_context(ctx); py_retval = PyObject_CallObject(ctx->callable, args); + decref_callback_context(ctx); Py_DECREF(args); } @@ -962,7 +1008,7 @@ step_callback(sqlite3_context *context, int argc, sqlite3_value **params) PyObject* stepmethod = NULL; callback_context *ctx = (callback_context *)sqlite3_user_data(context); - assert(ctx != NULL); + incref_callback_context(ctx); aggregate_instance = (PyObject**)sqlite3_aggregate_context(context, sizeof(PyObject*)); if (aggregate_instance == NULL) { @@ -1000,6 +1046,7 @@ step_callback(sqlite3_context *context, int argc, sqlite3_value **params) } error: + decref_callback_context(ctx); Py_XDECREF(stepmethod); Py_XDECREF(function_result); @@ -1031,9 +1078,10 @@ final_callback(sqlite3_context *context) PyObject *exc = PyErr_GetRaisedException(); callback_context *ctx = (callback_context *)sqlite3_user_data(context); - assert(ctx != NULL); + incref_callback_context(ctx); function_result = PyObject_CallMethodNoArgs(*aggregate_instance, ctx->state->str_finalize); + decref_callback_context(ctx); Py_DECREF(*aggregate_instance); ok = 0; @@ -1061,32 +1109,36 @@ final_callback(sqlite3_context *context) PyGILState_Release(threadstate); } -static void _pysqlite_drop_unused_cursor_references(pysqlite_Connection* self) +static int +_pysqlite_drop_unused_cursor_references(pysqlite_Connection* self) { /* we only need to do this once in a while */ if (self->created_cursors++ < 200) { - return; + return 0; } self->created_cursors = 0; PyObject* new_list = PyList_New(0); if (!new_list) { - return; + return -1; } - for (Py_ssize_t i = 0; i < PyList_Size(self->cursors); i++) { - PyObject* weakref = PyList_GetItem(self->cursors, i); + assert(PyList_CheckExact(self->cursors)); + Py_ssize_t imax = PyList_GET_SIZE(self->cursors); + for (Py_ssize_t i = 0; i < imax; i++) { + PyObject* weakref = PyList_GET_ITEM(self->cursors, i); if (_PyWeakref_IsDead(weakref)) { continue; } if (PyList_Append(new_list, weakref) != 0) { Py_DECREF(new_list); - return; + return -1; } } Py_SETREF(self->cursors, new_list); + return 0; } /* Allocate a UDF/callback context structure. In order to ensure that the state @@ -1099,12 +1151,16 @@ static callback_context * create_callback_context(PyTypeObject *cls, PyObject *callable) { callback_context *ctx = PyMem_Malloc(sizeof(callback_context)); - if (ctx != NULL) { - PyObject *module = PyType_GetModule(cls); - ctx->callable = Py_NewRef(callable); - ctx->module = Py_NewRef(module); - ctx->state = pysqlite_get_state(module); + if (ctx == NULL) { + PyErr_NoMemory(); + return NULL; } + + PyObject *module = PyType_GetModule(cls); + ctx->refcount = 1; + ctx->callable = Py_NewRef(callable); + ctx->module = Py_NewRef(module); + ctx->state = pysqlite_get_state(module); return ctx; } @@ -1112,11 +1168,33 @@ static void free_callback_context(callback_context *ctx) { assert(ctx != NULL); + assert(ctx->refcount == 0); Py_XDECREF(ctx->callable); Py_XDECREF(ctx->module); PyMem_Free(ctx); } +static inline void +incref_callback_context(callback_context *ctx) +{ + assert(PyGILState_Check()); + assert(ctx != NULL); + assert(ctx->refcount > 0); + ctx->refcount++; +} + +static inline void +decref_callback_context(callback_context *ctx) +{ + assert(PyGILState_Check()); + assert(ctx != NULL); + assert(ctx->refcount > 0); + ctx->refcount--; + if (ctx->refcount == 0) { + free_callback_context(ctx); + } +} + static void set_callback_context(callback_context **ctx_pp, callback_context *ctx) { @@ -1124,7 +1202,7 @@ set_callback_context(callback_context **ctx_pp, callback_context *ctx) callback_context *tmp = *ctx_pp; *ctx_pp = ctx; if (tmp != NULL) { - free_callback_context(tmp); + decref_callback_context(tmp); } } @@ -1135,7 +1213,7 @@ destructor_callback(void *ctx) // This function may be called without the GIL held, so we need to // ensure that we destroy 'ctx' with the GIL held. PyGILState_STATE gstate = PyGILState_Ensure(); - free_callback_context((callback_context *)ctx); + decref_callback_context((callback_context *)ctx); PyGILState_Release(gstate); } } @@ -1197,7 +1275,7 @@ pysqlite_connection_create_function_impl(pysqlite_Connection *self, func_callback, NULL, NULL, - &destructor_callback); // will decref func + &destructor_callback); // will free 'ctx' if (rc != SQLITE_OK) { /* Workaround for SQLite bug: no error code or string is available here */ @@ -1221,7 +1299,7 @@ inverse_callback(sqlite3_context *context, int argc, sqlite3_value **params) PyGILState_STATE gilstate = PyGILState_Ensure(); callback_context *ctx = (callback_context *)sqlite3_user_data(context); - assert(ctx != NULL); + incref_callback_context(ctx); int size = sizeof(PyObject *); PyObject **cls = (PyObject **)sqlite3_aggregate_context(context, size); @@ -1253,6 +1331,7 @@ inverse_callback(sqlite3_context *context, int argc, sqlite3_value **params) Py_DECREF(res); exit: + decref_callback_context(ctx); Py_XDECREF(method); PyGILState_Release(gilstate); } @@ -1269,7 +1348,7 @@ value_callback(sqlite3_context *context) PyGILState_STATE gilstate = PyGILState_Ensure(); callback_context *ctx = (callback_context *)sqlite3_user_data(context); - assert(ctx != NULL); + incref_callback_context(ctx); int size = sizeof(PyObject *); PyObject **cls = (PyObject **)sqlite3_aggregate_context(context, size); @@ -1277,6 +1356,8 @@ value_callback(sqlite3_context *context) assert(*cls != NULL); PyObject *res = PyObject_CallMethodNoArgs(*cls, ctx->state->str_value); + decref_callback_context(ctx); + if (res == NULL) { int attr_err = PyErr_ExceptionMatches(PyExc_AttributeError); set_sqlite_error(context, attr_err @@ -1399,7 +1480,7 @@ pysqlite_connection_create_aggregate_impl(pysqlite_Connection *self, 0, &step_callback, &final_callback, - &destructor_callback); // will decref func + &destructor_callback); // will free 'ctx' if (rc != SQLITE_OK) { /* Workaround for SQLite bug: no error code or string is available here */ PyErr_SetString(self->OperationalError, "Error creating aggregate"); @@ -1409,7 +1490,7 @@ pysqlite_connection_create_aggregate_impl(pysqlite_Connection *self, } static int -authorizer_callback(void *ctx, int action, const char *arg1, +authorizer_callback(void *ctx_vp, int action, const char *arg1, const char *arg2 , const char *dbname, const char *access_attempt_source) { @@ -1418,8 +1499,9 @@ authorizer_callback(void *ctx, int action, const char *arg1, PyObject *ret; int rc = SQLITE_DENY; - assert(ctx != NULL); - PyObject *callable = ((callback_context *)ctx)->callable; + callback_context *ctx = (callback_context *)ctx_vp; + incref_callback_context(ctx); + PyObject *callable = ctx->callable; ret = PyObject_CallFunction(callable, "issss", action, arg1, arg2, dbname, access_attempt_source); @@ -1441,21 +1523,23 @@ authorizer_callback(void *ctx, int action, const char *arg1, Py_DECREF(ret); } + decref_callback_context(ctx); PyGILState_Release(gilstate); return rc; } static int -progress_callback(void *ctx) +progress_callback(void *ctx_vp) { PyGILState_STATE gilstate = PyGILState_Ensure(); int rc; PyObject *ret; - assert(ctx != NULL); - PyObject *callable = ((callback_context *)ctx)->callable; - ret = PyObject_CallNoArgs(callable); + callback_context *ctx = (callback_context *)ctx_vp; + incref_callback_context(ctx); + + ret = PyObject_CallNoArgs(ctx->callable); if (!ret) { /* abort query if error occurred */ rc = -1; @@ -1468,6 +1552,7 @@ progress_callback(void *ctx) print_or_clear_traceback(ctx); } + decref_callback_context(ctx); PyGILState_Release(gilstate); return rc; } @@ -1479,7 +1564,7 @@ progress_callback(void *ctx) * to ensure future compatibility. */ static int -trace_callback(unsigned int type, void *ctx, void *stmt, void *sql) +trace_callback(unsigned int type, void *ctx_vp, void *stmt, void *sql) { if (type != SQLITE_TRACE_STMT) { return 0; @@ -1487,8 +1572,9 @@ trace_callback(unsigned int type, void *ctx, void *stmt, void *sql) PyGILState_STATE gilstate = PyGILState_Ensure(); - assert(ctx != NULL); - pysqlite_state *state = ((callback_context *)ctx)->state; + callback_context *ctx = (callback_context *)ctx_vp; + incref_callback_context(ctx); + pysqlite_state *state = ctx->state; assert(state != NULL); PyObject *py_statement = NULL; @@ -1502,7 +1588,7 @@ trace_callback(unsigned int type, void *ctx, void *stmt, void *sql) PyErr_SetString(state->DataError, "Expanded SQL string exceeds the maximum string length"); - print_or_clear_traceback((callback_context *)ctx); + print_or_clear_traceback(ctx); // Fall back to unexpanded sql py_statement = PyUnicode_FromString((const char *)sql); @@ -1512,16 +1598,16 @@ trace_callback(unsigned int type, void *ctx, void *stmt, void *sql) sqlite3_free((void *)expanded_sql); } if (py_statement) { - PyObject *callable = ((callback_context *)ctx)->callable; - PyObject *ret = PyObject_CallOneArg(callable, py_statement); + PyObject *ret = PyObject_CallOneArg(ctx->callable, py_statement); Py_DECREF(py_statement); Py_XDECREF(ret); } if (PyErr_Occurred()) { - print_or_clear_traceback((callback_context *)ctx); + print_or_clear_traceback(ctx); } exit: + decref_callback_context(ctx); PyGILState_Release(gilstate); return 0; } @@ -1574,8 +1660,8 @@ _sqlite3.Connection.set_progress_handler as pysqlite_connection_set_progress_han cls: defining_class progress_handler as callable: object A callable that takes no arguments. - If the callable returns non-zero, the current query is terminated, - and an exception is raised. + If the callable returns non-zero, the current query is + terminated, and an exception is raised. / [from 3.15] n: int The number of SQLite virtual machine instructions that are @@ -1583,14 +1669,15 @@ _sqlite3.Connection.set_progress_handler as pysqlite_connection_set_progress_han Set progress handler callback. -If 'progress_handler' is None or 'n' is 0, the progress handler is disabled. +If 'progress_handler' is None or 'n' is 0, the progress handler is +disabled. [clinic start generated code]*/ static PyObject * pysqlite_connection_set_progress_handler_impl(pysqlite_Connection *self, PyTypeObject *cls, PyObject *callable, int n) -/*[clinic end generated code: output=0739957fd8034a50 input=b4d6e2ef8b4d32f9]*/ +/*[clinic end generated code: output=0739957fd8034a50 input=d46887797f0c0ecf]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -1945,6 +2032,8 @@ collation_callback(void *context, int text1_length, const void *text1_data, PyObject* retval = NULL; long longval; int result = 0; + callback_context *ctx = (callback_context *)context; + incref_callback_context(ctx); /* This callback may be executed multiple times per sqlite3_step(). Bail if * the previous call failed */ @@ -1961,8 +2050,6 @@ collation_callback(void *context, int text1_length, const void *text1_data, goto finally; } - callback_context *ctx = (callback_context *)context; - assert(ctx != NULL); PyObject *args[] = { NULL, string1, string2 }; // Borrowed refs. size_t nargsf = 2 | PY_VECTORCALL_ARGUMENTS_OFFSET; retval = PyObject_Vectorcall(ctx->callable, args + 1, nargsf, NULL); @@ -1984,6 +2071,7 @@ collation_callback(void *context, int text1_length, const void *text1_data, } finally: + decref_callback_context(ctx); Py_XDECREF(string1); Py_XDECREF(string2); Py_XDECREF(retval); @@ -2207,7 +2295,7 @@ pysqlite_connection_create_collation_impl(pysqlite_Connection *self, * the context before returning. */ if (callable != Py_None) { - free_callback_context(ctx); + decref_callback_context(ctx); } set_error_from_db(self->state, self->db); return NULL; @@ -2226,15 +2314,15 @@ _sqlite3.Connection.serialize as serialize Serialize a database into a byte string. -For an ordinary on-disk database file, the serialization is just a copy of the -disk file. For an in-memory database or a "temp" database, the serialization is -the same sequence of bytes which would be written to disk if that database -were backed up to disk. +For an ordinary on-disk database file, the serialization is just +a copy of the disk file. For an in-memory database or a "temp" +database, the serialization is the same sequence of bytes which +would be written to disk if that database were backed up to disk. [clinic start generated code]*/ static PyObject * serialize_impl(pysqlite_Connection *self, const char *name) -/*[clinic end generated code: output=97342b0e55239dd3 input=d2eb5194a65abe2b]*/ +/*[clinic end generated code: output=97342b0e55239dd3 input=7e48654e8e082fa8]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -2279,18 +2367,19 @@ _sqlite3.Connection.deserialize as deserialize Load a serialized database. -The deserialize interface causes the database connection to disconnect from the -target database, and then reopen it as an in-memory database based on the given -serialized data. +The deserialize interface causes the database connection to +disconnect from the target database, and then reopen it as +an in-memory database based on the given serialized data. -The deserialize interface will fail with SQLITE_BUSY if the database is -currently in a read transaction or is involved in a backup operation. +The deserialize interface will fail with SQLITE_BUSY if the database +is currently in a read transaction or is involved in a backup +operation. [clinic start generated code]*/ static PyObject * deserialize_impl(pysqlite_Connection *self, Py_buffer *data, const char *name) -/*[clinic end generated code: output=e394c798b98bad89 input=1be4ca1faacf28f2]*/ +/*[clinic end generated code: output=e394c798b98bad89 input=5d20e028d98c0686]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -2363,13 +2452,14 @@ _sqlite3.Connection.__exit__ as pysqlite_connection_exit Called when the connection is used as a context manager. -If there was any exception, a rollback takes place; otherwise we commit. +If there was any exception, a rollback takes place; otherwise we +commit. [clinic start generated code]*/ static PyObject * pysqlite_connection_exit_impl(pysqlite_Connection *self, PyObject *exc_type, PyObject *exc_value, PyObject *exc_tb) -/*[clinic end generated code: output=0705200e9321202a input=bd66f1532c9c54a7]*/ +/*[clinic end generated code: output=0705200e9321202a input=8fdb0392ee6f3466]*/ { int commit = 0; PyObject* result; @@ -2409,20 +2499,20 @@ _sqlite3.Connection.setlimit as setlimit category: int The limit category to be set. limit: int - The new limit. If the new limit is a negative number, the limit is - unchanged. + The new limit. If the new limit is a negative number, the limit + is unchanged. / Set connection run-time limits. -Attempts to increase a limit above its hard upper bound are silently truncated -to the hard upper bound. Regardless of whether or not the limit was changed, -the prior value of the limit is returned. +Attempts to increase a limit above its hard upper bound are silently +truncated to the hard upper bound. Regardless of whether or not the +limit was changed, the prior value of the limit is returned. [clinic start generated code]*/ static PyObject * setlimit_impl(pysqlite_Connection *self, int category, int limit) -/*[clinic end generated code: output=0d208213f8d68ccd input=9bd469537e195635]*/ +/*[clinic end generated code: output=0d208213f8d68ccd input=5c2e430091206677]*/ { if (!pysqlite_check_thread(self) || !pysqlite_check_connection(self)) { return NULL; @@ -2623,6 +2713,10 @@ static PyGetSetDef connection_getset[] = { {"in_transaction", pysqlite_connection_get_in_transaction, NULL}, {"autocommit", get_autocommit, set_autocommit}, {"__text_signature__", get_sig, NULL}, + {"row_factory", connection_get_row_factory, + connection_set_row_factory}, + {"text_factory", connection_get_text_factory, + connection_set_text_factory}, {NULL} }; @@ -2670,8 +2764,6 @@ static struct PyMemberDef connection_members[] = {"InternalError", _Py_T_OBJECT, offsetof(pysqlite_Connection, InternalError), Py_READONLY}, {"ProgrammingError", _Py_T_OBJECT, offsetof(pysqlite_Connection, ProgrammingError), Py_READONLY}, {"NotSupportedError", _Py_T_OBJECT, offsetof(pysqlite_Connection, NotSupportedError), Py_READONLY}, - {"row_factory", _Py_T_OBJECT, offsetof(pysqlite_Connection, row_factory)}, - {"text_factory", _Py_T_OBJECT, offsetof(pysqlite_Connection, text_factory)}, {NULL} }; diff --git a/Modules/_sqlite/connection.h b/Modules/_sqlite/connection.h index 7a748ee3ea0c581..703396a0c8db53d 100644 --- a/Modules/_sqlite/connection.h +++ b/Modules/_sqlite/connection.h @@ -36,6 +36,7 @@ typedef struct _callback_context PyObject *callable; PyObject *module; pysqlite_state *state; + Py_ssize_t refcount; } callback_context; enum autocommit_mode { diff --git a/Modules/_sqlite/cursor.c b/Modules/_sqlite/cursor.c index 7943bfcca3679dd..cb0f9adcc45a96b 100644 --- a/Modules/_sqlite/cursor.c +++ b/Modules/_sqlite/cursor.c @@ -31,6 +31,7 @@ #include "util.h" #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() typedef enum { TYPE_LONG, @@ -185,9 +186,7 @@ cursor_dealloc(PyObject *op) pysqlite_Cursor *self = _pysqlite_Cursor_CAST(op); PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - if (self->in_weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, self->in_weakreflist); (void)tp->tp_clear(op); tp->tp_free(self); Py_DECREF(tp); @@ -472,6 +471,9 @@ static int check_cursor(pysqlite_Cursor* cur) return 0; } + assert(cur->connection != NULL); + assert(cur->connection->state != NULL); + if (cur->closed) { PyErr_SetString(cur->connection->state->ProgrammingError, "Cannot operate on a closed cursor."); @@ -568,43 +570,40 @@ bind_param(pysqlite_state *state, pysqlite_Statement *self, int pos, switch (paramtype) { case TYPE_LONG: { sqlite_int64 value = _pysqlite_long_as_int64(parameter); - if (value == -1 && PyErr_Occurred()) - rc = -1; - else - rc = sqlite3_bind_int64(self->st, pos, value); + rc = (value == -1 && PyErr_Occurred()) + ? SQLITE_ERROR + : sqlite3_bind_int64(self->st, pos, value); break; } case TYPE_FLOAT: { double value = PyFloat_AsDouble(parameter); - if (value == -1 && PyErr_Occurred()) { - rc = -1; - } - else { - rc = sqlite3_bind_double(self->st, pos, value); - } + rc = (value == -1 && PyErr_Occurred()) + ? SQLITE_ERROR + : sqlite3_bind_double(self->st, pos, value); break; } case TYPE_UNICODE: string = PyUnicode_AsUTF8AndSize(parameter, &buflen); - if (string == NULL) - return -1; + if (string == NULL) { + return SQLITE_ERROR; + } if (buflen > INT_MAX) { PyErr_SetString(PyExc_OverflowError, "string longer than INT_MAX bytes"); - return -1; + return SQLITE_ERROR; } rc = sqlite3_bind_text(self->st, pos, string, (int)buflen, SQLITE_TRANSIENT); break; case TYPE_BUFFER: { Py_buffer view; if (PyObject_GetBuffer(parameter, &view, PyBUF_SIMPLE) != 0) { - return -1; + return SQLITE_ERROR; } if (view.len > INT_MAX) { PyErr_SetString(PyExc_OverflowError, "BLOB longer than INT_MAX bytes"); PyBuffer_Release(&view); - return -1; + return SQLITE_ERROR; } rc = sqlite3_bind_blob(self->st, pos, view.buf, (int)view.len, SQLITE_TRANSIENT); PyBuffer_Release(&view); @@ -614,7 +613,7 @@ bind_param(pysqlite_state *state, pysqlite_Statement *self, int pos, PyErr_Format(state->ProgrammingError, "Error binding parameter %d: type '%s' is not supported", pos, Py_TYPE(parameter)->tp_name); - rc = -1; + rc = SQLITE_ERROR; } final: @@ -734,14 +733,17 @@ bind_parameters(pysqlite_state *state, pysqlite_Statement *self, } binding_name++; /* skip first char (the colon) */ - PyObject *current_param; - (void)PyMapping_GetOptionalItemString(parameters, binding_name, &current_param); - if (!current_param) { - if (!PyErr_Occurred() || PyErr_ExceptionMatches(PyExc_LookupError)) { - PyErr_Format(state->ProgrammingError, - "You did not supply a value for binding " - "parameter :%s.", binding_name); - } + PyObject *current_param = NULL; + int found = PyMapping_GetOptionalItemString(parameters, + binding_name, + &current_param); + if (found == -1) { + return; + } + else if (found == 0) { + PyErr_Format(state->ProgrammingError, + "You did not supply a value for binding " + "parameter :%s.", binding_name); return; } @@ -1160,35 +1162,31 @@ pysqlite_cursor_fetchone_impl(pysqlite_Cursor *self) /*[clinic input] _sqlite3.Cursor.fetchmany as pysqlite_cursor_fetchmany - size as maxrows: int(c_default='((pysqlite_Cursor *)self)->arraysize') = 1 + size as maxrows: uint32(c_default='((pysqlite_Cursor *)self)->arraysize') = 1 The default value is set by the Cursor.arraysize attribute. Fetches several rows from the resultset. [clinic start generated code]*/ static PyObject * -pysqlite_cursor_fetchmany_impl(pysqlite_Cursor *self, int maxrows) -/*[clinic end generated code: output=a8ef31fea64d0906 input=035dbe44a1005bf2]*/ +pysqlite_cursor_fetchmany_impl(pysqlite_Cursor *self, uint32_t maxrows) +/*[clinic end generated code: output=3325f2b477c71baf input=a509c412aa70b27e]*/ { PyObject* row; PyObject* list; - int counter = 0; list = PyList_New(0); if (!list) { return NULL; } - while ((row = pysqlite_cursor_iternext((PyObject *)self))) { - if (PyList_Append(list, row) < 0) { - Py_DECREF(row); - break; - } + while (maxrows > 0 && (row = pysqlite_cursor_iternext((PyObject *)self))) { + int rc = PyList_Append(list, row); Py_DECREF(row); - - if (++counter == maxrows) { + if (rc < 0) { break; } + maxrows--; } if (PyErr_Occurred()) { @@ -1302,6 +1300,30 @@ pysqlite_cursor_close_impl(pysqlite_Cursor *self) Py_RETURN_NONE; } +/*[clinic input] +@getter +_sqlite3.Cursor.arraysize +[clinic start generated code]*/ + +static PyObject * +_sqlite3_Cursor_arraysize_get_impl(pysqlite_Cursor *self) +/*[clinic end generated code: output=e0919d97175e6c50 input=3278f8d3ecbd90e3]*/ +{ + return PyLong_FromUInt32(self->arraysize); +} + +/*[clinic input] +@setter +_sqlite3.Cursor.arraysize +[clinic start generated code]*/ + +static int +_sqlite3_Cursor_arraysize_set_impl(pysqlite_Cursor *self, PyObject *value) +/*[clinic end generated code: output=af59a6b09f8cce6e input=ace48cb114e26060]*/ +{ + return PyLong_AsUInt32(value, &self->arraysize); +} + static PyMethodDef cursor_methods[] = { PYSQLITE_CURSOR_CLOSE_METHODDEF PYSQLITE_CURSOR_EXECUTEMANY_METHODDEF @@ -1319,7 +1341,6 @@ static struct PyMemberDef cursor_members[] = { {"connection", _Py_T_OBJECT, offsetof(pysqlite_Cursor, connection), Py_READONLY}, {"description", _Py_T_OBJECT, offsetof(pysqlite_Cursor, description), Py_READONLY}, - {"arraysize", Py_T_INT, offsetof(pysqlite_Cursor, arraysize), 0}, {"lastrowid", _Py_T_OBJECT, offsetof(pysqlite_Cursor, lastrowid), Py_READONLY}, {"rowcount", Py_T_LONG, offsetof(pysqlite_Cursor, rowcount), Py_READONLY}, {"row_factory", _Py_T_OBJECT, offsetof(pysqlite_Cursor, row_factory), 0}, @@ -1327,6 +1348,11 @@ static struct PyMemberDef cursor_members[] = {NULL} }; +static struct PyGetSetDef cursor_getsets[] = { + _SQLITE3_CURSOR_ARRAYSIZE_GETSETDEF + {NULL}, +}; + static const char cursor_doc[] = PyDoc_STR("SQLite database cursor class."); @@ -1337,6 +1363,7 @@ static PyType_Slot cursor_slots[] = { {Py_tp_iternext, pysqlite_cursor_iternext}, {Py_tp_methods, cursor_methods}, {Py_tp_members, cursor_members}, + {Py_tp_getset, cursor_getsets}, {Py_tp_init, pysqlite_cursor_init}, {Py_tp_traverse, cursor_traverse}, {Py_tp_clear, cursor_clear}, diff --git a/Modules/_sqlite/cursor.h b/Modules/_sqlite/cursor.h index 42f817af7c54ad4..c840a3d7ed0d156 100644 --- a/Modules/_sqlite/cursor.h +++ b/Modules/_sqlite/cursor.h @@ -35,7 +35,7 @@ typedef struct pysqlite_Connection* connection; PyObject* description; PyObject* row_cast_map; - int arraysize; + uint32_t arraysize; PyObject* lastrowid; long rowcount; PyObject* row_factory; diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 27e8dab92e0e678..fcc1d69c707ebe6 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -745,7 +745,6 @@ module_exec(PyObject *module) return 0; error: - sqlite3_shutdown(); return -1; } diff --git a/Modules/_sqlite/row.c b/Modules/_sqlite/row.c index 94565a01d18f245..8646f591ef5169f 100644 --- a/Modules/_sqlite/row.c +++ b/Modules/_sqlite/row.c @@ -232,7 +232,15 @@ static Py_hash_t pysqlite_row_hash(PyObject *op) { pysqlite_Row *self = _pysqlite_Row_CAST(op); - return PyObject_Hash(self->description) ^ PyObject_Hash(self->data); + Py_hash_t hash_description = PyObject_Hash(self->description); + if (hash_description == -1) { + return -1; + } + Py_hash_t hash_data = PyObject_Hash(self->data); + if (hash_data == -1) { + return -1; + } + return hash_description ^ hash_data; } static PyObject * diff --git a/Modules/_sre/sre.c b/Modules/_sre/sre.c index 602d0ab8588f62f..ace6b17ee2d93cf 100644 --- a/Modules/_sre/sre.c +++ b/Modules/_sre/sre.c @@ -44,6 +44,7 @@ static const char copyright[] = #include "pycore_long.h" // _PyLong_GetZero() #include "pycore_moduleobject.h" // _PyModule_GetState() #include "pycore_unicodeobject.h" // _PyUnicode_Copy +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "sre.h" // SRE_CODE @@ -736,10 +737,7 @@ pattern_dealloc(PyObject *self) { PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - PatternObject *obj = _PatternObject_CAST(self); - if (obj->weakreflist != NULL) { - PyObject_ClearWeakRefs(self); - } + FT_CLEAR_WEAKREFS(self, _PatternObject_CAST(self)->weakreflist); (void)pattern_clear(self); tp->tp_free(self); Py_DECREF(tp); @@ -1944,7 +1942,7 @@ _validate_inner(SRE_CODE *code, SRE_CODE *end, Py_ssize_t groups) sre_match() code is robust even if they don't, and the worst you can get is nonsensical match results. */ GET_ARG; - if (arg > 2 * (size_t)groups + 1) { + if (arg >= 2 * (size_t)groups) { VTRACE(("arg=%d, groups=%d\n", (int)arg, (int)groups)); FAIL; } @@ -2357,7 +2355,7 @@ match_getindex(MatchObject* self, PyObject* index) } // Check that i*2 cannot overflow to make static analyzers happy - assert(i <= SRE_MAXGROUPS); + assert((size_t)i <= SRE_MAXGROUPS); return i; } @@ -2837,20 +2835,25 @@ scanner_dealloc(PyObject *self) static int scanner_begin(ScannerObject* self) { - if (self->executing) { +#ifdef Py_GIL_DISABLED + int was_executing = _Py_atomic_exchange_int(&self->executing, 1); +#else + int was_executing = self->executing; + self->executing = 1; +#endif + if (was_executing) { PyErr_SetString(PyExc_ValueError, "regular expression scanner already executing"); return 0; } - self->executing = 1; return 1; } static void scanner_end(ScannerObject* self) { - assert(self->executing); - self->executing = 0; + assert(FT_ATOMIC_LOAD_INT_RELAXED(self->executing)); + FT_ATOMIC_STORE_INT(self->executing, 0); } /*[clinic input] @@ -3174,7 +3177,7 @@ static PyMethodDef pattern_methods[] = { _SRE_SRE_PATTERN___DEEPCOPY___METHODDEF _SRE_SRE_PATTERN__FAIL_AFTER_METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, - PyDoc_STR("See PEP 585")}, + PyDoc_STR("Patterns are generic over the type of string they handle (str or bytes)")}, {NULL, NULL} }; @@ -3230,7 +3233,7 @@ static PyMethodDef match_methods[] = { _SRE_SRE_MATCH___COPY___METHODDEF _SRE_SRE_MATCH___DEEPCOPY___METHODDEF {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, - PyDoc_STR("See PEP 585")}, + PyDoc_STR("Matches are generic over the type of string which was matched (str or bytes)")}, {NULL, NULL} }; diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 97a29f4d0e1830a..5258df3958ae1ad 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -26,6 +26,7 @@ #define OPENSSL_NO_DEPRECATED 1 #include "Python.h" +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() #include "pycore_fileutils.h" // _PyIsSelectable_fd() #include "pycore_long.h" // _PyLong_UnsignedLongLong_Converter() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() @@ -43,14 +44,14 @@ /* Redefined below for Windows debug builds after important #includes */ #define _PySSL_FIX_ERRNO -#define PySSL_BEGIN_ALLOW_THREADS_S(save) \ - do { (save) = PyEval_SaveThread(); } while(0) -#define PySSL_END_ALLOW_THREADS_S(save) \ - do { PyEval_RestoreThread(save); _PySSL_FIX_ERRNO; } while(0) -#define PySSL_BEGIN_ALLOW_THREADS { \ +#define PySSL_BEGIN_ALLOW_THREADS_S(save, mutex) \ + do { (save) = PyEval_SaveThread(); PyMutex_Lock(mutex); } while(0) +#define PySSL_END_ALLOW_THREADS_S(save, mutex) \ + do { PyMutex_Unlock(mutex); PyEval_RestoreThread(save); _PySSL_FIX_ERRNO; } while(0) +#define PySSL_BEGIN_ALLOW_THREADS(self) { \ PyThreadState *_save = NULL; \ - PySSL_BEGIN_ALLOW_THREADS_S(_save); -#define PySSL_END_ALLOW_THREADS PySSL_END_ALLOW_THREADS_S(_save); } + PySSL_BEGIN_ALLOW_THREADS_S(_save, &self->tstate_mutex); +#define PySSL_END_ALLOW_THREADS(self) PySSL_END_ALLOW_THREADS_S(_save, &self->tstate_mutex); } #if defined(HAVE_POLL_H) #include <poll.h> @@ -123,7 +124,7 @@ static void _PySSLFixErrno(void) { /* Include generated data (error codes) */ /* See make_ssl_data.h for notes on adding a new version. */ #if (OPENSSL_VERSION_NUMBER >= 0x30401000L) -#include "_ssl_data_34.h" +#include "_ssl_data_35.h" #elif (OPENSSL_VERSION_NUMBER >= 0x30100000L) #include "_ssl_data_340.h" #elif (OPENSSL_VERSION_NUMBER >= 0x30000000L) @@ -301,7 +302,7 @@ typedef struct { int post_handshake_auth; #endif PyObject *msg_cb; - PyObject *keylog_filename; + PyObject *keylog_filename; // can be anything accepted by Py_fopen() BIO *keylog_bio; /* Cached module state, also used in SSLSocket and SSLSession code. */ _sslmodulestate *state; @@ -309,6 +310,9 @@ typedef struct { PyObject *psk_client_callback; PyObject *psk_server_callback; #endif + /* Lock to synchronize calls when the thread state is detached. + See also gh-134698. */ + PyMutex tstate_mutex; } PySSLContext; #define PySSLContext_CAST(op) ((PySSLContext *)(op)) @@ -328,7 +332,7 @@ typedef struct { PySSLContext *ctx; /* weakref to SSL context */ char shutdown_seen_zero; enum py_ssl_server_or_client socket_type; - PyObject *owner; /* Python level "owner" passed to servername callback */ + PyObject *owner; /* weakref to Python level "owner" passed to servername callback */ PyObject *server_hostname; _PySSLError err; /* last seen error from various sources */ /* Some SSL callbacks don't have error reporting. Callback wrappers @@ -563,7 +567,7 @@ fill_and_set_sslerror(_sslmodulestate *state, goto fail; } } - if (PyUnicodeWriter_WriteUTF8(writer, "] ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, "] ", 2) < 0) { goto fail; } } @@ -574,7 +578,7 @@ fill_and_set_sslerror(_sslmodulestate *state, } else { if (PyUnicodeWriter_Format( - writer, "unknown error (0x%x)", errcode) < 0) { + writer, "unknown error (0x%lx)", errcode) < 0) { goto fail; } } @@ -889,12 +893,12 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, /* Make sure the SSL error state is initialized */ ERR_clear_error(); - PySSL_BEGIN_ALLOW_THREADS + PySSL_BEGIN_ALLOW_THREADS(sslctx) self->ssl = SSL_new(ctx); - PySSL_END_ALLOW_THREADS + PySSL_END_ALLOW_THREADS(sslctx) if (self->ssl == NULL) { + _setSSLError(get_state_ctx(sslctx), NULL, 0, __FILE__, __LINE__); Py_DECREF(self); - _setSSLError(get_state_ctx(self), NULL, 0, __FILE__, __LINE__); return NULL; } @@ -907,7 +911,7 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, } /* bpo43522 and OpenSSL < 1.1.1l: copy hostflags manually */ -#if OPENSSL_VERSION < 0x101010cf +#if OPENSSL_VERSION_NUMBER < 0x101010cf X509_VERIFY_PARAM *ssl_verification_params = SSL_get0_param(self->ssl); X509_VERIFY_PARAM *ssl_ctx_verification_params = SSL_CTX_get0_param(ctx); @@ -960,12 +964,12 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, BIO_set_nbio(SSL_get_wbio(self->ssl), 1); } - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS; if (socket_type == PY_SSL_CLIENT) SSL_set_connect_state(self->ssl); else SSL_set_accept_state(self->ssl); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS; self->socket_type = socket_type; if (sock != NULL) { @@ -1034,10 +1038,11 @@ _ssl__SSLSocket_do_handshake_impl(PySSLSocket *self) /* Actually negotiate SSL connection */ /* XXX If SSL_do_handshake() returns 0, it's also a failure. */ do { - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS ret = SSL_do_handshake(self->ssl); err = _PySSL_errno(ret < 1, self->ssl, ret); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS; + _PySSL_FIX_ERRNO; self->err = err; if (PyErr_CheckSignals()) @@ -1406,14 +1411,14 @@ _get_peer_alt_names (_sslmodulestate *state, X509 *certificate) { } PyTuple_SET_ITEM(t, 0, v); - if (name->d.ip->length == 4) { - unsigned char *p = name->d.ip->data; + if (ASN1_STRING_length(name->d.ip) == 4) { + const unsigned char *p = ASN1_STRING_get0_data(name->d.ip); v = PyUnicode_FromFormat( "%d.%d.%d.%d", p[0], p[1], p[2], p[3] ); - } else if (name->d.ip->length == 16) { - unsigned char *p = name->d.ip->data; + } else if (ASN1_STRING_length(name->d.ip) == 16) { + const unsigned char *p = ASN1_STRING_get0_data(name->d.ip); v = PyUnicode_FromFormat( "%X:%X:%X:%X:%X:%X:%X:%X", p[0] << 8 | p[1], @@ -1544,8 +1549,9 @@ _get_aia_uri(X509 *certificate, int nid) { continue; } uri = ad->location->d.uniformResourceIdentifier; - ostr = PyUnicode_FromStringAndSize((char *)uri->data, - uri->length); + ostr = PyUnicode_FromStringAndSize( + (const char *)ASN1_STRING_get0_data(uri), + ASN1_STRING_length(uri)); if (ostr == NULL) { goto fail; } @@ -1611,8 +1617,9 @@ _get_crl_dp(X509 *certificate) { continue; } uri = gn->d.uniformResourceIdentifier; - ouri = PyUnicode_FromStringAndSize((char *)uri->data, - uri->length); + ouri = PyUnicode_FromStringAndSize( + (const char *)ASN1_STRING_get0_data(uri), + ASN1_STRING_length(uri)); if (ouri == NULL) goto done; @@ -1827,14 +1834,14 @@ _certificate_to_der(_sslmodulestate *state, X509 *certificate) /*[clinic input] _ssl._test_decode_cert - path: object(converter="PyUnicode_FSConverter") + path: unicode_fs_encoded / [clinic start generated code]*/ static PyObject * _ssl__test_decode_cert_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=96becb9abb23c091 input=cdeaaf02d4346628]*/ +/*[clinic end generated code: output=96becb9abb23c091 input=cb4988d5e651a4f8]*/ { PyObject *retval = NULL; X509 *x=NULL; @@ -1864,7 +1871,6 @@ _ssl__test_decode_cert_impl(PyObject *module, PyObject *path) X509_free(x); fail0: - Py_DECREF(path); if (cert != NULL) BIO_free(cert); return retval; } @@ -2217,14 +2223,15 @@ _ssl._SSLSocket.context This changes the context associated with the SSLSocket. -This is typically used from within a callback function set by the sni_callback -on the SSLContext to change the certificate information associated with the -SSLSocket before the cryptographic exchange handshake messages. +This is typically used from within a callback function set by the +sni_callback on the SSLContext to change the certificate information +associated with the SSLSocket before the cryptographic exchange +handshake messages. [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_context_get_impl(PySSLSocket *self) -/*[clinic end generated code: output=d23e82f72f32e3d7 input=7cbb97407c2ace30]*/ +/*[clinic end generated code: output=d23e82f72f32e3d7 input=b845dea1f9710ebe]*/ { return Py_NewRef(self->ctx); } @@ -2340,6 +2347,10 @@ static int PySSL_clear(PyObject *op) { PySSLSocket *self = PySSLSocket_CAST(op); + Py_CLEAR(self->Socket); + Py_CLEAR(self->ctx); + Py_CLEAR(self->owner); + Py_CLEAR(self->server_hostname); Py_CLEAR(self->exc); return 0; } @@ -2364,10 +2375,7 @@ PySSL_dealloc(PyObject *op) SSL_set_shutdown(self->ssl, SSL_SENT_SHUTDOWN | SSL_get_shutdown(self->ssl)); SSL_free(self->ssl); } - Py_XDECREF(self->Socket); - Py_XDECREF(self->ctx); - Py_XDECREF(self->server_hostname); - Py_XDECREF(self->owner); + (void)PySSL_clear(op); PyObject_GC_Del(self); Py_DECREF(tp); } @@ -2414,9 +2422,10 @@ PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout) ms = (int)_PyTime_AsMilliseconds(timeout, _PyTime_ROUND_CEILING); assert(ms <= INT_MAX); - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS rc = poll(&pollfd, 1, (int)ms); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; #else /* Guard against socket too large for select*/ if (!_PyIsSelectable_fd(s->sock_fd)) @@ -2428,13 +2437,14 @@ PySSL_select(PySocketSockObject *s, int writing, PyTime_t timeout) FD_SET(s->sock_fd, &fds); /* Wait until the socket becomes ready */ - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS nfds = Py_SAFE_DOWNCAST(s->sock_fd+1, SOCKET_T, int); if (writing) rc = select(nfds, NULL, &fds, NULL, &tv); else rc = select(nfds, &fds, NULL, NULL, &tv); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; #endif /* Return SOCKET_TIMED_OUT on timeout, SOCKET_OPERATION_OK otherwise @@ -2505,10 +2515,11 @@ _ssl__SSLSocket_write_impl(PySSLSocket *self, Py_buffer *b) } do { - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS; retval = SSL_write_ex(self->ssl, b->buf, (size_t)b->len, &count); err = _PySSL_errno(retval == 0, self->ssl, retval); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS; + _PySSL_FIX_ERRNO; self->err = err; if (PyErr_CheckSignals()) @@ -2566,10 +2577,11 @@ _ssl__SSLSocket_pending_impl(PySSLSocket *self) int count = 0; _PySSLError err; - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS; count = SSL_pending(self->ssl); err = _PySSL_errno(count < 0, self->ssl, count); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS; + _PySSL_FIX_ERRNO; self->err = err; if (count < 0) @@ -2660,10 +2672,11 @@ _ssl__SSLSocket_read_impl(PySSLSocket *self, Py_ssize_t len, deadline = _PyDeadline_Init(timeout); do { - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS; retval = SSL_read_ex(self->ssl, mem, (size_t)len, &count); err = _PySSL_errno(retval == 0, self->ssl, retval); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS; + _PySSL_FIX_ERRNO; self->err = err; if (PyErr_CheckSignals()) @@ -2762,7 +2775,7 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) } while (1) { - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS; /* Disable read-ahead so that unwrap can work correctly. * Otherwise OpenSSL might read in too much data, * eating clear text data that happens to be @@ -2775,7 +2788,8 @@ _ssl__SSLSocket_shutdown_impl(PySSLSocket *self) SSL_set_read_ahead(self->ssl, 0); ret = SSL_shutdown(self->ssl); err = _PySSL_errno(ret < 0, self->ssl, ret); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS; + _PySSL_FIX_ERRNO; self->err = err; /* If err == 1, a secure shutdown with SSL_shutdown() is complete */ @@ -2848,15 +2862,16 @@ _ssl._SSLSocket.get_channel_binding Get channel binding data for current connection. -Raise ValueError if the requested `cb_type` is not supported. Return bytes -of the data or None if the data is not available (e.g. before the handshake). +Raise ValueError if the requested `cb_type` is not supported. +Return bytes of the data or None if the data is not available (e.g. +before the handshake). Only 'tls-unique' channel binding data from RFC 5929 is supported. [clinic start generated code]*/ static PyObject * _ssl__SSLSocket_get_channel_binding_impl(PySSLSocket *self, const char *cb_type) -/*[clinic end generated code: output=34bac9acb6a61d31 input=e008004fc08744db]*/ +/*[clinic end generated code: output=34bac9acb6a61d31 input=bed81ef7936535a0]*/ { char buf[PySSL_CB_MAXLEN]; size_t len; @@ -3167,9 +3182,10 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) // no other thread can be touching this object yet. // (Technically, we can't even lock if we wanted to, as the // lock hasn't been initialized yet.) - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS ctx = SSL_CTX_new(method); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; if (ctx == NULL) { _setSSLError(get_ssl_state(module), NULL, 0, __FILE__, __LINE__); @@ -3194,6 +3210,7 @@ _ssl__SSLContext_impl(PyTypeObject *type, int proto_version) self->psk_client_callback = NULL; self->psk_server_callback = NULL; #endif + self->tstate_mutex = (PyMutex){0}; /* Don't check host name by default */ if (proto_version == PY_SSL_VERSION_TLS_CLIENT) { @@ -3296,6 +3313,11 @@ context_traverse(PyObject *op, visitproc visit, void *arg) PySSLContext *self = PySSLContext_CAST(op); Py_VISIT(self->set_sni_cb); Py_VISIT(self->msg_cb); + Py_VISIT(self->keylog_filename); +#ifndef OPENSSL_NO_PSK + Py_VISIT(self->psk_client_callback); + Py_VISIT(self->psk_server_callback); +#endif Py_VISIT(Py_TYPE(self)); return 0; } @@ -3312,9 +3334,10 @@ context_clear(PyObject *op) Py_CLEAR(self->psk_server_callback); #endif if (self->keylog_bio != NULL) { - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS BIO_free_all(self->keylog_bio); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; self->keylog_bio = NULL; } return 0; @@ -3577,15 +3600,11 @@ _ssl__SSLContext_verify_flags_set_impl(PySSLContext *self, PyObject *value) static int set_min_max_proto_version(PySSLContext *self, PyObject *arg, int what) { - long v; + int v; int result; - if (!PyArg_Parse(arg, "l", &v)) - return -1; - if (v > INT_MAX) { - PyErr_SetString(PyExc_OverflowError, "Option is too long"); + if (!PyArg_Parse(arg, "i", &v)) return -1; - } switch(self->protocol) { case PY_SSL_VERSION_TLS_CLIENT: _Py_FALLTHROUGH; @@ -3620,7 +3639,7 @@ set_min_max_proto_version(PySSLContext *self, PyObject *arg, int what) break; default: PyErr_Format(PyExc_ValueError, - "Unsupported TLS/SSL version 0x%x", v); + "Unsupported TLS/SSL version 0x%x", (unsigned)v); return -1; } @@ -3654,7 +3673,7 @@ set_min_max_proto_version(PySSLContext *self, PyObject *arg, int what) } if (result == 0) { PyErr_Format(PyExc_ValueError, - "Unsupported protocol version 0x%x", v); + "Unsupported protocol version 0x%x", (unsigned)v); return -1; } return 0; @@ -4037,7 +4056,8 @@ _password_callback(char *buf, int size, int rwflag, void *userdata) _PySSLPasswordInfo *pw_info = (_PySSLPasswordInfo*) userdata; PyObject *fn_ret = NULL; - PySSL_END_ALLOW_THREADS_S(pw_info->thread_state); + pw_info->thread_state = PyThreadState_Swap(pw_info->thread_state); + _PySSL_FIX_ERRNO; if (pw_info->error) { /* already failed previously. OpenSSL 3.0.0-alpha14 invokes the @@ -4067,13 +4087,13 @@ _password_callback(char *buf, int size, int rwflag, void *userdata) goto error; } - PySSL_BEGIN_ALLOW_THREADS_S(pw_info->thread_state); + pw_info->thread_state = PyThreadState_Swap(pw_info->thread_state); memcpy(buf, pw_info->password, pw_info->size); return pw_info->size; error: Py_XDECREF(fn_ret); - PySSL_BEGIN_ALLOW_THREADS_S(pw_info->thread_state); + pw_info->thread_state = PyThreadState_Swap(pw_info->thread_state); pw_info->error = 1; return -1; } @@ -4126,10 +4146,10 @@ _ssl__SSLContext_load_cert_chain_impl(PySSLContext *self, PyObject *certfile, SSL_CTX_set_default_passwd_cb(self->ctx, _password_callback); SSL_CTX_set_default_passwd_cb_userdata(self->ctx, &pw_info); } - PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state); + PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); r = SSL_CTX_use_certificate_chain_file(self->ctx, PyBytes_AS_STRING(certfile_bytes)); - PySSL_END_ALLOW_THREADS_S(pw_info.thread_state); + PySSL_END_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); if (r != 1) { if (pw_info.error) { ERR_clear_error(); @@ -4144,11 +4164,11 @@ _ssl__SSLContext_load_cert_chain_impl(PySSLContext *self, PyObject *certfile, } goto error; } - PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state); + PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); r = SSL_CTX_use_PrivateKey_file(self->ctx, PyBytes_AS_STRING(keyfile ? keyfile_bytes : certfile_bytes), SSL_FILETYPE_PEM); - PySSL_END_ALLOW_THREADS_S(pw_info.thread_state); + PySSL_END_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); Py_CLEAR(keyfile_bytes); Py_CLEAR(certfile_bytes); if (r != 1) { @@ -4165,9 +4185,9 @@ _ssl__SSLContext_load_cert_chain_impl(PySSLContext *self, PyObject *certfile, } goto error; } - PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state); + PySSL_BEGIN_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); r = SSL_CTX_check_private_key(self->ctx); - PySSL_END_ALLOW_THREADS_S(pw_info.thread_state); + PySSL_END_ALLOW_THREADS_S(pw_info.thread_state, &self->tstate_mutex); if (r != 1) { _setSSLError(get_state_ctx(self), NULL, 0, __FILE__, __LINE__); goto error; @@ -4384,9 +4404,9 @@ _ssl__SSLContext_load_verify_locations_impl(PySSLContext *self, cafile_buf = PyBytes_AS_STRING(cafile_bytes); if (capath) capath_buf = PyBytes_AS_STRING(capath_bytes); - PySSL_BEGIN_ALLOW_THREADS + PySSL_BEGIN_ALLOW_THREADS(self) r = SSL_CTX_load_verify_locations(self->ctx, cafile_buf, capath_buf); - PySSL_END_ALLOW_THREADS + PySSL_END_ALLOW_THREADS(self) if (r != 1) { if (errno != 0) { PyErr_SetFromErrno(PyExc_OSError); @@ -4438,10 +4458,11 @@ _ssl__SSLContext_load_dh_params_impl(PySSLContext *self, PyObject *filepath) return NULL; errno = 0; - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS dh = PEM_read_DHparams(f, NULL, NULL, NULL); fclose(f); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; if (dh == NULL) { if (errno != 0) { PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, filepath); @@ -4593,6 +4614,7 @@ _ssl__SSLContext_set_default_verify_paths_impl(PySSLContext *self) Py_BEGIN_ALLOW_THREADS rc = SSL_CTX_set_default_verify_paths(self->ctx); Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; if (!rc) { _setSSLError(get_state_ctx(self), NULL, 0, __FILE__, __LINE__); return NULL; @@ -4650,12 +4672,15 @@ _servername_callback(SSL *s, int *al, void *args) PyObject *result; /* The high-level ssl.SSLSocket object */ PyObject *ssl_socket; + PyObject *sni_cb; const char *servername = SSL_get_servername(s, TLSEXT_NAMETYPE_host_name); PyGILState_STATE gstate = PyGILState_Ensure(); - if (sslctx->set_sni_cb == NULL) { - /* remove race condition in this the call back while if removing the - * callback is in progress */ + Py_BEGIN_CRITICAL_SECTION(sslctx); + sni_cb = Py_XNewRef(sslctx->set_sni_cb); + Py_END_CRITICAL_SECTION(); + + if (sni_cb == NULL) { PyGILState_Release(gstate); return SSL_TLSEXT_ERR_OK; } @@ -4682,7 +4707,7 @@ _servername_callback(SSL *s, int *al, void *args) goto error; if (servername == NULL) { - result = PyObject_CallFunctionObjArgs(sslctx->set_sni_cb, ssl_socket, + result = PyObject_CallFunctionObjArgs(sni_cb, ssl_socket, Py_None, sslctx, NULL); } else { @@ -4709,7 +4734,7 @@ _servername_callback(SSL *s, int *al, void *args) } Py_DECREF(servername_bytes); result = PyObject_CallFunctionObjArgs( - sslctx->set_sni_cb, ssl_socket, servername_str, + sni_cb, ssl_socket, servername_str, sslctx, NULL); Py_DECREF(servername_str); } @@ -4719,7 +4744,7 @@ _servername_callback(SSL *s, int *al, void *args) PyErr_FormatUnraisable("Exception ignored " "in ssl servername callback " "while calling set SNI callback %R", - sslctx->set_sni_cb); + sni_cb); *al = SSL_AD_HANDSHAKE_FAILURE; ret = SSL_TLSEXT_ERR_ALERT_FATAL; } @@ -4744,11 +4769,13 @@ _servername_callback(SSL *s, int *al, void *args) Py_DECREF(result); } + Py_DECREF(sni_cb); PyGILState_Release(gstate); return ret; error: - Py_DECREF(ssl_socket); + Py_XDECREF(ssl_socket); + Py_XDECREF(sni_cb); *al = SSL_AD_INTERNAL_ERROR; ret = SSL_TLSEXT_ERR_ALERT_FATAL; PyGILState_Release(gstate); @@ -4762,15 +4789,16 @@ _ssl._SSLContext.sni_callback Set a callback that will be called when a server name is provided by the SSL/TLS client in the SNI extension. -If the argument is None then the callback is disabled. The method is called -with the SSLSocket, the server name as a string, and the SSLContext object. +If the argument is None then the callback is disabled. The method +is called with the SSLSocket, the server name as a string, and the +SSLContext object. See RFC 6066 for details of the SNI extension. [clinic start generated code]*/ static PyObject * _ssl__SSLContext_sni_callback_get_impl(PySSLContext *self) -/*[clinic end generated code: output=961e6575cdfaf036 input=9b2473c5e984cfe6]*/ +/*[clinic end generated code: output=961e6575cdfaf036 input=280120c01d089dc3]*/ { PyObject *cb = self->set_sni_cb; if (cb == NULL) { @@ -4794,20 +4822,18 @@ _ssl__SSLContext_sni_callback_set_impl(PySSLContext *self, PyObject *value) "sni_callback cannot be set on TLS_CLIENT context"); return -1; } - Py_CLEAR(self->set_sni_cb); - if (value == Py_None) { + if (!PyCallable_Check(value)) { SSL_CTX_set_tlsext_servername_callback(self->ctx, NULL); - } - else { - if (!PyCallable_Check(value)) { - SSL_CTX_set_tlsext_servername_callback(self->ctx, NULL); - PyErr_SetString(PyExc_TypeError, - "not a callable object"); + Py_CLEAR(self->set_sni_cb); + if (value != Py_None) { + PyErr_SetString(PyExc_TypeError, "not a callable object"); return -1; } - self->set_sni_cb = Py_NewRef(value); - SSL_CTX_set_tlsext_servername_callback(self->ctx, _servername_callback); + } + else { + Py_XSETREF(self->set_sni_cb, Py_NewRef(value)); SSL_CTX_set_tlsext_servername_arg(self->ctx, self); + SSL_CTX_set_tlsext_servername_callback(self->ctx, _servername_callback); } return 0; } @@ -4862,16 +4888,16 @@ _ssl._SSLContext.cert_store_stats Returns quantities of loaded X.509 certificates. -X.509 certificates with a CA extension and certificate revocation lists -inside the context's cert store. +X.509 certificates with a CA extension and certificate revocation +lists inside the context's cert store. -NOTE: Certificates in a capath directory aren't loaded unless they have -been used at least once. +NOTE: Certificates in a capath directory aren't loaded unless they +have been used at least once. [clinic start generated code]*/ static PyObject * _ssl__SSLContext_cert_store_stats_impl(PySSLContext *self) -/*[clinic end generated code: output=5f356f4d9cca874d input=d13c6e3f2b48539b]*/ +/*[clinic end generated code: output=5f356f4d9cca874d input=9e5094e094b892a3]*/ { X509_STORE *store; STACK_OF(X509_OBJECT) *objs; @@ -4914,16 +4940,16 @@ _ssl._SSLContext.get_ca_certs Returns a list of dicts with information of loaded CA certs. -If the optional argument is True, returns a DER-encoded copy of the CA -certificate. +If the optional argument is True, returns a DER-encoded copy of the +CA certificate. -NOTE: Certificates in a capath directory aren't loaded unless they have -been used at least once. +NOTE: Certificates in a capath directory aren't loaded unless they +have been used at least once. [clinic start generated code]*/ static PyObject * _ssl__SSLContext_get_ca_certs_impl(PySSLContext *self, int binary_form) -/*[clinic end generated code: output=0d58f148f37e2938 input=eb0592909c9ad6e7]*/ +/*[clinic end generated code: output=0d58f148f37e2938 input=9f71af5aa4e67076]*/ { X509_STORE *store; STACK_OF(X509_OBJECT) *objs; @@ -5817,13 +5843,13 @@ _ssl.RAND_status Returns True if the OpenSSL PRNG has been seeded with enough data and False if not. -It is necessary to seed the PRNG with RAND_add() on some platforms before -using the ssl() function. +It is necessary to seed the PRNG with RAND_add() on some platforms +before using the ssl() function. [clinic start generated code]*/ static PyObject * _ssl_RAND_status_impl(PyObject *module) -/*[clinic end generated code: output=7e0aaa2d39fdc1ad input=636fb5659ea2e727]*/ +/*[clinic end generated code: output=7e0aaa2d39fdc1ad input=844b0dc0f2165e87]*/ { return PyBool_FromLong(RAND_status()); } @@ -6085,16 +6111,16 @@ _ssl.enum_certificates Retrieve certificates from Windows' cert store. -store_name may be one of 'CA', 'ROOT' or 'MY'. The system may provide -more cert storages, too. The function returns a list of (bytes, -encoding_type, trust) tuples. The encoding_type flag can be interpreted -with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either -a set of OIDs or the boolean True. +store_name may be one of 'CA', 'ROOT' or 'MY'. The system may +provide more cert storages, too. The function returns a list of +(bytes, encoding_type, trust) tuples. The encoding_type flag can be +interpreted with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The +trust setting is either a set of OIDs or the boolean True. [clinic start generated code]*/ static PyObject * _ssl_enum_certificates_impl(PyObject *module, const char *store_name) -/*[clinic end generated code: output=5134dc8bb3a3c893 input=263c22e6c6988cf3]*/ +/*[clinic end generated code: output=5134dc8bb3a3c893 input=ef81b4bd1b7ab8e9]*/ { HCERTSTORE hCollectionStore = NULL; PCCERT_CONTEXT pCertCtx = NULL; diff --git a/Modules/_ssl/debughelpers.c b/Modules/_ssl/debughelpers.c index 7c0b4876f4353ad..608ef07b5c5e432 100644 --- a/Modules/_ssl/debughelpers.c +++ b/Modules/_ssl/debughelpers.c @@ -140,13 +140,14 @@ _PySSL_keylog_callback(const SSL *ssl, const char *line) * critical debug helper. */ - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS PyThread_acquire_lock(lock, 1); res = BIO_printf(ssl_obj->ctx->keylog_bio, "%s\n", line); e = errno; (void)BIO_flush(ssl_obj->ctx->keylog_bio); PyThread_release_lock(lock); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; if (res == -1) { errno = e; @@ -187,9 +188,10 @@ _PySSLContext_set_keylog_filename(PyObject *op, PyObject *arg, if (self->keylog_bio != NULL) { BIO *bio = self->keylog_bio; self->keylog_bio = NULL; - PySSL_BEGIN_ALLOW_THREADS + Py_BEGIN_ALLOW_THREADS BIO_free_all(bio); - PySSL_END_ALLOW_THREADS + Py_END_ALLOW_THREADS + _PySSL_FIX_ERRNO; } if (arg == Py_None) { @@ -211,13 +213,13 @@ _PySSLContext_set_keylog_filename(PyObject *op, PyObject *arg, self->keylog_filename = Py_NewRef(arg); /* Write a header for seekable, empty files (this excludes pipes). */ - PySSL_BEGIN_ALLOW_THREADS + PySSL_BEGIN_ALLOW_THREADS(self) if (BIO_tell(self->keylog_bio) == 0) { BIO_puts(self->keylog_bio, "# TLS secrets log file, generated by OpenSSL / Python\n"); (void)BIO_flush(self->keylog_bio); } - PySSL_END_ALLOW_THREADS + PySSL_END_ALLOW_THREADS(self) SSL_CTX_set_keylog_callback(self->ctx, _PySSL_keylog_callback); return 0; } diff --git a/Modules/_ssl_data_34.h b/Modules/_ssl_data_35.h similarity index 98% rename from Modules/_ssl_data_34.h rename to Modules/_ssl_data_35.h index 99718c5e605acf2..89c97f3be51a7fd 100644 --- a/Modules/_ssl_data_34.h +++ b/Modules/_ssl_data_35.h @@ -1,6 +1,6 @@ /* File generated by Tools/ssl/make_ssl_data.py */ -/* Generated on 2025-03-26T13:47:34.223146+00:00 */ -/* Generated from Git commit openssl-3.4.1-0-ga26d85337d */ +/* Generated on 2026-05-03T20:36:01.807725+00:00 */ +/* Generated from Git commit openssl-3.5.6-0-g286ddeaac */ /* generated from args.lib2errnum */ static struct py_ssl_library_code library_codes[] = { @@ -1283,6 +1283,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"FAILED_BUILDING_OWN_CHAIN", 58, 164}, #endif + #ifdef CMP_R_FAILED_EXTRACTING_CENTRAL_GEN_KEY + {"FAILED_EXTRACTING_CENTRAL_GEN_KEY", ERR_LIB_CMP, CMP_R_FAILED_EXTRACTING_CENTRAL_GEN_KEY}, + #else + {"FAILED_EXTRACTING_CENTRAL_GEN_KEY", 58, 203}, + #endif #ifdef CMP_R_FAILED_EXTRACTING_PUBKEY {"FAILED_EXTRACTING_PUBKEY", ERR_LIB_CMP, CMP_R_FAILED_EXTRACTING_PUBKEY}, #else @@ -1343,6 +1348,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"INVALID_ROOTCAKEYUPDATE", 58, 195}, #endif + #ifdef CMP_R_MISSING_CENTRAL_GEN_KEY + {"MISSING_CENTRAL_GEN_KEY", ERR_LIB_CMP, CMP_R_MISSING_CENTRAL_GEN_KEY}, + #else + {"MISSING_CENTRAL_GEN_KEY", 58, 204}, + #endif #ifdef CMP_R_MISSING_CERTID {"MISSING_CERTID", ERR_LIB_CMP, CMP_R_MISSING_CERTID}, #else @@ -1513,6 +1523,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNCLEAN_CTX", 58, 191}, #endif + #ifdef CMP_R_UNEXPECTED_CENTRAL_GEN_KEY + {"UNEXPECTED_CENTRAL_GEN_KEY", ERR_LIB_CMP, CMP_R_UNEXPECTED_CENTRAL_GEN_KEY}, + #else + {"UNEXPECTED_CENTRAL_GEN_KEY", 58, 205}, + #endif #ifdef CMP_R_UNEXPECTED_CERTPROFILE {"UNEXPECTED_CERTPROFILE", ERR_LIB_CMP, CMP_R_UNEXPECTED_CERTPROFILE}, #else @@ -1653,6 +1668,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"CERTIFICATE_VERIFY_ERROR", 46, 100}, #endif + #ifdef CMS_R_CIPHER_AEAD_IN_ENVELOPED_DATA + {"CIPHER_AEAD_IN_ENVELOPED_DATA", ERR_LIB_CMS, CMS_R_CIPHER_AEAD_IN_ENVELOPED_DATA}, + #else + {"CIPHER_AEAD_IN_ENVELOPED_DATA", 46, 200}, + #endif #ifdef CMS_R_CIPHER_AEAD_SET_TAG_ERROR {"CIPHER_AEAD_SET_TAG_ERROR", ERR_LIB_CMS, CMS_R_CIPHER_AEAD_SET_TAG_ERROR}, #else @@ -2308,6 +2328,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"BAD_PBM_ITERATIONCOUNT", 56, 100}, #endif + #ifdef CRMF_R_CMS_NOT_SUPPORTED + {"CMS_NOT_SUPPORTED", ERR_LIB_CRMF, CRMF_R_CMS_NOT_SUPPORTED}, + #else + {"CMS_NOT_SUPPORTED", 56, 122}, + #endif #ifdef CRMF_R_CRMFERROR {"CRMFERROR", ERR_LIB_CRMF, CRMF_R_CRMFERROR}, #else @@ -2323,16 +2348,41 @@ static struct py_ssl_error_code error_codes[] = { #else {"ERROR_DECODING_CERTIFICATE", 56, 104}, #endif + #ifdef CRMF_R_ERROR_DECODING_ENCRYPTEDKEY + {"ERROR_DECODING_ENCRYPTEDKEY", ERR_LIB_CRMF, CRMF_R_ERROR_DECODING_ENCRYPTEDKEY}, + #else + {"ERROR_DECODING_ENCRYPTEDKEY", 56, 123}, + #endif #ifdef CRMF_R_ERROR_DECRYPTING_CERTIFICATE {"ERROR_DECRYPTING_CERTIFICATE", ERR_LIB_CRMF, CRMF_R_ERROR_DECRYPTING_CERTIFICATE}, #else {"ERROR_DECRYPTING_CERTIFICATE", 56, 105}, #endif + #ifdef CRMF_R_ERROR_DECRYPTING_ENCRYPTEDKEY + {"ERROR_DECRYPTING_ENCRYPTEDKEY", ERR_LIB_CRMF, CRMF_R_ERROR_DECRYPTING_ENCRYPTEDKEY}, + #else + {"ERROR_DECRYPTING_ENCRYPTEDKEY", 56, 124}, + #endif + #ifdef CRMF_R_ERROR_DECRYPTING_ENCRYPTEDVALUE + {"ERROR_DECRYPTING_ENCRYPTEDVALUE", ERR_LIB_CRMF, CRMF_R_ERROR_DECRYPTING_ENCRYPTEDVALUE}, + #else + {"ERROR_DECRYPTING_ENCRYPTEDVALUE", 56, 125}, + #endif #ifdef CRMF_R_ERROR_DECRYPTING_SYMMETRIC_KEY {"ERROR_DECRYPTING_SYMMETRIC_KEY", ERR_LIB_CRMF, CRMF_R_ERROR_DECRYPTING_SYMMETRIC_KEY}, #else {"ERROR_DECRYPTING_SYMMETRIC_KEY", 56, 106}, #endif + #ifdef CRMF_R_ERROR_SETTING_PURPOSE + {"ERROR_SETTING_PURPOSE", ERR_LIB_CRMF, CRMF_R_ERROR_SETTING_PURPOSE}, + #else + {"ERROR_SETTING_PURPOSE", 56, 126}, + #endif + #ifdef CRMF_R_ERROR_VERIFYING_ENCRYPTEDKEY + {"ERROR_VERIFYING_ENCRYPTEDKEY", ERR_LIB_CRMF, CRMF_R_ERROR_VERIFYING_ENCRYPTEDKEY}, + #else + {"ERROR_VERIFYING_ENCRYPTEDKEY", 56, 127}, + #endif #ifdef CRMF_R_FAILURE_OBTAINING_RANDOM {"FAILURE_OBTAINING_RANDOM", ERR_LIB_CRMF, CRMF_R_FAILURE_OBTAINING_RANDOM}, #else @@ -2358,6 +2408,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"POPOSKINPUT_NOT_SUPPORTED", 56, 113}, #endif + #ifdef CRMF_R_POPO_INCONSISTENT_CENTRAL_KEYGEN + {"POPO_INCONSISTENT_CENTRAL_KEYGEN", ERR_LIB_CRMF, CRMF_R_POPO_INCONSISTENT_CENTRAL_KEYGEN}, + #else + {"POPO_INCONSISTENT_CENTRAL_KEYGEN", 56, 128}, + #endif #ifdef CRMF_R_POPO_INCONSISTENT_PUBLIC_KEY {"POPO_INCONSISTENT_PUBLIC_KEY", ERR_LIB_CRMF, CRMF_R_POPO_INCONSISTENT_PUBLIC_KEY}, #else @@ -3963,6 +4018,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"PBKDF2_ERROR", 6, 181}, #endif + #ifdef EVP_R_PIPELINE_NOT_SUPPORTED + {"PIPELINE_NOT_SUPPORTED", ERR_LIB_EVP, EVP_R_PIPELINE_NOT_SUPPORTED}, + #else + {"PIPELINE_NOT_SUPPORTED", 6, 230}, + #endif #ifdef EVP_R_PKEY_APPLICATION_ASN1_METHOD_ALREADY_REGISTERED {"PKEY_APPLICATION_ASN1_METHOD_ALREADY_REGISTERED", ERR_LIB_EVP, EVP_R_PKEY_APPLICATION_ASN1_METHOD_ALREADY_REGISTERED}, #else @@ -3978,6 +4038,36 @@ static struct py_ssl_error_code error_codes[] = { #else {"PRIVATE_KEY_ENCODE_ERROR", 6, 146}, #endif + #ifdef EVP_R_PROVIDER_ASYM_CIPHER_FAILURE + {"PROVIDER_ASYM_CIPHER_FAILURE", ERR_LIB_EVP, EVP_R_PROVIDER_ASYM_CIPHER_FAILURE}, + #else + {"PROVIDER_ASYM_CIPHER_FAILURE", 6, 232}, + #endif + #ifdef EVP_R_PROVIDER_ASYM_CIPHER_NOT_SUPPORTED + {"PROVIDER_ASYM_CIPHER_NOT_SUPPORTED", ERR_LIB_EVP, EVP_R_PROVIDER_ASYM_CIPHER_NOT_SUPPORTED}, + #else + {"PROVIDER_ASYM_CIPHER_NOT_SUPPORTED", 6, 235}, + #endif + #ifdef EVP_R_PROVIDER_KEYMGMT_FAILURE + {"PROVIDER_KEYMGMT_FAILURE", ERR_LIB_EVP, EVP_R_PROVIDER_KEYMGMT_FAILURE}, + #else + {"PROVIDER_KEYMGMT_FAILURE", 6, 233}, + #endif + #ifdef EVP_R_PROVIDER_KEYMGMT_NOT_SUPPORTED + {"PROVIDER_KEYMGMT_NOT_SUPPORTED", ERR_LIB_EVP, EVP_R_PROVIDER_KEYMGMT_NOT_SUPPORTED}, + #else + {"PROVIDER_KEYMGMT_NOT_SUPPORTED", 6, 236}, + #endif + #ifdef EVP_R_PROVIDER_SIGNATURE_FAILURE + {"PROVIDER_SIGNATURE_FAILURE", ERR_LIB_EVP, EVP_R_PROVIDER_SIGNATURE_FAILURE}, + #else + {"PROVIDER_SIGNATURE_FAILURE", 6, 234}, + #endif + #ifdef EVP_R_PROVIDER_SIGNATURE_NOT_SUPPORTED + {"PROVIDER_SIGNATURE_NOT_SUPPORTED", ERR_LIB_EVP, EVP_R_PROVIDER_SIGNATURE_NOT_SUPPORTED}, + #else + {"PROVIDER_SIGNATURE_NOT_SUPPORTED", 6, 237}, + #endif #ifdef EVP_R_PUBLIC_KEY_NOT_RSA {"PUBLIC_KEY_NOT_RSA", ERR_LIB_EVP, EVP_R_PUBLIC_KEY_NOT_RSA}, #else @@ -3998,6 +4088,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"SIGNATURE_TYPE_AND_KEY_TYPE_INCOMPATIBLE", 6, 228}, #endif + #ifdef EVP_R_TOO_MANY_PIPES + {"TOO_MANY_PIPES", ERR_LIB_EVP, EVP_R_TOO_MANY_PIPES}, + #else + {"TOO_MANY_PIPES", 6, 231}, + #endif #ifdef EVP_R_TOO_MANY_RECORDS {"TOO_MANY_RECORDS", ERR_LIB_EVP, EVP_R_TOO_MANY_RECORDS}, #else @@ -4153,6 +4248,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"CONNECT_FAILURE", 61, 100}, #endif + #ifdef HTTP_R_CONTENT_TYPE_MISMATCH + {"CONTENT_TYPE_MISMATCH", ERR_LIB_HTTP, HTTP_R_CONTENT_TYPE_MISMATCH}, + #else + {"CONTENT_TYPE_MISMATCH", 61, 131}, + #endif #ifdef HTTP_R_ERROR_PARSING_ASN1_LENGTH {"ERROR_PARSING_ASN1_LENGTH", ERR_LIB_HTTP, HTTP_R_ERROR_PARSING_ASN1_LENGTH}, #else @@ -4753,6 +4853,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNSUPPORTED_PUBLIC_KEY_TYPE", 9, 110}, #endif + #ifdef PEM_R_UNSUPPORTED_PVK_KEY_TYPE + {"UNSUPPORTED_PVK_KEY_TYPE", ERR_LIB_PEM, PEM_R_UNSUPPORTED_PVK_KEY_TYPE}, + #else + {"UNSUPPORTED_PVK_KEY_TYPE", 9, 133}, + #endif #ifdef PKCS12_R_CALLBACK_FAILED {"CALLBACK_FAILED", ERR_LIB_PKCS12, PKCS12_R_CALLBACK_FAILED}, #else @@ -5243,6 +5348,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"FIPS_MODULE_ENTERING_ERROR_STATE", 57, 224}, #endif + #ifdef PROV_R_FIPS_MODULE_IMPORT_PCT_ERROR + {"FIPS_MODULE_IMPORT_PCT_ERROR", ERR_LIB_PROV, PROV_R_FIPS_MODULE_IMPORT_PCT_ERROR}, + #else + {"FIPS_MODULE_IMPORT_PCT_ERROR", 57, 253}, + #endif #ifdef PROV_R_FIPS_MODULE_IN_ERROR_STATE {"FIPS_MODULE_IN_ERROR_STATE", ERR_LIB_PROV, PROV_R_FIPS_MODULE_IN_ERROR_STATE}, #else @@ -5543,6 +5653,16 @@ static struct py_ssl_error_code error_codes[] = { #else {"MISSING_XCGHASH", 57, 135}, #endif + #ifdef PROV_R_ML_DSA_NO_FORMAT + {"ML_DSA_NO_FORMAT", ERR_LIB_PROV, PROV_R_ML_DSA_NO_FORMAT}, + #else + {"ML_DSA_NO_FORMAT", 57, 245}, + #endif + #ifdef PROV_R_ML_KEM_NO_FORMAT + {"ML_KEM_NO_FORMAT", ERR_LIB_PROV, PROV_R_ML_KEM_NO_FORMAT}, + #else + {"ML_KEM_NO_FORMAT", 57, 246}, + #endif #ifdef PROV_R_MODULE_INTEGRITY_FAILURE {"MODULE_INTEGRITY_FAILURE", ERR_LIB_PROV, PROV_R_MODULE_INTEGRITY_FAILURE}, #else @@ -5593,6 +5713,16 @@ static struct py_ssl_error_code error_codes[] = { #else {"NO_PARAMETERS_SET", 57, 177}, #endif + #ifdef PROV_R_NULL_LENGTH_POINTER + {"NULL_LENGTH_POINTER", ERR_LIB_PROV, PROV_R_NULL_LENGTH_POINTER}, + #else + {"NULL_LENGTH_POINTER", 57, 247}, + #endif + #ifdef PROV_R_NULL_OUTPUT_BUFFER + {"NULL_OUTPUT_BUFFER", ERR_LIB_PROV, PROV_R_NULL_OUTPUT_BUFFER}, + #else + {"NULL_OUTPUT_BUFFER", 57, 248}, + #endif #ifdef PROV_R_ONESHOT_CALL_OUT_OF_ORDER {"ONESHOT_CALL_OUT_OF_ORDER", ERR_LIB_PROV, PROV_R_ONESHOT_CALL_OUT_OF_ORDER}, #else @@ -5728,6 +5858,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNABLE_TO_RESEED", 57, 204}, #endif + #ifdef PROV_R_UNEXPECTED_KEY_PARAMETERS + {"UNEXPECTED_KEY_PARAMETERS", ERR_LIB_PROV, PROV_R_UNEXPECTED_KEY_PARAMETERS}, + #else + {"UNEXPECTED_KEY_PARAMETERS", 57, 249}, + #endif #ifdef PROV_R_UNSUPPORTED_CEK_ALG {"UNSUPPORTED_CEK_ALG", ERR_LIB_PROV, PROV_R_UNSUPPORTED_CEK_ALG}, #else @@ -5748,6 +5883,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"UNSUPPORTED_NUMBER_OF_ROUNDS", 57, 152}, #endif + #ifdef PROV_R_UNSUPPORTED_SELECTION + {"UNSUPPORTED_SELECTION", ERR_LIB_PROV, PROV_R_UNSUPPORTED_SELECTION}, + #else + {"UNSUPPORTED_SELECTION", 57, 250}, + #endif #ifdef PROV_R_UPDATE_CALL_OUT_OF_ORDER {"UPDATE_CALL_OUT_OF_ORDER", ERR_LIB_PROV, PROV_R_UPDATE_CALL_OUT_OF_ORDER}, #else @@ -5763,6 +5903,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"VALUE_ERROR", 57, 138}, #endif + #ifdef PROV_R_WRONG_CIPHERTEXT_SIZE + {"WRONG_CIPHERTEXT_SIZE", ERR_LIB_PROV, PROV_R_WRONG_CIPHERTEXT_SIZE}, + #else + {"WRONG_CIPHERTEXT_SIZE", 57, 251}, + #endif #ifdef PROV_R_WRONG_FINAL_BLOCK_LENGTH {"WRONG_FINAL_BLOCK_LENGTH", ERR_LIB_PROV, PROV_R_WRONG_FINAL_BLOCK_LENGTH}, #else @@ -5938,6 +6083,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"PRNG_NOT_SEEDED", 36, 100}, #endif + #ifdef RAND_R_RANDOM_POOL_IS_EMPTY + {"RANDOM_POOL_IS_EMPTY", ERR_LIB_RAND, RAND_R_RANDOM_POOL_IS_EMPTY}, + #else + {"RANDOM_POOL_IS_EMPTY", 36, 142}, + #endif #ifdef RAND_R_RANDOM_POOL_OVERFLOW {"RANDOM_POOL_OVERFLOW", ERR_LIB_RAND, RAND_R_RANDOM_POOL_OVERFLOW}, #else @@ -6923,6 +7073,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"DIGEST_CHECK_FAILED", 20, 149}, #endif + #ifdef SSL_R_DOMAIN_USE_ONLY + {"DOMAIN_USE_ONLY", ERR_LIB_SSL, SSL_R_DOMAIN_USE_ONLY}, + #else + {"DOMAIN_USE_ONLY", 20, 422}, + #endif #ifdef SSL_R_DTLS_MESSAGE_TOO_BIG {"DTLS_MESSAGE_TOO_BIG", ERR_LIB_SSL, SSL_R_DTLS_MESSAGE_TOO_BIG}, #else @@ -7213,6 +7368,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"LIBRARY_HAS_NO_CIPHERS", 20, 161}, #endif + #ifdef SSL_R_LISTENER_USE_ONLY + {"LISTENER_USE_ONLY", ERR_LIB_SSL, SSL_R_LISTENER_USE_ONLY}, + #else + {"LISTENER_USE_ONLY", 20, 421}, + #endif #ifdef SSL_R_MAXIMUM_ENCRYPTED_PKTS_REACHED {"MAXIMUM_ENCRYPTED_PKTS_REACHED", ERR_LIB_SSL, SSL_R_MAXIMUM_ENCRYPTED_PKTS_REACHED}, #else @@ -7243,6 +7403,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"MISSING_PSK_KEX_MODES_EXTENSION", 20, 310}, #endif + #ifdef SSL_R_MISSING_QUIC_TLS_FUNCTIONS + {"MISSING_QUIC_TLS_FUNCTIONS", ERR_LIB_SSL, SSL_R_MISSING_QUIC_TLS_FUNCTIONS}, + #else + {"MISSING_QUIC_TLS_FUNCTIONS", 20, 423}, + #endif #ifdef SSL_R_MISSING_RSA_CERTIFICATE {"MISSING_RSA_CERTIFICATE", ERR_LIB_SSL, SSL_R_MISSING_RSA_CERTIFICATE}, #else @@ -8983,6 +9148,11 @@ static struct py_ssl_error_code error_codes[] = { #else {"POLICY_WHEN_PROXY_LANGUAGE_REQUIRES_NO_POLICY", 34, 159}, #endif + #ifdef X509V3_R_PURPOSE_NOT_UNIQUE + {"PURPOSE_NOT_UNIQUE", ERR_LIB_X509V3, X509V3_R_PURPOSE_NOT_UNIQUE}, + #else + {"PURPOSE_NOT_UNIQUE", 34, 173}, + #endif #ifdef X509V3_R_SECTION_NOT_FOUND {"SECTION_NOT_FOUND", ERR_LIB_X509V3, X509V3_R_SECTION_NOT_FOUND}, #else diff --git a/Modules/_stat.c b/Modules/_stat.c index f11ca7d23b440d1..1dabf2f6d5b07fa 100644 --- a/Modules/_stat.c +++ b/Modules/_stat.c @@ -57,7 +57,7 @@ typedef unsigned short mode_t; * Only the names are defined by POSIX but not their value. All common file * types seems to have the same numeric value on all platforms, though. * - * pyport.h guarantees S_IFMT, S_IFDIR, S_IFCHR, S_IFREG and S_IFLNK + * fileutils.h guarantees S_IFMT, S_IFDIR, S_IFCHR, S_IFREG and S_IFLNK */ #ifndef S_IFBLK @@ -86,7 +86,7 @@ typedef unsigned short mode_t; /* S_ISXXX() - * pyport.h defines S_ISDIR(), S_ISREG() and S_ISCHR() + * fileutils.h defines S_ISDIR(), S_ISREG() and S_ISCHR() */ #ifndef S_ISBLK diff --git a/Modules/_struct.c b/Modules/_struct.c index c36079f1eb8886e..614512fe35f047d 100644 --- a/Modules/_struct.c +++ b/Modules/_struct.c @@ -1,7 +1,7 @@ /* struct module -- pack values into and (out of) bytes objects */ /* New version supporting byte order, alignment and size options, - character strings, and unsigned numbers */ + byte strings, and unsigned numbers */ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 @@ -9,8 +9,10 @@ #include "Python.h" #include "pycore_bytesobject.h" // _PyBytesWriter +#include "pycore_lock.h" // _PyOnceFlag_CallOnce() #include "pycore_long.h" // _PyLong_AsByteArray() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> // offsetof() @@ -760,14 +762,13 @@ np_halffloat(_structmodulestate *state, char *p, PyObject *v, const formatdef *f static int np_float(_structmodulestate *state, char *p, PyObject *v, const formatdef *f) { - float x = (float)PyFloat_AsDouble(v); + double x = PyFloat_AsDouble(v); if (x == -1 && PyErr_Occurred()) { PyErr_SetString(state->StructError, "required argument is not a float"); return -1; } - memcpy(p, &x, sizeof x); - return 0; + return PyFloat_Pack4(x, p, PY_LITTLE_ENDIAN); } static int @@ -1504,6 +1505,53 @@ static formatdef lilendian_table[] = { {0} }; +/* Ensure endian table optimization happens exactly once across all interpreters */ +static _PyOnceFlag endian_tables_init_once = {0}; + +static int +init_endian_tables(void *Py_UNUSED(arg)) +{ + const formatdef *native = native_table; + formatdef *other, *ptr; +#if PY_LITTLE_ENDIAN + other = lilendian_table; +#else + other = bigendian_table; +#endif + /* Scan through the native table, find a matching + entry in the endian table and swap in the + native implementations whenever possible + (64-bit platforms may not have "standard" sizes) */ + while (native->format != '\0' && other->format != '\0') { + ptr = other; + while (ptr->format != '\0') { + if (ptr->format == native->format) { + /* Match faster when formats are + listed in the same order */ + if (ptr == other) + other++; + /* Only use the trick if the + size matches */ + if (ptr->size != native->size) + break; + /* Skip float and double, could be + "unknown" float format */ + if (ptr->format == 'd' || ptr->format == 'f') + break; + /* Skip _Bool, semantics are different for standard size */ + if (ptr->format == '?') + break; + ptr->pack = native->pack; + ptr->unpack = native->unpack; + break; + } + ptr++; + } + native++; + } + return 0; +} + static const formatdef * whichtable(const char **pfmt) @@ -1571,11 +1619,11 @@ align(Py_ssize_t size, char c, const formatdef *e) /* calculate the size of a format string */ static int -prepare_s(PyStructObject *self) +prepare_s(PyStructObject *self, PyObject *format) { const formatdef *f; const formatdef *e; - formatcode *codes; + formatcode *codes, *codes0; const char *s; const char *fmt; @@ -1585,8 +1633,8 @@ prepare_s(PyStructObject *self) _structmodulestate *state = get_struct_state_structinst(self); - fmt = PyBytes_AS_STRING(self->s_format); - if (strlen(fmt) != (size_t)PyBytes_GET_SIZE(self->s_format)) { + fmt = PyBytes_AS_STRING(format); + if (strlen(fmt) != (size_t)PyBytes_GET_SIZE(format)) { PyErr_SetString(state->StructError, "embedded null character"); return -1; @@ -1627,9 +1675,23 @@ prepare_s(PyStructObject *self) switch (c) { case 's': _Py_FALLTHROUGH; - case 'p': len++; ncodes++; break; + case 'p': + if (len == PY_SSIZE_T_MAX) { + goto overflow; + } + len++; + ncodes++; + break; case 'x': break; - default: len += num; if (num) ncodes++; break; + default: + if (num > PY_SSIZE_T_MAX - len) { + goto overflow; + } + len += num; + if (num) { + ncodes++; + } + break; } itemsize = e->size; @@ -1649,18 +1711,12 @@ prepare_s(PyStructObject *self) return -1; } - self->s_size = size; - self->s_len = len; codes = PyMem_Malloc((ncodes + 1) * sizeof(formatcode)); if (codes == NULL) { PyErr_NoMemory(); return -1; } - /* Free any s_codes value left over from a previous initialization. */ - if (self->s_codes != NULL) - PyMem_Free(self->s_codes); - self->s_codes = codes; - + codes0 = codes; s = fmt; size = 0; while ((c = *s++) != '\0') { @@ -1700,6 +1756,14 @@ prepare_s(PyStructObject *self) codes->size = 0; codes->repeat = 0; + /* Free any s_codes value left over from a previous initialization. */ + if (self->s_codes != NULL) + PyMem_Free(self->s_codes); + self->s_codes = codes0; + self->s_size = size; + self->s_len = len; + Py_XSETREF(self->s_format, Py_NewRef(format)); + return 0; overflow: @@ -1765,9 +1829,8 @@ Struct___init___impl(PyStructObject *self, PyObject *format) return -1; } - Py_SETREF(self->s_format, format); - - ret = prepare_s(self); + ret = prepare_s(self, format); + Py_DECREF(format); return ret; } @@ -1794,9 +1857,7 @@ s_dealloc(PyObject *op) PyStructObject *s = PyStructObject_CAST(op); PyTypeObject *tp = Py_TYPE(s); PyObject_GC_UnTrack(s); - if (s->weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, s->weakreflist); if (s->s_codes != NULL) { PyMem_Free(s->s_codes); } @@ -1850,6 +1911,14 @@ s_unpack_internal(PyStructObject *soself, const char *startfrom, return NULL; } +#define ENSURE_STRUCT_IS_READY(self) \ + do { \ + if (!(self)->s_codes) { \ + PyErr_SetString(PyExc_RuntimeError, \ + "Struct object is not initialized"); \ + return NULL; \ + } \ + } while (0); /*[clinic input] Struct.unpack @@ -1870,7 +1939,7 @@ Struct_unpack_impl(PyStructObject *self, Py_buffer *buffer) /*[clinic end generated code: output=873a24faf02e848a input=3113f8e7038b2f6c]*/ { _structmodulestate *state = get_struct_state_structinst(self); - assert(self->s_codes != NULL); + ENSURE_STRUCT_IS_READY(self); if (buffer->len != self->s_size) { PyErr_Format(state->StructError, "unpack requires a buffer of %zd bytes", @@ -1891,7 +1960,8 @@ Return a tuple containing unpacked values. Values are unpacked according to the format string Struct.format. The buffer's size in bytes, starting at position offset, must be -at least Struct.size. +at least Struct.size. A negative offset counts from the end of the +buffer. See help(struct) for more on format strings. [clinic start generated code]*/ @@ -1899,10 +1969,10 @@ See help(struct) for more on format strings. static PyObject * Struct_unpack_from_impl(PyStructObject *self, Py_buffer *buffer, Py_ssize_t offset) -/*[clinic end generated code: output=57fac875e0977316 input=cafd4851d473c894]*/ +/*[clinic end generated code: output=57fac875e0977316 input=3451f778ed8a5345]*/ { _structmodulestate *state = get_struct_state_structinst(self); - assert(self->s_codes != NULL); + ENSURE_STRUCT_IS_READY(self); if (offset < 0) { if (offset + self->s_size > 0) { @@ -2054,8 +2124,7 @@ Struct_iter_unpack_impl(PyStructObject *self, PyObject *buffer) { _structmodulestate *state = get_struct_state_structinst(self); unpackiterobject *iter; - - assert(self->s_codes != NULL); + ENSURE_STRUCT_IS_READY(self); if (self->s_size == 0) { PyErr_Format(state->StructError, @@ -2090,7 +2159,7 @@ Struct_iter_unpack_impl(PyStructObject *self, PyObject *buffer) * * Takes a struct object, a tuple of arguments, and offset in that tuple of * argument for where to start processing the arguments for packing, and a - * character buffer for writing the packed string. The caller must insure + * character buffer for writing the packed data. The caller must ensure * that the buffer may contain the required length for packing the arguments. * 0 is returned on success, 1 is returned if there is an error. * @@ -2196,8 +2265,8 @@ s_pack(PyObject *self, PyObject *const *args, Py_ssize_t nargs) /* Validate arguments. */ soself = PyStructObject_CAST(self); + ENSURE_STRUCT_IS_READY(soself); assert(PyStruct_Check(self, state)); - assert(soself->s_codes != NULL); if (nargs != soself->s_len) { PyErr_Format(state->StructError, @@ -2228,8 +2297,9 @@ PyDoc_STRVAR(s_pack_into__doc__, \n\ Pack the values v1, v2, ... according to the format string S.format\n\ and write the packed bytes into the writable buffer buf starting at\n\ -offset. Note that the offset is a required argument. See\n\ -help(struct) for more on format strings."); +offset. Note that the offset is a required argument. A negative\n\ +offset counts from the end of the buffer. See help(struct) for more\n\ +on format strings."); static PyObject * s_pack_into(PyObject *self, PyObject *const *args, Py_ssize_t nargs) @@ -2241,8 +2311,8 @@ s_pack_into(PyObject *self, PyObject *const *args, Py_ssize_t nargs) /* Validate arguments. +1 is for the first arg as buffer. */ soself = PyStructObject_CAST(self); + ENSURE_STRUCT_IS_READY(soself); assert(PyStruct_Check(self, state)); - assert(soself->s_codes != NULL); if (nargs != (soself->s_len + 2)) { if (nargs == 0) { @@ -2329,6 +2399,7 @@ static PyObject * s_get_format(PyObject *op, void *Py_UNUSED(closure)) { PyStructObject *self = PyStructObject_CAST(op); + ENSURE_STRUCT_IS_READY(self); return PyUnicode_FromStringAndSize(PyBytes_AS_STRING(self->s_format), PyBytes_GET_SIZE(self->s_format)); } @@ -2347,6 +2418,7 @@ static PyObject * s_sizeof(PyObject *op, PyObject *Py_UNUSED(dummy)) { PyStructObject *self = PyStructObject_CAST(op); + ENSURE_STRUCT_IS_READY(self); size_t size = _PyObject_SIZE(Py_TYPE(self)) + sizeof(formatcode); for (formatcode *code = self->s_codes; code->fmtdef != NULL; code++) { size += sizeof(formatcode); @@ -2358,6 +2430,7 @@ static PyObject * s_repr(PyObject *op) { PyStructObject *self = PyStructObject_CAST(op); + ENSURE_STRUCT_IS_READY(self); PyObject* fmt = PyUnicode_FromStringAndSize( PyBytes_AS_STRING(self->s_format), PyBytes_GET_SIZE(self->s_format)); if (fmt == NULL) { @@ -2431,7 +2504,8 @@ static PyType_Spec PyStructType_spec = { static int cache_struct_converter(PyObject *module, PyObject *fmt, PyStructObject **ptr) { - PyObject * s_object; + PyObject *s_object; + PyObject *key; _structmodulestate *state = get_struct_state(module); if (fmt == NULL) { @@ -2439,24 +2513,41 @@ cache_struct_converter(PyObject *module, PyObject *fmt, PyStructObject **ptr) return 1; } - if (PyDict_GetItemRef(state->cache, fmt, &s_object) < 0) { + /* Use a str cache key: an equal str and bytes would collide and be + compared, raising BytesWarning under -bb. */ + if (PyBytes_Check(fmt)) { + key = PyUnicode_DecodeASCII(PyBytes_AS_STRING(fmt), + PyBytes_GET_SIZE(fmt), "surrogateescape"); + if (key == NULL) { + return 0; + } + } + else { + key = Py_NewRef(fmt); + } + + if (PyDict_GetItemRef(state->cache, key, &s_object) < 0) { + Py_DECREF(key); return 0; } if (s_object != NULL) { + Py_DECREF(key); *ptr = PyStructObject_CAST(s_object); return Py_CLEANUP_SUPPORTED; } - s_object = PyObject_CallOneArg(state->PyStructType, fmt); + s_object = PyObject_CallOneArg(state->PyStructType, key); if (s_object != NULL) { if (PyDict_GET_SIZE(state->cache) >= MAXCACHE) PyDict_Clear(state->cache); /* Attempt to cache the result */ - if (PyDict_SetItem(state->cache, fmt, s_object) == -1) + if (PyDict_SetItem(state->cache, key, s_object) == -1) PyErr_Clear(); + Py_DECREF(key); *ptr = (PyStructObject *)s_object; return Py_CLEANUP_SUPPORTED; } + Py_DECREF(key); return 0; } @@ -2522,8 +2613,8 @@ PyDoc_STRVAR(pack_into_doc, \n\ Pack the values v1, v2, ... according to the format string and write\n\ the packed bytes into the writable buffer buf starting at offset. Note\n\ -that the offset is a required argument. See help(struct) for more\n\ -on format strings."); +that the offset is a required argument. A negative offset counts from\n\ +the end of the buffer. See help(struct) for more on format strings."); static PyObject * pack_into(PyObject *module, PyObject *const *args, Py_ssize_t nargs) @@ -2577,6 +2668,7 @@ unpack_from Return a tuple containing values unpacked according to the format string. The buffer's size, minus offset, must be at least calcsize(format). +A negative offset counts from the end of the buffer. See help(struct) for more on format strings. [clinic start generated code]*/ @@ -2584,7 +2676,7 @@ See help(struct) for more on format strings. static PyObject * unpack_from_impl(PyObject *module, PyStructObject *s_object, Py_buffer *buffer, Py_ssize_t offset) -/*[clinic end generated code: output=1042631674c6e0d3 input=6e80a5398e985025]*/ +/*[clinic end generated code: output=1042631674c6e0d3 input=e383ecb97728be2b]*/ { return Struct_unpack_from_impl(s_object, buffer, offset); } @@ -2628,8 +2720,8 @@ static struct PyMethodDef module_functions[] = { PyDoc_STRVAR(module_doc, "Functions to convert between Python values and C structs.\n\ -Python bytes objects are used to hold the data representing the C struct\n\ -and also as format strings (explained below) to describe the layout of data\n\ +Python bytes objects are used to hold the data representing the C struct.\n\ +The format string (explained below) describes the layout of data\n\ in the C struct.\n\ \n\ The optional first format char indicates byte order, size and alignment:\n\ @@ -2639,19 +2731,18 @@ The optional first format char indicates byte order, size and alignment:\n\ >: big-endian, std. size & alignment\n\ !: same as >\n\ \n\ -The remaining chars indicate types of args and must match exactly;\n\ +The remaining characters indicate types of args and must match exactly;\n\ these can be preceded by a decimal repeat count:\n\ - x: pad byte (no data); c:char; b:signed byte; B:unsigned byte;\n\ - ?: _Bool (requires C99; if not available, char is used instead)\n\ - h:short; H:unsigned short; i:int; I:unsigned int;\n\ - l:long; L:unsigned long; f:float; d:double; e:half-float.\n\ + x: pad byte (no data); c: char; b: signed byte; B: unsigned byte;\n\ + ?: _Bool; h: short; H: unsigned short; i: int; I: unsigned int;\n\ + l: long; L: unsigned long; q: long long; Q: unsigned long long;\n\ + f: float; d: double; e: half-float;\n\ + F: float complex; D: double complex.\n\ Special cases (preceding decimal count indicates length):\n\ - s:string (array of char); p: pascal string (with count byte).\n\ + s: byte string (array of char); p: Pascal string (with count byte).\n\ Special cases (only available in native format):\n\ - n:ssize_t; N:size_t;\n\ - P:an integer type that is wide enough to hold a pointer.\n\ -Special case (not in native mode unless 'long long' in platform C):\n\ - q:long long; Q:unsigned long long\n\ + n: ssize_t; N: size_t;\n\ + P: an integer type that is wide enough to hold a pointer.\n\ Whitespace between formats is ignored.\n\ \n\ The variable struct.error is an exception raised on errors.\n"); @@ -2714,47 +2805,8 @@ _structmodule_exec(PyObject *m) return -1; } - /* Check endian and swap in faster functions */ - { - const formatdef *native = native_table; - formatdef *other, *ptr; -#if PY_LITTLE_ENDIAN - other = lilendian_table; -#else - other = bigendian_table; -#endif - /* Scan through the native table, find a matching - entry in the endian table and swap in the - native implementations whenever possible - (64-bit platforms may not have "standard" sizes) */ - while (native->format != '\0' && other->format != '\0') { - ptr = other; - while (ptr->format != '\0') { - if (ptr->format == native->format) { - /* Match faster when formats are - listed in the same order */ - if (ptr == other) - other++; - /* Only use the trick if the - size matches */ - if (ptr->size != native->size) - break; - /* Skip float and double, could be - "unknown" float format */ - if (ptr->format == 'd' || ptr->format == 'f') - break; - /* Skip _Bool, semantics are different for standard size */ - if (ptr->format == '?') - break; - ptr->pack = native->pack; - ptr->unpack = native->unpack; - break; - } - ptr++; - } - native++; - } - } + /* init cannot fail */ + (void)_PyOnceFlag_CallOnce(&endian_tables_init_once, init_endian_tables, NULL); /* Add some symbolic constants to the module */ state->StructError = PyErr_NewException("struct.error", NULL, NULL); diff --git a/Modules/_testcapi/float.c b/Modules/_testcapi/float.c index e3869134c84d43c..63de77ca6b86515 100644 --- a/Modules/_testcapi/float.c +++ b/Modules/_testcapi/float.c @@ -171,5 +171,9 @@ _PyTestCapi_Init_Float(PyObject *mod) return -1; } - return 0; +#if (defined(__mips__) && !defined(__mips_nan2008)) || defined(__hppa__) + return PyModule_Add(mod, "nan_msb_is_signaling", PyBool_FromLong(1)); +#else + return PyModule_Add(mod, "nan_msb_is_signaling", PyBool_FromLong(0)); +#endif } diff --git a/Modules/_testcapi/function.c b/Modules/_testcapi/function.c index ec1ba508df2ce91..40767adbd3f14a7 100644 --- a/Modules/_testcapi/function.c +++ b/Modules/_testcapi/function.c @@ -123,6 +123,13 @@ function_set_closure(PyObject *self, PyObject *args) } +static PyObject * +function_get_annotations(PyObject *self, PyObject *func) +{ + return Py_XNewRef(PyFunction_GetAnnotations(func)); +} + + static PyMethodDef test_methods[] = { {"function_get_code", function_get_code, METH_O, NULL}, {"function_get_globals", function_get_globals, METH_O, NULL}, @@ -133,6 +140,7 @@ static PyMethodDef test_methods[] = { {"function_set_kw_defaults", function_set_kw_defaults, METH_VARARGS, NULL}, {"function_get_closure", function_get_closure, METH_O, NULL}, {"function_set_closure", function_set_closure, METH_VARARGS, NULL}, + {"function_get_annotations", function_get_annotations, METH_O, NULL}, {NULL}, }; diff --git a/Modules/_testcapi/heaptype.c b/Modules/_testcapi/heaptype.c index 257e02566559763..09f18bbbf9a749e 100644 --- a/Modules/_testcapi/heaptype.c +++ b/Modules/_testcapi/heaptype.c @@ -528,6 +528,24 @@ pytype_getmodulebydef(PyObject *self, PyObject *type) return Py_XNewRef(mod); } +static PyType_Slot HeapCTypeWithBasesSlotNone_slots[] = { + {Py_tp_bases, NULL}, /* filled out with Py_None in runtime */ + {0, 0}, +}; + +static PyType_Spec HeapCTypeWithBasesSlotNone_spec = { + .name = "_testcapi.HeapCTypeWithBasesSlotNone", + .basicsize = sizeof(PyObject), + .flags = Py_TPFLAGS_DEFAULT, + .slots = HeapCTypeWithBasesSlotNone_slots +}; + +static PyObject * +create_heapctype_with_none_bases_slot(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + HeapCTypeWithBasesSlotNone_slots[0].pfunc = Py_None; + return PyType_FromSpec(&HeapCTypeWithBasesSlotNone_spec); +} static PyMethodDef TestMethods[] = { {"pytype_fromspec_meta", pytype_fromspec_meta, METH_O}, @@ -546,6 +564,8 @@ static PyMethodDef TestMethods[] = { {"get_tp_token", get_tp_token, METH_O}, {"pytype_getbasebytoken", pytype_getbasebytoken, METH_VARARGS}, {"pytype_getmodulebydef", pytype_getmodulebydef, METH_O}, + {"create_heapctype_with_none_bases_slot", + create_heapctype_with_none_bases_slot, METH_NOARGS}, {NULL}, }; @@ -879,6 +899,18 @@ static PyType_Spec HeapCTypeMetaclassNullNew_spec = { .slots = empty_type_slots }; +static PyType_Slot HeapCTypeWithBasesSlot_slots[] = { + {Py_tp_bases, NULL}, /* filled out in module init function */ + {0, 0}, +}; + +static PyType_Spec HeapCTypeWithBasesSlot_spec = { + .name = "_testcapi.HeapCTypeWithBasesSlot", + .basicsize = sizeof(PyLongObject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = HeapCTypeWithBasesSlot_slots +}; + typedef struct { PyObject_HEAD @@ -1419,6 +1451,18 @@ _PyTestCapi_Init_Heaptype(PyObject *m) { &PyType_Type, m, &HeapCTypeMetaclassNullNew_spec, (PyObject *) &PyType_Type); ADD("HeapCTypeMetaclassNullNew", HeapCTypeMetaclassNullNew); + PyObject *bases = PyTuple_Pack(1, &PyLong_Type); + if (bases == NULL) { + return -1; + } + HeapCTypeWithBasesSlot_slots[0].pfunc = bases; + PyObject *HeapCTypeWithBasesSlot = PyType_FromSpec(&HeapCTypeWithBasesSlot_spec); + Py_DECREF(bases); + if (HeapCTypeWithBasesSlot == NULL) { + return -1; + } + ADD("HeapCTypeWithBasesSlot", HeapCTypeWithBasesSlot); + ADD("Py_TP_USE_SPEC", PyLong_FromVoidPtr(Py_TP_USE_SPEC)); PyObject *HeapCCollection = PyType_FromMetaclass( diff --git a/Modules/_testcapi/immortal.c b/Modules/_testcapi/immortal.c index 0663c3781d426a2..c996ed242452599 100644 --- a/Modules/_testcapi/immortal.c +++ b/Modules/_testcapi/immortal.c @@ -31,13 +31,13 @@ test_immortal_small_ints(PyObject *self, PyObject *Py_UNUSED(ignored)) for (int i = -5; i <= 256; i++) { PyObject *obj = PyLong_FromLong(i); assert(verify_immortality(obj)); - int has_int_immortal_bit = ((PyLongObject *)obj)->long_value.lv_tag & IMMORTALITY_BIT_MASK; + int has_int_immortal_bit = _PyLong_IsSmallInt((PyLongObject *)obj); assert(has_int_immortal_bit); } for (int i = 257; i <= 260; i++) { PyObject *obj = PyLong_FromLong(i); assert(obj); - int has_int_immortal_bit = ((PyLongObject *)obj)->long_value.lv_tag & IMMORTALITY_BIT_MASK; + int has_int_immortal_bit = _PyLong_IsSmallInt((PyLongObject *)obj); assert(!has_int_immortal_bit); Py_DECREF(obj); } diff --git a/Modules/_testcapi/long.c b/Modules/_testcapi/long.c index 42243023a457680..6313abf5485fff1 100644 --- a/Modules/_testcapi/long.c +++ b/Modules/_testcapi/long.c @@ -228,7 +228,7 @@ pylongwriter_create(PyObject *module, PyObject *args) goto error; } - if (num < 0 || num >= PyLong_BASE) { + if (num < 0 || num >= (long)PyLong_BASE) { PyErr_SetString(PyExc_ValueError, "digit doesn't fit into digit"); goto error; } diff --git a/Modules/_testcapi/object.c b/Modules/_testcapi/object.c index 798ef97c495aeb4..4d53b3c678a470d 100644 --- a/Modules/_testcapi/object.c +++ b/Modules/_testcapi/object.c @@ -138,6 +138,15 @@ pyobject_is_unique_temporary(PyObject *self, PyObject *obj) return PyLong_FromLong(result); } +static PyObject * +pyobject_is_unique_temporary_new_object(PyObject *self, PyObject *unused) +{ + PyObject *obj = PyList_New(0); + int result = PyUnstable_Object_IsUniqueReferencedTemporary(obj); + Py_DECREF(obj); + return PyLong_FromLong(result); +} + static int MyObject_dealloc_called = 0; static void @@ -493,6 +502,7 @@ static PyMethodDef test_methods[] = { {"pyobject_clear_weakrefs_no_callbacks", pyobject_clear_weakrefs_no_callbacks, METH_O}, {"pyobject_enable_deferred_refcount", pyobject_enable_deferred_refcount, METH_O}, {"pyobject_is_unique_temporary", pyobject_is_unique_temporary, METH_O}, + {"pyobject_is_unique_temporary_new_object", pyobject_is_unique_temporary_new_object, METH_NOARGS}, {"test_py_try_inc_ref", test_py_try_inc_ref, METH_NOARGS}, {"test_xincref_doesnt_leak",test_xincref_doesnt_leak, METH_NOARGS}, {"test_incref_doesnt_leak", test_incref_doesnt_leak, METH_NOARGS}, diff --git a/Modules/_testcapi/parts.h b/Modules/_testcapi/parts.h index af6400162daf2b4..62794d4b944e934 100644 --- a/Modules/_testcapi/parts.h +++ b/Modules/_testcapi/parts.h @@ -65,5 +65,6 @@ int _PyTestCapi_Init_Import(PyObject *mod); int _PyTestCapi_Init_Frame(PyObject *mod); int _PyTestCapi_Init_Type(PyObject *mod); int _PyTestCapi_Init_Function(PyObject *mod); +int _PyTestCapi_Init_Weakref(PyObject *mod); #endif // Py_TESTCAPI_PARTS_H diff --git a/Modules/_testcapi/run.c b/Modules/_testcapi/run.c index c9db7ccdd965a2a..6812af994a43302 100644 --- a/Modules/_testcapi/run.c +++ b/Modules/_testcapi/run.c @@ -7,8 +7,44 @@ #include <errno.h> -static PyObject * -run_stringflags(PyObject *mod, PyObject *pos_args) +// Test functions, not macros +#undef PyRun_AnyFile +#undef PyRun_AnyFileEx +#undef PyRun_AnyFileFlags +#undef PyRun_File +#undef PyRun_FileEx +#undef PyRun_FileFlags +#undef PyRun_SimpleFile +#undef PyRun_SimpleFileEx +#undef PyRun_String +#undef PyRun_SimpleString +#undef Py_CompileString +#undef Py_CompileStringFlags +#undef PyRun_InteractiveOne +#undef PyRun_InteractiveLoop + + +// Test PyRun_String() +static PyObject* +run_string(PyObject *mod, PyObject *args) +{ + const char *str; + int start; + PyObject *globals = NULL; + PyObject *locals = NULL; + + if (!PyArg_ParseTuple(args, "yi|OO", &str, &start, &globals, &locals)) { + return NULL; + } + NULLABLE(globals); + NULLABLE(locals); + + return PyRun_String(str, start, globals, locals); +} + +// Test PyRun_StringFlags() +static PyObject* +run_stringflags(PyObject *mod, PyObject *args) { const char *str; Py_ssize_t size; @@ -20,12 +56,11 @@ run_stringflags(PyObject *mod, PyObject *pos_args) int cf_flags = 0; int cf_feature_version = 0; - if (!PyArg_ParseTuple(pos_args, "z#iO|Oii", + if (!PyArg_ParseTuple(args, "z#iO|Oii", &str, &size, &start, &globals, &locals, &cf_flags, &cf_feature_version)) { return NULL; } - NULLABLE(globals); NULLABLE(locals); if (cf_flags || cf_feature_version) { @@ -37,8 +72,381 @@ run_stringflags(PyObject *mod, PyObject *pos_args) return PyRun_StringFlags(str, start, globals, locals, pflags); } -static PyObject * -run_fileexflags(PyObject *mod, PyObject *pos_args) +static FILE* +open_file(PyObject *filename, int *closeit) +{ + if (PyUnicode_EqualToUTF8(filename, "<stdin>")) { + if (closeit) { + // override closeit + *closeit = 0; + } + return stdin; + } + return Py_fopen(filename, "r"); +} + + +// Test PyRun_SimpleFile() +static PyObject* +run_simplefile(PyObject *mod, PyObject *args) +{ + PyObject *open_filename; + const char *filename; + + if (!PyArg_ParseTuple(args, "Oy", &open_filename, &filename)) { + return NULL; + } + + FILE *fp = open_file(open_filename, NULL); + if (fp == NULL) { + return NULL; + } + int fd = fileno(fp); + + int res = PyRun_SimpleFile(fp, filename); + + assert(_Py_IsValidFD(fd)); + fclose(fp); + + assert(!PyErr_Occurred()); + return PyLong_FromLong(res); +} + +static int +close_fd(FILE *fp, int fd, int closeit) +{ + if (closeit && _Py_IsValidFD(fd)) { + PyErr_SetString(PyExc_AssertionError, "File was not closed after execution"); + fclose(fp); + return -1; + } + + if (!closeit && !_Py_IsValidFD(fd)) { + PyErr_SetString(PyExc_AssertionError, "Bad file descriptor after execution"); + return -1; + } + + if (!closeit) { + fclose(fp); + } + return 0; +} + +static int +close_fd_result(FILE *fp, int fd, int closeit, PyObject *result) +{ + if (closeit && result && _Py_IsValidFD(fd)) { + PyErr_SetString(PyExc_AssertionError, "File was not closed after execution"); + Py_DECREF(result); + fclose(fp); + return -1; + } + + if (!closeit && !_Py_IsValidFD(fd)) { + PyErr_SetString(PyExc_AssertionError, "Bad file descriptor after execution"); + Py_XDECREF(result); + return -1; + } + + if (!closeit) { + fclose(fp); + } + return 0; +} + +// Test PyRun_SimpleFileEx() +static PyObject* +run_simplefileex(PyObject *mod, PyObject *args) +{ + PyObject *open_filename; + const char *filename; + int closeit = 0; + + if (!PyArg_ParseTuple(args, "Oy|i", + &open_filename, &filename, &closeit)) { + return NULL; + } + + FILE *fp = open_file(open_filename, &closeit); + if (fp == NULL) { + return NULL; + } + int fd = fileno(fp); + + int res = PyRun_SimpleFileEx(fp, filename, closeit); + + if (close_fd(fp, fd, closeit) < 0) { + return NULL; + } + + assert(!PyErr_Occurred()); + return PyLong_FromLong(res); +} + +// Test PyRun_SimpleFileExFlags() +static PyObject* +run_simplefileexflags(PyObject *mod, PyObject *args) +{ + PyObject *open_filename; + const char *filename; + int closeit = 0; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + + if (!PyArg_ParseTuple(args, "Oy|ii", + &open_filename, &filename, &closeit, &cf_flags)) { + return NULL; + } + + FILE *fp = open_file(open_filename, &closeit); + if (fp == NULL) { + return NULL; + } + int fd = fileno(fp); + + if (cf_flags) { + flags.cf_flags = cf_flags; + pflags = &flags; + } + + int res = PyRun_SimpleFileExFlags(fp, filename, closeit, pflags); + + if (close_fd(fp, fd, closeit) < 0) { + return NULL; + } + + assert(!PyErr_Occurred()); + return PyLong_FromLong(res); +} + +// Test PyRun_AnyFile() +static PyObject* +run_anyfile(PyObject *mod, PyObject *args) +{ + PyObject *open_filename; + const char *filename; + + if (!PyArg_ParseTuple(args, "Oy", &open_filename, &filename)) { + return NULL; + } + + FILE *fp = open_file(open_filename, NULL); + if (fp == NULL) { + return NULL; + } + int fd = fileno(fp); + + int res = PyRun_AnyFile(fp, filename); + + assert(_Py_IsValidFD(fd)); + fclose(fp); + + assert(!PyErr_Occurred()); + return PyLong_FromLong(res); +} + +// Test PyRun_AnyFileFlags() +static PyObject* +run_anyfileflags(PyObject *mod, PyObject *args) +{ + PyObject *open_filename; + const char *filename; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + + if (!PyArg_ParseTuple(args, "Oy|i", + &open_filename, &filename, &cf_flags)) { + return NULL; + } + + FILE *fp = open_file(open_filename, NULL); + if (fp == NULL) { + return NULL; + } + int fd = fileno(fp); + + if (cf_flags) { + flags.cf_flags = cf_flags; + pflags = &flags; + } + + int res = PyRun_AnyFileFlags(fp, filename, pflags); + + assert(_Py_IsValidFD(fd)); + fclose(fp); + + assert(!PyErr_Occurred()); + return PyLong_FromLong(res); +} + +// Test PyRun_AnyFileEx() +static PyObject* +run_anyfileex(PyObject *mod, PyObject *args) +{ + PyObject *open_filename; + const char *filename; + int closeit = 0; + + if (!PyArg_ParseTuple(args, "Oy|i", + &open_filename, &filename, &closeit)) { + return NULL; + } + + FILE *fp = open_file(open_filename, &closeit); + if (fp == NULL) { + return NULL; + } + int fd = fileno(fp); + + int res = PyRun_AnyFileEx(fp, filename, closeit); + + if (close_fd(fp, fd, closeit) < 0) { + return NULL; + } + + assert(!PyErr_Occurred()); + return PyLong_FromLong(res); +} + +// Test PyRun_AnyFileExFlags() +static PyObject* +run_anyfileexflags(PyObject *mod, PyObject *args) +{ + PyObject *open_filename; + const char *filename; + int closeit = 0; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + + if (!PyArg_ParseTuple(args, "Oy|ii", + &open_filename, &filename, &closeit, &cf_flags)) { + return NULL; + } + + FILE *fp = open_file(open_filename, &closeit); + if (fp == NULL) { + return NULL; + } + int fd = fileno(fp); + + if (cf_flags) { + flags.cf_flags = cf_flags; + pflags = &flags; + } + + int res = PyRun_AnyFileExFlags(fp, filename, closeit, pflags); + + if (close_fd(fp, fd, closeit) < 0) { + return NULL; + } + + assert(!PyErr_Occurred()); + return PyLong_FromLong(res); +} + + +// Test PyRun_InteractiveOne() +static PyObject* +run_interactiveone(PyObject *mod, PyObject *args) +{ + PyObject *open_filename; + char *filename; + + if (!PyArg_ParseTuple(args, "Oy", &open_filename, &filename)) { + return NULL; + } + + FILE *fp = open_file(open_filename, NULL); + if (fp == NULL) { + return NULL; + } + int fd = fileno(fp); + + int res = PyRun_InteractiveOne(fp, filename); + assert(!PyErr_Occurred()); + + assert(_Py_IsValidFD(fd)); + fclose(fp); + + return PyLong_FromLong(res); +} + +// Test PyRun_InteractiveOneFlags() +static PyObject* +run_interactiveoneflags(PyObject *mod, PyObject *args) +{ + PyObject *open_filename; + char *filename; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + + if (!PyArg_ParseTuple(args, "Oy|i", + &open_filename, &filename, &cf_flags)) { + return NULL; + } + + FILE *fp = open_file(open_filename, NULL); + if (fp == NULL) { + return NULL; + } + int fd = fileno(fp); + + if (cf_flags) { + flags.cf_flags = cf_flags; + pflags = &flags; + } + + int res = PyRun_InteractiveOneFlags(fp, filename, pflags); + assert(!PyErr_Occurred()); + + assert(_Py_IsValidFD(fd)); + fclose(fp); + + return PyLong_FromLong(res); +} + +// Test PyRun_InteractiveOneObject() +static PyObject* +run_interactiveoneobject(PyObject *mod, PyObject *args) +{ + PyObject *open_filename; + PyObject *filename; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + + if (!PyArg_ParseTuple(args, "OO|i", + &open_filename, &filename, &cf_flags)) { + return NULL; + } + + FILE *fp = open_file(open_filename, NULL); + if (fp == NULL) { + return NULL; + } + int fd = fileno(fp); + + if (cf_flags) { + flags.cf_flags = cf_flags; + pflags = &flags; + } + + int res = PyRun_InteractiveOneObject(fp, filename, pflags); + assert(!PyErr_Occurred()); + + assert(_Py_IsValidFD(fd)); + fclose(fp); + + return PyLong_FromLong(res); +} + +// Test PyRun_File() +static PyObject* +run_file(PyObject *mod, PyObject *args) { PyObject *result = NULL; const char *filename = NULL; @@ -46,20 +454,128 @@ run_fileexflags(PyObject *mod, PyObject *pos_args) int start; PyObject *globals = NULL; PyObject *locals = NULL; + + if (!PyArg_ParseTuple(args, "z#iO|O", + &filename, &filename_size, &start, + &globals, &locals)) { + return NULL; + } + NULLABLE(globals); + NULLABLE(locals); + + FILE *fp = fopen(filename, "r"); + if (fp == NULL) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename); + return NULL; + } + int fd = fileno(fp); + + result = PyRun_File(fp, filename, start, globals, locals); + + assert(_Py_IsValidFD(fd)); + fclose(fp); + + return result; +} + +// Test PyRun_FileEx() +static PyObject* +run_fileex(PyObject *mod, PyObject *args) +{ + const char *filename = NULL; + Py_ssize_t filename_size; + int start; + PyObject *globals = NULL; + PyObject *locals = NULL; int closeit = 0; + + if (!PyArg_ParseTuple(args, "z#iO|Oi", + &filename, &filename_size, &start, &globals, &locals, + &closeit)) { + return NULL; + } + NULLABLE(globals); + NULLABLE(locals); + + FILE *fp = fopen(filename, "r"); + if (fp == NULL) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename); + return NULL; + } + int fd = fileno(fp); + + PyObject *result = PyRun_FileEx(fp, filename, start, globals, locals, + closeit); + + if (close_fd_result(fp, fd, closeit, result) < 0) { + return NULL; + } + + return result; +} + +// Test PyRun_FileFlags +static PyObject* +run_fileflags(PyObject *mod, PyObject *args) +{ + const char *filename = NULL; + Py_ssize_t filename_size; + int start; + PyObject *globals = NULL; + PyObject *locals = NULL; PyCompilerFlags flags = _PyCompilerFlags_INIT; PyCompilerFlags *pflags = NULL; int cf_flags = 0; int cf_feature_version = 0; - FILE *fp = NULL; + if (!PyArg_ParseTuple(args, "z#iO|Oii", + &filename, &filename_size, &start, &globals, &locals, + &cf_flags, &cf_feature_version)) { + return NULL; + } + NULLABLE(globals); + NULLABLE(locals); + if (cf_flags || cf_feature_version) { + flags.cf_flags = cf_flags; + flags.cf_feature_version = cf_feature_version; + pflags = &flags; + } + + FILE *fp = fopen(filename, "r"); + if (fp == NULL) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename); + return NULL; + } + int fd = fileno(fp); - if (!PyArg_ParseTuple(pos_args, "z#iO|Oiii", + PyObject *result = PyRun_FileFlags(fp, filename, start, globals, locals, + pflags); + + assert(_Py_IsValidFD(fd)); + fclose(fp); + + return result; +} + +static PyObject* +run_fileexflags(PyObject *mod, PyObject *args) +{ + const char *filename = NULL; + Py_ssize_t filename_size; + int start; + PyObject *globals = NULL; + PyObject *locals = NULL; + int closeit = 0; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + int cf_feature_version = 0; + + if (!PyArg_ParseTuple(args, "z#iO|Oiii", &filename, &filename_size, &start, &globals, &locals, &closeit, &cf_flags, &cf_feature_version)) { return NULL; } - NULLABLE(globals); NULLABLE(locals); if (cf_flags || cf_feature_version) { @@ -68,38 +584,226 @@ run_fileexflags(PyObject *mod, PyObject *pos_args) pflags = &flags; } - fp = fopen(filename, "r"); + FILE *fp = fopen(filename, "r"); if (fp == NULL) { PyErr_SetFromErrnoWithFilename(PyExc_OSError, filename); return NULL; } int fd = fileno(fp); - result = PyRun_FileExFlags(fp, filename, start, globals, locals, closeit, pflags); + PyObject *result = PyRun_FileExFlags(fp, filename, start, globals, locals, + closeit, pflags); - if (closeit && result && _Py_IsValidFD(fd)) { - PyErr_SetString(PyExc_AssertionError, "File was not closed after execution"); - Py_DECREF(result); - fclose(fp); + if (close_fd_result(fp, fd, closeit, result) < 0) { return NULL; } - if (!closeit && !_Py_IsValidFD(fd)) { - PyErr_SetString(PyExc_AssertionError, "Bad file descriptor after execution"); - Py_XDECREF(result); + return result; +} + +static PyObject* +run_simplestring(PyObject *mod, PyObject *args) +{ + const char *str; + if (!PyArg_ParseTuple(args, "y", &str)) { return NULL; } - if (!closeit) { - fclose(fp); /* don't need open file any more*/ + int res = PyRun_SimpleString(str); + assert(!PyErr_Occurred()); + return PyLong_FromLong(res); +} + +static PyObject* +run_simplestringflags(PyObject *mod, PyObject *args) +{ + const char *str; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + + if (!PyArg_ParseTuple(args, "y|i", &str, &flags)) { + return NULL; } - return result; + if (cf_flags) { + flags.cf_flags = cf_flags; + pflags = &flags; + } + + int res = PyRun_SimpleStringFlags(str, pflags); + assert(!PyErr_Occurred()); + return PyLong_FromLong(res); +} + + +// Test PyRun_InteractiveLoop() +static PyObject* +run_interactiveloop(PyObject *mod, PyObject *args) +{ + PyObject *open_filename; + char *filename; + + if (!PyArg_ParseTuple(args, "Oy|i", &open_filename, &filename)) { + return NULL; + } + + FILE *fp = open_file(open_filename, NULL); + if (fp == NULL) { + return NULL; + } + int fd = fileno(fp); + + int res = PyRun_InteractiveLoop(fp, filename); + assert(!PyErr_Occurred()); + + assert(_Py_IsValidFD(fd)); + fclose(fp); + + return PyLong_FromLong(res); +} + + +// Test PyRun_InteractiveLoopFlags() +static PyObject* +run_interactiveloopflags(PyObject *mod, PyObject *args) +{ + PyObject *open_filename; + char *filename; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + + if (!PyArg_ParseTuple(args, "Oy|i", + &open_filename, &filename, &cf_flags)) { + return NULL; + } + + FILE *fp = open_file(open_filename, NULL); + if (fp == NULL) { + return NULL; + } + int fd = fileno(fp); + + if (cf_flags) { + flags.cf_flags = cf_flags; + pflags = &flags; + } + + int res = PyRun_InteractiveLoopFlags(fp, filename, pflags); + assert(!PyErr_Occurred()); + + assert(_Py_IsValidFD(fd)); + fclose(fp); + + return PyLong_FromLong(res); +} + + +// Test Py_CompileStringFlags() +static PyObject* +run_compilestringflags(PyObject *mod, PyObject *args) +{ + const char *str; + const char *filename; + int start; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + int cf_feature_version = 0; + + if (!PyArg_ParseTuple(args, "yyi|ii", &str, &filename, &start, + &cf_flags, &cf_feature_version)) { + return NULL; + } + if (cf_flags || cf_feature_version) { + flags.cf_flags = cf_flags; + flags.cf_feature_version = cf_feature_version; + pflags = &flags; + } + + return Py_CompileStringFlags(str, filename, start, pflags); } + +// Test Py_CompileStringExFlags() +static PyObject* +run_compilestringexflags(PyObject *mod, PyObject *args) +{ + const char *str; + const char *filename; + int start; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + int cf_feature_version = 0; + int optimize = -1; + + if (!PyArg_ParseTuple(args, "yyi|iii", &str, &filename, &start, + &cf_flags, &cf_feature_version, &optimize)) { + return NULL; + } + if (cf_flags || cf_feature_version) { + flags.cf_flags = cf_flags; + flags.cf_feature_version = cf_feature_version; + pflags = &flags; + } + + return Py_CompileStringExFlags(str, filename, start, pflags, optimize); +} + + +// Test Py_CompileStringObject() +static PyObject* +run_compilestringobject(PyObject *mod, PyObject *args) +{ + const char *str; + PyObject *filename; + int start; + PyCompilerFlags flags = _PyCompilerFlags_INIT; + PyCompilerFlags *pflags = NULL; + int cf_flags = 0; + int cf_feature_version = 0; + int optimize = -1; + + if (!PyArg_ParseTuple(args, "yOi|iii", &str, &filename, &start, + &cf_flags, &cf_feature_version, &optimize)) { + return NULL; + } + if (cf_flags || cf_feature_version) { + flags.cf_flags = cf_flags; + flags.cf_feature_version = cf_feature_version; + pflags = &flags; + } + + return Py_CompileStringObject(str, filename, start, pflags, optimize); +} + + static PyMethodDef test_methods[] = { + {"run_string", run_string, METH_VARARGS}, {"run_stringflags", run_stringflags, METH_VARARGS}, + {"run_simplefile", run_simplefile, METH_VARARGS}, + {"run_simplefileex", run_simplefileex, METH_VARARGS}, + {"run_simplefileexflags", run_simplefileexflags, METH_VARARGS}, + {"run_anyfile", run_anyfile, METH_VARARGS}, + {"run_anyfileflags", run_anyfileflags, METH_VARARGS}, + {"run_anyfileex", run_anyfileex, METH_VARARGS}, + {"run_anyfileexflags", run_anyfileexflags, METH_VARARGS}, + {"run_interactiveone", run_interactiveone, METH_VARARGS}, + {"run_interactiveoneflags", run_interactiveoneflags, METH_VARARGS}, + {"run_interactiveoneobject", run_interactiveoneobject, METH_VARARGS}, + {"run_file", run_file, METH_VARARGS}, + {"run_fileex", run_fileex, METH_VARARGS}, + {"run_fileflags", run_fileflags, METH_VARARGS}, {"run_fileexflags", run_fileexflags, METH_VARARGS}, + {"run_simplestring", run_simplestring, METH_VARARGS}, + {"run_simplestringflags", run_simplestringflags, METH_VARARGS}, + {"run_interactiveloop", run_interactiveloop, METH_VARARGS}, + {"run_interactiveloopflags", run_interactiveloopflags, METH_VARARGS}, + {"run_compilestringflags", run_compilestringflags, METH_VARARGS}, + {"run_compilestringexflags", run_compilestringexflags, METH_VARARGS}, + {"run_compilestringobject", run_compilestringobject, METH_VARARGS}, {NULL}, }; @@ -109,5 +813,11 @@ _PyTestCapi_Init_Run(PyObject *mod) if (PyModule_AddFunctions(mod, test_methods) < 0) { return -1; } + if (PyModule_AddIntMacro(mod, PyCF_ONLY_AST) < 0) { + return -1; + } + if (PyModule_AddIntMacro(mod, PyCF_IGNORE_COOKIE) < 0) { + return -1; + } return 0; } diff --git a/Modules/_testcapi/unicode.c b/Modules/_testcapi/unicode.c index b8ecf53f4f8b9c5..3d591b5653466a2 100644 --- a/Modules/_testcapi/unicode.c +++ b/Modules/_testcapi/unicode.c @@ -295,16 +295,12 @@ writer_write_char(PyObject *self_raw, PyObject *args) return NULL; } - PyObject *str; - if (!PyArg_ParseTuple(args, "U", &str)) { + unsigned int ch; + if (!PyArg_ParseTuple(args, "I", &ch)) { return NULL; } - if (PyUnicode_GET_LENGTH(str) != 1) { - PyErr_SetString(PyExc_ValueError, "expect a single character"); - } - Py_UCS4 ch = PyUnicode_READ_CHAR(str, 0); - if (PyUnicodeWriter_WriteChar(self->writer, ch) < 0) { + if (PyUnicodeWriter_WriteChar(self->writer, (Py_UCS4)ch) < 0) { return NULL; } Py_RETURN_NONE; @@ -319,9 +315,9 @@ writer_write_utf8(PyObject *self_raw, PyObject *args) return NULL; } - char *str; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "yn", &str, &size)) { + const char *str; + Py_ssize_t bsize, size; + if (!PyArg_ParseTuple(args, "z#n", &str, &bsize, &size)) { return NULL; } @@ -333,27 +329,20 @@ writer_write_utf8(PyObject *self_raw, PyObject *args) static PyObject* -writer_write_widechar(PyObject *self_raw, PyObject *args) +writer_write_ascii(PyObject *self_raw, PyObject *args) { WriterObject *self = (WriterObject *)self_raw; if (writer_check(self) < 0) { return NULL; } - PyObject *str; - if (!PyArg_ParseTuple(args, "U", &str)) { - return NULL; - } - - Py_ssize_t size; - wchar_t *wstr = PyUnicode_AsWideCharString(str, &size); - if (wstr == NULL) { + const char *str; + Py_ssize_t bsize, size; + if (!PyArg_ParseTuple(args, "z#n", &str, &bsize, &size)) { return NULL; } - int res = PyUnicodeWriter_WriteWideChar(self->writer, wstr, size); - PyMem_Free(wstr); - if (res < 0) { + if (PyUnicodeWriter_WriteASCII(self->writer, str, size) < 0) { return NULL; } Py_RETURN_NONE; @@ -361,28 +350,30 @@ writer_write_widechar(PyObject *self_raw, PyObject *args) static PyObject* -writer_write_ucs4(PyObject *self_raw, PyObject *args) +writer_write_widechar(PyObject *self_raw, PyObject *args) { WriterObject *self = (WriterObject *)self_raw; if (writer_check(self) < 0) { return NULL; } - PyObject *str; - Py_ssize_t size; - if (!PyArg_ParseTuple(args, "Un", &str, &size)) { - return NULL; - } - Py_ssize_t len = PyUnicode_GET_LENGTH(str); - size = Py_MIN(size, len); + const char *s; + Py_ssize_t bsize; + Py_ssize_t size = -100; - Py_UCS4 *ucs4 = PyUnicode_AsUCS4Copy(str); - if (ucs4 == NULL) { + if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { return NULL; } + if (size == -100) { + if (bsize % SIZEOF_WCHAR_T) { + PyErr_SetString(PyExc_AssertionError, + "invalid size in writer.write_widechar()"); + return NULL; + } + size = bsize / SIZEOF_WCHAR_T; + } - int res = PyUnicodeWriter_WriteUCS4(self->writer, ucs4, size); - PyMem_Free(ucs4); + int res = PyUnicodeWriter_WriteWideChar(self->writer, (const wchar_t *)s, size); if (res < 0) { return NULL; } @@ -391,19 +382,31 @@ writer_write_ucs4(PyObject *self_raw, PyObject *args) static PyObject* -writer_write_str(PyObject *self_raw, PyObject *args) +writer_write_ucs4(PyObject *self_raw, PyObject *args) { WriterObject *self = (WriterObject *)self_raw; if (writer_check(self) < 0) { return NULL; } - PyObject *obj; - if (!PyArg_ParseTuple(args, "O", &obj)) { + const char *s; + Py_ssize_t bsize; + Py_ssize_t size = -100; + + if (!PyArg_ParseTuple(args, "z#|n", &s, &bsize, &size)) { return NULL; } + if (size == -100) { + if (bsize % sizeof(Py_UCS4)) { + PyErr_SetString(PyExc_AssertionError, + "invalid size in writer.write_ucs4()"); + return NULL; + } + size = bsize / sizeof(Py_UCS4); + } - if (PyUnicodeWriter_WriteStr(self->writer, obj) < 0) { + int res = PyUnicodeWriter_WriteUCS4(self->writer, (Py_UCS4 *)s, size); + if (res < 0) { return NULL; } Py_RETURN_NONE; @@ -411,18 +414,30 @@ writer_write_str(PyObject *self_raw, PyObject *args) static PyObject* -writer_write_repr(PyObject *self_raw, PyObject *args) +writer_write_str(PyObject *self_raw, PyObject *obj) { WriterObject *self = (WriterObject *)self_raw; if (writer_check(self) < 0) { return NULL; } - PyObject *obj; - if (!PyArg_ParseTuple(args, "O", &obj)) { + NULLABLE(obj); + if (PyUnicodeWriter_WriteStr(self->writer, obj) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + + +static PyObject* +writer_write_repr(PyObject *self_raw, PyObject *obj) +{ + WriterObject *self = (WriterObject *)self_raw; + if (writer_check(self) < 0) { return NULL; } + NULLABLE(obj); if (PyUnicodeWriter_WriteRepr(self->writer, obj) < 0) { return NULL; } @@ -440,9 +455,10 @@ writer_write_substring(PyObject *self_raw, PyObject *args) PyObject *str; Py_ssize_t start, end; - if (!PyArg_ParseTuple(args, "Unn", &str, &start, &end)) { + if (!PyArg_ParseTuple(args, "Onn", &str, &start, &end)) { return NULL; } + NULLABLE(str); if (PyUnicodeWriter_WriteSubstring(self->writer, str, start, end) < 0) { return NULL; @@ -460,10 +476,10 @@ writer_decodeutf8stateful(PyObject *self_raw, PyObject *args) } const char *str; - Py_ssize_t len; + Py_ssize_t bsize, len; const char *errors; int use_consumed = 0; - if (!PyArg_ParseTuple(args, "yny|i", &str, &len, &errors, &use_consumed)) { + if (!PyArg_ParseTuple(args, "z#nz#|p", &str, &bsize, &len, &errors, &bsize, &use_consumed)) { return NULL; } @@ -513,10 +529,11 @@ writer_finish(PyObject *self_raw, PyObject *Py_UNUSED(args)) static PyMethodDef writer_methods[] = { {"write_char", _PyCFunction_CAST(writer_write_char), METH_VARARGS}, {"write_utf8", _PyCFunction_CAST(writer_write_utf8), METH_VARARGS}, + {"write_ascii", _PyCFunction_CAST(writer_write_ascii), METH_VARARGS}, {"write_widechar", _PyCFunction_CAST(writer_write_widechar), METH_VARARGS}, {"write_ucs4", _PyCFunction_CAST(writer_write_ucs4), METH_VARARGS}, - {"write_str", _PyCFunction_CAST(writer_write_str), METH_VARARGS}, - {"write_repr", _PyCFunction_CAST(writer_write_repr), METH_VARARGS}, + {"write_str", _PyCFunction_CAST(writer_write_str), METH_O}, + {"write_repr", _PyCFunction_CAST(writer_write_repr), METH_O}, {"write_substring", _PyCFunction_CAST(writer_write_substring), METH_VARARGS}, {"decodeutf8stateful", _PyCFunction_CAST(writer_decodeutf8stateful), METH_VARARGS}, {"get_pointer", _PyCFunction_CAST(writer_get_pointer), METH_VARARGS}, diff --git a/Modules/_testcapi/vectorcall.c b/Modules/_testcapi/vectorcall.c index 03aaacb328e0b6d..f89dcb6c4cf03c1 100644 --- a/Modules/_testcapi/vectorcall.c +++ b/Modules/_testcapi/vectorcall.c @@ -179,14 +179,14 @@ _testcapi_VectorCallClass_set_vectorcall_impl(PyObject *self, if (!PyObject_TypeCheck(self, type)) { return PyErr_Format( PyExc_TypeError, - "expected %s instance", - PyType_GetName(type)); + "expected %N instance", + type); } if (!type->tp_vectorcall_offset) { return PyErr_Format( PyExc_TypeError, - "type %s has no vectorcall offset", - PyType_GetName(type)); + "type %N has no vectorcall offset", + type); } *(vectorcallfunc*)((char*)self + type->tp_vectorcall_offset) = ( VectorCallClass_vectorcall); diff --git a/Modules/_testcapi/watchers.c b/Modules/_testcapi/watchers.c index 6d061bb8d510408..66fd2f1746831de 100644 --- a/Modules/_testcapi/watchers.c +++ b/Modules/_testcapi/watchers.c @@ -9,6 +9,7 @@ #include "pycore_function.h" // FUNC_MAX_WATCHERS #include "pycore_interp_structs.h" // CODE_MAX_WATCHERS #include "pycore_context.h" // CONTEXT_MAX_WATCHERS +#include "pycore_lock.h" // _PyOnceFlag /*[clinic input] module _testcapi @@ -18,6 +19,14 @@ module _testcapi // Test dict watching static PyObject *g_dict_watch_events = NULL; static int g_dict_watchers_installed = 0; +static _PyOnceFlag g_dict_watch_once = {0}; + +static int +_init_dict_watch_events(void *arg) +{ + g_dict_watch_events = PyList_New(0); + return g_dict_watch_events ? 0 : -1; +} static int dict_watch_callback(PyDict_WatchEvent event, @@ -106,13 +115,10 @@ add_dict_watcher(PyObject *self, PyObject *kind) if (watcher_id < 0) { return NULL; } - if (!g_dict_watchers_installed) { - assert(!g_dict_watch_events); - if (!(g_dict_watch_events = PyList_New(0))) { - return NULL; - } + if (_PyOnceFlag_CallOnce(&g_dict_watch_once, _init_dict_watch_events, NULL) < 0) { + return NULL; } - g_dict_watchers_installed++; + _Py_atomic_add_int(&g_dict_watchers_installed, 1); return PyLong_FromLong(watcher_id); } @@ -122,10 +128,8 @@ clear_dict_watcher(PyObject *self, PyObject *watcher_id) if (PyDict_ClearWatcher(PyLong_AsLong(watcher_id))) { return NULL; } - g_dict_watchers_installed--; - if (!g_dict_watchers_installed) { - assert(g_dict_watch_events); - Py_CLEAR(g_dict_watch_events); + if (_Py_atomic_add_int(&g_dict_watchers_installed, -1) == 1) { + PyList_Clear(g_dict_watch_events); } Py_RETURN_NONE; } @@ -164,7 +168,7 @@ _testcapi_unwatch_dict_impl(PyObject *module, int watcher_id, PyObject *dict) static PyObject * get_dict_watcher_events(PyObject *self, PyObject *Py_UNUSED(args)) { - if (!g_dict_watch_events) { + if (_Py_atomic_load_int(&g_dict_watchers_installed) <= 0) { PyErr_SetString(PyExc_RuntimeError, "no watchers active"); return NULL; } @@ -364,7 +368,7 @@ add_code_watcher(PyObject *self, PyObject *which_watcher) watcher_id = PyCode_AddWatcher(error_code_event_handler); } else { - PyErr_Format(PyExc_ValueError, "invalid watcher %d", which_l); + PyErr_Format(PyExc_ValueError, "invalid watcher %ld", which_l); return NULL; } if (watcher_id < 0) { @@ -673,7 +677,7 @@ add_context_watcher(PyObject *self, PyObject *which_watcher) assert(PyLong_Check(which_watcher)); long which_l = PyLong_AsLong(which_watcher); if (which_l < 0 || which_l >= (long)Py_ARRAY_LENGTH(callbacks)) { - PyErr_Format(PyExc_ValueError, "invalid watcher %d", which_l); + PyErr_Format(PyExc_ValueError, "invalid watcher %ld", which_l); return NULL; } int watcher_id = PyContext_AddWatcher(callbacks[which_l]); diff --git a/Modules/_testcapi/weakref.c b/Modules/_testcapi/weakref.c new file mode 100644 index 000000000000000..7c3ad8565991b7e --- /dev/null +++ b/Modules/_testcapi/weakref.c @@ -0,0 +1,46 @@ +#include "parts.h" +#include "util.h" + + +static PyObject * +pyweakref_getref(PyObject *module, PyObject *ref) +{ + NULLABLE(ref); + PyObject *obj = UNINITIALIZED_PTR; + int rc = PyWeakref_GetRef(ref, &obj); + if (rc == -1 && PyErr_Occurred()) { + assert(obj == NULL); + return NULL; + } + if (obj == NULL) { + return Py_BuildValue("i", rc); + } + else { + assert(obj != UNINITIALIZED_PTR); + return Py_BuildValue("iN", rc, obj); + } +} + +static PyObject * +pyweakref_isdead(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + int rc = PyWeakref_IsDead(obj); + if (rc == -1 && PyErr_Occurred()) { + return NULL; + } + return PyLong_FromLong(rc); +} + + +static PyMethodDef test_methods[] = { + {"pyweakref_getref", pyweakref_getref, METH_O}, + {"pyweakref_isdead", pyweakref_isdead, METH_O}, + {NULL}, +}; + +int +_PyTestCapi_Init_Weakref(PyObject *m) +{ + return PyModule_AddFunctions(m, test_methods); +} diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 3aa6e4c9e43a263..5784482d38904df 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -15,6 +15,10 @@ #include "frameobject.h" // PyFrame_New() #include "marshal.h" // PyMarshal_WriteLongToFile() +#ifdef bool +# error "The public headers should not include <stdbool.h>, see gh-90904" +#endif + #include <float.h> // FLT_MAX #include <signal.h> #include <stddef.h> // offsetof() @@ -22,9 +26,8 @@ #ifdef HAVE_SYS_WAIT_H # include <sys/wait.h> // W_STOPCODE #endif - -#ifdef bool -# error "The public headers should not include <stdbool.h>, see gh-48924" +#ifdef HAVE_SYS_SYSCTL_H +# include <sys/sysctl.h> // sysctlbyname() #endif #include "_testcapi/util.h" @@ -116,8 +119,8 @@ test_sizeof_c_types(PyObject *self, PyObject *Py_UNUSED(ignored)) do { \ if (EXPECTED != sizeof(TYPE)) { \ PyErr_Format(get_testerror(self), \ - "sizeof(%s) = %u instead of %u", \ - #TYPE, sizeof(TYPE), EXPECTED); \ + "sizeof(%s) = %zu instead of %u", \ + #TYPE, sizeof(TYPE), (unsigned)(EXPECTED)); \ return (PyObject*)NULL; \ } \ } while (0) @@ -226,6 +229,18 @@ pycompilestring(PyObject* self, PyObject *obj) { return Py_CompileString(the_string, "<string>", Py_file_input); } +static PyObject* +pycompilestringexflags(PyObject *self, PyObject *args) { + const char *the_string, *filename; + int start, flags; + if (!PyArg_ParseTuple(args, "ysii", &the_string, &filename, &start, &flags)) { + return NULL; + } + PyCompilerFlags cf = _PyCompilerFlags_INIT; + cf.cf_flags = flags; + return Py_CompileStringExFlags(the_string, filename, start, &cf, -1); +} + static PyObject* test_lazy_hash_inheritance(PyObject* self, PyObject *Py_UNUSED(ignored)) { @@ -2419,12 +2434,22 @@ test_critical_sections(PyObject *module, PyObject *Py_UNUSED(args)) Py_BEGIN_CRITICAL_SECTION2(module, module); Py_END_CRITICAL_SECTION2(); +#ifdef Py_GIL_DISABLED + // avoid unused variable compiler warning on GIL-enabled build + PyMutex mut = {0}; + Py_BEGIN_CRITICAL_SECTION_MUTEX(&mut); + Py_END_CRITICAL_SECTION(); + + Py_BEGIN_CRITICAL_SECTION2_MUTEX(&mut, &mut); + Py_END_CRITICAL_SECTION2(); +#endif + Py_RETURN_NONE; } // Used by `finalize_thread_hang`. -#ifdef _POSIX_THREADS +#if defined(_POSIX_THREADS) && !defined(__wasi__) static void finalize_thread_hang_cleanup_callback(void *Py_UNUSED(arg)) { // Should not reach here. Py_FatalError("pthread thread termination was triggered unexpectedly"); @@ -2546,6 +2571,31 @@ toggle_reftrace_printer(PyObject *ob, PyObject *arg) Py_RETURN_NONE; } + +#ifdef HAVE_SYSCTLBYNAME +static PyObject* +uptime_bsd(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + struct timeval tv; + size_t size = sizeof(tv); + int res = sysctlbyname("kern.boottime", &tv, &size, NULL, 0); + if (res != 0) { + return PyErr_SetFromErrno(PyExc_OSError); + } + double boottime = (double)tv.tv_sec + tv.tv_usec * 1e-6; + + PyTime_t now_t; + if (PyTime_Time(&now_t) < 0) { + return NULL; + } + double now = PyTime_AsSecondsDouble(now_t); + + double uptime = now - boottime; + return PyFloat_FromDouble(uptime); +} +#endif + + static PyMethodDef TestMethods[] = { {"set_errno", set_errno, METH_VARARGS}, {"test_config", test_config, METH_NOARGS}, @@ -2610,6 +2660,7 @@ static PyMethodDef TestMethods[] = { {"return_result_with_error", return_result_with_error, METH_NOARGS}, {"getitem_with_error", getitem_with_error, METH_VARARGS}, {"Py_CompileString", pycompilestring, METH_O}, + {"Py_CompileStringExFlags", pycompilestringexflags, METH_VARARGS}, {"raise_SIGINT_then_send_None", raise_SIGINT_then_send_None, METH_VARARGS}, {"stack_pointer", stack_pointer, METH_NOARGS}, #ifdef W_STOPCODE @@ -2640,6 +2691,9 @@ static PyMethodDef TestMethods[] = { {"test_atexit", test_atexit, METH_NOARGS}, {"code_offset_to_line", _PyCFunction_CAST(code_offset_to_line), METH_FASTCALL}, {"toggle_reftrace_printer", toggle_reftrace_printer, METH_O}, +#ifdef HAVE_SYSCTLBYNAME + {"uptime_bsd", uptime_bsd, METH_NOARGS}, +#endif {NULL, NULL} /* sentinel */ }; @@ -3175,6 +3229,47 @@ create_manual_heap_type(void) return (PyObject *)type; } +typedef struct { + PyObject_VAR_HEAD +} ManagedDictObject; + +int ManagedDict_traverse(PyObject *self, visitproc visit, void *arg) { + Py_VISIT(Py_TYPE(self)); + return PyObject_VisitManagedDict(self, visit, arg); +} + +int ManagedDict_clear(PyObject *self) { + PyObject_ClearManagedDict(self); + return 0; +} + +static PyGetSetDef ManagedDict_getset[] = { + {"__dict__", PyObject_GenericGetDict, PyObject_GenericSetDict, NULL, NULL}, + {NULL, NULL, NULL, NULL, NULL}, +}; + +static PyType_Slot ManagedDict_slots[] = { + {Py_tp_new, (void *)PyType_GenericNew}, + {Py_tp_getset, (void *)ManagedDict_getset}, + {Py_tp_traverse, (void *)ManagedDict_traverse}, + {Py_tp_clear, (void *)ManagedDict_clear}, + {0} +}; + +static PyType_Spec ManagedDict_spec = { + "_testcapi.ManagedDictType", + sizeof(ManagedDictObject), + 0, // itemsize + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_MANAGED_DICT | Py_TPFLAGS_HEAPTYPE | Py_TPFLAGS_HAVE_GC, + ManagedDict_slots +}; + +static PyObject * +create_managed_dict_type(void) +{ + return PyType_FromSpec(&ManagedDict_spec); +} + static struct PyModuleDef _testcapimodule = { PyModuleDef_HEAD_INIT, .m_name = "_testcapi", @@ -3285,6 +3380,10 @@ PyInit__testcapi(void) PyModule_AddObject(m, "INT64_MAX", PyLong_FromInt64(INT64_MAX)); PyModule_AddObject(m, "UINT64_MAX", PyLong_FromUInt64(UINT64_MAX)); + if (PyModule_AddIntMacro(m, _Py_STACK_GROWS_DOWN)) { + return NULL; + } + if (PyModule_AddIntMacro(m, Py_single_input)) { return NULL; } @@ -3315,6 +3414,13 @@ PyInit__testcapi(void) return NULL; } + PyObject *managed_dict_type = create_managed_dict_type(); + if (managed_dict_type == NULL) { + return NULL; + } + if (PyModule_Add(m, "ManagedDictType", managed_dict_type) < 0) { + return NULL; + } /* Include tests from the _testcapi/ directory */ if (_PyTestCapi_Init_Vectorcall(m) < 0) { @@ -3428,6 +3534,9 @@ PyInit__testcapi(void) if (_PyTestCapi_Init_Function(m) < 0) { return NULL; } + if (_PyTestCapi_Init_Weakref(m) < 0) { + return NULL; + } PyState_AddModule(m, &_testcapimodule); return m; diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 3e903b6d87d89fc..1d23198dac52b2d 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -334,14 +334,14 @@ int_converter a: int = 12 b: int(accept={int}) = 34 - c: int(accept={str}) = 45 + c: int(accept={str}) = '-' / [clinic start generated code]*/ static PyObject * int_converter_impl(PyObject *module, int a, int b, int c) -/*[clinic end generated code: output=8e56b59be7d0c306 input=a1dbc6344853db7a]*/ +/*[clinic end generated code: output=8e56b59be7d0c306 input=9a306d4dc907e339]*/ { RETURN_PACKED_ARGS(3, PyLong_FromLong, long, a, b, c); } @@ -1360,6 +1360,7 @@ clone_f2_impl(PyObject *module, const char *path) class custom_t_converter(CConverter): type = 'custom_t' converter = 'custom_converter' + c_init_default = "<placeholder>" # overridden in pre_render(() def pre_render(self): self.c_default = f'''{{ @@ -1367,7 +1368,7 @@ class custom_t_converter(CConverter): }}''' [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=b2fb801e99a06bf6]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=78fe84e5ecc0481b]*/ /*[clinic input] diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index ae060c95fd5a01f..6df5032205e645e 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -21,6 +21,7 @@ #include "pycore_fileutils.h" // _Py_normpath() #include "pycore_flowgraph.h" // _PyCompile_OptimizeCfg() #include "pycore_frame.h" // _PyInterpreterFrame +#include "pycore_function.h" // _PyFunction_GET_BUILTINS #include "pycore_gc.h" // PyGC_Head #include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_import.h" // _PyImport_ClearExtension() @@ -120,10 +121,22 @@ get_c_recursion_remaining(PyObject *self, PyObject *Py_UNUSED(args)) PyThreadState *tstate = _PyThreadState_GET(); uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - int remaining = (int)((here_addr - _tstate->c_stack_soft_limit)/PYOS_STACK_MARGIN_BYTES * 50); + int remaining = (int)((here_addr - _tstate->c_stack_soft_limit) / _PyOS_STACK_MARGIN_BYTES * 50); return PyLong_FromLong(remaining); } +static PyObject* +get_stack_pointer(PyObject *self, PyObject *Py_UNUSED(args)) +{ + uintptr_t here_addr = _Py_get_machine_stack_pointer(); + return PyLong_FromSize_t(here_addr); +} + +static PyObject* +get_stack_margin(PyObject *self, PyObject *Py_UNUSED(args)) +{ + return PyLong_FromSize_t(_PyOS_STACK_MARGIN_BYTES); +} static PyObject* test_bswap(PyObject *self, PyObject *Py_UNUSED(args)) @@ -131,14 +144,14 @@ test_bswap(PyObject *self, PyObject *Py_UNUSED(args)) uint16_t u16 = _Py_bswap16(UINT16_C(0x3412)); if (u16 != UINT16_C(0x1234)) { PyErr_Format(PyExc_AssertionError, - "_Py_bswap16(0x3412) returns %u", u16); + "_Py_bswap16(0x3412) returns %d", u16); return NULL; } uint32_t u32 = _Py_bswap32(UINT32_C(0x78563412)); if (u32 != UINT32_C(0x12345678)) { PyErr_Format(PyExc_AssertionError, - "_Py_bswap32(0x78563412) returns %lu", u32); + "_Py_bswap32(0x78563412) returns %u", u32); return NULL; } @@ -417,7 +430,7 @@ test_edit_cost(PyObject *self, PyObject *Py_UNUSED(args)) static int check_bytes_find(const char *haystack0, const char *needle0, - int offset, Py_ssize_t expected) + Py_ssize_t offset, Py_ssize_t expected) { Py_ssize_t len_haystack = strlen(haystack0); Py_ssize_t len_needle = strlen(needle0); @@ -719,13 +732,17 @@ _testinternalcapi.compiler_codegen -> object compile_mode: int = 0 Apply compiler code generation to an AST. + +Return (instruction_sequence, metadata). metadata maps "argcount", +"posonlyargcount", "kwonlyargcount" to ints and "consts" to the list of +constants in LOAD_CONST index order (for use with optimize_cfg). [clinic start generated code]*/ static PyObject * _testinternalcapi_compiler_codegen_impl(PyObject *module, PyObject *ast, PyObject *filename, int optimize, int compile_mode) -/*[clinic end generated code: output=40a68f6e13951cc8 input=a0e00784f1517cd7]*/ +/*[clinic end generated code: output=40a68f6e13951cc8 input=e0c65e5c80efe30e]*/ { PyCompilerFlags *flags = NULL; return _PyCompile_CodeGen(ast, filename, flags, optimize, compile_mode); @@ -741,12 +758,15 @@ _testinternalcapi.optimize_cfg -> object nlocals: int Apply compiler optimizations to an instruction list. + +consts must be a list aligned with LOAD_CONST opargs (the "consts" entry +from the metadata dict returned by compiler_codegen for the same unit). [clinic start generated code]*/ static PyObject * _testinternalcapi_optimize_cfg_impl(PyObject *module, PyObject *instructions, PyObject *consts, int nlocals) -/*[clinic end generated code: output=57c53c3a3dfd1df0 input=6a96d1926d58d7e5]*/ +/*[clinic end generated code: output=57c53c3a3dfd1df0 input=905c3d935e063b27]*/ { return _PyCompile_OptimizeCfg(instructions, consts, nlocals); } @@ -835,7 +855,7 @@ get_interp_settings(PyObject *self, PyObject *args) } else { PyErr_Format(PyExc_NotImplementedError, - "%zd", interpid); + "%d", interpid); return NULL; } assert(interp != NULL); @@ -1022,7 +1042,7 @@ get_code_var_counts(PyObject *self, PyObject *_args, PyObject *_kwargs) globalsns = PyFunction_GET_GLOBALS(codearg); } if (builtinsns == NULL) { - builtinsns = PyFunction_GET_BUILTINS(codearg); + builtinsns = _PyFunction_GET_BUILTINS(codearg); } codearg = PyFunction_GET_CODE(codearg); } @@ -1045,6 +1065,9 @@ get_code_var_counts(PyObject *self, PyObject *_args, PyObject *_kwargs) #define SET_COUNT(DICT, STRUCT, NAME) \ do { \ PyObject *count = PyLong_FromLong(STRUCT.NAME); \ + if (count == NULL) { \ + goto error; \ + } \ int res = PyDict_SetItemString(DICT, #NAME, count); \ Py_DECREF(count); \ if (res < 0) { \ @@ -1165,6 +1188,47 @@ get_code_var_counts(PyObject *self, PyObject *_args, PyObject *_kwargs) return NULL; } +static PyObject * +verify_stateless_code(PyObject *self, PyObject *args, PyObject *kwargs) +{ + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *codearg; + PyObject *globalnames = NULL; + PyObject *globalsns = NULL; + PyObject *builtinsns = NULL; + static char *kwlist[] = {"code", "globalnames", + "globalsns", "builtinsns", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "O|O!O!O!:get_code_var_counts", kwlist, + &codearg, &PySet_Type, &globalnames, + &PyDict_Type, &globalsns, &PyDict_Type, &builtinsns)) + { + return NULL; + } + if (PyFunction_Check(codearg)) { + if (globalsns == NULL) { + globalsns = PyFunction_GET_GLOBALS(codearg); + } + if (builtinsns == NULL) { + builtinsns = _PyFunction_GET_BUILTINS(codearg); + } + codearg = PyFunction_GET_CODE(codearg); + } + else if (!PyCode_Check(codearg)) { + PyErr_SetString(PyExc_TypeError, + "argument must be a code object or a function"); + return NULL; + } + PyCodeObject *code = (PyCodeObject *)codearg; + + if (_PyCode_VerifyStateless( + tstate, code, globalnames, globalsns, builtinsns) < 0) + { + return NULL; + } + Py_RETURN_NONE; +} + #ifdef _Py_TIER2 static PyObject * @@ -1293,7 +1357,7 @@ pending_identify(PyObject *self, PyObject *args) PyThread_type_lock mutex = PyThread_allocate_lock(); if (mutex == NULL) { - return NULL; + return PyErr_NoMemory(); } PyThread_acquire_lock(mutex, WAIT_LOCK); /* It gets released in _pending_identify_callback(). */ @@ -1649,11 +1713,12 @@ create_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) static PyObject * destroy_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) { - static char *kwlist[] = {"id", NULL}; + static char *kwlist[] = {"id", "basic", NULL}; PyObject *idobj = NULL; + int basic = 0; if (!PyArg_ParseTupleAndKeywords(args, kwargs, - "O:destroy_interpreter", kwlist, - &idobj)) + "O|p:destroy_interpreter", kwlist, + &idobj, &basic)) { return NULL; } @@ -1663,7 +1728,27 @@ destroy_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } - _PyXI_EndInterpreter(interp, NULL, NULL); + if (basic) + { + // Test the basic Py_EndInterpreter with weird out of order thread states + PyThreadState *t1, *t2; + PyThreadState *prev; + t1 = interp->threads.head; + if (t1 == NULL) { + t1 = PyThreadState_New(interp); + } + t2 = PyThreadState_New(interp); + prev = PyThreadState_Swap(t2); + PyThreadState_Clear(t1); + PyThreadState_Delete(t1); + Py_EndInterpreter(t2); + PyThreadState_Swap(prev); + } + else + { + // use the cross interpreter _PyXI_EndInterpreter normally + _PyXI_EndInterpreter(interp, NULL, NULL); + } Py_RETURN_NONE; } @@ -1723,9 +1808,9 @@ exec_interpreter(PyObject *self, PyObject *args, PyObject *kwargs) /* To run some code in a sub-interpreter. -Generally you can use test.support.interpreters, +Generally you can use the interpreters module, but we keep this helper as a distinct implementation. -That's especially important for testing test.support.interpreters. +That's especially important for testing the interpreters module. */ static PyObject * run_in_subinterp_with_config(PyObject *self, PyObject *args, PyObject *kwargs) @@ -1929,7 +2014,14 @@ get_crossinterp_data(PyObject *self, PyObject *args, PyObject *kwargs) return NULL; } if (strcmp(mode, "xidata") == 0) { - if (_PyObject_GetXIData(tstate, obj, xidata) != 0) { + if (_PyObject_GetXIDataNoFallback(tstate, obj, xidata) != 0) { + goto error; + } + } + else if (strcmp(mode, "fallback") == 0) { + xidata_fallback_t fallback = _PyXIDATA_FULL_FALLBACK; + if (_PyObject_GetXIData(tstate, obj, fallback, xidata) != 0) + { goto error; } } @@ -1948,6 +2040,21 @@ get_crossinterp_data(PyObject *self, PyObject *args, PyObject *kwargs) goto error; } } + else if (strcmp(mode, "func") == 0) { + if (_PyFunction_GetXIData(tstate, obj, xidata) != 0) { + goto error; + } + } + else if (strcmp(mode, "script") == 0) { + if (_PyCode_GetScriptXIData(tstate, obj, xidata) != 0) { + goto error; + } + } + else if (strcmp(mode, "script-pure") == 0) { + if (_PyCode_GetPureScriptXIData(tstate, obj, xidata) != 0) { + goto error; + } + } else { PyErr_Format(PyExc_ValueError, "unsupported mode %R", modeobj); goto error; @@ -2150,6 +2257,13 @@ get_tlbc_id(PyObject *Py_UNUSED(module), PyObject *obj) } return PyLong_FromVoidPtr(bc); } + +static PyObject * +get_long_lived_total(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyLong_FromInt64(PyInterpreterState_Get()->gc.long_lived_total); +} + #endif static PyObject * @@ -2258,10 +2372,128 @@ incref_decref_delayed(PyObject *self, PyObject *op) Py_RETURN_NONE; } +#ifdef __EMSCRIPTEN__ +#include "emscripten.h" + +EM_JS(int, emscripten_set_up_async_input_device_js, (void), { + let idx = 0; + const encoder = new TextEncoder(); + const bufs = [ + encoder.encode("ab\n"), + encoder.encode("fi\n"), + encoder.encode("xy\n"), + ]; + function sleep(t) { + return new Promise(res => setTimeout(res, t)); + } + FS.createAsyncInputDevice("/dev", "blah", async () => { + await sleep(5); + return bufs[(idx ++) % 3]; + }); + return !!WebAssembly.promising; +}); + +static PyObject * +emscripten_set_up_async_input_device(PyObject *self, PyObject *Py_UNUSED(ignored)) { + if (emscripten_set_up_async_input_device_js()) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } +} +#endif + +static PyObject * +vectorcall_nop(PyObject *callable, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + Py_RETURN_NONE; +} + +static PyObject * +set_vectorcall_nop(PyObject *self, PyObject *func) +{ + if (!PyFunction_Check(func)) { + PyErr_SetString(PyExc_TypeError, "expected function"); + return NULL; + } + + ((PyFunctionObject*)func)->vectorcall = vectorcall_nop; + Py_RETURN_NONE; +} + + +static void +check_threadstate_set_stack_protection(PyThreadState *tstate, + void *start, size_t size) +{ + assert(PyUnstable_ThreadState_SetStackProtection(tstate, start, size) == 0); + assert(!PyErr_Occurred()); + + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(ts->c_stack_top == (uintptr_t)start + size); + assert(ts->c_stack_hard_limit <= ts->c_stack_soft_limit); + assert(ts->c_stack_soft_limit < ts->c_stack_top); +} + + +static PyObject * +test_threadstate_set_stack_protection(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyThreadState *tstate = PyThreadState_GET(); + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(!PyErr_Occurred()); + + uintptr_t init_base = ts->c_stack_init_base; + size_t init_top = ts->c_stack_init_top; + + // Test the minimum stack size + size_t size = _PyOS_MIN_STACK_SIZE; + void *start = (void*)(_Py_get_machine_stack_pointer() - size); + check_threadstate_set_stack_protection(tstate, start, size); + + // Test a larger size + size = 7654321; + assert(size > _PyOS_MIN_STACK_SIZE); + start = (void*)(_Py_get_machine_stack_pointer() - size); + check_threadstate_set_stack_protection(tstate, start, size); + + // Test invalid size (too small) + size = 5; + start = (void*)(_Py_get_machine_stack_pointer() - size); + assert(PyUnstable_ThreadState_SetStackProtection(tstate, start, size) == -1); + assert(PyErr_ExceptionMatches(PyExc_ValueError)); + PyErr_Clear(); + + // Test PyUnstable_ThreadState_ResetStackProtection() + PyUnstable_ThreadState_ResetStackProtection(tstate); + assert(ts->c_stack_init_base == init_base); + assert(ts->c_stack_init_top == init_top); + + Py_RETURN_NONE; +} + + +static PyObject * +_pyerr_setkeyerror(PyObject *self, PyObject *arg) +{ + // Test that _PyErr_SetKeyError() overrides the current exception + // if an exception is set + PyErr_NoMemory(); + + _PyErr_SetKeyError(arg); + + assert(PyErr_Occurred()); + return NULL; +} + + static PyMethodDef module_functions[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, {"get_c_recursion_remaining", get_c_recursion_remaining, METH_NOARGS}, + {"get_stack_pointer", get_stack_pointer, METH_NOARGS}, + {"get_stack_margin", get_stack_margin, METH_NOARGS}, {"test_bswap", test_bswap, METH_NOARGS}, {"test_popcount", test_popcount, METH_NOARGS}, {"test_bit_length", test_bit_length, METH_NOARGS}, @@ -2292,6 +2524,8 @@ static PyMethodDef module_functions[] = { {"get_co_localskinds", get_co_localskinds, METH_O, NULL}, {"get_code_var_counts", _PyCFunction_CAST(get_code_var_counts), METH_VARARGS | METH_KEYWORDS, NULL}, + {"verify_stateless_code", _PyCFunction_CAST(verify_stateless_code), + METH_VARARGS | METH_KEYWORDS, NULL}, #ifdef _Py_TIER2 {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL}, {"invalidate_executors", invalidate_executors, METH_O, NULL}, @@ -2346,6 +2580,7 @@ static PyMethodDef module_functions[] = { {"py_thread_id", get_py_thread_id, METH_NOARGS}, {"get_tlbc", get_tlbc, METH_O, NULL}, {"get_tlbc_id", get_tlbc_id, METH_O, NULL}, + {"get_long_lived_total", get_long_lived_total, METH_NOARGS}, #endif #ifdef _Py_TIER2 {"uop_symbols_test", _Py_uop_symbols_test, METH_NOARGS}, @@ -2358,6 +2593,13 @@ static PyMethodDef module_functions[] = { {"is_static_immortal", is_static_immortal, METH_O}, {"incref_decref_delayed", incref_decref_delayed, METH_O}, GET_NEXT_DICT_KEYS_VERSION_METHODDEF +#ifdef __EMSCRIPTEN__ + {"emscripten_set_up_async_input_device", emscripten_set_up_async_input_device, METH_NOARGS}, +#endif + {"set_vectorcall_nop", set_vectorcall_nop, METH_O}, + {"test_threadstate_set_stack_protection", + test_threadstate_set_stack_protection, METH_NOARGS}, + {"_pyerr_setkeyerror", _pyerr_setkeyerror, METH_O}, {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testinternalcapi/test_critical_sections.c b/Modules/_testinternalcapi/test_critical_sections.c index e0ba37abcdd3329..72a1fa2cdc7224d 100644 --- a/Modules/_testinternalcapi/test_critical_sections.c +++ b/Modules/_testinternalcapi/test_critical_sections.c @@ -4,6 +4,8 @@ #include "parts.h" #include "pycore_critical_section.h" +#include "pycore_pystate.h" +#include "pycore_pythread.h" #ifdef MS_WINDOWS # include <windows.h> // Sleep() @@ -284,13 +286,196 @@ test_critical_sections_gc(PyObject *self, PyObject *Py_UNUSED(args)) #endif +#ifdef Py_GIL_DISABLED + +static PyObject * +test_critical_section1_reacquisition(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyObject *a = PyDict_New(); + assert(a != NULL); + + PyCriticalSection cs1, cs2; + // First acquisition of critical section on object locks it + PyCriticalSection_Begin(&cs1, a); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert(_PyThreadState_GET()->critical_section == (uintptr_t)&cs1); + // Attempting to re-acquire critical section on same object which + // is already locked by top-most critical section is a no-op. + PyCriticalSection_Begin(&cs2, a); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert(_PyThreadState_GET()->critical_section == (uintptr_t)&cs1); + // Releasing second critical section is a no-op. + PyCriticalSection_End(&cs2); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert(_PyThreadState_GET()->critical_section == (uintptr_t)&cs1); + // Releasing first critical section unlocks the object + PyCriticalSection_End(&cs1); + assert(!PyMutex_IsLocked(&a->ob_mutex)); + + Py_DECREF(a); + Py_RETURN_NONE; +} + +static PyObject * +test_critical_section2_reacquisition(PyObject *self, PyObject *Py_UNUSED(args)) +{ + PyObject *a = PyDict_New(); + assert(a != NULL); + PyObject *b = PyDict_New(); + assert(b != NULL); + + PyCriticalSection2 cs; + // First acquisition of critical section on objects locks them + PyCriticalSection2_Begin(&cs, a, b); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + + // Attempting to re-acquire critical section on either of two + // objects already locked by top-most critical section is a no-op. + + // Check re-acquiring on first object + PyCriticalSection a_cs; + PyCriticalSection_Begin(&a_cs, a); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + // Releasing critical section on either object is a no-op. + PyCriticalSection_End(&a_cs); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + + // Check re-acquiring on second object + PyCriticalSection b_cs; + PyCriticalSection_Begin(&b_cs, b); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + // Releasing critical section on either object is a no-op. + PyCriticalSection_End(&b_cs); + assert(PyMutex_IsLocked(&a->ob_mutex)); + assert(PyMutex_IsLocked(&b->ob_mutex)); + assert(_PyCriticalSection_IsActive(PyThreadState_GET()->critical_section)); + assert((_PyThreadState_GET()->critical_section & + ~_Py_CRITICAL_SECTION_MASK) == (uintptr_t)&cs); + + // Releasing critical section on both objects unlocks them + PyCriticalSection2_End(&cs); + assert(!PyMutex_IsLocked(&a->ob_mutex)); + assert(!PyMutex_IsLocked(&b->ob_mutex)); + + Py_DECREF(a); + Py_DECREF(b); + Py_RETURN_NONE; +} + +#endif // Py_GIL_DISABLED + +#ifdef Py_CAN_START_THREADS + +// gh-144513: Test that critical sections don't deadlock with stop-the-world. +// This test is designed to deadlock (timeout) on builds without the fix. +struct test_data_stw { + PyObject *obj; + Py_ssize_t num_threads; + Py_ssize_t started; + PyEvent ready; +}; + +static void +thread_stw(void *arg) +{ + struct test_data_stw *test_data = arg; + PyGILState_STATE gil = PyGILState_Ensure(); + + if (_Py_atomic_add_ssize(&test_data->started, 1) == test_data->num_threads - 1) { + _PyEvent_Notify(&test_data->ready); + } + + // All threads: acquire critical section and hold it long enough to + // trigger TIME_TO_BE_FAIR_NS (1 ms), which causes direct handoff on unlock. + Py_BEGIN_CRITICAL_SECTION(test_data->obj); + pysleep(10); // 10 ms = 10 x TIME_TO_BE_FAIR_NS + Py_END_CRITICAL_SECTION(); + + PyGILState_Release(gil); +} + +static PyObject * +test_critical_sections_stw(PyObject *self, PyObject *Py_UNUSED(args)) +{ + // gh-144513: Test that critical sections don't deadlock during STW. + // + // The deadlock occurs when lock ownership is handed off (due to fairness + // after TIME_TO_BE_FAIR_NS) to a thread that has already suspended for + // stop-the-world. The STW requester then cannot acquire the lock. + // + // With the fix, the STW requester detects world_stopped and skips locking. + + #define STW_NUM_THREADS 2 + struct test_data_stw test_data = { + .obj = PyDict_New(), + .num_threads = STW_NUM_THREADS, + }; + if (test_data.obj == NULL) { + return NULL; + } + + PyThread_handle_t handles[STW_NUM_THREADS]; + PyThread_ident_t idents[STW_NUM_THREADS]; + for (Py_ssize_t i = 0; i < STW_NUM_THREADS; i++) { + PyThread_start_joinable_thread(&thread_stw, &test_data, + &idents[i], &handles[i]); + } + + // Wait for threads to start, then let them compete for the lock + PyEvent_Wait(&test_data.ready); + pysleep(5); + + // Request stop-the-world and try to acquire the critical section. + // Without the fix, this may deadlock. + PyInterpreterState *interp = PyInterpreterState_Get(); + _PyEval_StopTheWorld(interp); + + Py_BEGIN_CRITICAL_SECTION(test_data.obj); + Py_END_CRITICAL_SECTION(); + + _PyEval_StartTheWorld(interp); + + for (Py_ssize_t i = 0; i < STW_NUM_THREADS; i++) { + PyThread_join_thread(handles[i]); + } + #undef STW_NUM_THREADS + Py_DECREF(test_data.obj); + Py_RETURN_NONE; +} + +#endif // Py_CAN_START_THREADS + static PyMethodDef test_methods[] = { {"test_critical_sections", test_critical_sections, METH_NOARGS}, {"test_critical_sections_nest", test_critical_sections_nest, METH_NOARGS}, {"test_critical_sections_suspend", test_critical_sections_suspend, METH_NOARGS}, +#ifdef Py_GIL_DISABLED + {"test_critical_section1_reacquisition", test_critical_section1_reacquisition, METH_NOARGS}, + {"test_critical_section2_reacquisition", test_critical_section2_reacquisition, METH_NOARGS}, +#endif #ifdef Py_CAN_START_THREADS {"test_critical_sections_threads", test_critical_sections_threads, METH_NOARGS}, {"test_critical_sections_gc", test_critical_sections_gc, METH_NOARGS}, + {"test_critical_sections_stw", test_critical_sections_stw, METH_NOARGS}, #endif {NULL, NULL} /* sentinel */ }; diff --git a/Modules/_testinternalcapi/test_lock.c b/Modules/_testinternalcapi/test_lock.c index 8d678412fe71791..ded76ca9fe68194 100644 --- a/Modules/_testinternalcapi/test_lock.c +++ b/Modules/_testinternalcapi/test_lock.c @@ -57,7 +57,10 @@ lock_thread(void *arg) _Py_atomic_store_int(&test_data->started, 1); PyMutex_Lock(m); - assert(m->_bits == 1); + // gh-135641: in rare cases the lock may still have `_Py_HAS_PARKED` set + // (m->_bits == 3) due to bucket collisions in the parking lot hash table + // between this mutex and the `test_data.done` event. + assert(m->_bits == 1 || m->_bits == 3); PyMutex_Unlock(m); assert(m->_bits == 0); @@ -88,7 +91,8 @@ test_lock_two_threads(PyObject *self, PyObject *obj) } while (v != 3 && iters < 200); // both the "locked" and the "has parked" bits should be set - assert(test_data.m._bits == 3); + v = _Py_atomic_load_uint8_relaxed(&test_data.m._bits); + assert(v == 3); PyMutex_Unlock(&test_data.m); PyEvent_Wait(&test_data.done); diff --git a/Modules/_testlimitedcapi.c b/Modules/_testlimitedcapi.c index 4dae99ec92a0850..00addafd5b982c0 100644 --- a/Modules/_testlimitedcapi.c +++ b/Modules/_testlimitedcapi.c @@ -38,6 +38,9 @@ PyInit__testlimitedcapi(void) if (_PyTestLimitedCAPI_Init_Bytes(mod) < 0) { return NULL; } + if (_PyTestLimitedCAPI_Init_Capsule(mod) < 0) { + return NULL; + } if (_PyTestLimitedCAPI_Init_Codec(mod) < 0) { return NULL; } @@ -92,5 +95,11 @@ PyInit__testlimitedcapi(void) if (_PyTestLimitedCAPI_Init_File(mod) < 0) { return NULL; } + if (_PyTestLimitedCAPI_Init_Weakref(mod) < 0) { + return NULL; + } + if (_PyTestLimitedCAPI_Init_Run(mod) < 0) { + return NULL; + } return mod; } diff --git a/Modules/_testlimitedcapi/capsule.c b/Modules/_testlimitedcapi/capsule.c new file mode 100644 index 000000000000000..07b85823f399ba3 --- /dev/null +++ b/Modules/_testlimitedcapi/capsule.c @@ -0,0 +1,63 @@ +#include "parts.h" + +static void +capsule_destructor(PyObject *op) +{ + /* If non-NULL, the name and the pointer are the same allocation. */ + free((char *)PyCapsule_GetName(op)); +} + +static PyObject * +capsule_new(PyObject *self, PyObject *arg) +{ + const char *name; + Py_ssize_t size; + if (!PyArg_Parse(arg, "z#", &name, &size)) { + return NULL; + } + char *name_copy = NULL; + if (name != NULL) { + name_copy = strdup(name); + if (name_copy == NULL) { + return PyErr_NoMemory(); + } + } + static const char dummy = 0; + void *pointer = name_copy != NULL ? (void *)name_copy : (void *)&dummy; + PyObject *capsule = PyCapsule_New(pointer, name_copy, capsule_destructor); + if (capsule == NULL) { + free(name_copy); + } + return capsule; +} + +static PyObject * +pycapsule_import(PyObject *self, PyObject *args) +{ + const char *name; + Py_ssize_t size; + int no_block = 0; + if (!PyArg_ParseTuple(args, "z#|i", &name, &size, &no_block)) { + return NULL; + } + void *pointer = PyCapsule_Import(name, no_block); + if (pointer == NULL) { + return NULL; + } + /* Capsules created by capsule_new() store a copy of their name as the + pointer, so a successful import round-trips the name. Only use this + function with such capsules. */ + return PyUnicode_FromString((const char *)pointer); +} + +static PyMethodDef test_methods[] = { + {"capsule_new", capsule_new, METH_O}, + {"PyCapsule_Import", pycapsule_import, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Capsule(PyObject *m) +{ + return PyModule_AddFunctions(m, test_methods); +} diff --git a/Modules/_testlimitedcapi/clinic/long.c.h b/Modules/_testlimitedcapi/clinic/long.c.h index ebaeb53921a82f1..f9852aba266a570 100644 --- a/Modules/_testlimitedcapi/clinic/long.c.h +++ b/Modules/_testlimitedcapi/clinic/long.c.h @@ -84,8 +84,8 @@ PyDoc_STRVAR(_testlimitedcapi_test_long_as_size_t__doc__, "\n" "Test the PyLong_As{Size,Ssize}_t API.\n" "\n" -"At present this just tests that non-integer arguments are handled correctly.\n" -"It should be extended to test overflow handling."); +"At present this just tests that non-integer arguments are handled\n" +"correctly. It should be extended to test overflow handling."); #define _TESTLIMITEDCAPI_TEST_LONG_AS_SIZE_T_METHODDEF \ {"test_long_as_size_t", (PyCFunction)_testlimitedcapi_test_long_as_size_t, METH_NOARGS, _testlimitedcapi_test_long_as_size_t__doc__}, @@ -140,4 +140,4 @@ PyDoc_STRVAR(_testlimitedcapi_PyLong_AsInt__doc__, #define _TESTLIMITEDCAPI_PYLONG_ASINT_METHODDEF \ {"PyLong_AsInt", (PyCFunction)_testlimitedcapi_PyLong_AsInt, METH_O, _testlimitedcapi_PyLong_AsInt__doc__}, -/*[clinic end generated code: output=bc52b73c599f96c2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fb5c95bd0a4bdad8 input=a9049054013a1b77]*/ diff --git a/Modules/_testlimitedcapi/long.c b/Modules/_testlimitedcapi/long.c index 34bc7331da92477..99b9e96760d50df 100644 --- a/Modules/_testlimitedcapi/long.c +++ b/Modules/_testlimitedcapi/long.c @@ -451,13 +451,13 @@ _testlimitedcapi.test_long_as_size_t Test the PyLong_As{Size,Ssize}_t API. -At present this just tests that non-integer arguments are handled correctly. -It should be extended to test overflow handling. +At present this just tests that non-integer arguments are handled +correctly. It should be extended to test overflow handling. [clinic start generated code]*/ static PyObject * _testlimitedcapi_test_long_as_size_t_impl(PyObject *module) -/*[clinic end generated code: output=297a9f14a42f55af input=8923d8f2038c46f4]*/ +/*[clinic end generated code: output=297a9f14a42f55af input=692e73744b35bf6e]*/ { size_t out_u; Py_ssize_t out_s; diff --git a/Modules/_testlimitedcapi/parts.h b/Modules/_testlimitedcapi/parts.h index 60f6f03011a65c4..3d19e945bfdbc6c 100644 --- a/Modules/_testlimitedcapi/parts.h +++ b/Modules/_testlimitedcapi/parts.h @@ -25,6 +25,7 @@ int _PyTestLimitedCAPI_Init_Abstract(PyObject *module); int _PyTestLimitedCAPI_Init_ByteArray(PyObject *module); int _PyTestLimitedCAPI_Init_Bytes(PyObject *module); +int _PyTestLimitedCAPI_Init_Capsule(PyObject *module); int _PyTestLimitedCAPI_Init_Codec(PyObject *module); int _PyTestLimitedCAPI_Init_Complex(PyObject *module); int _PyTestLimitedCAPI_Init_Dict(PyObject *module); @@ -43,5 +44,7 @@ int _PyTestLimitedCAPI_Init_Unicode(PyObject *module); int _PyTestLimitedCAPI_Init_VectorcallLimited(PyObject *module); int _PyTestLimitedCAPI_Init_Version(PyObject *module); int _PyTestLimitedCAPI_Init_File(PyObject *module); +int _PyTestLimitedCAPI_Init_Weakref(PyObject *module); +int _PyTestLimitedCAPI_Init_Run(PyObject *module); #endif // Py_TESTLIMITEDCAPI_PARTS_H diff --git a/Modules/_testlimitedcapi/run.c b/Modules/_testlimitedcapi/run.c new file mode 100644 index 000000000000000..ca12e5faa5b3f96 --- /dev/null +++ b/Modules/_testlimitedcapi/run.c @@ -0,0 +1,35 @@ +#include "parts.h" +#include "util.h" + + +// Test functions, not macros +#undef Py_CompileString + + +// Test Py_CompileString() +static PyObject* +run_compilestring(PyObject *mod, PyObject *args) +{ + const char *str; + const char *filename; + int start; + + if (!PyArg_ParseTuple(args, "yyi", &str, &filename, &start)) { + return NULL; + } + + return Py_CompileString(str, filename, start); +} + + +static PyMethodDef test_methods[] = { + {"run_compilestring", run_compilestring, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Run(PyObject *m) +{ + return PyModule_AddFunctions(m, test_methods); +} + diff --git a/Modules/_testlimitedcapi/set.c b/Modules/_testlimitedcapi/set.c index 35da5fa5f008e1c..34ed6b1d60b5a41 100644 --- a/Modules/_testlimitedcapi/set.c +++ b/Modules/_testlimitedcapi/set.c @@ -155,6 +155,51 @@ test_frozenset_add_in_capi(PyObject *self, PyObject *Py_UNUSED(obj)) return NULL; } +static PyObject * +test_set_contains_does_not_convert_unhashable_key(PyObject *self, PyObject *Py_UNUSED(obj)) +{ + // See https://docs.python.org/3/c-api/set.html#c.PySet_Contains + PyObject *outer_set = PySet_New(NULL); + + PyObject *needle = PySet_New(NULL); + if (needle == NULL) { + Py_DECREF(outer_set); + return NULL; + } + + PyObject *num = PyLong_FromLong(42); + if (num == NULL) { + Py_DECREF(outer_set); + Py_DECREF(needle); + return NULL; + } + + if (PySet_Add(needle, num) < 0) { + Py_DECREF(outer_set); + Py_DECREF(needle); + Py_DECREF(num); + return NULL; + } + + int result = PySet_Contains(outer_set, needle); + + Py_DECREF(num); + Py_DECREF(needle); + Py_DECREF(outer_set); + + if (result < 0) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_Clear(); + Py_RETURN_NONE; + } + return NULL; + } + + PyErr_SetString(PyExc_AssertionError, + "PySet_Contains should have raised TypeError for unhashable key"); + return NULL; +} + static PyMethodDef test_methods[] = { {"set_check", set_check, METH_O}, {"set_checkexact", set_checkexact, METH_O}, @@ -174,6 +219,8 @@ static PyMethodDef test_methods[] = { {"set_clear", set_clear, METH_O}, {"test_frozenset_add_in_capi", test_frozenset_add_in_capi, METH_NOARGS}, + {"test_set_contains_does_not_convert_unhashable_key", + test_set_contains_does_not_convert_unhashable_key, METH_NOARGS}, {NULL}, }; diff --git a/Modules/_testlimitedcapi/weakref.c b/Modules/_testlimitedcapi/weakref.c new file mode 100644 index 000000000000000..e7f9d54d1a0d59e --- /dev/null +++ b/Modules/_testlimitedcapi/weakref.c @@ -0,0 +1,78 @@ +#include "pyconfig.h" // Py_GIL_DISABLED +#ifndef Py_GIL_DISABLED + // Need limited C API 3.5 for PyModule_AddFunctions() +# define Py_LIMITED_API 0x03050000 +#endif + +#include "parts.h" +#include "util.h" + + +static PyObject * +pyweakref_check(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyWeakref_Check(obj)); +} + +static PyObject * +pyweakref_checkref(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyWeakref_CheckRef(obj)); +} + +static PyObject * +pyweakref_checkrefexact(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyWeakref_CheckRefExact(obj)); +} + +static PyObject * +pyweakref_checkproxy(PyObject *module, PyObject *obj) +{ + NULLABLE(obj); + return PyLong_FromLong(PyWeakref_CheckProxy(obj)); +} + +static PyObject * +pyweakref_newref(PyObject *module, PyObject *args) +{ + PyObject *obj; + PyObject *callback = NULL; + if (!PyArg_ParseTuple(args, "O|O", &obj, &callback)) { + return NULL; + } + NULLABLE(obj); + return PyWeakref_NewRef(obj, callback); +} + +static PyObject * +pyweakref_newproxy(PyObject *module, PyObject *args) +{ + PyObject *obj; + PyObject *callback = NULL; + if (!PyArg_ParseTuple(args, "O|O", &obj, &callback)) { + return NULL; + } + NULLABLE(obj); + return PyWeakref_NewProxy(obj, callback); +} + + +static PyMethodDef test_methods[] = { + {"pyweakref_check", pyweakref_check, METH_O}, + {"pyweakref_checkref", pyweakref_checkref, METH_O}, + {"pyweakref_checkrefexact", pyweakref_checkrefexact, METH_O}, + {"pyweakref_checkproxy", pyweakref_checkproxy, METH_O}, + {"pyweakref_newref", pyweakref_newref, METH_VARARGS}, + {"pyweakref_newproxy", pyweakref_newproxy, METH_VARARGS}, + {NULL}, +}; + +int +_PyTestLimitedCAPI_Init_Weakref(PyObject *m) +{ + return PyModule_AddFunctions(m, test_methods); +} diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index bfec0678e2c6695..6b0e367c316d5ee 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -141,14 +141,14 @@ _testmultiphase.StateAccessType.get_defining_module Return the module of the defining class. -Also tests that result of PyType_GetModuleByDef matches defining_class's -module. +Also tests that result of PyType_GetModuleByDef matches +defining_class's module. [clinic start generated code]*/ static PyObject * _testmultiphase_StateAccessType_get_defining_module_impl(StateAccessTypeObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=ba2a14284a5d0921 input=d2c7245c8a9d06f8]*/ +/*[clinic end generated code: output=ba2a14284a5d0921 input=903e7f66555d65ae]*/ { PyObject *retval; retval = PyType_GetModule(cls); diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index 2c59085d15b5bee..f74b964faf35fbc 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -799,3 +799,11 @@ PyInit__testsinglephase_circular(void) } return Py_NewRef(static_module_circular); } + + +PyMODINIT_FUNC +PyInit__testsinglephase_raise_exception(void) +{ + PyErr_SetString(PyExc_RuntimeError, "evil"); + return NULL; +} diff --git a/Modules/_threadmodule.c b/Modules/_threadmodule.c index 9776a32755db686..3aa3648b8103cbf 100644 --- a/Modules/_threadmodule.c +++ b/Modules/_threadmodule.c @@ -281,6 +281,12 @@ _PyThread_AfterFork(struct _pythread_runtime_state *state) continue; } + // Keep handles for threads that have not been started yet. They are + // safe to start in the child process. + if (handle->state == THREAD_HANDLE_NOT_STARTED) { + continue; + } + // Mark all threads as done. Any attempts to join or detach the // underlying OS thread (if any) could crash. We are the only thread; // it's safe to set this non-atomically. @@ -661,12 +667,12 @@ PyThreadHandleObject_join(PyObject *op, PyObject *args) PyThreadHandleObject *self = PyThreadHandleObject_CAST(op); PyObject *timeout_obj = NULL; - if (!PyArg_ParseTuple(args, "|O?:join", &timeout_obj)) { + if (!PyArg_ParseTuple(args, "|O:join", &timeout_obj)) { return NULL; } PyTime_t timeout_ns = -1; - if (timeout_obj != NULL) { + if (timeout_obj != NULL && timeout_obj != Py_None) { if (_PyTime_FromSecondsObject(&timeout_ns, timeout_obj, _PyTime_ROUND_TIMEOUT) < 0) { return NULL; @@ -684,6 +690,9 @@ PyThreadHandleObject_is_done(PyObject *op, PyObject *Py_UNUSED(dummy)) { PyThreadHandleObject *self = PyThreadHandleObject_CAST(op); if (_PyEvent_IsSet(&self->handle->thread_is_exiting)) { + if (_PyOnceFlag_CallOnce(&self->handle->once, join_thread, self->handle) == -1) { + return NULL; + } Py_RETURN_TRUE; } else { @@ -1011,6 +1020,11 @@ rlock_traverse(PyObject *self, visitproc visit, void *arg) return 0; } +static int +rlock_locked_impl(rlockobject *self) +{ + return PyMutex_IsLocked(&self->lock.mutex); +} static void rlock_dealloc(PyObject *self) @@ -1100,7 +1114,7 @@ static PyObject * rlock_locked(PyObject *op, PyObject *Py_UNUSED(ignored)) { rlockobject *self = rlockobject_CAST(op); - int is_locked = _PyRecursiveMutex_IsLockedByCurrentThread(&self->lock); + int is_locked = rlock_locked_impl(self); return PyBool_FromLong(is_locked); } @@ -1201,11 +1215,18 @@ static PyObject * rlock_repr(PyObject *op) { rlockobject *self = rlockobject_CAST(op); - PyThread_ident_t owner = self->lock.thread; - size_t count = self->lock.level + 1; + PyThread_ident_t owner = FT_ATOMIC_LOAD_ULLONG_RELAXED(self->lock.thread); + int locked = rlock_locked_impl(self); + size_t count; + if (locked) { + count = self->lock.level + 1; + } + else { + count = 0; + } return PyUnicode_FromFormat( "<%s %s object owner=%" PY_FORMAT_THREAD_IDENT_T " count=%zu at %p>", - owner ? "locked" : "unlocked", + locked ? "locked" : "unlocked", Py_TYPE(self)->tp_name, owner, count, self); } @@ -1345,9 +1366,7 @@ static void localdummy_dealloc(PyObject *op) { localdummyobject *self = localdummyobject_CAST(op); - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, self->weakreflist); PyTypeObject *tp = Py_TYPE(self); tp->tp_free(self); Py_DECREF(tp); @@ -1929,10 +1948,10 @@ thread_PyThread_start_joinable_thread(PyObject *module, PyObject *fargs, PyObject *func = NULL; int daemon = 1; thread_module_state *state = get_thread_state(module); - PyObject *hobj = Py_None; + PyObject *hobj = NULL; if (!PyArg_ParseTupleAndKeywords(fargs, fkwargs, - "O|O!?p:start_joinable_thread", keywords, - &func, state->thread_handle_type, &hobj, &daemon)) { + "O|Op:start_joinable_thread", keywords, + &func, &hobj, &daemon)) { return NULL; } @@ -1942,6 +1961,14 @@ thread_PyThread_start_joinable_thread(PyObject *module, PyObject *fargs, return NULL; } + if (hobj == NULL) { + hobj = Py_None; + } + else if (hobj != Py_None && !Py_IS_TYPE(hobj, state->thread_handle_type)) { + PyErr_SetString(PyExc_TypeError, "'handle' must be a _ThreadHandle"); + return NULL; + } + if (PySys_Audit("_thread.start_joinable_thread", "OiO", func, daemon, hobj) < 0) { return NULL; @@ -2120,9 +2147,10 @@ thread_stack_size(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "|n:stack_size", &new_size)) return NULL; - if (new_size < 0) { - PyErr_SetString(PyExc_ValueError, - "size must be 0 or a positive value"); + Py_ssize_t min_size = _PyOS_MIN_STACK_SIZE + SYSTEM_PAGE_SIZE; + if (new_size != 0 && new_size < min_size) { + PyErr_Format(PyExc_ValueError, + "size must be at least %zi bytes", min_size); return NULL; } @@ -2301,7 +2329,7 @@ thread_excepthook(PyObject *module, PyObject *args) } PyDoc_STRVAR(excepthook_doc, -"_excepthook($module, (exc_type, exc_value, exc_traceback, thread), /)\n\ +"_excepthook($module, args, /)\n\ --\n\ \n\ Handle uncaught Thread.run() exception."); @@ -2447,7 +2475,9 @@ _thread__get_name_impl(PyObject *module) } #ifdef __sun - return PyUnicode_DecodeUTF8(name, strlen(name), "surrogateescape"); + // gh-138004: Decode Solaris/Illumos (e.g. OpenIndiana) thread names + // from ASCII, since OpenIndiana only supports ASCII names. + return PyUnicode_DecodeASCII(name, strlen(name), "surrogateescape"); #else return PyUnicode_DecodeFSDefault(name); #endif @@ -2485,8 +2515,9 @@ _thread_set_name_impl(PyObject *module, PyObject *name_obj) { #ifndef MS_WINDOWS #ifdef __sun - // Solaris always uses UTF-8 - const char *encoding = "utf-8"; + // gh-138004: Encode Solaris/Illumos thread names to ASCII, + // since OpenIndiana does not support non-ASCII names. + const char *encoding = "ascii"; #else // Encode the thread name to the filesystem encoding using the "replace" // error handler diff --git a/Modules/_tkinter.c b/Modules/_tkinter.c index 77695401919cb7a..3b373cdc366c170 100644 --- a/Modules/_tkinter.c +++ b/Modules/_tkinter.c @@ -152,18 +152,20 @@ _get_tcl_lib_path(void) } /* Check expected location for an installed Python first */ - tcl_library_path = PyUnicode_FromString("\\tcl\\tcl" TCL_VERSION); - if (tcl_library_path == NULL) { + PyObject* tmp_tcl_library_path = PyUnicode_FromString("\\tcl\\tcl" TCL_VERSION); + if (tmp_tcl_library_path == NULL) { Py_DECREF(prefix); return NULL; } - tcl_library_path = PyUnicode_Concat(prefix, tcl_library_path); + tcl_library_path = PyUnicode_Concat(prefix, tmp_tcl_library_path); + Py_DECREF(tmp_tcl_library_path); Py_DECREF(prefix); if (tcl_library_path == NULL) { return NULL; } stat_return_value = _Py_stat(tcl_library_path, &stat_buf); if (stat_return_value == -2) { + Py_DECREF(tcl_library_path); return NULL; } if (stat_return_value == -1) { @@ -178,16 +180,17 @@ _get_tcl_lib_path(void) } stat_return_value = _Py_stat(tcl_library_path, &stat_buf); if (stat_return_value == -2) { + Py_DECREF(tcl_library_path); return NULL; } if (stat_return_value == -1) { /* tcltkDir for a repository build doesn't exist either, reset errno and leave Tcl to its own devices */ errno = 0; - tcl_library_path = NULL; + Py_CLEAR(tcl_library_path); } #else - tcl_library_path = NULL; + Py_CLEAR(tcl_library_path); #endif } already_checked = 1; @@ -599,8 +602,12 @@ Tkapp_New(const char *screenName, const char *className, v->interp = Tcl_CreateInterp(); v->wantobjects = wantobjects; +#if TCL_MAJOR_VERSION >= 9 + v->threaded = 1; +#else v->threaded = Tcl_GetVar2Ex(v->interp, "tcl_platform", "threaded", TCL_GLOBAL_ONLY) != NULL; +#endif v->thread_id = Tcl_GetCurrentThread(); v->dispatching = 0; v->trace = NULL; @@ -728,11 +735,13 @@ Tkapp_New(const char *screenName, const char *className, if (!ret && GetLastError() == ERROR_ENVVAR_NOT_FOUND) { str_path = _get_tcl_lib_path(); if (str_path == NULL && PyErr_Occurred()) { + Py_DECREF(v); return NULL; } if (str_path != NULL) { utf8_path = PyUnicode_AsUTF8String(str_path); if (utf8_path == NULL) { + Py_DECREF(v); return NULL; } Tcl_SetVar(v->interp, @@ -964,6 +973,40 @@ asBignumObj(PyObject *value) return result; } +static Tcl_Obj* AsObj(PyObject *value); + +static Tcl_Obj* +TupleAsObj(PyObject *value, int wrapped) +{ + Tcl_Obj *result = NULL; + Py_ssize_t size = PyTuple_GET_SIZE(value); + if (size == 0) { + return Tcl_NewListObj(0, NULL); + } + if (!CHECK_SIZE(size, sizeof(Tcl_Obj *))) { + PyErr_SetString(PyExc_OverflowError, + wrapped ? "list is too long" : "tuple is too long"); + return NULL; + } + Tcl_Obj **argv = (Tcl_Obj **)PyMem_Malloc(((size_t)size) * sizeof(Tcl_Obj *)); + if (argv == NULL) { + PyErr_NoMemory(); + return NULL; + } + for (Py_ssize_t i = 0; i < size; i++) { + Tcl_Obj *item = AsObj(PyTuple_GET_ITEM(value, i)); + if (item == NULL) { + goto exit; + } + argv[i] = item; + } + result = Tcl_NewListObj((int)size, argv); + +exit: + PyMem_Free(argv); + return result; +} + static Tcl_Obj* AsObj(PyObject *value) { @@ -1010,28 +1053,17 @@ AsObj(PyObject *value) if (PyFloat_Check(value)) return Tcl_NewDoubleObj(PyFloat_AS_DOUBLE(value)); - if (PyTuple_Check(value) || PyList_Check(value)) { - Tcl_Obj **argv; - Py_ssize_t size, i; - - size = PySequence_Fast_GET_SIZE(value); - if (size == 0) - return Tcl_NewListObj(0, NULL); - if (!CHECK_SIZE(size, sizeof(Tcl_Obj *))) { - PyErr_SetString(PyExc_OverflowError, - PyTuple_Check(value) ? "tuple is too long" : - "list is too long"); + if (PyTuple_Check(value)) { + return TupleAsObj(value, false); + } + + if (PyList_Check(value)) { + PyObject *value_as_tuple = PyList_AsTuple(value); + if (value_as_tuple == NULL) { return NULL; } - argv = (Tcl_Obj **) PyMem_Malloc(((size_t)size) * sizeof(Tcl_Obj *)); - if (!argv) { - PyErr_NoMemory(); - return NULL; - } - for (i = 0; i < size; i++) - argv[i] = AsObj(PySequence_Fast_GET_ITEM(value,i)); - result = Tcl_NewListObj((int)size, argv); - PyMem_Free(argv); + result = TupleAsObj(value_as_tuple, true); + Py_DECREF(value_as_tuple); return result; } @@ -2399,7 +2431,7 @@ PythonCmdDelete(ClientData clientData) PythonCmd_ClientData *data = (PythonCmd_ClientData *)clientData; ENTER_PYTHON - Py_XDECREF(data->self); + /* data->self is borrowed. */ Py_XDECREF(data->func); PyMem_Free(data); LEAVE_PYTHON @@ -2468,7 +2500,9 @@ _tkinter_tkapp_createcommand_impl(TkappObject *self, const char *name, data = PyMem_NEW(PythonCmd_ClientData, 1); if (!data) return PyErr_NoMemory(); - Py_INCREF(self); + /* Borrow the interpreter: a strong reference would form an uncollectable + cycle (interp -> command -> data->self -> interp) and leak the command + (gh-80937). The command cannot outlive the interpreter. */ data->self = self; data->func = Py_NewRef(func); if (self->threaded && self->thread_id != Tcl_GetCurrentThread()) { @@ -2501,6 +2535,7 @@ _tkinter_tkapp_createcommand_impl(TkappObject *self, const char *name, } if (err) { PyErr_SetString(Tkinter_TclError, "can't create Tcl command"); + Py_DECREF(data->func); PyMem_Free(data); return NULL; } @@ -3064,10 +3099,24 @@ Tkapp_Dealloc(PyObject *op) { TkappObject *self = TkappObject_CAST(op); PyTypeObject *tp = Py_TYPE(self); - /*CHECK_TCL_APPARTMENT;*/ - ENTER_TCL - Tcl_DeleteInterp(Tkapp_Interp(self)); - LEAVE_TCL + if (self->threaded && self->thread_id != Tcl_GetCurrentThread()) { + /* Deleting the interpreter from another thread aborts the process + ("Tcl_AsyncDelete: async handler deleted by the wrong thread"). + Leak it instead (gh-83274). */ + if (PyErr_WarnEx(PyExc_RuntimeWarning, + "the Tcl interpreter is leaked because it was " + "deallocated in a thread other than the one it was " + "created in (see gh-83274)", 1) < 0) + { + PyErr_FormatUnraisable("Exception ignored while finalizing " + "a Tcl interpreter"); + } + } + else { + ENTER_TCL + Tcl_DeleteInterp(Tkapp_Interp(self)); + LEAVE_TCL + } Py_XDECREF(self->trace); PyObject_Free(self); Py_DECREF(tp); @@ -3207,6 +3256,20 @@ _tkinter_create_impl(PyObject *module, const char *screenName, CHECK_STRING_LENGTH(className); CHECK_STRING_LENGTH(use); +#if TCL_MAJOR_VERSION < 9 + /* className is title-cased during Tk initialization. Tcl 8.x does not + * support non-BMP characters (encoded as 4-byte UTF-8 sequences) there + * and crashes in Tcl_UtfToTitle (see gh-126219). Reject them up front. */ + for (const unsigned char *p = (const unsigned char *)className; *p; p++) { + if (*p >= 0xF0) { + PyErr_SetString(PyExc_ValueError, + "className must not contain non-BMP characters " + "with this version of Tcl/Tk"); + return NULL; + } + } +#endif + return (PyObject *) Tkapp_New(screenName, className, interactive, wantobjects, wantTk, sync, use); @@ -3220,12 +3283,13 @@ _tkinter.setbusywaitinterval Set the busy-wait interval in milliseconds between successive calls to Tcl_DoOneEvent in a threaded Python interpreter. -It should be set to a divisor of the maximum time between frames in an animation. +It should be set to a divisor of the maximum time between frames in +an animation. [clinic start generated code]*/ static PyObject * _tkinter_setbusywaitinterval_impl(PyObject *module, int new_val) -/*[clinic end generated code: output=42bf7757dc2d0ab6 input=deca1d6f9e6dae47]*/ +/*[clinic end generated code: output=42bf7757dc2d0ab6 input=984bbb86a3f916b7]*/ { if (new_val < 0) { PyErr_SetString(PyExc_ValueError, @@ -3338,14 +3402,20 @@ static PyMethodDef moduleMethods[] = }; #ifdef WAIT_FOR_STDIN +#ifndef MS_WINDOWS static int stdin_ready = 0; -#ifndef MS_WINDOWS static void MyFileProc(void *clientData, int mask) { + int tfile = (int)(Py_intptr_t)clientData; stdin_ready = 1; + /* Stop watching stdin now that input is available. Doing it here rather + than after the loop below ensures that a nested event loop (e.g. the one + started by wait_variable) does not keep waking up on the same unread + input, spinning at 100% CPU. */ + Tcl_DeleteFileHandler(tfile); } #endif @@ -3354,21 +3424,40 @@ static PyThreadState *event_tstate = NULL; static int EventHook(void) { -#ifndef MS_WINDOWS +#ifdef MS_WINDOWS + HANDLE hStdin; + DWORD type; +#else int tfile; + stdin_ready = 0; #endif PyEval_RestoreThread(event_tstate); - stdin_ready = 0; errorInCmd = 0; -#ifndef MS_WINDOWS +#ifdef MS_WINDOWS + hStdin = GetStdHandle(STD_INPUT_HANDLE); + type = GetFileType(hStdin); + while (1) { +#else tfile = fileno(stdin); - Tcl_CreateFileHandler(tfile, TCL_READABLE, MyFileProc, NULL); + Tcl_CreateFileHandler(tfile, TCL_READABLE, MyFileProc, + (void *)(Py_intptr_t)tfile); + while (!stdin_ready) { #endif - while (!errorInCmd && !stdin_ready) { int result; #ifdef MS_WINDOWS - if (_kbhit()) { - stdin_ready = 1; + if (type == FILE_TYPE_CHAR) { + if (_kbhit()) break; + } + else if (type == FILE_TYPE_PIPE) { + DWORD available; + if (PeekNamedPipe(hStdin, NULL, 0, NULL, &available, NULL)) { + if (available > 0) break; + } + else { + if (GetLastError() == ERROR_BROKEN_PIPE) break; + } + } + else if (type == FILE_TYPE_DISK) { break; } #endif @@ -3384,18 +3473,23 @@ EventHook(void) Sleep(Tkinter_busywaitinterval); Py_END_ALLOW_THREADS + /* Report an exception raised in a callback, but keep pumping events + instead of returning to the prompt: without readline there is no + input waiting on stdin yet, so returning here would block in fgets + until the user hits enter, freezing later callbacks. */ + if (errorInCmd) { + errorInCmd = 0; + PyErr_SetRaisedException(excInCmd); + excInCmd = NULL; + PyErr_Print(); + } if (result < 0) break; } #ifndef MS_WINDOWS - Tcl_DeleteFileHandler(tfile); + if (!stdin_ready) + Tcl_DeleteFileHandler(tfile); #endif - if (errorInCmd) { - errorInCmd = 0; - PyErr_SetRaisedException(excInCmd); - excInCmd = NULL; - PyErr_Print(); - } PyEval_SaveThread(); return 0; } @@ -3466,7 +3560,7 @@ PyInit__tkinter(void) tcl_lock = PyThread_allocate_lock(); if (tcl_lock == NULL) - return NULL; + return PyErr_NoMemory(); m = PyModule_Create(&_tkintermodule); if (m == NULL) diff --git a/Modules/_uuidmodule.c b/Modules/_uuidmodule.c index c5e78b1510b5e3a..c31a7e8fea56084 100644 --- a/Modules/_uuidmodule.c +++ b/Modules/_uuidmodule.c @@ -78,23 +78,47 @@ py_UuidCreate(PyObject *Py_UNUSED(context), return NULL; } +static int +py_windows_has_stable_node(void) +{ + UUID uuid; + RPC_STATUS res; + Py_BEGIN_ALLOW_THREADS + res = UuidCreateSequential(&uuid); + Py_END_ALLOW_THREADS + return res == RPC_S_OK; +} #endif /* MS_WINDOWS */ static int -uuid_exec(PyObject *module) { +uuid_exec(PyObject *module) +{ +#define ADD_INT(NAME, VALUE) \ + do { \ + if (PyModule_AddIntConstant(module, (NAME), (VALUE)) < 0) { \ + return -1; \ + } \ + } while (0) + assert(sizeof(uuid_t) == 16); #if defined(MS_WINDOWS) - int has_uuid_generate_time_safe = 0; + ADD_INT("has_uuid_generate_time_safe", 0); #elif defined(HAVE_UUID_GENERATE_TIME_SAFE) - int has_uuid_generate_time_safe = 1; + ADD_INT("has_uuid_generate_time_safe", 1); #else - int has_uuid_generate_time_safe = 0; + ADD_INT("has_uuid_generate_time_safe", 0); #endif - if (PyModule_AddIntConstant(module, "has_uuid_generate_time_safe", - has_uuid_generate_time_safe) < 0) { - return -1; - } + +#if defined(MS_WINDOWS) + ADD_INT("has_stable_extractable_node", py_windows_has_stable_node()); +#elif defined(HAVE_UUID_GENERATE_TIME_SAFE_STABLE_MAC) + ADD_INT("has_stable_extractable_node", 1); +#else + ADD_INT("has_stable_extractable_node", 0); +#endif + +#undef ADD_INT return 0; } diff --git a/Modules/_winapi.c b/Modules/_winapi.c index 02817e09b936f37..767a5b35f45664f 100644 --- a/Modules/_winapi.c +++ b/Modules/_winapi.c @@ -1187,8 +1187,10 @@ gethandlelist(PyObject *mapping, const char *name, Py_ssize_t *size) } ret = PyMem_Malloc(*size); - if (ret == NULL) + if (ret == NULL) { + PyErr_NoMemory(); goto cleanup; + } for (i = 0; i < PySequence_Fast_GET_SIZE(value_fast); i++) { ret[i] = PYNUM_TO_HANDLE(PySequence_Fast_GET_ITEM(value_fast, i)); @@ -1271,6 +1273,7 @@ getattributelist(PyObject *obj, const char *name, AttributeList *attribute_list) attribute_list->attribute_list = PyMem_Malloc(attribute_list_size); if (attribute_list->attribute_list == NULL) { ret = -1; + PyErr_NoMemory(); goto cleanup; } @@ -1668,6 +1671,9 @@ _winapi_GetShortPathName_impl(PyObject *module, LPCWSTR path) } PyMem_Free((void *)buffer); } + else { + PyErr_NoMemory(); + } } else { PyErr_SetFromWindowsErr(0); } @@ -2371,6 +2377,7 @@ _winapi_BatchedWaitForMultipleObjects_impl(PyObject *module, while (i < nhandles) { BatchedWaitData *data = (BatchedWaitData*)PyMem_Malloc(sizeof(BatchedWaitData)); if (!data) { + PyErr_NoMemory(); goto error; } thread_data[thread_count++] = data; @@ -2953,6 +2960,21 @@ _winapi_CopyFile2_impl(PyObject *module, LPCWSTR existing_file_name, } +/*[clinic input] +_winapi.GetTickCount64 + +Number of milliseconds that have elapsed since the system was started. +[clinic start generated code]*/ + +static PyObject * +_winapi_GetTickCount64_impl(PyObject *module) +/*[clinic end generated code: output=cb33c0568f0b3ed1 input=77ed6539ac7d6590]*/ +{ + ULONGLONG ticks = GetTickCount64(); + return PyLong_FromUnsignedLongLong(ticks); +} + + static PyMethodDef winapi_functions[] = { _WINAPI_CLOSEHANDLE_METHODDEF _WINAPI_CONNECTNAMEDPIPE_METHODDEF @@ -2999,6 +3021,7 @@ static PyMethodDef winapi_functions[] = { _WINAPI__MIMETYPES_READ_WINDOWS_REGISTRY_METHODDEF _WINAPI_NEEDCURRENTDIRECTORYFOREXEPATH_METHODDEF _WINAPI_COPYFILE2_METHODDEF + _WINAPI_GETTICKCOUNT64_METHODDEF {NULL, NULL} }; diff --git a/Modules/_zoneinfo.c b/Modules/_zoneinfo.c index abd53436b21b299..f3fb5127773fa2f 100644 --- a/Modules/_zoneinfo.c +++ b/Modules/_zoneinfo.c @@ -7,6 +7,7 @@ #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_typeobject.h" // _PyType_GetModuleState() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "datetime.h" // PyDateTime_TZInfo @@ -271,6 +272,7 @@ zoneinfo_new_instance(zoneinfo_state *state, PyTypeObject *type, PyObject *key) goto cleanup; error: + assert(PyErr_Occurred()); Py_CLEAR(self); cleanup: if (file_obj != NULL) { @@ -291,16 +293,11 @@ static PyObject * get_weak_cache(zoneinfo_state *state, PyTypeObject *type) { if (type == state->ZoneInfoType) { + Py_INCREF(state->ZONEINFO_WEAK_CACHE); return state->ZONEINFO_WEAK_CACHE; } else { - PyObject *cache = - PyObject_GetAttrString((PyObject *)type, "_weak_cache"); - // We are assuming that the type lives at least as long as the function - // that calls get_weak_cache, and that it holds a reference to the - // cache, so we'll return a "borrowed reference". - Py_XDECREF(cache); - return cache; + return PyObject_GetAttrString((PyObject *)type, "_weak_cache"); } } @@ -325,8 +322,12 @@ zoneinfo_ZoneInfo_impl(PyTypeObject *type, PyObject *key) } PyObject *weak_cache = get_weak_cache(state, type); + if (weak_cache == NULL) { + return NULL; + } instance = PyObject_CallMethod(weak_cache, "get", "O", key, Py_None); if (instance == NULL) { + Py_DECREF(weak_cache); return NULL; } @@ -334,19 +335,31 @@ zoneinfo_ZoneInfo_impl(PyTypeObject *type, PyObject *key) Py_DECREF(instance); PyObject *tmp = zoneinfo_new_instance(state, type, key); if (tmp == NULL) { + Py_DECREF(weak_cache); return NULL; } + ((PyZoneInfo_ZoneInfo *)tmp)->source = SOURCE_CACHE; instance = PyObject_CallMethod(weak_cache, "setdefault", "OO", key, tmp); Py_DECREF(tmp); if (instance == NULL) { + Py_DECREF(weak_cache); return NULL; } - ((PyZoneInfo_ZoneInfo *)instance)->source = SOURCE_CACHE; + } + + if (!PyObject_TypeCheck(instance, type)) { + PyErr_Format(PyExc_RuntimeError, + "Unexpected instance of %T in %s weak cache for key %R", + instance, _PyType_Name(type), key); + Py_DECREF(instance); + Py_DECREF(weak_cache); + return NULL; } update_strong_cache(state, type, key, instance); + Py_DECREF(weak_cache); return instance; } @@ -375,9 +388,7 @@ zoneinfo_dealloc(PyObject *obj_self) PyTypeObject *tp = Py_TYPE(self); PyObject_GC_UnTrack(self); - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs(obj_self); - } + FT_CLEAR_WEAKREFS(obj_self, self->weakreflist); if (self->trans_list_utc != NULL) { PyMem_Free(self->trans_list_utc); @@ -448,6 +459,7 @@ zoneinfo_ZoneInfo_from_file_impl(PyTypeObject *type, PyTypeObject *cls, return (PyObject *)self; error: + assert(PyErr_Occurred()); Py_XDECREF(file_repr); Py_XDECREF(self); return NULL; @@ -498,6 +510,9 @@ zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyTypeObject *cls, { zoneinfo_state *state = zoneinfo_get_state_by_cls(cls); PyObject *weak_cache = get_weak_cache(state, type); + if (weak_cache == NULL) { + return NULL; + } if (only_keys == NULL || only_keys == Py_None) { PyObject *rv = PyObject_CallMethod(weak_cache, "clear", NULL); @@ -511,12 +526,14 @@ zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyTypeObject *cls, PyObject *item = NULL; PyObject *pop = PyUnicode_FromString("pop"); if (pop == NULL) { + Py_DECREF(weak_cache); return NULL; } PyObject *iter = PyObject_GetIter(only_keys); if (iter == NULL) { Py_DECREF(pop); + Py_DECREF(weak_cache); return NULL; } @@ -541,6 +558,7 @@ zoneinfo_ZoneInfo_clear_cache_impl(PyTypeObject *type, PyTypeObject *cls, Py_DECREF(pop); } + Py_DECREF(weak_cache); if (PyErr_Occurred()) { return NULL; } @@ -970,7 +988,7 @@ load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj) } if (!PyTuple_CheckExact(data_tuple)) { - PyErr_Format(PyExc_TypeError, "Invalid data result type: %r", + PyErr_Format(PyExc_TypeError, "Invalid data result type: %R", data_tuple); goto error; } @@ -1024,10 +1042,12 @@ load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj) self->trans_list_utc = PyMem_Malloc(self->num_transitions * sizeof(int64_t)); if (self->trans_list_utc == NULL) { + PyErr_NoMemory(); goto error; } trans_idx = PyMem_Malloc(self->num_transitions * sizeof(Py_ssize_t)); if (trans_idx == NULL) { + PyErr_NoMemory(); goto error; } @@ -1052,7 +1072,7 @@ load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj) } trans_idx[i] = (size_t)cur_trans_idx; - if (trans_idx[i] > self->num_ttinfos) { + if (trans_idx[i] >= self->num_ttinfos) { PyErr_Format( PyExc_ValueError, "Invalid transition index found while reading TZif: %zd", @@ -1067,6 +1087,7 @@ load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj) isdst = PyMem_Malloc(self->num_ttinfos * sizeof(unsigned char)); if (utcoff == NULL || isdst == NULL) { + PyErr_NoMemory(); goto error; } for (size_t i = 0; i < self->num_ttinfos; ++i) { @@ -1096,6 +1117,7 @@ load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj) dstoff = PyMem_Calloc(self->num_ttinfos, sizeof(long)); if (dstoff == NULL) { + PyErr_NoMemory(); goto error; } @@ -1112,6 +1134,7 @@ load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj) // Build _ttinfo objects from utcoff, dstoff and abbr self->_ttinfos = PyMem_Malloc(self->num_ttinfos * sizeof(_ttinfo)); if (self->_ttinfos == NULL) { + PyErr_NoMemory(); goto error; } for (size_t i = 0; i < self->num_ttinfos; ++i) { @@ -1132,6 +1155,7 @@ load_data(zoneinfo_state *state, PyZoneInfo_ZoneInfo *self, PyObject *file_obj) self->trans_ttinfos = PyMem_Calloc(self->num_transitions, sizeof(_ttinfo *)); if (self->trans_ttinfos == NULL) { + PyErr_NoMemory(); goto error; } for (size_t i = 0; i < self->num_transitions; ++i) { @@ -1650,9 +1674,11 @@ parse_tz_str(zoneinfo_state *state, PyObject *tz_str_obj, _tzrule *out) p++; if (parse_transition_rule(&p, transitions[i])) { - PyErr_Format(PyExc_ValueError, - "Malformed transition rule in TZ string: %R", - tz_str_obj); + if (!PyErr_ExceptionMatches(PyExc_MemoryError)) { + PyErr_Format(PyExc_ValueError, + "Malformed transition rule in TZ string: %R", + tz_str_obj); + } goto error; } } @@ -1733,6 +1759,9 @@ parse_abbr(const char **p, PyObject **abbr) ptr++; } str_end = ptr; + if (str_end == str_start) { + return -1; + } ptr++; } else { @@ -1852,6 +1881,7 @@ parse_transition_rule(const char **p, TransitionRuleType **out) CalendarRule *rv = PyMem_Calloc(1, sizeof(CalendarRule)); if (rv == NULL) { + PyErr_NoMemory(); return -1; } @@ -1883,6 +1913,7 @@ parse_transition_rule(const char **p, TransitionRuleType **out) DayRule *rv = PyMem_Calloc(1, sizeof(DayRule)); if (rv == NULL) { + PyErr_NoMemory(); return -1; } @@ -2050,7 +2081,7 @@ utcoff_to_dstoff(size_t *trans_idx, long *utcoffs, long *dstoffs, dstoff = utcoff - utcoffs[comp_idx]; } - if (!dstoff && idx < (num_ttinfos - 1)) { + if (!dstoff && idx < (num_ttinfos - 1) && i + 1 < num_transitions) { comp_idx = trans_idx[i + 1]; // If the following transition is also DST and we couldn't find @@ -2116,6 +2147,7 @@ ts_to_local(size_t *trans_idx, int64_t *trans_utc, long *utcoff, for (size_t i = 0; i < 2; ++i) { trans_local[i] = PyMem_Malloc(num_transitions * sizeof(int64_t)); if (trans_local[i] == NULL) { + PyErr_NoMemory(); return -1; } diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index 4d046859a1540ef..8af6156a0da5756 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -1,14 +1,16 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorithm & the zstd library. */ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif +#include "Python.h" + #include "_zstdmodule.h" +#include <zstd.h> // ZSTD_*() +#include <zdict.h> // ZDICT_*() + /*[clinic input] module _zstd @@ -17,52 +19,91 @@ module _zstd #include "clinic/_zstdmodule.c.h" +ZstdDict * +_Py_parse_zstd_dict(const _zstd_state *state, PyObject *dict, int *ptype) +{ + if (state == NULL) { + return NULL; + } + + /* Check ZstdDict */ + if (PyObject_TypeCheck(dict, state->ZstdDict_type)) { + return (ZstdDict*)dict; + } + + /* Check (ZstdDict, type) */ + if (PyTuple_CheckExact(dict) && PyTuple_GET_SIZE(dict) == 2 + && PyObject_TypeCheck(PyTuple_GET_ITEM(dict, 0), state->ZstdDict_type) + && PyLong_Check(PyTuple_GET_ITEM(dict, 1))) + { + int type = PyLong_AsInt(PyTuple_GET_ITEM(dict, 1)); + if (type == -1 && PyErr_Occurred()) { + return NULL; + } + if (type == DICT_TYPE_DIGESTED + || type == DICT_TYPE_UNDIGESTED + || type == DICT_TYPE_PREFIX) + { + *ptype = type; + return (ZstdDict*)PyTuple_GET_ITEM(dict, 0); + } + } + + /* Wrong type */ + PyErr_SetString(PyExc_TypeError, + "zstd_dict argument should be a ZstdDict object."); + return NULL; +} + /* Format error message and set ZstdError. */ void -set_zstd_error(const _zstd_state* const state, - error_type type, size_t zstd_ret) +set_zstd_error(const _zstd_state *state, error_type type, size_t zstd_ret) { - char *msg; + const char *msg; assert(ZSTD_isError(zstd_ret)); - switch (type) - { - case ERR_DECOMPRESS: - msg = "Unable to decompress zstd data: %s"; - break; - case ERR_COMPRESS: - msg = "Unable to compress zstd data: %s"; - break; - case ERR_SET_PLEDGED_INPUT_SIZE: - msg = "Unable to set pledged uncompressed content size: %s"; - break; - - case ERR_LOAD_D_DICT: - msg = "Unable to load zstd dictionary or prefix for decompression: %s"; - break; - case ERR_LOAD_C_DICT: - msg = "Unable to load zstd dictionary or prefix for compression: %s"; - break; - - case ERR_GET_C_BOUNDS: - msg = "Unable to get zstd compression parameter bounds: %s"; - break; - case ERR_GET_D_BOUNDS: - msg = "Unable to get zstd decompression parameter bounds: %s"; - break; - case ERR_SET_C_LEVEL: - msg = "Unable to set zstd compression level: %s"; - break; - - case ERR_TRAIN_DICT: - msg = "Unable to train zstd dictionary: %s"; - break; - case ERR_FINALIZE_DICT: - msg = "Unable to finalize zstd dictionary: %s"; - break; - - default: - Py_UNREACHABLE(); + if (state == NULL) { + return; + } + switch (type) { + case ERR_DECOMPRESS: + msg = "Unable to decompress Zstandard data: %s"; + break; + case ERR_COMPRESS: + msg = "Unable to compress Zstandard data: %s"; + break; + case ERR_SET_PLEDGED_INPUT_SIZE: + msg = "Unable to set pledged uncompressed content size: %s"; + break; + + case ERR_LOAD_D_DICT: + msg = "Unable to load Zstandard dictionary or prefix for " + "decompression: %s"; + break; + case ERR_LOAD_C_DICT: + msg = "Unable to load Zstandard dictionary or prefix for " + "compression: %s"; + break; + + case ERR_GET_C_BOUNDS: + msg = "Unable to get zstd compression parameter bounds: %s"; + break; + case ERR_GET_D_BOUNDS: + msg = "Unable to get zstd decompression parameter bounds: %s"; + break; + case ERR_SET_C_LEVEL: + msg = "Unable to set zstd compression level: %s"; + break; + + case ERR_TRAIN_DICT: + msg = "Unable to train the Zstandard dictionary: %s"; + break; + case ERR_FINALIZE_DICT: + msg = "Unable to finalize the Zstandard dictionary: %s"; + break; + + default: + Py_UNREACHABLE(); } PyErr_Format(state->ZstdError, msg, ZSTD_getErrorName(zstd_ret)); } @@ -72,8 +113,7 @@ typedef struct { char parameter_name[32]; } ParameterInfo; -static const ParameterInfo cp_list[] = -{ +static const ParameterInfo cp_list[] = { {ZSTD_c_compressionLevel, "compression_level"}, {ZSTD_c_windowLog, "window_log"}, {ZSTD_c_hashLog, "hash_log"}, @@ -98,22 +138,18 @@ static const ParameterInfo cp_list[] = {ZSTD_c_overlapLog, "overlap_log"} }; -static const ParameterInfo dp_list[] = -{ +static const ParameterInfo dp_list[] = { {ZSTD_d_windowLogMax, "window_log_max"} }; void -set_parameter_error(const _zstd_state* const state, int is_compress, - int key_v, int value_v) +set_parameter_error(int is_compress, int key_v, int value_v) { ParameterInfo const *list; int list_size; - char const *name; char *type; ZSTD_bounds bounds; - int i; - char pos_msg[128]; + char pos_msg[64]; if (is_compress) { list = cp_list; @@ -127,8 +163,8 @@ set_parameter_error(const _zstd_state* const state, int is_compress, } /* Find parameter's name */ - name = NULL; - for (i = 0; i < list_size; i++) { + char const *name = NULL; + for (int i = 0; i < list_size; i++) { if (key_v == (list+i)->parameter) { name = (list+i)->parameter_name; break; @@ -150,20 +186,16 @@ set_parameter_error(const _zstd_state* const state, int is_compress, bounds = ZSTD_dParam_getBounds(key_v); } if (ZSTD_isError(bounds.error)) { - PyErr_Format(state->ZstdError, - "Zstd %s parameter \"%s\" is invalid. (zstd v%s)", - type, name, ZSTD_versionString()); + PyErr_Format(PyExc_ValueError, "invalid %s parameter '%s'", + type, name); return; } /* Error message */ - PyErr_Format(state->ZstdError, - "Error when setting zstd %s parameter \"%s\", it " - "should %d <= value <= %d, provided value is %d. " - "(zstd v%s, %d-bit build)", - type, name, - bounds.lowerBound, bounds.upperBound, value_v, - ZSTD_versionString(), 8*(int)sizeof(Py_ssize_t)); + PyErr_Format(PyExc_ValueError, + "%s parameter '%s' received an illegal value %d; " + "the valid range is [%d, %d]", + type, name, value_v, bounds.lowerBound, bounds.upperBound); } static inline _zstd_state* @@ -174,9 +206,57 @@ get_zstd_state(PyObject *module) return (_zstd_state *)state; } +static Py_ssize_t +calculate_samples_stats(PyBytesObject *samples_bytes, PyObject *samples_sizes, + size_t **chunk_sizes) +{ + Py_ssize_t chunks_number; + Py_ssize_t sizes_sum; + Py_ssize_t i; + + chunks_number = PyTuple_GET_SIZE(samples_sizes); + if ((size_t) chunks_number > UINT32_MAX) { + PyErr_Format(PyExc_ValueError, + "The number of samples should be <= %u.", UINT32_MAX); + return -1; + } + + /* Prepare chunk_sizes */ + *chunk_sizes = PyMem_New(size_t, chunks_number); + if (*chunk_sizes == NULL) { + PyErr_NoMemory(); + return -1; + } + + sizes_sum = PyBytes_GET_SIZE(samples_bytes); + for (i = 0; i < chunks_number; i++) { + size_t size = PyLong_AsSize_t(PyTuple_GET_ITEM(samples_sizes, i)); + (*chunk_sizes)[i] = size; + if (size == (size_t)-1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + goto sum_error; + } + return -1; + } + if ((size_t)sizes_sum < size) { + goto sum_error; + } + sizes_sum -= size; + } + + if (sizes_sum != 0) { +sum_error: + PyErr_SetString(PyExc_ValueError, + "The samples size tuple doesn't match the " + "concatenation's size."); + return -1; + } + return chunks_number; +} + /*[clinic input] -_zstd._train_dict +_zstd.train_dict samples_bytes: PyBytesObject Concatenation of samples. @@ -186,59 +266,30 @@ _zstd._train_dict The size of the dictionary. / -Internal function, train a zstd dictionary on sample data. +Train a Zstandard dictionary on sample data. [clinic start generated code]*/ static PyObject * -_zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, - PyObject *samples_sizes, Py_ssize_t dict_size) -/*[clinic end generated code: output=b5b4f36347c0addd input=2dce5b57d63923e2]*/ +_zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, + PyObject *samples_sizes, Py_ssize_t dict_size) +/*[clinic end generated code: output=8e87fe43935e8f77 input=d20dedb21c72cb62]*/ { - // TODO(emmatyping): The preamble and suffix to this function and _finalize_dict - // are pretty similar. We should see if we can refactor them to share that code. - Py_ssize_t chunks_number; - size_t *chunk_sizes = NULL; PyObject *dst_dict_bytes = NULL; + size_t *chunk_sizes = NULL; + Py_ssize_t chunks_number; size_t zstd_ret; - Py_ssize_t sizes_sum; - Py_ssize_t i; /* Check arguments */ if (dict_size <= 0) { - PyErr_SetString(PyExc_ValueError, "dict_size argument should be positive number."); - return NULL; - } - - chunks_number = Py_SIZE(samples_sizes); - if ((size_t) chunks_number > UINT32_MAX) { - PyErr_Format(PyExc_ValueError, - "The number of samples should be <= %u.", UINT32_MAX); + PyErr_SetString(PyExc_ValueError, + "dict_size argument should be positive number."); return NULL; } - /* Prepare chunk_sizes */ - chunk_sizes = PyMem_New(size_t, chunks_number); - if (chunk_sizes == NULL) { - PyErr_NoMemory(); - goto error; - } - - sizes_sum = 0; - for (i = 0; i < chunks_number; i++) { - PyObject *size = PyTuple_GetItem(samples_sizes, i); - chunk_sizes[i] = PyLong_AsSize_t(size); - if (chunk_sizes[i] == (size_t)-1 && PyErr_Occurred()) { - PyErr_Format(PyExc_ValueError, - "Items in samples_sizes should be an int " - "object, with a value between 0 and %u.", SIZE_MAX); - goto error; - } - sizes_sum += chunk_sizes[i]; - } - - if (sizes_sum != Py_SIZE(samples_bytes)) { - PyErr_SetString(PyExc_ValueError, - "The samples size tuple doesn't match the concatenation's size."); + /* Check that the samples are valid and get their sizes */ + chunks_number = calculate_samples_stats(samples_bytes, samples_sizes, + &chunk_sizes); + if (chunks_number < 0) { goto error; } @@ -250,16 +301,16 @@ _zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, /* Train the dictionary */ char *dst_dict_buffer = PyBytes_AS_STRING(dst_dict_bytes); - char *samples_buffer = PyBytes_AS_STRING(samples_bytes); + const char *samples_buffer = PyBytes_AS_STRING(samples_bytes); Py_BEGIN_ALLOW_THREADS zstd_ret = ZDICT_trainFromBuffer(dst_dict_buffer, dict_size, samples_buffer, chunk_sizes, (uint32_t)chunks_number); Py_END_ALLOW_THREADS - /* Check zstd dict error */ + /* Check Zstandard dict error */ if (ZDICT_isError(zstd_ret)) { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); set_zstd_error(mod_state, ERR_TRAIN_DICT, zstd_ret); goto error; } @@ -280,7 +331,7 @@ _zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, } /*[clinic input] -_zstd._finalize_dict +_zstd.finalize_dict custom_dict_bytes: PyBytesObject Custom dictionary content. @@ -291,63 +342,36 @@ _zstd._finalize_dict dict_size: Py_ssize_t The size of the dictionary. compression_level: int - Optimize for a specific zstd compression level, 0 means default. + Optimize for a specific Zstandard compression level, 0 means default. / -Internal function, finalize a zstd dictionary. +Finalize a Zstandard dictionary. [clinic start generated code]*/ static PyObject * -_zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, - PyBytesObject *samples_bytes, - PyObject *samples_sizes, Py_ssize_t dict_size, - int compression_level) -/*[clinic end generated code: output=5dc5b520fddba37f input=8afd42a249078460]*/ +_zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, + PyBytesObject *samples_bytes, + PyObject *samples_sizes, Py_ssize_t dict_size, + int compression_level) +/*[clinic end generated code: output=f91821ba5ae85bda input=3c7e2480aa08fb56]*/ { Py_ssize_t chunks_number; size_t *chunk_sizes = NULL; PyObject *dst_dict_bytes = NULL; size_t zstd_ret; ZDICT_params_t params; - Py_ssize_t sizes_sum; - Py_ssize_t i; /* Check arguments */ if (dict_size <= 0) { - PyErr_SetString(PyExc_ValueError, "dict_size argument should be positive number."); - return NULL; - } - - chunks_number = Py_SIZE(samples_sizes); - if ((size_t) chunks_number > UINT32_MAX) { - PyErr_Format(PyExc_ValueError, - "The number of samples should be <= %u.", UINT32_MAX); + PyErr_SetString(PyExc_ValueError, + "dict_size argument should be positive number."); return NULL; } - /* Prepare chunk_sizes */ - chunk_sizes = PyMem_New(size_t, chunks_number); - if (chunk_sizes == NULL) { - PyErr_NoMemory(); - goto error; - } - - sizes_sum = 0; - for (i = 0; i < chunks_number; i++) { - PyObject *size = PyTuple_GetItem(samples_sizes, i); - chunk_sizes[i] = PyLong_AsSize_t(size); - if (chunk_sizes[i] == (size_t)-1 && PyErr_Occurred()) { - PyErr_Format(PyExc_ValueError, - "Items in samples_sizes should be an int " - "object, with a value between 0 and %u.", SIZE_MAX); - goto error; - } - sizes_sum += chunk_sizes[i]; - } - - if (sizes_sum != Py_SIZE(samples_bytes)) { - PyErr_SetString(PyExc_ValueError, - "The samples size tuple doesn't match the concatenation's size."); + /* Check that the samples are valid and get their sizes */ + chunks_number = calculate_samples_stats(samples_bytes, samples_sizes, + &chunk_sizes); + if (chunks_number < 0) { goto error; } @@ -359,7 +383,7 @@ _zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, /* Parameters */ - /* Optimize for a specific zstd compression level, 0 means default. */ + /* Optimize for a specific Zstandard compression level, 0 means default. */ params.compressionLevel = compression_level; /* Write log to stderr, 0 = none. */ params.notificationLevel = 0; @@ -370,14 +394,15 @@ _zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, Py_BEGIN_ALLOW_THREADS zstd_ret = ZDICT_finalizeDictionary( PyBytes_AS_STRING(dst_dict_bytes), dict_size, - PyBytes_AS_STRING(custom_dict_bytes), Py_SIZE(custom_dict_bytes), + PyBytes_AS_STRING(custom_dict_bytes), + Py_SIZE(custom_dict_bytes), PyBytes_AS_STRING(samples_bytes), chunk_sizes, (uint32_t)chunks_number, params); Py_END_ALLOW_THREADS - /* Check zstd dict error */ + /* Check Zstandard dict error */ if (ZDICT_isError(zstd_ret)) { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); set_zstd_error(mod_state, ERR_FINALIZE_DICT, zstd_ret); goto error; } @@ -399,26 +424,25 @@ _zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, /*[clinic input] -_zstd._get_param_bounds +_zstd.get_param_bounds parameter: int The parameter to get bounds. is_compress: bool True for CompressionParameter, False for DecompressionParameter. -Internal function, get CompressionParameter/DecompressionParameter bounds. +Get CompressionParameter/DecompressionParameter bounds. [clinic start generated code]*/ static PyObject * -_zstd__get_param_bounds_impl(PyObject *module, int parameter, - int is_compress) -/*[clinic end generated code: output=9892cd822f937e79 input=884cd1a01125267d]*/ +_zstd_get_param_bounds_impl(PyObject *module, int parameter, int is_compress) +/*[clinic end generated code: output=4acf5a876f0620ca input=45742ef0a3531b65]*/ { ZSTD_bounds bound; if (is_compress) { bound = ZSTD_cParam_getBounds(parameter); if (ZSTD_isError(bound.error)) { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); set_zstd_error(mod_state, ERR_GET_C_BOUNDS, bound.error); return NULL; } @@ -426,7 +450,7 @@ _zstd__get_param_bounds_impl(PyObject *module, int parameter, else { bound = ZSTD_dParam_getBounds(parameter); if (ZSTD_isError(bound.error)) { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); set_zstd_error(mod_state, ERR_GET_D_BOUNDS, bound.error); return NULL; } @@ -442,24 +466,23 @@ _zstd.get_frame_size A bytes-like object, it should start from the beginning of a frame, and contains at least one complete frame. -Get the size of a zstd frame, including frame header and 4-byte checksum if it has one. - -It will iterate all blocks' headers within a frame, to accumulate the frame size. +Get the size of a Zstandard frame, including the header and optional checksum. [clinic start generated code]*/ static PyObject * _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer) -/*[clinic end generated code: output=a7384c2f8780f442 input=7d3ad24311893bf3]*/ +/*[clinic end generated code: output=a7384c2f8780f442 input=3b9f73f8c8129d38]*/ { size_t frame_size; - frame_size = ZSTD_findFrameCompressedSize(frame_buffer->buf, frame_buffer->len); + frame_size = ZSTD_findFrameCompressedSize(frame_buffer->buf, + frame_buffer->len); if (ZSTD_isError(frame_size)) { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); PyErr_Format(mod_state->ZstdError, - "Error when finding the compressed size of a zstd frame. " - "Make sure the frame_buffer argument starts from the " - "beginning of a frame, and its length not less than this " + "Error when finding the compressed size of a Zstandard frame. " + "Ensure the frame_buffer argument starts from the " + "beginning of a frame, and its length is not less than this " "complete frame. Zstd error message: %s.", ZSTD_getErrorName(frame_size)); return NULL; @@ -469,17 +492,17 @@ _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer) } /*[clinic input] -_zstd._get_frame_info +_zstd.get_frame_info frame_buffer: Py_buffer - A bytes-like object, containing the header of a zstd frame. + A bytes-like object, containing the header of a Zstandard frame. -Internal function, get zstd frame infomation from a frame header. +Get Zstandard frame infomation from a frame header. [clinic start generated code]*/ static PyObject * -_zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer) -/*[clinic end generated code: output=5462855464ecdf81 input=67f1f8e4b7b89c4d]*/ +_zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer) +/*[clinic end generated code: output=56e033cf48001929 input=94b240583ae22ca5]*/ { uint64_t decompressed_size; uint32_t dict_id; @@ -491,12 +514,12 @@ _zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer) /* #define ZSTD_CONTENTSIZE_UNKNOWN (0ULL - 1) #define ZSTD_CONTENTSIZE_ERROR (0ULL - 2) */ if (decompressed_size == ZSTD_CONTENTSIZE_ERROR) { - _zstd_state* const mod_state = get_zstd_state(module); + _zstd_state* mod_state = get_zstd_state(module); PyErr_SetString(mod_state->ZstdError, "Error when getting information from the header of " - "a zstd frame. Make sure the frame_buffer argument " + "a Zstandard frame. Ensure the frame_buffer argument " "starts from the beginning of a frame, and its length " - "not less than the frame header (6~18 bytes)."); + "is not less than the frame header (6~18 bytes)."); return NULL; } @@ -511,325 +534,169 @@ _zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer) } /*[clinic input] -_zstd._set_parameter_types +_zstd.set_parameter_types c_parameter_type: object(subclass_of='&PyType_Type') CompressionParameter IntEnum type object d_parameter_type: object(subclass_of='&PyType_Type') DecompressionParameter IntEnum type object -Internal function, set CompressionParameter/DecompressionParameter types for validity check. +Set CompressionParameter and DecompressionParameter types for validity check. [clinic start generated code]*/ static PyObject * -_zstd__set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type, - PyObject *d_parameter_type) -/*[clinic end generated code: output=a13d4890ccbd2873 input=4535545d903853d3]*/ +_zstd_set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type, + PyObject *d_parameter_type) +/*[clinic end generated code: output=f3313b1294f19502 input=75d7a953580fae5f]*/ { - _zstd_state* const mod_state = get_zstd_state(module); - - if (!PyType_Check(c_parameter_type) || !PyType_Check(d_parameter_type)) { - PyErr_SetString(PyExc_ValueError, - "The two arguments should be CompressionParameter and " - "DecompressionParameter types."); - return NULL; - } + _zstd_state* mod_state = get_zstd_state(module); - Py_XDECREF(mod_state->CParameter_type); Py_INCREF(c_parameter_type); - mod_state->CParameter_type = (PyTypeObject*)c_parameter_type; - - Py_XDECREF(mod_state->DParameter_type); + Py_XSETREF(mod_state->CParameter_type, (PyTypeObject*)c_parameter_type); Py_INCREF(d_parameter_type); - mod_state->DParameter_type = (PyTypeObject*)d_parameter_type; + Py_XSETREF(mod_state->DParameter_type, (PyTypeObject*)d_parameter_type); Py_RETURN_NONE; } static PyMethodDef _zstd_methods[] = { - _ZSTD__TRAIN_DICT_METHODDEF - _ZSTD__FINALIZE_DICT_METHODDEF - _ZSTD__GET_PARAM_BOUNDS_METHODDEF + _ZSTD_TRAIN_DICT_METHODDEF + _ZSTD_FINALIZE_DICT_METHODDEF + _ZSTD_GET_PARAM_BOUNDS_METHODDEF _ZSTD_GET_FRAME_SIZE_METHODDEF - _ZSTD__GET_FRAME_INFO_METHODDEF - _ZSTD__SET_PARAMETER_TYPES_METHODDEF - - {0} + _ZSTD_GET_FRAME_INFO_METHODDEF + _ZSTD_SET_PARAMETER_TYPES_METHODDEF + {NULL, NULL} }; - -#define ADD_INT_PREFIX_MACRO(module, macro) \ - do { \ - if (PyModule_AddIntConstant(module, "_" #macro, macro) < 0) { \ - return -1; \ - } \ - } while(0) - static int -add_parameters(PyObject *module) -{ - /* If add new parameters, please also add to cp_list/dp_list above. */ - - /* Compression parameters */ - ADD_INT_PREFIX_MACRO(module, ZSTD_c_compressionLevel); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_windowLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_hashLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_chainLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_searchLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_minMatch); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_targetLength); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_strategy); - - ADD_INT_PREFIX_MACRO(module, ZSTD_c_enableLongDistanceMatching); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmHashLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmMinMatch); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmBucketSizeLog); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_ldmHashRateLog); - - ADD_INT_PREFIX_MACRO(module, ZSTD_c_contentSizeFlag); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_checksumFlag); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_dictIDFlag); - - ADD_INT_PREFIX_MACRO(module, ZSTD_c_nbWorkers); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_jobSize); - ADD_INT_PREFIX_MACRO(module, ZSTD_c_overlapLog); - - /* Decompression parameters */ - ADD_INT_PREFIX_MACRO(module, ZSTD_d_windowLogMax); - - /* ZSTD_strategy enum */ - ADD_INT_PREFIX_MACRO(module, ZSTD_fast); - ADD_INT_PREFIX_MACRO(module, ZSTD_dfast); - ADD_INT_PREFIX_MACRO(module, ZSTD_greedy); - ADD_INT_PREFIX_MACRO(module, ZSTD_lazy); - ADD_INT_PREFIX_MACRO(module, ZSTD_lazy2); - ADD_INT_PREFIX_MACRO(module, ZSTD_btlazy2); - ADD_INT_PREFIX_MACRO(module, ZSTD_btopt); - ADD_INT_PREFIX_MACRO(module, ZSTD_btultra); - ADD_INT_PREFIX_MACRO(module, ZSTD_btultra2); - - return 0; -} - -static inline PyObject * -get_zstd_version_info(void) +_zstd_exec(PyObject *m) { - uint32_t ver = ZSTD_versionNumber(); - uint32_t major, minor, release; - - major = ver / 10000; - minor = (ver / 100) % 100; - release = ver % 100; - - return Py_BuildValue("III", major, minor, release); -} - -static inline int -add_vars_to_module(PyObject *module) -{ - PyObject *obj; - - /* zstd_version, a str. */ - if (PyModule_AddStringConstant(module, "zstd_version", - ZSTD_versionString()) < 0) { - return -1; - } - - /* zstd_version_info, a tuple. */ - obj = get_zstd_version_info(); - if (PyModule_AddObjectRef(module, "zstd_version_info", obj) < 0) { - Py_XDECREF(obj); - return -1; - } - Py_DECREF(obj); - - /* Add zstd parameters */ - if (add_parameters(module) < 0) { - return -1; - } - - /* _compressionLevel_values: (default, min, max) - ZSTD_defaultCLevel() was added in zstd v1.5.0 */ - obj = Py_BuildValue("iii", -#if ZSTD_VERSION_NUMBER < 10500 - ZSTD_CLEVEL_DEFAULT, -#else - ZSTD_defaultCLevel(), -#endif - ZSTD_minCLevel(), - ZSTD_maxCLevel()); - if (PyModule_AddObjectRef(module, - "_compressionLevel_values", - obj) < 0) { - Py_XDECREF(obj); - return -1; - } - Py_DECREF(obj); - - /* _ZSTD_CStreamSizes */ - obj = Py_BuildValue("II", - (uint32_t)ZSTD_CStreamInSize(), - (uint32_t)ZSTD_CStreamOutSize()); - if (PyModule_AddObjectRef(module, "_ZSTD_CStreamSizes", obj) < 0) { - Py_XDECREF(obj); - return -1; - } - Py_DECREF(obj); - - /* _ZSTD_DStreamSizes */ - obj = Py_BuildValue("II", - (uint32_t)ZSTD_DStreamInSize(), - (uint32_t)ZSTD_DStreamOutSize()); - if (PyModule_AddObjectRef(module, "_ZSTD_DStreamSizes", obj) < 0) { - Py_XDECREF(obj); - return -1; - } - Py_DECREF(obj); - - /* _ZSTD_CONFIG */ - obj = Py_BuildValue("isOOO", 8*(int)sizeof(Py_ssize_t), "c", - Py_False, - Py_True, -/* User mremap output buffer */ -#if defined(HAVE_MREMAP) - Py_True -#else - Py_False -#endif - ); - if (PyModule_AddObjectRef(module, "_ZSTD_CONFIG", obj) < 0) { - Py_XDECREF(obj); - return -1; - } - Py_DECREF(obj); - - return 0; -} - -#define ADD_STR_TO_STATE_MACRO(STR) \ - do { \ - mod_state->str_##STR = PyUnicode_FromString(#STR); \ - if (mod_state->str_##STR == NULL) { \ - return -1; \ - } \ - } while(0) - -static inline int -add_type_to_module(PyObject *module, const char *name, - PyType_Spec *type_spec, PyTypeObject **dest) -{ - PyObject *temp = PyType_FromModuleAndSpec(module, type_spec, NULL); - - if (PyModule_AddObjectRef(module, name, temp) < 0) { - Py_XDECREF(temp); - return -1; - } - - *dest = (PyTypeObject*) temp; - - return 0; -} - -static inline int -add_constant_to_type(PyTypeObject *type, const char *name, long value) -{ - PyObject *temp; - - temp = PyLong_FromLong(value); - if (temp == NULL) { - return -1; - } - - int rc = PyObject_SetAttrString((PyObject*) type, name, temp); - Py_DECREF(temp); - return rc; -} - -static int _zstd_exec(PyObject *module) { - _zstd_state* const mod_state = get_zstd_state(module); +#define ADD_TYPE(TYPE, SPEC) \ +do { \ + TYPE = (PyTypeObject *)PyType_FromModuleAndSpec(m, &(SPEC), NULL); \ + if (TYPE == NULL) { \ + return -1; \ + } \ + if (PyModule_AddType(m, TYPE) < 0) { \ + return -1; \ + } \ +} while (0) + +#define ADD_INT_MACRO(MACRO) \ + if (PyModule_AddIntConstant((m), #MACRO, (MACRO)) < 0) { \ + return -1; \ + } + +#define ADD_INT_CONST_TO_TYPE(TYPE, NAME, VALUE) \ +do { \ + PyObject *v = PyLong_FromLong((VALUE)); \ + if (v == NULL || PyObject_SetAttrString((PyObject *)(TYPE), \ + (NAME), v) < 0) { \ + Py_XDECREF(v); \ + return -1; \ + } \ + Py_DECREF(v); \ +} while (0) + + _zstd_state* mod_state = get_zstd_state(m); /* Reusable objects & variables */ - mod_state->empty_bytes = PyBytes_FromStringAndSize(NULL, 0); - if (mod_state->empty_bytes == NULL) { - return -1; - } - - mod_state->empty_readonly_memoryview = - PyMemoryView_FromMemory((char*)mod_state, 0, PyBUF_READ); - if (mod_state->empty_readonly_memoryview == NULL) { - return -1; - } - - /* Add str to module state */ - ADD_STR_TO_STATE_MACRO(read); - ADD_STR_TO_STATE_MACRO(readinto); - ADD_STR_TO_STATE_MACRO(write); - ADD_STR_TO_STATE_MACRO(flush); - mod_state->CParameter_type = NULL; mod_state->DParameter_type = NULL; - /* Add variables to module */ - if (add_vars_to_module(module) < 0) { - return -1; - } - - /* ZstdError */ + /* Create and add heap types */ + ADD_TYPE(mod_state->ZstdDict_type, zstd_dict_type_spec); + ADD_TYPE(mod_state->ZstdCompressor_type, zstd_compressor_type_spec); + ADD_TYPE(mod_state->ZstdDecompressor_type, zstd_decompressor_type_spec); mod_state->ZstdError = PyErr_NewExceptionWithDoc( - "_zstd.ZstdError", - "Call to the underlying zstd library failed.", - NULL, NULL); + "compression.zstd.ZstdError", + "An error occurred in the zstd library.", + NULL, NULL); if (mod_state->ZstdError == NULL) { return -1; } - - if (PyModule_AddObjectRef(module, "ZstdError", mod_state->ZstdError) < 0) { - Py_DECREF(mod_state->ZstdError); + if (PyModule_AddType(m, (PyTypeObject *)mod_state->ZstdError) < 0) { return -1; } - /* ZstdDict */ - if (add_type_to_module(module, - "ZstdDict", - &zstddict_type_spec, - &mod_state->ZstdDict_type) < 0) { + /* Add constants */ + if (PyModule_AddIntConstant(m, "zstd_version_number", + ZSTD_versionNumber()) < 0) { return -1; } - // ZstdCompressor - if (add_type_to_module(module, - "ZstdCompressor", - &zstdcompressor_type_spec, - &mod_state->ZstdCompressor_type) < 0) { + if (PyModule_AddStringConstant(m, "zstd_version", + ZSTD_versionString()) < 0) { return -1; } - // Add EndDirective enum to ZstdCompressor - if (add_constant_to_type(mod_state->ZstdCompressor_type, - "CONTINUE", - ZSTD_e_continue) < 0) { +#if ZSTD_VERSION_NUMBER >= 10500 + if (PyModule_AddIntConstant(m, "ZSTD_CLEVEL_DEFAULT", + ZSTD_defaultCLevel()) < 0) { return -1; } +#else + ADD_INT_MACRO(ZSTD_CLEVEL_DEFAULT); +#endif - if (add_constant_to_type(mod_state->ZstdCompressor_type, - "FLUSH_BLOCK", - ZSTD_e_flush) < 0) { + if (PyModule_Add(m, "ZSTD_DStreamOutSize", + PyLong_FromSize_t(ZSTD_DStreamOutSize())) < 0) { return -1; } - if (add_constant_to_type(mod_state->ZstdCompressor_type, - "FLUSH_FRAME", - ZSTD_e_end) < 0) { - return -1; - } - - // ZstdDecompressor - if (add_type_to_module(module, - "ZstdDecompressor", - &ZstdDecompressor_type_spec, - &mod_state->ZstdDecompressor_type) < 0) { - return -1; - } + /* Add zstd compression parameters. All should also be in cp_list. */ + ADD_INT_MACRO(ZSTD_c_compressionLevel); + ADD_INT_MACRO(ZSTD_c_windowLog); + ADD_INT_MACRO(ZSTD_c_hashLog); + ADD_INT_MACRO(ZSTD_c_chainLog); + ADD_INT_MACRO(ZSTD_c_searchLog); + ADD_INT_MACRO(ZSTD_c_minMatch); + ADD_INT_MACRO(ZSTD_c_targetLength); + ADD_INT_MACRO(ZSTD_c_strategy); + + ADD_INT_MACRO(ZSTD_c_enableLongDistanceMatching); + ADD_INT_MACRO(ZSTD_c_ldmHashLog); + ADD_INT_MACRO(ZSTD_c_ldmMinMatch); + ADD_INT_MACRO(ZSTD_c_ldmBucketSizeLog); + ADD_INT_MACRO(ZSTD_c_ldmHashRateLog); + + ADD_INT_MACRO(ZSTD_c_contentSizeFlag); + ADD_INT_MACRO(ZSTD_c_checksumFlag); + ADD_INT_MACRO(ZSTD_c_dictIDFlag); + + ADD_INT_MACRO(ZSTD_c_nbWorkers); + ADD_INT_MACRO(ZSTD_c_jobSize); + ADD_INT_MACRO(ZSTD_c_overlapLog); + + /* Add zstd decompression parameters. All should also be in dp_list. */ + ADD_INT_MACRO(ZSTD_d_windowLogMax); + + /* Add ZSTD_strategy enum members */ + ADD_INT_MACRO(ZSTD_fast); + ADD_INT_MACRO(ZSTD_dfast); + ADD_INT_MACRO(ZSTD_greedy); + ADD_INT_MACRO(ZSTD_lazy); + ADD_INT_MACRO(ZSTD_lazy2); + ADD_INT_MACRO(ZSTD_btlazy2); + ADD_INT_MACRO(ZSTD_btopt); + ADD_INT_MACRO(ZSTD_btultra); + ADD_INT_MACRO(ZSTD_btultra2); + + /* Add ZSTD_EndDirective enum members to ZstdCompressor */ + ADD_INT_CONST_TO_TYPE(mod_state->ZstdCompressor_type, + "CONTINUE", ZSTD_e_continue); + ADD_INT_CONST_TO_TYPE(mod_state->ZstdCompressor_type, + "FLUSH_BLOCK", ZSTD_e_flush); + ADD_INT_CONST_TO_TYPE(mod_state->ZstdCompressor_type, + "FLUSH_FRAME", ZSTD_e_end); + + /* Make ZstdCompressor immutable (set Py_TPFLAGS_IMMUTABLETYPE) */ + PyType_Freeze(mod_state->ZstdCompressor_type); + +#undef ADD_TYPE +#undef ADD_INT_MACRO +#undef ADD_ZSTD_COMPRESSOR_INT_CONST return 0; } @@ -837,14 +704,7 @@ static int _zstd_exec(PyObject *module) { static int _zstd_traverse(PyObject *module, visitproc visit, void *arg) { - _zstd_state* const mod_state = get_zstd_state(module); - - Py_VISIT(mod_state->empty_bytes); - Py_VISIT(mod_state->empty_readonly_memoryview); - Py_VISIT(mod_state->str_read); - Py_VISIT(mod_state->str_readinto); - Py_VISIT(mod_state->str_write); - Py_VISIT(mod_state->str_flush); + _zstd_state* mod_state = get_zstd_state(module); Py_VISIT(mod_state->ZstdDict_type); Py_VISIT(mod_state->ZstdCompressor_type); @@ -861,14 +721,7 @@ _zstd_traverse(PyObject *module, visitproc visit, void *arg) static int _zstd_clear(PyObject *module) { - _zstd_state* const mod_state = get_zstd_state(module); - - Py_CLEAR(mod_state->empty_bytes); - Py_CLEAR(mod_state->empty_readonly_memoryview); - Py_CLEAR(mod_state->str_read); - Py_CLEAR(mod_state->str_readinto); - Py_CLEAR(mod_state->str_write); - Py_CLEAR(mod_state->str_flush); + _zstd_state* mod_state = get_zstd_state(module); Py_CLEAR(mod_state->ZstdDict_type); Py_CLEAR(mod_state->ZstdCompressor_type); @@ -890,20 +743,21 @@ _zstd_free(void *module) static struct PyModuleDef_Slot _zstd_slots[] = { {Py_mod_exec, _zstd_exec}, + {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, {Py_mod_gil, Py_MOD_GIL_NOT_USED}, - - {0} + {0, NULL}, }; -struct PyModuleDef _zstdmodule = { - PyModuleDef_HEAD_INIT, +static struct PyModuleDef _zstdmodule = { + .m_base = PyModuleDef_HEAD_INIT, .m_name = "_zstd", + .m_doc = "Implementation module for Zstandard compression.", .m_size = sizeof(_zstd_state), .m_slots = _zstd_slots, .m_methods = _zstd_methods, .m_traverse = _zstd_traverse, .m_clear = _zstd_clear, - .m_free = _zstd_free + .m_free = _zstd_free, }; PyMODINIT_FUNC diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h index 9322ee259c5124b..82226ff8718e6b2 100644 --- a/Modules/_zstd/_zstdmodule.h +++ b/Modules/_zstd/_zstdmodule.h @@ -1,138 +1,28 @@ -#pragma once -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorithm & the zstd library. */ /* Declarations shared between different parts of the _zstd module*/ -#include "Python.h" +#ifndef ZSTD_MODULE_H +#define ZSTD_MODULE_H -#include "zstd.h" -#include "zdict.h" +#include "zstddict.h" +/* Type specs */ +extern PyType_Spec zstd_dict_type_spec; +extern PyType_Spec zstd_compressor_type_spec; +extern PyType_Spec zstd_decompressor_type_spec; -/* Forward declaration of module state */ -typedef struct _zstd_state _zstd_state; - -/* Forward reference of module def */ -extern PyModuleDef _zstdmodule; - -/* For clinic type calculations */ -static inline _zstd_state * -get_zstd_state_from_type(PyTypeObject *type) { - PyObject *module = PyType_GetModuleByDef(type, &_zstdmodule); - if (module == NULL) { - return NULL; - } - void *state = PyModule_GetState(module); - assert(state != NULL); - return (_zstd_state *)state; -} - -extern PyType_Spec zstddict_type_spec; -extern PyType_Spec zstdcompressor_type_spec; -extern PyType_Spec ZstdDecompressor_type_spec; - -struct _zstd_state { - PyObject *empty_bytes; - PyObject *empty_readonly_memoryview; - PyObject *str_read; - PyObject *str_readinto; - PyObject *str_write; - PyObject *str_flush; - +typedef struct { + /* Module heap types. */ PyTypeObject *ZstdDict_type; PyTypeObject *ZstdCompressor_type; PyTypeObject *ZstdDecompressor_type; PyObject *ZstdError; + /* enum types set by set_parameter_types. */ PyTypeObject *CParameter_type; PyTypeObject *DParameter_type; -}; - -typedef struct { - PyObject_HEAD - - /* Reusable compress/decompress dictionary, they are created once and - can be shared by multiple threads concurrently, since its usage is - read-only. - c_dicts is a dict, int(compressionLevel):PyCapsule(ZSTD_CDict*) */ - ZSTD_DDict *d_dict; - PyObject *c_dicts; - - /* Content of the dictionary, bytes object. */ - PyObject *dict_content; - /* Dictionary id */ - uint32_t dict_id; - - /* __init__ has been called, 0 or 1. */ - int inited; -} ZstdDict; - -typedef struct { - PyObject_HEAD - - /* Compression context */ - ZSTD_CCtx *cctx; - - /* ZstdDict object in use */ - PyObject *dict; - - /* Last mode, initialized to ZSTD_e_end */ - int last_mode; - - /* (nbWorker >= 1) ? 1 : 0 */ - int use_multithread; - - /* Compression level */ - int compression_level; - - /* __init__ has been called, 0 or 1. */ - int inited; -} ZstdCompressor; - -typedef struct { - PyObject_HEAD - - /* Decompression context */ - ZSTD_DCtx *dctx; - - /* ZstdDict object in use */ - PyObject *dict; - - /* Unconsumed input data */ - char *input_buffer; - size_t input_buffer_size; - size_t in_begin, in_end; - - /* Unused data */ - PyObject *unused_data; - - /* 0 if decompressor has (or may has) unconsumed input data, 0 or 1. */ - char needs_input; - - /* For decompress(), 0 or 1. - 1 when both input and output streams are at a frame edge, means a - frame is completely decoded and fully flushed, or the decompressor - just be initialized. */ - char at_frame_edge; - - /* For ZstdDecompressor, 0 or 1. - 1 means the end of the first frame has been reached. */ - char eof; - - /* Used for fast reset above three variables */ - char _unused_char_for_align; - - /* __init__ has been called, 0 or 1. */ - int inited; -} ZstdDecompressor; - -typedef enum { - TYPE_DECOMPRESSOR, // <D>, ZstdDecompressor class - TYPE_ENDLESS_DECOMPRESSOR, // <E>, decompress() function -} decompress_type; +} _zstd_state; typedef enum { ERR_DECOMPRESS, @@ -147,7 +37,7 @@ typedef enum { ERR_SET_C_LEVEL, ERR_TRAIN_DICT, - ERR_FINALIZE_DICT + ERR_FINALIZE_DICT, } error_type; typedef enum { @@ -156,41 +46,16 @@ typedef enum { DICT_TYPE_PREFIX = 2 } dictionary_type; -static inline int -mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out) { - return in->size == in->pos && out->size != out->pos; -} +extern ZstdDict * +_Py_parse_zstd_dict(const _zstd_state *state, + PyObject *dict, int *type); /* Format error message and set ZstdError. */ extern void -set_zstd_error(const _zstd_state* const state, - const error_type type, size_t zstd_ret); +set_zstd_error(const _zstd_state *state, + error_type type, size_t zstd_ret); extern void -set_parameter_error(const _zstd_state* const state, int is_compress, - int key_v, int value_v); - -static const char init_twice_msg[] = "__init__ method is called twice."; - -extern int -_PyZstd_load_c_dict(ZstdCompressor *self, PyObject *dict); - -extern int -_PyZstd_load_d_dict(ZstdDecompressor *self, PyObject *dict); - -extern int -_PyZstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options, - const char *arg_name, const char *arg_type); - -extern int -_PyZstd_set_d_parameters(ZstdDecompressor *self, PyObject *options); - -extern PyObject * -decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in, - Py_ssize_t max_length, - Py_ssize_t initial_size, - decompress_type type); +set_parameter_error(int is_compress, int key_v, int value_v); -extern PyObject * -compress_impl(ZstdCompressor *self, Py_buffer *data, - ZSTD_EndDirective end_directive); +#endif // !ZSTD_MODULE_H diff --git a/Modules/_zstd/buffer.h b/Modules/_zstd/buffer.h index 319b1214833fcf5..0ac7bcb4ddc4165 100644 --- a/Modules/_zstd/buffer.h +++ b/Modules/_zstd/buffer.h @@ -1,11 +1,12 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorithm & the zstd library. */ + +#ifndef ZSTD_BUFFER_H +#define ZSTD_BUFFER_H -#include "_zstdmodule.h" #include "pycore_blocks_output_buffer.h" +#include <zstd.h> // ZSTD_outBuffer + /* Blocks output buffer wrapper code */ /* Initialize the buffer, and grow the buffer. @@ -18,7 +19,8 @@ _OutputBuffer_InitAndGrow(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob, /* Ensure .list was set to NULL */ assert(buffer->list == NULL); - Py_ssize_t res = _BlocksOutputBuffer_InitAndGrow(buffer, max_length, &ob->dst); + Py_ssize_t res = _BlocksOutputBuffer_InitAndGrow(buffer, max_length, + &ob->dst); if (res < 0) { return -1; } @@ -33,8 +35,7 @@ _OutputBuffer_InitAndGrow(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob, Return -1 on failure */ static inline int _OutputBuffer_InitWithSize(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob, - Py_ssize_t max_length, - Py_ssize_t init_size) + Py_ssize_t max_length, Py_ssize_t init_size) { Py_ssize_t block_size; @@ -49,7 +50,8 @@ _OutputBuffer_InitWithSize(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob, block_size = init_size; } - Py_ssize_t res = _BlocksOutputBuffer_InitWithSize(buffer, block_size, &ob->dst); + Py_ssize_t res = _BlocksOutputBuffer_InitWithSize(buffer, block_size, + &ob->dst); if (res < 0) { return -1; } @@ -102,3 +104,5 @@ _OutputBuffer_ReachedMaxLength(_BlocksOutputBuffer *buffer, ZSTD_outBuffer *ob) return buffer->allocated == buffer->max_length; } + +#endif // !ZSTD_BUFFER_H diff --git a/Modules/_zstd/clinic/_zstdmodule.c.h b/Modules/_zstd/clinic/_zstdmodule.c.h index 2f8225389b7aea3..766e1cfa776767b 100644 --- a/Modules/_zstd/clinic/_zstdmodule.c.h +++ b/Modules/_zstd/clinic/_zstdmodule.c.h @@ -9,11 +9,11 @@ preserve #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_modsupport.h" // _PyArg_CheckPositional() -PyDoc_STRVAR(_zstd__train_dict__doc__, -"_train_dict($module, samples_bytes, samples_sizes, dict_size, /)\n" +PyDoc_STRVAR(_zstd_train_dict__doc__, +"train_dict($module, samples_bytes, samples_sizes, dict_size, /)\n" "--\n" "\n" -"Internal function, train a zstd dictionary on sample data.\n" +"Train a Zstandard dictionary on sample data.\n" "\n" " samples_bytes\n" " Concatenation of samples.\n" @@ -22,31 +22,31 @@ PyDoc_STRVAR(_zstd__train_dict__doc__, " dict_size\n" " The size of the dictionary."); -#define _ZSTD__TRAIN_DICT_METHODDEF \ - {"_train_dict", _PyCFunction_CAST(_zstd__train_dict), METH_FASTCALL, _zstd__train_dict__doc__}, +#define _ZSTD_TRAIN_DICT_METHODDEF \ + {"train_dict", _PyCFunction_CAST(_zstd_train_dict), METH_FASTCALL, _zstd_train_dict__doc__}, static PyObject * -_zstd__train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, - PyObject *samples_sizes, Py_ssize_t dict_size); +_zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, + PyObject *samples_sizes, Py_ssize_t dict_size); static PyObject * -_zstd__train_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +_zstd_train_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyBytesObject *samples_bytes; PyObject *samples_sizes; Py_ssize_t dict_size; - if (!_PyArg_CheckPositional("_train_dict", nargs, 3, 3)) { + if (!_PyArg_CheckPositional("train_dict", nargs, 3, 3)) { goto exit; } if (!PyBytes_Check(args[0])) { - _PyArg_BadArgument("_train_dict", "argument 1", "bytes", args[0]); + _PyArg_BadArgument("train_dict", "argument 1", "bytes", args[0]); goto exit; } samples_bytes = (PyBytesObject *)args[0]; if (!PyTuple_Check(args[1])) { - _PyArg_BadArgument("_train_dict", "argument 2", "tuple", args[1]); + _PyArg_BadArgument("train_dict", "argument 2", "tuple", args[1]); goto exit; } samples_sizes = args[1]; @@ -62,18 +62,18 @@ _zstd__train_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } dict_size = ival; } - return_value = _zstd__train_dict_impl(module, samples_bytes, samples_sizes, dict_size); + return_value = _zstd_train_dict_impl(module, samples_bytes, samples_sizes, dict_size); exit: return return_value; } -PyDoc_STRVAR(_zstd__finalize_dict__doc__, -"_finalize_dict($module, custom_dict_bytes, samples_bytes,\n" -" samples_sizes, dict_size, compression_level, /)\n" +PyDoc_STRVAR(_zstd_finalize_dict__doc__, +"finalize_dict($module, custom_dict_bytes, samples_bytes, samples_sizes,\n" +" dict_size, compression_level, /)\n" "--\n" "\n" -"Internal function, finalize a zstd dictionary.\n" +"Finalize a Zstandard dictionary.\n" "\n" " custom_dict_bytes\n" " Custom dictionary content.\n" @@ -84,19 +84,19 @@ PyDoc_STRVAR(_zstd__finalize_dict__doc__, " dict_size\n" " The size of the dictionary.\n" " compression_level\n" -" Optimize for a specific zstd compression level, 0 means default."); +" Optimize for a specific Zstandard compression level, 0 means default."); -#define _ZSTD__FINALIZE_DICT_METHODDEF \ - {"_finalize_dict", _PyCFunction_CAST(_zstd__finalize_dict), METH_FASTCALL, _zstd__finalize_dict__doc__}, +#define _ZSTD_FINALIZE_DICT_METHODDEF \ + {"finalize_dict", _PyCFunction_CAST(_zstd_finalize_dict), METH_FASTCALL, _zstd_finalize_dict__doc__}, static PyObject * -_zstd__finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, - PyBytesObject *samples_bytes, - PyObject *samples_sizes, Py_ssize_t dict_size, - int compression_level); +_zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, + PyBytesObject *samples_bytes, + PyObject *samples_sizes, Py_ssize_t dict_size, + int compression_level); static PyObject * -_zstd__finalize_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) +_zstd_finalize_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyBytesObject *custom_dict_bytes; @@ -105,21 +105,21 @@ _zstd__finalize_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) Py_ssize_t dict_size; int compression_level; - if (!_PyArg_CheckPositional("_finalize_dict", nargs, 5, 5)) { + if (!_PyArg_CheckPositional("finalize_dict", nargs, 5, 5)) { goto exit; } if (!PyBytes_Check(args[0])) { - _PyArg_BadArgument("_finalize_dict", "argument 1", "bytes", args[0]); + _PyArg_BadArgument("finalize_dict", "argument 1", "bytes", args[0]); goto exit; } custom_dict_bytes = (PyBytesObject *)args[0]; if (!PyBytes_Check(args[1])) { - _PyArg_BadArgument("_finalize_dict", "argument 2", "bytes", args[1]); + _PyArg_BadArgument("finalize_dict", "argument 2", "bytes", args[1]); goto exit; } samples_bytes = (PyBytesObject *)args[1]; if (!PyTuple_Check(args[2])) { - _PyArg_BadArgument("_finalize_dict", "argument 3", "tuple", args[2]); + _PyArg_BadArgument("finalize_dict", "argument 3", "tuple", args[2]); goto exit; } samples_sizes = args[2]; @@ -139,32 +139,31 @@ _zstd__finalize_dict(PyObject *module, PyObject *const *args, Py_ssize_t nargs) if (compression_level == -1 && PyErr_Occurred()) { goto exit; } - return_value = _zstd__finalize_dict_impl(module, custom_dict_bytes, samples_bytes, samples_sizes, dict_size, compression_level); + return_value = _zstd_finalize_dict_impl(module, custom_dict_bytes, samples_bytes, samples_sizes, dict_size, compression_level); exit: return return_value; } -PyDoc_STRVAR(_zstd__get_param_bounds__doc__, -"_get_param_bounds($module, /, parameter, is_compress)\n" +PyDoc_STRVAR(_zstd_get_param_bounds__doc__, +"get_param_bounds($module, /, parameter, is_compress)\n" "--\n" "\n" -"Internal function, get CompressionParameter/DecompressionParameter bounds.\n" +"Get CompressionParameter/DecompressionParameter bounds.\n" "\n" " parameter\n" " The parameter to get bounds.\n" " is_compress\n" " True for CompressionParameter, False for DecompressionParameter."); -#define _ZSTD__GET_PARAM_BOUNDS_METHODDEF \ - {"_get_param_bounds", _PyCFunction_CAST(_zstd__get_param_bounds), METH_FASTCALL|METH_KEYWORDS, _zstd__get_param_bounds__doc__}, +#define _ZSTD_GET_PARAM_BOUNDS_METHODDEF \ + {"get_param_bounds", _PyCFunction_CAST(_zstd_get_param_bounds), METH_FASTCALL|METH_KEYWORDS, _zstd_get_param_bounds__doc__}, static PyObject * -_zstd__get_param_bounds_impl(PyObject *module, int parameter, - int is_compress); +_zstd_get_param_bounds_impl(PyObject *module, int parameter, int is_compress); static PyObject * -_zstd__get_param_bounds(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_zstd_get_param_bounds(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -190,7 +189,7 @@ _zstd__get_param_bounds(PyObject *module, PyObject *const *args, Py_ssize_t narg static const char * const _keywords[] = {"parameter", "is_compress", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "_get_param_bounds", + .fname = "get_param_bounds", .kwtuple = KWTUPLE, }; #undef KWTUPLE @@ -211,7 +210,7 @@ _zstd__get_param_bounds(PyObject *module, PyObject *const *args, Py_ssize_t narg if (is_compress < 0) { goto exit; } - return_value = _zstd__get_param_bounds_impl(module, parameter, is_compress); + return_value = _zstd_get_param_bounds_impl(module, parameter, is_compress); exit: return return_value; @@ -221,13 +220,11 @@ PyDoc_STRVAR(_zstd_get_frame_size__doc__, "get_frame_size($module, /, frame_buffer)\n" "--\n" "\n" -"Get the size of a zstd frame, including frame header and 4-byte checksum if it has one.\n" +"Get the size of a Zstandard frame, including the header and optional checksum.\n" "\n" " frame_buffer\n" " A bytes-like object, it should start from the beginning of a frame,\n" -" and contains at least one complete frame.\n" -"\n" -"It will iterate all blocks\' headers within a frame, to accumulate the frame size."); +" and contains at least one complete frame."); #define _ZSTD_GET_FRAME_SIZE_METHODDEF \ {"get_frame_size", _PyCFunction_CAST(_zstd_get_frame_size), METH_FASTCALL|METH_KEYWORDS, _zstd_get_frame_size__doc__}, @@ -288,23 +285,23 @@ _zstd_get_frame_size(PyObject *module, PyObject *const *args, Py_ssize_t nargs, return return_value; } -PyDoc_STRVAR(_zstd__get_frame_info__doc__, -"_get_frame_info($module, /, frame_buffer)\n" +PyDoc_STRVAR(_zstd_get_frame_info__doc__, +"get_frame_info($module, /, frame_buffer)\n" "--\n" "\n" -"Internal function, get zstd frame infomation from a frame header.\n" +"Get Zstandard frame infomation from a frame header.\n" "\n" " frame_buffer\n" -" A bytes-like object, containing the header of a zstd frame."); +" A bytes-like object, containing the header of a Zstandard frame."); -#define _ZSTD__GET_FRAME_INFO_METHODDEF \ - {"_get_frame_info", _PyCFunction_CAST(_zstd__get_frame_info), METH_FASTCALL|METH_KEYWORDS, _zstd__get_frame_info__doc__}, +#define _ZSTD_GET_FRAME_INFO_METHODDEF \ + {"get_frame_info", _PyCFunction_CAST(_zstd_get_frame_info), METH_FASTCALL|METH_KEYWORDS, _zstd_get_frame_info__doc__}, static PyObject * -_zstd__get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer); +_zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer); static PyObject * -_zstd__get_frame_info(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_zstd_get_frame_info(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -330,7 +327,7 @@ _zstd__get_frame_info(PyObject *module, PyObject *const *args, Py_ssize_t nargs, static const char * const _keywords[] = {"frame_buffer", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "_get_frame_info", + .fname = "get_frame_info", .kwtuple = KWTUPLE, }; #undef KWTUPLE @@ -345,7 +342,7 @@ _zstd__get_frame_info(PyObject *module, PyObject *const *args, Py_ssize_t nargs, if (PyObject_GetBuffer(args[0], &frame_buffer, PyBUF_SIMPLE) != 0) { goto exit; } - return_value = _zstd__get_frame_info_impl(module, &frame_buffer); + return_value = _zstd_get_frame_info_impl(module, &frame_buffer); exit: /* Cleanup for frame_buffer */ @@ -356,26 +353,26 @@ _zstd__get_frame_info(PyObject *module, PyObject *const *args, Py_ssize_t nargs, return return_value; } -PyDoc_STRVAR(_zstd__set_parameter_types__doc__, -"_set_parameter_types($module, /, c_parameter_type, d_parameter_type)\n" +PyDoc_STRVAR(_zstd_set_parameter_types__doc__, +"set_parameter_types($module, /, c_parameter_type, d_parameter_type)\n" "--\n" "\n" -"Internal function, set CompressionParameter/DecompressionParameter types for validity check.\n" +"Set CompressionParameter and DecompressionParameter types for validity check.\n" "\n" " c_parameter_type\n" " CompressionParameter IntEnum type object\n" " d_parameter_type\n" " DecompressionParameter IntEnum type object"); -#define _ZSTD__SET_PARAMETER_TYPES_METHODDEF \ - {"_set_parameter_types", _PyCFunction_CAST(_zstd__set_parameter_types), METH_FASTCALL|METH_KEYWORDS, _zstd__set_parameter_types__doc__}, +#define _ZSTD_SET_PARAMETER_TYPES_METHODDEF \ + {"set_parameter_types", _PyCFunction_CAST(_zstd_set_parameter_types), METH_FASTCALL|METH_KEYWORDS, _zstd_set_parameter_types__doc__}, static PyObject * -_zstd__set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type, - PyObject *d_parameter_type); +_zstd_set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type, + PyObject *d_parameter_type); static PyObject * -_zstd__set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_zstd_set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) @@ -401,7 +398,7 @@ _zstd__set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t n static const char * const _keywords[] = {"c_parameter_type", "d_parameter_type", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, - .fname = "_set_parameter_types", + .fname = "set_parameter_types", .kwtuple = KWTUPLE, }; #undef KWTUPLE @@ -415,18 +412,18 @@ _zstd__set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t n goto exit; } if (!PyObject_TypeCheck(args[0], &PyType_Type)) { - _PyArg_BadArgument("_set_parameter_types", "argument 'c_parameter_type'", (&PyType_Type)->tp_name, args[0]); + _PyArg_BadArgument("set_parameter_types", "argument 'c_parameter_type'", (&PyType_Type)->tp_name, args[0]); goto exit; } c_parameter_type = args[0]; if (!PyObject_TypeCheck(args[1], &PyType_Type)) { - _PyArg_BadArgument("_set_parameter_types", "argument 'd_parameter_type'", (&PyType_Type)->tp_name, args[1]); + _PyArg_BadArgument("set_parameter_types", "argument 'd_parameter_type'", (&PyType_Type)->tp_name, args[1]); goto exit; } d_parameter_type = args[1]; - return_value = _zstd__set_parameter_types_impl(module, c_parameter_type, d_parameter_type); + return_value = _zstd_set_parameter_types_impl(module, c_parameter_type, d_parameter_type); exit: return return_value; } -/*[clinic end generated code: output=189c462236a7096c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=437b084f149e68e5 input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/clinic/compressor.c.h b/Modules/_zstd/clinic/compressor.c.h index d7909cdf89fcd18..6775ba4826a652b 100644 --- a/Modules/_zstd/clinic/compressor.c.h +++ b/Modules/_zstd/clinic/compressor.c.h @@ -8,30 +8,30 @@ preserve #endif #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() -PyDoc_STRVAR(_zstd_ZstdCompressor___init____doc__, +PyDoc_STRVAR(_zstd_ZstdCompressor_new__doc__, "ZstdCompressor(level=None, options=None, zstd_dict=None)\n" "--\n" "\n" "Create a compressor object for compressing data incrementally.\n" "\n" " level\n" -" The compression level to use, defaults to ZSTD_CLEVEL_DEFAULT.\n" +" The compression level to use. Defaults to COMPRESSION_LEVEL_DEFAULT.\n" " options\n" " A dict object that contains advanced compression parameters.\n" " zstd_dict\n" -" A ZstdDict object, a pre-trained zstd dictionary.\n" +" A ZstdDict object, a pre-trained Zstandard dictionary.\n" "\n" -"Thread-safe at method level. For one-shot compression, use the compress()\n" -"function instead."); +"Thread-safe at method level. For one-shot compression, use the\n" +"compress() function instead."); -static int -_zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level, - PyObject *options, PyObject *zstd_dict); +static PyObject * +_zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, + PyObject *options, PyObject *zstd_dict); -static int -_zstd_ZstdCompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs) +static PyObject * +_zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - int return_value = -1; + PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) #define NUM_KEYWORDS 3 @@ -89,7 +89,7 @@ _zstd_ZstdCompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs) } zstd_dict = fastargs[2]; skip_optional_pos: - return_value = _zstd_ZstdCompressor___init___impl((ZstdCompressor *)self, level, options, zstd_dict); + return_value = _zstd_ZstdCompressor_new_impl(type, level, options, zstd_dict); exit: return return_value; @@ -105,9 +105,9 @@ PyDoc_STRVAR(_zstd_ZstdCompressor_compress__doc__, " Can be these 3 values ZstdCompressor.CONTINUE,\n" " ZstdCompressor.FLUSH_BLOCK, ZstdCompressor.FLUSH_FRAME\n" "\n" -"Return a chunk of compressed data if possible, or b\'\' otherwise. When you have\n" -"finished providing data to the compressor, call the flush() method to finish\n" -"the compression process."); +"Return a chunk of compressed data if possible, or b\'\' otherwise.\n" +"When you have finished providing data to the compressor, call the\n" +"flush() method to finish the compression process."); #define _ZSTD_ZSTDCOMPRESSOR_COMPRESS_METHODDEF \ {"compress", _PyCFunction_CAST(_zstd_ZstdCompressor_compress), METH_FASTCALL|METH_KEYWORDS, _zstd_ZstdCompressor_compress__doc__}, @@ -189,9 +189,9 @@ PyDoc_STRVAR(_zstd_ZstdCompressor_flush__doc__, " Can be these 2 values ZstdCompressor.FLUSH_FRAME,\n" " ZstdCompressor.FLUSH_BLOCK\n" "\n" -"Flush any remaining data left in internal buffers. Since zstd data consists\n" -"of one or more independent frames, the compressor object can still be used\n" -"after this method is called."); +"Flush any remaining data left in internal buffers. Since Zstandard\n" +"data consists of one or more independent frames, the compressor\n" +"object can still be used after this method is called."); #define _ZSTD_ZSTDCOMPRESSOR_FLUSH_METHODDEF \ {"flush", _PyCFunction_CAST(_zstd_ZstdCompressor_flush), METH_FASTCALL|METH_KEYWORDS, _zstd_ZstdCompressor_flush__doc__}, @@ -252,4 +252,44 @@ _zstd_ZstdCompressor_flush(PyObject *self, PyObject *const *args, Py_ssize_t nar exit: return return_value; } -/*[clinic end generated code: output=ef69eab155be39f6 input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_zstd_ZstdCompressor_set_pledged_input_size__doc__, +"set_pledged_input_size($self, size, /)\n" +"--\n" +"\n" +"Set the uncompressed content size to be written into the frame header.\n" +"\n" +" size\n" +" The size of the uncompressed data to be provided to the compressor.\n" +"\n" +"This method can be used to ensure the header of the frame about to\n" +"be written includes the size of the data, unless the\n" +"CompressionParameter.content_size_flag is set to False.\n" +"If last_mode != FLUSH_FRAME, then a RuntimeError is raised.\n" +"\n" +"It is important to ensure that the pledged data size matches the\n" +"actual data size. If they do not match the compressed output data\n" +"may be corrupted and the final chunk written may be lost."); + +#define _ZSTD_ZSTDCOMPRESSOR_SET_PLEDGED_INPUT_SIZE_METHODDEF \ + {"set_pledged_input_size", (PyCFunction)_zstd_ZstdCompressor_set_pledged_input_size, METH_O, _zstd_ZstdCompressor_set_pledged_input_size__doc__}, + +static PyObject * +_zstd_ZstdCompressor_set_pledged_input_size_impl(ZstdCompressor *self, + unsigned long long size); + +static PyObject * +_zstd_ZstdCompressor_set_pledged_input_size(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + unsigned long long size; + + if (!zstd_contentsize_converter(arg, &size)) { + goto exit; + } + return_value = _zstd_ZstdCompressor_set_pledged_input_size_impl((ZstdCompressor *)self, size); + +exit: + return return_value; +} +/*[clinic end generated code: output=1a5e21476885866c input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/clinic/decompressor.c.h b/Modules/_zstd/clinic/decompressor.c.h index 9359c637203f8fb..fe3b76b8bb369df 100644 --- a/Modules/_zstd/clinic/decompressor.c.h +++ b/Modules/_zstd/clinic/decompressor.c.h @@ -7,31 +7,30 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_abstract.h" // _PyNumber_Index() -#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() -PyDoc_STRVAR(_zstd_ZstdDecompressor___init____doc__, +PyDoc_STRVAR(_zstd_ZstdDecompressor_new__doc__, "ZstdDecompressor(zstd_dict=None, options=None)\n" "--\n" "\n" "Create a decompressor object for decompressing data incrementally.\n" "\n" " zstd_dict\n" -" A ZstdDict object, a pre-trained zstd dictionary.\n" +" A ZstdDict object, a pre-trained Zstandard dictionary.\n" " options\n" " A dict object that contains advanced decompression parameters.\n" "\n" -"Thread-safe at method level. For one-shot decompression, use the decompress()\n" -"function instead."); +"Thread-safe at method level. For one-shot decompression, use the\n" +"decompress() function instead."); -static int -_zstd_ZstdDecompressor___init___impl(ZstdDecompressor *self, - PyObject *zstd_dict, PyObject *options); +static PyObject * +_zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, + PyObject *options); -static int -_zstd_ZstdDecompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs) +static PyObject * +_zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - int return_value = -1; + PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) #define NUM_KEYWORDS 2 @@ -82,7 +81,7 @@ _zstd_ZstdDecompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs } options = fastargs[1]; skip_optional_pos: - return_value = _zstd_ZstdDecompressor___init___impl((ZstdDecompressor *)self, zstd_dict, options); + return_value = _zstd_ZstdDecompressor_new_impl(type, zstd_dict, options); exit: return return_value; @@ -92,7 +91,8 @@ PyDoc_STRVAR(_zstd_ZstdDecompressor_unused_data__doc__, "A bytes object of un-consumed input data.\n" "\n" "When ZstdDecompressor object stops after a frame is\n" -"decompressed, unused input data after the frame. Otherwise this will be b\'\'."); +"decompressed, unused input data after the frame. Otherwise this\n" +"will be b\'\'."); #if defined(_zstd_ZstdDecompressor_unused_data_DOCSTR) # undef _zstd_ZstdDecompressor_unused_data_DOCSTR #endif @@ -114,13 +114,7 @@ _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self); static PyObject * _zstd_ZstdDecompressor_unused_data_get(PyObject *self, void *Py_UNUSED(context)) { - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(self); - return_value = _zstd_ZstdDecompressor_unused_data_get_impl((ZstdDecompressor *)self); - Py_END_CRITICAL_SECTION(); - - return return_value; + return _zstd_ZstdDecompressor_unused_data_get_impl((ZstdDecompressor *)self); } PyDoc_STRVAR(_zstd_ZstdDecompressor_decompress__doc__, @@ -130,24 +124,25 @@ PyDoc_STRVAR(_zstd_ZstdDecompressor_decompress__doc__, "Decompress *data*, returning uncompressed bytes if possible, or b\'\' otherwise.\n" "\n" " data\n" -" A bytes-like object, zstd data to be decompressed.\n" +" A bytes-like object, Zstandard data to be decompressed.\n" " max_length\n" " Maximum size of returned data. When it is negative, the size of\n" " output buffer is unlimited. When it is nonnegative, returns at\n" " most max_length bytes of decompressed data.\n" "\n" -"If *max_length* is nonnegative, returns at most *max_length* bytes of\n" -"decompressed data. If this limit is reached and further output can be\n" -"produced, *self.needs_input* will be set to ``False``. In this case, the next\n" -"call to *decompress()* may provide *data* as b\'\' to obtain more of the output.\n" +"If *max_length* is nonnegative, returns at most *max_length* bytes\n" +"of decompressed data. If this limit is reached and further output\n" +"can be produced, *self.needs_input* will be set to ``False``. In\n" +"this case, the next call to *decompress()* may provide *data* as b\'\'\n" +"to obtain more of the output.\n" "\n" -"If all of the input data was decompressed and returned (either because this\n" -"was less than *max_length* bytes, or because *max_length* was negative),\n" -"*self.needs_input* will be set to True.\n" +"If all of the input data was decompressed and returned (either\n" +"because this was less than *max_length* bytes, or because\n" +"*max_length* was negative), *self.needs_input* will be set to True.\n" "\n" -"Attempting to decompress data after the end of a frame is reached raises an\n" -"EOFError. Any data found after the end of the frame is ignored and saved in\n" -"the self.unused_data attribute."); +"Attempting to decompress data after the end of a frame is reached\n" +"raises an EOFError. Any data found after the end of the frame is\n" +"ignored and saved in the self.unused_data attribute."); #define _ZSTD_ZSTDDECOMPRESSOR_DECOMPRESS_METHODDEF \ {"decompress", _PyCFunction_CAST(_zstd_ZstdDecompressor_decompress), METH_FASTCALL|METH_KEYWORDS, _zstd_ZstdDecompressor_decompress__doc__}, @@ -227,4 +222,4 @@ _zstd_ZstdDecompressor_decompress(PyObject *self, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=ae703f0465a2906d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=70bc308e86463751 input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/clinic/zstddict.c.h b/Modules/_zstd/clinic/zstddict.c.h index 4e0f7b64172a744..18b049e3cbe37ef 100644 --- a/Modules/_zstd/clinic/zstddict.c.h +++ b/Modules/_zstd/clinic/zstddict.c.h @@ -6,38 +6,35 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif -#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() -PyDoc_STRVAR(_zstd_ZstdDict___init____doc__, -"ZstdDict(dict_content, is_raw=False)\n" +PyDoc_STRVAR(_zstd_ZstdDict_new__doc__, +"ZstdDict(dict_content, /, *, is_raw=False)\n" "--\n" "\n" -"Represents a zstd dictionary, which can be used for compression/decompression.\n" +"Represents a Zstandard dictionary.\n" "\n" " dict_content\n" -" A bytes-like object, dictionary\'s content.\n" +" The content of a Zstandard dictionary as a bytes-like object.\n" " is_raw\n" -" This parameter is for advanced user. True means dict_content\n" -" argument is a \"raw content\" dictionary, free of any format\n" -" restriction. False means dict_content argument is an ordinary\n" -" zstd dictionary, was created by zstd functions, follow a\n" -" specified format.\n" +" If true, perform no checks on *dict_content*, useful for some\n" +" advanced cases. Otherwise, check that the content represents\n" +" a Zstandard dictionary created by the zstd library or CLI.\n" "\n" -"It\'s thread-safe, and can be shared by multiple ZstdCompressor /\n" -"ZstdDecompressor objects."); +"The dictionary can be used for compression or decompression, and can be\n" +"shared by multiple ZstdCompressor or ZstdDecompressor objects."); -static int -_zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content, - int is_raw); +static PyObject * +_zstd_ZstdDict_new_impl(PyTypeObject *type, Py_buffer *dict_content, + int is_raw); -static int -_zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs) +static PyObject * +_zstd_ZstdDict_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - int return_value = -1; + PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 1 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -46,7 +43,7 @@ _zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs) } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(dict_content), &_Py_ID(is_raw), }, + .ob_item = { &_Py_ID(is_raw), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -55,7 +52,7 @@ _zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"dict_content", "is_raw", NULL}; + static const char * const _keywords[] = {"", "is_raw", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "ZstdDict", @@ -66,38 +63,73 @@ _zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs) PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; - PyObject *dict_content; + Py_buffer dict_content = {NULL, NULL}; int is_raw = 0; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, - /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } - dict_content = fastargs[0]; + if (PyObject_GetBuffer(fastargs[0], &dict_content, PyBUF_SIMPLE) != 0) { + goto exit; + } if (!noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; } is_raw = PyObject_IsTrue(fastargs[1]); if (is_raw < 0) { goto exit; } -skip_optional_pos: - return_value = _zstd_ZstdDict___init___impl((ZstdDict *)self, dict_content, is_raw); +skip_optional_kwonly: + return_value = _zstd_ZstdDict_new_impl(type, &dict_content, is_raw); exit: + /* Cleanup for dict_content */ + if (dict_content.obj) { + PyBuffer_Release(&dict_content); + } + return return_value; } +PyDoc_STRVAR(_zstd_ZstdDict_dict_content__doc__, +"The content of a Zstandard dictionary, as a bytes object."); +#if defined(_zstd_ZstdDict_dict_content_DOCSTR) +# undef _zstd_ZstdDict_dict_content_DOCSTR +#endif +#define _zstd_ZstdDict_dict_content_DOCSTR _zstd_ZstdDict_dict_content__doc__ + +#if !defined(_zstd_ZstdDict_dict_content_DOCSTR) +# define _zstd_ZstdDict_dict_content_DOCSTR NULL +#endif +#if defined(_ZSTD_ZSTDDICT_DICT_CONTENT_GETSETDEF) +# undef _ZSTD_ZSTDDICT_DICT_CONTENT_GETSETDEF +# define _ZSTD_ZSTDDICT_DICT_CONTENT_GETSETDEF {"dict_content", (getter)_zstd_ZstdDict_dict_content_get, (setter)_zstd_ZstdDict_dict_content_set, _zstd_ZstdDict_dict_content_DOCSTR}, +#else +# define _ZSTD_ZSTDDICT_DICT_CONTENT_GETSETDEF {"dict_content", (getter)_zstd_ZstdDict_dict_content_get, NULL, _zstd_ZstdDict_dict_content_DOCSTR}, +#endif + +static PyObject * +_zstd_ZstdDict_dict_content_get_impl(ZstdDict *self); + +static PyObject * +_zstd_ZstdDict_dict_content_get(PyObject *self, void *Py_UNUSED(context)) +{ + return _zstd_ZstdDict_dict_content_get_impl((ZstdDict *)self); +} + PyDoc_STRVAR(_zstd_ZstdDict_as_digested_dict__doc__, "Load as a digested dictionary to compressor.\n" "\n" -"Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_digested_dict)\n" -"1. Some advanced compression parameters of compressor may be overridden\n" -" by parameters of digested dictionary.\n" -"2. ZstdDict has a digested dictionaries cache for each compression level.\n" -" It\'s faster when loading again a digested dictionary with the same\n" -" compression level.\n" +"Pass this attribute as zstd_dict argument:\n" +"compress(dat, zstd_dict=zd.as_digested_dict)\n" +"\n" +"1. Some advanced compression parameters of compressor may be\n" +" overridden by parameters of digested dictionary.\n" +"2. ZstdDict has a digested dictionaries cache for each compression\n" +" level. It\'s faster when loading again a digested dictionary with\n" +" the same compression level.\n" "3. No need to use this for decompression."); #if defined(_zstd_ZstdDict_as_digested_dict_DOCSTR) # undef _zstd_ZstdDict_as_digested_dict_DOCSTR @@ -120,22 +152,19 @@ _zstd_ZstdDict_as_digested_dict_get_impl(ZstdDict *self); static PyObject * _zstd_ZstdDict_as_digested_dict_get(PyObject *self, void *Py_UNUSED(context)) { - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(self); - return_value = _zstd_ZstdDict_as_digested_dict_get_impl((ZstdDict *)self); - Py_END_CRITICAL_SECTION(); - - return return_value; + return _zstd_ZstdDict_as_digested_dict_get_impl((ZstdDict *)self); } PyDoc_STRVAR(_zstd_ZstdDict_as_undigested_dict__doc__, "Load as an undigested dictionary to compressor.\n" "\n" -"Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_undigested_dict)\n" -"1. The advanced compression parameters of compressor will not be overridden.\n" -"2. Loading an undigested dictionary is costly. If load an undigested dictionary\n" -" multiple times, consider reusing a compressor object.\n" +"Pass this attribute as zstd_dict argument:\n" +"compress(dat, zstd_dict=zd.as_undigested_dict)\n" +"\n" +"1. The advanced compression parameters of compressor will not be\n" +" overridden.\n" +"2. Loading an undigested dictionary is costly. If load an undigested\n" +" dictionary multiple times, consider reusing a compressor object.\n" "3. No need to use this for decompression."); #if defined(_zstd_ZstdDict_as_undigested_dict_DOCSTR) # undef _zstd_ZstdDict_as_undigested_dict_DOCSTR @@ -158,23 +187,20 @@ _zstd_ZstdDict_as_undigested_dict_get_impl(ZstdDict *self); static PyObject * _zstd_ZstdDict_as_undigested_dict_get(PyObject *self, void *Py_UNUSED(context)) { - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(self); - return_value = _zstd_ZstdDict_as_undigested_dict_get_impl((ZstdDict *)self); - Py_END_CRITICAL_SECTION(); - - return return_value; + return _zstd_ZstdDict_as_undigested_dict_get_impl((ZstdDict *)self); } PyDoc_STRVAR(_zstd_ZstdDict_as_prefix__doc__, "Load as a prefix to compressor/decompressor.\n" "\n" -"Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_prefix)\n" -"1. Prefix is compatible with long distance matching, while dictionary is not.\n" -"2. It only works for the first frame, then the compressor/decompressor will\n" -" return to no prefix state.\n" -"3. When decompressing, must use the same prefix as when compressing.\""); +"Pass this attribute as zstd_dict argument:\n" +"compress(dat, zstd_dict=zd.as_prefix)\n" +"\n" +"1. Prefix is compatible with long distance matching, while\n" +" dictionary is not.\n" +"2. It only works for the first frame, then the\n" +" compressor/decompressor will return to no prefix state.\n" +"3. When decompressing, must use the same prefix as when compressing."); #if defined(_zstd_ZstdDict_as_prefix_DOCSTR) # undef _zstd_ZstdDict_as_prefix_DOCSTR #endif @@ -196,12 +222,6 @@ _zstd_ZstdDict_as_prefix_get_impl(ZstdDict *self); static PyObject * _zstd_ZstdDict_as_prefix_get(PyObject *self, void *Py_UNUSED(context)) { - PyObject *return_value = NULL; - - Py_BEGIN_CRITICAL_SECTION(self); - return_value = _zstd_ZstdDict_as_prefix_get_impl((ZstdDict *)self); - Py_END_CRITICAL_SECTION(); - - return return_value; + return _zstd_ZstdDict_as_prefix_get_impl((ZstdDict *)self); } -/*[clinic end generated code: output=59257c053f74eda7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=49b66061b4fcdb5f input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index b735981e7476d55..fa31d3a0e90abde 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -1,123 +1,196 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorithm & the zstd library. */ /* ZstdCompressor class definitions */ /*[clinic input] module _zstd -class _zstd.ZstdCompressor "ZstdCompressor *" "clinic_state()->ZstdCompressor_type" +class _zstd.ZstdCompressor "ZstdCompressor *" "&zstd_compressor_type_spec" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=875bf614798f80cb]*/ - +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=7166021db1ef7df8]*/ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif -#include "_zstdmodule.h" +#include "Python.h" +#include "_zstdmodule.h" #include "buffer.h" +#include "internal/pycore_lock.h" // PyMutex_IsLocked #include <stddef.h> // offsetof() +#include <zstd.h> // ZSTD_*() + +typedef struct { + PyObject_HEAD + + /* Compression context */ + ZSTD_CCtx *cctx; + + /* ZstdDict object in use */ + PyObject *dict; + /* Last mode, initialized to ZSTD_e_end */ + int last_mode; + + /* (nbWorker >= 1) ? 1 : 0 */ + int use_multithread; + + /* Compression level */ + int compression_level; + + /* Lock to protect the compression context */ + PyMutex lock; +} ZstdCompressor; #define ZstdCompressor_CAST(op) ((ZstdCompressor *)op) -int -_PyZstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options, - const char *arg_name, const char* arg_type) +/*[python input] + +class zstd_contentsize_converter(CConverter): + type = 'unsigned long long' + converter = 'zstd_contentsize_converter' + +[python start generated code]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=0932c350d633c7de]*/ + + +static int +zstd_contentsize_converter(PyObject *size, unsigned long long *p) { - size_t zstd_ret; - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state == NULL) { - return -1; + // None means the user indicates the size is unknown. + if (size == Py_None) { + *p = ZSTD_CONTENTSIZE_UNKNOWN; } - - /* Integer compression level */ - if (PyLong_Check(level_or_options)) { - int level = PyLong_AsInt(level_or_options); - if (level == -1 && PyErr_Occurred()) { + else { + /* ZSTD_CONTENTSIZE_UNKNOWN is 0ULL - 1 + ZSTD_CONTENTSIZE_ERROR is 0ULL - 2 + Users should only pass values < ZSTD_CONTENTSIZE_ERROR */ + unsigned long long pledged_size = PyLong_AsUnsignedLongLong(size); + /* Here we check for (unsigned long long)-1 as a sign of an error in + PyLong_AsUnsignedLongLong */ + if (pledged_size == (unsigned long long)-1 && PyErr_Occurred()) { + *p = ZSTD_CONTENTSIZE_ERROR; + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_Format(PyExc_ValueError, + "size argument should be a positive int less " + "than %llu", ZSTD_CONTENTSIZE_ERROR); + return 0; + } + return 0; + } + if (pledged_size >= ZSTD_CONTENTSIZE_ERROR) { + *p = ZSTD_CONTENTSIZE_ERROR; PyErr_Format(PyExc_ValueError, - "Compression level should be an int value between %d and %d.", - ZSTD_minCLevel(), ZSTD_maxCLevel()); - return -1; + "size argument should be a positive int less " + "than %llu", ZSTD_CONTENTSIZE_ERROR); + return 0; } + *p = pledged_size; + } + return 1; +} + +#include "clinic/compressor.c.h" + +static int +_zstd_set_c_level(ZstdCompressor *self, int level) +{ + /* Set integer compression level */ + int min_level = ZSTD_minCLevel(); + int max_level = ZSTD_maxCLevel(); + if (level < min_level || level > max_level) { + PyErr_Format(PyExc_ValueError, + "illegal compression level %d; the valid range is [%d, %d]", + level, min_level, max_level); + return -1; + } - /* Save for generating ZSTD_CDICT */ - self->compression_level = level; + /* Save for generating ZSTD_CDICT */ + self->compression_level = level; - /* Set compressionLevel to compression context */ - zstd_ret = ZSTD_CCtx_setParameter(self->cctx, - ZSTD_c_compressionLevel, - level); + /* Set compressionLevel to compression context */ + size_t zstd_ret = ZSTD_CCtx_setParameter( + self->cctx, ZSTD_c_compressionLevel, level); - /* Check error */ - if (ZSTD_isError(zstd_ret)) { - set_zstd_error(mod_state, ERR_SET_C_LEVEL, zstd_ret); + /* Check error */ + if (ZSTD_isError(zstd_ret)) { + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); + set_zstd_error(mod_state, ERR_SET_C_LEVEL, zstd_ret); + return -1; + } + return 0; +} + +static int +_zstd_set_c_parameters(ZstdCompressor *self, PyObject *options) +{ + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); + if (mod_state == NULL) { + return -1; + } + + if (!PyDict_Check(options)) { + PyErr_Format(PyExc_TypeError, + "ZstdCompressor() argument 'options' must be dict, not %T", + options); + return -1; + } + + Py_ssize_t pos = 0; + PyObject *key, *value; + while (PyDict_Next(options, &pos, &key, &value)) { + /* Check key type */ + if (Py_TYPE(key) == mod_state->DParameter_type) { + PyErr_SetString(PyExc_TypeError, + "compression options dictionary key must not be a " + "DecompressionParameter attribute"); return -1; } - return 0; - } - /* Options dict */ - if (PyDict_Check(level_or_options)) { - PyObject *key, *value; - Py_ssize_t pos = 0; + Py_INCREF(key); + Py_INCREF(value); + int key_v = PyLong_AsInt(key); + Py_DECREF(key); + if (key_v == -1 && PyErr_Occurred()) { + Py_DECREF(value); + return -1; + } - while (PyDict_Next(level_or_options, &pos, &key, &value)) { - /* Check key type */ - if (Py_TYPE(key) == mod_state->DParameter_type) { - PyErr_SetString(PyExc_TypeError, - "Key of compression option dict should " - "NOT be DecompressionParameter."); - return -1; - } + int value_v = PyLong_AsInt(value); + Py_DECREF(value); + if (value_v == -1 && PyErr_Occurred()) { + return -1; + } - int key_v = PyLong_AsInt(key); - if (key_v == -1 && PyErr_Occurred()) { - PyErr_SetString(PyExc_ValueError, - "Key of options dict should be a CompressionParameter attribute."); + if (key_v == ZSTD_c_compressionLevel) { + if (_zstd_set_c_level(self, value_v) < 0) { return -1; } - - // TODO(emmatyping): check bounds when there is a value error here for better - // error message? - int value_v = PyLong_AsInt(value); - if (value_v == -1 && PyErr_Occurred()) { - PyErr_SetString(PyExc_ValueError, - "Value of option dict should be an int."); - return -1; + continue; + } + if (key_v == ZSTD_c_nbWorkers) { + /* From the zstd library docs: + 1. When nbWorkers >= 1, triggers asynchronous mode when + used with ZSTD_compressStream2(). + 2, Default value is `0`, aka "single-threaded mode" : no + worker is spawned, compression is performed inside + caller's thread, all invocations are blocking. */ + if (value_v != 0) { + self->use_multithread = 1; } + } - if (key_v == ZSTD_c_compressionLevel) { - /* Save for generating ZSTD_CDICT */ - self->compression_level = value_v; - } - else if (key_v == ZSTD_c_nbWorkers) { - /* From zstd library doc: - 1. When nbWorkers >= 1, triggers asynchronous mode when - used with ZSTD_compressStream2(). - 2, Default value is `0`, aka "single-threaded mode" : no - worker is spawned, compression is performed inside - caller's thread, all invocations are blocking. */ - if (value_v != 0) { - self->use_multithread = 1; - } - } + /* Set parameter to compression context */ + size_t zstd_ret = ZSTD_CCtx_setParameter(self->cctx, key_v, value_v); - /* Set parameter to compression context */ - zstd_ret = ZSTD_CCtx_setParameter(self->cctx, key_v, value_v); - if (ZSTD_isError(zstd_ret)) { - set_parameter_error(mod_state, 1, key_v, value_v); - return -1; - } + /* Check error */ + if (ZSTD_isError(zstd_ret)) { + set_parameter_error(1, key_v, value_v); + return -1; } - return 0; } - PyErr_Format(PyExc_TypeError, "Invalid type for %s. Expected %s", arg_name, arg_type); - return -1; + return 0; } static void @@ -130,12 +203,12 @@ capsule_free_cdict(PyObject *capsule) ZSTD_CDict * _get_CDict(ZstdDict *self, int compressionLevel) { + assert(PyMutex_IsLocked(&self->lock)); PyObject *level = NULL; - PyObject *capsule; + PyObject *capsule = NULL; ZSTD_CDict *cdict; + int ret; - // TODO(emmatyping): refactor critical section code into a lock_held function - Py_BEGIN_CRITICAL_SECTION(self); /* int level object */ level = PyLong_FromLong(compressionLevel); @@ -144,27 +217,23 @@ _get_CDict(ZstdDict *self, int compressionLevel) } /* Get PyCapsule object from self->c_dicts */ - capsule = PyDict_GetItemWithError(self->c_dicts, level); + ret = PyDict_GetItemRef(self->c_dicts, level, &capsule); + if (ret < 0) { + goto error; + } if (capsule == NULL) { - if (PyErr_Occurred()) { - goto error; - } - /* Create ZSTD_CDict instance */ - char *dict_buffer = PyBytes_AS_STRING(self->dict_content); - Py_ssize_t dict_len = Py_SIZE(self->dict_content); Py_BEGIN_ALLOW_THREADS - cdict = ZSTD_createCDict(dict_buffer, - dict_len, + cdict = ZSTD_createCDict(self->dict_buffer, self->dict_len, compressionLevel); Py_END_ALLOW_THREADS if (cdict == NULL) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state != NULL) { PyErr_SetString(mod_state->ZstdError, - "Failed to create ZSTD_CDict instance from zstd " - "dictionary content. Maybe the content is corrupted."); + "Failed to create a ZSTD_CDict instance from " + "Zstandard dictionary content."); } goto error; } @@ -177,11 +246,10 @@ _get_CDict(ZstdDict *self, int compressionLevel) } /* Add PyCapsule object to self->c_dicts */ - if (PyDict_SetItem(self->c_dicts, level, capsule) < 0) { - Py_DECREF(capsule); + ret = PyDict_SetItem(self->c_dicts, level, capsule); + if (ret < 0) { goto error; } - Py_DECREF(capsule); } else { /* ZSTD_CDict instance already exists */ @@ -193,61 +261,15 @@ _get_CDict(ZstdDict *self, int compressionLevel) cdict = NULL; success: Py_XDECREF(level); - Py_END_CRITICAL_SECTION(); + Py_XDECREF(capsule); return cdict; } -int -_PyZstd_load_c_dict(ZstdCompressor *self, PyObject *dict) { - +static int +_zstd_load_impl(ZstdCompressor *self, ZstdDict *zd, + _zstd_state *mod_state, int type) +{ size_t zstd_ret; - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state == NULL) { - return -1; - } - ZstdDict *zd; - int type, ret; - - /* Check ZstdDict */ - ret = PyObject_IsInstance(dict, (PyObject*)mod_state->ZstdDict_type); - if (ret < 0) { - return -1; - } - else if (ret > 0) { - /* When compressing, use undigested dictionary by default. */ - zd = (ZstdDict*)dict; - type = DICT_TYPE_UNDIGESTED; - goto load; - } - - /* Check (ZstdDict, type) */ - if (PyTuple_CheckExact(dict) && PyTuple_GET_SIZE(dict) == 2) { - /* Check ZstdDict */ - ret = PyObject_IsInstance(PyTuple_GET_ITEM(dict, 0), - (PyObject*)mod_state->ZstdDict_type); - if (ret < 0) { - return -1; - } - else if (ret > 0) { - /* type == -1 may indicate an error. */ - type = PyLong_AsInt(PyTuple_GET_ITEM(dict, 1)); - if (type == DICT_TYPE_DIGESTED || - type == DICT_TYPE_UNDIGESTED || - type == DICT_TYPE_PREFIX) - { - assert(type >= 0); - zd = (ZstdDict*)PyTuple_GET_ITEM(dict, 0); - goto load; - } - } - } - - /* Wrong type */ - PyErr_SetString(PyExc_TypeError, - "zstd_dict argument should be ZstdDict object."); - return -1; - -load: if (type == DICT_TYPE_DIGESTED) { /* Get ZSTD_CDict */ ZSTD_CDict *c_dict = _get_CDict(zd, self->compression_level); @@ -256,28 +278,18 @@ _PyZstd_load_c_dict(ZstdCompressor *self, PyObject *dict) { } /* Reference a prepared dictionary. It overrides some compression context's parameters. */ - Py_BEGIN_CRITICAL_SECTION(self); zstd_ret = ZSTD_CCtx_refCDict(self->cctx, c_dict); - Py_END_CRITICAL_SECTION(); } else if (type == DICT_TYPE_UNDIGESTED) { /* Load a dictionary. It doesn't override compression context's parameters. */ - Py_BEGIN_CRITICAL_SECTION2(self, zd); - zstd_ret = ZSTD_CCtx_loadDictionary( - self->cctx, - PyBytes_AS_STRING(zd->dict_content), - Py_SIZE(zd->dict_content)); - Py_END_CRITICAL_SECTION2(); + zstd_ret = ZSTD_CCtx_loadDictionary(self->cctx, zd->dict_buffer, + zd->dict_len); } else if (type == DICT_TYPE_PREFIX) { /* Load a prefix */ - Py_BEGIN_CRITICAL_SECTION2(self, zd); - zstd_ret = ZSTD_CCtx_refPrefix( - self->cctx, - PyBytes_AS_STRING(zd->dict_content), - Py_SIZE(zd->dict_content)); - Py_END_CRITICAL_SECTION2(); + zstd_ret = ZSTD_CCtx_refPrefix(self->cctx, zd->dict_buffer, + zd->dict_len); } else { Py_UNREACHABLE(); @@ -291,28 +303,58 @@ _PyZstd_load_c_dict(ZstdCompressor *self, PyObject *dict) { return 0; } -#define clinic_state() (get_zstd_state_from_type(type)) -#include "clinic/compressor.c.h" -#undef clinic_state +static int +_zstd_load_c_dict(ZstdCompressor *self, PyObject *dict) +{ + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); + /* When compressing, use undigested dictionary by default. */ + int type = DICT_TYPE_UNDIGESTED; + ZstdDict *zd = _Py_parse_zstd_dict(mod_state, dict, &type); + if (zd == NULL) { + return -1; + } + int ret; + PyMutex_Lock(&zd->lock); + ret = _zstd_load_impl(self, zd, mod_state, type); + PyMutex_Unlock(&zd->lock); + return ret; +} + +/*[clinic input] +@classmethod +_zstd.ZstdCompressor.__new__ as _zstd_ZstdCompressor_new + level: object = None + The compression level to use. Defaults to COMPRESSION_LEVEL_DEFAULT. + options: object = None + A dict object that contains advanced compression parameters. + zstd_dict: object = None + A ZstdDict object, a pre-trained Zstandard dictionary. + +Create a compressor object for compressing data incrementally. + +Thread-safe at method level. For one-shot compression, use the +compress() function instead. +[clinic start generated code]*/ static PyObject * -_zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwargs)) +_zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, + PyObject *options, PyObject *zstd_dict) +/*[clinic end generated code: output=cdef61eafecac3d7 input=bbfeeaa06fd3bd4d]*/ { - ZstdCompressor *self; - self = PyObject_GC_New(ZstdCompressor, type); + ZstdCompressor* self = PyObject_GC_New(ZstdCompressor, type); if (self == NULL) { goto error; } - self->inited = 0; - self->dict = NULL; self->use_multithread = 0; - + self->compression_level = ZSTD_CLEVEL_DEFAULT; + self->dict = NULL; + self->lock = (PyMutex){0}; /* Compression context */ self->cctx = ZSTD_createCCtx(); if (self->cctx == NULL) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state != NULL) { PyErr_SetString(mod_state->ZstdError, "Unable to create ZSTD_CCtx instance."); @@ -323,12 +365,56 @@ _zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject /* Last mode */ self->last_mode = ZSTD_e_end; + if (level != Py_None && options != Py_None) { + PyErr_SetString(PyExc_TypeError, + "Only one of level or options should be used."); + goto error; + } + + /* Set compression level */ + if (level != Py_None) { + if (!PyLong_Check(level)) { + PyErr_SetString(PyExc_TypeError, + "invalid type for level, expected int"); + goto error; + } + int level_v = PyLong_AsInt(level); + if (level_v == -1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_OverflowError)) { + PyErr_Format(PyExc_ValueError, + "illegal compression level; the valid range is [%d, %d]", + ZSTD_minCLevel(), ZSTD_maxCLevel()); + } + goto error; + } + if (_zstd_set_c_level(self, level_v) < 0) { + goto error; + } + } + + /* Set options dictionary */ + if (options != Py_None) { + if (_zstd_set_c_parameters(self, options) < 0) { + goto error; + } + } + + /* Load Zstandard dictionary to compression context */ + if (zstd_dict != Py_None) { + if (_zstd_load_c_dict(self, zstd_dict) < 0) { + goto error; + } + Py_INCREF(zstd_dict); + self->dict = zstd_dict; + } + + // We can only start GC tracking once self->dict is set. + PyObject_GC_Track(self); + return (PyObject*)self; error: - if (self != NULL) { - PyObject_GC_Del(self); - } + Py_XDECREF(self); return NULL; } @@ -340,7 +426,11 @@ ZstdCompressor_dealloc(PyObject *ob) PyObject_GC_UnTrack(self); /* Free compression context */ - ZSTD_freeCCtx(self->cctx); + if (self->cctx) { + ZSTD_freeCCtx(self->cctx); + } + + assert(!PyMutex_IsLocked(&self->lock)); /* Py_XDECREF the dict after free the compression context */ Py_CLEAR(self->dict); @@ -350,72 +440,11 @@ ZstdCompressor_dealloc(PyObject *ob) Py_DECREF(tp); } -/*[clinic input] -_zstd.ZstdCompressor.__init__ - - level: object = None - The compression level to use, defaults to ZSTD_CLEVEL_DEFAULT. - options: object = None - A dict object that contains advanced compression parameters. - zstd_dict: object = None - A ZstdDict object, a pre-trained zstd dictionary. - -Create a compressor object for compressing data incrementally. - -Thread-safe at method level. For one-shot compression, use the compress() -function instead. -[clinic start generated code]*/ - -static int -_zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level, - PyObject *options, PyObject *zstd_dict) -/*[clinic end generated code: output=215e6c4342732f96 input=9f79b0d8d34c8ef0]*/ -{ - /* Only called once */ - if (self->inited) { - PyErr_SetString(PyExc_RuntimeError, init_twice_msg); - return -1; - } - self->inited = 1; - - if (level != Py_None && options != Py_None) { - PyErr_SetString(PyExc_RuntimeError, "Only one of level or options should be used."); - return -1; - } - - /* Set compressLevel/options to compression context */ - if (level != Py_None) { - if (_PyZstd_set_c_parameters(self, level, "level", "int") < 0) { - return -1; - } - } - - if (options != Py_None) { - if (_PyZstd_set_c_parameters(self, options, "options", "dict") < 0) { - return -1; - } - } - - /* Load dictionary to compression context */ - if (zstd_dict != Py_None) { - if (_PyZstd_load_c_dict(self, zstd_dict) < 0) { - return -1; - } - - /* Py_INCREF the dict */ - Py_INCREF(zstd_dict); - self->dict = zstd_dict; - } - - // We can only start tracking self with the GC once self->dict is set. - PyObject_GC_Track(self); - return 0; -} - -PyObject * -compress_impl(ZstdCompressor *self, Py_buffer *data, - ZSTD_EndDirective end_directive) +static PyObject * +compress_lock_held(ZstdCompressor *self, Py_buffer *data, + ZSTD_EndDirective end_directive) { + assert(PyMutex_IsLocked(&self->lock)); ZSTD_inBuffer in; ZSTD_outBuffer out; _BlocksOutputBuffer buffer = {.list = NULL}; @@ -442,12 +471,12 @@ compress_impl(ZstdCompressor *self, Py_buffer *data, } if (_OutputBuffer_InitWithSize(&buffer, &out, -1, - (Py_ssize_t) output_buffer_size) < 0) { + (Py_ssize_t) output_buffer_size) < 0) { goto error; } - /* zstd stream compress */ + /* Zstandard stream compress */ while (1) { Py_BEGIN_ALLOW_THREADS zstd_ret = ZSTD_compressStream2(self->cctx, &out, &in, end_directive); @@ -455,10 +484,8 @@ compress_impl(ZstdCompressor *self, Py_buffer *data, /* Check error */ if (ZSTD_isError(zstd_ret)) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state != NULL) { - set_zstd_error(mod_state, ERR_COMPRESS, zstd_ret); - } + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); + set_zstd_error(mod_state, ERR_COMPRESS, zstd_ret); goto error; } @@ -487,9 +514,18 @@ compress_impl(ZstdCompressor *self, Py_buffer *data, return NULL; } +#ifndef NDEBUG +static inline int +mt_continue_should_break(ZSTD_inBuffer *in, ZSTD_outBuffer *out) +{ + return in->size == in->pos && out->size != out->pos; +} +#endif + static PyObject * -compress_mt_continue_impl(ZstdCompressor *self, Py_buffer *data) +compress_mt_continue_lock_held(ZstdCompressor *self, Py_buffer *data) { + assert(PyMutex_IsLocked(&self->lock)); ZSTD_inBuffer in; ZSTD_outBuffer out; _BlocksOutputBuffer buffer = {.list = NULL}; @@ -505,24 +541,25 @@ compress_mt_continue_impl(ZstdCompressor *self, Py_buffer *data) goto error; } - /* zstd stream compress */ + /* Zstandard stream compress */ while (1) { Py_BEGIN_ALLOW_THREADS do { - zstd_ret = ZSTD_compressStream2(self->cctx, &out, &in, ZSTD_e_continue); - } while (out.pos != out.size && in.pos != in.size && !ZSTD_isError(zstd_ret)); + zstd_ret = ZSTD_compressStream2(self->cctx, &out, &in, + ZSTD_e_continue); + } while (out.pos != out.size + && in.pos != in.size + && !ZSTD_isError(zstd_ret)); Py_END_ALLOW_THREADS /* Check error */ if (ZSTD_isError(zstd_ret)) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state != NULL) { - set_zstd_error(mod_state, ERR_COMPRESS, zstd_ret); - } + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); + set_zstd_error(mod_state, ERR_COMPRESS, zstd_ret); goto error; } - /* Like compress_impl(), output as much as possible. */ + /* Like compress_lock_held(), output as much as possible. */ if (out.pos == out.size) { if (_OutputBuffer_Grow(&buffer, &out) < 0) { goto error; @@ -556,15 +593,15 @@ _zstd.ZstdCompressor.compress Provide data to the compressor object. -Return a chunk of compressed data if possible, or b'' otherwise. When you have -finished providing data to the compressor, call the flush() method to finish -the compression process. +Return a chunk of compressed data if possible, or b'' otherwise. +When you have finished providing data to the compressor, call the +flush() method to finish the compression process. [clinic start generated code]*/ static PyObject * _zstd_ZstdCompressor_compress_impl(ZstdCompressor *self, Py_buffer *data, int mode) -/*[clinic end generated code: output=ed7982d1cf7b4f98 input=ac2c21d180f579ea]*/ +/*[clinic end generated code: output=ed7982d1cf7b4f98 input=11726dff64d7b2f9]*/ { PyObject *ret; @@ -581,14 +618,14 @@ _zstd_ZstdCompressor_compress_impl(ZstdCompressor *self, Py_buffer *data, } /* Thread-safe code */ - Py_BEGIN_CRITICAL_SECTION(self); + PyMutex_Lock(&self->lock); /* Compress */ if (self->use_multithread && mode == ZSTD_e_continue) { - ret = compress_mt_continue_impl(self, data); + ret = compress_mt_continue_lock_held(self, data); } else { - ret = compress_impl(self, data, mode); + ret = compress_lock_held(self, data, mode); } if (ret) { @@ -600,7 +637,7 @@ _zstd_ZstdCompressor_compress_impl(ZstdCompressor *self, Py_buffer *data, /* Resetting cctx's session never fail */ ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only); } - Py_END_CRITICAL_SECTION(); + PyMutex_Unlock(&self->lock); return ret; } @@ -614,14 +651,14 @@ _zstd.ZstdCompressor.flush Finish the compression process. -Flush any remaining data left in internal buffers. Since zstd data consists -of one or more independent frames, the compressor object can still be used -after this method is called. +Flush any remaining data left in internal buffers. Since Zstandard +data consists of one or more independent frames, the compressor +object can still be used after this method is called. [clinic start generated code]*/ static PyObject * _zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode) -/*[clinic end generated code: output=b7cf2c8d64dcf2e3 input=a766870301932b85]*/ +/*[clinic end generated code: output=b7cf2c8d64dcf2e3 input=130e0b1eddf0f498]*/ { PyObject *ret; @@ -635,8 +672,9 @@ _zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode) } /* Thread-safe code */ - Py_BEGIN_CRITICAL_SECTION(self); - ret = compress_impl(self, NULL, mode); + PyMutex_Lock(&self->lock); + + ret = compress_lock_held(self, NULL, mode); if (ret) { self->last_mode = mode; @@ -647,28 +685,80 @@ _zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode) /* Resetting cctx's session never fail */ ZSTD_CCtx_reset(self->cctx, ZSTD_reset_session_only); } - Py_END_CRITICAL_SECTION(); + PyMutex_Unlock(&self->lock); return ret; } + +/*[clinic input] +_zstd.ZstdCompressor.set_pledged_input_size + + size: zstd_contentsize + The size of the uncompressed data to be provided to the compressor. + / + +Set the uncompressed content size to be written into the frame header. + +This method can be used to ensure the header of the frame about to +be written includes the size of the data, unless the +CompressionParameter.content_size_flag is set to False. +If last_mode != FLUSH_FRAME, then a RuntimeError is raised. + +It is important to ensure that the pledged data size matches the +actual data size. If they do not match the compressed output data +may be corrupted and the final chunk written may be lost. +[clinic start generated code]*/ + +static PyObject * +_zstd_ZstdCompressor_set_pledged_input_size_impl(ZstdCompressor *self, + unsigned long long size) +/*[clinic end generated code: output=3a09e55cc0e3b4f9 input=d6d1669171af3da4]*/ +{ + // Error occured while converting argument, should be unreachable + assert(size != ZSTD_CONTENTSIZE_ERROR); + + /* Thread-safe code */ + PyMutex_Lock(&self->lock); + + /* Check the current mode */ + if (self->last_mode != ZSTD_e_end) { + PyErr_SetString(PyExc_ValueError, + "set_pledged_input_size() method must be called " + "when last_mode == FLUSH_FRAME"); + PyMutex_Unlock(&self->lock); + return NULL; + } + + /* Set pledged content size */ + size_t zstd_ret = ZSTD_CCtx_setPledgedSrcSize(self->cctx, size); + PyMutex_Unlock(&self->lock); + if (ZSTD_isError(zstd_ret)) { + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); + set_zstd_error(mod_state, ERR_SET_PLEDGED_INPUT_SIZE, zstd_ret); + return NULL; + } + + Py_RETURN_NONE; +} + static PyMethodDef ZstdCompressor_methods[] = { _ZSTD_ZSTDCOMPRESSOR_COMPRESS_METHODDEF _ZSTD_ZSTDCOMPRESSOR_FLUSH_METHODDEF - - {0} + _ZSTD_ZSTDCOMPRESSOR_SET_PLEDGED_INPUT_SIZE_METHODDEF + {NULL, NULL} }; PyDoc_STRVAR(ZstdCompressor_last_mode_doc, "The last mode used to this compressor object, its value can be .CONTINUE,\n" ".FLUSH_BLOCK, .FLUSH_FRAME. Initialized to .FLUSH_FRAME.\n\n" -"It can be used to get the current state of a compressor, such as, data flushed,\n" -"a frame ended."); +"It can be used to get the current state of a compressor, such as, data\n" +"flushed, or a frame ended."); static PyMemberDef ZstdCompressor_members[] = { {"last_mode", Py_T_INT, offsetof(ZstdCompressor, last_mode), - Py_READONLY, ZstdCompressor_last_mode_doc}, - {0} + Py_READONLY, ZstdCompressor_last_mode_doc}, + {NULL} }; static int @@ -690,18 +780,20 @@ ZstdCompressor_clear(PyObject *ob) static PyType_Slot zstdcompressor_slots[] = { {Py_tp_new, _zstd_ZstdCompressor_new}, {Py_tp_dealloc, ZstdCompressor_dealloc}, - {Py_tp_init, _zstd_ZstdCompressor___init__}, {Py_tp_methods, ZstdCompressor_methods}, {Py_tp_members, ZstdCompressor_members}, - {Py_tp_doc, (char*)_zstd_ZstdCompressor___init____doc__}, + {Py_tp_doc, (void *)_zstd_ZstdCompressor_new__doc__}, {Py_tp_traverse, ZstdCompressor_traverse}, {Py_tp_clear, ZstdCompressor_clear}, - {0} + {0, 0} }; -PyType_Spec zstdcompressor_type_spec = { - .name = "_zstd.ZstdCompressor", +PyType_Spec zstd_compressor_type_spec = { + .name = "compression.zstd.ZstdCompressor", .basicsize = sizeof(ZstdCompressor), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + // Py_TPFLAGS_IMMUTABLETYPE is not used here as several + // associated constants need to be added to the type. + // PyType_Freeze is called later to set the flag. + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, .slots = zstdcompressor_slots, }; diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index a4be180c0088fce..85b67d468864890 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -1,176 +1,143 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorithm & the zstd library. */ /* ZstdDecompressor class definition */ /*[clinic input] module _zstd -class _zstd.ZstdDecompressor "ZstdDecompressor *" "clinic_state()->ZstdDecompressor_type" +class _zstd.ZstdDecompressor "ZstdDecompressor *" "&zstd_decompressor_type_spec" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=4e6eae327c0c0c76]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=e2969ddf48a203e0]*/ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif -#include "_zstdmodule.h" +#include "Python.h" +#include "_zstdmodule.h" #include "buffer.h" +#include "internal/pycore_lock.h" // PyMutex_IsLocked +#include <stdbool.h> // bool #include <stddef.h> // offsetof() +#include <zstd.h> // ZSTD_*() + +typedef struct { + PyObject_HEAD + + /* Decompression context */ + ZSTD_DCtx *dctx; + + /* ZstdDict object in use */ + PyObject *dict; + + /* Unconsumed input data */ + char *input_buffer; + size_t input_buffer_size; + size_t in_begin, in_end; + + /* Unused data */ + PyObject *unused_data; + + /* 0 if decompressor has (or may has) unconsumed input data, 0 or 1. */ + bool needs_input; + + /* For ZstdDecompressor, 0 or 1. + 1 means the end of the first frame has been reached. */ + bool eof; + + /* Lock to protect the decompression context */ + PyMutex lock; +} ZstdDecompressor; #define ZstdDecompressor_CAST(op) ((ZstdDecompressor *)op) +#include "clinic/decompressor.c.h" + static inline ZSTD_DDict * _get_DDict(ZstdDict *self) { + assert(PyMutex_IsLocked(&self->lock)); ZSTD_DDict *ret; - /* Already created */ - if (self->d_dict != NULL) { - return self->d_dict; - } - - Py_BEGIN_CRITICAL_SECTION(self); if (self->d_dict == NULL) { /* Create ZSTD_DDict instance from dictionary content */ - char *dict_buffer = PyBytes_AS_STRING(self->dict_content); - Py_ssize_t dict_len = Py_SIZE(self->dict_content); Py_BEGIN_ALLOW_THREADS - self->d_dict = ZSTD_createDDict(dict_buffer, - dict_len); + ret = ZSTD_createDDict(self->dict_buffer, self->dict_len); Py_END_ALLOW_THREADS + self->d_dict = ret; if (self->d_dict == NULL) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state != NULL) { PyErr_SetString(mod_state->ZstdError, - "Failed to create ZSTD_DDict instance from zstd " - "dictionary content. Maybe the content is corrupted."); + "Failed to create a ZSTD_DDict instance from " + "Zstandard dictionary content."); } } } - /* Don't lose any exception */ - ret = self->d_dict; - Py_END_CRITICAL_SECTION(); - - return ret; + return self->d_dict; } -/* Set decompression parameters to decompression context */ -int -_PyZstd_set_d_parameters(ZstdDecompressor *self, PyObject *options) +static int +_zstd_set_d_parameters(ZstdDecompressor *self, PyObject *options) { - size_t zstd_ret; - PyObject *key, *value; - Py_ssize_t pos; - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state == NULL) { return -1; } if (!PyDict_Check(options)) { - PyErr_SetString(PyExc_TypeError, - "options argument should be dict object."); + PyErr_Format(PyExc_TypeError, + "ZstdDecompressor() argument 'options' must be dict, not %T", + options); return -1; } - pos = 0; + Py_ssize_t pos = 0; + PyObject *key, *value; while (PyDict_Next(options, &pos, &key, &value)) { /* Check key type */ if (Py_TYPE(key) == mod_state->CParameter_type) { PyErr_SetString(PyExc_TypeError, - "Key of decompression options dict should " - "NOT be CompressionParameter."); + "decompression options dictionary key must not be a " + "CompressionParameter attribute"); return -1; } - /* Both key & value should be 32-bit signed int */ + Py_INCREF(key); + Py_INCREF(value); int key_v = PyLong_AsInt(key); + Py_DECREF(key); if (key_v == -1 && PyErr_Occurred()) { - PyErr_SetString(PyExc_ValueError, - "Key of options dict should be a DecompressionParameter attribute."); + Py_DECREF(value); return -1; } - // TODO(emmatyping): check bounds when there is a value error here for better - // error message? int value_v = PyLong_AsInt(value); + Py_DECREF(value); if (value_v == -1 && PyErr_Occurred()) { - PyErr_SetString(PyExc_ValueError, - "Value of options dict should be an int."); return -1; } /* Set parameter to compression context */ - Py_BEGIN_CRITICAL_SECTION(self); - zstd_ret = ZSTD_DCtx_setParameter(self->dctx, key_v, value_v); - Py_END_CRITICAL_SECTION(); + size_t zstd_ret = ZSTD_DCtx_setParameter(self->dctx, key_v, value_v); /* Check error */ if (ZSTD_isError(zstd_ret)) { - set_parameter_error(mod_state, 0, key_v, value_v); + set_parameter_error(0, key_v, value_v); return -1; } } return 0; } -/* Load dictionary or prefix to decompression context */ -int -_PyZstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) +static int +_zstd_load_impl(ZstdDecompressor *self, ZstdDict *zd, + _zstd_state *mod_state, int type) { size_t zstd_ret; - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state == NULL) { - return -1; - } - ZstdDict *zd; - int type, ret; - - /* Check ZstdDict */ - ret = PyObject_IsInstance(dict, (PyObject*)mod_state->ZstdDict_type); - if (ret < 0) { - return -1; - } - else if (ret > 0) { - /* When decompressing, use digested dictionary by default. */ - zd = (ZstdDict*)dict; - type = DICT_TYPE_DIGESTED; - goto load; - } - - /* Check (ZstdDict, type) */ - if (PyTuple_CheckExact(dict) && PyTuple_GET_SIZE(dict) == 2) { - /* Check ZstdDict */ - ret = PyObject_IsInstance(PyTuple_GET_ITEM(dict, 0), - (PyObject*)mod_state->ZstdDict_type); - if (ret < 0) { - return -1; - } - else if (ret > 0) { - /* type == -1 may indicate an error. */ - type = PyLong_AsInt(PyTuple_GET_ITEM(dict, 1)); - if (type == DICT_TYPE_DIGESTED || - type == DICT_TYPE_UNDIGESTED || - type == DICT_TYPE_PREFIX) - { - assert(type >= 0); - zd = (ZstdDict*)PyTuple_GET_ITEM(dict, 0); - goto load; - } - } - } - - /* Wrong type */ - PyErr_SetString(PyExc_TypeError, - "zstd_dict argument should be ZstdDict object."); - return -1; - -load: if (type == DICT_TYPE_DIGESTED) { /* Get ZSTD_DDict */ ZSTD_DDict *d_dict = _get_DDict(zd); @@ -178,33 +145,20 @@ _PyZstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) return -1; } /* Reference a prepared dictionary */ - Py_BEGIN_CRITICAL_SECTION(self); zstd_ret = ZSTD_DCtx_refDDict(self->dctx, d_dict); - Py_END_CRITICAL_SECTION(); } else if (type == DICT_TYPE_UNDIGESTED) { /* Load a dictionary */ - Py_BEGIN_CRITICAL_SECTION2(self, zd); - zstd_ret = ZSTD_DCtx_loadDictionary( - self->dctx, - PyBytes_AS_STRING(zd->dict_content), - Py_SIZE(zd->dict_content)); - Py_END_CRITICAL_SECTION2(); + zstd_ret = ZSTD_DCtx_loadDictionary(self->dctx, zd->dict_buffer, + zd->dict_len); } else if (type == DICT_TYPE_PREFIX) { /* Load a prefix */ - Py_BEGIN_CRITICAL_SECTION2(self, zd); - zstd_ret = ZSTD_DCtx_refPrefix( - self->dctx, - PyBytes_AS_STRING(zd->dict_content), - Py_SIZE(zd->dict_content)); - Py_END_CRITICAL_SECTION2(); + zstd_ret = ZSTD_DCtx_refPrefix(self->dctx, zd->dict_buffer, + zd->dict_len); } else { - /* Impossible code path */ - PyErr_SetString(PyExc_SystemError, - "load_d_dict() impossible code path"); - return -1; + Py_UNREACHABLE(); } /* Check error */ @@ -215,22 +169,31 @@ _PyZstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) return 0; } - +/* Load dictionary or prefix to decompression context */ +static int +_zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) +{ + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); + /* When decompressing, use digested dictionary by default. */ + int type = DICT_TYPE_DIGESTED; + ZstdDict *zd = _Py_parse_zstd_dict(mod_state, dict, &type); + if (zd == NULL) { + return -1; + } + int ret; + PyMutex_Lock(&zd->lock); + ret = _zstd_load_impl(self, zd, mod_state, type); + PyMutex_Unlock(&zd->lock); + return ret; +} /* - Given the two types of decompressors (defined in _zstdmodule.h): - - typedef enum { - TYPE_DECOMPRESSOR, // <D>, ZstdDecompressor class - TYPE_ENDLESS_DECOMPRESSOR, // <E>, decompress() function - } decompress_type; - - Decompress implementation for <D>, <E>, pseudo code: + Decompress implementation in pseudo code: initialize_output_buffer while True: decompress_data - set_object_flag # .eof for <D>, .at_frame_edge for <E>. + set_object_flag # .eof if output_buffer_exhausted: if output_buffer_reached_max_length: @@ -240,76 +203,26 @@ _PyZstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) finish ZSTD_decompressStream()'s size_t return value: - - 0 when a frame is completely decoded and fully flushed, zstd's internal - buffer has no data. + - 0 when a frame is completely decoded and fully flushed, + zstd's internal buffer has no data. - An error code, which can be tested using ZSTD_isError(). - Or any other value > 0, which means there is still some decoding or flushing to do to complete current frame. Note, decompressing "an empty input" in any case will make it > 0. - - <E> supports multiple frames, has an .at_frame_edge flag, it means both the - input and output streams are at a frame edge. The flag can be set by this - statement: - - .at_frame_edge = (zstd_ret == 0) ? 1 : 0 - - But if decompressing "an empty input" at "a frame edge", zstd_ret will be - non-zero, then .at_frame_edge will be wrongly set to false. To solve this - problem, two AFE checks are needed to ensure that: when at "a frame edge", - empty input will not be decompressed. - - // AFE check - if (self->at_frame_edge && in->pos == in->size) { - finish - } - - In <E>, if .at_frame_edge is eventually set to true, but input stream has - unconsumed data (in->pos < in->size), then the outer function - stream_decompress() will set .at_frame_edge to false. In this case, - although the output stream is at a frame edge, for the caller, the input - stream is not at a frame edge, see below diagram. This behavior does not - affect the next AFE check, since (in->pos < in->size). - - input stream: --------------|--- - ^ - output stream: ====================| - ^ */ -PyObject * -decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in, - Py_ssize_t max_length, - Py_ssize_t initial_size, - decompress_type type) +static PyObject * +decompress_lock_held(ZstdDecompressor *self, ZSTD_inBuffer *in, + Py_ssize_t max_length) { size_t zstd_ret; ZSTD_outBuffer out; _BlocksOutputBuffer buffer = {.list = NULL}; PyObject *ret; - /* The first AFE check for setting .at_frame_edge flag */ - if (type == TYPE_ENDLESS_DECOMPRESSOR) { - if (self->at_frame_edge && in->pos == in->size) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state == NULL) { - return NULL; - } - ret = mod_state->empty_bytes; - Py_INCREF(ret); - return ret; - } - } - /* Initialize the output buffer */ - if (initial_size >= 0) { - if (_OutputBuffer_InitWithSize(&buffer, &out, max_length, initial_size) < 0) { - goto error; - } - } - else { - if (_OutputBuffer_InitAndGrow(&buffer, &out, max_length) < 0) { - goto error; - } + if (_OutputBuffer_InitAndGrow(&buffer, &out, max_length) < 0) { + goto error; } assert(out.pos == 0); @@ -321,33 +234,20 @@ decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in, /* Check error */ if (ZSTD_isError(zstd_ret)) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state != NULL) { - set_zstd_error(mod_state, ERR_DECOMPRESS, zstd_ret); - } + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); + set_zstd_error(mod_state, ERR_DECOMPRESS, zstd_ret); goto error; } - /* Set .eof/.af_frame_edge flag */ - if (type == TYPE_DECOMPRESSOR) { - /* ZstdDecompressor class stops when a frame is decompressed */ - if (zstd_ret == 0) { - self->eof = 1; - break; - } - } - else if (type == TYPE_ENDLESS_DECOMPRESSOR) { - /* decompress() function supports multiple frames */ - self->at_frame_edge = (zstd_ret == 0) ? 1 : 0; - - /* The second AFE check for setting .at_frame_edge flag */ - if (self->at_frame_edge && in->pos == in->size) { - break; - } + /* Set .eof flag */ + if (zstd_ret == 0) { + /* Stop when a frame is decompressed */ + self->eof = 1; + break; } /* Need to check out before in. Maybe zstd's internal buffer still has - a few bytes can be output, grow the buffer and continue. */ + a few bytes that can be output, grow the buffer and continue. */ if (out.pos == out.size) { /* Output buffer exhausted */ @@ -380,67 +280,40 @@ decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in, return NULL; } -void -decompressor_reset_session(ZstdDecompressor *self, - decompress_type type) +static void +decompressor_reset_session_lock_held(ZstdDecompressor *self) { - // TODO(emmatyping): use _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED here - // and ensure lock is always held + assert(PyMutex_IsLocked(&self->lock)); /* Reset variables */ self->in_begin = 0; self->in_end = 0; - if (type == TYPE_DECOMPRESSOR) { - Py_CLEAR(self->unused_data); - } + Py_CLEAR(self->unused_data); /* Reset variables in one operation */ self->needs_input = 1; - self->at_frame_edge = 1; self->eof = 0; - self->_unused_char_for_align = 0; - /* Resetting session never fail */ + /* Resetting session is guaranteed to never fail */ ZSTD_DCtx_reset(self->dctx, ZSTD_reset_session_only); } -PyObject * -stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length, - decompress_type type) +static PyObject * +stream_decompress_lock_held(ZstdDecompressor *self, Py_buffer *data, + Py_ssize_t max_length) { - Py_ssize_t initial_buffer_size = -1; + assert(PyMutex_IsLocked(&self->lock)); ZSTD_inBuffer in; PyObject *ret = NULL; int use_input_buffer; - if (type == TYPE_DECOMPRESSOR) { - /* Check .eof flag */ - if (self->eof) { - PyErr_SetString(PyExc_EOFError, "Already at the end of a zstd frame."); - assert(ret == NULL); - goto success; - } - } - else if (type == TYPE_ENDLESS_DECOMPRESSOR) { - /* Fast path for the first frame */ - if (self->at_frame_edge && self->in_begin == self->in_end) { - /* Read decompressed size */ - uint64_t decompressed_size = ZSTD_getFrameContentSize(data->buf, data->len); - - /* These two zstd constants always > PY_SSIZE_T_MAX: - ZSTD_CONTENTSIZE_UNKNOWN is (0ULL - 1) - ZSTD_CONTENTSIZE_ERROR is (0ULL - 2) - - Use ZSTD_findFrameCompressedSize() to check complete frame, - prevent allocating too much memory for small input chunk. */ - - if (decompressed_size <= (uint64_t) PY_SSIZE_T_MAX && - !ZSTD_isError(ZSTD_findFrameCompressedSize(data->buf, data->len)) ) - { - initial_buffer_size = (Py_ssize_t) decompressed_size; - } - } + /* Check .eof flag */ + if (self->eof) { + PyErr_SetString(PyExc_EOFError, + "Already at the end of a Zstandard frame."); + assert(ret == NULL); + return NULL; } /* Prepare input buffer w/wo unconsumed data */ @@ -527,30 +400,18 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length assert(in.pos == 0); /* Decompress */ - ret = decompress_impl(self, &in, - max_length, initial_buffer_size, - type); + ret = decompress_lock_held(self, &in, max_length); if (ret == NULL) { goto error; } /* Unconsumed input data */ if (in.pos == in.size) { - if (type == TYPE_DECOMPRESSOR) { - if (Py_SIZE(ret) == max_length || self->eof) { - self->needs_input = 0; - } - else { - self->needs_input = 1; - } + if (Py_SIZE(ret) == max_length || self->eof) { + self->needs_input = 0; } - else if (type == TYPE_ENDLESS_DECOMPRESSOR) { - if (Py_SIZE(ret) == max_length && !self->at_frame_edge) { - self->needs_input = 0; - } - else { - self->needs_input = 1; - } + else { + self->needs_input = 1; } if (use_input_buffer) { @@ -564,15 +425,11 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length self->needs_input = 0; - if (type == TYPE_ENDLESS_DECOMPRESSOR) { - self->at_frame_edge = 0; - } - if (!use_input_buffer) { /* Discard buffer if it's too small (resizing it may needlessly copy the current contents) */ - if (self->input_buffer != NULL && - self->input_buffer_size < data_size) + if (self->input_buffer != NULL + && self->input_buffer_size < data_size) { PyMem_Free(self->input_buffer); self->input_buffer = NULL; @@ -600,47 +457,57 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length } } - goto success; + return ret; error: /* Reset decompressor's states/session */ - decompressor_reset_session(self, type); + decompressor_reset_session_lock_held(self); Py_CLEAR(ret); -success: - - return ret; + return NULL; } +/*[clinic input] +@classmethod +_zstd.ZstdDecompressor.__new__ as _zstd_ZstdDecompressor_new + zstd_dict: object = None + A ZstdDict object, a pre-trained Zstandard dictionary. + options: object = None + A dict object that contains advanced decompression parameters. + +Create a decompressor object for decompressing data incrementally. + +Thread-safe at method level. For one-shot decompression, use the +decompress() function instead. +[clinic start generated code]*/ + static PyObject * -_zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +_zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, + PyObject *options) +/*[clinic end generated code: output=590ca65c1102ff4a input=73879de69bf89f59]*/ { - ZstdDecompressor *self; - self = PyObject_GC_New(ZstdDecompressor, type); + ZstdDecompressor* self = PyObject_GC_New(ZstdDecompressor, type); if (self == NULL) { goto error; } - self->inited = 0; - self->dict = NULL; self->input_buffer = NULL; self->input_buffer_size = 0; self->in_begin = -1; self->in_end = -1; self->unused_data = NULL; self->eof = 0; + self->dict = NULL; + self->lock = (PyMutex){0}; /* needs_input flag */ self->needs_input = 1; - /* at_frame_edge flag */ - self->at_frame_edge = 1; - /* Decompression context */ self->dctx = ZSTD_createDCtx(); if (self->dctx == NULL) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); + _zstd_state* mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state != NULL) { PyErr_SetString(mod_state->ZstdError, "Unable to create ZSTD_DCtx instance."); @@ -648,12 +515,29 @@ _zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwds) goto error; } + /* Load Zstandard dictionary to decompression context */ + if (zstd_dict != Py_None) { + if (_zstd_load_d_dict(self, zstd_dict) < 0) { + goto error; + } + Py_INCREF(zstd_dict); + self->dict = zstd_dict; + } + + /* Set options dictionary */ + if (options != Py_None) { + if (_zstd_set_d_parameters(self, options) < 0) { + goto error; + } + } + + // We can only start GC tracking once self->dict is set. + PyObject_GC_Track(self); + return (PyObject*)self; error: - if (self != NULL) { - PyObject_GC_Del(self); - } + Py_XDECREF(self); return NULL; } @@ -665,7 +549,11 @@ ZstdDecompressor_dealloc(PyObject *ob) PyObject_GC_UnTrack(self); /* Free decompression context */ - ZSTD_freeDCtx(self->dctx); + if (self->dctx) { + ZSTD_freeDCtx(self->dctx); + } + + assert(!PyMutex_IsLocked(&self->lock)); /* Py_CLEAR the dict after free decompression context */ Py_CLEAR(self->dict); @@ -682,87 +570,38 @@ ZstdDecompressor_dealloc(PyObject *ob) } /*[clinic input] -_zstd.ZstdDecompressor.__init__ - - zstd_dict: object = None - A ZstdDict object, a pre-trained zstd dictionary. - options: object = None - A dict object that contains advanced decompression parameters. - -Create a decompressor object for decompressing data incrementally. - -Thread-safe at method level. For one-shot decompression, use the decompress() -function instead. -[clinic start generated code]*/ - -static int -_zstd_ZstdDecompressor___init___impl(ZstdDecompressor *self, - PyObject *zstd_dict, PyObject *options) -/*[clinic end generated code: output=703af2f1ec226642 input=8fd72999acc1a146]*/ -{ - /* Only called once */ - if (self->inited) { - PyErr_SetString(PyExc_RuntimeError, init_twice_msg); - return -1; - } - self->inited = 1; - - /* Load dictionary to decompression context */ - if (zstd_dict != Py_None) { - if (_PyZstd_load_d_dict(self, zstd_dict) < 0) { - return -1; - } - - /* Py_INCREF the dict */ - Py_INCREF(zstd_dict); - self->dict = zstd_dict; - } - - /* Set option to decompression context */ - if (options != Py_None) { - if (_PyZstd_set_d_parameters(self, options) < 0) { - return -1; - } - } - - // We can only start tracking self with the GC once self->dict is set. - PyObject_GC_Track(self); - return 0; -} - -/*[clinic input] -@critical_section @getter _zstd.ZstdDecompressor.unused_data A bytes object of un-consumed input data. When ZstdDecompressor object stops after a frame is -decompressed, unused input data after the frame. Otherwise this will be b''. +decompressed, unused input data after the frame. Otherwise this +will be b''. [clinic start generated code]*/ static PyObject * _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self) -/*[clinic end generated code: output=f3a20940f11b6b09 input=5233800bef00df04]*/ +/*[clinic end generated code: output=f3a20940f11b6b09 input=0462065c5e60ba01]*/ { PyObject *ret; - /* Thread-safe code */ - Py_BEGIN_CRITICAL_SECTION(self); + PyMutex_Lock(&self->lock); if (!self->eof) { - _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); - if (mod_state == NULL) { - return NULL; - } - ret = mod_state->empty_bytes; - Py_INCREF(ret); + PyMutex_Unlock(&self->lock); + return Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); } else { if (self->unused_data == NULL) { - self->unused_data = PyBytes_FromStringAndSize( - self->input_buffer + self->in_begin, - self->in_end - self->in_begin); + if (self->input_buffer == NULL) { + self->unused_data = Py_GetConstant(Py_CONSTANT_EMPTY_BYTES); + } + else { + self->unused_data = PyBytes_FromStringAndSize( + self->input_buffer + self->in_begin, + self->in_end - self->in_begin); + } ret = self->unused_data; Py_XINCREF(ret); } @@ -772,8 +611,7 @@ _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self) } } - Py_END_CRITICAL_SECTION(); - + PyMutex_Unlock(&self->lock); return ret; } @@ -781,7 +619,7 @@ _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self) _zstd.ZstdDecompressor.decompress data: Py_buffer - A bytes-like object, zstd data to be decompressed. + A bytes-like object, Zstandard data to be decompressed. max_length: Py_ssize_t = -1 Maximum size of returned data. When it is negative, the size of output buffer is unlimited. When it is nonnegative, returns at @@ -789,43 +627,38 @@ _zstd.ZstdDecompressor.decompress Decompress *data*, returning uncompressed bytes if possible, or b'' otherwise. -If *max_length* is nonnegative, returns at most *max_length* bytes of -decompressed data. If this limit is reached and further output can be -produced, *self.needs_input* will be set to ``False``. In this case, the next -call to *decompress()* may provide *data* as b'' to obtain more of the output. +If *max_length* is nonnegative, returns at most *max_length* bytes +of decompressed data. If this limit is reached and further output +can be produced, *self.needs_input* will be set to ``False``. In +this case, the next call to *decompress()* may provide *data* as b'' +to obtain more of the output. -If all of the input data was decompressed and returned (either because this -was less than *max_length* bytes, or because *max_length* was negative), -*self.needs_input* will be set to True. +If all of the input data was decompressed and returned (either +because this was less than *max_length* bytes, or because +*max_length* was negative), *self.needs_input* will be set to True. -Attempting to decompress data after the end of a frame is reached raises an -EOFError. Any data found after the end of the frame is ignored and saved in -the self.unused_data attribute. +Attempting to decompress data after the end of a frame is reached +raises an EOFError. Any data found after the end of the frame is +ignored and saved in the self.unused_data attribute. [clinic start generated code]*/ static PyObject * _zstd_ZstdDecompressor_decompress_impl(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=a4302b3c940dbec6 input=830e455bc9a50b6e]*/ +/*[clinic end generated code: output=a4302b3c940dbec6 input=b55991a007f4c558]*/ { PyObject *ret; /* Thread-safe code */ - Py_BEGIN_CRITICAL_SECTION(self); - - ret = stream_decompress(self, data, max_length, TYPE_DECOMPRESSOR); - Py_END_CRITICAL_SECTION(); + PyMutex_Lock(&self->lock); + ret = stream_decompress_lock_held(self, data, max_length); + PyMutex_Unlock(&self->lock); return ret; } -#define clinic_state() (get_zstd_state_from_type(type)) -#include "clinic/decompressor.c.h" -#undef clinic_state - static PyMethodDef ZstdDecompressor_methods[] = { _ZSTD_ZSTDDECOMPRESSOR_DECOMPRESS_METHODDEF - - {0} + {NULL, NULL} }; PyDoc_STRVAR(ZstdDecompressor_eof_doc, @@ -833,24 +666,22 @@ PyDoc_STRVAR(ZstdDecompressor_eof_doc, "after that, an EOFError exception will be raised."); PyDoc_STRVAR(ZstdDecompressor_needs_input_doc, -"If the max_length output limit in .decompress() method has been reached, and\n" -"the decompressor has (or may has) unconsumed input data, it will be set to\n" -"False. In this case, pass b'' to .decompress() method may output further data."); +"If the max_length output limit in .decompress() method has been reached,\n" +"and the decompressor has (or may has) unconsumed input data, it will be set\n" +"to False. In this case, passing b'' to the .decompress() method may output\n" +"further data."); static PyMemberDef ZstdDecompressor_members[] = { {"eof", Py_T_BOOL, offsetof(ZstdDecompressor, eof), Py_READONLY, ZstdDecompressor_eof_doc}, - {"needs_input", Py_T_BOOL, offsetof(ZstdDecompressor, needs_input), Py_READONLY, ZstdDecompressor_needs_input_doc}, - - {0} + {NULL} }; static PyGetSetDef ZstdDecompressor_getset[] = { _ZSTD_ZSTDDECOMPRESSOR_UNUSED_DATA_GETSETDEF - - {0} + {NULL} }; static int @@ -873,19 +704,19 @@ ZstdDecompressor_clear(PyObject *ob) static PyType_Slot ZstdDecompressor_slots[] = { {Py_tp_new, _zstd_ZstdDecompressor_new}, {Py_tp_dealloc, ZstdDecompressor_dealloc}, - {Py_tp_init, _zstd_ZstdDecompressor___init__}, {Py_tp_methods, ZstdDecompressor_methods}, {Py_tp_members, ZstdDecompressor_members}, {Py_tp_getset, ZstdDecompressor_getset}, - {Py_tp_doc, (char*)_zstd_ZstdDecompressor___init____doc__}, + {Py_tp_doc, (void *)_zstd_ZstdDecompressor_new__doc__}, {Py_tp_traverse, ZstdDecompressor_traverse}, {Py_tp_clear, ZstdDecompressor_clear}, - {0} + {0, 0} }; -PyType_Spec ZstdDecompressor_type_spec = { - .name = "_zstd.ZstdDecompressor", +PyType_Spec zstd_decompressor_type_spec = { + .name = "compression.zstd.ZstdDecompressor", .basicsize = sizeof(ZstdDecompressor), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HAVE_GC, .slots = ZstdDecompressor_slots, }; diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index a19224c4a6403b3..e1b9d998e697fb8 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -1,38 +1,67 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorithm & the zstd library. */ /* ZstdDict class definitions */ /*[clinic input] module _zstd -class _zstd.ZstdDict "ZstdDict *" "clinic_state()->ZstdDict_type" +class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec" [clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=a5d1254c497e52ba]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=3dcc175ec974f81c]*/ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 #endif +#include "Python.h" + #include "_zstdmodule.h" +#include "clinic/zstddict.c.h" +#include "internal/pycore_lock.h" // PyMutex_IsLocked -#include <stddef.h> // offsetof() +#include <zstd.h> // ZSTD_freeDDict(), ZSTD_getDictID_fromDict() #define ZstdDict_CAST(op) ((ZstdDict *)op) +/*[clinic input] +@classmethod +_zstd.ZstdDict.__new__ as _zstd_ZstdDict_new + dict_content: Py_buffer + The content of a Zstandard dictionary as a bytes-like object. + / + * + is_raw: bool = False + If true, perform no checks on *dict_content*, useful for some + advanced cases. Otherwise, check that the content represents + a Zstandard dictionary created by the zstd library or CLI. + +Represents a Zstandard dictionary. + +The dictionary can be used for compression or decompression, and can be +shared by multiple ZstdCompressor or ZstdDecompressor objects. +[clinic start generated code]*/ + static PyObject * -_zstd_ZstdDict_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwargs)) +_zstd_ZstdDict_new_impl(PyTypeObject *type, Py_buffer *dict_content, + int is_raw) +/*[clinic end generated code: output=685b7406a48b0949 input=3bb66063c0240433]*/ { - ZstdDict *self; - self = PyObject_GC_New(ZstdDict, type); + /* All dictionaries must be at least 8 bytes */ + if (dict_content->len < 8) { + PyErr_SetString(PyExc_ValueError, + "Zstandard dictionary content too short " + "(must have at least eight bytes)"); + return NULL; + } + + ZstdDict* self = PyObject_GC_New(ZstdDict, type); if (self == NULL) { - goto error; + return NULL; } - self->dict_content = NULL; - self->inited = 0; self->d_dict = NULL; + self->dict_buffer = NULL; + self->dict_id = 0; + self->lock = (PyMutex){0}; /* ZSTD_CDict dict */ self->c_dicts = PyDict_New(); @@ -40,12 +69,29 @@ _zstd_ZstdDict_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_U goto error; } - return (PyObject*)self; + self->dict_buffer = PyMem_Malloc(dict_content->len); + if (!self->dict_buffer) { + PyErr_NoMemory(); + goto error; + } + memcpy(self->dict_buffer, dict_content->buf, dict_content->len); + self->dict_len = dict_content->len; -error: - if (self != NULL) { - PyObject_GC_Del(self); + /* Get dict_id, 0 means "raw content" dictionary. */ + self->dict_id = ZSTD_getDictID_fromDict(self->dict_buffer, self->dict_len); + + /* Check validity for ordinary dictionary */ + if (!is_raw && self->dict_id == 0) { + PyErr_SetString(PyExc_ValueError, "invalid Zstandard dictionary"); + goto error; } + + PyObject_GC_Track(self); + + return (PyObject *)self; + +error: + Py_XDECREF(self); return NULL; } @@ -57,194 +103,138 @@ ZstdDict_dealloc(PyObject *ob) PyObject_GC_UnTrack(self); /* Free ZSTD_DDict instance */ - ZSTD_freeDDict(self->d_dict); + if (self->d_dict) { + ZSTD_freeDDict(self->d_dict); + } + + assert(!PyMutex_IsLocked(&self->lock)); - /* Release dict_content after Free ZSTD_CDict/ZSTD_DDict instances */ - Py_CLEAR(self->dict_content); + /* Release dict_buffer after freeing ZSTD_CDict/ZSTD_DDict instances */ + PyMem_Free(self->dict_buffer); Py_CLEAR(self->c_dicts); PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_Del(ob); + tp->tp_free(self); Py_DECREF(tp); } -/*[clinic input] -_zstd.ZstdDict.__init__ - - dict_content: object - A bytes-like object, dictionary's content. - is_raw: bool = False - This parameter is for advanced user. True means dict_content - argument is a "raw content" dictionary, free of any format - restriction. False means dict_content argument is an ordinary - zstd dictionary, was created by zstd functions, follow a - specified format. - -Represents a zstd dictionary, which can be used for compression/decompression. - -It's thread-safe, and can be shared by multiple ZstdCompressor / -ZstdDecompressor objects. -[clinic start generated code]*/ - -static int -_zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content, - int is_raw) -/*[clinic end generated code: output=c5f5a0d8377d037c input=e6750f62a513b3ee]*/ -{ - /* Only called once */ - if (self->inited) { - PyErr_SetString(PyExc_RuntimeError, init_twice_msg); - return -1; - } - self->inited = 1; - - /* Check dict_content's type */ - self->dict_content = PyBytes_FromObject(dict_content); - if (self->dict_content == NULL) { - PyErr_SetString(PyExc_TypeError, - "dict_content argument should be bytes-like object."); - return -1; - } - - /* Both ordinary dictionary and "raw content" dictionary should - at least 8 bytes */ - if (Py_SIZE(self->dict_content) < 8) { - PyErr_SetString(PyExc_ValueError, - "Zstd dictionary content should at least 8 bytes."); - return -1; - } - - /* Get dict_id, 0 means "raw content" dictionary. */ - self->dict_id = ZSTD_getDictID_fromDict(PyBytes_AS_STRING(self->dict_content), - Py_SIZE(self->dict_content)); - - /* Check validity for ordinary dictionary */ - if (!is_raw && self->dict_id == 0) { - char *msg = "The dict_content argument is not a valid zstd " - "dictionary. The first 4 bytes of a valid zstd dictionary " - "should be a magic number: b'\\x37\\xA4\\x30\\xEC'.\n" - "If you are an advanced user, and can be sure that " - "dict_content argument is a \"raw content\" zstd " - "dictionary, set is_raw parameter to True."; - PyErr_SetString(PyExc_ValueError, msg); - return -1; - } - - // Can only track self once self->dict_content is included - PyObject_GC_Track(self); - return 0; -} - -#define clinic_state() (get_zstd_state(type)) -#include "clinic/zstddict.c.h" -#undef clinic_state - PyDoc_STRVAR(ZstdDict_dictid_doc, -"ID of zstd dictionary, a 32-bit unsigned int value.\n\n" -"Non-zero means ordinary dictionary, was created by zstd functions, follow\n" -"a specified format.\n\n" -"0 means a \"raw content\" dictionary, free of any format restriction, used\n" -"for advanced user."); - -PyDoc_STRVAR(ZstdDict_dictcontent_doc, -"The content of zstd dictionary, a bytes object, it's the same as dict_content\n" -"argument in ZstdDict.__init__() method. It can be used with other programs."); +"The Zstandard dictionary, an int between 0 and 2**32.\n\n" +"A non-zero value represents an ordinary Zstandard dictionary,\n" +"conforming to the standardised format.\n\n" +"A value of zero indicates a 'raw content' dictionary,\n" +"without any restrictions on format or content."); static PyObject * -ZstdDict_str(PyObject *ob) +ZstdDict_repr(PyObject *ob) { ZstdDict *dict = ZstdDict_CAST(ob); return PyUnicode_FromFormat("<ZstdDict dict_id=%u dict_size=%zd>", - dict->dict_id, Py_SIZE(dict->dict_content)); + (unsigned int)dict->dict_id, dict->dict_len); } static PyMemberDef ZstdDict_members[] = { {"dict_id", Py_T_UINT, offsetof(ZstdDict, dict_id), Py_READONLY, ZstdDict_dictid_doc}, - {"dict_content", Py_T_OBJECT_EX, offsetof(ZstdDict, dict_content), Py_READONLY, ZstdDict_dictcontent_doc}, - {0} + {NULL} }; /*[clinic input] -@critical_section +@getter +_zstd.ZstdDict.dict_content + +The content of a Zstandard dictionary, as a bytes object. +[clinic start generated code]*/ + +static PyObject * +_zstd_ZstdDict_dict_content_get_impl(ZstdDict *self) +/*[clinic end generated code: output=0d05caa5b550eabb input=4ed526d1c151c596]*/ +{ + return PyBytes_FromStringAndSize(self->dict_buffer, self->dict_len); +} + +/*[clinic input] @getter _zstd.ZstdDict.as_digested_dict Load as a digested dictionary to compressor. -Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_digested_dict) -1. Some advanced compression parameters of compressor may be overridden - by parameters of digested dictionary. -2. ZstdDict has a digested dictionaries cache for each compression level. - It's faster when loading again a digested dictionary with the same - compression level. +Pass this attribute as zstd_dict argument: +compress(dat, zstd_dict=zd.as_digested_dict) + +1. Some advanced compression parameters of compressor may be + overridden by parameters of digested dictionary. +2. ZstdDict has a digested dictionaries cache for each compression + level. It's faster when loading again a digested dictionary with + the same compression level. 3. No need to use this for decompression. [clinic start generated code]*/ static PyObject * _zstd_ZstdDict_as_digested_dict_get_impl(ZstdDict *self) -/*[clinic end generated code: output=09b086e7a7320dbb input=585448c79f31f74a]*/ +/*[clinic end generated code: output=09b086e7a7320dbb input=a9417d40f1d7fedd]*/ { return Py_BuildValue("Oi", self, DICT_TYPE_DIGESTED); } /*[clinic input] -@critical_section @getter _zstd.ZstdDict.as_undigested_dict Load as an undigested dictionary to compressor. -Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_undigested_dict) -1. The advanced compression parameters of compressor will not be overridden. -2. Loading an undigested dictionary is costly. If load an undigested dictionary - multiple times, consider reusing a compressor object. +Pass this attribute as zstd_dict argument: +compress(dat, zstd_dict=zd.as_undigested_dict) + +1. The advanced compression parameters of compressor will not be + overridden. +2. Loading an undigested dictionary is costly. If load an undigested + dictionary multiple times, consider reusing a compressor object. 3. No need to use this for decompression. [clinic start generated code]*/ static PyObject * _zstd_ZstdDict_as_undigested_dict_get_impl(ZstdDict *self) -/*[clinic end generated code: output=43c7a989e6d4253a input=022b0829ffb1c220]*/ +/*[clinic end generated code: output=43c7a989e6d4253a input=56443c9c4e589cd5]*/ { return Py_BuildValue("Oi", self, DICT_TYPE_UNDIGESTED); } /*[clinic input] -@critical_section @getter _zstd.ZstdDict.as_prefix Load as a prefix to compressor/decompressor. -Pass this attribute as zstd_dict argument: compress(dat, zstd_dict=zd.as_prefix) -1. Prefix is compatible with long distance matching, while dictionary is not. -2. It only works for the first frame, then the compressor/decompressor will - return to no prefix state. -3. When decompressing, must use the same prefix as when compressing." +Pass this attribute as zstd_dict argument: +compress(dat, zstd_dict=zd.as_prefix) + +1. Prefix is compatible with long distance matching, while + dictionary is not. +2. It only works for the first frame, then the + compressor/decompressor will return to no prefix state. +3. When decompressing, must use the same prefix as when compressing. [clinic start generated code]*/ static PyObject * _zstd_ZstdDict_as_prefix_get_impl(ZstdDict *self) -/*[clinic end generated code: output=6f7130c356595a16 input=09fb82a6a5407e87]*/ +/*[clinic end generated code: output=6f7130c356595a16 input=192681a899c6fad0]*/ { return Py_BuildValue("Oi", self, DICT_TYPE_PREFIX); } static PyGetSetDef ZstdDict_getset[] = { + _ZSTD_ZSTDDICT_DICT_CONTENT_GETSETDEF _ZSTD_ZSTDDICT_AS_DIGESTED_DICT_GETSETDEF - _ZSTD_ZSTDDICT_AS_UNDIGESTED_DICT_GETSETDEF - _ZSTD_ZSTDDICT_AS_PREFIX_GETSETDEF - - {0} + {NULL} }; static Py_ssize_t ZstdDict_length(PyObject *ob) { ZstdDict *self = ZstdDict_CAST(ob); - assert(PyBytes_Check(self->dict_content)); - return Py_SIZE(self->dict_content); + return self->dict_len; } static int @@ -252,7 +242,6 @@ ZstdDict_traverse(PyObject *ob, visitproc visit, void *arg) { ZstdDict *self = ZstdDict_CAST(ob); Py_VISIT(self->c_dicts); - Py_VISIT(self->dict_content); return 0; } @@ -260,7 +249,7 @@ static int ZstdDict_clear(PyObject *ob) { ZstdDict *self = ZstdDict_CAST(ob); - Py_CLEAR(self->dict_content); + Py_CLEAR(self->c_dicts); return 0; } @@ -269,18 +258,18 @@ static PyType_Slot zstddict_slots[] = { {Py_tp_getset, ZstdDict_getset}, {Py_tp_new, _zstd_ZstdDict_new}, {Py_tp_dealloc, ZstdDict_dealloc}, - {Py_tp_init, _zstd_ZstdDict___init__}, - {Py_tp_str, ZstdDict_str}, - {Py_tp_doc, (char*)_zstd_ZstdDict___init____doc__}, + {Py_tp_repr, ZstdDict_repr}, + {Py_tp_doc, (void *)_zstd_ZstdDict_new__doc__}, {Py_sq_length, ZstdDict_length}, {Py_tp_traverse, ZstdDict_traverse}, {Py_tp_clear, ZstdDict_clear}, - {0} + {0, 0} }; -PyType_Spec zstddict_type_spec = { - .name = "_zstd.ZstdDict", +PyType_Spec zstd_dict_type_spec = { + .name = "compression.zstd.ZstdDict", .basicsize = sizeof(ZstdDict), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HAVE_GC, .slots = zstddict_slots, }; diff --git a/Modules/_zstd/zstddict.h b/Modules/_zstd/zstddict.h new file mode 100644 index 000000000000000..e0d3f46b2b14a67 --- /dev/null +++ b/Modules/_zstd/zstddict.h @@ -0,0 +1,29 @@ +/* Low level interface to the Zstandard algorithm & the zstd library. */ + +#ifndef ZSTD_DICT_H +#define ZSTD_DICT_H + +#include <zstd.h> // ZSTD_DDict + +typedef struct { + PyObject_HEAD + + /* Reusable compress/decompress dictionary, they are created once and + can be shared by multiple threads concurrently, since its usage is + read-only. + c_dicts is a dict, int(compressionLevel):PyCapsule(ZSTD_CDict*) */ + ZSTD_DDict *d_dict; + PyObject *c_dicts; + + /* Dictionary content. */ + char *dict_buffer; + Py_ssize_t dict_len; + + /* Dictionary id */ + uint32_t dict_id; + + /* Lock to protect the digested dictionaries */ + PyMutex lock; +} ZstdDict; + +#endif // !ZSTD_DICT_H diff --git a/Modules/arraymodule.c b/Modules/arraymodule.c index 401a3a7072b8460..758a06f0e2b51ca 100644 --- a/Modules/arraymodule.c +++ b/Modules/arraymodule.c @@ -13,6 +13,8 @@ #include "pycore_ceval.h" // _PyEval_GetBuiltin() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_moduleobject.h" // _PyModule_GetState() +#include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> // offsetof() #include <stdbool.h> @@ -204,6 +206,33 @@ Note that the basic Get and Set functions do NOT check that the index is in bounds; that's the responsibility of the caller. ****************************************************************************/ +/* Macro to check array buffer validity and bounds after calling + user-defined methods (like __index__ or __float__) that might modify + the array during the call. +*/ +#define CHECK_ARRAY_BOUNDS(OP, IDX) \ + do { \ + if ((IDX) >= 0 && ((OP)->ob_item == NULL || \ + (IDX) >= Py_SIZE((OP)))) { \ + PyErr_SetString(PyExc_IndexError, \ + "array assignment index out of range"); \ + return -1; \ + } \ + } while (0) + +#define CHECK_ARRAY_BOUNDS_WITH_CLEANUP(OP, IDX, VAL, CLEANUP) \ + do { \ + if ((IDX) >= 0 && ((OP)->ob_item == NULL || \ + (IDX) >= Py_SIZE((OP)))) { \ + PyErr_SetString(PyExc_IndexError, \ + "array assignment index out of range"); \ + if (CLEANUP) { \ + Py_DECREF(VAL); \ + } \ + return -1; \ + } \ + } while (0) + static PyObject * b_getitem(arrayobject *ap, Py_ssize_t i) { @@ -220,7 +249,10 @@ b_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) the overflow checking */ if (!PyArg_Parse(v, "h;array item must be integer", &x)) return -1; - else if (x < -128) { + + CHECK_ARRAY_BOUNDS(ap, i); + + if (x < -128) { PyErr_SetString(PyExc_OverflowError, "signed char is less than minimum"); return -1; @@ -249,6 +281,9 @@ BB_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) /* 'B' == unsigned char, maps to PyArg_Parse's 'b' formatter */ if (!PyArg_Parse(v, "b;array item must be integer", &x)) return -1; + + CHECK_ARRAY_BOUNDS(ap, i); + if (i >= 0) ((unsigned char *)ap->ob_item)[i] = x; return 0; @@ -341,6 +376,9 @@ h_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) /* 'h' == signed short, maps to PyArg_Parse's 'h' formatter */ if (!PyArg_Parse(v, "h;array item must be integer", &x)) return -1; + + CHECK_ARRAY_BOUNDS(ap, i); + if (i >= 0) ((short *)ap->ob_item)[i] = x; return 0; @@ -370,6 +408,9 @@ HH_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) "unsigned short is greater than maximum"); return -1; } + + CHECK_ARRAY_BOUNDS(ap, i); + if (i >= 0) ((short *)ap->ob_item)[i] = (short)x; return 0; @@ -388,6 +429,9 @@ i_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) /* 'i' == signed int, maps to PyArg_Parse's 'i' formatter */ if (!PyArg_Parse(v, "i;array item must be integer", &x)) return -1; + + CHECK_ARRAY_BOUNDS(ap, i); + if (i >= 0) ((int *)ap->ob_item)[i] = x; return 0; @@ -428,6 +472,9 @@ II_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) } return -1; } + + CHECK_ARRAY_BOUNDS_WITH_CLEANUP(ap, i, v, do_decref); + if (i >= 0) ((unsigned int *)ap->ob_item)[i] = (unsigned int)x; @@ -449,6 +496,9 @@ l_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) long x; if (!PyArg_Parse(v, "l;array item must be integer", &x)) return -1; + + CHECK_ARRAY_BOUNDS(ap, i); + if (i >= 0) ((long *)ap->ob_item)[i] = x; return 0; @@ -480,6 +530,9 @@ LL_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) } return -1; } + + CHECK_ARRAY_BOUNDS_WITH_CLEANUP(ap, i, v, do_decref); + if (i >= 0) ((unsigned long *)ap->ob_item)[i] = x; @@ -501,6 +554,9 @@ q_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) long long x; if (!PyArg_Parse(v, "L;array item must be integer", &x)) return -1; + + CHECK_ARRAY_BOUNDS(ap, i); + if (i >= 0) ((long long *)ap->ob_item)[i] = x; return 0; @@ -533,6 +589,9 @@ QQ_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) } return -1; } + + CHECK_ARRAY_BOUNDS_WITH_CLEANUP(ap, i, v, do_decref); + if (i >= 0) ((unsigned long long *)ap->ob_item)[i] = x; @@ -554,6 +613,9 @@ f_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) float x; if (!PyArg_Parse(v, "f;array item must be float", &x)) return -1; + + CHECK_ARRAY_BOUNDS(ap, i); + if (i >= 0) ((float *)ap->ob_item)[i] = x; return 0; @@ -571,6 +633,9 @@ d_setitem(arrayobject *ap, Py_ssize_t i, PyObject *v) double x; if (!PyArg_Parse(v, "d;array item must be float", &x)) return -1; + + CHECK_ARRAY_BOUNDS(ap, i); + if (i >= 0) ((double *)ap->ob_item)[i] = x; return 0; @@ -632,7 +697,8 @@ class array.array "arrayobject *" "ArrayType" /*[clinic end generated code: output=da39a3ee5e6b4b0d input=a5c29edf59f176a3]*/ static PyObject * -newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *descr) +newarrayobject_untracked(PyTypeObject *type, Py_ssize_t size, + const struct arraydescr *descr) { arrayobject *op; size_t nbytes; @@ -647,7 +713,7 @@ newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *des return PyErr_NoMemory(); } nbytes = size * descr->itemsize; - op = (arrayobject *) type->tp_alloc(type, 0); + op = (arrayobject *) _PyType_AllocNoTrack(type, 0); if (op == NULL) { return NULL; } @@ -669,6 +735,16 @@ newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *des return (PyObject *) op; } +static PyObject * +newarrayobject(PyTypeObject *type, Py_ssize_t size, const struct arraydescr *descr) +{ + PyObject *op = newarrayobject_untracked(type, size, descr); + if (op != NULL) { + _PyObject_GC_TRACK(op); + } + return op; +} + static PyObject * getarrayitem(PyObject *op, Py_ssize_t i) { @@ -728,9 +804,7 @@ array_dealloc(PyObject *op) PyObject_GC_UnTrack(op); arrayobject *self = arrayobject_CAST(op); - if (self->weakreflist != NULL) { - PyObject_ClearWeakRefs(op); - } + FT_CLEAR_WEAKREFS(op, self->weakreflist); if (self->ob_item != NULL) { PyMem_Free(self->ob_item); } @@ -880,13 +954,14 @@ array_slice(arrayobject *a, Py_ssize_t ilow, Py_ssize_t ihigh) ihigh = ilow; else if (ihigh > Py_SIZE(a)) ihigh = Py_SIZE(a); - np = (arrayobject *) newarrayobject(state->ArrayType, ihigh - ilow, a->ob_descr); + np = (arrayobject *) newarrayobject_untracked(state->ArrayType, ihigh - ilow, a->ob_descr); if (np == NULL) return NULL; if (ihigh > ilow) { memcpy(np->ob_item, a->ob_item + ilow * a->ob_descr->itemsize, (ihigh-ilow) * a->ob_descr->itemsize); } + _PyObject_GC_TRACK(np); return (PyObject *)np; } @@ -957,7 +1032,7 @@ array_concat(PyObject *op, PyObject *bb) return PyErr_NoMemory(); } size = Py_SIZE(a) + Py_SIZE(b); - np = (arrayobject *) newarrayobject(state->ArrayType, size, a->ob_descr); + np = (arrayobject *) newarrayobject_untracked(state->ArrayType, size, a->ob_descr); if (np == NULL) { return NULL; } @@ -968,6 +1043,7 @@ array_concat(PyObject *op, PyObject *bb) memcpy(np->ob_item + Py_SIZE(a)*a->ob_descr->itemsize, b->ob_item, Py_SIZE(b)*b->ob_descr->itemsize); } + _PyObject_GC_TRACK(np); return (PyObject *)np; #undef b } @@ -985,16 +1061,19 @@ array_repeat(PyObject *op, Py_ssize_t n) return PyErr_NoMemory(); } Py_ssize_t size = array_length * n; - arrayobject* np = (arrayobject *) newarrayobject(state->ArrayType, size, a->ob_descr); + arrayobject* np = (arrayobject *) newarrayobject_untracked(state->ArrayType, size, a->ob_descr); if (np == NULL) return NULL; - if (size == 0) + if (size == 0) { + _PyObject_GC_TRACK(np); return (PyObject *)np; + } const Py_ssize_t oldbytes = array_length * a->ob_descr->itemsize; const Py_ssize_t newbytes = oldbytes * n; _PyBytes_Repeat(np->ob_item, newbytes, a->ob_item, oldbytes); + _PyObject_GC_TRACK(np); return (PyObject *)np; } @@ -1379,13 +1458,13 @@ array.array.buffer_info Return a tuple (address, length) giving the current memory address and the length in items of the buffer used to hold array's contents. -The length should be multiplied by the itemsize attribute to calculate -the buffer length in bytes. +The length should be multiplied by the itemsize attribute to +calculate the buffer length in bytes. [clinic start generated code]*/ static PyObject * array_array_buffer_info_impl(arrayobject *self) -/*[clinic end generated code: output=9b2a4ec3ae7e98e7 input=a58bae5c6e1ac6a6]*/ +/*[clinic end generated code: output=9b2a4ec3ae7e98e7 input=f4d34398d5dfc856]*/ { PyObject *retval = NULL, *v; @@ -1431,13 +1510,13 @@ array.array.byteswap Byteswap all items of the array. -If the items in the array are not 1, 2, 4, or 8 bytes in size, RuntimeError is -raised. +If the items in the array are not 1, 2, 4, or 8 bytes in size, +RuntimeError is raised. [clinic start generated code]*/ static PyObject * array_array_byteswap_impl(arrayobject *self) -/*[clinic end generated code: output=5f8236cbdf0d90b5 input=6a85591b950a0186]*/ +/*[clinic end generated code: output=5f8236cbdf0d90b5 input=3005a63cc263b839]*/ { char *p; Py_ssize_t i; @@ -1781,14 +1860,14 @@ array.array.fromunicode Extends this array with data from the unicode string ustr. -The array must be a unicode type array; otherwise a ValueError is raised. -Use array.frombytes(ustr.encode(...)) to append Unicode data to an array of -some other type. +The array must be a unicode type array; otherwise a ValueError is +raised. Use array.frombytes(ustr.encode(...)) to append Unicode +data to an array of some other type. [clinic start generated code]*/ static PyObject * array_array_fromunicode_impl(arrayobject *self, PyObject *ustr) -/*[clinic end generated code: output=24359f5e001a7f2b input=025db1fdade7a4ce]*/ +/*[clinic end generated code: output=24359f5e001a7f2b input=01fa592ec7b948b6]*/ { int typecode = self->ob_descr->typecode; if (typecode != 'u' && typecode != 'w') { @@ -1840,14 +1919,15 @@ array.array.tounicode Extends this array with data from the unicode string ustr. -Convert the array to a unicode string. The array must be a unicode type array; -otherwise a ValueError is raised. Use array.tobytes().decode() to obtain a -unicode string from an array of some other type. +Convert the array to a unicode string. The array must be a unicode +type array; otherwise a ValueError is raised. Use +array.tobytes().decode() to obtain a unicode string from an array of +some other type. [clinic start generated code]*/ static PyObject * array_array_tounicode_impl(arrayobject *self) -/*[clinic end generated code: output=08e442378336e1ef input=127242eebe70b66d]*/ +/*[clinic end generated code: output=08e442378336e1ef input=d4d5f398aa71a2be]*/ { int typecode = self->ob_descr->typecode; if (typecode != 'u' && typecode != 'w') { @@ -2406,7 +2486,8 @@ static PyMethodDef array_methods[] = { ARRAY_ARRAY_TOBYTES_METHODDEF ARRAY_ARRAY_TOUNICODE_METHODDEF ARRAY_ARRAY___SIZEOF___METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("Arrays are generic over the type of their elements")}, {NULL, NULL} /* sentinel */ }; @@ -2470,17 +2551,17 @@ array_subscr(PyObject *op, PyObject *item) return newarrayobject(state->ArrayType, 0, self->ob_descr); } else if (step == 1) { - PyObject *result = newarrayobject(state->ArrayType, - slicelength, self->ob_descr); + PyObject *result = newarrayobject_untracked(state->ArrayType, slicelength, self->ob_descr); if (result == NULL) return NULL; memcpy(((arrayobject *)result)->ob_item, self->ob_item + start * itemsize, slicelength * itemsize); + _PyObject_GC_TRACK(result); return result; } else { - result = newarrayobject(state->ArrayType, slicelength, self->ob_descr); + result = newarrayobject_untracked(state->ArrayType, slicelength, self->ob_descr); if (!result) return NULL; ar = (arrayobject*)result; @@ -2492,6 +2573,7 @@ array_subscr(PyObject *op, PyObject *item) itemsize); } + _PyObject_GC_TRACK(result); return result; } } @@ -2791,9 +2873,11 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) else len = 0; - a = newarrayobject(type, len, descr); - if (a == NULL) + a = newarrayobject_untracked(type, len, descr); + if (a == NULL) { + Py_XDECREF(it); return NULL; + } if (len > 0 && !array_Check(initial, state)) { Py_ssize_t i; @@ -2802,11 +2886,13 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PySequence_GetItem(initial, i); if (v == NULL) { Py_DECREF(a); + Py_XDECREF(it); return NULL; } if (setarrayitem(a, i, v) != 0) { Py_DECREF(v); Py_DECREF(a); + Py_XDECREF(it); return NULL; } Py_DECREF(v); @@ -2818,6 +2904,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) v = array_array_frombytes((PyObject *)a, initial); if (v == NULL) { Py_DECREF(a); + Py_XDECREF(it); return NULL; } Py_DECREF(v); @@ -2828,6 +2915,7 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) wchar_t *ustr = PyUnicode_AsWideCharString(initial, &n); if (ustr == NULL) { Py_DECREF(a); + Py_XDECREF(it); return NULL; } @@ -2839,12 +2927,16 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_SET_SIZE(self, n); self->allocated = n; } + else { + PyMem_Free(ustr); + } } else { // c == 'w' Py_ssize_t n = PyUnicode_GET_LENGTH(initial); Py_UCS4 *ustr = PyUnicode_AsUCS4Copy(initial); if (ustr == NULL) { Py_DECREF(a); + Py_XDECREF(it); return NULL; } @@ -2869,9 +2961,12 @@ array_new(PyTypeObject *type, PyObject *args, PyObject *kwds) } Py_DECREF(it); } + // Track only once fully built. + _PyObject_GC_TRACK(a); return a; } } + Py_XDECREF(it); PyErr_SetString(PyExc_ValueError, "bad typecode (must be b, B, u, w, h, H, i, I, l, L, q, Q, f or d)"); return NULL; @@ -2960,7 +3055,7 @@ static PyType_Slot array_slots[] = { {Py_tp_methods, array_methods}, {Py_tp_members, array_members}, {Py_tp_getset, array_getsets}, - {Py_tp_alloc, PyType_GenericAlloc}, + {Py_tp_alloc, _PyType_AllocNoTrack}, {Py_tp_new, array_new}, {Py_tp_traverse, array_tp_traverse}, @@ -3220,6 +3315,7 @@ array_modexec(PyObject *m) CREATE_TYPE(m, state->ArrayIterType, &arrayiter_spec); Py_SET_TYPE(state->ArrayIterType, &PyType_Type); + // Older undocumented alias: if (PyModule_AddObjectRef(m, "ArrayType", (PyObject *)state->ArrayType) < 0) { return -1; @@ -3228,14 +3324,12 @@ array_modexec(PyObject *m) PyObject *mutablesequence = PyImport_ImportModuleAttrString( "collections.abc", "MutableSequence"); if (!mutablesequence) { - Py_DECREF((PyObject *)state->ArrayType); return -1; } PyObject *res = PyObject_CallMethod(mutablesequence, "register", "O", (PyObject *)state->ArrayType); Py_DECREF(mutablesequence); if (!res) { - Py_DECREF((PyObject *)state->ArrayType); return -1; } Py_DECREF(res); diff --git a/Modules/atexitmodule.c b/Modules/atexitmodule.c index 4b068967a6ca6ea..d2f739cecfbb08d 100644 --- a/Modules/atexitmodule.c +++ b/Modules/atexitmodule.c @@ -184,6 +184,9 @@ atexit_register(PyObject *module, PyObject *args, PyObject *kwargs) return NULL; } PyObject *func_args = PyTuple_GetSlice(args, 1, PyTuple_GET_SIZE(args)); + if (func_args == NULL) { + return NULL; + } PyObject *func_kwargs = kwargs; if (func_kwargs == NULL) @@ -191,6 +194,7 @@ atexit_register(PyObject *module, PyObject *args, PyObject *kwargs) func_kwargs = Py_None; } PyObject *callback = PyTuple_Pack(3, func, func_args, func_kwargs); + Py_DECREF(func_args); if (callback == NULL) { return NULL; @@ -255,21 +259,36 @@ atexit_ncallbacks(PyObject *module, PyObject *Py_UNUSED(dummy)) static int atexit_unregister_locked(PyObject *callbacks, PyObject *func) { - for (Py_ssize_t i = 0; i < PyList_GET_SIZE(callbacks); ++i) { - PyObject *tuple = PyList_GET_ITEM(callbacks, i); + for (Py_ssize_t i = PyList_GET_SIZE(callbacks) - 1; i >= 0; --i) { + PyObject *tuple = Py_NewRef(PyList_GET_ITEM(callbacks, i)); assert(PyTuple_CheckExact(tuple)); PyObject *to_compare = PyTuple_GET_ITEM(tuple, 0); int cmp = PyObject_RichCompareBool(func, to_compare, Py_EQ); - if (cmp < 0) - { + if (cmp < 0) { + Py_DECREF(tuple); return -1; } if (cmp == 1) { // We found a callback! - if (PyList_SetSlice(callbacks, i, i + 1, NULL) < 0) { - return -1; + // But its index could have changed if it or other callbacks were + // unregistered during the comparison. + Py_ssize_t j = PyList_GET_SIZE(callbacks) - 1; + j = Py_MIN(j, i); + for (; j >= 0; --j) { + if (PyList_GET_ITEM(callbacks, j) == tuple) { + // We found the callback index! For real! + if (PyList_SetSlice(callbacks, j, j + 1, NULL) < 0) { + Py_DECREF(tuple); + return -1; + } + i = j; + break; + } } - --i; + } + Py_DECREF(tuple); + if (i >= PyList_GET_SIZE(callbacks)) { + i = PyList_GET_SIZE(callbacks); } } diff --git a/Modules/binascii.c b/Modules/binascii.c index 6bb01d148b6faac..6b762b809b5989d 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -219,6 +219,14 @@ binascii_a2b_uu_impl(PyObject *module, Py_buffer *data) assert(ascii_len >= 0); /* First byte: binary data length (in bytes) */ + if (ascii_len == 0) { + state = get_binascii_state(module); + if (state == NULL) { + return NULL; + } + PyErr_SetString(state->Error, "Missing length byte"); + return NULL; + } bin_len = (*ascii_data++ - ' ') & 077; ascii_len--; @@ -383,7 +391,6 @@ binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode) const unsigned char *ascii_data = data->buf; size_t ascii_len = data->len; binascii_state *state = NULL; - char padding_started = 0; /* Allocate the buffer */ Py_ssize_t bin_len = ((ascii_len+3)/4)*3; /* Upper bound, corrected later */ @@ -394,14 +401,6 @@ binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode) return NULL; unsigned char *bin_data_start = bin_data; - if (strict_mode && ascii_len > 0 && ascii_data[0] == '=') { - state = get_binascii_state(module); - if (state) { - PyErr_SetString(state->Error, "Leading padding not allowed"); - } - goto error_end; - } - int quad_pos = 0; unsigned char leftchar = 0; int pads = 0; @@ -412,35 +411,34 @@ binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode) ** the invalid ones. */ if (this_ch == BASE64_PAD) { - padding_started = 1; - - if (strict_mode && quad_pos == 0) { - state = get_binascii_state(module); - if (state) { - PyErr_SetString(state->Error, "Excess padding not allowed"); - } - goto error_end; + pads++; + if (quad_pos >= 2 && quad_pos + pads <= 4) { + continue; } - if (quad_pos >= 2 && quad_pos + ++pads >= 4) { - /* A pad sequence means we should not parse more input. - ** We've already interpreted the data from the quad at this point. - ** in strict mode, an error should raise if there's excess data after the padding. - */ - if (strict_mode && i + 1 < ascii_len) { - state = get_binascii_state(module); - if (state) { - PyErr_SetString(state->Error, "Excess data after padding"); - } - goto error_end; - } - - goto done; + // See RFC 4648, section-3.3: "specifications MAY ignore the + // pad character, "=", treating it as non-alphabet data, if + // it is present before the end of the encoded data" and + // "the excess pad characters MAY also be ignored." + if (!strict_mode) { + continue; } - continue; + if (quad_pos == 1) { + /* Set an error below. */ + break; + } + state = get_binascii_state(module); + if (state) { + PyErr_SetString(state->Error, + (quad_pos == 0 && i == 0) + ? "Leading padding not allowed" + : "Excess padding not allowed"); + } + goto error_end; } this_ch = table_a2b_base64[this_ch]; if (this_ch >= 64) { + // See RFC 4648, section-3.3. if (strict_mode) { state = get_binascii_state(module); if (state) { @@ -451,11 +449,14 @@ binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode) continue; } - // Characters that are not '=', in the middle of the padding, are not allowed - if (strict_mode && padding_started) { + // Characters that are not '=', in the middle of the padding, are + // not allowed (except when they are). See RFC 4648, section-3.3. + if (pads && strict_mode) { state = get_binascii_state(module); if (state) { - PyErr_SetString(state->Error, "Discontinuous padding not allowed"); + PyErr_SetString(state->Error, (quad_pos + pads == 4) + ? "Excess data after padding" + : "Discontinuous padding not allowed"); } goto error_end; } @@ -484,31 +485,35 @@ binascii_a2b_base64_impl(PyObject *module, Py_buffer *data, int strict_mode) } } - if (quad_pos != 0) { + if (quad_pos == 1) { + /* There is exactly one extra valid, non-padding, base64 character. + * * This is an invalid length, as there is no possible input that + ** could encoded into such a base64 string. + */ state = get_binascii_state(module); - if (state == NULL) { - /* error already set, from get_binascii_state */ - } else if (quad_pos == 1) { - /* - ** There is exactly one extra valid, non-padding, base64 character. - ** This is an invalid length, as there is no possible input that - ** could encoded into such a base64 string. - */ + if (state) { PyErr_Format(state->Error, "Invalid base64-encoded string: " "number of data characters (%zd) cannot be 1 more " "than a multiple of 4", (bin_data - bin_data_start) / 3 * 4 + 1); - } else { + } + goto error_end; + } + + if (quad_pos != 0 && quad_pos + pads < 4) { + state = get_binascii_state(module); + if (state) { PyErr_SetString(state->Error, "Incorrect padding"); } - error_end: - _PyBytesWriter_Dealloc(&writer); - return NULL; + goto error_end; } -done: return _PyBytesWriter_Finish(&writer, bin_data); + +error_end: + _PyBytesWriter_Dealloc(&writer); + return NULL; } diff --git a/Modules/blake2module.c b/Modules/blake2module.c index f9acc57f1b2fa3d..c4fbd204d566d70 100644 --- a/Modules/blake2module.c +++ b/Modules/blake2module.c @@ -43,12 +43,11 @@ // SIMD256 can't be compiled on macOS ARM64, and performance of SIMD128 isn't // great; but when compiling a universal2 binary, autoconf will set -// HACL_CAN_COMPILE_SIMD128 and HACL_CAN_COMPILE_SIMD256 because they *can* be -// compiled on x86_64. If we're on macOS ARM64, disable these preprocessor -// symbols. +// _Py_HACL_CAN_COMPILE_VEC{128,256} because they *can* be compiled on x86_64. +// If we're on macOS ARM64, we however disable these preprocessor symbols. #if defined(__APPLE__) && defined(__arm64__) -# undef HACL_CAN_COMPILE_SIMD128 -# undef HACL_CAN_COMPILE_SIMD256 +# undef _Py_HACL_CAN_COMPILE_VEC128 +# undef _Py_HACL_CAN_COMPILE_VEC256 #endif // ECX @@ -114,31 +113,32 @@ void detect_cpu_features(cpu_flags *flags) { } } -#ifdef HACL_CAN_COMPILE_SIMD128 +#if _Py_HACL_CAN_COMPILE_VEC128 static inline bool has_simd128(cpu_flags *flags) { - // For now this is Intel-only, could conceivably be #ifdef'd to something + // For now this is Intel-only, could conceivably be if'd to something // else. return flags->sse && flags->sse2 && flags->sse3 && flags->sse41 && flags->sse42 && flags->cmov; } #endif -#ifdef HACL_CAN_COMPILE_SIMD256 +#if _Py_HACL_CAN_COMPILE_VEC256 static inline bool has_simd256(cpu_flags *flags) { return flags->avx && flags->avx2; } #endif -// Small mismatch between the variable names Python defines as part of configure -// at the ones HACL* expects to be set in order to enable those headers. -#define HACL_CAN_COMPILE_VEC128 HACL_CAN_COMPILE_SIMD128 -#define HACL_CAN_COMPILE_VEC256 HACL_CAN_COMPILE_SIMD256 +// HACL* expects HACL_CAN_COMPILE_VEC* macros to be set in order to enable +// the corresponding SIMD instructions so we need to "forward" the values +// we just deduced above. +#define HACL_CAN_COMPILE_VEC128 _Py_HACL_CAN_COMPILE_VEC128 +#define HACL_CAN_COMPILE_VEC256 _Py_HACL_CAN_COMPILE_VEC256 #include "_hacl/Hacl_Hash_Blake2b.h" #include "_hacl/Hacl_Hash_Blake2s.h" -#if HACL_CAN_COMPILE_SIMD256 +#if _Py_HACL_CAN_COMPILE_VEC256 #include "_hacl/Hacl_Hash_Blake2b_Simd256.h" #endif -#if HACL_CAN_COMPILE_SIMD128 +#if _Py_HACL_CAN_COMPILE_VEC128 #include "_hacl/Hacl_Hash_Blake2s_Simd128.h" #endif @@ -165,7 +165,7 @@ blake2_get_state(PyObject *module) return (Blake2State *)state; } -#if defined(HACL_CAN_COMPILE_SIMD128) || defined(HACL_CAN_COMPILE_SIMD256) +#if defined(_Py_HACL_CAN_COMPILE_VEC128) || defined(_Py_HACL_CAN_COMPILE_VEC256) static inline Blake2State* blake2_get_state_from_type(PyTypeObject *module) { @@ -329,18 +329,18 @@ static inline bool is_blake2s(blake2_impl impl) { } static inline blake2_impl type_to_impl(PyTypeObject *type) { -#if defined(HACL_CAN_COMPILE_SIMD128) || defined(HACL_CAN_COMPILE_SIMD256) +#if defined(_Py_HACL_CAN_COMPILE_VEC128) || defined(_Py_HACL_CAN_COMPILE_VEC256) Blake2State* st = blake2_get_state_from_type(type); #endif if (!strcmp(type->tp_name, blake2b_type_spec.name)) { -#ifdef HACL_CAN_COMPILE_SIMD256 +#if _Py_HACL_CAN_COMPILE_VEC256 if (has_simd256(&st->flags)) return Blake2b_256; else #endif return Blake2b; } else if (!strcmp(type->tp_name, blake2s_type_spec.name)) { -#ifdef HACL_CAN_COMPILE_SIMD128 +#if _Py_HACL_CAN_COMPILE_VEC128 if (has_simd128(&st->flags)) return Blake2s_128; else @@ -356,10 +356,10 @@ typedef struct { union { Hacl_Hash_Blake2s_state_t *blake2s_state; Hacl_Hash_Blake2b_state_t *blake2b_state; -#ifdef HACL_CAN_COMPILE_SIMD128 +#if _Py_HACL_CAN_COMPILE_VEC128 Hacl_Hash_Blake2s_Simd128_state_t *blake2s_128_state; #endif -#ifdef HACL_CAN_COMPILE_SIMD256 +#if _Py_HACL_CAN_COMPILE_VEC256 Hacl_Hash_Blake2b_Simd256_state_t *blake2b_256_state; #endif }; @@ -425,14 +425,14 @@ static void update(Blake2Object *self, uint8_t *buf, Py_ssize_t len) { switch (self->impl) { - // These need to be ifdef'd out otherwise it's an unresolved symbol at - // link-time. -#ifdef HACL_CAN_COMPILE_SIMD256 + // blake2b_256_state and blake2s_128_state must be if'd since + // otherwise this results in an unresolved symbol at link-time. +#if _Py_HACL_CAN_COMPILE_VEC256 case Blake2b_256: HACL_UPDATE(Hacl_Hash_Blake2b_Simd256_update,self->blake2b_256_state, buf, len); return; #endif -#ifdef HACL_CAN_COMPILE_SIMD128 +#if _Py_HACL_CAN_COMPILE_VEC128 case Blake2s_128: HACL_UPDATE(Hacl_Hash_Blake2s_Simd128_update,self->blake2s_128_state, buf, len); return; @@ -468,12 +468,12 @@ py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, // Ensure that the states are NULL-initialized in case of an error. // See: py_blake2_clear() for more details. switch (self->impl) { -#if HACL_CAN_COMPILE_SIMD256 +#if _Py_HACL_CAN_COMPILE_VEC256 case Blake2b_256: self->blake2b_256_state = NULL; break; #endif -#if HACL_CAN_COMPILE_SIMD128 +#if _Py_HACL_CAN_COMPILE_VEC128 case Blake2s_128: self->blake2s_128_state = NULL; break; @@ -591,7 +591,7 @@ py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, }; switch (self->impl) { -#if HACL_CAN_COMPILE_SIMD256 +#if _Py_HACL_CAN_COMPILE_VEC256 case Blake2b_256: { self->blake2b_256_state = Hacl_Hash_Blake2b_Simd256_malloc_with_params_and_key(&params, last_node, key->buf); if (self->blake2b_256_state == NULL) { @@ -601,7 +601,7 @@ py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, break; } #endif -#if HACL_CAN_COMPILE_SIMD128 +#if _Py_HACL_CAN_COMPILE_VEC128 case Blake2s_128: { self->blake2s_128_state = Hacl_Hash_Blake2s_Simd128_malloc_with_params_and_key(&params, last_node, key->buf); if (self->blake2s_128_state == NULL) { @@ -655,13 +655,12 @@ py_blake2b_or_s_new(PyTypeObject *type, PyObject *data, int digest_size, /*[clinic input] @classmethod _blake2.blake2b.__new__ as py_blake2b_new - data: object(c_default="NULL") = b'' - / + data as data_obj: object(c_default="NULL") = b'' * digest_size: int(c_default="HACL_HASH_BLAKE2B_OUT_BYTES") = _blake2.blake2b.MAX_DIGEST_SIZE - key: Py_buffer(c_default="NULL", py_default="b''") = None - salt: Py_buffer(c_default="NULL", py_default="b''") = None - person: Py_buffer(c_default="NULL", py_default="b''") = None + key: Py_buffer = b'' + salt: Py_buffer = b'' + person: Py_buffer = b'' fanout: int = 1 depth: int = 1 leaf_size: unsigned_long = 0 @@ -670,31 +669,36 @@ _blake2.blake2b.__new__ as py_blake2b_new inner_size: int = 0 last_node: bool = False usedforsecurity: bool = True + string: object(c_default="NULL") = None Return a new BLAKE2b hash object. [clinic start generated code]*/ static PyObject * -py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size, +py_blake2b_new_impl(PyTypeObject *type, PyObject *data_obj, int digest_size, Py_buffer *key, Py_buffer *salt, Py_buffer *person, int fanout, int depth, unsigned long leaf_size, unsigned long long node_offset, int node_depth, - int inner_size, int last_node, int usedforsecurity) -/*[clinic end generated code: output=32bfd8f043c6896f input=8fee2b7b11428b2d]*/ + int inner_size, int last_node, int usedforsecurity, + PyObject *string) +/*[clinic end generated code: output=de64bd850606b6a0 input=32832fb37d13c03d]*/ { + PyObject *data; + if (_Py_hashlib_data_argument(&data, data_obj, string) < 0) { + return NULL; + } return py_blake2b_or_s_new(type, data, digest_size, key, salt, person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); } /*[clinic input] @classmethod _blake2.blake2s.__new__ as py_blake2s_new - data: object(c_default="NULL") = b'' - / + data as data_obj: object(c_default="NULL") = b'' * digest_size: int(c_default="HACL_HASH_BLAKE2S_OUT_BYTES") = _blake2.blake2s.MAX_DIGEST_SIZE - key: Py_buffer(c_default="NULL", py_default="b''") = None - salt: Py_buffer(c_default="NULL", py_default="b''") = None - person: Py_buffer(c_default="NULL", py_default="b''") = None + key: Py_buffer = b'' + salt: Py_buffer = b'' + person: Py_buffer = b'' fanout: int = 1 depth: int = 1 leaf_size: unsigned_long = 0 @@ -703,18 +707,24 @@ _blake2.blake2s.__new__ as py_blake2s_new inner_size: int = 0 last_node: bool = False usedforsecurity: bool = True + string: object(c_default="NULL") = None Return a new BLAKE2s hash object. [clinic start generated code]*/ static PyObject * -py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, +py_blake2s_new_impl(PyTypeObject *type, PyObject *data_obj, int digest_size, Py_buffer *key, Py_buffer *salt, Py_buffer *person, int fanout, int depth, unsigned long leaf_size, unsigned long long node_offset, int node_depth, - int inner_size, int last_node, int usedforsecurity) -/*[clinic end generated code: output=556181f73905c686 input=8165a11980eac7f3]*/ + int inner_size, int last_node, int usedforsecurity, + PyObject *string) +/*[clinic end generated code: output=582a0c4295cc3a3c input=da467fc9dae646bb]*/ { + PyObject *data; + if (_Py_hashlib_data_argument(&data, data_obj, string) < 0) { + return NULL; + } return py_blake2b_or_s_new(type, data, digest_size, key, salt, person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); } @@ -722,8 +732,13 @@ static int blake2_blake2b_copy_locked(Blake2Object *self, Blake2Object *cpy) { assert(cpy != NULL); + + // Ensure that the implementation type is consistent with the HACL* state. + // See https://github.com/python/cpython/issues/152851 for details. + cpy->impl = self->impl; + switch (self->impl) { -#if HACL_CAN_COMPILE_SIMD256 +#if _Py_HACL_CAN_COMPILE_VEC256 case Blake2b_256: { cpy->blake2b_256_state = Hacl_Hash_Blake2b_Simd256_copy(self->blake2b_256_state); if (cpy->blake2b_256_state == NULL) { @@ -732,7 +747,7 @@ blake2_blake2b_copy_locked(Blake2Object *self, Blake2Object *cpy) break; } #endif -#if HACL_CAN_COMPILE_SIMD128 +#if _Py_HACL_CAN_COMPILE_VEC128 case Blake2s_128: { cpy->blake2s_128_state = Hacl_Hash_Blake2s_Simd128_copy(self->blake2s_128_state); if (cpy->blake2s_128_state == NULL) { @@ -758,7 +773,6 @@ blake2_blake2b_copy_locked(Blake2Object *self, Blake2Object *cpy) default: Py_UNREACHABLE(); } - cpy->impl = self->impl; return 0; error: @@ -843,12 +857,12 @@ _blake2_blake2b_digest_impl(Blake2Object *self) ENTER_HASHLIB(self); uint8_t digest_length = 0; switch (self->impl) { -#if HACL_CAN_COMPILE_SIMD256 +#if _Py_HACL_CAN_COMPILE_VEC256 case Blake2b_256: digest_length = Hacl_Hash_Blake2b_Simd256_digest(self->blake2b_256_state, digest); break; #endif -#if HACL_CAN_COMPILE_SIMD128 +#if _Py_HACL_CAN_COMPILE_VEC128 case Blake2s_128: digest_length = Hacl_Hash_Blake2s_Simd128_digest(self->blake2s_128_state, digest); break; @@ -881,12 +895,12 @@ _blake2_blake2b_hexdigest_impl(Blake2Object *self) ENTER_HASHLIB(self); uint8_t digest_length = 0; switch (self->impl) { -#if HACL_CAN_COMPILE_SIMD256 +#if _Py_HACL_CAN_COMPILE_VEC256 case Blake2b_256: digest_length = Hacl_Hash_Blake2b_Simd256_digest(self->blake2b_256_state, digest); break; #endif -#if HACL_CAN_COMPILE_SIMD128 +#if _Py_HACL_CAN_COMPILE_VEC128 case Blake2s_128: digest_length = Hacl_Hash_Blake2s_Simd128_digest(self->blake2s_128_state, digest); break; @@ -937,11 +951,11 @@ py_blake2b_get_digest_size(PyObject *op, void *Py_UNUSED(closure)) { Blake2Object *self = _Blake2Object_CAST(op); switch (self->impl) { -#if HACL_CAN_COMPILE_SIMD256 +#if _Py_HACL_CAN_COMPILE_VEC256 case Blake2b_256: return PyLong_FromLong(Hacl_Hash_Blake2b_Simd256_info(self->blake2b_256_state).digest_length); #endif -#if HACL_CAN_COMPILE_SIMD128 +#if _Py_HACL_CAN_COMPILE_VEC128 case Blake2s_128: return PyLong_FromLong(Hacl_Hash_Blake2s_Simd128_info(self->blake2s_128_state).digest_length); #endif @@ -972,7 +986,7 @@ py_blake2_clear(PyObject *op) // it. If an error occurs in the constructor, we should only free // states that were allocated (i.e. that are not NULL). switch (self->impl) { -#if HACL_CAN_COMPILE_SIMD256 +#if _Py_HACL_CAN_COMPILE_VEC256 case Blake2b_256: if (self->blake2b_256_state != NULL) { Hacl_Hash_Blake2b_Simd256_free(self->blake2b_256_state); @@ -980,7 +994,7 @@ py_blake2_clear(PyObject *op) } break; #endif -#if HACL_CAN_COMPILE_SIMD128 +#if _Py_HACL_CAN_COMPILE_VEC128 case Blake2s_128: if (self->blake2s_128_state != NULL) { Hacl_Hash_Blake2s_Simd128_free(self->blake2s_128_state); diff --git a/Modules/cjkcodecs/_codecs_iso2022.c b/Modules/cjkcodecs/_codecs_iso2022.c index ef6faeb71274e1d..b1984df2695b172 100644 --- a/Modules/cjkcodecs/_codecs_iso2022.c +++ b/Modules/cjkcodecs/_codecs_iso2022.c @@ -802,10 +802,13 @@ jisx0213_encoder(const MultibyteCodec *codec, const Py_UCS4 *data, return coded; case 2: /* second character of unicode pair */ - coded = find_pairencmap((ucs2_t)data[0], (ucs2_t)data[1], - jisx0213_pair_encmap, JISX0213_ENCPAIRS); - if (coded != DBCINV) - return coded; + if (data[1] != 0) { /* Don't consume null char as part of pair */ + coded = find_pairencmap((ucs2_t)data[0], (ucs2_t)data[1], + jisx0213_pair_encmap, JISX0213_ENCPAIRS); + if (coded != DBCINV) { + return coded; + } + } _Py_FALLTHROUGH; case -1: /* flush unterminated */ diff --git a/Modules/cjkcodecs/_codecs_jp.c b/Modules/cjkcodecs/_codecs_jp.c index f7127487aa5f594..cd77888d5514b82 100644 --- a/Modules/cjkcodecs/_codecs_jp.c +++ b/Modules/cjkcodecs/_codecs_jp.c @@ -192,8 +192,11 @@ ENCODER(euc_jis_2004) JISX0213_ENCPAIRS); if (code == DBCINV) return 1; - } else + } + else if (c2 != 0) { + /* Don't consume null char as part of pair */ insize = 2; + } } } } @@ -611,8 +614,10 @@ ENCODER(shift_jis_2004) if (code == DBCINV) return 1; } - else + else if (ch2 != 0) { + /* Don't consume null char as part of pair */ insize = 2; + } } } } diff --git a/Modules/cjkcodecs/clinic/multibytecodec.c.h b/Modules/cjkcodecs/clinic/multibytecodec.c.h index b3663180d726e5e..32588b0561e1acc 100644 --- a/Modules/cjkcodecs/clinic/multibytecodec.c.h +++ b/Modules/cjkcodecs/clinic/multibytecodec.c.h @@ -14,10 +14,11 @@ PyDoc_STRVAR(_multibytecodec_MultibyteCodec_encode__doc__, "\n" "Return an encoded string version of \'input\'.\n" "\n" -"\'errors\' may be given to set a different error handling scheme. Default is\n" -"\'strict\' meaning that encoding errors raise a UnicodeEncodeError. Other possible\n" -"values are \'ignore\', \'replace\' and \'xmlcharrefreplace\' as well as any other name\n" -"registered with codecs.register_error that can handle UnicodeEncodeErrors."); +"\'errors\' may be given to set a different error handling scheme.\n" +"Default is \'strict\' meaning that encoding errors raise\n" +"a UnicodeEncodeError. Other possible values are \'ignore\', \'replace\'\n" +"and \'xmlcharrefreplace\' as well as any other name registered with\n" +"codecs.register_error that can handle UnicodeEncodeErrors."); #define _MULTIBYTECODEC_MULTIBYTECODEC_ENCODE_METHODDEF \ {"encode", _PyCFunction_CAST(_multibytecodec_MultibyteCodec_encode), METH_FASTCALL|METH_KEYWORDS, _multibytecodec_MultibyteCodec_encode__doc__}, @@ -103,9 +104,10 @@ PyDoc_STRVAR(_multibytecodec_MultibyteCodec_decode__doc__, "\n" "Decodes \'input\'.\n" "\n" -"\'errors\' may be given to set a different error handling scheme. Default is\n" -"\'strict\' meaning that encoding errors raise a UnicodeDecodeError. Other possible\n" -"values are \'ignore\' and \'replace\' as well as any other name registered with\n" +"\'errors\' may be given to set a different error handling scheme.\n" +"Default is \'strict\' meaning that encoding errors raise\n" +"a UnicodeDecodeError. Other possible values are \'ignore\' and\n" +"\'replace\' as well as any other name registered with\n" "codecs.register_error that is able to handle UnicodeDecodeErrors.\""); #define _MULTIBYTECODEC_MULTIBYTECODEC_DECODE_METHODDEF \ @@ -696,4 +698,4 @@ PyDoc_STRVAR(_multibytecodec___create_codec__doc__, #define _MULTIBYTECODEC___CREATE_CODEC_METHODDEF \ {"__create_codec", (PyCFunction)_multibytecodec___create_codec, METH_O, _multibytecodec___create_codec__doc__}, -/*[clinic end generated code: output=014f4f6bb9d29594 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a84b1544d7d01abb input=a9049054013a1b77]*/ diff --git a/Modules/cjkcodecs/multibytecodec.c b/Modules/cjkcodecs/multibytecodec.c index 08b74740bda4bf3..1841f62c7797c20 100644 --- a/Modules/cjkcodecs/multibytecodec.c +++ b/Modules/cjkcodecs/multibytecodec.c @@ -597,17 +597,18 @@ _multibytecodec.MultibyteCodec.encode Return an encoded string version of 'input'. -'errors' may be given to set a different error handling scheme. Default is -'strict' meaning that encoding errors raise a UnicodeEncodeError. Other possible -values are 'ignore', 'replace' and 'xmlcharrefreplace' as well as any other name -registered with codecs.register_error that can handle UnicodeEncodeErrors. +'errors' may be given to set a different error handling scheme. +Default is 'strict' meaning that encoding errors raise +a UnicodeEncodeError. Other possible values are 'ignore', 'replace' +and 'xmlcharrefreplace' as well as any other name registered with +codecs.register_error that can handle UnicodeEncodeErrors. [clinic start generated code]*/ static PyObject * _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self, PyObject *input, const char *errors) -/*[clinic end generated code: output=7b26652045ba56a9 input=2841745b95ed338f]*/ +/*[clinic end generated code: output=7b26652045ba56a9 input=980002ed1447697b]*/ { MultibyteCodec_State state; PyObject *errorcb, *r, *ucvt; @@ -662,9 +663,10 @@ _multibytecodec.MultibyteCodec.decode Decodes 'input'. -'errors' may be given to set a different error handling scheme. Default is -'strict' meaning that encoding errors raise a UnicodeDecodeError. Other possible -values are 'ignore' and 'replace' as well as any other name registered with +'errors' may be given to set a different error handling scheme. +Default is 'strict' meaning that encoding errors raise +a UnicodeDecodeError. Other possible values are 'ignore' and +'replace' as well as any other name registered with codecs.register_error that is able to handle UnicodeDecodeErrors." [clinic start generated code]*/ @@ -672,7 +674,7 @@ static PyObject * _multibytecodec_MultibyteCodec_decode_impl(MultibyteCodecObject *self, Py_buffer *input, const char *errors) -/*[clinic end generated code: output=ff419f65bad6cc77 input=e0c78fc7ab190def]*/ +/*[clinic end generated code: output=ff419f65bad6cc77 input=dbf93d8bb98ca440]*/ { MultibyteCodec_State state; MultibyteDecodeBuffer buf; diff --git a/Modules/clinic/_abc.c.h b/Modules/clinic/_abc.c.h index 04681fa2206a2a5..fa1c57dc26bf853 100644 --- a/Modules/clinic/_abc.c.h +++ b/Modules/clinic/_abc.c.h @@ -146,9 +146,9 @@ PyDoc_STRVAR(_abc_get_cache_token__doc__, "\n" "Returns the current ABC cache token.\n" "\n" -"The token is an opaque object (supporting equality testing) identifying the\n" -"current version of the ABC cache for virtual subclasses. The token changes\n" -"with every call to register() on any ABC."); +"The token is an opaque object (supporting equality testing) identifying\n" +"the current version of the ABC cache for virtual subclasses. The token\n" +"changes with every call to register() on any ABC."); #define _ABC_GET_CACHE_TOKEN_METHODDEF \ {"get_cache_token", (PyCFunction)_abc_get_cache_token, METH_NOARGS, _abc_get_cache_token__doc__}, @@ -161,4 +161,4 @@ _abc_get_cache_token(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _abc_get_cache_token_impl(module); } -/*[clinic end generated code: output=1989b6716c950e17 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=b05d599656aeb1e1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_asynciomodule.c.h b/Modules/clinic/_asynciomodule.c.h index 66953d74213b66d..f07a09df5ac7ae0 100644 --- a/Modules/clinic/_asynciomodule.c.h +++ b/Modules/clinic/_asynciomodule.c.h @@ -90,7 +90,8 @@ PyDoc_STRVAR(_asyncio_Future_result__doc__, "\n" "If the future has been cancelled, raises CancelledError. If the\n" "future\'s result isn\'t yet available, raises InvalidStateError. If\n" -"the future is done and has an exception set, this exception is raised."); +"the future is done and has an exception set, this exception is\n" +"raised."); #define _ASYNCIO_FUTURE_RESULT_METHODDEF \ {"result", (PyCFunction)_asyncio_Future_result, METH_NOARGS, _asyncio_Future_result__doc__}, @@ -250,8 +251,8 @@ PyDoc_STRVAR(_asyncio_Future_add_done_callback__doc__, "\n" "Add a callback to be run when the future becomes done.\n" "\n" -"The callback is called with a single argument - the future object. If\n" -"the future is already done when this is called, the callback is\n" +"The callback is called with a single argument - the future object.\n" +"If the future is already done when this is called, the callback is\n" "scheduled with call_soon."); #define _ASYNCIO_FUTURE_ADD_DONE_CALLBACK_METHODDEF \ @@ -371,9 +372,9 @@ PyDoc_STRVAR(_asyncio_Future_cancel__doc__, "\n" "Cancel the future and schedule callbacks.\n" "\n" -"If the future is already done or cancelled, return False. Otherwise,\n" -"change the future\'s state to cancelled, schedule the callbacks and\n" -"return True."); +"If the future is already done or cancelled, return False.\n" +"Otherwise, change the future\'s state to cancelled, schedule the\n" +"callbacks and return True."); #define _ASYNCIO_FUTURE_CANCEL_METHODDEF \ {"cancel", _PyCFunction_CAST(_asyncio_Future_cancel), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _asyncio_Future_cancel__doc__}, @@ -465,8 +466,8 @@ PyDoc_STRVAR(_asyncio_Future_done__doc__, "\n" "Return True if the future is done.\n" "\n" -"Done means either that a result / exception are available, or that the\n" -"future was cancelled."); +"Done means either that a result / exception are available, or that\n" +"the future was cancelled."); #define _ASYNCIO_FUTURE_DONE_METHODDEF \ {"done", (PyCFunction)_asyncio_Future_done, METH_NOARGS, _asyncio_Future_done__doc__}, @@ -2232,4 +2233,4 @@ _asyncio_future_discard_from_awaited_by(PyObject *module, PyObject *const *args, exit: return return_value; } -/*[clinic end generated code: output=b69948ed810591d9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=32996fb47c48245b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_bisectmodule.c.h b/Modules/clinic/_bisectmodule.c.h index 314208bc41d0c11..5031782a3ad3d13 100644 --- a/Modules/clinic/_bisectmodule.c.h +++ b/Modules/clinic/_bisectmodule.c.h @@ -16,8 +16,8 @@ PyDoc_STRVAR(_bisect_bisect_right__doc__, "Return the index where to insert item x in list a, assuming a is sorted.\n" "\n" "The return value i is such that all e in a[:i] have e <= x, and all e in\n" -"a[i:] have e > x. So if x already appears in the list, a.insert(i, x) will\n" -"insert just after the rightmost x already there.\n" +"a[i:] have e > x. So if x already appears in the list, a.insert(i, x)\n" +"will insert just after the rightmost x already there.\n" "\n" "Optional args lo (default 0) and hi (default len(a)) bound the\n" "slice of a to be searched.\n" @@ -235,8 +235,8 @@ PyDoc_STRVAR(_bisect_bisect_left__doc__, "Return the index where to insert item x in list a, assuming a is sorted.\n" "\n" "The return value i is such that all e in a[:i] have e < x, and all e in\n" -"a[i:] have e >= x. So if x already appears in the list, a.insert(i, x) will\n" -"insert just before the leftmost x already there.\n" +"a[i:] have e >= x. So if x already appears in the list, a.insert(i, x)\n" +"will insert just before the leftmost x already there.\n" "\n" "Optional args lo (default 0) and hi (default len(a)) bound the\n" "slice of a to be searched.\n" @@ -446,4 +446,4 @@ _bisect_insort_left(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P exit: return return_value; } -/*[clinic end generated code: output=729385c6a23828ab input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5e0b8c4e5e697eae input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_bz2module.c.h b/Modules/clinic/_bz2module.c.h index 2bc6524b6a973b8..30f2c7965e73ae1 100644 --- a/Modules/clinic/_bz2module.c.h +++ b/Modules/clinic/_bz2module.c.h @@ -116,18 +116,19 @@ PyDoc_STRVAR(_bz2_BZ2Decompressor_decompress__doc__, "\n" "Decompress *data*, returning uncompressed data as bytes.\n" "\n" -"If *max_length* is nonnegative, returns at most *max_length* bytes of\n" -"decompressed data. If this limit is reached and further output can be\n" -"produced, *self.needs_input* will be set to ``False``. In this case, the next\n" -"call to *decompress()* may provide *data* as b\'\' to obtain more of the output.\n" +"If *max_length* is nonnegative, returns at most *max_length* bytes\n" +"of decompressed data. If this limit is reached and further output\n" +"can be produced, *self.needs_input* will be set to ``False``. In\n" +"this case, the next call to *decompress()* may provide *data* as b\'\'\n" +"to obtain more of the output.\n" "\n" -"If all of the input data was decompressed and returned (either because this\n" -"was less than *max_length* bytes, or because *max_length* was negative),\n" -"*self.needs_input* will be set to True.\n" +"If all of the input data was decompressed and returned (either\n" +"because this was less than *max_length* bytes, or because\n" +"*max_length* was negative), *self.needs_input* will be set to True.\n" "\n" -"Attempting to decompress data after the end of stream is reached raises an\n" -"EOFError. Any data found after the end of the stream is ignored and saved in\n" -"the unused_data attribute."); +"Attempting to decompress data after the end of stream is reached\n" +"raises an EOFError. Any data found after the end of the stream is\n" +"ignored and saved in the unused_data attribute."); #define _BZ2_BZ2DECOMPRESSOR_DECOMPRESS_METHODDEF \ {"decompress", _PyCFunction_CAST(_bz2_BZ2Decompressor_decompress), METH_FASTCALL|METH_KEYWORDS, _bz2_BZ2Decompressor_decompress__doc__}, @@ -237,4 +238,4 @@ _bz2_BZ2Decompressor(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=552ac6d4c5a101b7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1dce5396d592bad7 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_codecsmodule.c.h b/Modules/clinic/_codecsmodule.c.h index b03103257593267..1b499bea2922a4e 100644 --- a/Modules/clinic/_codecsmodule.c.h +++ b/Modules/clinic/_codecsmodule.c.h @@ -14,9 +14,10 @@ PyDoc_STRVAR(_codecs_register__doc__, "\n" "Register a codec search function.\n" "\n" -"Search functions are expected to take one argument, the encoding name in\n" -"all lower case letters, and either return None, or a tuple of functions\n" -"(encoder, decoder, stream_reader, stream_writer) (or a CodecInfo object)."); +"Search functions are expected to take one argument, the encoding\n" +"name in all lower case letters, and either return None, or a tuple\n" +"of functions (encoder, decoder, stream_reader, stream_writer) (or\n" +"a CodecInfo object)."); #define _CODECS_REGISTER_METHODDEF \ {"register", (PyCFunction)_codecs_register, METH_O, _codecs_register__doc__}, @@ -76,10 +77,10 @@ PyDoc_STRVAR(_codecs_encode__doc__, "Encodes obj using the codec registered for encoding.\n" "\n" "The default encoding is \'utf-8\'. errors may be given to set a\n" -"different error handling scheme. Default is \'strict\' meaning that encoding\n" -"errors raise a ValueError. Other possible values are \'ignore\', \'replace\'\n" -"and \'backslashreplace\' as well as any other name registered with\n" -"codecs.register_error that can handle ValueErrors."); +"different error handling scheme. Default is \'strict\' meaning that\n" +"encoding errors raise a ValueError. Other possible values are \'ignore\',\n" +"\'replace\' and \'backslashreplace\' as well as any other name registered\n" +"with codecs.register_error that can handle ValueErrors."); #define _CODECS_ENCODE_METHODDEF \ {"encode", _PyCFunction_CAST(_codecs_encode), METH_FASTCALL|METH_KEYWORDS, _codecs_encode__doc__}, @@ -179,10 +180,10 @@ PyDoc_STRVAR(_codecs_decode__doc__, "Decodes obj using the codec registered for encoding.\n" "\n" "Default encoding is \'utf-8\'. errors may be given to set a\n" -"different error handling scheme. Default is \'strict\' meaning that encoding\n" -"errors raise a ValueError. Other possible values are \'ignore\', \'replace\'\n" -"and \'backslashreplace\' as well as any other name registered with\n" -"codecs.register_error that can handle ValueErrors."); +"different error handling scheme. Default is \'strict\' meaning that\n" +"encoding errors raise a ValueError. Other possible values are \'ignore\',\n" +"\'replace\' and \'backslashreplace\' as well as any other name registered\n" +"with codecs.register_error that can handle ValueErrors."); #define _CODECS_DECODE_METHODDEF \ {"decode", _PyCFunction_CAST(_codecs_decode), METH_FASTCALL|METH_KEYWORDS, _codecs_decode__doc__}, @@ -2649,8 +2650,9 @@ PyDoc_STRVAR(_codecs_register_error__doc__, "Register the specified error handler under the name errors.\n" "\n" "handler must be a callable object, that will be called with an exception\n" -"instance containing information about the location of the encoding/decoding\n" -"error and must return a (replacement, new position) tuple."); +"instance containing information about the location of the\n" +"encoding/decoding error and must return a (replacement, new position)\n" +"tuple."); #define _CODECS_REGISTER_ERROR_METHODDEF \ {"register_error", _PyCFunction_CAST(_codecs_register_error), METH_FASTCALL, _codecs_register_error__doc__}, @@ -2745,8 +2747,8 @@ PyDoc_STRVAR(_codecs_lookup_error__doc__, "\n" "lookup_error(errors) -> handler\n" "\n" -"Return the error handler for the specified error handling name or raise a\n" -"LookupError, if no handler exists under this name."); +"Return the error handler for the specified error handling name or raise\n" +"a LookupError, if no handler exists under this name."); #define _CODECS_LOOKUP_ERROR_METHODDEF \ {"lookup_error", (PyCFunction)_codecs_lookup_error, METH_O, _codecs_lookup_error__doc__}, @@ -2802,4 +2804,4 @@ _codecs_lookup_error(PyObject *module, PyObject *arg) #ifndef _CODECS_CODE_PAGE_ENCODE_METHODDEF #define _CODECS_CODE_PAGE_ENCODE_METHODDEF #endif /* !defined(_CODECS_CODE_PAGE_ENCODE_METHODDEF) */ -/*[clinic end generated code: output=ed13f20dfb09e306 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=eb221ef27f132476 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_collectionsmodule.c.h b/Modules/clinic/_collectionsmodule.c.h index b5c315c680e7821..6c60678a6fbd51a 100644 --- a/Modules/clinic/_collectionsmodule.c.h +++ b/Modules/clinic/_collectionsmodule.c.h @@ -340,7 +340,7 @@ deque_index(PyObject *deque, PyObject *const *args, Py_ssize_t nargs) PyObject *return_value = NULL; PyObject *v; Py_ssize_t start = 0; - Py_ssize_t stop = Py_SIZE(deque); + Py_ssize_t stop = PY_SSIZE_T_MAX; if (!_PyArg_CheckPositional("index", nargs, 1, 3)) { goto exit; @@ -632,4 +632,4 @@ tuplegetter_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=b9d4d647c221cb9f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f5a388add99d3d15 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_csv.c.h b/Modules/clinic/_csv.c.h index 416aeafccf917b2..b8dd8ac35fac599 100644 --- a/Modules/clinic/_csv.c.h +++ b/Modules/clinic/_csv.c.h @@ -12,9 +12,7 @@ PyDoc_STRVAR(_csv_list_dialects__doc__, "list_dialects($module, /)\n" "--\n" "\n" -"Return a list of all known dialect names.\n" -"\n" -" names = csv.list_dialects()"); +"Return a list of all known dialect names."); #define _CSV_LIST_DIALECTS_METHODDEF \ {"list_dialects", (PyCFunction)_csv_list_dialects, METH_NOARGS, _csv_list_dialects__doc__}, @@ -32,9 +30,7 @@ PyDoc_STRVAR(_csv_unregister_dialect__doc__, "unregister_dialect($module, /, name)\n" "--\n" "\n" -"Delete the name/dialect mapping associated with a string name.\n" -"\n" -" csv.unregister_dialect(name)"); +"Delete the name/dialect mapping associated with a string name."); #define _CSV_UNREGISTER_DIALECT_METHODDEF \ {"unregister_dialect", _PyCFunction_CAST(_csv_unregister_dialect), METH_FASTCALL|METH_KEYWORDS, _csv_unregister_dialect__doc__}, @@ -92,9 +88,7 @@ PyDoc_STRVAR(_csv_get_dialect__doc__, "get_dialect($module, /, name)\n" "--\n" "\n" -"Return the dialect instance associated with name.\n" -"\n" -" dialect = csv.get_dialect(name)"); +"Return the dialect instance associated with name."); #define _CSV_GET_DIALECT_METHODDEF \ {"get_dialect", _PyCFunction_CAST(_csv_get_dialect), METH_FASTCALL|METH_KEYWORDS, _csv_get_dialect__doc__}, @@ -154,8 +148,6 @@ PyDoc_STRVAR(_csv_field_size_limit__doc__, "\n" "Sets an upper limit on parsed fields.\n" "\n" -" csv.field_size_limit([limit])\n" -"\n" "Returns old limit. If limit is not given, no new limit is set and\n" "the old limit is returned"); @@ -215,4 +207,4 @@ _csv_field_size_limit(PyObject *module, PyObject *const *args, Py_ssize_t nargs, exit: return return_value; } -/*[clinic end generated code: output=1fb09d5e7667ad0d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ed77cb69fad9f3b4 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_curses_panel.c.h b/Modules/clinic/_curses_panel.c.h index 6f4966825ec4bf4..3ca11178c01eeef 100644 --- a/Modules/clinic/_curses_panel.c.h +++ b/Modules/clinic/_curses_panel.c.h @@ -35,7 +35,8 @@ PyDoc_STRVAR(_curses_panel_panel_hide__doc__, "\n" "Hide the panel.\n" "\n" -"This does not delete the object, it just makes the window on screen invisible."); +"This does not delete the object, it just makes the window on screen\n" +"invisible."); #define _CURSES_PANEL_PANEL_HIDE_METHODDEF \ {"hide", _PyCFunction_CAST(_curses_panel_panel_hide), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_hide__doc__}, @@ -57,7 +58,9 @@ PyDoc_STRVAR(_curses_panel_panel_show__doc__, "show($self, /)\n" "--\n" "\n" -"Display the panel (which might have been hidden)."); +"Display the panel (which might have been hidden).\n" +"\n" +"The panel is placed on top of the panel stack."); #define _CURSES_PANEL_PANEL_SHOW_METHODDEF \ {"show", _PyCFunction_CAST(_curses_panel_panel_show), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _curses_panel_panel_show__doc__}, @@ -362,7 +365,9 @@ PyDoc_STRVAR(_curses_panel_new_panel__doc__, "new_panel($module, win, /)\n" "--\n" "\n" -"Return a panel object, associating it with the given window win."); +"Return a panel object, associating it with the given window win.\n" +"\n" +"The new panel is placed on top of the panel stack."); #define _CURSES_PANEL_NEW_PANEL_METHODDEF \ {"new_panel", (PyCFunction)_curses_panel_new_panel, METH_O, _curses_panel_new_panel__doc__}, @@ -411,7 +416,8 @@ PyDoc_STRVAR(_curses_panel_update_panels__doc__, "\n" "Updates the virtual screen after changes in the panel stack.\n" "\n" -"This does not call curses.doupdate(), so you\'ll have to do this yourself."); +"This does not call curses.doupdate(), so you\'ll have to do this\n" +"yourself."); #define _CURSES_PANEL_UPDATE_PANELS_METHODDEF \ {"update_panels", (PyCFunction)_curses_panel_update_panels, METH_NOARGS, _curses_panel_update_panels__doc__}, @@ -424,4 +430,4 @@ _curses_panel_update_panels(PyObject *module, PyObject *Py_UNUSED(ignored)) { return _curses_panel_update_panels_impl(module); } -/*[clinic end generated code: output=36853ecb4a979814 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8efa92ec022f1047 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h index 552360eb80a5458..c1916d6e0d8e1b6 100644 --- a/Modules/clinic/_cursesmodule.c.h +++ b/Modules/clinic/_cursesmodule.c.h @@ -301,7 +301,7 @@ PyDoc_STRVAR(_curses_window_attron__doc__, "attron($self, attr, /)\n" "--\n" "\n" -"Add attribute attr from the \"background\" set."); +"Add attribute attr to the \"background\" set."); #define _CURSES_WINDOW_ATTRON_METHODDEF \ {"attron", (PyCFunction)_curses_window_attron, METH_O, _curses_window_attron__doc__}, @@ -422,10 +422,10 @@ PyDoc_STRVAR(_curses_window_border__doc__, " br\n" " Bottom-right corner.\n" "\n" -"Each parameter specifies the character to use for a specific part of the\n" -"border. The characters can be specified as integers or as one-character\n" -"strings. A 0 value for any parameter will cause the default character to be\n" -"used for that parameter."); +"Each parameter specifies the character to use for a specific part of\n" +"the border. The characters can be specified as integers or as\n" +"one-character strings. A 0 value for any parameter will cause the\n" +"default character to be used for that parameter."); #define _CURSES_WINDOW_BORDER_METHODDEF \ {"border", _PyCFunction_CAST(_curses_window_border), METH_FASTCALL, _curses_window_border__doc__}, @@ -500,8 +500,9 @@ PyDoc_STRVAR(_curses_window_box__doc__, " horch\n" " Top and bottom side.\n" "\n" -"Similar to border(), but both ls and rs are verch and both ts and bs are\n" -"horch. The default corner characters are always used by this function."); +"Similar to border(), but both ls and rs are verch and both ts and bs\n" +"are horch. The default corner characters are always used by this\n" +"function."); #define _CURSES_WINDOW_BOX_METHODDEF \ {"box", (PyCFunction)_curses_window_box, METH_VARARGS, _curses_window_box__doc__}, @@ -539,12 +540,15 @@ _curses_window_box(PyObject *self, PyObject *args) PyDoc_STRVAR(_curses_window_delch__doc__, "delch([y, x])\n" -"Delete any character at (y, x).\n" +"Delete the character under the cursor, or at (y, x) if specified.\n" "\n" " y\n" " Y-coordinate.\n" " x\n" -" X-coordinate."); +" X-coordinate.\n" +"\n" +"All characters to the right on the same line are shifted one\n" +"position left."); #define _CURSES_WINDOW_DELCH_METHODDEF \ {"delch", (PyCFunction)_curses_window_delch, METH_VARARGS, _curses_window_delch__doc__}, @@ -593,9 +597,9 @@ PyDoc_STRVAR(_curses_window_derwin__doc__, " begin_x\n" " Left side x-coordinate.\n" "\n" -"derwin() is the same as calling subwin(), except that begin_y and begin_x\n" -"are relative to the origin of the window, rather than relative to the entire\n" -"screen."); +"derwin() is the same as calling subwin(), except that begin_y and\n" +"begin_x are relative to the origin of the window, rather than\n" +"relative to the entire screen."); #define _CURSES_WINDOW_DERWIN_METHODDEF \ {"derwin", (PyCFunction)_curses_window_derwin, METH_VARARGS, _curses_window_derwin__doc__}, @@ -679,7 +683,7 @@ _curses_window_echochar(PyObject *self, PyObject *const *args, Py_ssize_t nargs) return return_value; } -#if defined(NCURSES_MOUSE_VERSION) +#if (defined(HAVE_CURSES_GETMOUSE) || defined(PDCURSES)) PyDoc_STRVAR(_curses_window_enclose__doc__, "enclose($self, y, x, /)\n" @@ -722,7 +726,7 @@ _curses_window_enclose(PyObject *self, PyObject *const *args, Py_ssize_t nargs) return return_value; } -#endif /* defined(NCURSES_MOUSE_VERSION) */ +#endif /* (defined(HAVE_CURSES_GETMOUSE) || defined(PDCURSES)) */ PyDoc_STRVAR(_curses_window_getbkgd__doc__, "getbkgd($self, /)\n" @@ -761,14 +765,15 @@ PyDoc_STRVAR(_curses_window_getch__doc__, " x\n" " X-coordinate.\n" "\n" -"The integer returned does not have to be in ASCII range: function keys,\n" -"keypad keys and so on return numbers higher than 256. In no-delay mode, -1\n" -"is returned if there is no input, else getch() waits until a key is pressed."); +"The integer returned does not have to be in ASCII range: function\n" +"keys, keypad keys and so on return numbers higher than 256. In\n" +"no-delay mode, -1 is returned if there is no input, else getch()\n" +"waits until a key is pressed."); #define _CURSES_WINDOW_GETCH_METHODDEF \ {"getch", (PyCFunction)_curses_window_getch, METH_VARARGS, _curses_window_getch__doc__}, -static int +static PyObject * _curses_window_getch_impl(PyCursesWindowObject *self, int group_right_1, int y, int x); @@ -779,7 +784,6 @@ _curses_window_getch(PyObject *self, PyObject *args) int group_right_1 = 0; int y = 0; int x = 0; - int _return_value; switch (PyTuple_GET_SIZE(args)) { case 0: @@ -794,11 +798,7 @@ _curses_window_getch(PyObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "_curses.window.getch requires 0 to 2 arguments"); goto exit; } - _return_value = _curses_window_getch_impl((PyCursesWindowObject *)self, group_right_1, y, x); - if ((_return_value == -1) && PyErr_Occurred()) { - goto exit; - } - return_value = PyLong_FromLong((long)_return_value); + return_value = _curses_window_getch_impl((PyCursesWindowObject *)self, group_right_1, y, x); exit: return return_value; @@ -813,9 +813,10 @@ PyDoc_STRVAR(_curses_window_getkey__doc__, " x\n" " X-coordinate.\n" "\n" -"Returning a string instead of an integer, as getch() does. Function keys,\n" -"keypad keys and other special keys return a multibyte string containing the\n" -"key name. In no-delay mode, an exception is raised if there is no input."); +"Returning a string instead of an integer, as getch() does. Function\n" +"keys, keypad keys and other special keys return a multibyte string\n" +"containing the key name. In no-delay mode, an exception is raised\n" +"if there is no input."); #define _CURSES_WINDOW_GETKEY_METHODDEF \ {"getkey", (PyCFunction)_curses_window_getkey, METH_VARARGS, _curses_window_getkey__doc__}, @@ -984,8 +985,8 @@ PyDoc_STRVAR(_curses_window_insch__doc__, " attr\n" " Attributes for the character.\n" "\n" -"All characters to the right of the cursor are shifted one position right, with\n" -"the rightmost characters on the line being lost."); +"All characters to the right of the cursor are shifted one position\n" +"right, with the rightmost characters on the line being lost."); #define _CURSES_WINDOW_INSCH_METHODDEF \ {"insch", (PyCFunction)_curses_window_insch, METH_VARARGS, _curses_window_insch__doc__}, @@ -1050,7 +1051,8 @@ PyDoc_STRVAR(_curses_window_inch__doc__, " x\n" " X-coordinate.\n" "\n" -"The bottom 8 bits are the character proper, and upper bits are the attributes."); +"The bottom 8 bits are the character proper, and upper bits are the\n" +"attributes."); #define _CURSES_WINDOW_INCH_METHODDEF \ {"inch", (PyCFunction)_curses_window_inch, METH_VARARGS, _curses_window_inch__doc__}, @@ -1104,11 +1106,11 @@ PyDoc_STRVAR(_curses_window_insstr__doc__, " attr\n" " Attributes for characters.\n" "\n" -"Insert a character string (as many characters as will fit on the line)\n" -"before the character under the cursor. All characters to the right of\n" -"the cursor are shifted right, with the rightmost characters on the line\n" -"being lost. The cursor position does not change (after moving to y, x,\n" -"if specified)."); +"Insert a character string (as many characters as will fit on the\n" +"line) before the character under the cursor. All characters to the\n" +"right of the cursor are shifted right, with the rightmost characters\n" +"on the line being lost. The cursor position does not change (after\n" +"moving to y, x, if specified)."); #define _CURSES_WINDOW_INSSTR_METHODDEF \ {"insstr", (PyCFunction)_curses_window_insstr, METH_VARARGS, _curses_window_insstr__doc__}, @@ -1179,12 +1181,12 @@ PyDoc_STRVAR(_curses_window_insnstr__doc__, " attr\n" " Attributes for characters.\n" "\n" -"Insert a character string (as many characters as will fit on the line)\n" -"before the character under the cursor, up to n characters. If n is zero\n" -"or negative, the entire string is inserted. All characters to the right\n" -"of the cursor are shifted right, with the rightmost characters on the line\n" -"being lost. The cursor position does not change (after moving to y, x, if\n" -"specified)."); +"Insert a character string (as many characters as will fit on the\n" +"line) before the character under the cursor, up to n characters. If\n" +"n is zero or negative, the entire string is inserted. All\n" +"characters to the right of the cursor are shifted right, with the\n" +"rightmost characters on the line being lost. The cursor position\n" +"does not change (after moving to y, x, if specified)."); #define _CURSES_WINDOW_INSNSTR_METHODDEF \ {"insnstr", (PyCFunction)_curses_window_insnstr, METH_VARARGS, _curses_window_insnstr__doc__}, @@ -1250,7 +1252,8 @@ PyDoc_STRVAR(_curses_window_is_linetouched__doc__, " line\n" " Line number.\n" "\n" -"Raise a curses.error exception if line is not valid for the given window."); +"Raise a curses.error exception if line is not valid for the given\n" +"window."); #define _CURSES_WINDOW_IS_LINETOUCHED_METHODDEF \ {"is_linetouched", (PyCFunction)_curses_window_is_linetouched, METH_O, _curses_window_is_linetouched__doc__}, @@ -1280,9 +1283,9 @@ PyDoc_STRVAR(_curses_window_noutrefresh__doc__, "noutrefresh([pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol])\n" "Mark for refresh but wait.\n" "\n" -"This function updates the data structure representing the desired state of the\n" -"window, but does not force an update of the physical screen. To accomplish\n" -"that, call doupdate()."); +"This function updates the data structure representing the desired\n" +"state of the window, but does not force an update of the physical\n" +"screen. To accomplish that, call doupdate()."); #define _CURSES_WINDOW_NOUTREFRESH_METHODDEF \ {"noutrefresh", (PyCFunction)_curses_window_noutrefresh, METH_VARARGS, _curses_window_noutrefresh__doc__}, @@ -1334,9 +1337,9 @@ PyDoc_STRVAR(_curses_window_noutrefresh__doc__, "\n" "Mark for refresh but wait.\n" "\n" -"This function updates the data structure representing the desired state of the\n" -"window, but does not force an update of the physical screen. To accomplish\n" -"that, call doupdate()."); +"This function updates the data structure representing the desired\n" +"state of the window, but does not force an update of the physical\n" +"screen. To accomplish that, call doupdate()."); #define _CURSES_WINDOW_NOUTREFRESH_METHODDEF \ {"noutrefresh", (PyCFunction)_curses_window_noutrefresh, METH_NOARGS, _curses_window_noutrefresh__doc__}, @@ -1356,14 +1359,15 @@ PyDoc_STRVAR(_curses_window_overlay__doc__, "overlay(destwin, [sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol])\n" "Overlay the window on top of destwin.\n" "\n" -"The windows need not be the same size, only the overlapping region is copied.\n" -"This copy is non-destructive, which means that the current background\n" -"character does not overwrite the old contents of destwin.\n" +"The windows need not be the same size, only the overlapping region\n" +"is copied. This copy is non-destructive, which means that the\n" +"current background character does not overwrite the old contents of\n" +"destwin.\n" "\n" -"To get fine-grained control over the copied region, the second form of\n" -"overlay() can be used. sminrow and smincol are the upper-left coordinates\n" -"of the source window, and the other variables mark a rectangle in the\n" -"destination window."); +"To get fine-grained control over the copied region, the second form\n" +"of overlay() can be used. sminrow and smincol are the upper-left\n" +"coordinates of the source window, and the other variables mark\n" +"a rectangle in the destination window."); #define _CURSES_WINDOW_OVERLAY_METHODDEF \ {"overlay", (PyCFunction)_curses_window_overlay, METH_VARARGS, _curses_window_overlay__doc__}, @@ -1414,14 +1418,15 @@ PyDoc_STRVAR(_curses_window_overwrite__doc__, " dmaxcol])\n" "Overwrite the window on top of destwin.\n" "\n" -"The windows need not be the same size, in which case only the overlapping\n" -"region is copied. This copy is destructive, which means that the current\n" -"background character overwrites the old contents of destwin.\n" +"The windows need not be the same size, in which case only the\n" +"overlapping region is copied. This copy is destructive, which means\n" +"that the current background character overwrites the old contents of\n" +"destwin.\n" "\n" -"To get fine-grained control over the copied region, the second form of\n" -"overwrite() can be used. sminrow and smincol are the upper-left coordinates\n" -"of the source window, the other variables mark a rectangle in the destination\n" -"window."); +"To get fine-grained control over the copied region, the second form\n" +"of overwrite() can be used. sminrow and smincol are the upper-left\n" +"coordinates of the source window, the other variables mark\n" +"a rectangle in the destination window."); #define _CURSES_WINDOW_OVERWRITE_METHODDEF \ {"overwrite", (PyCFunction)_curses_window_overwrite, METH_VARARGS, _curses_window_overwrite__doc__}, @@ -1540,16 +1545,17 @@ PyDoc_STRVAR(_curses_window_refresh__doc__, "Update the display immediately.\n" "\n" "Synchronize actual screen with previous drawing/deleting methods.\n" -"The 6 optional arguments can only be specified when the window is a pad\n" -"created with newpad(). The additional parameters are needed to indicate\n" -"what part of the pad and screen are involved. pminrow and pmincol specify\n" -"the upper left-hand corner of the rectangle to be displayed in the pad.\n" -"sminrow, smincol, smaxrow, and smaxcol specify the edges of the rectangle to\n" -"be displayed on the screen. The lower right-hand corner of the rectangle to\n" -"be displayed in the pad is calculated from the screen coordinates, since the\n" -"rectangles must be the same size. Both rectangles must be entirely contained\n" -"within their respective structures. Negative values of pminrow, pmincol,\n" -"sminrow, or smincol are treated as if they were zero."); +"The 6 optional arguments can only be specified when the window is\n" +"a pad created with newpad(). The additional parameters are needed\n" +"to indicate what part of the pad and screen are involved. pminrow\n" +"and pmincol specify the upper left-hand corner of the rectangle to\n" +"be displayed in the pad. sminrow, smincol, smaxrow, and smaxcol\n" +"specify the edges of the rectangle to be displayed on the screen.\n" +"The lower right-hand corner of the rectangle to be displayed in the\n" +"pad is calculated from the screen coordinates, since the rectangles\n" +"must be the same size. Both rectangles must be entirely contained\n" +"within their respective structures. Negative values of pminrow,\n" +"pmincol, sminrow, or smincol are treated as if they were zero."); #define _CURSES_WINDOW_REFRESH_METHODDEF \ {"refresh", (PyCFunction)_curses_window_refresh, METH_VARARGS, _curses_window_refresh__doc__}, @@ -1647,8 +1653,8 @@ PyDoc_STRVAR(_curses_window_subwin__doc__, " begin_x\n" " Left side x-coordinate.\n" "\n" -"By default, the sub-window will extend from the specified position to the\n" -"lower right corner of the window."); +"By default, the sub-window will extend from the specified position\n" +"to the lower right corner of the window."); #define _CURSES_WINDOW_SUBWIN_METHODDEF \ {"subwin", (PyCFunction)_curses_window_subwin, METH_VARARGS, _curses_window_subwin__doc__}, @@ -1696,7 +1702,8 @@ PyDoc_STRVAR(_curses_window_scroll__doc__, " lines\n" " Number of lines to scroll.\n" "\n" -"Scroll upward if the argument is positive and downward if it is negative."); +"Scroll upward if the argument is positive and downward if it is\n" +"negative."); #define _CURSES_WINDOW_SCROLL_METHODDEF \ {"scroll", (PyCFunction)_curses_window_scroll, METH_VARARGS, _curses_window_scroll__doc__}, @@ -1735,8 +1742,9 @@ PyDoc_STRVAR(_curses_window_touchline__doc__, "touchline(start, count, [changed=True])\n" "Pretend count lines have been changed, starting with line start.\n" "\n" -"If changed is supplied, it specifies whether the affected lines are marked\n" -"as having been changed (changed=True) or unchanged (changed=False)."); +"If changed is supplied, it specifies whether the affected lines are\n" +"marked as having been changed (changed=True) or unchanged\n" +"(changed=False)."); #define _CURSES_WINDOW_TOUCHLINE_METHODDEF \ {"touchline", (PyCFunction)_curses_window_touchline, METH_VARARGS, _curses_window_touchline__doc__}, @@ -1851,7 +1859,13 @@ _curses_window_vline(PyObject *self, PyObject *args) PyDoc_STRVAR(_curses_filter__doc__, "filter($module, /)\n" "--\n" -"\n"); +"\n" +"Restrict screen updates to the current line.\n" +"\n" +"Must be called before initscr(). Afterwards curses confines the cursor\n" +"and screen updates to a single line, which is useful for enabling\n" +"character-at-a-time line editing without touching the rest of the\n" +"screen."); #define _CURSES_FILTER_METHODDEF \ {"filter", (PyCFunction)_curses_filter, METH_NOARGS, _curses_filter__doc__}, @@ -1930,11 +1944,12 @@ PyDoc_STRVAR(_curses_cbreak__doc__, " flag\n" " If false, the effect is the same as calling nocbreak().\n" "\n" -"In cbreak mode (sometimes called \"rare\" mode) normal tty line buffering is\n" -"turned off and characters are available to be read one by one. However,\n" -"unlike raw mode, special characters (interrupt, quit, suspend, and flow\n" -"control) retain their effects on the tty driver and calling program.\n" -"Calling first raw() then cbreak() leaves the terminal in cbreak mode."); +"In cbreak mode (sometimes called \"rare\" mode) normal tty line buffering\n" +"is turned off and characters are available to be read one by one.\n" +"However, unlike raw mode, special characters (interrupt, quit, suspend,\n" +"and flow control) retain their effects on the tty driver and calling\n" +"program. Calling first raw() then cbreak() leaves the terminal in\n" +"cbreak mode."); #define _CURSES_CBREAK_METHODDEF \ {"cbreak", _PyCFunction_CAST(_curses_cbreak), METH_FASTCALL, _curses_cbreak__doc__}, @@ -1974,8 +1989,9 @@ PyDoc_STRVAR(_curses_color_content__doc__, " color_number\n" " The number of the color (0 - (COLORS-1)).\n" "\n" -"A 3-tuple is returned, containing the R, G, B values for the given color,\n" -"which will be between 0 (no component) and 1000 (maximum amount of component)."); +"A 3-tuple is returned, containing the R, G, B values for the given\n" +"color, which will be between 0 (no component) and 1000 (maximum amount\n" +"of component)."); #define _CURSES_COLOR_CONTENT_METHODDEF \ {"color_content", (PyCFunction)_curses_color_content, METH_O, _curses_color_content__doc__}, @@ -2008,7 +2024,8 @@ PyDoc_STRVAR(_curses_color_pair__doc__, " The number of the color pair.\n" "\n" "This attribute value can be combined with A_STANDOUT, A_REVERSE, and the\n" -"other A_* attributes. pair_number() is the counterpart to this function."); +"other A_* attributes. pair_number() is the counterpart to this\n" +"function."); #define _CURSES_COLOR_PAIR_METHODDEF \ {"color_pair", (PyCFunction)_curses_color_pair, METH_O, _curses_color_pair__doc__}, @@ -2042,9 +2059,9 @@ PyDoc_STRVAR(_curses_curs_set__doc__, " 0 for invisible, 1 for normal visible, or 2 for very visible.\n" "\n" "If the terminal supports the visibility requested, the previous cursor\n" -"state is returned; otherwise, an exception is raised. On many terminals,\n" -"the \"visible\" mode is an underline cursor and the \"very visible\" mode is\n" -"a block cursor."); +"state is returned; otherwise, an exception is raised. On many\n" +"terminals, the \"visible\" mode is an underline cursor and the \"very\n" +"visible\" mode is a block cursor."); #define _CURSES_CURS_SET_METHODDEF \ {"curs_set", (PyCFunction)_curses_curs_set, METH_O, _curses_curs_set__doc__}, @@ -2096,7 +2113,8 @@ PyDoc_STRVAR(_curses_def_shell_mode__doc__, "\n" "Save the current terminal mode as the \"shell\" mode.\n" "\n" -"The \"shell\" mode is the mode when the running program is not using curses.\n" +"The \"shell\" mode is the mode when the running program is not using\n" +"curses.\n" "\n" "Subsequent calls to reset_shell_mode() will restore this mode."); @@ -2170,7 +2188,8 @@ PyDoc_STRVAR(_curses_echo__doc__, " flag\n" " If false, the effect is the same as calling noecho().\n" "\n" -"In echo mode, each character input is echoed to the screen as it is entered."); +"In echo mode, each character input is echoed to the screen as it is\n" +"entered."); #define _CURSES_ECHO_METHODDEF \ {"echo", _PyCFunction_CAST(_curses_echo), METH_FASTCALL, _curses_echo__doc__}, @@ -2243,7 +2262,8 @@ PyDoc_STRVAR(_curses_flash__doc__, "\n" "Flash the screen.\n" "\n" -"That is, change it to reverse-video and then change it back in a short interval."); +"That is, change it to reverse-video and then change it back in a short\n" +"interval."); #define _CURSES_FLASH_METHODDEF \ {"flash", (PyCFunction)_curses_flash, METH_NOARGS, _curses_flash__doc__}, @@ -2263,8 +2283,8 @@ PyDoc_STRVAR(_curses_flushinp__doc__, "\n" "Flush all input buffers.\n" "\n" -"This throws away any typeahead that has been typed by the user and has not\n" -"yet been processed by the program."); +"This throws away any typeahead that has been typed by the user and has\n" +"not yet been processed by the program."); #define _CURSES_FLUSHINP_METHODDEF \ {"flushinp", (PyCFunction)_curses_flushinp, METH_NOARGS, _curses_flushinp__doc__}, @@ -2302,7 +2322,7 @@ _curses_getsyx(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* defined(getsyx) */ -#if defined(NCURSES_MOUSE_VERSION) +#if (defined(HAVE_CURSES_GETMOUSE) || defined(PDCURSES)) PyDoc_STRVAR(_curses_getmouse__doc__, "getmouse($module, /)\n" @@ -2325,9 +2345,9 @@ _curses_getmouse(PyObject *module, PyObject *Py_UNUSED(ignored)) return _curses_getmouse_impl(module); } -#endif /* defined(NCURSES_MOUSE_VERSION) */ +#endif /* (defined(HAVE_CURSES_GETMOUSE) || defined(PDCURSES)) */ -#if defined(NCURSES_MOUSE_VERSION) +#if (defined(HAVE_CURSES_GETMOUSE) || defined(PDCURSES)) PyDoc_STRVAR(_curses_ungetmouse__doc__, "ungetmouse($module, id, x, y, z, bstate, /)\n" @@ -2399,7 +2419,7 @@ _curses_ungetmouse(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -#endif /* defined(NCURSES_MOUSE_VERSION) */ +#endif /* (defined(HAVE_CURSES_GETMOUSE) || defined(PDCURSES)) */ PyDoc_STRVAR(_curses_getwin__doc__, "getwin($module, file, /)\n" @@ -2619,8 +2639,9 @@ PyDoc_STRVAR(_curses_init_pair__doc__, " bg\n" " Background color number (-1 - (COLORS-1)).\n" "\n" -"If the color-pair was previously initialized, the screen is refreshed and\n" -"all occurrences of that color-pair are changed to the new definition."); +"If the color-pair was previously initialized, the screen is refreshed\n" +"and all occurrences of that color-pair are changed to the new\n" +"definition."); #define _CURSES_INIT_PAIR_METHODDEF \ {"init_pair", _PyCFunction_CAST(_curses_init_pair), METH_FASTCALL, _curses_init_pair__doc__}, @@ -2771,7 +2792,7 @@ _curses_setupterm(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } -#if (defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20081102) +#if defined(HAVE_CURSES_ESCDELAY) PyDoc_STRVAR(_curses_get_escdelay__doc__, "get_escdelay($module, /)\n" @@ -2779,9 +2800,9 @@ PyDoc_STRVAR(_curses_get_escdelay__doc__, "\n" "Gets the curses ESCDELAY setting.\n" "\n" -"Gets the number of milliseconds to wait after reading an escape character,\n" -"to distinguish between an individual escape character entered on the\n" -"keyboard from escape sequences sent by cursor and function keys."); +"Gets the number of milliseconds to wait after reading an escape\n" +"character, to distinguish between an individual escape character entered\n" +"on the keyboard from escape sequences sent by cursor and function keys."); #define _CURSES_GET_ESCDELAY_METHODDEF \ {"get_escdelay", (PyCFunction)_curses_get_escdelay, METH_NOARGS, _curses_get_escdelay__doc__}, @@ -2795,9 +2816,9 @@ _curses_get_escdelay(PyObject *module, PyObject *Py_UNUSED(ignored)) return _curses_get_escdelay_impl(module); } -#endif /* (defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20081102) */ +#endif /* defined(HAVE_CURSES_ESCDELAY) */ -#if (defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20081102) +#if defined(HAVE_CURSES_SET_ESCDELAY) PyDoc_STRVAR(_curses_set_escdelay__doc__, "set_escdelay($module, ms, /)\n" @@ -2808,9 +2829,9 @@ PyDoc_STRVAR(_curses_set_escdelay__doc__, " ms\n" " length of the delay in milliseconds.\n" "\n" -"Sets the number of milliseconds to wait after reading an escape character,\n" -"to distinguish between an individual escape character entered on the\n" -"keyboard from escape sequences sent by cursor and function keys."); +"Sets the number of milliseconds to wait after reading an escape\n" +"character, to distinguish between an individual escape character entered\n" +"on the keyboard from escape sequences sent by cursor and function keys."); #define _CURSES_SET_ESCDELAY_METHODDEF \ {"set_escdelay", (PyCFunction)_curses_set_escdelay, METH_O, _curses_set_escdelay__doc__}, @@ -2834,9 +2855,9 @@ _curses_set_escdelay(PyObject *module, PyObject *arg) return return_value; } -#endif /* (defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20081102) */ +#endif /* defined(HAVE_CURSES_SET_ESCDELAY) */ -#if (defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20081102) +#if defined(HAVE_CURSES_TABSIZE) PyDoc_STRVAR(_curses_get_tabsize__doc__, "get_tabsize($module, /)\n" @@ -2844,8 +2865,8 @@ PyDoc_STRVAR(_curses_get_tabsize__doc__, "\n" "Gets the curses TABSIZE setting.\n" "\n" -"Gets the number of columns used by the curses library when converting a tab\n" -"character to spaces as it adds the tab to a window."); +"Gets the number of columns used by the curses library when converting\n" +"a tab character to spaces as it adds the tab to a window."); #define _CURSES_GET_TABSIZE_METHODDEF \ {"get_tabsize", (PyCFunction)_curses_get_tabsize, METH_NOARGS, _curses_get_tabsize__doc__}, @@ -2859,9 +2880,9 @@ _curses_get_tabsize(PyObject *module, PyObject *Py_UNUSED(ignored)) return _curses_get_tabsize_impl(module); } -#endif /* (defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20081102) */ +#endif /* defined(HAVE_CURSES_TABSIZE) */ -#if (defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20081102) +#if defined(HAVE_CURSES_SET_TABSIZE) PyDoc_STRVAR(_curses_set_tabsize__doc__, "set_tabsize($module, size, /)\n" @@ -2872,8 +2893,8 @@ PyDoc_STRVAR(_curses_set_tabsize__doc__, " size\n" " rendered cell width of a tab character.\n" "\n" -"Sets the number of columns used by the curses library when converting a tab\n" -"character to spaces as it adds the tab to a window."); +"Sets the number of columns used by the curses library when converting\n" +"a tab character to spaces as it adds the tab to a window."); #define _CURSES_SET_TABSIZE_METHODDEF \ {"set_tabsize", (PyCFunction)_curses_set_tabsize, METH_O, _curses_set_tabsize__doc__}, @@ -2897,12 +2918,17 @@ _curses_set_tabsize(PyObject *module, PyObject *arg) return return_value; } -#endif /* (defined(NCURSES_EXT_FUNCS) && NCURSES_EXT_FUNCS >= 20081102) */ +#endif /* defined(HAVE_CURSES_SET_TABSIZE) */ PyDoc_STRVAR(_curses_intrflush__doc__, "intrflush($module, flag, /)\n" "--\n" -"\n"); +"\n" +"Control flushing of the output buffer when an interrupt key is pressed.\n" +"\n" +"If flag is true, pressing an interrupt key (interrupt, break, or quit)\n" +"flushes all output in the terminal driver queue. If flag is false, no\n" +"flushing is done."); #define _CURSES_INTRFLUSH_METHODDEF \ {"intrflush", (PyCFunction)_curses_intrflush, METH_O, _curses_intrflush__doc__}, @@ -3044,8 +3070,8 @@ PyDoc_STRVAR(_curses_longname__doc__, "\n" "Return the terminfo long name field describing the current terminal.\n" "\n" -"The maximum length of a verbose description is 128 characters. It is defined\n" -"only after the call to initscr()."); +"The maximum length of a verbose description is 128 characters. It is\n" +"defined only after the call to initscr()."); #define _CURSES_LONGNAME_METHODDEF \ {"longname", (PyCFunction)_curses_longname, METH_NOARGS, _curses_longname__doc__}, @@ -3090,7 +3116,7 @@ _curses_meta(PyObject *module, PyObject *arg) return return_value; } -#if defined(NCURSES_MOUSE_VERSION) +#if (defined(HAVE_CURSES_GETMOUSE) || defined(PDCURSES)) PyDoc_STRVAR(_curses_mouseinterval__doc__, "mouseinterval($module, interval, /)\n" @@ -3102,8 +3128,8 @@ PyDoc_STRVAR(_curses_mouseinterval__doc__, " Time in milliseconds.\n" "\n" "Set the maximum time that can elapse between press and release events in\n" -"order for them to be recognized as a click, and return the previous interval\n" -"value."); +"order for them to be recognized as a click, and return the previous\n" +"interval value."); #define _CURSES_MOUSEINTERVAL_METHODDEF \ {"mouseinterval", (PyCFunction)_curses_mouseinterval, METH_O, _curses_mouseinterval__doc__}, @@ -3127,20 +3153,20 @@ _curses_mouseinterval(PyObject *module, PyObject *arg) return return_value; } -#endif /* defined(NCURSES_MOUSE_VERSION) */ +#endif /* (defined(HAVE_CURSES_GETMOUSE) || defined(PDCURSES)) */ -#if defined(NCURSES_MOUSE_VERSION) +#if (defined(HAVE_CURSES_GETMOUSE) || defined(PDCURSES)) PyDoc_STRVAR(_curses_mousemask__doc__, "mousemask($module, newmask, /)\n" "--\n" "\n" -"Set the mouse events to be reported, and return a tuple (availmask, oldmask).\n" +"Set the mouse events to be reported, and return (availmask, oldmask).\n" "\n" "Return a tuple (availmask, oldmask). availmask indicates which of the\n" -"specified mouse events can be reported; on complete failure it returns 0.\n" -"oldmask is the previous value of the given window\'s mouse event mask.\n" -"If this function is never called, no mouse events are ever reported."); +"specified mouse events can be reported; on complete failure it returns\n" +"0. oldmask is the previous value of the mouse event mask. If this\n" +"function is never called, no mouse events are ever reported."); #define _CURSES_MOUSEMASK_METHODDEF \ {"mousemask", (PyCFunction)_curses_mousemask, METH_O, _curses_mousemask__doc__}, @@ -3165,7 +3191,7 @@ _curses_mousemask(PyObject *module, PyObject *arg) return return_value; } -#endif /* defined(NCURSES_MOUSE_VERSION) */ +#endif /* (defined(HAVE_CURSES_GETMOUSE) || defined(PDCURSES)) */ PyDoc_STRVAR(_curses_napms__doc__, "napms($module, ms, /)\n" @@ -3257,8 +3283,8 @@ PyDoc_STRVAR(_curses_newwin__doc__, " begin_x\n" " Left side x-coordinate.\n" "\n" -"By default, the window will extend from the specified position to the lower\n" -"right corner of the screen."); +"By default, the window will extend from the specified position to the\n" +"lower right corner of the screen."); #define _CURSES_NEWWIN_METHODDEF \ {"newwin", (PyCFunction)_curses_newwin, METH_VARARGS, _curses_newwin__doc__}, @@ -3308,8 +3334,9 @@ PyDoc_STRVAR(_curses_nl__doc__, " flag\n" " If false, the effect is the same as calling nonl().\n" "\n" -"This mode translates the return key into newline on input, and translates\n" -"newline into return and line-feed on output. Newline mode is initially on."); +"This mode translates the return key into newline on input, and\n" +"translates newline into return and line-feed on output. Newline mode\n" +"is initially on."); #define _CURSES_NL_METHODDEF \ {"nl", _PyCFunction_CAST(_curses_nl), METH_FASTCALL, _curses_nl__doc__}, @@ -3386,8 +3413,8 @@ PyDoc_STRVAR(_curses_nonl__doc__, "\n" "Leave newline mode.\n" "\n" -"Disable translation of return into newline on input, and disable low-level\n" -"translation of newline into newline/return on output."); +"Disable translation of return into newline on input, and disable\n" +"low-level translation of newline into newline/return on output."); #define _CURSES_NONL_METHODDEF \ {"nonl", (PyCFunction)_curses_nonl, METH_NOARGS, _curses_nonl__doc__}, @@ -3577,7 +3604,10 @@ _curses_qiflush(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyDoc_STRVAR(_curses_update_lines_cols__doc__, "update_lines_cols($module, /)\n" "--\n" -"\n"); +"\n" +"Update the LINES and COLS module variables.\n" +"\n" +"This is useful for detecting manual screen resize."); #define _CURSES_UPDATE_LINES_COLS_METHODDEF \ {"update_lines_cols", (PyCFunction)_curses_update_lines_cols, METH_NOARGS, _curses_update_lines_cols__doc__}, @@ -3603,8 +3633,8 @@ PyDoc_STRVAR(_curses_raw__doc__, " If false, the effect is the same as calling noraw().\n" "\n" "In raw mode, normal line buffering and processing of interrupt, quit,\n" -"suspend, and flow control keys are turned off; characters are presented to\n" -"curses input functions one by one."); +"suspend, and flow control keys are turned off; characters are presented\n" +"to curses input functions one by one."); #define _CURSES_RAW_METHODDEF \ {"raw", _PyCFunction_CAST(_curses_raw), METH_FASTCALL, _curses_raw__doc__}, @@ -3702,8 +3732,8 @@ PyDoc_STRVAR(_curses_resizeterm__doc__, " ncols\n" " Width.\n" "\n" -"Adjusts other bookkeeping data used by the curses library that record the\n" -"window dimensions (in particular the SIGWINCH handler)."); +"Adjusts other bookkeeping data used by the curses library that record\n" +"the window dimensions (in particular the SIGWINCH handler)."); #define _CURSES_RESIZETERM_METHODDEF \ {"resizeterm", _PyCFunction_CAST(_curses_resizeterm), METH_FASTCALL, _curses_resizeterm__doc__}, @@ -3781,10 +3811,11 @@ PyDoc_STRVAR(_curses_resize_term__doc__, " Width.\n" "\n" "When resizing the windows, resize_term() blank-fills the areas that are\n" -"extended. The calling application should fill in these areas with appropriate\n" -"data. The resize_term() function attempts to resize all windows. However,\n" -"due to the calling convention of pads, it is not possible to resize these\n" -"without additional interaction with the application."); +"extended. The calling application should fill in these areas with\n" +"appropriate data. The resize_term() function attempts to resize all\n" +"windows. However, due to the calling convention of pads, it is not\n" +"possible to resize these without additional interaction with the\n" +"application."); #define _CURSES_RESIZE_TERM_METHODDEF \ {"resize_term", _PyCFunction_CAST(_curses_resize_term), METH_FASTCALL, _curses_resize_term__doc__}, @@ -3919,12 +3950,12 @@ PyDoc_STRVAR(_curses_start_color__doc__, "\n" "Initializes eight basic colors and global variables COLORS and COLOR_PAIRS.\n" "\n" -"Must be called if the programmer wants to use colors, and before any other\n" -"color manipulation routine is called. It is good practice to call this\n" -"routine right after initscr().\n" +"Must be called if the programmer wants to use colors, and before any\n" +"other color manipulation routine is called. It is good practice to call\n" +"this routine right after initscr().\n" "\n" -"It also restores the colors on the terminal to the values they had when the\n" -"terminal was just turned on."); +"It also restores the colors on the terminal to the values they had when\n" +"the terminal was just turned on."); #define _CURSES_START_COLOR_METHODDEF \ {"start_color", (PyCFunction)_curses_start_color, METH_NOARGS, _curses_start_color__doc__}, @@ -4026,8 +4057,8 @@ PyDoc_STRVAR(_curses_tigetnum__doc__, " capname\n" " The terminfo capability name.\n" "\n" -"The value -2 is returned if capname is not a numeric capability, or -1 if\n" -"it is canceled or absent from the terminal description."); +"The value -2 is returned if capname is not a numeric capability, or -1\n" +"if it is canceled or absent from the terminal description."); #define _CURSES_TIGETNUM_METHODDEF \ {"tigetnum", (PyCFunction)_curses_tigetnum, METH_O, _curses_tigetnum__doc__}, @@ -4069,8 +4100,8 @@ PyDoc_STRVAR(_curses_tigetstr__doc__, " capname\n" " The terminfo capability name.\n" "\n" -"None is returned if capname is not a string capability, or is canceled or\n" -"absent from the terminal description."); +"None is returned if capname is not a string capability, or is canceled\n" +"or absent from the terminal description."); #define _CURSES_TIGETSTR_METHODDEF \ {"tigetstr", (PyCFunction)_curses_tigetstr, METH_O, _curses_tigetstr__doc__}, @@ -4224,14 +4255,14 @@ PyDoc_STRVAR(_curses_use_env__doc__, "\n" "Use environment variables LINES and COLUMNS.\n" "\n" -"If used, this function should be called before initscr() or newterm() are\n" -"called.\n" +"If used, this function should be called before initscr() or newterm()\n" +"are called.\n" "\n" -"When flag is False, the values of lines and columns specified in the terminfo\n" -"database will be used, even if environment variables LINES and COLUMNS (used\n" -"by default) are set, or if curses is running in a window (in which case\n" -"default behavior would be to use the window size if LINES and COLUMNS are\n" -"not set)."); +"When flag is False, the values of lines and columns specified in the\n" +"terminfo database will be used, even if environment variables LINES and\n" +"COLUMNS (used by default) are set, or if curses is running in a window\n" +"(in which case default behavior would be to use the window size if LINES\n" +"and COLUMNS are not set)."); #define _CURSES_USE_ENV_METHODDEF \ {"use_env", (PyCFunction)_curses_use_env, METH_O, _curses_use_env__doc__}, @@ -4440,4 +4471,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #ifndef _CURSES_ASSUME_DEFAULT_COLORS_METHODDEF #define _CURSES_ASSUME_DEFAULT_COLORS_METHODDEF #endif /* !defined(_CURSES_ASSUME_DEFAULT_COLORS_METHODDEF) */ -/*[clinic end generated code: output=42b2923d88c8d0f6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=945fcc1c41f2ffe7 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_datetimemodule.c.h b/Modules/clinic/_datetimemodule.c.h index 18e6129fad8a89a..15281252ecf2275 100644 --- a/Modules/clinic/_datetimemodule.c.h +++ b/Modules/clinic/_datetimemodule.c.h @@ -14,8 +14,8 @@ PyDoc_STRVAR(datetime_date_fromtimestamp__doc__, "\n" "Create a date from a POSIX timestamp.\n" "\n" -"The timestamp is a number, e.g. created via time.time(), that is interpreted\n" -"as local time."); +"The timestamp is a number, e.g. created via time.time(), that is\n" +"interpreted as local time."); #define DATETIME_DATE_FROMTIMESTAMP_METHODDEF \ {"fromtimestamp", (PyCFunction)datetime_date_fromtimestamp, METH_O|METH_CLASS, datetime_date_fromtimestamp__doc__}, @@ -524,4 +524,4 @@ datetime_datetime_replace(PyObject *self, PyObject *const *args, Py_ssize_t narg exit: return return_value; } -/*[clinic end generated code: output=809640e747529c72 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=847b941002d485a8 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_dbmmodule.c.h b/Modules/clinic/_dbmmodule.c.h index 5e5031944087761..6c979a4b0081df3 100644 --- a/Modules/clinic/_dbmmodule.c.h +++ b/Modules/clinic/_dbmmodule.c.h @@ -5,6 +5,7 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_runtime.h" // _Py_SINGLETON() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_dbm_dbm_close__doc__, @@ -22,7 +23,13 @@ _dbm_dbm_close_impl(dbmobject *self); static PyObject * _dbm_dbm_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _dbm_dbm_close_impl((dbmobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _dbm_dbm_close_impl((dbmobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_dbm_dbm_keys__doc__, @@ -40,11 +47,18 @@ _dbm_dbm_keys_impl(dbmobject *self, PyTypeObject *cls); static PyObject * _dbm_dbm_keys(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "keys() takes no arguments"); - return NULL; + goto exit; } - return _dbm_dbm_keys_impl((dbmobject *)self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _dbm_dbm_keys_impl((dbmobject *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_dbm_dbm_get__doc__, @@ -85,7 +99,9 @@ _dbm_dbm_get(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_ &key, &key_length, &default_value)) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = _dbm_dbm_get_impl((dbmobject *)self, cls, key, key_length, default_value); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -97,7 +113,8 @@ PyDoc_STRVAR(_dbm_dbm_setdefault__doc__, "\n" "Return the value for key if present, otherwise default.\n" "\n" -"If key is not in the database, it is inserted with default as the value."); +"If key is not in the database, it is inserted with default as the\n" +"value."); #define _DBM_DBM_SETDEFAULT_METHODDEF \ {"setdefault", _PyCFunction_CAST(_dbm_dbm_setdefault), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _dbm_dbm_setdefault__doc__}, @@ -131,7 +148,9 @@ _dbm_dbm_setdefault(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py &key, &key_length, &default_value)) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = _dbm_dbm_setdefault_impl((dbmobject *)self, cls, key, key_length, default_value); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -152,11 +171,18 @@ _dbm_dbm_clear_impl(dbmobject *self, PyTypeObject *cls); static PyObject * _dbm_dbm_clear(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "clear() takes no arguments"); - return NULL; + goto exit; } - return _dbm_dbm_clear_impl((dbmobject *)self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _dbm_dbm_clear_impl((dbmobject *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(dbmopen__doc__, @@ -221,4 +247,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=3b456118f231b160 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=677deecf525167a5 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_functoolsmodule.c.h b/Modules/clinic/_functoolsmodule.c.h index 23f666310850312..6c847ef7d6c0b7e 100644 --- a/Modules/clinic/_functoolsmodule.c.h +++ b/Modules/clinic/_functoolsmodule.c.h @@ -76,9 +76,9 @@ PyDoc_STRVAR(_functools_reduce__doc__, "\n" "Apply a function of two arguments cumulatively to the items of an iterable, from left to right.\n" "\n" -"This effectively reduces the iterable to a single value. If initial is present,\n" -"it is placed before the items of the iterable in the calculation, and serves as\n" -"a default when the iterable is empty.\n" +"This effectively reduces the iterable to a single value. If initial is\n" +"present, it is placed before the items of the iterable in the\n" +"calculation, and serves as a default when the iterable is empty.\n" "\n" "For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5])\n" "calculates ((((1 + 2) + 3) + 4) + 5)."); @@ -192,4 +192,4 @@ _functools__lru_cache_wrapper_cache_clear(PyObject *self, PyObject *Py_UNUSED(ig return return_value; } -/*[clinic end generated code: output=7f2abc718fcc35d5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1c44abd7e56118dc input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_gdbmmodule.c.h b/Modules/clinic/_gdbmmodule.c.h index 00950f18e535413..fe993cc328fbd2f 100644 --- a/Modules/clinic/_gdbmmodule.c.h +++ b/Modules/clinic/_gdbmmodule.c.h @@ -5,6 +5,7 @@ preserve #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # include "pycore_runtime.h" // _Py_SINGLETON() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_gdbm_gdbm_get__doc__, @@ -70,7 +71,9 @@ _gdbm_gdbm_setdefault(PyObject *self, PyObject *const *args, Py_ssize_t nargs) } default_value = args[1]; skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _gdbm_gdbm_setdefault_impl((gdbmobject *)self, key, default_value); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -91,7 +94,13 @@ _gdbm_gdbm_close_impl(gdbmobject *self); static PyObject * _gdbm_gdbm_close(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _gdbm_gdbm_close_impl((gdbmobject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _gdbm_gdbm_close_impl((gdbmobject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_gdbm_gdbm_keys__doc__, @@ -109,11 +118,18 @@ _gdbm_gdbm_keys_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_keys(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "keys() takes no arguments"); - return NULL; + goto exit; } - return _gdbm_gdbm_keys_impl((gdbmobject *)self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _gdbm_gdbm_keys_impl((gdbmobject *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_gdbm_gdbm_firstkey__doc__, @@ -122,9 +138,9 @@ PyDoc_STRVAR(_gdbm_gdbm_firstkey__doc__, "\n" "Return the starting key for the traversal.\n" "\n" -"It\'s possible to loop over every key in the database using this method\n" -"and the nextkey() method. The traversal is ordered by GDBM\'s internal\n" -"hash values, and won\'t be sorted by the key values."); +"It\'s possible to loop over every key in the database using this\n" +"method and the nextkey() method. The traversal is ordered by GDBM\'s\n" +"internal hash values, and won\'t be sorted by the key values."); #define _GDBM_GDBM_FIRSTKEY_METHODDEF \ {"firstkey", _PyCFunction_CAST(_gdbm_gdbm_firstkey), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _gdbm_gdbm_firstkey__doc__}, @@ -135,11 +151,18 @@ _gdbm_gdbm_firstkey_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_firstkey(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "firstkey() takes no arguments"); - return NULL; + goto exit; } - return _gdbm_gdbm_firstkey_impl((gdbmobject *)self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _gdbm_gdbm_firstkey_impl((gdbmobject *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_gdbm_gdbm_nextkey__doc__, @@ -148,8 +171,8 @@ PyDoc_STRVAR(_gdbm_gdbm_nextkey__doc__, "\n" "Returns the key that follows key in the traversal.\n" "\n" -"The following code prints every key in the database db, without having\n" -"to create a list in memory that contains them all:\n" +"The following code prints every key in the database db, without\n" +"having to create a list in memory that contains them all:\n" "\n" " k = db.firstkey()\n" " while k is not None:\n" @@ -187,7 +210,9 @@ _gdbm_gdbm_nextkey(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ &key, &key_length)) { goto exit; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = _gdbm_gdbm_nextkey_impl((gdbmobject *)self, cls, key, key_length); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -201,9 +226,9 @@ PyDoc_STRVAR(_gdbm_gdbm_reorganize__doc__, "\n" "If you have carried out a lot of deletions and would like to shrink\n" "the space used by the GDBM file, this routine will reorganize the\n" -"database. GDBM will not shorten the length of a database file except\n" -"by using this reorganization; otherwise, deleted file space will be\n" -"kept and reused as new (key,value) pairs are added."); +"database. GDBM will not shorten the length of a database file\n" +"except by using this reorganization; otherwise, deleted file space\n" +"will be kept and reused as new (key,value) pairs are added."); #define _GDBM_GDBM_REORGANIZE_METHODDEF \ {"reorganize", _PyCFunction_CAST(_gdbm_gdbm_reorganize), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _gdbm_gdbm_reorganize__doc__}, @@ -214,11 +239,18 @@ _gdbm_gdbm_reorganize_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_reorganize(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "reorganize() takes no arguments"); - return NULL; + goto exit; } - return _gdbm_gdbm_reorganize_impl((gdbmobject *)self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _gdbm_gdbm_reorganize_impl((gdbmobject *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_gdbm_gdbm_sync__doc__, @@ -239,11 +271,18 @@ _gdbm_gdbm_sync_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_sync(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "sync() takes no arguments"); - return NULL; + goto exit; } - return _gdbm_gdbm_sync_impl((gdbmobject *)self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _gdbm_gdbm_sync_impl((gdbmobject *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_gdbm_gdbm_clear__doc__, @@ -261,11 +300,18 @@ _gdbm_gdbm_clear_impl(gdbmobject *self, PyTypeObject *cls); static PyObject * _gdbm_gdbm_clear(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "clear() takes no arguments"); - return NULL; + goto exit; } - return _gdbm_gdbm_clear_impl((gdbmobject *)self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _gdbm_gdbm_clear_impl((gdbmobject *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(dbmopen__doc__, @@ -343,4 +389,4 @@ dbmopen(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=d974cb39e4ee5d67 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=429b5db24568292e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h index 59ab46ca3f0978b..0e2f71d7e732479 100644 --- a/Modules/clinic/_hashopenssl.c.h +++ b/Modules/clinic/_hashopenssl.c.h @@ -232,8 +232,8 @@ EVPXOF_hexdigest(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje #endif /* defined(PY_OPENSSL_HAS_SHAKE) */ -PyDoc_STRVAR(EVP_new__doc__, -"new($module, /, name, string=b\'\', *, usedforsecurity=True)\n" +PyDoc_STRVAR(_hashlib_new__doc__, +"new($module, /, name, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new hash object using the named algorithm.\n" @@ -243,20 +243,20 @@ PyDoc_STRVAR(EVP_new__doc__, "\n" "The MD5 and SHA1 algorithms are always supported."); -#define EVP_NEW_METHODDEF \ - {"new", _PyCFunction_CAST(EVP_new), METH_FASTCALL|METH_KEYWORDS, EVP_new__doc__}, +#define _HASHLIB_NEW_METHODDEF \ + {"new", _PyCFunction_CAST(_hashlib_new), METH_FASTCALL|METH_KEYWORDS, _hashlib_new__doc__}, static PyObject * -EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj, - int usedforsecurity); +_hashlib_new_impl(PyObject *module, const char *name, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * -EVP_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_hashlib_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 3 + #define NUM_KEYWORDS 4 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -265,7 +265,7 @@ EVP_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(name), &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(name), &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -274,30 +274,43 @@ EVP_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"name", "string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"name", "data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "new", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[3]; + PyObject *argsbuf[4]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; - PyObject *name_obj; - PyObject *data_obj = NULL; + const char *name; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!args) { goto exit; } - name_obj = args[0]; + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("new", "argument 'name'", "str", args[0]); + goto exit; + } + Py_ssize_t name_length; + name = PyUnicode_AsUTF8AndSize(args[0], &name_length); + if (name == NULL) { + goto exit; + } + if (strlen(name) != (size_t)name_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } if (!noptargs) { goto skip_optional_pos; } if (args[1]) { - data_obj = args[1]; + data = args[1]; if (!--noptargs) { goto skip_optional_pos; } @@ -306,19 +319,25 @@ EVP_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwn if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[2]); - if (usedforsecurity < 0) { - goto exit; + if (args[2]) { + usedforsecurity = PyObject_IsTrue(args[2]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[3]; skip_optional_kwonly: - return_value = EVP_new_impl(module, name_obj, data_obj, usedforsecurity); + return_value = _hashlib_new_impl(module, name, data, usedforsecurity, string); exit: return return_value; } PyDoc_STRVAR(_hashlib_openssl_md5__doc__, -"openssl_md5($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_md5($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Returns a md5 hash object; optionally initialized with a string"); @@ -327,8 +346,8 @@ PyDoc_STRVAR(_hashlib_openssl_md5__doc__, {"openssl_md5", _PyCFunction_CAST(_hashlib_openssl_md5), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_md5__doc__}, static PyObject * -_hashlib_openssl_md5_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_md5_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -336,7 +355,7 @@ _hashlib_openssl_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -345,7 +364,7 @@ _hashlib_openssl_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -354,17 +373,18 @@ _hashlib_openssl_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_md5", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -375,7 +395,7 @@ _hashlib_openssl_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -384,19 +404,25 @@ _hashlib_openssl_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_md5_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_md5_impl(module, data, usedforsecurity, string); exit: return return_value; } PyDoc_STRVAR(_hashlib_openssl_sha1__doc__, -"openssl_sha1($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha1($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Returns a sha1 hash object; optionally initialized with a string"); @@ -405,8 +431,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha1__doc__, {"openssl_sha1", _PyCFunction_CAST(_hashlib_openssl_sha1), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha1__doc__}, static PyObject * -_hashlib_openssl_sha1_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha1_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -414,7 +440,7 @@ _hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -423,7 +449,7 @@ _hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -432,17 +458,18 @@ _hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha1", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -453,7 +480,7 @@ _hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -462,19 +489,26 @@ _hashlib_openssl_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha1_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha1_impl(module, data, usedforsecurity, string); exit: return return_value; } PyDoc_STRVAR(_hashlib_openssl_sha224__doc__, -"openssl_sha224($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha224($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha224 hash object; optionally initialized with a string"); @@ -483,8 +517,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha224__doc__, {"openssl_sha224", _PyCFunction_CAST(_hashlib_openssl_sha224), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha224__doc__}, static PyObject * -_hashlib_openssl_sha224_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha224_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -492,7 +526,7 @@ _hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -501,7 +535,7 @@ _hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t narg } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -510,17 +544,18 @@ _hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t narg # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha224", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -531,7 +566,7 @@ _hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t narg goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -540,19 +575,26 @@ _hashlib_openssl_sha224(PyObject *module, PyObject *const *args, Py_ssize_t narg if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha224_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha224_impl(module, data, usedforsecurity, string); exit: return return_value; } PyDoc_STRVAR(_hashlib_openssl_sha256__doc__, -"openssl_sha256($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha256($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha256 hash object; optionally initialized with a string"); @@ -561,8 +603,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha256__doc__, {"openssl_sha256", _PyCFunction_CAST(_hashlib_openssl_sha256), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha256__doc__}, static PyObject * -_hashlib_openssl_sha256_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha256_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -570,7 +612,7 @@ _hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -579,7 +621,7 @@ _hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t narg } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -588,17 +630,18 @@ _hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t narg # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha256", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -609,7 +652,7 @@ _hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t narg goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -618,19 +661,26 @@ _hashlib_openssl_sha256(PyObject *module, PyObject *const *args, Py_ssize_t narg if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha256_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha256_impl(module, data, usedforsecurity, string); exit: return return_value; } PyDoc_STRVAR(_hashlib_openssl_sha384__doc__, -"openssl_sha384($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha384($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha384 hash object; optionally initialized with a string"); @@ -639,8 +689,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha384__doc__, {"openssl_sha384", _PyCFunction_CAST(_hashlib_openssl_sha384), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha384__doc__}, static PyObject * -_hashlib_openssl_sha384_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha384_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -648,7 +698,7 @@ _hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -657,7 +707,7 @@ _hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t narg } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -666,17 +716,18 @@ _hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t narg # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha384", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -687,7 +738,7 @@ _hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t narg goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -696,19 +747,26 @@ _hashlib_openssl_sha384(PyObject *module, PyObject *const *args, Py_ssize_t narg if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha384_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha384_impl(module, data, usedforsecurity, string); exit: return return_value; } PyDoc_STRVAR(_hashlib_openssl_sha512__doc__, -"openssl_sha512($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha512($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha512 hash object; optionally initialized with a string"); @@ -717,8 +775,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha512__doc__, {"openssl_sha512", _PyCFunction_CAST(_hashlib_openssl_sha512), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha512__doc__}, static PyObject * -_hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha512_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -726,7 +784,7 @@ _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t narg PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -735,7 +793,7 @@ _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t narg } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -744,17 +802,18 @@ _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t narg # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha512", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -765,7 +824,7 @@ _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t narg goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -774,12 +833,18 @@ _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t narg if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha512_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha512_impl(module, data, usedforsecurity, string); exit: return return_value; @@ -788,7 +853,8 @@ _hashlib_openssl_sha512(PyObject *module, PyObject *const *args, Py_ssize_t narg #if defined(PY_OPENSSL_HAS_SHA3) PyDoc_STRVAR(_hashlib_openssl_sha3_224__doc__, -"openssl_sha3_224($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha3_224($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha3-224 hash object; optionally initialized with a string"); @@ -797,8 +863,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha3_224__doc__, {"openssl_sha3_224", _PyCFunction_CAST(_hashlib_openssl_sha3_224), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha3_224__doc__}, static PyObject * -_hashlib_openssl_sha3_224_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha3_224_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha3_224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -806,7 +872,7 @@ _hashlib_openssl_sha3_224(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -815,7 +881,7 @@ _hashlib_openssl_sha3_224(PyObject *module, PyObject *const *args, Py_ssize_t na } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -824,17 +890,18 @@ _hashlib_openssl_sha3_224(PyObject *module, PyObject *const *args, Py_ssize_t na # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha3_224", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -845,7 +912,7 @@ _hashlib_openssl_sha3_224(PyObject *module, PyObject *const *args, Py_ssize_t na goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -854,12 +921,18 @@ _hashlib_openssl_sha3_224(PyObject *module, PyObject *const *args, Py_ssize_t na if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha3_224_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha3_224_impl(module, data, usedforsecurity, string); exit: return return_value; @@ -870,7 +943,8 @@ _hashlib_openssl_sha3_224(PyObject *module, PyObject *const *args, Py_ssize_t na #if defined(PY_OPENSSL_HAS_SHA3) PyDoc_STRVAR(_hashlib_openssl_sha3_256__doc__, -"openssl_sha3_256($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha3_256($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha3-256 hash object; optionally initialized with a string"); @@ -879,8 +953,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha3_256__doc__, {"openssl_sha3_256", _PyCFunction_CAST(_hashlib_openssl_sha3_256), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha3_256__doc__}, static PyObject * -_hashlib_openssl_sha3_256_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha3_256_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha3_256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -888,7 +962,7 @@ _hashlib_openssl_sha3_256(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -897,7 +971,7 @@ _hashlib_openssl_sha3_256(PyObject *module, PyObject *const *args, Py_ssize_t na } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -906,17 +980,18 @@ _hashlib_openssl_sha3_256(PyObject *module, PyObject *const *args, Py_ssize_t na # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha3_256", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -927,7 +1002,7 @@ _hashlib_openssl_sha3_256(PyObject *module, PyObject *const *args, Py_ssize_t na goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -936,12 +1011,18 @@ _hashlib_openssl_sha3_256(PyObject *module, PyObject *const *args, Py_ssize_t na if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha3_256_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha3_256_impl(module, data, usedforsecurity, string); exit: return return_value; @@ -952,7 +1033,8 @@ _hashlib_openssl_sha3_256(PyObject *module, PyObject *const *args, Py_ssize_t na #if defined(PY_OPENSSL_HAS_SHA3) PyDoc_STRVAR(_hashlib_openssl_sha3_384__doc__, -"openssl_sha3_384($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha3_384($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha3-384 hash object; optionally initialized with a string"); @@ -961,8 +1043,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha3_384__doc__, {"openssl_sha3_384", _PyCFunction_CAST(_hashlib_openssl_sha3_384), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha3_384__doc__}, static PyObject * -_hashlib_openssl_sha3_384_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha3_384_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha3_384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -970,7 +1052,7 @@ _hashlib_openssl_sha3_384(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -979,7 +1061,7 @@ _hashlib_openssl_sha3_384(PyObject *module, PyObject *const *args, Py_ssize_t na } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -988,17 +1070,18 @@ _hashlib_openssl_sha3_384(PyObject *module, PyObject *const *args, Py_ssize_t na # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha3_384", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -1009,7 +1092,7 @@ _hashlib_openssl_sha3_384(PyObject *module, PyObject *const *args, Py_ssize_t na goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -1018,12 +1101,18 @@ _hashlib_openssl_sha3_384(PyObject *module, PyObject *const *args, Py_ssize_t na if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha3_384_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha3_384_impl(module, data, usedforsecurity, string); exit: return return_value; @@ -1034,7 +1123,8 @@ _hashlib_openssl_sha3_384(PyObject *module, PyObject *const *args, Py_ssize_t na #if defined(PY_OPENSSL_HAS_SHA3) PyDoc_STRVAR(_hashlib_openssl_sha3_512__doc__, -"openssl_sha3_512($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_sha3_512($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a sha3-512 hash object; optionally initialized with a string"); @@ -1043,8 +1133,8 @@ PyDoc_STRVAR(_hashlib_openssl_sha3_512__doc__, {"openssl_sha3_512", _PyCFunction_CAST(_hashlib_openssl_sha3_512), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_sha3_512__doc__}, static PyObject * -_hashlib_openssl_sha3_512_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_sha3_512_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_sha3_512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -1052,7 +1142,7 @@ _hashlib_openssl_sha3_512(PyObject *module, PyObject *const *args, Py_ssize_t na PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -1061,7 +1151,7 @@ _hashlib_openssl_sha3_512(PyObject *module, PyObject *const *args, Py_ssize_t na } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1070,17 +1160,18 @@ _hashlib_openssl_sha3_512(PyObject *module, PyObject *const *args, Py_ssize_t na # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_sha3_512", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -1091,7 +1182,7 @@ _hashlib_openssl_sha3_512(PyObject *module, PyObject *const *args, Py_ssize_t na goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -1100,12 +1191,18 @@ _hashlib_openssl_sha3_512(PyObject *module, PyObject *const *args, Py_ssize_t na if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_sha3_512_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_sha3_512_impl(module, data, usedforsecurity, string); exit: return return_value; @@ -1116,7 +1213,8 @@ _hashlib_openssl_sha3_512(PyObject *module, PyObject *const *args, Py_ssize_t na #if defined(PY_OPENSSL_HAS_SHAKE) PyDoc_STRVAR(_hashlib_openssl_shake_128__doc__, -"openssl_shake_128($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_shake_128($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a shake-128 variable hash object; optionally initialized with a string"); @@ -1125,8 +1223,8 @@ PyDoc_STRVAR(_hashlib_openssl_shake_128__doc__, {"openssl_shake_128", _PyCFunction_CAST(_hashlib_openssl_shake_128), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_shake_128__doc__}, static PyObject * -_hashlib_openssl_shake_128_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_shake_128_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_shake_128(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -1134,7 +1232,7 @@ _hashlib_openssl_shake_128(PyObject *module, PyObject *const *args, Py_ssize_t n PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -1143,7 +1241,7 @@ _hashlib_openssl_shake_128(PyObject *module, PyObject *const *args, Py_ssize_t n } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1152,17 +1250,18 @@ _hashlib_openssl_shake_128(PyObject *module, PyObject *const *args, Py_ssize_t n # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_shake_128", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -1173,7 +1272,7 @@ _hashlib_openssl_shake_128(PyObject *module, PyObject *const *args, Py_ssize_t n goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -1182,12 +1281,18 @@ _hashlib_openssl_shake_128(PyObject *module, PyObject *const *args, Py_ssize_t n if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_shake_128_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_shake_128_impl(module, data, usedforsecurity, string); exit: return return_value; @@ -1198,7 +1303,8 @@ _hashlib_openssl_shake_128(PyObject *module, PyObject *const *args, Py_ssize_t n #if defined(PY_OPENSSL_HAS_SHAKE) PyDoc_STRVAR(_hashlib_openssl_shake_256__doc__, -"openssl_shake_256($module, /, string=b\'\', *, usedforsecurity=True)\n" +"openssl_shake_256($module, /, data=b\'\', *, usedforsecurity=True,\n" +" string=None)\n" "--\n" "\n" "Returns a shake-256 variable hash object; optionally initialized with a string"); @@ -1207,8 +1313,8 @@ PyDoc_STRVAR(_hashlib_openssl_shake_256__doc__, {"openssl_shake_256", _PyCFunction_CAST(_hashlib_openssl_shake_256), METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_shake_256__doc__}, static PyObject * -_hashlib_openssl_shake_256_impl(PyObject *module, PyObject *data_obj, - int usedforsecurity); +_hashlib_openssl_shake_256_impl(PyObject *module, PyObject *data, + int usedforsecurity, PyObject *string); static PyObject * _hashlib_openssl_shake_256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -1216,7 +1322,7 @@ _hashlib_openssl_shake_256(PyObject *module, PyObject *const *args, Py_ssize_t n PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -1225,7 +1331,7 @@ _hashlib_openssl_shake_256(PyObject *module, PyObject *const *args, Py_ssize_t n } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -1234,17 +1340,18 @@ _hashlib_openssl_shake_256(PyObject *module, PyObject *const *args, Py_ssize_t n # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "openssl_shake_256", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *data_obj = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -1255,7 +1362,7 @@ _hashlib_openssl_shake_256(PyObject *module, PyObject *const *args, Py_ssize_t n goto skip_optional_pos; } if (args[0]) { - data_obj = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -1264,12 +1371,18 @@ _hashlib_openssl_shake_256(PyObject *module, PyObject *const *args, Py_ssize_t n if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = args[2]; skip_optional_kwonly: - return_value = _hashlib_openssl_shake_256_impl(module, data_obj, usedforsecurity); + return_value = _hashlib_openssl_shake_256_impl(module, data, usedforsecurity, string); exit: return return_value; @@ -1746,8 +1859,8 @@ PyDoc_STRVAR(_hashlib_HMAC_hexdigest__doc__, "\n" "Return hexadecimal digest of the bytes passed to the update() method so far.\n" "\n" -"This may be used to exchange the value safely in email or other non-binary\n" -"environments."); +"This may be used to exchange the value safely in email or other\n" +"non-binary environments."); #define _HASHLIB_HMAC_HEXDIGEST_METHODDEF \ {"hexdigest", (PyCFunction)_hashlib_HMAC_hexdigest, METH_NOARGS, _hashlib_HMAC_hexdigest__doc__}, @@ -1768,8 +1881,8 @@ PyDoc_STRVAR(_hashlib_get_fips_mode__doc__, "Determine the OpenSSL FIPS mode of operation.\n" "\n" "For OpenSSL 3.0.0 and newer it returns the state of the default provider\n" -"in the default OSSL context. It\'s not quite the same as FIPS_mode() but good\n" -"enough for unittests.\n" +"in the default OSSL context. It\'s not quite the same as FIPS_mode() but\n" +"good enough for unittests.\n" "\n" "Effectively any non-zero return value indicates FIPS mode;\n" "values other than 1 may have additional significance."); @@ -1871,4 +1984,4 @@ _hashlib_compare_digest(PyObject *module, PyObject *const *args, Py_ssize_t narg #ifndef _HASHLIB_SCRYPT_METHODDEF #define _HASHLIB_SCRYPT_METHODDEF #endif /* !defined(_HASHLIB_SCRYPT_METHODDEF) */ -/*[clinic end generated code: output=2c78822e38be64a8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=3e8d61a057978436 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_heapqmodule.c.h b/Modules/clinic/_heapqmodule.c.h index 81d108627265ab2..f3b8256efc02216 100644 --- a/Modules/clinic/_heapqmodule.c.h +++ b/Modules/clinic/_heapqmodule.c.h @@ -2,6 +2,7 @@ preserve [clinic start generated code]*/ +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_heapq_heappush__doc__, @@ -32,7 +33,9 @@ _heapq_heappush(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappush_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -61,7 +64,9 @@ _heapq_heappop(PyObject *module, PyObject *arg) goto exit; } heap = arg; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappop_impl(module, heap); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -103,7 +108,9 @@ _heapq_heapreplace(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heapreplace_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -140,7 +147,9 @@ _heapq_heappushpop(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappushpop_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -169,7 +178,9 @@ _heapq_heapify(PyObject *module, PyObject *arg) goto exit; } heap = arg; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heapify_impl(module, heap); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -203,7 +214,9 @@ _heapq_heappush_max(PyObject *module, PyObject *const *args, Py_ssize_t nargs) } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappush_max_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -232,7 +245,9 @@ _heapq_heappop_max(PyObject *module, PyObject *arg) goto exit; } heap = arg; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappop_max_impl(module, heap); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -266,7 +281,9 @@ _heapq_heapreplace_max(PyObject *module, PyObject *const *args, Py_ssize_t nargs } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heapreplace_max_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -295,7 +312,9 @@ _heapq_heapify_max(PyObject *module, PyObject *arg) goto exit; } heap = arg; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heapify_max_impl(module, heap); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -307,8 +326,8 @@ PyDoc_STRVAR(_heapq_heappushpop_max__doc__, "\n" "Maxheap variant of heappushpop.\n" "\n" -"The combined action runs more efficiently than heappush_max() followed by\n" -"a separate call to heappop_max()."); +"The combined action runs more efficiently than heappush_max()\n" +"followed by a separate call to heappop_max()."); #define _HEAPQ_HEAPPUSHPOP_MAX_METHODDEF \ {"heappushpop_max", _PyCFunction_CAST(_heapq_heappushpop_max), METH_FASTCALL, _heapq_heappushpop_max__doc__}, @@ -332,9 +351,11 @@ _heapq_heappushpop_max(PyObject *module, PyObject *const *args, Py_ssize_t nargs } heap = args[0]; item = args[1]; + Py_BEGIN_CRITICAL_SECTION(heap); return_value = _heapq_heappushpop_max_impl(module, heap, item); + Py_END_CRITICAL_SECTION(); exit: return return_value; } -/*[clinic end generated code: output=f55d8595ce150c76 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=21e4f248ef6e83d6 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lsprof.c.h b/Modules/clinic/_lsprof.c.h index 2918a6bc7abe748..acb4aaf27e377f2 100644 --- a/Modules/clinic/_lsprof.c.h +++ b/Modules/clinic/_lsprof.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_lsprof_Profiler_getstats__doc__, @@ -45,11 +46,18 @@ _lsprof_Profiler_getstats_impl(ProfilerObject *self, PyTypeObject *cls); static PyObject * _lsprof_Profiler_getstats(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { + PyObject *return_value = NULL; + if (nargs || (kwnames && PyTuple_GET_SIZE(kwnames))) { PyErr_SetString(PyExc_TypeError, "getstats() takes no arguments"); - return NULL; + goto exit; } - return _lsprof_Profiler_getstats_impl((ProfilerObject *)self, cls); + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _lsprof_Profiler_getstats_impl((ProfilerObject *)self, cls); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; } PyDoc_STRVAR(_lsprof_Profiler__pystart_callback__doc__, @@ -76,7 +84,44 @@ _lsprof_Profiler__pystart_callback(PyObject *self, PyObject *const *args, Py_ssi } code = args[0]; instruction_offset = args[1]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _lsprof_Profiler__pystart_callback_impl((ProfilerObject *)self, code, instruction_offset); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(_lsprof_Profiler__pythrow_callback__doc__, +"_pythrow_callback($self, code, instruction_offset, exception, /)\n" +"--\n" +"\n"); + +#define _LSPROF_PROFILER__PYTHROW_CALLBACK_METHODDEF \ + {"_pythrow_callback", _PyCFunction_CAST(_lsprof_Profiler__pythrow_callback), METH_FASTCALL, _lsprof_Profiler__pythrow_callback__doc__}, + +static PyObject * +_lsprof_Profiler__pythrow_callback_impl(ProfilerObject *self, PyObject *code, + PyObject *instruction_offset, + PyObject *exception); + +static PyObject * +_lsprof_Profiler__pythrow_callback(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *code; + PyObject *instruction_offset; + PyObject *exception; + + if (!_PyArg_CheckPositional("_pythrow_callback", nargs, 3, 3)) { + goto exit; + } + code = args[0]; + instruction_offset = args[1]; + exception = args[2]; + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _lsprof_Profiler__pythrow_callback_impl((ProfilerObject *)self, code, instruction_offset, exception); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -110,7 +155,9 @@ _lsprof_Profiler__pyreturn_callback(PyObject *self, PyObject *const *args, Py_ss code = args[0]; instruction_offset = args[1]; retval = args[2]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _lsprof_Profiler__pyreturn_callback_impl((ProfilerObject *)self, code, instruction_offset, retval); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -145,7 +192,9 @@ _lsprof_Profiler__ccall_callback(PyObject *self, PyObject *const *args, Py_ssize instruction_offset = args[1]; callable = args[2]; self_arg = args[3]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _lsprof_Profiler__ccall_callback_impl((ProfilerObject *)self, code, instruction_offset, callable, self_arg); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -182,7 +231,9 @@ _lsprof_Profiler__creturn_callback(PyObject *self, PyObject *const *args, Py_ssi instruction_offset = args[1]; callable = args[2]; self_arg = args[3]; + Py_BEGIN_CRITICAL_SECTION(self); return_value = _lsprof_Profiler__creturn_callback_impl((ProfilerObject *)self, code, instruction_offset, callable, self_arg); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -266,7 +317,9 @@ _lsprof_Profiler_enable(PyObject *self, PyObject *const *args, Py_ssize_t nargs, goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = _lsprof_Profiler_enable_impl((ProfilerObject *)self, subcalls, builtins); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -287,7 +340,13 @@ _lsprof_Profiler_disable_impl(ProfilerObject *self); static PyObject * _lsprof_Profiler_disable(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _lsprof_Profiler_disable_impl((ProfilerObject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _lsprof_Profiler_disable_impl((ProfilerObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(_lsprof_Profiler_clear__doc__, @@ -305,7 +364,13 @@ _lsprof_Profiler_clear_impl(ProfilerObject *self); static PyObject * _lsprof_Profiler_clear(PyObject *self, PyObject *Py_UNUSED(ignored)) { - return _lsprof_Profiler_clear_impl((ProfilerObject *)self); + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _lsprof_Profiler_clear_impl((ProfilerObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; } PyDoc_STRVAR(profiler_init__doc__, @@ -411,4 +476,4 @@ profiler_init(PyObject *self, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=fe231309776df7a7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=af26a0b0ddcc3351 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_lzmamodule.c.h b/Modules/clinic/_lzmamodule.c.h index ebdc81a0dac2f05..bba107e8f806daf 100644 --- a/Modules/clinic/_lzmamodule.c.h +++ b/Modules/clinic/_lzmamodule.c.h @@ -74,18 +74,19 @@ PyDoc_STRVAR(_lzma_LZMADecompressor_decompress__doc__, "\n" "Decompress *data*, returning uncompressed data as bytes.\n" "\n" -"If *max_length* is nonnegative, returns at most *max_length* bytes of\n" -"decompressed data. If this limit is reached and further output can be\n" -"produced, *self.needs_input* will be set to ``False``. In this case, the next\n" -"call to *decompress()* may provide *data* as b\'\' to obtain more of the output.\n" +"If *max_length* is nonnegative, returns at most *max_length* bytes\n" +"of decompressed data. If this limit is reached and further output\n" +"can be produced, *self.needs_input* will be set to ``False``. In\n" +"this case, the next call to *decompress()* may provide *data* as b\'\'\n" +"to obtain more of the output.\n" "\n" -"If all of the input data was decompressed and returned (either because this\n" -"was less than *max_length* bytes, or because *max_length* was negative),\n" -"*self.needs_input* will be set to True.\n" +"If all of the input data was decompressed and returned (either\n" +"because this was less than *max_length* bytes, or because\n" +"*max_length* was negative), *self.needs_input* will be set to True.\n" "\n" -"Attempting to decompress data after the end of stream is reached raises an\n" -"EOFError. Any data found after the end of the stream is ignored and saved in\n" -"the unused_data attribute."); +"Attempting to decompress data after the end of stream is reached\n" +"raises an EOFError. Any data found after the end of the stream is\n" +"ignored and saved in the unused_data attribute."); #define _LZMA_LZMADECOMPRESSOR_DECOMPRESS_METHODDEF \ {"decompress", _PyCFunction_CAST(_lzma_LZMADecompressor_decompress), METH_FASTCALL|METH_KEYWORDS, _lzma_LZMADecompressor_decompress__doc__}, @@ -333,4 +334,4 @@ _lzma__decode_filter_properties(PyObject *module, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=6386084cb43d2533 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ffc6d673d858048c input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_posixsubprocess.c.h b/Modules/clinic/_posixsubprocess.c.h index d52629cf6eaa5b7..e7e9707f182a2ed 100644 --- a/Modules/clinic/_posixsubprocess.c.h +++ b/Modules/clinic/_posixsubprocess.c.h @@ -14,15 +14,15 @@ PyDoc_STRVAR(subprocess_fork_exec__doc__, "\n" "Spawn a fresh new child process.\n" "\n" -"Fork a child process, close parent file descriptors as appropriate in the\n" -"child and duplicate the few that are needed before calling exec() in the\n" -"child process.\n" +"Fork a child process, close parent file descriptors as appropriate in\n" +"the child and duplicate the few that are needed before calling exec() in\n" +"the child process.\n" "\n" -"If close_fds is True, close file descriptors 3 and higher, except those listed\n" -"in the sorted tuple pass_fds.\n" +"If close_fds is True, close file descriptors 3 and higher, except those\n" +"listed in the sorted tuple pass_fds.\n" "\n" -"The preexec_fn, if supplied, will be called immediately before closing file\n" -"descriptors and exec.\n" +"The preexec_fn, if supplied, will be called immediately before closing\n" +"file descriptors and exec.\n" "\n" "WARNING: preexec_fn is NOT SAFE if your application uses threads.\n" " It may trigger infrequent, difficult to debug deadlocks.\n" @@ -150,4 +150,4 @@ subprocess_fork_exec(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=942bc2748a9c2785 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=138941c284792aa1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h index 1751d68716ba5fa..483b646b67ba379 100644 --- a/Modules/clinic/_queuemodule.c.h +++ b/Modules/clinic/_queuemodule.c.h @@ -44,8 +44,9 @@ PyDoc_STRVAR(_queue_SimpleQueue_put__doc__, "\n" "Put the item on the queue.\n" "\n" -"The optional \'block\' and \'timeout\' arguments are ignored, as this method\n" -"never blocks. They are provided for compatibility with the Queue class."); +"The optional \'block\' and \'timeout\' arguments are ignored, as this\n" +"method never blocks. They are provided for compatibility with the\n" +"Queue class."); #define _QUEUE_SIMPLEQUEUE_PUT_METHODDEF \ {"put", _PyCFunction_CAST(_queue_SimpleQueue_put), METH_FASTCALL|METH_KEYWORDS, _queue_SimpleQueue_put__doc__}, @@ -188,10 +189,11 @@ PyDoc_STRVAR(_queue_SimpleQueue_get__doc__, "\n" "Remove and return an item from the queue.\n" "\n" -"If optional args \'block\' is true and \'timeout\' is None (the default),\n" -"block if necessary until an item is available. If \'timeout\' is\n" -"a non-negative number, it blocks at most \'timeout\' seconds and raises\n" -"the Empty exception if no item was available within that time.\n" +"If optional args \'block\' is true and \'timeout\' is None (the\n" +"default), block if necessary until an item is available. If\n" +"\'timeout\' is a non-negative number, it blocks at most \'timeout\'\n" +"seconds and raises the Empty exception if no item was available\n" +"within that time.\n" "Otherwise (\'block\' is false), return an item if one is immediately\n" "available, else raise the Empty exception (\'timeout\' is ignored\n" "in that case)."); @@ -358,4 +360,4 @@ _queue_SimpleQueue_qsize(PyObject *self, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=1d3efe9df89997cf input=a9049054013a1b77]*/ +/*[clinic end generated code: output=58cfee868040d297 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_randommodule.c.h b/Modules/clinic/_randommodule.c.h index 1e989e970c9de58..ca9cad7a572dadf 100644 --- a/Modules/clinic/_randommodule.c.h +++ b/Modules/clinic/_randommodule.c.h @@ -3,6 +3,7 @@ preserve [clinic start generated code]*/ #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_long.h" // _PyLong_UInt64_Converter() #include "pycore_modsupport.h" // _PyArg_CheckPositional() PyDoc_STRVAR(_random_Random_random__doc__, @@ -124,16 +125,15 @@ PyDoc_STRVAR(_random_Random_getrandbits__doc__, {"getrandbits", (PyCFunction)_random_Random_getrandbits, METH_O, _random_Random_getrandbits__doc__}, static PyObject * -_random_Random_getrandbits_impl(RandomObject *self, int k); +_random_Random_getrandbits_impl(RandomObject *self, uint64_t k); static PyObject * _random_Random_getrandbits(PyObject *self, PyObject *arg) { PyObject *return_value = NULL; - int k; + uint64_t k; - k = PyLong_AsInt(arg); - if (k == -1 && PyErr_Occurred()) { + if (!_PyLong_UInt64_Converter(arg, &k)) { goto exit; } Py_BEGIN_CRITICAL_SECTION(self); @@ -143,4 +143,35 @@ _random_Random_getrandbits(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=4458b5a69201ebea input=a9049054013a1b77]*/ + +static int +random_init_impl(RandomObject *self, PyObject *seed); + +static int +random_init(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + PyTypeObject *base_tp = (PyTypeObject *)_randomstate_type(Py_TYPE(self))->Random_Type; + PyObject *seed = NULL; + + if ((Py_IS_TYPE(self, base_tp) || + Py_TYPE(self)->tp_new == base_tp->tp_new) && + !_PyArg_NoKeywords("Random", kwargs)) { + goto exit; + } + if (!_PyArg_CheckPositional("Random", PyTuple_GET_SIZE(args), 0, 1)) { + goto exit; + } + if (PyTuple_GET_SIZE(args) < 1) { + goto skip_optional; + } + seed = PyTuple_GET_ITEM(args, 0); +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = random_init_impl((RandomObject *)self, seed); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} +/*[clinic end generated code: output=ec95f7df0c3f3c19 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_remote_debugging_module.c.h b/Modules/clinic/_remote_debugging_module.c.h new file mode 100644 index 000000000000000..f6a51cdba6b4013 --- /dev/null +++ b/Modules/clinic/_remote_debugging_module.c.h @@ -0,0 +1,291 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) +# include "pycore_gc.h" // PyGC_Head +# include "pycore_runtime.h" // _Py_ID() +#endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() + +PyDoc_STRVAR(_remote_debugging_RemoteUnwinder___init____doc__, +"RemoteUnwinder(pid, *, all_threads=False, only_active_thread=False,\n" +" debug=False)\n" +"--\n" +"\n" +"Initialize a new RemoteUnwinder object for debugging a remote Python process.\n" +"\n" +"Args:\n" +" pid: Process ID of the target Python process to debug\n" +" all_threads: If True, initialize state for all threads in the process.\n" +" If False, only initialize for the main thread.\n" +" only_active_thread: If True, only sample the thread holding the GIL.\n" +" Cannot be used together with all_threads=True.\n" +" debug: If True, chain exceptions to explain the sequence of events that\n" +" lead to the exception.\n" +"\n" +"The RemoteUnwinder provides functionality to inspect and debug a running Python\n" +"process, including examining thread states, stack frames and other runtime data.\n" +"\n" +"Raises:\n" +" PermissionError: If access to the target process is denied\n" +" OSError: If unable to attach to the target process or access its memory\n" +" RuntimeError: If unable to read debug information from the target process\n" +" ValueError: If both all_threads and only_active_thread are True"); + +static int +_remote_debugging_RemoteUnwinder___init___impl(RemoteUnwinderObject *self, + int pid, int all_threads, + int only_active_thread, + int debug); + +static int +_remote_debugging_RemoteUnwinder___init__(PyObject *self, PyObject *args, PyObject *kwargs) +{ + int return_value = -1; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 4 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(pid), &_Py_ID(all_threads), &_Py_ID(only_active_thread), &_Py_ID(debug), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"pid", "all_threads", "only_active_thread", "debug", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "RemoteUnwinder", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[4]; + PyObject * const *fastargs; + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; + int pid; + int all_threads = 0; + int only_active_thread = 0; + int debug = 0; + + fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!fastargs) { + goto exit; + } + pid = PyLong_AsInt(fastargs[0]); + if (pid == -1 && PyErr_Occurred()) { + goto exit; + } + if (!noptargs) { + goto skip_optional_kwonly; + } + if (fastargs[1]) { + all_threads = PyObject_IsTrue(fastargs[1]); + if (all_threads < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + if (fastargs[2]) { + only_active_thread = PyObject_IsTrue(fastargs[2]); + if (only_active_thread < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } + } + debug = PyObject_IsTrue(fastargs[3]); + if (debug < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _remote_debugging_RemoteUnwinder___init___impl((RemoteUnwinderObject *)self, pid, all_threads, only_active_thread, debug); + +exit: + return return_value; +} + +PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_stack_trace__doc__, +"get_stack_trace($self, /)\n" +"--\n" +"\n" +"Returns a list of stack traces for threads in the target process.\n" +"\n" +"Each element in the returned list is a tuple of (thread_id, frame_list), where:\n" +"- thread_id is the OS thread identifier\n" +"- frame_list is a list of tuples (function_name, filename, line_number) representing\n" +" the Python stack frames for that thread, ordered from most recent to oldest\n" +"\n" +"The threads returned depend on the initialization parameters:\n" +"- If only_active_thread was True: returns only the thread holding the GIL\n" +"- If all_threads was True: returns all threads\n" +"- Otherwise: returns only the main thread\n" +"\n" +"Example:\n" +" [\n" +" (1234, [\n" +" (\'process_data\', \'worker.py\', 127),\n" +" (\'run_worker\', \'worker.py\', 45),\n" +" (\'main\', \'app.py\', 23)\n" +" ]),\n" +" (1235, [\n" +" (\'handle_request\', \'server.py\', 89),\n" +" (\'serve_forever\', \'server.py\', 52)\n" +" ])\n" +" ]\n" +"\n" +"Raises:\n" +" RuntimeError: If there is an error copying memory from the target process\n" +" OSError: If there is an error accessing the target process\n" +" PermissionError: If access to the target process is denied\n" +" UnicodeDecodeError: If there is an error decoding strings from the target process"); + +#define _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_STACK_TRACE_METHODDEF \ + {"get_stack_trace", (PyCFunction)_remote_debugging_RemoteUnwinder_get_stack_trace, METH_NOARGS, _remote_debugging_RemoteUnwinder_get_stack_trace__doc__}, + +static PyObject * +_remote_debugging_RemoteUnwinder_get_stack_trace_impl(RemoteUnwinderObject *self); + +static PyObject * +_remote_debugging_RemoteUnwinder_get_stack_trace(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _remote_debugging_RemoteUnwinder_get_stack_trace_impl((RemoteUnwinderObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_all_awaited_by__doc__, +"get_all_awaited_by($self, /)\n" +"--\n" +"\n" +"Get all tasks and their awaited_by relationships from the remote process.\n" +"\n" +"This provides a tree structure showing which tasks are waiting for other tasks.\n" +"\n" +"For each task, returns:\n" +"1. The call stack frames leading to where the task is currently executing\n" +"2. The name of the task\n" +"3. A list of tasks that this task is waiting for, with their own frames/names/etc\n" +"\n" +"Returns a list of [frames, task_name, subtasks] where:\n" +"- frames: List of (func_name, filename, lineno) showing the call stack\n" +"- task_name: String identifier for the task\n" +"- subtasks: List of tasks being awaited by this task, in same format\n" +"\n" +"Raises:\n" +" RuntimeError: If AsyncioDebug section is not available in the remote process\n" +" MemoryError: If memory allocation fails\n" +" OSError: If reading from the remote process fails\n" +"\n" +"Example output:\n" +"[\n" +" [\n" +" [(\"c5\", \"script.py\", 10), (\"c4\", \"script.py\", 14)],\n" +" \"c2_root\",\n" +" [\n" +" [\n" +" [(\"c1\", \"script.py\", 23)],\n" +" \"sub_main_2\",\n" +" [...]\n" +" ],\n" +" [...]\n" +" ]\n" +" ]\n" +"]"); + +#define _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_ALL_AWAITED_BY_METHODDEF \ + {"get_all_awaited_by", (PyCFunction)_remote_debugging_RemoteUnwinder_get_all_awaited_by, METH_NOARGS, _remote_debugging_RemoteUnwinder_get_all_awaited_by__doc__}, + +static PyObject * +_remote_debugging_RemoteUnwinder_get_all_awaited_by_impl(RemoteUnwinderObject *self); + +static PyObject * +_remote_debugging_RemoteUnwinder_get_all_awaited_by(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _remote_debugging_RemoteUnwinder_get_all_awaited_by_impl((RemoteUnwinderObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(_remote_debugging_RemoteUnwinder_get_async_stack_trace__doc__, +"get_async_stack_trace($self, /)\n" +"--\n" +"\n" +"Get the currently running async tasks and their dependency graphs from the remote process.\n" +"\n" +"This returns information about running tasks and all tasks that are waiting for them,\n" +"forming a complete dependency graph for each thread\'s active task.\n" +"\n" +"For each thread with a running task, returns the running task plus all tasks that\n" +"transitively depend on it (tasks waiting for the running task, tasks waiting for\n" +"those tasks, etc.).\n" +"\n" +"Returns a list of per-thread results, where each thread result contains:\n" +"- Thread ID\n" +"- List of task information for the running task and all its waiters\n" +"\n" +"Each task info contains:\n" +"- Task ID (memory address)\n" +"- Task name\n" +"- Call stack frames: List of (func_name, filename, lineno)\n" +"- List of tasks waiting for this task (recursive structure)\n" +"\n" +"Raises:\n" +" RuntimeError: If AsyncioDebug section is not available in the target process\n" +" MemoryError: If memory allocation fails\n" +" OSError: If reading from the remote process fails\n" +"\n" +"Example output (similar structure to get_all_awaited_by but only for running tasks):\n" +"[\n" +" (140234, [\n" +" (4345585712, \'main_task\',\n" +" [(\"run_server\", \"server.py\", 127), (\"main\", \"app.py\", 23)],\n" +" [\n" +" (4345585800, \'worker_1\', [...], [...]),\n" +" (4345585900, \'worker_2\', [...], [...])\n" +" ])\n" +" ])\n" +"]"); + +#define _REMOTE_DEBUGGING_REMOTEUNWINDER_GET_ASYNC_STACK_TRACE_METHODDEF \ + {"get_async_stack_trace", (PyCFunction)_remote_debugging_RemoteUnwinder_get_async_stack_trace, METH_NOARGS, _remote_debugging_RemoteUnwinder_get_async_stack_trace__doc__}, + +static PyObject * +_remote_debugging_RemoteUnwinder_get_async_stack_trace_impl(RemoteUnwinderObject *self); + +static PyObject * +_remote_debugging_RemoteUnwinder_get_async_stack_trace(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = _remote_debugging_RemoteUnwinder_get_async_stack_trace_impl((RemoteUnwinderObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} +/*[clinic end generated code: output=0dd1e6e8bab2a8b1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_ssl.c.h b/Modules/clinic/_ssl.c.h index c6e2abd4d934749..0f45b2f52c50902 100644 --- a/Modules/clinic/_ssl.c.h +++ b/Modules/clinic/_ssl.c.h @@ -47,7 +47,7 @@ static PyObject * _ssl__test_decode_cert(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - PyObject *path; + PyObject *path = NULL; if (!PyUnicode_FSConverter(arg, &path)) { goto exit; @@ -55,6 +55,9 @@ _ssl__test_decode_cert(PyObject *module, PyObject *arg) return_value = _ssl__test_decode_cert_impl(module, path); exit: + /* Cleanup for path */ + Py_XDECREF(path); + return return_value; } @@ -261,9 +264,10 @@ _ssl__SSLSocket_compression(PyObject *self, PyObject *Py_UNUSED(ignored)) PyDoc_STRVAR(_ssl__SSLSocket_context__doc__, "This changes the context associated with the SSLSocket.\n" "\n" -"This is typically used from within a callback function set by the sni_callback\n" -"on the SSLContext to change the certificate information associated with the\n" -"SSLSocket before the cryptographic exchange handshake messages."); +"This is typically used from within a callback function set by the\n" +"sni_callback on the SSLContext to change the certificate information\n" +"associated with the SSLSocket before the cryptographic exchange\n" +"handshake messages."); #if defined(_ssl__SSLSocket_context_DOCSTR) # undef _ssl__SSLSocket_context_DOCSTR #endif @@ -580,8 +584,9 @@ PyDoc_STRVAR(_ssl__SSLSocket_get_channel_binding__doc__, "\n" "Get channel binding data for current connection.\n" "\n" -"Raise ValueError if the requested `cb_type` is not supported. Return bytes\n" -"of the data or None if the data is not available (e.g. before the handshake).\n" +"Raise ValueError if the requested `cb_type` is not supported.\n" +"Return bytes of the data or None if the data is not available (e.g.\n" +"before the handshake).\n" "Only \'tls-unique\' channel binding data from RFC 5929 is supported."); #define _SSL__SSLSOCKET_GET_CHANNEL_BINDING_METHODDEF \ @@ -1808,8 +1813,9 @@ _ssl__SSLContext_set_ecdh_curve(PyObject *self, PyObject *name) PyDoc_STRVAR(_ssl__SSLContext_sni_callback__doc__, "Set a callback that will be called when a server name is provided by the SSL/TLS client in the SNI extension.\n" "\n" -"If the argument is None then the callback is disabled. The method is called\n" -"with the SSLSocket, the server name as a string, and the SSLContext object.\n" +"If the argument is None then the callback is disabled. The method\n" +"is called with the SSLSocket, the server name as a string, and the\n" +"SSLContext object.\n" "\n" "See RFC 6066 for details of the SNI extension."); #if defined(_ssl__SSLContext_sni_callback_DOCSTR) @@ -1873,11 +1879,11 @@ PyDoc_STRVAR(_ssl__SSLContext_cert_store_stats__doc__, "\n" "Returns quantities of loaded X.509 certificates.\n" "\n" -"X.509 certificates with a CA extension and certificate revocation lists\n" -"inside the context\'s cert store.\n" +"X.509 certificates with a CA extension and certificate revocation\n" +"lists inside the context\'s cert store.\n" "\n" -"NOTE: Certificates in a capath directory aren\'t loaded unless they have\n" -"been used at least once."); +"NOTE: Certificates in a capath directory aren\'t loaded unless they\n" +"have been used at least once."); #define _SSL__SSLCONTEXT_CERT_STORE_STATS_METHODDEF \ {"cert_store_stats", (PyCFunction)_ssl__SSLContext_cert_store_stats, METH_NOARGS, _ssl__SSLContext_cert_store_stats__doc__}, @@ -1903,11 +1909,11 @@ PyDoc_STRVAR(_ssl__SSLContext_get_ca_certs__doc__, "\n" "Returns a list of dicts with information of loaded CA certs.\n" "\n" -"If the optional argument is True, returns a DER-encoded copy of the CA\n" -"certificate.\n" +"If the optional argument is True, returns a DER-encoded copy of the\n" +"CA certificate.\n" "\n" -"NOTE: Certificates in a capath directory aren\'t loaded unless they have\n" -"been used at least once."); +"NOTE: Certificates in a capath directory aren\'t loaded unless they\n" +"have been used at least once."); #define _SSL__SSLCONTEXT_GET_CA_CERTS_METHODDEF \ {"get_ca_certs", _PyCFunction_CAST(_ssl__SSLContext_get_ca_certs), METH_FASTCALL|METH_KEYWORDS, _ssl__SSLContext_get_ca_certs__doc__}, @@ -2568,8 +2574,8 @@ PyDoc_STRVAR(_ssl_RAND_status__doc__, "\n" "Returns True if the OpenSSL PRNG has been seeded with enough data and False if not.\n" "\n" -"It is necessary to seed the PRNG with RAND_add() on some platforms before\n" -"using the ssl() function."); +"It is necessary to seed the PRNG with RAND_add() on some platforms\n" +"before using the ssl() function."); #define _SSL_RAND_STATUS_METHODDEF \ {"RAND_status", (PyCFunction)_ssl_RAND_status, METH_NOARGS, _ssl_RAND_status__doc__}, @@ -2738,11 +2744,11 @@ PyDoc_STRVAR(_ssl_enum_certificates__doc__, "\n" "Retrieve certificates from Windows\' cert store.\n" "\n" -"store_name may be one of \'CA\', \'ROOT\' or \'MY\'. The system may provide\n" -"more cert storages, too. The function returns a list of (bytes,\n" -"encoding_type, trust) tuples. The encoding_type flag can be interpreted\n" -"with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The trust setting is either\n" -"a set of OIDs or the boolean True."); +"store_name may be one of \'CA\', \'ROOT\' or \'MY\'. The system may\n" +"provide more cert storages, too. The function returns a list of\n" +"(bytes, encoding_type, trust) tuples. The encoding_type flag can be\n" +"interpreted with X509_ASN_ENCODING or PKCS_7_ASN_ENCODING. The\n" +"trust setting is either a set of OIDs or the boolean True."); #define _SSL_ENUM_CERTIFICATES_METHODDEF \ {"enum_certificates", _PyCFunction_CAST(_ssl_enum_certificates), METH_FASTCALL|METH_KEYWORDS, _ssl_enum_certificates__doc__}, @@ -2900,4 +2906,4 @@ _ssl_enum_crls(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje #ifndef _SSL_ENUM_CRLS_METHODDEF #define _SSL_ENUM_CRLS_METHODDEF #endif /* !defined(_SSL_ENUM_CRLS_METHODDEF) */ -/*[clinic end generated code: output=748650909fec8906 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6b5d14b14e152522 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_struct.c.h b/Modules/clinic/_struct.c.h index e4eaadb91eb2312..7c5186ebb3e157f 100644 --- a/Modules/clinic/_struct.c.h +++ b/Modules/clinic/_struct.c.h @@ -117,7 +117,8 @@ PyDoc_STRVAR(Struct_unpack_from__doc__, "Values are unpacked according to the format string Struct.format.\n" "\n" "The buffer\'s size in bytes, starting at position offset, must be\n" -"at least Struct.size.\n" +"at least Struct.size. A negative offset counts from the end of the\n" +"buffer.\n" "\n" "See help(struct) for more on format strings."); @@ -331,6 +332,7 @@ PyDoc_STRVAR(unpack_from__doc__, "Return a tuple containing values unpacked according to the format string.\n" "\n" "The buffer\'s size, minus offset, must be at least calcsize(format).\n" +"A negative offset counts from the end of the buffer.\n" "\n" "See help(struct) for more on format strings."); @@ -458,4 +460,4 @@ iter_unpack(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return return_value; } -/*[clinic end generated code: output=caa7f36443e91cb9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=000e11080c1bc7ef input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 970528ce9ea46d7..b652634892c27f0 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -273,19 +273,19 @@ char_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; char a = 'A'; - char b = '\x07'; - char c = '\x08'; + char b = '\a'; + char c = '\b'; char d = '\t'; char e = '\n'; - char f = '\x0b'; - char g = '\x0c'; + char f = '\v'; + char g = '\f'; char h = '\r'; char i = '"'; char j = '\''; char k = '?'; char l = '\\'; - char m = '\x00'; - char n = '\xff'; + char m = '\0'; + char n = '\377'; if (!_PyArg_CheckPositional("char_converter", nargs, 0, 14)) { goto exit; @@ -860,7 +860,7 @@ unsigned_short_converter(PyObject *module, PyObject *const *args, Py_ssize_t nar } PyDoc_STRVAR(int_converter__doc__, -"int_converter($module, a=12, b=34, c=45, /)\n" +"int_converter($module, a=12, b=34, c=\'-\', /)\n" "--\n" "\n"); @@ -876,7 +876,7 @@ int_converter(PyObject *module, PyObject *const *args, Py_ssize_t nargs) PyObject *return_value = NULL; int a = 12; int b = 34; - int c = 45; + int c = '-'; if (!_PyArg_CheckPositional("int_converter", nargs, 0, 3)) { goto exit; @@ -4481,4 +4481,4 @@ _testclinic_TestClass_posonly_poskw_varpos_array_no_fastcall(PyObject *type, PyO exit: return return_value; } -/*[clinic end generated code: output=84ffc31f27215baa input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8af194d826d6740d input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testinternalcapi.c.h b/Modules/clinic/_testinternalcapi.c.h index 21f4ee3201e5bf3..85edc6fbb5802f0 100644 --- a/Modules/clinic/_testinternalcapi.c.h +++ b/Modules/clinic/_testinternalcapi.c.h @@ -92,7 +92,11 @@ PyDoc_STRVAR(_testinternalcapi_compiler_codegen__doc__, "compiler_codegen($module, /, ast, filename, optimize, compile_mode=0)\n" "--\n" "\n" -"Apply compiler code generation to an AST."); +"Apply compiler code generation to an AST.\n" +"\n" +"Return (instruction_sequence, metadata). metadata maps \"argcount\",\n" +"\"posonlyargcount\", \"kwonlyargcount\" to ints and \"consts\" to the list of\n" +"constants in LOAD_CONST index order (for use with optimize_cfg)."); #define _TESTINTERNALCAPI_COMPILER_CODEGEN_METHODDEF \ {"compiler_codegen", _PyCFunction_CAST(_testinternalcapi_compiler_codegen), METH_FASTCALL|METH_KEYWORDS, _testinternalcapi_compiler_codegen__doc__}, @@ -169,7 +173,10 @@ PyDoc_STRVAR(_testinternalcapi_optimize_cfg__doc__, "optimize_cfg($module, /, instructions, consts, nlocals)\n" "--\n" "\n" -"Apply compiler optimizations to an instruction list."); +"Apply compiler optimizations to an instruction list.\n" +"\n" +"consts must be a list aligned with LOAD_CONST opargs (the \"consts\" entry\n" +"from the metadata dict returned by compiler_codegen for the same unit)."); #define _TESTINTERNALCAPI_OPTIMIZE_CFG_METHODDEF \ {"optimize_cfg", _PyCFunction_CAST(_testinternalcapi_optimize_cfg), METH_FASTCALL|METH_KEYWORDS, _testinternalcapi_optimize_cfg__doc__}, @@ -392,4 +399,4 @@ get_next_dict_keys_version(PyObject *module, PyObject *Py_UNUSED(ignored)) { return get_next_dict_keys_version_impl(module); } -/*[clinic end generated code: output=fbd8b7e0cae8bac7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ecb5d7ac85b153fa input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testmultiphase.c.h b/Modules/clinic/_testmultiphase.c.h index 311b64094767110..a0e4c66451101ed 100644 --- a/Modules/clinic/_testmultiphase.c.h +++ b/Modules/clinic/_testmultiphase.c.h @@ -14,8 +14,8 @@ PyDoc_STRVAR(_testmultiphase_StateAccessType_get_defining_module__doc__, "\n" "Return the module of the defining class.\n" "\n" -"Also tests that result of PyType_GetModuleByDef matches defining_class\'s\n" -"module."); +"Also tests that result of PyType_GetModuleByDef matches\n" +"defining_class\'s module."); #define _TESTMULTIPHASE_STATEACCESSTYPE_GET_DEFINING_MODULE_METHODDEF \ {"get_defining_module", _PyCFunction_CAST(_testmultiphase_StateAccessType_get_defining_module), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _testmultiphase_StateAccessType_get_defining_module__doc__}, @@ -165,4 +165,4 @@ _testmultiphase_StateAccessType_get_count(PyObject *self, PyTypeObject *cls, PyO } return _testmultiphase_StateAccessType_get_count_impl((StateAccessTypeObject *)self, cls); } -/*[clinic end generated code: output=8eed2f14292ec986 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=aff91f6219a7baca input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_tkinter.c.h b/Modules/clinic/_tkinter.c.h index 352c2b9e3d410c4..3147cf7f9d2cff3 100644 --- a/Modules/clinic/_tkinter.c.h +++ b/Modules/clinic/_tkinter.c.h @@ -907,7 +907,8 @@ PyDoc_STRVAR(_tkinter_setbusywaitinterval__doc__, "\n" "Set the busy-wait interval in milliseconds between successive calls to Tcl_DoOneEvent in a threaded Python interpreter.\n" "\n" -"It should be set to a divisor of the maximum time between frames in an animation."); +"It should be set to a divisor of the maximum time between frames in\n" +"an animation."); #define _TKINTER_SETBUSYWAITINTERVAL_METHODDEF \ {"setbusywaitinterval", (PyCFunction)_tkinter_setbusywaitinterval, METH_O, _tkinter_setbusywaitinterval__doc__}, @@ -966,4 +967,4 @@ _tkinter_getbusywaitinterval(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #define _TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF #endif /* !defined(_TKINTER_TKAPP_DELETEFILEHANDLER_METHODDEF) */ -/*[clinic end generated code: output=052c067aa69237be input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c807adb73e305725 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_winapi.c.h b/Modules/clinic/_winapi.c.h index f8b623fca082d2e..bd8a64bb57391ca 100644 --- a/Modules/clinic/_winapi.c.h +++ b/Modules/clinic/_winapi.c.h @@ -2161,4 +2161,22 @@ _winapi_CopyFile2(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } -/*[clinic end generated code: output=6cd07628af447d0a input=a9049054013a1b77]*/ + +PyDoc_STRVAR(_winapi_GetTickCount64__doc__, +"GetTickCount64($module, /)\n" +"--\n" +"\n" +"Number of milliseconds that have elapsed since the system was started."); + +#define _WINAPI_GETTICKCOUNT64_METHODDEF \ + {"GetTickCount64", (PyCFunction)_winapi_GetTickCount64, METH_NOARGS, _winapi_GetTickCount64__doc__}, + +static PyObject * +_winapi_GetTickCount64_impl(PyObject *module); + +static PyObject * +_winapi_GetTickCount64(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return _winapi_GetTickCount64_impl(module); +} +/*[clinic end generated code: output=4a408e816118d0a6 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/arraymodule.c.h b/Modules/clinic/arraymodule.c.h index 97e5ca771f3a90b..823629fe55179ca 100644 --- a/Modules/clinic/arraymodule.c.h +++ b/Modules/clinic/arraymodule.c.h @@ -292,8 +292,8 @@ PyDoc_STRVAR(array_array_buffer_info__doc__, "\n" "Return a tuple (address, length) giving the current memory address and the length in items of the buffer used to hold array\'s contents.\n" "\n" -"The length should be multiplied by the itemsize attribute to calculate\n" -"the buffer length in bytes."); +"The length should be multiplied by the itemsize attribute to\n" +"calculate the buffer length in bytes."); #define ARRAY_ARRAY_BUFFER_INFO_METHODDEF \ {"buffer_info", (PyCFunction)array_array_buffer_info, METH_NOARGS, array_array_buffer_info__doc__}, @@ -335,8 +335,8 @@ PyDoc_STRVAR(array_array_byteswap__doc__, "\n" "Byteswap all items of the array.\n" "\n" -"If the items in the array are not 1, 2, 4, or 8 bytes in size, RuntimeError is\n" -"raised."); +"If the items in the array are not 1, 2, 4, or 8 bytes in size,\n" +"RuntimeError is raised."); #define ARRAY_ARRAY_BYTESWAP_METHODDEF \ {"byteswap", (PyCFunction)array_array_byteswap, METH_NOARGS, array_array_byteswap__doc__}, @@ -566,9 +566,9 @@ PyDoc_STRVAR(array_array_fromunicode__doc__, "\n" "Extends this array with data from the unicode string ustr.\n" "\n" -"The array must be a unicode type array; otherwise a ValueError is raised.\n" -"Use array.frombytes(ustr.encode(...)) to append Unicode data to an array of\n" -"some other type."); +"The array must be a unicode type array; otherwise a ValueError is\n" +"raised. Use array.frombytes(ustr.encode(...)) to append Unicode\n" +"data to an array of some other type."); #define ARRAY_ARRAY_FROMUNICODE_METHODDEF \ {"fromunicode", (PyCFunction)array_array_fromunicode, METH_O, array_array_fromunicode__doc__}, @@ -599,9 +599,10 @@ PyDoc_STRVAR(array_array_tounicode__doc__, "\n" "Extends this array with data from the unicode string ustr.\n" "\n" -"Convert the array to a unicode string. The array must be a unicode type array;\n" -"otherwise a ValueError is raised. Use array.tobytes().decode() to obtain a\n" -"unicode string from an array of some other type."); +"Convert the array to a unicode string. The array must be a unicode\n" +"type array; otherwise a ValueError is raised. Use\n" +"array.tobytes().decode() to obtain a unicode string from an array of\n" +"some other type."); #define ARRAY_ARRAY_TOUNICODE_METHODDEF \ {"tounicode", (PyCFunction)array_array_tounicode, METH_NOARGS, array_array_tounicode__doc__}, @@ -773,4 +774,4 @@ array_arrayiterator___setstate__(PyObject *self, PyObject *state) return return_value; } -/*[clinic end generated code: output=dd49451ac1cc3f39 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bdd96ce6f32596b1 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/blake2module.c.h b/Modules/clinic/blake2module.c.h index bb2e308574a50a4..556f344e34740bf 100644 --- a/Modules/clinic/blake2module.c.h +++ b/Modules/clinic/blake2module.c.h @@ -10,20 +10,21 @@ preserve #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(py_blake2b_new__doc__, -"blake2b(data=b\'\', /, *, digest_size=_blake2.blake2b.MAX_DIGEST_SIZE,\n" +"blake2b(data=b\'\', *, digest_size=_blake2.blake2b.MAX_DIGEST_SIZE,\n" " key=b\'\', salt=b\'\', person=b\'\', fanout=1, depth=1, leaf_size=0,\n" " node_offset=0, node_depth=0, inner_size=0, last_node=False,\n" -" usedforsecurity=True)\n" +" usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new BLAKE2b hash object."); static PyObject * -py_blake2b_new_impl(PyTypeObject *type, PyObject *data, int digest_size, +py_blake2b_new_impl(PyTypeObject *type, PyObject *data_obj, int digest_size, Py_buffer *key, Py_buffer *salt, Py_buffer *person, int fanout, int depth, unsigned long leaf_size, unsigned long long node_offset, int node_depth, - int inner_size, int last_node, int usedforsecurity); + int inner_size, int last_node, int usedforsecurity, + PyObject *string); static PyObject * py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -31,7 +32,7 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 12 + #define NUM_KEYWORDS 14 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -40,7 +41,7 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(digest_size), &_Py_ID(key), &_Py_ID(salt), &_Py_ID(person), &_Py_ID(fanout), &_Py_ID(depth), &_Py_ID(leaf_size), &_Py_ID(node_offset), &_Py_ID(node_depth), &_Py_ID(inner_size), &_Py_ID(last_node), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(digest_size), &_Py_ID(key), &_Py_ID(salt), &_Py_ID(person), &_Py_ID(fanout), &_Py_ID(depth), &_Py_ID(leaf_size), &_Py_ID(node_offset), &_Py_ID(node_depth), &_Py_ID(inner_size), &_Py_ID(last_node), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -49,22 +50,22 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "blake2b", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[13]; + PyObject *argsbuf[14]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; - PyObject *data = NULL; + PyObject *data_obj = NULL; int digest_size = HACL_HASH_BLAKE2B_OUT_BYTES; - Py_buffer key = {NULL, NULL}; - Py_buffer salt = {NULL, NULL}; - Py_buffer person = {NULL, NULL}; + Py_buffer key = {.buf = "", .obj = NULL, .len = 0}; + Py_buffer salt = {.buf = "", .obj = NULL, .len = 0}; + Py_buffer person = {.buf = "", .obj = NULL, .len = 0}; int fanout = 1; int depth = 1; unsigned long leaf_size = 0; @@ -73,18 +74,23 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int inner_size = 0; int last_node = 0; int usedforsecurity = 1; + PyObject *string = NULL; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } - if (nargs < 1) { - goto skip_optional_posonly; + if (!noptargs) { + goto skip_optional_pos; } - noptargs--; - data = fastargs[0]; -skip_optional_posonly: + if (fastargs[0]) { + data_obj = fastargs[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } @@ -182,12 +188,18 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - usedforsecurity = PyObject_IsTrue(fastargs[12]); - if (usedforsecurity < 0) { - goto exit; + if (fastargs[12]) { + usedforsecurity = PyObject_IsTrue(fastargs[12]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = fastargs[13]; skip_optional_kwonly: - return_value = py_blake2b_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); + return_value = py_blake2b_new_impl(type, data_obj, digest_size, &key, &salt, &person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity, string); exit: /* Cleanup for key */ @@ -207,20 +219,21 @@ py_blake2b_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } PyDoc_STRVAR(py_blake2s_new__doc__, -"blake2s(data=b\'\', /, *, digest_size=_blake2.blake2s.MAX_DIGEST_SIZE,\n" +"blake2s(data=b\'\', *, digest_size=_blake2.blake2s.MAX_DIGEST_SIZE,\n" " key=b\'\', salt=b\'\', person=b\'\', fanout=1, depth=1, leaf_size=0,\n" " node_offset=0, node_depth=0, inner_size=0, last_node=False,\n" -" usedforsecurity=True)\n" +" usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new BLAKE2s hash object."); static PyObject * -py_blake2s_new_impl(PyTypeObject *type, PyObject *data, int digest_size, +py_blake2s_new_impl(PyTypeObject *type, PyObject *data_obj, int digest_size, Py_buffer *key, Py_buffer *salt, Py_buffer *person, int fanout, int depth, unsigned long leaf_size, unsigned long long node_offset, int node_depth, - int inner_size, int last_node, int usedforsecurity); + int inner_size, int last_node, int usedforsecurity, + PyObject *string); static PyObject * py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -228,7 +241,7 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 12 + #define NUM_KEYWORDS 14 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -237,7 +250,7 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(digest_size), &_Py_ID(key), &_Py_ID(salt), &_Py_ID(person), &_Py_ID(fanout), &_Py_ID(depth), &_Py_ID(leaf_size), &_Py_ID(node_offset), &_Py_ID(node_depth), &_Py_ID(inner_size), &_Py_ID(last_node), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(digest_size), &_Py_ID(key), &_Py_ID(salt), &_Py_ID(person), &_Py_ID(fanout), &_Py_ID(depth), &_Py_ID(leaf_size), &_Py_ID(node_offset), &_Py_ID(node_depth), &_Py_ID(inner_size), &_Py_ID(last_node), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -246,22 +259,22 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "digest_size", "key", "salt", "person", "fanout", "depth", "leaf_size", "node_offset", "node_depth", "inner_size", "last_node", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "blake2s", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[13]; + PyObject *argsbuf[14]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; - PyObject *data = NULL; + PyObject *data_obj = NULL; int digest_size = HACL_HASH_BLAKE2S_OUT_BYTES; - Py_buffer key = {NULL, NULL}; - Py_buffer salt = {NULL, NULL}; - Py_buffer person = {NULL, NULL}; + Py_buffer key = {.buf = "", .obj = NULL, .len = 0}; + Py_buffer salt = {.buf = "", .obj = NULL, .len = 0}; + Py_buffer person = {.buf = "", .obj = NULL, .len = 0}; int fanout = 1; int depth = 1; unsigned long leaf_size = 0; @@ -270,18 +283,23 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int inner_size = 0; int last_node = 0; int usedforsecurity = 1; + PyObject *string = NULL; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } - if (nargs < 1) { - goto skip_optional_posonly; + if (!noptargs) { + goto skip_optional_pos; } - noptargs--; - data = fastargs[0]; -skip_optional_posonly: + if (fastargs[0]) { + data_obj = fastargs[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } @@ -379,12 +397,18 @@ py_blake2s_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) goto skip_optional_kwonly; } } - usedforsecurity = PyObject_IsTrue(fastargs[12]); - if (usedforsecurity < 0) { - goto exit; + if (fastargs[12]) { + usedforsecurity = PyObject_IsTrue(fastargs[12]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = fastargs[13]; skip_optional_kwonly: - return_value = py_blake2s_new_impl(type, data, digest_size, &key, &salt, &person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity); + return_value = py_blake2s_new_impl(type, data_obj, digest_size, &key, &salt, &person, fanout, depth, leaf_size, node_offset, node_depth, inner_size, last_node, usedforsecurity, string); exit: /* Cleanup for key */ @@ -478,4 +502,4 @@ _blake2_blake2b_hexdigest(PyObject *self, PyObject *Py_UNUSED(ignored)) { return _blake2_blake2b_hexdigest_impl((Blake2Object *)self); } -/*[clinic end generated code: output=d30e8293bd8e2950 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bf30e70c312718cb input=a9049054013a1b77]*/ diff --git a/Modules/clinic/cmathmodule.c.h b/Modules/clinic/cmathmodule.c.h index 7f9e65baf120ea2..ecb5257cf6b2408 100644 --- a/Modules/clinic/cmathmodule.c.h +++ b/Modules/clinic/cmathmodule.c.h @@ -644,7 +644,8 @@ PyDoc_STRVAR(cmath_log__doc__, "\n" "log(z[, base]) -> the logarithm of z to the given base.\n" "\n" -"If the base is not specified, returns the natural logarithm (base e) of z."); +"If the base is not specified, returns the natural logarithm (base e)\n" +"of z."); #define CMATH_LOG_METHODDEF \ {"log", _PyCFunction_CAST(cmath_log), METH_FASTCALL, cmath_log__doc__}, @@ -882,11 +883,12 @@ PyDoc_STRVAR(cmath_isclose__doc__, "\n" "Return True if a is close in value to b, and False otherwise.\n" "\n" -"For the values to be considered close, the difference between them must be\n" -"smaller than at least one of the tolerances.\n" +"For the values to be considered close, the difference between them must\n" +"be smaller than at least one of the tolerances.\n" "\n" -"-inf, inf and NaN behave similarly to the IEEE 754 Standard. That is, NaN is\n" -"not close to anything, even itself. inf and -inf are only close to themselves."); +"-inf, inf and NaN behave similarly to the IEEE 754 Standard. That is,\n" +"NaN is not close to anything, even itself. inf and -inf are only close\n" +"to themselves."); #define CMATH_ISCLOSE_METHODDEF \ {"isclose", _PyCFunction_CAST(cmath_isclose), METH_FASTCALL|METH_KEYWORDS, cmath_isclose__doc__}, @@ -985,4 +987,4 @@ cmath_isclose(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec exit: return return_value; } -/*[clinic end generated code: output=631db17fb1c79d66 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7d5ad4cf258526cd input=a9049054013a1b77]*/ diff --git a/Modules/clinic/fcntlmodule.c.h b/Modules/clinic/fcntlmodule.c.h index 00a929064ba7006..85bfdefdded0271 100644 --- a/Modules/clinic/fcntlmodule.c.h +++ b/Modules/clinic/fcntlmodule.c.h @@ -6,17 +6,23 @@ PyDoc_STRVAR(fcntl_fcntl__doc__, "fcntl($module, fd, cmd, arg=0, /)\n" "--\n" "\n" -"Perform the operation `cmd` on file descriptor fd.\n" +"Perform the operation cmd on file descriptor fd.\n" "\n" -"The values used for `cmd` are operating system dependent, and are available\n" -"as constants in the fcntl module, using the same names as used in\n" -"the relevant C header files. The argument arg is optional, and\n" -"defaults to 0; it may be an int or a string. If arg is given as a string,\n" -"the return value of fcntl is a string of that length, containing the\n" -"resulting value put in the arg buffer by the operating system. The length\n" -"of the arg string is not allowed to exceed 1024 bytes. If the arg given\n" -"is an integer or if none is specified, the result value is an integer\n" -"corresponding to the return value of the fcntl call in the C code."); +"The values used for cmd are operating system dependent, and are\n" +"available as constants in the fcntl module, using the same names as used\n" +"in the relevant C header files. The argument arg is optional, and\n" +"defaults to 0; it may be an integer, a bytes-like object or a string.\n" +"If arg is given as a string, it will be encoded to binary using the\n" +"UTF-8 encoding.\n" +"\n" +"If the arg given is an integer or if none is specified, the result value\n" +"is an integer corresponding to the return value of the fcntl() call in\n" +"the C code.\n" +"\n" +"If arg is given as a bytes-like object, the return value of fcntl() is a\n" +"bytes object of that length, containing the resulting value put in the\n" +"arg buffer by the operating system. The length of the arg buffer is not\n" +"allowed to exceed 1024 bytes."); #define FCNTL_FCNTL_METHODDEF \ {"fcntl", (PyCFunction)(void(*)(void))fcntl_fcntl, METH_FASTCALL, fcntl_fcntl__doc__}, @@ -63,34 +69,34 @@ PyDoc_STRVAR(fcntl_ioctl__doc__, "ioctl($module, fd, request, arg=0, mutate_flag=True, /)\n" "--\n" "\n" -"Perform the operation `request` on file descriptor `fd`.\n" +"Perform the operation request on file descriptor fd.\n" "\n" -"The values used for `request` are operating system dependent, and are available\n" -"as constants in the fcntl or termios library modules, using the same names as\n" -"used in the relevant C header files.\n" +"The values used for request are operating system dependent, and are\n" +"available as constants in the fcntl or termios library modules, using\n" +"the same names as used in the relevant C header files.\n" "\n" -"The argument `arg` is optional, and defaults to 0; it may be an int or a\n" -"buffer containing character data (most likely a string or an array).\n" +"The argument arg is optional, and defaults to 0; it may be an integer, a\n" +"bytes-like object or a string. If arg is given as a string, it will be\n" +"encoded to binary using the UTF-8 encoding.\n" "\n" -"If the argument is a mutable buffer (such as an array) and if the\n" -"mutate_flag argument (which is only allowed in this case) is true then the\n" -"buffer is (in effect) passed to the operating system and changes made by\n" -"the OS will be reflected in the contents of the buffer after the call has\n" -"returned. The return value is the integer returned by the ioctl system\n" -"call.\n" +"If the arg given is an integer or if none is specified, the result value\n" +"is an integer corresponding to the return value of the ioctl() call in\n" +"the C code.\n" "\n" -"If the argument is a mutable buffer and the mutable_flag argument is false,\n" -"the behavior is as if a string had been passed.\n" +"If the argument is a mutable buffer (such as a bytearray) and the\n" +"mutate_flag argument is true (default) then the buffer is (in effect)\n" +"passed to the operating system and changes made by the OS will be\n" +"reflected in the contents of the buffer after the call has returned.\n" +"The return value is the integer returned by the ioctl() system call.\n" "\n" -"If the argument is an immutable buffer (most likely a string) then a copy\n" -"of the buffer is passed to the operating system and the return value is a\n" -"string of the same length containing whatever the operating system put in\n" -"the buffer. The length of the arg buffer in this case is not allowed to\n" -"exceed 1024 bytes.\n" +"If the argument is a mutable buffer and the mutable_flag argument is\n" +"false, the behavior is as if an immutable buffer had been passed.\n" "\n" -"If the arg given is an integer or if none is specified, the result value is\n" -"an integer corresponding to the return value of the ioctl call in the C\n" -"code."); +"If the argument is an immutable buffer then a copy of the buffer is\n" +"passed to the operating system and the return value is a bytes object of\n" +"the same length containing whatever the operating system put in the\n" +"buffer. The length of the arg buffer in this case is not allowed to\n" +"exceed 1024 bytes."); #define FCNTL_IOCTL_METHODDEF \ {"ioctl", (PyCFunction)(void(*)(void))fcntl_ioctl, METH_FASTCALL, fcntl_ioctl__doc__}, @@ -147,7 +153,7 @@ PyDoc_STRVAR(fcntl_flock__doc__, "flock($module, fd, operation, /)\n" "--\n" "\n" -"Perform the lock operation `operation` on file descriptor `fd`.\n" +"Perform the lock operation on file descriptor fd.\n" "\n" "See the Unix manual page for flock(2) for details (On some systems, this\n" "function is emulated using fcntl())."); @@ -189,22 +195,22 @@ PyDoc_STRVAR(fcntl_lockf__doc__, "\n" "A wrapper around the fcntl() locking calls.\n" "\n" -"`fd` is the file descriptor of the file to lock or unlock, and operation is one\n" -"of the following values:\n" +"fd is the file descriptor of the file to lock or unlock, and operation\n" +"is one of the following values:\n" "\n" " LOCK_UN - unlock\n" " LOCK_SH - acquire a shared lock\n" " LOCK_EX - acquire an exclusive lock\n" "\n" "When operation is LOCK_SH or LOCK_EX, it can also be bitwise ORed with\n" -"LOCK_NB to avoid blocking on lock acquisition. If LOCK_NB is used and the\n" -"lock cannot be acquired, an OSError will be raised and the exception will\n" -"have an errno attribute set to EACCES or EAGAIN (depending on the operating\n" -"system -- for portability, check for either value).\n" +"LOCK_NB to avoid blocking on lock acquisition. If LOCK_NB is used and\n" +"the lock cannot be acquired, an OSError will be raised and the exception\n" +"will have an errno attribute set to EACCES or EAGAIN (depending on the\n" +"operating system -- for portability, check for either value).\n" "\n" -"`len` is the number of bytes to lock, with the default meaning to lock to\n" -"EOF. `start` is the byte offset, relative to `whence`, to that the lock\n" -"starts. `whence` is as with fileobj.seek(), specifically:\n" +"len is the number of bytes to lock, with the default meaning to lock to\n" +"EOF. start is the byte offset, relative to whence, to that the lock\n" +"starts. whence is as with fileobj.seek(), specifically:\n" "\n" " 0 - relative to the start of the file (SEEK_SET)\n" " 1 - relative to the current buffer position (SEEK_CUR)\n" @@ -264,4 +270,4 @@ fcntl_lockf(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=65a16bc64c7b4de4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a9345d258926adb7 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/gcmodule.c.h b/Modules/clinic/gcmodule.c.h index 53ff9e4faf8b7cd..d9516700e95c8bb 100644 --- a/Modules/clinic/gcmodule.c.h +++ b/Modules/clinic/gcmodule.c.h @@ -377,8 +377,8 @@ PyDoc_STRVAR(gc_get_objects__doc__, " generation\n" " Generation to extract the objects from.\n" "\n" -"If generation is not None, return only the objects tracked by the collector\n" -"that are in that generation."); +"If generation is not None, return only the objects tracked by the\n" +"collector that are in that generation."); #define GC_GET_OBJECTS_METHODDEF \ {"get_objects", _PyCFunction_CAST(gc_get_objects), METH_FASTCALL|METH_KEYWORDS, gc_get_objects__doc__}, @@ -521,9 +521,10 @@ PyDoc_STRVAR(gc_freeze__doc__, "\n" "Freeze all current tracked objects and ignore them for future collections.\n" "\n" -"This can be used before a POSIX fork() call to make the gc copy-on-write friendly.\n" -"Note: collection before a POSIX fork() call may free pages for future allocation\n" -"which can cause copy-on-write."); +"This can be used before a POSIX fork() call to make the gc copy-on-write\n" +"friendly.\n" +"Note: collection before a POSIX fork() call may free pages for future\n" +"allocation which can cause copy-on-write."); #define GC_FREEZE_METHODDEF \ {"freeze", (PyCFunction)gc_freeze, METH_NOARGS, gc_freeze__doc__}, @@ -584,4 +585,4 @@ gc_get_freeze_count(PyObject *module, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=96d057eac558e6ca input=a9049054013a1b77]*/ +/*[clinic end generated code: output=41268fc41eac6b64 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/hmacmodule.c.h b/Modules/clinic/hmacmodule.c.h index 1ceb2d809e830a2..a31d2ab7f04463c 100644 --- a/Modules/clinic/hmacmodule.c.h +++ b/Modules/clinic/hmacmodule.c.h @@ -187,8 +187,8 @@ PyDoc_STRVAR(_hmac_HMAC_hexdigest__doc__, "\n" "Return hexadecimal digest of the bytes passed to the update() method so far.\n" "\n" -"This may be used to exchange the value safely in email or other non-binary\n" -"environments.\n" +"This may be used to exchange the value safely in email or other\n" +"non-binary environments.\n" "\n" "This method may raise a MemoryError."); @@ -670,4 +670,4 @@ _hmac_compute_blake2b_32(PyObject *module, PyObject *const *args, Py_ssize_t nar exit: return return_value; } -/*[clinic end generated code: output=30c0614482d963f5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6ec5948df1c5569a input=a9049054013a1b77]*/ diff --git a/Modules/clinic/itertoolsmodule.c.h b/Modules/clinic/itertoolsmodule.c.h index 0af82e7eb05be8d..0d87ca1e677b3e2 100644 --- a/Modules/clinic/itertoolsmodule.c.h +++ b/Modules/clinic/itertoolsmodule.c.h @@ -799,8 +799,8 @@ PyDoc_STRVAR(itertools_compress__doc__, "\n" "Return data elements corresponding to true selector elements.\n" "\n" -"Forms a shorter iterator from selected data elements using the selectors to\n" -"choose the data elements."); +"Forms a shorter iterator from selected data elements using the selectors\n" +"to choose the data elements."); static PyObject * itertools_compress_impl(PyTypeObject *type, PyObject *seq1, PyObject *seq2); @@ -965,4 +965,4 @@ itertools_count(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=999758202a532e0a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=59abbdf1cbb2e24b input=a9049054013a1b77]*/ diff --git a/Modules/clinic/mathmodule.c.h b/Modules/clinic/mathmodule.c.h index 4c2c8acd8f69d8b..0abce3b6baaa2d1 100644 --- a/Modules/clinic/mathmodule.c.h +++ b/Modules/clinic/mathmodule.c.h @@ -108,9 +108,7 @@ PyDoc_STRVAR(math_factorial__doc__, "factorial($module, n, /)\n" "--\n" "\n" -"Find n!.\n" -"\n" -"Raise a ValueError if x is negative or non-integral."); +"Find n!."); #define MATH_FACTORIAL_METHODDEF \ {"factorial", (PyCFunction)math_factorial, METH_O, math_factorial__doc__}, @@ -132,8 +130,9 @@ PyDoc_STRVAR(math_frexp__doc__, "\n" "Return the mantissa and exponent of x, as pair (m, e).\n" "\n" -"m is a float and e is an int, such that x = m * 2.**e.\n" -"If x is 0, m and e are both 0. Else 0.5 <= abs(m) < 1.0."); +"If x is a finite nonzero number, then m is a float with\n" +"0.5 <= abs(m) < 1.0 and an integer e is such that\n" +"x == m * 2**e exactly. Else, return (x, 0)."); #define MATH_FREXP_METHODDEF \ {"frexp", (PyCFunction)math_frexp, METH_O, math_frexp__doc__}, @@ -992,8 +991,8 @@ PyDoc_STRVAR(math_nextafter__doc__, "\n" "If steps is not specified or is None, it defaults to 1.\n" "\n" -"Raises a TypeError, if x or y is not a double, or if steps is not an integer.\n" -"Raises ValueError if steps is negative."); +"Raises a TypeError, if x or y is not a double, or if steps is not\n" +"an integer. Raises ValueError if steps is negative."); #define MATH_NEXTAFTER_METHODDEF \ {"nextafter", _PyCFunction_CAST(math_nextafter), METH_FASTCALL|METH_KEYWORDS, math_nextafter__doc__}, @@ -1112,4 +1111,4 @@ math_ulp(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=634773bd18cd3f93 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=18e16eef548c9c43 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/md5module.c.h b/Modules/clinic/md5module.c.h index 9ca4f6528ce8f52..f76902586dddb28 100644 --- a/Modules/clinic/md5module.c.h +++ b/Modules/clinic/md5module.c.h @@ -89,7 +89,7 @@ MD5Type_update(PyObject *self, PyObject *obj) } PyDoc_STRVAR(_md5_md5__doc__, -"md5($module, /, string=b\'\', *, usedforsecurity=True)\n" +"md5($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new MD5 hash object; optionally initialized with a string."); @@ -98,7 +98,8 @@ PyDoc_STRVAR(_md5_md5__doc__, {"md5", _PyCFunction_CAST(_md5_md5), METH_FASTCALL|METH_KEYWORDS, _md5_md5__doc__}, static PyObject * -_md5_md5_impl(PyObject *module, PyObject *string, int usedforsecurity); +_md5_md5_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj); static PyObject * _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -106,7 +107,7 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -115,7 +116,7 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -124,17 +125,18 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "md5", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string_obj = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -145,7 +147,7 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw goto skip_optional_pos; } if (args[0]) { - string = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -154,14 +156,20 @@ _md5_md5(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kw if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string_obj = args[2]; skip_optional_kwonly: - return_value = _md5_md5_impl(module, string, usedforsecurity); + return_value = _md5_md5_impl(module, data, usedforsecurity, string_obj); exit: return return_value; } -/*[clinic end generated code: output=73f4d2034d9fcc63 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=920fe54b9ed06f92 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/mmapmodule.c.h b/Modules/clinic/mmapmodule.c.h new file mode 100644 index 000000000000000..e00f82fb88bdd13 --- /dev/null +++ b/Modules/clinic/mmapmodule.c.h @@ -0,0 +1,769 @@ +/*[clinic input] +preserve +[clinic start generated code]*/ + +#include "pycore_abstract.h" // _PyNumber_Index() +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + +PyDoc_STRVAR(mmap_mmap_close__doc__, +"close($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_CLOSE_METHODDEF \ + {"close", (PyCFunction)mmap_mmap_close, METH_NOARGS, mmap_mmap_close__doc__}, + +static PyObject * +mmap_mmap_close_impl(mmap_object *self); + +static PyObject * +mmap_mmap_close(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_close_impl((mmap_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_read_byte__doc__, +"read_byte($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_READ_BYTE_METHODDEF \ + {"read_byte", (PyCFunction)mmap_mmap_read_byte, METH_NOARGS, mmap_mmap_read_byte__doc__}, + +static PyObject * +mmap_mmap_read_byte_impl(mmap_object *self); + +static PyObject * +mmap_mmap_read_byte(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_read_byte_impl((mmap_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_readline__doc__, +"readline($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_READLINE_METHODDEF \ + {"readline", (PyCFunction)mmap_mmap_readline, METH_NOARGS, mmap_mmap_readline__doc__}, + +static PyObject * +mmap_mmap_readline_impl(mmap_object *self); + +static PyObject * +mmap_mmap_readline(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_readline_impl((mmap_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_read__doc__, +"read($self, n=None, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_READ_METHODDEF \ + {"read", _PyCFunction_CAST(mmap_mmap_read), METH_FASTCALL, mmap_mmap_read__doc__}, + +static PyObject * +mmap_mmap_read_impl(mmap_object *self, Py_ssize_t num_bytes); + +static PyObject * +mmap_mmap_read(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t num_bytes = PY_SSIZE_T_MAX; + + if (!_PyArg_CheckPositional("read", nargs, 0, 1)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + if (!_Py_convert_optional_to_ssize_t(args[0], &num_bytes)) { + goto exit; + } +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_read_impl((mmap_object *)self, num_bytes); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_find__doc__, +"find($self, view, start=None, end=None, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_FIND_METHODDEF \ + {"find", _PyCFunction_CAST(mmap_mmap_find), METH_FASTCALL, mmap_mmap_find__doc__}, + +static PyObject * +mmap_mmap_find_impl(mmap_object *self, Py_buffer *view, PyObject *start, + PyObject *end); + +static PyObject * +mmap_mmap_find(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_buffer view = {NULL, NULL}; + PyObject *start = Py_None; + PyObject *end = Py_None; + + if (!_PyArg_CheckPositional("find", nargs, 1, 3)) { + goto exit; + } + if (PyObject_GetBuffer(args[0], &view, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + start = args[1]; + if (nargs < 3) { + goto skip_optional; + } + end = args[2]; +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_find_impl((mmap_object *)self, &view, start, end); + Py_END_CRITICAL_SECTION(); + +exit: + /* Cleanup for view */ + if (view.obj) { + PyBuffer_Release(&view); + } + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_rfind__doc__, +"rfind($self, view, start=None, end=None, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_RFIND_METHODDEF \ + {"rfind", _PyCFunction_CAST(mmap_mmap_rfind), METH_FASTCALL, mmap_mmap_rfind__doc__}, + +static PyObject * +mmap_mmap_rfind_impl(mmap_object *self, Py_buffer *view, PyObject *start, + PyObject *end); + +static PyObject * +mmap_mmap_rfind(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_buffer view = {NULL, NULL}; + PyObject *start = Py_None; + PyObject *end = Py_None; + + if (!_PyArg_CheckPositional("rfind", nargs, 1, 3)) { + goto exit; + } + if (PyObject_GetBuffer(args[0], &view, PyBUF_SIMPLE) != 0) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + start = args[1]; + if (nargs < 3) { + goto skip_optional; + } + end = args[2]; +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_rfind_impl((mmap_object *)self, &view, start, end); + Py_END_CRITICAL_SECTION(); + +exit: + /* Cleanup for view */ + if (view.obj) { + PyBuffer_Release(&view); + } + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_write__doc__, +"write($self, bytes, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_WRITE_METHODDEF \ + {"write", (PyCFunction)mmap_mmap_write, METH_O, mmap_mmap_write__doc__}, + +static PyObject * +mmap_mmap_write_impl(mmap_object *self, Py_buffer *data); + +static PyObject * +mmap_mmap_write(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_buffer data = {NULL, NULL}; + + if (PyObject_GetBuffer(arg, &data, PyBUF_SIMPLE) != 0) { + goto exit; + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_write_impl((mmap_object *)self, &data); + Py_END_CRITICAL_SECTION(); + +exit: + /* Cleanup for data */ + if (data.obj) { + PyBuffer_Release(&data); + } + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_write_byte__doc__, +"write_byte($self, byte, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_WRITE_BYTE_METHODDEF \ + {"write_byte", (PyCFunction)mmap_mmap_write_byte, METH_O, mmap_mmap_write_byte__doc__}, + +static PyObject * +mmap_mmap_write_byte_impl(mmap_object *self, unsigned char value); + +static PyObject * +mmap_mmap_write_byte(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + unsigned char value; + + { + long ival = PyLong_AsLong(arg); + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + else if (ival < 0) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is less than minimum"); + goto exit; + } + else if (ival > UCHAR_MAX) { + PyErr_SetString(PyExc_OverflowError, + "unsigned byte integer is greater than maximum"); + goto exit; + } + else { + value = (unsigned char) ival; + } + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_write_byte_impl((mmap_object *)self, value); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_size__doc__, +"size($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_SIZE_METHODDEF \ + {"size", (PyCFunction)mmap_mmap_size, METH_NOARGS, mmap_mmap_size__doc__}, + +static PyObject * +mmap_mmap_size_impl(mmap_object *self); + +static PyObject * +mmap_mmap_size(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_size_impl((mmap_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_resize__doc__, +"resize($self, newsize, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_RESIZE_METHODDEF \ + {"resize", (PyCFunction)mmap_mmap_resize, METH_O, mmap_mmap_resize__doc__}, + +static PyObject * +mmap_mmap_resize_impl(mmap_object *self, Py_ssize_t new_size); + +static PyObject * +mmap_mmap_resize(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_ssize_t new_size; + + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(arg); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + new_size = ival; + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_resize_impl((mmap_object *)self, new_size); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_tell__doc__, +"tell($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_TELL_METHODDEF \ + {"tell", (PyCFunction)mmap_mmap_tell, METH_NOARGS, mmap_mmap_tell__doc__}, + +static PyObject * +mmap_mmap_tell_impl(mmap_object *self); + +static PyObject * +mmap_mmap_tell(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_tell_impl((mmap_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_flush__doc__, +"flush($self, offset=0, size=None, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_FLUSH_METHODDEF \ + {"flush", _PyCFunction_CAST(mmap_mmap_flush), METH_FASTCALL, mmap_mmap_flush__doc__}, + +static PyObject * +mmap_mmap_flush_impl(mmap_object *self, Py_ssize_t offset, + PyObject *size_obj); + +static PyObject * +mmap_mmap_flush(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t offset = 0; + PyObject *size_obj = Py_None; + + if (!_PyArg_CheckPositional("flush", nargs, 0, 2)) { + goto exit; + } + if (nargs < 1) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + offset = ival; + } + if (nargs < 2) { + goto skip_optional; + } + size_obj = args[1]; +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_flush_impl((mmap_object *)self, offset, size_obj); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_seek__doc__, +"seek($self, pos, whence=0, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_SEEK_METHODDEF \ + {"seek", _PyCFunction_CAST(mmap_mmap_seek), METH_FASTCALL, mmap_mmap_seek__doc__}, + +static PyObject * +mmap_mmap_seek_impl(mmap_object *self, Py_ssize_t dist, int how); + +static PyObject * +mmap_mmap_seek(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t dist; + int how = 0; + + if (!_PyArg_CheckPositional("seek", nargs, 1, 2)) { + goto exit; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + dist = ival; + } + if (nargs < 2) { + goto skip_optional; + } + how = PyLong_AsInt(args[1]); + if (how == -1 && PyErr_Occurred()) { + goto exit; + } +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_seek_impl((mmap_object *)self, dist, how); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(mmap_mmap_seekable__doc__, +"seekable($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_SEEKABLE_METHODDEF \ + {"seekable", (PyCFunction)mmap_mmap_seekable, METH_NOARGS, mmap_mmap_seekable__doc__}, + +static PyObject * +mmap_mmap_seekable_impl(mmap_object *self); + +static PyObject * +mmap_mmap_seekable(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return mmap_mmap_seekable_impl((mmap_object *)self); +} + +PyDoc_STRVAR(mmap_mmap_move__doc__, +"move($self, dest, src, count, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_MOVE_METHODDEF \ + {"move", _PyCFunction_CAST(mmap_mmap_move), METH_FASTCALL, mmap_mmap_move__doc__}, + +static PyObject * +mmap_mmap_move_impl(mmap_object *self, Py_ssize_t dest, Py_ssize_t src, + Py_ssize_t cnt); + +static PyObject * +mmap_mmap_move(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + Py_ssize_t dest; + Py_ssize_t src; + Py_ssize_t cnt; + + if (!_PyArg_CheckPositional("move", nargs, 3, 3)) { + goto exit; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[0]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + dest = ival; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + src = ival; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[2]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + cnt = ival; + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_move_impl((mmap_object *)self, dest, src, cnt); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +PyDoc_STRVAR(mmap_mmap___enter____doc__, +"__enter__($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP___ENTER___METHODDEF \ + {"__enter__", (PyCFunction)mmap_mmap___enter__, METH_NOARGS, mmap_mmap___enter____doc__}, + +static PyObject * +mmap_mmap___enter___impl(mmap_object *self); + +static PyObject * +mmap_mmap___enter__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap___enter___impl((mmap_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(mmap_mmap___exit____doc__, +"__exit__($self, exc_type, exc_value, traceback, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP___EXIT___METHODDEF \ + {"__exit__", _PyCFunction_CAST(mmap_mmap___exit__), METH_FASTCALL, mmap_mmap___exit____doc__}, + +static PyObject * +mmap_mmap___exit___impl(mmap_object *self, PyObject *exc_type, + PyObject *exc_value, PyObject *traceback); + +static PyObject * +mmap_mmap___exit__(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + PyObject *exc_type; + PyObject *exc_value; + PyObject *traceback; + + if (!_PyArg_CheckPositional("__exit__", nargs, 3, 3)) { + goto exit; + } + exc_type = args[0]; + exc_value = args[1]; + traceback = args[2]; + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap___exit___impl((mmap_object *)self, exc_type, exc_value, traceback); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +#if defined(MS_WINDOWS) + +PyDoc_STRVAR(mmap_mmap___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP___SIZEOF___METHODDEF \ + {"__sizeof__", (PyCFunction)mmap_mmap___sizeof__, METH_NOARGS, mmap_mmap___sizeof____doc__}, + +static PyObject * +mmap_mmap___sizeof___impl(mmap_object *self); + +static PyObject * +mmap_mmap___sizeof__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap___sizeof___impl((mmap_object *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +#endif /* defined(MS_WINDOWS) */ + +#if (defined(MS_WINDOWS) && defined(Py_DEBUG)) + +PyDoc_STRVAR(mmap_mmap__protect__doc__, +"_protect($self, flNewProtect, start, length, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP__PROTECT_METHODDEF \ + {"_protect", _PyCFunction_CAST(mmap_mmap__protect), METH_FASTCALL, mmap_mmap__protect__doc__}, + +static PyObject * +mmap_mmap__protect_impl(mmap_object *self, unsigned int flNewProtect, + Py_ssize_t start, Py_ssize_t length); + +static PyObject * +mmap_mmap__protect(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + unsigned int flNewProtect; + Py_ssize_t start; + Py_ssize_t length; + + if (!_PyArg_CheckPositional("_protect", nargs, 3, 3)) { + goto exit; + } + flNewProtect = (unsigned int)PyLong_AsUnsignedLongMask(args[0]); + if (flNewProtect == (unsigned int)-1 && PyErr_Occurred()) { + goto exit; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + start = ival; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[2]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + length = ival; + } + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap__protect_impl((mmap_object *)self, flNewProtect, start, length); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +#endif /* (defined(MS_WINDOWS) && defined(Py_DEBUG)) */ + +#if defined(HAVE_MADVISE) + +PyDoc_STRVAR(mmap_mmap_madvise__doc__, +"madvise($self, option, start=0, length=None, /)\n" +"--\n" +"\n"); + +#define MMAP_MMAP_MADVISE_METHODDEF \ + {"madvise", _PyCFunction_CAST(mmap_mmap_madvise), METH_FASTCALL, mmap_mmap_madvise__doc__}, + +static PyObject * +mmap_mmap_madvise_impl(mmap_object *self, int option, Py_ssize_t start, + PyObject *length_obj); + +static PyObject * +mmap_mmap_madvise(PyObject *self, PyObject *const *args, Py_ssize_t nargs) +{ + PyObject *return_value = NULL; + int option; + Py_ssize_t start = 0; + PyObject *length_obj = Py_None; + + if (!_PyArg_CheckPositional("madvise", nargs, 1, 3)) { + goto exit; + } + option = PyLong_AsInt(args[0]); + if (option == -1 && PyErr_Occurred()) { + goto exit; + } + if (nargs < 2) { + goto skip_optional; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + start = ival; + } + if (nargs < 3) { + goto skip_optional; + } + length_obj = args[2]; +skip_optional: + Py_BEGIN_CRITICAL_SECTION(self); + return_value = mmap_mmap_madvise_impl((mmap_object *)self, option, start, length_obj); + Py_END_CRITICAL_SECTION(); + +exit: + return return_value; +} + +#endif /* defined(HAVE_MADVISE) */ + +#ifndef MMAP_MMAP___SIZEOF___METHODDEF + #define MMAP_MMAP___SIZEOF___METHODDEF +#endif /* !defined(MMAP_MMAP___SIZEOF___METHODDEF) */ + +#ifndef MMAP_MMAP__PROTECT_METHODDEF + #define MMAP_MMAP__PROTECT_METHODDEF +#endif /* !defined(MMAP_MMAP__PROTECT_METHODDEF) */ + +#ifndef MMAP_MMAP_MADVISE_METHODDEF + #define MMAP_MMAP_MADVISE_METHODDEF +#endif /* !defined(MMAP_MMAP_MADVISE_METHODDEF) */ +/*[clinic end generated code: output=fe4e2131abc1e971 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/overlapped.c.h b/Modules/clinic/overlapped.c.h index 7e2480bdace38d2..ba41eab15650e8f 100644 --- a/Modules/clinic/overlapped.c.h +++ b/Modules/clinic/overlapped.c.h @@ -529,8 +529,9 @@ PyDoc_STRVAR(_overlapped_Overlapped_getresult__doc__, "\n" "Retrieve result of operation.\n" "\n" -"If wait is true then it blocks until the operation is finished. If wait\n" -"is false and the operation is still pending then an error is raised."); +"If wait is true then it blocks until the operation is finished. If\n" +"wait is false and the operation is still pending then an error is\n" +"raised."); #define _OVERLAPPED_OVERLAPPED_GETRESULT_METHODDEF \ {"getresult", _PyCFunction_CAST(_overlapped_Overlapped_getresult), METH_FASTCALL, _overlapped_Overlapped_getresult__doc__}, @@ -1242,4 +1243,4 @@ _overlapped_Overlapped_WSARecvFromInto(PyObject *self, PyObject *const *args, Py return return_value; } -/*[clinic end generated code: output=3e4cb2b55342cd96 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=0ecaf45a09539599 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 6b8cc3d07ab01c3..094580e7c237b91 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -378,9 +378,9 @@ PyDoc_STRVAR(os_chdir__doc__, "\n" "Change the current working directory to the specified path.\n" "\n" -"path may always be specified as a string.\n" -"On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception."); +"path may always be specified as a string. On some platforms, path may\n" +"also be specified as an open file descriptor. If this functionality is\n" +"unavailable, using it raises an exception."); #define OS_CHDIR_METHODDEF \ {"chdir", _PyCFunction_CAST(os_chdir), METH_FASTCALL|METH_KEYWORDS, os_chdir__doc__}, @@ -515,14 +515,15 @@ PyDoc_STRVAR(os_chmod__doc__, "Change the access permissions of a file.\n" "\n" " path\n" -" Path to be modified. May always be specified as a str, bytes, or a path-like object.\n" -" On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception.\n" +" Path to be modified. May always be specified as a str, bytes, or\n" +" a path-like object. On some platforms, path may also be specified\n" +" as an open file descriptor. If this functionality is unavailable,\n" +" using it raises an exception.\n" " mode\n" " Operating-system mode bitfield.\n" -" Be careful when using number literals for *mode*. The conventional UNIX notation for\n" -" numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in\n" -" Python.\n" +" Be careful when using number literals for *mode*. The conventional\n" +" UNIX notation for numeric modes uses an octal base, which needs to\n" +" be indicated with a ``0o`` prefix in Python.\n" " dir_fd\n" " If not None, it should be a file descriptor open to a directory,\n" " and path should be relative; path will then be relative to that\n" @@ -631,9 +632,9 @@ PyDoc_STRVAR(os_fchmod__doc__, " The file descriptor of the file to be modified.\n" " mode\n" " Operating-system mode bitfield.\n" -" Be careful when using number literals for *mode*. The conventional UNIX notation for\n" -" numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in\n" -" Python.\n" +" Be careful when using number literals for *mode*. The conventional\n" +" UNIX notation for numeric modes uses an octal base, which needs to\n" +" be indicated with a ``0o`` prefix in Python.\n" "\n" "Equivalent to os.chmod(fd, mode)."); @@ -707,8 +708,8 @@ PyDoc_STRVAR(os_lchmod__doc__, "\n" "Change the access permissions of a file, without following symbolic links.\n" "\n" -"If path is a symlink, this affects the link itself rather than the target.\n" -"Equivalent to chmod(path, mode, follow_symlinks=False).\""); +"If path is a symlink, this affects the link itself rather than the\n" +"target. Equivalent to chmod(path, mode, follow_symlinks=False)."); #define OS_LCHMOD_METHODDEF \ {"lchmod", _PyCFunction_CAST(os_lchmod), METH_FASTCALL|METH_KEYWORDS, os_lchmod__doc__}, @@ -782,9 +783,9 @@ PyDoc_STRVAR(os_chflags__doc__, "\n" "Set file flags.\n" "\n" -"If follow_symlinks is False, and the last element of the path is a symbolic\n" -" link, chflags will change flags on the symbolic link itself instead of the\n" -" file the link points to.\n" +"If follow_symlinks is False, and the last element of the path is\n" +"a symbolic link, chflags() will change flags on the symbolic link itself\n" +"instead of the file the link points to.\n" "follow_symlinks may not be implemented on your platform. If it is\n" "unavailable, using it will raise a NotImplementedError."); @@ -1165,10 +1166,11 @@ PyDoc_STRVAR(os_chown__doc__, "chown($module, /, path, uid, gid, *, dir_fd=None, follow_symlinks=True)\n" "--\n" "\n" -"Change the owner and group id of path to the numeric uid and gid.\\\n" +"Change the owner and group id of path to the numeric uid and gid.\n" "\n" " path\n" -" Path to be examined; can be string, bytes, a path-like object, or open-file-descriptor int.\n" +" Path to be examined; can be string, bytes, a path-like object, or\n" +" open-file-descriptor int.\n" " dir_fd\n" " If not None, it should be a file descriptor open to a directory,\n" " and path should be relative; path will then be relative to that\n" @@ -1178,18 +1180,19 @@ PyDoc_STRVAR(os_chown__doc__, " stat will examine the symbolic link itself instead of the file\n" " the link points to.\n" "\n" -"path may always be specified as a string.\n" -"On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception.\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" -"If follow_symlinks is False, and the last element of the path is a symbolic\n" -" link, chown will modify the symbolic link itself instead of the file the\n" -" link points to.\n" +"path may always be specified as a string. On some platforms, path may\n" +"also be specified as an open file descriptor. If this functionality is\n" +"unavailable, using it raises an exception.\n" +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative to\n" +"that directory.\n" +"If follow_symlinks is False, and the last element of the path is\n" +"a symbolic link, chown will modify the symbolic link itself instead of\n" +"the file the link points to.\n" "It is an error to use dir_fd or follow_symlinks when specifying path as\n" -" an open file descriptor.\n" -"dir_fd and follow_symlinks may not be implemented on your platform.\n" -" If they are unavailable, using them will raise a NotImplementedError."); +"an open file descriptor.\n" +"dir_fd and follow_symlinks may not be implemented on your platform. If\n" +"they are unavailable, using them will raise a NotImplementedError."); #define OS_CHOWN_METHODDEF \ {"chown", _PyCFunction_CAST(os_chown), METH_FASTCALL|METH_KEYWORDS, os_chown__doc__}, @@ -1477,14 +1480,15 @@ PyDoc_STRVAR(os_link__doc__, "Create a hard link to a file.\n" "\n" "If either src_dir_fd or dst_dir_fd is not None, it should be a file\n" -" descriptor open to a directory, and the respective path string (src or dst)\n" -" should be relative; the path will then be relative to that directory.\n" +"descriptor open to a directory, and the respective path string (src or\n" +"dst) should be relative; the path will then be relative to that\n" +"directory.\n" "If follow_symlinks is False, and the last element of src is a symbolic\n" -" link, link will create a link to the symbolic link itself instead of the\n" -" file the link points to.\n" -"src_dir_fd, dst_dir_fd, and follow_symlinks may not be implemented on your\n" -" platform. If they are unavailable, using them will raise a\n" -" NotImplementedError."); +"link, link will create a link to the symbolic link itself instead of the\n" +"file the link points to.\n" +"src_dir_fd, dst_dir_fd, and follow_symlinks may not be implemented on\n" +"your platform. If they are unavailable, using them will raise\n" +"a NotImplementedError."); #define OS_LINK_METHODDEF \ {"link", _PyCFunction_CAST(os_link), METH_FASTCALL|METH_KEYWORDS, os_link__doc__}, @@ -1586,13 +1590,13 @@ PyDoc_STRVAR(os_listdir__doc__, "\n" "Return a list containing the names of the files in the directory.\n" "\n" -"path can be specified as either str, bytes, or a path-like object. If path is bytes,\n" -" the filenames returned will also be bytes; in all other circumstances\n" -" the filenames returned will be str.\n" +"path can be specified as either str, bytes, or a path-like object. If\n" +"path is bytes, the filenames returned will also be bytes; in all other\n" +"circumstances the filenames returned will be str.\n" "If path is None, uses the path=\'.\'.\n" -"On some platforms, path may also be specified as an open file descriptor;\\\n" -" the file descriptor must refer to a directory.\n" -" If this functionality is unavailable, using it raises NotImplementedError.\n" +"On some platforms, path may also be specified as an open file\n" +"descriptor; the file descriptor must refer to a directory. If this\n" +"functionality is unavailable, using it raises NotImplementedError.\n" "\n" "The list is in arbitrary order. It does not include the special\n" "entries \'.\' and \'..\' even if they are present in the directory."); @@ -2644,13 +2648,14 @@ PyDoc_STRVAR(os_mkdir__doc__, "\n" "Create a directory.\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" -"dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError.\n" +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative to\n" +"that directory.\n" +"dir_fd may not be implemented on your platform. If it is unavailable,\n" +"using it will raise a NotImplementedError.\n" "\n" -"The mode argument is ignored on Windows. Where it is used, the current umask\n" -"value is first masked out."); +"The mode argument is ignored on Windows. Where it is used, the current\n" +"umask value is first masked out."); #define OS_MKDIR_METHODDEF \ {"mkdir", _PyCFunction_CAST(os_mkdir), METH_FASTCALL|METH_KEYWORDS, os_mkdir__doc__}, @@ -2916,10 +2921,11 @@ PyDoc_STRVAR(os_rename__doc__, "Rename a file or directory.\n" "\n" "If either src_dir_fd or dst_dir_fd is not None, it should be a file\n" -" descriptor open to a directory, and the respective path string (src or dst)\n" -" should be relative; the path will then be relative to that directory.\n" +"descriptor open to a directory, and the respective path string (src or\n" +"dst) should be relative; the path will then be relative to that\n" +"directory.\n" "src_dir_fd and dst_dir_fd, may not be implemented on your platform.\n" -" If they are unavailable, using them will raise a NotImplementedError."); +"If they are unavailable, using them will raise a NotImplementedError."); #define OS_RENAME_METHODDEF \ {"rename", _PyCFunction_CAST(os_rename), METH_FASTCALL|METH_KEYWORDS, os_rename__doc__}, @@ -3010,10 +3016,11 @@ PyDoc_STRVAR(os_replace__doc__, "Rename a file or directory, overwriting the destination.\n" "\n" "If either src_dir_fd or dst_dir_fd is not None, it should be a file\n" -" descriptor open to a directory, and the respective path string (src or dst)\n" -" should be relative; the path will then be relative to that directory.\n" +"descriptor open to a directory, and the respective path string (src or\n" +"dst) should be relative; the path will then be relative to that\n" +"directory.\n" "src_dir_fd and dst_dir_fd, may not be implemented on your platform.\n" -" If they are unavailable, using them will raise a NotImplementedError."); +"If they are unavailable, using them will raise a NotImplementedError."); #define OS_REPLACE_METHODDEF \ {"replace", _PyCFunction_CAST(os_replace), METH_FASTCALL|METH_KEYWORDS, os_replace__doc__}, @@ -3103,10 +3110,11 @@ PyDoc_STRVAR(os_rmdir__doc__, "\n" "Remove a directory.\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative\n" +"to that directory.\n" "dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError."); +"If it is unavailable, using it will raise a NotImplementedError."); #define OS_RMDIR_METHODDEF \ {"rmdir", _PyCFunction_CAST(os_rmdir), METH_FASTCALL|METH_KEYWORDS, os_rmdir__doc__}, @@ -3361,10 +3369,11 @@ PyDoc_STRVAR(os_unlink__doc__, "\n" "Remove a file (same as remove()).\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative to\n" +"that directory.\n" "dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError."); +"If it is unavailable, using it will raise a NotImplementedError."); #define OS_UNLINK_METHODDEF \ {"unlink", _PyCFunction_CAST(os_unlink), METH_FASTCALL|METH_KEYWORDS, os_unlink__doc__}, @@ -3438,10 +3447,11 @@ PyDoc_STRVAR(os_remove__doc__, "\n" "Remove a file (same as unlink()).\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative\n" +"to that directory.\n" "dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError."); +"If it is unavailable, using it will raise a NotImplementedError."); #define OS_REMOVE_METHODDEF \ {"remove", _PyCFunction_CAST(os_remove), METH_FASTCALL|METH_KEYWORDS, os_remove__doc__}, @@ -3541,27 +3551,28 @@ PyDoc_STRVAR(os_utime__doc__, "\n" "Set the access and modified time of path.\n" "\n" -"path may always be specified as a string.\n" -"On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception.\n" +"path may always be specified as a string. On some platforms, path may\n" +"also be specified as an open file descriptor. If this functionality is\n" +"unavailable, using it raises an exception.\n" "\n" "If times is not None, it must be a tuple (atime, mtime);\n" -" atime and mtime should be expressed as float seconds since the epoch.\n" +"atime and mtime should be expressed as float seconds since the epoch.\n" "If ns is specified, it must be a tuple (atime_ns, mtime_ns);\n" -" atime_ns and mtime_ns should be expressed as integer nanoseconds\n" -" since the epoch.\n" +"atime_ns and mtime_ns should be expressed as integer nanoseconds\n" +"since the epoch.\n" "If times is None and ns is unspecified, utime uses the current time.\n" "Specifying tuples for both times and ns is an error.\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" -"If follow_symlinks is False, and the last element of the path is a symbolic\n" -" link, utime will modify the symbolic link itself instead of the file the\n" -" link points to.\n" -"It is an error to use dir_fd or follow_symlinks when specifying path\n" -" as an open file descriptor.\n" +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative to\n" +"that directory.\n" +"If follow_symlinks is False, and the last element of the path is\n" +"a symbolic link, utime will modify the symbolic link itself instead of\n" +"the file the link points to.\n" +"It is an error to use dir_fd or follow_symlinks when specifying path as\n" +"an open file descriptor.\n" "dir_fd and follow_symlinks may not be available on your platform.\n" -" If they are unavailable, using them will raise a NotImplementedError."); +"If they are unavailable, using them will raise a NotImplementedError."); #define OS_UTIME_METHODDEF \ {"utime", _PyCFunction_CAST(os_utime), METH_FASTCALL|METH_KEYWORDS, os_utime__doc__}, @@ -3845,8 +3856,8 @@ os_execve(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k PyDoc_STRVAR(os_posix_spawn__doc__, "posix_spawn($module, path, argv, env, /, *, file_actions=(),\n" -" setpgroup=<unrepresentable>, resetids=False, setsid=False,\n" -" setsigmask=(), setsigdef=(), scheduler=<unrepresentable>)\n" +" setpgroup=None, resetids=False, setsid=False,\n" +" setsigmask=(), setsigdef=(), scheduler=None)\n" "--\n" "\n" "Execute the program specified by path in a new process.\n" @@ -3864,7 +3875,8 @@ PyDoc_STRVAR(os_posix_spawn__doc__, " resetids\n" " If the value is `true` the POSIX_SPAWN_RESETIDS will be activated.\n" " setsid\n" -" If the value is `true` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated.\n" +" If the value is `true` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP\n" +" will be activated.\n" " setsigmask\n" " The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag.\n" " setsigdef\n" @@ -3998,8 +4010,8 @@ os_posix_spawn(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje PyDoc_STRVAR(os_posix_spawnp__doc__, "posix_spawnp($module, path, argv, env, /, *, file_actions=(),\n" -" setpgroup=<unrepresentable>, resetids=False, setsid=False,\n" -" setsigmask=(), setsigdef=(), scheduler=<unrepresentable>)\n" +" setpgroup=None, resetids=False, setsid=False,\n" +" setsigmask=(), setsigdef=(), scheduler=None)\n" "--\n" "\n" "Execute the program specified by path in a new process.\n" @@ -4017,7 +4029,8 @@ PyDoc_STRVAR(os_posix_spawnp__doc__, " resetids\n" " If the value is `True` the POSIX_SPAWN_RESETIDS will be activated.\n" " setsid\n" -" If the value is `True` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated.\n" +" If the value is `True` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP\n" +" will be activated.\n" " setsigmask\n" " The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag.\n" " setsigdef\n" @@ -4888,8 +4901,8 @@ PyDoc_STRVAR(os_posix_openpt__doc__, "Open and return a file descriptor for a master pseudo-terminal device.\n" "\n" "Performs a posix_openpt() C function call. The oflag argument is used to\n" -"set file status flags and file access modes as specified in the manual page\n" -"of posix_openpt() of your system."); +"set file status flags and file access modes as specified in the manual\n" +"page of posix_openpt() of your system."); #define OS_POSIX_OPENPT_METHODDEF \ {"posix_openpt", (PyCFunction)os_posix_openpt, METH_O, os_posix_openpt__doc__}, @@ -5345,9 +5358,9 @@ PyDoc_STRVAR(os_initgroups__doc__, "\n" "Initialize the group access list.\n" "\n" -"Call the system initgroups() to initialize the group access list with all of\n" -"the groups of which the specified username is a member, plus the specified\n" -"group id."); +"Call the system initgroups() to initialize the group access list with\n" +"all of the groups of which the specified username is a member, plus the\n" +"specified group id."); #define OS_INITGROUPS_METHODDEF \ {"initgroups", _PyCFunction_CAST(os_initgroups), METH_FASTCALL, os_initgroups__doc__}, @@ -5391,9 +5404,9 @@ PyDoc_STRVAR(os_initgroups__doc__, "\n" "Initialize the group access list.\n" "\n" -"Call the system initgroups() to initialize the group access list with all of\n" -"the groups of which the specified username is a member, plus the specified\n" -"group id."); +"Call the system initgroups() to initialize the group access list with\n" +"all of the groups of which the specified username is a member, plus the\n" +"specified group id."); #define OS_INITGROUPS_METHODDEF \ {"initgroups", _PyCFunction_CAST(os_initgroups), METH_FASTCALL, os_initgroups__doc__}, @@ -5546,7 +5559,8 @@ PyDoc_STRVAR(os_getppid__doc__, "Return the parent\'s process id.\n" "\n" "If the parent process has already exited, Windows machines will still\n" -"return its id; others systems will return the id of the \'init\' process (1)."); +"return its id; others systems will return the id of the \'init\' proces\n" +"(1)."); #define OS_GETPPID_METHODDEF \ {"getppid", (PyCFunction)os_getppid, METH_NOARGS, os_getppid__doc__}, @@ -6096,8 +6110,8 @@ PyDoc_STRVAR(os_waitid__doc__, " Constructed from the ORing of one or more of WEXITED, WSTOPPED\n" " or WCONTINUED and additionally may be ORed with WNOHANG or WNOWAIT.\n" "\n" -"Returns either waitid_result or None if WNOHANG is specified and there are\n" -"no children in a waitable state."); +"Returns either waitid_result or None if WNOHANG is specified and there\n" +"are no children in a waitable state."); #define OS_WAITID_METHODDEF \ {"waitid", _PyCFunction_CAST(os_waitid), METH_FASTCALL, os_waitid__doc__}, @@ -6258,8 +6272,8 @@ PyDoc_STRVAR(os_pidfd_open__doc__, "\n" "Return a file descriptor referring to the process *pid*.\n" "\n" -"The descriptor can be used to perform process management without races and\n" -"signals."); +"The descriptor can be used to perform process management without races\n" +"and signals."); #define OS_PIDFD_OPEN_METHODDEF \ {"pidfd_open", _PyCFunction_CAST(os_pidfd_open), METH_FASTCALL|METH_KEYWORDS, os_pidfd_open__doc__}, @@ -6483,8 +6497,9 @@ PyDoc_STRVAR(os_readlink__doc__, "\n" "Return a string representing the path to which the symbolic link points.\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -"and path should be relative; path will then be relative to that directory.\n" +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative to\n" +"that directory.\n" "\n" "dir_fd may not be implemented on your platform. If it is unavailable,\n" "using it will raise a NotImplementedError."); @@ -6566,14 +6581,15 @@ PyDoc_STRVAR(os_symlink__doc__, "Create a symbolic link pointing to src named dst.\n" "\n" "target_is_directory is required on Windows if the target is to be\n" -" interpreted as a directory. (On Windows, symlink requires\n" -" Windows 6.0 or greater, and raises a NotImplementedError otherwise.)\n" -" target_is_directory is ignored on non-Windows platforms.\n" +"interpreted as a directory. (On Windows, symlink requires Windows 6.0\n" +"or greater, and raises a NotImplementedError otherwise.)\n" +"target_is_directory is ignored on non-Windows platforms.\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" -"dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError."); +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative\n" +"to that directory.\n" +"dir_fd may not be implemented on your platform. If it is unavailable,\n" +"using it will raise a NotImplementedError."); #define OS_SYMLINK_METHODDEF \ {"symlink", _PyCFunction_CAST(os_symlink), METH_FASTCALL|METH_KEYWORDS, os_symlink__doc__}, @@ -7241,10 +7257,11 @@ PyDoc_STRVAR(os_open__doc__, "\n" "Open a file for low level IO. Returns a file descriptor (integer).\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" -"dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError."); +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative to\n" +"that directory.\n" +"dir_fd may not be implemented on your platform. If it is unavailable,\n" +"using it will raise a NotImplementedError."); #define OS_OPEN_METHODDEF \ {"open", _PyCFunction_CAST(os_open), METH_FASTCALL|METH_KEYWORDS, os_open__doc__}, @@ -7618,7 +7635,8 @@ PyDoc_STRVAR(os_lseek__doc__, " - SEEK_CUR: seek from the current file position.\n" " - SEEK_END: seek from the end of the file.\n" "\n" -"The return value is the number of bytes relative to the beginning of the file."); +"The return value is the number of bytes relative to the beginning of\n" +"the file."); #define OS_LSEEK_METHODDEF \ {"lseek", _PyCFunction_CAST(os_lseek), METH_FASTCALL, os_lseek__doc__}, @@ -7709,15 +7727,15 @@ PyDoc_STRVAR(os_readinto__doc__, "\n" "Read into a buffer object from a file descriptor.\n" "\n" -"The buffer should be mutable and bytes-like. On success, returns the number of\n" -"bytes read. Less bytes may be read than the size of the buffer. The underlying\n" -"system call will be retried when interrupted by a signal, unless the signal\n" -"handler raises an exception. Other errors will not be retried and an error will\n" -"be raised.\n" +"The buffer should be mutable and bytes-like. On success, returns the\n" +"number of bytes read. Less bytes may be read than the size of the\n" +"buffer. The underlying system call will be retried when interrupted by\n" +"a signal, unless the signal handler raises an exception. Other errors\n" +"will not be retried and an error will be raised.\n" "\n" -"Returns 0 if *fd* is at end of file or if the provided *buffer* has length 0\n" -"(which can be used to check for errors without reading data). Never returns\n" -"negative."); +"Returns 0 if *fd* is at end of file or if the provided *buffer* has\n" +"length 0 (which can be used to check for errors without reading data).\n" +"Never returns negative."); #define OS_READINTO_METHODDEF \ {"readinto", _PyCFunction_CAST(os_readinto), METH_FASTCALL, os_readinto__doc__}, @@ -7872,14 +7890,15 @@ PyDoc_STRVAR(os_preadv__doc__, "\n" "Reads from a file descriptor into a number of mutable bytes-like objects.\n" "\n" -"Combines the functionality of readv() and pread(). As readv(), it will\n" -"transfer data into each buffer until it is full and then move on to the next\n" -"buffer in the sequence to hold the rest of the data. Its fourth argument,\n" -"specifies the file offset at which the input operation is to be performed. It\n" -"will return the total number of bytes read (which can be less than the total\n" -"capacity of all the objects).\n" +"Combines the functionality of readv() and pread(). As readv(), it will\n" +"transfer data into each buffer until it is full and then move on to the\n" +"next buffer in the sequence to hold the rest of the data. Its fourth\n" +"argument, specifies the file offset at which the input operation is to\n" +"be performed. It will return the total number of bytes read (which can\n" +"be less than the total capacity of all the objects).\n" "\n" -"The flags argument contains a bitwise OR of zero or more of the following flags:\n" +"The flags argument contains a bitwise OR of zero or more of the\n" +"following flags:\n" "\n" "- RWF_HIPRI\n" "- RWF_NOWAIT\n" @@ -8602,14 +8621,16 @@ PyDoc_STRVAR(os_pwritev__doc__, "\n" "Writes the contents of bytes-like objects to a file descriptor at a given offset.\n" "\n" -"Combines the functionality of writev() and pwrite(). All buffers must be a sequence\n" -"of bytes-like objects. Buffers are processed in array order. Entire contents of first\n" -"buffer is written before proceeding to second, and so on. The operating system may\n" -"set a limit (sysconf() value SC_IOV_MAX) on the number of buffers that can be used.\n" -"This function writes the contents of each object to the file descriptor and returns\n" -"the total number of bytes written.\n" +"Combines the functionality of writev() and pwrite(). All buffers must be\n" +"a sequence of bytes-like objects. Buffers are processed in array order.\n" +"Entire contents of first buffer is written before proceeding to second,\n" +"and so on. The operating system may set a limit (sysconf() value\n" +"SC_IOV_MAX) on the number of buffers that can be used.\n" +"This function writes the contents of each object to the file descriptor\n" +"and returns the total number of bytes written.\n" "\n" -"The flags argument contains a bitwise OR of zero or more of the following flags:\n" +"The flags argument contains a bitwise OR of zero or more of the\n" +"following flags:\n" "\n" "- RWF_DSYNC\n" "- RWF_SYNC\n" @@ -8912,10 +8933,11 @@ PyDoc_STRVAR(os_mkfifo__doc__, "\n" "Create a \"fifo\" (a POSIX named pipe).\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" -"dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError."); +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative to\n" +"that directory.\n" +"dir_fd may not be implemented on your platform. If it is unavailable,\n" +"using it will raise a NotImplementedError."); #define OS_MKFIFO_METHODDEF \ {"mkfifo", _PyCFunction_CAST(os_mkfifo), METH_FASTCALL|METH_KEYWORDS, os_mkfifo__doc__}, @@ -9007,17 +9029,18 @@ PyDoc_STRVAR(os_mknod__doc__, "\n" "Create a node in the file system.\n" "\n" -"Create a node in the file system (file, device special file or named pipe)\n" -"at path. mode specifies both the permissions to use and the\n" +"Create a node in the file system (file, device special file or named\n" +"pipe) at path. mode specifies both the permissions to use and the\n" "type of node to be created, being combined (bitwise OR) with one of\n" -"S_IFREG, S_IFCHR, S_IFBLK, and S_IFIFO. If S_IFCHR or S_IFBLK is set on mode,\n" -"device defines the newly created device special file (probably using\n" -"os.makedev()). Otherwise device is ignored.\n" +"S_IFREG, S_IFCHR, S_IFBLK, and S_IFIFO. If S_IFCHR or S_IFBLK is set\n" +"on mode, device defines the newly created device special file (probably\n" +"using os.makedev()). Otherwise device is ignored.\n" "\n" -"If dir_fd is not None, it should be a file descriptor open to a directory,\n" -" and path should be relative; path will then be relative to that directory.\n" -"dir_fd may not be implemented on your platform.\n" -" If it is unavailable, using it will raise a NotImplementedError."); +"If dir_fd is not None, it should be a file descriptor open to\n" +"a directory, and path should be relative; path will then be relative\n" +"to that directory.\n" +"dir_fd may not be implemented on your platform. If it is unavailable,\n" +"using it will raise a NotImplementedError."); #define OS_MKNOD_METHODDEF \ {"mknod", _PyCFunction_CAST(os_mknod), METH_FASTCALL|METH_KEYWORDS, os_mknod__doc__}, @@ -9263,8 +9286,9 @@ PyDoc_STRVAR(os_truncate__doc__, "\n" "Truncate a file, specified by path, to a specific length.\n" "\n" -"On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception."); +"On some platforms, path may also be specified as an open file\n" +"descriptor. If this functionality is unavailable, using it raises\n" +"an exception."); #define OS_TRUNCATE_METHODDEF \ {"truncate", _PyCFunction_CAST(os_truncate), METH_FASTCALL|METH_KEYWORDS, os_truncate__doc__}, @@ -9338,7 +9362,8 @@ PyDoc_STRVAR(os_posix_fallocate__doc__, "Ensure a file has allocated at least a particular number of bytes on disk.\n" "\n" "Ensure that the file specified by fd encompasses a range of bytes\n" -"starting at offset bytes from the beginning and continuing for length bytes."); +"starting at offset bytes from the beginning and continuing for length\n" +"bytes."); #define OS_POSIX_FALLOCATE_METHODDEF \ {"posix_fallocate", _PyCFunction_CAST(os_posix_fallocate), METH_FASTCALL, os_posix_fallocate__doc__}, @@ -9384,8 +9409,8 @@ PyDoc_STRVAR(os_posix_fadvise__doc__, "\n" "Announce an intention to access data in a specific pattern.\n" "\n" -"Announce an intention to access data in a specific pattern, thus allowing\n" -"the kernel to make optimizations.\n" +"Announce an intention to access data in a specific pattern, thus\n" +"allowing the kernel to make optimizations.\n" "The advice applies to the region of the file specified by fd starting at\n" "offset and continuing for length bytes.\n" "advice is one of POSIX_FADV_NORMAL, POSIX_FADV_SEQUENTIAL,\n" @@ -10186,8 +10211,9 @@ PyDoc_STRVAR(os_statvfs__doc__, "Perform a statvfs system call on the given path.\n" "\n" "path may always be specified as a string.\n" -"On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception."); +"On some platforms, path may also be specified as an open file\n" +"descriptor. If this functionality is unavailable, using it raises\n" +"an exception."); #define OS_STATVFS_METHODDEF \ {"statvfs", _PyCFunction_CAST(os_statvfs), METH_FASTCALL|METH_KEYWORDS, os_statvfs__doc__}, @@ -10370,8 +10396,9 @@ PyDoc_STRVAR(os_pathconf__doc__, "Return the configuration limit name for the file or directory path.\n" "\n" "If there is no limit, return -1.\n" -"On some platforms, path may also be specified as an open file descriptor.\n" -" If this functionality is unavailable, using it raises an exception."); +"On some platforms, path may also be specified as an open file\n" +"descriptor. If this functionality is unavailable, using it raises\n" +"an exception."); #define OS_PATHCONF_METHODDEF \ {"pathconf", _PyCFunction_CAST(os_pathconf), METH_FASTCALL|METH_KEYWORDS, os_pathconf__doc__}, @@ -10514,8 +10541,8 @@ PyDoc_STRVAR(os_abort__doc__, "\n" "Abort the interpreter immediately.\n" "\n" -"This function \'dumps core\' or otherwise fails in the hardest way possible\n" -"on the hosting operating system. This function never returns."); +"This function \'dumps core\' or otherwise fails in the hardest way\n" +"possible on the hosting operating system. This function never returns."); #define OS_ABORT_METHODDEF \ {"abort", (PyCFunction)os_abort, METH_NOARGS, os_abort__doc__}, @@ -10903,10 +10930,11 @@ PyDoc_STRVAR(os_getxattr__doc__, "\n" "Return the value of extended attribute attribute on path.\n" "\n" -"path may be either a string, a path-like object, or an open file descriptor.\n" -"If follow_symlinks is False, and the last element of the path is a symbolic\n" -" link, getxattr will examine the symbolic link itself instead of the file\n" -" the link points to."); +"path may be either a string, a path-like object, or an open file\n" +"descriptor.\n" +"If follow_symlinks is False, and the last element of the path is\n" +"a symbolic link, getxattr will examine the symbolic link itself\n" +"instead of the file the link points to."); #define OS_GETXATTR_METHODDEF \ {"getxattr", _PyCFunction_CAST(os_getxattr), METH_FASTCALL|METH_KEYWORDS, os_getxattr__doc__}, @@ -10993,10 +11021,11 @@ PyDoc_STRVAR(os_setxattr__doc__, "\n" "Set extended attribute attribute on path to value.\n" "\n" -"path may be either a string, a path-like object, or an open file descriptor.\n" -"If follow_symlinks is False, and the last element of the path is a symbolic\n" -" link, setxattr will modify the symbolic link itself instead of the file\n" -" the link points to."); +"path may be either a string, a path-like object, or an open file\n" +"descriptor.\n" +"If follow_symlinks is False, and the last element of the path is\n" +"a symbolic link, setxattr will modify the symbolic link itself instead\n" +"of the file the link points to."); #define OS_SETXATTR_METHODDEF \ {"setxattr", _PyCFunction_CAST(os_setxattr), METH_FASTCALL|METH_KEYWORDS, os_setxattr__doc__}, @@ -11104,10 +11133,11 @@ PyDoc_STRVAR(os_removexattr__doc__, "\n" "Remove extended attribute attribute on path.\n" "\n" -"path may be either a string, a path-like object, or an open file descriptor.\n" -"If follow_symlinks is False, and the last element of the path is a symbolic\n" -" link, removexattr will modify the symbolic link itself instead of the file\n" -" the link points to."); +"path may be either a string, a path-like object, or an open file\n" +"descriptor.\n" +"If follow_symlinks is False, and the last element of the path is\n" +"a symbolic link, removexattr will modify the symbolic link itself\n" +"instead of the file the link points to."); #define OS_REMOVEXATTR_METHODDEF \ {"removexattr", _PyCFunction_CAST(os_removexattr), METH_FASTCALL|METH_KEYWORDS, os_removexattr__doc__}, @@ -11193,11 +11223,12 @@ PyDoc_STRVAR(os_listxattr__doc__, "\n" "Return a list of extended attributes on path.\n" "\n" -"path may be either None, a string, a path-like object, or an open file descriptor.\n" -"if path is None, listxattr will examine the current directory.\n" -"If follow_symlinks is False, and the last element of the path is a symbolic\n" -" link, listxattr will examine the symbolic link itself instead of the file\n" -" the link points to."); +"path may be either None, a string, a path-like object, or an open file\n" +"descriptor. If path is None, listxattr will examine the current\n" +"directory.\n" +"If follow_symlinks is False, and the last element of the path is\n" +"a symbolic link, listxattr will examine the symbolic link itself instead\n" +"of the file the link points to."); #define OS_LISTXATTR_METHODDEF \ {"listxattr", _PyCFunction_CAST(os_listxattr), METH_FASTCALL|METH_KEYWORDS, os_listxattr__doc__}, @@ -12202,9 +12233,9 @@ PyDoc_STRVAR(os_scandir__doc__, "\n" "Return an iterator of DirEntry objects for given path.\n" "\n" -"path can be specified as either str, bytes, or a path-like object. If path\n" -"is bytes, the names of yielded DirEntry objects will also be bytes; in\n" -"all other circumstances they will be str.\n" +"path can be specified as either str, bytes, or a path-like object. If\n" +"path is bytes, the names of yielded DirEntry objects will also be bytes;\n" +"in all other circumstances they will be str.\n" "\n" "If path is None, uses the path=\'.\'."); @@ -12276,9 +12307,9 @@ PyDoc_STRVAR(os_fspath__doc__, "\n" "Return the file system path representation of the object.\n" "\n" -"If the object is str or bytes, then allow it to pass through as-is. If the\n" -"object defines __fspath__(), then return the result of that method. All other\n" -"types raise a TypeError."); +"If the object is str or bytes, then allow it to pass through as-is. If\n" +"the object defines __fspath__(), then return the result of that method.\n" +"All other types raise a TypeError."); #define OS_FSPATH_METHODDEF \ {"fspath", _PyCFunction_CAST(os_fspath), METH_FASTCALL|METH_KEYWORDS, os_fspath__doc__}, @@ -12572,8 +12603,8 @@ PyDoc_STRVAR(os_waitstatus_to_exitcode__doc__, "On Windows, return status shifted right by 8 bits.\n" "\n" "On Unix, if the process is being traced or if waitpid() was called with\n" -"WUNTRACED option, the caller must first check if WIFSTOPPED(status) is true.\n" -"This function must not be called if WIFSTOPPED(status) is true."); +"WUNTRACED option, the caller must first check if WIFSTOPPED(status) is\n" +"true. This function must not be called if WIFSTOPPED(status) is true."); #define OS_WAITSTATUS_TO_EXITCODE_METHODDEF \ {"waitstatus_to_exitcode", _PyCFunction_CAST(os_waitstatus_to_exitcode), METH_FASTCALL|METH_KEYWORDS, os_waitstatus_to_exitcode__doc__}, @@ -12727,6 +12758,80 @@ os__emscripten_debugger(PyObject *module, PyObject *Py_UNUSED(ignored)) #endif /* defined(__EMSCRIPTEN__) */ +#if defined(__EMSCRIPTEN__) + +PyDoc_STRVAR(os__emscripten_log__doc__, +"_emscripten_log($module, /, arg)\n" +"--\n" +"\n" +"Log something to the JS console. Emscripten only."); + +#define OS__EMSCRIPTEN_LOG_METHODDEF \ + {"_emscripten_log", _PyCFunction_CAST(os__emscripten_log), METH_FASTCALL|METH_KEYWORDS, os__emscripten_log__doc__}, + +static PyObject * +os__emscripten_log_impl(PyObject *module, const char *arg); + +static PyObject * +os__emscripten_log(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(arg), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"arg", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_emscripten_log", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + const char *arg; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("_emscripten_log", "argument 'arg'", "str", args[0]); + goto exit; + } + Py_ssize_t arg_length; + arg = PyUnicode_AsUTF8AndSize(args[0], &arg_length); + if (arg == NULL) { + goto exit; + } + if (strlen(arg) != (size_t)arg_length) { + PyErr_SetString(PyExc_ValueError, "embedded null character"); + goto exit; + } + return_value = os__emscripten_log_impl(module, arg); + +exit: + return return_value; +} + +#endif /* defined(__EMSCRIPTEN__) */ + #ifndef OS_TTYNAME_METHODDEF #define OS_TTYNAME_METHODDEF #endif /* !defined(OS_TTYNAME_METHODDEF) */ @@ -13398,4 +13503,8 @@ os__emscripten_debugger(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__EMSCRIPTEN_DEBUGGER_METHODDEF #define OS__EMSCRIPTEN_DEBUGGER_METHODDEF #endif /* !defined(OS__EMSCRIPTEN_DEBUGGER_METHODDEF) */ -/*[clinic end generated code: output=f7b5635e0b948be4 input=a9049054013a1b77]*/ + +#ifndef OS__EMSCRIPTEN_LOG_METHODDEF + #define OS__EMSCRIPTEN_LOG_METHODDEF +#endif /* !defined(OS__EMSCRIPTEN_LOG_METHODDEF) */ +/*[clinic end generated code: output=37154d6d9778381e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/pyexpat.c.h b/Modules/clinic/pyexpat.c.h index 13210e3be0f747d..aebca46c91f493b 100644 --- a/Modules/clinic/pyexpat.c.h +++ b/Modules/clinic/pyexpat.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_SINGLETON() #endif +#include "pycore_long.h" // _PyLong_UnsignedLongLong_Converter() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(pyexpat_xmlparser_SetReparseDeferralEnabled__doc__, @@ -217,8 +218,9 @@ PyDoc_STRVAR(pyexpat_xmlparser_GetInputContext__doc__, "\n" "Return the untranslated text of the input that caused the current event.\n" "\n" -"If the event was generated by a large amount of text (such as a start tag\n" -"for an element with many attributes), not all of the text may be available."); +"If the event was generated by a large amount of text (such as\n" +"a start tag for an element with many attributes), not all of the\n" +"text may be available."); #define PYEXPAT_XMLPARSER_GETINPUTCONTEXT_METHODDEF \ {"GetInputContext", (PyCFunction)pyexpat_xmlparser_GetInputContext, METH_NOARGS, pyexpat_xmlparser_GetInputContext__doc__}, @@ -356,9 +358,10 @@ PyDoc_STRVAR(pyexpat_xmlparser_UseForeignDTD__doc__, "\n" "Allows the application to provide an artificial external subset if one is not specified as part of the document instance.\n" "\n" -"This readily allows the use of a \'default\' document type controlled by the\n" -"application, while still getting the advantage of providing document type\n" -"information to the parser. \'flag\' defaults to True if not provided."); +"This readily allows the use of a \'default\' document type controlled\n" +"by the application, while still getting the advantage of providing\n" +"document type information to the parser. \'flag\' defaults to True if\n" +"not provided."); #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF \ {"UseForeignDTD", _PyCFunction_CAST(pyexpat_xmlparser_UseForeignDTD), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_UseForeignDTD__doc__}, @@ -408,6 +411,268 @@ pyexpat_xmlparser_UseForeignDTD(PyObject *self, PyTypeObject *cls, PyObject *con #endif /* (XML_COMBINED_VERSION >= 19505) */ +#if (XML_COMBINED_VERSION >= 20400) + +PyDoc_STRVAR(pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold__doc__, +"SetBillionLaughsAttackProtectionActivationThreshold($self, threshold, /)\n" +"--\n" +"\n" +"Sets the number of output bytes needed to activate protection against billion laughs attacks.\n" +"\n" +"The number of output bytes includes amplification from entity expansion\n" +"and reading DTD files.\n" +"\n" +"Parser objects usually have a protection activation threshold of 8 MiB,\n" +"but the actual default value depends on the underlying Expat library.\n" +"\n" +"Activation thresholds below 4 MiB are known to break support for DITA 1.3\n" +"payload and are hence not recommended."); + +#define PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONACTIVATIONTHRESHOLD_METHODDEF \ + {"SetBillionLaughsAttackProtectionActivationThreshold", _PyCFunction_CAST(pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold__doc__}, + +static PyObject * +pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold_impl(xmlparseobject *self, + PyTypeObject *cls, + unsigned long long threshold); + +static PyObject * +pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "SetBillionLaughsAttackProtectionActivationThreshold", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + unsigned long long threshold; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!_PyLong_UnsignedLongLong_Converter(args[0], &threshold)) { + goto exit; + } + return_value = pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold_impl((xmlparseobject *)self, cls, threshold); + +exit: + return return_value; +} + +#endif /* (XML_COMBINED_VERSION >= 20400) */ + +#if (XML_COMBINED_VERSION >= 20400) + +PyDoc_STRVAR(pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification__doc__, +"SetBillionLaughsAttackProtectionMaximumAmplification($self, max_factor,\n" +" /)\n" +"--\n" +"\n" +"Sets the maximum tolerated amplification factor for protection against billion laughs attacks.\n" +"\n" +"The amplification factor is calculated as \"(direct + indirect) / direct\"\n" +"while parsing, where \"direct\" is the number of bytes read from the primary\n" +"document in parsing and \"indirect\" is the number of bytes added by expanding\n" +"entities and reading external DTD files, combined.\n" +"\n" +"The \'max_factor\' value must be a non-NaN floating point value greater than\n" +"or equal to 1.0. Amplification factors greater than 30,000 can be observed\n" +"in the middle of parsing even with benign files in practice. In particular,\n" +"the activation threshold should be carefully chosen to avoid false positives.\n" +"\n" +"Parser objects usually have a maximum amplification factor of 100,\n" +"but the actual default value depends on the underlying Expat library."); + +#define PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONMAXIMUMAMPLIFICATION_METHODDEF \ + {"SetBillionLaughsAttackProtectionMaximumAmplification", _PyCFunction_CAST(pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification__doc__}, + +static PyObject * +pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification_impl(xmlparseobject *self, + PyTypeObject *cls, + float max_factor); + +static PyObject * +pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "SetBillionLaughsAttackProtectionMaximumAmplification", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + float max_factor; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (PyFloat_CheckExact(args[0])) { + max_factor = (float) (PyFloat_AS_DOUBLE(args[0])); + } + else + { + max_factor = (float) PyFloat_AsDouble(args[0]); + if (max_factor == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + return_value = pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification_impl((xmlparseobject *)self, cls, max_factor); + +exit: + return return_value; +} + +#endif /* (XML_COMBINED_VERSION >= 20400) */ + +#if (XML_COMBINED_VERSION >= 20702) + +PyDoc_STRVAR(pyexpat_xmlparser_SetAllocTrackerActivationThreshold__doc__, +"SetAllocTrackerActivationThreshold($self, threshold, /)\n" +"--\n" +"\n" +"Sets the number of allocated bytes of dynamic memory needed to activate protection against disproportionate use of RAM.\n" +"\n" +"Parser objects usually have an allocation activation threshold of 64 MiB,\n" +"but the actual default value depends on the underlying Expat library."); + +#define PYEXPAT_XMLPARSER_SETALLOCTRACKERACTIVATIONTHRESHOLD_METHODDEF \ + {"SetAllocTrackerActivationThreshold", _PyCFunction_CAST(pyexpat_xmlparser_SetAllocTrackerActivationThreshold), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_SetAllocTrackerActivationThreshold__doc__}, + +static PyObject * +pyexpat_xmlparser_SetAllocTrackerActivationThreshold_impl(xmlparseobject *self, + PyTypeObject *cls, + unsigned long long threshold); + +static PyObject * +pyexpat_xmlparser_SetAllocTrackerActivationThreshold(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "SetAllocTrackerActivationThreshold", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + unsigned long long threshold; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!_PyLong_UnsignedLongLong_Converter(args[0], &threshold)) { + goto exit; + } + return_value = pyexpat_xmlparser_SetAllocTrackerActivationThreshold_impl((xmlparseobject *)self, cls, threshold); + +exit: + return return_value; +} + +#endif /* (XML_COMBINED_VERSION >= 20702) */ + +#if (XML_COMBINED_VERSION >= 20702) + +PyDoc_STRVAR(pyexpat_xmlparser_SetAllocTrackerMaximumAmplification__doc__, +"SetAllocTrackerMaximumAmplification($self, max_factor, /)\n" +"--\n" +"\n" +"Sets the maximum amplification factor between direct input and bytes of dynamic memory allocated.\n" +"\n" +"The amplification factor is calculated as \"allocated / direct\" while\n" +"parsing, where \"direct\" is the number of bytes read from the primary\n" +"document in parsing and \"allocated\" is the number of bytes of\n" +"dynamic memory allocated in the parser hierarchy.\n" +"\n" +"The \'max_factor\' value must be a non-NaN floating point value\n" +"greater than or equal to 1.0. Amplification factors greater than\n" +"100.0 can be observed near the start of parsing even with benign\n" +"files in practice. In particular, the activation threshold should\n" +"be carefully chosen to avoid false positives.\n" +"\n" +"Parser objects usually have a maximum amplification factor of 100,\n" +"but the actual default value depends on the underlying Expat library."); + +#define PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF \ + {"SetAllocTrackerMaximumAmplification", _PyCFunction_CAST(pyexpat_xmlparser_SetAllocTrackerMaximumAmplification), METH_METHOD|METH_FASTCALL|METH_KEYWORDS, pyexpat_xmlparser_SetAllocTrackerMaximumAmplification__doc__}, + +static PyObject * +pyexpat_xmlparser_SetAllocTrackerMaximumAmplification_impl(xmlparseobject *self, + PyTypeObject *cls, + float max_factor); + +static PyObject * +pyexpat_xmlparser_SetAllocTrackerMaximumAmplification(PyObject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + # define KWTUPLE (PyObject *)&_Py_SINGLETON(tuple_empty) + #else + # define KWTUPLE NULL + #endif + + static const char * const _keywords[] = {"", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "SetAllocTrackerMaximumAmplification", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; + float max_factor; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (PyFloat_CheckExact(args[0])) { + max_factor = (float) (PyFloat_AS_DOUBLE(args[0])); + } + else + { + max_factor = (float) PyFloat_AsDouble(args[0]); + if (max_factor == -1.0 && PyErr_Occurred()) { + goto exit; + } + } + return_value = pyexpat_xmlparser_SetAllocTrackerMaximumAmplification_impl((xmlparseobject *)self, cls, max_factor); + +exit: + return return_value; +} + +#endif /* (XML_COMBINED_VERSION >= 20702) */ + PyDoc_STRVAR(pyexpat_ParserCreate__doc__, "ParserCreate($module, /, encoding=None, namespace_separator=None,\n" " intern=<unrepresentable>)\n" @@ -552,4 +817,20 @@ pyexpat_ErrorString(PyObject *module, PyObject *arg) #ifndef PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #define PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF #endif /* !defined(PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF) */ -/*[clinic end generated code: output=4dbdc959c67dc2d5 input=a9049054013a1b77]*/ + +#ifndef PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONACTIVATIONTHRESHOLD_METHODDEF + #define PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONACTIVATIONTHRESHOLD_METHODDEF +#endif /* !defined(PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONACTIVATIONTHRESHOLD_METHODDEF) */ + +#ifndef PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONMAXIMUMAMPLIFICATION_METHODDEF + #define PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONMAXIMUMAMPLIFICATION_METHODDEF +#endif /* !defined(PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONMAXIMUMAMPLIFICATION_METHODDEF) */ + +#ifndef PYEXPAT_XMLPARSER_SETALLOCTRACKERACTIVATIONTHRESHOLD_METHODDEF + #define PYEXPAT_XMLPARSER_SETALLOCTRACKERACTIVATIONTHRESHOLD_METHODDEF +#endif /* !defined(PYEXPAT_XMLPARSER_SETALLOCTRACKERACTIVATIONTHRESHOLD_METHODDEF) */ + +#ifndef PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF + #define PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF +#endif /* !defined(PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF) */ +/*[clinic end generated code: output=b03765d16720ab5e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/resource.c.h b/Modules/clinic/resource.c.h index 9eda7de27532a17..e4ef93900d17978 100644 --- a/Modules/clinic/resource.c.h +++ b/Modules/clinic/resource.c.h @@ -2,6 +2,8 @@ preserve [clinic start generated code]*/ +#include "pycore_modsupport.h" // _PyArg_CheckPositional() + #if defined(HAVE_GETRUSAGE) PyDoc_STRVAR(resource_getrusage__doc__, @@ -66,7 +68,7 @@ PyDoc_STRVAR(resource_setrlimit__doc__, "\n"); #define RESOURCE_SETRLIMIT_METHODDEF \ - {"setrlimit", (PyCFunction)(void(*)(void))resource_setrlimit, METH_FASTCALL, resource_setrlimit__doc__}, + {"setrlimit", _PyCFunction_CAST(resource_setrlimit), METH_FASTCALL, resource_setrlimit__doc__}, static PyObject * resource_setrlimit_impl(PyObject *module, int resource, PyObject *limits); @@ -78,8 +80,7 @@ resource_setrlimit(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int resource; PyObject *limits; - if (nargs != 2) { - PyErr_Format(PyExc_TypeError, "setrlimit expected 2 arguments, got %zd", nargs); + if (!_PyArg_CheckPositional("setrlimit", nargs, 2, 2)) { goto exit; } resource = PyLong_AsInt(args[0]); @@ -101,7 +102,7 @@ PyDoc_STRVAR(resource_prlimit__doc__, "\n"); #define RESOURCE_PRLIMIT_METHODDEF \ - {"prlimit", (PyCFunction)(void(*)(void))resource_prlimit, METH_FASTCALL, resource_prlimit__doc__}, + {"prlimit", _PyCFunction_CAST(resource_prlimit), METH_FASTCALL, resource_prlimit__doc__}, static PyObject * resource_prlimit_impl(PyObject *module, pid_t pid, int resource, @@ -115,12 +116,7 @@ resource_prlimit(PyObject *module, PyObject *const *args, Py_ssize_t nargs) int resource; PyObject *limits = Py_None; - if (nargs < 2) { - PyErr_Format(PyExc_TypeError, "prlimit expected at least 2 arguments, got %zd", nargs); - goto exit; - } - if (nargs > 3) { - PyErr_Format(PyExc_TypeError, "prlimit expected at most 3 arguments, got %zd", nargs); + if (!_PyArg_CheckPositional("prlimit", nargs, 2, 3)) { goto exit; } pid = PyLong_AsPid(args[0]); @@ -178,4 +174,4 @@ resource_getpagesize(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef RESOURCE_PRLIMIT_METHODDEF #define RESOURCE_PRLIMIT_METHODDEF #endif /* !defined(RESOURCE_PRLIMIT_METHODDEF) */ -/*[clinic end generated code: output=e45883ace510414a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=8e905b2f5c35170e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/selectmodule.c.h b/Modules/clinic/selectmodule.c.h index 253ad8c9e78f00c..c6d0e5b1b81f50f 100644 --- a/Modules/clinic/selectmodule.c.h +++ b/Modules/clinic/selectmodule.c.h @@ -16,7 +16,8 @@ PyDoc_STRVAR(select_select__doc__, "\n" "Wait until one or more file descriptors are ready for some kind of I/O.\n" "\n" -"The first three arguments are iterables of file descriptors to be waited for:\n" +"The first three arguments are iterables of file descriptors to be waited\n" +"for:\n" "rlist -- wait until ready for reading\n" "wlist -- wait until ready for writing\n" "xlist -- wait for an \"exceptional condition\"\n" @@ -29,9 +30,9 @@ PyDoc_STRVAR(select_select__doc__, "a floating-point number to specify fractions of seconds. If it is absent\n" "or None, the call will never time out.\n" "\n" -"The return value is a tuple of three lists corresponding to the first three\n" -"arguments; each contains the subset of the corresponding file descriptors\n" -"that are ready.\n" +"The return value is a tuple of three lists corresponding to the first\n" +"three arguments; each contains the subset of the corresponding file\n" +"descriptors that are ready.\n" "\n" "*** IMPORTANT NOTICE ***\n" "On Windows, only sockets are supported; on Unix, all file\n" @@ -214,8 +215,8 @@ PyDoc_STRVAR(select_poll_poll__doc__, " The maximum time to wait in milliseconds, or else None (or a negative\n" " value) to wait indefinitely.\n" "\n" -"Returns a list containing any descriptors that have events or errors to\n" -"report, as a list of (fd, event) 2-tuples."); +"Returns a list containing any descriptors that have events or errors\n" +"to report, as a list of (fd, event) 2-tuples."); #define SELECT_POLL_POLL_METHODDEF \ {"poll", _PyCFunction_CAST(select_poll_poll), METH_FASTCALL, select_poll_poll__doc__}, @@ -396,11 +397,11 @@ PyDoc_STRVAR(select_devpoll_poll__doc__, "Polls the set of registered file descriptors.\n" "\n" " timeout\n" -" The maximum time to wait in milliseconds, or else None (or a negative\n" -" value) to wait indefinitely.\n" +" The maximum time to wait in milliseconds, or else None (or\n" +" a negative value) to wait indefinitely.\n" "\n" -"Returns a list containing any descriptors that have events or errors to\n" -"report, as a list of (fd, event) 2-tuples."); +"Returns a list containing any descriptors that have events or errors\n" +"to report, as a list of (fd, event) 2-tuples."); #define SELECT_DEVPOLL_POLL_METHODDEF \ {"poll", _PyCFunction_CAST(select_devpoll_poll), METH_FASTCALL, select_devpoll_poll__doc__}, @@ -498,8 +499,8 @@ PyDoc_STRVAR(select_poll__doc__, "\n" "Returns a polling object.\n" "\n" -"This object supports registering and unregistering file descriptors, and then\n" -"polling them for I/O events."); +"This object supports registering and unregistering file descriptors, and\n" +"then polling them for I/O events."); #define SELECT_POLL_METHODDEF \ {"poll", (PyCFunction)select_poll, METH_NOARGS, select_poll__doc__}, @@ -523,8 +524,8 @@ PyDoc_STRVAR(select_devpoll__doc__, "\n" "Returns a polling object.\n" "\n" -"This object supports registering and unregistering file descriptors, and then\n" -"polling them for I/O events."); +"This object supports registering and unregistering file descriptors, and\n" +"then polling them for I/O events."); #define SELECT_DEVPOLL_METHODDEF \ {"devpoll", (PyCFunction)select_devpoll, METH_NOARGS, select_devpoll__doc__}, @@ -954,8 +955,8 @@ PyDoc_STRVAR(select_epoll_poll__doc__, " maxevents\n" " the maximum number of events returned; -1 means no limit\n" "\n" -"Returns a list containing any descriptors that have events to report,\n" -"as a list of (fd, events) 2-tuples."); +"Returns a list containing any descriptors that have events to\n" +"report, as a list of (fd, events) 2-tuples."); #define SELECT_EPOLL_POLL_METHODDEF \ {"poll", _PyCFunction_CAST(select_epoll_poll), METH_FASTCALL|METH_KEYWORDS, select_epoll_poll__doc__}, @@ -1375,4 +1376,4 @@ select_kqueue_control(PyObject *self, PyObject *const *args, Py_ssize_t nargs) #ifndef SELECT_KQUEUE_CONTROL_METHODDEF #define SELECT_KQUEUE_CONTROL_METHODDEF #endif /* !defined(SELECT_KQUEUE_CONTROL_METHODDEF) */ -/*[clinic end generated code: output=6fc20d78802511d1 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=aac002a543ecb778 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha1module.c.h b/Modules/clinic/sha1module.c.h index 3e5fd1a41ce21fd..4a58d0cd9b82a41 100644 --- a/Modules/clinic/sha1module.c.h +++ b/Modules/clinic/sha1module.c.h @@ -89,7 +89,7 @@ SHA1Type_update(PyObject *self, PyObject *obj) } PyDoc_STRVAR(_sha1_sha1__doc__, -"sha1($module, /, string=b\'\', *, usedforsecurity=True)\n" +"sha1($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new SHA1 hash object; optionally initialized with a string."); @@ -98,7 +98,8 @@ PyDoc_STRVAR(_sha1_sha1__doc__, {"sha1", _PyCFunction_CAST(_sha1_sha1), METH_FASTCALL|METH_KEYWORDS, _sha1_sha1__doc__}, static PyObject * -_sha1_sha1_impl(PyObject *module, PyObject *string, int usedforsecurity); +_sha1_sha1_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj); static PyObject * _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -106,7 +107,7 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -115,7 +116,7 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -124,17 +125,18 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "sha1", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string_obj = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -145,7 +147,7 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * goto skip_optional_pos; } if (args[0]) { - string = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -154,14 +156,20 @@ _sha1_sha1(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject * if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string_obj = args[2]; skip_optional_kwonly: - return_value = _sha1_sha1_impl(module, string, usedforsecurity); + return_value = _sha1_sha1_impl(module, data, usedforsecurity, string_obj); exit: return return_value; } -/*[clinic end generated code: output=06161e87e2d645d4 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fd5a917404b68c4f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha2module.c.h b/Modules/clinic/sha2module.c.h index 26612125e75df97..07be91e4f6c68f2 100644 --- a/Modules/clinic/sha2module.c.h +++ b/Modules/clinic/sha2module.c.h @@ -169,7 +169,7 @@ SHA512Type_update(PyObject *self, PyObject *obj) } PyDoc_STRVAR(_sha2_sha256__doc__, -"sha256($module, /, string=b\'\', *, usedforsecurity=True)\n" +"sha256($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new SHA-256 hash object; optionally initialized with a string."); @@ -178,7 +178,8 @@ PyDoc_STRVAR(_sha2_sha256__doc__, {"sha256", _PyCFunction_CAST(_sha2_sha256), METH_FASTCALL|METH_KEYWORDS, _sha2_sha256__doc__}, static PyObject * -_sha2_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity); +_sha2_sha256_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj); static PyObject * _sha2_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -186,7 +187,7 @@ _sha2_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -195,7 +196,7 @@ _sha2_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -204,17 +205,18 @@ _sha2_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "sha256", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string_obj = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -225,7 +227,7 @@ _sha2_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto skip_optional_pos; } if (args[0]) { - string = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -234,19 +236,25 @@ _sha2_sha256(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string_obj = args[2]; skip_optional_kwonly: - return_value = _sha2_sha256_impl(module, string, usedforsecurity); + return_value = _sha2_sha256_impl(module, data, usedforsecurity, string_obj); exit: return return_value; } PyDoc_STRVAR(_sha2_sha224__doc__, -"sha224($module, /, string=b\'\', *, usedforsecurity=True)\n" +"sha224($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new SHA-224 hash object; optionally initialized with a string."); @@ -255,7 +263,8 @@ PyDoc_STRVAR(_sha2_sha224__doc__, {"sha224", _PyCFunction_CAST(_sha2_sha224), METH_FASTCALL|METH_KEYWORDS, _sha2_sha224__doc__}, static PyObject * -_sha2_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity); +_sha2_sha224_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj); static PyObject * _sha2_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -263,7 +272,7 @@ _sha2_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -272,7 +281,7 @@ _sha2_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -281,17 +290,18 @@ _sha2_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "sha224", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string_obj = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -302,7 +312,7 @@ _sha2_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto skip_optional_pos; } if (args[0]) { - string = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -311,19 +321,25 @@ _sha2_sha224(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string_obj = args[2]; skip_optional_kwonly: - return_value = _sha2_sha224_impl(module, string, usedforsecurity); + return_value = _sha2_sha224_impl(module, data, usedforsecurity, string_obj); exit: return return_value; } PyDoc_STRVAR(_sha2_sha512__doc__, -"sha512($module, /, string=b\'\', *, usedforsecurity=True)\n" +"sha512($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new SHA-512 hash object; optionally initialized with a string."); @@ -332,7 +348,8 @@ PyDoc_STRVAR(_sha2_sha512__doc__, {"sha512", _PyCFunction_CAST(_sha2_sha512), METH_FASTCALL|METH_KEYWORDS, _sha2_sha512__doc__}, static PyObject * -_sha2_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity); +_sha2_sha512_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj); static PyObject * _sha2_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -340,7 +357,7 @@ _sha2_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -349,7 +366,7 @@ _sha2_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -358,17 +375,18 @@ _sha2_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "sha512", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string_obj = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -379,7 +397,7 @@ _sha2_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto skip_optional_pos; } if (args[0]) { - string = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -388,19 +406,25 @@ _sha2_sha512(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string_obj = args[2]; skip_optional_kwonly: - return_value = _sha2_sha512_impl(module, string, usedforsecurity); + return_value = _sha2_sha512_impl(module, data, usedforsecurity, string_obj); exit: return return_value; } PyDoc_STRVAR(_sha2_sha384__doc__, -"sha384($module, /, string=b\'\', *, usedforsecurity=True)\n" +"sha384($module, /, data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new SHA-384 hash object; optionally initialized with a string."); @@ -409,7 +433,8 @@ PyDoc_STRVAR(_sha2_sha384__doc__, {"sha384", _PyCFunction_CAST(_sha2_sha384), METH_FASTCALL|METH_KEYWORDS, _sha2_sha384__doc__}, static PyObject * -_sha2_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity); +_sha2_sha384_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj); static PyObject * _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -417,7 +442,7 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -426,7 +451,7 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(string), &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -435,17 +460,18 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "sha384", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; - PyObject *string = NULL; + PyObject *data = NULL; int usedforsecurity = 1; + PyObject *string_obj = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); @@ -456,7 +482,7 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject goto skip_optional_pos; } if (args[0]) { - string = args[0]; + data = args[0]; if (!--noptargs) { goto skip_optional_pos; } @@ -465,14 +491,20 @@ _sha2_sha384(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(args[1]); - if (usedforsecurity < 0) { - goto exit; + if (args[1]) { + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string_obj = args[2]; skip_optional_kwonly: - return_value = _sha2_sha384_impl(module, string, usedforsecurity); + return_value = _sha2_sha384_impl(module, data, usedforsecurity, string_obj); exit: return return_value; } -/*[clinic end generated code: output=af11090855b7c85a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=90625b237c774a9f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/sha3module.c.h b/Modules/clinic/sha3module.c.h index 25f72b74f801dbb..121be2c07586956 100644 --- a/Modules/clinic/sha3module.c.h +++ b/Modules/clinic/sha3module.c.h @@ -10,13 +10,14 @@ preserve #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(py_sha3_new__doc__, -"sha3_224(data=b\'\', /, *, usedforsecurity=True)\n" +"sha3_224(data=b\'\', *, usedforsecurity=True, string=None)\n" "--\n" "\n" "Return a new SHA3 hash object."); static PyObject * -py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity); +py_sha3_new_impl(PyTypeObject *type, PyObject *data_obj, int usedforsecurity, + PyObject *string); static PyObject * py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) @@ -24,7 +25,7 @@ py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 1 + #define NUM_KEYWORDS 3 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -33,7 +34,7 @@ py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(usedforsecurity), }, + .ob_item = { &_Py_ID(data), &_Py_ID(usedforsecurity), &_Py_ID(string), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -42,40 +43,51 @@ py_sha3_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"", "usedforsecurity", NULL}; + static const char * const _keywords[] = {"data", "usedforsecurity", "string", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "sha3_224", .kwtuple = KWTUPLE, }; #undef KWTUPLE - PyObject *argsbuf[2]; + PyObject *argsbuf[3]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 0; - PyObject *data = NULL; + PyObject *data_obj = NULL; int usedforsecurity = 1; + PyObject *string = NULL; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, /*minpos*/ 0, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } - if (nargs < 1) { - goto skip_optional_posonly; + if (!noptargs) { + goto skip_optional_pos; + } + if (fastargs[0]) { + data_obj = fastargs[0]; + if (!--noptargs) { + goto skip_optional_pos; + } } - noptargs--; - data = fastargs[0]; -skip_optional_posonly: +skip_optional_pos: if (!noptargs) { goto skip_optional_kwonly; } - usedforsecurity = PyObject_IsTrue(fastargs[1]); - if (usedforsecurity < 0) { - goto exit; + if (fastargs[1]) { + usedforsecurity = PyObject_IsTrue(fastargs[1]); + if (usedforsecurity < 0) { + goto exit; + } + if (!--noptargs) { + goto skip_optional_kwonly; + } } + string = fastargs[2]; skip_optional_kwonly: - return_value = py_sha3_new_impl(type, data, usedforsecurity); + return_value = py_sha3_new_impl(type, data_obj, usedforsecurity, string); exit: return return_value; @@ -158,24 +170,57 @@ _sha3_sha3_224_update(PyObject *self, PyObject *data) } PyDoc_STRVAR(_sha3_shake_128_digest__doc__, -"digest($self, length, /)\n" +"digest($self, /, length)\n" "--\n" "\n" "Return the digest value as a bytes object."); #define _SHA3_SHAKE_128_DIGEST_METHODDEF \ - {"digest", (PyCFunction)_sha3_shake_128_digest, METH_O, _sha3_shake_128_digest__doc__}, + {"digest", _PyCFunction_CAST(_sha3_shake_128_digest), METH_FASTCALL|METH_KEYWORDS, _sha3_shake_128_digest__doc__}, static PyObject * _sha3_shake_128_digest_impl(SHA3object *self, unsigned long length); static PyObject * -_sha3_shake_128_digest(PyObject *self, PyObject *arg) +_sha3_shake_128_digest(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(length), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"length", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "digest", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; unsigned long length; - if (!_PyLong_UnsignedLong_Converter(arg, &length)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!_PyLong_UnsignedLong_Converter(args[0], &length)) { goto exit; } return_value = _sha3_shake_128_digest_impl((SHA3object *)self, length); @@ -185,24 +230,57 @@ _sha3_shake_128_digest(PyObject *self, PyObject *arg) } PyDoc_STRVAR(_sha3_shake_128_hexdigest__doc__, -"hexdigest($self, length, /)\n" +"hexdigest($self, /, length)\n" "--\n" "\n" "Return the digest value as a string of hexadecimal digits."); #define _SHA3_SHAKE_128_HEXDIGEST_METHODDEF \ - {"hexdigest", (PyCFunction)_sha3_shake_128_hexdigest, METH_O, _sha3_shake_128_hexdigest__doc__}, + {"hexdigest", _PyCFunction_CAST(_sha3_shake_128_hexdigest), METH_FASTCALL|METH_KEYWORDS, _sha3_shake_128_hexdigest__doc__}, static PyObject * _sha3_shake_128_hexdigest_impl(SHA3object *self, unsigned long length); static PyObject * -_sha3_shake_128_hexdigest(PyObject *self, PyObject *arg) +_sha3_shake_128_hexdigest(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + Py_hash_t ob_hash; + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_hash = -1, + .ob_item = { &_Py_ID(length), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"length", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "hexdigest", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; unsigned long length; - if (!_PyLong_UnsignedLong_Converter(arg, &length)) { + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + if (!args) { + goto exit; + } + if (!_PyLong_UnsignedLong_Converter(args[0], &length)) { goto exit; } return_value = _sha3_shake_128_hexdigest_impl((SHA3object *)self, length); @@ -210,4 +288,4 @@ _sha3_shake_128_hexdigest(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=5b3ac1c06c6899ea input=a9049054013a1b77]*/ +/*[clinic end generated code: output=65e437799472b89f input=a9049054013a1b77]*/ diff --git a/Modules/clinic/signalmodule.c.h b/Modules/clinic/signalmodule.c.h index 955861b4da37d9c..0c694a7d276876b 100644 --- a/Modules/clinic/signalmodule.c.h +++ b/Modules/clinic/signalmodule.c.h @@ -138,11 +138,12 @@ PyDoc_STRVAR(signal_signal__doc__, "Set the action for the given signal.\n" "\n" "The action can be SIG_DFL, SIG_IGN, or a callable Python object.\n" -"The previous action is returned. See getsignal() for possible return values.\n" +"The previous action is returned. See getsignal() for possible return\n" +"values.\n" "\n" "*** IMPORTANT NOTICE ***\n" -"A signal handler function is called with two arguments:\n" -"the first is the signal number, the second is the interrupted stack frame."); +"A signal handler function is called with two arguments: the first is\n" +"the signal number, the second is the interrupted stack frame."); #define SIGNAL_SIGNAL_METHODDEF \ {"signal", _PyCFunction_CAST(signal_signal), METH_FASTCALL, signal_signal__doc__}, @@ -362,8 +363,8 @@ PyDoc_STRVAR(signal_setitimer__doc__, "\n" "Sets given itimer (one of ITIMER_REAL, ITIMER_VIRTUAL or ITIMER_PROF).\n" "\n" -"The timer will fire after value seconds and after that every interval seconds.\n" -"The itimer can be cleared by setting seconds to zero.\n" +"The timer will fire after value seconds and after that every interval\n" +"seconds. The itimer can be cleared by setting seconds to zero.\n" "\n" "Returns old values as a tuple: (delay, interval)."); @@ -508,8 +509,8 @@ PyDoc_STRVAR(signal_sigwait__doc__, "Wait for a signal.\n" "\n" "Suspend execution of the calling thread until the delivery of one of the\n" -"signals specified in the signal set sigset. The function accepts the signal\n" -"and returns the signal number."); +"signals specified in the signal set sigset. The function accepts the\n" +"signal and returns the signal number."); #define SIGNAL_SIGWAIT_METHODDEF \ {"sigwait", (PyCFunction)signal_sigwait, METH_O, signal_sigwait__doc__}, @@ -779,4 +780,4 @@ signal_pidfd_send_signal(PyObject *module, PyObject *const *args, Py_ssize_t nar #ifndef SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #define SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF #endif /* !defined(SIGNAL_PIDFD_SEND_SIGNAL_METHODDEF) */ -/*[clinic end generated code: output=48bfaffeb25df5d2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a6158ef9ca1b5c2e input=a9049054013a1b77]*/ diff --git a/Modules/clinic/socketmodule.c.h b/Modules/clinic/socketmodule.c.h index 70ebbaa876b32b1..2b27640cd0aaf37 100644 --- a/Modules/clinic/socketmodule.c.h +++ b/Modules/clinic/socketmodule.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_long.h" // _PyLong_UInt16_Converter() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_socket_socket_close__doc__, @@ -314,7 +315,7 @@ static PyObject * _socket_if_nametoindex(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - PyObject *oname; + PyObject *oname = NULL; if (!PyUnicode_FSConverter(arg, &oname)) { goto exit; @@ -322,6 +323,9 @@ _socket_if_nametoindex(PyObject *module, PyObject *arg) return_value = _socket_if_nametoindex_impl(module, oname); exit: + /* Cleanup for oname */ + Py_XDECREF(oname); + return return_value; } @@ -369,4 +373,4 @@ _socket_if_indextoname(PyObject *module, PyObject *arg) #ifndef _SOCKET_IF_INDEXTONAME_METHODDEF #define _SOCKET_IF_INDEXTONAME_METHODDEF #endif /* !defined(_SOCKET_IF_INDEXTONAME_METHODDEF) */ -/*[clinic end generated code: output=c971b79d2193b426 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=1bbfda36b02c29c2 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/symtablemodule.c.h b/Modules/clinic/symtablemodule.c.h index 2ecd3afc00d2be4..bd55d77c5409e93 100644 --- a/Modules/clinic/symtablemodule.c.h +++ b/Modules/clinic/symtablemodule.c.h @@ -22,7 +22,7 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; PyObject *source; - PyObject *filename; + PyObject *filename = NULL; const char *startstr; if (!_PyArg_CheckPositional("symtable", nargs, 3, 3)) { @@ -48,6 +48,9 @@ _symtable_symtable(PyObject *module, PyObject *const *args, Py_ssize_t nargs) return_value = _symtable_symtable_impl(module, source, filename, startstr); exit: + /* Cleanup for filename */ + Py_XDECREF(filename); + return return_value; } -/*[clinic end generated code: output=931964a76a72f850 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7a8545d9a1efe837 input=a9049054013a1b77]*/ diff --git a/Modules/clinic/termios.c.h b/Modules/clinic/termios.c.h index 83f5a4f6e9f8820..35522bef1dcae9d 100644 --- a/Modules/clinic/termios.c.h +++ b/Modules/clinic/termios.c.h @@ -270,7 +270,8 @@ PyDoc_STRVAR(termios_tcsetwinsize__doc__, "Set the tty winsize for file descriptor fd.\n" "\n" "The winsize to be set is taken from the winsize argument, which\n" -"is a two-item tuple (ws_row, ws_col) like the one returned by tcgetwinsize()."); +"is a two-item tuple (ws_row, ws_col) like the one returned by\n" +"tcgetwinsize()."); #define TERMIOS_TCSETWINSIZE_METHODDEF \ {"tcsetwinsize", (PyCFunction)(void(*)(void))termios_tcsetwinsize, METH_FASTCALL, termios_tcsetwinsize__doc__}, @@ -299,4 +300,4 @@ termios_tcsetwinsize(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=c6c6192583b0da36 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d2176c4d9043d3cc input=a9049054013a1b77]*/ diff --git a/Modules/clinic/zlibmodule.c.h b/Modules/clinic/zlibmodule.c.h index 2710f65a840db93..e4ccd79e371bc98 100644 --- a/Modules/clinic/zlibmodule.c.h +++ b/Modules/clinic/zlibmodule.c.h @@ -205,7 +205,7 @@ zlib_decompress(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyDoc_STRVAR(zlib_compressobj__doc__, "compressobj($module, /, level=Z_DEFAULT_COMPRESSION, method=DEFLATED,\n" " wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL,\n" -" strategy=Z_DEFAULT_STRATEGY, zdict=None)\n" +" strategy=Z_DEFAULT_STRATEGY, zdict=<unrepresentable>)\n" "--\n" "\n" "Return a compressor object.\n" @@ -498,8 +498,8 @@ PyDoc_STRVAR(zlib_Decompress_decompress__doc__, " Unconsumed input data will be stored in\n" " the unconsumed_tail attribute.\n" "\n" -"After calling this function, some of the input data may still be stored in\n" -"internal buffers for later processing.\n" +"After calling this function, some of the input data may still be\n" +"stored in internal buffers for later processing.\n" "Call the flush() method to clear these buffers."); #define ZLIB_DECOMPRESS_DECOMPRESS_METHODDEF \ @@ -904,18 +904,19 @@ PyDoc_STRVAR(zlib_ZlibDecompressor_decompress__doc__, "\n" "Decompress *data*, returning uncompressed data as bytes.\n" "\n" -"If *max_length* is nonnegative, returns at most *max_length* bytes of\n" -"decompressed data. If this limit is reached and further output can be\n" -"produced, *self.needs_input* will be set to ``False``. In this case, the next\n" -"call to *decompress()* may provide *data* as b\'\' to obtain more of the output.\n" +"If *max_length* is nonnegative, returns at most *max_length* bytes\n" +"of decompressed data. If this limit is reached and further output\n" +"can be produced, *self.needs_input* will be set to ``False``. In\n" +"this case, the next call to *decompress()* may provide *data* as b\'\'\n" +"to obtain more of the output.\n" "\n" -"If all of the input data was decompressed and returned (either because this\n" -"was less than *max_length* bytes, or because *max_length* was negative),\n" -"*self.needs_input* will be set to True.\n" +"If all of the input data was decompressed and returned (either\n" +"because this was less than *max_length* bytes, or because\n" +"*max_length* was negative), *self.needs_input* will be set to True.\n" "\n" -"Attempting to decompress data after the end of stream is reached raises an\n" -"EOFError. Any data found after the end of the stream is ignored and saved in\n" -"the unused_data attribute."); +"Attempting to decompress data after the end of stream is reached\n" +"raises an EOFError. Any data found after the end of the stream is\n" +"ignored and saved in the unused_data attribute."); #define ZLIB_ZLIBDECOMPRESSOR_DECOMPRESS_METHODDEF \ {"decompress", _PyCFunction_CAST(zlib_ZlibDecompressor_decompress), METH_FASTCALL|METH_KEYWORDS, zlib_ZlibDecompressor_decompress__doc__}, @@ -1121,4 +1122,4 @@ zlib_crc32(PyObject *module, PyObject *const *args, Py_ssize_t nargs) #ifndef ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #define ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF #endif /* !defined(ZLIB_DECOMPRESS___DEEPCOPY___METHODDEF) */ -/*[clinic end generated code: output=33938c7613a8c1c7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=99936d9996e595ff input=a9049054013a1b77]*/ diff --git a/Modules/cmathmodule.c b/Modules/cmathmodule.c index 81cbf0d554de3c9..77ae7791ca0d695 100644 --- a/Modules/cmathmodule.c +++ b/Modules/cmathmodule.c @@ -163,8 +163,15 @@ special_type(double d) raised. */ -static Py_complex acos_special_values[7][7]; - +static Py_complex acos_special_values[7][7] = { + { {P34,INF}, {P,INF}, {P,INF}, {P,-INF}, {P,-INF}, {P34,-INF}, {N,INF} }, + { {P12,INF}, {U,U}, {U,U}, {U,U}, {U,U}, {P12,-INF}, {N,N} }, + { {P12,INF}, {U,U}, {P12,0.}, {P12,-0.}, {U,U}, {P12,-INF}, {P12,N} }, + { {P12,INF}, {U,U}, {P12,0.}, {P12,-0.}, {U,U}, {P12,-INF}, {P12,N} }, + { {P12,INF}, {U,U}, {U,U}, {U,U}, {U,U}, {P12,-INF}, {N,N} }, + { {P14,INF}, {0.,INF}, {0.,INF}, {0.,-INF}, {0.,-INF}, {P14,-INF}, {N,INF} }, + { {N,INF}, {N,N}, {N,N}, {N,N}, {N,N}, {N,-INF}, {N,N} } +}; /*[clinic input] cmath.acos -> Py_complex_protected @@ -202,7 +209,15 @@ cmath_acos_impl(PyObject *module, Py_complex z) } -static Py_complex acosh_special_values[7][7]; +static Py_complex acosh_special_values[7][7] = { + { {INF,-P34}, {INF,-P}, {INF,-P}, {INF,P}, {INF,P}, {INF,P34}, {INF,N} }, + { {INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {0.,-P12}, {0.,P12}, {U,U}, {INF,P12}, {N,P12} }, + { {INF,-P12}, {U,U}, {0.,-P12}, {0.,P12}, {U,U}, {INF,P12}, {N,P12} }, + { {INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P14}, {INF,-0.}, {INF,-0.}, {INF,0.}, {INF,0.}, {INF,P14}, {INF,N} }, + { {INF,N}, {N,N}, {N,N}, {N,N}, {N,N}, {INF,N}, {N,N} } +}; /*[clinic input] cmath.acosh = cmath.acos @@ -257,7 +272,15 @@ cmath_asin_impl(PyObject *module, Py_complex z) } -static Py_complex asinh_special_values[7][7]; +static Py_complex asinh_special_values[7][7] = { + { {-INF,-P14}, {-INF,-0.}, {-INF,-0.}, {-INF,0.}, {-INF,0.}, {-INF,P14}, {-INF,N} }, + { {-INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {-INF,P12}, {N,N} }, + { {-INF,-P12}, {U,U}, {-0.,-0.}, {-0.,0.}, {U,U}, {-INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P14}, {INF,-0.}, {INF,-0.}, {INF,0.}, {INF,0.}, {INF,P14}, {INF,N} }, + { {INF,N}, {N,N}, {N,-0.}, {N,0.}, {N,N}, {INF,N}, {N,N} } +}; /*[clinic input] cmath.asinh = cmath.acos @@ -318,7 +341,15 @@ cmath_atan_impl(PyObject *module, Py_complex z) } -static Py_complex atanh_special_values[7][7]; +static Py_complex atanh_special_values[7][7] = { + { {-0.,-P12}, {-0.,-P12}, {-0.,-P12}, {-0.,P12}, {-0.,P12}, {-0.,P12}, {-0.,N} }, + { {-0.,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {-0.,P12}, {N,N} }, + { {-0.,-P12}, {U,U}, {-0.,-0.}, {-0.,0.}, {U,U}, {-0.,P12}, {-0.,N} }, + { {0.,-P12}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {0.,P12}, {0.,N} }, + { {0.,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {0.,P12}, {N,N} }, + { {0.,-P12}, {0.,-P12}, {0.,-P12}, {0.,P12}, {0.,P12}, {0.,P12}, {0.,N} }, + { {0.,-P12}, {N,N}, {N,N}, {N,N}, {N,N}, {0.,P12}, {N,N} } +}; /*[clinic input] cmath.atanh = cmath.acos @@ -391,7 +422,15 @@ cmath_cos_impl(PyObject *module, Py_complex z) /* cosh(infinity + i*y) needs to be dealt with specially */ -static Py_complex cosh_special_values[7][7]; +static Py_complex cosh_special_values[7][7] = { + { {INF,N}, {U,U}, {INF,0.}, {INF,-0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {N,0.}, {U,U}, {1.,0.}, {1.,-0.}, {U,U}, {N,0.}, {N,0.} }, + { {N,0.}, {U,U}, {1.,-0.}, {1.,0.}, {U,U}, {N,0.}, {N,0.} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {INF,N}, {U,U}, {INF,-0.}, {INF,0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {N,N}, {N,0.}, {N,0.}, {N,N}, {N,N}, {N,N} } +}; /*[clinic input] cmath.cosh = cmath.acos @@ -453,7 +492,15 @@ cmath_cosh_impl(PyObject *module, Py_complex z) /* exp(infinity + i*y) and exp(-infinity + i*y) need special treatment for finite y */ -static Py_complex exp_special_values[7][7]; +static Py_complex exp_special_values[7][7] = { + { {0.,0.}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {0.,0.}, {0.,0.} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {N,N}, {U,U}, {1.,-0.}, {1.,0.}, {U,U}, {N,N}, {N,N} }, + { {N,N}, {U,U}, {1.,-0.}, {1.,0.}, {U,U}, {N,N}, {N,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {INF,N}, {U,U}, {INF,-0.}, {INF,0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {N,N}, {N,-0.}, {N,0.}, {N,N}, {N,N}, {N,N} } +}; /*[clinic input] cmath.exp = cmath.acos @@ -512,7 +559,15 @@ cmath_exp_impl(PyObject *module, Py_complex z) return r; } -static Py_complex log_special_values[7][7]; +static Py_complex log_special_values[7][7] = { + { {INF,-P34}, {INF,-P}, {INF,-P}, {INF,P}, {INF,P}, {INF,P34}, {INF,N} }, + { {INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {-INF,-P}, {-INF,P}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {-INF,-0.}, {-INF,0.}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P12}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,P12}, {N,N} }, + { {INF,-P14}, {INF,-0.}, {INF,-0.}, {INF,0.}, {INF,0.}, {INF,P14}, {INF,N} }, + { {INF,N}, {N,N}, {N,N}, {N,N}, {N,N}, {INF,N}, {N,N} } +}; static Py_complex c_log(Py_complex z) @@ -628,7 +683,15 @@ cmath_sin_impl(PyObject *module, Py_complex z) /* sinh(infinity + i*y) needs to be dealt with specially */ -static Py_complex sinh_special_values[7][7]; +static Py_complex sinh_special_values[7][7] = { + { {INF,N}, {U,U}, {-INF,-0.}, {-INF,0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {0.,N}, {U,U}, {-0.,-0.}, {-0.,0.}, {U,U}, {0.,N}, {0.,N} }, + { {0.,N}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {0.,N}, {0.,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {INF,N}, {U,U}, {INF,-0.}, {INF,0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {N,N}, {N,-0.}, {N,0.}, {N,N}, {N,N}, {N,N} } +}; /*[clinic input] cmath.sinh = cmath.acos @@ -687,7 +750,15 @@ cmath_sinh_impl(PyObject *module, Py_complex z) } -static Py_complex sqrt_special_values[7][7]; +static Py_complex sqrt_special_values[7][7] = { + { {INF,-INF}, {0.,-INF}, {0.,-INF}, {0.,INF}, {0.,INF}, {INF,INF}, {N,INF} }, + { {INF,-INF}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,INF}, {N,N} }, + { {INF,-INF}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {INF,INF}, {N,N} }, + { {INF,-INF}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {INF,INF}, {N,N} }, + { {INF,-INF}, {U,U}, {U,U}, {U,U}, {U,U}, {INF,INF}, {N,N} }, + { {INF,-INF}, {INF,-0.}, {INF,-0.}, {INF,0.}, {INF,0.}, {INF,INF}, {INF,N} }, + { {INF,-INF}, {N,N}, {N,N}, {N,N}, {N,N}, {INF,INF}, {N,N} } +}; /*[clinic input] cmath.sqrt = cmath.acos @@ -786,7 +857,15 @@ cmath_tan_impl(PyObject *module, Py_complex z) /* tanh(infinity + i*y) needs to be dealt with specially */ -static Py_complex tanh_special_values[7][7]; +static Py_complex tanh_special_values[7][7] = { + { {-1.,0.}, {U,U}, {-1.,-0.}, {-1.,0.}, {U,U}, {-1.,0.}, {-1.,0.} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {-0.0,N}, {U,U}, {-0.,-0.}, {-0.,0.}, {U,U}, {-0.0,N}, {-0.,N} }, + { {0.0,N}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {0.0,N}, {0.,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {1.,0.}, {U,U}, {1.,-0.}, {1.,0.}, {U,U}, {1.,0.}, {1.,0.} }, + { {N,N}, {N,N}, {N,-0.}, {N,0.}, {N,N}, {N,N}, {N,N} } +}; /*[clinic input] cmath.tanh = cmath.acos @@ -869,12 +948,13 @@ cmath.log log(z[, base]) -> the logarithm of z to the given base. -If the base is not specified, returns the natural logarithm (base e) of z. +If the base is not specified, returns the natural logarithm (base e) +of z. [clinic start generated code]*/ static PyObject * cmath_log_impl(PyObject *module, Py_complex x, PyObject *y_obj) -/*[clinic end generated code: output=4effdb7d258e0d94 input=e1f81d4fcfd26497]*/ +/*[clinic end generated code: output=4effdb7d258e0d94 input=eb25de0757baf4a0]*/ { Py_complex y; @@ -969,7 +1049,15 @@ cmath_polar_impl(PyObject *module, Py_complex z) */ -static Py_complex rect_special_values[7][7]; +static Py_complex rect_special_values[7][7] = { + { {INF,N}, {U,U}, {-INF,0.}, {-INF,-0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {0.,0.}, {U,U}, {-0.,0.}, {-0.,-0.}, {U,U}, {0.,0.}, {0.,0.} }, + { {0.,0.}, {U,U}, {0.,-0.}, {0.,0.}, {U,U}, {0.,0.}, {0.,0.} }, + { {N,N}, {U,U}, {U,U}, {U,U}, {U,U}, {N,N}, {N,N} }, + { {INF,N}, {U,U}, {INF,-0.}, {INF,0.}, {U,U}, {INF,N}, {INF,N} }, + { {N,N}, {N,N}, {N,0.}, {N,0.}, {N,N}, {N,N}, {N,N} } +}; /*[clinic input] cmath.rect @@ -1090,17 +1178,18 @@ Determine whether two complex numbers are close in value. Return True if a is close in value to b, and False otherwise. -For the values to be considered close, the difference between them must be -smaller than at least one of the tolerances. +For the values to be considered close, the difference between them must +be smaller than at least one of the tolerances. --inf, inf and NaN behave similarly to the IEEE 754 Standard. That is, NaN is -not close to anything, even itself. inf and -inf are only close to themselves. +-inf, inf and NaN behave similarly to the IEEE 754 Standard. That is, +NaN is not close to anything, even itself. inf and -inf are only close +to themselves. [clinic start generated code]*/ static int cmath_isclose_impl(PyObject *module, Py_complex a, Py_complex b, double rel_tol, double abs_tol) -/*[clinic end generated code: output=8a2486cc6e0014d1 input=df9636d7de1d4ac3]*/ +/*[clinic end generated code: output=8a2486cc6e0014d1 input=301b56c90d9a79de]*/ { double diff; @@ -1200,120 +1289,6 @@ cmath_exec(PyObject *mod) return -1; } - /* initialize special value tables */ - -#define INIT_SPECIAL_VALUES(NAME, BODY) { Py_complex* p = (Py_complex*)NAME; BODY } -#define C(REAL, IMAG) p->real = REAL; p->imag = IMAG; ++p; - - INIT_SPECIAL_VALUES(acos_special_values, { - C(P34,INF) C(P,INF) C(P,INF) C(P,-INF) C(P,-INF) C(P34,-INF) C(N,INF) - C(P12,INF) C(U,U) C(U,U) C(U,U) C(U,U) C(P12,-INF) C(N,N) - C(P12,INF) C(U,U) C(P12,0.) C(P12,-0.) C(U,U) C(P12,-INF) C(P12,N) - C(P12,INF) C(U,U) C(P12,0.) C(P12,-0.) C(U,U) C(P12,-INF) C(P12,N) - C(P12,INF) C(U,U) C(U,U) C(U,U) C(U,U) C(P12,-INF) C(N,N) - C(P14,INF) C(0.,INF) C(0.,INF) C(0.,-INF) C(0.,-INF) C(P14,-INF) C(N,INF) - C(N,INF) C(N,N) C(N,N) C(N,N) C(N,N) C(N,-INF) C(N,N) - }) - - INIT_SPECIAL_VALUES(acosh_special_values, { - C(INF,-P34) C(INF,-P) C(INF,-P) C(INF,P) C(INF,P) C(INF,P34) C(INF,N) - C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(0.,-P12) C(0.,P12) C(U,U) C(INF,P12) C(N,P12) - C(INF,-P12) C(U,U) C(0.,-P12) C(0.,P12) C(U,U) C(INF,P12) C(N,P12) - C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P14) C(INF,-0.) C(INF,-0.) C(INF,0.) C(INF,0.) C(INF,P14) C(INF,N) - C(INF,N) C(N,N) C(N,N) C(N,N) C(N,N) C(INF,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(asinh_special_values, { - C(-INF,-P14) C(-INF,-0.) C(-INF,-0.) C(-INF,0.) C(-INF,0.) C(-INF,P14) C(-INF,N) - C(-INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(-INF,P12) C(N,N) - C(-INF,-P12) C(U,U) C(-0.,-0.) C(-0.,0.) C(U,U) C(-INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P14) C(INF,-0.) C(INF,-0.) C(INF,0.) C(INF,0.) C(INF,P14) C(INF,N) - C(INF,N) C(N,N) C(N,-0.) C(N,0.) C(N,N) C(INF,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(atanh_special_values, { - C(-0.,-P12) C(-0.,-P12) C(-0.,-P12) C(-0.,P12) C(-0.,P12) C(-0.,P12) C(-0.,N) - C(-0.,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(-0.,P12) C(N,N) - C(-0.,-P12) C(U,U) C(-0.,-0.) C(-0.,0.) C(U,U) C(-0.,P12) C(-0.,N) - C(0.,-P12) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.,P12) C(0.,N) - C(0.,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(0.,P12) C(N,N) - C(0.,-P12) C(0.,-P12) C(0.,-P12) C(0.,P12) C(0.,P12) C(0.,P12) C(0.,N) - C(0.,-P12) C(N,N) C(N,N) C(N,N) C(N,N) C(0.,P12) C(N,N) - }) - - INIT_SPECIAL_VALUES(cosh_special_values, { - C(INF,N) C(U,U) C(INF,0.) C(INF,-0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(N,0.) C(U,U) C(1.,0.) C(1.,-0.) C(U,U) C(N,0.) C(N,0.) - C(N,0.) C(U,U) C(1.,-0.) C(1.,0.) C(U,U) C(N,0.) C(N,0.) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(INF,N) C(U,U) C(INF,-0.) C(INF,0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(N,N) C(N,0.) C(N,0.) C(N,N) C(N,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(exp_special_values, { - C(0.,0.) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.,0.) C(0.,0.) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(N,N) C(U,U) C(1.,-0.) C(1.,0.) C(U,U) C(N,N) C(N,N) - C(N,N) C(U,U) C(1.,-0.) C(1.,0.) C(U,U) C(N,N) C(N,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(INF,N) C(U,U) C(INF,-0.) C(INF,0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(N,N) C(N,-0.) C(N,0.) C(N,N) C(N,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(log_special_values, { - C(INF,-P34) C(INF,-P) C(INF,-P) C(INF,P) C(INF,P) C(INF,P34) C(INF,N) - C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(-INF,-P) C(-INF,P) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(-INF,-0.) C(-INF,0.) C(U,U) C(INF,P12) C(N,N) - C(INF,-P12) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,P12) C(N,N) - C(INF,-P14) C(INF,-0.) C(INF,-0.) C(INF,0.) C(INF,0.) C(INF,P14) C(INF,N) - C(INF,N) C(N,N) C(N,N) C(N,N) C(N,N) C(INF,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(sinh_special_values, { - C(INF,N) C(U,U) C(-INF,-0.) C(-INF,0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(0.,N) C(U,U) C(-0.,-0.) C(-0.,0.) C(U,U) C(0.,N) C(0.,N) - C(0.,N) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.,N) C(0.,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(INF,N) C(U,U) C(INF,-0.) C(INF,0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(N,N) C(N,-0.) C(N,0.) C(N,N) C(N,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(sqrt_special_values, { - C(INF,-INF) C(0.,-INF) C(0.,-INF) C(0.,INF) C(0.,INF) C(INF,INF) C(N,INF) - C(INF,-INF) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,INF) C(N,N) - C(INF,-INF) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(INF,INF) C(N,N) - C(INF,-INF) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(INF,INF) C(N,N) - C(INF,-INF) C(U,U) C(U,U) C(U,U) C(U,U) C(INF,INF) C(N,N) - C(INF,-INF) C(INF,-0.) C(INF,-0.) C(INF,0.) C(INF,0.) C(INF,INF) C(INF,N) - C(INF,-INF) C(N,N) C(N,N) C(N,N) C(N,N) C(INF,INF) C(N,N) - }) - - INIT_SPECIAL_VALUES(tanh_special_values, { - C(-1.,0.) C(U,U) C(-1.,-0.) C(-1.,0.) C(U,U) C(-1.,0.) C(-1.,0.) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(-0.0,N) C(U,U) C(-0.,-0.) C(-0.,0.) C(U,U) C(-0.0,N) C(-0.,N) - C(0.0,N) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.0,N) C(0.,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(1.,0.) C(U,U) C(1.,-0.) C(1.,0.) C(U,U) C(1.,0.) C(1.,0.) - C(N,N) C(N,N) C(N,-0.) C(N,0.) C(N,N) C(N,N) C(N,N) - }) - - INIT_SPECIAL_VALUES(rect_special_values, { - C(INF,N) C(U,U) C(-INF,0.) C(-INF,-0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(0.,0.) C(U,U) C(-0.,0.) C(-0.,-0.) C(U,U) C(0.,0.) C(0.,0.) - C(0.,0.) C(U,U) C(0.,-0.) C(0.,0.) C(U,U) C(0.,0.) C(0.,0.) - C(N,N) C(U,U) C(U,U) C(U,U) C(U,U) C(N,N) C(N,N) - C(INF,N) C(U,U) C(INF,-0.) C(INF,0.) C(U,U) C(INF,N) C(INF,N) - C(N,N) C(N,N) C(N,0.) C(N,0.) C(N,N) C(N,N) C(N,N) - }) return 0; } diff --git a/Modules/expat/COPYING b/Modules/expat/COPYING index ce9e5939291e45f..c6d184a8aae845c 100644 --- a/Modules/expat/COPYING +++ b/Modules/expat/COPYING @@ -1,5 +1,5 @@ Copyright (c) 1998-2000 Thai Open Source Software Center Ltd and Clark Cooper -Copyright (c) 2001-2022 Expat maintainers +Copyright (c) 2001-2025 Expat maintainers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/Modules/expat/expat.h b/Modules/expat/expat.h index 610e1ddc0e94ba6..c493c70441c4184 100644 --- a/Modules/expat/expat.h +++ b/Modules/expat/expat.h @@ -11,7 +11,7 @@ Copyright (c) 2000-2005 Fred L. Drake, Jr. <fdrake@users.sourceforge.net> Copyright (c) 2001-2002 Greg Stein <gstein@users.sourceforge.net> Copyright (c) 2002-2016 Karl Waclawek <karl@waclawek.net> - Copyright (c) 2016-2025 Sebastian Pipping <sebastian@pipping.org> + Copyright (c) 2016-2026 Sebastian Pipping <sebastian@pipping.org> Copyright (c) 2016 Cristian Rodríguez <crrodriguez@opensuse.org> Copyright (c) 2016 Thomas Beutlich <tc@tbeu.de> Copyright (c) 2017 Rhodri James <rhodri@wildebeest.org.uk> @@ -19,6 +19,7 @@ Copyright (c) 2023 Hanno Böck <hanno@gentoo.org> Copyright (c) 2023 Sony Corporation / Snild Dolkow <snild@sony.com> Copyright (c) 2024 Taichi Haradaguchi <20001722@ymail.ne.jp> + Copyright (c) 2025 Matthew Fernandez <matthew.fernandez@gmail.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -42,21 +43,22 @@ */ #ifndef Expat_INCLUDED -#define Expat_INCLUDED 1 +# define Expat_INCLUDED 1 -#include <stdlib.h> -#include "expat_external.h" +# include <stdint.h> // for uint8_t +# include <stdlib.h> +# include "expat_external.h" -#ifdef __cplusplus +# ifdef __cplusplus extern "C" { -#endif +# endif struct XML_ParserStruct; typedef struct XML_ParserStruct *XML_Parser; typedef unsigned char XML_Bool; -#define XML_TRUE ((XML_Bool)1) -#define XML_FALSE ((XML_Bool)0) +# define XML_TRUE ((XML_Bool)1) +# define XML_FALSE ((XML_Bool)0) /* The XML_Status enum gives the possible return values for several API functions. The preprocessor #defines are included so this @@ -73,11 +75,11 @@ typedef unsigned char XML_Bool; */ enum XML_Status { XML_STATUS_ERROR = 0, -#define XML_STATUS_ERROR XML_STATUS_ERROR +# define XML_STATUS_ERROR XML_STATUS_ERROR XML_STATUS_OK = 1, -#define XML_STATUS_OK XML_STATUS_OK +# define XML_STATUS_OK XML_STATUS_OK XML_STATUS_SUSPENDED = 2 -#define XML_STATUS_SUSPENDED XML_STATUS_SUSPENDED +# define XML_STATUS_SUSPENDED XML_STATUS_SUSPENDED }; enum XML_Error { @@ -276,7 +278,7 @@ XML_ParserCreate_MM(const XML_Char *encoding, /* Prepare a parser object to be reused. This is particularly valuable when memory allocation overhead is disproportionately high, - such as when a large number of small documnents need to be parsed. + such as when a large number of small documents need to be parsed. All handlers are cleared from the parser, except for the unknownEncodingHandler. The parser's external state is re-initialized except for the values of ns and ns_triplets. @@ -680,7 +682,7 @@ XMLPARSEAPI(void) XML_SetUserData(XML_Parser parser, void *userData); /* Returns the last value set by XML_SetUserData or NULL. */ -#define XML_GetUserData(parser) (*(void **)(parser)) +# define XML_GetUserData(parser) (*(void **)(parser)) /* This is equivalent to supplying an encoding argument to XML_ParserCreate. On success XML_SetEncoding returns non-zero, @@ -752,7 +754,7 @@ XML_GetSpecifiedAttributeCount(XML_Parser parser); XMLPARSEAPI(int) XML_GetIdAttributeIndex(XML_Parser parser); -#ifdef XML_ATTR_INFO +# ifdef XML_ATTR_INFO /* Source file byte offsets for the start and end of attribute names and values. The value indices are exclusive of surrounding quotes; thus in a UTF-8 source file an attribute value of "blah" will yield: @@ -773,7 +775,7 @@ typedef struct { */ XMLPARSEAPI(const XML_AttrInfo *) XML_GetAttributeInfo(XML_Parser parser); -#endif +# endif /* Parses some input. Returns XML_STATUS_ERROR if a fatal error is detected. The last call to XML_Parse must have isFinal true; len @@ -916,10 +918,21 @@ XML_SetParamEntityParsing(XML_Parser parser, function behavior. This must be called before parsing is started. Returns 1 if successful, 0 when called after parsing has started. Note: If parser == NULL, the function will do nothing and return 0. + DEPRECATED since Expat 2.8.0. */ XMLPARSEAPI(int) XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt); +/* Sets the hash salt to use for internal hash calculations. + Helps in preventing DoS attacks based on predicting hash function behavior. + This must be called before parsing is started. + Returns XML_TRUE if successful, XML_FALSE when called after parsing has + started or when parser is NULL. + Added in Expat 2.8.0. +*/ +XMLPARSEAPI(XML_Bool) +XML_SetHashSalt16Bytes(XML_Parser parser, const uint8_t entropy[16]); + /* If XML_Parse or XML_ParseBuffer have returned XML_STATUS_ERROR, then XML_GetErrorCode returns information about the error. */ @@ -970,9 +983,9 @@ XMLPARSEAPI(const char *) XML_GetInputContext(XML_Parser parser, int *offset, int *size); /* For backwards compatibility with previous versions. */ -#define XML_GetErrorLineNumber XML_GetCurrentLineNumber -#define XML_GetErrorColumnNumber XML_GetCurrentColumnNumber -#define XML_GetErrorByteIndex XML_GetCurrentByteIndex +# define XML_GetErrorLineNumber XML_GetCurrentLineNumber +# define XML_GetErrorColumnNumber XML_GetCurrentColumnNumber +# define XML_GetErrorByteIndex XML_GetCurrentByteIndex /* Frees the content model passed to the element declaration handler */ XMLPARSEAPI(void) @@ -1032,7 +1045,10 @@ enum XML_FeatureEnum { XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT, XML_FEATURE_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT, /* Added in Expat 2.6.0. */ - XML_FEATURE_GE + XML_FEATURE_GE, + /* Added in Expat 2.7.2. */ + XML_FEATURE_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT, + XML_FEATURE_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT, /* Additional features must be added to the end of this enum. */ }; @@ -1045,7 +1061,7 @@ typedef struct { XMLPARSEAPI(const XML_Feature *) XML_GetFeatureList(void); -#if defined(XML_DTD) || (defined(XML_GE) && XML_GE == 1) +# if defined(XML_DTD) || (defined(XML_GE) && XML_GE == 1) /* Added in Expat 2.4.0 for XML_DTD defined and * added in Expat 2.6.0 for XML_GE == 1. */ XMLPARSEAPI(XML_Bool) @@ -1057,7 +1073,17 @@ XML_SetBillionLaughsAttackProtectionMaximumAmplification( XMLPARSEAPI(XML_Bool) XML_SetBillionLaughsAttackProtectionActivationThreshold( XML_Parser parser, unsigned long long activationThresholdBytes); -#endif + +/* Added in Expat 2.7.2. */ +XMLPARSEAPI(XML_Bool) +XML_SetAllocTrackerMaximumAmplification(XML_Parser parser, + float maximumAmplificationFactor); + +/* Added in Expat 2.7.2. */ +XMLPARSEAPI(XML_Bool) +XML_SetAllocTrackerActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes); +# endif /* Added in Expat 2.6.0. */ XMLPARSEAPI(XML_Bool) @@ -1066,12 +1092,12 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); /* Expat follows the semantic versioning convention. See https://semver.org */ -#define XML_MAJOR_VERSION 2 -#define XML_MINOR_VERSION 7 -#define XML_MICRO_VERSION 1 +# define XML_MAJOR_VERSION 2 +# define XML_MINOR_VERSION 8 +# define XML_MICRO_VERSION 2 -#ifdef __cplusplus +# ifdef __cplusplus } -#endif +# endif #endif /* not Expat_INCLUDED */ diff --git a/Modules/expat/expat_config.h b/Modules/expat/expat_config.h index e7d9499d9078d95..70df73c8e00a5f7 100644 --- a/Modules/expat/expat_config.h +++ b/Modules/expat/expat_config.h @@ -3,7 +3,7 @@ * distribution. */ #ifndef EXPAT_CONFIG_H -#define EXPAT_CONFIG_H +#define EXPAT_CONFIG_H 1 #include <pyconfig.h> #ifdef WORDS_BIGENDIAN @@ -22,5 +22,10 @@ // bpo-30947: Python uses best available entropy sources to // call XML_SetHashSalt(), expat entropy sources are not needed #define XML_POOR_ENTROPY 1 +#undef HAVE_ARC4RANDOM +#undef HAVE_ARC4RANDOM_BUF +#undef HAVE_GETENTROPY +#undef HAVE_GETRANDOM +#undef HAVE_SYSCALL_GETRANDOM #endif /* EXPAT_CONFIG_H */ diff --git a/Modules/expat/expat_external.h b/Modules/expat/expat_external.h index 567872b09836e1c..cc945c424e471f7 100644 --- a/Modules/expat/expat_external.h +++ b/Modules/expat/expat_external.h @@ -12,9 +12,10 @@ Copyright (c) 2001-2002 Greg Stein <gstein@users.sourceforge.net> Copyright (c) 2002-2006 Karl Waclawek <karl@waclawek.net> Copyright (c) 2016 Cristian Rodríguez <crrodriguez@opensuse.org> - Copyright (c) 2016-2019 Sebastian Pipping <sebastian@pipping.org> + Copyright (c) 2016-2025 Sebastian Pipping <sebastian@pipping.org> Copyright (c) 2017 Rhodri James <rhodri@wildebeest.org.uk> Copyright (c) 2018 Yury Gribov <tetra2005@gmail.com> + Copyright (c) 2026 Matthew Fernandez <matthew.fernandez@gmail.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -38,8 +39,7 @@ */ #ifndef Expat_External_INCLUDED -#define Expat_External_INCLUDED 1 - +# define Expat_External_INCLUDED 1 /* Namespace external symbols to allow multiple libexpat version to co-exist. */ #include "pyexpatns.h" @@ -49,7 +49,7 @@ /* Expat tries very hard to make the API boundary very specifically defined. There are two macros defined to control this boundary; each of these can be defined before including this header to - achieve some different behavior, but doing so it not recommended or + achieve some different behavior, but doing so is not recommended or tested frequently. XMLCALL - The calling convention to use for all calls across the @@ -68,12 +68,12 @@ compiled with the cdecl calling convention as the default since system headers may assume the cdecl convention. */ -#ifndef XMLCALL -# if defined(_MSC_VER) -# define XMLCALL __cdecl -# elif defined(__GNUC__) && defined(__i386) && ! defined(__INTEL_COMPILER) -# define XMLCALL __attribute__((cdecl)) -# else +# ifndef XMLCALL +# if defined(_MSC_VER) +# define XMLCALL __cdecl +# elif defined(__GNUC__) && defined(__i386) && ! defined(__INTEL_COMPILER) +# define XMLCALL __attribute__((cdecl)) +# else /* For any platform which uses this definition and supports more than one calling convention, we need to extend this definition to declare the convention used on that platform, if it's possible to @@ -84,86 +84,86 @@ pre-processor and how to specify the same calling convention as the platform's malloc() implementation. */ -# define XMLCALL -# endif -#endif /* not defined XMLCALL */ +# define XMLCALL +# endif +# endif /* not defined XMLCALL */ -#if ! defined(XML_STATIC) && ! defined(XMLIMPORT) -# ifndef XML_BUILDING_EXPAT +# if ! defined(XML_STATIC) && ! defined(XMLIMPORT) +# ifndef XML_BUILDING_EXPAT /* using Expat from an application */ -# if defined(_MSC_EXTENSIONS) && ! defined(__BEOS__) && ! defined(__CYGWIN__) -# define XMLIMPORT __declspec(dllimport) +# if defined(_MSC_VER) && ! defined(__BEOS__) && ! defined(__CYGWIN__) +# define XMLIMPORT __declspec(dllimport) +# endif + # endif +# endif /* not defined XML_STATIC */ +# ifndef XML_ENABLE_VISIBILITY +# define XML_ENABLE_VISIBILITY 0 # endif -#endif /* not defined XML_STATIC */ - -#ifndef XML_ENABLE_VISIBILITY -# define XML_ENABLE_VISIBILITY 0 -#endif -#if ! defined(XMLIMPORT) && XML_ENABLE_VISIBILITY -# define XMLIMPORT __attribute__((visibility("default"))) -#endif +# if ! defined(XMLIMPORT) && XML_ENABLE_VISIBILITY +# define XMLIMPORT __attribute__((visibility("default"))) +# endif /* If we didn't define it above, define it away: */ -#ifndef XMLIMPORT -# define XMLIMPORT -#endif - -#if defined(__GNUC__) \ - && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96)) -# define XML_ATTR_MALLOC __attribute__((__malloc__)) -#else -# define XML_ATTR_MALLOC -#endif - -#if defined(__GNUC__) \ - && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) -# define XML_ATTR_ALLOC_SIZE(x) __attribute__((__alloc_size__(x))) -#else -# define XML_ATTR_ALLOC_SIZE(x) -#endif - -#define XMLPARSEAPI(type) XMLIMPORT type XMLCALL - -#ifdef __cplusplus -extern "C" { -#endif +# ifndef XMLIMPORT +# define XMLIMPORT +# endif -#ifdef XML_UNICODE_WCHAR_T -# ifndef XML_UNICODE -# define XML_UNICODE +# if defined(__GNUC__) \ + && (__GNUC__ > 2 || (__GNUC__ == 2 && __GNUC_MINOR__ >= 96)) +# define XML_ATTR_MALLOC __attribute__((__malloc__)) +# else +# define XML_ATTR_MALLOC # endif -# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2) -# error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc" + +# if defined(__GNUC__) \ + && ((__GNUC__ > 4) || (__GNUC__ == 4 && __GNUC_MINOR__ >= 3)) +# define XML_ATTR_ALLOC_SIZE(x) __attribute__((__alloc_size__(x))) +# else +# define XML_ATTR_ALLOC_SIZE(x) +# endif + +# define XMLPARSEAPI(type) XMLIMPORT type XMLCALL + +# ifdef __cplusplus +extern "C" { # endif -#endif -#ifdef XML_UNICODE /* Information is UTF-16 encoded. */ # ifdef XML_UNICODE_WCHAR_T +# ifndef XML_UNICODE +# define XML_UNICODE +# endif +# if defined(__SIZEOF_WCHAR_T__) && (__SIZEOF_WCHAR_T__ != 2) +# error "sizeof(wchar_t) != 2; Need -fshort-wchar for both Expat and libc" +# endif +# endif + +# ifdef XML_UNICODE /* Information is UTF-16 encoded. */ +# ifdef XML_UNICODE_WCHAR_T typedef wchar_t XML_Char; typedef wchar_t XML_LChar; -# else +# else typedef unsigned short XML_Char; typedef char XML_LChar; -# endif /* XML_UNICODE_WCHAR_T */ -#else /* Information is UTF-8 encoded. */ +# endif /* XML_UNICODE_WCHAR_T */ +# else /* Information is UTF-8 encoded. */ typedef char XML_Char; typedef char XML_LChar; -#endif /* XML_UNICODE */ +# endif /* XML_UNICODE */ -#ifdef XML_LARGE_SIZE /* Use large integers for file/stream positions. */ +# ifdef XML_LARGE_SIZE /* Use large integers for file/stream positions. */ typedef long long XML_Index; typedef unsigned long long XML_Size; -#else +# else typedef long XML_Index; typedef unsigned long XML_Size; -#endif /* XML_LARGE_SIZE */ +# endif /* XML_LARGE_SIZE */ -#ifdef __cplusplus +# ifdef __cplusplus } -#endif +# endif #endif /* not Expat_External_INCLUDED */ diff --git a/Modules/expat/fallthrough.h b/Modules/expat/fallthrough.h new file mode 100644 index 000000000000000..707dbdd44bfe53d --- /dev/null +++ b/Modules/expat/fallthrough.h @@ -0,0 +1,49 @@ +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2026 Nick Begg <nick@stunttruck.net> + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +#ifndef FALLTHROUGH_H +# define FALLTHROUGH_H 1 + +// Explicit fallthrough in switch case to avoid warnings +// with compiler flag -Wimplicit-fallthrough. + +# define EXPAT_FALLTHROUGH \ + do { \ + } while (0) + +# if defined(__has_attribute) +# if __has_attribute(fallthrough) +# undef EXPAT_FALLTHROUGH +# define EXPAT_FALLTHROUGH __attribute__((fallthrough)) +# endif +# endif + +#endif // FALLTHROUGH_H diff --git a/Modules/expat/internal.h b/Modules/expat/internal.h index 6bde6ae6b31ddd8..420d4217a569b1d 100644 --- a/Modules/expat/internal.h +++ b/Modules/expat/internal.h @@ -28,7 +28,7 @@ Copyright (c) 2002-2003 Fred L. Drake, Jr. <fdrake@users.sourceforge.net> Copyright (c) 2002-2006 Karl Waclawek <karl@waclawek.net> Copyright (c) 2003 Greg Stein <gstein@users.sourceforge.net> - Copyright (c) 2016-2025 Sebastian Pipping <sebastian@pipping.org> + Copyright (c) 2016-2026 Sebastian Pipping <sebastian@pipping.org> Copyright (c) 2018 Yury Gribov <tetra2005@gmail.com> Copyright (c) 2019 David Loffredo <loffredo@steptools.com> Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow <snild@sony.com> @@ -108,10 +108,12 @@ #endif #include <limits.h> // ULONG_MAX +#include <stddef.h> // size_t #if defined(_WIN32) \ && (! defined(__USE_MINGW_ANSI_STDIO) \ || (1 - __USE_MINGW_ANSI_STDIO - 1 == 0)) +# define EXPAT_FMT_LLX(midpart) "%" midpart "I64x" # define EXPAT_FMT_ULL(midpart) "%" midpart "I64u" # if defined(_WIN64) // Note: modifiers "td" and "zu" do not work for MinGW # define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "I64d" @@ -121,13 +123,14 @@ # define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" # endif #else +# define EXPAT_FMT_LLX(midpart) "%" midpart "llx" # define EXPAT_FMT_ULL(midpart) "%" midpart "llu" # if ! defined(ULONG_MAX) # error Compiler did not define ULONG_MAX for us # elif ULONG_MAX == 18446744073709551615u // 2^64-1 # define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "ld" # define EXPAT_FMT_SIZE_T(midpart) "%" midpart "lu" -# elif defined(EMSCRIPTEN) // 32bit mode Emscripten +# elif defined(__wasm32__) // 32bit mode Emscripten or WASI SDK # define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "ld" # define EXPAT_FMT_SIZE_T(midpart) "%" midpart "zu" # else @@ -148,6 +151,16 @@ 100.0f #define EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT \ 8388608 // 8 MiB, 2^23 + +#define EXPAT_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT 100.0f +#define EXPAT_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT \ + 67108864 // 64 MiB, 2^26 + +// NOTE: If function expat_alloc was user facing, EXPAT_MALLOC_ALIGNMENT would +// have to take sizeof(long double) into account +#define EXPAT_MALLOC_ALIGNMENT sizeof(long long) // largest parser (sub)member +#define EXPAT_MALLOC_PADDING ((EXPAT_MALLOC_ALIGNMENT) - sizeof(size_t)) + /* NOTE END */ #include "expat.h" // so we can use type XML_Parser below @@ -171,6 +184,9 @@ extern #endif XML_Bool g_reparseDeferralEnabledDefault; // written ONLY in runtests.c #if defined(XML_TESTING) +void *expat_malloc(XML_Parser parser, size_t size, int sourceLine); +void expat_free(XML_Parser parser, void *ptr, int sourceLine); +void *expat_realloc(XML_Parser parser, void *ptr, size_t size, int sourceLine); extern unsigned int g_bytesScanned; // used for testing only #endif diff --git a/Modules/expat/memory_sanitizer.h b/Modules/expat/memory_sanitizer.h new file mode 100644 index 000000000000000..a8a8006ccded12d --- /dev/null +++ b/Modules/expat/memory_sanitizer.h @@ -0,0 +1,51 @@ +/* + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2026 Matthew Fernandez <matthew.fernandez@gmail.com> + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +#if ! defined(MEMORY_SANITIZER_H) +# define MEMORY_SANITIZER_H 1 + +# if defined(__has_feature) +# if __has_feature(memory_sanitizer) +# include <sanitizer/msan_interface.h> + +// inform Memory Sanitizer that [base, base + extent) is now initialized +# define MSAN_UNPOISON(base, extent) __msan_unpoison((base), (extent)) + +# endif +# endif + +# if ! defined(MSAN_UNPOISON) +# define MSAN_UNPOISON(base, extent) \ + do { \ + } while (0) +# endif + +#endif // ! defined(MEMORY_SANITIZER_H) diff --git a/Modules/expat/pyexpatns.h b/Modules/expat/pyexpatns.h index 8ee03ef07928157..fc6b482d587e0d6 100644 --- a/Modules/expat/pyexpatns.h +++ b/Modules/expat/pyexpatns.h @@ -82,6 +82,8 @@ #define XmlPrologStateInit PyExpat_XmlPrologStateInit #define XmlPrologStateInitExternalEntity PyExpat_XmlPrologStateInitExternalEntity #define XML_ResumeParser PyExpat_XML_ResumeParser +#define XML_SetAllocTrackerActivationThreshold PyExpat_XML_SetAllocTrackerActivationThreshold +#define XML_SetAllocTrackerMaximumAmplification PyExpat_XML_SetAllocTrackerMaximumAmplification #define XML_SetAttlistDeclHandler PyExpat_XML_SetAttlistDeclHandler #define XML_SetBase PyExpat_XML_SetBase #define XML_SetBillionLaughsAttackProtectionActivationThreshold PyExpat_XML_SetBillionLaughsAttackProtectionActivationThreshold diff --git a/Modules/expat/refresh.sh b/Modules/expat/refresh.sh index 3904fc8afd63d28..327baafe1d5e4d4 100755 --- a/Modules/expat/refresh.sh +++ b/Modules/expat/refresh.sh @@ -12,9 +12,9 @@ fi # Update this when updating to a new version after verifying that the changes # the update brings in are good. These values are used for verifying the SBOM, too. -expected_libexpat_tag="R_2_7_1" -expected_libexpat_version="2.7.1" -expected_libexpat_sha256="0cce2e6e69b327fc607b8ff264f4b66bdf71ead55a87ffd5f3143f535f15cfa2" +expected_libexpat_tag="R_2_8_2" +expected_libexpat_version="2.8.2" +expected_libexpat_sha256="ef7d1994f533c9e7343d6c19f31064fc8ebbcbcaa144be3812b4f43052a05f4c" expat_dir="$(realpath "$(dirname -- "${BASH_SOURCE[0]}")")" cd ${expat_dir} @@ -24,19 +24,25 @@ curl --location "https://github.com/libexpat/libexpat/releases/download/${expect echo "${expected_libexpat_sha256} libexpat.tar.gz" | sha256sum --check # Step 2: Pull files from the libexpat distribution + +tar xzvf libexpat.tar.gz "expat-${expected_libexpat_version}/COPYING" --strip-components 2 + declare -a lib_files lib_files=( ascii.h asciitab.h expat.h expat_external.h + fallthrough.h iasciitab.h internal.h latin1tab.h + memory_sanitizer.h nametab.h siphash.h utf8tab.h winconfig.h + xcsinc.c xmlparse.c xmlrole.c xmlrole.h @@ -52,7 +58,19 @@ done rm libexpat.tar.gz # Step 3: Add the namespacing include to expat_external.h -sed -i 's/#define Expat_External_INCLUDED 1/&\n\n\/* Namespace external symbols to allow multiple libexpat version to\n co-exist. \*\/\n#include "pyexpatns.h"/' expat_external.h +sed -i 's/# define Expat_External_INCLUDED 1/&\n\/* Namespace external symbols to allow multiple libexpat version to\n co-exist. \*\/\n#include "pyexpatns.h"/' expat_external.h + +# Step 4: Skip the Windows rand_s entropy path in xmlparse.c when +# XML_POOR_ENTROPY is set. +sed -z -i 's|#if defined(_WIN32)\n# include "random_rand_s\.h"\n#endif /\* defined(_WIN32) \*/|#if defined(_WIN32) \&\& ! defined(XML_POOR_ENTROPY)\n# include "random_rand_s.h"\n#endif /* defined(_WIN32) \&\& ! defined(XML_POOR_ENTROPY) */|' xmlparse.c +sed -z -i 's|# ifdef _WIN32\n if (writeRandomBytes_rand_s|# if defined(_WIN32) \&\& ! defined(XML_POOR_ENTROPY)\n if (writeRandomBytes_rand_s|' xmlparse.c + +if ! grep -q '#if defined(_WIN32) && ! defined(XML_POOR_ENTROPY)' xmlparse.c; then + echo " +Error: rand_s gate not patched in xmlparse.c; +This may be due to source changes and will require updating this script" >&2 + exit 1 +fi echo " Updated! next steps: diff --git a/Modules/expat/siphash.h b/Modules/expat/siphash.h index 04f6f74585b5a2f..be216b4006dc847 100644 --- a/Modules/expat/siphash.h +++ b/Modules/expat/siphash.h @@ -100,6 +100,7 @@ #include <stddef.h> /* size_t */ #include <stdint.h> /* uint64_t uint32_t uint8_t */ +#include "fallthrough.h" /* * Workaround to not require a C++11 compiler for using ULL suffix @@ -234,25 +235,25 @@ sip24_final(struct siphash *H) { switch (left) { case 7: b |= (uint64_t)H->buf[6] << 48; - /* fall through */ + EXPAT_FALLTHROUGH; case 6: b |= (uint64_t)H->buf[5] << 40; - /* fall through */ + EXPAT_FALLTHROUGH; case 5: b |= (uint64_t)H->buf[4] << 32; - /* fall through */ + EXPAT_FALLTHROUGH; case 4: b |= (uint64_t)H->buf[3] << 24; - /* fall through */ + EXPAT_FALLTHROUGH; case 3: b |= (uint64_t)H->buf[2] << 16; - /* fall through */ + EXPAT_FALLTHROUGH; case 2: b |= (uint64_t)H->buf[1] << 8; - /* fall through */ + EXPAT_FALLTHROUGH; case 1: b |= (uint64_t)H->buf[0] << 0; - /* fall through */ + EXPAT_FALLTHROUGH; case 0: break; } diff --git a/Modules/expat/xcsinc.c b/Modules/expat/xcsinc.c new file mode 100644 index 000000000000000..3597c2480bc9617 --- /dev/null +++ b/Modules/expat/xcsinc.c @@ -0,0 +1,48 @@ +/* This file is included from other .c files! + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2022 Sebastian Pipping <sebastian@pipping.org> + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + +static size_t +xcslen(const XML_Char *s) { +#ifdef XML_UNICODE +# ifdef XML_UNICODE_WCHAR_T + return wcslen(s); +# else + // XML_Char is unsigned short + size_t len = 0; + while (s[len]) { + len++; + } + return len; +# endif +#else + return strlen(s); +#endif +} diff --git a/Modules/expat/xmlparse.c b/Modules/expat/xmlparse.c index 38a2d9657b6aeb0..d961ebf227e7405 100644 --- a/Modules/expat/xmlparse.c +++ b/Modules/expat/xmlparse.c @@ -1,4 +1,4 @@ -/* d19ae032c224863c1527ba44d228cc34b99192c3a4c5a27af1f4e054d45ee031 (2.7.1+) +/* 5de44e6750c6cc78818f06ed552f522a1241df0299395250e1792cb339389daf (2.8.2+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -13,7 +13,7 @@ Copyright (c) 2002-2016 Karl Waclawek <karl@waclawek.net> Copyright (c) 2005-2009 Steven Solie <steven@solie.ca> Copyright (c) 2016 Eric Rahm <erahm@mozilla.com> - Copyright (c) 2016-2025 Sebastian Pipping <sebastian@pipping.org> + Copyright (c) 2016-2026 Sebastian Pipping <sebastian@pipping.org> Copyright (c) 2016 Gaurav <g.gupta@samsung.com> Copyright (c) 2016 Thomas Beutlich <tc@tbeu.de> Copyright (c) 2016 Gustavo Grieco <gustavo.grieco@imag.fr> @@ -41,6 +41,15 @@ Copyright (c) 2023-2024 Sony Corporation / Snild Dolkow <snild@sony.com> Copyright (c) 2024-2025 Berkay Eren Ürün <berkay.ueruen@siemens.com> Copyright (c) 2024 Hanno Böck <hanno@gentoo.org> + Copyright (c) 2025-2026 Matthew Fernandez <matthew.fernandez@gmail.com> + Copyright (c) 2025 Atrem Borovik <polzovatellllk@gmail.com> + Copyright (c) 2025 Alfonso Gregory <gfunni234@gmail.com> + Copyright (c) 2026 Rosen Penev <rosenp@gmail.com> + Copyright (c) 2026 Francesco Bertolaccini + Copyright (c) 2026 Christian Ng <christianrng@berkeley.edu> + Copyright (c) 2026 Nick Begg <nick@stunttruck.net> + Copyright (c) 2026 Kartik Kenchi <netliomax25@gmail.com> + Copyright (c) 2026 Haris Hussain <hextheshadow0x@gmail.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -80,28 +89,16 @@ # error XML_CONTEXT_BYTES must be defined, non-empty and >=0 (0 to disable, >=1 to enable; 1024 is a common default) #endif -#if defined(HAVE_SYSCALL_GETRANDOM) -# if ! defined(_GNU_SOURCE) -# define _GNU_SOURCE 1 /* syscall prototype */ -# endif -#endif - -#ifdef _WIN32 -/* force stdlib to define rand_s() */ -# if ! defined(_CRT_RAND_S) -# define _CRT_RAND_S -# endif -#endif - #include <stdbool.h> #include <stddef.h> #include <string.h> /* memset(), memcpy() */ #include <assert.h> -#include <limits.h> /* UINT_MAX */ +#include <limits.h> /* INT_MAX, LLONG_MAX, LONG_MAX, UINT_MAX */ #include <stdio.h> /* fprintf */ -#include <stdlib.h> /* getenv, rand_s */ -#include <stdint.h> /* uintptr_t */ +#include <stdlib.h> /* getenv */ +#include <stdint.h> /* SIZE_MAX, uintptr_t */ #include <math.h> /* isnan */ +#include <errno.h> #ifdef _WIN32 # define getpid GetCurrentProcessId @@ -120,32 +117,36 @@ #include "ascii.h" #include "expat.h" #include "siphash.h" +#include "xcsinc.c" + +#if defined(HAVE_ARC4RANDOM) +# include "random_arc4random.h" +#endif /* defined(HAVE_ARC4RANDOM) */ + +#if defined(HAVE_ARC4RANDOM_BUF) +# include "random_arc4random_buf.h" +#endif // defined(HAVE_ARC4RANDOM_BUF) + +#if defined(XML_DEV_URANDOM) +# include "random_dev_urandom.h" +#endif /* defined(XML_DEV_URANDOM) */ + +#if defined(HAVE_GETENTROPY) +# include "random_getentropy.h" +#endif // defined(HAVE_GETENTROPY) #if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) -# if defined(HAVE_GETRANDOM) -# include <sys/random.h> /* getrandom */ -# else -# include <unistd.h> /* syscall */ -# include <sys/syscall.h> /* SYS_getrandom */ -# endif -# if ! defined(GRND_NONBLOCK) -# define GRND_NONBLOCK 0x0001 -# endif /* defined(GRND_NONBLOCK) */ -#endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ - -#if defined(HAVE_LIBBSD) \ - && (defined(HAVE_ARC4RANDOM_BUF) || defined(HAVE_ARC4RANDOM)) -# include <bsd/stdlib.h> -#endif +# include "random_getrandom.h" +#endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ -#if defined(_WIN32) && ! defined(LOAD_LIBRARY_SEARCH_SYSTEM32) -# define LOAD_LIBRARY_SEARCH_SYSTEM32 0x00000800 -#endif +#if defined(_WIN32) && ! defined(XML_POOR_ENTROPY) +# include "random_rand_s.h" +#endif /* defined(_WIN32) && ! defined(XML_POOR_ENTROPY) */ #if ! defined(HAVE_GETRANDOM) && ! defined(HAVE_SYSCALL_GETRANDOM) \ && ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) \ - && ! defined(XML_DEV_URANDOM) && ! defined(_WIN32) \ - && ! defined(XML_POOR_ENTROPY) + && ! defined(HAVE_GETENTROPY) && ! defined(XML_DEV_URANDOM) \ + && ! defined(_WIN32) && ! defined(XML_POOR_ENTROPY) # error You do not have support for any sources of high quality entropy \ enabled. For end user security, that is probably not what you want. \ \ @@ -154,13 +155,12 @@ * Linux >=3.17 + glibc (including <2.25) (syscall SYS_getrandom): HAVE_SYSCALL_GETRANDOM, \ * BSD / macOS >=10.7 / glibc >=2.36 (arc4random_buf): HAVE_ARC4RANDOM_BUF, \ * BSD / macOS (including <10.7) / glibc >=2.36 (arc4random): HAVE_ARC4RANDOM, \ - * libbsd (arc4random_buf): HAVE_ARC4RANDOM_BUF + HAVE_LIBBSD, \ - * libbsd (arc4random): HAVE_ARC4RANDOM + HAVE_LIBBSD, \ + * BSD / macOS >=10.12 / glibc >=2.25 (getentropy): HAVE_GETENTROPY, \ * Linux (including <3.17) / BSD / macOS (including <10.7) / Solaris >=8 (/dev/urandom): XML_DEV_URANDOM, \ * Windows >=Vista (rand_s): _WIN32. \ \ - If insist on not using any of these, bypass this error by defining \ - XML_POOR_ENTROPY; you have been warned. \ + If you insist on not using any of these, bypass this error by defining \ + XML_POOR_ENTROPY and be vulnerable to hash flooding; you have been warned. \ \ If you have reasons to patch this detection code away or need changes \ to the build system, please open a bug. Thank you! @@ -211,6 +211,12 @@ typedef char ICHAR; #endif +#ifdef XML_LARGE_SIZE +# define XML_INDEX_MAX LLONG_MAX +#else +# define XML_INDEX_MAX LONG_MAX +#endif + /* Round up n to be a multiple of sz, where sz is a power of 2. */ #define ROUND_UP(n, sz) (((n) + ((sz) - 1)) & ~((sz) - 1)) @@ -234,7 +240,7 @@ typedef struct { unsigned char power; size_t size; size_t used; - const XML_Memory_Handling_Suite *mem; + XML_Parser parser; } HASH_TABLE; static size_t keylen(KEY s); @@ -274,8 +280,8 @@ typedef struct binding { struct binding *prevPrefixBinding; const struct attribute_id *attId; XML_Char *uri; - int uriLen; - int uriAlloc; + size_t uriLen; + size_t uriAlloc; } BINDING; typedef struct prefix { @@ -287,9 +293,9 @@ typedef struct { const XML_Char *str; const XML_Char *localPart; const XML_Char *prefix; - int strLen; - int uriLen; - int prefixLen; + size_t strLen; + size_t uriLen; + size_t prefixLen; } TAG_NAME; /* TAG represents an open element. @@ -310,8 +316,11 @@ typedef struct tag { const char *rawName; /* tagName in the original encoding */ int rawNameLength; TAG_NAME name; /* tagName in the API encoding */ - char *buf; /* buffer for name components */ - char *bufEnd; /* end of the buffer */ + union { + char *raw; /* for byte-level access (rawName storage) */ + XML_Char *str; /* for character-level access (converted name) */ + } buf; /* buffer for name components */ + char *bufEnd; /* end of the buffer */ BINDING *bindings; } TAG; @@ -348,7 +357,7 @@ typedef struct { typedef struct block { struct block *next; int size; - XML_Char s[1]; + XML_Char s[]; } BLOCK; typedef struct { @@ -357,7 +366,7 @@ typedef struct { const XML_Char *end; XML_Char *ptr; XML_Char *start; - const XML_Memory_Handling_Suite *mem; + XML_Parser parser; } STRING_POOL; /* The XML_Char before the name is used to determine whether @@ -385,9 +394,10 @@ typedef struct { const XML_Char *name; PREFIX *prefix; const ATTRIBUTE_ID *idAtt; - int nDefaultAtts; - int allocDefaultAtts; + size_t nDefaultAtts; + size_t allocDefaultAtts; DEFAULT_ATTRIBUTE *defaultAtts; + HASH_TABLE defaultAttsNames; } ELEMENT_TYPE; typedef struct { @@ -417,6 +427,7 @@ typedef struct { unsigned scaffCount; int scaffLevel; int *scaffIndex; + size_t scaffIndexSize; } DTD; enum EntityType { @@ -452,6 +463,14 @@ typedef struct accounting { unsigned long long activationThresholdBytes; } ACCOUNTING; +typedef struct MALLOC_TRACKER { + XmlBigCount bytesAllocated; + XmlBigCount peakBytesAllocated; // updated live only for debug level >=2 + unsigned long debugLevel; + float maximumAmplificationFactor; // >=1.0 + XmlBigCount activationThresholdBytes; +} MALLOC_TRACKER; + typedef struct entity_stats { unsigned int countEverOpened; unsigned int currentDepth; @@ -555,27 +574,24 @@ static XML_Bool setContext(XML_Parser parser, const XML_Char *context); static void FASTCALL normalizePublicId(XML_Char *s); -static DTD *dtdCreate(const XML_Memory_Handling_Suite *ms); +static DTD *dtdCreate(XML_Parser parser); /* do not call if m_parentParser != NULL */ -static void dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms); -static void dtdDestroy(DTD *p, XML_Bool isDocEntity, - const XML_Memory_Handling_Suite *ms); +static void dtdReset(DTD *p, XML_Parser parser); +static void dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser); static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, - const XML_Memory_Handling_Suite *ms); + XML_Parser parser); static int copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, STRING_POOL *newPool, const HASH_TABLE *oldTable); static NAMED *lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize); -static void FASTCALL hashTableInit(HASH_TABLE *table, - const XML_Memory_Handling_Suite *ms); +static void FASTCALL hashTableInit(HASH_TABLE *table, XML_Parser parser); static void FASTCALL hashTableClear(HASH_TABLE *table); static void FASTCALL hashTableDestroy(HASH_TABLE *table); static void FASTCALL hashTableIterInit(HASH_TABLE_ITER *iter, const HASH_TABLE *table); static NAMED *FASTCALL hashTableIterNext(HASH_TABLE_ITER *iter); -static void FASTCALL poolInit(STRING_POOL *pool, - const XML_Memory_Handling_Suite *ms); +static void FASTCALL poolInit(STRING_POOL *pool, XML_Parser parser); static void FASTCALL poolClear(STRING_POOL *pool); static void FASTCALL poolDestroy(STRING_POOL *pool); static XML_Char *poolAppend(STRING_POOL *pool, const ENCODING *enc, @@ -583,8 +599,11 @@ static XML_Char *poolAppend(STRING_POOL *pool, const ENCODING *enc, static XML_Char *poolStoreString(STRING_POOL *pool, const ENCODING *enc, const char *ptr, const char *end); static XML_Bool FASTCALL poolGrow(STRING_POOL *pool); +static bool FASTCALL poolGrowUntil(STRING_POOL *pool, size_t needed); static const XML_Char *FASTCALL poolCopyString(STRING_POOL *pool, const XML_Char *s); +static const XML_Char *FASTCALL poolCopyStringNoFinish(STRING_POOL *pool, + const XML_Char *s); static const XML_Char *poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n); static const XML_Char *FASTCALL poolAppendString(STRING_POOL *pool, @@ -595,15 +614,15 @@ static XML_Content *build_model(XML_Parser parser); static ELEMENT_TYPE *getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, const char *end); -static XML_Char *copyString(const XML_Char *s, - const XML_Memory_Handling_Suite *memsuite); +static XML_Char *copyString(const XML_Char *s, XML_Parser parser); -static unsigned long generate_hash_secret_salt(XML_Parser parser); +static struct sipkey generate_hash_secret_salt(void); static XML_Bool startParsing(XML_Parser parser); static XML_Parser parserCreate(const XML_Char *encodingName, const XML_Memory_Handling_Suite *memsuite, - const XML_Char *nameSep, DTD *dtd); + const XML_Char *nameSep, DTD *dtd, + XML_Parser parentParser); static void parserInit(XML_Parser parser, const XML_Char *encodingName); @@ -627,24 +646,48 @@ static void entityTrackingOnOpen(XML_Parser parser, ENTITY *entity, int sourceLine); static void entityTrackingOnClose(XML_Parser parser, ENTITY *entity, int sourceLine); +#endif /* XML_GE == 1 */ static XML_Parser getRootParserOf(XML_Parser parser, unsigned int *outLevelDiff); -#endif /* XML_GE == 1 */ static unsigned long getDebugLevel(const char *variableName, unsigned long defaultDebugLevel); +static bool poolAppendChar(STRING_POOL *pool, XML_Char c); + +static bool poolAppendChars(STRING_POOL *pool, const XML_Char *s, size_t len); + #define poolStart(pool) ((pool)->start) #define poolLength(pool) ((pool)->ptr - (pool)->start) #define poolChop(pool) ((void)--(pool->ptr)) #define poolLastChar(pool) (((pool)->ptr)[-1]) #define poolDiscard(pool) ((pool)->ptr = (pool)->start) #define poolFinish(pool) ((pool)->start = (pool)->ptr) -#define poolAppendChar(pool, c) \ - (((pool)->ptr == (pool)->end && ! poolGrow(pool)) \ - ? 0 \ - : ((*((pool)->ptr)++ = c), 1)) + +bool +poolAppendChar(STRING_POOL *pool, XML_Char c) { + if (pool->ptr == pool->end && ! poolGrow(pool)) + return false; + + *(pool->ptr)++ = c; + return true; +} + +bool +poolAppendChars(STRING_POOL *pool, const XML_Char *s, size_t len) { + // Detect and prevent integer overflow + if (len > SIZE_MAX / sizeof(XML_Char)) + return false; + + if (! poolGrowUntil(pool, len)) + return false; + + memcpy(pool->ptr, s, len * sizeof(XML_Char)); + pool->ptr += len; + + return true; +} #if ! defined(XML_TESTING) const @@ -723,11 +766,9 @@ struct XML_ParserStruct { const char *m_eventEndPtr; const char *m_positionPtr; OPEN_INTERNAL_ENTITY *m_openInternalEntities; - OPEN_INTERNAL_ENTITY *m_freeInternalEntities; OPEN_INTERNAL_ENTITY *m_openAttributeEntities; - OPEN_INTERNAL_ENTITY *m_freeAttributeEntities; OPEN_INTERNAL_ENTITY *m_openValueEntities; - OPEN_INTERNAL_ENTITY *m_freeValueEntities; + OPEN_INTERNAL_ENTITY *m_freeEntities; XML_Bool m_defaultExpandInternalEntities; int m_tagLevel; ENTITY *m_declEntity; @@ -747,7 +788,7 @@ struct XML_ParserStruct { TAG *m_freeTagList; BINDING *m_inheritedBindings; BINDING *m_freeBindingList; - int m_attsSize; + size_t m_attsSize; int m_nSpecifiedAtts; int m_idAttIndex; ATTRIBUTE *m_atts; @@ -761,7 +802,7 @@ struct XML_ParserStruct { STRING_POOL m_tempPool; STRING_POOL m_temp2Pool; char *m_groupConnector; - unsigned int m_groupSize; + size_t m_groupSize; XML_Char m_namespaceSeparator; XML_Parser m_parentParser; XML_ParsingStatus m_parsingStatus; @@ -770,166 +811,266 @@ struct XML_ParserStruct { XML_Bool m_useForeignDTD; enum XML_ParamEntityParsing m_paramEntityParsing; #endif - unsigned long m_hash_secret_salt; + struct sipkey m_hash_secret_salt_128; + XML_Bool m_hash_secret_salt_set; #if XML_GE == 1 ACCOUNTING m_accounting; + MALLOC_TRACKER m_alloc_tracker; ENTITY_STATS m_entity_stats; #endif XML_Bool m_reenter; + unsigned m_handlerCallDepth; }; -#define MALLOC(parser, s) (parser->m_mem.malloc_fcn((s))) -#define REALLOC(parser, p, s) (parser->m_mem.realloc_fcn((p), (s))) -#define FREE(parser, p) (parser->m_mem.free_fcn((p))) +#if XML_GE == 1 +# define MALLOC(parser, s) (expat_malloc((parser), (s), __LINE__)) +# define REALLOC(parser, p, s) (expat_realloc((parser), (p), (s), __LINE__)) +# define FREE(parser, p) (expat_free((parser), (p), __LINE__)) +#else +# define MALLOC(parser, s) (parser->m_mem.malloc_fcn((s))) +# define REALLOC(parser, p, s) (parser->m_mem.realloc_fcn((p), (s))) +# define FREE(parser, p) (parser->m_mem.free_fcn((p))) +#endif -XML_Parser XMLCALL -XML_ParserCreate(const XML_Char *encodingName) { - return XML_ParserCreate_MM(encodingName, NULL, NULL); +#if XML_GE == 1 +static void +expat_heap_stat(XML_Parser rootParser, char operator, XmlBigCount absDiff, + XmlBigCount newTotal, XmlBigCount peakTotal, int sourceLine) { + // NOTE: This can be +infinity or -nan + const float amplification + = (float)newTotal / (float)rootParser->m_accounting.countBytesDirect; + fprintf( + stderr, + "expat: Allocations(%p): Direct " EXPAT_FMT_ULL("10") ", allocated %c" EXPAT_FMT_ULL( + "10") " to " EXPAT_FMT_ULL("10") " (" EXPAT_FMT_ULL("10") " peak), amplification %8.2f (xmlparse.c:%d)\n", + (void *)rootParser, rootParser->m_accounting.countBytesDirect, operator, + absDiff, newTotal, peakTotal, (double)amplification, sourceLine); } -XML_Parser XMLCALL -XML_ParserCreateNS(const XML_Char *encodingName, XML_Char nsSep) { - XML_Char tmp[2] = {nsSep, 0}; - return XML_ParserCreate_MM(encodingName, NULL, tmp); -} +static bool +expat_heap_increase_tolerable(XML_Parser rootParser, XmlBigCount increase, + int sourceLine) { + assert(rootParser != NULL); + assert(increase > 0); -// "xml=http://www.w3.org/XML/1998/namespace" -static const XML_Char implicitContext[] - = {ASCII_x, ASCII_m, ASCII_l, ASCII_EQUALS, ASCII_h, - ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, - ASCII_SLASH, ASCII_w, ASCII_w, ASCII_w, ASCII_PERIOD, - ASCII_w, ASCII_3, ASCII_PERIOD, ASCII_o, ASCII_r, - ASCII_g, ASCII_SLASH, ASCII_X, ASCII_M, ASCII_L, - ASCII_SLASH, ASCII_1, ASCII_9, ASCII_9, ASCII_8, - ASCII_SLASH, ASCII_n, ASCII_a, ASCII_m, ASCII_e, - ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, - '\0'}; + XmlBigCount newTotal = 0; + bool tolerable = true; -/* To avoid warnings about unused functions: */ -#if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) + // Detect integer overflow + if ((XmlBigCount)-1 - rootParser->m_alloc_tracker.bytesAllocated < increase) { + tolerable = false; + } else { + newTotal = rootParser->m_alloc_tracker.bytesAllocated + increase; -# if defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) + if (newTotal >= rootParser->m_alloc_tracker.activationThresholdBytes) { + assert(newTotal > 0); + // NOTE: This can be +infinity when dividing by zero but not -nan + const float amplification + = (float)newTotal / (float)rootParser->m_accounting.countBytesDirect; + if (amplification + > rootParser->m_alloc_tracker.maximumAmplificationFactor) { + tolerable = false; + } + } + } -/* Obtain entropy on Linux 3.17+ */ -static int -writeRandomBytes_getrandom_nonblock(void *target, size_t count) { - int success = 0; /* full count bytes written? */ - size_t bytesWrittenTotal = 0; - const unsigned int getrandomFlags = GRND_NONBLOCK; + if (! tolerable && (rootParser->m_alloc_tracker.debugLevel >= 1)) { + expat_heap_stat(rootParser, '+', increase, newTotal, newTotal, sourceLine); + } - do { - void *const currentTarget = (void *)((char *)target + bytesWrittenTotal); - const size_t bytesToWrite = count - bytesWrittenTotal; + return tolerable; +} - const int bytesWrittenMore = -# if defined(HAVE_GETRANDOM) - getrandom(currentTarget, bytesToWrite, getrandomFlags); -# else - syscall(SYS_getrandom, currentTarget, bytesToWrite, getrandomFlags); -# endif +# if defined(XML_TESTING) +void * +# else +static void * +# endif +expat_malloc(XML_Parser parser, size_t size, int sourceLine) { + // Detect integer overflow + if (SIZE_MAX - size < sizeof(size_t) + EXPAT_MALLOC_PADDING) { + return NULL; + } - if (bytesWrittenMore > 0) { - bytesWrittenTotal += bytesWrittenMore; - if (bytesWrittenTotal >= count) - success = 1; - } - } while (! success && (errno == EINTR)); + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(rootParser->m_parentParser == NULL); - return success; -} + const size_t bytesToAllocate = sizeof(size_t) + EXPAT_MALLOC_PADDING + size; -# endif /* defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) */ + if ((XmlBigCount)-1 - rootParser->m_alloc_tracker.bytesAllocated + < bytesToAllocate) { + return NULL; // i.e. signal integer overflow as out-of-memory + } -# if ! defined(_WIN32) && defined(XML_DEV_URANDOM) + if (! expat_heap_increase_tolerable(rootParser, bytesToAllocate, + sourceLine)) { + return NULL; // i.e. signal violation as out-of-memory + } -/* Extract entropy from /dev/urandom */ -static int -writeRandomBytes_dev_urandom(void *target, size_t count) { - int success = 0; /* full count bytes written? */ - size_t bytesWrittenTotal = 0; + // Actually allocate + void *const mallocedPtr = parser->m_mem.malloc_fcn(bytesToAllocate); - const int fd = open("/dev/urandom", O_RDONLY); - if (fd < 0) { - return 0; + if (mallocedPtr == NULL) { + return NULL; } - do { - void *const currentTarget = (void *)((char *)target + bytesWrittenTotal); - const size_t bytesToWrite = count - bytesWrittenTotal; + // Update in-block recorded size + *(size_t *)mallocedPtr = size; - const ssize_t bytesWrittenMore = read(fd, currentTarget, bytesToWrite); + // Update accounting + rootParser->m_alloc_tracker.bytesAllocated += bytesToAllocate; - if (bytesWrittenMore > 0) { - bytesWrittenTotal += bytesWrittenMore; - if (bytesWrittenTotal >= count) - success = 1; + // Report as needed + if (rootParser->m_alloc_tracker.debugLevel >= 2) { + if (rootParser->m_alloc_tracker.bytesAllocated + > rootParser->m_alloc_tracker.peakBytesAllocated) { + rootParser->m_alloc_tracker.peakBytesAllocated + = rootParser->m_alloc_tracker.bytesAllocated; } - } while (! success && (errno == EINTR)); + expat_heap_stat(rootParser, '+', bytesToAllocate, + rootParser->m_alloc_tracker.bytesAllocated, + rootParser->m_alloc_tracker.peakBytesAllocated, sourceLine); + } - close(fd); - return success; + return (char *)mallocedPtr + sizeof(size_t) + EXPAT_MALLOC_PADDING; } -# endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */ +# if defined(XML_TESTING) +void +# else +static void +# endif +expat_free(XML_Parser parser, void *ptr, int sourceLine) { + assert(parser != NULL); -#endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */ + if (ptr == NULL) { + return; + } -#if defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(rootParser->m_parentParser == NULL); -static void -writeRandomBytes_arc4random(void *target, size_t count) { - size_t bytesWrittenTotal = 0; + // Extract size (to the eyes of malloc_fcn/realloc_fcn) and + // the original pointer returned by malloc/realloc + void *const mallocedPtr = (char *)ptr - EXPAT_MALLOC_PADDING - sizeof(size_t); + const size_t bytesAllocated + = sizeof(size_t) + EXPAT_MALLOC_PADDING + *(size_t *)mallocedPtr; - while (bytesWrittenTotal < count) { - const uint32_t random32 = arc4random(); - size_t i = 0; + // Update accounting + assert(rootParser->m_alloc_tracker.bytesAllocated >= bytesAllocated); + rootParser->m_alloc_tracker.bytesAllocated -= bytesAllocated; - for (; (i < sizeof(random32)) && (bytesWrittenTotal < count); - i++, bytesWrittenTotal++) { - const uint8_t random8 = (uint8_t)(random32 >> (i * 8)); - ((uint8_t *)target)[bytesWrittenTotal] = random8; - } + // Report as needed + if (rootParser->m_alloc_tracker.debugLevel >= 2) { + expat_heap_stat(rootParser, '-', bytesAllocated, + rootParser->m_alloc_tracker.bytesAllocated, + rootParser->m_alloc_tracker.peakBytesAllocated, sourceLine); } + + // NOTE: This may be freeing rootParser, so freeing has to come last + parser->m_mem.free_fcn(mallocedPtr); } -#endif /* defined(HAVE_ARC4RANDOM) && ! defined(HAVE_ARC4RANDOM_BUF) */ +# if defined(XML_TESTING) +void * +# else +static void * +# endif +expat_realloc(XML_Parser parser, void *ptr, size_t size, int sourceLine) { + assert(parser != NULL); -#ifdef _WIN32 + if (ptr == NULL) { + return expat_malloc(parser, size, sourceLine); + } -/* Provide declaration of rand_s() for MinGW-32 (not 64, which has it), - as it didn't declare it in its header prior to version 5.3.0 of its - runtime package (mingwrt, containing stdlib.h). The upstream fix - was introduced at https://osdn.net/projects/mingw/ticket/39658 . */ -# if defined(__MINGW32__) && defined(__MINGW32_VERSION) \ - && __MINGW32_VERSION < 5003000L && ! defined(__MINGW64_VERSION_MAJOR) -__declspec(dllimport) int rand_s(unsigned int *); -# endif + if (size == 0) { + expat_free(parser, ptr, sourceLine); + return NULL; + } -/* Obtain entropy on Windows using the rand_s() function which - * generates cryptographically secure random numbers. Internally it - * uses RtlGenRandom API which is present in Windows XP and later. - */ -static int -writeRandomBytes_rand_s(void *target, size_t count) { - size_t bytesWrittenTotal = 0; + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(rootParser->m_parentParser == NULL); + + // Extract original size (to the eyes of the caller) and the original + // pointer returned by malloc/realloc + void *mallocedPtr = (char *)ptr - EXPAT_MALLOC_PADDING - sizeof(size_t); + const size_t prevSize = *(size_t *)mallocedPtr; + + // Classify upcoming change + const bool isIncrease = (size > prevSize); + const size_t absDiff + = (size > prevSize) ? (size - prevSize) : (prevSize - size); + + // Ask for permission from accounting + if (isIncrease) { + if (! expat_heap_increase_tolerable(rootParser, absDiff, sourceLine)) { + return NULL; // i.e. signal violation as out-of-memory + } + } + + // NOTE: Integer overflow detection has already been done for us + // by expat_heap_increase_tolerable(..) above + assert(SIZE_MAX - sizeof(size_t) - EXPAT_MALLOC_PADDING >= size); + + // Actually allocate + mallocedPtr = parser->m_mem.realloc_fcn( + mallocedPtr, sizeof(size_t) + EXPAT_MALLOC_PADDING + size); - while (bytesWrittenTotal < count) { - unsigned int random32 = 0; - size_t i = 0; + if (mallocedPtr == NULL) { + return NULL; + } - if (rand_s(&random32)) - return 0; /* failure */ + // Update accounting + if (isIncrease) { + assert((XmlBigCount)-1 - rootParser->m_alloc_tracker.bytesAllocated + >= absDiff); + rootParser->m_alloc_tracker.bytesAllocated += absDiff; + } else { // i.e. decrease + assert(rootParser->m_alloc_tracker.bytesAllocated >= absDiff); + rootParser->m_alloc_tracker.bytesAllocated -= absDiff; + } - for (; (i < sizeof(random32)) && (bytesWrittenTotal < count); - i++, bytesWrittenTotal++) { - const uint8_t random8 = (uint8_t)(random32 >> (i * 8)); - ((uint8_t *)target)[bytesWrittenTotal] = random8; + // Report as needed + if (rootParser->m_alloc_tracker.debugLevel >= 2) { + if (rootParser->m_alloc_tracker.bytesAllocated + > rootParser->m_alloc_tracker.peakBytesAllocated) { + rootParser->m_alloc_tracker.peakBytesAllocated + = rootParser->m_alloc_tracker.bytesAllocated; } + expat_heap_stat(rootParser, isIncrease ? '+' : '-', absDiff, + rootParser->m_alloc_tracker.bytesAllocated, + rootParser->m_alloc_tracker.peakBytesAllocated, sourceLine); } - return 1; /* success */ + + // Update in-block recorded size + *(size_t *)mallocedPtr = size; + + return (char *)mallocedPtr + sizeof(size_t) + EXPAT_MALLOC_PADDING; } +#endif // XML_GE == 1 -#endif /* _WIN32 */ +XML_Parser XMLCALL +XML_ParserCreate(const XML_Char *encodingName) { + return XML_ParserCreate_MM(encodingName, NULL, NULL); +} + +XML_Parser XMLCALL +XML_ParserCreateNS(const XML_Char *encodingName, XML_Char nsSep) { + XML_Char tmp[2] = {nsSep, 0}; + return XML_ParserCreate_MM(encodingName, NULL, tmp); +} + +// "xml=http://www.w3.org/XML/1998/namespace" +static const XML_Char implicitContext[] + = {ASCII_x, ASCII_m, ASCII_l, ASCII_EQUALS, ASCII_h, + ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, + ASCII_SLASH, ASCII_w, ASCII_w, ASCII_w, ASCII_PERIOD, + ASCII_w, ASCII_3, ASCII_PERIOD, ASCII_o, ASCII_r, + ASCII_g, ASCII_SLASH, ASCII_X, ASCII_M, ASCII_L, + ASCII_SLASH, ASCII_1, ASCII_9, ASCII_9, ASCII_8, + ASCII_SLASH, ASCII_n, ASCII_a, ASCII_m, ASCII_e, + ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, + '\0'}; #if ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) @@ -958,63 +1099,85 @@ gather_time_entropy(void) { #endif /* ! defined(HAVE_ARC4RANDOM_BUF) && ! defined(HAVE_ARC4RANDOM) */ -static unsigned long -ENTROPY_DEBUG(const char *label, unsigned long entropy) { +static struct sipkey +ENTROPY_DEBUG(const char *label, struct sipkey entropy_128) { if (getDebugLevel("EXPAT_ENTROPY_DEBUG", 0) >= 1u) { - fprintf(stderr, "expat: Entropy: %s --> 0x%0*lx (%lu bytes)\n", label, - (int)sizeof(entropy) * 2, entropy, (unsigned long)sizeof(entropy)); + fprintf(stderr, + "expat: Entropy: %s --> [0x" EXPAT_FMT_LLX( + "016") ", 0x" EXPAT_FMT_LLX("016") "] (16 bytes)\n", + label, (unsigned long long)entropy_128.k[0], + (unsigned long long)entropy_128.k[1]); } - return entropy; + return entropy_128; } -static unsigned long -generate_hash_secret_salt(XML_Parser parser) { - unsigned long entropy; - (void)parser; +static struct sipkey +generate_hash_secret_salt(void) { + struct sipkey entropy; /* "Failproof" high quality providers: */ #if defined(HAVE_ARC4RANDOM_BUF) - arc4random_buf(&entropy, sizeof(entropy)); + writeRandomBytes_arc4random_buf(&entropy, sizeof(entropy)); return ENTROPY_DEBUG("arc4random_buf", entropy); #elif defined(HAVE_ARC4RANDOM) - writeRandomBytes_arc4random((void *)&entropy, sizeof(entropy)); + writeRandomBytes_arc4random(&entropy, sizeof(entropy)); return ENTROPY_DEBUG("arc4random", entropy); #else /* Try high quality providers first .. */ -# ifdef _WIN32 - if (writeRandomBytes_rand_s((void *)&entropy, sizeof(entropy))) { +# if defined(_WIN32) && ! defined(XML_POOR_ENTROPY) + if (writeRandomBytes_rand_s(&entropy, sizeof(entropy))) { return ENTROPY_DEBUG("rand_s", entropy); } +# elif defined(HAVE_GETENTROPY) + if (writeRandomBytes_getentropy(&entropy, sizeof(entropy))) { + return ENTROPY_DEBUG("getentropy", entropy); + } + errno = 0; # elif defined(HAVE_GETRANDOM) || defined(HAVE_SYSCALL_GETRANDOM) - if (writeRandomBytes_getrandom_nonblock((void *)&entropy, sizeof(entropy))) { + if (writeRandomBytes_getrandom_nonblock(&entropy, sizeof(entropy))) { return ENTROPY_DEBUG("getrandom", entropy); } # endif # if ! defined(_WIN32) && defined(XML_DEV_URANDOM) - if (writeRandomBytes_dev_urandom((void *)&entropy, sizeof(entropy))) { + if (writeRandomBytes_dev_urandom(&entropy, sizeof(entropy))) { return ENTROPY_DEBUG("/dev/urandom", entropy); } # endif /* ! defined(_WIN32) && defined(XML_DEV_URANDOM) */ /* .. and self-made low quality for backup: */ + entropy.k[0] = 0; + entropy.k[1] = gather_time_entropy(); +# if ! defined(__wasi__) /* Process ID is 0 bits entropy if attacker has local access */ - entropy = gather_time_entropy() ^ getpid(); + entropy.k[1] ^= getpid(); +# endif /* Factors are 2^31-1 and 2^61-1 (Mersenne primes M31 and M61) */ if (sizeof(unsigned long) == 4) { - return ENTROPY_DEBUG("fallback(4)", entropy * 2147483647); + entropy.k[1] *= 2147483647; + return ENTROPY_DEBUG("fallback(4)", entropy); } else { - return ENTROPY_DEBUG("fallback(8)", - entropy * (unsigned long)2305843009213693951ULL); + entropy.k[1] *= 2305843009213693951ULL; + return ENTROPY_DEBUG("fallback(8)", entropy); } #endif } -static unsigned long -get_hash_secret_salt(XML_Parser parser) { - if (parser->m_parentParser != NULL) - return get_hash_secret_salt(parser->m_parentParser); - return parser->m_hash_secret_salt; +static void +beforeHandler(XML_Parser parser) { + assert(parser->m_handlerCallDepth < UINT_MAX); + parser->m_handlerCallDepth++; +} + +static void +afterHandler(XML_Parser parser) { + assert(parser->m_handlerCallDepth > 0); + parser->m_handlerCallDepth--; +} + +static bool +isCalledFromInsideHandler(XML_Parser parser) { + return parser->m_handlerCallDepth > 0; } static enum XML_Error @@ -1085,8 +1248,10 @@ callProcessor(XML_Parser parser, const char *start, const char *end, static XML_Bool /* only valid for root parser */ startParsing(XML_Parser parser) { /* hash functions must be initialized before setContext() is called */ - if (parser->m_hash_secret_salt == 0) - parser->m_hash_secret_salt = generate_hash_secret_salt(parser); + if (parser->m_hash_secret_salt_set != XML_TRUE) { + parser->m_hash_secret_salt_128 = generate_hash_secret_salt(); + parser->m_hash_secret_salt_set = XML_TRUE; + } if (parser->m_ns) { /* implicit context only set for root parser, since child parsers (i.e. external entity parsers) will inherit it @@ -1100,19 +1265,43 @@ XML_Parser XMLCALL XML_ParserCreate_MM(const XML_Char *encodingName, const XML_Memory_Handling_Suite *memsuite, const XML_Char *nameSep) { - return parserCreate(encodingName, memsuite, nameSep, NULL); + return parserCreate(encodingName, memsuite, nameSep, NULL, NULL); } static XML_Parser parserCreate(const XML_Char *encodingName, const XML_Memory_Handling_Suite *memsuite, const XML_Char *nameSep, - DTD *dtd) { - XML_Parser parser; + DTD *dtd, XML_Parser parentParser) { + XML_Parser parser = NULL; + +#if XML_GE == 1 + const size_t increase + = sizeof(size_t) + EXPAT_MALLOC_PADDING + sizeof(struct XML_ParserStruct); + + if (parentParser != NULL) { + const XML_Parser rootParser = getRootParserOf(parentParser, NULL); + if (! expat_heap_increase_tolerable(rootParser, increase, __LINE__)) { + return NULL; + } + } +#else + UNUSED_P(parentParser); +#endif if (memsuite) { XML_Memory_Handling_Suite *mtemp; +#if XML_GE == 1 + void *const sizeAndParser + = memsuite->malloc_fcn(sizeof(size_t) + EXPAT_MALLOC_PADDING + + sizeof(struct XML_ParserStruct)); + if (sizeAndParser != NULL) { + *(size_t *)sizeAndParser = sizeof(struct XML_ParserStruct); + parser = (XML_Parser)((char *)sizeAndParser + sizeof(size_t) + + EXPAT_MALLOC_PADDING); +#else parser = memsuite->malloc_fcn(sizeof(struct XML_ParserStruct)); if (parser != NULL) { +#endif mtemp = (XML_Memory_Handling_Suite *)&(parser->m_mem); mtemp->malloc_fcn = memsuite->malloc_fcn; mtemp->realloc_fcn = memsuite->realloc_fcn; @@ -1120,39 +1309,86 @@ parserCreate(const XML_Char *encodingName, } } else { XML_Memory_Handling_Suite *mtemp; - parser = (XML_Parser)malloc(sizeof(struct XML_ParserStruct)); +#if XML_GE == 1 + void *const sizeAndParser = malloc(sizeof(size_t) + EXPAT_MALLOC_PADDING + + sizeof(struct XML_ParserStruct)); + if (sizeAndParser != NULL) { + *(size_t *)sizeAndParser = sizeof(struct XML_ParserStruct); + parser = (XML_Parser)((char *)sizeAndParser + sizeof(size_t) + + EXPAT_MALLOC_PADDING); +#else + parser = malloc(sizeof(struct XML_ParserStruct)); if (parser != NULL) { +#endif mtemp = (XML_Memory_Handling_Suite *)&(parser->m_mem); mtemp->malloc_fcn = malloc; mtemp->realloc_fcn = realloc; mtemp->free_fcn = free; } - } + } // cppcheck-suppress[memleak symbolName=sizeAndParser] // Cppcheck >=2.18.0 if (! parser) return parser; +#if XML_GE == 1 + // Initialize .m_alloc_tracker + memset(&parser->m_alloc_tracker, 0, sizeof(MALLOC_TRACKER)); + if (parentParser == NULL) { + parser->m_alloc_tracker.debugLevel + = getDebugLevel("EXPAT_MALLOC_DEBUG", 0u); + parser->m_alloc_tracker.maximumAmplificationFactor + = EXPAT_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT; + parser->m_alloc_tracker.activationThresholdBytes + = EXPAT_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT; + + // NOTE: This initialization needs to come this early because these fields + // are read by allocation tracking code + parser->m_parentParser = NULL; + parser->m_accounting.countBytesDirect = 0; + } else { + parser->m_parentParser = parentParser; + } + + // Record XML_ParserStruct allocation we did a few lines up before + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(rootParser->m_parentParser == NULL); + assert(SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated >= increase); + rootParser->m_alloc_tracker.bytesAllocated += increase; + + // Report on allocation + if (rootParser->m_alloc_tracker.debugLevel >= 2) { + if (rootParser->m_alloc_tracker.bytesAllocated + > rootParser->m_alloc_tracker.peakBytesAllocated) { + rootParser->m_alloc_tracker.peakBytesAllocated + = rootParser->m_alloc_tracker.bytesAllocated; + } + + expat_heap_stat(rootParser, '+', increase, + rootParser->m_alloc_tracker.bytesAllocated, + rootParser->m_alloc_tracker.peakBytesAllocated, __LINE__); + } +#else + parser->m_parentParser = NULL; +#endif // XML_GE == 1 + parser->m_buffer = NULL; parser->m_bufferLim = NULL; parser->m_attsSize = INIT_ATTS_SIZE; - parser->m_atts - = (ATTRIBUTE *)MALLOC(parser, parser->m_attsSize * sizeof(ATTRIBUTE)); + parser->m_atts = MALLOC(parser, parser->m_attsSize * sizeof(ATTRIBUTE)); if (parser->m_atts == NULL) { FREE(parser, parser); return NULL; } #ifdef XML_ATTR_INFO - parser->m_attInfo = (XML_AttrInfo *)MALLOC( - parser, parser->m_attsSize * sizeof(XML_AttrInfo)); + parser->m_attInfo = MALLOC(parser, parser->m_attsSize * sizeof(XML_AttrInfo)); if (parser->m_attInfo == NULL) { FREE(parser, parser->m_atts); FREE(parser, parser); return NULL; } #endif - parser->m_dataBuf - = (XML_Char *)MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char)); + parser->m_dataBuf = MALLOC(parser, INIT_DATA_BUF_SIZE * sizeof(XML_Char)); if (parser->m_dataBuf == NULL) { FREE(parser, parser->m_atts); #ifdef XML_ATTR_INFO @@ -1166,7 +1402,7 @@ parserCreate(const XML_Char *encodingName, if (dtd) parser->m_dtd = dtd; else { - parser->m_dtd = dtdCreate(&parser->m_mem); + parser->m_dtd = dtdCreate(parser); if (parser->m_dtd == NULL) { FREE(parser, parser->m_dataBuf); FREE(parser, parser->m_atts); @@ -1180,9 +1416,7 @@ parserCreate(const XML_Char *encodingName, parser->m_freeBindingList = NULL; parser->m_freeTagList = NULL; - parser->m_freeInternalEntities = NULL; - parser->m_freeAttributeEntities = NULL; - parser->m_freeValueEntities = NULL; + parser->m_freeEntities = NULL; parser->m_groupSize = 0; parser->m_groupConnector = NULL; @@ -1200,8 +1434,8 @@ parserCreate(const XML_Char *encodingName, parser->m_protocolEncodingName = NULL; - poolInit(&parser->m_tempPool, &(parser->m_mem)); - poolInit(&parser->m_temp2Pool, &(parser->m_mem)); + poolInit(&parser->m_tempPool, parser); + poolInit(&parser->m_temp2Pool, parser); parserInit(parser, encodingName); if (encodingName && ! parser->m_protocolEncodingName) { @@ -1233,7 +1467,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_processor = prologInitProcessor; XmlPrologStateInit(&parser->m_prologState); if (encodingName != NULL) { - parser->m_protocolEncodingName = copyString(encodingName, &(parser->m_mem)); + parser->m_protocolEncodingName = copyString(encodingName, parser); } parser->m_curBase = NULL; XmlInitEncoding(&parser->m_initEncoding, &parser->m_encoding, 0); @@ -1295,16 +1529,18 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_unknownEncodingMem = NULL; parser->m_unknownEncodingRelease = NULL; parser->m_unknownEncodingData = NULL; - parser->m_parentParser = NULL; parser->m_parsingStatus.parsing = XML_INITIALIZED; // Reentry can only be triggered inside m_processor calls parser->m_reenter = XML_FALSE; + parser->m_handlerCallDepth = 0; #ifdef XML_DTD parser->m_isParamEntity = XML_FALSE; parser->m_useForeignDTD = XML_FALSE; parser->m_paramEntityParsing = XML_PARAM_ENTITY_PARSING_NEVER; #endif - parser->m_hash_secret_salt = 0; + parser->m_hash_secret_salt_128.k[0] = 0; + parser->m_hash_secret_salt_128.k[1] = 0; + parser->m_hash_secret_salt_set = XML_FALSE; #if XML_GE == 1 memset(&parser->m_accounting, 0, sizeof(ACCOUNTING)); @@ -1330,12 +1566,22 @@ moveToFreeBindingList(XML_Parser parser, BINDING *bindings) { } } +/* Moves a list of entities onto the start of another list. */ +static void +moveEntityList(OPEN_INTERNAL_ENTITY **dst, OPEN_INTERNAL_ENTITY **src) { + for (OPEN_INTERNAL_ENTITY *head = *src; head != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = head; + head = head->next; + openEntity->next = *dst; + *dst = openEntity; + } +} + XML_Bool XMLCALL XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { TAG *tStk; - OPEN_INTERNAL_ENTITY *openEntityList; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return XML_FALSE; if (parser->m_parentParser) @@ -1350,32 +1596,14 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { tag->bindings = NULL; parser->m_freeTagList = tag; } - /* move m_openInternalEntities to m_freeInternalEntities */ - openEntityList = parser->m_openInternalEntities; - while (openEntityList) { - OPEN_INTERNAL_ENTITY *openEntity = openEntityList; - openEntityList = openEntity->next; - openEntity->next = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = openEntity; - } - /* move m_openAttributeEntities to m_freeAttributeEntities (i.e. same task but - * for attributes) */ - openEntityList = parser->m_openAttributeEntities; - while (openEntityList) { - OPEN_INTERNAL_ENTITY *openEntity = openEntityList; - openEntityList = openEntity->next; - openEntity->next = parser->m_freeAttributeEntities; - parser->m_freeAttributeEntities = openEntity; - } - /* move m_openValueEntities to m_freeValueEntities (i.e. same task but - * for value entities) */ - openEntityList = parser->m_openValueEntities; - while (openEntityList) { - OPEN_INTERNAL_ENTITY *openEntity = openEntityList; - openEntityList = openEntity->next; - openEntity->next = parser->m_freeValueEntities; - parser->m_freeValueEntities = openEntity; - } + /* move m_openInternalEntities to m_freeEntities */ + moveEntityList(&parser->m_freeEntities, &parser->m_openInternalEntities); + /* move m_openAttributeEntities to m_freeEntities (i.e. same task but for + * attributes) */ + moveEntityList(&parser->m_freeEntities, &parser->m_openAttributeEntities); + /* move m_openValueEntities to m_freeEntities (i.e. same task but for value + * entities) */ + moveEntityList(&parser->m_freeEntities, &parser->m_openValueEntities); moveToFreeBindingList(parser, parser->m_inheritedBindings); FREE(parser, parser->m_unknownEncodingMem); if (parser->m_unknownEncodingRelease) @@ -1385,7 +1613,7 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { FREE(parser, (void *)parser->m_protocolEncodingName); parser->m_protocolEncodingName = NULL; parserInit(parser, encodingName); - dtdReset(parser->m_dtd, &parser->m_mem); + dtdReset(parser->m_dtd, parser); return XML_TRUE; } @@ -1421,7 +1649,7 @@ XML_SetEncoding(XML_Parser parser, const XML_Char *encodingName) { parser->m_protocolEncodingName = NULL; else { /* Copy the new encoding name into allocated memory */ - parser->m_protocolEncodingName = copyString(encodingName, &(parser->m_mem)); + parser->m_protocolEncodingName = copyString(encodingName, parser); if (! parser->m_protocolEncodingName) return XML_STATUS_ERROR; } @@ -1450,6 +1678,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, XML_ExternalEntityRefHandler oldExternalEntityRefHandler; XML_SkippedEntityHandler oldSkippedEntityHandler; XML_UnknownEncodingHandler oldUnknownEncodingHandler; + void *oldUnknownEncodingHandlerData; XML_ElementDeclHandler oldElementDeclHandler; XML_AttlistDeclHandler oldAttlistDeclHandler; XML_EntityDeclHandler oldEntityDeclHandler; @@ -1470,7 +1699,8 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, from hash tables associated with either parser without us having to worry which hash secrets each table has. */ - unsigned long oldhash_secret_salt; + struct sipkey oldhash_secret_salt_128; + XML_Bool oldhash_secret_salt_set; XML_Bool oldReparseDeferralEnabled; /* Validate the oldParser parameter before we pull everything out of it */ @@ -1495,6 +1725,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, oldExternalEntityRefHandler = parser->m_externalEntityRefHandler; oldSkippedEntityHandler = parser->m_skippedEntityHandler; oldUnknownEncodingHandler = parser->m_unknownEncodingHandler; + oldUnknownEncodingHandlerData = parser->m_unknownEncodingHandlerData; oldElementDeclHandler = parser->m_elementDeclHandler; oldAttlistDeclHandler = parser->m_attlistDeclHandler; oldEntityDeclHandler = parser->m_entityDeclHandler; @@ -1515,7 +1746,8 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, from hash tables associated with either parser without us having to worry which hash secrets each table has. */ - oldhash_secret_salt = parser->m_hash_secret_salt; + oldhash_secret_salt_128 = parser->m_hash_secret_salt_128; + oldhash_secret_salt_set = parser->m_hash_secret_salt_set; oldReparseDeferralEnabled = parser->m_reparseDeferralEnabled; #ifdef XML_DTD @@ -1523,16 +1755,12 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, newDtd = oldDtd; #endif /* XML_DTD */ - /* Note that the magical uses of the pre-processor to make field - access look more like C++ require that `parser' be overwritten - here. This makes this function more painful to follow than it - would be otherwise. - */ if (parser->m_ns) { XML_Char tmp[2] = {parser->m_namespaceSeparator, 0}; - parser = parserCreate(encodingName, &parser->m_mem, tmp, newDtd); + parser = parserCreate(encodingName, &parser->m_mem, tmp, newDtd, oldParser); } else { - parser = parserCreate(encodingName, &parser->m_mem, NULL, newDtd); + parser + = parserCreate(encodingName, &parser->m_mem, NULL, newDtd, oldParser); } if (! parser) @@ -1554,6 +1782,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, parser->m_externalEntityRefHandler = oldExternalEntityRefHandler; parser->m_skippedEntityHandler = oldSkippedEntityHandler; parser->m_unknownEncodingHandler = oldUnknownEncodingHandler; + parser->m_unknownEncodingHandlerData = oldUnknownEncodingHandlerData; parser->m_elementDeclHandler = oldElementDeclHandler; parser->m_attlistDeclHandler = oldAttlistDeclHandler; parser->m_entityDeclHandler = oldEntityDeclHandler; @@ -1568,7 +1797,8 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, parser->m_externalEntityRefHandlerArg = oldExternalEntityRefHandlerArg; parser->m_defaultExpandInternalEntities = oldDefaultExpandInternalEntities; parser->m_ns_triplets = oldns_triplets; - parser->m_hash_secret_salt = oldhash_secret_salt; + parser->m_hash_secret_salt_128 = oldhash_secret_salt_128; + parser->m_hash_secret_salt_set = oldhash_secret_salt_set; parser->m_reparseDeferralEnabled = oldReparseDeferralEnabled; parser->m_parentParser = oldParser; #ifdef XML_DTD @@ -1576,7 +1806,7 @@ XML_ExternalEntityParserCreate(XML_Parser oldParser, const XML_Char *context, parser->m_prologState.inEntityValue = oldInEntityValue; if (context) { #endif /* XML_DTD */ - if (! dtdCopy(oldParser, parser->m_dtd, oldDtd, &parser->m_mem) + if (! dtdCopy(oldParser, parser->m_dtd, oldDtd, parser) || ! setContext(parser, context)) { XML_ParserFree(parser); return NULL; @@ -1614,8 +1844,7 @@ destroyBindings(BINDING *bindings, XML_Parser parser) { void XMLCALL XML_ParserFree(XML_Parser parser) { TAG *tagList; - OPEN_INTERNAL_ENTITY *entityList; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return; /* free m_tagStack and m_freeTagList */ tagList = parser->m_tagStack; @@ -1629,52 +1858,39 @@ XML_ParserFree(XML_Parser parser) { } p = tagList; tagList = tagList->parent; - FREE(parser, p->buf); + FREE(parser, p->buf.raw); destroyBindings(p->bindings, parser); FREE(parser, p); } - /* free m_openInternalEntities and m_freeInternalEntities */ - entityList = parser->m_openInternalEntities; - for (;;) { - OPEN_INTERNAL_ENTITY *openEntity; - if (entityList == NULL) { - if (parser->m_freeInternalEntities == NULL) - break; - entityList = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = NULL; - } - openEntity = entityList; + /* free m_openInternalEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_openInternalEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; entityList = entityList->next; FREE(parser, openEntity); } - /* free m_openAttributeEntities and m_freeAttributeEntities */ - entityList = parser->m_openAttributeEntities; - for (;;) { - OPEN_INTERNAL_ENTITY *openEntity; - if (entityList == NULL) { - if (parser->m_freeAttributeEntities == NULL) - break; - entityList = parser->m_freeAttributeEntities; - parser->m_freeAttributeEntities = NULL; - } - openEntity = entityList; + /* free m_openAttributeEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_openAttributeEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; entityList = entityList->next; FREE(parser, openEntity); } - /* free m_openValueEntities and m_freeValueEntities */ - entityList = parser->m_openValueEntities; - for (;;) { - OPEN_INTERNAL_ENTITY *openEntity; - if (entityList == NULL) { - if (parser->m_freeValueEntities == NULL) - break; - entityList = parser->m_freeValueEntities; - parser->m_freeValueEntities = NULL; - } - openEntity = entityList; + /* free m_openValueEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_openValueEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; + entityList = entityList->next; + FREE(parser, openEntity); + } + /* free m_freeEntities */ + for (OPEN_INTERNAL_ENTITY *entityList = parser->m_freeEntities; + entityList != NULL;) { + OPEN_INTERNAL_ENTITY *const openEntity = entityList; entityList = entityList->next; FREE(parser, openEntity); } + parser->m_freeEntities = NULL; destroyBindings(parser->m_freeBindingList, parser); destroyBindings(parser->m_inheritedBindings, parser); poolDestroy(&parser->m_tempPool); @@ -1688,14 +1904,16 @@ XML_ParserFree(XML_Parser parser) { #else if (parser->m_dtd) #endif /* XML_DTD */ - dtdDestroy(parser->m_dtd, (XML_Bool)! parser->m_parentParser, - &parser->m_mem); - FREE(parser, (void *)parser->m_atts); + dtdDestroy(parser->m_dtd, (XML_Bool)! parser->m_parentParser, parser); + FREE(parser, parser->m_atts); #ifdef XML_ATTR_INFO - FREE(parser, (void *)parser->m_attInfo); + FREE(parser, parser->m_attInfo); #endif FREE(parser, parser->m_groupConnector); - FREE(parser, parser->m_buffer); + // NOTE: We are avoiding FREE(..) here because parser->m_buffer + // is not being allocated with MALLOC(..) but with plain + // .malloc_fcn(..). + parser->m_mem.free_fcn(parser->m_buffer); FREE(parser, parser->m_dataBuf); FREE(parser, parser->m_nsAtts); FREE(parser, parser->m_unknownEncodingMem); @@ -2010,19 +2228,58 @@ XML_SetParamEntityParsing(XML_Parser parser, #endif } +// DEPRECATED since Expat 2.8.0. int XMLCALL XML_SetHashSalt(XML_Parser parser, unsigned long hash_salt) { if (parser == NULL) return 0; - if (parser->m_parentParser) - return XML_SetHashSalt(parser->m_parentParser, hash_salt); + + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(! rootParser->m_parentParser); + /* block after XML_Parse()/XML_ParseBuffer() has been called */ - if (parserBusy(parser)) + if (parserBusy(rootParser)) return 0; - parser->m_hash_secret_salt = hash_salt; + + rootParser->m_hash_secret_salt_128.k[0] = 0; + rootParser->m_hash_secret_salt_128.k[1] = hash_salt; + + if (hash_salt != 0) { // to remain backwards compatible + rootParser->m_hash_secret_salt_set = XML_TRUE; + + if (sizeof(unsigned long) == 4) + ENTROPY_DEBUG("explicit(4)", rootParser->m_hash_secret_salt_128); + else + ENTROPY_DEBUG("explicit(8)", rootParser->m_hash_secret_salt_128); + } + return 1; } +XML_Bool XMLCALL +XML_SetHashSalt16Bytes(XML_Parser parser, const uint8_t entropy[16]) { + if (parser == NULL) + return XML_FALSE; + + if (entropy == NULL) + return XML_FALSE; + + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(! rootParser->m_parentParser); + + /* block after XML_Parse()/XML_ParseBuffer() has been called */ + if (parserBusy(rootParser)) + return XML_FALSE; + + sip_tokey(&(rootParser->m_hash_secret_salt_128), entropy); + + rootParser->m_hash_secret_salt_set = XML_TRUE; + + ENTROPY_DEBUG("explicit(16)", rootParser->m_hash_secret_salt_128); + + return XML_TRUE; +} + enum XML_Status XMLCALL XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { if ((parser == NULL) || (len < 0) || ((s == NULL) && (len != 0))) { @@ -2030,6 +2287,8 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; return XML_STATUS_ERROR; } + if (isCalledFromInsideHandler(parser)) + return XML_STATUS_ERROR; switch (parser->m_parsingStatus.parsing) { case XML_SUSPENDED: parser->m_errorCode = XML_ERROR_SUSPENDED; @@ -2042,7 +2301,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return XML_STATUS_ERROR; } - /* fall through */ + EXPAT_FALLTHROUGH; default: parser->m_parsingStatus.parsing = XML_PARSING; } @@ -2053,7 +2312,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { int nLeftOver; enum XML_Status result; /* Detect overflow (a+b > MAX <==> b > MAX-a) */ - if ((XML_Size)len > ((XML_Size)-1) / 2 - parser->m_parseEndByteIndex) { + if (len > XML_INDEX_MAX - parser->m_parseEndByteIndex) { parser->m_errorCode = XML_ERROR_NO_MEMORY; parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_processor = errorProcessor; @@ -2084,7 +2343,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_parsingStatus.parsing = XML_FINISHED; return XML_STATUS_OK; } - /* fall through */ + EXPAT_FALLTHROUGH; default: result = XML_STATUS_OK; } @@ -2139,7 +2398,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { const char *start; enum XML_Status result = XML_STATUS_OK; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return XML_STATUS_ERROR; if (len < 0) { @@ -2165,11 +2424,19 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return XML_STATUS_ERROR; } - /* fall through */ + EXPAT_FALLTHROUGH; default: parser->m_parsingStatus.parsing = XML_PARSING; } + // Detect and avoid integer overflow + if (len > XML_INDEX_MAX - parser->m_parseEndByteIndex) { + parser->m_errorCode = XML_ERROR_NO_MEMORY; + parser->m_eventPtr = parser->m_eventEndPtr = NULL; + parser->m_processor = errorProcessor; + return XML_STATUS_ERROR; + } + start = parser->m_bufferPtr; parser->m_positionPtr = start; parser->m_bufferEnd += len; @@ -2195,6 +2462,7 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { parser->m_parsingStatus.parsing = XML_FINISHED; return result; } + break; default:; /* should not happen */ } } @@ -2205,9 +2473,32 @@ XML_ParseBuffer(XML_Parser parser, int len, int isFinal) { return result; } +/* Modifies `parser`’s buffer to be backed by `newBuf`. */ +static void +setParserBuffer(XML_Parser parser, char *newBuf, int newBufSize, int keep) { + parser->m_bufferLim = newBuf + newBufSize; + if (parser->m_bufferPtr) { + const int parsing + = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr); + memcpy(newBuf, parser->m_bufferPtr - keep, parsing + keep); + // NOTE: We are avoiding FREE(..) here because parser->m_buffer + // is not being allocated with MALLOC(..) but with plain + // .malloc_fcn(..). + parser->m_mem.free_fcn(parser->m_buffer); + parser->m_buffer = newBuf; + parser->m_bufferEnd = newBuf + parsing + keep; + parser->m_bufferPtr = newBuf + keep; + } else { + /* This must be a brand new buffer with no data in it yet */ + parser->m_buffer = newBuf; + parser->m_bufferEnd = newBuf; + parser->m_bufferPtr = newBuf; + } +} + void *XMLCALL XML_GetBuffer(XML_Parser parser, int len) { - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return NULL; if (len < 0) { parser->m_errorCode = XML_ERROR_NO_MEMORY; @@ -2228,9 +2519,6 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_lastBufferRequestSize = len; if (len > EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_bufferEnd) || parser->m_buffer == NULL) { -#if XML_CONTEXT_BYTES > 0 - int keep; -#endif /* XML_CONTEXT_BYTES > 0 */ /* Do not invoke signed arithmetic overflow: */ int neededSize = (int)((unsigned)len + (unsigned)EXPAT_SAFE_PTR_DIFF( @@ -2240,7 +2528,9 @@ XML_GetBuffer(XML_Parser parser, int len) { return NULL; } #if XML_CONTEXT_BYTES > 0 - keep = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer); + const int parsed + = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer); + int keep = parsed; if (keep > XML_CONTEXT_BYTES) keep = XML_CONTEXT_BYTES; /* Detect and prevent integer overflow */ @@ -2248,16 +2538,16 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return NULL; } - neededSize += keep; +#else + int keep = 0; #endif /* XML_CONTEXT_BYTES > 0 */ + neededSize += keep; if (parser->m_buffer && parser->m_bufferPtr && neededSize <= EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer)) { #if XML_CONTEXT_BYTES > 0 - if (keep < EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer)) { - int offset - = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferPtr, parser->m_buffer) - - keep; + if (keep < parsed) { + int offset = parsed - keep; /* The buffer pointers cannot be NULL here; we have at least some bytes * in the buffer */ memmove(parser->m_buffer, &parser->m_buffer[offset], @@ -2274,7 +2564,6 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_bufferPtr = parser->m_buffer; #endif /* XML_CONTEXT_BYTES > 0 */ } else { - char *newBuf; int bufferSize = (int)EXPAT_SAFE_PTR_DIFF(parser->m_bufferLim, parser->m_buffer); if (bufferSize == 0) @@ -2287,43 +2576,14 @@ XML_GetBuffer(XML_Parser parser, int len) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return NULL; } - newBuf = (char *)MALLOC(parser, bufferSize); - if (newBuf == 0) { + // NOTE: We are avoiding MALLOC(..) here to leave limiting + // the input size to the application using Expat. + char *const newBuf = parser->m_mem.malloc_fcn(bufferSize); + if (newBuf == NULL) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return NULL; } - parser->m_bufferLim = newBuf + bufferSize; -#if XML_CONTEXT_BYTES > 0 - if (parser->m_bufferPtr) { - memcpy(newBuf, &parser->m_bufferPtr[-keep], - EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) - + keep); - FREE(parser, parser->m_buffer); - parser->m_buffer = newBuf; - parser->m_bufferEnd - = parser->m_buffer - + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr) - + keep; - parser->m_bufferPtr = parser->m_buffer + keep; - } else { - /* This must be a brand new buffer with no data in it yet */ - parser->m_bufferEnd = newBuf; - parser->m_bufferPtr = parser->m_buffer = newBuf; - } -#else - if (parser->m_bufferPtr) { - memcpy(newBuf, parser->m_bufferPtr, - EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr)); - FREE(parser, parser->m_buffer); - parser->m_bufferEnd - = newBuf - + EXPAT_SAFE_PTR_DIFF(parser->m_bufferEnd, parser->m_bufferPtr); - } else { - /* This must be a brand new buffer with no data in it yet */ - parser->m_bufferEnd = newBuf; - } - parser->m_bufferPtr = parser->m_buffer = newBuf; -#endif /* XML_CONTEXT_BYTES > 0 */ + setParserBuffer(parser, newBuf, bufferSize, keep); } parser->m_eventPtr = parser->m_eventEndPtr = NULL; parser->m_positionPtr = NULL; @@ -2376,7 +2636,7 @@ enum XML_Status XMLCALL XML_ResumeParser(XML_Parser parser) { enum XML_Status result = XML_STATUS_OK; - if (parser == NULL) + if ((parser == NULL) || isCalledFromInsideHandler(parser)) return XML_STATUS_ERROR; if (parser->m_parsingStatus.parsing != XML_SUSPENDED) { parser->m_errorCode = XML_ERROR_NOT_SUSPENDED; @@ -2402,6 +2662,7 @@ XML_ResumeParser(XML_Parser parser) { parser->m_parsingStatus.parsing = XML_FINISHED; return result; } + break; default:; } } @@ -2463,7 +2724,7 @@ XML_GetInputContext(XML_Parser parser, int *offset, int *size) { (void)offset; (void)size; #endif /* XML_CONTEXT_BYTES > 0 */ - return (const char *)0; + return NULL; } XML_Size XMLCALL @@ -2492,28 +2753,43 @@ XML_GetCurrentColumnNumber(XML_Parser parser) { void XMLCALL XML_FreeContentModel(XML_Parser parser, XML_Content *model) { - if (parser != NULL) - FREE(parser, model); + if (parser == NULL) + return; + + // NOTE: We are avoiding FREE(..) here because the content model + // has been created using plain .malloc_fcn(..) rather than MALLOC(..). + parser->m_mem.free_fcn(model); } void *XMLCALL XML_MemMalloc(XML_Parser parser, size_t size) { if (parser == NULL) return NULL; - return MALLOC(parser, size); + + // NOTE: We are avoiding MALLOC(..) here to not include + // user allocations with allocation tracking and limiting. + return parser->m_mem.malloc_fcn(size); } void *XMLCALL XML_MemRealloc(XML_Parser parser, void *ptr, size_t size) { if (parser == NULL) return NULL; - return REALLOC(parser, ptr, size); + + // NOTE: We are avoiding REALLOC(..) here to not include + // user allocations with allocation tracking and limiting. + return parser->m_mem.realloc_fcn(ptr, size); } void XMLCALL XML_MemFree(XML_Parser parser, void *ptr) { - if (parser != NULL) - FREE(parser, ptr); + if (parser == NULL) + return; + + // NOTE: We are avoiding FREE(..) here because XML_MemMalloc and + // XML_MemRealloc are not using MALLOC(..) and REALLOC(..) + // but plain .malloc_fcn(..) and .realloc_fcn(..), internally. + parser->m_mem.free_fcn(ptr); } void XMLCALL @@ -2713,6 +2989,13 @@ XML_GetFeatureList(void) { EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT}, /* Added in Expat 2.6.0. */ {XML_FEATURE_GE, XML_L("XML_GE"), 0}, + /* Added in Expat 2.7.2. */ + {XML_FEATURE_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT, + XML_L("XML_AT_MAX_AMP"), + (long int)EXPAT_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION_DEFAULT}, + {XML_FEATURE_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT, + XML_L("XML_AT_ACT_THRES"), + (long int)EXPAT_ALLOC_TRACKER_ACTIVATION_THRESHOLD_DEFAULT}, #endif {XML_FEATURE_END, NULL, 0}}; @@ -2741,6 +3024,29 @@ XML_SetBillionLaughsAttackProtectionActivationThreshold( parser->m_accounting.activationThresholdBytes = activationThresholdBytes; return XML_TRUE; } + +XML_Bool XMLCALL +XML_SetAllocTrackerMaximumAmplification(XML_Parser parser, + float maximumAmplificationFactor) { + if ((parser == NULL) || (parser->m_parentParser != NULL) + || isnan(maximumAmplificationFactor) + || (maximumAmplificationFactor < 1.0f)) { + return XML_FALSE; + } + parser->m_alloc_tracker.maximumAmplificationFactor + = maximumAmplificationFactor; + return XML_TRUE; +} + +XML_Bool XMLCALL +XML_SetAllocTrackerActivationThreshold( + XML_Parser parser, unsigned long long activationThresholdBytes) { + if ((parser == NULL) || (parser->m_parentParser != NULL)) { + return XML_FALSE; + } + parser->m_alloc_tracker.activationThresholdBytes = activationThresholdBytes; + return XML_TRUE; +} #endif /* XML_GE == 1 */ XML_Bool XMLCALL @@ -2761,10 +3067,10 @@ static XML_Bool storeRawNames(XML_Parser parser) { TAG *tag = parser->m_tagStack; while (tag) { - int bufSize; - int nameLen = sizeof(XML_Char) * (tag->name.strLen + 1); + size_t bufSize; + size_t nameLen = sizeof(XML_Char) * (tag->name.strLen + 1); size_t rawNameLen; - char *rawNameBuf = tag->buf + nameLen; + char *rawNameBuf = tag->buf.raw + nameLen; /* Stop if already stored. Since m_tagStack is a stack, we can stop at the first entry that has already been copied; everything below it in the stack is already been accounted for in a @@ -2777,25 +3083,25 @@ storeRawNames(XML_Parser parser) { */ rawNameLen = ROUND_UP(tag->rawNameLength, sizeof(XML_Char)); /* Detect and prevent integer overflow. */ - if (rawNameLen > (size_t)INT_MAX - nameLen) + if (rawNameLen > SIZE_MAX - nameLen) return XML_FALSE; - bufSize = nameLen + (int)rawNameLen; - if (bufSize > tag->bufEnd - tag->buf) { - char *temp = (char *)REALLOC(parser, tag->buf, bufSize); + bufSize = nameLen + rawNameLen; + if (bufSize > (size_t)(tag->bufEnd - tag->buf.raw)) { + char *temp = REALLOC(parser, tag->buf.raw, bufSize); if (temp == NULL) return XML_FALSE; - /* if tag->name.str points to tag->buf (only when namespace + /* if tag->name.str points to tag->buf.str (only when namespace processing is off) then we have to update it */ - if (tag->name.str == (XML_Char *)tag->buf) + if (tag->name.str == tag->buf.str) tag->name.str = (XML_Char *)temp; /* if tag->name.localPart is set (when namespace processing is on) then update it as well, since it will always point into tag->buf */ if (tag->name.localPart) tag->name.localPart - = (XML_Char *)temp + (tag->name.localPart - (XML_Char *)tag->buf); - tag->buf = temp; + = (XML_Char *)temp + (tag->name.localPart - tag->buf.str); + tag->buf.raw = temp; tag->bufEnd = temp + bufSize; rawNameBuf = temp + nameLen; } @@ -2904,7 +3210,7 @@ externalEntityInitProcessor3(XML_Parser parser, const char *start, if (parser->m_reenter) { return XML_ERROR_UNEXPECTED_STATE; // LCOV_EXCL_LINE } - /* Fall through */ + EXPAT_FALLTHROUGH; default: start = next; } @@ -2983,7 +3289,9 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, *eventEndPP = end; if (parser->m_characterDataHandler) { XML_Char c = 0xA; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &c, 1); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, end); /* We are at the end of the final buffer, should we check for @@ -3036,9 +3344,11 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, ((char *)&ch) + sizeof(XML_Char), __LINE__, XML_ACCOUNT_ENTITY_EXPANSION); #endif /* XML_GE == 1 */ - if (parser->m_characterDataHandler) + if (parser->m_characterDataHandler) { + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &ch, 1); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; } @@ -3058,9 +3368,11 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, else if (! entity->is_internal) return XML_ERROR_ENTITY_DECLARED_IN_PE; } else if (! entity) { - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; } @@ -3071,10 +3383,12 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, if (entity->textPtr) { enum XML_Error result; if (! parser->m_defaultExpandInternalEntities) { - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, entity->name, 0); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; } @@ -3088,9 +3402,12 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, entity->open = XML_FALSE; if (! context) return XML_ERROR_NO_MEMORY; - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, context, entity->base, - entity->systemId, entity->publicId)) + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, context, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) return XML_ERROR_EXTERNAL_ENTITY_HANDLING; poolDiscard(&parser->m_tempPool); } else if (parser->m_defaultHandler) @@ -3098,7 +3415,6 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, break; } case XML_TOK_START_TAG_NO_ATTS: - /* fall through */ case XML_TOK_START_TAG_WITH_ATTS: { TAG *tag; enum XML_Error result; @@ -3107,15 +3423,15 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, tag = parser->m_freeTagList; parser->m_freeTagList = parser->m_freeTagList->parent; } else { - tag = (TAG *)MALLOC(parser, sizeof(TAG)); + tag = MALLOC(parser, sizeof(TAG)); if (! tag) return XML_ERROR_NO_MEMORY; - tag->buf = (char *)MALLOC(parser, INIT_TAG_BUF_SIZE); - if (! tag->buf) { + tag->buf.raw = MALLOC(parser, INIT_TAG_BUF_SIZE); + if (! tag->buf.raw) { FREE(parser, tag); return XML_ERROR_NO_MEMORY; } - tag->bufEnd = tag->buf + INIT_TAG_BUF_SIZE; + tag->bufEnd = tag->buf.raw + INIT_TAG_BUF_SIZE; } tag->bindings = NULL; tag->parent = parser->m_tagStack; @@ -3128,46 +3444,47 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, { const char *rawNameEnd = tag->rawName + tag->rawNameLength; const char *fromPtr = tag->rawName; - toPtr = (XML_Char *)tag->buf; + toPtr = tag->buf.str; for (;;) { - int bufSize; - int convLen; const enum XML_Convert_Result convert_res = XmlConvert(enc, &fromPtr, rawNameEnd, (ICHAR **)&toPtr, (ICHAR *)tag->bufEnd - 1); - convLen = (int)(toPtr - (XML_Char *)tag->buf); + const size_t convLen = (size_t)(toPtr - tag->buf.str); if ((fromPtr >= rawNameEnd) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) { tag->name.strLen = convLen; break; } - bufSize = (int)(tag->bufEnd - tag->buf) << 1; + if (SIZE_MAX / 2 < (size_t)(tag->bufEnd - tag->buf.raw)) + return XML_ERROR_NO_MEMORY; + const size_t bufSize = (size_t)(tag->bufEnd - tag->buf.raw) * 2; { - char *temp = (char *)REALLOC(parser, tag->buf, bufSize); + char *temp = REALLOC(parser, tag->buf.raw, bufSize); if (temp == NULL) return XML_ERROR_NO_MEMORY; - tag->buf = temp; + tag->buf.raw = temp; tag->bufEnd = temp + bufSize; toPtr = (XML_Char *)temp + convLen; } } } - tag->name.str = (XML_Char *)tag->buf; + tag->name.str = tag->buf.str; *toPtr = XML_T('\0'); result = storeAtts(parser, enc, s, &(tag->name), &(tag->bindings), account); if (result) return result; - if (parser->m_startElementHandler) + if (parser->m_startElementHandler) { + beforeHandler(parser); parser->m_startElementHandler(parser->m_handlerArg, tag->name.str, (const XML_Char **)parser->m_atts); - else if (parser->m_defaultHandler) + afterHandler(parser); + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); poolClear(&parser->m_tempPool); break; } case XML_TOK_EMPTY_ELEMENT_NO_ATTS: - /* fall through */ case XML_TOK_EMPTY_ELEMENT_WITH_ATTS: { const char *rawName = s + enc->minBytesPerChar; enum XML_Error result; @@ -3187,14 +3504,18 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, } poolFinish(&parser->m_tempPool); if (parser->m_startElementHandler) { + beforeHandler(parser); parser->m_startElementHandler(parser->m_handlerArg, name.str, (const XML_Char **)parser->m_atts); + afterHandler(parser); noElmHandlers = XML_FALSE; } if (parser->m_endElementHandler) { if (parser->m_startElementHandler) *eventPP = *eventEndPP; + beforeHandler(parser); parser->m_endElementHandler(parser->m_handlerArg, name.str); + afterHandler(parser); noElmHandlers = XML_FALSE; } if (noElmHandlers && parser->m_defaultHandler) @@ -3252,14 +3573,19 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, } *uri = XML_T('\0'); } + beforeHandler(parser); parser->m_endElementHandler(parser->m_handlerArg, tag->name.str); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); while (tag->bindings) { BINDING *b = tag->bindings; - if (parser->m_endNamespaceDeclHandler) + if (parser->m_endNamespaceDeclHandler) { + beforeHandler(parser); parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name); + afterHandler(parser); + } tag->bindings = tag->bindings->nextTagBinding; b->nextTagBinding = parser->m_freeBindingList; parser->m_freeBindingList = b; @@ -3282,8 +3608,10 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, return XML_ERROR_BAD_CHAR_REF; if (parser->m_characterDataHandler) { XML_Char buf[XML_ENCODE_MAX]; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, buf, XmlEncode(n, (ICHAR *)buf)); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -3292,32 +3620,38 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, case XML_TOK_DATA_NEWLINE: if (parser->m_characterDataHandler) { XML_Char c = 0xA; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &c, 1); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; case XML_TOK_CDATA_SECT_OPEN: { enum XML_Error result; - if (parser->m_startCdataSectionHandler) + if (parser->m_startCdataSectionHandler) { + beforeHandler(parser); parser->m_startCdataSectionHandler(parser->m_handlerArg); - /* BEGIN disabled code */ - /* Suppose you doing a transformation on a document that involves - changing only the character data. You set up a defaultHandler - and a characterDataHandler. The defaultHandler simply copies - characters through. The characterDataHandler does the - transformation and writes the characters out escaping them as - necessary. This case will fail to work if we leave out the - following two lines (because & and < inside CDATA sections will - be incorrectly escaped). - - However, now we have a start/endCdataSectionHandler, so it seems - easier to let the user deal with this. - */ - else if ((0) && parser->m_characterDataHandler) + afterHandler(parser); + /* BEGIN disabled code */ + /* Suppose you doing a transformation on a document that involves + changing only the character data. You set up a defaultHandler + and a characterDataHandler. The defaultHandler simply copies + characters through. The characterDataHandler does the + transformation and writes the characters out escaping them as + necessary. This case will fail to work if we leave out the + following two lines (because & and < inside CDATA sections will + be incorrectly escaped). + + However, now we have a start/endCdataSectionHandler, so it seems + easier to let the user deal with this. + */ + } else if ((0) && parser->m_characterDataHandler) { + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0); - /* END disabled code */ - else if (parser->m_defaultHandler) + afterHandler(parser); + /* END disabled code */ + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); result = doCdataSection(parser, enc, &next, end, nextPtr, haveMore, account); @@ -3337,13 +3671,18 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, if (MUST_CONVERT(enc, s)) { ICHAR *dataPtr = (ICHAR *)parser->m_dataBuf; XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd); + beforeHandler(parser); parser->m_characterDataHandler( parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); - } else + afterHandler(parser); + } else { + beforeHandler(parser); parser->m_characterDataHandler( parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)end - (const XML_Char *)s)); + afterHandler(parser); + } } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, end); /* We are at the end of the final buffer, should we check for @@ -3368,16 +3707,21 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const enum XML_Convert_Result convert_res = XmlConvert( enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd); *eventEndPP = s; + beforeHandler(parser); charDataHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); + afterHandler(parser); if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; *eventPP = s; } - } else + } else { + beforeHandler(parser); charDataHandler(parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)next - (const XML_Char *)s)); + afterHandler(parser); + } } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -3415,7 +3759,7 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, *nextPtr = next; return XML_ERROR_NONE; } - /* Fall through */ + EXPAT_FALLTHROUGH; default:; *eventPP = s = next; } @@ -3435,8 +3779,11 @@ freeBindings(XML_Parser parser, BINDING *bindings) { /* m_startNamespaceDeclHandler will have been called for this * binding in addBindings(), so call the end handler now. */ - if (parser->m_endNamespaceDeclHandler) + if (parser->m_endNamespaceDeclHandler) { + beforeHandler(parser); parser->m_endNamespaceDeclHandler(parser->m_handlerArg, b->prefix->name); + afterHandler(parser); + } bindings = bindings->nextTagBinding; b->nextTagBinding = parser->m_freeBindingList; @@ -3460,20 +3807,14 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, TAG_NAME *tagNamePtr, BINDING **bindingsPtr, enum XML_Account account) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ - ELEMENT_TYPE *elementType; - int nDefaultAtts; - const XML_Char **appAtts; /* the attribute list for the application */ int attIndex = 0; - int prefixLen; - int i; - int n; XML_Char *uri; int nPrefixes = 0; BINDING *binding; const XML_Char *localPart; /* lookup the element type name */ - elementType + ELEMENT_TYPE *elementType = (ELEMENT_TYPE *)lookup(parser, &dtd->elementTypes, tagNamePtr->str, 0); if (! elementType) { const XML_Char *name = poolCopyString(&dtd->pool, tagNamePtr->str); @@ -3483,78 +3824,76 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sizeof(ELEMENT_TYPE)); if (! elementType) return XML_ERROR_NO_MEMORY; + if (! elementType->defaultAttsNames.parser) + hashTableInit(&(elementType->defaultAttsNames), parser); if (parser->m_ns && ! setElementTypePrefix(parser, elementType)) return XML_ERROR_NO_MEMORY; } - nDefaultAtts = elementType->nDefaultAtts; + const size_t nDefaultAtts = elementType->nDefaultAtts; + + /* Detect and prevent integer overflow. */ + if (parser->m_attsSize > (size_t)INT_MAX) + return XML_ERROR_NO_MEMORY; /* get the attributes from the tokenizer */ - n = XmlGetAttributes(enc, attStr, parser->m_attsSize, parser->m_atts); + size_t n = (size_t)XmlGetAttributes(enc, attStr, (int)parser->m_attsSize, + parser->m_atts); /* Detect and prevent integer overflow */ - if (n > INT_MAX - nDefaultAtts) { + if (n > SIZE_MAX - nDefaultAtts) { return XML_ERROR_NO_MEMORY; } if (n + nDefaultAtts > parser->m_attsSize) { - int oldAttsSize = parser->m_attsSize; - ATTRIBUTE *temp; -#ifdef XML_ATTR_INFO - XML_AttrInfo *temp2; -#endif + size_t oldAttsSize = parser->m_attsSize; /* Detect and prevent integer overflow */ - if ((nDefaultAtts > INT_MAX - INIT_ATTS_SIZE) - || (n > INT_MAX - (nDefaultAtts + INIT_ATTS_SIZE))) { + if ((nDefaultAtts > SIZE_MAX - INIT_ATTS_SIZE) + || (n > SIZE_MAX - (nDefaultAtts + INIT_ATTS_SIZE))) { return XML_ERROR_NO_MEMORY; } parser->m_attsSize = n + nDefaultAtts + INIT_ATTS_SIZE; - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)parser->m_attsSize > (size_t)(-1) / sizeof(ATTRIBUTE)) { + /* Detect and prevent integer overflow. */ + if (parser->m_attsSize > SIZE_MAX / sizeof(ATTRIBUTE)) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } -#endif - temp = (ATTRIBUTE *)REALLOC(parser, (void *)parser->m_atts, - parser->m_attsSize * sizeof(ATTRIBUTE)); + ATTRIBUTE *const temp = REALLOC(parser, parser->m_atts, + parser->m_attsSize * sizeof(ATTRIBUTE)); if (temp == NULL) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } parser->m_atts = temp; #ifdef XML_ATTR_INFO - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -# if UINT_MAX >= SIZE_MAX - if ((unsigned)parser->m_attsSize > (size_t)(-1) / sizeof(XML_AttrInfo)) { + /* Detect and prevent integer overflow. */ + if (parser->m_attsSize > SIZE_MAX / sizeof(XML_AttrInfo)) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } -# endif - temp2 = (XML_AttrInfo *)REALLOC(parser, (void *)parser->m_attInfo, - parser->m_attsSize * sizeof(XML_AttrInfo)); + XML_AttrInfo *const temp2 = REALLOC( + parser, parser->m_attInfo, parser->m_attsSize * sizeof(XML_AttrInfo)); if (temp2 == NULL) { parser->m_attsSize = oldAttsSize; return XML_ERROR_NO_MEMORY; } parser->m_attInfo = temp2; #endif - if (n > oldAttsSize) - XmlGetAttributes(enc, attStr, n, parser->m_atts); + if (n > oldAttsSize) { + /* Detect and prevent integer overflow. */ + if (n > (size_t)INT_MAX) + return XML_ERROR_NO_MEMORY; + XmlGetAttributes(enc, attStr, (int)n, parser->m_atts); + } } - appAtts = (const XML_Char **)parser->m_atts; - for (i = 0; i < n; i++) { + /* the attribute list for the application */ + const XML_Char **const appAtts = (const XML_Char **)parser->m_atts; + for (size_t i = 0; i < n; i++) { ATTRIBUTE *currAtt = &parser->m_atts[i]; #ifdef XML_ATTR_INFO XML_AttrInfo *currAttInfo = &parser->m_attInfo[i]; @@ -3587,13 +3926,11 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, (attId->name)[-1] = 1; appAtts[attIndex++] = attId->name; if (! parser->m_atts[i].normalized) { - enum XML_Error result; XML_Bool isCdata = XML_TRUE; /* figure out whether declared as other than CDATA */ if (attId->maybeTokenized) { - int j; - for (j = 0; j < nDefaultAtts; j++) { + for (size_t j = 0; j < nDefaultAtts; j++) { if (attId == elementType->defaultAtts[j].id) { isCdata = elementType->defaultAtts[j].isCdata; break; @@ -3602,7 +3939,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, } /* normalize the attribute value */ - result = storeAttributeValue( + const enum XML_Error result = storeAttributeValue( parser, enc, isCdata, parser->m_atts[i].valuePtr, parser->m_atts[i].valueEnd, &parser->m_tempPool, account); if (result) @@ -3640,7 +3977,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* set-up for XML_GetSpecifiedAttributeCount and XML_GetIdAttributeIndex */ parser->m_nSpecifiedAtts = attIndex; if (elementType->idAtt && (elementType->idAtt->name)[-1]) { - for (i = 0; i < attIndex; i += 2) + for (int i = 0; i < attIndex; i += 2) if (appAtts[i] == elementType->idAtt->name) { parser->m_idAttIndex = i; break; @@ -3649,7 +3986,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, parser->m_idAttIndex = -1; /* do attribute defaulting */ - for (i = 0; i < nDefaultAtts; i++) { + for (size_t i = 0; i < nDefaultAtts; i++) { const DEFAULT_ATTRIBUTE *da = elementType->defaultAtts + i; if (! (da->id->name)[-1] && da->value) { if (da->id->prefix) { @@ -3675,9 +4012,9 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* expand prefixed attribute names, check for duplicates, and clear flags that say whether attributes were specified */ - i = 0; + int i = 0; if (nPrefixes) { - int j; /* hash table index */ + unsigned int j; /* hash table index */ unsigned long version = parser->m_nsAttsVersion; /* Detect and prevent invalid shift */ @@ -3690,7 +4027,6 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* size of hash table must be at least 2 * (# of prefixed attributes) */ if ((nPrefixes << 1) >> parser->m_nsAttsPower) { /* true for m_nsAttsPower = 0 */ - NS_ATT *temp; /* hash table size must also be a power of 2 and >= 8 */ while (nPrefixes >> parser->m_nsAttsPower++) ; @@ -3711,15 +4047,15 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, * from -Wtype-limits on platforms where * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ #if UINT_MAX >= SIZE_MAX - if (nsAttsSize > (size_t)(-1) / sizeof(NS_ATT)) { + if (nsAttsSize > SIZE_MAX / sizeof(NS_ATT)) { /* Restore actual size of memory in m_nsAtts */ parser->m_nsAttsPower = oldNsAttsPower; return XML_ERROR_NO_MEMORY; } #endif - temp = (NS_ATT *)REALLOC(parser, parser->m_nsAtts, - nsAttsSize * sizeof(NS_ATT)); + NS_ATT *const temp + = REALLOC(parser, parser->m_nsAtts, nsAttsSize * sizeof(NS_ATT)); if (! temp) { /* Restore actual size of memory in m_nsAtts */ parser->m_nsAttsPower = oldNsAttsPower; @@ -3740,9 +4076,6 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, for (; i < attIndex; i += 2) { const XML_Char *s = appAtts[i]; if (s[-1] == 2) { /* prefixed */ - ATTRIBUTE_ID *id; - const BINDING *b; - unsigned long uriHash; struct siphash sip_state; struct sipkey sip_key; @@ -3750,7 +4083,8 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sip24_init(&sip_state, &sip_key); ((XML_Char *)s)[-1] = 0; /* clear flag */ - id = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0); + ATTRIBUTE_ID *const id + = (ATTRIBUTE_ID *)lookup(parser, &dtd->attributeIds, s, 0); if (! id || ! id->prefix) { /* This code is walking through the appAtts array, dealing * with (in this case) a prefixed attribute name. To be in @@ -3758,7 +4092,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, * has to have passed through the hash table lookup once * already. That implies that an entry for it already * exists, so the lookup above will return a pointer to - * already allocated memory. There is no opportunaity for + * already allocated memory. There is no opportunity for * the allocator to fail, so the condition above cannot be * fulfilled. * @@ -3768,15 +4102,12 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, */ return XML_ERROR_NO_MEMORY; /* LCOV_EXCL_LINE */ } - b = id->prefix->binding; + const BINDING *const b = id->prefix->binding; if (! b) return XML_ERROR_UNBOUND_PREFIX; - for (j = 0; j < b->uriLen; j++) { - const XML_Char c = b->uri[j]; - if (! poolAppendChar(&parser->m_tempPool, c)) - return XML_ERROR_NO_MEMORY; - } + if (! poolAppendChars(&parser->m_tempPool, b->uri, b->uriLen)) + return XML_ERROR_NO_MEMORY; sip24_update(&sip_state, b->uri, b->uriLen * sizeof(XML_Char)); @@ -3785,12 +4116,13 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, sip24_update(&sip_state, s, keylen(s) * sizeof(XML_Char)); - do { /* copies null terminator */ - if (! poolAppendChar(&parser->m_tempPool, *s)) + { + const size_t len = xcslen(s) + /*null terminator*/ 1; + if (! poolAppendChars(&parser->m_tempPool, s, len)) return XML_ERROR_NO_MEMORY; - } while (*s++); + } - uriHash = (unsigned long)sip24_final(&sip_state); + const unsigned long uriHash = (unsigned long)sip24_final(&sip_state); { /* Check hash table for duplicate of expanded name (uriName). Derived from code in lookup(parser, HASH_TABLE *table, ...). @@ -3818,10 +4150,9 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, if (parser->m_ns_triplets) { /* append namespace separator and prefix */ parser->m_tempPool.ptr[-1] = parser->m_namespaceSeparator; s = b->prefix->name; - do { - if (! poolAppendChar(&parser->m_tempPool, *s)) - return XML_ERROR_NO_MEMORY; - } while (*s++); + const size_t len = xcslen(s) + /*null terminator*/ 1; + if (! poolAppendChars(&parser->m_tempPool, s, len)) + return XML_ERROR_NO_MEMORY; } /* store expanded name in attribute list */ @@ -3864,48 +4195,36 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, localPart = tagNamePtr->str; } else return XML_ERROR_NONE; - prefixLen = 0; - if (parser->m_ns_triplets && binding->prefix->name) { - for (; binding->prefix->name[prefixLen++];) - ; /* prefixLen includes null terminator */ - } + size_t prefixLen = 0; + if (parser->m_ns_triplets && binding->prefix->name) + prefixLen = xcslen(binding->prefix->name) + /*null terminator*/ 1; tagNamePtr->localPart = localPart; tagNamePtr->uriLen = binding->uriLen; tagNamePtr->prefix = binding->prefix->name; tagNamePtr->prefixLen = prefixLen; - for (i = 0; localPart[i++];) - ; /* i includes null terminator */ + + const size_t localPartLen = xcslen(localPart) + /*null terminator*/ 1; /* Detect and prevent integer overflow */ - if (binding->uriLen > INT_MAX - prefixLen - || i > INT_MAX - (binding->uriLen + prefixLen)) { + if (binding->uriLen > SIZE_MAX - prefixLen + || localPartLen > SIZE_MAX - (binding->uriLen + prefixLen)) { return XML_ERROR_NO_MEMORY; } - n = i + binding->uriLen + prefixLen; - if (n > binding->uriAlloc) { - TAG *p; - + const size_t totalLen = localPartLen + binding->uriLen + prefixLen; + if (totalLen > binding->uriAlloc) { /* Detect and prevent integer overflow */ - if (n > INT_MAX - EXPAND_SPARE) { - return XML_ERROR_NO_MEMORY; - } - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)(n + EXPAND_SPARE) > (size_t)(-1) / sizeof(XML_Char)) { + if (totalLen > SIZE_MAX - EXPAND_SPARE + || totalLen + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) { return XML_ERROR_NO_MEMORY; } -#endif - uri = (XML_Char *)MALLOC(parser, (n + EXPAND_SPARE) * sizeof(XML_Char)); + uri = MALLOC(parser, (totalLen + EXPAND_SPARE) * sizeof(XML_Char)); if (! uri) return XML_ERROR_NO_MEMORY; - binding->uriAlloc = n + EXPAND_SPARE; + binding->uriAlloc = totalLen + EXPAND_SPARE; memcpy(uri, binding->uri, binding->uriLen * sizeof(XML_Char)); - for (p = parser->m_tagStack; p; p = p->parent) + for (TAG *p = parser->m_tagStack; p; p = p->parent) if (p->name.str == binding->uri) p->name.str = uri; FREE(parser, binding->uri); @@ -3913,10 +4232,14 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, } /* if m_namespaceSeparator != '\0' then uri includes it already */ uri = binding->uri + binding->uriLen; - memcpy(uri, localPart, i * sizeof(XML_Char)); + /* Detect and prevent integer overflow */ + if (localPartLen > SIZE_MAX / sizeof(XML_Char)) { + return XML_ERROR_NO_MEMORY; + } + memcpy(uri, localPart, localPartLen * sizeof(XML_Char)); /* we always have a namespace separator between localPart and prefix */ if (prefixLen) { - uri += i - 1; + uri += localPartLen - 1; *uri = parser->m_namespaceSeparator; /* replace null terminator */ memcpy(uri + 1, binding->prefix->name, prefixLen * sizeof(XML_Char)); } @@ -4051,7 +4374,7 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, ASCII_8, ASCII_SLASH, ASCII_n, ASCII_a, ASCII_m, ASCII_e, ASCII_s, ASCII_p, ASCII_a, ASCII_c, ASCII_e, '\0'}; - static const int xmlLen = (int)sizeof(xmlNamespace) / sizeof(XML_Char) - 1; + static const size_t xmlLen = sizeof(xmlNamespace) / sizeof(XML_Char) - 1; // "http://www.w3.org/2000/xmlns/" static const XML_Char xmlnsNamespace[] = {ASCII_h, ASCII_t, ASCII_t, ASCII_p, ASCII_COLON, ASCII_SLASH, @@ -4059,15 +4382,14 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, ASCII_3, ASCII_PERIOD, ASCII_o, ASCII_r, ASCII_g, ASCII_SLASH, ASCII_2, ASCII_0, ASCII_0, ASCII_0, ASCII_SLASH, ASCII_x, ASCII_m, ASCII_l, ASCII_n, ASCII_s, ASCII_SLASH, '\0'}; - static const int xmlnsLen - = (int)sizeof(xmlnsNamespace) / sizeof(XML_Char) - 1; + static const size_t xmlnsLen = sizeof(xmlnsNamespace) / sizeof(XML_Char) - 1; XML_Bool mustBeXML = XML_FALSE; XML_Bool isXML = XML_TRUE; XML_Bool isXMLNS = XML_TRUE; BINDING *b; - int len; + size_t len; /* empty URI is only valid for default namespace per XML NS 1.0 (not 1.1) */ if (*uri == XML_T('\0') && prefix->name) @@ -4086,6 +4408,10 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, } for (len = 0; uri[len]; len++) { + /* Detect and prevent integer overflow */ + if (len == SIZE_MAX) { + return XML_ERROR_NO_MEMORY; + } if (isXML && (len > xmlLen || uri[len] != xmlNamespace[len])) isXML = XML_FALSE; @@ -4126,28 +4452,24 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, if (isXMLNS) return XML_ERROR_RESERVED_NAMESPACE_URI; - if (parser->m_namespaceSeparator) + if (parser->m_namespaceSeparator) { + /* Detect and prevent integer overflow */ + if (len == SIZE_MAX) { + return XML_ERROR_NO_MEMORY; + } len++; + } if (parser->m_freeBindingList) { b = parser->m_freeBindingList; if (len > b->uriAlloc) { /* Detect and prevent integer overflow */ - if (len > INT_MAX - EXPAND_SPARE) { + if (len > SIZE_MAX - EXPAND_SPARE + || len + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) { return XML_ERROR_NO_MEMORY; } - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)(len + EXPAND_SPARE) > (size_t)(-1) / sizeof(XML_Char)) { - return XML_ERROR_NO_MEMORY; - } -#endif - - XML_Char *temp = (XML_Char *)REALLOC( - parser, b->uri, sizeof(XML_Char) * (len + EXPAND_SPARE)); + XML_Char *temp + = REALLOC(parser, b->uri, sizeof(XML_Char) * (len + EXPAND_SPARE)); if (temp == NULL) return XML_ERROR_NO_MEMORY; b->uri = temp; @@ -4155,26 +4477,17 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, } parser->m_freeBindingList = b->nextTagBinding; } else { - b = (BINDING *)MALLOC(parser, sizeof(BINDING)); + b = MALLOC(parser, sizeof(BINDING)); if (! b) return XML_ERROR_NO_MEMORY; /* Detect and prevent integer overflow */ - if (len > INT_MAX - EXPAND_SPARE) { - return XML_ERROR_NO_MEMORY; - } - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)(len + EXPAND_SPARE) > (size_t)(-1) / sizeof(XML_Char)) { + if (len > SIZE_MAX - EXPAND_SPARE + || len + EXPAND_SPARE > SIZE_MAX / sizeof(XML_Char)) { return XML_ERROR_NO_MEMORY; } -#endif - b->uri - = (XML_Char *)MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE)); + b->uri = MALLOC(parser, sizeof(XML_Char) * (len + EXPAND_SPARE)); if (! b->uri) { FREE(parser, b); return XML_ERROR_NO_MEMORY; @@ -4196,9 +4509,12 @@ addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, b->nextTagBinding = *bindingsPtr; *bindingsPtr = b; /* if attId == NULL then we are not starting a namespace scope */ - if (attId && parser->m_startNamespaceDeclHandler) + if (attId && parser->m_startNamespaceDeclHandler) { + beforeHandler(parser); parser->m_startNamespaceDeclHandler(parser->m_handlerArg, prefix->name, prefix->binding ? uri : 0); + afterHandler(parser); + } return XML_ERROR_NONE; } @@ -4260,15 +4576,20 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, *eventEndPP = next; switch (tok) { case XML_TOK_CDATA_SECT_CLOSE: - if (parser->m_endCdataSectionHandler) + if (parser->m_endCdataSectionHandler) { + beforeHandler(parser); parser->m_endCdataSectionHandler(parser->m_handlerArg); + afterHandler(parser); + } /* BEGIN disabled code */ /* see comment under XML_TOK_CDATA_SECT_OPEN */ - else if ((0) && parser->m_characterDataHandler) + else if ((0) && parser->m_characterDataHandler) { + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, parser->m_dataBuf, 0); - /* END disabled code */ - else if (parser->m_defaultHandler) + afterHandler(parser); + /* END disabled code */ + } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); *startPtr = next; *nextPtr = next; @@ -4279,7 +4600,9 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, case XML_TOK_DATA_NEWLINE: if (parser->m_characterDataHandler) { XML_Char c = 0xA; + beforeHandler(parser); parser->m_characterDataHandler(parser->m_handlerArg, &c, 1); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); break; @@ -4292,16 +4615,21 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, const enum XML_Convert_Result convert_res = XmlConvert( enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd); *eventEndPP = next; + beforeHandler(parser); charDataHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); + afterHandler(parser); if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; *eventPP = s; } - } else + } else { + beforeHandler(parser); charDataHandler(parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)next - (const XML_Char *)s)); + afterHandler(parser); + } } else if (parser->m_defaultHandler) reportDefault(parser, enc, s, next); } break; @@ -4346,7 +4674,7 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, if (parser->m_reenter) { return XML_ERROR_UNEXPECTED_STATE; // LCOV_EXCL_LINE } - /* Fall through */ + EXPAT_FALLTHROUGH; default:; *eventPP = s = next; } @@ -4540,8 +4868,10 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity, const char *s, if (! storedversion) return XML_ERROR_NO_MEMORY; } + beforeHandler(parser); parser->m_xmlDeclHandler(parser->m_handlerArg, storedversion, storedEncName, standalone); + afterHandler(parser); } else if (parser->m_defaultHandler) reportDefault(parser, parser->m_encoding, s, next); if (parser->m_protocolEncodingName == NULL) { @@ -4591,8 +4921,11 @@ handleUnknownEncoding(XML_Parser parser, const XML_Char *encodingName) { info.convert = NULL; info.data = NULL; info.release = NULL; - if (parser->m_unknownEncodingHandler(parser->m_unknownEncodingHandlerData, - encodingName, &info)) { + beforeHandler(parser); + const int status = parser->m_unknownEncodingHandler( + parser->m_unknownEncodingHandlerData, encodingName, &info); + afterHandler(parser); + if (status) { ENCODING *enc; parser->m_unknownEncodingMem = MALLOC(parser, XmlSizeOfUnknownEncoding()); if (! parser->m_unknownEncodingMem) { @@ -4720,7 +5053,7 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, } /* If we get this token, we have the start of what might be a normal tag, but not a declaration (i.e. it doesn't begin with - "<!"). In a DTD context, that isn't legal. + "<!" or "<?"). In a DTD context, that isn't legal. */ else if (tok == XML_TOK_INSTANCE_START) { *nextPtr = next; @@ -4809,6 +5142,15 @@ entityValueProcessor(XML_Parser parser, const char *s, const char *end, /* found end of entity value - can store it now */ return storeEntityValue(parser, enc, s, end, XML_ACCOUNT_DIRECT, NULL); } + /* If we get this token, we have the start of what might be a + normal tag, but not a declaration (i.e. it doesn't begin with + "<!" or "<?"). In a DTD context, that isn't legal. + */ + else if (tok == XML_TOK_INSTANCE_START) { + *nextPtr = next; + return XML_ERROR_SYNTAX; + } + start = next; } } @@ -4960,9 +5302,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, break; case XML_ROLE_DOCTYPE_INTERNAL_SUBSET: if (parser->m_startDoctypeDeclHandler) { + beforeHandler(parser); parser->m_startDoctypeDeclHandler( parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid, parser->m_doctypePubid, 1); + afterHandler(parser); parser->m_doctypeName = NULL; poolClear(&parser->m_tempPool); handleDefault = XML_FALSE; @@ -5001,7 +5345,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, handleDefault = XML_FALSE; goto alreadyChecked; } - /* fall through */ + EXPAT_FALLTHROUGH; case XML_ROLE_ENTITY_PUBLIC_ID: if (! XmlIsPublicId(enc, s, next, eventPP)) return XML_ERROR_PUBLICID; @@ -5029,9 +5373,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } if (parser->m_doctypeName) { + beforeHandler(parser); parser->m_startDoctypeDeclHandler( parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid, parser->m_doctypePubid, 0); + afterHandler(parser); poolClear(&parser->m_tempPool); handleDefault = XML_FALSE; } @@ -5058,14 +5404,22 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (parser->m_useForeignDTD) entity->base = parser->m_curBase; dtd->paramEntityRead = XML_FALSE; - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) return XML_ERROR_EXTERNAL_ENTITY_HANDLING; if (dtd->paramEntityRead) { - if (! dtd->standalone && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + if (! dtd->standalone && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int handlerStatus + = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! handlerStatus) + return XML_ERROR_NOT_STANDALONE; + } } /* if we didn't read the foreign DTD then this means that there is no external subset and we must reset dtd->hasParamEntityRefs @@ -5078,7 +5432,9 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } #endif /* XML_DTD */ if (parser->m_endDoctypeDeclHandler) { + beforeHandler(parser); parser->m_endDoctypeDeclHandler(parser->m_handlerArg); + afterHandler(parser); handleDefault = XML_FALSE; } break; @@ -5098,14 +5454,22 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_NO_MEMORY; entity->base = parser->m_curBase; dtd->paramEntityRead = XML_FALSE; - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) return XML_ERROR_EXTERNAL_ENTITY_HANDLING; if (dtd->paramEntityRead) { - if (! dtd->standalone && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + if (! dtd->standalone && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int handlerStatus + = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! handlerStatus) + return XML_ERROR_NOT_STANDALONE; + } } /* if we didn't read the foreign DTD then this means that there is no external subset and we must reset dtd->hasParamEntityRefs @@ -5198,10 +5562,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, poolFinish(&parser->m_tempPool); } *eventEndPP = s; + beforeHandler(parser); parser->m_attlistDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, parser->m_declAttributeId->name, parser->m_declAttributeType, 0, role == XML_ROLE_REQUIRED_ATTRIBUTE_VALUE); + afterHandler(parser); handleDefault = XML_FALSE; } } @@ -5236,10 +5602,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, poolFinish(&parser->m_tempPool); } *eventEndPP = s; + beforeHandler(parser); parser->m_attlistDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, parser->m_declAttributeId->name, parser->m_declAttributeType, attVal, role == XML_ROLE_FIXED_ATTRIBUTE_VALUE); + afterHandler(parser); poolClear(&parser->m_tempPool); handleDefault = XML_FALSE; } @@ -5254,16 +5622,22 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, parser, enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar, XML_ACCOUNT_NONE); if (parser->m_declEntity) { + /* Detect and prevent signed integer overflow */ + if ((size_t)poolLength(&dtd->entityValuePool) > (size_t)INT_MAX) { + return XML_ERROR_NO_MEMORY; + } parser->m_declEntity->textPtr = poolStart(&dtd->entityValuePool); parser->m_declEntity->textLen = (int)(poolLength(&dtd->entityValuePool)); poolFinish(&dtd->entityValuePool); if (parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, parser->m_declEntity->textPtr, parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); + afterHandler(parser); handleDefault = XML_FALSE; } } else @@ -5281,10 +5655,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, parser->m_declEntity->textPtr, parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); + afterHandler(parser); handleDefault = XML_FALSE; } } @@ -5315,9 +5691,13 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, #ifdef XML_DTD && ! parser->m_paramEntityParsing #endif /* XML_DTD */ - && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int status = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! status) + return XML_ERROR_NOT_STANDALONE; + } #ifndef XML_DTD break; #else /* XML_DTD */ @@ -5329,7 +5709,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, parser->m_declEntity->publicId = NULL; } #endif /* XML_DTD */ - /* fall through */ + EXPAT_FALLTHROUGH; case XML_ROLE_ENTITY_SYSTEM_ID: if (dtd->keepProcessing && parser->m_declEntity) { parser->m_declEntity->systemId @@ -5360,10 +5740,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (dtd->keepProcessing && parser->m_declEntity && parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, 0, 0, parser->m_declEntity->base, parser->m_declEntity->systemId, parser->m_declEntity->publicId, 0); + afterHandler(parser); handleDefault = XML_FALSE; } break; @@ -5376,17 +5758,21 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, poolFinish(&dtd->pool); if (parser->m_unparsedEntityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_unparsedEntityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->base, parser->m_declEntity->systemId, parser->m_declEntity->publicId, parser->m_declEntity->notation); + afterHandler(parser); handleDefault = XML_FALSE; } else if (parser->m_entityDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, 0, 0, 0, parser->m_declEntity->base, parser->m_declEntity->systemId, parser->m_declEntity->publicId, parser->m_declEntity->notation); + afterHandler(parser); handleDefault = XML_FALSE; } } @@ -5493,9 +5879,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (! systemId) return XML_ERROR_NO_MEMORY; *eventEndPP = s; + beforeHandler(parser); parser->m_notationDeclHandler( parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase, systemId, parser->m_declNotationPublicId); + afterHandler(parser); handleDefault = XML_FALSE; } poolClear(&parser->m_tempPool); @@ -5503,9 +5891,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, case XML_ROLE_NOTATION_NO_SYSTEM_ID: if (parser->m_declNotationPublicId && parser->m_notationDeclHandler) { *eventEndPP = s; + beforeHandler(parser); parser->m_notationDeclHandler( parser->m_handlerArg, parser->m_declNotationName, parser->m_curBase, 0, parser->m_declNotationPublicId); + afterHandler(parser); handleDefault = XML_FALSE; } poolClear(&parser->m_tempPool); @@ -5539,41 +5929,20 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, case XML_ROLE_GROUP_OPEN: if (parser->m_prologState.level >= parser->m_groupSize) { if (parser->m_groupSize) { - { - /* Detect and prevent integer overflow */ - if (parser->m_groupSize > (unsigned int)(-1) / 2u) { - return XML_ERROR_NO_MEMORY; - } - - char *const new_connector = (char *)REALLOC( - parser, parser->m_groupConnector, parser->m_groupSize *= 2); - if (new_connector == NULL) { - parser->m_groupSize /= 2; - return XML_ERROR_NO_MEMORY; - } - parser->m_groupConnector = new_connector; + /* Detect and prevent integer overflow */ + if (parser->m_groupSize > SIZE_MAX / 2) { + return XML_ERROR_NO_MEMORY; } - if (dtd->scaffIndex) { - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if (parser->m_groupSize > (size_t)(-1) / sizeof(int)) { - return XML_ERROR_NO_MEMORY; - } -#endif - - int *const new_scaff_index = (int *)REALLOC( - parser, dtd->scaffIndex, parser->m_groupSize * sizeof(int)); - if (new_scaff_index == NULL) - return XML_ERROR_NO_MEMORY; - dtd->scaffIndex = new_scaff_index; + char *const new_connector = REALLOC(parser, parser->m_groupConnector, + parser->m_groupSize *= 2); + if (new_connector == NULL) { + parser->m_groupSize /= 2; + return XML_ERROR_NO_MEMORY; } + parser->m_groupConnector = new_connector; } else { - parser->m_groupConnector - = (char *)MALLOC(parser, parser->m_groupSize = 32); + parser->m_groupConnector = MALLOC(parser, parser->m_groupSize = 32); if (! parser->m_groupConnector) { parser->m_groupSize = 0; return XML_ERROR_NO_MEMORY; @@ -5586,6 +5955,21 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (myindex < 0) return XML_ERROR_NO_MEMORY; assert(dtd->scaffIndex != NULL); + if ((size_t)dtd->scaffLevel >= dtd->scaffIndexSize) { + /* Detect and prevent integer overflow */ + if (dtd->scaffIndexSize > SIZE_MAX / 2 / sizeof(int)) { + return XML_ERROR_NO_MEMORY; + } + assert(dtd->scaffIndexSize > 0); + const size_t new_size = dtd->scaffIndexSize * 2; + int *const new_scaff_index + = REALLOC(parser, dtd->scaffIndex, new_size * sizeof(int)); + if (new_scaff_index == NULL) { + return XML_ERROR_NO_MEMORY; + } + dtd->scaffIndex = new_scaff_index; + dtd->scaffIndexSize = new_size; + } dtd->scaffIndex[dtd->scaffLevel] = myindex; dtd->scaffLevel++; dtd->scaffold[myindex].type = XML_CTYPE_SEQ; @@ -5666,7 +6050,9 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, /* cannot report skipped entities in declarations */ if ((role == XML_ROLE_PARAM_ENTITY_REF) && parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 1); + afterHandler(parser); handleDefault = XML_FALSE; } break; @@ -5687,9 +6073,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, dtd->paramEntityRead = XML_FALSE; entity->open = XML_TRUE; entityTrackingOnOpen(parser, entity, __LINE__); - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) { + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) { entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; return XML_ERROR_EXTERNAL_ENTITY_HANDLING; @@ -5707,9 +6096,13 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } } #endif /* XML_DTD */ - if (! dtd->standalone && parser->m_notStandaloneHandler - && ! parser->m_notStandaloneHandler(parser->m_handlerArg)) - return XML_ERROR_NOT_STANDALONE; + if (! dtd->standalone && parser->m_notStandaloneHandler) { + beforeHandler(parser); + const int status = parser->m_notStandaloneHandler(parser->m_handlerArg); + afterHandler(parser); + if (! status) + return XML_ERROR_NOT_STANDALONE; + } break; /* Element declaration stuff */ @@ -5730,8 +6123,11 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, case XML_ROLE_CONTENT_EMPTY: if (dtd->in_eldecl) { if (parser->m_elementDeclHandler) { - XML_Content *content - = (XML_Content *)MALLOC(parser, sizeof(XML_Content)); + // NOTE: We are avoiding MALLOC(..) here to so that + // applications that are not using XML_FreeContentModel but + // plain free(..) or .free_fcn() to free the content model's + // memory are safe. + XML_Content *content = parser->m_mem.malloc_fcn(sizeof(XML_Content)); if (! content) return XML_ERROR_NO_MEMORY; content->quant = XML_CQUANT_NONE; @@ -5741,8 +6137,10 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, content->type = ((role == XML_ROLE_CONTENT_ANY) ? XML_CTYPE_ANY : XML_CTYPE_EMPTY); *eventEndPP = s; + beforeHandler(parser); parser->m_elementDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, content); + afterHandler(parser); handleDefault = XML_FALSE; } dtd->in_eldecl = XML_FALSE; @@ -5786,9 +6184,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_NO_MEMORY; name = el->name; dtd->scaffold[myindex].name = name; - nameLen = 0; - for (; name[nameLen++];) - ; + nameLen = xcslen(name) + /*null terminator*/ 1; /* Detect and prevent integer overflow */ if (nameLen > UINT_MAX - dtd->contentStringLen) { @@ -5824,8 +6220,10 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, if (! model) return XML_ERROR_NO_MEMORY; *eventEndPP = s; + beforeHandler(parser); parser->m_elementDeclHandler( parser->m_handlerArg, parser->m_declElementType->name, model); + afterHandler(parser); } dtd->in_eldecl = XML_FALSE; dtd->contentStringLen = 0; @@ -5887,7 +6285,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, *nextPtr = next; return XML_ERROR_NONE; } - /* Fall through */ + EXPAT_FALLTHROUGH; default: s = next; tok = XmlPrologTok(enc, s, end, &next); @@ -5967,7 +6365,7 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, if (parser->m_reenter) { return XML_ERROR_UNEXPECTED_STATE; // LCOV_EXCL_LINE } - /* Fall through */ + EXPAT_FALLTHROUGH; default:; parser->m_eventPtr = s = next; } @@ -5977,20 +6375,18 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, static enum XML_Error processEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl, enum EntityType type) { - OPEN_INTERNAL_ENTITY *openEntity, **openEntityList, **freeEntityList; + OPEN_INTERNAL_ENTITY *openEntity, **openEntityList; + OPEN_INTERNAL_ENTITY **const freeEntityList = &parser->m_freeEntities; switch (type) { case ENTITY_INTERNAL: parser->m_processor = internalEntityProcessor; openEntityList = &parser->m_openInternalEntities; - freeEntityList = &parser->m_freeInternalEntities; break; case ENTITY_ATTRIBUTE: openEntityList = &parser->m_openAttributeEntities; - freeEntityList = &parser->m_freeAttributeEntities; break; case ENTITY_VALUE: openEntityList = &parser->m_openValueEntities; - freeEntityList = &parser->m_freeValueEntities; break; /* default case serves merely as a safety net in case of a * wrong entityType. Therefore we exclude the following lines @@ -6008,8 +6404,7 @@ processEntity(XML_Parser parser, ENTITY *entity, XML_Bool betweenDecl, openEntity = *freeEntityList; *freeEntityList = openEntity->next; } else { - openEntity - = (OPEN_INTERNAL_ENTITY *)MALLOC(parser, sizeof(OPEN_INTERNAL_ENTITY)); + openEntity = MALLOC(parser, sizeof(OPEN_INTERNAL_ENTITY)); if (! openEntity) return XML_ERROR_NO_MEMORY; } @@ -6087,6 +6482,10 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, // process its possible inner entities (which are added to the // m_openInternalEntities during doProlog or doContent calls above) entity->hasMore = XML_FALSE; + if (! entity->is_param + && (openEntity->startTagLevel != parser->m_tagLevel)) { + return XML_ERROR_ASYNC_ENTITY; + } triggerReenter(parser); return result; } // End of entity processing, "if" block will return here @@ -6104,8 +6503,8 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, parser->m_openInternalEntities = parser->m_openInternalEntities->next; /* put openEntity back in list of free instances */ - openEntity->next = parser->m_freeInternalEntities; - parser->m_freeInternalEntities = openEntity; + openEntity->next = parser->m_freeEntities; + parser->m_freeEntities = openEntity; if (parser->m_openInternalEntities == NULL) { parser->m_processor = entity->is_param ? prologProcessor : contentProcessor; @@ -6181,8 +6580,8 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, parser->m_openAttributeEntities = parser->m_openAttributeEntities->next; /* put openEntity back in list of free instances */ - openEntity->next = parser->m_freeAttributeEntities; - parser->m_freeAttributeEntities = openEntity; + openEntity->next = parser->m_freeEntities; + parser->m_freeEntities = openEntity; } // Break if an error occurred or there is nothing left to process @@ -6235,7 +6634,6 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, return XML_ERROR_INVALID_TOKEN; case XML_TOK_CHAR_REF: { XML_Char buf[XML_ENCODE_MAX]; - int i; int n = XmlCharRefNumber(enc, ptr); if (n < 0) { if (enc == parser->m_encoding) @@ -6255,10 +6653,9 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, * XmlEncode() is never passed a value it might return an * error for. */ - for (i = 0; i < n; i++) { - if (! poolAppendChar(pool, buf[i])) - return XML_ERROR_NO_MEMORY; - } + + if (! poolAppendChars(pool, buf, n)) + return XML_ERROR_NO_MEMORY; } break; case XML_TOK_DATA_CHARS: if (! poolAppend(pool, enc, ptr, next)) @@ -6266,7 +6663,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, break; case XML_TOK_TRAILING_CR: next = ptr + enc->minBytesPerChar; - /* fall through */ + EXPAT_FALLTHROUGH; case XML_TOK_ATTRIBUTE_VALUE_S: case XML_TOK_DATA_NEWLINE: if (! isCdata && (poolLength(pool) == 0 || poolLastChar(pool) == 0x20)) @@ -6277,7 +6674,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, case XML_TOK_ENTITY_REF: { const XML_Char *name; ENTITY *entity; - char checkEntityDecl; + bool checkEntityDecl; XML_Char ch = (XML_Char)XmlPredefinedEntityName( enc, ptr + enc->minBytesPerChar, next - enc->minBytesPerChar); if (ch) { @@ -6320,8 +6717,11 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, } else if (! entity) { /* Cannot report skipped entity here - see comments on parser->m_skippedEntityHandler. - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0); + afterHandler(parser); + } */ /* Cannot call the default handler because this would be out of sync with the call to the startElementHandler. @@ -6415,7 +6815,14 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, return XML_ERROR_NO_MEMORY; } - const char *next; + const char *next = entityTextPtr; + + /* Nothing to tokenize. */ + if (entityTextPtr >= entityTextEnd) { + result = XML_ERROR_NONE; + goto endEntityValue; + } + for (;;) { next = entityTextPtr; /* XmlEntityValueTok doesn't always set the last arg */ @@ -6447,8 +6854,11 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, /* not a well-formedness error - see XML 1.0: WFC Entity Declared */ /* cannot report skipped entity here - see comments on parser->m_skippedEntityHandler - if (parser->m_skippedEntityHandler) + if (parser->m_skippedEntityHandler) { + beforeHandler(parser); parser->m_skippedEntityHandler(parser->m_handlerArg, name, 0); + afterHandler(parser); + } */ dtd->keepProcessing = dtd->standalone; goto endEntityValue; @@ -6464,9 +6874,12 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, dtd->paramEntityRead = XML_FALSE; entity->open = XML_TRUE; entityTrackingOnOpen(parser, entity, __LINE__); - if (! parser->m_externalEntityRefHandler( - parser->m_externalEntityRefHandlerArg, 0, entity->base, - entity->systemId, entity->publicId)) { + beforeHandler(parser); + const int status = parser->m_externalEntityRefHandler( + parser->m_externalEntityRefHandlerArg, 0, entity->base, + entity->systemId, entity->publicId); + afterHandler(parser); + if (! status) { entityTrackingOnClose(parser, entity, __LINE__); entity->open = XML_FALSE; result = XML_ERROR_EXTERNAL_ENTITY_HANDLING; @@ -6502,17 +6915,15 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, break; case XML_TOK_TRAILING_CR: next = entityTextPtr + enc->minBytesPerChar; - /* fall through */ + EXPAT_FALLTHROUGH; case XML_TOK_DATA_NEWLINE: - if (pool->end == pool->ptr && ! poolGrow(pool)) { + if (! poolAppendChar(pool, 0xA)) { result = XML_ERROR_NO_MEMORY; goto endEntityValue; } - *(pool->ptr)++ = 0xA; break; case XML_TOK_CHAR_REF: { XML_Char buf[XML_ENCODE_MAX]; - int i; int n = XmlCharRefNumber(enc, entityTextPtr); if (n < 0) { if (enc == parser->m_encoding) @@ -6530,12 +6941,9 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, * XmlEncode() is never passed a value it might return an * error for. */ - for (i = 0; i < n; i++) { - if (pool->end == pool->ptr && ! poolGrow(pool)) { - result = XML_ERROR_NO_MEMORY; - goto endEntityValue; - } - *(pool->ptr)++ = buf[i]; + if (! poolAppendChars(pool, buf, n)) { + result = XML_ERROR_NO_MEMORY; + goto endEntityValue; } } break; case XML_TOK_PARTIAL: @@ -6632,8 +7040,8 @@ callStoreEntityValue(XML_Parser parser, const ENCODING *enc, parser->m_openValueEntities = parser->m_openValueEntities->next; /* put openEntity back in list of free instances */ - openEntity->next = parser->m_freeValueEntities; - parser->m_freeValueEntities = openEntity; + openEntity->next = parser->m_freeEntities; + parser->m_freeEntities = openEntity; } // Break if an error occurred or there is nothing left to process @@ -6663,6 +7071,11 @@ storeSelfEntityValue(XML_Parser parser, ENTITY *entity) { return XML_ERROR_NO_MEMORY; } + /* Detect and prevent signed integer overflow */ + if ((size_t)poolLength(pool) > (size_t)INT_MAX) { + poolDiscard(pool); + return XML_ERROR_NO_MEMORY; + } entity->textPtr = poolStart(pool); entity->textLen = (int)(poolLength(pool)); poolFinish(pool); @@ -6715,7 +7128,9 @@ reportProcessingInstruction(XML_Parser parser, const ENCODING *enc, if (! data) return 0; normalizeLines(data); + beforeHandler(parser); parser->m_processingInstructionHandler(parser->m_handlerArg, target, data); + afterHandler(parser); poolClear(&parser->m_tempPool); return 1; } @@ -6735,7 +7150,9 @@ reportComment(XML_Parser parser, const ENCODING *enc, const char *start, if (! data) return 0; normalizeLines(data); + beforeHandler(parser); parser->m_commentHandler(parser->m_handlerArg, data); + afterHandler(parser); poolClear(&parser->m_tempPool); return 1; } @@ -6776,15 +7193,20 @@ reportDefault(XML_Parser parser, const ENCODING *enc, const char *s, convert_res = XmlConvert(enc, &s, end, &dataPtr, (ICHAR *)parser->m_dataBufEnd); *eventEndPP = s; + beforeHandler(parser); parser->m_defaultHandler(parser->m_handlerArg, parser->m_dataBuf, (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); + afterHandler(parser); *eventPP = s; } while ((convert_res != XML_CONVERT_COMPLETED) && (convert_res != XML_CONVERT_INPUT_INCOMPLETE)); - } else + } else { + beforeHandler(parser); parser->m_defaultHandler( parser->m_handlerArg, (const XML_Char *)s, (int)((const XML_Char *)end - (const XML_Char *)s)); + afterHandler(parser); + } } static int @@ -6794,49 +7216,35 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, if (value || isId) { /* The handling of default attributes gets messed up if we have a default which duplicates a non-default. */ - int i; - for (i = 0; i < type->nDefaultAtts; i++) - if (attId == type->defaultAtts[i].id) - return 1; + NAMED *const nameFound + = lookup(parser, &(type->defaultAttsNames), attId->name, 0); + if (nameFound) + return 1; if (isId && ! type->idAtt && ! attId->xmlns) type->idAtt = attId; } if (type->nDefaultAtts == type->allocDefaultAtts) { - if (type->allocDefaultAtts == 0) { - type->allocDefaultAtts = 8; - type->defaultAtts = (DEFAULT_ATTRIBUTE *)MALLOC( - parser, type->allocDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); - if (! type->defaultAtts) { - type->allocDefaultAtts = 0; - return 0; - } - } else { - DEFAULT_ATTRIBUTE *temp; - - /* Detect and prevent integer overflow */ - if (type->allocDefaultAtts > INT_MAX / 2) { - return 0; - } - - int count = type->allocDefaultAtts * 2; + /* Detect and prevent integer overflow */ + if (type->allocDefaultAtts > SIZE_MAX / 2) { + return 0; + } - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((unsigned)count > (size_t)(-1) / sizeof(DEFAULT_ATTRIBUTE)) { - return 0; - } -#endif + size_t count = type->allocDefaultAtts * 2; + if (count == 0) { + count = 8; + } - temp = (DEFAULT_ATTRIBUTE *)REALLOC(parser, type->defaultAtts, - (count * sizeof(DEFAULT_ATTRIBUTE))); - if (temp == NULL) - return 0; - type->allocDefaultAtts = count; - type->defaultAtts = temp; + /* Detect and prevent integer overflow. */ + if (count > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) { + return 0; } + + DEFAULT_ATTRIBUTE *const temp = REALLOC( + parser, type->defaultAtts, (count * sizeof(DEFAULT_ATTRIBUTE))); + if (temp == NULL) + return 0; + type->allocDefaultAtts = count; + type->defaultAtts = temp; } att = type->defaultAtts + type->nDefaultAtts; att->id = attId; @@ -6844,6 +7252,12 @@ defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, att->isCdata = isCdata; if (! isCdata) attId->maybeTokenized = XML_TRUE; + + NAMED *const nameAddedOrFound + = lookup(parser, &(type->defaultAttsNames), attId->name, sizeof(NAMED)); + if (! nameAddedOrFound) + return 0; + type->nDefaultAtts += 1; return 1; } @@ -6913,13 +7327,14 @@ getAttributeId(XML_Parser parser, const ENCODING *enc, const char *start, } else { int i; for (i = 0; name[i]; i++) { + /* Detect and prevent signed integer overflow */ + if (i == INT_MAX) { + return NULL; + } /* attributes without prefix are *not* in the default namespace */ if (name[i] == XML_T(ASCII_COLON)) { - int j; - for (j = 0; j < i; j++) { - if (! poolAppendChar(&dtd->pool, name[j])) - return NULL; - } + if (! poolAppendChars(&dtd->pool, name, i)) + return NULL; if (! poolAppendChar(&dtd->pool, XML_T('\0'))) return NULL; id->prefix = (PREFIX *)lookup(parser, &dtd->prefixes, @@ -6947,46 +7362,39 @@ getContext(XML_Parser parser) { XML_Bool needSep = XML_FALSE; if (dtd->defaultPrefix.binding) { - int i; - int len; if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS))) return NULL; - len = dtd->defaultPrefix.binding->uriLen; + size_t len = dtd->defaultPrefix.binding->uriLen; if (parser->m_namespaceSeparator) len--; - for (i = 0; i < len; i++) { - if (! poolAppendChar(&parser->m_tempPool, - dtd->defaultPrefix.binding->uri[i])) { - /* Because of memory caching, I don't believe this line can be - * executed. - * - * This is part of a loop copying the default prefix binding - * URI into the parser's temporary string pool. Previously, - * that URI was copied into the same string pool, with a - * terminating NUL character, as part of setContext(). When - * the pool was cleared, that leaves a block definitely big - * enough to hold the URI on the free block list of the pool. - * The URI copy in getContext() therefore cannot run out of - * memory. - * - * If the pool is used between the setContext() and - * getContext() calls, the worst it can do is leave a bigger - * block on the front of the free list. Given that this is - * all somewhat inobvious and program logic can be changed, we - * don't delete the line but we do exclude it from the test - * coverage statistics. - */ - return NULL; /* LCOV_EXCL_LINE */ - } + if (! poolAppendChars(&parser->m_tempPool, dtd->defaultPrefix.binding->uri, + len)) { + /* Because of memory caching, I don't believe this line can be + * executed. + * + * This is part of a loop copying the default prefix binding + * URI into the parser's temporary string pool. Previously, + * that URI was copied into the same string pool, with a + * terminating NUL character, as part of setContext(). When + * the pool was cleared, that leaves a block definitely big + * enough to hold the URI on the free block list of the pool. + * The URI copy in getContext() therefore cannot run out of + * memory. + * + * If the pool is used between the setContext() and + * getContext() calls, the worst it can do is leave a bigger + * block on the front of the free list. Given that this is + * all somewhat inobvious and program logic can be changed, we + * don't delete the line but we do exclude it from the test + * coverage statistics. + */ + return NULL; /* LCOV_EXCL_LINE */ } needSep = XML_TRUE; } hashTableIterInit(&iter, &(dtd->prefixes)); for (;;) { - int i; - int len; - const XML_Char *s; PREFIX *prefix = (PREFIX *)hashTableIterNext(&iter); if (! prefix) break; @@ -7001,23 +7409,21 @@ getContext(XML_Parser parser) { } if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP)) return NULL; - for (s = prefix->name; *s; s++) - if (! poolAppendChar(&parser->m_tempPool, *s)) - return NULL; + if (! poolAppendChars(&parser->m_tempPool, prefix->name, + xcslen(prefix->name))) + return NULL; if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS))) return NULL; - len = prefix->binding->uriLen; + size_t len = prefix->binding->uriLen; if (parser->m_namespaceSeparator) len--; - for (i = 0; i < len; i++) - if (! poolAppendChar(&parser->m_tempPool, prefix->binding->uri[i])) - return NULL; + if (! poolAppendChars(&parser->m_tempPool, prefix->binding->uri, len)) + return NULL; needSep = XML_TRUE; } hashTableIterInit(&iter, &(dtd->generalEntities)); for (;;) { - const XML_Char *s; ENTITY *e = (ENTITY *)hashTableIterNext(&iter); if (! e) break; @@ -7025,9 +7431,8 @@ getContext(XML_Parser parser) { continue; if (needSep && ! poolAppendChar(&parser->m_tempPool, CONTEXT_SEP)) return NULL; - for (s = e->name; *s; s++) - if (! poolAppendChar(&parser->m_tempPool, *s)) - return 0; + if (! poolAppendChars(&parser->m_tempPool, e->name, xcslen(e->name))) + return NULL; needSep = XML_TRUE; } @@ -7065,16 +7470,24 @@ setContext(XML_Parser parser, const XML_Char *context) { else { if (! poolAppendChar(&parser->m_tempPool, XML_T('\0'))) return XML_FALSE; - prefix - = (PREFIX *)lookup(parser, &dtd->prefixes, - poolStart(&parser->m_tempPool), sizeof(PREFIX)); - if (! prefix) + const XML_Char *const prefixName = poolCopyStringNoFinish( + &dtd->pool, poolStart(&parser->m_tempPool)); + if (! prefixName) { return XML_FALSE; - if (prefix->name == poolStart(&parser->m_tempPool)) { - prefix->name = poolCopyString(&dtd->pool, prefix->name); - if (! prefix->name) - return XML_FALSE; } + + prefix = (PREFIX *)lookup(parser, &dtd->prefixes, prefixName, + sizeof(PREFIX)); + + const bool prefixNameUsed = prefix && prefix->name == prefixName; + if (prefixNameUsed) + poolFinish(&dtd->pool); + else + poolDiscard(&dtd->pool); + + if (! prefix) + return XML_FALSE; + poolDiscard(&parser->m_tempPool); } for (context = s + 1; *context != CONTEXT_SEP && *context != XML_T('\0'); @@ -7122,25 +7535,26 @@ normalizePublicId(XML_Char *publicId) { } static DTD * -dtdCreate(const XML_Memory_Handling_Suite *ms) { - DTD *p = ms->malloc_fcn(sizeof(DTD)); +dtdCreate(XML_Parser parser) { + DTD *p = MALLOC(parser, sizeof(DTD)); if (p == NULL) return p; - poolInit(&(p->pool), ms); - poolInit(&(p->entityValuePool), ms); - hashTableInit(&(p->generalEntities), ms); - hashTableInit(&(p->elementTypes), ms); - hashTableInit(&(p->attributeIds), ms); - hashTableInit(&(p->prefixes), ms); + poolInit(&(p->pool), parser); + poolInit(&(p->entityValuePool), parser); + hashTableInit(&(p->generalEntities), parser); + hashTableInit(&(p->elementTypes), parser); + hashTableInit(&(p->attributeIds), parser); + hashTableInit(&(p->prefixes), parser); #ifdef XML_DTD p->paramEntityRead = XML_FALSE; - hashTableInit(&(p->paramEntities), ms); + hashTableInit(&(p->paramEntities), parser); #endif /* XML_DTD */ p->defaultPrefix.name = NULL; p->defaultPrefix.binding = NULL; p->in_eldecl = XML_FALSE; p->scaffIndex = NULL; + p->scaffIndexSize = 0; p->scaffold = NULL; p->scaffLevel = 0; p->scaffSize = 0; @@ -7154,15 +7568,15 @@ dtdCreate(const XML_Memory_Handling_Suite *ms) { } static void -dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms) { +dtdReset(DTD *p, XML_Parser parser) { HASH_TABLE_ITER iter; hashTableIterInit(&iter, &(p->elementTypes)); for (;;) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; - if (e->allocDefaultAtts != 0) - ms->free_fcn(e->defaultAtts); + hashTableDestroy(&(e->defaultAttsNames)); + FREE(parser, e->defaultAtts); } hashTableClear(&(p->generalEntities)); #ifdef XML_DTD @@ -7179,9 +7593,10 @@ dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms) { p->in_eldecl = XML_FALSE; - ms->free_fcn(p->scaffIndex); + FREE(parser, p->scaffIndex); p->scaffIndex = NULL; - ms->free_fcn(p->scaffold); + p->scaffIndexSize = 0; + FREE(parser, p->scaffold); p->scaffold = NULL; p->scaffLevel = 0; @@ -7195,15 +7610,15 @@ dtdReset(DTD *p, const XML_Memory_Handling_Suite *ms) { } static void -dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms) { +dtdDestroy(DTD *p, XML_Bool isDocEntity, XML_Parser parser) { HASH_TABLE_ITER iter; hashTableIterInit(&iter, &(p->elementTypes)); for (;;) { ELEMENT_TYPE *e = (ELEMENT_TYPE *)hashTableIterNext(&iter); if (! e) break; - if (e->allocDefaultAtts != 0) - ms->free_fcn(e->defaultAtts); + hashTableDestroy(&(e->defaultAttsNames)); + FREE(parser, e->defaultAtts); } hashTableDestroy(&(p->generalEntities)); #ifdef XML_DTD @@ -7215,10 +7630,10 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms) { poolDestroy(&(p->pool)); poolDestroy(&(p->entityValuePool)); if (isDocEntity) { - ms->free_fcn(p->scaffIndex); - ms->free_fcn(p->scaffold); + FREE(parser, p->scaffIndex); + FREE(parser, p->scaffold); } - ms->free_fcn(p); + FREE(parser, p); } /* Do a deep copy of the DTD. Return 0 for out of memory, non-zero otherwise. @@ -7226,7 +7641,7 @@ dtdDestroy(DTD *p, XML_Bool isDocEntity, const XML_Memory_Handling_Suite *ms) { */ static int dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, - const XML_Memory_Handling_Suite *ms) { + XML_Parser parser) { HASH_TABLE_ITER iter; /* Copy the prefix table. */ @@ -7282,7 +7697,6 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, hashTableIterInit(&iter, &(oldDtd->elementTypes)); for (;;) { - int i; ELEMENT_TYPE *newE; const XML_Char *name; const ELEMENT_TYPE *oldE = (ELEMENT_TYPE *)hashTableIterNext(&iter); @@ -7295,19 +7709,17 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, sizeof(ELEMENT_TYPE)); if (! newE) return 0; + + if (! newE->defaultAttsNames.parser) + hashTableInit(&(newE->defaultAttsNames), parser); + if (oldE->nDefaultAtts) { - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if ((size_t)oldE->nDefaultAtts - > ((size_t)(-1) / sizeof(DEFAULT_ATTRIBUTE))) { + /* Detect and prevent integer overflow. */ + if (oldE->nDefaultAtts > SIZE_MAX / sizeof(DEFAULT_ATTRIBUTE)) { return 0; } -#endif newE->defaultAtts - = ms->malloc_fcn(oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); + = MALLOC(parser, oldE->nDefaultAtts * sizeof(DEFAULT_ATTRIBUTE)); if (! newE->defaultAtts) { return 0; } @@ -7319,9 +7731,10 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, if (oldE->prefix) newE->prefix = (PREFIX *)lookup(oldParser, &(newDtd->prefixes), oldE->prefix->name, 0); - for (i = 0; i < newE->nDefaultAtts; i++) { + for (size_t i = 0; i < newE->nDefaultAtts; i++) { + const XML_Char *const attributeName = oldE->defaultAtts[i].id->name; newE->defaultAtts[i].id = (ATTRIBUTE_ID *)lookup( - oldParser, &(newDtd->attributeIds), oldE->defaultAtts[i].id->name, 0); + oldParser, &(newDtd->attributeIds), attributeName, 0); newE->defaultAtts[i].isCdata = oldE->defaultAtts[i].isCdata; if (oldE->defaultAtts[i].value) { newE->defaultAtts[i].value @@ -7330,6 +7743,12 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, return 0; } else newE->defaultAtts[i].value = NULL; + + NAMED *const nameAddedOrFound = lookup(parser, &(newE->defaultAttsNames), + attributeName, sizeof(NAMED)); + if (! nameAddedOrFound) { + return 0; + } } } @@ -7356,6 +7775,7 @@ dtdCopy(XML_Parser oldParser, DTD *newDtd, const DTD *oldDtd, newDtd->scaffSize = oldDtd->scaffSize; newDtd->scaffLevel = oldDtd->scaffLevel; newDtd->scaffIndex = oldDtd->scaffIndex; + newDtd->scaffIndexSize = oldDtd->scaffIndexSize; return 1; } /* End dtdCopy */ @@ -7427,24 +7847,31 @@ copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, static XML_Bool FASTCALL keyeq(KEY s1, KEY s2) { +#ifdef XML_UNICODE +# ifdef XML_UNICODE_WCHAR_T + return (wcscmp(s1, s2) == 0) ? XML_TRUE : XML_FALSE; +# else for (; *s1 == *s2; s1++, s2++) if (*s1 == 0) return XML_TRUE; return XML_FALSE; +# endif +#else + return (strcmp(s1, s2) == 0) ? XML_TRUE : XML_FALSE; +#endif } static size_t keylen(KEY s) { - size_t len = 0; - for (; *s; s++, len++) - ; - return len; + return xcslen(s); } static void copy_salt_to_sipkey(XML_Parser parser, struct sipkey *key) { - key->k[0] = 0; - key->k[1] = get_hash_secret_salt(parser); + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(! rootParser->m_parentParser); + + *key = rootParser->m_hash_secret_salt_128; } static unsigned long FASTCALL @@ -7469,7 +7896,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { /* table->size is a power of 2 */ table->size = (size_t)1 << INIT_POWER; tsize = table->size * sizeof(NAMED *); - table->v = table->mem->malloc_fcn(tsize); + table->v = MALLOC(table->parser, tsize); if (! table->v) { table->size = 0; return NULL; @@ -7504,12 +7931,12 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { unsigned long newMask = (unsigned long)newSize - 1; /* Detect and prevent integer overflow */ - if (newSize > (size_t)(-1) / sizeof(NAMED *)) { + if (newSize > SIZE_MAX / sizeof(NAMED *)) { return NULL; } size_t tsize = newSize * sizeof(NAMED *); - NAMED **newV = table->mem->malloc_fcn(tsize); + NAMED **newV = MALLOC(table->parser, tsize); if (! newV) return NULL; memset(newV, 0, tsize); @@ -7525,7 +7952,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { } newV[j] = table->v[i]; } - table->mem->free_fcn(table->v); + FREE(table->parser, table->v); table->v = newV; table->power = newPower; table->size = newSize; @@ -7538,7 +7965,7 @@ lookup(XML_Parser parser, HASH_TABLE *table, KEY name, size_t createSize) { } } } - table->v[i] = table->mem->malloc_fcn(createSize); + table->v[i] = MALLOC(table->parser, createSize); if (! table->v[i]) return NULL; memset(table->v[i], 0, createSize); @@ -7551,7 +7978,7 @@ static void FASTCALL hashTableClear(HASH_TABLE *table) { size_t i; for (i = 0; i < table->size; i++) { - table->mem->free_fcn(table->v[i]); + FREE(table->parser, table->v[i]); table->v[i] = NULL; } table->used = 0; @@ -7561,17 +7988,17 @@ static void FASTCALL hashTableDestroy(HASH_TABLE *table) { size_t i; for (i = 0; i < table->size; i++) - table->mem->free_fcn(table->v[i]); - table->mem->free_fcn(table->v); + FREE(table->parser, table->v[i]); + FREE(table->parser, table->v); } static void FASTCALL -hashTableInit(HASH_TABLE *p, const XML_Memory_Handling_Suite *ms) { +hashTableInit(HASH_TABLE *p, XML_Parser parser) { p->power = 0; p->size = 0; p->used = 0; p->v = NULL; - p->mem = ms; + p->parser = parser; } static void FASTCALL @@ -7591,13 +8018,13 @@ hashTableIterNext(HASH_TABLE_ITER *iter) { } static void FASTCALL -poolInit(STRING_POOL *pool, const XML_Memory_Handling_Suite *ms) { +poolInit(STRING_POOL *pool, XML_Parser parser) { pool->blocks = NULL; pool->freeBlocks = NULL; pool->start = NULL; pool->ptr = NULL; pool->end = NULL; - pool->mem = ms; + pool->parser = parser; } static void FASTCALL @@ -7624,13 +8051,13 @@ poolDestroy(STRING_POOL *pool) { BLOCK *p = pool->blocks; while (p) { BLOCK *tem = p->next; - pool->mem->free_fcn(p); + FREE(pool->parser, p); p = tem; } p = pool->freeBlocks; while (p) { BLOCK *tem = p->next; - pool->mem->free_fcn(p); + FREE(pool->parser, p); p = tem; } } @@ -7654,15 +8081,30 @@ poolAppend(STRING_POOL *pool, const ENCODING *enc, const char *ptr, static const XML_Char *FASTCALL poolCopyString(STRING_POOL *pool, const XML_Char *s) { - do { - if (! poolAppendChar(pool, *s)) - return NULL; - } while (*s++); + if (! poolAppendChars(pool, s, xcslen(s) + /*null terminator*/ 1)) + return NULL; s = pool->start; poolFinish(pool); return s; } +// A version of `poolCopyString` that does not call `poolFinish` +// and reverts any partial advancement upon failure. +static const XML_Char *FASTCALL +poolCopyStringNoFinish(STRING_POOL *pool, const XML_Char *s) { + const XML_Char *const original = s; + do { + if (! poolAppendChar(pool, *s)) { + // Revert any previously successful advancement + const ptrdiff_t advancedBy = s - original; + if (advancedBy > 0) + pool->ptr -= advancedBy; + return NULL; + } + } while (*s++); + return pool->start; +} + static const XML_Char * poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { if (! pool->ptr && ! poolGrow(pool)) { @@ -7679,10 +8121,8 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { */ return NULL; /* LCOV_EXCL_LINE */ } - for (; n > 0; --n, s++) { - if (! poolAppendChar(pool, *s)) - return NULL; - } + if (n > 0 && ! poolAppendChars(pool, s, n)) + return NULL; s = pool->start; poolFinish(pool); return s; @@ -7690,11 +8130,8 @@ poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { static const XML_Char *FASTCALL poolAppendString(STRING_POOL *pool, const XML_Char *s) { - while (*s) { - if (! poolAppendChar(pool, *s)) - return NULL; - s++; - } + if (! poolAppendChars(pool, s, xcslen(s))) + return NULL; return pool->start; } @@ -7703,9 +8140,8 @@ poolStoreString(STRING_POOL *pool, const ENCODING *enc, const char *ptr, const char *end) { if (! poolAppend(pool, enc, ptr, end)) return NULL; - if (pool->ptr == pool->end && ! poolGrow(pool)) + if (! poolAppendChar(pool, 0)) return NULL; - *(pool->ptr)++ = 0; return pool->start; } @@ -7740,7 +8176,7 @@ poolBytesToAllocateFor(int blockSize) { static XML_Bool FASTCALL poolGrow(STRING_POOL *pool) { if (pool->freeBlocks) { - if (pool->start == 0) { + if (pool->start == NULL) { pool->blocks = pool->freeBlocks; pool->freeBlocks = pool->freeBlocks->next; pool->blocks->next = NULL; @@ -7785,8 +8221,7 @@ poolGrow(STRING_POOL *pool) { if (bytesToAllocate == 0) return XML_FALSE; - temp = (BLOCK *)pool->mem->realloc_fcn(pool->blocks, - (unsigned)bytesToAllocate); + temp = REALLOC(pool->parser, pool->blocks, bytesToAllocate); if (temp == NULL) return XML_FALSE; pool->blocks = temp; @@ -7826,7 +8261,7 @@ poolGrow(STRING_POOL *pool) { if (bytesToAllocate == 0) return XML_FALSE; - tem = pool->mem->malloc_fcn(bytesToAllocate); + tem = MALLOC(pool->parser, bytesToAllocate); if (! tem) return XML_FALSE; tem->size = blockSize; @@ -7841,6 +8276,19 @@ poolGrow(STRING_POOL *pool) { return XML_TRUE; } +static bool FASTCALL +poolGrowUntil(STRING_POOL *pool, size_t needed) { + for (;;) { + const size_t available = pool->end - pool->ptr; + if (available >= needed) { + return true; + } + if (! poolGrow(pool)) { + return false; + } + } +} + static int FASTCALL nextScaffoldPart(XML_Parser parser) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ @@ -7848,21 +8296,22 @@ nextScaffoldPart(XML_Parser parser) { int next; if (! dtd->scaffIndex) { - /* Detect and prevent integer overflow. - * The preprocessor guard addresses the "always false" warning - * from -Wtype-limits on platforms where - * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ -#if UINT_MAX >= SIZE_MAX - if (parser->m_groupSize > ((size_t)(-1) / sizeof(int))) { + /* Detect and prevent integer overflow. */ + if (parser->m_groupSize > SIZE_MAX / sizeof(int)) { return -1; } -#endif - dtd->scaffIndex = (int *)MALLOC(parser, parser->m_groupSize * sizeof(int)); + dtd->scaffIndex = MALLOC(parser, parser->m_groupSize * sizeof(int)); if (! dtd->scaffIndex) return -1; + dtd->scaffIndexSize = parser->m_groupSize; dtd->scaffIndex[0] = 0; } + // Will casting to int be safe further down? + if (dtd->scaffCount > INT_MAX) { + return -1; + } + if (dtd->scaffCount >= dtd->scaffSize) { CONTENT_SCAFFOLD *temp; if (dtd->scaffold) { @@ -7875,26 +8324,25 @@ nextScaffoldPart(XML_Parser parser) { * from -Wtype-limits on platforms where * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ #if UINT_MAX >= SIZE_MAX - if (dtd->scaffSize > (size_t)(-1) / 2u / sizeof(CONTENT_SCAFFOLD)) { + if (dtd->scaffSize > SIZE_MAX / 2u / sizeof(CONTENT_SCAFFOLD)) { return -1; } #endif - temp = (CONTENT_SCAFFOLD *)REALLOC( - parser, dtd->scaffold, dtd->scaffSize * 2 * sizeof(CONTENT_SCAFFOLD)); + temp = REALLOC(parser, dtd->scaffold, + dtd->scaffSize * 2 * sizeof(CONTENT_SCAFFOLD)); if (temp == NULL) return -1; dtd->scaffSize *= 2; } else { - temp = (CONTENT_SCAFFOLD *)MALLOC(parser, INIT_SCAFFOLD_ELEMENTS - * sizeof(CONTENT_SCAFFOLD)); + temp = MALLOC(parser, INIT_SCAFFOLD_ELEMENTS * sizeof(CONTENT_SCAFFOLD)); if (temp == NULL) return -1; dtd->scaffSize = INIT_SCAFFOLD_ELEMENTS; } dtd->scaffold = temp; } - next = dtd->scaffCount++; + next = (int)dtd->scaffCount++; me = &dtd->scaffold[next]; if (dtd->scaffLevel) { CONTENT_SCAFFOLD *parent @@ -7926,22 +8374,25 @@ build_model(XML_Parser parser) { * from -Wtype-limits on platforms where * sizeof(unsigned int) < sizeof(size_t), e.g. on x86_64. */ #if UINT_MAX >= SIZE_MAX - if (dtd->scaffCount > (size_t)(-1) / sizeof(XML_Content)) { + if (dtd->scaffCount > SIZE_MAX / sizeof(XML_Content)) { return NULL; } - if (dtd->contentStringLen > (size_t)(-1) / sizeof(XML_Char)) { + if (dtd->contentStringLen > SIZE_MAX / sizeof(XML_Char)) { return NULL; } #endif if (dtd->scaffCount * sizeof(XML_Content) - > (size_t)(-1) - dtd->contentStringLen * sizeof(XML_Char)) { + > SIZE_MAX - dtd->contentStringLen * sizeof(XML_Char)) { return NULL; } const size_t allocsize = (dtd->scaffCount * sizeof(XML_Content) + (dtd->contentStringLen * sizeof(XML_Char))); - ret = (XML_Content *)MALLOC(parser, allocsize); + // NOTE: We are avoiding MALLOC(..) here to so that + // applications that are not using XML_FreeContentModel but plain + // free(..) or .free_fcn() to free the content model's memory are safe. + ret = parser->m_mem.malloc_fcn(allocsize); if (! ret) return NULL; @@ -8013,12 +8464,21 @@ build_model(XML_Parser parser) { const XML_Char *src; dest->name = str; src = dtd->scaffold[src_node].name; - for (;;) { - *str++ = *src; - if (! *src) - break; - src++; + + const size_t nameLen = xcslen(src) + /* null terminator*/ 1; + + // Detect and prevent integer overflow + if (nameLen > SIZE_MAX / sizeof(XML_Char)) { + // NOTE: We are avoiding FREE(..) here because the model + // is not being allocated with MALLOC(..) but with plain + // .malloc_fcn(..). + parser->m_mem.free_fcn(ret); + return NULL; } + + memcpy(str, src, nameLen * sizeof(XML_Char)); + str += nameLen; + dest->numchildren = 0; dest->children = NULL; } else { @@ -8051,6 +8511,8 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, sizeof(ELEMENT_TYPE)); if (! ret) return NULL; + if (! ret->defaultAttsNames.parser) + hashTableInit(&(ret->defaultAttsNames), getRootParserOf(parser, NULL)); if (ret->name != name) poolDiscard(&dtd->pool); else { @@ -8062,23 +8524,25 @@ getElementType(XML_Parser parser, const ENCODING *enc, const char *ptr, } static XML_Char * -copyString(const XML_Char *s, const XML_Memory_Handling_Suite *memsuite) { - size_t charsRequired = 0; - XML_Char *result; - +copyString(const XML_Char *s, XML_Parser parser) { /* First determine how long the string is */ - while (s[charsRequired] != 0) { - charsRequired++; - } - /* Include the terminator */ - charsRequired++; + const size_t charsRequired = xcslen(s) + /*null terminator*/ 1; + + /* Detect and prevent integer overflow */ + if (charsRequired > SIZE_MAX / sizeof(XML_Char)) + return NULL; + + const size_t bytesRequired = charsRequired * sizeof(XML_Char); /* Now allocate space for the copy */ - result = memsuite->malloc_fcn(charsRequired * sizeof(XML_Char)); + XML_Char *const result = MALLOC(parser, bytesRequired); + if (result == NULL) return NULL; + /* Copy the original into place */ - memcpy(result, s, charsRequired * sizeof(XML_Char)); + memcpy(result, s, bytesRequired); + return result; } @@ -8093,10 +8557,10 @@ accountingGetCurrentAmplification(XML_Parser rootParser) { + rootParser->m_accounting.countBytesIndirect; const float amplificationFactor = rootParser->m_accounting.countBytesDirect - ? (countBytesOutput + ? ((float)countBytesOutput / (float)(rootParser->m_accounting.countBytesDirect)) - : ((lenOfShortestInclude - + rootParser->m_accounting.countBytesIndirect) + : ((float)(lenOfShortestInclude + + rootParser->m_accounting.countBytesIndirect) / (float)lenOfShortestInclude); assert(! rootParser->m_parentParser); return amplificationFactor; @@ -8245,15 +8709,25 @@ entityTrackingReportStats(XML_Parser rootParser, ENTITY *entity, const char *const entityName = entity->name; # endif + const bool limitingWanted = rootParser->m_entity_stats.debugLevel < 2; + const int maxLimitedDepth = 10; // somewhat arbitrary + const int candidateIndentDepth + = (int)rootParser->m_entity_stats.currentDepth - 1; + const bool limitingNeeded + = limitingWanted && (candidateIndentDepth > maxLimitedDepth); + const char *const ellipisOrEmpty = limitingNeeded ? " [..] " : ""; + const int indentDepth + = limitingNeeded ? (maxLimitedDepth - /* make space for ellipis */ 2) + : candidateIndentDepth; + fprintf( stderr, - "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s; %s length %d (xmlparse.c:%d)\n", + "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s%s; %s length %d (xmlparse.c:%d)\n", (void *)rootParser, rootParser->m_entity_stats.countEverOpened, rootParser->m_entity_stats.currentDepth, - rootParser->m_entity_stats.maximumDepthSeen, - ((int)rootParser->m_entity_stats.currentDepth - 1) * 2, "", - entity->is_param ? "%" : "&", entityName, action, entity->textLen, - sourceLine); + rootParser->m_entity_stats.maximumDepthSeen, indentDepth * 2, "", + ellipisOrEmpty, entity->is_param ? "%" : "&", entityName, action, + entity->textLen, sourceLine); } static void @@ -8280,6 +8754,8 @@ entityTrackingOnClose(XML_Parser originParser, ENTITY *entity, int sourceLine) { rootParser->m_entity_stats.currentDepth--; } +#endif /* XML_GE == 1 */ + static XML_Parser getRootParserOf(XML_Parser parser, unsigned int *outLevelDiff) { XML_Parser rootParser = parser; @@ -8295,6 +8771,8 @@ getRootParserOf(XML_Parser parser, unsigned int *outLevelDiff) { return rootParser; } +#if XML_GE == 1 + const char * unsignedCharToPrintable(unsigned char c) { switch (c) { diff --git a/Modules/expat/xmlrole.c b/Modules/expat/xmlrole.c index 2c48bf408679538..d56bee82dd2d13f 100644 --- a/Modules/expat/xmlrole.c +++ b/Modules/expat/xmlrole.c @@ -16,6 +16,7 @@ Copyright (c) 2017 Rhodri James <rhodri@wildebeest.org.uk> Copyright (c) 2019 David Loffredo <loffredo@steptools.com> Copyright (c) 2021 Donghee Na <donghee.na@python.org> + Copyright (c) 2025 Alfonso Gregory <gfunni234@gmail.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -46,7 +47,6 @@ # include "winconfig.h" #endif -#include "expat_external.h" #include "internal.h" #include "xmlrole.h" #include "ascii.h" diff --git a/Modules/expat/xmlrole.h b/Modules/expat/xmlrole.h index a7904274c91d4ec..9d0d4ff11b7f982 100644 --- a/Modules/expat/xmlrole.h +++ b/Modules/expat/xmlrole.h @@ -10,7 +10,7 @@ Copyright (c) 2000 Clark Cooper <coopercc@users.sourceforge.net> Copyright (c) 2002 Karl Waclawek <karl@waclawek.net> Copyright (c) 2002 Fred L. Drake, Jr. <fdrake@users.sourceforge.net> - Copyright (c) 2017-2024 Sebastian Pipping <sebastian@pipping.org> + Copyright (c) 2017-2025 Sebastian Pipping <sebastian@pipping.org> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -34,19 +34,13 @@ */ #ifndef XmlRole_INCLUDED -#define XmlRole_INCLUDED 1 +# define XmlRole_INCLUDED 1 -#ifdef __VMS -/* 0 1 2 3 0 1 2 3 - 1234567890123456789012345678901 1234567890123456789012345678901 */ -# define XmlPrologStateInitExternalEntity XmlPrologStateInitExternalEnt -#endif +# include "xmltok.h" -#include "xmltok.h" - -#ifdef __cplusplus +# ifdef __cplusplus extern "C" { -#endif +# endif enum { XML_ROLE_ERROR = -1, @@ -107,11 +101,11 @@ enum { XML_ROLE_CONTENT_ELEMENT_PLUS, XML_ROLE_PI, XML_ROLE_COMMENT, -#ifdef XML_DTD +# ifdef XML_DTD XML_ROLE_TEXT_DECL, XML_ROLE_IGNORE_SECT, XML_ROLE_INNER_PARAM_ENTITY_REF, -#endif /* XML_DTD */ +# endif /* XML_DTD */ XML_ROLE_PARAM_ENTITY_REF }; @@ -120,23 +114,23 @@ typedef struct prolog_state { const char *end, const ENCODING *enc); unsigned level; int role_none; -#ifdef XML_DTD +# ifdef XML_DTD unsigned includeLevel; int documentEntity; int inEntityValue; -#endif /* XML_DTD */ +# endif /* XML_DTD */ } PROLOG_STATE; void XmlPrologStateInit(PROLOG_STATE *state); -#ifdef XML_DTD +# ifdef XML_DTD void XmlPrologStateInitExternalEntity(PROLOG_STATE *state); -#endif /* XML_DTD */ +# endif /* XML_DTD */ -#define XmlTokenRole(state, tok, ptr, end, enc) \ - (((state)->handler)(state, tok, ptr, end, enc)) +# define XmlTokenRole(state, tok, ptr, end, enc) \ + (((state)->handler)(state, tok, ptr, end, enc)) -#ifdef __cplusplus +# ifdef __cplusplus } -#endif +# endif #endif /* not XmlRole_INCLUDED */ diff --git a/Modules/expat/xmltok.c b/Modules/expat/xmltok.c index 29a66d72ceea5e5..387c6e44a25f80c 100644 --- a/Modules/expat/xmltok.c +++ b/Modules/expat/xmltok.c @@ -24,6 +24,8 @@ Copyright (c) 2022 Martin Ettl <ettl.martin78@googlemail.com> Copyright (c) 2022 Sean McBride <sean@rogue-research.com> Copyright (c) 2023 Hanno Böck <hanno@gentoo.org> + Copyright (c) 2025 Alfonso Gregory <gfunni234@gmail.com> + Copyright (c) 2026 Nick Begg <nick@stunttruck.net> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -56,8 +58,8 @@ # include "winconfig.h" #endif -#include "expat_external.h" #include "internal.h" +#include "fallthrough.h" #include "xmltok.h" #include "nametab.h" @@ -641,7 +643,7 @@ unicode_byte_type(char hi, char lo) { *(*toP)++ = lo; \ break; \ } \ - /* fall through */ \ + EXPAT_FALLTHROUGH; \ case 0x1: \ case 0x2: \ case 0x3: \ @@ -1398,7 +1400,7 @@ unknown_toUtf16(const ENCODING *enc, const char **fromP, const char *fromLim, } ENCODING * -XmlInitUnknownEncoding(void *mem, int *table, CONVERTER convert, +XmlInitUnknownEncoding(void *mem, const int *table, CONVERTER convert, void *userData) { int i; struct unknown_encoding *e = (struct unknown_encoding *)mem; @@ -1557,7 +1559,7 @@ initScan(const ENCODING *const *encodingTable, const INIT_ENCODING *enc, case 0xEF: /* possibly first byte of UTF-8 BOM */ if (INIT_ENC_INDEX(enc) == ISO_8859_1_ENC && state == XML_CONTENT_STATE) break; - /* fall through */ + EXPAT_FALLTHROUGH; case 0x00: case 0x3C: return XML_TOK_PARTIAL; @@ -1661,7 +1663,7 @@ initScan(const ENCODING *const *encodingTable, const INIT_ENCODING *enc, # undef ns ENCODING * -XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert, +XmlInitUnknownEncodingNS(void *mem, const int *table, CONVERTER convert, void *userData) { ENCODING *enc = XmlInitUnknownEncoding(mem, table, convert, userData); if (enc) diff --git a/Modules/expat/xmltok.h b/Modules/expat/xmltok.h index c51fce1ec1518be..79a9fb76871f10b 100644 --- a/Modules/expat/xmltok.h +++ b/Modules/expat/xmltok.h @@ -35,113 +35,113 @@ */ #ifndef XmlTok_INCLUDED -#define XmlTok_INCLUDED 1 +# define XmlTok_INCLUDED 1 -#ifdef __cplusplus +# ifdef __cplusplus extern "C" { -#endif +# endif /* The following token may be returned by XmlContentTok */ -#define XML_TOK_TRAILING_RSQB \ - -5 /* ] or ]] at the end of the scan; might be \ - start of illegal ]]> sequence */ +# define XML_TOK_TRAILING_RSQB \ + -5 /* ] or ]] at the end of the scan; might be \ + start of illegal ]]> sequence */ /* The following tokens may be returned by both XmlPrologTok and XmlContentTok. */ -#define XML_TOK_NONE -4 /* The string to be scanned is empty */ -#define XML_TOK_TRAILING_CR \ - -3 /* A CR at the end of the scan; \ - might be part of CRLF sequence */ -#define XML_TOK_PARTIAL_CHAR -2 /* only part of a multibyte sequence */ -#define XML_TOK_PARTIAL -1 /* only part of a token */ -#define XML_TOK_INVALID 0 +# define XML_TOK_NONE -4 /* The string to be scanned is empty */ +# define XML_TOK_TRAILING_CR \ + -3 /* A CR at the end of the scan; \ + might be part of CRLF sequence */ +# define XML_TOK_PARTIAL_CHAR -2 /* only part of a multibyte sequence */ +# define XML_TOK_PARTIAL -1 /* only part of a token */ +# define XML_TOK_INVALID 0 /* The following tokens are returned by XmlContentTok; some are also returned by XmlAttributeValueTok, XmlEntityTok, XmlCdataSectionTok. */ -#define XML_TOK_START_TAG_WITH_ATTS 1 -#define XML_TOK_START_TAG_NO_ATTS 2 -#define XML_TOK_EMPTY_ELEMENT_WITH_ATTS 3 /* empty element tag <e/> */ -#define XML_TOK_EMPTY_ELEMENT_NO_ATTS 4 -#define XML_TOK_END_TAG 5 -#define XML_TOK_DATA_CHARS 6 -#define XML_TOK_DATA_NEWLINE 7 -#define XML_TOK_CDATA_SECT_OPEN 8 -#define XML_TOK_ENTITY_REF 9 -#define XML_TOK_CHAR_REF 10 /* numeric character reference */ +# define XML_TOK_START_TAG_WITH_ATTS 1 +# define XML_TOK_START_TAG_NO_ATTS 2 +# define XML_TOK_EMPTY_ELEMENT_WITH_ATTS 3 /* empty element tag <e/> */ +# define XML_TOK_EMPTY_ELEMENT_NO_ATTS 4 +# define XML_TOK_END_TAG 5 +# define XML_TOK_DATA_CHARS 6 +# define XML_TOK_DATA_NEWLINE 7 +# define XML_TOK_CDATA_SECT_OPEN 8 +# define XML_TOK_ENTITY_REF 9 +# define XML_TOK_CHAR_REF 10 /* numeric character reference */ /* The following tokens may be returned by both XmlPrologTok and XmlContentTok. */ -#define XML_TOK_PI 11 /* processing instruction */ -#define XML_TOK_XML_DECL 12 /* XML decl or text decl */ -#define XML_TOK_COMMENT 13 -#define XML_TOK_BOM 14 /* Byte order mark */ +# define XML_TOK_PI 11 /* processing instruction */ +# define XML_TOK_XML_DECL 12 /* XML decl or text decl */ +# define XML_TOK_COMMENT 13 +# define XML_TOK_BOM 14 /* Byte order mark */ /* The following tokens are returned only by XmlPrologTok */ -#define XML_TOK_PROLOG_S 15 -#define XML_TOK_DECL_OPEN 16 /* <!foo */ -#define XML_TOK_DECL_CLOSE 17 /* > */ -#define XML_TOK_NAME 18 -#define XML_TOK_NMTOKEN 19 -#define XML_TOK_POUND_NAME 20 /* #name */ -#define XML_TOK_OR 21 /* | */ -#define XML_TOK_PERCENT 22 -#define XML_TOK_OPEN_PAREN 23 -#define XML_TOK_CLOSE_PAREN 24 -#define XML_TOK_OPEN_BRACKET 25 -#define XML_TOK_CLOSE_BRACKET 26 -#define XML_TOK_LITERAL 27 -#define XML_TOK_PARAM_ENTITY_REF 28 -#define XML_TOK_INSTANCE_START 29 +# define XML_TOK_PROLOG_S 15 +# define XML_TOK_DECL_OPEN 16 /* <!foo */ +# define XML_TOK_DECL_CLOSE 17 /* > */ +# define XML_TOK_NAME 18 +# define XML_TOK_NMTOKEN 19 +# define XML_TOK_POUND_NAME 20 /* #name */ +# define XML_TOK_OR 21 /* | */ +# define XML_TOK_PERCENT 22 +# define XML_TOK_OPEN_PAREN 23 +# define XML_TOK_CLOSE_PAREN 24 +# define XML_TOK_OPEN_BRACKET 25 +# define XML_TOK_CLOSE_BRACKET 26 +# define XML_TOK_LITERAL 27 +# define XML_TOK_PARAM_ENTITY_REF 28 +# define XML_TOK_INSTANCE_START 29 /* The following occur only in element type declarations */ -#define XML_TOK_NAME_QUESTION 30 /* name? */ -#define XML_TOK_NAME_ASTERISK 31 /* name* */ -#define XML_TOK_NAME_PLUS 32 /* name+ */ -#define XML_TOK_COND_SECT_OPEN 33 /* <![ */ -#define XML_TOK_COND_SECT_CLOSE 34 /* ]]> */ -#define XML_TOK_CLOSE_PAREN_QUESTION 35 /* )? */ -#define XML_TOK_CLOSE_PAREN_ASTERISK 36 /* )* */ -#define XML_TOK_CLOSE_PAREN_PLUS 37 /* )+ */ -#define XML_TOK_COMMA 38 +# define XML_TOK_NAME_QUESTION 30 /* name? */ +# define XML_TOK_NAME_ASTERISK 31 /* name* */ +# define XML_TOK_NAME_PLUS 32 /* name+ */ +# define XML_TOK_COND_SECT_OPEN 33 /* <![ */ +# define XML_TOK_COND_SECT_CLOSE 34 /* ]]> */ +# define XML_TOK_CLOSE_PAREN_QUESTION 35 /* )? */ +# define XML_TOK_CLOSE_PAREN_ASTERISK 36 /* )* */ +# define XML_TOK_CLOSE_PAREN_PLUS 37 /* )+ */ +# define XML_TOK_COMMA 38 /* The following token is returned only by XmlAttributeValueTok */ -#define XML_TOK_ATTRIBUTE_VALUE_S 39 +# define XML_TOK_ATTRIBUTE_VALUE_S 39 /* The following token is returned only by XmlCdataSectionTok */ -#define XML_TOK_CDATA_SECT_CLOSE 40 +# define XML_TOK_CDATA_SECT_CLOSE 40 /* With namespace processing this is returned by XmlPrologTok for a name with a colon. */ -#define XML_TOK_PREFIXED_NAME 41 +# define XML_TOK_PREFIXED_NAME 41 -#ifdef XML_DTD -# define XML_TOK_IGNORE_SECT 42 -#endif /* XML_DTD */ +# ifdef XML_DTD +# define XML_TOK_IGNORE_SECT 42 +# endif /* XML_DTD */ -#ifdef XML_DTD -# define XML_N_STATES 4 -#else /* not XML_DTD */ -# define XML_N_STATES 3 -#endif /* not XML_DTD */ +# ifdef XML_DTD +# define XML_N_STATES 4 +# else /* not XML_DTD */ +# define XML_N_STATES 3 +# endif /* not XML_DTD */ -#define XML_PROLOG_STATE 0 -#define XML_CONTENT_STATE 1 -#define XML_CDATA_SECTION_STATE 2 -#ifdef XML_DTD -# define XML_IGNORE_SECTION_STATE 3 -#endif /* XML_DTD */ +# define XML_PROLOG_STATE 0 +# define XML_CONTENT_STATE 1 +# define XML_CDATA_SECTION_STATE 2 +# ifdef XML_DTD +# define XML_IGNORE_SECTION_STATE 3 +# endif /* XML_DTD */ -#define XML_N_LITERAL_TYPES 2 -#define XML_ATTRIBUTE_VALUE_LITERAL 0 -#define XML_ENTITY_VALUE_LITERAL 1 +# define XML_N_LITERAL_TYPES 2 +# define XML_ATTRIBUTE_VALUE_LITERAL 0 +# define XML_ENTITY_VALUE_LITERAL 1 /* The size of the buffer passed to XmlUtf8Encode must be at least this. */ -#define XML_UTF8_ENCODE_MAX 4 +# define XML_UTF8_ENCODE_MAX 4 /* The size of the buffer passed to XmlUtf16Encode must be at least this. */ -#define XML_UTF16_ENCODE_MAX 2 +# define XML_UTF16_ENCODE_MAX 2 typedef struct position { /* first line and first column are 0 not 1 */ @@ -220,63 +220,63 @@ struct encoding { the prolog outside literals, comments and processing instructions. */ -#define XmlTok(enc, state, ptr, end, nextTokPtr) \ - (((enc)->scanners[state])(enc, ptr, end, nextTokPtr)) +# define XmlTok(enc, state, ptr, end, nextTokPtr) \ + (((enc)->scanners[state])(enc, ptr, end, nextTokPtr)) -#define XmlPrologTok(enc, ptr, end, nextTokPtr) \ - XmlTok(enc, XML_PROLOG_STATE, ptr, end, nextTokPtr) +# define XmlPrologTok(enc, ptr, end, nextTokPtr) \ + XmlTok(enc, XML_PROLOG_STATE, ptr, end, nextTokPtr) -#define XmlContentTok(enc, ptr, end, nextTokPtr) \ - XmlTok(enc, XML_CONTENT_STATE, ptr, end, nextTokPtr) +# define XmlContentTok(enc, ptr, end, nextTokPtr) \ + XmlTok(enc, XML_CONTENT_STATE, ptr, end, nextTokPtr) -#define XmlCdataSectionTok(enc, ptr, end, nextTokPtr) \ - XmlTok(enc, XML_CDATA_SECTION_STATE, ptr, end, nextTokPtr) +# define XmlCdataSectionTok(enc, ptr, end, nextTokPtr) \ + XmlTok(enc, XML_CDATA_SECTION_STATE, ptr, end, nextTokPtr) -#ifdef XML_DTD +# ifdef XML_DTD -# define XmlIgnoreSectionTok(enc, ptr, end, nextTokPtr) \ - XmlTok(enc, XML_IGNORE_SECTION_STATE, ptr, end, nextTokPtr) +# define XmlIgnoreSectionTok(enc, ptr, end, nextTokPtr) \ + XmlTok(enc, XML_IGNORE_SECTION_STATE, ptr, end, nextTokPtr) -#endif /* XML_DTD */ +# endif /* XML_DTD */ /* This is used for performing a 2nd-level tokenization on the content of a literal that has already been returned by XmlTok. */ -#define XmlLiteralTok(enc, literalType, ptr, end, nextTokPtr) \ - (((enc)->literalScanners[literalType])(enc, ptr, end, nextTokPtr)) +# define XmlLiteralTok(enc, literalType, ptr, end, nextTokPtr) \ + (((enc)->literalScanners[literalType])(enc, ptr, end, nextTokPtr)) -#define XmlAttributeValueTok(enc, ptr, end, nextTokPtr) \ - XmlLiteralTok(enc, XML_ATTRIBUTE_VALUE_LITERAL, ptr, end, nextTokPtr) +# define XmlAttributeValueTok(enc, ptr, end, nextTokPtr) \ + XmlLiteralTok(enc, XML_ATTRIBUTE_VALUE_LITERAL, ptr, end, nextTokPtr) -#define XmlEntityValueTok(enc, ptr, end, nextTokPtr) \ - XmlLiteralTok(enc, XML_ENTITY_VALUE_LITERAL, ptr, end, nextTokPtr) +# define XmlEntityValueTok(enc, ptr, end, nextTokPtr) \ + XmlLiteralTok(enc, XML_ENTITY_VALUE_LITERAL, ptr, end, nextTokPtr) -#define XmlNameMatchesAscii(enc, ptr1, end1, ptr2) \ - (((enc)->nameMatchesAscii)(enc, ptr1, end1, ptr2)) +# define XmlNameMatchesAscii(enc, ptr1, end1, ptr2) \ + (((enc)->nameMatchesAscii)(enc, ptr1, end1, ptr2)) -#define XmlNameLength(enc, ptr) (((enc)->nameLength)(enc, ptr)) +# define XmlNameLength(enc, ptr) (((enc)->nameLength)(enc, ptr)) -#define XmlSkipS(enc, ptr) (((enc)->skipS)(enc, ptr)) +# define XmlSkipS(enc, ptr) (((enc)->skipS)(enc, ptr)) -#define XmlGetAttributes(enc, ptr, attsMax, atts) \ - (((enc)->getAtts)(enc, ptr, attsMax, atts)) +# define XmlGetAttributes(enc, ptr, attsMax, atts) \ + (((enc)->getAtts)(enc, ptr, attsMax, atts)) -#define XmlCharRefNumber(enc, ptr) (((enc)->charRefNumber)(enc, ptr)) +# define XmlCharRefNumber(enc, ptr) (((enc)->charRefNumber)(enc, ptr)) -#define XmlPredefinedEntityName(enc, ptr, end) \ - (((enc)->predefinedEntityName)(enc, ptr, end)) +# define XmlPredefinedEntityName(enc, ptr, end) \ + (((enc)->predefinedEntityName)(enc, ptr, end)) -#define XmlUpdatePosition(enc, ptr, end, pos) \ - (((enc)->updatePosition)(enc, ptr, end, pos)) +# define XmlUpdatePosition(enc, ptr, end, pos) \ + (((enc)->updatePosition)(enc, ptr, end, pos)) -#define XmlIsPublicId(enc, ptr, end, badPtr) \ - (((enc)->isPublicId)(enc, ptr, end, badPtr)) +# define XmlIsPublicId(enc, ptr, end, badPtr) \ + (((enc)->isPublicId)(enc, ptr, end, badPtr)) -#define XmlUtf8Convert(enc, fromP, fromLim, toP, toLim) \ - (((enc)->utf8Convert)(enc, fromP, fromLim, toP, toLim)) +# define XmlUtf8Convert(enc, fromP, fromLim, toP, toLim) \ + (((enc)->utf8Convert)(enc, fromP, fromLim, toP, toLim)) -#define XmlUtf16Convert(enc, fromP, fromLim, toP, toLim) \ - (((enc)->utf16Convert)(enc, fromP, fromLim, toP, toLim)) +# define XmlUtf16Convert(enc, fromP, fromLim, toP, toLim) \ + (((enc)->utf16Convert)(enc, fromP, fromLim, toP, toLim)) typedef struct { ENCODING initEnc; @@ -299,7 +299,7 @@ int XmlSizeOfUnknownEncoding(void); typedef int(XMLCALL *CONVERTER)(void *userData, const char *p); -ENCODING *XmlInitUnknownEncoding(void *mem, int *table, CONVERTER convert, +ENCODING *XmlInitUnknownEncoding(void *mem, const int *table, CONVERTER convert, void *userData); int XmlParseXmlDeclNS(int isGeneralTextEntity, const ENCODING *enc, @@ -312,10 +312,10 @@ int XmlInitEncodingNS(INIT_ENCODING *p, const ENCODING **encPtr, const char *name); const ENCODING *XmlGetUtf8InternalEncodingNS(void); const ENCODING *XmlGetUtf16InternalEncodingNS(void); -ENCODING *XmlInitUnknownEncodingNS(void *mem, int *table, CONVERTER convert, - void *userData); -#ifdef __cplusplus +ENCODING *XmlInitUnknownEncodingNS(void *mem, const int *table, + CONVERTER convert, void *userData); +# ifdef __cplusplus } -#endif +# endif #endif /* not XmlTok_INCLUDED */ diff --git a/Modules/expat/xmltok_impl.c b/Modules/expat/xmltok_impl.c index 239a2d06c4512ce..eae11bdd968aee4 100644 --- a/Modules/expat/xmltok_impl.c +++ b/Modules/expat/xmltok_impl.c @@ -17,6 +17,7 @@ Copyright (c) 2019 David Loffredo <loffredo@steptools.com> Copyright (c) 2020 Boris Kolpackov <boris@codesynthesis.com> Copyright (c) 2022 Martin Ettl <ettl.martin78@googlemail.com> + Copyright (c) 2026 Nick Begg <nick@stunttruck.net> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -83,7 +84,7 @@ *nextTokPtr = ptr; \ return XML_TOK_INVALID; \ } \ - /* fall through */ \ + EXPAT_FALLTHROUGH; \ case BT_NMSTRT: \ case BT_HEX: \ case BT_DIGIT: \ @@ -112,7 +113,7 @@ *nextTokPtr = ptr; \ return XML_TOK_INVALID; \ } \ - /* fall through */ \ + EXPAT_FALLTHROUGH; \ case BT_NMSTRT: \ case BT_HEX: \ ptr += MINBPC(enc); \ @@ -209,7 +210,7 @@ PREFIX(scanDecl)(const ENCODING *enc, const char *ptr, const char *end, *nextTokPtr = ptr; return XML_TOK_INVALID; } - /* fall through */ + EXPAT_FALLTHROUGH; case BT_S: case BT_CR: case BT_LF: @@ -323,7 +324,7 @@ PREFIX(scanPi)(const ENCODING *enc, const char *ptr, const char *end, *nextTokPtr = ptr + MINBPC(enc); return tok; } - /* fall through */ + EXPAT_FALLTHROUGH; default: *nextTokPtr = ptr; return XML_TOK_INVALID; @@ -615,7 +616,7 @@ PREFIX(scanAtts)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_INVALID; } } - /* fall through */ + EXPAT_FALLTHROUGH; case BT_EQUALS: { int open; # ifdef XML_NS @@ -898,7 +899,7 @@ PREFIX(contentTok)(const ENCODING *enc, const char *ptr, const char *end, return XML_TOK_INVALID; } } - /* fall through */ + EXPAT_FALLTHROUGH; case BT_AMP: case BT_LT: case BT_NONXML: @@ -1059,7 +1060,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, /* indicate that this might be part of a CR/LF pair */ return -XML_TOK_PROLOG_S; } - /* fall through */ + EXPAT_FALLTHROUGH; case BT_S: case BT_LF: for (;;) { @@ -1074,7 +1075,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, /* don't split CR/LF pair */ if (ptr + MINBPC(enc) != end) break; - /* fall through */ + EXPAT_FALLTHROUGH; default: *nextTokPtr = ptr; return XML_TOK_PROLOG_S; @@ -1189,7 +1190,7 @@ PREFIX(prologTok)(const ENCODING *enc, const char *ptr, const char *end, tok = XML_TOK_NMTOKEN; break; } - /* fall through */ + EXPAT_FALLTHROUGH; default: *nextTokPtr = ptr; return XML_TOK_INVALID; @@ -1484,7 +1485,7 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end, case BT_NMSTRT: if (! (BYTE_TO_ASCII(enc, ptr) & ~0x7f)) break; - /* fall through */ + EXPAT_FALLTHROUGH; default: switch (BYTE_TO_ASCII(enc, ptr)) { case 0x24: /* $ */ diff --git a/Modules/expat/xmltok_ns.c b/Modules/expat/xmltok_ns.c index fbdd3e3c7b79996..810ca2c6d0485eb 100644 --- a/Modules/expat/xmltok_ns.c +++ b/Modules/expat/xmltok_ns.c @@ -12,6 +12,7 @@ Copyright (c) 2002 Fred L. Drake, Jr. <fdrake@users.sourceforge.net> Copyright (c) 2002-2006 Karl Waclawek <karl@waclawek.net> Copyright (c) 2017-2021 Sebastian Pipping <sebastian@pipping.org> + Copyright (c) 2025 Alfonso Gregory <gfunni234@gmail.com> Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -98,13 +99,13 @@ NS(findEncoding)(const ENCODING *enc, const char *ptr, const char *end) { int i; XmlUtf8Convert(enc, &ptr, end, &p, p + ENCODING_MAX - 1); if (ptr != end) - return 0; + return NULL; *p = 0; if (streqci(buf, KW_UTF_16) && enc->minBytesPerChar == 2) return enc; i = getEncodingIndex(buf); if (i == UNKNOWN_ENC) - return 0; + return NULL; return NS(encodings)[i]; } diff --git a/Modules/faulthandler.c b/Modules/faulthandler.c index d49ce794d886745..09b910f4cf7d192 100644 --- a/Modules/faulthandler.c +++ b/Modules/faulthandler.c @@ -525,6 +525,11 @@ faulthandler_enable(void) } #endif + // gh-137185: Initialize C stack trace dumping outside of the signal + // handler. Specifically, we call backtrace() to ensure that libgcc is + // dynamically loaded outside of the signal handler. + _Py_InitDumpStack(); + for (size_t i=0; i < faulthandler_nsignals; i++) { fault_handler_t *handler; int err; @@ -1243,20 +1248,6 @@ faulthandler_stack_overflow(PyObject *self, PyObject *Py_UNUSED(ignored)) #endif /* defined(FAULTHANDLER_USE_ALT_STACK) && defined(HAVE_SIGACTION) */ -static int -faulthandler_traverse(PyObject *module, visitproc visit, void *arg) -{ - Py_VISIT(thread.file); -#ifdef FAULTHANDLER_USER - if (user_signals != NULL) { - for (size_t signum=0; signum < Py_NSIG; signum++) - Py_VISIT(user_signals[signum].file); - } -#endif - Py_VISIT(fatal_error.file); - return 0; -} - #ifdef MS_WINDOWS static PyObject * faulthandler_raise_exception(PyObject *self, PyObject *args) @@ -1389,7 +1380,6 @@ static struct PyModuleDef module_def = { .m_name = "faulthandler", .m_doc = module_doc, .m_methods = module_methods, - .m_traverse = faulthandler_traverse, .m_slots = faulthandler_slots }; diff --git a/Modules/fcntlmodule.c b/Modules/fcntlmodule.c index 220ee9ecdffc8ab..690882e34a8e2c1 100644 --- a/Modules/fcntlmodule.c +++ b/Modules/fcntlmodule.c @@ -41,22 +41,28 @@ fcntl.fcntl arg: object(c_default='NULL') = 0 / -Perform the operation `cmd` on file descriptor fd. - -The values used for `cmd` are operating system dependent, and are available -as constants in the fcntl module, using the same names as used in -the relevant C header files. The argument arg is optional, and -defaults to 0; it may be an int or a string. If arg is given as a string, -the return value of fcntl is a string of that length, containing the -resulting value put in the arg buffer by the operating system. The length -of the arg string is not allowed to exceed 1024 bytes. If the arg given -is an integer or if none is specified, the result value is an integer -corresponding to the return value of the fcntl call in the C code. +Perform the operation cmd on file descriptor fd. + +The values used for cmd are operating system dependent, and are +available as constants in the fcntl module, using the same names as used +in the relevant C header files. The argument arg is optional, and +defaults to 0; it may be an integer, a bytes-like object or a string. +If arg is given as a string, it will be encoded to binary using the +UTF-8 encoding. + +If the arg given is an integer or if none is specified, the result value +is an integer corresponding to the return value of the fcntl() call in +the C code. + +If arg is given as a bytes-like object, the return value of fcntl() is a +bytes object of that length, containing the resulting value put in the +arg buffer by the operating system. The length of the arg buffer is not +allowed to exceed 1024 bytes. [clinic start generated code]*/ static PyObject * fcntl_fcntl_impl(PyObject *module, int fd, int code, PyObject *arg) -/*[clinic end generated code: output=888fc93b51c295bd input=7955340198e5f334]*/ +/*[clinic end generated code: output=888fc93b51c295bd input=56c6d6196a4854df]*/ { int ret; int async_err = 0; @@ -135,40 +141,40 @@ fcntl.ioctl mutate_flag as mutate_arg: bool = True / -Perform the operation `request` on file descriptor `fd`. +Perform the operation request on file descriptor fd. -The values used for `request` are operating system dependent, and are available -as constants in the fcntl or termios library modules, using the same names as -used in the relevant C header files. +The values used for request are operating system dependent, and are +available as constants in the fcntl or termios library modules, using +the same names as used in the relevant C header files. -The argument `arg` is optional, and defaults to 0; it may be an int or a -buffer containing character data (most likely a string or an array). +The argument arg is optional, and defaults to 0; it may be an integer, a +bytes-like object or a string. If arg is given as a string, it will be +encoded to binary using the UTF-8 encoding. -If the argument is a mutable buffer (such as an array) and if the -mutate_flag argument (which is only allowed in this case) is true then the -buffer is (in effect) passed to the operating system and changes made by -the OS will be reflected in the contents of the buffer after the call has -returned. The return value is the integer returned by the ioctl system -call. +If the arg given is an integer or if none is specified, the result value +is an integer corresponding to the return value of the ioctl() call in +the C code. -If the argument is a mutable buffer and the mutable_flag argument is false, -the behavior is as if a string had been passed. +If the argument is a mutable buffer (such as a bytearray) and the +mutate_flag argument is true (default) then the buffer is (in effect) +passed to the operating system and changes made by the OS will be +reflected in the contents of the buffer after the call has returned. +The return value is the integer returned by the ioctl() system call. -If the argument is an immutable buffer (most likely a string) then a copy -of the buffer is passed to the operating system and the return value is a -string of the same length containing whatever the operating system put in -the buffer. The length of the arg buffer in this case is not allowed to -exceed 1024 bytes. +If the argument is a mutable buffer and the mutable_flag argument is +false, the behavior is as if an immutable buffer had been passed. -If the arg given is an integer or if none is specified, the result value is -an integer corresponding to the return value of the ioctl call in the C -code. +If the argument is an immutable buffer then a copy of the buffer is +passed to the operating system and the return value is a bytes object of +the same length containing whatever the operating system put in the +buffer. The length of the arg buffer in this case is not allowed to +exceed 1024 bytes. [clinic start generated code]*/ static PyObject * fcntl_ioctl_impl(PyObject *module, int fd, unsigned long code, PyObject *arg, int mutate_arg) -/*[clinic end generated code: output=f72baba2454d7a62 input=9c6cca5e2c339622]*/ +/*[clinic end generated code: output=f72baba2454d7a62 input=b69717a5588e086e]*/ { /* We use the unsigned non-checked 'I' format for the 'code' parameter because the system expects it to be a 32bit bit field value @@ -290,7 +296,7 @@ fcntl.flock operation as code: int / -Perform the lock operation `operation` on file descriptor `fd`. +Perform the lock operation on file descriptor fd. See the Unix manual page for flock(2) for details (On some systems, this function is emulated using fcntl()). @@ -298,7 +304,7 @@ function is emulated using fcntl()). static PyObject * fcntl_flock_impl(PyObject *module, int fd, int code) -/*[clinic end generated code: output=84059e2b37d2fc64 input=0bfc00f795953452]*/ +/*[clinic end generated code: output=84059e2b37d2fc64 input=ade68943e8599f0a]*/ { int ret; int async_err = 0; @@ -361,22 +367,22 @@ fcntl.lockf A wrapper around the fcntl() locking calls. -`fd` is the file descriptor of the file to lock or unlock, and operation is one -of the following values: +fd is the file descriptor of the file to lock or unlock, and operation +is one of the following values: LOCK_UN - unlock LOCK_SH - acquire a shared lock LOCK_EX - acquire an exclusive lock When operation is LOCK_SH or LOCK_EX, it can also be bitwise ORed with -LOCK_NB to avoid blocking on lock acquisition. If LOCK_NB is used and the -lock cannot be acquired, an OSError will be raised and the exception will -have an errno attribute set to EACCES or EAGAIN (depending on the operating -system -- for portability, check for either value). +LOCK_NB to avoid blocking on lock acquisition. If LOCK_NB is used and +the lock cannot be acquired, an OSError will be raised and the exception +will have an errno attribute set to EACCES or EAGAIN (depending on the +operating system -- for portability, check for either value). -`len` is the number of bytes to lock, with the default meaning to lock to -EOF. `start` is the byte offset, relative to `whence`, to that the lock -starts. `whence` is as with fileobj.seek(), specifically: +len is the number of bytes to lock, with the default meaning to lock to +EOF. start is the byte offset, relative to whence, to that the lock +starts. whence is as with fileobj.seek(), specifically: 0 - relative to the start of the file (SEEK_SET) 1 - relative to the current buffer position (SEEK_CUR) @@ -386,7 +392,7 @@ starts. `whence` is as with fileobj.seek(), specifically: static PyObject * fcntl_lockf_impl(PyObject *module, int fd, int code, PyObject *lenobj, PyObject *startobj, int whence) -/*[clinic end generated code: output=4985e7a172e7461a input=5480479fc63a04b8]*/ +/*[clinic end generated code: output=4985e7a172e7461a input=369bef4d7a1c5ff4]*/ { int ret; int async_err = 0; diff --git a/Modules/gcmodule.c b/Modules/gcmodule.c index ad13496b06deafe..7bb1dcc076ee495 100644 --- a/Modules/gcmodule.c +++ b/Modules/gcmodule.c @@ -159,6 +159,17 @@ gc_set_threshold_impl(PyObject *module, int threshold0, int group_right_1, { GCState *gcstate = get_gc_state(); +#ifndef Py_GIL_DISABLED + gcstate->generations[0].threshold = threshold0; + if (group_right_1) { + gcstate->generations[1].threshold = threshold1; + } + if (group_right_2) { + gcstate->generations[2].threshold = threshold2; + } +#else + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); gcstate->young.threshold = threshold0; if (group_right_1) { gcstate->old[0].threshold = threshold1; @@ -166,6 +177,8 @@ gc_set_threshold_impl(PyObject *module, int threshold0, int group_right_1, if (group_right_2) { gcstate->old[1].threshold = threshold2; } + _PyEval_StartTheWorld(interp); +#endif Py_RETURN_NONE; } @@ -180,10 +193,17 @@ gc_get_threshold_impl(PyObject *module) /*[clinic end generated code: output=7902bc9f41ecbbd8 input=286d79918034d6e6]*/ { GCState *gcstate = get_gc_state(); +#ifndef Py_GIL_DISABLED + return Py_BuildValue("(iii)", + gcstate->generations[0].threshold, + gcstate->generations[1].threshold, + gcstate->generations[2].threshold); +#else return Py_BuildValue("(iii)", gcstate->young.threshold, gcstate->old[0].threshold, - 0); + gcstate->old[1].threshold); +#endif } /*[clinic input] @@ -207,10 +227,17 @@ gc_get_count_impl(PyObject *module) gc->alloc_count = 0; #endif +#ifndef Py_GIL_DISABLED + return Py_BuildValue("(iii)", + gcstate->generations[0].count, + gcstate->generations[1].count, + gcstate->generations[2].count); +#else return Py_BuildValue("(iii)", - gcstate->young.count, - gcstate->old[gcstate->visited_space].count, - gcstate->old[gcstate->visited_space^1].count); + _Py_atomic_load_int_relaxed(&gcstate->young.count), + gcstate->old[0].count, + gcstate->old[1].count); +#endif } /*[clinic input] @@ -302,13 +329,13 @@ gc.get_objects Return a list of objects tracked by the collector (excluding the list returned). -If generation is not None, return only the objects tracked by the collector -that are in that generation. +If generation is not None, return only the objects tracked by the +collector that are in that generation. [clinic start generated code]*/ static PyObject * gc_get_objects_impl(PyObject *module, Py_ssize_t generation) -/*[clinic end generated code: output=48b35fea4ba6cb0e input=ef7da9df9806754c]*/ +/*[clinic end generated code: output=48b35fea4ba6cb0e input=3a819826fbde5eef]*/ { if (PySys_Audit("gc.get_objects", "n", generation) < 0) { return NULL; @@ -418,14 +445,15 @@ gc.freeze Freeze all current tracked objects and ignore them for future collections. -This can be used before a POSIX fork() call to make the gc copy-on-write friendly. -Note: collection before a POSIX fork() call may free pages for future allocation -which can cause copy-on-write. +This can be used before a POSIX fork() call to make the gc copy-on-write +friendly. +Note: collection before a POSIX fork() call may free pages for future +allocation which can cause copy-on-write. [clinic start generated code]*/ static PyObject * gc_freeze_impl(PyObject *module) -/*[clinic end generated code: output=502159d9cdc4c139 input=b602b16ac5febbe5]*/ +/*[clinic end generated code: output=502159d9cdc4c139 input=989012d0ba5a066f]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); _PyGC_Freeze(interp); @@ -476,7 +504,7 @@ PyDoc_STRVAR(gc__doc__, "set_debug() -- Set debugging flags.\n" "get_debug() -- Get debugging flags.\n" "set_threshold() -- Set the collection thresholds.\n" -"get_threshold() -- Return the current the collection thresholds.\n" +"get_threshold() -- Return the current collection thresholds.\n" "get_objects() -- Return a list of all objects tracked by the collector.\n" "is_tracked() -- Returns true if a given object is tracked.\n" "is_finalized() -- Returns true if a given object has been already finalized.\n" diff --git a/Modules/getpath.py b/Modules/getpath.py index be2210345afbda5..0e4f1e87e7342a3 100644 --- a/Modules/getpath.py +++ b/Modules/getpath.py @@ -129,8 +129,7 @@ # checked by looking for the BUILDDIR_TXT file, which contains the # relative path to the platlib dir. The executable_dir value is # derived from joining the VPATH preprocessor variable to the -# directory containing pybuilddir.txt. If it is not found, the -# BUILD_LANDMARK file is found, which is part of the source tree. +# directory containing pybuilddir.txt. # prefix is then found by searching up for a file that should only # exist in the source tree, and the stdlib dir is set to prefix/Lib. @@ -177,7 +176,6 @@ if os_name == 'posix' or os_name == 'darwin': BUILDDIR_TXT = 'pybuilddir.txt' - BUILD_LANDMARK = 'Modules/Setup.local' DEFAULT_PROGRAM_NAME = f'python{VERSION_MAJOR}' STDLIB_SUBDIR = f'{platlibdir}/python{VERSION_MAJOR}.{VERSION_MINOR}{ABI_THREAD}' STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}/os.py', f'{STDLIB_SUBDIR}/os.pyc'] @@ -190,7 +188,6 @@ elif os_name == 'nt': BUILDDIR_TXT = 'pybuilddir.txt' - BUILD_LANDMARK = f'{VPATH}\\Modules\\Setup.local' DEFAULT_PROGRAM_NAME = f'python' STDLIB_SUBDIR = 'Lib' STDLIB_LANDMARKS = [f'{STDLIB_SUBDIR}\\os.py', f'{STDLIB_SUBDIR}\\os.pyc'] @@ -364,10 +361,9 @@ def search_up(prefix, *landmarks, test=isfile): venv_prefix = None pyvenvcfg = [] - # Search for the 'home' key in pyvenv.cfg. Currently, we don't consider the - # presence of a pyvenv.cfg file without a 'home' key to signify the - # existence of a virtual environment — we quietly ignore them. - # XXX: If we don't find a 'home' key, we don't look for another pyvenv.cfg! + # Search for the 'home' key in pyvenv.cfg. If a home key isn't found, + # then it means a venv is active and home is based on the venv's + # executable (if its a symlink, home is where the symlink points). for line in pyvenvcfg: key, had_equ, value = line.partition('=') if had_equ and key.strip().lower() == 'home': @@ -412,10 +408,8 @@ def search_up(prefix, *landmarks, test=isfile): if isfile(candidate): base_executable = candidate break + # home key found; stop iterating over lines break - else: - # We didn't find a 'home' key in pyvenv.cfg (no break), reset venv_prefix. - venv_prefix = None # ****************************************************************************** @@ -515,13 +509,9 @@ def search_up(prefix, *landmarks, test=isfile): platstdlib_dir = real_executable_dir build_prefix = joinpath(real_executable_dir, VPATH) except (FileNotFoundError, PermissionError): - if isfile(joinpath(real_executable_dir, BUILD_LANDMARK)): - build_prefix = joinpath(real_executable_dir, VPATH) - if os_name == 'nt': - # QUIRK: Windows builds need platstdlib_dir to be the executable - # dir. Normally the builddir marker handles this, but in this - # case we need to correct manually. - platstdlib_dir = real_executable_dir + # We used to check for an alternate landmark here, but now we require + # BUILDDIR_TXT to exist. (gh-151544; CVE-2026-12003) + pass if build_prefix: if os_name == 'nt': diff --git a/Modules/grpmodule.c b/Modules/grpmodule.c index 29da9936b65504f..652958618a2c4c8 100644 --- a/Modules/grpmodule.c +++ b/Modules/grpmodule.c @@ -55,6 +55,11 @@ get_grp_state(PyObject *module) static struct PyModuleDef grpmodule; +/* Mutex to protect calls to getgrgid(), getgrnam(), and getgrent(). + * These functions return pointer to static data structure, which + * may be overwritten by any subsequent calls. */ +static PyMutex group_db_mutex = {0}; + #define DEFAULT_BUFFER_SIZE 1024 static PyObject * @@ -168,9 +173,15 @@ grp_getgrgid_impl(PyObject *module, PyObject *id) Py_END_ALLOW_THREADS #else + PyMutex_Lock(&group_db_mutex); + // The getgrgid() function need not be thread-safe. + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/getgrgid.html p = getgrgid(gid); #endif if (p == NULL) { +#ifndef HAVE_GETGRGID_R + PyMutex_Unlock(&group_db_mutex); +#endif PyMem_RawFree(buf); if (nomem == 1) { return PyErr_NoMemory(); @@ -185,6 +196,8 @@ grp_getgrgid_impl(PyObject *module, PyObject *id) retval = mkgrent(module, p); #ifdef HAVE_GETGRGID_R PyMem_RawFree(buf); +#else + PyMutex_Unlock(&group_db_mutex); #endif return retval; } @@ -249,9 +262,15 @@ grp_getgrnam_impl(PyObject *module, PyObject *name) Py_END_ALLOW_THREADS #else + PyMutex_Lock(&group_db_mutex); + // The getgrnam() function need not be thread-safe. + // https://pubs.opengroup.org/onlinepubs/9699919799/functions/getgrnam.html p = getgrnam(name_chars); #endif if (p == NULL) { +#ifndef HAVE_GETGRNAM_R + PyMutex_Unlock(&group_db_mutex); +#endif if (nomem == 1) { PyErr_NoMemory(); } @@ -261,6 +280,9 @@ grp_getgrnam_impl(PyObject *module, PyObject *name) goto out; } retval = mkgrent(module, p); +#ifndef HAVE_GETGRNAM_R + PyMutex_Unlock(&group_db_mutex); +#endif out: PyMem_RawFree(buf); Py_DECREF(bytes); @@ -285,8 +307,7 @@ grp_getgrall_impl(PyObject *module) return NULL; } - static PyMutex getgrall_mutex = {0}; - PyMutex_Lock(&getgrall_mutex); + PyMutex_Lock(&group_db_mutex); setgrent(); struct group *p; @@ -306,7 +327,7 @@ grp_getgrall_impl(PyObject *module) done: endgrent(); - PyMutex_Unlock(&getgrall_mutex); + PyMutex_Unlock(&group_db_mutex); return d; } diff --git a/Modules/hashlib.h b/Modules/hashlib.h index 7105e68af7b806f..a80b195a7657927 100644 --- a/Modules/hashlib.h +++ b/Modules/hashlib.h @@ -76,3 +76,32 @@ * to allow the user to optimize based on the platform they're using. */ #define HASHLIB_GIL_MINSIZE 2048 +static inline int +_Py_hashlib_data_argument(PyObject **res, PyObject *data, PyObject *string) +{ + if (data != NULL && string == NULL) { + // called as H(data) or H(data=...) + *res = data; + return 1; + } + else if (data == NULL && string != NULL) { + // called as H(string=...) + *res = string; + return 1; + } + else if (data == NULL && string == NULL) { + // fast path when no data is given + assert(!PyErr_Occurred()); + *res = NULL; + return 0; + } + else { + // called as H(data=..., string) + *res = NULL; + PyErr_SetString(PyExc_TypeError, + "'data' and 'string' are mutually exclusive " + "and support for 'string' keyword parameter " + "is slated for removal in a future version."); + return -1; + } +} diff --git a/Modules/hmacmodule.c b/Modules/hmacmodule.c index c7b49d4dee3d0a9..d3f86065125e1d5 100644 --- a/Modules/hmacmodule.c +++ b/Modules/hmacmodule.c @@ -31,14 +31,15 @@ #endif #if defined(__APPLE__) && defined(__arm64__) -# undef HACL_CAN_COMPILE_SIMD128 -# undef HACL_CAN_COMPILE_SIMD256 +# undef _Py_HACL_CAN_COMPILE_VEC128 +# undef _Py_HACL_CAN_COMPILE_VEC256 #endif -// Small mismatch between the variable names Python defines as part of configure -// at the ones HACL* expects to be set in order to enable those headers. -#define HACL_CAN_COMPILE_VEC128 HACL_CAN_COMPILE_SIMD128 -#define HACL_CAN_COMPILE_VEC256 HACL_CAN_COMPILE_SIMD256 +// HACL* expects HACL_CAN_COMPILE_VEC* macros to be set in order to enable +// the corresponding SIMD instructions so we need to "forward" the values +// we just deduced above. +#define HACL_CAN_COMPILE_VEC128 _Py_HACL_CAN_COMPILE_VEC128 +#define HACL_CAN_COMPILE_VEC256 _Py_HACL_CAN_COMPILE_VEC256 #include "_hacl/Hacl_HMAC.h" #include "_hacl/Hacl_Streaming_HMAC.h" // Hacl_Agile_Hash_* identifiers @@ -234,24 +235,24 @@ typedef struct py_hmac_hacl_api { * * The formal signature of this macro is: * - * (HACL_HMAC_state *, uint8_t *, uint32_t, PyObject *, (C statements)) + * (HACL_HMAC_state *, uint8_t *, uint32_t, (C statements)) */ #ifndef NDEBUG #define Py_HMAC_HACL_UPDATE_ONCE( \ HACL_STATE, BUF, LEN, \ - ALGORITHM, ERRACTION \ + ERRACTION \ ) \ do { \ Py_CHECK_HACL_UINT32_T_LENGTH(LEN); \ hacl_errno_t code = Py_HMAC_HACL_UPDATE_CALL(HACL_STATE, BUF, LEN); \ - if (_hacl_convert_errno(code, (ALGORITHM)) < 0) { \ + if (_hacl_convert_errno(code) < 0) { \ ERRACTION; \ } \ } while (0) #else #define Py_HMAC_HACL_UPDATE_ONCE( \ HACL_STATE, BUF, LEN, \ - _ALGORITHM, _ERRACTION \ + _ERRACTION \ ) \ do { \ (void)Py_HMAC_HACL_UPDATE_CALL(HACL_STATE, BUF, (LEN)); \ @@ -274,17 +275,17 @@ typedef struct py_hmac_hacl_api { * * The formal signature of this macro is: * - * (HACL_HMAC_state *, uint8_t *, C integer, PyObject *, (C statements)) + * (HACL_HMAC_state *, uint8_t *, C integer, (C statements)) */ #ifdef Py_HMAC_SSIZE_LARGER_THAN_UINT32 #define Py_HMAC_HACL_UPDATE_LOOP( \ HACL_STATE, BUF, LEN, \ - ALGORITHM, ERRACTION \ + ERRACTION \ ) \ do { \ while ((Py_ssize_t)LEN > UINT32_MAX_AS_SSIZE_T) { \ Py_HMAC_HACL_UPDATE_ONCE(HACL_STATE, BUF, UINT32_MAX, \ - ALGORITHM, ERRACTION); \ + ERRACTION); \ BUF += UINT32_MAX; \ LEN -= UINT32_MAX; \ } \ @@ -292,7 +293,7 @@ typedef struct py_hmac_hacl_api { #else #define Py_HMAC_HACL_UPDATE_LOOP( \ HACL_STATE, BUF, LEN, \ - _ALGORITHM, _ERRACTION \ + _ERRACTION \ ) #endif @@ -301,17 +302,17 @@ typedef struct py_hmac_hacl_api { * * The formal signature of this macro is: * - * (HACL_HMAC_state *, uint8_t *, C integer, PyObject *, (C statements)) + * (HACL_HMAC_state *, uint8_t *, C integer, (C statements)) */ #define Py_HMAC_HACL_UPDATE( \ HACL_STATE, BUF, LEN, \ - ALGORITHM, ERRACTION \ + ERRACTION \ ) \ do { \ Py_HMAC_HACL_UPDATE_LOOP(HACL_STATE, BUF, LEN, \ - ALGORITHM, ERRACTION); \ + ERRACTION); \ Py_HMAC_HACL_UPDATE_ONCE(HACL_STATE, BUF, LEN, \ - ALGORITHM, ERRACTION); \ + ERRACTION); \ } while (0) /* @@ -464,7 +465,7 @@ narrow_hmac_hash_kind(hmacmodule_state *state, HMAC_Hash_Kind kind) { switch (kind) { case Py_hmac_kind_hmac_blake2s_32: { -#if HACL_CAN_COMPILE_SIMD128 +#if _Py_HACL_CAN_COMPILE_VEC128 if (state->can_run_simd128) { return Py_hmac_kind_hmac_vectorized_blake2s_32; } @@ -472,7 +473,7 @@ narrow_hmac_hash_kind(hmacmodule_state *state, HMAC_Hash_Kind kind) return kind; } case Py_hmac_kind_hmac_blake2b_32: { -#if HACL_CAN_COMPILE_SIMD256 +#if _Py_HACL_CAN_COMPILE_VEC256 if (state->can_run_simd256) { return Py_hmac_kind_hmac_vectorized_blake2b_32; } @@ -491,38 +492,40 @@ narrow_hmac_hash_kind(hmacmodule_state *state, HMAC_Hash_Kind kind) * Otherwise, this sets an appropriate exception and returns -1. */ static int -_hacl_convert_errno(hacl_errno_t code, PyObject *algorithm) +_hacl_convert_errno(hacl_errno_t code) { + assert(PyGILState_GetThisThreadState() != NULL); + if (code == Hacl_Streaming_Types_Success) { + return 0; + } + + PyGILState_STATE gstate = PyGILState_Ensure(); switch (code) { - case Hacl_Streaming_Types_Success: { - return 0; - } case Hacl_Streaming_Types_InvalidAlgorithm: { - // only makes sense if an algorithm is known at call time - assert(algorithm != NULL); - assert(PyUnicode_CheckExact(algorithm)); - PyErr_Format(PyExc_ValueError, "invalid algorithm: %U", algorithm); - return -1; + PyErr_SetString(PyExc_ValueError, "invalid HACL* algorithm"); + break; } case Hacl_Streaming_Types_InvalidLength: { PyErr_SetString(PyExc_ValueError, "invalid length"); - return -1; + break; } case Hacl_Streaming_Types_MaximumLengthExceeded: { PyErr_SetString(PyExc_OverflowError, "maximum length exceeded"); - return -1; + break; } case Hacl_Streaming_Types_OutOfMemory: { PyErr_NoMemory(); - return -1; + break; } default: { PyErr_Format(PyExc_RuntimeError, - "HACL* internal routine failed with error code: %d", + "HACL* internal routine failed with error code: %u", code); - return -1; + break; } } + PyGILState_Release(gstate); + return -1; } /* @@ -536,7 +539,7 @@ _hacl_hmac_state_new(HMAC_Hash_Kind kind, uint8_t *key, uint32_t len) assert(kind != Py_hmac_kind_hash_unknown); HACL_HMAC_state *state = NULL; hacl_errno_t retcode = Hacl_Streaming_HMAC_malloc_(kind, key, len, &state); - if (_hacl_convert_errno(retcode, NULL) < 0) { + if (_hacl_convert_errno(retcode) < 0) { assert(state == NULL); return NULL; } @@ -804,13 +807,13 @@ hmac_feed_initial_data(HMACObject *self, uint8_t *msg, Py_ssize_t len) } if (len < HASHLIB_GIL_MINSIZE) { - Py_HMAC_HACL_UPDATE(self->state, msg, len, self->name, return -1); + Py_HMAC_HACL_UPDATE(self->state, msg, len, return -1); return 0; } int res = 0; Py_BEGIN_ALLOW_THREADS - Py_HMAC_HACL_UPDATE(self->state, msg, len, self->name, goto error); + Py_HMAC_HACL_UPDATE(self->state, msg, len, goto error); goto done; #ifndef NDEBUG error: @@ -978,7 +981,7 @@ hmac_update_state_with_lock(HMACObject *self, uint8_t *buf, Py_ssize_t len) int res = 0; Py_BEGIN_ALLOW_THREADS PyMutex_Lock(&self->mutex); // unconditionally acquire a lock - Py_HMAC_HACL_UPDATE(self->state, buf, len, self->name, goto error); + Py_HMAC_HACL_UPDATE(self->state, buf, len, goto error); goto done; #ifndef NDEBUG error: @@ -1005,7 +1008,7 @@ static int hmac_update_state_cond_lock(HMACObject *self, uint8_t *buf, Py_ssize_t len) { ENTER_HASHLIB(self); // conditionally acquire a lock - Py_HMAC_HACL_UPDATE(self->state, buf, len, self->name, goto error); + Py_HMAC_HACL_UPDATE(self->state, buf, len, goto error); LEAVE_HASHLIB(self); return 0; @@ -1076,7 +1079,7 @@ hmac_digest_compute_cond_lock(HMACObject *self, uint8_t *digest) rc == Hacl_Streaming_Types_Success || rc == Hacl_Streaming_Types_OutOfMemory ); - return _hacl_convert_errno(rc, NULL); + return _hacl_convert_errno(rc); } /*[clinic input] @@ -1104,15 +1107,15 @@ _hmac.HMAC.hexdigest Return hexadecimal digest of the bytes passed to the update() method so far. -This may be used to exchange the value safely in email or other non-binary -environments. +This may be used to exchange the value safely in email or other +non-binary environments. This method may raise a MemoryError. [clinic start generated code]*/ static PyObject * _hmac_HMAC_hexdigest_impl(HMACObject *self) -/*[clinic end generated code: output=6659807a09ae14ec input=493b2db8013982b9]*/ +/*[clinic end generated code: output=6659807a09ae14ec input=5ae03f21d69c970c]*/ { assert(self->digest_size <= Py_hmac_hash_max_digest_size); uint8_t digest[Py_hmac_hash_max_digest_size]; @@ -1526,7 +1529,6 @@ static void py_hmac_hinfo_ht_free(void *hinfo) { py_hmac_hinfo *entry = (py_hmac_hinfo *)hinfo; - assert(entry->display_name != NULL); if (--(entry->refcnt) == 0) { Py_CLEAR(entry->display_name); PyMem_Free(hinfo); @@ -1601,16 +1603,19 @@ py_hmac_hinfo_ht_new(void) assert(value->display_name == NULL); value->refcnt = 0; -#define Py_HMAC_HINFO_LINK(KEY) \ - do { \ - int rc = py_hmac_hinfo_ht_add(table, KEY, value); \ - if (rc < 0) { \ - PyMem_Free(value); \ - goto error; \ - } \ - else if (rc == 1) { \ - value->refcnt++; \ - } \ +#define Py_HMAC_HINFO_LINK(KEY) \ + do { \ + int rc = py_hmac_hinfo_ht_add(table, (KEY), value); \ + if (rc < 0) { \ + /* entry may already be in ht, freed upon exit */ \ + if (value->refcnt == 0) { \ + PyMem_Free(value); \ + } \ + goto error; \ + } \ + else if (rc == 1) { \ + value->refcnt++; \ + } \ } while (0) Py_HMAC_HINFO_LINK(e->name); Py_HMAC_HINFO_LINK(e->hashlib_name); @@ -1622,7 +1627,8 @@ py_hmac_hinfo_ht_new(void) e->hashlib_name == NULL ? e->name : e->hashlib_name ); if (value->display_name == NULL) { - PyMem_Free(value); + /* 'value' is owned by the table (refcnt > 0), + so _Py_hashtable_destroy() will free it. */ goto error; } } @@ -1759,7 +1765,7 @@ hmacmodule_init_cpu_features(hmacmodule_state *state) #undef ECX_SSE3 #undef EBX_AVX2 -#if HACL_CAN_COMPILE_SIMD128 +#if _Py_HACL_CAN_COMPILE_VEC128 // TODO(picnixz): use py_cpuid_features (gh-125022) to improve detection state->can_run_simd128 = sse && sse2 && sse3 && sse41 && sse42 && cmov; #else @@ -1769,7 +1775,7 @@ hmacmodule_init_cpu_features(hmacmodule_state *state) state->can_run_simd128 = false; #endif -#if HACL_CAN_COMPILE_SIMD256 +#if _Py_HACL_CAN_COMPILE_VEC256 // TODO(picnixz): use py_cpuid_features (gh-125022) to improve detection state->can_run_simd256 = state->can_run_simd128 && avx && avx2; #else diff --git a/Modules/itertoolsmodule.c b/Modules/itertoolsmodule.c index 943c1e8607b38ff..2470a08885ad5c1 100644 --- a/Modules/itertoolsmodule.c +++ b/Modules/itertoolsmodule.c @@ -376,7 +376,7 @@ pairwise_next(PyObject *op) } result = po->result; - if (Py_REFCNT(result) == 1) { + if (_PyObject_IsUniquelyReferenced(result)) { Py_INCREF(result); PyObject *last_old = PyTuple_GET_ITEM(result, 0); PyObject *last_new = PyTuple_GET_ITEM(result, 1); @@ -544,9 +544,19 @@ groupby_next(PyObject *op) else if (gbo->tgtkey == NULL) break; else { - int rcmp; + /* A user-defined __eq__ can re-enter groupby and advance the iterator, + mutating gbo->tgtkey / gbo->currkey while we are comparing them. + Take local snapshots and hold strong references so INCREF/DECREF + apply to the same objects even under re-entrancy. */ + PyObject *tgtkey = gbo->tgtkey; + PyObject *currkey = gbo->currkey; + + Py_INCREF(tgtkey); + Py_INCREF(currkey); + int rcmp = PyObject_RichCompareBool(tgtkey, currkey, Py_EQ); + Py_DECREF(tgtkey); + Py_DECREF(currkey); - rcmp = PyObject_RichCompareBool(gbo->tgtkey, gbo->currkey, Py_EQ); if (rcmp == -1) return NULL; else if (rcmp == 0) @@ -668,7 +678,16 @@ _grouper_next(PyObject *op) } assert(gbo->currkey != NULL); - rcmp = PyObject_RichCompareBool(igo->tgtkey, gbo->currkey, Py_EQ); + /* A user-defined __eq__ can re-enter the grouper and advance the iterator, + mutating gbo->currkey while we are comparing them. + Take local snapshots and hold strong references so INCREF/DECREF + apply to the same objects even under re-entrancy. */ + PyObject *tgtkey = Py_NewRef(igo->tgtkey); + PyObject *currkey = Py_NewRef(gbo->currkey); + rcmp = PyObject_RichCompareBool(tgtkey, currkey, Py_EQ); + Py_DECREF(tgtkey); + Py_DECREF(currkey); + if (rcmp <= 0) /* got any error or current group is end */ return NULL; @@ -753,13 +772,17 @@ teedataobject_newinternal(itertools_state *state, PyObject *it) static PyObject * teedataobject_jumplink(itertools_state *state, teedataobject *tdo) { + PyObject *link; + Py_BEGIN_CRITICAL_SECTION(tdo); if (tdo->nextlink == NULL) tdo->nextlink = teedataobject_newinternal(state, tdo->it); - return Py_XNewRef(tdo->nextlink); + link = Py_XNewRef(tdo->nextlink); + Py_END_CRITICAL_SECTION(); + return link; } static PyObject * -teedataobject_getitem(teedataobject *tdo, int i) +teedataobject_getitem_lock_held(teedataobject *tdo, int i) { PyObject *value; @@ -785,6 +808,16 @@ teedataobject_getitem(teedataobject *tdo, int i) return Py_NewRef(value); } +static PyObject * +teedataobject_getitem(teedataobject *tdo, int i) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(tdo); + result = teedataobject_getitem_lock_held(tdo, i); + Py_END_CRITICAL_SECTION(); + return result; +} + static int teedataobject_traverse(PyObject *op, visitproc visit, void * arg) { @@ -802,10 +835,13 @@ teedataobject_traverse(PyObject *op, visitproc visit, void * arg) static void teedataobject_safe_decref(PyObject *obj) { - while (obj && Py_REFCNT(obj) == 1) { + while (obj && _PyObject_IsUniquelyReferenced(obj)) { teedataobject *tmp = teedataobject_CAST(obj); - PyObject *nextlink = tmp->nextlink; + PyObject *nextlink; + Py_BEGIN_CRITICAL_SECTION(obj); + nextlink = tmp->nextlink; tmp->nextlink = NULL; + Py_END_CRITICAL_SECTION(); Py_SETREF(obj, nextlink); } Py_XDECREF(obj); @@ -818,11 +854,13 @@ teedataobject_clear(PyObject *op) PyObject *tmp; teedataobject *tdo = teedataobject_CAST(op); + Py_BEGIN_CRITICAL_SECTION(op); Py_CLEAR(tdo->it); for (i=0 ; i<tdo->numread ; i++) Py_CLEAR(tdo->values[i]); tmp = tdo->nextlink; tdo->nextlink = NULL; + Py_END_CRITICAL_SECTION(); teedataobject_safe_decref(tmp); return 0; } @@ -915,20 +953,67 @@ static PyObject * tee_next(PyObject *op) { teeobject *to = teeobject_CAST(op); - PyObject *value, *link; + PyObject *value; +#ifndef Py_GIL_DISABLED + /* The GIL already serializes access, so keep the simple path without the + snapshot and revalidation that the free-threaded build needs. */ if (to->index >= LINKCELLS) { - link = teedataobject_jumplink(to->state, to->dataobj); - if (link == NULL) + PyObject *link = teedataobject_jumplink(to->state, to->dataobj); + if (link == NULL) { return NULL; + } Py_SETREF(to->dataobj, (teedataobject *)link); to->index = 0; } value = teedataobject_getitem(to->dataobj, to->index); - if (value == NULL) + if (value == NULL) { return NULL; + } to->index++; return value; +#else + for (;;) { + teedataobject *dataobj; + int index; + + /* Snapshot the branch position (strong ref to the shared data object) + under the tee lock; the data object is locked separately, not nested, + then the advance is revalidated. */ + Py_BEGIN_CRITICAL_SECTION(op); + dataobj = (teedataobject *)Py_NewRef((PyObject *)to->dataobj); + index = to->index; + Py_END_CRITICAL_SECTION(); + + if (index < LINKCELLS) { + value = teedataobject_getitem(dataobj, index); + if (value != NULL) { + Py_BEGIN_CRITICAL_SECTION(op); + if (to->dataobj == dataobj && to->index == index) { + to->index = index + 1; + } + Py_END_CRITICAL_SECTION(); + } + Py_DECREF(dataobj); + return value; + } + + PyObject *link = teedataobject_jumplink(to->state, dataobj); + if (link == NULL) { + Py_DECREF(dataobj); + return NULL; + } + Py_BEGIN_CRITICAL_SECTION(op); + if (to->dataobj == dataobj) { + Py_SETREF(to->dataobj, (teedataobject *)link); + to->index = 0; + link = NULL; + } + Py_END_CRITICAL_SECTION(); + Py_XDECREF(link); + Py_DECREF(dataobj); + } +#endif } static int @@ -947,8 +1032,10 @@ tee_copy_impl(teeobject *to) if (newto == NULL) { return NULL; } + Py_BEGIN_CRITICAL_SECTION(to); newto->dataobj = (teedataobject *)Py_NewRef(to->dataobj); newto->index = to->index; + Py_END_CRITICAL_SECTION(); newto->weakreflist = NULL; newto->state = to->state; PyObject_GC_Track(newto); @@ -1124,7 +1211,6 @@ typedef struct { PyObject *it; PyObject *saved; Py_ssize_t index; - int firstpass; } cycleobject; #define cycleobject_CAST(op) ((cycleobject *)(op)) @@ -1165,8 +1251,7 @@ itertools_cycle_impl(PyTypeObject *type, PyObject *iterable) } lz->it = it; lz->saved = saved; - lz->index = 0; - lz->firstpass = 0; + lz->index = -1; return (PyObject *)lz; } @@ -1199,11 +1284,11 @@ cycle_next(PyObject *op) cycleobject *lz = cycleobject_CAST(op); PyObject *item; - if (lz->it != NULL) { + Py_ssize_t index = FT_ATOMIC_LOAD_SSIZE_RELAXED(lz->index); + + if (index < 0) { item = PyIter_Next(lz->it); if (item != NULL) { - if (lz->firstpass) - return item; if (PyList_Append(lz->saved, item)) { Py_DECREF(item); return NULL; @@ -1213,15 +1298,22 @@ cycle_next(PyObject *op) /* Note: StopIteration is already cleared by PyIter_Next() */ if (PyErr_Occurred()) return NULL; + index = 0; + FT_ATOMIC_STORE_SSIZE_RELAXED(lz->index, 0); +#ifndef Py_GIL_DISABLED Py_CLEAR(lz->it); +#endif } if (PyList_GET_SIZE(lz->saved) == 0) return NULL; - item = PyList_GET_ITEM(lz->saved, lz->index); - lz->index++; - if (lz->index >= PyList_GET_SIZE(lz->saved)) - lz->index = 0; - return Py_NewRef(item); + item = PyList_GetItemRef(lz->saved, index); + assert(item); + index++; + if (index >= PyList_GET_SIZE(lz->saved)) { + index = 0; + } + FT_ATOMIC_STORE_SSIZE_RELAXED(lz->index, index); + return item; } static PyType_Slot cycle_slots[] = { @@ -1875,8 +1967,8 @@ chain_traverse(PyObject *op, visitproc visit, void *arg) return 0; } -static PyObject * -chain_next(PyObject *op) +static inline PyObject * +chain_next_lock_held(PyObject *op) { chainobject *lz = chainobject_CAST(op); PyObject *item; @@ -1914,6 +2006,16 @@ chain_next(PyObject *op) return NULL; } +static PyObject * +chain_next(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = chain_next_lock_held(op); + Py_END_CRITICAL_SECTION() + return result; +} + PyDoc_STRVAR(chain_doc, "chain(*iterables)\n\ --\n\ @@ -1922,10 +2024,14 @@ Return a chain object whose .__next__() method returns elements from the\n\ first iterable until it is exhausted, then elements from the next\n\ iterable, until all of the iterables are exhausted."); +PyDoc_STRVAR(chain_class_getitem_doc, +"chain is generic over the type of its contents.\n\ +This is the union of the types of the input iterable contents."); + static PyMethodDef chain_methods[] = { ITERTOOLS_CHAIN_FROM_ITERABLE_METHODDEF {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, chain_class_getitem_doc}, {NULL, NULL} /* sentinel */ }; @@ -2081,7 +2187,7 @@ product_traverse(PyObject *op, visitproc visit, void *arg) } static PyObject * -product_next(PyObject *op) +product_next_lock_held(PyObject *op) { productobject *lz = productobject_CAST(op); PyObject *pool; @@ -2114,7 +2220,7 @@ product_next(PyObject *op) Py_ssize_t *indices = lz->indices; /* Copy the previous result tuple or re-use it if available */ - if (Py_REFCNT(result) > 1) { + if (!_PyObject_IsUniquelyReferenced(result)) { PyObject *old_result = result; result = _PyTuple_FromArray(_PyTuple_ITEMS(old_result), npools); if (result == NULL) @@ -2167,6 +2273,16 @@ product_next(PyObject *op) return NULL; } +static PyObject * +product_next(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = product_next_lock_held(op); + Py_END_CRITICAL_SECTION() + return result; +} + static PyMethodDef product_methods[] = { {"__sizeof__", product_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ @@ -2314,7 +2430,7 @@ combinations_traverse(PyObject *op, visitproc visit, void *arg) } static PyObject * -combinations_next(PyObject *op) +combinations_next_lock_held(PyObject *op) { combinationsobject *co = combinationsobject_CAST(op); PyObject *elem; @@ -2343,7 +2459,7 @@ combinations_next(PyObject *op) } } else { /* Copy the previous result tuple or re-use it if available */ - if (Py_REFCNT(result) > 1) { + if (!_PyObject_IsUniquelyReferenced(result)) { PyObject *old_result = result; result = _PyTuple_FromArray(_PyTuple_ITEMS(old_result), r); if (result == NULL) @@ -2399,6 +2515,16 @@ combinations_next(PyObject *op) return NULL; } +static PyObject * +combinations_next(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = combinations_next_lock_held(op); + Py_END_CRITICAL_SECTION() + return result; +} + static PyMethodDef combinations_methods[] = { {"__sizeof__", combinations_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ @@ -2558,7 +2684,7 @@ cwr_traverse(PyObject *op, visitproc visit, void *arg) } static PyObject * -cwr_next(PyObject *op) +cwr_next_lock_held(PyObject *op) { cwrobject *co = cwrobject_CAST(op); PyObject *elem; @@ -2589,7 +2715,7 @@ cwr_next(PyObject *op) } } else { /* Copy the previous result tuple or re-use it if available */ - if (Py_REFCNT(result) > 1) { + if (!_PyObject_IsUniquelyReferenced(result)) { PyObject *old_result = result; result = _PyTuple_FromArray(_PyTuple_ITEMS(old_result), r); if (result == NULL) @@ -2637,6 +2763,16 @@ cwr_next(PyObject *op) return NULL; } +static PyObject * +cwr_next(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = cwr_next_lock_held(op); + Py_END_CRITICAL_SECTION() + return result; +} + static PyMethodDef cwr_methods[] = { {"__sizeof__", cwr_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ @@ -2817,7 +2953,7 @@ permutations_traverse(PyObject *op, visitproc visit, void *arg) } static PyObject * -permutations_next(PyObject *op) +permutations_next_lock_held(PyObject *op) { permutationsobject *po = permutationsobject_CAST(op); PyObject *elem; @@ -2850,7 +2986,7 @@ permutations_next(PyObject *op) goto empty; /* Copy the previous result tuple or re-use it if available */ - if (Py_REFCNT(result) > 1) { + if (!_PyObject_IsUniquelyReferenced(result)) { PyObject *old_result = result; result = _PyTuple_FromArray(_PyTuple_ITEMS(old_result), r); if (result == NULL) @@ -2907,6 +3043,16 @@ permutations_next(PyObject *op) return NULL; } +static PyObject * +permutations_next(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = permutations_next_lock_held(op); + Py_END_CRITICAL_SECTION() + return result; +} + static PyMethodDef permuations_methods[] = { {"__sizeof__", permutations_sizeof, METH_NOARGS, sizeof_doc}, {NULL, NULL} /* sentinel */ @@ -3014,7 +3160,7 @@ accumulate_traverse(PyObject *op, visitproc visit, void *arg) } static PyObject * -accumulate_next(PyObject *op) +accumulate_next_lock_held(PyObject *op) { accumulateobject *lz = accumulateobject_CAST(op); PyObject *val, *newtotal; @@ -3046,6 +3192,16 @@ accumulate_next(PyObject *op) return newtotal; } +static PyObject * +accumulate_next(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = accumulate_next_lock_held(op); + Py_END_CRITICAL_SECTION() + return result; +} + static PyType_Slot accumulate_slots[] = { {Py_tp_dealloc, accumulate_dealloc}, {Py_tp_getattro, PyObject_GenericGetAttr}, @@ -3091,13 +3247,13 @@ itertools.compress.__new__ selectors as seq2: object Return data elements corresponding to true selector elements. -Forms a shorter iterator from selected data elements using the selectors to -choose the data elements. +Forms a shorter iterator from selected data elements using the selectors +to choose the data elements. [clinic start generated code]*/ static PyObject * itertools_compress_impl(PyTypeObject *type, PyObject *seq1, PyObject *seq2) -/*[clinic end generated code: output=7e67157212ed09e0 input=79596d7cd20c77e5]*/ +/*[clinic end generated code: output=7e67157212ed09e0 input=32ca4347dbc46749]*/ { PyObject *data=NULL, *selectors=NULL; compressobject *lz; @@ -3525,9 +3681,11 @@ static PyObject * count_repr(PyObject *op) { countobject *lz = countobject_CAST(op); - if (lz->long_cnt == NULL) + if (lz->long_cnt == NULL) { + Py_ssize_t cnt = FT_ATOMIC_LOAD_SSIZE_RELAXED(lz->cnt); return PyUnicode_FromFormat("%s(%zd)", - _PyType_Name(Py_TYPE(lz)), lz->cnt); + _PyType_Name(Py_TYPE(lz)), cnt); + } if (PyLong_Check(lz->long_step)) { long step = PyLong_AsLong(lz->long_step); @@ -3804,7 +3962,7 @@ zip_longest_traverse(PyObject *op, visitproc visit, void *arg) } static PyObject * -zip_longest_next(PyObject *op) +zip_longest_next_lock_held(PyObject *op) { ziplongestobject *lz = ziplongestobject_CAST(op); Py_ssize_t i; @@ -3818,7 +3976,7 @@ zip_longest_next(PyObject *op) return NULL; if (lz->numactive == 0) return NULL; - if (Py_REFCNT(result) == 1) { + if (_PyObject_IsUniquelyReferenced(result)) { Py_INCREF(result); for (i=0 ; i < tuplesize ; i++) { it = PyTuple_GET_ITEM(lz->ittuple, i); @@ -3875,6 +4033,16 @@ zip_longest_next(PyObject *op) return result; } +static PyObject * +zip_longest_next(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = zip_longest_next_lock_held(op); + Py_END_CRITICAL_SECTION() + return result; +} + PyDoc_STRVAR(zip_longest_doc, "zip_longest(*iterables, fillvalue=None)\n\ --\n\ diff --git a/Modules/main.c b/Modules/main.c index ea1239ecc57f00e..f9df068e6dcea17 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -10,7 +10,7 @@ #include "pycore_pathconfig.h" // _PyPathConfig_ComputeSysPath0() #include "pycore_pylifecycle.h" // _Py_PreInitializeFromPyArgv() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_pythonrun.h" // _PyRun_AnyFileObject() +#include "pycore_pythonrun.h" // _PyRun_AnyFile() #include "pycore_unicodeobject.h" // _PyUnicode_Dedent() /* Includes for exit_sigint() */ @@ -79,6 +79,12 @@ pymain_init(const _PyArgv *args) /* --- pymain_run_python() ---------------------------------------- */ +static int +pymain_check_signals(void) +{ + return Py_MakePendingCalls(); +} + /* Non-zero if filename, command (-c) or module (-m) is set on the command line */ static inline int config_run_code(const PyConfig *config) @@ -200,17 +206,21 @@ pymain_header(const PyConfig *config) } -static void +static int pymain_import_readline(const PyConfig *config) { + if (pymain_check_signals() < 0) { + return -1; + } + if (config->isolated) { - return; + return 0; } if (!config->inspect && config_run_code(config)) { - return; + return 0; } if (!isatty(fileno(stdin))) { - return; + return 0; } PyObject *mod = PyImport_ImportModule("readline"); @@ -227,6 +237,7 @@ pymain_import_readline(const PyConfig *config) else { Py_DECREF(mod); } + return 0; } @@ -234,7 +245,6 @@ static int pymain_run_command(wchar_t *command) { PyObject *unicode, *bytes; - int ret; unicode = PyUnicode_FromWideChar(command, -1); if (unicode == NULL) { @@ -258,9 +268,18 @@ pymain_run_command(wchar_t *command) PyCompilerFlags cf = _PyCompilerFlags_INIT; cf.cf_flags |= PyCF_IGNORE_COOKIE; - ret = _PyRun_SimpleStringFlagsWithName(PyBytes_AsString(bytes), "<string>", &cf); + PyObject *result = NULL; + PyObject *string_obj = PyUnicode_FromString("<string>"); + if (string_obj != NULL) { + result = _PyRun_SimpleString(PyBytes_AsString(bytes), + string_obj, &cf); + Py_DECREF(string_obj); + } Py_DECREF(bytes); - return (ret != 0); + if (result == NULL) { + return pymain_exit_err_print(); + } + return 0; error: PySys_WriteStderr("Unable to decode the command from the command line:\n"); @@ -269,49 +288,65 @@ pymain_run_command(wchar_t *command) static int -pymain_start_pyrepl_no_main(void) +pymain_run_pyrepl(int pythonstartup) { - int res = 0; + int exitcode = 0; PyObject *console = NULL; PyObject *empty_tuple = NULL; PyObject *kwargs = NULL; PyObject *console_result = NULL; + PyObject *main_module = NULL; PyObject *pyrepl = PyImport_ImportModule("_pyrepl.main"); if (pyrepl == NULL) { fprintf(stderr, "Could not import _pyrepl.main\n"); - res = pymain_exit_err_print(); - goto done; + goto error; } + console = PyObject_GetAttrString(pyrepl, "interactive_console"); if (console == NULL) { fprintf(stderr, "Could not access _pyrepl.main.interactive_console\n"); - res = pymain_exit_err_print(); - goto done; + goto error; } + empty_tuple = PyTuple_New(0); if (empty_tuple == NULL) { - res = pymain_exit_err_print(); - goto done; + goto error; } kwargs = PyDict_New(); if (kwargs == NULL) { - res = pymain_exit_err_print(); - goto done; + goto error; } - if (!PyDict_SetItemString(kwargs, "pythonstartup", _PyLong_GetOne())) { - console_result = PyObject_Call(console, empty_tuple, kwargs); - if (console_result == NULL) { - res = pymain_exit_err_print(); - } + + main_module = PyImport_AddModuleRef("__main__"); + if (main_module == NULL) { + goto error; } + + PyObject *value = pythonstartup ? Py_True : Py_False; + if (PyDict_SetItemString(kwargs, "mainmodule", main_module) + || PyDict_SetItemString(kwargs, "pythonstartup", value)) + { + goto error; + } + + console_result = PyObject_Call(console, empty_tuple, kwargs); + if (console_result == NULL) { + goto error; + } + done: Py_XDECREF(console_result); Py_XDECREF(kwargs); Py_XDECREF(empty_tuple); Py_XDECREF(console); Py_XDECREF(pyrepl); - return res; + Py_XDECREF(main_module); + return exitcode; + +error: + exitcode = pymain_exit_err_print(); + goto done; } @@ -391,16 +426,19 @@ pymain_run_file_obj(PyObject *program_name, PyObject *filename, return 1; } - // Call pending calls like signal handlers (SIGINT) - if (Py_MakePendingCalls() == -1) { + if (pymain_check_signals() < 0) { fclose(fp); return pymain_exit_err_print(); } - /* PyRun_AnyFileExFlags(closeit=1) calls fclose(fp) before running code */ + /* _PyRun_AnyFile(closeit=1) calls fclose(fp) before running code */ PyCompilerFlags cf = _PyCompilerFlags_INIT; - int run = _PyRun_AnyFileObject(fp, filename, 1, &cf); - return (run != 0); + PyObject *result = _PyRun_AnyFile(fp, filename, 1, &cf); + if (result == NULL) { + return pymain_exit_err_print(); + } + Py_DECREF(result); + return 0; } static int @@ -408,28 +446,26 @@ pymain_run_file(const PyConfig *config) { PyObject *filename = PyUnicode_FromWideChar(config->run_filename, -1); if (filename == NULL) { - PyErr_Print(); - return -1; + return pymain_exit_err_print(); } PyObject *program_name = PyUnicode_FromWideChar(config->program_name, -1); if (program_name == NULL) { Py_DECREF(filename); - PyErr_Print(); - return -1; + return pymain_exit_err_print(); } - int res = pymain_run_file_obj(program_name, filename, - config->skip_source_first_line); + int exitcode = pymain_run_file_obj(program_name, filename, + config->skip_source_first_line); Py_DECREF(filename); Py_DECREF(program_name); - return res; + return exitcode; } static int pymain_run_startup(PyConfig *config, int *exitcode) { - int ret; + int ret = 0; if (!config->use_environment) { return 0; } @@ -469,10 +505,12 @@ pymain_run_startup(PyConfig *config, int *exitcode) } PyCompilerFlags cf = _PyCompilerFlags_INIT; - (void) _PyRun_SimpleFileObject(fp, startup, 0, &cf); - PyErr_Clear(); + PyObject *result = _PyRun_SimpleFile(fp, startup, 0, &cf); fclose(fp); - ret = 0; + if (result == NULL) { + goto error; + } + Py_DECREF(result); done: Py_XDECREF(startup); @@ -501,6 +539,7 @@ pymain_run_interactive_hook(int *exitcode) } if (PySys_Audit("cpython.run_interactivehook", "O", hook) < 0) { + Py_DECREF(hook); goto error; } @@ -522,14 +561,58 @@ pymain_run_interactive_hook(int *exitcode) static void pymain_set_inspect(PyConfig *config, int inspect) { - config->inspect = inspect; -_Py_COMP_DIAG_PUSH -_Py_COMP_DIAG_IGNORE_DEPR_DECLS - Py_InspectFlag = inspect; -_Py_COMP_DIAG_POP + PyObject *value = PyLong_FromLong(inspect); + if (value == NULL || PyConfig_Set("inspect", value) < 0) { + fprintf(stderr, "Could not set the inspect flag\n"); + PyErr_Print(); + } + else { + assert(config->inspect == inspect); + } + Py_XDECREF(value); } +static int +_pymain_run_repl(PyConfig *config, int startup) +{ + if (pymain_check_signals() < 0) { + return pymain_exit_err_print(); + } + + if (PySys_Audit("cpython.run_stdin", NULL) < 0) { + return pymain_exit_err_print(); + } + + if (isatty(fileno(stdin)) + && !_Py_GetEnv(config->use_environment, "PYTHON_BASIC_REPL")) { + PyObject *pyrepl = PyImport_ImportModule("_pyrepl"); + if (pyrepl != NULL) { + int exitcode = pymain_run_pyrepl(startup); + Py_DECREF(pyrepl); + return exitcode; + } + if (!PyErr_ExceptionMatches(PyExc_ModuleNotFoundError)) { + fprintf(stderr, "Could not import _pyrepl.main\n"); + return pymain_exit_err_print(); + } + PyErr_Clear(); + } + + PyCompilerFlags cf = _PyCompilerFlags_INIT; + PyObject *result = NULL; + PyObject *stdin_obj = PyUnicode_FromString("<stdin>"); + if (stdin_obj != NULL) { + result = _PyRun_AnyFile(stdin, stdin_obj, 0, &cf); + Py_DECREF(stdin_obj); + } + if (result == NULL) { + return pymain_exit_err_print(); + } + Py_DECREF(result); + return 0; +} + static int pymain_run_stdin(PyConfig *config) { @@ -547,22 +630,7 @@ pymain_run_stdin(PyConfig *config) } } - /* call pending calls like signal handlers (SIGINT) */ - if (Py_MakePendingCalls() == -1) { - return pymain_exit_err_print(); - } - - if (PySys_Audit("cpython.run_stdin", NULL) < 0) { - return pymain_exit_err_print(); - } - - if (!isatty(fileno(stdin)) - || _Py_GetEnv(config->use_environment, "PYTHON_BASIC_REPL")) { - PyCompilerFlags cf = _PyCompilerFlags_INIT; - int run = PyRun_AnyFileExFlags(stdin, "<stdin>", 0, &cf); - return (run != 0); - } - return pymain_run_module(L"_pyrepl", 0); + return _pymain_run_repl(config, 0); } @@ -584,33 +652,24 @@ pymain_repl(PyConfig *config, int *exitcode) return; } - if (PySys_Audit("cpython.run_stdin", NULL) < 0) { - return; - } - - if (!isatty(fileno(stdin)) - || _Py_GetEnv(config->use_environment, "PYTHON_BASIC_REPL")) { - PyCompilerFlags cf = _PyCompilerFlags_INIT; - int run = PyRun_AnyFileExFlags(stdin, "<stdin>", 0, &cf); - *exitcode = (run != 0); - return; - } - int run = pymain_start_pyrepl_no_main(); - *exitcode = (run != 0); - return; + *exitcode = _pymain_run_repl(config, 1); } static void pymain_run_python(int *exitcode) { + int set_running_main = 0; + PyObject *main_importer_path = NULL; PyInterpreterState *interp = _PyInterpreterState_GET(); - /* pymain_run_stdin() modify the config */ + /* pymain_repl() and pymain_run_stdin() modify the config */ PyConfig *config = (PyConfig*)_PyInterpreterState_GetConfig(interp); /* ensure path config is written into global variables */ - if (_PyStatus_EXCEPTION(_PyPathConfig_UpdateGlobal(config))) { + PyStatus status = _PyPathConfig_UpdateGlobal(config); + if (_PyStatus_EXCEPTION(status)) { + _PyErr_SetFromPyStatus(status); goto error; } @@ -632,7 +691,9 @@ pymain_run_python(int *exitcode) } // import readline and rlcompleter before script dir is added to sys.path - pymain_import_readline(config); + if (pymain_import_readline(config) < 0) { + goto error; + } PyObject *path0 = NULL; if (main_importer_path != NULL) { @@ -671,8 +732,13 @@ pymain_run_python(int *exitcode) pymain_header(config); _PyInterpreterState_SetRunningMain(interp); + set_running_main = 1; assert(!PyErr_Occurred()); + if (pymain_check_signals() < 0) { + goto error; + } + if (config->run_command) { *exitcode = pymain_run_command(config->run_command); } @@ -689,6 +755,10 @@ pymain_run_python(int *exitcode) *exitcode = pymain_run_stdin(config); } + if (pymain_check_signals() < 0) { + goto error; + } + pymain_repl(config, exitcode); goto done; @@ -696,7 +766,9 @@ pymain_run_python(int *exitcode) *exitcode = pymain_exit_err_print(); done: - _PyInterpreterState_SetNotRunningMain(interp); + if (set_running_main) { + _PyInterpreterState_SetNotRunningMain(interp); + } Py_XDECREF(main_importer_path); } diff --git a/Modules/mathmodule.c b/Modules/mathmodule.c index 11d9b7418a25a21..51efd38892ca88e 100644 --- a/Modules/mathmodule.c +++ b/Modules/mathmodule.c @@ -1793,16 +1793,20 @@ math_isqrt(PyObject *module, PyObject *n) /* The correct result is either a or a - 1. Figure out which, and decrement a if necessary. */ - /* a_too_large = n < a * a */ + /* a_too_large = n < a * a. Compare by value: n can be an instance + of an int subclass with an overridden __lt__ method. */ b = PyNumber_Multiply(a, a); if (b == NULL) { goto error; } - a_too_large = PyObject_RichCompareBool(n, b, Py_LT); + PyObject *cmp = PyLong_Type.tp_richcompare(n, b, Py_LT); Py_DECREF(b); - if (a_too_large == -1) { + if (cmp == NULL) { goto error; } + assert(PyBool_Check(cmp)); + a_too_large = (cmp == Py_True); + Py_DECREF(cmp); if (a_too_large) { Py_SETREF(a, PyNumber_Subtract(a, _PyLong_GetOne())); @@ -2008,13 +2012,11 @@ math.factorial / Find n!. - -Raise a ValueError if x is negative or non-integral. [clinic start generated code]*/ static PyObject * math_factorial(PyObject *module, PyObject *arg) -/*[clinic end generated code: output=6686f26fae00e9ca input=713fb771677e8c31]*/ +/*[clinic end generated code: output=6686f26fae00e9ca input=366cc321df3d4773]*/ { long x, two_valuation; int overflow; @@ -2092,13 +2094,14 @@ math.frexp Return the mantissa and exponent of x, as pair (m, e). -m is a float and e is an int, such that x = m * 2.**e. -If x is 0, m and e are both 0. Else 0.5 <= abs(m) < 1.0. +If x is a finite nonzero number, then m is a float with +0.5 <= abs(m) < 1.0 and an integer e is such that +x == m * 2**e exactly. Else, return (x, 0). [clinic start generated code]*/ static PyObject * math_frexp_impl(PyObject *module, double x) -/*[clinic end generated code: output=03e30d252a15ad4a input=96251c9e208bc6e9]*/ +/*[clinic end generated code: output=03e30d252a15ad4a input=215cf8ea28a0959b]*/ { int i; /* deal with special cases directly, to sidestep platform @@ -2163,6 +2166,27 @@ math_ldexp_impl(PyObject *module, double x, PyObject *i) } else { errno = 0; r = ldexp(x, (int)exp); +#ifdef _MSC_VER + if (DBL_MIN > r && r > -DBL_MIN) { + /* Denormal (or zero) results can be incorrectly rounded here (rather, + truncated). Fixed in newer versions of the C runtime, included + with Windows 11. */ + int original_exp; + frexp(x, &original_exp); + if (original_exp > DBL_MIN_EXP) { + /* Shift down to the smallest normal binade. No bits lost. */ + int shift = DBL_MIN_EXP - original_exp; + x = ldexp(x, shift); + exp -= shift; + } + /* Multiplying by 2**exp finishes the job, and the HW will round as + appropriate. Note: if exp < -DBL_MANT_DIG, all of x is shifted + to be < 0.5ULP of smallest denorm, so should be thrown away. If + exp is so very negative that ldexp underflows to 0, that's fine; + no need to check in advance. */ + r = x*ldexp(1.0, (int)exp); + } +#endif if (isinf(r)) errno = ERANGE; } @@ -3922,13 +3946,13 @@ Return the floating-point value the given number of steps after x towards y. If steps is not specified or is None, it defaults to 1. -Raises a TypeError, if x or y is not a double, or if steps is not an integer. -Raises ValueError if steps is negative. +Raises a TypeError, if x or y is not a double, or if steps is not +an integer. Raises ValueError if steps is negative. [clinic start generated code]*/ static PyObject * math_nextafter_impl(PyObject *module, double x, double y, PyObject *steps) -/*[clinic end generated code: output=cc6511f02afc099e input=7f2a5842112af2b4]*/ +/*[clinic end generated code: output=cc6511f02afc099e input=89764144d1a33160]*/ { #if defined(_AIX) if (x == y) { diff --git a/Modules/md5module.c b/Modules/md5module.c index c36eb41d4d201e9..f3855ec3f37faa1 100644 --- a/Modules/md5module.c +++ b/Modules/md5module.c @@ -87,7 +87,10 @@ static void MD5_dealloc(PyObject *op) { MD5object *ptr = _MD5object_CAST(op); - Hacl_Hash_MD5_free(ptr->hash_state); + if (ptr->hash_state != NULL) { + Hacl_Hash_MD5_free(ptr->hash_state); + ptr->hash_state = NULL; + } PyTypeObject *tp = Py_TYPE(op); PyObject_GC_UnTrack(ptr); PyObject_GC_Del(ptr); @@ -276,17 +279,24 @@ static PyType_Spec md5_type_spec = { /*[clinic input] _md5.md5 - string: object(c_default="NULL") = b'' + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string as string_obj: object(c_default="NULL") = None Return a new MD5 hash object; optionally initialized with a string. [clinic start generated code]*/ static PyObject * -_md5_md5_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=587071f76254a4ac input=7a144a1905636985]*/ +_md5_md5_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj) +/*[clinic end generated code: output=d45e187d3d16f3a8 input=7ea5c5366dbb44bf]*/ { + PyObject *string; + if (_Py_hashlib_data_argument(&string, data, string_obj) < 0) { + return NULL; + } + MD5object *new; Py_buffer buf; diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 6a3855628458497..f7b8ab336fed73e 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -23,8 +23,10 @@ #endif #include <Python.h> +#include "pycore_abstract.h" // _Py_convert_optional_to_ssize_t() #include "pycore_bytesobject.h" // _PyBytes_Find() #include "pycore_fileutils.h" // _Py_stat_struct +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> // offsetof() #ifndef MS_WINDOWS @@ -89,6 +91,12 @@ my_getpagesize(void) # define MAP_ANONYMOUS MAP_ANON #endif +/*[clinic input] +module mmap +class mmap.mmap "mmap_object *" "" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=82a9f8a529905b9b]*/ + typedef enum { ACCESS_DEFAULT, @@ -118,6 +126,7 @@ typedef struct { #ifdef UNIX int fd; _Bool trackfd; + int flags; #endif PyObject *weakreflist; @@ -126,6 +135,27 @@ typedef struct { #define mmap_object_CAST(op) ((mmap_object *)(op)) +#include "clinic/mmapmodule.c.h" + + +/* Return a Py_ssize_t from the object arg. This conversion logic is similar + to what AC uses for `Py_ssize_t` arguments. + + Returns -1 on error. Use PyErr_Occurred() to disambiguate. +*/ +static Py_ssize_t +_As_Py_ssize_t(PyObject *arg) { + assert(arg != NULL); + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(arg); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + return ival; +} + + static int mmap_object_traverse(PyObject *op, visitproc visit, void *arg) { @@ -163,17 +193,22 @@ mmap_object_dealloc(PyObject *op) Py_END_ALLOW_THREADS #endif /* UNIX */ - if (m_obj->weakreflist != NULL) - PyObject_ClearWeakRefs(op); + FT_CLEAR_WEAKREFS(op, m_obj->weakreflist); tp->tp_free(m_obj); Py_DECREF(tp); } +/*[clinic input] +@critical_section +mmap.mmap.close + +[clinic start generated code]*/ + static PyObject * -mmap_close_method(PyObject *op, PyObject *Py_UNUSED(ignored)) +mmap_mmap_close_impl(mmap_object *self) +/*[clinic end generated code: output=a1ae0c727546f78d input=25020035f047eae1]*/ { - mmap_object *self = mmap_object_CAST(op); if (self->exports > 0) { PyErr_SetString(PyExc_BufferError, "cannot close "\ "exported pointers exist"); @@ -461,10 +496,16 @@ _safe_PyBytes_FromStringAndSize(char *start, size_t num_bytes) { } } +/*[clinic input] +@critical_section +mmap.mmap.read_byte + +[clinic start generated code]*/ + static PyObject * -mmap_read_byte_method(PyObject *op, PyObject *Py_UNUSED(ignored)) +mmap_mmap_read_byte_impl(mmap_object *self) +/*[clinic end generated code: output=d931da1319f3869b input=5b8c6a904bdddda9]*/ { - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); if (self->pos >= self->size) { PyErr_SetString(PyExc_ValueError, "read byte out of range"); @@ -478,12 +519,18 @@ mmap_read_byte_method(PyObject *op, PyObject *Py_UNUSED(ignored)) return PyLong_FromLong((unsigned char) dest); } +/*[clinic input] +@critical_section +mmap.mmap.readline + +[clinic start generated code]*/ + static PyObject * -mmap_read_line_method(PyObject *op, PyObject *Py_UNUSED(ignored)) +mmap_mmap_readline_impl(mmap_object *self) +/*[clinic end generated code: output=b9d2bf9999283311 input=2c4efd1d06e1cdd1]*/ { Py_ssize_t remaining; char *start, *eol; - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); @@ -508,16 +555,22 @@ mmap_read_line_method(PyObject *op, PyObject *Py_UNUSED(ignored)) return result; } +/*[clinic input] +@critical_section +mmap.mmap.read + + n as num_bytes: object(converter='_Py_convert_optional_to_ssize_t', type='Py_ssize_t', c_default='PY_SSIZE_T_MAX') = None + / + +[clinic start generated code]*/ + static PyObject * -mmap_read_method(PyObject *op, PyObject *args) +mmap_mmap_read_impl(mmap_object *self, Py_ssize_t num_bytes) +/*[clinic end generated code: output=3b4d4f3704ed0969 input=8f97f361d435e357]*/ { - Py_ssize_t num_bytes = PY_SSIZE_T_MAX, remaining; - mmap_object *self = mmap_object_CAST(op); + Py_ssize_t remaining; CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "|n?:read", &num_bytes)) - return NULL; - CHECK_VALID(NULL); /* silently 'adjust' out-of-range requests */ remaining = (self->pos < self->size) ? self->size - self->pos : 0; @@ -533,81 +586,105 @@ mmap_read_method(PyObject *op, PyObject *args) } static PyObject * -mmap_gfind(mmap_object *self, - PyObject *args, - int reverse) +mmap_gfind_lock_held(mmap_object *self, Py_buffer *view, PyObject *start_obj, + PyObject *end_obj, int reverse) { Py_ssize_t start = self->pos; Py_ssize_t end = self->size; - Py_buffer view; CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, reverse ? "y*|nn:rfind" : "y*|nn:find", - &view, &start, &end)) { - return NULL; - } - else { - if (start < 0) - start += self->size; - if (start < 0) - start = 0; - else if (start > self->size) - start = self->size; - - if (end < 0) - end += self->size; - if (end < 0) - end = 0; - else if (end > self->size) - end = self->size; - - Py_ssize_t index; - PyObject *result; - CHECK_VALID_OR_RELEASE(NULL, view); - if (end < start) { - result = PyLong_FromSsize_t(-1); + if (start_obj != Py_None) { + start = _As_Py_ssize_t(start_obj); + if (start == -1 && PyErr_Occurred()) { + return NULL; } - else if (reverse) { - assert(0 <= start && start <= end && end <= self->size); - if (_safe_PyBytes_ReverseFind(&index, self, - self->data + start, end - start, - view.buf, view.len, start) < 0) - { - result = NULL; - } - else { - result = PyLong_FromSsize_t(index); + + if (end_obj != Py_None) { + end = _As_Py_ssize_t(end_obj); + if (end == -1 && PyErr_Occurred()) { + return NULL; } } + } + + if (start < 0) + start += self->size; + if (start < 0) + start = 0; + else if (start > self->size) + start = self->size; + + if (end < 0) + end += self->size; + if (end < 0) + end = 0; + else if (end > self->size) + end = self->size; + + Py_ssize_t index; + PyObject *result; + CHECK_VALID(NULL); + if (end < start) { + result = PyLong_FromSsize_t(-1); + } + else if (reverse) { + assert(0 <= start && start <= end && end <= self->size); + if (_safe_PyBytes_ReverseFind(&index, self, + self->data + start, end - start, + view->buf, view->len, start) < 0) + { + result = NULL; + } else { - assert(0 <= start && start <= end && end <= self->size); - if (_safe_PyBytes_Find(&index, self, - self->data + start, end - start, - view.buf, view.len, start) < 0) - { - result = NULL; - } - else { - result = PyLong_FromSsize_t(index); - } + result = PyLong_FromSsize_t(index); } - PyBuffer_Release(&view); - return result; } + else { + assert(0 <= start && start <= end && end <= self->size); + if (_safe_PyBytes_Find(&index, self, + self->data + start, end - start, + view->buf, view->len, start) < 0) + { + result = NULL; + } + else { + result = PyLong_FromSsize_t(index); + } + } + return result; } +/*[clinic input] +@critical_section +mmap.mmap.find + + view: Py_buffer + start: object = None + end: object = None + / + +[clinic start generated code]*/ + static PyObject * -mmap_find_method(PyObject *op, PyObject *args) +mmap_mmap_find_impl(mmap_object *self, Py_buffer *view, PyObject *start, + PyObject *end) +/*[clinic end generated code: output=ef8878a322f00192 input=0135504494b52c2b]*/ { - mmap_object *self = mmap_object_CAST(op); - return mmap_gfind(self, args, 0); + return mmap_gfind_lock_held(self, view, start, end, 0); } +/*[clinic input] +@critical_section +mmap.mmap.rfind = mmap.mmap.find + +[clinic start generated code]*/ + static PyObject * -mmap_rfind_method(PyObject *op, PyObject *args) +mmap_mmap_rfind_impl(mmap_object *self, Py_buffer *view, PyObject *start, + PyObject *end) +/*[clinic end generated code: output=73b918940d67c2b8 input=8aecdd1f70c06c62]*/ { - mmap_object *self = mmap_object_CAST(op); - return mmap_gfind(self, args, 1); + return mmap_gfind_lock_held(self, view, start, end, 1); } static int @@ -643,50 +720,55 @@ is_resizeable(mmap_object *self) } +/*[clinic input] +@critical_section +mmap.mmap.write + + bytes as data: Py_buffer + / + +[clinic start generated code]*/ + static PyObject * -mmap_write_method(PyObject *op, PyObject *args) +mmap_mmap_write_impl(mmap_object *self, Py_buffer *data) +/*[clinic end generated code: output=9e97063efb6fb27b input=3f16fa79aa89d6f7]*/ { - Py_buffer data; - mmap_object *self = mmap_object_CAST(op); - CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "y*:write", &data)) - return NULL; - if (!is_writable(self)) { - PyBuffer_Release(&data); return NULL; } - if (self->pos > self->size || self->size - self->pos < data.len) { - PyBuffer_Release(&data); + if (self->pos > self->size || self->size - self->pos < data->len) { PyErr_SetString(PyExc_ValueError, "data out of range"); return NULL; } - CHECK_VALID_OR_RELEASE(NULL, data); + CHECK_VALID(NULL); PyObject *result; - if (safe_memcpy(self->data + self->pos, data.buf, data.len) < 0) { + if (safe_memcpy(self->data + self->pos, data->buf, data->len) < 0) { result = NULL; } else { - self->pos += data.len; - result = PyLong_FromSsize_t(data.len); + self->pos += data->len; + result = PyLong_FromSsize_t(data->len); } - PyBuffer_Release(&data); return result; } +/*[clinic input] +@critical_section +mmap.mmap.write_byte + + byte as value: unsigned_char + / + +[clinic start generated code]*/ + static PyObject * -mmap_write_byte_method(PyObject *op, PyObject *args) +mmap_mmap_write_byte_impl(mmap_object *self, unsigned char value) +/*[clinic end generated code: output=aa11adada9b17510 input=32740bfa174f0991]*/ { - char value; - mmap_object *self = mmap_object_CAST(op); - CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "b:write_byte", &value)) - return(NULL); - if (!is_writable(self)) return NULL; @@ -696,17 +778,23 @@ mmap_write_byte_method(PyObject *op, PyObject *args) return NULL; } - if (safe_byte_copy(self->data + self->pos, &value) < 0) { + if (safe_byte_copy(self->data + self->pos, (const char*)&value) < 0) { return NULL; } self->pos++; Py_RETURN_NONE; } +/*[clinic input] +@critical_section +mmap.mmap.size + +[clinic start generated code]*/ + static PyObject * -mmap_size_method(PyObject *op, PyObject *Py_UNUSED(ignored)) +mmap_mmap_size_impl(mmap_object *self) +/*[clinic end generated code: output=c177e65e83a648ff input=f69c072efd2e1595]*/ { - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); #ifdef MS_WINDOWS @@ -753,14 +841,21 @@ mmap_size_method(PyObject *op, PyObject *Py_UNUSED(ignored)) / new size? */ +/*[clinic input] +@critical_section +mmap.mmap.resize + + newsize as new_size: Py_ssize_t + / + +[clinic start generated code]*/ + static PyObject * -mmap_resize_method(PyObject *op, PyObject *args) +mmap_mmap_resize_impl(mmap_object *self, Py_ssize_t new_size) +/*[clinic end generated code: output=6f262537ce9c2dcc input=b6b5dee52a41b79f]*/ { - Py_ssize_t new_size; - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "n:resize", &new_size) || - !is_resizeable(self)) { + if (!is_resizeable(self)) { return NULL; } if (new_size < 0 || PY_SSIZE_T_MAX - new_size < self->offset) { @@ -874,6 +969,16 @@ mmap_resize_method(PyObject *op, PyObject *args) #else void *newmap; +#if defined(__linux__) || defined(__NetBSD__) + // Linux mremap() refuses to grow a shared anonymous mapping, and + // NetBSD mremap() returns a mapping whose grown region is not backed, + // so accessing it crashes. Reject it here in both cases. + if (self->fd == -1 && !(self->flags & MAP_PRIVATE) && new_size > self->size) { + PyErr_Format(PyExc_ValueError, + "mmap: can't expand a shared anonymous mapping"); + return NULL; + } +#endif if (self->fd != -1 && ftruncate(self->fd, self->offset + new_size) == -1) { PyErr_SetFromErrno(PyExc_OSError); return NULL; @@ -901,23 +1006,45 @@ mmap_resize_method(PyObject *op, PyObject *args) } } +/*[clinic input] +@critical_section +mmap.mmap.tell + +[clinic start generated code]*/ + static PyObject * -mmap_tell_method(PyObject *op, PyObject *Py_UNUSED(ignored)) +mmap_mmap_tell_impl(mmap_object *self) +/*[clinic end generated code: output=6034958630e1b1d1 input=fd163acacf45c3a5]*/ { - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); return PyLong_FromSize_t(self->pos); } +/*[clinic input] +@critical_section +mmap.mmap.flush + + offset: Py_ssize_t = 0 + size as size_obj: object = None + / + +[clinic start generated code]*/ + static PyObject * -mmap_flush_method(PyObject *op, PyObject *args) +mmap_mmap_flush_impl(mmap_object *self, Py_ssize_t offset, + PyObject *size_obj) +/*[clinic end generated code: output=41a10c349ed1608a input=7e6bb8f9462f53e6]*/ { - Py_ssize_t offset = 0; - mmap_object *self = mmap_object_CAST(op); Py_ssize_t size = self->size; CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "|nn:flush", &offset, &size)) - return NULL; + + if (size_obj != Py_None) { + size = _As_Py_ssize_t(size_obj); + if (size == -1 && PyErr_Occurred()) { + return NULL; + } + } + if (size < 0 || offset < 0 || self->size - offset < size) { PyErr_SetString(PyExc_ValueError, "flush values out of range"); return NULL; @@ -945,60 +1072,80 @@ mmap_flush_method(PyObject *op, PyObject *args) #endif } +/*[clinic input] +@critical_section +mmap.mmap.seek + + pos as dist: Py_ssize_t + whence as how: int = 0 + / + +[clinic start generated code]*/ + static PyObject * -mmap_seek_method(PyObject *op, PyObject *args) +mmap_mmap_seek_impl(mmap_object *self, Py_ssize_t dist, int how) +/*[clinic end generated code: output=00310494e8b8c592 input=e2fda5d081c3db22]*/ { - Py_ssize_t dist; - mmap_object *self = mmap_object_CAST(op); - int how=0; CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "n|i:seek", &dist, &how)) - return NULL; - else { - Py_ssize_t where; - switch (how) { - case 0: /* relative to start */ - where = dist; - break; - case 1: /* relative to current position */ - if (PY_SSIZE_T_MAX - self->pos < dist) - goto onoutofrange; - where = self->pos + dist; - break; - case 2: /* relative to end */ - if (PY_SSIZE_T_MAX - self->size < dist) - goto onoutofrange; - where = self->size + dist; - break; - default: - PyErr_SetString(PyExc_ValueError, "unknown seek type"); - return NULL; - } - if (where > self->size || where < 0) + Py_ssize_t where; + switch (how) { + case 0: /* relative to start */ + where = dist; + break; + case 1: /* relative to current position */ + if (PY_SSIZE_T_MAX - self->pos < dist) + goto onoutofrange; + where = self->pos + dist; + break; + case 2: /* relative to end */ + if (PY_SSIZE_T_MAX - self->size < dist) goto onoutofrange; - self->pos = where; - return PyLong_FromSsize_t(self->pos); + where = self->size + dist; + break; + default: + PyErr_SetString(PyExc_ValueError, "unknown seek type"); + return NULL; } + if (where > self->size || where < 0) + goto onoutofrange; + self->pos = where; + return PyLong_FromSsize_t(self->pos); onoutofrange: PyErr_SetString(PyExc_ValueError, "seek out of range"); return NULL; } +/*[clinic input] +mmap.mmap.seekable + +[clinic start generated code]*/ + static PyObject * -mmap_seekable_method(PyObject *op, PyObject *Py_UNUSED(ignored)) +mmap_mmap_seekable_impl(mmap_object *self) +/*[clinic end generated code: output=6311dc3ea300fa38 input=5132505f6e259001]*/ { Py_RETURN_TRUE; } +/*[clinic input] +@critical_section +mmap.mmap.move + + dest: Py_ssize_t + src: Py_ssize_t + count as cnt: Py_ssize_t + / + +[clinic start generated code]*/ + static PyObject * -mmap_move_method(PyObject *op, PyObject *args) +mmap_mmap_move_impl(mmap_object *self, Py_ssize_t dest, Py_ssize_t src, + Py_ssize_t cnt) +/*[clinic end generated code: output=391f549a44181793 input=cf8cfe10d9f6b448]*/ { - Py_ssize_t dest, src, cnt; - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "nnn:move", &dest, &src, &cnt) || - !is_writable(self)) { + if (!is_writable(self)) { return NULL; } else { /* bounds check the values */ @@ -1024,30 +1171,53 @@ static PyObject * mmap_closed_get(PyObject *op, void *Py_UNUSED(closure)) { mmap_object *self = mmap_object_CAST(op); + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); #ifdef MS_WINDOWS - return PyBool_FromLong(self->map_handle == NULL ? 1 : 0); + result = PyBool_FromLong(self->map_handle == NULL ? 1 : 0); #elif defined(UNIX) - return PyBool_FromLong(self->data == NULL ? 1 : 0); + result = PyBool_FromLong(self->data == NULL ? 1 : 0); #endif + Py_END_CRITICAL_SECTION(); + return result; } +/*[clinic input] +@critical_section +mmap.mmap.__enter__ + +[clinic start generated code]*/ + static PyObject * -mmap__enter__method(PyObject *op, PyObject *Py_UNUSED(ignored)) +mmap_mmap___enter___impl(mmap_object *self) +/*[clinic end generated code: output=92cfc59f4c4e2d26 input=a446541fbfe0b890]*/ { - mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); return Py_NewRef(self); } +/*[clinic input] +@critical_section +mmap.mmap.__exit__ + + exc_type: object + exc_value: object + traceback: object + / + +[clinic start generated code]*/ + static PyObject * -mmap__exit__method(PyObject *op, PyObject *Py_UNUSED(args)) +mmap_mmap___exit___impl(mmap_object *self, PyObject *exc_type, + PyObject *exc_value, PyObject *traceback) +/*[clinic end generated code: output=bec7e3e319c1f07e input=5f28e91cf752bc64]*/ { - return mmap_close_method(op, NULL); + return mmap_mmap_close_impl(self); } static PyObject * -mmap__repr__method(PyObject *op) +mmap__repr__method_lock_held(PyObject *op) { mmap_object *mobj = mmap_object_CAST(op); @@ -1091,11 +1261,27 @@ mmap__repr__method(PyObject *op) } } +static PyObject * +mmap__repr__method(PyObject *op) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = mmap__repr__method_lock_held(op); + Py_END_CRITICAL_SECTION(); + return result; +} + #ifdef MS_WINDOWS +/*[clinic input] +@critical_section +mmap.mmap.__sizeof__ + +[clinic start generated code]*/ + static PyObject * -mmap__sizeof__method(PyObject *op, PyObject *Py_UNUSED(dummy)) +mmap_mmap___sizeof___impl(mmap_object *self) +/*[clinic end generated code: output=1aed30daff807d09 input=8a648868a089553c]*/ { - mmap_object *self = mmap_object_CAST(op); size_t res = _PyObject_SIZE(Py_TYPE(self)); if (self->tagname) { res += (wcslen(self->tagname) + 1) * sizeof(self->tagname[0]); @@ -1105,18 +1291,26 @@ mmap__sizeof__method(PyObject *op, PyObject *Py_UNUSED(dummy)) #endif #if defined(MS_WINDOWS) && defined(Py_DEBUG) +/*[clinic input] +@critical_section +mmap.mmap._protect + + flNewProtect: unsigned_int(bitwise=True) + start: Py_ssize_t + length: Py_ssize_t + / + +[clinic start generated code]*/ + static PyObject * -mmap_protect_method(PyObject *op, PyObject *args) { - DWORD flNewProtect, flOldProtect; - Py_ssize_t start, length; - mmap_object *self = mmap_object_CAST(op); +mmap_mmap__protect_impl(mmap_object *self, unsigned int flNewProtect, + Py_ssize_t start, Py_ssize_t length) +/*[clinic end generated code: output=a87271a34d1ad6cf input=9170498c5e1482da]*/ +{ + DWORD flOldProtect; CHECK_VALID(NULL); - if (!PyArg_ParseTuple(args, "Inn:protect", &flNewProtect, &start, &length)) { - return NULL; - } - if (!VirtualProtect((void *) (self->data + start), length, flNewProtect, &flOldProtect)) { @@ -1129,18 +1323,32 @@ mmap_protect_method(PyObject *op, PyObject *args) { #endif #ifdef HAVE_MADVISE +/*[clinic input] +@critical_section +mmap.mmap.madvise + + option: int + start: Py_ssize_t = 0 + length as length_obj: object = None + / + +[clinic start generated code]*/ + static PyObject * -mmap_madvise_method(PyObject *op, PyObject *args) +mmap_mmap_madvise_impl(mmap_object *self, int option, Py_ssize_t start, + PyObject *length_obj) +/*[clinic end generated code: output=816be656f08c0e3c input=2d37f7a4c87f1053]*/ { - int option; - Py_ssize_t start = 0, length; - mmap_object *self = mmap_object_CAST(op); + Py_ssize_t length; CHECK_VALID(NULL); - length = self->size; - - if (!PyArg_ParseTuple(args, "i|nn:madvise", &option, &start, &length)) { - return NULL; + if (length_obj == Py_None) { + length = self->size; + } else { + length = _As_Py_ssize_t(length_obj); + if (length == -1 && PyErr_Occurred()) { + return NULL; + } } if (start < 0 || start >= self->size) { @@ -1176,32 +1384,26 @@ static struct PyMemberDef mmap_object_members[] = { }; static struct PyMethodDef mmap_object_methods[] = { - {"close", mmap_close_method, METH_NOARGS}, - {"find", mmap_find_method, METH_VARARGS}, - {"rfind", mmap_rfind_method, METH_VARARGS}, - {"flush", mmap_flush_method, METH_VARARGS}, -#ifdef HAVE_MADVISE - {"madvise", mmap_madvise_method, METH_VARARGS}, -#endif - {"move", mmap_move_method, METH_VARARGS}, - {"read", mmap_read_method, METH_VARARGS}, - {"read_byte", mmap_read_byte_method, METH_NOARGS}, - {"readline", mmap_read_line_method, METH_NOARGS}, - {"resize", mmap_resize_method, METH_VARARGS}, - {"seek", mmap_seek_method, METH_VARARGS}, - {"seekable", mmap_seekable_method, METH_NOARGS}, - {"size", mmap_size_method, METH_NOARGS}, - {"tell", mmap_tell_method, METH_NOARGS}, - {"write", mmap_write_method, METH_VARARGS}, - {"write_byte", mmap_write_byte_method, METH_VARARGS}, - {"__enter__", mmap__enter__method, METH_NOARGS}, - {"__exit__", mmap__exit__method, METH_VARARGS}, -#ifdef MS_WINDOWS - {"__sizeof__", mmap__sizeof__method, METH_NOARGS}, -#ifdef Py_DEBUG - {"_protect", mmap_protect_method, METH_VARARGS}, -#endif // Py_DEBUG -#endif // MS_WINDOWS + MMAP_MMAP_CLOSE_METHODDEF + MMAP_MMAP_FIND_METHODDEF + MMAP_MMAP_RFIND_METHODDEF + MMAP_MMAP_FLUSH_METHODDEF + MMAP_MMAP_MADVISE_METHODDEF + MMAP_MMAP_MOVE_METHODDEF + MMAP_MMAP_READ_METHODDEF + MMAP_MMAP_READ_BYTE_METHODDEF + MMAP_MMAP_READLINE_METHODDEF + MMAP_MMAP_RESIZE_METHODDEF + MMAP_MMAP_SEEK_METHODDEF + MMAP_MMAP_SEEKABLE_METHODDEF + MMAP_MMAP_SIZE_METHODDEF + MMAP_MMAP_TELL_METHODDEF + MMAP_MMAP_WRITE_METHODDEF + MMAP_MMAP_WRITE_BYTE_METHODDEF + MMAP_MMAP___ENTER___METHODDEF + MMAP_MMAP___EXIT___METHODDEF + MMAP_MMAP___SIZEOF___METHODDEF + MMAP_MMAP__PROTECT_METHODDEF {NULL, NULL} /* sentinel */ }; @@ -1214,7 +1416,7 @@ static PyGetSetDef mmap_object_getset[] = { /* Functions for treating an mmap'ed file as a buffer */ static int -mmap_buffer_getbuf(PyObject *op, Py_buffer *view, int flags) +mmap_buffer_getbuf_lock_held(PyObject *op, Py_buffer *view, int flags) { mmap_object *self = mmap_object_CAST(op); CHECK_VALID(-1); @@ -1225,23 +1427,45 @@ mmap_buffer_getbuf(PyObject *op, Py_buffer *view, int flags) return 0; } +static int +mmap_buffer_getbuf(PyObject *op, Py_buffer *view, int flags) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(op); + result = mmap_buffer_getbuf_lock_held(op, view, flags); + Py_END_CRITICAL_SECTION(); + return result; +} + static void mmap_buffer_releasebuf(PyObject *op, Py_buffer *Py_UNUSED(view)) { mmap_object *self = mmap_object_CAST(op); + Py_BEGIN_CRITICAL_SECTION(self); self->exports--; + Py_END_CRITICAL_SECTION(); } static Py_ssize_t -mmap_length(PyObject *op) +mmap_length_lock_held(PyObject *op) { mmap_object *self = mmap_object_CAST(op); CHECK_VALID(-1); return self->size; } +static Py_ssize_t +mmap_length(PyObject *op) +{ + Py_ssize_t result; + Py_BEGIN_CRITICAL_SECTION(op); + result = mmap_length_lock_held(op); + Py_END_CRITICAL_SECTION(); + return result; +} + static PyObject * -mmap_item(PyObject *op, Py_ssize_t i) +mmap_item_lock_held(PyObject *op, Py_ssize_t i) { mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); @@ -1258,7 +1482,16 @@ mmap_item(PyObject *op, Py_ssize_t i) } static PyObject * -mmap_subscript(PyObject *op, PyObject *item) +mmap_item(PyObject *op, Py_ssize_t i) { + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = mmap_item_lock_held(op, i); + Py_END_CRITICAL_SECTION(); + return result; +} + +static PyObject * +mmap_subscript_lock_held(PyObject *op, PyObject *item) { mmap_object *self = mmap_object_CAST(op); CHECK_VALID(NULL); @@ -1320,8 +1553,18 @@ mmap_subscript(PyObject *op, PyObject *item) } } +static PyObject * +mmap_subscript(PyObject *op, PyObject *item) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(op); + result = mmap_subscript_lock_held(op, item); + Py_END_CRITICAL_SECTION(); + return result; +} + static int -mmap_ass_item(PyObject *op, Py_ssize_t i, PyObject *v) +mmap_ass_item_lock_held(PyObject *op, Py_ssize_t i, PyObject *v) { const char *buf; mmap_object *self = mmap_object_CAST(op); @@ -1352,7 +1595,17 @@ mmap_ass_item(PyObject *op, Py_ssize_t i, PyObject *v) } static int -mmap_ass_subscript(PyObject *op, PyObject *item, PyObject *value) +mmap_ass_item(PyObject *op, Py_ssize_t i, PyObject *v) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(op); + result = mmap_ass_item_lock_held(op, i, v); + Py_END_CRITICAL_SECTION(); + return result; +} + +static int +mmap_ass_subscript_lock_held(PyObject *op, PyObject *item, PyObject *value) { mmap_object *self = mmap_object_CAST(op); CHECK_VALID(-1); @@ -1448,6 +1701,16 @@ mmap_ass_subscript(PyObject *op, PyObject *item, PyObject *value) } } +static int +mmap_ass_subscript(PyObject *op, PyObject *item, PyObject *value) +{ + int result; + Py_BEGIN_CRITICAL_SECTION(op); + result = mmap_ass_subscript_lock_held(op, item, value); + Py_END_CRITICAL_SECTION(); + return result; +} + static PyObject * new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict); @@ -1670,6 +1933,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) else { m_obj->fd = -1; } + m_obj->flags = flags; Py_BEGIN_ALLOW_THREADS m_obj->data = mmap(NULL, map_size, prot, flags, fd, offset); @@ -1709,7 +1973,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) DWORD off_lo; /* lower 32 bits of offset */ DWORD size_hi; /* upper 32 bits of size */ DWORD size_lo; /* lower 32 bits of size */ - PyObject *tagname = NULL; + PyObject *tagname = Py_None; DWORD dwErr = 0; int fileno; HANDLE fh = 0; @@ -1719,7 +1983,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) "tagname", "access", "offset", NULL }; - if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|U?iL", keywords, + if (!PyArg_ParseTupleAndKeywords(args, kwdict, "in|OiL", keywords, &fileno, &map_size, &tagname, &access, &offset)) { return NULL; @@ -1852,7 +2116,13 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) m_obj->weakreflist = NULL; m_obj->exports = 0; /* set the tag name */ - if (tagname != NULL) { + if (!Py_IsNone(tagname)) { + if (!PyUnicode_Check(tagname)) { + Py_DECREF(m_obj); + return PyErr_Format(PyExc_TypeError, "expected str or None for " + "'tagname', not %.200s", + Py_TYPE(tagname)->tp_name); + } m_obj->tagname = PyUnicode_AsWideCharString(tagname, NULL); if (m_obj->tagname == NULL) { Py_DECREF(m_obj); diff --git a/Modules/overlapped.c b/Modules/overlapped.c index 29b7b356648a539..bf29afabb955730 100644 --- a/Modules/overlapped.c +++ b/Modules/overlapped.c @@ -884,13 +884,14 @@ _overlapped.Overlapped.getresult Retrieve result of operation. -If wait is true then it blocks until the operation is finished. If wait -is false and the operation is still pending then an error is raised. +If wait is true then it blocks until the operation is finished. If +wait is false and the operation is still pending then an error is +raised. [clinic start generated code]*/ static PyObject * _overlapped_Overlapped_getresult_impl(OverlappedObject *self, BOOL wait) -/*[clinic end generated code: output=8c9bd04d08994f6c input=aa5b03e9897ca074]*/ +/*[clinic end generated code: output=8c9bd04d08994f6c input=852fbd817cbd2b3d]*/ { DWORD transferred = 0; BOOL ret; @@ -1806,7 +1807,7 @@ _overlapped_Overlapped_WSASendTo_impl(OverlappedObject *self, HANDLE handle, case ERROR_IO_PENDING: Py_RETURN_NONE; default: - self->type = TYPE_NOT_STARTED; + Overlapped_clear(self); return SetFromWindowsErr(err); } } @@ -1873,7 +1874,7 @@ _overlapped_Overlapped_WSARecvFrom_impl(OverlappedObject *self, case ERROR_IO_PENDING: Py_RETURN_NONE; default: - self->type = TYPE_NOT_STARTED; + Overlapped_clear(self); return SetFromWindowsErr(err); } } @@ -1914,6 +1915,11 @@ _overlapped_Overlapped_WSARecvFromInto_impl(OverlappedObject *self, } #endif + if (bufobj->len < (Py_ssize_t)size) { + PyErr_SetString(PyExc_ValueError, "nbytes is greater than the length of the buffer"); + return NULL; + } + wsabuf.buf = bufobj->buf; wsabuf.len = size; @@ -1940,7 +1946,7 @@ _overlapped_Overlapped_WSARecvFromInto_impl(OverlappedObject *self, case ERROR_IO_PENDING: Py_RETURN_NONE; default: - self->type = TYPE_NOT_STARTED; + Overlapped_clear(self); return SetFromWindowsErr(err); } } diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 922694fa367ac3c..26e62a1a7c50dc8 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -685,7 +685,16 @@ static void reset_remotedebug_data(PyThreadState *tstate) { tstate->remote_debugger_support.debugger_pending_call = 0; - memset(tstate->remote_debugger_support.debugger_script_path, 0, MAX_SCRIPT_PATH_SIZE); + memset(tstate->remote_debugger_support.debugger_script_path, 0, + _Py_MAX_SCRIPT_PATH_SIZE); +} + +static void +reset_asyncio_state(_PyThreadStateImpl *tstate) +{ + llist_init(&tstate->asyncio_tasks_head); + tstate->asyncio_running_loop = NULL; + tstate->asyncio_running_task = NULL; } @@ -724,6 +733,8 @@ PyOS_AfterFork_Child(void) reset_remotedebug_data(tstate); + reset_asyncio_state((_PyThreadStateImpl *)tstate); + // Remove the dead thread states. We "start the world" once we are the only // thread state left to undo the stop the world call in `PyOS_BeforeFork`. // That needs to happen before `_PyThreadState_DeleteList`, because that @@ -1240,6 +1251,8 @@ get_posix_state(PyObject *module) * Contains a file descriptor if path.accept_fd was true * and the caller provided a signed integer instead of any * sort of string. + * path.is_fd + * True if path was provided as a file descriptor. * * WARNING: if your "path" parameter is optional, and is * unspecified, path_converter will never get called. @@ -1292,6 +1305,7 @@ typedef struct { const wchar_t *wide; const char *narrow; int fd; + bool is_fd; int value_error; Py_ssize_t length; PyObject *object; @@ -1301,7 +1315,7 @@ typedef struct { #define PATH_T_INITIALIZE(function_name, argument_name, nullable, nonstrict, \ make_wide, suppress_value_error, allow_fd) \ {function_name, argument_name, nullable, nonstrict, make_wide, \ - suppress_value_error, allow_fd, NULL, NULL, -1, 0, 0, NULL, NULL} + suppress_value_error, allow_fd, NULL, NULL, -1, false, 0, 0, NULL, NULL} #ifdef MS_WINDOWS #define PATH_T_INITIALIZE_P(function_name, argument_name, nullable, \ nonstrict, suppress_value_error, allow_fd) \ @@ -1435,6 +1449,7 @@ path_converter(PyObject *o, void *p) } path->wide = NULL; path->narrow = NULL; + path->is_fd = true; goto success_exit; } else { @@ -1597,7 +1612,7 @@ static int fd_and_follow_symlinks_invalid(const char *function_name, int fd, int follow_symlinks) { - if ((fd > 0) && (!follow_symlinks)) { + if ((fd >= 0) && (!follow_symlinks)) { PyErr_Format(PyExc_ValueError, "%s: cannot use fd and follow_symlinks together", function_name); @@ -1775,7 +1790,7 @@ convertenviron(void) #ifdef MS_WINDOWS k = PyUnicode_FromWideChar(*e, (Py_ssize_t)(p-*e)); #else - k = PyBytes_FromStringAndSize(*e, (int)(p-*e)); + k = PyBytes_FromStringAndSize(*e, (Py_ssize_t)(p-*e)); #endif if (k == NULL) { Py_DECREF(d); @@ -2312,7 +2327,7 @@ PyDoc_STRVAR(stat_result__doc__, "stat_result: Result from stat, fstat, or lstat.\n\n\ This object may be accessed either as a tuple of\n\ (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime)\n\ -or via the attributes st_mode, st_ino, st_dev, st_nlink, st_uid, and so on.\n\ +or via the attributes st_mode, st_ino, st_dev, st_nlink, and so on.\n\ \n\ Posix/windows: If your platform supports st_blksize, st_blocks, st_rdev,\n\ or st_flags, they are available as attributes only.\n\ @@ -2743,7 +2758,7 @@ _pystat_fromstructstat(PyObject *module, STRUCT_STAT *st) SET_ITEM(ST_BLOCKS_IDX, PyLong_FromLong((long)st->st_blocks)); #endif #ifdef HAVE_STRUCT_STAT_ST_RDEV - SET_ITEM(ST_RDEV_IDX, PyLong_FromLong((long)st->st_rdev)); + SET_ITEM(ST_RDEV_IDX, _PyLong_FromDev(st->st_rdev)); #endif #ifdef HAVE_STRUCT_STAT_ST_GEN SET_ITEM(ST_GEN_IDX, PyLong_FromLong((long)st->st_gen)); @@ -3042,25 +3057,22 @@ class path_t_converter(CConverter): type = "path_t" impl_by_reference = True parse_by_reference = True + default_type = () + c_init_default = "<placeholder>" # overridden in pre_render() converter = 'path_converter' def converter_init(self, *, allow_fd=False, make_wide=None, nonstrict=False, nullable=False, suppress_value_error=False): - # right now path_t doesn't support default values. - # to support a default value, you'll need to override initialize(). - if self.default not in (unspecified, None): - fail("Can't specify a default to the path_t converter!") - - if self.c_default not in (None, 'Py_None'): - raise RuntimeError("Can't specify a c_default to the path_t converter!") self.nullable = nullable self.nonstrict = nonstrict self.make_wide = make_wide self.suppress_value_error = suppress_value_error self.allow_fd = allow_fd + if nullable: + self.default_type = NoneType def pre_render(self): def strify(value): @@ -3095,6 +3107,8 @@ class path_t_converter(CConverter): class dir_fd_converter(CConverter): type = 'int' + default_type = NoneType + c_init_default = 'DEFAULT_DIR_FD' def converter_init(self, requires=None): if self.default in (unspecified, None): @@ -3104,6 +3118,9 @@ class dir_fd_converter(CConverter): else: self.converter = 'dir_fd_converter' + def c_default_init(self): + self.c_default = 'DEFAULT_DIR_FD' + class uid_t_converter(CConverter): type = "uid_t" converter = '_Py_Uid_Converter' @@ -3121,17 +3138,6 @@ class dev_t_return_converter(unsigned_long_return_converter): conversion_fn = '_PyLong_FromDev' unsigned_cast = '(dev_t)' -class FSConverter_converter(CConverter): - type = 'PyObject *' - converter = 'PyUnicode_FSConverter' - def converter_init(self): - if self.default is not unspecified: - fail("FSConverter_converter does not support default values") - self.c_default = 'NULL' - - def cleanup(self): - return "Py_XDECREF(" + self.name + ");\n" - class pid_t_converter(CConverter): type = 'pid_t' format_unit = '" _Py_PARSE_PID "' @@ -3195,7 +3201,7 @@ class confname_converter(CConverter): """, argname=argname, converter=self.converter, table=self.table) [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=8189d5ae78244626]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=ddbf3ac90a981122]*/ /*[clinic input] @@ -3474,14 +3480,14 @@ os.chdir Change the current working directory to the specified path. -path may always be specified as a string. -On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. +path may always be specified as a string. On some platforms, path may +also be specified as an open file descriptor. If this functionality is +unavailable, using it raises an exception. [clinic start generated code]*/ static PyObject * os_chdir_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=3be6400eee26eaae input=1a4a15b4d12cb15d]*/ +/*[clinic end generated code: output=3be6400eee26eaae input=64673c342e4369f1]*/ { int result; @@ -3594,15 +3600,16 @@ win32_fchmod(int fd, int mode) os.chmod path: path_t(allow_fd='PATH_HAVE_FCHMOD') - Path to be modified. May always be specified as a str, bytes, or a path-like object. - On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. + Path to be modified. May always be specified as a str, bytes, or + a path-like object. On some platforms, path may also be specified + as an open file descriptor. If this functionality is unavailable, + using it raises an exception. mode: int Operating-system mode bitfield. - Be careful when using number literals for *mode*. The conventional UNIX notation for - numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in - Python. + Be careful when using number literals for *mode*. The conventional + UNIX notation for numeric modes uses an octal base, which needs to + be indicated with a ``0o`` prefix in Python. * @@ -3629,7 +3636,7 @@ dir_fd and follow_symlinks may not be implemented on your platform. static PyObject * os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, int follow_symlinks) -/*[clinic end generated code: output=5cf6a94915cc7bff input=fcf115d174b9f3d8]*/ +/*[clinic end generated code: output=5cf6a94915cc7bff input=7b6e2eeadd8bf199]*/ { int result; @@ -3764,9 +3771,9 @@ os.fchmod The file descriptor of the file to be modified. mode: int Operating-system mode bitfield. - Be careful when using number literals for *mode*. The conventional UNIX notation for - numeric modes uses an octal base, which needs to be indicated with a ``0o`` prefix in - Python. + Be careful when using number literals for *mode*. The conventional + UNIX notation for numeric modes uses an octal base, which needs to + be indicated with a ``0o`` prefix in Python. Change the access permissions of the file given by file descriptor fd. @@ -3775,7 +3782,7 @@ Equivalent to os.chmod(fd, mode). static PyObject * os_fchmod_impl(PyObject *module, int fd, int mode) -/*[clinic end generated code: output=afd9bc05b4e426b3 input=b5594618bbbc22df]*/ +/*[clinic end generated code: output=afd9bc05b4e426b3 input=d24331f9fdc17f49]*/ { int res; @@ -3816,13 +3823,13 @@ os.lchmod Change the access permissions of a file, without following symbolic links. -If path is a symlink, this affects the link itself rather than the target. -Equivalent to chmod(path, mode, follow_symlinks=False)." +If path is a symlink, this affects the link itself rather than the +target. Equivalent to chmod(path, mode, follow_symlinks=False). [clinic start generated code]*/ static PyObject * os_lchmod_impl(PyObject *module, path_t *path, int mode) -/*[clinic end generated code: output=082344022b51a1d5 input=90c5663c7465d24f]*/ +/*[clinic end generated code: output=082344022b51a1d5 input=074b9b520f1f3008]*/ { int res; if (PySys_Audit("os.chmod", "Oii", path->object, mode, -1) < 0) { @@ -3860,9 +3867,9 @@ os.chflags Set file flags. -If follow_symlinks is False, and the last element of the path is a symbolic - link, chflags will change flags on the symbolic link itself instead of the - file the link points to. +If follow_symlinks is False, and the last element of the path is +a symbolic link, chflags() will change flags on the symbolic link itself +instead of the file the link points to. follow_symlinks may not be implemented on your platform. If it is unavailable, using it will raise a NotImplementedError. @@ -3871,7 +3878,7 @@ unavailable, using it will raise a NotImplementedError. static PyObject * os_chflags_impl(PyObject *module, path_t *path, unsigned long flags, int follow_symlinks) -/*[clinic end generated code: output=85571c6737661ce9 input=0327e29feb876236]*/ +/*[clinic end generated code: output=85571c6737661ce9 input=31391927707be1de]*/ { int result; @@ -4021,7 +4028,8 @@ os_fdatasync_impl(PyObject *module, int fd) os.chown path : path_t(allow_fd='PATH_HAVE_FCHOWN') - Path to be examined; can be string, bytes, a path-like object, or open-file-descriptor int. + Path to be examined; can be string, bytes, a path-like object, or + open-file-descriptor int. uid: uid_t @@ -4029,7 +4037,7 @@ os.chown * - dir_fd : dir_fd(requires='fchownat') = None + dir_fd: dir_fd(requires='fchownat') = None If not None, it should be a file descriptor open to a directory, and path should be relative; path will then be relative to that directory. @@ -4039,27 +4047,28 @@ os.chown stat will examine the symbolic link itself instead of the file the link points to. -Change the owner and group id of path to the numeric uid and gid.\ +Change the owner and group id of path to the numeric uid and gid. -path may always be specified as a string. -On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. -If follow_symlinks is False, and the last element of the path is a symbolic - link, chown will modify the symbolic link itself instead of the file the - link points to. +path may always be specified as a string. On some platforms, path may +also be specified as an open file descriptor. If this functionality is +unavailable, using it raises an exception. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative to +that directory. +If follow_symlinks is False, and the last element of the path is +a symbolic link, chown will modify the symbolic link itself instead of +the file the link points to. It is an error to use dir_fd or follow_symlinks when specifying path as - an open file descriptor. -dir_fd and follow_symlinks may not be implemented on your platform. - If they are unavailable, using them will raise a NotImplementedError. +an open file descriptor. +dir_fd and follow_symlinks may not be implemented on your platform. If +they are unavailable, using them will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_chown_impl(PyObject *module, path_t *path, uid_t uid, gid_t gid, int dir_fd, int follow_symlinks) -/*[clinic end generated code: output=4beadab0db5f70cd input=b08c5ec67996a97d]*/ +/*[clinic end generated code: output=4beadab0db5f70cd input=509c91b7a0e72f52]*/ { int result; @@ -4355,20 +4364,21 @@ os.link Create a hard link to a file. If either src_dir_fd or dst_dir_fd is not None, it should be a file - descriptor open to a directory, and the respective path string (src or dst) - should be relative; the path will then be relative to that directory. +descriptor open to a directory, and the respective path string (src or +dst) should be relative; the path will then be relative to that +directory. If follow_symlinks is False, and the last element of src is a symbolic - link, link will create a link to the symbolic link itself instead of the - file the link points to. -src_dir_fd, dst_dir_fd, and follow_symlinks may not be implemented on your - platform. If they are unavailable, using them will raise a - NotImplementedError. +link, link will create a link to the symbolic link itself instead of the +file the link points to. +src_dir_fd, dst_dir_fd, and follow_symlinks may not be implemented on +your platform. If they are unavailable, using them will raise +a NotImplementedError. [clinic start generated code]*/ static PyObject * os_link_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd, int follow_symlinks) -/*[clinic end generated code: output=7f00f6007fd5269a input=1d5e602d115fed7b]*/ +/*[clinic end generated code: output=7f00f6007fd5269a input=a28e6866fbd20a01]*/ { #ifdef MS_WINDOWS BOOL result = FALSE; @@ -4448,15 +4458,16 @@ os_link_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, #if defined(MS_WINDOWS) && !defined(HAVE_OPENDIR) +static wchar_t * +join_path_filenameW(const wchar_t *path_wide, const wchar_t *filename, + int normalize); + static PyObject * _listdir_windows_no_opendir(path_t *path, PyObject *list) { PyObject *v; HANDLE hFindFile = INVALID_HANDLE_VALUE; BOOL result, return_bytes; - wchar_t namebuf[MAX_PATH+4]; /* Overallocate for "\*.*" */ - /* only claim to have space for MAX_PATH */ - Py_ssize_t len = Py_ARRAY_LENGTH(namebuf)-4; wchar_t *wnamebuf = NULL; WIN32_FIND_DATAW wFileData; @@ -4464,26 +4475,17 @@ _listdir_windows_no_opendir(path_t *path, PyObject *list) if (!path->wide) { /* Default arg: "." */ po_wchars = L"."; - len = 1; return_bytes = 0; } else { po_wchars = path->wide; - len = wcslen(path->wide); return_bytes = PyBytes_Check(path->object); } - /* The +5 is so we can append "\\*.*\0" */ - wnamebuf = PyMem_New(wchar_t, len + 5); - if (!wnamebuf) { - PyErr_NoMemory(); + + wnamebuf = join_path_filenameW(po_wchars, L"*.*", 1); + if (wnamebuf == NULL) { goto exit; } - wcscpy(wnamebuf, po_wchars); - if (len > 0) { - wchar_t wch = wnamebuf[len-1]; - if (wch != SEP && wch != ALTSEP && wch != L':') - wnamebuf[len++] = SEP; - wcscpy(wnamebuf + len, L"*.*"); - } + if ((list = PyList_New(0)) == NULL) { goto exit; } @@ -4668,23 +4670,22 @@ os.listdir Return a list containing the names of the files in the directory. -path can be specified as either str, bytes, or a path-like object. If path is bytes, - the filenames returned will also be bytes; in all other circumstances - the filenames returned will be str. +path can be specified as either str, bytes, or a path-like object. If +path is bytes, the filenames returned will also be bytes; in all other +circumstances the filenames returned will be str. If path is None, uses the path='.'. -On some platforms, path may also be specified as an open file descriptor;\ - the file descriptor must refer to a directory. - If this functionality is unavailable, using it raises NotImplementedError. +On some platforms, path may also be specified as an open file +descriptor; the file descriptor must refer to a directory. If this +functionality is unavailable, using it raises NotImplementedError. The list is in arbitrary order. It does not include the special entries '.' and '..' even if they are present in the directory. - [clinic start generated code]*/ static PyObject * os_listdir_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=293045673fcd1a75 input=e3f58030f538295d]*/ +/*[clinic end generated code: output=293045673fcd1a75 input=4eefe7c6a42ec9b2]*/ { if (PySys_Audit("os.listdir", "O", path->object ? path->object : Py_None) < 0) { @@ -5231,7 +5232,7 @@ os__path_splitroot_impl(PyObject *module, path_t *path) buffer = (wchar_t*)PyMem_Malloc(sizeof(wchar_t) * (wcslen(path->wide) + 1)); if (!buffer) { - return NULL; + return PyErr_NoMemory(); } wcscpy(buffer, path->wide); for (wchar_t *p = wcschr(buffer, L'/'); p; p = wcschr(p, L'/')) { @@ -5675,6 +5676,9 @@ os__path_normpath_impl(PyObject *module, path_t *path) else { result = PyUnicode_FromWideChar(norm_path, norm_len); } + if (result == NULL) { + return NULL; + } if (PyBytes_Check(path->object)) { Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); } @@ -5692,22 +5696,21 @@ os.mkdir dir_fd : dir_fd(requires='mkdirat') = None -# "mkdir(path, mode=0o777, *, dir_fd=None)\n\n\ - Create a directory. -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. -dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative to +that directory. +dir_fd may not be implemented on your platform. If it is unavailable, +using it will raise a NotImplementedError. -The mode argument is ignored on Windows. Where it is used, the current umask -value is first masked out. +The mode argument is ignored on Windows. Where it is used, the current +umask value is first masked out. [clinic start generated code]*/ static PyObject * os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) -/*[clinic end generated code: output=a70446903abe821f input=a61722e1576fab03]*/ +/*[clinic end generated code: output=a70446903abe821f input=30270d369599634b]*/ { int result; #ifdef MS_WINDOWS @@ -5968,16 +5971,17 @@ os.rename Rename a file or directory. If either src_dir_fd or dst_dir_fd is not None, it should be a file - descriptor open to a directory, and the respective path string (src or dst) - should be relative; the path will then be relative to that directory. +descriptor open to a directory, and the respective path string (src or +dst) should be relative; the path will then be relative to that +directory. src_dir_fd and dst_dir_fd, may not be implemented on your platform. - If they are unavailable, using them will raise a NotImplementedError. +If they are unavailable, using them will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_rename_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd) -/*[clinic end generated code: output=59e803072cf41230 input=faa61c847912c850]*/ +/*[clinic end generated code: output=59e803072cf41230 input=7d320d687c715fd6]*/ { return internal_rename(src, dst, src_dir_fd, dst_dir_fd, 0); } @@ -5989,16 +5993,17 @@ os.replace = os.rename Rename a file or directory, overwriting the destination. If either src_dir_fd or dst_dir_fd is not None, it should be a file - descriptor open to a directory, and the respective path string (src or dst) - should be relative; the path will then be relative to that directory. +descriptor open to a directory, and the respective path string (src or +dst) should be relative; the path will then be relative to that +directory. src_dir_fd and dst_dir_fd, may not be implemented on your platform. - If they are unavailable, using them will raise a NotImplementedError. +If they are unavailable, using them will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_replace_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd) -/*[clinic end generated code: output=1968c02e7857422b input=c003f0def43378ef]*/ +/*[clinic end generated code: output=1968c02e7857422b input=44ed6b762d5953fc]*/ { return internal_rename(src, dst, src_dir_fd, dst_dir_fd, 1); } @@ -6013,15 +6018,16 @@ os.rmdir Remove a directory. -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative +to that directory. dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If it is unavailable, using it will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_rmdir_impl(PyObject *module, path_t *path, int dir_fd) -/*[clinic end generated code: output=080eb54f506e8301 input=38c8b375ca34a7e2]*/ +/*[clinic end generated code: output=080eb54f506e8301 input=84325211e33a98e0]*/ { int result; #ifdef HAVE_UNLINKAT @@ -6097,14 +6103,14 @@ os_system_impl(PyObject *module, const wchar_t *command) /*[clinic input] os.system -> long - command: FSConverter + command: unicode_fs_encoded Execute the command in a subshell. [clinic start generated code]*/ static long os_system_impl(PyObject *module, PyObject *command) -/*[clinic end generated code: output=290fc437dd4f33a0 input=86a58554ba6094af]*/ +/*[clinic end generated code: output=290fc437dd4f33a0 input=47c6f24b6dc92881]*/ { long result; const char *bytes = PyBytes_AsString(command); @@ -6192,16 +6198,17 @@ os.unlink Remove a file (same as remove()). -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative to +that directory. dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If it is unavailable, using it will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_unlink_impl(PyObject *module, path_t *path, int dir_fd) -/*[clinic end generated code: output=621797807b9963b1 input=d7bcde2b1b2a2552]*/ +/*[clinic end generated code: output=621797807b9963b1 input=1a2ef2579207eab1]*/ { int result; #ifdef HAVE_UNLINKAT @@ -6253,15 +6260,16 @@ os.remove = os.unlink Remove a file (same as unlink()). -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative +to that directory. dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If it is unavailable, using it will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_remove_impl(PyObject *module, path_t *path, int dir_fd) -/*[clinic end generated code: output=a8535b28f0068883 input=e05c5ab55cd30983]*/ +/*[clinic end generated code: output=a8535b28f0068883 input=9f6e66912126bd56]*/ { return os_unlink_impl(module, path, dir_fd); } @@ -6583,38 +6591,37 @@ os.utime dir_fd: dir_fd(requires='futimensat') = None follow_symlinks: bool=True -# "utime(path, times=None, *[, ns], dir_fd=None, follow_symlinks=True)\n\ - Set the access and modified time of path. -path may always be specified as a string. -On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. +path may always be specified as a string. On some platforms, path may +also be specified as an open file descriptor. If this functionality is +unavailable, using it raises an exception. If times is not None, it must be a tuple (atime, mtime); - atime and mtime should be expressed as float seconds since the epoch. +atime and mtime should be expressed as float seconds since the epoch. If ns is specified, it must be a tuple (atime_ns, mtime_ns); - atime_ns and mtime_ns should be expressed as integer nanoseconds - since the epoch. +atime_ns and mtime_ns should be expressed as integer nanoseconds +since the epoch. If times is None and ns is unspecified, utime uses the current time. Specifying tuples for both times and ns is an error. -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. -If follow_symlinks is False, and the last element of the path is a symbolic - link, utime will modify the symbolic link itself instead of the file the - link points to. -It is an error to use dir_fd or follow_symlinks when specifying path - as an open file descriptor. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative to +that directory. +If follow_symlinks is False, and the last element of the path is +a symbolic link, utime will modify the symbolic link itself instead of +the file the link points to. +It is an error to use dir_fd or follow_symlinks when specifying path as +an open file descriptor. dir_fd and follow_symlinks may not be available on your platform. - If they are unavailable, using them will raise a NotImplementedError. +If they are unavailable, using them will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns, int dir_fd, int follow_symlinks) -/*[clinic end generated code: output=cfcac69d027b82cf input=2fbd62a2f228f8f4]*/ +/*[clinic end generated code: output=cfcac69d027b82cf input=5ab470b2bc250788]*/ { #ifdef MS_WINDOWS HANDLE hFile; @@ -6835,8 +6842,8 @@ static EXECV_CHAR** parse_envlist(PyObject* env, Py_ssize_t *envc_ptr) { Py_ssize_t i, pos, envc; - PyObject *keys=NULL, *vals=NULL; - PyObject *key2, *val2, *keyval; + PyObject *keys = NULL, *vals = NULL; + PyObject *key = NULL, *val = NULL, *key2 = NULL, *val2 = NULL; EXECV_CHAR **envlist; i = PyMapping_Size(env); @@ -6861,20 +6868,22 @@ parse_envlist(PyObject* env, Py_ssize_t *envc_ptr) } for (pos = 0; pos < i; pos++) { - PyObject *key = PyList_GetItem(keys, pos); // Borrowed ref. + // The 'key' and 'val' must be strong references because of + // possible side-effects by PyUnicode_FS{Converter,Decoder}(). + key = PyList_GetItemRef(keys, pos); if (key == NULL) { goto error; } - PyObject *val = PyList_GetItem(vals, pos); // Borrowed ref. + val = PyList_GetItemRef(vals, pos); if (val == NULL) { goto error; } #if defined(HAVE_WEXECV) || defined(HAVE_WSPAWNV) - if (!PyUnicode_FSDecoder(key, &key2)) + if (!PyUnicode_FSDecoder(key, &key2)) { goto error; + } if (!PyUnicode_FSDecoder(val, &val2)) { - Py_DECREF(key2); goto error; } /* Search from index 1 because on Windows starting '=' is allowed for @@ -6883,39 +6892,38 @@ parse_envlist(PyObject* env, Py_ssize_t *envc_ptr) PyUnicode_FindChar(key2, '=', 1, PyUnicode_GET_LENGTH(key2), 1) != -1) { PyErr_SetString(PyExc_ValueError, "illegal environment variable name"); - Py_DECREF(key2); - Py_DECREF(val2); goto error; } - keyval = PyUnicode_FromFormat("%U=%U", key2, val2); + PyObject *keyval = PyUnicode_FromFormat("%U=%U", key2, val2); #else - if (!PyUnicode_FSConverter(key, &key2)) + if (!PyUnicode_FSConverter(key, &key2)) { goto error; + } if (!PyUnicode_FSConverter(val, &val2)) { - Py_DECREF(key2); goto error; } if (PyBytes_GET_SIZE(key2) == 0 || strchr(PyBytes_AS_STRING(key2) + 1, '=') != NULL) { PyErr_SetString(PyExc_ValueError, "illegal environment variable name"); - Py_DECREF(key2); - Py_DECREF(val2); goto error; } - keyval = PyBytes_FromFormat("%s=%s", PyBytes_AS_STRING(key2), - PyBytes_AS_STRING(val2)); + PyObject *keyval = PyBytes_FromFormat("%s=%s", PyBytes_AS_STRING(key2), + PyBytes_AS_STRING(val2)); #endif - Py_DECREF(key2); - Py_DECREF(val2); - if (!keyval) + if (!keyval) { goto error; + } if (!fsconvert_strdup(keyval, &envlist[envc++])) { Py_DECREF(keyval); goto error; } + Py_CLEAR(key); + Py_CLEAR(val); + Py_CLEAR(key2); + Py_CLEAR(val2); Py_DECREF(keyval); } Py_DECREF(vals); @@ -6926,6 +6934,10 @@ parse_envlist(PyObject* env, Py_ssize_t *envc_ptr) return envlist; error: + Py_XDECREF(key); + Py_XDECREF(val); + Py_XDECREF(key2); + Py_XDECREF(val2); Py_XDECREF(keys); Py_XDECREF(vals); free_string_array(envlist, envc); @@ -7150,6 +7162,7 @@ parse_posix_spawn_flags(PyObject *module, const char *func_name, PyObject *setpg PyObject *setsigdef, PyObject *scheduler, posix_spawnattr_t *attrp) { + assert(scheduler == NULL || scheduler == Py_None || PyTuple_Check(scheduler)); long all_flags = 0; errno = posix_spawnattr_init(attrp); @@ -7158,7 +7171,7 @@ parse_posix_spawn_flags(PyObject *module, const char *func_name, PyObject *setpg return -1; } - if (setpgroup) { + if (setpgroup && setpgroup != Py_None) { pid_t pgid = PyLong_AsPid(setpgroup); if (pgid == (pid_t)-1 && PyErr_Occurred()) { goto fail; @@ -7231,7 +7244,7 @@ parse_posix_spawn_flags(PyObject *module, const char *func_name, PyObject *setpg } #endif - if (scheduler) { + if (scheduler && scheduler != Py_None) { #ifdef POSIX_SPAWN_SETSCHEDULER PyObject *py_schedpolicy; PyObject *schedparam_obj; @@ -7447,7 +7460,7 @@ py_posix_spawn(int use_posix_spawnp, PyObject *module, path_t *path, PyObject *a if (argc < 1) { PyErr_Format(PyExc_ValueError, "%s: argv must not be empty", func_name); - return NULL; + goto exit; } if (!PyMapping_Check(env) && env != Py_None) { @@ -7456,6 +7469,12 @@ py_posix_spawn(int use_posix_spawnp, PyObject *module, path_t *path, PyObject *a goto exit; } + if (scheduler && !PyTuple_Check(scheduler) && scheduler != Py_None) { + PyErr_Format(PyExc_TypeError, + "%s: scheduler must be a tuple or None", func_name); + goto exit; + } + argvlist = parse_arglist(argv, &argc); if (argvlist == NULL) { goto exit; @@ -7567,17 +7586,18 @@ os.posix_spawn * file_actions: object(c_default='NULL') = () A sequence of file action tuples. - setpgroup: object = NULL + setpgroup: object(c_default='NULL') = None The pgroup to use with the POSIX_SPAWN_SETPGROUP flag. resetids: bool = False If the value is `true` the POSIX_SPAWN_RESETIDS will be activated. setsid: bool = False - If the value is `true` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated. + If the value is `true` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP + will be activated. setsigmask: object(c_default='NULL') = () The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag. setsigdef: object(c_default='NULL') = () The sigmask to use with the POSIX_SPAWN_SETSIGDEF flag. - scheduler: object = NULL + scheduler: object(c_default='NULL') = None A tuple with the scheduler policy (optional) and parameters. Execute the program specified by path in a new process. @@ -7589,7 +7609,7 @@ os_posix_spawn_impl(PyObject *module, path_t *path, PyObject *argv, PyObject *setpgroup, int resetids, int setsid, PyObject *setsigmask, PyObject *setsigdef, PyObject *scheduler) -/*[clinic end generated code: output=14a1098c566bc675 input=808aed1090d84e33]*/ +/*[clinic end generated code: output=14a1098c566bc675 input=c7592dcbc96e8114]*/ { return py_posix_spawn(0, module, path, argv, env, file_actions, setpgroup, resetids, setsid, setsigmask, setsigdef, @@ -7613,17 +7633,18 @@ os.posix_spawnp * file_actions: object(c_default='NULL') = () A sequence of file action tuples. - setpgroup: object = NULL + setpgroup: object(c_default='NULL') = None The pgroup to use with the POSIX_SPAWN_SETPGROUP flag. resetids: bool = False If the value is `True` the POSIX_SPAWN_RESETIDS will be activated. setsid: bool = False - If the value is `True` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP will be activated. + If the value is `True` the POSIX_SPAWN_SETSID or POSIX_SPAWN_SETSID_NP + will be activated. setsigmask: object(c_default='NULL') = () The sigmask to use with the POSIX_SPAWN_SETSIGMASK flag. setsigdef: object(c_default='NULL') = () The sigmask to use with the POSIX_SPAWN_SETSIGDEF flag. - scheduler: object = NULL + scheduler: object(c_default='NULL') = None A tuple with the scheduler policy (optional) and parameters. Execute the program specified by path in a new process. @@ -7635,7 +7656,7 @@ os_posix_spawnp_impl(PyObject *module, path_t *path, PyObject *argv, PyObject *setpgroup, int resetids, int setsid, PyObject *setsigmask, PyObject *setsigdef, PyObject *scheduler) -/*[clinic end generated code: output=7b9aaefe3031238d input=9e89e616116752a1]*/ +/*[clinic end generated code: output=7b9aaefe3031238d input=43ccc1452cae2be3]*/ { return py_posix_spawn(1, module, path, argv, env, file_actions, setpgroup, resetids, setsid, setsigmask, setsigdef, @@ -7700,18 +7721,15 @@ os_spawnv_impl(PyObject *module, int mode, path_t *path, PyObject *argv) int i; Py_ssize_t argc; intptr_t spawnval; - PyObject *(*getitem)(PyObject *, Py_ssize_t); /* spawnv has three arguments: (mode, path, argv), where argv is a list or tuple of strings. */ if (PyList_Check(argv)) { argc = PyList_Size(argv); - getitem = PyList_GetItem; } else if (PyTuple_Check(argv)) { argc = PyTuple_Size(argv); - getitem = PyTuple_GetItem; } else { PyErr_SetString(PyExc_TypeError, @@ -7729,14 +7747,29 @@ os_spawnv_impl(PyObject *module, int mode, path_t *path, PyObject *argv) return PyErr_NoMemory(); } for (i = 0; i < argc; i++) { - if (!fsconvert_strdup((*getitem)(argv, i), - &argvlist[i])) { + // The item must be a strong reference because of possible + // side-effects of PyUnicode_FS{Converter,Decoder}() in + // fsconvert_strdup(): an item's __fspath__() can mutate a list + // *argv*, releasing the list's reference to the item (gh-151416). + PyObject *item = PySequence_ITEM(argv, i); + if (item == NULL) { free_string_array(argvlist, i); - PyErr_SetString( - PyExc_TypeError, - "spawnv() arg 2 must contain only strings"); return NULL; } + if (!fsconvert_strdup(item, &argvlist[i])) { + Py_DECREF(item); + free_string_array(argvlist, i); + // Add argument context to the converter's terse TypeError, but + // let MemoryError, codec errors, embedded-null ValueError, etc. + // propagate unmasked. + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + PyErr_SetString( + PyExc_TypeError, + "spawnv() arg 2 must contain only strings"); + } + return NULL; + } + Py_DECREF(item); if (i == 0 && !argvlist[0][0]) { free_string_array(argvlist, i + 1); PyErr_SetString( @@ -7807,7 +7840,6 @@ os_spawnve_impl(PyObject *module, int mode, path_t *path, PyObject *argv, PyObject *res = NULL; Py_ssize_t argc, i, envc; intptr_t spawnval; - PyObject *(*getitem)(PyObject *, Py_ssize_t); Py_ssize_t lastarg = 0; /* spawnve has four arguments: (mode, path, argv, env), where @@ -7816,11 +7848,9 @@ os_spawnve_impl(PyObject *module, int mode, path_t *path, PyObject *argv, if (PyList_Check(argv)) { argc = PyList_Size(argv); - getitem = PyList_GetItem; } else if (PyTuple_Check(argv)) { argc = PyTuple_Size(argv); - getitem = PyTuple_GetItem; } else { PyErr_SetString(PyExc_TypeError, @@ -7844,12 +7874,21 @@ os_spawnve_impl(PyObject *module, int mode, path_t *path, PyObject *argv, goto fail_0; } for (i = 0; i < argc; i++) { - if (!fsconvert_strdup((*getitem)(argv, i), - &argvlist[i])) - { + // The item must be a strong reference because of possible + // side-effects of PyUnicode_FS{Converter,Decoder}() in + // fsconvert_strdup(): an item's __fspath__() can mutate a list + // *argv*, releasing the list's reference to the item (gh-151416). + PyObject *item = PySequence_ITEM(argv, i); + if (item == NULL) { + lastarg = i; + goto fail_1; + } + if (!fsconvert_strdup(item, &argvlist[i])) { + Py_DECREF(item); lastarg = i; goto fail_1; } + Py_DECREF(item); if (i == 0 && !argvlist[0][0]) { lastarg = i + 1; PyErr_SetString( @@ -7975,53 +8014,19 @@ os_register_at_fork_impl(PyObject *module, PyObject *before, // running in the process. Best effort, silent if unable to count threads. // Constraint: Quick. Never overcounts. Never leaves an error set. // -// This should only be called from the parent process after +// This MUST only be called from the parent process after // PyOS_AfterFork_Parent(). static void -warn_about_fork_with_threads(const char* name) +warn_about_fork_with_threads( + const char* name, // Name of the API to use in the warning message. + const Py_ssize_t num_os_threads // Only trusted when >= 1. +) { // It's not safe to issue the warning while the world is stopped, because // other threads might be holding locks that we need, which would deadlock. assert(!_PyRuntime.stoptheworld.world_stopped); - // TODO: Consider making an `os` module API to return the current number - // of threads in the process. That'd presumably use this platform code but - // raise an error rather than using the inaccurate fallback. - Py_ssize_t num_python_threads = 0; -#if defined(__APPLE__) && defined(HAVE_GETPID) - mach_port_t macos_self = mach_task_self(); - mach_port_t macos_task; - if (task_for_pid(macos_self, getpid(), &macos_task) == KERN_SUCCESS) { - thread_array_t macos_threads; - mach_msg_type_number_t macos_n_threads; - if (task_threads(macos_task, &macos_threads, - &macos_n_threads) == KERN_SUCCESS) { - num_python_threads = macos_n_threads; - } - } -#elif defined(__linux__) - // Linux /proc/self/stat 20th field is the number of threads. - FILE* proc_stat = fopen("/proc/self/stat", "r"); - if (proc_stat) { - size_t n; - // Size chosen arbitrarily. ~60% more bytes than a 20th column index - // observed on the author's workstation. - char stat_line[160]; - n = fread(&stat_line, 1, 159, proc_stat); - stat_line[n] = '\0'; - fclose(proc_stat); - - char *saveptr = NULL; - char *field = strtok_r(stat_line, " ", &saveptr); - unsigned int idx; - for (idx = 19; idx && field; --idx) { - field = strtok_r(NULL, " ", &saveptr); - } - if (idx == 0 && field) { // found the 20th field - num_python_threads = atoi(field); // 0 on error - } - } -#endif + Py_ssize_t num_python_threads = num_os_threads; if (num_python_threads <= 0) { // Fall back to just the number our threading module knows about. // An incomplete view of the world, but better than nothing. @@ -8074,6 +8079,51 @@ warn_about_fork_with_threads(const char* name) PyErr_Clear(); } } + +// If this returns <= 0, we were unable to successfully use any OS APIs. +// Returns a positive number of threads otherwise. +static Py_ssize_t get_number_of_os_threads(void) +{ + // TODO: Consider making an `os` module API to return the current number + // of threads in the process. That'd presumably use this platform code but + // raise an error rather than using the inaccurate fallback. + Py_ssize_t num_python_threads = 0; +#if defined(__APPLE__) && defined(HAVE_GETPID) + mach_port_t macos_self = mach_task_self(); + mach_port_t macos_task; + if (task_for_pid(macos_self, getpid(), &macos_task) == KERN_SUCCESS) { + thread_array_t macos_threads; + mach_msg_type_number_t macos_n_threads; + if (task_threads(macos_task, &macos_threads, + &macos_n_threads) == KERN_SUCCESS) { + num_python_threads = macos_n_threads; + } + } +#elif defined(__linux__) + // Linux /proc/self/stat 20th field is the number of threads. + FILE* proc_stat = fopen("/proc/self/stat", "r"); + if (proc_stat) { + size_t n; + // Size chosen arbitrarily. ~60% more bytes than a 20th column index + // observed on the author's workstation. + char stat_line[160]; + n = fread(&stat_line, 1, 159, proc_stat); + stat_line[n] = '\0'; + fclose(proc_stat); + + char *saveptr = NULL; + char *field = strtok_r(stat_line, " ", &saveptr); + unsigned int idx; + for (idx = 19; idx && field; --idx) { + field = strtok_r(NULL, " ", &saveptr); + } + if (idx == 0 && field) { // found the 20th field + num_python_threads = atoi(field); // 0 on error + } + } +#endif + return num_python_threads; +} #endif // HAVE_FORK1 || HAVE_FORKPTY || HAVE_FORK #ifdef HAVE_FORK1 @@ -8108,10 +8158,12 @@ os_fork1_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + // Called before AfterFork_Parent in case those hooks start threads. + Py_ssize_t num_os_threads = get_number_of_os_threads(); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. - warn_about_fork_with_threads("fork1"); + warn_about_fork_with_threads("fork1", num_os_threads); } if (pid == -1) { errno = saved_errno; @@ -8157,10 +8209,12 @@ os_fork_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + // Called before AfterFork_Parent in case those hooks start threads. + Py_ssize_t num_os_threads = get_number_of_os_threads(); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. - warn_about_fork_with_threads("fork"); + warn_about_fork_with_threads("fork", num_os_threads); } if (pid == -1) { errno = saved_errno; @@ -8185,10 +8239,10 @@ static PyObject * os_sched_get_priority_max_impl(PyObject *module, int policy) /*[clinic end generated code: output=9e465c6e43130521 input=2097b7998eca6874]*/ { - int max; - - max = sched_get_priority_max(policy); - if (max < 0) + /* make sure that errno is cleared before the call */ + errno = 0; + int max = sched_get_priority_max(policy); + if (max == -1 && errno) return posix_error(); return PyLong_FromLong(max); } @@ -8206,8 +8260,10 @@ static PyObject * os_sched_get_priority_min_impl(PyObject *module, int policy) /*[clinic end generated code: output=7595c1138cc47a6d input=21bc8fa0d70983bf]*/ { + /* make sure that errno is cleared before the call */ + errno = 0; int min = sched_get_priority_min(policy); - if (min < 0) + if (min == -1 && errno) return posix_error(); return PyLong_FromLong(min); } @@ -8268,7 +8324,7 @@ os_sched_param_impl(PyTypeObject *type, PyObject *sched_priority) static PyObject * os_sched_param_reduce(PyObject *self, PyObject *Py_UNUSED(dummy)) { - return Py_BuildValue("(O(N))", Py_TYPE(self), PyStructSequence_GetItem(self, 0)); + return Py_BuildValue("(O(O))", Py_TYPE(self), PyStructSequence_GetItem(self, 0)); } static PyMethodDef os_sched_param_reduce_method = { @@ -8649,17 +8705,19 @@ os.posix_openpt -> int Open and return a file descriptor for a master pseudo-terminal device. Performs a posix_openpt() C function call. The oflag argument is used to -set file status flags and file access modes as specified in the manual page -of posix_openpt() of your system. +set file status flags and file access modes as specified in the manual +page of posix_openpt() of your system. [clinic start generated code]*/ static int os_posix_openpt_impl(PyObject *module, int oflag) -/*[clinic end generated code: output=ee0bc2624305fc79 input=0de33d0e29693caa]*/ +/*[clinic end generated code: output=ee0bc2624305fc79 input=3ce4eb297fa64307]*/ { int fd; -#if defined(O_CLOEXEC) + // OpenBSD posix_openpt() rejects any flag other than O_RDWR and + // O_NOCTTY; the fd is made non-inheritable below in any case. +#if defined(O_CLOEXEC) && !defined(__OpenBSD__) oflag |= O_CLOEXEC; #endif @@ -8806,14 +8864,14 @@ os_ptsname_impl(PyObject *module, int fd) #if defined(HAVE_OPENPTY) || defined(HAVE_FORKPTY) || defined(HAVE_LOGIN_TTY) || defined(HAVE_DEV_PTMX) #ifdef HAVE_PTY_H #include <pty.h> -#ifdef HAVE_UTMP_H -#include <utmp.h> -#endif /* HAVE_UTMP_H */ #elif defined(HAVE_LIBUTIL_H) #include <libutil.h> #elif defined(HAVE_UTIL_H) #include <util.h> #endif /* HAVE_PTY_H */ +#ifdef HAVE_UTMP_H +#include <utmp.h> +#endif /* HAVE_UTMP_H */ #ifdef HAVE_STROPTS_H #include <stropts.h> #endif @@ -8902,11 +8960,30 @@ os_openpty_impl(PyObject *module) goto posix_error; #if !defined(__CYGWIN__) && !defined(__ANDROID__) && !defined(HAVE_DEV_PTC) + // Pushing "ptem" makes the slave a terminal, which a session leader + // without a controlling terminal then acquires as one despite O_NOCTTY. + // Note whether we already had one, so a new one can be disowned below. + int had_ctty = 0; +#ifdef TIOCNOTTY + int tty_fd = open("/dev/tty", O_RDONLY | O_NOCTTY); + if (tty_fd >= 0) { + had_ctty = 1; + close(tty_fd); + } +#endif ioctl(slave_fd, I_PUSH, "ptem"); /* push ptem */ ioctl(slave_fd, I_PUSH, "ldterm"); /* push ldterm */ #ifndef __hpux ioctl(slave_fd, I_PUSH, "ttcompat"); /* push ttcompat */ #endif /* __hpux */ +#ifdef TIOCNOTTY + if (!had_ctty && getsid(0) == getpid()) { + // Disown it; TIOCNOTTY sends SIGHUP to the session leader. + PyOS_sighandler_t sig_saved = PyOS_setsig(SIGHUP, SIG_IGN); + ioctl(slave_fd, TIOCNOTTY); + PyOS_setsig(SIGHUP, sig_saved); + } +#endif #endif /* HAVE_CYGWIN */ #endif /* HAVE_OPENPTY */ @@ -9012,10 +9089,12 @@ os_forkpty_impl(PyObject *module) /* child: this clobbers and resets the import lock. */ PyOS_AfterFork_Child(); } else { + // Called before AfterFork_Parent in case those hooks start threads. + Py_ssize_t num_os_threads = get_number_of_os_threads(); /* parent: release the import lock. */ PyOS_AfterFork_Parent(); // After PyOS_AfterFork_Parent() starts the world to avoid deadlock. - warn_about_fork_with_threads("forkpty"); + warn_about_fork_with_threads("forkpty", num_os_threads); } if (pid == -1) { return posix_error(); @@ -9282,38 +9361,38 @@ os_getgroups_impl(PyObject *module) /*[clinic input] os.initgroups - username as oname: FSConverter + username as oname: unicode_fs_encoded gid: int / Initialize the group access list. -Call the system initgroups() to initialize the group access list with all of -the groups of which the specified username is a member, plus the specified -group id. +Call the system initgroups() to initialize the group access list with +all of the groups of which the specified username is a member, plus the +specified group id. [clinic start generated code]*/ static PyObject * os_initgroups_impl(PyObject *module, PyObject *oname, int gid) -/*[clinic end generated code: output=7f074d30a425fd3a input=df3d54331b0af204]*/ +/*[clinic end generated code: output=7f074d30a425fd3a input=35f2d4fb7fcc0bdf]*/ #else /*[clinic input] os.initgroups - username as oname: FSConverter + username as oname: unicode_fs_encoded gid: gid_t / Initialize the group access list. -Call the system initgroups() to initialize the group access list with all of -the groups of which the specified username is a member, plus the specified -group id. +Call the system initgroups() to initialize the group access list with +all of the groups of which the specified username is a member, plus the +specified group id. [clinic start generated code]*/ static PyObject * os_initgroups_impl(PyObject *module, PyObject *oname, gid_t gid) -/*[clinic end generated code: output=59341244521a9e3f input=0cb91bdc59a4c564]*/ +/*[clinic end generated code: output=59341244521a9e3f input=7e4514dff4526a95]*/ #endif { const char *username = PyBytes_AS_STRING(oname); @@ -9510,12 +9589,13 @@ os.getppid Return the parent's process id. If the parent process has already exited, Windows machines will still -return its id; others systems will return the id of the 'init' process (1). +return its id; others systems will return the id of the 'init' proces +(1). [clinic start generated code]*/ static PyObject * os_getppid_impl(PyObject *module) -/*[clinic end generated code: output=43b2a946a8c603b4 input=e637cb87539c030e]*/ +/*[clinic end generated code: output=43b2a946a8c603b4 input=e17c1de18f41316b]*/ { #ifdef MS_WINDOWS return win32_getppid(); @@ -9548,6 +9628,24 @@ os_getlogin_impl(PyObject *module) } else result = PyErr_SetFromWindowsErr(GetLastError()); +#elif defined (HAVE_GETLOGIN_R) +# if defined (HAVE_MAXLOGNAME) + char name[MAXLOGNAME + 1]; +# elif defined (HAVE_UT_NAMESIZE) + char name[UT_NAMESIZE + 1]; +# else + char name[256]; +# endif + int err = getlogin_r(name, sizeof(name)); + if (err) { + int old_errno = errno; + errno = -err; + posix_error(); + errno = old_errno; + } + else { + result = PyUnicode_DecodeFSDefault(name); + } #else char *name; int old_errno = errno; @@ -10051,13 +10149,13 @@ os.waitid Returns the result of waiting for a process or processes. -Returns either waitid_result or None if WNOHANG is specified and there are -no children in a waitable state. +Returns either waitid_result or None if WNOHANG is specified and there +are no children in a waitable state. [clinic start generated code]*/ static PyObject * os_waitid_impl(PyObject *module, idtype_t idtype, id_t id, int options) -/*[clinic end generated code: output=5d2e1c0bde61f4d8 input=d8e7f76e052b7920]*/ +/*[clinic end generated code: output=5d2e1c0bde61f4d8 input=14956bc8d102b5db]*/ { PyObject *result; int res; @@ -10224,13 +10322,13 @@ os.pidfd_open Return a file descriptor referring to the process *pid*. -The descriptor can be used to perform process management without races and -signals. +The descriptor can be used to perform process management without races +and signals. [clinic start generated code]*/ static PyObject * os_pidfd_open_impl(PyObject *module, pid_t pid, unsigned int flags) -/*[clinic end generated code: output=5c7252698947dc41 input=c3fd99ce947ccfef]*/ +/*[clinic end generated code: output=5c7252698947dc41 input=03058b32c389f874]*/ { int fd = syscall(__NR_pidfd_open, pid, flags); if (fd < 0) { @@ -10309,8 +10407,9 @@ os.readlink Return a string representing the path to which the symbolic link points. -If dir_fd is not None, it should be a file descriptor open to a directory, -and path should be relative; path will then be relative to that directory. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative to +that directory. dir_fd may not be implemented on your platform. If it is unavailable, using it will raise a NotImplementedError. @@ -10318,7 +10417,7 @@ using it will raise a NotImplementedError. static PyObject * os_readlink_impl(PyObject *module, path_t *path, int dir_fd) -/*[clinic end generated code: output=d21b732a2e814030 input=113c87e0db1ecaf2]*/ +/*[clinic end generated code: output=d21b732a2e814030 input=03d10130870dbca8]*/ { #if defined(HAVE_READLINK) char buffer[MAXPATHLEN+1]; @@ -10513,26 +10612,25 @@ os.symlink * dir_fd: dir_fd(requires='symlinkat')=None -# "symlink(src, dst, target_is_directory=False, *, dir_fd=None)\n\n\ - Create a symbolic link pointing to src named dst. target_is_directory is required on Windows if the target is to be - interpreted as a directory. (On Windows, symlink requires - Windows 6.0 or greater, and raises a NotImplementedError otherwise.) - target_is_directory is ignored on non-Windows platforms. +interpreted as a directory. (On Windows, symlink requires Windows 6.0 +or greater, and raises a NotImplementedError otherwise.) +target_is_directory is ignored on non-Windows platforms. -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. -dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative +to that directory. +dir_fd may not be implemented on your platform. If it is unavailable, +using it will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_symlink_impl(PyObject *module, path_t *src, path_t *dst, int target_is_directory, int dir_fd) -/*[clinic end generated code: output=08ca9f3f3cf960f6 input=e820ec4472547bc3]*/ +/*[clinic end generated code: output=08ca9f3f3cf960f6 input=71b75467b31c45f7]*/ { #ifdef MS_WINDOWS DWORD result; @@ -11092,19 +11190,18 @@ os.open -> int * dir_fd: dir_fd(requires='openat') = None -# "open(path, flags, mode=0o777, *, dir_fd=None)\n\n\ - Open a file for low level IO. Returns a file descriptor (integer). -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. -dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative to +that directory. +dir_fd may not be implemented on your platform. If it is unavailable, +using it will raise a NotImplementedError. [clinic start generated code]*/ static int os_open_impl(PyObject *module, path_t *path, int flags, int mode, int dir_fd) -/*[clinic end generated code: output=abc7227888c8bc73 input=ad8623b29acd2934]*/ +/*[clinic end generated code: output=abc7227888c8bc73 input=75f7b4eaf92f2225]*/ { int fd; int async_err = 0; @@ -11393,12 +11490,13 @@ os.lseek -> Py_off_t Set the position of a file descriptor. Return the new position. -The return value is the number of bytes relative to the beginning of the file. +The return value is the number of bytes relative to the beginning of +the file. [clinic start generated code]*/ static Py_off_t os_lseek_impl(PyObject *module, int fd, Py_off_t position, int how) -/*[clinic end generated code: output=971e1efb6b30bd2f input=f096e754c5367504]*/ +/*[clinic end generated code: output=971e1efb6b30bd2f input=32ea0788da7cb44b]*/ { Py_off_t result; @@ -11474,20 +11572,20 @@ os.readinto -> Py_ssize_t Read into a buffer object from a file descriptor. -The buffer should be mutable and bytes-like. On success, returns the number of -bytes read. Less bytes may be read than the size of the buffer. The underlying -system call will be retried when interrupted by a signal, unless the signal -handler raises an exception. Other errors will not be retried and an error will -be raised. +The buffer should be mutable and bytes-like. On success, returns the +number of bytes read. Less bytes may be read than the size of the +buffer. The underlying system call will be retried when interrupted by +a signal, unless the signal handler raises an exception. Other errors +will not be retried and an error will be raised. -Returns 0 if *fd* is at end of file or if the provided *buffer* has length 0 -(which can be used to check for errors without reading data). Never returns -negative. +Returns 0 if *fd* is at end of file or if the provided *buffer* has +length 0 (which can be used to check for errors without reading data). +Never returns negative. [clinic start generated code]*/ static Py_ssize_t os_readinto_impl(PyObject *module, int fd, Py_buffer *buffer) -/*[clinic end generated code: output=8091a3513c683a80 input=d40074d0a68de575]*/ +/*[clinic end generated code: output=8091a3513c683a80 input=2a5f8b212cb5730c]*/ { assert(buffer->len >= 0); Py_ssize_t result = _Py_read(fd, buffer->buf, buffer->len); @@ -11682,14 +11780,15 @@ os.preadv -> Py_ssize_t Reads from a file descriptor into a number of mutable bytes-like objects. -Combines the functionality of readv() and pread(). As readv(), it will -transfer data into each buffer until it is full and then move on to the next -buffer in the sequence to hold the rest of the data. Its fourth argument, -specifies the file offset at which the input operation is to be performed. It -will return the total number of bytes read (which can be less than the total -capacity of all the objects). +Combines the functionality of readv() and pread(). As readv(), it will +transfer data into each buffer until it is full and then move on to the +next buffer in the sequence to hold the rest of the data. Its fourth +argument, specifies the file offset at which the input operation is to +be performed. It will return the total number of bytes read (which can +be less than the total capacity of all the objects). -The flags argument contains a bitwise OR of zero or more of the following flags: +The flags argument contains a bitwise OR of zero or more of the +following flags: - RWF_HIPRI - RWF_NOWAIT @@ -11700,7 +11799,7 @@ Using non-zero flags requires Linux 4.6 or newer. static Py_ssize_t os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, int flags) -/*[clinic end generated code: output=26fc9c6e58e7ada5 input=4173919dc1f7ed99]*/ +/*[clinic end generated code: output=26fc9c6e58e7ada5 input=a7911fe42877e9de]*/ { Py_ssize_t cnt, n; int async_err = 0; @@ -12326,14 +12425,16 @@ os.pwritev -> Py_ssize_t Writes the contents of bytes-like objects to a file descriptor at a given offset. -Combines the functionality of writev() and pwrite(). All buffers must be a sequence -of bytes-like objects. Buffers are processed in array order. Entire contents of first -buffer is written before proceeding to second, and so on. The operating system may -set a limit (sysconf() value SC_IOV_MAX) on the number of buffers that can be used. -This function writes the contents of each object to the file descriptor and returns -the total number of bytes written. +Combines the functionality of writev() and pwrite(). All buffers must be +a sequence of bytes-like objects. Buffers are processed in array order. +Entire contents of first buffer is written before proceeding to second, +and so on. The operating system may set a limit (sysconf() value +SC_IOV_MAX) on the number of buffers that can be used. +This function writes the contents of each object to the file descriptor +and returns the total number of bytes written. -The flags argument contains a bitwise OR of zero or more of the following flags: +The flags argument contains a bitwise OR of zero or more of the +following flags: - RWF_DSYNC - RWF_SYNC @@ -12345,7 +12446,7 @@ Using non-zero flags requires Linux 4.7 or newer. static Py_ssize_t os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, int flags) -/*[clinic end generated code: output=e3dd3e9d11a6a5c7 input=35358c327e1a2a8e]*/ +/*[clinic end generated code: output=e3dd3e9d11a6a5c7 input=43ee144dd99584e5]*/ { Py_ssize_t cnt; Py_ssize_t result; @@ -12567,15 +12668,16 @@ os.mkfifo Create a "fifo" (a POSIX named pipe). -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. -dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative to +that directory. +dir_fd may not be implemented on your platform. If it is unavailable, +using it will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_mkfifo_impl(PyObject *module, path_t *path, int mode, int dir_fd) -/*[clinic end generated code: output=ce41cfad0e68c940 input=73032e98a36e0e19]*/ +/*[clinic end generated code: output=ce41cfad0e68c940 input=d2fb917c01e888d6]*/ { int result; int async_err = 0; @@ -12628,23 +12730,24 @@ os.mknod Create a node in the file system. -Create a node in the file system (file, device special file or named pipe) -at path. mode specifies both the permissions to use and the +Create a node in the file system (file, device special file or named +pipe) at path. mode specifies both the permissions to use and the type of node to be created, being combined (bitwise OR) with one of -S_IFREG, S_IFCHR, S_IFBLK, and S_IFIFO. If S_IFCHR or S_IFBLK is set on mode, -device defines the newly created device special file (probably using -os.makedev()). Otherwise device is ignored. +S_IFREG, S_IFCHR, S_IFBLK, and S_IFIFO. If S_IFCHR or S_IFBLK is set +on mode, device defines the newly created device special file (probably +using os.makedev()). Otherwise device is ignored. -If dir_fd is not None, it should be a file descriptor open to a directory, - and path should be relative; path will then be relative to that directory. -dir_fd may not be implemented on your platform. - If it is unavailable, using it will raise a NotImplementedError. +If dir_fd is not None, it should be a file descriptor open to +a directory, and path should be relative; path will then be relative +to that directory. +dir_fd may not be implemented on your platform. If it is unavailable, +using it will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_mknod_impl(PyObject *module, path_t *path, int mode, dev_t device, int dir_fd) -/*[clinic end generated code: output=92e55d3ca8917461 input=ee44531551a4d83b]*/ +/*[clinic end generated code: output=92e55d3ca8917461 input=7d0099e85c6b4cba]*/ { int result; int async_err = 0; @@ -12813,13 +12916,14 @@ os.truncate Truncate a file, specified by path, to a specific length. -On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. +On some platforms, path may also be specified as an open file +descriptor. If this functionality is unavailable, using it raises +an exception. [clinic start generated code]*/ static PyObject * os_truncate_impl(PyObject *module, path_t *path, Py_off_t length) -/*[clinic end generated code: output=43009c8df5c0a12b input=77229cf0b50a9b77]*/ +/*[clinic end generated code: output=43009c8df5c0a12b input=ce33fd7808a511c4]*/ { int result; #ifdef MS_WINDOWS @@ -12882,13 +12986,14 @@ os.posix_fallocate Ensure a file has allocated at least a particular number of bytes on disk. Ensure that the file specified by fd encompasses a range of bytes -starting at offset bytes from the beginning and continuing for length bytes. +starting at offset bytes from the beginning and continuing for length +bytes. [clinic start generated code]*/ static PyObject * os_posix_fallocate_impl(PyObject *module, int fd, Py_off_t offset, Py_off_t length) -/*[clinic end generated code: output=73f107139564aa9d input=d7a2ef0ab2ca52fb]*/ +/*[clinic end generated code: output=73f107139564aa9d input=54f630ac8b5d3733]*/ { int result; int async_err = 0; @@ -12897,6 +13002,10 @@ os_posix_fallocate_impl(PyObject *module, int fd, Py_off_t offset, Py_BEGIN_ALLOW_THREADS result = posix_fallocate(fd, offset, length); Py_END_ALLOW_THREADS + // DragonFly BSD returns -1 and sets errno. + if (result == -1) { + result = errno; + } } while (result == EINTR && !(async_err = PyErr_CheckSignals())); if (result == 0) @@ -12923,8 +13032,8 @@ os.posix_fadvise Announce an intention to access data in a specific pattern. -Announce an intention to access data in a specific pattern, thus allowing -the kernel to make optimizations. +Announce an intention to access data in a specific pattern, thus +allowing the kernel to make optimizations. The advice applies to the region of the file specified by fd starting at offset and continuing for length bytes. advice is one of POSIX_FADV_NORMAL, POSIX_FADV_SEQUENTIAL, @@ -12935,7 +13044,7 @@ POSIX_FADV_DONTNEED. static PyObject * os_posix_fadvise_impl(PyObject *module, int fd, Py_off_t offset, Py_off_t length, int advice) -/*[clinic end generated code: output=412ef4aa70c98642 input=0fbe554edc2f04b5]*/ +/*[clinic end generated code: output=412ef4aa70c98642 input=961b01a4518ef727]*/ { int result; int async_err = 0; @@ -12944,6 +13053,10 @@ os_posix_fadvise_impl(PyObject *module, int fd, Py_off_t offset, Py_BEGIN_ALLOW_THREADS result = posix_fadvise(fd, offset, length, advice); Py_END_ALLOW_THREADS + // DragonFly BSD returns -1 and sets errno. + if (result == -1) { + result = errno; + } } while (result == EINTR && !(async_err = PyErr_CheckSignals())); if (result == 0) @@ -13046,8 +13159,8 @@ os_putenv_impl(PyObject *module, PyObject *name, PyObject *value) /*[clinic input] os.putenv - name: FSConverter - value: FSConverter + name: unicode_fs_encoded + value: unicode_fs_encoded / Change or add an environment variable. @@ -13055,7 +13168,7 @@ Change or add an environment variable. static PyObject * os_putenv_impl(PyObject *module, PyObject *name, PyObject *value) -/*[clinic end generated code: output=d29a567d6b2327d2 input=a97bc6152f688d31]*/ +/*[clinic end generated code: output=d29a567d6b2327d2 input=84fcd30f873c8c45]*/ { const char *name_string = PyBytes_AS_STRING(name); const char *value_string = PyBytes_AS_STRING(value); @@ -13098,7 +13211,7 @@ os_unsetenv_impl(PyObject *module, PyObject *name) #else /*[clinic input] os.unsetenv - name: FSConverter + name: unicode_fs_encoded / Delete an environment variable. @@ -13106,7 +13219,7 @@ Delete an environment variable. static PyObject * os_unsetenv_impl(PyObject *module, PyObject *name) -/*[clinic end generated code: output=54c4137ab1834f02 input=2bb5288a599c7107]*/ +/*[clinic end generated code: output=54c4137ab1834f02 input=78ff12e505ade80a]*/ { if (PySys_Audit("os.unsetenv", "(O)", name) < 0) { return NULL; @@ -13495,13 +13608,14 @@ os.statvfs Perform a statvfs system call on the given path. path may always be specified as a string. -On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. +On some platforms, path may also be specified as an open file +descriptor. If this functionality is unavailable, using it raises +an exception. [clinic start generated code]*/ static PyObject * os_statvfs_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=87106dd1beb8556e input=3f5c35791c669bd9]*/ +/*[clinic end generated code: output=87106dd1beb8556e input=1cfd9a4fd36f7425]*/ { int result; @@ -13780,20 +13894,22 @@ os.pathconf -> long Return the configuration limit name for the file or directory path. If there is no limit, return -1. -On some platforms, path may also be specified as an open file descriptor. - If this functionality is unavailable, using it raises an exception. +On some platforms, path may also be specified as an open file +descriptor. If this functionality is unavailable, using it raises +an exception. [clinic start generated code]*/ static long os_pathconf_impl(PyObject *module, path_t *path, int name) -/*[clinic end generated code: output=5bedee35b293a089 input=6f6072f57b10c787]*/ +/*[clinic end generated code: output=5bedee35b293a089 input=e86f6eacfa006426]*/ { long limit; errno = 0; #ifdef HAVE_FPATHCONF - if (path->fd != -1) + if (path->is_fd) { limit = fpathconf(path->fd, name); + } else #endif limit = pathconf(path->narrow, name); @@ -14590,13 +14706,13 @@ os.abort Abort the interpreter immediately. -This function 'dumps core' or otherwise fails in the hardest way possible -on the hosting operating system. This function never returns. +This function 'dumps core' or otherwise fails in the hardest way +possible on the hosting operating system. This function never returns. [clinic start generated code]*/ static PyObject * os_abort_impl(PyObject *module) -/*[clinic end generated code: output=dcf52586dad2467c input=cf2c7d98bc504047]*/ +/*[clinic end generated code: output=dcf52586dad2467c input=ee8bd0ed690440ab]*/ { abort(); /*NOTREACHED*/ @@ -14856,17 +14972,18 @@ os.getxattr Return the value of extended attribute attribute on path. -path may be either a string, a path-like object, or an open file descriptor. -If follow_symlinks is False, and the last element of the path is a symbolic - link, getxattr will examine the symbolic link itself instead of the file - the link points to. +path may be either a string, a path-like object, or an open file +descriptor. +If follow_symlinks is False, and the last element of the path is +a symbolic link, getxattr will examine the symbolic link itself +instead of the file the link points to. [clinic start generated code]*/ static PyObject * os_getxattr_impl(PyObject *module, path_t *path, path_t *attribute, int follow_symlinks) -/*[clinic end generated code: output=5f2f44200a43cff2 input=025789491708f7eb]*/ +/*[clinic end generated code: output=5f2f44200a43cff2 input=db1021ed738d9754]*/ { Py_ssize_t i; PyObject *buffer = NULL; @@ -14934,17 +15051,18 @@ os.setxattr Set extended attribute attribute on path to value. -path may be either a string, a path-like object, or an open file descriptor. -If follow_symlinks is False, and the last element of the path is a symbolic - link, setxattr will modify the symbolic link itself instead of the file - the link points to. +path may be either a string, a path-like object, or an open file +descriptor. +If follow_symlinks is False, and the last element of the path is +a symbolic link, setxattr will modify the symbolic link itself instead +of the file the link points to. [clinic start generated code]*/ static PyObject * os_setxattr_impl(PyObject *module, path_t *path, path_t *attribute, Py_buffer *value, int flags, int follow_symlinks) -/*[clinic end generated code: output=98b83f63fdde26bb input=c17c0103009042f0]*/ +/*[clinic end generated code: output=98b83f63fdde26bb input=6c4ee6724e8947a4]*/ { ssize_t result; @@ -14987,17 +15105,18 @@ os.removexattr Remove extended attribute attribute on path. -path may be either a string, a path-like object, or an open file descriptor. -If follow_symlinks is False, and the last element of the path is a symbolic - link, removexattr will modify the symbolic link itself instead of the file - the link points to. +path may be either a string, a path-like object, or an open file +descriptor. +If follow_symlinks is False, and the last element of the path is +a symbolic link, removexattr will modify the symbolic link itself +instead of the file the link points to. [clinic start generated code]*/ static PyObject * os_removexattr_impl(PyObject *module, path_t *path, path_t *attribute, int follow_symlinks) -/*[clinic end generated code: output=521a51817980cda6 input=3d9a7d36fe2f7c4e]*/ +/*[clinic end generated code: output=521a51817980cda6 input=a7ec62a86aa83f01]*/ { ssize_t result; @@ -15034,16 +15153,17 @@ os.listxattr Return a list of extended attributes on path. -path may be either None, a string, a path-like object, or an open file descriptor. -if path is None, listxattr will examine the current directory. -If follow_symlinks is False, and the last element of the path is a symbolic - link, listxattr will examine the symbolic link itself instead of the file - the link points to. +path may be either None, a string, a path-like object, or an open file +descriptor. If path is None, listxattr will examine the current +directory. +If follow_symlinks is False, and the last element of the path is +a symbolic link, listxattr will examine the symbolic link itself instead +of the file the link points to. [clinic start generated code]*/ static PyObject * os_listxattr_impl(PyObject *module, path_t *path, int follow_symlinks) -/*[clinic end generated code: output=bebdb4e2ad0ce435 input=9826edf9fdb90869]*/ +/*[clinic end generated code: output=bebdb4e2ad0ce435 input=cb4a6414afaa99bd]*/ { Py_ssize_t i; PyObject *result = NULL; @@ -15164,14 +15284,14 @@ os_urandom_impl(PyObject *module, Py_ssize_t size) /*[clinic input] os.memfd_create - name: FSConverter + name: unicode_fs_encoded flags: unsigned_int(bitwise=True, c_default="MFD_CLOEXEC") = MFD_CLOEXEC [clinic start generated code]*/ static PyObject * os_memfd_create_impl(PyObject *module, PyObject *name, unsigned int flags) -/*[clinic end generated code: output=6681ede983bdb9a6 input=a42cfc199bcd56e9]*/ +/*[clinic end generated code: output=6681ede983bdb9a6 input=cd0eb092cfac474b]*/ { int fd; const char *bytes = PyBytes_AS_STRING(name); @@ -15945,7 +16065,7 @@ static PyMemberDef DirEntry_members[] = { {"name", Py_T_OBJECT_EX, offsetof(DirEntry, name), Py_READONLY, "the entry's base filename, relative to scandir() \"path\" argument"}, {"path", Py_T_OBJECT_EX, offsetof(DirEntry, path), Py_READONLY, - "the entry's full path name; equivalent to os.path.join(scandir_path, entry.name)"}, + "the entry's full path name; equivalent to\nos.path.join(scandir_path, entry.name)"}, {NULL} }; @@ -15960,7 +16080,7 @@ static PyMethodDef DirEntry_methods[] = { OS_DIRENTRY_INODE_METHODDEF OS_DIRENTRY___FSPATH___METHODDEF {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("DirEntry is generic over the type of the path (str or bytes)")}, {NULL} }; @@ -15983,13 +16103,19 @@ static PyType_Spec DirEntryType_spec = { #ifdef MS_WINDOWS +static int +is_extended_path(const wchar_t *path) +{ + return wcsncmp(path, L"\\\\?\\", 4) == 0; +} + static wchar_t * -join_path_filenameW(const wchar_t *path_wide, const wchar_t *filename) +join_path_filenameW(const wchar_t *path_wide, const wchar_t *filename, + int normalize) { Py_ssize_t path_len; - Py_ssize_t size; wchar_t *result; - wchar_t ch; + wchar_t *path_allocated = NULL; if (!path_wide) { /* Default arg: "." */ path_wide = L"."; @@ -15999,20 +16125,44 @@ join_path_filenameW(const wchar_t *path_wide, const wchar_t *filename) path_len = wcslen(path_wide); } - /* The +1's are for the path separator and the NUL */ - size = path_len + 1 + wcslen(filename) + 1; + if (path_len == 0) { + result = PyMem_New(wchar_t, 1); + if (result == NULL) { + PyErr_NoMemory(); + return NULL; + } + result[0] = L'\0'; + return result; + } + + if (normalize && !is_extended_path(path_wide)) { + int err = _PyOS_getfullpathname(path_wide, &path_allocated); + if (err < 0) { + PyErr_SetFromWindowsErr(0); + return NULL; + } + if (path_allocated == NULL) { + PyErr_NoMemory(); + return NULL; + } + path_wide = path_allocated; + path_len = wcslen(path_wide); + } + + size_t size = (size_t)path_len + 1 + wcslen(filename) + 1; result = PyMem_New(wchar_t, size); - if (!result) { + if (result == NULL) { + PyMem_RawFree(path_allocated); PyErr_NoMemory(); return NULL; } wcscpy(result, path_wide); - if (path_len > 0) { - ch = result[path_len - 1]; - if (ch != SEP && ch != ALTSEP && ch != L':') - result[path_len++] = SEP; - wcscpy(result + path_len, filename); + wchar_t ch = result[path_len - 1]; + if (ch != SEP && ch != ALTSEP && ch != L':') { + result[path_len++] = SEP; } + wcscpy(result + path_len, filename); + PyMem_RawFree(path_allocated); return result; } @@ -16044,7 +16194,7 @@ DirEntry_from_find_data(PyObject *module, path_t *path, WIN32_FIND_DATAW *dataW) goto error; } - joined_path = join_path_filenameW(path->wide, dataW->cFileName); + joined_path = join_path_filenameW(path->wide, dataW->cFileName, 0); if (!joined_path) goto error; @@ -16432,16 +16582,16 @@ os.scandir Return an iterator of DirEntry objects for given path. -path can be specified as either str, bytes, or a path-like object. If path -is bytes, the names of yielded DirEntry objects will also be bytes; in -all other circumstances they will be str. +path can be specified as either str, bytes, or a path-like object. If +path is bytes, the names of yielded DirEntry objects will also be bytes; +in all other circumstances they will be str. If path is None, uses the path='.'. [clinic start generated code]*/ static PyObject * os_scandir_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=6eb2668b675ca89e input=6bdd312708fc3bb0]*/ +/*[clinic end generated code: output=6eb2668b675ca89e input=6ab9600993f51577]*/ { ScandirIterator *iterator; #ifdef MS_WINDOWS @@ -16476,7 +16626,7 @@ os_scandir_impl(PyObject *module, path_t *path) #ifdef MS_WINDOWS iterator->first_time = 1; - path_strW = join_path_filenameW(iterator->path.wide, L"*.*"); + path_strW = join_path_filenameW(iterator->path.wide, L"*.*", 1); if (!path_strW) goto error; @@ -16594,14 +16744,14 @@ os.fspath Return the file system path representation of the object. -If the object is str or bytes, then allow it to pass through as-is. If the -object defines __fspath__(), then return the result of that method. All other -types raise a TypeError. +If the object is str or bytes, then allow it to pass through as-is. If +the object defines __fspath__(), then return the result of that method. +All other types raise a TypeError. [clinic start generated code]*/ static PyObject * os_fspath_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=c3c3b78ecff2914f input=e357165f7b22490f]*/ +/*[clinic end generated code: output=c3c3b78ecff2914f input=d3c54404240d5da0]*/ { return PyOS_FSPath(path); } @@ -16785,13 +16935,13 @@ On Unix: On Windows, return status shifted right by 8 bits. On Unix, if the process is being traced or if waitpid() was called with -WUNTRACED option, the caller must first check if WIFSTOPPED(status) is true. -This function must not be called if WIFSTOPPED(status) is true. +WUNTRACED option, the caller must first check if WIFSTOPPED(status) is +true. This function must not be called if WIFSTOPPED(status) is true. [clinic start generated code]*/ static PyObject * os_waitstatus_to_exitcode_impl(PyObject *module, PyObject *status_obj) -/*[clinic end generated code: output=db50b1b0ba3c7153 input=7fe2d7fdaea3db42]*/ +/*[clinic end generated code: output=db50b1b0ba3c7153 input=3b44a23f5090006c]*/ { #ifndef MS_WINDOWS int status = PyLong_AsInt(status_obj); @@ -16934,6 +17084,25 @@ os__emscripten_debugger_impl(PyObject *module) emscripten_debugger(); Py_RETURN_NONE; } + +EM_JS(void, emscripten_log_impl_js, (const char* arg), { + console.warn(UTF8ToString(arg)); +}); + +/*[clinic input] +os._emscripten_log + arg: str + +Log something to the JS console. Emscripten only. +[clinic start generated code]*/ + +static PyObject * +os__emscripten_log_impl(PyObject *module, const char *arg) +/*[clinic end generated code: output=9749e5e293c42784 input=350aa1f70bc1e905]*/ +{ + emscripten_log_impl_js(arg); + Py_RETURN_NONE; +} #endif /* __EMSCRIPTEN__ */ @@ -17153,6 +17322,7 @@ static PyMethodDef posix_methods[] = { OS__IS_INPUTHOOK_INSTALLED_METHODDEF OS__CREATE_ENVIRON_METHODDEF OS__EMSCRIPTEN_DEBUGGER_METHODDEF + OS__EMSCRIPTEN_LOG_METHODDEF {NULL, NULL} /* Sentinel */ }; diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index fa153d86543e999..688f01839a20cc1 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -3,12 +3,15 @@ #endif #include "Python.h" +#include "pycore_ceval.h" // _Py_EnterRecursiveCall() #include "pycore_import.h" // _PyImport_SetModule() #include "pycore_pyhash.h" // _Py_HashSecret #include "pycore_traceback.h" // _PyTraceback_Add() #include <stdbool.h> #include <stddef.h> // offsetof() + +#include "expat_config.h" #include "expat.h" #include "pyexpat.h" @@ -74,6 +77,15 @@ typedef struct { PyObject_HEAD XML_Parser itself; + /* + * Strong reference to a parent `xmlparseobject` if this parser + * is a child parser. Set to NULL if this parser is a root parser. + * This is needed to keep the parent parser alive as long as it has + * at least one child parser. + * + * See https://github.com/python/cpython/issues/139400 for details. + */ + PyObject *parent; int ordered_attributes; /* Return attributes as a list. */ int specified_attributes; /* Report only specified attributes. */ int in_callback; /* Is a callback active? */ @@ -120,50 +132,89 @@ CALL_XML_HANDLER_SETTER(const struct HandlerInfo *handler_info, setter(xml_parser, xml_handler); } +static int +set_xml_error_attr_code(PyObject *err, enum XML_Error code) +{ + PyObject *v = PyLong_FromLong((long)code); + int ok = v != NULL && PyObject_SetAttr(err, &_Py_ID(code), v) != -1; + Py_XDECREF(v); + return ok; +} + /* Set an integer attribute on the error object; return true on success, * false on an exception. */ static int -set_error_attr(PyObject *err, const char *name, int value) +set_xml_error_attr_location(PyObject *err, const char *name, XML_Size value) { - PyObject *v = PyLong_FromLong(value); + PyObject *v = PyLong_FromSize_t((size_t)value); + int ok = v != NULL && PyObject_SetAttrString(err, name, v) != -1; + Py_XDECREF(v); + return ok; +} - if (v == NULL || PyObject_SetAttrString(err, name, v) == -1) { - Py_XDECREF(v); - return 0; + +static PyObject * +set_xml_error(pyexpat_state *state, + enum XML_Error code, XML_Size lineno, XML_Size column, + const char *errmsg) +{ + PyObject *arg; + if (errmsg == NULL) { + arg = PyUnicode_FromFormat( + "%s: line %zu, column %zu", + XML_ErrorString(code), + (size_t)lineno, (size_t)column + ); + } + else { + arg = PyUnicode_FromStringAndSize(errmsg, strlen(errmsg)); + } + if (arg == NULL) { + return NULL; } - Py_DECREF(v); - return 1; + PyObject *res = PyObject_CallOneArg(state->error, arg); + Py_DECREF(arg); + if ( + res != NULL + && set_xml_error_attr_code(res, code) + && set_xml_error_attr_location(res, "lineno", lineno) + && set_xml_error_attr_location(res, "offset", column) + ) { + PyErr_SetObject(state->error, res); + } + Py_XDECREF(res); + return NULL; } +#define SET_XML_ERROR(STATE, SELF, CODE, ERRMSG) \ + do { \ + XML_Parser parser = SELF->itself; \ + assert(parser != NULL); \ + XML_Size lineno = XML_GetCurrentLineNumber(parser); \ + XML_Size column = XML_GetCurrentColumnNumber(parser); \ + (void)set_xml_error(state, CODE, lineno, column, ERRMSG); \ + } while (0) + /* Build and set an Expat exception, including positioning * information. Always returns NULL. */ static PyObject * set_error(pyexpat_state *state, xmlparseobject *self, enum XML_Error code) { - PyObject *err; - PyObject *buffer; - XML_Parser parser = self->itself; - int lineno = XML_GetErrorLineNumber(parser); - int column = XML_GetErrorColumnNumber(parser); + SET_XML_ERROR(state, self, code, NULL); + return NULL; +} - buffer = PyUnicode_FromFormat("%s: line %i, column %i", - XML_ErrorString(code), lineno, column); - if (buffer == NULL) - return NULL; - err = PyObject_CallOneArg(state->error, buffer); - Py_DECREF(buffer); - if ( err != NULL - && set_error_attr(err, "code", code) - && set_error_attr(err, "offset", column) - && set_error_attr(err, "lineno", lineno)) { - PyErr_SetObject(state->error, err); - } - Py_XDECREF(err); +static PyObject * +set_invalid_arg(pyexpat_state *state, xmlparseobject *self, const char *errmsg) +{ + SET_XML_ERROR(state, self, XML_ERROR_INVALID_ARGUMENT, errmsg); return NULL; } +#undef SET_XML_ERROR + static int have_handler(xmlparseobject *self, int type) { @@ -338,7 +389,7 @@ my_CharacterDataHandler(void *userData, const XML_Char *data, int len) if (self->buffer == NULL) call_character_handler(self, data, len); else { - if ((self->buffer_used + len) > self->buffer_size) { + if (len > (self->buffer_size - self->buffer_used)) { if (flush_character_buffer(self) < 0) return; /* handler might have changed; drop the rest on the floor @@ -553,6 +604,10 @@ static PyObject * conv_content_model(XML_Content * const model, PyObject *(*conv_string)(void *)) { + if (_Py_EnterRecursiveCall(" in conv_content_model")) { + return NULL; + } + PyObject *result = NULL; PyObject *children = PyTuple_New(model->numchildren); int i; @@ -564,7 +619,7 @@ conv_content_model(XML_Content * const model, conv_string); if (child == NULL) { Py_XDECREF(children); - return NULL; + goto done; } PyTuple_SET_ITEM(children, i, child); } @@ -572,6 +627,8 @@ conv_content_model(XML_Content * const model, model->type, model->quant, conv_string, model->name, children); } +done: + _Py_LeaveRecursiveCall(); return result; } @@ -588,7 +645,7 @@ my_ElementDeclHandler(void *userData, PyObject *modelobj, *nameobj; if (PyErr_Occurred()) - return; + goto finally; if (flush_character_buffer(self) < 0) goto finally; @@ -970,13 +1027,14 @@ pyexpat.xmlparser.GetInputContext Return the untranslated text of the input that caused the current event. -If the event was generated by a large amount of text (such as a start tag -for an element with many attributes), not all of the text may be available. +If the event was generated by a large amount of text (such as +a start tag for an element with many attributes), not all of the +text may be available. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_GetInputContext_impl(xmlparseobject *self) -/*[clinic end generated code: output=a88026d683fc22cc input=034df8712db68379]*/ +/*[clinic end generated code: output=a88026d683fc22cc input=3ff7cb00783c8f98]*/ { if (self->in_callback) { int offset, size; @@ -1028,6 +1086,10 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self, new_parser->ns_prefixes = self->ns_prefixes; new_parser->itself = XML_ExternalEntityParserCreate(self->itself, context, encoding); + // The new subparser will make use of the parent XML_Parser inside of Expat. + // So we need to take subparsers into account with the reference counting + // of their parent parser. + new_parser->parent = Py_NewRef(self); new_parser->handlers = 0; new_parser->intern = Py_XNewRef(self->intern); @@ -1104,15 +1166,16 @@ pyexpat.xmlparser.UseForeignDTD Allows the application to provide an artificial external subset if one is not specified as part of the document instance. -This readily allows the use of a 'default' document type controlled by the -application, while still getting the advantage of providing document type -information to the parser. 'flag' defaults to True if not provided. +This readily allows the use of a 'default' document type controlled +by the application, while still getting the advantage of providing +document type information to the parser. 'flag' defaults to True if +not provided. [clinic start generated code]*/ static PyObject * pyexpat_xmlparser_UseForeignDTD_impl(xmlparseobject *self, PyTypeObject *cls, int flag) -/*[clinic end generated code: output=d7d98252bd25a20f input=23440ecb0573fb29]*/ +/*[clinic end generated code: output=d7d98252bd25a20f input=0387bce44fd53d99]*/ { pyexpat_state *state = PyType_GetModuleState(cls); enum XML_Error rc; @@ -1125,6 +1188,185 @@ pyexpat_xmlparser_UseForeignDTD_impl(xmlparseobject *self, PyTypeObject *cls, } #endif +#if XML_COMBINED_VERSION >= 20400 +static PyObject * +set_activation_threshold(xmlparseobject *self, + PyTypeObject *cls, + unsigned long long threshold, + XML_Bool (*setter)(XML_Parser, unsigned long long)) +{ + assert(self->itself != NULL); + if (setter(self->itself, threshold) == XML_TRUE) { + Py_RETURN_NONE; + } + // The setter fails if self->itself is NULL (which is not possible here) + // or is a non-root parser, which currently only happens for parsers + // created by ExternalEntityParserCreate(). + pyexpat_state *state = PyType_GetModuleState(cls); + return set_invalid_arg(state, self, "parser must be a root parser"); +} + +static PyObject * +set_maximum_amplification(xmlparseobject *self, + PyTypeObject *cls, + float max_factor, + XML_Bool (*setter)(XML_Parser, float)) +{ + assert(self->itself != NULL); + if (setter(self->itself, max_factor) == XML_TRUE) { + Py_RETURN_NONE; + } + // The setter fails if self->itself is NULL (which is not possible here), + // is a non-root parser, which currently only happens for parsers created + // by ExternalEntityParserCreate(), or if 'max_factor' is NaN or < 1.0. + pyexpat_state *state = PyType_GetModuleState(cls); + // Note: Expat has no API to determine whether a parser is a root parser, + // and since the Expat functions for defining the various maximum allowed + // amplifcation factors fail when a bad parser or an out-of-range factor + // is given without specifying which check failed, we check whether the + // factor is out-of-range to improve the error message. See also gh-90949. + const char *message = (isnan(max_factor) || max_factor < 1.0f) + ? "'max_factor' must be at least 1.0" + : "parser must be a root parser"; + return set_invalid_arg(state, self, message); +} +#endif + +#if XML_COMBINED_VERSION >= 20400 +/*[clinic input] +pyexpat.xmlparser.SetBillionLaughsAttackProtectionActivationThreshold + + cls: defining_class + threshold: unsigned_long_long + / + +Sets the number of output bytes needed to activate protection against billion laughs attacks. + +The number of output bytes includes amplification from entity expansion +and reading DTD files. + +Parser objects usually have a protection activation threshold of 8 MiB, +but the actual default value depends on the underlying Expat library. + +Activation thresholds below 4 MiB are known to break support for DITA 1.3 +payload and are hence not recommended. +[clinic start generated code]*/ + +static PyObject * +pyexpat_xmlparser_SetBillionLaughsAttackProtectionActivationThreshold_impl(xmlparseobject *self, + PyTypeObject *cls, + unsigned long long threshold) +/*[clinic end generated code: output=0c082342f1c78114 input=5a51695a481def92]*/ +{ + return set_activation_threshold( + self, cls, threshold, + XML_SetBillionLaughsAttackProtectionActivationThreshold + ); +} +#endif + +#if XML_COMBINED_VERSION >= 20400 +/*[clinic input] +pyexpat.xmlparser.SetBillionLaughsAttackProtectionMaximumAmplification + + cls: defining_class + max_factor: float + / + +Sets the maximum tolerated amplification factor for protection against billion laughs attacks. + +The amplification factor is calculated as "(direct + indirect) / direct" +while parsing, where "direct" is the number of bytes read from the primary +document in parsing and "indirect" is the number of bytes added by expanding +entities and reading external DTD files, combined. + +The 'max_factor' value must be a non-NaN floating point value greater than +or equal to 1.0. Amplification factors greater than 30,000 can be observed +in the middle of parsing even with benign files in practice. In particular, +the activation threshold should be carefully chosen to avoid false positives. + +Parser objects usually have a maximum amplification factor of 100, +but the actual default value depends on the underlying Expat library. +[clinic start generated code]*/ + +static PyObject * +pyexpat_xmlparser_SetBillionLaughsAttackProtectionMaximumAmplification_impl(xmlparseobject *self, + PyTypeObject *cls, + float max_factor) +/*[clinic end generated code: output=c590439eadf463fa input=5de7c6dd7169b3b0]*/ +{ + return set_maximum_amplification( + self, cls, max_factor, + XML_SetBillionLaughsAttackProtectionMaximumAmplification + ); +} +#endif + +#if XML_COMBINED_VERSION >= 20702 +/*[clinic input] +pyexpat.xmlparser.SetAllocTrackerActivationThreshold + + cls: defining_class + threshold: unsigned_long_long + / + +Sets the number of allocated bytes of dynamic memory needed to activate protection against disproportionate use of RAM. + +Parser objects usually have an allocation activation threshold of 64 MiB, +but the actual default value depends on the underlying Expat library. +[clinic start generated code]*/ + +static PyObject * +pyexpat_xmlparser_SetAllocTrackerActivationThreshold_impl(xmlparseobject *self, + PyTypeObject *cls, + unsigned long long threshold) +/*[clinic end generated code: output=bed7e93207ba08c5 input=b74171709a77f2d9]*/ +{ + return set_activation_threshold( + self, cls, threshold, + XML_SetAllocTrackerActivationThreshold + ); +} +#endif + +#if XML_COMBINED_VERSION >= 20702 +/*[clinic input] +pyexpat.xmlparser.SetAllocTrackerMaximumAmplification + + cls: defining_class + max_factor: float + / + +Sets the maximum amplification factor between direct input and bytes of dynamic memory allocated. + +The amplification factor is calculated as "allocated / direct" while +parsing, where "direct" is the number of bytes read from the primary +document in parsing and "allocated" is the number of bytes of +dynamic memory allocated in the parser hierarchy. + +The 'max_factor' value must be a non-NaN floating point value +greater than or equal to 1.0. Amplification factors greater than +100.0 can be observed near the start of parsing even with benign +files in practice. In particular, the activation threshold should +be carefully chosen to avoid false positives. + +Parser objects usually have a maximum amplification factor of 100, +but the actual default value depends on the underlying Expat library. +[clinic start generated code]*/ + +static PyObject * +pyexpat_xmlparser_SetAllocTrackerMaximumAmplification_impl(xmlparseobject *self, + PyTypeObject *cls, + float max_factor) +/*[clinic end generated code: output=6e44bd48c9b112a0 input=db40271991462073]*/ +{ + return set_maximum_amplification( + self, cls, max_factor, + XML_SetAllocTrackerMaximumAmplification + ); +} +#endif + static struct PyMethodDef xmlparse_methods[] = { PYEXPAT_XMLPARSER_PARSE_METHODDEF PYEXPAT_XMLPARSER_PARSEFILE_METHODDEF @@ -1133,9 +1375,11 @@ static struct PyMethodDef xmlparse_methods[] = { PYEXPAT_XMLPARSER_GETINPUTCONTEXT_METHODDEF PYEXPAT_XMLPARSER_EXTERNALENTITYPARSERCREATE_METHODDEF PYEXPAT_XMLPARSER_SETPARAMENTITYPARSING_METHODDEF -#if XML_COMBINED_VERSION >= 19505 PYEXPAT_XMLPARSER_USEFOREIGNDTD_METHODDEF -#endif + PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONACTIVATIONTHRESHOLD_METHODDEF + PYEXPAT_XMLPARSER_SETBILLIONLAUGHSATTACKPROTECTIONMAXIMUMAMPLIFICATION_METHODDEF + PYEXPAT_XMLPARSER_SETALLOCTRACKERACTIVATIONTHRESHOLD_METHODDEF + PYEXPAT_XMLPARSER_SETALLOCTRACKERMAXIMUMAMPLIFICATION_METHODDEF PYEXPAT_XMLPARSER_SETREPARSEDEFERRALENABLED_METHODDEF PYEXPAT_XMLPARSER_GETREPARSEDEFERRALENABLED_METHODDEF {NULL, NULL} /* sentinel */ @@ -1242,13 +1486,17 @@ newxmlparseobject(pyexpat_state *state, const char *encoding, /* namespace_separator is either NULL or contains one char + \0 */ self->itself = XML_ParserCreate_MM(encoding, &ExpatMemoryHandler, namespace_separator); + self->parent = NULL; if (self->itself == NULL) { PyErr_SetString(PyExc_RuntimeError, "XML_ParserCreate failed"); Py_DECREF(self); return NULL; } -#if XML_COMBINED_VERSION >= 20100 +#if XML_COMBINED_VERSION >= 20800 + /* This feature was added upstream in libexpat 2.8.0. */ + XML_SetHashSalt16Bytes(self->itself, _Py_HashSecret.expat.hashsalt16); +#elif XML_COMBINED_VERSION >= 20100 /* This feature was added upstream in libexpat 2.1.0. */ XML_SetHashSalt(self->itself, (unsigned long)_Py_HashSecret.expat.hashsalt); @@ -1278,6 +1526,7 @@ xmlparse_traverse(PyObject *op, visitproc visit, void *arg) for (size_t i = 0; handler_info[i].name != NULL; i++) { Py_VISIT(self->handlers[i]); } + Py_VISIT(self->parent); Py_VISIT(Py_TYPE(op)); return 0; } @@ -1288,6 +1537,10 @@ xmlparse_clear(PyObject *op) xmlparseobject *self = xmlparseobject_CAST(op); clear_handlers(self, 0); Py_CLEAR(self->intern); + // NOTE: We cannot call Py_CLEAR(self->parent) prior to calling + // XML_ParserFree(self->itself), or a subparser could lose its parent + // XML_Parser while still making use of it internally. + // https://github.com/python/cpython/issues/139400 return 0; } @@ -1301,6 +1554,7 @@ xmlparse_dealloc(PyObject *op) XML_ParserFree(self->itself); } self->itself = NULL; + Py_CLEAR(self->parent); if (self->handlers != NULL) { PyMem_Free(self->handlers); @@ -2136,11 +2390,30 @@ pyexpat_exec(PyObject *mod) #else capi->SetHashSalt = NULL; #endif +#if XML_COMBINED_VERSION >= 20800 + capi->SetHashSalt16Bytes = XML_SetHashSalt16Bytes; +#else + capi->SetHashSalt16Bytes = NULL; +#endif #if XML_COMBINED_VERSION >= 20600 capi->SetReparseDeferralEnabled = XML_SetReparseDeferralEnabled; #else capi->SetReparseDeferralEnabled = NULL; #endif +#if XML_COMBINED_VERSION >= 20702 + capi->SetAllocTrackerActivationThreshold = XML_SetAllocTrackerActivationThreshold; + capi->SetAllocTrackerMaximumAmplification = XML_SetAllocTrackerMaximumAmplification; +#else + capi->SetAllocTrackerActivationThreshold = NULL; + capi->SetAllocTrackerMaximumAmplification = NULL; +#endif +#if XML_COMBINED_VERSION >= 20400 + capi->SetBillionLaughsAttackProtectionActivationThreshold = XML_SetBillionLaughsAttackProtectionActivationThreshold; + capi->SetBillionLaughsAttackProtectionMaximumAmplification = XML_SetBillionLaughsAttackProtectionMaximumAmplification; +#else + capi->SetBillionLaughsAttackProtectionActivationThreshold = NULL; + capi->SetBillionLaughsAttackProtectionMaximumAmplification = NULL; +#endif /* export using capsule */ PyObject *capi_object = PyCapsule_New(capi, PyExpat_CAPSULE_NAME, @@ -2211,6 +2484,9 @@ PyInit_pyexpat(void) static void clear_handlers(xmlparseobject *self, int initial) { + if (self->handlers == NULL) { + return; + } for (size_t i = 0; handler_info[i].name != NULL; i++) { if (initial) { self->handlers[i] = NULL; diff --git a/Modules/readline.c b/Modules/readline.c index 0dd99dc66c08e9d..7708f47d4d03279 100644 --- a/Modules/readline.c +++ b/Modules/readline.c @@ -255,6 +255,7 @@ readline_read_init_file_impl(PyObject *module, PyObject *filename_obj) if (!PyUnicode_FSConverter(filename_obj, &filename_bytes)) return NULL; if (PySys_Audit("open", "OCi", filename_obj, 'r', 0) < 0) { + Py_DECREF(filename_bytes); return NULL; } errno = rl_read_init_file(PyBytes_AS_STRING(filename_bytes)); @@ -298,6 +299,7 @@ readline_read_history_file_impl(PyObject *module, PyObject *filename_obj) if (!PyUnicode_FSConverter(filename_obj, &filename_bytes)) return NULL; if (PySys_Audit("open", "OCi", filename_obj, 'r', 0) < 0) { + Py_DECREF(filename_bytes); return NULL; } errno = read_history(PyBytes_AS_STRING(filename_bytes)); @@ -343,6 +345,7 @@ readline_write_history_file_impl(PyObject *module, PyObject *filename_obj) return NULL; filename = PyBytes_AS_STRING(filename_bytes); if (PySys_Audit("open", "OCi", filename_obj, 'w', 0) < 0) { + Py_DECREF(filename_bytes); return NULL; } } else { @@ -388,7 +391,7 @@ readline_append_history_file_impl(PyObject *module, int nelements, { if (nelements < 0) { - PyErr_SetString(PyExc_ValueError, "nelements must be positive"); + PyErr_SetString(PyExc_ValueError, "nelements must be non-negative"); return NULL; } @@ -400,6 +403,7 @@ readline_append_history_file_impl(PyObject *module, int nelements, return NULL; filename = PyBytes_AS_STRING(filename_bytes); if (PySys_Audit("open", "OCi", filename_obj, 'a', 0) < 0) { + Py_DECREF(filename_bytes); return NULL; } } else { @@ -1378,6 +1382,10 @@ setup_readline(readlinestate *mod_state) completer_word_break_characters = strdup(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?"); /* All nonalphanums except '.' */ + + if (!completer_word_break_characters) { + goto error; + } #ifdef WITH_EDITLINE // libedit uses rl_basic_word_break_characters instead of // rl_completer_word_break_characters as complete delimiter @@ -1421,6 +1429,10 @@ setup_readline(readlinestate *mod_state) RESTORE_LOCALE(saved_locale) return 0; + +error: + RESTORE_LOCALE(saved_locale) + return -1; } /* Wrapper around GNU readline that handles signals differently. */ diff --git a/Modules/resource.c b/Modules/resource.c index 3fe18e7c98e3d81..2353bc6653abd88 100644 --- a/Modules/resource.c +++ b/Modules/resource.c @@ -1,7 +1,5 @@ -// Need limited C API version 3.13 for PySys_Audit() -#include "pyconfig.h" // Py_GIL_DISABLED -#ifndef Py_GIL_DISABLED -# define Py_LIMITED_API 0x030d0000 +#ifndef Py_BUILD_CORE_BUILTIN +# define Py_BUILD_CORE_MODULE 1 #endif #include "Python.h" @@ -150,6 +148,35 @@ resource_getrusage_impl(PyObject *module, int who) } #endif +static int +py2rlim(PyObject *obj, rlim_t *out) +{ + obj = PyNumber_Index(obj); + if (obj == NULL) { + return -1; + } + int neg = PyLong_IsNegative(obj); + assert(neg >= 0); + Py_ssize_t bytes = PyLong_AsNativeBytes(obj, out, sizeof(*out), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + Py_DECREF(obj); + if (bytes < 0) { + return -1; + } + else if (neg && (*out != RLIM_INFINITY || bytes > (Py_ssize_t)sizeof(*out))) { + PyErr_SetString(PyExc_ValueError, + "Cannot convert negative int"); + return -1; + } + else if (bytes > (Py_ssize_t)sizeof(*out)) { + PyErr_SetString(PyExc_OverflowError, + "Python int too large to convert to C rlim_t"); + return -1; + } + return 0; +} + static int py2rlimit(PyObject *limits, struct rlimit *rl_out) { @@ -166,26 +193,13 @@ py2rlimit(PyObject *limits, struct rlimit *rl_out) } curobj = PyTuple_GetItem(limits, 0); // borrowed maxobj = PyTuple_GetItem(limits, 1); // borrowed -#if !defined(HAVE_LARGEFILE_SUPPORT) - rl_out->rlim_cur = PyLong_AsLong(curobj); - if (rl_out->rlim_cur == (rlim_t)-1 && PyErr_Occurred()) - goto error; - rl_out->rlim_max = PyLong_AsLong(maxobj); - if (rl_out->rlim_max == (rlim_t)-1 && PyErr_Occurred()) - goto error; -#else - /* The limits are probably bigger than a long */ - rl_out->rlim_cur = PyLong_AsLongLong(curobj); - if (rl_out->rlim_cur == (rlim_t)-1 && PyErr_Occurred()) - goto error; - rl_out->rlim_max = PyLong_AsLongLong(maxobj); - if (rl_out->rlim_max == (rlim_t)-1 && PyErr_Occurred()) + if (py2rlim(curobj, &rl_out->rlim_cur) < 0 || + py2rlim(maxobj, &rl_out->rlim_max) < 0) + { goto error; -#endif + } Py_DECREF(limits); - rl_out->rlim_cur = rl_out->rlim_cur & RLIM_INFINITY; - rl_out->rlim_max = rl_out->rlim_max & RLIM_INFINITY; return 0; error: @@ -193,15 +207,24 @@ py2rlimit(PyObject *limits, struct rlimit *rl_out) return -1; } +static PyObject* +rlim2py(rlim_t value) +{ + if (value == RLIM_INFINITY) { + return PyLong_FromNativeBytes(&value, sizeof(value), -1); + } + return PyLong_FromUnsignedNativeBytes(&value, sizeof(value), -1); +} + static PyObject* rlimit2py(struct rlimit rl) { - if (sizeof(rl.rlim_cur) > sizeof(long)) { - return Py_BuildValue("LL", - (long long) rl.rlim_cur, - (long long) rl.rlim_max); + PyObject *cur = rlim2py(rl.rlim_cur); + if (cur == NULL) { + return NULL; } - return Py_BuildValue("ll", (long) rl.rlim_cur, (long) rl.rlim_max); + PyObject *max = rlim2py(rl.rlim_max); + return Py_BuildValue("NN", cur, max); } /*[clinic input] @@ -495,14 +518,7 @@ resource_exec(PyObject *module) ADD_INT(module, RLIMIT_KQUEUES); #endif - PyObject *v; - if (sizeof(RLIM_INFINITY) > sizeof(long)) { - v = PyLong_FromLongLong((long long) RLIM_INFINITY); - } else - { - v = PyLong_FromLong((long) RLIM_INFINITY); - } - if (PyModule_Add(module, "RLIM_INFINITY", v) < 0) { + if (PyModule_Add(module, "RLIM_INFINITY", rlim2py(RLIM_INFINITY)) < 0) { return -1; } return 0; diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c index d234d504cb51674..42bf1157ccc5224 100644 --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -251,7 +251,8 @@ select.select Wait until one or more file descriptors are ready for some kind of I/O. -The first three arguments are iterables of file descriptors to be waited for: +The first three arguments are iterables of file descriptors to be waited +for: rlist -- wait until ready for reading wlist -- wait until ready for writing xlist -- wait for an "exceptional condition" @@ -264,9 +265,9 @@ The optional 4th argument specifies a timeout in seconds; it may be a floating-point number to specify fractions of seconds. If it is absent or None, the call will never time out. -The return value is a tuple of three lists corresponding to the first three -arguments; each contains the subset of the corresponding file descriptors -that are ready. +The return value is a tuple of three lists corresponding to the first +three arguments; each contains the subset of the corresponding file +descriptors that are ready. *** IMPORTANT NOTICE *** On Windows, only sockets are supported; on Unix, all file @@ -276,7 +277,7 @@ descriptors can be used. static PyObject * select_select_impl(PyObject *module, PyObject *rlist, PyObject *wlist, PyObject *xlist, PyObject *timeout_obj) -/*[clinic end generated code: output=2b3cfa824f7ae4cf input=1199d5e101abca4a]*/ +/*[clinic end generated code: output=2b3cfa824f7ae4cf input=34a2c2075ca9830e]*/ { #ifdef SELECT_USES_HEAP pylist *rfd2obj, *wfd2obj, *efd2obj; @@ -613,13 +614,13 @@ select.poll.poll Polls the set of registered file descriptors. -Returns a list containing any descriptors that have events or errors to -report, as a list of (fd, event) 2-tuples. +Returns a list containing any descriptors that have events or errors +to report, as a list of (fd, event) 2-tuples. [clinic start generated code]*/ static PyObject * select_poll_poll_impl(pollObject *self, PyObject *timeout_obj) -/*[clinic end generated code: output=876e837d193ed7e4 input=54310631457efdec]*/ +/*[clinic end generated code: output=876e837d193ed7e4 input=e0a9c0aa283de8c8]*/ { PyObject *result_list = NULL; int poll_result, i, j; @@ -941,19 +942,19 @@ select_devpoll_unregister_impl(devpollObject *self, int fd) @critical_section select.devpoll.poll timeout as timeout_obj: object = None - The maximum time to wait in milliseconds, or else None (or a negative - value) to wait indefinitely. + The maximum time to wait in milliseconds, or else None (or + a negative value) to wait indefinitely. / Polls the set of registered file descriptors. -Returns a list containing any descriptors that have events or errors to -report, as a list of (fd, event) 2-tuples. +Returns a list containing any descriptors that have events or errors +to report, as a list of (fd, event) 2-tuples. [clinic start generated code]*/ static PyObject * select_devpoll_poll_impl(devpollObject *self, PyObject *timeout_obj) -/*[clinic end generated code: output=2654e5457cca0b3c input=fe7a3f6dcbc118c5]*/ +/*[clinic end generated code: output=2654e5457cca0b3c input=9e1672658d728539]*/ { struct dvpoll dvp; PyObject *result_list = NULL; @@ -1203,13 +1204,13 @@ select.poll Returns a polling object. -This object supports registering and unregistering file descriptors, and then -polling them for I/O events. +This object supports registering and unregistering file descriptors, and +then polling them for I/O events. [clinic start generated code]*/ static PyObject * select_poll_impl(PyObject *module) -/*[clinic end generated code: output=16a665a4e1d228c5 input=3f877909d5696bbf]*/ +/*[clinic end generated code: output=16a665a4e1d228c5 input=0aefd4527e99e0aa]*/ { return (PyObject *)newPollObject(module); } @@ -1221,13 +1222,13 @@ select.devpoll Returns a polling object. -This object supports registering and unregistering file descriptors, and then -polling them for I/O events. +This object supports registering and unregistering file descriptors, and +then polling them for I/O events. [clinic start generated code]*/ static PyObject * select_devpoll_impl(PyObject *module) -/*[clinic end generated code: output=ea9213cc87fd9581 input=53a1af94564f00a3]*/ +/*[clinic end generated code: output=ea9213cc87fd9581 input=4c2ac27d10248526]*/ { return (PyObject *)newDevPollObject(module); } @@ -1569,14 +1570,14 @@ select.epoll.poll Wait for events on the epoll file descriptor. -Returns a list containing any descriptors that have events to report, -as a list of (fd, events) 2-tuples. +Returns a list containing any descriptors that have events to +report, as a list of (fd, events) 2-tuples. [clinic start generated code]*/ static PyObject * select_epoll_poll_impl(pyEpoll_Object *self, PyObject *timeout_obj, int maxevents) -/*[clinic end generated code: output=e02d121a20246c6c input=33d34a5ea430fd5b]*/ +/*[clinic end generated code: output=e02d121a20246c6c input=5a49d65788c70c7a]*/ { int nfds, i; PyObject *elist = NULL, *etuple = NULL; diff --git a/Modules/sha1module.c b/Modules/sha1module.c index f4a00cdb4221567..a746bf74f8d4c19 100644 --- a/Modules/sha1module.c +++ b/Modules/sha1module.c @@ -272,19 +272,25 @@ static PyType_Spec sha1_type_spec = { /*[clinic input] _sha1.sha1 - string: object(c_default="NULL") = b'' + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string as string_obj: object(c_default="NULL") = None Return a new SHA1 hash object; optionally initialized with a string. [clinic start generated code]*/ static PyObject * -_sha1_sha1_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=6f8b3af05126e18e input=bd54b68e2bf36a8a]*/ +_sha1_sha1_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj) +/*[clinic end generated code: output=0d453775924f88a7 input=807f25264e0ac656]*/ { SHA1object *new; Py_buffer buf; + PyObject *string; + if (_Py_hashlib_data_argument(&string, data, string_obj) < 0) { + return NULL; + } if (string) { GET_BUFFER_VIEW_OR_ERROUT(string, &buf); diff --git a/Modules/sha2module.c b/Modules/sha2module.c index e88d7cb2d456bf0..72931910c5d7207 100644 --- a/Modules/sha2module.c +++ b/Modules/sha2module.c @@ -594,18 +594,24 @@ static PyType_Spec sha512_type_spec = { /*[clinic input] _sha2.sha256 - string: object(c_default="NULL") = b'' + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string as string_obj: object(c_default="NULL") = None Return a new SHA-256 hash object; optionally initialized with a string. [clinic start generated code]*/ static PyObject * -_sha2_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=243c9dd289931f87 input=6249da1de607280a]*/ +_sha2_sha256_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj) +/*[clinic end generated code: output=49828a7bcd418f45 input=9ce1d70e669abc14]*/ { Py_buffer buf; + PyObject *string; + if (_Py_hashlib_data_argument(&string, data, string_obj) < 0) { + return NULL; + } if (string) { GET_BUFFER_VIEW_OR_ERROUT(string, &buf); @@ -651,18 +657,25 @@ _sha2_sha256_impl(PyObject *module, PyObject *string, int usedforsecurity) /*[clinic input] _sha2.sha224 - string: object(c_default="NULL") = b'' + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string as string_obj: object(c_default="NULL") = None Return a new SHA-224 hash object; optionally initialized with a string. [clinic start generated code]*/ static PyObject * -_sha2_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=68191f232e4a3843 input=c42bcba47fd7d2b7]*/ +_sha2_sha224_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj) +/*[clinic end generated code: output=2163cb03b6cf6157 input=612f7682a889bc2a]*/ { Py_buffer buf; + PyObject *string; + if (_Py_hashlib_data_argument(&string, data, string_obj) < 0) { + return NULL; + } + if (string) { GET_BUFFER_VIEW_OR_ERROUT(string, &buf); } @@ -706,19 +719,25 @@ _sha2_sha224_impl(PyObject *module, PyObject *string, int usedforsecurity) /*[clinic input] _sha2.sha512 - string: object(c_default="NULL") = b'' + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string as string_obj: object(c_default="NULL") = None Return a new SHA-512 hash object; optionally initialized with a string. [clinic start generated code]*/ static PyObject * -_sha2_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=d55c8996eca214d7 input=0576ae2a6ebfad25]*/ +_sha2_sha512_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj) +/*[clinic end generated code: output=cc3fcfce001a4538 input=19c9f2c06d59563a]*/ { SHA512object *new; Py_buffer buf; + PyObject *string; + if (_Py_hashlib_data_argument(&string, data, string_obj) < 0) { + return NULL; + } sha2_state *state = sha2_get_state(module); @@ -763,19 +782,25 @@ _sha2_sha512_impl(PyObject *module, PyObject *string, int usedforsecurity) /*[clinic input] _sha2.sha384 - string: object(c_default="NULL") = b'' + data: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string as string_obj: object(c_default="NULL") = None Return a new SHA-384 hash object; optionally initialized with a string. [clinic start generated code]*/ static PyObject * -_sha2_sha384_impl(PyObject *module, PyObject *string, int usedforsecurity) -/*[clinic end generated code: output=b29a0d81d51d1368 input=4e9199d8de0d2f9b]*/ +_sha2_sha384_impl(PyObject *module, PyObject *data, int usedforsecurity, + PyObject *string_obj) +/*[clinic end generated code: output=b6e3db593b5a0330 input=9fd50c942ad9e0bf]*/ { SHA512object *new; Py_buffer buf; + PyObject *string; + if (_Py_hashlib_data_argument(&string, data, string_obj) < 0) { + return NULL; + } sha2_state *state = sha2_get_state(module); diff --git a/Modules/sha3module.c b/Modules/sha3module.c index a7edf5c66a1e76e..cfbf0cbcc042c5e 100644 --- a/Modules/sha3module.c +++ b/Modules/sha3module.c @@ -105,18 +105,25 @@ sha3_update(Hacl_Hash_SHA3_state_t *state, uint8_t *buf, Py_ssize_t len) /*[clinic input] @classmethod _sha3.sha3_224.__new__ as py_sha3_new - data: object(c_default="NULL") = b'' - / + + data as data_obj: object(c_default="NULL") = b'' * usedforsecurity: bool = True + string: object(c_default="NULL") = None Return a new SHA3 hash object. [clinic start generated code]*/ static PyObject * -py_sha3_new_impl(PyTypeObject *type, PyObject *data, int usedforsecurity) -/*[clinic end generated code: output=90409addc5d5e8b0 input=637e5f8f6a93982a]*/ +py_sha3_new_impl(PyTypeObject *type, PyObject *data_obj, int usedforsecurity, + PyObject *string) +/*[clinic end generated code: output=dcec1eca20395f2a input=c106e0b4e2d67d58]*/ { + PyObject *data; + if (_Py_hashlib_data_argument(&data, data_obj, string) < 0) { + return NULL; + } + Py_buffer buf = {NULL, NULL}; SHA3State *state = _PyType_GetModuleState(type); SHA3object *self = newSHA3object(type); @@ -503,14 +510,13 @@ _SHAKE_digest(PyObject *op, unsigned long digestlen, int hex) _sha3.shake_128.digest length: unsigned_long - / Return the digest value as a bytes object. [clinic start generated code]*/ static PyObject * _sha3_shake_128_digest_impl(SHA3object *self, unsigned long length) -/*[clinic end generated code: output=2313605e2f87bb8f input=418ef6a36d2e6082]*/ +/*[clinic end generated code: output=2313605e2f87bb8f input=93d6d6ff32904f18]*/ { return _SHAKE_digest((PyObject *)self, length, 0); } @@ -520,14 +526,13 @@ _sha3_shake_128_digest_impl(SHA3object *self, unsigned long length) _sha3.shake_128.hexdigest length: unsigned_long - / Return the digest value as a string of hexadecimal digits. [clinic start generated code]*/ static PyObject * _sha3_shake_128_hexdigest_impl(SHA3object *self, unsigned long length) -/*[clinic end generated code: output=bf8e2f1e490944a8 input=69fb29b0926ae321]*/ +/*[clinic end generated code: output=bf8e2f1e490944a8 input=562d74e7060b56ab]*/ { return _SHAKE_digest((PyObject *)self, length, 1); } diff --git a/Modules/signalmodule.c b/Modules/signalmodule.c index 54bcd3270ef31ac..ee279703c798d82 100644 --- a/Modules/signalmodule.c +++ b/Modules/signalmodule.c @@ -469,16 +469,17 @@ signal.signal Set the action for the given signal. The action can be SIG_DFL, SIG_IGN, or a callable Python object. -The previous action is returned. See getsignal() for possible return values. +The previous action is returned. See getsignal() for possible return +values. *** IMPORTANT NOTICE *** -A signal handler function is called with two arguments: -the first is the signal number, the second is the interrupted stack frame. +A signal handler function is called with two arguments: the first is +the signal number, the second is the interrupted stack frame. [clinic start generated code]*/ static PyObject * signal_signal_impl(PyObject *module, int signalnum, PyObject *handler) -/*[clinic end generated code: output=b44cfda43780f3a1 input=deee84af5fa0432c]*/ +/*[clinic end generated code: output=b44cfda43780f3a1 input=99ce4035ec56ffc1]*/ { _signal_module_state *modstate = get_signal_state(module); PyObject *old_handler; @@ -856,8 +857,8 @@ signal.setitimer Sets given itimer (one of ITIMER_REAL, ITIMER_VIRTUAL or ITIMER_PROF). -The timer will fire after value seconds and after that every interval seconds. -The itimer can be cleared by setting seconds to zero. +The timer will fire after value seconds and after that every interval +seconds. The itimer can be cleared by setting seconds to zero. Returns old values as a tuple: (delay, interval). [clinic start generated code]*/ @@ -865,7 +866,7 @@ Returns old values as a tuple: (delay, interval). static PyObject * signal_setitimer_impl(PyObject *module, int which, PyObject *seconds, PyObject *interval) -/*[clinic end generated code: output=65f9dcbddc35527b input=de43daf194e6f66f]*/ +/*[clinic end generated code: output=65f9dcbddc35527b input=bd9f0d2ed8614193]*/ { _signal_module_state *modstate = get_signal_state(module); @@ -1026,13 +1027,13 @@ signal.sigwait Wait for a signal. Suspend execution of the calling thread until the delivery of one of the -signals specified in the signal set sigset. The function accepts the signal -and returns the signal number. +signals specified in the signal set sigset. The function accepts the +signal and returns the signal number. [clinic start generated code]*/ static PyObject * signal_sigwait_impl(PyObject *module, sigset_t sigset) -/*[clinic end generated code: output=f43770699d682f96 input=a6fbd47b1086d119]*/ +/*[clinic end generated code: output=f43770699d682f96 input=91773742dd416a3e]*/ { int err, signum; @@ -1180,7 +1181,13 @@ signal_sigwaitinfo_impl(PyObject *module, sigset_t sigset) err = sigwaitinfo(&sigset, &si); Py_END_ALLOW_THREADS } while (err == -1 - && errno == EINTR && !(async_err = PyErr_CheckSignals())); + && (errno == EINTR +#if defined(__NetBSD__) + /* NetBSD's implementation violates POSIX by setting + * errno to ECANCELED instead of EINTR. */ + || errno == ECANCELED +#endif + ) && !(async_err = PyErr_CheckSignals())); if (err == -1) return (!async_err) ? PyErr_SetFromErrno(PyExc_OSError) : NULL; @@ -1623,7 +1630,7 @@ signal_module_exec(PyObject *m) modstate->ignore_handler = state->ignore_handler; // borrowed ref #ifdef PYHAVE_ITIMER_ERROR - modstate->itimer_error = PyErr_NewException("signal.itimer_error", + modstate->itimer_error = PyErr_NewException("signal.ItimerError", PyExc_OSError, NULL); if (modstate->itimer_error == NULL) { return -1; @@ -1940,7 +1947,7 @@ signal_install_handlers(void) /* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. * * All of the code in this function must only use async-signal-safe functions, - * listed at `man 7 signal` or + * listed at `man 7 signal-safety` or * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html. * * If this function is updated, update also _posix_spawn() of subprocess.py. diff --git a/Modules/socketmodule.c b/Modules/socketmodule.c index 479583792637934..73f55ff05742a25 100644 --- a/Modules/socketmodule.c +++ b/Modules/socketmodule.c @@ -111,7 +111,6 @@ Local naming conventions: #include "pycore_moduleobject.h" // _PyModule_GetState #include "pycore_time.h" // _PyTime_AsMilliseconds() #include "pycore_pystate.h" // _Py_AssertHoldsTstate() -#include "pycore_pyatomic_ft_wrappers.h" #ifdef _Py_MEMORY_SANITIZER # include <sanitizer/msan_interface.h> @@ -151,7 +150,7 @@ listen([n]) -- start listening for incoming connections\n\ recv(buflen[, flags]) -- receive data\n\ recv_into(buffer[, nbytes[, flags]]) -- receive data (into a buffer)\n\ recvfrom(buflen[, flags]) -- receive data and sender\'s address\n\ -recvfrom_into(buffer[, nbytes, [, flags])\n\ +recvfrom_into(buffer[, nbytes, [, flags]])\n\ -- receive data and sender\'s address (into a buffer)\n\ sendall(data[, flags]) -- send all data\n\ send(data[, flags]) -- send data, may not send all of it\n\ @@ -565,7 +564,6 @@ static int sock_cloexec_works = -1; static inline void set_sock_fd(PySocketSockObject *s, SOCKET_T fd) { -#ifdef Py_GIL_DISABLED #if SIZEOF_SOCKET_T == SIZEOF_INT _Py_atomic_store_int_relaxed((int *)&s->sock_fd, (int)fd); #elif SIZEOF_SOCKET_T == SIZEOF_LONG @@ -575,15 +573,11 @@ set_sock_fd(PySocketSockObject *s, SOCKET_T fd) #else #error "Unsupported SIZEOF_SOCKET_T" #endif -#else - s->sock_fd = fd; -#endif } static inline SOCKET_T get_sock_fd(PySocketSockObject *s) { -#ifdef Py_GIL_DISABLED #if SIZEOF_SOCKET_T == SIZEOF_INT return (SOCKET_T)_Py_atomic_load_int_relaxed((int *)&s->sock_fd); #elif SIZEOF_SOCKET_T == SIZEOF_LONG @@ -593,9 +587,6 @@ get_sock_fd(PySocketSockObject *s) #else #error "Unsupported SIZEOF_SOCKET_T" #endif -#else - return s->sock_fd; -#endif } #define _PySocketSockObject_CAST(op) ((PySocketSockObject *)(op)) @@ -638,33 +629,22 @@ _PyLong_##NAME##_Converter(PyObject *obj, void *ptr) \ return 1; \ } -UNSIGNED_INT_CONVERTER(UInt16, uint16_t) -UNSIGNED_INT_CONVERTER(UInt32, uint32_t) - #if defined(HAVE_IF_NAMEINDEX) || defined(MS_WINDOWS) # ifdef MS_WINDOWS UNSIGNED_INT_CONVERTER(NetIfindex, NET_IFINDEX) # else - UNSIGNED_INT_CONVERTER(NetIfindex, unsigned int) +# define _PyLong_NetIfindex_Converter _PyLong_UnsignedInt_Converter # define NET_IFINDEX unsigned int # endif #endif // defined(HAVE_IF_NAMEINDEX) || defined(MS_WINDOWS) /*[python input] -class uint16_converter(CConverter): - type = "uint16_t" - converter = '_PyLong_UInt16_Converter' - -class uint32_converter(CConverter): - type = "uint32_t" - converter = '_PyLong_UInt32_Converter' - class NET_IFINDEX_converter(CConverter): type = "NET_IFINDEX" converter = '_PyLong_NetIfindex_Converter' [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=3de2e4a03fbf83b8]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=1cf809c40a407c34]*/ /*[clinic input] module _socket @@ -736,12 +716,6 @@ select_error(void) # define SOCK_INPROGRESS_ERR EINPROGRESS #endif -#ifdef _MSC_VER -# define SUPPRESS_DEPRECATED_CALL __pragma(warning(suppress: 4996)) -#else -# define SUPPRESS_DEPRECATED_CALL -#endif - /* Convenience function to raise an error according to errno and return a NULL pointer from a function. */ @@ -2794,6 +2768,12 @@ getsockaddrlen(PySocketSockObject *s, socklen_t *len_ret) _Py_FALLTHROUGH; #endif /* AF_RDS */ +#ifdef AF_DIVERT + case AF_DIVERT: + /* FreeBSD divert(4) sockets use sockaddr_in: fall-through */ + _Py_FALLTHROUGH; +#endif /* AF_DIVERT */ + case AF_INET: { *len_ret = sizeof (struct sockaddr_in); @@ -3386,7 +3366,7 @@ sock_setsockopt(PyObject *self, PyObject *args) &level, &optname, &flag)) { #ifdef MS_WINDOWS if (optname == SIO_TCP_SET_ACK_FREQUENCY) { - int dummy; + DWORD dummy; res = WSAIoctl(get_sock_fd(s), SIO_TCP_SET_ACK_FREQUENCY, &flag, sizeof(flag), NULL, 0, &dummy, NULL, NULL); if (res >= 0) { @@ -4517,17 +4497,19 @@ sock_recvmsg_into(PyObject *self, PyObject *args) struct iovec *iovs = NULL; Py_ssize_t i, nitems, nbufs = 0; Py_buffer *bufs = NULL; - PyObject *buffers_arg, *fast, *retval = NULL; + PyObject *buffers_arg, *buffers_tuple, *retval = NULL; if (!PyArg_ParseTuple(args, "O|ni:recvmsg_into", &buffers_arg, &ancbufsize, &flags)) return NULL; - if ((fast = PySequence_Fast(buffers_arg, - "recvmsg_into() argument 1 must be an " - "iterable")) == NULL) + buffers_tuple = PySequence_Tuple(buffers_arg); + if (buffers_tuple == NULL) { + PyErr_SetString(PyExc_TypeError, + "recvmsg_into() argument 1 must be an iterable"); return NULL; - nitems = PySequence_Fast_GET_SIZE(fast); + } + nitems = PyTuple_GET_SIZE(buffers_tuple); if (nitems > INT_MAX) { PyErr_SetString(PyExc_OSError, "recvmsg_into() argument 1 is too long"); goto finally; @@ -4541,7 +4523,7 @@ sock_recvmsg_into(PyObject *self, PyObject *args) goto finally; } for (; nbufs < nitems; nbufs++) { - if (!PyArg_Parse(PySequence_Fast_GET_ITEM(fast, nbufs), + if (!PyArg_Parse(PyTuple_GET_ITEM(buffers_tuple, nbufs), "w*;recvmsg_into() argument 1 must be an iterable " "of single-segment read-write buffers", &bufs[nbufs])) @@ -4557,7 +4539,7 @@ sock_recvmsg_into(PyObject *self, PyObject *args) PyBuffer_Release(&bufs[i]); PyMem_Free(bufs); PyMem_Free(iovs); - Py_DECREF(fast); + Py_DECREF(buffers_tuple); return retval; } @@ -4809,6 +4791,7 @@ sock_sendto(PyObject *self, PyObject *args) } if (PySys_Audit("socket.sendto", "OO", s, addro) < 0) { + PyBuffer_Release(&pbuf); return NULL; } @@ -4855,14 +4838,14 @@ sock_sendmsg_iovec(PySocketSockObject *s, PyObject *data_arg, /* Fill in an iovec for each message part, and save the Py_buffer structs to release afterwards. */ - data_fast = PySequence_Fast(data_arg, - "sendmsg() argument 1 must be an " - "iterable"); + data_fast = PySequence_Tuple(data_arg); if (data_fast == NULL) { + PyErr_SetString(PyExc_TypeError, + "sendmsg() argument 1 must be an iterable"); goto finally; } - ndataparts = PySequence_Fast_GET_SIZE(data_fast); + ndataparts = PyTuple_GET_SIZE(data_fast); if (ndataparts > INT_MAX) { PyErr_SetString(PyExc_OSError, "sendmsg() argument 1 is too long"); goto finally; @@ -4884,7 +4867,7 @@ sock_sendmsg_iovec(PySocketSockObject *s, PyObject *data_arg, } } for (; ndatabufs < ndataparts; ndatabufs++) { - if (!PyArg_Parse(PySequence_Fast_GET_ITEM(data_fast, ndatabufs), + if (!PyArg_Parse(PyTuple_GET_ITEM(data_fast, ndatabufs), "y*;sendmsg() argument 1 must be an iterable of " "bytes-like objects", &databufs[ndatabufs])) @@ -4966,11 +4949,13 @@ sock_sendmsg(PyObject *self, PyObject *args) if (cmsg_arg == NULL) ncmsgs = 0; else { - if ((cmsg_fast = PySequence_Fast(cmsg_arg, - "sendmsg() argument 2 must be an " - "iterable")) == NULL) + cmsg_fast = PySequence_Tuple(cmsg_arg); + if (cmsg_fast == NULL) { + PyErr_SetString(PyExc_TypeError, + "sendmsg() argument 2 must be an iterable"); goto finally; - ncmsgs = PySequence_Fast_GET_SIZE(cmsg_fast); + } + ncmsgs = PyTuple_GET_SIZE(cmsg_fast); } #ifndef CMSG_SPACE @@ -4990,8 +4975,9 @@ sock_sendmsg(PyObject *self, PyObject *args) controllen = controllen_last = 0; while (ncmsgbufs < ncmsgs) { size_t bufsize, space; + PyObject *item = PyTuple_GET_ITEM(cmsg_fast, ncmsgbufs); - if (!PyArg_Parse(PySequence_Fast_GET_ITEM(cmsg_fast, ncmsgbufs), + if (!PyArg_Parse(item, "(iiy*):[sendmsg() ancillary data items]", &cmsgs[ncmsgbufs].level, &cmsgs[ncmsgbufs].type, @@ -6215,8 +6201,10 @@ socket_gethostbyname_ex(PyObject *self, PyObject *args) #ifdef USE_GETHOSTBYNAME_LOCK PyThread_acquire_lock(netdb_lock, 1); #endif - SUPPRESS_DEPRECATED_CALL + _Py_COMP_DIAG_PUSH + _Py_COMP_DIAG_IGNORE_DEPR_DECLS h = gethostbyname(name); + _Py_COMP_DIAG_POP #endif /* HAVE_GETHOSTBYNAME_R */ Py_END_ALLOW_THREADS /* Some C libraries would require addr.__ss_family instead of @@ -6320,8 +6308,10 @@ socket_gethostbyaddr(PyObject *self, PyObject *args) #ifdef USE_GETHOSTBYNAME_LOCK PyThread_acquire_lock(netdb_lock, 1); #endif - SUPPRESS_DEPRECATED_CALL + _Py_COMP_DIAG_PUSH + _Py_COMP_DIAG_IGNORE_DEPR_DECLS h = gethostbyaddr(ap, al, af); + _Py_COMP_DIAG_POP #endif /* HAVE_GETHOSTBYNAME_R */ Py_END_ALLOW_THREADS ret = gethost_common(state, h, SAS2SA(&addr), sizeof(addr), af); @@ -6738,8 +6728,10 @@ _socket_inet_aton_impl(PyObject *module, const char *ip_addr) packed_addr = INADDR_BROADCAST; } else { - SUPPRESS_DEPRECATED_CALL + _Py_COMP_DIAG_PUSH + _Py_COMP_DIAG_IGNORE_DEPR_DECLS packed_addr = inet_addr(ip_addr); + _Py_COMP_DIAG_POP if (packed_addr == INADDR_NONE) { /* invalid address */ PyErr_SetString(PyExc_OSError, @@ -6782,8 +6774,10 @@ _socket_inet_ntoa_impl(PyObject *module, Py_buffer *packed_ip) memcpy(&packed_addr, packed_ip->buf, packed_ip->len); PyBuffer_Release(packed_ip); - SUPPRESS_DEPRECATED_CALL + _Py_COMP_DIAG_PUSH + _Py_COMP_DIAG_IGNORE_DEPR_DECLS return PyUnicode_FromString(inet_ntoa(packed_addr)); + _Py_COMP_DIAG_POP } #endif // HAVE_INET_NTOA @@ -6976,7 +6970,7 @@ socket_getaddrinfo(PyObject *self, PyObject *args, PyObject* kwargs) if (PySys_Audit("socket.getaddrinfo", "OOiii", hobj, pobj, family, socktype, protocol) < 0) { - return NULL; + goto err; } memset(&hints, 0, sizeof(hints)); @@ -7281,7 +7275,7 @@ Returns a list of network interface information (index, name) tuples."); /*[clinic input] _socket.if_nametoindex - oname: object(converter="PyUnicode_FSConverter") + oname: unicode_fs_encoded / Returns the interface index corresponding to the interface name if_name. @@ -7289,7 +7283,7 @@ Returns the interface index corresponding to the interface name if_name. static PyObject * _socket_if_nametoindex_impl(PyObject *module, PyObject *oname) -/*[clinic end generated code: output=289a411614f30244 input=01e0f1205307fb77]*/ +/*[clinic end generated code: output=289a411614f30244 input=6125dc20683560cf]*/ { #ifdef MS_WINDOWS NET_IFINDEX index; @@ -7297,11 +7291,10 @@ _socket_if_nametoindex_impl(PyObject *module, PyObject *oname) unsigned long index; #endif + errno = ENODEV; // in case 'if_nametoindex' does not set errno index = if_nametoindex(PyBytes_AS_STRING(oname)); - Py_DECREF(oname); if (index == 0) { - /* if_nametoindex() doesn't set errno */ - PyErr_SetString(PyExc_OSError, "no interface with this name"); + PyErr_SetFromErrno(PyExc_OSError); return NULL; } @@ -7321,6 +7314,7 @@ static PyObject * _socket_if_indextoname_impl(PyObject *module, NET_IFINDEX index) /*[clinic end generated code: output=e48bc324993052e0 input=c93f753d0cf6d7d1]*/ { + errno = ENXIO; // in case 'if_indextoname' does not set errno char name[IF_NAMESIZE + 1]; if (if_indextoname(index, name) == NULL) { PyErr_SetFromErrno(PyExc_OSError); @@ -9239,6 +9233,10 @@ socket_exec(PyObject *m) /* Initialize gethostbyname lock */ #if defined(USE_GETHOSTBYNAME_LOCK) netdb_lock = PyThread_allocate_lock(); + if (netdb_lock == NULL) { + PyErr_NoMemory(); + goto error; + } #endif #ifdef MS_WINDOWS diff --git a/Modules/socketmodule.h b/Modules/socketmodule.h index 63624d511c35a05..6f8f4b21599cfb7 100644 --- a/Modules/socketmodule.h +++ b/Modules/socketmodule.h @@ -18,6 +18,10 @@ */ #ifdef AF_BTH # include <ws2bth.h> +# ifdef __clang__ +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wpragma-pack" +# endif # include <pshpack1.h> /* @@ -51,7 +55,10 @@ struct SOCKADDR_BTH_REDEF { }; # include <poppack.h> -#endif +# ifdef __clang__ +# pragma clang diagnostic pop +# endif +#endif /* AF_BTH */ /* Windows 'supports' CMSG_LEN, but does not follow the POSIX standard * interface at all, so there is no point including the code that diff --git a/Modules/symtablemodule.c b/Modules/symtablemodule.c index d0d5223e5acea85..d353f406831ecd2 100644 --- a/Modules/symtablemodule.c +++ b/Modules/symtablemodule.c @@ -13,7 +13,7 @@ module _symtable _symtable.symtable source: object - filename: object(converter='PyUnicode_FSDecoder') + filename: unicode_fs_decoded startstr: str / @@ -23,7 +23,7 @@ Return symbol and scope dictionaries used internally by compiler. static PyObject * _symtable_symtable_impl(PyObject *module, PyObject *source, PyObject *filename, const char *startstr) -/*[clinic end generated code: output=59eb0d5fc7285ac4 input=9dd8a50c0c36a4d7]*/ +/*[clinic end generated code: output=59eb0d5fc7285ac4 input=436ffff90d02e4f6]*/ { struct symtable *st; PyObject *t; @@ -47,12 +47,10 @@ _symtable_symtable_impl(PyObject *module, PyObject *source, else { PyErr_SetString(PyExc_ValueError, "symtable() arg 3 must be 'exec' or 'eval' or 'single'"); - Py_DECREF(filename); Py_XDECREF(source_copy); return NULL; } st = _Py_SymtableStringObjectFlags(str, filename, start, &cf); - Py_DECREF(filename); Py_XDECREF(source_copy); if (st == NULL) { return NULL; diff --git a/Modules/termios.c b/Modules/termios.c index efb5fcc17fa5ef9..6205f6026ef4cfc 100644 --- a/Modules/termios.c +++ b/Modules/termios.c @@ -483,12 +483,13 @@ termios.tcsetwinsize Set the tty winsize for file descriptor fd. The winsize to be set is taken from the winsize argument, which -is a two-item tuple (ws_row, ws_col) like the one returned by tcgetwinsize(). +is a two-item tuple (ws_row, ws_col) like the one returned by +tcgetwinsize(). [clinic start generated code]*/ static PyObject * termios_tcsetwinsize_impl(PyObject *module, int fd, PyObject *winsz) -/*[clinic end generated code: output=2ac3c9bb6eda83e1 input=4a06424465b24aee]*/ +/*[clinic end generated code: output=2ac3c9bb6eda83e1 input=efc9beb16d06382a]*/ { if (!PySequence_Check(winsz) || PySequence_Size(winsz) != 2) { PyErr_SetString(PyExc_TypeError, diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 1bfbf3f6a0b991e..5e43adb25f17b60 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -763,6 +763,13 @@ Other codes may be available on your platform. See documentation for\n\ the C library strftime function.\n" #ifdef HAVE_STRFTIME +// gh-154460: OpenBSD's wcsftime() computes %V incorrectly: it returns 53 +// whenever the ISO 8601 week belongs to a different year than tm_year. +// strftime() is not affected. +#ifdef __OpenBSD__ +# undef HAVE_WCSFTIME +#endif + #ifdef HAVE_WCSFTIME #define time_char wchar_t #define format_time wcsftime @@ -820,12 +827,15 @@ time_strftime1(time_char **outbuf, size_t *bufsize, PyErr_NoMemory(); return NULL; } - *outbuf = (time_char *)PyMem_Realloc(*outbuf, - *bufsize*sizeof(time_char)); - if (*outbuf == NULL) { + time_char *tmp = (time_char *)PyMem_Realloc(*outbuf, + *bufsize*sizeof(time_char)); + if (tmp == NULL) { + PyMem_Free(*outbuf); + *outbuf = NULL; PyErr_NoMemory(); return NULL; } + *outbuf = tmp; #if defined _MSC_VER && _MSC_VER >= 1400 && defined(__STDC_SECURE_LIB__) errno = 0; #endif diff --git a/Modules/unicodedata.c b/Modules/unicodedata.c index ef8cf3d0d274592..d9b477a12b4b475 100644 --- a/Modules/unicodedata.c +++ b/Modules/unicodedata.c @@ -230,9 +230,9 @@ unicodedata_UCD_numeric_impl(PyObject *self, int chr, have_old = 1; rc = -1.0; } - else if (old->decimal_changed != 0xFF) { + else if (old->numeric_changed != 0.0) { have_old = 1; - rc = old->decimal_changed; + rc = old->numeric_changed; } } @@ -388,6 +388,17 @@ unicodedata_UCD_east_asian_width_impl(PyObject *self, int chr) return PyUnicode_FromString(_PyUnicode_EastAsianWidthNames[index]); } +// For Hangul decomposition +#define SBase 0xAC00 +#define LBase 0x1100 +#define VBase 0x1161 +#define TBase 0x11A7 +#define LCount 19 +#define VCount 21 +#define TCount 28 +#define NCount (VCount*TCount) +#define SCount (LCount*NCount) + /*[clinic input] unicodedata.UCD.decomposition @@ -418,6 +429,25 @@ unicodedata_UCD_decomposition_impl(PyObject *self, int chr) return Py_GetConstant(Py_CONSTANT_EMPTY_STR); /* unassigned */ } + // Hangul Decomposition. + // See section 3.12.2, "Hangul Syllable Decomposition" + // https://www.unicode.org/versions/latest/core-spec/chapter-3/#G56669 + if (SBase <= code && code < (SBase + SCount)) { + int SIndex = code - SBase; + int L = LBase + SIndex / NCount; + int V = VBase + (SIndex % NCount) / TCount; + int T = TBase + SIndex % TCount; + if (T != TBase) { + PyOS_snprintf(decomp, sizeof(decomp), + "%04X %04X %04X", L, V, T); + } + else { + PyOS_snprintf(decomp, sizeof(decomp), + "%04X %04X", L, V); + } + return PyUnicode_FromString(decomp); + } + if (code < 0 || code >= 0x110000) index = 0; else { @@ -480,15 +510,63 @@ get_decomp_record(PyObject *self, Py_UCS4 code, (*index)++; } -#define SBase 0xAC00 -#define LBase 0x1100 -#define VBase 0x1161 -#define TBase 0x11A7 -#define LCount 19 -#define VCount 21 -#define TCount 28 -#define NCount (VCount*TCount) -#define SCount (LCount*NCount) +/* Small combining runs are usually cheaper with insertion sort. */ +#define CANONICAL_ORDERING_COUNTING_SORT_THRESHOLD 20 + +static void +canonical_ordering_sort_insertion(int kind, void *data, + Py_ssize_t start, Py_ssize_t end) +{ + for (Py_ssize_t i = start + 1; i < end; i++) { + Py_UCS4 code = PyUnicode_READ(kind, data, i); + unsigned char combining = _getrecord_ex(code)->combining; + Py_ssize_t j = i; + + while (j > start) { + Py_UCS4 previous = PyUnicode_READ(kind, data, j - 1); + if (_getrecord_ex(previous)->combining <= combining) { + break; + } + PyUnicode_WRITE(kind, data, j, previous); + j--; + } + if (j != i) { + PyUnicode_WRITE(kind, data, j, code); + } + } +} + +static void +canonical_ordering_sort_counting(int kind, void *data, + Py_ssize_t start, Py_ssize_t end, + Py_UCS4 *sortbuf) +{ + Py_ssize_t counts[256] = {0}; + Py_ssize_t run_length = end - start; + Py_ssize_t total = 0; + + for (Py_ssize_t i = start; i < end; i++) { + Py_UCS4 code = PyUnicode_READ(kind, data, i); + unsigned char combining = _getrecord_ex(code)->combining; + counts[combining]++; + } + + for (size_t i = 0; i < Py_ARRAY_LENGTH(counts); i++) { + Py_ssize_t count = counts[i]; + counts[i] = total; + total += count; + } + + /* Reuse counts[] as the next output slot for each CCC. */ + for (Py_ssize_t i = start; i < end; i++) { + Py_UCS4 code = PyUnicode_READ(kind, data, i); + unsigned char combining = _getrecord_ex(code)->combining; + sortbuf[counts[combining]++] = code; + } + for (Py_ssize_t i = 0; i < run_length; i++) { + PyUnicode_WRITE(kind, data, start + i, sortbuf[i]); + } +} static PyObject* nfd_nfkd(PyObject *self, PyObject *input, int k) @@ -496,13 +574,16 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) PyObject *result; Py_UCS4 *output; Py_ssize_t i, o, osize; - int kind; - const void *data; + int input_kind, result_kind; + const void *input_data; + void *result_data; /* Longest decomposition in Unicode 3.2: U+FDFA */ Py_UCS4 stack[20]; Py_ssize_t space, isize; int index, prefix, count, stackptr; unsigned char prev, cur; + Py_UCS4 *sortbuf = NULL; + Py_ssize_t sortbuflen = 0; stackptr = 0; isize = PyUnicode_GET_LENGTH(input); @@ -522,11 +603,11 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) return NULL; } i = o = 0; - kind = PyUnicode_KIND(input); - data = PyUnicode_DATA(input); + input_kind = PyUnicode_KIND(input); + input_data = PyUnicode_DATA(input); while (i < isize) { - stack[stackptr++] = PyUnicode_READ(kind, data, i++); + stack[stackptr++] = PyUnicode_READ(input_kind, input_data, i++); while(stackptr) { Py_UCS4 code = stack[--stackptr]; /* Hangul Decomposition adds three characters in @@ -543,7 +624,9 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) } output = new_output; } - /* Hangul Decomposition. */ + // Hangul Decomposition. + // See section 3.12.2, "Hangul Syllable Decomposition" + // https://www.unicode.org/versions/latest/core-spec/chapter-3/#G56669 if (SBase <= code && code < (SBase+SCount)) { int SIndex = code - SBase; int L = LBase + SIndex / NCount; @@ -592,34 +675,64 @@ nfd_nfkd(PyObject *self, PyObject *input, int k) if (!result) return NULL; - kind = PyUnicode_KIND(result); - data = PyUnicode_DATA(result); + result_kind = PyUnicode_KIND(result); + result_data = PyUnicode_DATA(result); - /* Sort canonically. */ + /* Sort each consecutive combining-character run canonically. */ i = 0; - prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining; - for (i++; i < PyUnicode_GET_LENGTH(result); i++) { - cur = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining; - if (prev == 0 || cur == 0 || prev <= cur) { - prev = cur; + while (i < o) { + Py_ssize_t run_length, run_start; + int needs_sort = 0; + + Py_UCS4 ch = PyUnicode_READ(result_kind, result_data, i); + prev = _getrecord_ex(ch)->combining; + if (prev == 0) { + i++; continue; } - /* Non-canonical order. Need to switch *i with previous. */ - o = i - 1; - while (1) { - Py_UCS4 tmp = PyUnicode_READ(kind, data, o+1); - PyUnicode_WRITE(kind, data, o+1, - PyUnicode_READ(kind, data, o)); - PyUnicode_WRITE(kind, data, o, tmp); - o--; - if (o < 0) - break; - prev = _getrecord_ex(PyUnicode_READ(kind, data, o))->combining; - if (prev == 0 || prev <= cur) + + run_start = i++; + while (i < o) { + Py_UCS4 ch = PyUnicode_READ(result_kind, result_data, i); + cur = _getrecord_ex(ch)->combining; + if (cur == 0) { break; + } + if (prev > cur) { + needs_sort = 1; + } + prev = cur; + i++; } - prev = _getrecord_ex(PyUnicode_READ(kind, data, i))->combining; + if (!needs_sort) { + continue; + } + + run_length = i - run_start; + if (run_length < CANONICAL_ORDERING_COUNTING_SORT_THRESHOLD) { + canonical_ordering_sort_insertion(result_kind, result_data, + run_start, i); + continue; + } + + if (run_length > sortbuflen) { + Py_UCS4 *new_sortbuf = PyMem_Resize(sortbuf, + Py_UCS4, + run_length); + if (new_sortbuf == NULL) { + PyErr_NoMemory(); + PyMem_Free(sortbuf); + Py_DECREF(result); + return NULL; + } + sortbuf = new_sortbuf; + sortbuflen = run_length; + } + + canonical_ordering_sort_counting(result_kind, result_data, + run_start, i, sortbuf); } + PyMem_Free(sortbuf); return result; } @@ -1010,21 +1123,18 @@ static const char * const hangul_syllables[][3] = { { 0, 0, "H" } }; -/* These ranges need to match makeunicodedata.py:cjk_ranges. */ static int -is_unified_ideograph(Py_UCS4 code) +find_prefix_id(Py_UCS4 code) { - return - (0x3400 <= code && code <= 0x4DBF) || /* CJK Ideograph Extension A */ - (0x4E00 <= code && code <= 0x9FFF) || /* CJK Ideograph */ - (0x20000 <= code && code <= 0x2A6DF) || /* CJK Ideograph Extension B */ - (0x2A700 <= code && code <= 0x2B739) || /* CJK Ideograph Extension C */ - (0x2B740 <= code && code <= 0x2B81D) || /* CJK Ideograph Extension D */ - (0x2B820 <= code && code <= 0x2CEA1) || /* CJK Ideograph Extension E */ - (0x2CEB0 <= code && code <= 0x2EBE0) || /* CJK Ideograph Extension F */ - (0x2EBF0 <= code && code <= 0x2EE5D) || /* CJK Ideograph Extension I */ - (0x30000 <= code && code <= 0x3134A) || /* CJK Ideograph Extension G */ - (0x31350 <= code && code <= 0x323AF); /* CJK Ideograph Extension H */ + for (int i = 0; i < (int)Py_ARRAY_LENGTH(derived_name_ranges); i++) { + if (code < derived_name_ranges[i].first) { + return -1; + } + if (code <= derived_name_ranges[i].last) { + return derived_name_ranges[i].prefixid; + } + } + return -1; } /* macros used to determine if the given code point is in the PUA range that @@ -1302,7 +1412,9 @@ _getucname(PyObject *self, } } - if (SBase <= code && code < SBase+SCount) { + int prefixid = find_prefix_id(code); + if (prefixid == 0) { + assert(SBase <= code && code < SBase+SCount); /* Hangul syllable. */ int SIndex = code - SBase; int L = SIndex / NCount; @@ -1324,11 +1436,13 @@ _getucname(PyObject *self, return 1; } - if (is_unified_ideograph(code)) { - if (buflen < 28) - /* Worst case: CJK UNIFIED IDEOGRAPH-20000 */ + /* Only support CJK unified ideographs. + * Support for Tangut ideographs is a new feature in 3.15. */ + if (prefixid == 1) { + const char *prefix = derived_name_prefixes[prefixid]; + if (snprintf(buffer, buflen, "%s%04X", prefix, code) >= buflen) { return 0; - sprintf(buffer, "CJK UNIFIED IDEOGRAPH-%X", code); + } return 1; } @@ -1362,7 +1476,7 @@ find_syllable(const char *str, int *len, int *pos, int count, int column) len1 = Py_SAFE_DOWNCAST(strlen(s), size_t, int); if (len1 <= *len) continue; - if (strncmp(str, s, len1) == 0) { + if (PyOS_strnicmp(str, s, len1) == 0) { *len = len1; *pos = i; } @@ -1385,6 +1499,35 @@ _check_alias_and_seq(Py_UCS4* code, int with_named_seq) return 1; } +static Py_UCS4 +parse_hex_code(const char *name, int namelen) +{ + if (namelen < 4 || namelen > 6) { + return (Py_UCS4)-1; + } + if (*name == '0') { + return (Py_UCS4)-1; + } + int v = 0; + while (namelen--) { + v *= 16; + Py_UCS1 c = Py_TOUPPER(*name); + if (c >= '0' && c <= '9') { + v += c - '0'; + } + else if (c >= 'A' && c <= 'F') { + v += c - 'A' + 10; + } + else { + return (Py_UCS4)-1; + } + name++; + } + if (v > 0x10ffff) { + return (Py_UCS4)-1; + } + return v; +} static int _getcode(const char* name, int namelen, Py_UCS4* code) @@ -1393,8 +1536,19 @@ _getcode(const char* name, int namelen, Py_UCS4* code) * Named aliases are not resolved, they are returned as a code point in the * PUA */ - /* Check for hangul syllables. */ - if (strncmp(name, "HANGUL SYLLABLE ", 16) == 0) { + int i = 0; + size_t prefixlen; + for (; i < (int)Py_ARRAY_LENGTH(derived_name_prefixes); i++) { + const char *prefix = derived_name_prefixes[i]; + prefixlen = strlen(derived_name_prefixes[i]); + if (PyOS_strnicmp(name, prefix, prefixlen) == 0) { + break; + } + } + + if (i == 0) { + /* Hangul syllables. */ + assert(PyOS_strnicmp(name, "HANGUL SYLLABLE ", 16) == 0); int len, L = -1, V = -1, T = -1; const char *pos = name + 16; find_syllable(pos, &len, &L, LCount, 0); @@ -1411,27 +1565,11 @@ _getcode(const char* name, int namelen, Py_UCS4* code) return 0; } - /* Check for unified ideographs. */ - if (strncmp(name, "CJK UNIFIED IDEOGRAPH-", 22) == 0) { - /* Four or five hexdigits must follow. */ - unsigned int v; - v = 0; - name += 22; - namelen -= 22; - if (namelen != 4 && namelen != 5) + if (i < (int)Py_ARRAY_LENGTH(derived_name_prefixes)) { + Py_UCS4 v = parse_hex_code(name + prefixlen, namelen - prefixlen); + if (find_prefix_id(v) != i) { return 0; - while (namelen--) { - v *= 16; - if (*name >= '0' && *name <= '9') - v += *name - '0'; - else if (*name >= 'A' && *name <= 'F') - v += *name - 'A' + 10; - else - return 0; - name++; } - if (!is_unified_ideograph(v)) - return 0; *code = v; return 1; } @@ -1456,32 +1594,17 @@ capi_getcode(const char* name, int namelen, Py_UCS4* code, return _check_alias_and_seq(code, with_named_seq); } -static void -unicodedata_destroy_capi(PyObject *capsule) -{ - void *capi = PyCapsule_GetPointer(capsule, PyUnicodeData_CAPSULE_NAME); - PyMem_Free(capi); -} - static PyObject * unicodedata_create_capi(void) { - _PyUnicode_Name_CAPI *capi = PyMem_Malloc(sizeof(_PyUnicode_Name_CAPI)); - if (capi == NULL) { - PyErr_NoMemory(); - return NULL; - } - capi->getname = capi_getucname; - capi->getcode = capi_getcode; - - PyObject *capsule = PyCapsule_New(capi, - PyUnicodeData_CAPSULE_NAME, - unicodedata_destroy_capi); - if (capsule == NULL) { - PyMem_Free(capi); - } - return capsule; -}; + // Statically allocated so that any cached pointers stay valid after unicodedata + // is removed from sys.modules and the capsule is gc'd (gh-149449). + static _PyUnicode_Name_CAPI capi = { + .getname = capi_getucname, + .getcode = capi_getcode, + }; + return PyCapsule_New(&capi, PyUnicodeData_CAPSULE_NAME, NULL); +} /* -------------------------------------------------------------------- */ diff --git a/Modules/unicodename_db.h b/Modules/unicodename_db.h index 0697e259b390191..acc85cb0b4ef3c1 100644 --- a/Modules/unicodename_db.h +++ b/Modules/unicodename_db.h @@ -19473,3 +19473,30 @@ static const named_sequence named_sequences[] = { {2, {0x02E5, 0x02E9}}, {2, {0x02E9, 0x02E5}}, }; + +typedef struct { + Py_UCS4 first; + Py_UCS4 last; + int prefixid; +} derived_name_range; + +static const derived_name_range derived_name_ranges[] = { + {0x3400, 0x4DBF, 1}, + {0x4E00, 0x9FFF, 1}, + {0xAC00, 0xD7A3, 0}, + {0x17000, 0x187F7, 2}, + {0x18D00, 0x18D08, 2}, + {0x20000, 0x2A6DF, 1}, + {0x2A700, 0x2B739, 1}, + {0x2B740, 0x2B81D, 1}, + {0x2B820, 0x2CEA1, 1}, + {0x2CEB0, 0x2EBE0, 1}, + {0x2EBF0, 0x2EE5D, 1}, + {0x30000, 0x3134A, 1}, + {0x31350, 0x323AF, 1}, +}; +static const char * const derived_name_prefixes[] = { + "HANGUL SYLLABLE ", + "CJK UNIFIED IDEOGRAPH-", + "TANGUT IDEOGRAPH-", +}; diff --git a/Modules/xxlimited.c b/Modules/xxlimited.c index 26ac35734fb0604..09c8d9487f54266 100644 --- a/Modules/xxlimited.c +++ b/Modules/xxlimited.c @@ -14,7 +14,9 @@ This module roughly corresponds to:: class Xxo: - """A class that explicitly stores attributes in an internal dict""" + """A class that explicitly stores attributes in an internal dict + (to simulate custom attribute handling). + """ def __init__(self): # In the C class, "_x_attr" is not accessible from Python code @@ -85,13 +87,16 @@ typedef struct { // Instance state typedef struct { PyObject_HEAD - PyObject *x_attr; /* Attributes dictionary */ + PyObject *x_attr; /* Attributes dictionary. + * May be NULL, which acts as an + * empty dict. + */ char x_buffer[BUFSIZE]; /* buffer for Py_buffer */ Py_ssize_t x_exports; /* how many buffer are exported */ } XxoObject; #define XxoObject_CAST(op) ((XxoObject *)(op)) -// XXX: no good way to do this yet +// TODO: full support for type-checking was added in 3.14 (Py_tp_token) // #define XxoObject_Check(v) Py_IS_TYPE(v, Xxo_Type) static XxoObject * @@ -112,8 +117,13 @@ newXxoObject(PyObject *module) return self; } -/* Xxo finalization */ +/* Xxo finalization. + * + * Types that store references to other PyObjects generally need to implement + * the GC slots: traverse, clear, dealloc, and (optionally) finalize. + */ +// traverse: Visit all references from an object, including its type static int Xxo_traverse(PyObject *op, visitproc visit, void *arg) { @@ -126,6 +136,7 @@ Xxo_traverse(PyObject *op, visitproc visit, void *arg) return 0; } +// clear: drop references in order to break all reference cycles static int Xxo_clear(PyObject *op) { @@ -134,6 +145,8 @@ Xxo_clear(PyObject *op) return 0; } +// finalize: like clear, but should leave the object in a consistent state. +// Equivalent to `__del__` in Python. static void Xxo_finalize(PyObject *op) { @@ -141,6 +154,7 @@ Xxo_finalize(PyObject *op) Py_CLEAR(self->x_attr); } +// dealloc: drop all remaining references and free memory static void Xxo_dealloc(PyObject *self) { @@ -155,6 +169,7 @@ Xxo_dealloc(PyObject *self) /* Xxo attribute handling */ +// Get an attribute. static PyObject * Xxo_getattro(PyObject *op, PyObject *name) { @@ -168,9 +183,12 @@ Xxo_getattro(PyObject *op, PyObject *name) return NULL; } } + // Fall back to generic implementation (this handles special attributes, + // raising AttributeError, etc.) return PyObject_GenericGetAttr(op, name); } +// Set or delete an attribute. static int Xxo_setattro(PyObject *op, PyObject *name, PyObject *v) { @@ -198,7 +216,9 @@ Xxo_setattro(PyObject *op, PyObject *name, PyObject *v) } } -/* Xxo methods */ +/* Xxo methods: C functions plus a PyMethodDef array that lists them and + * specifies metadata. + */ static PyObject * Xxo_demo(PyObject *op, PyTypeObject *defining_class, @@ -234,7 +254,10 @@ static PyMethodDef Xxo_methods[] = { {NULL, NULL} /* sentinel */ }; -/* Xxo buffer interface */ +/* Xxo buffer interface: C functions later referenced from PyType_Slot array. + * Other interfaces (e.g. for sequence-like or number-like types) are defined + * similarly. + */ static int Xxo_getbuffer(PyObject *op, Py_buffer *view, int flags) @@ -300,6 +323,7 @@ static PyType_Spec Xxo_Type_spec = { /* Str type definition*/ static PyType_Slot Str_Type_slots[] = { + // slots array intentionally kept empty {0, 0}, /* sentinel */ }; @@ -400,12 +424,32 @@ xx_modexec(PyObject *m) } static PyModuleDef_Slot xx_slots[] = { + + /* exec function to initialize the module (called as part of import + * after the object was added to sys.modules) + */ {Py_mod_exec, xx_modexec}, + + /* Signal that this module supports being loaded in multiple interpreters + * with separate GILs (global interpreter locks). + * See "Isolating Extension Modules" on how to prepare a module for this: + * https://docs.python.org/3/howto/isolating-extensions.html + */ {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED}, + + /* Signal that this module does not rely on the GIL for its own needs. + * Without this slot, free-threaded builds of CPython will enable + * the GIL when this module is loaded. + */ {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {0, NULL} }; +// Module finalization: modules that hold references in their module state +// need to implement the fullowing GC hooks. They're similar to the ones for +// types (see "Xxo finalization"). + static int xx_traverse(PyObject *module, visitproc visit, void *arg) { @@ -424,6 +468,13 @@ xx_clear(PyObject *module) return 0; } +static void +xx_free(void *module) +{ + // allow xx_modexec to omit calling xx_clear on error + (void)xx_clear((PyObject *)module); +} + static struct PyModuleDef xxmodule = { PyModuleDef_HEAD_INIT, .m_name = "xxlimited", @@ -433,13 +484,13 @@ static struct PyModuleDef xxmodule = { .m_slots = xx_slots, .m_traverse = xx_traverse, .m_clear = xx_clear, - /* m_free is not necessary here: xx_clear clears all references, - * and the module state is deallocated along with the module. - */ + .m_free = xx_free, }; -/* Export function for the module (*must* be called PyInit_xx) */ +/* Export function for the module. *Must* be called PyInit_xx; usually it is + * the only non-`static` object in a module definition. + */ PyMODINIT_FUNC PyInit_xxlimited(void) diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c index d4b4b91697c08e7..2bccc7740f28238 100644 --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -263,7 +263,9 @@ static compobject * newcompobject(PyTypeObject *type) { compobject *self; - self = PyObject_New(compobject, type); + assert(type != NULL); + assert(type->tp_alloc != NULL); + self = _compobject_CAST(type->tp_alloc(type, 0)); if (self == NULL) return NULL; self->eof = 0; @@ -554,7 +556,7 @@ zlib.compressobj strategy: int(c_default="Z_DEFAULT_STRATEGY") = Z_DEFAULT_STRATEGY Used to tune the compression algorithm. Possible values are Z_DEFAULT_STRATEGY, Z_FILTERED, and Z_HUFFMAN_ONLY. - zdict: Py_buffer = None + zdict: Py_buffer = NULL The predefined compression dictionary - a sequence of bytes containing subsequences that are likely to occur in the input data. @@ -564,7 +566,7 @@ Return a compressor object. static PyObject * zlib_compressobj_impl(PyObject *module, int level, int method, int wbits, int memLevel, int strategy, Py_buffer *zdict) -/*[clinic end generated code: output=8b5bed9c8fc3814d input=2fa3d026f90ab8d5]*/ +/*[clinic end generated code: output=8b5bed9c8fc3814d input=1a6f61d8a8885c0d]*/ { zlibstate *state = get_zlib_state(module); if (zdict->buf != NULL && (size_t)zdict->len > UINT_MAX) { @@ -706,33 +708,41 @@ zlib_decompressobj_impl(PyObject *module, int wbits, PyObject *zdict) } static void -Dealloc(compobject *self) +compobject_dealloc_impl(PyObject *op, int (*dealloc)(z_streamp)) { - PyTypeObject *type = Py_TYPE(self); + PyTypeObject *type = Py_TYPE(op); + PyObject_GC_UnTrack(op); + compobject *self = _compobject_CAST(op); + if (self->is_initialised) { + (void)dealloc(&self->zst); + } PyThread_free_lock(self->lock); Py_XDECREF(self->unused_data); Py_XDECREF(self->unconsumed_tail); Py_XDECREF(self->zdict); - PyObject_Free(self); + type->tp_free(self); Py_DECREF(type); } +static int +compobject_traverse(PyObject *op, visitproc visit, void *arg) +{ + compobject *self = _compobject_CAST(op); + Py_VISIT(Py_TYPE(op)); + Py_VISIT(self->zdict); + return 0; +} + static void Comp_dealloc(PyObject *op) { - compobject *self = _compobject_CAST(op); - if (self->is_initialised) - (void)deflateEnd(&self->zst); - Dealloc(self); + compobject_dealloc_impl(op, &deflateEnd); } static void Decomp_dealloc(PyObject *op) { - compobject *self = _compobject_CAST(op); - if (self->is_initialised) - (void)inflateEnd(&self->zst); - Dealloc(self); + compobject_dealloc_impl(op, &inflateEnd); } /*[clinic input] @@ -865,15 +875,15 @@ zlib.Decompress.decompress Return a bytes object containing the decompressed version of the data. -After calling this function, some of the input data may still be stored in -internal buffers for later processing. +After calling this function, some of the input data may still be +stored in internal buffers for later processing. Call the flush() method to clear these buffers. [clinic start generated code]*/ static PyObject * zlib_Decompress_decompress_impl(compobject *self, PyTypeObject *cls, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=b024a93c2c922d57 input=bfb37b3864cfb606]*/ +/*[clinic end generated code: output=b024a93c2c922d57 input=b8c9e2d124fe4720]*/ { int err = Z_OK; Py_ssize_t ibuflen; @@ -1357,6 +1367,8 @@ typedef struct { char needs_input; } ZlibDecompressor; +#define ZlibDecompressor_CAST(op) ((ZlibDecompressor *)(op)) + /*[clinic input] class zlib.ZlibDecompressor "ZlibDecompressor *" "&ZlibDecompressorType" [clinic start generated code]*/ @@ -1365,8 +1377,9 @@ class zlib.ZlibDecompressor "ZlibDecompressor *" "&ZlibDecompressorType" static void ZlibDecompressor_dealloc(PyObject *op) { - ZlibDecompressor *self = (ZlibDecompressor*)op; - PyObject *type = (PyObject *)Py_TYPE(self); + PyTypeObject *type = Py_TYPE(op); + PyObject_GC_UnTrack(op); + ZlibDecompressor *self = ZlibDecompressor_CAST(op); PyThread_free_lock(self->lock); if (self->is_initialised) { inflateEnd(&self->zst); @@ -1374,10 +1387,19 @@ ZlibDecompressor_dealloc(PyObject *op) PyMem_Free(self->input_buffer); Py_CLEAR(self->unused_data); Py_CLEAR(self->zdict); - PyObject_Free(self); + type->tp_free(self); Py_DECREF(type); } +static int +ZlibDecompressor_traverse(PyObject *op, visitproc visit, void *arg) +{ + ZlibDecompressor *self = ZlibDecompressor_CAST(op); + Py_VISIT(Py_TYPE(op)); + Py_VISIT(self->zdict); + return 0; +} + static int set_inflate_zdict_ZlibDecompressor(zlibstate *state, ZlibDecompressor *self) { @@ -1653,6 +1675,7 @@ decompress(ZlibDecompressor *self, uint8_t *data, return result; error: + self->zst.next_in = NULL; Py_XDECREF(result); return NULL; } @@ -1665,24 +1688,25 @@ zlib.ZlibDecompressor.decompress Decompress *data*, returning uncompressed data as bytes. -If *max_length* is nonnegative, returns at most *max_length* bytes of -decompressed data. If this limit is reached and further output can be -produced, *self.needs_input* will be set to ``False``. In this case, the next -call to *decompress()* may provide *data* as b'' to obtain more of the output. +If *max_length* is nonnegative, returns at most *max_length* bytes +of decompressed data. If this limit is reached and further output +can be produced, *self.needs_input* will be set to ``False``. In +this case, the next call to *decompress()* may provide *data* as b'' +to obtain more of the output. -If all of the input data was decompressed and returned (either because this -was less than *max_length* bytes, or because *max_length* was negative), -*self.needs_input* will be set to True. +If all of the input data was decompressed and returned (either +because this was less than *max_length* bytes, or because +*max_length* was negative), *self.needs_input* will be set to True. -Attempting to decompress data after the end of stream is reached raises an -EOFError. Any data found after the end of the stream is ignored and saved in -the unused_data attribute. +Attempting to decompress data after the end of stream is reached +raises an EOFError. Any data found after the end of the stream is +ignored and saved in the unused_data attribute. [clinic start generated code]*/ static PyObject * zlib_ZlibDecompressor_decompress_impl(ZlibDecompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=990d32787b775f85 input=0b29d99715250b96]*/ +/*[clinic end generated code: output=990d32787b775f85 input=6fb56d60b48cd843]*/ { PyObject *result = NULL; @@ -1729,7 +1753,9 @@ ZlibDecompressor__new__(PyTypeObject *cls, args, kwargs, format, keywords, &wbits, &zdict)) { return NULL; } - ZlibDecompressor *self = PyObject_New(ZlibDecompressor, cls); + + assert(cls != NULL && cls->tp_alloc != NULL); + ZlibDecompressor *self = ZlibDecompressor_CAST(cls->tp_alloc(cls, 0)); if (self == NULL) { return NULL; } @@ -1937,6 +1963,7 @@ static PyMethodDef zlib_methods[] = static PyType_Slot Comptype_slots[] = { {Py_tp_dealloc, Comp_dealloc}, + {Py_tp_traverse, compobject_traverse}, {Py_tp_methods, comp_methods}, {0, 0}, }; @@ -1944,12 +1971,17 @@ static PyType_Slot Comptype_slots[] = { static PyType_Spec Comptype_spec = { .name = "zlib.Compress", .basicsize = sizeof(compobject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .flags = ( + Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_DISALLOW_INSTANTIATION + | Py_TPFLAGS_HAVE_GC + ), .slots= Comptype_slots, }; static PyType_Slot Decomptype_slots[] = { {Py_tp_dealloc, Decomp_dealloc}, + {Py_tp_traverse, compobject_traverse}, {Py_tp_methods, Decomp_methods}, {Py_tp_members, Decomp_members}, {0, 0}, @@ -1958,12 +1990,17 @@ static PyType_Slot Decomptype_slots[] = { static PyType_Spec Decomptype_spec = { .name = "zlib.Decompress", .basicsize = sizeof(compobject), - .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, + .flags = ( + Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_DISALLOW_INSTANTIATION + | Py_TPFLAGS_HAVE_GC + ), .slots = Decomptype_slots, }; static PyType_Slot ZlibDecompressor_type_slots[] = { {Py_tp_dealloc, ZlibDecompressor_dealloc}, + {Py_tp_traverse, ZlibDecompressor_traverse}, {Py_tp_members, ZlibDecompressor_members}, {Py_tp_new, ZlibDecompressor__new__}, {Py_tp_doc, (char *)ZlibDecompressor__new____doc__}, @@ -1978,7 +2015,11 @@ static PyType_Spec ZlibDecompressor_type_spec = { // ZlibDecompressor_type_spec does not have Py_TPFLAGS_BASETYPE flag // which prevents to create a subclass. // So calling PyType_GetModuleState() in this file is always safe. - .flags = (Py_TPFLAGS_DEFAULT | Py_TPFLAGS_IMMUTABLETYPE), + .flags = ( + Py_TPFLAGS_DEFAULT + | Py_TPFLAGS_IMMUTABLETYPE + | Py_TPFLAGS_HAVE_GC + ), .slots = ZlibDecompressor_type_slots, }; PyDoc_STRVAR(zlib_module_documentation, diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index b5d5ca9178ebdba..1209125c70bd8ed 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -82,6 +82,25 @@ bytearray_releasebuffer(PyObject *self, Py_buffer *view) Py_END_CRITICAL_SECTION(); } +typedef PyObject* (*_ba_bytes_op)(const char *buf, Py_ssize_t len, + PyObject *sub, Py_ssize_t start, + Py_ssize_t end); + +static PyObject * +_bytearray_with_buffer(PyByteArrayObject *self, _ba_bytes_op op, PyObject *sub, + Py_ssize_t start, Py_ssize_t end) +{ + PyObject *res; + + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self); + + /* Increase exports to prevent bytearray storage from changing during op. */ + self->ob_exports++; + res = op(PyByteArray_AS_STRING(self), Py_SIZE(self), sub, start, end); + self->ob_exports--; + return res; +} + static int _canresize(PyByteArrayObject *self) { @@ -709,7 +728,9 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(op); PyByteArrayObject *self = _PyByteArray_CAST(op); Py_ssize_t start, stop, step, slicelen; - char *buf = PyByteArray_AS_STRING(self); + // Do not store a reference to the internal buffer since + // index.__index__() or _getbytevalue() may alter 'self'. + // See https://github.com/python/cpython/issues/91153. if (_PyIndex_Check(index)) { Py_ssize_t i = PyNumber_AsSsize_t(index, PyExc_IndexError); @@ -744,7 +765,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value } else { assert(0 <= ival && ival < 256); - buf[i] = (char)ival; + PyByteArray_AS_STRING(self)[i] = (char)ival; return 0; } } @@ -805,6 +826,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value /* Delete slice */ size_t cur; Py_ssize_t i; + char *buf = PyByteArray_AS_STRING(self); if (!_canresize(self)) return -1; @@ -845,6 +867,7 @@ bytearray_ass_subscript_lock_held(PyObject *op, PyObject *index, PyObject *value /* Assign slice */ Py_ssize_t i; size_t cur; + char *buf = PyByteArray_AS_STRING(self); if (needed != slicelen) { PyErr_Format(PyExc_ValueError, @@ -1296,8 +1319,7 @@ bytearray_find_impl(PyByteArrayObject *self, PyObject *sub, Py_ssize_t start, Py_ssize_t end) /*[clinic end generated code: output=413e1cab2ae87da0 input=1de9f4558df68336]*/ { - return _Py_bytes_find(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), - sub, start, end); + return _bytearray_with_buffer(self, _Py_bytes_find, sub, start, end); } /*[clinic input] @@ -1312,8 +1334,7 @@ bytearray_count_impl(PyByteArrayObject *self, PyObject *sub, Py_ssize_t start, Py_ssize_t end) /*[clinic end generated code: output=a21ee2692e4f1233 input=2608c30644614724]*/ { - return _Py_bytes_count(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), - sub, start, end); + return _bytearray_with_buffer(self, _Py_bytes_count, sub, start, end); } /*[clinic input] @@ -1360,8 +1381,7 @@ bytearray_index_impl(PyByteArrayObject *self, PyObject *sub, Py_ssize_t start, Py_ssize_t end) /*[clinic end generated code: output=067a1e78efc672a7 input=0086ba0ab9bf44a5]*/ { - return _Py_bytes_index(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), - sub, start, end); + return _bytearray_with_buffer(self, _Py_bytes_index, sub, start, end); } /*[clinic input] @@ -1378,8 +1398,7 @@ bytearray_rfind_impl(PyByteArrayObject *self, PyObject *sub, Py_ssize_t start, Py_ssize_t end) /*[clinic end generated code: output=51bf886f932b283c input=ac73593305d5c1d1]*/ { - return _Py_bytes_rfind(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), - sub, start, end); + return _bytearray_with_buffer(self, _Py_bytes_rfind, sub, start, end); } /*[clinic input] @@ -1396,18 +1415,22 @@ bytearray_rindex_impl(PyByteArrayObject *self, PyObject *sub, Py_ssize_t start, Py_ssize_t end) /*[clinic end generated code: output=38e1cf66bafb08b9 input=0cf331bf5ebe0e91]*/ { - return _Py_bytes_rindex(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), - sub, start, end); + return _bytearray_with_buffer(self, _Py_bytes_rindex, sub, start, end); } static int bytearray_contains(PyObject *self, PyObject *arg) { - int ret; + int ret = -1; Py_BEGIN_CRITICAL_SECTION(self); - ret = _Py_bytes_contains(PyByteArray_AS_STRING(self), + PyByteArrayObject *ba = _PyByteArray_CAST(self); + + /* Increase exports to prevent bytearray storage from changing during _Py_bytes_contains(). */ + ba->ob_exports++; + ret = _Py_bytes_contains(PyByteArray_AS_STRING(ba), PyByteArray_GET_SIZE(self), arg); + ba->ob_exports--; Py_END_CRITICAL_SECTION(); return ret; } @@ -1433,8 +1456,7 @@ bytearray_startswith_impl(PyByteArrayObject *self, PyObject *subobj, Py_ssize_t start, Py_ssize_t end) /*[clinic end generated code: output=a3d9b6d44d3662a6 input=ea8d036d09df34b2]*/ { - return _Py_bytes_startswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), - subobj, start, end); + return _bytearray_with_buffer(self, _Py_bytes_startswith, subobj, start, end); } /*[clinic input] @@ -1458,8 +1480,7 @@ bytearray_endswith_impl(PyByteArrayObject *self, PyObject *subobj, Py_ssize_t start, Py_ssize_t end) /*[clinic end generated code: output=e75ea8c227954caa input=c61b90bb23a689ce]*/ { - return _Py_bytes_endswith(PyByteArray_AS_STRING(self), PyByteArray_GET_SIZE(self), - subobj, start, end); + return _bytearray_with_buffer(self, _Py_bytes_endswith, subobj, start, end); } /*[clinic input] @@ -1531,19 +1552,20 @@ bytearray_removesuffix_impl(PyByteArrayObject *self, Py_buffer *suffix) /*[clinic input] +@critical_section bytearray.resize size: Py_ssize_t - New size to resize to.. + New size to resize to. / Resize the internal buffer of bytearray to len. [clinic start generated code]*/ static PyObject * bytearray_resize_impl(PyByteArrayObject *self, Py_ssize_t size) -/*[clinic end generated code: output=f73524922990b2d9 input=75fd4d17c4aa47d3]*/ +/*[clinic end generated code: output=f73524922990b2d9 input=116046316a2b5cfc]*/ { Py_ssize_t start_size = PyByteArray_GET_SIZE(self); - int result = PyByteArray_Resize((PyObject *)self, size); + int result = bytearray_resize_lock_held((PyObject *)self, size); if (result < 0) { return NULL; } @@ -1566,14 +1588,15 @@ bytearray.translate Return a copy with each character mapped by the given translation table. -All characters occurring in the optional argument delete are removed. -The remaining characters are mapped through the given translation table. +All characters occurring in the optional argument delete are +removed. The remaining characters are mapped through the given +translation table. [clinic start generated code]*/ static PyObject * bytearray_translate_impl(PyByteArrayObject *self, PyObject *table, PyObject *deletechars) -/*[clinic end generated code: output=b6a8f01c2a74e446 input=cd6fa93ca04e05bc]*/ +/*[clinic end generated code: output=b6a8f01c2a74e446 input=e5b770ffaf0e40eb]*/ { char *input, *output; const char *table_chars; @@ -1671,15 +1694,15 @@ bytearray.maketrans Return a translation table usable for the bytes or bytearray translate method. -The returned table will be one where each byte in frm is mapped to the byte at -the same position in to. +The returned table will be one where each byte in frm is mapped to +the byte at the same position in to. The bytes objects frm and to must be of the same length. [clinic start generated code]*/ static PyObject * bytearray_maketrans_impl(Py_buffer *frm, Py_buffer *to) -/*[clinic end generated code: output=1df267d99f56b15e input=b10de38c85950a63]*/ +/*[clinic end generated code: output=1df267d99f56b15e input=b1e7b0acbbaeb48a]*/ { return _Py_bytes_maketrans(frm, to); } @@ -1718,8 +1741,8 @@ bytearray.split sep: object = None The delimiter according which to split the bytearray. - None (the default value) means split on ASCII whitespace characters - (space, tab, return, newline, formfeed, vertical tab). + None (the default value) means split on ASCII whitespace + characters (space, tab, return, newline, formfeed, vertical tab). maxsplit: Py_ssize_t = -1 Maximum number of splits to do. -1 (the default value) means no limit. @@ -1730,28 +1753,34 @@ Return a list of the sections in the bytearray, using sep as the delimiter. static PyObject * bytearray_split_impl(PyByteArrayObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=833e2cf385d9a04d input=1c367486b9938909]*/ +/*[clinic end generated code: output=833e2cf385d9a04d input=8776ed42f71b707f]*/ { - Py_ssize_t len = PyByteArray_GET_SIZE(self), n; - const char *s = PyByteArray_AS_STRING(self), *sub; - PyObject *list; - Py_buffer vsub; + PyObject *list = NULL; + + /* Increase exports to prevent bytearray storage from changing during _Py_bytes_contains(). */ + self->ob_exports++; + const char *sbuf = PyByteArray_AS_STRING(self); + Py_ssize_t slen = PyByteArray_GET_SIZE((PyObject *)self); if (maxsplit < 0) maxsplit = PY_SSIZE_T_MAX; - if (sep == Py_None) - return stringlib_split_whitespace((PyObject*) self, s, len, maxsplit); + if (sep == Py_None) { + list = stringlib_split_whitespace((PyObject*)self, sbuf, slen, maxsplit); + goto done; + } - if (PyObject_GetBuffer(sep, &vsub, PyBUF_SIMPLE) != 0) - return NULL; - sub = vsub.buf; - n = vsub.len; + Py_buffer vsub; + if (PyObject_GetBuffer(sep, &vsub, PyBUF_SIMPLE) != 0) { + goto done; + } - list = stringlib_split( - (PyObject*) self, s, len, sub, n, maxsplit - ); + list = stringlib_split((PyObject*)self, sbuf, slen, + (const char *)vsub.buf, vsub.len, maxsplit); PyBuffer_Release(&vsub); + +done: + self->ob_exports--; return list; } @@ -1764,17 +1793,18 @@ bytearray.partition Partition the bytearray into three parts using the given separator. -This will search for the separator sep in the bytearray. If the separator is -found, returns a 3-tuple containing the part before the separator, the -separator itself, and the part after it as new bytearray objects. +This will search for the separator sep in the bytearray. If the +separator is found, returns a 3-tuple containing the part before the +separator, the separator itself, and the part after it as new +bytearray objects. -If the separator is not found, returns a 3-tuple containing the copy of the -original bytearray object and two empty bytearray objects. +If the separator is not found, returns a 3-tuple containing the copy +of the original bytearray object and two empty bytearray objects. [clinic start generated code]*/ static PyObject * bytearray_partition_impl(PyByteArrayObject *self, PyObject *sep) -/*[clinic end generated code: output=b5fa1e03f10cfccb input=632855f986733f34]*/ +/*[clinic end generated code: output=b5fa1e03f10cfccb input=d76673ed03acf5dd]*/ { PyObject *bytesep, *result; @@ -1802,18 +1832,19 @@ bytearray.rpartition Partition the bytearray into three parts using the given separator. -This will search for the separator sep in the bytearray, starting at the end. -If the separator is found, returns a 3-tuple containing the part before the -separator, the separator itself, and the part after it as new bytearray -objects. +This will search for the separator sep in the bytearray, starting at +the end. If the separator is found, returns a 3-tuple containing +the part before the separator, the separator itself, and the part +after it as new bytearray objects. -If the separator is not found, returns a 3-tuple containing two empty bytearray -objects and the copy of the original bytearray object. +If the separator is not found, returns a 3-tuple containing two +empty bytearray objects and the copy of the original bytearray +object. [clinic start generated code]*/ static PyObject * bytearray_rpartition_impl(PyByteArrayObject *self, PyObject *sep) -/*[clinic end generated code: output=0186ce7b1ef61289 input=4318e3d125497450]*/ +/*[clinic end generated code: output=0186ce7b1ef61289 input=b9216a2074174a36]*/ { PyObject *bytesep, *result; @@ -1838,34 +1869,41 @@ bytearray.rsplit = bytearray.split Return a list of the sections in the bytearray, using sep as the delimiter. -Splitting is done starting at the end of the bytearray and working to the front. +Splitting is done starting at the end of the bytearray and working +to the front. [clinic start generated code]*/ static PyObject * bytearray_rsplit_impl(PyByteArrayObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=a55e0b5a03cb6190 input=3cd513c2b94a53c1]*/ +/*[clinic end generated code: output=a55e0b5a03cb6190 input=c12efb1a77e16c90]*/ { - Py_ssize_t len = PyByteArray_GET_SIZE(self), n; - const char *s = PyByteArray_AS_STRING(self), *sub; - PyObject *list; - Py_buffer vsub; + PyObject *list = NULL; + + /* Increase exports to prevent bytearray storage from changing during _Py_bytes_contains(). */ + self->ob_exports++; + const char *sbuf = PyByteArray_AS_STRING(self); + Py_ssize_t slen = PyByteArray_GET_SIZE((PyObject *)self); if (maxsplit < 0) maxsplit = PY_SSIZE_T_MAX; - if (sep == Py_None) - return stringlib_rsplit_whitespace((PyObject*) self, s, len, maxsplit); + if (sep == Py_None) { + list = stringlib_rsplit_whitespace((PyObject*)self, sbuf, slen, maxsplit); + goto done; + } - if (PyObject_GetBuffer(sep, &vsub, PyBUF_SIMPLE) != 0) - return NULL; - sub = vsub.buf; - n = vsub.len; + Py_buffer vsub; + if (PyObject_GetBuffer(sep, &vsub, PyBUF_SIMPLE) != 0) { + goto done; + } - list = stringlib_rsplit( - (PyObject*) self, s, len, sub, n, maxsplit - ); + list = stringlib_rsplit((PyObject*)self, sbuf, slen, + (const char *)vsub.buf, vsub.len, maxsplit); PyBuffer_Release(&vsub); + +done: + self->ob_exports--; return list; } @@ -2149,7 +2187,6 @@ bytearray_extend_impl(PyByteArrayObject *self, PyObject *iterable_of_ints) Py_DECREF(bytearray_obj); return NULL; } - buf[len++] = value; Py_DECREF(item); if (len >= buf_size) { @@ -2159,7 +2196,7 @@ bytearray_extend_impl(PyByteArrayObject *self, PyObject *iterable_of_ints) Py_DECREF(bytearray_obj); return PyErr_NoMemory(); } - addition = len >> 1; + addition = len ? len >> 1 : 1; if (addition > PY_SSIZE_T_MAX - len - 1) buf_size = PY_SSIZE_T_MAX; else @@ -2173,6 +2210,7 @@ bytearray_extend_impl(PyByteArrayObject *self, PyObject *iterable_of_ints) have invalidated it. */ buf = PyByteArray_AS_STRING(bytearray_obj); } + buf[len++] = value; } Py_DECREF(it); @@ -2326,12 +2364,13 @@ bytearray.strip Strip leading and trailing bytes contained in the argument. -If the argument is omitted or None, strip leading and trailing ASCII whitespace. +If the argument is omitted or None, strip leading and trailing ASCII +whitespace. [clinic start generated code]*/ static PyObject * bytearray_strip_impl(PyByteArrayObject *self, PyObject *bytes) -/*[clinic end generated code: output=760412661a34ad5a input=1f9026e5ad35388a]*/ +/*[clinic end generated code: output=760412661a34ad5a input=f4ec5fa609df7d14]*/ { return bytearray_strip_impl_helper(self, bytes, BOTHSTRIP); } @@ -2431,11 +2470,11 @@ bytearray.decode encoding: str(c_default="NULL") = 'utf-8' The encoding with which to decode the bytearray. errors: str(c_default="NULL") = 'strict' - The error handling scheme to use for the handling of decoding errors. - The default is 'strict' meaning that decoding errors raise a - UnicodeDecodeError. Other possible values are 'ignore' and 'replace' - as well as any other name registered with codecs.register_error that - can handle UnicodeDecodeErrors. + The error handling scheme to use for the handling of decoding + errors. The default is 'strict' meaning that decoding errors + raise a UnicodeDecodeError. Other possible values are 'ignore' + and 'replace' as well as any other name registered with + codecs.register_error that can handle UnicodeDecodeErrors. Decode the bytearray using the codec registered for encoding. [clinic start generated code]*/ @@ -2443,7 +2482,7 @@ Decode the bytearray using the codec registered for encoding. static PyObject * bytearray_decode_impl(PyByteArrayObject *self, const char *encoding, const char *errors) -/*[clinic end generated code: output=f57d43f4a00b42c5 input=86c303ee376b8453]*/ +/*[clinic end generated code: output=f57d43f4a00b42c5 input=e51ce9b82b51e2ca]*/ { if (encoding == NULL) encoding = PyUnicode_GetDefaultEncoding(); @@ -2471,14 +2510,15 @@ bytearray.join Concatenate any number of bytes/bytearray objects. -The bytearray whose method is called is inserted in between each pair. +The bytearray whose method is called is inserted in between each +pair. The result is returned as a new bytearray object. [clinic start generated code]*/ static PyObject * bytearray_join_impl(PyByteArrayObject *self, PyObject *iterable_of_bytes) -/*[clinic end generated code: output=0ced382b5846a7ee input=49627e07ca31ca26]*/ +/*[clinic end generated code: output=0ced382b5846a7ee input=0a31db349efcd7fa]*/ { PyObject *ret; self->ob_exports++; // this protects `self` from being cleared/resized if `iterable_of_bytes` is a custom iterator @@ -2515,13 +2555,13 @@ bytearray.splitlines Return a list of the lines in the bytearray, breaking at line boundaries. -Line breaks are not included in the resulting list unless keepends is given and -true. +Line breaks are not included in the resulting list unless keepends +is given and true. [clinic start generated code]*/ static PyObject * bytearray_splitlines_impl(PyByteArrayObject *self, int keepends) -/*[clinic end generated code: output=4223c94b895f6ad9 input=874cd662866a66a1]*/ +/*[clinic end generated code: output=4223c94b895f6ad9 input=73512aabe215e0ec]*/ { return stringlib_splitlines( (PyObject*) self, PyByteArray_AS_STRING(self), @@ -2539,12 +2579,13 @@ bytearray.fromhex Create a bytearray object from a string of hexadecimal numbers. Spaces between two numbers are accepted. -Example: bytearray.fromhex('B9 01EF') -> bytearray(b'\\xb9\\x01\\xef') +Example: + bytearray.fromhex('B9 01EF') -> bytearray(b'\\xb9\\x01\\xef') [clinic start generated code]*/ static PyObject * bytearray_fromhex_impl(PyTypeObject *type, PyObject *string) -/*[clinic end generated code: output=8f0f0b6d30fb3ba0 input=7e314e5b2d7ab484]*/ +/*[clinic end generated code: output=8f0f0b6d30fb3ba0 input=2243a8b0b9e66cd5]*/ { PyObject *result = _PyBytes_FromHex(string, type == &PyByteArray_Type); if (type != &PyByteArray_Type && result != NULL) { @@ -2560,8 +2601,8 @@ bytearray.hex sep: object = NULL An optional single character or byte to separate hex bytes. bytes_per_sep: int = 1 - How many bytes between separators. Positive values count from the - right, negative values count from the left. + How many bytes between separators. Positive values count from + the right, negative values count from the left. Create a string of hexadecimal numbers from a bytearray object. @@ -2579,11 +2620,17 @@ Create a string of hexadecimal numbers from a bytearray object. static PyObject * bytearray_hex_impl(PyByteArrayObject *self, PyObject *sep, int bytes_per_sep) -/*[clinic end generated code: output=29c4e5ef72c565a0 input=7784107de7048873]*/ +/*[clinic end generated code: output=29c4e5ef72c565a0 input=88d6628560fdd413]*/ { char* argbuf = PyByteArray_AS_STRING(self); Py_ssize_t arglen = PyByteArray_GET_SIZE(self); - return _Py_strhex_with_sep(argbuf, arglen, sep, bytes_per_sep); + // Prevent 'self' from being freed if computing len(sep) mutates 'self' + // in _Py_strhex_with_sep(). + // See: https://github.com/python/cpython/issues/143195. + self->ob_exports++; + PyObject *res = _Py_strhex_with_sep(argbuf, arglen, sep, bytes_per_sep); + self->ob_exports--; + return res; } static PyObject * @@ -2751,7 +2798,15 @@ bytearray_mod_lock_held(PyObject *v, PyObject *w) _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(v); if (!PyByteArray_Check(v)) Py_RETURN_NOTIMPLEMENTED; - return _PyBytes_FormatEx(PyByteArray_AS_STRING(v), PyByteArray_GET_SIZE(v), w, 1); + + PyByteArrayObject *self = _PyByteArray_CAST(v); + /* Increase exports to prevent bytearray storage from changing during op. */ + self->ob_exports++; + PyObject *res = _PyBytes_FormatEx( + PyByteArray_AS_STRING(v), PyByteArray_GET_SIZE(v), w, 1 + ); + self->ob_exports--; + return res; } static PyObject * diff --git a/Objects/bytes_methods.c b/Objects/bytes_methods.c index c239ae18a593e3c..c9849c8ec4dad09 100644 --- a/Objects/bytes_methods.c +++ b/Objects/bytes_methods.c @@ -277,8 +277,8 @@ _Py_bytes_upper(char *result, const char *cptr, Py_ssize_t len) PyDoc_STRVAR_shared(_Py_title__doc__, "B.title() -> copy of B\n\ \n\ -Return a titlecased version of B, i.e. ASCII words start with uppercase\n\ -characters, all remaining cased characters have lowercase."); +Return a titlecased version of B, i.e. ASCII words start with\n\ +uppercase characters, all remaining cased characters have lowercase."); void _Py_bytes_title(char *result, const char *s, Py_ssize_t len) diff --git a/Objects/bytesobject.c b/Objects/bytesobject.c index fc407ec6bf99d67..03245788bb10403 100644 --- a/Objects/bytesobject.c +++ b/Objects/bytesobject.c @@ -10,6 +10,7 @@ #include "pycore_global_objects.h"// _Py_GET_GLOBAL_OBJECT() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_long.h" // _PyLong_DigitValue +#include "pycore_list.h" // _PyList_GetItemRef #include "pycore_object.h" // _PyObject_GC_TRACK #include "pycore_pymem.h" // PYMEM_CLEANBYTE #include "pycore_strhex.h" // _Py_strhex_with_sep() @@ -975,8 +976,10 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, /* 2: size preallocated for %s */ if (alloc > 2) { res = _PyBytesWriter_Prepare(&writer, res, alloc - 2); - if (res == NULL) + if (res == NULL) { + Py_XDECREF(temp); goto error; + } } #ifndef NDEBUG char *before = res; @@ -1075,10 +1078,11 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len, } /* Unescape a backslash-escaped string. */ -PyObject *_PyBytes_DecodeEscape(const char *s, +PyObject *_PyBytes_DecodeEscape2(const char *s, Py_ssize_t len, const char *errors, - const char **first_invalid_escape) + int *first_invalid_escape_char, + const char **first_invalid_escape_ptr) { int c; char *p; @@ -1092,7 +1096,8 @@ PyObject *_PyBytes_DecodeEscape(const char *s, return NULL; writer.overallocate = 1; - *first_invalid_escape = NULL; + *first_invalid_escape_char = -1; + *first_invalid_escape_ptr = NULL; end = s + len; while (s < end) { @@ -1130,9 +1135,10 @@ PyObject *_PyBytes_DecodeEscape(const char *s, c = (c<<3) + *s++ - '0'; } if (c > 0377) { - if (*first_invalid_escape == NULL) { - *first_invalid_escape = s-3; /* Back up 3 chars, since we've - already incremented s. */ + if (*first_invalid_escape_char == -1) { + *first_invalid_escape_char = c; + /* Back up 3 chars, since we've already incremented s. */ + *first_invalid_escape_ptr = s - 3; } } *p++ = c; @@ -1173,9 +1179,10 @@ PyObject *_PyBytes_DecodeEscape(const char *s, break; default: - if (*first_invalid_escape == NULL) { - *first_invalid_escape = s-1; /* Back up one char, since we've - already incremented s. */ + if (*first_invalid_escape_char == -1) { + *first_invalid_escape_char = (unsigned char)s[-1]; + /* Back up one char, since we've already incremented s. */ + *first_invalid_escape_ptr = s - 1; } *p++ = '\\'; s--; @@ -1195,18 +1202,19 @@ PyObject *PyBytes_DecodeEscape(const char *s, Py_ssize_t Py_UNUSED(unicode), const char *Py_UNUSED(recode_encoding)) { - const char* first_invalid_escape; - PyObject *result = _PyBytes_DecodeEscape(s, len, errors, - &first_invalid_escape); + int first_invalid_escape_char; + const char *first_invalid_escape_ptr; + PyObject *result = _PyBytes_DecodeEscape2(s, len, errors, + &first_invalid_escape_char, + &first_invalid_escape_ptr); if (result == NULL) return NULL; - if (first_invalid_escape != NULL) { - unsigned char c = *first_invalid_escape; - if ('4' <= c && c <= '7') { + if (first_invalid_escape_char != -1) { + if (first_invalid_escape_char > 0xff) { if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "b\"\\%.3s\" is an invalid octal escape sequence. " + "b\"\\%o\" is an invalid octal escape sequence. " "Such sequences will not work in the future. ", - first_invalid_escape) < 0) + first_invalid_escape_char) < 0) { Py_DECREF(result); return NULL; @@ -1216,7 +1224,7 @@ PyObject *PyBytes_DecodeEscape(const char *s, if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, "b\"\\%c\" is an invalid escape sequence. " "Such sequences will not work in the future. ", - c) < 0) + first_invalid_escape_char) < 0) { Py_DECREF(result); return NULL; @@ -1748,8 +1756,8 @@ bytes.split sep: object = None The delimiter according which to split the bytes. - None (the default value) means split on ASCII whitespace characters - (space, tab, return, newline, formfeed, vertical tab). + None (the default value) means split on ASCII whitespace + characters (space, tab, return, newline, formfeed, vertical tab). maxsplit: Py_ssize_t = -1 Maximum number of splits to do. -1 (the default value) means no limit. @@ -1759,7 +1767,7 @@ Return a list of the sections in the bytes, using sep as the delimiter. static PyObject * bytes_split_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=52126b5844c1d8ef input=8b809b39074abbfa]*/ +/*[clinic end generated code: output=52126b5844c1d8ef input=9c0faeffc870f672]*/ { Py_ssize_t len = PyBytes_GET_SIZE(self), n; const char *s = PyBytes_AS_STRING(self), *sub; @@ -1788,17 +1796,17 @@ bytes.partition Partition the bytes into three parts using the given separator. -This will search for the separator sep in the bytes. If the separator is found, -returns a 3-tuple containing the part before the separator, the separator -itself, and the part after it. +This will search for the separator sep in the bytes. If the +separator is found, returns a 3-tuple containing the part before the +separator, the separator itself, and the part after it. -If the separator is not found, returns a 3-tuple containing the original bytes -object and two empty bytes objects. +If the separator is not found, returns a 3-tuple containing the +original bytes object and two empty bytes objects. [clinic start generated code]*/ static PyObject * bytes_partition_impl(PyBytesObject *self, Py_buffer *sep) -/*[clinic end generated code: output=f532b392a17ff695 input=61cca95519406099]*/ +/*[clinic end generated code: output=f532b392a17ff695 input=2e6e551ea4f8b95a]*/ { return stringlib_partition( (PyObject*) self, @@ -1815,17 +1823,18 @@ bytes.rpartition Partition the bytes into three parts using the given separator. -This will search for the separator sep in the bytes, starting at the end. If -the separator is found, returns a 3-tuple containing the part before the -separator, the separator itself, and the part after it. +This will search for the separator sep in the bytes, starting at the +end. If the separator is found, returns a 3-tuple containing the +part before the separator, the separator itself, and the part after +it. -If the separator is not found, returns a 3-tuple containing two empty bytes -objects and the original bytes object. +If the separator is not found, returns a 3-tuple containing two +empty bytes objects and the original bytes object. [clinic start generated code]*/ static PyObject * bytes_rpartition_impl(PyBytesObject *self, Py_buffer *sep) -/*[clinic end generated code: output=191b114cbb028e50 input=d78db010c8cfdbe1]*/ +/*[clinic end generated code: output=191b114cbb028e50 input=f7d24f722a5470a4]*/ { return stringlib_rpartition( (PyObject*) self, @@ -1839,12 +1848,13 @@ bytes.rsplit = bytes.split Return a list of the sections in the bytes, using sep as the delimiter. -Splitting is done starting at the end of the bytes and working to the front. +Splitting is done starting at the end of the bytes and working to +the front. [clinic start generated code]*/ static PyObject * bytes_rsplit_impl(PyBytesObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=ba698d9ea01e1c8f input=0f86c9f28f7d7b7b]*/ +/*[clinic end generated code: output=ba698d9ea01e1c8f input=7fd643d4337b6a9b]*/ { Py_ssize_t len = PyBytes_GET_SIZE(self), n; const char *s = PyBytes_AS_STRING(self), *sub; @@ -2067,12 +2077,13 @@ bytes.strip Strip leading and trailing bytes contained in the argument. -If the argument is omitted or None, strip leading and trailing ASCII whitespace. +If the argument is omitted or None, strip leading and trailing ASCII +whitespace. [clinic start generated code]*/ static PyObject * bytes_strip_impl(PyBytesObject *self, PyObject *bytes) -/*[clinic end generated code: output=c7c228d3bd104a1b input=8a354640e4e0b3ef]*/ +/*[clinic end generated code: output=c7c228d3bd104a1b input=9ffea5f752032bd0]*/ { return do_argstrip(self, BOTHSTRIP, bytes); } @@ -2140,14 +2151,15 @@ bytes.translate Return a copy with each character mapped by the given translation table. -All characters occurring in the optional argument delete are removed. -The remaining characters are mapped through the given translation table. +All characters occurring in the optional argument delete are +removed. The remaining characters are mapped through the given +translation table. [clinic start generated code]*/ static PyObject * bytes_translate_impl(PyBytesObject *self, PyObject *table, PyObject *deletechars) -/*[clinic end generated code: output=43be3437f1956211 input=0ecdf159f654233c]*/ +/*[clinic end generated code: output=43be3437f1956211 input=4e4460a981d768c5]*/ { const char *input; char *output; @@ -2269,15 +2281,15 @@ bytes.maketrans Return a translation table usable for the bytes or bytearray translate method. -The returned table will be one where each byte in frm is mapped to the byte at -the same position in to. +The returned table will be one where each byte in frm is mapped to +the byte at the same position in to. The bytes objects frm and to must be of the same length. [clinic start generated code]*/ static PyObject * bytes_maketrans_impl(Py_buffer *frm, Py_buffer *to) -/*[clinic end generated code: output=a36f6399d4b77f6f input=a3bd00d430a0979f]*/ +/*[clinic end generated code: output=a36f6399d4b77f6f input=c88bcc17621f785d]*/ { return _Py_bytes_maketrans(frm, to); } @@ -2319,13 +2331,14 @@ bytes.removeprefix as bytes_removeprefix Return a bytes object with the given prefix string removed if present. -If the bytes starts with the prefix string, return bytes[len(prefix):]. -Otherwise, return a copy of the original bytes. +If the bytes starts with the prefix string, return +bytes[len(prefix):]. Otherwise, return a copy of the original +bytes. [clinic start generated code]*/ static PyObject * bytes_removeprefix_impl(PyBytesObject *self, Py_buffer *prefix) -/*[clinic end generated code: output=f006865331a06ab6 input=0c93bac817a8502c]*/ +/*[clinic end generated code: output=f006865331a06ab6 input=8f371f9421b8addd]*/ { const char *self_start = PyBytes_AS_STRING(self); Py_ssize_t self_len = PyBytes_GET_SIZE(self); @@ -2355,14 +2368,14 @@ bytes.removesuffix as bytes_removesuffix Return a bytes object with the given suffix string removed if present. -If the bytes ends with the suffix string and that suffix is not empty, -return bytes[:-len(prefix)]. Otherwise, return a copy of the original -bytes. +If the bytes ends with the suffix string and that suffix is not +empty, return bytes[:-len(prefix)]. Otherwise, return a copy of the +original bytes. [clinic start generated code]*/ static PyObject * bytes_removesuffix_impl(PyBytesObject *self, Py_buffer *suffix) -/*[clinic end generated code: output=d887d308e3242eeb input=9f4e1da8c637bbf1]*/ +/*[clinic end generated code: output=d887d308e3242eeb input=35eada0260d1171b]*/ { const char *self_start = PyBytes_AS_STRING(self); Py_ssize_t self_len = PyBytes_GET_SIZE(self); @@ -2440,11 +2453,11 @@ bytes.decode encoding: str(c_default="NULL") = 'utf-8' The encoding with which to decode the bytes. errors: str(c_default="NULL") = 'strict' - The error handling scheme to use for the handling of decoding errors. - The default is 'strict' meaning that decoding errors raise a - UnicodeDecodeError. Other possible values are 'ignore' and 'replace' - as well as any other name registered with codecs.register_error that - can handle UnicodeDecodeErrors. + The error handling scheme to use for the handling of decoding + errors. The default is 'strict' meaning that decoding errors + raise a UnicodeDecodeError. Other possible values are 'ignore' + and 'replace' as well as any other name registered with + codecs.register_error that can handle UnicodeDecodeErrors. Decode the bytes using the codec registered for encoding. [clinic start generated code]*/ @@ -2452,7 +2465,7 @@ Decode the bytes using the codec registered for encoding. static PyObject * bytes_decode_impl(PyBytesObject *self, const char *encoding, const char *errors) -/*[clinic end generated code: output=5649a53dde27b314 input=958174769d2a40ca]*/ +/*[clinic end generated code: output=5649a53dde27b314 input=94e9b8524f1d7f37]*/ { return PyUnicode_FromEncodedObject((PyObject*)self, encoding, errors); } @@ -2465,13 +2478,13 @@ bytes.splitlines Return a list of the lines in the bytes, breaking at line boundaries. -Line breaks are not included in the resulting list unless keepends is given and -true. +Line breaks are not included in the resulting list unless keepends +is given and true. [clinic start generated code]*/ static PyObject * bytes_splitlines_impl(PyBytesObject *self, int keepends) -/*[clinic end generated code: output=3484149a5d880ffb input=5d7b898af2fe55c0]*/ +/*[clinic end generated code: output=3484149a5d880ffb input=8b7b6915be775bcf]*/ { return stringlib_splitlines( (PyObject*) self, PyBytes_AS_STRING(self), @@ -2619,9 +2632,10 @@ bytes.hex sep: object = NULL An optional single character or byte to separate hex bytes. + bytes_per_sep: int = 1 - How many bytes between separators. Positive values count from the - right, negative values count from the left. + How many bytes between separators. Positive values count from + the right, negative values count from the left. Create a string of hexadecimal numbers from a bytes object. @@ -2639,7 +2653,7 @@ Create a string of hexadecimal numbers from a bytes object. static PyObject * bytes_hex_impl(PyBytesObject *self, PyObject *sep, int bytes_per_sep) -/*[clinic end generated code: output=1f134da504064139 input=1a21282b1f1ae595]*/ +/*[clinic end generated code: output=1f134da504064139 input=67d7bb5f70f0d6f2]*/ { const char *argbuf = PyBytes_AS_STRING(self); Py_ssize_t arglen = PyBytes_GET_SIZE(self); @@ -2859,7 +2873,6 @@ _PyBytes_FromList(PyObject *x) Py_ssize_t i, size = PyList_GET_SIZE(x); Py_ssize_t value; char *str; - PyObject *item; _PyBytesWriter writer; _PyBytesWriter_Init(&writer); @@ -2870,8 +2883,10 @@ _PyBytes_FromList(PyObject *x) size = writer.allocated; for (i = 0; i < PyList_GET_SIZE(x); i++) { - item = PyList_GET_ITEM(x, i); - Py_INCREF(item); + PyObject *item = _PyList_GetItemRef((PyListObject *)x, i); + if (item == NULL) { + goto error; + } value = PyNumber_AsSsize_t(item, NULL); Py_DECREF(item); if (value == -1 && PyErr_Occurred()) @@ -3142,7 +3157,7 @@ PyBytes_Concat(PyObject **pv, PyObject *w) return; } - if (Py_REFCNT(*pv) == 1 && PyBytes_CheckExact(*pv)) { + if (_PyObject_IsUniquelyReferenced(*pv) && PyBytes_CheckExact(*pv)) { /* Only one reference, so we can resize in place */ Py_ssize_t oldsize; Py_buffer wb; @@ -3227,7 +3242,7 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) Py_DECREF(v); return 0; } - if (Py_REFCNT(v) != 1) { + if (!_PyObject_IsUniquelyReferenced(v)) { if (oldsize < newsize) { *pv = _PyBytes_FromSize(newsize, 0); if (*pv) { diff --git a/Objects/call.c b/Objects/call.c index b1610dababd4660..0299e458b1d8792 100644 --- a/Objects/call.c +++ b/Objects/call.c @@ -726,6 +726,7 @@ _PyObject_CallMethodId(PyObject *obj, _Py_Identifier *name, PyObject * _PyObject_CallMethodFormat(PyThreadState *tstate, PyObject *callable, const char *format, ...) { + assert(callable != NULL); va_list va; va_start(va, format); PyObject *retval = callmethod(tstate, callable, format, va); @@ -824,6 +825,60 @@ object_vacall(PyThreadState *tstate, PyObject *base, return result; } +PyObject * +_PyObject_VectorcallPrepend(PyThreadState *tstate, PyObject *callable, + PyObject *arg, PyObject *const *args, + size_t nargsf, PyObject *kwnames) +{ + Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); + assert(nargs == 0 || args[nargs-1]); + + PyObject *result; + if (nargsf & PY_VECTORCALL_ARGUMENTS_OFFSET) { + /* PY_VECTORCALL_ARGUMENTS_OFFSET is set, so we are allowed to mutate the vector */ + PyObject **newargs = (PyObject**)args - 1; + nargs += 1; + PyObject *tmp = newargs[0]; + newargs[0] = arg; + assert(newargs[nargs-1]); + result = _PyObject_VectorcallTstate(tstate, callable, newargs, + nargs, kwnames); + newargs[0] = tmp; + } + else { + Py_ssize_t nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames); + Py_ssize_t totalargs = nargs + nkwargs; + if (totalargs == 0) { + return _PyObject_VectorcallTstate(tstate, callable, &arg, 1, NULL); + } + + PyObject *newargs_stack[_PY_FASTCALL_SMALL_STACK]; + PyObject **newargs; + if (totalargs <= (Py_ssize_t)Py_ARRAY_LENGTH(newargs_stack) - 1) { + newargs = newargs_stack; + } + else { + newargs = PyMem_Malloc((totalargs+1) * sizeof(PyObject *)); + if (newargs == NULL) { + _PyErr_NoMemory(tstate); + return NULL; + } + } + /* use borrowed references */ + newargs[0] = arg; + /* bpo-37138: since totalargs > 0, it's impossible that args is NULL. + * We need this, since calling memcpy() with a NULL pointer is + * undefined behaviour. */ + assert(args != NULL); + memcpy(newargs + 1, args, totalargs * sizeof(PyObject *)); + result = _PyObject_VectorcallTstate(tstate, callable, + newargs, nargs+1, kwnames); + if (newargs != newargs_stack) { + PyMem_Free(newargs); + } + } + return result; +} PyObject * PyObject_VectorcallMethod(PyObject *name, PyObject *const *args, @@ -834,28 +889,44 @@ PyObject_VectorcallMethod(PyObject *name, PyObject *const *args, assert(PyVectorcall_NARGS(nargsf) >= 1); PyThreadState *tstate = _PyThreadState_GET(); - PyObject *callable = NULL; + _PyCStackRef self, method; + _PyThreadState_PushCStackRef(tstate, &self); + _PyThreadState_PushCStackRef(tstate, &method); /* Use args[0] as "self" argument */ - int unbound = _PyObject_GetMethod(args[0], name, &callable); - if (callable == NULL) { + self.ref = PyStackRef_FromPyObjectBorrow(args[0]); + int unbound = _PyObject_GetMethodStackRef(tstate, &self.ref, name, &method.ref); + if (unbound < 0) { + _PyThreadState_PopCStackRef(tstate, &method); + _PyThreadState_PopCStackRef(tstate, &self); return NULL; } - if (unbound) { + PyObject *callable = PyStackRef_AsPyObjectBorrow(method.ref); + PyObject *self_obj = PyStackRef_AsPyObjectBorrow(self.ref); + PyObject *result; + + EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_METHOD, callable); + if (self_obj == NULL) { + /* Skip "self". We can keep PY_VECTORCALL_ARGUMENTS_OFFSET since + * args[-1] in the onward call is args[0] here. */ + result = _PyObject_VectorcallTstate(tstate, callable, + args + 1, nargsf - 1, kwnames); + } + else if (self_obj == args[0]) { /* We must remove PY_VECTORCALL_ARGUMENTS_OFFSET since * that would be interpreted as allowing to change args[-1] */ - nargsf &= ~PY_VECTORCALL_ARGUMENTS_OFFSET; + result = _PyObject_VectorcallTstate(tstate, callable, args, + nargsf & ~PY_VECTORCALL_ARGUMENTS_OFFSET, + kwnames); } else { - /* Skip "self". We can keep PY_VECTORCALL_ARGUMENTS_OFFSET since - * args[-1] in the onward call is args[0] here. */ - args++; - nargsf--; + /* classmethod: self_obj is the type, not args[0]. Replace + * args[0] with self_obj and call the underlying callable. */ + result = _PyObject_VectorcallPrepend(tstate, callable, self_obj, + args + 1, nargsf - 1, kwnames); } - EVAL_CALL_STAT_INC_IF_FUNCTION(EVAL_CALL_METHOD, callable); - PyObject *result = _PyObject_VectorcallTstate(tstate, callable, - args, nargsf, kwnames); - Py_DECREF(callable); + _PyThreadState_PopCStackRef(tstate, &method); + _PyThreadState_PopCStackRef(tstate, &self); return result; } @@ -868,19 +939,26 @@ PyObject_CallMethodObjArgs(PyObject *obj, PyObject *name, ...) return null_error(tstate); } - PyObject *callable = NULL; - int is_method = _PyObject_GetMethod(obj, name, &callable); - if (callable == NULL) { + _PyCStackRef self, method; + _PyThreadState_PushCStackRef(tstate, &self); + _PyThreadState_PushCStackRef(tstate, &method); + self.ref = PyStackRef_FromPyObjectBorrow(obj); + int res = _PyObject_GetMethodStackRef(tstate, &self.ref, name, &method.ref); + if (res < 0) { + _PyThreadState_PopCStackRef(tstate, &method); + _PyThreadState_PopCStackRef(tstate, &self); return NULL; } - obj = is_method ? obj : NULL; + PyObject *callable = PyStackRef_AsPyObjectBorrow(method.ref); + PyObject *self_obj = PyStackRef_AsPyObjectBorrow(self.ref); va_list vargs; va_start(vargs, name); - PyObject *result = object_vacall(tstate, obj, callable, vargs); + PyObject *result = object_vacall(tstate, self_obj, callable, vargs); va_end(vargs); - Py_DECREF(callable); + _PyThreadState_PopCStackRef(tstate, &method); + _PyThreadState_PopCStackRef(tstate, &self); return result; } diff --git a/Objects/classobject.c b/Objects/classobject.c index 58e1d17977322e6..d511e077be35643 100644 --- a/Objects/classobject.c +++ b/Objects/classobject.c @@ -7,6 +7,7 @@ #include "pycore_object.h" #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "clinic/classobject.c.h" @@ -50,54 +51,7 @@ method_vectorcall(PyObject *method, PyObject *const *args, PyThreadState *tstate = _PyThreadState_GET(); PyObject *self = PyMethod_GET_SELF(method); PyObject *func = PyMethod_GET_FUNCTION(method); - Py_ssize_t nargs = PyVectorcall_NARGS(nargsf); - assert(nargs == 0 || args[nargs-1]); - - PyObject *result; - if (nargsf & PY_VECTORCALL_ARGUMENTS_OFFSET) { - /* PY_VECTORCALL_ARGUMENTS_OFFSET is set, so we are allowed to mutate the vector */ - PyObject **newargs = (PyObject**)args - 1; - nargs += 1; - PyObject *tmp = newargs[0]; - newargs[0] = self; - assert(newargs[nargs-1]); - result = _PyObject_VectorcallTstate(tstate, func, newargs, - nargs, kwnames); - newargs[0] = tmp; - } - else { - Py_ssize_t nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames); - Py_ssize_t totalargs = nargs + nkwargs; - if (totalargs == 0) { - return _PyObject_VectorcallTstate(tstate, func, &self, 1, NULL); - } - - PyObject *newargs_stack[_PY_FASTCALL_SMALL_STACK]; - PyObject **newargs; - if (totalargs <= (Py_ssize_t)Py_ARRAY_LENGTH(newargs_stack) - 1) { - newargs = newargs_stack; - } - else { - newargs = PyMem_Malloc((totalargs+1) * sizeof(PyObject *)); - if (newargs == NULL) { - _PyErr_NoMemory(tstate); - return NULL; - } - } - /* use borrowed references */ - newargs[0] = self; - /* bpo-37138: since totalargs > 0, it's impossible that args is NULL. - * We need this, since calling memcpy() with a NULL pointer is - * undefined behaviour. */ - assert(args != NULL); - memcpy(newargs + 1, args, totalargs * sizeof(PyObject *)); - result = _PyObject_VectorcallTstate(tstate, func, - newargs, nargs+1, kwnames); - if (newargs != newargs_stack) { - PyMem_Free(newargs); - } - } - return result; + return _PyObject_VectorcallPrepend(tstate, func, self, args, nargsf, kwnames); } @@ -245,8 +199,7 @@ method_dealloc(PyObject *self) { PyMethodObject *im = _PyMethodObject_CAST(self); _PyObject_GC_UNTRACK(im); - if (im->im_weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *)im); + FT_CLEAR_WEAKREFS(self, im->im_weakreflist); Py_DECREF(im->im_func); Py_XDECREF(im->im_self); assert(Py_IS_TYPE(self, &PyMethod_Type)); diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index ffb45ade11f6dc6..dd64b9eae41e723 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -599,7 +599,7 @@ PyDoc_STRVAR(bytearray_resize__doc__, "Resize the internal buffer of bytearray to len.\n" "\n" " size\n" -" New size to resize to.."); +" New size to resize to."); #define BYTEARRAY_RESIZE_METHODDEF \ {"resize", (PyCFunction)bytearray_resize, METH_O, bytearray_resize__doc__}, @@ -625,7 +625,9 @@ bytearray_resize(PyObject *self, PyObject *arg) } size = ival; } + Py_BEGIN_CRITICAL_SECTION(self); return_value = bytearray_resize_impl((PyByteArrayObject *)self, size); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -640,8 +642,9 @@ PyDoc_STRVAR(bytearray_translate__doc__, " table\n" " Translation table, which must be a bytes object of length 256.\n" "\n" -"All characters occurring in the optional argument delete are removed.\n" -"The remaining characters are mapped through the given translation table."); +"All characters occurring in the optional argument delete are\n" +"removed. The remaining characters are mapped through the given\n" +"translation table."); #define BYTEARRAY_TRANSLATE_METHODDEF \ {"translate", _PyCFunction_CAST(bytearray_translate), METH_FASTCALL|METH_KEYWORDS, bytearray_translate__doc__}, @@ -711,8 +714,8 @@ PyDoc_STRVAR(bytearray_maketrans__doc__, "\n" "Return a translation table usable for the bytes or bytearray translate method.\n" "\n" -"The returned table will be one where each byte in frm is mapped to the byte at\n" -"the same position in to.\n" +"The returned table will be one where each byte in frm is mapped to\n" +"the byte at the same position in to.\n" "\n" "The bytes objects frm and to must be of the same length."); @@ -831,8 +834,8 @@ PyDoc_STRVAR(bytearray_split__doc__, "\n" " sep\n" " The delimiter according which to split the bytearray.\n" -" None (the default value) means split on ASCII whitespace characters\n" -" (space, tab, return, newline, formfeed, vertical tab).\n" +" None (the default value) means split on ASCII whitespace\n" +" characters (space, tab, return, newline, formfeed, vertical tab).\n" " maxsplit\n" " Maximum number of splits to do.\n" " -1 (the default value) means no limit."); @@ -921,12 +924,13 @@ PyDoc_STRVAR(bytearray_partition__doc__, "\n" "Partition the bytearray into three parts using the given separator.\n" "\n" -"This will search for the separator sep in the bytearray. If the separator is\n" -"found, returns a 3-tuple containing the part before the separator, the\n" -"separator itself, and the part after it as new bytearray objects.\n" +"This will search for the separator sep in the bytearray. If the\n" +"separator is found, returns a 3-tuple containing the part before the\n" +"separator, the separator itself, and the part after it as new\n" +"bytearray objects.\n" "\n" -"If the separator is not found, returns a 3-tuple containing the copy of the\n" -"original bytearray object and two empty bytearray objects."); +"If the separator is not found, returns a 3-tuple containing the copy\n" +"of the original bytearray object and two empty bytearray objects."); #define BYTEARRAY_PARTITION_METHODDEF \ {"partition", (PyCFunction)bytearray_partition, METH_O, bytearray_partition__doc__}, @@ -952,13 +956,14 @@ PyDoc_STRVAR(bytearray_rpartition__doc__, "\n" "Partition the bytearray into three parts using the given separator.\n" "\n" -"This will search for the separator sep in the bytearray, starting at the end.\n" -"If the separator is found, returns a 3-tuple containing the part before the\n" -"separator, the separator itself, and the part after it as new bytearray\n" -"objects.\n" +"This will search for the separator sep in the bytearray, starting at\n" +"the end. If the separator is found, returns a 3-tuple containing\n" +"the part before the separator, the separator itself, and the part\n" +"after it as new bytearray objects.\n" "\n" -"If the separator is not found, returns a 3-tuple containing two empty bytearray\n" -"objects and the copy of the original bytearray object."); +"If the separator is not found, returns a 3-tuple containing two\n" +"empty bytearray objects and the copy of the original bytearray\n" +"object."); #define BYTEARRAY_RPARTITION_METHODDEF \ {"rpartition", (PyCFunction)bytearray_rpartition, METH_O, bytearray_rpartition__doc__}, @@ -986,13 +991,14 @@ PyDoc_STRVAR(bytearray_rsplit__doc__, "\n" " sep\n" " The delimiter according which to split the bytearray.\n" -" None (the default value) means split on ASCII whitespace characters\n" -" (space, tab, return, newline, formfeed, vertical tab).\n" +" None (the default value) means split on ASCII whitespace\n" +" characters (space, tab, return, newline, formfeed, vertical tab).\n" " maxsplit\n" " Maximum number of splits to do.\n" " -1 (the default value) means no limit.\n" "\n" -"Splitting is done starting at the end of the bytearray and working to the front."); +"Splitting is done starting at the end of the bytearray and working\n" +"to the front."); #define BYTEARRAY_RSPLIT_METHODDEF \ {"rsplit", _PyCFunction_CAST(bytearray_rsplit), METH_FASTCALL|METH_KEYWORDS, bytearray_rsplit__doc__}, @@ -1294,7 +1300,8 @@ PyDoc_STRVAR(bytearray_strip__doc__, "\n" "Strip leading and trailing bytes contained in the argument.\n" "\n" -"If the argument is omitted or None, strip leading and trailing ASCII whitespace."); +"If the argument is omitted or None, strip leading and trailing ASCII\n" +"whitespace."); #define BYTEARRAY_STRIP_METHODDEF \ {"strip", _PyCFunction_CAST(bytearray_strip), METH_FASTCALL, bytearray_strip__doc__}, @@ -1405,11 +1412,11 @@ PyDoc_STRVAR(bytearray_decode__doc__, " encoding\n" " The encoding with which to decode the bytearray.\n" " errors\n" -" The error handling scheme to use for the handling of decoding errors.\n" -" The default is \'strict\' meaning that decoding errors raise a\n" -" UnicodeDecodeError. Other possible values are \'ignore\' and \'replace\'\n" -" as well as any other name registered with codecs.register_error that\n" -" can handle UnicodeDecodeErrors."); +" The error handling scheme to use for the handling of decoding\n" +" errors. The default is \'strict\' meaning that decoding errors\n" +" raise a UnicodeDecodeError. Other possible values are \'ignore\'\n" +" and \'replace\' as well as any other name registered with\n" +" codecs.register_error that can handle UnicodeDecodeErrors."); #define BYTEARRAY_DECODE_METHODDEF \ {"decode", _PyCFunction_CAST(bytearray_decode), METH_FASTCALL|METH_KEYWORDS, bytearray_decode__doc__}, @@ -1508,7 +1515,8 @@ PyDoc_STRVAR(bytearray_join__doc__, "\n" "Concatenate any number of bytes/bytearray objects.\n" "\n" -"The bytearray whose method is called is inserted in between each pair.\n" +"The bytearray whose method is called is inserted in between each\n" +"pair.\n" "\n" "The result is returned as a new bytearray object."); @@ -1536,8 +1544,8 @@ PyDoc_STRVAR(bytearray_splitlines__doc__, "\n" "Return a list of the lines in the bytearray, breaking at line boundaries.\n" "\n" -"Line breaks are not included in the resulting list unless keepends is given and\n" -"true."); +"Line breaks are not included in the resulting list unless keepends\n" +"is given and true."); #define BYTEARRAY_SPLITLINES_METHODDEF \ {"splitlines", _PyCFunction_CAST(bytearray_splitlines), METH_FASTCALL|METH_KEYWORDS, bytearray_splitlines__doc__}, @@ -1608,7 +1616,8 @@ PyDoc_STRVAR(bytearray_fromhex__doc__, "Create a bytearray object from a string of hexadecimal numbers.\n" "\n" "Spaces between two numbers are accepted.\n" -"Example: bytearray.fromhex(\'B9 01EF\') -> bytearray(b\'\\\\xb9\\\\x01\\\\xef\')"); +"Example:\n" +" bytearray.fromhex(\'B9 01EF\') -> bytearray(b\'\\\\xb9\\\\x01\\\\xef\')"); #define BYTEARRAY_FROMHEX_METHODDEF \ {"fromhex", (PyCFunction)bytearray_fromhex, METH_O|METH_CLASS, bytearray_fromhex__doc__}, @@ -1635,8 +1644,8 @@ PyDoc_STRVAR(bytearray_hex__doc__, " sep\n" " An optional single character or byte to separate hex bytes.\n" " bytes_per_sep\n" -" How many bytes between separators. Positive values count from the\n" -" right, negative values count from the left.\n" +" How many bytes between separators. Positive values count from\n" +" the right, negative values count from the left.\n" "\n" "Example:\n" ">>> value = bytearray([0xb9, 0x01, 0xef])\n" @@ -1796,4 +1805,4 @@ bytearray_sizeof(PyObject *self, PyObject *Py_UNUSED(ignored)) { return bytearray_sizeof_impl((PyByteArrayObject *)self); } -/*[clinic end generated code: output=be6d28193bc96a2c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d13c01fc42a84ff5 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/bytesobject.c.h b/Objects/clinic/bytesobject.c.h index 00cf13d422d900d..986cc87c8df2a19 100644 --- a/Objects/clinic/bytesobject.c.h +++ b/Objects/clinic/bytesobject.c.h @@ -35,8 +35,8 @@ PyDoc_STRVAR(bytes_split__doc__, "\n" " sep\n" " The delimiter according which to split the bytes.\n" -" None (the default value) means split on ASCII whitespace characters\n" -" (space, tab, return, newline, formfeed, vertical tab).\n" +" None (the default value) means split on ASCII whitespace\n" +" characters (space, tab, return, newline, formfeed, vertical tab).\n" " maxsplit\n" " Maximum number of splits to do.\n" " -1 (the default value) means no limit."); @@ -122,12 +122,12 @@ PyDoc_STRVAR(bytes_partition__doc__, "\n" "Partition the bytes into three parts using the given separator.\n" "\n" -"This will search for the separator sep in the bytes. If the separator is found,\n" -"returns a 3-tuple containing the part before the separator, the separator\n" -"itself, and the part after it.\n" +"This will search for the separator sep in the bytes. If the\n" +"separator is found, returns a 3-tuple containing the part before the\n" +"separator, the separator itself, and the part after it.\n" "\n" -"If the separator is not found, returns a 3-tuple containing the original bytes\n" -"object and two empty bytes objects."); +"If the separator is not found, returns a 3-tuple containing the\n" +"original bytes object and two empty bytes objects."); #define BYTES_PARTITION_METHODDEF \ {"partition", (PyCFunction)bytes_partition, METH_O, bytes_partition__doc__}, @@ -161,12 +161,13 @@ PyDoc_STRVAR(bytes_rpartition__doc__, "\n" "Partition the bytes into three parts using the given separator.\n" "\n" -"This will search for the separator sep in the bytes, starting at the end. If\n" -"the separator is found, returns a 3-tuple containing the part before the\n" -"separator, the separator itself, and the part after it.\n" +"This will search for the separator sep in the bytes, starting at the\n" +"end. If the separator is found, returns a 3-tuple containing the\n" +"part before the separator, the separator itself, and the part after\n" +"it.\n" "\n" -"If the separator is not found, returns a 3-tuple containing two empty bytes\n" -"objects and the original bytes object."); +"If the separator is not found, returns a 3-tuple containing two\n" +"empty bytes objects and the original bytes object."); #define BYTES_RPARTITION_METHODDEF \ {"rpartition", (PyCFunction)bytes_rpartition, METH_O, bytes_rpartition__doc__}, @@ -202,13 +203,14 @@ PyDoc_STRVAR(bytes_rsplit__doc__, "\n" " sep\n" " The delimiter according which to split the bytes.\n" -" None (the default value) means split on ASCII whitespace characters\n" -" (space, tab, return, newline, formfeed, vertical tab).\n" +" None (the default value) means split on ASCII whitespace\n" +" characters (space, tab, return, newline, formfeed, vertical tab).\n" " maxsplit\n" " Maximum number of splits to do.\n" " -1 (the default value) means no limit.\n" "\n" -"Splitting is done starting at the end of the bytes and working to the front."); +"Splitting is done starting at the end of the bytes and working to\n" +"the front."); #define BYTES_RSPLIT_METHODDEF \ {"rsplit", _PyCFunction_CAST(bytes_rsplit), METH_FASTCALL|METH_KEYWORDS, bytes_rsplit__doc__}, @@ -523,7 +525,8 @@ PyDoc_STRVAR(bytes_strip__doc__, "\n" "Strip leading and trailing bytes contained in the argument.\n" "\n" -"If the argument is omitted or None, strip leading and trailing ASCII whitespace."); +"If the argument is omitted or None, strip leading and trailing ASCII\n" +"whitespace."); #define BYTES_STRIP_METHODDEF \ {"strip", _PyCFunction_CAST(bytes_strip), METH_FASTCALL, bytes_strip__doc__}, @@ -677,8 +680,9 @@ PyDoc_STRVAR(bytes_translate__doc__, " table\n" " Translation table, which must be a bytes object of length 256.\n" "\n" -"All characters occurring in the optional argument delete are removed.\n" -"The remaining characters are mapped through the given translation table."); +"All characters occurring in the optional argument delete are\n" +"removed. The remaining characters are mapped through the given\n" +"translation table."); #define BYTES_TRANSLATE_METHODDEF \ {"translate", _PyCFunction_CAST(bytes_translate), METH_FASTCALL|METH_KEYWORDS, bytes_translate__doc__}, @@ -746,8 +750,8 @@ PyDoc_STRVAR(bytes_maketrans__doc__, "\n" "Return a translation table usable for the bytes or bytearray translate method.\n" "\n" -"The returned table will be one where each byte in frm is mapped to the byte at\n" -"the same position in to.\n" +"The returned table will be one where each byte in frm is mapped to\n" +"the byte at the same position in to.\n" "\n" "The bytes objects frm and to must be of the same length."); @@ -862,8 +866,9 @@ PyDoc_STRVAR(bytes_removeprefix__doc__, "\n" "Return a bytes object with the given prefix string removed if present.\n" "\n" -"If the bytes starts with the prefix string, return bytes[len(prefix):].\n" -"Otherwise, return a copy of the original bytes."); +"If the bytes starts with the prefix string, return\n" +"bytes[len(prefix):]. Otherwise, return a copy of the original\n" +"bytes."); #define BYTES_REMOVEPREFIX_METHODDEF \ {"removeprefix", (PyCFunction)bytes_removeprefix, METH_O, bytes_removeprefix__doc__}, @@ -897,9 +902,9 @@ PyDoc_STRVAR(bytes_removesuffix__doc__, "\n" "Return a bytes object with the given suffix string removed if present.\n" "\n" -"If the bytes ends with the suffix string and that suffix is not empty,\n" -"return bytes[:-len(prefix)]. Otherwise, return a copy of the original\n" -"bytes."); +"If the bytes ends with the suffix string and that suffix is not\n" +"empty, return bytes[:-len(prefix)]. Otherwise, return a copy of the\n" +"original bytes."); #define BYTES_REMOVESUFFIX_METHODDEF \ {"removesuffix", (PyCFunction)bytes_removesuffix, METH_O, bytes_removesuffix__doc__}, @@ -1038,11 +1043,11 @@ PyDoc_STRVAR(bytes_decode__doc__, " encoding\n" " The encoding with which to decode the bytes.\n" " errors\n" -" The error handling scheme to use for the handling of decoding errors.\n" -" The default is \'strict\' meaning that decoding errors raise a\n" -" UnicodeDecodeError. Other possible values are \'ignore\' and \'replace\'\n" -" as well as any other name registered with codecs.register_error that\n" -" can handle UnicodeDecodeErrors."); +" The error handling scheme to use for the handling of decoding\n" +" errors. The default is \'strict\' meaning that decoding errors\n" +" raise a UnicodeDecodeError. Other possible values are \'ignore\'\n" +" and \'replace\' as well as any other name registered with\n" +" codecs.register_error that can handle UnicodeDecodeErrors."); #define BYTES_DECODE_METHODDEF \ {"decode", _PyCFunction_CAST(bytes_decode), METH_FASTCALL|METH_KEYWORDS, bytes_decode__doc__}, @@ -1139,8 +1144,8 @@ PyDoc_STRVAR(bytes_splitlines__doc__, "\n" "Return a list of the lines in the bytes, breaking at line boundaries.\n" "\n" -"Line breaks are not included in the resulting list unless keepends is given and\n" -"true."); +"Line breaks are not included in the resulting list unless keepends\n" +"is given and true."); #define BYTES_SPLITLINES_METHODDEF \ {"splitlines", _PyCFunction_CAST(bytes_splitlines), METH_FASTCALL|METH_KEYWORDS, bytes_splitlines__doc__}, @@ -1236,8 +1241,8 @@ PyDoc_STRVAR(bytes_hex__doc__, " sep\n" " An optional single character or byte to separate hex bytes.\n" " bytes_per_sep\n" -" How many bytes between separators. Positive values count from the\n" -" right, negative values count from the left.\n" +" How many bytes between separators. Positive values count from\n" +" the right, negative values count from the left.\n" "\n" "Example:\n" ">>> value = b\'\\xb9\\x01\\xef\'\n" @@ -1411,4 +1416,4 @@ bytes_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=08b9507244f73638 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f2845457d3d7b36e input=a9049054013a1b77]*/ diff --git a/Objects/clinic/codeobject.c.h b/Objects/clinic/codeobject.c.h index 0cd6e0b56620e85..88333e9d3363eb4 100644 --- a/Objects/clinic/codeobject.c.h +++ b/Objects/clinic/codeobject.c.h @@ -414,7 +414,8 @@ PyDoc_STRVAR(code__varname_from_oparg__doc__, "\n" "(internal-only) Return the local variable name for the given oparg.\n" "\n" -"WARNING: this method is for internal use only and may change or go away."); +"WARNING: this method is for internal use only and may change or go\n" +"away."); #define CODE__VARNAME_FROM_OPARG_METHODDEF \ {"_varname_from_oparg", _PyCFunction_CAST(code__varname_from_oparg), METH_FASTCALL|METH_KEYWORDS, code__varname_from_oparg__doc__}, @@ -470,4 +471,4 @@ code__varname_from_oparg(PyObject *self, PyObject *const *args, Py_ssize_t nargs exit: return return_value; } -/*[clinic end generated code: output=c5c6e40fc357defe input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5c22e29e430401b4 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/exceptions.c.h b/Objects/clinic/exceptions.c.h index 9baac8b1cc660b0..5047a673e579c66 100644 --- a/Objects/clinic/exceptions.c.h +++ b/Objects/clinic/exceptions.c.h @@ -44,7 +44,7 @@ BaseException___setstate__(PyObject *self, PyObject *state) { PyObject *return_value = NULL; - Py_BEGIN_CRITICAL_SECTION(self); + Py_BEGIN_CRITICAL_SECTION(state); return_value = BaseException___setstate___impl((PyBaseExceptionObject *)self, state); Py_END_CRITICAL_SECTION(); @@ -380,4 +380,4 @@ BaseExceptionGroup_subgroup(PyObject *self, PyObject *matcher_value) return return_value; } -/*[clinic end generated code: output=fcf70b3b71f3d14a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e63b88d0443b4f92 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/floatobject.c.h b/Objects/clinic/floatobject.c.h index 4051131f480ccb6..f62dc94594930f4 100644 --- a/Objects/clinic/floatobject.c.h +++ b/Objects/clinic/floatobject.c.h @@ -290,9 +290,9 @@ PyDoc_STRVAR(float___getformat____doc__, "\n" "It exists mainly to be used in Python\'s test suite.\n" "\n" -"This function returns whichever of \'unknown\', \'IEEE, big-endian\' or \'IEEE,\n" -"little-endian\' best describes the format of floating-point numbers used by the\n" -"C type named by typestr."); +"This function returns whichever of \'unknown\', \'IEEE, big-endian\' or\n" +"\'IEEE, little-endian\' best describes the format of floating-point\n" +"numbers used by the C type named by typestr."); #define FLOAT___GETFORMAT___METHODDEF \ {"__getformat__", (PyCFunction)float___getformat__, METH_O|METH_CLASS, float___getformat____doc__}, @@ -353,4 +353,4 @@ float___format__(PyObject *self, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=927035897ea3573f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=cb5263e0ed588ac7 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/interpolationobject.c.h b/Objects/clinic/interpolationobject.c.h index 7a94dabafc92f24..2030e17e49e47a7 100644 --- a/Objects/clinic/interpolationobject.c.h +++ b/Objects/clinic/interpolationobject.c.h @@ -47,26 +47,31 @@ interpolation_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *argsbuf[4]; PyObject * const *fastargs; Py_ssize_t nargs = PyTuple_GET_SIZE(args); - Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 2; + Py_ssize_t noptargs = nargs + (kwargs ? PyDict_GET_SIZE(kwargs) : 0) - 1; PyObject *value; - PyObject *expression; + PyObject *expression = &_Py_STR(empty); PyObject *conversion = Py_None; PyObject *format_spec = &_Py_STR(empty); fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, - /*minpos*/ 2, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + /*minpos*/ 1, /*maxpos*/ 4, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } value = fastargs[0]; - if (!PyUnicode_Check(fastargs[1])) { - _PyArg_BadArgument("Interpolation", "argument 'expression'", "str", fastargs[1]); - goto exit; - } - expression = fastargs[1]; if (!noptargs) { goto skip_optional_pos; } + if (fastargs[1]) { + if (!PyUnicode_Check(fastargs[1])) { + _PyArg_BadArgument("Interpolation", "argument 'expression'", "str", fastargs[1]); + goto exit; + } + expression = fastargs[1]; + if (!--noptargs) { + goto skip_optional_pos; + } + } if (fastargs[2]) { if (!_conversion_converter(fastargs[2], &conversion)) { goto exit; @@ -86,4 +91,4 @@ interpolation_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=599742a5ccd6f060 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2391391e2d7708c0 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/listobject.c.h b/Objects/clinic/listobject.c.h index 26ba5b954336da5..f3821ef5f70c21c 100644 --- a/Objects/clinic/listobject.c.h +++ b/Objects/clinic/listobject.c.h @@ -200,11 +200,11 @@ PyDoc_STRVAR(list_sort__doc__, "\n" "Sort the list in ascending order and return None.\n" "\n" -"The sort is in-place (i.e. the list itself is modified) and stable (i.e. the\n" -"order of two equal elements is maintained).\n" +"The sort is in-place (i.e. the list itself is modified) and stable\n" +"(i.e. the order of two equal elements is maintained).\n" "\n" -"If a key function is given, apply it once to each list item and sort them,\n" -"ascending or descending, according to their function values.\n" +"If a key function is given, apply it once to each list item and sort\n" +"them, ascending or descending, according to their function values.\n" "\n" "The reverse flag can be set to sort in descending order."); @@ -468,4 +468,4 @@ list___reversed__(PyObject *self, PyObject *Py_UNUSED(ignored)) { return list___reversed___impl((PyListObject *)self); } -/*[clinic end generated code: output=ae13fc2b56dc27c2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=06c21b0bffbe8d84 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/longobject.c.h b/Objects/clinic/longobject.c.h index a236a32c091c4c6..cfdcfd77ab1760c 100644 --- a/Objects/clinic/longobject.c.h +++ b/Objects/clinic/longobject.c.h @@ -262,19 +262,20 @@ PyDoc_STRVAR(int_to_bytes__doc__, "Return an array of bytes representing an integer.\n" "\n" " length\n" -" Length of bytes object to use. An OverflowError is raised if the\n" -" integer is not representable with the given number of bytes. Default\n" -" is length 1.\n" +" Length of bytes object to use. An OverflowError is raised if\n" +" the integer is not representable with the given number of bytes.\n" +" Default is length 1.\n" " byteorder\n" -" The byte order used to represent the integer. If byteorder is \'big\',\n" -" the most significant byte is at the beginning of the byte array. If\n" -" byteorder is \'little\', the most significant byte is at the end of the\n" -" byte array. To request the native byte order of the host system, use\n" -" sys.byteorder as the byte order value. Default is to use \'big\'.\n" +" The byte order used to represent the integer. If byteorder is\n" +" \'big\', the most significant byte is at the beginning of the byte\n" +" array. If byteorder is \'little\', the most significant byte is at\n" +" the end of the byte array. To request the native byte order of\n" +" the host system, use sys.byteorder as the byte order value.\n" +" Default is to use \'big\'.\n" " signed\n" -" Determines whether two\'s complement is used to represent the integer.\n" -" If signed is False and a negative integer is given, an OverflowError\n" -" is raised."); +" Determines whether two\'s complement is used to represent the\n" +" integer. If signed is False and a negative integer is given,\n" +" an OverflowError is raised."); #define INT_TO_BYTES_METHODDEF \ {"to_bytes", _PyCFunction_CAST(int_to_bytes), METH_FASTCALL|METH_KEYWORDS, int_to_bytes__doc__}, @@ -378,17 +379,19 @@ PyDoc_STRVAR(int_from_bytes__doc__, "\n" " bytes\n" " Holds the array of bytes to convert. The argument must either\n" -" support the buffer protocol or be an iterable object producing bytes.\n" -" Bytes and bytearray are examples of built-in objects that support the\n" -" buffer protocol.\n" +" support the buffer protocol or be an iterable object producing\n" +" bytes. Bytes and bytearray are examples of built-in objects that\n" +" support the buffer protocol.\n" " byteorder\n" -" The byte order used to represent the integer. If byteorder is \'big\',\n" -" the most significant byte is at the beginning of the byte array. If\n" -" byteorder is \'little\', the most significant byte is at the end of the\n" -" byte array. To request the native byte order of the host system, use\n" -" sys.byteorder as the byte order value. Default is to use \'big\'.\n" +" The byte order used to represent the integer. If byteorder is\n" +" \'big\', the most significant byte is at the beginning of the byte\n" +" array. If byteorder is \'little\', the most significant byte is at\n" +" the end of the byte array. To request the native byte order of\n" +" the host system, use sys.byteorder as the byte order value.\n" +" Default is to use \'big\'.\n" " signed\n" -" Indicates whether two\'s complement is used to represent the integer."); +" Indicates whether two\'s complement is used to represent the\n" +" integer."); #define INT_FROM_BYTES_METHODDEF \ {"from_bytes", _PyCFunction_CAST(int_from_bytes), METH_FASTCALL|METH_KEYWORDS|METH_CLASS, int_from_bytes__doc__}, @@ -485,4 +488,4 @@ int_is_integer(PyObject *self, PyObject *Py_UNUSED(ignored)) { return int_is_integer_impl(self); } -/*[clinic end generated code: output=d23f8ce5bdf08a30 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=93d4c37f650e1a65 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/memoryobject.c.h b/Objects/clinic/memoryobject.c.h index 28cfd1a22080c9b..f2e66563722e3f8 100644 --- a/Objects/clinic/memoryobject.c.h +++ b/Objects/clinic/memoryobject.c.h @@ -258,11 +258,12 @@ PyDoc_STRVAR(memoryview_tobytes__doc__, "\n" "Return the data in the buffer as a byte string.\n" "\n" -"Order can be {\'C\', \'F\', \'A\'}. When order is \'C\' or \'F\', the data of the\n" -"original array is converted to C or Fortran order. For contiguous views,\n" -"\'A\' returns an exact copy of the physical memory. In particular, in-memory\n" -"Fortran order is preserved. For non-contiguous views, the data is converted\n" -"to C first. order=None is the same as order=\'C\'."); +"Order can be {\'C\', \'F\', \'A\'}. When order is \'C\' or \'F\', the data of\n" +"the original array is converted to C or Fortran order. For\n" +"contiguous views, \'A\' returns an exact copy of the physical memory.\n" +"In particular, in-memory Fortran order is preserved. For\n" +"non-contiguous views, the data is converted to C first. order=None\n" +"is the same as order=\'C\'."); #define MEMORYVIEW_TOBYTES_METHODDEF \ {"tobytes", _PyCFunction_CAST(memoryview_tobytes), METH_FASTCALL|METH_KEYWORDS, memoryview_tobytes__doc__}, @@ -347,8 +348,8 @@ PyDoc_STRVAR(memoryview_hex__doc__, " sep\n" " An optional single character or byte to separate hex bytes.\n" " bytes_per_sep\n" -" How many bytes between separators. Positive values count from the\n" -" right, negative values count from the left.\n" +" How many bytes between separators. Positive values count from\n" +" the right, negative values count from the left.\n" "\n" "Example:\n" ">>> value = memoryview(b\'\\xb9\\x01\\xef\')\n" @@ -496,4 +497,4 @@ memoryview_index(PyObject *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=154f4c04263ccb24 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6d0886bee65a8f9f input=a9049054013a1b77]*/ diff --git a/Objects/clinic/odictobject.c.h b/Objects/clinic/odictobject.c.h index e71c29a1b268d05..92129e6444810f4 100644 --- a/Objects/clinic/odictobject.c.h +++ b/Objects/clinic/odictobject.c.h @@ -6,6 +6,7 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif +#include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(OrderedDict_fromkeys__doc__, @@ -73,6 +74,53 @@ OrderedDict_fromkeys(PyObject *type, PyObject *const *args, Py_ssize_t nargs, Py return return_value; } +PyDoc_STRVAR(OrderedDict___sizeof____doc__, +"__sizeof__($self, /)\n" +"--\n" +"\n"); + +#define ORDEREDDICT___SIZEOF___METHODDEF \ + {"__sizeof__", (PyCFunction)OrderedDict___sizeof__, METH_NOARGS, OrderedDict___sizeof____doc__}, + +static Py_ssize_t +OrderedDict___sizeof___impl(PyODictObject *self); + +static PyObject * +OrderedDict___sizeof__(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + Py_ssize_t _return_value; + + Py_BEGIN_CRITICAL_SECTION(self); + _return_value = OrderedDict___sizeof___impl((PyODictObject *)self); + Py_END_CRITICAL_SECTION(); + if ((_return_value == -1) && PyErr_Occurred()) { + goto exit; + } + return_value = PyLong_FromSsize_t(_return_value); + +exit: + return return_value; +} + +PyDoc_STRVAR(OrderedDict___reduce____doc__, +"__reduce__($self, /)\n" +"--\n" +"\n" +"Return state information for pickling"); + +#define ORDEREDDICT___REDUCE___METHODDEF \ + {"__reduce__", (PyCFunction)OrderedDict___reduce__, METH_NOARGS, OrderedDict___reduce____doc__}, + +static PyObject * +OrderedDict___reduce___impl(PyODictObject *od); + +static PyObject * +OrderedDict___reduce__(PyObject *od, PyObject *Py_UNUSED(ignored)) +{ + return OrderedDict___reduce___impl((PyODictObject *)od); +} + PyDoc_STRVAR(OrderedDict_setdefault__doc__, "setdefault($self, /, key, default=None)\n" "--\n" @@ -135,7 +183,9 @@ OrderedDict_setdefault(PyObject *self, PyObject *const *args, Py_ssize_t nargs, } default_value = args[1]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = OrderedDict_setdefault_impl((PyODictObject *)self, key, default_value); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -204,7 +254,9 @@ OrderedDict_pop(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyObjec } default_value = args[1]; skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = OrderedDict_pop_impl((PyODictObject *)self, key, default_value); + Py_END_CRITICAL_SECTION(); exit: return return_value; @@ -216,7 +268,8 @@ PyDoc_STRVAR(OrderedDict_popitem__doc__, "\n" "Remove and return a (key, value) pair from the dictionary.\n" "\n" -"Pairs are returned in LIFO order if last is true or FIFO order if false."); +"Pairs are returned in LIFO order if last is true or FIFO order if\n" +"false."); #define ORDEREDDICT_POPITEM_METHODDEF \ {"popitem", _PyCFunction_CAST(OrderedDict_popitem), METH_FASTCALL|METH_KEYWORDS, OrderedDict_popitem__doc__}, @@ -272,12 +325,62 @@ OrderedDict_popitem(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyO goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = OrderedDict_popitem_impl((PyODictObject *)self, last); + Py_END_CRITICAL_SECTION(); exit: return return_value; } +PyDoc_STRVAR(OrderedDict_clear__doc__, +"clear($self, /)\n" +"--\n" +"\n" +"Remove all items from ordered dict."); + +#define ORDEREDDICT_CLEAR_METHODDEF \ + {"clear", (PyCFunction)OrderedDict_clear, METH_NOARGS, OrderedDict_clear__doc__}, + +static PyObject * +OrderedDict_clear_impl(PyODictObject *self); + +static PyObject * +OrderedDict_clear(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(self); + return_value = OrderedDict_clear_impl((PyODictObject *)self); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + +PyDoc_STRVAR(OrderedDict_copy__doc__, +"copy($self, /)\n" +"--\n" +"\n" +"A shallow copy of ordered dict."); + +#define ORDEREDDICT_COPY_METHODDEF \ + {"copy", (PyCFunction)OrderedDict_copy, METH_NOARGS, OrderedDict_copy__doc__}, + +static PyObject * +OrderedDict_copy_impl(PyObject *od); + +static PyObject * +OrderedDict_copy(PyObject *od, PyObject *Py_UNUSED(ignored)) +{ + PyObject *return_value = NULL; + + Py_BEGIN_CRITICAL_SECTION(od); + return_value = OrderedDict_copy_impl(od); + Py_END_CRITICAL_SECTION(); + + return return_value; +} + PyDoc_STRVAR(OrderedDict_move_to_end__doc__, "move_to_end($self, /, key, last=True)\n" "--\n" @@ -342,9 +445,11 @@ OrderedDict_move_to_end(PyObject *self, PyObject *const *args, Py_ssize_t nargs, goto exit; } skip_optional_pos: + Py_BEGIN_CRITICAL_SECTION(self); return_value = OrderedDict_move_to_end_impl((PyODictObject *)self, key, last); + Py_END_CRITICAL_SECTION(); exit: return return_value; } -/*[clinic end generated code: output=7d8206823bb1f419 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=89f7e92de998f9a4 input=a9049054013a1b77]*/ diff --git a/Objects/clinic/unicodeobject.c.h b/Objects/clinic/unicodeobject.c.h index 1819fbaea220a32..0e06406cf8369d1 100644 --- a/Objects/clinic/unicodeobject.c.h +++ b/Objects/clinic/unicodeobject.c.h @@ -33,8 +33,8 @@ PyDoc_STRVAR(unicode_title__doc__, "\n" "Return a version of the string where each word is titlecased.\n" "\n" -"More specifically, words start with uppercased characters and all remaining\n" -"cased characters have lower case."); +"More specifically, words start with uppercased characters and all\n" +"remaining cased characters have lower case."); #define UNICODE_TITLE_METHODDEF \ {"title", (PyCFunction)unicode_title, METH_NOARGS, unicode_title__doc__}, @@ -54,8 +54,8 @@ PyDoc_STRVAR(unicode_capitalize__doc__, "\n" "Return a capitalized version of the string.\n" "\n" -"More specifically, make the first character have upper case and the rest lower\n" -"case."); +"More specifically, make the first character have upper case and the\n" +"rest lower case."); #define UNICODE_CAPITALIZE_METHODDEF \ {"capitalize", (PyCFunction)unicode_capitalize, METH_NOARGS, unicode_capitalize__doc__}, @@ -93,7 +93,8 @@ PyDoc_STRVAR(unicode_center__doc__, "\n" "Return a centered string of length width.\n" "\n" -"Padding is done using the specified fill character (default is a space)."); +"Padding is done using the specified fill character (default is\n" +"a space)."); #define UNICODE_CENTER_METHODDEF \ {"center", _PyCFunction_CAST(unicode_center), METH_FASTCALL, unicode_center__doc__}, @@ -142,7 +143,8 @@ PyDoc_STRVAR(unicode_count__doc__, "\n" "Return the number of non-overlapping occurrences of substring sub in string S[start:end].\n" "\n" -"Optional arguments start and end are interpreted as in slice notation."); +"Optional arguments start and end are interpreted as in slice\n" +"notation."); #define UNICODE_COUNT_METHODDEF \ {"count", _PyCFunction_CAST(unicode_count), METH_FASTCALL, unicode_count__doc__}, @@ -202,8 +204,8 @@ PyDoc_STRVAR(unicode_encode__doc__, " errors\n" " The error handling scheme to use for encoding errors.\n" " The default is \'strict\' meaning that encoding errors raise a\n" -" UnicodeEncodeError. Other possible values are \'ignore\', \'replace\' and\n" -" \'xmlcharrefreplace\' as well as any other name registered with\n" +" UnicodeEncodeError. Other possible values are \'ignore\', \'replace\'\n" +" and \'xmlcharrefreplace\' as well as any other name registered with\n" " codecs.register_error that can handle UnicodeEncodeErrors."); #define UNICODE_ENCODE_METHODDEF \ @@ -368,8 +370,8 @@ PyDoc_STRVAR(unicode_find__doc__, "\n" "Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end].\n" "\n" -"Optional arguments start and end are interpreted as in slice notation.\n" -"Return -1 on failure."); +"Optional arguments start and end are interpreted as in slice\n" +"notation. Return -1 on failure."); #define UNICODE_FIND_METHODDEF \ {"find", _PyCFunction_CAST(unicode_find), METH_FASTCALL, unicode_find__doc__}, @@ -424,8 +426,8 @@ PyDoc_STRVAR(unicode_index__doc__, "\n" "Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end].\n" "\n" -"Optional arguments start and end are interpreted as in slice notation.\n" -"Raises ValueError when the substring is not found."); +"Optional arguments start and end are interpreted as in slice\n" +"notation. Raises ValueError when the substring is not found."); #define UNICODE_INDEX_METHODDEF \ {"index", _PyCFunction_CAST(unicode_index), METH_FASTCALL, unicode_index__doc__}, @@ -501,8 +503,8 @@ PyDoc_STRVAR(unicode_islower__doc__, "\n" "Return True if the string is a lowercase string, False otherwise.\n" "\n" -"A string is lowercase if all cased characters in the string are lowercase and\n" -"there is at least one cased character in the string."); +"A string is lowercase if all cased characters in the string are\n" +"lowercase and there is at least one cased character in the string."); #define UNICODE_ISLOWER_METHODDEF \ {"islower", (PyCFunction)unicode_islower, METH_NOARGS, unicode_islower__doc__}, @@ -522,8 +524,8 @@ PyDoc_STRVAR(unicode_isupper__doc__, "\n" "Return True if the string is an uppercase string, False otherwise.\n" "\n" -"A string is uppercase if all cased characters in the string are uppercase and\n" -"there is at least one cased character in the string."); +"A string is uppercase if all cased characters in the string are\n" +"uppercase and there is at least one cased character in the string."); #define UNICODE_ISUPPER_METHODDEF \ {"isupper", (PyCFunction)unicode_isupper, METH_NOARGS, unicode_isupper__doc__}, @@ -564,8 +566,8 @@ PyDoc_STRVAR(unicode_isspace__doc__, "\n" "Return True if the string is a whitespace string, False otherwise.\n" "\n" -"A string is whitespace if all characters in the string are whitespace and there\n" -"is at least one character in the string."); +"A string is whitespace if all characters in the string are\n" +"whitespace and there is at least one character in the string."); #define UNICODE_ISSPACE_METHODDEF \ {"isspace", (PyCFunction)unicode_isspace, METH_NOARGS, unicode_isspace__doc__}, @@ -585,8 +587,8 @@ PyDoc_STRVAR(unicode_isalpha__doc__, "\n" "Return True if the string is an alphabetic string, False otherwise.\n" "\n" -"A string is alphabetic if all characters in the string are alphabetic and there\n" -"is at least one character in the string."); +"A string is alphabetic if all characters in the string are\n" +"alphabetic and there is at least one character in the string."); #define UNICODE_ISALPHA_METHODDEF \ {"isalpha", (PyCFunction)unicode_isalpha, METH_NOARGS, unicode_isalpha__doc__}, @@ -606,8 +608,8 @@ PyDoc_STRVAR(unicode_isalnum__doc__, "\n" "Return True if the string is an alpha-numeric string, False otherwise.\n" "\n" -"A string is alpha-numeric if all characters in the string are alpha-numeric and\n" -"there is at least one character in the string."); +"A string is alpha-numeric if all characters in the string are\n" +"alpha-numeric and there is at least one character in the string."); #define UNICODE_ISALNUM_METHODDEF \ {"isalnum", (PyCFunction)unicode_isalnum, METH_NOARGS, unicode_isalnum__doc__}, @@ -627,8 +629,8 @@ PyDoc_STRVAR(unicode_isdecimal__doc__, "\n" "Return True if the string is a decimal string, False otherwise.\n" "\n" -"A string is a decimal string if all characters in the string are decimal and\n" -"there is at least one character in the string."); +"A string is a decimal string if all characters in the string are\n" +"decimal and there is at least one character in the string."); #define UNICODE_ISDECIMAL_METHODDEF \ {"isdecimal", (PyCFunction)unicode_isdecimal, METH_NOARGS, unicode_isdecimal__doc__}, @@ -648,8 +650,8 @@ PyDoc_STRVAR(unicode_isdigit__doc__, "\n" "Return True if the string is a digit string, False otherwise.\n" "\n" -"A string is a digit string if all characters in the string are digits and there\n" -"is at least one character in the string."); +"A string is a digit string if all characters in the string are\n" +"digits and there is at least one character in the string."); #define UNICODE_ISDIGIT_METHODDEF \ {"isdigit", (PyCFunction)unicode_isdigit, METH_NOARGS, unicode_isdigit__doc__}, @@ -669,8 +671,8 @@ PyDoc_STRVAR(unicode_isnumeric__doc__, "\n" "Return True if the string is a numeric string, False otherwise.\n" "\n" -"A string is numeric if all characters in the string are numeric and there is at\n" -"least one character in the string."); +"A string is numeric if all characters in the string are numeric and\n" +"there is at least one character in the string."); #define UNICODE_ISNUMERIC_METHODDEF \ {"isnumeric", (PyCFunction)unicode_isnumeric, METH_NOARGS, unicode_isnumeric__doc__}, @@ -690,8 +692,8 @@ PyDoc_STRVAR(unicode_isidentifier__doc__, "\n" "Return True if the string is a valid Python identifier, False otherwise.\n" "\n" -"Call keyword.iskeyword(s) to test whether string s is a reserved identifier,\n" -"such as \"def\" or \"class\"."); +"Call keyword.iskeyword(s) to test whether string s is a reserved\n" +"identifier, such as \"def\" or \"class\"."); #define UNICODE_ISIDENTIFIER_METHODDEF \ {"isidentifier", (PyCFunction)unicode_isidentifier, METH_NOARGS, unicode_isidentifier__doc__}, @@ -731,8 +733,8 @@ PyDoc_STRVAR(unicode_join__doc__, "\n" "Concatenate any number of strings.\n" "\n" -"The string whose method is called is inserted in between each given string.\n" -"The result is returned as a new string.\n" +"The string whose method is called is inserted in between each given\n" +"string. The result is returned as a new string.\n" "\n" "Example: \'.\'.join([\'ab\', \'pq\', \'rs\']) -> \'ab.pq.rs\'"); @@ -745,7 +747,8 @@ PyDoc_STRVAR(unicode_ljust__doc__, "\n" "Return a left-justified string of length width.\n" "\n" -"Padding is done using the specified fill character (default is a space)."); +"Padding is done using the specified fill character (default is\n" +"a space)."); #define UNICODE_LJUST_METHODDEF \ {"ljust", _PyCFunction_CAST(unicode_ljust), METH_FASTCALL, unicode_ljust__doc__}, @@ -1008,8 +1011,9 @@ PyDoc_STRVAR(unicode_removeprefix__doc__, "\n" "Return a str with the given prefix string removed if present.\n" "\n" -"If the string starts with the prefix string, return string[len(prefix):].\n" -"Otherwise, return a copy of the original string."); +"If the string starts with the prefix string, return\n" +"string[len(prefix):]. Otherwise, return a copy of the original\n" +"string."); #define UNICODE_REMOVEPREFIX_METHODDEF \ {"removeprefix", (PyCFunction)unicode_removeprefix, METH_O, unicode_removeprefix__doc__}, @@ -1040,9 +1044,9 @@ PyDoc_STRVAR(unicode_removesuffix__doc__, "\n" "Return a str with the given suffix string removed if present.\n" "\n" -"If the string ends with the suffix string and that suffix is not empty,\n" -"return string[:-len(suffix)]. Otherwise, return a copy of the original\n" -"string."); +"If the string ends with the suffix string and that suffix is not\n" +"empty, return string[:-len(suffix)]. Otherwise, return a copy of\n" +"the original string."); #define UNICODE_REMOVESUFFIX_METHODDEF \ {"removesuffix", (PyCFunction)unicode_removesuffix, METH_O, unicode_removesuffix__doc__}, @@ -1073,8 +1077,8 @@ PyDoc_STRVAR(unicode_rfind__doc__, "\n" "Return the highest index in S where substring sub is found, such that sub is contained within S[start:end].\n" "\n" -"Optional arguments start and end are interpreted as in slice notation.\n" -"Return -1 on failure."); +"Optional arguments start and end are interpreted as in slice\n" +"notation. Return -1 on failure."); #define UNICODE_RFIND_METHODDEF \ {"rfind", _PyCFunction_CAST(unicode_rfind), METH_FASTCALL, unicode_rfind__doc__}, @@ -1129,8 +1133,8 @@ PyDoc_STRVAR(unicode_rindex__doc__, "\n" "Return the highest index in S where substring sub is found, such that sub is contained within S[start:end].\n" "\n" -"Optional arguments start and end are interpreted as in slice notation.\n" -"Raises ValueError when the substring is not found."); +"Optional arguments start and end are interpreted as in slice\n" +"notation. Raises ValueError when the substring is not found."); #define UNICODE_RINDEX_METHODDEF \ {"rindex", _PyCFunction_CAST(unicode_rindex), METH_FASTCALL, unicode_rindex__doc__}, @@ -1185,7 +1189,8 @@ PyDoc_STRVAR(unicode_rjust__doc__, "\n" "Return a right-justified string of length width.\n" "\n" -"Padding is done using the specified fill character (default is a space)."); +"Padding is done using the specified fill character (default is\n" +"a space)."); #define UNICODE_RJUST_METHODDEF \ {"rjust", _PyCFunction_CAST(unicode_rjust), METH_FASTCALL, unicode_rjust__doc__}, @@ -1237,18 +1242,18 @@ PyDoc_STRVAR(unicode_split__doc__, " sep\n" " The separator used to split the string.\n" "\n" -" When set to None (the default value), will split on any whitespace\n" -" character (including \\n \\r \\t \\f and spaces) and will discard\n" -" empty strings from the result.\n" +" When set to None (the default value), will split on any\n" +" whitespace character (including \\n \\r \\t \\f and spaces) and\n" +" will discard empty strings from the result.\n" " maxsplit\n" " Maximum number of splits.\n" " -1 (the default value) means no limit.\n" "\n" "Splitting starts at the front of the string and works to the end.\n" "\n" -"Note, str.split() is mainly useful for data that has been intentionally\n" -"delimited. With natural text that includes punctuation, consider using\n" -"the regular expression module."); +"Note, str.split() is mainly useful for data that has been\n" +"intentionally delimited. With natural text that includes\n" +"punctuation, consider using the regular expression module."); #define UNICODE_SPLIT_METHODDEF \ {"split", _PyCFunction_CAST(unicode_split), METH_FASTCALL|METH_KEYWORDS, unicode_split__doc__}, @@ -1331,12 +1336,12 @@ PyDoc_STRVAR(unicode_partition__doc__, "\n" "Partition the string into three parts using the given separator.\n" "\n" -"This will search for the separator in the string. If the separator is found,\n" -"returns a 3-tuple containing the part before the separator, the separator\n" -"itself, and the part after it.\n" +"This will search for the separator in the string. If the separator\n" +"is found, returns a 3-tuple containing the part before the\n" +"separator, the separator itself, and the part after it.\n" "\n" -"If the separator is not found, returns a 3-tuple containing the original string\n" -"and two empty strings."); +"If the separator is not found, returns a 3-tuple containing\n" +"the original string and two empty strings."); #define UNICODE_PARTITION_METHODDEF \ {"partition", (PyCFunction)unicode_partition, METH_O, unicode_partition__doc__}, @@ -1347,12 +1352,13 @@ PyDoc_STRVAR(unicode_rpartition__doc__, "\n" "Partition the string into three parts using the given separator.\n" "\n" -"This will search for the separator in the string, starting at the end. If\n" -"the separator is found, returns a 3-tuple containing the part before the\n" -"separator, the separator itself, and the part after it.\n" +"This will search for the separator in the string, starting at the\n" +"end. If the separator is found, returns a 3-tuple containing the\n" +"part before the separator, the separator itself, and the part after\n" +"it.\n" "\n" -"If the separator is not found, returns a 3-tuple containing two empty strings\n" -"and the original string."); +"If the separator is not found, returns a 3-tuple containing two\n" +"empty strings and the original string."); #define UNICODE_RPARTITION_METHODDEF \ {"rpartition", (PyCFunction)unicode_rpartition, METH_O, unicode_rpartition__doc__}, @@ -1366,9 +1372,9 @@ PyDoc_STRVAR(unicode_rsplit__doc__, " sep\n" " The separator used to split the string.\n" "\n" -" When set to None (the default value), will split on any whitespace\n" -" character (including \\n \\r \\t \\f and spaces) and will discard\n" -" empty strings from the result.\n" +" When set to None (the default value), will split on any\n" +" whitespace character (including \\n \\r \\t \\f and spaces) and\n" +" will discard empty strings from the result.\n" " maxsplit\n" " Maximum number of splits.\n" " -1 (the default value) means no limit.\n" @@ -1456,8 +1462,8 @@ PyDoc_STRVAR(unicode_splitlines__doc__, "\n" "Return a list of the lines in the string, breaking at line boundaries.\n" "\n" -"Line breaks are not included in the resulting list unless keepends is given and\n" -"true."); +"Line breaks are not included in the resulting list unless keepends\n" +"is given and true."); #define UNICODE_SPLITLINES_METHODDEF \ {"splitlines", _PyCFunction_CAST(unicode_splitlines), METH_FASTCALL|METH_KEYWORDS, unicode_splitlines__doc__}, @@ -1543,13 +1549,14 @@ PyDoc_STRVAR(unicode_maketrans__doc__, "\n" "Return a translation table usable for str.translate().\n" "\n" -"If there is only one argument, it must be a dictionary mapping Unicode\n" -"ordinals (integers) or characters to Unicode ordinals, strings or None.\n" -"Character keys will be then converted to ordinals.\n" -"If there are two arguments, they must be strings of equal length, and\n" -"in the resulting dictionary, each character in x will be mapped to the\n" -"character at the same position in y. If there is a third argument, it\n" -"must be a string, whose characters will be mapped to None in the result."); +"If there is only one argument, it must be a dictionary mapping\n" +"Unicode ordinals (integers) or characters to Unicode ordinals,\n" +"strings or None. Character keys will be then converted to ordinals.\n" +"If there are two arguments, they must be strings of equal length,\n" +"and in the resulting dictionary, each character in x will be mapped\n" +"to the character at the same position in y. If there is a third\n" +"argument, it must be a string, whose characters will be mapped to\n" +"None in the result."); #define UNICODE_MAKETRANS_METHODDEF \ {"maketrans", _PyCFunction_CAST(unicode_maketrans), METH_FASTCALL|METH_STATIC, unicode_maketrans__doc__}, @@ -1599,12 +1606,13 @@ PyDoc_STRVAR(unicode_translate__doc__, "Replace each character in the string using the given translation table.\n" "\n" " table\n" -" Translation table, which must be a mapping of Unicode ordinals to\n" -" Unicode ordinals, strings, or None.\n" +" Translation table, which must be a mapping of Unicode ordinals\n" +" to Unicode ordinals, strings, or None.\n" "\n" -"The table must implement lookup/indexing via __getitem__, for instance a\n" -"dictionary or list. If this operation raises LookupError, the character is\n" -"left untouched. Characters mapped to None are deleted."); +"The table must implement lookup/indexing via __getitem__, for\n" +"instance a dictionary or list. If this operation raises\n" +"LookupError, the character is left untouched. Characters mapped to\n" +"None are deleted."); #define UNICODE_TRANSLATE_METHODDEF \ {"translate", (PyCFunction)unicode_translate, METH_O, unicode_translate__doc__}, @@ -1908,4 +1916,4 @@ unicode_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) exit: return return_value; } -/*[clinic end generated code: output=238917fe66120bde input=a9049054013a1b77]*/ +/*[clinic end generated code: output=6c3a5d1565fad1d6 input=a9049054013a1b77]*/ diff --git a/Objects/codeobject.c b/Objects/codeobject.c index 92eaf5e00bc7dd6..ff85b11ecbc23f1 100644 --- a/Objects/codeobject.c +++ b/Objects/codeobject.c @@ -17,6 +17,7 @@ #include "pycore_tuple.h" // _PyTuple_ITEMS() #include "pycore_unicodeobject.h" // _PyUnicode_InternImmortal() #include "pycore_uniqueid.h" // _PyObject_AssignUniqueId() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "clinic/codeobject.c.h" #include <stdbool.h> @@ -744,6 +745,10 @@ _PyCode_New(struct _PyCodeConstructor *con) return NULL; } +#ifdef Py_GIL_DISABLED + co->_co_unique_id = _Py_INVALID_UNIQUE_ID; +#endif + if (init_code(co, con) < 0) { Py_DECREF(co); return NULL; @@ -1018,8 +1023,12 @@ PyCode_Addr2Line(PyCodeObject *co, int addrq) if (addrq < 0) { return co->co_firstlineno; } - if (co->_co_monitoring && co->_co_monitoring->lines) { - return _Py_Instrumentation_GetLine(co, addrq/sizeof(_Py_CODEUNIT)); + _PyCoMonitoringData *data = _Py_atomic_load_ptr_acquire(&co->_co_monitoring); + if (data) { + _PyCoLineInstrumentationData *lines = _Py_atomic_load_ptr_acquire(&data->lines); + if (lines) { + return _Py_Instrumentation_GetLine(co, lines, addrq/sizeof(_Py_CODEUNIT)); + } } assert(addrq >= 0 && addrq < _PyCode_NBYTES(co)); PyCodeAddressRange bounds; @@ -1027,6 +1036,23 @@ PyCode_Addr2Line(PyCodeObject *co, int addrq) return _PyCode_CheckLineNumber(addrq, &bounds); } +int +_PyCode_SafeAddr2Line(PyCodeObject *co, int addrq) +{ + if (addrq < 0) { + return co->co_firstlineno; + } + if (co->_co_monitoring && co->_co_monitoring->lines) { + return _Py_Instrumentation_GetLine(co, co->_co_monitoring->lines, addrq/sizeof(_Py_CODEUNIT)); + } + if (!(addrq >= 0 && addrq < _PyCode_NBYTES(co))) { + return -1; + } + PyCodeAddressRange bounds; + _PyCode_InitAddressRange(co, &bounds); + return _PyCode_CheckLineNumber(addrq, &bounds); +} + void _PyLineTable_InitAddressRange(const char *linetable, Py_ssize_t length, int firstlineno, PyCodeAddressRange *range) { @@ -1554,6 +1580,67 @@ typedef struct { } _PyCodeObjectExtra; +static inline size_t +code_extra_size(Py_ssize_t n) +{ + return sizeof(_PyCodeObjectExtra) + (n - 1) * sizeof(void *); +} + +#ifdef Py_GIL_DISABLED +static int +code_extra_grow_ft(PyCodeObject *co, _PyCodeObjectExtra *old_co_extra, + Py_ssize_t old_ce_size, Py_ssize_t new_ce_size, + Py_ssize_t index, void *extra) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(co); + _PyCodeObjectExtra *new_co_extra = PyMem_Malloc( + code_extra_size(new_ce_size)); + if (new_co_extra == NULL) { + PyErr_NoMemory(); + return -1; + } + + if (old_ce_size > 0) { + memcpy(new_co_extra->ce_extras, old_co_extra->ce_extras, + old_ce_size * sizeof(void *)); + } + for (Py_ssize_t i = old_ce_size; i < new_ce_size; i++) { + new_co_extra->ce_extras[i] = NULL; + } + new_co_extra->ce_size = new_ce_size; + new_co_extra->ce_extras[index] = extra; + + // Publish new buffer and its contents to lock-free readers. + FT_ATOMIC_STORE_PTR_RELEASE(co->co_extra, new_co_extra); + if (old_co_extra != NULL) { + // QSBR: defer old-buffer free until lock-free readers quiesce. + _PyMem_FreeDelayed(old_co_extra, code_extra_size(old_ce_size)); + } + return 0; +} +#else +static int +code_extra_grow_gil(PyCodeObject *co, _PyCodeObjectExtra *old_co_extra, + Py_ssize_t old_ce_size, Py_ssize_t new_ce_size, + Py_ssize_t index, void *extra) +{ + _PyCodeObjectExtra *new_co_extra = PyMem_Realloc( + old_co_extra, code_extra_size(new_ce_size)); + if (new_co_extra == NULL) { + PyErr_NoMemory(); + return -1; + } + + for (Py_ssize_t i = old_ce_size; i < new_ce_size; i++) { + new_co_extra->ce_extras[i] = NULL; + } + new_co_extra->ce_size = new_ce_size; + new_co_extra->ce_extras[index] = extra; + co->co_extra = new_co_extra; + return 0; +} +#endif + int PyUnstable_Code_GetExtra(PyObject *code, Py_ssize_t index, void **extra) { @@ -1562,15 +1649,19 @@ PyUnstable_Code_GetExtra(PyObject *code, Py_ssize_t index, void **extra) return -1; } - PyCodeObject *o = (PyCodeObject*) code; - _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra*) o->co_extra; + PyCodeObject *co = (PyCodeObject *)code; + *extra = NULL; - if (co_extra == NULL || index < 0 || co_extra->ce_size <= index) { - *extra = NULL; + if (index < 0) { return 0; } - *extra = co_extra->ce_extras[index]; + // Lock-free read; pairs with release stores in SetExtra. + _PyCodeObjectExtra *co_extra = FT_ATOMIC_LOAD_PTR_ACQUIRE(co->co_extra); + if (co_extra != NULL && index < co_extra->ce_size) { + *extra = FT_ATOMIC_LOAD_PTR_ACQUIRE(co_extra->ce_extras[index]); + } + return 0; } @@ -1580,40 +1671,59 @@ PyUnstable_Code_SetExtra(PyObject *code, Py_ssize_t index, void *extra) { PyInterpreterState *interp = _PyInterpreterState_GET(); - if (!PyCode_Check(code) || index < 0 || - index >= interp->co_extra_user_count) { + // co_extra_user_count is monotonically increasing and published with + // release store in RequestCodeExtraIndex, so once an index is valid + // it stays valid. + Py_ssize_t user_count = FT_ATOMIC_LOAD_SSIZE_ACQUIRE( + interp->co_extra_user_count); + + if (!PyCode_Check(code) || index < 0 || index >= user_count) { PyErr_BadInternalCall(); return -1; } - PyCodeObject *o = (PyCodeObject*) code; - _PyCodeObjectExtra *co_extra = (_PyCodeObjectExtra *) o->co_extra; + PyCodeObject *co = (PyCodeObject *)code; + int result = 0; + void *old_slot_value = NULL; - if (co_extra == NULL || co_extra->ce_size <= index) { - Py_ssize_t i = (co_extra == NULL ? 0 : co_extra->ce_size); - co_extra = PyMem_Realloc( - co_extra, - sizeof(_PyCodeObjectExtra) + - (interp->co_extra_user_count-1) * sizeof(void*)); - if (co_extra == NULL) { - return -1; - } - for (; i < interp->co_extra_user_count; i++) { - co_extra->ce_extras[i] = NULL; - } - co_extra->ce_size = interp->co_extra_user_count; - o->co_extra = co_extra; + Py_BEGIN_CRITICAL_SECTION(co); + + _PyCodeObjectExtra *old_co_extra = (_PyCodeObjectExtra *)co->co_extra; + Py_ssize_t old_ce_size = (old_co_extra == NULL) + ? 0 : old_co_extra->ce_size; + + // Fast path: slot already exists, update in place. + if (index < old_ce_size) { + old_slot_value = old_co_extra->ce_extras[index]; + FT_ATOMIC_STORE_PTR_RELEASE(old_co_extra->ce_extras[index], extra); + goto done; } - if (co_extra->ce_extras[index] != NULL) { - freefunc free = interp->co_extra_freefuncs[index]; - if (free != NULL) { - free(co_extra->ce_extras[index]); + // Slow path: buffer needs to grow. + Py_ssize_t new_ce_size = user_count; +#ifdef Py_GIL_DISABLED + // FT build: allocate new buffer and swap; QSBR reclaims the old one. + result = code_extra_grow_ft( + co, old_co_extra, old_ce_size, new_ce_size, index, extra); +#else + // GIL build: grow with realloc. + result = code_extra_grow_gil( + co, old_co_extra, old_ce_size, new_ce_size, index, extra); +#endif + +done:; + Py_END_CRITICAL_SECTION(); + if (old_slot_value != NULL) { + // Free the old slot value if a free function was registered. + // The caller must ensure no other thread can still access the old + // value after this overwrite. + freefunc free_extra = interp->co_extra_freefuncs[index]; + if (free_extra != NULL) { + free_extra(old_slot_value); } } - co_extra->ce_extras[index] = extra; - return 0; + return result; } @@ -1714,7 +1824,7 @@ static int identify_unbound_names(PyThreadState *tstate, PyCodeObject *co, PyObject *globalnames, PyObject *attrnames, PyObject *globalsns, PyObject *builtinsns, - struct co_unbound_counts *counts) + struct co_unbound_counts *counts, int *p_numdupes) { // This function is inspired by inspect.getclosurevars(). // It would be nicer if we had something similar to co_localspluskinds, @@ -1729,6 +1839,7 @@ identify_unbound_names(PyThreadState *tstate, PyCodeObject *co, assert(builtinsns == NULL || PyDict_Check(builtinsns)); assert(counts == NULL || counts->total == 0); struct co_unbound_counts unbound = {0}; + int numdupes = 0; Py_ssize_t len = Py_SIZE(co); for (int i = 0; i < len; i += _PyInstruction_GetLength(co, i)) { _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(co, i); @@ -1747,6 +1858,12 @@ identify_unbound_names(PyThreadState *tstate, PyCodeObject *co, if (PySet_Add(attrnames, name) < 0) { return -1; } + if (PySet_Contains(globalnames, name)) { + if (_PyErr_Occurred(tstate)) { + return -1; + } + numdupes += 1; + } } else if (inst.op.code == LOAD_GLOBAL) { int oparg = GET_OPARG(co, i, inst.op.arg); @@ -1778,11 +1895,20 @@ identify_unbound_names(PyThreadState *tstate, PyCodeObject *co, if (PySet_Add(globalnames, name) < 0) { return -1; } + if (PySet_Contains(attrnames, name)) { + if (_PyErr_Occurred(tstate)) { + return -1; + } + numdupes += 1; + } } } if (counts != NULL) { *counts = unbound; } + if (p_numdupes != NULL) { + *p_numdupes = numdupes; + } return 0; } @@ -1932,20 +2058,24 @@ _PyCode_SetUnboundVarCounts(PyThreadState *tstate, // Fill in unbound.globals and unbound.numattrs. struct co_unbound_counts unbound = {0}; + int numdupes = 0; Py_BEGIN_CRITICAL_SECTION(co); res = identify_unbound_names( tstate, co, globalnames, attrnames, globalsns, builtinsns, - &unbound); + &unbound, &numdupes); Py_END_CRITICAL_SECTION(); if (res < 0) { goto finally; } assert(unbound.numunknown == 0); - assert(unbound.total <= counts->unbound.total); + assert(unbound.total - numdupes <= counts->unbound.total); assert(counts->unbound.numunknown == counts->unbound.total); - unbound.numunknown = counts->unbound.total - unbound.total; - unbound.total = counts->unbound.total; + // There may be a name that is both a global and an attr. + int totalunbound = counts->unbound.total + numdupes; + unbound.numunknown = totalunbound - unbound.total; + unbound.total = totalunbound; counts->unbound = unbound; + counts->total += numdupes; res = 0; finally: @@ -1955,12 +2085,129 @@ _PyCode_SetUnboundVarCounts(PyThreadState *tstate, } +int +_PyCode_CheckNoInternalState(PyCodeObject *co, const char **p_errmsg) +{ + const char *errmsg = NULL; + // We don't worry about co_executors, co_instrumentation, + // or co_monitoring. They are essentially ephemeral. + if (co->co_extra != NULL) { + errmsg = "only basic code objects are supported"; + } + + if (errmsg != NULL) { + if (p_errmsg != NULL) { + *p_errmsg = errmsg; + } + return 0; + } + return 1; +} + +int +_PyCode_CheckNoExternalState(PyCodeObject *co, _PyCode_var_counts_t *counts, + const char **p_errmsg) +{ + const char *errmsg = NULL; + if (counts->numfree > 0) { // It's a closure. + errmsg = "closures not supported"; + } + else if (counts->unbound.globals.numglobal > 0) { + errmsg = "globals not supported"; + } + else if (counts->unbound.globals.numbuiltin > 0 + && counts->unbound.globals.numunknown > 0) + { + errmsg = "globals not supported"; + } + // Otherwise we don't check counts.unbound.globals.numunknown since we can't + // distinguish beween globals and builtins here. + + if (errmsg != NULL) { + if (p_errmsg != NULL) { + *p_errmsg = errmsg; + } + return 0; + } + return 1; +} + +int +_PyCode_VerifyStateless(PyThreadState *tstate, + PyCodeObject *co, PyObject *globalnames, + PyObject *globalsns, PyObject *builtinsns) +{ + const char *errmsg; + _PyCode_var_counts_t counts = {0}; + _PyCode_GetVarCounts(co, &counts); + if (_PyCode_SetUnboundVarCounts( + tstate, co, &counts, globalnames, NULL, + globalsns, builtinsns) < 0) + { + return -1; + } + // We may consider relaxing the internal state constraints + // if it becomes a problem. + if (!_PyCode_CheckNoInternalState(co, &errmsg)) { + _PyErr_SetString(tstate, PyExc_ValueError, errmsg); + return -1; + } + if (builtinsns != NULL) { + // Make sure the next check will fail for globals, + // even if there aren't any builtins. + counts.unbound.globals.numbuiltin += 1; + } + if (!_PyCode_CheckNoExternalState(co, &counts, &errmsg)) { + _PyErr_SetString(tstate, PyExc_ValueError, errmsg); + return -1; + } + // Note that we don't check co->co_flags & CO_NESTED for anything here. + return 0; +} + + +int +_PyCode_CheckPureFunction(PyCodeObject *co, const char **p_errmsg) +{ + const char *errmsg = NULL; + if (co->co_flags & CO_GENERATOR) { + errmsg = "generators not supported"; + } + else if (co->co_flags & CO_COROUTINE) { + errmsg = "coroutines not supported"; + } + else if (co->co_flags & CO_ITERABLE_COROUTINE) { + errmsg = "coroutines not supported"; + } + else if (co->co_flags & CO_ASYNC_GENERATOR) { + errmsg = "generators not supported"; + } + + if (errmsg != NULL) { + if (p_errmsg != NULL) { + *p_errmsg = errmsg; + } + return 0; + } + return 1; +} + /* Here "value" means a non-None value, since a bare return is identical * to returning None explicitly. Likewise a missing return statement * at the end of the function is turned into "return None". */ static int code_returns_only_none(PyCodeObject *co) { + if (!_PyCode_CheckPureFunction(co, NULL)) { + return 0; + } + int len = (int)Py_SIZE(co); + assert(len > 0); + + // The last instruction either returns or raises. We can take advantage + // of that for a quick exit. + _Py_CODEUNIT final = _Py_GetBaseCodeUnit(co, len-1); + // Look up None in co_consts. Py_ssize_t nconsts = PyTuple_Size(co->co_consts); int none_index = 0; @@ -1971,26 +2218,42 @@ code_returns_only_none(PyCodeObject *co) } if (none_index == nconsts) { // None wasn't there, which means there was no implicit return, - // "return", or "return None". That means there must be - // an explicit return (non-None). - return 0; - } + // "return", or "return None". - // Walk the bytecode, looking for RETURN_VALUE. - Py_ssize_t len = Py_SIZE(co); - for (int i = 0; i < len; i += _PyInstruction_GetLength(co, i)) { - _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(co, i); - if (IS_RETURN_OPCODE(inst.op.code)) { - assert(i != 0); - // Ignore it if it returns None. - _Py_CODEUNIT prev = _Py_GetBaseCodeUnit(co, i-1); - if (prev.op.code == LOAD_CONST) { - // We don't worry about EXTENDED_ARG for now. - if (prev.op.arg == none_index) { - continue; + // That means there must be + // an explicit return (non-None), or it only raises. + if (IS_RETURN_OPCODE(final.op.code)) { + // It was an explicit return (non-None). + return 0; + } + // It must end with a raise then. We still have to walk the + // bytecode to see if there's any explicit return (non-None). + assert(IS_RAISE_OPCODE(final.op.code)); + for (int i = 0; i < len; i += _PyInstruction_GetLength(co, i)) { + _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(co, i); + if (IS_RETURN_OPCODE(inst.op.code)) { + // We alraedy know it isn't returning None. + return 0; + } + } + // It must only raise. + } + else { + // Walk the bytecode, looking for RETURN_VALUE. + for (int i = 0; i < len; i += _PyInstruction_GetLength(co, i)) { + _Py_CODEUNIT inst = _Py_GetBaseCodeUnit(co, i); + if (IS_RETURN_OPCODE(inst.op.code)) { + assert(i != 0); + // Ignore it if it returns None. + _Py_CODEUNIT prev = _Py_GetBaseCodeUnit(co, i-1); + if (prev.op.code == LOAD_CONST) { + // We don't worry about EXTENDED_ARG for now. + if (prev.op.arg == none_index) { + continue; + } } + return 0; } - return 0; } } return 1; @@ -2230,6 +2493,8 @@ free_monitoring_data(_PyCoMonitoringData *data) static void code_dealloc(PyObject *self) { + PyThreadState *tstate = PyThreadState_GET(); + _Py_atomic_add_uint64(&tstate->interp->_code_object_generation, 1); PyCodeObject *co = _PyCodeObject_CAST(self); _PyObject_ResurrectStart(self); notify_code_watchers(PY_CODE_EVENT_DESTROY, co); @@ -2281,20 +2546,20 @@ code_dealloc(PyObject *self) Py_XDECREF(co->_co_cached->_co_varnames); PyMem_Free(co->_co_cached); } - if (co->co_weakreflist != NULL) { - PyObject_ClearWeakRefs(self); - } + FT_CLEAR_WEAKREFS(self, co->co_weakreflist); free_monitoring_data(co->_co_monitoring); #ifdef Py_GIL_DISABLED - // The first element always points to the mutable bytecode at the end of - // the code object, which will be freed when the code object is freed. - for (Py_ssize_t i = 1; i < co->co_tlbc->size; i++) { - char *entry = co->co_tlbc->entries[i]; - if (entry != NULL) { - PyMem_Free(entry); + if (co->co_tlbc != NULL) { + // The first element always points to the mutable bytecode at the end of + // the code object, which will be freed when the code object is freed. + for (Py_ssize_t i = 1; i < co->co_tlbc->size; i++) { + char *entry = co->co_tlbc->entries[i]; + if (entry != NULL) { + PyMem_Free(entry); + } } + PyMem_Free(co->co_tlbc); } - PyMem_Free(co->co_tlbc); #endif PyObject_Free(co); } @@ -2689,12 +2954,13 @@ code._varname_from_oparg (internal-only) Return the local variable name for the given oparg. -WARNING: this method is for internal use only and may change or go away. +WARNING: this method is for internal use only and may change or go +away. [clinic start generated code]*/ static PyObject * code__varname_from_oparg_impl(PyCodeObject *self, int oparg) -/*[clinic end generated code: output=1fd1130413184206 input=c5fa3ee9bac7d4ca]*/ +/*[clinic end generated code: output=1fd1130413184206 input=6ba7d6df0d566463]*/ { PyObject *name = PyTuple_GetItem(self->co_localsplusnames, oparg); if (name == NULL) { @@ -3176,12 +3442,29 @@ _PyCodeArray_New(Py_ssize_t size) return arr; } +// Get the underlying code unit, leaving instrumentation +static _Py_CODEUNIT +deopt_code_unit(PyCodeObject *code, int i) +{ + _Py_CODEUNIT *src_instr = _PyCode_CODE(code) + i; + _Py_CODEUNIT inst = { + .cache = FT_ATOMIC_LOAD_UINT16_RELAXED(*(uint16_t *)src_instr)}; + int opcode = inst.op.code; + if (opcode < MIN_INSTRUMENTED_OPCODE) { + inst.op.code = _PyOpcode_Deopt[opcode]; + assert(inst.op.code < MIN_SPECIALIZED_OPCODE); + } + // JIT should not be enabled with free-threading + assert(inst.op.code != ENTER_EXECUTOR); + return inst; +} + static void copy_code(_Py_CODEUNIT *dst, PyCodeObject *co) { int code_len = (int) Py_SIZE(co); for (int i = 0; i < code_len; i += _PyInstruction_GetLength(co, i)) { - dst[i] = _Py_GetBaseCodeUnit(co, i); + dst[i] = deopt_code_unit(co, i); } _PyCode_Quicken(dst, code_len, 1); } @@ -3214,7 +3497,7 @@ create_tlbc_lock_held(PyCodeObject *co, Py_ssize_t idx) } memcpy(new_tlbc->entries, tlbc->entries, tlbc->size * sizeof(void *)); _Py_atomic_store_ptr_release(&co->co_tlbc, new_tlbc); - _PyMem_FreeDelayed(tlbc); + _PyMem_FreeDelayed(tlbc, tlbc->size * sizeof(void *)); tlbc = new_tlbc; } char *bc = PyMem_Calloc(1, _PyCode_NBYTES(co)); diff --git a/Objects/descrobject.c b/Objects/descrobject.c index 10c465b95ac1928..344f2713af8e07c 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -150,7 +150,7 @@ method_get(PyObject *self, PyObject *obj, PyObject *type) } else { PyErr_Format(PyExc_TypeError, "descriptor '%V' needs a type, not '%s', as arg 2", - descr_name((PyDescrObject *)descr), + descr_name((PyDescrObject *)descr), "?", Py_TYPE(type)->tp_name); return NULL; } @@ -1178,7 +1178,7 @@ static PyMethodDef mappingproxy_methods[] = { {"copy", mappingproxy_copy, METH_NOARGS, PyDoc_STR("D.copy() -> a shallow copy of D")}, {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, - PyDoc_STR("See PEP 585")}, + PyDoc_STR("mappingproxy objects are generic over two types, signifying (respectively) the types of their keys and values")}, {"__reversed__", mappingproxy_reversed, METH_NOARGS, PyDoc_STR("D.__reversed__() -> reverse iterator")}, {0} @@ -1607,7 +1607,7 @@ property_set_name(PyObject *self, PyObject *args) { if (PyTuple_GET_SIZE(args) != 2) { PyErr_Format( PyExc_TypeError, - "__set_name__() takes 2 positional arguments but %d were given", + "__set_name__() takes 2 positional arguments but %zd were given", PyTuple_GET_SIZE(args)); return NULL; } diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 59b0cf1ce7d4228..1f95ff5e0ef16a1 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -547,13 +547,13 @@ static inline uint8_t calculate_log2_keysize(Py_ssize_t minsize) { #if SIZEOF_LONG == SIZEOF_SIZE_T - minsize = (minsize | PyDict_MINSIZE) - 1; - return _Py_bit_length(minsize | (PyDict_MINSIZE-1)); + minsize = Py_MAX(minsize, PyDict_MINSIZE); + return _Py_bit_length(minsize - 1); #elif defined(_MSC_VER) - // On 64bit Windows, sizeof(long) == 4. - minsize = (minsize | PyDict_MINSIZE) - 1; + // On 64bit Windows, sizeof(long) == 4. We cannot use _Py_bit_length. + minsize = Py_MAX(minsize, PyDict_MINSIZE); unsigned long msb; - _BitScanReverse64(&msb, (uint64_t)minsize); + _BitScanReverse64(&msb, (uint64_t)minsize - 1); return (uint8_t)(msb + 1); #else uint8_t log2_size; @@ -813,7 +813,7 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr) { #ifdef Py_GIL_DISABLED if (use_qsbr) { - _PyMem_FreeDelayed(keys); + _PyMem_FreeDelayed(keys, _PyDict_KeysSize(keys)); return; } #endif @@ -858,7 +858,7 @@ free_values(PyDictValues *values, bool use_qsbr) assert(values->embedded == 0); #ifdef Py_GIL_DISABLED if (use_qsbr) { - _PyMem_FreeDelayed(values); + _PyMem_FreeDelayed(values, values_size_from_count(values->capacity)); return; } #endif @@ -1082,7 +1082,7 @@ compare_unicode_unicode(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; - PyObject *ep_key = FT_ATOMIC_LOAD_PTR_RELAXED(ep->me_key); + PyObject *ep_key = FT_ATOMIC_LOAD_PTR_CONSUME(ep->me_key); assert(ep_key != NULL); assert(PyUnicode_CheckExact(ep_key)); if (ep_key == key || @@ -1375,7 +1375,7 @@ compare_unicode_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; - PyObject *startkey = _Py_atomic_load_ptr_relaxed(&ep->me_key); + PyObject *startkey = _Py_atomic_load_ptr_consume(&ep->me_key); assert(startkey == NULL || PyUnicode_CheckExact(ep->me_key)); assert(!PyUnicode_CheckExact(key)); @@ -1418,7 +1418,7 @@ compare_unicode_unicode_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { PyDictUnicodeEntry *ep = &((PyDictUnicodeEntry *)ep0)[ix]; - PyObject *startkey = _Py_atomic_load_ptr_relaxed(&ep->me_key); + PyObject *startkey = _Py_atomic_load_ptr_consume(&ep->me_key); if (startkey == key) { assert(PyUnicode_CheckExact(startkey)); return 1; @@ -1454,7 +1454,7 @@ compare_generic_threadsafe(PyDictObject *mp, PyDictKeysObject *dk, void *ep0, Py_ssize_t ix, PyObject *key, Py_hash_t hash) { PyDictKeyEntry *ep = &((PyDictKeyEntry *)ep0)[ix]; - PyObject *startkey = _Py_atomic_load_ptr_relaxed(&ep->me_key); + PyObject *startkey = _Py_atomic_load_ptr_consume(&ep->me_key); if (startkey == key) { return 1; } @@ -1584,32 +1584,49 @@ _Py_dict_lookup_threadsafe(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyOb return ix; } +static Py_ssize_t +lookup_threadsafe_unicode(PyDictKeysObject *dk, PyObject *key, Py_hash_t hash, _PyStackRef *value_addr) +{ + assert(dk->dk_kind == DICT_KEYS_UNICODE); + assert(PyUnicode_CheckExact(key)); + + Py_ssize_t ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash); + if (ix == DKIX_EMPTY) { + *value_addr = PyStackRef_NULL; + return ix; + } + else if (ix >= 0) { + PyObject **addr_of_value = &DK_UNICODE_ENTRIES(dk)[ix].me_value; + PyObject *value = _Py_atomic_load_ptr(addr_of_value); + if (value == NULL) { + *value_addr = PyStackRef_NULL; + return DKIX_EMPTY; + } + if (_PyObject_HasDeferredRefcount(value)) { + *value_addr = (_PyStackRef){ .bits = (uintptr_t)value | Py_TAG_DEFERRED }; + return ix; + } + if (_Py_TryIncrefCompare(addr_of_value, value)) { + *value_addr = PyStackRef_FromPyObjectSteal(value); + return ix; + } + return DKIX_KEY_CHANGED; + } + assert(ix == DKIX_KEY_CHANGED); + return ix; +} + Py_ssize_t _Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject *key, Py_hash_t hash, _PyStackRef *value_addr) { - PyDictKeysObject *dk = _Py_atomic_load_ptr(&mp->ma_keys); + ensure_shared_on_read(mp); + + PyDictKeysObject *dk = _Py_atomic_load_ptr_acquire(&mp->ma_keys); if (dk->dk_kind == DICT_KEYS_UNICODE && PyUnicode_CheckExact(key)) { - Py_ssize_t ix = unicodekeys_lookup_unicode_threadsafe(dk, key, hash); - if (ix == DKIX_EMPTY) { - *value_addr = PyStackRef_NULL; + Py_ssize_t ix = lookup_threadsafe_unicode(dk, key, hash, value_addr); + if (ix != DKIX_KEY_CHANGED) { return ix; } - else if (ix >= 0) { - PyObject **addr_of_value = &DK_UNICODE_ENTRIES(dk)[ix].me_value; - PyObject *value = _Py_atomic_load_ptr(addr_of_value); - if (value == NULL) { - *value_addr = PyStackRef_NULL; - return DKIX_EMPTY; - } - if (_PyObject_HasDeferredRefcount(value)) { - *value_addr = (_PyStackRef){ .bits = (uintptr_t)value | Py_TAG_DEFERRED }; - return ix; - } - if (_Py_TryIncrefCompare(addr_of_value, value)) { - *value_addr = PyStackRef_FromPyObjectSteal(value); - return ix; - } - } } PyObject *obj; @@ -1649,6 +1666,54 @@ _Py_dict_lookup_threadsafe_stackref(PyDictObject *mp, PyObject *key, Py_hash_t h #endif +// Looks up the unicode key `key` in the dictionary. Note that `*method` may +// already contain a valid value! See _PyObject_GetMethodStackRef(). +int +_PyDict_GetMethodStackRef(PyDictObject *mp, PyObject *key, _PyStackRef *method) +{ + assert(PyUnicode_CheckExact(key)); + Py_hash_t hash = hash_unicode_key(key); + +#ifdef Py_GIL_DISABLED + // NOTE: We can only do the fast-path lookup if we are on the owning + // thread or if the dict is already marked as shared so that the load + // of ma_keys is safe without a lock. We cannot call ensure_shared_on_read() + // in this code path without incref'ing the dict because the dict is a + // borrowed reference protected by QSBR, and acquiring the lock could lead + // to a quiescent state (allowing the dict to be freed). + if (_Py_IsOwnedByCurrentThread((PyObject *)mp) || IS_DICT_SHARED(mp)) { + PyDictKeysObject *dk = _Py_atomic_load_ptr_acquire(&mp->ma_keys); + if (dk->dk_kind == DICT_KEYS_UNICODE) { + _PyStackRef ref; + Py_ssize_t ix = lookup_threadsafe_unicode(dk, key, hash, &ref); + if (ix >= 0) { + assert(!PyStackRef_IsNull(ref)); + PyStackRef_XSETREF(*method, ref); + return 1; + } + else if (ix == DKIX_EMPTY) { + return 0; + } + assert(ix == DKIX_KEY_CHANGED); + } + } +#endif + + PyObject *obj; + Py_INCREF(mp); + Py_ssize_t ix = _Py_dict_lookup_threadsafe(mp, key, hash, &obj); + Py_DECREF(mp); + if (ix == DKIX_ERROR) { + PyStackRef_CLEAR(*method); + return -1; + } + else if (ix >= 0 && obj != NULL) { + PyStackRef_XSETREF(*method, PyStackRef_FromPyObjectSteal(obj)); + return 1; + } + return 0; // not found +} + int _PyDict_HasOnlyStringKeys(PyObject *dict) { @@ -1723,6 +1788,14 @@ static inline int insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp, Py_hash_t hash, PyObject *key, PyObject *value) { + // gh-140551: If dict was cleared in _Py_dict_lookup, + // we have to resize one more time to force general key kind. + if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { + if (insertion_resize(interp, mp, 0) < 0) + return -1; + assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); + } + if (mp->ma_keys->dk_usable <= 0) { /* Need to resize. */ if (insertion_resize(interp, mp, 1) < 0) { @@ -1818,39 +1891,35 @@ static int insertdict(PyInterpreterState *interp, PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject *value) { - PyObject *old_value; + PyObject *old_value = NULL; + Py_ssize_t ix; ASSERT_DICT_LOCKED(mp); - if (DK_IS_UNICODE(mp->ma_keys) && !PyUnicode_CheckExact(key)) { - if (insertion_resize(interp, mp, 0) < 0) - goto Fail; - assert(mp->ma_keys->dk_kind == DICT_KEYS_GENERAL); - } - - if (_PyDict_HasSplitTable(mp)) { - Py_ssize_t ix = insert_split_key(mp->ma_keys, key, hash); + if (_PyDict_HasSplitTable(mp) && PyUnicode_CheckExact(key)) { + ix = insert_split_key(mp->ma_keys, key, hash); if (ix != DKIX_EMPTY) { insert_split_value(interp, mp, key, value, ix); Py_DECREF(key); Py_DECREF(value); return 0; } - - /* No space in shared keys. Resize and continue below. */ - if (insertion_resize(interp, mp, 1) < 0) { + // No space in shared keys. Go to insert_combined_dict() below. + } + else { + ix = _Py_dict_lookup(mp, key, hash, &old_value); + if (ix == DKIX_ERROR) goto Fail; - } } - Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &old_value); - if (ix == DKIX_ERROR) - goto Fail; - - if (ix == DKIX_EMPTY) { - assert(!_PyDict_HasSplitTable(mp)); - /* Insert into new slot. */ - assert(old_value == NULL); + if (old_value == NULL) { + // insert_combined_dict() will convert from non DICT_KEYS_GENERAL table + // into DICT_KEYS_GENERAL table if key is not Unicode. + // We don't convert it before _Py_dict_lookup because non-Unicode key + // may change generic table into Unicode table. + // + // NOTE: ix may not be DKIX_EMPTY because split table may have key + // without value. if (insert_combined_dict(interp, mp, hash, key, value) < 0) { goto Fail; } @@ -1862,10 +1931,14 @@ insertdict(PyInterpreterState *interp, PyDictObject *mp, if (old_value != value) { _PyDict_NotifyEvent(interp, PyDict_EVENT_MODIFIED, mp, key, value); assert(old_value != NULL); - assert(!_PyDict_HasSplitTable(mp)); if (DK_IS_UNICODE(mp->ma_keys)) { - PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; - STORE_VALUE(ep, value); + if (_PyDict_HasSplitTable(mp)) { + STORE_SPLIT_VALUE(mp, ix, value); + } + else { + PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix]; + STORE_VALUE(ep, value); + } } else { PyDictKeyEntry *ep = &DK_ENTRIES(mp->ma_keys)[ix]; @@ -2770,8 +2843,8 @@ PyDict_DelItem(PyObject *op, PyObject *key) return _PyDict_DelItem_KnownHash(op, key, hash); } -static int -delitem_knownhash_lock_held(PyObject *op, PyObject *key, Py_hash_t hash) +int +_PyDict_DelItem_KnownHash_LockHeld(PyObject *op, PyObject *key, Py_hash_t hash) { Py_ssize_t ix; PyDictObject *mp; @@ -2806,7 +2879,7 @@ _PyDict_DelItem_KnownHash(PyObject *op, PyObject *key, Py_hash_t hash) { int res; Py_BEGIN_CRITICAL_SECTION(op); - res = delitem_knownhash_lock_held(op, key, hash); + res = _PyDict_DelItem_KnownHash_LockHeld(op, key, hash); Py_END_CRITICAL_SECTION(); return res; } @@ -2868,6 +2941,21 @@ _PyDict_DelItemIf(PyObject *op, PyObject *key, return res; } +static void +clear_embedded_values(PyDictValues *values, Py_ssize_t nentries) +{ + PyObject *refs[SHARED_KEYS_MAX_SIZE]; + assert(nentries <= SHARED_KEYS_MAX_SIZE); + for (Py_ssize_t i = 0; i < nentries; i++) { + refs[i] = values->values[i]; + values->values[i] = NULL; + } + values->size = 0; + for (Py_ssize_t i = 0; i < nentries; i++) { + Py_XDECREF(refs[i]); + } +} + static void clear_lock_held(PyObject *op) { @@ -2897,24 +2985,27 @@ clear_lock_held(PyObject *op) assert(oldkeys->dk_refcnt == 1); dictkeys_decref(interp, oldkeys, IS_DICT_SHARED(mp)); } + else if (oldvalues->embedded) { + clear_embedded_values(oldvalues, oldkeys->dk_nentries); + } else { + set_values(mp, NULL); + set_keys(mp, Py_EMPTY_KEYS); n = oldkeys->dk_nentries; for (i = 0; i < n; i++) { Py_CLEAR(oldvalues->values[i]); } - if (oldvalues->embedded) { - oldvalues->size = 0; - } - else { - set_values(mp, NULL); - set_keys(mp, Py_EMPTY_KEYS); - free_values(oldvalues, IS_DICT_SHARED(mp)); - dictkeys_decref(interp, oldkeys, false); - } + free_values(oldvalues, IS_DICT_SHARED(mp)); + dictkeys_decref(interp, oldkeys, false); } ASSERT_CONSISTENT(mp); } +void +_PyDict_Clear_LockHeld(PyObject *op) { + clear_lock_held(op); +} + void PyDict_Clear(PyObject *op) { @@ -3178,9 +3269,10 @@ dict_set_fromkeys(PyInterpreterState *interp, PyDictObject *mp, Py_ssize_t pos = 0; PyObject *key; Py_hash_t hash; - - if (dictresize(interp, mp, - estimate_log2_keysize(PySet_GET_SIZE(iterable)), 0)) { + uint8_t new_size = Py_MAX( + estimate_log2_keysize(PySet_GET_SIZE(iterable)), + DK_LOG_SIZE(mp->ma_keys)); + if (dictresize(interp, mp, new_size, 0)) { Py_DECREF(mp); return NULL; } @@ -3852,7 +3944,7 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe } } - Py_ssize_t orig_size = other->ma_keys->dk_nentries; + Py_ssize_t orig_size = other->ma_used; Py_ssize_t pos = 0; Py_hash_t hash; PyObject *key, *value; @@ -3886,7 +3978,7 @@ dict_dict_merge(PyInterpreterState *interp, PyDictObject *mp, PyDictObject *othe if (err != 0) return -1; - if (orig_size != other->ma_keys->dk_nentries) { + if (orig_size != other->ma_used) { PyErr_SetString(PyExc_RuntimeError, "dict mutated during update"); return -1; @@ -4320,6 +4412,7 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu PyDictObject *mp = (PyDictObject *)d; PyObject *value; Py_hash_t hash; + Py_ssize_t ix; PyInterpreterState *interp = _PyInterpreterState_GET(); ASSERT_DICT_LOCKED(d); @@ -4355,17 +4448,8 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu return 0; } - if (!PyUnicode_CheckExact(key) && DK_IS_UNICODE(mp->ma_keys)) { - if (insertion_resize(interp, mp, 0) < 0) { - if (result) { - *result = NULL; - } - return -1; - } - } - - if (_PyDict_HasSplitTable(mp)) { - Py_ssize_t ix = insert_split_key(mp->ma_keys, key, hash); + if (_PyDict_HasSplitTable(mp) && PyUnicode_CheckExact(key)) { + ix = insert_split_key(mp->ma_keys, key, hash); if (ix != DKIX_EMPTY) { PyObject *value = mp->ma_values->values[ix]; int already_present = value != NULL; @@ -4378,33 +4462,29 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu } return already_present; } - - /* No space in shared keys. Resize and continue below. */ - if (insertion_resize(interp, mp, 1) < 0) { - goto error; - } + // No space in shared keys. Go to insert_combined_dict() below. } - - assert(!_PyDict_HasSplitTable(mp)); - - Py_ssize_t ix = _Py_dict_lookup(mp, key, hash, &value); - if (ix == DKIX_ERROR) { - if (result) { - *result = NULL; + else { + ix = _Py_dict_lookup(mp, key, hash, &value); + if (ix == DKIX_ERROR) { + if (result) { + *result = NULL; + } + return -1; } - return -1; } if (ix == DKIX_EMPTY) { - assert(!_PyDict_HasSplitTable(mp)); value = default_value; + // See comment to this function in insertdict. if (insert_combined_dict(interp, mp, hash, Py_NewRef(key), Py_NewRef(value)) < 0) { Py_DECREF(key); Py_DECREF(value); if (result) { *result = NULL; } + return -1; } STORE_USED(mp, mp->ma_used + 1); @@ -4422,12 +4502,6 @@ dict_setdefault_ref_lock_held(PyObject *d, PyObject *key, PyObject *default_valu *result = incref_result ? Py_NewRef(value) : value; } return 1; - -error: - if (result) { - *result = NULL; - } - return -1; } int @@ -4613,10 +4687,8 @@ dict_traverse(PyObject *op, visitproc visit, void *arg) if (DK_IS_UNICODE(keys)) { if (_PyDict_HasSplitTable(mp)) { - if (!mp->ma_values->embedded) { - for (i = 0; i < n; i++) { - Py_VISIT(mp->ma_values->values[i]); - } + for (i = 0; i < n; i++) { + Py_VISIT(mp->ma_values->values[i]); } } else { @@ -4647,9 +4719,11 @@ dict_tp_clear(PyObject *op) static PyObject *dictiter_new(PyDictObject *, PyTypeObject *); -static Py_ssize_t -sizeof_lock_held(PyDictObject *mp) +Py_ssize_t +_PyDict_SizeOf_LockHeld(PyDictObject *mp) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(mp); + size_t res = _PyObject_SIZE(Py_TYPE(mp)); if (_PyDict_HasSplitTable(mp)) { res += shared_keys_usable_size(mp->ma_keys) * sizeof(PyObject*); @@ -4668,7 +4742,7 @@ _PyDict_SizeOf(PyDictObject *mp) { Py_ssize_t res; Py_BEGIN_CRITICAL_SECTION(mp); - res = sizeof_lock_held(mp); + res = _PyDict_SizeOf_LockHeld(mp); Py_END_CRITICAL_SECTION(); return res; @@ -4753,7 +4827,8 @@ static PyMethodDef mapp_methods[] = { DICT_CLEAR_METHODDEF DICT_COPY_METHODDEF DICT___REVERSED___METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("dicts are generic over two types, signifying (respectively) the types of their keys and values")}, {NULL, NULL} /* sentinel */ }; @@ -5524,7 +5599,7 @@ dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, k = _Py_atomic_load_ptr_acquire(&d->ma_keys); assert(i >= 0); if (_PyDict_HasSplitTable(d)) { - PyDictValues *values = _Py_atomic_load_ptr_relaxed(&d->ma_values); + PyDictValues *values = _Py_atomic_load_ptr_consume(&d->ma_values); if (values == NULL) { goto concurrent_modification; } @@ -5609,22 +5684,10 @@ dictiter_iternext_threadsafe(PyDictObject *d, PyObject *self, #endif -static bool -has_unique_reference(PyObject *op) -{ -#ifdef Py_GIL_DISABLED - return (_Py_IsOwnedByCurrentThread(op) && - op->ob_ref_local == 1 && - _Py_atomic_load_ssize_relaxed(&op->ob_ref_shared) == 0); -#else - return Py_REFCNT(op) == 1; -#endif -} - static bool acquire_iter_result(PyObject *result) { - if (has_unique_reference(result)) { + if (_PyObject_IsUniquelyReferenced(result)) { Py_INCREF(result); return true; } @@ -5661,8 +5724,11 @@ dictiter_iternextitem(PyObject *self) } else { result = PyTuple_New(2); - if (result == NULL) + if (result == NULL) { + Py_DECREF(key); + Py_DECREF(value); return NULL; + } PyTuple_SET_ITEM(result, 0, key); PyTuple_SET_ITEM(result, 1, value); } @@ -5770,7 +5836,7 @@ dictreviter_iter_lock_held(PyDictObject *d, PyObject *self) } else if (Py_IS_TYPE(di, &PyDictRevIterItem_Type)) { result = di->di_result; - if (Py_REFCNT(result) == 1) { + if (_PyObject_IsUniquelyReferenced(result)) { PyObject *oldkey = PyTuple_GET_ITEM(result, 0); PyObject *oldvalue = PyTuple_GET_ITEM(result, 1); PyTuple_SET_ITEM(result, 0, Py_NewRef(key)); @@ -6858,7 +6924,7 @@ _PyDict_SetItem_LockHeld(PyDictObject *dict, PyObject *name, PyObject *value) dict_unhashable_type(name); return -1; } - return delitem_knownhash_lock_held((PyObject *)dict, name, hash); + return _PyDict_DelItem_KnownHash_LockHeld((PyObject *)dict, name, hash); } else { return setitem_lock_held(dict, name, value); } @@ -7125,7 +7191,7 @@ _PyObject_TryGetInstanceAttribute(PyObject *obj, PyObject *name, PyObject **attr Py_BEGIN_CRITICAL_SECTION(dict); if (dict->ma_values == values && FT_ATOMIC_LOAD_UINT8(values->valid)) { - value = _Py_atomic_load_ptr_relaxed(&values->values[ix]); + value = _Py_atomic_load_ptr_consume(&values->values[ix]); *attr = _Py_XNewRefWithLock(value); success = true; } else { @@ -7184,16 +7250,21 @@ PyObject_VisitManagedDict(PyObject *obj, visitproc visit, void *arg) if((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT) == 0) { return 0; } - if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { + PyDictObject *dict = _PyObject_ManagedDictPointer(obj)->dict; + if (dict != NULL) { + // GH-130327: If there's a managed dictionary available, we should + // *always* traverse it. The dict is responsible for traversing the + // inline values if it points to them. + Py_VISIT(dict); + } + else if (tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) { PyDictValues *values = _PyObject_InlineValues(obj); if (values->valid) { for (Py_ssize_t i = 0; i < values->capacity; i++) { Py_VISIT(values->values[i]); } - return 0; } } - Py_VISIT(_PyObject_ManagedDictPointer(obj)->dict); return 0; } @@ -7630,13 +7701,19 @@ validate_watcher_id(PyInterpreterState *interp, int watcher_id) PyErr_Format(PyExc_ValueError, "Invalid dict watcher ID %d", watcher_id); return -1; } - if (!interp->dict_state.watchers[watcher_id]) { + PyDict_WatchCallback cb = FT_ATOMIC_LOAD_PTR_RELAXED( + interp->dict_state.watchers[watcher_id]); + if (cb == NULL) { PyErr_Format(PyExc_ValueError, "No dict watcher set for ID %d", watcher_id); return -1; } return 0; } +// In free-threaded builds, Add/Clear serialize on watcher_mutex and publish +// callbacks with release stores. SendEvent reads them lock-free using +// acquire loads. + int PyDict_Watch(int watcher_id, PyObject* dict) { @@ -7648,7 +7725,8 @@ PyDict_Watch(int watcher_id, PyObject* dict) if (validate_watcher_id(interp, watcher_id)) { return -1; } - ((PyDictObject*)dict)->_ma_watcher_tag |= (1LL << watcher_id); + FT_ATOMIC_OR_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, + 1ULL << watcher_id); return 0; } @@ -7663,36 +7741,48 @@ PyDict_Unwatch(int watcher_id, PyObject* dict) if (validate_watcher_id(interp, watcher_id)) { return -1; } - ((PyDictObject*)dict)->_ma_watcher_tag &= ~(1LL << watcher_id); + FT_ATOMIC_AND_UINT64(((PyDictObject*)dict)->_ma_watcher_tag, + ~(1ULL << watcher_id)); return 0; } int PyDict_AddWatcher(PyDict_WatchCallback callback) { + int watcher_id = -1; PyInterpreterState *interp = _PyInterpreterState_GET(); + FT_MUTEX_LOCK_FLAGS(&interp->dict_state.watcher_mutex, + _Py_LOCK_DONT_DETACH); /* Start at 2, as 0 and 1 are reserved for CPython */ for (int i = 2; i < DICT_MAX_WATCHERS; i++) { if (!interp->dict_state.watchers[i]) { - interp->dict_state.watchers[i] = callback; - return i; + FT_ATOMIC_STORE_PTR_RELEASE(interp->dict_state.watchers[i], callback); + watcher_id = i; + goto done; } } - PyErr_SetString(PyExc_RuntimeError, "no more dict watcher IDs available"); - return -1; +done: + FT_MUTEX_UNLOCK(&interp->dict_state.watcher_mutex); + return watcher_id; } int PyDict_ClearWatcher(int watcher_id) { + int res = 0; PyInterpreterState *interp = _PyInterpreterState_GET(); + FT_MUTEX_LOCK_FLAGS(&interp->dict_state.watcher_mutex, + _Py_LOCK_DONT_DETACH); if (validate_watcher_id(interp, watcher_id)) { - return -1; + res = -1; + goto done; } - interp->dict_state.watchers[watcher_id] = NULL; - return 0; + FT_ATOMIC_STORE_PTR_RELEASE(interp->dict_state.watchers[watcher_id], NULL); +done: + FT_MUTEX_UNLOCK(&interp->dict_state.watcher_mutex); + return res; } static const char * @@ -7717,7 +7807,8 @@ _PyDict_SendEvent(int watcher_bits, PyInterpreterState *interp = _PyInterpreterState_GET(); for (int i = 0; i < DICT_MAX_WATCHERS; i++) { if (watcher_bits & 1) { - PyDict_WatchCallback cb = interp->dict_state.watchers[i]; + PyDict_WatchCallback cb = FT_ATOMIC_LOAD_PTR_ACQUIRE( + interp->dict_state.watchers[i]); if (cb && (cb(event, (PyObject*)mp, key, value) < 0)) { // We don't want to resurrect the dict by potentially having an // unraisablehook keep a reference to it, so we don't pass the diff --git a/Objects/enumobject.c b/Objects/enumobject.c index 1123b140c7fda80..e94a66bbd5c6afa 100644 --- a/Objects/enumobject.c +++ b/Objects/enumobject.c @@ -148,7 +148,7 @@ enumerate_vectorcall(PyObject *type, PyObject *const *args, } PyErr_Format(PyExc_TypeError, - "enumerate() takes at most 2 arguments (%d given)", nargs + nkwargs); + "enumerate() takes at most 2 arguments (%zd given)", nargs + nkwargs); return NULL; } @@ -291,10 +291,13 @@ enum_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) enumobject *en = _enumobject_CAST(op); PyObject *result; Py_BEGIN_CRITICAL_SECTION(en); - if (en->en_longindex != NULL) + if (en->en_longindex != NULL) { result = Py_BuildValue("O(OO)", Py_TYPE(en), en->en_sit, en->en_longindex); - else - result = Py_BuildValue("O(On)", Py_TYPE(en), en->en_sit, en->en_index); + } + else { + Py_ssize_t en_index = FT_ATOMIC_LOAD_SSIZE_RELAXED(en->en_index); + result = Py_BuildValue("O(On)", Py_TYPE(en), en->en_sit, en_index); + } Py_END_CRITICAL_SECTION(); return result; } @@ -304,7 +307,7 @@ PyDoc_STRVAR(reduce_doc, "Return state information for pickling."); static PyMethodDef enum_methods[] = { {"__reduce__", enum_reduce, METH_NOARGS, reduce_doc}, {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("'enumerate' objects are generic over the type of their values")}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/exceptions.c b/Objects/exceptions.c index b17cac835516707..833e283c17603c8 100644 --- a/Objects/exceptions.c +++ b/Objects/exceptions.c @@ -233,7 +233,7 @@ BaseException___reduce___impl(PyBaseExceptionObject *self) */ /*[clinic input] -@critical_section +@critical_section state BaseException.__setstate__ state: object / @@ -241,7 +241,7 @@ BaseException.__setstate__ static PyObject * BaseException___setstate___impl(PyBaseExceptionObject *self, PyObject *state) -/*[clinic end generated code: output=f3834889950453ab input=5524b61cfe9b9856]*/ +/*[clinic end generated code: output=f3834889950453ab input=f9b1aea70382cdb6]*/ { PyObject *d_key, *d_value; Py_ssize_t i = 0; @@ -695,12 +695,12 @@ PyTypeObject _PyExc_ ## EXCNAME = { \ #define ComplexExtendsException(EXCBASE, EXCNAME, EXCSTORE, EXCNEW, \ EXCMETHODS, EXCMEMBERS, EXCGETSET, \ - EXCSTR, EXCDOC) \ + EXCSTR, EXCREPR, EXCDOC) \ static PyTypeObject _PyExc_ ## EXCNAME = { \ PyVarObject_HEAD_INIT(NULL, 0) \ # EXCNAME, \ sizeof(Py ## EXCSTORE ## Object), 0, \ - EXCSTORE ## _dealloc, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, \ + EXCSTORE ## _dealloc, 0, 0, 0, 0, EXCREPR, 0, 0, 0, 0, 0, \ EXCSTR, 0, 0, 0, \ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, \ PyDoc_STR(EXCDOC), EXCSTORE ## _traverse, \ @@ -793,7 +793,7 @@ StopIteration_traverse(PyObject *op, visitproc visit, void *arg) } ComplexExtendsException(PyExc_Exception, StopIteration, StopIteration, - 0, 0, StopIteration_members, 0, 0, + 0, 0, StopIteration_members, 0, 0, 0, "Signal the end from iterator.__next__()."); @@ -866,7 +866,7 @@ static PyMemberDef SystemExit_members[] = { }; ComplexExtendsException(PyExc_BaseException, SystemExit, SystemExit, - 0, 0, SystemExit_members, 0, 0, + 0, 0, SystemExit_members, 0, 0, 0, "Request to exit from the interpreter."); /* @@ -891,6 +891,7 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) PyObject *message = NULL; PyObject *exceptions = NULL; + PyObject *exceptions_str = NULL; if (!PyArg_ParseTuple(args, "UO:BaseExceptionGroup.__new__", @@ -906,6 +907,18 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } + /* Save initial exceptions sequence as a string in case sequence is mutated */ + if (!PyList_Check(exceptions) && !PyTuple_Check(exceptions)) { + exceptions_str = PyObject_Repr(exceptions); + if (exceptions_str == NULL) { + /* We don't hold a reference to exceptions, so clear it before + * attempting a decref in the cleanup. + */ + exceptions = NULL; + goto error; + } + } + exceptions = PySequence_Tuple(exceptions); if (!exceptions) { return NULL; @@ -930,7 +943,7 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (!PyExceptionInstance_Check(exc)) { PyErr_Format( PyExc_ValueError, - "Item %d of second argument (exceptions) is not an exception", + "Item %zd of second argument (exceptions) is not an exception", i); goto error; } @@ -989,9 +1002,11 @@ BaseExceptionGroup_new(PyTypeObject *type, PyObject *args, PyObject *kwds) self->msg = Py_NewRef(message); self->excs = exceptions; + self->excs_str = exceptions_str; return (PyObject*)self; error: - Py_DECREF(exceptions); + Py_XDECREF(exceptions); + Py_XDECREF(exceptions_str); return NULL; } @@ -1030,6 +1045,7 @@ BaseExceptionGroup_clear(PyObject *op) PyBaseExceptionGroupObject *self = PyBaseExceptionGroupObject_CAST(op); Py_CLEAR(self->msg); Py_CLEAR(self->excs); + Py_CLEAR(self->excs_str); return BaseException_clear(op); } @@ -1047,6 +1063,7 @@ BaseExceptionGroup_traverse(PyObject *op, visitproc visit, void *arg) PyBaseExceptionGroupObject *self = PyBaseExceptionGroupObject_CAST(op); Py_VISIT(self->msg); Py_VISIT(self->excs); + Py_VISIT(self->excs_str); return BaseException_traverse(op, visit, arg); } @@ -1064,6 +1081,54 @@ BaseExceptionGroup_str(PyObject *op) self->msg, num_excs, num_excs > 1 ? "s" : ""); } +static PyObject * +BaseExceptionGroup_repr(PyObject *op) +{ + PyBaseExceptionGroupObject *self = PyBaseExceptionGroupObject_CAST(op); + assert(self->msg); + + PyObject *exceptions_str = NULL; + + /* Use the saved exceptions string for custom sequences. */ + if (self->excs_str) { + exceptions_str = Py_NewRef(self->excs_str); + } + else { + assert(self->excs); + + /* Older versions delegated to BaseException, inserting the current + * value of self.args[1]; but this can be mutable and go out-of-sync + * with self.exceptions. Instead, use self.exceptions for accuracy, + * making it look like self.args[1] for backwards compatibility. */ + if (PyList_Check(PyTuple_GET_ITEM(self->args, 1))) { + PyObject *exceptions_list = PySequence_List(self->excs); + if (!exceptions_list) { + return NULL; + } + + exceptions_str = PyObject_Repr(exceptions_list); + Py_DECREF(exceptions_list); + } + else { + exceptions_str = PyObject_Repr(self->excs); + } + + if (!exceptions_str) { + return NULL; + } + } + + assert(exceptions_str != NULL); + + const char *name = _PyType_Name(Py_TYPE(self)); + PyObject *repr = PyUnicode_FromFormat( + "%s(%R, %U)", name, + self->msg, exceptions_str); + + Py_DECREF(exceptions_str); + return repr; +} + /*[clinic input] @critical_section BaseExceptionGroup.derive @@ -1659,7 +1724,7 @@ PyUnstable_Exc_PrepReraiseStar(PyObject *orig, PyObject *excs) PyObject *exc = PyList_GET_ITEM(excs, i); if (exc == NULL || !(PyExceptionInstance_Check(exc) || Py_IsNone(exc))) { PyErr_Format(PyExc_TypeError, - "item %d of excs is not an exception", i); + "item %zd of excs is not an exception", i); return NULL; } } @@ -1688,7 +1753,8 @@ static PyMemberDef BaseExceptionGroup_members[] = { static PyMethodDef BaseExceptionGroup_methods[] = { {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + PyDoc_STR("Exception groups are generic over the type of their contained exceptions")}, BASEEXCEPTIONGROUP_DERIVE_METHODDEF BASEEXCEPTIONGROUP_SPLIT_METHODDEF BASEEXCEPTIONGROUP_SUBGROUP_METHODDEF @@ -1698,7 +1764,7 @@ static PyMethodDef BaseExceptionGroup_methods[] = { ComplexExtendsException(PyExc_BaseException, BaseExceptionGroup, BaseExceptionGroup, BaseExceptionGroup_new /* new */, BaseExceptionGroup_methods, BaseExceptionGroup_members, - 0 /* getset */, BaseExceptionGroup_str, + 0 /* getset */, BaseExceptionGroup_str, BaseExceptionGroup_repr, "A combination of multiple unrelated exceptions."); /* @@ -1884,7 +1950,7 @@ static PyMethodDef ImportError_methods[] = { ComplexExtendsException(PyExc_Exception, ImportError, ImportError, 0 /* new */, ImportError_methods, ImportError_members, - 0 /* getset */, ImportError_str, + 0 /* getset */, ImportError_str, 0, "Import can't find module, or can't find name in " "module."); @@ -2008,10 +2074,10 @@ oserror_init(PyOSErrorObject *self, PyObject **p_args, return -1; } else { - self->filename = Py_NewRef(filename); + Py_XSETREF(self->filename, Py_NewRef(filename)); if (filename2 && filename2 != Py_None) { - self->filename2 = Py_NewRef(filename2); + Py_XSETREF(self->filename2, Py_NewRef(filename2)); } if (nargs >= 2 && nargs <= 5) { @@ -2026,10 +2092,10 @@ oserror_init(PyOSErrorObject *self, PyObject **p_args, } } } - self->myerrno = Py_XNewRef(myerrno); - self->strerror = Py_XNewRef(strerror); + Py_XSETREF(self->myerrno, Py_XNewRef(myerrno)); + Py_XSETREF(self->strerror, Py_XNewRef(strerror)); #ifdef MS_WINDOWS - self->winerror = Py_XNewRef(winerror); + Py_XSETREF(self->winerror, Py_XNewRef(winerror)); #endif /* Steals the reference to args */ @@ -2356,7 +2422,7 @@ static PyGetSetDef OSError_getset[] = { ComplexExtendsException(PyExc_Exception, OSError, OSError, OSError_new, OSError_methods, OSError_members, OSError_getset, - OSError_str, + OSError_str, 0, "Base class for I/O related errors."); @@ -2497,7 +2563,7 @@ static PyMethodDef NameError_methods[] = { ComplexExtendsException(PyExc_Exception, NameError, NameError, 0, NameError_methods, NameError_members, - 0, BaseException_str, "Name not found globally."); + 0, BaseException_str, 0, "Name not found globally."); /* * UnboundLocalError extends NameError @@ -2631,7 +2697,7 @@ static PyMethodDef AttributeError_methods[] = { ComplexExtendsException(PyExc_Exception, AttributeError, AttributeError, 0, AttributeError_methods, AttributeError_members, - 0, BaseException_str, "Attribute not found."); + 0, BaseException_str, 0, "Attribute not found."); /* * SyntaxError extends Exception @@ -2664,23 +2730,25 @@ SyntaxError_init(PyObject *op, PyObject *args, PyObject *kwds) return -1; } - self->end_lineno = NULL; - self->end_offset = NULL; + PyObject *filename, *lineno, *offset, *text; + PyObject *end_lineno = NULL; + PyObject *end_offset = NULL; + PyObject *metadata = NULL; if (!PyArg_ParseTuple(info, "OOOO|OOO", - &self->filename, &self->lineno, - &self->offset, &self->text, - &self->end_lineno, &self->end_offset, &self->metadata)) { + &filename, &lineno, + &offset, &text, + &end_lineno, &end_offset, &metadata)) { Py_DECREF(info); return -1; } - Py_INCREF(self->filename); - Py_INCREF(self->lineno); - Py_INCREF(self->offset); - Py_INCREF(self->text); - Py_XINCREF(self->end_lineno); - Py_XINCREF(self->end_offset); - Py_XINCREF(self->metadata); + Py_XSETREF(self->filename, Py_NewRef(filename)); + Py_XSETREF(self->lineno, Py_NewRef(lineno)); + Py_XSETREF(self->offset, Py_NewRef(offset)); + Py_XSETREF(self->text, Py_NewRef(text)); + Py_XSETREF(self->end_lineno, Py_XNewRef(end_lineno)); + Py_XSETREF(self->end_offset, Py_XNewRef(end_offset)); + Py_XSETREF(self->metadata, Py_XNewRef(metadata)); Py_DECREF(info); if (self->end_lineno != NULL && self->end_offset == NULL) { @@ -2830,7 +2898,7 @@ static PyMemberDef SyntaxError_members[] = { ComplexExtendsException(PyExc_Exception, SyntaxError, SyntaxError, 0, 0, SyntaxError_members, 0, - SyntaxError_str, "Invalid syntax."); + SyntaxError_str, 0, "Invalid syntax."); /* @@ -2890,7 +2958,7 @@ KeyError_str(PyObject *op) } ComplexExtendsException(PyExc_LookupError, KeyError, BaseException, - 0, 0, 0, 0, KeyError_str, "Mapping key not found."); + 0, 0, 0, 0, KeyError_str, 0, "Mapping key not found."); /* diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 93e1973d6b32fc7..a9d521634a62dfa 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -139,7 +139,11 @@ _PyStackRef _PyFloat_FromDouble_ConsumeInputs(_PyStackRef left, _PyStackRef righ { PyStackRef_CLOSE_SPECIALIZED(left, _PyFloat_ExactDealloc); PyStackRef_CLOSE_SPECIALIZED(right, _PyFloat_ExactDealloc); - return PyStackRef_FromPyObjectSteal(PyFloat_FromDouble(value)); + PyObject *obj = PyFloat_FromDouble(value); + if (obj == NULL) { + return PyStackRef_NULL; + } + return PyStackRef_FromPyObjectSteal(obj); } static PyObject * @@ -442,82 +446,67 @@ float_richcompare(PyObject *v, PyObject *w, int op) assert(vsign != 0); /* if vsign were 0, then since wsign is * not 0, we would have taken the * vsign != wsign branch at the start */ - /* We want to work with non-negative numbers. */ - if (vsign < 0) { - /* "Multiply both sides" by -1; this also swaps the - * comparator. - */ - i = -i; - op = _Py_SwappedOp[op]; - } - assert(i > 0.0); (void) frexp(i, &exponent); /* exponent is the # of bits in v before the radix point; * we know that nbits (the # of bits in w) > 48 at this point */ if (exponent < nbits) { - i = 1.0; - j = 2.0; + j = i; + i = 0.0; goto Compare; } if (exponent > nbits) { - i = 2.0; - j = 1.0; + j = 0.0; goto Compare; } /* v and w have the same number of bits before the radix - * point. Construct two ints that have the same comparison - * outcome. + * point. Construct an int from the integer part of v and + * update op if necessary, so comparing two ints has the same outcome. */ { double fracpart; double intpart; PyObject *result = NULL; PyObject *vv = NULL; - PyObject *ww = w; - if (wsign < 0) { - ww = PyNumber_Negative(w); - if (ww == NULL) - goto Error; + fracpart = modf(i, &intpart); + if (fracpart != 0.0) { + switch (op) { + /* Non-integer float never equals to an int. */ + case Py_EQ: + Py_RETURN_FALSE; + case Py_NE: + Py_RETURN_TRUE; + /* For non-integer float, v <= w <=> v < w. + * If v > 0: trunc(v) < v < trunc(v) + 1 + * v < w => trunc(v) < w + * trunc(v) < w => trunc(v) + 1 <= w => v < w + * If v < 0: trunc(v) - 1 < v < trunc(v) + * v < w => trunc(v) - 1 < w => trunc(v) <= w + * trunc(v) <= w => v < w + */ + case Py_LT: + case Py_LE: + op = vsign > 0 ? Py_LT : Py_LE; + break; + /* The same as above, but with opposite directions. */ + case Py_GT: + case Py_GE: + op = vsign > 0 ? Py_GE : Py_GT; + break; + } } - else - Py_INCREF(ww); - fracpart = modf(i, &intpart); vv = PyLong_FromDouble(intpart); if (vv == NULL) goto Error; - if (fracpart != 0.0) { - /* Shift left, and or a 1 bit into vv - * to represent the lost fraction. - */ - PyObject *temp; - - temp = _PyLong_Lshift(ww, 1); - if (temp == NULL) - goto Error; - Py_SETREF(ww, temp); - - temp = _PyLong_Lshift(vv, 1); - if (temp == NULL) - goto Error; - Py_SETREF(vv, temp); - - temp = PyNumber_Or(vv, _PyLong_GetOne()); - if (temp == NULL) - goto Error; - Py_SETREF(vv, temp); - } - - r = PyObject_RichCompareBool(vv, ww, op); + r = PyObject_RichCompareBool(vv, w, op); if (r < 0) goto Error; result = PyBool_FromLong(r); Error: Py_XDECREF(vv); - Py_XDECREF(ww); return result; } } /* else if (PyLong_Check(w)) */ @@ -1709,14 +1698,14 @@ You probably don't want to use this function. It exists mainly to be used in Python's test suite. -This function returns whichever of 'unknown', 'IEEE, big-endian' or 'IEEE, -little-endian' best describes the format of floating-point numbers used by the -C type named by typestr. +This function returns whichever of 'unknown', 'IEEE, big-endian' or +'IEEE, little-endian' best describes the format of floating-point +numbers used by the C type named by typestr. [clinic start generated code]*/ static PyObject * float___getformat___impl(PyTypeObject *type, const char *typestr) -/*[clinic end generated code: output=2bfb987228cc9628 input=90d5e246409a246e]*/ +/*[clinic end generated code: output=2bfb987228cc9628 input=933cc4bdcf8fa8d3]*/ { float_format_type r; @@ -2028,6 +2017,10 @@ PyFloat_Pack2(double x, char *data, int le) memcpy(&v, &x, sizeof(v)); v &= 0xffc0000000000ULL; bits = (unsigned short)(v >> 42); /* NaN's type & payload */ + /* set qNaN if no payload */ + if (!bits) { + bits |= (1<<9); + } } else { sign = (x < 0.0); @@ -2200,16 +2193,16 @@ PyFloat_Pack4(double x, char *data, int le) if ((v & (1ULL << 51)) == 0) { uint32_t u32; memcpy(&u32, &y, 4); - u32 &= ~(1 << 22); /* make sNaN */ + /* if have payload, make sNaN */ + if (u32 & 0x3fffff) { + u32 &= ~(1 << 22); + } memcpy(&y, &u32, 4); } #else uint32_t u32; memcpy(&u32, &y, 4); - if ((v & (1ULL << 51)) == 0) { - u32 &= ~(1 << 22); - } /* Workaround RISC-V: "If a NaN value is converted to a * different floating-point type, the result is the * canonical NaN of the new type". The canonical NaN here @@ -2220,6 +2213,10 @@ PyFloat_Pack4(double x, char *data, int le) /* add payload */ u32 -= (u32 & 0x3fffff); u32 += (uint32_t)((v & 0x7ffffffffffffULL) >> 29); + /* if have payload, make sNaN */ + if ((v & (1ULL << 51)) == 0 && (u32 & 0x3fffff)) { + u32 &= ~(1 << 22); + } memcpy(&y, &u32, 4); #endif diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 76b52efccf804f2..b4597a937074b8f 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -1036,11 +1036,11 @@ static PyObject * frame_lasti_get_impl(PyFrameObject *self) /*[clinic end generated code: output=03275b4f0327d1a2 input=0225ed49cb1fbeeb]*/ { - int lasti = _PyInterpreterFrame_LASTI(self->f_frame); + int lasti = PyUnstable_InterpreterFrame_GetLasti(self->f_frame); if (lasti < 0) { return PyLong_FromLong(-1); } - return PyLong_FromLong(lasti * sizeof(_Py_CODEUNIT)); + return PyLong_FromLong(lasti); } /*[clinic input] @@ -1109,7 +1109,7 @@ frame_back_get_impl(PyFrameObject *self) /*[clinic end generated code: output=3a84c22a55a63c79 input=9e528570d0e1f44a]*/ { PyObject *res = (PyObject *)PyFrame_GetBack(self); - if (res == NULL) { + if (res == NULL && !PyErr_Occurred()) { Py_RETURN_NONE; } return res; @@ -1671,6 +1671,8 @@ frame_lineno_set_impl(PyFrameObject *self, PyObject *value) case PY_MONITORING_EVENT_PY_RESUME: case PY_MONITORING_EVENT_JUMP: case PY_MONITORING_EVENT_BRANCH: + case PY_MONITORING_EVENT_BRANCH_LEFT: + case PY_MONITORING_EVENT_BRANCH_RIGHT: case PY_MONITORING_EVENT_LINE: case PY_MONITORING_EVENT_PY_YIELD: /* Setting f_lineno is allowed for the above events */ @@ -2042,11 +2044,15 @@ static PyObject * frame_repr(PyObject *op) { PyFrameObject *f = PyFrameObject_CAST(op); + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(f); int lineno = PyFrame_GetLineNumber(f); PyCodeObject *code = _PyFrame_GetCode(f->f_frame); - return PyUnicode_FromFormat( + result = PyUnicode_FromFormat( "<frame at %p, file %R, line %d, code %S>", f, code->co_filename, lineno, code->co_name); + Py_END_CRITICAL_SECTION(); + return result; } static PyMethodDef frame_methods[] = { @@ -2282,6 +2288,9 @@ _PyFrame_GetLocals(_PyInterpreterFrame *frame) } PyFrameObject* f = _PyFrame_GetFrameObject(frame); + if (f == NULL) { + return NULL; + } return _PyFrameLocalsProxy_New(f); } @@ -2391,6 +2400,9 @@ PyFrame_GetBack(PyFrameObject *frame) prev = _PyFrame_GetFirstComplete(prev); if (prev) { back = _PyFrame_GetFrameObject(prev); + if (back == NULL) { + return NULL; + } } } return (PyFrameObject*)Py_XNewRef(back); diff --git a/Objects/funcobject.c b/Objects/funcobject.c index 56df5730db0c55b..6d1e093ddab7ce9 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -1,13 +1,16 @@ /* Function object implementation */ #include "Python.h" +#include "pycore_code.h" // _PyCode_VerifyStateless() #include "pycore_dict.h" // _Py_INCREF_DICT() #include "pycore_function.h" // _PyFunction_Vectorcall #include "pycore_long.h" // _PyLong_GetOne() #include "pycore_modsupport.h" // _PyArg_NoKeywords() #include "pycore_object.h" // _PyObject_GC_UNTRACK() #include "pycore_pyerrors.h" // _PyErr_Occurred() +#include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_stats.h" +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() static const char * @@ -678,7 +681,7 @@ func_set_code(PyObject *self, PyObject *value, void *Py_UNUSED(ignored)) if (nclosure != nfree) { PyErr_Format(PyExc_ValueError, "%U() requires a code object with %zd free vars," - " not %zd", + " not %d", op->func_name, nclosure, nfree); return -1; @@ -1064,7 +1067,7 @@ func_new_impl(PyTypeObject *type, PyCodeObject *code, PyObject *globals, nclosure = closure == Py_None ? 0 : PyTuple_GET_SIZE(closure); if (code->co_nfreevars != nclosure) return PyErr_Format(PyExc_ValueError, - "%U requires closure of length %zd, not %zd", + "%U requires closure of length %d, not %zd", code->co_name, code->co_nfreevars, nclosure); if (nclosure) { Py_ssize_t i; @@ -1146,9 +1149,7 @@ func_dealloc(PyObject *self) return; } _PyObject_GC_UNTRACK(op); - if (op->func_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *) op); - } + FT_CLEAR_WEAKREFS(self, op->func_weakreflist); (void)func_clear((PyObject*)op); // These aren't cleared by func_clear(). _Py_DECREF_CODE((PyCodeObject *)op->func_code); @@ -1240,6 +1241,64 @@ PyTypeObject PyFunction_Type = { }; +int +_PyFunction_VerifyStateless(PyThreadState *tstate, PyObject *func) +{ + assert(!PyErr_Occurred()); + assert(PyFunction_Check(func)); + + // Check the globals. + PyObject *globalsns = PyFunction_GET_GLOBALS(func); + if (globalsns != NULL && !PyDict_Check(globalsns)) { + _PyErr_Format(tstate, PyExc_TypeError, + "unsupported globals %R", globalsns); + return -1; + } + // Check the builtins. + PyObject *builtinsns = _PyFunction_GET_BUILTINS(func); + if (builtinsns != NULL && !PyDict_Check(builtinsns)) { + _PyErr_Format(tstate, PyExc_TypeError, + "unsupported builtins %R", builtinsns); + return -1; + } + // Disallow __defaults__. + PyObject *defaults = PyFunction_GET_DEFAULTS(func); + if (defaults != NULL) { + assert(PyTuple_Check(defaults)); // per PyFunction_New() + if (PyTuple_GET_SIZE(defaults) > 0) { + _PyErr_SetString(tstate, PyExc_ValueError, + "defaults not supported"); + return -1; + } + } + // Disallow __kwdefaults__. + PyObject *kwdefaults = PyFunction_GET_KW_DEFAULTS(func); + if (kwdefaults != NULL) { + assert(PyDict_Check(kwdefaults)); // per PyFunction_New() + if (PyDict_Size(kwdefaults) > 0) { + _PyErr_SetString(tstate, PyExc_ValueError, + "keyword defaults not supported"); + return -1; + } + } + // Disallow __closure__. + PyObject *closure = PyFunction_GET_CLOSURE(func); + if (closure != NULL) { + assert(PyTuple_Check(closure)); // per PyFunction_New() + if (PyTuple_GET_SIZE(closure) > 0) { + _PyErr_SetString(tstate, PyExc_ValueError, "closures not supported"); + return -1; + } + } + // Check the code. + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + if (_PyCode_VerifyStateless(tstate, co, NULL, globalsns, builtinsns) < 0) { + return -1; + } + return 0; +} + + static int functools_copy_attr(PyObject *wrapper, PyObject *wrapped, PyObject *name) { @@ -1411,6 +1470,19 @@ cm_descr_get(PyObject *self, PyObject *obj, PyObject *type) return PyMethod_New(cm->cm_callable, type); } +static PyObject * +cm_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + classmethod *cm = (classmethod *)PyType_GenericAlloc(type, 0); + if (cm == NULL) { + return NULL; + } + cm->cm_callable = Py_None; + cm->cm_dict = NULL; + _PyObject_SetDeferredRefcount((PyObject *)cm); + return (PyObject *)cm; +} + static int cm_init(PyObject *self, PyObject *args, PyObject *kwds) { @@ -1557,7 +1629,7 @@ PyTypeObject PyClassMethod_Type = { offsetof(classmethod, cm_dict), /* tp_dictoffset */ cm_init, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ + cm_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ }; @@ -1642,6 +1714,19 @@ sm_descr_get(PyObject *self, PyObject *obj, PyObject *type) return Py_NewRef(sm->sm_callable); } +static PyObject * +sm_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + staticmethod *sm = (staticmethod *)PyType_GenericAlloc(type, 0); + if (sm == NULL) { + return NULL; + } + sm->sm_callable = Py_None; + sm->sm_dict = NULL; + _PyObject_SetDeferredRefcount((PyObject *)sm); + return (PyObject *)sm; +} + static int sm_init(PyObject *self, PyObject *args, PyObject *kwds) { @@ -1792,7 +1877,7 @@ PyTypeObject PyStaticMethod_Type = { offsetof(staticmethod, sm_dict), /* tp_dictoffset */ sm_init, /* tp_init */ PyType_GenericAlloc, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ + sm_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ }; @@ -1806,3 +1891,17 @@ PyStaticMethod_New(PyObject *callable) } return (PyObject *)sm; } + +PyObject * +_PyClassMethod_GetFunc(PyObject *self) +{ + classmethod *cm = _PyClassMethod_CAST(self); + return cm->cm_callable; +} + +PyObject * +_PyStaticMethod_GetFunc(PyObject *self) +{ + staticmethod *sm = _PyStaticMethod_CAST(self); + return sm->sm_callable; +} diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index ec3d01f00a3c3c2..010707cd0ef78a8 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -7,6 +7,7 @@ #include "pycore_typevarobject.h" // _Py_typing_type_repr #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() #include "pycore_unionobject.h" // _Py_union_type_or, _PyGenericAlias_Check +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stdbool.h> @@ -33,9 +34,7 @@ ga_dealloc(PyObject *self) gaobject *alias = (gaobject *)self; _PyObject_GC_UNTRACK(self); - if (alias->weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *)alias); - } + FT_CLEAR_WEAKREFS(self, alias->weakreflist); Py_XDECREF(alias->origin); Py_XDECREF(alias->args); Py_XDECREF(alias->parameters); @@ -65,14 +64,19 @@ ga_repr_items_list(PyUnicodeWriter *writer, PyObject *p) for (Py_ssize_t i = 0; i < len; i++) { if (i > 0) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { return -1; } } - PyObject *item = PyList_GET_ITEM(p, i); + PyObject *item = PyList_GetItemRef(p, i); + if (item == NULL) { + return -1; // list can be mutated in a callback + } if (_Py_typing_type_repr(writer, item) < 0) { + Py_DECREF(item); return -1; } + Py_DECREF(item); } if (PyUnicodeWriter_WriteChar(writer, ']') < 0) { @@ -109,7 +113,7 @@ ga_repr(PyObject *self) } for (Py_ssize_t i = 0; i < len; i++) { if (i > 0) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { goto error; } } @@ -126,7 +130,7 @@ ga_repr(PyObject *self) } if (len == 0) { // for something like tuple[()] we should print a "()" - if (PyUnicodeWriter_WriteUTF8(writer, "()", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, "()", 2) < 0) { goto error; } } @@ -182,20 +186,23 @@ PyObject * _Py_make_parameters(PyObject *args) { assert(PyTuple_Check(args) || PyList_Check(args)); + if (Py_EnterRecursiveCall(" in __parameter__ calculation")) { + return NULL; + } + const bool is_args_list = PyList_Check(args); PyObject *tuple_args = NULL; if (is_args_list) { args = tuple_args = PySequence_Tuple(args); if (args == NULL) { - return NULL; + goto cleanup; } } Py_ssize_t nargs = PyTuple_GET_SIZE(args); Py_ssize_t len = nargs; PyObject *parameters = PyTuple_New(len); if (parameters == NULL) { - Py_XDECREF(tuple_args); - return NULL; + goto error; } Py_ssize_t iparam = 0; for (Py_ssize_t iarg = 0; iarg < nargs; iarg++) { @@ -206,9 +213,7 @@ _Py_make_parameters(PyObject *args) } int rc = PyObject_HasAttrWithError(t, &_Py_ID(__typing_subst__)); if (rc < 0) { - Py_DECREF(parameters); - Py_XDECREF(tuple_args); - return NULL; + goto error; } if (rc) { iparam += tuple_add(parameters, iparam, t); @@ -217,18 +222,14 @@ _Py_make_parameters(PyObject *args) PyObject *subparams; if (PyObject_GetOptionalAttr(t, &_Py_ID(__parameters__), &subparams) < 0) { - Py_DECREF(parameters); - Py_XDECREF(tuple_args); - return NULL; + goto error; } if (!subparams && (PyTuple_Check(t) || PyList_Check(t))) { // Recursively call _Py_make_parameters for lists/tuples and // add the results to the current parameters. subparams = _Py_make_parameters(t); if (subparams == NULL) { - Py_DECREF(parameters); - Py_XDECREF(tuple_args); - return NULL; + goto error; } } if (subparams && PyTuple_Check(subparams)) { @@ -238,9 +239,8 @@ _Py_make_parameters(PyObject *args) len += needed; if (_PyTuple_Resize(&parameters, len) < 0) { Py_DECREF(subparams); - Py_DECREF(parameters); Py_XDECREF(tuple_args); - return NULL; + goto cleanup; } } for (Py_ssize_t j = 0; j < len2; j++) { @@ -253,13 +253,19 @@ _Py_make_parameters(PyObject *args) } if (iparam < len) { if (_PyTuple_Resize(&parameters, iparam) < 0) { - Py_XDECREF(parameters); - Py_XDECREF(tuple_args); - return NULL; + goto error; } } Py_XDECREF(tuple_args); + Py_LeaveRecursiveCall(); return parameters; + +error: + Py_XDECREF(parameters); + Py_XDECREF(tuple_args); +cleanup: + Py_LeaveRecursiveCall(); + return NULL; } /* If obj is a generic alias, substitute type variables params @@ -407,6 +413,9 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje self); } item = _unpack_args(item); + if (item == NULL) { + return NULL; + } for (Py_ssize_t i = 0; i < nparams; i++) { PyObject *param = PyTuple_GET_ITEM(parameters, i); PyObject *prepare, *tmp; @@ -526,12 +535,24 @@ _Py_subs_parameters(PyObject *self, PyObject *args, PyObject *parameters, PyObje return NULL; } if (unpack) { + if (!PyTuple_Check(arg)) { + Py_DECREF(newargs); + Py_DECREF(item); + Py_XDECREF(tuple_args); + PyObject *original = PyTuple_GET_ITEM(args, iarg); + PyErr_Format(PyExc_TypeError, + "expected __typing_subst__ of %T objects to return a tuple, not %T", + original, arg); + Py_DECREF(arg); + return NULL; + } jarg = tuple_extend(&newargs, jarg, &PyTuple_GET_ITEM(arg, 0), PyTuple_GET_SIZE(arg)); Py_DECREF(arg); if (jarg < 0) { Py_DECREF(item); Py_XDECREF(tuple_args); + assert(newargs == NULL); return NULL; } } @@ -551,7 +572,8 @@ PyDoc_STRVAR(genericalias__doc__, "--\n\n" "Represent a PEP 585 generic type\n" "\n" -"E.g. for t = list[int], t.__origin__ is list and t.__args__ is (int,)."); +"For example, for t = list[int], t.__origin__ is list and t.__args__\n" +"is (int,)."); static PyObject * ga_getitem(PyObject *self, PyObject *item) @@ -631,13 +653,12 @@ ga_vectorcall(PyObject *self, PyObject *const *args, size_t nargsf, PyObject *kwnames) { gaobject *alias = (gaobject *) self; - PyObject *obj = PyVectorcall_Function(alias->origin)(alias->origin, args, nargsf, kwnames); + PyObject *obj = PyObject_Vectorcall(alias->origin, args, nargsf, kwnames); return set_orig_class(obj, self); } static const char* const attr_exceptions[] = { "__class__", - "__bases__", "__origin__", "__args__", "__unpacked__", @@ -646,6 +667,11 @@ static const char* const attr_exceptions[] = { "__mro_entries__", "__reduce_ex__", // needed so we don't look up object.__reduce_ex__ "__reduce__", + NULL, +}; + +static const char* const attr_blocked[] = { + "__bases__", "__copy__", "__deepcopy__", NULL, @@ -656,15 +682,29 @@ ga_getattro(PyObject *self, PyObject *name) { gaobject *alias = (gaobject *)self; if (PyUnicode_Check(name)) { + // When we check blocked attrs, we don't allow to proxy them to `__origin__`. + // Otherwise, we can break existing code. + for (const char * const *p = attr_blocked; ; p++) { + if (*p == NULL) { + break; + } + if (_PyUnicode_EqualToASCIIString(name, *p)) { + goto generic_getattr; + } + } + + // When we see own attrs, it has a priority over `__origin__`'s attr. for (const char * const *p = attr_exceptions; ; p++) { if (*p == NULL) { return PyObject_GetAttr(alias->origin, name); } if (_PyUnicode_EqualToASCIIString(name, *p)) { - break; + goto generic_getattr; } } } + +generic_getattr: return PyObject_GenericGetAttr(self, name); } @@ -801,7 +841,7 @@ static PyMemberDef ga_members[] = { }; static PyObject * -ga_parameters(PyObject *self, void *unused) +ga_parameters_lock_held(PyObject *self) { gaobject *alias = (gaobject *)self; if (alias->parameters == NULL) { @@ -813,6 +853,16 @@ ga_parameters(PyObject *self, void *unused) return Py_NewRef(alias->parameters); } +static PyObject * +ga_parameters(PyObject *self, void *unused) +{ + PyObject *result; + Py_BEGIN_CRITICAL_SECTION(self); + result = ga_parameters_lock_held(self); + Py_END_CRITICAL_SECTION(); + return result; +} + static PyObject * ga_unpacked_tuple_args(PyObject *self, void *unused) { diff --git a/Objects/genobject.c b/Objects/genobject.c index 98b2c5004df8ac2..bb2b8fdf1598c8b 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -17,6 +17,7 @@ #include "pycore_pyerrors.h" // _PyErr_ClearExcState() #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_warnings.h" // _PyErr_WarnUnawaitedCoroutine() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "opcode_ids.h" // RESUME, etc @@ -161,8 +162,7 @@ gen_dealloc(PyObject *self) _PyObject_GC_UNTRACK(gen); - if (gen->gi_weakreflist != NULL) - PyObject_ClearWeakRefs(self); + FT_CLEAR_WEAKREFS(self, gen->gi_weakreflist); _PyObject_GC_TRACK(self); @@ -315,7 +315,7 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc, int closing) } PyDoc_STRVAR(send_doc, -"send(arg) -> send 'arg' into generator,\n\ +"send(value) -> send 'value' into generator,\n\ return next yielded value or raise StopIteration."); static PyObject * @@ -390,6 +390,7 @@ gen_close(PyObject *self, PyObject *args) if (gen->gi_frame_state == FRAME_CREATED) { gen->gi_frame_state = FRAME_COMPLETED; + gen_clear_frame(gen); Py_RETURN_NONE; } if (FRAME_STATE_FINISHED(gen->gi_frame_state)) { @@ -407,11 +408,12 @@ gen_close(PyObject *self, PyObject *args) } _PyInterpreterFrame *frame = &gen->gi_iframe; if (is_resume(frame->instr_ptr)) { + bool no_unwind_tools = _PyEval_NoToolsForUnwind(_PyThreadState_GET()); /* We can safely ignore the outermost try block * as it is automatically generated to handle * StopIteration. */ int oparg = frame->instr_ptr->op.arg; - if (oparg & RESUME_OPARG_DEPTH1_MASK) { + if (oparg & RESUME_OPARG_DEPTH1_MASK && no_unwind_tools) { // RESUME after YIELD_VALUE and exception depth is 1 assert((oparg & RESUME_OPARG_LOCATION_MASK) != RESUME_AT_FUNC_START); gen->gi_frame_state = FRAME_COMPLETED; @@ -844,7 +846,8 @@ static PyMethodDef gen_methods[] = { {"throw", _PyCFunction_CAST(gen_throw), METH_FASTCALL, throw_doc}, {"close", gen_close, METH_NOARGS, close_doc}, {"__sizeof__", gen_sizeof, METH_NOARGS, sizeof__doc__}, - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("generators are generic over the types of their yield, send, and return values")}, {NULL, NULL} /* Sentinel */ }; @@ -922,6 +925,7 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) gen->gi_weakreflist = NULL; gen->gi_exc_state.exc_value = NULL; gen->gi_exc_state.previous_item = NULL; + gen->gi_iframe.f_executable = PyStackRef_None; assert(func->func_name != NULL); gen->gi_name = Py_NewRef(func->func_name); assert(func->func_qualname != NULL); @@ -1204,7 +1208,8 @@ static PyMethodDef coro_methods[] = { {"throw",_PyCFunction_CAST(gen_throw), METH_FASTCALL, coro_throw_doc}, {"close", gen_close, METH_NOARGS, coro_close_doc}, {"__sizeof__", gen_sizeof, METH_NOARGS, sizeof__doc__}, - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("coroutines are generic over the types of their yield, send, and return values")}, {NULL, NULL} /* Sentinel */ }; @@ -1636,7 +1641,7 @@ static PyMethodDef async_gen_methods[] = { {"aclose", async_gen_aclose, METH_NOARGS, async_aclose_doc}, {"__sizeof__", gen_sizeof, METH_NOARGS, sizeof__doc__}, {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, PyDoc_STR("async generators are generic over the types of their yield and send values")}, {NULL, NULL} /* Sentinel */ }; diff --git a/Objects/interpolationobject.c b/Objects/interpolationobject.c index aaea3b8c0670c92..e37724fb7852a27 100644 --- a/Objects/interpolationobject.c +++ b/Objects/interpolationobject.c @@ -54,7 +54,7 @@ typedef struct { Interpolation.__new__ as interpolation_new value: object - expression: object(subclass_of='&PyUnicode_Type') + expression: object(subclass_of='&PyUnicode_Type', c_default='&_Py_STR(empty)') = "" conversion: object(converter='_conversion_converter') = None format_spec: object(subclass_of='&PyUnicode_Type', c_default='&_Py_STR(empty)') = "" [clinic start generated code]*/ @@ -63,7 +63,7 @@ static PyObject * interpolation_new_impl(PyTypeObject *type, PyObject *value, PyObject *expression, PyObject *conversion, PyObject *format_spec) -/*[clinic end generated code: output=6488e288765bc1a9 input=d91711024068528c]*/ +/*[clinic end generated code: output=6488e288765bc1a9 input=fc5c285c1dd23d36]*/ { interpolationobject *self = PyObject_GC_New(interpolationobject, type); if (!self) { @@ -137,6 +137,8 @@ interpolation_reduce(PyObject *op, PyObject *Py_UNUSED(dummy)) static PyMethodDef interpolation_methods[] = { {"__reduce__", interpolation_reduce, METH_NOARGS, PyDoc_STR("__reduce__() -> (cls, state)")}, + {"__class_getitem__", Py_GenericAlias, + METH_O|METH_CLASS, PyDoc_STR("Interpolations are generic over the types of their values")}, {NULL, NULL}, }; diff --git a/Objects/listobject.c b/Objects/listobject.c index c5895645a2dd121..bd5d21b2ac12c7b 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -61,7 +61,8 @@ free_list_items(PyObject** items, bool use_qsbr) #ifdef Py_GIL_DISABLED _PyListArray *array = _Py_CONTAINER_OF(items, _PyListArray, ob_item); if (use_qsbr) { - _PyMem_FreeDelayed(array); + size_t size = sizeof(_PyListArray) + array->allocated * sizeof(PyObject *); + _PyMem_FreeDelayed(array, size); } else { PyMem_Free(array); @@ -594,7 +595,7 @@ list_repr_impl(PyListObject *v) so must refetch the list size on each iteration. */ for (Py_ssize_t i = 0; i < Py_SIZE(v); ++i) { /* Hold a strong reference since repr(item) can mutate the list */ - item = Py_NewRef(v->ob_item[i]); + item = Py_XNewRef(v->ob_item[i]); if (i > 0) { if (PyUnicodeWriter_WriteChar(writer, ',') < 0) { @@ -2886,18 +2887,18 @@ list.sort Sort the list in ascending order and return None. -The sort is in-place (i.e. the list itself is modified) and stable (i.e. the -order of two equal elements is maintained). +The sort is in-place (i.e. the list itself is modified) and stable +(i.e. the order of two equal elements is maintained). -If a key function is given, apply it once to each list item and sort them, -ascending or descending, according to their function values. +If a key function is given, apply it once to each list item and sort +them, ascending or descending, according to their function values. The reverse flag can be set to sort in descending order. [clinic start generated code]*/ static PyObject * list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) -/*[clinic end generated code: output=57b9f9c5e23fbe42 input=667bf25d0e3a3676]*/ +/*[clinic end generated code: output=57b9f9c5e23fbe42 input=c145526281e1fb9f]*/ { MergeState ms; Py_ssize_t nremaining; @@ -3537,8 +3538,14 @@ list___sizeof___impl(PyListObject *self) /*[clinic end generated code: output=3417541f95f9a53e input=b8030a5d5ce8a187]*/ { size_t res = _PyObject_SIZE(Py_TYPE(self)); - Py_ssize_t allocated = FT_ATOMIC_LOAD_SSIZE_RELAXED(self->allocated); - res += (size_t)allocated * sizeof(void*); +#ifdef Py_GIL_DISABLED + PyObject **ob_item = _Py_atomic_load_ptr(&self->ob_item); + if (ob_item != NULL) { + res += list_capacity(ob_item) * sizeof(PyObject *); + } +#else + res += (size_t)self->allocated * sizeof(PyObject *); +#endif return PyLong_FromSize_t(res); } @@ -3561,7 +3568,8 @@ static PyMethodDef list_methods[] = { LIST_COUNT_METHODDEF LIST_REVERSE_METHODDEF LIST_SORT_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("lists are generic over the type of their contents")}, {NULL, NULL} /* sentinel */ }; diff --git a/Objects/longobject.c b/Objects/longobject.c index 40d90ecf4fa0682..daa1f9005e206ee 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -24,7 +24,7 @@ class int "PyObject *" "&PyLong_Type" #define medium_value(x) ((stwodigits)_PyLong_CompactValue(x)) -#define IS_SMALL_INT(ival) (-_PY_NSMALLNEGINTS <= (ival) && (ival) < _PY_NSMALLPOSINTS) +#define IS_SMALL_INT(ival) _PY_IS_SMALL_INT(ival) #define IS_SMALL_UINT(ival) ((ival) < _PY_NSMALLPOSINTS) #define _MAX_STR_DIGITS_ERROR_FMT_TO_INT "Exceeds the limit (%d digits) for integer string conversion: value has %zd digits; use sys.set_int_max_str_digits() to increase the limit" @@ -324,7 +324,7 @@ _PyLong_Negate(PyLongObject **x_p) PyLongObject *x; x = (PyLongObject *)*x_p; - if (Py_REFCNT(x) == 1) { + if (_PyObject_IsUniquelyReferenced((PyObject *)x)) { _PyLong_FlipSign(x); return; } @@ -971,16 +971,9 @@ _PyLong_FromByteArray(const unsigned char* bytes, size_t n, ++numsignificantbytes; } - /* How many Python int digits do we need? We have - 8*numsignificantbytes bits, and each Python int digit has - PyLong_SHIFT bits, so it's the ceiling of the quotient. */ - /* catch overflow before it happens */ - if (numsignificantbytes > (PY_SSIZE_T_MAX - PyLong_SHIFT) / 8) { - PyErr_SetString(PyExc_OverflowError, - "byte array too long to convert to int"); - return NULL; - } - ndigits = (numsignificantbytes * 8 + PyLong_SHIFT - 1) / PyLong_SHIFT; + /* avoid integer overflow */ + ndigits = numsignificantbytes / PyLong_SHIFT * 8 + + (numsignificantbytes % PyLong_SHIFT * 8 + PyLong_SHIFT - 1) / PyLong_SHIFT; v = long_alloc(ndigits); if (v == NULL) return NULL; @@ -1145,13 +1138,19 @@ _PyLong_AsByteArray(PyLongObject* v, *p = (unsigned char)(accum & 0xff); p += pincr; } - else if (j == n && n > 0 && is_signed) { + else if (j == n && is_signed) { /* The main loop filled the byte array exactly, so the code just above didn't get to ensure there's a sign bit, and the loop below wouldn't add one either. Make sure a sign bit exists. */ - unsigned char msb = *(p - pincr); - int sign_bit_set = msb >= 0x80; + int sign_bit_set; + if (n > 0) { + unsigned char msb = *(p - pincr); + sign_bit_set = msb >= 0x80; + } + else { + sign_bit_set = 0; + } assert(accumbits == 0); if (sign_bit_set == do_twos_comp) return 0; @@ -1760,6 +1759,10 @@ UNSIGNED_INT_CONVERTER(UnsignedInt, unsigned int) UNSIGNED_INT_CONVERTER(UnsignedLong, unsigned long) UNSIGNED_INT_CONVERTER(UnsignedLongLong, unsigned long long) UNSIGNED_INT_CONVERTER(Size_t, size_t) +UNSIGNED_INT_CONVERTER(UInt8, uint8_t) +UNSIGNED_INT_CONVERTER(UInt16, uint16_t) +UNSIGNED_INT_CONVERTER(UInt32, uint32_t) +UNSIGNED_INT_CONVERTER(UInt64, uint64_t) #define CHECK_BINOP(v,w) \ @@ -2046,7 +2049,7 @@ long_to_decimal_string_internal(PyObject *aa, if (size_a >= 10 * _PY_LONG_MAX_STR_DIGITS_THRESHOLD / (3 * PyLong_SHIFT) + 2) { PyInterpreterState *interp = _PyInterpreterState_GET(); - int max_str_digits = interp->long_state.max_str_digits; + int max_str_digits = _Py_atomic_load_int(&interp->long_state.max_str_digits); if ((max_str_digits > 0) && (max_str_digits / (3 * PyLong_SHIFT) <= (size_a - 11) / 10)) { PyErr_Format(PyExc_ValueError, _MAX_STR_DIGITS_ERROR_FMT_TO_STR, @@ -2127,7 +2130,7 @@ long_to_decimal_string_internal(PyObject *aa, } if (strlen > _PY_LONG_MAX_STR_DIGITS_THRESHOLD) { PyInterpreterState *interp = _PyInterpreterState_GET(); - int max_str_digits = interp->long_state.max_str_digits; + int max_str_digits = _Py_atomic_load_int(&interp->long_state.max_str_digits); Py_ssize_t strlen_nosign = strlen - negative; if ((max_str_digits > 0) && (strlen_nosign > max_str_digits)) { Py_DECREF(scratch); @@ -2939,7 +2942,7 @@ long_from_string_base(const char **str, int base, PyLongObject **res) * quadratic algorithm. */ if (digits > _PY_LONG_MAX_STR_DIGITS_THRESHOLD) { PyInterpreterState *interp = _PyInterpreterState_GET(); - int max_str_digits = interp->long_state.max_str_digits; + int max_str_digits = _Py_atomic_load_int(&interp->long_state.max_str_digits); if ((max_str_digits > 0) && (digits > max_str_digits)) { PyErr_Format(PyExc_ValueError, _MAX_STR_DIGITS_ERROR_FMT_TO_INT, max_str_digits, digits); @@ -3044,11 +3047,11 @@ PyLong_FromString(const char *str, char **pend, int base) } /* Set sign and normalize */ - if (sign < 0) { - _PyLong_FlipSign(z); - } long_normalize(z); z = maybe_small_long(z); + if (sign < 0) { + _PyLong_Negate(&z); + } if (pend != NULL) { *pend = (char *)str; @@ -3548,21 +3551,11 @@ long_richcompare(PyObject *self, PyObject *other, int op) Py_RETURN_RICHCOMPARE(result, 0, op); } -static inline int -/// Return 1 if the object is one of the immortal small ints -_long_is_small_int(PyObject *op) -{ - PyLongObject *long_object = (PyLongObject *)op; - int is_small_int = (long_object->long_value.lv_tag & IMMORTALITY_BIT_MASK) != 0; - assert((!is_small_int) || PyLong_CheckExact(op)); - return is_small_int; -} - void _PyLong_ExactDealloc(PyObject *self) { assert(PyLong_CheckExact(self)); - if (_long_is_small_int(self)) { + if (_PyLong_IsSmallInt((PyLongObject *)self)) { // See PEP 683, section Accidental De-Immortalizing for details _Py_SetImmortal(self); return; @@ -3577,7 +3570,7 @@ _PyLong_ExactDealloc(PyObject *self) static void long_dealloc(PyObject *self) { - if (_long_is_small_int(self)) { + if (_PyLong_IsSmallInt((PyLongObject *)self)) { /* This should never get called, but we also don't want to SEGV if * we accidentally decref small Ints out of existence. Instead, * since small Ints are immortal, re-set the reference count. @@ -4336,10 +4329,10 @@ pylong_int_divmod(PyLongObject *v, PyLongObject *w, if (result == NULL) { return -1; } - if (!PyTuple_Check(result)) { + if (!PyTuple_Check(result) || PyTuple_GET_SIZE(result) != 2) { Py_DECREF(result); PyErr_SetString(PyExc_ValueError, - "tuple is required from int_divmod()"); + "tuple of length 2 is required from int_divmod()"); return -1; } PyObject *q = PyTuple_GET_ITEM(result, 0); @@ -5751,7 +5744,7 @@ _PyLong_GCD(PyObject *aarg, PyObject *barg) assert(size_a >= 0); _PyLong_SetSignAndDigitCount(c, 1, size_a); } - else if (Py_REFCNT(a) == 1) { + else if (_PyObject_IsUniquelyReferenced((PyObject *)a)) { c = (PyLongObject*)Py_NewRef(a); } else { @@ -5765,7 +5758,8 @@ _PyLong_GCD(PyObject *aarg, PyObject *barg) assert(size_a >= 0); _PyLong_SetSignAndDigitCount(d, 1, size_a); } - else if (Py_REFCNT(b) == 1 && size_a <= alloc_b) { + else if (_PyObject_IsUniquelyReferenced((PyObject *)b) + && size_a <= alloc_b) { d = (PyLongObject*)Py_NewRef(b); assert(size_a >= 0); _PyLong_SetSignAndDigitCount(d, 1, size_a); @@ -6283,20 +6277,21 @@ int_as_integer_ratio_impl(PyObject *self) int.to_bytes length: Py_ssize_t = 1 - Length of bytes object to use. An OverflowError is raised if the - integer is not representable with the given number of bytes. Default - is length 1. + Length of bytes object to use. An OverflowError is raised if + the integer is not representable with the given number of bytes. + Default is length 1. byteorder: unicode(c_default="NULL") = "big" - The byte order used to represent the integer. If byteorder is 'big', - the most significant byte is at the beginning of the byte array. If - byteorder is 'little', the most significant byte is at the end of the - byte array. To request the native byte order of the host system, use - sys.byteorder as the byte order value. Default is to use 'big'. + The byte order used to represent the integer. If byteorder is + 'big', the most significant byte is at the beginning of the byte + array. If byteorder is 'little', the most significant byte is at + the end of the byte array. To request the native byte order of + the host system, use sys.byteorder as the byte order value. + Default is to use 'big'. * signed as is_signed: bool = False - Determines whether two's complement is used to represent the integer. - If signed is False and a negative integer is given, an OverflowError - is raised. + Determines whether two's complement is used to represent the + integer. If signed is False and a negative integer is given, + an OverflowError is raised. Return an array of bytes representing an integer. [clinic start generated code]*/ @@ -6304,7 +6299,7 @@ Return an array of bytes representing an integer. static PyObject * int_to_bytes_impl(PyObject *self, Py_ssize_t length, PyObject *byteorder, int is_signed) -/*[clinic end generated code: output=89c801df114050a3 input=a0103d0e9ad85c2b]*/ +/*[clinic end generated code: output=89c801df114050a3 input=661d7e615a4f132d]*/ { int little_endian; PyObject *bytes; @@ -6347,18 +6342,20 @@ int.from_bytes bytes as bytes_obj: object Holds the array of bytes to convert. The argument must either - support the buffer protocol or be an iterable object producing bytes. - Bytes and bytearray are examples of built-in objects that support the - buffer protocol. + support the buffer protocol or be an iterable object producing + bytes. Bytes and bytearray are examples of built-in objects that + support the buffer protocol. byteorder: unicode(c_default="NULL") = "big" - The byte order used to represent the integer. If byteorder is 'big', - the most significant byte is at the beginning of the byte array. If - byteorder is 'little', the most significant byte is at the end of the - byte array. To request the native byte order of the host system, use - sys.byteorder as the byte order value. Default is to use 'big'. + The byte order used to represent the integer. If byteorder is + 'big', the most significant byte is at the beginning of the byte + array. If byteorder is 'little', the most significant byte is at + the end of the byte array. To request the native byte order of + the host system, use sys.byteorder as the byte order value. + Default is to use 'big'. * signed as is_signed: bool = False - Indicates whether two's complement is used to represent the integer. + Indicates whether two's complement is used to represent the + integer. Return the integer represented by the given array of bytes. [clinic start generated code]*/ @@ -6366,7 +6363,7 @@ Return the integer represented by the given array of bytes. static PyObject * int_from_bytes_impl(PyTypeObject *type, PyObject *bytes_obj, PyObject *byteorder, int is_signed) -/*[clinic end generated code: output=efc5d68e31f9314f input=2ff527997fe7b0c5]*/ +/*[clinic end generated code: output=efc5d68e31f9314f input=95801e50b942e164]*/ { int little_endian; PyObject *long_obj, *bytes; @@ -6501,7 +6498,8 @@ If x is not a number or if base is given, then x must be a string,\n\ bytes, or bytearray instance representing an integer literal in the\n\ given base. The literal can be preceded by '+' or '-' and be surrounded\n\ by whitespace. The base defaults to 10. Valid bases are 0 and 2-36.\n\ -Base 0 means to interpret the base from the string as an integer literal.\n\ +Base 0 means to interpret the base from the string as an integer\n\ +iteral.\n\ >>> int('0b100', base=0)\n\ 4"); diff --git a/Objects/memoryobject.c b/Objects/memoryobject.c index cf673fb379edcd9..f3677f0e5384a88 100644 --- a/Objects/memoryobject.c +++ b/Objects/memoryobject.c @@ -1600,11 +1600,7 @@ memory_getbuf(PyObject *_self, Py_buffer *view, int flags) view->obj = Py_NewRef(self); -#ifdef Py_GIL_DISABLED - _Py_atomic_add_ssize(&self->exports, 1); -#else - self->exports++; -#endif + FT_ATOMIC_ADD_SSIZE(self->exports, 1); return 0; } @@ -1613,11 +1609,7 @@ static void memory_releasebuf(PyObject *_self, Py_buffer *view) { PyMemoryViewObject *self = (PyMemoryViewObject *)_self; -#ifdef Py_GIL_DISABLED - _Py_atomic_add_ssize(&self->exports, -1); -#else - self->exports--; -#endif + FT_ATOMIC_ADD_SSIZE(self->exports, -1); return; /* PyBuffer_Release() decrements view->obj after this function returns. */ } @@ -1672,6 +1664,10 @@ fix_error_int(const char *fmt) return -1; } +// UNPACK_TO_BOOL: Return 0 if PTR represents "false", and 1 otherwise. +static const _Bool bool_false = 0; +#define UNPACK_TO_BOOL(PTR) (memcmp((PTR), &bool_false, sizeof(_Bool)) != 0) + /* Accept integer objects or objects with an __index__() method. */ static long pylong_as_ld(PyObject *item) @@ -1807,7 +1803,7 @@ unpack_single(PyMemoryViewObject *self, const char *ptr, const char *fmt) case 'l': UNPACK_SINGLE(ld, ptr, long); goto convert_ld; /* boolean */ - case '?': UNPACK_SINGLE(ld, ptr, _Bool); goto convert_bool; + case '?': ld = UNPACK_TO_BOOL(ptr); goto convert_bool; /* unsigned integers */ case 'H': UNPACK_SINGLE(lu, ptr, unsigned short); goto convert_lu; @@ -2271,16 +2267,17 @@ memoryview.tobytes Return the data in the buffer as a byte string. -Order can be {'C', 'F', 'A'}. When order is 'C' or 'F', the data of the -original array is converted to C or Fortran order. For contiguous views, -'A' returns an exact copy of the physical memory. In particular, in-memory -Fortran order is preserved. For non-contiguous views, the data is converted -to C first. order=None is the same as order='C'. +Order can be {'C', 'F', 'A'}. When order is 'C' or 'F', the data of +the original array is converted to C or Fortran order. For +contiguous views, 'A' returns an exact copy of the physical memory. +In particular, in-memory Fortran order is preserved. For +non-contiguous views, the data is converted to C first. order=None +is the same as order='C'. [clinic start generated code]*/ static PyObject * memoryview_tobytes_impl(PyMemoryViewObject *self, const char *order) -/*[clinic end generated code: output=1288b62560a32a23 input=0efa3ddaeda573a8]*/ +/*[clinic end generated code: output=1288b62560a32a23 input=119c70aa91791dc8]*/ { Py_buffer *src = VIEW_ADDR(self); char ord = 'C'; @@ -2320,8 +2317,8 @@ memoryview.hex sep: object = NULL An optional single character or byte to separate hex bytes. bytes_per_sep: int = 1 - How many bytes between separators. Positive values count from the - right, negative values count from the left. + How many bytes between separators. Positive values count from + the right, negative values count from the left. Return the data in the buffer as a str of hexadecimal numbers. @@ -2340,7 +2337,7 @@ Return the data in the buffer as a str of hexadecimal numbers. static PyObject * memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, int bytes_per_sep) -/*[clinic end generated code: output=430ca760f94f3ca7 input=539f6a3a5fb56946]*/ +/*[clinic end generated code: output=430ca760f94f3ca7 input=94c2495f886c786b]*/ { Py_buffer *src = VIEW_ADDR(self); PyObject *bytes; @@ -2349,7 +2346,13 @@ memoryview_hex_impl(PyMemoryViewObject *self, PyObject *sep, CHECK_RELEASED(self); if (MV_C_CONTIGUOUS(self->flags)) { - return _Py_strhex_with_sep(src->buf, src->len, sep, bytes_per_sep); + // Prevent 'self' from being freed if computing len(sep) mutates 'self' + // in _Py_strhex_with_sep(). + // See: https://github.com/python/cpython/issues/143195. + FT_ATOMIC_ADD_SSIZE(self->exports, 1); + PyObject *ret = _Py_strhex_with_sep(src->buf, src->len, sep, bytes_per_sep); + FT_ATOMIC_ADD_SSIZE(self->exports, -1); + return ret; } bytes = PyBytes_FromStringAndSize(NULL, src->len); @@ -2426,7 +2429,7 @@ ptr_from_tuple(const Py_buffer *view, PyObject *tup) if (nindices > view->ndim) { PyErr_Format(PyExc_TypeError, - "cannot index %zd-dimension view with %zd-element tuple", + "cannot index %d-dimension view with %zd-element tuple", view->ndim, nindices); return NULL; } @@ -2983,7 +2986,7 @@ unpack_cmp(const char *p, const char *q, char fmt, case 'l': CMP_SINGLE(p, q, long); return equal; /* boolean */ - case '?': CMP_SINGLE(p, q, _Bool); return equal; + case '?': return UNPACK_TO_BOOL(p) == UNPACK_TO_BOOL(q); /* unsigned integers */ case 'H': CMP_SINGLE(p, q, unsigned short); return equal; @@ -3222,9 +3225,16 @@ memory_hash(PyObject *_self) "memoryview: hashing is restricted to formats 'B', 'b' or 'c'"); return -1; } - if (view->obj != NULL && PyObject_Hash(view->obj) == -1) { - /* Keep the original error message */ - return -1; + if (view->obj != NULL) { + // Prevent 'self' from being freed when computing the item's hash. + // See https://github.com/python/cpython/issues/142664. + FT_ATOMIC_ADD_SSIZE(self->exports, 1); + Py_hash_t h = PyObject_Hash(view->obj); + FT_ATOMIC_ADD_SSIZE(self->exports, -1); + if (h == -1) { + /* Keep the original error message */ + return -1; + } } if (!MV_C_CONTIGUOUS(self->flags)) { @@ -3442,7 +3452,8 @@ static PyMethodDef memory_methods[] = { MEMORYVIEW_INDEX_METHODDEF {"__enter__", memory_enter, METH_NOARGS, NULL}, {"__exit__", memory_exit, METH_VARARGS, memory_exit_doc}, - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("memoryviews are generic over the type of their underlying data")}, {NULL, NULL} }; diff --git a/Objects/methodobject.c b/Objects/methodobject.c index c3dcd09ad1cdb6f..e6e469ca270ac96 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -8,6 +8,7 @@ #include "pycore_object.h" #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() /* undefine macro trampoline to PyCFunction_NewEx */ @@ -167,9 +168,7 @@ meth_dealloc(PyObject *self) { PyCFunctionObject *m = _PyCFunctionObject_CAST(self); PyObject_GC_UnTrack(m); - if (m->m_weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject*) m); - } + FT_CLEAR_WEAKREFS(self, m->m_weakreflist); // We need to access ml_flags here rather than later. // `m->m_ml` might have the same lifetime // as `m_self` when it's dynamically allocated. diff --git a/Objects/mimalloc/heap.c b/Objects/mimalloc/heap.c index d92dc768e5ec284..5fbfb82baa02040 100644 --- a/Objects/mimalloc/heap.c +++ b/Objects/mimalloc/heap.c @@ -100,7 +100,10 @@ static bool mi_heap_page_collect(mi_heap_t* heap, mi_page_queue_t* pq, mi_page_t // note: this will free retired pages as well. bool freed = _PyMem_mi_page_maybe_free(page, pq, collect >= MI_FORCE); if (!freed && collect == MI_ABANDON) { - _mi_page_abandon(page, pq); + // _PyMem_mi_page_maybe_free may have moved the page to a different + // page queue, so we need to re-fetch the correct queue. + uint8_t bin = (mi_page_is_in_full(page) ? MI_BIN_FULL : _mi_bin(page->xblock_size)); + _mi_page_abandon(page, &heap->pages[bin]); } } else if (collect == MI_ABANDON) { diff --git a/Objects/mimalloc/init.c b/Objects/mimalloc/init.c index 81b241063ff40fc..7711c827a58b1ca 100644 --- a/Objects/mimalloc/init.c +++ b/Objects/mimalloc/init.c @@ -183,9 +183,9 @@ mi_heap_t* _mi_heap_main_get(void) { // note: in x64 in release build `sizeof(mi_thread_data_t)` is under 4KiB (= OS page size). typedef struct mi_thread_data_s { - mi_heap_t heap; // must come first due to cast in `_mi_heap_done` + mi_heap_t heap; // must come first due to cast in `_mi_heap_done` mi_tld_t tld; - mi_memid_t memid; + mi_memid_t memid; // must come last due to zero'ing } mi_thread_data_t; @@ -231,7 +231,7 @@ static mi_thread_data_t* mi_thread_data_zalloc(void) { } if (td != NULL && !is_zero) { - _mi_memzero_aligned(td, sizeof(*td)); + _mi_memzero_aligned(td, offsetof(mi_thread_data_t,memid)); } return td; } diff --git a/Objects/mimalloc/page.c b/Objects/mimalloc/page.c index ff7444cce10923c..ded59f8eb1ccaac 100644 --- a/Objects/mimalloc/page.c +++ b/Objects/mimalloc/page.c @@ -213,6 +213,13 @@ static void _mi_page_thread_free_collect(mi_page_t* page) // update counts now page->used -= count; + + if (page->used == 0) { + // The page may have had a QSBR goal set from a previous point when it + // was all-free. That goal is no longer valid because the page was + // allocated from and then freed again by other threads. + _PyMem_mi_page_clear_qsbr(page); + } } void _mi_page_free_collect(mi_page_t* page, bool force) { @@ -225,9 +232,6 @@ void _mi_page_free_collect(mi_page_t* page, bool force) { // and the local free list if (page->local_free != NULL) { - // any previous QSBR goals are no longer valid because we reused the page - _PyMem_mi_page_clear_qsbr(page); - if mi_likely(page->free == NULL) { // usual case page->free = page->local_free; diff --git a/Objects/mimalloc/segment.c b/Objects/mimalloc/segment.c index 9b092b9b734d4c5..9dad69c995e7a05 100644 --- a/Objects/mimalloc/segment.c +++ b/Objects/mimalloc/segment.c @@ -1286,6 +1286,7 @@ static bool mi_segment_check_free(mi_segment_t* segment, size_t slices_needed, s _mi_stat_decrease(&tld->stats->pages_abandoned, 1); #ifdef Py_GIL_DISABLED page->qsbr_goal = 0; + mi_assert_internal(page->qsbr_node.next == NULL); #endif segment->abandoned--; slice = mi_segment_page_clear(page, tld); // re-assign slice due to coalesce! @@ -1361,6 +1362,7 @@ static mi_segment_t* mi_segment_reclaim(mi_segment_t* segment, mi_heap_t* heap, // if everything free by now, free the page #ifdef Py_GIL_DISABLED page->qsbr_goal = 0; + mi_assert_internal(page->qsbr_node.next == NULL); #endif slice = mi_segment_page_clear(page, tld); // set slice again due to coalesceing } diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index f363ef173cbd469..b68584b5dd571de 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -14,6 +14,7 @@ #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_sysmodule.h" // _PySys_GetOptionalAttrString() #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "osdefs.h" // MAXPATHLEN @@ -827,8 +828,7 @@ module_dealloc(PyObject *self) if (verbose && m->md_name) { PySys_FormatStderr("# destroy %U\n", m->md_name); } - if (m->md_weaklist != NULL) - PyObject_ClearWeakRefs((PyObject *) m); + FT_CLEAR_WEAKREFS(self, m->md_weaklist); /* bpo-39824: Don't call m_free() if m_size > 0 and md_state=NULL */ if (m->md_def && m->md_def->m_free diff --git a/Objects/namespaceobject.c b/Objects/namespaceobject.c index caebe6bf5435675..f39f1b35ebb6fd9 100644 --- a/Objects/namespaceobject.c +++ b/Objects/namespaceobject.c @@ -13,6 +13,7 @@ typedef struct { } _PyNamespaceObject; #define _PyNamespace_CAST(op) _Py_CAST(_PyNamespaceObject*, (op)) +#define _PyNamespace_Check(op) PyObject_TypeCheck((op), &_PyNamespace_Type) static PyMemberDef namespace_members[] = { @@ -124,9 +125,10 @@ namespace_repr(PyObject *ns) if (PyUnicode_Check(key) && PyUnicode_GET_LENGTH(key) > 0) { PyObject *value, *item; - value = PyDict_GetItemWithError(d, key); - if (value != NULL) { + int has_key = PyDict_GetItemRef(d, key, &value); + if (has_key == 1) { item = PyUnicode_FromFormat("%U=%R", key, value); + Py_DECREF(value); if (item == NULL) { loop_error = 1; } @@ -135,7 +137,7 @@ namespace_repr(PyObject *ns) Py_DECREF(item); } } - else if (PyErr_Occurred()) { + else if (has_key < 0) { loop_error = 1; } } @@ -229,6 +231,14 @@ namespace_replace(PyObject *self, PyObject *args, PyObject *kwargs) if (!result) { return NULL; } + if (!_PyNamespace_Check(result)) { + PyErr_Format(PyExc_TypeError, + "expect %N type, but %T() returned '%T' object", + &_PyNamespace_Type, self, result); + Py_DECREF(result); + return NULL; + } + if (PyDict_Update(((_PyNamespaceObject*)result)->ns_dict, ((_PyNamespaceObject*)self)->ns_dict) < 0) { diff --git a/Objects/object.c b/Objects/object.c index 723b0427e692515..3a104bcd932cd4f 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -10,6 +10,7 @@ #include "pycore_descrobject.h" // _PyMethodWrapper_Type #include "pycore_dict.h" // _PyObject_MaterializeManagedDict() #include "pycore_floatobject.h" // _PyFloat_DebugMallocStats() +#include "pycore_function.h" // _PyClassMethod_GetFunc() #include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_genobject.h" // _PyAsyncGenAThrow_Type #include "pycore_hamt.h" // _PyHamtItems_Type @@ -685,6 +686,8 @@ PyObject_Print(PyObject *op, FILE *fp, int flags) ret = -1; } } + + _Py_LeaveRecursiveCall(); return ret; } @@ -711,7 +714,7 @@ _PyObject_IsFreed(PyObject *op) } -/* For debugging convenience. See Misc/gdbinit for some useful gdb hooks */ +/* For debugging convenience. */ void _PyObject_Dump(PyObject* op) { @@ -1664,6 +1667,142 @@ _PyObject_GetMethod(PyObject *obj, PyObject *name, PyObject **method) return 0; } +// Look up a method on `self` by `name`. +// +// On success, `*method` is set and the function returns 0 or 1. If the +// return value is 1, the call is an unbound method and `*self` is the +// "self" or "cls" argument to pass. If the return value is 0, the call is +// a regular function and `*self` is cleared. +// +// On error, returns -1, clears `*self`, and sets an exception. +int +_PyObject_GetMethodStackRef(PyThreadState *ts, _PyStackRef *self, + PyObject *name, _PyStackRef *method) +{ + int meth_found = 0; + PyObject *obj = PyStackRef_AsPyObjectBorrow(*self); + + assert(PyStackRef_IsNull(*method)); + + PyTypeObject *tp = Py_TYPE(obj); + if (!_PyType_IsReady(tp)) { + if (PyType_Ready(tp) < 0) { + PyStackRef_CLEAR(*self); + return -1; + } + } + + if (tp->tp_getattro != PyObject_GenericGetAttr || !PyUnicode_CheckExact(name)) { + PyObject *res = PyObject_GetAttr(obj, name); + PyStackRef_CLEAR(*self); + if (res != NULL) { + *method = PyStackRef_FromPyObjectSteal(res); + return 0; + } + return -1; + } + + _PyType_LookupStackRefAndVersion(tp, name, method); + PyObject *descr = PyStackRef_AsPyObjectBorrow(*method); + descrgetfunc f = NULL; + if (descr != NULL) { + if (_PyType_HasFeature(Py_TYPE(descr), Py_TPFLAGS_METHOD_DESCRIPTOR)) { + meth_found = 1; + } + else { + f = Py_TYPE(descr)->tp_descr_get; + if (f != NULL && PyDescr_IsData(descr)) { + PyObject *value = f(descr, obj, (PyObject *)Py_TYPE(obj)); + PyStackRef_CLEAR(*method); + PyStackRef_CLEAR(*self); + if (value != NULL) { + *method = PyStackRef_FromPyObjectSteal(value); + return 0; + } + return -1; + } + } + } + PyObject *dict, *attr; + if ((tp->tp_flags & Py_TPFLAGS_INLINE_VALUES) && + _PyObject_TryGetInstanceAttribute(obj, name, &attr)) { + if (attr != NULL) { + PyStackRef_XSETREF(*method, PyStackRef_FromPyObjectSteal(attr)); + PyStackRef_CLEAR(*self); + return 0; + } + dict = NULL; + } + else if ((tp->tp_flags & Py_TPFLAGS_MANAGED_DICT)) { + dict = (PyObject *)_PyObject_GetManagedDict(obj); + } + else { + PyObject **dictptr = _PyObject_ComputedDictPointer(obj); + if (dictptr != NULL) { + dict = FT_ATOMIC_LOAD_PTR_ACQUIRE(*dictptr); + } + else { + dict = NULL; + } + } + if (dict != NULL) { + assert(PyUnicode_CheckExact(name)); + int found = _PyDict_GetMethodStackRef((PyDictObject *)dict, name, method); + if (found < 0) { + assert(PyStackRef_IsNull(*method)); + PyStackRef_CLEAR(*self); + return -1; + } + else if (found) { + PyStackRef_CLEAR(*self); + return 0; + } + } + + if (meth_found) { + assert(!PyStackRef_IsNull(*method)); + return 1; + } + + if (f != NULL) { + if (Py_IS_TYPE(descr, &PyClassMethod_Type)) { + PyObject *callable = _PyClassMethod_GetFunc(descr); + PyStackRef_XSETREF(*method, PyStackRef_FromPyObjectNew(callable)); + PyStackRef_XSETREF(*self, PyStackRef_FromPyObjectNew((PyObject *)tp)); + return 1; + } + else if (Py_IS_TYPE(descr, &PyStaticMethod_Type)) { + PyObject *callable = _PyStaticMethod_GetFunc(descr); + PyStackRef_XSETREF(*method, PyStackRef_FromPyObjectNew(callable)); + PyStackRef_CLEAR(*self); + return 0; + } + PyObject *value = f(descr, obj, (PyObject *)tp); + PyStackRef_CLEAR(*method); + PyStackRef_CLEAR(*self); + if (value) { + *method = PyStackRef_FromPyObjectSteal(value); + return 0; + } + return -1; + } + + if (descr != NULL) { + assert(!PyStackRef_IsNull(*method)); + PyStackRef_CLEAR(*self); + return 0; + } + + PyErr_Format(PyExc_AttributeError, + "'%.100s' object has no attribute '%U'", + tp->tp_name, name); + + _PyObject_SetAttributeErrorContext(obj, name); + assert(PyStackRef_IsNull(*method)); + PyStackRef_CLEAR(*self); + return -1; +} + /* Generic GetAttr functions - put these in your tp_[gs]etattro slot. */ PyObject * @@ -1817,7 +1956,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, } Py_INCREF(name); - Py_INCREF(tp); + _Py_INCREF_TYPE(tp); PyThreadState *tstate = _PyThreadState_GET(); _PyCStackRef cref; @@ -1892,7 +2031,7 @@ _PyObject_GenericSetAttrWithDict(PyObject *obj, PyObject *name, } done: _PyThreadState_PopCStackRef(tstate, &cref); - Py_DECREF(tp); + _Py_DECREF_TYPE(tp); Py_DECREF(name); return res; } @@ -1906,34 +2045,11 @@ PyObject_GenericSetAttr(PyObject *obj, PyObject *name, PyObject *value) int PyObject_GenericSetDict(PyObject *obj, PyObject *value, void *context) { - PyObject **dictptr = _PyObject_GetDictPtr(obj); - if (dictptr == NULL) { - if (_PyType_HasFeature(Py_TYPE(obj), Py_TPFLAGS_INLINE_VALUES) && - _PyObject_GetManagedDict(obj) == NULL - ) { - /* Was unable to convert to dict */ - PyErr_NoMemory(); - } - else { - PyErr_SetString(PyExc_AttributeError, - "This object has no __dict__"); - } - return -1; - } if (value == NULL) { PyErr_SetString(PyExc_TypeError, "cannot delete __dict__"); return -1; } - if (!PyDict_Check(value)) { - PyErr_Format(PyExc_TypeError, - "__dict__ must be set to a dictionary, " - "not a '%.200s'", Py_TYPE(value)->tp_name); - return -1; - } - Py_BEGIN_CRITICAL_SECTION(obj); - Py_XSETREF(*dictptr, Py_NewRef(value)); - Py_END_CRITICAL_SECTION(); - return 0; + return _PyObject_SetDict(obj, value); } @@ -1996,9 +2112,25 @@ _dir_locals(void) PyObject *names; PyObject *locals; - locals = _PyEval_GetFrameLocals(); - if (locals == NULL) + if (_PyEval_GetFrame() != NULL) { + locals = _PyEval_GetFrameLocals(); + } + else { + PyThreadState *tstate = _PyThreadState_GET(); + locals = _PyEval_GetGlobalsFromRunningMain(tstate); + if (locals == NULL) { + if (!_PyErr_Occurred(tstate)) { + locals = _PyEval_GetFrameLocals(); + assert(_PyErr_Occurred(tstate)); + } + } + else { + Py_INCREF(locals); + } + } + if (locals == NULL) { return NULL; + } names = PyMapping_Keys(locals); Py_DECREF(locals); @@ -2548,21 +2680,14 @@ _Py_NewReferenceNoTotal(PyObject *op) void _Py_SetImmortalUntracked(PyObject *op) { -#ifdef Py_DEBUG - // For strings, use _PyUnicode_InternImmortal instead. - if (PyUnicode_CheckExact(op)) { - assert(PyUnicode_CHECK_INTERNED(op) == SSTATE_INTERNED_IMMORTAL - || PyUnicode_CHECK_INTERNED(op) == SSTATE_INTERNED_IMMORTAL_STATIC); - } -#endif // Check if already immortal to avoid degrading from static immortal to plain immortal if (_Py_IsImmortal(op)) { return; } #ifdef Py_GIL_DISABLED - op->ob_tid = _Py_UNOWNED_TID; - op->ob_ref_local = _Py_IMMORTAL_REFCNT_LOCAL; - op->ob_ref_shared = 0; + _Py_atomic_store_uintptr_relaxed(&op->ob_tid, _Py_UNOWNED_TID); + _Py_atomic_store_uint32_relaxed(&op->ob_ref_local, _Py_IMMORTAL_REFCNT_LOCAL); + _Py_atomic_store_ssize_relaxed(&op->ob_ref_shared, 0); _Py_atomic_or_uint8(&op->ob_gc_bits, _PyGC_BITS_DEFERRED); #elif SIZEOF_VOID_P > 4 op->ob_flags = _Py_IMMORTAL_FLAGS; @@ -2603,6 +2728,13 @@ PyUnstable_Object_EnableDeferredRefcount(PyObject *op) return 0; } + if (!PyObject_GC_IsTracked(op)) { + // When deferred refcount is enabled, the object will only be + // deallocated by the tracing garbage collector. So it must be tracked + // by the garbage collector. + return 0; + } + uint8_t bits = _Py_atomic_load_uint8(&op->ob_gc_bits); if ((bits & _PyGC_BITS_DEFERRED) != 0) { @@ -2638,8 +2770,12 @@ PyUnstable_Object_IsUniqueReferencedTemporary(PyObject *op) _PyStackRef *stackpointer = frame->stackpointer; while (stackpointer > base) { stackpointer--; - if (op == PyStackRef_AsPyObjectBorrow(*stackpointer)) { - return PyStackRef_IsHeapSafe(*stackpointer); + _PyStackRef ref = *stackpointer; + if (PyStackRef_IsTaggedInt(ref)) { + continue; + } + if (op == PyStackRef_AsPyObjectBorrow(ref)) { + return PyStackRef_IsHeapSafe(ref); } } return 0; @@ -2930,57 +3066,28 @@ Py_ReprLeave(PyObject *obj) /* Trashcan support. */ -#ifndef Py_GIL_DISABLED -/* We need to store a pointer in the refcount field of - * an object. It is important that we never store 0 (NULL). -* It is also important to not make the object appear immortal, -* or it might be untracked by the cycle GC. */ -static uintptr_t -pointer_to_safe_refcount(void *ptr) -{ - uintptr_t full = (uintptr_t)ptr; - assert((full & 3) == 0); -#if SIZEOF_VOID_P > 4 - uint32_t refcnt = (uint32_t)full; - if (refcnt >= (uint32_t)_Py_IMMORTAL_MINIMUM_REFCNT) { - full = full - ((uintptr_t)_Py_IMMORTAL_MINIMUM_REFCNT) + 1; - } - return full + 2; -#else - // Make the top two bits 0, so it appears mortal. - return (full >> 2) + 1; -#endif -} - -static void * -safe_refcount_to_pointer(uintptr_t refcnt) -{ -#if SIZEOF_VOID_P > 4 - if (refcnt & 1) { - refcnt += _Py_IMMORTAL_MINIMUM_REFCNT - 1; - } - return (void *)(refcnt - 2); -#else - return (void *)((refcnt -1) << 2); -#endif -} -#endif - /* Add op to the gcstate->trash_delete_later list. Called when the current - * call-stack depth gets large. op must be a currently untracked gc'ed - * object, with refcount 0. Py_DECREF must already have been called on it. + * call-stack depth gets large. op must be a gc'ed object, with refcount 0. + * Py_DECREF must already have been called on it. */ void _PyTrash_thread_deposit_object(PyThreadState *tstate, PyObject *op) { _PyObject_ASSERT(op, Py_REFCNT(op) == 0); + PyTypeObject *tp = Py_TYPE(op); + assert(tp->tp_flags & Py_TPFLAGS_HAVE_GC); + int tracked = 0; + if (tp->tp_is_gc == NULL || tp->tp_is_gc(op)) { + tracked = _PyObject_GC_IS_TRACKED(op); + if (tracked) { + _PyObject_GC_UNTRACK(op); + } + } + uintptr_t tagged_ptr = ((uintptr_t)tstate->delete_later) | tracked; #ifdef Py_GIL_DISABLED - op->ob_tid = (uintptr_t)tstate->delete_later; + op->ob_tid = tagged_ptr; #else - /* Store the delete_later pointer in the refcnt field. */ - uintptr_t refcnt = pointer_to_safe_refcount(tstate->delete_later); - *((uintptr_t*)op) = refcnt; - assert(!_Py_IsImmortal(op)); + _Py_AS_GC(op)->_gc_next = tagged_ptr; #endif tstate->delete_later = op; } @@ -2995,17 +3102,17 @@ _PyTrash_thread_destroy_chain(PyThreadState *tstate) destructor dealloc = Py_TYPE(op)->tp_dealloc; #ifdef Py_GIL_DISABLED - tstate->delete_later = (PyObject*) op->ob_tid; + uintptr_t tagged_ptr = op->ob_tid; op->ob_tid = 0; _Py_atomic_store_ssize_relaxed(&op->ob_ref_shared, _Py_REF_MERGED); #else - /* Get the delete_later pointer from the refcnt field. - * See _PyTrash_thread_deposit_object(). */ - uintptr_t refcnt = *((uintptr_t*)op); - tstate->delete_later = safe_refcount_to_pointer(refcnt); - op->ob_refcnt = 0; + uintptr_t tagged_ptr = _Py_AS_GC(op)->_gc_next; + _Py_AS_GC(op)->_gc_next = 0; #endif - + tstate->delete_later = (PyObject *)(tagged_ptr & ~1); + if (tagged_ptr & 1) { + _PyObject_GC_TRACK(op); + } /* Call the deallocator directly. This used to try to * fool Py_DECREF into calling it indirectly, but * Py_DECREF was already called on this object, and in @@ -3079,10 +3186,11 @@ void _Py_Dealloc(PyObject *op) { PyTypeObject *type = Py_TYPE(op); + unsigned long gc_flag = type->tp_flags & Py_TPFLAGS_HAVE_GC; destructor dealloc = type->tp_dealloc; PyThreadState *tstate = _PyThreadState_GET(); intptr_t margin = _Py_RecursionLimit_GetMargin(tstate); - if (margin < 2) { + if (margin < 2 && gc_flag) { _PyTrash_thread_deposit_object(tstate, (PyObject *)op); return; } @@ -3128,7 +3236,7 @@ _Py_Dealloc(PyObject *op) Py_XDECREF(old_exc); Py_DECREF(type); #endif - if (tstate->delete_later && margin >= 4) { + if (tstate->delete_later && margin >= 4 && gc_flag) { _PyTrash_thread_destroy_chain(tstate); } } diff --git a/Objects/object_layout_312.png b/Objects/object_layout_312.png index a63d095ea0b19eb..9ccb99f9db1b6d6 100644 Binary files a/Objects/object_layout_312.png and b/Objects/object_layout_312.png differ diff --git a/Objects/object_layout_313.png b/Objects/object_layout_313.png index f7059c286c84e6c..f3df59253da1310 100644 Binary files a/Objects/object_layout_313.png and b/Objects/object_layout_313.png differ diff --git a/Objects/object_layout_full_312.png b/Objects/object_layout_full_312.png index 4f46ca86091d452..8c0aca4e06a3029 100644 Binary files a/Objects/object_layout_full_312.png and b/Objects/object_layout_full_312.png differ diff --git a/Objects/object_layout_full_313.png b/Objects/object_layout_full_313.png index 7352ec69e5d8db2..2d5c8569f628dee 100644 Binary files a/Objects/object_layout_full_313.png and b/Objects/object_layout_full_313.png differ diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index b209808da902daa..9af59f13dc82d0c 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -124,22 +124,74 @@ _PyMem_mi_page_is_safe_to_free(mi_page_t *page) } +#ifdef Py_GIL_DISABLED + +// If we are deferring collection of more than this amount of memory for +// mimalloc pages, advance the write sequence. Advancing allows these +// pages to be re-used in a different thread or for a different size class. +#define QSBR_PAGE_MEM_LIMIT 4096*20 + +// Return true if the global write sequence should be advanced for a mimalloc +// page that is deferred from collection. +static bool +should_advance_qsbr_for_page(struct _qsbr_thread_state *qsbr, mi_page_t *page) +{ + size_t bsize = mi_page_block_size(page); + size_t page_size = page->capacity*bsize; + if (page_size > QSBR_PAGE_MEM_LIMIT) { + qsbr->deferred_page_memory = 0; + return true; + } + qsbr->deferred_page_memory += page_size; + if (qsbr->deferred_page_memory > QSBR_PAGE_MEM_LIMIT) { + qsbr->deferred_page_memory = 0; + return true; + } + return false; +} + +static _PyThreadStateImpl * +tstate_from_heap(mi_heap_t *heap) +{ + return _Py_CONTAINER_OF(heap->tld, _PyThreadStateImpl, mimalloc.tld); +} +#endif + static bool _PyMem_mi_page_maybe_free(mi_page_t *page, mi_page_queue_t *pq, bool force) { #ifdef Py_GIL_DISABLED assert(mi_page_all_free(page)); if (page->use_qsbr) { - _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)PyThreadState_GET(); - if (page->qsbr_goal != 0 && _Py_qbsr_goal_reached(tstate->qsbr, page->qsbr_goal)) { + struct _qsbr_thread_state *qsbr = ((_PyThreadStateImpl *)PyThreadState_GET())->qsbr; + if (page->qsbr_goal != 0 && _Py_qbsr_goal_reached(qsbr, page->qsbr_goal)) { _PyMem_mi_page_clear_qsbr(page); _mi_page_free(page, pq, force); return true; } + // gh-145615: since we are not freeing this page yet, we want to + // make it available for allocations. Note that the QSBR goal and + // linked list node remain set even if the page is later used for + // an allocation. We only detect and clear the QSBR goal when the + // page becomes empty again (used == 0). + if (mi_page_is_in_full(page)) { + _mi_page_unfull(page); + } + _PyMem_mi_page_clear_qsbr(page); page->retire_expire = 0; - page->qsbr_goal = _Py_qsbr_deferred_advance(tstate->qsbr); + + if (should_advance_qsbr_for_page(qsbr, page)) { + page->qsbr_goal = _Py_qsbr_advance(qsbr->shared); + } + else { + page->qsbr_goal = _Py_qsbr_shared_next(qsbr->shared); + } + + // We may be freeing a page belonging to a different thread during a + // stop-the-world event. Find the _PyThreadStateImpl for the page. + _PyThreadStateImpl *tstate = tstate_from_heap(mi_page_heap(page)); llist_insert_tail(&tstate->mimalloc.page_list, &page->qsbr_node); return false; } @@ -156,7 +208,8 @@ _PyMem_mi_page_reclaimed(mi_page_t *page) if (page->qsbr_goal != 0) { if (mi_page_all_free(page)) { assert(page->qsbr_node.next == NULL); - _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)PyThreadState_GET(); + _PyThreadStateImpl *tstate = tstate_from_heap(mi_page_heap(page)); + assert(tstate == (_PyThreadStateImpl *)_PyThreadState_GET()); page->retire_expire = 0; llist_insert_tail(&tstate->mimalloc.page_list, &page->qsbr_node); } @@ -1141,8 +1194,44 @@ free_work_item(uintptr_t ptr, delayed_dealloc_cb cb, void *state) } } + +#ifdef Py_GIL_DISABLED + +// For deferred advance on free: the number of deferred items before advancing +// the write sequence. This is based on WORK_ITEMS_PER_CHUNK. We ideally +// want to process a chunk before it overflows. +#define QSBR_DEFERRED_LIMIT 127 + +// If the deferred memory exceeds 1 MiB, advance the write sequence. This +// helps limit memory usage due to QSBR delaying frees too long. +#define QSBR_FREE_MEM_LIMIT 1024*1024 + +// Return true if the global write sequence should be advanced for a deferred +// memory free. +static bool +should_advance_qsbr_for_free(struct _qsbr_thread_state *qsbr, size_t size) +{ + if (size > QSBR_FREE_MEM_LIMIT) { + qsbr->deferred_count = 0; + qsbr->deferred_memory = 0; + qsbr->should_process = true; + return true; + } + qsbr->deferred_count++; + qsbr->deferred_memory += size; + if (qsbr->deferred_count > QSBR_DEFERRED_LIMIT || + qsbr->deferred_memory > QSBR_FREE_MEM_LIMIT) { + qsbr->deferred_count = 0; + qsbr->deferred_memory = 0; + qsbr->should_process = true; + return true; + } + return false; +} +#endif + static void -free_delayed(uintptr_t ptr) +free_delayed(uintptr_t ptr, size_t size) { #ifndef Py_GIL_DISABLED free_work_item(ptr, NULL, NULL); @@ -1200,23 +1289,32 @@ free_delayed(uintptr_t ptr) } assert(buf != NULL && buf->wr_idx < WORK_ITEMS_PER_CHUNK); - uint64_t seq = _Py_qsbr_deferred_advance(tstate->qsbr); + uint64_t seq; + if (should_advance_qsbr_for_free(tstate->qsbr, size)) { + seq = _Py_qsbr_advance(tstate->qsbr->shared); + } + else { + seq = _Py_qsbr_shared_next(tstate->qsbr->shared); + } buf->array[buf->wr_idx].ptr = ptr; buf->array[buf->wr_idx].qsbr_goal = seq; buf->wr_idx++; if (buf->wr_idx == WORK_ITEMS_PER_CHUNK) { + // Normally the processing of delayed items is done from the eval + // breaker. Processing here is a safety measure to ensure too much + // work does not accumulate. _PyMem_ProcessDelayed((PyThreadState *)tstate); } #endif } void -_PyMem_FreeDelayed(void *ptr) +_PyMem_FreeDelayed(void *ptr, size_t size) { assert(!((uintptr_t)ptr & 0x01)); if (ptr != NULL) { - free_delayed((uintptr_t)ptr); + free_delayed((uintptr_t)ptr, size); } } @@ -1226,7 +1324,10 @@ _PyObject_XDecRefDelayed(PyObject *ptr) { assert(!((uintptr_t)ptr & 0x01)); if (ptr != NULL) { - free_delayed(((uintptr_t)ptr)|0x01); + // We use 0 as the size since we don't have an easy way to know the + // actual size. If we are freeing many objects, the write sequence + // will be advanced due to QSBR_DEFERRED_LIMIT. + free_delayed(((uintptr_t)ptr)|0x01, 0); } } #endif @@ -1238,7 +1339,7 @@ work_queue_first(struct llist_node *head) } static void -process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr, +process_queue(struct llist_node *head, _PyThreadStateImpl *tstate, bool keep_empty, delayed_dealloc_cb cb, void *state) { while (!llist_empty(head)) { @@ -1246,7 +1347,7 @@ process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr, if (buf->rd_idx < buf->wr_idx) { struct _mem_work_item *item = &buf->array[buf->rd_idx]; - if (!_Py_qsbr_poll(qsbr, item->qsbr_goal)) { + if (!_Py_qsbr_poll(tstate->qsbr, item->qsbr_goal)) { return; } @@ -1270,11 +1371,11 @@ process_queue(struct llist_node *head, struct _qsbr_thread_state *qsbr, static void process_interp_queue(struct _Py_mem_interp_free_queue *queue, - struct _qsbr_thread_state *qsbr, delayed_dealloc_cb cb, + _PyThreadStateImpl *tstate, delayed_dealloc_cb cb, void *state) { assert(PyMutex_IsLocked(&queue->mutex)); - process_queue(&queue->head, qsbr, false, cb, state); + process_queue(&queue->head, tstate, false, cb, state); int more_work = !llist_empty(&queue->head); _Py_atomic_store_int_relaxed(&queue->has_work, more_work); @@ -1282,7 +1383,7 @@ process_interp_queue(struct _Py_mem_interp_free_queue *queue, static void maybe_process_interp_queue(struct _Py_mem_interp_free_queue *queue, - struct _qsbr_thread_state *qsbr, delayed_dealloc_cb cb, + _PyThreadStateImpl *tstate, delayed_dealloc_cb cb, void *state) { if (!_Py_atomic_load_int_relaxed(&queue->has_work)) { @@ -1291,7 +1392,7 @@ maybe_process_interp_queue(struct _Py_mem_interp_free_queue *queue, // Try to acquire the lock, but don't block if it's already held. if (_PyMutex_LockTimed(&queue->mutex, 0, 0) == PY_LOCK_ACQUIRED) { - process_interp_queue(queue, qsbr, cb, state); + process_interp_queue(queue, tstate, cb, state); PyMutex_Unlock(&queue->mutex); } } @@ -1302,11 +1403,13 @@ _PyMem_ProcessDelayed(PyThreadState *tstate) PyInterpreterState *interp = tstate->interp; _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + tstate_impl->qsbr->should_process = false; + // Process thread-local work - process_queue(&tstate_impl->mem_free_queue, tstate_impl->qsbr, true, NULL, NULL); + process_queue(&tstate_impl->mem_free_queue, tstate_impl, true, NULL, NULL); // Process shared interpreter work - maybe_process_interp_queue(&interp->mem_free_queue, tstate_impl->qsbr, NULL, NULL); + maybe_process_interp_queue(&interp->mem_free_queue, tstate_impl, NULL, NULL); } void @@ -1316,10 +1419,10 @@ _PyMem_ProcessDelayedNoDealloc(PyThreadState *tstate, delayed_dealloc_cb cb, voi _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; // Process thread-local work - process_queue(&tstate_impl->mem_free_queue, tstate_impl->qsbr, true, cb, state); + process_queue(&tstate_impl->mem_free_queue, tstate_impl, true, cb, state); // Process shared interpreter work - maybe_process_interp_queue(&interp->mem_free_queue, tstate_impl->qsbr, cb, state); + maybe_process_interp_queue(&interp->mem_free_queue, tstate_impl, cb, state); } void @@ -1348,7 +1451,7 @@ _PyMem_AbandonDelayed(PyThreadState *tstate) // Process the merged queue now (see gh-130794). _PyThreadStateImpl *this_tstate = (_PyThreadStateImpl *)_PyThreadState_GET(); - process_interp_queue(&interp->mem_free_queue, this_tstate->qsbr, NULL, NULL); + process_interp_queue(&interp->mem_free_queue, this_tstate, NULL, NULL); PyMutex_Unlock(&interp->mem_free_queue.mutex); diff --git a/Objects/odictobject.c b/Objects/odictobject.c index 891f6197401503d..5c6ac56da5fed5e 100644 --- a/Objects/odictobject.c +++ b/Objects/odictobject.c @@ -473,6 +473,7 @@ Potential Optimizations #include "pycore_pyerrors.h" // _PyErr_ChainExceptions1() #include "pycore_tuple.h" // _PyTuple_Recycle() #include <stddef.h> // offsetof() +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "clinic/odictobject.c.h" @@ -535,6 +536,7 @@ struct _odictnode { static Py_ssize_t _odict_get_index_raw(PyODictObject *od, PyObject *key, Py_hash_t hash) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); PyObject *value = NULL; PyDictKeysObject *keys = ((PyDictObject *)od)->ma_keys; Py_ssize_t ix; @@ -559,6 +561,7 @@ _odict_get_index_raw(PyODictObject *od, PyObject *key, Py_hash_t hash) static int _odict_resize(PyODictObject *od) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); Py_ssize_t size, i; _ODictNode **fast_nodes, *node; @@ -595,6 +598,7 @@ _odict_resize(PyODictObject *od) static Py_ssize_t _odict_get_index(PyODictObject *od, PyObject *key, Py_hash_t hash) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); PyDictKeysObject *keys; assert(key != NULL); @@ -615,6 +619,7 @@ _odict_get_index(PyODictObject *od, PyObject *key, Py_hash_t hash) static _ODictNode * _odict_find_node_hash(PyODictObject *od, PyObject *key, Py_hash_t hash) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); Py_ssize_t index; if (_odict_EMPTY(od)) @@ -629,6 +634,7 @@ _odict_find_node_hash(PyODictObject *od, PyObject *key, Py_hash_t hash) static _ODictNode * _odict_find_node(PyODictObject *od, PyObject *key) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); Py_ssize_t index; Py_hash_t hash; @@ -647,6 +653,7 @@ _odict_find_node(PyODictObject *od, PyObject *key) static void _odict_add_head(PyODictObject *od, _ODictNode *node) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); _odictnode_PREV(node) = NULL; _odictnode_NEXT(node) = _odict_FIRST(od); if (_odict_FIRST(od) == NULL) @@ -660,6 +667,7 @@ _odict_add_head(PyODictObject *od, _ODictNode *node) static void _odict_add_tail(PyODictObject *od, _ODictNode *node) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); _odictnode_PREV(node) = _odict_LAST(od); _odictnode_NEXT(node) = NULL; if (_odict_LAST(od) == NULL) @@ -674,6 +682,7 @@ _odict_add_tail(PyODictObject *od, _ODictNode *node) static int _odict_add_new_node(PyODictObject *od, PyObject *key, Py_hash_t hash) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); Py_ssize_t i; _ODictNode *node; @@ -718,6 +727,7 @@ _odict_add_new_node(PyODictObject *od, PyObject *key, Py_hash_t hash) static void _odict_remove_node(PyODictObject *od, _ODictNode *node) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); if (_odict_FIRST(od) == node) _odict_FIRST(od) = _odictnode_NEXT(node); else if (_odictnode_PREV(node) != NULL) @@ -753,6 +763,7 @@ static int _odict_clear_node(PyODictObject *od, _ODictNode *node, PyObject *key, Py_hash_t hash) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); Py_ssize_t i; assert(key != NULL); @@ -950,31 +961,34 @@ OrderedDict_fromkeys_impl(PyTypeObject *type, PyObject *seq, PyObject *value) return _PyDict_FromKeys((PyObject *)type, seq, value); } -/* __sizeof__() */ - -/* OrderedDict.__sizeof__() does not have a docstring. */ -PyDoc_STRVAR(odict_sizeof__doc__, ""); +/*[clinic input] +@critical_section +OrderedDict.__sizeof__ -> Py_ssize_t +[clinic start generated code]*/ -static PyObject * -odict_sizeof(PyObject *op, PyObject *Py_UNUSED(ignored)) +static Py_ssize_t +OrderedDict___sizeof___impl(PyODictObject *self) +/*[clinic end generated code: output=1a8560db8cf83ac5 input=655e989ae24daa6a]*/ { - PyODictObject *od = _PyODictObject_CAST(op); - Py_ssize_t res = _PyDict_SizeOf((PyDictObject *)od); - res += sizeof(_ODictNode *) * od->od_fast_nodes_size; /* od_fast_nodes */ - if (!_odict_EMPTY(od)) { - res += sizeof(_ODictNode) * PyODict_SIZE(od); /* linked-list */ + Py_ssize_t res = _PyDict_SizeOf_LockHeld((PyDictObject *)self); + res += sizeof(_ODictNode *) * self->od_fast_nodes_size; /* od_fast_nodes */ + if (!_odict_EMPTY(self)) { + res += sizeof(_ODictNode) * PyODict_SIZE(self); /* linked-list */ } - return PyLong_FromSsize_t(res); + return res; } -/* __reduce__() */ +/*[clinic input] +OrderedDict.__reduce__ + self as od: self(type="PyODictObject *") -PyDoc_STRVAR(odict_reduce__doc__, "Return state information for pickling"); +Return state information for pickling +[clinic start generated code]*/ static PyObject * -odict_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) +OrderedDict___reduce___impl(PyODictObject *od) +/*[clinic end generated code: output=71eeb81f760a6a8e input=b0467c7ec400fe5e]*/ { - register PyODictObject *od = _PyODictObject_CAST(op); PyObject *state, *result = NULL; PyObject *items_iter, *items, *args = NULL; @@ -1009,8 +1023,10 @@ odict_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) /* setdefault(): Skips __missing__() calls. */ +static int PyODict_SetItem_LockHeld(PyObject *self, PyObject *key, PyObject *value); /*[clinic input] +@critical_section OrderedDict.setdefault key: object @@ -1024,7 +1040,7 @@ Return the value for key if key is in the dictionary, else default. static PyObject * OrderedDict_setdefault_impl(PyODictObject *self, PyObject *key, PyObject *default_value) -/*[clinic end generated code: output=97537cb7c28464b6 input=38e098381c1efbc6]*/ +/*[clinic end generated code: output=97537cb7c28464b6 input=d7b93e92734f99b5]*/ { PyObject *result = NULL; @@ -1034,7 +1050,7 @@ OrderedDict_setdefault_impl(PyODictObject *self, PyObject *key, if (PyErr_Occurred()) return NULL; assert(_odict_find_node(self, key) == NULL); - if (PyODict_SetItem((PyObject *)self, key, default_value) >= 0) { + if (PyODict_SetItem_LockHeld((PyObject *)self, key, default_value) >= 0) { result = Py_NewRef(default_value); } } @@ -1064,10 +1080,9 @@ static PyObject * _odict_popkey_hash(PyObject *od, PyObject *key, PyObject *failobj, Py_hash_t hash) { - PyObject *value = NULL; - - Py_BEGIN_CRITICAL_SECTION(od); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); + PyObject *value = NULL; _ODictNode *node = _odict_find_node_hash(_PyODictObject_CAST(od), key, hash); if (node != NULL) { /* Pop the node first to avoid a possible dict resize (due to @@ -1092,7 +1107,6 @@ _odict_popkey_hash(PyObject *od, PyObject *key, PyObject *failobj, PyErr_SetObject(PyExc_KeyError, key); } } - Py_END_CRITICAL_SECTION(); done: return value; @@ -1100,6 +1114,7 @@ _odict_popkey_hash(PyObject *od, PyObject *key, PyObject *failobj, /* Skips __missing__() calls. */ /*[clinic input] +@critical_section OrderedDict.pop key: object @@ -1114,7 +1129,7 @@ raise a KeyError. static PyObject * OrderedDict_pop_impl(PyODictObject *self, PyObject *key, PyObject *default_value) -/*[clinic end generated code: output=7a6447d104e7494b input=7efe36601007dff7]*/ +/*[clinic end generated code: output=7a6447d104e7494b input=a79988887b4a651f]*/ { Py_hash_t hash = PyObject_Hash(key); if (hash == -1) @@ -1126,18 +1141,20 @@ OrderedDict_pop_impl(PyODictObject *self, PyObject *key, /* popitem() */ /*[clinic input] +@critical_section OrderedDict.popitem last: bool = True Remove and return a (key, value) pair from the dictionary. -Pairs are returned in LIFO order if last is true or FIFO order if false. +Pairs are returned in LIFO order if last is true or FIFO order if +false. [clinic start generated code]*/ static PyObject * OrderedDict_popitem_impl(PyODictObject *self, int last) -/*[clinic end generated code: output=98e7d986690d49eb input=d992ac5ee8305e1a]*/ +/*[clinic end generated code: output=98e7d986690d49eb input=ebf1cc91579c9e54]*/ { PyObject *key, *value, *item = NULL; _ODictNode *node; @@ -1152,8 +1169,10 @@ OrderedDict_popitem_impl(PyODictObject *self, int last) node = last ? _odict_LAST(self) : _odict_FIRST(self); key = Py_NewRef(_odictnode_KEY(node)); value = _odict_popkey_hash((PyObject *)self, key, NULL, _odictnode_HASH(node)); - if (value == NULL) + if (value == NULL) { + Py_DECREF(key); return NULL; + } item = PyTuple_Pack(2, key, value); Py_DECREF(key); Py_DECREF(value); @@ -1166,6 +1185,9 @@ OrderedDict_popitem_impl(PyODictObject *self, int last) PyDoc_STRVAR(odict_keys__doc__, ""); static PyObject * odictkeys_new(PyObject *od, PyObject *Py_UNUSED(ignored)); /* forward */ +static int +_PyODict_SetItem_KnownHash_LockHeld(PyObject *od, PyObject *key, PyObject *value, + Py_hash_t hash); /* forward */ /* values() */ @@ -1191,32 +1213,36 @@ static PyObject * mutablemapping_update(PyObject *, PyObject *, PyObject *); #define odict_update mutablemapping_update -/* clear() */ +/*[clinic input] +@critical_section +OrderedDict.clear -PyDoc_STRVAR(odict_clear__doc__, - "od.clear() -> None. Remove all items from od."); +Remove all items from ordered dict. +[clinic start generated code]*/ static PyObject * -odict_clear(PyObject *op, PyObject *Py_UNUSED(ignored)) +OrderedDict_clear_impl(PyODictObject *self) +/*[clinic end generated code: output=a1a76d1322f556c5 input=08b12322e74c535c]*/ { - register PyODictObject *od = _PyODictObject_CAST(op); - PyDict_Clear(op); - _odict_clear_nodes(od); + _PyDict_Clear_LockHeld((PyObject *)self); + _odict_clear_nodes(self); Py_RETURN_NONE; } /* copy() */ -/* forward */ -static int _PyODict_SetItem_KnownHash(PyObject *, PyObject *, PyObject *, - Py_hash_t); +/*[clinic input] +@critical_section +OrderedDict.copy + self as od: self -PyDoc_STRVAR(odict_copy__doc__, "od.copy() -> a shallow copy of od"); +A shallow copy of ordered dict. +[clinic start generated code]*/ static PyObject * -odict_copy(PyObject *op, PyObject *Py_UNUSED(ignored)) +OrderedDict_copy_impl(PyObject *od) +/*[clinic end generated code: output=9cdbe7394aecc576 input=e329951ae617ed48]*/ { - register PyODictObject *od = _PyODictObject_CAST(op); _ODictNode *node; PyObject *od_copy; @@ -1227,36 +1253,52 @@ odict_copy(PyObject *op, PyObject *Py_UNUSED(ignored)) if (od_copy == NULL) return NULL; + /* The loop body may run arbitrary Python code which could mutate od and + free its nodes (gh-148660); detect that the same way __eq__ does. */ + size_t state = _PyODictObject_CAST(od)->od_state; + if (PyODict_CheckExact(od)) { _odict_FOREACH(od, node) { - PyObject *key = _odictnode_KEY(node); - PyObject *value = _odictnode_VALUE(node, od); + PyObject *key = Py_NewRef(_odictnode_KEY(node)); + Py_hash_t hash = _odictnode_HASH(node); + PyObject *value = PyODict_GetItemWithError(od, key); if (value == NULL) { if (!PyErr_Occurred()) PyErr_SetObject(PyExc_KeyError, key); + Py_DECREF(key); goto fail; } - if (_PyODict_SetItem_KnownHash((PyObject *)od_copy, key, value, - _odictnode_HASH(node)) != 0) + int res = _PyODict_SetItem_KnownHash_LockHeld((PyObject *)od_copy, + key, value, hash); + Py_DECREF(key); + if (res != 0) goto fail; + if (_PyODictObject_CAST(od)->od_state != state) + goto mutated; } } else { _odict_FOREACH(od, node) { - int res; - PyObject *value = PyObject_GetItem((PyObject *)od, - _odictnode_KEY(node)); - if (value == NULL) + PyObject *key = Py_NewRef(_odictnode_KEY(node)); + PyObject *value = PyObject_GetItem(od, key); + if (value == NULL) { + Py_DECREF(key); goto fail; - res = PyObject_SetItem((PyObject *)od_copy, - _odictnode_KEY(node), value); + } + int res = PyObject_SetItem((PyObject *)od_copy, key, value); Py_DECREF(value); + Py_DECREF(key); if (res != 0) goto fail; + if (_PyODictObject_CAST(od)->od_state != state) + goto mutated; } } return od_copy; +mutated: + PyErr_SetString(PyExc_RuntimeError, + "OrderedDict mutated during iteration"); fail: Py_DECREF(od_copy); return NULL; @@ -1285,6 +1327,7 @@ odict_reversed(PyObject *op, PyObject *Py_UNUSED(ignored)) /* move_to_end() */ /*[clinic input] +@critical_section OrderedDict.move_to_end key: object @@ -1297,7 +1340,7 @@ Raise KeyError if the element does not exist. static PyObject * OrderedDict_move_to_end_impl(PyODictObject *self, PyObject *key, int last) -/*[clinic end generated code: output=fafa4c5cc9b92f20 input=d6ceff7132a2fcd7]*/ +/*[clinic end generated code: output=fafa4c5cc9b92f20 input=09f8bc7053c0f6d4]*/ { _ODictNode *node; @@ -1338,10 +1381,8 @@ static PyMethodDef odict_methods[] = { /* overridden dict methods */ ORDEREDDICT_FROMKEYS_METHODDEF - {"__sizeof__", odict_sizeof, METH_NOARGS, - odict_sizeof__doc__}, - {"__reduce__", odict_reduce, METH_NOARGS, - odict_reduce__doc__}, + ORDEREDDICT___SIZEOF___METHODDEF + ORDEREDDICT___REDUCE___METHODDEF ORDEREDDICT_SETDEFAULT_METHODDEF ORDEREDDICT_POP_METHODDEF ORDEREDDICT_POPITEM_METHODDEF @@ -1353,11 +1394,8 @@ static PyMethodDef odict_methods[] = { odict_items__doc__}, {"update", _PyCFunction_CAST(odict_update), METH_VARARGS | METH_KEYWORDS, odict_update__doc__}, - {"clear", odict_clear, METH_NOARGS, - odict_clear__doc__}, - {"copy", odict_copy, METH_NOARGS, - odict_copy__doc__}, - + ORDEREDDICT_CLEAR_METHODDEF + ORDEREDDICT_COPY_METHODDEF /* new methods */ {"__reversed__", odict_reversed, METH_NOARGS, odict_reversed__doc__}, @@ -1391,8 +1429,7 @@ odict_dealloc(PyObject *op) PyObject_GC_UnTrack(self); Py_XDECREF(self->od_inst_dict); - if (self->od_weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *)self); + FT_CLEAR_WEAKREFS(op, self->od_weakreflist); _odict_clear_nodes(self); PyDict_Type.tp_dealloc((PyObject *)self); @@ -1457,7 +1494,8 @@ odict_tp_clear(PyObject *op) { PyODictObject *od = _PyODictObject_CAST(op); Py_CLEAR(od->od_inst_dict); - PyDict_Clear((PyObject *)od); + // cannot use lock held variant as critical section is not held here + PyDict_Clear(op); _odict_clear_nodes(od); return 0; } @@ -1465,7 +1503,7 @@ odict_tp_clear(PyObject *op) /* tp_richcompare */ static PyObject * -odict_richcompare(PyObject *v, PyObject *w, int op) +odict_richcompare_lock_held(PyObject *v, PyObject *w, int op) { if (!PyODict_Check(v) || !PyDict_Check(w)) { Py_RETURN_NOTIMPLEMENTED; @@ -1498,6 +1536,16 @@ odict_richcompare(PyObject *v, PyObject *w, int op) } } +static PyObject * +odict_richcompare(PyObject *v, PyObject *w, int op) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION2(v, w); + res = odict_richcompare_lock_held(v, w, op); + Py_END_CRITICAL_SECTION2(); + return res; +} + /* tp_iter */ static PyObject * @@ -1517,7 +1565,7 @@ odict_init(PyObject *self, PyObject *args, PyObject *kwds) if (len == -1) return -1; if (len > 1) { - const char *msg = "expected at most 1 arguments, got %zd"; + const char *msg = "expected at most 1 argument, got %zd"; PyErr_Format(PyExc_TypeError, msg, len); return -1; } @@ -1588,10 +1636,11 @@ PyODict_New(void) } static int -_PyODict_SetItem_KnownHash(PyObject *od, PyObject *key, PyObject *value, - Py_hash_t hash) +_PyODict_SetItem_KnownHash_LockHeld(PyObject *od, PyObject *key, PyObject *value, + Py_hash_t hash) { - int res = _PyDict_SetItem_KnownHash(od, key, value, hash); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); + int res = _PyDict_SetItem_KnownHash_LockHeld((PyDictObject *)od, key, value, hash); if (res == 0) { res = _odict_add_new_node(_PyODictObject_CAST(od), key, hash); if (res < 0) { @@ -1604,18 +1653,32 @@ _PyODict_SetItem_KnownHash(PyObject *od, PyObject *key, PyObject *value, return res; } -int -PyODict_SetItem(PyObject *od, PyObject *key, PyObject *value) + +static int +PyODict_SetItem_LockHeld(PyObject *od, PyObject *key, PyObject *value) { + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); Py_hash_t hash = PyObject_Hash(key); - if (hash == -1) + if (hash == -1) { return -1; - return _PyODict_SetItem_KnownHash(od, key, value, hash); + } + return _PyODict_SetItem_KnownHash_LockHeld(od, key, value, hash); } int -PyODict_DelItem(PyObject *od, PyObject *key) +PyODict_SetItem(PyObject *od, PyObject *key, PyObject *value) { + int res; + Py_BEGIN_CRITICAL_SECTION(od); + res = PyODict_SetItem_LockHeld(od, key, value); + Py_END_CRITICAL_SECTION(); + return res; +} + +int +PyODict_DelItem_LockHeld(PyObject *od, PyObject *key) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(od); int res; Py_hash_t hash = PyObject_Hash(key); if (hash == -1) @@ -1623,9 +1686,18 @@ PyODict_DelItem(PyObject *od, PyObject *key) res = _odict_clear_node(_PyODictObject_CAST(od), NULL, key, hash); if (res < 0) return -1; - return _PyDict_DelItem_KnownHash(od, key, hash); + return _PyDict_DelItem_KnownHash_LockHeld(od, key, hash); } +int +PyODict_DelItem(PyObject *od, PyObject *key) +{ + int res; + Py_BEGIN_CRITICAL_SECTION(od); + res = PyODict_DelItem_LockHeld(od, key); + Py_END_CRITICAL_SECTION(); + return res; +} /* ------------------------------------------- * The OrderedDict views (keys/values/items) @@ -1667,14 +1739,14 @@ odictiter_traverse(PyObject *op, visitproc visit, void *arg) /* In order to protect against modifications during iteration, we track * the current key instead of the current node. */ static PyObject * -odictiter_nextkey(odictiterobject *di) +odictiter_nextkey_lock_held(odictiterobject *di) { + assert(di->di_odict != NULL); + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(di->di_odict); PyObject *key = NULL; _ODictNode *node; int reversed = di->kind & _odict_ITER_REVERSED; - if (di->di_odict == NULL) - return NULL; if (di->di_current == NULL) goto done; /* We're already done. */ @@ -1719,8 +1791,23 @@ odictiter_nextkey(odictiterobject *di) return key; } + static PyObject * -odictiter_iternext(PyObject *op) +odictiter_nextkey(odictiterobject *di) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(di); + if (di->di_odict == NULL) { + return NULL; + } + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(di->di_odict); + res = odictiter_nextkey_lock_held(di); + Py_END_CRITICAL_SECTION(); + return res; +} + +static PyObject * +odictiter_iternext_lock_held(PyObject *op) { odictiterobject *di = (odictiterobject*)op; PyObject *result, *value; @@ -1734,14 +1821,12 @@ odictiter_iternext(PyObject *op) return key; } - value = PyODict_GetItem((PyObject *)di->di_odict, key); /* borrowed */ - if (value == NULL) { + if (PyDict_GetItemRef((PyObject *)di->di_odict, key, &value) != 1) { if (!PyErr_Occurred()) PyErr_SetObject(PyExc_KeyError, key); Py_DECREF(key); goto done; } - Py_INCREF(value); /* Handle the values case. */ if (!(di->kind & _odict_ITER_KEYS)) { @@ -1752,7 +1837,7 @@ odictiter_iternext(PyObject *op) /* Handle the items case. */ result = di->di_result; - if (Py_REFCNT(result) == 1) { + if (_PyObject_IsUniquelyReferenced(result)) { /* not in use so we can reuse it * (the common case during iteration) */ Py_INCREF(result); @@ -1781,6 +1866,17 @@ odictiter_iternext(PyObject *op) return NULL; } +static PyObject * +odictiter_iternext(PyObject *op) +{ + PyObject *res; + Py_BEGIN_CRITICAL_SECTION(op); + res = odictiter_iternext_lock_held(op); + Py_END_CRITICAL_SECTION(); + return res; +} + + /* No need for tp_clear because odictiterobject is not mutable. */ PyDoc_STRVAR(reduce_doc, "Return state information for pickling"); diff --git a/Objects/picklebufobject.c b/Objects/picklebufobject.c index 3ce800de04c2082..50f17687bc43650 100644 --- a/Objects/picklebufobject.c +++ b/Objects/picklebufobject.c @@ -1,6 +1,7 @@ /* PickleBuffer object implementation */ #include "Python.h" +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include <stddef.h> typedef struct { @@ -111,8 +112,7 @@ picklebuf_dealloc(PyObject *op) { PyPickleBufferObject *self = (PyPickleBufferObject*)op; PyObject_GC_UnTrack(self); - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) self); + FT_CLEAR_WEAKREFS(op, self->weakreflist); PyBuffer_Release(&self->view); Py_TYPE(self)->tp_free((PyObject *) self); } diff --git a/Objects/rangeobject.c b/Objects/rangeobject.c index f8cdfe68a6435e2..e93346fb27703f6 100644 --- a/Objects/rangeobject.c +++ b/Objects/rangeobject.c @@ -1042,6 +1042,11 @@ longrangeiter_reduce(PyObject *op, PyObject *Py_UNUSED(ignored)) static PyObject * longrangeiter_setstate(PyObject *op, PyObject *state) { + if (!PyLong_CheckExact(state)) { + PyErr_Format(PyExc_TypeError, "state must be an int, not %T", state); + return NULL; + } + longrangeiterobject *r = (longrangeiterobject*)op; PyObject *zero = _PyLong_GetZero(); // borrowed reference int cmp; diff --git a/Objects/setobject.c b/Objects/setobject.c index 8aa6b0d180907ba..ac1efa26b458c6b 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -40,6 +40,7 @@ #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_SSIZE_RELAXED() #include "pycore_pyerrors.h" // _PyErr_SetKeyError() #include "pycore_setobject.h" // _PySet_NextEntry() definition +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "stringlib/eq.h" // unicode_eq() #include <stddef.h> // offsetof() @@ -85,6 +86,8 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) int probes; int cmp; + int frozenset = PyFrozenSet_CheckExact(so); + while (1) { entry = &so->table[i]; probes = (i + LINEAR_PROBES <= mask) ? LINEAR_PROBES: 0; @@ -101,13 +104,20 @@ set_lookkey(PySetObject *so, PyObject *key, Py_hash_t hash) && unicode_eq(startkey, key)) return entry; table = so->table; - Py_INCREF(startkey); - cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); - Py_DECREF(startkey); - if (cmp < 0) - return NULL; - if (table != so->table || entry->key != startkey) - return set_lookkey(so, key, hash); + if (frozenset) { + cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + if (cmp < 0) + return NULL; + } else { + // incref startkey because it can be removed from the set by the compare + Py_INCREF(startkey); + cmp = PyObject_RichCompareBool(startkey, key, Py_EQ); + Py_DECREF(startkey); + if (cmp < 0) + return NULL; + if (table != so->table || entry->key != startkey) + return set_lookkey(so, key, hash); + } if (cmp > 0) return entry; mask = so->mask; @@ -180,6 +190,9 @@ set_add_entry_takeref(PySetObject *so, PyObject *key, Py_hash_t hash) found_unused_or_dummy: if (freeslot == NULL) goto found_unused; + if (freeslot->hash != -1) { + goto restart; + } FT_ATOMIC_STORE_SSIZE_RELAXED(so->used, so->used + 1); freeslot->key = key; freeslot->hash = hash; @@ -536,8 +549,7 @@ set_dealloc(PyObject *self) /* bpo-31095: UnTrack is needed before calling any callbacks */ PyObject_GC_UnTrack(so); - if (so->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) so); + FT_CLEAR_WEAKREFS(self, so->weakreflist); for (entry = so->table; used > 0; entry++) { if (entry->key && entry->key != dummy) { @@ -1125,13 +1137,15 @@ set_update_impl(PySetObject *so, PyObject * const *others, can be retrieved or updated in a single cache line. */ +// Build a set/frozenset left GC-untracked; the caller must _PyObject_GC_TRACK() +// it once fully built, so a half-built set is never exposed during filling. static PyObject * -make_new_set(PyTypeObject *type, PyObject *iterable) +make_new_set_untracked(PyTypeObject *type, PyObject *iterable) { assert(PyType_Check(type)); PySetObject *so; - so = (PySetObject *)type->tp_alloc(type, 0); + so = (PySetObject *)_PyType_AllocNoTrack(type, 0); if (so == NULL) return NULL; @@ -1154,7 +1168,17 @@ make_new_set(PyTypeObject *type, PyObject *iterable) } static PyObject * -make_new_set_basetype(PyTypeObject *type, PyObject *iterable) +make_new_set(PyTypeObject *type, PyObject *iterable) +{ + PyObject *so = make_new_set_untracked(type, iterable); + if (so != NULL) { + _PyObject_GC_TRACK(so); + } + return so; +} + +static PyObject * +make_new_set_basetype_untracked(PyTypeObject *type, PyObject *iterable) { if (type != &PySet_Type && type != &PyFrozenSet_Type) { if (PyType_IsSubtype(type, &PySet_Type)) @@ -1162,7 +1186,17 @@ make_new_set_basetype(PyTypeObject *type, PyObject *iterable) else type = &PyFrozenSet_Type; } - return make_new_set(type, iterable); + return make_new_set_untracked(type, iterable); +} + +static PyObject * +make_new_set_basetype(PyTypeObject *type, PyObject *iterable) +{ + PyObject *so = make_new_set_basetype_untracked(type, iterable); + if (so != NULL) { + _PyObject_GC_TRACK(so); + } + return so; } static PyObject * @@ -1270,6 +1304,21 @@ set_swap_bodies(PySetObject *a, PySetObject *b) } } +static PyObject * +set_copy_untracked_lock_held(PySetObject *so) +{ + _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); + PyObject *copy = make_new_set_basetype_untracked(Py_TYPE(so), NULL); + if (copy == NULL) { + return NULL; + } + if (set_merge_lock_held((PySetObject *)copy, (PyObject *)so) < 0) { + Py_DECREF(copy); + return NULL; + } + return copy; +} + /*[clinic input] @critical_section set.copy @@ -1282,14 +1331,9 @@ static PyObject * set_copy_impl(PySetObject *so) /*[clinic end generated code: output=c9223a1e1cc6b041 input=c169a4fbb8209257]*/ { - _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(so); - PyObject *copy = make_new_set_basetype(Py_TYPE(so), NULL); - if (copy == NULL) { - return NULL; - } - if (set_merge_lock_held((PySetObject *)copy, (PyObject *)so) < 0) { - Py_DECREF(copy); - return NULL; + PyObject *copy = set_copy_untracked_lock_held(so); + if (copy != NULL) { + _PyObject_GC_TRACK(copy); } return copy; } @@ -1345,7 +1389,8 @@ set_union_impl(PySetObject *so, PyObject * const *others, PyObject *other; Py_ssize_t i; - result = (PySetObject *)set_copy((PyObject *)so, NULL); + result = (PySetObject *)make_new_set_basetype_untracked(Py_TYPE(so), + (PyObject *)so); if (result == NULL) return NULL; @@ -1358,6 +1403,7 @@ set_union_impl(PySetObject *so, PyObject * const *others, return NULL; } } + _PyObject_GC_TRACK(result); return (PyObject *)result; } @@ -1407,7 +1453,7 @@ set_intersection(PySetObject *so, PyObject *other) if ((PyObject *)so == other) return set_copy_impl(so); - result = (PySetObject *)make_new_set_basetype(Py_TYPE(so), NULL); + result = (PySetObject *)make_new_set_basetype_untracked(Py_TYPE(so), NULL); if (result == NULL) return NULL; @@ -1440,6 +1486,7 @@ set_intersection(PySetObject *so, PyObject *other) } Py_DECREF(key); } + _PyObject_GC_TRACK(result); return (PyObject *)result; } @@ -1471,6 +1518,7 @@ set_intersection(PySetObject *so, PyObject *other) Py_DECREF(result); return NULL; } + _PyObject_GC_TRACK(result); return (PyObject *)result; error: Py_DECREF(it); @@ -1746,11 +1794,11 @@ set_difference_update_impl(PySetObject *so, PyObject * const *others, } static PyObject * -set_copy_and_difference(PySetObject *so, PyObject *other) +set_copy_and_difference_untracked(PySetObject *so, PyObject *other) { PyObject *result; - result = set_copy_impl(so); + result = set_copy_untracked_lock_held(so); if (result == NULL) return NULL; if (set_difference_update_internal((PySetObject *) result, other) == 0) @@ -1760,7 +1808,7 @@ set_copy_and_difference(PySetObject *so, PyObject *other) } static PyObject * -set_difference(PySetObject *so, PyObject *other) +set_difference_untracked(PySetObject *so, PyObject *other) { PyObject *result; PyObject *key; @@ -1776,16 +1824,16 @@ set_difference(PySetObject *so, PyObject *other) other_size = PyDict_GET_SIZE(other); } else { - return set_copy_and_difference(so, other); + return set_copy_and_difference_untracked(so, other); } /* If len(so) much more than len(other), it's more efficient to simply copy * so and then iterate other looking for common elements. */ if ((PySet_GET_SIZE(so) >> 2) > other_size) { - return set_copy_and_difference(so, other); + return set_copy_and_difference_untracked(so, other); } - result = make_new_set_basetype(Py_TYPE(so), NULL); + result = make_new_set_basetype_untracked(Py_TYPE(so), NULL); if (result == NULL) return NULL; @@ -1857,7 +1905,7 @@ set_difference_multi_impl(PySetObject *so, PyObject * const *others, other = others[0]; Py_BEGIN_CRITICAL_SECTION2(so, other); - result = set_difference(so, other); + result = set_difference_untracked(so, other); Py_END_CRITICAL_SECTION2(); if (result == NULL) return NULL; @@ -1873,6 +1921,7 @@ set_difference_multi_impl(PySetObject *so, PyObject * const *others, return NULL; } } + _PyObject_GC_TRACK(result); return result; } @@ -1885,8 +1934,11 @@ set_sub(PyObject *self, PyObject *other) PyObject *rv; Py_BEGIN_CRITICAL_SECTION2(so, other); - rv = set_difference(so, other); + rv = set_difference_untracked(so, other); Py_END_CRITICAL_SECTION2(); + if (rv != NULL) { + _PyObject_GC_TRACK(rv); + } return rv; } @@ -2021,7 +2073,8 @@ static PyObject * set_symmetric_difference_impl(PySetObject *so, PyObject *other) /*[clinic end generated code: output=270ee0b5d42b0797 input=624f6e7bbdf70db1]*/ { - PySetObject *result = (PySetObject *)make_new_set_basetype(Py_TYPE(so), NULL); + PySetObject *result = + (PySetObject *)make_new_set_basetype_untracked(Py_TYPE(so), NULL); if (result == NULL) { return NULL; } @@ -2033,6 +2086,7 @@ set_symmetric_difference_impl(PySetObject *so, PyObject *other) Py_DECREF(result); return NULL; } + _PyObject_GC_TRACK(result); return (PyObject *)result; } @@ -2234,10 +2288,16 @@ set_contains_lock_held(PySetObject *so, PyObject *key) int _PySet_Contains(PySetObject *so, PyObject *key) { + assert(so); + int rv; - Py_BEGIN_CRITICAL_SECTION(so); - rv = set_contains_lock_held(so, key); - Py_END_CRITICAL_SECTION(); + if (PyFrozenSet_CheckExact(so)) { + rv = set_contains_lock_held(so, key); + } else { + Py_BEGIN_CRITICAL_SECTION(so); + rv = set_contains_lock_held(so, key); + Py_END_CRITICAL_SECTION(); + } return rv; } @@ -2428,7 +2488,7 @@ set_init(PyObject *so, PyObject *args, PyObject *kwds) if (!PyArg_UnpackTuple(args, Py_TYPE(self)->tp_name, 0, 1, &iterable)) return -1; - if (Py_REFCNT(self) == 1 && self->fill == 0) { + if (_PyObject_IsUniquelyReferenced((PyObject *)self) && self->fill == 0) { self->hash = -1; if (iterable == NULL) { return 0; @@ -2502,7 +2562,8 @@ static PyMethodDef set_methods[] = { SET_SYMMETRIC_DIFFERENCE_UPDATE_METHODDEF SET_UNION_METHODDEF SET_UPDATE_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("sets are generic over the type of their elements")}, {NULL, NULL} /* sentinel */ }; @@ -2584,7 +2645,7 @@ PyTypeObject PySet_Type = { 0, /* tp_descr_set */ 0, /* tp_dictoffset */ set_init, /* tp_init */ - PyType_GenericAlloc, /* tp_alloc */ + _PyType_AllocNoTrack, /* tp_alloc */ set_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ .tp_vectorcall = set_vectorcall, @@ -2606,7 +2667,8 @@ static PyMethodDef frozenset_methods[] = { SET___SIZEOF___METHODDEF SET_SYMMETRIC_DIFFERENCE_METHODDEF SET_UNION_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, + PyDoc_STR("frozensets are generic over the type of their elements")}, {NULL, NULL} /* sentinel */ }; @@ -2675,7 +2737,7 @@ PyTypeObject PyFrozenSet_Type = { 0, /* tp_descr_set */ 0, /* tp_dictoffset */ 0, /* tp_init */ - PyType_GenericAlloc, /* tp_alloc */ + _PyType_AllocNoTrack, /* tp_alloc */ frozenset_new, /* tp_new */ PyObject_GC_Del, /* tp_free */ .tp_vectorcall = frozenset_vectorcall, @@ -2731,7 +2793,9 @@ PySet_Contains(PyObject *anyset, PyObject *key) PyErr_BadInternalCall(); return -1; } - + if (PyFrozenSet_CheckExact(anyset)) { + return set_contains_key((PySetObject *)anyset, key); + } int rv; Py_BEGIN_CRITICAL_SECTION(anyset); rv = set_contains_key((PySetObject *)anyset, key); @@ -2758,7 +2822,7 @@ int PySet_Add(PyObject *anyset, PyObject *key) { if (!PySet_Check(anyset) && - (!PyFrozenSet_Check(anyset) || Py_REFCNT(anyset) != 1)) { + (!PyFrozenSet_Check(anyset) || !_PyObject_IsUniquelyReferenced(anyset))) { PyErr_BadInternalCall(); return -1; } diff --git a/Objects/sliceobject.c b/Objects/sliceobject.c index 5186ff4f6f0cf5f..8bafac07a6cf492 100644 --- a/Objects/sliceobject.c +++ b/Objects/sliceobject.c @@ -341,7 +341,9 @@ PyDoc_STRVAR(slice_doc, "slice(stop)\n\ slice(start, stop[, step])\n\ \n\ -Create a slice object. This is used for extended slicing (e.g. a[0:10:2])."); +Create a slice object.\n\ +\n\ +This is used for extended slicing (e.g. a[0:10:2])."); static void slice_dealloc(PyObject *op) diff --git a/Objects/stringlib/fastsearch.h b/Objects/stringlib/fastsearch.h index 05e700b06258f02..26bb0555ca9b6cf 100644 --- a/Objects/stringlib/fastsearch.h +++ b/Objects/stringlib/fastsearch.h @@ -69,8 +69,8 @@ STRINGLIB(find_char)(const STRINGLIB_CHAR* s, Py_ssize_t n, STRINGLIB_CHAR ch) and UCS4 representations. */ if (needle != 0) { do { - void *candidate = memchr(p, needle, - (e - p) * sizeof(STRINGLIB_CHAR)); + const void *candidate = memchr(p, needle, + (e - p) * sizeof(STRINGLIB_CHAR)); if (candidate == NULL) return -1; s1 = p; @@ -595,7 +595,7 @@ STRINGLIB(default_find)(const STRINGLIB_CHAR* s, Py_ssize_t n, continue; } /* miss: check if next character is part of pattern */ - if (!STRINGLIB_BLOOM(mask, ss[i+1])) { + if (i + 1 <= w && !STRINGLIB_BLOOM(mask, ss[i+1])) { i = i + m; } else { @@ -604,7 +604,7 @@ STRINGLIB(default_find)(const STRINGLIB_CHAR* s, Py_ssize_t n, } else { /* skip: check if next character is part of pattern */ - if (!STRINGLIB_BLOOM(mask, ss[i+1])) { + if (i + 1 <= w && !STRINGLIB_BLOOM(mask, ss[i+1])) { i = i + m; } } @@ -668,7 +668,7 @@ STRINGLIB(adaptive_find)(const STRINGLIB_CHAR* s, Py_ssize_t n, } } /* miss: check if next character is part of pattern */ - if (!STRINGLIB_BLOOM(mask, ss[i+1])) { + if (i + 1 <= w && !STRINGLIB_BLOOM(mask, ss[i+1])) { i = i + m; } else { @@ -677,7 +677,7 @@ STRINGLIB(adaptive_find)(const STRINGLIB_CHAR* s, Py_ssize_t n, } else { /* skip: check if next character is part of pattern */ - if (!STRINGLIB_BLOOM(mask, ss[i+1])) { + if (i + 1 <= w && !STRINGLIB_BLOOM(mask, ss[i+1])) { i = i + m; } } diff --git a/Objects/stringlib/join.h b/Objects/stringlib/join.h index de6bd83ffe4c8bc..deebfeadc0f4fd8 100644 --- a/Objects/stringlib/join.h +++ b/Objects/stringlib/join.h @@ -68,13 +68,18 @@ STRINGLIB(bytes_join)(PyObject *sep, PyObject *iterable) buffers[i].len = PyBytes_GET_SIZE(item); } else { + /* item is only borrowed; its __buffer__() may run Python that + drops the sequence's last reference to it. */ + Py_INCREF(item); if (PyObject_GetBuffer(item, &buffers[i], PyBUF_SIMPLE) != 0) { + Py_DECREF(item); PyErr_Format(PyExc_TypeError, "sequence item %zd: expected a bytes-like object, " "%.80s found", i, Py_TYPE(item)->tp_name); goto error; } + Py_DECREF(item); /* If the backing objects are mutable, then dropping the GIL * opens up race conditions where another thread tries to modify * the object which we hold a buffer on it. Such code has data diff --git a/Objects/structseq.c b/Objects/structseq.c index c05bcde24c441b5..2dc59b0ff64a382 100644 --- a/Objects/structseq.c +++ b/Objects/structseq.c @@ -445,6 +445,7 @@ structseq_replace(PyObject *op, PyObject *args, PyObject *kwargs) } } + _PyObject_GC_TRACK(result); return (PyObject *)result; error: diff --git a/Objects/templateobject.c b/Objects/templateobject.c index 7d356980b56cbbe..1609e82b444516c 100644 --- a/Objects/templateobject.c +++ b/Objects/templateobject.c @@ -23,11 +23,15 @@ templateiter_next(PyObject *op) if (self->from_strings) { item = PyIter_Next(self->stringsiter); self->from_strings = 0; + if (item == NULL) { + return NULL; + } if (PyUnicode_GET_LENGTH(item) == 0) { Py_SETREF(item, PyIter_Next(self->interpolationsiter)); self->from_strings = 1; } - } else { + } + else { item = PyIter_Next(self->interpolationsiter); self->from_strings = 1; } @@ -144,13 +148,14 @@ template_new(PyTypeObject *type, PyObject *args, PyObject *kwds) if (last_was_str) { PyObject *laststring = PyTuple_GET_ITEM(strings, stringsidx - 1); PyObject *concat = PyUnicode_Concat(laststring, item); - Py_DECREF(laststring); if (!concat) { Py_DECREF(strings); Py_DECREF(interpolations); return NULL; } + /* Replace laststring with concat */ PyTuple_SET_ITEM(strings, stringsidx - 1, concat); + Py_DECREF(laststring); } else { PyTuple_SET_ITEM(strings, stringsidx++, Py_NewRef(item)); @@ -242,54 +247,6 @@ template_iter(PyObject *op) return (PyObject *)iter; } -static PyObject * -template_strings_append_str(PyObject *strings, PyObject *str) -{ - Py_ssize_t stringslen = PyTuple_GET_SIZE(strings); - PyObject *string = PyTuple_GET_ITEM(strings, stringslen - 1); - PyObject *concat = PyUnicode_Concat(string, str); - if (concat == NULL) { - return NULL; - } - - PyObject *newstrings = PyTuple_New(stringslen); - if (newstrings == NULL) { - Py_DECREF(concat); - return NULL; - } - - for (Py_ssize_t i = 0; i < stringslen - 1; i++) { - PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i))); - } - PyTuple_SET_ITEM(newstrings, stringslen - 1, concat); - - return newstrings; -} - -static PyObject * -template_strings_prepend_str(PyObject *strings, PyObject *str) -{ - Py_ssize_t stringslen = PyTuple_GET_SIZE(strings); - PyObject *string = PyTuple_GET_ITEM(strings, 0); - PyObject *concat = PyUnicode_Concat(str, string); - if (concat == NULL) { - return NULL; - } - - PyObject *newstrings = PyTuple_New(stringslen); - if (newstrings == NULL) { - Py_DECREF(concat); - return NULL; - } - - PyTuple_SET_ITEM(newstrings, 0, concat); - for (Py_ssize_t i = 1; i < stringslen; i++) { - PyTuple_SET_ITEM(newstrings, i, Py_NewRef(PyTuple_GET_ITEM(strings, i))); - } - - return newstrings; -} - static PyObject * template_strings_concat(PyObject *left, PyObject *right) { @@ -341,47 +298,17 @@ template_concat_templates(templateobject *self, templateobject *other) return newtemplate; } -static PyObject * -template_concat_template_str(templateobject *self, PyObject *other) -{ - PyObject *newstrings = template_strings_append_str(self->strings, other); - if (newstrings == NULL) { - return NULL; - } - - PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations); - Py_DECREF(newstrings); - return newtemplate; -} - -static PyObject * -template_concat_str_template(templateobject *self, PyObject *other) -{ - PyObject *newstrings = template_strings_prepend_str(self->strings, other); - if (newstrings == NULL) { - return NULL; - } - - PyObject *newtemplate = _PyTemplate_Build(newstrings, self->interpolations); - Py_DECREF(newstrings); - return newtemplate; -} - PyObject * _PyTemplate_Concat(PyObject *self, PyObject *other) { if (_PyTemplate_CheckExact(self) && _PyTemplate_CheckExact(other)) { return template_concat_templates((templateobject *) self, (templateobject *) other); } - else if ((_PyTemplate_CheckExact(self)) && PyUnicode_Check(other)) { - return template_concat_template_str((templateobject *) self, other); - } - else if (PyUnicode_Check(self) && (_PyTemplate_CheckExact(other))) { - return template_concat_str_template((templateobject *) other, self); - } - else { - Py_RETURN_NOTIMPLEMENTED; - } + + PyErr_Format(PyExc_TypeError, + "can only concatenate string.templatelib.Template (not \"%T\") to string.templatelib.Template", + other); + return NULL; } static PyObject * @@ -444,6 +371,12 @@ template_reduce(PyObject *op, PyObject *Py_UNUSED(dummy)) static PyMethodDef template_methods[] = { {"__reduce__", template_reduce, METH_NOARGS, NULL}, + {"__class_getitem__", Py_GenericAlias, + METH_O|METH_CLASS, + // note that this is not supported in typeshed, and it is not clear if the + // type for this is a simple TypeVar or a TypeVarTuple + // for details, see: https://github.com/python/typeshed/issues/14878 + PyDoc_STR("Template supports [] for generic usage")}, {NULL, NULL}, }; diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 9b31758485ca5e5..0f7a37faa91a577 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -118,7 +118,7 @@ int PyTuple_SetItem(PyObject *op, Py_ssize_t i, PyObject *newitem) { PyObject **p; - if (!PyTuple_Check(op) || Py_REFCNT(op) != 1) { + if (!PyTuple_Check(op) || !_PyObject_IsUniquelyReferenced(op)) { Py_XDECREF(newitem); PyErr_BadInternalCall(); return -1; @@ -156,6 +156,18 @@ _PyTuple_MaybeUntrack(PyObject *op) _PyObject_GC_UNTRACK(op); } +/* Fast, but conservative check if an object maybe tracked + May return true for an object that is not tracked, + Will always return true for an object that is tracked. + This is a temporary workaround until _PyObject_GC_IS_TRACKED + becomes fast and safe to call on non-GC objects. +*/ +static bool +maybe_tracked(PyObject *ob) +{ + return _PyType_IS_GC(Py_TYPE(ob)); +} + PyObject * PyTuple_Pack(Py_ssize_t n, ...) { @@ -163,6 +175,7 @@ PyTuple_Pack(Py_ssize_t n, ...) PyObject *o; PyObject **items; va_list vargs; + bool track = false; if (n == 0) { return tuple_get_empty(); @@ -177,10 +190,15 @@ PyTuple_Pack(Py_ssize_t n, ...) items = result->ob_item; for (i = 0; i < n; i++) { o = va_arg(vargs, PyObject *); + if (!track && maybe_tracked(o)) { + track = true; + } items[i] = Py_NewRef(o); } va_end(vargs); - _PyObject_GC_TRACK(result); + if (track) { + _PyObject_GC_TRACK(result); + } return (PyObject *)result; } @@ -377,11 +395,17 @@ _PyTuple_FromArray(PyObject *const *src, Py_ssize_t n) return NULL; } PyObject **dst = tuple->ob_item; + bool track = false; for (Py_ssize_t i = 0; i < n; i++) { PyObject *item = src[i]; + if (!track && maybe_tracked(item)) { + track = true; + } dst[i] = Py_NewRef(item); } - _PyObject_GC_TRACK(tuple); + if (track) { + _PyObject_GC_TRACK(tuple); + } return (PyObject *)tuple; } @@ -396,10 +420,17 @@ _PyTuple_FromStackRefStealOnSuccess(const _PyStackRef *src, Py_ssize_t n) return NULL; } PyObject **dst = tuple->ob_item; + bool track = false; for (Py_ssize_t i = 0; i < n; i++) { - dst[i] = PyStackRef_AsPyObjectSteal(src[i]); + PyObject *item = PyStackRef_AsPyObjectSteal(src[i]); + if (!track && maybe_tracked(item)) { + track = true; + } + dst[i] = item; + } + if (track) { + _PyObject_GC_TRACK(tuple); } - _PyObject_GC_TRACK(tuple); return (PyObject *)tuple; } @@ -844,11 +875,19 @@ tuple___getnewargs___impl(PyTupleObject *self) return Py_BuildValue("(N)", tuple_slice(self, 0, Py_SIZE(self))); } + +PyDoc_STRVAR(tuple_class_getitem_doc, +"Tuples are generic over the types of their contents.\n\n\ +For example, use ``tuple[int, str]`` for a pair whose first element\n\ +is an int and second element is a string.\n\n\ +Tuples also support the form ``tuple[T, ...]`` to indicate\n\ +an arbitrary length tuple of elements of type T."); + static PyMethodDef tuple_methods[] = { TUPLE___GETNEWARGS___METHODDEF TUPLE_INDEX_METHODDEF TUPLE_COUNT_METHODDEF - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, tuple_class_getitem_doc}, {NULL, NULL} /* sentinel */ }; @@ -923,7 +962,7 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize) v = (PyTupleObject *) *pv; if (v == NULL || !Py_IS_TYPE(v, &PyTuple_Type) || - (Py_SIZE(v) != 0 && Py_REFCNT(v) != 1)) { + (Py_SIZE(v) != 0 && !_PyObject_IsUniquelyReferenced(*pv))) { *pv = 0; Py_XDECREF(v); PyErr_BadInternalCall(); diff --git a/Objects/typeobject.c b/Objects/typeobject.c index a7ab69fef4c7214..c0138933fead47c 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -51,10 +51,9 @@ class object "PyObject *" "&PyBaseObject_Type" MCACHE_HASH(FT_ATOMIC_LOAD_UINT32_RELAXED((type)->tp_version_tag), \ ((Py_ssize_t)(name)) >> 3) #define MCACHE_CACHEABLE_NAME(name) \ - PyUnicode_CheckExact(name) && \ - (PyUnicode_GET_LENGTH(name) <= MCACHE_MAX_ATTR_SIZE) + (PyUnicode_CheckExact(name) && \ + (PyUnicode_GET_LENGTH(name) <= MCACHE_MAX_ATTR_SIZE)) -#define NEXT_GLOBAL_VERSION_TAG _PyRuntime.types.next_version_tag #define NEXT_VERSION_TAG(interp) \ (interp)->types.next_version_tag @@ -66,17 +65,73 @@ class object "PyObject *" "&PyBaseObject_Type" // be released and reacquired during a subclass update if there's contention // on the subclass lock. #define TYPE_LOCK &PyInterpreterState_Get()->types.mutex -#define BEGIN_TYPE_LOCK() Py_BEGIN_CRITICAL_SECTION_MUT(TYPE_LOCK) +#define BEGIN_TYPE_LOCK() Py_BEGIN_CRITICAL_SECTION_MUTEX(TYPE_LOCK) #define END_TYPE_LOCK() Py_END_CRITICAL_SECTION() #define BEGIN_TYPE_DICT_LOCK(d) \ - Py_BEGIN_CRITICAL_SECTION2_MUT(TYPE_LOCK, &_PyObject_CAST(d)->ob_mutex) + Py_BEGIN_CRITICAL_SECTION2_MUTEX(TYPE_LOCK, &_PyObject_CAST(d)->ob_mutex) #define END_TYPE_DICT_LOCK() Py_END_CRITICAL_SECTION2() #define ASSERT_TYPE_LOCK_HELD() \ _Py_CRITICAL_SECTION_ASSERT_MUTEX_LOCKED(TYPE_LOCK) +static void +types_stop_world(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); +} + +static void +types_start_world(void) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StartTheWorld(interp); +} + +// This is used to temporarily prevent the TYPE_LOCK from being suspended +// when held by the topmost critical section. +static void +type_lock_prevent_release(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + uintptr_t *tagptr = &tstate->critical_section; + PyCriticalSection *c = (PyCriticalSection *)(*tagptr & ~_Py_CRITICAL_SECTION_MASK); + if (!(*tagptr & _Py_CRITICAL_SECTION_TWO_MUTEXES)) { + assert(c->_cs_mutex == TYPE_LOCK); + c->_cs_mutex = NULL; + } + else { + PyCriticalSection2 *c2 = (PyCriticalSection2 *)c; + if (c->_cs_mutex == TYPE_LOCK) { + c->_cs_mutex = c2->_cs_mutex2; + c2->_cs_mutex2 = NULL; + } + else { + assert(c2->_cs_mutex2 == TYPE_LOCK); + c2->_cs_mutex2 = NULL; + } + } +} + +static void +type_lock_allow_release(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + uintptr_t *tagptr = &tstate->critical_section; + PyCriticalSection *c = (PyCriticalSection *)(*tagptr & ~_Py_CRITICAL_SECTION_MASK); + if (!(*tagptr & _Py_CRITICAL_SECTION_TWO_MUTEXES)) { + assert(c->_cs_mutex == NULL); + c->_cs_mutex = TYPE_LOCK; + } + else { + PyCriticalSection2 *c2 = (PyCriticalSection2 *)c; + assert(c2->_cs_mutex2 == NULL); + c2->_cs_mutex2 = TYPE_LOCK; + } +} + #else #define BEGIN_TYPE_LOCK() @@ -84,6 +139,10 @@ class object "PyObject *" "&PyBaseObject_Type" #define BEGIN_TYPE_DICT_LOCK(d) #define END_TYPE_DICT_LOCK() #define ASSERT_TYPE_LOCK_HELD() +#define types_stop_world() +#define types_start_world() +#define type_lock_prevent_release() +#define type_lock_allow_release() #endif @@ -157,8 +216,8 @@ static_ext_type_lookup(PyInterpreterState *interp, size_t index, assert(index < _Py_MAX_MANAGED_STATIC_EXT_TYPES); size_t full_index = index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; - int64_t interp_count = - _PyRuntime.types.managed_static.types[full_index].interp_count; + int64_t interp_count = _Py_atomic_load_int64( + &_PyRuntime.types.managed_static.types[full_index].interp_count); assert((interp_count == 0) == (_PyRuntime.types.managed_static.types[full_index].type == NULL)); *p_interp_count = interp_count; @@ -235,7 +294,7 @@ managed_static_type_state_init(PyInterpreterState *interp, PyTypeObject *self, : index + _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; assert((initial == 1) == - (_PyRuntime.types.managed_static.types[full_index].interp_count == 0)); + (_Py_atomic_load_int64(&_PyRuntime.types.managed_static.types[full_index].interp_count) == 0)); (void)_Py_atomic_add_int64( &_PyRuntime.types.managed_static.types[full_index].interp_count, 1); @@ -284,7 +343,7 @@ managed_static_type_state_clear(PyInterpreterState *interp, PyTypeObject *self, : &(interp->types.for_extensions.initialized[index]); assert(state != NULL); - assert(_PyRuntime.types.managed_static.types[full_index].interp_count > 0); + assert(_Py_atomic_load_int64(&_PyRuntime.types.managed_static.types[full_index].interp_count) > 0); assert(_PyRuntime.types.managed_static.types[full_index].type == state->type); assert(state->type != NULL); @@ -294,7 +353,7 @@ managed_static_type_state_clear(PyInterpreterState *interp, PyTypeObject *self, (void)_Py_atomic_add_int64( &_PyRuntime.types.managed_static.types[full_index].interp_count, -1); if (final) { - assert(!_PyRuntime.types.managed_static.types[full_index].interp_count); + assert(!_Py_atomic_load_int64(&_PyRuntime.types.managed_static.types[full_index].interp_count)); _PyRuntime.types.managed_static.types[full_index].type = NULL; managed_static_type_index_clear(self); @@ -542,7 +601,6 @@ clear_tp_bases(PyTypeObject *self, int final) static inline PyObject * lookup_tp_mro(PyTypeObject *self) { - ASSERT_TYPE_LOCK_HELD(); return self->tp_mro; } @@ -581,8 +639,19 @@ set_tp_mro(PyTypeObject *self, PyObject *mro, int initial) /* Other checks are done via set_tp_bases. */ _Py_SetImmortal(mro); } + else { + PyUnstable_Object_EnableDeferredRefcount(mro); + } + } + if (!initial) { + type_lock_prevent_release(); + types_stop_world(); } self->tp_mro = mro; + if (!initial) { + types_start_world(); + type_lock_allow_release(); + } } static inline void @@ -1258,6 +1327,19 @@ _PyType_GetVersionForCurrentState(PyTypeObject *tp) #error "_Py_ATTR_CACHE_UNUSED must be bigger than max" #endif +static inline unsigned int +next_global_version_tag(void) +{ + unsigned int old; + do { + old = _Py_atomic_load_uint_relaxed(&_PyRuntime.types.next_version_tag); + if (old >= _Py_MAX_GLOBAL_TYPE_VERSION_TAG) { + return 0; + } + } while (!_Py_atomic_compare_exchange_uint(&_PyRuntime.types.next_version_tag, &old, old + 1)); + return old + 1; +} + static int assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) { @@ -1288,11 +1370,12 @@ assign_version_tag(PyInterpreterState *interp, PyTypeObject *type) } if (type->tp_flags & Py_TPFLAGS_IMMUTABLETYPE) { /* static types */ - if (NEXT_GLOBAL_VERSION_TAG > _Py_MAX_GLOBAL_TYPE_VERSION_TAG) { + unsigned int next_version_tag = next_global_version_tag(); + if (next_version_tag == 0) { /* We have run out of version numbers */ return 0; } - set_version_unlocked(type, NEXT_GLOBAL_VERSION_TAG++); + set_version_unlocked(type, next_version_tag); assert (type->tp_version_tag <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); } else { @@ -1421,9 +1504,13 @@ type_set_name(PyObject *tp, PyObject *value, void *Py_UNUSED(closure)) return -1; } + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); type->tp_name = tp_name; - Py_SETREF(((PyHeapTypeObject*)type)->ht_name, Py_NewRef(value)); - + PyObject *old_name = ((PyHeapTypeObject*)type)->ht_name; + ((PyHeapTypeObject*)type)->ht_name = Py_NewRef(value); + _PyEval_StartTheWorld(interp); + Py_DECREF(old_name); return 0; } @@ -1610,18 +1697,11 @@ static PyObject * type_get_mro(PyObject *tp, void *Py_UNUSED(closure)) { PyTypeObject *type = PyTypeObject_CAST(tp); - PyObject *mro; - - BEGIN_TYPE_LOCK(); - mro = lookup_tp_mro(type); + PyObject *mro = lookup_tp_mro(type); if (mro == NULL) { - mro = Py_None; - } else { - Py_INCREF(mro); + Py_RETURN_NONE; } - - END_TYPE_LOCK(); - return mro; + return Py_NewRef(mro); } static PyTypeObject *best_base(PyObject *); @@ -1641,7 +1721,7 @@ static int recurse_down_subclasses(PyTypeObject *type, PyObject *name, update_callback callback, void *data); static int -mro_hierarchy(PyTypeObject *type, PyObject *temp) +mro_hierarchy_for_complete_type(PyTypeObject *type, PyObject *temp) { ASSERT_TYPE_LOCK_HELD(); @@ -1652,6 +1732,7 @@ mro_hierarchy(PyTypeObject *type, PyObject *temp) return res; } PyObject *new_mro = lookup_tp_mro(type); + assert(new_mro != NULL); PyObject *tuple; if (old_mro != NULL) { @@ -1696,7 +1777,7 @@ mro_hierarchy(PyTypeObject *type, PyObject *temp) Py_ssize_t n = PyList_GET_SIZE(subclasses); for (Py_ssize_t i = 0; i < n; i++) { PyTypeObject *subclass = _PyType_CAST(PyList_GET_ITEM(subclasses, i)); - res = mro_hierarchy(subclass, temp); + res = mro_hierarchy_for_complete_type(subclass, temp); if (res < 0) { break; } @@ -1778,7 +1859,7 @@ type_set_bases_unlocked(PyTypeObject *type, PyObject *new_bases) if (temp == NULL) { goto bail; } - if (mro_hierarchy(type, temp) < 0) { + if (mro_hierarchy_for_complete_type(type, temp) < 0) { goto undo; } Py_DECREF(temp); @@ -2065,19 +2146,46 @@ type_set_annotations(PyObject *tp, PyObject *value, void *Py_UNUSED(closure)) return -1; } - int result; PyObject *dict = PyType_GetDict(type); - if (value != NULL) { - /* set */ - result = PyDict_SetItem(dict, &_Py_ID(__annotations_cache__), value); - } else { - /* delete */ - result = PyDict_Pop(dict, &_Py_ID(__annotations_cache__), NULL); - if (result == 0) { - PyErr_SetString(PyExc_AttributeError, "__annotations__"); + int result = PyDict_ContainsString(dict, "__annotations__"); + if (result < 0) { + Py_DECREF(dict); + return -1; + } + if (result) { + // If __annotations__ is currently in the dict, we update it, + if (value != NULL) { + result = PyDict_SetItem(dict, &_Py_ID(__annotations__), value); + } else { + result = PyDict_Pop(dict, &_Py_ID(__annotations__), NULL); + if (result == 0) { + // Somebody else just deleted it? + PyErr_SetString(PyExc_AttributeError, "__annotations__"); + Py_DECREF(dict); + return -1; + } + } + if (result < 0) { Py_DECREF(dict); return -1; } + // Also clear __annotations_cache__ just in case. + result = PyDict_Pop(dict, &_Py_ID(__annotations_cache__), NULL); + } + else { + // Else we update only __annotations_cache__. + if (value != NULL) { + /* set */ + result = PyDict_SetItem(dict, &_Py_ID(__annotations_cache__), value); + } else { + /* delete */ + result = PyDict_Pop(dict, &_Py_ID(__annotations_cache__), NULL); + if (result == 0) { + PyErr_SetString(PyExc_AttributeError, "__annotations__"); + Py_DECREF(dict); + return -1; + } + } } if (result < 0) { Py_DECREF(dict); @@ -3247,6 +3355,7 @@ mro_implementation_unlocked(PyTypeObject *type) */ PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(bases, 0)); PyObject *base_mro = lookup_tp_mro(base); + assert(base_mro != NULL); Py_ssize_t k = PyTuple_GET_SIZE(base_mro); PyObject *result = PyTuple_New(k + 1); if (result == NULL) { @@ -3281,9 +3390,12 @@ mro_implementation_unlocked(PyTypeObject *type) return NULL; } + PyObject *mro_to_merge; for (Py_ssize_t i = 0; i < n; i++) { PyTypeObject *base = _PyType_CAST(PyTuple_GET_ITEM(bases, i)); - to_merge[i] = lookup_tp_mro(base); + mro_to_merge = lookup_tp_mro(base); + assert(mro_to_merge != NULL); + to_merge[i] = mro_to_merge; } to_merge[n] = bases; @@ -3649,10 +3761,39 @@ subtype_dict(PyObject *obj, void *context) return PyObject_GenericGetDict(obj, context); } +int +_PyObject_SetDict(PyObject *obj, PyObject *value) +{ + if (value != NULL && !PyDict_Check(value)) { + PyErr_Format(PyExc_TypeError, + "__dict__ must be set to a dictionary, " + "not a '%.200s'", Py_TYPE(value)->tp_name); + return -1; + } + if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) { + return _PyObject_SetManagedDict(obj, value); + } + PyObject **dictptr = _PyObject_ComputedDictPointer(obj); + if (dictptr == NULL) { + PyErr_SetString(PyExc_AttributeError, + "This object has no __dict__"); + return -1; + } + Py_BEGIN_CRITICAL_SECTION(obj); + PyObject *olddict = *dictptr; + FT_ATOMIC_STORE_PTR_RELEASE(*dictptr, Py_NewRef(value)); +#ifdef Py_GIL_DISABLED + _PyObject_XDecRefDelayed(olddict); +#else + Py_XDECREF(olddict); +#endif + Py_END_CRITICAL_SECTION(); + return 0; +} + static int subtype_setdict(PyObject *obj, PyObject *value, void *context) { - PyObject **dictptr; PyTypeObject *base; base = get_builtin_base_with_dict(Py_TYPE(obj)); @@ -3670,28 +3811,7 @@ subtype_setdict(PyObject *obj, PyObject *value, void *context) } return func(descr, obj, value); } - /* Almost like PyObject_GenericSetDict, but allow __dict__ to be deleted. */ - if (value != NULL && !PyDict_Check(value)) { - PyErr_Format(PyExc_TypeError, - "__dict__ must be set to a dictionary, " - "not a '%.200s'", Py_TYPE(value)->tp_name); - return -1; - } - - if (Py_TYPE(obj)->tp_flags & Py_TPFLAGS_MANAGED_DICT) { - return _PyObject_SetManagedDict(obj, value); - } - else { - dictptr = _PyObject_ComputedDictPointer(obj); - if (dictptr == NULL) { - PyErr_SetString(PyExc_AttributeError, - "This object has no __dict__"); - return -1; - } - Py_CLEAR(*dictptr); - *dictptr = Py_XNewRef(value); - } - return 0; + return _PyObject_SetDict(obj, value); } static PyObject * @@ -4613,6 +4733,13 @@ type_new_get_bases(type_new_ctx *ctx, PyObject **type) if (winner != ctx->metatype) { if (winner->tp_new != type_new) { + /* Check if tp_new is NULL (cannot instantiate this type) */ + if (winner->tp_new == NULL) { + PyErr_Format(PyExc_TypeError, + "cannot create '%.400s' instances", + winner->tp_name); + return -1; + } /* Pass it to the winner */ *type = winner->tp_new(winner, ctx->args, ctx->kwds); if (*type == NULL) { @@ -4774,28 +4901,28 @@ check_basicsize_includes_size_and_offsets(PyTypeObject* type) if (type->tp_base && type->tp_base->tp_basicsize > type->tp_basicsize) { PyErr_Format(PyExc_TypeError, - "tp_basicsize for type '%s' (%d) is too small for base '%s' (%d)", + "tp_basicsize for type '%s' (%zd) is too small for base '%s' (%zd)", type->tp_name, type->tp_basicsize, type->tp_base->tp_name, type->tp_base->tp_basicsize); return 0; } if (type->tp_weaklistoffset + (Py_ssize_t)sizeof(PyObject*) > max) { PyErr_Format(PyExc_TypeError, - "weaklist offset %d is out of bounds for type '%s' (tp_basicsize = %d)", + "weaklist offset %zd is out of bounds for type '%s' (tp_basicsize = %zd)", type->tp_weaklistoffset, type->tp_name, type->tp_basicsize); return 0; } if (type->tp_dictoffset + (Py_ssize_t)sizeof(PyObject*) > max) { PyErr_Format(PyExc_TypeError, - "dict offset %d is out of bounds for type '%s' (tp_basicsize = %d)", + "dict offset %zd is out of bounds for type '%s' (tp_basicsize = %zd)", type->tp_dictoffset, type->tp_name, type->tp_basicsize); return 0; } if (type->tp_vectorcall_offset + (Py_ssize_t)sizeof(vectorcallfunc*) > max) { PyErr_Format(PyExc_TypeError, - "vectorcall offset %d is out of bounds for type '%s' (tp_basicsize = %d)", + "vectorcall offset %zd is out of bounds for type '%s' (tp_basicsize = %zd)", type->tp_vectorcall_offset, type->tp_name, type->tp_basicsize); return 0; @@ -4994,6 +5121,7 @@ PyType_FromMetaclass( Py_ssize_t name_buf_len = strlen(spec->name) + 1; _ht_tpname = PyMem_Malloc(name_buf_len); if (_ht_tpname == NULL) { + PyErr_NoMemory(); goto finally; } memcpy(_ht_tpname, spec->name, name_buf_len); @@ -5277,7 +5405,7 @@ PyType_FromMetaclass( assert(_PyType_CheckConsistency(type)); - finally: +finally: if (PyErr_Occurred()) { Py_CLEAR(res); } @@ -5476,23 +5604,15 @@ get_base_by_token_recursive(PyObject *bases, void *token) } int -PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) +_PyType_GetBaseByToken_Borrow(PyTypeObject *type, void *token, PyTypeObject **result) { + assert(token != NULL); + assert(PyType_Check(type)); + if (result != NULL) { *result = NULL; } - if (token == NULL) { - PyErr_Format(PyExc_SystemError, - "PyType_GetBaseByToken called with token=NULL"); - return -1; - } - if (!PyType_Check(type)) { - PyErr_Format(PyExc_TypeError, - "expected a type, got a '%T' object", type); - return -1; - } - if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) { // No static type has a heaptype superclass, // which is ensured by type_ready_mro(). @@ -5501,7 +5621,7 @@ PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) if (((PyHeapTypeObject*)type)->ht_token == token) { found: if (result != NULL) { - *result = (PyTypeObject *)Py_NewRef(type); + *result = type; } return 1; } @@ -5535,6 +5655,30 @@ PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) return 0; } +int +PyType_GetBaseByToken(PyTypeObject *type, void *token, PyTypeObject **result) +{ + if (result != NULL) { + *result = NULL; + } + if (token == NULL) { + PyErr_Format(PyExc_SystemError, + "PyType_GetBaseByToken called with token=NULL"); + return -1; + } + if (!PyType_Check(type)) { + PyErr_Format(PyExc_TypeError, + "expected a type, got a '%T' object", type); + return -1; + } + + int res = _PyType_GetBaseByToken_Borrow(type, token, result); + if (res > 0 && result) { + Py_INCREF(*result); + } + return res; +} + void * PyObject_GetTypeData(PyObject *obj, PyTypeObject *cls) @@ -5566,17 +5710,16 @@ PyObject_GetItemData(PyObject *obj) } /* Internal API to look for a name through the MRO, bypassing the method cache. - This returns a borrowed reference, and might set an exception. - 'error' is set to: -1: error with exception; 1: error without exception; 0: ok */ -static PyObject * -find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) + The result is stored as a _PyStackRef in `out`. It never set an exception. + Returns -1 if there was an error, 0 if the name was not found, and 1 if + the name was found. */ +static int +find_name_in_mro(PyTypeObject *type, PyObject *name, _PyStackRef *out) { - ASSERT_TYPE_LOCK_HELD(); - Py_hash_t hash = _PyObject_HashFast(name); if (hash == -1) { - *error = -1; - return NULL; + PyErr_Clear(); + return -1; } /* Look in tp_dict of types in MRO */ @@ -5584,37 +5727,42 @@ find_name_in_mro(PyTypeObject *type, PyObject *name, int *error) if (mro == NULL) { if (!is_readying(type)) { if (PyType_Ready(type) < 0) { - *error = -1; - return NULL; + PyErr_Clear(); + return -1; } mro = lookup_tp_mro(type); } if (mro == NULL) { - *error = 1; - return NULL; + return -1; } } - PyObject *res = NULL; + int res = 0; + PyThreadState *tstate = _PyThreadState_GET(); /* Keep a strong reference to mro because type->tp_mro can be replaced during dict lookup, e.g. when comparing to non-string keys. */ - Py_INCREF(mro); + _PyCStackRef mro_ref; + _PyThreadState_PushCStackRef(tstate, &mro_ref); + mro_ref.ref = PyStackRef_FromPyObjectNew(mro); Py_ssize_t n = PyTuple_GET_SIZE(mro); for (Py_ssize_t i = 0; i < n; i++) { PyObject *base = PyTuple_GET_ITEM(mro, i); PyObject *dict = lookup_tp_dict(_PyType_CAST(base)); assert(dict && PyDict_Check(dict)); - if (_PyDict_GetItemRef_KnownHash((PyDictObject *)dict, name, hash, &res) < 0) { - *error = -1; + Py_ssize_t ix = _Py_dict_lookup_threadsafe_stackref( + (PyDictObject *)dict, name, hash, out); + if (ix == DKIX_ERROR) { + PyErr_Clear(); + res = -1; goto done; } - if (res != NULL) { + if (!PyStackRef_IsNull(*out)) { + res = 1; break; } } - *error = 0; done: - Py_DECREF(mro); + _PyThreadState_PopCStackRef(tstate, &mro_ref); return res; } @@ -5718,6 +5866,14 @@ _PyType_LookupRefAndVersion(PyTypeObject *type, PyObject *name, unsigned int *ve return PyStackRef_AsPyObjectSteal(out); } +static int +should_assign_version_tag(PyTypeObject *type, PyObject *name, unsigned int version_tag) +{ + return (version_tag == 0 + && FT_ATOMIC_LOAD_UINT16_RELAXED(type->tp_versions_used) < MAX_VERSIONS_PER_CLASS + && MCACHE_CACHEABLE_NAME(name)); +} + unsigned int _PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef *out) { @@ -5766,52 +5922,39 @@ _PyType_LookupStackRefAndVersion(PyTypeObject *type, PyObject *name, _PyStackRef /* We may end up clearing live exceptions below, so make sure it's ours. */ assert(!PyErr_Occurred()); - // We need to atomically do the lookup and capture the version before - // anyone else can modify our mro or mutate the type. - - PyObject *res; - int error; + int res; PyInterpreterState *interp = _PyInterpreterState_GET(); - int has_version = 0; - unsigned int assigned_version = 0; - BEGIN_TYPE_LOCK(); - // We must assign the version before doing the lookup. If - // find_name_in_mro() blocks and releases the critical section - // then the type version can change. - if (MCACHE_CACHEABLE_NAME(name)) { - has_version = assign_version_tag(interp, type); - assigned_version = type->tp_version_tag; - } - res = find_name_in_mro(type, name, &error); - END_TYPE_LOCK(); + + unsigned int version_tag = FT_ATOMIC_LOAD_UINT(type->tp_version_tag); + if (should_assign_version_tag(type, name, version_tag)) { + BEGIN_TYPE_LOCK(); + assign_version_tag(interp, type); + version_tag = type->tp_version_tag; + res = find_name_in_mro(type, name, out); + END_TYPE_LOCK(); + } + else { + res = find_name_in_mro(type, name, out); + } /* Only put NULL results into cache if there was no error. */ - if (error) { - /* It's not ideal to clear the error condition, - but this function is documented as not setting - an exception, and I don't want to change that. - E.g., when PyType_Ready() can't proceed, it won't - set the "ready" flag, so future attempts to ready - the same type will call it again -- hopefully - in a context that propagates the exception out. - */ - if (error == -1) { - PyErr_Clear(); - } + if (res < 0) { *out = PyStackRef_NULL; return 0; } - if (has_version) { + if (version_tag == 0 || !MCACHE_CACHEABLE_NAME(name)) { + return 0; + } + + PyObject *res_obj = PyStackRef_AsPyObjectBorrow(*out); #if Py_GIL_DISABLED - update_cache_gil_disabled(entry, name, assigned_version, res); + update_cache_gil_disabled(entry, name, version_tag, res_obj); #else - PyObject *old_value = update_cache(entry, name, assigned_version, res); - Py_DECREF(old_value); + PyObject *old_value = update_cache(entry, name, version_tag, res_obj); + Py_DECREF(old_value); #endif - } - *out = res ? PyStackRef_FromPyObjectSteal(res) : PyStackRef_NULL; - return has_version ? assigned_version : 0; + return version_tag; } /* Internal API to look for a name through the MRO. @@ -5931,6 +6074,15 @@ void _PyType_SetFlagsRecursive(PyTypeObject *self, unsigned long mask, unsigned long flags) { BEGIN_TYPE_LOCK(); + /* Ideally, changing flags and invalidating the old version tag would happen + in one step. In 3.14, invalidate first while holding TYPE_LOCK so readers + cannot assign a fresh tag from stale flags. Immutable types are skipped by + set_flags_recursive(). */ + if (!PyType_HasFeature(self, Py_TPFLAGS_IMMUTABLETYPE) && + (self->tp_flags & mask) != flags) + { + type_modified_unlocked(self); + } set_flags_recursive(self, mask, flags); END_TYPE_LOCK(); } @@ -6124,6 +6276,18 @@ type_setattro(PyObject *self, PyObject *name, PyObject *value) assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_INLINE_VALUES)); assert(!_PyType_HasFeature(metatype, Py_TPFLAGS_MANAGED_DICT)); +#ifdef Py_GIL_DISABLED + // gh-139103: Enable deferred refcounting for functions assigned + // to type objects. This is important for `dataclass.__init__`, + // which is generated dynamically. + if (value != NULL && + PyFunction_Check(value) && + !_PyObject_HasDeferredRefcount(value)) + { + PyUnstable_Object_EnableDeferredRefcount(value); + } +#endif + PyObject *old_value = NULL; PyObject *descr = _PyType_LookupRef(metatype, name); if (descr != NULL) { @@ -6651,12 +6815,6 @@ PyTypeObject PyType_Type = { symmetrically, __new__() complains about excess arguments unless __init__() is overridden and __new__() is not overridden (IOW, if __new__() is overridden or __init__() is not overridden). - - However, for backwards compatibility, this breaks too much code. - Therefore, in 2.6, we'll *warn* about excess arguments when both - methods are overridden; for all other cases we'll use the above - rules. - */ /* Forward */ @@ -7041,7 +7199,11 @@ object_set_class_world_stopped(PyObject *self, PyTypeObject *newto) assert(_PyObject_GetManagedDict(self) == dict); - if (_PyDict_DetachFromObject(dict, self) < 0) { + int err; + Py_BEGIN_CRITICAL_SECTION(dict); + err = _PyDict_DetachFromObject(dict, self); + Py_END_CRITICAL_SECTION(); + if (err < 0) { return -1; } @@ -7083,12 +7245,17 @@ object_set_class(PyObject *self, PyObject *value, void *closure) #ifdef Py_GIL_DISABLED PyInterpreterState *interp = _PyInterpreterState_GET(); - _PyEval_StopTheWorld(interp); + int unique = _PyObject_IsUniquelyReferenced(self); + if (!unique) { + _PyEval_StopTheWorld(interp); + } #endif PyTypeObject *oldto = Py_TYPE(self); int res = object_set_class_world_stopped(self, newto); #ifdef Py_GIL_DISABLED - _PyEval_StartTheWorld(interp); + if (!unique) { + _PyEval_StartTheWorld(interp); + } #endif if (res == 0) { if (oldto->tp_flags & Py_TPFLAGS_HEAPTYPE) { @@ -8571,6 +8738,7 @@ type_ready_inherit(PyTypeObject *type) // Inherit slots PyObject *mro = lookup_tp_mro(type); + assert(mro != NULL); Py_ssize_t n = PyTuple_GET_SIZE(mro); for (Py_ssize_t i = 1; i < n; i++) { PyObject *b = PyTuple_GET_ITEM(mro, i); @@ -8669,7 +8837,11 @@ type_ready_set_new(PyTypeObject *type, int initial) && base == &PyBaseObject_Type && !(type->tp_flags & Py_TPFLAGS_HEAPTYPE)) { - type_add_flags(type, Py_TPFLAGS_DISALLOW_INSTANTIATION); + if (initial) { + type_add_flags(type, Py_TPFLAGS_DISALLOW_INSTANTIATION); + } else { + assert(type->tp_flags & Py_TPFLAGS_DISALLOW_INSTANTIATION); + } } if (!(type->tp_flags & Py_TPFLAGS_DISALLOW_INSTANTIATION)) { @@ -8683,13 +8855,17 @@ type_ready_set_new(PyTypeObject *type, int initial) } } else { - // tp_new is NULL: inherit tp_new from base - type->tp_new = base->tp_new; + if (initial) { + // tp_new is NULL: inherit tp_new from base + type->tp_new = base->tp_new; + } } } else { // Py_TPFLAGS_DISALLOW_INSTANTIATION sets tp_new to NULL - type->tp_new = NULL; + if (initial) { + type->tp_new = NULL; + } } return 0; } @@ -8749,6 +8925,7 @@ type_ready_post_checks(PyTypeObject *type) PyErr_Format(PyExc_SystemError, "type %s has a tp_dictoffset that is too small", type->tp_name); + return -1; } } return 0; @@ -8822,7 +8999,12 @@ type_ready(PyTypeObject *type, int initial) } /* All done -- set the ready flag */ - type_add_flags(type, Py_TPFLAGS_READY); + if (initial) { + type_add_flags(type, Py_TPFLAGS_READY); + } else { + assert(type->tp_flags & Py_TPFLAGS_READY); + } + stop_readying(type); assert(_PyType_CheckConsistency(type)); @@ -8871,15 +9053,16 @@ init_static_type(PyInterpreterState *interp, PyTypeObject *self, assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_DICT)); assert(!(self->tp_flags & Py_TPFLAGS_MANAGED_WEAKREF)); - if ((self->tp_flags & Py_TPFLAGS_READY) == 0) { - assert(initial); + if (initial) { + assert((self->tp_flags & Py_TPFLAGS_READY) == 0); type_add_flags(self, _Py_TPFLAGS_STATIC_BUILTIN); type_add_flags(self, Py_TPFLAGS_IMMUTABLETYPE); - assert(NEXT_GLOBAL_VERSION_TAG <= _Py_MAX_GLOBAL_TYPE_VERSION_TAG); if (self->tp_version_tag == 0) { - _PyType_SetVersion(self, NEXT_GLOBAL_VERSION_TAG++); + unsigned int next_version_tag = next_global_version_tag(); + assert(next_version_tag != 0); + _PyType_SetVersion(self, next_version_tag); } } else { @@ -9682,6 +9865,11 @@ tp_new_wrapper(PyObject *self, PyObject *args, PyObject *kwds) /* If staticbase is NULL now, it is a really weird type. In the spirit of backwards compatibility (?), just shut up. */ if (staticbase && staticbase->tp_new != type->tp_new) { + if (staticbase->tp_new == NULL) { + PyErr_Format(PyExc_TypeError, + "cannot create '%s' instances", subtype->tp_name); + return NULL; + } PyErr_Format(PyExc_TypeError, "%s.__new__(%s) is not safe, use %s.__new__()", type->tp_name, @@ -10124,6 +10312,7 @@ slot_tp_hash(PyObject *self) return PyObject_HashNotImplemented(self); } if (!PyLong_Check(res)) { + Py_DECREF(res); PyErr_SetString(PyExc_TypeError, "__hash__ method should return an integer"); return -1; @@ -10211,7 +10400,10 @@ _Py_slot_tp_getattr_hook(PyObject *self, PyObject *name) getattr = _PyType_LookupRef(tp, &_Py_ID(__getattr__)); if (getattr == NULL) { /* No __getattr__ hook: use a simpler dispatcher */ +#ifndef Py_GIL_DISABLED + // Replacing the slot is only thread-safe if there is a GIL. tp->tp_getattro = _Py_slot_tp_getattro; +#endif return _Py_slot_tp_getattro(self, name); } /* speed hack: we could use lookup_maybe, but that would resolve the @@ -10336,9 +10528,11 @@ slot_tp_descr_get(PyObject *self, PyObject *obj, PyObject *type) get = _PyType_LookupRef(tp, &_Py_ID(__get__)); if (get == NULL) { +#ifndef Py_GIL_DISABLED /* Avoid further slowdowns */ if (tp->tp_descr_get == slot_tp_descr_get) tp->tp_descr_get = NULL; +#endif return Py_NewRef(self); } if (obj == NULL) @@ -11129,7 +11323,6 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) int use_generic = 0; int offset = p->offset; - int error; void **ptr = slotptr(type, offset); if (ptr == NULL) { @@ -11142,19 +11335,15 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) assert(!PyErr_Occurred()); do { /* Use faster uncached lookup as we won't get any cache hits during type setup. */ - descr = find_name_in_mro(type, p->name_strobj, &error); - if (descr == NULL) { - if (error == -1) { - /* It is unlikely but not impossible that there has been an exception - during lookup. Since this function originally expected no errors, - we ignore them here in order to keep up the interface. */ - PyErr_Clear(); - } + _PyStackRef descr_ref; + int res = find_name_in_mro(type, p->name_strobj, &descr_ref); + if (res <= 0) { if (ptr == (void**)&type->tp_iternext) { specific = (void *)_PyObject_NextNotImplemented; } continue; } + descr = PyStackRef_AsPyObjectBorrow(descr_ref); if (Py_IS_TYPE(descr, &PyWrapperDescr_Type) && ((PyWrapperDescrObject *)descr)->d_base->name_strobj == p->name_strobj) { void **tptr = resolve_slotdups(type, p->name_strobj); @@ -11222,7 +11411,7 @@ update_one_slot(PyTypeObject *type, pytype_slotdef *p) type_clear_flags(type, Py_TPFLAGS_HAVE_VECTORCALL); } } - Py_DECREF(descr); + PyStackRef_CLOSE(descr_ref); } while ((++p)->offset == offset); if (specific && !use_generic) *ptr = specific; @@ -11653,18 +11842,16 @@ _super_lookup_descr(PyTypeObject *su_type, PyTypeObject *su_obj_type, PyObject * PyObject *mro, *res; Py_ssize_t i, n; - BEGIN_TYPE_LOCK(); mro = lookup_tp_mro(su_obj_type); - /* keep a strong reference to mro because su_obj_type->tp_mro can be - replaced during PyDict_GetItemRef(dict, name, &res) and because - another thread can modify it after we end the critical section - below */ - Py_XINCREF(mro); - END_TYPE_LOCK(); - if (mro == NULL) return NULL; + /* Keep a strong reference to mro because su_obj_type->tp_mro can be + replaced during PyDict_GetItemRef(dict, name, &res). */ + PyThreadState *tstate = _PyThreadState_GET(); + _PyCStackRef mro_ref; + _PyThreadState_PushCStackRefNew(tstate, &mro_ref, mro); + assert(PyTuple_Check(mro)); n = PyTuple_GET_SIZE(mro); @@ -11675,7 +11862,7 @@ _super_lookup_descr(PyTypeObject *su_type, PyTypeObject *su_obj_type, PyObject * } i++; /* skip su->type (if any) */ if (i >= n) { - Py_DECREF(mro); + _PyThreadState_PopCStackRef(tstate, &mro_ref); return NULL; } @@ -11686,13 +11873,13 @@ _super_lookup_descr(PyTypeObject *su_type, PyTypeObject *su_obj_type, PyObject * if (PyDict_GetItemRef(dict, name, &res) != 0) { // found or error - Py_DECREF(mro); + _PyThreadState_PopCStackRef(tstate, &mro_ref); return res; } i++; } while (i < n); - Py_DECREF(mro); + _PyThreadState_PopCStackRef(tstate, &mro_ref); return NULL; } @@ -12022,7 +12209,8 @@ PyDoc_STRVAR(super_doc, "super() -> same as super(__class__, <first argument>)\n" "super(type) -> unbound super object\n" "super(type, obj) -> bound super object; requires isinstance(obj, type)\n" -"super(type, type2) -> bound super object; requires issubclass(type2, type)\n" +"super(type, type2) -> bound super object; requires\n" +" issubclass(type2, type)\n" "Typical use to call a cooperative superclass method:\n" "class C(B):\n" " def meth(self, arg):\n" diff --git a/Objects/typevarobject.c b/Objects/typevarobject.c index 6c199a52aa0ae61..80d3fd488afae21 100644 --- a/Objects/typevarobject.c +++ b/Objects/typevarobject.c @@ -192,7 +192,7 @@ constevaluator_call(PyObject *self, PyObject *args, PyObject *kwargs) for (Py_ssize_t i = 0; i < PyTuple_GET_SIZE(value); i++) { PyObject *item = PyTuple_GET_ITEM(value, i); if (i > 0) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { PyUnicodeWriter_Discard(writer); return NULL; } @@ -273,7 +273,7 @@ _Py_typing_type_repr(PyUnicodeWriter *writer, PyObject *p) } if (p == (PyObject *)&_PyNone_Type) { - return PyUnicodeWriter_WriteUTF8(writer, "None", 4); + return PyUnicodeWriter_WriteASCII(writer, "None", 4); } if ((rc = PyObject_HasAttrWithError(p, &_Py_ID(__origin__))) > 0 && @@ -472,7 +472,7 @@ typevar_dealloc(PyObject *self) _PyObject_GC_UNTRACK(self); - Py_DECREF(tv->name); + Py_XDECREF(tv->name); Py_XDECREF(tv->bound); Py_XDECREF(tv->evaluate_bound); Py_XDECREF(tv->constraints); @@ -491,20 +491,21 @@ typevar_traverse(PyObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); typevarobject *tv = typevarobject_CAST(self); + Py_VISIT(tv->name); Py_VISIT(tv->bound); Py_VISIT(tv->evaluate_bound); Py_VISIT(tv->constraints); Py_VISIT(tv->evaluate_constraints); Py_VISIT(tv->default_value); Py_VISIT(tv->evaluate_default); - PyObject_VisitManagedDict(self, visit, arg); - return 0; + return PyObject_VisitManagedDict(self, visit, arg); } static int typevar_clear(PyObject *op) { typevarobject *self = typevarobject_CAST(op); + Py_CLEAR(self->name); Py_CLEAR(self->bound); Py_CLEAR(self->evaluate_bound); Py_CLEAR(self->constraints); @@ -814,7 +815,7 @@ typevar_typing_prepare_subst_impl(typevarobject *self, PyObject *alias, } Py_DECREF(params); PyErr_Format(PyExc_TypeError, - "Too few arguments for %S; actual %d, expected at least %d", + "Too few arguments for %S; actual %zd, expected at least %zd", alias, args_len, i + 1); return NULL; } @@ -1171,7 +1172,7 @@ paramspec_dealloc(PyObject *self) _PyObject_GC_UNTRACK(self); - Py_DECREF(ps->name); + Py_XDECREF(ps->name); Py_XDECREF(ps->bound); Py_XDECREF(ps->default_value); Py_XDECREF(ps->evaluate_default); @@ -1187,17 +1188,18 @@ paramspec_traverse(PyObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); paramspecobject *ps = paramspecobject_CAST(self); + Py_VISIT(ps->name); Py_VISIT(ps->bound); Py_VISIT(ps->default_value); Py_VISIT(ps->evaluate_default); - PyObject_VisitManagedDict(self, visit, arg); - return 0; + return PyObject_VisitManagedDict(self, visit, arg); } static int paramspec_clear(PyObject *op) { paramspecobject *self = paramspecobject_CAST(op); + Py_CLEAR(self->name); Py_CLEAR(self->bound); Py_CLEAR(self->default_value); Py_CLEAR(self->evaluate_default); @@ -1446,13 +1448,13 @@ The following syntax creates a parameter specification that defaults\n\ to a callable accepting two positional-only arguments of types int\n\ and str:\n\ \n\ - type IntFuncDefault[**P = (int, str)] = Callable[P, int]\n\ + type IntFuncDefault[**P = [int, str]] = Callable[P, int]\n\ \n\ For compatibility with Python 3.11 and earlier, ParamSpec objects\n\ can also be created as follows::\n\ \n\ P = ParamSpec('P')\n\ - DefaultP = ParamSpec('DefaultP', default=(int, str))\n\ + DefaultP = ParamSpec('DefaultP', default=[int, str])\n\ \n\ Parameter specification variables exist primarily for the benefit of\n\ static type checkers. They are used to forward the parameter types of\n\ @@ -1519,7 +1521,7 @@ typevartuple_dealloc(PyObject *self) _PyObject_GC_UNTRACK(self); typevartupleobject *tvt = typevartupleobject_CAST(self); - Py_DECREF(tvt->name); + Py_XDECREF(tvt->name); Py_XDECREF(tvt->default_value); Py_XDECREF(tvt->evaluate_default); PyObject_ClearManagedDict(self); @@ -1683,16 +1685,17 @@ typevartuple_traverse(PyObject *self, visitproc visit, void *arg) { Py_VISIT(Py_TYPE(self)); typevartupleobject *tvt = typevartupleobject_CAST(self); + Py_VISIT(tvt->name); Py_VISIT(tvt->default_value); Py_VISIT(tvt->evaluate_default); - PyObject_VisitManagedDict(self, visit, arg); - return 0; + return PyObject_VisitManagedDict(self, visit, arg); } static int typevartuple_clear(PyObject *self) { typevartupleobject *tvt = typevartupleobject_CAST(self); + Py_CLEAR(tvt->name); Py_CLEAR(tvt->default_value); Py_CLEAR(tvt->evaluate_default); PyObject_ClearManagedDict(self); @@ -1851,7 +1854,7 @@ typealias_dealloc(PyObject *self) PyTypeObject *tp = Py_TYPE(self); _PyObject_GC_UNTRACK(self); typealiasobject *ta = typealiasobject_CAST(self); - Py_DECREF(ta->name); + Py_XDECREF(ta->name); Py_XDECREF(ta->type_params); Py_XDECREF(ta->compute_value); Py_XDECREF(ta->value); @@ -2032,6 +2035,7 @@ static int typealias_traverse(PyObject *op, visitproc visit, void *arg) { typealiasobject *self = typealiasobject_CAST(op); + Py_VISIT(self->name); Py_VISIT(self->type_params); Py_VISIT(self->compute_value); Py_VISIT(self->value); @@ -2043,6 +2047,7 @@ static int typealias_clear(PyObject *op) { typealiasobject *self = typealiasobject_CAST(op); + Py_CLEAR(self->name); Py_CLEAR(self->type_params); Py_CLEAR(self->compute_value); Py_CLEAR(self->value); @@ -2213,8 +2218,9 @@ PyDoc_STRVAR(generic_class_getitem_doc, "Parameterizes a generic class.\n\ \n\ At least, parameterizing a generic class is the *main* thing this\n\ -method does. For example, for some generic class `Foo`, this is called\n\ -when we do `Foo[int]` - there, with `cls=Foo` and `params=int`.\n\ +method does. For example, for some generic class `Foo`, this is\n\ +called when we do `Foo[int]` - there, with `cls=Foo` and\n\ +`params=int`.\n\ \n\ However, note that this method is also called when defining generic\n\ classes in the first place with `class Foo[T]: ...`.\n\ diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index eb3e1c48fd40509..6bd9e7d607cc414 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -56,7 +56,6 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_pyhash.h" // _Py_HashSecret_t #include "pycore_pylifecycle.h" // _Py_SetFileSystemEncoding() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_template.h" // _PyTemplate_Concat() #include "pycore_tuple.h" // _PyTuple_FromArray() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI #include "pycore_unicodeobject.h" // struct _Py_unicode_state @@ -88,14 +87,12 @@ class Py_UCS4_converter(CConverter): type = 'Py_UCS4' converter = 'convert_uc' - def converter_init(self): - if self.default is not unspecified: - self.c_default = ascii(self.default) - if len(self.c_default) > 4 or self.c_default[0] != "'": - self.c_default = hex(ord(self.default)) + def c_default_init(self): + import libclinic + self.c_default = libclinic.c_unichar_repr(self.default) [python start generated code]*/ -/*[python end generated code: output=da39a3ee5e6b4b0d input=88f5dd06cd8e7a61]*/ +/*[python end generated code: output=da39a3ee5e6b4b0d input=22f057b68fd9a65a]*/ /* --- Globals ------------------------------------------------------------ @@ -116,7 +113,9 @@ NOTE: In the interpreter's initialization phase, some globals are currently #endif #ifdef Py_GIL_DISABLED -# define LOCK_INTERNED(interp) PyMutex_Lock(&_Py_INTERP_CACHED_OBJECT(interp, interned_mutex)) +# define LOCK_INTERNED(interp) \ + PyMutex_LockFlags(&_Py_INTERP_CACHED_OBJECT(interp, interned_mutex), \ + _Py_LOCK_DONT_DETACH) # define UNLOCK_INTERNED(interp) PyMutex_Unlock(&_Py_INTERP_CACHED_OBJECT(interp, interned_mutex)) #else # define LOCK_INTERNED(interp) @@ -679,6 +678,14 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) { #define CHECK(expr) \ do { if (!(expr)) { _PyObject_ASSERT_FAILED_MSG(op, Py_STRINGIFY(expr)); } } while (0) +#ifdef Py_GIL_DISABLED +# define CHECK_IF_GIL(expr) (void)(expr) +# define CHECK_IF_FT(expr) CHECK(expr) +#else +# define CHECK_IF_GIL(expr) CHECK(expr) +# define CHECK_IF_FT(expr) (void)(expr) +#endif + assert(op != NULL); CHECK(PyUnicode_Check(op)); @@ -759,11 +766,9 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) /* Check interning state */ #ifdef Py_DEBUG - // Note that we do not check `_Py_IsImmortal(op)`, since stable ABI - // extensions can make immortal strings mortal (but with a high enough - // refcount). - // The other way is extremely unlikely (worth a potential failed assertion - // in a debug build), so we do check `!_Py_IsImmortal(op)`. + // Note that we do not check `_Py_IsImmortal(op)` in the GIL-enabled build + // since stable ABI extensions can make immortal strings mortal (but with a + // high enough refcount). switch (PyUnicode_CHECK_INTERNED(op)) { case SSTATE_NOT_INTERNED: if (ascii->state.statically_allocated) { @@ -773,18 +778,20 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content) // are static but use SSTATE_NOT_INTERNED } else { - CHECK(!_Py_IsImmortal(op)); + CHECK_IF_GIL(!_Py_IsImmortal(op)); } break; case SSTATE_INTERNED_MORTAL: CHECK(!ascii->state.statically_allocated); - CHECK(!_Py_IsImmortal(op)); + CHECK_IF_GIL(!_Py_IsImmortal(op)); break; case SSTATE_INTERNED_IMMORTAL: CHECK(!ascii->state.statically_allocated); + CHECK_IF_FT(_Py_IsImmortal(op)); break; case SSTATE_INTERNED_IMMORTAL_STATIC: CHECK(ascii->state.statically_allocated); + CHECK_IF_FT(_Py_IsImmortal(op)); break; default: Py_UNREACHABLE(); @@ -5498,7 +5505,6 @@ unicode_decode_utf8(const char *s, Py_ssize_t size, if (maxchr <= 255) { memcpy(PyUnicode_1BYTE_DATA(u), s, pos); s += pos; - size -= pos; writer.pos = pos; } @@ -5546,7 +5552,6 @@ unicode_decode_utf8_writer(_PyUnicodeWriter *writer, return 0; } s += decoded; - size -= decoded; } return unicode_decode_utf8_impl(writer, starts, s, end, @@ -6621,13 +6626,15 @@ _PyUnicode_GetNameCAPI(void) /* --- Unicode Escape Codec ----------------------------------------------- */ PyObject * -_PyUnicode_DecodeUnicodeEscapeInternal(const char *s, +_PyUnicode_DecodeUnicodeEscapeInternal2(const char *s, Py_ssize_t size, const char *errors, Py_ssize_t *consumed, - const char **first_invalid_escape) + int *first_invalid_escape_char, + const char **first_invalid_escape_ptr) { const char *starts = s; + const char *initial_starts = starts; _PyUnicodeWriter writer; const char *end; PyObject *errorHandler = NULL; @@ -6635,7 +6642,8 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, _PyUnicode_Name_CAPI *ucnhash_capi; // so we can remember if we've seen an invalid escape char or not - *first_invalid_escape = NULL; + *first_invalid_escape_char = -1; + *first_invalid_escape_ptr = NULL; if (size == 0) { if (consumed) { @@ -6723,9 +6731,12 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, } } if (ch > 0377) { - if (*first_invalid_escape == NULL) { - *first_invalid_escape = s-3; /* Back up 3 chars, since we've - already incremented s. */ + if (*first_invalid_escape_char == -1) { + *first_invalid_escape_char = ch; + if (starts == initial_starts) { + /* Back up 3 chars, since we've already incremented s. */ + *first_invalid_escape_ptr = s - 3; + } } } WRITE_CHAR(ch); @@ -6820,9 +6831,12 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, goto error; default: - if (*first_invalid_escape == NULL) { - *first_invalid_escape = s-1; /* Back up one char, since we've - already incremented s. */ + if (*first_invalid_escape_char == -1) { + *first_invalid_escape_char = c; + if (starts == initial_starts) { + /* Back up one char, since we've already incremented s. */ + *first_invalid_escape_ptr = s - 1; + } } WRITE_ASCII_CHAR('\\'); WRITE_CHAR(c); @@ -6867,19 +6881,20 @@ _PyUnicode_DecodeUnicodeEscapeStateful(const char *s, const char *errors, Py_ssize_t *consumed) { - const char *first_invalid_escape; - PyObject *result = _PyUnicode_DecodeUnicodeEscapeInternal(s, size, errors, + int first_invalid_escape_char; + const char *first_invalid_escape_ptr; + PyObject *result = _PyUnicode_DecodeUnicodeEscapeInternal2(s, size, errors, consumed, - &first_invalid_escape); + &first_invalid_escape_char, + &first_invalid_escape_ptr); if (result == NULL) return NULL; - if (first_invalid_escape != NULL) { - unsigned char c = *first_invalid_escape; - if ('4' <= c && c <= '7') { + if (first_invalid_escape_char != -1) { + if (first_invalid_escape_char > 0xff) { if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, - "\"\\%.3s\" is an invalid octal escape sequence. " + "\"\\%o\" is an invalid octal escape sequence. " "Such sequences will not work in the future. ", - first_invalid_escape) < 0) + first_invalid_escape_char) < 0) { Py_DECREF(result); return NULL; @@ -6889,7 +6904,7 @@ _PyUnicode_DecodeUnicodeEscapeStateful(const char *s, if (PyErr_WarnFormat(PyExc_DeprecationWarning, 1, "\"\\%c\" is an invalid escape sequence. " "Such sequences will not work in the future. ", - c) < 0) + first_invalid_escape_char) < 0) { Py_DECREF(result); return NULL; @@ -7704,10 +7719,6 @@ code_page_name(UINT code_page, PyObject **obj) *obj = NULL; if (code_page == CP_ACP) return "mbcs"; - if (code_page == CP_UTF7) - return "CP_UTF7"; - if (code_page == CP_UTF8) - return "CP_UTF8"; *obj = PyBytes_FromFormat("cp%u", code_page); if (*obj == NULL) @@ -8549,7 +8560,7 @@ charmap_decode_mapping(const char *s, goto Undefined; if (value < 0 || value > MAX_UNICODE) { PyErr_Format(PyExc_TypeError, - "character mapping must be in range(0x%x)", + "character mapping must be in range(0x%lx)", (unsigned long)MAX_UNICODE + 1); goto onError; } @@ -9298,8 +9309,8 @@ charmaptranslate_lookup(Py_UCS4 c, PyObject *mapping, PyObject **result, Py_UCS4 long value = PyLong_AsLong(x); if (value < 0 || value > MAX_UNICODE) { PyErr_Format(PyExc_ValueError, - "character mapping must be in range(0x%x)", - MAX_UNICODE+1); + "character mapping must be in range(0x%lx)", + (unsigned long)MAX_UNICODE + 1); Py_DECREF(x); return -1; } @@ -11037,9 +11048,9 @@ replace(PyObject *self, PyObject *str1, } done: - assert(srelease == (sbuf != PyUnicode_DATA(self))); - assert(release1 == (buf1 != PyUnicode_DATA(str1))); - assert(release2 == (buf2 != PyUnicode_DATA(str2))); + assert(srelease == (sbuf != NULL && sbuf != PyUnicode_DATA(self))); + assert(release1 == (buf1 != NULL && buf1 != PyUnicode_DATA(str1))); + assert(release2 == (buf2 != NULL && buf2 != PyUnicode_DATA(str2))); if (srelease) PyMem_Free((void *)sbuf); if (release1) @@ -11051,9 +11062,9 @@ replace(PyObject *self, PyObject *str1, nothing: /* nothing to replace; return original string (when possible) */ - assert(srelease == (sbuf != PyUnicode_DATA(self))); - assert(release1 == (buf1 != PyUnicode_DATA(str1))); - assert(release2 == (buf2 != PyUnicode_DATA(str2))); + assert(srelease == (sbuf != NULL && sbuf != PyUnicode_DATA(self))); + assert(release1 == (buf1 != NULL && buf1 != PyUnicode_DATA(str1))); + assert(release2 == (buf2 != NULL && buf2 != PyUnicode_DATA(str2))); if (srelease) PyMem_Free((void *)sbuf); if (release1) @@ -11063,9 +11074,9 @@ replace(PyObject *self, PyObject *str1, return unicode_result_unchanged(self); error: - assert(srelease == (sbuf != PyUnicode_DATA(self))); - assert(release1 == (buf1 != PyUnicode_DATA(str1))); - assert(release2 == (buf2 != PyUnicode_DATA(str2))); + assert(srelease == (sbuf != NULL && sbuf != PyUnicode_DATA(self))); + assert(release1 == (buf1 != NULL && buf1 != PyUnicode_DATA(str1))); + assert(release2 == (buf2 != NULL && buf2 != PyUnicode_DATA(str2))); if (srelease) PyMem_Free((void *)sbuf); if (release1) @@ -11082,13 +11093,13 @@ str.title as unicode_title Return a version of the string where each word is titlecased. -More specifically, words start with uppercased characters and all remaining -cased characters have lower case. +More specifically, words start with uppercased characters and all +remaining cased characters have lower case. [clinic start generated code]*/ static PyObject * unicode_title_impl(PyObject *self) -/*[clinic end generated code: output=c75ae03809574902 input=fa945d669b26e683]*/ +/*[clinic end generated code: output=c75ae03809574902 input=2a07e2c7df94627a]*/ { return case_operation(self, do_title); } @@ -11098,13 +11109,13 @@ str.capitalize as unicode_capitalize Return a capitalized version of the string. -More specifically, make the first character have upper case and the rest lower -case. +More specifically, make the first character have upper case and the +rest lower case. [clinic start generated code]*/ static PyObject * unicode_capitalize_impl(PyObject *self) -/*[clinic end generated code: output=e49a4c333cdb7667 input=f4cbf1016938da6d]*/ +/*[clinic end generated code: output=e49a4c333cdb7667 input=e50e50ed45a654cf]*/ { if (PyUnicode_GET_LENGTH(self) == 0) return unicode_result_unchanged(self); @@ -11158,12 +11169,13 @@ str.center as unicode_center Return a centered string of length width. -Padding is done using the specified fill character (default is a space). +Padding is done using the specified fill character (default is +a space). [clinic start generated code]*/ static PyObject * unicode_center_impl(PyObject *self, Py_ssize_t width, Py_UCS4 fillchar) -/*[clinic end generated code: output=420c8859effc7c0c input=b42b247eb26e6519]*/ +/*[clinic end generated code: output=420c8859effc7c0c input=df91017dfd186a78]*/ { Py_ssize_t marg, left; @@ -11629,16 +11641,10 @@ PyUnicode_Concat(PyObject *left, PyObject *right) return NULL; if (!PyUnicode_Check(right)) { - if (_PyTemplate_CheckExact(right)) { - // str + tstring is implemented in the tstring type - return _PyTemplate_Concat(left, right); - } - else { - PyErr_Format(PyExc_TypeError, - "can only concatenate str (not \"%.200s\") to str", - Py_TYPE(right)->tp_name); - return NULL; - } + PyErr_Format(PyExc_TypeError, + "can only concatenate str (not \"%.200s\") to str", + Py_TYPE(right)->tp_name); + return NULL; } /* Shortcuts */ @@ -11769,13 +11775,14 @@ str.count as unicode_count -> Py_ssize_t Return the number of non-overlapping occurrences of substring sub in string S[start:end]. -Optional arguments start and end are interpreted as in slice notation. +Optional arguments start and end are interpreted as in slice +notation. [clinic start generated code]*/ static Py_ssize_t unicode_count_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=8fcc3aef0b18edbf input=6f168ffd94be8785]*/ +/*[clinic end generated code: output=8fcc3aef0b18edbf input=0db5f2367599b50d]*/ { assert(PyUnicode_Check(str)); assert(PyUnicode_Check(substr)); @@ -11848,8 +11855,8 @@ str.encode as unicode_encode errors: str(c_default="NULL") = 'strict' The error handling scheme to use for encoding errors. The default is 'strict' meaning that encoding errors raise a - UnicodeEncodeError. Other possible values are 'ignore', 'replace' and - 'xmlcharrefreplace' as well as any other name registered with + UnicodeEncodeError. Other possible values are 'ignore', 'replace' + and 'xmlcharrefreplace' as well as any other name registered with codecs.register_error that can handle UnicodeEncodeErrors. Encode the string using the codec registered for encoding. @@ -11857,7 +11864,7 @@ Encode the string using the codec registered for encoding. static PyObject * unicode_encode_impl(PyObject *self, const char *encoding, const char *errors) -/*[clinic end generated code: output=bf78b6e2a9470e3c input=f0a9eb293d08fe02]*/ +/*[clinic end generated code: output=bf78b6e2a9470e3c input=b85a9645cb33b729]*/ { return PyUnicode_AsEncodedString(self, encoding, errors); } @@ -11953,14 +11960,14 @@ str.find as unicode_find = str.count Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end]. -Optional arguments start and end are interpreted as in slice notation. -Return -1 on failure. +Optional arguments start and end are interpreted as in slice +notation. Return -1 on failure. [clinic start generated code]*/ static Py_ssize_t unicode_find_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=51dbe6255712e278 input=4a89d2d68ef57256]*/ +/*[clinic end generated code: output=51dbe6255712e278 input=6be5f4af6fc545d8]*/ { Py_ssize_t result = any_find_slice(str, substr, start, end, 1); if (result < 0) { @@ -12016,14 +12023,14 @@ str.index as unicode_index = str.count Return the lowest index in S where substring sub is found, such that sub is contained within S[start:end]. -Optional arguments start and end are interpreted as in slice notation. -Raises ValueError when the substring is not found. +Optional arguments start and end are interpreted as in slice +notation. Raises ValueError when the substring is not found. [clinic start generated code]*/ static Py_ssize_t unicode_index_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=77558288837cdf40 input=d986aeac0be14a1c]*/ +/*[clinic end generated code: output=77558288837cdf40 input=39f29462607c38cf]*/ { Py_ssize_t result = any_find_slice(str, substr, start, end, 1); if (result == -1) { @@ -12056,13 +12063,13 @@ str.islower as unicode_islower Return True if the string is a lowercase string, False otherwise. -A string is lowercase if all cased characters in the string are lowercase and -there is at least one cased character in the string. +A string is lowercase if all cased characters in the string are +lowercase and there is at least one cased character in the string. [clinic start generated code]*/ static PyObject * unicode_islower_impl(PyObject *self) -/*[clinic end generated code: output=dbd41995bd005b81 input=acec65ac6821ae47]*/ +/*[clinic end generated code: output=dbd41995bd005b81 input=1879b48dfc628366]*/ { Py_ssize_t i, length; int kind; @@ -12099,13 +12106,13 @@ str.isupper as unicode_isupper Return True if the string is an uppercase string, False otherwise. -A string is uppercase if all cased characters in the string are uppercase and -there is at least one cased character in the string. +A string is uppercase if all cased characters in the string are +uppercase and there is at least one cased character in the string. [clinic start generated code]*/ static PyObject * unicode_isupper_impl(PyObject *self) -/*[clinic end generated code: output=049209c8e7f15f59 input=e9b1feda5d17f2d3]*/ +/*[clinic end generated code: output=049209c8e7f15f59 input=77d29904aef0e3a0]*/ { Py_ssize_t i, length; int kind; @@ -12198,13 +12205,13 @@ str.isspace as unicode_isspace Return True if the string is a whitespace string, False otherwise. -A string is whitespace if all characters in the string are whitespace and there -is at least one character in the string. +A string is whitespace if all characters in the string are +whitespace and there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isspace_impl(PyObject *self) -/*[clinic end generated code: output=163a63bfa08ac2b9 input=fe462cb74f8437d8]*/ +/*[clinic end generated code: output=163a63bfa08ac2b9 input=29e09560fc23fbeb]*/ { Py_ssize_t i, length; int kind; @@ -12236,13 +12243,13 @@ str.isalpha as unicode_isalpha Return True if the string is an alphabetic string, False otherwise. -A string is alphabetic if all characters in the string are alphabetic and there -is at least one character in the string. +A string is alphabetic if all characters in the string are +alphabetic and there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isalpha_impl(PyObject *self) -/*[clinic end generated code: output=cc81b9ac3883ec4f input=d0fd18a96cbca5eb]*/ +/*[clinic end generated code: output=cc81b9ac3883ec4f input=9906a07f3e04892e]*/ { Py_ssize_t i, length; int kind; @@ -12273,13 +12280,13 @@ str.isalnum as unicode_isalnum Return True if the string is an alpha-numeric string, False otherwise. -A string is alpha-numeric if all characters in the string are alpha-numeric and -there is at least one character in the string. +A string is alpha-numeric if all characters in the string are +alpha-numeric and there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isalnum_impl(PyObject *self) -/*[clinic end generated code: output=a5a23490ffc3660c input=5c6579bf2e04758c]*/ +/*[clinic end generated code: output=a5a23490ffc3660c input=fd90c03fd83af0c7]*/ { int kind; const void *data; @@ -12312,13 +12319,13 @@ str.isdecimal as unicode_isdecimal Return True if the string is a decimal string, False otherwise. -A string is a decimal string if all characters in the string are decimal and -there is at least one character in the string. +A string is a decimal string if all characters in the string are +decimal and there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isdecimal_impl(PyObject *self) -/*[clinic end generated code: output=fb2dcdb62d3fc548 input=336bc97ab4c8268f]*/ +/*[clinic end generated code: output=fb2dcdb62d3fc548 input=63b0453c48cad0af]*/ { Py_ssize_t i, length; int kind; @@ -12349,13 +12356,13 @@ str.isdigit as unicode_isdigit Return True if the string is a digit string, False otherwise. -A string is a digit string if all characters in the string are digits and there -is at least one character in the string. +A string is a digit string if all characters in the string are +digits and there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isdigit_impl(PyObject *self) -/*[clinic end generated code: output=10a6985311da6858 input=901116c31deeea4c]*/ +/*[clinic end generated code: output=10a6985311da6858 input=353b03747b062e4b]*/ { Py_ssize_t i, length; int kind; @@ -12387,13 +12394,13 @@ str.isnumeric as unicode_isnumeric Return True if the string is a numeric string, False otherwise. -A string is numeric if all characters in the string are numeric and there is at -least one character in the string. +A string is numeric if all characters in the string are numeric and +there is at least one character in the string. [clinic start generated code]*/ static PyObject * unicode_isnumeric_impl(PyObject *self) -/*[clinic end generated code: output=9172a32d9013051a input=722507db976f826c]*/ +/*[clinic end generated code: output=9172a32d9013051a input=83b2a072ed7aff48]*/ { Py_ssize_t i, length; int kind; @@ -12467,13 +12474,13 @@ str.isidentifier as unicode_isidentifier Return True if the string is a valid Python identifier, False otherwise. -Call keyword.iskeyword(s) to test whether string s is a reserved identifier, -such as "def" or "class". +Call keyword.iskeyword(s) to test whether string s is a reserved +identifier, such as "def" or "class". [clinic start generated code]*/ static PyObject * unicode_isidentifier_impl(PyObject *self) -/*[clinic end generated code: output=fe585a9666572905 input=2d807a104f21c0c5]*/ +/*[clinic end generated code: output=fe585a9666572905 input=82e830f25b2a7945]*/ { return PyBool_FromLong(PyUnicode_IsIdentifier(self)); } @@ -12519,15 +12526,15 @@ str.join as unicode_join Concatenate any number of strings. -The string whose method is called is inserted in between each given string. -The result is returned as a new string. +The string whose method is called is inserted in between each given +string. The result is returned as a new string. Example: '.'.join(['ab', 'pq', 'rs']) -> 'ab.pq.rs' [clinic start generated code]*/ static PyObject * unicode_join(PyObject *self, PyObject *iterable) -/*[clinic end generated code: output=6857e7cecfe7bf98 input=2f70422bfb8fa189]*/ +/*[clinic end generated code: output=6857e7cecfe7bf98 input=fd330a11ee845fb2]*/ { return PyUnicode_Join(self, iterable); } @@ -12547,12 +12554,13 @@ str.ljust as unicode_ljust Return a left-justified string of length width. -Padding is done using the specified fill character (default is a space). +Padding is done using the specified fill character (default is +a space). [clinic start generated code]*/ static PyObject * unicode_ljust_impl(PyObject *self, Py_ssize_t width, Py_UCS4 fillchar) -/*[clinic end generated code: output=1cce0e0e0a0b84b3 input=3ab599e335e60a32]*/ +/*[clinic end generated code: output=1cce0e0e0a0b84b3 input=8a55f06694c20ed6]*/ { if (PyUnicode_GET_LENGTH(self) >= width) return unicode_result_unchanged(self); @@ -12902,13 +12910,14 @@ str.removeprefix as unicode_removeprefix Return a str with the given prefix string removed if present. -If the string starts with the prefix string, return string[len(prefix):]. -Otherwise, return a copy of the original string. +If the string starts with the prefix string, return +string[len(prefix):]. Otherwise, return a copy of the original +string. [clinic start generated code]*/ static PyObject * unicode_removeprefix_impl(PyObject *self, PyObject *prefix) -/*[clinic end generated code: output=f1e5945e9763bcb9 input=27ec40b99a37eb88]*/ +/*[clinic end generated code: output=f1e5945e9763bcb9 input=90d162724944bfa7]*/ { int match = tailmatch(self, prefix, 0, PY_SSIZE_T_MAX, -1); if (match == -1) { @@ -12929,14 +12938,14 @@ str.removesuffix as unicode_removesuffix Return a str with the given suffix string removed if present. -If the string ends with the suffix string and that suffix is not empty, -return string[:-len(suffix)]. Otherwise, return a copy of the original -string. +If the string ends with the suffix string and that suffix is not +empty, return string[:-len(suffix)]. Otherwise, return a copy of +the original string. [clinic start generated code]*/ static PyObject * unicode_removesuffix_impl(PyObject *self, PyObject *suffix) -/*[clinic end generated code: output=d36629e227636822 input=12cc32561e769be4]*/ +/*[clinic end generated code: output=d36629e227636822 input=6efc96152d4bfcd5]*/ { int match = tailmatch(self, suffix, 0, PY_SSIZE_T_MAX, +1); if (match == -1) { @@ -13045,14 +13054,14 @@ str.rfind as unicode_rfind = str.count Return the highest index in S where substring sub is found, such that sub is contained within S[start:end]. -Optional arguments start and end are interpreted as in slice notation. -Return -1 on failure. +Optional arguments start and end are interpreted as in slice +notation. Return -1 on failure. [clinic start generated code]*/ static Py_ssize_t unicode_rfind_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=880b29f01dd014c8 input=898361fb71f59294]*/ +/*[clinic end generated code: output=880b29f01dd014c8 input=130dfb5a94532dd6]*/ { Py_ssize_t result = any_find_slice(str, substr, start, end, -1); if (result < 0) { @@ -13066,14 +13075,14 @@ str.rindex as unicode_rindex = str.count Return the highest index in S where substring sub is found, such that sub is contained within S[start:end]. -Optional arguments start and end are interpreted as in slice notation. -Raises ValueError when the substring is not found. +Optional arguments start and end are interpreted as in slice +notation. Raises ValueError when the substring is not found. [clinic start generated code]*/ static Py_ssize_t unicode_rindex_impl(PyObject *str, PyObject *substr, Py_ssize_t start, Py_ssize_t end) -/*[clinic end generated code: output=5f3aef124c867fe1 input=35943dead6c1ea9d]*/ +/*[clinic end generated code: output=5f3aef124c867fe1 input=97a766ba968a214e]*/ { Py_ssize_t result = any_find_slice(str, substr, start, end, -1); if (result == -1) { @@ -13094,12 +13103,13 @@ str.rjust as unicode_rjust Return a right-justified string of length width. -Padding is done using the specified fill character (default is a space). +Padding is done using the specified fill character (default is +a space). [clinic start generated code]*/ static PyObject * unicode_rjust_impl(PyObject *self, Py_ssize_t width, Py_UCS4 fillchar) -/*[clinic end generated code: output=804a1a57fbe8d5cf input=d05f550b5beb1f72]*/ +/*[clinic end generated code: output=804a1a57fbe8d5cf input=1256a8d659589907]*/ { if (PyUnicode_GET_LENGTH(self) >= width) return unicode_result_unchanged(self); @@ -13122,9 +13132,9 @@ str.split as unicode_split sep: object = None The separator used to split the string. - When set to None (the default value), will split on any whitespace - character (including \n \r \t \f and spaces) and will discard - empty strings from the result. + When set to None (the default value), will split on any + whitespace character (including \n \r \t \f and spaces) and + will discard empty strings from the result. maxsplit: Py_ssize_t = -1 Maximum number of splits. -1 (the default value) means no limit. @@ -13133,15 +13143,15 @@ Return a list of the substrings in the string, using sep as the separator string Splitting starts at the front of the string and works to the end. -Note, str.split() is mainly useful for data that has been intentionally -delimited. With natural text that includes punctuation, consider using -the regular expression module. +Note, str.split() is mainly useful for data that has been +intentionally delimited. With natural text that includes +punctuation, consider using the regular expression module. [clinic start generated code]*/ static PyObject * unicode_split_impl(PyObject *self, PyObject *sep, Py_ssize_t maxsplit) -/*[clinic end generated code: output=3a65b1db356948dc input=a29bcc0c7a5af0eb]*/ +/*[clinic end generated code: output=3a65b1db356948dc input=9dc157701983897d]*/ { if (sep == Py_None) return split(self, NULL, maxsplit); @@ -13265,17 +13275,17 @@ str.partition as unicode_partition Partition the string into three parts using the given separator. -This will search for the separator in the string. If the separator is found, -returns a 3-tuple containing the part before the separator, the separator -itself, and the part after it. +This will search for the separator in the string. If the separator +is found, returns a 3-tuple containing the part before the +separator, the separator itself, and the part after it. -If the separator is not found, returns a 3-tuple containing the original string -and two empty strings. +If the separator is not found, returns a 3-tuple containing +the original string and two empty strings. [clinic start generated code]*/ static PyObject * unicode_partition(PyObject *self, PyObject *sep) -/*[clinic end generated code: output=e4ced7bd253ca3c4 input=f29b8d06c63e50be]*/ +/*[clinic end generated code: output=e4ced7bd253ca3c4 input=e45faa8c26270cb1]*/ { return PyUnicode_Partition(self, sep); } @@ -13285,17 +13295,18 @@ str.rpartition as unicode_rpartition = str.partition Partition the string into three parts using the given separator. -This will search for the separator in the string, starting at the end. If -the separator is found, returns a 3-tuple containing the part before the -separator, the separator itself, and the part after it. +This will search for the separator in the string, starting at the +end. If the separator is found, returns a 3-tuple containing the +part before the separator, the separator itself, and the part after +it. -If the separator is not found, returns a 3-tuple containing two empty strings -and the original string. +If the separator is not found, returns a 3-tuple containing two +empty strings and the original string. [clinic start generated code]*/ static PyObject * unicode_rpartition(PyObject *self, PyObject *sep) -/*[clinic end generated code: output=1aa13cf1156572aa input=c4b7db3ef5cf336a]*/ +/*[clinic end generated code: output=1aa13cf1156572aa input=53a7f8cb19975b7c]*/ { return PyUnicode_RPartition(self, sep); } @@ -13339,13 +13350,13 @@ str.splitlines as unicode_splitlines Return a list of the lines in the string, breaking at line boundaries. -Line breaks are not included in the resulting list unless keepends is given and -true. +Line breaks are not included in the resulting list unless keepends +is given and true. [clinic start generated code]*/ static PyObject * unicode_splitlines_impl(PyObject *self, int keepends) -/*[clinic end generated code: output=f664dcdad153ec40 input=ba6ad05ee85d2b55]*/ +/*[clinic end generated code: output=f664dcdad153ec40 input=bf780246bee5462b]*/ { return PyUnicode_Splitlines(self, keepends); } @@ -13369,6 +13380,45 @@ unicode_swapcase_impl(PyObject *self) return case_operation(self, do_swapcase); } +static int +unicode_maketrans_from_dict(PyObject *x, PyObject *newdict) +{ + PyObject *key, *value; + Py_ssize_t i = 0; + int res; + while (PyDict_Next(x, &i, &key, &value)) { + if (PyUnicode_Check(key)) { + PyObject *newkey; + int kind; + const void *data; + if (PyUnicode_GET_LENGTH(key) != 1) { + PyErr_SetString(PyExc_ValueError, "string keys in translate" + "table must be of length 1"); + return -1; + } + kind = PyUnicode_KIND(key); + data = PyUnicode_DATA(key); + newkey = PyLong_FromLong(PyUnicode_READ(kind, data, 0)); + if (!newkey) + return -1; + res = PyDict_SetItem(newdict, newkey, value); + Py_DECREF(newkey); + if (res < 0) + return -1; + } + else if (PyLong_Check(key)) { + if (PyDict_SetItem(newdict, key, value) < 0) + return -1; + } + else { + PyErr_SetString(PyExc_TypeError, "keys in translate table must" + "be strings or integers"); + return -1; + } + } + return 0; +} + /*[clinic input] @staticmethod @@ -13384,18 +13434,19 @@ str.maketrans as unicode_maketrans Return a translation table usable for str.translate(). -If there is only one argument, it must be a dictionary mapping Unicode -ordinals (integers) or characters to Unicode ordinals, strings or None. -Character keys will be then converted to ordinals. -If there are two arguments, they must be strings of equal length, and -in the resulting dictionary, each character in x will be mapped to the -character at the same position in y. If there is a third argument, it -must be a string, whose characters will be mapped to None in the result. +If there is only one argument, it must be a dictionary mapping +Unicode ordinals (integers) or characters to Unicode ordinals, +strings or None. Character keys will be then converted to ordinals. +If there are two arguments, they must be strings of equal length, +and in the resulting dictionary, each character in x will be mapped +to the character at the same position in y. If there is a third +argument, it must be a string, whose characters will be mapped to +None in the result. [clinic start generated code]*/ static PyObject * unicode_maketrans_impl(PyObject *x, PyObject *y, PyObject *z) -/*[clinic end generated code: output=a925c89452bd5881 input=7bfbf529a293c6c5]*/ +/*[clinic end generated code: output=a925c89452bd5881 input=66bc00a1b4258a6e]*/ { PyObject *new = NULL, *key, *value; Py_ssize_t i = 0; @@ -13454,9 +13505,6 @@ unicode_maketrans_impl(PyObject *x, PyObject *y, PyObject *z) } } } else { - int kind; - const void *data; - /* x must be a dict */ if (!PyDict_CheckExact(x)) { PyErr_SetString(PyExc_TypeError, "if you give only one argument " @@ -13464,34 +13512,12 @@ unicode_maketrans_impl(PyObject *x, PyObject *y, PyObject *z) goto err; } /* copy entries into the new dict, converting string keys to int keys */ - while (PyDict_Next(x, &i, &key, &value)) { - if (PyUnicode_Check(key)) { - /* convert string keys to integer keys */ - PyObject *newkey; - if (PyUnicode_GET_LENGTH(key) != 1) { - PyErr_SetString(PyExc_ValueError, "string keys in translate " - "table must be of length 1"); - goto err; - } - kind = PyUnicode_KIND(key); - data = PyUnicode_DATA(key); - newkey = PyLong_FromLong(PyUnicode_READ(kind, data, 0)); - if (!newkey) - goto err; - res = PyDict_SetItem(new, newkey, value); - Py_DECREF(newkey); - if (res < 0) - goto err; - } else if (PyLong_Check(key)) { - /* just keep integer keys */ - if (PyDict_SetItem(new, key, value) < 0) - goto err; - } else { - PyErr_SetString(PyExc_TypeError, "keys in translate table must " - "be strings or integers"); - goto err; - } - } + int errcode; + Py_BEGIN_CRITICAL_SECTION(x); + errcode = unicode_maketrans_from_dict(x, new); + Py_END_CRITICAL_SECTION(); + if (errcode < 0) + goto err; } return new; err: @@ -13503,20 +13529,21 @@ unicode_maketrans_impl(PyObject *x, PyObject *y, PyObject *z) str.translate as unicode_translate table: object - Translation table, which must be a mapping of Unicode ordinals to - Unicode ordinals, strings, or None. + Translation table, which must be a mapping of Unicode ordinals + to Unicode ordinals, strings, or None. / Replace each character in the string using the given translation table. -The table must implement lookup/indexing via __getitem__, for instance a -dictionary or list. If this operation raises LookupError, the character is -left untouched. Characters mapped to None are deleted. +The table must implement lookup/indexing via __getitem__, for +instance a dictionary or list. If this operation raises +LookupError, the character is left untouched. Characters mapped to +None are deleted. [clinic start generated code]*/ static PyObject * unicode_translate(PyObject *self, PyObject *table) -/*[clinic end generated code: output=3cb448ff2fd96bf3 input=6d38343db63d8eb0]*/ +/*[clinic end generated code: output=3cb448ff2fd96bf3 input=9874c06808f58900]*/ { return _PyUnicode_TranslateCharmap(self, table, "ignore"); } @@ -13944,7 +13971,12 @@ _PyUnicodeWriter_WriteStr(_PyUnicodeWriter *writer, PyObject *str) int PyUnicodeWriter_WriteStr(PyUnicodeWriter *writer, PyObject *obj) { - if (Py_TYPE(obj) == &PyLong_Type) { + PyTypeObject *type = Py_TYPE(obj); + if (type == &PyUnicode_Type) { + return _PyUnicodeWriter_WriteStr((_PyUnicodeWriter*)writer, obj); + } + + if (type == &PyLong_Type) { return _PyLong_FormatWriter((_PyUnicodeWriter*)writer, obj, 10, 0); } @@ -13962,6 +13994,10 @@ PyUnicodeWriter_WriteStr(PyUnicodeWriter *writer, PyObject *obj) int PyUnicodeWriter_WriteRepr(PyUnicodeWriter *writer, PyObject *obj) { + if (obj == NULL) { + return _PyUnicodeWriter_WriteASCIIString((_PyUnicodeWriter*)writer, "<NULL>", 6); + } + if (Py_TYPE(obj) == &PyLong_Type) { return _PyLong_FormatWriter((_PyUnicodeWriter*)writer, obj, 10, 0); } @@ -14040,6 +14076,10 @@ _PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, if (len == -1) len = strlen(ascii); + if (len == 0) { + return 0; + } + assert(ucs1lib_find_max_char((const Py_UCS1*)ascii, (const Py_UCS1*)ascii + len) < 128); if (writer->buffer == NULL && !writer->overallocate) { @@ -14093,6 +14133,20 @@ _PyUnicodeWriter_WriteASCIIString(_PyUnicodeWriter *writer, return 0; } + +int +PyUnicodeWriter_WriteASCII(PyUnicodeWriter *writer, + const char *str, + Py_ssize_t size) +{ + assert(writer != NULL); + _Py_AssertHoldsTstate(); + + _PyUnicodeWriter *priv_writer = (_PyUnicodeWriter*)writer; + return _PyUnicodeWriter_WriteASCIIString(priv_writer, str, size); +} + + int PyUnicodeWriter_WriteUTF8(PyUnicodeWriter *writer, const char *str, @@ -15918,10 +15972,25 @@ immortalize_interned(PyObject *s) _Py_DecRefTotal(_PyThreadState_GET()); } #endif - FT_ATOMIC_STORE_UINT8_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_IMMORTAL); _Py_SetImmortal(s); + // The switch to SSTATE_INTERNED_IMMORTAL must be the last thing done here + // to synchronize with the check in intern_common() that avoids locking if + // the string is already immortal. + FT_ATOMIC_STORE_UINT8(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_IMMORTAL); } +#ifdef Py_GIL_DISABLED +static bool +can_immortalize_safely(PyObject *s) +{ + if (_Py_IsOwnedByCurrentThread(s) || _Py_IsImmortal(s)) { + return true; + } + Py_ssize_t shared = _Py_atomic_load_ssize(&s->ob_ref_shared); + return _Py_REF_IS_MERGED(shared); +} +#endif + static /* non-null */ PyObject* intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, bool immortalize) @@ -15950,11 +16019,16 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, // no, go on break; case SSTATE_INTERNED_MORTAL: +#ifndef Py_GIL_DISABLED // yes but we might need to make it immortal if (immortalize) { immortalize_interned(s); } return s; +#else + // not fully interned yet; fall through to the locking path + break; +#endif default: // all done return s; @@ -15999,6 +16073,41 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, /* Do a setdefault on the per-interpreter cache. */ PyObject *interned = get_interned_dict(interp); assert(interned != NULL); +#ifdef Py_GIL_DISABLED + // Lock-free fast path: check if there's already an interned copy that + // is in its final immortal state. + PyObject *r; + int res = PyDict_GetItemRef(interned, s, &r); + if (res < 0) { + PyErr_Clear(); + return s; + } + if (res > 0) { + unsigned int state = _Py_atomic_load_uint8(&_PyUnicode_STATE(r).interned); + if (state == SSTATE_INTERNED_IMMORTAL) { + Py_DECREF(s); + return r; + } + // Not yet fully interned; fall through to the locking path. + Py_DECREF(r); + } +#endif + +#ifdef Py_GIL_DISABLED + // Immortalization writes to the refcount fields non-atomically. That + // races with Py_INCREF / Py_DECREF on the thread that owns `s`. If we + // don't own it (and its refcount hasn't been merged), intern a copy + // we own instead. + if (!can_immortalize_safely(s)) { + PyObject *copy = _PyUnicode_Copy(s); + if (copy == NULL) { + PyErr_Clear(); + return s; + } + Py_DECREF(s); + s = copy; + } +#endif LOCK_INTERNED(interp); PyObject *t; @@ -16036,7 +16145,7 @@ intern_common(PyInterpreterState *interp, PyObject *s /* stolen */, Py_DECREF(s); Py_DECREF(s); } - FT_ATOMIC_STORE_UINT8_RELAXED(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_MORTAL); + FT_ATOMIC_STORE_UINT8(_PyUnicode_STATE(s).interned, SSTATE_INTERNED_MORTAL); /* INTERNED_MORTAL -> INTERNED_IMMORTAL (if needed) */ diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 66435924b6c6c31..cf592342daf4baa 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -4,6 +4,7 @@ #include "pycore_typevarobject.h" // _PyTypeAlias_Type, _Py_typing_type_repr #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString #include "pycore_unionobject.h" +#include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() typedef struct { @@ -21,9 +22,7 @@ unionobject_dealloc(PyObject *self) unionobject *alias = (unionobject *)self; _PyObject_GC_UNTRACK(self); - if (alias->weakreflist != NULL) { - PyObject_ClearWeakRefs((PyObject *)alias); - } + FT_CLEAR_WEAKREFS(self, alias->weakreflist); Py_XDECREF(alias->args); Py_XDECREF(alias->hashable_args); @@ -62,7 +61,7 @@ union_hash(PyObject *self) } // The unhashable values somehow became hashable again. Still raise // an error. - PyErr_Format(PyExc_TypeError, "union contains %d unhashable elements", n); + PyErr_Format(PyExc_TypeError, "union contains %zd unhashable elements", n); return -1; } return PyObject_Hash(alias->hashable_args); @@ -290,7 +289,7 @@ union_repr(PyObject *self) } for (Py_ssize_t i = 0; i < len; i++) { - if (i > 0 && PyUnicodeWriter_WriteUTF8(writer, " | ", 3) < 0) { + if (i > 0 && PyUnicodeWriter_WriteASCII(writer, " | ", 3) < 0) { goto error; } PyObject *p = PyTuple_GET_ITEM(alias->args, i); @@ -300,12 +299,12 @@ union_repr(PyObject *self) } #if 0 - PyUnicodeWriter_WriteUTF8(writer, "|args=", 6); + PyUnicodeWriter_WriteASCII(writer, "|args=", 6); PyUnicodeWriter_WriteRepr(writer, alias->args); - PyUnicodeWriter_WriteUTF8(writer, "|h=", 3); + PyUnicodeWriter_WriteASCII(writer, "|h=", 3); PyUnicodeWriter_WriteRepr(writer, alias->hashable_args); if (alias->unhashable_args) { - PyUnicodeWriter_WriteUTF8(writer, "|u=", 3); + PyUnicodeWriter_WriteASCII(writer, "|u=", 3); PyUnicodeWriter_WriteRepr(writer, alias->unhashable_args); } #endif @@ -394,8 +393,23 @@ static PyGetSetDef union_properties[] = { {0} }; +static PyObject * +union_nb_or(PyObject *a, PyObject *b) +{ + unionbuilder ub; + if (!unionbuilder_init(&ub, true)) { + return NULL; + } + if (!unionbuilder_add_single(&ub, a) || + !unionbuilder_add_single(&ub, b)) { + unionbuilder_finalize(&ub); + return NULL; + } + return make_union(&ub); +} + static PyNumberMethods union_as_number = { - .nb_or = _Py_union_type_or, // Add __or__ function + .nb_or = union_nb_or, // Add __or__ function }; static const char* const cls_attrs[] = { @@ -475,11 +489,13 @@ _Py_union_from_tuple(PyObject *args) } if (PyTuple_CheckExact(args)) { if (!unionbuilder_add_tuple(&ub, args)) { + unionbuilder_finalize(&ub); return NULL; } } else { if (!unionbuilder_add_single(&ub, args)) { + unionbuilder_finalize(&ub); return NULL; } } @@ -501,7 +517,8 @@ union_mro_entries(PyObject *self, PyObject *args) static PyMethodDef union_methods[] = { {"__mro_entries__", union_mro_entries, METH_O}, - {"__class_getitem__", union_class_getitem, METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + {"__class_getitem__", union_class_getitem, METH_O|METH_CLASS, + PyDoc_STR("Create a union containing the given types")}, {0} }; diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index bd4c4ac9b3475a4..d3d5723fbe32e7f 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -491,7 +491,8 @@ static PyMemberDef weakref_members[] = { static PyMethodDef weakref_methods[] = { {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + PyDoc_STR("Weakrefs are generic over the type of the referenced object.")}, {NULL} /* Sentinel */ }; diff --git a/PC/_wmimodule.cpp b/PC/_wmimodule.cpp index b6efb3e4a207b4c..b9a229b1398ec8d 100644 --- a/PC/_wmimodule.cpp +++ b/PC/_wmimodule.cpp @@ -57,11 +57,11 @@ _query_thread(LPVOID param) IEnumWbemClassObject* enumerator = NULL; HRESULT hr = S_OK; BSTR bstrQuery = NULL; - struct _query_data *data = (struct _query_data*)param; + _query_data data = *(struct _query_data*)param; // gh-125315: Copy the query string first, so that if the main thread gives // up on waiting we aren't left with a dangling pointer (and a likely crash) - bstrQuery = SysAllocString(data->query); + bstrQuery = SysAllocString(data.query); if (!bstrQuery) { hr = HRESULT_FROM_WIN32(ERROR_NOT_ENOUGH_MEMORY); } @@ -71,7 +71,7 @@ _query_thread(LPVOID param) } if (FAILED(hr)) { - CloseHandle(data->writePipe); + CloseHandle(data.writePipe); if (bstrQuery) { SysFreeString(bstrQuery); } @@ -96,7 +96,7 @@ _query_thread(LPVOID param) IID_IWbemLocator, (LPVOID *)&locator ); } - if (SUCCEEDED(hr) && !SetEvent(data->initEvent)) { + if (SUCCEEDED(hr) && !SetEvent(data.initEvent)) { hr = HRESULT_FROM_WIN32(GetLastError()); } if (SUCCEEDED(hr)) { @@ -105,7 +105,7 @@ _query_thread(LPVOID param) NULL, NULL, 0, NULL, 0, 0, &services ); } - if (SUCCEEDED(hr) && !SetEvent(data->connectEvent)) { + if (SUCCEEDED(hr) && !SetEvent(data.connectEvent)) { hr = HRESULT_FROM_WIN32(GetLastError()); } if (SUCCEEDED(hr)) { @@ -143,7 +143,7 @@ _query_thread(LPVOID param) if (FAILED(hr) || got != 1 || !value) { continue; } - if (!startOfEnum && !WriteFile(data->writePipe, (LPVOID)L"\0", 2, &written, NULL)) { + if (!startOfEnum && !WriteFile(data.writePipe, (LPVOID)L"\0", 2, &written, NULL)) { hr = HRESULT_FROM_WIN32(GetLastError()); break; } @@ -171,10 +171,10 @@ _query_thread(LPVOID param) DWORD cbStr1, cbStr2; cbStr1 = (DWORD)(wcslen(propName) * sizeof(propName[0])); cbStr2 = (DWORD)(wcslen(propStr) * sizeof(propStr[0])); - if (!WriteFile(data->writePipe, propName, cbStr1, &written, NULL) || - !WriteFile(data->writePipe, (LPVOID)L"=", 2, &written, NULL) || - !WriteFile(data->writePipe, propStr, cbStr2, &written, NULL) || - !WriteFile(data->writePipe, (LPVOID)L"\0", 2, &written, NULL) + if (!WriteFile(data.writePipe, propName, cbStr1, &written, NULL) || + !WriteFile(data.writePipe, (LPVOID)L"=", 2, &written, NULL) || + !WriteFile(data.writePipe, propStr, cbStr2, &written, NULL) || + !WriteFile(data.writePipe, (LPVOID)L"\0", 2, &written, NULL) ) { hr = HRESULT_FROM_WIN32(GetLastError()); } @@ -200,7 +200,7 @@ _query_thread(LPVOID param) locator->Release(); } CoUninitialize(); - CloseHandle(data->writePipe); + CloseHandle(data.writePipe); return (DWORD)hr; } @@ -230,13 +230,13 @@ _wmi.exec_query Runs a WMI query against the local machine. -This returns a single string with 'name=value' pairs in a flat array separated -by null characters. +This returns a single string with 'name=value' pairs in a flat array +separated by null characters. [clinic start generated code]*/ static PyObject * _wmi_exec_query_impl(PyObject *module, PyObject *query) -/*[clinic end generated code: output=a62303d5bb5e003f input=48d2d0a1e1a7e3c2]*/ +/*[clinic end generated code: output=a62303d5bb5e003f input=a8d5710acdfbf515]*/ /*[clinic end generated code]*/ { diff --git a/PC/clinic/_wmimodule.cpp.h b/PC/clinic/_wmimodule.cpp.h index 38d52d0329dcc0d..6c18990f056b5f7 100644 --- a/PC/clinic/_wmimodule.cpp.h +++ b/PC/clinic/_wmimodule.cpp.h @@ -14,8 +14,8 @@ PyDoc_STRVAR(_wmi_exec_query__doc__, "\n" "Runs a WMI query against the local machine.\n" "\n" -"This returns a single string with \'name=value\' pairs in a flat array separated\n" -"by null characters."); +"This returns a single string with \'name=value\' pairs in a flat array\n" +"separated by null characters."); #define _WMI_EXEC_QUERY_METHODDEF \ {"exec_query", _PyCFunction_CAST(_wmi_exec_query), METH_FASTCALL|METH_KEYWORDS, _wmi_exec_query__doc__}, @@ -72,4 +72,4 @@ _wmi_exec_query(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj exit: return return_value; } -/*[clinic end generated code: output=802bcbcba69e8d0e input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f246d0e568cc2d2c input=a9049054013a1b77]*/ diff --git a/PC/icons/idlex150.png b/PC/icons/idlex150.png index 806cb0c8aa219be..dbdca5b00f88123 100644 Binary files a/PC/icons/idlex150.png and b/PC/icons/idlex150.png differ diff --git a/PC/launcher2.c b/PC/launcher2.c index 832935c5cc6c1cf..4dd18c8eb5462e9 100644 --- a/PC/launcher2.c +++ b/PC/launcher2.c @@ -922,6 +922,20 @@ _readIni(const wchar_t *section, const wchar_t *settingName, wchar_t *buffer, in { wchar_t iniPath[MAXLEN]; int n; + // Check for _PYLAUNCHER_INIDIR override (used for test isolation) + DWORD len = GetEnvironmentVariableW(L"_PYLAUNCHER_INIDIR", iniPath, MAXLEN); + if (len && len < MAXLEN) { + if (join(iniPath, MAXLEN, L"py.ini")) { + debug(L"# Reading from %s for %s/%s\n", iniPath, section, settingName); + n = GetPrivateProfileStringW(section, settingName, NULL, buffer, bufferLength, iniPath); + if (n) { + debug(L"# Found %s in %s\n", settingName, iniPath); + return n; + } + } + // When _PYLAUNCHER_INIDIR is set, skip the default locations + return 0; + } if (SUCCEEDED(SHGetFolderPathW(NULL, CSIDL_LOCAL_APPDATA, NULL, 0, iniPath)) && join(iniPath, MAXLEN, L"py.ini")) { debug(L"# Reading from %s for %s/%s\n", iniPath, section, settingName); diff --git a/PC/layout/main.py b/PC/layout/main.py index 7324a135133b668..8543e7c56e1c419 100644 --- a/PC/layout/main.py +++ b/PC/layout/main.py @@ -247,9 +247,15 @@ def in_build(f, dest="", new_name=None, no_lib=False): if ns.include_freethreaded: yield from in_build("venvlaunchert.exe", "Lib/venv/scripts/nt/") yield from in_build("venvwlaunchert.exe", "Lib/venv/scripts/nt/") - else: + elif (VER_MAJOR, VER_MINOR) > (3, 12): yield from in_build("venvlauncher.exe", "Lib/venv/scripts/nt/") yield from in_build("venvwlauncher.exe", "Lib/venv/scripts/nt/") + else: + # Older versions of venv expected the scripts to be named 'python' + # and they were renamed at this stage. We need to replicate that + # when packaging older versions. + yield from in_build("venvlauncher.exe", "Lib/venv/scripts/nt/", "python") + yield from in_build("venvwlauncher.exe", "Lib/venv/scripts/nt/", "pythonw") if ns.include_tools: @@ -652,15 +658,6 @@ def main(): ns.doc_build = (Path.cwd() / ns.doc_build).resolve() if ns.include_cat and not ns.include_cat.is_absolute(): ns.include_cat = (Path.cwd() / ns.include_cat).resolve() - if not ns.arch: - # TODO: Calculate arch from files in ns.build instead - if sys.winver.endswith("-arm64"): - ns.arch = "arm64" - elif sys.winver.endswith("-32"): - ns.arch = "win32" - else: - ns.arch = "amd64" - if ns.zip and not ns.zip.is_absolute(): ns.zip = (Path.cwd() / ns.zip).resolve() if ns.catalog and not ns.catalog.is_absolute(): @@ -668,6 +665,17 @@ def main(): configure_logger(ns) + if not ns.arch: + from .support.arch import calculate_from_build_dir + ns.arch = calculate_from_build_dir(ns.build) + + expect = f"{VER_MAJOR}.{VER_MINOR}.{VER_MICRO}{VER_SUFFIX}" + actual = check_patchlevel_version(ns.source) + if actual and actual != expect: + log_error(f"Inferred version {expect} does not match {actual} from patchlevel.h. " + "You should set %PYTHONINCLUDE% or %PYTHON_HEXVERSION% before launching.") + return 5 + log_info( """OPTIONS Source: {ns.source} diff --git a/PC/layout/support/arch.py b/PC/layout/support/arch.py new file mode 100644 index 000000000000000..daf4efbc7ab6742 --- /dev/null +++ b/PC/layout/support/arch.py @@ -0,0 +1,34 @@ +from struct import unpack +from .constants import * +from .logging import * + +def calculate_from_build_dir(root): + candidates = [ + root / PYTHON_DLL_NAME, + root / FREETHREADED_PYTHON_DLL_NAME, + *root.glob("*.dll"), + *root.glob("*.pyd"), + # Check EXE last because it's easier to have cross-platform EXE + *root.glob("*.exe"), + ] + + ARCHS = { + b"PE\0\0\x4c\x01": "win32", + b"PE\0\0\x64\x86": "amd64", + b"PE\0\0\x64\xAA": "arm64" + } + + first_exc = None + for pe in candidates: + try: + # Read the PE header to grab the machine type + with open(pe, "rb") as f: + f.seek(0x3C) + offset = int.from_bytes(f.read(4), "little") + f.seek(offset) + arch = ARCHS[f.read(6)] + except (FileNotFoundError, PermissionError, LookupError) as ex: + log_debug("Failed to open {}: {}", pe, ex) + continue + log_info("Inferred architecture {} from {}", arch, pe) + return arch diff --git a/PC/layout/support/constants.py b/PC/layout/support/constants.py index ae22aa16ebfa5da..6b8c915e519743f 100644 --- a/PC/layout/support/constants.py +++ b/PC/layout/support/constants.py @@ -6,6 +6,8 @@ __version__ = "3.8" import os +import pathlib +import re import struct import sys @@ -13,9 +15,15 @@ def _unpack_hexversion(): try: hexversion = int(os.getenv("PYTHON_HEXVERSION"), 16) + return struct.pack(">i", hexversion) except (TypeError, ValueError): - hexversion = sys.hexversion - return struct.pack(">i", hexversion) + pass + if os.getenv("PYTHONINCLUDE"): + try: + return _read_patchlevel_version(pathlib.Path(os.getenv("PYTHONINCLUDE"))) + except OSError: + pass + return struct.pack(">i", sys.hexversion) def _get_suffix(field4): @@ -26,6 +34,39 @@ def _get_suffix(field4): return "" +def _read_patchlevel_version(sources): + if not sources.match("Include"): + sources /= "Include" + values = {} + with open(sources / "patchlevel.h", "r", encoding="utf-8") as f: + for line in f: + m = re.match(r'#\s*define\s+(PY_\S+?)\s+(\S+)', line.strip(), re.I) + if m and m.group(2): + v = m.group(2) + if v.startswith('"'): + v = v[1:-1] + else: + v = values.get(v, v) + if isinstance(v, str): + try: + v = int(v, 16 if v.startswith("0x") else 10) + except ValueError: + pass + values[m.group(1)] = v + return ( + values["PY_MAJOR_VERSION"], + values["PY_MINOR_VERSION"], + values["PY_MICRO_VERSION"], + values["PY_RELEASE_LEVEL"] << 4 | values["PY_RELEASE_SERIAL"], + ) + + +def check_patchlevel_version(sources): + got = _read_patchlevel_version(sources) + if got != (VER_MAJOR, VER_MINOR, VER_MICRO, VER_FIELD4): + return f"{got[0]}.{got[1]}.{got[2]}{_get_suffix(got[3])}" + + VER_MAJOR, VER_MINOR, VER_MICRO, VER_FIELD4 = _unpack_hexversion() VER_SUFFIX = _get_suffix(VER_FIELD4) VER_FIELD3 = VER_MICRO << 8 | VER_FIELD4 diff --git a/PC/layout/support/pymanager.py b/PC/layout/support/pymanager.py index 667c89cdd2cc7a6..831d49ea3f9b46f 100644 --- a/PC/layout/support/pymanager.py +++ b/PC/layout/support/pymanager.py @@ -80,7 +80,9 @@ def calculate_install_json(ns, *, for_embed=False, for_test=False): # Tag used in runtime ID (for side-by-side install/updates) ID_TAG = XY_ARCH_TAG - # Tag shown in 'py list' output + # Tag shown in 'py list' output. + # gh-139810: We only include '-dev' here for prereleases, even though it + # works for final releases too. DISPLAY_TAG = f"{XY_TAG}-dev{TAG_ARCH}" if VER_SUFFIX else XY_ARCH_TAG # Tag used for PEP 514 registration SYS_WINVER = XY_TAG + (TAG_ARCH if TAG_ARCH != '-64' else '') @@ -102,9 +104,10 @@ def calculate_install_json(ns, *, for_embed=False, for_test=False): FULL_ARCH_TAG, XY_ARCH_TAG, X_ARCH_TAG, - # X_TAG and XY_TAG doesn't include VER_SUFFIX, so create -dev versions - f"{XY_TAG}-dev{TAG_ARCH}" if XY_TAG and VER_SUFFIX else "", - f"{X_TAG}-dev{TAG_ARCH}" if X_TAG and VER_SUFFIX else "", + # gh-139810: The -dev tags are always included so that the latest + # release is chosen, no matter whether it's prerelease or final. + f"{XY_TAG}-dev{TAG_ARCH}" if XY_TAG else "", + f"{X_TAG}-dev{TAG_ARCH}" if X_TAG else "", ] # Generate run-for entries for each target. @@ -115,16 +118,15 @@ def calculate_install_json(ns, *, for_embed=False, for_test=False): ]: if not base["target"]: continue - STD_RUN_FOR.append({**base, "tag": FULL_ARCH_TAG}) + STD_RUN_FOR.extend([ + {**base, "tag": FULL_ARCH_TAG}, + {**base, "tag": f"{XY_TAG}-dev{TAG_ARCH}" if XY_TAG else ""}, + {**base, "tag": f"{X_TAG}-dev{TAG_ARCH}" if X_TAG else ""}, + ]) if XY_TAG: STD_RUN_FOR.append({**base, "tag": XY_ARCH_TAG}) if X_TAG: STD_RUN_FOR.append({**base, "tag": X_ARCH_TAG}) - if VER_SUFFIX: - STD_RUN_FOR.extend([ - {**base, "tag": f"{XY_TAG}-dev{TAG_ARCH}" if XY_TAG else ""}, - {**base, "tag": f"{X_TAG}-dev{TAG_ARCH}" if X_TAG else ""}, - ]) # Generate alias entries for each target. We need both arch and non-arch # versions as well as windowed/non-windowed versions to make sure that all diff --git a/PC/pyconfig.h.in b/PC/pyconfig.h similarity index 97% rename from PC/pyconfig.h.in rename to PC/pyconfig.h index 9e70303868e5de7..710a737ebcc0819 100644 --- a/PC/pyconfig.h.in +++ b/PC/pyconfig.h @@ -94,12 +94,17 @@ WIN32 is still required for the locale module. #endif #endif /* Py_BUILD_CORE || Py_BUILD_CORE_BUILTIN || Py_BUILD_CORE_MODULE */ -/* Define to 1 if you want to disable the GIL */ -/* Uncomment the definition for free-threaded builds, or define it manually - * when compiling extension modules. Note that we test with #ifdef, so - * defining as 0 will still disable the GIL. */ -#ifndef Py_GIL_DISABLED -/* #define Py_GIL_DISABLED 1 */ +/* Define to 1 when compiling for experimental free-threaded builds */ +#ifdef Py_GIL_DISABLED +/* We undefine if it was set to zero because all later checks are #ifdef. + * Note that non-Windows builds do not do this, and so every effort should + * be made to avoid defining the variable at all when not desired. However, + * sysconfig.get_config_var always returns a 1 or a 0, and so it seems likely + * that a build backend will define it with the value. + */ +#if Py_GIL_DISABLED == 0 +#undef Py_GIL_DISABLED +#endif #endif /* Compiler specific defines */ diff --git a/PC/winreg.c b/PC/winreg.c index c0de5c1353a349f..159df46bc639d66 100644 --- a/PC/winreg.c +++ b/PC/winreg.c @@ -51,6 +51,7 @@ PyDoc_STRVAR(module_doc, "CreateKey() - Creates the specified key, or opens it if it already exists.\n" "DeleteKey() - Deletes the specified key.\n" "DeleteValue() - Removes a named value from the specified registry key.\n" +"DeleteTree() - Deletes the specified key and all its subkeys and values recursively.\n" "EnumKey() - Enumerates subkeys of the specified open registry key.\n" "EnumValue() - Enumerates values of the specified open registry key.\n" "ExpandEnvironmentStrings() - Expand the env strings in a REG_EXPAND_SZ\n" @@ -102,7 +103,8 @@ PyDoc_STRVAR(PyHKEY_doc, "Operations:\n" "__bool__ - Handles with an open object return true, otherwise false.\n" "__int__ - Converting a handle to an integer returns the Win32 handle.\n" -"rich comparison - Handle objects are compared using the handle value."); +"__enter__, __exit__ - Context manager support for 'with' statement,\n" +"automatically closes handle."); @@ -426,7 +428,9 @@ PyHKEY_Close(winreg_state *st, PyObject *ob_handle) if (PyHKEY_Check(st, ob_handle)) { ((PyHKEYObject*)ob_handle)->hkey = 0; } + Py_BEGIN_ALLOW_THREADS rc = key ? RegCloseKey(key) : ERROR_SUCCESS; + Py_END_ALLOW_THREADS if (rc != ERROR_SUCCESS) PyErr_SetFromWindowsErrWithFunction(rc, "RegCloseKey"); return rc == ERROR_SUCCESS; @@ -499,14 +503,21 @@ PyWinObject_CloseHKEY(winreg_state *st, PyObject *obHandle) } #if SIZEOF_LONG >= SIZEOF_HKEY else if (PyLong_Check(obHandle)) { - long rc = RegCloseKey((HKEY)PyLong_AsLong(obHandle)); + long rc; + Py_BEGIN_ALLOW_THREADS + rc = RegCloseKey((HKEY)PyLong_AsLong(obHandle)); + Py_END_ALLOW_THREADS ok = (rc == ERROR_SUCCESS); if (!ok) PyErr_SetFromWindowsErrWithFunction(rc, "RegCloseKey"); } #else else if (PyLong_Check(obHandle)) { - long rc = RegCloseKey((HKEY)PyLong_AsVoidPtr(obHandle)); + long rc; + HKEY hkey = (HKEY)PyLong_AsVoidPtr(obHandle); + Py_BEGIN_ALLOW_THREADS + rc = RegCloseKey(hkey); + Py_END_ALLOW_THREADS ok = (rc == ERROR_SUCCESS); if (!ok) PyErr_SetFromWindowsErrWithFunction(rc, "RegCloseKey"); @@ -924,7 +935,9 @@ winreg_CreateKey_impl(PyObject *module, HKEY key, const wchar_t *sub_key) (Py_ssize_t)KEY_WRITE) < 0) { return NULL; } + Py_BEGIN_ALLOW_THREADS rc = RegCreateKeyW(key, sub_key, &retKey); + Py_END_ALLOW_THREADS if (rc != ERROR_SUCCESS) { PyErr_SetFromWindowsErrWithFunction(rc, "CreateKey"); return NULL; @@ -973,8 +986,10 @@ winreg_CreateKeyEx_impl(PyObject *module, HKEY key, const wchar_t *sub_key, (Py_ssize_t)access) < 0) { return NULL; } + Py_BEGIN_ALLOW_THREADS rc = RegCreateKeyExW(key, sub_key, reserved, NULL, 0, access, NULL, &retKey, NULL); + Py_END_ALLOW_THREADS if (rc != ERROR_SUCCESS) { PyErr_SetFromWindowsErrWithFunction(rc, "CreateKeyEx"); return NULL; @@ -1187,10 +1202,12 @@ winreg_EnumValue_impl(PyObject *module, HKEY key, int index) (Py_ssize_t)key, index) < 0) { return NULL; } - if ((rc = RegQueryInfoKeyW(key, NULL, NULL, NULL, NULL, NULL, NULL, - NULL, - &retValueSize, &retDataSize, NULL, NULL)) - != ERROR_SUCCESS) + + Py_BEGIN_ALLOW_THREADS + rc = RegQueryInfoKeyW(key, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + &retValueSize, &retDataSize, NULL, NULL); + Py_END_ALLOW_THREADS + if (rc != ERROR_SUCCESS) return PyErr_SetFromWindowsErrWithFunction(rc, "RegQueryInfoKey"); ++retValueSize; /* include null terminators */ @@ -1477,9 +1494,11 @@ winreg_QueryInfoKey_impl(PyObject *module, HKEY key) if (PySys_Audit("winreg.QueryInfoKey", "n", (Py_ssize_t)key) < 0) { return NULL; } - if ((rc = RegQueryInfoKeyW(key, NULL, NULL, 0, &nSubKeys, NULL, NULL, - &nValues, NULL, NULL, NULL, &ft)) - != ERROR_SUCCESS) { + Py_BEGIN_ALLOW_THREADS + rc = RegQueryInfoKeyW(key, NULL, NULL, 0, &nSubKeys, NULL, NULL, + &nValues, NULL, NULL, NULL, &ft); + Py_END_ALLOW_THREADS + if (rc != ERROR_SUCCESS) { return PyErr_SetFromWindowsErrWithFunction(rc, "RegQueryInfoKey"); } li.LowPart = ft.dwLowDateTime; @@ -1587,7 +1606,9 @@ winreg_QueryValue_impl(PyObject *module, HKEY key, const wchar_t *sub_key) PyMem_Free(pbuf); } if (childKey != key) { + Py_BEGIN_ALLOW_THREADS RegCloseKey(childKey); + Py_END_ALLOW_THREADS } return result; } @@ -1625,7 +1646,9 @@ winreg_QueryValueEx_impl(PyObject *module, HKEY key, const wchar_t *name) (Py_ssize_t)key, NULL, name) < 0) { return NULL; } + Py_BEGIN_ALLOW_THREADS rc = RegQueryValueExW(key, name, NULL, NULL, NULL, &bufSize); + Py_END_ALLOW_THREADS if (rc == ERROR_MORE_DATA) bufSize = 256; else if (rc != ERROR_SUCCESS) @@ -1637,8 +1660,10 @@ winreg_QueryValueEx_impl(PyObject *module, HKEY key, const wchar_t *name) while (1) { retSize = bufSize; + Py_BEGIN_ALLOW_THREADS rc = RegQueryValueExW(key, name, NULL, &typ, (BYTE *)retBuf, &retSize); + Py_END_ALLOW_THREADS if (rc != ERROR_MORE_DATA) break; @@ -1656,7 +1681,7 @@ winreg_QueryValueEx_impl(PyObject *module, HKEY key, const wchar_t *name) return PyErr_SetFromWindowsErrWithFunction(rc, "RegQueryValueEx"); } - obData = Reg2Py(retBuf, bufSize, typ); + obData = Reg2Py(retBuf, retSize, typ); PyMem_Free(retBuf); if (obData == NULL) return NULL; diff --git a/PCbuild/_elementtree.vcxproj b/PCbuild/_elementtree.vcxproj index 3eb9c89bcb67405..46e34a5ea82d5d0 100644 --- a/PCbuild/_elementtree.vcxproj +++ b/PCbuild/_elementtree.vcxproj @@ -103,9 +103,11 @@ <ClInclude Include="..\Modules\expat\expat.h" /> <ClInclude Include="..\Modules\expat\expat_config.h" /> <ClInclude Include="..\Modules\expat\expat_external.h" /> + <ClInclude Include="..\Modules\expat\fallthrough.h" /> <ClInclude Include="..\Modules\expat\iasciitab.h" /> <ClInclude Include="..\Modules\expat\internal.h" /> <ClInclude Include="..\Modules\expat\latin1tab.h" /> + <ClInclude Include="..\Modules\expat\memory_sanitizer.h" /> <ClInclude Include="..\Modules\expat\nametab.h" /> <ClInclude Include="..\Modules\expat\pyexpatns.h" /> <ClInclude Include="..\Modules\expat\utf8tab.h" /> diff --git a/PCbuild/_elementtree.vcxproj.filters b/PCbuild/_elementtree.vcxproj.filters index a5368024ccebfd4..f6b55d8ff82a68c 100644 --- a/PCbuild/_elementtree.vcxproj.filters +++ b/PCbuild/_elementtree.vcxproj.filters @@ -33,6 +33,9 @@ <ClInclude Include="..\Modules\expat\expat_external.h"> <Filter>Header Files\expat</Filter> </ClInclude> + <ClInclude Include="..\Modules\expat\fallthrough.h"> + <Filter>Header Files\expat</Filter> + </ClInclude> <ClInclude Include="..\Modules\expat\iasciitab.h"> <Filter>Header Files\expat</Filter> </ClInclude> @@ -42,6 +45,9 @@ <ClInclude Include="..\Modules\expat\latin1tab.h"> <Filter>Header Files\expat</Filter> </ClInclude> + <ClInclude Include="..\Modules\expat\memory_sanitizer.h"> + <Filter>Header Files\expat</Filter> + </ClInclude> <ClInclude Include="..\Modules\expat\nametab.h"> <Filter>Header Files\expat</Filter> </ClInclude> diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 8c161835af9a8c3..5ceddf759b8f3bc 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -106,6 +106,7 @@ </ItemGroup> <ItemGroup> <ClCompile Include="..\Modules\atexitmodule.c" /> + <ClCompile Include="..\Modules\_datetimemodule.c" /> <ClCompile Include="..\Modules\faulthandler.c" /> <ClCompile Include="..\Modules\gcmodule.c" /> <ClCompile Include="..\Modules\getbuildinfo.c" /> @@ -276,7 +277,7 @@ <ClCompile Include="..\Python\uniqueid.c" /> </ItemGroup> <ItemGroup> - <ClInclude Include="..\PC\pyconfig.h.in" /> + <ClInclude Include="..\PC\pyconfig.h" /> </ItemGroup> <ItemGroup> <!-- BEGIN frozen modules --> @@ -436,31 +437,6 @@ <ImportGroup Label="ExtensionTargets"> </ImportGroup> - <!-- Direct copy from pythoncore.vcxproj, but this one is only used for our - own build. All other extension modules will use the copy that pythoncore - generates. --> - <Target Name="_UpdatePyconfig" BeforeTargets="PrepareForBuild"> - <MakeDir Directories="$(IntDir)" Condition="!Exists($(IntDir))" /> - <ItemGroup> - <PyConfigH Remove="@(PyConfigH)" /> - <PyConfigH Include="@(ClInclude)" Condition="'%(Filename)%(Extension)' == 'pyconfig.h.in'" /> - </ItemGroup> - <Error Text="Did not find pyconfig.h" Condition="@(ClInclude) == ''" /> - <PropertyGroup> - <PyConfigH>@(PyConfigH->'%(FullPath)', ';')</PyConfigH> - <PyConfigHText>$([System.IO.File]::ReadAllText($(PyConfigH)))</PyConfigHText> - <OldPyConfigH Condition="Exists('$(IntDir)pyconfig.h')">$([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h'))</OldPyConfigH> - </PropertyGroup> - <PropertyGroup Condition="$(DisableGil) == 'true'"> - <PyConfigHText>$(PyConfigHText.Replace('/* #define Py_GIL_DISABLED 1 */', '#define Py_GIL_DISABLED 1'))</PyConfigHText> - </PropertyGroup> - <Message Text="Updating pyconfig.h" Condition="$(PyConfigHText.TrimEnd()) != $(OldPyConfigH.TrimEnd())" /> - <WriteLinesToFile File="$(IntDir)pyconfig.h" - Lines="$(PyConfigHText)" - Overwrite="true" - Condition="$(PyConfigHText.TrimEnd()) != $(OldPyConfigH.TrimEnd())" /> - </Target> - <Target Name="_RebuildGetPath" AfterTargets="_RebuildFrozen" Condition="$(Configuration) != 'PGUpdate'"> <Exec Command='"$(TargetPath)" "%(GetPath.ModName)" "%(GetPath.FullPath)" "%(GetPath.IntFile)"' /> diff --git a/PCbuild/_testcapi.vcxproj b/PCbuild/_testcapi.vcxproj index a68f15d25aabb76..51c4bcc6b47bff1 100644 --- a/PCbuild/_testcapi.vcxproj +++ b/PCbuild/_testcapi.vcxproj @@ -131,6 +131,7 @@ <ClCompile Include="..\Modules\_testcapi\frame.c" /> <ClCompile Include="..\Modules\_testcapi\type.c" /> <ClCompile Include="..\Modules\_testcapi\function.c" /> + <ClCompile Include="..\Modules\_testcapi\weakref.c" /> </ItemGroup> <ItemGroup> <ResourceCompile Include="..\PC\python_nt.rc" /> diff --git a/PCbuild/_testcapi.vcxproj.filters b/PCbuild/_testcapi.vcxproj.filters index 21091e9dc1aa166..d0b2b3462c6bfb1 100644 --- a/PCbuild/_testcapi.vcxproj.filters +++ b/PCbuild/_testcapi.vcxproj.filters @@ -126,6 +126,9 @@ <ClCompile Include="..\Modules\_testcapi\function.c"> <Filter>Source Files</Filter> </ClCompile> + <ClCompile Include="..\Modules\_testcapi\weakref.c"> + <Filter>Source Files</Filter> + </ClCompile> </ItemGroup> <ItemGroup> <ResourceCompile Include="..\PC\python_nt.rc"> diff --git a/PCbuild/_testclinic_limited.vcxproj b/PCbuild/_testclinic_limited.vcxproj index 183a55080e86938..95c205309b1f308 100644 --- a/PCbuild/_testclinic_limited.vcxproj +++ b/PCbuild/_testclinic_limited.vcxproj @@ -70,6 +70,7 @@ <ProjectGuid>{01FDF29A-40A1-46DF-84F5-85EBBD2A2410}</ProjectGuid> <RootNamespace>_testclinic_limited</RootNamespace> <Keyword>Win32Proj</Keyword> + <SupportPGO>false</SupportPGO> </PropertyGroup> <Import Project="python.props" /> <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> diff --git a/PCbuild/_testlimitedcapi.vcxproj b/PCbuild/_testlimitedcapi.vcxproj index 36c41fc9824fdaf..f9be7b7e60c5498 100644 --- a/PCbuild/_testlimitedcapi.vcxproj +++ b/PCbuild/_testlimitedcapi.vcxproj @@ -97,6 +97,7 @@ <ClCompile Include="..\Modules\_testlimitedcapi\abstract.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\bytearray.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\bytes.c" /> + <ClCompile Include="..\Modules\_testlimitedcapi\capsule.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\codec.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\complex.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\dict.c" /> @@ -115,6 +116,8 @@ <ClCompile Include="..\Modules\_testlimitedcapi\vectorcall_limited.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\version.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\file.c" /> + <ClCompile Include="..\Modules\_testlimitedcapi\weakref.c" /> + <ClCompile Include="..\Modules\_testlimitedcapi\run.c" /> </ItemGroup> <ItemGroup> <ResourceCompile Include="..\PC\python_nt.rc" /> diff --git a/PCbuild/_testlimitedcapi.vcxproj.filters b/PCbuild/_testlimitedcapi.vcxproj.filters index 62ecb2f70ffa2dd..b4509df01a28da2 100644 --- a/PCbuild/_testlimitedcapi.vcxproj.filters +++ b/PCbuild/_testlimitedcapi.vcxproj.filters @@ -12,6 +12,7 @@ <ClCompile Include="..\Modules\_testlimitedcapi\abstract.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\bytearray.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\bytes.c" /> + <ClCompile Include="..\Modules\_testlimitedcapi\capsule.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\codec.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\complex.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\dict.c" /> @@ -31,6 +32,8 @@ <ClCompile Include="..\Modules\_testlimitedcapi\vectorcall_limited.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\version.c" /> <ClCompile Include="..\Modules\_testlimitedcapi\file.c" /> + <ClCompile Include="..\Modules\_testlimitedcapi\weakref.c" /> + <ClCompile Include="..\Modules\_testlimitedcapi\run.c" /> <ClCompile Include="..\Modules\_testlimitedcapi.c" /> </ItemGroup> <ItemGroup> diff --git a/PCbuild/_zstd.vcxproj b/PCbuild/_zstd.vcxproj index b53f93a6af8d113..6f91b8d05cca20b 100644 --- a/PCbuild/_zstd.vcxproj +++ b/PCbuild/_zstd.vcxproj @@ -137,6 +137,7 @@ <ItemGroup> <ClInclude Include="..\Modules\_zstd\_zstdmodule.h" /> <ClInclude Include="..\Modules\_zstd\buffer.h" /> + <ClInclude Include="..\Modules\_zstd\zstddict.h" /> <ClInclude Include="$(zstdDir)lib\common\bitstream.h" /> <ClInclude Include="$(zstdDir)lib\common\error_private.h" /> <ClInclude Include="$(zstdDir)lib\common\fse.h" /> diff --git a/PCbuild/_zstd.vcxproj.filters b/PCbuild/_zstd.vcxproj.filters index d4d36063c85d09c..eec666e5eaf4390 100644 --- a/PCbuild/_zstd.vcxproj.filters +++ b/PCbuild/_zstd.vcxproj.filters @@ -128,6 +128,9 @@ <ClInclude Include="..\Modules\_zstd\buffer.h"> <Filter>Header Files</Filter> </ClInclude> + <ClInclude Include="..\Modules\_zstd\zstddict.h"> + <Filter>Header Files</Filter> + </ClInclude> <ClInclude Include="$(zstdDir)lib\zstd.h"> <Filter>Header Files\zstd</Filter> </ClInclude> diff --git a/PCbuild/build.bat b/PCbuild/build.bat index 2f358991e484ce4..602357048867d68 100644 --- a/PCbuild/build.bat +++ b/PCbuild/build.bat @@ -33,7 +33,7 @@ echo. -k Attempt to kill any running Pythons before building (usually done echo. automatically by the pythoncore project) echo. --pgo Build with Profile-Guided Optimization. This flag echo. overrides -c and -d -echo. --disable-gil Enable experimental support for running without the GIL. +echo. --disable-gil Enable support for running without the GIL. echo. --test-marker Enable the test marker within the build. echo. --regen Regenerate all opcodes, grammar and tokens. echo. --experimental-jit Enable the experimental just-in-time compiler. diff --git a/PCbuild/find_python.bat b/PCbuild/find_python.bat index d65d080ca71a901..4824e89de41d690 100644 --- a/PCbuild/find_python.bat +++ b/PCbuild/find_python.bat @@ -47,7 +47,7 @@ @rem If py.exe finds a recent enough version, use that one @rem It is fine to add new versions to this list when they have released, @rem but we do not use prerelease builds here. -@for %%p in (3.13 3.12 3.11 3.10) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found +@for %%p in (3.14 3.13 3.12 3.11 3.10) do @py -%%p -EV >nul 2>&1 && (set PYTHON=py -%%p) && (set _Py_Python_Source=found %%p with py.exe) && goto :found @if NOT exist "%_Py_EXTERNALS_DIR%" mkdir "%_Py_EXTERNALS_DIR%" @set _Py_NUGET=%NUGET% @@ -55,7 +55,7 @@ @set _Py_HOST_PYTHON=%HOST_PYTHON% @if "%_Py_HOST_PYTHON%"=="" set _Py_HOST_PYTHON=py @if "%_Py_NUGET%"=="" (set _Py_NUGET=%_Py_EXTERNALS_DIR%\nuget.exe) -@if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://aka.ms/nugetclidl) +@if "%_Py_NUGET_URL%"=="" (set _Py_NUGET_URL=https://dist.nuget.org/win-x86-commandline/latest/nuget.exe) @if NOT exist "%_Py_NUGET%" ( @if not "%_Py_Quiet%"=="1" @echo Downloading nuget... @rem NB: Must use single quotes around NUGET here, NOT double! @@ -67,6 +67,9 @@ ) ) +@rem ensure nuget.org source exist +@"%_Py_NUGET%" sources add -Name nuget.org -Source https://api.nuget.org/v3/index.json >nul 2>nul + @if not "%_Py_Quiet%"=="1" @echo Installing Python via nuget... @if not "%_Py_Quiet%"=="1" ( @"%_Py_NUGET%" install pythonx86 -ExcludeVersion -OutputDirectory "%_Py_EXTERNALS_DIR%" diff --git a/PCbuild/get_external.py b/PCbuild/get_external.py index 4ecc8925349c930..a78aa6a23041adf 100755 --- a/PCbuild/get_external.py +++ b/PCbuild/get_external.py @@ -5,8 +5,28 @@ import pathlib import sys import time +import urllib.error +import urllib.request import zipfile -from urllib.request import urlretrieve + + +def retrieve_with_retries(download_location, output_path, reporthook, + max_retries=7): + """Download a file with exponential backoff retry and save to disk.""" + for attempt in range(max_retries + 1): + try: + resp = urllib.request.urlretrieve( + download_location, + output_path, + reporthook=reporthook, + ) + except (urllib.error.URLError, ConnectionError) as ex: + if attempt == max_retries: + msg = f"Download from {download_location} failed." + raise OSError(msg) from ex + time.sleep(2.25**attempt) + else: + return resp def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose): @@ -16,10 +36,10 @@ def fetch_zip(commit_hash, zip_dir, *, org='python', binary=False, verbose): if verbose: reporthook = print zip_dir.mkdir(parents=True, exist_ok=True) - filename, headers = urlretrieve( + filename, _headers = retrieve_with_retries( url, zip_dir / f'{commit_hash}.zip', - reporthook=reporthook, + reporthook ) return filename diff --git a/PCbuild/get_externals.bat b/PCbuild/get_externals.bat index e29054f5734d49f..4ce04d46ce2207b 100644 --- a/PCbuild/get_externals.bat +++ b/PCbuild/get_externals.bat @@ -54,9 +54,9 @@ echo.Fetching external libraries... set libraries= set libraries=%libraries% bzip2-1.0.8 if NOT "%IncludeLibffiSrc%"=="false" set libraries=%libraries% libffi-3.4.4 -if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.0.16 +if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-3.5.7 set libraries=%libraries% mpdecimal-4.0.0 -set libraries=%libraries% sqlite-3.49.1.0 +set libraries=%libraries% sqlite-3.50.4.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.15.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.15.0 set libraries=%libraries% xz-5.2.5 @@ -79,7 +79,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeLibffi%"=="false" set binaries=%binaries% libffi-3.4.4 -if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.0.16.2 +if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-3.5.7 if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.15.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 if NOT "%IncludeLLVM%"=="false" set binaries=%binaries% llvm-19.1.7.0 diff --git a/PCbuild/pyexpat.vcxproj b/PCbuild/pyexpat.vcxproj index dc9161a8b290f96..8499402f0a9040b 100644 --- a/PCbuild/pyexpat.vcxproj +++ b/PCbuild/pyexpat.vcxproj @@ -95,6 +95,8 @@ </ClCompile> </ItemDefinitionGroup> <ItemGroup> + <ClInclude Include="..\Modules\expat\fallthrough.h" /> + <ClInclude Include="..\Modules\expat\memory_sanitizer.h" /> <ClInclude Include="..\Modules\expat\xmlrole.h" /> <ClInclude Include="..\Modules\expat\xmltok.h" /> </ItemGroup> diff --git a/PCbuild/pyexpat.vcxproj.filters b/PCbuild/pyexpat.vcxproj.filters index fd22fc8c477df02..230531033fccd87 100644 --- a/PCbuild/pyexpat.vcxproj.filters +++ b/PCbuild/pyexpat.vcxproj.filters @@ -12,6 +12,12 @@ </Filter> </ItemGroup> <ItemGroup> + <ClInclude Include="..\Modules\expat\fallthrough.h"> + <Filter>Header Files</Filter> + </ClInclude> + <ClInclude Include="..\Modules\expat\memory_sanitizer.h"> + <Filter>Header Files</Filter> + </ClInclude> <ClInclude Include="..\Modules\expat\xmlrole.h"> <Filter>Header Files</Filter> </ClInclude> diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index 4e414dc913b9d5d..53bfe5e3ea95ccf 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -10,10 +10,10 @@ <Py_IntDir Condition="'$(Py_IntDir)' == ''">$(MSBuildThisFileDirectory)obj\</Py_IntDir> <IntDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\$(ProjectName)\</IntDir> <IntDir>$(IntDir.Replace(`\\`, `\`))</IntDir> - <!-- pyconfig.h is updated by pythoncore.vcxproj, so it's always in pythoncore's IntDir --> - <GeneratedPyConfigDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\pythoncore\</GeneratedPyConfigDir> <GeneratedFrozenModulesDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_frozen\</GeneratedFrozenModulesDir> <GeneratedZlibNgDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)$(ArchName)_$(Configuration)\zlib-ng\</GeneratedZlibNgDir> + <GeneratedJitStencilsDir>$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_$(Configuration)</GeneratedJitStencilsDir> + <GeneratedJitStencilsDir Condition="$(Configuration) == 'PGUpdate'">$(Py_IntDir)\$(MajorVersionNumber)$(MinorVersionNumber)_PGInstrument</GeneratedJitStencilsDir> <TargetName Condition="'$(TargetName)' == ''">$(ProjectName)</TargetName> <TargetName>$(TargetName)$(PyDebugExt)</TargetName> <GenerateManifest>false</GenerateManifest> @@ -49,11 +49,12 @@ <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64'">_WIN64;</_PlatformPreprocessorDefinition> <_PlatformPreprocessorDefinition Condition="$(Platform) == 'x64' and $(PlatformToolset) != 'ClangCL'">_M_X64;$(_PlatformPreprocessorDefinition)</_PlatformPreprocessorDefinition> <_Py3NamePreprocessorDefinition>PY3_DLLNAME=L"$(Py3DllName)$(PyDebugExt)";</_Py3NamePreprocessorDefinition> + <_FreeThreadedPreprocessorDefinition Condition="$(DisableGil) == 'true'">Py_GIL_DISABLED=1;</_FreeThreadedPreprocessorDefinition> </PropertyGroup> <ItemDefinitionGroup> <ClCompile> - <AdditionalIncludeDirectories>$(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)Include\internal\mimalloc;$(GeneratedPyConfigDir);$(PySourcePath)PC;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> - <PreprocessorDefinitions>WIN32;$(_Py3NamePreprocessorDefinition);$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PyStatsPreprocessorDefinition)$(_PydPreprocessorDefinition)%(PreprocessorDefinitions)</PreprocessorDefinitions> + <AdditionalIncludeDirectories>$(PySourcePath)Include;$(PySourcePath)Include\internal;$(PySourcePath)Include\internal\mimalloc;$(PySourcePath)PC;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <PreprocessorDefinitions>WIN32;$(_Py3NamePreprocessorDefinition)$(_PlatformPreprocessorDefinition)$(_DebugPreprocessorDefinition)$(_PyStatsPreprocessorDefinition)$(_PydPreprocessorDefinition)$(_FreeThreadedPreprocessorDefinition)%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions Condition="'$(SupportPGO)' and ($(Configuration) == 'PGInstrument' or $(Configuration) == 'PGUpdate')">_Py_USING_PGO=1;%(PreprocessorDefinitions)</PreprocessorDefinitions> <Optimization>MaxSpeed</Optimization> @@ -96,19 +97,16 @@ <TargetMachine Condition="'$(Platform)' == 'x64'">MachineX64</TargetMachine> <TargetMachine Condition="'$(Platform)'=='ARM'">MachineARM</TargetMachine> <TargetMachine Condition="'$(Platform)'=='ARM64'">MachineARM64</TargetMachine> - <ProfileGuidedDatabase Condition="$(SupportPGO)">$(OutDir)$(TargetName).pgd</ProfileGuidedDatabase> - <LinkTimeCodeGeneration Condition="$(Configuration) == 'Release'">UseLinkTimeCodeGeneration</LinkTimeCodeGeneration> - <LinkTimeCodeGeneration Condition="$(SupportPGO) and $(Configuration) == 'PGInstrument'">PGInstrument</LinkTimeCodeGeneration> - <LinkTimeCodeGeneration Condition="$(SupportPGO) and $(Configuration) == 'PGUpdate'">PGUpdate</LinkTimeCodeGeneration> + <LinkTimeCodeGeneration Condition="$(Configuration) != 'Debug'">UseLinkTimeCodeGeneration</LinkTimeCodeGeneration> <AdditionalDependencies>advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;%(AdditionalDependencies)</AdditionalDependencies> <AdditionalOptions Condition="$(Configuration) != 'Debug'">/OPT:REF,NOICF %(AdditionalOptions)</AdditionalOptions> <AdditionalOptions Condition="$(MSVCHasBrokenARM64Clamping) == 'true' and $(Platform) == 'ARM64'">-d2:-pattern-opt-disable:-932189325 %(AdditionalOptions)</AdditionalOptions> + <AdditionalOptions Condition="$(SupportPGO) and $(Configuration) == 'PGInstrument' and $(PlatformToolset) != 'ClangCL'">/GENPROFILE %(AdditionalOptions)</AdditionalOptions> + <AdditionalOptions Condition="$(SupportPGO) and $(Configuration) == 'PGUpdate' and $(PlatformToolset) != 'ClangCL'">/USEPROFILE %(AdditionalOptions)</AdditionalOptions> </Link> <Lib> <LinkTimeCodeGeneration>false</LinkTimeCodeGeneration> - <LinkTimeCodeGeneration Condition="$(Configuration) == 'Release'">true</LinkTimeCodeGeneration> - <LinkTimeCodeGeneration Condition="$(SupportPGO) and $(Configuration) == 'PGInstrument'">true</LinkTimeCodeGeneration> - <LinkTimeCodeGeneration Condition="$(SupportPGO) and $(Configuration) == 'PGUpdate'">true</LinkTimeCodeGeneration> + <LinkTimeCodeGeneration Condition="$(Configuration) != 'Debug'">true</LinkTimeCodeGeneration> </Lib> <ResourceCompile> <AdditionalIncludeDirectories>$(PySourcePath)PC;$(PySourcePath)Include;$(IntDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> diff --git a/PCbuild/python.props b/PCbuild/python.props index ddc7696d2762fec..ce4a7781fbd0762 100644 --- a/PCbuild/python.props +++ b/PCbuild/python.props @@ -11,6 +11,7 @@ We set BasePlatformToolset for ICC's benefit, it's otherwise ignored. --> + <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and '$(VisualStudioVersion)' == '18.0'">v143</BasePlatformToolset> <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and '$(VisualStudioVersion)' == '17.0'">v143</BasePlatformToolset> <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and '$(VisualStudioVersion)' == '16.0'">v142</BasePlatformToolset> <BasePlatformToolset Condition="'$(BasePlatformToolset)' == '' and ('$(MSBuildToolsVersion)' == '15.0' or '$(VisualStudioVersion)' == '15.0')">v141</BasePlatformToolset> @@ -74,15 +75,15 @@ <Import Project="$(ExternalProps)" Condition="$(ExternalProps) != '' and Exists('$(ExternalProps)')" /> <PropertyGroup> - <sqlite3Dir Condition="$(sqlite3Dir) == ''">$(ExternalsDir)sqlite-3.49.1.0\</sqlite3Dir> + <sqlite3Dir Condition="$(sqlite3Dir) == ''">$(ExternalsDir)sqlite-3.50.4.0\</sqlite3Dir> <bz2Dir Condition="$(bz2Dir) == ''">$(ExternalsDir)bzip2-1.0.8\</bz2Dir> <lzmaDir Condition="$(lzmaDir) == ''">$(ExternalsDir)xz-5.2.5\</lzmaDir> <libffiDir Condition="$(libffiDir) == ''">$(ExternalsDir)libffi-3.4.4\</libffiDir> <libffiOutDir Condition="$(libffiOutDir) == ''">$(libffiDir)$(ArchName)\</libffiOutDir> <libffiIncludeDir Condition="$(libffiIncludeDir) == ''">$(libffiOutDir)include</libffiIncludeDir> <mpdecimalDir Condition="$(mpdecimalDir) == ''">$(ExternalsDir)\mpdecimal-4.0.0\</mpdecimalDir> - <opensslDir Condition="$(opensslDir) == ''">$(ExternalsDir)openssl-3.0.16\</opensslDir> - <opensslOutDir Condition="$(opensslOutDir) == ''">$(ExternalsDir)openssl-bin-3.0.16.2\$(ArchName)\</opensslOutDir> + <opensslDir Condition="$(opensslDir) == ''">$(ExternalsDir)openssl-3.5.7\</opensslDir> + <opensslOutDir Condition="$(opensslOutDir) == ''">$(ExternalsDir)openssl-bin-3.5.7\$(ArchName)\</opensslOutDir> <opensslIncludeDir Condition="$(opensslIncludeDir) == ''">$(opensslOutDir)include</opensslIncludeDir> <nasmDir Condition="$(nasmDir) == ''">$(ExternalsDir)\nasm-2.11.06\</nasmDir> <zlibDir Condition="$(zlibDir) == ''">$(ExternalsDir)\zlib-1.3.1\</zlibDir> diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 549d6284972afc3..b911c9385634d77 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -102,6 +102,7 @@ <AdditionalOptions>/Zm200 %(AdditionalOptions)</AdditionalOptions> <AdditionalIncludeDirectories>$(PySourcePath)Modules\_hacl;$(PySourcePath)Modules\_hacl\include;$(PySourcePath)Python;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <AdditionalIncludeDirectories Condition="$(IncludeExternals)">$(zlibNgDir);$(GeneratedZlibNgDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> + <AdditionalIncludeDirectories Condition="'$(UseJIT)' == 'true'">$(GeneratedJitStencilsDir);%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories> <PreprocessorDefinitions>_USRDLL;Py_BUILD_CORE;Py_BUILD_CORE_BUILTIN;Py_ENABLE_SHARED;MS_DLL_ID="$(SysWinVer)";ZLIB_COMPAT;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions Condition="$(IncludeExternals)">_Py_HAVE_ZLIB;%(PreprocessorDefinitions)</PreprocessorDefinitions> <PreprocessorDefinitions Condition="'$(UseJIT)' == 'true'">_Py_JIT;%(PreprocessorDefinitions)</PreprocessorDefinitions> @@ -409,7 +410,7 @@ <ClInclude Include="..\Parser\string_parser.h" /> <ClInclude Include="..\Parser\pegen.h" /> <ClInclude Include="..\PC\errmap.h" /> - <ClInclude Include="..\PC\pyconfig.h.in" /> + <ClInclude Include="..\PC\pyconfig.h" /> <ClInclude Include="..\Python\condvar.h" /> <ClInclude Include="..\Python\stdlib_module_names.h" /> <ClInclude Include="..\Python\thread_nt.h" /> @@ -418,8 +419,12 @@ <ClCompile Include="..\Modules\_abc.c" /> <ClCompile Include="..\Modules\_bisectmodule.c" /> <ClCompile Include="..\Modules\blake2module.c"> - <PreprocessorDefinitions Condition="'$(Platform)' == 'x64'">HACL_CAN_COMPILE_SIMD128;%(PreprocessorDefinitions)</PreprocessorDefinitions> - <PreprocessorDefinitions Condition="'$(Platform)' == 'x64'">HACL_CAN_COMPILE_SIMD256;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <PreprocessorDefinitions Condition="'$(Platform)' == 'x64'"> + _Py_HACL_CAN_COMPILE_VEC128;%(PreprocessorDefinitions) + </PreprocessorDefinitions> + <PreprocessorDefinitions Condition="'$(Platform)' == 'x64'"> + _Py_HACL_CAN_COMPILE_VEC256;%(PreprocessorDefinitions) + </PreprocessorDefinitions> </ClCompile> <ClCompile Include="..\Modules\_codecsmodule.c" /> <ClCompile Include="..\Modules\_collectionsmodule.c" /> @@ -688,34 +693,6 @@ </ImportGroup> <Target Name="_TriggerRegen" BeforeTargets="PrepareForBuild" DependsOnTargets="Regen" /> - <Target Name="_UpdatePyconfig" BeforeTargets="PrepareForBuild"> - <MakeDir Directories="$(IntDir)" Condition="!Exists($(IntDir))" /> - <ItemGroup> - <PyConfigH Remove="@(PyConfigH)" /> - <PyConfigH Include="@(ClInclude)" Condition="'%(Filename)%(Extension)' == 'pyconfig.h.in'" /> - </ItemGroup> - <Error Text="Did not find pyconfig.h" Condition="@(ClInclude) == ''" /> - <PropertyGroup> - <PyConfigH>@(PyConfigH->'%(FullPath)', ';')</PyConfigH> - <PyConfigHText>$([System.IO.File]::ReadAllText($(PyConfigH)))</PyConfigHText> - <OldPyConfigH Condition="Exists('$(IntDir)pyconfig.h')">$([System.IO.File]::ReadAllText('$(IntDir)pyconfig.h'))</OldPyConfigH> - </PropertyGroup> - <PropertyGroup Condition="$(DisableGil) == 'true'"> - <PyConfigHText>$(PyConfigHText.Replace('/* #define Py_GIL_DISABLED 1 */', '#define Py_GIL_DISABLED 1'))</PyConfigHText> - </PropertyGroup> - <Message Text="Updating pyconfig.h" Condition="$(PyConfigHText.TrimEnd()) != $(OldPyConfigH.TrimEnd())" /> - <WriteLinesToFile File="$(IntDir)pyconfig.h" - Lines="$(PyConfigHText)" - Overwrite="true" - Condition="$(PyConfigHText.TrimEnd()) != $(OldPyConfigH.TrimEnd())" /> - </Target> - <Target Name="_CopyPyconfig" Inputs="$(IntDir)pyconfig.h" Outputs="$(OutDir)pyconfig.h" AfterTargets="Build" DependsOnTargets="_UpdatePyconfig"> - <Copy SourceFiles="$(IntDir)pyconfig.h" DestinationFolder="$(OutDir)" /> - </Target> - <Target Name="_CleanPyconfig" AfterTargets="Clean"> - <Delete Files="$(IntDir)pyconfig.h;$(OutDir)pyconfig.h" /> - </Target> - <Target Name="_GetBuildInfo" BeforeTargets="PrepareForBuild"> <PropertyGroup> <GIT Condition="$(GIT) == ''">git</GIT> diff --git a/PCbuild/readme.txt b/PCbuild/readme.txt index 01e19aabdecdc71..13ab06a3cf73a67 100644 --- a/PCbuild/readme.txt +++ b/PCbuild/readme.txt @@ -211,7 +211,7 @@ _lzma Homepage: https://tukaani.org/xz/ _ssl - Python wrapper for version 3.0.15 of the OpenSSL secure sockets + Python wrapper for version 3.5 of the OpenSSL secure sockets library, which is downloaded from our binaries repository at https://github.com/python/cpython-bin-deps. @@ -230,7 +230,7 @@ _ssl again when building. _sqlite3 - Wraps SQLite 3.49.1, which is itself built by sqlite3.vcxproj + Wraps SQLite 3.50.4, which is itself built by sqlite3.vcxproj Homepage: https://www.sqlite.org/ _tkinter diff --git a/PCbuild/regen.targets b/PCbuild/regen.targets index 3ad177378072359..742597f5cb5ebd2 100644 --- a/PCbuild/regen.targets +++ b/PCbuild/regen.targets @@ -29,12 +29,12 @@ <_KeywordSources Include="$(PySourcePath)Grammar\python.gram;$(PySourcePath)Grammar\Tokens" /> <_KeywordOutputs Include="$(PySourcePath)Lib\keyword.py" /> <!-- Taken from _Target._compute_digest in Tools\jit\_targets.py: --> - <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(GeneratedPyConfigDir)pyconfig.h;$(PySourcePath)Tools\jit\**"/> + <_JITSources Include="$(PySourcePath)Python\executor_cases.c.h;$(PySourcePath)PC\pyconfig.h;$(PySourcePath)Tools\jit\**"/> <!-- Need to explicitly enumerate these, since globbing doesn't work for missing outputs: --> - <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils.h"/> - <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils-aarch64-pc-windows-msvc.h" Condition="$(Platform) == 'ARM64'"/> - <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils-i686-pc-windows-msvc.h" Condition="$(Platform) == 'Win32'"/> - <_JITOutputs Include="$(GeneratedPyConfigDir)jit_stencils-x86_64-pc-windows-msvc.h" Condition="$(Platform) == 'x64'"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils.h"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-aarch64-pc-windows-msvc.h" Condition="$(Platform) == 'ARM64'"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-i686-pc-windows-msvc.h" Condition="$(Platform) == 'Win32'"/> + <_JITOutputs Include="$(GeneratedJitStencilsDir)jit_stencils-x86_64-pc-windows-msvc.h" Condition="$(Platform) == 'x64'"/> <_CasesSources Include="$(PySourcePath)Python\bytecodes.c;$(PySourcePath)Python\optimizer_bytecodes.c;"/> <_CasesOutputs Include="$(PySourcePath)Python\generated_cases.c.h;$(PySourcePath)Include\opcode_ids.h;$(PySourcePath)Include\internal\pycore_uop_ids.h;$(PySourcePath)Python\opcode_targets.h;$(PySourcePath)Include\internal\pycore_opcode_metadata.h;$(PySourcePath)Include\internal\pycore_uop_metadata.h;$(PySourcePath)Python\optimizer_cases.c.h;$(PySourcePath)Lib\_opcode_metadata.py"/> <_SbomSources Include="$(PySourcePath)PCbuild\get_externals.bat" /> @@ -116,7 +116,7 @@ <Target Name="_RegenJIT" Condition="'$(UseJIT)' == 'true'" - DependsOnTargets="_UpdatePyconfig;FindPythonForBuild" + DependsOnTargets="FindPythonForBuild" Inputs="@(_JITSources)" Outputs="@(_JITOutputs)"> <PropertyGroup> @@ -125,8 +125,7 @@ <JITArgs Condition="$(Platform) == 'x64'">x86_64-pc-windows-msvc</JITArgs> <JITArgs Condition="$(Configuration) == 'Debug'">$(JITArgs) --debug</JITArgs> </PropertyGroup> - <Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\jit\build.py" $(JITArgs)' - WorkingDirectory="$(GeneratedPyConfigDir)"/> + <Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\jit\build.py" $(JITArgs) --output-dir "$(GeneratedJitStencilsDir)" --pyconfig-dir "$(PySourcePath)PC"'/> </Target> <Target Name="_CleanJIT" AfterTargets="Clean"> <Delete Files="@(_JITOutputs)"/> diff --git a/Parser/Python.asdl b/Parser/Python.asdl index 96f3914b029d4c7..9c7529c479916d0 100644 --- a/Parser/Python.asdl +++ b/Parser/Python.asdl @@ -114,7 +114,7 @@ module Python attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) arguments = (arg* posonlyargs, arg* args, arg? vararg, arg* kwonlyargs, - expr* kw_defaults, arg? kwarg, expr* defaults) + expr?* kw_defaults, arg? kwarg, expr* defaults) arg = (identifier arg, expr? annotation, string? type_comment) attributes (int lineno, int col_offset, int? end_lineno, int? end_col_offset) diff --git a/Parser/action_helpers.c b/Parser/action_helpers.c index 3bcc0870882a29a..bbf303abbb48e4f 100644 --- a/Parser/action_helpers.c +++ b/Parser/action_helpers.c @@ -254,7 +254,11 @@ _set_seq_context(Parser *p, asdl_expr_seq *seq, expr_context_ty ctx) } for (Py_ssize_t i = 0; i < len; i++) { expr_ty e = asdl_seq_GET(seq, i); - asdl_seq_SET(new_seq, i, _PyPegen_set_expr_context(p, e, ctx)); + expr_ty new_e = _PyPegen_set_expr_context(p, e, ctx); + if (!new_e) { + return NULL; + } + asdl_seq_SET(new_seq, i, new_e); } return new_seq; } @@ -268,19 +272,21 @@ _set_name_context(Parser *p, expr_ty e, expr_context_ty ctx) static expr_ty _set_tuple_context(Parser *p, expr_ty e, expr_context_ty ctx) { - return _PyAST_Tuple( - _set_seq_context(p, e->v.Tuple.elts, ctx), - ctx, - EXTRA_EXPR(e, e)); + asdl_expr_seq *seq = _set_seq_context(p, e->v.Tuple.elts, ctx); + if (!seq && PyErr_Occurred()) { + return NULL; + } + return _PyAST_Tuple(seq, ctx, EXTRA_EXPR(e, e)); } static expr_ty _set_list_context(Parser *p, expr_ty e, expr_context_ty ctx) { - return _PyAST_List( - _set_seq_context(p, e->v.List.elts, ctx), - ctx, - EXTRA_EXPR(e, e)); + asdl_expr_seq *seq = _set_seq_context(p, e->v.List.elts, ctx); + if (!seq && PyErr_Occurred()) { + return NULL; + } + return _PyAST_List(seq, ctx, EXTRA_EXPR(e, e)); } static expr_ty @@ -300,8 +306,11 @@ _set_attribute_context(Parser *p, expr_ty e, expr_context_ty ctx) static expr_ty _set_starred_context(Parser *p, expr_ty e, expr_context_ty ctx) { - return _PyAST_Starred(_PyPegen_set_expr_context(p, e->v.Starred.value, ctx), - ctx, EXTRA_EXPR(e, e)); + expr_ty inner = _PyPegen_set_expr_context(p, e->v.Starred.value, ctx); + if (!inner) { + return NULL; + } + return _PyAST_Starred(inner, ctx, EXTRA_EXPR(e, e)); } /* Creates an `expr_ty` equivalent to `expr` but with `ctx` as context */ @@ -435,6 +444,9 @@ _PyPegen_name_default_pair(Parser *p, arg_ty arg, expr_ty value, Token *tc) return NULL; } a->arg = _PyPegen_add_type_comment_to_arg(p, arg, tc); + if (!a->arg) { + return NULL; + } a->value = value; return a; } @@ -965,7 +977,7 @@ _PyPegen_check_fstring_conversion(Parser *p, Token* conv_token, expr_ty conv) if (conv_token->lineno != conv->lineno || conv_token->end_col_offset != conv->col_offset) { return RAISE_SYNTAX_ERROR_KNOWN_RANGE( conv_token, conv, - "%c-string: conversion type must come right after the exclamanation mark", + "%c-string: conversion type must come right after the exclamation mark", TOK_GET_STRING_PREFIX(p->tok) ); } @@ -1136,7 +1148,14 @@ expr_ty _PyPegen_collect_call_seqs(Parser *p, asdl_expr_seq *a, asdl_seq *b, } asdl_expr_seq *starreds = _PyPegen_seq_extract_starred_exprs(p, b); + if (!starreds && PyErr_Occurred()) { + return NULL; + } + asdl_keyword_seq *keywords = _PyPegen_seq_delete_starred_exprs(p, b); + if (!keywords && PyErr_Occurred()) { + return NULL; + } if (starreds) { total_len += asdl_seq_LEN(starreds); @@ -1384,6 +1403,9 @@ expr_ty _PyPegen_template_str(Parser *p, Token *a, asdl_expr_seq *raw_expressions, Token *b) { asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions, b, TSTRING); + if (resized_exprs == NULL) { + return NULL; + } return _PyAST_TemplateStr(resized_exprs, a->lineno, a->col_offset, b->end_lineno, b->end_col_offset, p->arena); @@ -1393,6 +1415,9 @@ expr_ty _PyPegen_joined_str(Parser *p, Token* a, asdl_expr_seq* raw_expressions, Token*b) { asdl_expr_seq *resized_exprs = _get_resized_exprs(p, a, raw_expressions, b, FSTRING); + if (resized_exprs == NULL) { + return NULL; + } return _PyAST_JoinedStr(resized_exprs, a->lineno, a->col_offset, b->end_lineno, b->end_col_offset, p->arena); @@ -1404,7 +1429,15 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) { if (PyBytes_AsStringAndSize(tok->bytes, &bstr, &bsize) == -1) { return NULL; } - PyObject* str = _PyPegen_decode_string(p, 0, bstr, bsize, tok); + + // Check if we're inside a raw f-string for format spec decoding + int is_raw = 0; + if (INSIDE_FSTRING(p->tok)) { + tokenizer_mode *mode = TOK_GET_MODE(p->tok); + is_raw = mode->raw; + } + + PyObject* str = _PyPegen_decode_string(p, is_raw, bstr, bsize, tok); if (str == NULL) { return NULL; } @@ -1534,7 +1567,7 @@ expr_ty _PyPegen_interpolation(Parser *p, expr_ty expression, Token *debug, Resu end_col_offset, arena ); - if (!debug) { + if (!interpolation || !debug) { return interpolation; } @@ -1545,6 +1578,9 @@ expr_ty _PyPegen_interpolation(Parser *p, expr_ty expression, Token *debug, Resu } asdl_expr_seq *values = _Py_asdl_expr_seq_new(2, arena); + if (!values) { + return NULL; + } asdl_seq_SET(values, 0, debug_text); asdl_seq_SET(values, 1, interpolation); return _PyAST_JoinedStr(values, lineno, col_offset, debug_end_line, debug_end_offset, p->arena); @@ -1561,7 +1597,7 @@ expr_ty _PyPegen_formatted_value(Parser *p, expr_ty expression, Token *debug, Re end_col_offset, arena ); - if (!debug) { + if (!formatted_value || !debug) { return formatted_value; } @@ -1591,6 +1627,9 @@ expr_ty _PyPegen_formatted_value(Parser *p, expr_ty expression, Token *debug, Re } asdl_expr_seq *values = _Py_asdl_expr_seq_new(2, arena); + if (!values) { + return NULL; + } asdl_seq_SET(values, 0, debug_text); asdl_seq_SET(values, 1, formatted_value); return _PyAST_JoinedStr(values, lineno, col_offset, debug_end_line, debug_end_offset, p->arena); @@ -1831,16 +1870,22 @@ _build_concatenated_joined_str(Parser *p, asdl_expr_seq *strings, { asdl_expr_seq *values = _build_concatenated_str(p, strings, lineno, col_offset, end_lineno, end_col_offset, arena); + if (!values) { + return NULL; + } return _PyAST_JoinedStr(values, lineno, col_offset, end_lineno, end_col_offset, p->arena); } -static expr_ty -_build_concatenated_template_str(Parser *p, asdl_expr_seq *strings, +expr_ty +_PyPegen_concatenate_tstrings(Parser *p, asdl_expr_seq *strings, int lineno, int col_offset, int end_lineno, int end_col_offset, PyArena *arena) { asdl_expr_seq *values = _build_concatenated_str(p, strings, lineno, col_offset, end_lineno, end_col_offset, arena); + if (!values) { + return NULL; + } return _PyAST_TemplateStr(values, lineno, col_offset, end_lineno, end_col_offset, arena); } @@ -1853,7 +1898,6 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, Py_ssize_t len = asdl_seq_LEN(strings); assert(len > 0); - int t_string_found = 0; int f_string_found = 0; int unicode_string_found = 0; int bytes_found = 0; @@ -1873,7 +1917,8 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, f_string_found = 1; break; case TemplateStr_kind: - t_string_found = 1; + // python.gram handles this; we should never get here + assert(0); break; default: f_string_found = 1; @@ -1882,13 +1927,13 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, } // Cannot mix unicode and bytes - if ((unicode_string_found || f_string_found || t_string_found) && bytes_found) { + if ((unicode_string_found || f_string_found) && bytes_found) { RAISE_SYNTAX_ERROR("cannot mix bytes and nonbytes literals"); return NULL; } // If it's only bytes or only unicode string, do a simple concat - if (!f_string_found && !t_string_found) { + if (!f_string_found) { if (len == 1) { return asdl_seq_GET(strings, 0); } @@ -1902,11 +1947,6 @@ _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *strings, } } - if (t_string_found) { - return _build_concatenated_template_str(p, strings, lineno, - col_offset, end_lineno, end_col_offset, arena); - } - return _build_concatenated_joined_str(p, strings, lineno, col_offset, end_lineno, end_col_offset, arena); } @@ -1936,6 +1976,9 @@ _PyPegen_register_stmts(Parser *p, asdl_stmt_seq* stmts) { return stmts; } stmt_ty last_stmt = asdl_seq_GET(stmts, len - 1); + if (p->last_stmt_location.lineno > last_stmt->lineno) { + return stmts; + } p->last_stmt_location.lineno = last_stmt->lineno; p->last_stmt_location.col_offset = last_stmt->col_offset; p->last_stmt_location.end_lineno = last_stmt->end_lineno; diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 09e014534fbabc3..5ad20d49fa4b310 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -942,7 +942,7 @@ def visitModule(self, mod): } if (p == 0) { PyErr_Format(PyExc_TypeError, - "%.400s got multiple values for argument '%U'", + "%.400s got multiple values for argument %R", Py_TYPE(self)->tp_name, key); res = -1; goto cleanup; @@ -965,7 +965,7 @@ def visitModule(self, mod): else if (contains == 0) { if (PyErr_WarnFormat( PyExc_DeprecationWarning, 1, - "%.400s.__init__ got an unexpected keyword argument '%U'. " + "%.400s.__init__ got an unexpected keyword argument %R. " "Support for arbitrary keyword arguments is deprecated " "and will be removed in Python 3.15.", Py_TYPE(self)->tp_name, key @@ -1009,7 +1009,7 @@ def visitModule(self, mod): else { if (PyErr_WarnFormat( PyExc_DeprecationWarning, 1, - "Field '%U' is missing from %.400s._field_types. " + "Field %R is missing from %.400s._field_types. " "This will become an error in Python 3.15.", name, Py_TYPE(self)->tp_name ) < 0) { @@ -1044,7 +1044,7 @@ def visitModule(self, mod): // simple field (e.g., identifier) if (PyErr_WarnFormat( PyExc_DeprecationWarning, 1, - "%.400s.__init__ missing 1 required positional argument: '%U'. " + "%.400s.__init__ missing 1 required positional argument: %R. " "This will become an error in Python 3.15.", Py_TYPE(self)->tp_name, name ) < 0) { @@ -1207,7 +1207,7 @@ def visitModule(self, mod): if (rc == 0) { PyErr_Format(PyExc_TypeError, "%.400s.__replace__ got an unexpected keyword " - "argument '%U'.", Py_TYPE(self)->tp_name, key); + "argument %R.", Py_TYPE(self)->tp_name, key); Py_DECREF(expecting); return -1; } @@ -1244,6 +1244,32 @@ def visitModule(self, mod): Py_DECREF(unused); } } + + // Discard fields from 'expecting' that default to None + PyObject *field_types = NULL; + if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), + &_Py_ID(_field_types), + &field_types) < 0) + { + Py_DECREF(expecting); + return -1; + } + if (field_types != NULL) { + Py_ssize_t pos = 0; + PyObject *field_name, *field_type; + while (PyDict_Next(field_types, &pos, &field_name, &field_type)) { + if (_PyUnion_Check(field_type)) { + // optional field + if (PySet_Discard(expecting, field_name) < 0) { + Py_DECREF(expecting); + Py_DECREF(field_types); + return -1; + } + } + } + Py_DECREF(field_types); + } + // Now 'expecting' contains the fields or attributes // that would not be filled inside ast_type_replace(). Py_ssize_t m = PySet_GET_SIZE(expecting); @@ -1486,7 +1512,7 @@ def visitModule(self, mod): for (Py_ssize_t i = 0; i < Py_MIN(length, 2); i++) { if (i > 0) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { goto error; } } @@ -1510,7 +1536,7 @@ def visitModule(self, mod): } if (i == 0 && length > 2) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ...", 5) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ...", 5) < 0) { goto error; } } @@ -1614,7 +1640,7 @@ def visitModule(self, mod): } if (i > 0) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { Py_DECREF(name); Py_DECREF(value_repr); goto error; diff --git a/Parser/lexer/buffer.c b/Parser/lexer/buffer.c index 63aa1ea2ad4f604..e122fd0d9878ea2 100644 --- a/Parser/lexer/buffer.c +++ b/Parser/lexer/buffer.c @@ -13,8 +13,8 @@ _PyLexer_remember_fstring_buffers(struct tok_state *tok) for (index = tok->tok_mode_stack_index; index >= 0; --index) { mode = &(tok->tok_mode_stack[index]); - mode->start_offset = mode->start - tok->buf; - mode->multi_line_start_offset = mode->multi_line_start - tok->buf; + mode->start_offset = mode->start == NULL ? -1 : mode->start - tok->buf; + mode->multi_line_start_offset = mode->multi_line_start == NULL ? -1 : mode->multi_line_start - tok->buf; } } @@ -27,8 +27,8 @@ _PyLexer_restore_fstring_buffers(struct tok_state *tok) for (index = tok->tok_mode_stack_index; index >= 0; --index) { mode = &(tok->tok_mode_stack[index]); - mode->start = tok->buf + mode->start_offset; - mode->multi_line_start = tok->buf + mode->multi_line_start_offset; + mode->start = mode->start_offset < 0 ? NULL : tok->buf + mode->start_offset; + mode->multi_line_start = mode->multi_line_start_offset < 0 ? NULL : tok->buf + mode->multi_line_start_offset; } } diff --git a/Parser/lexer/lexer.c b/Parser/lexer/lexer.c index 4d10bccf0a53f2f..7f25afec302c225 100644 --- a/Parser/lexer/lexer.c +++ b/Parser/lexer/lexer.c @@ -121,38 +121,88 @@ set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { } PyObject *res = NULL; - // Check if there is a # character in the expression + // Look for a # character outside of string literals int hash_detected = 0; + int in_string = 0; + char quote_char = 0; + for (Py_ssize_t i = 0; i < tok_mode->last_expr_size - tok_mode->last_expr_end; i++) { - if (tok_mode->last_expr_buffer[i] == '#') { + char ch = tok_mode->last_expr_buffer[i]; + + // Skip escaped characters + if (ch == '\\') { + i++; + continue; + } + + // Handle quotes + if (ch == '"' || ch == '\'') { + // The following if/else block works becase there is an off number + // of quotes in STRING tokens and the lexer only ever reaches this + // function with valid STRING tokens. + // For example: """hello""" + // First quote: in_string = 1 + // Second quote: in_string = 0 + // Third quote: in_string = 1 + if (!in_string) { + in_string = 1; + quote_char = ch; + } + else if (ch == quote_char) { + in_string = 0; + } + continue; + } + + // Check for # outside strings + if (ch == '#' && !in_string) { hash_detected = 1; break; } } - + // If we found a # character in the expression, we need to handle comments if (hash_detected) { - Py_ssize_t input_length = tok_mode->last_expr_size - tok_mode->last_expr_end; - char *result = (char *)PyMem_Malloc((input_length + 1) * sizeof(char)); + // Allocate buffer for processed result + char *result = (char *)PyMem_Malloc((tok_mode->last_expr_size - tok_mode->last_expr_end + 1) * sizeof(char)); if (!result) { return -1; } - Py_ssize_t i = 0; - Py_ssize_t j = 0; + Py_ssize_t i = 0; // Input position + Py_ssize_t j = 0; // Output position + in_string = 0; // Whether we're in a string + quote_char = 0; // Current string quote char - for (i = 0, j = 0; i < input_length; i++) { - if (tok_mode->last_expr_buffer[i] == '#') { - // Skip characters until newline or end of string - while (i < input_length && tok_mode->last_expr_buffer[i] != '\0') { - if (tok_mode->last_expr_buffer[i] == '\n') { - result[j++] = tok_mode->last_expr_buffer[i]; - break; - } + // Process each character + while (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { + char ch = tok_mode->last_expr_buffer[i]; + + // Handle string quotes + if (ch == '"' || ch == '\'') { + // See comment above to understand this part + if (!in_string) { + in_string = 1; + quote_char = ch; + } else if (ch == quote_char) { + in_string = 0; + } + result[j++] = ch; + } + // Skip comments + else if (ch == '#' && !in_string) { + while (i < tok_mode->last_expr_size - tok_mode->last_expr_end && + tok_mode->last_expr_buffer[i] != '\n') { i++; } - } else { - result[j++] = tok_mode->last_expr_buffer[i]; + if (i < tok_mode->last_expr_size - tok_mode->last_expr_end) { + result[j++] = '\n'; + } } + // Copy other chars + else { + result[j++] = ch; + } + i++; } result[j] = '\0'; // Null-terminate the result string @@ -164,11 +214,9 @@ set_ftstring_expr(struct tok_state* tok, struct token *token, char c) { tok_mode->last_expr_size - tok_mode->last_expr_end, NULL ); - } - - if (!res) { + if (!res) { return -1; } token->metadata = res; @@ -491,6 +539,9 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t return MAKE_TOKEN(ERRORTOKEN); } } + else if (c == EOF && PyErr_Occurred()) { + return MAKE_TOKEN(ERRORTOKEN); + } else { break; } @@ -1328,7 +1379,7 @@ tok_get_normal_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct t return MAKE_TOKEN(_PyTokenizer_syntaxerror(tok, "invalid non-printable character U+%04X", c)); } - if( c == '=' && INSIDE_FSTRING_EXPR(current_tok)) { + if( c == '=' && INSIDE_FSTRING_EXPR_AT_TOP(current_tok)) { current_tok->in_debug = 1; } @@ -1421,7 +1472,8 @@ tok_get_fstring_mode(struct tok_state *tok, tokenizer_mode* current_tok, struct return MAKE_TOKEN( _PyTokenizer_syntaxerror( tok, - "f-string: newlines are not allowed in format specifiers for single quoted f-strings" + "%c-string: newlines are not allowed in format specifiers for single quoted %c-strings", + TOK_GET_STRING_PREFIX(tok), TOK_GET_STRING_PREFIX(tok) ) ); } diff --git a/Parser/lexer/state.c b/Parser/lexer/state.c index 2de9004fe084f2f..a1f645029021ec8 100644 --- a/Parser/lexer/state.c +++ b/Parser/lexer/state.c @@ -15,8 +15,11 @@ _PyTokenizer_tok_new(void) struct tok_state *tok = (struct tok_state *)PyMem_Calloc( 1, sizeof(struct tok_state)); - if (tok == NULL) + if (tok == NULL) { + PyErr_NoMemory(); return NULL; + } + tok->buf = tok->cur = tok->inp = NULL; tok->fp_interactive = 0; tok->interactive_src_start = NULL; diff --git a/Parser/lexer/state.h b/Parser/lexer/state.h index 5e8cac7249b21c5..877127125a76524 100644 --- a/Parser/lexer/state.h +++ b/Parser/lexer/state.h @@ -9,6 +9,8 @@ #define INSIDE_FSTRING(tok) (tok->tok_mode_stack_index > 0) #define INSIDE_FSTRING_EXPR(tok) (tok->curly_bracket_expr_start_depth >= 0) +#define INSIDE_FSTRING_EXPR_AT_TOP(tok) \ + (tok->curly_bracket_depth - tok->curly_bracket_expr_start_depth == 1) enum decoding_state { STATE_INIT, diff --git a/Parser/myreadline.c b/Parser/myreadline.c index 64e8f5383f06022..ee77479ba7bdccb 100644 --- a/Parser/myreadline.c +++ b/Parser/myreadline.c @@ -344,7 +344,7 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt) break; } n += strlen(p + n); - } while (p[n-1] != '\n'); + } while (n == 0 || p[n-1] != '\n'); pr = (char *)PyMem_RawRealloc(p, n+1); if (pr == NULL) { diff --git a/Parser/parser.c b/Parser/parser.c index 509fac7df6e3717..3f06abc2a4854a2 100644 --- a/Parser/parser.c +++ b/Parser/parser.c @@ -14,7 +14,7 @@ # define MAXSTACK 4000 # endif #else -# define MAXSTACK 4000 +# define MAXSTACK 6000 #endif static const int n_keyword_lists = 9; static KeywordToken *reserved_keywords[] = { @@ -348,185 +348,187 @@ static char *soft_keywords[] = { #define invalid_fstring_conversion_character_type 1261 #define invalid_tstring_replacement_field_type 1262 #define invalid_tstring_conversion_character_type 1263 -#define invalid_arithmetic_type 1264 -#define invalid_factor_type 1265 -#define invalid_type_params_type 1266 -#define _loop0_1_type 1267 -#define _loop1_2_type 1268 -#define _loop0_3_type 1269 -#define _gather_4_type 1270 -#define _tmp_5_type 1271 -#define _tmp_6_type 1272 -#define _tmp_7_type 1273 -#define _tmp_8_type 1274 -#define _tmp_9_type 1275 -#define _tmp_10_type 1276 -#define _tmp_11_type 1277 -#define _loop1_12_type 1278 -#define _tmp_13_type 1279 -#define _loop0_14_type 1280 -#define _gather_15_type 1281 -#define _tmp_16_type 1282 -#define _tmp_17_type 1283 -#define _loop0_18_type 1284 -#define _loop1_19_type 1285 -#define _loop0_20_type 1286 -#define _gather_21_type 1287 -#define _tmp_22_type 1288 -#define _loop0_23_type 1289 -#define _gather_24_type 1290 -#define _loop1_25_type 1291 -#define _tmp_26_type 1292 -#define _tmp_27_type 1293 -#define _loop0_28_type 1294 -#define _loop0_29_type 1295 -#define _loop1_30_type 1296 -#define _loop1_31_type 1297 -#define _loop0_32_type 1298 -#define _loop1_33_type 1299 -#define _loop0_34_type 1300 -#define _gather_35_type 1301 -#define _tmp_36_type 1302 -#define _loop1_37_type 1303 -#define _loop1_38_type 1304 -#define _loop1_39_type 1305 -#define _loop0_40_type 1306 -#define _gather_41_type 1307 -#define _tmp_42_type 1308 -#define _tmp_43_type 1309 -#define _tmp_44_type 1310 -#define _loop0_45_type 1311 -#define _gather_46_type 1312 -#define _loop0_47_type 1313 -#define _gather_48_type 1314 -#define _tmp_49_type 1315 -#define _loop0_50_type 1316 -#define _gather_51_type 1317 -#define _loop0_52_type 1318 -#define _gather_53_type 1319 -#define _loop0_54_type 1320 -#define _gather_55_type 1321 -#define _loop1_56_type 1322 -#define _loop1_57_type 1323 -#define _loop0_58_type 1324 -#define _gather_59_type 1325 -#define _loop1_60_type 1326 -#define _loop1_61_type 1327 -#define _loop1_62_type 1328 -#define _tmp_63_type 1329 -#define _loop0_64_type 1330 -#define _gather_65_type 1331 -#define _tmp_66_type 1332 -#define _tmp_67_type 1333 -#define _tmp_68_type 1334 -#define _tmp_69_type 1335 -#define _tmp_70_type 1336 -#define _loop0_71_type 1337 -#define _loop0_72_type 1338 -#define _loop1_73_type 1339 -#define _loop1_74_type 1340 -#define _loop0_75_type 1341 -#define _loop1_76_type 1342 -#define _loop0_77_type 1343 -#define _loop0_78_type 1344 -#define _loop0_79_type 1345 -#define _loop0_80_type 1346 -#define _loop1_81_type 1347 -#define _tmp_82_type 1348 -#define _loop0_83_type 1349 -#define _gather_84_type 1350 -#define _loop1_85_type 1351 -#define _loop0_86_type 1352 -#define _tmp_87_type 1353 -#define _loop0_88_type 1354 -#define _gather_89_type 1355 -#define _tmp_90_type 1356 -#define _loop0_91_type 1357 -#define _gather_92_type 1358 -#define _loop0_93_type 1359 -#define _gather_94_type 1360 -#define _loop0_95_type 1361 -#define _loop0_96_type 1362 -#define _gather_97_type 1363 -#define _loop1_98_type 1364 -#define _tmp_99_type 1365 -#define _loop0_100_type 1366 -#define _gather_101_type 1367 -#define _loop0_102_type 1368 -#define _gather_103_type 1369 -#define _tmp_104_type 1370 -#define _tmp_105_type 1371 -#define _loop0_106_type 1372 -#define _gather_107_type 1373 -#define _tmp_108_type 1374 -#define _tmp_109_type 1375 -#define _tmp_110_type 1376 -#define _tmp_111_type 1377 -#define _tmp_112_type 1378 -#define _loop1_113_type 1379 -#define _tmp_114_type 1380 -#define _tmp_115_type 1381 -#define _tmp_116_type 1382 -#define _tmp_117_type 1383 -#define _tmp_118_type 1384 -#define _loop0_119_type 1385 -#define _loop0_120_type 1386 -#define _tmp_121_type 1387 -#define _tmp_122_type 1388 -#define _tmp_123_type 1389 -#define _tmp_124_type 1390 -#define _tmp_125_type 1391 -#define _tmp_126_type 1392 -#define _tmp_127_type 1393 -#define _tmp_128_type 1394 -#define _tmp_129_type 1395 -#define _loop0_130_type 1396 -#define _gather_131_type 1397 -#define _tmp_132_type 1398 -#define _tmp_133_type 1399 -#define _tmp_134_type 1400 -#define _tmp_135_type 1401 -#define _loop0_136_type 1402 -#define _gather_137_type 1403 -#define _tmp_138_type 1404 -#define _loop0_139_type 1405 -#define _gather_140_type 1406 -#define _loop0_141_type 1407 -#define _gather_142_type 1408 -#define _tmp_143_type 1409 -#define _loop0_144_type 1410 -#define _tmp_145_type 1411 -#define _tmp_146_type 1412 -#define _tmp_147_type 1413 -#define _tmp_148_type 1414 -#define _tmp_149_type 1415 -#define _tmp_150_type 1416 -#define _tmp_151_type 1417 -#define _tmp_152_type 1418 -#define _tmp_153_type 1419 -#define _tmp_154_type 1420 -#define _tmp_155_type 1421 -#define _tmp_156_type 1422 -#define _tmp_157_type 1423 -#define _tmp_158_type 1424 -#define _tmp_159_type 1425 -#define _tmp_160_type 1426 -#define _tmp_161_type 1427 -#define _tmp_162_type 1428 -#define _tmp_163_type 1429 -#define _tmp_164_type 1430 -#define _tmp_165_type 1431 -#define _tmp_166_type 1432 -#define _tmp_167_type 1433 -#define _tmp_168_type 1434 -#define _tmp_169_type 1435 -#define _tmp_170_type 1436 -#define _loop0_171_type 1437 -#define _tmp_172_type 1438 -#define _tmp_173_type 1439 -#define _tmp_174_type 1440 -#define _tmp_175_type 1441 -#define _tmp_176_type 1442 +#define invalid_string_tstring_concat_type 1264 +#define invalid_arithmetic_type 1265 +#define invalid_factor_type 1266 +#define invalid_type_params_type 1267 +#define _loop0_1_type 1268 +#define _loop1_2_type 1269 +#define _loop0_3_type 1270 +#define _gather_4_type 1271 +#define _tmp_5_type 1272 +#define _tmp_6_type 1273 +#define _tmp_7_type 1274 +#define _tmp_8_type 1275 +#define _tmp_9_type 1276 +#define _tmp_10_type 1277 +#define _tmp_11_type 1278 +#define _loop1_12_type 1279 +#define _tmp_13_type 1280 +#define _loop0_14_type 1281 +#define _gather_15_type 1282 +#define _tmp_16_type 1283 +#define _tmp_17_type 1284 +#define _loop0_18_type 1285 +#define _loop1_19_type 1286 +#define _loop0_20_type 1287 +#define _gather_21_type 1288 +#define _tmp_22_type 1289 +#define _loop0_23_type 1290 +#define _gather_24_type 1291 +#define _loop1_25_type 1292 +#define _tmp_26_type 1293 +#define _tmp_27_type 1294 +#define _loop0_28_type 1295 +#define _loop0_29_type 1296 +#define _loop1_30_type 1297 +#define _loop1_31_type 1298 +#define _loop0_32_type 1299 +#define _loop1_33_type 1300 +#define _loop0_34_type 1301 +#define _gather_35_type 1302 +#define _tmp_36_type 1303 +#define _loop1_37_type 1304 +#define _loop1_38_type 1305 +#define _loop1_39_type 1306 +#define _loop0_40_type 1307 +#define _gather_41_type 1308 +#define _tmp_42_type 1309 +#define _tmp_43_type 1310 +#define _tmp_44_type 1311 +#define _loop0_45_type 1312 +#define _gather_46_type 1313 +#define _loop0_47_type 1314 +#define _gather_48_type 1315 +#define _tmp_49_type 1316 +#define _loop0_50_type 1317 +#define _gather_51_type 1318 +#define _loop0_52_type 1319 +#define _gather_53_type 1320 +#define _loop0_54_type 1321 +#define _gather_55_type 1322 +#define _loop1_56_type 1323 +#define _loop1_57_type 1324 +#define _loop0_58_type 1325 +#define _gather_59_type 1326 +#define _loop1_60_type 1327 +#define _loop1_61_type 1328 +#define _loop1_62_type 1329 +#define _tmp_63_type 1330 +#define _loop0_64_type 1331 +#define _gather_65_type 1332 +#define _tmp_66_type 1333 +#define _tmp_67_type 1334 +#define _tmp_68_type 1335 +#define _tmp_69_type 1336 +#define _tmp_70_type 1337 +#define _loop0_71_type 1338 +#define _loop0_72_type 1339 +#define _loop1_73_type 1340 +#define _loop1_74_type 1341 +#define _loop0_75_type 1342 +#define _loop1_76_type 1343 +#define _loop0_77_type 1344 +#define _loop0_78_type 1345 +#define _loop0_79_type 1346 +#define _loop0_80_type 1347 +#define _loop1_81_type 1348 +#define _loop1_82_type 1349 +#define _tmp_83_type 1350 +#define _loop0_84_type 1351 +#define _gather_85_type 1352 +#define _loop1_86_type 1353 +#define _loop0_87_type 1354 +#define _tmp_88_type 1355 +#define _loop0_89_type 1356 +#define _gather_90_type 1357 +#define _tmp_91_type 1358 +#define _loop0_92_type 1359 +#define _gather_93_type 1360 +#define _loop0_94_type 1361 +#define _gather_95_type 1362 +#define _loop0_96_type 1363 +#define _loop0_97_type 1364 +#define _gather_98_type 1365 +#define _loop1_99_type 1366 +#define _tmp_100_type 1367 +#define _loop0_101_type 1368 +#define _gather_102_type 1369 +#define _loop0_103_type 1370 +#define _gather_104_type 1371 +#define _tmp_105_type 1372 +#define _tmp_106_type 1373 +#define _loop0_107_type 1374 +#define _gather_108_type 1375 +#define _tmp_109_type 1376 +#define _tmp_110_type 1377 +#define _tmp_111_type 1378 +#define _tmp_112_type 1379 +#define _tmp_113_type 1380 +#define _loop1_114_type 1381 +#define _tmp_115_type 1382 +#define _tmp_116_type 1383 +#define _tmp_117_type 1384 +#define _tmp_118_type 1385 +#define _tmp_119_type 1386 +#define _loop0_120_type 1387 +#define _loop0_121_type 1388 +#define _tmp_122_type 1389 +#define _tmp_123_type 1390 +#define _tmp_124_type 1391 +#define _tmp_125_type 1392 +#define _tmp_126_type 1393 +#define _tmp_127_type 1394 +#define _tmp_128_type 1395 +#define _tmp_129_type 1396 +#define _tmp_130_type 1397 +#define _loop0_131_type 1398 +#define _gather_132_type 1399 +#define _tmp_133_type 1400 +#define _tmp_134_type 1401 +#define _tmp_135_type 1402 +#define _tmp_136_type 1403 +#define _loop0_137_type 1404 +#define _gather_138_type 1405 +#define _tmp_139_type 1406 +#define _loop0_140_type 1407 +#define _gather_141_type 1408 +#define _loop0_142_type 1409 +#define _gather_143_type 1410 +#define _tmp_144_type 1411 +#define _loop0_145_type 1412 +#define _tmp_146_type 1413 +#define _tmp_147_type 1414 +#define _tmp_148_type 1415 +#define _tmp_149_type 1416 +#define _tmp_150_type 1417 +#define _tmp_151_type 1418 +#define _tmp_152_type 1419 +#define _tmp_153_type 1420 +#define _tmp_154_type 1421 +#define _tmp_155_type 1422 +#define _tmp_156_type 1423 +#define _tmp_157_type 1424 +#define _tmp_158_type 1425 +#define _tmp_159_type 1426 +#define _tmp_160_type 1427 +#define _tmp_161_type 1428 +#define _tmp_162_type 1429 +#define _tmp_163_type 1430 +#define _tmp_164_type 1431 +#define _tmp_165_type 1432 +#define _tmp_166_type 1433 +#define _tmp_167_type 1434 +#define _tmp_168_type 1435 +#define _tmp_169_type 1436 +#define _tmp_170_type 1437 +#define _tmp_171_type 1438 +#define _loop0_172_type 1439 +#define _tmp_173_type 1440 +#define _tmp_174_type 1441 +#define _tmp_175_type 1442 +#define _tmp_176_type 1443 +#define _tmp_177_type 1444 static mod_ty file_rule(Parser *p); static mod_ty interactive_rule(Parser *p); @@ -792,6 +794,7 @@ static void *invalid_fstring_replacement_field_rule(Parser *p); static void *invalid_fstring_conversion_character_rule(Parser *p); static void *invalid_tstring_replacement_field_rule(Parser *p); static void *invalid_tstring_conversion_character_rule(Parser *p); +static void *invalid_string_tstring_concat_rule(Parser *p); static void *invalid_arithmetic_rule(Parser *p); static void *invalid_factor_rule(Parser *p); static void *invalid_type_params_rule(Parser *p); @@ -876,46 +879,46 @@ static asdl_seq *_loop0_78_rule(Parser *p); static asdl_seq *_loop0_79_rule(Parser *p); static asdl_seq *_loop0_80_rule(Parser *p); static asdl_seq *_loop1_81_rule(Parser *p); -static void *_tmp_82_rule(Parser *p); -static asdl_seq *_loop0_83_rule(Parser *p); -static asdl_seq *_gather_84_rule(Parser *p); -static asdl_seq *_loop1_85_rule(Parser *p); -static asdl_seq *_loop0_86_rule(Parser *p); -static void *_tmp_87_rule(Parser *p); -static asdl_seq *_loop0_88_rule(Parser *p); -static asdl_seq *_gather_89_rule(Parser *p); -static void *_tmp_90_rule(Parser *p); -static asdl_seq *_loop0_91_rule(Parser *p); -static asdl_seq *_gather_92_rule(Parser *p); -static asdl_seq *_loop0_93_rule(Parser *p); -static asdl_seq *_gather_94_rule(Parser *p); -static asdl_seq *_loop0_95_rule(Parser *p); +static asdl_seq *_loop1_82_rule(Parser *p); +static void *_tmp_83_rule(Parser *p); +static asdl_seq *_loop0_84_rule(Parser *p); +static asdl_seq *_gather_85_rule(Parser *p); +static asdl_seq *_loop1_86_rule(Parser *p); +static asdl_seq *_loop0_87_rule(Parser *p); +static void *_tmp_88_rule(Parser *p); +static asdl_seq *_loop0_89_rule(Parser *p); +static asdl_seq *_gather_90_rule(Parser *p); +static void *_tmp_91_rule(Parser *p); +static asdl_seq *_loop0_92_rule(Parser *p); +static asdl_seq *_gather_93_rule(Parser *p); +static asdl_seq *_loop0_94_rule(Parser *p); +static asdl_seq *_gather_95_rule(Parser *p); static asdl_seq *_loop0_96_rule(Parser *p); -static asdl_seq *_gather_97_rule(Parser *p); -static asdl_seq *_loop1_98_rule(Parser *p); -static void *_tmp_99_rule(Parser *p); -static asdl_seq *_loop0_100_rule(Parser *p); -static asdl_seq *_gather_101_rule(Parser *p); -static asdl_seq *_loop0_102_rule(Parser *p); -static asdl_seq *_gather_103_rule(Parser *p); -static void *_tmp_104_rule(Parser *p); +static asdl_seq *_loop0_97_rule(Parser *p); +static asdl_seq *_gather_98_rule(Parser *p); +static asdl_seq *_loop1_99_rule(Parser *p); +static void *_tmp_100_rule(Parser *p); +static asdl_seq *_loop0_101_rule(Parser *p); +static asdl_seq *_gather_102_rule(Parser *p); +static asdl_seq *_loop0_103_rule(Parser *p); +static asdl_seq *_gather_104_rule(Parser *p); static void *_tmp_105_rule(Parser *p); -static asdl_seq *_loop0_106_rule(Parser *p); -static asdl_seq *_gather_107_rule(Parser *p); -static void *_tmp_108_rule(Parser *p); +static void *_tmp_106_rule(Parser *p); +static asdl_seq *_loop0_107_rule(Parser *p); +static asdl_seq *_gather_108_rule(Parser *p); static void *_tmp_109_rule(Parser *p); static void *_tmp_110_rule(Parser *p); static void *_tmp_111_rule(Parser *p); static void *_tmp_112_rule(Parser *p); -static asdl_seq *_loop1_113_rule(Parser *p); -static void *_tmp_114_rule(Parser *p); +static void *_tmp_113_rule(Parser *p); +static asdl_seq *_loop1_114_rule(Parser *p); static void *_tmp_115_rule(Parser *p); static void *_tmp_116_rule(Parser *p); static void *_tmp_117_rule(Parser *p); static void *_tmp_118_rule(Parser *p); -static asdl_seq *_loop0_119_rule(Parser *p); +static void *_tmp_119_rule(Parser *p); static asdl_seq *_loop0_120_rule(Parser *p); -static void *_tmp_121_rule(Parser *p); +static asdl_seq *_loop0_121_rule(Parser *p); static void *_tmp_122_rule(Parser *p); static void *_tmp_123_rule(Parser *p); static void *_tmp_124_rule(Parser *p); @@ -924,22 +927,22 @@ static void *_tmp_126_rule(Parser *p); static void *_tmp_127_rule(Parser *p); static void *_tmp_128_rule(Parser *p); static void *_tmp_129_rule(Parser *p); -static asdl_seq *_loop0_130_rule(Parser *p); -static asdl_seq *_gather_131_rule(Parser *p); -static void *_tmp_132_rule(Parser *p); +static void *_tmp_130_rule(Parser *p); +static asdl_seq *_loop0_131_rule(Parser *p); +static asdl_seq *_gather_132_rule(Parser *p); static void *_tmp_133_rule(Parser *p); static void *_tmp_134_rule(Parser *p); static void *_tmp_135_rule(Parser *p); -static asdl_seq *_loop0_136_rule(Parser *p); -static asdl_seq *_gather_137_rule(Parser *p); -static void *_tmp_138_rule(Parser *p); -static asdl_seq *_loop0_139_rule(Parser *p); -static asdl_seq *_gather_140_rule(Parser *p); -static asdl_seq *_loop0_141_rule(Parser *p); -static asdl_seq *_gather_142_rule(Parser *p); -static void *_tmp_143_rule(Parser *p); -static asdl_seq *_loop0_144_rule(Parser *p); -static void *_tmp_145_rule(Parser *p); +static void *_tmp_136_rule(Parser *p); +static asdl_seq *_loop0_137_rule(Parser *p); +static asdl_seq *_gather_138_rule(Parser *p); +static void *_tmp_139_rule(Parser *p); +static asdl_seq *_loop0_140_rule(Parser *p); +static asdl_seq *_gather_141_rule(Parser *p); +static asdl_seq *_loop0_142_rule(Parser *p); +static asdl_seq *_gather_143_rule(Parser *p); +static void *_tmp_144_rule(Parser *p); +static asdl_seq *_loop0_145_rule(Parser *p); static void *_tmp_146_rule(Parser *p); static void *_tmp_147_rule(Parser *p); static void *_tmp_148_rule(Parser *p); @@ -965,12 +968,13 @@ static void *_tmp_167_rule(Parser *p); static void *_tmp_168_rule(Parser *p); static void *_tmp_169_rule(Parser *p); static void *_tmp_170_rule(Parser *p); -static asdl_seq *_loop0_171_rule(Parser *p); -static void *_tmp_172_rule(Parser *p); +static void *_tmp_171_rule(Parser *p); +static asdl_seq *_loop0_172_rule(Parser *p); static void *_tmp_173_rule(Parser *p); static void *_tmp_174_rule(Parser *p); static void *_tmp_175_rule(Parser *p); static void *_tmp_176_rule(Parser *p); +static void *_tmp_177_rule(Parser *p); // file: statements? $ @@ -1002,7 +1006,7 @@ file_rule(Parser *p) { D(fprintf(stderr, "%*c+ file[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "statements? $")); _res = _PyPegen_make_module ( p , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -1045,7 +1049,7 @@ interactive_rule(Parser *p) { D(fprintf(stderr, "%*c+ interactive[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "statement_newline")); _res = _PyAST_Interactive ( a , p -> arena ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -1094,7 +1098,7 @@ eval_rule(Parser *p) { D(fprintf(stderr, "%*c+ eval[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions NEWLINE* $")); _res = _PyAST_Expression ( a , p -> arena ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -1155,7 +1159,7 @@ func_type_rule(Parser *p) { D(fprintf(stderr, "%*c+ func_type[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' type_expressions? ')' '->' expression NEWLINE* $")); _res = _PyAST_FunctionType ( a , b , p -> arena ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -1197,8 +1201,8 @@ statements_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ statements[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "statement+")); - _res = _PyPegen_register_stmts ( p , ( asdl_stmt_seq* ) _PyPegen_seq_flatten ( p , a ) ); - if (_res == NULL && PyErr_Occurred()) { + _res = ( asdl_stmt_seq* ) _PyPegen_seq_flatten ( p , a ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -1240,8 +1244,8 @@ statement_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ statement[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "compound_stmt")); - _res = ( asdl_stmt_seq* ) _PyPegen_singleton_seq ( p , a ); - if (_res == NULL && PyErr_Occurred()) { + _res = _PyPegen_register_stmts ( p , ( asdl_stmt_seq* ) _PyPegen_singleton_seq ( p , a ) ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -1265,7 +1269,7 @@ statement_rule(Parser *p) { D(fprintf(stderr, "%*c+ statement[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "simple_stmts")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -1308,7 +1312,7 @@ single_compound_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ single_compound_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "compound_stmt")); _res = _PyPegen_register_stmts ( p , ( asdl_stmt_seq* ) _PyPegen_singleton_seq ( p , a ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -1363,7 +1367,7 @@ statement_newline_rule(Parser *p) { D(fprintf(stderr, "%*c+ statement_newline[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "single_compound_stmt NEWLINE")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -1415,7 +1419,7 @@ statement_newline_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = ( asdl_stmt_seq* ) _PyPegen_singleton_seq ( p , CHECK ( stmt_ty , _PyAST_Pass ( EXTRA ) ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -1439,7 +1443,7 @@ statement_newline_rule(Parser *p) { D(fprintf(stderr, "%*c+ statement_newline[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "$")); _res = _PyPegen_interactive_exit ( p ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -1487,7 +1491,7 @@ simple_stmts_rule(Parser *p) { D(fprintf(stderr, "%*c+ simple_stmts[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "simple_stmt !';' NEWLINE")); _res = ( asdl_stmt_seq* ) _PyPegen_singleton_seq ( p , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -1518,7 +1522,7 @@ simple_stmts_rule(Parser *p) { D(fprintf(stderr, "%*c+ simple_stmts[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "';'.simple_stmt+ ';'? NEWLINE")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -1637,7 +1641,7 @@ simple_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Expr ( e , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -1677,7 +1681,7 @@ simple_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> simple_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('import' | 'from') import_stmt")); stmt_ty import_stmt_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_5_rule, p) + _PyPegen_lookahead(1, _tmp_5_rule, p) && (import_stmt_var = import_stmt_rule(p)) // import_stmt ) @@ -1915,7 +1919,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('def' | '@' | 'async') function_def")); stmt_ty function_def_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_6_rule, p) + _PyPegen_lookahead(1, _tmp_6_rule, p) && (function_def_var = function_def_rule(p)) // function_def ) @@ -1957,7 +1961,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('class' | '@') class_def")); stmt_ty class_def_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_7_rule, p) + _PyPegen_lookahead(1, _tmp_7_rule, p) && (class_def_var = class_def_rule(p)) // class_def ) @@ -1978,7 +1982,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('with' | 'async') with_stmt")); stmt_ty with_stmt_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_8_rule, p) + _PyPegen_lookahead(1, _tmp_8_rule, p) && (with_stmt_var = with_stmt_rule(p)) // with_stmt ) @@ -1999,7 +2003,7 @@ compound_stmt_rule(Parser *p) D(fprintf(stderr, "%*c> compound_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&('for' | 'async') for_stmt")); stmt_ty for_stmt_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_9_rule, p) + _PyPegen_lookahead(1, _tmp_9_rule, p) && (for_stmt_var = for_stmt_rule(p)) // for_stmt ) @@ -2137,7 +2141,7 @@ assignment_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = CHECK_VERSION ( stmt_ty , 6 , "Variable annotation syntax is" , _PyAST_AnnAssign ( CHECK ( expr_ty , _PyPegen_set_expr_context ( p , a , Store ) ) , b , c , 1 , EXTRA ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -2179,7 +2183,7 @@ assignment_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = CHECK_VERSION ( stmt_ty , 6 , "Variable annotations syntax is" , _PyAST_AnnAssign ( a , b , c , 0 , EXTRA ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -2220,7 +2224,7 @@ assignment_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Assign ( a , b , NEW_TYPE_COMMENT ( p , tc ) , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -2262,7 +2266,7 @@ assignment_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_AugAssign ( a , b -> kind , c , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -2398,7 +2402,7 @@ augassign_rule(Parser *p) { D(fprintf(stderr, "%*c+ augassign[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+='")); _res = _PyPegen_augoperator ( p , Add ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -2422,7 +2426,7 @@ augassign_rule(Parser *p) { D(fprintf(stderr, "%*c+ augassign[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-='")); _res = _PyPegen_augoperator ( p , Sub ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -2446,7 +2450,7 @@ augassign_rule(Parser *p) { D(fprintf(stderr, "%*c+ augassign[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*='")); _res = _PyPegen_augoperator ( p , Mult ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -2470,7 +2474,7 @@ augassign_rule(Parser *p) { D(fprintf(stderr, "%*c+ augassign[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@='")); _res = CHECK_VERSION ( AugOperator* , 5 , "The '@' operator is" , _PyPegen_augoperator ( p , MatMult ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -2494,7 +2498,7 @@ augassign_rule(Parser *p) { D(fprintf(stderr, "%*c+ augassign[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/='")); _res = _PyPegen_augoperator ( p , Div ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -2518,7 +2522,7 @@ augassign_rule(Parser *p) { D(fprintf(stderr, "%*c+ augassign[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'%='")); _res = _PyPegen_augoperator ( p , Mod ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -2542,7 +2546,7 @@ augassign_rule(Parser *p) { D(fprintf(stderr, "%*c+ augassign[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'&='")); _res = _PyPegen_augoperator ( p , BitAnd ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -2566,7 +2570,7 @@ augassign_rule(Parser *p) { D(fprintf(stderr, "%*c+ augassign[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'|='")); _res = _PyPegen_augoperator ( p , BitOr ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -2590,7 +2594,7 @@ augassign_rule(Parser *p) { D(fprintf(stderr, "%*c+ augassign[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'^='")); _res = _PyPegen_augoperator ( p , BitXor ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -2614,7 +2618,7 @@ augassign_rule(Parser *p) { D(fprintf(stderr, "%*c+ augassign[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'<<='")); _res = _PyPegen_augoperator ( p , LShift ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -2638,7 +2642,7 @@ augassign_rule(Parser *p) { D(fprintf(stderr, "%*c+ augassign[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'>>='")); _res = _PyPegen_augoperator ( p , RShift ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -2662,7 +2666,7 @@ augassign_rule(Parser *p) { D(fprintf(stderr, "%*c+ augassign[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**='")); _res = _PyPegen_augoperator ( p , Pow ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -2686,7 +2690,7 @@ augassign_rule(Parser *p) { D(fprintf(stderr, "%*c+ augassign[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'//='")); _res = _PyPegen_augoperator ( p , FloorDiv ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -2750,7 +2754,7 @@ return_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Return ( a , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -2817,7 +2821,7 @@ raise_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Raise ( a , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -2850,7 +2854,7 @@ raise_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Raise ( NULL , NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -2911,7 +2915,7 @@ pass_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Pass ( EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -2972,7 +2976,7 @@ break_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Break ( EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -3033,7 +3037,7 @@ continue_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Continue ( EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -3097,7 +3101,7 @@ global_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Global ( CHECK ( asdl_identifier_seq* , _PyPegen_map_names_to_ids ( p , a ) ) , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -3161,7 +3165,7 @@ nonlocal_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Nonlocal ( CHECK ( asdl_identifier_seq* , _PyPegen_map_names_to_ids ( p , a ) ) , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -3213,7 +3217,7 @@ del_stmt_rule(Parser *p) && (a = del_targets_rule(p)) // del_targets && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_16_rule, p) + _PyPegen_lookahead(1, _tmp_16_rule, p) ) { D(fprintf(stderr, "%*c+ del_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'del' del_targets &(';' | NEWLINE)")); @@ -3227,7 +3231,7 @@ del_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Delete ( a , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -3307,7 +3311,7 @@ yield_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Expr ( y , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -3374,7 +3378,7 @@ assert_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Assert ( a , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -3514,7 +3518,7 @@ import_name_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Import ( a , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -3589,7 +3593,7 @@ import_from_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyPegen_checked_future_import ( p , b -> v . Name . id , c , _PyPegen_seq_count_dots ( a ) , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -3631,7 +3635,7 @@ import_from_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_ImportFrom ( NULL , b , _PyPegen_seq_count_dots ( a ) , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -3697,7 +3701,7 @@ import_from_targets_rule(Parser *p) { D(fprintf(stderr, "%*c+ import_from_targets[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' import_from_as_names ','? ')'")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -3751,7 +3755,7 @@ import_from_targets_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = ( asdl_alias_seq* ) _PyPegen_singleton_seq ( p , CHECK ( alias_ty , _PyPegen_alias_for_star ( p , EXTRA ) ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -3813,7 +3817,7 @@ import_from_as_names_rule(Parser *p) { D(fprintf(stderr, "%*c+ import_from_as_names[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.import_from_as_name+")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -3896,7 +3900,7 @@ import_from_as_name_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_alias ( a -> v . Name . id , ( b ) ? ( ( expr_ty ) b ) -> v . Name . id : NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -3939,7 +3943,7 @@ dotted_as_names_rule(Parser *p) { D(fprintf(stderr, "%*c+ dotted_as_names[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.dotted_as_name+")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -4022,7 +4026,7 @@ dotted_as_name_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_alias ( a -> v . Name . id , ( b ) ? ( ( expr_ty ) b ) -> v . Name . id : NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -4107,7 +4111,7 @@ dotted_name_raw(Parser *p) { D(fprintf(stderr, "%*c+ dotted_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name '.' NAME")); _res = _PyPegen_join_names_with_dot ( p , a , b ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -4182,7 +4186,7 @@ block_rule(Parser *p) { D(fprintf(stderr, "%*c+ block[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT statements DEDENT")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -4264,7 +4268,7 @@ decorators_rule(Parser *p) { D(fprintf(stderr, "%*c+ decorators[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(('@' named_expression NEWLINE))+")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -4310,7 +4314,7 @@ class_def_rule(Parser *p) { D(fprintf(stderr, "%*c+ class_def[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "decorators class_def_raw")); _res = _PyPegen_class_def_decorators ( p , a , b ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -4426,7 +4430,7 @@ class_def_raw_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_ClassDef ( a -> v . Name . id , ( b ) ? ( ( expr_ty ) b ) -> v . Call . args : NULL , ( b ) ? ( ( expr_ty ) b ) -> v . Call . keywords : NULL , c , NULL , t , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -4472,7 +4476,7 @@ function_def_rule(Parser *p) { D(fprintf(stderr, "%*c+ function_def[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "decorators function_def_raw")); _res = _PyPegen_function_def_decorators ( p , d , f ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -4601,7 +4605,7 @@ function_def_raw_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_FunctionDef ( n -> v . Name . id , ( params ) ? params : CHECK ( arguments_ty , _PyPegen_empty_arguments ( p ) ) , b , NULL , a , NEW_TYPE_COMMENT ( p , tc ) , t , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -4664,7 +4668,7 @@ function_def_raw_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = CHECK_VERSION ( stmt_ty , 5 , "Async functions are" , _PyAST_AsyncFunctionDef ( n -> v . Name . id , ( params ) ? params : CHECK ( arguments_ty , _PyPegen_empty_arguments ( p ) ) , b , NULL , a , NEW_TYPE_COMMENT ( p , tc ) , t , EXTRA ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -4778,7 +4782,7 @@ parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default param_no_default* param_with_default* star_etc?")); _res = CHECK_VERSION ( arguments_ty , 8 , "Positional-only parameters are" , _PyPegen_make_arguments ( p , a , NULL , b , c , d ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -4808,7 +4812,7 @@ parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default param_with_default* star_etc?")); _res = CHECK_VERSION ( arguments_ty , 8 , "Positional-only parameters are" , _PyPegen_make_arguments ( p , NULL , a , NULL , b , c ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -4838,7 +4842,7 @@ parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default+ param_with_default* star_etc?")); _res = _PyPegen_make_arguments ( p , NULL , NULL , a , b , c ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -4865,7 +4869,7 @@ parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_with_default+ star_etc?")); _res = _PyPegen_make_arguments ( p , NULL , NULL , NULL , a , b ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -4889,7 +4893,7 @@ parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_etc")); _res = _PyPegen_make_arguments ( p , NULL , NULL , NULL , NULL , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -4938,7 +4942,7 @@ slash_no_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ slash_no_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default+ '/' ','")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -4967,7 +4971,7 @@ slash_no_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ slash_no_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default+ '/' &')'")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -5021,7 +5025,7 @@ slash_with_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ slash_with_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default* param_with_default+ '/' ','")); _res = _PyPegen_slash_with_default ( p , ( asdl_arg_seq* ) a , b ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -5053,7 +5057,7 @@ slash_with_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ slash_with_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default* param_with_default+ '/' &')'")); _res = _PyPegen_slash_with_default ( p , ( asdl_arg_seq* ) a , b ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -5129,7 +5133,7 @@ star_etc_rule(Parser *p) { D(fprintf(stderr, "%*c+ star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' param_no_default param_maybe_default* kwds?")); _res = _PyPegen_star_etc ( p , a , b , c ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -5162,7 +5166,7 @@ star_etc_rule(Parser *p) { D(fprintf(stderr, "%*c+ star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' param_no_default_star_annotation param_maybe_default* kwds?")); _res = _PyPegen_star_etc ( p , a , b , c ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -5195,7 +5199,7 @@ star_etc_rule(Parser *p) { D(fprintf(stderr, "%*c+ star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' ',' param_maybe_default+ kwds?")); _res = _PyPegen_star_etc ( p , NULL , b , c ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -5219,7 +5223,7 @@ star_etc_rule(Parser *p) { D(fprintf(stderr, "%*c+ star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwds")); _res = _PyPegen_star_etc ( p , NULL , NULL , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -5284,7 +5288,7 @@ kwds_rule(Parser *p) { D(fprintf(stderr, "%*c+ kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' param_no_default")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -5333,7 +5337,7 @@ param_no_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ param_no_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param ',' TYPE_COMMENT?")); _res = _PyPegen_add_type_comment_to_arg ( p , a , tc ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -5362,7 +5366,7 @@ param_no_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ param_no_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param TYPE_COMMENT? &')'")); _res = _PyPegen_add_type_comment_to_arg ( p , a , tc ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -5413,7 +5417,7 @@ param_no_default_star_annotation_rule(Parser *p) { D(fprintf(stderr, "%*c+ param_no_default_star_annotation[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_star_annotation ',' TYPE_COMMENT?")); _res = _PyPegen_add_type_comment_to_arg ( p , a , tc ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -5442,7 +5446,7 @@ param_no_default_star_annotation_rule(Parser *p) { D(fprintf(stderr, "%*c+ param_no_default_star_annotation[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_star_annotation TYPE_COMMENT? &')'")); _res = _PyPegen_add_type_comment_to_arg ( p , a , tc ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -5494,7 +5498,7 @@ param_with_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ param_with_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param default ',' TYPE_COMMENT?")); _res = _PyPegen_name_default_pair ( p , a , c , tc ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -5526,7 +5530,7 @@ param_with_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ param_with_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param default TYPE_COMMENT? &')'")); _res = _PyPegen_name_default_pair ( p , a , c , tc ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -5580,7 +5584,7 @@ param_maybe_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ param_maybe_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param default? ',' TYPE_COMMENT?")); _res = _PyPegen_name_default_pair ( p , a , c , tc ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -5612,7 +5616,7 @@ param_maybe_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ param_maybe_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param default? TYPE_COMMENT? &')'")); _res = _PyPegen_name_default_pair ( p , a , c , tc ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -5676,7 +5680,7 @@ param_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_arg ( a -> v . Name . id , b , NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -5740,7 +5744,7 @@ param_star_annotation_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_arg ( a -> v . Name . id , b , NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -5786,7 +5790,7 @@ annotation_rule(Parser *p) { D(fprintf(stderr, "%*c+ annotation[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':' expression")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -5832,7 +5836,7 @@ star_annotation_rule(Parser *p) { D(fprintf(stderr, "%*c+ star_annotation[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':' star_expression")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -5878,7 +5882,7 @@ default_rule(Parser *p) { D(fprintf(stderr, "%*c+ default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' expression")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -5992,7 +5996,7 @@ if_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_If ( a , b , CHECK ( asdl_stmt_seq* , _PyPegen_singleton_seq ( p , c ) ) , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -6037,7 +6041,7 @@ if_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_If ( a , b , c , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -6132,7 +6136,7 @@ elif_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_If ( a , b , CHECK ( asdl_stmt_seq* , _PyPegen_singleton_seq ( p , c ) ) , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -6177,7 +6181,7 @@ elif_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_If ( a , b , c , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -6245,7 +6249,7 @@ else_block_rule(Parser *p) { D(fprintf(stderr, "%*c+ else_block[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else' &&':' block")); _res = b; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -6337,7 +6341,7 @@ while_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_While ( a , b , c , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -6445,7 +6449,7 @@ for_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_For ( t , ex , b , el , NEW_TYPE_COMMENT ( p , tc ) , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -6509,7 +6513,7 @@ for_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = CHECK_VERSION ( stmt_ty , 5 , "Async for loops are" , _PyAST_AsyncFor ( t , ex , b , el , NEW_TYPE_COMMENT ( p , tc ) , EXTRA ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -6640,7 +6644,7 @@ with_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_With ( a , b , NEW_TYPE_COMMENT ( p , tc ) , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -6685,7 +6689,7 @@ with_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_With ( a , b , NEW_TYPE_COMMENT ( p , tc ) , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -6740,7 +6744,7 @@ with_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = CHECK_VERSION ( stmt_ty , 5 , "Async with statements are" , _PyAST_AsyncWith ( a , b , NULL , EXTRA ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -6788,7 +6792,7 @@ with_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = CHECK_VERSION ( stmt_ty , 5 , "Async with statements are" , _PyAST_AsyncWith ( a , b , NEW_TYPE_COMMENT ( p , tc ) , EXTRA ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -6856,12 +6860,12 @@ with_item_rule(Parser *p) && (t = star_target_rule(p)) // star_target && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_36_rule, p) + _PyPegen_lookahead(1, _tmp_36_rule, p) ) { D(fprintf(stderr, "%*c+ with_item[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression 'as' star_target &(',' | ')' | ':')")); _res = _PyAST_withitem ( e , t , p -> arena ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -6904,7 +6908,7 @@ with_item_rule(Parser *p) { D(fprintf(stderr, "%*c+ with_item[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression")); _res = _PyAST_withitem ( e , NULL , p -> arena ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -6997,7 +7001,7 @@ try_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Try ( b , NULL , NULL , f , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -7045,7 +7049,7 @@ try_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Try ( b , ex , el , f , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -7093,7 +7097,7 @@ try_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = CHECK_VERSION ( stmt_ty , 11 , "Exception groups are" , _PyAST_TryStar ( b , ex , el , f , EXTRA ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -7188,7 +7192,7 @@ except_block_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_ExceptHandler ( e , NULL , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -7236,7 +7240,7 @@ except_block_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_ExceptHandler ( e , ( ( expr_ty ) t ) -> v . Name . id , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -7278,7 +7282,7 @@ except_block_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = CHECK_VERSION ( excepthandler_ty , 14 , "except expressions without parentheses are" , _PyAST_ExceptHandler ( e , NULL , b , EXTRA ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -7317,7 +7321,7 @@ except_block_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_ExceptHandler ( NULL , NULL , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -7433,7 +7437,7 @@ except_star_block_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_ExceptHandler ( e , NULL , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -7484,7 +7488,7 @@ except_star_block_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_ExceptHandler ( e , ( ( expr_ty ) t ) -> v . Name . id , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -7529,7 +7533,7 @@ except_star_block_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = CHECK_VERSION ( excepthandler_ty , 14 , "except expressions without parentheses are" , _PyAST_ExceptHandler ( e , NULL , b , EXTRA ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -7616,7 +7620,7 @@ finally_block_rule(Parser *p) { D(fprintf(stderr, "%*c+ finally_block[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally' &&':' block")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -7697,7 +7701,7 @@ match_stmt_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = CHECK_VERSION ( stmt_ty , 10 , "Pattern matching is" , _PyAST_Match ( subject , cases , EXTRA ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -7783,7 +7787,7 @@ subject_expr_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Tuple ( CHECK ( asdl_expr_seq* , _PyPegen_seq_insert_in_front ( p , value , values ) ) , Load , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -7876,7 +7880,7 @@ case_block_rule(Parser *p) { D(fprintf(stderr, "%*c+ case_block[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "\"case\" patterns guard? ':' block")); _res = _PyAST_match_case ( pattern , guard , body , p -> arena ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -7922,7 +7926,7 @@ guard_rule(Parser *p) { D(fprintf(stderr, "%*c+ guard[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' named_expression")); _res = guard; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -7983,7 +7987,7 @@ patterns_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchSequence ( patterns , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -8126,7 +8130,7 @@ as_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchAs ( pattern , target -> v . Name . id , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -8206,7 +8210,7 @@ or_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = asdl_seq_LEN ( patterns ) == 1 ? asdl_seq_GET ( patterns , 0 ) : _PyAST_MatchOr ( patterns , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -8445,7 +8449,7 @@ literal_pattern_rule(Parser *p) if ( (value = signed_number_rule(p)) // signed_number && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_42_rule, p) + _PyPegen_lookahead(0, _tmp_42_rule, p) ) { D(fprintf(stderr, "%*c+ literal_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number !('+' | '-')")); @@ -8459,7 +8463,7 @@ literal_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchValue ( value , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -8492,7 +8496,7 @@ literal_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchValue ( value , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -8525,7 +8529,7 @@ literal_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchValue ( value , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -8558,7 +8562,7 @@ literal_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchSingleton ( Py_None , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -8591,7 +8595,7 @@ literal_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchSingleton ( Py_True , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -8624,7 +8628,7 @@ literal_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchSingleton ( Py_False , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -8679,7 +8683,7 @@ literal_expr_rule(Parser *p) if ( (signed_number_var = signed_number_rule(p)) // signed_number && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_42_rule, p) + _PyPegen_lookahead(0, _tmp_42_rule, p) ) { D(fprintf(stderr, "%*c+ literal_expr[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "signed_number !('+' | '-')")); @@ -8717,7 +8721,7 @@ literal_expr_rule(Parser *p) D(fprintf(stderr, "%*c> literal_expr[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&(STRING | FSTRING_START | TSTRING_START) strings")); expr_ty strings_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_43_rule, p) + _PyPegen_lookahead(1, _tmp_43_rule, p) && (strings_var = strings_rule(p)) // strings ) @@ -8752,7 +8756,7 @@ literal_expr_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Constant ( Py_None , NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -8785,7 +8789,7 @@ literal_expr_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Constant ( Py_True , NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -8818,7 +8822,7 @@ literal_expr_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Constant ( Py_False , NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -8887,7 +8891,7 @@ complex_number_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_BinOp ( real , Add , imag , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -8926,7 +8930,7 @@ complex_number_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_BinOp ( real , Sub , imag , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -9009,7 +9013,7 @@ signed_number_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_UnaryOp ( USub , number , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -9092,7 +9096,7 @@ signed_real_number_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_UnaryOp ( USub , real , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -9135,7 +9139,7 @@ real_number_rule(Parser *p) { D(fprintf(stderr, "%*c+ real_number[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NUMBER")); _res = _PyPegen_ensure_real ( p , real ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -9178,7 +9182,7 @@ imaginary_number_rule(Parser *p) { D(fprintf(stderr, "%*c+ imaginary_number[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NUMBER")); _res = _PyPegen_ensure_imaginary ( p , imag ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -9239,7 +9243,7 @@ capture_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchAs ( NULL , target -> v . Name . id , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -9281,12 +9285,12 @@ pattern_capture_target_rule(Parser *p) && (name = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_44_rule, p) + _PyPegen_lookahead(0, _tmp_44_rule, p) ) { D(fprintf(stderr, "%*c+ pattern_capture_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!\"_\" NAME !('.' | '(' | '=')")); _res = _PyPegen_set_expr_context ( p , name , Store ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -9347,7 +9351,7 @@ wildcard_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchAs ( NULL , NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -9396,7 +9400,7 @@ value_pattern_rule(Parser *p) if ( (attr = attr_rule(p)) // attr && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_44_rule, p) + _PyPegen_lookahead(0, _tmp_44_rule, p) ) { D(fprintf(stderr, "%*c+ value_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "attr !('.' | '(' | '=')")); @@ -9410,7 +9414,7 @@ value_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchValue ( attr , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -9513,7 +9517,7 @@ attr_raw(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Attribute ( value , attr -> v . Name . id , Load , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -9620,7 +9624,7 @@ group_pattern_rule(Parser *p) { D(fprintf(stderr, "%*c+ group_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' pattern ')'")); _res = pattern; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -9687,7 +9691,7 @@ sequence_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchSequence ( patterns , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -9726,7 +9730,7 @@ sequence_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchSequence ( patterns , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -9775,7 +9779,7 @@ open_sequence_pattern_rule(Parser *p) { D(fprintf(stderr, "%*c+ open_sequence_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "maybe_star_pattern ',' maybe_sequence_pattern?")); _res = _PyPegen_seq_insert_in_front ( p , pattern , patterns ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -9822,7 +9826,7 @@ maybe_sequence_pattern_rule(Parser *p) { D(fprintf(stderr, "%*c+ maybe_sequence_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.maybe_star_pattern+ ','?")); _res = patterns; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -9947,7 +9951,7 @@ star_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchStar ( target -> v . Name . id , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -9983,7 +9987,7 @@ star_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchStar ( NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -10052,7 +10056,7 @@ mapping_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchMapping ( NULL , NULL , NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -10095,7 +10099,7 @@ mapping_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchMapping ( NULL , NULL , rest -> v . Name . id , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -10144,7 +10148,7 @@ mapping_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchMapping ( CHECK ( asdl_expr_seq* , _PyPegen_get_pattern_keys ( p , items ) ) , CHECK ( asdl_pattern_seq* , _PyPegen_get_patterns ( p , items ) ) , rest -> v . Name . id , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -10187,7 +10191,7 @@ mapping_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchMapping ( CHECK ( asdl_expr_seq* , _PyPegen_get_pattern_keys ( p , items ) ) , CHECK ( asdl_pattern_seq* , _PyPegen_get_patterns ( p , items ) ) , NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -10274,7 +10278,7 @@ key_value_pattern_rule(Parser *p) { D(fprintf(stderr, "%*c+ key_value_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(literal_expr | attr) ':' pattern")); _res = _PyPegen_key_pattern_pair ( p , key , pattern ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -10320,7 +10324,7 @@ double_star_pattern_rule(Parser *p) { D(fprintf(stderr, "%*c+ double_star_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' pattern_capture_target")); _res = target; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -10392,7 +10396,7 @@ class_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchClass ( cls , NULL , NULL , NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -10438,7 +10442,7 @@ class_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchClass ( cls , patterns , NULL , NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -10484,7 +10488,7 @@ class_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchClass ( cls , NULL , CHECK ( asdl_identifier_seq* , _PyPegen_map_names_to_ids ( p , CHECK ( asdl_expr_seq* , _PyPegen_get_pattern_keys ( p , keywords ) ) ) ) , CHECK ( asdl_pattern_seq* , _PyPegen_get_patterns ( p , keywords ) ) , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -10536,7 +10540,7 @@ class_pattern_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_MatchClass ( cls , patterns , CHECK ( asdl_identifier_seq* , _PyPegen_map_names_to_ids ( p , CHECK ( asdl_expr_seq* , _PyPegen_get_pattern_keys ( p , keywords ) ) ) ) , CHECK ( asdl_pattern_seq* , _PyPegen_get_patterns ( p , keywords ) ) , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -10598,7 +10602,7 @@ positional_patterns_rule(Parser *p) { D(fprintf(stderr, "%*c+ positional_patterns[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.pattern+")); _res = args; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -10685,7 +10689,7 @@ keyword_pattern_rule(Parser *p) { D(fprintf(stderr, "%*c+ keyword_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '=' pattern")); _res = _PyPegen_key_pattern_pair ( p , arg , value ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -10758,7 +10762,7 @@ type_alias_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = CHECK_VERSION ( stmt_ty , 12 , "Type statement is" , _PyAST_TypeAlias ( CHECK ( expr_ty , _PyPegen_set_expr_context ( p , n , Store ) ) , t , b , EXTRA ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -10826,7 +10830,7 @@ type_params_rule(Parser *p) { D(fprintf(stderr, "%*c+ type_params[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'[' type_param_seq ']'")); _res = CHECK_VERSION ( asdl_type_param_seq* , 12 , "Type parameter lists are" , t ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -10873,7 +10877,7 @@ type_param_seq_rule(Parser *p) { D(fprintf(stderr, "%*c+ type_param_seq[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.type_param+ ','?")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -10948,7 +10952,7 @@ type_param_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_TypeVar ( a -> v . Name . id , b , c , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -11006,7 +11010,7 @@ type_param_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_TypeVarTuple ( a -> v . Name . id , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -11045,7 +11049,7 @@ type_param_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_ParamSpec ( a -> v . Name . id , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -11092,7 +11096,7 @@ type_param_bound_rule(Parser *p) { D(fprintf(stderr, "%*c+ type_param_bound[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':' expression")); _res = e; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -11138,7 +11142,7 @@ type_param_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ type_param_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' expression")); _res = CHECK_VERSION ( expr_ty , 13 , "Type parameter defaults are" , e ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -11184,7 +11188,7 @@ type_param_starred_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ type_param_starred_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' star_expression")); _res = CHECK_VERSION ( expr_ty , 13 , "Type parameter defaults are" , e ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -11252,7 +11256,7 @@ expressions_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Tuple ( CHECK ( asdl_expr_seq* , _PyPegen_seq_insert_in_front ( p , a , b ) ) , Load , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -11288,7 +11292,7 @@ expressions_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Tuple ( CHECK ( asdl_expr_seq* , _PyPegen_singleton_seq ( p , a ) ) , Load , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -11427,7 +11431,7 @@ expression_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_IfExp ( b , a , c , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -11533,7 +11537,7 @@ yield_expr_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_YieldFrom ( a , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -11569,7 +11573,7 @@ yield_expr_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Yield ( a , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -11640,7 +11644,7 @@ star_expressions_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Tuple ( CHECK ( asdl_expr_seq* , _PyPegen_seq_insert_in_front ( p , a , b ) ) , Load , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -11676,7 +11680,7 @@ star_expressions_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Tuple ( CHECK ( asdl_expr_seq* , _PyPegen_singleton_seq ( p , a ) ) , Load , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -11763,7 +11767,7 @@ star_expression_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Starred ( a , Load , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -11830,7 +11834,7 @@ star_named_expressions_rule(Parser *p) { D(fprintf(stderr, "%*c+ star_named_expressions[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.star_named_expression+ ','?")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -11894,7 +11898,7 @@ star_named_expression_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Starred ( a , Load , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -11983,7 +11987,7 @@ assignment_expression_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = CHECK_VERSION ( expr_ty , 8 , "Assignment expressions are" , _PyAST_NamedExpr ( CHECK ( expr_ty , _PyPegen_set_expr_context ( p , a , Store ) ) , b , EXTRA ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -12133,7 +12137,7 @@ disjunction_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_BoolOp ( Or , CHECK ( asdl_expr_seq* , _PyPegen_seq_insert_in_front ( p , a , b ) ) , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -12221,7 +12225,7 @@ conjunction_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_BoolOp ( And , CHECK ( asdl_expr_seq* , _PyPegen_seq_insert_in_front ( p , a , b ) ) , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -12309,7 +12313,7 @@ inversion_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_UnaryOp ( Not , a , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -12393,7 +12397,7 @@ comparison_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Compare ( a , CHECK ( asdl_int_seq* , _PyPegen_get_cmpops ( p , b ) ) , CHECK ( asdl_expr_seq* , _PyPegen_get_exprs ( p , b ) ) , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -12677,7 +12681,7 @@ eq_bitwise_or_rule(Parser *p) { D(fprintf(stderr, "%*c+ eq_bitwise_or[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'==' bitwise_or")); _res = _PyPegen_cmpop_expr_pair ( p , Eq , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -12723,7 +12727,7 @@ noteq_bitwise_or_rule(Parser *p) { D(fprintf(stderr, "%*c+ noteq_bitwise_or[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "('!=') bitwise_or")); _res = _PyPegen_cmpop_expr_pair ( p , NotEq , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -12769,7 +12773,7 @@ lte_bitwise_or_rule(Parser *p) { D(fprintf(stderr, "%*c+ lte_bitwise_or[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'<=' bitwise_or")); _res = _PyPegen_cmpop_expr_pair ( p , LtE , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -12815,7 +12819,7 @@ lt_bitwise_or_rule(Parser *p) { D(fprintf(stderr, "%*c+ lt_bitwise_or[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'<' bitwise_or")); _res = _PyPegen_cmpop_expr_pair ( p , Lt , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -12861,7 +12865,7 @@ gte_bitwise_or_rule(Parser *p) { D(fprintf(stderr, "%*c+ gte_bitwise_or[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'>=' bitwise_or")); _res = _PyPegen_cmpop_expr_pair ( p , GtE , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -12907,7 +12911,7 @@ gt_bitwise_or_rule(Parser *p) { D(fprintf(stderr, "%*c+ gt_bitwise_or[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'>' bitwise_or")); _res = _PyPegen_cmpop_expr_pair ( p , Gt , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -12956,7 +12960,7 @@ notin_bitwise_or_rule(Parser *p) { D(fprintf(stderr, "%*c+ notin_bitwise_or[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'not' 'in' bitwise_or")); _res = _PyPegen_cmpop_expr_pair ( p , NotIn , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -13002,7 +13006,7 @@ in_bitwise_or_rule(Parser *p) { D(fprintf(stderr, "%*c+ in_bitwise_or[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'in' bitwise_or")); _res = _PyPegen_cmpop_expr_pair ( p , In , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -13051,7 +13055,7 @@ isnot_bitwise_or_rule(Parser *p) { D(fprintf(stderr, "%*c+ isnot_bitwise_or[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'is' 'not' bitwise_or")); _res = _PyPegen_cmpop_expr_pair ( p , IsNot , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -13097,7 +13101,7 @@ is_bitwise_or_rule(Parser *p) { D(fprintf(stderr, "%*c+ is_bitwise_or[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'is' bitwise_or")); _res = _PyPegen_cmpop_expr_pair ( p , Is , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -13200,7 +13204,7 @@ bitwise_or_raw(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_BinOp ( a , BitOr , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -13322,7 +13326,7 @@ bitwise_xor_raw(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_BinOp ( a , BitXor , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -13444,7 +13448,7 @@ bitwise_and_raw(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_BinOp ( a , BitAnd , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -13566,7 +13570,7 @@ shift_expr_raw(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_BinOp ( a , LShift , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -13605,7 +13609,7 @@ shift_expr_raw(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_BinOp ( a , RShift , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -13746,7 +13750,7 @@ sum_raw(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_BinOp ( a , Add , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -13785,7 +13789,7 @@ sum_raw(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_BinOp ( a , Sub , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -13914,7 +13918,7 @@ term_raw(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_BinOp ( a , Mult , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -13953,7 +13957,7 @@ term_raw(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_BinOp ( a , Div , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -13992,7 +13996,7 @@ term_raw(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_BinOp ( a , FloorDiv , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -14031,7 +14035,7 @@ term_raw(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_BinOp ( a , Mod , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -14070,7 +14074,7 @@ term_raw(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = CHECK_VERSION ( expr_ty , 5 , "The '@' operator is" , _PyAST_BinOp ( a , MatMult , b , EXTRA ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -14176,7 +14180,7 @@ factor_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_UnaryOp ( UAdd , a , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -14212,7 +14216,7 @@ factor_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_UnaryOp ( USub , a , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -14248,7 +14252,7 @@ factor_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_UnaryOp ( Invert , a , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -14335,7 +14339,7 @@ power_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_BinOp ( a , Pow , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -14422,7 +14426,7 @@ await_primary_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = CHECK_VERSION ( expr_ty , 5 , "Await expressions are" , _PyAST_Await ( a , EXTRA ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -14550,7 +14554,7 @@ primary_raw(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Attribute ( a , b -> v . Name . id , Load , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -14586,7 +14590,7 @@ primary_raw(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Call ( a , CHECK ( asdl_expr_seq* , ( asdl_expr_seq* ) _PyPegen_singleton_seq ( p , b ) ) , NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -14628,7 +14632,7 @@ primary_raw(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Call ( a , ( b ) ? ( ( expr_ty ) b ) -> v . Call . args : NULL , ( b ) ? ( ( expr_ty ) b ) -> v . Call . keywords : NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -14670,7 +14674,7 @@ primary_raw(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Subscript ( a , b , Load , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -14743,7 +14747,7 @@ slices_rule(Parser *p) { D(fprintf(stderr, "%*c+ slices[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice !','")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -14780,7 +14784,7 @@ slices_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Tuple ( a , Load , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -14850,7 +14854,7 @@ slice_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Slice ( a , b , c , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -14874,7 +14878,7 @@ slice_rule(Parser *p) { D(fprintf(stderr, "%*c+ slice[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "named_expression")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -14964,7 +14968,7 @@ atom_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Constant ( Py_True , NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -14997,7 +15001,7 @@ atom_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Constant ( Py_False , NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -15030,7 +15034,7 @@ atom_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Constant ( Py_None , NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -15049,7 +15053,7 @@ atom_rule(Parser *p) D(fprintf(stderr, "%*c> atom[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "&(STRING | FSTRING_START | TSTRING_START) strings")); expr_ty strings_var; if ( - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_43_rule, p) + _PyPegen_lookahead(1, _tmp_43_rule, p) && (strings_var = strings_rule(p)) // strings ) @@ -15166,7 +15170,7 @@ atom_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Constant ( Py_Ellipsis , NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -15215,7 +15219,7 @@ group_rule(Parser *p) { D(fprintf(stderr, "%*c+ group[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' (yield_expr | named_expression) ')'")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -15304,7 +15308,7 @@ lambdef_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Lambda ( ( a ) ? a : CHECK ( arguments_ty , _PyPegen_empty_arguments ( p ) ) , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -15418,7 +15422,7 @@ lambda_parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ lambda_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default lambda_param_no_default* lambda_param_with_default* lambda_star_etc?")); _res = CHECK_VERSION ( arguments_ty , 8 , "Positional-only parameters are" , _PyPegen_make_arguments ( p , a , NULL , b , c , d ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -15448,7 +15452,7 @@ lambda_parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ lambda_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default lambda_param_with_default* lambda_star_etc?")); _res = CHECK_VERSION ( arguments_ty , 8 , "Positional-only parameters are" , _PyPegen_make_arguments ( p , NULL , a , NULL , b , c ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -15478,7 +15482,7 @@ lambda_parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ lambda_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default+ lambda_param_with_default* lambda_star_etc?")); _res = _PyPegen_make_arguments ( p , NULL , NULL , a , b , c ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -15505,7 +15509,7 @@ lambda_parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ lambda_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_with_default+ lambda_star_etc?")); _res = _PyPegen_make_arguments ( p , NULL , NULL , NULL , a , b ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -15529,7 +15533,7 @@ lambda_parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ lambda_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_star_etc")); _res = _PyPegen_make_arguments ( p , NULL , NULL , NULL , NULL , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -15580,7 +15584,7 @@ lambda_slash_no_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ lambda_slash_no_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default+ '/' ','")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -15609,7 +15613,7 @@ lambda_slash_no_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ lambda_slash_no_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default+ '/' &':'")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -15663,7 +15667,7 @@ lambda_slash_with_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ lambda_slash_with_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default* lambda_param_with_default+ '/' ','")); _res = _PyPegen_slash_with_default ( p , ( asdl_arg_seq* ) a , b ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -15695,7 +15699,7 @@ lambda_slash_with_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ lambda_slash_with_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default* lambda_param_with_default+ '/' &':'")); _res = _PyPegen_slash_with_default ( p , ( asdl_arg_seq* ) a , b ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -15770,7 +15774,7 @@ lambda_star_etc_rule(Parser *p) { D(fprintf(stderr, "%*c+ lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' lambda_param_no_default lambda_param_maybe_default* lambda_kwds?")); _res = _PyPegen_star_etc ( p , a , b , c ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -15803,7 +15807,7 @@ lambda_star_etc_rule(Parser *p) { D(fprintf(stderr, "%*c+ lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' ',' lambda_param_maybe_default+ lambda_kwds?")); _res = _PyPegen_star_etc ( p , NULL , b , c ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -15827,7 +15831,7 @@ lambda_star_etc_rule(Parser *p) { D(fprintf(stderr, "%*c+ lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_kwds")); _res = _PyPegen_star_etc ( p , NULL , NULL , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -15892,7 +15896,7 @@ lambda_kwds_rule(Parser *p) { D(fprintf(stderr, "%*c+ lambda_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' lambda_param_no_default")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -15938,7 +15942,7 @@ lambda_param_no_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ lambda_param_no_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param ','")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -15964,7 +15968,7 @@ lambda_param_no_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ lambda_param_no_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param &':'")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -16013,7 +16017,7 @@ lambda_param_with_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ lambda_param_with_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param default ','")); _res = _PyPegen_name_default_pair ( p , a , c , NULL ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -16042,7 +16046,7 @@ lambda_param_with_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ lambda_param_with_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param default &':'")); _res = _PyPegen_name_default_pair ( p , a , c , NULL ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -16091,7 +16095,7 @@ lambda_param_maybe_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ lambda_param_maybe_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param default? ','")); _res = _PyPegen_name_default_pair ( p , a , c , NULL ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -16120,7 +16124,7 @@ lambda_param_maybe_default_rule(Parser *p) { D(fprintf(stderr, "%*c+ lambda_param_maybe_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param default? &':'")); _res = _PyPegen_name_default_pair ( p , a , c , NULL ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -16181,7 +16185,7 @@ lambda_param_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_arg ( a -> v . Name . id , NULL , NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -16243,7 +16247,7 @@ fstring_middle_rule(Parser *p) { D(fprintf(stderr, "%*c+ fstring_middle[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "FSTRING_MIDDLE")); _res = _PyPegen_constant_from_token ( p , t ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -16321,7 +16325,7 @@ fstring_replacement_field_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyPegen_formatted_value ( p , a , debug_expr , conversion , format , rbrace , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -16386,7 +16390,7 @@ fstring_conversion_rule(Parser *p) { D(fprintf(stderr, "%*c+ fstring_conversion[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "\"!\" NAME")); _res = _PyPegen_check_fstring_conversion ( p , conv_token , conv ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -16450,7 +16454,7 @@ fstring_full_format_spec_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyPegen_setup_full_format_spec ( p , colon , ( asdl_expr_seq* ) spec , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -16493,7 +16497,7 @@ fstring_format_spec_rule(Parser *p) { D(fprintf(stderr, "%*c+ fstring_format_spec[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "FSTRING_MIDDLE")); _res = _PyPegen_decoded_constant_from_token ( p , t ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -16561,7 +16565,7 @@ fstring_rule(Parser *p) { D(fprintf(stderr, "%*c+ fstring[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "FSTRING_START fstring_middle* FSTRING_END")); _res = _PyPegen_joined_str ( p , a , ( asdl_expr_seq* ) b , c ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -16639,7 +16643,7 @@ tstring_format_spec_replacement_field_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyPegen_formatted_value ( p , a , debug_expr , conversion , format , rbrace , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -16701,7 +16705,7 @@ tstring_format_spec_rule(Parser *p) { D(fprintf(stderr, "%*c+ tstring_format_spec[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "TSTRING_MIDDLE")); _res = _PyPegen_decoded_constant_from_token ( p , t ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -16784,7 +16788,7 @@ tstring_full_format_spec_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyPegen_setup_full_format_spec ( p , colon , ( asdl_expr_seq* ) spec , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -16862,7 +16866,7 @@ tstring_replacement_field_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyPegen_interpolation ( p , a , debug_expr , conversion , format , rbrace , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -16943,7 +16947,7 @@ tstring_middle_rule(Parser *p) { D(fprintf(stderr, "%*c+ tstring_middle[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "TSTRING_MIDDLE")); _res = _PyPegen_constant_from_token ( p , t ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -16996,7 +17000,7 @@ tstring_rule(Parser *p) { D(fprintf(stderr, "%*c+ tstring[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "TSTRING_START tstring_middle* TSTRING_END")); _res = CHECK_VERSION ( expr_ty , 14 , "t-strings are" , _PyPegen_template_str ( p , a , ( asdl_expr_seq* ) b , c ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -17040,7 +17044,7 @@ string_rule(Parser *p) { D(fprintf(stderr, "%*c+ string[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "STRING")); _res = _PyPegen_constant_from_string ( p , s ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -17057,7 +17061,7 @@ string_rule(Parser *p) return _res; } -// strings: ((fstring | string | tstring))+ +// strings: invalid_string_tstring_concat | ((fstring | string))+ | tstring+ static expr_ty strings_rule(Parser *p) { @@ -17083,18 +17087,37 @@ strings_rule(Parser *p) UNUSED(_start_lineno); // Only used by EXTRA macro int _start_col_offset = p->tokens[_mark]->col_offset; UNUSED(_start_col_offset); // Only used by EXTRA macro - { // ((fstring | string | tstring))+ + if (p->call_invalid_rules) { // invalid_string_tstring_concat if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> strings[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((fstring | string | tstring))+")); + D(fprintf(stderr, "%*c> strings[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "invalid_string_tstring_concat")); + void *invalid_string_tstring_concat_var; + if ( + (invalid_string_tstring_concat_var = invalid_string_tstring_concat_rule(p)) // invalid_string_tstring_concat + ) + { + D(fprintf(stderr, "%*c+ strings[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_string_tstring_concat")); + _res = invalid_string_tstring_concat_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s strings[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "invalid_string_tstring_concat")); + } + { // ((fstring | string))+ + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> strings[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((fstring | string))+")); asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_loop1_81_rule(p)) // ((fstring | string | tstring))+ + (a = (asdl_expr_seq*)_loop1_81_rule(p)) // ((fstring | string))+ ) { - D(fprintf(stderr, "%*c+ strings[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((fstring | string | tstring))+")); + D(fprintf(stderr, "%*c+ strings[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((fstring | string))+")); Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); if (_token == NULL) { p->level--; @@ -17105,7 +17128,7 @@ strings_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyPegen_concatenate_strings ( p , a , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -17114,7 +17137,40 @@ strings_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s strings[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "((fstring | string | tstring))+")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "((fstring | string))+")); + } + { // tstring+ + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> strings[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring+")); + asdl_expr_seq* a; + if ( + (a = (asdl_expr_seq*)_loop1_82_rule(p)) // tstring+ + ) + { + D(fprintf(stderr, "%*c+ strings[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tstring+")); + Token *_token = _PyPegen_get_last_nonnwhitespace_token(p); + if (_token == NULL) { + p->level--; + return NULL; + } + int _end_lineno = _token->end_lineno; + UNUSED(_end_lineno); // Only used by EXTRA macro + int _end_col_offset = _token->end_col_offset; + UNUSED(_end_col_offset); // Only used by EXTRA macro + _res = _PyPegen_concatenate_tstrings ( p , a , EXTRA ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s strings[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tstring+")); } _res = NULL; done: @@ -17173,7 +17229,7 @@ list_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_List ( a , Load , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -17224,7 +17280,7 @@ tuple_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = _tmp_82_rule(p), !p->error_indicator) // [star_named_expression ',' star_named_expressions?] + (a = _tmp_83_rule(p), !p->error_indicator) // [star_named_expression ',' star_named_expressions?] && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' ) @@ -17240,7 +17296,7 @@ tuple_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Tuple ( a , Load , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -17307,7 +17363,7 @@ set_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Set ( a , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -17374,7 +17430,7 @@ dict_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Dict ( CHECK ( asdl_expr_seq* , _PyPegen_get_keys ( p , a ) ) , CHECK ( asdl_expr_seq* , _PyPegen_get_values ( p , a ) ) , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -17439,14 +17495,14 @@ double_starred_kvpairs_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_seq * a; if ( - (a = _gather_84_rule(p)) // ','.double_starred_kvpair+ + (a = _gather_85_rule(p)) // ','.double_starred_kvpair+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ','?")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -17492,7 +17548,7 @@ double_starred_kvpair_rule(Parser *p) { D(fprintf(stderr, "%*c+ double_starred_kvpair[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' bitwise_or")); _res = _PyPegen_key_value_pair ( p , NULL , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -17560,7 +17616,7 @@ kvpair_rule(Parser *p) { D(fprintf(stderr, "%*c+ kvpair[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' expression")); _res = _PyPegen_key_value_pair ( p , a , b ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -17598,12 +17654,12 @@ for_if_clauses_rule(Parser *p) D(fprintf(stderr, "%*c> for_if_clauses[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "for_if_clause+")); asdl_comprehension_seq* a; if ( - (a = (asdl_comprehension_seq*)_loop1_85_rule(p)) // for_if_clause+ + (a = (asdl_comprehension_seq*)_loop1_86_rule(p)) // for_if_clause+ ) { D(fprintf(stderr, "%*c+ for_if_clauses[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "for_if_clause+")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -17663,12 +17719,12 @@ for_if_clause_rule(Parser *p) && (b = disjunction_rule(p)) // disjunction && - (c = (asdl_expr_seq*)_loop0_86_rule(p)) // (('if' disjunction))* + (c = (asdl_expr_seq*)_loop0_87_rule(p)) // (('if' disjunction))* ) { D(fprintf(stderr, "%*c+ for_if_clause[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async' 'for' star_targets 'in' ~ disjunction (('if' disjunction))*")); _res = CHECK_VERSION ( comprehension_ty , 6 , "Async comprehensions are" , _PyAST_comprehension ( a , b , c , 1 , p -> arena ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -17706,12 +17762,12 @@ for_if_clause_rule(Parser *p) && (b = disjunction_rule(p)) // disjunction && - (c = (asdl_expr_seq*)_loop0_86_rule(p)) // (('if' disjunction))* + (c = (asdl_expr_seq*)_loop0_87_rule(p)) // (('if' disjunction))* ) { D(fprintf(stderr, "%*c+ for_if_clause[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'for' star_targets 'in' ~ disjunction (('if' disjunction))*")); _res = _PyAST_comprehension ( a , b , c , 0 , p -> arena ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -17823,7 +17879,7 @@ listcomp_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_ListComp ( a , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -17912,7 +17968,7 @@ setcomp_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_SetComp ( a , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -17985,7 +18041,7 @@ genexp_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (a = _tmp_87_rule(p)) // assignment_expression | expression !':=' + (a = _tmp_88_rule(p)) // assignment_expression | expression !':=' && (b = for_if_clauses_rule(p)) // for_if_clauses && @@ -18003,7 +18059,7 @@ genexp_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_GeneratorExp ( a , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -18092,7 +18148,7 @@ dictcomp_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_DictComp ( a -> key , a -> value , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -18164,7 +18220,7 @@ arguments_rule(Parser *p) { D(fprintf(stderr, "%*c+ arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args ','? &')'")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -18234,9 +18290,9 @@ args_rule(Parser *p) asdl_expr_seq* a; void *b; if ( - (a = (asdl_expr_seq*)_gather_89_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ + (a = (asdl_expr_seq*)_gather_90_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ && - (b = _tmp_90_rule(p), !p->error_indicator) // [',' kwargs] + (b = _tmp_91_rule(p), !p->error_indicator) // [',' kwargs] ) { D(fprintf(stderr, "%*c+ args[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ [',' kwargs]")); @@ -18250,7 +18306,7 @@ args_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyPegen_collect_call_seqs ( p , a , b , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -18283,7 +18339,7 @@ args_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Call ( _PyPegen_dummy_name ( p ) , CHECK_NULL_ALLOWED ( asdl_expr_seq* , _PyPegen_seq_extract_starred_exprs ( p , a ) ) , CHECK_NULL_ALLOWED ( asdl_keyword_seq* , _PyPegen_seq_delete_starred_exprs ( p , a ) ) , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -18326,16 +18382,16 @@ kwargs_rule(Parser *p) asdl_seq * a; asdl_seq * b; if ( - (a = _gather_92_rule(p)) // ','.kwarg_or_starred+ + (a = _gather_93_rule(p)) // ','.kwarg_or_starred+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (b = _gather_94_rule(p)) // ','.kwarg_or_double_starred+ + (b = _gather_95_rule(p)) // ','.kwarg_or_double_starred+ ) { D(fprintf(stderr, "%*c+ kwargs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_starred+ ',' ','.kwarg_or_double_starred+")); _res = _PyPegen_join_sequences ( p , a , b ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -18352,13 +18408,13 @@ kwargs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> kwargs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_starred+")); - asdl_seq * _gather_92_var; + asdl_seq * _gather_93_var; if ( - (_gather_92_var = _gather_92_rule(p)) // ','.kwarg_or_starred+ + (_gather_93_var = _gather_93_rule(p)) // ','.kwarg_or_starred+ ) { D(fprintf(stderr, "%*c+ kwargs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_starred+")); - _res = _gather_92_var; + _res = _gather_93_var; goto done; } p->mark = _mark; @@ -18371,13 +18427,13 @@ kwargs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> kwargs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_double_starred+")); - asdl_seq * _gather_94_var; + asdl_seq * _gather_95_var; if ( - (_gather_94_var = _gather_94_rule(p)) // ','.kwarg_or_double_starred+ + (_gather_95_var = _gather_95_rule(p)) // ','.kwarg_or_double_starred+ ) { D(fprintf(stderr, "%*c+ kwargs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.kwarg_or_double_starred+")); - _res = _gather_94_var; + _res = _gather_95_var; goto done; } p->mark = _mark; @@ -18459,7 +18515,7 @@ starred_expression_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Starred ( a , Load , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -18564,7 +18620,7 @@ kwarg_or_starred_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyPegen_keyword_or_starred ( p , CHECK ( keyword_ty , _PyAST_keyword ( a -> v . Name . id , b , EXTRA ) ) , 1 ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -18588,7 +18644,7 @@ kwarg_or_starred_rule(Parser *p) { D(fprintf(stderr, "%*c+ kwarg_or_starred[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = _PyPegen_keyword_or_starred ( p , a , 0 ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -18674,7 +18730,7 @@ kwarg_or_double_starred_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyPegen_keyword_or_starred ( p , CHECK ( keyword_ty , _PyAST_keyword ( a -> v . Name . id , b , EXTRA ) ) , 1 ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -18710,7 +18766,7 @@ kwarg_or_double_starred_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyPegen_keyword_or_starred ( p , CHECK ( keyword_ty , _PyAST_keyword ( NULL , a , EXTRA ) ) , 1 ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -18764,7 +18820,7 @@ star_targets_rule(Parser *p) { D(fprintf(stderr, "%*c+ star_targets[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_target !','")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -18788,7 +18844,7 @@ star_targets_rule(Parser *p) if ( (a = star_target_rule(p)) // star_target && - (b = _loop0_95_rule(p)) // ((',' star_target))* + (b = _loop0_96_rule(p)) // ((',' star_target))* && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) @@ -18804,7 +18860,7 @@ star_targets_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Tuple ( CHECK ( asdl_expr_seq* , _PyPegen_seq_insert_in_front ( p , a , b ) ) , Store , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -18844,14 +18900,14 @@ star_targets_list_seq_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_97_rule(p)) // ','.star_target+ + (a = (asdl_expr_seq*)_gather_98_rule(p)) // ','.star_target+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ star_targets_list_seq[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.star_target+ ','?")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -18894,14 +18950,14 @@ star_targets_tuple_seq_rule(Parser *p) if ( (a = star_target_rule(p)) // star_target && - (b = _loop1_98_rule(p)) // ((',' star_target))+ + (b = _loop1_99_rule(p)) // ((',' star_target))+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ star_targets_tuple_seq[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_target ((',' star_target))+ ','?")); _res = ( asdl_expr_seq* ) _PyPegen_seq_insert_in_front ( p , a , b ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -18928,7 +18984,7 @@ star_targets_tuple_seq_rule(Parser *p) { D(fprintf(stderr, "%*c+ star_targets_tuple_seq[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_target ','")); _res = ( asdl_expr_seq* ) _PyPegen_singleton_seq ( p , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -18982,7 +19038,7 @@ star_target_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (a = _tmp_99_rule(p)) // !'*' star_target + (a = _tmp_100_rule(p)) // !'*' star_target ) { D(fprintf(stderr, "%*c+ star_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (!'*' star_target)")); @@ -18996,7 +19052,7 @@ star_target_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Starred ( CHECK ( expr_ty , _PyPegen_set_expr_context ( p , a , Store ) ) , Store , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -19078,7 +19134,7 @@ target_with_star_atom_rule(Parser *p) && (b = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(0, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ target_with_star_atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '.' NAME !t_lookahead")); @@ -19092,7 +19148,7 @@ target_with_star_atom_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Attribute ( a , b -> v . Name . id , Store , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -19122,7 +19178,7 @@ target_with_star_atom_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(0, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ target_with_star_atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '[' slices ']' !t_lookahead")); @@ -19136,7 +19192,7 @@ target_with_star_atom_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Subscript ( a , b , Store , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -19212,7 +19268,7 @@ star_atom_rule(Parser *p) { D(fprintf(stderr, "%*c+ star_atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME")); _res = _PyPegen_set_expr_context ( p , a , Store ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -19242,7 +19298,7 @@ star_atom_rule(Parser *p) { D(fprintf(stderr, "%*c+ star_atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' target_with_star_atom ')'")); _res = _PyPegen_set_expr_context ( p , a , Store ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -19281,7 +19337,7 @@ star_atom_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Tuple ( a , Store , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -19320,7 +19376,7 @@ star_atom_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_List ( a , Store , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -19382,7 +19438,7 @@ single_target_rule(Parser *p) { D(fprintf(stderr, "%*c+ single_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME")); _res = _PyPegen_set_expr_context ( p , a , Store ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -19412,7 +19468,7 @@ single_target_rule(Parser *p) { D(fprintf(stderr, "%*c+ single_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' single_target ')'")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -19469,7 +19525,7 @@ single_subscript_attribute_target_rule(Parser *p) && (b = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(0, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ single_subscript_attribute_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '.' NAME !t_lookahead")); @@ -19483,7 +19539,7 @@ single_subscript_attribute_target_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Attribute ( a , b -> v . Name . id , Store , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -19513,7 +19569,7 @@ single_subscript_attribute_target_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(0, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ single_subscript_attribute_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '[' slices ']' !t_lookahead")); @@ -19527,7 +19583,7 @@ single_subscript_attribute_target_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Subscript ( a , b , Store , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -19623,7 +19679,7 @@ t_primary_raw(Parser *p) && (b = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(1, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(1, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ t_primary[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '.' NAME &t_lookahead")); @@ -19637,7 +19693,7 @@ t_primary_raw(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Attribute ( a , b -> v . Name . id , Load , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -19667,7 +19723,7 @@ t_primary_raw(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(1, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ t_primary[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '[' slices ']' &t_lookahead")); @@ -19681,7 +19737,7 @@ t_primary_raw(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Subscript ( a , b , Load , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -19705,7 +19761,7 @@ t_primary_raw(Parser *p) && (b = genexp_rule(p)) // genexp && - _PyPegen_lookahead(1, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(1, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ t_primary[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary genexp &t_lookahead")); @@ -19719,7 +19775,7 @@ t_primary_raw(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Call ( a , CHECK ( asdl_expr_seq* , ( asdl_expr_seq* ) _PyPegen_singleton_seq ( p , b ) ) , NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -19749,7 +19805,7 @@ t_primary_raw(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 8)) // token=')' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(1, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ t_primary[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '(' arguments? ')' &t_lookahead")); @@ -19763,7 +19819,7 @@ t_primary_raw(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Call ( a , ( b ) ? ( ( expr_ty ) b ) -> v . Call . args : NULL , ( b ) ? ( ( expr_ty ) b ) -> v . Call . keywords : NULL , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -19784,12 +19840,12 @@ t_primary_raw(Parser *p) if ( (a = atom_rule(p)) // atom && - _PyPegen_lookahead(1, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(1, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ t_primary[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "atom &t_lookahead")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -19905,14 +19961,14 @@ del_targets_rule(Parser *p) UNUSED(_opt_var); // Silence compiler warnings asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_101_rule(p)) // ','.del_target+ + (a = (asdl_expr_seq*)_gather_102_rule(p)) // ','.del_target+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { D(fprintf(stderr, "%*c+ del_targets[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.del_target+ ','?")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -19974,7 +20030,7 @@ del_target_rule(Parser *p) && (b = _PyPegen_name_token(p)) // NAME && - _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(0, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ del_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '.' NAME !t_lookahead")); @@ -19988,7 +20044,7 @@ del_target_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Attribute ( a , b -> v . Name . id , Del , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20018,7 +20074,7 @@ del_target_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 10)) // token=']' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) t_lookahead_rule, p) + _PyPegen_lookahead(0, t_lookahead_rule, p) ) { D(fprintf(stderr, "%*c+ del_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "t_primary '[' slices ']' !t_lookahead")); @@ -20032,7 +20088,7 @@ del_target_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Subscript ( a , b , Del , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20104,7 +20160,7 @@ del_t_atom_rule(Parser *p) { D(fprintf(stderr, "%*c+ del_t_atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME")); _res = _PyPegen_set_expr_context ( p , a , Del ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20134,7 +20190,7 @@ del_t_atom_rule(Parser *p) { D(fprintf(stderr, "%*c+ del_t_atom[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' del_target ')'")); _res = _PyPegen_set_expr_context ( p , a , Del ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20173,7 +20229,7 @@ del_t_atom_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_Tuple ( a , Del , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20212,7 +20268,7 @@ del_t_atom_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_List ( a , Del , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20263,7 +20319,7 @@ type_expressions_rule(Parser *p) expr_ty b; expr_ty c; if ( - (a = _gather_103_rule(p)) // ','.expression+ + (a = _gather_104_rule(p)) // ','.expression+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -20280,7 +20336,7 @@ type_expressions_rule(Parser *p) { D(fprintf(stderr, "%*c+ type_expressions[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.expression+ ',' '*' expression ',' '**' expression")); _res = ( asdl_expr_seq* ) _PyPegen_seq_append_to_end ( p , CHECK ( asdl_seq* , _PyPegen_seq_append_to_end ( p , a , b ) ) , c ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20302,7 +20358,7 @@ type_expressions_rule(Parser *p) asdl_seq * a; expr_ty b; if ( - (a = _gather_103_rule(p)) // ','.expression+ + (a = _gather_104_rule(p)) // ','.expression+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -20313,7 +20369,7 @@ type_expressions_rule(Parser *p) { D(fprintf(stderr, "%*c+ type_expressions[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.expression+ ',' '*' expression")); _res = ( asdl_expr_seq* ) _PyPegen_seq_append_to_end ( p , a , b ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20335,7 +20391,7 @@ type_expressions_rule(Parser *p) asdl_seq * a; expr_ty b; if ( - (a = _gather_103_rule(p)) // ','.expression+ + (a = _gather_104_rule(p)) // ','.expression+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -20346,7 +20402,7 @@ type_expressions_rule(Parser *p) { D(fprintf(stderr, "%*c+ type_expressions[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.expression+ ',' '**' expression")); _res = ( asdl_expr_seq* ) _PyPegen_seq_append_to_end ( p , a , b ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20382,7 +20438,7 @@ type_expressions_rule(Parser *p) { D(fprintf(stderr, "%*c+ type_expressions[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' expression ',' '**' expression")); _res = ( asdl_expr_seq* ) _PyPegen_seq_append_to_end ( p , CHECK ( asdl_seq* , _PyPegen_singleton_seq ( p , a ) ) , b ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20409,7 +20465,7 @@ type_expressions_rule(Parser *p) { D(fprintf(stderr, "%*c+ type_expressions[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' expression")); _res = ( asdl_expr_seq* ) _PyPegen_singleton_seq ( p , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20436,7 +20492,7 @@ type_expressions_rule(Parser *p) { D(fprintf(stderr, "%*c+ type_expressions[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' expression")); _res = ( asdl_expr_seq* ) _PyPegen_singleton_seq ( p , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20455,12 +20511,12 @@ type_expressions_rule(Parser *p) D(fprintf(stderr, "%*c> type_expressions[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.expression+")); asdl_expr_seq* a; if ( - (a = (asdl_expr_seq*)_gather_103_rule(p)) // ','.expression+ + (a = (asdl_expr_seq*)_gather_104_rule(p)) // ','.expression+ ) { D(fprintf(stderr, "%*c+ type_expressions[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.expression+")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20506,12 +20562,12 @@ func_type_comment_rule(Parser *p) && (t = _PyPegen_expect_token(p, TYPE_COMMENT)) // token='TYPE_COMMENT' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_104_rule, p) + _PyPegen_lookahead(1, _tmp_105_rule, p) ) { D(fprintf(stderr, "%*c+ func_type_comment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE TYPE_COMMENT &(NEWLINE INDENT)")); _res = t; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20592,20 +20648,20 @@ invalid_arguments_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_arguments[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' ','.(starred_expression !'=')+")); - asdl_seq * _gather_107_var; - void *_tmp_105_var; + asdl_seq * _gather_108_var; + void *_tmp_106_var; Token * a; if ( - (_tmp_105_var = _tmp_105_rule(p)) // (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs + (_tmp_106_var = _tmp_106_rule(p)) // (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs && (a = _PyPegen_expect_token(p, 12)) // token=',' && - (_gather_107_var = _gather_107_rule(p)) // ','.(starred_expression !'=')+ + (_gather_108_var = _gather_108_rule(p)) // ','.(starred_expression !'=')+ ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) | kwargs) ',' ','.(starred_expression !'=')+")); _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( a , "iterable argument unpacking follows keyword argument unpacking" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20634,12 +20690,12 @@ invalid_arguments_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_opt_var = _tmp_108_rule(p), !p->error_indicator) // [args | expression for_if_clauses] + (_opt_var = _tmp_109_rule(p), !p->error_indicator) // [args | expression for_if_clauses] ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses ',' [args | expression for_if_clauses]")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , _PyPegen_get_last_comprehension_item ( PyPegen_last_item ( b , comprehension_ty ) ) , "Generator expression must be parenthesized" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20672,7 +20728,7 @@ invalid_arguments_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '=' expression for_if_clauses")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "invalid syntax. Maybe you meant '==' or ':=' instead of '='?" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20694,18 +20750,18 @@ invalid_arguments_rule(Parser *p) expr_ty a; Token * b; if ( - (_opt_var = _tmp_109_rule(p), !p->error_indicator) // [(args ',')] + (_opt_var = _tmp_110_rule(p), !p->error_indicator) // [(args ',')] && (a = _PyPegen_name_token(p)) // NAME && (b = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_110_rule, p) + _PyPegen_lookahead(1, _tmp_111_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "[(args ',')] NAME '=' &(',' | ')')")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "expected argument value expression" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20732,7 +20788,7 @@ invalid_arguments_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args for_if_clauses")); _res = _PyPegen_nonparen_genexp_in_call ( p , a , b ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20765,7 +20821,7 @@ invalid_arguments_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args ',' expression for_if_clauses")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , _PyPegen_get_last_comprehension_item ( PyPegen_last_item ( b , comprehension_ty ) ) , "Generator expression must be parenthesized" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20795,7 +20851,7 @@ invalid_arguments_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_arguments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args ',' args")); _res = _PyPegen_arguments_parsing_error ( p , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20838,14 +20894,14 @@ invalid_kwarg_rule(Parser *p) Token* a; Token * b; if ( - (a = (Token*)_tmp_111_rule(p)) // 'True' | 'False' | 'None' + (a = (Token*)_tmp_112_rule(p)) // 'True' | 'False' | 'None' && (b = _PyPegen_expect_token(p, 22)) // token='=' ) { D(fprintf(stderr, "%*c+ invalid_kwarg[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "('True' | 'False' | 'None') '='")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "cannot assign to %s" , PyBytes_AS_STRING ( a -> bytes ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20878,7 +20934,7 @@ invalid_kwarg_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_kwarg[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '=' expression for_if_clauses")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "invalid syntax. Maybe you meant '==' or ':=' instead of '='?" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20898,7 +20954,7 @@ invalid_kwarg_rule(Parser *p) expr_ty a; Token * b; if ( - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_112_rule, p) + _PyPegen_lookahead(0, _tmp_113_rule, p) && (a = expression_rule(p)) // expression && @@ -20907,7 +20963,7 @@ invalid_kwarg_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_kwarg[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!(NAME '=') expression '='")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "expression cannot contain assignment, perhaps you meant \"==\"?" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -20940,7 +20996,7 @@ invalid_kwarg_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_kwarg[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' expression '=' expression")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "cannot assign to keyword argument unpacking" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21022,7 +21078,7 @@ expression_without_invalid_rule(Parser *p) int _end_col_offset = _token->end_col_offset; UNUSED(_end_col_offset); // Only used by EXTRA macro _res = _PyAST_IfExp ( b , a , c , EXTRA ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->call_invalid_rules = _prev_call_invalid; p->level--; @@ -21112,7 +21168,7 @@ invalid_legacy_expression_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_legacy_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME !'(' star_expressions")); _res = _PyPegen_check_legacy_stmt ( p , a ) ? RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "Missing parentheses in call to '%U'. Did you mean %U(...)?" , a -> v . Name . id , a -> v . Name . id ) : NULL; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21164,7 +21220,7 @@ invalid_type_param_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' NAME ':' expression")); _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( colon , e -> kind == Tuple_kind ? "cannot use constraints with TypeVarTuple" : "cannot use bound with TypeVarTuple" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21197,7 +21253,7 @@ invalid_type_param_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_type_param[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' NAME ':' expression")); _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( colon , e -> kind == Tuple_kind ? "cannot use constraints with ParamSpec" : "cannot use bound with ParamSpec" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21246,14 +21302,14 @@ invalid_expression_rule(Parser *p) if ( (string_var = _PyPegen_string_token(p)) // STRING && - (a = _loop1_113_rule(p)) // ((!STRING expression_without_invalid))+ + (a = _loop1_114_rule(p)) // ((!STRING expression_without_invalid))+ && (string_var_1 = _PyPegen_string_token(p)) // STRING ) { D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "STRING ((!STRING expression_without_invalid))+ STRING")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( PyPegen_first_item ( a , expr_ty ) , PyPegen_last_item ( a , expr_ty ) , "invalid syntax. Is this intended to be part of the string?" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21273,7 +21329,7 @@ invalid_expression_rule(Parser *p) expr_ty a; expr_ty b; if ( - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_114_rule, p) + _PyPegen_lookahead(0, _tmp_115_rule, p) && (a = disjunction_rule(p)) // disjunction && @@ -21282,7 +21338,7 @@ invalid_expression_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!(NAME STRING | SOFT_KEYWORD) disjunction expression_without_invalid")); _res = _PyPegen_check_legacy_stmt ( p , a ) ? NULL : p -> tokens [p -> mark - 1] -> level == 0 ? NULL : RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "invalid syntax. Perhaps you forgot a comma?" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21309,12 +21365,12 @@ invalid_expression_rule(Parser *p) && (b = disjunction_rule(p)) // disjunction && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_115_rule, p) + _PyPegen_lookahead(0, _tmp_116_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "disjunction 'if' disjunction !('else' | ':')")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "expected 'else' after 'if' expression" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21344,12 +21400,12 @@ invalid_expression_rule(Parser *p) && (_keyword_1 = _PyPegen_expect_token(p, 686)) // token='else' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) expression_rule, p) + _PyPegen_lookahead_for_expr(0, expression_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "disjunction 'if' disjunction 'else' !expression")); _res = RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "expected expression after 'else', but statement is given" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21372,7 +21428,7 @@ invalid_expression_rule(Parser *p) expr_ty b; stmt_ty c; if ( - (a = (stmt_ty)_tmp_116_rule(p)) // pass_stmt | break_stmt | continue_stmt + (a = (stmt_ty)_tmp_117_rule(p)) // pass_stmt | break_stmt | continue_stmt && (_keyword = _PyPegen_expect_token(p, 682)) // token='if' && @@ -21385,7 +21441,7 @@ invalid_expression_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(pass_stmt | break_stmt | continue_stmt) 'if' disjunction 'else' simple_stmt")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "expected expression before 'if', but statement is given" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21418,7 +21474,7 @@ invalid_expression_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'lambda' lambda_params? ':' &FSTRING_MIDDLE")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "f-string: lambda expressions are not allowed without parentheses" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21451,7 +21507,7 @@ invalid_expression_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'lambda' lambda_params? ':' &TSTRING_MIDDLE")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "t-string: lambda expressions are not allowed without parentheses" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21507,7 +21563,7 @@ invalid_named_expression_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':=' expression")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "cannot use assignment expressions with %s" , _PyPegen_get_expr_name ( a ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21534,12 +21590,12 @@ invalid_named_expression_rule(Parser *p) && (b = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_117_rule, p) + _PyPegen_lookahead(0, _tmp_118_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '=' bitwise_or !('=' | ':=')")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "invalid syntax. Maybe you meant '==' or ':=' instead of '='?" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21560,7 +21616,7 @@ invalid_named_expression_rule(Parser *p) Token * b; expr_ty bitwise_or_var; if ( - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_118_rule, p) + _PyPegen_lookahead(0, _tmp_119_rule, p) && (a = bitwise_or_rule(p)) // bitwise_or && @@ -21568,12 +21624,12 @@ invalid_named_expression_rule(Parser *p) && (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_117_rule, p) + _PyPegen_lookahead(0, _tmp_118_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_named_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!(list | tuple | genexp | 'True' | 'None' | 'False') bitwise_or '=' bitwise_or !('=' | ':=')")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "cannot assign to %s here. Maybe you meant '==' instead of '='?" , _PyPegen_get_expr_name ( a ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21629,7 +21685,7 @@ invalid_assignment_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "invalid_ann_assign_target ':' expression")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "only single target (not %s) can be annotated" , _PyPegen_get_expr_name ( a ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21648,7 +21704,7 @@ invalid_assignment_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions* ':' expression")); Token * _literal; Token * _literal_1; - asdl_seq * _loop0_119_var; + asdl_seq * _loop0_120_var; expr_ty a; expr_ty expression_var; if ( @@ -21656,7 +21712,7 @@ invalid_assignment_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_loop0_119_var = _loop0_119_rule(p)) // star_named_expressions* + (_loop0_120_var = _loop0_120_rule(p)) // star_named_expressions* && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -21665,7 +21721,7 @@ invalid_assignment_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions* ':' expression")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "only single target (not tuple) can be annotated" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21695,7 +21751,7 @@ invalid_assignment_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' expression")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "illegal target for annotation" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21713,10 +21769,10 @@ invalid_assignment_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((star_targets '='))* star_expressions '='")); Token * _literal; - asdl_seq * _loop0_120_var; + asdl_seq * _loop0_121_var; expr_ty a; if ( - (_loop0_120_var = _loop0_120_rule(p)) // ((star_targets '='))* + (_loop0_121_var = _loop0_121_rule(p)) // ((star_targets '='))* && (a = star_expressions_rule(p)) // star_expressions && @@ -21725,7 +21781,7 @@ invalid_assignment_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((star_targets '='))* star_expressions '='")); _res = RAISE_SYNTAX_ERROR_INVALID_TARGET ( STAR_TARGETS , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21743,10 +21799,10 @@ invalid_assignment_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_assignment[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((star_targets '='))* yield_expr '='")); Token * _literal; - asdl_seq * _loop0_120_var; + asdl_seq * _loop0_121_var; expr_ty a; if ( - (_loop0_120_var = _loop0_120_rule(p)) // ((star_targets '='))* + (_loop0_121_var = _loop0_121_rule(p)) // ((star_targets '='))* && (a = yield_expr_rule(p)) // yield_expr && @@ -21755,7 +21811,7 @@ invalid_assignment_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((star_targets '='))* yield_expr '='")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "assignment to yield expression not possible" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21785,7 +21841,7 @@ invalid_assignment_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_assignment[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_expressions augassign annotated_rhs")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "'%s' is an illegal expression for augmented assignment" , _PyPegen_get_expr_name ( a ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21872,7 +21928,7 @@ invalid_ann_assign_target_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_ann_assign_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' invalid_ann_assign_target ')'")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21918,7 +21974,7 @@ invalid_del_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_del_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'del' star_expressions")); _res = RAISE_SYNTAX_ERROR_INVALID_TARGET ( DEL_TARGETS , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -21963,7 +22019,7 @@ invalid_block_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_block[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE !INDENT")); _res = RAISE_INDENTATION_ERROR ( "expected an indented block" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22002,11 +22058,11 @@ invalid_comprehension_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '(' | '{') starred_expression for_if_clauses")); - void *_tmp_121_var; + void *_tmp_122_var; expr_ty a; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_121_var = _tmp_121_rule(p)) // '[' | '(' | '{' + (_tmp_122_var = _tmp_122_rule(p)) // '[' | '(' | '{' && (a = starred_expression_rule(p)) // starred_expression && @@ -22015,7 +22071,7 @@ invalid_comprehension_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_comprehension[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "('[' | '(' | '{') starred_expression for_if_clauses")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "iterable unpacking cannot be used in comprehension" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22033,12 +22089,12 @@ invalid_comprehension_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '{') star_named_expression ',' star_named_expressions for_if_clauses")); Token * _literal; - void *_tmp_122_var; + void *_tmp_123_var; expr_ty a; asdl_expr_seq* b; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_122_var = _tmp_122_rule(p)) // '[' | '{' + (_tmp_123_var = _tmp_123_rule(p)) // '[' | '{' && (a = star_named_expression_rule(p)) // star_named_expression && @@ -22051,7 +22107,7 @@ invalid_comprehension_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_comprehension[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "('[' | '{') star_named_expression ',' star_named_expressions for_if_clauses")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , PyPegen_last_item ( b , expr_ty ) , "did you forget parentheses around the comprehension target?" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22068,12 +22124,12 @@ invalid_comprehension_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_comprehension[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('[' | '{') star_named_expression ',' for_if_clauses")); - void *_tmp_122_var; + void *_tmp_123_var; expr_ty a; Token * b; asdl_comprehension_seq* for_if_clauses_var; if ( - (_tmp_122_var = _tmp_122_rule(p)) // '[' | '{' + (_tmp_123_var = _tmp_123_rule(p)) // '[' | '{' && (a = star_named_expression_rule(p)) // star_named_expression && @@ -22084,7 +22140,7 @@ invalid_comprehension_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_comprehension[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "('[' | '{') star_named_expression ',' for_if_clauses")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "did you forget parentheses around the comprehension target?" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22139,7 +22195,7 @@ invalid_dict_comprehension_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_dict_comprehension[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' '**' bitwise_or for_if_clauses '}'")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "dict unpacking cannot be used in dict comprehension" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22191,7 +22247,7 @@ invalid_parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "\"/\" ','")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "at least one argument must precede /" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22209,10 +22265,10 @@ invalid_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(slash_no_default | slash_with_default) param_maybe_default* '/'")); asdl_seq * _loop0_32_var; - void *_tmp_123_var; + void *_tmp_124_var; Token * a; if ( - (_tmp_123_var = _tmp_123_rule(p)) // slash_no_default | slash_with_default + (_tmp_124_var = _tmp_124_rule(p)) // slash_no_default | slash_with_default && (_loop0_32_var = _loop0_32_rule(p)) // param_maybe_default* && @@ -22221,7 +22277,7 @@ invalid_parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(slash_no_default | slash_with_default) param_maybe_default* '/'")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "/ may appear only once" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22255,7 +22311,7 @@ invalid_parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default? param_no_default* invalid_parameters_helper param_no_default")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "parameter without a default follows parameter with a default" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22292,7 +22348,7 @@ invalid_parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default* '(' param_no_default+ ','? ')'")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "Function parameters cannot be parenthesized" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22314,16 +22370,16 @@ invalid_parameters_rule(Parser *p) asdl_seq * _loop0_32_var_1; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_124_var; + void *_tmp_125_var; Token * a; if ( - (_opt_var = _tmp_123_rule(p), !p->error_indicator) // [(slash_no_default | slash_with_default)] + (_opt_var = _tmp_124_rule(p), !p->error_indicator) // [(slash_no_default | slash_with_default)] && (_loop0_32_var = _loop0_32_rule(p)) // param_maybe_default* && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_124_var = _tmp_124_rule(p)) // ',' | param_no_default + (_tmp_125_var = _tmp_125_rule(p)) // ',' | param_no_default && (_loop0_32_var_1 = _loop0_32_rule(p)) // param_maybe_default* && @@ -22332,7 +22388,7 @@ invalid_parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "[(slash_no_default | slash_with_default)] param_maybe_default* '*' (',' | param_no_default) param_maybe_default* '/'")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "/ must be ahead of *" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22362,7 +22418,7 @@ invalid_parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_maybe_default+ '/' '*'")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "expected comma between / and *" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22402,12 +22458,12 @@ invalid_default_rule(Parser *p) if ( (a = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_125_rule, p) + _PyPegen_lookahead(1, _tmp_126_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_default[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' &(')' | ',')")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "expected default value expression" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22447,17 +22503,17 @@ invalid_star_etc_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (')' | ',' (')' | '**'))")); - void *_tmp_126_var; + void *_tmp_127_var; Token * a; if ( (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_126_var = _tmp_126_rule(p)) // ')' | ',' (')' | '**') + (_tmp_127_var = _tmp_127_rule(p)) // ')' | ',' (')' | '**') ) { D(fprintf(stderr, "%*c+ invalid_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (')' | ',' (')' | '**'))")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "named arguments must follow bare *" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22487,7 +22543,7 @@ invalid_star_etc_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' ',' TYPE_COMMENT")); _res = RAISE_SYNTAX_ERROR ( "bare * has associated type comment" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22517,7 +22573,7 @@ invalid_star_etc_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' param '='")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "var-positional argument cannot have default value" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22536,24 +22592,24 @@ invalid_star_etc_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (param_no_default | ',') param_maybe_default* '*' (param_no_default | ',')")); Token * _literal; asdl_seq * _loop0_32_var; - void *_tmp_127_var; - void *_tmp_127_var_1; + void *_tmp_128_var; + void *_tmp_128_var_1; Token * a; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_127_var = _tmp_127_rule(p)) // param_no_default | ',' + (_tmp_128_var = _tmp_128_rule(p)) // param_no_default | ',' && (_loop0_32_var = _loop0_32_rule(p)) // param_maybe_default* && (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_127_var_1 = _tmp_127_rule(p)) // param_no_default | ',' + (_tmp_128_var_1 = _tmp_128_rule(p)) // param_no_default | ',' ) { D(fprintf(stderr, "%*c+ invalid_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (param_no_default | ',') param_maybe_default* '*' (param_no_default | ',')")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "* argument may appear only once" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22602,7 +22658,7 @@ invalid_kwds_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' param '='")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "var-keyword argument cannot have default value" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22635,7 +22691,7 @@ invalid_kwds_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' param ',' param")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "arguments cannot follow var-keyword argument" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22663,12 +22719,12 @@ invalid_kwds_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (a = (Token*)_tmp_128_rule(p)) // '*' | '**' | '/' + (a = (Token*)_tmp_129_rule(p)) // '*' | '**' | '/' ) { D(fprintf(stderr, "%*c+ invalid_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' param ',' ('*' | '**' | '/')")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "arguments cannot follow var-keyword argument" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22711,7 +22767,7 @@ invalid_parameters_helper_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_parameters_helper[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); _res = _PyPegen_singleton_seq ( p , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22782,7 +22838,7 @@ invalid_lambda_parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_lambda_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "\"/\" ','")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "at least one argument must precede /" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22800,10 +22856,10 @@ invalid_lambda_parameters_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(lambda_slash_no_default | lambda_slash_with_default) lambda_param_maybe_default* '/'")); asdl_seq * _loop0_75_var; - void *_tmp_129_var; + void *_tmp_130_var; Token * a; if ( - (_tmp_129_var = _tmp_129_rule(p)) // lambda_slash_no_default | lambda_slash_with_default + (_tmp_130_var = _tmp_130_rule(p)) // lambda_slash_no_default | lambda_slash_with_default && (_loop0_75_var = _loop0_75_rule(p)) // lambda_param_maybe_default* && @@ -22812,7 +22868,7 @@ invalid_lambda_parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_lambda_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(lambda_slash_no_default | lambda_slash_with_default) lambda_param_maybe_default* '/'")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "/ may appear only once" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22846,7 +22902,7 @@ invalid_lambda_parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_lambda_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default? lambda_param_no_default* invalid_lambda_parameters_helper lambda_param_no_default")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "parameter without a default follows parameter with a default" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22863,7 +22919,7 @@ invalid_lambda_parameters_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_lambda_parameters[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default* '(' ','.lambda_param+ ','? ')'")); - asdl_seq * _gather_131_var; + asdl_seq * _gather_132_var; asdl_seq * _loop0_71_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings @@ -22874,7 +22930,7 @@ invalid_lambda_parameters_rule(Parser *p) && (a = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_131_var = _gather_131_rule(p)) // ','.lambda_param+ + (_gather_132_var = _gather_132_rule(p)) // ','.lambda_param+ && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -22883,7 +22939,7 @@ invalid_lambda_parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_lambda_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default* '(' ','.lambda_param+ ','? ')'")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "Lambda expression parameters cannot be parenthesized" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22905,16 +22961,16 @@ invalid_lambda_parameters_rule(Parser *p) asdl_seq * _loop0_75_var_1; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_132_var; + void *_tmp_133_var; Token * a; if ( - (_opt_var = _tmp_129_rule(p), !p->error_indicator) // [(lambda_slash_no_default | lambda_slash_with_default)] + (_opt_var = _tmp_130_rule(p), !p->error_indicator) // [(lambda_slash_no_default | lambda_slash_with_default)] && (_loop0_75_var = _loop0_75_rule(p)) // lambda_param_maybe_default* && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_132_var = _tmp_132_rule(p)) // ',' | lambda_param_no_default + (_tmp_133_var = _tmp_133_rule(p)) // ',' | lambda_param_no_default && (_loop0_75_var_1 = _loop0_75_rule(p)) // lambda_param_maybe_default* && @@ -22923,7 +22979,7 @@ invalid_lambda_parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_lambda_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "[(lambda_slash_no_default | lambda_slash_with_default)] lambda_param_maybe_default* '*' (',' | lambda_param_no_default) lambda_param_maybe_default* '/'")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "/ must be ahead of *" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22953,7 +23009,7 @@ invalid_lambda_parameters_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_lambda_parameters[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_maybe_default+ '/' '*'")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "expected comma between / and *" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -22998,7 +23054,7 @@ invalid_lambda_parameters_helper_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_lambda_parameters_helper[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); _res = _PyPegen_singleton_seq ( p , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23057,16 +23113,16 @@ invalid_lambda_star_etc_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_lambda_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (':' | ',' (':' | '**'))")); Token * _literal; - void *_tmp_133_var; + void *_tmp_134_var; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_133_var = _tmp_133_rule(p)) // ':' | ',' (':' | '**') + (_tmp_134_var = _tmp_134_rule(p)) // ':' | ',' (':' | '**') ) { D(fprintf(stderr, "%*c+ invalid_lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (':' | ',' (':' | '**'))")); _res = RAISE_SYNTAX_ERROR ( "named arguments must follow bare *" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23096,7 +23152,7 @@ invalid_lambda_star_etc_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' lambda_param '='")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "var-positional argument cannot have default value" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23115,24 +23171,24 @@ invalid_lambda_star_etc_rule(Parser *p) D(fprintf(stderr, "%*c> invalid_lambda_star_etc[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*' (lambda_param_no_default | ',') lambda_param_maybe_default* '*' (lambda_param_no_default | ',')")); Token * _literal; asdl_seq * _loop0_75_var; - void *_tmp_134_var; - void *_tmp_134_var_1; + void *_tmp_135_var; + void *_tmp_135_var_1; Token * a; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_134_var = _tmp_134_rule(p)) // lambda_param_no_default | ',' + (_tmp_135_var = _tmp_135_rule(p)) // lambda_param_no_default | ',' && (_loop0_75_var = _loop0_75_rule(p)) // lambda_param_maybe_default* && (a = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_134_var_1 = _tmp_134_rule(p)) // lambda_param_no_default | ',' + (_tmp_135_var_1 = _tmp_135_rule(p)) // lambda_param_no_default | ',' ) { D(fprintf(stderr, "%*c+ invalid_lambda_star_etc[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' (lambda_param_no_default | ',') lambda_param_maybe_default* '*' (lambda_param_no_default | ',')")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "* argument may appear only once" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23184,7 +23240,7 @@ invalid_lambda_kwds_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_lambda_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' lambda_param '='")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "var-keyword argument cannot have default value" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23217,7 +23273,7 @@ invalid_lambda_kwds_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_lambda_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' lambda_param ',' lambda_param")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "arguments cannot follow var-keyword argument" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23245,12 +23301,12 @@ invalid_lambda_kwds_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 12)) // token=',' && - (a = (Token*)_tmp_128_rule(p)) // '*' | '**' | '/' + (a = (Token*)_tmp_129_rule(p)) // '*' | '**' | '/' ) { D(fprintf(stderr, "%*c+ invalid_lambda_kwds[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**' lambda_param ',' ('*' | '**' | '/')")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "arguments cannot follow var-keyword argument" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23305,7 +23361,7 @@ invalid_double_type_comments_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_double_type_comments[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "TYPE_COMMENT NEWLINE TYPE_COMMENT NEWLINE INDENT")); _res = RAISE_SYNTAX_ERROR ( "Cannot have two type comments on def" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23351,12 +23407,12 @@ invalid_with_item_rule(Parser *p) && (a = expression_rule(p)) // expression && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_36_rule, p) + _PyPegen_lookahead(1, _tmp_36_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_with_item[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression 'as' expression &(',' | ')' | ':')")); _res = RAISE_SYNTAX_ERROR_INVALID_TARGET ( STAR_TARGETS , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23395,20 +23451,20 @@ invalid_for_if_clause_rule(Parser *p) Token * _keyword; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings - void *_tmp_135_var; + void *_tmp_136_var; if ( (_opt_var = _PyPegen_expect_token(p, 698), !p->error_indicator) // 'async'? && (_keyword = _PyPegen_expect_token(p, 694)) // token='for' && - (_tmp_135_var = _tmp_135_rule(p)) // bitwise_or ((',' bitwise_or))* ','? + (_tmp_136_var = _tmp_136_rule(p)) // bitwise_or ((',' bitwise_or))* ','? && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 695) // token='in' ) { D(fprintf(stderr, "%*c+ invalid_for_if_clause[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'for' (bitwise_or ((',' bitwise_or))* ','?) !'in'")); _res = RAISE_SYNTAX_ERROR ( "'in' expected after for-loop variables" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23458,7 +23514,7 @@ invalid_for_target_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_for_target[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'for' star_expressions")); _res = RAISE_SYNTAX_ERROR_INVALID_TARGET ( FOR_TARGETS , a ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23507,7 +23563,7 @@ invalid_group_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_group[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' starred_expression ')'")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "cannot use starred expression here" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23540,7 +23596,7 @@ invalid_group_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_group[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' '**' expression ')'")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "cannot use double starred expression here" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23576,14 +23632,14 @@ invalid_import_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_import[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'import' ','.dotted_name+ 'from' dotted_name")); - asdl_seq * _gather_137_var; + asdl_seq * _gather_138_var; Token * _keyword; Token * a; expr_ty dotted_name_var; if ( (a = _PyPegen_expect_token(p, 634)) // token='import' && - (_gather_137_var = _gather_137_rule(p)) // ','.dotted_name+ + (_gather_138_var = _gather_138_rule(p)) // ','.dotted_name+ && (_keyword = _PyPegen_expect_token(p, 633)) // token='from' && @@ -23592,7 +23648,7 @@ invalid_import_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_import[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'import' ','.dotted_name+ 'from' dotted_name")); _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( a , "Did you mean to use 'from ... import ...' instead?" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23619,7 +23675,7 @@ invalid_import_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_import[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'import' NEWLINE")); _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( token , "Expected one or more names after 'import'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23636,7 +23692,7 @@ invalid_import_rule(Parser *p) return _res; } -// invalid_dotted_as_name: dotted_name 'as' !(NAME (',' | ')' | NEWLINE)) expression +// invalid_dotted_as_name: dotted_name 'as' !(NAME (',' | ')' | ';' | NEWLINE)) expression static void * invalid_dotted_as_name_rule(Parser *p) { @@ -23649,12 +23705,12 @@ invalid_dotted_as_name_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // dotted_name 'as' !(NAME (',' | ')' | NEWLINE)) expression + { // dotted_name 'as' !(NAME (',' | ')' | ';' | NEWLINE)) expression if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_dotted_as_name[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_name 'as' !(NAME (',' | ')' | NEWLINE)) expression")); + D(fprintf(stderr, "%*c> invalid_dotted_as_name[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_name 'as' !(NAME (',' | ')' | ';' | NEWLINE)) expression")); Token * _keyword; expr_ty a; expr_ty dotted_name_var; @@ -23663,14 +23719,14 @@ invalid_dotted_as_name_rule(Parser *p) && (_keyword = _PyPegen_expect_token(p, 680)) // token='as' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_138_rule, p) + _PyPegen_lookahead(0, _tmp_139_rule, p) && (a = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ invalid_dotted_as_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name 'as' !(NAME (',' | ')' | NEWLINE)) expression")); + D(fprintf(stderr, "%*c+ invalid_dotted_as_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name 'as' !(NAME (',' | ')' | ';' | NEWLINE)) expression")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "cannot use %s as import target" , _PyPegen_get_expr_name ( a ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23679,7 +23735,7 @@ invalid_dotted_as_name_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_dotted_as_name[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_name 'as' !(NAME (',' | ')' | NEWLINE)) expression")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_name 'as' !(NAME (',' | ')' | ';' | NEWLINE)) expression")); } _res = NULL; done: @@ -23687,7 +23743,7 @@ invalid_dotted_as_name_rule(Parser *p) return _res; } -// invalid_import_from_as_name: NAME 'as' !(NAME (',' | ')' | NEWLINE)) expression +// invalid_import_from_as_name: NAME 'as' !(NAME (',' | ')' | ';' | NEWLINE)) expression static void * invalid_import_from_as_name_rule(Parser *p) { @@ -23700,12 +23756,12 @@ invalid_import_from_as_name_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // NAME 'as' !(NAME (',' | ')' | NEWLINE)) expression + { // NAME 'as' !(NAME (',' | ')' | ';' | NEWLINE)) expression if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_import_from_as_name[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME 'as' !(NAME (',' | ')' | NEWLINE)) expression")); + D(fprintf(stderr, "%*c> invalid_import_from_as_name[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME 'as' !(NAME (',' | ')' | ';' | NEWLINE)) expression")); Token * _keyword; expr_ty a; expr_ty name_var; @@ -23714,14 +23770,14 @@ invalid_import_from_as_name_rule(Parser *p) && (_keyword = _PyPegen_expect_token(p, 680)) // token='as' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_138_rule, p) + _PyPegen_lookahead(0, _tmp_139_rule, p) && (a = expression_rule(p)) // expression ) { - D(fprintf(stderr, "%*c+ invalid_import_from_as_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME 'as' !(NAME (',' | ')' | NEWLINE)) expression")); + D(fprintf(stderr, "%*c+ invalid_import_from_as_name[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME 'as' !(NAME (',' | ')' | ';' | NEWLINE)) expression")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "cannot use %s as import target" , _PyPegen_get_expr_name ( a ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23730,7 +23786,7 @@ invalid_import_from_as_name_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_import_from_as_name[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME 'as' !(NAME (',' | ')' | NEWLINE)) expression")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME 'as' !(NAME (',' | ')' | ';' | NEWLINE)) expression")); } _res = NULL; done: @@ -23770,7 +23826,7 @@ invalid_import_from_targets_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_import_from_targets[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "import_from_as_names ',' NEWLINE")); _res = RAISE_SYNTAX_ERROR ( "trailing comma not allowed without surrounding parentheses" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23794,7 +23850,7 @@ invalid_import_from_targets_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_import_from_targets[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( token , "Expected one or more names after 'import'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23832,7 +23888,7 @@ invalid_with_stmt_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' ','.(expression ['as' star_target])+ NEWLINE")); - asdl_seq * _gather_140_var; + asdl_seq * _gather_141_var; Token * _keyword; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings @@ -23842,14 +23898,14 @@ invalid_with_stmt_rule(Parser *p) && (_keyword = _PyPegen_expect_token(p, 647)) // token='with' && - (_gather_140_var = _gather_140_rule(p)) // ','.(expression ['as' star_target])+ + (_gather_141_var = _gather_141_rule(p)) // ','.(expression ['as' star_target])+ && (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { D(fprintf(stderr, "%*c+ invalid_with_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'with' ','.(expression ['as' star_target])+ NEWLINE")); _res = RAISE_SYNTAX_ERROR ( "expected ':'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23866,7 +23922,7 @@ invalid_with_stmt_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE")); - asdl_seq * _gather_142_var; + asdl_seq * _gather_143_var; Token * _keyword; Token * _literal; Token * _literal_1; @@ -23882,7 +23938,7 @@ invalid_with_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_142_var = _gather_142_rule(p)) // ','.(expressions ['as' star_target])+ + (_gather_143_var = _gather_143_rule(p)) // ','.(expressions ['as' star_target])+ && (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -23893,7 +23949,7 @@ invalid_with_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_with_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' NEWLINE")); _res = RAISE_SYNTAX_ERROR ( "expected ':'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23931,7 +23987,7 @@ invalid_with_stmt_indent_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt_indent[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' ','.(expression ['as' star_target])+ ':' NEWLINE !INDENT")); - asdl_seq * _gather_140_var; + asdl_seq * _gather_141_var; Token * _literal; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings @@ -23942,7 +23998,7 @@ invalid_with_stmt_indent_rule(Parser *p) && (a = _PyPegen_expect_token(p, 647)) // token='with' && - (_gather_140_var = _gather_140_rule(p)) // ','.(expression ['as' star_target])+ + (_gather_141_var = _gather_141_rule(p)) // ','.(expression ['as' star_target])+ && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && @@ -23953,7 +24009,7 @@ invalid_with_stmt_indent_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_with_stmt_indent[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'with' ','.(expression ['as' star_target])+ ':' NEWLINE !INDENT")); _res = RAISE_INDENTATION_ERROR ( "expected an indented block after 'with' statement on line %d" , a -> lineno ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -23970,7 +24026,7 @@ invalid_with_stmt_indent_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_with_stmt_indent[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'async'? 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' ':' NEWLINE !INDENT")); - asdl_seq * _gather_142_var; + asdl_seq * _gather_143_var; Token * _literal; Token * _literal_1; Token * _literal_2; @@ -23987,7 +24043,7 @@ invalid_with_stmt_indent_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 7)) // token='(' && - (_gather_142_var = _gather_142_rule(p)) // ','.(expressions ['as' star_target])+ + (_gather_143_var = _gather_143_rule(p)) // ','.(expressions ['as' star_target])+ && (_opt_var_1 = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? && @@ -24002,7 +24058,7 @@ invalid_with_stmt_indent_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_with_stmt_indent[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'with' '(' ','.(expressions ['as' star_target])+ ','? ')' ':' NEWLINE !INDENT")); _res = RAISE_INDENTATION_ERROR ( "expected an indented block after 'with' statement on line %d" , a -> lineno ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24057,7 +24113,7 @@ invalid_try_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_try_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'try' ':' NEWLINE !INDENT")); _res = RAISE_INDENTATION_ERROR ( "expected an indented block after 'try' statement on line %d" , a -> lineno ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24084,12 +24140,12 @@ invalid_try_stmt_rule(Parser *p) && (block_var = block_rule(p)) // block && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_143_rule, p) + _PyPegen_lookahead(0, _tmp_144_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_try_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'try' ':' block !('except' | 'finally')")); _res = RAISE_SYNTAX_ERROR ( "expected 'except' or 'finally' block" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24109,7 +24165,7 @@ invalid_try_stmt_rule(Parser *p) Token * _keyword; Token * _literal; Token * _literal_1; - asdl_seq * _loop0_144_var; + asdl_seq * _loop0_145_var; asdl_seq * _loop1_37_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings @@ -24121,7 +24177,7 @@ invalid_try_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_144_var = _loop0_144_rule(p)) // block* + (_loop0_145_var = _loop0_145_rule(p)) // block* && (_loop1_37_var = _loop1_37_rule(p)) // except_block+ && @@ -24138,7 +24194,7 @@ invalid_try_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_try_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'try' ':' block* except_block+ 'except' '*' expression ['as' NAME] ':'")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "cannot have both 'except' and 'except*' on the same 'try'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24158,7 +24214,7 @@ invalid_try_stmt_rule(Parser *p) Token * _keyword; Token * _literal; Token * _literal_1; - asdl_seq * _loop0_144_var; + asdl_seq * _loop0_145_var; asdl_seq * _loop1_38_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings @@ -24168,20 +24224,20 @@ invalid_try_stmt_rule(Parser *p) && (_literal = _PyPegen_expect_token(p, 11)) // token=':' && - (_loop0_144_var = _loop0_144_rule(p)) // block* + (_loop0_145_var = _loop0_145_rule(p)) // block* && (_loop1_38_var = _loop1_38_rule(p)) // except_star_block+ && (a = _PyPegen_expect_token(p, 677)) // token='except' && - (_opt_var = _tmp_145_rule(p), !p->error_indicator) // [expression ['as' NAME]] + (_opt_var = _tmp_146_rule(p), !p->error_indicator) // [expression ['as' NAME]] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' ) { D(fprintf(stderr, "%*c+ invalid_try_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'try' ':' block* except_star_block+ 'except' [expression ['as' NAME]] ':'")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "cannot have both 'except' and 'except*' on the same 'try'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24202,7 +24258,7 @@ invalid_try_stmt_rule(Parser *p) // | 'except' expression ',' expressions 'as' NAME ':' // | 'except' expression ['as' NAME] NEWLINE // | 'except' NEWLINE -// | 'except' expression 'as' expression +// | 'except' expression 'as' expression ':' block static void * invalid_except_stmt_rule(Parser *p) { @@ -24246,7 +24302,7 @@ invalid_except_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_except_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' expression ',' expressions 'as' NAME ':'")); _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( a , "multiple exception types must be parenthesized when using 'as'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24280,7 +24336,7 @@ invalid_except_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_except_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' expression ['as' NAME] NEWLINE")); _res = RAISE_SYNTAX_ERROR ( "expected ':'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24307,7 +24363,7 @@ invalid_except_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_except_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' NEWLINE")); _res = RAISE_SYNTAX_ERROR ( "expected ':'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24318,15 +24374,17 @@ invalid_except_stmt_rule(Parser *p) D(fprintf(stderr, "%*c%s invalid_except_stmt[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except' NEWLINE")); } - { // 'except' expression 'as' expression + { // 'except' expression 'as' expression ':' block if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_except_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' expression 'as' expression")); + D(fprintf(stderr, "%*c> invalid_except_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' expression 'as' expression ':' block")); Token * _keyword; Token * _keyword_1; + Token * _literal; expr_ty a; + asdl_stmt_seq* block_var; expr_ty expression_var; if ( (_keyword = _PyPegen_expect_token(p, 677)) // token='except' @@ -24336,11 +24394,15 @@ invalid_except_stmt_rule(Parser *p) (_keyword_1 = _PyPegen_expect_token(p, 680)) // token='as' && (a = expression_rule(p)) // expression + && + (_literal = _PyPegen_expect_token(p, 11)) // token=':' + && + (block_var = block_rule(p)) // block ) { - D(fprintf(stderr, "%*c+ invalid_except_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' expression 'as' expression")); + D(fprintf(stderr, "%*c+ invalid_except_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' expression 'as' expression ':' block")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "cannot use except statement with %s" , _PyPegen_get_expr_name ( a ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24349,7 +24411,7 @@ invalid_except_stmt_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_except_stmt[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except' expression 'as' expression")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except' expression 'as' expression ':' block")); } _res = NULL; done: @@ -24361,7 +24423,7 @@ invalid_except_stmt_rule(Parser *p) // | 'except' '*' expression ',' expressions 'as' NAME ':' // | 'except' '*' expression ['as' NAME] NEWLINE // | 'except' '*' (NEWLINE | ':') -// | 'except' '*' expression 'as' expression +// | 'except' '*' expression 'as' expression ':' block static void * invalid_except_star_stmt_rule(Parser *p) { @@ -24408,7 +24470,7 @@ invalid_except_star_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_except_star_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' '*' expression ',' expressions 'as' NAME ':'")); _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( a , "multiple exception types must be parenthesized when using 'as'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24445,7 +24507,7 @@ invalid_except_star_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_except_star_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' '*' expression ['as' NAME] NEWLINE")); _res = RAISE_SYNTAX_ERROR ( "expected ':'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24463,19 +24525,19 @@ invalid_except_star_stmt_rule(Parser *p) } D(fprintf(stderr, "%*c> invalid_except_star_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' '*' (NEWLINE | ':')")); Token * _literal; - void *_tmp_146_var; + void *_tmp_147_var; Token * a; if ( (a = _PyPegen_expect_token(p, 677)) // token='except' && (_literal = _PyPegen_expect_token(p, 16)) // token='*' && - (_tmp_146_var = _tmp_146_rule(p)) // NEWLINE | ':' + (_tmp_147_var = _tmp_147_rule(p)) // NEWLINE | ':' ) { D(fprintf(stderr, "%*c+ invalid_except_star_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' '*' (NEWLINE | ':')")); _res = RAISE_SYNTAX_ERROR ( "expected one or more exception types" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24486,16 +24548,18 @@ invalid_except_star_stmt_rule(Parser *p) D(fprintf(stderr, "%*c%s invalid_except_star_stmt[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except' '*' (NEWLINE | ':')")); } - { // 'except' '*' expression 'as' expression + { // 'except' '*' expression 'as' expression ':' block if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> invalid_except_star_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' '*' expression 'as' expression")); + D(fprintf(stderr, "%*c> invalid_except_star_stmt[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except' '*' expression 'as' expression ':' block")); Token * _keyword; Token * _keyword_1; Token * _literal; + Token * _literal_1; expr_ty a; + asdl_stmt_seq* block_var; expr_ty expression_var; if ( (_keyword = _PyPegen_expect_token(p, 677)) // token='except' @@ -24507,11 +24571,15 @@ invalid_except_star_stmt_rule(Parser *p) (_keyword_1 = _PyPegen_expect_token(p, 680)) // token='as' && (a = expression_rule(p)) // expression + && + (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' + && + (block_var = block_rule(p)) // block ) { - D(fprintf(stderr, "%*c+ invalid_except_star_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' '*' expression 'as' expression")); + D(fprintf(stderr, "%*c+ invalid_except_star_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' '*' expression 'as' expression ':' block")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "cannot use except* statement with %s" , _PyPegen_get_expr_name ( a ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24520,7 +24588,7 @@ invalid_except_star_stmt_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s invalid_except_star_stmt[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except' '*' expression 'as' expression")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except' '*' expression 'as' expression ':' block")); } _res = NULL; done: @@ -24562,7 +24630,7 @@ invalid_finally_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_finally_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally' ':' NEWLINE !INDENT")); _res = RAISE_INDENTATION_ERROR ( "expected an indented block after 'finally' statement on line %d" , a -> lineno ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24622,7 +24690,7 @@ invalid_except_stmt_indent_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_except_stmt_indent[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' expression ['as' NAME] ':' NEWLINE !INDENT")); _res = RAISE_INDENTATION_ERROR ( "expected an indented block after 'except' statement on line %d" , a -> lineno ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24654,7 +24722,7 @@ invalid_except_stmt_indent_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_except_stmt_indent[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' ':' NEWLINE !INDENT")); _res = RAISE_INDENTATION_ERROR ( "expected an indented block after 'except' statement on line %d" , a -> lineno ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24716,7 +24784,7 @@ invalid_except_star_stmt_indent_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_except_star_stmt_indent[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except' '*' expression ['as' NAME] ':' NEWLINE !INDENT")); _res = RAISE_INDENTATION_ERROR ( "expected an indented block after 'except*' statement on line %d" , a -> lineno ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24767,7 +24835,7 @@ invalid_match_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_match_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "\"match\" subject_expr NEWLINE")); _res = CHECK_VERSION ( void* , 10 , "Pattern matching is" , RAISE_SYNTAX_ERROR ( "expected ':'" ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24802,7 +24870,7 @@ invalid_match_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_match_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "\"match\" subject_expr ':' NEWLINE !INDENT")); _res = RAISE_INDENTATION_ERROR ( "expected an indented block after 'match' statement on line %d" , a -> lineno ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24857,7 +24925,7 @@ invalid_case_block_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_case_block[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "\"case\" patterns guard? NEWLINE")); _res = RAISE_SYNTAX_ERROR ( "expected ':'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24896,7 +24964,7 @@ invalid_case_block_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_case_block[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "\"case\" patterns guard? ':' NEWLINE !INDENT")); _res = RAISE_INDENTATION_ERROR ( "expected an indented block after 'case' statement on line %d" , a -> lineno ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24945,7 +25013,7 @@ invalid_as_pattern_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_as_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "or_pattern 'as' \"_\"")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "cannot use '_' as a target" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -24975,7 +25043,7 @@ invalid_as_pattern_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_as_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "or_pattern 'as' expression")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "cannot use %s as pattern target" , _PyPegen_get_expr_name ( a ) ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -25024,7 +25092,7 @@ invalid_class_pattern_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_class_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "name_or_attr '(' invalid_class_argument_pattern")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( PyPegen_first_item ( a , pattern_ty ) , PyPegen_last_item ( a , pattern_ty ) , "positional patterns follow keyword patterns" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -25067,7 +25135,7 @@ invalid_class_argument_pattern_rule(Parser *p) asdl_pattern_seq* a; asdl_seq* keyword_patterns_var; if ( - (_opt_var = _tmp_147_rule(p), !p->error_indicator) // [positional_patterns ','] + (_opt_var = _tmp_148_rule(p), !p->error_indicator) // [positional_patterns ','] && (keyword_patterns_var = keyword_patterns_rule(p)) // keyword_patterns && @@ -25078,7 +25146,7 @@ invalid_class_argument_pattern_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_class_argument_pattern[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "[positional_patterns ','] keyword_patterns ',' positional_patterns")); _res = a; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -25129,7 +25197,7 @@ invalid_if_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_if_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' named_expression NEWLINE")); _res = RAISE_SYNTAX_ERROR ( "expected ':'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -25164,7 +25232,7 @@ invalid_if_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_if_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' named_expression ':' NEWLINE !INDENT")); _res = RAISE_INDENTATION_ERROR ( "expected an indented block after 'if' statement on line %d" , a -> lineno ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -25215,7 +25283,7 @@ invalid_elif_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_elif_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'elif' named_expression NEWLINE")); _res = RAISE_SYNTAX_ERROR ( "expected ':'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -25250,7 +25318,7 @@ invalid_elif_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_elif_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'elif' named_expression ':' NEWLINE !INDENT")); _res = RAISE_INDENTATION_ERROR ( "expected an indented block after 'elif' statement on line %d" , a -> lineno ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -25301,7 +25369,7 @@ invalid_else_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_else_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else' ':' NEWLINE !INDENT")); _res = RAISE_INDENTATION_ERROR ( "expected an indented block after 'else' statement on line %d" , a -> lineno ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -25334,7 +25402,7 @@ invalid_else_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_else_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else' ':' block 'elif'")); _res = RAISE_SYNTAX_ERROR ( "'elif' block follows an 'else' block" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -25385,7 +25453,7 @@ invalid_while_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_while_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'while' named_expression NEWLINE")); _res = RAISE_SYNTAX_ERROR ( "expected ':'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -25420,7 +25488,7 @@ invalid_while_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_while_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'while' named_expression ':' NEWLINE !INDENT")); _res = RAISE_INDENTATION_ERROR ( "expected an indented block after 'while' statement on line %d" , a -> lineno ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -25481,7 +25549,7 @@ invalid_for_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_for_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'for' star_targets 'in' star_expressions NEWLINE")); _res = RAISE_SYNTAX_ERROR ( "expected ':'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -25526,7 +25594,7 @@ invalid_for_stmt_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_for_stmt[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'for' star_targets 'in' star_expressions ':' NEWLINE !INDENT")); _res = RAISE_INDENTATION_ERROR ( "expected an indented block after 'for' statement on line %d" , a -> lineno ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -25604,7 +25672,7 @@ invalid_def_raw_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'async'? 'def' NAME type_params? '(' params? ')' ['->' expression] ':' NEWLINE !INDENT")); _res = RAISE_INDENTATION_ERROR ( "expected an indented block after function definition on line %d" , a -> lineno ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -25717,7 +25785,7 @@ invalid_class_def_raw_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_class_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'class' NAME type_params? ['(' arguments? ')'] NEWLINE")); _res = RAISE_SYNTAX_ERROR ( "expected ':'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -25760,7 +25828,7 @@ invalid_class_def_raw_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_class_def_raw[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'class' NAME type_params? ['(' arguments? ')'] ':' NEWLINE !INDENT")); _res = RAISE_INDENTATION_ERROR ( "expected an indented block after class definition on line %d" , a -> lineno ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -25799,11 +25867,11 @@ invalid_double_starred_kvpairs_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_double_starred_kvpairs[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - asdl_seq * _gather_84_var; + asdl_seq * _gather_85_var; Token * _literal; void *invalid_kvpair_var; if ( - (_gather_84_var = _gather_84_rule(p)) // ','.double_starred_kvpair+ + (_gather_85_var = _gather_85_rule(p)) // ','.double_starred_kvpair+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && @@ -25811,7 +25879,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.double_starred_kvpair+ ',' invalid_kvpair")); - _res = _PyPegen_dummy_name(p, _gather_84_var, _literal, invalid_kvpair_var); + _res = _PyPegen_dummy_name(p, _gather_85_var, _literal, invalid_kvpair_var); goto done; } p->mark = _mark; @@ -25840,7 +25908,7 @@ invalid_double_starred_kvpairs_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' '*' bitwise_or")); _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( a , "cannot use a starred expression in a dictionary value" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -25864,12 +25932,12 @@ invalid_double_starred_kvpairs_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_148_rule, p) + _PyPegen_lookahead(1, _tmp_149_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_double_starred_kvpairs[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "expression expected after dictionary key and ':'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -25917,7 +25985,7 @@ invalid_kvpair_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_kvpair[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !(':')")); _res = RAISE_ERROR_KNOWN_LOCATION ( p , PyExc_SyntaxError , a -> lineno , a -> end_col_offset - 1 , a -> end_lineno , - 1 , "':' expected after dictionary key" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -25950,7 +26018,7 @@ invalid_kvpair_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_kvpair[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' '*' bitwise_or")); _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( a , "cannot use a starred expression in a dictionary value" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -25974,12 +26042,12 @@ invalid_kvpair_rule(Parser *p) && (a = _PyPegen_expect_token(p, 11)) // token=':' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_148_rule, p) + _PyPegen_lookahead(1, _tmp_149_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_kvpair[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ':' &('}' | ',')")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "expression expected after dictionary key and ':'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26031,7 +26099,7 @@ invalid_starred_expression_unpacking_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_starred_expression_unpacking[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*' expression '=' expression")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "cannot assign to iterable argument unpacking" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26074,7 +26142,7 @@ invalid_starred_expression_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_starred_expression[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); _res = RAISE_SYNTAX_ERROR ( "Invalid star expression" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26131,7 +26199,7 @@ invalid_fstring_replacement_field_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' '='")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "f-string: valid expression required before '='" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26158,7 +26226,7 @@ invalid_fstring_replacement_field_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' '!'")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "f-string: valid expression required before '!'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26185,7 +26253,7 @@ invalid_fstring_replacement_field_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' ':'")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "f-string: valid expression required before ':'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26212,7 +26280,7 @@ invalid_fstring_replacement_field_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' '}'")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "f-string: valid expression required before '}'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26233,12 +26301,12 @@ invalid_fstring_replacement_field_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) annotated_rhs_rule, p) + _PyPegen_lookahead_for_expr(0, annotated_rhs_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' !annotated_rhs")); _res = RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "f-string: expecting a valid expression after '{'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26262,12 +26330,12 @@ invalid_fstring_replacement_field_rule(Parser *p) && (annotated_rhs_var = annotated_rhs_rule(p)) // annotated_rhs && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_149_rule, p) + _PyPegen_lookahead(0, _tmp_150_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs !('=' | '!' | ':' | '}')")); _res = PyErr_Occurred ( ) ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "f-string: expecting '=', or '!', or ':', or '}'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26294,12 +26362,12 @@ invalid_fstring_replacement_field_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_150_rule, p) + _PyPegen_lookahead(0, _tmp_151_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '=' !('!' | ':' | '}')")); _res = PyErr_Occurred ( ) ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "f-string: expecting '!', or ':', or '}'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26358,14 +26426,14 @@ invalid_fstring_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_151_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_152_rule(p), !p->error_indicator) // ['!' NAME] && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_152_rule, p) + _PyPegen_lookahead(0, _tmp_153_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] !(':' | '}')")); _res = PyErr_Occurred ( ) ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "f-string: expecting ':' or '}'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26397,7 +26465,7 @@ invalid_fstring_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_151_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_152_rule(p), !p->error_indicator) // ['!' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -26408,7 +26476,7 @@ invalid_fstring_replacement_field_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}'")); _res = PyErr_Occurred ( ) ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "f-string: expecting '}', or format specs" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26438,14 +26506,14 @@ invalid_fstring_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_151_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_152_rule(p), !p->error_indicator) // ['!' NAME] && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) { D(fprintf(stderr, "%*c+ invalid_fstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] !'}'")); _res = PyErr_Occurred ( ) ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "f-string: expecting '}'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26485,12 +26553,12 @@ invalid_fstring_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_152_rule, p) + _PyPegen_lookahead(1, _tmp_153_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' &(':' | '}')")); _res = RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "f-string: missing conversion character" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26511,12 +26579,12 @@ invalid_fstring_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead_with_name(0, _PyPegen_name_token, p) + _PyPegen_lookahead_for_expr(0, _PyPegen_name_token, p) ) { D(fprintf(stderr, "%*c+ invalid_fstring_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' !NAME")); _res = RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "f-string: invalid conversion character" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26573,7 +26641,7 @@ invalid_tstring_replacement_field_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_tstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' '='")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "t-string: valid expression required before '='" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26600,7 +26668,7 @@ invalid_tstring_replacement_field_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_tstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' '!'")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "t-string: valid expression required before '!'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26627,7 +26695,7 @@ invalid_tstring_replacement_field_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_tstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' ':'")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "t-string: valid expression required before ':'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26654,7 +26722,7 @@ invalid_tstring_replacement_field_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_tstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' '}'")); _res = RAISE_SYNTAX_ERROR_KNOWN_LOCATION ( a , "t-string: valid expression required before '}'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26675,12 +26743,12 @@ invalid_tstring_replacement_field_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) annotated_rhs_rule, p) + _PyPegen_lookahead_for_expr(0, annotated_rhs_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' !annotated_rhs")); _res = RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "t-string: expecting a valid expression after '{'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26704,12 +26772,12 @@ invalid_tstring_replacement_field_rule(Parser *p) && (annotated_rhs_var = annotated_rhs_rule(p)) // annotated_rhs && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_149_rule, p) + _PyPegen_lookahead(0, _tmp_150_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs !('=' | '!' | ':' | '}')")); _res = PyErr_Occurred ( ) ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "t-string: expecting '=', or '!', or ':', or '}'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26736,12 +26804,12 @@ invalid_tstring_replacement_field_rule(Parser *p) && (_literal_1 = _PyPegen_expect_token(p, 22)) // token='=' && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_150_rule, p) + _PyPegen_lookahead(0, _tmp_151_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '=' !('!' | ':' | '}')")); _res = PyErr_Occurred ( ) ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "t-string: expecting '!', or ':', or '}'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26800,14 +26868,14 @@ invalid_tstring_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_151_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_152_rule(p), !p->error_indicator) // ['!' NAME] && - _PyPegen_lookahead(0, (void *(*)(Parser *)) _tmp_152_rule, p) + _PyPegen_lookahead(0, _tmp_153_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] !(':' | '}')")); _res = PyErr_Occurred ( ) ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "t-string: expecting ':' or '}'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26839,7 +26907,7 @@ invalid_tstring_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_151_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_152_rule(p), !p->error_indicator) // ['!' NAME] && (_literal_1 = _PyPegen_expect_token(p, 11)) // token=':' && @@ -26850,7 +26918,7 @@ invalid_tstring_replacement_field_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_tstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] ':' fstring_format_spec* !'}'")); _res = PyErr_Occurred ( ) ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "t-string: expecting '}', or format specs" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26880,14 +26948,14 @@ invalid_tstring_replacement_field_rule(Parser *p) && (_opt_var = _PyPegen_expect_token(p, 22), !p->error_indicator) // '='? && - (_opt_var_1 = _tmp_151_rule(p), !p->error_indicator) // ['!' NAME] + (_opt_var_1 = _tmp_152_rule(p), !p->error_indicator) // ['!' NAME] && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 26) // token='}' ) { D(fprintf(stderr, "%*c+ invalid_tstring_replacement_field[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{' annotated_rhs '='? ['!' NAME] !'}'")); _res = PyErr_Occurred ( ) ? NULL : RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "t-string: expecting '}'" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26927,12 +26995,12 @@ invalid_tstring_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead(1, (void *(*)(Parser *)) _tmp_152_rule, p) + _PyPegen_lookahead(1, _tmp_153_rule, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' &(':' | '}')")); _res = RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "t-string: missing conversion character" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26953,12 +27021,12 @@ invalid_tstring_conversion_character_rule(Parser *p) if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' && - _PyPegen_lookahead_with_name(0, _PyPegen_name_token, p) + _PyPegen_lookahead_for_expr(0, _PyPegen_name_token, p) ) { D(fprintf(stderr, "%*c+ invalid_tstring_conversion_character[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' !NAME")); _res = RAISE_SYNTAX_ERROR_ON_NEXT_TOKEN ( "t-string: invalid conversion character" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -26975,6 +27043,81 @@ invalid_tstring_conversion_character_rule(Parser *p) return _res; } +// invalid_string_tstring_concat: +// | ((fstring | string))+ tstring +// | tstring+ (fstring | string) +static void * +invalid_string_tstring_concat_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // ((fstring | string))+ tstring + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_string_tstring_concat[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "((fstring | string))+ tstring")); + asdl_seq * a; + expr_ty b; + if ( + (a = _loop1_81_rule(p)) // ((fstring | string))+ + && + (b = (expr_ty)tstring_rule(p)) // tstring + ) + { + D(fprintf(stderr, "%*c+ invalid_string_tstring_concat[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "((fstring | string))+ tstring")); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( PyPegen_last_item ( a , expr_ty ) , b , "cannot mix t-string literals with string or bytes literals" ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_string_tstring_concat[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "((fstring | string))+ tstring")); + } + { // tstring+ (fstring | string) + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> invalid_string_tstring_concat[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring+ (fstring | string)")); + asdl_seq * a; + expr_ty b; + if ( + (a = _loop1_82_rule(p)) // tstring+ + && + (b = (expr_ty)_tmp_154_rule(p)) // fstring | string + ) + { + D(fprintf(stderr, "%*c+ invalid_string_tstring_concat[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tstring+ (fstring | string)")); + _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( PyPegen_last_item ( a , expr_ty ) , b , "cannot mix t-string literals with string or bytes literals" ); + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { + p->error_indicator = 1; + p->level--; + return NULL; + } + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s invalid_string_tstring_concat[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tstring+ (fstring | string)")); + } + _res = NULL; + done: + p->level--; + return _res; +} + // invalid_arithmetic: sum ('+' | '-' | '*' | '/' | '%' | '//' | '@') 'not' inversion static void * invalid_arithmetic_rule(Parser *p) @@ -26994,14 +27137,14 @@ invalid_arithmetic_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_arithmetic[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "sum ('+' | '-' | '*' | '/' | '%' | '//' | '@') 'not' inversion")); - void *_tmp_153_var; + void *_tmp_155_var; Token * a; expr_ty b; expr_ty sum_var; if ( (sum_var = sum_rule(p)) // sum && - (_tmp_153_var = _tmp_153_rule(p)) // '+' | '-' | '*' | '/' | '%' | '//' | '@' + (_tmp_155_var = _tmp_155_rule(p)) // '+' | '-' | '*' | '/' | '%' | '//' | '@' && (a = _PyPegen_expect_token(p, 703)) // token='not' && @@ -27010,7 +27153,7 @@ invalid_arithmetic_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_arithmetic[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "sum ('+' | '-' | '*' | '/' | '%' | '//' | '@') 'not' inversion")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "'not' after an operator must be parenthesized" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -27046,11 +27189,11 @@ invalid_factor_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> invalid_factor[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('+' | '-' | '~') 'not' factor")); - void *_tmp_154_var; + void *_tmp_156_var; Token * a; expr_ty b; if ( - (_tmp_154_var = _tmp_154_rule(p)) // '+' | '-' | '~' + (_tmp_156_var = _tmp_156_rule(p)) // '+' | '-' | '~' && (a = _PyPegen_expect_token(p, 703)) // token='not' && @@ -27059,7 +27202,7 @@ invalid_factor_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_factor[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "('+' | '-' | '~') 'not' factor")); _res = RAISE_SYNTAX_ERROR_KNOWN_RANGE ( a , b , "'not' after an operator must be parenthesized" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -27105,7 +27248,7 @@ invalid_type_params_rule(Parser *p) { D(fprintf(stderr, "%*c+ invalid_type_params[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'[' ']'")); _res = RAISE_SYNTAX_ERROR_STARTING_FROM ( token , "Type parameter list cannot be empty" ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -27298,7 +27441,7 @@ _loop0_3_rule(Parser *p) ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -27711,7 +27854,7 @@ _tmp_10_rule(Parser *p) { D(fprintf(stderr, "%*c+ _tmp_10[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'=' annotated_rhs")); _res = d; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -27760,7 +27903,7 @@ _tmp_11_rule(Parser *p) { D(fprintf(stderr, "%*c+ _tmp_11[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' single_target ')'")); _res = b; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -27824,12 +27967,12 @@ _loop1_12_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_12[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_155_var; + void *_tmp_157_var; while ( - (_tmp_155_var = _tmp_155_rule(p)) // star_targets '=' + (_tmp_157_var = _tmp_157_rule(p)) // star_targets '=' ) { - _res = _tmp_155_var; + _res = _tmp_157_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -27897,7 +28040,7 @@ _tmp_13_rule(Parser *p) { D(fprintf(stderr, "%*c+ _tmp_13[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'from' expression")); _res = z; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -27951,7 +28094,7 @@ _loop0_14_rule(Parser *p) ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -28117,7 +28260,7 @@ _tmp_17_rule(Parser *p) { D(fprintf(stderr, "%*c+ _tmp_17[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' expression")); _res = z; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -28162,12 +28305,12 @@ _loop0_18_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop0_18[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_156_var; + void *_tmp_158_var; while ( - (_tmp_156_var = _tmp_156_rule(p)) // '.' | '...' + (_tmp_158_var = _tmp_158_rule(p)) // '.' | '...' ) { - _res = _tmp_156_var; + _res = _tmp_158_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28229,12 +28372,12 @@ _loop1_19_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_19[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('.' | '...')")); - void *_tmp_156_var; + void *_tmp_158_var; while ( - (_tmp_156_var = _tmp_156_rule(p)) // '.' | '...' + (_tmp_158_var = _tmp_158_rule(p)) // '.' | '...' ) { - _res = _tmp_156_var; + _res = _tmp_158_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28310,7 +28453,7 @@ _loop0_20_rule(Parser *p) ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -28419,7 +28562,7 @@ _tmp_22_rule(Parser *p) { D(fprintf(stderr, "%*c+ _tmp_22[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' NAME")); _res = z; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -28473,7 +28616,7 @@ _loop0_23_rule(Parser *p) ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -28581,12 +28724,12 @@ _loop1_25_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_25[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('@' named_expression NEWLINE)")); - void *_tmp_157_var; + void *_tmp_159_var; while ( - (_tmp_157_var = _tmp_157_rule(p)) // '@' named_expression NEWLINE + (_tmp_159_var = _tmp_159_rule(p)) // '@' named_expression NEWLINE ) { - _res = _tmp_157_var; + _res = _tmp_159_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -28657,7 +28800,7 @@ _tmp_26_rule(Parser *p) { D(fprintf(stderr, "%*c+ _tmp_26[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'(' arguments? ')'")); _res = z; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -28703,7 +28846,7 @@ _tmp_27_rule(Parser *p) { D(fprintf(stderr, "%*c+ _tmp_27[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'->' expression")); _res = z; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -29174,7 +29317,7 @@ _loop0_34_rule(Parser *p) ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -29583,7 +29726,7 @@ _loop0_40_rule(Parser *p) ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -29909,7 +30052,7 @@ _loop0_45_rule(Parser *p) ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -30026,7 +30169,7 @@ _loop0_47_rule(Parser *p) ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -30200,7 +30343,7 @@ _loop0_50_rule(Parser *p) ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -30317,7 +30460,7 @@ _loop0_52_rule(Parser *p) ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -30434,7 +30577,7 @@ _loop0_54_rule(Parser *p) ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -30614,12 +30757,12 @@ _loop1_57_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_57[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_expression)")); - void *_tmp_158_var; + void *_tmp_160_var; while ( - (_tmp_158_var = _tmp_158_rule(p)) // ',' star_expression + (_tmp_160_var = _tmp_160_rule(p)) // ',' star_expression ) { - _res = _tmp_158_var; + _res = _tmp_160_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30695,7 +30838,7 @@ _loop0_58_rule(Parser *p) ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -30803,12 +30946,12 @@ _loop1_60_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_60[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('or' conjunction)")); - void *_tmp_159_var; + void *_tmp_161_var; while ( - (_tmp_159_var = _tmp_159_rule(p)) // 'or' conjunction + (_tmp_161_var = _tmp_161_rule(p)) // 'or' conjunction ) { - _res = _tmp_159_var; + _res = _tmp_161_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -30875,12 +31018,12 @@ _loop1_61_rule(Parser *p) return NULL; } D(fprintf(stderr, "%*c> _loop1_61[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('and' inversion)")); - void *_tmp_160_var; + void *_tmp_162_var; while ( - (_tmp_160_var = _tmp_160_rule(p)) // 'and' inversion + (_tmp_162_var = _tmp_162_rule(p)) // 'and' inversion ) { - _res = _tmp_160_var; + _res = _tmp_162_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -31017,7 +31160,7 @@ _tmp_63_rule(Parser *p) { D(fprintf(stderr, "%*c+ _tmp_63[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!='")); _res = _PyPegen_check_barry_as_flufl ( p , tok ) ? NULL : tok; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -31067,11 +31210,11 @@ _loop0_64_rule(Parser *p) while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_161_rule(p)) // slice | starred_expression + (elem = _tmp_163_rule(p)) // slice | starred_expression ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -31132,7 +31275,7 @@ _gather_65_rule(Parser *p) void *elem; asdl_seq * seq; if ( - (elem = _tmp_161_rule(p)) // slice | starred_expression + (elem = _tmp_163_rule(p)) // slice | starred_expression && (seq = _loop0_64_rule(p)) // _loop0_64 ) @@ -31180,7 +31323,7 @@ _tmp_66_rule(Parser *p) { D(fprintf(stderr, "%*c+ _tmp_66[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':' expression?")); _res = d; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -32167,7 +32310,7 @@ _loop0_80_rule(Parser *p) return _seq; } -// _loop1_81: (fstring | string | tstring) +// _loop1_81: (fstring | string) static asdl_seq * _loop1_81_rule(Parser *p) { @@ -32189,18 +32332,18 @@ _loop1_81_rule(Parser *p) } Py_ssize_t _children_capacity = 1; Py_ssize_t _n = 0; - { // (fstring | string | tstring) + { // (fstring | string) if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_81[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(fstring | string | tstring)")); - void *_tmp_162_var; + D(fprintf(stderr, "%*c> _loop1_81[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(fstring | string)")); + void *_tmp_154_var; while ( - (_tmp_162_var = _tmp_162_rule(p)) // fstring | string | tstring + (_tmp_154_var = _tmp_154_rule(p)) // fstring | string ) { - _res = _tmp_162_var; + _res = _tmp_154_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32218,7 +32361,7 @@ _loop1_81_rule(Parser *p) } p->mark = _mark; D(fprintf(stderr, "%*c%s _loop1_81[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(fstring | string | tstring)")); + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(fstring | string)")); } if (_n == 0 || p->error_indicator) { PyMem_Free(_children); @@ -32239,9 +32382,81 @@ _loop1_81_rule(Parser *p) return _seq; } -// _tmp_82: star_named_expression ',' star_named_expressions? +// _loop1_82: tstring +static asdl_seq * +_loop1_82_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void *_res = NULL; + int _mark = p->mark; + void **_children = PyMem_Malloc(sizeof(void *)); + if (!_children) { + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + Py_ssize_t _children_capacity = 1; + Py_ssize_t _n = 0; + { // tstring + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _loop1_82[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring")); + expr_ty tstring_var; + while ( + (tstring_var = tstring_rule(p)) // tstring + ) + { + _res = tstring_var; + if (_n == _children_capacity) { + _children_capacity *= 2; + void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); + if (!_new_children) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + _children = _new_children; + } + _children[_n++] = _res; + _mark = p->mark; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _loop1_82[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tstring")); + } + if (_n == 0 || p->error_indicator) { + PyMem_Free(_children); + p->level--; + return NULL; + } + asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); + if (!_seq) { + PyMem_Free(_children); + p->error_indicator = 1; + PyErr_NoMemory(); + p->level--; + return NULL; + } + for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]); + PyMem_Free(_children); + p->level--; + return _seq; +} + +// _tmp_83: star_named_expression ',' star_named_expressions? static void * -_tmp_82_rule(Parser *p) +_tmp_83_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32257,7 +32472,7 @@ _tmp_82_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_82[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions?")); + D(fprintf(stderr, "%*c> _tmp_83[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions?")); Token * _literal; expr_ty y; void *z; @@ -32269,9 +32484,9 @@ _tmp_82_rule(Parser *p) (z = star_named_expressions_rule(p), !p->error_indicator) // star_named_expressions? ) { - D(fprintf(stderr, "%*c+ _tmp_82[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions?")); + D(fprintf(stderr, "%*c+ _tmp_83[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_named_expression ',' star_named_expressions?")); _res = _PyPegen_seq_insert_in_front ( p , y , z ); - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -32279,7 +32494,7 @@ _tmp_82_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_82[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_83[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expression ',' star_named_expressions?")); } _res = NULL; @@ -32288,9 +32503,9 @@ _tmp_82_rule(Parser *p) return _res; } -// _loop0_83: ',' double_starred_kvpair +// _loop0_84: ',' double_starred_kvpair static asdl_seq * -_loop0_83_rule(Parser *p) +_loop0_84_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32315,7 +32530,7 @@ _loop0_83_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_83[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); + D(fprintf(stderr, "%*c> _loop0_84[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' double_starred_kvpair")); Token * _literal; KeyValuePair* elem; while ( @@ -32325,7 +32540,7 @@ _loop0_83_rule(Parser *p) ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -32347,7 +32562,7 @@ _loop0_83_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_83[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_84[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' double_starred_kvpair")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32364,9 +32579,9 @@ _loop0_83_rule(Parser *p) return _seq; } -// _gather_84: double_starred_kvpair _loop0_83 +// _gather_85: double_starred_kvpair _loop0_84 static asdl_seq * -_gather_84_rule(Parser *p) +_gather_85_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32377,27 +32592,27 @@ _gather_84_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // double_starred_kvpair _loop0_83 + { // double_starred_kvpair _loop0_84 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_84[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_83")); + D(fprintf(stderr, "%*c> _gather_85[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_84")); KeyValuePair* elem; asdl_seq * seq; if ( (elem = double_starred_kvpair_rule(p)) // double_starred_kvpair && - (seq = _loop0_83_rule(p)) // _loop0_83 + (seq = _loop0_84_rule(p)) // _loop0_84 ) { - D(fprintf(stderr, "%*c+ _gather_84[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_83")); + D(fprintf(stderr, "%*c+ _gather_85[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "double_starred_kvpair _loop0_84")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_84[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_83")); + D(fprintf(stderr, "%*c%s _gather_85[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "double_starred_kvpair _loop0_84")); } _res = NULL; done: @@ -32405,9 +32620,9 @@ _gather_84_rule(Parser *p) return _res; } -// _loop1_85: for_if_clause +// _loop1_86: for_if_clause static asdl_seq * -_loop1_85_rule(Parser *p) +_loop1_86_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32432,7 +32647,7 @@ _loop1_85_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_85[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "for_if_clause")); + D(fprintf(stderr, "%*c> _loop1_86[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "for_if_clause")); comprehension_ty for_if_clause_var; while ( (for_if_clause_var = for_if_clause_rule(p)) // for_if_clause @@ -32455,7 +32670,7 @@ _loop1_85_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_85[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_86[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "for_if_clause")); } if (_n == 0 || p->error_indicator) { @@ -32477,9 +32692,9 @@ _loop1_85_rule(Parser *p) return _seq; } -// _loop0_86: ('if' disjunction) +// _loop0_87: ('if' disjunction) static asdl_seq * -_loop0_86_rule(Parser *p) +_loop0_87_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32504,13 +32719,13 @@ _loop0_86_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_86[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); - void *_tmp_163_var; + D(fprintf(stderr, "%*c> _loop0_87[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "('if' disjunction)")); + void *_tmp_164_var; while ( - (_tmp_163_var = _tmp_163_rule(p)) // 'if' disjunction + (_tmp_164_var = _tmp_164_rule(p)) // 'if' disjunction ) { - _res = _tmp_163_var; + _res = _tmp_164_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -32527,7 +32742,7 @@ _loop0_86_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_86[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_87[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "('if' disjunction)")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32544,9 +32759,9 @@ _loop0_86_rule(Parser *p) return _seq; } -// _tmp_87: assignment_expression | expression !':=' +// _tmp_88: assignment_expression | expression !':=' static void * -_tmp_87_rule(Parser *p) +_tmp_88_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32562,18 +32777,18 @@ _tmp_87_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_87[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c> _tmp_88[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "assignment_expression")); expr_ty assignment_expression_var; if ( (assignment_expression_var = assignment_expression_rule(p)) // assignment_expression ) { - D(fprintf(stderr, "%*c+ _tmp_87[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); + D(fprintf(stderr, "%*c+ _tmp_88[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "assignment_expression")); _res = assignment_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_87[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_88[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "assignment_expression")); } { // expression !':=' @@ -32581,7 +32796,7 @@ _tmp_87_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_87[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c> _tmp_88[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression !':='")); expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression @@ -32589,12 +32804,12 @@ _tmp_87_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 53) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_87[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); + D(fprintf(stderr, "%*c+ _tmp_88[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression !':='")); _res = expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_87[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_88[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression !':='")); } _res = NULL; @@ -32603,9 +32818,9 @@ _tmp_87_rule(Parser *p) return _res; } -// _loop0_88: ',' (starred_expression | (assignment_expression | expression !':=') !'=') +// _loop0_89: ',' (starred_expression | (assignment_expression | expression !':=') !'=') static asdl_seq * -_loop0_88_rule(Parser *p) +_loop0_89_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32630,17 +32845,17 @@ _loop0_88_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_88[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); + D(fprintf(stderr, "%*c> _loop0_89[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_164_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_165_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -32662,7 +32877,7 @@ _loop0_88_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_88[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_89[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (starred_expression | (assignment_expression | expression !':=') !'=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32679,10 +32894,10 @@ _loop0_88_rule(Parser *p) return _seq; } -// _gather_89: -// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_88 +// _gather_90: +// | (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_89 static asdl_seq * -_gather_89_rule(Parser *p) +_gather_90_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32693,27 +32908,27 @@ _gather_89_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_88 + { // (starred_expression | (assignment_expression | expression !':=') !'=') _loop0_89 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_89[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_88")); + D(fprintf(stderr, "%*c> _gather_90[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_89")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_164_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' + (elem = _tmp_165_rule(p)) // starred_expression | (assignment_expression | expression !':=') !'=' && - (seq = _loop0_88_rule(p)) // _loop0_88 + (seq = _loop0_89_rule(p)) // _loop0_89 ) { - D(fprintf(stderr, "%*c+ _gather_89[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_88")); + D(fprintf(stderr, "%*c+ _gather_90[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_89")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_89[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_88")); + D(fprintf(stderr, "%*c%s _gather_90[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression | (assignment_expression | expression !':=') !'=') _loop0_89")); } _res = NULL; done: @@ -32721,9 +32936,9 @@ _gather_89_rule(Parser *p) return _res; } -// _tmp_90: ',' kwargs +// _tmp_91: ',' kwargs static void * -_tmp_90_rule(Parser *p) +_tmp_91_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32739,7 +32954,7 @@ _tmp_90_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_90[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwargs")); + D(fprintf(stderr, "%*c> _tmp_91[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwargs")); Token * _literal; asdl_seq* k; if ( @@ -32748,9 +32963,9 @@ _tmp_90_rule(Parser *p) (k = kwargs_rule(p)) // kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_90[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' kwargs")); + D(fprintf(stderr, "%*c+ _tmp_91[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' kwargs")); _res = k; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -32758,7 +32973,7 @@ _tmp_90_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_90[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_91[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwargs")); } _res = NULL; @@ -32767,9 +32982,9 @@ _tmp_90_rule(Parser *p) return _res; } -// _loop0_91: ',' kwarg_or_starred +// _loop0_92: ',' kwarg_or_starred static asdl_seq * -_loop0_91_rule(Parser *p) +_loop0_92_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32794,7 +33009,7 @@ _loop0_91_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_91[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_starred")); + D(fprintf(stderr, "%*c> _loop0_92[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_starred")); Token * _literal; KeywordOrStarred* elem; while ( @@ -32804,7 +33019,7 @@ _loop0_91_rule(Parser *p) ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -32826,7 +33041,7 @@ _loop0_91_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_91[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_92[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_starred")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32843,9 +33058,9 @@ _loop0_91_rule(Parser *p) return _seq; } -// _gather_92: kwarg_or_starred _loop0_91 +// _gather_93: kwarg_or_starred _loop0_92 static asdl_seq * -_gather_92_rule(Parser *p) +_gather_93_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32856,27 +33071,27 @@ _gather_92_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // kwarg_or_starred _loop0_91 + { // kwarg_or_starred _loop0_92 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_92[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_91")); + D(fprintf(stderr, "%*c> _gather_93[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_92")); KeywordOrStarred* elem; asdl_seq * seq; if ( (elem = kwarg_or_starred_rule(p)) // kwarg_or_starred && - (seq = _loop0_91_rule(p)) // _loop0_91 + (seq = _loop0_92_rule(p)) // _loop0_92 ) { - D(fprintf(stderr, "%*c+ _gather_92[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_91")); + D(fprintf(stderr, "%*c+ _gather_93[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_starred _loop0_92")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_92[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_starred _loop0_91")); + D(fprintf(stderr, "%*c%s _gather_93[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_starred _loop0_92")); } _res = NULL; done: @@ -32884,9 +33099,9 @@ _gather_92_rule(Parser *p) return _res; } -// _loop0_93: ',' kwarg_or_double_starred +// _loop0_94: ',' kwarg_or_double_starred static asdl_seq * -_loop0_93_rule(Parser *p) +_loop0_94_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32911,7 +33126,7 @@ _loop0_93_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_93[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_double_starred")); + D(fprintf(stderr, "%*c> _loop0_94[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' kwarg_or_double_starred")); Token * _literal; KeywordOrStarred* elem; while ( @@ -32921,7 +33136,7 @@ _loop0_93_rule(Parser *p) ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -32943,7 +33158,7 @@ _loop0_93_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_93[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_94[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' kwarg_or_double_starred")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -32960,9 +33175,9 @@ _loop0_93_rule(Parser *p) return _seq; } -// _gather_94: kwarg_or_double_starred _loop0_93 +// _gather_95: kwarg_or_double_starred _loop0_94 static asdl_seq * -_gather_94_rule(Parser *p) +_gather_95_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -32973,27 +33188,27 @@ _gather_94_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // kwarg_or_double_starred _loop0_93 + { // kwarg_or_double_starred _loop0_94 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_94[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_93")); + D(fprintf(stderr, "%*c> _gather_95[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_94")); KeywordOrStarred* elem; asdl_seq * seq; if ( (elem = kwarg_or_double_starred_rule(p)) // kwarg_or_double_starred && - (seq = _loop0_93_rule(p)) // _loop0_93 + (seq = _loop0_94_rule(p)) // _loop0_94 ) { - D(fprintf(stderr, "%*c+ _gather_94[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_93")); + D(fprintf(stderr, "%*c+ _gather_95[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwarg_or_double_starred _loop0_94")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_94[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_double_starred _loop0_93")); + D(fprintf(stderr, "%*c%s _gather_95[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwarg_or_double_starred _loop0_94")); } _res = NULL; done: @@ -33001,9 +33216,9 @@ _gather_94_rule(Parser *p) return _res; } -// _loop0_95: (',' star_target) +// _loop0_96: (',' star_target) static asdl_seq * -_loop0_95_rule(Parser *p) +_loop0_96_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33028,13 +33243,13 @@ _loop0_95_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_95[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_165_var; + D(fprintf(stderr, "%*c> _loop0_96[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); + void *_tmp_166_var; while ( - (_tmp_165_var = _tmp_165_rule(p)) // ',' star_target + (_tmp_166_var = _tmp_166_rule(p)) // ',' star_target ) { - _res = _tmp_165_var; + _res = _tmp_166_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -33051,7 +33266,7 @@ _loop0_95_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_95[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_96[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_target)")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33068,9 +33283,9 @@ _loop0_95_rule(Parser *p) return _seq; } -// _loop0_96: ',' star_target +// _loop0_97: ',' star_target static asdl_seq * -_loop0_96_rule(Parser *p) +_loop0_97_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33095,7 +33310,7 @@ _loop0_96_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_96[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _loop0_97[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty elem; while ( @@ -33105,7 +33320,7 @@ _loop0_96_rule(Parser *p) ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -33127,7 +33342,7 @@ _loop0_96_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_96[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_97[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33144,9 +33359,9 @@ _loop0_96_rule(Parser *p) return _seq; } -// _gather_97: star_target _loop0_96 +// _gather_98: star_target _loop0_97 static asdl_seq * -_gather_97_rule(Parser *p) +_gather_98_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33157,27 +33372,27 @@ _gather_97_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // star_target _loop0_96 + { // star_target _loop0_97 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_97[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_target _loop0_96")); + D(fprintf(stderr, "%*c> _gather_98[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_target _loop0_97")); expr_ty elem; asdl_seq * seq; if ( (elem = star_target_rule(p)) // star_target && - (seq = _loop0_96_rule(p)) // _loop0_96 + (seq = _loop0_97_rule(p)) // _loop0_97 ) { - D(fprintf(stderr, "%*c+ _gather_97[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_target _loop0_96")); + D(fprintf(stderr, "%*c+ _gather_98[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_target _loop0_97")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_97[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_target _loop0_96")); + D(fprintf(stderr, "%*c%s _gather_98[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_target _loop0_97")); } _res = NULL; done: @@ -33185,9 +33400,9 @@ _gather_97_rule(Parser *p) return _res; } -// _loop1_98: (',' star_target) +// _loop1_99: (',' star_target) static asdl_seq * -_loop1_98_rule(Parser *p) +_loop1_99_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33212,13 +33427,13 @@ _loop1_98_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_98[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); - void *_tmp_165_var; + D(fprintf(stderr, "%*c> _loop1_99[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' star_target)")); + void *_tmp_166_var; while ( - (_tmp_165_var = _tmp_165_rule(p)) // ',' star_target + (_tmp_166_var = _tmp_166_rule(p)) // ',' star_target ) { - _res = _tmp_165_var; + _res = _tmp_166_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -33235,7 +33450,7 @@ _loop1_98_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_98[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_99[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' star_target)")); } if (_n == 0 || p->error_indicator) { @@ -33257,9 +33472,9 @@ _loop1_98_rule(Parser *p) return _seq; } -// _tmp_99: !'*' star_target +// _tmp_100: !'*' star_target static void * -_tmp_99_rule(Parser *p) +_tmp_100_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33275,7 +33490,7 @@ _tmp_99_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_99[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); + D(fprintf(stderr, "%*c> _tmp_100[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); expr_ty star_target_var; if ( _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 16) // token='*' @@ -33283,12 +33498,12 @@ _tmp_99_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_99[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); + D(fprintf(stderr, "%*c+ _tmp_100[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!'*' star_target")); _res = star_target_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_99[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_100[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "!'*' star_target")); } _res = NULL; @@ -33297,9 +33512,9 @@ _tmp_99_rule(Parser *p) return _res; } -// _loop0_100: ',' del_target +// _loop0_101: ',' del_target static asdl_seq * -_loop0_100_rule(Parser *p) +_loop0_101_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33324,7 +33539,7 @@ _loop0_100_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_100[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' del_target")); + D(fprintf(stderr, "%*c> _loop0_101[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' del_target")); Token * _literal; expr_ty elem; while ( @@ -33334,7 +33549,7 @@ _loop0_100_rule(Parser *p) ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -33356,7 +33571,7 @@ _loop0_100_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_100[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_101[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' del_target")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33373,9 +33588,9 @@ _loop0_100_rule(Parser *p) return _seq; } -// _gather_101: del_target _loop0_100 +// _gather_102: del_target _loop0_101 static asdl_seq * -_gather_101_rule(Parser *p) +_gather_102_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33386,27 +33601,27 @@ _gather_101_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // del_target _loop0_100 + { // del_target _loop0_101 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_101[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "del_target _loop0_100")); + D(fprintf(stderr, "%*c> _gather_102[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "del_target _loop0_101")); expr_ty elem; asdl_seq * seq; if ( (elem = del_target_rule(p)) // del_target && - (seq = _loop0_100_rule(p)) // _loop0_100 + (seq = _loop0_101_rule(p)) // _loop0_101 ) { - D(fprintf(stderr, "%*c+ _gather_101[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "del_target _loop0_100")); + D(fprintf(stderr, "%*c+ _gather_102[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "del_target _loop0_101")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_101[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "del_target _loop0_100")); + D(fprintf(stderr, "%*c%s _gather_102[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "del_target _loop0_101")); } _res = NULL; done: @@ -33414,9 +33629,9 @@ _gather_101_rule(Parser *p) return _res; } -// _loop0_102: ',' expression +// _loop0_103: ',' expression static asdl_seq * -_loop0_102_rule(Parser *p) +_loop0_103_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33441,7 +33656,7 @@ _loop0_102_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_102[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); + D(fprintf(stderr, "%*c> _loop0_103[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' expression")); Token * _literal; expr_ty elem; while ( @@ -33451,7 +33666,7 @@ _loop0_102_rule(Parser *p) ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -33473,7 +33688,7 @@ _loop0_102_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_102[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_103[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' expression")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33490,9 +33705,9 @@ _loop0_102_rule(Parser *p) return _seq; } -// _gather_103: expression _loop0_102 +// _gather_104: expression _loop0_103 static asdl_seq * -_gather_103_rule(Parser *p) +_gather_104_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33503,27 +33718,27 @@ _gather_103_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // expression _loop0_102 + { // expression _loop0_103 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_103[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_102")); + D(fprintf(stderr, "%*c> _gather_104[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression _loop0_103")); expr_ty elem; asdl_seq * seq; if ( (elem = expression_rule(p)) // expression && - (seq = _loop0_102_rule(p)) // _loop0_102 + (seq = _loop0_103_rule(p)) // _loop0_103 ) { - D(fprintf(stderr, "%*c+ _gather_103[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_102")); + D(fprintf(stderr, "%*c+ _gather_104[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression _loop0_103")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_103[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_102")); + D(fprintf(stderr, "%*c%s _gather_104[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression _loop0_103")); } _res = NULL; done: @@ -33531,9 +33746,9 @@ _gather_103_rule(Parser *p) return _res; } -// _tmp_104: NEWLINE INDENT +// _tmp_105: NEWLINE INDENT static void * -_tmp_104_rule(Parser *p) +_tmp_105_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33549,7 +33764,7 @@ _tmp_104_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_104[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); + D(fprintf(stderr, "%*c> _tmp_105[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); Token * indent_var; Token * newline_var; if ( @@ -33558,12 +33773,12 @@ _tmp_104_rule(Parser *p) (indent_var = _PyPegen_expect_token(p, INDENT)) // token='INDENT' ) { - D(fprintf(stderr, "%*c+ _tmp_104[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); + D(fprintf(stderr, "%*c+ _tmp_105[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE INDENT")); _res = _PyPegen_dummy_name(p, newline_var, indent_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_104[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_105[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE INDENT")); } _res = NULL; @@ -33572,11 +33787,11 @@ _tmp_104_rule(Parser *p) return _res; } -// _tmp_105: +// _tmp_106: // | (','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs) // | kwargs static void * -_tmp_105_rule(Parser *p) +_tmp_106_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33592,18 +33807,18 @@ _tmp_105_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_105[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); - void *_tmp_166_var; + D(fprintf(stderr, "%*c> _tmp_106[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); + void *_tmp_167_var; if ( - (_tmp_166_var = _tmp_166_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs + (_tmp_167_var = _tmp_167_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_105[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); - _res = _tmp_166_var; + D(fprintf(stderr, "%*c+ _tmp_106[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); + _res = _tmp_167_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_105[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_106[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs)")); } { // kwargs @@ -33611,18 +33826,18 @@ _tmp_105_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_105[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwargs")); + D(fprintf(stderr, "%*c> _tmp_106[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "kwargs")); asdl_seq* kwargs_var; if ( (kwargs_var = kwargs_rule(p)) // kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_105[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwargs")); + D(fprintf(stderr, "%*c+ _tmp_106[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "kwargs")); _res = kwargs_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_105[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_106[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "kwargs")); } _res = NULL; @@ -33631,9 +33846,9 @@ _tmp_105_rule(Parser *p) return _res; } -// _loop0_106: ',' (starred_expression !'=') +// _loop0_107: ',' (starred_expression !'=') static asdl_seq * -_loop0_106_rule(Parser *p) +_loop0_107_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33658,17 +33873,17 @@ _loop0_106_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_106[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression !'=')")); + D(fprintf(stderr, "%*c> _loop0_107[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (starred_expression !'=')")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_167_rule(p)) // starred_expression !'=' + (elem = _tmp_168_rule(p)) // starred_expression !'=' ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -33690,7 +33905,7 @@ _loop0_106_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_106[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_107[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (starred_expression !'=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -33707,9 +33922,9 @@ _loop0_106_rule(Parser *p) return _seq; } -// _gather_107: (starred_expression !'=') _loop0_106 +// _gather_108: (starred_expression !'=') _loop0_107 static asdl_seq * -_gather_107_rule(Parser *p) +_gather_108_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33720,27 +33935,27 @@ _gather_107_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (starred_expression !'=') _loop0_106 + { // (starred_expression !'=') _loop0_107 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_107[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression !'=') _loop0_106")); + D(fprintf(stderr, "%*c> _gather_108[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(starred_expression !'=') _loop0_107")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_167_rule(p)) // starred_expression !'=' + (elem = _tmp_168_rule(p)) // starred_expression !'=' && - (seq = _loop0_106_rule(p)) // _loop0_106 + (seq = _loop0_107_rule(p)) // _loop0_107 ) { - D(fprintf(stderr, "%*c+ _gather_107[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression !'=') _loop0_106")); + D(fprintf(stderr, "%*c+ _gather_108[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(starred_expression !'=') _loop0_107")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_107[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression !'=') _loop0_106")); + D(fprintf(stderr, "%*c%s _gather_108[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(starred_expression !'=') _loop0_107")); } _res = NULL; done: @@ -33748,9 +33963,9 @@ _gather_107_rule(Parser *p) return _res; } -// _tmp_108: args | expression for_if_clauses +// _tmp_109: args | expression for_if_clauses static void * -_tmp_108_rule(Parser *p) +_tmp_109_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33766,18 +33981,18 @@ _tmp_108_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_108[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args")); + D(fprintf(stderr, "%*c> _tmp_109[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args")); expr_ty args_var; if ( (args_var = args_rule(p)) // args ) { - D(fprintf(stderr, "%*c+ _tmp_108[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args")); + D(fprintf(stderr, "%*c+ _tmp_109[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args")); _res = args_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_108[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_109[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "args")); } { // expression for_if_clauses @@ -33785,7 +34000,7 @@ _tmp_108_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_108[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); + D(fprintf(stderr, "%*c> _tmp_109[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); expr_ty expression_var; asdl_comprehension_seq* for_if_clauses_var; if ( @@ -33794,12 +34009,12 @@ _tmp_108_rule(Parser *p) (for_if_clauses_var = for_if_clauses_rule(p)) // for_if_clauses ) { - D(fprintf(stderr, "%*c+ _tmp_108[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); + D(fprintf(stderr, "%*c+ _tmp_109[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression for_if_clauses")); _res = _PyPegen_dummy_name(p, expression_var, for_if_clauses_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_108[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_109[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression for_if_clauses")); } _res = NULL; @@ -33808,9 +34023,9 @@ _tmp_108_rule(Parser *p) return _res; } -// _tmp_109: args ',' +// _tmp_110: args ',' static void * -_tmp_109_rule(Parser *p) +_tmp_110_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33826,7 +34041,7 @@ _tmp_109_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_109[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args ','")); + D(fprintf(stderr, "%*c> _tmp_110[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "args ','")); Token * _literal; expr_ty args_var; if ( @@ -33835,12 +34050,12 @@ _tmp_109_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_109[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args ','")); + D(fprintf(stderr, "%*c+ _tmp_110[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "args ','")); _res = _PyPegen_dummy_name(p, args_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_109[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_110[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "args ','")); } _res = NULL; @@ -33849,9 +34064,9 @@ _tmp_109_rule(Parser *p) return _res; } -// _tmp_110: ',' | ')' +// _tmp_111: ',' | ')' static void * -_tmp_110_rule(Parser *p) +_tmp_111_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33867,18 +34082,18 @@ _tmp_110_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_110[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_111[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_110[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_111[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_110[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_111[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // ')' @@ -33886,18 +34101,18 @@ _tmp_110_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_110[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_111[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_110[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_111[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_110[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_111[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } _res = NULL; @@ -33906,9 +34121,9 @@ _tmp_110_rule(Parser *p) return _res; } -// _tmp_111: 'True' | 'False' | 'None' +// _tmp_112: 'True' | 'False' | 'None' static void * -_tmp_111_rule(Parser *p) +_tmp_112_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -33924,18 +34139,18 @@ _tmp_111_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_111[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c> _tmp_112[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 622)) // token='True' ) { - D(fprintf(stderr, "%*c+ _tmp_111[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c+ _tmp_112[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_111[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_112[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'True'")); } { // 'False' @@ -33943,18 +34158,18 @@ _tmp_111_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_111[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c> _tmp_112[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 624)) // token='False' ) { - D(fprintf(stderr, "%*c+ _tmp_111[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c+ _tmp_112[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_111[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_112[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'False'")); } { // 'None' @@ -33962,18 +34177,18 @@ _tmp_111_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_111[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c> _tmp_112[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 623)) // token='None' ) { - D(fprintf(stderr, "%*c+ _tmp_111[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c+ _tmp_112[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_111[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_112[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'None'")); } _res = NULL; @@ -33982,9 +34197,9 @@ _tmp_111_rule(Parser *p) return _res; } -// _tmp_112: NAME '=' +// _tmp_113: NAME '=' static void * -_tmp_112_rule(Parser *p) +_tmp_113_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34000,7 +34215,7 @@ _tmp_112_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_112[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME '='")); + D(fprintf(stderr, "%*c> _tmp_113[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME '='")); Token * _literal; expr_ty name_var; if ( @@ -34009,12 +34224,12 @@ _tmp_112_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_112[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '='")); + D(fprintf(stderr, "%*c+ _tmp_113[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME '='")); _res = _PyPegen_dummy_name(p, name_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_112[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_113[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME '='")); } _res = NULL; @@ -34023,9 +34238,9 @@ _tmp_112_rule(Parser *p) return _res; } -// _loop1_113: (!STRING expression_without_invalid) +// _loop1_114: (!STRING expression_without_invalid) static asdl_seq * -_loop1_113_rule(Parser *p) +_loop1_114_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34050,13 +34265,13 @@ _loop1_113_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop1_113[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(!STRING expression_without_invalid)")); - void *_tmp_168_var; + D(fprintf(stderr, "%*c> _loop1_114[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(!STRING expression_without_invalid)")); + void *_tmp_169_var; while ( - (_tmp_168_var = _tmp_168_rule(p)) // !STRING expression_without_invalid + (_tmp_169_var = _tmp_169_rule(p)) // !STRING expression_without_invalid ) { - _res = _tmp_168_var; + _res = _tmp_169_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -34073,7 +34288,7 @@ _loop1_113_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop1_113[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop1_114[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(!STRING expression_without_invalid)")); } if (_n == 0 || p->error_indicator) { @@ -34095,9 +34310,9 @@ _loop1_113_rule(Parser *p) return _seq; } -// _tmp_114: NAME STRING | SOFT_KEYWORD +// _tmp_115: NAME STRING | SOFT_KEYWORD static void * -_tmp_114_rule(Parser *p) +_tmp_115_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34113,7 +34328,7 @@ _tmp_114_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_114[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME STRING")); + D(fprintf(stderr, "%*c> _tmp_115[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME STRING")); expr_ty name_var; expr_ty string_var; if ( @@ -34122,12 +34337,12 @@ _tmp_114_rule(Parser *p) (string_var = _PyPegen_string_token(p)) // STRING ) { - D(fprintf(stderr, "%*c+ _tmp_114[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME STRING")); + D(fprintf(stderr, "%*c+ _tmp_115[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME STRING")); _res = _PyPegen_dummy_name(p, name_var, string_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_114[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_115[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME STRING")); } { // SOFT_KEYWORD @@ -34135,18 +34350,18 @@ _tmp_114_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_114[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); + D(fprintf(stderr, "%*c> _tmp_115[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); expr_ty soft_keyword_var; if ( (soft_keyword_var = _PyPegen_soft_keyword_token(p)) // SOFT_KEYWORD ) { - D(fprintf(stderr, "%*c+ _tmp_114[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); + D(fprintf(stderr, "%*c+ _tmp_115[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "SOFT_KEYWORD")); _res = soft_keyword_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_114[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_115[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "SOFT_KEYWORD")); } _res = NULL; @@ -34155,9 +34370,9 @@ _tmp_114_rule(Parser *p) return _res; } -// _tmp_115: 'else' | ':' +// _tmp_116: 'else' | ':' static void * -_tmp_115_rule(Parser *p) +_tmp_116_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34173,18 +34388,18 @@ _tmp_115_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_115[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'else'")); + D(fprintf(stderr, "%*c> _tmp_116[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'else'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 686)) // token='else' ) { - D(fprintf(stderr, "%*c+ _tmp_115[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else'")); + D(fprintf(stderr, "%*c+ _tmp_116[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'else'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_115[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_116[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'else'")); } { // ':' @@ -34192,18 +34407,18 @@ _tmp_115_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_115[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_116[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_115[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_116[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_115[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_116[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -34212,9 +34427,9 @@ _tmp_115_rule(Parser *p) return _res; } -// _tmp_116: pass_stmt | break_stmt | continue_stmt +// _tmp_117: pass_stmt | break_stmt | continue_stmt static void * -_tmp_116_rule(Parser *p) +_tmp_117_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34230,18 +34445,18 @@ _tmp_116_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_116[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "pass_stmt")); + D(fprintf(stderr, "%*c> _tmp_117[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "pass_stmt")); stmt_ty pass_stmt_var; if ( (pass_stmt_var = pass_stmt_rule(p)) // pass_stmt ) { - D(fprintf(stderr, "%*c+ _tmp_116[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "pass_stmt")); + D(fprintf(stderr, "%*c+ _tmp_117[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "pass_stmt")); _res = pass_stmt_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_116[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_117[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "pass_stmt")); } { // break_stmt @@ -34249,18 +34464,18 @@ _tmp_116_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_116[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "break_stmt")); + D(fprintf(stderr, "%*c> _tmp_117[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "break_stmt")); stmt_ty break_stmt_var; if ( (break_stmt_var = break_stmt_rule(p)) // break_stmt ) { - D(fprintf(stderr, "%*c+ _tmp_116[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "break_stmt")); + D(fprintf(stderr, "%*c+ _tmp_117[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "break_stmt")); _res = break_stmt_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_116[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_117[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "break_stmt")); } { // continue_stmt @@ -34268,18 +34483,18 @@ _tmp_116_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_116[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "continue_stmt")); + D(fprintf(stderr, "%*c> _tmp_117[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "continue_stmt")); stmt_ty continue_stmt_var; if ( (continue_stmt_var = continue_stmt_rule(p)) // continue_stmt ) { - D(fprintf(stderr, "%*c+ _tmp_116[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "continue_stmt")); + D(fprintf(stderr, "%*c+ _tmp_117[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "continue_stmt")); _res = continue_stmt_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_116[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_117[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "continue_stmt")); } _res = NULL; @@ -34288,9 +34503,9 @@ _tmp_116_rule(Parser *p) return _res; } -// _tmp_117: '=' | ':=' +// _tmp_118: '=' | ':=' static void * -_tmp_117_rule(Parser *p) +_tmp_118_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34306,18 +34521,18 @@ _tmp_117_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_117[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_117[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_117[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_118[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // ':=' @@ -34325,18 +34540,18 @@ _tmp_117_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_117[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 53)) // token=':=' ) { - D(fprintf(stderr, "%*c+ _tmp_117[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); + D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_117[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_118[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':='")); } _res = NULL; @@ -34345,9 +34560,9 @@ _tmp_117_rule(Parser *p) return _res; } -// _tmp_118: list | tuple | genexp | 'True' | 'None' | 'False' +// _tmp_119: list | tuple | genexp | 'True' | 'None' | 'False' static void * -_tmp_118_rule(Parser *p) +_tmp_119_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34363,18 +34578,18 @@ _tmp_118_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); + D(fprintf(stderr, "%*c> _tmp_119[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "list")); expr_ty list_var; if ( (list_var = list_rule(p)) // list ) { - D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); + D(fprintf(stderr, "%*c+ _tmp_119[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "list")); _res = list_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_118[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_119[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "list")); } { // tuple @@ -34382,18 +34597,18 @@ _tmp_118_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); + D(fprintf(stderr, "%*c> _tmp_119[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tuple")); expr_ty tuple_var; if ( (tuple_var = tuple_rule(p)) // tuple ) { - D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); + D(fprintf(stderr, "%*c+ _tmp_119[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tuple")); _res = tuple_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_118[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_119[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tuple")); } { // genexp @@ -34401,18 +34616,18 @@ _tmp_118_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); + D(fprintf(stderr, "%*c> _tmp_119[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "genexp")); expr_ty genexp_var; if ( (genexp_var = genexp_rule(p)) // genexp ) { - D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); + D(fprintf(stderr, "%*c+ _tmp_119[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "genexp")); _res = genexp_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_118[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_119[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "genexp")); } { // 'True' @@ -34420,18 +34635,18 @@ _tmp_118_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c> _tmp_119[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'True'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 622)) // token='True' ) { - D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); + D(fprintf(stderr, "%*c+ _tmp_119[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'True'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_118[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_119[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'True'")); } { // 'None' @@ -34439,18 +34654,18 @@ _tmp_118_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c> _tmp_119[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'None'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 623)) // token='None' ) { - D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); + D(fprintf(stderr, "%*c+ _tmp_119[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'None'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_118[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_119[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'None'")); } { // 'False' @@ -34458,18 +34673,18 @@ _tmp_118_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_118[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c> _tmp_119[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'False'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 624)) // token='False' ) { - D(fprintf(stderr, "%*c+ _tmp_118[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); + D(fprintf(stderr, "%*c+ _tmp_119[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'False'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_118[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_119[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'False'")); } _res = NULL; @@ -34478,9 +34693,9 @@ _tmp_118_rule(Parser *p) return _res; } -// _loop0_119: star_named_expressions +// _loop0_120: star_named_expressions static asdl_seq * -_loop0_119_rule(Parser *p) +_loop0_120_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34505,7 +34720,7 @@ _loop0_119_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_119[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expressions")); + D(fprintf(stderr, "%*c> _loop0_120[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_named_expressions")); asdl_expr_seq* star_named_expressions_var; while ( (star_named_expressions_var = star_named_expressions_rule(p)) // star_named_expressions @@ -34528,7 +34743,7 @@ _loop0_119_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_119[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_120[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_named_expressions")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -34545,9 +34760,9 @@ _loop0_119_rule(Parser *p) return _seq; } -// _loop0_120: (star_targets '=') +// _loop0_121: (star_targets '=') static asdl_seq * -_loop0_120_rule(Parser *p) +_loop0_121_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34572,13 +34787,13 @@ _loop0_120_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_120[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); - void *_tmp_155_var; + D(fprintf(stderr, "%*c> _loop0_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(star_targets '=')")); + void *_tmp_157_var; while ( - (_tmp_155_var = _tmp_155_rule(p)) // star_targets '=' + (_tmp_157_var = _tmp_157_rule(p)) // star_targets '=' ) { - _res = _tmp_155_var; + _res = _tmp_157_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -34595,7 +34810,7 @@ _loop0_120_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_120[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_121[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(star_targets '=')")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -34612,9 +34827,9 @@ _loop0_120_rule(Parser *p) return _seq; } -// _tmp_121: '[' | '(' | '{' +// _tmp_122: '[' | '(' | '{' static void * -_tmp_121_rule(Parser *p) +_tmp_122_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34630,18 +34845,18 @@ _tmp_121_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c> _tmp_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' ) { - D(fprintf(stderr, "%*c+ _tmp_121[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c+ _tmp_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_121[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_122[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); } { // '(' @@ -34649,18 +34864,18 @@ _tmp_121_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c> _tmp_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'('")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 7)) // token='(' ) { - D(fprintf(stderr, "%*c+ _tmp_121[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); + D(fprintf(stderr, "%*c+ _tmp_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'('")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_121[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_122[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'('")); } { // '{' @@ -34668,18 +34883,18 @@ _tmp_121_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_121[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c> _tmp_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' ) { - D(fprintf(stderr, "%*c+ _tmp_121[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c+ _tmp_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_121[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_122[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); } _res = NULL; @@ -34688,9 +34903,9 @@ _tmp_121_rule(Parser *p) return _res; } -// _tmp_122: '[' | '{' +// _tmp_123: '[' | '{' static void * -_tmp_122_rule(Parser *p) +_tmp_123_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34706,18 +34921,18 @@ _tmp_122_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c> _tmp_123[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'['")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 9)) // token='[' ) { - D(fprintf(stderr, "%*c+ _tmp_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); + D(fprintf(stderr, "%*c+ _tmp_123[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'['")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_122[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_123[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'['")); } { // '{' @@ -34725,18 +34940,18 @@ _tmp_122_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_122[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c> _tmp_123[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'{'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 25)) // token='{' ) { - D(fprintf(stderr, "%*c+ _tmp_122[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); + D(fprintf(stderr, "%*c+ _tmp_123[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'{'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_122[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_123[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'{'")); } _res = NULL; @@ -34745,9 +34960,9 @@ _tmp_122_rule(Parser *p) return _res; } -// _tmp_123: slash_no_default | slash_with_default +// _tmp_124: slash_no_default | slash_with_default static void * -_tmp_123_rule(Parser *p) +_tmp_124_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34763,18 +34978,18 @@ _tmp_123_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_123[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_124[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_no_default")); asdl_arg_seq* slash_no_default_var; if ( (slash_no_default_var = slash_no_default_rule(p)) // slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_123[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_124[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_no_default")); _res = slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_123[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_124[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_no_default")); } { // slash_with_default @@ -34782,18 +34997,18 @@ _tmp_123_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_123[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_124[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slash_with_default")); SlashWithDefault* slash_with_default_var; if ( (slash_with_default_var = slash_with_default_rule(p)) // slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_123[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_124[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slash_with_default")); _res = slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_123[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_124[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slash_with_default")); } _res = NULL; @@ -34802,9 +35017,9 @@ _tmp_123_rule(Parser *p) return _res; } -// _tmp_124: ',' | param_no_default +// _tmp_125: ',' | param_no_default static void * -_tmp_124_rule(Parser *p) +_tmp_125_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34820,18 +35035,18 @@ _tmp_124_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_124[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_125[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_124[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_125[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_124[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_125[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // param_no_default @@ -34839,18 +35054,18 @@ _tmp_124_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_124[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _tmp_125[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; if ( (param_no_default_var = param_no_default_rule(p)) // param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_124[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_125[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); _res = param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_124[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_125[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } _res = NULL; @@ -34859,9 +35074,9 @@ _tmp_124_rule(Parser *p) return _res; } -// _tmp_125: ')' | ',' +// _tmp_126: ')' | ',' static void * -_tmp_125_rule(Parser *p) +_tmp_126_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34877,18 +35092,18 @@ _tmp_125_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_125[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_126[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_125[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_126[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_125[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_126[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // ',' @@ -34896,18 +35111,18 @@ _tmp_125_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_125[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_126[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_125[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_126[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_125[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_126[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -34916,9 +35131,9 @@ _tmp_125_rule(Parser *p) return _res; } -// _tmp_126: ')' | ',' (')' | '**') +// _tmp_127: ')' | ',' (')' | '**') static void * -_tmp_126_rule(Parser *p) +_tmp_127_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34934,18 +35149,18 @@ _tmp_126_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_126[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_127[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_126[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_127[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_126[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_127[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // ',' (')' | '**') @@ -34953,21 +35168,21 @@ _tmp_126_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_126[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); + D(fprintf(stderr, "%*c> _tmp_127[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); Token * _literal; - void *_tmp_169_var; + void *_tmp_170_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_169_var = _tmp_169_rule(p)) // ')' | '**' + (_tmp_170_var = _tmp_170_rule(p)) // ')' | '**' ) { - D(fprintf(stderr, "%*c+ _tmp_126[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_169_var); + D(fprintf(stderr, "%*c+ _tmp_127[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (')' | '**')")); + _res = _PyPegen_dummy_name(p, _literal, _tmp_170_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_126[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_127[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (')' | '**')")); } _res = NULL; @@ -34976,9 +35191,9 @@ _tmp_126_rule(Parser *p) return _res; } -// _tmp_127: param_no_default | ',' +// _tmp_128: param_no_default | ',' static void * -_tmp_127_rule(Parser *p) +_tmp_128_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -34994,18 +35209,18 @@ _tmp_127_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_127[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c> _tmp_128[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "param_no_default")); arg_ty param_no_default_var; if ( (param_no_default_var = param_no_default_rule(p)) // param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_127[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_128[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "param_no_default")); _res = param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_127[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_128[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "param_no_default")); } { // ',' @@ -35013,18 +35228,18 @@ _tmp_127_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_127[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_128[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_127[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_128[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_127[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_128[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -35033,9 +35248,9 @@ _tmp_127_rule(Parser *p) return _res; } -// _tmp_128: '*' | '**' | '/' +// _tmp_129: '*' | '**' | '/' static void * -_tmp_128_rule(Parser *p) +_tmp_129_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35051,18 +35266,18 @@ _tmp_128_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_128[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c> _tmp_129[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' ) { - D(fprintf(stderr, "%*c+ _tmp_128[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c+ _tmp_129[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_128[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_129[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); } { // '**' @@ -35070,18 +35285,18 @@ _tmp_128_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_128[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_129[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_128[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_129[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_128[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_129[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } { // '/' @@ -35089,18 +35304,18 @@ _tmp_128_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_128[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c> _tmp_129[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 17)) // token='/' ) { - D(fprintf(stderr, "%*c+ _tmp_128[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c+ _tmp_129[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_128[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_129[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); } _res = NULL; @@ -35109,9 +35324,9 @@ _tmp_128_rule(Parser *p) return _res; } -// _tmp_129: lambda_slash_no_default | lambda_slash_with_default +// _tmp_130: lambda_slash_no_default | lambda_slash_with_default static void * -_tmp_129_rule(Parser *p) +_tmp_130_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35127,18 +35342,18 @@ _tmp_129_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_129[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c> _tmp_130[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); asdl_arg_seq* lambda_slash_no_default_var; if ( (lambda_slash_no_default_var = lambda_slash_no_default_rule(p)) // lambda_slash_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_129[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); + D(fprintf(stderr, "%*c+ _tmp_130[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_no_default")); _res = lambda_slash_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_129[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_130[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_no_default")); } { // lambda_slash_with_default @@ -35146,18 +35361,18 @@ _tmp_129_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_129[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c> _tmp_130[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); SlashWithDefault* lambda_slash_with_default_var; if ( (lambda_slash_with_default_var = lambda_slash_with_default_rule(p)) // lambda_slash_with_default ) { - D(fprintf(stderr, "%*c+ _tmp_129[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); + D(fprintf(stderr, "%*c+ _tmp_130[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_slash_with_default")); _res = lambda_slash_with_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_129[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_130[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_slash_with_default")); } _res = NULL; @@ -35166,9 +35381,9 @@ _tmp_129_rule(Parser *p) return _res; } -// _loop0_130: ',' lambda_param +// _loop0_131: ',' lambda_param static asdl_seq * -_loop0_130_rule(Parser *p) +_loop0_131_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35193,7 +35408,7 @@ _loop0_130_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_130[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' lambda_param")); + D(fprintf(stderr, "%*c> _loop0_131[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' lambda_param")); Token * _literal; arg_ty elem; while ( @@ -35203,7 +35418,7 @@ _loop0_130_rule(Parser *p) ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -35225,7 +35440,7 @@ _loop0_130_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_130[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_131[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' lambda_param")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35242,9 +35457,9 @@ _loop0_130_rule(Parser *p) return _seq; } -// _gather_131: lambda_param _loop0_130 +// _gather_132: lambda_param _loop0_131 static asdl_seq * -_gather_131_rule(Parser *p) +_gather_132_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35255,27 +35470,27 @@ _gather_131_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // lambda_param _loop0_130 + { // lambda_param _loop0_131 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_131[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_130")); + D(fprintf(stderr, "%*c> _gather_132[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_131")); arg_ty elem; asdl_seq * seq; if ( (elem = lambda_param_rule(p)) // lambda_param && - (seq = _loop0_130_rule(p)) // _loop0_130 + (seq = _loop0_131_rule(p)) // _loop0_131 ) { - D(fprintf(stderr, "%*c+ _gather_131[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_130")); + D(fprintf(stderr, "%*c+ _gather_132[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param _loop0_131")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_131[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param _loop0_130")); + D(fprintf(stderr, "%*c%s _gather_132[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param _loop0_131")); } _res = NULL; done: @@ -35283,9 +35498,9 @@ _gather_131_rule(Parser *p) return _res; } -// _tmp_132: ',' | lambda_param_no_default +// _tmp_133: ',' | lambda_param_no_default static void * -_tmp_132_rule(Parser *p) +_tmp_133_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35301,18 +35516,18 @@ _tmp_132_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_132[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_133[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_132[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_133[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_132[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_133[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // lambda_param_no_default @@ -35320,18 +35535,18 @@ _tmp_132_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_132[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _tmp_133[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; if ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_132[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_133[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); _res = lambda_param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_132[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_133[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } _res = NULL; @@ -35340,9 +35555,9 @@ _tmp_132_rule(Parser *p) return _res; } -// _tmp_133: ':' | ',' (':' | '**') +// _tmp_134: ':' | ',' (':' | '**') static void * -_tmp_133_rule(Parser *p) +_tmp_134_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35358,18 +35573,18 @@ _tmp_133_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_133[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_134[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_133[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_134[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_133[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_134[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // ',' (':' | '**') @@ -35377,21 +35592,21 @@ _tmp_133_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_133[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); + D(fprintf(stderr, "%*c> _tmp_134[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); Token * _literal; - void *_tmp_170_var; + void *_tmp_171_var; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (_tmp_170_var = _tmp_170_rule(p)) // ':' | '**' + (_tmp_171_var = _tmp_171_rule(p)) // ':' | '**' ) { - D(fprintf(stderr, "%*c+ _tmp_133[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); - _res = _PyPegen_dummy_name(p, _literal, _tmp_170_var); + D(fprintf(stderr, "%*c+ _tmp_134[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' (':' | '**')")); + _res = _PyPegen_dummy_name(p, _literal, _tmp_171_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_133[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_134[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (':' | '**')")); } _res = NULL; @@ -35400,9 +35615,9 @@ _tmp_133_rule(Parser *p) return _res; } -// _tmp_134: lambda_param_no_default | ',' +// _tmp_135: lambda_param_no_default | ',' static void * -_tmp_134_rule(Parser *p) +_tmp_135_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35418,18 +35633,18 @@ _tmp_134_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_134[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c> _tmp_135[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); arg_ty lambda_param_no_default_var; if ( (lambda_param_no_default_var = lambda_param_no_default_rule(p)) // lambda_param_no_default ) { - D(fprintf(stderr, "%*c+ _tmp_134[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); + D(fprintf(stderr, "%*c+ _tmp_135[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "lambda_param_no_default")); _res = lambda_param_no_default_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_134[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_135[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "lambda_param_no_default")); } { // ',' @@ -35437,18 +35652,18 @@ _tmp_134_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_134[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_135[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_134[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_135[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_134[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_135[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -35457,9 +35672,9 @@ _tmp_134_rule(Parser *p) return _res; } -// _tmp_135: bitwise_or ((',' bitwise_or))* ','? +// _tmp_136: bitwise_or ((',' bitwise_or))* ','? static void * -_tmp_135_rule(Parser *p) +_tmp_136_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35475,25 +35690,25 @@ _tmp_135_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_135[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); - asdl_seq * _loop0_171_var; + D(fprintf(stderr, "%*c> _tmp_136[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); + asdl_seq * _loop0_172_var; void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty bitwise_or_var; if ( (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or && - (_loop0_171_var = _loop0_171_rule(p)) // ((',' bitwise_or))* + (_loop0_172_var = _loop0_172_rule(p)) // ((',' bitwise_or))* && (_opt_var = _PyPegen_expect_token(p, 12), !p->error_indicator) // ','? ) { - D(fprintf(stderr, "%*c+ _tmp_135[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); - _res = _PyPegen_dummy_name(p, bitwise_or_var, _loop0_171_var, _opt_var); + D(fprintf(stderr, "%*c+ _tmp_136[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); + _res = _PyPegen_dummy_name(p, bitwise_or_var, _loop0_172_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_135[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_136[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "bitwise_or ((',' bitwise_or))* ','?")); } _res = NULL; @@ -35502,9 +35717,9 @@ _tmp_135_rule(Parser *p) return _res; } -// _loop0_136: ',' dotted_name +// _loop0_137: ',' dotted_name static asdl_seq * -_loop0_136_rule(Parser *p) +_loop0_137_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35529,7 +35744,7 @@ _loop0_136_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_136[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_name")); + D(fprintf(stderr, "%*c> _loop0_137[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' dotted_name")); Token * _literal; expr_ty elem; while ( @@ -35539,7 +35754,7 @@ _loop0_136_rule(Parser *p) ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -35561,7 +35776,7 @@ _loop0_136_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_136[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_137[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' dotted_name")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35578,9 +35793,9 @@ _loop0_136_rule(Parser *p) return _seq; } -// _gather_137: dotted_name _loop0_136 +// _gather_138: dotted_name _loop0_137 static asdl_seq * -_gather_137_rule(Parser *p) +_gather_138_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35591,27 +35806,27 @@ _gather_137_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // dotted_name _loop0_136 + { // dotted_name _loop0_137 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_137[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_136")); + D(fprintf(stderr, "%*c> _gather_138[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_137")); expr_ty elem; asdl_seq * seq; if ( (elem = dotted_name_rule(p)) // dotted_name && - (seq = _loop0_136_rule(p)) // _loop0_136 + (seq = _loop0_137_rule(p)) // _loop0_137 ) { - D(fprintf(stderr, "%*c+ _gather_137[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_136")); + D(fprintf(stderr, "%*c+ _gather_138[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "dotted_name _loop0_137")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_137[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_name _loop0_136")); + D(fprintf(stderr, "%*c%s _gather_138[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "dotted_name _loop0_137")); } _res = NULL; done: @@ -35619,9 +35834,9 @@ _gather_137_rule(Parser *p) return _res; } -// _tmp_138: NAME (',' | ')' | NEWLINE) +// _tmp_139: NAME (',' | ')' | ';' | NEWLINE) static void * -_tmp_138_rule(Parser *p) +_tmp_139_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35632,27 +35847,27 @@ _tmp_138_rule(Parser *p) } void * _res = NULL; int _mark = p->mark; - { // NAME (',' | ')' | NEWLINE) + { // NAME (',' | ')' | ';' | NEWLINE) if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_138[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME (',' | ')' | NEWLINE)")); - void *_tmp_172_var; + D(fprintf(stderr, "%*c> _tmp_139[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NAME (',' | ')' | ';' | NEWLINE)")); + void *_tmp_173_var; expr_ty name_var; if ( (name_var = _PyPegen_name_token(p)) // NAME && - (_tmp_172_var = _tmp_172_rule(p)) // ',' | ')' | NEWLINE + (_tmp_173_var = _tmp_173_rule(p)) // ',' | ')' | ';' | NEWLINE ) { - D(fprintf(stderr, "%*c+ _tmp_138[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME (',' | ')' | NEWLINE)")); - _res = _PyPegen_dummy_name(p, name_var, _tmp_172_var); + D(fprintf(stderr, "%*c+ _tmp_139[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NAME (',' | ')' | ';' | NEWLINE)")); + _res = _PyPegen_dummy_name(p, name_var, _tmp_173_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_138[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME (',' | ')' | NEWLINE)")); + D(fprintf(stderr, "%*c%s _tmp_139[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NAME (',' | ')' | ';' | NEWLINE)")); } _res = NULL; done: @@ -35660,9 +35875,9 @@ _tmp_138_rule(Parser *p) return _res; } -// _loop0_139: ',' (expression ['as' star_target]) +// _loop0_140: ',' (expression ['as' star_target]) static asdl_seq * -_loop0_139_rule(Parser *p) +_loop0_140_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35687,17 +35902,17 @@ _loop0_139_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_139[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_140[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expression ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_173_rule(p)) // expression ['as' star_target] + (elem = _tmp_174_rule(p)) // expression ['as' star_target] ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -35719,7 +35934,7 @@ _loop0_139_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_139[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_140[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expression ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35736,9 +35951,9 @@ _loop0_139_rule(Parser *p) return _seq; } -// _gather_140: (expression ['as' star_target]) _loop0_139 +// _gather_141: (expression ['as' star_target]) _loop0_140 static asdl_seq * -_gather_140_rule(Parser *p) +_gather_141_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35749,27 +35964,27 @@ _gather_140_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expression ['as' star_target]) _loop0_139 + { // (expression ['as' star_target]) _loop0_140 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_140[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_139")); + D(fprintf(stderr, "%*c> _gather_141[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_140")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_173_rule(p)) // expression ['as' star_target] + (elem = _tmp_174_rule(p)) // expression ['as' star_target] && - (seq = _loop0_139_rule(p)) // _loop0_139 + (seq = _loop0_140_rule(p)) // _loop0_140 ) { - D(fprintf(stderr, "%*c+ _gather_140[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_139")); + D(fprintf(stderr, "%*c+ _gather_141[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expression ['as' star_target]) _loop0_140")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_140[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_139")); + D(fprintf(stderr, "%*c%s _gather_141[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expression ['as' star_target]) _loop0_140")); } _res = NULL; done: @@ -35777,9 +35992,9 @@ _gather_140_rule(Parser *p) return _res; } -// _loop0_141: ',' (expressions ['as' star_target]) +// _loop0_142: ',' (expressions ['as' star_target]) static asdl_seq * -_loop0_141_rule(Parser *p) +_loop0_142_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35804,17 +36019,17 @@ _loop0_141_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_141[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); + D(fprintf(stderr, "%*c> _loop0_142[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' (expressions ['as' star_target])")); Token * _literal; void *elem; while ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' && - (elem = _tmp_174_rule(p)) // expressions ['as' star_target] + (elem = _tmp_175_rule(p)) // expressions ['as' star_target] ) { _res = elem; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; PyMem_Free(_children); p->level--; @@ -35836,7 +36051,7 @@ _loop0_141_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_141[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_142[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' (expressions ['as' star_target])")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -35853,9 +36068,9 @@ _loop0_141_rule(Parser *p) return _seq; } -// _gather_142: (expressions ['as' star_target]) _loop0_141 +// _gather_143: (expressions ['as' star_target]) _loop0_142 static asdl_seq * -_gather_142_rule(Parser *p) +_gather_143_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35866,27 +36081,27 @@ _gather_142_rule(Parser *p) } asdl_seq * _res = NULL; int _mark = p->mark; - { // (expressions ['as' star_target]) _loop0_141 + { // (expressions ['as' star_target]) _loop0_142 if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _gather_142[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_141")); + D(fprintf(stderr, "%*c> _gather_143[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_142")); void *elem; asdl_seq * seq; if ( - (elem = _tmp_174_rule(p)) // expressions ['as' star_target] + (elem = _tmp_175_rule(p)) // expressions ['as' star_target] && - (seq = _loop0_141_rule(p)) // _loop0_141 + (seq = _loop0_142_rule(p)) // _loop0_142 ) { - D(fprintf(stderr, "%*c+ _gather_142[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_141")); + D(fprintf(stderr, "%*c+ _gather_143[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(expressions ['as' star_target]) _loop0_142")); _res = _PyPegen_seq_insert_in_front(p, elem, seq); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _gather_142[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_141")); + D(fprintf(stderr, "%*c%s _gather_143[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(expressions ['as' star_target]) _loop0_142")); } _res = NULL; done: @@ -35894,9 +36109,9 @@ _gather_142_rule(Parser *p) return _res; } -// _tmp_143: 'except' | 'finally' +// _tmp_144: 'except' | 'finally' static void * -_tmp_143_rule(Parser *p) +_tmp_144_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35912,18 +36127,18 @@ _tmp_143_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_143[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); + D(fprintf(stderr, "%*c> _tmp_144[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'except'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 677)) // token='except' ) { - D(fprintf(stderr, "%*c+ _tmp_143[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); + D(fprintf(stderr, "%*c+ _tmp_144[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'except'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_143[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_144[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'except'")); } { // 'finally' @@ -35931,18 +36146,18 @@ _tmp_143_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_143[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); + D(fprintf(stderr, "%*c> _tmp_144[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'finally'")); Token * _keyword; if ( (_keyword = _PyPegen_expect_token(p, 673)) // token='finally' ) { - D(fprintf(stderr, "%*c+ _tmp_143[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); + D(fprintf(stderr, "%*c+ _tmp_144[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'finally'")); _res = _keyword; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_143[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_144[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'finally'")); } _res = NULL; @@ -35951,9 +36166,9 @@ _tmp_143_rule(Parser *p) return _res; } -// _loop0_144: block +// _loop0_145: block static asdl_seq * -_loop0_144_rule(Parser *p) +_loop0_145_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -35978,7 +36193,7 @@ _loop0_144_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_144[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); + D(fprintf(stderr, "%*c> _loop0_145[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "block")); asdl_stmt_seq* block_var; while ( (block_var = block_rule(p)) // block @@ -36001,7 +36216,7 @@ _loop0_144_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_144[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_145[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "block")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -36018,9 +36233,9 @@ _loop0_144_rule(Parser *p) return _seq; } -// _tmp_145: expression ['as' NAME] +// _tmp_146: expression ['as' NAME] static void * -_tmp_145_rule(Parser *p) +_tmp_146_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36036,7 +36251,7 @@ _tmp_145_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_145[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); + D(fprintf(stderr, "%*c> _tmp_146[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; @@ -36046,12 +36261,12 @@ _tmp_145_rule(Parser *p) (_opt_var = _tmp_22_rule(p), !p->error_indicator) // ['as' NAME] ) { - D(fprintf(stderr, "%*c+ _tmp_145[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); + D(fprintf(stderr, "%*c+ _tmp_146[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' NAME]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_145[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_146[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' NAME]")); } _res = NULL; @@ -36060,9 +36275,9 @@ _tmp_145_rule(Parser *p) return _res; } -// _tmp_146: NEWLINE | ':' +// _tmp_147: NEWLINE | ':' static void * -_tmp_146_rule(Parser *p) +_tmp_147_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36078,18 +36293,18 @@ _tmp_146_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_146[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_147[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); Token * newline_var; if ( (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_146[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_147[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); _res = newline_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_146[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_147[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE")); } { // ':' @@ -36097,18 +36312,18 @@ _tmp_146_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_146[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_147[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_146[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_147[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_146[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_147[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } _res = NULL; @@ -36117,9 +36332,9 @@ _tmp_146_rule(Parser *p) return _res; } -// _tmp_147: positional_patterns ',' +// _tmp_148: positional_patterns ',' static void * -_tmp_147_rule(Parser *p) +_tmp_148_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36135,7 +36350,7 @@ _tmp_147_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_147[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c> _tmp_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); Token * _literal; asdl_pattern_seq* positional_patterns_var; if ( @@ -36144,12 +36359,12 @@ _tmp_147_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_147[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); + D(fprintf(stderr, "%*c+ _tmp_148[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "positional_patterns ','")); _res = _PyPegen_dummy_name(p, positional_patterns_var, _literal); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_147[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_148[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "positional_patterns ','")); } _res = NULL; @@ -36158,9 +36373,9 @@ _tmp_147_rule(Parser *p) return _res; } -// _tmp_148: '}' | ',' +// _tmp_149: '}' | ',' static void * -_tmp_148_rule(Parser *p) +_tmp_149_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36176,18 +36391,18 @@ _tmp_148_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_148[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_148[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } { // ',' @@ -36195,18 +36410,18 @@ _tmp_148_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_148[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_148[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_148[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } _res = NULL; @@ -36215,9 +36430,9 @@ _tmp_148_rule(Parser *p) return _res; } -// _tmp_149: '=' | '!' | ':' | '}' +// _tmp_150: '=' | '!' | ':' | '}' static void * -_tmp_149_rule(Parser *p) +_tmp_150_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36233,18 +36448,18 @@ _tmp_149_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'='")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); + D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'='")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'='")); } { // '!' @@ -36252,18 +36467,18 @@ _tmp_149_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' ) { - D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); } { // ':' @@ -36271,18 +36486,18 @@ _tmp_149_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -36290,18 +36505,18 @@ _tmp_149_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_149[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_149[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_149[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -36310,9 +36525,9 @@ _tmp_149_rule(Parser *p) return _res; } -// _tmp_150: '!' | ':' | '}' +// _tmp_151: '!' | ':' | '}' static void * -_tmp_150_rule(Parser *p) +_tmp_151_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36328,18 +36543,18 @@ _tmp_150_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c> _tmp_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 54)) // token='!' ) { - D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); + D(fprintf(stderr, "%*c+ _tmp_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_151[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!'")); } { // ':' @@ -36347,18 +36562,18 @@ _tmp_150_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_151[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -36366,18 +36581,18 @@ _tmp_150_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_150[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_150[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_150[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_151[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -36386,9 +36601,9 @@ _tmp_150_rule(Parser *p) return _res; } -// _tmp_151: '!' NAME +// _tmp_152: '!' NAME static void * -_tmp_151_rule(Parser *p) +_tmp_152_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36404,7 +36619,7 @@ _tmp_151_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_151[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'!' NAME")); Token * _literal; expr_ty name_var; if ( @@ -36413,12 +36628,12 @@ _tmp_151_rule(Parser *p) (name_var = _PyPegen_name_token(p)) // NAME ) { - D(fprintf(stderr, "%*c+ _tmp_151[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); + D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'!' NAME")); _res = _PyPegen_dummy_name(p, _literal, name_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_151[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'!' NAME")); } _res = NULL; @@ -36427,9 +36642,9 @@ _tmp_151_rule(Parser *p) return _res; } -// _tmp_152: ':' | '}' +// _tmp_153: ':' | '}' static void * -_tmp_152_rule(Parser *p) +_tmp_153_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36445,18 +36660,18 @@ _tmp_152_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '}' @@ -36464,18 +36679,18 @@ _tmp_152_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_152[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'}'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 26)) // token='}' ) { - D(fprintf(stderr, "%*c+ _tmp_152[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); + D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'}'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_152[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'}'")); } _res = NULL; @@ -36484,9 +36699,66 @@ _tmp_152_rule(Parser *p) return _res; } -// _tmp_153: '+' | '-' | '*' | '/' | '%' | '//' | '@' +// _tmp_154: fstring | string static void * -_tmp_153_rule(Parser *p) +_tmp_154_rule(Parser *p) +{ + if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { + _Pypegen_stack_overflow(p); + } + if (p->error_indicator) { + p->level--; + return NULL; + } + void * _res = NULL; + int _mark = p->mark; + { // fstring + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); + expr_ty fstring_var; + if ( + (fstring_var = fstring_rule(p)) // fstring + ) + { + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); + _res = fstring_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring")); + } + { // string + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); + expr_ty string_var; + if ( + (string_var = string_rule(p)) // string + ) + { + D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); + _res = string_var; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "string")); + } + _res = NULL; + done: + p->level--; + return _res; +} + +// _tmp_155: '+' | '-' | '*' | '/' | '%' | '//' | '@' +static void * +_tmp_155_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36502,18 +36774,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 14)) // token='+' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); } { // '-' @@ -36521,18 +36793,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 15)) // token='-' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); } { // '*' @@ -36540,18 +36812,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'*'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 16)) // token='*' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'*'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'*'")); } { // '/' @@ -36559,18 +36831,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'/'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 17)) // token='/' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'/'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'/'")); } { // '%' @@ -36578,18 +36850,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'%'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'%'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 24)) // token='%' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'%'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'%'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'%'")); } { // '//' @@ -36597,18 +36869,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'//'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'//'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 47)) // token='//' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'//'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'//'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'//'")); } { // '@' @@ -36616,18 +36888,18 @@ _tmp_153_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_153[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@'")); + D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 49)) // token='@' ) { - D(fprintf(stderr, "%*c+ _tmp_153[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@'")); + D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_153[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@'")); } _res = NULL; @@ -36636,9 +36908,9 @@ _tmp_153_rule(Parser *p) return _res; } -// _tmp_154: '+' | '-' | '~' +// _tmp_156: '+' | '-' | '~' static void * -_tmp_154_rule(Parser *p) +_tmp_156_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36654,18 +36926,18 @@ _tmp_154_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'+'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 14)) // token='+' ) { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); + D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'+'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'+'")); } { // '-' @@ -36673,18 +36945,18 @@ _tmp_154_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'-'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 15)) // token='-' ) { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); + D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'-'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'-'")); } { // '~' @@ -36692,18 +36964,18 @@ _tmp_154_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_154[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'~'")); + D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'~'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 31)) // token='~' ) { - D(fprintf(stderr, "%*c+ _tmp_154[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'~'")); + D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'~'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_154[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'~'")); } _res = NULL; @@ -36712,9 +36984,9 @@ _tmp_154_rule(Parser *p) return _res; } -// _tmp_155: star_targets '=' +// _tmp_157: star_targets '=' static void * -_tmp_155_rule(Parser *p) +_tmp_157_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36730,7 +37002,7 @@ _tmp_155_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_155[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "star_targets '='")); Token * _literal; expr_ty z; if ( @@ -36739,9 +37011,9 @@ _tmp_155_rule(Parser *p) (_literal = _PyPegen_expect_token(p, 22)) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_155[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); + D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "star_targets '='")); _res = z; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -36749,7 +37021,7 @@ _tmp_155_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_155[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "star_targets '='")); } _res = NULL; @@ -36758,9 +37030,9 @@ _tmp_155_rule(Parser *p) return _res; } -// _tmp_156: '.' | '...' +// _tmp_158: '.' | '...' static void * -_tmp_156_rule(Parser *p) +_tmp_158_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36776,18 +37048,18 @@ _tmp_156_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'.'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 23)) // token='.' ) { - D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'.'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'.'")); } { // '...' @@ -36795,18 +37067,18 @@ _tmp_156_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_156[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'...'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 52)) // token='...' ) { - D(fprintf(stderr, "%*c+ _tmp_156[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); + D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'...'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_156[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'...'")); } _res = NULL; @@ -36815,9 +37087,9 @@ _tmp_156_rule(Parser *p) return _res; } -// _tmp_157: '@' named_expression NEWLINE +// _tmp_159: '@' named_expression NEWLINE static void * -_tmp_157_rule(Parser *p) +_tmp_159_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36833,7 +37105,7 @@ _tmp_157_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_157[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); Token * _literal; expr_ty f; Token * newline_var; @@ -36845,9 +37117,9 @@ _tmp_157_rule(Parser *p) (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_157[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'@' named_expression NEWLINE")); _res = f; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -36855,7 +37127,7 @@ _tmp_157_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_157[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'@' named_expression NEWLINE")); } _res = NULL; @@ -36864,9 +37136,9 @@ _tmp_157_rule(Parser *p) return _res; } -// _tmp_158: ',' star_expression +// _tmp_160: ',' star_expression static void * -_tmp_158_rule(Parser *p) +_tmp_160_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36882,7 +37154,7 @@ _tmp_158_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_158[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_expression")); Token * _literal; expr_ty c; if ( @@ -36891,9 +37163,9 @@ _tmp_158_rule(Parser *p) (c = star_expression_rule(p)) // star_expression ) { - D(fprintf(stderr, "%*c+ _tmp_158[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); + D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_expression")); _res = c; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -36901,7 +37173,7 @@ _tmp_158_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_158[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_expression")); } _res = NULL; @@ -36910,9 +37182,9 @@ _tmp_158_rule(Parser *p) return _res; } -// _tmp_159: 'or' conjunction +// _tmp_161: 'or' conjunction static void * -_tmp_159_rule(Parser *p) +_tmp_161_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36928,7 +37200,7 @@ _tmp_159_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_159[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); Token * _keyword; expr_ty c; if ( @@ -36937,9 +37209,9 @@ _tmp_159_rule(Parser *p) (c = conjunction_rule(p)) // conjunction ) { - D(fprintf(stderr, "%*c+ _tmp_159[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); + D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'or' conjunction")); _res = c; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -36947,7 +37219,7 @@ _tmp_159_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_159[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'or' conjunction")); } _res = NULL; @@ -36956,9 +37228,9 @@ _tmp_159_rule(Parser *p) return _res; } -// _tmp_160: 'and' inversion +// _tmp_162: 'and' inversion static void * -_tmp_160_rule(Parser *p) +_tmp_162_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -36974,7 +37246,7 @@ _tmp_160_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_160[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'and' inversion")); Token * _keyword; expr_ty c; if ( @@ -36983,9 +37255,9 @@ _tmp_160_rule(Parser *p) (c = inversion_rule(p)) // inversion ) { - D(fprintf(stderr, "%*c+ _tmp_160[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); + D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'and' inversion")); _res = c; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -36993,7 +37265,7 @@ _tmp_160_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_160[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'and' inversion")); } _res = NULL; @@ -37002,9 +37274,9 @@ _tmp_160_rule(Parser *p) return _res; } -// _tmp_161: slice | starred_expression +// _tmp_163: slice | starred_expression static void * -_tmp_161_rule(Parser *p) +_tmp_163_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37020,18 +37292,18 @@ _tmp_161_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c> _tmp_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "slice")); expr_ty slice_var; if ( (slice_var = slice_rule(p)) // slice ) { - D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); + D(fprintf(stderr, "%*c+ _tmp_163[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "slice")); _res = slice_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_163[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "slice")); } { // starred_expression @@ -37039,18 +37311,18 @@ _tmp_161_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_161[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_161[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_163[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_161[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_163[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } _res = NULL; @@ -37059,85 +37331,9 @@ _tmp_161_rule(Parser *p) return _res; } -// _tmp_162: fstring | string | tstring -static void * -_tmp_162_rule(Parser *p) -{ - if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { - _Pypegen_stack_overflow(p); - } - if (p->error_indicator) { - p->level--; - return NULL; - } - void * _res = NULL; - int _mark = p->mark; - { // fstring - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "fstring")); - expr_ty fstring_var; - if ( - (fstring_var = fstring_rule(p)) // fstring - ) - { - D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "fstring")); - _res = fstring_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "fstring")); - } - { // string - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "string")); - expr_ty string_var; - if ( - (string_var = string_rule(p)) // string - ) - { - D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "string")); - _res = string_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "string")); - } - { // tstring - if (p->error_indicator) { - p->level--; - return NULL; - } - D(fprintf(stderr, "%*c> _tmp_162[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "tstring")); - expr_ty tstring_var; - if ( - (tstring_var = tstring_rule(p)) // tstring - ) - { - D(fprintf(stderr, "%*c+ _tmp_162[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "tstring")); - _res = tstring_var; - goto done; - } - p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_162[%d-%d]: %s failed!\n", p->level, ' ', - p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "tstring")); - } - _res = NULL; - done: - p->level--; - return _res; -} - -// _tmp_163: 'if' disjunction +// _tmp_164: 'if' disjunction static void * -_tmp_163_rule(Parser *p) +_tmp_164_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37153,7 +37349,7 @@ _tmp_163_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_163[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); Token * _keyword; expr_ty z; if ( @@ -37162,9 +37358,9 @@ _tmp_163_rule(Parser *p) (z = disjunction_rule(p)) // disjunction ) { - D(fprintf(stderr, "%*c+ _tmp_163[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); + D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'if' disjunction")); _res = z; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -37172,7 +37368,7 @@ _tmp_163_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_163[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'if' disjunction")); } _res = NULL; @@ -37181,9 +37377,9 @@ _tmp_163_rule(Parser *p) return _res; } -// _tmp_164: starred_expression | (assignment_expression | expression !':=') !'=' +// _tmp_165: starred_expression | (assignment_expression | expression !':=') !'=' static void * -_tmp_164_rule(Parser *p) +_tmp_165_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37199,18 +37395,18 @@ _tmp_164_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression ) { - D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); + D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression")); } { // (assignment_expression | expression !':=') !'=' @@ -37218,20 +37414,20 @@ _tmp_164_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_164[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - void *_tmp_87_var; + D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + void *_tmp_88_var; if ( - (_tmp_87_var = _tmp_87_rule(p)) // assignment_expression | expression !':=' + (_tmp_88_var = _tmp_88_rule(p)) // assignment_expression | expression !':=' && _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_164[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); - _res = _tmp_87_var; + D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "(assignment_expression | expression !':=') !'='")); + _res = _tmp_88_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_164[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(assignment_expression | expression !':=') !'='")); } _res = NULL; @@ -37240,9 +37436,9 @@ _tmp_164_rule(Parser *p) return _res; } -// _tmp_165: ',' star_target +// _tmp_166: ',' star_target static void * -_tmp_165_rule(Parser *p) +_tmp_166_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37258,7 +37454,7 @@ _tmp_165_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_165[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' star_target")); Token * _literal; expr_ty c; if ( @@ -37267,9 +37463,9 @@ _tmp_165_rule(Parser *p) (c = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_165[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); + D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' star_target")); _res = c; - if (_res == NULL && PyErr_Occurred()) { + if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) { p->error_indicator = 1; p->level--; return NULL; @@ -37277,7 +37473,7 @@ _tmp_165_rule(Parser *p) goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_165[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' star_target")); } _res = NULL; @@ -37286,10 +37482,10 @@ _tmp_165_rule(Parser *p) return _res; } -// _tmp_166: +// _tmp_167: // | ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs static void * -_tmp_166_rule(Parser *p) +_tmp_167_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37305,24 +37501,24 @@ _tmp_166_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_166[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); - asdl_seq * _gather_89_var; + D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); + asdl_seq * _gather_90_var; Token * _literal; asdl_seq* kwargs_var; if ( - (_gather_89_var = _gather_89_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ + (_gather_90_var = _gather_90_rule(p)) // ','.(starred_expression | (assignment_expression | expression !':=') !'=')+ && (_literal = _PyPegen_expect_token(p, 12)) // token=',' && (kwargs_var = kwargs_rule(p)) // kwargs ) { - D(fprintf(stderr, "%*c+ _tmp_166[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); - _res = _PyPegen_dummy_name(p, _gather_89_var, _literal, kwargs_var); + D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); + _res = _PyPegen_dummy_name(p, _gather_90_var, _literal, kwargs_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_166[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','.(starred_expression | (assignment_expression | expression !':=') !'=')+ ',' kwargs")); } _res = NULL; @@ -37331,9 +37527,9 @@ _tmp_166_rule(Parser *p) return _res; } -// _tmp_167: starred_expression !'=' +// _tmp_168: starred_expression !'=' static void * -_tmp_167_rule(Parser *p) +_tmp_168_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37349,7 +37545,7 @@ _tmp_167_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_167[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); + D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); expr_ty starred_expression_var; if ( (starred_expression_var = starred_expression_rule(p)) // starred_expression @@ -37357,12 +37553,12 @@ _tmp_167_rule(Parser *p) _PyPegen_lookahead_with_int(0, _PyPegen_expect_token, p, 22) // token='=' ) { - D(fprintf(stderr, "%*c+ _tmp_167[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); + D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "starred_expression !'='")); _res = starred_expression_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_167[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "starred_expression !'='")); } _res = NULL; @@ -37371,9 +37567,9 @@ _tmp_167_rule(Parser *p) return _res; } -// _tmp_168: !STRING expression_without_invalid +// _tmp_169: !STRING expression_without_invalid static void * -_tmp_168_rule(Parser *p) +_tmp_169_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37389,20 +37585,20 @@ _tmp_168_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_168[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "!STRING expression_without_invalid")); + D(fprintf(stderr, "%*c> _tmp_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "!STRING expression_without_invalid")); expr_ty expression_without_invalid_var; if ( - _PyPegen_lookahead(0, (void *(*)(Parser *)) _PyPegen_string_token, p) + _PyPegen_lookahead(0, _PyPegen_string_token, p) && (expression_without_invalid_var = expression_without_invalid_rule(p)) // expression_without_invalid ) { - D(fprintf(stderr, "%*c+ _tmp_168[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!STRING expression_without_invalid")); + D(fprintf(stderr, "%*c+ _tmp_169[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "!STRING expression_without_invalid")); _res = expression_without_invalid_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_168[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_169[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "!STRING expression_without_invalid")); } _res = NULL; @@ -37411,9 +37607,9 @@ _tmp_168_rule(Parser *p) return _res; } -// _tmp_169: ')' | '**' +// _tmp_170: ')' | '**' static void * -_tmp_169_rule(Parser *p) +_tmp_170_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37429,18 +37625,18 @@ _tmp_169_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_169[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_170[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_169[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_170[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } { // '**' @@ -37448,18 +37644,18 @@ _tmp_169_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_169[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_169[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_170[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_169[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_170[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -37468,9 +37664,9 @@ _tmp_169_rule(Parser *p) return _res; } -// _tmp_170: ':' | '**' +// _tmp_171: ':' | '**' static void * -_tmp_170_rule(Parser *p) +_tmp_171_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37486,18 +37682,18 @@ _tmp_170_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c> _tmp_171[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "':'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 11)) // token=':' ) { - D(fprintf(stderr, "%*c+ _tmp_170[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); + D(fprintf(stderr, "%*c+ _tmp_171[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "':'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_170[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_171[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "':'")); } { // '**' @@ -37505,18 +37701,18 @@ _tmp_170_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_170[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c> _tmp_171[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'**'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 35)) // token='**' ) { - D(fprintf(stderr, "%*c+ _tmp_170[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); + D(fprintf(stderr, "%*c+ _tmp_171[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'**'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_170[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_171[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'**'")); } _res = NULL; @@ -37525,9 +37721,9 @@ _tmp_170_rule(Parser *p) return _res; } -// _loop0_171: (',' bitwise_or) +// _loop0_172: (',' bitwise_or) static asdl_seq * -_loop0_171_rule(Parser *p) +_loop0_172_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37552,13 +37748,13 @@ _loop0_171_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _loop0_171[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' bitwise_or)")); - void *_tmp_175_var; + D(fprintf(stderr, "%*c> _loop0_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "(',' bitwise_or)")); + void *_tmp_176_var; while ( - (_tmp_175_var = _tmp_175_rule(p)) // ',' bitwise_or + (_tmp_176_var = _tmp_176_rule(p)) // ',' bitwise_or ) { - _res = _tmp_175_var; + _res = _tmp_176_var; if (_n == _children_capacity) { _children_capacity *= 2; void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *)); @@ -37575,7 +37771,7 @@ _loop0_171_rule(Parser *p) _mark = p->mark; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _loop0_171[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _loop0_172[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "(',' bitwise_or)")); } asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena); @@ -37592,9 +37788,9 @@ _loop0_171_rule(Parser *p) return _seq; } -// _tmp_172: ',' | ')' | NEWLINE +// _tmp_173: ',' | ')' | ';' | NEWLINE static void * -_tmp_172_rule(Parser *p) +_tmp_173_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37610,18 +37806,18 @@ _tmp_172_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "','")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 12)) // token=',' ) { - D(fprintf(stderr, "%*c+ _tmp_172[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); + D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "','")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_172[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "','")); } { // ')' @@ -37629,37 +37825,56 @@ _tmp_172_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "')'")); Token * _literal; if ( (_literal = _PyPegen_expect_token(p, 8)) // token=')' ) { - D(fprintf(stderr, "%*c+ _tmp_172[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); + D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "')'")); _res = _literal; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_172[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "')'")); } + { // ';' + if (p->error_indicator) { + p->level--; + return NULL; + } + D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "';'")); + Token * _literal; + if ( + (_literal = _PyPegen_expect_token(p, 13)) // token=';' + ) + { + D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "';'")); + _res = _literal; + goto done; + } + p->mark = _mark; + D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', + p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "';'")); + } { // NEWLINE if (p->error_indicator) { p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_172[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "NEWLINE")); Token * newline_var; if ( (newline_var = _PyPegen_expect_token(p, NEWLINE)) // token='NEWLINE' ) { - D(fprintf(stderr, "%*c+ _tmp_172[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); + D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "NEWLINE")); _res = newline_var; goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_172[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "NEWLINE")); } _res = NULL; @@ -37668,9 +37883,9 @@ _tmp_172_rule(Parser *p) return _res; } -// _tmp_173: expression ['as' star_target] +// _tmp_174: expression ['as' star_target] static void * -_tmp_173_rule(Parser *p) +_tmp_174_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37686,22 +37901,22 @@ _tmp_173_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_173[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expression_var; if ( (expression_var = expression_rule(p)) // expression && - (_opt_var = _tmp_176_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_177_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_173[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_174[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expression ['as' star_target]")); _res = _PyPegen_dummy_name(p, expression_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_173[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_174[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expression ['as' star_target]")); } _res = NULL; @@ -37710,9 +37925,9 @@ _tmp_173_rule(Parser *p) return _res; } -// _tmp_174: expressions ['as' star_target] +// _tmp_175: expressions ['as' star_target] static void * -_tmp_174_rule(Parser *p) +_tmp_175_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37728,22 +37943,22 @@ _tmp_174_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_174[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c> _tmp_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); void *_opt_var; UNUSED(_opt_var); // Silence compiler warnings expr_ty expressions_var; if ( (expressions_var = expressions_rule(p)) // expressions && - (_opt_var = _tmp_176_rule(p), !p->error_indicator) // ['as' star_target] + (_opt_var = _tmp_177_rule(p), !p->error_indicator) // ['as' star_target] ) { - D(fprintf(stderr, "%*c+ _tmp_174[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); + D(fprintf(stderr, "%*c+ _tmp_175[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "expressions ['as' star_target]")); _res = _PyPegen_dummy_name(p, expressions_var, _opt_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_174[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_175[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "expressions ['as' star_target]")); } _res = NULL; @@ -37752,9 +37967,9 @@ _tmp_174_rule(Parser *p) return _res; } -// _tmp_175: ',' bitwise_or +// _tmp_176: ',' bitwise_or static void * -_tmp_175_rule(Parser *p) +_tmp_176_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37770,7 +37985,7 @@ _tmp_175_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_175[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); + D(fprintf(stderr, "%*c> _tmp_176[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); Token * _literal; expr_ty bitwise_or_var; if ( @@ -37779,12 +37994,12 @@ _tmp_175_rule(Parser *p) (bitwise_or_var = bitwise_or_rule(p)) // bitwise_or ) { - D(fprintf(stderr, "%*c+ _tmp_175[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); + D(fprintf(stderr, "%*c+ _tmp_176[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "',' bitwise_or")); _res = _PyPegen_dummy_name(p, _literal, bitwise_or_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_175[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_176[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "',' bitwise_or")); } _res = NULL; @@ -37793,9 +38008,9 @@ _tmp_175_rule(Parser *p) return _res; } -// _tmp_176: 'as' star_target +// _tmp_177: 'as' star_target static void * -_tmp_176_rule(Parser *p) +_tmp_177_rule(Parser *p) { if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) { _Pypegen_stack_overflow(p); @@ -37811,7 +38026,7 @@ _tmp_176_rule(Parser *p) p->level--; return NULL; } - D(fprintf(stderr, "%*c> _tmp_176[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c> _tmp_177[%d-%d]: %s\n", p->level, ' ', _mark, p->mark, "'as' star_target")); Token * _keyword; expr_ty star_target_var; if ( @@ -37820,12 +38035,12 @@ _tmp_176_rule(Parser *p) (star_target_var = star_target_rule(p)) // star_target ) { - D(fprintf(stderr, "%*c+ _tmp_176[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); + D(fprintf(stderr, "%*c+ _tmp_177[%d-%d]: %s succeeded!\n", p->level, ' ', _mark, p->mark, "'as' star_target")); _res = _PyPegen_dummy_name(p, _keyword, star_target_var); goto done; } p->mark = _mark; - D(fprintf(stderr, "%*c%s _tmp_176[%d-%d]: %s failed!\n", p->level, ' ', + D(fprintf(stderr, "%*c%s _tmp_177[%d-%d]: %s failed!\n", p->level, ' ', p->error_indicator ? "ERROR!" : "-", _mark, p->mark, "'as' star_target")); } _res = NULL; diff --git a/Parser/pegen.c b/Parser/pegen.c index 3efeba78450d1af..608adf3694117ab 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -9,6 +9,7 @@ #include "lexer/lexer.h" #include "tokenizer/tokenizer.h" +#include "tokenizer/helpers.h" #include "pegen.h" // Internal parser functions @@ -379,44 +380,34 @@ _PyPegen_is_memoized(Parser *p, int type, void *pres) return 0; } -int -_PyPegen_lookahead_with_name(int positive, expr_ty (func)(Parser *), Parser *p) -{ - int mark = p->mark; - void *res = func(p); - p->mark = mark; - return (res != NULL) == positive; -} +#define LOOKAHEAD1(NAME, RES_TYPE) \ + int \ + NAME (int positive, RES_TYPE (func)(Parser *), Parser *p) \ + { \ + int mark = p->mark; \ + void *res = func(p); \ + p->mark = mark; \ + return (res != NULL) == positive; \ + } -int -_PyPegen_lookahead_with_string(int positive, expr_ty (func)(Parser *, const char*), Parser *p, const char* arg) -{ - int mark = p->mark; - void *res = func(p, arg); - p->mark = mark; - return (res != NULL) == positive; -} +LOOKAHEAD1(_PyPegen_lookahead, void *) +LOOKAHEAD1(_PyPegen_lookahead_for_expr, expr_ty) +LOOKAHEAD1(_PyPegen_lookahead_for_stmt, stmt_ty) +#undef LOOKAHEAD1 -int -_PyPegen_lookahead_with_int(int positive, Token *(func)(Parser *, int), Parser *p, int arg) -{ - int mark = p->mark; - void *res = func(p, arg); - p->mark = mark; - return (res != NULL) == positive; -} +#define LOOKAHEAD2(NAME, RES_TYPE, T) \ + int \ + NAME (int positive, RES_TYPE (func)(Parser *, T), Parser *p, T arg) \ + { \ + int mark = p->mark; \ + void *res = func(p, arg); \ + p->mark = mark; \ + return (res != NULL) == positive; \ + } -// gh-111178: Use _Py_NO_SANITIZE_UNDEFINED to disable sanitizer checks on -// undefined behavior (UBsan) in this function, rather than changing 'func' -// callback API. -int _Py_NO_SANITIZE_UNDEFINED -_PyPegen_lookahead(int positive, void *(func)(Parser *), Parser *p) -{ - int mark = p->mark; - void *res = func(p); - p->mark = mark; - return (res != NULL) == positive; -} +LOOKAHEAD2(_PyPegen_lookahead_with_int, Token *, int) +LOOKAHEAD2(_PyPegen_lookahead_with_string, expr_ty, const char *) +#undef LOOKAHEAD2 Token * _PyPegen_expect_token(Parser *p, int type) @@ -549,6 +540,21 @@ _PyPegen_new_identifier(Parser *p, const char *n) } id = id2; } + static const char * const forbidden[] = { + "None", + "True", + "False", + NULL + }; + for (int i = 0; forbidden[i] != NULL; i++) { + if (_PyUnicode_EqualToASCIIString(id, forbidden[i])) { + PyErr_Format(PyExc_ValueError, + "identifier field can't represent '%s' constant", + forbidden[i]); + Py_DECREF(id); + goto error; + } + } PyInterpreterState *interp = _PyInterpreterState_GET(); _PyUnicode_InternImmortal(interp, &id); if (_PyArena_AddPyObject(p->arena, id) < 0) @@ -605,7 +611,8 @@ expr_ty _PyPegen_soft_keyword_token(Parser *p) { Py_ssize_t size; PyBytes_AsStringAndSize(t->bytes, &the_token, &size); for (char **keyword = p->soft_keywords; *keyword != NULL; keyword++) { - if (strncmp(*keyword, the_token, (size_t)size) == 0) { + if (strlen(*keyword) == (size_t)size && + strncmp(*keyword, the_token, (size_t)size) == 0) { return _PyPegen_name_from_token(p, t); } } @@ -942,6 +949,11 @@ _PyPegen_run_parser(Parser *p) { void *res = _PyPegen_parse(p); assert(p->level == 0); + if (res != NULL && PyErr_Occurred()) { + // Discard a result returned with an exception still pending + // (e.g. a MemoryError from a recovered-from allocation failure). + return NULL; + } if (res == NULL) { if ((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) && _is_end_of_source(p)) { PyErr_Clear(); @@ -996,8 +1008,11 @@ _PyPegen_run_parser_from_file_pointer(FILE *fp, int start_rule, PyObject *filena struct tok_state *tok = _PyTokenizer_FromFile(fp, enc, ps1, ps2); if (tok == NULL) { if (PyErr_Occurred()) { - _PyPegen_raise_tokenizer_init_error(filename_ob); - return NULL; + _PyTokenizer_raise_init_error(filename_ob); + } + else { + // The only silent tokenizer init failure is a failed allocation. + PyErr_NoMemory(); } return NULL; } @@ -1049,7 +1064,11 @@ _PyPegen_run_parser_from_string(const char *str, int start_rule, PyObject *filen } if (tok == NULL) { if (PyErr_Occurred()) { - _PyPegen_raise_tokenizer_init_error(filename_ob); + _PyTokenizer_raise_init_error(filename_ob); + } + else { + // The only silent tokenizer init failure is a failed allocation. + PyErr_NoMemory(); } return NULL; } diff --git a/Parser/pegen.h b/Parser/pegen.h index 1862fd7407ec3ca..1330d08bd6a10a2 100644 --- a/Parser/pegen.h +++ b/Parser/pegen.h @@ -145,10 +145,11 @@ int _PyPegen_insert_memo(Parser *p, int mark, int type, void *node); int _PyPegen_update_memo(Parser *p, int mark, int type, void *node); int _PyPegen_is_memoized(Parser *p, int type, void *pres); -int _PyPegen_lookahead_with_name(int, expr_ty (func)(Parser *), Parser *); -int _PyPegen_lookahead_with_int(int, Token *(func)(Parser *, int), Parser *, int); -int _PyPegen_lookahead_with_string(int , expr_ty (func)(Parser *, const char*), Parser *, const char*); int _PyPegen_lookahead(int, void *(func)(Parser *), Parser *); +int _PyPegen_lookahead_for_expr(int, expr_ty (func)(Parser *), Parser *); +int _PyPegen_lookahead_for_stmt(int, stmt_ty (func)(Parser *), Parser *); +int _PyPegen_lookahead_with_int(int, Token *(func)(Parser *, int), Parser *, int); +int _PyPegen_lookahead_with_string(int, expr_ty (func)(Parser *, const char*), Parser *, const char*); Token *_PyPegen_expect_token(Parser *p, int type); void* _PyPegen_expect_forced_result(Parser *p, void* result, const char* expected); @@ -174,7 +175,6 @@ typedef enum { } TARGETS_TYPE; int _Pypegen_raise_decode_error(Parser *p); -void _PyPegen_raise_tokenizer_init_error(PyObject *filename); int _Pypegen_tokenizer_error(Parser *p); void *_PyPegen_raise_error(Parser *p, PyObject *errtype, int use_mark, const char *errmsg, ...); void *_PyPegen_raise_error_known_location(Parser *p, PyObject *errtype, @@ -350,6 +350,7 @@ expr_ty _PyPegen_collect_call_seqs(Parser *, asdl_expr_seq *, asdl_seq *, expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok); expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok); expr_ty _PyPegen_constant_from_string(Parser* p, Token* tok); +expr_ty _PyPegen_concatenate_tstrings(Parser *p, asdl_expr_seq *, int, int, int, int, PyArena *); expr_ty _PyPegen_concatenate_strings(Parser *p, asdl_expr_seq *, int, int, int, int, PyArena *); expr_ty _PyPegen_FetchRawForm(Parser *p, int, int, int, int); expr_ty _PyPegen_ensure_imaginary(Parser *p, expr_ty); diff --git a/Parser/pegen_errors.c b/Parser/pegen_errors.c index f62b8695995617f..4d4f9a11865d88e 100644 --- a/Parser/pegen_errors.c +++ b/Parser/pegen_errors.c @@ -2,52 +2,13 @@ #include <errcode.h> #include "pycore_pyerrors.h" // _PyErr_ProgramDecodedTextObject() +#include "pycore_runtime.h" // _Py_ID() #include "lexer/state.h" #include "lexer/lexer.h" #include "pegen.h" // TOKENIZER ERRORS -void -_PyPegen_raise_tokenizer_init_error(PyObject *filename) -{ - if (!(PyErr_ExceptionMatches(PyExc_LookupError) - || PyErr_ExceptionMatches(PyExc_SyntaxError) - || PyErr_ExceptionMatches(PyExc_ValueError) - || PyErr_ExceptionMatches(PyExc_UnicodeDecodeError))) { - return; - } - PyObject *errstr = NULL; - PyObject *tuple = NULL; - PyObject *type; - PyObject *value; - PyObject *tback; - PyErr_Fetch(&type, &value, &tback); - errstr = PyObject_Str(value); - if (!errstr) { - goto error; - } - - PyObject *tmp = Py_BuildValue("(OiiO)", filename, 0, -1, Py_None); - if (!tmp) { - goto error; - } - - tuple = PyTuple_Pack(2, errstr, tmp); - Py_DECREF(tmp); - if (!value) { - goto error; - } - PyErr_SetObject(PyExc_SyntaxError, tuple); - -error: - Py_XDECREF(type); - Py_XDECREF(value); - Py_XDECREF(tback); - Py_XDECREF(errstr); - Py_XDECREF(tuple); -} - static inline void raise_unclosed_parentheses_error(Parser *p) { int error_lineno = p->tok->parenlinenostack[p->tok->level-1]; diff --git a/Parser/string_parser.c b/Parser/string_parser.c index d3631b114c5a3ca..ebe68989d1af58f 100644 --- a/Parser/string_parser.c +++ b/Parser/string_parser.c @@ -196,15 +196,18 @@ decode_unicode_with_escapes(Parser *parser, const char *s, size_t len, Token *t) len = (size_t)(p - buf); s = buf; - const char *first_invalid_escape; - v = _PyUnicode_DecodeUnicodeEscapeInternal(s, (Py_ssize_t)len, NULL, NULL, &first_invalid_escape); + int first_invalid_escape_char; + const char *first_invalid_escape_ptr; + v = _PyUnicode_DecodeUnicodeEscapeInternal2(s, (Py_ssize_t)len, NULL, NULL, + &first_invalid_escape_char, + &first_invalid_escape_ptr); // HACK: later we can simply pass the line no, since we don't preserve the tokens // when we are decoding the string but we preserve the line numbers. - if (v != NULL && first_invalid_escape != NULL && t != NULL) { - if (warn_invalid_escape_sequence(parser, s, first_invalid_escape, t) < 0) { - /* We have not decref u before because first_invalid_escape points - inside u. */ + if (v != NULL && first_invalid_escape_ptr != NULL && t != NULL) { + if (warn_invalid_escape_sequence(parser, s, first_invalid_escape_ptr, t) < 0) { + /* We have not decref u before because first_invalid_escape_ptr + points inside u. */ Py_XDECREF(u); Py_DECREF(v); return NULL; @@ -217,14 +220,17 @@ decode_unicode_with_escapes(Parser *parser, const char *s, size_t len, Token *t) static PyObject * decode_bytes_with_escapes(Parser *p, const char *s, Py_ssize_t len, Token *t) { - const char *first_invalid_escape; - PyObject *result = _PyBytes_DecodeEscape(s, len, NULL, &first_invalid_escape); + int first_invalid_escape_char; + const char *first_invalid_escape_ptr; + PyObject *result = _PyBytes_DecodeEscape2(s, len, NULL, + &first_invalid_escape_char, + &first_invalid_escape_ptr); if (result == NULL) { return NULL; } - if (first_invalid_escape != NULL) { - if (warn_invalid_escape_sequence(p, s, first_invalid_escape, t) < 0) { + if (first_invalid_escape_ptr != NULL) { + if (warn_invalid_escape_sequence(p, s, first_invalid_escape_ptr, t) < 0) { Py_DECREF(result); return NULL; } diff --git a/Parser/tokenizer/file_tokenizer.c b/Parser/tokenizer/file_tokenizer.c index 01e473f58a07779..a11702557a07af3 100644 --- a/Parser/tokenizer/file_tokenizer.c +++ b/Parser/tokenizer/file_tokenizer.c @@ -282,10 +282,8 @@ tok_underflow_interactive(struct tok_state *tok) { } static int -tok_underflow_file(struct tok_state *tok) { - if (tok->start == NULL && !INSIDE_FSTRING(tok)) { - tok->cur = tok->inp = tok->buf; - } +tok_underflow_file(struct tok_state *tok) +{ if (tok->decoding_state == STATE_INIT) { /* We have not yet determined the encoding. If an encoding is found, use the file-pointer @@ -296,8 +294,16 @@ tok_underflow_file(struct tok_state *tok) { } assert(tok->decoding_state != STATE_INIT); } + int raw = tok->decoding_readline == NULL; + if (raw && tok->decoding_state != STATE_NORMAL) { + /* Keep the first line in the buffer to validate it later if + * the encoding has not yet been determined. */ + } + else if (tok->start == NULL && !INSIDE_FSTRING(tok)) { + tok->cur = tok->inp = tok->buf; + } /* Read until '\n' or EOF */ - if (tok->decoding_readline != NULL) { + if (!raw) { /* We already have a codec associated with this input. */ if (!tok_readline_recode(tok)) { return 0; @@ -328,20 +334,35 @@ tok_underflow_file(struct tok_state *tok) { ADVANCE_LINENO(); if (tok->decoding_state != STATE_NORMAL) { - if (tok->lineno > 2) { - tok->decoding_state = STATE_NORMAL; - } - else if (!_PyTokenizer_check_coding_spec(tok->cur, strlen(tok->cur), + if (!_PyTokenizer_check_coding_spec(tok->cur, strlen(tok->cur), tok, fp_setreadl)) { return 0; } + if (tok->lineno >= 2) { + tok->decoding_state = STATE_NORMAL; + } } - /* The default encoding is UTF-8, so make sure we don't have any - non-UTF-8 sequences in it. */ - if (!tok->encoding && !_PyTokenizer_ensure_utf8(tok->cur, tok)) { - _PyTokenizer_error_ret(tok); - return 0; + if (raw && tok->decoding_state == STATE_NORMAL) { + const char *line = tok->lineno <= 2 ? tok->buf : tok->cur; + int lineno = tok->lineno <= 2 ? 1 : tok->lineno; + if (!tok->encoding) { + /* The default encoding is UTF-8, so make sure we don't have any + non-UTF-8 sequences in it. */ + if (!_PyTokenizer_ensure_utf8(line, tok, lineno)) { + _PyTokenizer_error_ret(tok); + return 0; + } + } + else { + PyObject *tmp = PyUnicode_Decode(line, strlen(line), + tok->encoding, NULL); + if (tmp == NULL) { + _PyTokenizer_error_ret(tok); + return 0; + } + Py_DECREF(tmp); + } } assert(tok->done == E_OK); return tok->done == E_OK; @@ -357,6 +378,7 @@ _PyTokenizer_FromFile(FILE *fp, const char* enc, return NULL; if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) { _PyTokenizer_Free(tok); + PyErr_NoMemory(); return NULL; } tok->cur = tok->inp = tok->buf; diff --git a/Parser/tokenizer/helpers.c b/Parser/tokenizer/helpers.c index 5a416adb875aa12..92d439e90363c8e 100644 --- a/Parser/tokenizer/helpers.c +++ b/Parser/tokenizer/helpers.c @@ -1,5 +1,6 @@ #include "Python.h" #include "errcode.h" +#include "pycore_runtime.h" // _Py_ID() #include "pycore_token.h" #include "../lexer/state.h" @@ -47,8 +48,10 @@ _syntaxerror_range(struct tok_state *tok, const char *format, goto error; } - args = Py_BuildValue("(O(OiiNii))", errmsg, tok->filename, tok->lineno, - col_offset, errtext, tok->lineno, end_col_offset); + args = Py_BuildValue("(O(OiiNii))", errmsg, + tok->filename ? tok->filename : Py_None, + tok->lineno, col_offset, errtext, + tok->lineno, end_col_offset); if (args) { PyErr_SetObject(PyExc_SyntaxError, args); Py_DECREF(args); @@ -147,6 +150,53 @@ _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_inval return 0; } +void +_PyTokenizer_raise_init_error(PyObject *filename) +{ + if (!(PyErr_ExceptionMatches(PyExc_LookupError) + || PyErr_ExceptionMatches(PyExc_SyntaxError) + || PyErr_ExceptionMatches(PyExc_ValueError) + || PyErr_ExceptionMatches(PyExc_UnicodeDecodeError))) { + return; + } + PyObject *errstr = NULL; + PyObject *tuple = NULL; + PyObject *type; + PyObject *value; + PyObject *tback; + PyErr_Fetch(&type, &value, &tback); + if (PyErr_GivenExceptionMatches(value, PyExc_SyntaxError)) { + if (PyObject_SetAttr(value, &_Py_ID(filename), filename)) { + goto error; + } + PyErr_Restore(type, value, tback); + return; + } + errstr = PyObject_Str(value); + if (!errstr) { + goto error; + } + + PyObject *tmp = Py_BuildValue("(OiiO)", filename, 0, -1, Py_None); + if (!tmp) { + goto error; + } + + tuple = PyTuple_Pack(2, errstr, tmp); + Py_DECREF(tmp); + if (!tuple) { + goto error; + } + PyErr_SetObject(PyExc_SyntaxError, tuple); + +error: + Py_XDECREF(type); + Py_XDECREF(value); + Py_XDECREF(tback); + Py_XDECREF(errstr); + Py_XDECREF(tuple); +} + int _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...) { @@ -191,6 +241,7 @@ _PyTokenizer_new_string(const char *s, Py_ssize_t len, struct tok_state *tok) char* result = (char *)PyMem_Malloc(len + 1); if (!result) { tok->done = E_NOMEM; + PyErr_NoMemory(); return NULL; } memcpy(result, s, len); @@ -219,6 +270,7 @@ _PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf buf = PyMem_Malloc(needed_length); if (buf == NULL) { tok->done = E_NOMEM; + PyErr_NoMemory(); return NULL; } for (current = buf; *s; s++, current++) { @@ -414,18 +466,21 @@ _PyTokenizer_check_coding_spec(const char* line, Py_ssize_t size, struct tok_sta if (tok->encoding == NULL) { assert(tok->decoding_readline == NULL); if (strcmp(cs, "utf-8") != 0 && !set_readline(tok, cs)) { + _PyTokenizer_raise_init_error(tok->filename); _PyTokenizer_error_ret(tok); - PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs); PyMem_Free(cs); return 0; } tok->encoding = cs; } else { /* then, compare cs with BOM */ if (strcmp(tok->encoding, cs) != 0) { - _PyTokenizer_error_ret(tok); - PyErr_Format(PyExc_SyntaxError, - "encoding problem: %s with BOM", cs); + tok->line_start = line; + tok->cur = (char *)line; + assert(size <= INT_MAX); + _PyTokenizer_syntaxerror_known_range(tok, 0, (int)size, + "encoding problem: %s with BOM", cs); PyMem_Free(cs); + _PyTokenizer_error_ret(tok); return 0; } PyMem_Free(cs); @@ -489,31 +544,47 @@ valid_utf8(const unsigned char* s) return 0; } length = expected + 1; - for (; expected; expected--) - if (s[expected] < 0x80 || s[expected] >= 0xC0) + for (int i = 1; i <= expected; i++) { + if (s[i] < 0x80 || s[i] >= 0xC0) { return 0; + } + } return length; } int -_PyTokenizer_ensure_utf8(char *line, struct tok_state *tok) +_PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno) { - int badchar = 0; - unsigned char *c; + const char *badchar = NULL; + const char *c; int length; - for (c = (unsigned char *)line; *c; c += length) { - if (!(length = valid_utf8(c))) { - badchar = *c; + int col_offset = 0; + const char *line_start = line; + for (c = line; *c; c += length) { + if (!(length = valid_utf8((const unsigned char *)c))) { + badchar = c; break; } + col_offset++; + if (*c == '\n') { + lineno++; + col_offset = 0; + line_start = c + 1; + } } if (badchar) { - PyErr_Format(PyExc_SyntaxError, - "Non-UTF-8 code starting with '\\x%.2x' " - "in file %U on line %i, " - "but no encoding declared; " - "see https://peps.python.org/pep-0263/ for details", - badchar, tok->filename, tok->lineno); + tok->lineno = lineno; + tok->line_start = line_start; + tok->cur = (char *)badchar; + _PyTokenizer_syntaxerror_known_range(tok, + col_offset + 1, col_offset + 1, + "Non-UTF-8 code starting with '\\x%.2x'" + "%s%V on line %i, " + "but no encoding declared; " + "see https://peps.python.org/pep-0263/ for details", + (unsigned char)*badchar, + tok->filename ? " in file " : "", tok->filename, "", + lineno); return 0; } return 1; diff --git a/Parser/tokenizer/helpers.h b/Parser/tokenizer/helpers.h index 42ea13cd1f853f9..34303999a60aff7 100644 --- a/Parser/tokenizer/helpers.h +++ b/Parser/tokenizer/helpers.h @@ -15,6 +15,7 @@ int _PyTokenizer_indenterror(struct tok_state *tok); int _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_invalid_escape_char); int _PyTokenizer_parser_warn(struct tok_state *tok, PyObject *category, const char *format, ...); char *_PyTokenizer_error_ret(struct tok_state *tok); +void _PyTokenizer_raise_init_error(PyObject *filename); char *_PyTokenizer_new_string(const char *s, Py_ssize_t len, struct tok_state *tok); char *_PyTokenizer_translate_newlines(const char *s, int exec_input, int preserve_crlf, struct tok_state *tok); @@ -26,7 +27,7 @@ int _PyTokenizer_check_bom(int get_char(struct tok_state *), struct tok_state *tok); int _PyTokenizer_check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok, int set_readline(struct tok_state *, const char *)); -int _PyTokenizer_ensure_utf8(char *line, struct tok_state *tok); +int _PyTokenizer_ensure_utf8(const char *line, struct tok_state *tok, int lineno); #ifdef Py_DEBUG void _PyTokenizer_print_escape(FILE *f, const char *s, Py_ssize_t size); diff --git a/Parser/tokenizer/readline_tokenizer.c b/Parser/tokenizer/readline_tokenizer.c index 22f84c77a12b47f..917f7b40cfbbfed 100644 --- a/Parser/tokenizer/readline_tokenizer.c +++ b/Parser/tokenizer/readline_tokenizer.c @@ -97,7 +97,7 @@ tok_underflow_readline(struct tok_state* tok) { ADVANCE_LINENO(); /* The default encoding is UTF-8, so make sure we don't have any non-UTF-8 sequences in it. */ - if (!tok->encoding && !_PyTokenizer_ensure_utf8(tok->cur, tok)) { + if (!tok->encoding && !_PyTokenizer_ensure_utf8(tok->cur, tok, tok->lineno)) { _PyTokenizer_error_ret(tok); return 0; } @@ -114,6 +114,7 @@ _PyTokenizer_FromReadline(PyObject* readline, const char* enc, return NULL; if ((tok->buf = (char *)PyMem_Malloc(BUFSIZ)) == NULL) { _PyTokenizer_Free(tok); + PyErr_NoMemory(); return NULL; } tok->cur = tok->inp = tok->buf; diff --git a/Parser/tokenizer/string_tokenizer.c b/Parser/tokenizer/string_tokenizer.c index 0c26d5df8d4a40e..7f07cca37ee0191 100644 --- a/Parser/tokenizer/string_tokenizer.c +++ b/Parser/tokenizer/string_tokenizer.c @@ -86,15 +86,18 @@ decode_str(const char *input, int single, struct tok_state *tok, int preserve_cr /* need to check line 1 and 2 separately since check_coding_spec assumes a single line as input */ if (newl[0]) { + tok->lineno = 1; if (!_PyTokenizer_check_coding_spec(str, newl[0] - str, tok, buf_setreadl)) { return NULL; } if (tok->enc == NULL && tok->decoding_state != STATE_NORMAL && newl[1]) { + tok->lineno = 2; if (!_PyTokenizer_check_coding_spec(newl[0]+1, newl[1] - newl[0], tok, buf_setreadl)) return NULL; } } + tok->lineno = 0; if (tok->enc != NULL) { assert(utf8 == NULL); utf8 = _PyTokenizer_translate_into_utf8(str, tok->enc); @@ -102,6 +105,22 @@ decode_str(const char *input, int single, struct tok_state *tok, int preserve_cr return _PyTokenizer_error_ret(tok); str = PyBytes_AS_STRING(utf8); } + else if (!_PyTokenizer_ensure_utf8(str, tok, 1)) { + return _PyTokenizer_error_ret(tok); + } + if (utf8 != NULL) { + char *translated = _PyTokenizer_translate_newlines( + str, single, preserve_crlf, tok); + if (translated == NULL) { + Py_DECREF(utf8); + return _PyTokenizer_error_ret(tok); + } + PyMem_Free(tok->input); + tok->input = translated; + str = translated; + Py_CLEAR(utf8); + } + tok->str = str; assert(tok->decoding_buffer == NULL); tok->decoding_buffer = utf8; /* CAUTION */ return str; diff --git a/Tools/wasm/README.md b/Platforms/emscripten/README.md similarity index 94% rename from Tools/wasm/README.md rename to Platforms/emscripten/README.md index 232321c515721ef..82d338c7781c820 100644 --- a/Tools/wasm/README.md +++ b/Platforms/emscripten/README.md @@ -22,7 +22,7 @@ https://github.com/psf/webassembly for more information. ### Build To cross compile to the ``wasm32-emscripten`` platform you need -[the Emscripten compiler toolchain](https://emscripten.org/), +[the Emscripten compiler toolchain](https://emscripten.org/), a Python interpreter, and an installation of Node version 18 or newer. Emscripten version 4.0.2 is recommended; newer versions may also work, but all official testing is performed with that version. All commands below are relative @@ -84,13 +84,24 @@ make a node application that "embeds" the interpreter instead of acting like the CLI you will need to write your own alternative to `node_entry.mjs`. +### Running tests + +After building, you can run the full test suite with: +```shell +./cross-build/wasm32-emscripten/build/python/python.sh -m test -uall +``` +You can run the browser smoke test with: +```shell +./Platforms/emscripten/browser_test/run_test.sh +``` + ### The Web Example -When building for Emscripten, the web example will be built automatically. It is -in the ``web_example`` directory. To run the web example, ``cd`` into the +When building for Emscripten, the web example will be built automatically. It +is in the ``web_example`` directory. To run the web example, ``cd`` into the ``web_example`` directory, then run ``python server.py``. This will start a web -server; you can then visit ``http://localhost:8000/python.html`` in a browser to -see a simple REPL example. +server; you can then visit ``http://localhost:8000/`` in a browser to see a +simple REPL example. The web example relies on a bug fix in Emscripten version 3.1.73 so if you build with earlier versions of Emscripten it may not work. The web example uses @@ -215,8 +226,8 @@ await createEmscriptenModule({ e.g. ``ctypes``, ``readline``, ``ssl``, and more. - Shared extension modules are not implemented yet. All extension modules are statically linked into the main binary. The experimental configure - option ``--enable-wasm-dynamic-linking`` enables dynamic extensions - supports. It's currently known to crash in combination with threading. + option ``--enable-wasm-dynamic-linking`` enables dynamic extension + support. It's currently known to crash in combination with threading. - glibc extensions for date and time formatting are not available. - ``locales`` module is affected by musl libc issues, [gh-90548](https://github.com/python/cpython/issues/90548). @@ -232,8 +243,8 @@ await createEmscriptenModule({ are not shipped. All other modules are bundled as pre-compiled ``pyc`` files. - In-memory file system (MEMFS) is not persistent and limited. -- Test modules are disabled by default. Use ``--enable-test-modules`` build - test modules like ``_testcapi``. +- Test modules are built by default. Use ``--disable-test-modules`` to disable + building test modules like ``_testcapi``. ## WASI (wasm32-wasi) diff --git a/Platforms/emscripten/__main__.py b/Platforms/emscripten/__main__.py new file mode 100644 index 000000000000000..84a1589b81f6a20 --- /dev/null +++ b/Platforms/emscripten/__main__.py @@ -0,0 +1,889 @@ +#!/usr/bin/env python3 + +import argparse +import contextlib +import functools +import hashlib +import json +import os +import shutil +import subprocess +import sys +import sysconfig +import tempfile +from pathlib import Path +from textwrap import dedent +from urllib.request import urlopen + +import tomllib + +try: + from os import process_cpu_count as cpu_count +except ImportError: + from os import cpu_count + + +EMSCRIPTEN_DIR = Path(__file__).parent +CHECKOUT = EMSCRIPTEN_DIR.parent.parent +CONFIG_FILE = EMSCRIPTEN_DIR / "config.toml" + +DEFAULT_CROSS_BUILD_DIR = CHECKOUT / "cross-build" +HOST_TRIPLE = "wasm32-emscripten" + + +@functools.cache +def load_config_toml(): + with CONFIG_FILE.open("rb") as file: + return tomllib.load(file) + + +@functools.cache +def required_emscripten_version(): + return load_config_toml()["emscripten-version"] + + +@functools.cache +def emsdk_cache_root(emsdk_cache): + required_version = required_emscripten_version() + return Path(emsdk_cache).absolute() / required_version + + +@functools.cache +def emsdk_activate_path(emsdk_cache): + return emsdk_cache_root(emsdk_cache) / "emsdk/emsdk_env.sh" + + +def get_build_paths(cross_build_dir=None, emsdk_cache=None): + """Compute all build paths from the given cross-build directory.""" + if cross_build_dir is None: + cross_build_dir = DEFAULT_CROSS_BUILD_DIR + cross_build_dir = Path(cross_build_dir).absolute() + host_triple_dir = cross_build_dir / HOST_TRIPLE + prefix_dir = host_triple_dir / "prefix" + if emsdk_cache: + prefix_dir = emsdk_cache_root(emsdk_cache) / "prefix" + + return { + "cross_build_dir": cross_build_dir, + "native_build_dir": cross_build_dir / "build", + "host_triple_dir": host_triple_dir, + "host_build_dir": host_triple_dir / "build", + "host_dir": host_triple_dir / "build" / "python", + "prefix_dir": prefix_dir, + } + + +LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" +LOCAL_SETUP_MARKER = b"# Generated by Platforms/wasm/emscripten.py\n" + + +@functools.cache +def validate_emsdk_version(emsdk_cache): + """Validate that the emsdk cache contains the required emscripten version.""" + if emsdk_cache is None: + print("Build will use EMSDK from current environment.") + return + required_version = required_emscripten_version() + emsdk_env = emsdk_activate_path(emsdk_cache) + if not emsdk_env.is_file(): + print( + f"Required emscripten version {required_version} not found in {emsdk_cache}", + file=sys.stderr, + ) + sys.exit(1) + print(f"✅ Emscripten version {required_version} found in {emsdk_cache}") + + +def parse_env(text): + result = {} + for line in text.splitlines(): + key, val = line.split("=", 1) + result[key] = val + return result + + +@functools.cache +def get_emsdk_environ(emsdk_cache): + """Returns os.environ updated by sourcing emsdk_env.sh""" + if not emsdk_cache: + return os.environ + env_text = subprocess.check_output( + [ + "bash", + "-c", + f"EMSDK_QUIET=1 source {emsdk_activate_path(emsdk_cache)} && env", + ], + text=True, + ) + return parse_env(env_text) + + +def updated_env(updates, emsdk_cache): + """Create a new dict representing the environment to use. + + The changes made to the execution environment are printed out. + """ + env_defaults = {} + # https://reproducible-builds.org/docs/source-date-epoch/ + git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] + try: + epoch = subprocess.check_output( + git_epoch_cmd, encoding="utf-8" + ).strip() + env_defaults["SOURCE_DATE_EPOCH"] = epoch + except subprocess.CalledProcessError: + pass # Might be building from a tarball. + # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. + environment = env_defaults | get_emsdk_environ(emsdk_cache) | updates + env_diff = {} + for key, value in environment.items(): + if os.environ.get(key) != value: + env_diff[key] = value + + print("🌎 Environment changes:") + for key in sorted(env_diff.keys()): + print(f" {key}={env_diff[key]}") + + return environment + + +def subdir(path_key, *, clean_ok=False): + """Decorator to change to a working directory. + + path_key is a key into context.build_paths, used to resolve the working + directory at call time. + """ + + def decorator(func): + @functools.wraps(func) + def wrapper(context): + working_dir = context.build_paths[path_key] + try: + tput_output = subprocess.check_output( + ["tput", "cols"], encoding="utf-8" + ) + terminal_width = int(tput_output.strip()) + except subprocess.CalledProcessError: + terminal_width = 80 + print("⎯" * terminal_width) + print("📁", working_dir) + if ( + clean_ok + and getattr(context, "clean", False) + and working_dir.exists() + ): + print("🚮 Deleting directory (--clean)...") + shutil.rmtree(working_dir) + + working_dir.mkdir(parents=True, exist_ok=True) + + with contextlib.chdir(working_dir): + return func(context, working_dir) + + return wrapper + + return decorator + + +def call(command, *, quiet, **kwargs): + """Execute a command. + + If 'quiet' is true, then redirect stdout and stderr to a temporary file. + """ + print("❯", " ".join(map(str, command))) + if not quiet: + stdout = None + stderr = None + else: + stdout = tempfile.NamedTemporaryFile( + "w", + encoding="utf-8", + delete=False, + prefix="cpython-emscripten-", + suffix=".log", + ) + stderr = subprocess.STDOUT + print(f"📝 Logging output to {stdout.name} (--quiet)...") + + subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) + + +def build_platform(): + """The name of the build/host platform.""" + # Can also be found via `config.guess`.` + return sysconfig.get_config_var("BUILD_GNU_TYPE") + + +def build_python_path(context): + """The path to the build Python binary.""" + native_build_dir = context.build_paths["native_build_dir"] + binary = native_build_dir / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError( + f"Unable to find `python(.exe)` in {native_build_dir}" + ) + + return binary + + +def install_emscripten(context): + emsdk_cache = context.emsdk_cache + if emsdk_cache is None: + print("install-emscripten requires --emsdk-cache", file=sys.stderr) + sys.exit(1) + version = required_emscripten_version() + emsdk_target = emsdk_cache_root(emsdk_cache) / "emsdk" + if emsdk_target.exists(): + if not context.quiet: + print(f"Emscripten version {version} already installed") + return + if not context.quiet: + print(f"Installing emscripten version {version}") + emsdk_target.mkdir(parents=True) + call( + [ + "git", + "clone", + "https://github.com/emscripten-core/emsdk.git", + emsdk_target, + ], + quiet=context.quiet, + ) + call([emsdk_target / "emsdk", "install", version], quiet=context.quiet) + call([emsdk_target / "emsdk", "activate", version], quiet=context.quiet) + if not context.quiet: + print(f"Installed emscripten version {version}") + + +@subdir("native_build_dir", clean_ok=True) +def configure_build_python(context, working_dir): + """Configure the build/host Python.""" + if LOCAL_SETUP.exists(): + print(f"👍 {LOCAL_SETUP} exists ...") + else: + print(f"📝 Touching {LOCAL_SETUP} ...") + LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) + + configure = [os.path.relpath(CHECKOUT / "configure", working_dir)] + if context.args: + configure.extend(context.args) + + call(configure, quiet=context.quiet) + + +@subdir("native_build_dir") +def make_build_python(context, working_dir): + """Make/build the build Python.""" + call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet) + + binary = build_python_path(context) + cmd = [ + binary, + "-c", + "import sys; " + "print(f'{sys.version_info.major}.{sys.version_info.minor}')", + ] + version = subprocess.check_output(cmd, encoding="utf-8").strip() + + print(f"🎉 {binary} {version}") + + +@subdir("native_build_dir") +def pythoninfo_build_python(context, working_dir): + """Display build info of the build Python.""" + call(["make", "pythoninfo"], quiet=context.quiet) + + +def check_shasum(file: str, expected_shasum: str): + with open(file, "rb") as f: + digest = hashlib.file_digest(f, "sha256") + if digest.hexdigest() != expected_shasum: + raise RuntimeError(f"Unexpected shasum for {file}") + + +def download_and_unpack(working_dir: Path, url: str, expected_shasum: str): + with tempfile.NamedTemporaryFile( + suffix=".tar.gz", delete_on_close=False + ) as tmp_file: + with urlopen(url) as response: + shutil.copyfileobj(response, tmp_file) + tmp_file.close() + check_shasum(tmp_file.name, expected_shasum) + shutil.unpack_archive(tmp_file.name, working_dir) + + +def should_build_library(prefix, name, config, quiet): + cached_config = prefix / (name + ".json") + if not cached_config.exists(): + if not quiet: + print( + f"No cached build of {name} version {config['version']} found, building" + ) + return True + + try: + with cached_config.open("rb") as f: + cached_config = json.load(f) + except json.JSONDecodeError: + if not quiet: + print(f"Cached data for {name} invalid, rebuilding") + return True + if config == cached_config: + if not quiet: + print( + f"Found cached build of {name} version {config['version']}, not rebuilding" + ) + return False + + if not quiet: + print( + f"Found cached build of {name} version {config['version']} but it's out of date, rebuilding" + ) + return True + + +def write_library_config(prefix, name, config, quiet): + cached_config = prefix / (name + ".json") + with cached_config.open("w") as f: + json.dump(config, f) + if not quiet: + print(f"Succeded building {name}, wrote config to {cached_config}") + + +@subdir("host_build_dir", clean_ok=True) +def make_emscripten_libffi(context, working_dir): + validate_emsdk_version(context.emsdk_cache) + prefix = context.build_paths["prefix_dir"] + libffi_config = load_config_toml()["dependencies"]["libffi"] + with open(EMSCRIPTEN_DIR / "make_libffi.sh", "rb") as f: + libffi_config["make_libffi_shasum"] = hashlib.file_digest(f, "sha256").hexdigest() + if not should_build_library( + prefix, "libffi", libffi_config, context.quiet + ): + return + + if context.check_up_to_date: + print("libffi out of date, expected to be up to date", file=sys.stderr) + sys.exit(1) + + url = libffi_config["url"] + version = libffi_config["version"] + shasum = libffi_config["shasum"] + libffi_dir = working_dir / f"libffi-{version}" + shutil.rmtree(libffi_dir, ignore_errors=True) + download_and_unpack( + working_dir, + url.format(version=version), + shasum, + ) + call( + [EMSCRIPTEN_DIR / "make_libffi.sh"], + env=updated_env({"PREFIX": prefix}, context.emsdk_cache), + cwd=libffi_dir, + quiet=context.quiet, + ) + write_library_config(prefix, "libffi", libffi_config, context.quiet) + + +@subdir("host_build_dir", clean_ok=True) +def make_mpdec(context, working_dir): + validate_emsdk_version(context.emsdk_cache) + prefix = context.build_paths["prefix_dir"] + mpdec_config = load_config_toml()["dependencies"]["mpdec"] + if not should_build_library(prefix, "mpdec", mpdec_config, context.quiet): + return + + if context.check_up_to_date: + print("libmpdec out of date, expected to be up to date", file=sys.stderr) + sys.exit(1) + + url = mpdec_config["url"] + version = mpdec_config["version"] + shasum = mpdec_config["shasum"] + mpdec_dir = working_dir / f"mpdecimal-{version}" + shutil.rmtree(mpdec_dir, ignore_errors=True) + download_and_unpack( + working_dir, + url.format(version=version), + shasum, + ) + call( + [ + "emconfigure", + mpdec_dir / "configure", + "CFLAGS=-fPIC", + "--prefix", + prefix, + "--disable-shared", + ], + cwd=mpdec_dir, + quiet=context.quiet, + env=updated_env({}, context.emsdk_cache), + ) + call( + ["make", "install"], + cwd=mpdec_dir, + quiet=context.quiet, + ) + write_library_config(prefix, "mpdec", mpdec_config, context.quiet) + + +def make_dependencies(context): + make_emscripten_libffi(context) + make_mpdec(context) + + +def calculate_node_path(): + node_version = os.environ.get("PYTHON_NODE_VERSION", None) + if node_version is None: + node_version = load_config_toml()["node-version"] + + subprocess.run( + [ + "bash", + "-c", + f"source ~/.nvm/nvm.sh && nvm install {node_version}", + ], + check=True, + ) + + res = subprocess.run( + [ + "bash", + "-c", + f"source ~/.nvm/nvm.sh && nvm which {node_version}", + ], + text=True, + capture_output=True, + check=True, + ) + return res.stdout.strip() + + +@subdir("host_dir", clean_ok=True) +def configure_emscripten_python(context, working_dir): + """Configure the emscripten/host build.""" + validate_emsdk_version(context.emsdk_cache) + host_runner = context.host_runner + if host_runner is None: + host_runner = calculate_node_path() + + paths = context.build_paths + config_site = os.fsdecode(EMSCRIPTEN_DIR / "config.site-wasm32-emscripten") + + emscripten_build_dir = working_dir.relative_to(CHECKOUT) + + python_build_dir = paths["native_build_dir"] / "build" + lib_dirs = list(python_build_dir.glob("lib.*")) + assert len(lib_dirs) == 1, ( + f"Expected a single lib.* directory in {python_build_dir}" + ) + lib_dir = os.fsdecode(lib_dirs[0]) + pydebug = lib_dir.endswith("-pydebug") + python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] + sysconfig_data = ( + f"{emscripten_build_dir}/build/lib.emscripten-wasm32-{python_version}" + ) + if pydebug: + sysconfig_data += "-pydebug" + pkg_config_path_dir = (paths["prefix_dir"] / "lib/pkgconfig/").resolve() + env_additions = { + "CONFIG_SITE": config_site, + "HOSTRUNNER": host_runner, + "EM_PKG_CONFIG_PATH": str(pkg_config_path_dir), + } + build_python = os.fsdecode(build_python_path(context)) + configure = [ + "emconfigure", + os.path.relpath(CHECKOUT / "configure", working_dir), + "CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2", + "PKG_CONFIG=pkg-config", + f"--host={HOST_TRIPLE}", + f"--build={build_platform()}", + f"--with-build-python={build_python}", + "--without-pymalloc", + "--disable-shared", + "--disable-ipv6", + "--enable-big-digits=30", + "--enable-wasm-dynamic-linking", + f"--prefix={paths['prefix_dir']}", + ] + if pydebug: + configure.append("--with-pydebug") + if context.args: + configure.extend(context.args) + call( + configure, + env=updated_env(env_additions, context.emsdk_cache), + quiet=context.quiet, + ) + + shutil.copy( + EMSCRIPTEN_DIR / "node_entry.mjs", working_dir / "node_entry.mjs" + ) + + shutil.copy( + EMSCRIPTEN_DIR / "streams.mjs", working_dir / "streams.mjs" + ) + + node_entry = working_dir / "node_entry.mjs" + exec_script = working_dir / "python.sh" + exec_script.write_text( + dedent( + f"""\ + #!/bin/sh + + # Macs come with FreeBSD coreutils which doesn't have the -s option + # so feature detect and work around it. + if which grealpath > /dev/null 2>&1; then + # It has brew installed gnu core utils, use that + REALPATH="grealpath -s" + elif which realpath > /dev/null 2>&1 && realpath --version > /dev/null 2>&1 && realpath --version | grep GNU > /dev/null 2>&1; then + # realpath points to GNU realpath so use it. + REALPATH="realpath -s" + else + # Shim for macs without GNU coreutils + abs_path () {{ + echo "$(cd "$(dirname "$1")" || exit; pwd)/$(basename "$1")" + }} + REALPATH=abs_path + fi + + # Before node 24, --experimental-wasm-jspi uses different API, + # After node 24 JSPI is on by default. + ARGS=$({host_runner} -e "$(cat <<"EOF" + const major_version = Number(process.version.split(".")[0].slice(1)); + if (major_version === 24) {{ + process.stdout.write("--experimental-wasm-jspi"); + }} + EOF + )") + + # We compute our own path, not following symlinks and pass it in so that + # node_entry.mjs can set sys.executable correctly. + # Intentionally allow word splitting on NODEFLAGS. + exec {host_runner} $NODEFLAGS $ARGS {node_entry} --this-program="$($REALPATH "$0")" "$@" + """ + ) + ) + exec_script.chmod(0o755) + print(f"🏃‍♀️ Created {exec_script} ... ") + sys.stdout.flush() + + +@subdir("host_dir") +def make_emscripten_python(context, working_dir): + """Run `make` for the emscripten/host build.""" + call( + ["make", "--jobs", str(cpu_count()), "all"], + env=updated_env({}, context.emsdk_cache), + quiet=context.quiet, + ) + + exec_script = working_dir / "python.sh" + subprocess.check_call([exec_script, "--version"]) + + +def run_emscripten_python(context, args=None): + """Run the built emscripten Python.""" + host_dir = context.build_paths["host_dir"] + exec_script = host_dir / "python.sh" + if not exec_script.is_file(): + print("Emscripten not built", file=sys.stderr) + sys.exit(1) + + if args is None: + args = context.args + # Strip the "--" separator if present + if args and args[0] == "--": + args = args[1:] + + if context.test: + args = load_config_toml()["test-args"] + args + elif context.pythoninfo: + args = load_config_toml()["pythoninfo-args"] + args + + os.execv(str(exec_script), [str(exec_script), *args]) + + +def pythoninfo_emscripten_python(context): + """Display build info of the host/Emscripten Python.""" + run_emscripten_python(context, ["-m", "test.pythoninfo"]) + + +def build_target(context): + """Build one or more targets.""" + steps = [] + if context.target in {"build", "all"}: + steps.extend([ + configure_build_python, + make_build_python, + pythoninfo_build_python, + ]) + if context.target in {"host", "all"}: + steps.extend([ + make_emscripten_libffi, + make_mpdec, + configure_emscripten_python, + make_emscripten_python, + pythoninfo_emscripten_python, + ]) + + for step in steps: + step(context) + + +def clean_contents(context): + """Delete all files created by this script.""" + if context.target in {"all", "build"}: + build_dir = context.build_paths["native_build_dir"] + if build_dir.exists(): + print(f"🧹 Deleting {build_dir} ...") + shutil.rmtree(build_dir) + + if context.target in {"all", "host"}: + host_triple_dir = context.build_paths["host_triple_dir"] + if host_triple_dir.exists(): + print(f"🧹 Deleting {host_triple_dir} ...") + shutil.rmtree(host_triple_dir) + + if LOCAL_SETUP.exists(): + with LOCAL_SETUP.open("rb") as file: + if file.read(len(LOCAL_SETUP_MARKER)) == LOCAL_SETUP_MARKER: + print(f"🧹 Deleting generated {LOCAL_SETUP} ...") + + +def add_cross_build_dir_option(subcommand): + subcommand.add_argument( + "--cross-build-dir", + action="store", + default=os.environ.get("CROSS_BUILD_DIR"), + dest="cross_build_dir", + help=( + "Path to the cross-build directory " + f"(default: {DEFAULT_CROSS_BUILD_DIR}). " + "Can also be set with the CROSS_BUILD_DIR environment variable." + ), + ) + + +def main(): + parser = argparse.ArgumentParser() + subcommands = parser.add_subparsers(dest="subcommand") + + install_emscripten_cmd = subcommands.add_parser( + "install-emscripten", + help="Install the appropriate version of Emscripten", + ) + + build = subcommands.add_parser("build", help="Build everything") + build.add_argument( + "target", + nargs="?", + default="all", + choices=["all", "host", "build"], + help=( + "What should be built. 'build' for just the build platform, or " + "'host' for the host platform, or 'all' for both. Defaults to 'all'." + ), + ) + + configure_build = subcommands.add_parser( + "configure-build-python", help="Run `configure` for the build Python" + ) + + make_mpdec_cmd = subcommands.add_parser( + "make-mpdec", + help="Clone mpdec repo, configure and build it for emscripten", + ) + + make_libffi_cmd = subcommands.add_parser( + "make-libffi", + help="Clone libffi repo, configure and build it for emscripten", + ) + + make_dependencies_cmd = subcommands.add_parser( + "make-dependencies", + help="Build all static library dependencies", + ) + + for cmd in [make_mpdec_cmd, make_libffi_cmd, make_dependencies_cmd]: + cmd.add_argument( + "--check-up-to-date", + action="store_true", + default=False, + help=("If passed, will fail if dependency is out of date"), + ) + + make_build = subcommands.add_parser( + "make-build-python", help="Run `make` for the build Python" + ) + + pythoninfo_build = subcommands.add_parser( + "pythoninfo-build", help="Display build info of the build Python" + ) + + configure_host = subcommands.add_parser( + "configure-host", + help=( + "Run `configure` for the host/emscripten " + "(pydebug builds are inferred from the build Python)" + ), + ) + + make_host = subcommands.add_parser( + "make-host", help="Run `make` for the host/emscripten" + ) + + pythoninfo_host = subcommands.add_parser( + "pythoninfo-host", help="Display build info of the host/Emscripten Python" + ) + + run = subcommands.add_parser( + "run", + help="Run the built emscripten Python", + ) + run.add_argument( + "--test", + action="store_true", + default=False, + help=( + "Add the default test arguments to the beginning of the command. " + "Default arguments loaded from Platforms/emscripten/config.toml" + ), + ) + run.add_argument( + "--pythoninfo", + action="store_true", + default=False, + help="Run -m test.pythoninfo", + ) + run.add_argument( + "args", + nargs=argparse.REMAINDER, + help=( + "Arguments to pass to the emscripten Python " + "(use '--' to separate from run options)" + ), + ) + add_cross_build_dir_option(run) + + clean = subcommands.add_parser( + "clean", help="Delete files and directories created by this script" + ) + clean.add_argument( + "target", + nargs="?", + default="host", + choices=["all", "host", "build"], + help=( + "What should be cleaned. 'build' for just the build platform, or " + "'host' for the host platform, or 'all' for both. Defaults to 'host'." + ), + ) + + for subcommand in ( + install_emscripten_cmd, + build, + configure_build, + make_libffi_cmd, + make_mpdec_cmd, + make_dependencies_cmd, + make_build, + pythoninfo_build, + configure_host, + make_host, + pythoninfo_host, + clean, + ): + subcommand.add_argument( + "--quiet", + action="store_true", + default="QUIET" in os.environ, + dest="quiet", + help=( + "Redirect output from subprocesses to a log file. " + "Can also be set with the QUIET environment variable." + ), + ) + add_cross_build_dir_option(subcommand) + subcommand.add_argument( + "--emsdk-cache", + action="store", + default=os.environ.get("EMSDK_CACHE"), + dest="emsdk_cache", + help=( + "Path to emsdk cache directory. If provided, validates that " + "the required emscripten version is installed. " + "Can also be set with the EMSDK_CACHE environment variable." + ), + ) + + for subcommand in configure_build, configure_host: + subcommand.add_argument( + "--clean", + action="store_true", + default=False, + dest="clean", + help="Delete any relevant directories before building", + ) + + for subcommand in build, configure_build, configure_host: + subcommand.add_argument( + "args", nargs="*", help="Extra arguments to pass to `configure`" + ) + + for subcommand in build, configure_host: + subcommand.add_argument( + "--host-runner", + action="store", + default=None, + dest="host_runner", + help="Command template for running the emscripten host " + "(default: use nvm to install the node version specified in config.toml)", + ) + + context = parser.parse_args() + context.emsdk_cache = getattr(context, "emsdk_cache", None) + context.cross_build_dir = getattr(context, "cross_build_dir", None) + context.check_up_to_date = getattr(context, "check_up_to_date", False) + + if context.emsdk_cache: + context.emsdk_cache = Path(context.emsdk_cache).absolute() + + context.build_paths = get_build_paths( + context.cross_build_dir, context.emsdk_cache + ) + + dispatch = { + "install-emscripten": install_emscripten, + "make-libffi": make_emscripten_libffi, + "make-mpdec": make_mpdec, + "make-dependencies": make_dependencies, + "configure-build-python": configure_build_python, + "make-build-python": make_build_python, + "pythoninfo-build": pythoninfo_build_python, + "configure-host": configure_emscripten_python, + "make-host": make_emscripten_python, + "pythoninfo-host": pythoninfo_emscripten_python, + "build": build_target, + "run": run_emscripten_python, + "clean": clean_contents, + } + + if not context.subcommand: + # No command provided, display help and exit + print( + "Expected one of", + ", ".join(sorted(dispatch.keys())), + file=sys.stderr, + ) + parser.print_help(sys.stderr) + sys.exit(1) + dispatch[context.subcommand](context) + + +if __name__ == "__main__": + main() diff --git a/Platforms/emscripten/browser_test/.gitignore b/Platforms/emscripten/browser_test/.gitignore new file mode 100644 index 000000000000000..3d54eab82a0d714 --- /dev/null +++ b/Platforms/emscripten/browser_test/.gitignore @@ -0,0 +1,4 @@ +node_modules +test-results +playwright-report +test_log.txt diff --git a/Platforms/emscripten/browser_test/index.spec.ts b/Platforms/emscripten/browser_test/index.spec.ts new file mode 100644 index 000000000000000..2594b73022bcc17 --- /dev/null +++ b/Platforms/emscripten/browser_test/index.spec.ts @@ -0,0 +1,15 @@ +import { test, expect } from '@playwright/test'; + +test('has title', async ({ page }) => { + await page.goto('/'); + + await expect(page).toHaveTitle("Emscripten PyRepl Example"); + const xterm = await page.locator('css=#terminal'); + await expect(xterm).toHaveText(/Python.*on emscripten.*Type.*for more information/); + const xtermInput = await page.getByRole('textbox'); + await xtermInput.pressSequentially(`def f():\nprint("hello", "emscripten repl!")\n\n`); + await xtermInput.pressSequentially(`f()\n`); + await expect(xterm).toHaveText(/hello emscripten repl!/); + +}); + diff --git a/Platforms/emscripten/browser_test/package-lock.json b/Platforms/emscripten/browser_test/package-lock.json new file mode 100644 index 000000000000000..67acfba9daf62ea --- /dev/null +++ b/Platforms/emscripten/browser_test/package-lock.json @@ -0,0 +1,1276 @@ +{ + "name": "browser_test", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "browser_test", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "@playwright/test": "^1.61.1", + "@types/node": "^24.12.0", + "get-port-cli": "^3.0.0", + "http-server": "^14.1.1", + "playwright": "^1.61.1" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.29.0", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", + "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.28.5", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.28.5", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", + "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@playwright/test": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.61.1.tgz", + "integrity": "sha512-8nKv6+0RJSL9FE4jYOEGXnPeM/Hg12qZpmqzZjRh3qM0Y7c3z1mrOTfFLids72RDQYVh9WpLEfR5WdpNX4fkig==", + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@types/minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@types/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-hov8bUuiLiyFPGyFPE1lwWhmzYbirOXQNNo40+y3zow8aFVTeyn3VWL0VFFfdNddA8S4Vf0Tc062rzyNr7Paag==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "24.12.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-24.12.0.tgz", + "integrity": "sha512-GYDxsZi3ChgmckRT9HPU0WEhKLP08ev/Yfcq2AstjrDASOYCSXeyjDsHg4v5t4jOj7cyDX3vmprafKlWIG9MXQ==", + "license": "MIT", + "dependencies": { + "undici-types": "~7.16.0" + } + }, + "node_modules/@types/normalize-package-data": { + "version": "2.4.4", + "resolved": "https://registry.npmjs.org/@types/normalize-package-data/-/normalize-package-data-2.4.4.tgz", + "integrity": "sha512-37i+OaWTh9qeK4LSHPsyRC7NahnGotNuZvjLSgcPzblpHB3rrCJxAOgI5gCdKm7coonsaX1Of0ILiTcnZjbfxA==", + "license": "MIT" + }, + "node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/arrify": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-1.0.1.tgz", + "integrity": "sha512-3CYzex9M9FGQjCGMGyi6/31c8GJbgb0qGyrx5HWxPd0aCwh4cB2YjMb2Xf9UuoogrMrlO9cTqnB5rI5GHZTcUA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "license": "MIT" + }, + "node_modules/basic-auth": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", + "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.1.2" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/camelcase-keys": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-7.0.2.tgz", + "integrity": "sha512-Rjs1H+A9R+Ig+4E/9oyB66UC5Mj9Xq3N//vcLf2WzgdTi/3gUu3Z9KoqmlrEG4VuuLK8wJHofxzdQXz/knhiYg==", + "license": "MIT", + "dependencies": { + "camelcase": "^6.3.0", + "map-obj": "^4.1.0", + "quick-lru": "^5.1.1", + "type-fest": "^1.2.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/chalk": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", + "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^4.1.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "license": "MIT" + }, + "node_modules/corser": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/corser/-/corser-2.0.1.tgz", + "integrity": "sha512-utCYNzRSQIZNPIcGZdQc92UVJYAhtGAteCFg0yRaFm8f0P+CPtyGyHXJcGXnffjCybUCEx3FQ2G7U3/o9eIkVQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/debug": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.1.tgz", + "integrity": "sha512-KcKCqiftBJcZr++7ykoDIEwSa3XWowTfNPo92BYxjXiyYEVrUQh2aLyhxBCwww+heortUFxEJYcRzosstTEBYQ==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decamelize": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-5.0.1.tgz", + "integrity": "sha512-VfxadyCECXgQlkoEAjeghAr5gY3Hf+IKjKb+X8tGVDtveCjN+USwprd2q3QXBR9T1+x2DG0XZF5/w+7HAtSaXA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/decamelize-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/decamelize-keys/-/decamelize-keys-1.1.1.tgz", + "integrity": "sha512-WiPxgEirIV0/eIOMcnFBA3/IJZAZqKnwAwWyvvdi4lsr1WCN22nhdf/3db3DoZcUjTV2SqfzIwNyp6y2xs3nmg==", + "license": "MIT", + "dependencies": { + "decamelize": "^1.1.0", + "map-obj": "^1.0.0" + }, + "engines": { + "node": ">=0.10.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/decamelize-keys/node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/decamelize-keys/node_modules/map-obj": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-1.0.1.tgz", + "integrity": "sha512-7N/q3lyZ+LVCp7PzuxrJr4KMbBE2hW7BT7YNia330OFxIf4d3r5zVpicP2650l7CPN6RM9zOJRl3NGpqSiw3Eg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/error-ex": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz", + "integrity": "sha512-sqQamAnR14VgCr1A618A3sGrygcpK+HEbenA/HiEAkkUwcZIIB/tgWqHFxWgOyDh4nB4JCRimh79dR5Ywc9MDQ==", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.2.1" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "license": "MIT" + }, + "node_modules/find-up": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-5.0.0.tgz", + "integrity": "sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==", + "license": "MIT", + "dependencies": { + "locate-path": "^6.0.0", + "path-exists": "^4.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "license": "MIT", + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-port": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/get-port/-/get-port-6.1.2.tgz", + "integrity": "sha512-BrGGraKm2uPqurfGVj/z97/zv8dPleC6x9JBNRTrDNtCkkRF4rPwrQXFgL7+I+q8QSdU4ntLQX2D7KIxSy8nGw==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-port-cli": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/get-port-cli/-/get-port-cli-3.0.0.tgz", + "integrity": "sha512-060GMr81KapTzSobWNrQVAqHeUaFRZhPj/lNnzdCcfVodFN497wRgEamnTCNgldJuiR6TXxdtkFidcYQ/nSVDA==", + "license": "MIT", + "dependencies": { + "get-port": "^6.0.0", + "meow": "^10.1.1" + }, + "bin": { + "get-port": "cli.js" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hard-rejection": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/hard-rejection/-/hard-rejection-2.1.0.tgz", + "integrity": "sha512-VIZB+ibDhx7ObhAe7OVtoEbuP4h/MuOTHJ+J8h/eBXotJYl0fBgR72xDFCKgIh22OJZIOVNxBMWuhAr10r8HdA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/he": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", + "integrity": "sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==", + "license": "MIT", + "bin": { + "he": "bin/he" + } + }, + "node_modules/hosted-git-info": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-4.1.0.tgz", + "integrity": "sha512-kyCuEOWjJqZuDbRHzL8V93NzQhwIB71oFWSyzVo+KPZI+pnQPPxucdkrOZvkLRnrf5URsQM+IJ09Dw29cRALIA==", + "license": "ISC", + "dependencies": { + "lru-cache": "^6.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/html-encoding-sniffer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz", + "integrity": "sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==", + "license": "MIT", + "dependencies": { + "whatwg-encoding": "^2.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/http-proxy": { + "version": "1.18.1", + "resolved": "https://registry.npmjs.org/http-proxy/-/http-proxy-1.18.1.tgz", + "integrity": "sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.0", + "follow-redirects": "^1.0.0", + "requires-port": "^1.0.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/http-server": { + "version": "14.1.1", + "resolved": "https://registry.npmjs.org/http-server/-/http-server-14.1.1.tgz", + "integrity": "sha512-+cbxadF40UXd9T01zUHgA+rlo2Bg1Srer4+B4NwIHdaGxAGGv59nYRnGGDJ9LBk7alpS0US+J+bLLdQOOkJq4A==", + "license": "MIT", + "dependencies": { + "basic-auth": "^2.0.1", + "chalk": "^4.1.2", + "corser": "^2.0.1", + "he": "^1.2.0", + "html-encoding-sniffer": "^3.0.0", + "http-proxy": "^1.18.1", + "mime": "^1.6.0", + "minimist": "^1.2.6", + "opener": "^1.5.1", + "portfinder": "^1.0.28", + "secure-compare": "3.0.1", + "union": "~0.5.0", + "url-join": "^4.0.1" + }, + "bin": { + "http-server": "bin/http-server" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/iconv-lite": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", + "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/indent-string": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-5.0.0.tgz", + "integrity": "sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-arrayish": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", + "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==", + "license": "MIT" + }, + "node_modules/is-core-module": { + "version": "2.16.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", + "integrity": "sha512-UfoeMA6fIJ8wTYFEUjelnaGI67v6+N7qXJEvQuIGa99l4xsCruSYOVSQ0uPANn4dAzm8lkYPaKLrrijLq7x23w==", + "license": "MIT", + "dependencies": { + "hasown": "^2.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-plain-obj": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-1.1.0.tgz", + "integrity": "sha512-yvkRyxmFKEOQ4pNXCmJG5AEQNlXJS5LaONXo5/cLdTZdWvsZ1ioJEonLGAosKlMWE8lwUy/bJzMjcw8az73+Fg==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/json-parse-even-better-errors": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", + "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", + "license": "MIT" + }, + "node_modules/kind-of": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz", + "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/lines-and-columns": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.2.4.tgz", + "integrity": "sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==", + "license": "MIT" + }, + "node_modules/locate-path": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz", + "integrity": "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==", + "license": "MIT", + "dependencies": { + "p-locate": "^5.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/lru-cache": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", + "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", + "license": "ISC", + "dependencies": { + "yallist": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/map-obj": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/map-obj/-/map-obj-4.3.0.tgz", + "integrity": "sha512-hdN1wVrZbb29eBGiGjJbeP8JbKjq1urkHJ/LIP/NY48MZ1QVXUsQBV1G1zvYFHn1XE06cwjBsOI2K3Ulnj1YXQ==", + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/meow": { + "version": "10.1.5", + "resolved": "https://registry.npmjs.org/meow/-/meow-10.1.5.tgz", + "integrity": "sha512-/d+PQ4GKmGvM9Bee/DPa8z3mXs/pkvJE2KEThngVNOqtmljC6K7NMPxtc2JeZYTmpWb9k/TmxjeL18ez3h7vCw==", + "license": "MIT", + "dependencies": { + "@types/minimist": "^1.2.2", + "camelcase-keys": "^7.0.0", + "decamelize": "^5.0.0", + "decamelize-keys": "^1.1.0", + "hard-rejection": "^2.1.0", + "minimist-options": "4.1.0", + "normalize-package-data": "^3.0.2", + "read-pkg-up": "^8.0.0", + "redent": "^4.0.0", + "trim-newlines": "^4.0.2", + "type-fest": "^1.2.2", + "yargs-parser": "^20.2.9" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/minimist-options": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/minimist-options/-/minimist-options-4.1.0.tgz", + "integrity": "sha512-Q4r8ghd80yhO/0j1O3B2BjweX3fiHg9cdOwjJd2J76Q135c+NDxGCqdYKQ1SKBuFfgWbAUzBfvYjPUEeNgqN1A==", + "license": "MIT", + "dependencies": { + "arrify": "^1.0.1", + "is-plain-obj": "^1.1.0", + "kind-of": "^6.0.3" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/normalize-package-data": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-3.0.3.tgz", + "integrity": "sha512-p2W1sgqij3zMMyRC067Dg16bfzVH+w7hyegmpIvZ4JNjqtGOVAIvLmjBx3yP7YTe9vKJgkoNOPjwQGogDoMXFA==", + "license": "BSD-2-Clause", + "dependencies": { + "hosted-git-info": "^4.0.1", + "is-core-module": "^2.5.0", + "semver": "^7.3.4", + "validate-npm-package-license": "^3.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/opener": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/opener/-/opener-1.5.2.tgz", + "integrity": "sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==", + "license": "(WTFPL OR MIT)", + "bin": { + "opener": "bin/opener-bin.js" + } + }, + "node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-5.0.0.tgz", + "integrity": "sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==", + "license": "MIT", + "dependencies": { + "p-limit": "^3.0.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/parse-json": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.2.0.tgz", + "integrity": "sha512-ayCKvm/phCGxOkYRSCM82iDwct8/EonSEgCSxWxD7ve6jHggsFl4fZVQBPRNgQoKiuV/odhFrGzQXZwbifC8Rg==", + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.0.0", + "error-ex": "^1.3.1", + "json-parse-even-better-errors": "^2.3.0", + "lines-and-columns": "^1.1.6" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "license": "ISC" + }, + "node_modules/playwright": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.61.1.tgz", + "integrity": "sha512-DWnY5o3YbLWK4GovuAVwpqL+1VwGNdUGrRr++8j8PtQQzvAVZUIMjKQ90fY689sEJZJBbZVw1rXaOKSTitkzPQ==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.61.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.61.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.61.1.tgz", + "integrity": "sha512-h7Qlt6m4REp25qvIdvbDtVmD4LqVXfpRxhORv9L0jzETM05p4fuPJ3dKyuSXQxDSbXnmS79HAgi9589lGSpLkg==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/portfinder": { + "version": "1.0.37", + "resolved": "https://registry.npmjs.org/portfinder/-/portfinder-1.0.37.tgz", + "integrity": "sha512-yuGIEjDAYnnOex9ddMnKZEMFE0CcGo6zbfzDklkmT1m5z734ss6JMzN9rNB3+RR7iS+F10D4/BVIaXOyh8PQKw==", + "license": "MIT", + "dependencies": { + "async": "^3.2.6", + "debug": "^4.3.6" + }, + "engines": { + "node": ">= 10.12" + } + }, + "node_modules/qs": { + "version": "6.15.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.0.tgz", + "integrity": "sha512-mAZTtNCeetKMH+pSjrb76NAM8V9a05I9aBZOHztWy/UqcJdQYNsf59vrRKWnojAT9Y+GbIvoTBC++CPHqpDBhQ==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/quick-lru": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/quick-lru/-/quick-lru-5.1.1.tgz", + "integrity": "sha512-WuyALRjWPDGtt/wzJiadO5AXY+8hZ80hVpe6MyivgraREW751X3SbhRvG3eLKOYN+8VEvqLcf3wdnt44Z4S4SA==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/read-pkg": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/read-pkg/-/read-pkg-6.0.0.tgz", + "integrity": "sha512-X1Fu3dPuk/8ZLsMhEj5f4wFAF0DWoK7qhGJvgaijocXxBmSToKfbFtqbxMO7bVjNA1dmE5huAzjXj/ey86iw9Q==", + "license": "MIT", + "dependencies": { + "@types/normalize-package-data": "^2.4.0", + "normalize-package-data": "^3.0.2", + "parse-json": "^5.2.0", + "type-fest": "^1.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/read-pkg-up": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/read-pkg-up/-/read-pkg-up-8.0.0.tgz", + "integrity": "sha512-snVCqPczksT0HS2EC+SxUndvSzn6LRCwpfSvLrIfR5BKDQQZMaI6jPRC9dYvYFDRAuFEAnkwww8kBBNE/3VvzQ==", + "license": "MIT", + "dependencies": { + "find-up": "^5.0.0", + "read-pkg": "^6.0.0", + "type-fest": "^1.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/redent": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/redent/-/redent-4.0.0.tgz", + "integrity": "sha512-tYkDkVVtYkSVhuQ4zBgfvciymHaeuel+zFKXShfDnFP5SyVEP7qo70Rf1jTOTCx3vGNAbnEi/xFkcfQVMIBWag==", + "license": "MIT", + "dependencies": { + "indent-string": "^5.0.0", + "strip-indent": "^4.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/requires-port": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/requires-port/-/requires-port-1.0.0.tgz", + "integrity": "sha512-KigOCHcocU3XODJxsu8i/j8T9tzT4adHiecwORRQ0ZZFcp7ahwXuRU1m+yuO90C5ZUyGeGfocHDI14M3L3yDAQ==", + "license": "MIT" + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/secure-compare": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/secure-compare/-/secure-compare-3.0.1.tgz", + "integrity": "sha512-AckIIV90rPDcBcglUwXPF3kg0P0qmPsPXAj6BBEENQE1p5yA1xfmDJzfi1Tappj37Pv2mVbKpL3Z1T+Nn7k1Qw==", + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.7.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.4.tgz", + "integrity": "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/spdx-correct": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.2.0.tgz", + "integrity": "sha512-kN9dJbvnySHULIluDHy32WHRUu3Og7B9sbY7tsFLctQkIqnMh3hErYgdMjTYuqmcXX+lK5T1lnUt3G7zNswmZA==", + "license": "Apache-2.0", + "dependencies": { + "spdx-expression-parse": "^3.0.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/spdx-exceptions": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/spdx-exceptions/-/spdx-exceptions-2.5.0.tgz", + "integrity": "sha512-PiU42r+xO4UbUS1buo3LPJkjlO7430Xn5SVAhdpzzsPHsjbYVflnnFdATgabnLude+Cqu25p6N+g2lw/PFsa4w==", + "license": "CC-BY-3.0" + }, + "node_modules/spdx-expression-parse": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/spdx-expression-parse/-/spdx-expression-parse-3.0.1.tgz", + "integrity": "sha512-cbqHunsQWnJNE6KhVSMsMeH5H/L9EpymbzqTQ3uLwNCLZ1Q481oWaofqH7nO6V07xlXwY6PhQdQ2IedWx/ZK4Q==", + "license": "MIT", + "dependencies": { + "spdx-exceptions": "^2.1.0", + "spdx-license-ids": "^3.0.0" + } + }, + "node_modules/spdx-license-ids": { + "version": "3.0.23", + "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.23.tgz", + "integrity": "sha512-CWLcCCH7VLu13TgOH+r8p1O/Znwhqv/dbb6lqWy67G+pT1kHmeD/+V36AVb/vq8QMIQwVShJ6Ssl5FPh0fuSdw==", + "license": "CC0-1.0" + }, + "node_modules/strip-indent": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/strip-indent/-/strip-indent-4.1.1.tgz", + "integrity": "sha512-SlyRoSkdh1dYP0PzclLE7r0M9sgbFKKMFXpFRUMNuKhQSbC6VQIGzq3E0qsfvGJaUFJPGv6Ws1NZ/haTAjfbMA==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/trim-newlines": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-4.1.1.tgz", + "integrity": "sha512-jRKj0n0jXWo6kh62nA5TEh3+4igKDXLvzBJcPpiizP7oOolUrYIxmVBG9TOtHYFHoddUk6YvAkGeGoSVTXfQXQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/type-fest": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-1.4.0.tgz", + "integrity": "sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/undici-types": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", + "integrity": "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==", + "license": "MIT" + }, + "node_modules/union": { + "version": "0.5.0", + "resolved": "https://registry.npmjs.org/union/-/union-0.5.0.tgz", + "integrity": "sha512-N6uOhuW6zO95P3Mel2I2zMsbsanvvtgn6jVqJv4vbVcz/JN0OkL9suomjQGmWtxJQXOCqUJvquc1sMeNz/IwlA==", + "dependencies": { + "qs": "^6.4.0" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/url-join": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/url-join/-/url-join-4.0.1.tgz", + "integrity": "sha512-jk1+QP6ZJqyOiuEI9AEWQfju/nB2Pw466kbA0LEZljHwKeMgd9WrAEgEGxjPDD2+TNbbb37rTyhEfrCXfuKXnA==", + "license": "MIT" + }, + "node_modules/validate-npm-package-license": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/validate-npm-package-license/-/validate-npm-package-license-3.0.4.tgz", + "integrity": "sha512-DpKm2Ui/xN7/HQKCtpZxoRWBhZ9Z0kqtygG8XCgNQ8ZlDnxuQmWhj566j8fN4Cu3/JmbhsDo7fcAJq4s9h27Ew==", + "license": "Apache-2.0", + "dependencies": { + "spdx-correct": "^3.0.0", + "spdx-expression-parse": "^3.0.0" + } + }, + "node_modules/whatwg-encoding": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz", + "integrity": "sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==", + "license": "MIT", + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/yallist": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", + "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==", + "license": "ISC" + }, + "node_modules/yargs-parser": { + "version": "20.2.9", + "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-20.2.9.tgz", + "integrity": "sha512-y11nGElTIV+CT3Zv9t7VKl+Q3hTQoT9a1Qzezhhl6Rp21gJ/IVTW7Z3y9EWXhuUBC2Shnf+DX0antecpAwSP8w==", + "license": "ISC", + "engines": { + "node": ">=10" + } + }, + "node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + } + } +} diff --git a/Platforms/emscripten/browser_test/package.json b/Platforms/emscripten/browser_test/package.json new file mode 100644 index 000000000000000..fc8e1ce33975ef6 --- /dev/null +++ b/Platforms/emscripten/browser_test/package.json @@ -0,0 +1,19 @@ +{ + "name": "browser_test", + "version": "1.0.0", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "description": "", + "dependencies": { + "@playwright/test": "^1.61.1", + "@types/node": "^24.12.0", + "get-port-cli": "^3.0.0", + "http-server": "^14.1.1", + "playwright": "^1.61.1" + } +} diff --git a/Platforms/emscripten/browser_test/playwright.config.ts b/Platforms/emscripten/browser_test/playwright.config.ts new file mode 100644 index 000000000000000..d170789a5970ec1 --- /dev/null +++ b/Platforms/emscripten/browser_test/playwright.config.ts @@ -0,0 +1,26 @@ +import { defineConfig, devices } from '@playwright/test'; +import { resolve } from "node:path"; + +const port = process.env.PORT ?? "8787"; +const crossBuildDir = resolve("../../../", process.env.CROSS_BUILD_DIR ?? "cross-build"); + +export default defineConfig({ + testDir: '.', + forbidOnly: true, + retries: 2, + reporter: process.env.CI ? 'dot' : 'html', + use: { + baseURL: `http://localhost:${port}`, + trace: 'on-first-retry', + }, + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + ], + webServer: { + command: `npx http-server ${crossBuildDir}/wasm32-emscripten/build/python/web_example_pyrepl_jspi/ -p ${port}`, + url: `http://localhost:${port}`, + }, +}); diff --git a/Platforms/emscripten/browser_test/run_test.sh b/Platforms/emscripten/browser_test/run_test.sh new file mode 100755 index 000000000000000..da6e24cba2e3c9e --- /dev/null +++ b/Platforms/emscripten/browser_test/run_test.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -euo pipefail +cd "$(dirname "$0")" +rm -f test_log.txt +echo "Installing node packages" | tee -a test_log.txt +npm ci >> test_log.txt 2>&1 +echo "Installing playwright browsers" | tee -a test_log.txt +npx playwright install 2>> test_log.txt +# Disable color so FORCE_COLOR=1 (set in CI) doesn't wrap the port number in +# ANSI escape codes, which would produce an invalid baseURL in Playwright. +export PORT=$(FORCE_COLOR=0 npx get-port-cli) +echo "Running tests with webserver on port $PORT" | tee -a test_log.txt +CI=1 npx playwright test | tee -a test_log.txt diff --git a/Platforms/emscripten/browser_test/tsconfig.json b/Platforms/emscripten/browser_test/tsconfig.json new file mode 100644 index 000000000000000..29a2d833656b532 --- /dev/null +++ b/Platforms/emscripten/browser_test/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "esnext", + "module": "nodenext", + "lib": ["ES2020"], + "strict": true, + "esModuleInterop": true, + "types": ["node"] + }, + "include": ["**/*.ts"], + "exclude": ["node_modules"] +} diff --git a/Tools/wasm/config.site-wasm32-emscripten b/Platforms/emscripten/config.site-wasm32-emscripten similarity index 97% rename from Tools/wasm/config.site-wasm32-emscripten rename to Platforms/emscripten/config.site-wasm32-emscripten index 1471546a5eec170..f69dbb8e779a426 100644 --- a/Tools/wasm/config.site-wasm32-emscripten +++ b/Platforms/emscripten/config.site-wasm32-emscripten @@ -1,6 +1,6 @@ # config.site override for cross compiling to wasm32-emscripten platform # -# CONFIG_SITE=Tools/wasm/config.site-wasm32-emscripten \ +# CONFIG_SITE=Platforms/emscripten/config.site-wasm32-emscripten \ # emconfigure ./configure --host=wasm32-unknown-emscripten --build=... # # Written by Christian Heimes <christian@python.org> @@ -69,7 +69,6 @@ ac_cv_func_posix_fallocate=no # Syscalls that resulted in a segfault ac_cv_func_utimensat=no -ac_cv_header_sys_ioctl_h=no # sockets are supported, but only AF_INET / AF_INET6 in non-blocking mode. # Disable AF_UNIX and AF_PACKET support, see socketmodule.h. diff --git a/Platforms/emscripten/config.toml b/Platforms/emscripten/config.toml new file mode 100644 index 000000000000000..ba2dc8f4a482bfa --- /dev/null +++ b/Platforms/emscripten/config.toml @@ -0,0 +1,26 @@ +# Any data that can vary between Python versions is to be kept in this file. +# This allows for blanket copying of the Emscripten build code between supported +# Python versions. +emscripten-version = "4.0.12" +node-version = "24" +test-args = [ + "-m", "test", + "-v", + "-uall", + "--rerun", + "--single-process", + "-W", +] +pythoninfo-args = [ + "-m", "test.pythoninfo", +] + +[dependencies.libffi] +url = "https://github.com/libffi/libffi/releases/download/v{version}/libffi-{version}.tar.gz" +version = "3.4.6" +shasum = "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e" + +[dependencies.mpdec] +url = "https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{version}.tar.gz" +version = "4.0.1" +shasum = "96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8" diff --git a/Tools/wasm/emscripten/make_libffi.sh b/Platforms/emscripten/make_libffi.sh similarity index 100% rename from Tools/wasm/emscripten/make_libffi.sh rename to Platforms/emscripten/make_libffi.sh diff --git a/Tools/wasm/emscripten/node_entry.mjs b/Platforms/emscripten/node_entry.mjs similarity index 92% rename from Tools/wasm/emscripten/node_entry.mjs rename to Platforms/emscripten/node_entry.mjs index 166df40742b7fc4..9478b7714adbc80 100644 --- a/Tools/wasm/emscripten/node_entry.mjs +++ b/Platforms/emscripten/node_entry.mjs @@ -1,5 +1,6 @@ import EmscriptenModule from "./python.mjs"; import fs from "node:fs"; +import { initializeStreams } from "./streams.mjs"; if (process?.versions?.node) { const nodeVersion = Number(process.versions.node.split(".", 1)[0]); @@ -39,6 +40,9 @@ const settings = { Object.assign(Module.ENV, process.env); delete Module.ENV.PATH; }, + onRuntimeInitialized() { + initializeStreams(Module.FS); + }, // Ensure that sys.executable, sys._base_executable, etc point to python.sh // not to this file. To properly handle symlinks, python.sh needs to compute // its own path. @@ -49,7 +53,7 @@ const settings = { try { await EmscriptenModule(settings); -} catch(e) { +} catch (e) { // Show JavaScript exception and traceback console.warn(e); // Show Python exception and traceback diff --git a/Platforms/emscripten/prepare_external_wasm.py b/Platforms/emscripten/prepare_external_wasm.py new file mode 100644 index 000000000000000..1b0a9de4b1fe8d8 --- /dev/null +++ b/Platforms/emscripten/prepare_external_wasm.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +import argparse +import sys +from pathlib import Path + +JS_TEMPLATE = """ +#include "emscripten.h" + +EM_JS(void, {function_name}, (void), {{ + return new WebAssembly.Module(hexStringToUTF8Array("{hex_string}")); +}} +function hexStringToUTF8Array(hex) {{ + const bytes = []; + for (let i = 0; i < hex.length; i += 2) {{ + bytes.push(parseInt(hex.substr(i, 2), 16)); + }} + return new Uint8Array(bytes); +}}); +""" + + +def prepare_wasm(input_file, output_file, function_name): + # Read the compiled WASM as binary and convert to hex + wasm_bytes = Path(input_file).read_bytes() + + hex_string = "".join(f"{byte:02x}" for byte in wasm_bytes) + + # Generate JavaScript module + js_content = JS_TEMPLATE.format( + function_name=function_name, hex_string=hex_string + ) + Path(output_file).write_text(js_content) + + print(f"Successfully compiled {input_file} and generated {output_file}") + return 0 + + +def main(): + parser = argparse.ArgumentParser( + description="Compile WebAssembly text files using wasm-as" + ) + parser.add_argument("input_file", help="Input .wat file to compile") + parser.add_argument("output_file", help="Output file name") + parser.add_argument("function_name", help="Name of the export function") + + args = parser.parse_args() + + return prepare_wasm(args.input_file, args.output_file, args.function_name) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/Platforms/emscripten/streams.mjs b/Platforms/emscripten/streams.mjs new file mode 100644 index 000000000000000..76ad79f9247f4cf --- /dev/null +++ b/Platforms/emscripten/streams.mjs @@ -0,0 +1,241 @@ +/** + * This is a pared down version of + * https://github.com/pyodide/pyodide/blob/main/src/js/streams.ts + * + * It replaces the standard streams devices that Emscripten provides with our + * own better ones. It fixes the following deficiencies: + * + * 1. The emscripten std streams always have isatty set to true. These set + * isatty to match the value for the stdin/stdout/stderr that node sees. + * 2. The emscripten std streams don't support the ttygetwinsize ioctl. If + * isatty() returns true, then these do, and it returns the actual window + * size as the OS reports it to Node. + * 3. The emscripten std streams introduce an extra layer of buffering which has + * to be flushed with fsync(). + * 4. The emscripten std streams are slow and complex because they go through a + * character-based handler layer. This is particularly awkward because both + * sides of this character based layer deal with buffers and so we need + * complex adaptors, buffering, etc on both sides. Removing this + * character-based middle layer makes everything better. + * https://github.com/emscripten-core/emscripten/blob/1aa7fb531f11e11e7ae49b75a24e1a8fe6fa4a7d/src/lib/libtty.js?plain=1#L104-L114 + * + * Ideally some version of this should go upstream to Emscripten since it is not + * in any way specific to Python. But I (Hood) haven't gotten around to it yet. + */ + +import * as tty from "node:tty"; +import * as fs from "node:fs"; + +let FS; +const DEVOPS = {}; +const DEVS = {}; + +function isErrnoError(e) { + return e && typeof e === "object" && "errno" in e; +} + +const waitBuffer = new Int32Array( + new WebAssembly.Memory({ shared: true, initial: 1, maximum: 1 }).buffer, +); +function syncSleep(timeout) { + try { + Atomics.wait(waitBuffer, 0, 0, timeout); + return true; + } catch (_) { + return false; + } +} + +/** + * Calls the callback and handle node EAGAIN errors. + */ +function handleEAGAIN(cb) { + while (true) { + try { + return cb(); + } catch (e) { + if (e && e.code === "EAGAIN") { + // Presumably this means we're in node and tried to read from/write to + // an O_NONBLOCK file descriptor. Synchronously sleep for 10ms then try + // again. In case for some reason we fail to sleep, propagate the error + // (it will turn into an EOFError). + if (syncSleep(10)) { + continue; + } + } + throw e; + } + } +} + +function readWriteHelper(stream, cb, method) { + let nbytes; + try { + nbytes = handleEAGAIN(cb); + } catch (e) { + if (e && e.code && Module.ERRNO_CODES[e.code]) { + throw new FS.ErrnoError(Module.ERRNO_CODES[e.code]); + } + if (isErrnoError(e)) { + // the handler set an errno, propagate it + throw e; + } + console.error("Error thrown in read:"); + console.error(e); + throw new FS.ErrnoError(Module.ERRNO_CODES.EIO); + } + if (nbytes === undefined) { + // Prevent an infinite loop caused by incorrect code that doesn't return a + // value. + // Maybe we should set nbytes = buffer.length here instead? + console.warn( + `${method} returned undefined; a correct implementation must return a number`, + ); + throw new FS.ErrnoError(Module.ERRNO_CODES.EIO); + } + if (nbytes !== 0) { + stream.node.timestamp = Date.now(); + } + return nbytes; +} + +function asUint8Array(arg) { + if (ArrayBuffer.isView(arg)) { + return new Uint8Array(arg.buffer, arg.byteOffset, arg.byteLength); + } else { + return new Uint8Array(arg); + } +} + +const prepareBuffer = (buffer, offset, length) => + asUint8Array(buffer).subarray(offset, offset + length); + +const TTY_OPS = { + ioctl_tiocgwinsz(tty) { + return tty.devops.ioctl_tiocgwinsz?.(); + }, +}; + +const stream_ops = { + open: function (stream) { + const devops = DEVOPS[stream.node.rdev]; + if (!devops) { + throw new FS.ErrnoError(Module.ERRNO_CODES.ENODEV); + } + stream.devops = devops; + stream.tty = stream.devops.isatty ? { ops: TTY_OPS, devops } : undefined; + stream.seekable = false; + }, + close: function (stream) { + // flush any pending line data + stream.stream_ops.fsync(stream); + }, + fsync: function (stream) { + const ops = stream.devops; + if (ops.fsync) { + ops.fsync(); + } + }, + read: function (stream, buffer, offset, length, pos /* ignored */) { + buffer = prepareBuffer(buffer, offset, length); + return readWriteHelper(stream, () => stream.devops.read(buffer), "read"); + }, + write: function (stream, buffer, offset, length, pos /* ignored */) { + buffer = prepareBuffer(buffer, offset, length); + return readWriteHelper(stream, () => stream.devops.write(buffer), "write"); + }, +}; + +function nodeFsync(fd) { + try { + fs.fsyncSync(fd); + } catch (e) { + if (e?.code === "EINVAL") { + return; + } + // On Mac, calling fsync when not isatty returns ENOTSUP + // On Windows, stdin/stdout/stderr may be closed, returning EBADF or EPERM + if ( + e?.code === "ENOTSUP" || e?.code === "EBADF" || e?.code === "EPERM" + ) { + return; + } + + throw e; + } +} + +class NodeReader { + constructor(nodeStream) { + this.nodeStream = nodeStream; + this.isatty = tty.isatty(nodeStream.fd); + } + + read(buffer) { + try { + return fs.readSync(this.nodeStream.fd, buffer); + } catch (e) { + // Platform differences: on Windows, reading EOF throws an exception, + // but on other OSes, reading EOF returns 0. Uniformize behavior by + // catching the EOF exception and returning 0. + if (e.toString().includes("EOF")) { + return 0; + } + throw e; + } + } + + fsync() { + nodeFsync(this.nodeStream.fd); + } +} + +class NodeWriter { + constructor(nodeStream) { + this.nodeStream = nodeStream; + this.isatty = tty.isatty(nodeStream.fd); + } + + write(buffer) { + return fs.writeSync(this.nodeStream.fd, buffer); + } + + fsync() { + nodeFsync(this.nodeStream.fd); + } + + ioctl_tiocgwinsz() { + return [this.nodeStream.rows ?? 24, this.nodeStream.columns ?? 80]; + } +} + +export function initializeStreams(fsarg) { + FS = fsarg; + const major = FS.createDevice.major++; + DEVS.stdin = FS.makedev(major, 0); + DEVS.stdout = FS.makedev(major, 1); + DEVS.stderr = FS.makedev(major, 2); + + FS.registerDevice(DEVS.stdin, stream_ops); + FS.registerDevice(DEVS.stdout, stream_ops); + FS.registerDevice(DEVS.stderr, stream_ops); + + FS.unlink("/dev/stdin"); + FS.unlink("/dev/stdout"); + FS.unlink("/dev/stderr"); + + FS.mkdev("/dev/stdin", DEVS.stdin); + FS.mkdev("/dev/stdout", DEVS.stdout); + FS.mkdev("/dev/stderr", DEVS.stderr); + + DEVOPS[DEVS.stdin] = new NodeReader(process.stdin); + DEVOPS[DEVS.stdout] = new NodeWriter(process.stdout); + DEVOPS[DEVS.stderr] = new NodeWriter(process.stderr); + + FS.closeStream(0 /* stdin */); + FS.closeStream(1 /* stdout */); + FS.closeStream(2 /* stderr */); + FS.open("/dev/stdin", 0 /* O_RDONLY */); + FS.open("/dev/stdout", 1 /* O_WRONLY */); + FS.open("/dev/stderr", 1 /* O_WRONLY */); +} diff --git a/Tools/wasm/emscripten/web_example/wasm_assets.py b/Platforms/emscripten/wasm_assets.py similarity index 95% rename from Tools/wasm/emscripten/web_example/wasm_assets.py rename to Platforms/emscripten/wasm_assets.py index deeb9229a4412bc..8743e76e4449afd 100755 --- a/Tools/wasm/emscripten/web_example/wasm_assets.py +++ b/Platforms/emscripten/wasm_assets.py @@ -15,10 +15,9 @@ import sys import sysconfig import zipfile -from typing import Dict # source directory -SRCDIR = pathlib.Path(__file__).parents[4].absolute() +SRCDIR = pathlib.Path(__file__).parents[2].absolute() SRCDIR_LIB = SRCDIR / "Lib" @@ -27,7 +26,9 @@ WASM_STDLIB_ZIP = ( WASM_LIB / f"python{sys.version_info.major}{sys.version_info.minor}.zip" ) -WASM_STDLIB = WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}" +WASM_STDLIB = ( + WASM_LIB / f"python{sys.version_info.major}.{sys.version_info.minor}" +) WASM_DYNLOAD = WASM_STDLIB / "lib-dynload" @@ -58,7 +59,6 @@ # socket.create_connection() raises an exception: # "BlockingIOError: [Errno 26] Operation in progress". OMIT_NETWORKING_FILES = ( - "email/", "ftplib.py", "http/", "imaplib.py", @@ -84,7 +84,6 @@ "_json": ["json/"], "_multiprocessing": ["concurrent/futures/process.py", "multiprocessing/"], "pyexpat": ["xml/", "xmlrpc/"], - "readline": ["rlcompleter.py"], "_sqlite3": ["sqlite3/"], "_ssl": ["ssl.py"], "_tkinter": ["idlelib/", "tkinter/", "turtle.py", "turtledemo/"], @@ -134,7 +133,7 @@ def filterfunc(filename: str) -> bool: pzf.writepy(entry, filterfunc=filterfunc) -def detect_extension_modules(args: argparse.Namespace) -> Dict[str, bool]: +def detect_extension_modules(args: argparse.Namespace) -> dict[str, bool]: modules = {} # disabled by Modules/Setup.local ? @@ -149,7 +148,7 @@ def detect_extension_modules(args: argparse.Namespace) -> Dict[str, bool]: # disabled by configure? with open(args.sysconfig_data) as f: data = f.read() - loc: Dict[str, Dict[str, str]] = {} + loc: dict[str, dict[str, str]] = {} exec(data, globals(), loc) for key, value in loc["build_time_vars"].items(): diff --git a/Tools/wasm/emscripten/web_example/python.html b/Platforms/emscripten/web_example/index.html similarity index 52% rename from Tools/wasm/emscripten/web_example/python.html rename to Platforms/emscripten/web_example/index.html index 078f86eb764419a..3a207b92015451b 100644 --- a/Tools/wasm/emscripten/web_example/python.html +++ b/Platforms/emscripten/web_example/index.html @@ -1,31 +1,39 @@ <!doctype html> <html lang="en"> <head> - <meta charset="UTF-8" /> - <meta http-equiv="X-UA-Compatible" content="IE=edge" /> - <meta name="viewport" content="width=device-width, initial-scale=1.0" /> - <meta name="author" content="Katie Bell" /> - <meta name="description" content="Simple REPL for Python WASM" /> + <meta charset="UTF-8"> + <meta http-equiv="X-UA-Compatible" content="IE=edge"> + <meta name="viewport" content="width=device-width, initial-scale=1.0"> + <meta name="author" content="Katie Bell, Adam Hartz"> + <meta name="description" content="Simple REPL for Python WASM"> <title>wasm-python terminal</title> <link rel="stylesheet" href="https://unpkg.com/xterm@4.18.0/css/xterm.css" crossorigin integrity="sha384-4eEEn/eZgVHkElpKAzzPx/Kow/dTSgFk1BNe+uHdjHa+NkZJDh5Vqkq31+y7Eycd" - /> + > <style> body { font-family: arial; max-width: 800px; margin: 0 auto; } - #code { + #editor { + padding: 5px; + border: 1px solid black; width: 100%; - height: 180px; + height: 300px; } #info { padding-top: 20px; } + .error { + border: 1px solid red; + background-color: #ffd9d9; + padding: 5px; + margin-top: 20px; + } .button-container { display: flex; justify-content: end; @@ -41,8 +49,14 @@ src="https://unpkg.com/xterm@4.18.0/lib/xterm.js" crossorigin integrity="sha384-yYdNmem1ioP5Onm7RpXutin5A8TimLheLNQ6tnMi01/ZpxXdAwIm2t4fJMx1Djs+" - /> + ></script> + <script + src="https://cdnjs.cloudflare.com/ajax/libs/ace/1.43.1/ace.js" + crossorigin + integrity="sha512-kmA5vhcxOkZI0ReiKJMGNb8/KKbgbExIlnt6aXuPtl86AgHBEi6OHHOz2wsTazBDGZKxe7fmiE+pIuZJQks4+A==" + ></script> <script type="module"> + const _magic_ctrlc_string = "__WASM_REPL_CTRLC_" + (Date.now()) + "__"; class WorkerManager { constructor( workerURL, @@ -132,11 +146,14 @@ class WasmTerminal { constructor() { - this.inputBuffer = new BufferQueue(); - this.input = ""; - this.resolveInput = null; - this.activeInput = false; - this.inputStartCursor = null; + try { + this.history = JSON.parse(sessionStorage.getItem('__python_wasm_repl.history')); + this.historyBuffer = this.history.slice(); + } catch(e) { + this.history = []; + this.historyBuffer = []; + } + this.reset(); this.xterm = new Terminal({ scrollback: 10000, @@ -155,6 +172,18 @@ this.xterm.onData(this.handleTermData); } + reset() { + this.inputBuffer = new BufferQueue(); + this.input = ""; + this.resolveInput = null; + this.activeInput = false; + this.inputStartCursor = null; + + this.cursorPosition = 0; + this.historyIndex = -1; + this.beforeHistoryNav = ""; + } + open(container) { this.xterm.open(container); } @@ -186,9 +215,34 @@ if (!(ord === 0x1b || ord == 0x7f || ord < 32)) { this.inputBuffer.addData(data); } - // TODO: Handle ANSI escape sequences + // TODO: Handle more escape sequences? } else if (ord === 0x1b) { // Handle special characters + switch (data.slice(1)) { + case "[A": // up + this.historyBack(); + break; + case "[B": // down + this.historyForward(); + break; + case "[C": // right + this.cursorRight(); + break; + case "[D": // left + this.cursorLeft(); + break; + case "[H": // home key + this.cursorHome(true); + break; + case "[F": // end key + this.cursorEnd(true); + break; + case "[3~": // delete key + this.deleteAtCursor(); + break; + default: + break; + } } else if (ord < 32 || ord === 0x7f) { switch (data) { case "\x0c": // CTRL+L @@ -201,8 +255,18 @@ this.input + this.writeLine("\n"), ); this.input = ""; + this.cursorPosition = 0; this.activeInput = false; break; + case "\x03": // CTRL+C + this.input = ""; + this.cursorPosition = 0; + this.historyIndex = -1; + this.resolveInput(_magic_ctrlc_string + "\n"); + break; + case "\x09": // TAB + this.handleTab(); + break; case "\x7F": // BACKSPACE case "\x08": // CTRL+H this.handleCursorErase(true); @@ -211,14 +275,20 @@ // Send empty input if (this.input === "") { this.resolveInput(""); + this.cursorPosition = 0; this.activeInput = false; } } } else { this.handleCursorInsert(data); + this.updateHistory(); } }; + clearLine() { + this.xterm.write("\x1b[K"); + } + writeLine(line) { this.xterm.write(line.slice(0, -1)); this.xterm.write("\r\n"); @@ -226,8 +296,36 @@ } handleCursorInsert(data) { - this.input += data; + const trailing = this.input.slice(this.cursorPosition); + this.input = + this.input.slice(0, this.cursorPosition) + + data + + trailing; + this.cursorPosition += data.length; this.xterm.write(data); + if (trailing.length !== 0) { + this.xterm.write(trailing); + this.xterm.write("\x1b[" + trailing.length + "D"); + } + this.updateHistory(); + } + + handleTab() { + // handle tabs: from the current position, add spaces until + // this.cursorPosition is a multiple of 4. + const prefix = this.input.slice(0, this.cursorPosition); + const suffix = this.input.slice(this.cursorPosition); + const count = 4 - (this.cursorPosition % 4); + const toAdd = " ".repeat(count); + this.input = prefix + toAdd + suffix; + this.cursorHome(false); + this.clearLine(); + this.xterm.write(this.input); + if (suffix) { + this.xterm.write("\x1b[" + suffix.length + "D"); + } + this.cursorPosition += count; + this.updateHistory(); } handleCursorErase() { @@ -238,9 +336,113 @@ ) { return; } - this.input = this.input.slice(0, -1); - this.xterm.write("\x1B[D"); - this.xterm.write("\x1B[P"); + const trailing = this.input.slice(this.cursorPosition); + this.input = + this.input.slice(0, this.cursorPosition - 1) + trailing; + this.cursorLeft(); + this.clearLine(); + if (trailing.length !== 0) { + this.xterm.write(trailing); + this.xterm.write("\x1b[" + trailing.length + "D"); + } + this.updateHistory(); + } + + deleteAtCursor() { + if (this.cursorPosition < this.input.length) { + const trailing = this.input.slice( + this.cursorPosition + 1, + ); + this.input = + this.input.slice(0, this.cursorPosition) + trailing; + this.clearLine(); + if (trailing.length !== 0) { + this.xterm.write(trailing); + this.xterm.write("\x1b[" + trailing.length + "D"); + } + this.updateHistory(); + } + } + + cursorRight() { + if (this.cursorPosition < this.input.length) { + this.cursorPosition += 1; + this.xterm.write("\x1b[C"); + } + } + + cursorLeft() { + if (this.cursorPosition > 0) { + this.cursorPosition -= 1; + this.xterm.write("\x1b[D"); + } + } + + cursorHome(updatePosition) { + if (this.cursorPosition > 0) { + this.xterm.write("\x1b[" + this.cursorPosition + "D"); + if (updatePosition) { + this.cursorPosition = 0; + } + } + } + + cursorEnd() { + if (this.cursorPosition < this.input.length) { + this.xterm.write( + "\x1b[" + + (this.input.length - this.cursorPosition) + + "C", + ); + this.cursorPosition = this.input.length; + } + } + + updateHistory() { + if (this.historyIndex !== -1) { + this.historyBuffer[this.historyIndex] = this.input; + } else { + this.beforeHistoryNav = this.input; + } + } + + historyBack() { + if (this.history.length === 0) { + return; + } else if (this.historyIndex === -1) { + // we're not currently navigating the history; store + // the current command and then look at the end of our + // history buffer + this.beforeHistoryNav = this.input; + this.historyIndex = this.history.length - 1; + } else if (this.historyIndex > 0) { + this.historyIndex -= 1; + } + this.input = this.historyBuffer[this.historyIndex]; + this.cursorHome(false); + this.clearLine(); + this.xterm.write(this.input); + this.cursorPosition = this.input.length; + } + + historyForward() { + if (this.history.length === 0 || this.historyIndex === -1) { + // we're not currently navigating the history; NOP. + return; + } else if (this.historyIndex < this.history.length - 1) { + this.historyIndex += 1; + this.input = this.historyBuffer[this.historyIndex]; + } else if (this.historyIndex == this.history.length - 1) { + // we're coming back from the last history value; reset + // the input to whatever it was when we started going + // through the history + this.input = this.beforeHistoryNav; + this.historyIndex = -1; + } + this.cursorHome(false); + this.clearLine(); + this.xterm.write(this.input); + this.cursorPosition = this.input.length; } prompt = async () => { @@ -263,12 +465,29 @@ // Hack to ensure cursor input start doesn't end up after user input setTimeout(() => { this.handleCursorInsert( - this.inputBuffer.nextLine(), + this.inputBuffer.nextLine() ); }, 1); } return new Promise((resolve, reject) => { this.resolveInput = (value) => { + if ( + value.replace(/\s/g, "").length != 0 && + value != _magic_ctrlc_string + "\n" + ) { + if (this.historyIndex !== -1) { + this.historyBuffer[this.historyIndex] = + this.history[this.historyIndex]; + } + this.history.push(value.slice(0, -1)); + this.historyBuffer.push(value.slice(0, -1)); + this.historyIndex = -1; + this.cursorPosition = 0; + try { + sessionStorage.setItem('__python_wasm_repl.history', JSON.stringify(this.history)); + } catch(e) { + } + } resolve(value); }; }); @@ -327,8 +546,6 @@ const stopButton = document.getElementById("stop"); const clearButton = document.getElementById("clear"); - const codeBox = document.getElementById("codebox"); - window.onload = () => { const terminal = new WasmTerminal(); terminal.open(document.getElementById("terminal")); @@ -362,8 +579,9 @@ runButton.addEventListener("click", (e) => { terminal.clear(); + terminal.reset(); // reset the history programRunning(true); - const code = codeBox.value; + const code = editor.getValue(); pythonWorkerManager.run({ args: ["main.py"], files: { "main.py": code }, @@ -372,10 +590,28 @@ replButton.addEventListener("click", (e) => { terminal.clear(); + terminal.reset(); // reset the history + const REPL = ` +class WASMREPLKeyboardInterrupt(KeyboardInterrupt): + pass + +import sys +import code +import builtins + +def _interrupt_aware_input(prompt=''): + line = builtins.input(prompt) + if line.strip() == "${_magic_ctrlc_string}": + raise KeyboardInterrupt() + return line + +cprt = 'Type "help", "copyright", "credits" or "license" for more information.' +banner = f'Python {sys.version} on {sys.platform}\\n{cprt}' + +code.interact(banner=banner, readfunc=_interrupt_aware_input, exitmsg='') +`; programRunning(true); - // Need to use "-i -" to force interactive mode. - // Looks like isatty always returns false in emscripten - pythonWorkerManager.run({ args: ["-i", "-"], files: {} }); + pythonWorkerManager.run({ args: ["-c", REPL], files: {} }); }); stopButton.addEventListener("click", (e) => { @@ -395,6 +631,7 @@ const finishedCallback = () => { programRunning(false); + pythonWorkerManager.reset(); }; const pythonWorkerManager = new WorkerManager( @@ -404,30 +641,63 @@ finishedCallback, ); }; + var editor; + document.addEventListener("DOMContentLoaded", () => { + editor = ace.edit("editor"); + editor.session.setMode("ace/mode/python"); + }); </script> </head> <body> - <h1>Simple REPL for Python WASM</h1> - <textarea id="codebox" cols="108" rows="16"> -print('Welcome to WASM!') -</textarea - > - <div class="button-container"> - <button id="run" disabled>Run</button> - <button id="repl" disabled>Start REPL</button> - <button id="stop" disabled>Stop</button> - <button id="clear" disabled>Clear</button> + <div id="repldemo"> + <h1>Simple REPL for Python WASM</h1> + <div id="editor">print('Welcome to WASM!')</div> + <div class="button-container"> + <button id="run" disabled>Run code</button> + <button id="repl" disabled>Start REPL</button> + <button id="stop" disabled>Stop</button> + <button id="clear" disabled>Clear</button> + </div> + <div id="terminal"></div> + <div id="info"> + The simple REPL provides a limited Python experience in the + browser. + <a + href="https://github.com/python/cpython/blob/main/Platforms/emscripten/README.md" + > + Platforms/emscripten/README.md + </a> + contains a list of known limitations and issues. Networking, + subprocesses, and threading are not available. + </div> </div> - <div id="terminal"></div> - <div id="info"> - The simple REPL provides a limited Python experience in the browser. - <a - href="https://github.com/python/cpython/blob/main/Tools/wasm/README.md" - > - Tools/wasm/README.md - </a> - contains a list of known limitations and issues. Networking, - subprocesses, and threading are not available. + <div id="buffererror" class="error" style="display: none"> + <p> + <code>SharedArrayBuffer</code>, which is required for this demo, + is not available in your browser environment. One common cause + of this failure is loading <code>index.html</code> directly in + your browser instead of using <code>server.py</code> as + described in + <a + href="https://github.com/python/cpython/blob/main/Platforms/emscripten/README.md#the-web-example" + > + Platforms/emscripten/README.md + </a>. + </p> + <p> + For more details about security requirements for + <code>SharedArrayBuffer</code>, see + <a + href="https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/SharedArrayBuffer#security_requirements" + >this MDN page</a + >. + </p> + <script> + if (typeof SharedArrayBuffer === 'undefined') { + document.getElementById('repldemo').style.display = 'none'; + document.getElementById('buffererror').style.display = 'block'; + } + </script> </div> </body> </html> diff --git a/Tools/wasm/emscripten/web_example/python.worker.mjs b/Platforms/emscripten/web_example/python.worker.mjs similarity index 100% rename from Tools/wasm/emscripten/web_example/python.worker.mjs rename to Platforms/emscripten/web_example/python.worker.mjs diff --git a/Tools/wasm/emscripten/web_example/server.py b/Platforms/emscripten/web_example/server.py similarity index 82% rename from Tools/wasm/emscripten/web_example/server.py rename to Platforms/emscripten/web_example/server.py index 768e6f84e07798e..f2e6ed56c6bcff5 100755 --- a/Tools/wasm/emscripten/web_example/server.py +++ b/Platforms/emscripten/web_example/server.py @@ -6,10 +6,16 @@ description="Start a local webserver with a Python terminal." ) parser.add_argument( - "--port", type=int, default=8000, help="port for the http server to listen on" + "--port", + type=int, + default=8000, + help="port for the http server to listen on", ) parser.add_argument( - "--bind", type=str, default="127.0.0.1", help="Bind address (empty for all)" + "--bind", + type=str, + default="127.0.0.1", + help="Bind address (empty for all)", ) diff --git a/Platforms/emscripten/web_example_pyrepl_jspi/index.html b/Platforms/emscripten/web_example_pyrepl_jspi/index.html new file mode 100644 index 000000000000000..cc7413b5caa9820 --- /dev/null +++ b/Platforms/emscripten/web_example_pyrepl_jspi/index.html @@ -0,0 +1,35 @@ +<!doctype html> +<html> + <head> + <title>Emscripten PyRepl Example</title> + <link + rel="stylesheet" + href="https://unpkg.com/xterm@4.18.0/css/xterm.css" + /> + <style> + body { + background-color: #300a24; + } + + .xterm-dom-renderer-owner-1 .xterm-fg-3 { + color: #c4a000 !important; + } + + .xterm-dom-renderer-owner-1 .xterm-fg-6 { + color: #2aa1b3 !important; + } + + .xterm-dom-renderer-owner-1 .xterm-fg-12 { + color: #1054a6 !important; + } + + .xterm-dom-renderer-owner-1 .xterm-fg-13 { + color: #a347ba !important; + } + </style> + </head> + <body> + <div id="terminal"></div> + <script type="module" src="src.mjs"></script> + </body> +</html> diff --git a/Platforms/emscripten/web_example_pyrepl_jspi/src.mjs b/Platforms/emscripten/web_example_pyrepl_jspi/src.mjs new file mode 100644 index 000000000000000..5642372c9d24728 --- /dev/null +++ b/Platforms/emscripten/web_example_pyrepl_jspi/src.mjs @@ -0,0 +1,194 @@ +// Much of this is adapted from here: +// https://github.com/mame/xterm-pty/blob/main/emscripten-pty.js +// Thanks to xterm-pty for making this possible! + +import createEmscriptenModule from "./python.mjs"; +import { openpty } from "https://unpkg.com/xterm-pty/index.mjs"; +import "https://unpkg.com/@xterm/xterm/lib/xterm.js"; + +var term = new Terminal(); +term.open(document.getElementById("terminal")); +const { master, slave: PTY } = openpty(); +term.loadAddon(master); +globalThis.PTY = PTY; + +async function setupStdlib(Module) { + const versionInt = Module.HEAPU32[Module._Py_Version >>> 2]; + const major = (versionInt >>> 24) & 0xff; + const minor = (versionInt >>> 16) & 0xff; + // Prevent complaints about not finding exec-prefix by making a lib-dynload directory + Module.FS.mkdirTree(`/lib/python${major}.${minor}/lib-dynload/`); + const resp = await fetch(`python${major}.${minor}.zip`); + const stdlibBuffer = await resp.arrayBuffer(); + Module.FS.writeFile( + `/lib/python${major}${minor}.zip`, + new Uint8Array(stdlibBuffer), + { canOwn: true }, + ); +} + +const tty_ops = { + ioctl_tcgets: () => { + const termios = PTY.ioctl("TCGETS"); + const data = { + c_iflag: termios.iflag, + c_oflag: termios.oflag, + c_cflag: termios.cflag, + c_lflag: termios.lflag, + c_cc: termios.cc, + }; + return data; + }, + + ioctl_tcsets: (_tty, _optional_actions, data) => { + PTY.ioctl("TCSETS", { + iflag: data.c_iflag, + oflag: data.c_oflag, + cflag: data.c_cflag, + lflag: data.c_lflag, + cc: data.c_cc, + }); + return 0; + }, + + ioctl_tiocgwinsz: () => PTY.ioctl("TIOCGWINSZ").reverse(), + + get_char: () => { + throw new Error("Should not happen"); + }, + put_char: () => { + throw new Error("Should not happen"); + }, + + fsync: () => {}, +}; + +const POLLIN = 1; +const POLLOUT = 4; + +const waitResult = { + READY: 0, + SIGNAL: 1, + TIMEOUT: 2, +}; + +function onReadable() { + var handle; + var promise = new Promise((resolve) => { + handle = PTY.onReadable(() => resolve(waitResult.READY)); + }); + return [promise, handle]; +} + +function onSignal() { + // TODO: signal handling + var handle = { dispose() {} }; + var promise = new Promise((resolve) => {}); + return [promise, handle]; +} + +function onTimeout(timeout) { + var id; + var promise = new Promise((resolve) => { + if (timeout > 0) { + id = setTimeout(resolve, timeout, waitResult.TIMEOUT); + } + }); + var handle = { + dispose() { + if (id) { + clearTimeout(id); + } + }, + }; + return [promise, handle]; +} + +async function waitForReadable(timeout) { + let p1, p2, p3; + let h1, h2, h3; + try { + [p1, h1] = onReadable(); + [p2, h2] = onTimeout(timeout); + [p3, h3] = onSignal(); + return await Promise.race([p1, p2, p3]); + } finally { + h1.dispose(); + h2.dispose(); + h3.dispose(); + } +} + +const FIONREAD = 0x541b; + +const tty_stream_ops = { + async readAsync(stream, buffer, offset, length, pos /* ignored */) { + let readBytes = PTY.read(length); + if (length && !readBytes.length) { + const status = await waitForReadable(-1); + if (status === waitResult.READY) { + readBytes = PTY.read(length); + } else { + throw new Error("Not implemented"); + } + } + buffer.set(readBytes, offset); + return readBytes.length; + }, + + write: (stream, buffer, offset, length) => { + // Note: default `buffer` is for some reason `HEAP8` (signed), while we want unsigned `HEAPU8`. + buffer = new Uint8Array( + buffer.buffer, + buffer.byteOffset, + buffer.byteLength, + ); + const toWrite = Array.from(buffer.subarray(offset, offset + length)); + PTY.write(toWrite); + return length; + }, + + async pollAsync(stream, timeout) { + if (!PTY.readable && timeout) { + await waitForReadable(timeout); + } + return (PTY.readable ? POLLIN : 0) | (PTY.writable ? POLLOUT : 0); + }, + ioctl(stream, request, varargs) { + if (request === FIONREAD) { + const res = PTY.fromLdiscToUpperBuffer.length; + Module.HEAPU32[varargs / 4] = res; + return 0; + } + throw new Error("Unimplemented ioctl request"); + }, +}; + +async function setupStdio(Module) { + Object.assign(Module.TTY.default_tty_ops, tty_ops); + Object.assign(Module.TTY.stream_ops, tty_stream_ops); +} + +const emscriptenSettings = { + async preRun(Module) { + Module.addRunDependency("pre-run"); + Module.ENV.TERM = "xterm-256color"; + // Uncomment next line to turn on tracing (messages go to browser console). + // Module.ENV.PYREPL_TRACE = "1"; + + // Leak module so we can try to show traceback if we crash on startup + globalThis.Module = Module; + await Promise.all([setupStdlib(Module), setupStdio(Module)]); + Module.removeRunDependency("pre-run"); + }, +}; + +try { + await createEmscriptenModule(emscriptenSettings); +} catch (e) { + // Show JavaScript exception and traceback + console.warn(e); + // Show Python exception and traceback + Module.__Py_DumpTraceback(2, Module._PyGILState_GetThisThreadState()); + process.exit(1); +} diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 8a7412c7019e607..6845cc7130a505d 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -53,6 +53,53 @@ static void error(const char *msg) } +static void error_fmt(const char *format, ...) +{ + va_list vargs; + va_start(vargs, format); + fprintf(stderr, "ERROR: "); + vfprintf(stderr, format, vargs); + fprintf(stderr, "\n"); + va_end(vargs); + fflush(stderr); +} + + +static wchar_t* py_getenv(const char *name) +{ + const char *env = getenv(name); + if (env == NULL) { + error_fmt("need %s env var", name); + return NULL; + } + + wchar_t *result = Py_DecodeLocale(env, NULL); + if (result == NULL) { + error("Py_DecodeLocale() failed"); + return NULL; + } + return result; +} + + +static wchar_t* get_cmdline_arg(const char *arg_name) +{ + if (main_argc < 3) { + const char *test = main_argv[1]; + fprintf(stderr, "usage: %s %s %s\n", PROGRAM, test, arg_name); + return NULL; + } + const char *arg = main_argv[2]; + + wchar_t *result = Py_DecodeLocale(arg, NULL); + if (result == NULL) { + error_fmt("failed to decode %s command line argument", arg_name); + return NULL; + } + return result; +} + + static void config_set_string(PyConfig *config, wchar_t **config_str, const wchar_t *str) { PyStatus status = PyConfig_SetString(config, config_str, str); @@ -360,8 +407,18 @@ static int test_pre_initialization_sys_options(void) size_t xoption_len = wcslen(static_xoption); wchar_t *dynamic_once_warnoption = \ (wchar_t *) calloc(warnoption_len+1, sizeof(wchar_t)); + if (dynamic_once_warnoption == NULL) { + error("out of memory allocating warnoption"); + return 1; + } wchar_t *dynamic_xoption = \ (wchar_t *) calloc(xoption_len+1, sizeof(wchar_t)); + if (dynamic_xoption == NULL) { + free(dynamic_once_warnoption); + error("out of memory allocating xoption"); + return 1; + } + wcsncpy(dynamic_once_warnoption, static_warnoption, warnoption_len+1); wcsncpy(dynamic_xoption, static_xoption, xoption_len+1); @@ -403,9 +460,9 @@ static int test_pre_initialization_sys_options(void) /* bpo-20891: Avoid race condition when initialising the GIL */ -static void bpo20891_thread(void *lockp) +static void bpo20891_thread(void *eventp) { - PyThread_type_lock lock = *((PyThread_type_lock*)lockp); + PyEvent *event = (PyEvent *)eventp; PyGILState_STATE state = PyGILState_Ensure(); if (!PyGILState_Check()) { @@ -414,8 +471,7 @@ static void bpo20891_thread(void *lockp) } PyGILState_Release(state); - - PyThread_release_lock(lock); + _PyEvent_Notify(event); } static int test_bpo20891(void) @@ -425,27 +481,17 @@ static int test_bpo20891(void) /* bpo-20891: Calling PyGILState_Ensure in a non-Python thread must not crash. */ - PyThread_type_lock lock = PyThread_allocate_lock(); - if (!lock) { - error("PyThread_allocate_lock failed!"); - return 1; - } _testembed_Py_InitializeFromConfig(); + PyEvent event = {0}; - unsigned long thrd = PyThread_start_new_thread(bpo20891_thread, &lock); + unsigned long thrd = PyThread_start_new_thread(bpo20891_thread, &event); if (thrd == PYTHREAD_INVALID_THREAD_ID) { error("PyThread_start_new_thread failed!"); return 1; } - PyThread_acquire_lock(lock, WAIT_LOCK); - Py_BEGIN_ALLOW_THREADS - /* wait until the thread exit */ - PyThread_acquire_lock(lock, WAIT_LOCK); - Py_END_ALLOW_THREADS - - PyThread_free_lock(lock); + PyEvent_Wait(&event); Py_Finalize(); @@ -1424,9 +1470,12 @@ static int test_audit_subinterpreter(void) PySys_AddAuditHook(_audit_subinterpreter_hook, NULL); _testembed_Py_InitializeFromConfig(); - Py_NewInterpreter(); - Py_NewInterpreter(); - Py_NewInterpreter(); + PyThreadState *tstate = PyThreadState_Get(); + for (int i = 0; i < 3; ++i) + { + Py_EndInterpreter(Py_NewInterpreter()); + PyThreadState_Swap(tstate); + } Py_Finalize(); @@ -1615,14 +1664,8 @@ static int test_init_sys_add(void) static int test_init_setpath(void) { - char *env = getenv("TESTPATH"); - if (!env) { - error("missing TESTPATH env var"); - return 1; - } - wchar_t *path = Py_DecodeLocale(env, NULL); + wchar_t *path = py_getenv("TESTPATH"); if (path == NULL) { - error("failed to decode TESTPATH"); return 1; } Py_SetPath(path); @@ -1648,14 +1691,8 @@ static int test_init_setpath_config(void) Py_ExitStatusException(status); } - char *env = getenv("TESTPATH"); - if (!env) { - error("missing TESTPATH env var"); - return 1; - } - wchar_t *path = Py_DecodeLocale(env, NULL); + wchar_t *path = py_getenv("TESTPATH"); if (path == NULL) { - error("failed to decode TESTPATH"); return 1; } Py_SetPath(path); @@ -1677,14 +1714,8 @@ static int test_init_setpath_config(void) static int test_init_setpythonhome(void) { - char *env = getenv("TESTHOME"); - if (!env) { - error("missing TESTHOME env var"); - return 1; - } - wchar_t *home = Py_DecodeLocale(env, NULL); + wchar_t *home = py_getenv("TESTHOME"); if (home == NULL) { - error("failed to decode TESTHOME"); return 1; } Py_SetPythonHome(home); @@ -1702,14 +1733,8 @@ static int test_init_is_python_build(void) { // gh-91985: in-tree builds fail to check for build directory landmarks // under the effect of 'home' or PYTHONHOME environment variable. - char *env = getenv("TESTHOME"); - if (!env) { - error("missing TESTHOME env var"); - return 1; - } - wchar_t *home = Py_DecodeLocale(env, NULL); + wchar_t *home = py_getenv("TESTHOME"); if (home == NULL) { - error("failed to decode TESTHOME"); return 1; } @@ -1723,7 +1748,7 @@ static int test_init_is_python_build(void) // Use an impossible value so we can detect whether it isn't updated // during initialization. config._is_python_build = INT_MAX; - env = getenv("NEGATIVE_ISPYTHONBUILD"); + char *env = getenv("NEGATIVE_ISPYTHONBUILD"); if (env && strcmp(env, "0") != 0) { config._is_python_build = INT_MIN; } @@ -2040,6 +2065,82 @@ static int test_init_run_main(void) } +static int test_init_run_main_exitcode(Py_ssize_t argc, wchar_t * const *argv) +{ + PyConfig config; + PyConfig_InitPythonConfig(&config); + + config.parse_argv = 1; + config_set_argv(&config, argc, argv); + config_set_string(&config, &config.program_name, L"./python3"); + + init_from_config_clear(&config); + + int exitcode = Py_RunMain(); + if (exitcode != 123) { + error_fmt("Py_RunMain() returned %i, expected 123", exitcode); + return 1; + } + + // If Py_RunMain() calls Py_Exit(), this message is not written to stdout + printf("ok! Py_RunMain() returned 123\n"); + + return 0; +} + + +static int test_init_run_main_script_exitcode(void) +{ + wchar_t *filename = get_cmdline_arg("FILENAME"); + if (filename == NULL) { + return 1; + } + + wchar_t* argv[] = {L"python3", filename}; + int res = test_init_run_main_exitcode(Py_ARRAY_LENGTH(argv), argv); + PyMem_RawFree(filename); + + return res; +} + + +static int test_init_run_main_module_exitcode(void) +{ + wchar_t *module = get_cmdline_arg("MODULE"); + if (module == NULL) { + return 1; + } + + wchar_t* argv[] = {L"python3", L"-m", module}; + int res = test_init_run_main_exitcode(Py_ARRAY_LENGTH(argv), argv); + PyMem_RawFree(module); + + return res; +} + + +static int test_init_run_main_interactive_exitcode(void) +{ + wchar_t* argv[] = {L"python3", L"-i"}; + return test_init_run_main_exitcode(Py_ARRAY_LENGTH(argv), argv); +} + + +static int test_init_run_main_code_exitcode(void) +{ + wchar_t *code = get_cmdline_arg("CODE"); + if (code == NULL) { + return 1; + } + + wchar_t* argv[] = {L"python3", L"-c", code}; + int res = test_init_run_main_exitcode(Py_ARRAY_LENGTH(argv), argv); + PyMem_RawFree(code); + + return res; +} + + static int test_run_main(void) { PyConfig config; @@ -2114,15 +2215,20 @@ static int check_use_frozen_modules(const char *rawval) if (rawval == NULL) { wcscpy(optval, L"frozen_modules"); } - else if (swprintf(optval, 100, -#if defined(_MSC_VER) - L"frozen_modules=%S", -#else - L"frozen_modules=%s", -#endif - rawval) < 0) { - error("rawval is too long"); - return -1; + else { + wchar_t *val = Py_DecodeLocale(rawval, NULL); + if (val == NULL) { + error("unable to decode TESTFROZEN"); + return -1; + } + wcscpy(optval, L"frozen_modules="); + if ((wcslen(optval) + wcslen(val)) >= Py_ARRAY_LENGTH(optval)) { + error("TESTFROZEN is too long"); + PyMem_RawFree(val); + return -1; + } + wcscat(optval, val); + PyMem_RawFree(val); } PyConfig config; @@ -2274,6 +2380,177 @@ static int test_repeated_init_and_inittab(void) return 0; } +/// Multi-phase initialization package & submodule /// + +int +mp_pkg_exec(PyObject *mod) +{ + // make this a namespace package + // empty list = namespace package + if (PyModule_Add(mod, "__path__", PyList_New(0)) < 0) { + return -1; + } + if (PyModule_AddStringConstant(mod, "mp_pkg_exec_slot_ran", "yes") < 0) { + return -1; + } + return 0; +} + +static PyModuleDef_Slot mp_pkg_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {Py_mod_exec, mp_pkg_exec}, + {0, NULL} +}; + +static struct PyModuleDef mp_pkg_def = { + PyModuleDef_HEAD_INIT, + .m_name = "mp_pkg", + .m_size = 0, + .m_slots = mp_pkg_slots, +}; + +PyMODINIT_FUNC +PyInit_mp_pkg(void) +{ + return PyModuleDef_Init(&mp_pkg_def); +} + +static PyObject * +submod_greet(PyObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyUnicode_FromString("Hello from sub-module"); +} + +static PyMethodDef submod_methods[] = { + {"greet", submod_greet, METH_NOARGS, NULL}, + {NULL}, +}; + +int +mp_submod_exec(PyObject *mod) +{ + return PyModule_AddStringConstant(mod, "mp_submod_exec_slot_ran", "yes"); +} + +static PyModuleDef_Slot mp_submod_slots[] = { + {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {Py_mod_exec, mp_submod_exec}, + {0, NULL} +}; + +static struct PyModuleDef mp_submod_def = { + PyModuleDef_HEAD_INIT, + .m_name = "mp_pkg.mp_submod", + .m_size = 0, + .m_methods = submod_methods, + .m_slots = mp_submod_slots, +}; + +PyMODINIT_FUNC +PyInit_mp_submod(void) +{ + return PyModuleDef_Init(&mp_submod_def); +} + +static int +test_inittab_submodule_multiphase(void) +{ + wchar_t* argv[] = { + PROGRAM_NAME, + L"-c", + L"import sys;" + L"import mp_pkg.mp_submod;" + L"print(mp_pkg.mp_submod);" + L"print(sys.modules['mp_pkg.mp_submod']);" + L"print(mp_pkg.mp_submod.greet());" + L"print(f'{mp_pkg.mp_submod.mp_submod_exec_slot_ran=}');" + L"print(f'{mp_pkg.mp_pkg_exec_slot_ran=}');" + }; + PyConfig config; + if (PyImport_AppendInittab("mp_pkg", + &PyInit_mp_pkg) != 0) { + fprintf(stderr, "PyImport_AppendInittab() failed\n"); + return 1; + } + if (PyImport_AppendInittab("mp_pkg.mp_submod", + &PyInit_mp_submod) != 0) { + fprintf(stderr, "PyImport_AppendInittab() failed\n"); + return 1; + } + PyConfig_InitPythonConfig(&config); + config.isolated = 1; + config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv); + init_from_config_clear(&config); + return Py_RunMain(); +} + +/// Single-phase initialization package & submodule /// + +static struct PyModuleDef sp_pkg_def = { + PyModuleDef_HEAD_INIT, + .m_name = "sp_pkg", + .m_size = 0, +}; + +PyMODINIT_FUNC +PyInit_sp_pkg(void) +{ + PyObject *mod = PyModule_Create(&sp_pkg_def); + if (mod == NULL) { + return NULL; + } + // make this a namespace package + // empty list = namespace package + if (PyModule_Add(mod, "__path__", PyList_New(0)) < 0) { + Py_DECREF(mod); + return NULL; + } + return mod; +} + +static struct PyModuleDef sp_submod_def = { + PyModuleDef_HEAD_INIT, + .m_name = "sp_pkg.sp_submod", + .m_size = 0, + .m_methods = submod_methods, +}; + +PyMODINIT_FUNC +PyInit_sp_submod(void) +{ + return PyModule_Create(&sp_submod_def); +} + +static int +test_inittab_submodule_singlephase(void) +{ + wchar_t* argv[] = { + PROGRAM_NAME, + L"-c", + L"import sys;" + L"import sp_pkg.sp_submod;" + L"print(sp_pkg.sp_submod);" + L"print(sys.modules['sp_pkg.sp_submod']);" + L"print(sp_pkg.sp_submod.greet());" + }; + PyConfig config; + if (PyImport_AppendInittab("sp_pkg", + &PyInit_sp_pkg) != 0) { + fprintf(stderr, "PyImport_AppendInittab() failed\n"); + return 1; + } + if (PyImport_AppendInittab("sp_pkg.sp_submod", + &PyInit_sp_submod) != 0) { + fprintf(stderr, "PyImport_AppendInittab() failed\n"); + return 1; + } + PyConfig_InitPythonConfig(&config); + config.isolated = 1; + config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv); + init_from_config_clear(&config); + return Py_RunMain(); +} + static void wrap_allocator(PyMemAllocatorEx *allocator); static void unwrap_allocator(PyMemAllocatorEx *allocator); @@ -2397,6 +2674,10 @@ static struct TestCase TestCases[] = { {"test_preinit_dont_parse_argv", test_preinit_dont_parse_argv}, {"test_init_read_set", test_init_read_set}, {"test_init_run_main", test_init_run_main}, + {"test_init_run_main_code_exitcode", test_init_run_main_code_exitcode}, + {"test_init_run_main_script_exitcode", test_init_run_main_script_exitcode}, + {"test_init_run_main_module_exitcode", test_init_run_main_module_exitcode}, + {"test_init_run_main_interactive_exitcode", test_init_run_main_interactive_exitcode}, {"test_init_sys_add", test_init_sys_add}, {"test_init_setpath", test_init_setpath}, {"test_init_setpath_config", test_init_setpath_config}, @@ -2431,7 +2712,8 @@ static struct TestCase TestCases[] = { {"test_frozenmain", test_frozenmain}, #endif {"test_get_incomplete_frame", test_get_incomplete_frame}, - + {"test_inittab_submodule_multiphase", test_inittab_submodule_multiphase}, + {"test_inittab_submodule_singlephase", test_inittab_submodule_singlephase}, {NULL, NULL} }; diff --git a/Python/Python-ast.c b/Python/Python-ast.c index df035568f84be1c..1cc88dc179e120d 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -5226,7 +5226,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) } if (p == 0) { PyErr_Format(PyExc_TypeError, - "%.400s got multiple values for argument '%U'", + "%.400s got multiple values for argument %R", Py_TYPE(self)->tp_name, key); res = -1; goto cleanup; @@ -5249,7 +5249,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) else if (contains == 0) { if (PyErr_WarnFormat( PyExc_DeprecationWarning, 1, - "%.400s.__init__ got an unexpected keyword argument '%U'. " + "%.400s.__init__ got an unexpected keyword argument %R. " "Support for arbitrary keyword arguments is deprecated " "and will be removed in Python 3.15.", Py_TYPE(self)->tp_name, key @@ -5293,7 +5293,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) else { if (PyErr_WarnFormat( PyExc_DeprecationWarning, 1, - "Field '%U' is missing from %.400s._field_types. " + "Field %R is missing from %.400s._field_types. " "This will become an error in Python 3.15.", name, Py_TYPE(self)->tp_name ) < 0) { @@ -5328,7 +5328,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) // simple field (e.g., identifier) if (PyErr_WarnFormat( PyExc_DeprecationWarning, 1, - "%.400s.__init__ missing 1 required positional argument: '%U'. " + "%.400s.__init__ missing 1 required positional argument: %R. " "This will become an error in Python 3.15.", Py_TYPE(self)->tp_name, name ) < 0) { @@ -5491,7 +5491,7 @@ ast_type_replace_check(PyObject *self, if (rc == 0) { PyErr_Format(PyExc_TypeError, "%.400s.__replace__ got an unexpected keyword " - "argument '%U'.", Py_TYPE(self)->tp_name, key); + "argument %R.", Py_TYPE(self)->tp_name, key); Py_DECREF(expecting); return -1; } @@ -5528,6 +5528,32 @@ ast_type_replace_check(PyObject *self, Py_DECREF(unused); } } + + // Discard fields from 'expecting' that default to None + PyObject *field_types = NULL; + if (PyObject_GetOptionalAttr((PyObject*)Py_TYPE(self), + &_Py_ID(_field_types), + &field_types) < 0) + { + Py_DECREF(expecting); + return -1; + } + if (field_types != NULL) { + Py_ssize_t pos = 0; + PyObject *field_name, *field_type; + while (PyDict_Next(field_types, &pos, &field_name, &field_type)) { + if (_PyUnion_Check(field_type)) { + // optional field + if (PySet_Discard(expecting, field_name) < 0) { + Py_DECREF(expecting); + Py_DECREF(field_types); + return -1; + } + } + } + Py_DECREF(field_types); + } + // Now 'expecting' contains the fields or attributes // that would not be filled inside ast_type_replace(). Py_ssize_t m = PySet_GET_SIZE(expecting); @@ -5770,7 +5796,7 @@ ast_repr_list(PyObject *list, int depth) for (Py_ssize_t i = 0; i < Py_MIN(length, 2); i++) { if (i > 0) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { goto error; } } @@ -5794,7 +5820,7 @@ ast_repr_list(PyObject *list, int depth) } if (i == 0 && length > 2) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ...", 5) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ...", 5) < 0) { goto error; } } @@ -5898,7 +5924,7 @@ ast_repr_max_depth(AST_object *self, int depth) } if (i > 0) { - if (PyUnicodeWriter_WriteUTF8(writer, ", ", 2) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, ", ", 2) < 0) { Py_DECREF(name); Py_DECREF(value_repr); goto error; @@ -6787,7 +6813,7 @@ init_types(void *arg) return -1; state->arguments_type = make_type(state, "arguments", state->AST_type, arguments_fields, 7, - "arguments(arg* posonlyargs, arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults, arg? kwarg, expr* defaults)"); + "arguments(arg* posonlyargs, arg* args, arg? vararg, arg* kwonlyargs, expr?* kw_defaults, arg? kwarg, expr* defaults)"); if (!state->arguments_type) return -1; if (add_attributes(state, state->arguments_type, NULL, 0) < 0) return -1; if (PyObject_SetAttr(state->arguments_type, state->vararg, Py_None) == -1) diff --git a/Python/_warnings.c b/Python/_warnings.c index 39bf1b225ccb0c6..912468d2a59a95e 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1479,28 +1479,6 @@ PyErr_WarnExplicitObject(PyObject *category, PyObject *message, return 0; } -/* Like PyErr_WarnExplicitObject, but automatically sets up context */ -int -_PyErr_WarnExplicitObjectWithContext(PyObject *category, PyObject *message, - PyObject *filename, int lineno) -{ - PyObject *unused_filename, *module, *registry; - int unused_lineno; - int stack_level = 1; - - if (!setup_context(stack_level, NULL, &unused_filename, &unused_lineno, - &module, &registry)) { - return -1; - } - - int rc = PyErr_WarnExplicitObject(category, message, filename, lineno, - module, registry); - Py_DECREF(unused_filename); - Py_DECREF(registry); - Py_DECREF(module); - return rc; -} - int PyErr_WarnExplicit(PyObject *category, const char *text, const char *filename_str, int lineno, diff --git a/Python/asm_trampoline.S b/Python/asm_trampoline.S index 0a3265dfeee2044..2513cde4e764951 100644 --- a/Python/asm_trampoline.S +++ b/Python/asm_trampoline.S @@ -1,3 +1,5 @@ +#include "asm_trampoline_aarch64.h" + .text .globl _Py_trampoline_func_start # The following assembly is equivalent to: @@ -9,18 +11,24 @@ # } _Py_trampoline_func_start: #ifdef __x86_64__ - sub $8, %rsp - call *%rcx - add $8, %rsp +#if defined(__CET__) && (__CET__ & 1) + endbr64 +#endif + push %rbp + mov %rsp, %rbp + call *%rcx + pop %rbp ret #endif // __x86_64__ #if defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) // ARM64 little endian, 64bit ABI // generate with aarch64-linux-gnu-gcc 12.1 + SIGN_LR stp x29, x30, [sp, -16]! mov x29, sp blr x3 ldp x29, x30, [sp], 16 + VERIFY_LR ret #endif #ifdef __riscv @@ -34,3 +42,22 @@ _Py_trampoline_func_start: .globl _Py_trampoline_func_end _Py_trampoline_func_end: .section .note.GNU-stack,"",@progbits +# Note for indicating the assembly code supports CET +#if defined(__x86_64__) && defined(__CET__) && (__CET__ & 1) + .section .note.gnu.property,"a" + .align 8 + .long 1f - 0f + .long 4f - 1f + .long 5 +0: + .string "GNU" +1: + .align 8 + .long 0xc0000002 + .long 3f - 2f +2: + .long 0x3 +3: + .align 8 +4: +#endif // __x86_64__ diff --git a/Python/asm_trampoline_aarch64.h b/Python/asm_trampoline_aarch64.h new file mode 100644 index 000000000000000..bc83aa460b6860d --- /dev/null +++ b/Python/asm_trampoline_aarch64.h @@ -0,0 +1,56 @@ +#ifndef ASM_TRAMPOLINE_AARCH_64_H_ +#define ASM_TRAMPOLINE_AARCH_64_H_ + +/* + * References: + * - https://developer.arm.com/documentation/101028/0012/5--Feature-test-macros + * - https://github.com/ARM-software/abi-aa/blob/main/aaelf64/aaelf64.rst + */ + +#if defined(__ARM_FEATURE_BTI_DEFAULT) && __ARM_FEATURE_BTI_DEFAULT == 1 + #define BTI_J hint 36 /* bti j: for jumps, IE br instructions */ + #define BTI_C hint 34 /* bti c: for calls, IE bl instructions */ + #define GNU_PROPERTY_AARCH64_BTI 1 /* bit 0 GNU Notes is for BTI support */ +#else + #define BTI_J + #define BTI_C + #define GNU_PROPERTY_AARCH64_BTI 0 +#endif + +#if defined(__ARM_FEATURE_PAC_DEFAULT) + #if __ARM_FEATURE_PAC_DEFAULT & 1 + #define SIGN_LR hint 25 /* paciasp: sign with the A key */ + #define VERIFY_LR hint 29 /* autiasp: verify with the A key */ + #elif __ARM_FEATURE_PAC_DEFAULT & 2 + #define SIGN_LR hint 27 /* pacibsp: sign with the b key */ + #define VERIFY_LR hint 31 /* autibsp: verify with the b key */ + #endif + #define GNU_PROPERTY_AARCH64_POINTER_AUTH 2 /* bit 1 GNU Notes is for PAC support */ +#else + #define SIGN_LR BTI_C + #define VERIFY_LR + #define GNU_PROPERTY_AARCH64_POINTER_AUTH 0 +#endif + +#if defined(__ARM_FEATURE_GCS_DEFAULT) && __ARM_FEATURE_GCS_DEFAULT == 1 + #define GNU_PROPERTY_AARCH64_GCS 4 /* bit 2 GNU Notes is for GCS support */ +#else + #define GNU_PROPERTY_AARCH64_GCS 0 +#endif + +/* Add the BTI, PAC and GCS support to GNU Notes section */ +#if GNU_PROPERTY_AARCH64_BTI != 0 || GNU_PROPERTY_AARCH64_POINTER_AUTH != 0 || GNU_PROPERTY_AARCH64_GCS != 0 + .pushsection .note.gnu.property, "a"; /* Start a new allocatable section */ + .balign 8; /* align it on a byte boundry */ + .long 4; /* size of "GNU\0" */ + .long 0x10; /* size of descriptor */ + .long 0x5; /* NT_GNU_PROPERTY_TYPE_0 */ + .asciz "GNU"; + .long 0xc0000000; /* GNU_PROPERTY_AARCH64_FEATURE_1_AND */ + .long 4; /* Four bytes of data */ + .long (GNU_PROPERTY_AARCH64_BTI|GNU_PROPERTY_AARCH64_POINTER_AUTH|GNU_PROPERTY_AARCH64_GCS); /* BTI, PAC or GCS is enabled */ + .long 0; /* padding for 8 byte alignment */ + .popsection; /* end the section */ +#endif + +#endif diff --git a/Python/assemble.c b/Python/assemble.c index 8cc2d50a3227f8a..761ac813bf2425f 100644 --- a/Python/assemble.c +++ b/Python/assemble.c @@ -80,9 +80,9 @@ assemble_init(struct assembler *a, int firstlineno) } return SUCCESS; error: - Py_XDECREF(a->a_bytecode); - Py_XDECREF(a->a_linetable); - Py_XDECREF(a->a_except_table); + Py_CLEAR(a->a_bytecode); + Py_CLEAR(a->a_linetable); + Py_CLEAR(a->a_except_table); return ERROR; } diff --git a/Python/ast_preprocess.c b/Python/ast_preprocess.c index bafd67ed790b209..fe6fd9479d1531f 100644 --- a/Python/ast_preprocess.c +++ b/Python/ast_preprocess.c @@ -19,6 +19,7 @@ typedef struct { int optimize; int ff_features; int syntax_check_only; + int enable_warnings; _Py_c_array_t cf_finally; /* context for PEP 765 check */ int cf_finally_used; @@ -78,7 +79,7 @@ control_flow_in_finally_warning(const char *kw, stmt_ty n, _PyASTPreprocessState static int before_return(_PyASTPreprocessState *state, stmt_ty node_) { - if (state->cf_finally_used > 0) { + if (state->enable_warnings && state->cf_finally_used > 0) { ControlFlowInFinallyContext *ctx = get_cf_finally_top(state); if (ctx->in_finally && ! ctx->in_funcdef) { if (!control_flow_in_finally_warning("return", node_, state)) { @@ -92,7 +93,7 @@ before_return(_PyASTPreprocessState *state, stmt_ty node_) static int before_loop_exit(_PyASTPreprocessState *state, stmt_ty node_, const char *kw) { - if (state->cf_finally_used > 0) { + if (state->enable_warnings && state->cf_finally_used > 0) { ControlFlowInFinallyContext *ctx = get_cf_finally_top(state); if (ctx->in_finally && ! ctx->in_loop) { if (!control_flow_in_finally_warning(kw, node_, state)) { @@ -435,13 +436,38 @@ stmt_seq_remove_item(asdl_stmt_seq *stmts, Py_ssize_t idx) return 1; } +static int +remove_docstring(asdl_stmt_seq *stmts, Py_ssize_t idx, PyArena *ctx_) +{ + assert(_PyAST_GetDocString(stmts) != NULL); + // In case there's just the docstring in the body, replace it with `pass` + // keyword, so body won't be empty. + if (asdl_seq_LEN(stmts) == 1) { + stmt_ty docstring = (stmt_ty)asdl_seq_GET(stmts, 0); + stmt_ty pass = _PyAST_Pass( + docstring->lineno, docstring->col_offset, + // we know that `pass` always takes 4 chars and a single line, + // while docstring can span on multiple lines + docstring->lineno, docstring->col_offset + 4, + ctx_ + ); + if (pass == NULL) { + return 0; + } + asdl_seq_SET(stmts, 0, pass); + return 1; + } + // In case there are more than 1 body items, just remove the docstring. + return stmt_seq_remove_item(stmts, idx); +} + static int astfold_body(asdl_stmt_seq *stmts, PyArena *ctx_, _PyASTPreprocessState *state) { int docstring = _PyAST_GetDocString(stmts) != NULL; if (docstring && (state->optimize >= 2)) { /* remove the docstring */ - if (!stmt_seq_remove_item(stmts, 0)) { + if (!remove_docstring(stmts, 0, ctx_)) { return 0; } docstring = 0; @@ -943,7 +969,7 @@ astfold_type_param(type_param_ty node_, PyArena *ctx_, _PyASTPreprocessState *st int _PyAST_Preprocess(mod_ty mod, PyArena *arena, PyObject *filename, int optimize, - int ff_features, int syntax_check_only) + int ff_features, int syntax_check_only, int enable_warnings) { _PyASTPreprocessState state; memset(&state, 0, sizeof(_PyASTPreprocessState)); @@ -951,6 +977,7 @@ _PyAST_Preprocess(mod_ty mod, PyArena *arena, PyObject *filename, int optimize, state.optimize = optimize; state.ff_features = ff_features; state.syntax_check_only = syntax_check_only; + state.enable_warnings = enable_warnings; if (_Py_CArray_Init(&state.cf_finally, sizeof(ControlFlowInFinallyContext), 20) < 0) { return -1; } diff --git a/Python/ast_unparse.c b/Python/ast_unparse.c index c121ec096aebf45..c25699978cf6511 100644 --- a/Python/ast_unparse.c +++ b/Python/ast_unparse.c @@ -663,27 +663,15 @@ build_ftstring_body(asdl_expr_seq *values, bool is_format_spec) } static int -_write_values_subarray(PyUnicodeWriter *writer, asdl_expr_seq *values, Py_ssize_t first_idx, - Py_ssize_t last_idx, char prefix, PyArena *arena) +append_templatestr(PyUnicodeWriter *writer, expr_ty e) { int result = -1; - - asdl_expr_seq *new_values = _Py_asdl_expr_seq_new(last_idx - first_idx + 1, arena); - if (!new_values) { - return result; - } - - Py_ssize_t j = 0; - for (Py_ssize_t i = first_idx; i <= last_idx; ++i) { - asdl_seq_SET(new_values, j++, asdl_seq_GET(values, i)); - } - - PyObject *body = build_ftstring_body(new_values, false); + PyObject *body = build_ftstring_body(e->v.TemplateStr.values, false); if (!body) { - return result; + return -1; } - if (-1 != append_char(writer, prefix) && + if (-1 != append_charp(writer, "t") && -1 != append_repr(writer, body)) { result = 0; @@ -692,64 +680,6 @@ _write_values_subarray(PyUnicodeWriter *writer, asdl_expr_seq *values, Py_ssize_ return result; } -static int -append_templatestr(PyUnicodeWriter *writer, expr_ty e) -{ - PyArena *arena = _PyArena_New(); - if (!arena) { - return -1; - } - - Py_ssize_t last_idx = 0; - Py_ssize_t len = asdl_seq_LEN(e->v.TemplateStr.values); - for (Py_ssize_t i = 0; i < len; i++) { - expr_ty value = asdl_seq_GET(e->v.TemplateStr.values, i); - - // Handle implicit concat of t-strings with f-strings - if (value->kind == FormattedValue_kind) { - if (i > last_idx) { - // Create a new TemplateStr with the values between last_idx and i - // and append it to the writer. - if (_write_values_subarray(writer, e->v.TemplateStr.values, - last_idx, i - 1, 't', arena) == -1) { - goto error; - } - - if (append_charp(writer, " ") == -1) { - goto error; - } - } - - // Append the FormattedValue to the writer. - if (_write_values_subarray(writer, e->v.TemplateStr.values, - i, i, 'f', arena) == -1) { - goto error; - } - - if (i + 1 < len) { - if (append_charp(writer, " ") == -1) { - goto error; - } - } - - last_idx = i + 1; - } - } - - if (last_idx < len) { - if (_write_values_subarray(writer, e->v.TemplateStr.values, - last_idx, len - 1, 't', arena) == -1) { - goto error; - } - } - - return 0; - -error: - _PyArena_Free(arena); - return -1; -} - static int append_joinedstr(PyUnicodeWriter *writer, expr_ty e, bool is_format_spec) { @@ -774,32 +704,37 @@ append_joinedstr(PyUnicodeWriter *writer, expr_ty e, bool is_format_spec) } static int -append_interpolation_value(PyUnicodeWriter *writer, expr_ty e) +append_interpolation_str(PyUnicodeWriter *writer, PyObject *str) { const char *outer_brace = "{"; - /* Grammar allows PR_TUPLE, but use >PR_TEST for adding parenthesis - around a lambda with ':' */ - PyObject *temp_fv_str = expr_as_unicode(e, PR_TEST + 1); - if (!temp_fv_str) { - return -1; - } - if (PyUnicode_Find(temp_fv_str, _Py_LATIN1_CHR('{'), 0, 1, 1) == 0) { + if (PyUnicode_Find(str, _Py_LATIN1_CHR('{'), 0, 1, 1) == 0) { /* Expression starts with a brace, split it with a space from the outer one. */ outer_brace = "{ "; } if (-1 == append_charp(writer, outer_brace)) { - Py_DECREF(temp_fv_str); return -1; } - if (-1 == PyUnicodeWriter_WriteStr(writer, temp_fv_str)) { - Py_DECREF(temp_fv_str); + if (-1 == PyUnicodeWriter_WriteStr(writer, str)) { return -1; } - Py_DECREF(temp_fv_str); return 0; } +static int +append_interpolation_value(PyUnicodeWriter *writer, expr_ty e) +{ + /* Grammar allows PR_TUPLE, but use >PR_TEST for adding parenthesis + around a lambda with ':' */ + PyObject *temp_fv_str = expr_as_unicode(e, PR_TEST + 1); + if (!temp_fv_str) { + return -1; + } + int result = append_interpolation_str(writer, temp_fv_str); + Py_DECREF(temp_fv_str); + return result; +} + static int append_interpolation_conversion(PyUnicodeWriter *writer, int conversion) { @@ -843,7 +778,7 @@ append_interpolation_format_spec(PyUnicodeWriter *writer, expr_ty e) static int append_interpolation(PyUnicodeWriter *writer, expr_ty e) { - if (-1 == append_interpolation_value(writer, e->v.Interpolation.value)) { + if (-1 == append_interpolation_str(writer, e->v.Interpolation.str)) { return -1; } diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 3d0295ee3883f29..a8e9a17d656741f 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -272,15 +272,16 @@ should be a list of names to emulate ``from name import ...``, or an empty list to emulate ``import name``. When importing a module from a package, note that __import__('A.B', ...) returns package A when fromlist is empty, but its submodule B when -fromlist is not empty. The level argument is used to determine whether to -perform absolute or relative imports: 0 is absolute, while a positive number -is the number of parent directories to search relative to the current module. +fromlist is not empty. The level argument is used to determine whether +to perform absolute or relative imports: 0 is absolute, while a positive +number is the number of parent directories to search relative to the +current module. [clinic start generated code]*/ static PyObject * builtin___import___impl(PyObject *module, PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) -/*[clinic end generated code: output=4febeda88a0cd245 input=73f4b960ea5b9dd6]*/ +/*[clinic end generated code: output=4febeda88a0cd245 input=e3096a230383f72d]*/ { return PyImport_ImportModuleLevelObject(name, globals, locals, fromlist, level); @@ -634,8 +635,9 @@ PyDoc_STRVAR(filter_doc, "filter(function, iterable, /)\n\ --\n\ \n\ -Return an iterator yielding those items of iterable for which function(item)\n\ -is true. If function is None, return the items that are true."); +Return an iterator yielding those items of iterable for which\n\ +function(item) is true. If function is None, return the items that\n\ +are true."); PyTypeObject PyFilter_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) @@ -745,7 +747,7 @@ builtin_chr(PyObject *module, PyObject *i) compile as builtin_compile source: object - filename: object(converter="PyUnicode_FSDecoder") + filename: unicode_fs_decoded mode: str flags: int = 0 dont_inherit: bool = False @@ -755,23 +757,24 @@ compile as builtin_compile Compile source into a code object that can be executed by exec() or eval(). -The source code may represent a Python module, statement or expression. +The source code may represent a Python module, statement or +expression. The filename will be used for run-time error messages. The mode must be 'exec' to compile a module, 'single' to compile a single (interactive) statement, or 'eval' to compile an expression. -The flags argument, if present, controls which future statements influence -the compilation of the code. +The flags argument, if present, controls which future statements +influence the compilation of the code. The dont_inherit argument, if true, stops the compilation inheriting the effects of any future statements in effect in the code calling -compile; if absent or false these statements do influence the compilation, -in addition to any features explicitly specified. +compile; if absent or false these statements do influence the +compilation, in addition to any features explicitly specified. [clinic start generated code]*/ static PyObject * builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, const char *mode, int flags, int dont_inherit, int optimize, int feature_version) -/*[clinic end generated code: output=b0c09c84f116d3d7 input=cc78e20e7c7682ba]*/ +/*[clinic end generated code: output=b0c09c84f116d3d7 input=d69ec2180ff24031]*/ { PyObject *source_copy; const char *str; @@ -889,7 +892,6 @@ builtin_compile_impl(PyObject *module, PyObject *source, PyObject *filename, error: result = NULL; finally: - Py_DECREF(filename); return result; } @@ -908,10 +910,10 @@ PyDoc_STRVAR(dir_doc, "dir([object]) -> list of strings\n" "\n" "If called without an argument, return the names in the current scope.\n" -"Else, return an alphabetized list of names comprising (some of) the attributes\n" -"of the given object, and of attributes reachable from it.\n" -"If the object supplies a method named __dir__, it will be used; otherwise\n" -"the default dir() logic is used and returns:\n" +"Else, return an alphabetized list of names comprising (some of) the\n" +"attributes of the given object, and of attributes reachable from it.\n" +"If the object supplies a method named __dir__, it will be used;\n" +"otherwise the default dir() logic is used and returns:\n" " for a module object: the module's attributes.\n" " for a class object: its attributes, and recursively the attributes\n" " of its bases.\n" @@ -958,6 +960,7 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals) /*[clinic end generated code: output=0a0824aa70093116 input=7c7bce5299a89062]*/ { + PyThreadState *tstate = _PyThreadState_GET(); PyObject *result = NULL, *source_copy; const char *str; @@ -971,35 +974,46 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, : "globals must be a dict"); return NULL; } - if (globals == Py_None) { + + int fromframe = 0; + if (globals != Py_None) { + Py_INCREF(globals); + } + else if (_PyEval_GetFrame() != NULL) { + fromframe = 1; globals = PyEval_GetGlobals(); - if (locals == Py_None) { - locals = _PyEval_GetFrameLocals(); - if (locals == NULL) - return NULL; - } - else { - Py_INCREF(locals); - } + assert(globals != NULL); + Py_INCREF(globals); } - else if (locals == Py_None) - locals = Py_NewRef(globals); else { - Py_INCREF(locals); + globals = _PyEval_GetGlobalsFromRunningMain(tstate); + if (globals == NULL) { + if (!_PyErr_Occurred(tstate)) { + PyErr_SetString(PyExc_TypeError, + "eval must be given globals and locals " + "when called without a frame"); + } + return NULL; + } + Py_INCREF(globals); } - if (globals == NULL || locals == NULL) { - PyErr_SetString(PyExc_TypeError, - "eval must be given globals and locals " - "when called without a frame"); - goto error; + if (locals != Py_None) { + Py_INCREF(locals); } - - int r = PyDict_Contains(globals, &_Py_ID(__builtins__)); - if (r == 0) { - r = PyDict_SetItem(globals, &_Py_ID(__builtins__), PyEval_GetBuiltins()); + else if (fromframe) { + locals = _PyEval_GetFrameLocals(); + if (locals == NULL) { + assert(PyErr_Occurred()); + Py_DECREF(globals); + return NULL; + } } - if (r < 0) { + else { + locals = Py_NewRef(globals); + } + + if (_PyEval_EnsureBuiltins(tstate, globals, NULL) < 0) { goto error; } @@ -1040,6 +1054,7 @@ builtin_eval_impl(PyObject *module, PyObject *source, PyObject *globals, } error: + Py_XDECREF(globals); Py_XDECREF(locals); return result; } @@ -1070,29 +1085,44 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, PyObject *locals, PyObject *closure) /*[clinic end generated code: output=7579eb4e7646743d input=25e989b6d87a3a21]*/ { + PyThreadState *tstate = _PyThreadState_GET(); PyObject *v; - if (globals == Py_None) { + int fromframe = 0; + if (globals != Py_None) { + Py_INCREF(globals); + } + else if (_PyEval_GetFrame() != NULL) { + fromframe = 1; globals = PyEval_GetGlobals(); - if (locals == Py_None) { - locals = _PyEval_GetFrameLocals(); - if (locals == NULL) - return NULL; - } - else { - Py_INCREF(locals); + assert(globals != NULL); + Py_INCREF(globals); + } + else { + globals = _PyEval_GetGlobalsFromRunningMain(tstate); + if (globals == NULL) { + if (!_PyErr_Occurred(tstate)) { + PyErr_SetString(PyExc_SystemError, + "globals and locals cannot be NULL"); + } + goto error; } - if (!globals || !locals) { - PyErr_SetString(PyExc_SystemError, - "globals and locals cannot be NULL"); + Py_INCREF(globals); + } + + if (locals != Py_None) { + Py_INCREF(locals); + } + else if (fromframe) { + locals = _PyEval_GetFrameLocals(); + if (locals == NULL) { + assert(PyErr_Occurred()); + Py_DECREF(globals); return NULL; } } - else if (locals == Py_None) { - locals = Py_NewRef(globals); - } else { - Py_INCREF(locals); + locals = Py_NewRef(globals); } if (!PyDict_Check(globals)) { @@ -1106,11 +1136,8 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, Py_TYPE(locals)->tp_name); goto error; } - int r = PyDict_Contains(globals, &_Py_ID(__builtins__)); - if (r == 0) { - r = PyDict_SetItem(globals, &_Py_ID(__builtins__), PyEval_GetBuiltins()); - } - if (r < 0) { + + if (_PyEval_EnsureBuiltins(tstate, globals, NULL) < 0) { goto error; } @@ -1187,11 +1214,13 @@ builtin_exec_impl(PyObject *module, PyObject *source, PyObject *globals, } if (v == NULL) goto error; + Py_DECREF(globals); Py_DECREF(locals); Py_DECREF(v); Py_RETURN_NONE; error: + Py_XDECREF(globals); Py_XDECREF(locals); return NULL; } @@ -1223,9 +1252,11 @@ builtin_getattr(PyObject *self, PyObject *const *args, Py_ssize_t nargs) PyDoc_STRVAR(getattr_doc, "getattr(object, name[, default]) -> value\n\ \n\ -Get a named attribute from an object; getattr(x, 'y') is equivalent to x.y.\n\ -When a default argument is given, it is returned when the attribute doesn't\n\ -exist; without it, an exception is raised in that case."); +Get a named attribute from an object.\n\ +\n\ +getattr(x, 'y') is equivalent to x.y.\n\ +When a default argument is given, it is returned when the attribute\n\ +doesn't exist; without it, an exception is raised in that case."); /*[clinic input] @@ -1233,18 +1264,29 @@ globals as builtin_globals Return the dictionary containing the current scope's global variables. -NOTE: Updates to this dictionary *will* affect name lookups in the current -global scope and vice-versa. +NOTE: Updates to this dictionary *will* affect name lookups in the +current global scope and vice-versa. [clinic start generated code]*/ static PyObject * builtin_globals_impl(PyObject *module) -/*[clinic end generated code: output=e5dd1527067b94d2 input=9327576f92bb48ba]*/ +/*[clinic end generated code: output=e5dd1527067b94d2 input=6d725a9b48d1eaeb]*/ { - PyObject *d; - - d = PyEval_GetGlobals(); - return Py_XNewRef(d); + PyObject *globals; + if (_PyEval_GetFrame() != NULL) { + globals = PyEval_GetGlobals(); + assert(globals != NULL); + return Py_NewRef(globals); + } + PyThreadState *tstate = _PyThreadState_GET(); + globals = _PyEval_GetGlobalsFromRunningMain(tstate); + if (globals == NULL) { + if (_PyErr_Occurred(tstate)) { + return NULL; + } + Py_RETURN_NONE; + } + return Py_NewRef(globals); } @@ -1464,34 +1506,27 @@ map_next(PyObject *self) } Py_ssize_t nargs = 0; - for (i=0; i < niters; i++) { + for (i = 0; i < niters; i++) { PyObject *it = PyTuple_GET_ITEM(lz->iters, i); PyObject *val = Py_TYPE(it)->tp_iternext(it); if (val == NULL) { if (lz->strict) { goto check; } - goto exit; + goto exit_no_result; } stack[i] = val; nargs++; } result = _PyObject_VectorcallTstate(tstate, lz->func, stack, nargs, NULL); + goto exit; -exit: - for (i=0; i < nargs; i++) { - Py_DECREF(stack[i]); - } - if (stack != small_stack) { - PyMem_Free(stack); - } - return result; check: if (PyErr_Occurred()) { if (!PyErr_ExceptionMatches(PyExc_StopIteration)) { // next() on argument i raised an exception (not StopIteration) - return NULL; + goto exit_no_result; } PyErr_Clear(); } @@ -1499,9 +1534,10 @@ map_next(PyObject *self) // ValueError: map() argument 2 is shorter than argument 1 // ValueError: map() argument 3 is shorter than arguments 1-2 const char* plural = i == 1 ? " " : "s 1-"; - return PyErr_Format(PyExc_ValueError, - "map() argument %d is shorter than argument%s%d", - i + 1, plural, i); + PyErr_Format(PyExc_ValueError, + "map() argument %zd is shorter than argument%s%zd", + i + 1, plural, i); + goto exit_no_result; } for (i = 1; i < niters; i++) { PyObject *it = PyTuple_GET_ITEM(lz->iters, i); @@ -1509,21 +1545,33 @@ map_next(PyObject *self) if (val) { Py_DECREF(val); const char* plural = i == 1 ? " " : "s 1-"; - return PyErr_Format(PyExc_ValueError, - "map() argument %d is longer than argument%s%d", - i + 1, plural, i); + PyErr_Format(PyExc_ValueError, + "map() argument %zd is longer than argument%s%zd", + i + 1, plural, i); + goto exit_no_result; } if (PyErr_Occurred()) { if (!PyErr_ExceptionMatches(PyExc_StopIteration)) { // next() on argument i raised an exception (not StopIteration) - return NULL; + goto exit_no_result; } PyErr_Clear(); } // Argument i is exhausted. So far so good... } // All arguments are exhausted. Success! - goto exit; + +exit_no_result: + assert(result == NULL); + +exit: + for (i = 0; i < nargs; i++) { + Py_DECREF(stack[i]); + } + if (stack != small_stack) { + PyMem_Free(stack); + } + return result; } static PyObject * @@ -1575,8 +1623,8 @@ PyDoc_STRVAR(map_doc, Make an iterator that computes the function using arguments from\n\ each of the iterables. Stops when the shortest iterable is exhausted.\n\ \n\ -If strict is true and one of the arguments is exhausted before the others,\n\ -raise a ValueError."); +If strict is true and one of the arguments is exhausted before the\n\ +others, raise a ValueError."); PyTypeObject PyMap_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) @@ -1663,8 +1711,8 @@ builtin_next(PyObject *self, PyObject *const *args, Py_ssize_t nargs) PyDoc_STRVAR(next_doc, "next(iterator[, default])\n\ \n\ -Return the next item from the iterator. If default is given and the iterator\n\ -is exhausted, it is returned instead of raising StopIteration."); +Return the next item from the iterator. If default is given and the\n\ +iterator is exhausted, it is returned instead of raising StopIteration."); /*[clinic input] @@ -1720,15 +1768,17 @@ hash as builtin_hash obj: object / -Return the hash value for the given object. +Return the integer hash value for the given object. -Two objects that compare equal must also have the same hash value, but the -reverse is not necessarily true. +Two objects that compare equal must also have the same hash value, but +the reverse is not necessarily true. Hash values may differ between +Python processes. Not all objects are hashable; calling hash() on an +unhashable object raises TypeError. [clinic start generated code]*/ static PyObject * builtin_hash(PyObject *module, PyObject *obj) -/*[clinic end generated code: output=237668e9d7688db7 input=58c48be822bf9c54]*/ +/*[clinic end generated code: output=237668e9d7688db7 input=70a242ff65f6717c]*/ { Py_hash_t x; @@ -1785,7 +1835,8 @@ iter(callable, sentinel) -> iterator\n\ \n\ Get an iterator from an object. In the first form, the argument must\n\ supply its own iterator, or be a sequence.\n\ -In the second form, the callable is called until it returns the sentinel."); +In the second form, the callable is called until it returns the\n\ +sentinel."); /*[clinic input] @@ -1879,16 +1930,31 @@ locals as builtin_locals Return a dictionary containing the current scope's local variables. -NOTE: Whether or not updates to this dictionary will affect name lookups in -the local scope and vice-versa is *implementation dependent* and not -covered by any backwards compatibility guarantees. +NOTE: Whether or not updates to this dictionary will affect name +lookups in the local scope and vice-versa is *implementation +dependent* and not covered by any backwards compatibility +guarantees. [clinic start generated code]*/ static PyObject * builtin_locals_impl(PyObject *module) -/*[clinic end generated code: output=b46c94015ce11448 input=7874018d478d5c4b]*/ +/*[clinic end generated code: output=b46c94015ce11448 input=989cc75c22167c42]*/ { - return _PyEval_GetFrameLocals(); + PyObject *locals; + if (_PyEval_GetFrame() != NULL) { + locals = _PyEval_GetFrameLocals(); + assert(locals != NULL || PyErr_Occurred()); + return locals; + } + PyThreadState *tstate = _PyThreadState_GET(); + locals = _PyEval_GetGlobalsFromRunningMain(tstate); + if (locals == NULL) { + if (_PyErr_Occurred(tstate)) { + return NULL; + } + Py_RETURN_NONE; + } + return Py_NewRef(locals); } @@ -2067,15 +2133,21 @@ builtin_oct(PyObject *module, PyObject *number) /*[clinic input] ord as builtin_ord - c: object + character as c: object / -Return the Unicode code point for a one-character string. +Return the ordinal value of a character. + +If the argument is a one-character string, return the Unicode code +point of that character. + +If the argument is a bytes or bytearray object of length 1, return its +single byte value. [clinic start generated code]*/ static PyObject * builtin_ord(PyObject *module, PyObject *c) -/*[clinic end generated code: output=4fa5e87a323bae71 input=3064e5d6203ad012]*/ +/*[clinic end generated code: output=4fa5e87a323bae71 input=98d38480432e1177]*/ { long ord; Py_ssize_t size; @@ -2126,14 +2198,14 @@ pow as builtin_pow Equivalent to base**exp with 2 arguments or base**exp % mod with 3 arguments -Some types, such as ints, are able to use a more efficient algorithm when -invoked using the three argument form. +Some types, such as ints, are able to use a more efficient algorithm +when invoked using the three argument form. [clinic start generated code]*/ static PyObject * builtin_pow_impl(PyObject *module, PyObject *base, PyObject *exp, PyObject *mod) -/*[clinic end generated code: output=3ca1538221bbf15f input=435dbd48a12efb23]*/ +/*[clinic end generated code: output=3ca1538221bbf15f input=6133ded72c7db82e]*/ { return PyNumber_Power(base, exp, mod); } @@ -2254,13 +2326,14 @@ Read a string from standard input. The trailing newline is stripped. The prompt string, if given, is printed to standard output without a trailing newline before reading input. -If the user hits EOF (*nix: Ctrl-D, Windows: Ctrl-Z+Return), raise EOFError. +If the user hits EOF (*nix: Ctrl-D, Windows: Ctrl-Z+Return), raise +EOFError. On *nix systems, readline is used if available. [clinic start generated code]*/ static PyObject * builtin_input_impl(PyObject *module, PyObject *prompt) -/*[clinic end generated code: output=83db5a191e7a0d60 input=159c46d4ae40977e]*/ +/*[clinic end generated code: output=83db5a191e7a0d60 input=ebb939c954639427]*/ { PyObject *fin = NULL; PyObject *fout = NULL; @@ -2528,13 +2601,14 @@ round as builtin_round Round a number to a given precision in decimal digits. -The return value is an integer if ndigits is omitted or None. Otherwise -the return value has the same type as the number. ndigits may be negative. +The return value is an integer if ndigits is omitted or None. +Otherwise the return value has the same type as the number. ndigits +may be negative. [clinic start generated code]*/ static PyObject * builtin_round_impl(PyObject *module, PyObject *number, PyObject *ndigits) -/*[clinic end generated code: output=ff0d9dd176c02ede input=275678471d7aca15]*/ +/*[clinic end generated code: output=ff0d9dd176c02ede input=bdcb7c67bf4a4320]*/ { PyObject *result; if (ndigits == Py_None) { @@ -2566,8 +2640,8 @@ sorted as builtin_sorted Return a new list containing all items from the iterable in ascending order. -A custom key function can be supplied to customize the sort order, and the -reverse flag can be set to request the result in descending order. +A custom key function can be supplied to customize the sort order, and +the reverse flag can be set to request the result in descending order. [end disabled clinic input]*/ PyDoc_STRVAR(builtin_sorted__doc__, @@ -2624,7 +2698,22 @@ builtin_vars(PyObject *self, PyObject *args) if (!PyArg_UnpackTuple(args, "vars", 0, 1, &v)) return NULL; if (v == NULL) { - d = _PyEval_GetFrameLocals(); + if (_PyEval_GetFrame() != NULL) { + d = _PyEval_GetFrameLocals(); + } + else { + PyThreadState *tstate = _PyThreadState_GET(); + d = _PyEval_GetGlobalsFromRunningMain(tstate); + if (d == NULL) { + if (!_PyErr_Occurred(tstate)) { + d = _PyEval_GetFrameLocals(); + assert(_PyErr_Occurred(tstate)); + } + } + else { + Py_INCREF(d); + } + } } else { if (PyObject_GetOptionalAttr(v, &_Py_ID(__dict__), &d) == 0) { @@ -2695,13 +2784,13 @@ sum as builtin_sum Return the sum of a 'start' value (default: 0) plus an iterable of numbers When the iterable is empty, return the start value. -This function is intended specifically for use with numeric values and may -reject non-numeric types. +This function is intended specifically for use with numeric values and +may reject non-numeric types. [clinic start generated code]*/ static PyObject * builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) -/*[clinic end generated code: output=df758cec7d1d302f input=162b50765250d222]*/ +/*[clinic end generated code: output=df758cec7d1d302f input=83d1bc7fd7f7ad3b]*/ { PyObject *result = start; PyObject *temp, *item, *iter; @@ -2945,15 +3034,15 @@ isinstance as builtin_isinstance Return whether an object is an instance of a class or of a subclass thereof. -A tuple, as in ``isinstance(x, (A, B, ...))``, may be given as the target to -check against. This is equivalent to ``isinstance(x, A) or isinstance(x, B) -or ...`` etc. +A tuple, as in ``isinstance(x, (A, B, ...))``, may be given as the +target to check against. This is equivalent to ``isinstance(x, A) or +isinstance(x, B) or ...`` etc. [clinic start generated code]*/ static PyObject * builtin_isinstance_impl(PyObject *module, PyObject *obj, PyObject *class_or_tuple) -/*[clinic end generated code: output=6faf01472c13b003 input=ffa743db1daf7549]*/ +/*[clinic end generated code: output=6faf01472c13b003 input=477d04768621c24b]*/ { int retval; @@ -2973,15 +3062,15 @@ issubclass as builtin_issubclass Return whether 'cls' is derived from another class or is the same class. -A tuple, as in ``issubclass(x, (A, B, ...))``, may be given as the target to -check against. This is equivalent to ``issubclass(x, A) or issubclass(x, B) -or ...``. +A tuple, as in ``issubclass(x, (A, B, ...))``, may be given as the +target to check against. This is equivalent to ``issubclass(x, A) or +issubclass(x, B) or ...``. [clinic start generated code]*/ static PyObject * builtin_issubclass_impl(PyObject *module, PyObject *cls, PyObject *class_or_tuple) -/*[clinic end generated code: output=358412410cd7a250 input=a24b9f3d58c370d6]*/ +/*[clinic end generated code: output=358412410cd7a250 input=a91ce96345a6705d]*/ { int retval; @@ -3152,7 +3241,7 @@ zip_next(PyObject *self) // ValueError: zip() argument 3 is shorter than arguments 1-2 const char* plural = i == 1 ? " " : "s 1-"; return PyErr_Format(PyExc_ValueError, - "zip() argument %d is shorter than argument%s%d", + "zip() argument %zd is shorter than argument%s%zd", i + 1, plural, i); } for (i = 1; i < tuplesize; i++) { @@ -3162,7 +3251,7 @@ zip_next(PyObject *self) Py_DECREF(item); const char* plural = i == 1 ? " " : "s 1-"; return PyErr_Format(PyExc_ValueError, - "zip() argument %d is longer than argument%s%d", + "zip() argument %zd is longer than argument%s%zd", i + 1, plural, i); } if (PyErr_Occurred()) { @@ -3211,13 +3300,13 @@ PyDoc_STRVAR(zip_doc, "zip(*iterables, strict=False)\n\ --\n\ \n\ -The zip object yields n-length tuples, where n is the number of iterables\n\ -passed as positional arguments to zip(). The i-th element in every tuple\n\ -comes from the i-th iterable argument to zip(). This continues until the\n\ -shortest argument is exhausted.\n\ +The zip object yields n-length tuples, where n is the number of\n\ +iterables passed as positional arguments to zip(). The i-th element\n\ +in every tuple comes from the i-th iterable argument to zip(). This\n\ +continues until the shortest argument is exhausted.\n\ \n\ -If strict is true and one of the arguments is exhausted before the others,\n\ -raise a ValueError.\n\ +If strict is true and one of the arguments is exhausted before the\n\ +others, raise a ValueError.\n\ \n\ >>> list(zip('abcdefg', range(3), range(4)))\n\ [('a', 0, 0), ('b', 1, 1), ('c', 2, 2)]"); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6a0766626402b09..509a5a6e13f8570 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -181,7 +181,15 @@ dummy_func( } tier1 op(_MAYBE_INSTRUMENT, (--)) { - if (tstate->tracing == 0) { + #ifdef Py_GIL_DISABLED + // For thread-safety, we need to check instrumentation version + // even when tracing. Otherwise, another thread may concurrently + // re-write the bytecode while we are executing this function. + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { @@ -785,9 +793,12 @@ dummy_func( PyObject *temp = PyStackRef_AsPyObjectSteal(*target_local); PyObject *right_o = PyStackRef_AsPyObjectSteal(right); PyUnicode_Append(&temp, right_o); - *target_local = PyStackRef_FromPyObjectSteal(temp); Py_DECREF(right_o); - ERROR_IF(PyStackRef_IsNull(*target_local)); + if (temp == NULL) { + *target_local = PyStackRef_NULL; + ERROR_IF(true); + } + *target_local = PyStackRef_FromPyObjectSteal(temp); #if TIER_ONE // The STORE_FAST is already done. This is done here in tier one, // and during trace projection in tier two: @@ -816,6 +827,7 @@ dummy_func( PyObject *res_o = d->action(left_o, right_o); DECREF_INPUTS(); + ERROR_IF(res_o == NULL); res = PyStackRef_FromPyObjectSteal(res_o); } @@ -1376,13 +1388,12 @@ dummy_func( gen_frame->previous = NULL; /* We don't know which of these is relevant here, so keep them equal */ assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #if TIER_ONE && defined(Py_DEBUG) + if (!PyStackRef_IsNone(frame->f_executable)) { + int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + assert(opcode == SEND || opcode == FOR_ITER); + } #endif RELOAD_STACK(); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); @@ -2283,39 +2294,19 @@ dummy_func( op(_LOAD_ATTR, (owner -- attr, self_or_null[oparg&1])) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - PyObject *attr_o; if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ - attr_o = NULL; - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); - if (is_meth) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */ - assert(attr_o != NULL); // No errors on this branch - self_or_null[0] = owner; // Transfer ownership - DEAD(owner); - } - else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - meth | NULL | arg1 | ... | argN - */ - PyStackRef_CLOSE(owner); - ERROR_IF(attr_o == NULL); - self_or_null[0] = PyStackRef_NULL; - } + attr = _Py_LoadAttr_StackRefSteal(tstate, owner, name, self_or_null); + DEAD(owner); + ERROR_IF(PyStackRef_IsNull(attr)); } else { /* Classic, pushes one value. */ - attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); PyStackRef_CLOSE(owner); ERROR_IF(attr_o == NULL); + attr = PyStackRef_FromPyObjectSteal(attr_o); } - attr = PyStackRef_FromPyObjectSteal(attr_o); } macro(LOAD_ATTR) = @@ -3155,7 +3146,14 @@ dummy_func( replaced op(_FOR_ITER, (iter -- iter, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + iternextfunc func = Py_TYPE(iter_o)->tp_iternext; + if (func == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.100s' object is not an iterator", + Py_TYPE(iter_o)->tp_name); + ERROR_NO_POP(); + } + PyObject *next_o = func(iter_o); if (next_o == NULL) { if (_PyErr_Occurred(tstate)) { int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); @@ -3179,7 +3177,14 @@ dummy_func( op(_FOR_ITER_TIER_TWO, (iter -- iter, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + iternextfunc func = Py_TYPE(iter_o)->tp_iternext; + if (func == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.100s' object is not an iterator", + Py_TYPE(iter_o)->tp_name); + ERROR_NO_POP(); + } + PyObject *next_o = func(iter_o); if (next_o == NULL) { if (_PyErr_Occurred(tstate)) { int matches = _PyErr_ExceptionMatches(tstate, PyExc_StopIteration); @@ -3202,7 +3207,14 @@ dummy_func( inst(INSTRUMENTED_FOR_ITER, (unused/1, iter -- iter, next)) { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + iternextfunc func = Py_TYPE(iter_o)->tp_iternext; + if (func == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.100s' object is not an iterator", + Py_TYPE(iter_o)->tp_name); + ERROR_NO_POP(); + } + PyObject *next_o = func(iter_o); if (next_o != NULL) { next = PyStackRef_FromPyObjectSteal(next_o); INSTRUMENTED_JUMP(this_instr, next_instr, PY_MONITORING_EVENT_BRANCH_LEFT); @@ -4719,6 +4731,7 @@ dummy_func( unused/1 + // Skip over the counter _CHECK_PEP_523 + _CHECK_FUNCTION_VERSION_KW + + _CHECK_RECURSION_REMAINING + _PY_FRAME_KW + _SAVE_RETURN_OFFSET + _PUSH_FRAME; @@ -4956,6 +4969,13 @@ dummy_func( PyObject **ptr = (PyObject **)(((char *)func) + offset); assert(*ptr == NULL); *ptr = attr; + if (oparg == MAKE_FUNCTION_ANNOTATE && PyFunction_Check(attr)) { + // gh-137814: Fix the qualname of __annotate__ functions + PyFunctionObject *func_obj = (PyFunctionObject *)attr; + PyObject *fixed_qualname = PyUnicode_FromFormat("%U.__annotate__", ((PyFunctionObject *)func)->func_qualname); + ERROR_IF(fixed_qualname == NULL); + Py_SETREF(func_obj->func_qualname, fixed_qualname); + } } inst(RETURN_GENERATOR, (-- res)) { diff --git a/Python/ceval.c b/Python/ceval.c index 490b653f132a6a3..b3dab2e9da5a716 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -8,7 +8,7 @@ #include "pycore_backoff.h" #include "pycore_call.h" // _PyObject_CallNoArgs() #include "pycore_cell.h" // PyCell_GetRef() -#include "pycore_ceval.h" +#include "pycore_ceval.h" // SPECIAL___ENTER__ #include "pycore_code.h" #include "pycore_dict.h" #include "pycore_emscripten_signal.h" // _Py_CHECK_EMSCRIPTEN_SIGNALS @@ -333,13 +333,23 @@ _Py_ReachedRecursionLimitWithMargin(PyThreadState *tstate, int margin_count) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; - if (here_addr > _tstate->c_stack_soft_limit + margin_count * PYOS_STACK_MARGIN_BYTES) { +#if _Py_STACK_GROWS_DOWN + if (here_addr > _tstate->c_stack_soft_limit + margin_count * _PyOS_STACK_MARGIN_BYTES) { +#else + if (here_addr <= _tstate->c_stack_soft_limit - margin_count * _PyOS_STACK_MARGIN_BYTES) { +#endif return 0; } if (_tstate->c_stack_hard_limit == 0) { _Py_InitializeRecursionLimits(tstate); } - return here_addr <= _tstate->c_stack_soft_limit + margin_count * PYOS_STACK_MARGIN_BYTES; +#if _Py_STACK_GROWS_DOWN + return here_addr <= _tstate->c_stack_soft_limit + margin_count * _PyOS_STACK_MARGIN_BYTES && + here_addr >= _tstate->c_stack_soft_limit - 2 * _PyOS_STACK_MARGIN_BYTES; +#else + return here_addr > _tstate->c_stack_soft_limit - margin_count * _PyOS_STACK_MARGIN_BYTES && + here_addr <= _tstate->c_stack_soft_limit + 2 * _PyOS_STACK_MARGIN_BYTES; +#endif } void @@ -347,12 +357,18 @@ _Py_EnterRecursiveCallUnchecked(PyThreadState *tstate) { uintptr_t here_addr = _Py_get_machine_stack_pointer(); _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; +#if _Py_STACK_GROWS_DOWN if (here_addr < _tstate->c_stack_hard_limit) { +#else + if (here_addr > _tstate->c_stack_hard_limit) { +#endif Py_FatalError("Unchecked stack overflow."); } } -#if defined(__s390x__) +#if defined(_Py_LINKER_THREAD_STACK_SIZE) +# define Py_C_STACK_SIZE _Py_LINKER_THREAD_STACK_SIZE +#elif defined(__s390x__) # define Py_C_STACK_SIZE 320000 #elif defined(_WIN32) // Don't define Py_C_STACK_SIZE, ask the O/S @@ -360,9 +376,6 @@ _Py_EnterRecursiveCallUnchecked(PyThreadState *tstate) # define Py_C_STACK_SIZE 1200000 #elif defined(__sparc__) # define Py_C_STACK_SIZE 1600000 -#elif defined(__wasi__) - /* Web assembly has two stacks, so this isn't really the stack depth */ -# define Py_C_STACK_SIZE 131072 // wasi-libc DEFAULT_STACK_SIZE #elif defined(__hppa__) || defined(__powerpc64__) # define Py_C_STACK_SIZE 2000000 #else @@ -427,22 +440,28 @@ int pthread_attr_destroy(pthread_attr_t *a) #endif - -void -_Py_InitializeRecursionLimits(PyThreadState *tstate) +static void +hardware_stack_limits(uintptr_t *base, uintptr_t *top, uintptr_t sp) { - _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; #ifdef WIN32 ULONG_PTR low, high; GetCurrentThreadStackLimits(&low, &high); - _tstate->c_stack_top = (uintptr_t)high; + *top = (uintptr_t)high; ULONG guarantee = 0; SetThreadStackGuarantee(&guarantee); - _tstate->c_stack_hard_limit = ((uintptr_t)low) + guarantee + PYOS_STACK_MARGIN_BYTES; - _tstate->c_stack_soft_limit = _tstate->c_stack_hard_limit + PYOS_STACK_MARGIN_BYTES; + *base = (uintptr_t)low + guarantee; +#elif defined(__APPLE__) + pthread_t this_thread = pthread_self(); + void *stack_addr = pthread_get_stackaddr_np(this_thread); // top of the stack + size_t stack_size = pthread_get_stacksize_np(this_thread); + *top = (uintptr_t)stack_addr; + *base = ((uintptr_t)stack_addr) - stack_size; #else - uintptr_t here_addr = _Py_get_machine_stack_pointer(); -# if defined(HAVE_PTHREAD_GETATTR_NP) && !defined(_AIX) && !defined(__NetBSD__) + /// XXX musl supports HAVE_PTHRED_GETATTR_NP, but the resulting stack size + /// (on alpine at least) is much smaller than expected and imposes undue limits + /// compared to the old stack size estimation. (We assume musl is not glibc.) +# if defined(HAVE_PTHREAD_GETATTR_NP) && !defined(_AIX) && \ + !defined(__NetBSD__) && (defined(__GLIBC__) || !defined(__linux__)) size_t stack_size, guard_size; void *stack_addr; pthread_attr_t attr; @@ -453,28 +472,118 @@ _Py_InitializeRecursionLimits(PyThreadState *tstate) err |= pthread_attr_destroy(&attr); } if (err == 0) { - uintptr_t base = ((uintptr_t)stack_addr) + guard_size; - _tstate->c_stack_top = base + stack_size; + *base = ((uintptr_t)stack_addr) + guard_size; + *top = (uintptr_t)stack_addr + stack_size; + return; + } +# endif + // Add some space for caller function then round to minimum page size + // This is a guess at the top of the stack, but should be a reasonably + // good guess if called from _PyThreadState_Attach when creating a thread. + // If the thread is attached deep in a call stack, then the guess will be poor. +#if _Py_STACK_GROWS_DOWN + uintptr_t top_addr = _Py_SIZE_ROUND_UP(sp + 8*sizeof(void*), SYSTEM_PAGE_SIZE); + *top = top_addr; + *base = top_addr - Py_C_STACK_SIZE; +# else + uintptr_t base_addr = _Py_SIZE_ROUND_DOWN(sp - 8*sizeof(void*), SYSTEM_PAGE_SIZE); + *base = base_addr; + *top = base_addr + Py_C_STACK_SIZE; +#endif +#endif +} + +static void +tstate_set_stack(PyThreadState *tstate, + uintptr_t base, uintptr_t top) +{ + assert(base < top); + assert((top - base) >= _PyOS_MIN_STACK_SIZE); + #ifdef _Py_THREAD_SANITIZER - // Thread sanitizer crashes if we use a bit more than half the stack. - _tstate->c_stack_soft_limit = base + (stack_size / 2); + // Thread sanitizer crashes if we use more than half the stack. + uintptr_t stacksize = top - base; +# if _Py_STACK_GROWS_DOWN + base += stacksize / 2; +# else + top -= stacksize / 2; +# endif +#endif + _PyThreadStateImpl *_tstate = (_PyThreadStateImpl *)tstate; +#if _Py_STACK_GROWS_DOWN + _tstate->c_stack_top = top; + _tstate->c_stack_hard_limit = base + _PyOS_STACK_MARGIN_BYTES; + _tstate->c_stack_soft_limit = base + _PyOS_STACK_MARGIN_BYTES * 2; +# ifndef NDEBUG + // Sanity checks + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(ts->c_stack_hard_limit <= ts->c_stack_soft_limit); + assert(ts->c_stack_soft_limit < ts->c_stack_top); +# endif #else - _tstate->c_stack_soft_limit = base + PYOS_STACK_MARGIN_BYTES * 2; + _tstate->c_stack_top = base; + _tstate->c_stack_hard_limit = top - _PyOS_STACK_MARGIN_BYTES; + _tstate->c_stack_soft_limit = top - _PyOS_STACK_MARGIN_BYTES * 2; +# ifndef NDEBUG + // Sanity checks + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + assert(ts->c_stack_hard_limit >= ts->c_stack_soft_limit); + assert(ts->c_stack_soft_limit > ts->c_stack_top); +# endif #endif - _tstate->c_stack_hard_limit = base + PYOS_STACK_MARGIN_BYTES; - assert(_tstate->c_stack_soft_limit < here_addr); - assert(here_addr < _tstate->c_stack_top); +} + + +void +_Py_InitializeRecursionLimits(PyThreadState *tstate) +{ + uintptr_t base, top; + uintptr_t here_addr = _Py_get_machine_stack_pointer(); + hardware_stack_limits(&base, &top, here_addr); + assert(top != 0); + + tstate_set_stack(tstate, base, top); + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + ts->c_stack_init_base = base; + ts->c_stack_init_top = top; +} + + +int +PyUnstable_ThreadState_SetStackProtection(PyThreadState *tstate, + void *stack_start_addr, size_t stack_size) +{ + if (stack_size < _PyOS_MIN_STACK_SIZE) { + PyErr_Format(PyExc_ValueError, + "stack_size must be at least %zu bytes", + _PyOS_MIN_STACK_SIZE); + return -1; + } + + uintptr_t base = (uintptr_t)stack_start_addr; + uintptr_t top = base + stack_size; + tstate_set_stack(tstate, base, top); + return 0; +} + + +void +PyUnstable_ThreadState_ResetStackProtection(PyThreadState *tstate) +{ + _PyThreadStateImpl *ts = (_PyThreadStateImpl *)tstate; + if (ts->c_stack_init_top != 0) { + tstate_set_stack(tstate, + ts->c_stack_init_base, + ts->c_stack_init_top); return; } -# endif - _tstate->c_stack_top = _Py_SIZE_ROUND_UP(here_addr, 4096); - _tstate->c_stack_soft_limit = _tstate->c_stack_top - Py_C_STACK_SIZE; - _tstate->c_stack_hard_limit = _tstate->c_stack_top - (Py_C_STACK_SIZE + PYOS_STACK_MARGIN_BYTES); -#endif + + _Py_InitializeRecursionLimits(tstate); } + /* The function _Py_EnterRecursiveCallTstate() only calls _Py_CheckRecursiveCall() - if the recursion_depth reaches recursion_limit. */ + if the stack pointer is between the stack base and c_stack_hard_limit. */ int _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) { @@ -482,9 +591,17 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) uintptr_t here_addr = _Py_get_machine_stack_pointer(); assert(_tstate->c_stack_soft_limit != 0); assert(_tstate->c_stack_hard_limit != 0); +#if _Py_STACK_GROWS_DOWN + assert(here_addr >= _tstate->c_stack_hard_limit - _PyOS_STACK_MARGIN_BYTES); if (here_addr < _tstate->c_stack_hard_limit) { /* Overflowing while handling an overflow. Give up. */ int kbytes_used = (int)(_tstate->c_stack_top - here_addr)/1024; +#else + assert(here_addr <= _tstate->c_stack_hard_limit + _PyOS_STACK_MARGIN_BYTES); + if (here_addr > _tstate->c_stack_hard_limit) { + /* Overflowing while handling an overflow. Give up. */ + int kbytes_used = (int)(here_addr - _tstate->c_stack_top)/1024; +#endif char buffer[80]; snprintf(buffer, 80, "Unrecoverable stack overflow (used %d kB)%s", kbytes_used, where); Py_FatalError(buffer); @@ -493,7 +610,11 @@ _Py_CheckRecursiveCall(PyThreadState *tstate, const char *where) return 0; } else { +#if _Py_STACK_GROWS_DOWN int kbytes_used = (int)(_tstate->c_stack_top - here_addr)/1024; +#else + int kbytes_used = (int)(here_addr - _tstate->c_stack_top)/1024; +#endif tstate->recursion_headroom++; _PyErr_Format(tstate, PyExc_RecursionError, "Stack overflow (used %d kB)%s", @@ -617,15 +738,19 @@ _PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys) PyObject *seen = NULL; PyObject *dummy = NULL; PyObject *values = NULL; - PyObject *get = NULL; // We use the two argument form of map.get(key, default) for two reasons: // - Atomically check for a key and get its value without error handling. // - Don't cause key creation or resizing in dict subclasses like // collections.defaultdict that define __missing__ (or similar). - int meth_found = _PyObject_GetMethod(map, &_Py_ID(get), &get); - if (get == NULL) { + _PyCStackRef self, method; + _PyThreadState_PushCStackRef(tstate, &self); + _PyThreadState_PushCStackRef(tstate, &method); + self.ref = PyStackRef_FromPyObjectBorrow(map); + int res = _PyObject_GetMethodStackRef(tstate, &self.ref, &_Py_ID(get), &method.ref); + if (res < 0) { goto fail; } + PyObject *get = PyStackRef_AsPyObjectBorrow(method.ref); seen = PySet_New(NULL); if (seen == NULL) { goto fail; @@ -649,9 +774,10 @@ _PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys) } goto fail; } - PyObject *args[] = { map, key, dummy }; + PyObject *self_obj = PyStackRef_AsPyObjectBorrow(self.ref); + PyObject *args[] = { self_obj, key, dummy }; PyObject *value = NULL; - if (meth_found) { + if (!PyStackRef_IsNull(self.ref)) { value = PyObject_Vectorcall(get, args, 3, NULL); } else { @@ -672,12 +798,14 @@ _PyEval_MatchKeys(PyThreadState *tstate, PyObject *map, PyObject *keys) } // Success: done: - Py_DECREF(get); + _PyThreadState_PopCStackRef(tstate, &method); + _PyThreadState_PopCStackRef(tstate, &self); Py_DECREF(seen); Py_DECREF(dummy); return values; fail: - Py_XDECREF(get); + _PyThreadState_PopCStackRef(tstate, &method); + _PyThreadState_PopCStackRef(tstate, &self); Py_XDECREF(seen); Py_XDECREF(dummy); Py_XDECREF(values); @@ -764,7 +892,7 @@ _PyEval_MatchClass(PyThreadState *tstate, PyObject *subject, PyObject *type, if (allowed < nargs) { const char *plural = (allowed == 1) ? "" : "s"; _PyErr_Format(tstate, PyExc_TypeError, - "%s() accepts %d positional sub-pattern%s (%d given)", + "%s() accepts %zd positional sub-pattern%s (%zd given)", ((PyTypeObject*)type)->tp_name, allowed, plural, nargs); goto fail; @@ -878,6 +1006,26 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) #include "ceval_macros.h" + +_PyStackRef +_Py_LoadAttr_StackRefSteal( + PyThreadState *tstate, _PyStackRef owner, + PyObject *name, _PyStackRef *self_or_null) +{ + // Use _PyCStackRefs to ensure that both method and self are visible to + // the GC. Even though self_or_null is on the evaluation stack, it may be + // after the stackpointer and therefore not visible to the GC. + _PyCStackRef method, self; + _PyThreadState_PushCStackRef(tstate, &method); + _PyThreadState_PushCStackRef(tstate, &self); + self.ref = owner; // steal reference to owner + // NOTE: method.ref is initialized to PyStackRef_NULL and remains null on + // error, so we don't need to explicitly use the return code from the call. + _PyObject_GetMethodStackRef(tstate, &self.ref, name, &method.ref); + *self_or_null = _PyThreadState_PopCStackRefSteal(tstate, &self); + return _PyThreadState_PopCStackRefSteal(tstate, &method); +} + int _Py_CheckRecursiveCallPy( PyThreadState *tstate) { @@ -935,6 +1083,7 @@ _PyObjectArray_FromStackRefArray(_PyStackRef *input, Py_ssize_t nargs, PyObject // +1 in case PY_VECTORCALL_ARGUMENTS_OFFSET is set. result = PyMem_Malloc((nargs + 1) * sizeof(PyObject *)); if (result == NULL) { + PyErr_NoMemory(); return NULL; } result++; @@ -1290,7 +1439,7 @@ format_missing(PyThreadState *tstate, const char *kind, if (name_str == NULL) return; _PyErr_Format(tstate, PyExc_TypeError, - "%U() missing %i required %s argument%s: %U", + "%U() missing %zd required %s argument%s: %U", qualname, len, kind, @@ -1801,9 +1950,9 @@ clear_gen_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); + frame->previous = NULL; _PyFrame_ClearExceptCode(frame); _PyErr_ClearExcState(&gen->gi_exc_state); - frame->previous = NULL; } void @@ -2107,6 +2256,7 @@ do_raise(PyThreadState *tstate, PyObject *exc, PyObject *cause) "calling %R should have returned an instance of " "BaseException, not %R", cause, Py_TYPE(fixed_cause)); + Py_DECREF(fixed_cause); goto raise_error; } Py_DECREF(cause); @@ -2173,14 +2323,17 @@ _PyEval_ExceptionGroupMatch(_PyInterpreterFrame *frame, PyObject* exc_value, return -1; } PyFrameObject *f = _PyFrame_GetFrameObject(frame); - if (f != NULL) { - PyObject *tb = _PyTraceBack_FromFrame(NULL, f); - if (tb == NULL) { - return -1; - } - PyException_SetTraceback(wrapped, tb); - Py_DECREF(tb); + if (f == NULL) { + Py_DECREF(wrapped); + return -1; + } + + PyObject *tb = _PyTraceBack_FromFrame(NULL, f); + if (tb == NULL) { + return -1; } + PyException_SetTraceback(wrapped, tb); + Py_DECREF(tb); *match = wrapped; } *rest = Py_NewRef(Py_None); @@ -2425,6 +2578,10 @@ monitor_unwind(PyThreadState *tstate, do_monitor_exc(tstate, frame, instr, PY_MONITORING_EVENT_PY_UNWIND); } +bool +_PyEval_NoToolsForUnwind(PyThreadState *tstate) { + return no_tools_for_global_event(tstate, PY_MONITORING_EVENT_PY_UNWIND); +} static int monitor_handled(PyThreadState *tstate, @@ -2492,21 +2649,10 @@ PyEval_SetProfile(Py_tracefunc func, PyObject *arg) void PyEval_SetProfileAllThreads(Py_tracefunc func, PyObject *arg) { - PyThreadState *this_tstate = _PyThreadState_GET(); - PyInterpreterState* interp = this_tstate->interp; - - _PyRuntimeState *runtime = &_PyRuntime; - HEAD_LOCK(runtime); - PyThreadState* ts = PyInterpreterState_ThreadHead(interp); - HEAD_UNLOCK(runtime); - - while (ts) { - if (_PyEval_SetProfile(ts, func, arg) < 0) { - PyErr_FormatUnraisable("Exception ignored in PyEval_SetProfileAllThreads"); - } - HEAD_LOCK(runtime); - ts = PyThreadState_Next(ts); - HEAD_UNLOCK(runtime); + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyEval_SetProfileAllThreads(interp, func, arg) < 0) { + /* Log _PySys_Audit() error */ + PyErr_FormatUnraisable("Exception ignored in PyEval_SetProfileAllThreads"); } } @@ -2523,21 +2669,10 @@ PyEval_SetTrace(Py_tracefunc func, PyObject *arg) void PyEval_SetTraceAllThreads(Py_tracefunc func, PyObject *arg) { - PyThreadState *this_tstate = _PyThreadState_GET(); - PyInterpreterState* interp = this_tstate->interp; - - _PyRuntimeState *runtime = &_PyRuntime; - HEAD_LOCK(runtime); - PyThreadState* ts = PyInterpreterState_ThreadHead(interp); - HEAD_UNLOCK(runtime); - - while (ts) { - if (_PyEval_SetTrace(ts, func, arg) < 0) { - PyErr_FormatUnraisable("Exception ignored in PyEval_SetTraceAllThreads"); - } - HEAD_LOCK(runtime); - ts = PyThreadState_Next(ts); - HEAD_UNLOCK(runtime); + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyEval_SetTraceAllThreads(interp, func, arg) < 0) { + /* Log _PySys_Audit() error */ + PyErr_FormatUnraisable("Exception ignored in PyEval_SetTraceAllThreads"); } } @@ -2676,6 +2811,11 @@ PyEval_GetLocals(void) if (PyFrameLocalsProxy_Check(locals)) { PyFrameObject *f = _PyFrame_GetFrameObject(current_frame); + if (f == NULL) { + Py_DECREF(locals); + return NULL; + } + PyObject *ret = f->f_locals_cache; if (ret == NULL) { ret = PyDict_New(); @@ -2735,10 +2875,9 @@ _PyEval_GetFrameLocals(void) return locals; } -PyObject * -PyEval_GetGlobals(void) +static PyObject * +_PyEval_GetGlobals(PyThreadState *tstate) { - PyThreadState *tstate = _PyThreadState_GET(); _PyInterpreterFrame *current_frame = _PyThreadState_GetFrame(tstate); if (current_frame == NULL) { return NULL; @@ -2746,6 +2885,120 @@ PyEval_GetGlobals(void) return current_frame->f_globals; } +PyObject * +PyEval_GetGlobals(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + return _PyEval_GetGlobals(tstate); +} + +PyObject * +_PyEval_GetGlobalsFromRunningMain(PyThreadState *tstate) +{ + if (!_PyInterpreterState_IsRunningMain(tstate->interp)) { + return NULL; + } + PyObject *mod = _Py_GetMainModule(tstate); + if (_Py_CheckMainModule(mod) < 0) { + Py_XDECREF(mod); + return NULL; + } + PyObject *globals = PyModule_GetDict(mod); // borrowed + Py_DECREF(mod); + return globals; +} + +static PyObject * +get_globals_builtins(PyObject *globals) +{ + PyObject *builtins = NULL; + if (PyDict_Check(globals)) { + if (PyDict_GetItemRef(globals, &_Py_ID(__builtins__), &builtins) < 0) { + return NULL; + } + } + else { + if (PyMapping_GetOptionalItem( + globals, &_Py_ID(__builtins__), &builtins) < 0) + { + return NULL; + } + } + return builtins; +} + +static int +set_globals_builtins(PyObject *globals, PyObject *builtins) +{ + if (PyDict_Check(globals)) { + if (PyDict_SetItem(globals, &_Py_ID(__builtins__), builtins) < 0) { + return -1; + } + } + else { + if (PyObject_SetItem(globals, &_Py_ID(__builtins__), builtins) < 0) { + return -1; + } + } + return 0; +} + +int +_PyEval_EnsureBuiltins(PyThreadState *tstate, PyObject *globals, + PyObject **p_builtins) +{ + PyObject *builtins = get_globals_builtins(globals); + if (builtins == NULL) { + if (_PyErr_Occurred(tstate)) { + return -1; + } + builtins = PyEval_GetBuiltins(); // borrowed + if (builtins == NULL) { + assert(_PyErr_Occurred(tstate)); + return -1; + } + Py_INCREF(builtins); + if (set_globals_builtins(globals, builtins) < 0) { + Py_DECREF(builtins); + return -1; + } + } + if (p_builtins != NULL) { + *p_builtins = builtins; + } + else { + Py_DECREF(builtins); + } + return 0; +} + +int +_PyEval_EnsureBuiltinsWithModule(PyThreadState *tstate, PyObject *globals, + PyObject **p_builtins) +{ + PyObject *builtins = get_globals_builtins(globals); + if (builtins == NULL) { + if (_PyErr_Occurred(tstate)) { + return -1; + } + builtins = PyImport_ImportModuleLevel("builtins", NULL, NULL, NULL, 0); + if (builtins == NULL) { + return -1; + } + if (set_globals_builtins(globals, builtins) < 0) { + Py_DECREF(builtins); + return -1; + } + } + if (p_builtins != NULL) { + *p_builtins = builtins; + } + else { + Py_DECREF(builtins); + } + return 0; +} + PyObject* PyEval_GetFrameLocals(void) { @@ -3271,11 +3524,27 @@ PyUnstable_Eval_RequestCodeExtraIndex(freefunc free) PyInterpreterState *interp = _PyInterpreterState_GET(); Py_ssize_t new_index; - if (interp->co_extra_user_count == MAX_CO_EXTRA_USERS - 1) { +#ifdef Py_GIL_DISABLED + struct _py_code_state *state = &interp->code_state; + FT_MUTEX_LOCK(&state->mutex); +#endif + + if (interp->co_extra_user_count >= MAX_CO_EXTRA_USERS - 1) { +#ifdef Py_GIL_DISABLED + FT_MUTEX_UNLOCK(&state->mutex); +#endif return -1; } - new_index = interp->co_extra_user_count++; + + new_index = interp->co_extra_user_count; interp->co_extra_freefuncs[new_index] = free; + + // Publish freefuncs[new_index] before making the index visible. + FT_ATOMIC_STORE_SSIZE_RELEASE(interp->co_extra_user_count, new_index + 1); + +#ifdef Py_GIL_DISABLED + FT_MUTEX_UNLOCK(&state->mutex); +#endif return new_index; } diff --git a/Python/ceval_gil.c b/Python/ceval_gil.c index 5b5018a63731ab1..aa68371ac8febff 100644 --- a/Python/ceval_gil.c +++ b/Python/ceval_gil.c @@ -1218,30 +1218,30 @@ static inline int run_remote_debugger_source(PyObject *source) // Note that this function is inline to avoid creating a PLT entry // that would be an easy target for a ROP gadget. -static inline void run_remote_debugger_script(const char *path) +static inline void run_remote_debugger_script(PyObject *path) { - if (0 != PySys_Audit("remote_debugger_script", "s", path)) { + if (0 != PySys_Audit("cpython.remote_debugger_script", "O", path)) { PyErr_FormatUnraisable( - "Audit hook failed for remote debugger script %s", path); + "Audit hook failed for remote debugger script %U", path); return; } // Open the debugger script with the open code hook, and reopen the // resulting file object to get a C FILE* object. - PyObject* fileobj = PyFile_OpenCode(path); + PyObject* fileobj = PyFile_OpenCodeObject(path); if (!fileobj) { - PyErr_FormatUnraisable("Can't open debugger script %s", path); + PyErr_FormatUnraisable("Can't open debugger script %U", path); return; } PyObject* source = PyObject_CallMethodNoArgs(fileobj, &_Py_ID(read)); if (!source) { - PyErr_FormatUnraisable("Error reading debugger script %s", path); + PyErr_FormatUnraisable("Error reading debugger script %U", path); } PyObject* res = PyObject_CallMethodNoArgs(fileobj, &_Py_ID(close)); if (!res) { - PyErr_FormatUnraisable("Error closing debugger script %s", path); + PyErr_FormatUnraisable("Error closing debugger script %U", path); } else { Py_DECREF(res); } @@ -1249,7 +1249,7 @@ static inline void run_remote_debugger_script(const char *path) if (source) { if (0 != run_remote_debugger_source(source)) { - PyErr_FormatUnraisable("Error executing debugger script %s", path); + PyErr_FormatUnraisable("Error executing debugger script %U", path); } Py_DECREF(source); } @@ -1278,7 +1278,14 @@ int _PyRunRemoteDebugger(PyThreadState *tstate) pathsz); path[pathsz - 1] = '\0'; if (*path) { - run_remote_debugger_script(path); + PyObject *path_obj = PyUnicode_DecodeFSDefault(path); + if (path_obj == NULL) { + PyErr_FormatUnraisable("Can't decode debugger script"); + } + else { + run_remote_debugger_script(path_obj); + Py_DECREF(path_obj); + } } PyMem_Free(path); } @@ -1380,6 +1387,10 @@ _Py_HandlePending(PyThreadState *tstate) _Py_unset_eval_breaker_bit(tstate, _PY_EVAL_EXPLICIT_MERGE_BIT); _Py_brc_merge_refcounts(tstate); } + /* Process deferred memory frees held by QSBR */ + if (_Py_qsbr_should_process(((_PyThreadStateImpl *)tstate)->qsbr)) { + _PyMem_ProcessDelayed(tstate); + } #endif /* GC scheduled to run */ diff --git a/Python/ceval_macros.h b/Python/ceval_macros.h index 187ec8fdd265840..95a8e719d4d0d1c 100644 --- a/Python/ceval_macros.h +++ b/Python/ceval_macros.h @@ -79,6 +79,14 @@ #endif #if Py_TAIL_CALL_INTERP +# if defined(__clang__) || defined(__GNUC__) +# if !_Py__has_attribute(preserve_none) || !_Py__has_attribute(musttail) +# error "This compiler does not have support for efficient tail calling." +# endif +# elif defined(_MSC_VER) +# error "Tail calling not supported for MSVC." +# endif + // Note: [[clang::musttail]] works for GCC 15, but not __attribute__((musttail)) at the moment. # define Py_MUSTTAIL [[clang::musttail]] # define Py_PRESERVE_NONE_CC __attribute__((preserve_none)) @@ -312,14 +320,15 @@ GETITEM(PyObject *v, Py_ssize_t i) { // for an exception handler, displaying the traceback, and so on #define INSTRUMENTED_JUMP(src, dest, event) \ do { \ + _Py_CODEUNIT *_dest = (dest); \ if (tstate->tracing) {\ - next_instr = dest; \ + next_instr = _dest; \ } else { \ _PyFrame_SetStackPointer(frame, stack_pointer); \ - next_instr = _Py_call_instrumentation_jump(this_instr, tstate, event, frame, src, dest); \ + next_instr = _Py_call_instrumentation_jump(this_instr, tstate, event, frame, src, _dest); \ stack_pointer = _PyFrame_GetStackPointer(frame); \ if (next_instr == NULL) { \ - next_instr = (dest)+1; \ + next_instr = _dest + 1; \ JUMP_TO_LABEL(error); \ } \ } \ @@ -368,7 +377,9 @@ do { \ frame = tstate->current_frame; \ stack_pointer = _PyFrame_GetStackPointer(frame); \ if (next_instr == NULL) { \ - next_instr = frame->instr_ptr; \ + /* gh-140104: The exception handler expects frame->instr_ptr + to after this_instr, not this_instr! */ \ + next_instr = frame->instr_ptr + 1; \ JUMP_TO_LABEL(error); \ } \ DISPATCH(); \ @@ -396,7 +407,9 @@ do { \ stack_pointer = _PyFrame_GetStackPointer(frame); \ if (next_instr == NULL) \ { \ - next_instr = frame->instr_ptr; \ + /* gh-140104: The exception handler expects frame->instr_ptr + to after this_instr, not this_instr! */ \ + next_instr = frame->instr_ptr + 1; \ goto error; \ } \ DISPATCH(); \ diff --git a/Python/clinic/bltinmodule.c.h b/Python/clinic/bltinmodule.c.h index c826a5724f769cd..57f307ef7ad9dbb 100644 --- a/Python/clinic/bltinmodule.c.h +++ b/Python/clinic/bltinmodule.c.h @@ -25,9 +25,10 @@ PyDoc_STRVAR(builtin___import____doc__, "empty list to emulate ``import name``.\n" "When importing a module from a package, note that __import__(\'A.B\', ...)\n" "returns package A when fromlist is empty, but its submodule B when\n" -"fromlist is not empty. The level argument is used to determine whether to\n" -"perform absolute or relative imports: 0 is absolute, while a positive number\n" -"is the number of parent directories to search relative to the current module."); +"fromlist is not empty. The level argument is used to determine whether\n" +"to perform absolute or relative imports: 0 is absolute, while a positive\n" +"number is the number of parent directories to search relative to the\n" +"current module."); #define BUILTIN___IMPORT___METHODDEF \ {"__import__", _PyCFunction_CAST(builtin___import__), METH_FASTCALL|METH_KEYWORDS, builtin___import____doc__}, @@ -243,16 +244,17 @@ PyDoc_STRVAR(builtin_compile__doc__, "\n" "Compile source into a code object that can be executed by exec() or eval().\n" "\n" -"The source code may represent a Python module, statement or expression.\n" +"The source code may represent a Python module, statement or\n" +"expression.\n" "The filename will be used for run-time error messages.\n" "The mode must be \'exec\' to compile a module, \'single\' to compile a\n" "single (interactive) statement, or \'eval\' to compile an expression.\n" -"The flags argument, if present, controls which future statements influence\n" -"the compilation of the code.\n" +"The flags argument, if present, controls which future statements\n" +"influence the compilation of the code.\n" "The dont_inherit argument, if true, stops the compilation inheriting\n" "the effects of any future statements in effect in the code calling\n" -"compile; if absent or false these statements do influence the compilation,\n" -"in addition to any features explicitly specified."); +"compile; if absent or false these statements do influence the\n" +"compilation, in addition to any features explicitly specified."); #define BUILTIN_COMPILE_METHODDEF \ {"compile", _PyCFunction_CAST(builtin_compile), METH_FASTCALL|METH_KEYWORDS, builtin_compile__doc__}, @@ -296,7 +298,7 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj PyObject *argsbuf[7]; Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 3; PyObject *source; - PyObject *filename; + PyObject *filename = NULL; const char *mode; int flags = 0; int dont_inherit = 0; @@ -367,6 +369,9 @@ builtin_compile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj return_value = builtin_compile_impl(module, source, filename, mode, flags, dont_inherit, optimize, feature_version); exit: + /* Cleanup for filename */ + Py_XDECREF(filename); + return return_value; } @@ -577,8 +582,8 @@ PyDoc_STRVAR(builtin_globals__doc__, "\n" "Return the dictionary containing the current scope\'s global variables.\n" "\n" -"NOTE: Updates to this dictionary *will* affect name lookups in the current\n" -"global scope and vice-versa."); +"NOTE: Updates to this dictionary *will* affect name lookups in the\n" +"current global scope and vice-versa."); #define BUILTIN_GLOBALS_METHODDEF \ {"globals", (PyCFunction)builtin_globals, METH_NOARGS, builtin_globals__doc__}, @@ -720,10 +725,12 @@ PyDoc_STRVAR(builtin_hash__doc__, "hash($module, obj, /)\n" "--\n" "\n" -"Return the hash value for the given object.\n" +"Return the integer hash value for the given object.\n" "\n" -"Two objects that compare equal must also have the same hash value, but the\n" -"reverse is not necessarily true."); +"Two objects that compare equal must also have the same hash value, but\n" +"the reverse is not necessarily true. Hash values may differ between\n" +"Python processes. Not all objects are hashable; calling hash() on an\n" +"unhashable object raises TypeError."); #define BUILTIN_HASH_METHODDEF \ {"hash", (PyCFunction)builtin_hash, METH_O, builtin_hash__doc__}, @@ -802,9 +809,10 @@ PyDoc_STRVAR(builtin_locals__doc__, "\n" "Return a dictionary containing the current scope\'s local variables.\n" "\n" -"NOTE: Whether or not updates to this dictionary will affect name lookups in\n" -"the local scope and vice-versa is *implementation dependent* and not\n" -"covered by any backwards compatibility guarantees."); +"NOTE: Whether or not updates to this dictionary will affect name\n" +"lookups in the local scope and vice-versa is *implementation\n" +"dependent* and not covered by any backwards compatibility\n" +"guarantees."); #define BUILTIN_LOCALS_METHODDEF \ {"locals", (PyCFunction)builtin_locals, METH_NOARGS, builtin_locals__doc__}, @@ -831,10 +839,16 @@ PyDoc_STRVAR(builtin_oct__doc__, {"oct", (PyCFunction)builtin_oct, METH_O, builtin_oct__doc__}, PyDoc_STRVAR(builtin_ord__doc__, -"ord($module, c, /)\n" +"ord($module, character, /)\n" "--\n" "\n" -"Return the Unicode code point for a one-character string."); +"Return the ordinal value of a character.\n" +"\n" +"If the argument is a one-character string, return the Unicode code\n" +"point of that character.\n" +"\n" +"If the argument is a bytes or bytearray object of length 1, return its\n" +"single byte value."); #define BUILTIN_ORD_METHODDEF \ {"ord", (PyCFunction)builtin_ord, METH_O, builtin_ord__doc__}, @@ -845,8 +859,8 @@ PyDoc_STRVAR(builtin_pow__doc__, "\n" "Equivalent to base**exp with 2 arguments or base**exp % mod with 3 arguments\n" "\n" -"Some types, such as ints, are able to use a more efficient algorithm when\n" -"invoked using the three argument form."); +"Some types, such as ints, are able to use a more efficient algorithm\n" +"when invoked using the three argument form."); #define BUILTIN_POW_METHODDEF \ {"pow", _PyCFunction_CAST(builtin_pow), METH_FASTCALL|METH_KEYWORDS, builtin_pow__doc__}, @@ -1022,7 +1036,8 @@ PyDoc_STRVAR(builtin_input__doc__, "The prompt string, if given, is printed to standard output without a\n" "trailing newline before reading input.\n" "\n" -"If the user hits EOF (*nix: Ctrl-D, Windows: Ctrl-Z+Return), raise EOFError.\n" +"If the user hits EOF (*nix: Ctrl-D, Windows: Ctrl-Z+Return), raise\n" +"EOFError.\n" "On *nix systems, readline is used if available."); #define BUILTIN_INPUT_METHODDEF \ @@ -1068,8 +1083,9 @@ PyDoc_STRVAR(builtin_round__doc__, "\n" "Round a number to a given precision in decimal digits.\n" "\n" -"The return value is an integer if ndigits is omitted or None. Otherwise\n" -"the return value has the same type as the number. ndigits may be negative."); +"The return value is an integer if ndigits is omitted or None.\n" +"Otherwise the return value has the same type as the number. ndigits\n" +"may be negative."); #define BUILTIN_ROUND_METHODDEF \ {"round", _PyCFunction_CAST(builtin_round), METH_FASTCALL|METH_KEYWORDS, builtin_round__doc__}, @@ -1137,8 +1153,8 @@ PyDoc_STRVAR(builtin_sum__doc__, "Return the sum of a \'start\' value (default: 0) plus an iterable of numbers\n" "\n" "When the iterable is empty, return the start value.\n" -"This function is intended specifically for use with numeric values and may\n" -"reject non-numeric types."); +"This function is intended specifically for use with numeric values and\n" +"may reject non-numeric types."); #define BUILTIN_SUM_METHODDEF \ {"sum", _PyCFunction_CAST(builtin_sum), METH_FASTCALL|METH_KEYWORDS, builtin_sum__doc__}, @@ -1205,9 +1221,9 @@ PyDoc_STRVAR(builtin_isinstance__doc__, "\n" "Return whether an object is an instance of a class or of a subclass thereof.\n" "\n" -"A tuple, as in ``isinstance(x, (A, B, ...))``, may be given as the target to\n" -"check against. This is equivalent to ``isinstance(x, A) or isinstance(x, B)\n" -"or ...`` etc."); +"A tuple, as in ``isinstance(x, (A, B, ...))``, may be given as the\n" +"target to check against. This is equivalent to ``isinstance(x, A) or\n" +"isinstance(x, B) or ...`` etc."); #define BUILTIN_ISINSTANCE_METHODDEF \ {"isinstance", _PyCFunction_CAST(builtin_isinstance), METH_FASTCALL, builtin_isinstance__doc__}, @@ -1240,9 +1256,9 @@ PyDoc_STRVAR(builtin_issubclass__doc__, "\n" "Return whether \'cls\' is derived from another class or is the same class.\n" "\n" -"A tuple, as in ``issubclass(x, (A, B, ...))``, may be given as the target to\n" -"check against. This is equivalent to ``issubclass(x, A) or issubclass(x, B)\n" -"or ...``."); +"A tuple, as in ``issubclass(x, (A, B, ...))``, may be given as the\n" +"target to check against. This is equivalent to ``issubclass(x, A) or\n" +"issubclass(x, B) or ...``."); #define BUILTIN_ISSUBCLASS_METHODDEF \ {"issubclass", _PyCFunction_CAST(builtin_issubclass), METH_FASTCALL, builtin_issubclass__doc__}, @@ -1268,4 +1284,4 @@ builtin_issubclass(PyObject *module, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=e7a5d0851d7f2cfb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bc267a3ce1596f24 input=a9049054013a1b77]*/ diff --git a/Python/clinic/context.c.h b/Python/clinic/context.c.h index 5ed74e6e6ddb6bf..ece7341d65d5fb6 100644 --- a/Python/clinic/context.c.h +++ b/Python/clinic/context.c.h @@ -10,8 +10,8 @@ PyDoc_STRVAR(_contextvars_Context_get__doc__, "\n" "Return the value for `key` if `key` has the value in the context object.\n" "\n" -"If `key` does not exist, return `default`. If `default` is not given,\n" -"return None."); +"If `key` does not exist, return `default`. If `default` is not\n" +"given, return None."); #define _CONTEXTVARS_CONTEXT_GET_METHODDEF \ {"get", _PyCFunction_CAST(_contextvars_Context_get), METH_FASTCALL, _contextvars_Context_get__doc__}, @@ -122,10 +122,12 @@ PyDoc_STRVAR(_contextvars_ContextVar_get__doc__, "\n" "Return a value for the context variable for the current context.\n" "\n" -"If there is no value for the variable in the current context, the method will:\n" -" * return the value of the default argument of the method, if provided; or\n" -" * return the default value for the context variable, if it was created\n" -" with one; or\n" +"If there is no value for the variable in the current context, the\n" +"method will:\n" +" * return the value of the default argument of the method, if\n" +" provided; or\n" +" * return the default value for the context variable, if it was\n" +" created with one; or\n" " * raise a LookupError."); #define _CONTEXTVARS_CONTEXTVAR_GET_METHODDEF \ @@ -160,10 +162,11 @@ PyDoc_STRVAR(_contextvars_ContextVar_set__doc__, "\n" "Call to set a new value for the context variable in the current context.\n" "\n" -"The required value argument is the new value for the context variable.\n" +"The required value argument is the new value for the context\n" +"variable.\n" "\n" -"Returns a Token object that can be used to restore the variable to its previous\n" -"value via the `ContextVar.reset()` method."); +"Returns a Token object that can be used to restore the variable to\n" +"its previous value via the `ContextVar.reset()` method."); #define _CONTEXTVARS_CONTEXTVAR_SET_METHODDEF \ {"set", (PyCFunction)_contextvars_ContextVar_set, METH_O, _contextvars_ContextVar_set__doc__}, @@ -187,8 +190,8 @@ PyDoc_STRVAR(_contextvars_ContextVar_reset__doc__, "\n" "Reset the context variable.\n" "\n" -"The variable is reset to the value it had before the `ContextVar.set()` that\n" -"created the token was used."); +"The variable is reset to the value it had before the\n" +"`ContextVar.set()` that created the token was used."); #define _CONTEXTVARS_CONTEXTVAR_RESET_METHODDEF \ {"reset", (PyCFunction)_contextvars_ContextVar_reset, METH_O, _contextvars_ContextVar_reset__doc__}, @@ -256,4 +259,4 @@ token_exit(PyObject *self, PyObject *const *args, Py_ssize_t nargs) exit: return return_value; } -/*[clinic end generated code: output=3a04b2fddf24c3e9 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=90ec3e4375804e9b input=a9049054013a1b77]*/ diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h index 9bbb13f75665174..ab7fe513b2c69fc 100644 --- a/Python/clinic/import.c.h +++ b/Python/clinic/import.c.h @@ -34,8 +34,9 @@ PyDoc_STRVAR(_imp_acquire_lock__doc__, "\n" "Acquires the interpreter\'s import lock for the current thread.\n" "\n" -"This lock should be used by import hooks to ensure thread-safety when importing\n" -"modules. On platforms without threads, this function does nothing."); +"This lock should be used by import hooks to ensure thread-safety when\n" +"importing modules. On platforms without threads, this function does\n" +"nothing."); #define _IMP_ACQUIRE_LOCK_METHODDEF \ {"acquire_lock", (PyCFunction)_imp_acquire_lock, METH_NOARGS, _imp_acquire_lock__doc__}, @@ -629,4 +630,4 @@ _imp_source_hash(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb #ifndef _IMP_EXEC_DYNAMIC_METHODDEF #define _IMP_EXEC_DYNAMIC_METHODDEF #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */ -/*[clinic end generated code: output=24f597d6b0f3feed input=a9049054013a1b77]*/ +/*[clinic end generated code: output=394455063b83475b input=a9049054013a1b77]*/ diff --git a/Python/clinic/marshal.c.h b/Python/clinic/marshal.c.h index 6c00b2b31b007fa..ec0d2eb8a2af543 100644 --- a/Python/clinic/marshal.c.h +++ b/Python/clinic/marshal.c.h @@ -195,8 +195,8 @@ PyDoc_STRVAR(marshal_dumps__doc__, " allow_code\n" " Allow to write code objects.\n" "\n" -"Raise a ValueError exception if value has (or contains an object that has) an\n" -"unsupported type."); +"Raise a ValueError exception if value has (or contains an object that\n" +"has) an unsupported type."); #define MARSHAL_DUMPS_METHODDEF \ {"dumps", _PyCFunction_CAST(marshal_dumps), METH_FASTCALL|METH_KEYWORDS, marshal_dumps__doc__}, @@ -280,8 +280,8 @@ PyDoc_STRVAR(marshal_loads__doc__, " allow_code\n" " Allow to load code objects.\n" "\n" -"If no valid value is found, raise EOFError, ValueError or TypeError. Extra\n" -"bytes in the input are ignored."); +"If no valid value is found, raise EOFError, ValueError or TypeError.\n" +"Extra bytes in the input are ignored."); #define MARSHAL_LOADS_METHODDEF \ {"loads", _PyCFunction_CAST(marshal_loads), METH_FASTCALL|METH_KEYWORDS, marshal_loads__doc__}, @@ -351,4 +351,4 @@ marshal_loads(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObjec return return_value; } -/*[clinic end generated code: output=3e4bfc070a3c78ac input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a574570c3717f60e input=a9049054013a1b77]*/ diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index a47e4d11b54441d..edef016ac6010fb 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -1396,7 +1396,8 @@ PyDoc_STRVAR(sys__stats_dump__doc__, "\n" "Dump stats to file, and clears the stats.\n" "\n" -"Return False if no statistics were not dumped because stats gathering was off."); +"Return False if no statistics were not dumped because stats gathering\n" +"was off."); #define SYS__STATS_DUMP_METHODDEF \ {"_stats_dump", (PyCFunction)sys__stats_dump, METH_NOARGS, sys__stats_dump__doc__}, @@ -1544,16 +1545,16 @@ PyDoc_STRVAR(sys_remote_exec__doc__, "Executes a file containing Python code in a given remote Python process.\n" "\n" "This function returns immediately, and the code will be executed by the\n" -"target process\'s main thread at the next available opportunity, similarly\n" -"to how signals are handled. There is no interface to determine when the\n" -"code has been executed. The caller is responsible for making sure that\n" -"the file still exists whenever the remote process tries to read it and that\n" -"it hasn\'t been overwritten.\n" +"target process\'s main thread at the next available opportunity,\n" +"similarly to how signals are handled. There is no interface to\n" +"determine when the code has been executed. The caller is responsible\n" +"for making sure that the file still exists whenever the remote process\n" +"tries to read it and that it hasn\'t been overwritten.\n" "\n" -"The remote process must be running a CPython interpreter of the same major\n" -"and minor version as the local process. If either the local or remote\n" -"interpreter is pre-release (alpha, beta, or release candidate) then the\n" -"local and remote interpreters must be the same exact version.\n" +"The remote process must be running a CPython interpreter of the same\n" +"major and minor version as the local process. If either the local or\n" +"remote interpreter is pre-release (alpha, beta, or release candidate)\n" +"then the local and remote interpreters must be the same exact version.\n" "\n" "Args:\n" " pid (int): The process ID of the target Python process.\n" @@ -1793,6 +1794,37 @@ sys__baserepl(PyObject *module, PyObject *Py_UNUSED(ignored)) return sys__baserepl_impl(module); } +PyDoc_STRVAR(sys__clear_type_descriptors__doc__, +"_clear_type_descriptors($module, type, /)\n" +"--\n" +"\n" +"Private function for clearing certain descriptors from a type\'s dictionary.\n" +"\n" +"See gh-135228 for context."); + +#define SYS__CLEAR_TYPE_DESCRIPTORS_METHODDEF \ + {"_clear_type_descriptors", (PyCFunction)sys__clear_type_descriptors, METH_O, sys__clear_type_descriptors__doc__}, + +static PyObject * +sys__clear_type_descriptors_impl(PyObject *module, PyObject *type); + +static PyObject * +sys__clear_type_descriptors(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *type; + + if (!PyObject_TypeCheck(arg, &PyType_Type)) { + _PyArg_BadArgument("_clear_type_descriptors", "argument", (&PyType_Type)->tp_name, arg); + goto exit; + } + type = arg; + return_value = sys__clear_type_descriptors_impl(module, type); + +exit: + return return_value; +} + PyDoc_STRVAR(sys__is_gil_enabled__doc__, "_is_gil_enabled($module, /)\n" "--\n" @@ -1948,4 +1980,4 @@ _jit_is_active(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=449d16326e69dcf6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=407915aef6734c56 input=a9049054013a1b77]*/ diff --git a/Python/codecs.c b/Python/codecs.c index caf8d9d5f3c1887..3ca0ff1fc502f76 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -10,6 +10,7 @@ Copyright (c) Corporation for National Research Initiatives. #include "Python.h" #include "pycore_call.h" // _PyObject_CallNoArgs() +#include "pycore_initconfig.h" // _Py_DumpPathConfig() #include "pycore_interp.h" // PyInterpreterState.codec_search_path #include "pycore_pyerrors.h" // _PyErr_FormatNote() #include "pycore_pystate.h" // _PyInterpreterState_GET() @@ -1204,7 +1205,7 @@ get_standard_encoding_impl(const char *encoding, int *bytelength) } } } - else if (strcmp(encoding, "CP_UTF8") == 0) { + else if (strcmp(encoding, "cp65001") == 0) { *bytelength = 3; return ENC_UTF8; } @@ -1691,6 +1692,8 @@ _PyCodec_InitRegistry(PyInterpreterState *interp) // search functions, so this is done after everything else is initialized. PyObject *mod = PyImport_ImportModule("encodings"); if (mod == NULL) { + PyThreadState *tstate = _PyThreadState_GET(); + _Py_DumpPathConfig(tstate); return PyStatus_Error("Failed to import encodings module"); } Py_DECREF(mod); diff --git a/Python/codegen.c b/Python/codegen.c index 683601103ec99d4..766d54d069927ec 100644 --- a/Python/codegen.c +++ b/Python/codegen.c @@ -28,6 +28,8 @@ #include "pycore_pystate.h" // _Py_GetConfig() #include "pycore_symtable.h" // PySTEntryObject #include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString +#include "pycore_ceval.h" // SPECIAL___ENTER__ +#include "pycore_template.h" // _PyTemplate_Type #define NEED_OPCODE_METADATA #include "pycore_opcode_metadata.h" // _PyOpcode_opcode_metadata, _PyOpcode_num_popped/pushed @@ -1198,11 +1200,11 @@ codegen_type_param_bound_or_default(compiler *c, expr_ty e, ADDOP_LOAD_CONST_NEW(c, LOC(e), defaults); RETURN_IF_ERROR(codegen_setup_annotations_scope(c, LOC(e), key, name)); if (allow_starred && e->kind == Starred_kind) { - VISIT(c, expr, e->v.Starred.value); - ADDOP_I(c, LOC(e), UNPACK_SEQUENCE, (Py_ssize_t)1); + VISIT_IN_SCOPE(c, expr, e->v.Starred.value); + ADDOP_I_IN_SCOPE(c, LOC(e), UNPACK_SEQUENCE, (Py_ssize_t)1); } else { - VISIT(c, expr, e); + VISIT_IN_SCOPE(c, expr, e); } ADDOP_IN_SCOPE(c, LOC(e), RETURN_VALUE); PyCodeObject *co = _PyCompile_OptimizeAndAssemble(c, 1); @@ -3606,10 +3608,11 @@ infer_type(expr_ty e) return &PyGen_Type; case Lambda_kind: return &PyFunction_Type; - case JoinedStr_kind: case TemplateStr_kind: - case FormattedValue_kind: case Interpolation_kind: + return &_PyTemplate_Type; + case JoinedStr_kind: + case FormattedValue_kind: return &PyUnicode_Type; case Constant_kind: return Py_TYPE(e->v.Constant.value); @@ -3663,6 +3666,8 @@ check_subscripter(compiler *c, expr_ty e) case Set_kind: case SetComp_kind: case GeneratorExp_kind: + case TemplateStr_kind: + case Interpolation_kind: case Lambda_kind: { location loc = LOC(e); return _PyCompile_Warn(c, loc, "'%.200s' object is not subscriptable; " @@ -3697,9 +3702,7 @@ check_index(compiler *c, expr_ty e, expr_ty s) case List_kind: case ListComp_kind: case JoinedStr_kind: - case TemplateStr_kind: - case FormattedValue_kind: - case Interpolation_kind: { + case FormattedValue_kind: { location loc = LOC(e); return _PyCompile_Warn(c, loc, "%.200s indices must be integers " "or slices, not %.200s; " @@ -3858,6 +3861,14 @@ maybe_optimize_function_call(compiler *c, expr_ty e, jump_target_label end) return 0; } + expr_ty generator_exp = asdl_seq_GET(args, 0); + PySTEntryObject *generator_entry = _PySymtable_Lookup(SYMTABLE(c), (void *)generator_exp); + if (generator_entry->ste_coroutine) { + Py_DECREF(generator_entry); + return 0; + } + Py_DECREF(generator_entry); + location loc = LOC(func); int optimized = 0; @@ -3889,7 +3900,6 @@ maybe_optimize_function_call(compiler *c, expr_ty e, jump_target_label end) if (const_oparg == CONSTANT_BUILTIN_TUPLE) { ADDOP_I(c, loc, BUILD_LIST, 0); } - expr_ty generator_exp = asdl_seq_GET(args, 0); VISIT(c, expr, generator_exp); NEW_JUMP_TARGET_LABEL(c, loop); @@ -4069,16 +4079,6 @@ codegen_template_str(compiler *c, expr_ty e) } else { VISIT(c, expr, value); - Py_ssize_t j; - for (j = i + 1; j < value_count; j++) { - value = asdl_seq_GET(e->v.TemplateStr.values, j); - if (value->kind == Interpolation_kind) { - break; - } - VISIT(c, expr, value); - ADDOP_INPLACE(c, loc, Add); - } - i = j - 1; stringslen++; last_was_interpolation = 0; } @@ -4411,7 +4411,6 @@ codegen_sync_comprehension_generator(compiler *c, location loc, comprehension_ty gen = (comprehension_ty)asdl_seq_GET(generators, gen_index); - if (!iter_on_stack) { if (gen_index == 0) { assert(METADATA(c)->u_argcount == 1); @@ -4449,7 +4448,6 @@ codegen_sync_comprehension_generator(compiler *c, location loc, if (IS_JUMP_TARGET_LABEL(start)) { depth++; - ADDOP(c, LOC(gen->iter), GET_ITER); USE_LABEL(c, start); ADDOP_JUMP(c, LOC(gen->iter), FOR_ITER, anchor); } @@ -4865,7 +4863,10 @@ codegen_comprehension(compiler *c, expr_ty e, int type, } Py_CLEAR(co); - VISIT(c, expr, outermost->iter); + if (codegen_comprehension_iter(c, outermost)) { + goto error; + } + ADDOP_I(c, loc, CALL, 0); if (is_async_comprehension && type != COMP_GENEXP) { @@ -5506,10 +5507,12 @@ codegen_annassign(compiler *c, stmt_ty s) RETURN_IF_ERROR(_PyCompile_AddDeferredAnnotation( c, s, &conditional_annotation_index)); if (conditional_annotation_index != NULL) { - ADDOP_NAME( - c, loc, - SCOPE_TYPE(c) == COMPILE_SCOPE_CLASS ? LOAD_DEREF : LOAD_NAME, - &_Py_ID(__conditional_annotations__), cellvars); + if (SCOPE_TYPE(c) == COMPILE_SCOPE_CLASS) { + ADDOP_NAME(c, loc, LOAD_DEREF, &_Py_ID(__conditional_annotations__), cellvars); + } + else { + ADDOP_NAME(c, loc, LOAD_NAME, &_Py_ID(__conditional_annotations__), names); + } ADDOP_LOAD_CONST_NEW(c, loc, conditional_annotation_index); ADDOP_I(c, loc, SET_ADD, 1); ADDOP(c, loc, POP_TOP); diff --git a/Python/compile.c b/Python/compile.c index c04391e682f9ac9..6c5ef5329ffc3aa 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -103,6 +103,7 @@ typedef struct _PyCompiler { bool c_save_nested_seqs; /* if true, construct recursive instruction sequences * (including instructions for nested code objects) */ + int c_disable_warning; } compiler; static int @@ -135,7 +136,7 @@ compiler_setup(compiler *c, mod_ty mod, PyObject *filename, c->c_optimize = (optimize == -1) ? _Py_GetConfig()->optimization_level : optimize; c->c_save_nested_seqs = false; - if (!_PyAST_Preprocess(mod, arena, filename, c->c_optimize, merged, 0)) { + if (!_PyAST_Preprocess(mod, arena, filename, c->c_optimize, merged, 0, 1)) { return ERROR; } c->c_st = _PySymtable_Build(mod, filename, &c->c_future); @@ -166,6 +167,7 @@ new_compiler(mod_ty mod, PyObject *filename, PyCompilerFlags *pflags, { compiler *c = PyMem_Calloc(1, sizeof(compiler)); if (c == NULL) { + PyErr_NoMemory(); return NULL; } if (compiler_setup(c, mod, filename, pflags, optimize, arena) < 0) { @@ -627,7 +629,7 @@ _PyCompile_EnterScope(compiler *c, identifier name, int scope_type, } } if (u->u_ste->ste_has_conditional_annotations) { - /* Cook up an implicit __conditional__annotations__ cell */ + /* Cook up an implicit __conditional_annotations__ cell */ Py_ssize_t res; assert(u->u_scope_type == COMPILE_SCOPE_CLASS || u->u_scope_type == COMPILE_SCOPE_MODULE); res = _PyCompile_DictAddObj(u->u_metadata.u_cellvars, &_Py_ID(__conditional_annotations__)); @@ -765,6 +767,9 @@ _PyCompile_PushFBlock(compiler *c, location loc, f->fb_loc = loc; f->fb_exit = exit; f->fb_datum = datum; + if (t == COMPILE_FBLOCK_FINALLY_END) { + c->c_disable_warning++; + } return SUCCESS; } @@ -776,6 +781,9 @@ _PyCompile_PopFBlock(compiler *c, fblocktype t, jump_target_label block_label) u->u_nfblocks--; assert(u->u_fblock[u->u_nfblocks].fb_type == t); assert(SAME_JUMP_TARGET_LABEL(u->u_fblock[u->u_nfblocks].fb_block, block_label)); + if (t == COMPILE_FBLOCK_FINALLY_END) { + c->c_disable_warning--; + } } fblockinfo * @@ -846,12 +854,15 @@ compiler_mod(compiler *c, mod_ty mod) { PyCodeObject *co = NULL; int addNone = mod->kind != Expression_kind; + assert(c->u == NULL); if (compiler_codegen(c, mod) < 0) { goto finally; } co = _PyCompile_OptimizeAndAssemble(c, addNone); finally: - _PyCompile_ExitScope(c); + if (c->u != NULL) { + _PyCompile_ExitScope(c); + } return co; } @@ -1203,6 +1214,9 @@ _PyCompile_Error(compiler *c, location loc, const char *format, ...) int _PyCompile_Warn(compiler *c, location loc, const char *format, ...) { + if (c->c_disable_warning) { + return 0; + } va_list vargs; va_start(vargs, format); PyObject *msg = PyUnicode_FromFormatV(format, vargs); @@ -1492,7 +1506,7 @@ _PyCompile_AstPreprocess(mod_ty mod, PyObject *filename, PyCompilerFlags *cf, if (optimize == -1) { optimize = _Py_GetConfig()->optimization_level; } - if (!_PyAST_Preprocess(mod, arena, filename, optimize, flags, no_const_folding)) { + if (!_PyAST_Preprocess(mod, arena, filename, optimize, flags, no_const_folding, 0)) { return -1; } return 0; @@ -1600,6 +1614,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, { PyObject *res = NULL; PyObject *metadata = NULL; + PyObject *consts_list = NULL; if (!PyAST_Check(ast)) { PyErr_SetString(PyExc_TypeError, "expected an AST"); @@ -1626,7 +1641,7 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, metadata = PyDict_New(); if (metadata == NULL) { - return NULL; + goto finally; } if (compiler_codegen(c, mod) < 0) { @@ -1654,12 +1669,23 @@ _PyCompile_CodeGen(PyObject *ast, PyObject *filename, PyCompilerFlags *pflags, } if (_PyInstructionSequence_ApplyLabelMap(_PyCompile_InstrSequence(c)) < 0) { - return NULL; + goto finally; + } + + /* After AddReturnAtEnd: co_consts indices match the final instruction stream. */ + consts_list = consts_dict_keys_inorder(umd->u_consts); + if (consts_list == NULL) { + goto finally; + } + if (PyDict_SetItemString(metadata, "consts", consts_list) < 0) { + goto finally; } + /* Allocate a copy of the instruction sequence on the heap */ res = PyTuple_Pack(2, _PyCompile_InstrSequence(c), metadata); finally: + Py_XDECREF(consts_list); Py_XDECREF(metadata); _PyCompile_ExitScope(c); compiler_free(c); diff --git a/Python/context.c b/Python/context.c index dceaae9b42979da..9cec2eec1893910 100644 --- a/Python/context.c +++ b/Python/context.c @@ -343,12 +343,6 @@ PyContextVar_Set(PyObject *ovar, PyObject *val) ENSURE_ContextVar(ovar, NULL) PyContextVar *var = (PyContextVar *)ovar; - if (!PyContextVar_CheckExact(var)) { - PyErr_SetString( - PyExc_TypeError, "an instance of ContextVar was expected"); - return NULL; - } - PyContext *ctx = context_get(); if (ctx == NULL) { return NULL; @@ -363,6 +357,9 @@ PyContextVar_Set(PyObject *ovar, PyObject *val) Py_XINCREF(old_val); PyContextToken *tok = token_new(ctx, var, old_val); Py_XDECREF(old_val); + if (tok == NULL) { + return NULL; + } if (contextvar_set(var, val)) { Py_DECREF(tok); @@ -626,14 +623,14 @@ _contextvars.Context.get Return the value for `key` if `key` has the value in the context object. -If `key` does not exist, return `default`. If `default` is not given, -return None. +If `key` does not exist, return `default`. If `default` is not +given, return None. [clinic start generated code]*/ static PyObject * _contextvars_Context_get_impl(PyContext *self, PyObject *key, PyObject *default_value) -/*[clinic end generated code: output=0c54aa7664268189 input=c8eeb81505023995]*/ +/*[clinic end generated code: output=0c54aa7664268189 input=d1be897231334ea9]*/ { if (context_check_key_type(key)) { return NULL; @@ -979,7 +976,7 @@ contextvar_tp_repr(PyObject *op) return NULL; } - if (PyUnicodeWriter_WriteUTF8(writer, "<ContextVar name=", 17) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, "<ContextVar name=", 17) < 0) { goto error; } if (PyUnicodeWriter_WriteRepr(writer, self->var_name) < 0) { @@ -987,7 +984,7 @@ contextvar_tp_repr(PyObject *op) } if (self->var_default != NULL) { - if (PyUnicodeWriter_WriteUTF8(writer, " default=", 9) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, " default=", 9) < 0) { goto error; } if (PyUnicodeWriter_WriteRepr(writer, self->var_default) < 0) { @@ -1013,23 +1010,19 @@ _contextvars.ContextVar.get Return a value for the context variable for the current context. -If there is no value for the variable in the current context, the method will: - * return the value of the default argument of the method, if provided; or - * return the default value for the context variable, if it was created - with one; or +If there is no value for the variable in the current context, the +method will: + * return the value of the default argument of the method, if + provided; or + * return the default value for the context variable, if it was + created with one; or * raise a LookupError. [clinic start generated code]*/ static PyObject * _contextvars_ContextVar_get_impl(PyContextVar *self, PyObject *default_value) -/*[clinic end generated code: output=0746bd0aa2ced7bf input=30aa2ab9e433e401]*/ +/*[clinic end generated code: output=0746bd0aa2ced7bf input=83814c6aef4a9fe3]*/ { - if (!PyContextVar_CheckExact(self)) { - PyErr_SetString( - PyExc_TypeError, "an instance of ContextVar was expected"); - return NULL; - } - PyObject *val; if (PyContextVar_Get((PyObject *)self, default_value, &val) < 0) { return NULL; @@ -1050,15 +1043,16 @@ _contextvars.ContextVar.set Call to set a new value for the context variable in the current context. -The required value argument is the new value for the context variable. +The required value argument is the new value for the context +variable. -Returns a Token object that can be used to restore the variable to its previous -value via the `ContextVar.reset()` method. +Returns a Token object that can be used to restore the variable to +its previous value via the `ContextVar.reset()` method. [clinic start generated code]*/ static PyObject * _contextvars_ContextVar_set_impl(PyContextVar *self, PyObject *value) -/*[clinic end generated code: output=1b562d35cc79c806 input=c0a6887154227453]*/ +/*[clinic end generated code: output=1b562d35cc79c806 input=6ffee66796d67896]*/ { return PyContextVar_Set((PyObject *)self, value); } @@ -1070,13 +1064,13 @@ _contextvars.ContextVar.reset Reset the context variable. -The variable is reset to the value it had before the `ContextVar.set()` that -created the token was used. +The variable is reset to the value it had before the +`ContextVar.set()` that created the token was used. [clinic start generated code]*/ static PyObject * _contextvars_ContextVar_reset_impl(PyContextVar *self, PyObject *token) -/*[clinic end generated code: output=3205d2bdff568521 input=ebe2881e5af4ffda]*/ +/*[clinic end generated code: output=3205d2bdff568521 input=dd33cfcb18c00e37]*/ { if (!PyContextToken_CheckExact(token)) { PyErr_Format(PyExc_TypeError, @@ -1102,7 +1096,8 @@ static PyMethodDef PyContextVar_methods[] = { _CONTEXTVARS_CONTEXTVAR_SET_METHODDEF _CONTEXTVARS_CONTEXTVAR_RESET_METHODDEF {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + PyDoc_STR("ContextVars are generic over the type of their contained values")}, {NULL, NULL} }; @@ -1182,15 +1177,15 @@ token_tp_repr(PyObject *op) if (writer == NULL) { return NULL; } - if (PyUnicodeWriter_WriteUTF8(writer, "<Token", 6) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, "<Token", 6) < 0) { goto error; } if (self->tok_used) { - if (PyUnicodeWriter_WriteUTF8(writer, " used", 5) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, " used", 5) < 0) { goto error; } } - if (PyUnicodeWriter_WriteUTF8(writer, " var=", 5) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, " var=", 5) < 0) { goto error; } if (PyUnicodeWriter_WriteRepr(writer, (PyObject *)self->tok_var) < 0) { @@ -1268,7 +1263,8 @@ token_exit_impl(PyContextToken *self, PyObject *type, PyObject *val, static PyMethodDef PyContextTokenType_methods[] = { {"__class_getitem__", Py_GenericAlias, - METH_O|METH_CLASS, PyDoc_STR("See PEP 585")}, + METH_O|METH_CLASS, + PyDoc_STR("Tokens are generic over the same type as the ContextVar which created them.")}, TOKEN_ENTER_METHODDEF TOKEN_EXIT_METHODDEF {NULL} @@ -1357,11 +1353,8 @@ get_token_missing(void) PyStatus _PyContext_Init(PyInterpreterState *interp) { - if (!_Py_IsMainInterpreter(interp)) { - return _PyStatus_OK(); - } - PyObject *missing = get_token_missing(); + assert(PyUnstable_IsImmortal(missing)); if (PyDict_SetItemString( _PyType_GetDict(&PyContextToken_Type), "MISSING", missing)) { diff --git a/Python/critical_section.c b/Python/critical_section.c index 73857b85496316c..d3df5caddae6d07 100644 --- a/Python/critical_section.c +++ b/Python/critical_section.c @@ -1,7 +1,8 @@ #include "Python.h" -#include "pycore_lock.h" #include "pycore_critical_section.h" +#include "pycore_interp.h" +#include "pycore_lock.h" #ifdef Py_GIL_DISABLED static_assert(_Alignof(PyCriticalSection) >= 4, @@ -24,8 +25,30 @@ _PyCriticalSection_BeginSlow(PyCriticalSection *c, PyMutex *m) // As an optimisation for locking the same object recursively, skip // locking if the mutex is currently locked by the top-most critical // section. - if (tstate->critical_section && - untag_critical_section(tstate->critical_section)->_cs_mutex == m) { + // If the top-most critical section is a two-mutex critical section, + // then locking is skipped if either mutex is m. + if (tstate->critical_section) { + PyCriticalSection *prev = untag_critical_section(tstate->critical_section); + if (prev->_cs_mutex == m) { + c->_cs_mutex = NULL; + c->_cs_prev = 0; + return; + } + if (tstate->critical_section & _Py_CRITICAL_SECTION_TWO_MUTEXES) { + PyCriticalSection2 *prev2 = (PyCriticalSection2 *) + untag_critical_section(tstate->critical_section); + if (prev2->_cs_mutex2 == m) { + c->_cs_mutex = NULL; + c->_cs_prev = 0; + return; + } + } + } + // If the world is stopped, we don't need to acquire the lock because + // there are no other threads that could be accessing the object. + // Without this check, acquiring a critical section while the world is + // stopped could lead to a deadlock. + if (tstate->interp->stoptheworld.world_stopped) { c->_cs_mutex = NULL; c->_cs_prev = 0; return; @@ -45,6 +68,28 @@ _PyCriticalSection2_BeginSlow(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2, { #ifdef Py_GIL_DISABLED PyThreadState *tstate = _PyThreadState_GET(); + if (tstate->interp->stoptheworld.world_stopped) { + c->_cs_base._cs_mutex = NULL; + c->_cs_mutex2 = NULL; + c->_cs_base._cs_prev = 0; + return; + } + // Same optimization as in _PyCriticalSection_BeginSlow: skip locking when + // recursively acquiring the same locks. + if (tstate->critical_section && + tstate->critical_section & _Py_CRITICAL_SECTION_TWO_MUTEXES) { + PyCriticalSection2 *prev2 = (PyCriticalSection2 *) + untag_critical_section(tstate->critical_section); + assert((uintptr_t)m1 < (uintptr_t)m2); + assert((uintptr_t)prev2->_cs_base._cs_mutex < + (uintptr_t)prev2->_cs_mutex2); + if (prev2->_cs_base._cs_mutex == m1 && prev2->_cs_mutex2 == m2) { + c->_cs_base._cs_mutex = NULL; + c->_cs_mutex2 = NULL; + c->_cs_base._cs_prev = 0; + return; + } + } c->_cs_base._cs_mutex = NULL; c->_cs_mutex2 = NULL; c->_cs_base._cs_prev = tstate->critical_section; @@ -130,6 +175,15 @@ PyCriticalSection_Begin(PyCriticalSection *c, PyObject *op) #endif } +#undef PyCriticalSection_BeginMutex +void +PyCriticalSection_BeginMutex(PyCriticalSection *c, PyMutex *m) +{ +#ifdef Py_GIL_DISABLED + _PyCriticalSection_BeginMutex(c, m); +#endif +} + #undef PyCriticalSection_End void PyCriticalSection_End(PyCriticalSection *c) @@ -148,6 +202,15 @@ PyCriticalSection2_Begin(PyCriticalSection2 *c, PyObject *a, PyObject *b) #endif } +#undef PyCriticalSection2_BeginMutex +void +PyCriticalSection2_BeginMutex(PyCriticalSection2 *c, PyMutex *m1, PyMutex *m2) +{ +#ifdef Py_GIL_DISABLED + _PyCriticalSection2_BeginMutex(c, m1, m2); +#endif +} + #undef PyCriticalSection2_End void PyCriticalSection2_End(PyCriticalSection2 *c) diff --git a/Python/crossinterp.c b/Python/crossinterp.c index 74ce02f1a26401b..143b77996481467 100644 --- a/Python/crossinterp.c +++ b/Python/crossinterp.c @@ -6,8 +6,14 @@ #include "osdefs.h" // MAXPATHLEN #include "pycore_ceval.h" // _Py_simple_func #include "pycore_crossinterp.h" // _PyXIData_t +#include "pycore_function.h" // _PyFunction_VerifyStateless() +#include "pycore_global_strings.h" // _Py_ID() +#include "pycore_import.h" // _PyImport_SetModule() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_namespace.h" // _PyNamespace_New() +#include "pycore_pythonrun.h" // _Py_SourceAsString() +#include "pycore_runtime.h" // _PyRuntime +#include "pycore_setobject.h" // _PySet_NextEntry() #include "pycore_typeobject.h" // _PyStaticType_InitBuiltin() @@ -19,6 +25,7 @@ _Py_GetMainfile(char *buffer, size_t maxlen) PyThreadState *tstate = _PyThreadState_GET(); PyObject *module = _Py_GetMainModule(tstate); if (_Py_CheckMainModule(module) < 0) { + Py_XDECREF(module); return -1; } Py_ssize_t size = _PyModule_GetFilenameUTF8(module, buffer, maxlen); @@ -27,27 +34,6 @@ _Py_GetMainfile(char *buffer, size_t maxlen) } -static PyObject * -import_get_module(PyThreadState *tstate, const char *modname) -{ - PyObject *module = NULL; - if (strcmp(modname, "__main__") == 0) { - module = _Py_GetMainModule(tstate); - if (_Py_CheckMainModule(module) < 0) { - assert(_PyErr_Occurred(tstate)); - return NULL; - } - } - else { - module = PyImport_ImportModule(modname); - if (module == NULL) { - return NULL; - } - } - return module; -} - - static PyObject * runpy_run_path(const char *filename, const char *modname) { @@ -67,97 +53,192 @@ runpy_run_path(const char *filename, const char *modname) } -static PyObject * -pyerr_get_message(PyObject *exc) +static void +set_exc_with_cause(PyObject *exctype, const char *msg) { - assert(!PyErr_Occurred()); - PyObject *args = PyException_GetArgs(exc); - if (args == NULL || args == Py_None || PyObject_Size(args) < 1) { - return NULL; - } - if (PyUnicode_Check(args)) { - return args; - } - PyObject *msg = PySequence_GetItem(args, 0); - Py_DECREF(args); - if (msg == NULL) { - PyErr_Clear(); - return NULL; - } - if (!PyUnicode_Check(msg)) { - Py_DECREF(msg); - return NULL; - } - return msg; + PyObject *cause = PyErr_GetRaisedException(); + PyErr_SetString(exctype, msg); + PyObject *exc = PyErr_GetRaisedException(); + PyException_SetCause(exc, cause); + PyErr_SetRaisedException(exc); } -#define MAX_MODNAME (255) -#define MAX_ATTRNAME (255) -struct attributeerror_info { - char modname[MAX_MODNAME+1]; - char attrname[MAX_ATTRNAME+1]; +/****************************/ +/* module duplication utils */ +/****************************/ + +struct sync_module_result { + PyObject *module; + PyObject *loaded; + PyObject *failed; +}; + +struct sync_module { + const char *filename; + char _filename[MAXPATHLEN+1]; + struct sync_module_result cached; }; +static void +sync_module_clear(struct sync_module *data) +{ + data->filename = NULL; + Py_CLEAR(data->cached.module); + Py_CLEAR(data->cached.loaded); + Py_CLEAR(data->cached.failed); +} + +static void +sync_module_capture_exc(PyThreadState *tstate, struct sync_module *data) +{ + assert(_PyErr_Occurred(tstate)); + PyObject *context = data->cached.failed; + PyObject *exc = _PyErr_GetRaisedException(tstate); + _PyErr_SetRaisedException(tstate, Py_NewRef(exc)); + if (context != NULL) { + PyException_SetContext(exc, context); + } + data->cached.failed = exc; +} + + static int -_parse_attributeerror(PyObject *exc, struct attributeerror_info *info) +ensure_isolated_main(PyThreadState *tstate, struct sync_module *main) { - assert(exc != NULL); - assert(PyErr_GivenExceptionMatches(exc, PyExc_AttributeError)); - int res = -1; + // Load the module from the original file (or from a cache). - PyObject *msgobj = pyerr_get_message(exc); - if (msgobj == NULL) { + // First try the local cache. + if (main->cached.failed != NULL) { + // We'll deal with this in apply_isolated_main(). + assert(main->cached.module == NULL); + assert(main->cached.loaded == NULL); + return 0; + } + else if (main->cached.loaded != NULL) { + assert(main->cached.module != NULL); + return 0; + } + assert(main->cached.module == NULL); + + if (main->filename == NULL) { + _PyErr_SetString(tstate, PyExc_NotImplementedError, ""); return -1; } - const char *err = PyUnicode_AsUTF8(msgobj); - if (strncmp(err, "module '", 8) != 0) { - goto finally; + // It wasn't in the local cache so we'll need to populate it. + PyObject *mod = _Py_GetMainModule(tstate); + if (_Py_CheckMainModule(mod) < 0) { + // This is probably unrecoverable, so don't bother caching the error. + assert(_PyErr_Occurred(tstate)); + Py_XDECREF(mod); + return -1; } - err += 8; + PyObject *loaded = NULL; - const char *matched = strchr(err, '\''); - if (matched == NULL) { - goto finally; + // Try the per-interpreter cache for the loaded module. + // XXX Store it in sys.modules? + PyObject *interpns = PyInterpreterState_GetDict(tstate->interp); + assert(interpns != NULL); + PyObject *key = PyUnicode_FromString("CACHED_MODULE_NS___main__"); + if (key == NULL) { + // It's probably unrecoverable, so don't bother caching the error. + Py_DECREF(mod); + return -1; } - Py_ssize_t len = matched - err; - if (len > MAX_MODNAME) { - goto finally; + else if (PyDict_GetItemRef(interpns, key, &loaded) < 0) { + // It's probably unrecoverable, so don't bother caching the error. + Py_DECREF(mod); + Py_DECREF(key); + return -1; } - (void)strncpy(info->modname, err, len); - info->modname[len] = '\0'; - err = matched; + else if (loaded == NULL) { + // It wasn't already loaded from file. + loaded = PyModule_NewObject(&_Py_ID(__main__)); + if (loaded == NULL) { + goto error; + } + PyObject *ns = _PyModule_GetDict(loaded); - if (strncmp(err, "' has no attribute '", 20) != 0) { - goto finally; - } - err += 20; + // We don't want to trigger "if __name__ == '__main__':", + // so we use a bogus module name. + PyObject *loaded_ns = + runpy_run_path(main->filename, "<fake __main__>"); + if (loaded_ns == NULL) { + goto error; + } + int res = PyDict_Update(ns, loaded_ns); + Py_DECREF(loaded_ns); + if (res < 0) { + goto error; + } - matched = strchr(err, '\''); - if (matched == NULL) { - goto finally; - } - len = matched - err; - if (len > MAX_ATTRNAME) { - goto finally; + // Set the per-interpreter cache entry. + if (PyDict_SetItem(interpns, key, loaded) < 0) { + goto error; + } } - (void)strncpy(info->attrname, err, len); - info->attrname[len] = '\0'; - err = matched + 1; - if (strlen(err) > 0) { - goto finally; + Py_DECREF(key); + main->cached = (struct sync_module_result){ + .module = mod, + .loaded = loaded, + }; + return 0; + +error: + sync_module_capture_exc(tstate, main); + Py_XDECREF(loaded); + Py_DECREF(mod); + Py_XDECREF(key); + return -1; +} + +#ifndef NDEBUG +static int +main_mod_matches(PyObject *expected) +{ + PyObject *mod = PyImport_GetModule(&_Py_ID(__main__)); + Py_XDECREF(mod); + return mod == expected; +} +#endif + +static int +apply_isolated_main(PyThreadState *tstate, struct sync_module *main) +{ + assert((main->cached.loaded == NULL) == (main->cached.loaded == NULL)); + if (main->cached.failed != NULL) { + // It must have failed previously. + assert(main->cached.loaded == NULL); + _PyErr_SetRaisedException(tstate, main->cached.failed); + return -1; } - res = 0; + assert(main->cached.loaded != NULL); -finally: - Py_DECREF(msgobj); - return res; + assert(main_mod_matches(main->cached.module)); + if (_PyImport_SetModule(&_Py_ID(__main__), main->cached.loaded) < 0) { + sync_module_capture_exc(tstate, main); + return -1; + } + return 0; } -#undef MAX_MODNAME -#undef MAX_ATTRNAME +static void +restore_main(PyThreadState *tstate, struct sync_module *main) +{ + assert(main->cached.failed == NULL); + assert(main->cached.module != NULL); + assert(main->cached.loaded != NULL); + PyObject *exc = _PyErr_GetRaisedException(tstate); + assert(main_mod_matches(main->cached.loaded)); + int res = _PyImport_SetModule(&_Py_ID(__main__), main->cached.module); + assert(res == 0); + if (res < 0) { + PyErr_FormatUnraisable("Exception ignored while restoring __main__"); + } + _PyErr_SetRaisedException(tstate, exc); +} /**************/ @@ -207,16 +288,16 @@ _Py_CallInInterpreterAndRawFree(PyInterpreterState *interp, /* cross-interpreter data */ /**************************/ -/* registry of {type -> xidatafunc} */ +/* registry of {type -> _PyXIData_getdata_t} */ -/* For now we use a global registry of shareable classes. An - alternative would be to add a tp_* slot for a class's - xidatafunc. It would be simpler and more efficient. */ +/* For now we use a global registry of shareable classes. + An alternative would be to add a tp_* slot for a class's + _PyXIData_getdata_t. It would be simpler and more efficient. */ static void xid_lookup_init(_PyXIData_lookup_t *); static void xid_lookup_fini(_PyXIData_lookup_t *); struct _dlcontext; -static xidatafunc lookup_getdata(struct _dlcontext *, PyObject *); +static _PyXIData_getdata_t lookup_getdata(struct _dlcontext *, PyObject *); #include "crossinterp_data_lookup.h" @@ -340,7 +421,7 @@ _set_xid_lookup_failure(PyThreadState *tstate, PyObject *obj, const char *msg, set_notshareableerror(tstate, cause, 0, msg); } else { - msg = "%S does not support cross-interpreter data"; + msg = "%R does not support cross-interpreter data"; format_notshareableerror(tstate, cause, 0, msg, obj); } } @@ -353,8 +434,8 @@ _PyObject_CheckXIData(PyThreadState *tstate, PyObject *obj) if (get_lookup_context(tstate, &ctx) < 0) { return -1; } - xidatafunc getdata = lookup_getdata(&ctx, obj); - if (getdata == NULL) { + _PyXIData_getdata_t getdata = lookup_getdata(&ctx, obj); + if (getdata.basic == NULL && getdata.fallback == NULL) { if (!_PyErr_Occurred(tstate)) { _set_xid_lookup_failure(tstate, obj, NULL, NULL); } @@ -385,9 +466,9 @@ _check_xidata(PyThreadState *tstate, _PyXIData_t *xidata) return 0; } -int -_PyObject_GetXIData(PyThreadState *tstate, - PyObject *obj, _PyXIData_t *xidata) +static int +_get_xidata(PyThreadState *tstate, + PyObject *obj, xidata_fallback_t fallback, _PyXIData_t *xidata) { PyInterpreterState *interp = tstate->interp; @@ -395,6 +476,7 @@ _PyObject_GetXIData(PyThreadState *tstate, assert(xidata->obj == NULL); if (xidata->data != NULL || xidata->obj != NULL) { _PyErr_SetString(tstate, PyExc_ValueError, "xidata not cleared"); + return -1; } // Call the "getdata" func for the object. @@ -403,8 +485,8 @@ _PyObject_GetXIData(PyThreadState *tstate, return -1; } Py_INCREF(obj); - xidatafunc getdata = lookup_getdata(&ctx, obj); - if (getdata == NULL) { + _PyXIData_getdata_t getdata = lookup_getdata(&ctx, obj); + if (getdata.basic == NULL && getdata.fallback == NULL) { if (PyErr_Occurred()) { Py_DECREF(obj); return -1; @@ -416,7 +498,9 @@ _PyObject_GetXIData(PyThreadState *tstate, } return -1; } - int res = getdata(tstate, obj, xidata); + int res = getdata.basic != NULL + ? getdata.basic(tstate, obj, xidata) + : getdata.fallback(tstate, obj, fallback, xidata); Py_DECREF(obj); if (res != 0) { PyObject *cause = _PyErr_GetRaisedException(tstate); @@ -436,6 +520,51 @@ _PyObject_GetXIData(PyThreadState *tstate, return 0; } +int +_PyObject_GetXIDataNoFallback(PyThreadState *tstate, + PyObject *obj, _PyXIData_t *xidata) +{ + return _get_xidata(tstate, obj, _PyXIDATA_XIDATA_ONLY, xidata); +} + +int +_PyObject_GetXIData(PyThreadState *tstate, + PyObject *obj, xidata_fallback_t fallback, + _PyXIData_t *xidata) +{ + switch (fallback) { + case _PyXIDATA_XIDATA_ONLY: + return _get_xidata(tstate, obj, fallback, xidata); + case _PyXIDATA_FULL_FALLBACK: + if (_get_xidata(tstate, obj, fallback, xidata) == 0) { + return 0; + } + PyObject *exc = _PyErr_GetRaisedException(tstate); + if (PyFunction_Check(obj)) { + if (_PyFunction_GetXIData(tstate, obj, xidata) == 0) { + Py_DECREF(exc); + return 0; + } + _PyErr_Clear(tstate); + } + // We could try _PyMarshal_GetXIData() but we won't for now. + if (_PyPickle_GetXIData(tstate, obj, xidata) == 0) { + Py_DECREF(exc); + return 0; + } + // Raise the original exception. + _PyErr_SetRaisedException(tstate, exc); + return -1; + default: +#ifdef Py_DEBUG + Py_FatalError("unsupported xidata fallback option"); +#endif + _PyErr_SetString(tstate, PyExc_SystemError, + "unsupported xidata fallback option"); + return -1; + } +} + /* pickle C-API */ @@ -456,28 +585,6 @@ _PyPickle_Dumps(struct _pickle_context *ctx, PyObject *obj) } -struct sync_module_result { - PyObject *module; - PyObject *loaded; - PyObject *failed; -}; - -struct sync_module { - const char *filename; - char _filename[MAXPATHLEN+1]; - struct sync_module_result cached; -}; - -static void -sync_module_clear(struct sync_module *data) -{ - data->filename = NULL; - Py_CLEAR(data->cached.module); - Py_CLEAR(data->cached.loaded); - Py_CLEAR(data->cached.failed); -} - - struct _unpickle_context { PyThreadState *tstate; // We only special-case the __main__ module, @@ -491,142 +598,89 @@ _unpickle_context_clear(struct _unpickle_context *ctx) sync_module_clear(&ctx->main); } -static struct sync_module_result -_unpickle_context_get_module(struct _unpickle_context *ctx, - const char *modname) +static int +check_missing___main___attr(PyObject *exc) { - if (strcmp(modname, "__main__") == 0) { - return ctx->main.cached; + assert(!PyErr_Occurred()); + if (!PyErr_GivenExceptionMatches(exc, PyExc_AttributeError)) { + return 0; } - else { - return (struct sync_module_result){ - .failed = PyExc_NotImplementedError, - }; + + // Get the error message. + PyObject *args = PyException_GetArgs(exc); + if (args == NULL || args == Py_None || PyObject_Size(args) < 1) { + Py_XDECREF(args); + assert(!PyErr_Occurred()); + return 0; + } + PyObject *msgobj = args; + if (!PyUnicode_Check(msgobj)) { + msgobj = PySequence_GetItem(args, 0); + Py_DECREF(args); + if (msgobj == NULL) { + PyErr_Clear(); + return 0; + } } + const char *err = PyUnicode_AsUTF8(msgobj); + + // Check if it's a missing __main__ attr. + int cmp = strncmp(err, "module '__main__' has no attribute '", 36); + Py_DECREF(msgobj); + return cmp == 0; } -static struct sync_module_result -_unpickle_context_set_module(struct _unpickle_context *ctx, - const char *modname) +static PyObject * +_PyPickle_Loads(struct _unpickle_context *ctx, PyObject *pickled) { - struct sync_module_result res = {0}; - struct sync_module_result *cached = NULL; - const char *filename = NULL; - const char *run_modname = modname; - if (strcmp(modname, "__main__") == 0) { - cached = &ctx->main.cached; - filename = ctx->main.filename; - // We don't want to trigger "if __name__ == '__main__':". - run_modname = "<fake __main__>"; - } - else { - res.failed = PyExc_NotImplementedError; - goto finally; - } + PyThreadState *tstate = ctx->tstate; - res.module = import_get_module(ctx->tstate, modname); - if (res.module == NULL) { - res.failed = _PyErr_GetRaisedException(ctx->tstate); - assert(res.failed != NULL); - goto finally; + PyObject *exc = NULL; + PyObject *loads = PyImport_ImportModuleAttrString("pickle", "loads"); + if (loads == NULL) { + return NULL; } - if (filename == NULL) { - Py_CLEAR(res.module); - res.failed = PyExc_NotImplementedError; + // Make an initial attempt to unpickle. + PyObject *obj = PyObject_CallOneArg(loads, pickled); + if (obj != NULL) { goto finally; } - res.loaded = runpy_run_path(filename, run_modname); - if (res.loaded == NULL) { - Py_CLEAR(res.module); - res.failed = _PyErr_GetRaisedException(ctx->tstate); - assert(res.failed != NULL); + assert(_PyErr_Occurred(tstate)); + if (ctx == NULL) { goto finally; } - -finally: - if (cached != NULL) { - assert(cached->module == NULL); - assert(cached->loaded == NULL); - assert(cached->failed == NULL); - *cached = res; - } - return res; -} - - -static int -_handle_unpickle_missing_attr(struct _unpickle_context *ctx, PyObject *exc) -{ - // The caller must check if an exception is set or not when -1 is returned. - assert(!_PyErr_Occurred(ctx->tstate)); - assert(PyErr_GivenExceptionMatches(exc, PyExc_AttributeError)); - struct attributeerror_info info; - if (_parse_attributeerror(exc, &info) < 0) { - return -1; + exc = _PyErr_GetRaisedException(tstate); + if (!check_missing___main___attr(exc)) { + goto finally; } - // Get the module. - struct sync_module_result mod = _unpickle_context_get_module(ctx, info.modname); - if (mod.failed != NULL) { - // It must have failed previously. - return -1; - } - if (mod.module == NULL) { - mod = _unpickle_context_set_module(ctx, info.modname); - if (mod.failed != NULL) { - return -1; - } - assert(mod.module != NULL); + // Temporarily swap in a fake __main__ loaded from the original + // file and cached. Note that functions will use the cached ns + // for __globals__, // not the actual module. + if (ensure_isolated_main(tstate, &ctx->main) < 0) { + goto finally; } - - // Bail out if it is unexpectedly set already. - if (PyObject_HasAttrString(mod.module, info.attrname)) { - return -1; + if (apply_isolated_main(tstate, &ctx->main) < 0) { + goto finally; } - // Try setting the attribute. - PyObject *value = NULL; - if (PyDict_GetItemStringRef(mod.loaded, info.attrname, &value) <= 0) { - return -1; - } - assert(value != NULL); - int res = PyObject_SetAttrString(mod.module, info.attrname, value); - Py_DECREF(value); - if (res < 0) { - return -1; + // Try to unpickle once more. + obj = PyObject_CallOneArg(loads, pickled); + restore_main(tstate, &ctx->main); + if (obj == NULL) { + goto finally; } + Py_CLEAR(exc); - return 0; -} - -static PyObject * -_PyPickle_Loads(struct _unpickle_context *ctx, PyObject *pickled) -{ - PyObject *loads = PyImport_ImportModuleAttrString("pickle", "loads"); - if (loads == NULL) { - return NULL; - } - PyObject *obj = PyObject_CallOneArg(loads, pickled); - if (ctx != NULL) { - while (obj == NULL) { - assert(_PyErr_Occurred(ctx->tstate)); - if (!PyErr_ExceptionMatches(PyExc_AttributeError)) { - // We leave other failures unhandled. - break; - } - // Try setting the attr if not set. - PyObject *exc = _PyErr_GetRaisedException(ctx->tstate); - if (_handle_unpickle_missing_attr(ctx, exc) < 0) { - // Any resulting exceptions are ignored - // in favor of the original. - _PyErr_SetRaisedException(ctx->tstate, exc); - break; - } - Py_CLEAR(exc); - // Retry with the attribute set. - obj = PyObject_CallOneArg(loads, pickled); +finally: + if (exc != NULL) { + if (_PyErr_Occurred(tstate)) { + sync_module_capture_exc(tstate, &ctx->main); } + // We restore the original exception. + // It might make sense to chain it (__context__). + _PyErr_SetRaisedException(tstate, exc); } Py_DECREF(loads); return obj; @@ -784,35 +838,167 @@ _PyMarshal_GetXIData(PyThreadState *tstate, PyObject *obj, _PyXIData_t *xidata) } -/* using cross-interpreter data */ - -PyObject * -_PyXIData_NewObject(_PyXIData_t *xidata) -{ - return xidata->new_object(xidata); -} +/* script wrapper */ static int -_call_clear_xidata(void *data) +verify_script(PyThreadState *tstate, PyCodeObject *co, int checked, int pure) { - _xidata_clear((_PyXIData_t *)data); + // Make sure it isn't a closure and (optionally) doesn't use globals. + PyObject *builtins = NULL; + if (pure) { + builtins = _PyEval_GetBuiltins(tstate); + assert(builtins != NULL); + } + if (checked) { + assert(_PyCode_VerifyStateless(tstate, co, NULL, NULL, builtins) == 0); + } + else if (_PyCode_VerifyStateless(tstate, co, NULL, NULL, builtins) < 0) { + return -1; + } + // Make sure it doesn't have args. + if (co->co_argcount > 0 + || co->co_posonlyargcount > 0 + || co->co_kwonlyargcount > 0 + || co->co_flags & (CO_VARARGS | CO_VARKEYWORDS)) + { + _PyErr_SetString(tstate, PyExc_ValueError, + "code with args not supported"); + return -1; + } + // Make sure it doesn't return anything. + if (!_PyCode_ReturnsOnlyNone(co)) { + _PyErr_SetString(tstate, PyExc_ValueError, + "code that returns a value is not a script"); + return -1; + } return 0; } static int -_xidata_release(_PyXIData_t *xidata, int rawfree) -{ - if ((xidata->data == NULL || xidata->free == NULL) && xidata->obj == NULL) { - // Nothing to release! - if (rawfree) { - PyMem_RawFree(xidata); - } - else { - xidata->data = NULL; +get_script_xidata(PyThreadState *tstate, PyObject *obj, int pure, + _PyXIData_t *xidata) +{ + // Get the corresponding code object. + PyObject *code = NULL; + int checked = 0; + if (PyCode_Check(obj)) { + code = obj; + Py_INCREF(code); + } + else if (PyFunction_Check(obj)) { + code = PyFunction_GET_CODE(obj); + assert(code != NULL); + Py_INCREF(code); + if (pure) { + if (_PyFunction_VerifyStateless(tstate, obj) < 0) { + goto error; + } + checked = 1; } - return 0; } - + else { + const char *filename = "<script>"; + int optimize = 0; + PyCompilerFlags cf = _PyCompilerFlags_INIT; + cf.cf_flags = PyCF_SOURCE_IS_UTF8; + PyObject *ref = NULL; + const char *script = _Py_SourceAsString(obj, "???", "???", &cf, &ref); + if (script == NULL) { + if (!_PyObject_SupportedAsScript(obj)) { + // We discard the raised exception. + _PyErr_Format(tstate, PyExc_TypeError, + "unsupported script %R", obj); + } + goto error; + } +#ifdef Py_GIL_DISABLED + // Don't immortalize code constants to avoid memory leaks. + ((_PyThreadStateImpl *)tstate)->suppress_co_const_immortalization++; +#endif + code = Py_CompileStringExFlags( + script, filename, Py_file_input, &cf, optimize); +#ifdef Py_GIL_DISABLED + ((_PyThreadStateImpl *)tstate)->suppress_co_const_immortalization--; +#endif + Py_XDECREF(ref); + if (code == NULL) { + goto error; + } + // Compiled text can't have args or any return statements, + // nor be a closure. It can use globals though. + if (!pure) { + // We don't need to check for globals either. + checked = 1; + } + } + + // Make sure it's actually a script. + if (verify_script(tstate, (PyCodeObject *)code, checked, pure) < 0) { + goto error; + } + + // Convert the code object. + int res = _PyCode_GetXIData(tstate, code, xidata); + Py_DECREF(code); + if (res < 0) { + return -1; + } + return 0; + +error: + Py_XDECREF(code); + PyObject *cause = _PyErr_GetRaisedException(tstate); + assert(cause != NULL); + _set_xid_lookup_failure( + tstate, NULL, "object not a valid script", cause); + Py_DECREF(cause); + return -1; +} + +int +_PyCode_GetScriptXIData(PyThreadState *tstate, + PyObject *obj, _PyXIData_t *xidata) +{ + return get_script_xidata(tstate, obj, 0, xidata); +} + +int +_PyCode_GetPureScriptXIData(PyThreadState *tstate, + PyObject *obj, _PyXIData_t *xidata) +{ + return get_script_xidata(tstate, obj, 1, xidata); +} + + +/* using cross-interpreter data */ + +PyObject * +_PyXIData_NewObject(_PyXIData_t *xidata) +{ + return xidata->new_object(xidata); +} + +static int +_call_clear_xidata(void *data) +{ + _xidata_clear((_PyXIData_t *)data); + return 0; +} + +static int +_xidata_release(_PyXIData_t *xidata, int rawfree) +{ + if ((xidata->data == NULL || xidata->free == NULL) && xidata->obj == NULL) { + // Nothing to release! + if (rawfree) { + PyMem_RawFree(xidata); + } + else { + xidata->data = NULL; + } + return 0; + } + // Switch to the original interpreter. PyInterpreterState *interp = _PyInterpreterState_LookUpID( _PyXIData_INTERPID(xidata)); @@ -853,7 +1039,7 @@ _PyXIData_ReleaseAndRawFree(_PyXIData_t *xidata) /* convenience utilities */ /*************************/ -static const char * +static char * _copy_string_obj_raw(PyObject *strobj, Py_ssize_t *p_size) { Py_ssize_t size = -1; @@ -954,11 +1140,16 @@ _format_TracebackException(PyObject *tbexc) } Py_ssize_t size = -1; - const char *formatted = _copy_string_obj_raw(formatted_obj, &size); + char *formatted = _copy_string_obj_raw(formatted_obj, &size); Py_DECREF(formatted_obj); - // We remove trailing the newline added by TracebackException.format(). - assert(formatted[size-1] == '\n'); - ((char *)formatted)[size-1] = '\0'; + if (formatted == NULL || size == 0) { + return formatted; + } + assert(formatted[size] == '\0'); + // Remove a trailing newline if needed. + if (formatted[size-1] == '\n') { + formatted[size-1] = '\0'; + } return formatted; } @@ -968,8 +1159,8 @@ _release_xid_data(_PyXIData_t *xidata, int rawfree) { PyObject *exc = PyErr_GetRaisedException(); int res = rawfree - ? _PyXIData_Release(xidata) - : _PyXIData_ReleaseAndRawFree(xidata); + ? _PyXIData_ReleaseAndRawFree(xidata) + : _PyXIData_Release(xidata); if (res < 0) { /* The owning interpreter is already destroyed. */ _PyXIData_Clear(NULL, xidata); @@ -1130,8 +1321,14 @@ _excinfo_normalize_type(struct _excinfo_type *info, *p_module = module; } +static int +excinfo_is_set(_PyXI_excinfo *info) +{ + return info->type.name != NULL || info->msg != NULL; +} + static void -_PyXI_excinfo_Clear(_PyXI_excinfo *info) +_PyXI_excinfo_clear(_PyXI_excinfo *info) { _excinfo_clear_type(&info->type); if (info->msg != NULL) { @@ -1181,7 +1378,7 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) assert(exc != NULL); if (PyErr_GivenExceptionMatches(exc, PyExc_MemoryError)) { - _PyXI_excinfo_Clear(info); + _PyXI_excinfo_clear(info); return NULL; } const char *failure = NULL; @@ -1227,7 +1424,7 @@ _PyXI_excinfo_InitFromException(_PyXI_excinfo *info, PyObject *exc) error: assert(failure != NULL); - _PyXI_excinfo_Clear(info); + _PyXI_excinfo_clear(info); return failure; } @@ -1278,7 +1475,7 @@ _PyXI_excinfo_InitFromObject(_PyXI_excinfo *info, PyObject *obj) error: assert(failure != NULL); - _PyXI_excinfo_Clear(info); + _PyXI_excinfo_clear(info); return failure; } @@ -1291,6 +1488,11 @@ _PyXI_excinfo_Apply(_PyXI_excinfo *info, PyObject *exctype) if (tbexc == NULL) { PyErr_Clear(); } + else { + PyErr_SetObject(exctype, tbexc); + Py_DECREF(tbexc); + return; + } } PyObject *formatted = _PyXI_excinfo_format(info); @@ -1436,13 +1638,18 @@ _PyXI_excinfo_AsObject(_PyXI_excinfo *info) } -int -_PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc) +_PyXI_excinfo * +_PyXI_NewExcInfo(PyObject *exc) { assert(!PyErr_Occurred()); if (exc == NULL || exc == Py_None) { PyErr_SetString(PyExc_ValueError, "missing exc"); - return -1; + return NULL; + } + _PyXI_excinfo *info = PyMem_RawCalloc(1, sizeof(_PyXI_excinfo)); + if (info == NULL) { + PyErr_NoMemory(); + return NULL; } const char *failure; if (PyExceptionInstance_Check(exc) || PyExceptionClass_Check(exc)) { @@ -1452,10 +1659,19 @@ _PyXI_InitExcInfo(_PyXI_excinfo *info, PyObject *exc) failure = _PyXI_excinfo_InitFromObject(info, exc); } if (failure != NULL) { - PyErr_SetString(PyExc_Exception, failure); - return -1; + PyMem_RawFree(info); + set_exc_with_cause(PyExc_Exception, failure); + return NULL; } - return 0; + return info; +} + +void +_PyXI_FreeExcInfo(_PyXI_excinfo *info) +{ + assert(info != NULL); + _PyXI_excinfo_clear(info); + PyMem_RawFree(info); } PyObject * @@ -1470,12 +1686,6 @@ _PyXI_ExcInfoAsObject(_PyXI_excinfo *info) return _PyXI_excinfo_AsObject(info); } -void -_PyXI_ClearExcInfo(_PyXI_excinfo *info) -{ - _PyXI_excinfo_Clear(info); -} - /***************************/ /* short-term data sharing */ @@ -1489,14 +1699,9 @@ _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp) PyThreadState *tstate = _PyThreadState_GET(); assert(!PyErr_Occurred()); + assert(code != _PyXI_ERR_NO_ERROR); + assert(code != _PyXI_ERR_UNCAUGHT_EXCEPTION); switch (code) { - case _PyXI_ERR_NO_ERROR: _Py_FALLTHROUGH; - case _PyXI_ERR_UNCAUGHT_EXCEPTION: - // There is nothing to apply. -#ifdef Py_DEBUG - Py_UNREACHABLE(); -#endif - return 0; case _PyXI_ERR_OTHER: // XXX msg? PyErr_SetNone(PyExc_InterpreterError); @@ -1516,12 +1721,20 @@ _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp) PyErr_SetString(PyExc_InterpreterError, "failed to apply namespace to __main__"); break; + case _PyXI_ERR_PRESERVE_FAILURE: + PyErr_SetString(PyExc_InterpreterError, + "failed to preserve objects across session"); + break; + case _PyXI_ERR_EXC_PROPAGATION_FAILURE: + PyErr_SetString(PyExc_InterpreterError, + "failed to transfer exception between interpreters"); + break; case _PyXI_ERR_NOT_SHAREABLE: _set_xid_lookup_failure(tstate, NULL, NULL, NULL); break; default: #ifdef Py_DEBUG - Py_UNREACHABLE(); + Py_FatalError("unsupported error code"); #else PyErr_Format(PyExc_RuntimeError, "unsupported error code %d", code); #endif @@ -1530,70 +1743,276 @@ _PyXI_ApplyErrorCode(_PyXI_errcode code, PyInterpreterState *interp) return -1; } +/* basic failure info */ + +struct xi_failure { + // The kind of error to propagate. + _PyXI_errcode code; + // The propagated message. + const char *msg; + int msg_owned; +}; // _PyXI_failure + +#define XI_FAILURE_INIT (_PyXI_failure){ .code = _PyXI_ERR_NO_ERROR } + +static void +clear_xi_failure(_PyXI_failure *failure) +{ + if (failure->msg != NULL && failure->msg_owned) { + PyMem_RawFree((void*)failure->msg); + } + *failure = XI_FAILURE_INIT; +} + +static void +copy_xi_failure(_PyXI_failure *dest, _PyXI_failure *src) +{ + *dest = *src; + dest->msg_owned = 0; +} + +_PyXI_failure * +_PyXI_NewFailure(void) +{ + _PyXI_failure *failure = PyMem_RawMalloc(sizeof(_PyXI_failure)); + if (failure == NULL) { + PyErr_NoMemory(); + return NULL; + } + *failure = XI_FAILURE_INIT; + return failure; +} + +void +_PyXI_FreeFailure(_PyXI_failure *failure) +{ + clear_xi_failure(failure); + PyMem_RawFree(failure); +} + +_PyXI_errcode +_PyXI_GetFailureCode(_PyXI_failure *failure) +{ + if (failure == NULL) { + return _PyXI_ERR_NO_ERROR; + } + return failure->code; +} + +void +_PyXI_InitFailureUTF8(_PyXI_failure *failure, + _PyXI_errcode code, const char *msg) +{ + *failure = (_PyXI_failure){ + .code = code, + .msg = msg, + .msg_owned = 0, + }; +} + +int +_PyXI_InitFailure(_PyXI_failure *failure, _PyXI_errcode code, PyObject *obj) +{ + *failure = (_PyXI_failure){ + .code = code, + .msg = NULL, + .msg_owned = 0, + }; + if (obj == NULL) { + return 0; + } + + PyObject *msgobj = PyObject_Str(obj); + if (msgobj == NULL) { + return -1; + } + // This will leak if not paired with clear_xi_failure(). + // That happens automatically in _capture_current_exception(). + const char *msg = _copy_string_obj_raw(msgobj, NULL); + Py_DECREF(msgobj); + if (msg == NULL) { + return -1; + } + *failure = (_PyXI_failure){ + .code = code, + .msg = msg, + .msg_owned = 1, + }; + return 0; +} + /* shared exceptions */ -static const char * -_PyXI_InitError(_PyXI_error *error, PyObject *excobj, _PyXI_errcode code) +typedef struct { + // The originating interpreter. + PyInterpreterState *interp; + // The error to propagate, if different from the uncaught exception. + _PyXI_failure *override; + _PyXI_failure _override; + // The exception information to propagate, if applicable. + // This is populated only for some error codes, + // but always for _PyXI_ERR_UNCAUGHT_EXCEPTION. + _PyXI_excinfo uncaught; +} _PyXI_error; + +static void +xi_error_clear(_PyXI_error *err) { - if (error->interp == NULL) { - error->interp = PyInterpreterState_Get(); + err->interp = NULL; + if (err->override != NULL) { + clear_xi_failure(err->override); } + _PyXI_excinfo_clear(&err->uncaught); +} - const char *failure = NULL; - if (code == _PyXI_ERR_UNCAUGHT_EXCEPTION) { - // There is an unhandled exception we need to propagate. - failure = _PyXI_excinfo_InitFromException(&error->uncaught, excobj); - if (failure != NULL) { - // We failed to initialize error->uncaught. - // XXX Print the excobj/traceback? Emit a warning? - // XXX Print the current exception/traceback? - if (PyErr_ExceptionMatches(PyExc_MemoryError)) { - error->code = _PyXI_ERR_NO_MEMORY; - } - else { - error->code = _PyXI_ERR_OTHER; - } - PyErr_Clear(); +static int +xi_error_is_set(_PyXI_error *error) +{ + if (error->override != NULL) { + assert(error->override->code != _PyXI_ERR_NO_ERROR); + assert(error->override->code != _PyXI_ERR_UNCAUGHT_EXCEPTION + || excinfo_is_set(&error->uncaught)); + return 1; + } + return excinfo_is_set(&error->uncaught); +} + +static int +xi_error_has_override(_PyXI_error *err) +{ + if (err->override == NULL) { + return 0; + } + return (err->override->code != _PyXI_ERR_NO_ERROR + && err->override->code != _PyXI_ERR_UNCAUGHT_EXCEPTION); +} + +static PyObject * +xi_error_resolve_current_exc(PyThreadState *tstate, + _PyXI_failure *override) +{ + assert(override == NULL || override->code != _PyXI_ERR_NO_ERROR); + + PyObject *exc = _PyErr_GetRaisedException(tstate); + if (exc == NULL) { + assert(override == NULL + || override->code != _PyXI_ERR_UNCAUGHT_EXCEPTION); + } + else if (override == NULL) { + // This is equivalent to _PyXI_ERR_UNCAUGHT_EXCEPTION. + } + else if (override->code == _PyXI_ERR_UNCAUGHT_EXCEPTION) { + // We want to actually capture the current exception. + } + else if (exc != NULL) { + // It might make sense to do similarly for other codes. + if (override->code == _PyXI_ERR_ALREADY_RUNNING) { + // We don't need the exception info. + Py_CLEAR(exc); + } + // ...else we want to actually capture the current exception. + } + return exc; +} + +static void +xi_error_set_override(PyThreadState *tstate, _PyXI_error *err, + _PyXI_failure *override) +{ + assert(err->override == NULL); + assert(override != NULL); + assert(override->code != _PyXI_ERR_NO_ERROR); + // Use xi_error_set_exc() instead of setting _PyXI_ERR_UNCAUGHT_EXCEPTION.. + assert(override->code != _PyXI_ERR_UNCAUGHT_EXCEPTION); + err->override = &err->_override; + // The caller still owns override->msg. + copy_xi_failure(&err->_override, override); + err->interp = tstate->interp; +} + +static void +xi_error_set_override_code(PyThreadState *tstate, _PyXI_error *err, + _PyXI_errcode code) +{ + _PyXI_failure override = XI_FAILURE_INIT; + override.code = code; + xi_error_set_override(tstate, err, &override); +} + +static const char * +xi_error_set_exc(PyThreadState *tstate, _PyXI_error *err, PyObject *exc) +{ + assert(!_PyErr_Occurred(tstate)); + assert(!xi_error_is_set(err)); + assert(err->override == NULL); + assert(err->interp == NULL); + assert(exc != NULL); + const char *failure = + _PyXI_excinfo_InitFromException(&err->uncaught, exc); + if (failure != NULL) { + // We failed to initialize err->uncaught. + // XXX Print the excobj/traceback? Emit a warning? + // XXX Print the current exception/traceback? + if (_PyErr_ExceptionMatches(tstate, PyExc_MemoryError)) { + xi_error_set_override_code(tstate, err, _PyXI_ERR_NO_MEMORY); } else { - error->code = code; + xi_error_set_override_code(tstate, err, _PyXI_ERR_OTHER); } - assert(error->code != _PyXI_ERR_NO_ERROR); - } - else { - // There is an error code we need to propagate. - assert(excobj == NULL); - assert(code != _PyXI_ERR_NO_ERROR); - error->code = code; - _PyXI_excinfo_Clear(&error->uncaught); + PyErr_Clear(); } return failure; } -PyObject * -_PyXI_ApplyError(_PyXI_error *error) +static PyObject * +_PyXI_ApplyError(_PyXI_error *error, const char *failure) { PyThreadState *tstate = PyThreadState_Get(); - if (error->code == _PyXI_ERR_UNCAUGHT_EXCEPTION) { - // Raise an exception that proxies the propagated exception. + + if (failure != NULL) { + xi_error_clear(error); + return NULL; + } + + _PyXI_errcode code = _PyXI_ERR_UNCAUGHT_EXCEPTION; + if (error->override != NULL) { + code = error->override->code; + assert(code != _PyXI_ERR_NO_ERROR); + } + + if (code == _PyXI_ERR_UNCAUGHT_EXCEPTION) { + // We will raise an exception that proxies the propagated exception. return _PyXI_excinfo_AsObject(&error->uncaught); } - else if (error->code == _PyXI_ERR_NOT_SHAREABLE) { + else if (code == _PyXI_ERR_NOT_SHAREABLE) { // Propagate the exception directly. assert(!_PyErr_Occurred(tstate)); - _set_xid_lookup_failure(tstate, NULL, error->uncaught.msg, NULL); + PyObject *cause = NULL; + if (excinfo_is_set(&error->uncaught)) { + // Maybe instead set a PyExc_ExceptionSnapshot as __cause__? + // That type doesn't exist currently + // but would look like interpreters.ExecutionFailed. + _PyXI_excinfo_Apply(&error->uncaught, PyExc_Exception); + cause = _PyErr_GetRaisedException(tstate); + } + const char *msg = error->override != NULL + ? error->override->msg + : error->uncaught.msg; + _set_xid_lookup_failure(tstate, NULL, msg, cause); + Py_XDECREF(cause); } else { // Raise an exception corresponding to the code. - assert(error->code != _PyXI_ERR_NO_ERROR); - (void)_PyXI_ApplyErrorCode(error->code, error->interp); - if (error->uncaught.type.name != NULL || error->uncaught.msg != NULL) { + (void)_PyXI_ApplyErrorCode(code, error->interp); + assert(error->override == NULL || error->override->msg == NULL); + if (excinfo_is_set(&error->uncaught)) { // __context__ will be set to a proxy of the propagated exception. - PyObject *exc = PyErr_GetRaisedException(); + // (or use PyExc_ExceptionSnapshot like _PyXI_ERR_NOT_SHAREABLE?) + PyObject *exc = _PyErr_GetRaisedException(tstate); _PyXI_excinfo_Apply(&error->uncaught, PyExc_InterpreterError); - PyObject *exc2 = PyErr_GetRaisedException(); + PyObject *exc2 = _PyErr_GetRaisedException(tstate); PyException_SetContext(exc, exc2); - PyErr_SetRaisedException(exc); + _PyErr_SetRaisedException(tstate, exc); } } assert(PyErr_Occurred()); @@ -1624,6 +2043,7 @@ typedef struct _sharednsitem { // in a different interpreter to release the XI data. } _PyXI_namespace_item; +#ifndef NDEBUG static int _sharednsitem_is_initialized(_PyXI_namespace_item *item) { @@ -1632,6 +2052,7 @@ _sharednsitem_is_initialized(_PyXI_namespace_item *item) } return 0; } +#endif static int _sharednsitem_init(_PyXI_namespace_item *item, PyObject *key) @@ -1659,7 +2080,8 @@ _sharednsitem_has_value(_PyXI_namespace_item *item, int64_t *p_interpid) } static int -_sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value) +_sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value, + xidata_fallback_t fallback) { assert(_sharednsitem_is_initialized(item)); assert(item->xidata == NULL); @@ -1668,7 +2090,7 @@ _sharednsitem_set_value(_PyXI_namespace_item *item, PyObject *value) return -1; } PyThreadState *tstate = PyThreadState_Get(); - if (_PyObject_GetXIData(tstate, value, item->xidata) != 0) { + if (_PyObject_GetXIData(tstate, value, fallback, item->xidata) < 0) { PyMem_RawFree(item->xidata); item->xidata = NULL; // The caller may want to propagate PyExc_NotShareableError @@ -1700,7 +2122,8 @@ _sharednsitem_clear(_PyXI_namespace_item *item) } static int -_sharednsitem_copy_from_ns(struct _sharednsitem *item, PyObject *ns) +_sharednsitem_copy_from_ns(struct _sharednsitem *item, PyObject *ns, + xidata_fallback_t fallback) { assert(item->name != NULL); assert(item->xidata == NULL); @@ -1712,7 +2135,7 @@ _sharednsitem_copy_from_ns(struct _sharednsitem *item, PyObject *ns) // When applied, this item will be set to the default (or fail). return 0; } - if (_sharednsitem_set_value(item, value) < 0) { + if (_sharednsitem_set_value(item, value, fallback) < 0) { return -1; } return 0; @@ -1742,156 +2165,212 @@ _sharednsitem_apply(_PyXI_namespace_item *item, PyObject *ns, PyObject *dflt) return res; } -struct _sharedns { - Py_ssize_t len; - _PyXI_namespace_item *items; -}; -static _PyXI_namespace * -_sharedns_new(void) -{ - _PyXI_namespace *ns = PyMem_RawCalloc(sizeof(_PyXI_namespace), 1); - if (ns == NULL) { - PyErr_NoMemory(); - return NULL; - } - *ns = (_PyXI_namespace){ 0 }; - return ns; -} +typedef struct { + Py_ssize_t maxitems; + Py_ssize_t numnames; + Py_ssize_t numvalues; + _PyXI_namespace_item items[1]; +} _PyXI_namespace; +#ifndef NDEBUG static int -_sharedns_is_initialized(_PyXI_namespace *ns) +_sharedns_check_counts(_PyXI_namespace *ns) { - if (ns->len == 0) { - assert(ns->items == NULL); + if (ns->maxitems <= 0) { + return 0; + } + if (ns->numnames < 0) { + return 0; + } + if (ns->numnames > ns->maxitems) { + return 0; + } + if (ns->numvalues < 0) { + return 0; + } + if (ns->numvalues > ns->numnames) { return 0; } - - assert(ns->len > 0); - assert(ns->items != NULL); - assert(_sharednsitem_is_initialized(&ns->items[0])); - assert(ns->len == 1 - || _sharednsitem_is_initialized(&ns->items[ns->len - 1])); return 1; } -#define HAS_COMPLETE_DATA 1 -#define HAS_PARTIAL_DATA 2 - static int -_sharedns_has_xidata(_PyXI_namespace *ns, int64_t *p_interpid) +_sharedns_check_consistency(_PyXI_namespace *ns) { - // We expect _PyXI_namespace to always be initialized. - assert(_sharedns_is_initialized(ns)); - int res = 0; - _PyXI_namespace_item *item0 = &ns->items[0]; - if (!_sharednsitem_is_initialized(item0)) { + if (!_sharedns_check_counts(ns)) { return 0; } - int64_t interpid0 = -1; - if (!_sharednsitem_has_value(item0, &interpid0)) { - return 0; + + Py_ssize_t i = 0; + _PyXI_namespace_item *item; + if (ns->numvalues > 0) { + item = &ns->items[0]; + if (!_sharednsitem_is_initialized(item)) { + return 0; + } + int64_t interpid0 = -1; + if (!_sharednsitem_has_value(item, &interpid0)) { + return 0; + } + i += 1; + for (; i < ns->numvalues; i++) { + item = &ns->items[i]; + if (!_sharednsitem_is_initialized(item)) { + return 0; + } + int64_t interpid = -1; + if (!_sharednsitem_has_value(item, &interpid)) { + return 0; + } + if (interpid != interpid0) { + return 0; + } + } } - if (ns->len > 1) { - // At this point we know it is has at least partial data. - _PyXI_namespace_item *itemN = &ns->items[ns->len-1]; - if (!_sharednsitem_is_initialized(itemN)) { - res = HAS_PARTIAL_DATA; - goto finally; + for (; i < ns->numnames; i++) { + item = &ns->items[i]; + if (!_sharednsitem_is_initialized(item)) { + return 0; } - int64_t interpidN = -1; - if (!_sharednsitem_has_value(itemN, &interpidN)) { - res = HAS_PARTIAL_DATA; - goto finally; + if (_sharednsitem_has_value(item, NULL)) { + return 0; } - assert(interpidN == interpid0); } - res = HAS_COMPLETE_DATA; - *p_interpid = interpid0; - -finally: - return res; + for (; i < ns->maxitems; i++) { + item = &ns->items[i]; + if (_sharednsitem_is_initialized(item)) { + return 0; + } + if (_sharednsitem_has_value(item, NULL)) { + return 0; + } + } + return 1; } +#endif -static void -_sharedns_clear(_PyXI_namespace *ns) +static _PyXI_namespace * +_sharedns_alloc(Py_ssize_t maxitems) { - if (!_sharedns_is_initialized(ns)) { - return; + if (maxitems < 0) { + if (!PyErr_Occurred()) { + PyErr_BadInternalCall(); + } + return NULL; } - - // If the cross-interpreter data were allocated as part of - // _PyXI_namespace_item (instead of dynamically), this is where - // we would need verify that we are clearing the items in the - // correct interpreter, to avoid a race with releasing the XI data - // via a pending call. See _sharedns_has_xidata(). - for (Py_ssize_t i=0; i < ns->len; i++) { - _sharednsitem_clear(&ns->items[i]); + else if (maxitems == 0) { + PyErr_SetString(PyExc_ValueError, "empty namespaces not allowed"); + return NULL; + } + + // Check for overflow. + size_t fixedsize = sizeof(_PyXI_namespace) - sizeof(_PyXI_namespace_item); + if ((size_t)maxitems > + ((size_t)PY_SSIZE_T_MAX - fixedsize) / sizeof(_PyXI_namespace_item)) + { + PyErr_NoMemory(); + return NULL; + } + + // Allocate the value, including items. + size_t size = fixedsize + sizeof(_PyXI_namespace_item) * maxitems; + + _PyXI_namespace *ns = PyMem_RawCalloc(size, 1); + if (ns == NULL) { + PyErr_NoMemory(); + return NULL; } - PyMem_RawFree(ns->items); - ns->items = NULL; - ns->len = 0; + ns->maxitems = maxitems; + assert(_sharedns_check_consistency(ns)); + return ns; } static void _sharedns_free(_PyXI_namespace *ns) { - _sharedns_clear(ns); + // If we weren't always dynamically allocating the cross-interpreter + // data in each item then we would need to use a pending call + // to call _sharedns_free(), to avoid the race between freeing + // the shared namespace and releasing the XI data. + assert(_sharedns_check_counts(ns)); + Py_ssize_t i = 0; + _PyXI_namespace_item *item; + if (ns->numvalues > 0) { + // One or more items may have interpreter-specific data. +#ifndef NDEBUG + int64_t interpid = PyInterpreterState_GetID(PyInterpreterState_Get()); + int64_t interpid_i; +#endif + for (; i < ns->numvalues; i++) { + item = &ns->items[i]; + assert(_sharednsitem_is_initialized(item)); + // While we do want to ensure consistency across items, + // technically they don't need to match the current + // interpreter. However, we keep the constraint for + // simplicity, by giving _PyXI_FreeNamespace() the exclusive + // responsibility of dealing with the owning interpreter. + assert(_sharednsitem_has_value(item, &interpid_i)); + assert(interpid_i == interpid); + _sharednsitem_clear(item); + } + } + for (; i < ns->numnames; i++) { + item = &ns->items[i]; + assert(_sharednsitem_is_initialized(item)); + assert(!_sharednsitem_has_value(item, NULL)); + _sharednsitem_clear(item); + } +#ifndef NDEBUG + for (; i < ns->maxitems; i++) { + item = &ns->items[i]; + assert(!_sharednsitem_is_initialized(item)); + assert(!_sharednsitem_has_value(item, NULL)); + } +#endif + PyMem_RawFree(ns); } -static int -_sharedns_init(_PyXI_namespace *ns, PyObject *names) +static _PyXI_namespace * +_create_sharedns(PyObject *names) { - assert(!_sharedns_is_initialized(ns)); assert(names != NULL); - Py_ssize_t len = PyDict_CheckExact(names) + Py_ssize_t numnames = PyDict_CheckExact(names) ? PyDict_Size(names) : PySequence_Size(names); - if (len < 0) { - return -1; - } - if (len == 0) { - PyErr_SetString(PyExc_ValueError, "empty namespaces not allowed"); - return -1; - } - assert(len > 0); - // Allocate the items. - _PyXI_namespace_item *items = - PyMem_RawCalloc(sizeof(struct _sharednsitem), len); - if (items == NULL) { - PyErr_NoMemory(); - return -1; + _PyXI_namespace *ns = _sharedns_alloc(numnames); + if (ns == NULL) { + return NULL; } + _PyXI_namespace_item *items = ns->items; // Fill in the names. - Py_ssize_t i = -1; if (PyDict_CheckExact(names)) { + Py_ssize_t i = 0; Py_ssize_t pos = 0; - for (i=0; i < len; i++) { - PyObject *key; - if (!PyDict_Next(names, &pos, &key, NULL)) { - // This should not be possible. - assert(0); - goto error; - } - if (_sharednsitem_init(&items[i], key) < 0) { + PyObject *name; + while(PyDict_Next(names, &pos, &name, NULL)) { + if (_sharednsitem_init(&items[i], name) < 0) { goto error; } + ns->numnames += 1; + i += 1; } } else if (PySequence_Check(names)) { - for (i=0; i < len; i++) { - PyObject *key = PySequence_GetItem(names, i); - if (key == NULL) { + for (Py_ssize_t i = 0; i < numnames; i++) { + PyObject *name = PySequence_GetItem(names, i); + if (name == NULL) { goto error; } - int res = _sharednsitem_init(&items[i], key); - Py_DECREF(key); + int res = _sharednsitem_init(&items[i], name); + Py_DECREF(name); if (res < 0) { goto error; } + ns->numnames += 1; } } else { @@ -1899,140 +2378,84 @@ _sharedns_init(_PyXI_namespace *ns, PyObject *names) "non-sequence namespace not supported"); goto error; } - - ns->items = items; - ns->len = len; - assert(_sharedns_is_initialized(ns)); - return 0; + assert(ns->numnames == ns->maxitems); + return ns; error: - for (Py_ssize_t j=0; j < i; j++) { - _sharednsitem_clear(&items[j]); - } - PyMem_RawFree(items); - assert(!_sharedns_is_initialized(ns)); - return -1; + _sharedns_free(ns); + return NULL; } -void -_PyXI_FreeNamespace(_PyXI_namespace *ns) -{ - if (!_sharedns_is_initialized(ns)) { - return; - } - - int64_t interpid = -1; - if (!_sharedns_has_xidata(ns, &interpid)) { - _sharedns_free(ns); - return; - } +static void _propagate_not_shareable_error(PyThreadState *, + _PyXI_failure *); - if (interpid == PyInterpreterState_GetID(PyInterpreterState_Get())) { - _sharedns_free(ns); - } - else { - // If we weren't always dynamically allocating the cross-interpreter - // data in each item then we would need to using a pending call - // to call _sharedns_free(), to avoid the race between freeing - // the shared namespace and releasing the XI data. - _sharedns_free(ns); - } -} - -_PyXI_namespace * -_PyXI_NamespaceFromNames(PyObject *names) +static int +_fill_sharedns(_PyXI_namespace *ns, PyObject *nsobj, + xidata_fallback_t fallback, _PyXI_failure *p_err) { - if (names == NULL || names == Py_None) { - return NULL; - } - - _PyXI_namespace *ns = _sharedns_new(); - if (ns == NULL) { - return NULL; - } - - if (_sharedns_init(ns, names) < 0) { - PyMem_RawFree(ns); - if (PySequence_Size(names) == 0) { - PyErr_Clear(); - } - return NULL; - } - - return ns; -} - -#ifndef NDEBUG -static int _session_is_active(_PyXI_session *); -#endif -static void _propagate_not_shareable_error(_PyXI_session *); - -int -_PyXI_FillNamespaceFromDict(_PyXI_namespace *ns, PyObject *nsobj, - _PyXI_session *session) -{ - // session must be entered already, if provided. - assert(session == NULL || _session_is_active(session)); - assert(_sharedns_is_initialized(ns)); - for (Py_ssize_t i=0; i < ns->len; i++) { - _PyXI_namespace_item *item = &ns->items[i]; - if (_sharednsitem_copy_from_ns(item, nsobj) < 0) { - _propagate_not_shareable_error(session); + // All items are expected to be shareable. + assert(_sharedns_check_counts(ns)); + assert(ns->numnames == ns->maxitems); + assert(ns->numvalues == 0); + PyThreadState *tstate = PyThreadState_Get(); + for (Py_ssize_t i=0; i < ns->maxitems; i++) { + if (_sharednsitem_copy_from_ns(&ns->items[i], nsobj, fallback) < 0) { + if (p_err != NULL) { + _propagate_not_shareable_error(tstate, p_err); + } // Clear out the ones we set so far. for (Py_ssize_t j=0; j < i; j++) { _sharednsitem_clear_value(&ns->items[j]); + ns->numvalues -= 1; } return -1; } + ns->numvalues += 1; } return 0; } -// All items are expected to be shareable. -static _PyXI_namespace * -_PyXI_NamespaceFromDict(PyObject *nsobj, _PyXI_session *session) +static int +_sharedns_free_pending(void *data) { - // session must be entered already, if provided. - assert(session == NULL || _session_is_active(session)); - if (nsobj == NULL || nsobj == Py_None) { - return NULL; - } - if (!PyDict_CheckExact(nsobj)) { - PyErr_SetString(PyExc_TypeError, "expected a dict"); - return NULL; - } + _sharedns_free((_PyXI_namespace *)data); + return 0; +} - _PyXI_namespace *ns = _sharedns_new(); - if (ns == NULL) { - return NULL; +static void +_destroy_sharedns(_PyXI_namespace *ns) +{ + assert(_sharedns_check_counts(ns)); + assert(ns->numnames == ns->maxitems); + if (ns->numvalues == 0) { + _sharedns_free(ns); + return; } - if (_sharedns_init(ns, nsobj) < 0) { - if (PyDict_Size(nsobj) == 0) { - PyMem_RawFree(ns); - PyErr_Clear(); - return NULL; - } - goto error; + int64_t interpid0; + if (!_sharednsitem_has_value(&ns->items[0], &interpid0)) { + // This shouldn't have been possible. + // We can deal with it in _sharedns_free(). + _sharedns_free(ns); + return; } - - if (_PyXI_FillNamespaceFromDict(ns, nsobj, session) < 0) { - goto error; + PyInterpreterState *interp = _PyInterpreterState_LookUpID(interpid0); + if (interp == PyInterpreterState_Get()) { + _sharedns_free(ns); + return; } - return ns; - -error: - assert(PyErr_Occurred() - || (session != NULL && session->error_override != NULL)); - _sharedns_free(ns); - return NULL; + // One or more items may have interpreter-specific data. + // Currently the xidata for each value is dynamically allocated, + // so technically we don't need to worry about that. + // However, explicitly adding a pending call here is simpler. + (void)_Py_CallInInterpreter(interp, _sharedns_free_pending, ns); } -int -_PyXI_ApplyNamespace(_PyXI_namespace *ns, PyObject *nsobj, PyObject *dflt) +static int +_apply_sharedns(_PyXI_namespace *ns, PyObject *nsobj, PyObject *dflt) { - for (Py_ssize_t i=0; i < ns->len; i++) { + for (Py_ssize_t i=0; i < ns->maxitems; i++) { if (_sharednsitem_apply(&ns->items[i], nsobj, dflt) != 0) { return -1; } @@ -2041,9 +2464,69 @@ _PyXI_ApplyNamespace(_PyXI_namespace *ns, PyObject *nsobj, PyObject *dflt) } -/**********************/ -/* high-level helpers */ -/**********************/ +/*********************************/ +/* switched-interpreter sessions */ +/*********************************/ + +struct xi_session { +#define SESSION_UNUSED 0 +#define SESSION_ACTIVE 1 + int status; + int switched; + + // Once a session has been entered, this is the tstate that was + // current before the session. If it is different from cur_tstate + // then we must have switched interpreters. Either way, this will + // be the current tstate once we exit the session. + PyThreadState *prev_tstate; + // Once a session has been entered, this is the current tstate. + // It must be current when the session exits. + PyThreadState *init_tstate; + // This is true if init_tstate needs cleanup during exit. + int own_init_tstate; + + // This is true if, while entering the session, init_thread took + // "ownership" of the interpreter's __main__ module. This means + // it is the only thread that is allowed to run code there. + // (Caveat: for now, users may still run exec() against the + // __main__ module's dict, though that isn't advisable.) + int running; + // This is a cached reference to the __dict__ of the entered + // interpreter's __main__ module. It is looked up when at the + // beginning of the session as a convenience. + PyObject *main_ns; + + // This is a dict of objects that will be available (via sharing) + // once the session exits. Do not access this directly; use + // _PyXI_Preserve() and _PyXI_GetPreserved() instead; + PyObject *_preserved; +}; + +_PyXI_session * +_PyXI_NewSession(void) +{ + _PyXI_session *session = PyMem_RawCalloc(1, sizeof(_PyXI_session)); + if (session == NULL) { + PyErr_NoMemory(); + return NULL; + } + return session; +} + +void +_PyXI_FreeSession(_PyXI_session *session) +{ + assert(session->status == SESSION_UNUSED); + PyMem_RawFree(session); +} + + +static inline int +_session_is_active(_PyXI_session *session) +{ + return session->status == SESSION_ACTIVE; +} + /* enter/exit a cross-interpreter session */ @@ -2051,29 +2534,33 @@ static void _enter_session(_PyXI_session *session, PyInterpreterState *interp) { // Set here and cleared in _exit_session(). + assert(session->status == SESSION_UNUSED); assert(!session->own_init_tstate); assert(session->init_tstate == NULL); assert(session->prev_tstate == NULL); // Set elsewhere and cleared in _exit_session(). assert(!session->running); assert(session->main_ns == NULL); - // Set elsewhere and cleared in _capture_current_exception(). - assert(session->error_override == NULL); - // Set elsewhere and cleared in _PyXI_ApplyCapturedException(). - assert(session->error == NULL); // Switch to interpreter. PyThreadState *tstate = PyThreadState_Get(); PyThreadState *prev = tstate; - if (interp != tstate->interp) { + int same_interp = (interp == tstate->interp); + if (!same_interp) { tstate = _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_EXEC); // XXX Possible GILState issues? - session->prev_tstate = PyThreadState_Swap(tstate); - assert(session->prev_tstate == prev); - session->own_init_tstate = 1; + PyThreadState *swapped = PyThreadState_Swap(tstate); + assert(swapped == prev); + (void)swapped; } - session->init_tstate = tstate; - session->prev_tstate = prev; + + *session = (_PyXI_session){ + .status = SESSION_ACTIVE, + .switched = !same_interp, + .init_tstate = tstate, + .prev_tstate = prev, + .own_init_tstate = !same_interp, + }; } static void @@ -2082,16 +2569,16 @@ _exit_session(_PyXI_session *session) PyThreadState *tstate = session->init_tstate; assert(tstate != NULL); assert(PyThreadState_Get() == tstate); + assert(!_PyErr_Occurred(tstate)); // Release any of the entered interpreters resources. - if (session->main_ns != NULL) { - Py_CLEAR(session->main_ns); - } + Py_CLEAR(session->main_ns); + Py_CLEAR(session->_preserved); // Ensure this thread no longer owns __main__. if (session->running) { _PyInterpreterState_SetNotRunningMain(tstate->interp); - assert(!PyErr_Occurred()); + assert(!_PyErr_Occurred(tstate)); session->running = 0; } @@ -2107,25 +2594,15 @@ _exit_session(_PyXI_session *session) else { assert(!session->own_init_tstate); } - session->prev_tstate = NULL; - session->init_tstate = NULL; -} -#ifndef NDEBUG -static int -_session_is_active(_PyXI_session *session) -{ - return (session->init_tstate != NULL); + *session = (_PyXI_session){0}; } -#endif static void -_propagate_not_shareable_error(_PyXI_session *session) +_propagate_not_shareable_error(PyThreadState *tstate, + _PyXI_failure *override) { - if (session == NULL) { - return; - } - PyThreadState *tstate = PyThreadState_Get(); + assert(override != NULL); PyObject *exctype = get_notshareableerror_type(tstate); if (exctype == NULL) { PyErr_FormatUnraisable( @@ -2134,166 +2611,463 @@ _propagate_not_shareable_error(_PyXI_session *session) } if (PyErr_ExceptionMatches(exctype)) { // We want to propagate the exception directly. - session->_error_override = _PyXI_ERR_NOT_SHAREABLE; - session->error_override = &session->_error_override; + *override = (_PyXI_failure){ + .code = _PyXI_ERR_NOT_SHAREABLE, + }; } } -static void -_capture_current_exception(_PyXI_session *session) + +static int _ensure_main_ns(_PyXI_session *, _PyXI_failure *); +static const char * capture_session_error(_PyXI_session *, _PyXI_error *, + _PyXI_failure *); + +int +_PyXI_Enter(_PyXI_session *session, + PyInterpreterState *interp, PyObject *nsupdates, + _PyXI_session_result *result) { - assert(session->error == NULL); - if (!PyErr_Occurred()) { - assert(session->error_override == NULL); - return; +#ifndef NDEBUG + PyThreadState *tstate = _PyThreadState_GET(); // Only used for asserts +#endif + + // Convert the attrs for cross-interpreter use. + _PyXI_namespace *sharedns = NULL; + if (nsupdates != NULL) { + assert(PyDict_Check(nsupdates)); + Py_ssize_t len = PyDict_Size(nsupdates); + if (len < 0) { + if (result != NULL) { + result->errcode = _PyXI_ERR_APPLY_NS_FAILURE; + } + return -1; + } + if (len > 0) { + sharedns = _create_sharedns(nsupdates); + if (sharedns == NULL) { + if (result != NULL) { + result->errcode = _PyXI_ERR_APPLY_NS_FAILURE; + } + return -1; + } + // For now we limit it to shareable objects. + xidata_fallback_t fallback = _PyXIDATA_XIDATA_ONLY; + _PyXI_failure _err = XI_FAILURE_INIT; + if (_fill_sharedns(sharedns, nsupdates, fallback, &_err) < 0) { + assert(_PyErr_Occurred(tstate)); + if (_err.code == _PyXI_ERR_NO_ERROR) { + _err.code = _PyXI_ERR_UNCAUGHT_EXCEPTION; + } + _destroy_sharedns(sharedns); + if (result != NULL) { + assert(_err.msg == NULL); + result->errcode = _err.code; + } + return -1; + } + } } - // Handle the exception override. - _PyXI_errcode *override = session->error_override; - session->error_override = NULL; - _PyXI_errcode errcode = override != NULL - ? *override - : _PyXI_ERR_UNCAUGHT_EXCEPTION; + // Switch to the requested interpreter (if necessary). + _enter_session(session, interp); + _PyXI_failure override = XI_FAILURE_INIT; + override.code = _PyXI_ERR_UNCAUGHT_EXCEPTION; +#ifndef NDEBUG + tstate = _PyThreadState_GET(); +#endif - // Pop the exception object. - PyObject *excval = NULL; - if (errcode == _PyXI_ERR_UNCAUGHT_EXCEPTION) { - // We want to actually capture the current exception. - excval = PyErr_GetRaisedException(); + // Ensure this thread owns __main__. + if (_PyInterpreterState_SetRunningMain(interp) < 0) { + // In the case where we didn't switch interpreters, it would + // be more efficient to leave the exception in place and return + // immediately. However, life is simpler if we don't. + override.code = _PyXI_ERR_ALREADY_RUNNING; + goto error; } - else if (errcode == _PyXI_ERR_ALREADY_RUNNING) { - // We don't need the exception info. - PyErr_Clear(); + session->running = 1; + + // Apply the cross-interpreter data. + if (sharedns != NULL) { + if (_ensure_main_ns(session, &override) < 0) { + goto error; + } + if (_apply_sharedns(sharedns, session->main_ns, NULL) < 0) { + override.code = _PyXI_ERR_APPLY_NS_FAILURE; + goto error; + } + _destroy_sharedns(sharedns); } - else { - // We could do a variety of things here, depending on errcode. - // However, for now we simply capture the exception and save - // the errcode. - excval = PyErr_GetRaisedException(); + + override.code = _PyXI_ERR_NO_ERROR; + assert(!_PyErr_Occurred(tstate)); + return 0; + +error: + // We want to propagate all exceptions here directly (best effort). + assert(override.code != _PyXI_ERR_NO_ERROR); + _PyXI_error err = {0}; + const char *failure = capture_session_error(session, &err, &override); + + // Exit the session. + _exit_session(session); +#ifndef NDEBUG + tstate = _PyThreadState_GET(); +#endif + + if (sharedns != NULL) { + _destroy_sharedns(sharedns); } - // Capture the exception. - _PyXI_error *err = &session->_error; - *err = (_PyXI_error){ - .interp = session->init_tstate->interp, - }; - const char *failure; - if (excval == NULL) { - failure = _PyXI_InitError(err, NULL, errcode); + // Apply the error from the other interpreter. + PyObject *excinfo = _PyXI_ApplyError(&err, failure); + xi_error_clear(&err); + if (excinfo != NULL) { + if (result != NULL) { + result->excinfo = excinfo; + } + else { +#ifdef Py_DEBUG + fprintf(stderr, "_PyXI_Enter(): uncaught exception discarded"); +#endif + Py_DECREF(excinfo); + } + } + assert(_PyErr_Occurred(tstate)); + + return -1; +} + +static int _pop_preserved(_PyXI_session *, _PyXI_namespace **, PyObject **, + _PyXI_failure *); +static int _finish_preserved(_PyXI_namespace *, PyObject **); + +int +_PyXI_Exit(_PyXI_session *session, _PyXI_failure *override, + _PyXI_session_result *result) +{ + PyThreadState *tstate = _PyThreadState_GET(); + int res = 0; + + // Capture the raised exception, if any. + _PyXI_error err = {0}; + const char *failure = NULL; + if (override != NULL && override->code == _PyXI_ERR_NO_ERROR) { + assert(override->msg == NULL); + override = NULL; + } + if (_PyErr_Occurred(tstate)) { + failure = capture_session_error(session, &err, override); } else { - failure = _PyXI_InitError(err, excval, _PyXI_ERR_UNCAUGHT_EXCEPTION); - Py_DECREF(excval); - if (failure == NULL && override != NULL) { - err->code = errcode; - } + assert(override == NULL); } - // Handle capture failure. - if (failure != NULL) { - // XXX Make this error message more generic. - fprintf(stderr, - "RunFailedError: script raised an uncaught exception (%s)", - failure); - err = NULL; + // Capture the preserved namespace. + _PyXI_namespace *preserved = NULL; + PyObject *preservedobj = NULL; + if (result != NULL) { + assert(!_PyErr_Occurred(tstate)); + _PyXI_failure _override = XI_FAILURE_INIT; + if (_pop_preserved( + session, &preserved, &preservedobj, &_override) < 0) + { + assert(preserved == NULL); + assert(preservedobj == NULL); + if (xi_error_is_set(&err)) { + // XXX Chain the exception (i.e. set __context__)? + PyErr_FormatUnraisable( + "Exception ignored while capturing preserved objects"); + clear_xi_failure(&_override); + } + else { + if (_override.code == _PyXI_ERR_NO_ERROR) { + _override.code = _PyXI_ERR_UNCAUGHT_EXCEPTION; + } + failure = capture_session_error(session, &err, &_override); + } + } } - // Finished! - assert(!PyErr_Occurred()); - session->error = err; -} + // Exit the session. + assert(!_PyErr_Occurred(tstate)); + _exit_session(session); + tstate = _PyThreadState_GET(); + + // Restore the preserved namespace. + assert(preserved == NULL || preservedobj == NULL); + if (_finish_preserved(preserved, &preservedobj) < 0) { + assert(preservedobj == NULL); + if (xi_error_is_set(&err)) { + // XXX Chain the exception (i.e. set __context__)? + PyErr_FormatUnraisable( + "Exception ignored while capturing preserved objects"); + } + else { + xi_error_set_override_code( + tstate, &err, _PyXI_ERR_PRESERVE_FAILURE); + _propagate_not_shareable_error(tstate, err.override); + } + } + if (result != NULL) { + result->preserved = preservedobj; + result->errcode = err.override != NULL + ? err.override->code + : _PyXI_ERR_NO_ERROR; + } -PyObject * -_PyXI_ApplyCapturedException(_PyXI_session *session) -{ - assert(!PyErr_Occurred()); - assert(session->error != NULL); - PyObject *res = _PyXI_ApplyError(session->error); - assert((res == NULL) != (PyErr_Occurred() == NULL)); - session->error = NULL; + // Apply the error from the other interpreter, if any. + if (xi_error_is_set(&err)) { + res = -1; + assert(!_PyErr_Occurred(tstate)); + PyObject *excinfo = _PyXI_ApplyError(&err, failure); + if (excinfo == NULL) { + assert(_PyErr_Occurred(tstate)); + if (result != NULL && !xi_error_has_override(&err)) { + _PyXI_ClearResult(result); + *result = (_PyXI_session_result){ + .errcode = _PyXI_ERR_EXC_PROPAGATION_FAILURE, + }; + } + } + else if (result != NULL) { + result->excinfo = excinfo; + } + else { +#ifdef Py_DEBUG + fprintf(stderr, "_PyXI_Exit(): uncaught exception discarded"); +#endif + Py_DECREF(excinfo); + } + xi_error_clear(&err); + } return res; } -int -_PyXI_HasCapturedException(_PyXI_session *session) -{ - return session->error != NULL; -} -int -_PyXI_Enter(_PyXI_session *session, - PyInterpreterState *interp, PyObject *nsupdates) +/* in an active cross-interpreter session */ + +static const char * +capture_session_error(_PyXI_session *session, _PyXI_error *err, + _PyXI_failure *override) { - // Convert the attrs for cross-interpreter use. - _PyXI_namespace *sharedns = NULL; - if (nsupdates != NULL) { - sharedns = _PyXI_NamespaceFromDict(nsupdates, NULL); - if (sharedns == NULL && PyErr_Occurred()) { - assert(session->error == NULL); - return -1; + assert(_session_is_active(session)); + assert(!xi_error_is_set(err)); + PyThreadState *tstate = session->init_tstate; + + // Normalize the exception override. + if (override != NULL) { + if (override->code == _PyXI_ERR_UNCAUGHT_EXCEPTION) { + assert(override->msg == NULL); + override = NULL; + } + else { + assert(override->code != _PyXI_ERR_NO_ERROR); } } - // Switch to the requested interpreter (if necessary). - _enter_session(session, interp); - PyThreadState *session_tstate = session->init_tstate; - _PyXI_errcode errcode = _PyXI_ERR_UNCAUGHT_EXCEPTION; + // Handle the exception, if any. + const char *failure = NULL; + PyObject *exc = xi_error_resolve_current_exc(tstate, override); + if (exc != NULL) { + // There is an unhandled exception we need to preserve. + failure = xi_error_set_exc(tstate, err, exc); + Py_DECREF(exc); + if (_PyErr_Occurred(tstate)) { + PyErr_FormatUnraisable(failure); + } + } - // Ensure this thread owns __main__. - if (_PyInterpreterState_SetRunningMain(interp) < 0) { - // In the case where we didn't switch interpreters, it would - // be more efficient to leave the exception in place and return - // immediately. However, life is simpler if we don't. - errcode = _PyXI_ERR_ALREADY_RUNNING; - goto error; + // Handle the override. + if (override != NULL && failure == NULL) { + xi_error_set_override(tstate, err, override); } - session->running = 1; + // Finished! + assert(!_PyErr_Occurred(tstate)); + return failure; +} + +static int +_ensure_main_ns(_PyXI_session *session, _PyXI_failure *failure) +{ + assert(_session_is_active(session)); + PyThreadState *tstate = session->init_tstate; + if (session->main_ns != NULL) { + return 0; + } // Cache __main__.__dict__. - PyObject *main_mod = _Py_GetMainModule(session_tstate); + PyObject *main_mod = _Py_GetMainModule(tstate); if (_Py_CheckMainModule(main_mod) < 0) { - errcode = _PyXI_ERR_MAIN_NS_FAILURE; - goto error; + Py_XDECREF(main_mod); + if (failure != NULL) { + *failure = (_PyXI_failure){ + .code = _PyXI_ERR_MAIN_NS_FAILURE, + }; + } + return -1; } PyObject *ns = PyModule_GetDict(main_mod); // borrowed Py_DECREF(main_mod); if (ns == NULL) { - errcode = _PyXI_ERR_MAIN_NS_FAILURE; - goto error; + if (failure != NULL) { + *failure = (_PyXI_failure){ + .code = _PyXI_ERR_MAIN_NS_FAILURE, + }; + } + return -1; } session->main_ns = Py_NewRef(ns); + return 0; +} - // Apply the cross-interpreter data. - if (sharedns != NULL) { - if (_PyXI_ApplyNamespace(sharedns, ns, NULL) < 0) { - errcode = _PyXI_ERR_APPLY_NS_FAILURE; +PyObject * +_PyXI_GetMainNamespace(_PyXI_session *session, _PyXI_failure *failure) +{ + if (!_session_is_active(session)) { + PyErr_SetString(PyExc_RuntimeError, "session not active"); + return NULL; + } + if (_ensure_main_ns(session, failure) < 0) { + return NULL; + } + return session->main_ns; +} + + +static int +_pop_preserved(_PyXI_session *session, + _PyXI_namespace **p_xidata, PyObject **p_obj, + _PyXI_failure *p_failure) +{ + _PyXI_failure failure = XI_FAILURE_INIT; + _PyXI_namespace *xidata = NULL; + assert(_PyThreadState_GET() == session->init_tstate); // active session + + if (session->_preserved == NULL) { + *p_xidata = NULL; + *p_obj = NULL; + return 0; + } + if (session->init_tstate == session->prev_tstate) { + // We did not switch interpreters. + *p_xidata = NULL; + *p_obj = session->_preserved; + session->_preserved = NULL; + return 0; + } + *p_obj = NULL; + + // We did switch interpreters. + Py_ssize_t len = PyDict_Size(session->_preserved); + if (len < 0) { + failure.code = _PyXI_ERR_PRESERVE_FAILURE; + goto error; + } + else if (len == 0) { + *p_xidata = NULL; + } + else { + xidata = _create_sharedns(session->_preserved); + if (xidata == NULL) { + failure.code = _PyXI_ERR_PRESERVE_FAILURE; + goto error; + } + if (_fill_sharedns(xidata, session->_preserved, + _PyXIDATA_FULL_FALLBACK, &failure) < 0) + { + if (failure.code != _PyXI_ERR_NOT_SHAREABLE) { + assert(failure.msg != NULL); + failure.code = _PyXI_ERR_PRESERVE_FAILURE; + } goto error; } - _PyXI_FreeNamespace(sharedns); + *p_xidata = xidata; } + Py_CLEAR(session->_preserved); + return 0; - errcode = _PyXI_ERR_NO_ERROR; - assert(!PyErr_Occurred()); +error: + if (p_failure != NULL) { + *p_failure = failure; + } + if (xidata != NULL) { + _destroy_sharedns(xidata); + } + return -1; +} + +static int +_finish_preserved(_PyXI_namespace *xidata, PyObject **p_preserved) +{ + if (xidata == NULL) { + return 0; + } + int res = -1; + if (p_preserved != NULL) { + PyObject *ns = PyDict_New(); + if (ns == NULL) { + goto finally; + } + if (_apply_sharedns(xidata, ns, NULL) < 0) { + Py_CLEAR(ns); + goto finally; + } + *p_preserved = ns; + } + res = 0; + +finally: + _destroy_sharedns(xidata); + return res; +} + +int +_PyXI_Preserve(_PyXI_session *session, const char *name, PyObject *value, + _PyXI_failure *p_failure) +{ + _PyXI_failure failure = XI_FAILURE_INIT; + if (!_session_is_active(session)) { + PyErr_SetString(PyExc_RuntimeError, "session not active"); + return -1; + } + if (session->_preserved == NULL) { + session->_preserved = PyDict_New(); + if (session->_preserved == NULL) { + set_exc_with_cause(PyExc_RuntimeError, + "failed to initialize preserved objects"); + failure.code = _PyXI_ERR_PRESERVE_FAILURE; + goto error; + } + } + if (PyDict_SetItemString(session->_preserved, name, value) < 0) { + set_exc_with_cause(PyExc_RuntimeError, "failed to preserve object"); + failure.code = _PyXI_ERR_PRESERVE_FAILURE; + goto error; + } return 0; error: - assert(PyErr_Occurred()); - // We want to propagate all exceptions here directly (best effort). - assert(errcode != _PyXI_ERR_UNCAUGHT_EXCEPTION); - session->error_override = &errcode; - _capture_current_exception(session); - _exit_session(session); - if (sharedns != NULL) { - _PyXI_FreeNamespace(sharedns); + if (p_failure != NULL) { + *p_failure = failure; } return -1; } +PyObject * +_PyXI_GetPreserved(_PyXI_session_result *result, const char *name) +{ + PyObject *value = NULL; + if (result->preserved != NULL) { + (void)PyDict_GetItemStringRef(result->preserved, name, &value); + } + return value; +} + void -_PyXI_Exit(_PyXI_session *session) +_PyXI_ClearResult(_PyXI_session_result *result) { - _capture_current_exception(session); - _exit_session(session); + Py_CLEAR(result->preserved); + Py_CLEAR(result->excinfo); } diff --git a/Python/crossinterp_data_lookup.h b/Python/crossinterp_data_lookup.h index 231537c66d78f6d..cf84633e10e356c 100644 --- a/Python/crossinterp_data_lookup.h +++ b/Python/crossinterp_data_lookup.h @@ -12,7 +12,8 @@ typedef _PyXIData_regitem_t dlregitem_t; // forward static void _xidregistry_init(dlregistry_t *); static void _xidregistry_fini(dlregistry_t *); -static xidatafunc _lookup_getdata_from_registry(dlcontext_t *, PyObject *); +static _PyXIData_getdata_t _lookup_getdata_from_registry( + dlcontext_t *, PyObject *); /* used in crossinterp.c */ @@ -49,7 +50,7 @@ get_lookup_context(PyThreadState *tstate, dlcontext_t *res) return 0; } -static xidatafunc +static _PyXIData_getdata_t lookup_getdata(dlcontext_t *ctx, PyObject *obj) { /* Cross-interpreter objects are looked up by exact match on the class. @@ -87,25 +88,52 @@ _PyXIData_FormatNotShareableError(PyThreadState *tstate, va_end(vargs); } +int +_PyXI_UnwrapNotShareableError(PyThreadState * tstate, _PyXI_failure *failure) +{ + PyObject *exctype = get_notshareableerror_type(tstate); + assert(exctype != NULL); + if (!_PyErr_ExceptionMatches(tstate, exctype)) { + return -1; + } + PyObject *exc = _PyErr_GetRaisedException(tstate); + if (failure != NULL) { + _PyXI_errcode code = _PyXI_ERR_NOT_SHAREABLE; + if (_PyXI_InitFailure(failure, code, exc) < 0) { + return -1; + } + } + PyObject *cause = PyException_GetCause(exc); + if (cause != NULL) { + Py_DECREF(exc); + exc = cause; + } + else { + assert(PyException_GetContext(exc) == NULL); + } + _PyErr_SetRaisedException(tstate, exc); + return 0; +} + -xidatafunc +_PyXIData_getdata_t _PyXIData_Lookup(PyThreadState *tstate, PyObject *obj) { dlcontext_t ctx; if (get_lookup_context(tstate, &ctx) < 0) { - return NULL; + return (_PyXIData_getdata_t){0}; } return lookup_getdata(&ctx, obj); } /***********************************************/ -/* a registry of {type -> xidatafunc} */ +/* a registry of {type -> _PyXIData_getdata_t} */ /***********************************************/ -/* For now we use a global registry of shareable classes. An - alternative would be to add a tp_* slot for a class's - xidatafunc. It would be simpler and more efficient. */ +/* For now we use a global registry of shareable classes. + An alternative would be to add a tp_* slot for a class's + _PyXIData_getdata_t. It would be simpler and more efficient. */ /* registry lifecycle */ @@ -200,7 +228,7 @@ _xidregistry_find_type(dlregistry_t *xidregistry, PyTypeObject *cls) return NULL; } -static xidatafunc +static _PyXIData_getdata_t _lookup_getdata_from_registry(dlcontext_t *ctx, PyObject *obj) { PyTypeObject *cls = Py_TYPE(obj); @@ -209,10 +237,12 @@ _lookup_getdata_from_registry(dlcontext_t *ctx, PyObject *obj) _xidregistry_lock(xidregistry); dlregitem_t *matched = _xidregistry_find_type(xidregistry, cls); - xidatafunc func = matched != NULL ? matched->getdata : NULL; + _PyXIData_getdata_t getdata = matched != NULL + ? matched->getdata + : (_PyXIData_getdata_t){0}; _xidregistry_unlock(xidregistry); - return func; + return getdata; } @@ -220,12 +250,13 @@ _lookup_getdata_from_registry(dlcontext_t *ctx, PyObject *obj) static int _xidregistry_add_type(dlregistry_t *xidregistry, - PyTypeObject *cls, xidatafunc getdata) + PyTypeObject *cls, _PyXIData_getdata_t getdata) { dlregitem_t *newhead = PyMem_RawMalloc(sizeof(dlregitem_t)); if (newhead == NULL) { return -1; } + assert((getdata.basic == NULL) != (getdata.fallback == NULL)); *newhead = (dlregitem_t){ // We do not keep a reference, to avoid keeping the class alive. .cls = cls, @@ -283,13 +314,13 @@ _xidregistry_clear(dlregistry_t *xidregistry) int _PyXIData_RegisterClass(PyThreadState *tstate, - PyTypeObject *cls, xidatafunc getdata) + PyTypeObject *cls, _PyXIData_getdata_t getdata) { if (!PyType_Check(cls)) { PyErr_Format(PyExc_ValueError, "only classes may be registered"); return -1; } - if (getdata == NULL) { + if (getdata.basic == NULL && getdata.fallback == NULL) { PyErr_Format(PyExc_ValueError, "missing 'getdata' func"); return -1; } @@ -304,7 +335,8 @@ _PyXIData_RegisterClass(PyThreadState *tstate, dlregitem_t *matched = _xidregistry_find_type(xidregistry, cls); if (matched != NULL) { - assert(matched->getdata == getdata); + assert(matched->getdata.basic == getdata.basic); + assert(matched->getdata.fallback == getdata.fallback); matched->refcount += 1; goto finally; } @@ -423,7 +455,7 @@ _PyBytes_GetXIDataWrapped(PyThreadState *tstate, return NULL; } if (size < sizeof(_PyBytes_data_t)) { - PyErr_Format(PyExc_ValueError, "expected size >= %d, got %d", + PyErr_Format(PyExc_ValueError, "expected size >= %zu, got %zu", sizeof(_PyBytes_data_t), size); return NULL; } @@ -608,7 +640,8 @@ _tuple_shared_free(void* data) } static int -_tuple_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *xidata) +_tuple_shared(PyThreadState *tstate, PyObject *obj, xidata_fallback_t fallback, + _PyXIData_t *xidata) { Py_ssize_t len = PyTuple_GET_SIZE(obj); if (len < 0) { @@ -636,7 +669,7 @@ _tuple_shared(PyThreadState *tstate, PyObject *obj, _PyXIData_t *xidata) int res = -1; if (!_Py_EnterRecursiveCallTstate(tstate, " while sharing a tuple")) { - res = _PyObject_GetXIData(tstate, item, xidata_i); + res = _PyObject_GetXIData(tstate, item, fallback, xidata_i); _Py_LeaveRecursiveCallTstate(tstate); } if (res < 0) { @@ -677,44 +710,126 @@ _PyCode_GetXIData(PyThreadState *tstate, PyObject *obj, _PyXIData_t *xidata) return 0; } +// function + +PyObject * +_PyFunction_FromXIData(_PyXIData_t *xidata) +{ + // For now "stateless" functions are the only ones we must accommodate. + + PyObject *code = _PyMarshal_ReadObjectFromXIData(xidata); + if (code == NULL) { + return NULL; + } + // Create a new function. + // For stateless functions (no globals) we use __main__ as __globals__, + // just like we do for builtins like exec(). + assert(PyCode_Check(code)); + PyThreadState *tstate = _PyThreadState_GET(); + PyObject *globals = _PyEval_GetGlobalsFromRunningMain(tstate); // borrowed + if (globals == NULL) { + if (_PyErr_Occurred(tstate)) { + Py_DECREF(code); + return NULL; + } + globals = PyDict_New(); + if (globals == NULL) { + Py_DECREF(code); + return NULL; + } + } + else { + Py_INCREF(globals); + } + if (_PyEval_EnsureBuiltins(tstate, globals, NULL) < 0) { + Py_DECREF(code); + Py_DECREF(globals); + return NULL; + } + PyObject *func = PyFunction_New(code, globals); + Py_DECREF(code); + Py_DECREF(globals); + return func; +} + +int +_PyFunction_GetXIData(PyThreadState *tstate, PyObject *func, + _PyXIData_t *xidata) +{ + if (!PyFunction_Check(func)) { + const char *msg = "expected a function, got %R"; + format_notshareableerror(tstate, NULL, 0, msg, func); + return -1; + } + if (_PyFunction_VerifyStateless(tstate, func) < 0) { + PyObject *cause = _PyErr_GetRaisedException(tstate); + assert(cause != NULL); + const char *msg = "only stateless functions are shareable"; + set_notshareableerror(tstate, cause, 0, msg); + Py_DECREF(cause); + return -1; + } + PyObject *code = PyFunction_GET_CODE(func); + + // Ideally code objects would be immortal and directly shareable. + // In the meantime, we use marshal. + if (_PyMarshal_GetXIData(tstate, code, xidata) < 0) { + return -1; + } + // Replace _PyMarshal_ReadObjectFromXIData. + // (_PyFunction_FromXIData() will call it.) + _PyXIData_SET_NEW_OBJECT(xidata, _PyFunction_FromXIData); + return 0; +} + // registration static void _register_builtins_for_crossinterpreter_data(dlregistry_t *xidregistry) { +#define REGISTER(TYPE, GETDATA) \ + _xidregistry_add_type(xidregistry, (PyTypeObject *)TYPE, \ + ((_PyXIData_getdata_t){.basic=(GETDATA)})) +#define REGISTER_FALLBACK(TYPE, GETDATA) \ + _xidregistry_add_type(xidregistry, (PyTypeObject *)TYPE, \ + ((_PyXIData_getdata_t){.fallback=(GETDATA)})) // None - if (_xidregistry_add_type(xidregistry, (PyTypeObject *)PyObject_Type(Py_None), _none_shared) != 0) { + if (REGISTER(Py_TYPE(Py_None), _none_shared) != 0) { Py_FatalError("could not register None for cross-interpreter sharing"); } // int - if (_xidregistry_add_type(xidregistry, &PyLong_Type, _long_shared) != 0) { + if (REGISTER(&PyLong_Type, _long_shared) != 0) { Py_FatalError("could not register int for cross-interpreter sharing"); } // bytes - if (_xidregistry_add_type(xidregistry, &PyBytes_Type, _PyBytes_GetXIData) != 0) { + if (REGISTER(&PyBytes_Type, _PyBytes_GetXIData) != 0) { Py_FatalError("could not register bytes for cross-interpreter sharing"); } // str - if (_xidregistry_add_type(xidregistry, &PyUnicode_Type, _str_shared) != 0) { + if (REGISTER(&PyUnicode_Type, _str_shared) != 0) { Py_FatalError("could not register str for cross-interpreter sharing"); } // bool - if (_xidregistry_add_type(xidregistry, &PyBool_Type, _bool_shared) != 0) { + if (REGISTER(&PyBool_Type, _bool_shared) != 0) { Py_FatalError("could not register bool for cross-interpreter sharing"); } // float - if (_xidregistry_add_type(xidregistry, &PyFloat_Type, _float_shared) != 0) { + if (REGISTER(&PyFloat_Type, _float_shared) != 0) { Py_FatalError("could not register float for cross-interpreter sharing"); } // tuple - if (_xidregistry_add_type(xidregistry, &PyTuple_Type, _tuple_shared) != 0) { + if (REGISTER_FALLBACK(&PyTuple_Type, _tuple_shared) != 0) { Py_FatalError("could not register tuple for cross-interpreter sharing"); } + + // For now, we do not register PyCode_Type or PyFunction_Type. +#undef REGISTER +#undef REGISTER_FALLBACK } diff --git a/Python/crossinterp_exceptions.h b/Python/crossinterp_exceptions.h index ca4ca1cf123e494..98411adc5eb3f6f 100644 --- a/Python/crossinterp_exceptions.h +++ b/Python/crossinterp_exceptions.h @@ -7,13 +7,6 @@ _ensure_current_cause(PyThreadState *tstate, PyObject *cause) } PyObject *exc = _PyErr_GetRaisedException(tstate); assert(exc != NULL); - PyObject *ctx = PyException_GetContext(exc); - if (ctx == NULL) { - PyException_SetContext(exc, Py_NewRef(cause)); - } - else { - Py_DECREF(ctx); - } assert(PyException_GetCause(exc) == NULL); PyException_SetCause(exc, Py_NewRef(cause)); _PyErr_SetRaisedException(tstate, exc); @@ -24,7 +17,7 @@ _ensure_current_cause(PyThreadState *tstate, PyObject *cause) static PyTypeObject _PyExc_InterpreterError = { PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "interpreters.InterpreterError", + .tp_name = "concurrent.interpreters.InterpreterError", .tp_doc = PyDoc_STR("A cross-interpreter operation failed"), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, //.tp_traverse = ((PyTypeObject *)PyExc_Exception)->tp_traverse, @@ -37,7 +30,7 @@ PyObject *PyExc_InterpreterError = (PyObject *)&_PyExc_InterpreterError; static PyTypeObject _PyExc_InterpreterNotFoundError = { PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "interpreters.InterpreterNotFoundError", + .tp_name = "concurrent.interpreters.InterpreterNotFoundError", .tp_doc = PyDoc_STR("An interpreter was not found"), .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, //.tp_traverse = ((PyTypeObject *)PyExc_Exception)->tp_traverse, @@ -51,7 +44,7 @@ PyObject *PyExc_InterpreterNotFoundError = (PyObject *)&_PyExc_InterpreterNotFou static int _init_notshareableerror(exceptions_t *state) { - const char *name = "interpreters.NotShareableError"; + const char *name = "concurrent.interpreters.NotShareableError"; PyObject *base = PyExc_TypeError; PyObject *ns = NULL; PyObject *exctype = PyErr_NewException(name, base, ns); diff --git a/Python/emscripten_syscalls.c b/Python/emscripten_syscalls.c new file mode 100644 index 000000000000000..b1236e6b123fc94 --- /dev/null +++ b/Python/emscripten_syscalls.c @@ -0,0 +1,319 @@ +#include "emscripten.h" +#include "stdio.h" + +// If we're running in node, report the UID of the user in the native system as +// the UID of the user. Since the nodefs will report the uid correctly, if we +// don't make getuid report it correctly too we'll see some permission errors. +// Normally __syscall_getuid32 is a stub that always returns 0 but it is +// defined with weak linkage so we can override it. +EM_JS(int, __syscall_getuid32_js, (void), { + // If we're in node and we can, report the native uid + if (ENVIRONMENT_IS_NODE) { + return process.getuid(); + } + // Fall back to the stub case of returning 0. + return 0; +}) + +int __syscall_getuid32(void) { + return __syscall_getuid32_js(); +} + +EM_JS(int, __syscall_umask_js, (int mask), { + // If we're in node and we can, call native process.umask() + if (ENVIRONMENT_IS_NODE) { + try { + return process.umask(mask); + } catch(e) { + // oops... + // NodeJS docs: "In Worker threads, process.umask(mask) will throw an exception." + // umask docs: "This system call always succeeds" + return 0; + } + } + // Fall back to the stub case of returning 0. + return 0; +}) + +int __syscall_umask(int mask) { + return __syscall_umask_js(mask); +} + +#include <wasi/api.h> +#include <errno.h> +#include <fcntl.h> + +// Variant of EM_JS that does C preprocessor substitution on the body +#define EM_JS_MACROS(ret, func_name, args, body...) \ + EM_JS(ret, func_name, args, body) + +EM_JS_MACROS(void, _emscripten_promising_main_js, (void), { + // Define FS.createAsyncInputDevice(), This is quite similar to + // FS.createDevice() defined here: + // https://github.com/emscripten-core/emscripten/blob/4.0.11/src/lib/libfs.js?plain=1#L1642 + // but instead of returning one byte at a time, the input() function should + // return a Uint8Array. This makes the handler code simpler, the + // `createAsyncInputDevice` simpler, and everything faster. + FS.createAsyncInputDevice = function(parent, name, input) { + parent = typeof parent == 'string' ? parent : FS.getPath(parent); + var path = PATH.join2(parent, name); + var mode = FS_getMode(true, false); + FS.createDevice.major ||= 64; + var dev = FS.makedev(FS.createDevice.major++, 0); + async function getDataBuf() { + var buf; + try { + buf = await input(); + } catch (e) { + throw new FS.ErrnoError(EIO); + } + if (!buf?.byteLength) { + throw new FS.ErrnoError(EAGAIN); + } + ops._dataBuf = buf; + } + + var ops = { + _dataBuf: new Uint8Array(0), + open(stream) { + stream.seekable = false; + }, + async readAsync(stream, buffer, offset, length, pos /* ignored */) { + buffer = buffer.subarray(offset, offset + length); + if (!ops._dataBuf.byteLength) { + await getDataBuf(); + } + var toRead = Math.min(ops._dataBuf.byteLength, buffer.byteLength); + buffer.subarray(0, toRead).set(ops._dataBuf); + buffer = buffer.subarray(toRead); + ops._dataBuf = ops._dataBuf.subarray(toRead); + if (toRead) { + stream.node.atime = Date.now(); + } + return toRead; + }, + }; + FS.registerDevice(dev, ops); + return FS.mkdev(path, mode, dev); + }; + if (!WebAssembly.promising) { + // No stack switching support =( + return; + } + const origResolveGlobalSymbol = resolveGlobalSymbol; + if (ENVIRONMENT_IS_NODE && !Module.onExit) { + Module.onExit = (code) => process.exit(code); + } + // * wrap the main symbol with WebAssembly.promising, + // * call exit_with_live_runtime() to prevent emscripten from shutting down + // the runtime before the promise resolves, + // * call onExit / process.exit ourselves, since exit_with_live_runtime() + // prevented Emscripten from calling it normally. + resolveGlobalSymbol = function (name, direct = false) { + const orig = origResolveGlobalSymbol(name, direct); + if (name === "main") { + const main = WebAssembly.promising(orig.sym); + orig.sym = (...args) => { + (async () => { + const ret = await main(...args); + Module.onExit?.(ret); + })(); + _emscripten_exit_with_live_runtime(); + }; + } + return orig; + }; +}) + +__attribute__((constructor)) void _emscripten_promising_main(void) { + _emscripten_promising_main_js(); +} + + +#define IOVEC_T_BUF_OFFSET 0 +#define IOVEC_T_BUF_LEN_OFFSET 4 +#define IOVEC_T_SIZE 8 +_Static_assert(offsetof(__wasi_iovec_t, buf) == IOVEC_T_BUF_OFFSET, + "Unexpected __wasi_iovec_t layout"); +_Static_assert(offsetof(__wasi_iovec_t, buf_len) == IOVEC_T_BUF_LEN_OFFSET, + "Unexpected __wasi_iovec_t layout"); +_Static_assert(sizeof(__wasi_iovec_t) == IOVEC_T_SIZE, + "Unexpected __wasi_iovec_t layout"); + +// If the stream has a readAsync handler, read to buffer defined in iovs, write +// number of bytes read to *nread, and return a promise that resolves to the +// errno. Otherwise, return null. +EM_JS_MACROS(__externref_t, __maybe_fd_read_async, ( + __wasi_fd_t fd, + const __wasi_iovec_t *iovs, + size_t iovcnt, + __wasi_size_t *nread +), { + if (!WebAssembly.promising) { + return null; + } + var stream; + try { + stream = SYSCALLS.getStreamFromFD(fd); + } catch (e) { + // If the fd was already closed or never existed, getStreamFromFD() + // raises. We'll let fd_read_orig() handle setting errno. + return null; + } + if (!stream.stream_ops.readAsync) { + // Not an async device. Fall back to __wasi_fd_read_orig(). + return null; + } + return (async () => { + // This is the same as libwasi.js fd_read() and doReadv() except we use + // readAsync and we await it. + // https://github.com/emscripten-core/emscripten/blob/4.0.11/src/lib/libwasi.js?plain=1#L331 + // https://github.com/emscripten-core/emscripten/blob/4.0.11/src/lib/libwasi.js?plain=1#L197 + try { + var ret = 0; + for (var i = 0; i < iovcnt; i++) { + var ptr = HEAP32[(iovs + IOVEC_T_BUF_OFFSET)/4]; + var len = HEAP32[(iovs + IOVEC_T_BUF_LEN_OFFSET)/4]; + iovs += IOVEC_T_SIZE; + var curr = await stream.stream_ops.readAsync(stream, HEAP8, ptr, len); + if (curr < 0) return -1; + ret += curr; + if (curr < len) break; // nothing more to read + } + HEAP32[nread/4] = ret; + return 0; + } catch (e) { + if (e.name !== 'ErrnoError') { + throw e; + } + return e["errno"]; + } + })(); +}; +); + +// Bind original fd_read syscall to __wasi_fd_read_orig(). +__wasi_errno_t __wasi_fd_read_orig(__wasi_fd_t fd, const __wasi_iovec_t *iovs, + size_t iovs_len, __wasi_size_t *nread) + __attribute__((__import_module__("wasi_snapshot_preview1"), + __import_name__("fd_read"), __warn_unused_result__)); + +// Take a promise that resolves to __wasi_errno_t and suspend until it resolves, +// get the output. +EM_JS(int, __block_for_int, (__externref_t p), { + return p; +} +if (WebAssembly.Suspending) { + __block_for_int = new WebAssembly.Suspending(__block_for_int); +} +) + +// Replacement for fd_read syscall. Call __maybe_fd_read_async. If it returned +// null, delegate back to __wasi_fd_read_orig. Otherwise, use __block_for_int +// to get the result. +__wasi_errno_t __wasi_fd_read(__wasi_fd_t fd, const __wasi_iovec_t *iovs, + size_t iovs_len, __wasi_size_t *nread) { + __externref_t p = __maybe_fd_read_async(fd, iovs, iovs_len, nread); + if (__builtin_wasm_ref_is_null_extern(p)) { + return __wasi_fd_read_orig(fd, iovs, iovs_len, nread); + } + return __block_for_int(p); +} + +#include <poll.h> +#define POLLFD_FD 0 +#define POLLFD_EVENTS 4 +#define POLLFD_REVENTS 6 +#define POLLFD_SIZE 8 +_Static_assert(offsetof(struct pollfd, fd) == 0, "Unepxected pollfd struct layout"); +_Static_assert(offsetof(struct pollfd, events) == 4, "Unepxected pollfd struct layout"); +_Static_assert(offsetof(struct pollfd, revents) == 6, "Unepxected pollfd struct layout"); +_Static_assert(sizeof(struct pollfd) == 8, "Unepxected pollfd struct layout"); + +EM_JS_MACROS(__externref_t, __maybe_poll_async, (intptr_t fds, int nfds, int timeout), { + if (!WebAssembly.promising) { + return null; + } + return (async function() { + try { + var nonzero = 0; + var promises = []; + for (var i = 0; i < nfds; i++) { + var pollfd = fds + POLLFD_SIZE * i; + var fd = HEAP32[(pollfd + POLLFD_FD)/4]; + var events = HEAP16[(pollfd + POLLFD_EVENTS)/2]; + var mask = POLLNVAL; + var stream = FS.getStream(fd); + if (stream) { + mask = POLLIN | POLLOUT; + if (stream.stream_ops.pollAsync) { + promises.push(stream.stream_ops.pollAsync(stream, timeout).then((mask) => { + mask &= events | POLLERR | POLLHUP; + HEAP16[(pollfd + POLLFD_REVENTS)/2] = mask; + if (mask) { + nonzero ++; + } + })); + } else if (stream.stream_ops.poll) { + var mask = stream.stream_ops.poll(stream, timeout); + mask &= events | POLLERR | POLLHUP; + HEAP16[(pollfd + POLLFD_REVENTS)/2] = mask; + if (mask) { + nonzero ++; + } + } + } + } + await Promise.all(promises); + return nonzero; + } catch(e) { + if (e?.name !== "ErrnoError") throw e; + return -e["errno"]; + } + })(); +}); + +// Bind original poll syscall to syscall_poll_orig(). +int syscall_poll_orig(intptr_t fds, int nfds, int timeout) + __attribute__((__import_module__("env"), + __import_name__("__syscall_poll"), __warn_unused_result__)); + +int __syscall_poll(intptr_t fds, int nfds, int timeout) { + __externref_t p = __maybe_poll_async(fds, nfds, timeout); + if (__builtin_wasm_ref_is_null_extern(p)) { + return syscall_poll_orig(fds, nfds, timeout); + } + return __block_for_int(p); +} + +#include <sys/ioctl.h> + +int syscall_ioctl_orig(int fd, int request, void* varargs) + __attribute__((__import_module__("env"), + __import_name__("__syscall_ioctl"), __warn_unused_result__)); + +int __syscall_ioctl(int fd, int request, void* varargs) { + if (request == FIOCLEX || request == FIONCLEX) { + return 0; + } + if (request == FIONBIO) { + // Implement FIONBIO via fcntl. + // TODO: Upstream this. + int flags = fcntl(fd, F_GETFL, 0); + int nonblock = **((int**)varargs); + if (flags < 0) { + return -errno; + } + if (nonblock) { + flags |= O_NONBLOCK; + } else { + flags &= (~O_NONBLOCK); + } + int res = fcntl(fd, F_SETFL, flags); + if (res < 0) { + return -errno; + } + return res; + } + return syscall_ioctl_orig(fd, request, varargs); +} diff --git a/Python/emscripten_trampoline.c b/Python/emscripten_trampoline.c index a7bb685bf3dc6db..1833311ca74d9dd 100644 --- a/Python/emscripten_trampoline.c +++ b/Python/emscripten_trampoline.c @@ -4,222 +4,115 @@ #include <Python.h> #include "pycore_runtime.h" // _PyRuntime -typedef int (*CountArgsFunc)(PyCFunctionWithKeywords func); +// We use the _PyRuntime.emscripten_trampoline field to store a function pointer +// for a wasm-gc based trampoline if it works. Otherwise fall back to JS +// trampoline. The JS trampoline breaks stack switching but every runtime that +// supports stack switching also supports wasm-gc. +// +// We'd like to make the trampoline call into a direct call but currently we +// need to import the wasmTable to compile trampolineModule. emcc >= 4.0.19 +// defines the table in WebAssembly and exports it so we won't have access to it +// until after the main module is compiled. +// +// To fix this, one natural solution would be to pass a funcref to the +// trampoline instead of a table index. Several PRs would be needed to fix +// things in llvm and emscripten in order to make this possible. +// +// The performance costs of an extra call_indirect aren't that large anyways. +// The JIT should notice that the target is always the same and turn into a +// check +// +// if (call_target != expected) deoptimize; +// direct_call(call_target, args); -// Offset of emscripten_count_args_function in _PyRuntimeState. There's a couple -// of alternatives: -// 1. Just make emscripten_count_args_function a real C global variable instead -// of a field of _PyRuntimeState. This would violate our rule against mutable +// Offset of emscripten_trampoline in _PyRuntimeState. There's a couple of +// alternatives: +// +// 1. Just make emscripten_trampoline a real C global variable instead of a +// field of _PyRuntimeState. This would violate our rule against mutable // globals. +// // 2. #define a preprocessor constant equal to a hard coded number and make a -// _Static_assert(offsetof(_PyRuntimeState, emscripten_count_args_function) -// == OURCONSTANT) This has the disadvantage that we have to update the hard -// coded constant when _PyRuntimeState changes +// _Static_assert(offsetof(_PyRuntimeState, emscripten_trampoline) == OURCONSTANT) +// This has the disadvantage that we have to update the hard coded constant +// when _PyRuntimeState changes // // So putting the mutable constant in _PyRuntime and using a immutable global to // record the offset so we can access it from JS is probably the best way. -EMSCRIPTEN_KEEPALIVE const int _PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET = offsetof(_PyRuntimeState, emscripten_count_args_function); +EMSCRIPTEN_KEEPALIVE const int _PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET = offsetof(_PyRuntimeState, emscripten_trampoline); -EM_JS(CountArgsFunc, _PyEM_GetCountArgsPtr, (), { - return Module._PyEM_CountArgsPtr; // initialized below -} -// Binary module for the checks. It has to be done in web assembly because -// clang/llvm have no support yet for the reference types yet. In fact, the wasm -// binary toolkit doesn't yet support the ref.test instruction either. To -// convert the following textual wasm to a binary, you can build wabt from this -// branch: https://github.com/WebAssembly/wabt/pull/2529 and then use that -// wat2wasm binary. -// -// (module -// (type $type0 (func (param) (result i32))) -// (type $type1 (func (param i32) (result i32))) -// (type $type2 (func (param i32 i32) (result i32))) -// (type $type3 (func (param i32 i32 i32) (result i32))) -// (type $blocktype (func (param i32) (result))) -// (table $funcs (import "e" "t") 0 funcref) -// (export "f" (func $f)) -// (func $f (param $fptr i32) (result i32) -// (local $fref funcref) -// local.get $fptr -// table.get $funcs -// local.tee $fref -// ref.test $type3 -// (block $b (type $blocktype) -// i32.eqz -// br_if $b -// i32.const 3 -// return -// ) -// local.get $fref -// ref.test $type2 -// (block $b (type $blocktype) -// i32.eqz -// br_if $b -// i32.const 2 -// return -// ) -// local.get $fref -// ref.test $type1 -// (block $b (type $blocktype) -// i32.eqz -// br_if $b -// i32.const 1 -// return -// ) -// local.get $fref -// ref.test $type0 -// (block $b (type $blocktype) -// i32.eqz -// br_if $b -// i32.const 0 -// return -// ) -// i32.const -1 -// ) -// ) +typedef PyObject* (*TrampolineFunc)(int* success, + PyCFunctionWithKeywords func, + PyObject* self, + PyObject* args, + PyObject* kw); -function getPyEMCountArgsPtr() { - let isIOS = globalThis.navigator && /iPad|iPhone|iPod/.test(navigator.platform); +/** + * Backwards compatible trampoline works with all JS runtimes + */ +EM_JS(PyObject*, _PyEM_TrampolineCall_JS, (PyCFunctionWithKeywords func, PyObject *arg1, PyObject *arg2, PyObject *arg3), { + return wasmTable.get(func)(arg1, arg2, arg3); +} +// Try to compile wasm-gc trampoline if possible. +function getPyEMTrampolinePtr() { + // Starting with iOS 18.3.1, WebKit on iOS has an issue with the garbage + // collector that breaks the call trampoline. See #130418 and + // https://bugs.webkit.org/show_bug.cgi?id=293113 for details. + let isIOS = globalThis.navigator && ( + /iPad|iPhone|iPod/.test(navigator.userAgent) || + // Starting with iPadOS 13, iPads might send a platform string that looks like a desktop Mac. + // To differentiate, we check if the platform is 'MacIntel' (common for Macs and newer iPads) + // AND if the device has multi-touch capabilities (navigator.maxTouchPoints > 1) + (navigator.platform === 'MacIntel' && typeof navigator.maxTouchPoints !== 'undefined' && navigator.maxTouchPoints > 1) + ); if (isIOS) { return 0; } - - // Try to initialize countArgsFunc - const code = new Uint8Array([ - 0x00, 0x61, 0x73, 0x6d, // \0asm magic number - 0x01, 0x00, 0x00, 0x00, // version 1 - 0x01, 0x1b, // Type section, body is 0x1b bytes - 0x05, // 6 entries - 0x60, 0x00, 0x01, 0x7f, // (type $type0 (func (param) (result i32))) - 0x60, 0x01, 0x7f, 0x01, 0x7f, // (type $type1 (func (param i32) (result i32))) - 0x60, 0x02, 0x7f, 0x7f, 0x01, 0x7f, // (type $type2 (func (param i32 i32) (result i32))) - 0x60, 0x03, 0x7f, 0x7f, 0x7f, 0x01, 0x7f, // (type $type3 (func (param i32 i32 i32) (result i32))) - 0x60, 0x01, 0x7f, 0x00, // (type $blocktype (func (param i32) (result))) - 0x02, 0x09, // Import section, 0x9 byte body - 0x01, // 1 import (table $funcs (import "e" "t") 0 funcref) - 0x01, 0x65, // "e" - 0x01, 0x74, // "t" - 0x01, // importing a table - 0x70, // of entry type funcref - 0x00, 0x00, // table limits: no max, min of 0 - 0x03, 0x02, // Function section - 0x01, 0x01, // We're going to define one function of type 1 (func (param i32) (result i32)) - 0x07, 0x05, // export section - 0x01, // 1 export - 0x01, 0x66, // called "f" - 0x00, // a function - 0x00, // at index 0 - - 0x0a, 0x44, // Code section, - 0x01, 0x42, // one entry of length 50 - 0x01, 0x01, 0x70, // one local of type funcref - // Body of the function - 0x20, 0x00, // local.get $fptr - 0x25, 0x00, // table.get $funcs - 0x22, 0x01, // local.tee $fref - 0xfb, 0x14, 0x03, // ref.test $type3 - 0x02, 0x04, // block $b (type $blocktype) - 0x45, // i32.eqz - 0x0d, 0x00, // br_if $b - 0x41, 0x03, // i32.const 3 - 0x0f, // return - 0x0b, // end block - - 0x20, 0x01, // local.get $fref - 0xfb, 0x14, 0x02, // ref.test $type2 - 0x02, 0x04, // block $b (type $blocktype) - 0x45, // i32.eqz - 0x0d, 0x00, // br_if $b - 0x41, 0x02, // i32.const 2 - 0x0f, // return - 0x0b, // end block - - 0x20, 0x01, // local.get $fref - 0xfb, 0x14, 0x01, // ref.test $type1 - 0x02, 0x04, // block $b (type $blocktype) - 0x45, // i32.eqz - 0x0d, 0x00, // br_if $b - 0x41, 0x01, // i32.const 1 - 0x0f, // return - 0x0b, // end block - - 0x20, 0x01, // local.get $fref - 0xfb, 0x14, 0x00, // ref.test $type0 - 0x02, 0x04, // block $b (type $blocktype) - 0x45, // i32.eqz - 0x0d, 0x00, // br_if $b - 0x41, 0x00, // i32.const 0 - 0x0f, // return - 0x0b, // end block - - 0x41, 0x7f, // i32.const -1 - 0x0b // end function - ]); + let trampolineModule; try { - const mod = new WebAssembly.Module(code); - const inst = new WebAssembly.Instance(mod, { e: { t: wasmTable } }); - return addFunction(inst.exports.f); + trampolineModule = getWasmTrampolineModule(); } catch (e) { - // If something goes wrong, we'll null out _PyEM_CountFuncParams and fall - // back to the JS trampoline. + // Compilation error due to missing wasm-gc support, fall back to JS + // trampoline return 0; } + const trampolineInstance = new WebAssembly.Instance(trampolineModule, { + env: { __indirect_function_table: wasmTable, memory: wasmMemory }, + }); + return addFunction(trampolineInstance.exports.trampoline_call); } - -addOnPreRun(() => { - const ptr = getPyEMCountArgsPtr(); - Module._PyEM_CountArgsPtr = ptr; - const offset = HEAP32[__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET / 4]; +// We have to be careful to work correctly with memory snapshots -- the value of +// _PyRuntimeState.emscripten_trampoline needs to reflect whether wasm-gc is +// available in the current runtime, not in the runtime the snapshot was taken +// in. This writes the appropriate value to +// _PyRuntimeState.emscripten_trampoline from JS startup code that runs every +// time, whether we are restoring a snapshot or not. +addOnPreRun(function setEmscriptenTrampoline() { + const ptr = getPyEMTrampolinePtr(); + const offset = HEAP32[__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET / 4]; HEAP32[(__PyRuntime + offset) / 4] = ptr; }); ); -void -_Py_EmscriptenTrampoline_Init(_PyRuntimeState *runtime) -{ - runtime->emscripten_count_args_function = _PyEM_GetCountArgsPtr(); -} - -// We have to be careful to work correctly with memory snapshots. Even if we are -// loading a memory snapshot, we need to perform the JS initialization work. -// That means we can't call the initialization code from C. Instead, we export -// this function pointer to JS and then fill it in a preRun function which runs -// unconditionally. -/** - * Backwards compatible trampoline works with all JS runtimes - */ -EM_JS(PyObject*, _PyEM_TrampolineCall_JS, (PyCFunctionWithKeywords func, PyObject *arg1, PyObject *arg2, PyObject *arg3), { - return wasmTable.get(func)(arg1, arg2, arg3); -}); - -typedef PyObject* (*zero_arg)(void); -typedef PyObject* (*one_arg)(PyObject*); -typedef PyObject* (*two_arg)(PyObject*, PyObject*); -typedef PyObject* (*three_arg)(PyObject*, PyObject*, PyObject*); - PyObject* _PyEM_TrampolineCall(PyCFunctionWithKeywords func, PyObject* self, PyObject* args, PyObject* kw) { - CountArgsFunc count_args = _PyRuntime.emscripten_count_args_function; - if (count_args == 0) { + TrampolineFunc trampoline = _PyRuntime.emscripten_trampoline; + if (trampoline == 0) { return _PyEM_TrampolineCall_JS(func, self, args, kw); } - switch (count_args(func)) { - case 0: - return ((zero_arg)func)(); - case 1: - return ((one_arg)func)(self); - case 2: - return ((two_arg)func)(self, args); - case 3: - return ((three_arg)func)(self, args, kw); - default: - PyErr_SetString(PyExc_SystemError, "Handler takes too many arguments"); - return NULL; + int success = 1; + PyObject *result = trampoline(&success, func, self, args, kw); + if (!success) { + PyErr_SetString(PyExc_SystemError, "Handler takes too many arguments"); } + return result; } +#else +// This is exported so we need to define it even when it isn't used +__attribute__((used)) const int _PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET = 0; #endif diff --git a/Python/emscripten_trampoline_inner.c b/Python/emscripten_trampoline_inner.c new file mode 100644 index 000000000000000..a2bad4857ed089c --- /dev/null +++ b/Python/emscripten_trampoline_inner.c @@ -0,0 +1,38 @@ +// This file must be compiled with -mgc to enable the extra wasm-gc +// instructions. It has to be compiled separately because not enough JS runtimes +// support wasm-gc yet. If the JS runtime does not support wasm-gc (or has buggy +// support like iOS), we will use the JS trampoline fallback. + +// We can't import Python.h here because it is compiled/linked with -nostdlib. +// We don't need to know what's inside PyObject* anyways. We could just call it +// void* everywhere. There are two reasons to do this: +// 1. to improve readability +// 2. eventually when we are comfortable requiring wasm-gc, we can merge this +// into emscripten_trampoline.c without worrying about it. +typedef void PyObject; + +typedef PyObject* (*three_arg)(PyObject*, PyObject*, PyObject*); +typedef PyObject* (*two_arg)(PyObject*, PyObject*); +typedef PyObject* (*one_arg)(PyObject*); +typedef PyObject* (*zero_arg)(void); + +#define TRY_RETURN_CALL(ty, args...) \ + if (__builtin_wasm_test_function_pointer_signature((ty)func)) { \ + return ((ty)func)(args); \ + } + +__attribute__((export_name("trampoline_call"))) PyObject* +trampoline_call(int* success, + void* func, + PyObject* self, + PyObject* args, + PyObject* kw) +{ + *success = 1; + TRY_RETURN_CALL(three_arg, self, args, kw); + TRY_RETURN_CALL(two_arg, self, args); + TRY_RETURN_CALL(one_arg, self); + TRY_RETURN_CALL(zero_arg); + *success = 0; + return 0; +} diff --git a/Python/errors.c b/Python/errors.c index 81f267b043afafc..13633cb20c419c5 100644 --- a/Python/errors.c +++ b/Python/errors.c @@ -247,13 +247,23 @@ PyErr_SetObject(PyObject *exception, PyObject *value) _PyErr_SetObject(tstate, exception, value); } -/* Set a key error with the specified argument, wrapping it in a - * tuple automatically so that tuple keys are not unpacked as the - * exception arguments. */ +/* Set a key error with the specified argument. This function should be used to + * raise a KeyError with an argument instead of PyErr_SetObject(PyExc_KeyError, + * arg) which has a special behavior. PyErr_SetObject() unpacks arg if it's a + * tuple, and it uses arg instead of creating a new exception if arg is an + * exception. + * + * If an exception is already set, override the exception. */ void _PyErr_SetKeyError(PyObject *arg) { PyThreadState *tstate = _PyThreadState_GET(); + + // PyObject_CallOneArg() must not be called with an exception set, + // otherwise _Py_CheckFunctionResult() can fail if the function returned + // a result with an excception set. + _PyErr_Clear(tstate); + PyObject *exc = PyObject_CallOneArg(PyExc_KeyError, arg); if (!exc) { /* caller will expect error to be set anyway */ @@ -1632,6 +1642,7 @@ format_unraisable_v(const char *format, va_list va, PyObject *obj) _Py_EnsureTstateNotNULL(tstate); PyObject *err_msg = NULL; + PyObject *hook = NULL; PyObject *exc_type, *exc_value, *exc_tb; _PyErr_Fetch(tstate, &exc_type, &exc_value, &exc_tb); @@ -1676,7 +1687,6 @@ format_unraisable_v(const char *format, va_list va, PyObject *obj) goto error; } - PyObject *hook; if (_PySys_GetOptionalAttr(&_Py_ID(unraisablehook), &hook) < 0) { Py_DECREF(hook_args); err_msg_str = NULL; @@ -1689,7 +1699,6 @@ format_unraisable_v(const char *format, va_list va, PyObject *obj) } if (_PySys_Audit(tstate, "sys.unraisablehook", "OO", hook, hook_args) < 0) { - Py_DECREF(hook); Py_DECREF(hook_args); err_msg_str = "Exception ignored in audit hook"; obj = NULL; @@ -1697,13 +1706,11 @@ format_unraisable_v(const char *format, va_list va, PyObject *obj) } if (hook == Py_None) { - Py_DECREF(hook); Py_DECREF(hook_args); goto default_hook; } PyObject *res = PyObject_CallOneArg(hook, hook_args); - Py_DECREF(hook); Py_DECREF(hook_args); if (res != NULL) { Py_DECREF(res); @@ -1733,6 +1740,7 @@ format_unraisable_v(const char *format, va_list va, PyObject *obj) Py_XDECREF(exc_value); Py_XDECREF(exc_tb); Py_XDECREF(err_msg); + Py_XDECREF(hook); _PyErr_Clear(tstate); /* Just in case */ } @@ -1938,8 +1946,8 @@ int _PyErr_EmitSyntaxWarning(PyObject *msg, PyObject *filename, int lineno, int col_offset, int end_lineno, int end_col_offset) { - if (_PyErr_WarnExplicitObjectWithContext(PyExc_SyntaxWarning, msg, - filename, lineno) < 0) + if (PyErr_WarnExplicitObject(PyExc_SyntaxWarning, msg, + filename, lineno, NULL, NULL) < 0) { if (PyErr_ExceptionMatches(PyExc_SyntaxWarning)) { /* Replace the SyntaxWarning exception with a SyntaxError diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 3e51ac41fa3a2ed..f5c11e0f98dae68 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -1131,14 +1131,13 @@ assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyUnicode_Append(&temp, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - *target_local = PyStackRef_FromPyObjectSteal(temp); - _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(right_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*target_local)) { + if (temp == NULL) { + *target_local = PyStackRef_NULL; JUMP_TO_ERROR(); } + *target_local = PyStackRef_FromPyObjectSteal(temp); #if TIER_ONE assert(next_instr->op.code == STORE_FAST); @@ -1193,6 +1192,9 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + JUMP_TO_ERROR(); + } res = PyStackRef_FromPyObjectSteal(res_o); stack_pointer[0] = res; stack_pointer += 1; @@ -1576,7 +1578,7 @@ new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; - frame->return_offset = 6 ; + frame->return_offset = 6u ; stack_pointer[-3].bits = (uintptr_t)new_frame; stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); @@ -1944,8 +1946,8 @@ gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - assert( 2 + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2 + oparg); + assert( 2u + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2u + oparg); gen_frame->previous = frame; stack_pointer[-1].bits = (uintptr_t)gen_frame; break; @@ -1973,13 +1975,12 @@ frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #if TIER_ONE && defined(Py_DEBUG) + if (!PyStackRef_IsNone(frame->f_executable)) { + int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + assert(opcode == SEND || opcode == FOR_ITER); + } #endif stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); @@ -3187,32 +3188,20 @@ owner = stack_pointer[-1]; self_or_null = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - PyObject *attr_o; if (oparg & 1) { - attr_o = NULL; _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + attr = _Py_LoadAttr_StackRefSteal(tstate, owner, name, self_or_null); stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_meth) { - assert(attr_o != NULL); - self_or_null[0] = owner; - } - else { - stack_pointer += -1; + if (PyStackRef_IsNull(attr)) { + stack_pointer[-1] = attr; + stack_pointer += (oparg&1); assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - JUMP_TO_ERROR(); - } - self_or_null[0] = PyStackRef_NULL; - stack_pointer += 1; + JUMP_TO_ERROR(); } } else { _PyFrame_SetStackPointer(frame, stack_pointer); - attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -3222,9 +3211,9 @@ if (attr_o == NULL) { JUMP_TO_ERROR(); } + attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer += 1; } - attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer[-1] = attr; stack_pointer += (oparg&1); assert(WITHIN_STACK_BOUNDS()); @@ -4273,8 +4262,17 @@ _PyStackRef next; iter = stack_pointer[-1]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + iternextfunc func = Py_TYPE(iter_o)->tp_iternext; + if (func == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'%.100s' object is not an iterator", + Py_TYPE(iter_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_ERROR(); + } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + PyObject *next_o = func(iter_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (next_o == NULL) { if (_PyErr_Occurred(tstate)) { @@ -4535,7 +4533,7 @@ gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; gen_frame->previous = frame; - frame->return_offset = (uint16_t)( 2 + oparg); + frame->return_offset = (uint16_t)( 2u + oparg); stack_pointer[0].bits = (uintptr_t)gen_frame; stack_pointer += 1; assert(WITHIN_STACK_BOUNDS()); @@ -6616,6 +6614,22 @@ PyObject **ptr = (PyObject **)(((char *)func) + offset); assert(*ptr == NULL); *ptr = attr; + if (oparg == MAKE_FUNCTION_ANNOTATE && PyFunction_Check(attr)) { + PyFunctionObject *func_obj = (PyFunctionObject *)attr; + stack_pointer[-2] = func_out; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *fixed_qualname = PyUnicode_FromFormat("%U.__annotate__", ((PyFunctionObject *)func)->func_qualname); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (fixed_qualname == NULL) { + JUMP_TO_ERROR(); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_SETREF(func_obj->func_qualname, fixed_qualname); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 1; + } stack_pointer[-2] = func_out; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); diff --git a/Python/flowgraph.c b/Python/flowgraph.c index 78ef02a911a72bc..5557f72fabac5a6 100644 --- a/Python/flowgraph.c +++ b/Python/flowgraph.c @@ -6,6 +6,7 @@ #include "pycore_intrinsics.h" #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_long.h" // _PY_IS_SMALL_INT() +#include "pycore_hashtable.h" // _Py_hashtable_t #include "pycore_opcode_utils.h" #include "pycore_opcode_metadata.h" // OPCODE_HAS_ARG, etc @@ -391,7 +392,6 @@ cfg_builder_maybe_start_new_block(cfg_builder *g) static bool cfg_builder_check(cfg_builder *g) { - assert(g->g_entryblock->b_iused > 0); for (basicblock *block = g->g_block_list; block != NULL; block = block->b_list) { assert(!_PyMem_IsPtrFreed(block)); if (block->b_instr != NULL) { @@ -1295,6 +1295,14 @@ get_const_value(int opcode, int oparg, PyObject *co_consts) PyObject *constant = NULL; assert(loads_const(opcode)); if (opcode == LOAD_CONST) { + assert(PyList_Check(co_consts)); + Py_ssize_t n = PyList_GET_SIZE(co_consts); + if (oparg < 0 || oparg >= n) { + PyErr_Format(PyExc_ValueError, + "LOAD_CONST index %d is out of range for consts (len=%zd)", + oparg, n); + return NULL; + } constant = PyList_GET_ITEM(co_consts, oparg); } if (opcode == LOAD_SMALL_INT) { @@ -1311,30 +1319,38 @@ get_const_value(int opcode, int oparg, PyObject *co_consts) // Steals a reference to newconst. static int -add_const(PyObject *newconst, PyObject *consts, PyObject *const_cache) +add_const(PyObject *newconst, PyObject *consts, PyObject *const_cache, + _Py_hashtable_t *consts_index) { if (_PyCompile_ConstCacheMergeOne(const_cache, &newconst) < 0) { Py_DECREF(newconst); return -1; } - Py_ssize_t index; - for (index = 0; index < PyList_GET_SIZE(consts); index++) { - if (PyList_GET_ITEM(consts, index) == newconst) { - break; - } + _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(consts_index, (void *)newconst); + if (entry != NULL) { + Py_DECREF(newconst); + return (int)(uintptr_t)entry->value; } - if (index == PyList_GET_SIZE(consts)) { - if ((size_t)index >= (size_t)INT_MAX - 1) { - PyErr_SetString(PyExc_OverflowError, "too many constants"); - Py_DECREF(newconst); - return -1; - } - if (PyList_Append(consts, newconst)) { - Py_DECREF(newconst); - return -1; - } + + Py_ssize_t index = PyList_GET_SIZE(consts); + if ((size_t)index >= (size_t)INT_MAX - 1) { + PyErr_SetString(PyExc_OverflowError, "too many constants"); + Py_DECREF(newconst); + return -1; } + if (PyList_Append(consts, newconst)) { + Py_DECREF(newconst); + return -1; + } + + if (_Py_hashtable_set(consts_index, (void *)newconst, (void *)(uintptr_t)index) < 0) { + PyList_SetSlice(consts, index, index + 1, NULL); + Py_DECREF(newconst); + PyErr_NoMemory(); + return -1; + } + Py_DECREF(newconst); return (int)index; } @@ -1397,7 +1413,7 @@ maybe_instr_make_load_smallint(cfg_instr *instr, PyObject *newconst, if (val == -1 && PyErr_Occurred()) { return -1; } - if (!overflow && _PY_IS_SMALL_INT(val)) { + if (!overflow && _PY_IS_SMALL_INT(val) && 0 <= val && val <= 255) { assert(_Py_IsImmortal(newconst)); INSTR_SET_OP1(instr, LOAD_SMALL_INT, (int)val); return 1; @@ -1410,7 +1426,8 @@ maybe_instr_make_load_smallint(cfg_instr *instr, PyObject *newconst, /* Steals reference to "newconst" */ static int instr_make_load_const(cfg_instr *instr, PyObject *newconst, - PyObject *consts, PyObject *const_cache) + PyObject *consts, PyObject *const_cache, + _Py_hashtable_t *consts_index) { int res = maybe_instr_make_load_smallint(instr, newconst, consts, const_cache); if (res < 0) { @@ -1420,7 +1437,7 @@ instr_make_load_const(cfg_instr *instr, PyObject *newconst, if (res > 0) { return SUCCESS; } - int oparg = add_const(newconst, consts, const_cache); + int oparg = add_const(newconst, consts, const_cache, consts_index); RETURN_IF_ERROR(oparg); INSTR_SET_OP1(instr, LOAD_CONST, oparg); return SUCCESS; @@ -1433,7 +1450,8 @@ instr_make_load_const(cfg_instr *instr, PyObject *newconst, Called with codestr pointing to the first LOAD_CONST. */ static int -fold_tuple_of_constants(basicblock *bb, int i, PyObject *consts, PyObject *const_cache) +fold_tuple_of_constants(basicblock *bb, int i, PyObject *consts, + PyObject *const_cache, _Py_hashtable_t *consts_index) { /* Pre-conditions */ assert(PyDict_CheckExact(const_cache)); @@ -1470,7 +1488,7 @@ fold_tuple_of_constants(basicblock *bb, int i, PyObject *consts, PyObject *const } nop_out(const_instrs, seq_size); - return instr_make_load_const(instr, const_tuple, consts, const_cache); + return instr_make_load_const(instr, const_tuple, consts, const_cache, consts_index); } /* Replace: @@ -1488,7 +1506,8 @@ fold_tuple_of_constants(basicblock *bb, int i, PyObject *consts, PyObject *const */ static int fold_constant_intrinsic_list_to_tuple(basicblock *bb, int i, - PyObject *consts, PyObject *const_cache) + PyObject *consts, PyObject *const_cache, + _Py_hashtable_t *consts_index) { assert(PyDict_CheckExact(const_cache)); assert(PyList_CheckExact(consts)); @@ -1540,7 +1559,7 @@ fold_constant_intrinsic_list_to_tuple(basicblock *bb, int i, nop_out(&instr, 1); } assert(consts_found == 0); - return instr_make_load_const(intrinsic, newconst, consts, const_cache); + return instr_make_load_const(intrinsic, newconst, consts, const_cache, consts_index); } if (expect_append) { @@ -1576,7 +1595,8 @@ Optimize lists and sets for: */ static int optimize_lists_and_sets(basicblock *bb, int i, int nextop, - PyObject *consts, PyObject *const_cache) + PyObject *consts, PyObject *const_cache, + _Py_hashtable_t *consts_index) { assert(PyDict_CheckExact(const_cache)); assert(PyList_CheckExact(consts)); @@ -1626,7 +1646,7 @@ optimize_lists_and_sets(basicblock *bb, int i, int nextop, Py_SETREF(const_result, frozenset); } - int index = add_const(const_result, consts, const_cache); + int index = add_const(const_result, consts, const_cache, consts_index); RETURN_IF_ERROR(index); nop_out(const_instrs, seq_size); @@ -1823,7 +1843,8 @@ eval_const_binop(PyObject *left, int op, PyObject *right) } static int -fold_const_binop(basicblock *bb, int i, PyObject *consts, PyObject *const_cache) +fold_const_binop(basicblock *bb, int i, PyObject *consts, + PyObject *const_cache, _Py_hashtable_t *consts_index) { #define BINOP_OPERAND_COUNT 2 assert(PyDict_CheckExact(const_cache)); @@ -1865,7 +1886,7 @@ fold_const_binop(basicblock *bb, int i, PyObject *consts, PyObject *const_cache) } nop_out(operands_instrs, BINOP_OPERAND_COUNT); - return instr_make_load_const(binop, newconst, consts, const_cache); + return instr_make_load_const(binop, newconst, consts, const_cache, consts_index); } static PyObject * @@ -1884,6 +1905,10 @@ eval_const_unaryop(PyObject *operand, int opcode, int oparg) result = PyNumber_Negative(operand); break; case UNARY_INVERT: + // XXX: This should be removed once the ~bool depreciation expires. + if (PyBool_Check(operand)) { + return NULL; + } result = PyNumber_Invert(operand); break; case UNARY_NOT: { @@ -1907,7 +1932,8 @@ eval_const_unaryop(PyObject *operand, int opcode, int oparg) } static int -fold_const_unaryop(basicblock *bb, int i, PyObject *consts, PyObject *const_cache) +fold_const_unaryop(basicblock *bb, int i, PyObject *consts, + PyObject *const_cache, _Py_hashtable_t *consts_index) { #define UNARYOP_OPERAND_COUNT 1 assert(PyDict_CheckExact(const_cache)); @@ -1944,7 +1970,7 @@ fold_const_unaryop(basicblock *bb, int i, PyObject *consts, PyObject *const_cach assert(PyBool_Check(newconst)); } nop_out(&operand_instr, UNARYOP_OPERAND_COUNT); - return instr_make_load_const(unaryop, newconst, consts, const_cache); + return instr_make_load_const(unaryop, newconst, consts, const_cache, consts_index); } #define VISITED (-1) @@ -2139,7 +2165,8 @@ apply_static_swaps(basicblock *block, int i) } static int -basicblock_optimize_load_const(PyObject *const_cache, basicblock *bb, PyObject *consts) +basicblock_optimize_load_const(PyObject *const_cache, basicblock *bb, + PyObject *consts, _Py_hashtable_t *consts_index) { assert(PyDict_CheckExact(const_cache)); assert(PyList_CheckExact(consts)); @@ -2149,6 +2176,9 @@ basicblock_optimize_load_const(PyObject *const_cache, basicblock *bb, PyObject * cfg_instr *inst = &bb->b_instr[i]; if (inst->i_opcode == LOAD_CONST) { PyObject *constant = get_const_value(inst->i_opcode, inst->i_oparg, consts); + if (constant == NULL) { + return ERROR; + } int res = maybe_instr_make_load_smallint(inst, constant, consts, const_cache); Py_DECREF(constant); if (res < 0) { @@ -2254,7 +2284,7 @@ basicblock_optimize_load_const(PyObject *const_cache, basicblock *bb, PyObject * return ERROR; } cnt = PyBool_FromLong(is_true); - int index = add_const(cnt, consts, const_cache); + int index = add_const(cnt, consts, const_cache, consts_index); if (index < 0) { return ERROR; } @@ -2268,15 +2298,17 @@ basicblock_optimize_load_const(PyObject *const_cache, basicblock *bb, PyObject * } static int -optimize_load_const(PyObject *const_cache, cfg_builder *g, PyObject *consts) { +optimize_load_const(PyObject *const_cache, cfg_builder *g, PyObject *consts, + _Py_hashtable_t *consts_index) { for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - RETURN_IF_ERROR(basicblock_optimize_load_const(const_cache, b, consts)); + RETURN_IF_ERROR(basicblock_optimize_load_const(const_cache, b, consts, consts_index)); } return SUCCESS; } static int -optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) +optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts, + _Py_hashtable_t *consts_index) { assert(PyDict_CheckExact(const_cache)); assert(PyList_CheckExact(consts)); @@ -2316,11 +2348,11 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) continue; } } - RETURN_IF_ERROR(fold_tuple_of_constants(bb, i, consts, const_cache)); + RETURN_IF_ERROR(fold_tuple_of_constants(bb, i, consts, const_cache, consts_index)); break; case BUILD_LIST: case BUILD_SET: - RETURN_IF_ERROR(optimize_lists_and_sets(bb, i, nextop, consts, const_cache)); + RETURN_IF_ERROR(optimize_lists_and_sets(bb, i, nextop, consts, const_cache, consts_index)); break; case POP_JUMP_IF_NOT_NONE: case POP_JUMP_IF_NONE: @@ -2455,7 +2487,7 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) _Py_FALLTHROUGH; case UNARY_INVERT: case UNARY_NEGATIVE: - RETURN_IF_ERROR(fold_const_unaryop(bb, i, consts, const_cache)); + RETURN_IF_ERROR(fold_const_unaryop(bb, i, consts, const_cache, consts_index)); break; case CALL_INTRINSIC_1: if (oparg == INTRINSIC_LIST_TO_TUPLE) { @@ -2463,15 +2495,15 @@ optimize_basic_block(PyObject *const_cache, basicblock *bb, PyObject *consts) INSTR_SET_OP0(inst, NOP); } else { - RETURN_IF_ERROR(fold_constant_intrinsic_list_to_tuple(bb, i, consts, const_cache)); + RETURN_IF_ERROR(fold_constant_intrinsic_list_to_tuple(bb, i, consts, const_cache, consts_index)); } } else if (oparg == INTRINSIC_UNARY_POSITIVE) { - RETURN_IF_ERROR(fold_const_unaryop(bb, i, consts, const_cache)); + RETURN_IF_ERROR(fold_const_unaryop(bb, i, consts, const_cache, consts_index)); } break; case BINARY_OP: - RETURN_IF_ERROR(fold_const_binop(bb, i, consts, const_cache)); + RETURN_IF_ERROR(fold_const_binop(bb, i, consts, const_cache, consts_index)); break; } } @@ -2516,16 +2548,17 @@ remove_redundant_nops_and_jumps(cfg_builder *g) NOPs. Later those NOPs are removed. */ static int -optimize_cfg(cfg_builder *g, PyObject *consts, PyObject *const_cache, int firstlineno) +optimize_cfg(cfg_builder *g, PyObject *consts, PyObject *const_cache, + _Py_hashtable_t *consts_index, int firstlineno) { assert(PyDict_CheckExact(const_cache)); RETURN_IF_ERROR(check_cfg(g)); RETURN_IF_ERROR(inline_small_or_no_lineno_blocks(g->g_entryblock)); RETURN_IF_ERROR(remove_unreachable(g->g_entryblock)); RETURN_IF_ERROR(resolve_line_numbers(g, firstlineno)); - RETURN_IF_ERROR(optimize_load_const(const_cache, g, consts)); + RETURN_IF_ERROR(optimize_load_const(const_cache, g, consts, consts_index)); for (basicblock *b = g->g_entryblock; b != NULL; b = b->b_next) { - RETURN_IF_ERROR(optimize_basic_block(const_cache, b, consts)); + RETURN_IF_ERROR(optimize_basic_block(const_cache, b, consts, consts_index)); } RETURN_IF_ERROR(remove_redundant_nops_and_pairs(g->g_entryblock)); RETURN_IF_ERROR(remove_unreachable(g->g_entryblock)); @@ -2862,8 +2895,10 @@ optimize_load_fast(cfg_builder *g) // how many inputs should be left on the stack. // Opcodes that consume no inputs + case FORMAT_SIMPLE: case GET_ANEXT: case GET_LEN: + case GET_YIELD_FROM_ITER: case IMPORT_FROM: case MATCH_KEYS: case MATCH_MAPPING: @@ -2898,6 +2933,16 @@ optimize_load_fast(cfg_builder *g) break; } + case END_SEND: + case SET_FUNCTION_ATTRIBUTE: { + assert(_PyOpcode_num_popped(opcode, oparg) == 2); + assert(_PyOpcode_num_pushed(opcode, oparg) == 1); + ref tos = ref_stack_pop(&refs); + ref_stack_pop(&refs); + PUSH_REF(tos.instr, tos.local); + break; + } + // Opcodes that consume some inputs and push new values case CHECK_EXC_MATCH: { ref_stack_pop(&refs); @@ -2927,6 +2972,14 @@ optimize_load_fast(cfg_builder *g) break; } + case LOAD_SPECIAL: + case PUSH_EXC_INFO: { + ref tos = ref_stack_pop(&refs); + PUSH_REF(i, NOT_LOCAL); + PUSH_REF(tos.instr, tos.local); + break; + } + case SEND: { load_fast_push_block(&sp, instr->i_target, refs.size); ref_stack_pop(&refs); @@ -3131,6 +3184,7 @@ remove_unused_consts(basicblock *entryblock, PyObject *consts) index_map = PyMem_Malloc(nconsts * sizeof(Py_ssize_t)); if (index_map == NULL) { + PyErr_NoMemory(); goto end; } for (Py_ssize_t i = 1; i < nconsts; i++) { @@ -3183,6 +3237,7 @@ remove_unused_consts(basicblock *entryblock, PyObject *consts) /* adjust const indices in the bytecode */ reverse_index_map = PyMem_Malloc(nconsts * sizeof(Py_ssize_t)); if (reverse_index_map == NULL) { + PyErr_NoMemory(); goto end; } for (Py_ssize_t i = 0; i < nconsts; i++) { @@ -3439,11 +3494,13 @@ convert_pseudo_conditional_jumps(cfg_builder *g) instr->i_opcode = instr->i_opcode == JUMP_IF_FALSE ? POP_JUMP_IF_FALSE : POP_JUMP_IF_TRUE; location loc = instr->i_loc; + basicblock *except = instr->i_except; cfg_instr copy = { .i_opcode = COPY, .i_oparg = 1, .i_loc = loc, .i_target = NULL, + .i_except = except, }; RETURN_IF_ERROR(basicblock_insert_instruction(b, i++, &copy)); cfg_instr to_bool = { @@ -3451,6 +3508,7 @@ convert_pseudo_conditional_jumps(cfg_builder *g) .i_oparg = 0, .i_loc = loc, .i_target = NULL, + .i_except = except, }; RETURN_IF_ERROR(basicblock_insert_instruction(b, i++, &to_bool)); } @@ -3603,6 +3661,7 @@ _PyCfg_OptimizeCodeUnit(cfg_builder *g, PyObject *consts, PyObject *const_cache, int nlocals, int nparams, int firstlineno) { assert(cfg_builder_check(g)); + assert(g->g_entryblock->b_iused > 0); /** Preprocessing **/ /* Map labels to targets and mark exception handlers */ RETURN_IF_ERROR(translate_jump_labels_to_targets(g->g_entryblock)); @@ -3610,7 +3669,33 @@ _PyCfg_OptimizeCodeUnit(cfg_builder *g, PyObject *consts, PyObject *const_cache, RETURN_IF_ERROR(label_exception_targets(g->g_entryblock)); /** Optimization **/ - RETURN_IF_ERROR(optimize_cfg(g, consts, const_cache, firstlineno)); + + _Py_hashtable_t *consts_index = _Py_hashtable_new( + _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct); + if (consts_index == NULL) { + PyErr_NoMemory(); + return ERROR; + } + + for (Py_ssize_t i = 0; i < PyList_GET_SIZE(consts); i++) { + PyObject *item = PyList_GET_ITEM(consts, i); + if (_Py_hashtable_get_entry(consts_index, (void *)item) != NULL) { + continue; + } + if (_Py_hashtable_set(consts_index, (void *)item, + (void *)(uintptr_t)i) < 0) { + _Py_hashtable_destroy(consts_index); + PyErr_NoMemory(); + return ERROR; + } + } + + int ret = optimize_cfg(g, consts, const_cache, consts_index, firstlineno); + + _Py_hashtable_destroy(consts_index); + + RETURN_IF_ERROR(ret); + RETURN_IF_ERROR(remove_unused_consts(g->g_entryblock, consts)); RETURN_IF_ERROR( add_checks_for_loads_of_uninitialized_variables( @@ -3693,6 +3778,7 @@ insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entrybl .i_oparg = 0, .i_loc = loc, .i_target = NULL, + .i_except = NULL, }; RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 0, &make_gen)); cfg_instr pop_top = { @@ -3700,6 +3786,7 @@ insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entrybl .i_oparg = 0, .i_loc = loc, .i_target = NULL, + .i_except = NULL, }; RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 1, &pop_top)); } @@ -3730,6 +3817,7 @@ insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entrybl .i_oparg = oldindex, .i_loc = NO_LOCATION, .i_target = NULL, + .i_except = NULL, }; if (basicblock_insert_instruction(entryblock, ncellsused, &make_cell) < 0) { PyMem_RawFree(sorted); @@ -3746,6 +3834,7 @@ insert_prefix_instructions(_PyCompile_CodeUnitMetadata *umd, basicblock *entrybl .i_oparg = nfreevars, .i_loc = NO_LOCATION, .i_target = NULL, + .i_except = NULL, }; RETURN_IF_ERROR(basicblock_insert_instruction(entryblock, 0, &copy_frees)); } @@ -4042,6 +4131,10 @@ _PyCompile_OptimizeCfg(PyObject *seq, PyObject *consts, int nlocals) PyErr_SetString(PyExc_ValueError, "expected an instruction sequence"); return NULL; } + if (!PyList_Check(consts)) { + PyErr_SetString(PyExc_TypeError, "consts must be a list"); + return NULL; + } PyObject *const_cache = PyDict_New(); if (const_cache == NULL) { return NULL; diff --git a/Python/frame.c b/Python/frame.c index ce216797e47cda6..1196154d8949c9a 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -54,7 +54,7 @@ take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) _PyFrame_Copy(frame, new_frame); // _PyFrame_Copy takes the reference to the executable, // so we need to restore it. - frame->f_executable = PyStackRef_DUP(new_frame->f_executable); + new_frame->f_executable = PyStackRef_DUP(new_frame->f_executable); f->f_frame = new_frame; new_frame->owner = FRAME_OWNED_BY_FRAME_OBJECT; if (_PyFrame_IsIncomplete(new_frame)) { @@ -135,14 +135,14 @@ PyUnstable_InterpreterFrame_GetCode(struct _PyInterpreterFrame *frame) return PyStackRef_AsPyObjectNew(frame->f_executable); } -int +// NOTE: We allow racy accesses to the instruction pointer from other threads +// for sys._current_frames() and similar APIs. +int _Py_NO_SANITIZE_THREAD PyUnstable_InterpreterFrame_GetLasti(struct _PyInterpreterFrame *frame) { return _PyInterpreterFrame_LASTI(frame) * sizeof(_Py_CODEUNIT); } -// NOTE: We allow racy accesses to the instruction pointer from other threads -// for sys._current_frames() and similar APIs. int _Py_NO_SANITIZE_THREAD PyUnstable_InterpreterFrame_GetLine(_PyInterpreterFrame *frame) { diff --git a/Python/gc.c b/Python/gc.c index 7b0e6d6e8035047..469282683604b83 100644 --- a/Python/gc.c +++ b/Python/gc.c @@ -6,17 +6,18 @@ #include "pycore_ceval.h" // _Py_set_eval_breaker_bit() #include "pycore_dict.h" // _PyInlineValuesSize() #include "pycore_initconfig.h" // _PyStatus_OK() +#include "pycore_context.h" #include "pycore_interp.h" // PyInterpreterState.gc #include "pycore_interpframe.h" // _PyFrame_GetLocalsArray() +#include "pycore_object.h" #include "pycore_object_alloc.h" // _PyObject_MallocWithType() +#include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() #include "pycore_tuple.h" // _PyTuple_MaybeUntrack() #include "pycore_weakref.h" // _PyWeakref_ClearRef() - #include "pydtrace.h" - -#ifndef Py_GIL_DISABLED +#if !defined(Py_GIL_DISABLED) typedef struct _gc_runtime_state GCState; @@ -24,10 +25,6 @@ typedef struct _gc_runtime_state GCState; # define GC_DEBUG #endif -// Define this when debugging the GC -// #define GC_EXTRA_DEBUG - - #define GC_NEXT _PyGCHead_NEXT #define GC_PREV _PyGCHead_PREV @@ -50,7 +47,7 @@ typedef struct _gc_runtime_state GCState; // move_legacy_finalizers() removes this flag instead. // Between them, unreachable list is not normal list and we can not use // most gc_list_* functions for it. -#define NEXT_MASK_UNREACHABLE 2 +#define NEXT_MASK_UNREACHABLE (1) #define AS_GC(op) _Py_AS_GC(op) #define FROM_GC(gc) _Py_FROM_GC(gc) @@ -100,48 +97,9 @@ gc_decref(PyGC_Head *g) g->_gc_prev -= 1 << _PyGC_PREV_SHIFT; } -static inline int -gc_old_space(PyGC_Head *g) -{ - return g->_gc_next & _PyGC_NEXT_MASK_OLD_SPACE_1; -} - -static inline int -other_space(int space) -{ - assert(space == 0 || space == 1); - return space ^ _PyGC_NEXT_MASK_OLD_SPACE_1; -} - -static inline void -gc_flip_old_space(PyGC_Head *g) -{ - g->_gc_next ^= _PyGC_NEXT_MASK_OLD_SPACE_1; -} -static inline void -gc_set_old_space(PyGC_Head *g, int space) -{ - assert(space == 0 || space == _PyGC_NEXT_MASK_OLD_SPACE_1); - g->_gc_next &= ~_PyGC_NEXT_MASK_OLD_SPACE_1; - g->_gc_next |= space; -} +#define GEN_HEAD(gcstate, n) (&(gcstate)->generations[n].head) -static PyGC_Head * -GEN_HEAD(GCState *gcstate, int n) -{ - assert((gcstate->visited_space & (~1)) == 0); - switch(n) { - case 0: - return &gcstate->young.head; - case 1: - return &gcstate->old[gcstate->visited_space].head; - case 2: - return &gcstate->old[gcstate->visited_space^1].head; - default: - Py_UNREACHABLE(); - } -} static GCState * get_gc_state(void) @@ -160,12 +118,11 @@ _PyGC_InitState(GCState *gcstate) GEN.head._gc_prev = (uintptr_t)&GEN.head; \ } while (0) - assert(gcstate->young.count == 0); - assert(gcstate->old[0].count == 0); - assert(gcstate->old[1].count == 0); - INIT_HEAD(gcstate->young); - INIT_HEAD(gcstate->old[0]); - INIT_HEAD(gcstate->old[1]); + for (int i = 0; i < NUM_GENERATIONS; i++) { + assert(gcstate->generations[i].count == 0); + INIT_HEAD(gcstate->generations[i]); + }; + gcstate->generation0 = GEN_HEAD(gcstate, 0); INIT_HEAD(gcstate->permanent_generation); #undef INIT_HEAD @@ -186,7 +143,6 @@ _PyGC_Init(PyInterpreterState *interp) if (gcstate->callbacks == NULL) { return _PyStatus_NO_MEMORY(); } - gcstate->heap_size = 0; return _PyStatus_OK(); } @@ -264,7 +220,6 @@ gc_list_is_empty(PyGC_Head *list) static inline void gc_list_append(PyGC_Head *node, PyGC_Head *list) { - assert((list->_gc_prev & ~_PyGC_PREV_MASK) == 0); PyGC_Head *last = (PyGC_Head *)list->_gc_prev; // last <-> node @@ -322,8 +277,6 @@ gc_list_merge(PyGC_Head *from, PyGC_Head *to) PyGC_Head *from_tail = GC_PREV(from); assert(from_head != from); assert(from_tail != from); - assert(gc_list_is_empty(to) || - gc_old_space(to_tail) == gc_old_space(from_tail)); _PyGCHead_SET_NEXT(to_tail, from_head); _PyGCHead_SET_PREV(from_head, to_tail); @@ -392,8 +345,8 @@ enum flagstates {collecting_clear_unreachable_clear, static void validate_list(PyGC_Head *head, enum flagstates flags) { - assert((head->_gc_prev & ~_PyGC_PREV_MASK) == 0); - assert((head->_gc_next & ~_PyGC_PREV_MASK) == 0); + assert((head->_gc_prev & PREV_MASK_COLLECTING) == 0); + assert((head->_gc_next & NEXT_MASK_UNREACHABLE) == 0); uintptr_t prev_value = 0, next_value = 0; switch (flags) { case collecting_clear_unreachable_clear: @@ -415,7 +368,7 @@ validate_list(PyGC_Head *head, enum flagstates flags) PyGC_Head *gc = GC_NEXT(head); while (gc != head) { PyGC_Head *trueprev = GC_PREV(gc); - PyGC_Head *truenext = GC_NEXT(gc); + PyGC_Head *truenext = (PyGC_Head *)(gc->_gc_next & ~NEXT_MASK_UNREACHABLE); assert(truenext != NULL); assert(trueprev == prev); assert((gc->_gc_prev & PREV_MASK_COLLECTING) == prev_value); @@ -425,58 +378,10 @@ validate_list(PyGC_Head *head, enum flagstates flags) } assert(prev == GC_PREV(head)); } - #else #define validate_list(x, y) do{}while(0) #endif -#ifdef GC_EXTRA_DEBUG - - -static void -gc_list_validate_space(PyGC_Head *head, int space) { - PyGC_Head *gc = GC_NEXT(head); - while (gc != head) { - assert(gc_old_space(gc) == space); - gc = GC_NEXT(gc); - } -} - -static void -validate_spaces(GCState *gcstate) -{ - int visited = gcstate->visited_space; - int not_visited = other_space(visited); - gc_list_validate_space(&gcstate->young.head, not_visited); - for (int space = 0; space < 2; space++) { - gc_list_validate_space(&gcstate->old[space].head, space); - } - gc_list_validate_space(&gcstate->permanent_generation.head, visited); -} - -static void -validate_consistent_old_space(PyGC_Head *head) -{ - PyGC_Head *gc = GC_NEXT(head); - if (gc == head) { - return; - } - int old_space = gc_old_space(gc); - while (gc != head) { - PyGC_Head *truenext = GC_NEXT(gc); - assert(truenext != NULL); - assert(gc_old_space(gc) == old_space); - gc = truenext; - } -} - - -#else -#define validate_spaces(g) do{}while(0) -#define validate_consistent_old_space(l) do{}while(0) -#define gc_list_validate_space(l, s) do{}while(0) -#endif - /*** end of list stuff ***/ @@ -495,8 +400,8 @@ update_refs(PyGC_Head *containers) if (_Py_IsImmortal(op)) { assert(!_Py_IsStaticImmortal(op)); _PyObject_GC_UNTRACK(op); - gc = next; - continue; + gc = next; + continue; } gc_reset_refs(gc, Py_REFCNT(op)); /* Python's cyclic gc should never see an incoming refcount @@ -570,8 +475,7 @@ _PyGC_VisitFrameStack(_PyInterpreterFrame *frame, visitproc visit, void *arg) } /* Subtract internal references from gc_refs. After this, gc_refs is >= 0 - * for all objects in containers, and is GC_REACHABLE for all tracked gc - * objects not in containers. The ones with gc_refs > 0 are directly + * for all objects in containers. The ones with gc_refs > 0 are directly * reachable from outside containers, and so can't be collected. */ static void @@ -622,13 +526,12 @@ visit_reachable(PyObject *op, void *arg) // Manually unlink gc from unreachable list because the list functions // don't work right in the presence of NEXT_MASK_UNREACHABLE flags. PyGC_Head *prev = GC_PREV(gc); - PyGC_Head *next = GC_NEXT(gc); + PyGC_Head *next = (PyGC_Head*)(gc->_gc_next & ~NEXT_MASK_UNREACHABLE); _PyObject_ASSERT(FROM_GC(prev), prev->_gc_next & NEXT_MASK_UNREACHABLE); _PyObject_ASSERT(FROM_GC(next), next->_gc_next & NEXT_MASK_UNREACHABLE); - prev->_gc_next = gc->_gc_next; // copy flag bits - gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; + prev->_gc_next = gc->_gc_next; // copy NEXT_MASK_UNREACHABLE _PyGCHead_SET_PREV(next, prev); gc_list_append(gc, reachable); @@ -680,9 +583,6 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) * or to the right have been scanned yet. */ - validate_consistent_old_space(young); - /* Record which old space we are in, and set NEXT_MASK_UNREACHABLE bit for convenience */ - uintptr_t flags = NEXT_MASK_UNREACHABLE | (gc->_gc_next & _PyGC_NEXT_MASK_OLD_SPACE_1); while (gc != young) { if (gc_get_refs(gc)) { /* gc is definitely reachable from outside the @@ -728,18 +628,17 @@ move_unreachable(PyGC_Head *young, PyGC_Head *unreachable) // But this may pollute the unreachable list head's 'next' pointer // too. That's semantically senseless but expedient here - the // damage is repaired when this function ends. - last->_gc_next = flags | (uintptr_t)gc; + last->_gc_next = (NEXT_MASK_UNREACHABLE | (uintptr_t)gc); _PyGCHead_SET_PREV(gc, last); - gc->_gc_next = flags | (uintptr_t)unreachable; + gc->_gc_next = (NEXT_MASK_UNREACHABLE | (uintptr_t)unreachable); unreachable->_gc_prev = (uintptr_t)gc; } - gc = _PyGCHead_NEXT(prev); + gc = (PyGC_Head*)prev->_gc_next; } // young->_gc_prev must be last element remained in the list. young->_gc_prev = (uintptr_t)prev; - young->_gc_next &= _PyGC_PREV_MASK; // don't let the pollution of the list head's next pointer leak - unreachable->_gc_next &= _PyGC_PREV_MASK; + unreachable->_gc_next &= ~NEXT_MASK_UNREACHABLE; } /* In theory, all tuples should be younger than the @@ -795,8 +694,8 @@ move_legacy_finalizers(PyGC_Head *unreachable, PyGC_Head *finalizers) PyObject *op = FROM_GC(gc); _PyObject_ASSERT(op, gc->_gc_next & NEXT_MASK_UNREACHABLE); - next = GC_NEXT(gc); gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; + next = (PyGC_Head*)gc->_gc_next; if (has_legacy_finalizer(op)) { gc_clear_collecting(gc); @@ -819,8 +718,8 @@ clear_unreachable_mask(PyGC_Head *unreachable) PyGC_Head *gc, *next; for (gc = GC_NEXT(unreachable); gc != unreachable; gc = next) { _PyObject_ASSERT((PyObject*)FROM_GC(gc), gc->_gc_next & NEXT_MASK_UNREACHABLE); - next = GC_NEXT(gc); gc->_gc_next &= ~NEXT_MASK_UNREACHABLE; + next = (PyGC_Head*)gc->_gc_next; } validate_list(unreachable, collecting_set_unreachable_clear); } @@ -870,7 +769,7 @@ move_legacy_finalizer_reachable(PyGC_Head *finalizers) * no object in `unreachable` is weakly referenced anymore. */ static int -handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) +handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old, bool allow_callbacks) { PyGC_Head *gc; PyObject *op; /* generally FROM_GC(gc) */ @@ -879,7 +778,9 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) PyGC_Head *next; int num_freed = 0; - gc_list_init(&wrcb_to_call); + if (allow_callbacks) { + gc_list_init(&wrcb_to_call); + } /* Clear all weakrefs to the objects in unreachable. If such a weakref * also has a callback, move it into `wrcb_to_call` if the callback @@ -935,6 +836,11 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) _PyObject_ASSERT((PyObject *)wr, wr->wr_object == op); _PyWeakref_ClearRef(wr); _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); + + if (!allow_callbacks) { + continue; + } + if (wr->wr_callback == NULL) { /* no callback */ continue; @@ -987,10 +893,13 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) } } + if (!allow_callbacks) { + return 0; + } + /* Invoke the callbacks we decided to honor. It's safe to invoke them * because they can't reference unreachable objects. */ - int visited_space = get_gc_state()->visited_space; while (! gc_list_is_empty(&wrcb_to_call)) { PyObject *temp; PyObject *callback; @@ -1026,7 +935,6 @@ handle_weakrefs(PyGC_Head *unreachable, PyGC_Head *old) Py_DECREF(op); if (wrcb_to_call._gc_next == (uintptr_t)gc) { /* object is still alive -- move it */ - gc_set_old_space(gc, visited_space); gc_list_move(gc, old); } else { @@ -1155,6 +1063,25 @@ delete_garbage(PyThreadState *tstate, GCState *gcstate, } +// Show stats for objects in each generations +static void +show_stats_each_generations(GCState *gcstate) +{ + char buf[100]; + size_t pos = 0; + + for (int i = 0; i < NUM_GENERATIONS && pos < sizeof(buf); i++) { + pos += PyOS_snprintf(buf+pos, sizeof(buf)-pos, + " %zd", + gc_list_size(GEN_HEAD(gcstate, i))); + } + + PySys_FormatStderr( + "gc: objects in each generation:%s\n" + "gc: objects in permanent generation: %zd\n", + buf, gc_list_size(&gcstate->permanent_generation.head)); +} + /* Deduce which objects among "base" are unreachable from outside the list and move them to 'unreachable'. The process consist in the following steps: @@ -1228,6 +1155,7 @@ deduce_unreachable(PyGC_Head *base, PyGC_Head *unreachable) { * the reachable objects instead. But this is a one-time cost, probably not * worth complicating the code to speed just a little. */ + gc_list_init(unreachable); move_unreachable(base, unreachable); // gc_prev is pointer again validate_list(base, collecting_clear_unreachable_clear); validate_list(unreachable, collecting_set_unreachable_set); @@ -1265,455 +1193,226 @@ handle_resurrected_objects(PyGC_Head *unreachable, PyGC_Head* still_unreachable, gc_list_merge(resurrected, old_generation); } -static void -gc_collect_region(PyThreadState *tstate, - PyGC_Head *from, - PyGC_Head *to, - struct gc_collection_stats *stats); - -static inline Py_ssize_t -gc_list_set_space(PyGC_Head *list, int space) -{ - Py_ssize_t size = 0; - PyGC_Head *gc; - for (gc = GC_NEXT(list); gc != list; gc = GC_NEXT(gc)) { - gc_set_old_space(gc, space); - size++; - } - return size; -} -/* Making progress in the incremental collector - * In order to eventually collect all cycles - * the incremental collector must progress through the old - * space faster than objects are added to the old space. - * - * Each young or incremental collection adds a number of - * objects, S (for survivors) to the old space, and - * incremental collectors scan I objects from the old space. - * I > S must be true. We also want I > S * N to be where - * N > 1. Higher values of N mean that the old space is - * scanned more rapidly. - * The default incremental threshold of 10 translates to - * N == 1.4 (1 + 4/threshold) +/* Invoke progress callbacks to notify clients that garbage collection + * is starting or stopping */ - -/* Divide by 10, so that the default incremental threshold of 10 - * scans objects at 1% of the heap size */ -#define SCAN_RATE_DIVISOR 10 - static void -add_stats(GCState *gcstate, int gen, struct gc_collection_stats *stats) +invoke_gc_callback(PyThreadState *tstate, const char *phase, + int generation, Py_ssize_t collected, + Py_ssize_t uncollectable) { - gcstate->generation_stats[gen].collected += stats->collected; - gcstate->generation_stats[gen].uncollectable += stats->uncollectable; - gcstate->generation_stats[gen].collections += 1; -} + assert(!_PyErr_Occurred(tstate)); -static void -gc_collect_young(PyThreadState *tstate, - struct gc_collection_stats *stats) -{ + /* we may get called very early */ GCState *gcstate = &tstate->interp->gc; - validate_spaces(gcstate); - PyGC_Head *young = &gcstate->young.head; - PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; - untrack_tuples(young); - GC_STAT_ADD(0, collections, 1); -#ifdef Py_STATS - { - Py_ssize_t count = 0; - PyGC_Head *gc; - for (gc = GC_NEXT(young); gc != young; gc = GC_NEXT(gc)) { - count++; + if (gcstate->callbacks == NULL) { + return; + } + + /* The local variable cannot be rebound, check it for sanity */ + assert(PyList_CheckExact(gcstate->callbacks)); + PyObject *info = NULL; + if (PyList_GET_SIZE(gcstate->callbacks) != 0) { + info = Py_BuildValue("{sisnsn}", + "generation", generation, + "collected", collected, + "uncollectable", uncollectable); + if (info == NULL) { + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; } } -#endif - PyGC_Head survivors; - gc_list_init(&survivors); - gc_list_set_space(young, gcstate->visited_space); - gc_collect_region(tstate, young, &survivors, stats); - gc_list_merge(&survivors, visited); - validate_spaces(gcstate); - gcstate->young.count = 0; - gcstate->old[gcstate->visited_space].count++; - add_stats(gcstate, 0, stats); - validate_spaces(gcstate); -} + PyObject *phase_obj = PyUnicode_FromString(phase); + if (phase_obj == NULL) { + Py_XDECREF(info); + PyErr_FormatUnraisable("Exception ignored on invoking gc callbacks"); + return; + } -#ifndef NDEBUG -static inline int -IS_IN_VISITED(PyGC_Head *gc, int visited_space) -{ - assert(visited_space == 0 || other_space(visited_space) == 0); - return gc_old_space(gc) == visited_space; + PyObject *stack[] = {phase_obj, info}; + for (Py_ssize_t i=0; i<PyList_GET_SIZE(gcstate->callbacks); i++) { + PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); + Py_INCREF(cb); /* make sure cb doesn't go away */ + r = PyObject_Vectorcall(cb, stack, 2, NULL); + if (r == NULL) { + PyErr_FormatUnraisable("Exception ignored while " + "calling GC callback %R", cb); + } + else { + Py_DECREF(r); + } + Py_DECREF(cb); + } + Py_DECREF(phase_obj); + Py_XDECREF(info); + assert(!_PyErr_Occurred(tstate)); } -#endif -struct container_and_flag { - PyGC_Head *container; - int visited_space; - intptr_t size; -}; -/* A traversal callback for adding to container) */ +/* Find the oldest generation (highest numbered) where the count + * exceeds the threshold. Objects in the that generation and + * generations younger than it will be collected. */ static int -visit_add_to_container(PyObject *op, void *arg) -{ - OBJECT_STAT_INC(object_visits); - struct container_and_flag *cf = (struct container_and_flag *)arg; - int visited = cf->visited_space; - assert(visited == get_gc_state()->visited_space); - if (!_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { - PyGC_Head *gc = AS_GC(op); - if (_PyObject_GC_IS_TRACKED(op) && - gc_old_space(gc) != visited) { - gc_flip_old_space(gc); - gc_list_move(gc, cf->container); - cf->size++; +gc_select_generation(GCState *gcstate) +{ + for (int i = NUM_GENERATIONS-1; i >= 0; i--) { + if (gcstate->generations[i].count > gcstate->generations[i].threshold) { + /* Avoid quadratic performance degradation in number + of tracked objects (see also issue #4074): + + To limit the cost of garbage collection, there are two strategies; + - make each collection faster, e.g. by scanning fewer objects + - do less collections + This heuristic is about the latter strategy. + + In addition to the various configurable thresholds, we only trigger a + full collection if the ratio + + long_lived_pending / long_lived_total + + is above a given value (hardwired to 25%). + + The reason is that, while "non-full" collections (i.e., collections of + the young and middle generations) will always examine roughly the same + number of objects -- determined by the aforementioned thresholds --, + the cost of a full collection is proportional to the total number of + long-lived objects, which is virtually unbounded. + + Indeed, it has been remarked that doing a full collection every + <constant number> of object creations entails a dramatic performance + degradation in workloads which consist in creating and storing lots of + long-lived objects (e.g. building a large list of GC-tracked objects would + show quadratic performance, instead of linear as expected: see issue #4074). + + Using the above ratio, instead, yields amortized linear performance in + the total number of objects (the effect of which can be summarized + thusly: "each full garbage collection is more and more costly as the + number of objects grows, but we do fewer and fewer of them"). + + This heuristic was suggested by Martin von Löwis on python-dev in + June 2008. His original analysis and proposal can be found at: + http://mail.python.org/pipermail/python-dev/2008-June/080579.html + */ + if (i == NUM_GENERATIONS - 1 + && gcstate->long_lived_pending < gcstate->long_lived_total / 4) + { + continue; + } + return i; } } - return 0; + return -1; } -static intptr_t -expand_region_transitively_reachable(PyGC_Head *container, PyGC_Head *gc, GCState *gcstate) + +/* This is the main function. Read this to understand how the + * collection process works. */ +static Py_ssize_t +gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason) { - struct container_and_flag arg = { - .container = container, - .visited_space = gcstate->visited_space, - .size = 0 - }; - assert(GC_NEXT(gc) == container); - while (gc != container) { - /* Survivors will be moved to visited space, so they should - * have been marked as visited */ - assert(IS_IN_VISITED(gc, gcstate->visited_space)); - PyObject *op = FROM_GC(gc); - assert(_PyObject_GC_IS_TRACKED(op)); - if (_Py_IsImmortal(op)) { - PyGC_Head *next = GC_NEXT(gc); - gc_list_move(gc, &get_gc_state()->permanent_generation.head); - gc = next; - continue; - } - traverseproc traverse = Py_TYPE(op)->tp_traverse; - (void) traverse(op, - visit_add_to_container, - &arg); - gc = GC_NEXT(gc); - } - return arg.size; -} + int i; + Py_ssize_t m = 0; /* # objects collected */ + Py_ssize_t n = 0; /* # unreachable objects that couldn't be collected */ + PyGC_Head *young; /* the generation we are examining */ + PyGC_Head *old; /* next older generation */ + PyGC_Head unreachable; /* non-problematic unreachable trash */ + PyGC_Head finalizers; /* objects with, & reachable from, __del__ */ + PyGC_Head *gc; + PyTime_t t1 = 0; /* initialize to prevent a compiler warning */ + GCState *gcstate = &tstate->interp->gc; -/* Do bookkeeping for a completed GC cycle */ -static void -completed_scavenge(GCState *gcstate) -{ - /* We must observe two invariants: - * 1. Members of the permanent generation must be marked visited. - * 2. We cannot touch members of the permanent generation. */ - int visited; - if (gc_list_is_empty(&gcstate->permanent_generation.head)) { - /* Permanent generation is empty so we can flip spaces bit */ - int not_visited = gcstate->visited_space; - visited = other_space(not_visited); - gcstate->visited_space = visited; - /* Make sure all objects have visited bit set correctly */ - gc_list_set_space(&gcstate->young.head, not_visited); - } - else { - /* We must move the objects from visited to pending space. */ - visited = gcstate->visited_space; - int not_visited = other_space(visited); - assert(gc_list_is_empty(&gcstate->old[not_visited].head)); - gc_list_merge(&gcstate->old[visited].head, &gcstate->old[not_visited].head); - gc_list_set_space(&gcstate->old[not_visited].head, not_visited); + // gc_collect_main() must not be called before _PyGC_Init + // or after _PyGC_Fini() + assert(gcstate->garbage != NULL); + assert(!_PyErr_Occurred(tstate)); + + int expected = 0; + if (!_Py_atomic_compare_exchange_int(&gcstate->collecting, &expected, 1)) { + // Don't start a garbage collection if one is already in progress. + return 0; } - assert(gc_list_is_empty(&gcstate->old[visited].head)); - gcstate->work_to_do = 0; - gcstate->phase = GC_PHASE_MARK; -} -static intptr_t -move_to_reachable(PyObject *op, PyGC_Head *reachable, int visited_space) -{ - if (op != NULL && !_Py_IsImmortal(op) && _PyObject_IS_GC(op)) { - PyGC_Head *gc = AS_GC(op); - if (_PyObject_GC_IS_TRACKED(op) && - gc_old_space(gc) != visited_space) { - gc_flip_old_space(gc); - gc_list_move(gc, reachable); - return 1; + if (generation == GENERATION_AUTO) { + // Select the oldest generation that needs collecting. We will collect + // objects from that generation and all generations younger than it. + generation = gc_select_generation(gcstate); + if (generation < 0) { + // No generation needs to be collected. + _Py_atomic_store_int(&gcstate->collecting, 0); + return 0; } } - return 0; -} -static intptr_t -mark_all_reachable(PyGC_Head *reachable, PyGC_Head *visited, int visited_space) -{ - // Transitively traverse all objects from reachable, until empty - struct container_and_flag arg = { - .container = reachable, - .visited_space = visited_space, - .size = 0 - }; - while (!gc_list_is_empty(reachable)) { - PyGC_Head *gc = _PyGCHead_NEXT(reachable); - assert(gc_old_space(gc) == visited_space); - gc_list_move(gc, visited); - PyObject *op = FROM_GC(gc); - traverseproc traverse = Py_TYPE(op)->tp_traverse; - (void) traverse(op, - visit_add_to_container, - &arg); - } - gc_list_validate_space(visited, visited_space); - return arg.size; -} - -static intptr_t -mark_stacks(PyInterpreterState *interp, PyGC_Head *visited, int visited_space, bool start) -{ - PyGC_Head reachable; - gc_list_init(&reachable); - Py_ssize_t objects_marked = 0; - // Move all objects on stacks to reachable - _PyRuntimeState *runtime = &_PyRuntime; - HEAD_LOCK(runtime); - PyThreadState* ts = PyInterpreterState_ThreadHead(interp); - HEAD_UNLOCK(runtime); - while (ts) { - _PyInterpreterFrame *frame = ts->current_frame; - while (frame) { - if (frame->owner >= FRAME_OWNED_BY_INTERPRETER) { - frame = frame->previous; - continue; - } - _PyStackRef *locals = frame->localsplus; - _PyStackRef *sp = frame->stackpointer; - objects_marked += move_to_reachable(frame->f_locals, &reachable, visited_space); - PyObject *func = PyStackRef_AsPyObjectBorrow(frame->f_funcobj); - objects_marked += move_to_reachable(func, &reachable, visited_space); - while (sp > locals) { - sp--; - if (PyStackRef_IsNullOrInt(*sp)) { - continue; - } - PyObject *op = PyStackRef_AsPyObjectBorrow(*sp); - if (_Py_IsImmortal(op)) { - continue; - } - if (_PyObject_IS_GC(op)) { - PyGC_Head *gc = AS_GC(op); - if (_PyObject_GC_IS_TRACKED(op) && - gc_old_space(gc) != visited_space) { - gc_flip_old_space(gc); - objects_marked++; - gc_list_move(gc, &reachable); - } - } - } - if (!start && frame->visited) { - // If this frame has already been visited, then the lower frames - // will have already been visited and will not have changed - break; - } - frame->visited = 1; - frame = frame->previous; - } - HEAD_LOCK(runtime); - ts = PyThreadState_Next(ts); - HEAD_UNLOCK(runtime); + assert(generation >= 0 && generation < NUM_GENERATIONS); + +#ifdef Py_STATS + if (_Py_stats) { + _Py_stats->object_stats.object_visits = 0; } - objects_marked += mark_all_reachable(&reachable, visited, visited_space); - assert(gc_list_is_empty(&reachable)); - return objects_marked; -} +#endif + GC_STAT_ADD(generation, collections, 1); -static intptr_t -mark_global_roots(PyInterpreterState *interp, PyGC_Head *visited, int visited_space) -{ - PyGC_Head reachable; - gc_list_init(&reachable); - Py_ssize_t objects_marked = 0; - objects_marked += move_to_reachable(interp->sysdict, &reachable, visited_space); - objects_marked += move_to_reachable(interp->builtins, &reachable, visited_space); - objects_marked += move_to_reachable(interp->dict, &reachable, visited_space); - struct types_state *types = &interp->types; - for (int i = 0; i < _Py_MAX_MANAGED_STATIC_BUILTIN_TYPES; i++) { - objects_marked += move_to_reachable(types->builtins.initialized[i].tp_dict, &reachable, visited_space); - objects_marked += move_to_reachable(types->builtins.initialized[i].tp_subclasses, &reachable, visited_space); + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(tstate, "start", generation, 0, 0); } - for (int i = 0; i < _Py_MAX_MANAGED_STATIC_EXT_TYPES; i++) { - objects_marked += move_to_reachable(types->for_extensions.initialized[i].tp_dict, &reachable, visited_space); - objects_marked += move_to_reachable(types->for_extensions.initialized[i].tp_subclasses, &reachable, visited_space); + + if (gcstate->debug & _PyGC_DEBUG_STATS) { + PySys_WriteStderr("gc: collecting generation %d...\n", generation); + show_stats_each_generations(gcstate); + // ignore error: don't interrupt the GC if reading the clock fails + (void)PyTime_PerfCounterRaw(&t1); } - objects_marked += mark_all_reachable(&reachable, visited, visited_space); - assert(gc_list_is_empty(&reachable)); - return objects_marked; -} -static intptr_t -mark_at_start(PyThreadState *tstate) -{ - // TO DO -- Make this incremental - GCState *gcstate = &tstate->interp->gc; - PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; - Py_ssize_t objects_marked = mark_global_roots(tstate->interp, visited, gcstate->visited_space); - objects_marked += mark_stacks(tstate->interp, visited, gcstate->visited_space, true); - gcstate->work_to_do -= objects_marked; - gcstate->phase = GC_PHASE_COLLECT; - validate_spaces(gcstate); - return objects_marked; -} - -static intptr_t -assess_work_to_do(GCState *gcstate) -{ - /* The amount of work we want to do depends on three things. - * 1. The number of new objects created - * 2. The growth in heap size since the last collection - * 3. The heap size (up to the number of new objects, to avoid quadratic effects) - * - * For a steady state heap, the amount of work to do is three times the number - * of new objects added to the heap. This ensures that we stay ahead in the - * worst case of all new objects being garbage. - * - * This could be improved by tracking survival rates, but it is still a - * large improvement on the non-marking approach. - */ - intptr_t scale_factor = gcstate->old[0].threshold; - if (scale_factor < 2) { - scale_factor = 2; + if (PyDTrace_GC_START_ENABLED()) { + PyDTrace_GC_START(generation); } - intptr_t new_objects = gcstate->young.count; - intptr_t max_heap_fraction = new_objects*3/2; - intptr_t heap_fraction = gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; - if (heap_fraction > max_heap_fraction) { - heap_fraction = max_heap_fraction; + + /* update collection and allocation counters */ + if (generation+1 < NUM_GENERATIONS) { + gcstate->generations[generation+1].count += 1; + } + for (i = 0; i <= generation; i++) { + gcstate->generations[i].count = 0; } - gcstate->young.count = 0; - return new_objects + heap_fraction; -} -static void -gc_collect_increment(PyThreadState *tstate, struct gc_collection_stats *stats) -{ - GC_STAT_ADD(1, collections, 1); - GCState *gcstate = &tstate->interp->gc; - gcstate->work_to_do += assess_work_to_do(gcstate); - untrack_tuples(&gcstate->young.head); - if (gcstate->phase == GC_PHASE_MARK) { - Py_ssize_t objects_marked = mark_at_start(tstate); - GC_STAT_ADD(1, objects_transitively_reachable, objects_marked); - gcstate->work_to_do -= objects_marked; - validate_spaces(gcstate); - return; + /* merge younger generations with one we are currently collecting */ + for (i = 0; i < generation; i++) { + gc_list_merge(GEN_HEAD(gcstate, i), GEN_HEAD(gcstate, generation)); } - PyGC_Head *not_visited = &gcstate->old[gcstate->visited_space^1].head; - PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; - PyGC_Head increment; - gc_list_init(&increment); - int scale_factor = gcstate->old[0].threshold; - if (scale_factor < 2) { - scale_factor = 2; - } - intptr_t objects_marked = mark_stacks(tstate->interp, visited, gcstate->visited_space, false); - GC_STAT_ADD(1, objects_transitively_reachable, objects_marked); - gcstate->work_to_do -= objects_marked; - gc_list_set_space(&gcstate->young.head, gcstate->visited_space); - gc_list_merge(&gcstate->young.head, &increment); - gc_list_validate_space(&increment, gcstate->visited_space); - Py_ssize_t increment_size = gc_list_size(&increment); - while (increment_size < gcstate->work_to_do) { - if (gc_list_is_empty(not_visited)) { - break; - } - PyGC_Head *gc = _PyGCHead_NEXT(not_visited); - gc_list_move(gc, &increment); - increment_size++; - assert(!_Py_IsImmortal(FROM_GC(gc))); - gc_set_old_space(gc, gcstate->visited_space); - increment_size += expand_region_transitively_reachable(&increment, gc, gcstate); - } - GC_STAT_ADD(1, objects_not_transitively_reachable, increment_size); - validate_list(&increment, collecting_clear_unreachable_clear); - gc_list_validate_space(&increment, gcstate->visited_space); - PyGC_Head survivors; - gc_list_init(&survivors); - gc_collect_region(tstate, &increment, &survivors, stats); - gc_list_merge(&survivors, visited); - assert(gc_list_is_empty(&increment)); - gcstate->work_to_do += gcstate->heap_size / SCAN_RATE_DIVISOR / scale_factor; - gcstate->work_to_do -= increment_size; - - add_stats(gcstate, 1, stats); - if (gc_list_is_empty(not_visited)) { - completed_scavenge(gcstate); - } - validate_spaces(gcstate); -} -static void -gc_collect_full(PyThreadState *tstate, - struct gc_collection_stats *stats) -{ - GC_STAT_ADD(2, collections, 1); - GCState *gcstate = &tstate->interp->gc; - validate_spaces(gcstate); - PyGC_Head *young = &gcstate->young.head; - PyGC_Head *pending = &gcstate->old[gcstate->visited_space^1].head; - PyGC_Head *visited = &gcstate->old[gcstate->visited_space].head; - untrack_tuples(young); - /* merge all generations into visited */ - gc_list_merge(young, pending); - gc_list_validate_space(pending, 1-gcstate->visited_space); - gc_list_set_space(pending, gcstate->visited_space); - gcstate->young.count = 0; - gc_list_merge(pending, visited); - validate_spaces(gcstate); - - gc_collect_region(tstate, visited, visited, - stats); - validate_spaces(gcstate); - gcstate->young.count = 0; - gcstate->old[0].count = 0; - gcstate->old[1].count = 0; - completed_scavenge(gcstate); - _PyGC_ClearAllFreeLists(tstate->interp); - validate_spaces(gcstate); - add_stats(gcstate, 2, stats); -} - -/* This is the main function. Read this to understand how the - * collection process works. */ -static void -gc_collect_region(PyThreadState *tstate, - PyGC_Head *from, - PyGC_Head *to, - struct gc_collection_stats *stats) -{ - PyGC_Head unreachable; /* non-problematic unreachable trash */ - PyGC_Head finalizers; /* objects with, & reachable from, __del__ */ - PyGC_Head *gc; /* initialize to prevent a compiler warning */ - GCState *gcstate = &tstate->interp->gc; + /* handy references */ + young = GEN_HEAD(gcstate, generation); + if (generation < NUM_GENERATIONS-1) { + old = GEN_HEAD(gcstate, generation+1); + } + else { + old = young; + } + validate_list(old, collecting_clear_unreachable_clear); - assert(gcstate->garbage != NULL); - assert(!_PyErr_Occurred(tstate)); + deduce_unreachable(young, &unreachable); - gc_list_init(&unreachable); - deduce_unreachable(from, &unreachable); - validate_consistent_old_space(from); - untrack_tuples(from); - validate_consistent_old_space(to); - if (from != to) { - gc_list_merge(from, to); - } - validate_consistent_old_space(to); + untrack_tuples(young); /* Move reachable objects to next generation. */ + if (young != old) { + if (generation == NUM_GENERATIONS - 2) { + gcstate->long_lived_pending += gc_list_size(young); + } + gc_list_merge(young, old); + } + else { + // In Python <= 3.13, we called untrack_dicts(young) here to untrack + // atomic-only dicts (see issue #14775). Python 3.14 removed the lazy + // dict tracking machinery entirely (GH-127010) -- dicts are always + // tracked from creation and never untracked by GC. That way, we don't + // have to restore MAINTAIN_TRACKING across every PyDict_SetItem call + // site; the cost is slightly more work for full collections on dicts + // with only atomic values. + gcstate->long_lived_pending = 0; + gcstate->long_lived_total = gc_list_size(young); + } /* All objects in unreachable are trash, but objects reachable from * legacy finalizers (e.g. tp_del) can't safely be deleted. @@ -1727,8 +1426,10 @@ gc_collect_region(PyThreadState *tstate, * and we move those into the finalizers list too. */ move_legacy_finalizer_reachable(&finalizers); + validate_list(&finalizers, collecting_clear_unreachable_clear); validate_list(&unreachable, collecting_set_unreachable_clear); + /* Print debugging information. */ if (gcstate->debug & _PyGC_DEBUG_COLLECTABLE) { for (gc = GC_NEXT(&unreachable); gc != &unreachable; gc = GC_NEXT(gc)) { @@ -1737,102 +1438,98 @@ gc_collect_region(PyThreadState *tstate, } /* Clear weakrefs and invoke callbacks as necessary. */ - stats->collected += handle_weakrefs(&unreachable, to); - gc_list_validate_space(to, gcstate->visited_space); - validate_list(to, collecting_clear_unreachable_clear); + m += handle_weakrefs(&unreachable, old, true); + validate_list(old, collecting_clear_unreachable_clear); validate_list(&unreachable, collecting_set_unreachable_clear); /* Call tp_finalize on objects which have one. */ finalize_garbage(tstate, &unreachable); + /* Handle any objects that may have resurrected after the call * to 'finalize_garbage' and continue the collection with the * objects that are still unreachable */ PyGC_Head final_unreachable; - gc_list_init(&final_unreachable); - handle_resurrected_objects(&unreachable, &final_unreachable, to); + handle_resurrected_objects(&unreachable, &final_unreachable, old); + + /* Clear weakrefs to objects in the unreachable set. No Python-level + * code must be allowed to access those unreachable objects. During + * delete_garbage(), finalizers outside the unreachable set might run + * and create new weakrefs. If those weakrefs were not cleared, they + * could reveal unreachable objects. Callbacks are not executed. + */ + handle_weakrefs(&final_unreachable, NULL, false); /* Call tp_clear on objects in the final_unreachable set. This will cause * the reference cycles to be broken. It may also cause some objects * in finalizers to be freed. */ - stats->collected += gc_list_size(&final_unreachable); - delete_garbage(tstate, gcstate, &final_unreachable, to); + m += gc_list_size(&final_unreachable); + delete_garbage(tstate, gcstate, &final_unreachable, old); /* Collect statistics on uncollectable objects found and print * debugging information. */ - Py_ssize_t n = 0; for (gc = GC_NEXT(&finalizers); gc != &finalizers; gc = GC_NEXT(gc)) { n++; - if (gcstate->debug & _PyGC_DEBUG_COLLECTABLE) + if (gcstate->debug & _PyGC_DEBUG_UNCOLLECTABLE) debug_cycle("uncollectable", FROM_GC(gc)); } - stats->uncollectable = n; + if (gcstate->debug & _PyGC_DEBUG_STATS) { + PyTime_t t2; + (void)PyTime_PerfCounterRaw(&t2); + double d = PyTime_AsSecondsDouble(t2 - t1); + PySys_WriteStderr( + "gc: done, %zd unreachable, %zd uncollectable, %.4fs elapsed\n", + n+m, n, d); + } + /* Append instances in the uncollectable set to a Python * reachable list of garbage. The programmer has to deal with * this if they insist on creating this type of structure. */ - handle_legacy_finalizers(tstate, gcstate, &finalizers, to); - gc_list_validate_space(to, gcstate->visited_space); - validate_list(to, collecting_clear_unreachable_clear); -} + handle_legacy_finalizers(tstate, gcstate, &finalizers, old); + validate_list(old, collecting_clear_unreachable_clear); -/* Invoke progress callbacks to notify clients that garbage collection - * is starting or stopping - */ -static void -do_gc_callback(GCState *gcstate, const char *phase, - int generation, struct gc_collection_stats *stats) -{ - assert(!PyErr_Occurred()); + /* Clear free list only during the collection of the highest + * generation */ + if (generation == NUM_GENERATIONS-1) { + _PyGC_ClearAllFreeLists(tstate->interp); + } - /* The local variable cannot be rebound, check it for sanity */ - assert(PyList_CheckExact(gcstate->callbacks)); - PyObject *info = NULL; - if (PyList_GET_SIZE(gcstate->callbacks) != 0) { - info = Py_BuildValue("{sisnsn}", - "generation", generation, - "collected", stats->collected, - "uncollectable", stats->uncollectable); - if (info == NULL) { - PyErr_FormatUnraisable("Exception ignored while invoking gc callbacks"); - return; + if (_PyErr_Occurred(tstate)) { + if (reason == _Py_GC_REASON_SHUTDOWN) { + _PyErr_Clear(tstate); + } + else { + PyErr_FormatUnraisable("Exception ignored in garbage collection"); } } - PyObject *phase_obj = PyUnicode_FromString(phase); - if (phase_obj == NULL) { - Py_XDECREF(info); - PyErr_FormatUnraisable("Exception ignored while invoking gc callbacks"); - return; + /* Update stats */ + struct gc_generation_stats *stats = &gcstate->generation_stats[generation]; + stats->collections++; + stats->collected += m; + stats->uncollectable += n; + + GC_STAT_ADD(generation, objects_collected, m); +#ifdef Py_STATS + if (_Py_stats) { + GC_STAT_ADD(generation, object_visits, + _Py_stats->object_stats.object_visits); + _Py_stats->object_stats.object_visits = 0; } +#endif - PyObject *stack[] = {phase_obj, info}; - for (Py_ssize_t i=0; i<PyList_GET_SIZE(gcstate->callbacks); i++) { - PyObject *r, *cb = PyList_GET_ITEM(gcstate->callbacks, i); - Py_INCREF(cb); /* make sure cb doesn't go away */ - r = PyObject_Vectorcall(cb, stack, 2, NULL); - if (r == NULL) { - PyErr_FormatUnraisable("Exception ignored while " - "calling GC callback %R", cb); - } - else { - Py_DECREF(r); - } - Py_DECREF(cb); + if (PyDTrace_GC_DONE_ENABLED()) { + PyDTrace_GC_DONE(n + m); } - Py_DECREF(phase_obj); - Py_XDECREF(info); - assert(!PyErr_Occurred()); -} -static void -invoke_gc_callback(GCState *gcstate, const char *phase, - int generation, struct gc_collection_stats *stats) -{ - if (gcstate->callbacks == NULL) { - return; + if (reason != _Py_GC_REASON_SHUTDOWN) { + invoke_gc_callback(tstate, "stop", generation, m, n); } - do_gc_callback(gcstate, phase, generation, stats); + + assert(!_PyErr_Occurred(tstate)); + _Py_atomic_store_int(&gcstate->collecting, 0); + return n + m; } static int @@ -1894,25 +1591,20 @@ _PyGC_GetObjects(PyInterpreterState *interp, int generation) GCState *gcstate = &interp->gc; PyObject *result = PyList_New(0); - /* Generation: - * -1: Return all objects - * 0: All young objects - * 1: No objects - * 2: All old objects - */ - if (result == NULL || generation == 1) { - return result; + if (result == NULL) { + return NULL; } - if (generation <= 0) { - if (append_objects(result, &gcstate->young.head)) { - goto error; + + if (generation == -1) { + /* If generation is -1, get all objects from all generations */ + for (int i = 0; i < NUM_GENERATIONS; i++) { + if (append_objects(result, GEN_HEAD(gcstate, i))) { + goto error; + } } } - if (generation != 0) { - if (append_objects(result, &gcstate->old[0].head)) { - goto error; - } - if (append_objects(result, &gcstate->old[1].head)) { + else { + if (append_objects(result, GEN_HEAD(gcstate, generation))) { goto error; } } @@ -1927,23 +1619,10 @@ void _PyGC_Freeze(PyInterpreterState *interp) { GCState *gcstate = &interp->gc; - /* The permanent_generation must be visited */ - gc_list_set_space(&gcstate->young.head, gcstate->visited_space); - gc_list_merge(&gcstate->young.head, &gcstate->permanent_generation.head); - gcstate->young.count = 0; - PyGC_Head*old0 = &gcstate->old[0].head; - PyGC_Head*old1 = &gcstate->old[1].head; - if (gcstate->visited_space) { - gc_list_set_space(old0, 1); + for (int i = 0; i < NUM_GENERATIONS; ++i) { + gc_list_merge(GEN_HEAD(gcstate, i), &gcstate->permanent_generation.head); + gcstate->generations[i].count = 0; } - else { - gc_list_set_space(old1, 0); - } - gc_list_merge(old0, &gcstate->permanent_generation.head); - gcstate->old[0].count = 0; - gc_list_merge(old1, &gcstate->permanent_generation.head); - gcstate->old[1].count = 0; - validate_spaces(gcstate); } void @@ -1951,8 +1630,7 @@ _PyGC_Unfreeze(PyInterpreterState *interp) { GCState *gcstate = &interp->gc; gc_list_merge(&gcstate->permanent_generation.head, - &gcstate->old[gcstate->visited_space].head); - validate_spaces(gcstate); + GEN_HEAD(gcstate, NUM_GENERATIONS-1)); } Py_ssize_t @@ -1988,86 +1666,29 @@ PyGC_IsEnabled(void) return gcstate->enabled; } -// Show stats for objects in each generations -static void -show_stats_each_generations(GCState *gcstate) -{ - char buf[100]; - size_t pos = 0; - - for (int i = 0; i < NUM_GENERATIONS && pos < sizeof(buf); i++) { - pos += PyOS_snprintf(buf+pos, sizeof(buf)-pos, - " %zd", - gc_list_size(GEN_HEAD(gcstate, i))); - } - PySys_FormatStderr( - "gc: objects in each generation:%s\n" - "gc: objects in permanent generation: %zd\n", - buf, gc_list_size(&gcstate->permanent_generation.head)); -} - +/* Public API to invoke gc.collect() from C */ Py_ssize_t -_PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) +PyGC_Collect(void) { + PyThreadState *tstate = _PyThreadState_GET(); GCState *gcstate = &tstate->interp->gc; - assert(tstate->current_frame == NULL || tstate->current_frame->stackpointer != NULL); - int expected = 0; - if (!_Py_atomic_compare_exchange_int(&gcstate->collecting, &expected, 1)) { - // Don't start a garbage collection if one is already in progress. + if (!gcstate->enabled) { return 0; } - struct gc_collection_stats stats = { 0 }; - if (reason != _Py_GC_REASON_SHUTDOWN) { - invoke_gc_callback(gcstate, "start", generation, &stats); - } - if (gcstate->debug & _PyGC_DEBUG_STATS) { - PySys_WriteStderr("gc: collecting generation %d...\n", generation); - show_stats_each_generations(gcstate); - } - if (PyDTrace_GC_START_ENABLED()) { - PyDTrace_GC_START(generation); - } + Py_ssize_t n; PyObject *exc = _PyErr_GetRaisedException(tstate); - switch(generation) { - case 0: - gc_collect_young(tstate, &stats); - break; - case 1: - gc_collect_increment(tstate, &stats); - break; - case 2: - gc_collect_full(tstate, &stats); - break; - default: - Py_UNREACHABLE(); - } - if (PyDTrace_GC_DONE_ENABLED()) { - PyDTrace_GC_DONE(stats.uncollectable + stats.collected); - } - if (reason != _Py_GC_REASON_SHUTDOWN) { - invoke_gc_callback(gcstate, "stop", generation, &stats); - } + n = gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_MANUAL); _PyErr_SetRaisedException(tstate, exc); - GC_STAT_ADD(generation, objects_collected, stats.collected); -#ifdef Py_STATS - if (_Py_stats) { - GC_STAT_ADD(generation, object_visits, - _Py_stats->object_stats.object_visits); - _Py_stats->object_stats.object_visits = 0; - } -#endif - validate_spaces(gcstate); - _Py_atomic_store_int(&gcstate->collecting, 0); - return stats.uncollectable + stats.collected; + + return n; } -/* Public API to invoke gc.collect() from C */ Py_ssize_t -PyGC_Collect(void) +_PyGC_Collect(PyThreadState *tstate, int generation, _PyGC_Reason reason) { - return _PyGC_Collect(_PyThreadState_GET(), 2, _Py_GC_REASON_MANUAL); + return gc_collect_main(tstate, generation, reason); } void @@ -2079,7 +1700,7 @@ _PyGC_CollectNoFail(PyThreadState *tstate) during interpreter shutdown (and then never finish it). See http://bugs.python.org/issue8713#msg195178 for an example. */ - _PyGC_Collect(_PyThreadState_GET(), 2, _Py_GC_REASON_SHUTDOWN); + gc_collect_main(tstate, NUM_GENERATIONS - 1, _Py_GC_REASON_SHUTDOWN); } void @@ -2154,9 +1775,9 @@ _PyGC_Fini(PyInterpreterState *interp) * This bug was originally fixed when reported as gh-90228. The bug was * re-introduced in gh-94673. */ - finalize_unlink_gc_head(&gcstate->young.head); - finalize_unlink_gc_head(&gcstate->old[0].head); - finalize_unlink_gc_head(&gcstate->old[1].head); + for (int i = 0; i < NUM_GENERATIONS; i++) { + finalize_unlink_gc_head(&gcstate->generations[i].head); + } finalize_unlink_gc_head(&gcstate->permanent_generation.head); } @@ -2241,11 +1862,10 @@ _PyObject_GC_Link(PyObject *op) GCState *gcstate = &tstate->interp->gc; gc->_gc_next = 0; gc->_gc_prev = 0; - gcstate->young.count++; /* number of allocated GC objects */ - gcstate->heap_size++; - if (gcstate->young.count > gcstate->young.threshold && + gcstate->generations[0].count++; /* number of allocated GC objects */ + if (gcstate->generations[0].count > gcstate->generations[0].threshold && gcstate->enabled && - gcstate->young.threshold && + gcstate->generations[0].threshold && !_Py_atomic_load_int_relaxed(&gcstate->collecting) && !_PyErr_Occurred(tstate)) { @@ -2256,9 +1876,11 @@ _PyObject_GC_Link(PyObject *op) void _Py_RunGC(PyThreadState *tstate) { - if (tstate->interp->gc.enabled) { - _PyGC_Collect(tstate, 1, _Py_GC_REASON_HEAP); + GCState *gcstate = get_gc_state(); + if (!gcstate->enabled) { + return; } + gc_collect_main(tstate, GENERATION_AUTO, _Py_GC_REASON_HEAP); } static PyObject * @@ -2359,6 +1981,8 @@ PyObject_GC_Del(void *op) PyGC_Head *g = AS_GC(op); if (_PyObject_GC_IS_TRACKED(op)) { gc_list_remove(g); + GCState *gcstate = get_gc_state(); + gcstate->heap_size--; #ifdef Py_DEBUG PyObject *exc = PyErr_GetRaisedException(); if (PyErr_WarnExplicitFormat(PyExc_ResourceWarning, "gc", 0, @@ -2373,10 +1997,9 @@ PyObject_GC_Del(void *op) #endif } GCState *gcstate = get_gc_state(); - if (gcstate->young.count > 0) { - gcstate->young.count--; + if (gcstate->generations[0].count > 0) { + gcstate->generations[0].count--; } - gcstate->heap_size--; PyObject_Free(((char *)op)-presize); } @@ -2421,18 +2044,14 @@ PyUnstable_GC_VisitObjects(gcvisitobjects_t callback, void *arg) GCState *gcstate = get_gc_state(); int original_state = gcstate->enabled; gcstate->enabled = 0; - if (visit_generation(callback, arg, &gcstate->young) < 0) { - goto done; - } - if (visit_generation(callback, arg, &gcstate->old[0]) < 0) { - goto done; - } - if (visit_generation(callback, arg, &gcstate->old[1]) < 0) { - goto done; + for (size_t i = 0; i < NUM_GENERATIONS; i++) { + if (visit_generation(callback, arg, &gcstate->generations[i]) < 0) { + goto done; + } } visit_generation(callback, arg, &gcstate->permanent_generation); done: gcstate->enabled = original_state; } -#endif // Py_GIL_DISABLED +#endif // !Py_GIL_DISABLED diff --git a/Python/gc_free_threading.c b/Python/gc_free_threading.c index 757e9cb3227e260..d1b8d282415337a 100644 --- a/Python/gc_free_threading.c +++ b/Python/gc_free_threading.c @@ -374,6 +374,19 @@ op_from_block(void *block, void *arg, bool include_frozen) return op; } +// As above but returns untracked and frozen objects as well. +static PyObject * +op_from_block_all_gc(void *block, void *arg) +{ + struct visitor_args *a = arg; + if (block == NULL) { + return NULL; + } + PyObject *op = (PyObject *)((char*)block + a->offset); + assert(PyObject_IS_GC(op)); + return op; +} + static int gc_visit_heaps_lock_held(PyInterpreterState *interp, mi_block_visit_fun *visitor, struct visitor_args *arg) @@ -675,10 +688,11 @@ gc_mark_span_push(gc_span_stack_t *ss, PyObject **start, PyObject **end) else { ss->capacity *= 2; } - ss->stack = (gc_span_t *)PyMem_Realloc(ss->stack, ss->capacity * sizeof(gc_span_t)); - if (ss->stack == NULL) { + gc_span_t *new_stack = (gc_span_t *)PyMem_Realloc(ss->stack, ss->capacity * sizeof(gc_span_t)); + if (new_stack == NULL) { return -1; } + ss->stack = new_stack; } assert(end > start); ss->stack[ss->size].start = start; @@ -891,7 +905,11 @@ gc_visit_thread_stacks_mark_alive(PyInterpreterState *interp, gc_mark_args_t *ar static void queue_untracked_obj_decref(PyObject *op, struct collection_state *state) { - if (!_PyObject_GC_IS_TRACKED(op)) { + assert(Py_REFCNT(op) == 0); + // gh-142975: We have to treat frozen objects as untracked in this function + // or else they might be picked up in a future collection, which breaks the + // assumption that all incoming objects have a non-zero reference count. + if (!_PyObject_GC_IS_TRACKED(op) || gc_is_frozen(op)) { // GC objects with zero refcount are handled subsequently by the // GC as if they were cyclic trash, but we have to handle dead // non-GC objects here. Add one to the refcount so that we can @@ -1174,12 +1192,20 @@ static bool scan_heap_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, void *block, size_t block_size, void *args) { - PyObject *op = op_from_block(block, args, false); + PyObject *op = op_from_block_all_gc(block, args); if (op == NULL) { return true; } - struct collection_state *state = (struct collection_state *)args; + // The free-threaded GC cost is proportional to the number of objects in + // the mimalloc GC heap and so we should include the counts for untracked + // and frozen objects as well. This is especially important if many + // tuples have been untracked. + state->long_lived_total++; + if (!_PyObject_GC_IS_TRACKED(op) || gc_is_frozen(op)) { + return true; + } + if (gc_is_unreachable(op)) { // Disable deferred refcounting for unreachable objects so that they // are collected immediately after finalization. @@ -1197,6 +1223,9 @@ scan_heap_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, else { worklist_push(&state->unreachable, op); } + // It is possible this object will be resurrected but + // for now we assume it will be deallocated. + state->long_lived_total--; return true; } @@ -1210,7 +1239,6 @@ scan_heap_visitor(const mi_heap_t *heap, const mi_heap_area_t *area, // object is reachable, restore `ob_tid`; we're done with these objects gc_restore_tid(op); gc_clear_alive(op); - state->long_lived_total++; return true; } @@ -1492,9 +1520,9 @@ move_legacy_finalizer_reachable(struct collection_state *state) } // Clear all weakrefs to unreachable objects. Weakrefs with callbacks are -// enqueued in `wrcb_to_call`, but not invoked yet. +// optionally enqueued in `wrcb_to_call`, but not invoked yet. static void -clear_weakrefs(struct collection_state *state) +clear_weakrefs(struct collection_state *state, bool enqueue_callbacks) { PyObject *op; WORKSTACK_FOR_EACH(&state->unreachable, op) { @@ -1526,6 +1554,10 @@ clear_weakrefs(struct collection_state *state) _PyWeakref_ClearRef(wr); _PyObject_ASSERT((PyObject *)wr, wr->wr_object == Py_None); + if (!enqueue_callbacks) { + continue; + } + // We do not invoke callbacks for weakrefs that are themselves // unreachable. This is partly for historical reasons: weakrefs // predate safe object finalization, and a weakref that is itself @@ -1817,6 +1849,7 @@ handle_resurrected_objects(struct collection_state *state) _PyObject_ASSERT(op, Py_REFCNT(op) > 1); worklist_remove(&iter); merge_refcount(op, -1); // remove worklist reference + state->long_lived_total++; } } } @@ -1927,8 +1960,7 @@ get_process_mem_usage(void) } #elif __linux__ - // Linux, use smaps_rollup (Kernel >= 4.4) for RSS + Swap - FILE* fp = fopen("/proc/self/smaps_rollup", "r"); + FILE* fp = fopen("/proc/self/status", "r"); if (fp == NULL) { return -1; } @@ -1938,11 +1970,11 @@ get_process_mem_usage(void) long long swap_kb = -1; while (fgets(line_buffer, sizeof(line_buffer), fp) != NULL) { - if (rss_kb == -1 && strncmp(line_buffer, "Rss:", 4) == 0) { - sscanf(line_buffer + 4, "%lld", &rss_kb); + if (rss_kb == -1 && strncmp(line_buffer, "VmRSS:", 6) == 0) { + sscanf(line_buffer + 6, "%lld", &rss_kb); } - else if (swap_kb == -1 && strncmp(line_buffer, "Swap:", 5) == 0) { - sscanf(line_buffer + 5, "%lld", &swap_kb); + else if (swap_kb == -1 && strncmp(line_buffer, "VmSwap:", 7) == 0) { + sscanf(line_buffer + 7, "%lld", &swap_kb); } if (rss_kb != -1 && swap_kb != -1) { break; // Found both @@ -2063,7 +2095,7 @@ gc_should_collect_mem_usage(GCState *gcstate) // 70,000 new container objects. return true; } - Py_ssize_t last_mem = gcstate->last_mem; + Py_ssize_t last_mem = _Py_atomic_load_ssize_relaxed(&gcstate->last_mem); Py_ssize_t mem_threshold = Py_MAX(last_mem / 10, 128); if ((mem - last_mem) > mem_threshold) { // The process memory usage has increased too much, do a collection. @@ -2134,7 +2166,19 @@ record_deallocation(PyThreadState *tstate) gc->alloc_count--; if (gc->alloc_count <= -LOCAL_ALLOC_COUNT_THRESHOLD) { GCState *gcstate = &tstate->interp->gc; - _Py_atomic_add_int(&gcstate->young.count, (int)gc->alloc_count); + int count = _Py_atomic_load_int_relaxed(&gcstate->young.count); + int new_count; + do { + if (count == 0) { + break; + } + new_count = count + (int)gc->alloc_count; + if (new_count < 0) { + new_count = 0; + } + } while (!_Py_atomic_compare_exchange_int(&gcstate->young.count, + &count, + new_count)); gc->alloc_count = 0; } } @@ -2208,11 +2252,8 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, } } - // Record the number of live GC objects - interp->gc.long_lived_total = state->long_lived_total; - // Clear weakrefs and enqueue callbacks (but do not call them). - clear_weakrefs(state); + clear_weakrefs(state, true); _PyEval_StartTheWorld(interp); // Deallocate any object from the refcount merge step @@ -2223,11 +2264,21 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, call_weakref_callbacks(state); finalize_garbage(state); - // Handle any objects that may have resurrected after the finalization. _PyEval_StopTheWorld(interp); + // Handle any objects that may have resurrected after the finalization. err = handle_resurrected_objects(state); // Clear free lists in all threads _PyGC_ClearAllFreeLists(interp); + if (err == 0) { + // Clear weakrefs to objects in the unreachable set. No Python-level + // code must be allowed to access those unreachable objects. During + // delete_garbage(), finalizers outside the unreachable set might + // run and create new weakrefs. If those weakrefs were not cleared, + // they could reveal unreachable objects. + clear_weakrefs(state, false); + } + // Record the number of live GC objects + interp->gc.long_lived_total = state->long_lived_total; _PyEval_StartTheWorld(interp); if (err < 0) { @@ -2246,7 +2297,8 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state, // Store the current memory usage, can be smaller now if breaking cycles // freed some memory. - state->gcstate->last_mem = get_process_mem_usage(); + Py_ssize_t last_mem = get_process_mem_usage(); + _Py_atomic_store_ssize_relaxed(&state->gcstate->last_mem, last_mem); // Append objects with legacy finalizers to the "gc.garbage" list. handle_legacy_finalizers(state); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 7e8b05b747ed8f0..9e825c529c9dd2d 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -315,6 +315,9 @@ stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -2; assert(WITHIN_STACK_BOUNDS()); + if (res_o == NULL) { + JUMP_TO_LABEL(error); + } res = PyStackRef_FromPyObjectSteal(res_o); } stack_pointer[0] = res; @@ -389,14 +392,13 @@ assert(WITHIN_STACK_BOUNDS()); _PyFrame_SetStackPointer(frame, stack_pointer); PyUnicode_Append(&temp, right_o); - stack_pointer = _PyFrame_GetStackPointer(frame); - *target_local = PyStackRef_FromPyObjectSteal(temp); - _PyFrame_SetStackPointer(frame, stack_pointer); Py_DECREF(right_o); stack_pointer = _PyFrame_GetStackPointer(frame); - if (PyStackRef_IsNull(*target_local)) { + if (temp == NULL) { + *target_local = PyStackRef_NULL; JUMP_TO_LABEL(error); } + *target_local = PyStackRef_FromPyObjectSteal(temp); #if TIER_ONE assert(next_instr->op.code == STORE_FAST); @@ -653,7 +655,7 @@ new_frame = _PyFrame_PushUnchecked(tstate, getitem, 2, frame); new_frame->localsplus[0] = container; new_frame->localsplus[1] = sub; - frame->return_offset = 6 ; + frame->return_offset = 6u ; } // _PUSH_FRAME { @@ -1586,7 +1588,7 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } - frame->return_offset = 4 ; + frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -2641,7 +2643,7 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } - assert( 1 == 1); + assert( 1u == 1); frame->return_offset = 1; DISPATCH_INLINED(new_frame); } @@ -2922,8 +2924,8 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } - assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); - frame->return_offset = 4 ; + assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -3324,6 +3326,14 @@ JUMP_TO_PREDICTED(CALL_KW); } } + // _CHECK_RECURSION_REMAINING + { + if (tstate->py_recursion_remaining <= 1) { + UPDATE_MISS_STATS(CALL_KW); + assert(_PyOpcode_Deopt[opcode] == (CALL_KW)); + JUMP_TO_PREDICTED(CALL_KW); + } + } // _PY_FRAME_KW { kwnames = stack_pointer[-1]; @@ -5724,8 +5734,17 @@ // _FOR_ITER { PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + iternextfunc func = Py_TYPE(iter_o)->tp_iternext; + if (func == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'%.100s' object is not an iterator", + Py_TYPE(iter_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + PyObject *next_o = func(iter_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (next_o == NULL) { if (_PyErr_Occurred(tstate)) { @@ -5804,7 +5823,7 @@ gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; gen_frame->previous = frame; - frame->return_offset = (uint16_t)( 2 + oparg); + frame->return_offset = (uint16_t)( 2u + oparg); } // _PUSH_FRAME { @@ -6443,7 +6462,7 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } - frame->return_offset = 4 ; + frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -6664,7 +6683,7 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } - assert( 1 == 1); + assert( 1u == 1); frame->return_offset = 1; DISPATCH_INLINED(new_frame); } @@ -6815,8 +6834,8 @@ if (new_frame == NULL) { JUMP_TO_LABEL(error); } - assert( 4 == 1 + INLINE_CACHE_ENTRIES_CALL_KW); - frame->return_offset = 4 ; + assert( 4u == 1 + INLINE_CACHE_ENTRIES_CALL_KW); + frame->return_offset = 4u ; DISPATCH_INLINED(new_frame); } STACKREFS_TO_PYOBJECTS(arguments, total_args, args_o); @@ -7037,8 +7056,17 @@ /* Skip 1 cache entry */ iter = stack_pointer[-1]; PyObject *iter_o = PyStackRef_AsPyObjectBorrow(iter); + iternextfunc func = Py_TYPE(iter_o)->tp_iternext; + if (func == NULL) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyErr_Format(tstate, PyExc_TypeError, + "'%.100s' object is not an iterator", + Py_TYPE(iter_o)->tp_name); + stack_pointer = _PyFrame_GetStackPointer(frame); + JUMP_TO_LABEL(error); + } _PyFrame_SetStackPointer(frame, stack_pointer); - PyObject *next_o = (*Py_TYPE(iter_o)->tp_iternext)(iter_o); + PyObject *next_o = func(iter_o); stack_pointer = _PyFrame_GetStackPointer(frame); if (next_o != NULL) { next = PyStackRef_FromPyObjectSteal(next_o); @@ -7478,7 +7506,13 @@ } // _MAYBE_INSTRUMENT { - if (tstate->tracing == 0) { + #ifdef Py_GIL_DISABLED + + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { @@ -7583,6 +7617,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(INSTRUMENTED_YIELD_VALUE); + opcode = INSTRUMENTED_YIELD_VALUE; _PyStackRef val; _PyStackRef retval; _PyStackRef value; @@ -7622,13 +7657,12 @@ frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #if TIER_ONE && defined(Py_DEBUG) + if (!PyStackRef_IsNone(frame->f_executable)) { + int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + assert(opcode == SEND || opcode == FOR_ITER); + } #endif stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); @@ -7980,32 +8014,17 @@ { self_or_null = &stack_pointer[0]; PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - PyObject *attr_o; if (oparg & 1) { - attr_o = NULL; _PyFrame_SetStackPointer(frame, stack_pointer); - int is_meth = _PyObject_GetMethod(PyStackRef_AsPyObjectBorrow(owner), name, &attr_o); + attr = _Py_LoadAttr_StackRefSteal(tstate, owner, name, self_or_null); stack_pointer = _PyFrame_GetStackPointer(frame); - if (is_meth) { - assert(attr_o != NULL); - self_or_null[0] = owner; - } - else { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - _PyFrame_SetStackPointer(frame, stack_pointer); - PyStackRef_CLOSE(owner); - stack_pointer = _PyFrame_GetStackPointer(frame); - if (attr_o == NULL) { - JUMP_TO_LABEL(error); - } - self_or_null[0] = PyStackRef_NULL; - stack_pointer += 1; + if (PyStackRef_IsNull(attr)) { + JUMP_TO_LABEL(pop_1_error); } } else { _PyFrame_SetStackPointer(frame, stack_pointer); - attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); + PyObject *attr_o = PyObject_GetAttr(PyStackRef_AsPyObjectBorrow(owner), name); stack_pointer = _PyFrame_GetStackPointer(frame); stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -8015,9 +8034,9 @@ if (attr_o == NULL) { JUMP_TO_LABEL(error); } + attr = PyStackRef_FromPyObjectSteal(attr_o); stack_pointer += 1; } - attr = PyStackRef_FromPyObjectSteal(attr_o); } stack_pointer[-1] = attr; stack_pointer += (oparg&1); @@ -8205,7 +8224,7 @@ stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); new_frame->localsplus[1] = PyStackRef_FromPyObjectNew(name); - frame->return_offset = 10 ; + frame->return_offset = 10u ; DISPATCH_INLINED(new_frame); } @@ -10454,7 +10473,13 @@ } // _MAYBE_INSTRUMENT { - if (tstate->tracing == 0) { + #ifdef Py_GIL_DISABLED + + int check_instrumentation = 1; + #else + int check_instrumentation = (tstate->tracing == 0); + #endif + if (check_instrumentation) { uintptr_t global_version = _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) & ~_PY_EVAL_EVENTS_MASK; uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(_PyFrame_GetCode(frame)->_co_instrumentation_version); if (code_version != global_version) { @@ -10652,8 +10677,8 @@ gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - assert( 2 + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2 + oparg); + assert( 2u + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2u + oparg); assert(gen_frame->previous == NULL); gen_frame->previous = frame; DISPATCH_INLINED(gen_frame); @@ -10753,8 +10778,8 @@ gen->gi_frame_state = FRAME_EXECUTING; gen->gi_exc_state.previous_item = tstate->exc_info; tstate->exc_info = &gen->gi_exc_state; - assert( 2 + oparg <= UINT16_MAX); - frame->return_offset = (uint16_t)( 2 + oparg); + assert( 2u + oparg <= UINT16_MAX); + frame->return_offset = (uint16_t)( 2u + oparg); gen_frame->previous = frame; } // _PUSH_FRAME @@ -10868,6 +10893,22 @@ PyObject **ptr = (PyObject **)(((char *)func) + offset); assert(*ptr == NULL); *ptr = attr; + if (oparg == MAKE_FUNCTION_ANNOTATE && PyFunction_Check(attr)) { + PyFunctionObject *func_obj = (PyFunctionObject *)attr; + stack_pointer[-2] = func_out; + stack_pointer += -1; + assert(WITHIN_STACK_BOUNDS()); + _PyFrame_SetStackPointer(frame, stack_pointer); + PyObject *fixed_qualname = PyUnicode_FromFormat("%U.__annotate__", ((PyFunctionObject *)func)->func_qualname); + stack_pointer = _PyFrame_GetStackPointer(frame); + if (fixed_qualname == NULL) { + JUMP_TO_LABEL(error); + } + _PyFrame_SetStackPointer(frame, stack_pointer); + Py_SETREF(func_obj->func_qualname, fixed_qualname); + stack_pointer = _PyFrame_GetStackPointer(frame); + stack_pointer += 1; + } stack_pointer[-2] = func_out; stack_pointer += -1; assert(WITHIN_STACK_BOUNDS()); @@ -12279,6 +12320,7 @@ frame->instr_ptr = next_instr; next_instr += 1; INSTRUCTION_STATS(YIELD_VALUE); + opcode = YIELD_VALUE; _PyStackRef retval; _PyStackRef value; retval = stack_pointer[-1]; @@ -12299,13 +12341,12 @@ frame = tstate->current_frame = frame->previous; gen_frame->previous = NULL; assert(INLINE_CACHE_ENTRIES_SEND == INLINE_CACHE_ENTRIES_FOR_ITER); - #if TIER_ONE - assert(frame->instr_ptr->op.code == INSTRUMENTED_LINE || - frame->instr_ptr->op.code == INSTRUMENTED_INSTRUCTION || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == SEND || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == FOR_ITER || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == INTERPRETER_EXIT || - _PyOpcode_Deopt[frame->instr_ptr->op.code] == ENTER_EXECUTOR); + #if TIER_ONE && defined(Py_DEBUG) + if (!PyStackRef_IsNone(frame->f_executable)) { + int i = frame->instr_ptr - _PyFrame_GetBytecode(frame); + int opcode = _Py_GetBaseCodeUnit(_PyFrame_GetCode(frame), i).op.code; + assert(opcode == SEND || opcode == FOR_ITER); + } #endif stack_pointer = _PyFrame_GetStackPointer(frame); LOAD_IP(1 + INLINE_CACHE_ENTRIES_SEND); diff --git a/Python/getargs.c b/Python/getargs.c index 0cf596285cc7b58..43ceb2bf8359715 100644 --- a/Python/getargs.c +++ b/Python/getargs.c @@ -1,8 +1,6 @@ /* New getargs implementation */ -#include <stdbool.h> - #define PY_CXX_CONST const #include "Python.h" #include "pycore_abstract.h" // _PyNumber_Index() @@ -468,12 +466,9 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *format = *p_format; int i; Py_ssize_t len; - bool nullable = false; int istuple = PyTuple_Check(arg); int mustbetuple = istuple; - assert(*format == '('); - format++; for (;;) { int c = *format++; if (c == '(') { @@ -482,12 +477,8 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, level++; } else if (c == ')') { - if (level == 0) { - if (*format == '?') { - nullable = true; - } + if (level == 0) break; - } level--; } else if (c == ':' || c == ';' || c == '\0') @@ -524,13 +515,6 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } } - if (arg == Py_None && nullable) { - const char *msg = skipitem(p_format, p_va, flags); - if (msg != NULL) { - levels[0] = 0; - } - return msg; - } if (istuple) { /* fallthrough */ } @@ -539,10 +523,9 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, { levels[0] = 0; PyOS_snprintf(msgbuf, bufsize, - "must be %d-item tuple%s, not %.50s", - n, - nullable ? " or None" : "", - arg == Py_None ? "None" : Py_TYPE(arg)->tp_name); + "must be %d-item tuple, not %.50s", + n, + arg == Py_None ? "None" : Py_TYPE(arg)->tp_name); return msgbuf; } else { @@ -579,7 +562,7 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, return msgbuf; } - format = *p_format + 1; + format = *p_format; for (i = 0; i < n; i++) { const char *msg; PyObject *item = PyTuple_GET_ITEM(arg, i); @@ -594,10 +577,6 @@ converttuple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } } - format++; - if (*format == '?') { - format++; - } *p_format = format; if (!istuple) { Py_DECREF(arg); @@ -616,8 +595,11 @@ convertitem(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *format = *p_format; if (*format == '(' /* ')' */) { + format++; msg = converttuple(arg, &format, p_va, flags, levels, msgbuf, bufsize, freelist); + if (msg == NULL) + format++; } else { msg = convertsimple(arg, &format, p_va, flags, @@ -647,7 +629,7 @@ _PyArg_BadArgument(const char *fname, const char *displayname, } static const char * -converterr(bool nullable, const char *expected, PyObject *arg, char *msgbuf, size_t bufsize) +converterr(const char *expected, PyObject *arg, char *msgbuf, size_t bufsize) { assert(expected != NULL); assert(arg != NULL); @@ -657,23 +639,20 @@ converterr(bool nullable, const char *expected, PyObject *arg, char *msgbuf, siz } else { PyOS_snprintf(msgbuf, bufsize, - "must be %.50s%s, not %.50s", expected, - nullable ? " or None" : "", + "must be %.50s, not %.50s", expected, arg == Py_None ? "None" : Py_TYPE(arg)->tp_name); } return msgbuf; } static const char * -convertcharerr(bool nullable, const char *expected, const char *what, Py_ssize_t size, +convertcharerr(const char *expected, const char *what, Py_ssize_t size, char *msgbuf, size_t bufsize) { assert(expected != NULL); PyOS_snprintf(msgbuf, bufsize, - "must be %.50s%s, not %.50s of length %zd", - expected, - nullable ? " or None" : "", - what, size); + "must be %.50s, not %.50s of length %zd", + expected, what, size); return msgbuf; } @@ -693,26 +672,15 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, char *msgbuf, size_t bufsize, freelist_t *freelist) { #define RETURN_ERR_OCCURRED return msgbuf -#define HANDLE_NULLABLE \ - if (*format == '?') { \ - format++; \ - if (arg == Py_None) { \ - break; \ - } \ - nullable = true; \ - } - const char *format = *p_format; char c = *format++; const char *sarg; - bool nullable = false; switch (c) { case 'b': { /* unsigned byte -- very short int */ unsigned char *p = va_arg(*p_va, unsigned char *); - HANDLE_NULLABLE; long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -726,6 +694,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, "unsigned byte integer is greater than maximum"); RETURN_ERR_OCCURRED; } + else *p = (unsigned char) ival; break; } @@ -733,7 +702,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'B': {/* byte sized bitfield - both signed and unsigned values allowed */ unsigned char *p = va_arg(*p_va, unsigned char *); - HANDLE_NULLABLE; unsigned long ival = PyLong_AsUnsignedLongMask(arg); if (ival == (unsigned long)-1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -744,7 +712,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'h': {/* signed short int */ short *p = va_arg(*p_va, short *); - HANDLE_NULLABLE; long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -766,7 +733,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'H': { /* short int sized bitfield, both signed and unsigned allowed */ unsigned short *p = va_arg(*p_va, unsigned short *); - HANDLE_NULLABLE; unsigned long ival = PyLong_AsUnsignedLongMask(arg); if (ival == (unsigned long)-1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -777,7 +743,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'i': {/* signed int */ int *p = va_arg(*p_va, int *); - HANDLE_NULLABLE; long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -799,7 +764,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'I': { /* int sized bitfield, both signed and unsigned allowed */ unsigned int *p = va_arg(*p_va, unsigned int *); - HANDLE_NULLABLE; unsigned long ival = PyLong_AsUnsignedLongMask(arg); if (ival == (unsigned long)-1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -812,7 +776,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, { PyObject *iobj; Py_ssize_t *p = va_arg(*p_va, Py_ssize_t *); - HANDLE_NULLABLE; Py_ssize_t ival = -1; iobj = _PyNumber_Index(arg); if (iobj != NULL) { @@ -826,7 +789,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } case 'l': {/* long int */ long *p = va_arg(*p_va, long *); - HANDLE_NULLABLE; long ival = PyLong_AsLong(arg); if (ival == -1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -837,10 +799,9 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'k': { /* long sized bitfield */ unsigned long *p = va_arg(*p_va, unsigned long *); - HANDLE_NULLABLE; unsigned long ival; if (!PyIndex_Check(arg)) { - return converterr(nullable, "int", arg, msgbuf, bufsize); + return converterr("int", arg, msgbuf, bufsize); } ival = PyLong_AsUnsignedLongMask(arg); if (ival == (unsigned long)(long)-1 && PyErr_Occurred()) { @@ -852,7 +813,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'L': {/* long long */ long long *p = va_arg( *p_va, long long * ); - HANDLE_NULLABLE; long long ival = PyLong_AsLongLong(arg); if (ival == (long long)-1 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -863,10 +823,9 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'K': { /* long long sized bitfield */ unsigned long long *p = va_arg(*p_va, unsigned long long *); - HANDLE_NULLABLE; unsigned long long ival; if (!PyIndex_Check(arg)) { - return converterr(nullable, "int", arg, msgbuf, bufsize); + return converterr("int", arg, msgbuf, bufsize); } ival = PyLong_AsUnsignedLongLongMask(arg); if (ival == (unsigned long long)(long long)-1 && PyErr_Occurred()) { @@ -878,7 +837,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'f': {/* float */ float *p = va_arg(*p_va, float *); - HANDLE_NULLABLE; double dval = PyFloat_AsDouble(arg); if (dval == -1.0 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -889,7 +847,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'd': {/* double */ double *p = va_arg(*p_va, double *); - HANDLE_NULLABLE; double dval = PyFloat_AsDouble(arg); if (dval == -1.0 && PyErr_Occurred()) RETURN_ERR_OCCURRED; @@ -900,7 +857,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'D': {/* complex double */ Py_complex *p = va_arg(*p_va, Py_complex *); - HANDLE_NULLABLE; Py_complex cval; cval = PyComplex_AsCComplex(arg); if (PyErr_Occurred()) @@ -912,10 +868,9 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'c': {/* char */ char *p = va_arg(*p_va, char *); - HANDLE_NULLABLE; if (PyBytes_Check(arg)) { if (PyBytes_GET_SIZE(arg) != 1) { - return convertcharerr(nullable, "a byte string of length 1", + return convertcharerr("a byte string of length 1", "a bytes object", PyBytes_GET_SIZE(arg), msgbuf, bufsize); } @@ -923,28 +878,27 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } else if (PyByteArray_Check(arg)) { if (PyByteArray_GET_SIZE(arg) != 1) { - return convertcharerr(nullable, "a byte string of length 1", + return convertcharerr("a byte string of length 1", "a bytearray object", PyByteArray_GET_SIZE(arg), msgbuf, bufsize); } *p = PyByteArray_AS_STRING(arg)[0]; } else - return converterr(nullable, "a byte string of length 1", arg, msgbuf, bufsize); + return converterr("a byte string of length 1", arg, msgbuf, bufsize); break; } case 'C': {/* unicode char */ int *p = va_arg(*p_va, int *); - HANDLE_NULLABLE; int kind; const void *data; if (!PyUnicode_Check(arg)) - return converterr(nullable, "a unicode character", arg, msgbuf, bufsize); + return converterr("a unicode character", arg, msgbuf, bufsize); if (PyUnicode_GET_LENGTH(arg) != 1) { - return convertcharerr(nullable, "a unicode character", + return convertcharerr("a unicode character", "a string", PyUnicode_GET_LENGTH(arg), msgbuf, bufsize); } @@ -957,7 +911,6 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'p': {/* boolean *p*redicate */ int *p = va_arg(*p_va, int *); - HANDLE_NULLABLE; int val = PyObject_IsTrue(arg); if (val > 0) *p = 1; @@ -976,31 +929,24 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *buf; Py_ssize_t count; if (*format == '*') { - format++; - HANDLE_NULLABLE; if (getbuffer(arg, (Py_buffer*)p, &buf) < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); + return converterr(buf, arg, msgbuf, bufsize); + format++; if (addcleanup(p, freelist, cleanup_buffer)) { return converterr( - nullable, "(cleanup problem)", + "(cleanup problem)", arg, msgbuf, bufsize); } break; } - else if (*format == '#') { + count = convertbuffer(arg, (const void **)p, &buf); + if (count < 0) + return converterr(buf, arg, msgbuf, bufsize); + if (*format == '#') { Py_ssize_t *psize = va_arg(*p_va, Py_ssize_t*); - format++; - HANDLE_NULLABLE; - count = convertbuffer(arg, (const void **)p, &buf); - if (count < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); *psize = count; - } - else { - HANDLE_NULLABLE; - count = convertbuffer(arg, (const void **)p, &buf); - if (count < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); + format++; + } else { if (strlen(*p) != (size_t)count) { PyErr_SetString(PyExc_ValueError, "embedded null byte"); RETURN_ERR_OCCURRED; @@ -1016,35 +962,32 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, /* "s*" or "z*" */ Py_buffer *p = (Py_buffer *)va_arg(*p_va, Py_buffer *); - format++; - HANDLE_NULLABLE; if (c == 'z' && arg == Py_None) PyBuffer_FillInfo(p, NULL, NULL, 0, 1, 0); else if (PyUnicode_Check(arg)) { Py_ssize_t len; sarg = PyUnicode_AsUTF8AndSize(arg, &len); if (sarg == NULL) - return converterr(nullable, CONV_UNICODE, + return converterr(CONV_UNICODE, arg, msgbuf, bufsize); PyBuffer_FillInfo(p, arg, (void *)sarg, len, 1, 0); } else { /* any bytes-like object */ const char *buf; if (getbuffer(arg, p, &buf) < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); + return converterr(buf, arg, msgbuf, bufsize); } if (addcleanup(p, freelist, cleanup_buffer)) { return converterr( - nullable, "(cleanup problem)", + "(cleanup problem)", arg, msgbuf, bufsize); } + format++; } else if (*format == '#') { /* a string or read-only bytes-like object */ /* "s#" or "z#" */ const void **p = (const void **)va_arg(*p_va, const char **); Py_ssize_t *psize = va_arg(*p_va, Py_ssize_t*); - format++; - HANDLE_NULLABLE; if (c == 'z' && arg == Py_None) { *p = NULL; *psize = 0; @@ -1053,7 +996,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, Py_ssize_t len; sarg = PyUnicode_AsUTF8AndSize(arg, &len); if (sarg == NULL) - return converterr(nullable, CONV_UNICODE, + return converterr(CONV_UNICODE, arg, msgbuf, bufsize); *p = sarg; *psize = len; @@ -1063,22 +1006,22 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, const char *buf; Py_ssize_t count = convertbuffer(arg, p, &buf); if (count < 0) - return converterr(nullable, buf, arg, msgbuf, bufsize); + return converterr(buf, arg, msgbuf, bufsize); *psize = count; } + format++; } else { /* "s" or "z" */ const char **p = va_arg(*p_va, const char **); Py_ssize_t len; sarg = NULL; - HANDLE_NULLABLE; if (c == 'z' && arg == Py_None) *p = NULL; else if (PyUnicode_Check(arg)) { sarg = PyUnicode_AsUTF8AndSize(arg, &len); if (sarg == NULL) - return converterr(nullable, CONV_UNICODE, + return converterr(CONV_UNICODE, arg, msgbuf, bufsize); if (strlen(sarg) != (size_t)len) { PyErr_SetString(PyExc_ValueError, "embedded null character"); @@ -1087,7 +1030,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, *p = sarg; } else - return converterr(c == 'z' || nullable, "str", + return converterr(c == 'z' ? "str or None" : "str", arg, msgbuf, bufsize); } break; @@ -1116,46 +1059,13 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, recode_strings = 0; else return converterr( - nullable, "(unknown parser marker combination)", + "(unknown parser marker combination)", arg, msgbuf, bufsize); buffer = (char **)va_arg(*p_va, char **); format++; if (buffer == NULL) - return converterr(nullable, "(buffer is NULL)", + return converterr("(buffer is NULL)", arg, msgbuf, bufsize); - Py_ssize_t *psize = NULL; - if (*format == '#') { - /* Using buffer length parameter '#': - - - if *buffer is NULL, a new buffer of the - needed size is allocated and the data - copied into it; *buffer is updated to point - to the new buffer; the caller is - responsible for PyMem_Free()ing it after - usage - - - if *buffer is not NULL, the data is - copied to *buffer; *buffer_len has to be - set to the size of the buffer on input; - buffer overflow is signalled with an error; - buffer has to provide enough room for the - encoded string plus the trailing 0-byte - - - in both cases, *buffer_len is updated to - the size of the buffer /excluding/ the - trailing 0-byte - - */ - psize = va_arg(*p_va, Py_ssize_t*); - - format++; - if (psize == NULL) { - return converterr( - nullable, "(buffer_len is NULL)", - arg, msgbuf, bufsize); - } - } - HANDLE_NULLABLE; /* Encode object */ if (!recode_strings && @@ -1176,7 +1086,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, encoding, NULL); if (s == NULL) - return converterr(nullable, "(encoding failed)", + return converterr("(encoding failed)", arg, msgbuf, bufsize); assert(PyBytes_Check(s)); size = PyBytes_GET_SIZE(s); @@ -1186,15 +1096,42 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } else { return converterr( - nullable, - recode_strings ? "str" - : nullable ? "str, bytes, bytearray" - : "str, bytes or bytearray", + recode_strings ? "str" : "str, bytes or bytearray", arg, msgbuf, bufsize); } /* Write output; output is guaranteed to be 0-terminated */ - if (psize != NULL) { + if (*format == '#') { + /* Using buffer length parameter '#': + + - if *buffer is NULL, a new buffer of the + needed size is allocated and the data + copied into it; *buffer is updated to point + to the new buffer; the caller is + responsible for PyMem_Free()ing it after + usage + + - if *buffer is not NULL, the data is + copied to *buffer; *buffer_len has to be + set to the size of the buffer on input; + buffer overflow is signalled with an error; + buffer has to provide enough room for the + encoded string plus the trailing 0-byte + + - in both cases, *buffer_len is updated to + the size of the buffer /excluding/ the + trailing 0-byte + + */ + Py_ssize_t *psize = va_arg(*p_va, Py_ssize_t*); + + format++; + if (psize == NULL) { + Py_DECREF(s); + return converterr( + "(buffer_len is NULL)", + arg, msgbuf, bufsize); + } if (*buffer == NULL) { *buffer = PyMem_NEW(char, size + 1); if (*buffer == NULL) { @@ -1205,7 +1142,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, if (addcleanup(buffer, freelist, cleanup_ptr)) { Py_DECREF(s); return converterr( - nullable, "(cleanup problem)", + "(cleanup problem)", arg, msgbuf, bufsize); } } else { @@ -1239,7 +1176,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, if ((Py_ssize_t)strlen(ptr) != size) { Py_DECREF(s); return converterr( - nullable, "encoded string without null bytes", + "encoded string without null bytes", arg, msgbuf, bufsize); } *buffer = PyMem_NEW(char, size + 1); @@ -1250,7 +1187,7 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, } if (addcleanup(buffer, freelist, cleanup_ptr)) { Py_DECREF(s); - return converterr(nullable, "(cleanup problem)", + return converterr("(cleanup problem)", arg, msgbuf, bufsize); } memcpy(*buffer, ptr, size+1); @@ -1261,32 +1198,29 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, case 'S': { /* PyBytes object */ PyObject **p = va_arg(*p_va, PyObject **); - HANDLE_NULLABLE; if (PyBytes_Check(arg)) *p = arg; else - return converterr(nullable, "bytes", arg, msgbuf, bufsize); + return converterr("bytes", arg, msgbuf, bufsize); break; } case 'Y': { /* PyByteArray object */ PyObject **p = va_arg(*p_va, PyObject **); - HANDLE_NULLABLE; if (PyByteArray_Check(arg)) *p = arg; else - return converterr(nullable, "bytearray", arg, msgbuf, bufsize); + return converterr("bytearray", arg, msgbuf, bufsize); break; } case 'U': { /* PyUnicode object */ PyObject **p = va_arg(*p_va, PyObject **); - HANDLE_NULLABLE; if (PyUnicode_Check(arg)) { *p = arg; } else - return converterr(nullable, "str", arg, msgbuf, bufsize); + return converterr("str", arg, msgbuf, bufsize); break; } @@ -1297,11 +1231,10 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, type = va_arg(*p_va, PyTypeObject*); p = va_arg(*p_va, PyObject **); format++; - HANDLE_NULLABLE; if (PyType_IsSubtype(Py_TYPE(arg), type)) *p = arg; else - return converterr(nullable, type->tp_name, arg, msgbuf, bufsize); + return converterr(type->tp_name, arg, msgbuf, bufsize); } else if (*format == '&') { @@ -1310,18 +1243,16 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, void *addr = va_arg(*p_va, void *); int res; format++; - HANDLE_NULLABLE; if (! (res = (*convert)(arg, addr))) - return converterr(nullable, "(unspecified)", + return converterr("(unspecified)", arg, msgbuf, bufsize); if (res == Py_CLEANUP_SUPPORTED && addcleanup(addr, freelist, convert) == -1) - return converterr(nullable, "(cleanup problem)", + return converterr("(cleanup problem)", arg, msgbuf, bufsize); } else { p = va_arg(*p_va, PyObject **); - HANDLE_NULLABLE; *p = arg; } break; @@ -1333,30 +1264,29 @@ convertsimple(PyObject *arg, const char **p_format, va_list *p_va, int flags, if (*format != '*') return converterr( - nullable, "(invalid use of 'w' format character)", + "(invalid use of 'w' format character)", arg, msgbuf, bufsize); format++; - HANDLE_NULLABLE; /* Caller is interested in Py_buffer, and the object supports it directly. The request implicitly asks for PyBUF_SIMPLE, so the result is C-contiguous with format 'B'. */ if (PyObject_GetBuffer(arg, (Py_buffer*)p, PyBUF_WRITABLE) < 0) { PyErr_Clear(); - return converterr(nullable, "read-write bytes-like object", + return converterr("read-write bytes-like object", arg, msgbuf, bufsize); } assert(PyBuffer_IsContiguous((Py_buffer *)p, 'C')); if (addcleanup(p, freelist, cleanup_buffer)) { return converterr( - nullable, "(cleanup problem)", + "(cleanup problem)", arg, msgbuf, bufsize); } break; } default: - return converterr(nullable, "(impossible<bad format char>)", arg, msgbuf, bufsize); + return converterr("(impossible<bad format char>)", arg, msgbuf, bufsize); } @@ -2348,7 +2278,7 @@ vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, if (i < parser->min) { /* Less arguments than required */ if (i < pos) { - Py_ssize_t min = Py_MIN(pos, parser->min); + int min = Py_MIN(pos, parser->min); PyErr_Format(PyExc_TypeError, "%.200s%s takes %s %d positional argument%s" " (%zd given)", @@ -2362,7 +2292,7 @@ vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, else { keyword = PyTuple_GET_ITEM(kwtuple, i - pos); PyErr_Format(PyExc_TypeError, "%.200s%s missing required " - "argument '%U' (pos %d)", + "argument '%U' (pos %zd)", (parser->fname == NULL) ? "function" : parser->fname, (parser->fname == NULL) ? "" : "()", keyword, i+1); @@ -2403,7 +2333,7 @@ vgetargskeywordsfast_impl(PyObject *const *args, Py_ssize_t nargs, /* arg present in tuple and in dict */ PyErr_Format(PyExc_TypeError, "argument for %.200s%s given by name ('%U') " - "and position (%d)", + "and position (%zd)", (parser->fname == NULL) ? "function" : parser->fname, (parser->fname == NULL) ? "" : "()", keyword, i+1); @@ -2751,9 +2681,6 @@ skipitem(const char **p_format, va_list *p_va, int flags) return "impossible<bad format char>"; } - if (*format == '?') { - format++; - } *p_format = format; return NULL; diff --git a/Python/getversion.c b/Python/getversion.c index 226b2f999a6bfd1..8d8bc6ea70048c9 100644 --- a/Python/getversion.c +++ b/Python/getversion.c @@ -15,7 +15,7 @@ void _Py_InitVersion(void) } initialized = 1; #ifdef Py_GIL_DISABLED - const char *buildinfo_format = "%.80s experimental free-threading build (%.80s) %.80s"; + const char *buildinfo_format = "%.80s free-threading build (%.80s) %.80s"; #else const char *buildinfo_format = "%.80s (%.80s) %.80s"; #endif diff --git a/Python/hamt.c b/Python/hamt.c index f9bbf63961d8de1..98ef96df2c54e37 100644 --- a/Python/hamt.c +++ b/Python/hamt.c @@ -256,9 +256,9 @@ Debug ===== The HAMT datatype is accessible for testing purposes under the -`_testcapi` module: +`_testinternalcapi` module: - >>> from _testcapi import hamt + >>> from _testinternalcapi import hamt >>> h = hamt() >>> h2 = h.set('a', 2) >>> h3 = h2.set('b', 3) @@ -701,6 +701,7 @@ hamt_node_bitmap_assoc(PyHamtNode_Bitmap *self, PyHamtNode_Bitmap *ret = hamt_node_bitmap_clone(self); if (ret == NULL) { + Py_DECREF(sub_node); return NULL; } Py_SETREF(ret->b_array[val_idx], (PyObject*)sub_node); @@ -993,6 +994,7 @@ hamt_node_bitmap_without(PyHamtNode_Bitmap *self, PyHamtNode_Bitmap *clone = hamt_node_bitmap_clone(self); if (clone == NULL) { + Py_DECREF(sub_node); return W_ERROR; } @@ -1176,7 +1178,7 @@ hamt_node_bitmap_dump(PyHamtNode_Bitmap *node, } if (key_or_null == NULL) { - if (PyUnicodeWriter_WriteUTF8(writer, "NULL:\n", -1) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, "NULL:\n", 6) < 0) { goto error; } @@ -1194,7 +1196,7 @@ hamt_node_bitmap_dump(PyHamtNode_Bitmap *node, } } - if (PyUnicodeWriter_WriteUTF8(writer, "\n", 1) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, "\n", 1) < 0) { goto error; } } @@ -1915,7 +1917,7 @@ hamt_node_array_dump(PyHamtNode_Array *node, goto error; } - if (PyUnicodeWriter_WriteUTF8(writer, "\n", 1) < 0) { + if (PyUnicodeWriter_WriteASCII(writer, "\n", 1) < 0) { goto error; } } @@ -2328,6 +2330,10 @@ _PyHamt_Eq(PyHamtObject *v, PyHamtObject *w) return 0; } + Py_INCREF(v); + Py_INCREF(w); + + int res = 1; PyHamtIteratorState iter; hamt_iter_t iter_res; hamt_find_t find_res; @@ -2343,25 +2349,38 @@ _PyHamt_Eq(PyHamtObject *v, PyHamtObject *w) find_res = hamt_find(w, v_key, &w_val); switch (find_res) { case F_ERROR: - return -1; + res = -1; + goto done; case F_NOT_FOUND: - return 0; + res = 0; + goto done; case F_FOUND: { + Py_INCREF(v_key); + Py_INCREF(v_val); + Py_INCREF(w_val); int cmp = PyObject_RichCompareBool(v_val, w_val, Py_EQ); + Py_DECREF(v_key); + Py_DECREF(v_val); + Py_DECREF(w_val); if (cmp < 0) { - return -1; + res = -1; + goto done; } if (cmp == 0) { - return 0; + res = 0; + goto done; } } } } } while (iter_res != I_END); - return 1; +done: + Py_DECREF(v); + Py_DECREF(w); + return res; } Py_ssize_t diff --git a/Python/import.c b/Python/import.c index afdc28eda31b9b9..d18ca3d87e09ebd 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3,6 +3,7 @@ #include "Python.h" #include "pycore_audit.h" // _PySys_Audit() #include "pycore_ceval.h" +#include "pycore_critical_section.h" // Py_BEGIN_CRITICAL_SECTION() #include "pycore_hashtable.h" // _Py_hashtable_new_full() #include "pycore_import.h" // _PyImport_BootstrapImp() #include "pycore_initconfig.h" // _PyStatus_OK() @@ -296,12 +297,32 @@ PyImport_GetModule(PyObject *name) mod = import_get_module(tstate, name); if (mod != NULL && mod != Py_None) { if (import_ensure_initialized(tstate->interp, mod, name) < 0) { + goto error; + } + /* Verify the module is still in sys.modules. Another thread may have + removed it (due to import failure) between our import_get_module() + call and the _initializing check in import_ensure_initialized(). */ + PyObject *mod_check = import_get_module(tstate, name); + if (mod_check != mod) { + Py_XDECREF(mod_check); + if (_PyErr_Occurred(tstate)) { + goto error; + } + /* The module was removed or replaced. Return NULL to report + "not found" rather than trying to keep up with racing + modifications to sys.modules; returning the new value would + require looping to redo the ensure_initialized check. */ Py_DECREF(mod); - remove_importlib_frames(tstate); return NULL; } + Py_DECREF(mod_check); } return mod; + +error: + Py_DECREF(mod); + remove_importlib_frames(tstate); + return NULL; } /* Get the module object corresponding to a module name. @@ -309,13 +330,8 @@ PyImport_GetModule(PyObject *name) if not, create a new one and insert it in the modules dictionary. */ static PyObject * -import_add_module(PyThreadState *tstate, PyObject *name) +import_add_module_lock_held(PyObject *modules, PyObject *name) { - PyObject *modules = get_modules_dict(tstate, false); - if (modules == NULL) { - return NULL; - } - PyObject *m; if (PyMapping_GetOptionalItem(modules, name, &m) < 0) { return NULL; @@ -335,6 +351,21 @@ import_add_module(PyThreadState *tstate, PyObject *name) return m; } +static PyObject * +import_add_module(PyThreadState *tstate, PyObject *name) +{ + PyObject *modules = get_modules_dict(tstate, false); + if (modules == NULL) { + return NULL; + } + + PyObject *m; + Py_BEGIN_CRITICAL_SECTION(modules); + m = import_add_module_lock_held(modules, name); + Py_END_CRITICAL_SECTION(); + return m; +} + PyObject * PyImport_AddModuleRef(const char *name) { @@ -825,13 +856,13 @@ _PyImport_SwapPackageContext(const char *newcontext) int _PyImport_GetDLOpenFlags(PyInterpreterState *interp) { - return DLOPENFLAGS(interp); + return FT_ATOMIC_LOAD_INT_RELAXED(DLOPENFLAGS(interp)); } void _PyImport_SetDLOpenFlags(PyInterpreterState *interp, int new_val) { - DLOPENFLAGS(interp) = new_val; + FT_ATOMIC_STORE_INT_RELAXED(DLOPENFLAGS(interp), new_val); } #endif // HAVE_DLOPEN @@ -2116,13 +2147,29 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, } main_finally: + if (rc < 0) { + _Py_ext_module_loader_result_apply_error(&res, name_buf); + } + /* Switch back to the subinterpreter. */ if (switched) { + // gh-144601: The exception object can't be transferred across + // interpreters. Instead, we print out an unraisable exception, and + // then raise a different exception for the calling interpreter. + if (rc < 0) { + assert(PyErr_Occurred()); + PyErr_FormatUnraisable("Exception while importing from subinterpreter"); + } assert(main_tstate != tstate); switch_back_from_main_interpreter(tstate, main_tstate, mod); /* Any module we got from the init function will have to be * reloaded in the subinterpreter. */ mod = NULL; + if (rc < 0) { + PyErr_SetString(PyExc_ImportError, + "failed to import from subinterpreter due to exception"); + goto error; + } } /*****************************************************************/ @@ -2131,7 +2178,6 @@ import_run_extension(PyThreadState *tstate, PyModInitFunction p0, /* Finally we handle the error return from _PyImport_RunModInitFunc(). */ if (rc < 0) { - _Py_ext_module_loader_result_apply_error(&res, name_buf); goto error; } @@ -2979,7 +3025,7 @@ find_frozen(PyObject *nameobj, struct frozen_info *info) if (nameobj == NULL || nameobj == Py_None) { return FROZEN_BAD_NAME; } - const char *name = PyUnicode_AsUTF8(nameobj); + const char *name = _PyUnicode_AsUTF8NoNUL(nameobj); if (name == NULL) { // Note that this function previously used // _PyUnicode_EqualToASCIIString(). We clear the error here @@ -3802,6 +3848,27 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, if (import_ensure_initialized(tstate->interp, mod, abs_name) < 0) { goto error; } + /* Verify the module is still in sys.modules. Another thread may have + removed it (due to import failure) between our import_get_module() + call and the _initializing check in import_ensure_initialized(). + If removed, we retry the import to preserve normal semantics: the + caller gets the exception from the actual import failure rather + than a synthetic error. */ + PyObject *mod_check = import_get_module(tstate, abs_name); + if (mod_check != mod) { + Py_XDECREF(mod_check); + if (_PyErr_Occurred(tstate)) { + goto error; + } + Py_DECREF(mod); + mod = import_find_and_load(tstate, abs_name); + if (mod == NULL) { + goto error; + } + } + else { + Py_DECREF(mod_check); + } } else { Py_XDECREF(mod); @@ -3852,15 +3919,17 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } final_mod = import_get_module(tstate, to_return); - Py_DECREF(to_return); if (final_mod == NULL) { if (!_PyErr_Occurred(tstate)) { _PyErr_Format(tstate, PyExc_KeyError, "%R not in sys.modules as expected", to_return); } + Py_DECREF(to_return); goto error; } + + Py_DECREF(to_return); } } else { @@ -3956,23 +4025,28 @@ PyImport_Import(PyObject *module_name) } /* Get the builtins from current globals */ - globals = PyEval_GetGlobals(); + globals = PyEval_GetGlobals(); // borrowed if (globals != NULL) { Py_INCREF(globals); + // XXX Use _PyEval_EnsureBuiltins()? builtins = PyObject_GetItem(globals, &_Py_ID(__builtins__)); - if (builtins == NULL) + if (builtins == NULL) { + // XXX Fall back to interp->builtins or sys.modules['builtins']? goto err; + } + } + else if (_PyErr_Occurred(tstate)) { + goto err; } else { /* No globals -- use standard builtins, and fake globals */ - builtins = PyImport_ImportModuleLevel("builtins", - NULL, NULL, NULL, 0); - if (builtins == NULL) { + globals = PyDict_New(); + if (globals == NULL) { goto err; } - globals = Py_BuildValue("{OO}", &_Py_ID(__builtins__), builtins); - if (globals == NULL) + if (_PyEval_EnsureBuiltinsWithModule(tstate, globals, &builtins) < 0) { goto err; + } } /* Get the __import__ function from the builtins */ @@ -4254,13 +4328,14 @@ _imp.acquire_lock Acquires the interpreter's import lock for the current thread. -This lock should be used by import hooks to ensure thread-safety when importing -modules. On platforms without threads, this function does nothing. +This lock should be used by import hooks to ensure thread-safety when +importing modules. On platforms without threads, this function does +nothing. [clinic start generated code]*/ static PyObject * _imp_acquire_lock_impl(PyObject *module) -/*[clinic end generated code: output=1aff58cb0ee1b026 input=4a2d4381866d5fdc]*/ +/*[clinic end generated code: output=1aff58cb0ee1b026 input=60e9c1b4ab471ead]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); _PyImport_AcquireLock(interp); @@ -4669,6 +4744,7 @@ static PyObject * _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) /*[clinic end generated code: output=83249b827a4fde77 input=c31b954f4cf4e09d]*/ { + FILE *fp = NULL; PyObject *mod = NULL; PyThreadState *tstate = _PyThreadState_GET(); @@ -4711,16 +4787,12 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) /* We would move this (and the fclose() below) into * _PyImport_GetModInitFunc(), but it isn't clear if the intervening * code relies on fp still being open. */ - FILE *fp; if (file != NULL) { fp = Py_fopen(info.filename, "r"); if (fp == NULL) { goto finally; } } - else { - fp = NULL; - } PyModInitFunction p0 = _PyImport_GetModInitFunc(&info, fp); if (p0 == NULL) { @@ -4744,12 +4816,10 @@ _imp_create_dynamic_impl(PyObject *module, PyObject *spec, PyObject *file) } #endif - // XXX Shouldn't this happen in the error cases too (i.e. in "finally")? - if (fp) { +finally: + if (fp != NULL) { fclose(fp); } - -finally: _Py_ext_module_loader_info_clear(&info); return mod; } diff --git a/Python/importdl.c b/Python/importdl.c index 802843fe7b9dceb..98e71e643ef3319 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -175,7 +175,6 @@ _Py_ext_module_loader_info_init_for_builtin( PyObject *name) { assert(PyUnicode_Check(name)); - assert(PyUnicode_FindChar(name, '.', 0, PyUnicode_GetLength(name), -1) == -1); assert(PyUnicode_GetLength(name) > 0); PyObject *name_encoded = PyUnicode_AsEncodedString(name, "ascii", NULL); diff --git a/Python/index_pool.c b/Python/index_pool.c index 007c81a0fc16ec3..520a65938ec6c7b 100644 --- a/Python/index_pool.c +++ b/Python/index_pool.c @@ -172,6 +172,9 @@ _PyIndexPool_AllocIndex(_PyIndexPool *pool) else { index = heap_pop(free_indices); } + + pool->tlbc_generation++; + UNLOCK_POOL(pool); return index; } @@ -180,6 +183,7 @@ void _PyIndexPool_FreeIndex(_PyIndexPool *pool, int32_t index) { LOCK_POOL(pool); + pool->tlbc_generation++; heap_add(&pool->free_indices, index); UNLOCK_POOL(pool); } diff --git a/Python/initconfig.c b/Python/initconfig.c index e827091172162e8..fd5ad650531f4d6 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -84,118 +84,137 @@ typedef struct { config_sys_flag_setter flag_setter; } PyConfigSysSpec; +typedef struct { + int *ptr; + int not; +} PyConfigGlobalVar; + typedef struct { const char *name; size_t offset; PyConfigMemberType type; PyConfigMemberVisibility visibility; PyConfigSysSpec sys; + PyConfigGlobalVar global_var; } PyConfigSpec; -#define SPEC(MEMBER, TYPE, VISIBILITY, sys) \ +#define SPEC(MEMBER, TYPE, VISIBILITY, sys, global_var) \ {#MEMBER, offsetof(PyConfig, MEMBER), \ - PyConfig_MEMBER_##TYPE, PyConfig_MEMBER_##VISIBILITY, sys} + PyConfig_MEMBER_##TYPE, PyConfig_MEMBER_##VISIBILITY, sys, global_var} #define SYS_ATTR(name) {name, -1, NULL} #define SYS_FLAG_SETTER(index, setter) {NULL, index, setter} #define SYS_FLAG(index) SYS_FLAG_SETTER(index, NULL) #define NO_SYS SYS_ATTR(NULL) +#define GLOBAL(ptr, not) {ptr, not} +#define NO_GLOBAL GLOBAL(NULL, 0) + +// Ignore deprecations on global variables such as Py_IsolatedFlag +_Py_COMP_DIAG_PUSH +_Py_COMP_DIAG_IGNORE_DEPR_DECLS + // Update _test_embed_set_config when adding new members static const PyConfigSpec PYCONFIG_SPEC[] = { // --- Public options ----------- - SPEC(argv, WSTR_LIST, PUBLIC, SYS_ATTR("argv")), - SPEC(base_exec_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("base_exec_prefix")), - SPEC(base_executable, WSTR_OPT, PUBLIC, SYS_ATTR("_base_executable")), - SPEC(base_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("base_prefix")), - SPEC(bytes_warning, UINT, PUBLIC, SYS_FLAG(9)), - SPEC(cpu_count, INT, PUBLIC, NO_SYS), - SPEC(exec_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("exec_prefix")), - SPEC(executable, WSTR_OPT, PUBLIC, SYS_ATTR("executable")), - SPEC(inspect, BOOL, PUBLIC, SYS_FLAG(1)), - SPEC(int_max_str_digits, UINT, PUBLIC, NO_SYS), - SPEC(interactive, BOOL, PUBLIC, SYS_FLAG(2)), - SPEC(module_search_paths, WSTR_LIST, PUBLIC, SYS_ATTR("path")), - SPEC(optimization_level, UINT, PUBLIC, SYS_FLAG(3)), - SPEC(parser_debug, BOOL, PUBLIC, SYS_FLAG(0)), - SPEC(platlibdir, WSTR, PUBLIC, SYS_ATTR("platlibdir")), - SPEC(prefix, WSTR_OPT, PUBLIC, SYS_ATTR("prefix")), - SPEC(pycache_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("pycache_prefix")), - SPEC(quiet, BOOL, PUBLIC, SYS_FLAG(10)), - SPEC(stdlib_dir, WSTR_OPT, PUBLIC, SYS_ATTR("_stdlib_dir")), - SPEC(use_environment, BOOL, PUBLIC, SYS_FLAG_SETTER(7, config_sys_flag_not)), - SPEC(verbose, UINT, PUBLIC, SYS_FLAG(8)), - SPEC(warnoptions, WSTR_LIST, PUBLIC, SYS_ATTR("warnoptions")), - SPEC(write_bytecode, BOOL, PUBLIC, SYS_FLAG_SETTER(4, config_sys_flag_not)), - SPEC(xoptions, WSTR_LIST, PUBLIC, SYS_ATTR("_xoptions")), + SPEC(argv, WSTR_LIST, PUBLIC, SYS_ATTR("argv"), NO_GLOBAL), + SPEC(base_exec_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("base_exec_prefix"), NO_GLOBAL), + SPEC(base_executable, WSTR_OPT, PUBLIC, SYS_ATTR("_base_executable"), NO_GLOBAL), + SPEC(base_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("base_prefix"), NO_GLOBAL), + SPEC(bytes_warning, UINT, PUBLIC, SYS_FLAG(9), GLOBAL(&Py_BytesWarningFlag, 0)), + SPEC(cpu_count, INT, PUBLIC, NO_SYS, NO_GLOBAL), + SPEC(exec_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("exec_prefix"), NO_GLOBAL), + SPEC(executable, WSTR_OPT, PUBLIC, SYS_ATTR("executable"), NO_GLOBAL), + SPEC(inspect, BOOL, PUBLIC, SYS_FLAG(1), GLOBAL(&Py_InspectFlag, 0)), + SPEC(int_max_str_digits, UINT, PUBLIC, NO_SYS, NO_GLOBAL), + SPEC(interactive, BOOL, PUBLIC, SYS_FLAG(2), GLOBAL(&Py_InteractiveFlag, 0)), + SPEC(module_search_paths, WSTR_LIST, PUBLIC, SYS_ATTR("path"), NO_GLOBAL), + SPEC(optimization_level, UINT, PUBLIC, SYS_FLAG(3), GLOBAL(&Py_OptimizeFlag, 0)), + SPEC(parser_debug, BOOL, PUBLIC, SYS_FLAG(0), GLOBAL(&Py_DebugFlag, 0)), + SPEC(platlibdir, WSTR, PUBLIC, SYS_ATTR("platlibdir"), NO_GLOBAL), + SPEC(prefix, WSTR_OPT, PUBLIC, SYS_ATTR("prefix"), NO_GLOBAL), + SPEC(pycache_prefix, WSTR_OPT, PUBLIC, SYS_ATTR("pycache_prefix"), NO_GLOBAL), + SPEC(quiet, BOOL, PUBLIC, SYS_FLAG(10), GLOBAL(&Py_QuietFlag, 0)), + SPEC(stdlib_dir, WSTR_OPT, PUBLIC, SYS_ATTR("_stdlib_dir"), NO_GLOBAL), + SPEC(use_environment, BOOL, PUBLIC, + SYS_FLAG_SETTER(7, config_sys_flag_not), GLOBAL(&Py_IgnoreEnvironmentFlag, 1)), + SPEC(verbose, UINT, PUBLIC, SYS_FLAG(8), GLOBAL(&Py_VerboseFlag, 0)), + SPEC(warnoptions, WSTR_LIST, PUBLIC, SYS_ATTR("warnoptions"), NO_GLOBAL), + SPEC(write_bytecode, BOOL, PUBLIC, SYS_FLAG_SETTER(4, config_sys_flag_not), + GLOBAL(&Py_DontWriteBytecodeFlag, 1)), + SPEC(xoptions, WSTR_LIST, PUBLIC, SYS_ATTR("_xoptions"), NO_GLOBAL), // --- Read-only options ----------- #ifdef Py_STATS - SPEC(_pystats, BOOL, READ_ONLY, NO_SYS), + SPEC(_pystats, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), #endif - SPEC(buffered_stdio, BOOL, READ_ONLY, NO_SYS), - SPEC(check_hash_pycs_mode, WSTR, READ_ONLY, NO_SYS), - SPEC(code_debug_ranges, BOOL, READ_ONLY, NO_SYS), - SPEC(configure_c_stdio, BOOL, READ_ONLY, NO_SYS), - SPEC(dev_mode, BOOL, READ_ONLY, NO_SYS), // sys.flags.dev_mode - SPEC(dump_refs, BOOL, READ_ONLY, NO_SYS), - SPEC(dump_refs_file, WSTR_OPT, READ_ONLY, NO_SYS), + SPEC(buffered_stdio, BOOL, READ_ONLY, NO_SYS, + GLOBAL(&Py_UnbufferedStdioFlag, 1)), + SPEC(check_hash_pycs_mode, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(code_debug_ranges, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(configure_c_stdio, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(dev_mode, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), // sys.flags.dev_mode + SPEC(dump_refs, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(dump_refs_file, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL), #ifdef Py_GIL_DISABLED - SPEC(enable_gil, INT, READ_ONLY, NO_SYS), - SPEC(tlbc_enabled, INT, READ_ONLY, NO_SYS), + SPEC(enable_gil, INT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(tlbc_enabled, INT, READ_ONLY, NO_SYS, NO_GLOBAL), #endif - SPEC(faulthandler, BOOL, READ_ONLY, NO_SYS), - SPEC(filesystem_encoding, WSTR, READ_ONLY, NO_SYS), - SPEC(filesystem_errors, WSTR, READ_ONLY, NO_SYS), - SPEC(hash_seed, ULONG, READ_ONLY, NO_SYS), - SPEC(home, WSTR_OPT, READ_ONLY, NO_SYS), - SPEC(thread_inherit_context, INT, READ_ONLY, NO_SYS), - SPEC(context_aware_warnings, INT, READ_ONLY, NO_SYS), - SPEC(import_time, UINT, READ_ONLY, NO_SYS), - SPEC(install_signal_handlers, BOOL, READ_ONLY, NO_SYS), - SPEC(isolated, BOOL, READ_ONLY, NO_SYS), // sys.flags.isolated + SPEC(faulthandler, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(filesystem_encoding, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(filesystem_errors, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(hash_seed, ULONG, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(home, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(thread_inherit_context, INT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(context_aware_warnings, INT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(import_time, UINT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(install_signal_handlers, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(isolated, BOOL, READ_ONLY, NO_SYS, GLOBAL(&Py_IsolatedFlag, 0)), // sys.flags.isolated #ifdef MS_WINDOWS - SPEC(legacy_windows_stdio, BOOL, READ_ONLY, NO_SYS), + SPEC(legacy_windows_stdio, BOOL, READ_ONLY, NO_SYS, + GLOBAL(&Py_LegacyWindowsStdioFlag, 0)), #endif - SPEC(malloc_stats, BOOL, READ_ONLY, NO_SYS), - SPEC(orig_argv, WSTR_LIST, READ_ONLY, SYS_ATTR("orig_argv")), - SPEC(parse_argv, BOOL, READ_ONLY, NO_SYS), - SPEC(pathconfig_warnings, BOOL, READ_ONLY, NO_SYS), - SPEC(perf_profiling, UINT, READ_ONLY, NO_SYS), - SPEC(remote_debug, BOOL, READ_ONLY, NO_SYS), - SPEC(program_name, WSTR, READ_ONLY, NO_SYS), - SPEC(run_command, WSTR_OPT, READ_ONLY, NO_SYS), - SPEC(run_filename, WSTR_OPT, READ_ONLY, NO_SYS), - SPEC(run_module, WSTR_OPT, READ_ONLY, NO_SYS), + SPEC(malloc_stats, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(orig_argv, WSTR_LIST, READ_ONLY, SYS_ATTR("orig_argv"), NO_GLOBAL), + SPEC(parse_argv, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(pathconfig_warnings, BOOL, READ_ONLY, NO_SYS, + GLOBAL(&Py_FrozenFlag, 1)), + SPEC(perf_profiling, UINT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(remote_debug, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(program_name, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(run_command, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(run_filename, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(run_module, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL), #ifdef Py_DEBUG - SPEC(run_presite, WSTR_OPT, READ_ONLY, NO_SYS), + SPEC(run_presite, WSTR_OPT, READ_ONLY, NO_SYS, NO_GLOBAL), #endif - SPEC(safe_path, BOOL, READ_ONLY, NO_SYS), - SPEC(show_ref_count, BOOL, READ_ONLY, NO_SYS), - SPEC(site_import, BOOL, READ_ONLY, NO_SYS), // sys.flags.no_site - SPEC(skip_source_first_line, BOOL, READ_ONLY, NO_SYS), - SPEC(stdio_encoding, WSTR, READ_ONLY, NO_SYS), - SPEC(stdio_errors, WSTR, READ_ONLY, NO_SYS), - SPEC(tracemalloc, UINT, READ_ONLY, NO_SYS), - SPEC(use_frozen_modules, BOOL, READ_ONLY, NO_SYS), - SPEC(use_hash_seed, BOOL, READ_ONLY, NO_SYS), + SPEC(safe_path, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(show_ref_count, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(site_import, BOOL, READ_ONLY, NO_SYS, GLOBAL(&Py_NoSiteFlag, 1)), // sys.flags.no_site + SPEC(skip_source_first_line, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(stdio_encoding, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(stdio_errors, WSTR, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(tracemalloc, UINT, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(use_frozen_modules, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), + SPEC(use_hash_seed, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), #ifdef __APPLE__ - SPEC(use_system_logger, BOOL, READ_ONLY, NO_SYS), + SPEC(use_system_logger, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), #endif - SPEC(user_site_directory, BOOL, READ_ONLY, NO_SYS), // sys.flags.no_user_site - SPEC(warn_default_encoding, BOOL, READ_ONLY, NO_SYS), + SPEC(user_site_directory, BOOL, READ_ONLY, NO_SYS, + GLOBAL(&Py_NoUserSiteDirectory, 1)), // sys.flags.no_user_site + SPEC(warn_default_encoding, BOOL, READ_ONLY, NO_SYS, NO_GLOBAL), // --- Init-only options ----------- - SPEC(_config_init, UINT, INIT_ONLY, NO_SYS), - SPEC(_init_main, BOOL, INIT_ONLY, NO_SYS), - SPEC(_install_importlib, BOOL, INIT_ONLY, NO_SYS), - SPEC(_is_python_build, BOOL, INIT_ONLY, NO_SYS), - SPEC(module_search_paths_set, BOOL, INIT_ONLY, NO_SYS), - SPEC(pythonpath_env, WSTR_OPT, INIT_ONLY, NO_SYS), - SPEC(sys_path_0, WSTR_OPT, INIT_ONLY, NO_SYS), + SPEC(_config_init, UINT, INIT_ONLY, NO_SYS, NO_GLOBAL), + SPEC(_init_main, BOOL, INIT_ONLY, NO_SYS, NO_GLOBAL), + SPEC(_install_importlib, BOOL, INIT_ONLY, NO_SYS, NO_GLOBAL), + SPEC(_is_python_build, BOOL, INIT_ONLY, NO_SYS, NO_GLOBAL), + SPEC(module_search_paths_set, BOOL, INIT_ONLY, NO_SYS, NO_GLOBAL), + SPEC(pythonpath_env, WSTR_OPT, INIT_ONLY, NO_SYS, NO_GLOBAL), + SPEC(sys_path_0, WSTR_OPT, INIT_ONLY, NO_SYS, NO_GLOBAL), // Array terminator {NULL, 0, 0, 0, NO_SYS}, @@ -231,17 +250,24 @@ static const PyConfigSpec PYPRECONFIG_SPEC[] = { {NULL, 0, 0, 0, NO_SYS}, }; +// End of ignoring deprecations on global variables +_Py_COMP_DIAG_POP + #undef SPEC #undef SYS_ATTR #undef SYS_FLAG_SETTER #undef SYS_FLAG #undef NO_SYS +#undef GLOBAL +#undef NO_GLOBAL // Forward declarations -static PyObject* -config_get(const PyConfig *config, const PyConfigSpec *spec, - int use_sys); +static PyObject* config_get(const PyConfig *config, const PyConfigSpec *spec, + int use_sys); +static void initconfig_free_wstr(wchar_t *member); +static void initconfig_free_wstr_list(PyWideStringList *list); +static void initconfig_free_config(const PyConfig *config); /* --- Command line options --------------------------------------- */ @@ -299,9 +325,15 @@ arg ...: arguments passed to program in sys.argv[1:]\n\ static const char usage_xoptions[] = "\ The following implementation-specific options are available:\n\ +-X context_aware_warnings=[0|1]: if true (1) then the warnings module will\n\ + use a context variables; if false (0) then the warnings module will\n\ + use module globals, which is not concurrent-safe; set to true for\n\ + free-threaded builds and false otherwise; also\n\ + PYTHON_CONTEXT_AWARE_WARNINGS\n\ -X cpu_count=N: override the return value of os.cpu_count();\n\ -X cpu_count=default cancels overriding; also PYTHON_CPU_COUNT\n\ -X dev : enable Python Development Mode; also PYTHONDEVMODE\n\ +-X disable-remote-debug: disable remote debugging; also PYTHON_DISABLE_REMOTE_DEBUG\n\ -X faulthandler: dump the Python traceback on fatal errors;\n\ also PYTHONFAULTHANDLER\n\ -X frozen_modules=[on|off]: whether to use frozen modules; the default is \"on\"\n\ @@ -312,7 +344,7 @@ The following implementation-specific options are available:\n\ "-X gil=[0|1]: enable (1) or disable (0) the GIL; also PYTHON_GIL\n" #endif "\ --X importtime[=2]: show how long each import takes; use -X importtime=2 to\ +-X importtime[=2]: show how long each import takes; use -X importtime=2 to\n\ log imports of already-loaded modules; also PYTHONPROFILEIMPORTTIME\n\ -X int_max_str_digits=N: limit the size of int<->str conversions;\n\ 0 disables the limit; also PYTHONINTMAXSTRDIGITS\n\ @@ -321,7 +353,6 @@ The following implementation-specific options are available:\n\ -X perf: support the Linux \"perf\" profiler; also PYTHONPERFSUPPORT=1\n\ -X perf_jit: support the Linux \"perf\" profiler with DWARF support;\n\ also PYTHON_PERF_JIT_SUPPORT=1\n\ --X disable-remote-debug: disable remote debugging; also PYTHON_DISABLE_REMOTE_DEBUG\n\ " #ifdef Py_DEBUG "-X presite=MOD: import this module before site; also PYTHON_PRESITE\n" @@ -336,21 +367,17 @@ The following implementation-specific options are available:\n\ "\ -X showrefcount: output the total reference count and number of used\n\ memory blocks when the program finishes or after each statement in\n\ - the interactive interpreter; only works on debug builds\n" + the interactive interpreter; only works on debug builds\n\ +-X thread_inherit_context=[0|1]: enable (1) or disable (0) threads inheriting\n\ + context vars by default; enabled by default in the free-threaded\n\ + build and disabled otherwise; also PYTHON_THREAD_INHERIT_CONTEXT\n\ +" #ifdef Py_GIL_DISABLED "-X tlbc=[0|1]: enable (1) or disable (0) thread-local bytecode. Also\n\ PYTHON_TLBC\n" #endif "\ --X thread_inherit_context=[0|1]: enable (1) or disable (0) threads inheriting\n\ - context vars by default; enabled by default in the free-threaded\n\ - build and disabled otherwise; also PYTHON_THREAD_INHERIT_CONTEXT\n\ --X context_aware_warnings=[0|1]: if true (1) then the warnings module will\n\ - use a context variables; if false (0) then the warnings module will\n\ - use module globals, which is not concurrent-safe; set to true for\n\ - free-threaded builds and false otherwise; also\n\ - PYTHON_CONTEXT_AWARE_WARNINGS\n\ --X tracemalloc[=N]: trace Python memory allocations; N sets a traceback limit\n \ +-X tracemalloc[=N]: trace Python memory allocations; N sets a traceback limit\n\ of N frames (default: 1); also PYTHONTRACEMALLOC=N\n\ -X utf8[=0|1]: enable (1) or disable (0) UTF-8 mode; also PYTHONUTF8\n\ -X warn_default_encoding: enable opt-in EncodingWarning for 'encoding=None';\n\ @@ -360,34 +387,19 @@ The following implementation-specific options are available:\n\ /* Envvars that don't have equivalent command-line options are listed first */ static const char usage_envvars[] = "Environment variables that change behavior:\n" -"PYTHONSTARTUP : file executed on interactive startup (no default)\n" -"PYTHONPATH : '%lc'-separated list of directories prefixed to the\n" -" default module search path. The result is sys.path.\n" -"PYTHONHOME : alternate <prefix> directory (or <prefix>%lc<exec_prefix>).\n" -" The default module search path uses %s.\n" -"PYTHONPLATLIBDIR: override sys.platlibdir\n" +"PYTHONASYNCIODEBUG: enable asyncio debug mode\n" +"PYTHON_BASIC_REPL: use the traditional parser-based REPL\n" +"PYTHONBREAKPOINT: if this variable is set to 0, it disables the default\n" +" debugger. It can be set to the callable of your debugger of\n" +" choice.\n" "PYTHONCASEOK : ignore case in 'import' statements (Windows)\n" -"PYTHONIOENCODING: encoding[:errors] used for stdin/stdout/stderr\n" -"PYTHONHASHSEED : if this variable is set to 'random', a random value is used\n" -" to seed the hashes of str and bytes objects. It can also be\n" -" set to an integer in the range [0,4294967295] to get hash\n" -" values with a predictable seed.\n" -"PYTHONMALLOC : set the Python memory allocators and/or install debug hooks\n" -" on Python memory allocators. Use PYTHONMALLOC=debug to\n" -" install debug hooks.\n" -"PYTHONMALLOCSTATS: print memory allocator statistics\n" "PYTHONCOERCECLOCALE: if this variable is set to 0, it disables the locale\n" " coercion behavior. Use PYTHONCOERCECLOCALE=warn to request\n" " display of locale coercion and locale compatibility warnings\n" " on stderr.\n" -"PYTHONBREAKPOINT: if this variable is set to 0, it disables the default\n" -" debugger. It can be set to the callable of your debugger of\n" -" choice.\n" "PYTHON_COLORS : if this variable is set to 1, the interpreter will colorize\n" " various kinds of output. Setting it to 0 deactivates\n" " this behavior.\n" -"PYTHON_HISTORY : the location of a .python_history file.\n" -"PYTHONASYNCIODEBUG: enable asyncio debug mode\n" #ifdef Py_TRACE_REFS "PYTHONDUMPREFS : dump objects and reference counts still alive after shutdown\n" "PYTHONDUMPREFSFILE: dump objects and reference counts to the specified file\n" @@ -395,14 +407,31 @@ static const char usage_envvars[] = #ifdef __APPLE__ "PYTHONEXECUTABLE: set sys.argv[0] to this value (macOS only)\n" #endif +"PYTHONHASHSEED : if this variable is set to 'random', a random value is used\n" +" to seed the hashes of str and bytes objects. It can also be\n" +" set to an integer in the range [0,4294967295] to get hash\n" +" values with a predictable seed.\n" +"PYTHON_HISTORY : the location of a .python_history file.\n" +"PYTHONHOME : alternate <prefix> directory (or <prefix>%lc<exec_prefix>).\n" +" The default module search path uses %s.\n" +"PYTHONIOENCODING: encoding[:errors] used for stdin/stdout/stderr\n" #ifdef MS_WINDOWS "PYTHONLEGACYWINDOWSFSENCODING: use legacy \"mbcs\" encoding for file system\n" "PYTHONLEGACYWINDOWSSTDIO: use legacy Windows stdio\n" #endif +"PYTHONMALLOC : set the Python memory allocators and/or install debug hooks\n" +" on Python memory allocators. Use PYTHONMALLOC=debug to\n" +" install debug hooks.\n" +"PYTHONMALLOCSTATS: print memory allocator statistics\n" +"PYTHONPATH : '%lc'-separated list of directories prefixed to the\n" +" default module search path. The result is sys.path.\n" +"PYTHONPLATLIBDIR: override sys.platlibdir\n" +"PYTHONSTARTUP : file executed on interactive startup (no default)\n" "PYTHONUSERBASE : defines the user base directory (site.USER_BASE)\n" -"PYTHON_BASIC_REPL: use the traditional parser-based REPL\n" "\n" "These variables have equivalent command-line options (see --help for details):\n" +"PYTHON_CONTEXT_AWARE_WARNINGS: if true (1), enable thread-safe warnings\n" +" module behaviour (-X context_aware_warnings)\n" "PYTHON_CPU_COUNT: override the return value of os.cpu_count() (-X cpu_count)\n" "PYTHONDEBUG : enable parser debug mode (-d)\n" "PYTHONDEVMODE : enable Python Development Mode (-X dev)\n" @@ -421,9 +450,9 @@ static const char usage_envvars[] = " (-X no_debug_ranges)\n" "PYTHONNOUSERSITE: disable user site directory (-s)\n" "PYTHONOPTIMIZE : enable level 1 optimizations (-O)\n" -"PYTHONPERFSUPPORT: support the Linux \"perf\" profiler (-X perf)\n" "PYTHON_PERF_JIT_SUPPORT: enable Linux \"perf\" profiler support with JIT\n" " (-X perf_jit)\n" +"PYTHONPERFSUPPORT: support the Linux \"perf\" profiler (-X perf)\n" #ifdef Py_DEBUG "PYTHON_PRESITE: import this module before site (-X presite)\n" #endif @@ -434,13 +463,11 @@ static const char usage_envvars[] = #ifdef Py_STATS "PYTHONSTATS : turns on statistics gathering (-X pystats)\n" #endif +"PYTHON_THREAD_INHERIT_CONTEXT: if true (1), threads inherit context vars\n" +" (-X thread_inherit_context)\n" #ifdef Py_GIL_DISABLED "PYTHON_TLBC : when set to 0, disables thread-local bytecode (-X tlbc)\n" #endif -"PYTHON_THREAD_INHERIT_CONTEXT: if true (1), threads inherit context vars\n" -" (-X thread_inherit_context)\n" -"PYTHON_CONTEXT_AWARE_WARNINGS: if true (1), enable thread-safe warnings module\n" -" behaviour (-X context_aware_warnings)\n" "PYTHONTRACEMALLOC: trace Python memory allocations (-X tracemalloc)\n" "PYTHONUNBUFFERED: disable stdout/stderr buffering (-u)\n" "PYTHONUTF8 : control the UTF-8 mode (-X utf8)\n" @@ -499,7 +526,7 @@ _Py_COMP_DIAG_IGNORE_DEPR_DECLS do { \ obj = (EXPR); \ if (obj == NULL) { \ - return NULL; \ + goto fail; \ } \ int res = PyDict_SetItemString(dict, (KEY), obj); \ Py_DECREF(obj); \ @@ -1843,7 +1870,9 @@ config_read_env_vars(PyConfig *config) _Py_get_env_flag(use_env, &config->parser_debug, "PYTHONDEBUG"); _Py_get_env_flag(use_env, &config->verbose, "PYTHONVERBOSE"); _Py_get_env_flag(use_env, &config->optimization_level, "PYTHONOPTIMIZE"); - _Py_get_env_flag(use_env, &config->inspect, "PYTHONINSPECT"); + if (!config->inspect && _Py_GetEnv(use_env, "PYTHONINSPECT")) { + config->inspect = 1; + } int dont_write_bytecode = 0; _Py_get_env_flag(use_env, &dont_write_bytecode, "PYTHONDONTWRITEBYTECODE"); @@ -2835,7 +2864,7 @@ config_usage(int error, const wchar_t* program) static void config_envvars_usage(void) { - printf(usage_envvars, (wint_t)DELIM, (wint_t)DELIM, PYTHONHOMEHELP); + printf(usage_envvars, (wint_t)DELIM, PYTHONHOMEHELP, (wint_t)DELIM); } static void @@ -2962,8 +2991,6 @@ config_parse_cmdline(PyConfig *config, PyWideStringList *warnoptions, /* option handled by _PyPreCmdline_Read() */ break; - /* case 'J': reserved for Jython */ - case 'O': config->optimization_level++; break; @@ -3727,6 +3754,9 @@ PyInitConfig_Free(PyInitConfig *config) if (config == NULL) { return; } + + initconfig_free_config(&config->config); + PyMem_RawFree(config->inittab); free(config->err_msg); free(config); } @@ -4095,13 +4125,51 @@ PyInitConfig_SetStr(PyInitConfig *config, const char *name, const char* value) } +static void +initconfig_free_wstr(wchar_t *member) +{ + if (member) { + free(member); + } +} + + +static void +initconfig_free_wstr_list(PyWideStringList *list) +{ + for (Py_ssize_t i = 0; i < list->length; i++) { + free(list->items[i]); + } + free(list->items); +} + + +static void +initconfig_free_config(const PyConfig *config) +{ + const PyConfigSpec *spec = PYCONFIG_SPEC; + for (; spec->name != NULL; spec++) { + void *member = config_get_spec_member(config, spec); + if (spec->type == PyConfig_MEMBER_WSTR + || spec->type == PyConfig_MEMBER_WSTR_OPT) + { + wchar_t *wstr = *(wchar_t **)member; + initconfig_free_wstr(wstr); + } + else if (spec->type == PyConfig_MEMBER_WSTR_LIST) { + initconfig_free_wstr_list(member); + } + } +} + + static int -_PyWideStringList_FromUTF8(PyInitConfig *config, PyWideStringList *list, - Py_ssize_t length, char * const *items) +initconfig_set_str_list(PyInitConfig *config, PyWideStringList *list, + Py_ssize_t length, char * const *items) { PyWideStringList wlist = _PyWideStringList_INIT; size_t size = sizeof(wchar_t*) * length; - wlist.items = (wchar_t **)PyMem_RawMalloc(size); + wlist.items = (wchar_t **)malloc(size); if (wlist.items == NULL) { config->status = _PyStatus_NO_MEMORY(); return -1; @@ -4110,14 +4178,14 @@ _PyWideStringList_FromUTF8(PyInitConfig *config, PyWideStringList *list, for (Py_ssize_t i = 0; i < length; i++) { wchar_t *arg = utf8_to_wstr(config, items[i]); if (arg == NULL) { - _PyWideStringList_Clear(&wlist); + initconfig_free_wstr_list(&wlist); return -1; } wlist.items[i] = arg; wlist.length++; } - _PyWideStringList_Clear(list); + initconfig_free_wstr_list(list); *list = wlist; return 0; } @@ -4138,7 +4206,7 @@ PyInitConfig_SetStrList(PyInitConfig *config, const char *name, return -1; } PyWideStringList *list = raw_member; - if (_PyWideStringList_FromUTF8(config, list, length, items) < 0) { + if (initconfig_set_str_list(config, list, length, items) < 0) { return -1; } @@ -4330,7 +4398,8 @@ config_get(const PyConfig *config, const PyConfigSpec *spec, if (strcmp(spec->name, "int_max_str_digits") == 0) { PyInterpreterState *interp = _PyInterpreterState_GET(); - return PyLong_FromLong(interp->long_state.max_str_digits); + int maxdigits = _Py_atomic_load_int(&interp->long_state.max_str_digits); + return PyLong_FromLong(maxdigits); } } @@ -4684,6 +4753,16 @@ PyConfig_Set(const char *name, PyObject *value) Py_UNREACHABLE(); } + // Set the global variable + if (spec->global_var.ptr != NULL) { + assert(has_int_value); + int value = int_value; + if (spec->global_var.not) { + value = !value; + } + *spec->global_var.ptr = value; + } + if (spec->sys.attr != NULL) { // Set the sys attribute, but don't set PyInterpreterState.config // to keep the code simple. diff --git a/Python/instrumentation.c b/Python/instrumentation.c index 13bdd041becd697..d06cda7c589c74b 100644 --- a/Python/instrumentation.c +++ b/Python/instrumentation.c @@ -1040,6 +1040,8 @@ set_version_raw(uintptr_t *ptr, uint32_t version) static void set_global_version(PyThreadState *tstate, uint32_t version) { + ASSERT_WORLD_STOPPED(); + assert((version & _PY_EVAL_EVENTS_MASK) == 0); PyInterpreterState *interp = tstate->interp; set_version_raw(&interp->ceval.instrumentation_version, version); @@ -1190,9 +1192,10 @@ call_instrumentation_vector( break; } else { - LOCK_CODE(code); + PyInterpreterState *interp = tstate->interp; + _PyEval_StopTheWorld(interp); remove_tools(code, offset, event, 1 << tool); - UNLOCK_CODE(); + _PyEval_StartTheWorld(interp); } } } @@ -1282,13 +1285,10 @@ _Py_call_instrumentation_exc2( } int -_Py_Instrumentation_GetLine(PyCodeObject *code, int index) +_Py_Instrumentation_GetLine(PyCodeObject *code, _PyCoLineInstrumentationData *line_data, int index) { - _PyCoMonitoringData *monitoring = code->_co_monitoring; - assert(monitoring != NULL); - assert(monitoring->lines != NULL); + assert(line_data != NULL); assert(index < Py_SIZE(code)); - _PyCoLineInstrumentationData *line_data = monitoring->lines; int line_delta = get_line_delta(line_data, index); int line = compute_line(code, line_delta); return line; @@ -1306,11 +1306,11 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, _PyCoMonitoringData *monitoring = code->_co_monitoring; _PyCoLineInstrumentationData *line_data = monitoring->lines; PyInterpreterState *interp = tstate->interp; - int line = _Py_Instrumentation_GetLine(code, i); + int line = _Py_Instrumentation_GetLine(code, line_data, i); assert(line >= 0); assert(prev != NULL); int prev_index = (int)(prev - bytecode); - int prev_line = _Py_Instrumentation_GetLine(code, prev_index); + int prev_line = _Py_Instrumentation_GetLine(code, line_data, prev_index); if (prev_line == line) { int prev_opcode = bytecode[prev_index].op.code; /* RESUME and INSTRUMENTED_RESUME are needed for the operation of @@ -1381,9 +1381,10 @@ _Py_call_instrumentation_line(PyThreadState *tstate, _PyInterpreterFrame* frame, } else { /* DISABLE */ - LOCK_CODE(code); + PyInterpreterState *interp = tstate->interp; + _PyEval_StopTheWorld(interp); remove_line_tools(code, i, 1 << tool); - UNLOCK_CODE(); + _PyEval_StartTheWorld(interp); } } while (tools); Py_DECREF(line_obj); @@ -1438,9 +1439,10 @@ _Py_call_instrumentation_instruction(PyThreadState *tstate, _PyInterpreterFrame* } else { /* DISABLE */ - LOCK_CODE(code); + PyInterpreterState *interp = tstate->interp; + _PyEval_StopTheWorld(interp); remove_per_instruction_tools(code, offset, 1 << tool); - UNLOCK_CODE(); + _PyEval_StartTheWorld(interp); } } Py_DECREF(offset_obj); @@ -1505,11 +1507,9 @@ initialize_tools(PyCodeObject *code) } static void -initialize_lines(PyCodeObject *code, int bytes_per_entry) +initialize_lines(_PyCoLineInstrumentationData *line_data, PyCodeObject *code, int bytes_per_entry) { ASSERT_WORLD_STOPPED_OR_LOCKED(code); - _PyCoLineInstrumentationData *line_data = code->_co_monitoring->lines; - assert(line_data != NULL); line_data->bytes_per_entry = bytes_per_entry; int code_len = (int)Py_SIZE(code); @@ -1650,18 +1650,19 @@ allocate_instrumentation_data(PyCodeObject *code) ASSERT_WORLD_STOPPED_OR_LOCKED(code); if (code->_co_monitoring == NULL) { - code->_co_monitoring = PyMem_Malloc(sizeof(_PyCoMonitoringData)); - if (code->_co_monitoring == NULL) { + _PyCoMonitoringData *monitoring = PyMem_Malloc(sizeof(_PyCoMonitoringData)); + if (monitoring == NULL) { PyErr_NoMemory(); return -1; } - code->_co_monitoring->local_monitors = (_Py_LocalMonitors){ 0 }; - code->_co_monitoring->active_monitors = (_Py_LocalMonitors){ 0 }; - code->_co_monitoring->tools = NULL; - code->_co_monitoring->lines = NULL; - code->_co_monitoring->line_tools = NULL; - code->_co_monitoring->per_instruction_opcodes = NULL; - code->_co_monitoring->per_instruction_tools = NULL; + monitoring->local_monitors = (_Py_LocalMonitors){ 0 }; + monitoring->active_monitors = (_Py_LocalMonitors){ 0 }; + monitoring->tools = NULL; + monitoring->lines = NULL; + monitoring->line_tools = NULL; + monitoring->per_instruction_opcodes = NULL; + monitoring->per_instruction_tools = NULL; + _Py_atomic_store_ptr_release(&code->_co_monitoring, monitoring); } return 0; } @@ -1726,12 +1727,13 @@ update_instrumentation_data(PyCodeObject *code, PyInterpreterState *interp) else { bytes_per_entry = 5; } - code->_co_monitoring->lines = PyMem_Malloc(1 + code_len * bytes_per_entry); - if (code->_co_monitoring->lines == NULL) { + _PyCoLineInstrumentationData *lines = PyMem_Malloc(1 + code_len * bytes_per_entry); + if (lines == NULL) { PyErr_NoMemory(); return -1; } - initialize_lines(code, bytes_per_entry); + initialize_lines(lines, code, bytes_per_entry); + _Py_atomic_store_ptr_release(&code->_co_monitoring->lines, lines); } if (multitools && code->_co_monitoring->line_tools == NULL) { code->_co_monitoring->line_tools = PyMem_Malloc(code_len); @@ -1936,28 +1938,26 @@ _Py_Instrument(PyCodeObject *code, PyInterpreterState *interp) static int -instrument_all_executing_code_objects(PyInterpreterState *interp) { +instrument_all_executing_code_objects(PyInterpreterState *interp) +{ ASSERT_WORLD_STOPPED(); - _PyRuntimeState *runtime = &_PyRuntime; - HEAD_LOCK(runtime); - PyThreadState* ts = PyInterpreterState_ThreadHead(interp); - HEAD_UNLOCK(runtime); - while (ts) { + int err = 0; + _Py_FOR_EACH_TSTATE_BEGIN(interp, ts) { _PyInterpreterFrame *frame = ts->current_frame; while (frame) { if (frame->owner < FRAME_OWNED_BY_INTERPRETER) { - if (instrument_lock_held(_PyFrame_GetCode(frame), interp)) { - return -1; + err = instrument_lock_held(_PyFrame_GetCode(frame), interp); + if (err) { + goto done; } } frame = frame->previous; } - HEAD_LOCK(runtime); - ts = PyThreadState_Next(ts); - HEAD_UNLOCK(runtime); } - return 0; +done: + _Py_FOR_EACH_TSTATE_END(interp); + return err; } static void @@ -2003,6 +2003,7 @@ check_tool(PyInterpreterState *interp, int tool_id) int _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) { + ASSERT_WORLD_STOPPED(); assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); PyThreadState *tstate = _PyThreadState_GET(); PyInterpreterState *interp = tstate->interp; @@ -2011,33 +2012,28 @@ _PyMonitoring_SetEvents(int tool_id, _PyMonitoringEventSet events) return -1; } - int res; - _PyEval_StopTheWorld(interp); uint32_t existing_events = get_events(&interp->monitors, tool_id); if (existing_events == events) { - res = 0; - goto done; + return 0; } - set_events(&interp->monitors, tool_id, events); uint32_t new_version = global_version(interp) + MONITORING_VERSION_INCREMENT; if (new_version == 0) { PyErr_Format(PyExc_OverflowError, "events set too many times"); - res = -1; - goto done; + return -1; } + set_events(&interp->monitors, tool_id, events); set_global_version(tstate, new_version); #ifdef _Py_TIER2 _Py_Executors_InvalidateAll(interp, 1); #endif - res = instrument_all_executing_code_objects(interp); -done: - _PyEval_StartTheWorld(interp); - return res; + return instrument_all_executing_code_objects(interp); } int _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEventSet events) { + ASSERT_WORLD_STOPPED(); + assert(0 <= tool_id && tool_id < PY_MONITORING_TOOL_IDS); PyInterpreterState *interp = _PyInterpreterState_GET(); assert(events < (1 << _PY_MONITORING_LOCAL_EVENTS)); @@ -2049,11 +2045,8 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent return -1; } - int res; - _PyEval_StopTheWorld(interp); if (allocate_instrumentation_data(code)) { - res = -1; - goto done; + return -1; } code->_co_monitoring->tool_versions[tool_id] = interp->monitoring_tool_versions[tool_id]; @@ -2061,16 +2054,11 @@ _PyMonitoring_SetLocalEvents(PyCodeObject *code, int tool_id, _PyMonitoringEvent _Py_LocalMonitors *local = &code->_co_monitoring->local_monitors; uint32_t existing_events = get_local_events(local, tool_id); if (existing_events == events) { - res = 0; - goto done; + return 0; } set_local_events(local, tool_id, events); - res = force_instrument_lock_held(code, interp); - -done: - _PyEval_StartTheWorld(interp); - return res; + return force_instrument_lock_held(code, interp); } int @@ -2102,11 +2090,12 @@ int _PyMonitoring_ClearToolId(int tool_id) } } + _PyEval_StopTheWorld(interp); if (_PyMonitoring_SetEvents(tool_id, 0) < 0) { + _PyEval_StartTheWorld(interp); return -1; } - _PyEval_StopTheWorld(interp); uint32_t version = global_version(interp) + MONITORING_VERSION_INCREMENT; if (version == 0) { PyErr_Format(PyExc_OverflowError, "events set too many times"); @@ -2343,7 +2332,11 @@ monitoring_set_events_impl(PyObject *module, int tool_id, int event_set) event_set &= ~(1 << PY_MONITORING_EVENT_BRANCH); event_set |= (1 << PY_MONITORING_EVENT_BRANCH_RIGHT) | (1 << PY_MONITORING_EVENT_BRANCH_LEFT); } - if (_PyMonitoring_SetEvents(tool_id, event_set)) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); + int err = _PyMonitoring_SetEvents(tool_id, event_set); + _PyEval_StartTheWorld(interp); + if (err) { return NULL; } Py_RETURN_NONE; @@ -2424,7 +2417,11 @@ monitoring_set_local_events_impl(PyObject *module, int tool_id, return NULL; } - if (_PyMonitoring_SetLocalEvents((PyCodeObject*)code, tool_id, event_set)) { + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); + int err = _PyMonitoring_SetLocalEvents((PyCodeObject*)code, tool_id, event_set); + _PyEval_StartTheWorld(interp); + if (err) { return NULL; } Py_RETURN_NONE; @@ -2558,18 +2555,22 @@ PyObject *_Py_CreateMonitoringObject(void) err = PyObject_SetAttrString(events, "NO_EVENTS", _PyLong_GetZero()); if (err) goto error; PyObject *val = PyLong_FromLong(PY_MONITORING_DEBUGGER_ID); + assert(val != NULL); /* Can't return NULL because the int is small. */ err = PyObject_SetAttrString(mod, "DEBUGGER_ID", val); Py_DECREF(val); if (err) goto error; val = PyLong_FromLong(PY_MONITORING_COVERAGE_ID); + assert(val != NULL); err = PyObject_SetAttrString(mod, "COVERAGE_ID", val); Py_DECREF(val); if (err) goto error; val = PyLong_FromLong(PY_MONITORING_PROFILER_ID); + assert(val != NULL); err = PyObject_SetAttrString(mod, "PROFILER_ID", val); Py_DECREF(val); if (err) goto error; val = PyLong_FromLong(PY_MONITORING_OPTIMIZER_ID); + assert(val != NULL); err = PyObject_SetAttrString(mod, "OPTIMIZER_ID", val); Py_DECREF(val); if (err) goto error; @@ -2991,9 +2992,10 @@ branch_handler_vectorcall( // Orphaned NOT_TAKEN -- Jump removed by the compiler return res; } - LOCK_CODE(code); + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); remove_tools(code, offset, other_event, 1 << self->tool_id); - UNLOCK_CODE(); + _PyEval_StartTheWorld(interp); } return res; } diff --git a/Python/interpconfig.c b/Python/interpconfig.c index 1add8a81425b9a8..a37bd3f5b23a01e 100644 --- a/Python/interpconfig.c +++ b/Python/interpconfig.c @@ -208,7 +208,7 @@ interp_config_from_dict(PyObject *origdict, PyInterpreterConfig *config, } else if (unused > 0) { PyErr_Format(PyExc_ValueError, - "config dict has %d extra items (%R)", unused, dict); + "config dict has %zd extra items (%R)", unused, dict); goto error; } diff --git a/Python/jit.c b/Python/jit.c index e232cc1f7d92506..35ff9a663cd140d 100644 --- a/Python/jit.c +++ b/Python/jit.c @@ -69,10 +69,6 @@ jit_alloc(size_t size) #else int flags = MAP_ANONYMOUS | MAP_PRIVATE; int prot = PROT_READ | PROT_WRITE; -# ifdef MAP_JIT - flags |= MAP_JIT; - prot |= PROT_EXEC; -# endif unsigned char *memory = mmap(NULL, size, prot, flags, -1, 0); int failed = memory == MAP_FAILED; #endif @@ -118,11 +114,8 @@ mark_executable(unsigned char *memory, size_t size) int old; int failed = !VirtualProtect(memory, size, PAGE_EXECUTE_READ, &old); #else - int failed = 0; __builtin___clear_cache((char *)memory, (char *)memory + size); -#ifndef MAP_JIT - failed = mprotect(memory, size, PROT_EXEC | PROT_READ); -#endif + int failed = mprotect(memory, size, PROT_EXEC | PROT_READ); #endif if (failed) { jit_error("unable to protect executable memory"); @@ -255,18 +248,6 @@ patch_aarch64_12(unsigned char *location, uint64_t value) set_bits(loc32, 10, value, shift, 12); } -// Relaxable 12-bit low part of an absolute address. Pairs nicely with -// patch_aarch64_21rx (below). -void -patch_aarch64_12x(unsigned char *location, uint64_t value) -{ - // This can *only* be relaxed if it occurs immediately before a matching - // patch_aarch64_21rx. If that happens, the JIT build step will replace both - // calls with a single call to patch_aarch64_33rx. Otherwise, we end up - // here, and the instruction is patched normally: - patch_aarch64_12(location, value); -} - // 16-bit low part of an absolute address. void patch_aarch64_16a(unsigned char *location, uint64_t value) @@ -327,18 +308,6 @@ patch_aarch64_21r(unsigned char *location, uint64_t value) set_bits(loc32, 5, value, 2, 19); } -// Relaxable 21-bit count of pages between this page and an absolute address's -// page. Pairs nicely with patch_aarch64_12x (above). -void -patch_aarch64_21rx(unsigned char *location, uint64_t value) -{ - // This can *only* be relaxed if it occurs immediately before a matching - // patch_aarch64_12x. If that happens, the JIT build step will replace both - // calls with a single call to patch_aarch64_33rx. Otherwise, we end up - // here, and the instruction is patched normally: - patch_aarch64_21r(location, value); -} - // 28-bit relative branch. void patch_aarch64_26r(unsigned char *location, uint64_t value) @@ -354,46 +323,6 @@ patch_aarch64_26r(unsigned char *location, uint64_t value) set_bits(loc32, 0, value, 2, 26); } -// A pair of patch_aarch64_21rx and patch_aarch64_12x. -void -patch_aarch64_33rx(unsigned char *location, uint64_t value) -{ - uint32_t *loc32 = (uint32_t *)location; - // Try to relax the pair of GOT loads into an immediate value: - assert(IS_AARCH64_ADRP(*loc32)); - unsigned char reg = get_bits(loc32[0], 0, 5); - assert(IS_AARCH64_LDR_OR_STR(loc32[1])); - // There should be only one register involved: - assert(reg == get_bits(loc32[1], 0, 5)); // ldr's output register. - assert(reg == get_bits(loc32[1], 5, 5)); // ldr's input register. - uint64_t relaxed = *(uint64_t *)value; - if (relaxed < (1UL << 16)) { - // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; nop - loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | reg; - loc32[1] = 0xD503201F; - return; - } - if (relaxed < (1ULL << 32)) { - // adrp reg, AAA; ldr reg, [reg + BBB] -> movz reg, XXX; movk reg, YYY - loc32[0] = 0xD2800000 | (get_bits(relaxed, 0, 16) << 5) | reg; - loc32[1] = 0xF2A00000 | (get_bits(relaxed, 16, 16) << 5) | reg; - return; - } - relaxed = value - (uintptr_t)location; - if ((relaxed & 0x3) == 0 && - (int64_t)relaxed >= -(1L << 19) && - (int64_t)relaxed < (1L << 19)) - { - // adrp reg, AAA; ldr reg, [reg + BBB] -> ldr reg, XXX; nop - loc32[0] = 0x58000000 | (get_bits(relaxed, 2, 19) << 5) | reg; - loc32[1] = 0xD503201F; - return; - } - // Couldn't do it. Just patch the two instructions normally: - patch_aarch64_21rx(location, value); - patch_aarch64_12x(location + 4, value); -} - // Relaxable 32-bit relative address. void patch_x86_64_32rx(unsigned char *location, uint64_t value) @@ -528,9 +457,6 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz if (memory == NULL) { return -1; } -#ifdef MAP_JIT - pthread_jit_write_protect_np(0); -#endif // Collect memory stats OPT_STAT_ADD(jit_total_memory_size, total_size); OPT_STAT_ADD(jit_code_size, code_size); @@ -568,9 +494,6 @@ _PyJIT_Compile(_PyExecutorObject *executor, const _PyUOpInstruction trace[], siz data += group->data_size; assert(code == memory + code_size); assert(data == memory + code_size + state.trampolines.size + data_size); -#ifdef MAP_JIT - pthread_jit_write_protect_np(1); -#endif if (mark_executable(memory, total_size)) { jit_free(memory, total_size); return -1; diff --git a/Python/legacy_tracing.c b/Python/legacy_tracing.c index dbd19d7755c237e..2e29e1aa208ac0e 100644 --- a/Python/legacy_tracing.c +++ b/Python/legacy_tracing.c @@ -133,16 +133,10 @@ sys_profile_call_or_return( Py_RETURN_NONE; } -int -_PyEval_SetOpcodeTrace( - PyFrameObject *frame, - bool enable -) { - assert(frame != NULL); - - PyCodeObject *code = _PyFrame_GetCode(frame->f_frame); +static int +set_opcode_trace_world_stopped(PyCodeObject *code, bool enable) +{ _PyMonitoringEventSet events = 0; - if (_PyMonitoring_GetLocalEvents(code, PY_MONITORING_SYS_TRACE_ID, &events) < 0) { return -1; } @@ -161,6 +155,32 @@ _PyEval_SetOpcodeTrace( return _PyMonitoring_SetLocalEvents(code, PY_MONITORING_SYS_TRACE_ID, events); } +int +_PyEval_SetOpcodeTrace(PyFrameObject *frame, bool enable) +{ + assert(frame != NULL); + + PyCodeObject *code = _PyFrame_GetCode(frame->f_frame); + +#ifdef Py_GIL_DISABLED + // First check if a change is necessary outside of the stop-the-world pause + _PyMonitoringEventSet events = 0; + if (_PyMonitoring_GetLocalEvents(code, PY_MONITORING_SYS_TRACE_ID, &events) < 0) { + return -1; + } + int is_enabled = (events & (1 << PY_MONITORING_EVENT_INSTRUCTION)) != 0; + if (is_enabled == enable) { + return 0; // No change needed + } +#endif + + PyInterpreterState *interp = _PyInterpreterState_GET(); + _PyEval_StopTheWorld(interp); + int res = set_opcode_trace_world_stopped(code, enable); + _PyEval_StartTheWorld(interp); + return res; +} + static PyObject * call_trace_func(_PyLegacyEventHandler *self, PyObject *arg) { @@ -378,8 +398,8 @@ sys_trace_jump_func( assert(PyCode_Check(code)); /* We can call _Py_Instrumentation_GetLine because we always set * line events for tracing */ - int to_line = _Py_Instrumentation_GetLine(code, to); - int from_line = _Py_Instrumentation_GetLine(code, from); + int to_line = _Py_Instrumentation_GetLine(code, code->_co_monitoring->lines, to); + int from_line = _Py_Instrumentation_GetLine(code, code->_co_monitoring->lines, from); if (to_line != from_line) { /* Will be handled by target INSTRUMENTED_LINE */ return &_PyInstrumentation_DISABLE; @@ -438,59 +458,74 @@ is_tstate_valid(PyThreadState *tstate) } #endif -static Py_ssize_t -setup_profile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg, PyObject **old_profileobj) +static int +setup_profile_callbacks(void *Py_UNUSED(arg)) { - *old_profileobj = NULL; /* Setup PEP 669 monitoring callbacks and events. */ - if (!tstate->interp->sys_profile_initialized) { - tstate->interp->sys_profile_initialized = true; - if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - sys_profile_start, PyTrace_CALL, - PY_MONITORING_EVENT_PY_START, - PY_MONITORING_EVENT_PY_RESUME)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - sys_profile_throw, PyTrace_CALL, - PY_MONITORING_EVENT_PY_THROW, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - sys_profile_return, PyTrace_RETURN, - PY_MONITORING_EVENT_PY_RETURN, - PY_MONITORING_EVENT_PY_YIELD)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - sys_profile_unwind, PyTrace_RETURN, - PY_MONITORING_EVENT_PY_UNWIND, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - sys_profile_call_or_return, PyTrace_C_CALL, - PY_MONITORING_EVENT_CALL, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - sys_profile_call_or_return, PyTrace_C_RETURN, - PY_MONITORING_EVENT_C_RETURN, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, - sys_profile_call_or_return, PyTrace_C_EXCEPTION, - PY_MONITORING_EVENT_C_RAISE, -1)) { - return -1; - } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + sys_profile_start, PyTrace_CALL, + PY_MONITORING_EVENT_PY_START, + PY_MONITORING_EVENT_PY_RESUME)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + sys_profile_throw, PyTrace_CALL, + PY_MONITORING_EVENT_PY_THROW, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + sys_profile_return, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_RETURN, + PY_MONITORING_EVENT_PY_YIELD)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + sys_profile_unwind, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_UNWIND, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + sys_profile_call_or_return, PyTrace_C_CALL, + PY_MONITORING_EVENT_CALL, -1)) { + return -1; } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + sys_profile_call_or_return, PyTrace_C_RETURN, + PY_MONITORING_EVENT_C_RETURN, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_PROFILE_ID, + sys_profile_call_or_return, PyTrace_C_EXCEPTION, + PY_MONITORING_EVENT_C_RAISE, -1)) { + return -1; + } + return 0; +} +static PyObject * +swap_profile_func_arg(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +{ int delta = (func != NULL) - (tstate->c_profilefunc != NULL); tstate->c_profilefunc = func; - *old_profileobj = tstate->c_profileobj; + PyObject *old_profileobj = tstate->c_profileobj; tstate->c_profileobj = Py_XNewRef(arg); tstate->interp->sys_profiling_threads += delta; assert(tstate->interp->sys_profiling_threads >= 0); - return tstate->interp->sys_profiling_threads; + return old_profileobj; +} + +static int +set_monitoring_profile_events(PyInterpreterState *interp) +{ + uint32_t events = 0; + if (interp->sys_profiling_threads) { + events = + (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | + (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | + (1 << PY_MONITORING_EVENT_CALL) | (1 << PY_MONITORING_EVENT_PY_UNWIND) | + (1 << PY_MONITORING_EVENT_PY_THROW); + } + return _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, events); } int @@ -507,87 +542,155 @@ _PyEval_SetProfile(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) return -1; } - // needs to be decref'd outside of the lock - PyObject *old_profileobj; - LOCK_SETUP(); - Py_ssize_t profiling_threads = setup_profile(tstate, func, arg, &old_profileobj); - UNLOCK_SETUP(); - Py_XDECREF(old_profileobj); + PyInterpreterState *interp = tstate->interp; + if (_PyOnceFlag_CallOnce(&interp->sys_profile_once_flag, + setup_profile_callbacks, NULL) < 0) { + return -1; + } + + _PyEval_StopTheWorld(interp); + PyObject *old_profileobj = swap_profile_func_arg(tstate, func, arg); + int ret = set_monitoring_profile_events(interp); + _PyEval_StartTheWorld(interp); + Py_XDECREF(old_profileobj); // needs to be decref'd outside of stop-the-world + return ret; +} + +int +_PyEval_SetProfileAllThreads(PyInterpreterState *interp, Py_tracefunc func, PyObject *arg) +{ + PyThreadState *current_tstate = _PyThreadState_GET(); + assert(is_tstate_valid(current_tstate)); + assert(current_tstate->interp == interp); + + if (_PySys_Audit(current_tstate, "sys.setprofile", NULL) < 0) { + return -1; + } + + if (_PyOnceFlag_CallOnce(&interp->sys_profile_once_flag, + setup_profile_callbacks, NULL) < 0) { + return -1; + } - uint32_t events = 0; - if (profiling_threads) { - events = - (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | - (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | - (1 << PY_MONITORING_EVENT_CALL) | (1 << PY_MONITORING_EVENT_PY_UNWIND) | - (1 << PY_MONITORING_EVENT_PY_THROW); + PyObject *old_profileobjs = NULL; + _PyEval_StopTheWorld(interp); + HEAD_LOCK(&_PyRuntime); + Py_ssize_t num_thread_states = 0; + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, p) { + num_thread_states++; } - return _PyMonitoring_SetEvents(PY_MONITORING_SYS_PROFILE_ID, events); + old_profileobjs = PyTuple_New(num_thread_states); + if (old_profileobjs == NULL) { + HEAD_UNLOCK(&_PyRuntime); + _PyEval_StartTheWorld(interp); + return -1; + } + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { + PyObject *old = swap_profile_func_arg(tstate, func, arg); + PyTuple_SET_ITEM(old_profileobjs, --num_thread_states, old); + } + HEAD_UNLOCK(&_PyRuntime); + int ret = set_monitoring_profile_events(interp); + _PyEval_StartTheWorld(interp); + Py_XDECREF(old_profileobjs); // needs to be decref'd outside of stop-the-world + return ret; } -static Py_ssize_t -setup_tracing(PyThreadState *tstate, Py_tracefunc func, PyObject *arg, PyObject **old_traceobj) +static int +setup_trace_callbacks(void *Py_UNUSED(arg)) { - *old_traceobj = NULL; /* Setup PEP 669 monitoring callbacks and events. */ - if (!tstate->interp->sys_trace_initialized) { - tstate->interp->sys_trace_initialized = true; - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_start, PyTrace_CALL, - PY_MONITORING_EVENT_PY_START, - PY_MONITORING_EVENT_PY_RESUME)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_throw, PyTrace_CALL, - PY_MONITORING_EVENT_PY_THROW, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_return, PyTrace_RETURN, - PY_MONITORING_EVENT_PY_RETURN, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_yield, PyTrace_RETURN, - PY_MONITORING_EVENT_PY_YIELD, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_exception_func, PyTrace_EXCEPTION, - PY_MONITORING_EVENT_RAISE, - PY_MONITORING_EVENT_STOP_ITERATION)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_line_func, PyTrace_LINE, - PY_MONITORING_EVENT_LINE, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_unwind, PyTrace_RETURN, - PY_MONITORING_EVENT_PY_UNWIND, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_jump_func, PyTrace_LINE, - PY_MONITORING_EVENT_JUMP, -1)) { - return -1; - } - if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, - sys_trace_instruction_func, PyTrace_OPCODE, - PY_MONITORING_EVENT_INSTRUCTION, -1)) { - return -1; - } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_start, PyTrace_CALL, + PY_MONITORING_EVENT_PY_START, + PY_MONITORING_EVENT_PY_RESUME)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_throw, PyTrace_CALL, + PY_MONITORING_EVENT_PY_THROW, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_return, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_RETURN, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_yield, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_YIELD, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_exception_func, PyTrace_EXCEPTION, + PY_MONITORING_EVENT_RAISE, + PY_MONITORING_EVENT_STOP_ITERATION)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_line_func, PyTrace_LINE, + PY_MONITORING_EVENT_LINE, -1)) { + return -1; } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_unwind, PyTrace_RETURN, + PY_MONITORING_EVENT_PY_UNWIND, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_jump_func, PyTrace_LINE, + PY_MONITORING_EVENT_JUMP, -1)) { + return -1; + } + if (set_callbacks(PY_MONITORING_SYS_TRACE_ID, + sys_trace_instruction_func, PyTrace_OPCODE, + PY_MONITORING_EVENT_INSTRUCTION, -1)) { + return -1; + } + return 0; +} +static PyObject * +swap_trace_func_arg(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) +{ int delta = (func != NULL) - (tstate->c_tracefunc != NULL); tstate->c_tracefunc = func; - *old_traceobj = tstate->c_traceobj; + PyObject *old_traceobj = tstate->c_traceobj; tstate->c_traceobj = Py_XNewRef(arg); tstate->interp->sys_tracing_threads += delta; assert(tstate->interp->sys_tracing_threads >= 0); - return tstate->interp->sys_tracing_threads; + return old_traceobj; +} + +static int +set_monitoring_trace_events(PyInterpreterState *interp) +{ + uint32_t events = 0; + if (interp->sys_tracing_threads) { + events = + (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | + (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | + (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | + (1 << PY_MONITORING_EVENT_JUMP) | + (1 << PY_MONITORING_EVENT_PY_UNWIND) | (1 << PY_MONITORING_EVENT_PY_THROW) | + (1 << PY_MONITORING_EVENT_STOP_ITERATION); + } + return _PyMonitoring_SetEvents(PY_MONITORING_SYS_TRACE_ID, events); +} + +// Enable opcode tracing for the thread's current frame if needed. +static int +maybe_set_opcode_trace(PyThreadState *tstate) +{ + _PyInterpreterFrame *iframe = tstate->current_frame; + if (iframe == NULL) { + return 0; + } + PyFrameObject *frame = iframe->frame_obj; + if (frame == NULL || !frame->f_trace_opcodes) { + return 0; + } + return set_opcode_trace_world_stopped(_PyFrame_GetCode(iframe), true); } int @@ -603,35 +706,76 @@ _PyEval_SetTrace(PyThreadState *tstate, Py_tracefunc func, PyObject *arg) if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) { return -1; } - // needs to be decref'd outside of the lock - PyObject *old_traceobj; - LOCK_SETUP(); - assert(tstate->interp->sys_tracing_threads >= 0); - Py_ssize_t tracing_threads = setup_tracing(tstate, func, arg, &old_traceobj); - UNLOCK_SETUP(); - Py_XDECREF(old_traceobj); - if (tracing_threads < 0) { + + PyInterpreterState *interp = tstate->interp; + if (_PyOnceFlag_CallOnce(&interp->sys_trace_once_flag, + setup_trace_callbacks, NULL) < 0) { return -1; } - uint32_t events = 0; - if (tracing_threads) { - events = - (1 << PY_MONITORING_EVENT_PY_START) | (1 << PY_MONITORING_EVENT_PY_RESUME) | - (1 << PY_MONITORING_EVENT_PY_RETURN) | (1 << PY_MONITORING_EVENT_PY_YIELD) | - (1 << PY_MONITORING_EVENT_RAISE) | (1 << PY_MONITORING_EVENT_LINE) | - (1 << PY_MONITORING_EVENT_JUMP) | - (1 << PY_MONITORING_EVENT_PY_UNWIND) | (1 << PY_MONITORING_EVENT_PY_THROW) | - (1 << PY_MONITORING_EVENT_STOP_ITERATION); + int err = 0; + _PyEval_StopTheWorld(interp); + PyObject *old_traceobj = swap_trace_func_arg(tstate, func, arg); + err = set_monitoring_trace_events(interp); + if (err != 0) { + goto done; + } + if (interp->sys_tracing_threads) { + err = maybe_set_opcode_trace(tstate); + } +done: + _PyEval_StartTheWorld(interp); + Py_XDECREF(old_traceobj); // needs to be decref'd outside stop-the-world + return err; +} + +int +_PyEval_SetTraceAllThreads(PyInterpreterState *interp, Py_tracefunc func, PyObject *arg) +{ + PyThreadState *current_tstate = _PyThreadState_GET(); + assert(is_tstate_valid(current_tstate)); + assert(current_tstate->interp == interp); + + if (_PySys_Audit(current_tstate, "sys.settrace", NULL) < 0) { + return -1; + } + + if (_PyOnceFlag_CallOnce(&interp->sys_trace_once_flag, + setup_trace_callbacks, NULL) < 0) { + return -1; + } - PyFrameObject* frame = PyEval_GetFrame(); - if (frame && frame->f_trace_opcodes) { - int ret = _PyEval_SetOpcodeTrace(frame, true); - if (ret != 0) { - return ret; + PyObject *old_trace_objs = NULL; + _PyEval_StopTheWorld(interp); + HEAD_LOCK(&_PyRuntime); + Py_ssize_t num_thread_states = 0; + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, p) { + num_thread_states++; + } + old_trace_objs = PyTuple_New(num_thread_states); + if (old_trace_objs == NULL) { + HEAD_UNLOCK(&_PyRuntime); + _PyEval_StartTheWorld(interp); + return -1; + } + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { + PyObject *old = swap_trace_func_arg(tstate, func, arg); + PyTuple_SET_ITEM(old_trace_objs, --num_thread_states, old); + } + if (interp->sys_tracing_threads) { + _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { + int err = maybe_set_opcode_trace(tstate); + if (err != 0) { + HEAD_UNLOCK(&_PyRuntime); + _PyEval_StartTheWorld(interp); + Py_XDECREF(old_trace_objs); + return -1; } } } - - return _PyMonitoring_SetEvents(PY_MONITORING_SYS_TRACE_ID, events); + HEAD_UNLOCK(&_PyRuntime); + int err = set_monitoring_trace_events(interp); + _PyEval_StartTheWorld(interp); + Py_XDECREF(old_trace_objs); // needs to be decref'd outside of stop-the-world + return err; } diff --git a/Python/lock.c b/Python/lock.c index 28a12ad18352d1e..62f2637bcfb4531 100644 --- a/Python/lock.c +++ b/Python/lock.c @@ -58,7 +58,7 @@ _PyMutex_LockTimed(PyMutex *m, PyTime_t timeout, _PyLockFlags flags) return PY_LOCK_ACQUIRED; } } - else if (timeout == 0) { + if (timeout == 0) { return PY_LOCK_FAILURE; } @@ -212,7 +212,16 @@ _PyRawMutex_LockSlow(_PyRawMutex *m) // Wait for us to be woken up. Note that we still have to lock the // mutex ourselves: it is NOT handed off to us. - _PySemaphore_Wait(&waiter.sema, -1, /*detach=*/0); + // + // Loop until we observe an actual wakeup. A return of Py_PARK_INTR + // could otherwise let us exit _PySemaphore_Wait and destroy + // `waiter.sema` while _PyRawMutex_UnlockSlow's matching + // _PySemaphore_Wakeup is still pending, since the unlocker has + // already CAS-removed us from the waiter list without any handshake. + int res; + do { + res = _PySemaphore_Wait(&waiter.sema, -1, /*detach=*/0); + } while (res != Py_PARK_OK); } _PySemaphore_Destroy(&waiter.sema); @@ -619,3 +628,11 @@ PyMutex_Unlock(PyMutex *m) Py_FatalError("unlocking mutex that is not locked"); } } + + +#undef PyMutex_IsLocked +int +PyMutex_IsLocked(PyMutex *m) +{ + return _PyMutex_IsLocked(m); +} diff --git a/Python/marshal.c b/Python/marshal.c index b39c1a5b1ade50e..7e9a1af2076fa25 100644 --- a/Python/marshal.c +++ b/Python/marshal.c @@ -11,6 +11,7 @@ #include "pycore_code.h" // _PyCode_New() #include "pycore_hashtable.h" // _Py_hashtable_t #include "pycore_long.h" // _PyLong_IsZero() +#include "pycore_object.h" // _PyObject_IsUniquelyReferenced #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_setobject.h" // _PySet_NextEntryRef() #include "pycore_unicodeobject.h" // _PyUnicode_InternImmortal() @@ -309,7 +310,7 @@ w_PyLong(const PyLongObject *ob, char flag, WFILE *p) } if (!long_export.digits) { int8_t sign = long_export.value < 0 ? -1 : 1; - uint64_t abs_value = Py_ABS(long_export.value); + uint64_t abs_value = _Py_ABS_CAST(uint64_t, long_export.value); uint64_t d = abs_value; long l = 0; @@ -379,7 +380,6 @@ static int w_ref(PyObject *v, char *flag, WFILE *p) { _Py_hashtable_entry_t *entry; - int w; if (p->version < 3 || p->hashtable == NULL) return 0; /* not writing object references */ @@ -388,7 +388,7 @@ w_ref(PyObject *v, char *flag, WFILE *p) * But we use TYPE_REF always for interned string, to PYC file stable * as possible. */ - if (Py_REFCNT(v) == 1 && + if (_PyObject_IsUniquelyReferenced(v) && !(PyUnicode_CheckExact(v) && PyUnicode_CHECK_INTERNED(v))) { return 0; } @@ -396,20 +396,28 @@ w_ref(PyObject *v, char *flag, WFILE *p) entry = _Py_hashtable_get_entry(p->hashtable, v); if (entry != NULL) { /* write the reference index to the stream */ - w = (int)(uintptr_t)entry->value; + uintptr_t w = (uintptr_t)entry->value; + if (w & 0x80000000LU) { + PyErr_Format(PyExc_ValueError, "cannot marshal recursion %T objects", v); + goto err; + } /* we don't store "long" indices in the dict */ - assert(0 <= w && w <= 0x7fffffff); + assert(w <= 0x7fffffff); w_byte(TYPE_REF, p); - w_long(w, p); + w_long((int)w, p); return 1; } else { - size_t s = p->hashtable->nentries; + size_t w = p->hashtable->nentries; /* we don't support long indices */ - if (s >= 0x7fffffff) { + if (w >= 0x7fffffff) { PyErr_SetString(PyExc_ValueError, "too many objects"); goto err; } - w = (int)s; + // Corresponding code should call w_complete() after + // writing the object. + if (PyCode_Check(v) || PySlice_Check(v)) { + w |= 0x80000000LU; + } if (_Py_hashtable_set(p->hashtable, Py_NewRef(v), (void *)(uintptr_t)w) < 0) { Py_DECREF(v); @@ -423,6 +431,27 @@ w_ref(PyObject *v, char *flag, WFILE *p) return 1; } +static void +w_complete(PyObject *v, WFILE *p) +{ + if (p->version < 3 || p->hashtable == NULL) { + return; + } + if (_PyObject_IsUniquelyReferenced(v)) { + return; + } + + _Py_hashtable_entry_t *entry = _Py_hashtable_get_entry(p->hashtable, v); + if (entry == NULL) { + return; + } + assert(entry != NULL); + uintptr_t w = (uintptr_t)entry->value; + assert(w & 0x80000000LU); + w &= ~0x80000000LU; + entry->value = (void *)(uintptr_t)w; +} + static void w_complex_object(PyObject *v, char flag, WFILE *p); @@ -431,6 +460,10 @@ w_object(PyObject *v, WFILE *p) { char flag = '\0'; + if (p->error != WFERR_OK) { + return; + } + p->depth++; if (p->depth > MAX_MARSHAL_STACK_DEPTH) { @@ -668,6 +701,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(co->co_linetable, p); w_object(co->co_exceptiontable, p); Py_DECREF(co_code); + w_complete(v, p); } else if (PyObject_CheckBuffer(v)) { /* Write unknown bytes-like objects as a bytes object */ @@ -693,6 +727,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p) w_object(slice->start, p); w_object(slice->stop, p); w_object(slice->step, p); + w_complete(v, p); } else { W_TYPE(TYPE_UNKNOWN, p); @@ -1573,7 +1608,7 @@ r_object(RFILE *p) goto code_error; firstlineno = (int)r_long(p); if (firstlineno == -1 && PyErr_Occurred()) - break; + goto code_error; linetable = r_object(p); if (linetable == NULL) goto code_error; @@ -1656,6 +1691,9 @@ r_object(RFILE *p) case TYPE_SLICE: { Py_ssize_t idx = r_ref_reserve(flag, p); + if (idx < 0) { + break; + } PyObject *stop = NULL; PyObject *step = NULL; PyObject *start = r_object(p); @@ -2007,14 +2045,14 @@ marshal.dumps Return the bytes object that would be written to a file by dump(value, file). -Raise a ValueError exception if value has (or contains an object that has) an -unsupported type. +Raise a ValueError exception if value has (or contains an object that +has) an unsupported type. [clinic start generated code]*/ static PyObject * marshal_dumps_impl(PyObject *module, PyObject *value, int version, int allow_code) -/*[clinic end generated code: output=115f90da518d1d49 input=167eaecceb63f0a8]*/ +/*[clinic end generated code: output=115f90da518d1d49 input=d9609c4dee4507fb]*/ { return _PyMarshal_WriteObjectToString(value, version, allow_code); } @@ -2030,13 +2068,13 @@ marshal.loads Convert the bytes-like object to a value. -If no valid value is found, raise EOFError, ValueError or TypeError. Extra -bytes in the input are ignored. +If no valid value is found, raise EOFError, ValueError or TypeError. +Extra bytes in the input are ignored. [clinic start generated code]*/ static PyObject * marshal_loads_impl(PyObject *module, Py_buffer *bytes, int allow_code) -/*[clinic end generated code: output=62c0c538d3edc31f input=14de68965b45aaa7]*/ +/*[clinic end generated code: output=62c0c538d3edc31f input=286f1dbd6811d2ad]*/ { RFILE rf; char *s = bytes->buf; diff --git a/Python/optimizer.c b/Python/optimizer.c index dde3dd8ebe745a8..3a66aeb4f827b36 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -109,13 +109,21 @@ uop_optimize(_PyInterpreterFrame *frame, _Py_CODEUNIT *instr, /* Returns 1 if optimized, 0 if not optimized, and -1 for an error. * If optimized, *executor_ptr contains a new reference to the executor */ -int +// gh-137573: inlining this function causes stack overflows +Py_NO_INLINE int _PyOptimizer_Optimize( _PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyExecutorObject **executor_ptr, int chain_depth) { _PyStackRef *stack_pointer = frame->stackpointer; - assert(_PyInterpreterState_GET()->jit); + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (!interp->jit) { + // gh-140936: It is possible that interp->jit will become false during + // interpreter finalization. However, the specialized JUMP_BACKWARD_JIT + // instruction may still be present. In this case, we should + // return immediately without optimization. + return 0; + } // The first executor in a chain and the MAX_CHAIN_DEPTH'th executor *must* // make progress in order to avoid infinite loops or excessively-long // side-exit chains. We can only insert the executor into the bytecode if @@ -1205,6 +1213,10 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil assert(next_exit == -1); assert(dest == executor->trace); assert(dest->opcode == _START_EXECUTOR); + // Note: we MUST track it here before any Py_DECREF(executor) or + // linking of executor. Otherwise, the GC tries to untrack a + // still untracked object during dealloc. + _PyObject_GC_TRACK(executor); _Py_ExecutorInit(executor, dependencies); #ifdef Py_DEBUG char *python_lltrace = Py_GETENV("PYTHON_LLTRACE"); @@ -1234,7 +1246,6 @@ make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFil return NULL; } #endif - _PyObject_GC_TRACK(executor); return executor; } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 8b0bd1e9518c6ee..5584ae67945971a 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -18,6 +18,7 @@ #include "pycore_opcode_metadata.h" #include "pycore_opcode_utils.h" #include "pycore_pystate.h" // _PyInterpreterState_GET() +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_* #include "pycore_uop_metadata.h" #include "pycore_long.h" #include "pycore_interpframe.h" // _PyFrame_GetCode @@ -54,14 +55,15 @@ static int get_mutations(PyObject* dict) { assert(PyDict_CheckExact(dict)); PyDictObject *d = (PyDictObject *)dict; - return (d->_ma_watcher_tag >> DICT_MAX_WATCHERS) & ((1 << DICT_WATCHED_MUTATION_BITS)-1); + uint64_t tag = FT_ATOMIC_LOAD_UINT64_RELAXED(d->_ma_watcher_tag); + return (tag >> DICT_MAX_WATCHERS) & ((1 << DICT_WATCHED_MUTATION_BITS) - 1); } static void increment_mutations(PyObject* dict) { assert(PyDict_CheckExact(dict)); PyDictObject *d = (PyDictObject *)dict; - d->_ma_watcher_tag += (1 << DICT_MAX_WATCHERS); + FT_ATOMIC_ADD_UINT64(d->_ma_watcher_tag, 1ULL << DICT_MAX_WATCHERS); } /* The first two dict watcher IDs are reserved for CPython, @@ -90,6 +92,17 @@ type_watcher_callback(PyTypeObject* type) return 0; } +static int +_setup_optimizer_watchers(void *Py_UNUSED(arg)) +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + FT_ATOMIC_STORE_PTR_RELEASE( + interp->dict_state.watchers[GLOBALS_WATCHER_ID], + globals_watcher_callback); + interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback; + return 0; +} + static PyObject * convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj, bool pop) { @@ -103,6 +116,10 @@ convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj, bool pop) if ((int)index >= dict->ma_keys->dk_nentries) { return NULL; } + PyDictKeysObject *keys = dict->ma_keys; + if (keys->dk_version != inst->operand0) { + return NULL; + } PyObject *res = entries[index].me_value; if (res == NULL) { return NULL; @@ -167,12 +184,8 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, uint32_t builtins_watched = 0; uint32_t globals_watched = 0; uint32_t prechecked_function_version = 0; - if (interp->dict_state.watchers[GLOBALS_WATCHER_ID] == NULL) { - interp->dict_state.watchers[GLOBALS_WATCHER_ID] = globals_watcher_callback; - } - if (interp->type_watchers[TYPE_WATCHER_ID] == NULL) { - interp->type_watchers[TYPE_WATCHER_ID] = type_watcher_callback; - } + _PyOnceFlag_CallOnce(&interp->dict_state.watcher_setup_once, + _setup_optimizer_watchers, NULL); for (int pc = 0; pc < buffer_size; pc++) { _PyUOpInstruction *inst = &buffer[pc]; int opcode = inst->opcode; @@ -445,13 +458,10 @@ optimize_uops( _Py_uop_abstractcontext_init(ctx); _Py_UOpsAbstractFrame *frame = _Py_uop_frame_new(ctx, co, curr_stacklen, NULL, 0); if (frame == NULL) { - return -1; + return 0; } ctx->curr_frame_depth++; ctx->frame = frame; - ctx->done = false; - ctx->out_of_space = false; - ctx->contradiction = false; _PyUOpInstruction *this_instr = NULL; for (int i = 0; !ctx->done; i++) { diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index e99421a3aff4baa..e3927d6c690a8b0 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -580,7 +580,13 @@ dummy_func(void) { PyDict_Watch(GLOBALS_WATCHER_ID, dict); _Py_BloomFilter_Add(dependencies, dict); PyObject *res = convert_global_to_const(this_instr, dict, true); - attr = sym_new_const(ctx, res); + if (res == NULL) { + attr = sym_new_not_null(ctx); + } + else { + attr = sym_new_const(ctx, res); + } + } } } diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 56b4b9983fbaaa0..53ebdc3b562fe07 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -1235,7 +1235,12 @@ PyDict_Watch(GLOBALS_WATCHER_ID, dict); _Py_BloomFilter_Add(dependencies, dict); PyObject *res = convert_global_to_const(this_instr, dict, true); - attr = sym_new_const(ctx, res); + if (res == NULL) { + attr = sym_new_not_null(ctx); + } + else { + attr = sym_new_const(ctx, res); + } } } } diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index e8a4f87031b76a4..2387f215178624e 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -691,6 +691,13 @@ _Py_uop_abstractcontext_init(JitOptContext *ctx) // Frame setup ctx->curr_frame_depth = 0; + + // Ctx signals. + // Note: this must happen before frame_new, as it might override + // the result should frame_new set things to bottom. + ctx->done = false; + ctx->out_of_space = false; + ctx->contradiction = false; } int diff --git a/Python/parking_lot.c b/Python/parking_lot.c index 8edf43235942ab3..01901f5c09be1b5 100644 --- a/Python/parking_lot.c +++ b/Python/parking_lot.c @@ -61,7 +61,9 @@ _PySemaphore_Init(_PySemaphore *sema) NULL // unnamed ); if (!sema->platform_sem) { - Py_FatalError("parking_lot: CreateSemaphore failed"); + _Py_FatalErrorFormat(__func__, + "parking_lot: CreateSemaphore failed (error: %u)", + GetLastError()); } #elif defined(_Py_USE_SEMAPHORES) if (sem_init(&sema->platform_sem, /*pshared=*/0, /*value=*/0) < 0) { @@ -112,17 +114,27 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, PyTime_t timeout) } } - // NOTE: we wait on the sigint event even in non-main threads to match the - // behavior of the other platforms. Non-main threads will ignore the - // Py_PARK_INTR result. - HANDLE sigint_event = _PyOS_SigintEvent(); - HANDLE handles[2] = { sema->platform_sem, sigint_event }; - DWORD count = sigint_event != NULL ? 2 : 1; + HANDLE handles[2] = { sema->platform_sem, NULL }; + HANDLE sigint_event = NULL; + DWORD count = 1; + if (_Py_IsMainThread()) { + // gh-135099: Wait on the SIGINT event only in the main thread. Other + // threads would ignore the result anyways, and accessing + // `_PyOS_SigintEvent()` from non-main threads may race with + // interpreter shutdown, which closes the event handle. Note that + // non-main interpreters will ignore the result. + sigint_event = _PyOS_SigintEvent(); + if (sigint_event != NULL) { + handles[1] = sigint_event; + count = 2; + } + } wait = WaitForMultipleObjects(count, handles, FALSE, millis); if (wait == WAIT_OBJECT_0) { res = Py_PARK_OK; } else if (wait == WAIT_OBJECT_0 + 1) { + assert(sigint_event != NULL); ResetEvent(sigint_event); res = Py_PARK_INTR; } @@ -131,8 +143,8 @@ _PySemaphore_PlatformWait(_PySemaphore *sema, PyTime_t timeout) } else { _Py_FatalErrorFormat(__func__, - "unexpected error from semaphore: %u (error: %u)", - wait, GetLastError()); + "unexpected error from semaphore: %u (error: %u, handle: %p)", + wait, GetLastError(), sema->platform_sem); } #elif defined(_Py_USE_SEMAPHORES) int err; @@ -241,7 +253,9 @@ _PySemaphore_Wakeup(_PySemaphore *sema) { #if defined(MS_WINDOWS) if (!ReleaseSemaphore(sema->platform_sem, 1, NULL)) { - Py_FatalError("parking_lot: ReleaseSemaphore failed"); + _Py_FatalErrorFormat(__func__, + "parking_lot: ReleaseSemaphore failed (error: %u, handle: %p)", + GetLastError(), sema->platform_sem); } #elif defined(_Py_USE_SEMAPHORES) int err = sem_post(&sema->platform_sem); @@ -332,7 +346,19 @@ _PyParkingLot_Park(const void *addr, const void *expected, size_t size, enqueue(bucket, addr, &wait); _PyRawMutex_Unlock(&bucket->mutex); - int res = _PySemaphore_Wait(&wait.sema, timeout_ns, detach); + PyThreadState *tstate = NULL; + if (detach) { + tstate = _PyThreadState_GET(); + if (tstate && _PyThreadState_IsAttached(tstate)) { + // Only detach if we are attached + PyEval_ReleaseThread(tstate); + } + else { + tstate = NULL; + } + } + + int res = _PySemaphore_Wait(&wait.sema, timeout_ns, 0); if (res == Py_PARK_OK) { goto done; } @@ -344,7 +370,7 @@ _PyParkingLot_Park(const void *addr, const void *expected, size_t size, // Another thread has started to unpark us. Wait until we process the // wakeup signal. do { - res = _PySemaphore_Wait(&wait.sema, -1, detach); + res = _PySemaphore_Wait(&wait.sema, -1, 0); } while (res != Py_PARK_OK); goto done; } @@ -356,6 +382,9 @@ _PyParkingLot_Park(const void *addr, const void *expected, size_t size, done: _PySemaphore_Destroy(&wait.sema); + if (tstate) { + PyEval_AcquireThread(tstate); + } return res; } diff --git a/Python/perf_jit_trampoline.c b/Python/perf_jit_trampoline.c index 1211e0e9f112b76..33c37e2f0ddb414 100644 --- a/Python/perf_jit_trampoline.c +++ b/Python/perf_jit_trampoline.c @@ -1,241 +1,353 @@ +/* + * Python Perf Trampoline Support - JIT Dump Implementation + * + * This file implements the perf jitdump API for Python's performance profiling + * integration. It allows perf (Linux performance analysis tool) to understand + * and profile dynamically generated Python bytecode by creating JIT dump files + * that perf can inject into its analysis. + * + * + * IMPORTANT: This file exports specific callback functions that are part of + * Python's internal API. Do not modify the function signatures or behavior + * of exported functions without coordinating with the Python core team. + * + * Usually the binary and libraries are mapped in separate region like below: + * + * address -> + * --+---------------------+--//--+---------------------+-- + * | .text | .data | ... | | .text | .data | ... | + * --+---------------------+--//--+---------------------+-- + * myprog libc.so + * + * So it'd be easy and straight-forward to find a mapped binary or library from an + * address. + * + * But for JIT code, the code arena only cares about the code section. But the + * resulting DSOs (which is generated by perf inject -j) contain ELF headers and + * unwind info too. Then it'd generate following address space with synthesized + * MMAP events. Let's say it has a sample between address B and C. + * + * sample + * | + * address -> A B v C + * --------------------------------------------------------------------------------------------------- + * /tmp/jitted-PID-0.so | (headers) | .text | unwind info | + * /tmp/jitted-PID-1.so | (headers) | .text | unwind info | + * /tmp/jitted-PID-2.so | (headers) | .text | unwind info | + * ... + * --------------------------------------------------------------------------------------------------- + * + * If it only maps the .text section, it'd find the jitted-PID-1.so but cannot see + * the unwind info. If it maps both .text section and unwind sections, the sample + * could be mapped to either jitted-PID-0.so or jitted-PID-1.so and it's confusing + * which one is right. So to make perf happy we have non-overlapping ranges for each + * DSO: + * + * address -> + * ------------------------------------------------------------------------------------------------------- + * /tmp/jitted-PID-0.so | (headers) | .text | unwind info | + * /tmp/jitted-PID-1.so | (headers) | .text | unwind info | + * /tmp/jitted-PID-2.so | (headers) | .text | unwind info | + * ... + * ------------------------------------------------------------------------------------------------------- + * + * As the trampolines are constant, we add a constant padding but in general the padding needs to have the + * size of the unwind info rounded to 16 bytes. In general, for our trampolines this is 0x50 + */ + + + #include "Python.h" #include "pycore_ceval.h" // _PyPerf_Callbacks #include "pycore_frame.h" #include "pycore_interp.h" #include "pycore_runtime.h" // _PyRuntime - #ifdef PY_HAVE_PERF_TRAMPOLINE -#include <fcntl.h> -#include <stdio.h> -#include <stdlib.h> -#include <sys/mman.h> // mmap() -#include <sys/types.h> -#include <unistd.h> // sysconf() -#include <sys/time.h> // gettimeofday() -#include <sys/syscall.h> - -// ---------------------------------- -// Perf jitdump API -// ---------------------------------- - -typedef struct { - FILE* perf_map; - PyThread_type_lock map_lock; - void* mapped_buffer; - size_t mapped_size; - int code_id; -} PerfMapJitState; - -static PerfMapJitState perf_jit_map_state; +/* Standard library includes for perf jitdump implementation */ +#include <elf.h> // ELF architecture constants +#include <fcntl.h> // File control operations +#include <stdio.h> // Standard I/O operations +#include <stdlib.h> // Standard library functions +#include <sys/mman.h> // Memory mapping functions (mmap) +#include <sys/types.h> // System data types +#include <unistd.h> // System calls (sysconf, getpid) +#include <sys/time.h> // Time functions (gettimeofday) +#include <sys/syscall.h> // System call interface + +// ============================================================================= +// CONSTANTS AND CONFIGURATION +// ============================================================================= /* -Usually the binary and libraries are mapped in separate region like below: - - address -> - --+---------------------+--//--+---------------------+-- - | .text | .data | ... | | .text | .data | ... | - --+---------------------+--//--+---------------------+-- - myprog libc.so - -So it'd be easy and straight-forward to find a mapped binary or library from an -address. - -But for JIT code, the code arena only cares about the code section. But the -resulting DSOs (which is generated by perf inject -j) contain ELF headers and -unwind info too. Then it'd generate following address space with synthesized -MMAP events. Let's say it has a sample between address B and C. - - sample - | - address -> A B v C - --------------------------------------------------------------------------------------------------- - /tmp/jitted-PID-0.so | (headers) | .text | unwind info | - /tmp/jitted-PID-1.so | (headers) | .text | unwind info | - /tmp/jitted-PID-2.so | (headers) | .text | unwind info | - ... - --------------------------------------------------------------------------------------------------- - -If it only maps the .text section, it'd find the jitted-PID-1.so but cannot see -the unwind info. If it maps both .text section and unwind sections, the sample -could be mapped to either jitted-PID-0.so or jitted-PID-1.so and it's confusing -which one is right. So to make perf happy we have non-overlapping ranges for each -DSO: - - address -> - ------------------------------------------------------------------------------------------------------- - /tmp/jitted-PID-0.so | (headers) | .text | unwind info | - /tmp/jitted-PID-1.so | (headers) | .text | unwind info | - /tmp/jitted-PID-2.so | (headers) | .text | unwind info | - ... - ------------------------------------------------------------------------------------------------------- - -As the trampolines are constant, we add a constant padding but in general the padding needs to have the -size of the unwind info rounded to 16 bytes. In general, for our trampolines this is 0x50 + * Memory layout considerations for perf jitdump: + * + * Perf expects non-overlapping memory regions for each JIT-compiled function. + * When perf processes the jitdump file, it creates synthetic DSO (Dynamic + * Shared Object) files that contain: + * - ELF headers + * - .text section (actual machine code) + * - Unwind information (for stack traces) + * + * To ensure proper address space layout, we add padding between code regions. + * This prevents address conflicts when perf maps the synthesized DSOs. + * + * Memory layout example: + * /tmp/jitted-PID-0.so: [headers][.text][unwind_info][padding] + * /tmp/jitted-PID-1.so: [headers][.text][unwind_info][padding] + * + * The padding size is now calculated automatically during initialization + * based on the actual unwind information requirements. */ -#define PERF_JIT_CODE_PADDING 0x100 +/* Convenient access to the global trampoline API state */ #define trampoline_api _PyRuntime.ceval.perf.trampoline_api -typedef uint64_t uword; -typedef const char* CodeComments; +/* Type aliases for clarity and portability */ +typedef uint64_t uword; // Word-sized unsigned integer +typedef const char* CodeComments; // Code comment strings -#define Pd "d" -#define MB (1024 * 1024) +/* Memory size constants */ +#define MB (1024 * 1024) // 1 Megabyte for buffer sizing -#define EM_386 3 -#define EM_X86_64 62 -#define EM_ARM 40 -#define EM_AARCH64 183 -#define EM_RISCV 243 +// ============================================================================= +// ARCHITECTURE-SPECIFIC DEFINITIONS +// ============================================================================= -#define TARGET_ARCH_IA32 0 -#define TARGET_ARCH_X64 0 -#define TARGET_ARCH_ARM 0 -#define TARGET_ARCH_ARM64 0 -#define TARGET_ARCH_RISCV32 0 -#define TARGET_ARCH_RISCV64 0 - -#define FLAG_generate_perf_jitdump 0 -#define FLAG_write_protect_code 0 -#define FLAG_write_protect_vm_isolate 0 -#define FLAG_code_comments 0 - -#define UNREACHABLE() - -static uword GetElfMachineArchitecture(void) { -#if TARGET_ARCH_IA32 - return EM_386; -#elif TARGET_ARCH_X64 +/* + * Returns the ELF machine architecture constant for the current platform. + * This is required for the jitdump header to correctly identify the target + * architecture for perf processing. + * + */ +static uint64_t GetElfMachineArchitecture(void) { +#if defined(__x86_64__) || defined(_M_X64) return EM_X86_64; -#elif TARGET_ARCH_ARM - return EM_ARM; -#elif TARGET_ARCH_ARM64 +#elif defined(__i386__) || defined(_M_IX86) + return EM_386; +#elif defined(__aarch64__) return EM_AARCH64; -#elif TARGET_ARCH_RISCV32 || TARGET_ARCH_RISCV64 +#elif defined(__arm__) || defined(_M_ARM) + return EM_ARM; +#elif defined(__riscv) return EM_RISCV; #else - UNREACHABLE(); + Py_UNREACHABLE(); // Unsupported architecture - should never reach here return 0; #endif } +// ============================================================================= +// PERF JITDUMP DATA STRUCTURES +// ============================================================================= + +/* + * Perf jitdump file format structures + * + * These structures define the binary format that perf expects for JIT dump files. + * The format is documented in the Linux perf tools source code and must match + * exactly for proper perf integration. + */ + +/* + * Jitdump file header - written once at the beginning of each jitdump file + * Contains metadata about the process and jitdump format version + */ typedef struct { - uint32_t magic; - uint32_t version; - uint32_t size; - uint32_t elf_mach_target; - uint32_t reserved; - uint32_t process_id; - uint64_t time_stamp; - uint64_t flags; + uint32_t magic; // Magic number (0x4A695444 = "JiTD") + uint32_t version; // Jitdump format version (currently 1) + uint32_t size; // Size of this header structure + uint32_t elf_mach_target; // Target architecture (from GetElfMachineArchitecture) + uint32_t reserved; // Reserved field (must be 0) + uint32_t process_id; // Process ID of the JIT compiler + uint64_t time_stamp; // Timestamp when jitdump was created + uint64_t flags; // Feature flags (currently unused) } Header; - enum PerfEvent { - PerfLoad = 0, - PerfMove = 1, - PerfDebugInfo = 2, - PerfClose = 3, - PerfUnwindingInfo = 4 +/* + * Perf event types supported by the jitdump format + * Each event type has a corresponding structure format + */ +enum PerfEvent { + PerfLoad = 0, // Code load event (new JIT function) + PerfMove = 1, // Code move event (function relocated) + PerfDebugInfo = 2, // Debug information event + PerfClose = 3, // JIT session close event + PerfUnwindingInfo = 4 // Stack unwinding information event }; +/* + * Base event structure - common header for all perf events + * Every event in the jitdump file starts with this structure + */ struct BaseEvent { - uint32_t event; - uint32_t size; - uint64_t time_stamp; - }; + uint32_t event; // Event type (from PerfEvent enum) + uint32_t size; // Total size of this event including payload + uint64_t time_stamp; // Timestamp when event occurred +}; +/* + * Code load event - indicates a new JIT-compiled function is available + * This is the most important event type for Python profiling + */ typedef struct { - struct BaseEvent base; - uint32_t process_id; - uint32_t thread_id; - uint64_t vma; - uint64_t code_address; - uint64_t code_size; - uint64_t code_id; + struct BaseEvent base; // Common event header + uint32_t process_id; // Process ID where code was generated + uint32_t thread_id; // Thread ID where code was generated + uint64_t vma; // Virtual memory address where code is loaded + uint64_t code_address; // Address of the actual machine code + uint64_t code_size; // Size of the machine code in bytes + uint64_t code_id; // Unique identifier for this code region + /* Followed by: + * - null-terminated function name string + * - raw machine code bytes + */ } CodeLoadEvent; +/* + * Code unwinding information event - provides DWARF data for stack traces + * Essential for proper stack unwinding during profiling + */ typedef struct { - struct BaseEvent base; - uint64_t unwind_data_size; - uint64_t eh_frame_hdr_size; - uint64_t mapped_size; + struct BaseEvent base; // Common event header + uint64_t unwind_data_size; // Size of the unwinding data + uint64_t eh_frame_hdr_size; // Size of the EH frame header + uint64_t mapped_size; // Total mapped size (with padding) + /* Followed by: + * - EH frame header + * - DWARF unwinding information + * - Padding to alignment boundary + */ } CodeUnwindingInfoEvent; -static const intptr_t nanoseconds_per_second = 1000000000; - -// Dwarf encoding constants +// ============================================================================= +// GLOBAL STATE MANAGEMENT +// ============================================================================= -static const uint8_t DwarfUData4 = 0x03; -static const uint8_t DwarfSData4 = 0x0b; -static const uint8_t DwarfPcRel = 0x10; -static const uint8_t DwarfDataRel = 0x30; -// static uint8_t DwarfOmit = 0xff; +/* + * Global state for the perf jitdump implementation + * + * This structure maintains all the state needed for generating jitdump files. + * It's designed as a singleton since there's typically only one jitdump file + * per Python process. + */ typedef struct { - unsigned char version; - unsigned char eh_frame_ptr_enc; - unsigned char fde_count_enc; - unsigned char table_enc; - int32_t eh_frame_ptr; - int32_t eh_fde_count; - int32_t from; - int32_t to; -} EhFrameHeader; + FILE* perf_map; // File handle for the jitdump file + PyThread_type_lock map_lock; // Thread synchronization lock + void* mapped_buffer; // Memory-mapped region (signals perf we're active) + size_t mapped_size; // Size of the mapped region + int code_id; // Counter for unique code region identifiers +} PerfMapJitState; + +/* Global singleton instance */ +static PerfMapJitState perf_jit_map_state; + +// ============================================================================= +// TIME UTILITIES +// ============================================================================= + +/* Time conversion constant */ +static const intptr_t nanoseconds_per_second = 1000000000; +/* + * Get current monotonic time in nanoseconds + * + * Monotonic time is preferred for event timestamps because it's not affected + * by system clock adjustments. This ensures consistent timing relationships + * between events even if the system clock is changed. + * + * Returns: Current monotonic time in nanoseconds since an arbitrary epoch + */ static int64_t get_current_monotonic_ticks(void) { struct timespec ts; if (clock_gettime(CLOCK_MONOTONIC, &ts) != 0) { - UNREACHABLE(); + Py_UNREACHABLE(); // Should never fail on supported systems return 0; } - // Convert to nanoseconds. + + /* Convert to nanoseconds for maximum precision */ int64_t result = ts.tv_sec; result *= nanoseconds_per_second; result += ts.tv_nsec; return result; } +/* + * Get current wall clock time in microseconds + * + * Used for the jitdump file header timestamp. Unlike monotonic time, + * this represents actual wall clock time that can be correlated with + * other system events. + * + * Returns: Current time in microseconds since Unix epoch + */ static int64_t get_current_time_microseconds(void) { - // gettimeofday has microsecond resolution. - struct timeval tv; - if (gettimeofday(&tv, NULL) < 0) { - UNREACHABLE(); - return 0; - } - return ((int64_t)(tv.tv_sec) * 1000000) + tv.tv_usec; + struct timeval tv; + if (gettimeofday(&tv, NULL) < 0) { + Py_UNREACHABLE(); // Should never fail on supported systems + return 0; + } + return ((int64_t)(tv.tv_sec) * 1000000) + tv.tv_usec; } +// ============================================================================= +// UTILITY FUNCTIONS +// ============================================================================= +/* + * Round up a value to the next multiple of a given number + * + * This is essential for maintaining proper alignment requirements in the + * jitdump format. Many structures need to be aligned to specific boundaries + * (typically 8 or 16 bytes) for efficient processing by perf. + * + * Args: + * value: The value to round up + * multiple: The multiple to round up to + * + * Returns: The smallest value >= input that is a multiple of 'multiple' + */ static size_t round_up(int64_t value, int64_t multiple) { if (multiple == 0) { - // Avoid division by zero - return value; + return value; // Avoid division by zero } int64_t remainder = value % multiple; if (remainder == 0) { - // Value is already a multiple of 'multiple' - return value; + return value; // Already aligned } - // Calculate the difference to the next multiple + /* Calculate how much to add to reach the next multiple */ int64_t difference = multiple - remainder; - - // Add the difference to the value int64_t rounded_up_value = value + difference; return rounded_up_value; } +// ============================================================================= +// FILE I/O UTILITIES +// ============================================================================= +/* + * Write data to the jitdump file with error handling + * + * This function ensures that all data is written to the file, handling + * partial writes that can occur with large buffers or when the system + * is under load. + * + * Args: + * buffer: Pointer to data to write + * size: Number of bytes to write + */ static void perf_map_jit_write_fully(const void* buffer, size_t size) { FILE* out_file = perf_jit_map_state.perf_map; const char* ptr = (const char*)(buffer); + while (size > 0) { const size_t written = fwrite(ptr, 1, size, out_file); if (written == 0) { - UNREACHABLE(); + Py_UNREACHABLE(); // Write failure - should be very rare break; } size -= written; @@ -243,284 +355,835 @@ static void perf_map_jit_write_fully(const void* buffer, size_t size) { } } +/* + * Write the jitdump file header + * + * The header must be written exactly once at the beginning of each jitdump + * file. It provides metadata that perf uses to parse the rest of the file. + * + * Args: + * pid: Process ID to include in the header + * out_file: File handle to write to (currently unused, uses global state) + */ static void perf_map_jit_write_header(int pid, FILE* out_file) { Header header; - header.magic = 0x4A695444; - header.version = 1; - header.size = sizeof(Header); - header.elf_mach_target = GetElfMachineArchitecture(); - header.process_id = pid; - header.time_stamp = get_current_time_microseconds(); - header.flags = 0; - perf_map_jit_write_fully(&header, sizeof(header)); -} - -static void* perf_map_jit_init(void) { - char filename[100]; - int pid = getpid(); - snprintf(filename, sizeof(filename) - 1, "/tmp/jit-%d.dump", pid); - const int fd = open(filename, O_CREAT | O_TRUNC | O_RDWR, 0666); - if (fd == -1) { - return NULL; - } - - const long page_size = sysconf(_SC_PAGESIZE); // NOLINT(runtime/int) - if (page_size == -1) { - close(fd); - return NULL; - } - // The perf jit interface forces us to map the first page of the file - // to signal that we are using the interface. - perf_jit_map_state.mapped_buffer = mmap(NULL, page_size, PROT_READ | PROT_EXEC, MAP_PRIVATE, fd, 0); - if (perf_jit_map_state.mapped_buffer == NULL) { - close(fd); - return NULL; - } - perf_jit_map_state.mapped_size = page_size; - perf_jit_map_state.perf_map = fdopen(fd, "w+"); - if (perf_jit_map_state.perf_map == NULL) { - close(fd); - return NULL; - } - setvbuf(perf_jit_map_state.perf_map, NULL, _IOFBF, 2 * MB); - perf_map_jit_write_header(pid, perf_jit_map_state.perf_map); - - perf_jit_map_state.map_lock = PyThread_allocate_lock(); - if (perf_jit_map_state.map_lock == NULL) { - fclose(perf_jit_map_state.perf_map); - return NULL; - } - perf_jit_map_state.code_id = 0; + /* Initialize header with required values */ + header.magic = 0x4A695444; // "JiTD" magic number + header.version = 1; // Current jitdump version + header.size = sizeof(Header); // Header size for validation + header.elf_mach_target = GetElfMachineArchitecture(); // Target architecture + header.reserved = 0; // padding reserved for future use + header.process_id = pid; // Process identifier + header.time_stamp = get_current_time_microseconds(); // Creation time + header.flags = 0; // No special flags currently used - trampoline_api.code_padding = PERF_JIT_CODE_PADDING; - return &perf_jit_map_state; + perf_map_jit_write_fully(&header, sizeof(header)); } -/* DWARF definitions. */ +// ============================================================================= +// DWARF CONSTANTS AND UTILITIES +// ============================================================================= + +/* + * DWARF (Debug With Arbitrary Record Formats) constants + * + * DWARF is a debugging data format used to provide stack unwinding information. + * These constants define the various encoding types and opcodes used in + * DWARF Call Frame Information (CFI) records. + */ +/* DWARF Call Frame Information version */ #define DWRF_CIE_VERSION 1 +/* DWARF CFA (Call Frame Address) opcodes */ enum { - DWRF_CFA_nop = 0x0, - DWRF_CFA_offset_extended = 0x5, - DWRF_CFA_def_cfa = 0xc, - DWRF_CFA_def_cfa_offset = 0xe, - DWRF_CFA_offset_extended_sf = 0x11, - DWRF_CFA_advance_loc = 0x40, - DWRF_CFA_offset = 0x80 + DWRF_CFA_nop = 0x0, // No operation + DWRF_CFA_offset_extended = 0x5, // Extended offset instruction + DWRF_CFA_def_cfa = 0xc, // Define CFA rule + DWRF_CFA_def_cfa_register = 0xd, // Define CFA register + DWRF_CFA_def_cfa_offset = 0xe, // Define CFA offset + DWRF_CFA_offset_extended_sf = 0x11, // Extended signed offset + DWRF_CFA_advance_loc = 0x40, // Advance location counter + DWRF_CFA_offset = 0x80, // Simple offset instruction +#if defined(__aarch64__) + DWRF_CFA_AARCH64_negate_ra_state = 0x2d, // Toggle return address signing state +#endif + DWRF_CFA_restore = 0xc0 // Restore register }; -enum - { - DWRF_EH_PE_absptr = 0x00, - DWRF_EH_PE_omit = 0xff, - - /* FDE data encoding. */ - DWRF_EH_PE_uleb128 = 0x01, - DWRF_EH_PE_udata2 = 0x02, - DWRF_EH_PE_udata4 = 0x03, - DWRF_EH_PE_udata8 = 0x04, - DWRF_EH_PE_sleb128 = 0x09, - DWRF_EH_PE_sdata2 = 0x0a, - DWRF_EH_PE_sdata4 = 0x0b, - DWRF_EH_PE_sdata8 = 0x0c, - DWRF_EH_PE_signed = 0x08, - - /* FDE flags. */ - DWRF_EH_PE_pcrel = 0x10, - DWRF_EH_PE_textrel = 0x20, - DWRF_EH_PE_datarel = 0x30, - DWRF_EH_PE_funcrel = 0x40, - DWRF_EH_PE_aligned = 0x50, - - DWRF_EH_PE_indirect = 0x80 - }; +/* DWARF Exception Handling pointer encodings */ +enum { + DWRF_EH_PE_absptr = 0x00, // Absolute pointer + DWRF_EH_PE_omit = 0xff, // Omitted value + + /* Data type encodings */ + DWRF_EH_PE_uleb128 = 0x01, // Unsigned LEB128 + DWRF_EH_PE_udata2 = 0x02, // Unsigned 2-byte + DWRF_EH_PE_udata4 = 0x03, // Unsigned 4-byte + DWRF_EH_PE_udata8 = 0x04, // Unsigned 8-byte + DWRF_EH_PE_sleb128 = 0x09, // Signed LEB128 + DWRF_EH_PE_sdata2 = 0x0a, // Signed 2-byte + DWRF_EH_PE_sdata4 = 0x0b, // Signed 4-byte + DWRF_EH_PE_sdata8 = 0x0c, // Signed 8-byte + DWRF_EH_PE_signed = 0x08, // Signed flag + + /* Reference type encodings */ + DWRF_EH_PE_pcrel = 0x10, // PC-relative + DWRF_EH_PE_textrel = 0x20, // Text-relative + DWRF_EH_PE_datarel = 0x30, // Data-relative + DWRF_EH_PE_funcrel = 0x40, // Function-relative + DWRF_EH_PE_aligned = 0x50, // Aligned + DWRF_EH_PE_indirect = 0x80 // Indirect +}; +/* Additional DWARF constants for debug information */ enum { DWRF_TAG_compile_unit = 0x11 }; - enum { DWRF_children_no = 0, DWRF_children_yes = 1 }; +enum { + DWRF_AT_name = 0x03, // Name attribute + DWRF_AT_stmt_list = 0x10, // Statement list + DWRF_AT_low_pc = 0x11, // Low PC address + DWRF_AT_high_pc = 0x12 // High PC address +}; +enum { + DWRF_FORM_addr = 0x01, // Address form + DWRF_FORM_data4 = 0x06, // 4-byte data + DWRF_FORM_string = 0x08 // String form +}; -enum { DWRF_AT_name = 0x03, DWRF_AT_stmt_list = 0x10, DWRF_AT_low_pc = 0x11, DWRF_AT_high_pc = 0x12 }; - -enum { DWRF_FORM_addr = 0x01, DWRF_FORM_data4 = 0x06, DWRF_FORM_string = 0x08 }; - -enum { DWRF_LNS_extended_op = 0, DWRF_LNS_copy = 1, DWRF_LNS_advance_pc = 2, DWRF_LNS_advance_line = 3 }; +/* Line number program opcodes */ +enum { + DWRF_LNS_extended_op = 0, // Extended opcode + DWRF_LNS_copy = 1, // Copy operation + DWRF_LNS_advance_pc = 2, // Advance program counter + DWRF_LNS_advance_line = 3 // Advance line number +}; -enum { DWRF_LNE_end_sequence = 1, DWRF_LNE_set_address = 2 }; +/* Line number extended opcodes */ +enum { + DWRF_LNE_end_sequence = 1, // End of sequence + DWRF_LNE_set_address = 2 // Set address +}; +/* + * Architecture-specific DWARF register numbers + * + * These constants define the register numbering scheme used by DWARF + * for each supported architecture. The numbers must match the ABI + * specification for proper stack unwinding. + */ enum { #ifdef __x86_64__ - /* Yes, the order is strange, but correct. */ - DWRF_REG_AX, - DWRF_REG_DX, - DWRF_REG_CX, - DWRF_REG_BX, - DWRF_REG_SI, - DWRF_REG_DI, - DWRF_REG_BP, - DWRF_REG_SP, - DWRF_REG_8, - DWRF_REG_9, - DWRF_REG_10, - DWRF_REG_11, - DWRF_REG_12, - DWRF_REG_13, - DWRF_REG_14, - DWRF_REG_15, - DWRF_REG_RA, + /* x86_64 register numbering (note: order is defined by x86_64 ABI) */ + DWRF_REG_AX, // RAX + DWRF_REG_DX, // RDX + DWRF_REG_CX, // RCX + DWRF_REG_BX, // RBX + DWRF_REG_SI, // RSI + DWRF_REG_DI, // RDI + DWRF_REG_BP, // RBP + DWRF_REG_SP, // RSP + DWRF_REG_8, // R8 + DWRF_REG_9, // R9 + DWRF_REG_10, // R10 + DWRF_REG_11, // R11 + DWRF_REG_12, // R12 + DWRF_REG_13, // R13 + DWRF_REG_14, // R14 + DWRF_REG_15, // R15 + DWRF_REG_RA, // Return address (RIP) #elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) - DWRF_REG_SP = 31, - DWRF_REG_RA = 30, + /* AArch64 register numbering */ + DWRF_REG_FP = 29, // Frame Pointer + DWRF_REG_RA = 30, // Link register (return address) + DWRF_REG_SP = 31, // Stack pointer #else # error "Unsupported target architecture" #endif }; -typedef struct ELFObjectContext -{ - uint8_t* p; /* Pointer to next address in obj.space. */ - uint8_t* startp; /* Pointer to start address in obj.space. */ - uint8_t* eh_frame_p; /* Pointer to start address in obj.space. */ - uint32_t code_size; /* Size of machine code. */ +/* DWARF encoding constants used in EH frame headers */ +static const uint8_t DwarfUData4 = 0x03; // Unsigned 4-byte data +static const uint8_t DwarfSData4 = 0x0b; // Signed 4-byte data +static const uint8_t DwarfPcRel = 0x10; // PC-relative encoding +static const uint8_t DwarfDataRel = 0x30; // Data-relative encoding + +// ============================================================================= +// ELF OBJECT CONTEXT +// ============================================================================= + +/* + * Context for building ELF/DWARF structures + * + * This structure maintains state while constructing DWARF unwind information. + * It acts as a simple buffer manager with pointers to track current position + * and important landmarks within the buffer. + */ +typedef struct ELFObjectContext { + uint8_t* p; // Current write position in buffer + uint8_t* startp; // Start of buffer (for offset calculations) + uint8_t* eh_frame_p; // Start of EH frame data (for relative offsets) + uint8_t* fde_p; // Start of FDE data (for PC-relative calculations) + uint32_t code_size; // Size of the code being described } ELFObjectContext; -/* Append a null-terminated string. */ -static uint32_t -elfctx_append_string(ELFObjectContext* ctx, const char* str) -{ +/* + * EH Frame Header structure for DWARF unwinding + * + * This structure provides metadata about the DWARF unwinding information + * that follows. It's required by the perf jitdump format to enable proper + * stack unwinding during profiling. + */ +typedef struct { + unsigned char version; // EH frame version (always 1) + unsigned char eh_frame_ptr_enc; // Encoding of EH frame pointer + unsigned char fde_count_enc; // Encoding of FDE count + unsigned char table_enc; // Encoding of table entries + int32_t eh_frame_ptr; // Pointer to EH frame data + int32_t eh_fde_count; // Number of FDEs (Frame Description Entries) + int32_t from; // Start address of code range + int32_t to; // End address of code range +} EhFrameHeader; + +// ============================================================================= +// DWARF GENERATION UTILITIES +// ============================================================================= + +/* + * Append a null-terminated string to the ELF context buffer + * + * Args: + * ctx: ELF object context + * str: String to append (must be null-terminated) + * + * Returns: Offset from start of buffer where string was written + */ +static uint32_t elfctx_append_string(ELFObjectContext* ctx, const char* str) { uint8_t* p = ctx->p; uint32_t ofs = (uint32_t)(p - ctx->startp); + + /* Copy string including null terminator */ do { *p++ = (uint8_t)*str; } while (*str++); + ctx->p = p; return ofs; } -/* Append a SLEB128 value. */ -static void -elfctx_append_sleb128(ELFObjectContext* ctx, int32_t v) -{ +/* + * Append a SLEB128 (Signed Little Endian Base 128) value + * + * SLEB128 is a variable-length encoding used extensively in DWARF. + * It efficiently encodes small numbers in fewer bytes. + * + * Args: + * ctx: ELF object context + * v: Signed value to encode + */ +static void elfctx_append_sleb128(ELFObjectContext* ctx, int32_t v) { uint8_t* p = ctx->p; + + /* Encode 7 bits at a time, with continuation bit in MSB */ for (; (uint32_t)(v + 0x40) >= 0x80; v >>= 7) { - *p++ = (uint8_t)((v & 0x7f) | 0x80); + *p++ = (uint8_t)((v & 0x7f) | 0x80); // Set continuation bit } - *p++ = (uint8_t)(v & 0x7f); + *p++ = (uint8_t)(v & 0x7f); // Final byte without continuation bit + ctx->p = p; } -/* Append a ULEB128 to buffer. */ -static void -elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) -{ +/* + * Append a ULEB128 (Unsigned Little Endian Base 128) value + * + * Similar to SLEB128 but for unsigned values. + * + * Args: + * ctx: ELF object context + * v: Unsigned value to encode + */ +static void elfctx_append_uleb128(ELFObjectContext* ctx, uint32_t v) { uint8_t* p = ctx->p; + + /* Encode 7 bits at a time, with continuation bit in MSB */ for (; v >= 0x80; v >>= 7) { - *p++ = (char)((v & 0x7f) | 0x80); + *p++ = (char)((v & 0x7f) | 0x80); // Set continuation bit } - *p++ = (char)v; + *p++ = (char)v; // Final byte without continuation bit + ctx->p = p; } -/* Shortcuts to generate DWARF structures. */ -#define DWRF_U8(x) (*p++ = (x)) -#define DWRF_I8(x) (*(int8_t*)p = (x), p++) -#define DWRF_U16(x) (*(uint16_t*)p = (x), p += 2) -#define DWRF_U32(x) (*(uint32_t*)p = (x), p += 4) -#define DWRF_ADDR(x) (*(uintptr_t*)p = (x), p += sizeof(uintptr_t)) -#define DWRF_UV(x) (ctx->p = p, elfctx_append_uleb128(ctx, (x)), p = ctx->p) -#define DWRF_SV(x) (ctx->p = p, elfctx_append_sleb128(ctx, (x)), p = ctx->p) -#define DWRF_STR(str) (ctx->p = p, elfctx_append_string(ctx, (str)), p = ctx->p) -#define DWRF_ALIGNNOP(s) \ - while ((uintptr_t)p & ((s)-1)) { \ - *p++ = DWRF_CFA_nop; \ +/* + * Macros for generating DWARF structures + * + * These macros provide a convenient way to write various data types + * to the DWARF buffer while automatically advancing the pointer. + */ +#define DWRF_U8(x) (*p++ = (x)) // Write unsigned 8-bit +#define DWRF_I8(x) (*(int8_t*)p = (x), p++) // Write signed 8-bit +#define DWRF_U16(x) (*(uint16_t*)p = (x), p += 2) // Write unsigned 16-bit +#define DWRF_U32(x) (*(uint32_t*)p = (x), p += 4) // Write unsigned 32-bit +#define DWRF_ADDR(x) (*(uintptr_t*)p = (x), p += sizeof(uintptr_t)) // Write address +#define DWRF_UV(x) (ctx->p = p, elfctx_append_uleb128(ctx, (x)), p = ctx->p) // Write ULEB128 +#define DWRF_SV(x) (ctx->p = p, elfctx_append_sleb128(ctx, (x)), p = ctx->p) // Write SLEB128 +#define DWRF_STR(str) (ctx->p = p, elfctx_append_string(ctx, (str)), p = ctx->p) // Write string + +/* Align to specified boundary with NOP instructions */ +#define DWRF_ALIGNNOP(s) \ + while ((uintptr_t)p & ((s)-1)) { \ + *p++ = DWRF_CFA_nop; \ } -#define DWRF_SECTION(name, stmt) \ - { \ - uint32_t* szp_##name = (uint32_t*)p; \ - p += 4; \ - stmt; \ - *szp_##name = (uint32_t)((p - (uint8_t*)szp_##name) - 4); \ + +/* Write a DWARF section with automatic size calculation */ +#define DWRF_SECTION(name, stmt) \ + { \ + uint32_t* szp_##name = (uint32_t*)p; \ + p += 4; \ + stmt; \ + *szp_##name = (uint32_t)((p - (uint8_t*)szp_##name) - 4); \ } -/* Initialize .eh_frame section. */ -static void -elf_init_ehframe(ELFObjectContext* ctx) -{ +// ============================================================================= +// DWARF EH FRAME GENERATION +// ============================================================================= + +static void elf_init_ehframe(ELFObjectContext* ctx); + +/* + * Initialize DWARF .eh_frame section for a code region + * + * The .eh_frame section contains Call Frame Information (CFI) that describes + * how to unwind the stack at any point in the code. This is essential for + * proper profiling as it allows perf to generate accurate call graphs. + * + * The function generates two main components: + * 1. CIE (Common Information Entry) - describes calling conventions + * 2. FDE (Frame Description Entry) - describes specific function unwinding + * + * Args: + * ctx: ELF object context containing code size and buffer pointers + */ +static size_t calculate_eh_frame_size(void) { + /* Calculate the EH frame size for the trampoline function */ + extern void *_Py_trampoline_func_start; + extern void *_Py_trampoline_func_end; + + size_t code_size = (char*)&_Py_trampoline_func_end - (char*)&_Py_trampoline_func_start; + + ELFObjectContext ctx; + char buffer[1024]; // Buffer for DWARF data (1KB should be sufficient) + ctx.code_size = code_size; + ctx.startp = ctx.p = (uint8_t*)buffer; + ctx.fde_p = NULL; + + elf_init_ehframe(&ctx); + return ctx.p - ctx.startp; +} + +static void elf_init_ehframe(ELFObjectContext* ctx) { uint8_t* p = ctx->p; - uint8_t* framep = p; - - /* Emit DWARF EH CIE. */ - DWRF_SECTION(CIE, DWRF_U32(0); /* Offset to CIE itself. */ - DWRF_U8(DWRF_CIE_VERSION); - DWRF_STR("zR"); /* Augmentation. */ - DWRF_UV(1); /* Code alignment factor. */ - DWRF_SV(-(int64_t)sizeof(uintptr_t)); /* Data alignment factor. */ - DWRF_U8(DWRF_REG_RA); /* Return address register. */ - DWRF_UV(1); - DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); /* Augmentation data. */ - DWRF_U8(DWRF_CFA_def_cfa); DWRF_UV(DWRF_REG_SP); DWRF_UV(sizeof(uintptr_t)); - DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); DWRF_UV(1); - DWRF_ALIGNNOP(sizeof(uintptr_t)); - ) + uint8_t* framep = p; // Remember start of frame data + + /* + * DWARF Unwind Table for Trampoline Function + * + * This section defines DWARF Call Frame Information (CFI) using encoded macros + * like `DWRF_U8`, `DWRF_UV`, and `DWRF_SECTION` to describe how the trampoline function + * preserves and restores registers. This is used by profiling tools (e.g., `perf`) + * and debuggers for stack unwinding in JIT-compiled code. + * + * ------------------------------------------------- + * TO REGENERATE THIS TABLE FROM GCC OBJECTS: + * ------------------------------------------------- + * + * 1. Create a trampoline source file (e.g., `trampoline.c`): + * + * #include <Python.h> + * typedef PyObject* (*py_evaluator)(void*, void*, int); + * PyObject* trampoline(void *ts, void *f, int throwflag, py_evaluator evaluator) { + * return evaluator(ts, f, throwflag); + * } + * + * 2. Compile to an object file with frame pointer preservation: + * + * gcc trampoline.c -I. -I./Include -O2 -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -c + * + * 3. Extract DWARF unwind info from the object file: + * + * readelf -w trampoline.o + * + * Example output from `.eh_frame`: + * + * 00000000 CIE + * Version: 1 + * Augmentation: "zR" + * Code alignment factor: 4 + * Data alignment factor: -8 + * Return address column: 30 + * DW_CFA_def_cfa: r31 (sp) ofs 0 + * + * 00000014 FDE cie=00000000 pc=0..14 + * DW_CFA_advance_loc: 4 + * DW_CFA_def_cfa_offset: 16 + * DW_CFA_offset: r29 at cfa-16 + * DW_CFA_offset: r30 at cfa-8 + * DW_CFA_advance_loc: 12 + * DW_CFA_restore: r30 + * DW_CFA_restore: r29 + * DW_CFA_def_cfa_offset: 0 + * + * -- These values can be verified by comparing with `readelf -w` or `llvm-dwarfdump --eh-frame`. + * + * ---------------------------------- + * HOW TO TRANSLATE TO DWRF_* MACROS: + * ---------------------------------- + * + * After compiling your trampoline with: + * + * gcc trampoline.c -I. -I./Include -O2 -fno-omit-frame-pointer -mno-omit-leaf-frame-pointer -c + * + * run: + * + * readelf -w trampoline.o + * + * to inspect the generated `.eh_frame` data. You will see two main components: + * + * 1. A CIE (Common Information Entry): shared configuration used by all FDEs. + * 2. An FDE (Frame Description Entry): function-specific unwind instructions. + * + * --------------------- + * Translating the CIE: + * --------------------- + * From `readelf -w`, you might see: + * + * 00000000 0000000000000010 00000000 CIE + * Version: 1 + * Augmentation: "zR" + * Code alignment factor: 4 + * Data alignment factor: -8 + * Return address column: 30 + * Augmentation data: 1b + * DW_CFA_def_cfa: r31 (sp) ofs 0 + * + * Map this to: + * + * DWRF_SECTION(CIE, + * DWRF_U32(0); // CIE ID (always 0 for CIEs) + * DWRF_U8(DWRF_CIE_VERSION); // Version: 1 + * DWRF_STR("zR"); // Augmentation string "zR" + * DWRF_UV(4); // Code alignment factor = 4 + * DWRF_SV(-8); // Data alignment factor = -8 + * DWRF_U8(DWRF_REG_RA); // Return address register (e.g., x30 = 30) + * DWRF_UV(1); // Augmentation data length = 1 + * DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); // Encoding for FDE pointers + * + * DWRF_U8(DWRF_CFA_def_cfa); // DW_CFA_def_cfa + * DWRF_UV(DWRF_REG_SP); // Register: SP (r31) + * DWRF_UV(0); // Offset = 0 + * + * DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer size boundary + * ) + * + * Notes: + * - Use `DWRF_UV` for unsigned LEB128, `DWRF_SV` for signed LEB128. + * - `DWRF_REG_RA` and `DWRF_REG_SP` are architecture-defined constants. + * + * --------------------- + * Translating the FDE: + * --------------------- + * From `readelf -w`: + * + * 00000014 0000000000000020 00000018 FDE cie=00000000 pc=0000000000000000..0000000000000014 + * DW_CFA_advance_loc: 4 + * DW_CFA_def_cfa_offset: 16 + * DW_CFA_offset: r29 at cfa-16 + * DW_CFA_offset: r30 at cfa-8 + * DW_CFA_advance_loc: 12 + * DW_CFA_restore: r30 + * DW_CFA_restore: r29 + * DW_CFA_def_cfa_offset: 0 + * + * Map the FDE header and instructions to: + * + * DWRF_SECTION(FDE, + * DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (relative from here) + * DWRF_U32(pc_relative_offset); // PC-relative location of the code (calculated dynamically) + * DWRF_U32(ctx->code_size); // Code range covered by this FDE + * DWRF_U8(0); // Augmentation data length (none) + * + * DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance location by 1 unit (1 * 4 = 4 bytes) + * DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 + * DWRF_UV(16); + * + * DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Save x29 (frame pointer) + * DWRF_UV(2); // At offset 2 * 8 = 16 bytes + * + * DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Save x30 (return address) + * DWRF_UV(1); // At offset 1 * 8 = 8 bytes + * + * DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance location by 3 units (3 * 4 = 12 bytes) + * + * DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // Restore x30 + * DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // Restore x29 + * + * DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + * DWRF_UV(0); + * ) + * + * To regenerate: + * 1. Get the `code alignment factor`, `data alignment factor`, and `RA column` from the CIE. + * 2. Note the range of the function from the FDE's `pc=...` line and map it to the JIT code as + * the code is in a different address space every time. + * 3. For each `DW_CFA_*` entry, use the corresponding `DWRF_*` macro: + * - `DW_CFA_def_cfa_offset` → DWRF_U8(DWRF_CFA_def_cfa_offset), DWRF_UV(value) + * - `DW_CFA_offset: rX` → DWRF_U8(DWRF_CFA_offset | reg), DWRF_UV(offset) + * - `DW_CFA_restore: rX` → DWRF_U8(DWRF_CFA_offset | reg) // restore is same as reusing offset + * - `DW_CFA_advance_loc: N` → DWRF_U8(DWRF_CFA_advance_loc | (N / code_alignment_factor)) + * 4. Use `DWRF_REG_FP`, `DWRF_REG_RA`, etc., for register numbers. + * 5. Use `sizeof(uintptr_t)` (typically 8) for pointer size calculations and alignment. + */ + + /* + * Emit DWARF EH CIE (Common Information Entry) + * + * The CIE describes the calling conventions and basic unwinding rules + * that apply to all functions in this compilation unit. + */ + DWRF_SECTION(CIE, + DWRF_U32(0); // CIE ID (0 indicates this is a CIE) + DWRF_U8(DWRF_CIE_VERSION); // CIE version (1) + DWRF_STR("zR"); // Augmentation string ("zR" = has LSDA) +#ifdef __x86_64__ + DWRF_UV(1); // Code alignment factor (x86_64: 1 byte) +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + DWRF_UV(4); // Code alignment factor (AArch64: 4 bytes per instruction) +#endif + DWRF_SV(-(int64_t)sizeof(uintptr_t)); // Data alignment factor (negative) + DWRF_U8(DWRF_REG_RA); // Return address register number + DWRF_UV(1); // Augmentation data length + DWRF_U8(DWRF_EH_PE_pcrel | DWRF_EH_PE_sdata4); // FDE pointer encoding - ctx->eh_frame_p = p; + /* Initial CFI instructions - describe default calling convention */ +#ifdef __x86_64__ + /* x86_64 initial CFI state */ + DWRF_U8(DWRF_CFA_def_cfa); // Define CFA (Call Frame Address) + DWRF_UV(DWRF_REG_SP); // CFA = SP register + DWRF_UV(sizeof(uintptr_t)); // CFA = SP + pointer_size + DWRF_U8(DWRF_CFA_offset|DWRF_REG_RA); // Return address is saved + DWRF_UV(1); // At offset 1 from CFA +#elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) + /* AArch64 initial CFI state */ + DWRF_U8(DWRF_CFA_def_cfa); // Define CFA (Call Frame Address) + DWRF_UV(DWRF_REG_SP); // CFA = SP register + DWRF_UV(0); // CFA = SP + 0 (AArch64 starts with offset 0) + // No initial register saves in AArch64 CIE +#endif + DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer boundary + ) - /* Emit DWARF EH FDE. */ - DWRF_SECTION(FDE, DWRF_U32((uint32_t)(p - framep)); /* Offset to CIE. */ - DWRF_U32(-0x30); /* Machine code offset relative to .text. */ - DWRF_U32(ctx->code_size); /* Machine code length. */ - DWRF_U8(0); /* Augmentation data. */ - /* Registers saved in CFRAME. */ + ctx->eh_frame_p = p; // Remember start of FDE data + + /* + * Emit DWARF EH FDE (Frame Description Entry) + * + * The FDE describes unwinding information specific to this function. + * It references the CIE and provides function-specific CFI instructions. + * + * The PC-relative offset is calculated after the entire EH frame is built + * to ensure accurate positioning relative to the synthesized DSO layout. + */ + DWRF_SECTION(FDE, + DWRF_U32((uint32_t)(p - framep)); // Offset to CIE (backwards reference) + ctx->fde_p = p; // Remember where PC offset field is located for later calculation + DWRF_U32(0); // Placeholder for PC-relative offset (calculated at end of elf_init_ehframe) + DWRF_U32(ctx->code_size); // Address range covered by this FDE (code length) + DWRF_U8(0); // Augmentation data length (none) + + /* + * Architecture-specific CFI instructions + * + * These instructions describe how registers are saved and restored + * during function calls. Each architecture has different calling + * conventions and register usage patterns. + */ #ifdef __x86_64__ - DWRF_U8(DWRF_CFA_advance_loc | 4); - DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); - DWRF_U8(DWRF_CFA_advance_loc | 6); - DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(8); - /* Extra registers saved for JIT-compiled code. */ + /* x86_64 calling convention unwinding rules with frame pointer */ +# if defined(__CET__) && (__CET__ & 1) + DWRF_U8(DWRF_CFA_advance_loc | 4); // Advance past endbr64 (4 bytes) +# endif + DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance past push %rbp (1 byte) + DWRF_U8(DWRF_CFA_def_cfa_offset); // def_cfa_offset 16 + DWRF_UV(16); // New offset: SP + 16 + DWRF_U8(DWRF_CFA_offset | DWRF_REG_BP); // offset r6 at cfa-16 + DWRF_UV(2); // Offset factor: 2 * 8 = 16 bytes + DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance past mov %rsp,%rbp (3 bytes) + DWRF_U8(DWRF_CFA_def_cfa_register); // def_cfa_register r6 + DWRF_UV(DWRF_REG_BP); // Use base pointer register + DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance past call *%rcx (2 bytes) + pop %rbp (1 byte) = 3 + DWRF_U8(DWRF_CFA_def_cfa); // def_cfa r7 ofs 8 + DWRF_UV(DWRF_REG_SP); // Use stack pointer register + DWRF_UV(8); // New offset: SP + 8 #elif defined(__aarch64__) && defined(__AARCH64EL__) && !defined(__ILP32__) - DWRF_U8(DWRF_CFA_advance_loc | 1); - DWRF_U8(DWRF_CFA_def_cfa_offset); DWRF_UV(16); - DWRF_U8(DWRF_CFA_offset | 29); DWRF_UV(2); - DWRF_U8(DWRF_CFA_offset | 30); DWRF_UV(1); - DWRF_U8(DWRF_CFA_advance_loc | 3); - DWRF_U8(DWRF_CFA_offset | -(64 - 29)); - DWRF_U8(DWRF_CFA_offset | -(64 - 30)); - DWRF_U8(DWRF_CFA_def_cfa_offset); - DWRF_UV(0); + /* AArch64 calling convention unwinding rules */ +#if defined(__ARM_FEATURE_PAC_DEFAULT) || \ + (defined(__ARM_FEATURE_BTI_DEFAULT) && __ARM_FEATURE_BTI_DEFAULT == 1) + DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance past SIGN_LR (4 bytes) +#endif +#if defined(__ARM_FEATURE_PAC_DEFAULT) + DWRF_U8(DWRF_CFA_AARCH64_negate_ra_state); // Saved LR is PAC-signed from here +#endif + DWRF_U8(DWRF_CFA_advance_loc | 1); // Advance by 1 instruction (4 bytes) + DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 16 + DWRF_UV(16); // Stack pointer moved by 16 bytes + DWRF_U8(DWRF_CFA_offset | DWRF_REG_FP); // x29 (frame pointer) saved + DWRF_UV(2); // At CFA-16 (2 * 8 = 16 bytes from CFA) + DWRF_U8(DWRF_CFA_offset | DWRF_REG_RA); // x30 (link register) saved + DWRF_UV(1); // At CFA-8 (1 * 8 = 8 bytes from CFA) + DWRF_U8(DWRF_CFA_advance_loc | 3); // Advance by 3 instructions (12 bytes) +#if defined(__ARM_FEATURE_PAC_DEFAULT) + DWRF_U8(DWRF_CFA_AARCH64_negate_ra_state); // LR is authenticated, no longer PAC-signed +#endif + DWRF_U8(DWRF_CFA_restore | DWRF_REG_RA); // Restore x30 - NO DWRF_UV() after this! + DWRF_U8(DWRF_CFA_restore | DWRF_REG_FP); // Restore x29 - NO DWRF_UV() after this! + DWRF_U8(DWRF_CFA_def_cfa_offset); // CFA = SP + 0 (stack restored) + DWRF_UV(0); // Back to original stack position #else # error "Unsupported target architecture" #endif - DWRF_ALIGNNOP(sizeof(uintptr_t));) - ctx->p = p; + DWRF_ALIGNNOP(sizeof(uintptr_t)); // Align to pointer boundary + ) + + ctx->p = p; // Update context pointer to end of generated data + + /* Calculate and update the PC-relative offset in the FDE + * + * When perf processes the jitdump, it creates a synthesized DSO with this layout: + * + * Synthesized DSO Memory Layout: + * ┌─────────────────────────────────────────────────────────────┐ < code_start + * │ Code Section │ + * │ (round_up(code_size, 8) bytes) │ + * ├─────────────────────────────────────────────────────────────┤ < start of EH frame data + * │ EH Frame Data │ + * │ ┌─────────────────────────────────────────────────────┐ │ + * │ │ CIE data │ │ + * │ └─────────────────────────────────────────────────────┘ │ + * │ ┌─────────────────────────────────────────────────────┐ │ + * │ │ FDE Header: │ │ + * │ │ - CIE offset (4 bytes) │ │ + * │ │ - PC offset (4 bytes) <─ fde_offset_in_frame ─────┼────┼─> points to code_start + * │ │ - address range (4 bytes) │ │ (this specific field) + * │ │ CFI Instructions... │ │ + * │ └─────────────────────────────────────────────────────┘ │ + * ├─────────────────────────────────────────────────────────────┤ < reference_point + * │ EhFrameHeader │ + * │ (navigation metadata) │ + * └─────────────────────────────────────────────────────────────┘ + * + * The PC offset field in the FDE must contain the distance from itself to code_start: + * + * distance = code_start - fde_pc_field + * + * Where: + * fde_pc_field_location = reference_point - eh_frame_size + fde_offset_in_frame + * code_start_location = reference_point - eh_frame_size - round_up(code_size, 8) + * + * Therefore: + * distance = code_start_location - fde_pc_field_location + * = (ref - eh_frame_size - rounded_code_size) - (ref - eh_frame_size + fde_offset_in_frame) + * = -rounded_code_size - fde_offset_in_frame + * = -(round_up(code_size, 8) + fde_offset_in_frame) + * + * Note: fde_offset_in_frame is the offset from EH frame start to the PC offset field, + * + */ + if (ctx->fde_p != NULL) { + int32_t fde_offset_in_frame = (ctx->fde_p - ctx->startp); + int32_t rounded_code_size = round_up(ctx->code_size, 8); + int32_t pc_relative_offset = -(rounded_code_size + fde_offset_in_frame); + + + // Update the PC-relative offset in the FDE + *(int32_t*)ctx->fde_p = pc_relative_offset; + } } +// ============================================================================= +// JITDUMP INITIALIZATION +// ============================================================================= + +/* + * Initialize the perf jitdump interface + * + * This function sets up everything needed to generate jitdump files: + * 1. Creates the jitdump file with a unique name + * 2. Maps the first page to signal perf that we're using the interface + * 3. Writes the jitdump header + * 4. Initializes synchronization primitives + * + * The memory mapping is crucial - perf detects jitdump files by scanning + * for processes that have mapped files matching the pattern /tmp/jit-*.dump + * + * Returns: Pointer to initialized state, or NULL on failure + */ +static void* perf_map_jit_init(void) { + char filename[100]; + int pid = getpid(); + + /* Create unique filename based on process ID */ + snprintf(filename, sizeof(filename) - 1, "/tmp/jit-%d.dump", pid); + + /* Create/open the jitdump file with appropriate permissions */ + const int fd = open(filename, O_CREAT | O_TRUNC | O_RDWR, 0666); + if (fd == -1) { + return NULL; // Failed to create file + } + + /* Get system page size for memory mapping */ + const long page_size = sysconf(_SC_PAGESIZE); + if (page_size == -1) { + close(fd); + return NULL; // Failed to get page size + } + + /* + * Map the first page of the jitdump file + * + * This memory mapping serves as a signal to perf that this process + * is generating JIT code. Perf scans /proc/.../maps looking for mapped + * files that match the jitdump naming pattern. + * + * The mapping must be PROT_READ | PROT_EXEC to be detected by perf. + */ + perf_jit_map_state.mapped_buffer = mmap( + NULL, // Let kernel choose address + page_size, // Map one page + PROT_READ | PROT_EXEC, // Read and execute permissions (required by perf) + MAP_PRIVATE, // Private mapping + fd, // File descriptor + 0 // Offset 0 (first page) + ); + + if (perf_jit_map_state.mapped_buffer == MAP_FAILED) { + perf_jit_map_state.mapped_buffer = NULL; + close(fd); + return NULL; // Memory mapping failed + } + + perf_jit_map_state.mapped_size = page_size; + + /* Convert file descriptor to FILE* for easier I/O operations */ + perf_jit_map_state.perf_map = fdopen(fd, "w+"); + if (perf_jit_map_state.perf_map == NULL) { + close(fd); + return NULL; // Failed to create FILE* + } + + /* + * Set up file buffering for better performance + * + * We use a large buffer (2MB) because jitdump files can be written + * frequently during program execution. Buffering reduces system call + * overhead and improves overall performance. + */ + setvbuf(perf_jit_map_state.perf_map, NULL, _IOFBF, 2 * MB); + + /* Write the jitdump file header */ + perf_map_jit_write_header(pid, perf_jit_map_state.perf_map); + + /* + * Initialize thread synchronization lock + * + * Multiple threads may attempt to write to the jitdump file + * simultaneously. This lock ensures thread-safe access to the + * global jitdump state. + */ + perf_jit_map_state.map_lock = PyThread_allocate_lock(); + if (perf_jit_map_state.map_lock == NULL) { + fclose(perf_jit_map_state.perf_map); + return NULL; // Failed to create lock + } + + /* Initialize code ID counter */ + perf_jit_map_state.code_id = 0; + + /* Calculate padding size based on actual unwind info requirements */ + size_t eh_frame_size = calculate_eh_frame_size(); + size_t unwind_data_size = sizeof(EhFrameHeader) + eh_frame_size; + trampoline_api.code_padding = round_up(unwind_data_size, 16); + + return &perf_jit_map_state; +} + +// ============================================================================= +// MAIN JITDUMP ENTRY WRITING +// ============================================================================= + +/* + * Write a complete jitdump entry for a Python function + * + * This is the main function called by Python's trampoline system whenever + * a new piece of JIT-compiled code needs to be recorded. It writes both + * the unwinding information and the code load event to the jitdump file. + * + * The function performs these steps: + * 1. Initialize jitdump system if not already done + * 2. Extract function name and filename from Python code object + * 3. Generate DWARF unwinding information + * 4. Write unwinding info event to jitdump file + * 5. Write code load event to jitdump file + * + * Args: + * state: Jitdump state (currently unused, uses global state) + * code_addr: Address where the compiled code resides + * code_size: Size of the compiled code in bytes + * co: Python code object containing metadata + * + * IMPORTANT: This function signature is part of Python's internal API + * and must not be changed without coordinating with core Python development. + */ static void perf_map_jit_write_entry(void *state, const void *code_addr, - unsigned int code_size, PyCodeObject *co) + unsigned int code_size, PyCodeObject *co) { - + /* Initialize jitdump system on first use */ if (perf_jit_map_state.perf_map == NULL) { void* ret = perf_map_jit_init(); if(ret == NULL){ - return; + return; // Initialization failed, silently abort } } + /* + * Extract function information from Python code object + * + * We create a human-readable function name by combining the qualified + * name (includes class/module context) with the filename. This helps + * developers identify functions in perf reports. + */ const char *entry = ""; if (co->co_qualname != NULL) { entry = PyUnicode_AsUTF8(co->co_qualname); } + const char *filename = ""; if (co->co_filename != NULL) { filename = PyUnicode_AsUTF8(co->co_filename); } - + /* + * Create formatted function name for perf display + * + * Format: "py::<function_name>:<filename>" + * The "py::" prefix helps identify Python functions in mixed-language + * profiles (e.g., when profiling C extensions alongside Python code). + */ size_t perf_map_entry_size = snprintf(NULL, 0, "py::%s:%s", entry, filename) + 1; char* perf_map_entry = (char*) PyMem_RawMalloc(perf_map_entry_size); if (perf_map_entry == NULL) { - return; + return; // Memory allocation failed } snprintf(perf_map_entry, perf_map_entry_size, "py::%s:%s", entry, filename); @@ -528,90 +1191,186 @@ static void perf_map_jit_write_entry(void *state, const void *code_addr, uword base = (uword)code_addr; uword size = code_size; - // Write the code unwinding info event. - - // Create unwinding information (eh frame) + /* + * Generate DWARF unwinding information + * + * DWARF data is essential for proper stack unwinding during profiling. + * Without it, perf cannot generate accurate call graphs, especially + * in optimized code where frame pointers may be omitted. + */ ELFObjectContext ctx; - char buffer[1024]; + char buffer[1024]; // Buffer for DWARF data (1KB should be sufficient) ctx.code_size = code_size; ctx.startp = ctx.p = (uint8_t*)buffer; + ctx.fde_p = NULL; // Initialize to NULL, will be set when FDE is written + + /* Generate EH frame (Exception Handling frame) data */ elf_init_ehframe(&ctx); int eh_frame_size = ctx.p - ctx.startp; - // Populate the unwind info event for perf + /* + * Write Code Unwinding Information Event + * + * This event must be written before the code load event to ensure + * perf has the unwinding information available when it processes + * the code region. + */ CodeUnwindingInfoEvent ev2; ev2.base.event = PerfUnwindingInfo; ev2.base.time_stamp = get_current_monotonic_ticks(); ev2.unwind_data_size = sizeof(EhFrameHeader) + eh_frame_size; - // Ensure we have enough space between DSOs when perf maps them - assert(ev2.unwind_data_size <= PERF_JIT_CODE_PADDING); + + /* Verify we don't exceed our padding budget */ + assert(ev2.unwind_data_size <= (uint64_t)trampoline_api.code_padding); + ev2.eh_frame_hdr_size = sizeof(EhFrameHeader); - ev2.mapped_size = round_up(ev2.unwind_data_size, 16); + ev2.mapped_size = round_up(ev2.unwind_data_size, 16); // 16-byte alignment + + /* Calculate total event size with padding */ int content_size = sizeof(ev2) + sizeof(EhFrameHeader) + eh_frame_size; - int padding_size = round_up(content_size, 8) - content_size; + int padding_size = round_up(content_size, 8) - content_size; // 8-byte align ev2.base.size = content_size + padding_size; - perf_map_jit_write_fully(&ev2, sizeof(ev2)); + /* Write the unwinding info event header */ + perf_map_jit_write_fully(&ev2, sizeof(ev2)); - // Populate the eh Frame header + /* + * Write EH Frame Header + * + * The EH frame header provides metadata about the DWARF unwinding + * information that follows. It includes pointers and counts that + * help perf navigate the unwinding data efficiently. + */ EhFrameHeader f; f.version = 1; - f.eh_frame_ptr_enc = DwarfSData4 | DwarfPcRel; - f.fde_count_enc = DwarfUData4; - f.table_enc = DwarfSData4 | DwarfDataRel; + f.eh_frame_ptr_enc = DwarfSData4 | DwarfPcRel; // PC-relative signed 4-byte + f.fde_count_enc = DwarfUData4; // Unsigned 4-byte count + f.table_enc = DwarfSData4 | DwarfDataRel; // Data-relative signed 4-byte + + /* Calculate relative offsets for EH frame navigation */ f.eh_frame_ptr = -(eh_frame_size + 4 * sizeof(unsigned char)); - f.eh_fde_count = 1; + f.eh_fde_count = 1; // We generate exactly one FDE per function f.from = -(round_up(code_size, 8) + eh_frame_size); + int cie_size = ctx.eh_frame_p - ctx.startp; f.to = -(eh_frame_size - cie_size); + /* Write EH frame data and header */ perf_map_jit_write_fully(ctx.startp, eh_frame_size); perf_map_jit_write_fully(&f, sizeof(f)); + /* Write padding to maintain alignment */ char padding_bytes[] = "\0\0\0\0\0\0\0\0"; perf_map_jit_write_fully(&padding_bytes, padding_size); - // Write the code load event. + /* + * Write Code Load Event + * + * This event tells perf about the new code region. It includes: + * - Memory addresses and sizes + * - Process and thread identification + * - Function name for symbol resolution + * - The actual machine code bytes + */ CodeLoadEvent ev; ev.base.event = PerfLoad; ev.base.size = sizeof(ev) + (name_length+1) + size; ev.base.time_stamp = get_current_monotonic_ticks(); ev.process_id = getpid(); - ev.thread_id = syscall(SYS_gettid); - ev.vma = base; - ev.code_address = base; + ev.thread_id = syscall(SYS_gettid); // Get thread ID via system call + ev.vma = base; // Virtual memory address + ev.code_address = base; // Same as VMA for our use case ev.code_size = size; + + /* Assign unique code ID and increment counter */ perf_jit_map_state.code_id += 1; ev.code_id = perf_jit_map_state.code_id; + /* Write code load event and associated data */ perf_map_jit_write_fully(&ev, sizeof(ev)); - perf_map_jit_write_fully(perf_map_entry, name_length+1); - perf_map_jit_write_fully((void*)(base), size); - return; + perf_map_jit_write_fully(perf_map_entry, name_length+1); // Include null terminator + perf_map_jit_write_fully((void*)(base), size); // Copy actual machine code + + /* Clean up allocated memory */ + PyMem_RawFree(perf_map_entry); } +// ============================================================================= +// CLEANUP AND FINALIZATION +// ============================================================================= + +/* + * Finalize and cleanup the perf jitdump system + * + * This function is called when Python is shutting down or when the + * perf trampoline system is being disabled. It ensures all resources + * are properly released and all buffered data is flushed to disk. + * + * Args: + * state: Jitdump state (currently unused, uses global state) + * + * Returns: 0 on success + * + * IMPORTANT: This function signature is part of Python's internal API + * and must not be changed without coordinating with core Python development. + */ static int perf_map_jit_fini(void* state) { + /* + * Close jitdump file with proper synchronization + * + * We need to acquire the lock to ensure no other threads are + * writing to the file when we close it. This prevents corruption + * and ensures all data is properly flushed. + */ if (perf_jit_map_state.perf_map != NULL) { - // close the file PyThread_acquire_lock(perf_jit_map_state.map_lock, 1); - fclose(perf_jit_map_state.perf_map); + fclose(perf_jit_map_state.perf_map); // This also flushes buffers PyThread_release_lock(perf_jit_map_state.map_lock); - // clean up the lock and state + /* Clean up synchronization primitive */ PyThread_free_lock(perf_jit_map_state.map_lock); perf_jit_map_state.perf_map = NULL; } + + /* + * Unmap the memory region + * + * This removes the signal to perf that we were generating JIT code. + * After this point, perf will no longer detect this process as + * having JIT capabilities. + */ if (perf_jit_map_state.mapped_buffer != NULL) { munmap(perf_jit_map_state.mapped_buffer, perf_jit_map_state.mapped_size); + perf_jit_map_state.mapped_buffer = NULL; } + + /* Clear global state reference */ trampoline_api.state = NULL; - return 0; + + return 0; // Success } +// ============================================================================= +// PUBLIC API EXPORT +// ============================================================================= + +/* + * Python Perf Callbacks Structure + * + * This structure defines the callback interface that Python's trampoline + * system uses to integrate with perf profiling. It contains function + * pointers for initialization, event writing, and cleanup. + * + * CRITICAL: This structure and its contents are part of Python's internal + * API. The function signatures and behavior must remain stable to maintain + * compatibility with the Python interpreter's perf integration system. + * + * Used by: Python's _PyPerf_Callbacks system in pycore_ceval.h + */ _PyPerf_Callbacks _Py_perfmap_jit_callbacks = { - &perf_map_jit_init, - &perf_map_jit_write_entry, - &perf_map_jit_fini, + &perf_map_jit_init, // Initialization function + &perf_map_jit_write_entry, // Event writing function + &perf_map_jit_fini, // Cleanup function }; -#endif +#endif /* PY_HAVE_PERF_TRAMPOLINE */ diff --git a/Python/perf_trampoline.c b/Python/perf_trampoline.c index 996e54b82b61e83..8e7cfff33ebfcd4 100644 --- a/Python/perf_trampoline.c +++ b/Python/perf_trampoline.c @@ -162,6 +162,8 @@ static void invalidate_icache(char* begin, char*end) { } #endif +#define CODE_ALIGNMENT 32 + /* The function pointer is passed as last argument. The other three arguments * are passed in the same order as the function requires. This results in * shorter, more efficient ASM code for trampoline. @@ -202,6 +204,49 @@ enum perf_trampoline_type { #define perf_map_file _PyRuntime.ceval.perf.map_file #define persist_after_fork _PyRuntime.ceval.perf.persist_after_fork #define perf_trampoline_type _PyRuntime.ceval.perf.perf_trampoline_type +#define prev_eval_frame _PyRuntime.ceval.perf.prev_eval_frame +#define trampoline_refcount _PyRuntime.ceval.perf.trampoline_refcount +#define code_watcher_id _PyRuntime.ceval.perf.code_watcher_id + +static void free_code_arenas(void); + +static void +perf_trampoline_clear_code_watcher(void) +{ + if (code_watcher_id >= 0) { + PyCode_ClearWatcher(code_watcher_id); + code_watcher_id = -1; + } + extra_code_index = -1; +} + +static void +perf_trampoline_reset_state(void) +{ + free_code_arenas(); + perf_trampoline_clear_code_watcher(); +} + +static int +perf_trampoline_code_watcher(PyCodeEvent event, PyCodeObject *co) +{ + if (event != PY_CODE_EVENT_DESTROY) { + return 0; + } + if (extra_code_index == -1) { + return 0; + } + py_trampoline f = NULL; + int ret = _PyCode_GetExtra((PyObject *)co, extra_code_index, (void **)&f); + if (ret != 0 || f == NULL) { + return 0; + } + trampoline_refcount--; + if (trampoline_refcount == 0) { + perf_trampoline_reset_state(); + } + return 0; +} static void perf_map_write_entry(void *state, const void *code_addr, @@ -291,7 +336,9 @@ new_code_arena(void) void *start = &_Py_trampoline_func_start; void *end = &_Py_trampoline_func_end; size_t code_size = end - start; - size_t chunk_size = round_up(code_size + trampoline_api.code_padding, 16); + size_t unaligned_size = code_size + trampoline_api.code_padding; + size_t chunk_size = round_up(unaligned_size, CODE_ALIGNMENT); + assert(chunk_size % CODE_ALIGNMENT == 0); // TODO: Check the effect of alignment of the code chunks. Initial investigation // showed that this has no effect on performance in x86-64 or aarch64 and the current // version has the advantage that the unwinder in GDB can unwind across JIT-ed code. @@ -356,7 +403,9 @@ static inline py_trampoline code_arena_new_code(code_arena_t *code_arena) { py_trampoline trampoline = (py_trampoline)code_arena->current_addr; - size_t total_code_size = round_up(code_arena->code_size + trampoline_api.code_padding, 16); + size_t total_code_size = round_up(code_arena->code_size + trampoline_api.code_padding, + CODE_ALIGNMENT); + assert(total_code_size % CODE_ALIGNMENT == 0); code_arena->size_left -= total_code_size; code_arena->current_addr += total_code_size; return trampoline; @@ -399,6 +448,7 @@ py_trampoline_evaluator(PyThreadState *ts, _PyInterpreterFrame *frame, perf_code_arena->code_size, co); _PyCode_SetExtra((PyObject *)co, extra_code_index, (void *)new_trampoline); + trampoline_refcount++; f = new_trampoline; } assert(f != NULL); @@ -422,6 +472,7 @@ int PyUnstable_PerfTrampoline_CompileCode(PyCodeObject *co) } trampoline_api.write_state(trampoline_api.state, new_trampoline, perf_code_arena->code_size, co); + trampoline_refcount++; return _PyCode_SetExtra((PyObject *)co, extra_code_index, (void *)new_trampoline); } @@ -476,6 +527,10 @@ _PyPerfTrampoline_Init(int activate) { #ifdef PY_HAVE_PERF_TRAMPOLINE PyThreadState *tstate = _PyThreadState_GET(); + if (code_watcher_id == 0) { + // Initialize to -1 since 0 is a valid watcher ID + code_watcher_id = -1; + } if (tstate->interp->eval_frame && tstate->interp->eval_frame != py_trampoline_evaluator) { PyErr_SetString(PyExc_RuntimeError, @@ -489,9 +544,6 @@ _PyPerfTrampoline_Init(int activate) } else { _PyInterpreterState_SetEvalFrameFunc(tstate->interp, py_trampoline_evaluator); - if (new_code_arena() < 0) { - return -1; - } extra_code_index = _PyEval_RequestCodeExtraIndex(NULL); if (extra_code_index == -1) { return -1; @@ -499,6 +551,16 @@ _PyPerfTrampoline_Init(int activate) if (trampoline_api.state == NULL && trampoline_api.init_state != NULL) { trampoline_api.state = trampoline_api.init_state(); } + if (new_code_arena() < 0) { + return -1; + } + code_watcher_id = PyCode_AddWatcher(perf_trampoline_code_watcher); + if (code_watcher_id < 0) { + PyErr_FormatUnraisable("Failed to register code watcher for perf trampoline"); + free_code_arenas(); + return -1; + } + trampoline_refcount = 1; // Base refcount held by the system perf_status = PERF_STATUS_OK; } #endif @@ -520,17 +582,19 @@ _PyPerfTrampoline_Fini(void) trampoline_api.free_state(trampoline_api.state); perf_trampoline_type = PERF_TRAMPOLINE_UNSET; } - extra_code_index = -1; + + // Prevent new trampolines from being created perf_status = PERF_STATUS_NO_INIT; -#endif - return 0; -} -void _PyPerfTrampoline_FreeArenas(void) { -#ifdef PY_HAVE_PERF_TRAMPOLINE - free_code_arenas(); + // Decrement base refcount. If refcount reaches 0, all code objects are already + // dead so clean up now. Otherwise, watcher remains active to clean up when last + // code object dies; extra_code_index stays valid so watcher can identify them. + trampoline_refcount--; + if (trampoline_refcount == 0) { + perf_trampoline_reset_state(); + } #endif - return; + return 0; } int @@ -562,6 +626,13 @@ _PyPerfTrampoline_AfterFork_Child(void) int was_active = _PyIsPerfTrampolineActive(); _PyPerfTrampoline_Fini(); if (was_active) { + // After fork, Fini may leave the old code watcher registered + // if trampolined code objects from the parent still exist + // (trampoline_refcount > 0). Clear it unconditionally before + // Init registers a new one, but keep the old arenas mapped: the + // child may still need to return through trampoline frames that + // were on the C stack at fork(). + perf_trampoline_clear_code_watcher(); _PyPerfTrampoline_Init(1); } } diff --git a/Python/preconfig.c b/Python/preconfig.c index 5b26c75de8b3a00..7b4168c466712a8 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -584,7 +584,7 @@ _Py_get_xoption(const PyWideStringList *xoptions, const wchar_t *name) for (Py_ssize_t i=0; i < xoptions->length; i++) { const wchar_t *option = xoptions->items[i]; size_t len; - wchar_t *sep = wcschr(option, L'='); + const wchar_t *sep = wcschr(option, L'='); if (sep != NULL) { len = (sep - option); } @@ -615,7 +615,7 @@ preconfig_init_utf8_mode(PyPreConfig *config, const _PyPreCmdline *cmdline) const wchar_t *xopt; xopt = _Py_get_xoption(&cmdline->xoptions, L"utf8"); if (xopt) { - wchar_t *sep = wcschr(xopt, L'='); + const wchar_t *sep = wcschr(xopt, L'='); if (sep) { xopt = sep + 1; if (wcscmp(xopt, L"1") == 0) { diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index c4c1d9fd9e13805..fe55ec7cafffec7 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -417,7 +417,8 @@ interpreter_update_config(PyThreadState *tstate, int only_update_path_config) } } - tstate->interp->long_state.max_str_digits = config->int_max_str_digits; + _Py_atomic_store_int(&tstate->interp->long_state.max_str_digits, + config->int_max_str_digits); // Update the sys module for the new configuration if (_PySys_UpdateConfig(tstate) < 0) { @@ -503,6 +504,7 @@ pycore_init_runtime(_PyRuntimeState *runtime, _PyRuntimeState_SetFinalizing(runtime, NULL); _Py_InitVersion(); + _Py_DumpTraceback_Init(); status = _Py_HashRandomization_Init(config); if (_PyStatus_EXCEPTION(status)) { @@ -760,6 +762,11 @@ pycore_init_types(PyInterpreterState *interp) return status; } + status = _PyDateTime_InitTypes(interp); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + return _PyStatus_OK(); } @@ -1595,6 +1602,7 @@ finalize_remove_modules(PyObject *modules, int verbose) PyObject *value = PyObject_GetItem(modules, key); if (value == NULL) { PyErr_FormatUnraisable("Exception ignored while removing modules"); + Py_DECREF(key); continue; } CLEAR_MODULE(key, value); @@ -1702,8 +1710,10 @@ finalize_modules(PyThreadState *tstate) #endif // Stop watching __builtin__ modifications - PyDict_Unwatch(0, interp->builtins); - + if (PyDict_Unwatch(0, interp->builtins) < 0) { + // might happen if interp is cleared before watching the __builtin__ + PyErr_Clear(); + } PyObject *modules = _PyImport_GetModules(interp); if (modules == NULL) { // Already done @@ -1915,7 +1925,6 @@ finalize_interp_clear(PyThreadState *tstate) _PyArg_Fini(); _Py_ClearFileSystemEncoding(); _PyPerfTrampoline_Fini(); - _PyPerfTrampoline_FreeArenas(); } finalize_interp_types(tstate->interp); @@ -1992,6 +2001,7 @@ resolve_final_tstate(_PyRuntimeState *runtime) } else { /* Fall back to the current tstate. It's better than nothing. */ + // XXX No it's not main_tstate = tstate; } } @@ -2037,6 +2047,16 @@ _Py_Finalize(_PyRuntimeState *runtime) _PyAtExit_Call(tstate->interp); + /* Clean up any lingering subinterpreters. + + Two preconditions need to be met here: + + - This has to happen before _PyRuntimeState_SetFinalizing is + called, or else threads might get prematurely blocked. + - The world must not be stopped, as finalizers can run. + */ + finalize_subinterpreters(); + assert(_PyThreadState_GET() == tstate); /* Copy the core config, PyInterpreterState_Delete() free @@ -2124,9 +2144,6 @@ _Py_Finalize(_PyRuntimeState *runtime) _PyImport_FiniExternal(tstate->interp); finalize_modules(tstate); - /* Clean up any lingering subinterpreters. */ - finalize_subinterpreters(); - /* Print debug stats if any */ _PyEval_Fini(); @@ -2291,18 +2308,17 @@ new_interpreter(PyThreadState **tstate_p, interpreters: disable PyGILState_Check(). */ runtime->gilstate.check_enabled = 0; - PyInterpreterState *interp = PyInterpreterState_New(); + // XXX Might new_interpreter() have been called without the GIL held? + PyThreadState *save_tstate = _PyThreadState_GET(); + PyThreadState *tstate = NULL; + PyInterpreterState *interp; + status = _PyInterpreterState_New(save_tstate, &interp); if (interp == NULL) { - *tstate_p = NULL; - return _PyStatus_OK(); + goto error; } _PyInterpreterState_SetWhence(interp, whence); interp->_ready = 1; - // XXX Might new_interpreter() have been called without the GIL held? - PyThreadState *save_tstate = _PyThreadState_GET(); - PyThreadState *tstate = NULL; - /* From this point until the init_interp_create_gil() call, we must not do anything that requires that the GIL be held (or otherwise exist). That applies whether or not the new @@ -2377,15 +2393,13 @@ new_interpreter(PyThreadState **tstate_p, error: *tstate_p = NULL; if (tstate != NULL) { - PyThreadState_Clear(tstate); - _PyThreadState_Detach(tstate); - PyThreadState_Delete(tstate); + Py_EndInterpreter(tstate); + } else if (interp != NULL) { + PyInterpreterState_Delete(interp); } if (save_tstate != NULL) { _PyThreadState_Attach(save_tstate); } - PyInterpreterState_Delete(interp); - return status; } @@ -2410,9 +2424,8 @@ Py_NewInterpreter(void) return tstate; } -/* Delete an interpreter and its last thread. This requires that the - given thread state is current, that the thread has no remaining - frames, and that it is its interpreter's only remaining thread. +/* Delete an interpreter. This requires that the given thread state + is current, and that the thread has no remaining frames. It is a fatal error to violate these constraints. (Py_FinalizeEx() doesn't have these constraints -- it zaps @@ -2442,15 +2455,20 @@ Py_EndInterpreter(PyThreadState *tstate) _Py_FinishPendingCalls(tstate); _PyAtExit_Call(tstate->interp); - - if (tstate != interp->threads.head || tstate->next != NULL) { - Py_FatalError("not the last thread"); - } - + _PyRuntimeState *runtime = interp->runtime; + _PyEval_StopTheWorldAll(runtime); /* Remaining daemon threads will automatically exit when they attempt to take the GIL (ex: PyEval_RestoreThread()). */ _PyInterpreterState_SetFinalizing(interp, tstate); + PyThreadState *list = _PyThreadState_RemoveExcept(tstate); + for (PyThreadState *p = list; p != NULL; p = p->next) { + _PyThreadState_SetShuttingDown(p); + } + + _PyEval_StartTheWorldAll(runtime); + _PyThreadState_DeleteList(list, /*is_after_fork=*/0); + // XXX Call something like _PyImport_Disable() here? _PyImport_FiniExternal(tstate->interp); @@ -2480,6 +2498,8 @@ finalize_subinterpreters(void) PyInterpreterState *main_interp = _PyInterpreterState_Main(); assert(final_tstate->interp == main_interp); _PyRuntimeState *runtime = main_interp->runtime; + assert(!runtime->stoptheworld.world_stopped); + assert(_PyRuntimeState_GetFinalizing(runtime) == NULL); struct pyinterpreters *interpreters = &runtime->interpreters; /* Get the first interpreter in the list. */ @@ -2499,7 +2519,7 @@ finalize_subinterpreters(void) (void)PyErr_WarnEx( PyExc_RuntimeWarning, "remaining subinterpreters; " - "destroy them with _interpreters.destroy()", + "close them with Interpreter.close()", 0); /* Swap out the current tstate, which we know must belong @@ -2508,27 +2528,17 @@ finalize_subinterpreters(void) /* Clean up all remaining subinterpreters. */ while (interp != NULL) { - assert(!_PyInterpreterState_IsRunningMain(interp)); - - /* Find the tstate to use for fini. We assume the interpreter - will have at most one tstate at this point. */ - PyThreadState *tstate = interp->threads.head; - if (tstate != NULL) { - /* Ideally we would be able to use tstate as-is, and rely - on it being in a ready state: no exception set, not - running anything (tstate->current_frame), matching the - current thread ID (tstate->thread_id). To play it safe, - we always delete it and use a fresh tstate instead. */ - assert(tstate != final_tstate); - _PyThreadState_Attach(tstate); - PyThreadState_Clear(tstate); - _PyThreadState_Detach(tstate); - PyThreadState_Delete(tstate); + /* Make a tstate for finalization. */ + PyThreadState *tstate = _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_FINI); + if (tstate == NULL) { + // XXX Some graceful way to always get a thread state? + Py_FatalError("thread state allocation failed"); } - tstate = _PyThreadState_NewBound(interp, _PyThreadState_WHENCE_FINI); - /* Destroy the subinterpreter. */ + /* Enter the subinterpreter. */ _PyThreadState_Attach(tstate); + + /* Destroy the subinterpreter. */ Py_EndInterpreter(tstate); assert(_PyThreadState_GET() == NULL); @@ -2963,7 +2973,9 @@ apple_log_write_impl(PyObject *self, PyObject *args) // Pass the user-provided text through explicit %s formatting // to avoid % literals being interpreted as a formatting directive. - os_log_with_type(OS_LOG_DEFAULT, logtype, "%s", text); + // Using {public} ensures "dynamic" string messages are visible + // in the log without special configuration. + os_log_with_type(OS_LOG_DEFAULT, logtype, "%{public}s", text); Py_RETURN_NONE; } @@ -3351,7 +3363,9 @@ fatal_error(int fd, int header, const char *prefix, const char *msg, This function already did its best to display a traceback. Disable faulthandler to prevent writing a second traceback on abort(). */ - _PyFaulthandler_Fini(); + if (has_tstate_and_gil) { + _PyFaulthandler_Fini(); + } /* Check if the current Python thread hold the GIL */ if (has_tstate_and_gil) { @@ -3589,7 +3603,7 @@ PyOS_getsig(int sig) /* * All of the code in this function must only use async-signal-safe functions, - * listed at `man 7 signal` or + * listed at `man 7 signal-safety` or * http://www.opengroup.org/onlinepubs/009695399/functions/xsh_chap02_04.html. */ PyOS_sighandler_t diff --git a/Python/pystate.c b/Python/pystate.c index 1ac134400856d4b..d1505be4acf50f2 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -8,7 +8,6 @@ #include "pycore_codecs.h" // _PyCodec_Fini() #include "pycore_critical_section.h" // _PyCriticalSection_Resume() #include "pycore_dtoa.h" // _dtoa_state_INIT() -#include "pycore_emscripten_trampoline.h" // _Py_EmscriptenTrampoline_Init() #include "pycore_freelist.h" // _PyObject_ClearFreeLists() #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interpframe.h" // _PyThreadState_HasStackSpace() @@ -404,12 +403,12 @@ _Py_COMP_DIAG_POP &(runtime)->unicode_state.ids.mutex, \ &(runtime)->imports.extensions.mutex, \ &(runtime)->ceval.pending_mainthread.mutex, \ - &(runtime)->ceval.sys_trace_profile_mutex, \ &(runtime)->atexit.mutex, \ &(runtime)->audit_hooks.mutex, \ &(runtime)->allocators.mutex, \ &(runtime)->_main_interpreter.types.mutex, \ &(runtime)->_main_interpreter.code_state.mutex, \ + &(runtime)->_main_interpreter.dict_state.watcher_mutex, \ } static void @@ -434,11 +433,6 @@ init_runtime(_PyRuntimeState *runtime, runtime->main_thread = PyThread_get_thread_ident(); runtime->unicode_state.ids.next_index = unicode_next_index; - -#if defined(__EMSCRIPTEN__) && defined(PY_CALL_TRAMPOLINE) - _Py_EmscriptenTrampoline_Init(runtime); -#endif - runtime->_initialized = 1; } @@ -572,16 +566,19 @@ _PyInterpreterState_Enable(_PyRuntimeState *runtime) static PyInterpreterState * alloc_interpreter(void) { + // Aligned allocation for PyInterpreterState. + // the first word of the memory block is used to store + // the original pointer to be used later to free the memory. size_t alignment = _Alignof(PyInterpreterState); - size_t allocsize = sizeof(PyInterpreterState) + alignment - 1; + size_t allocsize = sizeof(PyInterpreterState) + sizeof(void *) + alignment - 1; void *mem = PyMem_RawCalloc(1, allocsize); if (mem == NULL) { return NULL; } - PyInterpreterState *interp = _Py_ALIGN_UP(mem, alignment); - assert(_Py_IS_ALIGNED(interp, alignment)); - interp->_malloced = mem; - return interp; + void *ptr = _Py_ALIGN_UP((char *)mem + sizeof(void *), alignment); + ((void **)ptr)[-1] = mem; + assert(_Py_IS_ALIGNED(ptr, alignment)); + return ptr; } static void @@ -596,7 +593,7 @@ free_interpreter(PyInterpreterState *interp) interp->obmalloc = NULL; } assert(_Py_IS_ALIGNED(interp, _Alignof(PyInterpreterState))); - PyMem_RawFree(interp->_malloced); + PyMem_RawFree(((void **)interp)[-1]); } } @@ -674,8 +671,7 @@ init_interpreter(PyInterpreterState *interp, } interp->monitoring_tool_versions[t] = 0; } - interp->sys_profile_initialized = false; - interp->sys_trace_initialized = false; + interp->_code_object_generation = 0; interp->jit = false; interp->executor_list_head = NULL; interp->executor_deletion_list_head = NULL; @@ -698,6 +694,9 @@ init_interpreter(PyInterpreterState *interp, NULL, &alloc ); + if (interp->open_stackrefs_table == NULL) { + return _PyStatus_NO_MEMORY(); + } # ifdef Py_STACKREF_CLOSE_DEBUG interp->closed_stackrefs_table = _Py_hashtable_new_full( _Py_hashtable_hash_ptr, @@ -706,6 +705,9 @@ init_interpreter(PyInterpreterState *interp, NULL, &alloc ); + if (interp->closed_stackrefs_table == NULL) { + return _PyStatus_NO_MEMORY(); + } # endif _Py_stackref_associate(interp, Py_None, PyStackRef_None); _Py_stackref_associate(interp, Py_False, PyStackRef_False); @@ -868,10 +870,11 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->audit_hooks); - // At this time, all the threads should be cleared so we don't need atomic - // operations for instrumentation_version or eval_breaker. + // gh-140257: Threads have already been cleared, but daemon threads may + // still access eval_breaker atomically via take_gil() right before they + // hang. Use an atomic store to prevent data races during finalization. interp->ceval.instrumentation_version = 0; - tstate->eval_breaker = 0; + _Py_atomic_store_uintptr(&tstate->eval_breaker, 0); for (int i = 0; i < _PY_MONITORING_UNGROUPED_EVENTS; i++) { interp->monitors.tools[i] = 0; @@ -881,11 +884,13 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->monitoring_callables[t][e]); } } - interp->sys_profile_initialized = false; - interp->sys_trace_initialized = false; for (int t = 0; t < PY_MONITORING_TOOL_IDS; t++) { Py_CLEAR(interp->monitoring_tool_names[t]); } + interp->_code_object_generation = 0; +#ifdef Py_GIL_DISABLED + interp->tlbc_indices.tlbc_generation = 0; +#endif PyConfig_Clear(&interp->config); _PyCodec_Fini(interp); @@ -909,7 +914,6 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) _Py_ClearExecutorDeletionList(interp); #endif _PyAST_Fini(interp); - _PyWarnings_Fini(interp); _PyAtExit_Fini(interp); // All Python types must be destroyed before the last GC collection. Python @@ -920,6 +924,9 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) _PyGC_CollectNoFail(tstate); _PyGC_Fini(interp); + // Finalize warnings after last gc so that any finalizers can + // access warnings state + _PyWarnings_Fini(interp); /* We don't clear sysdict and builtins until the end of this function. Because clearing other attributes can execute arbitrary Python code which requires sysdict and builtins. */ @@ -1049,6 +1056,8 @@ PyInterpreterState_Delete(PyInterpreterState *interp) _PyObject_FiniState(interp); + PyConfig_Clear(&interp->config); + free_interpreter(interp); } @@ -1246,6 +1255,7 @@ _Py_CheckMainModule(PyObject *module) PyObject *msg = PyUnicode_FromString("invalid __main__ module"); if (msg != NULL) { (void)PyErr_SetImportError(msg, &_Py_ID(__main__), NULL); + Py_DECREF(msg); } return -1; } @@ -1457,9 +1467,6 @@ tstate_is_alive(PyThreadState *tstate) // lifecycle //---------- -/* Minimum size of data stack chunk */ -#define DATA_STACK_CHUNK_SIZE (16*1024) - static _PyStackChunk* allocate_chunk(int size_in_bytes, _PyStackChunk* previous) { @@ -1575,6 +1582,7 @@ init_threadstate(_PyThreadStateImpl *_tstate, tstate->datastack_chunk = NULL; tstate->datastack_top = NULL; tstate->datastack_limit = NULL; + tstate->datastack_cached_chunk = NULL; tstate->what_event = -1; tstate->current_executor = NULL; tstate->dict_global_version = 0; @@ -1583,6 +1591,9 @@ init_threadstate(_PyThreadStateImpl *_tstate, _tstate->c_stack_top = 0; _tstate->c_stack_hard_limit = 0; + _tstate->c_stack_init_base = 0; + _tstate->c_stack_init_top = 0; + _tstate->asyncio_running_loop = NULL; _tstate->asyncio_running_task = NULL; @@ -1711,6 +1722,11 @@ clear_datastack(PyThreadState *tstate) _PyObject_VirtualFree(chunk, chunk->size); chunk = prev; } + if (tstate->datastack_cached_chunk != NULL) { + _PyObject_VirtualFree(tstate->datastack_cached_chunk, + tstate->datastack_cached_chunk->size); + tstate->datastack_cached_chunk = NULL; + } } void @@ -1791,13 +1807,14 @@ PyThreadState_Clear(PyThreadState *tstate) } if (tstate->c_profilefunc != NULL) { - tstate->interp->sys_profiling_threads--; + FT_ATOMIC_ADD_SSIZE(tstate->interp->sys_profiling_threads, -1); tstate->c_profilefunc = NULL; } if (tstate->c_tracefunc != NULL) { - tstate->interp->sys_tracing_threads--; + FT_ATOMIC_ADD_SSIZE(tstate->interp->sys_tracing_threads, -1); tstate->c_tracefunc = NULL; } + Py_CLEAR(tstate->c_profileobj); Py_CLEAR(tstate->c_traceobj); @@ -1811,16 +1828,23 @@ PyThreadState_Clear(PyThreadState *tstate) struct _Py_freelists *freelists = _Py_freelists_GET(); _PyObject_ClearFreeLists(freelists, 1); + // Flush the thread's local GC allocation count to the global count + // before the thread state is cleared, otherwise the count is lost. + _PyThreadStateImpl *tstate_impl = (_PyThreadStateImpl *)tstate; + _Py_atomic_add_int(&tstate->interp->gc.young.count, + (int)tstate_impl->gc.alloc_count); + tstate_impl->gc.alloc_count = 0; + // Merge our thread-local refcounts into the type's own refcount and // free our local refcount array. - _PyObject_FinalizePerThreadRefcounts((_PyThreadStateImpl *)tstate); + _PyObject_FinalizePerThreadRefcounts(tstate_impl); // Remove ourself from the biased reference counting table of threads. _Py_brc_remove_thread(tstate); // Release our thread-local copies of the bytecode for reuse by another // thread - _Py_ClearTLBCIndex((_PyThreadStateImpl *)tstate); + _Py_ClearTLBCIndex(tstate_impl); #endif // Merge our queue of pointers to be freed into the interpreter queue. @@ -1908,9 +1932,14 @@ tstate_delete_common(PyThreadState *tstate, int release_gil) static void zapthreads(PyInterpreterState *interp) { + PyThreadState *tstate; /* No need to lock the mutex here because this should only happen - when the threads are all really dead (XXX famous last words). */ - _Py_FOR_EACH_TSTATE_UNLOCKED(interp, tstate) { + when the threads are all really dead (XXX famous last words). + + Cannot use _Py_FOR_EACH_TSTATE_UNLOCKED because we are freeing + the thread states here. + */ + while ((tstate = interp->threads.head) != NULL) { tstate_verify_not_active(tstate); tstate_delete_common(tstate, 0); free_threadstate((_PyThreadStateImpl *)tstate); @@ -2345,13 +2374,15 @@ stop_the_world(struct _stoptheworld_state *stw) { _PyRuntimeState *runtime = &_PyRuntime; - PyMutex_Lock(&stw->mutex); + // gh-137433: Acquire the rwmutex first to avoid deadlocks with daemon + // threads that may hang when blocked on lock acquisition. if (stw->is_global) { _PyRWMutex_Lock(&runtime->stoptheworld_mutex); } else { _PyRWMutex_RLock(&runtime->stoptheworld_mutex); } + PyMutex_Lock(&stw->mutex); HEAD_LOCK(runtime); stw->requested = 1; @@ -2417,13 +2448,13 @@ start_the_world(struct _stoptheworld_state *stw) } stw->requester = NULL; HEAD_UNLOCK(runtime); + PyMutex_Unlock(&stw->mutex); if (stw->is_global) { _PyRWMutex_Unlock(&runtime->stoptheworld_mutex); } else { _PyRWMutex_RUnlock(&runtime->stoptheworld_mutex); } - PyMutex_Unlock(&stw->mutex); } #endif // Py_GIL_DISABLED @@ -3007,13 +3038,24 @@ _PyInterpreterState_HasFeature(PyInterpreterState *interp, unsigned long feature static PyObject ** push_chunk(PyThreadState *tstate, int size) { - int allocate_size = DATA_STACK_CHUNK_SIZE; + int allocate_size = _PY_DATA_STACK_CHUNK_SIZE; while (allocate_size < (int)sizeof(PyObject*)*(size + MINIMUM_OVERHEAD)) { allocate_size *= 2; } - _PyStackChunk *new = allocate_chunk(allocate_size, tstate->datastack_chunk); - if (new == NULL) { - return NULL; + _PyStackChunk *new; + if (tstate->datastack_cached_chunk != NULL + && (size_t)allocate_size <= tstate->datastack_cached_chunk->size) + { + new = tstate->datastack_cached_chunk; + tstate->datastack_cached_chunk = NULL; + new->previous = tstate->datastack_chunk; + new->top = 0; + } + else { + new = allocate_chunk(allocate_size, tstate->datastack_chunk); + if (new == NULL) { + return NULL; + } } if (tstate->datastack_chunk) { tstate->datastack_chunk->top = tstate->datastack_top - @@ -3049,12 +3091,17 @@ _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame * frame) if (base == &tstate->datastack_chunk->data[0]) { _PyStackChunk *chunk = tstate->datastack_chunk; _PyStackChunk *previous = chunk->previous; + _PyStackChunk *cached = tstate->datastack_cached_chunk; // push_chunk ensures that the root chunk is never popped: assert(previous); tstate->datastack_top = &previous->data[previous->top]; tstate->datastack_chunk = previous; - _PyObject_VirtualFree(chunk, chunk->size); tstate->datastack_limit = (PyObject **)(((char *)previous) + previous->size); + chunk->previous = NULL; + if (cached != NULL) { + _PyObject_VirtualFree(cached, cached->size); + } + tstate->datastack_cached_chunk = chunk; } else { assert(tstate->datastack_top); diff --git a/Python/pystrhex.c b/Python/pystrhex.c index 38484f5a7d4227b..af2f5c5dce5fca5 100644 --- a/Python/pystrhex.c +++ b/Python/pystrhex.c @@ -42,8 +42,7 @@ static PyObject *_Py_strhex_impl(const char* argbuf, const Py_ssize_t arglen, else { bytes_per_sep_group = 0; } - - unsigned int abs_bytes_per_sep = Py_ABS(bytes_per_sep_group); + unsigned int abs_bytes_per_sep = _Py_ABS_CAST(unsigned int, bytes_per_sep_group); Py_ssize_t resultlen = 0; if (bytes_per_sep_group && arglen > 0) { /* How many sep characters we'll be inserting. */ diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 4ee287af72fdb2d..a97bc7dbb8e5a1e 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -22,7 +22,7 @@ #include "pycore_pyerrors.h" // _PyErr_GetRaisedException() #include "pycore_pylifecycle.h" // _Py_FdIsInteractive() #include "pycore_pystate.h" // _PyInterpreterState_GET() -#include "pycore_pythonrun.h" // export _PyRun_InteractiveLoopObject() +#include "pycore_pythonrun.h" // export _PyRun_AnyFile() #include "pycore_sysmodule.h" // _PySys_SetAttr() #include "pycore_traceback.h" // _PyTraceBack_Print() #include "pycore_unicodeobject.h" // _PyUnicode_Equal() @@ -41,50 +41,53 @@ # include "windows.h" #endif -/* Forward */ +/* Forward declarations */ static void flush_io(void); static PyObject *run_mod(mod_ty, PyObject *, PyObject *, PyObject *, PyCompilerFlags *, PyArena *, PyObject*, int); static PyObject *run_pyc_file(FILE *, PyObject *, PyObject *, PyCompilerFlags *); -static int PyRun_InteractiveOneObjectEx(FILE *, PyObject *, PyCompilerFlags *); -static PyObject* pyrun_file(FILE *fp, PyObject *filename, int start, - PyObject *globals, PyObject *locals, int closeit, - PyCompilerFlags *flags); static PyObject * -_PyRun_StringFlagsWithName(const char *str, PyObject* name, int start, - PyObject *globals, PyObject *locals, PyCompilerFlags *flags, - int generate_new_source); +_PyRun_String(const char *str, PyObject* name, int start, + PyObject *globals, PyObject *locals, PyCompilerFlags *flags, + int generate_new_source); +static PyObject* +_PyRun_InteractiveLoop(FILE *fp, PyObject *filename, PyCompilerFlags *flags); +static int +_PyRun_InteractiveOne(FILE *fp, PyObject *filename, PyCompilerFlags *flags); +static PyObject * +_PyRun_File(FILE *fp, PyObject *filename, int start, PyObject *globals, + PyObject *locals, int closeit, PyCompilerFlags *flags); -int -_PyRun_AnyFileObject(FILE *fp, PyObject *filename, int closeit, - PyCompilerFlags *flags) + +PyObject* +_PyRun_AnyFile(FILE *fp, PyObject *filename, int closeit, + PyCompilerFlags *flags) { int decref_filename = 0; if (filename == NULL) { filename = PyUnicode_FromString("???"); if (filename == NULL) { - PyErr_Print(); - return -1; + return NULL; } decref_filename = 1; } - int res; + PyObject *result; if (_Py_FdIsInteractive(fp, filename)) { - res = _PyRun_InteractiveLoopObject(fp, filename, flags); + result = _PyRun_InteractiveLoop(fp, filename, flags); if (closeit) { fclose(fp); } } else { - res = _PyRun_SimpleFileObject(fp, filename, closeit, flags); + result = _PyRun_SimpleFile(fp, filename, closeit, flags); } if (decref_filename) { Py_DECREF(filename); } - return res; + return result; } int @@ -99,14 +102,20 @@ PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit, return -1; } } - int res = _PyRun_AnyFileObject(fp, filename_obj, closeit, flags); + + PyObject *result = _PyRun_AnyFile(fp, filename_obj, closeit, flags); Py_XDECREF(filename_obj); - return res; + if (result == NULL) { + PyErr_Print(); + return -1; + } + Py_DECREF(result); + return 0; } -int -_PyRun_InteractiveLoopObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags) +static PyObject* +_PyRun_InteractiveLoop(FILE *fp, PyObject *filename, PyCompilerFlags *flags) { PyCompilerFlags local_flags = _PyCompilerFlags_INIT; if (flags == NULL) { @@ -115,8 +124,7 @@ _PyRun_InteractiveLoopObject(FILE *fp, PyObject *filename, PyCompilerFlags *flag PyObject *v; if (_PySys_GetOptionalAttr(&_Py_ID(ps1), &v) < 0) { - PyErr_Print(); - return -1; + return NULL; } if (v == NULL) { v = PyUnicode_FromString(">>> "); @@ -129,8 +137,7 @@ _PyRun_InteractiveLoopObject(FILE *fp, PyObject *filename, PyCompilerFlags *flag } Py_XDECREF(v); if (_PySys_GetOptionalAttr(&_Py_ID(ps2), &v) < 0) { - PyErr_Print(); - return -1; + return NULL; } if (v == NULL) { v = PyUnicode_FromString("... "); @@ -146,36 +153,41 @@ _PyRun_InteractiveLoopObject(FILE *fp, PyObject *filename, PyCompilerFlags *flag #ifdef Py_REF_DEBUG int show_ref_count = _Py_GetConfig()->show_ref_count; #endif - int err = 0; - int ret; int nomem_count = 0; + int ret; do { - ret = PyRun_InteractiveOneObjectEx(fp, filename, flags); + ret = _PyRun_InteractiveOne(fp, filename, flags); if (ret == -1 && PyErr_Occurred()) { /* Prevent an endless loop after multiple consecutive MemoryErrors * while still allowing an interactive command to fail with a * MemoryError. */ if (PyErr_ExceptionMatches(PyExc_MemoryError)) { if (++nomem_count > 16) { - PyErr_Clear(); - err = -1; - break; + return NULL; } } else { nomem_count = 0; } + + int inspect = _Py_GetConfig()->inspect; + if (!inspect && PyErr_ExceptionMatches(PyExc_SystemExit)) { + return NULL; + } + PyErr_Print(); - flush_io(); - } else { + } + else { nomem_count = 0; } + #ifdef Py_REF_DEBUG if (show_ref_count) { _PyDebug_PrintTotalRefs(); } #endif } while (ret != E_EOF); - return err; + + Py_RETURN_NONE; } @@ -188,10 +200,14 @@ PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flag return -1; } - int err = _PyRun_InteractiveLoopObject(fp, filename_obj, flags); + PyObject *result = _PyRun_InteractiveLoop(fp, filename_obj, flags); Py_DECREF(filename_obj); - return err; - + if (result == NULL) { + PyErr_Print(); + return -1; + } + Py_DECREF(result); + return 0; } @@ -287,9 +303,14 @@ pyrun_one_parse_ast(FILE *fp, PyObject *filename, /* A PyRun_InteractiveOneObject() auxiliary function that does not print the * error on failure. */ static int -PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename, - PyCompilerFlags *flags) +_PyRun_InteractiveOne(FILE *fp, PyObject *filename, PyCompilerFlags *flags) { + if (!PyUnicode_Check(filename)) { + PyErr_Format(PyExc_TypeError, "expect str for filename, got %T", + filename); + return -1; + } + PyArena *arena = _PyArena_New(); if (arena == NULL) { return -1; @@ -356,9 +377,7 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename, int PyRun_InteractiveOneObject(FILE *fp, PyObject *filename, PyCompilerFlags *flags) { - int res; - - res = PyRun_InteractiveOneObjectEx(fp, filename, flags); + int res = _PyRun_InteractiveOne(fp, filename, flags); if (res == -1) { PyErr_Print(); flush_io(); @@ -458,24 +477,26 @@ set_main_loader(PyObject *d, PyObject *filename, const char *loader_name) } -int -_PyRun_SimpleFileObject(FILE *fp, PyObject *filename, int closeit, - PyCompilerFlags *flags) +PyObject* +_PyRun_SimpleFile(FILE *fp, PyObject *filename, int closeit, + PyCompilerFlags *flags) { - int ret = -1; + PyObject *res = NULL; + int set_file_name = 0; PyObject *main_module = PyImport_AddModuleRef("__main__"); - if (main_module == NULL) - return -1; + if (main_module == NULL) { + goto done; + } PyObject *dict = PyModule_GetDict(main_module); // borrowed ref - int set_file_name = 0; int has_file = PyDict_ContainsString(dict, "__file__"); if (has_file < 0) { goto done; } if (!has_file) { if (PyDict_SetItemString(dict, "__file__", filename) < 0) { + fprintf(stderr, "python: failed to set __main__.__file__\n"); goto done; } if (PyDict_SetItemString(dict, "__cached__", Py_None) < 0) { @@ -489,7 +510,6 @@ _PyRun_SimpleFileObject(FILE *fp, PyObject *filename, int closeit, goto done; } - PyObject *v; if (pyc) { FILE *pyc_fp; /* Try to run a pyc file. First, re-open in binary */ @@ -505,42 +525,34 @@ _PyRun_SimpleFileObject(FILE *fp, PyObject *filename, int closeit, if (set_main_loader(dict, filename, "SourcelessFileLoader") < 0) { fprintf(stderr, "python: failed to set __main__.__loader__\n"); - ret = -1; fclose(pyc_fp); goto done; } - v = run_pyc_file(pyc_fp, dict, dict, flags); + res = run_pyc_file(pyc_fp, dict, dict, flags); } else { /* When running from stdin, leave __main__.__loader__ alone */ if ((!PyUnicode_Check(filename) || !PyUnicode_EqualToUTF8(filename, "<stdin>")) && set_main_loader(dict, filename, "SourceFileLoader") < 0) { fprintf(stderr, "python: failed to set __main__.__loader__\n"); - ret = -1; goto done; } - v = pyrun_file(fp, filename, Py_file_input, dict, dict, - closeit, flags); + res = _PyRun_File(fp, filename, Py_file_input, dict, dict, + closeit, flags); } flush_io(); - if (v == NULL) { - Py_CLEAR(main_module); - PyErr_Print(); - goto done; - } - Py_DECREF(v); - ret = 0; done: if (set_file_name) { if (PyDict_PopString(dict, "__file__", NULL) < 0) { - PyErr_Print(); + fprintf(stderr, "python: failed to delete __main__.__file__\n"); + Py_CLEAR(res); } if (PyDict_PopString(dict, "__cached__", NULL) < 0) { PyErr_Print(); } } Py_XDECREF(main_module); - return ret; + return res; } @@ -550,50 +562,47 @@ PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit, { PyObject *filename_obj = PyUnicode_DecodeFSDefault(filename); if (filename_obj == NULL) { + PyErr_Print(); return -1; } - int res = _PyRun_SimpleFileObject(fp, filename_obj, closeit, flags); + PyObject *result = _PyRun_SimpleFile(fp, filename_obj, closeit, flags); Py_DECREF(filename_obj); - return res; + if (result == NULL) { + PyErr_Print(); + return -1; + } + return 0; } -int -_PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompilerFlags *flags) { +PyObject* +_PyRun_SimpleString(const char *command, PyObject* name, + PyCompilerFlags *flags) +{ PyObject *main_module = PyImport_AddModuleRef("__main__"); if (main_module == NULL) { - return -1; + return NULL; } PyObject *dict = PyModule_GetDict(main_module); // borrowed ref - PyObject *res = NULL; - if (name == NULL) { - res = PyRun_StringFlags(command, Py_file_input, dict, dict, flags); - } else { - PyObject* the_name = PyUnicode_FromString(name); - if (!the_name) { - PyErr_Print(); - return -1; - } - res = _PyRun_StringFlagsWithName(command, the_name, Py_file_input, dict, dict, flags, 0); - Py_DECREF(the_name); - } + PyObject *res = _PyRun_String(command, name, Py_file_input, + dict, dict, flags, 0); Py_DECREF(main_module); + return res; +} + +int +PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags) +{ + PyObject *res = _PyRun_SimpleString(command, NULL, flags); if (res == NULL) { PyErr_Print(); return -1; } - Py_DECREF(res); return 0; } -int -PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags) -{ - return _PyRun_SimpleStringFlagsWithName(command, NULL, flags); -} - static int parse_exit_code(PyObject *code, int *exitcode_p) { @@ -1151,6 +1160,7 @@ _PyErr_Display(PyObject *file, PyObject *unused, PyObject *value, PyObject *tb) "traceback", "_print_exception_bltin"); if (print_exception_fn == NULL || !PyCallable_Check(print_exception_fn)) { + Py_XDECREF(print_exception_fn); goto fallback; } @@ -1228,11 +1238,27 @@ void PyErr_DisplayException(PyObject *exc) PyErr_Display(NULL, exc, NULL); } +static int +check_start(int start) +{ + if (start == Py_single_input || start == Py_file_input + || start == Py_eval_input || start == Py_func_type_input) + { + return 0; + } + PyErr_SetString(PyExc_ValueError, "invalid start argument"); + return -1; +} + static PyObject * -_PyRun_StringFlagsWithName(const char *str, PyObject* name, int start, - PyObject *globals, PyObject *locals, PyCompilerFlags *flags, - int generate_new_source) +_PyRun_String(const char *str, PyObject* name, int start, + PyObject *globals, PyObject *locals, PyCompilerFlags *flags, + int generate_new_source) { + if (check_start(start) < 0) { + return NULL; + } + PyObject *ret = NULL; mod_ty mod; PyArena *arena; @@ -1268,13 +1294,17 @@ PyObject * PyRun_StringFlags(const char *str, int start, PyObject *globals, PyObject *locals, PyCompilerFlags *flags) { - return _PyRun_StringFlagsWithName(str, NULL, start, globals, locals, flags, 0); + return _PyRun_String(str, NULL, start, globals, locals, flags, 0); } static PyObject * -pyrun_file(FILE *fp, PyObject *filename, int start, PyObject *globals, - PyObject *locals, int closeit, PyCompilerFlags *flags) +_PyRun_File(FILE *fp, PyObject *filename, int start, PyObject *globals, + PyObject *locals, int closeit, PyCompilerFlags *flags) { + if (check_start(start) < 0) { + return NULL; + } + PyArena *arena = _PyArena_New(); if (arena == NULL) { return NULL; @@ -1307,11 +1337,12 @@ PyRun_FileExFlags(FILE *fp, const char *filename, int start, PyObject *globals, { PyObject *filename_obj = PyUnicode_DecodeFSDefault(filename); if (filename_obj == NULL) { + PyErr_Print(); return NULL; } - PyObject *res = pyrun_file(fp, filename_obj, start, globals, - locals, closeit, flags); + PyObject *res = _PyRun_File(fp, filename_obj, start, globals, + locals, closeit, flags); Py_DECREF(filename_obj); return res; @@ -1365,6 +1396,29 @@ run_eval_code_obj(PyThreadState *tstate, PyCodeObject *co, PyObject *globals, Py return PyEval_EvalCode((PyObject*)co, globals, locals); } +static PyObject * +get_interactive_filename(PyObject *filename, Py_ssize_t count) +{ + PyObject *result; + Py_ssize_t len = PyUnicode_GET_LENGTH(filename); + + if (len >= 2 + && PyUnicode_ReadChar(filename, 0) == '<' + && PyUnicode_ReadChar(filename, len - 1) == '>') { + PyObject *middle = PyUnicode_Substring(filename, 1, len-1); + if (middle == NULL) { + return NULL; + } + result = PyUnicode_FromFormat("<%U-%zd>", middle, count); + Py_DECREF(middle); + } else { + result = PyUnicode_FromFormat( + "%U-%zd", filename, count); + } + return result; + +} + static PyObject * run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals, PyCompilerFlags *flags, PyArena *arena, PyObject* interactive_src, @@ -1375,8 +1429,8 @@ run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals, if (interactive_src) { PyInterpreterState *interp = tstate->interp; if (generate_new_source) { - interactive_filename = PyUnicode_FromFormat( - "%U-%d", filename, interp->_interactive_src_count++); + interactive_filename = get_interactive_filename( + filename, interp->_interactive_src_count++); } else { Py_INCREF(interactive_filename); } @@ -1485,6 +1539,10 @@ PyObject * Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize) { + if (check_start(start) < 0) { + return NULL; + } + PyCodeObject *co; mod_ty mod; PyArena *arena = _PyArena_New(); @@ -1524,6 +1582,26 @@ Py_CompileStringExFlags(const char *str, const char *filename_str, int start, return co; } +int +_PyObject_SupportedAsScript(PyObject *cmd) +{ + if (PyUnicode_Check(cmd)) { + return 1; + } + else if (PyBytes_Check(cmd)) { + return 1; + } + else if (PyByteArray_Check(cmd)) { + return 1; + } + else if (PyObject_CheckBuffer(cmd)) { + return 1; + } + else { + return 0; + } +} + const char * _Py_SourceAsString(PyObject *cmd, const char *funcname, const char *what, PyCompilerFlags *cf, PyObject **cmd_copy) { @@ -1671,7 +1749,7 @@ Py_CompileString(const char *str, const char *p, int s) } #undef Py_CompileStringFlags -PyAPI_FUNC(PyObject *) +PyObject* Py_CompileStringFlags(const char *str, const char *p, int s, PyCompilerFlags *flags) { diff --git a/Python/pytime.c b/Python/pytime.c index 67cf64372644909..894f6e7ca54c3eb 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -107,7 +107,7 @@ static void pytime_overflow(void) { PyErr_SetString(PyExc_OverflowError, - "timestamp too large to convert to C PyTime_t"); + "timestamp out of range for C PyTime_t"); } @@ -571,7 +571,7 @@ pytime_from_double(PyTime_t *tp, double value, _PyTime_round_t round, /* See comments in pytime_double_to_denominator */ if (!((double)PyTime_MIN <= d && d < -(double)PyTime_MIN)) { - pytime_time_t_overflow(); + pytime_overflow(); *tp = 0; return -1; } diff --git a/Python/qsbr.c b/Python/qsbr.c index bf34fb2523dfc8d..203daa0d3071729 100644 --- a/Python/qsbr.c +++ b/Python/qsbr.c @@ -41,10 +41,6 @@ // Starting size of the array of qsbr thread states #define MIN_ARRAY_SIZE 8 -// For _Py_qsbr_deferred_advance(): the number of deferrals before advancing -// the write sequence. -#define QSBR_DEFERRED_LIMIT 10 - // Allocate a QSBR thread state from the freelist static struct _qsbr_thread_state * qsbr_allocate(struct _qsbr_shared *shared) @@ -88,22 +84,29 @@ grow_thread_array(struct _qsbr_shared *shared) new_size = MIN_ARRAY_SIZE; } - struct _qsbr_pad *array = PyMem_RawCalloc(new_size, sizeof(*array)); - if (array == NULL) { + // Overallocate by 63 bytes so we can align to a 64-byte boundary. + // This avoids potential false sharing between the first entry and other + // allocations. + size_t alignment = 64; + size_t alloc_size = (size_t)new_size * sizeof(struct _qsbr_pad) + alignment - 1; + void *raw = PyMem_RawCalloc(1, alloc_size); + if (raw == NULL) { return -1; } + struct _qsbr_pad *array = _Py_ALIGN_UP(raw, alignment); - struct _qsbr_pad *old = shared->array; - if (old != NULL) { + void *old_raw = shared->array_raw; + if (shared->array != NULL) { memcpy(array, shared->array, shared->size * sizeof(*array)); } shared->array = array; + shared->array_raw = raw; shared->size = new_size; shared->freelist = NULL; initialize_new_array(shared); - PyMem_RawFree(old); + PyMem_RawFree(old_raw); return 0; } @@ -117,13 +120,9 @@ _Py_qsbr_advance(struct _qsbr_shared *shared) } uint64_t -_Py_qsbr_deferred_advance(struct _qsbr_thread_state *qsbr) +_Py_qsbr_shared_next(struct _qsbr_shared *shared) { - if (++qsbr->deferrals < QSBR_DEFERRED_LIMIT) { - return _Py_qsbr_shared_current(qsbr->shared) + QSBR_INCR; - } - qsbr->deferrals = 0; - return _Py_qsbr_advance(qsbr->shared); + return _Py_qsbr_shared_current(shared) + QSBR_INCR; } static uint64_t @@ -264,8 +263,9 @@ void _Py_qsbr_fini(PyInterpreterState *interp) { struct _qsbr_shared *shared = &interp->qsbr; - PyMem_RawFree(shared->array); + PyMem_RawFree(shared->array_raw); shared->array = NULL; + shared->array_raw = NULL; shared->size = 0; shared->freelist = NULL; } diff --git a/Python/remote_debug.h b/Python/remote_debug.h index edc77c302916ca1..38dee987e2f5032 100644 --- a/Python/remote_debug.h +++ b/Python/remote_debug.h @@ -13,6 +13,16 @@ If you need to add a new function ensure that is declared 'static'. extern "C" { #endif +#ifdef __clang__ + #define UNUSED __attribute__((unused)) +#elif defined(__GNUC__) + #define UNUSED __attribute__((unused)) +#elif defined(_MSC_VER) + #define UNUSED __pragma(warning(suppress: 4505)) +#else + #define UNUSED +#endif + #if !defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) # error "this header requires Py_BUILD_CORE or Py_BUILD_CORE_MODULE define" #endif @@ -35,7 +45,7 @@ extern "C" { # include <sys/mman.h> #endif -#if defined(__APPLE__) && TARGET_OS_OSX +#if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX # include <libproc.h> # include <mach-o/fat.h> # include <mach-o/loader.h> @@ -73,27 +83,137 @@ extern "C" { # define HAVE_PROCESS_VM_READV 0 #endif +static inline int +_Py_RemoteDebug_HasPermissionError(void) +{ + return PyErr_Occurred() + && PyErr_ExceptionMatches(PyExc_PermissionError); +} + +#define _set_debug_exception_cause(exception, format, ...) \ + do { \ + if (!PyErr_ExceptionMatches(PyExc_PermissionError)) { \ + PyThreadState *tstate = _PyThreadState_GET(); \ + if (!_PyErr_Occurred(tstate)) { \ + _PyErr_Format(tstate, exception, format, ##__VA_ARGS__); \ + } else { \ + _PyErr_FormatFromCause(exception, format, ##__VA_ARGS__); \ + } \ + } \ + } while (0) + +static inline size_t +get_page_size(void) { + size_t page_size = 0; + if (page_size == 0) { +#ifdef MS_WINDOWS + SYSTEM_INFO si; + GetSystemInfo(&si); + page_size = si.dwPageSize; +#else + page_size = (size_t)getpagesize(); +#endif + } + return page_size; +} + +typedef struct page_cache_entry { + uintptr_t page_addr; // page-aligned base address + char *data; + int valid; + struct page_cache_entry *next; +} page_cache_entry_t; + +#define MAX_PAGES 1024 + // Define a platform-independent process handle structure typedef struct { pid_t pid; -#ifdef MS_WINDOWS +#if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX + mach_port_t task; +#elif defined(MS_WINDOWS) HANDLE hProcess; +#elif defined(__linux__) + int memfd; #endif + page_cache_entry_t pages[MAX_PAGES]; + Py_ssize_t page_size; } proc_handle_t; +// Forward declaration for use in validation function +static int +_Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address, size_t len, void* dst); + +// Optional callback to validate a candidate section address found during +// memory map searches. Returns 1 if the address is valid, 0 to skip it. +// This allows callers to filter out duplicate/stale mappings (e.g. from +// ctypes dlopen) whose sections were never initialized. +typedef int (*section_validator_t)(proc_handle_t *handle, uintptr_t address); + +// Validate that a candidate address starts with _Py_Debug_Cookie. +static int +_Py_RemoteDebug_ValidatePyRuntimeCookie(proc_handle_t *handle, uintptr_t address) +{ + if (address == 0) { + return 0; + } + char buf[sizeof(_Py_Debug_Cookie) - 1]; + if (_Py_RemoteDebug_ReadRemoteMemory(handle, address, sizeof(buf), buf) != 0) { + PyErr_Clear(); + return 0; + } + return memcmp(buf, _Py_Debug_Cookie, sizeof(buf)) == 0; +} + +static void +_Py_RemoteDebug_FreePageCache(proc_handle_t *handle) +{ + for (int i = 0; i < MAX_PAGES; i++) { + PyMem_RawFree(handle->pages[i].data); + handle->pages[i].data = NULL; + handle->pages[i].valid = 0; + } +} + +UNUSED static void +_Py_RemoteDebug_ClearCache(proc_handle_t *handle) +{ + for (int i = 0; i < MAX_PAGES; i++) { + handle->pages[i].valid = 0; + } +} + +#if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX +static mach_port_t pid_to_task(pid_t pid); +#endif + // Initialize the process handle static int _Py_RemoteDebug_InitProcHandle(proc_handle_t *handle, pid_t pid) { handle->pid = pid; -#ifdef MS_WINDOWS +#if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX + handle->task = pid_to_task(handle->pid); + if (handle->task == 0) { + _set_debug_exception_cause(PyExc_RuntimeError, "Failed to initialize macOS process handle"); + return -1; + } +#elif defined(MS_WINDOWS) handle->hProcess = OpenProcess( PROCESS_VM_READ | PROCESS_VM_WRITE | PROCESS_VM_OPERATION | PROCESS_QUERY_INFORMATION, FALSE, pid); if (handle->hProcess == NULL) { PyErr_SetFromWindowsErr(0); + _set_debug_exception_cause(PyExc_RuntimeError, "Failed to initialize Windows process handle"); return -1; } +#elif defined(__linux__) + handle->memfd = -1; #endif + handle->page_size = get_page_size(); + for (int i = 0; i < MAX_PAGES; i++) { + handle->pages[i].data = NULL; + handle->pages[i].valid = 0; + } return 0; } @@ -105,11 +225,17 @@ _Py_RemoteDebug_CleanupProcHandle(proc_handle_t *handle) { CloseHandle(handle->hProcess); handle->hProcess = NULL; } +#elif defined(__linux__) + if (handle->memfd != -1) { + close(handle->memfd); + handle->memfd = -1; + } #endif handle->pid = 0; + _Py_RemoteDebug_FreePageCache(handle); } -#if defined(__APPLE__) && TARGET_OS_OSX +#if defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX static uintptr_t return_section_address64( @@ -148,8 +274,10 @@ return_section_address64( &object_name ); if (ret != KERN_SUCCESS) { - PyErr_SetString( - PyExc_RuntimeError, "Cannot get any more VM maps.\n"); + PyErr_Format(PyExc_RuntimeError, + "mach_vm_region failed while parsing 64-bit Mach-O binary " + "at base address 0x%lx (kern_return_t: %d)", + base, ret); return 0; } } @@ -169,9 +297,6 @@ return_section_address64( cmd = (struct segment_command_64*)((void*)cmd + cmd->cmdsize); } - // We should not be here, but if we are there, we should say about this - PyErr_SetString( - PyExc_RuntimeError, "Cannot find section address.\n"); return 0; } @@ -212,8 +337,10 @@ return_section_address32( &object_name ); if (ret != KERN_SUCCESS) { - PyErr_SetString( - PyExc_RuntimeError, "Cannot get any more VM maps.\n"); + PyErr_Format(PyExc_RuntimeError, + "mach_vm_region failed while parsing 32-bit Mach-O binary " + "at base address 0x%lx (kern_return_t: %d)", + base, ret); return 0; } } @@ -233,9 +360,6 @@ return_section_address32( cmd = (struct segment_command*)((void*)cmd + cmd->cmdsize); } - // We should not be here, but if we are there, we should say about this - PyErr_SetString( - PyExc_RuntimeError, "Cannot find section address.\n"); return 0; } @@ -253,8 +377,20 @@ return_section_address_fat( int is_abi64; size_t cpu_size = sizeof(cpu), abi64_size = sizeof(is_abi64); - sysctlbyname("hw.cputype", &cpu, &cpu_size, NULL, 0); - sysctlbyname("hw.cpu64bit_capable", &is_abi64, &abi64_size, NULL, 0); + if (sysctlbyname("hw.cputype", &cpu, &cpu_size, NULL, 0) != 0) { + PyErr_Format(PyExc_OSError, + "Failed to determine CPU type via sysctlbyname " + "for fat binary analysis at 0x%lx: %s", + base, strerror(errno)); + return 0; + } + if (sysctlbyname("hw.cpu64bit_capable", &is_abi64, &abi64_size, NULL, 0) != 0) { + PyErr_Format(PyExc_OSError, + "Failed to determine CPU ABI capability via sysctlbyname " + "for fat binary analysis at 0x%lx: %s", + base, strerror(errno)); + return 0; + } cpu |= is_abi64 * CPU_ARCH_ABI64; @@ -285,13 +421,18 @@ return_section_address_fat( return return_section_address64(section, proc_ref, base, (void*)hdr); default: - PyErr_SetString(PyExc_RuntimeError, "Unknown Mach-O magic in fat binary.\n"); + PyErr_Format(PyExc_RuntimeError, + "Unknown Mach-O magic number 0x%x in fat binary architecture %u at base 0x%lx", + hdr->magic, i, base); return 0; } } } - PyErr_SetString(PyExc_RuntimeError, "No matching architecture found in fat binary.\n"); + PyErr_Format(PyExc_RuntimeError, + "No matching architecture found for CPU type 0x%x " + "in fat binary at base 0x%lx (%u architectures examined)", + cpu, base, nfat_arch); return 0; } @@ -300,20 +441,26 @@ search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_ { int fd = open(path, O_RDONLY); if (fd == -1) { - PyErr_Format(PyExc_RuntimeError, "Cannot open binary %s\n", path); + PyErr_Format(PyExc_OSError, + "Cannot open binary file '%s' for section '%s' search: %s", + path, secname, strerror(errno)); return 0; } struct stat fs; if (fstat(fd, &fs) == -1) { - PyErr_Format(PyExc_RuntimeError, "Cannot get size of binary %s\n", path); + PyErr_Format(PyExc_OSError, + "Cannot get file size for binary '%s' during section '%s' search: %s", + path, secname, strerror(errno)); close(fd); return 0; } void* map = mmap(0, fs.st_size, PROT_READ, MAP_SHARED, fd, 0); if (map == MAP_FAILED) { - PyErr_Format(PyExc_RuntimeError, "Cannot map binary %s\n", path); + PyErr_Format(PyExc_OSError, + "Cannot memory map binary file '%s' (size: %lld bytes) for section '%s' search: %s", + path, (long long)fs.st_size, secname, strerror(errno)); close(fd); return 0; } @@ -335,13 +482,22 @@ search_section_in_file(const char* secname, char* path, uintptr_t base, mach_vm_ result = return_section_address_fat(secname, proc_ref, base, map); break; default: - PyErr_SetString(PyExc_RuntimeError, "Unknown Mach-O magic"); + PyErr_Format(PyExc_RuntimeError, + "Unrecognized Mach-O magic number 0x%x in binary file '%s' for section '%s' search", + magic, path, secname); break; } - munmap(map, fs.st_size); + if (munmap(map, fs.st_size) != 0) { + PyErr_Format(PyExc_OSError, + "Failed to unmap binary file '%s' (size: %lld bytes): %s", + path, (long long)fs.st_size, strerror(errno)); + result = 0; + } if (close(fd) != 0) { - PyErr_SetFromErrno(PyExc_OSError); + PyErr_Format(PyExc_OSError, + "Failed to close binary file '%s': %s", + path, strerror(errno)); result = 0; } return result; @@ -356,14 +512,18 @@ pid_to_task(pid_t pid) result = task_for_pid(mach_task_self(), pid, &task); if (result != KERN_SUCCESS) { - PyErr_Format(PyExc_PermissionError, "Cannot get task for PID %d", pid); + PyErr_Format(PyExc_PermissionError, + "Cannot get task port for PID %d (kern_return_t: %d). " + "This typically requires running as root or having the 'com.apple.system-task-ports' entitlement.", + pid, result); return 0; } return task; } static uintptr_t -search_map_for_section(proc_handle_t *handle, const char* secname, const char* substr) { +search_map_for_section(proc_handle_t *handle, const char* secname, const char* substr, + section_validator_t validator) { mach_vm_address_t address = 0; mach_vm_size_t size = 0; mach_msg_type_number_t count = sizeof(vm_region_basic_info_data_64_t); @@ -373,13 +533,15 @@ search_map_for_section(proc_handle_t *handle, const char* secname, const char* s mach_port_t proc_ref = pid_to_task(handle->pid); if (proc_ref == 0) { if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_PermissionError, "Cannot get task for PID"); + PyErr_Format(PyExc_PermissionError, + "Cannot get task port for PID %d during section search", + handle->pid); } return 0; } - int match_found = 0; char map_filename[MAXPATHLEN + 1]; + while (mach_vm_region( proc_ref, &address, @@ -389,6 +551,7 @@ search_map_for_section(proc_handle_t *handle, const char* secname, const char* s &count, &object_name) == KERN_SUCCESS) { + if ((region_info.protection & VM_PROT_READ) == 0 || (region_info.protection & VM_PROT_EXECUTE) == 0) { address += size; @@ -409,21 +572,23 @@ search_map_for_section(proc_handle_t *handle, const char* secname, const char* s filename = map_filename; // No path, use the whole string } - if (!match_found && strncmp(filename, substr, strlen(substr)) == 0) { - match_found = 1; - return search_section_in_file( + if (strncmp(filename, substr, strlen(substr)) == 0) { + uintptr_t result = search_section_in_file( secname, map_filename, address, size, proc_ref); + if (result != 0 + && (validator == NULL || validator(handle, result))) + { + return result; + } } address += size; } - PyErr_SetString(PyExc_RuntimeError, - "mach_vm_region failed to find the section"); return 0; } -#endif // (__APPLE__ && TARGET_OS_OSX) +#endif // (__APPLE__ && defined(TARGET_OS_OSX) && TARGET_OS_OSX) #if defined(__linux__) && HAVE_PROCESS_VM_READV static uintptr_t @@ -442,24 +607,38 @@ search_elf_file_for_section( int fd = open(elf_file, O_RDONLY); if (fd < 0) { - PyErr_SetFromErrno(PyExc_OSError); + PyErr_Format(PyExc_OSError, + "Cannot open ELF file '%s' for section '%s' search: %s", + elf_file, secname, strerror(errno)); goto exit; } struct stat file_stats; if (fstat(fd, &file_stats) != 0) { - PyErr_SetFromErrno(PyExc_OSError); + PyErr_Format(PyExc_OSError, + "Cannot get file size for ELF file '%s' during section '%s' search: %s", + elf_file, secname, strerror(errno)); goto exit; } file_memory = mmap(NULL, file_stats.st_size, PROT_READ, MAP_PRIVATE, fd, 0); if (file_memory == MAP_FAILED) { - PyErr_SetFromErrno(PyExc_OSError); + PyErr_Format(PyExc_OSError, + "Cannot memory map ELF file '%s' (size: %lld bytes) for section '%s' search: %s", + elf_file, (long long)file_stats.st_size, secname, strerror(errno)); goto exit; } Elf_Ehdr* elf_header = (Elf_Ehdr*)file_memory; + // Validate ELF header + if (elf_header->e_shstrndx >= elf_header->e_shnum) { + PyErr_Format(PyExc_RuntimeError, + "Invalid ELF file '%s': string table index %u >= section count %u", + elf_file, elf_header->e_shstrndx, elf_header->e_shnum); + goto exit; + } + Elf_Shdr* section_header_table = (Elf_Shdr*)(file_memory + elf_header->e_shoff); Elf_Shdr* shstrtab_section = &section_header_table[elf_header->e_shstrndx]; @@ -476,6 +655,10 @@ search_elf_file_for_section( } } + if (section == NULL) { + goto exit; + } + Elf_Phdr* program_header_table = (Elf_Phdr*)(file_memory + elf_header->e_phoff); // Find the first PT_LOAD segment Elf_Phdr* first_load_segment = NULL; @@ -486,32 +669,142 @@ search_elf_file_for_section( } } - if (section != NULL && first_load_segment != NULL) { - uintptr_t elf_load_addr = first_load_segment->p_vaddr - - (first_load_segment->p_vaddr % first_load_segment->p_align); - result = start_address + (uintptr_t)section->sh_addr - elf_load_addr; + if (first_load_segment == NULL) { + PyErr_Format(PyExc_RuntimeError, + "No PT_LOAD segment found in ELF file '%s' (%u program headers examined)", + elf_file, elf_header->e_phnum); + goto exit; } + uintptr_t elf_load_addr = first_load_segment->p_vaddr + - (first_load_segment->p_vaddr % first_load_segment->p_align); + result = start_address + (uintptr_t)section->sh_addr - elf_load_addr; + exit: if (file_memory != NULL) { munmap(file_memory, file_stats.st_size); } if (fd >= 0 && close(fd) != 0) { - PyErr_SetFromErrno(PyExc_OSError); + PyErr_Format(PyExc_OSError, + "Failed to close ELF file '%s': %s", + elf_file, strerror(errno)); result = 0; } return result; } +static const char * +find_debug_cookie(const char *buffer, size_t len) +{ + const char *cookie = _Py_Debug_Cookie; + const size_t cookie_len = sizeof(_Py_Debug_Cookie) - 1; + if (len < cookie_len) { + return NULL; + } + + size_t pos = 0; + size_t last = len - cookie_len; + while (pos <= last) { + const char *candidate = memchr( + buffer + pos, cookie[0], last - pos + 1); + if (candidate == NULL) { + return NULL; + } + pos = (size_t)(candidate - buffer); + if (memcmp(candidate, cookie, cookie_len) == 0) { + return candidate; + } + pos++; + } + return NULL; +} + +static int +linux_map_path_is_deleted(const char *path) +{ + static const char deleted_suffix[] = " (deleted)"; + size_t path_len = strlen(path); + size_t suffix_len = sizeof(deleted_suffix) - 1; + return path_len >= suffix_len + && strcmp(path + path_len - suffix_len, deleted_suffix) == 0; +} + +static int +linux_map_perms_are_readwrite(const char *perms) +{ + return perms[0] == 'r' && perms[1] == 'w'; +} + static uintptr_t -search_linux_map_for_section(proc_handle_t *handle, const char* secname, const char* substr) +scan_linux_mapping_for_pyruntime_cookie( + proc_handle_t *handle, + uintptr_t start, + uintptr_t end) +{ + if (end <= start) { + return 0; + } + + const size_t cookie_len = sizeof(_Py_Debug_Cookie) - 1; + const size_t overlap = cookie_len - 1; + const size_t chunk_size = 1024 * 1024; + char *buffer = PyMem_Malloc(chunk_size); + if (buffer == NULL) { + PyErr_NoMemory(); + _set_debug_exception_cause(PyExc_MemoryError, + "Cannot allocate memory while scanning PID %d for PyRuntime cookie", + handle->pid); + return 0; + } + + uintptr_t retval = 0; + uintptr_t mapping_size = end - start; + uintptr_t offset = 0; + while (offset < mapping_size) { + uintptr_t remaining = mapping_size - offset; + size_t wanted = remaining > chunk_size + ? chunk_size : (size_t)remaining; + if (_Py_RemoteDebug_ReadRemoteMemory( + handle, start + offset, wanted, buffer) < 0) { + if (_Py_RemoteDebug_HasPermissionError()) { + goto exit; + } + // A candidate mapping can disappear or contain unreadable holes while + // the target process keeps running. Treat those as non-matches and + // keep scanning other candidate mappings. + PyErr_Clear(); + } + else { + const char *hit = find_debug_cookie(buffer, wanted); + if (hit != NULL) { + retval = start + offset + (uintptr_t)(hit - buffer); + goto exit; + } + } + + if (wanted <= overlap) { + break; + } + offset += wanted - overlap; + } + +exit: + PyMem_Free(buffer); + return retval; +} + +static uintptr_t +search_linux_map_for_section(proc_handle_t *handle, const char* secname, const char* substr, + section_validator_t validator) { char maps_file_path[64]; sprintf(maps_file_path, "/proc/%d/maps", handle->pid); FILE* maps_file = fopen(maps_file_path, "r"); if (maps_file == NULL) { - PyErr_SetFromErrno(PyExc_OSError); + PyErr_Format(PyExc_OSError, + "Cannot open process memory map file '%s' for PID %d section search: %s", + maps_file_path, handle->pid, strerror(errno)); return 0; } @@ -520,11 +813,14 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c char *line = PyMem_Malloc(linesz); if (!line) { fclose(maps_file); - PyErr_NoMemory(); + _set_debug_exception_cause(PyExc_MemoryError, + "Cannot allocate memory for reading process map file '%s'", + maps_file_path); return 0; } uintptr_t retval = 0; + while (fgets(line + linelen, linesz - linelen, maps_file) != NULL) { linelen = strlen(line); if (line[linelen - 1] != '\n') { @@ -535,7 +831,9 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c if (!biggerline) { PyMem_Free(line); fclose(maps_file); - PyErr_NoMemory(); + _set_debug_exception_cause(PyExc_MemoryError, + "Cannot reallocate memory while reading process map file '%s' (attempted size: %zu)", + maps_file_path, linesz); return 0; } line = biggerline; @@ -548,16 +846,27 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c linelen = 0; unsigned long start = 0; - unsigned long path_pos = 0; - sscanf(line, "%lx-%*x %*s %*s %*s %*s %ln", &start, &path_pos); + unsigned long end = 0; + int path_pos = 0; + char perms[5] = ""; + int fields = sscanf(line, "%lx-%lx %4s %*s %*s %*s %n", + &start, &end, perms, &path_pos); - if (!path_pos) { + if (fields < 3 || !path_pos) { // Line didn't match our format string. This shouldn't be // possible, but let's be defensive and skip the line. continue; } const char *path = line + path_pos; + if (path[0] == '\0') { + continue; + } + if (path[0] == '[' && path[strlen(path)-1] == ']') { + // Skip [heap], [stack], [anon:cpython:pymalloc], etc. + continue; + } + const char *filename = strrchr(path, '/'); if (filename) { filename++; // Move past the '/' @@ -566,16 +875,42 @@ search_linux_map_for_section(proc_handle_t *handle, const char* secname, const c } if (strstr(filename, substr)) { - retval = search_elf_file_for_section(handle, secname, start, path); + int deleted_pyruntime_mapping = + strcmp(secname, "PyRuntime") == 0 + && linux_map_path_is_deleted(path); + if (deleted_pyruntime_mapping + && linux_map_perms_are_readwrite(perms)) { + PyErr_Clear(); + retval = scan_linux_mapping_for_pyruntime_cookie( + handle, (uintptr_t)start, (uintptr_t)end); + } + if (!deleted_pyruntime_mapping + && retval == 0 && !PyErr_Occurred()) { + PyErr_Clear(); + retval = search_elf_file_for_section( + handle, secname, start, path); + } if (retval) { + if (validator == NULL || validator(handle, retval)) { + break; + } + if (_Py_RemoteDebug_HasPermissionError()) { + retval = 0; + break; + } + } + else if (_Py_RemoteDebug_HasPermissionError()) { break; } + retval = 0; } } PyMem_Free(line); if (fclose(maps_file) != 0) { - PyErr_SetFromErrno(PyExc_OSError); + PyErr_Format(PyExc_OSError, + "Failed to close process map file '%s': %s", + maps_file_path, strerror(errno)); retval = 0; } @@ -591,11 +926,20 @@ static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char* HANDLE hFile = CreateFileW(mod_path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); if (hFile == INVALID_HANDLE_VALUE) { PyErr_SetFromWindowsErr(0); + DWORD error = GetLastError(); + PyErr_Format(PyExc_OSError, + "Cannot open PE file for section '%s' analysis (error %lu)", + secname, error); return NULL; } + HANDLE hMap = CreateFileMapping(hFile, NULL, PAGE_READONLY, 0, 0, 0); if (!hMap) { PyErr_SetFromWindowsErr(0); + DWORD error = GetLastError(); + PyErr_Format(PyExc_OSError, + "Cannot create file mapping for PE file section '%s' analysis (error %lu)", + secname, error); CloseHandle(hFile); return NULL; } @@ -603,6 +947,10 @@ static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char* BYTE* mapView = (BYTE*)MapViewOfFile(hMap, FILE_MAP_READ, 0, 0, 0); if (!mapView) { PyErr_SetFromWindowsErr(0); + DWORD error = GetLastError(); + PyErr_Format(PyExc_OSError, + "Cannot map view of PE file for section '%s' analysis (error %lu)", + secname, error); CloseHandle(hMap); CloseHandle(hFile); return NULL; @@ -610,7 +958,9 @@ static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char* IMAGE_DOS_HEADER* pDOSHeader = (IMAGE_DOS_HEADER*)mapView; if (pDOSHeader->e_magic != IMAGE_DOS_SIGNATURE) { - PyErr_SetString(PyExc_RuntimeError, "Invalid DOS signature."); + PyErr_Format(PyExc_RuntimeError, + "Invalid DOS signature (0x%x) in PE file for section '%s' analysis (expected 0x%x)", + pDOSHeader->e_magic, secname, IMAGE_DOS_SIGNATURE); UnmapViewOfFile(mapView); CloseHandle(hMap); CloseHandle(hFile); @@ -619,7 +969,9 @@ static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char* IMAGE_NT_HEADERS* pNTHeaders = (IMAGE_NT_HEADERS*)(mapView + pDOSHeader->e_lfanew); if (pNTHeaders->Signature != IMAGE_NT_SIGNATURE) { - PyErr_SetString(PyExc_RuntimeError, "Invalid NT signature."); + PyErr_Format(PyExc_RuntimeError, + "Invalid NT signature (0x%lx) in PE file for section '%s' analysis (expected 0x%lx)", + pNTHeaders->Signature, secname, IMAGE_NT_SIGNATURE); UnmapViewOfFile(mapView); CloseHandle(hMap); CloseHandle(hFile); @@ -646,14 +998,20 @@ static void* analyze_pe(const wchar_t* mod_path, BYTE* remote_base, const char* static uintptr_t -search_windows_map_for_section(proc_handle_t* handle, const char* secname, const wchar_t* substr) { +search_windows_map_for_section(proc_handle_t* handle, const char* secname, const wchar_t* substr, + section_validator_t validator) { HANDLE hProcSnap; do { hProcSnap = CreateToolhelp32Snapshot(TH32CS_SNAPMODULE, handle->pid); } while (hProcSnap == INVALID_HANDLE_VALUE && GetLastError() == ERROR_BAD_LENGTH); if (hProcSnap == INVALID_HANDLE_VALUE) { - PyErr_SetString(PyExc_PermissionError, "Unable to create module snapshot. Check permissions or PID."); + PyErr_SetFromWindowsErr(0); + DWORD error = GetLastError(); + PyErr_Format(PyExc_PermissionError, + "Unable to create module snapshot for PID %d section '%s' " + "search (error %lu). Check permissions or PID validity", + handle->pid, secname, error); return 0; } @@ -664,14 +1022,18 @@ search_windows_map_for_section(proc_handle_t* handle, const char* secname, const for (BOOL hasModule = Module32FirstW(hProcSnap, &moduleEntry); hasModule; hasModule = Module32NextW(hProcSnap, &moduleEntry)) { // Look for either python executable or DLL if (wcsstr(moduleEntry.szModule, substr)) { - runtime_addr = analyze_pe(moduleEntry.szExePath, moduleEntry.modBaseAddr, secname); - if (runtime_addr != NULL) { + void *candidate = analyze_pe(moduleEntry.szExePath, moduleEntry.modBaseAddr, secname); + if (candidate != NULL + && (validator == NULL || validator(handle, (uintptr_t)candidate))) + { + runtime_addr = candidate; break; } } } CloseHandle(hProcSnap); + return (uintptr_t)runtime_addr; } @@ -685,29 +1047,46 @@ _Py_RemoteDebug_GetPyRuntimeAddress(proc_handle_t* handle) #ifdef MS_WINDOWS // On Windows, search for 'python' in executable or DLL - address = search_windows_map_for_section(handle, "PyRuntime", L"python"); + address = search_windows_map_for_section(handle, "PyRuntime", L"python", + _Py_RemoteDebug_ValidatePyRuntimeCookie); if (address == 0) { // Error out: 'python' substring covers both executable and DLL PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(PyExc_RuntimeError, "Failed to find the PyRuntime section in the process."); + PyErr_Format(PyExc_RuntimeError, + "Failed to find the PyRuntime section in process %d on Windows platform", + handle->pid); _PyErr_ChainExceptions1(exc); } #elif defined(__linux__) // On Linux, search for 'python' in executable or DLL - address = search_linux_map_for_section(handle, "PyRuntime", "python"); + address = search_linux_map_for_section(handle, "PyRuntime", "python", + _Py_RemoteDebug_ValidatePyRuntimeCookie); if (address == 0) { // Error out: 'python' substring covers both executable and DLL PyObject *exc = PyErr_GetRaisedException(); - PyErr_SetString(PyExc_RuntimeError, "Failed to find the PyRuntime section in the process."); + PyErr_Format(PyExc_RuntimeError, + "Failed to find the PyRuntime section in process %d on Linux platform", + handle->pid); _PyErr_ChainExceptions1(exc); } -#elif defined(__APPLE__) && TARGET_OS_OSX +#elif defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX // On macOS, try libpython first, then fall back to python - address = search_map_for_section(handle, "PyRuntime", "libpython"); - if (address == 0) { - // TODO: Differentiate between not found and error + const char* candidates[] = {"libpython", "python", "Python", NULL}; + for (const char** candidate = candidates; *candidate; candidate++) { PyErr_Clear(); - address = search_map_for_section(handle, "PyRuntime", "python"); + address = search_map_for_section(handle, "PyRuntime", *candidate, + _Py_RemoteDebug_ValidatePyRuntimeCookie); + if (address != 0) { + break; + } + } + if (address == 0) { + PyObject *exc = PyErr_GetRaisedException(); + PyErr_Format(PyExc_RuntimeError, + "Failed to find the PyRuntime section in process %d " + "on macOS platform (tried both libpython and python)", + handle->pid); + _PyErr_ChainExceptions1(exc); } #else Py_UNREACHABLE(); @@ -716,6 +1095,61 @@ _Py_RemoteDebug_GetPyRuntimeAddress(proc_handle_t* handle) return address; } +#if defined(__linux__) && HAVE_PROCESS_VM_READV + +static int +open_proc_mem_fd(proc_handle_t *handle) +{ + char mem_file_path[64]; + sprintf(mem_file_path, "/proc/%d/mem", handle->pid); + + handle->memfd = open(mem_file_path, O_RDWR); + if (handle->memfd == -1) { + PyErr_SetFromErrno(PyExc_OSError); + _set_debug_exception_cause(PyExc_OSError, + "failed to open file %s: %s", mem_file_path, strerror(errno)); + return -1; + } + return 0; +} + +// Why is pwritev not guarded? Except on Android API level 23 (no longer +// supported), HAVE_PROCESS_VM_READV is sufficient. +static int +read_remote_memory_fallback(proc_handle_t *handle, uintptr_t remote_address, size_t len, void* dst) +{ + if (handle->memfd == -1) { + if (open_proc_mem_fd(handle) < 0) { + return -1; + } + } + + struct iovec local[1]; + Py_ssize_t result = 0; + Py_ssize_t read_bytes = 0; + + do { + local[0].iov_base = (char*)dst + result; + local[0].iov_len = len - result; + off_t offset = remote_address + result; + + read_bytes = preadv(handle->memfd, local, 1, offset); + if (read_bytes < 0) { + PyErr_SetFromErrno(PyExc_OSError); + _set_debug_exception_cause(PyExc_OSError, + "preadv failed for PID %d at address 0x%lx " + "(size %zu, partial read %zd bytes): %s", + handle->pid, remote_address + result, len - result, result, strerror(errno)); + return -1; + } + + result += read_bytes; + } while ((size_t)read_bytes != local[0].iov_len); + return 0; +} + +#endif // __linux__ + // Platform-independent memory read function static int _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address, size_t len, void* dst) @@ -726,12 +1160,20 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address do { if (!ReadProcessMemory(handle->hProcess, (LPCVOID)(remote_address + result), (char*)dst + result, len - result, &read_bytes)) { PyErr_SetFromWindowsErr(0); + DWORD error = GetLastError(); + _set_debug_exception_cause(PyExc_OSError, + "ReadProcessMemory failed for PID %d at address 0x%lx " + "(size %zu, partial read %zu bytes): Windows error %lu", + handle->pid, remote_address + result, len - result, result, error); return -1; } result += read_bytes; } while (result < len); return 0; #elif defined(__linux__) && HAVE_PROCESS_VM_READV + if (handle->memfd != -1) { + return read_remote_memory_fallback(handle, remote_address, len, dst); + } struct iovec local[1]; struct iovec remote[1]; Py_ssize_t result = 0; @@ -745,17 +1187,24 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address read_bytes = process_vm_readv(handle->pid, local, 1, remote, 1, 0); if (read_bytes < 0) { + if (errno == ENOSYS) { + return read_remote_memory_fallback(handle, remote_address, len, dst); + } PyErr_SetFromErrno(PyExc_OSError); + _set_debug_exception_cause(PyExc_OSError, + "process_vm_readv failed for PID %d at address 0x%lx " + "(size %zu, partial read %zd bytes): %s", + handle->pid, remote_address + result, len - result, result, strerror(errno)); return -1; } result += read_bytes; } while ((size_t)read_bytes != local[0].iov_len); return 0; -#elif defined(__APPLE__) && TARGET_OS_OSX +#elif defined(__APPLE__) && defined(TARGET_OS_OSX) && TARGET_OS_OSX Py_ssize_t result = -1; kern_return_t kr = mach_vm_read_overwrite( - pid_to_task(handle->pid), + handle->task, (mach_vm_address_t)remote_address, len, (mach_vm_address_t)dst, @@ -764,13 +1213,22 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address if (kr != KERN_SUCCESS) { switch (kr) { case KERN_PROTECTION_FAILURE: - PyErr_SetString(PyExc_PermissionError, "Not enough permissions to read memory"); + PyErr_Format(PyExc_PermissionError, + "Memory protection failure reading from PID %d at address " + "0x%lx (size %zu): insufficient permissions", + handle->pid, remote_address, len); break; case KERN_INVALID_ARGUMENT: - PyErr_SetString(PyExc_PermissionError, "Invalid argument to mach_vm_read_overwrite"); + PyErr_Format(PyExc_ValueError, + "Invalid argument to mach_vm_read_overwrite for PID %d at " + "address 0x%lx (size %zu)", + handle->pid, remote_address, len); break; default: - PyErr_SetString(PyExc_RuntimeError, "Unknown error reading memory"); + PyErr_Format(PyExc_RuntimeError, + "mach_vm_read_overwrite failed for PID %d at address 0x%lx " + "(size %zu): kern_return_t %d", + handle->pid, remote_address, len, kr); } return -1; } @@ -780,6 +1238,63 @@ _Py_RemoteDebug_ReadRemoteMemory(proc_handle_t *handle, uintptr_t remote_address #endif } +UNUSED static int +_Py_RemoteDebug_PagedReadRemoteMemory(proc_handle_t *handle, + uintptr_t addr, + size_t size, + void *out) +{ + size_t page_size = handle->page_size; + uintptr_t page_base = addr & ~(page_size - 1); + size_t offset_in_page = addr - page_base; + + if (offset_in_page + size > page_size) { + return _Py_RemoteDebug_ReadRemoteMemory(handle, addr, size, out); + } + + // Search for valid cached page + for (int i = 0; i < MAX_PAGES; i++) { + page_cache_entry_t *entry = &handle->pages[i]; + if (entry->valid && entry->page_addr == page_base) { + memcpy(out, entry->data + offset_in_page, size); + return 0; + } + } + + // Find reusable slot + for (int i = 0; i < MAX_PAGES; i++) { + page_cache_entry_t *entry = &handle->pages[i]; + if (!entry->valid) { + if (entry->data == NULL) { + entry->data = PyMem_RawMalloc(page_size); + if (entry->data == NULL) { + PyErr_NoMemory(); + _set_debug_exception_cause(PyExc_MemoryError, + "Cannot allocate %zu bytes for page cache entry " + "during read from PID %d at address 0x%lx", + page_size, handle->pid, addr); + return -1; + } + } + + if (_Py_RemoteDebug_ReadRemoteMemory(handle, page_base, page_size, entry->data) < 0) { + // Try to just copy the exact amount as a fallback + PyErr_Clear(); + goto fallback; + } + + entry->page_addr = page_base; + entry->valid = 1; + memcpy(out, entry->data + offset_in_page, size); + return 0; + } + } + +fallback: + // Cache full — fallback to uncached read + return _Py_RemoteDebug_ReadRemoteMemory(handle, addr, size, out); +} + static int _Py_RemoteDebug_ReadDebugOffsets( proc_handle_t *handle, @@ -789,13 +1304,16 @@ _Py_RemoteDebug_ReadDebugOffsets( *runtime_start_address = _Py_RemoteDebug_GetPyRuntimeAddress(handle); if (!*runtime_start_address) { if (!PyErr_Occurred()) { - PyErr_SetString( - PyExc_RuntimeError, "Failed to get PyRuntime address"); + PyErr_Format(PyExc_RuntimeError, + "Failed to locate PyRuntime address for PID %d", + handle->pid); } + _set_debug_exception_cause(PyExc_RuntimeError, "PyRuntime address lookup failed during debug offsets initialization"); return -1; } size_t size = sizeof(struct _Py_DebugOffsets); if (0 != _Py_RemoteDebug_ReadRemoteMemory(handle, *runtime_start_address, size, debug_offsets)) { + _set_debug_exception_cause(PyExc_RuntimeError, "Failed to read debug offsets structure from remote process"); return -1; } return 0; diff --git a/Python/remote_debugging.c b/Python/remote_debugging.c index dd55b7812d4deec..71ffb17ed68b1d0 100644 --- a/Python/remote_debugging.c +++ b/Python/remote_debugging.c @@ -19,11 +19,44 @@ cleanup_proc_handle(proc_handle_t *handle) { } static int -read_memory(proc_handle_t *handle, uint64_t remote_address, size_t len, void* dst) +read_memory(proc_handle_t *handle, uintptr_t remote_address, size_t len, void* dst) { return _Py_RemoteDebug_ReadRemoteMemory(handle, remote_address, len, dst); } +// Why is pwritev not guarded? Except on Android API level 23 (no longer +// supported), HAVE_PROCESS_VM_READV is sufficient. +#if defined(__linux__) && HAVE_PROCESS_VM_READV +static int +write_memory_fallback(proc_handle_t *handle, uintptr_t remote_address, size_t len, const void* src) +{ + if (handle->memfd == -1) { + if (open_proc_mem_fd(handle) < 0) { + return -1; + } + } + + struct iovec local[1]; + Py_ssize_t result = 0; + Py_ssize_t written = 0; + + do { + local[0].iov_base = (char*)src + result; + local[0].iov_len = len - result; + off_t offset = remote_address + result; + + written = pwritev(handle->memfd, local, 1, offset); + if (written < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + + result += written; + } while ((size_t)written != local[0].iov_len); + return 0; +} +#endif // __linux__ + static int write_memory(proc_handle_t *handle, uintptr_t remote_address, size_t len, const void* src) { @@ -39,6 +72,9 @@ write_memory(proc_handle_t *handle, uintptr_t remote_address, size_t len, const } while (result < len); return 0; #elif defined(__linux__) && HAVE_PROCESS_VM_READV + if (handle->memfd != -1) { + return write_memory_fallback(handle, remote_address, len, src); + } struct iovec local[1]; struct iovec remote[1]; Py_ssize_t result = 0; @@ -52,6 +88,9 @@ write_memory(proc_handle_t *handle, uintptr_t remote_address, size_t len, const written = process_vm_writev(handle->pid, local, 1, remote, 1, 0); if (written < 0) { + if (errno == ENOSYS) { + return write_memory_fallback(handle, remote_address, len, src); + } PyErr_SetFromErrno(PyExc_OSError); return -1; } @@ -196,7 +235,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc int is_remote_debugging_enabled = 0; if (0 != read_memory( handle, - interpreter_state_addr + debug_offsets.debugger_support.remote_debugging_enabled, + interpreter_state_addr + (uintptr_t)debug_offsets.debugger_support.remote_debugging_enabled, sizeof(int), &is_remote_debugging_enabled)) { @@ -216,7 +255,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc if (tid != 0) { if (0 != read_memory( handle, - interpreter_state_addr + debug_offsets.interpreter_state.threads_head, + interpreter_state_addr + (uintptr_t)debug_offsets.interpreter_state.threads_head, sizeof(void*), &thread_state_addr)) { @@ -225,7 +264,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc while (thread_state_addr != 0) { if (0 != read_memory( handle, - thread_state_addr + debug_offsets.thread_state.native_thread_id, + thread_state_addr + (uintptr_t)debug_offsets.thread_state.native_thread_id, sizeof(this_tid), &this_tid)) { @@ -238,7 +277,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc if (0 != read_memory( handle, - thread_state_addr + debug_offsets.thread_state.next, + thread_state_addr + (uintptr_t)debug_offsets.thread_state.next, sizeof(void*), &thread_state_addr)) { @@ -255,7 +294,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc } else { if (0 != read_memory( handle, - interpreter_state_addr + debug_offsets.interpreter_state.threads_main, + interpreter_state_addr + (uintptr_t)debug_offsets.interpreter_state.threads_main, sizeof(void*), &thread_state_addr)) { @@ -307,7 +346,7 @@ send_exec_to_proc_handle(proc_handle_t *handle, int tid, const char *debugger_sc uintptr_t eval_breaker; if (0 != read_memory( handle, - thread_state_addr + debug_offsets.debugger_support.eval_breaker, + thread_state_addr + (uintptr_t)debug_offsets.debugger_support.eval_breaker, sizeof(uintptr_t), &eval_breaker)) { diff --git a/Python/specialize.c b/Python/specialize.c index bbe725c8ec8381b..4ba854d2ae8c3a8 100644 --- a/Python/specialize.c +++ b/Python/specialize.c @@ -118,6 +118,7 @@ _Py_GetSpecializationStats(void) { err += add_stat_dict(stats, LOAD_GLOBAL, "load_global"); err += add_stat_dict(stats, STORE_SUBSCR, "store_subscr"); err += add_stat_dict(stats, STORE_ATTR, "store_attr"); + err += add_stat_dict(stats, JUMP_BACKWARD, "jump_backward"); err += add_stat_dict(stats, CALL, "call"); err += add_stat_dict(stats, CALL_KW, "call_kw"); err += add_stat_dict(stats, BINARY_OP, "binary_op"); @@ -629,6 +630,7 @@ _PyCode_Quicken(_Py_CODEUNIT *instructions, Py_ssize_t size, int enable_counters #define SPEC_FAIL_CALL_INIT_NOT_PYTHON 21 #define SPEC_FAIL_CALL_PEP_523 22 #define SPEC_FAIL_CALL_BOUND_METHOD 23 +#define SPEC_FAIL_CALL_VECTORCALL 24 #define SPEC_FAIL_CALL_CLASS_MUTABLE 26 #define SPEC_FAIL_CALL_METHOD_WRAPPER 28 #define SPEC_FAIL_CALL_OPERATOR_WRAPPER 29 @@ -934,8 +936,7 @@ analyze_descriptor_load(PyTypeObject *type, PyObject *name, PyObject **descr, un PyObject *getattr = _PyType_Lookup(type, &_Py_ID(__getattr__)); has_getattr = getattr != NULL; if (has_custom_getattribute) { - if (getattro_slot == _Py_slot_tp_getattro && - !has_getattr && + if (!has_getattr && Py_IS_TYPE(getattribute, &PyFunction_Type)) { *descr = getattribute; *tp_version = ga_version; @@ -1258,12 +1259,6 @@ do_specialize_instance_load_attr(PyObject* owner, _Py_CODEUNIT* instr, PyObject* return -1; case GETATTRIBUTE_IS_PYTHON_FUNCTION: { - #ifndef Py_GIL_DISABLED - // In free-threaded builds it's possible for tp_getattro to change - // after the call to analyze_descriptor. That is fine: the version - // guard will fail. - assert(type->tp_getattro == _Py_slot_tp_getattro); - #endif assert(Py_IS_TYPE(descr, &PyFunction_Type)); _PyLoadMethodCache *lm_cache = (_PyLoadMethodCache *)(instr + 1); if (!function_check_args(descr, 2, LOAD_ATTR)) { @@ -2077,6 +2072,10 @@ specialize_py_call(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523); return -1; } + if (func->vectorcall != _PyFunction_Vectorcall) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_VECTORCALL); + return -1; + } int argcount = -1; if (kind == SPEC_FAIL_CODE_NOT_OPTIMIZED) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CODE_NOT_OPTIMIZED); @@ -2116,6 +2115,10 @@ specialize_py_call_kw(PyFunctionObject *func, _Py_CODEUNIT *instr, int nargs, SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_PEP_523); return -1; } + if (func->vectorcall != _PyFunction_Vectorcall) { + SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CALL_VECTORCALL); + return -1; + } if (kind == SPEC_FAIL_CODE_NOT_OPTIMIZED) { SPECIALIZATION_FAIL(CALL, SPEC_FAIL_CODE_NOT_OPTIMIZED); return -1; diff --git a/Python/stackrefs.c b/Python/stackrefs.c index 979a6b1c62820a2..69d4e8b943159f5 100644 --- a/Python/stackrefs.c +++ b/Python/stackrefs.c @@ -1,4 +1,3 @@ - #include "Python.h" #include "pycore_object.h" @@ -34,6 +33,7 @@ make_table_entry(PyObject *obj, const char *filename, int linenumber) result->filename = filename; result->linenumber = linenumber; result->filename_borrow = NULL; + result->linenumber_borrow = 0; return result; } diff --git a/Python/structmember.c b/Python/structmember.c index 574acf296157f3a..b6c9de85155a12d 100644 --- a/Python/structmember.c +++ b/Python/structmember.c @@ -161,19 +161,10 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) PyErr_SetString(PyExc_AttributeError, "readonly attribute"); return -1; } - if (v == NULL) { - if (l->type == Py_T_OBJECT_EX) { - /* Check if the attribute is set. */ - if (*(PyObject **)addr == NULL) { - PyErr_SetString(PyExc_AttributeError, l->name); - return -1; - } - } - else if (l->type != _Py_T_OBJECT) { - PyErr_SetString(PyExc_TypeError, - "can't delete numeric/char attribute"); - return -1; - } + if (v == NULL && l->type != Py_T_OBJECT_EX && l->type != _Py_T_OBJECT) { + PyErr_SetString(PyExc_TypeError, + "can't delete numeric/char attribute"); + return -1; } switch (l->type) { case Py_T_BOOL:{ @@ -324,6 +315,15 @@ PyMember_SetOne(char *addr, PyMemberDef *l, PyObject *v) oldv = *(PyObject **)addr; FT_ATOMIC_STORE_PTR_RELEASE(*(PyObject **)addr, Py_XNewRef(v)); Py_END_CRITICAL_SECTION(); + if (v == NULL && oldv == NULL && l->type == Py_T_OBJECT_EX) { + // Raise an exception when attempting to delete an already deleted + // attribute. + // Differently from Py_T_OBJECT_EX, _Py_T_OBJECT does not raise an + // exception here (PyMember_GetOne will return Py_None instead of + // NULL). + PyErr_SetString(PyExc_AttributeError, l->name); + return -1; + } Py_XDECREF(oldv); break; case Py_T_CHAR: { diff --git a/Python/symtable.c b/Python/symtable.c index f633e281019720f..6847f97cf476e97 100644 --- a/Python/symtable.c +++ b/Python/symtable.c @@ -806,6 +806,8 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, PyObject *k, *v; Py_ssize_t pos = 0; int remove_dunder_class = 0; + int remove_dunder_classdict = 0; + int remove_dunder_cond_annotations = 0; while (PyDict_Next(comp->ste_symbols, &pos, &k, &v)) { // skip comprehension parameter @@ -828,15 +830,32 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, if (existing == NULL && PyErr_Occurred()) { return 0; } - // __class__ is never allowed to be free through a class scope (see - // drop_class_free) + // __class__, __classdict__ and __conditional_annotations__ are + // not allowed to be free through a class scope (see + // drop_class_free) unless children scopes need it if (scope == FREE && ste->ste_type == ClassBlock && - _PyUnicode_EqualToASCIIString(k, "__class__")) { + (_PyUnicode_EqualToASCIIString(k, "__class__") || + _PyUnicode_EqualToASCIIString(k, "__classdict__") || + _PyUnicode_EqualToASCIIString(k, "__conditional_annotations__"))) { scope = GLOBAL_IMPLICIT; - if (PySet_Discard(comp_free, k) < 0) { + int child_needs_free = is_free_in_any_child(comp, k); + if (child_needs_free < 0) { return 0; } - remove_dunder_class = 1; + if (!child_needs_free) { + if (PySet_Discard(comp_free, k) < 0) { + return 0; + } + } + if (_PyUnicode_EqualToASCIIString(k, "__class__")) { + remove_dunder_class = 1; + } + else if (_PyUnicode_EqualToASCIIString(k, "__conditional_annotations__")) { + remove_dunder_cond_annotations = 1; + } + else { + remove_dunder_classdict = 1; + } } if (!existing) { // name does not exist in scope, copy from comprehension @@ -876,6 +895,12 @@ inline_comprehension(PySTEntryObject *ste, PySTEntryObject *comp, if (remove_dunder_class && PyDict_DelItemString(comp->ste_symbols, "__class__") < 0) { return 0; } + if (remove_dunder_classdict && PyDict_DelItemString(comp->ste_symbols, "__classdict__") < 0) { + return 0; + } + if (remove_dunder_cond_annotations && PyDict_DelItemString(comp->ste_symbols, "__conditional_annotations__") < 0) { + return 0; + } return 1; } @@ -2583,6 +2608,9 @@ symtable_visit_type_param_bound_or_default( PyObject *error_msg = PyUnicode_FromFormat("reserved name '%U' cannot be " "used for type parameter", name); + if (error_msg == NULL) { + return 0; + } PyErr_SetObject(PyExc_SyntaxError, error_msg); Py_DECREF(error_msg); SET_ERROR_LOCATION(st->st_filename, LOCATION(tp)); @@ -2780,6 +2808,7 @@ symtable_visit_annotation(struct symtable *st, expr_ty annotation, void *key) int future_annotations = st->st_future->ff_features & CO_FUTURE_ANNOTATIONS; if (current_type == ClassBlock && !future_annotations) { st->st_cur->ste_can_see_class_scope = 1; + parent_ste->ste_needs_classdict = 1; if (!symtable_add_def(st, &_Py_ID(__classdict__), USE, LOCATION(annotation))) { return 0; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 00dce4527fbb90d..eeb9ef2fbffd64a 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1183,9 +1183,10 @@ sys__settraceallthreads(PyObject *module, PyObject *arg) argument = arg; } - - PyEval_SetTraceAllThreads(func, argument); - + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyEval_SetTraceAllThreads(interp, func, argument) < 0) { + return NULL; + } Py_RETURN_NONE; } @@ -1263,8 +1264,10 @@ sys__setprofileallthreads(PyObject *module, PyObject *arg) argument = arg; } - PyEval_SetProfileAllThreads(func, argument); - + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (_PyEval_SetProfileAllThreads(interp, func, argument) < 0) { + return NULL; + } Py_RETURN_NONE; } @@ -1670,6 +1673,9 @@ _sys_getwindowsversion_from_kernel32(void) !GetFileVersionInfoW(kernel32_path, 0, verblock_size, verblock) || !VerQueryValueW(verblock, L"", (LPVOID)&ffi, &ffi_len)) { PyErr_SetFromWindowsErr(0); + if (verblock) { + PyMem_RawFree(verblock); + } return NULL; } @@ -1753,7 +1759,7 @@ sys_getwindowsversion_impl(PyObject *module) PyObject *realVersion = _sys_getwindowsversion_from_kernel32(); if (!realVersion) { if (!PyErr_ExceptionMatches(PyExc_WindowsError)) { - return NULL; + goto error; } PyErr_Clear(); @@ -1893,7 +1899,8 @@ sys_get_int_max_str_digits_impl(PyObject *module) /*[clinic end generated code: output=0042f5e8ae0e8631 input=61bf9f99bc8b112d]*/ { PyInterpreterState *interp = _PyInterpreterState_GET(); - return PyLong_FromLong(interp->long_state.max_str_digits); + int maxdigits = _Py_atomic_load_int(&interp->long_state.max_str_digits); + return PyLong_FromLong(maxdigits); } @@ -2308,12 +2315,13 @@ sys._stats_dump -> bool Dump stats to file, and clears the stats. -Return False if no statistics were not dumped because stats gathering was off. +Return False if no statistics were not dumped because stats gathering +was off. [clinic start generated code]*/ static int sys__stats_dump_impl(PyObject *module) -/*[clinic end generated code: output=6e346b4ba0de4489 input=31a489e39418b2a5]*/ +/*[clinic end generated code: output=6e346b4ba0de4489 input=7f3b7758cb59d2ff]*/ { int res = _Py_PrintSpecializationStats(1); _Py_StatsClear(); @@ -2367,14 +2375,14 @@ sys_activate_stack_trampoline_impl(PyObject *module, const char *backend) return NULL; } } - else if (strcmp(backend, "perf_jit") == 0) { - _PyPerf_Callbacks cur_cb; - _PyPerfTrampoline_GetCallbacks(&cur_cb); - if (cur_cb.write_state != _Py_perfmap_jit_callbacks.write_state) { - if (_PyPerfTrampoline_SetCallbacks(&_Py_perfmap_jit_callbacks) < 0 ) { - PyErr_SetString(PyExc_ValueError, "can't activate perf jit trampoline"); - return NULL; - } + } + else if (strcmp(backend, "perf_jit") == 0) { + _PyPerf_Callbacks cur_cb; + _PyPerfTrampoline_GetCallbacks(&cur_cb); + if (cur_cb.write_state != _Py_perfmap_jit_callbacks.write_state) { + if (_PyPerfTrampoline_SetCallbacks(&_Py_perfmap_jit_callbacks) < 0 ) { + PyErr_SetString(PyExc_ValueError, "can't activate perf jit trampoline"); + return NULL; } } } @@ -2448,26 +2456,63 @@ sys_is_remote_debug_enabled_impl(PyObject *module) #endif } +/*[clinic input] +sys.remote_exec + + pid: int + script: object + +Executes a file containing Python code in a given remote Python process. + +This function returns immediately, and the code will be executed by the +target process's main thread at the next available opportunity, +similarly to how signals are handled. There is no interface to +determine when the code has been executed. The caller is responsible +for making sure that the file still exists whenever the remote process +tries to read it and that it hasn't been overwritten. + +The remote process must be running a CPython interpreter of the same +major and minor version as the local process. If either the local or +remote interpreter is pre-release (alpha, beta, or release candidate) +then the local and remote interpreters must be the same exact version. + +Args: + pid (int): The process ID of the target Python process. + script (str|bytes): The path to a file containing + the Python code to be executed. +[clinic start generated code]*/ + static PyObject * -sys_remote_exec_unicode_path(PyObject *module, int pid, PyObject *script) +sys_remote_exec_impl(PyObject *module, int pid, PyObject *script) +/*[clinic end generated code: output=7d94c56afe4a52c0 input=7bd58f8da20cb74c]*/ { - const char *debugger_script_path = PyUnicode_AsUTF8(script); - if (debugger_script_path == NULL) { + PyObject *path; + const char *debugger_script_path; + + if (PyUnicode_FSConverter(script, &path) == 0) { return NULL; } + if (PySys_Audit("sys.remote_exec", "iO", pid, script) < 0) { + return NULL; + } + + debugger_script_path = PyBytes_AS_STRING(path); #ifdef MS_WINDOWS + PyObject *unicode_path; + if (PyUnicode_FSDecoder(path, &unicode_path) < 0) { + goto error; + } // Use UTF-16 (wide char) version of the path for permission checks - wchar_t *debugger_script_path_w = PyUnicode_AsWideCharString(script, NULL); + wchar_t *debugger_script_path_w = PyUnicode_AsWideCharString(unicode_path, NULL); + Py_DECREF(unicode_path); if (debugger_script_path_w == NULL) { - return NULL; + goto error; } - - // Check file attributes using wide character version (W) instead of ANSI (A) DWORD attr = GetFileAttributesW(debugger_script_path_w); - PyMem_Free(debugger_script_path_w); if (attr == INVALID_FILE_ATTRIBUTES) { DWORD err = GetLastError(); + PyMem_Free(debugger_script_path_w); if (err == ERROR_FILE_NOT_FOUND || err == ERROR_PATH_NOT_FOUND) { PyErr_SetString(PyExc_FileNotFoundError, "Script file does not exist"); } @@ -2475,11 +2520,12 @@ sys_remote_exec_unicode_path(PyObject *module, int pid, PyObject *script) PyErr_SetString(PyExc_PermissionError, "Script file cannot be read"); } else { - PyErr_SetFromWindowsErr(0); + PyErr_SetFromWindowsErr(err); } - return NULL; + goto error; } -#else + PyMem_Free(debugger_script_path_w); +#else // MS_WINDOWS if (access(debugger_script_path, F_OK | R_OK) != 0) { switch (errno) { case ENOENT: @@ -2491,54 +2537,19 @@ sys_remote_exec_unicode_path(PyObject *module, int pid, PyObject *script) default: PyErr_SetFromErrno(PyExc_OSError); } - return NULL; + goto error; } -#endif - +#endif // MS_WINDOWS if (_PySysRemoteDebug_SendExec(pid, 0, debugger_script_path) < 0) { - return NULL; + goto error; } + Py_DECREF(path); Py_RETURN_NONE; -} - -/*[clinic input] -sys.remote_exec - - pid: int - script: object - -Executes a file containing Python code in a given remote Python process. - -This function returns immediately, and the code will be executed by the -target process's main thread at the next available opportunity, similarly -to how signals are handled. There is no interface to determine when the -code has been executed. The caller is responsible for making sure that -the file still exists whenever the remote process tries to read it and that -it hasn't been overwritten. - -The remote process must be running a CPython interpreter of the same major -and minor version as the local process. If either the local or remote -interpreter is pre-release (alpha, beta, or release candidate) then the -local and remote interpreters must be the same exact version. - -Args: - pid (int): The process ID of the target Python process. - script (str|bytes): The path to a file containing - the Python code to be executed. -[clinic start generated code]*/ -static PyObject * -sys_remote_exec_impl(PyObject *module, int pid, PyObject *script) -/*[clinic end generated code: output=7d94c56afe4a52c0 input=39908ca2c5fe1eb0]*/ -{ - PyObject *ret = NULL; - PyObject *path; - if (PyUnicode_FSDecoder(script, &path)) { - ret = sys_remote_exec_unicode_path(module, pid, path); - Py_DECREF(path); - } - return ret; +error: + Py_DECREF(path); + return NULL; } @@ -2634,6 +2645,47 @@ sys__baserepl_impl(PyObject *module) Py_RETURN_NONE; } +/*[clinic input] +sys._clear_type_descriptors + + type: object(subclass_of='&PyType_Type') + / + +Private function for clearing certain descriptors from a type's dictionary. + +See gh-135228 for context. +[clinic start generated code]*/ + +static PyObject * +sys__clear_type_descriptors_impl(PyObject *module, PyObject *type) +/*[clinic end generated code: output=5ad17851b762b6d9 input=dc536c97fde07251]*/ +{ + PyTypeObject *typeobj = (PyTypeObject *)type; + if (_PyType_HasFeature(typeobj, Py_TPFLAGS_IMMUTABLETYPE)) { + PyErr_SetString(PyExc_TypeError, "argument is immutable"); + return NULL; + } + PyObject *dict = _PyType_GetDict(typeobj); + PyObject *dunder_dict = NULL; + if (PyDict_Pop(dict, &_Py_ID(__dict__), &dunder_dict) < 0) { + return NULL; + } + PyObject *dunder_weakref = NULL; + if (PyDict_Pop(dict, &_Py_ID(__weakref__), &dunder_weakref) < 0) { + PyType_Modified(typeobj); + Py_XDECREF(dunder_dict); + return NULL; + } + PyType_Modified(typeobj); + // We try to hold onto a reference to these until after we call + // PyType_Modified(), in case their deallocation triggers somer user code + // that tries to do something to the type. + Py_XDECREF(dunder_dict); + Py_XDECREF(dunder_weakref); + Py_RETURN_NONE; +} + + /*[clinic input] sys._is_gil_enabled -> bool @@ -2736,20 +2788,31 @@ PyAPI_FUNC(int) PyUnstable_CopyPerfMapFile(const char* parent_filename) { } char buf[4096]; PyThread_acquire_lock(perf_map_state.map_lock, 1); - int fflush_result = 0, result = 0; + int result = 0; while (1) { size_t bytes_read = fread(buf, 1, sizeof(buf), from); + if (bytes_read == 0) { + if (ferror(from)) { + result = -1; + } + break; + } + size_t bytes_written = fwrite(buf, 1, bytes_read, perf_map_state.perf_map); - fflush_result = fflush(perf_map_state.perf_map); - if (fflush_result != 0 || bytes_read == 0 || bytes_written < bytes_read) { + if (bytes_written < bytes_read) { result = -1; - goto close_and_release; + break; } + + if (fflush(perf_map_state.perf_map) != 0) { + result = -1; + break; + } + if (bytes_read < sizeof(buf) && feof(from)) { - goto close_and_release; + break; } } -close_and_release: fclose(from); PyThread_release_lock(perf_map_state.map_lock); return result; @@ -2830,6 +2893,7 @@ static PyMethodDef sys_methods[] = { SYS__STATS_DUMP_METHODDEF #endif SYS__GET_CPU_COUNT_CONFIG_METHODDEF + SYS__CLEAR_TYPE_DESCRIPTORS_METHODDEF SYS__IS_GIL_ENABLED_METHODDEF SYS__DUMP_TRACELETS_METHODDEF {NULL, NULL} // sentinel @@ -3371,14 +3435,39 @@ sys_set_flag(PyObject *flags, Py_ssize_t pos, PyObject *value) int _PySys_SetFlagObj(Py_ssize_t pos, PyObject *value) { - PyObject *flags = _PySys_GetRequiredAttrString("flags"); - if (flags == NULL) { - return -1; + PyObject *new_flags = NULL; + PyObject *flags_str = &_Py_ID(flags); // immortal ref + + PyObject *old_flags = _PySys_GetRequiredAttr(flags_str); + if (old_flags == NULL) { + goto error; } - sys_set_flag(flags, pos, value); - Py_DECREF(flags); - return 0; + new_flags = PyStructSequence_New(&FlagsType); + if (new_flags == NULL) { + goto error; + } + + for (Py_ssize_t i = 0; i < (Py_ssize_t)(Py_ARRAY_LENGTH(flags_fields) - 1); i++) { + if (i != pos) { + PyObject *old_value; + old_value = PyStructSequence_GET_ITEM(old_flags, i); // borrowed ref + sys_set_flag(new_flags, i, old_value); + } + else { + sys_set_flag(new_flags, pos, value); + } + } + + int res = _PySys_SetAttr(flags_str, new_flags); + Py_DECREF(old_flags); + Py_DECREF(new_flags); + return res; + +error: + Py_XDECREF(old_flags); + Py_XDECREF(new_flags); + return -1; } @@ -3402,8 +3491,6 @@ set_flags_from_config(PyInterpreterState *interp, PyObject *flags) const PyPreConfig *preconfig = &interp->runtime->preconfig; const PyConfig *config = _PyInterpreterState_GetConfig(interp); - // _PySys_UpdateConfig() modifies sys.flags in-place: - // Py_XDECREF() is needed in this case. Py_ssize_t pos = 0; #define SetFlagObj(expr) \ do { \ @@ -3600,6 +3687,18 @@ make_impl_info(PyObject *version_info) goto error; #endif + // PEP-734 +#if defined(__wasi__) || defined(__EMSCRIPTEN__) + // It is not enabled on WASM builds just yet + value = Py_False; +#else + value = Py_True; +#endif + res = PyDict_SetItemString(impl_info, "supports_isolated_interpreters", value); + if (res < 0) { + goto error; + } + /* dict ready */ ns = _PyNamespace_New(impl_info); @@ -3814,7 +3913,7 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) /* implementation */ SET_SYS("implementation", make_impl_info(version_info)); - // sys.flags: updated in-place later by _PySys_UpdateConfig() + // sys.flags: updated later by _PySys_UpdateConfig() ENSURE_INFO_TYPE(FlagsType, flags_desc); SET_SYS("flags", make_flags(tstate->interp)); @@ -3932,16 +4031,21 @@ _PySys_UpdateConfig(PyThreadState *tstate) #undef COPY_LIST #undef COPY_WSTR - // sys.flags - PyObject *flags = _PySys_GetRequiredAttrString("flags"); - if (flags == NULL) { + // replace sys.flags + PyObject *new_flags = PyStructSequence_New(&FlagsType); + if (new_flags == NULL) { return -1; } - if (set_flags_from_config(interp, flags) < 0) { - Py_DECREF(flags); + if (set_flags_from_config(interp, new_flags) < 0) { + Py_DECREF(new_flags); + return -1; + } + + res = _PySys_SetAttr(&_Py_ID(flags), new_flags); + Py_DECREF(new_flags); + if (res < 0) { return -1; } - Py_DECREF(flags); SET_SYS("dont_write_bytecode", PyBool_FromLong(!config->write_bytecode)); @@ -4442,7 +4546,7 @@ _PySys_SetIntMaxStrDigits(int maxdigits) // Set PyInterpreterState.long_state.max_str_digits // and PyInterpreterState.config.int_max_str_digits. PyInterpreterState *interp = _PyInterpreterState_GET(); - interp->long_state.max_str_digits = maxdigits; - interp->config.int_max_str_digits = maxdigits; + _Py_atomic_store_int(&interp->long_state.max_str_digits, maxdigits); + _Py_atomic_store_int(&interp->config.int_max_str_digits, maxdigits); return 0; } diff --git a/Python/thread_pthread.h b/Python/thread_pthread.h index da4058242448f3e..8e7b9589615b124 100644 --- a/Python/thread_pthread.h +++ b/Python/thread_pthread.h @@ -407,8 +407,8 @@ PyThread_get_thread_native_id(void) lwpid_t native_id; native_id = _lwp_self(); #elif defined(__DragonFly__) - lwpid_t native_id; - native_id = lwp_gettid(); + // lwp_gettid() is only unique within a process, so combine it with the pid. + unsigned long native_id = (unsigned long)getpid() << 32 | lwp_gettid(); #endif return (unsigned long) native_id; } diff --git a/Python/traceback.c b/Python/traceback.c index c06cb1a59089e29..c8c13d16d4c79c3 100644 --- a/Python/traceback.c +++ b/Python/traceback.c @@ -42,7 +42,7 @@ #if defined(__STDC_NO_VLA__) && (__STDC_NO_VLA__ == 1) /* Use alloca() for VLAs. */ -# define VLA(type, name, size) type *name = alloca(size) +# define VLA(type, name, size) type *name = alloca(sizeof(type) * (size)) #elif !defined(__STDC_NO_VLA__) || (__STDC_NO_VLA__ == 0) /* Use actual C VLAs.*/ # define VLA(type, name, size) type name[size] @@ -70,6 +70,13 @@ class traceback "PyTracebackObject *" "&PyTraceback_Type" #include "clinic/traceback.c.h" + +#ifdef MS_WINDOWS +typedef HRESULT (WINAPI *PF_GET_THREAD_DESCRIPTION)(HANDLE, PCWSTR*); +static PF_GET_THREAD_DESCRIPTION pGetThreadDescription = NULL; +#endif + + static PyObject * tb_create_raw(PyTracebackObject *next, PyFrameObject *frame, int lasti, int lineno) @@ -409,6 +416,9 @@ _Py_FindSourceFile(PyObject *filename, char* namebuf, size_t namelen, PyObject * npath = PyList_Size(syspath); open = PyObject_GetAttr(io, &_Py_ID(open)); + if (open == NULL) { + goto error; + } for (i = 0; i < npath; i++) { v = PyList_GetItem(syspath, i); if (v == NULL) { @@ -974,16 +984,72 @@ _Py_DumpASCII(int fd, PyObject *text) } } + +#ifdef MS_WINDOWS +static void +_Py_DumpWideString(int fd, wchar_t *str) +{ + Py_ssize_t size = wcslen(str); + int truncated; + if (MAX_STRING_LENGTH < size) { + size = MAX_STRING_LENGTH; + truncated = 1; + } + else { + truncated = 0; + } + + for (Py_ssize_t i=0; i < size; i++) { + Py_UCS4 ch = str[i]; + if (' ' <= ch && ch <= 126) { + /* printable ASCII character */ + dump_char(fd, (char)ch); + } + else if (ch <= 0xff) { + PUTS(fd, "\\x"); + _Py_DumpHexadecimal(fd, ch, 2); + } + else if (Py_UNICODE_IS_HIGH_SURROGATE(ch) + && Py_UNICODE_IS_LOW_SURROGATE(str[i+1])) { + ch = Py_UNICODE_JOIN_SURROGATES(ch, str[i+1]); + i++; // Skip the low surrogate character + PUTS(fd, "\\U"); + _Py_DumpHexadecimal(fd, ch, 8); + } + else { + Py_BUILD_ASSERT(sizeof(wchar_t) == 2); + PUTS(fd, "\\u"); + _Py_DumpHexadecimal(fd, ch, 4); + } + } + + if (truncated) { + PUTS(fd, "..."); + } +} +#endif + + /* Write a frame into the file fd: "File "xxx", line xxx in xxx". - This function is signal safe. */ + This function is signal safe. -static void + Return 0 on success. Return -1 if the frame is invalid. */ + +static int dump_frame(int fd, _PyInterpreterFrame *frame) { - assert(frame->owner < FRAME_OWNED_BY_INTERPRETER); + if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { + /* Ignore trampoline frame */ + return 0; + } + + PyCodeObject *code = _PyFrame_SafeGetCode(frame); + if (code == NULL) { + return -1; + } - PyCodeObject *code =_PyFrame_GetCode(frame); + int res = 0; PUTS(fd, " File "); if (code->co_filename != NULL && PyUnicode_Check(code->co_filename)) @@ -991,29 +1057,36 @@ dump_frame(int fd, _PyInterpreterFrame *frame) PUTS(fd, "\""); _Py_DumpASCII(fd, code->co_filename); PUTS(fd, "\""); - } else { + } + else { PUTS(fd, "???"); + res = -1; } - int lineno = PyUnstable_InterpreterFrame_GetLine(frame); PUTS(fd, ", line "); + int lasti = _PyFrame_SafeGetLasti(frame); + int lineno = -1; + if (lasti >= 0) { + lineno = _PyCode_SafeAddr2Line(code, lasti); + } if (lineno >= 0) { _Py_DumpDecimal(fd, (size_t)lineno); } else { PUTS(fd, "???"); + res = -1; } - PUTS(fd, " in "); - if (code->co_name != NULL - && PyUnicode_Check(code->co_name)) { + PUTS(fd, " in "); + if (code->co_name != NULL && PyUnicode_Check(code->co_name)) { _Py_DumpASCII(fd, code->co_name); } else { PUTS(fd, "???"); + res = -1; } - PUTS(fd, "\n"); + return res; } static int @@ -1025,6 +1098,9 @@ tstate_is_freed(PyThreadState *tstate) if (_PyMem_IsPtrFreed(tstate->interp)) { return 1; } + if (_PyMem_IsULongFreed(tstate->thread_id)) { + return 1; + } return 0; } @@ -1044,7 +1120,7 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) } if (tstate_is_freed(tstate)) { - PUTS(fd, " <tstate is freed>\n"); + PUTS(fd, " <freed thread state>\n"); return; } @@ -1056,17 +1132,6 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) unsigned int depth = 0; while (1) { - if (frame->owner == FRAME_OWNED_BY_INTERPRETER) { - /* Trampoline frame */ - frame = frame->previous; - if (frame == NULL) { - break; - } - - /* Can't have more than one shim frame in a row */ - assert(frame->owner != FRAME_OWNED_BY_INTERPRETER); - } - if (MAX_FRAME_DEPTH <= depth) { if (MAX_FRAME_DEPTH < depth) { PUTS(fd, "plus "); @@ -1076,8 +1141,20 @@ dump_traceback(int fd, PyThreadState *tstate, int write_header) break; } - dump_frame(fd, frame); - frame = frame->previous; + if (_PyMem_IsPtrFreed(frame)) { + PUTS(fd, " <freed frame>\n"); + break; + } + // Read frame->previous early since memory can be freed during + // dump_frame() + _PyInterpreterFrame *previous = frame->previous; + + if (dump_frame(fd, frame) < 0) { + PUTS(fd, " <invalid frame>\n"); + break; + } + + frame = previous; if (frame == NULL) { break; } @@ -1108,23 +1185,12 @@ _Py_DumpTraceback(int fd, PyThreadState *tstate) # endif #endif -/* Write the thread identifier into the file 'fd': "Current thread 0xHHHH:\" if - is_current is true, "Thread 0xHHHH:\n" otherwise. - - This function is signal safe. */ +// Write the thread name static void -write_thread_id(int fd, PyThreadState *tstate, int is_current) +write_thread_name(int fd, PyThreadState *tstate) { - if (is_current) - PUTS(fd, "Current thread 0x"); - else - PUTS(fd, "Thread 0x"); - _Py_DumpHexadecimal(fd, - tstate->thread_id, - sizeof(unsigned long) * 2); - - // Write the thread name +#ifndef MS_WINDOWS #if defined(HAVE_PTHREAD_GETNAME_NP) || defined(HAVE_PTHREAD_GET_NAME_NP) char name[100]; pthread_t thread = (pthread_t)tstate->thread_id; @@ -1143,6 +1209,51 @@ write_thread_id(int fd, PyThreadState *tstate, int is_current) } } #endif +#else + // Windows implementation + if (pGetThreadDescription == NULL) { + return; + } + + HANDLE thread = OpenThread(THREAD_QUERY_LIMITED_INFORMATION, FALSE, tstate->thread_id); + if (thread == NULL) { + return; + } + + wchar_t *name; + HRESULT hr = pGetThreadDescription(thread, &name); + if (!FAILED(hr)) { + if (name[0] != 0) { + PUTS(fd, " ["); + _Py_DumpWideString(fd, name); + PUTS(fd, "]"); + } + LocalFree(name); + } + CloseHandle(thread); +#endif +} + + +/* Write the thread identifier into the file 'fd': "Current thread 0xHHHH:\" if + is_current is true, "Thread 0xHHHH:\n" otherwise. + + This function is signal safe (except on Windows). */ + +static void +write_thread_id(int fd, PyThreadState *tstate, int is_current) +{ + if (is_current) + PUTS(fd, "Current thread 0x"); + else + PUTS(fd, "Thread 0x"); + _Py_DumpHexadecimal(fd, + tstate->thread_id, + sizeof(unsigned long) * 2); + + if (!_PyMem_IsULongFreed(tstate->thread_id)) { + write_thread_name(fd, tstate); + } PUTS(fd, " (most recent call first):\n"); } @@ -1200,7 +1311,6 @@ _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, return "unable to get the thread head state"; /* Dump the traceback of each thread */ - tstate = PyInterpreterState_ThreadHead(interp); unsigned int nthreads = 0; _Py_BEGIN_SUPPRESS_IPH do @@ -1211,11 +1321,18 @@ _Py_DumpTracebackThreads(int fd, PyInterpreterState *interp, PUTS(fd, "...\n"); break; } + + if (tstate_is_freed(tstate)) { + PUTS(fd, "<freed thread state>\n"); + break; + } + write_thread_id(fd, tstate, tstate == current_tstate); if (tstate == current_tstate && tstate->interp->gc.collecting) { PUTS(fd, " Garbage-collecting\n"); } dump_traceback(fd, tstate, 0); + tstate = PyThreadState_Next(tstate); nthreads++; } while (tstate != NULL); @@ -1327,3 +1444,30 @@ _Py_DumpStack(int fd) PUTS(fd, " <cannot get C stack on this system>\n"); } #endif + +void +_Py_InitDumpStack(void) +{ +#ifdef CAN_C_BACKTRACE + // gh-137185: Call backtrace() once to force libgcc to be loaded early. + void *callstack[1]; + (void)backtrace(callstack, 1); +#endif +} + + +void +_Py_DumpTraceback_Init(void) +{ +#ifdef MS_WINDOWS + if (pGetThreadDescription != NULL) { + return; + } + + HMODULE kernelbase = GetModuleHandleW(L"kernelbase.dll"); + if (kernelbase != NULL) { + pGetThreadDescription = (PF_GET_THREAD_DESCRIPTION)GetProcAddress( + kernelbase, "GetThreadDescription"); + } +#endif +} diff --git a/Python/tracemalloc.c b/Python/tracemalloc.c index 7066a214f1065ba..4795068d0a06188 100644 --- a/Python/tracemalloc.c +++ b/Python/tracemalloc.c @@ -224,13 +224,20 @@ tracemalloc_get_frame(_PyInterpreterFrame *pyframe, frame_t *frame) assert(PyStackRef_CodeCheck(pyframe->f_executable)); frame->filename = &_Py_STR(anon_unknown); - int lineno = PyUnstable_InterpreterFrame_GetLine(pyframe); + int lineno = -1; + PyCodeObject *code = _PyFrame_GetCode(pyframe); + // PyUnstable_InterpreterFrame_GetLine() cannot but used, since it uses + // a critical section which can trigger a deadlock. + int lasti = _PyFrame_SafeGetLasti(pyframe); + if (lasti >= 0) { + lineno = _PyCode_SafeAddr2Line(code, lasti); + } if (lineno < 0) { lineno = 0; } frame->lineno = (unsigned int)lineno; - PyObject *filename = filename = _PyFrame_GetCode(pyframe)->co_filename; + PyObject *filename = code->co_filename; if (filename == NULL) { #ifdef TRACE_DEBUG tracemalloc_error("failed to get the filename of the code object"); @@ -840,7 +847,7 @@ _PyTraceMalloc_Start(int max_nframe) /* everything is ready: start tracing Python memory allocations */ TABLES_LOCK(); - tracemalloc_config.tracing = 1; + _Py_atomic_store_int_relaxed(&tracemalloc_config.tracing, 1); TABLES_UNLOCK(); return 0; @@ -853,11 +860,12 @@ _PyTraceMalloc_Stop(void) TABLES_LOCK(); if (!tracemalloc_config.tracing) { - goto done; + TABLES_UNLOCK(); + return; } /* stop tracing Python memory allocations */ - tracemalloc_config.tracing = 0; + _Py_atomic_store_int_relaxed(&tracemalloc_config.tracing, 0); /* unregister the hook on memory allocators */ PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); @@ -870,10 +878,12 @@ _PyTraceMalloc_Stop(void) raw_free(tracemalloc_traceback); tracemalloc_traceback = NULL; - (void)PyRefTracer_SetTracer(NULL, NULL); - -done: TABLES_UNLOCK(); + + // Call it after TABLES_UNLOCK() since it calls _PyEval_StopTheWorldAll() + // which would lead to a deadlock with TABLES_LOCK() which doesn't detach + // the thread state. + (void)PyRefTracer_SetTracer(NULL, NULL); } @@ -1197,6 +1207,10 @@ int PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, size_t size) { + if (_Py_atomic_load_int_relaxed(&tracemalloc_config.tracing) == 0) { + /* tracemalloc is not tracing: do nothing */ + return -2; + } PyGILState_STATE gil_state = PyGILState_Ensure(); TABLES_LOCK(); @@ -1218,6 +1232,11 @@ PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) { + if (_Py_atomic_load_int_relaxed(&tracemalloc_config.tracing) == 0) { + /* tracemalloc is not tracing: do nothing */ + return -2; + } + TABLES_LOCK(); int result; diff --git a/README.rst b/README.rst index 897d8995b63d9b7..e72349798dab0d8 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -This is Python version 3.14.0 beta 1 -==================================== +This is Python version 3.14.6 +============================= .. image:: https://github.com/python/cpython/actions/workflows/build.yml/badge.svg?branch=main&event=push :alt: CPython build status on GitHub Actions @@ -153,7 +153,7 @@ Documentation updated daily. It can also be downloaded in many formats for faster access. The documentation -is downloadable in HTML, PDF, and reStructuredText formats; the latter version +is downloadable in HTML, EPUB, and reStructuredText formats; the latter version is primarily for documentation authors, translators, and people with special formatting requirements. @@ -232,4 +232,4 @@ This Python distribution contains *no* GNU General Public License (GPL) code, so it may be used in proprietary projects. There are interfaces to some GNU code but these are entirely optional. -All trademarks referenced herein are property of their respective holders. \ No newline at end of file +All trademarks referenced herein are property of their respective holders. diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index fa7689d45dbcb7d..996f725fdcb9b57 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -1,7 +1,7 @@ extend = "../../.ruff.toml" # Inherit the project-wide settings [per-file-target-version] -"deepfreeze.py" = "py310" +"deepfreeze.py" = "py311" # requires `code.co_exceptiontable` "stable_abi.py" = "py311" # requires 'tomllib' [format] @@ -29,7 +29,6 @@ ignore = [ "F541", # f-string without any placeholders "PYI024", # Use `typing.NamedTuple` instead of `collections.namedtuple` "PYI025", # Use `from collections.abc import Set as AbstractSet` - "UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)` ] [lint.per-file-ignores] diff --git a/Tools/build/.warningignore_macos b/Tools/build/.warningignore_macos index d7b62bc6a43ba65..67f50119db73100 100644 --- a/Tools/build/.warningignore_macos +++ b/Tools/build/.warningignore_macos @@ -3,7 +3,3 @@ # Keep lines sorted lexicographically to help avoid merge conflicts. # Format example: # /path/to/file (number of warnings in file) -Modules/expat/siphash.h 7 -Modules/expat/xmlparse.c 13 -Modules/expat/xmltok.c 3 -Modules/expat/xmltok_impl.c 26 diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index 9815bcfe27d9953..d6b0db0684cdaee 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -17,9 +17,11 @@ See --help for more information """ + +from __future__ import annotations + import _imp import argparse -import collections import enum import logging import os @@ -29,13 +31,16 @@ import sysconfig import warnings from collections.abc import Iterable -from importlib._bootstrap import _load as bootstrap_load +from importlib._bootstrap import ( # type: ignore[attr-defined] + _load as bootstrap_load, +) from importlib.machinery import ( BuiltinImporter, ExtensionFileLoader, ModuleSpec, ) from importlib.util import spec_from_file_location, spec_from_loader +from typing import NamedTuple SRC_DIR = pathlib.Path(__file__).parent.parent.parent @@ -112,6 +117,7 @@ ) +@enum.unique class ModuleState(enum.Enum): # Makefile state "yes" BUILTIN = "builtin" @@ -123,11 +129,13 @@ class ModuleState(enum.Enum): # disabled by Setup / makesetup rule DISABLED_SETUP = "disabled_setup" - def __bool__(self): + def __bool__(self) -> bool: return self.value in {"builtin", "shared"} -ModuleInfo = collections.namedtuple("ModuleInfo", "name state") +class ModuleInfo(NamedTuple): + name: str + state: ModuleState class ModuleChecker: @@ -135,9 +143,9 @@ class ModuleChecker: setup_files = ( # see end of configure.ac - "Modules/Setup.local", - "Modules/Setup.stdlib", - "Modules/Setup.bootstrap", + pathlib.Path("Modules/Setup.local"), + pathlib.Path("Modules/Setup.stdlib"), + pathlib.Path("Modules/Setup.bootstrap"), SRC_DIR / "Modules/Setup", ) @@ -149,15 +157,15 @@ def __init__(self, cross_compiling: bool = False, strict: bool = False): self.builddir = self.get_builddir() self.modules = self.get_modules() - self.builtin_ok = [] - self.shared_ok = [] - self.failed_on_import = [] - self.missing = [] - self.disabled_configure = [] - self.disabled_setup = [] - self.notavailable = [] + self.builtin_ok: list[ModuleInfo] = [] + self.shared_ok: list[ModuleInfo] = [] + self.failed_on_import: list[ModuleInfo] = [] + self.missing: list[ModuleInfo] = [] + self.disabled_configure: list[ModuleInfo] = [] + self.disabled_setup: list[ModuleInfo] = [] + self.notavailable: list[ModuleInfo] = [] - def check(self): + def check(self) -> None: if not hasattr(_imp, 'create_dynamic'): logger.warning( ('Dynamic extensions not supported ' @@ -189,10 +197,10 @@ def check(self): assert modinfo.state == ModuleState.SHARED self.shared_ok.append(modinfo) - def summary(self, *, verbose: bool = False): + def summary(self, *, verbose: bool = False) -> None: longest = max([len(e.name) for e in self.modules], default=0) - def print_three_column(modinfos: list[ModuleInfo]): + def print_three_column(modinfos: list[ModuleInfo]) -> None: names = [modinfo.name for modinfo in modinfos] names.sort(key=str.lower) # guarantee zip() doesn't drop anything @@ -262,12 +270,12 @@ def print_three_column(modinfos: list[ModuleInfo]): f"{len(self.failed_on_import)} failed on import)" ) - def check_strict_build(self): + def check_strict_build(self) -> None: """Fail if modules are missing and it's a strict build""" if self.strict_extensions_build and (self.failed_on_import or self.missing): raise RuntimeError("Failed to build some stdlib modules") - def list_module_names(self, *, all: bool = False) -> set: + def list_module_names(self, *, all: bool = False) -> set[str]: names = {modinfo.name for modinfo in self.modules} if all: names.update(WINDOWS_MODULES) @@ -280,9 +288,9 @@ def get_builddir(self) -> pathlib.Path: except FileNotFoundError: logger.error("%s must be run from the top build directory", __file__) raise - builddir = pathlib.Path(builddir) - logger.debug("%s: %s", self.pybuilddir_txt, builddir) - return builddir + builddir_path = pathlib.Path(builddir) + logger.debug("%s: %s", self.pybuilddir_txt, builddir_path) + return builddir_path def get_modules(self) -> list[ModuleInfo]: """Get module info from sysconfig and Modules/Setup* files""" @@ -367,7 +375,7 @@ def parse_setup_file(self, setup_file: pathlib.Path) -> Iterable[ModuleInfo]: case ["*disabled*"]: state = ModuleState.DISABLED case ["*noconfig*"]: - state = None + continue case [*items]: if state == ModuleState.DISABLED: # *disabled* can disable multiple modules per line @@ -384,26 +392,33 @@ def parse_setup_file(self, setup_file: pathlib.Path) -> Iterable[ModuleInfo]: def get_spec(self, modinfo: ModuleInfo) -> ModuleSpec: """Get ModuleSpec for builtin or extension module""" if modinfo.state == ModuleState.SHARED: - location = os.fspath(self.get_location(modinfo)) + mod_location = self.get_location(modinfo) + assert mod_location is not None + location = os.fspath(mod_location) loader = ExtensionFileLoader(modinfo.name, location) - return spec_from_file_location(modinfo.name, location, loader=loader) + spec = spec_from_file_location(modinfo.name, location, loader=loader) + assert spec is not None + return spec elif modinfo.state == ModuleState.BUILTIN: - return spec_from_loader(modinfo.name, loader=BuiltinImporter) + spec = spec_from_loader(modinfo.name, loader=BuiltinImporter) + assert spec is not None + return spec else: raise ValueError(modinfo) - def get_location(self, modinfo: ModuleInfo) -> pathlib.Path: + def get_location(self, modinfo: ModuleInfo) -> pathlib.Path | None: """Get shared library location in build directory""" if modinfo.state == ModuleState.SHARED: return self.builddir / f"{modinfo.name}{self.ext_suffix}" else: return None - def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec): + def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec) -> None: """Check that the module file is present and not empty""" - if spec.loader is BuiltinImporter: - return + if spec.loader is BuiltinImporter: # type: ignore[comparison-overlap] + return # type: ignore[unreachable] try: + assert spec.origin is not None st = os.stat(spec.origin) except FileNotFoundError: logger.error("%s (%s) is missing", modinfo.name, spec.origin) @@ -411,7 +426,7 @@ def _check_file(self, modinfo: ModuleInfo, spec: ModuleSpec): if not st.st_size: raise ImportError(f"{spec.origin} is an empty file") - def check_module_import(self, modinfo: ModuleInfo): + def check_module_import(self, modinfo: ModuleInfo) -> None: """Attempt to import module and report errors""" spec = self.get_spec(modinfo) self._check_file(modinfo, spec) @@ -430,7 +445,7 @@ def check_module_import(self, modinfo: ModuleInfo): logger.exception("Importing extension '%s' failed!", modinfo.name) raise - def check_module_cross(self, modinfo: ModuleInfo): + def check_module_cross(self, modinfo: ModuleInfo) -> None: """Sanity check for cross compiling""" spec = self.get_spec(modinfo) self._check_file(modinfo, spec) @@ -443,6 +458,7 @@ def rename_module(self, modinfo: ModuleInfo) -> None: failed_name = f"{modinfo.name}_failed{self.ext_suffix}" builddir_path = self.get_location(modinfo) + assert builddir_path is not None if builddir_path.is_symlink(): symlink = builddir_path module_path = builddir_path.resolve().relative_to(os.getcwd()) @@ -466,7 +482,7 @@ def rename_module(self, modinfo: ModuleInfo) -> None: logger.debug("Rename '%s' -> '%s'", module_path, failed_path) -def main(): +def main() -> None: args = parser.parse_args() if args.debug: args.verbose = True diff --git a/Tools/build/check_warnings.py b/Tools/build/check_warnings.py index 3f49d8e7f2ee484..44ccf9708ad72f0 100644 --- a/Tools/build/check_warnings.py +++ b/Tools/build/check_warnings.py @@ -8,21 +8,29 @@ import sys from collections import defaultdict from pathlib import Path -from typing import NamedTuple +from typing import NamedTuple, TypedDict class IgnoreRule(NamedTuple): file_path: str - count: int + count: int # type: ignore[assignment] ignore_all: bool = False is_directory: bool = False +class CompileWarning(TypedDict): + file: str + line: str + column: str + message: str + option: str + + def parse_warning_ignore_file(file_path: str) -> set[IgnoreRule]: """ Parses the warning ignore file and returns a set of IgnoreRules """ - files_with_expected_warnings = set() + files_with_expected_warnings: set[IgnoreRule] = set() with Path(file_path).open(encoding="UTF-8") as ignore_rules_file: files_with_expected_warnings = set() for i, line in enumerate(ignore_rules_file): @@ -46,7 +54,7 @@ def parse_warning_ignore_file(file_path: str) -> set[IgnoreRule]: ) sys.exit(1) if ignore_all: - count = 0 + count = "0" files_with_expected_warnings.add( IgnoreRule( @@ -61,7 +69,7 @@ def extract_warnings_from_compiler_output( compiler_output: str, compiler_output_type: str, path_prefix: str = "", -) -> list[dict]: +) -> list[CompileWarning]: """ Extracts warnings from the compiler output based on compiler output type. Removes path prefix from file paths if provided. @@ -78,8 +86,12 @@ def extract_warnings_from_compiler_output( r"(?P<file>.*):(?P<line>\d+):(?P<column>\d+): warning: " r"(?P<message>.*) (?P<option>\[-[^\]]+\])$" ) + else: + raise RuntimeError( + f"Unsupported compiler output type: {compiler_output_type}", + ) compiled_regex = re.compile(regex_pattern) - compiler_warnings = [] + compiler_warnings: list[CompileWarning] = [] for i, line in enumerate(compiler_output.splitlines(), start=1): if match := compiled_regex.match(line): try: @@ -100,7 +112,9 @@ def extract_warnings_from_compiler_output( return compiler_warnings -def get_warnings_by_file(warnings: list[dict]) -> dict[str, list[dict]]: +def get_warnings_by_file( + warnings: list[CompileWarning], +) -> dict[str, list[CompileWarning]]: """ Returns a dictionary where the key is the file and the data is the warnings in that file. Does not include duplicate warnings for a @@ -138,7 +152,7 @@ def is_file_ignored( def get_unexpected_warnings( ignore_rules: set[IgnoreRule], - files_with_warnings: set[IgnoreRule], + files_with_warnings: dict[str, list[CompileWarning]], ) -> int: """ Returns failure status if warnings discovered in list of warnings @@ -180,7 +194,7 @@ def get_unexpected_warnings( def get_unexpected_improvements( ignore_rules: set[IgnoreRule], - files_with_warnings: set[IgnoreRule], + files_with_warnings: dict[str, list[CompileWarning]], ) -> int: """ Returns failure status if the number of warnings for a file is greater diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index cfdd55fd1925fd6..8839ee42b3714a9 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -11,7 +11,7 @@ import os import subprocess -from dataclasses import dataclass +from dataclasses import dataclass, fields from pathlib import Path TYPE_CHECKING = False @@ -19,14 +19,16 @@ from collections.abc import Set GITHUB_DEFAULT_BRANCH = os.environ["GITHUB_DEFAULT_BRANCH"] -GITHUB_CODEOWNERS_PATH = Path(".github/CODEOWNERS") GITHUB_WORKFLOWS_PATH = Path(".github/workflows") -CONFIGURATION_FILE_NAMES = frozenset({ - ".pre-commit-config.yaml", - ".ruff.toml", - "mypy.ini", +RUN_TESTS_IGNORE = frozenset({ + Path("Tools/check-c-api-docs/ignored_c_api.txt"), + Path(".github/CODEOWNERS"), + Path(".pre-commit-config.yaml"), + Path(".ruff.toml"), + Path("mypy.ini"), }) + UNIX_BUILD_SYSTEM_FILE_NAMES = frozenset({ Path("aclocal.m4"), Path("config.guess"), @@ -45,25 +47,97 @@ SUFFIXES_C_OR_CPP = frozenset({".c", ".h", ".cpp"}) SUFFIXES_DOCUMENTATION = frozenset({".rst", ".md"}) +ANDROID_DIRS = frozenset({"Android"}) +EMSCRIPTEN_DIRS = frozenset({Path("Platforms", "emscripten")}) +IOS_DIRS = frozenset({"Apple", "iOS"}) +MACOS_DIRS = frozenset({"Mac"}) +WASI_DIRS = frozenset({Path("Tools", "wasm")}) + +LIBRARY_FUZZER_PATHS = frozenset({ + # All C/CPP fuzzers. + Path("configure"), + Path(".github/workflows/reusable-cifuzz.yml"), + # ast + Path("Lib/ast.py"), + Path("Python/ast.c"), + # configparser + Path("Lib/configparser.py"), + # csv + Path("Lib/csv.py"), + Path("Modules/_csv.c"), + # decode + Path("Lib/encodings/"), + Path("Modules/_codecsmodule.c"), + Path("Modules/cjkcodecs/"), + Path("Modules/unicodedata*"), + # difflib + Path("Lib/difflib.py"), + # email + Path("Lib/email/"), + # html + Path("Lib/html/"), + Path("Lib/_markupbase.py"), + # http.client + Path("Lib/http/client.py"), + # json + Path("Lib/json/"), + Path("Modules/_json.c"), + # plist + Path("Lib/plistlib.py"), + # re + Path("Lib/re/"), + Path("Modules/_sre/"), + # tarfile + Path("Lib/tarfile.py"), + # tomllib + Path("Modules/tomllib/"), + # xml + Path("Lib/xml/"), + Path("Lib/_markupbase.py"), + Path("Modules/expat/"), + Path("Modules/pyexpat.c"), + # zipfile + Path("Lib/zipfile/"), + # zoneinfo + Path("Lib/zoneinfo/"), + Path("Modules/_zoneinfo.c"), +}) + @dataclass(kw_only=True, slots=True) class Outputs: + run_android: bool = False run_ci_fuzz: bool = False + run_ci_fuzz_stdlib: bool = False run_docs: bool = False + run_emscripten: bool = False + run_ios: bool = False + run_macos: bool = False run_tests: bool = False + run_ubuntu: bool = False + run_wasi: bool = False run_windows_msi: bool = False run_windows_tests: bool = False def compute_changes() -> None: target_branch, head_ref = git_refs() - if target_branch and head_ref: + if os.environ.get("GITHUB_EVENT_NAME", "") == "pull_request": # Getting changed files only makes sense on a pull request files = get_changed_files(target_branch, head_ref) outputs = process_changed_files(files) else: # Otherwise, just run the tests - outputs = Outputs(run_tests=True, run_windows_tests=True) + outputs = Outputs( + run_android=True, + run_emscripten=True, + run_ios=True, + run_macos=True, + run_tests=True, + run_ubuntu=True, + run_wasi=True, + run_windows_tests=True, + ) outputs = process_target_branch(outputs, target_branch) if outputs.run_tests: @@ -76,6 +150,11 @@ def compute_changes() -> None: else: print("Branch too old for CIFuzz tests; or no C files were changed") + if outputs.run_ci_fuzz_stdlib: + print("Run CIFuzz tests for stdlib") + else: + print("Branch too old for CIFuzz tests; or no stdlib files were changed") + if outputs.run_docs: print("Build documentation") @@ -111,35 +190,85 @@ def get_changed_files( return frozenset(map(Path, filter(None, map(str.strip, changed_files)))) +def get_file_platform(file: Path) -> str | None: + if not file.parts: + return None + first_part = file.parts[0] + if first_part in MACOS_DIRS: + return "macos" + if first_part in IOS_DIRS: + return "ios" + if first_part in ANDROID_DIRS: + return "android" + if len(file.parts) >= 2 and Path(*file.parts[:2]) in EMSCRIPTEN_DIRS: + return "emscripten" + if len(file.parts) >= 2 and Path(*file.parts[:2]) in WASI_DIRS: # Tools/wasm/ + return "wasi" + return None + + +def is_fuzzable_library_file(file: Path) -> bool: + return any( + (file.is_relative_to(needs_fuzz) and needs_fuzz.is_dir()) + or (file == needs_fuzz and file.is_file()) + for needs_fuzz in LIBRARY_FUZZER_PATHS + ) + + def process_changed_files(changed_files: Set[Path]) -> Outputs: run_tests = False run_ci_fuzz = False + run_ci_fuzz_stdlib = False run_docs = False run_windows_tests = False run_windows_msi = False + platforms_changed = set() + has_platform_specific_change = True + for file in changed_files: # Documentation files doc_or_misc = file.parts[0] in {"Doc", "Misc"} doc_file = file.suffix in SUFFIXES_DOCUMENTATION or doc_or_misc if file.parent == GITHUB_WORKFLOWS_PATH: - if file.name == "build.yml": - run_tests = run_ci_fuzz = True + if file.name in ("build.yml", "reusable-cifuzz.yml"): + run_tests = run_ci_fuzz = run_ci_fuzz_stdlib = run_windows_tests = True + has_platform_specific_change = False + continue if file.name == "reusable-docs.yml": run_docs = True + continue + if file.name == "reusable-windows.yml": + run_tests = True + run_windows_tests = True + continue if file.name == "reusable-windows-msi.yml": run_windows_msi = True - - if not ( - doc_file - or file == GITHUB_CODEOWNERS_PATH - or file.name in CONFIGURATION_FILE_NAMES - ): + continue + if file.name == "reusable-macos.yml": + run_tests = True + platforms_changed.add("macos") + continue + if file.name == "reusable-emscripten.yml": + run_tests = True + platforms_changed.add("emscripten") + continue + if file.name == "reusable-wasi.yml": + run_tests = True + platforms_changed.add("wasi") + continue + + if not doc_file and file not in RUN_TESTS_IGNORE: run_tests = True - if file not in UNIX_BUILD_SYSTEM_FILE_NAMES: - run_windows_tests = True + platform = get_file_platform(file) + if platform is not None: + platforms_changed.add(platform) + else: + has_platform_specific_change = False + if file not in UNIX_BUILD_SYSTEM_FILE_NAMES: + run_windows_tests = True # The fuzz tests are pretty slow so they are executed only for PRs # changing relevant files. @@ -150,6 +279,8 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: ("Modules", "_xxtestfuzz"), }: run_ci_fuzz = True + if not run_ci_fuzz_stdlib and is_fuzzable_library_file(file): + run_ci_fuzz_stdlib = True # Check for changed documentation-related files if doc_file: @@ -159,12 +290,43 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs: if file.parts[:2] == ("Tools", "msi"): run_windows_msi = True + # Check which platform specific tests to run + if run_tests: + if not has_platform_specific_change or not platforms_changed: + run_android = True + run_emscripten = True + run_ios = True + run_macos = True + run_ubuntu = True + run_wasi = True + else: + run_android = "android" in platforms_changed + run_emscripten = "emscripten" in platforms_changed + run_ios = "ios" in platforms_changed + run_macos = "macos" in platforms_changed + run_ubuntu = False + run_wasi = "wasi" in platforms_changed + else: + run_android = False + run_emscripten = False + run_ios = False + run_macos = False + run_ubuntu = False + run_wasi = False + return Outputs( + run_android=run_android, run_ci_fuzz=run_ci_fuzz, + run_ci_fuzz_stdlib=run_ci_fuzz_stdlib, run_docs=run_docs, + run_emscripten=run_emscripten, + run_ios=run_ios, + run_macos=run_macos, run_tests=run_tests, - run_windows_tests=run_windows_tests, + run_ubuntu=run_ubuntu, + run_wasi=run_wasi, run_windows_msi=run_windows_msi, + run_windows_tests=run_windows_tests, ) @@ -191,11 +353,10 @@ def write_github_output(outputs: Outputs) -> None: return with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f: - f.write(f"run-ci-fuzz={bool_lower(outputs.run_ci_fuzz)}\n") - f.write(f"run-docs={bool_lower(outputs.run_docs)}\n") - f.write(f"run-tests={bool_lower(outputs.run_tests)}\n") - f.write(f"run-windows-tests={bool_lower(outputs.run_windows_tests)}\n") - f.write(f"run-windows-msi={bool_lower(outputs.run_windows_msi)}\n") + for field in fields(outputs): + name = field.name.replace("_", "-") + val = bool_lower(getattr(outputs, field.name)) + f.write(f"{name}={val}\n") def bool_lower(value: bool, /) -> str: diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index 23f584479379765..2b9f03aebb6d7e8 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -2,9 +2,12 @@ The script may be executed by _bootstrap_python interpreter. Shared library extension modules are not available in that case. -On Windows, and in cross-compilation cases, it is executed -by Python 3.10, and 3.11 features are not available. +Requires 3.11+ to be executed, +because relies on `code.co_qualname` and `code.co_exceptiontable`. """ + +from __future__ import annotations + import argparse import builtins import collections @@ -13,10 +16,14 @@ import re import time import types -from typing import TextIO import umarshal +TYPE_CHECKING = False +if TYPE_CHECKING: + from collections.abc import Iterator + from typing import Any, TextIO + ROOT = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) verbose = False @@ -45,8 +52,8 @@ def make_string_literal(b: bytes) -> str: next_code_version = 1 -def get_localsplus(code: types.CodeType): - a = collections.defaultdict(int) +def get_localsplus(code: types.CodeType) -> tuple[tuple[str, ...], bytes]: + a: collections.defaultdict[str, int] = collections.defaultdict(int) for name in code.co_varnames: a[name] |= CO_FAST_LOCAL for name in code.co_cellvars: @@ -136,7 +143,7 @@ def get_identifiers_and_strings(self) -> tuple[set[str], dict[str, str]]: return identifiers, strings @contextlib.contextmanager - def indent(self) -> None: + def indent(self) -> Iterator[None]: save_level = self.level try: self.level += 1 @@ -148,7 +155,7 @@ def write(self, arg: str) -> None: self.file.writelines((" "*self.level, arg, "\n")) @contextlib.contextmanager - def block(self, prefix: str, suffix: str = "") -> None: + def block(self, prefix: str, suffix: str = "") -> Iterator[None]: self.write(prefix + " {") with self.indent(): yield @@ -250,9 +257,17 @@ def generate_code(self, name: str, code: types.CodeType) -> str: co_names = self.generate(name + "_names", code.co_names) co_filename = self.generate(name + "_filename", code.co_filename) co_name = self.generate(name + "_name", code.co_name) - co_qualname = self.generate(name + "_qualname", code.co_qualname) co_linetable = self.generate(name + "_linetable", code.co_linetable) - co_exceptiontable = self.generate(name + "_exceptiontable", code.co_exceptiontable) + # We use 3.10 for type checking, but this module requires 3.11 + # TODO: bump python version for this script. + co_qualname = self.generate( + name + "_qualname", + code.co_qualname, # type: ignore[attr-defined] + ) + co_exceptiontable = self.generate( + name + "_exceptiontable", + code.co_exceptiontable, # type: ignore[attr-defined] + ) # These fields are not directly accessible localsplusnames, localspluskinds = get_localsplus(code) co_localsplusnames = self.generate(name + "_localsplusnames", localsplusnames) @@ -379,13 +394,13 @@ def generate_complex(self, name: str, z: complex) -> str: self.write(f".cval = {{ {z.real}, {z.imag} }},") return f"&{name}.ob_base" - def generate_frozenset(self, name: str, fs: frozenset[object]) -> str: + def generate_frozenset(self, name: str, fs: frozenset[Any]) -> str: try: - fs = sorted(fs) + fs_sorted = sorted(fs) except TypeError: # frozen set with incompatible types, fallback to repr() - fs = sorted(fs, key=repr) - ret = self.generate_tuple(name, tuple(fs)) + fs_sorted = sorted(fs, key=repr) + ret = self.generate_tuple(name, tuple(fs_sorted)) self.write("// TODO: The above tuple should be a frozenset") return ret @@ -402,7 +417,7 @@ def generate(self, name: str, obj: object) -> str: # print(f"Cache hit {key!r:.40}: {self.cache[key]!r:.40}") return self.cache[key] self.misses += 1 - if isinstance(obj, (types.CodeType, umarshal.Code)) : + if isinstance(obj, types.CodeType) : val = self.generate_code(name, obj) elif isinstance(obj, tuple): val = self.generate_tuple(name, obj) @@ -458,7 +473,7 @@ def decode_frozen_data(source: str) -> types.CodeType: if re.match(FROZEN_DATA_LINE, line): values.extend([int(x) for x in line.split(",") if x.strip()]) data = bytes(values) - return umarshal.loads(data) + return umarshal.loads(data) # type: ignore[no-any-return] def generate(args: list[str], output: TextIO) -> None: @@ -494,12 +509,12 @@ def generate(args: list[str], output: TextIO) -> None: help="Input file and module name (required) in file:modname format") @contextlib.contextmanager -def report_time(label: str): - t0 = time.time() +def report_time(label: str) -> Iterator[None]: + t0 = time.perf_counter() try: yield finally: - t1 = time.time() + t1 = time.perf_counter() if verbose: print(f"{label}: {t1-t0:.3f} sec") diff --git a/Tools/build/generate-build-details.py b/Tools/build/generate-build-details.py index 0da6c2948d66881..ed9ab2844d250aa 100644 --- a/Tools/build/generate-build-details.py +++ b/Tools/build/generate-build-details.py @@ -3,6 +3,8 @@ # Script initially imported from: # https://github.com/FFY00/python-instrospection/blob/main/python_introspection/scripts/generate-build-details.py +from __future__ import annotations + import argparse import collections import importlib.machinery @@ -11,19 +13,23 @@ import sys import sysconfig +TYPE_CHECKING = False +if TYPE_CHECKING: + from typing import Any + -def version_info_to_dict(obj): # (object) -> dict[str, Any] +def version_info_to_dict(obj: sys._version_info) -> dict[str, Any]: field_names = ('major', 'minor', 'micro', 'releaselevel', 'serial') return {field: getattr(obj, field) for field in field_names} -def get_dict_key(container, key): # (dict[str, Any], str) -> dict[str, Any] +def get_dict_key(container: dict[str, Any], key: str) -> dict[str, Any]: for part in key.split('.'): container = container[part] return container -def generate_data(schema_version): +def generate_data(schema_version: str) -> collections.defaultdict[str, Any]: """Generate the build-details.json data (PEP 739). :param schema_version: The schema version of the data we want to generate. @@ -32,7 +38,9 @@ def generate_data(schema_version): if schema_version != '1.0': raise ValueError(f'Unsupported schema_version: {schema_version}') - data = collections.defaultdict(lambda: collections.defaultdict(dict)) + data: collections.defaultdict[str, Any] = collections.defaultdict( + lambda: collections.defaultdict(dict), + ) data['schema_version'] = schema_version @@ -47,7 +55,7 @@ def generate_data(schema_version): data['language']['version'] = sysconfig.get_python_version() data['language']['version_info'] = version_info_to_dict(sys.version_info) - data['implementation'] = vars(sys.implementation) + data['implementation'] = vars(sys.implementation).copy() data['implementation']['version'] = version_info_to_dict(sys.implementation.version) # Fix cross-compilation if '_multiarch' in data['implementation']: @@ -67,7 +75,7 @@ def generate_data(schema_version): PY3LIBRARY = sysconfig.get_config_var('PY3LIBRARY') LIBPYTHON = sysconfig.get_config_var('LIBPYTHON') LIBPC = sysconfig.get_config_var('LIBPC') - INCLUDEDIR = sysconfig.get_config_var('INCLUDEDIR') + INCLUDEPY = sysconfig.get_config_var('INCLUDEPY') if os.name == 'posix': # On POSIX, LIBRARY is always the static library, while LDLIBRARY is the @@ -96,7 +104,7 @@ def generate_data(schema_version): data['abi']['extension_suffix'] = sysconfig.get_config_var('EXT_SUFFIX') # EXTENSION_SUFFIXES has been constant for a long time, and currently we - # don't have a better information source to find the stable ABI suffix. + # don't have a better information source to find the stable ABI suffix. for suffix in importlib.machinery.EXTENSION_SUFFIXES: if suffix.startswith('.abi'): data['abi']['stable_abi_suffix'] = suffix @@ -115,44 +123,62 @@ def generate_data(schema_version): if has_static_library: data['libpython']['static'] = os.path.join(LIBDIR, LIBRARY) - data['c_api']['include'] = INCLUDEDIR + data['c_api']['headers'] = INCLUDEPY if LIBPC: data['c_api']['pkgconfig_path'] = LIBPC return data -def make_paths_relative(data, config_path=None): # (dict[str, Any], str | None) -> None +def make_paths_relative(data: dict[str, Any], config_path: str | None = None) -> None: # Make base_prefix relative to the config_path directory if config_path: - data['base_prefix'] = os.path.relpath(data['base_prefix'], os.path.dirname(config_path)) + data['base_prefix'] = relative_path(data['base_prefix'], + os.path.dirname(config_path)) + base_prefix = data['base_prefix'] + # Update path values to make them relative to base_prefix - PATH_KEYS = [ + PATH_KEYS = ( 'base_interpreter', 'libpython.dynamic', 'libpython.dynamic_stableabi', 'libpython.static', 'c_api.headers', 'c_api.pkgconfig_path', - ] + ) for entry in PATH_KEYS: - parent, _, child = entry.rpartition('.') + *parents, child = entry.split('.') # Get the key container object try: container = data - for part in parent.split('.'): + for part in parents: container = container[part] + if child not in container: + raise KeyError(child) current_path = container[child] except KeyError: continue # Get the relative path - new_path = os.path.relpath(current_path, data['base_prefix']) + new_path = relative_path(current_path, base_prefix) # Join '.' so that the path is formated as './path' instead of 'path' new_path = os.path.join('.', new_path) container[child] = new_path -def main(): # () -> None +def relative_path(path: str, base: str) -> str: + if os.name != 'nt': + return os.path.relpath(path, base) + + # There are no relative paths between drives on Windows. + path_drv, _ = os.path.splitdrive(path) + base_drv, _ = os.path.splitdrive(base) + if path_drv.lower() == base_drv.lower(): + return os.path.relpath(path, base) + + return path + + +def main() -> None: parser = argparse.ArgumentParser(exit_on_error=False) parser.add_argument('location') parser.add_argument( @@ -178,8 +204,9 @@ def main(): # () -> None make_paths_relative(data, args.config_file_path) json_output = json.dumps(data, indent=2) - with open(args.location, 'w') as f: - print(json_output, file=f) + with open(args.location, 'w', encoding='utf-8') as f: + f.write(json_output) + f.write('\n') if __name__ == '__main__': diff --git a/Tools/build/generate_levenshtein_examples.py b/Tools/build/generate_levenshtein_examples.py index 30dcc7cf1a14794..2396c8040ca5396 100644 --- a/Tools/build/generate_levenshtein_examples.py +++ b/Tools/build/generate_levenshtein_examples.py @@ -13,7 +13,7 @@ _CASE_COST = 1 -def _substitution_cost(ch_a, ch_b): +def _substitution_cost(ch_a: str, ch_b: str) -> int: if ch_a == ch_b: return 0 if ch_a.lower() == ch_b.lower(): @@ -22,7 +22,7 @@ def _substitution_cost(ch_a, ch_b): @lru_cache(None) -def levenshtein(a, b): +def levenshtein(a: str, b: str) -> int: if not a or not b: return (len(a) + len(b)) * _MOVE_COST option1 = levenshtein(a[:-1], b[:-1]) + _substitution_cost(a[-1], b[-1]) @@ -31,7 +31,7 @@ def levenshtein(a, b): return min(option1, option2, option3) -def main(): +def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument('output_path', metavar='FILE', type=str) parser.add_argument('--overwrite', dest='overwrite', action='store_const', @@ -48,7 +48,7 @@ def main(): ) return - examples = set() + examples: set[tuple[str, str, int]] = set() # Create a lot of non-empty examples, which should end up with a Gauss-like # distribution for even costs (moves) and odd costs (case substitutions). while len(examples) < 9990: diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py index db01426e9722c3d..f5a2f26cbb085cd 100644 --- a/Tools/build/generate_sbom.py +++ b/Tools/build/generate_sbom.py @@ -4,10 +4,13 @@ import hashlib import json import os +import random import re import subprocess import sys +import time import typing +import urllib.error import urllib.request from pathlib import Path, PurePosixPath, PureWindowsPath @@ -161,6 +164,23 @@ def get_externals() -> list[str]: return externals +def download_with_retries(download_location: str, + max_retries: int = 7, + base_delay: float = 2.25, + max_jitter: float = 1.0) -> typing.Any: + """Download a file with exponential backoff retry.""" + for attempt in range(max_retries + 1): + try: + resp = urllib.request.urlopen(download_location) + except (urllib.error.URLError, ConnectionError) as ex: + if attempt == max_retries: + msg = f"Download from {download_location} failed." + raise OSError(msg) from ex + time.sleep(base_delay**attempt + random.uniform(0, max_jitter)) + else: + return resp + + def check_sbom_packages(sbom_data: dict[str, typing.Any]) -> None: """Make a bunch of assertions about the SBOM package data to ensure it's consistent.""" @@ -175,7 +195,7 @@ def check_sbom_packages(sbom_data: dict[str, typing.Any]) -> None: # and that the download URL is valid. if "checksums" not in package or "CI" in os.environ: download_location = package["downloadLocation"] - resp = urllib.request.urlopen(download_location) + resp = download_with_retries(download_location) error_if(resp.status != 200, f"Couldn't access URL: {download_location}'") package["checksums"] = [{ @@ -222,14 +242,14 @@ def check_sbom_packages(sbom_data: dict[str, typing.Any]) -> None: ) # libexpat specifies its expected rev in a refresh script. - if package["name"] == "libexpat": + if package["name"] == "expat": libexpat_refresh_sh = (CPYTHON_ROOT_DIR / "Modules/expat/refresh.sh").read_text() libexpat_expected_version_match = re.search( r"expected_libexpat_version=\"([0-9]+\.[0-9]+\.[0-9]+)\"", libexpat_refresh_sh ) libexpat_expected_sha256_match = re.search( - r"expected_libexpat_sha256=\"[a-f0-9]{40}\"", + r"expected_libexpat_sha256=\"([a-f0-9]{64})\"", libexpat_refresh_sh ) libexpat_expected_version = libexpat_expected_version_match and libexpat_expected_version_match.group(1) diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index 88414cdbb37a8d0..bda725396406118 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -1,9 +1,12 @@ # This script lists the names of standard library modules # to update Python/stdlib_module_names.h +from __future__ import annotations + import _imp import os.path import sys import sysconfig +from typing import TextIO from check_extension_modules import ModuleChecker @@ -48,12 +51,12 @@ } # Built-in modules -def list_builtin_modules(names): +def list_builtin_modules(names: set[str]) -> None: names |= set(sys.builtin_module_names) # Pure Python modules (Lib/*.py) -def list_python_modules(names): +def list_python_modules(names: set[str]) -> None: for filename in os.listdir(STDLIB_PATH): if not filename.endswith(".py"): continue @@ -62,7 +65,7 @@ def list_python_modules(names): # Packages in Lib/ -def list_packages(names): +def list_packages(names: set[str]) -> None: for name in os.listdir(STDLIB_PATH): if name in IGNORE: continue @@ -76,16 +79,16 @@ def list_packages(names): # Built-in and extension modules built by Modules/Setup* # includes Windows and macOS extensions. -def list_modules_setup_extensions(names): +def list_modules_setup_extensions(names: set[str]) -> None: checker = ModuleChecker() names.update(checker.list_module_names(all=True)) # List frozen modules of the PyImport_FrozenModules list (Python/frozen.c). # Use the "./Programs/_testembed list_frozen" command. -def list_frozen(names): +def list_frozen(names: set[str]) -> None: submodules = set() - for name in _imp._frozen_module_names(): + for name in _imp._frozen_module_names(): # type: ignore[attr-defined] # To skip __hello__, __hello_alias__ and etc. if name.startswith('__'): continue @@ -101,8 +104,8 @@ def list_frozen(names): raise Exception(f'unexpected frozen submodules: {sorted(submodules)}') -def list_modules(): - names = set() +def list_modules() -> set[str]: + names: set[str] = set() list_builtin_modules(names) list_modules_setup_extensions(names) @@ -127,7 +130,7 @@ def list_modules(): return names -def write_modules(fp, names): +def write_modules(fp: TextIO, names: set[str]) -> None: print(f"// Auto-generated by {SCRIPT_NAME}.", file=fp) print("// List used to create sys.stdlib_module_names.", file=fp) @@ -138,7 +141,7 @@ def write_modules(fp, names): print("};", file=fp) -def main(): +def main() -> None: if not sysconfig.is_python_build(): print(f"ERROR: {sys.executable} is not a Python build", file=sys.stderr) diff --git a/Tools/build/mypy.ini b/Tools/build/mypy.ini index 06224163884a984..d1f193e25601c5b 100644 --- a/Tools/build/mypy.ini +++ b/Tools/build/mypy.ini @@ -1,7 +1,20 @@ [mypy] + +# Please, when adding new files here, also add them to: +# .github/workflows/mypy.yml files = + Tools/build/check_extension_modules.py, + Tools/build/check_warnings.py, Tools/build/compute-changes.py, - Tools/build/generate_sbom.py + Tools/build/deepfreeze.py, + Tools/build/generate-build-details.py, + Tools/build/generate_levenshtein_examples.py, + Tools/build/generate_sbom.py, + Tools/build/generate_stdlib_module_names.py, + Tools/build/verify_ensurepip_wheels.py, + Tools/build/update_file.py, + Tools/build/umarshal.py + pretty = True # Make sure Python can still be built diff --git a/Tools/build/stable_abi.py b/Tools/build/stable_abi.py index 1ddd76cdd9bf64d..39115b331ba6428 100644 --- a/Tools/build/stable_abi.py +++ b/Tools/build/stable_abi.py @@ -232,7 +232,7 @@ def sort_key(item): 'data': 'data', 'struct': 'type', 'macro': 'macro', - # 'const': 'const', # all undocumented + 'const': 'macro', 'typedef': 'type', } diff --git a/Tools/build/umarshal.py b/Tools/build/umarshal.py index 679fa7caf9f931f..865cffc24401226 100644 --- a/Tools/build/umarshal.py +++ b/Tools/build/umarshal.py @@ -145,12 +145,12 @@ def r_PyLong(self) -> int: def r_float_bin(self) -> float: buf = self.r_string(8) import struct # Lazy import to avoid breaking UNIX build - return struct.unpack("d", buf)[0] + return struct.unpack("d", buf)[0] # type: ignore[no-any-return] def r_float_str(self) -> float: n = self.r_byte() buf = self.r_string(n) - return ast.literal_eval(buf.decode("ascii")) + return ast.literal_eval(buf.decode("ascii")) # type: ignore[no-any-return] def r_ref_reserve(self, flag: int) -> int: if flag: @@ -306,7 +306,7 @@ def loads(data: bytes) -> Any: return r.r_object() -def main(): +def main() -> None: # Test import marshal import pprint @@ -314,8 +314,9 @@ def main(): data = marshal.dumps(sample) retval = loads(data) assert retval == sample, retval - sample = main.__code__ - data = marshal.dumps(sample) + + sample2 = main.__code__ + data = marshal.dumps(sample2) retval = loads(data) assert isinstance(retval, Code), retval pprint.pprint(retval.__dict__) diff --git a/Tools/build/update_file.py b/Tools/build/update_file.py index b4182c1d0cb6388..b4a5fb6e778ae8c 100644 --- a/Tools/build/update_file.py +++ b/Tools/build/update_file.py @@ -6,14 +6,27 @@ actually change the in-tree generated code. """ +from __future__ import annotations + import contextlib import os import os.path import sys +TYPE_CHECKING = False +if TYPE_CHECKING: + import typing + from collections.abc import Iterator + from io import TextIOWrapper + + _Outcome: typing.TypeAlias = typing.Literal['created', 'updated', 'same'] + @contextlib.contextmanager -def updating_file_with_tmpfile(filename, tmpfile=None): +def updating_file_with_tmpfile( + filename: str, + tmpfile: str | None = None, +) -> Iterator[tuple[TextIOWrapper, TextIOWrapper]]: """A context manager for updating a file via a temp file. The context manager provides two open files: the source file open @@ -46,13 +59,18 @@ def updating_file_with_tmpfile(filename, tmpfile=None): update_file_with_tmpfile(filename, tmpfile) -def update_file_with_tmpfile(filename, tmpfile, *, create=False): +def update_file_with_tmpfile( + filename: str, + tmpfile: str, + *, + create: bool = False, +) -> _Outcome: try: targetfile = open(filename, 'rb') except FileNotFoundError: if not create: raise # re-raise - outcome = 'created' + outcome: _Outcome = 'created' os.replace(tmpfile, filename) else: with targetfile: diff --git a/Tools/build/verify_ensurepip_wheels.py b/Tools/build/verify_ensurepip_wheels.py index a37da2f70757e58..46c42916d9354da 100755 --- a/Tools/build/verify_ensurepip_wheels.py +++ b/Tools/build/verify_ensurepip_wheels.py @@ -20,13 +20,13 @@ GITHUB_ACTIONS = os.getenv("GITHUB_ACTIONS") == "true" -def print_notice(file_path: str, message: str) -> None: +def print_notice(file_path: str | Path, message: str) -> None: if GITHUB_ACTIONS: message = f"::notice file={file_path}::{message}" print(message, end="\n\n") -def print_error(file_path: str, message: str) -> None: +def print_error(file_path: str | Path, message: str) -> None: if GITHUB_ACTIONS: message = f"::error file={file_path}::{message}" print(message, end="\n\n") @@ -67,6 +67,7 @@ def verify_wheel(package_name: str) -> bool: return False release_files = json.loads(raw_text)["releases"][package_version] + expected_digest = "" for release_info in release_files: if package_path.name != release_info["filename"]: continue @@ -95,6 +96,7 @@ def verify_wheel(package_name: str) -> bool: return True + if __name__ == "__main__": exit_status = int(not verify_wheel("pip")) raise SystemExit(exit_status) diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 037fe11ea223c79..f5dcd5c76c55f24 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -1,5 +1,4 @@ import os.path -import re from c_parser.preprocessor import ( get_preprocessor as _get_preprocessor, @@ -18,16 +17,16 @@ def _abs(relfile): return os.path.join(REPO_ROOT, relfile) -def clean_lines(text): - """Clear out comments, blank lines, and leading/trailing whitespace.""" - lines = (line.strip() for line in text.splitlines()) - lines = (line.partition('#')[0].rstrip() - for line in lines - if line and not line.startswith('#')) - glob_all = f'{GLOB_ALL} ' - lines = (re.sub(r'^[*] ', glob_all, line) for line in lines) - lines = (_abs(line) for line in lines) - return list(lines) +def format_conf_lines(lines): + """Format conf entries for use in a TSV table.""" + return [_abs(entry) for entry in lines] + + +def format_tsv_lines(lines): + """Format entries for use in a TSV table.""" + lines = ((_abs(first), *rest) for first, *rest in lines) + lines = map('\t'.join, lines) + return list(map(str.rstrip, lines)) ''' @@ -47,252 +46,241 @@ def clean_lines(text): ''' # XXX Handle these. -# Tab separated: -EXCLUDED = clean_lines(''' -# @begin=conf@ - -# OSX -Modules/_scproxy.c # SystemConfiguration/SystemConfiguration.h - -# Windows -Modules/_winapi.c # windows.h -Modules/expat/winconfig.h -Modules/overlapped.c # winsock.h -Python/dynload_win.c # windows.h -Python/thread_nt.h - -# other OS-dependent -Python/dynload_aix.c # sys/ldr.h -Python/dynload_dl.c # dl.h -Python/dynload_hpux.c # dl.h -Python/emscripten_signal.c -Python/thread_pthread.h -Python/thread_pthread_stubs.h - -# only huge constants (safe but parsing is slow) -Modules/_ssl_data_*.h -Modules/cjkcodecs/mappings_*.h -Modules/unicodedata_db.h -Modules/unicodename_db.h -Objects/unicodetype_db.h - -# generated -Python/deepfreeze/*.c -Python/frozen_modules/*.h -Python/generated_cases.c.h -Python/executor_cases.c.h -Python/optimizer_cases.c.h - -# not actually source -Python/bytecodes.c -Python/optimizer_bytecodes.c - -# mimalloc -Objects/mimalloc/*.c -Include/internal/mimalloc/*.h -Include/internal/mimalloc/mimalloc/*.h - -# @end=conf@ -''') +EXCLUDED = format_conf_lines([ + # macOS + 'Modules/_scproxy.c', # SystemConfiguration/SystemConfiguration.h + + # Windows + 'Modules/_winapi.c', # windows.h + 'Modules/expat/winconfig.h', + 'Modules/overlapped.c', # winsock.h + 'Python/dynload_win.c', # windows.h + 'Python/thread_nt.h', + + # other OS-dependent + 'Python/dynload_aix.c', # sys/ldr.h + 'Python/dynload_dl.c', # dl.h + 'Python/dynload_hpux.c', # dl.h + 'Python/emscripten_signal.c', + 'Python/emscripten_syscalls.c', + 'Python/emscripten_trampoline_inner.c', + 'Python/thread_pthread.h', + 'Python/thread_pthread_stubs.h', + + # only huge constants (safe but parsing is slow) + 'Modules/_ssl_data_*.h', + 'Modules/cjkcodecs/mappings_*.h', + 'Modules/unicodedata_db.h', + 'Modules/unicodename_db.h', + 'Objects/unicodetype_db.h', + + # generated + 'Python/deepfreeze/*.c', + 'Python/frozen_modules/*.h', + 'Python/generated_cases.c.h', + 'Python/executor_cases.c.h', + 'Python/optimizer_cases.c.h', + # XXX: Throws errors if PY_VERSION_HEX is not mocked out + 'Modules/clinic/_testclinic_depr.c.h', + + # not actually source + 'Python/bytecodes.c', + 'Python/optimizer_bytecodes.c', + + # mimalloc + 'Objects/mimalloc/*.c', + 'Include/internal/mimalloc/*.h', + 'Include/internal/mimalloc/mimalloc/*.h', +]) # XXX Fix the parser. -EXCLUDED += clean_lines(''' -# The tool should be able to parse these... - -# The problem with xmlparse.c is that something -# has gone wrong where # we handle "maybe inline actual" -# in Tools/c-analyzer/c_parser/parser/_global.py. -Modules/expat/internal.h -Modules/expat/xmlparse.c -''') - -INCL_DIRS = clean_lines(''' -# @begin=tsv@ - -glob dirname -* . -* ./Include -* ./Include/internal -* ./Include/internal/mimalloc - -Modules/_decimal/**/*.c Modules/_decimal/libmpdec -Modules/_elementtree.c Modules/expat -Modules/_hacl/*.c Modules/_hacl/include -Modules/_hacl/*.c Modules/_hacl/ -Modules/_hacl/*.h Modules/_hacl/include -Modules/_hacl/*.h Modules/_hacl/ -Modules/md5module.c Modules/_hacl/include -Modules/sha1module.c Modules/_hacl/include -Modules/sha2module.c Modules/_hacl/include -Modules/sha3module.c Modules/_hacl/include -Modules/blake2module.c Modules/_hacl/include -Modules/hmacmodule.c Modules/_hacl/include -Objects/stringlib/*.h Objects - -# possible system-installed headers, just in case -Modules/_tkinter.c /usr/include/tcl8.6 -Modules/_uuidmodule.c /usr/include/uuid -Modules/tkappinit.c /usr/include/tcl - -# @end=tsv@ -''')[1:] - -INCLUDES = clean_lines(''' -# @begin=tsv@ - -glob include - -**/*.h Python.h -Include/**/*.h object.h - -# for Py_HAVE_CONDVAR -Include/internal/pycore_gil.h pycore_condvar.h -Python/thread_pthread.h pycore_condvar.h - -# other - -Objects/stringlib/join.h stringlib/stringdefs.h -Objects/stringlib/ctype.h stringlib/stringdefs.h -Objects/stringlib/transmogrify.h stringlib/stringdefs.h -#Objects/stringlib/fastsearch.h stringlib/stringdefs.h -#Objects/stringlib/count.h stringlib/stringdefs.h -#Objects/stringlib/find.h stringlib/stringdefs.h -#Objects/stringlib/partition.h stringlib/stringdefs.h -#Objects/stringlib/split.h stringlib/stringdefs.h -Objects/stringlib/fastsearch.h stringlib/ucs1lib.h -Objects/stringlib/count.h stringlib/ucs1lib.h -Objects/stringlib/find.h stringlib/ucs1lib.h -Objects/stringlib/partition.h stringlib/ucs1lib.h -Objects/stringlib/split.h stringlib/ucs1lib.h -Objects/stringlib/find_max_char.h Objects/stringlib/ucs1lib.h -Objects/stringlib/count.h Objects/stringlib/fastsearch.h -Objects/stringlib/find.h Objects/stringlib/fastsearch.h -Objects/stringlib/partition.h Objects/stringlib/fastsearch.h -Objects/stringlib/replace.h Objects/stringlib/fastsearch.h -Objects/stringlib/repr.h Objects/stringlib/fastsearch.h -Objects/stringlib/split.h Objects/stringlib/fastsearch.h - -# @end=tsv@ -''')[1:] - -MACROS = clean_lines(''' -# @begin=tsv@ - -glob name value - -Include/internal/*.h Py_BUILD_CORE 1 -Python/**/*.c Py_BUILD_CORE 1 -Python/**/*.h Py_BUILD_CORE 1 -Parser/**/*.c Py_BUILD_CORE 1 -Parser/**/*.h Py_BUILD_CORE 1 -Objects/**/*.c Py_BUILD_CORE 1 -Objects/**/*.h Py_BUILD_CORE 1 - -Modules/_asynciomodule.c Py_BUILD_CORE 1 -Modules/_codecsmodule.c Py_BUILD_CORE 1 -Modules/_collectionsmodule.c Py_BUILD_CORE 1 -Modules/_ctypes/_ctypes.c Py_BUILD_CORE 1 -Modules/_ctypes/cfield.c Py_BUILD_CORE 1 -Modules/_cursesmodule.c Py_BUILD_CORE 1 -Modules/_datetimemodule.c Py_BUILD_CORE 1 -Modules/_functoolsmodule.c Py_BUILD_CORE 1 -Modules/_heapqmodule.c Py_BUILD_CORE 1 -Modules/_io/*.c Py_BUILD_CORE 1 -Modules/_io/*.h Py_BUILD_CORE 1 -Modules/_localemodule.c Py_BUILD_CORE 1 -Modules/_operator.c Py_BUILD_CORE 1 -Modules/_posixsubprocess.c Py_BUILD_CORE 1 -Modules/_sre/sre.c Py_BUILD_CORE 1 -Modules/_threadmodule.c Py_BUILD_CORE 1 -Modules/_tracemalloc.c Py_BUILD_CORE 1 -Modules/_weakref.c Py_BUILD_CORE 1 -Modules/_zoneinfo.c Py_BUILD_CORE 1 -Modules/atexitmodule.c Py_BUILD_CORE 1 -Modules/cmathmodule.c Py_BUILD_CORE 1 -Modules/faulthandler.c Py_BUILD_CORE 1 -Modules/gcmodule.c Py_BUILD_CORE 1 -Modules/getpath.c Py_BUILD_CORE 1 -Modules/getpath_noop.c Py_BUILD_CORE 1 -Modules/itertoolsmodule.c Py_BUILD_CORE 1 -Modules/main.c Py_BUILD_CORE 1 -Modules/mathmodule.c Py_BUILD_CORE 1 -Modules/posixmodule.c Py_BUILD_CORE 1 -Modules/sha256module.c Py_BUILD_CORE 1 -Modules/sha512module.c Py_BUILD_CORE 1 -Modules/signalmodule.c Py_BUILD_CORE 1 -Modules/symtablemodule.c Py_BUILD_CORE 1 -Modules/timemodule.c Py_BUILD_CORE 1 -Modules/unicodedata.c Py_BUILD_CORE 1 - -Modules/_json.c Py_BUILD_CORE_BUILTIN 1 -Modules/_pickle.c Py_BUILD_CORE_BUILTIN 1 -Modules/_testinternalcapi.c Py_BUILD_CORE_BUILTIN 1 - -Include/cpython/abstract.h Py_CPYTHON_ABSTRACTOBJECT_H 1 -Include/cpython/bytearrayobject.h Py_CPYTHON_BYTEARRAYOBJECT_H 1 -Include/cpython/bytesobject.h Py_CPYTHON_BYTESOBJECT_H 1 -Include/cpython/ceval.h Py_CPYTHON_CEVAL_H 1 -Include/cpython/code.h Py_CPYTHON_CODE_H 1 -Include/cpython/dictobject.h Py_CPYTHON_DICTOBJECT_H 1 -Include/cpython/fileobject.h Py_CPYTHON_FILEOBJECT_H 1 -Include/cpython/fileutils.h Py_CPYTHON_FILEUTILS_H 1 -Include/cpython/frameobject.h Py_CPYTHON_FRAMEOBJECT_H 1 -Include/cpython/import.h Py_CPYTHON_IMPORT_H 1 -Include/cpython/interpreteridobject.h Py_CPYTHON_INTERPRETERIDOBJECT_H 1 -Include/cpython/listobject.h Py_CPYTHON_LISTOBJECT_H 1 -Include/cpython/methodobject.h Py_CPYTHON_METHODOBJECT_H 1 -Include/cpython/object.h Py_CPYTHON_OBJECT_H 1 -Include/cpython/objimpl.h Py_CPYTHON_OBJIMPL_H 1 -Include/cpython/pyerrors.h Py_CPYTHON_ERRORS_H 1 -Include/cpython/pylifecycle.h Py_CPYTHON_PYLIFECYCLE_H 1 -Include/cpython/pymem.h Py_CPYTHON_PYMEM_H 1 -Include/cpython/pystate.h Py_CPYTHON_PYSTATE_H 1 -Include/cpython/sysmodule.h Py_CPYTHON_SYSMODULE_H 1 -Include/cpython/traceback.h Py_CPYTHON_TRACEBACK_H 1 -Include/cpython/tupleobject.h Py_CPYTHON_TUPLEOBJECT_H 1 -Include/cpython/unicodeobject.h Py_CPYTHON_UNICODEOBJECT_H 1 - -# implied include of <unistd.h> -Include/**/*.h _POSIX_THREADS 1 -Include/**/*.h HAVE_PTHREAD_H 1 - -# from pyconfig.h -Include/cpython/pthread_stubs.h HAVE_PTHREAD_STUBS 1 -Python/thread_pthread_stubs.h HAVE_PTHREAD_STUBS 1 - -# from Objects/bytesobject.c -Objects/stringlib/partition.h STRINGLIB_GET_EMPTY() bytes_get_empty() -Objects/stringlib/join.h STRINGLIB_MUTABLE 0 -Objects/stringlib/partition.h STRINGLIB_MUTABLE 0 -Objects/stringlib/split.h STRINGLIB_MUTABLE 0 -Objects/stringlib/transmogrify.h STRINGLIB_MUTABLE 0 - -# from Makefile -Modules/getpath.c PYTHONPATH 1 -Modules/getpath.c PREFIX ... -Modules/getpath.c EXEC_PREFIX ... -Modules/getpath.c VERSION ... -Modules/getpath.c VPATH ... -Modules/getpath.c PLATLIBDIR ... -#Modules/_dbmmodule.c USE_GDBM_COMPAT 1 -Modules/_dbmmodule.c USE_NDBM 1 -#Modules/_dbmmodule.c USE_BERKDB 1 - -# See: setup.py -Modules/_decimal/**/*.c CONFIG_64 1 -Modules/_decimal/**/*.c ASM 1 -Modules/expat/xmlparse.c HAVE_EXPAT_CONFIG_H 1 -Modules/expat/xmlparse.c XML_POOR_ENTROPY 1 -Modules/_dbmmodule.c HAVE_GDBM_DASH_NDBM_H 1 - -# others -Modules/_sre/sre_lib.h LOCAL(type) static inline type -Modules/_sre/sre_lib.h SRE(F) sre_ucs2_##F -Objects/stringlib/codecs.h STRINGLIB_IS_UNICODE 1 -Include/internal/pycore_crossinterp_data_registry.h Py_CORE_CROSSINTERP_DATA_REGISTRY_H 1 - -# @end=tsv@ -''')[1:] +EXCLUDED += format_conf_lines([ + # The tool should be able to parse these... + + # The problem with xmlparse.c is that something + # has gone wrong where # we handle "maybe inline actual" + # in Tools/c-analyzer/c_parser/parser/_global.py. + 'Modules/expat/internal.h', + 'Modules/expat/xmlparse.c', +]) + +INCL_DIRS = format_tsv_lines([ + # (glob, dirname) + + ('*', '.'), + ('*', './Include'), + ('*', './Include/internal'), + ('*', './Include/internal/mimalloc'), + + ('Modules/_decimal/**/*.c', 'Modules/_decimal/libmpdec'), + ('Modules/_elementtree.c', 'Modules/expat'), + ('Modules/_hacl/*.c', 'Modules/_hacl/include'), + ('Modules/_hacl/*.c', 'Modules/_hacl/'), + ('Modules/_hacl/*.h', 'Modules/_hacl/include'), + ('Modules/_hacl/*.h', 'Modules/_hacl/'), + ('Modules/md5module.c', 'Modules/_hacl/include'), + ('Modules/sha1module.c', 'Modules/_hacl/include'), + ('Modules/sha2module.c', 'Modules/_hacl/include'), + ('Modules/sha3module.c', 'Modules/_hacl/include'), + ('Modules/blake2module.c', 'Modules/_hacl/include'), + ('Modules/hmacmodule.c', 'Modules/_hacl/include'), + ('Objects/stringlib/*.h', 'Objects'), + + # possible system-installed headers, just in case + ('Modules/_tkinter.c', '/usr/include/tcl8.6'), + ('Modules/_uuidmodule.c', '/usr/include/uuid'), + ('Modules/tkappinit.c', '/usr/include/tcl'), + +]) + +INCLUDES = format_tsv_lines([ + # (glob, include) + + ('**/*.h', 'Python.h'), + ('Include/**/*.h', 'object.h'), + + # for Py_HAVE_CONDVAR + ('Include/internal/pycore_gil.h', 'pycore_condvar.h'), + ('Python/thread_pthread.h', 'pycore_condvar.h'), + + # other + + ('Objects/stringlib/join.h', 'stringlib/stringdefs.h'), + ('Objects/stringlib/ctype.h', 'stringlib/stringdefs.h'), + ('Objects/stringlib/transmogrify.h', 'stringlib/stringdefs.h'), + # ('Objects/stringlib/fastsearch.h', 'stringlib/stringdefs.h'), + # ('Objects/stringlib/count.h', 'stringlib/stringdefs.h'), + # ('Objects/stringlib/find.h', 'stringlib/stringdefs.h'), + # ('Objects/stringlib/partition.h', 'stringlib/stringdefs.h'), + # ('Objects/stringlib/split.h', 'stringlib/stringdefs.h'), + ('Objects/stringlib/fastsearch.h', 'stringlib/ucs1lib.h'), + ('Objects/stringlib/count.h', 'stringlib/ucs1lib.h'), + ('Objects/stringlib/find.h', 'stringlib/ucs1lib.h'), + ('Objects/stringlib/partition.h', 'stringlib/ucs1lib.h'), + ('Objects/stringlib/split.h', 'stringlib/ucs1lib.h'), + ('Objects/stringlib/find_max_char.h', 'Objects/stringlib/ucs1lib.h'), + ('Objects/stringlib/count.h', 'Objects/stringlib/fastsearch.h'), + ('Objects/stringlib/find.h', 'Objects/stringlib/fastsearch.h'), + ('Objects/stringlib/partition.h', 'Objects/stringlib/fastsearch.h'), + ('Objects/stringlib/replace.h', 'Objects/stringlib/fastsearch.h'), + ('Objects/stringlib/repr.h', 'Objects/stringlib/fastsearch.h'), + ('Objects/stringlib/split.h', 'Objects/stringlib/fastsearch.h'), +]) + +MACROS = format_tsv_lines([ + # (glob, name, value) + + ('Include/internal/*.h', 'Py_BUILD_CORE', '1'), + ('Python/**/*.c', 'Py_BUILD_CORE', '1'), + ('Python/**/*.h', 'Py_BUILD_CORE', '1'), + ('Parser/**/*.c', 'Py_BUILD_CORE', '1'), + ('Parser/**/*.h', 'Py_BUILD_CORE', '1'), + ('Objects/**/*.c', 'Py_BUILD_CORE', '1'), + ('Objects/**/*.h', 'Py_BUILD_CORE', '1'), + + ('Modules/_asynciomodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/_codecsmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/_collectionsmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/_ctypes/_ctypes.c', 'Py_BUILD_CORE', '1'), + ('Modules/_ctypes/cfield.c', 'Py_BUILD_CORE', '1'), + ('Modules/_cursesmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/_datetimemodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/_functoolsmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/_heapqmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/_io/*.c', 'Py_BUILD_CORE', '1'), + ('Modules/_io/*.h', 'Py_BUILD_CORE', '1'), + ('Modules/_localemodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/_operator.c', 'Py_BUILD_CORE', '1'), + ('Modules/_posixsubprocess.c', 'Py_BUILD_CORE', '1'), + ('Modules/_sre/sre.c', 'Py_BUILD_CORE', '1'), + ('Modules/_threadmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/_tracemalloc.c', 'Py_BUILD_CORE', '1'), + ('Modules/_weakref.c', 'Py_BUILD_CORE', '1'), + ('Modules/_zoneinfo.c', 'Py_BUILD_CORE', '1'), + ('Modules/atexitmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/cmathmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/faulthandler.c', 'Py_BUILD_CORE', '1'), + ('Modules/gcmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/getpath.c', 'Py_BUILD_CORE', '1'), + ('Modules/getpath_noop.c', 'Py_BUILD_CORE', '1'), + ('Modules/itertoolsmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/main.c', 'Py_BUILD_CORE', '1'), + ('Modules/mathmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/posixmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/sha256module.c', 'Py_BUILD_CORE', '1'), + ('Modules/sha512module.c', 'Py_BUILD_CORE', '1'), + ('Modules/signalmodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/symtablemodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/timemodule.c', 'Py_BUILD_CORE', '1'), + ('Modules/unicodedata.c', 'Py_BUILD_CORE', '1'), + + ('Modules/_json.c', 'Py_BUILD_CORE_BUILTIN', '1'), + ('Modules/_pickle.c', 'Py_BUILD_CORE_BUILTIN', '1'), + ('Modules/_testinternalcapi.c', 'Py_BUILD_CORE_BUILTIN', '1'), + + ('Include/cpython/abstract.h', 'Py_CPYTHON_ABSTRACTOBJECT_H', '1'), + ('Include/cpython/bytearrayobject.h', 'Py_CPYTHON_BYTEARRAYOBJECT_H', '1'), + ('Include/cpython/bytesobject.h', 'Py_CPYTHON_BYTESOBJECT_H', '1'), + ('Include/cpython/ceval.h', 'Py_CPYTHON_CEVAL_H', '1'), + ('Include/cpython/code.h', 'Py_CPYTHON_CODE_H', '1'), + ('Include/cpython/dictobject.h', 'Py_CPYTHON_DICTOBJECT_H', '1'), + ('Include/cpython/fileobject.h', 'Py_CPYTHON_FILEOBJECT_H', '1'), + ('Include/cpython/fileutils.h', 'Py_CPYTHON_FILEUTILS_H', '1'), + ('Include/cpython/frameobject.h', 'Py_CPYTHON_FRAMEOBJECT_H', '1'), + ('Include/cpython/import.h', 'Py_CPYTHON_IMPORT_H', '1'), + ('Include/cpython/interpreteridobject.h', 'Py_CPYTHON_INTERPRETERIDOBJECT_H', '1'), + ('Include/cpython/listobject.h', 'Py_CPYTHON_LISTOBJECT_H', '1'), + ('Include/cpython/methodobject.h', 'Py_CPYTHON_METHODOBJECT_H', '1'), + ('Include/cpython/object.h', 'Py_CPYTHON_OBJECT_H', '1'), + ('Include/cpython/objimpl.h', 'Py_CPYTHON_OBJIMPL_H', '1'), + ('Include/cpython/pyerrors.h', 'Py_CPYTHON_ERRORS_H', '1'), + ('Include/cpython/pylifecycle.h', 'Py_CPYTHON_PYLIFECYCLE_H', '1'), + ('Include/cpython/pymem.h', 'Py_CPYTHON_PYMEM_H', '1'), + ('Include/cpython/pystate.h', 'Py_CPYTHON_PYSTATE_H', '1'), + ('Include/cpython/sysmodule.h', 'Py_CPYTHON_SYSMODULE_H', '1'), + ('Include/cpython/traceback.h', 'Py_CPYTHON_TRACEBACK_H', '1'), + ('Include/cpython/tupleobject.h', 'Py_CPYTHON_TUPLEOBJECT_H', '1'), + ('Include/cpython/unicodeobject.h', 'Py_CPYTHON_UNICODEOBJECT_H', '1'), + + # implied include of <unistd.h> + ('Include/**/*.h', '_POSIX_THREADS', '1'), + ('Include/**/*.h', 'HAVE_PTHREAD_H', '1'), + + # from pyconfig.h + ('Include/cpython/pthread_stubs.h', 'HAVE_PTHREAD_STUBS', '1'), + ('Python/thread_pthread_stubs.h', 'HAVE_PTHREAD_STUBS', '1'), + + # from Objects/bytesobject.c + ('Objects/stringlib/partition.h', 'STRINGLIB_GET_EMPTY()', 'bytes_get_empty()'), + ('Objects/stringlib/join.h', 'STRINGLIB_MUTABLE', '0'), + ('Objects/stringlib/partition.h', 'STRINGLIB_MUTABLE', '0'), + ('Objects/stringlib/split.h', 'STRINGLIB_MUTABLE', '0'), + ('Objects/stringlib/transmogrify.h', 'STRINGLIB_MUTABLE', '0'), + + # from Makefile + ('Modules/getpath.c', 'PYTHONPATH', '1'), + ('Modules/getpath.c', 'PREFIX', '...'), + ('Modules/getpath.c', 'EXEC_PREFIX', '...'), + ('Modules/getpath.c', 'VERSION', '...'), + ('Modules/getpath.c', 'VPATH', '...'), + ('Modules/getpath.c', 'PLATLIBDIR', '...'), + # ('Modules/_dbmmodule.c', 'USE_GDBM_COMPAT', '1'), + ('Modules/_dbmmodule.c', 'USE_NDBM', '1'), + # ('Modules/_dbmmodule.c', 'USE_BERKDB', '1'), + + # See: setup.py + ('Modules/_decimal/**/*.c', 'CONFIG_64', '1'), + ('Modules/_decimal/**/*.c', 'ASM', '1'), + ('Modules/expat/xmlparse.c', 'HAVE_EXPAT_CONFIG_H', '1'), + ('Modules/expat/xmlparse.c', 'XML_POOR_ENTROPY', '1'), + ('Modules/_dbmmodule.c', 'HAVE_GDBM_DASH_NDBM_H', '1'), + + # others + ('Modules/_sre/sre_lib.h', 'LOCAL(type)', 'static inline type'), + ('Modules/_sre/sre_lib.h', 'SRE(F)', 'sre_ucs2_##F'), + ('Objects/stringlib/codecs.h', 'STRINGLIB_IS_UNICODE', '1'), + ('Include/internal/pycore_crossinterp_data_registry.h', 'Py_CORE_CROSSINTERP_DATA_REGISTRY_H', '1'), +]) # -pthread # -Wno-unused-result @@ -345,6 +333,7 @@ def clean_lines(text): _abs('Modules/_ssl_data_300.h'): (80_000, 10_000), _abs('Modules/_ssl_data_111.h'): (80_000, 10_000), _abs('Modules/cjkcodecs/mappings_*.h'): (160_000, 2_000), + _abs('Modules/clinic/_testclinic.c.h'): (120_000, 5_000), _abs('Modules/unicodedata_db.h'): (180_000, 3_000), _abs('Modules/unicodename_db.h'): (1_200_000, 15_000), _abs('Objects/unicodetype_db.h'): (240_000, 3_000), diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 3c3cb2f9c86f16b..515577516f759e0 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -354,7 +354,7 @@ Modules/_testclinic.c - TestClass - ################################## ## global non-objects to fix in builtin modules -# <none> +Objects/memoryobject.c - bool_false - ################################## diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index b128abca39fb41b..57fc14400d38a92 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -24,6 +24,7 @@ Modules/posixmodule.c os_dup2_impl dup3_works - ## guards around resource init Python/thread_pthread.h PyThread__init_thread lib_initialized - +Modules/_struct.c - endian_tables_init_once - ##----------------------- ## other values (not Python-specific) @@ -167,6 +168,7 @@ Python/sysmodule.c - _preinit_xoptions - # XXX need race protection? Modules/faulthandler.c faulthandler_dump_traceback reentrant - Modules/faulthandler.c faulthandler_dump_c_stack reentrant - +Modules/grpmodule.c - group_db_mutex - Python/pylifecycle.c _Py_FatalErrorFormat reentrant - Python/pylifecycle.c fatal_error reentrant - @@ -225,6 +227,7 @@ Modules/_datetimemodule.c datetime_isoformat specs - Modules/_datetimemodule.c parse_hh_mm_ss_ff correction - Modules/_datetimemodule.c time_isoformat specs - Modules/_datetimemodule.c - capi_types - +Modules/_datetimemodule.c normalize_century cache - Modules/_decimal/_decimal.c - cond_map_template - Modules/_decimal/_decimal.c - dec_signal_string - Modules/_decimal/_decimal.c - dflt_ctx - @@ -320,6 +323,7 @@ Modules/pyexpat.c - error_info_of - Modules/pyexpat.c - handler_info - Modules/termios.c - termios_constants - Modules/timemodule.c init_timezone YEAR - +Modules/unicodedata.c unicodedata_create_capi capi - Objects/bytearrayobject.c - _PyByteArray_empty_string - Objects/complexobject.c - c_1 - Objects/exceptions.c - static_exceptions - @@ -451,6 +455,7 @@ Modules/_testcapi/object.c - MyObject_dealloc_called - Modules/_testcapi/object.c - MyType - Modules/_testcapi/structmember.c - test_structmembersType_OldAPI - Modules/_testcapi/watchers.c - g_dict_watch_events - +Modules/_testcapi/watchers.c - g_dict_watch_once - Modules/_testcapi/watchers.c - g_dict_watchers_installed - Modules/_testcapi/watchers.c - g_type_modified_events - Modules/_testcapi/watchers.c - g_type_watchers_installed - diff --git a/Tools/cases_generator/generators_common.py b/Tools/cases_generator/generators_common.py index 9e60d219a71c4ad..0b3d89660099ded 100644 --- a/Tools/cases_generator/generators_common.py +++ b/Tools/cases_generator/generators_common.py @@ -443,7 +443,7 @@ def instruction_size(self, """Replace the INSTRUCTION_SIZE macro with the size of the current instruction.""" if uop.instruction_size is None: raise analysis_error("The INSTRUCTION_SIZE macro requires uop.instruction_size to be set", tkn) - self.out.emit(f" {uop.instruction_size} ") + self.out.emit(f" {uop.instruction_size}u ") return True def _print_storage(self, reason:str, storage: Storage) -> None: diff --git a/Tools/cases_generator/interpreter_definition.md b/Tools/cases_generator/interpreter_definition.md index 1ee4306f3eac1d9..72020133738fa51 100644 --- a/Tools/cases_generator/interpreter_definition.md +++ b/Tools/cases_generator/interpreter_definition.md @@ -81,7 +81,7 @@ and a piece of C code describing its semantics: (definition | family | pseudo)+ definition: - "inst" "(" NAME ["," stack_effect] ")" "{" C-code "}" + "inst" "(" NAME "," stack_effect ")" "{" C-code "}" | "op" "(" NAME "," stack_effect ")" "{" C-code "}" | diff --git a/Tools/cases_generator/opcode_metadata_generator.py b/Tools/cases_generator/opcode_metadata_generator.py index 620e4b6f1f4a69f..10567204dcc5993 100644 --- a/Tools/cases_generator/opcode_metadata_generator.py +++ b/Tools/cases_generator/opcode_metadata_generator.py @@ -157,6 +157,13 @@ def generate_deopt_table(analysis: Analysis, out: CWriter) -> None: if inst.family is not None: deopt = inst.family.name deopts.append((inst.name, deopt)) + defined = set(analysis.opmap.values()) + for i in range(256): + if i not in defined: + deopts.append((f'{i}', f'{i}')) + + assert len(deopts) == 256 + assert len(set(x[0] for x in deopts)) == 256 for name, deopt in sorted(deopts): out.emit(f"[{name}] = {deopt},\n") out.emit("};\n\n") diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index 276f306dfffa187..fc3bc47286f7f63 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -91,7 +91,7 @@ def deopt_if( self.emit("}\n") return not always_true(first_tkn) - def exit_if( # type: ignore[override] + def exit_if( self, tkn: Token, tkn_iter: TokenIterator, diff --git a/Tools/check-c-api-docs/ignored_c_api.txt b/Tools/check-c-api-docs/ignored_c_api.txt new file mode 100644 index 000000000000000..642a1e07f745cd6 --- /dev/null +++ b/Tools/check-c-api-docs/ignored_c_api.txt @@ -0,0 +1,106 @@ +# pydtrace_probes.h +PyDTrace_AUDIT +PyDTrace_FUNCTION_ENTRY +PyDTrace_FUNCTION_RETURN +PyDTrace_GC_DONE +PyDTrace_GC_START +PyDTrace_IMPORT_FIND_LOAD_DONE +PyDTrace_IMPORT_FIND_LOAD_START +PyDTrace_INSTANCE_DELETE_DONE +PyDTrace_INSTANCE_DELETE_START +PyDTrace_INSTANCE_NEW_DONE +PyDTrace_INSTANCE_NEW_START +PyDTrace_LINE +# fileobject.h +Py_FileSystemDefaultEncodeErrors +Py_FileSystemDefaultEncoding +Py_HasFileSystemDefaultEncoding +Py_UTF8Mode +# pyhash.h +Py_HASH_EXTERNAL +# modsupport.h +PyABIInfo_FREETHREADING_AGNOSTIC +# object.h +Py_INVALID_SIZE +# pyexpat.h +PyExpat_CAPI_MAGIC +PyExpat_CAPSULE_NAME +# pyport.h +PYLONG_BITS_IN_DIGIT +PY_DWORD_MAX +PY_FORMAT_SIZE_T +PY_INT32_T +PY_INT64_T +PY_LITTLE_ENDIAN +PY_LLONG_MAX +PY_LLONG_MIN +PY_LONG_LONG +PY_SIZE_MAX +PY_UINT32_T +PY_UINT64_T +PY_ULLONG_MAX +# patchlevel.h +PYTHON_ABI_STRING +PYTHON_API_STRING +PY_RELEASE_LEVEL_ALPHA +PY_RELEASE_LEVEL_BETA +PY_RELEASE_LEVEL_FINAL +PY_RELEASE_LEVEL_GAMMA +PY_VERSION +# unicodeobject.h +Py_UNICODE_SIZE +# cpython/methodobject.h +PyCFunction_GET_CLASS +# cpython/descrobject.h +PyDescr_COMMON +PyDescr_NAME +PyDescr_TYPE +PyWrapperFlag_KEYWORDS +# cpython/fileobject.h +Py_UniversalNewlineFgets +# cpython/pylifecycle.h +Py_FrozenMain +# pythonrun.h +PyErr_Display +# cpython/objimpl.h +PyObject_GET_WEAKREFS_LISTPTR +# cpython/pythonrun.h +PyOS_Readline +# cpython/warnings.h +PyErr_Warn +# fileobject.h +PY_STDIOTEXTMODE +# structmember.h +PY_WRITE_RESTRICTED +# pythread.h +PY_TIMEOUT_T +PY_TIMEOUT_MAX +# cpython/pyctype.h +PY_CTF_ALNUM +PY_CTF_ALPHA +PY_CTF_DIGIT +PY_CTF_LOWER +PY_CTF_SPACE +PY_CTF_UPPER +PY_CTF_XDIGIT +# cpython/code.h +PY_DEF_EVENT +PY_FOREACH_CODE_EVENT +# cpython/funcobject.h +PY_DEF_EVENT +PY_FOREACH_FUNC_EVENT +# cpython/monitoring.h +PY_MONITORING_EVENT_BRANCH +# cpython/dictobject.h +PY_DEF_EVENT +PY_FOREACH_DICT_EVENT +# cpython/pystats.h +PYSTATS_MAX_UOP_ID +# 3.14 only +Py_TPFLAGS_PREHEADER +PyUnicode_AsDecodedObject +PyUnicode_AsDecodedUnicode +PyUnicode_AsEncodedObject +PyUnicode_AsEncodedUnicode +# Soft-deprecated in 3.15; will remain undocumented in 3.14 and 3.13 +PyException_HEAD diff --git a/Tools/check-c-api-docs/main.py b/Tools/check-c-api-docs/main.py new file mode 100644 index 000000000000000..3debb9ed09da78e --- /dev/null +++ b/Tools/check-c-api-docs/main.py @@ -0,0 +1,188 @@ +import re +from pathlib import Path +import sys +import _colorize +import textwrap + +SIMPLE_FUNCTION_REGEX = re.compile(r"PyAPI_FUNC(.+) (\w+)\(") +SIMPLE_MACRO_REGEX = re.compile(r"# *define *(\w+)(\(.+\))? ") +SIMPLE_INLINE_REGEX = re.compile(r"static inline .+( |\n)(\w+)") +SIMPLE_DATA_REGEX = re.compile(r"PyAPI_DATA\(.+\) (\w+)") +API_NAME_REGEX = re.compile(r'\bP[yY][a-zA-Z0-9_]+') + +CPYTHON = Path(__file__).parent.parent.parent +INCLUDE = CPYTHON / "Include" +C_API_DOCS = CPYTHON / "Doc" / "c-api" +IGNORED = ( + (CPYTHON / "Tools" / "check-c-api-docs" / "ignored_c_api.txt") + .read_text() + .split("\n") +) + +for index, line in enumerate(IGNORED): + if line.startswith("#"): + IGNORED.pop(index) + +MISTAKE = """ +If this is a mistake and this script should not be failing, create an +issue and tag Peter (@ZeroIntensity) on it.\ +""" + + +def found_undocumented(singular: bool) -> str: + some = "an" if singular else "some" + s = "" if singular else "s" + these = "this" if singular else "these" + them = "it" if singular else "them" + were = "was" if singular else "were" + + return ( + textwrap.dedent( + f""" + Found {some} undocumented C API{s}! + + Python requires documentation on all public C API symbols, macros, and types. + If {these} API{s} {were} not meant to be public, prefix {them} with a + leading underscore (_PySomething_API) or move {them} to the internal C API + (pycore_*.h files). + + In exceptional cases, certain APIs can be ignored by adding them to + Tools/check-c-api-docs/ignored_c_api.txt + """ + ) + + MISTAKE + ) + + +def found_ignored_documented(singular: bool) -> str: + some = "a" if singular else "some" + s = "" if singular else "s" + them = "it" if singular else "them" + were = "was" if singular else "were" + they = "it" if singular else "they" + + return ( + textwrap.dedent( + f""" + Found {some} C API{s} listed in Tools/c-api-docs-check/ignored_c_api.txt, but + {they} {were} found in the documentation. To fix this, remove {them} from + ignored_c_api.txt. + """ + ) + + MISTAKE + ) + + +def scan_file_for_docs( + filename: str, + text: str, + names: set[str]) -> tuple[list[str], list[str]]: + """ + Scan a header file for C API functions. + """ + undocumented: list[str] = [] + documented_ignored: list[str] = [] + colors = _colorize.get_colors() + + def check_for_name(name: str) -> None: + documented = name in names + if documented and (name in IGNORED): + documented_ignored.append(name) + elif not documented and (name not in IGNORED): + undocumented.append(name) + + for function in SIMPLE_FUNCTION_REGEX.finditer(text): + name = function.group(2) + if not API_NAME_REGEX.fullmatch(name): + continue + + check_for_name(name) + + for macro in SIMPLE_MACRO_REGEX.finditer(text): + name = macro.group(1) + if not API_NAME_REGEX.fullmatch(name): + continue + + if "(" in name: + name = name[: name.index("(")] + + check_for_name(name) + + for inline in SIMPLE_INLINE_REGEX.finditer(text): + name = inline.group(2) + if not API_NAME_REGEX.fullmatch(name): + continue + + check_for_name(name) + + for data in SIMPLE_DATA_REGEX.finditer(text): + name = data.group(1) + if not API_NAME_REGEX.fullmatch(name): + continue + + check_for_name(name) + + # Remove duplicates and sort alphabetically to keep the output deterministic + undocumented = list(set(undocumented)) + undocumented.sort() + + if undocumented or documented_ignored: + print(f"{filename} {colors.RED}BAD{colors.RESET}") + for name in undocumented: + print(f"{colors.BOLD_RED}UNDOCUMENTED:{colors.RESET} {name}") + for name in documented_ignored: + print(f"{colors.BOLD_YELLOW}DOCUMENTED BUT IGNORED:{colors.RESET} {name}") + else: + print(f"{filename} {colors.GREEN}OK{colors.RESET}") + + return undocumented, documented_ignored + + +def main() -> None: + print("Gathering C API names from docs...") + names = set() + for path in C_API_DOCS.glob('**/*.rst'): + text = path.read_text(encoding="utf-8") + for name in API_NAME_REGEX.findall(text): + names.add(name) + print(f"Got {len(names)} names!") + + print("Scanning for undocumented C API functions...") + files = [*INCLUDE.iterdir(), *(INCLUDE / "cpython").iterdir()] + all_missing: list[str] = [] + all_found_ignored: list[str] = [] + + for file in files: + if file.is_dir(): + continue + assert file.exists() + text = file.read_text(encoding="utf-8") + missing, ignored = scan_file_for_docs(str(file.relative_to(INCLUDE)), text, names) + all_found_ignored += ignored + all_missing += missing + + fail = False + to_check = [ + (all_missing, "missing", found_undocumented(len(all_missing) == 1)), + ( + all_found_ignored, + "documented but ignored", + found_ignored_documented(len(all_found_ignored) == 1), + ), + ] + for name_list, what, message in to_check: + if not name_list: + continue + + s = "s" if len(name_list) != 1 else "" + print(f"-- {len(name_list)} {what} C API{s} --") + for name in name_list: + print(f" - {name}") + print(message) + fail = True + + sys.exit(1 if fail else 0) + + +if __name__ == "__main__": + main() diff --git a/Tools/check-c-api-docs/mypy.ini b/Tools/check-c-api-docs/mypy.ini new file mode 100644 index 000000000000000..f42eb2836e2fd85 --- /dev/null +++ b/Tools/check-c-api-docs/mypy.ini @@ -0,0 +1,19 @@ +[mypy] +files = Tools/check-c-api-docs/ +pretty = True + +# We need `_colorize` import: +mypy_path = $MYPY_CONFIG_FILE_DIR/../../Misc/mypy + +# Make sure Python can still be built +# using Python 3.13 for `PYTHON_FOR_REGEN`... +python_version = 3.13 + +# ...And be strict: +strict = True +extra_checks = True +enable_error_code = + ignore-without-code, + redundant-expr, + truthy-bool, + possibly-undefined, diff --git a/Tools/clinic/.ruff.toml b/Tools/clinic/.ruff.toml index 5033887df0c1cda..944d17ee3e98551 100644 --- a/Tools/clinic/.ruff.toml +++ b/Tools/clinic/.ruff.toml @@ -17,9 +17,6 @@ ignore = [ # Use f-strings instead of format specifiers. # Doesn't always make code more readable. "UP032", - # Use PEP-604 unions rather than tuples for isinstance() checks. - # Makes code slower and more verbose. https://github.com/astral-sh/ruff/issues/7871. - "UP038", ] unfixable = [ # The autofixes sometimes do the wrong things for these; diff --git a/Tools/clinic/libclinic/__init__.py b/Tools/clinic/libclinic/__init__.py index 7c5cede23966777..742f1448146a0fa 100644 --- a/Tools/clinic/libclinic/__init__.py +++ b/Tools/clinic/libclinic/__init__.py @@ -7,7 +7,9 @@ ) from .formatting import ( SIG_END_MARKER, - c_repr, + c_str_repr, + c_bytes_repr, + c_unichar_repr, docstring_for_c_string, format_escape, indent_all_lines, @@ -26,7 +28,7 @@ from .utils import ( FormatCounterFormatter, NULL, - Null, + NullType, Sentinels, VersionTuple, compute_checksum, @@ -45,7 +47,9 @@ # Formatting helpers "SIG_END_MARKER", - "c_repr", + "c_str_repr", + "c_bytes_repr", + "c_unichar_repr", "docstring_for_c_string", "format_escape", "indent_all_lines", @@ -64,7 +68,7 @@ # Utility functions "FormatCounterFormatter", "NULL", - "Null", + "NullType", "Sentinels", "VersionTuple", "compute_checksum", diff --git a/Tools/clinic/libclinic/clanguage.py b/Tools/clinic/libclinic/clanguage.py index 9e7fa7a7f58f956..7f02c7790f015aa 100644 --- a/Tools/clinic/libclinic/clanguage.py +++ b/Tools/clinic/libclinic/clanguage.py @@ -6,7 +6,7 @@ from operator import attrgetter from collections.abc import Iterable -import libclinic +import libclinic.cpp from libclinic import ( unspecified, fail, Sentinels, VersionTuple) from libclinic.codegen import CRenderData, TemplateDict, CodeGen @@ -101,7 +101,7 @@ def compiler_deprecated_warning( code = self.COMPILER_DEPRECATION_WARNING_PROTOTYPE.format( major=minversion[0], minor=minversion[1], - message=libclinic.c_repr(message), + message=libclinic.c_str_repr(message), ) return libclinic.normalize_snippet(code) diff --git a/Tools/clinic/libclinic/converter.py b/Tools/clinic/libclinic/converter.py index 2c93dda35410308..3d375dd3fdd70db 100644 --- a/Tools/clinic/libclinic/converter.py +++ b/Tools/clinic/libclinic/converter.py @@ -6,7 +6,7 @@ import libclinic from libclinic import fail -from libclinic import Sentinels, unspecified, unknown +from libclinic import Sentinels, unspecified, unknown, NULL from libclinic.codegen import CRenderData, Include, TemplateDict from libclinic.function import Function, Parameter @@ -83,9 +83,9 @@ class CConverter(metaclass=CConverterAutoRegister): # at runtime). default: object = unspecified - # If not None, default must be isinstance() of this type. + # default must be isinstance() of this type. # (You can also specify a tuple of types.) - default_type: bltns.type[object] | tuple[bltns.type[object], ...] | None = None + default_type: bltns.type[object] | tuple[bltns.type[object], ...] = object # "default" converted into a C value, as a string. # Or None if there is no default. @@ -95,6 +95,13 @@ class CConverter(metaclass=CConverterAutoRegister): # Or None if there is no default. py_default: str | None = None + # The default value used to initialize the C variable when + # there is no default. + # + # Every non-abstract subclass with non-trivial cleanup() should supply + # a valid value. + c_init_default: str = '' + # The default value used to initialize the C variable when # there is no default, but not specifying a default may # result in an "uninitialized variable" warning. This can @@ -105,7 +112,7 @@ class CConverter(metaclass=CConverterAutoRegister): # # This value is specified as a string. # Every non-abstract subclass should supply a valid value. - c_ignored_default: str = 'NULL' + c_ignored_default: str = '' # If true, wrap with Py_UNUSED. unused = False @@ -182,9 +189,25 @@ def __init__(self, self.unused = unused self._includes: list[Include] = [] + if c_default: + self.c_default = c_default + if py_default: + self.py_default = py_default + + if annotation is not unspecified: + fail("The 'annotation' parameter is not currently permitted.") + + # Make sure not to set self.function until after converter_init() has been called. + # This prevents you from caching information + # about the function in converter_init(). + # (That breaks if we get cloned.) + self.converter_init(**kwargs) + if default is not unspecified: - if (self.default_type - and default is not unknown + if self.default_type == (): + conv_name = self.__class__.__name__.removesuffix('_converter') + fail(f"A '{conv_name}' parameter cannot be marked optional.") + if (default is not unknown and not isinstance(default, self.default_type) ): if isinstance(self.default_type, type): @@ -197,19 +220,38 @@ def __init__(self, f"{name!r} is not of type {types_str!r}") self.default = default - if c_default: - self.c_default = c_default - if py_default: - self.py_default = py_default - - if annotation is not unspecified: - fail("The 'annotation' parameter is not currently permitted.") + if not self.c_default: + if default is unspecified: + if self.c_init_default: + self.c_default = self.c_init_default + elif default is NULL: + self.c_default = self.c_ignored_default or self.c_init_default + if not self.c_default: + cls_name = self.__class__.__name__ + fail(f"{cls_name}: c_default is required for " + f"default value NULL") + else: + assert default is not unknown + self.c_default_init() + if not self.c_default: + if default is None: + self.c_default = self.c_init_default + if not self.c_default: + cls_name = self.__class__.__name__ + fail(f"{cls_name}: c_default is required for " + f"default value None") + elif isinstance(default, str): + self.c_default = libclinic.c_str_repr(default) + elif isinstance(default, bytes): + self.c_default = libclinic.c_bytes_repr(default) + elif isinstance(default, (int, float)): + self.c_default = repr(default) + else: + cls_name = self.__class__.__name__ + fail(f"{cls_name}: c_default is required for " + f"default value {default!r}") + fail(f"Unsupported default value {default!r}.") - # Make sure not to set self.function until after converter_init() has been called. - # This prevents you from caching information - # about the function in converter_init(). - # (That breaks if we get cloned.) - self.converter_init(**kwargs) self.function = function # Add a custom __getattr__ method to improve the error message @@ -233,6 +275,9 @@ def __getattr__(self, attr): def converter_init(self) -> None: pass + def c_default_init(self) -> None: + return + def is_optional(self) -> bool: return (self.default is not unspecified) @@ -324,7 +369,7 @@ def parse_argument(self, args: list[str]) -> None: args.append(self.converter) if self.encoding: - args.append(libclinic.c_repr(self.encoding)) + args.append(libclinic.c_str_repr(self.encoding)) elif self.subclass_of: args.append(self.subclass_of) @@ -371,7 +416,7 @@ def declaration(self, *, in_parser: bool = False) -> str: declaration = [self.simple_declaration(in_parser=True)] default = self.c_default if not default and self.parameter.group: - default = self.c_ignored_default + default = self.c_ignored_default or self.c_init_default if default: declaration.append(" = ") declaration.append(default) diff --git a/Tools/clinic/libclinic/converters.py b/Tools/clinic/libclinic/converters.py index 633fb5f56a6693b..64fc1e95007516e 100644 --- a/Tools/clinic/libclinic/converters.py +++ b/Tools/clinic/libclinic/converters.py @@ -4,7 +4,7 @@ from types import NoneType from typing import Any -from libclinic import fail, Null, unspecified, unknown +from libclinic import fail, NullType, unspecified, NULL, c_bytes_repr, c_unichar_repr from libclinic.function import ( Function, Parameter, CALLABLE, STATIC_METHOD, CLASS_METHOD, METHOD_INIT, METHOD_NEW, @@ -17,6 +17,57 @@ TypeSet = set[bltns.type[object]] +class BaseUnsignedIntConverter(CConverter): + bitwise = False + default_type = int + c_ignored_default = '0' + + def use_converter(self) -> None: + if self.converter: + self.add_include('pycore_long.h', + f'{self.converter}()') + + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: + if not limited_capi: + return super().parse_arg(argname, displayname, limited_capi=limited_capi) + return self.format_code(""" + {{{{ + Py_ssize_t _bytes = PyLong_AsNativeBytes({argname}, &{paramname}, sizeof({type}), + Py_ASNATIVEBYTES_NATIVE_ENDIAN | + Py_ASNATIVEBYTES_ALLOW_INDEX | + Py_ASNATIVEBYTES_REJECT_NEGATIVE | + Py_ASNATIVEBYTES_UNSIGNED_BUFFER); + if (_bytes < 0) {{{{ + goto exit; + }}}} + if ((size_t)_bytes > sizeof({type})) {{{{ + PyErr_SetString(PyExc_OverflowError, + "Python int too large for C {type}"); + goto exit; + }}}} + }}}} + """, + argname=argname, + type=self.type) + + +class uint8_converter(BaseUnsignedIntConverter): + type = "uint8_t" + converter = '_PyLong_UInt8_Converter' + +class uint16_converter(BaseUnsignedIntConverter): + type = "uint16_t" + converter = '_PyLong_UInt16_Converter' + +class uint32_converter(BaseUnsignedIntConverter): + type = "uint32_t" + converter = '_PyLong_UInt32_Converter' + +class uint64_converter(BaseUnsignedIntConverter): + type = "uint64_t" + converter = '_PyLong_UInt64_Converter' + + class bool_converter(CConverter): type = 'int' default_type = bool @@ -26,12 +77,13 @@ class bool_converter(CConverter): def converter_init(self, *, accept: TypeSet = {object}) -> None: if accept == {int}: self.format_unit = 'i' + self.default_type = int # type: ignore[assignment] elif accept != {object}: fail(f"bool_converter: illegal 'accept' argument {accept!r}") - if self.default is not unspecified and self.default is not unknown: - self.default = bool(self.default) - if self.c_default in {'Py_True', 'Py_False'}: - self.c_default = str(int(self.default)) + + def c_default_init(self) -> None: + assert isinstance(self.default, int) + self.c_default = str(int(self.default)) def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'i': @@ -59,6 +111,7 @@ class defining_class_converter(CConverter): this is the default converter used for the defining class. """ type = 'PyTypeObject *' + default_type = () format_unit = '' show_in_signature = False specified_type: str | None = None @@ -75,7 +128,7 @@ def set_template_dict(self, template_dict: TemplateDict) -> None: class char_converter(CConverter): type = 'char' - default_type = (bytes, bytearray) + default_type = bytes format_unit = 'c' c_ignored_default = "'\0'" @@ -84,9 +137,18 @@ def converter_init(self) -> None: if len(self.default) != 1: fail(f"char_converter: illegal default value {self.default!r}") - self.c_default = repr(bytes(self.default))[1:] - if self.c_default == '"\'"': - self.c_default = r"'\''" + def c_default_init(self) -> None: + default = self.default + assert isinstance(default, bytes) + if default == b"'": + self.c_default = r"'\''" + elif default == b'"': + self.c_default = r"""'"'""" + elif default == b'\0': + self.c_default = r"'\0'" + else: + r = c_bytes_repr(default)[1:-1] + self.c_default = "'" + r + "'" def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'c': @@ -126,7 +188,6 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st @add_legacy_c_converter('B', bitwise=True) class unsigned_char_converter(CConverter): type = 'unsigned char' - default_type = int format_unit = 'b' c_ignored_default = "'\0'" @@ -211,32 +272,8 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st return super().parse_arg(argname, displayname, limited_capi=limited_capi) -def format_inline_unsigned_int_converter(self: CConverter, argname: str) -> str: - return self.format_code(""" - {{{{ - Py_ssize_t _bytes = PyLong_AsNativeBytes({argname}, &{paramname}, sizeof({type}), - Py_ASNATIVEBYTES_NATIVE_ENDIAN | - Py_ASNATIVEBYTES_ALLOW_INDEX | - Py_ASNATIVEBYTES_REJECT_NEGATIVE | - Py_ASNATIVEBYTES_UNSIGNED_BUFFER); - if (_bytes < 0) {{{{ - goto exit; - }}}} - if ((size_t)_bytes > sizeof({type})) {{{{ - PyErr_SetString(PyExc_OverflowError, - "Python int too large for C {type}"); - goto exit; - }}}} - }}}} - """, - argname=argname, - type=self.type) - - -class unsigned_short_converter(CConverter): +class unsigned_short_converter(BaseUnsignedIntConverter): type = 'unsigned short' - default_type = int - c_ignored_default = "0" def converter_init(self, *, bitwise: bool = False) -> None: if bitwise: @@ -244,11 +281,6 @@ def converter_init(self, *, bitwise: bool = False) -> None: else: self.converter = '_PyLong_UnsignedShort_Converter' - def use_converter(self) -> None: - if self.converter == '_PyLong_UnsignedShort_Converter': - self.add_include('pycore_long.h', - '_PyLong_UnsignedShort_Converter()') - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'H': return self.format_code(""" @@ -258,9 +290,7 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st }}}} """, argname=argname) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - return format_inline_unsigned_int_converter(self, argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) @add_legacy_c_converter('C', accept={str}) @@ -275,11 +305,19 @@ def converter_init( ) -> None: if accept == {str}: self.format_unit = 'C' + self.default_type = str # type: ignore[assignment] + if isinstance(self.default, str): + if len(self.default) != 1: + fail(f"int_converter: illegal default value {self.default!r}") elif accept != {int}: fail(f"int_converter: illegal 'accept' argument {accept!r}") if type is not None: self.type = type + def c_default_init(self) -> None: + if isinstance(self.default, str): + self.c_default = c_unichar_repr(self.default) + def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'i': return self.format_code(""" @@ -311,10 +349,8 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st return super().parse_arg(argname, displayname, limited_capi=limited_capi) -class unsigned_int_converter(CConverter): +class unsigned_int_converter(BaseUnsignedIntConverter): type = 'unsigned int' - default_type = int - c_ignored_default = "0" def converter_init(self, *, bitwise: bool = False) -> None: if bitwise: @@ -322,11 +358,6 @@ def converter_init(self, *, bitwise: bool = False) -> None: else: self.converter = '_PyLong_UnsignedInt_Converter' - def use_converter(self) -> None: - if self.converter == '_PyLong_UnsignedInt_Converter': - self.add_include('pycore_long.h', - '_PyLong_UnsignedInt_Converter()') - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'I': return self.format_code(""" @@ -336,9 +367,7 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st }}}} """, argname=argname) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - return format_inline_unsigned_int_converter(self, argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) class long_converter(CConverter): @@ -359,10 +388,8 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st return super().parse_arg(argname, displayname, limited_capi=limited_capi) -class unsigned_long_converter(CConverter): +class unsigned_long_converter(BaseUnsignedIntConverter): type = 'unsigned long' - default_type = int - c_ignored_default = "0" def converter_init(self, *, bitwise: bool = False) -> None: if bitwise: @@ -370,11 +397,6 @@ def converter_init(self, *, bitwise: bool = False) -> None: else: self.converter = '_PyLong_UnsignedLong_Converter' - def use_converter(self) -> None: - if self.converter == '_PyLong_UnsignedLong_Converter': - self.add_include('pycore_long.h', - '_PyLong_UnsignedLong_Converter()') - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'k': return self.format_code(""" @@ -387,9 +409,7 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st argname=argname, bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), ) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - return format_inline_unsigned_int_converter(self, argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) class long_long_converter(CConverter): @@ -410,10 +430,8 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st return super().parse_arg(argname, displayname, limited_capi=limited_capi) -class unsigned_long_long_converter(CConverter): +class unsigned_long_long_converter(BaseUnsignedIntConverter): type = 'unsigned long long' - default_type = int - c_ignored_default = "0" def converter_init(self, *, bitwise: bool = False) -> None: if bitwise: @@ -421,11 +439,6 @@ def converter_init(self, *, bitwise: bool = False) -> None: else: self.converter = '_PyLong_UnsignedLongLong_Converter' - def use_converter(self) -> None: - if self.converter == '_PyLong_UnsignedLongLong_Converter': - self.add_include('pycore_long.h', - '_PyLong_UnsignedLongLong_Converter()') - def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'K': return self.format_code(""" @@ -438,19 +451,18 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st argname=argname, bad_argument=self.bad_argument(displayname, 'int', limited_capi=limited_capi), ) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - return format_inline_unsigned_int_converter(self, argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) class Py_ssize_t_converter(CConverter): type = 'Py_ssize_t' + default_type = (int, NoneType) c_ignored_default = "0" def converter_init(self, *, accept: TypeSet = {int}) -> None: if accept == {int}: self.format_unit = 'n' - self.default_type = int + self.default_type = int # type: ignore[assignment] elif accept == {int, NoneType}: self.converter = '_Py_convert_optional_to_ssize_t' else: @@ -507,10 +519,13 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st class slice_index_converter(CConverter): type = 'Py_ssize_t' + default_type = (int, NoneType) + c_ignored_default = "0" def converter_init(self, *, accept: TypeSet = {int, NoneType}) -> None: if accept == {int}: self.converter = '_PyEval_SliceIndexNotNone' + self.default_type = int # type: ignore[assignment] self.nullable = False elif accept == {int, NoneType}: self.converter = '_PyEval_SliceIndex' @@ -557,14 +572,9 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st argname=argname) -class size_t_converter(CConverter): +class size_t_converter(BaseUnsignedIntConverter): type = 'size_t' converter = '_PyLong_Size_t_Converter' - c_ignored_default = "0" - - def use_converter(self) -> None: - self.add_include('pycore_long.h', - '_PyLong_Size_t_Converter()') def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'n': @@ -575,9 +585,7 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st }}}} """, argname=argname) - if not limited_capi: - return super().parse_arg(argname, displayname, limited_capi=limited_capi) - return format_inline_unsigned_int_converter(self, argname) + return super().parse_arg(argname, displayname, limited_capi=limited_capi) class fildes_converter(CConverter): @@ -685,6 +693,7 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st class object_converter(CConverter): type = 'PyObject *' format_unit = 'O' + c_ignored_default = 'NULL' def converter_init( self, *, @@ -704,6 +713,10 @@ def converter_init( if type is not None: self.type = type + def c_default_init(self) -> None: + default = self.default + if default is None or isinstance(default, bool): + self.c_default = "Py_" + repr(default) # # We define three conventions for buffer types in the 'accept' argument: @@ -733,8 +746,9 @@ def str_converter_key( class str_converter(CConverter): type = 'const char *' - default_type = (str, Null, NoneType) + default_type = (str, bytes, NullType, NoneType) format_unit = 's' + c_ignored_default = 'NULL' def converter_init( self, @@ -752,14 +766,16 @@ def converter_init( self.format_unit = format_unit self.length = bool(zeroes) if encoding: - if self.default not in (Null, None, unspecified): + if self.default not in (NULL, None, unspecified): fail("str_converter: Argument Clinic doesn't support default values for encoded strings") self.encoding = encoding self.type = 'char *' # sorry, clinic can't support preallocated buffers # for es# and et# self.c_default = "NULL" - if NoneType in accept and self.c_default == "Py_None": + + def c_default_init(self) -> None: + if self.default is None: self.c_default = "NULL" def post_parsing(self) -> str: @@ -872,6 +888,7 @@ class PyBytesObject_converter(CConverter): type = 'PyBytesObject *' format_unit = 'S' # accept = {bytes} + c_ignored_default = 'NULL' def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'S': @@ -892,6 +909,7 @@ class PyByteArrayObject_converter(CConverter): type = 'PyByteArrayObject *' format_unit = 'Y' # accept = {bytearray} + c_ignored_default = 'NULL' def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'Y': @@ -910,8 +928,9 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st class unicode_converter(CConverter): type = 'PyObject *' - default_type = (str, Null, NoneType) + default_type = (str, NullType, NoneType) format_unit = 'U' + c_ignored_default = 'NULL' def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: if self.format_unit == 'U': @@ -928,13 +947,34 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st return super().parse_arg(argname, displayname, limited_capi=limited_capi) +class _unicode_fs_converter_base(CConverter): + type = 'PyObject *' + default_type = NullType + c_init_default = 'NULL' + + def c_default_init(self) -> None: + fail(f"{self.__class__.__name__} does not support default values") + + def cleanup(self) -> str: + return f"Py_XDECREF({self.parser_name});" + + +class unicode_fs_encoded_converter(_unicode_fs_converter_base): + converter = 'PyUnicode_FSConverter' + + +class unicode_fs_decoded_converter(_unicode_fs_converter_base): + converter = 'PyUnicode_FSDecoder' + + @add_legacy_c_converter('u') @add_legacy_c_converter('u#', zeroes=True) @add_legacy_c_converter('Z', accept={str, NoneType}) @add_legacy_c_converter('Z#', accept={str, NoneType}, zeroes=True) class Py_UNICODE_converter(CConverter): type = 'const wchar_t *' - default_type = (str, Null, NoneType) + default_type = (str, NullType, NoneType) + c_ignored_default = 'NULL' def converter_init( self, *, @@ -950,6 +990,7 @@ def converter_init( self.accept = accept if accept == {str}: self.converter = '_PyUnicode_WideCharString_Converter' + self.default_type = (str, NullType) # type: ignore[assignment] elif accept == {str, NoneType}: self.converter = '_PyUnicode_WideCharString_Opt_Converter' else: @@ -1005,28 +1046,34 @@ def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> st @add_legacy_c_converter('w*', accept={rwbuffer}) class Py_buffer_converter(CConverter): type = 'Py_buffer' + default_type = (str, bytes, NullType, NoneType) format_unit = 'y*' impl_by_reference = True - c_ignored_default = "{NULL, NULL}" + c_init_default = "{NULL, NULL}" def converter_init(self, *, accept: TypeSet = {buffer}) -> None: - if self.default not in (unspecified, None): - fail("The only legal default value for Py_buffer is None.") - - self.c_default = self.c_ignored_default - if accept == {str, buffer, NoneType}: - format_unit = 'z*' + self.format_unit = 'z*' + self.default_type = (str, bytes, NullType, NoneType) elif accept == {str, buffer}: - format_unit = 's*' + self.format_unit = 's*' + self.default_type = (str, bytes, NullType) # type: ignore[assignment] elif accept == {buffer}: - format_unit = 'y*' + self.format_unit = 'y*' + self.default_type = (bytes, NullType) # type: ignore[assignment] elif accept == {rwbuffer}: - format_unit = 'w*' + self.format_unit = 'w*' + self.default_type = NullType # type: ignore[assignment] else: fail("Py_buffer_converter: illegal combination of arguments") - self.format_unit = format_unit + def c_default_init(self) -> None: + default = self.default + if isinstance(default, bytes): + self.c_default = f'{{.buf = {c_bytes_repr(default)}, .obj = NULL, .len = {len(default)}}}' + elif isinstance(default, str): + default = default.encode() + self.c_default = f'{{.buf = {c_bytes_repr(default)}, .obj = NULL, .len = {len(default)}}}' def cleanup(self) -> str: name = self.name @@ -1107,6 +1154,7 @@ class self_converter(CConverter): this is the default converter used for "self". """ type: str | None = None + default_type = () format_unit = '' specified_type: str | None = None @@ -1221,6 +1269,7 @@ def use_pyobject_self(self, func: Function) -> bool: # Converters for var-positional parameter. class VarPosCConverter(CConverter): + default_type = () format_unit = '' def parse_arg(self, argname: str, displayname: str, *, limited_capi: bool) -> str | None: @@ -1233,8 +1282,7 @@ def parse_vararg(self, *, pos_only: int, min_pos: int, max_pos: int, class varpos_tuple_converter(VarPosCConverter): type = 'PyObject *' - format_unit = '' - c_default = 'NULL' + c_init_default = 'NULL' def cleanup(self) -> str: return f"""Py_XDECREF({self.parser_name});\n""" @@ -1292,7 +1340,6 @@ def parse_vararg(self, *, pos_only: int, min_pos: int, max_pos: int, class varpos_array_converter(VarPosCConverter): type = 'PyObject * const *' length = True - c_ignored_default = '' def parse_vararg(self, *, pos_only: int, min_pos: int, max_pos: int, fastcall: bool, limited_capi: bool) -> str: diff --git a/Tools/clinic/libclinic/dsl_parser.py b/Tools/clinic/libclinic/dsl_parser.py index 282ff64cd33089c..6ead9bf20228334 100644 --- a/Tools/clinic/libclinic/dsl_parser.py +++ b/Tools/clinic/libclinic/dsl_parser.py @@ -7,7 +7,7 @@ import shlex import sys from collections.abc import Callable -from types import FunctionType, NoneType +from types import FunctionType from typing import TYPE_CHECKING, Any, NamedTuple import libclinic @@ -877,43 +877,16 @@ def parse_parameter(self, line: str) -> None: # handle "as" for parameters too c_name = None - name, have_as_token, trailing = line.partition(' as ') - if have_as_token: - name = name.strip() - if ' ' not in name: - fields = trailing.strip().split(' ') - if not fields: - fail("Invalid 'as' clause!") - c_name = fields[0] - if c_name.endswith(':'): - name += ':' - c_name = c_name[:-1] - fields[0] = name - line = ' '.join(fields) - - default: str | None - base, equals, default = line.rpartition('=') - if not equals: - base = default - default = None - - module = None + m = re.match(r'(?:\* *)?\w+( +as +(\w+))', line) + if m: + c_name = m[2] + line = line[:m.start(1)] + line[m.end(1):] + try: - ast_input = f"def x({base}): pass" + ast_input = f"def x({line}\n): pass" module = ast.parse(ast_input) except SyntaxError: - try: - # the last = was probably inside a function call, like - # c: int(accept={str}) - # so assume there was no actual default value. - default = None - ast_input = f"def x({line}): pass" - module = ast.parse(ast_input) - except SyntaxError: - pass - if not module: - fail(f"Function {self.function.name!r} has an invalid parameter declaration:\n\t", - repr(line)) + fail(f"Function {self.function.name!r} has an invalid parameter declaration: {line!r}") function = module.body[0] assert isinstance(function, ast.FunctionDef) @@ -922,9 +895,6 @@ def parse_parameter(self, line: str) -> None: if len(function_args.args) > 1: fail(f"Function {self.function.name!r} has an " f"invalid parameter declaration (comma?): {line!r}") - if function_args.defaults or function_args.kw_defaults: - fail(f"Function {self.function.name!r} has an " - f"invalid parameter declaration (default value?): {line!r}") if function_args.kwarg: fail(f"Function {self.function.name!r} has an " f"invalid parameter declaration (**kwargs?): {line!r}") @@ -944,29 +914,26 @@ def parse_parameter(self, line: str) -> None: name = 'varpos_' + name value: object - if not default: - if is_vararg: - value = NULL - else: - if self.parameter_state is ParamState.OPTIONAL: - fail(f"Can't have a parameter without a default ({parameter_name!r}) " - "after a parameter with a default!") - value = unspecified + has_c_default = 'c_default' in kwargs + if not function_args.defaults: + value = unspecified + if (not is_vararg + and self.parameter_state is ParamState.OPTIONAL): + fail(f"Can't have a parameter without a default ({parameter_name!r}) " + "after a parameter with a default!") if 'py_default' in kwargs: fail("You can't specify py_default without specifying a default value!") + if has_c_default: + fail("You can't specify c_default without specifying a default value!") else: - if is_vararg: - fail("Vararg can't take a default value!") + expr = function_args.defaults[0] + default = ast_input[expr.col_offset: expr.end_col_offset].strip() if self.parameter_state is ParamState.REQUIRED: self.parameter_state = ParamState.OPTIONAL - default = default.strip() bad = False - ast_input = f"x = {default}" try: - module = ast.parse(ast_input) - - if 'c_default' not in kwargs: + if not has_c_default: # we can only represent very simple data values in C. # detect whether default is okay, via a denylist # of disallowed ast nodes. @@ -992,13 +959,14 @@ def bad_node(self, node: ast.AST) -> None: visit_Starred = bad_node denylist = DetectBadNodes() - denylist.visit(module) + denylist.visit(expr) bad = denylist.bad else: # if they specify a c_default, we can be more lenient about the default value. # but at least make an attempt at ensuring it's a valid expression. + code = compile(ast.Expression(expr), '<expr>', 'eval') try: - value = eval(default) + value = eval(code) except NameError: pass # probably a named constant except Exception as e: @@ -1010,22 +978,16 @@ def bad_node(self, node: ast.AST) -> None: if bad: fail(f"Unsupported expression as default value: {default!r}") - assignment = module.body[0] - assert isinstance(assignment, ast.Assign) - expr = assignment.value # mild hack: explicitly support NULL as a default value - c_default: str | None if isinstance(expr, ast.Name) and expr.id == 'NULL': value = NULL py_default = '<unrepresentable>' - c_default = "NULL" elif (isinstance(expr, ast.BinOp) or (isinstance(expr, ast.UnaryOp) and not (isinstance(expr.operand, ast.Constant) and type(expr.operand.value) in {int, float, complex}) )): - c_default = kwargs.get("c_default") - if not (isinstance(c_default, str) and c_default): + if not has_c_default: fail(f"When you specify an expression ({default!r}) " f"as your default value, " f"you MUST specify a valid c_default.", @@ -1044,8 +1006,7 @@ def bad_node(self, node: ast.AST) -> None: a.append(n.id) py_default = ".".join(reversed(a)) - c_default = kwargs.get("c_default") - if not (isinstance(c_default, str) and c_default): + if not has_c_default: fail(f"When you specify a named constant ({py_default!r}) " "as your default value, " "you MUST specify a valid c_default.") @@ -1057,25 +1018,15 @@ def bad_node(self, node: ast.AST) -> None: else: value = ast.literal_eval(expr) py_default = repr(value) - if isinstance(value, (bool, NoneType)): - c_default = "Py_" + py_default - elif isinstance(value, str): - c_default = libclinic.c_repr(value) - else: - c_default = py_default - except SyntaxError as e: - fail(f"Syntax error: {e.text!r}") except (ValueError, AttributeError): value = unknown - c_default = kwargs.get("c_default") py_default = default - if not (isinstance(c_default, str) and c_default): + if not has_c_default: fail("When you specify a named constant " f"({py_default!r}) as your default value, " "you MUST specify a valid c_default.") - kwargs.setdefault('c_default', c_default) kwargs.setdefault('py_default', py_default) dict = legacy_converters if legacy else converters @@ -1096,12 +1047,10 @@ def bad_node(self, node: ast.AST) -> None: if isinstance(converter, self_converter): if len(self.function.parameters) == 1: - if self.parameter_state is not ParamState.REQUIRED: - fail("A 'self' parameter cannot be marked optional.") - if value is not unspecified: - fail("A 'self' parameter cannot have a default value.") if self.group: fail("A 'self' parameter cannot be in an optional group.") + assert self.parameter_state is ParamState.REQUIRED + assert value is unspecified kind = inspect.Parameter.POSITIONAL_ONLY self.parameter_state = ParamState.START self.function.parameters.clear() @@ -1112,14 +1061,12 @@ def bad_node(self, node: ast.AST) -> None: if isinstance(converter, defining_class_converter): _lp = len(self.function.parameters) if _lp == 1: - if self.parameter_state is not ParamState.REQUIRED: - fail("A 'defining_class' parameter cannot be marked optional.") - if value is not unspecified: - fail("A 'defining_class' parameter cannot have a default value.") if self.group: fail("A 'defining_class' parameter cannot be in an optional group.") if self.function.cls is None: fail("A 'defining_class' parameter cannot be defined at module level.") + assert self.parameter_state is ParamState.REQUIRED + assert value is unspecified kind = inspect.Parameter.POSITIONAL_ONLY else: fail("A 'defining_class' parameter, if specified, must either " diff --git a/Tools/clinic/libclinic/formatting.py b/Tools/clinic/libclinic/formatting.py index 873ece6210017a8..264327818c1d193 100644 --- a/Tools/clinic/libclinic/formatting.py +++ b/Tools/clinic/libclinic/formatting.py @@ -39,8 +39,55 @@ def _quoted_for_c_string(text: str) -> str: return text -def c_repr(text: str) -> str: - return '"' + text + '"' +# Use octals, because \x... in C has arbitrary number of hexadecimal digits. +_c_repr = [chr(i) if 32 <= i < 127 else fr'\{i:03o}' for i in range(256)] +_c_repr[ord('"')] = r'\"' +_c_repr[ord('\\')] = r'\\' +_c_repr[ord('\a')] = r'\a' +_c_repr[ord('\b')] = r'\b' +_c_repr[ord('\f')] = r'\f' +_c_repr[ord('\n')] = r'\n' +_c_repr[ord('\r')] = r'\r' +_c_repr[ord('\t')] = r'\t' +_c_repr[ord('\v')] = r'\v' + +def _break_trigraphs(s: str) -> str: + # Prevent trigraphs from being interpreted inside string literals. + if '??' in s: + s = s.replace('??', r'?\?') + s = s.replace(r'\??', r'\?\?') + # Also Argument Clinic does not like comment-like sequences + # in string literals. + s = s.replace(r'/*', r'/\*') + s = s.replace(r'*/', r'*\/') + return s + +def c_bytes_repr(data: bytes) -> str: + r = ''.join(_c_repr[i] for i in data) + r = _break_trigraphs(r) + return '"' + r + '"' + +def c_str_repr(text: str) -> str: + r = ''.join(_c_repr[i] if i < 0x80 + else fr'\u{i:04x}' if i < 0x10000 + else fr'\U{i:08x}' + for i in map(ord, text)) + r = _break_trigraphs(r) + return '"' + r + '"' + +def c_unichar_repr(char: str) -> str: + if char == "'": + return r"'\''" + if char == '"': + return """'"'""" + if char == '\0': + return '0' + i = ord(char) + if i < 0x80: + r = _c_repr[i] + if not r.startswith((r'\0', r'\1')): + return "'" + r + "'" + return f'0x{i:02x}' def wrapped_c_string_literal( @@ -58,8 +105,8 @@ def wrapped_c_string_literal( drop_whitespace=False, break_on_hyphens=False, ) - separator = c_repr(suffix + "\n" + subsequent_indent * " ") - return initial_indent * " " + c_repr(separator.join(wrapped)) + separator = '"' + suffix + "\n" + subsequent_indent * " " + '"' + return initial_indent * " " + '"' + separator.join(wrapped) + '"' def _add_prefix_and_suffix(text: str, *, prefix: str = "", suffix: str = "") -> str: diff --git a/Tools/clinic/libclinic/utils.py b/Tools/clinic/libclinic/utils.py index 17e8f35be73bf42..3df64f270dd074a 100644 --- a/Tools/clinic/libclinic/utils.py +++ b/Tools/clinic/libclinic/utils.py @@ -85,9 +85,9 @@ def __repr__(self) -> str: # This one needs to be a distinct class, unlike the other two -class Null: +class NullType: def __repr__(self) -> str: return '<Null>' -NULL = Null() +NULL = NullType() diff --git a/Tools/ftscalingbench/ftscalingbench.py b/Tools/ftscalingbench/ftscalingbench.py index 926bc66b944c6ff..955d06d4c04c4c1 100644 --- a/Tools/ftscalingbench/ftscalingbench.py +++ b/Tools/ftscalingbench/ftscalingbench.py @@ -27,6 +27,7 @@ import sys import threading import time +from dataclasses import dataclass # The iterations in individual benchmarks are scaled by this factor. WORK_SCALE = 100 @@ -189,6 +190,65 @@ def thread_local_read(): _ = tmp.x +@dataclass +class MyDataClass: + x: int + y: int + z: int + +@register_benchmark +def instantiate_dataclass(): + for _ in range(1000 * WORK_SCALE): + obj = MyDataClass(x=1, y=2, z=3) + +@register_benchmark +def super_call(): + # TODO: super() on the same class from multiple threads still doesn't + # scale well, so use a class per-thread here for now. + class Base: + def method(self): + return 1 + + class Derived(Base): + def method(self): + return super().method() + + obj = Derived() + for _ in range(1000 * WORK_SCALE): + obj.method() + + +class MyClassMethod: + @classmethod + def my_classmethod(cls): + return cls + + @staticmethod + def my_staticmethod(): + pass + +@register_benchmark +def classmethod_call(): + obj = MyClassMethod() + for _ in range(1000 * WORK_SCALE): + obj.my_classmethod() + +@register_benchmark +def staticmethod_call(): + obj = MyClassMethod() + for _ in range(1000 * WORK_SCALE): + obj.my_staticmethod() + +@register_benchmark +def setattr_non_interned(): + prefix = "prefix" + obj = MyObject() + for _ in range(1000 * WORK_SCALE): + setattr(obj, f"{prefix}_a", None) + setattr(obj, f"{prefix}_b", None) + setattr(obj, f"{prefix}_c", None) + + def bench_one_thread(func): t0 = time.perf_counter_ns() func() diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 27aa6b0cc266d39..f765d96d9cb7a97 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -43,7 +43,6 @@ import gdb import os -import locale import sys @@ -107,8 +106,6 @@ def interp_frame_has_tlbc_index(): USED_TAGS = 0b11 -ENCODING = locale.getpreferredencoding() - FRAME_INFO_OPTIMIZED_OUT = '(frame information optimized out)' UNABLE_READ_INFO_PYTHON_FRAME = 'Unable to read information on python frame' EVALFRAME = '_PyEval_EvalFrameDefault' @@ -152,6 +149,11 @@ def write(self, data): def getvalue(self): return self._val + +def _PyStackRef_AsPyObjectBorrow(gdbval): + return gdb.Value(int(gdbval['bits']) & ~USED_TAGS) + + class PyObjectPtr(object): """ Class wrapping a gdb.Value that's either a (PyObject*) within the @@ -170,7 +172,7 @@ def __init__(self, gdbval, cast_to=None): if gdbval.type.name == '_PyStackRef': if cast_to is None: cast_to = gdb.lookup_type('PyObject').pointer() - self._gdbval = gdb.Value(int(gdbval['bits']) & ~USED_TAGS).cast(cast_to) + self._gdbval = _PyStackRef_AsPyObjectBorrow(gdbval).cast(cast_to) elif cast_to: self._gdbval = gdbval.cast(cast_to) else: @@ -1034,30 +1036,49 @@ def write_repr(self, out, visited): return return self._frame.write_repr(out, visited) - def print_traceback(self): - if self.is_optimized_out(): - sys.stdout.write(' %s\n' % FRAME_INFO_OPTIMIZED_OUT) - return - return self._frame.print_traceback() - class PyFramePtr: def __init__(self, gdbval): self._gdbval = gdbval + if self.is_optimized_out(): + return + self.co = self._f_code() + if self.is_shim(): + return + self.co_name = self.co.pyop_field('co_name') + self.co_filename = self.co.pyop_field('co_filename') - if not self.is_optimized_out(): + self.f_lasti = self._f_lasti() + self.co_nlocals = int_from_int(self.co.field('co_nlocals')) + pnames = self.co.field('co_localsplusnames') + self.co_localsplusnames = PyTupleObjectPtr.from_pyobject_ptr(pnames) + + @staticmethod + def get_thread_state(): + exprs = [ + '_Py_tss_gilstate', # 3.15+ + '_Py_tss_tstate', # 3.12+ (and not when GIL is released) + 'pthread_getspecific(_PyRuntime.autoTSSkey._key)', # only live programs + '((struct pthread*)$fs_base)->specific_1stblock[_PyRuntime.autoTSSkey._key].data' # x86-64 + ] + for expr in exprs: try: - self.co = self._f_code() - self.co_name = self.co.pyop_field('co_name') - self.co_filename = self.co.pyop_field('co_filename') - - self.f_lasti = self._f_lasti() - self.co_nlocals = int_from_int(self.co.field('co_nlocals')) - pnames = self.co.field('co_localsplusnames') - self.co_localsplusnames = PyTupleObjectPtr.from_pyobject_ptr(pnames) - self._is_code = True - except: - self._is_code = False + val = gdb.parse_and_eval(f'(PyThreadState*)({expr})') + except gdb.error: + continue + if int(val) != 0: + return val + return None + + @staticmethod + def get_thread_local_frame(): + thread_state = PyFramePtr.get_thread_state() + if thread_state is None: + return None + current_frame = thread_state['current_frame'] + if int(current_frame) == 0: + return None + return PyFramePtr(current_frame) def is_optimized_out(self): return self._gdbval.is_optimized_out @@ -1115,6 +1136,8 @@ def is_shim(self): return self._f_special("owner", int) == FRAME_OWNED_BY_INTERPRETER def previous(self): + if int(self._gdbval['previous']) == 0: + return None return self._f_special("previous", PyFramePtr) def iter_globals(self): @@ -1243,6 +1266,27 @@ def print_traceback(self): lineno, self.co_name.proxyval(visited))) + def print_traceback_until_shim(self, frame_index=None): + # Print traceback for _PyInterpreterFrame and return previous frame + interp_frame = self + while True: + if not interp_frame: + sys.stdout.write(' (unable to read python frame information)\n') + return None + if interp_frame.is_shim(): + return interp_frame.previous() + + if frame_index is not None: + line = interp_frame.get_truncated_repr(MAX_OUTPUT_LEN) + sys.stdout.write('#%i %s\n' % (frame_index, line)) + else: + interp_frame.print_traceback() + if not interp_frame.is_optimized_out(): + line = interp_frame.current_line() + if line is not None: + sys.stdout.write(' %s\n' % line.strip()) + interp_frame = interp_frame.previous() + def get_truncated_repr(self, maxlen): ''' Get a repr-like string for the data, but truncate it at "maxlen" bytes @@ -1446,6 +1490,10 @@ def proxyval(self, visited): def write_repr(self, out, visited): # Write this out as a Python str literal + # gdb writes its output in the host charset, so a character is escaped + # unless it is printable and encodable in that charset. + encoding = gdb.host_charset() + # Get a PyUnicodeObject* within the Python gdb process: proxy = self.proxyval(visited) @@ -1493,8 +1541,10 @@ def write_repr(self, out, visited): printable = ucs.isprintable() if printable: try: - ucs.encode(ENCODING) - except UnicodeEncodeError: + ucs.encode(encoding) + # LookupError or ValueError if the host charset is unknown + # or invalid. + except (UnicodeEncodeError, LookupError, ValueError): printable = False # Map Unicode whitespace and control characters @@ -1855,20 +1905,10 @@ def get_selected_bytecode_frame(cls): def print_summary(self): if self.is_evalframe(): interp_frame = self.get_pyop() - while True: - if interp_frame: - if interp_frame.is_shim(): - break - line = interp_frame.get_truncated_repr(MAX_OUTPUT_LEN) - sys.stdout.write('#%i %s\n' % (self.get_index(), line)) - if not interp_frame.is_optimized_out(): - line = interp_frame.current_line() - if line is not None: - sys.stdout.write(' %s\n' % line.strip()) - else: - sys.stdout.write('#%i (unable to read python frame information)\n' % self.get_index()) - break - interp_frame = interp_frame.previous() + if interp_frame: + interp_frame.print_traceback_until_shim(self.get_index()) + else: + sys.stdout.write('#%i (unable to read python frame information)\n' % self.get_index()) else: info = self.is_other_python_frame() if info: @@ -1876,29 +1916,6 @@ def print_summary(self): else: sys.stdout.write('#%i\n' % self.get_index()) - def print_traceback(self): - if self.is_evalframe(): - interp_frame = self.get_pyop() - while True: - if interp_frame: - if interp_frame.is_shim(): - break - interp_frame.print_traceback() - if not interp_frame.is_optimized_out(): - line = interp_frame.current_line() - if line is not None: - sys.stdout.write(' %s\n' % line.strip()) - else: - sys.stdout.write(' (unable to read python frame information)\n') - break - interp_frame = interp_frame.previous() - else: - info = self.is_other_python_frame() - if info: - sys.stdout.write(' %s\n' % info) - else: - sys.stdout.write(' (not a python frame)\n') - class PyList(gdb.Command): '''List the current Python source code, if any @@ -2042,6 +2059,41 @@ def invoke(self, args, from_tty): PyUp() PyDown() + +def print_traceback_helper(full_info): + frame = Frame.get_selected_python_frame() + interp_frame = PyFramePtr.get_thread_local_frame() + if not frame and not interp_frame: + print('Unable to locate python frame') + return + + sys.stdout.write('Traceback (most recent call first):\n') + if frame: + while frame: + frame_index = frame.get_index() if full_info else None + if frame.is_evalframe(): + pyop = frame.get_pyop() + if pyop is not None: + # Use the _PyInterpreterFrame from the gdb frame + interp_frame = pyop + if interp_frame: + interp_frame = interp_frame.print_traceback_until_shim(frame_index) + else: + sys.stdout.write(' (unable to read python frame information)\n') + else: + info = frame.is_other_python_frame() + if full_info: + if info: + sys.stdout.write('#%i %s\n' % (frame_index, info)) + elif info: + sys.stdout.write(' %s\n' % info) + frame = frame.older() + else: + # Fall back to just using the thread-local frame + while interp_frame: + interp_frame = interp_frame.print_traceback_until_shim() + + class PyBacktraceFull(gdb.Command): 'Display the current python frame and all the frames within its call stack (if any)' def __init__(self): @@ -2052,15 +2104,7 @@ def __init__(self): def invoke(self, args, from_tty): - frame = Frame.get_selected_python_frame() - if not frame: - print('Unable to locate python frame') - return - - while frame: - if frame.is_python_frame(): - frame.print_summary() - frame = frame.older() + print_traceback_helper(full_info=True) PyBacktraceFull() @@ -2072,18 +2116,8 @@ def __init__(self): gdb.COMMAND_STACK, gdb.COMPLETE_NONE) - def invoke(self, args, from_tty): - frame = Frame.get_selected_python_frame() - if not frame: - print('Unable to locate python frame') - return - - sys.stdout.write('Traceback (most recent call first):\n') - while frame: - if frame.is_python_frame(): - frame.print_traceback() - frame = frame.older() + print_traceback_helper(full_info=False) PyBacktrace() diff --git a/Tools/i18n/.ruff.toml b/Tools/i18n/.ruff.toml new file mode 100644 index 000000000000000..a8f4f2f5a96d7b8 --- /dev/null +++ b/Tools/i18n/.ruff.toml @@ -0,0 +1,10 @@ +extend = "../../.ruff.toml" # Inherit the project-wide settings + +target-version = "py313" + +[lint] +select = [ + "F", # pyflakes + "I", # isort + "UP", # pyupgrade +] diff --git a/Tools/i18n/makelocalealias.py b/Tools/i18n/makelocalealias.py index b407a8a643be7c0..085da1c63b701c3 100755 --- a/Tools/i18n/makelocalealias.py +++ b/Tools/i18n/makelocalealias.py @@ -8,6 +8,7 @@ """ import locale import sys + _locale = locale # Location of the X11 alias file. @@ -89,16 +90,15 @@ def parse_glibc_supported(filename): def pprint(data): items = sorted(data.items()) for k, v in items: - print(' %-40s%a,' % ('%a:' % k, v)) + print(f" {k!a:<40}{v!a},") def print_differences(data, olddata): items = sorted(olddata.items()) for k, v in items: if k not in data: - print('# removed %a' % k) + print(f'# removed {k!a}') elif olddata[k] != data[k]: - print('# updated %a -> %a to %a' % \ - (k, olddata[k], data[k])) + print(f'# updated {k!a} -> {olddata[k]!a} to {data[k]!a}') # Additions are not mentioned def optimize(data): @@ -121,7 +121,7 @@ def check(data): errors = 0 for k, v in data.items(): if locale.normalize(k) != v: - print('ERROR: %a -> %a != %a' % (k, locale.normalize(k), v), + print(f'ERROR: {k!a} -> {locale.normalize(k)!a} != {v!a}', file=sys.stderr) errors += 1 return errors @@ -131,15 +131,18 @@ def check(data): parser = argparse.ArgumentParser() parser.add_argument('--locale-alias', default=LOCALE_ALIAS, help='location of the X11 alias file ' - '(default: %a)' % LOCALE_ALIAS) + f'(default: {LOCALE_ALIAS})') parser.add_argument('--glibc-supported', default=SUPPORTED, help='location of the glibc SUPPORTED locales file ' - '(default: %a)' % SUPPORTED) + f'(default: {SUPPORTED})') args = parser.parse_args() data = locale.locale_alias.copy() data.update(parse_glibc_supported(args.glibc_supported)) data.update(parse(args.locale_alias)) + # Hardcode 'c.utf8' -> 'C.UTF-8' because 'en_US.UTF-8' does not exist + # on all platforms. + data['c.utf8'] = 'C.UTF-8' while True: # Repeat optimization while the size is decreased. n = len(data) diff --git a/Tools/i18n/msgfmt.py b/Tools/i18n/msgfmt.py index cd5f1ed9f3e2683..29b41b4c3c63113 100755 --- a/Tools/i18n/msgfmt.py +++ b/Tools/i18n/msgfmt.py @@ -25,14 +25,14 @@ Display version information and exit. """ -import os -import sys +import array import ast +import codecs import getopt +import os import struct -import array +import sys from email.parser import HeaderParser -import codecs __version__ = "1.2" @@ -114,7 +114,7 @@ def make(filename, outfile): try: with open(infile, 'rb') as f: lines = f.readlines() - except IOError as msg: + except OSError as msg: print(msg, file=sys.stderr) sys.exit(1) @@ -127,6 +127,7 @@ def make(filename, outfile): sys.exit(1) section = msgctxt = None + msgid = msgstr = b'' fuzzy = 0 # Start off assuming Latin-1, so everything decodes without failure, @@ -178,7 +179,7 @@ def make(filename, outfile): # This is a message with plural forms elif l.startswith('msgid_plural'): if section != ID: - print('msgid_plural not preceded by msgid on %s:%d' % (infile, lno), + print(f'msgid_plural not preceded by msgid on {infile}:{lno}', file=sys.stderr) sys.exit(1) l = l[12:] @@ -189,7 +190,7 @@ def make(filename, outfile): section = STR if l.startswith('msgstr['): if not is_plural: - print('plural without msgid_plural on %s:%d' % (infile, lno), + print(f'plural without msgid_plural on {infile}:{lno}', file=sys.stderr) sys.exit(1) l = l.split(']', 1)[1] @@ -197,7 +198,7 @@ def make(filename, outfile): msgstr += b'\0' # Separator of the various plural forms else: if is_plural: - print('indexed msgstr required for plural on %s:%d' % (infile, lno), + print(f'indexed msgstr required for plural on {infile}:{lno}', file=sys.stderr) sys.exit(1) l = l[6:] @@ -213,8 +214,7 @@ def make(filename, outfile): elif section == STR: msgstr += l.encode(encoding) else: - print('Syntax error on %s:%d' % (infile, lno), \ - 'before:', file=sys.stderr) + print(f'Syntax error on {infile}:{lno} before:', file=sys.stderr) print(l, file=sys.stderr) sys.exit(1) # Add last entry @@ -227,7 +227,7 @@ def make(filename, outfile): try: with open(outfile,"wb") as f: f.write(output) - except IOError as msg: + except OSError as msg: print(msg, file=sys.stderr) diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index f46b05067d7fde8..ddf4474d2bce55f 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -193,7 +193,7 @@ def make_escapes(pass_nonascii): escape = escape_ascii else: escape = escape_nonascii - escapes = [r"\%03o" % i for i in range(256)] + escapes = [fr"\{i:03o}" for i in range(256)] for i in range(32, 127): escapes[i] = chr(i) escapes[ord('\\')] = r'\\' @@ -796,7 +796,7 @@ class Options: try: with open(options.excludefilename) as fp: options.toexclude = fp.readlines() - except IOError: + except OSError: print(f"Can't read --exclude-file: {options.excludefilename}", file=sys.stderr) sys.exit(1) diff --git a/Tools/jit/_llvm.py b/Tools/jit/_llvm.py index f09a8404871b24b..79b6b7ec53005ef 100644 --- a/Tools/jit/_llvm.py +++ b/Tools/jit/_llvm.py @@ -11,7 +11,9 @@ import _targets _LLVM_VERSION = 19 -_LLVM_VERSION_PATTERN = re.compile(rf"version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+") +_LLVM_VERSION_PATTERN = re.compile( + rf"(?<!Apple )(LLVM|clang) version\s+{_LLVM_VERSION}\.\d+\.\d+\S*\s+" +) _EXTERNALS_LLVM_TAG = "llvm-19.1.7.0" _P = typing.ParamSpec("_P") diff --git a/Tools/jit/_stencils.py b/Tools/jit/_stencils.py index 03b0ba647b0db77..6059b34c8d75202 100644 --- a/Tools/jit/_stencils.py +++ b/Tools/jit/_stencils.py @@ -55,8 +55,8 @@ class HoleValue(enum.Enum): _PATCH_FUNCS = { # aarch64-apple-darwin: "ARM64_RELOC_BRANCH26": "patch_aarch64_26r", - "ARM64_RELOC_GOT_LOAD_PAGE21": "patch_aarch64_21rx", - "ARM64_RELOC_GOT_LOAD_PAGEOFF12": "patch_aarch64_12x", + "ARM64_RELOC_GOT_LOAD_PAGE21": "patch_aarch64_21r", + "ARM64_RELOC_GOT_LOAD_PAGEOFF12": "patch_aarch64_12", "ARM64_RELOC_PAGE21": "patch_aarch64_21r", "ARM64_RELOC_PAGEOFF12": "patch_aarch64_12", "ARM64_RELOC_UNSIGNED": "patch_64", @@ -64,20 +64,20 @@ class HoleValue(enum.Enum): "IMAGE_REL_AMD64_REL32": "patch_x86_64_32rx", # aarch64-pc-windows-msvc: "IMAGE_REL_ARM64_BRANCH26": "patch_aarch64_26r", - "IMAGE_REL_ARM64_PAGEBASE_REL21": "patch_aarch64_21rx", + "IMAGE_REL_ARM64_PAGEBASE_REL21": "patch_aarch64_21r", "IMAGE_REL_ARM64_PAGEOFFSET_12A": "patch_aarch64_12", - "IMAGE_REL_ARM64_PAGEOFFSET_12L": "patch_aarch64_12x", + "IMAGE_REL_ARM64_PAGEOFFSET_12L": "patch_aarch64_12", # i686-pc-windows-msvc: "IMAGE_REL_I386_DIR32": "patch_32", "IMAGE_REL_I386_REL32": "patch_x86_64_32rx", # aarch64-unknown-linux-gnu: "R_AARCH64_ABS64": "patch_64", "R_AARCH64_ADD_ABS_LO12_NC": "patch_aarch64_12", - "R_AARCH64_ADR_GOT_PAGE": "patch_aarch64_21rx", + "R_AARCH64_ADR_GOT_PAGE": "patch_aarch64_21r", "R_AARCH64_ADR_PREL_PG_HI21": "patch_aarch64_21r", "R_AARCH64_CALL26": "patch_aarch64_26r", "R_AARCH64_JUMP26": "patch_aarch64_26r", - "R_AARCH64_LD64_GOT_LO12_NC": "patch_aarch64_12x", + "R_AARCH64_LD64_GOT_LO12_NC": "patch_aarch64_12", "R_AARCH64_MOVW_UABS_G0_NC": "patch_aarch64_16a", "R_AARCH64_MOVW_UABS_G1_NC": "patch_aarch64_16b", "R_AARCH64_MOVW_UABS_G2_NC": "patch_aarch64_16c", @@ -140,38 +140,6 @@ class Hole: def __post_init__(self) -> None: self.func = _PATCH_FUNCS[self.kind] - def fold( - self, - other: typing.Self, - body: bytes | bytearray, - ) -> typing.Self | None: - """Combine two holes into a single hole, if possible.""" - instruction_a = int.from_bytes( - body[self.offset : self.offset + 4], byteorder=sys.byteorder - ) - instruction_b = int.from_bytes( - body[other.offset : other.offset + 4], byteorder=sys.byteorder - ) - reg_a = instruction_a & 0b11111 - reg_b1 = instruction_b & 0b11111 - reg_b2 = (instruction_b >> 5) & 0b11111 - - if ( - self.offset + 4 == other.offset - and self.value == other.value - and self.symbol == other.symbol - and self.addend == other.addend - and self.func == "patch_aarch64_21rx" - and other.func == "patch_aarch64_12x" - and reg_a == reg_b1 == reg_b2 - ): - # These can *only* be properly relaxed when they appear together and - # patch the same value: - folded = self.replace() - folded.func = "patch_aarch64_33rx" - return folded - return None - def as_c(self, where: str) -> str: """Dump this hole as a call to a patch_* function.""" location = f"{where} + {self.offset:#x}" diff --git a/Tools/jit/_targets.py b/Tools/jit/_targets.py index 6ceb4404e74ce7e..f1085cc9bf081de 100644 --- a/Tools/jit/_targets.py +++ b/Tools/jit/_targets.py @@ -10,6 +10,7 @@ import sys import tempfile import typing +import shlex import _llvm import _schema @@ -46,7 +47,9 @@ class _Target(typing.Generic[_S, _R]): stable: bool = False debug: bool = False verbose: bool = False + cflags: str = "" known_symbols: dict[str, int] = dataclasses.field(default_factory=dict) + pyconfig_dir: pathlib.Path = pathlib.Path.cwd().resolve() def _get_nop(self) -> bytes: if re.fullmatch(r"aarch64-.*", self.triple): @@ -57,15 +60,19 @@ def _get_nop(self) -> bytes: raise ValueError(f"NOP not defined for {self.triple}") return nop - def _compute_digest(self, out: pathlib.Path) -> str: + def _compute_digest(self) -> str: hasher = hashlib.sha256() hasher.update(self.triple.encode()) hasher.update(self.debug.to_bytes()) + hasher.update(self.cflags.encode()) # These dependencies are also reflected in _JITSources in regen.targets: hasher.update(PYTHON_EXECUTOR_CASES_C_H.read_bytes()) - hasher.update((out / "pyconfig.h").read_bytes()) + hasher.update((self.pyconfig_dir / "pyconfig.h").read_bytes()) for dirpath, _, filenames in sorted(os.walk(TOOLS_JIT)): - for filename in filenames: + # Exclude cache files from digest computation to ensure reproducible builds. + if dirpath.endswith("__pycache__"): + continue + for filename in sorted(filenames): hasher.update(pathlib.Path(dirpath, filename).read_bytes()) return hasher.hexdigest() @@ -110,7 +117,7 @@ def _handle_section(self, section: _S, group: _stencils.StencilGroup) -> None: raise NotImplementedError(type(self)) def _handle_relocation( - self, base: int, relocation: _R, raw: bytes | bytearray + self, base: int, relocation: _R, raw: bytearray ) -> _stencils.Hole: raise NotImplementedError(type(self)) @@ -125,7 +132,7 @@ async def _compile( f"-D_JIT_OPCODE={opname}", "-D_PyJIT_ACTIVE", "-D_Py_JIT", - "-I.", + f"-I{self.pyconfig_dir}", f"-I{CPYTHON / 'Include'}", f"-I{CPYTHON / 'Include' / 'internal'}", f"-I{CPYTHON / 'Include' / 'internal' / 'mimalloc'}", @@ -154,6 +161,8 @@ async def _compile( f"{o}", f"{c}", *self.args, + # Allow user-provided CFLAGS to override any defaults + *shlex.split(self.cflags), ] await _llvm.run("clang", args, echo=self.verbose) return await self._parse(o) @@ -193,20 +202,19 @@ async def _build_stencils(self) -> dict[str, _stencils.StencilGroup]: def build( self, - out: pathlib.Path, *, comment: str = "", force: bool = False, - stencils_h: str = "jit_stencils.h", + jit_stencils: pathlib.Path, ) -> None: """Build jit_stencils.h in the given directory.""" + jit_stencils.parent.mkdir(parents=True, exist_ok=True) if not self.stable: warning = f"JIT support for {self.triple} is still experimental!" request = "Please report any issues you encounter.".center(len(warning)) outline = "=" * len(warning) print("\n".join(["", outline, warning, request, outline, ""])) - digest = f"// {self._compute_digest(out)}\n" - jit_stencils = out / stencils_h + digest = f"// {self._compute_digest()}\n" if ( not force and jit_stencils.exists() @@ -214,7 +222,7 @@ def build( ): return stencil_groups = ASYNCIO_RUNNER.run(self._build_stencils()) - jit_stencils_new = out / "jit_stencils.h.new" + jit_stencils_new = jit_stencils.parent / "jit_stencils.h.new" try: with jit_stencils_new.open("w") as file: file.write(digest) @@ -277,10 +285,7 @@ def _unwrap_dllimport(self, name: str) -> tuple[_stencils.HoleValue, str | None] return _stencils.symbol_to_value(name) def _handle_relocation( - self, - base: int, - relocation: _schema.COFFRelocation, - raw: bytes | bytearray, + self, base: int, relocation: _schema.COFFRelocation, raw: bytearray ) -> _stencils.Hole: match relocation: case { @@ -375,10 +380,7 @@ def _handle_section( }, section_type def _handle_relocation( - self, - base: int, - relocation: _schema.ELFRelocation, - raw: bytes | bytearray, + self, base: int, relocation: _schema.ELFRelocation, raw: bytearray ) -> _stencils.Hole: symbol: str | None match relocation: @@ -454,10 +456,7 @@ def _handle_section( stencil.holes.append(hole) def _handle_relocation( - self, - base: int, - relocation: _schema.MachORelocation, - raw: bytes | bytearray, + self, base: int, relocation: _schema.MachORelocation, raw: bytearray ) -> _stencils.Hole: symbol: str | None match relocation: diff --git a/Tools/jit/_writer.py b/Tools/jit/_writer.py index 090b52660f009c0..a7f3d3965827d3c 100644 --- a/Tools/jit/_writer.py +++ b/Tools/jit/_writer.py @@ -1,6 +1,5 @@ """Utilities for writing StencilGroups out to a C header file.""" -import itertools import typing import math @@ -60,15 +59,8 @@ def _dump_stencil(opname: str, group: _stencils.StencilGroup) -> typing.Iterator for part, stencil in [("data", group.data), ("code", group.code)]: if stencil.body.rstrip(b"\x00"): yield f" memcpy({part}, {part}_body, sizeof({part}_body));" - skip = False stencil.holes.sort(key=lambda hole: hole.offset) - for hole, pair in itertools.zip_longest(stencil.holes, stencil.holes[1:]): - if skip: - skip = False - continue - if pair and (folded := hole.fold(pair, stencil.body)): - skip = True - hole = folded + for hole in stencil.holes: yield f" {hole.as_c(part)}" yield "}" yield "" diff --git a/Tools/jit/build.py b/Tools/jit/build.py index 49b08f477dbed7b..a0733005929bf24 100644 --- a/Tools/jit/build.py +++ b/Tools/jit/build.py @@ -8,7 +8,6 @@ import _targets if __name__ == "__main__": - out = pathlib.Path.cwd().resolve() comment = f"$ {shlex.join([pathlib.Path(sys.executable).name] + sys.argv)}" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -23,21 +22,39 @@ parser.add_argument( "-f", "--force", action="store_true", help="force the entire JIT to be rebuilt" ) + parser.add_argument( + "-o", + "--output-dir", + help="where to output generated files", + required=True, + type=lambda p: pathlib.Path(p).resolve(), + ) + parser.add_argument( + "-p", + "--pyconfig-dir", + help="where to find pyconfig.h", + required=True, + type=lambda p: pathlib.Path(p).resolve(), + ) parser.add_argument( "-v", "--verbose", action="store_true", help="echo commands as they are run" ) + parser.add_argument( + "--cflags", help="additional flags to pass to the compiler", default="" + ) args = parser.parse_args() for target in args.target: target.debug = args.debug target.force = args.force target.verbose = args.verbose + target.cflags = args.cflags + target.pyconfig_dir = args.pyconfig_dir target.build( - out, comment=comment, - stencils_h=f"jit_stencils-{target.triple}.h", force=args.force, + jit_stencils=args.output_dir / f"jit_stencils-{target.triple}.h", ) - jit_stencils_h = out / "jit_stencils.h" + jit_stencils_h = args.output_dir / "jit_stencils.h" lines = [f"// {comment}\n"] guard = "#if" for target in args.target: diff --git a/Tools/jit/template.c b/Tools/jit/template.c index 5ee26f93f1e2666..d042699680c639c 100644 --- a/Tools/jit/template.c +++ b/Tools/jit/template.c @@ -70,9 +70,11 @@ do { \ } while (0) #undef LLTRACE_RESUME_FRAME -#define LLTRACE_RESUME_FRAME() \ - do { \ - } while (0) +#ifdef Py_DEBUG +#define LLTRACE_RESUME_FRAME() (frame->lltrace = 0) +#else +#define LLTRACE_RESUME_FRAME() do {} while (0) +#endif #define PATCH_JUMP(ALIAS) \ do { \ diff --git a/Tools/msi/bundle/Default.wxl b/Tools/msi/bundle/Default.wxl index 0bd3644b58b7476..335c1d922d97b52 100644 --- a/Tools/msi/bundle/Default.wxl +++ b/Tools/msi/bundle/Default.wxl @@ -92,7 +92,7 @@ Select Customize to review current options.</String> <String Id="PrecompileLabel">&amp;Precompile standard library</String> <String Id="Include_symbolsLabel">Download debugging &amp;symbols</String> <String Id="Include_debugLabel">Download debu&amp;g binaries (requires VS 2017 or later)</String> - <String Id="Include_freethreadedLabel">Download &amp;free-threaded binaries (experimental)</String> + <String Id="Include_freethreadedLabel">Download &amp;free-threaded binaries</String> <String Id="ProgressHeader">[ActionLikeInstallation] Progress</String> <String Id="ProgressLabel">[ActionLikeInstalling]:</String> diff --git a/Tools/msi/bundle/bundle.wxs b/Tools/msi/bundle/bundle.wxs index abfeb88784890c8..3fcb00553f5edd3 100644 --- a/Tools/msi/bundle/bundle.wxs +++ b/Tools/msi/bundle/bundle.wxs @@ -106,11 +106,11 @@ <Variable Name="SimpleInstallDescription" Value="" bal:Overridable="yes" /> <Chain ParallelCache="yes"> + <PackageGroupRef Id="core" /> + <PackageGroupRef Id="exe" /> <?if $(var.Platform)!="ARM64" ?> <PackageGroupRef Id="crt" /> <?endif ?> - <PackageGroupRef Id="core" /> - <PackageGroupRef Id="exe" /> <PackageGroupRef Id="dev" /> <PackageGroupRef Id="lib" /> <?if $(var.IncludeFreethreaded)~="true" ?> diff --git a/Tools/msi/dev/dev_files.wxs b/Tools/msi/dev/dev_files.wxs index 4357dc86d9d3564..21f9c848cc6be58 100644 --- a/Tools/msi/dev/dev_files.wxs +++ b/Tools/msi/dev/dev_files.wxs @@ -3,7 +3,7 @@ <Fragment> <ComponentGroup Id="dev_pyconfig"> <Component Id="include_pyconfig.h" Directory="include" Guid="*"> - <File Id="include_pyconfig.h" Name="pyconfig.h" Source="pyconfig.h" KeyPath="yes" /> + <File Id="include_pyconfig.h" Name="pyconfig.h" Source="!(bindpath.src)PC\pyconfig.h" KeyPath="yes" /> </Component> </ComponentGroup> </Fragment> diff --git a/Tools/msi/freethreaded/freethreaded_files.wxs b/Tools/msi/freethreaded/freethreaded_files.wxs index 86d9a8b83f65350..0707e77b5e9ab2e 100644 --- a/Tools/msi/freethreaded/freethreaded_files.wxs +++ b/Tools/msi/freethreaded/freethreaded_files.wxs @@ -103,7 +103,7 @@ </ComponentGroup> </Fragment> - <?define exts=pyexpat;select;unicodedata;winsound;_bz2;_elementtree;_socket;_ssl;_ctypes;_hashlib;_multiprocessing;_lzma;_decimal;_overlapped;_sqlite3;_asyncio;_queue;_uuid;_wmi;_zoneinfo;_zstd;_testcapi;_ctypes_test;_testbuffer;_testimportmultiple;_testmultiphase;_testsinglephase;_testconsole;_testinternalcapi;_testclinic;_testclinic_limited;_tkinter ?> + <?define exts=pyexpat;select;unicodedata;winsound;_bz2;_elementtree;_socket;_ssl;_ctypes;_hashlib;_multiprocessing;_lzma;_decimal;_overlapped;_sqlite3;_asyncio;_queue;_remote_debugging;_uuid;_wmi;_zoneinfo;_zstd;_testcapi;_ctypes_test;_testbuffer;_testimportmultiple;_testmultiphase;_testsinglephase;_testconsole;_testinternalcapi;_testclinic;_testclinic_limited;_tkinter ?> <Fragment> <DirectoryRef Id="Lib_venv_scripts_nt__freethreaded" /> diff --git a/Tools/msi/get_externals.bat b/Tools/msi/get_externals.bat index f6602ce9588ff41..c7c7e9f470bc250 100644 --- a/Tools/msi/get_externals.bat +++ b/Tools/msi/get_externals.bat @@ -6,7 +6,7 @@ set HERE=%~dp0 if "%PCBUILD%"=="" (set PCBUILD=%HERE%..\..\PCbuild\) if "%EXTERNALS_DIR%"=="" (set EXTERNALS_DIR=%HERE%..\..\externals\windows-installer) if "%NUGET%"=="" (set NUGET=%EXTERNALS_DIR%\..\nuget.exe) -if "%NUGET_URL%"=="" (set NUGET_URL=https://aka.ms/nugetclidl) +if "%NUGET_URL%"=="" (set NUGET_URL=https://dist.nuget.org/win-x86-commandline/latest/nuget.exe) set DO_FETCH=true set DO_CLEAN=false diff --git a/Tools/msi/lib/lib.wixproj b/Tools/msi/lib/lib.wixproj index 02078e503d74a4d..3ea46dd40ea4ce9 100644 --- a/Tools/msi/lib/lib.wixproj +++ b/Tools/msi/lib/lib.wixproj @@ -15,12 +15,11 @@ <EmbeddedResource Include="*.wxl" /> </ItemGroup> <ItemGroup> - <ExcludeFolders Include="Lib\test;Lib\tests;Lib\tkinter;Lib\idlelib;Lib\turtledemo" /> + <ExcludeFolders Include="Lib\site-packages;Lib\test;Lib\tests;Lib\tkinter;Lib\idlelib;Lib\turtledemo" /> <InstallFiles Include="$(PySourcePath)Lib\**\*" Exclude="$(PySourcePath)Lib\**\*.pyc; $(PySourcePath)Lib\**\*.pyo; $(PySourcePath)Lib\turtle.py; - $(PySourcePath)Lib\site-packages\README; @(ExcludeFolders->'$(PySourcePath)%(Identity)\*'); @(ExcludeFolders->'$(PySourcePath)%(Identity)\**\*')"> <SourceBase>$(PySourcePath)Lib</SourceBase> diff --git a/Tools/msi/lib/lib_files.wxs b/Tools/msi/lib/lib_files.wxs index 8439518bcbd804b..4d44299f7839099 100644 --- a/Tools/msi/lib/lib_files.wxs +++ b/Tools/msi/lib/lib_files.wxs @@ -1,6 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?> <Wix xmlns="http://schemas.microsoft.com/wix/2006/wi"> - <?define exts=pyexpat;select;unicodedata;winsound;_bz2;_elementtree;_socket;_ssl;_ctypes;_hashlib;_multiprocessing;_lzma;_decimal;_overlapped;_sqlite3;_asyncio;_queue;_uuid;_wmi;_zoneinfo;_zstd ?> + <?define exts=pyexpat;select;unicodedata;winsound;_bz2;_elementtree;_socket;_ssl;_ctypes;_hashlib;_multiprocessing;_lzma;_decimal;_overlapped;_sqlite3;_asyncio;_queue;_remote_debugging;_uuid;_wmi;_zoneinfo;_zstd ?> <Fragment> <DirectoryRef Id="Lib_venv_scripts_nt" /> diff --git a/Tools/patchcheck/patchcheck.py b/Tools/patchcheck/patchcheck.py index 0dcf6ef844a0487..afd010a52544a3d 100755 --- a/Tools/patchcheck/patchcheck.py +++ b/Tools/patchcheck/patchcheck.py @@ -53,19 +53,43 @@ def get_git_branch(): def get_git_upstream_remote(): - """Get the remote name to use for upstream branches + """ + Get the remote name to use for upstream branches - Uses "upstream" if it exists, "origin" otherwise + Check for presence of "https://github.com/python/cpython" remote URL. + If only one is found, return that remote name. If multiple are found, + check for and return "upstream", "origin", or "python", in that + order. Raise an error if no valid matches are found. """ - cmd = "git remote get-url upstream".split() - try: - subprocess.check_output(cmd, - stderr=subprocess.DEVNULL, - cwd=SRCDIR, - encoding='UTF-8') - except subprocess.CalledProcessError: - return "origin" - return "upstream" + cmd = "git remote -v".split() + output = subprocess.check_output( + cmd, + stderr=subprocess.DEVNULL, + cwd=SRCDIR, + encoding="UTF-8" + ) + # Filter to desired remotes, accounting for potential uppercasing + filtered_remotes = { + remote.split("\t")[0].lower() for remote in output.split('\n') + if "python/cpython" in remote.lower() and remote.endswith("(fetch)") + } + if len(filtered_remotes) == 1: + [remote] = filtered_remotes + return remote + for remote_name in ["upstream", "origin", "python"]: + if remote_name in filtered_remotes: + return remote_name + remotes_found = "\n".join( + {remote for remote in output.split('\n') if remote.endswith("(fetch)")} + ) + raise ValueError( + f"Patchcheck was unable to find an unambiguous upstream remote, " + f"with URL matching 'https://github.com/python/cpython'. " + f"For help creating an upstream remote, see Dev Guide: " + f"https://devguide.python.org/getting-started/" + f"git-boot-camp/#cloning-a-forked-cpython-repository " + f"\nRemotes found: \n{remotes_found}" + ) def get_git_remote_default_branch(remote_name): diff --git a/Tools/peg_generator/.ruff.toml b/Tools/peg_generator/.ruff.toml new file mode 100644 index 000000000000000..bcf57248713df48 --- /dev/null +++ b/Tools/peg_generator/.ruff.toml @@ -0,0 +1,22 @@ +extend = "../../.ruff.toml" # Inherit the project-wide settings + +extend-exclude = [ + # Generated files: + "Tools/peg_generator/pegen/grammar_parser.py", +] + +[lint] +select = [ + "F", # pyflakes + "I", # isort + "UP", # pyupgrade + "RUF100", # Ban unused `# noqa` comments + "PGH004", # Ban blanket `# noqa` comments (only ignore specific error codes) +] +unfixable = [ + # The autofixes sometimes do the wrong things for these; + # it's better to have to manually look at the code and see how it needs fixing + "F841", # Detects unused variables + "F601", # Detects dictionaries that have duplicate keys + "F602", # Also detects dictionaries that have duplicate keys +] diff --git a/Tools/peg_generator/pegen/__main__.py b/Tools/peg_generator/pegen/__main__.py index 0b0b4b291c2b0ec..a6bc627d47cbbba 100755 --- a/Tools/peg_generator/pegen/__main__.py +++ b/Tools/peg_generator/pegen/__main__.py @@ -10,7 +10,6 @@ import time import token import traceback -from typing import Tuple from pegen.grammar import Grammar from pegen.parser import Parser @@ -21,7 +20,7 @@ def generate_c_code( args: argparse.Namespace, -) -> Tuple[Grammar, Parser, Tokenizer, ParserGenerator]: +) -> tuple[Grammar, Parser, Tokenizer, ParserGenerator]: from pegen.build import build_c_parser_and_generator verbose = args.verbose @@ -50,7 +49,7 @@ def generate_c_code( def generate_python_code( args: argparse.Namespace, -) -> Tuple[Grammar, Parser, Tokenizer, ParserGenerator]: +) -> tuple[Grammar, Parser, Tokenizer, ParserGenerator]: from pegen.build import build_python_parser_and_generator verbose = args.verbose @@ -188,7 +187,7 @@ def main() -> None: if __name__ == "__main__": - if sys.version_info < (3, 8): + if sys.version_info < (3, 8): # noqa: UP036 print("ERROR: using pegen requires at least Python 3.8!", file=sys.stderr) sys.exit(1) main() diff --git a/Tools/peg_generator/pegen/ast_dump.py b/Tools/peg_generator/pegen/ast_dump.py index 07f8799c114f5d6..60dc1b04672e2fa 100644 --- a/Tools/peg_generator/pegen/ast_dump.py +++ b/Tools/peg_generator/pegen/ast_dump.py @@ -6,7 +6,7 @@ TODO: Remove the above-described hack. """ -from typing import Any, Optional, Tuple +from typing import Any def ast_dump( @@ -14,9 +14,9 @@ def ast_dump( annotate_fields: bool = True, include_attributes: bool = False, *, - indent: Optional[str] = None, + indent: str | None = None, ) -> str: - def _format(node: Any, level: int = 0) -> Tuple[str, bool]: + def _format(node: Any, level: int = 0) -> tuple[str, bool]: if indent is not None: level += 1 prefix = "\n" + indent * level @@ -41,7 +41,7 @@ def _format(node: Any, level: int = 0) -> Tuple[str, bool]: value, simple = _format(value, level) allsimple = allsimple and simple if keywords: - args.append("%s=%s" % (name, value)) + args.append(f"{name}={value}") else: args.append(value) if include_attributes and node._attributes: @@ -54,16 +54,16 @@ def _format(node: Any, level: int = 0) -> Tuple[str, bool]: continue value, simple = _format(value, level) allsimple = allsimple and simple - args.append("%s=%s" % (name, value)) + args.append(f"{name}={value}") if allsimple and len(args) <= 3: - return "%s(%s)" % (node.__class__.__name__, ", ".join(args)), not args - return "%s(%s%s)" % (node.__class__.__name__, prefix, sep.join(args)), False + return "{}({})".format(node.__class__.__name__, ", ".join(args)), not args + return f"{node.__class__.__name__}({prefix}{sep.join(args)})", False elif isinstance(node, list): if not node: return "[]", True - return "[%s%s]" % (prefix, sep.join(_format(x, level)[0] for x in node)), False + return f"[{prefix}{sep.join(_format(x, level)[0] for x in node)}]", False return repr(node), True if all(cls.__name__ != "AST" for cls in node.__class__.__mro__): - raise TypeError("expected AST, got %r" % node.__class__.__name__) + raise TypeError(f"expected AST, got {node.__class__.__name__!r}") return _format(node)[0] diff --git a/Tools/peg_generator/pegen/build.py b/Tools/peg_generator/pegen/build.py index 41338c29bdd9ebd..44a58581686a4f7 100644 --- a/Tools/peg_generator/pegen/build.py +++ b/Tools/peg_generator/pegen/build.py @@ -6,7 +6,7 @@ import sysconfig import tempfile import tokenize -from typing import IO, Any, Dict, List, Optional, Set, Tuple +from typing import IO, Any from pegen.c_generator import CParserGenerator from pegen.grammar import Grammar @@ -18,11 +18,11 @@ MOD_DIR = pathlib.Path(__file__).resolve().parent -TokenDefinitions = Tuple[Dict[int, str], Dict[str, int], Set[str]] +TokenDefinitions = tuple[dict[int, str], dict[str, int], set[str]] Incomplete = Any # TODO: install `types-setuptools` and remove this alias -def get_extra_flags(compiler_flags: str, compiler_py_flags_nodist: str) -> List[str]: +def get_extra_flags(compiler_flags: str, compiler_py_flags_nodist: str) -> list[str]: flags = sysconfig.get_config_var(compiler_flags) py_flags_nodist = sysconfig.get_config_var(compiler_py_flags_nodist) if flags is None or py_flags_nodist is None: @@ -71,11 +71,11 @@ def fixup_build_ext(cmd: Incomplete) -> None: def compile_c_extension( generated_source_path: str, - build_dir: Optional[str] = None, + build_dir: str | None = None, verbose: bool = False, keep_asserts: bool = True, disable_optimization: bool = False, - library_dir: Optional[str] = None, + library_dir: str | None = None, ) -> pathlib.Path: """Compile the generated source for a parser generator into an extension module. @@ -93,11 +93,10 @@ def compile_c_extension( """ import setuptools.command.build_ext import setuptools.logging - - from setuptools import Extension, Distribution - from setuptools.modified import newer_group + from setuptools import Distribution, Extension from setuptools._distutils.ccompiler import new_compiler from setuptools._distutils.sysconfig import customize_compiler + from setuptools.modified import newer_group if verbose: setuptools.logging.set_threshold(logging.DEBUG) @@ -108,6 +107,8 @@ def compile_c_extension( extra_compile_args.append("-DPy_BUILD_CORE_MODULE") # Define _Py_TEST_PEGEN to not call PyAST_Validate() in Parser/pegen.c extra_compile_args.append("-D_Py_TEST_PEGEN") + if sys.platform == "win32" and sysconfig.get_config_var("Py_GIL_DISABLED"): + extra_compile_args.append("-DPy_GIL_DISABLED") extra_link_args = get_extra_flags("LDFLAGS", "PY_LDFLAGS_NODIST") if keep_asserts: extra_compile_args.append("-UNDEBUG") @@ -239,7 +240,7 @@ def compile_c_extension( def build_parser( grammar_file: str, verbose_tokenizer: bool = False, verbose_parser: bool = False -) -> Tuple[Grammar, Parser, Tokenizer]: +) -> tuple[Grammar, Parser, Tokenizer]: with open(grammar_file) as file: tokenizer = Tokenizer(tokenize.generate_tokens(file.readline), verbose=verbose_tokenizer) parser = GrammarParser(tokenizer, verbose=verbose_parser) @@ -290,7 +291,7 @@ def build_c_generator( keep_asserts_in_extension: bool = True, skip_actions: bool = False, ) -> ParserGenerator: - with open(tokens_file, "r") as tok_file: + with open(tokens_file) as tok_file: all_tokens, exact_tok, non_exact_tok = generate_token_definitions(tok_file) with open(output_file, "w") as file: gen: ParserGenerator = CParserGenerator( @@ -331,7 +332,7 @@ def build_c_parser_and_generator( verbose_c_extension: bool = False, keep_asserts_in_extension: bool = True, skip_actions: bool = False, -) -> Tuple[Grammar, Parser, Tokenizer, ParserGenerator]: +) -> tuple[Grammar, Parser, Tokenizer, ParserGenerator]: """Generate rules, C parser, tokenizer, parser generator for a given grammar Args: @@ -371,7 +372,7 @@ def build_python_parser_and_generator( verbose_tokenizer: bool = False, verbose_parser: bool = False, skip_actions: bool = False, -) -> Tuple[Grammar, Parser, Tokenizer, ParserGenerator]: +) -> tuple[Grammar, Parser, Tokenizer, ParserGenerator]: """Generate rules, python parser, tokenizer, parser generator for a given grammar Args: diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index 2be85a163b4043e..7bdfe44fbde9ddf 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -1,9 +1,10 @@ import ast import os.path import re +from collections.abc import Callable from dataclasses import dataclass, field from enum import Enum -from typing import IO, Any, Callable, Dict, List, Optional, Set, Text, Tuple +from typing import IO, Any from pegen import grammar from pegen.grammar import ( @@ -44,7 +45,7 @@ # define MAXSTACK 4000 # endif #else -# define MAXSTACK 4000 +# define MAXSTACK 6000 #endif """ @@ -86,13 +87,13 @@ class NodeTypes(Enum): @dataclass class FunctionCall: function: str - arguments: List[Any] = field(default_factory=list) - assigned_variable: Optional[str] = None - assigned_variable_type: Optional[str] = None - return_type: Optional[str] = None - nodetype: Optional[NodeTypes] = None + arguments: list[Any] = field(default_factory=list) + assigned_variable: str | None = None + assigned_variable_type: str | None = None + return_type: str | None = None + nodetype: NodeTypes | None = None force_true: bool = False - comment: Optional[str] = None + comment: str | None = None def __str__(self) -> str: parts = [] @@ -124,14 +125,14 @@ class CCallMakerVisitor(GrammarVisitor): def __init__( self, parser_generator: ParserGenerator, - exact_tokens: Dict[str, int], - non_exact_tokens: Set[str], + exact_tokens: dict[str, int], + non_exact_tokens: set[str], ): self.gen = parser_generator self.exact_tokens = exact_tokens self.non_exact_tokens = non_exact_tokens - self.cache: Dict[str, str] = {} - self.cleanup_statements: List[str] = [] + self.cache: dict[str, str] = {} + self.cleanup_statements: list[str] = [] def keyword_helper(self, keyword: str) -> FunctionCall: return FunctionCall( @@ -167,7 +168,7 @@ def visit_NameLeaf(self, node: NameLeaf) -> FunctionCall: ) return FunctionCall( assigned_variable=f"{name.lower()}_var", - function=f"_PyPegen_expect_token", + function="_PyPegen_expect_token", arguments=["p", name], nodetype=NodeTypes.GENERIC_TOKEN, return_type="Token *", @@ -199,7 +200,7 @@ def visit_StringLeaf(self, node: StringLeaf) -> FunctionCall: type = self.exact_tokens[val] return FunctionCall( assigned_variable="_literal", - function=f"_PyPegen_expect_token", + function="_PyPegen_expect_token", arguments=["p", type], nodetype=NodeTypes.GENERIC_TOKEN, return_type="Token *", @@ -214,33 +215,47 @@ def visit_NamedItem(self, node: NamedItem) -> FunctionCall: call.assigned_variable_type = node.type return call + def assert_no_undefined_behavior( + self, call: FunctionCall, wrapper: str, expected_rtype: str | None, + ) -> None: + if call.return_type != expected_rtype: + raise RuntimeError( + f"{call.function} return type is incompatible with {wrapper}: " + f"expect: {expected_rtype}, actual: {call.return_type}" + ) + def lookahead_call_helper(self, node: Lookahead, positive: int) -> FunctionCall: call = self.generate_call(node.node) - if call.nodetype == NodeTypes.NAME_TOKEN: - return FunctionCall( - function=f"_PyPegen_lookahead_with_name", - arguments=[positive, call.function, *call.arguments], - return_type="int", - ) + comment = None + if call.nodetype is NodeTypes.NAME_TOKEN: + function = "_PyPegen_lookahead_for_expr" + self.assert_no_undefined_behavior(call, function, "expr_ty") + elif call.nodetype is NodeTypes.STRING_TOKEN: + # _PyPegen_string_token() returns 'void *' instead of 'Token *'; + # in addition, the overall function call would return 'expr_ty'. + assert call.function == "_PyPegen_string_token" + function = "_PyPegen_lookahead" + self.assert_no_undefined_behavior(call, function, "expr_ty") elif call.nodetype == NodeTypes.SOFT_KEYWORD: - return FunctionCall( - function=f"_PyPegen_lookahead_with_string", - arguments=[positive, call.function, *call.arguments], - return_type="int", - ) + function = "_PyPegen_lookahead_with_string" + self.assert_no_undefined_behavior(call, function, "expr_ty") elif call.nodetype in {NodeTypes.GENERIC_TOKEN, NodeTypes.KEYWORD}: - return FunctionCall( - function=f"_PyPegen_lookahead_with_int", - arguments=[positive, call.function, *call.arguments], - return_type="int", - comment=f"token={node.node}", - ) + function = "_PyPegen_lookahead_with_int" + self.assert_no_undefined_behavior(call, function, "Token *") + comment = f"token={node.node}" + elif call.return_type == "expr_ty": + function = "_PyPegen_lookahead_for_expr" + elif call.return_type == "stmt_ty": + function = "_PyPegen_lookahead_for_stmt" else: - return FunctionCall( - function=f"_PyPegen_lookahead", - arguments=[positive, f"(void *(*)(Parser *)) {call.function}", *call.arguments], - return_type="int", - ) + function = "_PyPegen_lookahead" + self.assert_no_undefined_behavior(call, function, None) + return FunctionCall( + function=function, + arguments=[positive, call.function, *call.arguments], + return_type="int", + comment=comment, + ) def visit_PositiveLookahead(self, node: PositiveLookahead) -> FunctionCall: return self.lookahead_call_helper(node, 1) @@ -257,7 +272,7 @@ def visit_Forced(self, node: Forced) -> FunctionCall: type = self.exact_tokens[val] return FunctionCall( assigned_variable="_literal", - function=f"_PyPegen_expect_forced_token", + function="_PyPegen_expect_forced_token", arguments=["p", type, f'"{val}"'], nodetype=NodeTypes.GENERIC_TOKEN, return_type="Token *", @@ -269,7 +284,7 @@ def visit_Forced(self, node: Forced) -> FunctionCall: call.comment = None return FunctionCall( assigned_variable="_literal", - function=f"_PyPegen_expect_forced_result", + function="_PyPegen_expect_forced_result", arguments=["p", str(call), f'"{node.node.rhs!s}"'], return_type="void *", comment=f"forced_token=({node.node.rhs!s})", @@ -292,7 +307,7 @@ def _generate_artificial_rule_call( node: Any, prefix: str, rule_generation_func: Callable[[], str], - return_type: Optional[str] = None, + return_type: str | None = None, ) -> FunctionCall: node_str = f"{node}" key = f"{prefix}_{node_str}" @@ -363,10 +378,10 @@ class CParserGenerator(ParserGenerator, GrammarVisitor): def __init__( self, grammar: grammar.Grammar, - tokens: Dict[int, str], - exact_tokens: Dict[str, int], - non_exact_tokens: Set[str], - file: Optional[IO[Text]], + tokens: dict[int, str], + exact_tokens: dict[str, int], + non_exact_tokens: set[str], + file: IO[str] | None, debug: bool = False, skip_actions: bool = False, ): @@ -377,7 +392,7 @@ def __init__( self._varname_counter = 0 self.debug = debug self.skip_actions = skip_actions - self.cleanup_statements: List[str] = [] + self.cleanup_statements: list[str] = [] def add_level(self) -> None: self.print("if (p->level++ == MAXSTACK || _Py_ReachedRecursionLimitWithMargin(PyThreadState_Get(), 1)) {") @@ -413,12 +428,12 @@ def call_with_errorcheck_goto(self, call_text: str, goto_target: str) -> None: self.print(f"if ({error_var}) {{") with self.indent(): self.print(f"goto {goto_target};") - self.print(f"}}") + self.print("}") def out_of_memory_return( self, expr: str, - cleanup_code: Optional[str] = None, + cleanup_code: str | None = None, ) -> None: self.print(f"if ({expr}) {{") with self.indent(): @@ -427,14 +442,14 @@ def out_of_memory_return( self.print("p->error_indicator = 1;") self.print("PyErr_NoMemory();") self.add_return("NULL") - self.print(f"}}") + self.print("}") def out_of_memory_goto(self, expr: str, goto_target: str) -> None: self.print(f"if ({expr}) {{") with self.indent(): self.print("PyErr_NoMemory();") self.print(f"goto {goto_target};") - self.print(f"}}") + self.print("}") def generate(self, filename: str) -> None: self.collect_rules() @@ -477,8 +492,8 @@ def generate(self, filename: str) -> None: if trailer: self.print(trailer.rstrip("\n") % dict(mode=mode, modulename=modulename)) - def _group_keywords_by_length(self) -> Dict[int, List[Tuple[str, int]]]: - groups: Dict[int, List[Tuple[str, int]]] = {} + def _group_keywords_by_length(self) -> dict[int, list[tuple[str, int]]]: + groups: dict[int, list[tuple[str, int]]] = {} for keyword_str, keyword_type in self.keywords.items(): length = len(keyword_str) if length in groups: @@ -570,10 +585,10 @@ def _set_up_rule_memoization(self, node: Rule, result_type: str) -> None: self.print("if (_raw == NULL || p->mark <= _resmark)") with self.indent(): self.print("break;") - self.print(f"_resmark = p->mark;") + self.print("_resmark = p->mark;") self.print("_res = _raw;") self.print("}") - self.print(f"p->mark = _resmark;") + self.print("p->mark = _resmark;") self.add_return("_res") self.print("}") self.print(f"static {result_type}") @@ -629,7 +644,7 @@ def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None: if memoize: self.print("int _start_mark = p->mark;") self.print("void **_children = PyMem_Malloc(sizeof(void *));") - self.out_of_memory_return(f"!_children") + self.out_of_memory_return("!_children") self.print("Py_ssize_t _children_capacity = 1;") self.print("Py_ssize_t _n = 0;") if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts): @@ -647,7 +662,7 @@ def _handle_loop_rule_body(self, node: Rule, rhs: Rhs) -> None: self.add_return("NULL") self.print("}") self.print("asdl_seq *_seq = (asdl_seq*)_Py_asdl_generic_seq_new(_n, p->arena);") - self.out_of_memory_return(f"!_seq", cleanup_code="PyMem_Free(_children);") + self.out_of_memory_return("!_seq", cleanup_code="PyMem_Free(_children);") self.print("for (Py_ssize_t i = 0; i < _n; i++) asdl_seq_SET_UNTYPED(_seq, i, _children[i]);") self.print("PyMem_Free(_children);") if memoize and node.name: @@ -701,7 +716,7 @@ def visit_NamedItem(self, node: NamedItem) -> None: self.print(call) def visit_Rhs( - self, node: Rhs, is_loop: bool, is_gather: bool, rulename: Optional[str] + self, node: Rhs, is_loop: bool, is_gather: bool, rulename: str | None ) -> None: if is_loop: assert len(node.alts) == 1 @@ -720,10 +735,10 @@ def join_conditions(self, keyword: str, node: Any) -> None: self.visit(item) self.print(")") - def emit_action(self, node: Alt, cleanup_code: Optional[str] = None) -> None: + def emit_action(self, node: Alt, cleanup_code: str | None = None) -> None: self.print(f"_res = {node.action};") - self.print("if (_res == NULL && PyErr_Occurred()) {") + self.print("if ((_res == NULL || p->error_indicator) && PyErr_Occurred()) {") with self.indent(): self.print("p->error_indicator = 1;") if cleanup_code: @@ -762,7 +777,7 @@ def emit_default_action(self, is_gather: bool, node: Alt) -> None: def emit_dummy_action(self) -> None: self.print("_res = _PyPegen_dummy_name(p);") - def handle_alt_normal(self, node: Alt, is_gather: bool, rulename: Optional[str]) -> None: + def handle_alt_normal(self, node: Alt, is_gather: bool, rulename: str | None) -> None: self.join_conditions(keyword="if", node=node) self.print("{") # We have parsed successfully all the conditions for the option. @@ -782,10 +797,10 @@ def handle_alt_normal(self, node: Alt, is_gather: bool, rulename: Optional[str]) self.emit_default_action(is_gather, node) # As the current option has parsed correctly, do not continue with the rest. - self.print(f"goto done;") + self.print("goto done;") self.print("}") - def handle_alt_loop(self, node: Alt, is_gather: bool, rulename: Optional[str]) -> None: + def handle_alt_loop(self, node: Alt, is_gather: bool, rulename: str | None) -> None: # Condition of the main body of the alternative self.join_conditions(keyword="while", node=node) self.print("{") @@ -809,7 +824,7 @@ def handle_alt_loop(self, node: Alt, is_gather: bool, rulename: Optional[str]) - self.print( "void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));" ) - self.out_of_memory_return(f"!_new_children", cleanup_code="PyMem_Free(_children);") + self.out_of_memory_return("!_new_children", cleanup_code="PyMem_Free(_children);") self.print("_children = _new_children;") self.print("}") self.print("_children[_n++] = _res;") @@ -817,7 +832,7 @@ def handle_alt_loop(self, node: Alt, is_gather: bool, rulename: Optional[str]) - self.print("}") def visit_Alt( - self, node: Alt, is_loop: bool, is_gather: bool, rulename: Optional[str] + self, node: Alt, is_loop: bool, is_gather: bool, rulename: str | None ) -> None: if len(node.items) == 1 and str(node.items[0]).startswith("invalid_"): self.print(f"if (p->call_invalid_rules) {{ // {node}") @@ -861,7 +876,7 @@ def visit_Alt( self.print("}") self.print("}") - def collect_vars(self, node: Alt) -> Dict[Optional[str], Optional[str]]: + def collect_vars(self, node: Alt) -> dict[str | None, str | None]: types = {} with self.local_variable_context(): for item in node.items: @@ -869,7 +884,7 @@ def collect_vars(self, node: Alt) -> Dict[Optional[str], Optional[str]]: types[name] = type return types - def add_var(self, node: NamedItem) -> Tuple[Optional[str], Optional[str]]: + def add_var(self, node: NamedItem) -> tuple[str | None, str | None]: call = self.callmakervisitor.generate_call(node.item) name = node.name if node.name else call.assigned_variable if name is not None: diff --git a/Tools/peg_generator/pegen/first_sets.py b/Tools/peg_generator/pegen/first_sets.py index 6d794ffa4bfa8b5..f6b83c0ca31e3f4 100755 --- a/Tools/peg_generator/pegen/first_sets.py +++ b/Tools/peg_generator/pegen/first_sets.py @@ -3,7 +3,6 @@ import argparse import pprint import sys -from typing import Dict, Set from pegen.build import build_parser from pegen.grammar import ( @@ -33,20 +32,20 @@ class FirstSetCalculator(GrammarVisitor): - def __init__(self, rules: Dict[str, Rule]) -> None: + def __init__(self, rules: dict[str, Rule]) -> None: self.rules = rules self.nullables = compute_nullables(rules) - self.first_sets: Dict[str, Set[str]] = dict() - self.in_process: Set[str] = set() + self.first_sets: dict[str, set[str]] = dict() + self.in_process: set[str] = set() - def calculate(self) -> Dict[str, Set[str]]: + def calculate(self) -> dict[str, set[str]]: for name, rule in self.rules.items(): self.visit(rule) return self.first_sets - def visit_Alt(self, item: Alt) -> Set[str]: - result: Set[str] = set() - to_remove: Set[str] = set() + def visit_Alt(self, item: Alt) -> set[str]: + result: set[str] = set() + to_remove: set[str] = set() for other in item.items: new_terminals = self.visit(other) if isinstance(other.item, NegativeLookahead): @@ -71,34 +70,34 @@ def visit_Alt(self, item: Alt) -> Set[str]: return result - def visit_Cut(self, item: Cut) -> Set[str]: + def visit_Cut(self, item: Cut) -> set[str]: return set() - def visit_Group(self, item: Group) -> Set[str]: + def visit_Group(self, item: Group) -> set[str]: return self.visit(item.rhs) - def visit_PositiveLookahead(self, item: Lookahead) -> Set[str]: + def visit_PositiveLookahead(self, item: Lookahead) -> set[str]: return self.visit(item.node) - def visit_NegativeLookahead(self, item: NegativeLookahead) -> Set[str]: + def visit_NegativeLookahead(self, item: NegativeLookahead) -> set[str]: return self.visit(item.node) - def visit_NamedItem(self, item: NamedItem) -> Set[str]: + def visit_NamedItem(self, item: NamedItem) -> set[str]: return self.visit(item.item) - def visit_Opt(self, item: Opt) -> Set[str]: + def visit_Opt(self, item: Opt) -> set[str]: return self.visit(item.node) - def visit_Gather(self, item: Gather) -> Set[str]: + def visit_Gather(self, item: Gather) -> set[str]: return self.visit(item.node) - def visit_Repeat0(self, item: Repeat0) -> Set[str]: + def visit_Repeat0(self, item: Repeat0) -> set[str]: return self.visit(item.node) - def visit_Repeat1(self, item: Repeat1) -> Set[str]: + def visit_Repeat1(self, item: Repeat1) -> set[str]: return self.visit(item.node) - def visit_NameLeaf(self, item: NameLeaf) -> Set[str]: + def visit_NameLeaf(self, item: NameLeaf) -> set[str]: if item.value not in self.rules: return {item.value} @@ -110,16 +109,16 @@ def visit_NameLeaf(self, item: NameLeaf) -> Set[str]: return self.first_sets[item.value] - def visit_StringLeaf(self, item: StringLeaf) -> Set[str]: + def visit_StringLeaf(self, item: StringLeaf) -> set[str]: return {item.value} - def visit_Rhs(self, item: Rhs) -> Set[str]: - result: Set[str] = set() + def visit_Rhs(self, item: Rhs) -> set[str]: + result: set[str] = set() for alt in item.alts: result |= self.visit(alt) return result - def visit_Rule(self, item: Rule) -> Set[str]: + def visit_Rule(self, item: Rule) -> set[str]: if item.name in self.in_process: return set() elif item.name not in self.first_sets: @@ -138,7 +137,7 @@ def main() -> None: try: grammar, parser, tokenizer = build_parser(args.grammar_file) except Exception as err: - print("ERROR: Failed to parse grammar file", file=sys.stderr) + print("ERROR: Failed to parse grammar file", err, file=sys.stderr) sys.exit(1) firs_sets = FirstSetCalculator(grammar.rules).calculate() diff --git a/Tools/peg_generator/pegen/grammar.py b/Tools/peg_generator/pegen/grammar.py index 1ee9d100b61f490..cca8584a6320718 100644 --- a/Tools/peg_generator/pegen/grammar.py +++ b/Tools/peg_generator/pegen/grammar.py @@ -1,15 +1,7 @@ from __future__ import annotations -from typing import ( - AbstractSet, - Any, - Iterable, - Iterator, - List, - Optional, - Tuple, - Union, -) +from collections.abc import Iterable, Iterator, Set +from typing import Any class GrammarError(Exception): @@ -34,7 +26,7 @@ def generic_visit(self, node: Iterable[Any], *args: Any, **kwargs: Any) -> Any: class Grammar: - def __init__(self, rules: Iterable[Rule], metas: Iterable[Tuple[str, Optional[str]]]): + def __init__(self, rules: Iterable[Rule], metas: Iterable[tuple[str, str | None]]): # Check if there are repeated rules in "rules" all_rules = {} for rule in rules: @@ -66,7 +58,7 @@ def __iter__(self) -> Iterator[Rule]: class Rule: - def __init__(self, name: str, type: Optional[str], rhs: Rhs, memo: Optional[object] = None): + def __init__(self, name: str, type: str | None, rhs: Rhs, memo: object | None = None): self.name = name self.type = type self.rhs = rhs @@ -141,9 +133,9 @@ def __repr__(self) -> str: class Rhs: - def __init__(self, alts: List[Alt]): + def __init__(self, alts: list[Alt]): self.alts = alts - self.memo: Optional[Tuple[Optional[str], str]] = None + self.memo: tuple[str | None, str] | None = None def __str__(self) -> str: return " | ".join(str(alt) for alt in self.alts) @@ -151,7 +143,7 @@ def __str__(self) -> str: def __repr__(self) -> str: return f"Rhs({self.alts!r})" - def __iter__(self) -> Iterator[List[Alt]]: + def __iter__(self) -> Iterator[list[Alt]]: yield self.alts @property @@ -165,7 +157,7 @@ def can_be_inlined(self) -> bool: class Alt: - def __init__(self, items: List[NamedItem], *, icut: int = -1, action: Optional[str] = None): + def __init__(self, items: list[NamedItem], *, icut: int = -1, action: str | None = None): self.items = items self.icut = icut self.action = action @@ -185,12 +177,12 @@ def __repr__(self) -> str: args.append(f"action={self.action!r}") return f"Alt({', '.join(args)})" - def __iter__(self) -> Iterator[List[NamedItem]]: + def __iter__(self) -> Iterator[list[NamedItem]]: yield self.items class NamedItem: - def __init__(self, name: Optional[str], item: Item, type: Optional[str] = None): + def __init__(self, name: str | None, item: Item, type: str | None = None): self.name = name self.item = item self.type = type @@ -271,7 +263,7 @@ class Repeat: def __init__(self, node: Plain): self.node = node - self.memo: Optional[Tuple[Optional[str], str]] = None + self.memo: tuple[str | None, str] | None = None def __iter__(self) -> Iterator[Plain]: yield self.node @@ -334,12 +326,12 @@ def __init__(self) -> None: pass def __repr__(self) -> str: - return f"Cut()" + return "Cut()" def __str__(self) -> str: - return f"~" + return "~" - def __iter__(self) -> Iterator[Tuple[str, str]]: + def __iter__(self) -> Iterator[tuple[str, str]]: yield from () def __eq__(self, other: object) -> bool: @@ -347,15 +339,15 @@ def __eq__(self, other: object) -> bool: return NotImplemented return True - def initial_names(self) -> AbstractSet[str]: + def initial_names(self) -> Set[str]: return set() -Plain = Union[Leaf, Group] -Item = Union[Plain, Opt, Repeat, Forced, Lookahead, Rhs, Cut] -RuleName = Tuple[str, Optional[str]] -MetaTuple = Tuple[str, Optional[str]] -MetaList = List[MetaTuple] -RuleList = List[Rule] -NamedItemList = List[NamedItem] -LookaheadOrCut = Union[Lookahead, Cut] +Plain = Leaf | Group +Item = Plain | Opt | Repeat | Forced | Lookahead | Rhs | Cut +RuleName = tuple[str, str | None] +MetaTuple = tuple[str, str | None] +MetaList = list[MetaTuple] +RuleList = list[Rule] +NamedItemList = list[NamedItem] +LookaheadOrCut = Lookahead | Cut diff --git a/Tools/peg_generator/pegen/grammar_visualizer.py b/Tools/peg_generator/pegen/grammar_visualizer.py index 11f784f45b66b8a..eadb6987389b887 100644 --- a/Tools/peg_generator/pegen/grammar_visualizer.py +++ b/Tools/peg_generator/pegen/grammar_visualizer.py @@ -1,6 +1,7 @@ import argparse import sys -from typing import Any, Callable, Iterator +from collections.abc import Callable, Iterator +from typing import Any from pegen.build import build_parser from pegen.grammar import Grammar, Rule @@ -52,7 +53,7 @@ def main() -> None: try: grammar, parser, tokenizer = build_parser(args.filename) except Exception as err: - print("ERROR: Failed to parse grammar file", file=sys.stderr) + print("ERROR: Failed to parse grammar file", err, file=sys.stderr) sys.exit(1) visitor = ASTGrammarPrinter() diff --git a/Tools/peg_generator/pegen/parser.py b/Tools/peg_generator/pegen/parser.py index a987d30a9d6438b..806003c8350c033 100644 --- a/Tools/peg_generator/pegen/parser.py +++ b/Tools/peg_generator/pegen/parser.py @@ -5,7 +5,8 @@ import tokenize import traceback from abc import abstractmethod -from typing import Any, Callable, ClassVar, Dict, Optional, Tuple, Type, TypeVar, cast +from collections.abc import Callable +from typing import Any, ClassVar, TypeVar, cast from pegen.tokenizer import Mark, Tokenizer, exact_token_types @@ -74,12 +75,12 @@ def memoize_wrapper(self: "Parser", *args: object) -> Any: def memoize_left_rec( - method: Callable[["Parser"], Optional[T]] -) -> Callable[["Parser"], Optional[T]]: + method: Callable[["Parser"], T | None] +) -> Callable[["Parser"], T | None]: """Memoize a left-recursive symbol method.""" method_name = method.__name__ - def memoize_left_rec_wrapper(self: "Parser") -> Optional[T]: + def memoize_left_rec_wrapper(self: "Parser") -> T | None: mark = self._mark() key = mark, method_name, () # Fast path: cache hit, and not verbose. @@ -160,15 +161,15 @@ def memoize_left_rec_wrapper(self: "Parser") -> Optional[T]: class Parser: """Parsing base class.""" - KEYWORDS: ClassVar[Tuple[str, ...]] + KEYWORDS: ClassVar[tuple[str, ...]] - SOFT_KEYWORDS: ClassVar[Tuple[str, ...]] + SOFT_KEYWORDS: ClassVar[tuple[str, ...]] def __init__(self, tokenizer: Tokenizer, *, verbose: bool = False): self._tokenizer = tokenizer self._verbose = verbose self._level = 0 - self._cache: Dict[Tuple[Mark, str, Tuple[Any, ...]], Tuple[Any, Mark]] = {} + self._cache: dict[tuple[Mark, str, tuple[Any, ...]], tuple[Any, Mark]] = {} # Integer tracking whether we are in a left recursive rule or not. Can be useful # for error reporting. self.in_recursive_rule = 0 @@ -185,28 +186,28 @@ def showpeek(self) -> str: return f"{tok.start[0]}.{tok.start[1]}: {token.tok_name[tok.type]}:{tok.string!r}" @memoize - def name(self) -> Optional[tokenize.TokenInfo]: + def name(self) -> tokenize.TokenInfo | None: tok = self._tokenizer.peek() if tok.type == token.NAME and tok.string not in self.KEYWORDS: return self._tokenizer.getnext() return None @memoize - def number(self) -> Optional[tokenize.TokenInfo]: + def number(self) -> tokenize.TokenInfo | None: tok = self._tokenizer.peek() if tok.type == token.NUMBER: return self._tokenizer.getnext() return None @memoize - def string(self) -> Optional[tokenize.TokenInfo]: + def string(self) -> tokenize.TokenInfo | None: tok = self._tokenizer.peek() if tok.type == token.STRING: return self._tokenizer.getnext() return None @memoize - def fstring_start(self) -> Optional[tokenize.TokenInfo]: + def fstring_start(self) -> tokenize.TokenInfo | None: FSTRING_START = getattr(token, "FSTRING_START", None) if not FSTRING_START: return None @@ -216,7 +217,7 @@ def fstring_start(self) -> Optional[tokenize.TokenInfo]: return None @memoize - def fstring_middle(self) -> Optional[tokenize.TokenInfo]: + def fstring_middle(self) -> tokenize.TokenInfo | None: FSTRING_MIDDLE = getattr(token, "FSTRING_MIDDLE", None) if not FSTRING_MIDDLE: return None @@ -226,7 +227,7 @@ def fstring_middle(self) -> Optional[tokenize.TokenInfo]: return None @memoize - def fstring_end(self) -> Optional[tokenize.TokenInfo]: + def fstring_end(self) -> tokenize.TokenInfo | None: FSTRING_END = getattr(token, "FSTRING_END", None) if not FSTRING_END: return None @@ -236,28 +237,28 @@ def fstring_end(self) -> Optional[tokenize.TokenInfo]: return None @memoize - def op(self) -> Optional[tokenize.TokenInfo]: + def op(self) -> tokenize.TokenInfo | None: tok = self._tokenizer.peek() if tok.type == token.OP: return self._tokenizer.getnext() return None @memoize - def type_comment(self) -> Optional[tokenize.TokenInfo]: + def type_comment(self) -> tokenize.TokenInfo | None: tok = self._tokenizer.peek() if tok.type == token.TYPE_COMMENT: return self._tokenizer.getnext() return None @memoize - def soft_keyword(self) -> Optional[tokenize.TokenInfo]: + def soft_keyword(self) -> tokenize.TokenInfo | None: tok = self._tokenizer.peek() if tok.type == token.NAME and tok.string in self.SOFT_KEYWORDS: return self._tokenizer.getnext() return None @memoize - def expect(self, type: str) -> Optional[tokenize.TokenInfo]: + def expect(self, type: str) -> tokenize.TokenInfo | None: tok = self._tokenizer.peek() if tok.string == type: return self._tokenizer.getnext() @@ -271,7 +272,7 @@ def expect(self, type: str) -> Optional[tokenize.TokenInfo]: return self._tokenizer.getnext() return None - def expect_forced(self, res: Any, expectation: str) -> Optional[tokenize.TokenInfo]: + def expect_forced(self, res: Any, expectation: str) -> tokenize.TokenInfo | None: if res is None: raise self.make_syntax_error(f"expected {expectation}") return res @@ -293,7 +294,7 @@ def make_syntax_error(self, message: str, filename: str = "<unknown>") -> Syntax return SyntaxError(message, (filename, tok.start[0], 1 + tok.start[1], tok.line)) -def simple_parser_main(parser_class: Type[Parser]) -> None: +def simple_parser_main(parser_class: type[Parser]) -> None: argparser = argparse.ArgumentParser() argparser.add_argument( "-v", @@ -330,7 +331,7 @@ def simple_parser_main(parser_class: Type[Parser]) -> None: endpos = 0 else: endpos = file.tell() - except IOError: + except OSError: endpos = 0 finally: if file is not sys.stdin: diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py index 6ce0649aefe7ff1..43054c258d2a056 100644 --- a/Tools/peg_generator/pegen/parser_generator.py +++ b/Tools/peg_generator/pegen/parser_generator.py @@ -1,22 +1,10 @@ -import sys import ast import contextlib import re +import sys from abc import abstractmethod -from typing import ( - IO, - AbstractSet, - Any, - Dict, - Iterable, - Iterator, - List, - Optional, - Set, - Text, - Tuple, - Union, -) +from collections.abc import Iterable, Iterator, Set +from typing import IO, Any from pegen import sccutils from pegen.grammar import ( @@ -44,8 +32,8 @@ class RuleCollectorVisitor(GrammarVisitor): """Visitor that invokes a provided callmaker visitor with just the NamedItem nodes""" - def __init__(self, rules: Dict[str, Rule], callmakervisitor: GrammarVisitor) -> None: - self.rulses = rules + def __init__(self, rules: dict[str, Rule], callmakervisitor: GrammarVisitor) -> None: + self.rules = rules self.callmaker = callmakervisitor def visit_Rule(self, rule: Rule) -> None: @@ -56,9 +44,9 @@ def visit_NamedItem(self, item: NamedItem) -> None: class KeywordCollectorVisitor(GrammarVisitor): - """Visitor that collects all the keywods and soft keywords in the Grammar""" + """Visitor that collects all the keywords and soft keywords in the Grammar""" - def __init__(self, gen: "ParserGenerator", keywords: Dict[str, int], soft_keywords: Set[str]): + def __init__(self, gen: "ParserGenerator", keywords: dict[str, int], soft_keywords: set[str]): self.generator = gen self.keywords = keywords self.soft_keywords = soft_keywords @@ -73,7 +61,7 @@ def visit_StringLeaf(self, node: StringLeaf) -> None: class RuleCheckingVisitor(GrammarVisitor): - def __init__(self, rules: Dict[str, Rule], tokens: Set[str]): + def __init__(self, rules: dict[str, Rule], tokens: set[str]): self.rules = rules self.tokens = tokens # If python < 3.12 add the virtual fstring tokens @@ -81,6 +69,11 @@ def __init__(self, rules: Dict[str, Rule], tokens: Set[str]): self.tokens.add("FSTRING_START") self.tokens.add("FSTRING_END") self.tokens.add("FSTRING_MIDDLE") + # If python < 3.14 add the virtual tstring tokens + if sys.version_info < (3, 14, 0, 'beta', 1): + self.tokens.add("TSTRING_START") + self.tokens.add("TSTRING_END") + self.tokens.add("TSTRING_MIDDLE") def visit_NameLeaf(self, node: NameLeaf) -> None: if node.value not in self.rules and node.value not in self.tokens: @@ -95,11 +88,11 @@ def visit_NamedItem(self, node: NamedItem) -> None: class ParserGenerator: callmakervisitor: GrammarVisitor - def __init__(self, grammar: Grammar, tokens: Set[str], file: Optional[IO[Text]]): + def __init__(self, grammar: Grammar, tokens: set[str], file: IO[str] | None): self.grammar = grammar self.tokens = tokens - self.keywords: Dict[str, int] = {} - self.soft_keywords: Set[str] = set() + self.keywords: dict[str, int] = {} + self.soft_keywords: set[str] = set() self.rules = grammar.rules self.validate_rule_names() if "trailer" not in grammar.metas and "start" not in self.rules: @@ -112,8 +105,8 @@ def __init__(self, grammar: Grammar, tokens: Set[str], file: Optional[IO[Text]]) self.first_graph, self.first_sccs = compute_left_recursives(self.rules) self.counter = 0 # For name_rule()/name_loop() self.keyword_counter = 499 # For keyword_type() - self.all_rules: Dict[str, Rule] = self.rules.copy() # Rules + temporal rules - self._local_variable_stack: List[List[str]] = [] + self.all_rules: dict[str, Rule] = self.rules.copy() # Rules + temporal rules + self._local_variable_stack: list[list[str]] = [] def validate_rule_names(self) -> None: for rule in self.rules: @@ -127,7 +120,7 @@ def local_variable_context(self) -> Iterator[None]: self._local_variable_stack.pop() @property - def local_variable_names(self) -> List[str]: + def local_variable_names(self) -> list[str]: return self._local_variable_stack[-1] @abstractmethod @@ -159,7 +152,7 @@ def collect_rules(self) -> None: keyword_collector.visit(rule) rule_collector = RuleCollectorVisitor(self.rules, self.callmakervisitor) - done: Set[str] = set() + done: set[str] = set() while True: computed_rules = list(self.all_rules) todo = [i for i in computed_rules if i not in done] @@ -224,10 +217,10 @@ def dedupe(self, name: str) -> str: class NullableVisitor(GrammarVisitor): - def __init__(self, rules: Dict[str, Rule]) -> None: + def __init__(self, rules: dict[str, Rule]) -> None: self.rules = rules - self.visited: Set[Any] = set() - self.nullables: Set[Union[Rule, NamedItem]] = set() + self.visited: set[Any] = set() + self.nullables: set[Rule | NamedItem] = set() def visit_Rule(self, rule: Rule) -> bool: if rule in self.visited: @@ -289,7 +282,7 @@ def visit_StringLeaf(self, node: StringLeaf) -> bool: return not node.value -def compute_nullables(rules: Dict[str, Rule]) -> Set[Any]: +def compute_nullables(rules: dict[str, Rule]) -> set[Any]: """Compute which rules in a grammar are nullable. Thanks to TatSu (tatsu/leftrec.py) for inspiration. @@ -301,12 +294,12 @@ def compute_nullables(rules: Dict[str, Rule]) -> Set[Any]: class InitialNamesVisitor(GrammarVisitor): - def __init__(self, rules: Dict[str, Rule]) -> None: + def __init__(self, rules: dict[str, Rule]) -> None: self.rules = rules self.nullables = compute_nullables(rules) - def generic_visit(self, node: Iterable[Any], *args: Any, **kwargs: Any) -> Set[Any]: - names: Set[str] = set() + def generic_visit(self, node: Iterable[Any], *args: Any, **kwargs: Any) -> set[Any]: + names: set[str] = set() for value in node: if isinstance(value, list): for item in value: @@ -315,33 +308,33 @@ def generic_visit(self, node: Iterable[Any], *args: Any, **kwargs: Any) -> Set[A names |= self.visit(value, *args, **kwargs) return names - def visit_Alt(self, alt: Alt) -> Set[Any]: - names: Set[str] = set() + def visit_Alt(self, alt: Alt) -> set[Any]: + names: set[str] = set() for item in alt.items: names |= self.visit(item) if item not in self.nullables: break return names - def visit_Forced(self, force: Forced) -> Set[Any]: + def visit_Forced(self, force: Forced) -> set[Any]: return set() - def visit_LookAhead(self, lookahead: Lookahead) -> Set[Any]: + def visit_LookAhead(self, lookahead: Lookahead) -> set[Any]: return set() - def visit_Cut(self, cut: Cut) -> Set[Any]: + def visit_Cut(self, cut: Cut) -> set[Any]: return set() - def visit_NameLeaf(self, node: NameLeaf) -> Set[Any]: + def visit_NameLeaf(self, node: NameLeaf) -> set[Any]: return {node.value} - def visit_StringLeaf(self, node: StringLeaf) -> Set[Any]: + def visit_StringLeaf(self, node: StringLeaf) -> set[Any]: return set() def compute_left_recursives( - rules: Dict[str, Rule] -) -> Tuple[Dict[str, AbstractSet[str]], List[AbstractSet[str]]]: + rules: dict[str, Rule] +) -> tuple[dict[str, Set[str]], list[Set[str]]]: graph = make_first_graph(rules) sccs = list(sccutils.strongly_connected_components(graph.keys(), graph)) for scc in sccs: @@ -369,7 +362,7 @@ def compute_left_recursives( return graph, sccs -def make_first_graph(rules: Dict[str, Rule]) -> Dict[str, AbstractSet[str]]: +def make_first_graph(rules: dict[str, Rule]) -> dict[str, Set[str]]: """Compute the graph of left-invocations. There's an edge from A to B if A may invoke B at its initial @@ -379,7 +372,7 @@ def make_first_graph(rules: Dict[str, Rule]) -> Dict[str, AbstractSet[str]]: """ initial_name_visitor = InitialNamesVisitor(rules) graph = {} - vertices: Set[str] = set() + vertices: set[str] = set() for rulename, rhs in rules.items(): graph[rulename] = names = initial_name_visitor.visit(rhs) vertices |= names diff --git a/Tools/peg_generator/pegen/python_generator.py b/Tools/peg_generator/pegen/python_generator.py index 4bb26480ebc0af8..e69ea4b5f4e14c4 100644 --- a/Tools/peg_generator/pegen/python_generator.py +++ b/Tools/peg_generator/pegen/python_generator.py @@ -1,6 +1,7 @@ import os.path import token -from typing import IO, Any, Callable, Dict, Optional, Sequence, Set, Text, Tuple +from collections.abc import Callable, Sequence +from typing import IO, Any from pegen import grammar from pegen.grammar import ( @@ -74,10 +75,10 @@ def visit_NegativeLookahead(self, node: NegativeLookahead) -> bool: def visit_Opt(self, node: Opt) -> bool: return self.visit(node.node) - def visit_Repeat(self, node: Repeat0) -> Tuple[str, str]: + def visit_Repeat(self, node: Repeat0) -> tuple[str, str]: return self.visit(node.node) - def visit_Gather(self, node: Gather) -> Tuple[str, str]: + def visit_Gather(self, node: Gather) -> tuple[str, str]: return self.visit(node.node) def visit_Group(self, node: Group) -> bool: @@ -93,9 +94,9 @@ def visit_Forced(self, node: Forced) -> bool: class PythonCallMakerVisitor(GrammarVisitor): def __init__(self, parser_generator: ParserGenerator): self.gen = parser_generator - self.cache: Dict[str, Tuple[str, str]] = {} + self.cache: dict[str, tuple[str, str]] = {} - def visit_NameLeaf(self, node: NameLeaf) -> Tuple[Optional[str], str]: + def visit_NameLeaf(self, node: NameLeaf) -> tuple[str | None, str]: name = node.value if name == "SOFT_KEYWORD": return "soft_keyword", "self.soft_keyword()" @@ -108,31 +109,31 @@ def visit_NameLeaf(self, node: NameLeaf) -> Tuple[Optional[str], str]: return "_" + name.lower(), f"self.expect({name!r})" return name, f"self.{name}()" - def visit_StringLeaf(self, node: StringLeaf) -> Tuple[str, str]: + def visit_StringLeaf(self, node: StringLeaf) -> tuple[str, str]: return "literal", f"self.expect({node.value})" - def visit_NamedItem(self, node: NamedItem) -> Tuple[Optional[str], str]: + def visit_NamedItem(self, node: NamedItem) -> tuple[str | None, str]: name, call = self.visit(node.item) if node.name: name = node.name return name, call - def lookahead_call_helper(self, node: Lookahead) -> Tuple[str, str]: + def lookahead_call_helper(self, node: Lookahead) -> tuple[str, str]: name, call = self.visit(node.node) head, tail = call.split("(", 1) assert tail[-1] == ")" tail = tail[:-1] return head, tail - def visit_PositiveLookahead(self, node: PositiveLookahead) -> Tuple[None, str]: + def visit_PositiveLookahead(self, node: PositiveLookahead) -> tuple[None, str]: head, tail = self.lookahead_call_helper(node) return None, f"self.positive_lookahead({head}, {tail})" - def visit_NegativeLookahead(self, node: NegativeLookahead) -> Tuple[None, str]: + def visit_NegativeLookahead(self, node: NegativeLookahead) -> tuple[None, str]: head, tail = self.lookahead_call_helper(node) return None, f"self.negative_lookahead({head}, {tail})" - def visit_Opt(self, node: Opt) -> Tuple[str, str]: + def visit_Opt(self, node: Opt) -> tuple[str, str]: name, call = self.visit(node.node) # Note trailing comma (the call may already have one comma # at the end, for example when rules have both repeat0 and optional @@ -148,7 +149,7 @@ def _generate_artificial_rule_call( prefix: str, call_by_name_func: Callable[[str], str], rule_generation_func: Callable[[], str], - ) -> Tuple[str, str]: + ) -> tuple[str, str]: node_str = f"{node}" key = f"{prefix}_{node_str}" if key in self.cache: @@ -159,7 +160,7 @@ def _generate_artificial_rule_call( self.cache[key] = name, call return self.cache[key] - def visit_Rhs(self, node: Rhs) -> Tuple[str, str]: + def visit_Rhs(self, node: Rhs) -> tuple[str, str]: if len(node.alts) == 1 and len(node.alts[0].items) == 1: return self.visit(node.alts[0].items[0]) @@ -170,7 +171,7 @@ def visit_Rhs(self, node: Rhs) -> Tuple[str, str]: lambda: self.gen.artificial_rule_from_rhs(node), ) - def visit_Repeat0(self, node: Repeat0) -> Tuple[str, str]: + def visit_Repeat0(self, node: Repeat0) -> tuple[str, str]: return self._generate_artificial_rule_call( node, "repeat0", @@ -178,7 +179,7 @@ def visit_Repeat0(self, node: Repeat0) -> Tuple[str, str]: lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=False), ) - def visit_Repeat1(self, node: Repeat1) -> Tuple[str, str]: + def visit_Repeat1(self, node: Repeat1) -> tuple[str, str]: return self._generate_artificial_rule_call( node, "repeat1", @@ -186,7 +187,7 @@ def visit_Repeat1(self, node: Repeat1) -> Tuple[str, str]: lambda: self.gen.artificial_rule_from_repeat(node.node, is_repeat1=True), ) - def visit_Gather(self, node: Gather) -> Tuple[str, str]: + def visit_Gather(self, node: Gather) -> tuple[str, str]: return self._generate_artificial_rule_call( node, "gather", @@ -194,13 +195,13 @@ def visit_Gather(self, node: Gather) -> Tuple[str, str]: lambda: self.gen.artificial_rule_from_gather(node), ) - def visit_Group(self, node: Group) -> Tuple[Optional[str], str]: + def visit_Group(self, node: Group) -> tuple[str | None, str]: return self.visit(node.rhs) - def visit_Cut(self, node: Cut) -> Tuple[str, str]: + def visit_Cut(self, node: Cut) -> tuple[str, str]: return "cut", "True" - def visit_Forced(self, node: Forced) -> Tuple[str, str]: + def visit_Forced(self, node: Forced) -> tuple[str, str]: if isinstance(node.node, Group): _, val = self.visit(node.node.rhs) return "forced", f"self.expect_forced({val}, '''({node.node.rhs!s})''')" @@ -215,10 +216,10 @@ class PythonParserGenerator(ParserGenerator, GrammarVisitor): def __init__( self, grammar: grammar.Grammar, - file: Optional[IO[Text]], - tokens: Set[str] = set(token.tok_name.values()), - location_formatting: Optional[str] = None, - unreachable_formatting: Optional[str] = None, + file: IO[str] | None, + tokens: set[str] = set(token.tok_name.values()), + location_formatting: str | None = None, + unreachable_formatting: str | None = None, ): tokens.add("SOFT_KEYWORD") super().__init__(grammar, tokens, file) @@ -355,7 +356,7 @@ def visit_Alt(self, node: Alt, is_loop: bool, is_gather: bool) -> None: if is_loop: self.print(f"children.append({action})") - self.print(f"mark = self._mark()") + self.print("mark = self._mark()") else: if "UNREACHABLE" in action: action = action.replace("UNREACHABLE", self.unreachable_formatting) diff --git a/Tools/peg_generator/pegen/sccutils.py b/Tools/peg_generator/pegen/sccutils.py index da4c9331625dd93..51f618a14d936b8 100644 --- a/Tools/peg_generator/pegen/sccutils.py +++ b/Tools/peg_generator/pegen/sccutils.py @@ -1,11 +1,11 @@ # Adapted from mypy (mypy/build.py) under the MIT license. -from typing import * +from collections.abc import Iterable, Iterator, Set def strongly_connected_components( - vertices: AbstractSet[str], edges: Dict[str, AbstractSet[str]] -) -> Iterator[AbstractSet[str]]: + vertices: Set[str], edges: dict[str, Set[str]] +) -> Iterator[Set[str]]: """Compute Strongly Connected Components of a directed graph. Args: @@ -20,12 +20,12 @@ def strongly_connected_components( From https://code.activestate.com/recipes/578507-strongly-connected-components-of-a-directed-graph/. """ - identified: Set[str] = set() - stack: List[str] = [] - index: Dict[str, int] = {} - boundaries: List[int] = [] + identified: set[str] = set() + stack: list[str] = [] + index: dict[str, int] = {} + boundaries: list[int] = [] - def dfs(v: str) -> Iterator[Set[str]]: + def dfs(v: str) -> Iterator[set[str]]: index[v] = len(stack) stack.append(v) boundaries.append(index[v]) @@ -50,8 +50,8 @@ def dfs(v: str) -> Iterator[Set[str]]: def topsort( - data: Dict[AbstractSet[str], Set[AbstractSet[str]]] -) -> Iterable[AbstractSet[AbstractSet[str]]]: + data: dict[Set[str], set[Set[str]]] +) -> Iterable[Set[Set[str]]]: """Topological sort. Args: @@ -94,12 +94,12 @@ def topsort( break yield ready data = {item: (dep - ready) for item, dep in data.items() if item not in ready} - assert not data, "A cyclic dependency exists amongst %r" % data + assert not data, f"A cyclic dependency exists amongst {data}" def find_cycles_in_scc( - graph: Dict[str, AbstractSet[str]], scc: AbstractSet[str], start: str -) -> Iterable[List[str]]: + graph: dict[str, Set[str]], scc: Set[str], start: str +) -> Iterable[list[str]]: """Find cycles in SCC emanating from start. Yields lists of the form ['A', 'B', 'C', 'A'], which means there's @@ -117,7 +117,7 @@ def find_cycles_in_scc( assert start in graph # Recursive helper that yields cycles. - def dfs(node: str, path: List[str]) -> Iterator[List[str]]: + def dfs(node: str, path: list[str]) -> Iterator[list[str]]: if node in path: yield path + [node] return diff --git a/Tools/peg_generator/pegen/testutil.py b/Tools/peg_generator/pegen/testutil.py index 0e85b844ef152b7..46f6c7fec1bda05 100644 --- a/Tools/peg_generator/pegen/testutil.py +++ b/Tools/peg_generator/pegen/testutil.py @@ -6,7 +6,7 @@ import textwrap import token import tokenize -from typing import IO, Any, Dict, Final, Optional, Type, cast +from typing import IO, Any, Final, cast from pegen.build import compile_c_extension from pegen.c_generator import CParserGenerator @@ -23,19 +23,19 @@ } -def generate_parser(grammar: Grammar) -> Type[Parser]: +def generate_parser(grammar: Grammar) -> type[Parser]: # Generate a parser. out = io.StringIO() genr = PythonParserGenerator(grammar, out) genr.generate("<string>") # Load the generated parser class. - ns: Dict[str, Any] = {} + ns: dict[str, Any] = {} exec(out.getvalue(), ns) return ns["GeneratedParser"] -def run_parser(file: IO[bytes], parser_class: Type[Parser], *, verbose: bool = False) -> Any: +def run_parser(file: IO[bytes], parser_class: type[Parser], *, verbose: bool = False) -> Any: # Run a parser on a file (stream). tokenizer = Tokenizer(tokenize.generate_tokens(file.readline)) # type: ignore[arg-type] # typeshed issue #3515 parser = parser_class(tokenizer, verbose=verbose) @@ -46,7 +46,7 @@ def run_parser(file: IO[bytes], parser_class: Type[Parser], *, verbose: bool = F def parse_string( - source: str, parser_class: Type[Parser], *, dedent: bool = True, verbose: bool = False + source: str, parser_class: type[Parser], *, dedent: bool = True, verbose: bool = False ) -> Any: # Run the parser on a string. if dedent: @@ -55,7 +55,7 @@ def parse_string( return run_parser(file, parser_class, verbose=verbose) # type: ignore[arg-type] # typeshed issue #3515 -def make_parser(source: str) -> Type[Parser]: +def make_parser(source: str) -> type[Parser]: # Combine parse_string() and generate_parser(). grammar = parse_string(source, GrammarParser) return generate_parser(grammar) @@ -86,7 +86,7 @@ def generate_parser_c_extension( grammar: Grammar, path: pathlib.PurePath, debug: bool = False, - library_dir: Optional[str] = None, + library_dir: str | None = None, ) -> Any: """Generate a parser c extension for the given grammar in the given path diff --git a/Tools/peg_generator/pegen/tokenizer.py b/Tools/peg_generator/pegen/tokenizer.py index 7ee49e1432b77a6..fba5bac95178956 100644 --- a/Tools/peg_generator/pegen/tokenizer.py +++ b/Tools/peg_generator/pegen/tokenizer.py @@ -1,6 +1,6 @@ import token import tokenize -from typing import Dict, Iterator, List +from collections.abc import Iterator Mark = int # NewType('Mark', int) @@ -8,7 +8,11 @@ def shorttok(tok: tokenize.TokenInfo) -> str: - return "%-25.25s" % f"{tok.start[0]}.{tok.start[1]}: {token.tok_name[tok.type]}:{tok.string!r}" + formatted = ( + f"{tok.start[0]}.{tok.start[1]}: " + f"{token.tok_name[tok.type]}:{tok.string!r}" + ) + return f"{formatted:<25.25}" class Tokenizer: @@ -17,7 +21,7 @@ class Tokenizer: This is pretty tied to Python's syntax. """ - _tokens: List[tokenize.TokenInfo] + _tokens: list[tokenize.TokenInfo] def __init__( self, tokengen: Iterator[tokenize.TokenInfo], *, path: str = "", verbose: bool = False @@ -26,7 +30,7 @@ def __init__( self._tokens = [] self._index = 0 self._verbose = verbose - self._lines: Dict[int, str] = {} + self._lines: dict[int, str] = {} self._path = path if verbose: self.report(False, False) @@ -72,7 +76,7 @@ def get_last_non_whitespace_token(self) -> tokenize.TokenInfo: break return tok - def get_lines(self, line_numbers: List[int]) -> List[str]: + def get_lines(self, line_numbers: list[int]) -> list[str]: """Retrieve source lines corresponding to line numbers.""" if self._lines: lines = self._lines diff --git a/Tools/peg_generator/pegen/validator.py b/Tools/peg_generator/pegen/validator.py index 4699d5712d95226..5e2bc238a1e9661 100644 --- a/Tools/peg_generator/pegen/validator.py +++ b/Tools/peg_generator/pegen/validator.py @@ -1,4 +1,4 @@ -from typing import Optional +from typing import Any from pegen import grammar from pegen.grammar import Alt, GrammarVisitor, Rhs, Rule @@ -11,7 +11,7 @@ class ValidationError(Exception): class GrammarValidator(GrammarVisitor): def __init__(self, grammar: grammar.Grammar) -> None: self.grammar = grammar - self.rulename: Optional[str] = None + self.rulename: str | None = None def validate_rule(self, rulename: str, node: Rule) -> None: self.rulename = rulename @@ -46,6 +46,37 @@ def visit_Alt(self, node: Alt) -> None: ) +class CutValidator(GrammarValidator): + """Fail if Cut is not directly in a rule. + + For simplicity, we currently document that a Cut affects alternatives + of the *rule* it is in. + However, the implementation makes cuts local to enclosing Rhs + (e.g. parenthesized list of choices). + Additionally, in academic papers about PEG, repeats and optional items + are "desugared" to choices with an empty alternative, and thus contain + a Cut's effect. + + Please update documentation and tests when adding this cut, + then get rid of this validator. + + See gh-143054. + """ + + def visit(self, node: Any, parents: tuple[Any, ...] = ()) -> None: + super().visit(node, parents=(*parents, node)) + + def visit_Cut(self, node: Alt, parents: tuple[Any, ...] = ()) -> None: + parent_types = [type(p).__name__ for p in parents] + if parent_types != ['Rule', 'Rhs', 'Alt', 'NamedItem', 'Cut']: + raise ValidationError( + f"Rule {self.rulename!r} contains cut that's not on the " + "top level. " + "The intended semantics of such cases need " + "to be clarified; see the CutValidator docstring." + f"\nThe cut is inside: {parent_types}" + ) + def validate_grammar(the_grammar: grammar.Grammar) -> None: for validator_cls in GrammarValidator.__subclasses__(): validator = validator_cls(the_grammar) diff --git a/Tools/requirements-dev.txt b/Tools/requirements-dev.txt index 5bf180bb30a310e..46381ea58a12382 100644 --- a/Tools/requirements-dev.txt +++ b/Tools/requirements-dev.txt @@ -1,7 +1,7 @@ # Requirements file for external linters and checks we run on # Tools/clinic, Tools/cases_generator/, and Tools/peg_generator/ in CI -mypy==1.15 +mypy==2.1.0 # needed for peg_generator: -types-psutil==6.0.0.20240901 -types-setuptools==74.0.0.20240831 +types-psutil==7.2.2.20260508 +types-setuptools==82.0.0.20260508 diff --git a/Tools/requirements-hypothesis.txt b/Tools/requirements-hypothesis.txt index 66898885c0a4121..e5deac497fbe3f2 100644 --- a/Tools/requirements-hypothesis.txt +++ b/Tools/requirements-hypothesis.txt @@ -1,4 +1,4 @@ # Requirements file for hypothesis that # we use to run our property-based tests in CI. -hypothesis==6.111.2 +hypothesis==6.135.26 diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 68cfad3f92cdc78..905af9dcfd802ea 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -492,7 +492,7 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: ): (trace_too_long, attempts), Doc( "Trace too short", - "A potential trace is abandoned because it it too short.", + "A potential trace is abandoned because it is too short.", ): (trace_too_short, attempts), Doc( "Inner loop found", "A trace is truncated because it has an inner loop" diff --git a/Tools/ssl/multissltests.py b/Tools/ssl/multissltests.py index b1a5df91901fc6d..cb349236a034c40 100755 --- a/Tools/ssl/multissltests.py +++ b/Tools/ssl/multissltests.py @@ -44,14 +44,16 @@ OPENSSL_OLD_VERSIONS = [ "1.1.1w", + "3.1.8", + "3.2.6", + "3.3.7", ] OPENSSL_RECENT_VERSIONS = [ - "3.0.16", - "3.1.8", - "3.2.4", - "3.3.3", - "3.4.1", + "3.0.21", + "3.4.6", + "3.5.7", + "3.6.3", # See make_ssl_data.py for notes on adding a new version. ] @@ -70,9 +72,8 @@ parser = argparse.ArgumentParser( prog='multissl', description=( - "Run CPython tests with multiple OpenSSL and LibreSSL " - "versions." - ) + "Run CPython tests with multiple cryptography libraries/versions." + ), ) parser.add_argument( '--debug', @@ -146,6 +147,12 @@ dest='keep_sources', help="Keep original sources for debugging." ) +parser.add_argument( + '--tsan', + action='store_true', + dest='tsan', + help="Build with thread sanitizer. (Disables fips in OpenSSL 3.x)." +) class AbstractBuilder(object): @@ -294,12 +301,14 @@ def _unpack_src(self): raise ValueError(member.name, base) member.name = member.name[len(base):].lstrip('/') log.info("Unpacking files to {}".format(self.build_dir)) - tf.extractall(self.build_dir, members) + tf.extractall(self.build_dir, members, filter='data') def _build_src(self, config_args=()): """Now build openssl""" log.info("Running build in {}".format(self.build_dir)) cwd = self.build_dir + if self.args.tsan: + config_args += ("-fsanitize=thread",) cmd = [ "./config", *config_args, "shared", "--debug", diff --git a/Tools/tsan/suppressions_free_threading.txt b/Tools/tsan/suppressions_free_threading.txt index 21224e490b8160b..c10af91faae17cc 100644 --- a/Tools/tsan/suppressions_free_threading.txt +++ b/Tools/tsan/suppressions_free_threading.txt @@ -46,4 +46,9 @@ race:list_inplace_repeat_lock_held # PyObject_Realloc internally does memcpy which isn't atomic so can race # with non-locking reads. See #132070 -race:PyObject_Realloc \ No newline at end of file +race:PyObject_Realloc + +# gh-133467. Some of these could be hard to trigger. +race_top:update_one_slot +race_top:set_tp_bases +race_top:type_set_bases_unlocked diff --git a/Tools/unicode/makeunicodedata.py b/Tools/unicode/makeunicodedata.py index 889ae8fc869b8ad..6b1d7da26e28e65 100644 --- a/Tools/unicode/makeunicodedata.py +++ b/Tools/unicode/makeunicodedata.py @@ -43,7 +43,7 @@ # When changing UCD version please update # * Doc/library/stdtypes.rst, and # * Doc/library/unicodedata.rst -# * Doc/reference/lexical_analysis.rst (two occurrences) +# * Doc/reference/lexical_analysis.rst (three occurrences) UNIDATA_VERSION = "16.0.0" UNICODE_DATA = "UnicodeData%s.txt" COMPOSITION_EXCLUSIONS = "CompositionExclusions%s.txt" @@ -99,18 +99,13 @@ CASED_MASK = 0x2000 EXTENDED_CASE_MASK = 0x4000 -# these ranges need to match unicodedata.c:is_unified_ideograph -cjk_ranges = [ - ('3400', '4DBF'), # CJK Ideograph Extension A CJK - ('4E00', '9FFF'), # CJK Ideograph - ('20000', '2A6DF'), # CJK Ideograph Extension B - ('2A700', '2B739'), # CJK Ideograph Extension C - ('2B740', '2B81D'), # CJK Ideograph Extension D - ('2B820', '2CEA1'), # CJK Ideograph Extension E - ('2CEB0', '2EBE0'), # CJK Ideograph Extension F - ('2EBF0', '2EE5D'), # CJK Ideograph Extension I - ('30000', '3134A'), # CJK Ideograph Extension G - ('31350', '323AF'), # CJK Ideograph Extension H +# Maps the range names in UnicodeData.txt to prefixes for +# derived names specified by rule NR2. +# Hangul should always be at index 0, since it uses special format. +derived_name_range_names = [ + ("Hangul Syllable", "HANGUL SYLLABLE "), + ("CJK Ideograph", "CJK UNIFIED IDEOGRAPH-"), + ("Tangut Ideograph", "TANGUT IDEOGRAPH-"), ] @@ -124,7 +119,7 @@ def maketables(trace=0): for version in old_versions: print("--- Reading", UNICODE_DATA % ("-"+version), "...") - old_unicode = UnicodeData(version, cjk_check=False) + old_unicode = UnicodeData(version, ideograph_check=False) print(len(list(filter(None, old_unicode.table))), "characters") merge_old_version(version, unicode, old_unicode) @@ -698,6 +693,23 @@ def makeunicodename(unicode, trace): fprint(' {%d, {%s}},' % (len(sequence), seq_str)) fprint('};') + fprint(dedent(""" + typedef struct { + Py_UCS4 first; + Py_UCS4 last; + int prefixid; + } derived_name_range; + """)) + + fprint('static const derived_name_range derived_name_ranges[] = {') + for name_range in unicode.derived_name_ranges: + fprint(' {0x%s, 0x%s, %d},' % name_range) + fprint('};') + + fprint('static const char * const derived_name_prefixes[] = {') + for _, prefix in derived_name_range_names: + fprint(' "%s",' % prefix) + fprint('};') def merge_old_version(version, new, old): # Changes to exclusion file not implemented yet @@ -905,14 +917,14 @@ def from_row(row: List[str]) -> UcdRecord: class UnicodeData: # table: List[Optional[UcdRecord]] # index is codepoint; None means unassigned - def __init__(self, version, cjk_check=True): + def __init__(self, version, ideograph_check=True): self.changed = [] table = [None] * 0x110000 for s in UcdFile(UNICODE_DATA, version): char = int(s[0], 16) table[char] = from_row(s) - cjk_ranges_found = [] + self.derived_name_ranges = [] # expand first-last ranges field = None @@ -926,15 +938,15 @@ def __init__(self, version, cjk_check=True): s.name = "" field = dataclasses.astuple(s)[:15] elif s.name[-5:] == "Last>": - if s.name.startswith("<CJK Ideograph"): - cjk_ranges_found.append((field[0], - s.codepoint)) + for j, (rangename, _) in enumerate(derived_name_range_names): + if s.name.startswith("<" + rangename): + self.derived_name_ranges.append( + (field[0], s.codepoint, j)) + break s.name = "" field = None elif field: table[i] = from_row(('%X' % i,) + field[1:]) - if cjk_check and cjk_ranges != cjk_ranges_found: - raise ValueError("CJK ranges deviate: have %r" % cjk_ranges_found) # public attributes self.filename = UNICODE_DATA % '' diff --git a/Tools/wasm/.ruff.toml b/Tools/wasm/.ruff.toml new file mode 100644 index 000000000000000..3d8e59fa3f22c41 --- /dev/null +++ b/Tools/wasm/.ruff.toml @@ -0,0 +1,25 @@ +extend = "../../.ruff.toml" # Inherit the project-wide settings + +[format] +preview = true +docstring-code-format = true + +[lint] +select = [ + "C4", # flake8-comprehensions + "E", # pycodestyle + "F", # pyflakes + "I", # isort + "ISC", # flake8-implicit-str-concat + "LOG", # flake8-logging + "PGH", # pygrep-hooks + "PT", # flake8-pytest-style + "PYI", # flake8-pyi + "RUF100", # Ban unused `# noqa` comments + "UP", # pyupgrade + "W", # pycodestyle + "YTT", # flake8-2020 +] +ignore = [ + "E501", # Line too long +] diff --git a/Tools/wasm/emscripten/__main__.py b/Tools/wasm/emscripten/__main__.py index 849bd5de44eb7b6..29890cc1a2f3652 100644 --- a/Tools/wasm/emscripten/__main__.py +++ b/Tools/wasm/emscripten/__main__.py @@ -1,398 +1,14 @@ -#!/usr/bin/env python3 - -import argparse -import contextlib -import functools -import os -import shutil -import subprocess -import sys -import sysconfig -import tempfile -from urllib.request import urlopen -from pathlib import Path -from textwrap import dedent - -try: - from os import process_cpu_count as cpu_count -except ImportError: - from os import cpu_count - - -EMSCRIPTEN_DIR = Path(__file__).parent -CHECKOUT = EMSCRIPTEN_DIR.parent.parent.parent - -CROSS_BUILD_DIR = CHECKOUT / "cross-build" -NATIVE_BUILD_DIR = CROSS_BUILD_DIR / "build" -HOST_TRIPLE = "wasm32-emscripten" - -DOWNLOAD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build" -HOST_BUILD_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "build" -HOST_DIR = HOST_BUILD_DIR / "python" -PREFIX_DIR = CROSS_BUILD_DIR / HOST_TRIPLE / "prefix" - -LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" -LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/emscripten.py\n".encode("utf-8") - - -def updated_env(updates={}): - """Create a new dict representing the environment to use. - - The changes made to the execution environment are printed out. - """ - env_defaults = {} - # https://reproducible-builds.org/docs/source-date-epoch/ - git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] - try: - epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() - env_defaults["SOURCE_DATE_EPOCH"] = epoch - except subprocess.CalledProcessError: - pass # Might be building from a tarball. - # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. - environment = env_defaults | os.environ | updates - - env_diff = {} - for key, value in environment.items(): - if os.environ.get(key) != value: - env_diff[key] = value - - print("🌎 Environment changes:") - for key in sorted(env_diff.keys()): - print(f" {key}={env_diff[key]}") - - return environment - - -def subdir(working_dir, *, clean_ok=False): - """Decorator to change to a working directory.""" - - def decorator(func): - @functools.wraps(func) - def wrapper(context): - try: - tput_output = subprocess.check_output( - ["tput", "cols"], encoding="utf-8" - ) - terminal_width = int(tput_output.strip()) - except subprocess.CalledProcessError: - terminal_width = 80 - print("⎯" * terminal_width) - print("📁", working_dir) - if clean_ok and getattr(context, "clean", False) and working_dir.exists(): - print("🚮 Deleting directory (--clean)...") - shutil.rmtree(working_dir) - - working_dir.mkdir(parents=True, exist_ok=True) - - with contextlib.chdir(working_dir): - return func(context, working_dir) - - return wrapper - - return decorator - - -def call(command, *, quiet, **kwargs): - """Execute a command. - - If 'quiet' is true, then redirect stdout and stderr to a temporary file. - """ - print("❯", " ".join(map(str, command))) - if not quiet: - stdout = None - stderr = None - else: - stdout = tempfile.NamedTemporaryFile( - "w", - encoding="utf-8", - delete=False, - prefix="cpython-emscripten-", - suffix=".log", - ) - stderr = subprocess.STDOUT - print(f"📝 Logging output to {stdout.name} (--quiet)...") - - subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) - - -def build_platform(): - """The name of the build/host platform.""" - # Can also be found via `config.guess`.` - return sysconfig.get_config_var("BUILD_GNU_TYPE") - - -def build_python_path(): - """The path to the build Python binary.""" - binary = NATIVE_BUILD_DIR / "python" - if not binary.is_file(): - binary = binary.with_suffix(".exe") - if not binary.is_file(): - raise FileNotFoundError("Unable to find `python(.exe)` in " f"{NATIVE_BUILD_DIR}") - - return binary - - -@subdir(NATIVE_BUILD_DIR, clean_ok=True) -def configure_build_python(context, working_dir): - """Configure the build/host Python.""" - if LOCAL_SETUP.exists(): - print(f"👍 {LOCAL_SETUP} exists ...") - else: - print(f"📝 Touching {LOCAL_SETUP} ...") - LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) - - configure = [os.path.relpath(CHECKOUT / "configure", working_dir)] - if context.args: - configure.extend(context.args) - - call(configure, quiet=context.quiet) - - -@subdir(NATIVE_BUILD_DIR) -def make_build_python(context, working_dir): - """Make/build the build Python.""" - call(["make", "--jobs", str(cpu_count()), "all"], quiet=context.quiet) - - binary = build_python_path() - cmd = [ - binary, - "-c", - "import sys; " "print(f'{sys.version_info.major}.{sys.version_info.minor}')", - ] - version = subprocess.check_output(cmd, encoding="utf-8").strip() - - print(f"🎉 {binary} {version}") - - -@subdir(HOST_BUILD_DIR, clean_ok=True) -def make_emscripten_libffi(context, working_dir): - shutil.rmtree(working_dir / "libffi-3.4.6", ignore_errors=True) - with tempfile.NamedTemporaryFile(suffix=".tar.gz") as tmp_file: - with urlopen( - "https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz" - ) as response: - shutil.copyfileobj(response, tmp_file) - shutil.unpack_archive(tmp_file.name, working_dir) - call( - [EMSCRIPTEN_DIR / "make_libffi.sh"], - env=updated_env({"PREFIX": PREFIX_DIR}), - cwd=working_dir / "libffi-3.4.6", - quiet=context.quiet, - ) - - -@subdir(HOST_DIR, clean_ok=True) -def configure_emscripten_python(context, working_dir): - """Configure the emscripten/host build.""" - config_site = os.fsdecode( - CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-emscripten" - ) - - emscripten_build_dir = working_dir.relative_to(CHECKOUT) - - python_build_dir = NATIVE_BUILD_DIR / "build" - lib_dirs = list(python_build_dir.glob("lib.*")) - assert ( - len(lib_dirs) == 1 - ), f"Expected a single lib.* directory in {python_build_dir}" - lib_dir = os.fsdecode(lib_dirs[0]) - pydebug = lib_dir.endswith("-pydebug") - python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] - sysconfig_data = ( - f"{emscripten_build_dir}/build/lib.emscripten-wasm32-{python_version}" - ) - if pydebug: - sysconfig_data += "-pydebug" - - host_runner = context.host_runner - pkg_config_path_dir = (PREFIX_DIR / "lib/pkgconfig/").resolve() - env_additions = { - "CONFIG_SITE": config_site, - "HOSTRUNNER": host_runner, - "EM_PKG_CONFIG_PATH": str(pkg_config_path_dir), - } - build_python = os.fsdecode(build_python_path()) - configure = [ - "emconfigure", - os.path.relpath(CHECKOUT / "configure", working_dir), - "CFLAGS=-DPY_CALL_TRAMPOLINE -sUSE_BZIP2", - "PKG_CONFIG=pkg-config", - f"--host={HOST_TRIPLE}", - f"--build={build_platform()}", - f"--with-build-python={build_python}", - "--without-pymalloc", - "--disable-shared", - "--disable-ipv6", - "--enable-big-digits=30", - "--enable-wasm-dynamic-linking", - f"--prefix={PREFIX_DIR}", - ] - if pydebug: - configure.append("--with-pydebug") - if context.args: - configure.extend(context.args) - call( - configure, - env=updated_env(env_additions), - quiet=context.quiet, - ) - - shutil.copy(EMSCRIPTEN_DIR / "node_entry.mjs", working_dir / "node_entry.mjs") - - node_entry = working_dir / "node_entry.mjs" - exec_script = working_dir / "python.sh" - exec_script.write_text( - dedent( - f"""\ - #!/bin/sh - - # Macs come with FreeBSD coreutils which doesn't have the -s option - # so feature detect and work around it. - if which grealpath > /dev/null 2>&1; then - # It has brew installed gnu core utils, use that - REALPATH="grealpath -s" - elif which realpath > /dev/null 2>&1 && realpath --version > /dev/null 2>&1 && realpath --version | grep GNU > /dev/null 2>&1; then - # realpath points to GNU realpath so use it. - REALPATH="realpath -s" - else - # Shim for macs without GNU coreutils - abs_path () {{ - echo "$(cd "$(dirname "$1")" || exit; pwd)/$(basename "$1")" - }} - REALPATH=abs_path - fi - - # We compute our own path, not following symlinks and pass it in so that - # node_entry.mjs can set sys.executable correctly. - # Intentionally allow word splitting on NODEFLAGS. - exec {host_runner} $NODEFLAGS {node_entry} --this-program="$($REALPATH "$0")" "$@" - """ - ) - ) - exec_script.chmod(0o755) - print(f"🏃‍♀️ Created {exec_script} ... ") - sys.stdout.flush() - - -@subdir(HOST_DIR) -def make_emscripten_python(context, working_dir): - """Run `make` for the emscripten/host build.""" - call( - ["make", "--jobs", str(cpu_count()), "all"], - env=updated_env(), - quiet=context.quiet, - ) - - exec_script = working_dir / "python.sh" - subprocess.check_call([exec_script, "--version"]) - - -def build_all(context): - """Build everything.""" - steps = [ - configure_build_python, - make_build_python, - make_emscripten_libffi, - configure_emscripten_python, - make_emscripten_python, - ] - for step in steps: - step(context) - - -def clean_contents(context): - """Delete all files created by this script.""" - if CROSS_BUILD_DIR.exists(): - print(f"🧹 Deleting {CROSS_BUILD_DIR} ...") - shutil.rmtree(CROSS_BUILD_DIR) - - if LOCAL_SETUP.exists(): - with LOCAL_SETUP.open("rb") as file: - if file.read(len(LOCAL_SETUP_MARKER)) == LOCAL_SETUP_MARKER: - print(f"🧹 Deleting generated {LOCAL_SETUP} ...") - - -def main(): - default_host_runner = "node" +if __name__ == "__main__": + import pathlib + import runpy + import sys - parser = argparse.ArgumentParser() - subcommands = parser.add_subparsers(dest="subcommand") - build = subcommands.add_parser("build", help="Build everything") - configure_build = subcommands.add_parser( - "configure-build-python", help="Run `configure` for the " "build Python" - ) - make_libffi_cmd = subcommands.add_parser( - "make-libffi", help="Clone libffi repo, configure and build it for emscripten" + print( + "⚠️ WARNING: This script is deprecated and slated for removal in Python 3.20; " + "execute the `Platforms/emscripten/` directory instead (i.e. `python Platforms/emscripten`)\n", + file=sys.stderr, ) - make_build = subcommands.add_parser( - "make-build-python", help="Run `make` for the build Python" - ) - configure_host = subcommands.add_parser( - "configure-host", - help="Run `configure` for the host/emscripten (pydebug builds are inferred from the build Python)", - ) - make_host = subcommands.add_parser( - "make-host", help="Run `make` for the host/emscripten" - ) - clean = subcommands.add_parser( - "clean", help="Delete files and directories created by this script" - ) - for subcommand in ( - build, - configure_build, - make_libffi_cmd, - make_build, - configure_host, - make_host, - ): - subcommand.add_argument( - "--quiet", - action="store_true", - default=False, - dest="quiet", - help="Redirect output from subprocesses to a log file", - ) - for subcommand in configure_build, configure_host: - subcommand.add_argument( - "--clean", - action="store_true", - default=False, - dest="clean", - help="Delete any relevant directories before building", - ) - for subcommand in build, configure_build, configure_host: - subcommand.add_argument( - "args", nargs="*", help="Extra arguments to pass to `configure`" - ) - for subcommand in build, configure_host: - subcommand.add_argument( - "--host-runner", - action="store", - default=default_host_runner, - dest="host_runner", - help="Command template for running the emscripten host" - f"`{default_host_runner}`)", - ) - - context = parser.parse_args() - dispatch = { - "make-libffi": make_emscripten_libffi, - "configure-build-python": configure_build_python, - "make-build-python": make_build_python, - "configure-host": configure_emscripten_python, - "make-host": make_emscripten_python, - "build": build_all, - "clean": clean_contents, - } - - if not context.subcommand: - # No command provided, display help and exit - print("Expected one of", ", ".join(sorted(dispatch.keys())), file=sys.stderr) - parser.print_help(sys.stderr) - sys.exit(1) - dispatch[context.subcommand](context) - - -if __name__ == "__main__": - main() + checkout = pathlib.Path(__file__).parents[3] + emscripten_dir = (checkout / "Platforms/emscripten").absolute() + runpy.run_path(str(emscripten_dir), run_name="__main__") diff --git a/Tools/wasm/emscripten/browser_test/run_test.sh b/Tools/wasm/emscripten/browser_test/run_test.sh new file mode 100755 index 000000000000000..ed8cae7bf23b29e --- /dev/null +++ b/Tools/wasm/emscripten/browser_test/run_test.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# Redirect to new location +exec "$(dirname "$0")/../../../../Platforms/emscripten/browser_test/run_test.sh" "$@" diff --git a/Tools/wasm/mypy.ini b/Tools/wasm/mypy.ini deleted file mode 100644 index 4de0a30c260f5fb..000000000000000 --- a/Tools/wasm/mypy.ini +++ /dev/null @@ -1,11 +0,0 @@ -[mypy] -files = Tools/wasm/wasm_*.py -pretty = True -show_traceback = True - -# Make sure the wasm can be run using Python 3.8: -python_version = 3.8 - -# Be strict... -strict = True -enable_error_code = truthy-bool,ignore-without-code diff --git a/Tools/wasm/wasi-env b/Tools/wasm/wasi-env index 4c5078a1f675e21..08d4f499baa85c3 100755 --- a/Tools/wasm/wasi-env +++ b/Tools/wasm/wasi-env @@ -1,7 +1,8 @@ #!/bin/sh set -e -# NOTE: to be removed once no longer used in https://github.com/python/buildmaster-config/blob/main/master/custom/factories.py . +# NOTE: to be removed once no longer used in https://github.com/python/buildmaster-config/blob/main/master/custom/factories.py ; +# expected in Python 3.18 as 3.13 is when `wasi.py` was introduced. # function usage() { diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index a742043e4be1d2d..af55e03d10f7548 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -1,367 +1,12 @@ -#!/usr/bin/env python3 +if __name__ == "__main__": + import pathlib + import runpy + import sys -import argparse -import contextlib -import functools -import os -try: - from os import process_cpu_count as cpu_count -except ImportError: - from os import cpu_count -import pathlib -import shutil -import subprocess -import sys -import sysconfig -import tempfile - - -CHECKOUT = pathlib.Path(__file__).parent.parent.parent - -CROSS_BUILD_DIR = CHECKOUT / "cross-build" -BUILD_DIR = CROSS_BUILD_DIR / "build" - -LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" -LOCAL_SETUP_MARKER = "# Generated by Tools/wasm/wasi.py\n".encode("utf-8") - -WASMTIME_VAR_NAME = "WASMTIME" -WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}" - - -def updated_env(updates={}): - """Create a new dict representing the environment to use. - - The changes made to the execution environment are printed out. - """ - env_defaults = {} - # https://reproducible-builds.org/docs/source-date-epoch/ - git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] - try: - epoch = subprocess.check_output(git_epoch_cmd, encoding="utf-8").strip() - env_defaults["SOURCE_DATE_EPOCH"] = epoch - except subprocess.CalledProcessError: - pass # Might be building from a tarball. - # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. - environment = env_defaults | os.environ | updates - - env_diff = {} - for key, value in environment.items(): - if os.environ.get(key) != value: - env_diff[key] = value - - print("🌎 Environment changes:") - for key in sorted(env_diff.keys()): - print(f" {key}={env_diff[key]}") - - return environment - - -def subdir(working_dir, *, clean_ok=False): - """Decorator to change to a working directory.""" - def decorator(func): - @functools.wraps(func) - def wrapper(context): - nonlocal working_dir - - if callable(working_dir): - working_dir = working_dir(context) - try: - tput_output = subprocess.check_output(["tput", "cols"], - encoding="utf-8") - except subprocess.CalledProcessError: - terminal_width = 80 - else: - terminal_width = int(tput_output.strip()) - print("⎯" * terminal_width) - print("📁", working_dir) - if (clean_ok and getattr(context, "clean", False) and - working_dir.exists()): - print(f"🚮 Deleting directory (--clean)...") - shutil.rmtree(working_dir) - - working_dir.mkdir(parents=True, exist_ok=True) - - with contextlib.chdir(working_dir): - return func(context, working_dir) - - return wrapper - - return decorator - - -def call(command, *, quiet, **kwargs): - """Execute a command. - - If 'quiet' is true, then redirect stdout and stderr to a temporary file. - """ - print("❯", " ".join(map(str, command))) - if not quiet: - stdout = None - stderr = None - else: - stdout = tempfile.NamedTemporaryFile("w", encoding="utf-8", - delete=False, - prefix="cpython-wasi-", - suffix=".log") - stderr = subprocess.STDOUT - print(f"📝 Logging output to {stdout.name} (--quiet)...") - - subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) - - -def build_platform(): - """The name of the build/host platform.""" - # Can also be found via `config.guess`.` - return sysconfig.get_config_var("BUILD_GNU_TYPE") - - -def build_python_path(): - """The path to the build Python binary.""" - binary = BUILD_DIR / "python" - if not binary.is_file(): - binary = binary.with_suffix(".exe") - if not binary.is_file(): - raise FileNotFoundError("Unable to find `python(.exe)` in " - f"{BUILD_DIR}") - - return binary - - -@subdir(BUILD_DIR, clean_ok=True) -def configure_build_python(context, working_dir): - """Configure the build/host Python.""" - if LOCAL_SETUP.exists(): - print(f"👍 {LOCAL_SETUP} exists ...") - else: - print(f"📝 Touching {LOCAL_SETUP} ...") - LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) - - configure = [os.path.relpath(CHECKOUT / 'configure', working_dir)] - if context.args: - configure.extend(context.args) - - call(configure, quiet=context.quiet) - - -@subdir(BUILD_DIR) -def make_build_python(context, working_dir): - """Make/build the build Python.""" - call(["make", "--jobs", str(cpu_count()), "all"], - quiet=context.quiet) - - binary = build_python_path() - cmd = [binary, "-c", - "import sys; " - "print(f'{sys.version_info.major}.{sys.version_info.minor}')"] - version = subprocess.check_output(cmd, encoding="utf-8").strip() - - print(f"🎉 {binary} {version}") - - -def find_wasi_sdk(): - """Find the path to wasi-sdk.""" - if wasi_sdk_path := os.environ.get("WASI_SDK_PATH"): - return pathlib.Path(wasi_sdk_path) - elif (default_path := pathlib.Path("/opt/wasi-sdk")).exists(): - return default_path - - -def wasi_sdk_env(context): - """Calculate environment variables for building with wasi-sdk.""" - wasi_sdk_path = context.wasi_sdk_path - sysroot = wasi_sdk_path / "share" / "wasi-sysroot" - env = {"CC": "clang", "CPP": "clang-cpp", "CXX": "clang++", - "AR": "llvm-ar", "RANLIB": "ranlib"} - - for env_var, binary_name in list(env.items()): - env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) - - if wasi_sdk_path != pathlib.Path("/opt/wasi-sdk"): - for compiler in ["CC", "CPP", "CXX"]: - env[compiler] += f" --sysroot={sysroot}" - - env["PKG_CONFIG_PATH"] = "" - env["PKG_CONFIG_LIBDIR"] = os.pathsep.join( - map(os.fsdecode, - [sysroot / "lib" / "pkgconfig", - sysroot / "share" / "pkgconfig"])) - env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot) - - env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path) - env["WASI_SYSROOT"] = os.fsdecode(sysroot) - - env["PATH"] = os.pathsep.join([os.fsdecode(wasi_sdk_path / "bin"), - os.environ["PATH"]]) - - return env - - -@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True) -def configure_wasi_python(context, working_dir): - """Configure the WASI/host build.""" - if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): - raise ValueError("WASI-SDK not found; " - "download from " - "https://github.com/WebAssembly/wasi-sdk and/or " - "specify via $WASI_SDK_PATH or --wasi-sdk") - - config_site = os.fsdecode(CHECKOUT / "Tools" / "wasm" / "config.site-wasm32-wasi") - - wasi_build_dir = working_dir.relative_to(CHECKOUT) - - python_build_dir = BUILD_DIR / "build" - lib_dirs = list(python_build_dir.glob("lib.*")) - assert len(lib_dirs) == 1, f"Expected a single lib.* directory in {python_build_dir}" - lib_dir = os.fsdecode(lib_dirs[0]) - pydebug = lib_dir.endswith("-pydebug") - python_version = lib_dir.removesuffix("-pydebug").rpartition("-")[-1] - sysconfig_data = f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" - if pydebug: - sysconfig_data += "-pydebug" - - # Use PYTHONPATH to include sysconfig data which must be anchored to the - # WASI guest's `/` directory. - args = {"GUEST_DIR": "/", - "HOST_DIR": CHECKOUT, - "ENV_VAR_NAME": "PYTHONPATH", - "ENV_VAR_VALUE": f"/{sysconfig_data}", - "PYTHON_WASM": working_dir / "python.wasm"} - # Check dynamically for wasmtime in case it was specified manually via - # `--host-runner`. - if WASMTIME_HOST_RUNNER_VAR in context.host_runner: - if wasmtime := shutil.which("wasmtime"): - args[WASMTIME_VAR_NAME] = wasmtime - else: - raise FileNotFoundError("wasmtime not found; download from " - "https://github.com/bytecodealliance/wasmtime") - host_runner = context.host_runner.format_map(args) - env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} - build_python = os.fsdecode(build_python_path()) - # The path to `configure` MUST be relative, else `python.wasm` is unable - # to find the stdlib due to Python not recognizing that it's being - # executed from within a checkout. - configure = [os.path.relpath(CHECKOUT / 'configure', working_dir), - f"--host={context.host_triple}", - f"--build={build_platform()}", - f"--with-build-python={build_python}"] - if pydebug: - configure.append("--with-pydebug") - if context.args: - configure.extend(context.args) - call(configure, - env=updated_env(env_additions | wasi_sdk_env(context)), - quiet=context.quiet) - - python_wasm = working_dir / "python.wasm" - exec_script = working_dir / "python.sh" - with exec_script.open("w", encoding="utf-8") as file: - file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n') - exec_script.chmod(0o755) - print(f"🏃‍♀️ Created {exec_script} ... ") - sys.stdout.flush() - - -@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple) -def make_wasi_python(context, working_dir): - """Run `make` for the WASI/host build.""" - call(["make", "--jobs", str(cpu_count()), "all"], - env=updated_env(), - quiet=context.quiet) - - exec_script = working_dir / "python.sh" - subprocess.check_call([exec_script, "--version"]) print( - f"🎉 Use '{exec_script.relative_to(context.init_dir)}' " - "to run CPython in wasm runtime" + "⚠️ WARNING: This script is deprecated and slated for removal in Python 3.20; " + "execute the `wasi/` directory instead (i.e. `python Tools/wasm/wasi`)\n", + file=sys.stderr, ) - -def build_all(context): - """Build everything.""" - steps = [configure_build_python, make_build_python, configure_wasi_python, - make_wasi_python] - for step in steps: - step(context) - -def clean_contents(context): - """Delete all files created by this script.""" - if CROSS_BUILD_DIR.exists(): - print(f"🧹 Deleting {CROSS_BUILD_DIR} ...") - shutil.rmtree(CROSS_BUILD_DIR) - - if LOCAL_SETUP.exists(): - with LOCAL_SETUP.open("rb") as file: - if file.read(len(LOCAL_SETUP_MARKER)) == LOCAL_SETUP_MARKER: - print(f"🧹 Deleting generated {LOCAL_SETUP} ...") - - -def main(): - default_host_runner = (f"{WASMTIME_HOST_RUNNER_VAR} run " - # Make sure the stack size will work for a pydebug - # build. - # Use 16 MiB stack. - "--wasm max-wasm-stack=16777216 " - # Enable thread support; causes use of preview1. - #"--wasm threads=y --wasi threads=y " - # Map the checkout to / to load the stdlib from /Lib. - "--dir {HOST_DIR}::{GUEST_DIR} " - # Set PYTHONPATH to the sysconfig data. - "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}") - - parser = argparse.ArgumentParser() - subcommands = parser.add_subparsers(dest="subcommand") - build = subcommands.add_parser("build", help="Build everything") - configure_build = subcommands.add_parser("configure-build-python", - help="Run `configure` for the " - "build Python") - make_build = subcommands.add_parser("make-build-python", - help="Run `make` for the build Python") - configure_host = subcommands.add_parser("configure-host", - help="Run `configure` for the " - "host/WASI (pydebug builds " - "are inferred from the build " - "Python)") - make_host = subcommands.add_parser("make-host", - help="Run `make` for the host/WASI") - clean = subcommands.add_parser("clean", help="Delete files and directories " - "created by this script") - for subcommand in build, configure_build, make_build, configure_host, make_host: - subcommand.add_argument("--quiet", action="store_true", default=False, - dest="quiet", - help="Redirect output from subprocesses to a log file") - for subcommand in configure_build, configure_host: - subcommand.add_argument("--clean", action="store_true", default=False, - dest="clean", - help="Delete any relevant directories before building") - for subcommand in build, configure_build, configure_host: - subcommand.add_argument("args", nargs="*", - help="Extra arguments to pass to `configure`") - for subcommand in build, configure_host: - subcommand.add_argument("--wasi-sdk", type=pathlib.Path, - dest="wasi_sdk_path", - default=find_wasi_sdk(), - help="Path to wasi-sdk; defaults to " - "$WASI_SDK_PATH or /opt/wasi-sdk") - subcommand.add_argument("--host-runner", action="store", - default=default_host_runner, dest="host_runner", - help="Command template for running the WASI host " - "(default designed for wasmtime 14 or newer: " - f"`{default_host_runner}`)") - for subcommand in build, configure_host, make_host: - subcommand.add_argument("--host-triple", action="store", default="wasm32-wasip1", - help="The target triple for the WASI host build") - - context = parser.parse_args() - context.init_dir = pathlib.Path().absolute() - - dispatch = {"configure-build-python": configure_build_python, - "make-build-python": make_build_python, - "configure-host": configure_wasi_python, - "make-host": make_wasi_python, - "build": build_all, - "clean": clean_contents} - dispatch[context.subcommand](context) - - -if __name__ == "__main__": - main() + runpy.run_path(pathlib.Path(__file__).parent / "wasi", run_name="__main__") diff --git a/Tools/wasm/wasi/__main__.py b/Tools/wasm/wasi/__main__.py new file mode 100644 index 000000000000000..26b14d24e952ac9 --- /dev/null +++ b/Tools/wasm/wasi/__main__.py @@ -0,0 +1,589 @@ +#!/usr/bin/env python3 + +import argparse +import contextlib +import functools +import os + +try: + from os import process_cpu_count as cpu_count +except ImportError: + from os import cpu_count +import pathlib +import shutil +import subprocess +import sys +import sysconfig +import tempfile + +CHECKOUT = pathlib.Path(__file__).parent.parent.parent.parent +assert (CHECKOUT / "configure").is_file(), ( + "Please update the location of the file" +) + +CROSS_BUILD_DIR = CHECKOUT / "cross-build" +# Build platform can also be found via `config.guess`. +BUILD_DIR = CROSS_BUILD_DIR / sysconfig.get_config_var("BUILD_GNU_TYPE") + +LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local" +LOCAL_SETUP_MARKER = ( + b"# Generated by Tools/wasm/wasi .\n" + b"# Required to statically build extension modules." +) + +WASI_SDK_VERSION = 24 + +WASMTIME_VAR_NAME = "WASMTIME" +WASMTIME_HOST_RUNNER_VAR = f"{{{WASMTIME_VAR_NAME}}}" + + +def separator(): + """Print a separator line across the terminal width.""" + try: + tput_output = subprocess.check_output( + ["tput", "cols"], encoding="utf-8" + ) + except subprocess.CalledProcessError: + terminal_width = 80 + else: + terminal_width = int(tput_output.strip()) + print("⎯" * terminal_width) + + +def log(emoji, message, *, spacing=None): + """Print a notification with an emoji. + + If 'spacing' is None, calculate the spacing based on the number of code points + in the emoji as terminals "eat" a space when the emoji has multiple code points. + """ + if spacing is None: + spacing = " " if len(emoji) == 1 else " " + print("".join([emoji, spacing, message])) + + +def updated_env(updates={}): + """Create a new dict representing the environment to use. + + The changes made to the execution environment are printed out. + """ + env_defaults = {} + # https://reproducible-builds.org/docs/source-date-epoch/ + git_epoch_cmd = ["git", "log", "-1", "--pretty=%ct"] + try: + epoch = subprocess.check_output( + git_epoch_cmd, encoding="utf-8" + ).strip() + env_defaults["SOURCE_DATE_EPOCH"] = epoch + except subprocess.CalledProcessError: + pass # Might be building from a tarball. + # This layering lets SOURCE_DATE_EPOCH from os.environ takes precedence. + environment = env_defaults | os.environ | updates + + env_diff = {} + for key, value in environment.items(): + if os.environ.get(key) != value: + env_diff[key] = value + + env_vars = ( + f"\n {key}={item}" for key, item in sorted(env_diff.items()) + ) + log("🌎", f"Environment changes:{''.join(env_vars)}") + + return environment + + +def subdir(working_dir, *, clean_ok=False): + """Decorator to change to a working directory.""" + + def decorator(func): + @functools.wraps(func) + def wrapper(context): + nonlocal working_dir + + if callable(working_dir): + working_dir = working_dir(context) + separator() + log("📁", os.fsdecode(working_dir)) + if ( + clean_ok + and getattr(context, "clean", False) + and working_dir.exists() + ): + log("🚮", "Deleting directory (--clean)...") + shutil.rmtree(working_dir) + + working_dir.mkdir(parents=True, exist_ok=True) + + with contextlib.chdir(working_dir): + return func(context, working_dir) + + return wrapper + + return decorator + + +def call(command, *, context=None, quiet=False, logdir=None, **kwargs): + """Execute a command. + + If 'quiet' is true, then redirect stdout and stderr to a temporary file. + """ + if context is not None: + quiet = context.quiet + logdir = context.logdir + elif quiet and logdir is None: + raise ValueError("When quiet is True, logdir must be specified") + + log("❯", " ".join(map(str, command)), spacing=" ") + if not quiet: + stdout = None + stderr = None + else: + stdout = tempfile.NamedTemporaryFile( + "w", + encoding="utf-8", + delete=False, + dir=logdir, + prefix="cpython-wasi-", + suffix=".log", + ) + stderr = subprocess.STDOUT + log("📝", f"Logging output to {stdout.name} (--quiet)...") + + subprocess.check_call(command, **kwargs, stdout=stdout, stderr=stderr) + + +def build_python_path(): + """The path to the build Python binary.""" + binary = BUILD_DIR / "python" + if not binary.is_file(): + binary = binary.with_suffix(".exe") + if not binary.is_file(): + raise FileNotFoundError( + f"Unable to find `python(.exe)` in {BUILD_DIR}" + ) + + return binary + + +def build_python_is_pydebug(): + """Find out if the build Python is a pydebug build.""" + test = "import sys, test.support; sys.exit(test.support.Py_DEBUG)" + result = subprocess.run( + [build_python_path(), "-c", test], + capture_output=True, + ) + return bool(result.returncode) + + +@subdir(BUILD_DIR, clean_ok=True) +def configure_build_python(context, working_dir): + """Configure the build/host Python.""" + if LOCAL_SETUP.exists(): + if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER: + log("👍", f"{LOCAL_SETUP} exists ...") + else: + log("⚠️", f"{LOCAL_SETUP} exists, but has unexpected contents") + else: + log("📝", f"Creating {LOCAL_SETUP} ...") + LOCAL_SETUP.write_bytes(LOCAL_SETUP_MARKER) + + configure = [os.path.relpath(CHECKOUT / "configure", working_dir)] + if context.args: + configure.extend(context.args) + + call(configure, context=context) + + +@subdir(BUILD_DIR) +def make_build_python(context, working_dir): + """Make/build the build Python.""" + call(["make", "--jobs", str(cpu_count()), "all"], context=context) + + binary = build_python_path() + cmd = [ + binary, + "-c", + "import sys; " + "print(f'{sys.version_info.major}.{sys.version_info.minor}')", + ] + version = subprocess.check_output(cmd, encoding="utf-8").strip() + + log("🎉", f"{binary} {version}") + + +@subdir(BUILD_DIR) +def pythoninfo_build_python(context, working_dir): + """Display build info of the build Python.""" + call(["make", "pythoninfo"], context=context) + + +def find_wasi_sdk(): + """Find the path to the WASI SDK.""" + wasi_sdk_path = None + + if wasi_sdk_path_env_var := os.environ.get("WASI_SDK_PATH"): + wasi_sdk_path = pathlib.Path(wasi_sdk_path_env_var) + else: + opt_path = pathlib.Path("/opt") + # WASI SDK versions have a ``.0`` suffix, but it's a constant; the WASI SDK team + # has said they don't plan to ever do a point release and all of their Git tags + # lack the ``.0`` suffix. + # Starting with WASI SDK 23, the tarballs went from containing a directory named + # ``wasi-sdk-{WASI_SDK_VERSION}.0`` to e.g. + # ``wasi-sdk-{WASI_SDK_VERSION}.0-x86_64-linux``. + potential_sdks = [ + path + for path in opt_path.glob(f"wasi-sdk-{WASI_SDK_VERSION}.0*") + if path.is_dir() + ] + if len(potential_sdks) == 1: + wasi_sdk_path = potential_sdks[0] + elif (default_path := opt_path / "wasi-sdk").is_dir(): + wasi_sdk_path = default_path + + # Starting with WASI SDK 25, a VERSION file is included in the root + # of the SDK directory that we can read to warn folks when they are using + # an unsupported version. + if wasi_sdk_path and (version_file := wasi_sdk_path / "VERSION").is_file(): + version_details = version_file.read_text(encoding="utf-8") + found_version = version_details.splitlines()[0] + # Make sure there's a trailing dot to avoid false positives if somehow the + # supported version is a prefix of the found version (e.g. `25` and `2567`). + if not found_version.startswith(f"{WASI_SDK_VERSION}."): + major_version = found_version.partition(".")[0] + log( + "⚠️", + f" Found WASI SDK {major_version}, " + f"but WASI SDK {WASI_SDK_VERSION} is the supported version", + ) + + return wasi_sdk_path + + +def wasi_sdk_env(context): + """Calculate environment variables for building with wasi-sdk.""" + wasi_sdk_path = context.wasi_sdk_path + sysroot = wasi_sdk_path / "share" / "wasi-sysroot" + env = { + "CC": "clang", + "CPP": "clang-cpp", + "CXX": "clang++", + "AR": "llvm-ar", + "RANLIB": "ranlib", + } + + for env_var, binary_name in list(env.items()): + env[env_var] = os.fsdecode(wasi_sdk_path / "bin" / binary_name) + + if not wasi_sdk_path.name.startswith("wasi-sdk"): + for compiler in ["CC", "CPP", "CXX"]: + env[compiler] += f" --sysroot={sysroot}" + + env["PKG_CONFIG_PATH"] = "" + env["PKG_CONFIG_LIBDIR"] = os.pathsep.join( + map( + os.fsdecode, + [sysroot / "lib" / "pkgconfig", sysroot / "share" / "pkgconfig"], + ) + ) + env["PKG_CONFIG_SYSROOT_DIR"] = os.fsdecode(sysroot) + + env["WASI_SDK_PATH"] = os.fsdecode(wasi_sdk_path) + env["WASI_SYSROOT"] = os.fsdecode(sysroot) + + env["PATH"] = os.pathsep.join([ + os.fsdecode(wasi_sdk_path / "bin"), + os.environ["PATH"], + ]) + + return env + + +@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple, clean_ok=True) +def configure_wasi_python(context, working_dir): + """Configure the WASI/host build.""" + if not context.wasi_sdk_path or not context.wasi_sdk_path.exists(): + raise ValueError( + "WASI-SDK not found; " + "download from " + "https://github.com/WebAssembly/wasi-sdk and/or " + "specify via $WASI_SDK_PATH or --wasi-sdk" + ) + + config_site = os.fsdecode( + CHECKOUT / "Tools" / "wasm" / "wasi" / "config.site-wasm32-wasi" + ) + + wasi_build_dir = working_dir.relative_to(CHECKOUT) + + python_build_dir = BUILD_DIR / "build" + lib_dirs = list(python_build_dir.glob("lib.*")) + assert len(lib_dirs) == 1, ( + f"Expected a single lib.* directory in {python_build_dir}" + ) + lib_dir = os.fsdecode(lib_dirs[0]) + python_version = lib_dir.rpartition("-")[-1] + sysconfig_data_dir = ( + f"{wasi_build_dir}/build/lib.wasi-wasm32-{python_version}" + ) + + # Use PYTHONPATH to include sysconfig data which must be anchored to the + # WASI guest's `/` directory. + args = { + "GUEST_DIR": "/", + "HOST_DIR": CHECKOUT, + "ENV_VAR_NAME": "PYTHONPATH", + "ENV_VAR_VALUE": f"/{sysconfig_data_dir}", + "PYTHON_WASM": working_dir / "python.wasm", + "ARGV0": wasi_build_dir / "python.wasm", + } + # Check dynamically for wasmtime in case it was specified manually via + # `--host-runner`. + if WASMTIME_HOST_RUNNER_VAR in context.host_runner: + if wasmtime := shutil.which("wasmtime"): + args[WASMTIME_VAR_NAME] = wasmtime + else: + raise FileNotFoundError( + "wasmtime not found; download from " + "https://github.com/bytecodealliance/wasmtime" + ) + host_runner = context.host_runner.format_map(args) + env_additions = {"CONFIG_SITE": config_site, "HOSTRUNNER": host_runner} + build_python = os.fsdecode(build_python_path()) + # The path to `configure` MUST be relative, else `python.wasm` is unable + # to find the stdlib due to Python not recognizing that it's being + # executed from within a checkout. + configure = [ + os.path.relpath(CHECKOUT / "configure", working_dir), + f"--host={context.host_triple}", + f"--build={BUILD_DIR.name}", + f"--with-build-python={build_python}", + ] + if build_python_is_pydebug(): + configure.append("--with-pydebug") + if context.args: + configure.extend(context.args) + call( + configure, + env=updated_env(env_additions | wasi_sdk_env(context)), + context=context, + ) + + python_wasm = working_dir / "python.wasm" + exec_script = working_dir / "python.sh" + with exec_script.open("w", encoding="utf-8") as file: + file.write(f'#!/bin/sh\nexec {host_runner} {python_wasm} "$@"\n') + exec_script.chmod(0o755) + log("🏃", f"Created {exec_script} (--host-runner)... ") + sys.stdout.flush() + + +@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple) +def make_wasi_python(context, working_dir): + """Run `make` for the WASI/host build.""" + call( + ["make", "--jobs", str(cpu_count()), "all"], + env=updated_env(), + context=context, + ) + + exec_script = working_dir / "python.sh" + call([exec_script, "--version"], quiet=False) + log( + "🎉", + f"Use `{exec_script.relative_to(context.init_dir)}` " + "to run CPython w/ the WASI host specified by --host-runner", + ) + + +@subdir(lambda context: CROSS_BUILD_DIR / context.host_triple) +def pythoninfo_wasi_python(context, working_dir): + """Display build info of the host/WASI Python.""" + call(["make", "pythoninfo"], context=context) + + +def clean_contents(context): + """Delete all files created by this script.""" + if CROSS_BUILD_DIR.exists(): + log("🧹", f"Deleting {CROSS_BUILD_DIR} ...") + shutil.rmtree(CROSS_BUILD_DIR) + + if LOCAL_SETUP.exists(): + if LOCAL_SETUP.read_bytes() == LOCAL_SETUP_MARKER: + log("🧹", f"Deleting generated {LOCAL_SETUP} ...") + + +def build_steps(*steps): + """Construct a command from other steps.""" + + def builder(context): + for step in steps: + step(context) + + return builder + + +def main(): + default_host_triple = "wasm32-wasip1" + default_wasi_sdk = find_wasi_sdk() + default_host_runner = ( + f"{WASMTIME_HOST_RUNNER_VAR} run " + # Make sure the stack size will work for a pydebug + # build. + # Use 16 MiB stack. + "--wasm max-wasm-stack=16777216 " + # Enable thread support; causes use of preview1. + # "--wasm threads=y --wasi threads=y " + # Explicitly set the argv[0] value + "--argv0 {ARGV0} " + # Map the checkout to / to load the stdlib from /Lib. + "--dir {HOST_DIR}::{GUEST_DIR} " + # Set PYTHONPATH to the sysconfig data. + "--env {ENV_VAR_NAME}={ENV_VAR_VALUE}" + ) + default_logdir = pathlib.Path(tempfile.gettempdir()) + + parser = argparse.ArgumentParser() + subcommands = parser.add_subparsers(dest="subcommand") + build = subcommands.add_parser("build", help="Build everything") + configure_build = subcommands.add_parser( + "configure-build-python", help="Run `configure` for the build Python" + ) + make_build = subcommands.add_parser( + "make-build-python", help="Run `make` for the build Python" + ) + build_python = subcommands.add_parser( + "build-python", help="Build the build Python" + ) + pythoninfo_build = subcommands.add_parser( + "pythoninfo-build", help="Display build info of the build Python" + ) + configure_host = subcommands.add_parser( + "configure-host", + help="Run `configure` for the " + "host/WASI (pydebug builds " + "are inferred from the build " + "Python)", + ) + make_host = subcommands.add_parser( + "make-host", help="Run `make` for the host/WASI" + ) + build_host = subcommands.add_parser( + "build-host", help="Build the host/WASI Python" + ) + pythoninfo_host = subcommands.add_parser( + "pythoninfo-host", help="Display build info of the host/WASI Python" + ) + subcommands.add_parser( + "clean", help="Delete files and directories created by this script" + ) + for subcommand in ( + build, + configure_build, + make_build, + build_python, + pythoninfo_build, + configure_host, + make_host, + pythoninfo_host, + build_host, + ): + subcommand.add_argument( + "--quiet", + action="store_true", + default=False, + dest="quiet", + help="Redirect output from subprocesses to a log file", + ) + subcommand.add_argument( + "--logdir", + type=pathlib.Path, + default=default_logdir, + help=f"Directory to store log files; defaults to {default_logdir}", + ) + for subcommand in ( + configure_build, + configure_host, + build_python, + build_host, + ): + subcommand.add_argument( + "--clean", + action="store_true", + default=False, + dest="clean", + help="Delete any relevant directories before building", + ) + for subcommand in ( + build, + configure_build, + configure_host, + build_python, + build_host, + ): + subcommand.add_argument( + "args", nargs="*", help="Extra arguments to pass to `configure`" + ) + for subcommand in build, configure_host, build_host: + subcommand.add_argument( + "--wasi-sdk", + type=pathlib.Path, + dest="wasi_sdk_path", + default=default_wasi_sdk, + help=f"Path to the WASI SDK; defaults to {default_wasi_sdk}", + ) + subcommand.add_argument( + "--host-runner", + action="store", + default=default_host_runner, + dest="host_runner", + help="Command template for running the WASI host; defaults to " + f"`{default_host_runner}`", + ) + for subcommand in ( + build, + configure_host, + make_host, + build_host, + pythoninfo_host, + ): + subcommand.add_argument( + "--host-triple", + action="store", + default=default_host_triple, + help="The target triple for the WASI host build; " + f"defaults to {default_host_triple}", + ) + + context = parser.parse_args() + context.init_dir = pathlib.Path().absolute() + + build_build_python = build_steps( + configure_build_python, + make_build_python, + pythoninfo_build_python, + ) + build_wasi_python = build_steps( + configure_wasi_python, + make_wasi_python, + pythoninfo_wasi_python, + ) + + dispatch = { + "configure-build-python": configure_build_python, + "make-build-python": make_build_python, + "build-python": build_build_python, + "pythoninfo-build": pythoninfo_build_python, + "configure-host": configure_wasi_python, + "make-host": make_wasi_python, + "build-host": build_wasi_python, + "pythoninfo-host": pythoninfo_wasi_python, + "build": build_steps(build_build_python, build_wasi_python), + "clean": clean_contents, + None: lambda args: parser.print_help(), + } + dispatch[context.subcommand](context) + + +if __name__ == "__main__": + main() diff --git a/Tools/wasm/config.site-wasm32-wasi b/Tools/wasm/wasi/config.site-wasm32-wasi similarity index 100% rename from Tools/wasm/config.site-wasm32-wasi rename to Tools/wasm/wasi/config.site-wasm32-wasi diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index bcb80212362b71f..3f1bde3853fce7a 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -21,9 +21,10 @@ ./Tools/wasm/wasm_builder.py --clean build build """ + import argparse -import enum import dataclasses +import enum import logging import os import pathlib @@ -38,18 +39,12 @@ import time import warnings import webbrowser +from collections.abc import Callable, Iterable # for Python 3.8 from typing import ( - cast, Any, - Callable, - Dict, - Iterable, - List, - Optional, - Tuple, - Union, + cast, ) logger = logging.getLogger("wasm_build") @@ -67,7 +62,9 @@ # path to Emscripten SDK config file. # auto-detect's EMSDK in /opt/emsdk without ". emsdk_env.sh". -EM_CONFIG = pathlib.Path(os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten")) +EM_CONFIG = pathlib.Path( + os.environ.setdefault("EM_CONFIG", "/opt/emsdk/.emscripten") +) EMSDK_MIN_VERSION = (3, 1, 19) EMSDK_BROKEN_VERSION = { (3, 1, 14): "https://github.com/emscripten-core/emscripten/issues/17338", @@ -119,7 +116,7 @@ def parse_emconfig( emconfig: pathlib.Path = EM_CONFIG, -) -> Tuple[pathlib.Path, pathlib.Path]: +) -> tuple[pathlib.Path, pathlib.Path]: """Parse EM_CONFIG file and lookup EMSCRIPTEN_ROOT and NODE_JS. The ".emscripten" config file is a Python snippet that uses "EM_CONFIG" @@ -131,7 +128,7 @@ def parse_emconfig( with open(emconfig, encoding="utf-8") as f: code = f.read() # EM_CONFIG file is a Python snippet - local: Dict[str, Any] = {} + local: dict[str, Any] = {} exec(code, globals(), local) emscripten_root = pathlib.Path(local["EMSCRIPTEN_ROOT"]) node_js = pathlib.Path(local["NODE_JS"]) @@ -189,16 +186,16 @@ class Platform: name: str pythonexe: str - config_site: Optional[pathlib.PurePath] - configure_wrapper: Optional[pathlib.Path] - make_wrapper: Optional[pathlib.PurePath] - environ: Dict[str, Any] + config_site: pathlib.PurePath | None + configure_wrapper: pathlib.Path | None + make_wrapper: pathlib.PurePath | None + environ: dict[str, Any] check: Callable[[], None] # Used for build_emports(). - ports: Optional[pathlib.PurePath] - cc: Optional[pathlib.PurePath] + ports: pathlib.PurePath | None + cc: pathlib.PurePath | None - def getenv(self, profile: "BuildProfile") -> Dict[str, Any]: + def getenv(self, profile: "BuildProfile") -> dict[str, Any]: return self.environ.copy() @@ -261,8 +258,7 @@ def _check_emscripten() -> None: # git / upstream / tot-upstream installation version = version[:-4] version_tuple = cast( - Tuple[int, int, int], - tuple(int(v) for v in version.split(".")) + tuple[int, int, int], tuple(int(v) for v in version.split(".")) ) if version_tuple < EMSDK_MIN_VERSION: raise ConditionError( @@ -386,7 +382,7 @@ def get_extra_paths(self) -> Iterable[pathlib.PurePath]: return [] @property - def emport_args(self) -> List[str]: + def emport_args(self) -> list[str]: """Host-specific port args (Emscripten).""" cls = type(self) if self is cls.wasm64_emscripten: @@ -397,7 +393,7 @@ def emport_args(self) -> List[str]: return [] @property - def embuilder_args(self) -> List[str]: + def embuilder_args(self) -> list[str]: """Host-specific embuilder args (Emscripten).""" cls = type(self) if self is cls.wasm64_emscripten: @@ -420,7 +416,7 @@ def is_browser(self) -> bool: return self in {cls.browser, cls.browser_debug} @property - def emport_args(self) -> List[str]: + def emport_args(self) -> list[str]: """Target-specific port args.""" cls = type(self) if self in {cls.browser_debug, cls.node_debug}: @@ -446,9 +442,9 @@ class BuildProfile: name: str support_level: SupportLevel host: Host - target: Union[EmscriptenTarget, None] = None - dynamic_linking: Union[bool, None] = None - pthreads: Union[bool, None] = None + target: EmscriptenTarget | None = None + dynamic_linking: bool | None = None + pthreads: bool | None = None default_testopts: str = "-j2" @property @@ -472,7 +468,7 @@ def makefile(self) -> pathlib.Path: return self.builddir / "Makefile" @property - def configure_cmd(self) -> List[str]: + def configure_cmd(self) -> list[str]: """Generate configure command""" # use relative path, so WASI tests can find lib prefix. # pathlib.Path.relative_to() does not work here. @@ -507,7 +503,7 @@ def configure_cmd(self) -> List[str]: return cmd @property - def make_cmd(self) -> List[str]: + def make_cmd(self) -> list[str]: """Generate make command""" cmd = ["make"] platform = self.host.platform @@ -515,10 +511,10 @@ def make_cmd(self) -> List[str]: cmd.insert(0, os.fspath(platform.make_wrapper)) return cmd - def getenv(self) -> Dict[str, Any]: + def getenv(self) -> dict[str, Any]: """Generate environ dict for platform""" env = os.environ.copy() - if hasattr(os, 'process_cpu_count'): + if hasattr(os, "process_cpu_count"): cpu_count = os.process_cpu_count() else: cpu_count = os.cpu_count() @@ -529,7 +525,7 @@ def getenv(self) -> Dict[str, Any]: env.pop(key, None) elif key == "PATH": # list of path items, prefix with extra paths - new_path: List[pathlib.PurePath] = [] + new_path: list[pathlib.PurePath] = [] new_path.extend(self.host.get_extra_paths()) new_path.extend(value) env[key] = os.pathsep.join(os.fspath(p) for p in new_path) @@ -547,7 +543,7 @@ def _run_cmd( self, cmd: Iterable[str], args: Iterable[str] = (), - cwd: Optional[pathlib.Path] = None, + cwd: pathlib.Path | None = None, ) -> int: cmd = list(cmd) cmd.extend(args) @@ -585,7 +581,7 @@ def run_pythoninfo(self, *args: str) -> int: self._check_execute() return self.run_make("pythoninfo", *args) - def run_test(self, target: str, testopts: Optional[str] = None) -> int: + def run_test(self, target: str, testopts: str | None = None) -> int: """Run buildbottests""" self._check_execute() if testopts is None: @@ -596,7 +592,9 @@ def run_py(self, *args: str) -> int: """Run Python with hostrunner""" self._check_execute() return self.run_make( - "--eval", f"run: all; $(HOSTRUNNER) ./$(PYTHON) {shlex.join(args)}", "run" + "--eval", + f"run: all; $(HOSTRUNNER) ./$(PYTHON) {shlex.join(args)}", + "run", ) def run_browser(self, bind: str = "127.0.0.1", port: int = 8000) -> None: @@ -666,9 +664,12 @@ def build_emports(self, force: bool = False) -> None: # Pre-build libbz2, libsqlite3, libz, and some system libs. ports_cmd.extend(["-sUSE_ZLIB", "-sUSE_BZIP2", "-sUSE_SQLITE3"]) # Multi-threaded sqlite3 has different suffix - embuilder_cmd.extend( - ["build", "bzip2", "sqlite3-mt" if self.pthreads else "sqlite3", "zlib"] - ) + embuilder_cmd.extend([ + "build", + "bzip2", + "sqlite3-mt" if self.pthreads else "sqlite3", + "zlib", + ]) self._run_cmd(embuilder_cmd, cwd=SRCDIR) @@ -816,8 +817,8 @@ def build_emports(self, force: bool = False) -> None: ) # Don't list broken and experimental variants in help -platforms_choices = list(p.name for p in _profiles) + ["cleanall"] -platforms_help = list(p.name for p in _profiles if p.support_level) + ["cleanall"] +platforms_choices = [p.name for p in _profiles] + ["cleanall"] +platforms_help = [p.name for p in _profiles if p.support_level] + ["cleanall"] parser.add_argument( "platform", metavar="PLATFORM", @@ -825,18 +826,18 @@ def build_emports(self, force: bool = False) -> None: choices=platforms_choices, ) -ops = dict( - build="auto build (build 'build' Python, emports, configure, compile)", - configure="run ./configure", - compile="run 'make all'", - pythoninfo="run 'make pythoninfo'", - test="run 'make buildbottest TESTOPTS=...' (supports parallel tests)", - hostrunnertest="run 'make hostrunnertest TESTOPTS=...'", - repl="start interactive REPL / webserver + browser session", - clean="run 'make clean'", - cleanall="remove all build directories", - emports="build Emscripten port with embuilder (only Emscripten)", -) +ops = { + "build": "auto build (build 'build' Python, emports, configure, compile)", + "configure": "run ./configure", + "compile": "run 'make all'", + "pythoninfo": "run 'make pythoninfo'", + "test": "run 'make buildbottest TESTOPTS=...' (supports parallel tests)", + "hostrunnertest": "run 'make hostrunnertest TESTOPTS=...'", + "repl": "start interactive REPL / webserver + browser session", + "clean": "run 'make clean'", + "cleanall": "remove all build directories", + "emports": "build Emscripten port with embuilder (only Emscripten)", +} ops_help = "\n".join(f"{op:16s} {help}" for op, help in ops.items()) parser.add_argument( "ops", diff --git a/configure b/configure index c51192f12c82230..7be3c3ec7c94d91 100755 --- a/configure +++ b/configure @@ -826,6 +826,7 @@ OPENSSL_LDFLAGS OPENSSL_LIBS OPENSSL_INCLUDES ENSUREPIP +CFLAGS_CEVAL SRCDIRS THREADHEADERS PANEL_LIBS @@ -965,6 +966,7 @@ LDLIBRARY LIBRARY BUILDEXEEXT NO_AS_NEEDED +_Py_STACK_GROWS_DOWN MULTIARCH_CPPFLAGS PLATFORM_TRIPLET MULTIARCH @@ -1826,8 +1828,8 @@ Optional Features: no) --enable-profiling enable C-level code profiling with gprof (default is no) - --disable-gil enable experimental support for running without the - GIL (default is no) + --disable-gil enable support for running without the GIL (default + is no) --enable-pystats enable internal statistics gathering (default is no) --enable-optimizations enable expensive, stable optimizations (PGO, etc.) (default is no) @@ -3816,7 +3818,7 @@ fi -for ac_prog in python$PACKAGE_VERSION python3.13 python3.12 python3.11 python3.10 python3 python +for ac_prog in python$PACKAGE_VERSION python3.15 python3.14 python3.13 python3.12 python3.11 python3.10 python3 python do # Extract the first word of "$ac_prog", so it can be a program name with args. set dummy $ac_prog; ac_word=$2 @@ -4359,7 +4361,7 @@ then : yes) case $ac_sys_system in Darwin) enableval=/Library/Frameworks ;; - iOS) enableval=iOS/Frameworks/\$\(MULTIARCH\) ;; + iOS) enableval=Apple/iOS/Frameworks/\$\(MULTIARCH\) ;; *) as_fn_error $? "Unknown platform for framework build" "$LINENO" 5 esac esac @@ -4470,9 +4472,9 @@ then : prefix=$PYTHONFRAMEWORKPREFIX PYTHONFRAMEWORKINSTALLNAMEPREFIX="@rpath/$PYTHONFRAMEWORKDIR" - RESSRCDIR=iOS/Resources + RESSRCDIR=Apple/iOS/Resources - ac_config_files="$ac_config_files iOS/Resources/Info.plist" + ac_config_files="$ac_config_files Apple/iOS/Resources/Info.plist" ;; *) @@ -6594,7 +6596,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6643,7 +6645,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6675,7 +6677,7 @@ printf "%s\n" "no" >&6; } fi if test "x$ac_pt_CXX" = x; then - CXX="g++" + CXX="notfound" else case $cross_compiling:$ac_tool_warned in yes:) @@ -6704,7 +6706,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6753,7 +6755,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6785,7 +6787,7 @@ printf "%s\n" "no" >&6; } fi if test "x$ac_pt_CXX" = x; then - CXX="c++" + CXX="notfound" else case $cross_compiling:$ac_tool_warned in yes:) @@ -6814,7 +6816,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6863,7 +6865,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6895,7 +6897,7 @@ printf "%s\n" "no" >&6; } fi if test "x$ac_pt_CXX" = x; then - CXX="clang++" + CXX="notfound" else case $cross_compiling:$ac_tool_warned in yes:) @@ -6924,7 +6926,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -6973,7 +6975,7 @@ else case e in #( ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR -for as_dir in notfound +for as_dir in $PATH do IFS=$as_save_IFS case $as_dir in #((( @@ -7005,7 +7007,7 @@ printf "%s\n" "no" >&6; } fi if test "x$ac_pt_CXX" = x; then - CXX="icpc" + CXX="notfound" else case $cross_compiling:$ac_tool_warned in yes:) @@ -7211,6 +7213,18 @@ if test x$MULTIARCH != x; then fi +# Guess C stack direction +case $host in #( + hppa*) : + _Py_STACK_GROWS_DOWN=0 ;; #( + *) : + _Py_STACK_GROWS_DOWN=1 ;; +esac + +printf "%s\n" "#define _Py_STACK_GROWS_DOWN $_Py_STACK_GROWS_DOWN" >>confdefs.h + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for PEP 11 support tier" >&5 printf %s "checking for PEP 11 support tier... " >&6; } case $host/$ac_cv_cc_name in #( @@ -7254,6 +7268,8 @@ case $host/$ac_cv_cc_name in #( PY_SUPPORT_TIER=3 ;; #( x86_64-*-linux-android/clang) : PY_SUPPORT_TIER=3 ;; #( + wasm32-*-emscripten/emcc) : + PY_SUPPORT_TIER=3 ;; #( *) : PY_SUPPORT_TIER=0 @@ -9034,7 +9050,100 @@ case "$ac_cv_cc_name" in fi ;; gcc) + # Check for 32-bit x86 ISA + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for i686" >&5 +printf %s "checking for i686... " >&6; } +if test ${ac_cv_i686+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + + #ifdef __i386__ + # error "i386" + #endif + +int +main (void) +{ + + ; + return 0; +} + +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_i686=no +else case e in #( + e) ac_cv_i686=yes ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_i686" >&5 +printf "%s\n" "$ac_cv_i686" >&6; } + PGO_PROF_GEN_FLAG="-fprofile-generate" + + # Use -fprofile-update=atomic to fix a random GCC internal error on PGO + # build (gh-145801) caused by corruption of profile data (.gcda files). + # + # gh-148535: On i686, using -fprofile-update=atomic makes the PGO build + # way slower (up to 47x slower). So far, the GCC internal error on PGO + # build was not seen on i686, so don't use this flag on i686. + if test "x$ac_cv_i686" = xno +then : + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C compiler accepts -fprofile-update=atomic" >&5 +printf %s "checking whether C compiler accepts -fprofile-update=atomic... " >&6; } +if test ${ax_cv_check_cflags___fprofile_update_atomic+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + ax_check_save_flags=$CFLAGS + CFLAGS="$CFLAGS -fprofile-update=atomic" + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main (void) +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ax_cv_check_cflags___fprofile_update_atomic=yes +else case e in #( + e) ax_cv_check_cflags___fprofile_update_atomic=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + CFLAGS=$ax_check_save_flags ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ax_cv_check_cflags___fprofile_update_atomic" >&5 +printf "%s\n" "$ax_cv_check_cflags___fprofile_update_atomic" >&6; } +if test "x$ax_cv_check_cflags___fprofile_update_atomic" = xyes +then : + PGO_PROF_GEN_FLAG="$PGO_PROF_GEN_FLAG -fprofile-update=atomic" +else case e in #( + e) : ;; +esac +fi + + +fi + PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" LLVM_PROF_MERGER="true" LLVM_PROF_FILE="" @@ -9601,9 +9710,10 @@ fi as_fn_append LDFLAGS_NODIST " -sWASM_BIGINT" as_fn_append LINKFORSHARED " -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js" - as_fn_append LINKFORSHARED " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV" - as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET,_PyGILState_GetThisThreadState,__Py_DumpTraceback" + as_fn_append LINKFORSHARED " -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY,ERRNO_CODES" + as_fn_append LINKFORSHARED " -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,_PyGILState_GetThisThreadState,__Py_DumpTraceback,__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET" as_fn_append LINKFORSHARED " -sSTACK_SIZE=5MB" + as_fn_append LINKFORSHARED " -sTEXTDECODER=2" if test "x$enable_wasm_dynamic_linking" = xyes then : @@ -9677,6 +9787,65 @@ fi ;; esac +if test "$ac_sys_system" = "Linux" -a "$cross_compiling" = no; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for thread stack size" >&5 +printf %s "checking for thread stack size... " >&6; } +if test ${ac_cv_thread_stack_size+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) + cat > conftest.c <<EOF +#include <pthread.h> + +int main() +{ + pthread_attr_t attrs; + size_t size; + + int rc = pthread_attr_init(&attrs); + if (rc != 0) { + return 2; + } + + rc = pthread_attr_getstacksize(&attrs, &size); + if (rc != 0) { + return 2; + } + + if (size < 1024 * 1024) { + return 1; + } + return 0; +} +EOF + + ac_cv_thread_stack_size=unknown + if $CC -pthread $CFLAGS conftest.c -o conftest &>/dev/null; then + ./conftest &>/dev/null + exitcode=$? + if test $exitcode -eq 1; then + ac_cv_thread_stack_size=1048576 + elif test $exitcode -eq 0; then + ac_cv_thread_stack_size="default" + fi + fi + rm -f conftest.c conftest + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_thread_stack_size" >&5 +printf "%s\n" "$ac_cv_thread_stack_size" >&6; } + + if test "$ac_cv_thread_stack_size" != "default" -a "$ac_cv_thread_stack_size" != "unknown"; then + LDFLAGS="$LDFLAGS -Wl,-z,stack-size=$ac_cv_thread_stack_size" + # Stack size used by Python/ceval.c to set Py_C_STACK_SIZE + +printf "%s\n" "#define _Py_LINKER_THREAD_STACK_SIZE $ac_cv_thread_stack_size" >>confdefs.h + + fi +fi + case $enable_wasm_dynamic_linking in #( yes) : ac_cv_func_dlopen=yes ;; #( @@ -10863,7 +11032,7 @@ then : else case e in #( e) as_fn_append CFLAGS_NODIST " $jit_flags" - REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host}" + REGEN_JIT_COMMAND="\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\"" JIT_STENCILS_H="jit_stencils.h" if test "x$Py_DEBUG" = xtrue then : @@ -11591,6 +11760,12 @@ if test "x$ac_cv_header_sys_syscall_h" = xyes then : printf "%s\n" "#define HAVE_SYS_SYSCALL_H 1" >>confdefs.h +fi +ac_fn_c_check_header_compile "$LINENO" "sys/sysctl.h" "ac_cv_header_sys_sysctl_h" "$ac_includes_default" +if test "x$ac_cv_header_sys_sysctl_h" = xyes +then : + printf "%s\n" "#define HAVE_SYS_SYSCTL_H 1" >>confdefs.h + fi ac_fn_c_check_header_compile "$LINENO" "sys/sysmacros.h" "ac_cv_header_sys_sysmacros_h" "$ac_includes_default" if test "x$ac_cv_header_sys_sysmacros_h" = xyes @@ -12979,13 +13154,6 @@ if test "$ac_cv_sizeof_off_t" -gt "$ac_cv_sizeof_long" -a \ else have_largefile_support="no" fi -case $ac_sys_system in #( - Emscripten) : - have_largefile_support="no" - ;; #( - *) : - ;; -esac if test "x$have_largefile_support" = xyes then : @@ -14052,6 +14220,7 @@ fi + have_uuid=missing for ac_header in uuid.h @@ -14061,6 +14230,7 @@ if test "x$ac_cv_header_uuid_h" = xyes then : printf "%s\n" "#define HAVE_UUID_H 1" >>confdefs.h + for ac_func in uuid_create uuid_enc_be do : as_ac_var=`printf "%s\n" "ac_cv_func_$ac_func" | sed "$as_sed_sh"` @@ -14070,7 +14240,9 @@ then : cat >>confdefs.h <<_ACEOF #define `printf "%s\n" "HAVE_$ac_func" | sed "$as_sed_cpp"` 1 _ACEOF - have_uuid=yes + + have_uuid=yes + ac_cv_have_uuid_h=yes LIBUUID_CFLAGS=${LIBUUID_CFLAGS-""} LIBUUID_LIBS=${LIBUUID_LIBS-""} @@ -14160,6 +14332,7 @@ if test "x$ac_cv_header_uuid_uuid_h" = xyes then : printf "%s\n" "#define HAVE_UUID_UUID_H 1" >>confdefs.h + ac_cv_have_uuid_uuid_h=yes py_check_lib_save_LIBS=$LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for uuid_generate_time in -luuid" >&5 printf %s "checking for uuid_generate_time in -luuid... " >&6; } @@ -14257,8 +14430,9 @@ fi printf "%s\n" "$ac_cv_lib_uuid_uuid_generate_time_safe" >&6; } if test "x$ac_cv_lib_uuid_uuid_generate_time_safe" = xyes then : - have_uuid=yes - printf "%s\n" "#define HAVE_UUID_GENERATE_TIME_SAFE 1" >>confdefs.h + + have_uuid=yes + ac_cv_have_uuid_generate_time_safe=yes fi @@ -14302,6 +14476,7 @@ if test "x$ac_cv_header_uuid_uuid_h" = xyes then : printf "%s\n" "#define HAVE_UUID_UUID_H 1" >>confdefs.h + ac_cv_have_uuid_uuid_h=yes py_check_lib_save_LIBS=$LIBS { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for uuid_generate_time in -luuid" >&5 printf %s "checking for uuid_generate_time in -luuid... " >&6; } @@ -14399,8 +14574,9 @@ fi printf "%s\n" "$ac_cv_lib_uuid_uuid_generate_time_safe" >&6; } if test "x$ac_cv_lib_uuid_uuid_generate_time_safe" = xyes then : - have_uuid=yes - printf "%s\n" "#define HAVE_UUID_GENERATE_TIME_SAFE 1" >>confdefs.h + + have_uuid=yes + ac_cv_have_uuid_generate_time_safe=yes fi @@ -14431,10 +14607,16 @@ else { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } have_uuid=yes - printf "%s\n" "#define HAVE_UUID_H 1" >>confdefs.h - - printf "%s\n" "#define HAVE_UUID_GENERATE_TIME_SAFE 1" >>confdefs.h - + ac_cv_have_uuid_generate_time_safe=yes + # The uuid.h file to include may be <uuid.h> *or* <uuid/uuid.h>. + # Since pkg-config --cflags uuid may return -I/usr/include/uuid, + # it's possible to write '#include <uuid.h>' in _uuidmodule.c, + # assuming that the compiler flags are properly updated. + # + # Ideally, we should have defined HAVE_UUID_H if and only if + # #include <uuid.h> can be written, *without* assuming extra + # include path. + ac_cv_have_uuid_h=yes fi @@ -14455,6 +14637,7 @@ if test "x$ac_cv_func_uuid_generate_time" = xyes then : have_uuid=yes + ac_cv_have_uuid_uuid_h=yes LIBUUID_CFLAGS=${LIBUUID_CFLAGS-""} LIBUUID_LIBS=${LIBUUID_LIBS-""} @@ -14467,9 +14650,28 @@ done fi -# gh-124228: While the libuuid library is available on NetBSD, it supports only UUID version 4. +if test "x$ac_cv_have_uuid_h" = xyes +then : + printf "%s\n" "#define HAVE_UUID_H 1" >>confdefs.h + +fi +if test "x$ac_cv_have_uuid_uuid_h" = xyes +then : + printf "%s\n" "#define HAVE_UUID_UUID_H 1" >>confdefs.h + +fi +if test "x$ac_cv_have_uuid_generate_time_safe" = xyes +then : + + printf "%s\n" "#define HAVE_UUID_GENERATE_TIME_SAFE 1" >>confdefs.h + + +fi + +# gh-124228: While the libuuid library is available on NetBSD and OpenBSD, +# it supports only UUID version 4. # This restriction inhibits the proper generation of time-based UUIDs. -if test "$ac_sys_system" = "NetBSD"; then +if test "$ac_sys_system" = "NetBSD" || test "$ac_sys_system" = "OpenBSD"; then have_uuid=missing printf "%s\n" "#define HAVE_UUID_H 0" >>confdefs.h @@ -14480,6 +14682,164 @@ then : have_uuid=no fi +# gh-132710: The UUID node is fetched by using libuuid when possible +# and cached. While the node is constant within the same process, +# different interpreters may have different values as libuuid may +# randomize the node value if the latter cannot be deduced. +# +# Consumers may define HAVE_UUID_GENERATE_TIME_SAFE_STABLE_MAC +# to indicate that libuuid is unstable and should not be relied +# upon to deduce the MAC address. + + +if test "$have_uuid" = "yes" -a "$HAVE_UUID_GENERATE_TIME_SAFE" = "1" +then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if uuid_generate_time_safe() node value is stable" >&5 +printf %s "checking if uuid_generate_time_safe() node value is stable... " >&6; } + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + # Be sure to add the extra include path if we used pkg-config + # as HAVE_UUID_H may be set even though <uuid.h> is only reachable + # by adding extra -I flags. + # + # If the following script does not compile, we simply assume that + # libuuid is missing. + CFLAGS="$CFLAGS $LIBUUID_CFLAGS" + LIBS="$LIBS $LIBUUID_LIBS" + if test "$cross_compiling" = yes +then : + + +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + #include <inttypes.h> // PRIu64 + #include <stdint.h> // uint64_t + #include <stdio.h> // fopen(), fclose() + + #ifdef HAVE_UUID_H + #include <uuid.h> + #else + #include <uuid/uuid.h> + #endif + + #define ERR 1 + int main(void) { + uuid_t uuid; // unsigned char[16] + (void)uuid_generate_time_safe(uuid); + uint64_t node = 0; + for (size_t i = 0; i < 6; i++) { + node |= (uint64_t)uuid[15 - i] << (8 * i); + } + FILE *fp = fopen("conftest.out", "w"); + if (fp == NULL) { + return ERR; + } + int rc = fprintf(fp, "%" PRIu64 "\n", node) >= 0; + rc |= fclose(fp); + return rc == 0 ? 0 : ERR; + } +_ACEOF +if ac_fn_c_try_run "$LINENO" +then : + + py_cv_uuid_node1=`cat conftest.out` + +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + # Be sure to add the extra include path if we used pkg-config + # as HAVE_UUID_H may be set even though <uuid.h> is only reachable + # by adding extra -I flags. + # + # If the following script does not compile, we simply assume that + # libuuid is missing. + CFLAGS="$CFLAGS $LIBUUID_CFLAGS" + LIBS="$LIBS $LIBUUID_LIBS" + if test "$cross_compiling" = yes +then : + + +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + #include <inttypes.h> // PRIu64 + #include <stdint.h> // uint64_t + #include <stdio.h> // fopen(), fclose() + + #ifdef HAVE_UUID_H + #include <uuid.h> + #else + #include <uuid/uuid.h> + #endif + + #define ERR 1 + int main(void) { + uuid_t uuid; // unsigned char[16] + (void)uuid_generate_time_safe(uuid); + uint64_t node = 0; + for (size_t i = 0; i < 6; i++) { + node |= (uint64_t)uuid[15 - i] << (8 * i); + } + FILE *fp = fopen("conftest.out", "w"); + if (fp == NULL) { + return ERR; + } + int rc = fprintf(fp, "%" PRIu64 "\n", node) >= 0; + rc |= fclose(fp); + return rc == 0 ? 0 : ERR; + } +_ACEOF +if ac_fn_c_try_run "$LINENO" +then : + + py_cv_uuid_node2=`cat conftest.out` + +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + + if test -n "$py_cv_uuid_node1" -a "$py_cv_uuid_node1" = "$py_cv_uuid_node2" + then + printf "%s\n" "#define HAVE_UUID_GENERATE_TIME_SAFE_STABLE_MAC 1" >>confdefs.h + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: stable" >&5 +printf "%s\n" "stable" >&6; } + else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: unstable" >&5 +printf "%s\n" "unstable" >&6; } + fi +fi + # 'Real Time' functions on Solaris # posix4 on Solaris 2.6 # pthread (first!) on Linux @@ -15558,7 +15918,7 @@ fi printf "%s\n" "$ac_cv_ffi_complex_double_supported" >&6; } if test "$ac_cv_ffi_complex_double_supported" = "yes"; then -printf "%s\n" "#define Py_FFI_SUPPORT_C_COMPLEX 1" >>confdefs.h +printf "%s\n" "#define _Py_FFI_SUPPORT_C_COMPLEX 1" >>confdefs.h fi @@ -15711,15 +16071,15 @@ LIBS=$save_LIBS else case e in #( - e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&5 -printf "%s\n" "$as_me: WARNING: the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; consider using a system installed mpdecimal library." >&2;} ;; + e) { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: the bundled copy of libmpdec is scheduled for removal in Python 3.16; consider using a system installed mpdecimal library." >&5 +printf "%s\n" "$as_me: WARNING: the bundled copy of libmpdec is scheduled for removal in Python 3.16; consider using a system installed mpdecimal library." >&2;} ;; esac fi if test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no" then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: no system libmpdecimal found; falling back to bundled libmpdecimal (deprecated and scheduled for removal in Python 3.15)" >&5 -printf "%s\n" "$as_me: WARNING: no system libmpdecimal found; falling back to bundled libmpdecimal (deprecated and scheduled for removal in Python 3.15)" >&2;} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: no system libmpdec found; falling back to bundled libmpdec (deprecated and scheduled for removal in Python 3.16)" >&5 +printf "%s\n" "$as_me: WARNING: no system libmpdec found; falling back to bundled libmpdec (deprecated and scheduled for removal in Python 3.16)" >&2;} LIBMPDEC_CFLAGS="-I\$(srcdir)/Modules/_decimal/libmpdec" LIBMPDEC_LIBS="-lm \$(LIBMPDEC_A)" LIBMPDEC_INTERNAL="\$(LIBMPDEC_HEADERS) \$(LIBMPDEC_A)" @@ -16963,8 +17323,8 @@ int main (void) { - void *x1 = Tcl_Init; - void *x2 = Tk_Init; + Tcl_Init(NULL); + Tk_Init(NULL); ; return 0; @@ -18864,15 +19224,27 @@ printf "%s\n" "#define WITH_DTRACE 1" >>confdefs.h # linked into the binary. Correspondingly, dtrace(1) is missing the ELF # generation flag '-G'. We check for presence of this flag, rather than # hardcoding support by OS, in the interest of robustness. + # + # NetBSD DTrace requires the -x nolibs flag to avoid system library conflicts + # and uses header generation for testing instead of object generation. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether DTrace probes require linking" >&5 printf %s "checking whether DTrace probes require linking... " >&6; } if test ${ac_cv_dtrace_link+y} then : printf %s "(cached) " >&6 else case e in #( - e) ac_cv_dtrace_link=no + e) + ac_cv_dtrace_link=no echo 'BEGIN{}' > conftest.d - "$DTRACE" $DFLAGS -G -s conftest.d -o conftest.o > /dev/null 2>&1 && \ + case $host in + *netbsd*) + DTRACE_TEST_FLAGS="-x nolibs -h" + ;; + *) + DTRACE_TEST_FLAGS="-G" + ;; + esac + "$DTRACE" $DFLAGS $DTRACE_TEST_FLAGS -s conftest.d -o conftest.o > /dev/null 2>&1 && \ ac_cv_dtrace_link=yes ;; esac @@ -18882,6 +19254,12 @@ printf "%s\n" "$ac_cv_dtrace_link" >&6; } if test "$ac_cv_dtrace_link" = "yes"; then DTRACE_OBJS="Python/pydtrace.o" fi + # Set NetBSD-specific DTrace flags in DFLAGS + case $host in + *netbsd*) + DFLAGS="$DFLAGS -x nolibs" + ;; + esac fi PLATFORM_HEADERS= @@ -18890,7 +19268,7 @@ PLATFORM_OBJS= case $ac_sys_system in #( Emscripten) : - as_fn_append PLATFORM_OBJS ' Python/emscripten_signal.o Python/emscripten_trampoline.o' + as_fn_append PLATFORM_OBJS ' Python/emscripten_signal.o Python/emscripten_trampoline.o Python/emscripten_trampoline_wasm.o Python/emscripten_syscalls.o' as_fn_append PLATFORM_HEADERS ' $(srcdir)/Include/internal/pycore_emscripten_signal.h $(srcdir)/Include/internal/pycore_emscripten_trampoline.h' ;; #( *) : @@ -19063,12 +19441,6 @@ if test "x$ac_cv_func_dup" = xyes then : printf "%s\n" "#define HAVE_DUP 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "dup3" "ac_cv_func_dup3" -if test "x$ac_cv_func_dup3" = xyes -then : - printf "%s\n" "#define HAVE_DUP3 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "execv" "ac_cv_func_execv" if test "x$ac_cv_func_execv" = xyes @@ -19267,6 +19639,12 @@ if test "x$ac_cv_func_getlogin" = xyes then : printf "%s\n" "#define HAVE_GETLOGIN 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "getlogin_r" "ac_cv_func_getlogin_r" +if test "x$ac_cv_func_getlogin_r" = xyes +then : + printf "%s\n" "#define HAVE_GETLOGIN_R 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "getpeername" "ac_cv_func_getpeername" if test "x$ac_cv_func_getpeername" = xyes @@ -19537,12 +19915,6 @@ if test "x$ac_cv_func_pipe" = xyes then : printf "%s\n" "#define HAVE_PIPE 1" >>confdefs.h -fi -ac_fn_c_check_func "$LINENO" "pipe2" "ac_cv_func_pipe2" -if test "x$ac_cv_func_pipe2" = xyes -then : - printf "%s\n" "#define HAVE_PIPE2 1" >>confdefs.h - fi ac_fn_c_check_func "$LINENO" "plock" "ac_cv_func_plock" if test "x$ac_cv_func_plock" = xyes @@ -20005,6 +20377,12 @@ if test "x$ac_cv_func_sysconf" = xyes then : printf "%s\n" "#define HAVE_SYSCONF 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "sysctlbyname" "ac_cv_func_sysctlbyname" +if test "x$ac_cv_func_sysctlbyname" = xyes +then : + printf "%s\n" "#define HAVE_SYSCTLBYNAME 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "tcgetpgrp" "ac_cv_func_tcgetpgrp" if test "x$ac_cv_func_tcgetpgrp" = xyes @@ -20188,7 +20566,13 @@ fi # header definition prevents usage - autoconf doesn't use the headers), or # raise an error if used at runtime. Force these symbols off. if test "$ac_sys_system" != "iOS" ; then - ac_fn_c_check_func "$LINENO" "getentropy" "ac_cv_func_getentropy" + ac_fn_c_check_func "$LINENO" "dup3" "ac_cv_func_dup3" +if test "x$ac_cv_func_dup3" = xyes +then : + printf "%s\n" "#define HAVE_DUP3 1" >>confdefs.h + +fi +ac_fn_c_check_func "$LINENO" "getentropy" "ac_cv_func_getentropy" if test "x$ac_cv_func_getentropy" = xyes then : printf "%s\n" "#define HAVE_GETENTROPY 1" >>confdefs.h @@ -20199,6 +20583,12 @@ if test "x$ac_cv_func_getgroups" = xyes then : printf "%s\n" "#define HAVE_GETGROUPS 1" >>confdefs.h +fi +ac_fn_c_check_func "$LINENO" "pipe2" "ac_cv_func_pipe2" +if test "x$ac_cv_func_pipe2" = xyes +then : + printf "%s\n" "#define HAVE_PIPE2 1" >>confdefs.h + fi ac_fn_c_check_func "$LINENO" "system" "ac_cv_func_system" if test "x$ac_cv_func_system" = xyes @@ -22451,11 +22841,301 @@ fi # Put the nasty error message in config.log where it belongs echo "$LIBZSTD_PKG_ERRORS" >&5 + + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + CPPFLAGS="$CPPFLAGS $LIBZSTD_CFLAGS" + CFLAGS="$CFLAGS $LIBZSTD_CFLAGS" + LIBS="$LIBS $LIBZSTD_LIBS" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing ZDICT_finalizeDictionary" >&5 +printf %s "checking for library containing ZDICT_finalizeDictionary... " >&6; } +if test ${ac_cv_search_ZDICT_finalizeDictionary+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char ZDICT_finalizeDictionary (void); +int +main (void) +{ +return ZDICT_finalizeDictionary (); + ; + return 0; +} +_ACEOF +for ac_lib in '' zstd +do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO" +then : + ac_cv_search_ZDICT_finalizeDictionary=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext + if test ${ac_cv_search_ZDICT_finalizeDictionary+y} +then : + break +fi +done +if test ${ac_cv_search_ZDICT_finalizeDictionary+y} +then : + +else case e in #( + e) ac_cv_search_ZDICT_finalizeDictionary=no ;; +esac +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_ZDICT_finalizeDictionary" >&5 +printf "%s\n" "$ac_cv_search_ZDICT_finalizeDictionary" >&6; } +ac_res=$ac_cv_search_ZDICT_finalizeDictionary +if test "$ac_res" != no +then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking ZSTD_VERSION_NUMBER >= 1.4.5" >&5 +printf %s "checking ZSTD_VERSION_NUMBER >= 1.4.5... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + #include "zstd.h" +int +main (void) +{ + + #if ZSTD_VERSION_NUMBER < 10405 + # error "zstd version is too old" + #endif + + ; + return 0; +} + +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + for ac_header in zstd.h zdict.h +do : + as_ac_Header=`printf "%s\n" "ac_cv_header_$ac_header" | sed "$as_sed_sh"` +ac_fn_c_check_header_compile "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" +if eval test \"x\$"$as_ac_Header"\" = x"yes" +then : + cat >>confdefs.h <<_ACEOF +#define `printf "%s\n" "HAVE_$ac_header" | sed "$as_sed_cpp"` 1 +_ACEOF + have_libzstd=yes +else case e in #( + e) have_libzstd=no ;; +esac +fi + +done + +else case e in #( + e) + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } have_libzstd=no + ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + +else case e in #( + e) have_libzstd=no ;; +esac +fi + + if test "x$have_libzstd" = xyes +then : + + LIBZSTD_CFLAGS=${LIBZSTD_CFLAGS-""} + LIBZSTD_LIBS=${LIBZSTD_LIBS-"-lzstd"} + +fi + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + + elif test $pkg_failed = untried; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } + + save_CFLAGS=$CFLAGS +save_CPPFLAGS=$CPPFLAGS +save_LDFLAGS=$LDFLAGS +save_LIBS=$LIBS + + + CPPFLAGS="$CPPFLAGS $LIBZSTD_CFLAGS" + CFLAGS="$CFLAGS $LIBZSTD_CFLAGS" + LIBS="$LIBS $LIBZSTD_LIBS" + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for library containing ZDICT_finalizeDictionary" >&5 +printf %s "checking for library containing ZDICT_finalizeDictionary... " >&6; } +if test ${ac_cv_search_ZDICT_finalizeDictionary+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. + The 'extern "C"' is for builds by C++ compilers; + although this is not generally supported in C code supporting it here + has little cost and some practical benefit (sr 110532). */ +#ifdef __cplusplus +extern "C" +#endif +char ZDICT_finalizeDictionary (void); +int +main (void) +{ +return ZDICT_finalizeDictionary (); + ; + return 0; +} +_ACEOF +for ac_lib in '' zstd +do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO" +then : + ac_cv_search_ZDICT_finalizeDictionary=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext + if test ${ac_cv_search_ZDICT_finalizeDictionary+y} +then : + break +fi +done +if test ${ac_cv_search_ZDICT_finalizeDictionary+y} +then : + +else case e in #( + e) ac_cv_search_ZDICT_finalizeDictionary=no ;; +esac +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_ZDICT_finalizeDictionary" >&5 +printf "%s\n" "$ac_cv_search_ZDICT_finalizeDictionary" >&6; } +ac_res=$ac_cv_search_ZDICT_finalizeDictionary +if test "$ac_res" != no +then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking ZSTD_VERSION_NUMBER >= 1.4.5" >&5 +printf %s "checking ZSTD_VERSION_NUMBER >= 1.4.5... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + #include "zstd.h" +int +main (void) +{ + + #if ZSTD_VERSION_NUMBER < 10405 + # error "zstd version is too old" + #endif + + ; + return 0; +} + +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + for ac_header in zstd.h zdict.h +do : + as_ac_Header=`printf "%s\n" "ac_cv_header_$ac_header" | sed "$as_sed_sh"` +ac_fn_c_check_header_compile "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default" +if eval test \"x\$"$as_ac_Header"\" = x"yes" +then : + cat >>confdefs.h <<_ACEOF +#define `printf "%s\n" "HAVE_$ac_header" | sed "$as_sed_cpp"` 1 +_ACEOF + have_libzstd=yes +else case e in #( + e) have_libzstd=no ;; +esac +fi + +done + +else case e in #( + e) + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } have_libzstd=no + ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + +else case e in #( + e) have_libzstd=no ;; +esac +fi + + if test "x$have_libzstd" = xyes +then : + + LIBZSTD_CFLAGS=${LIBZSTD_CFLAGS-""} + LIBZSTD_LIBS=${LIBZSTD_LIBS-"-lzstd"} + +fi + +CFLAGS=$save_CFLAGS +CPPFLAGS=$save_CPPFLAGS +LDFLAGS=$save_LDFLAGS +LIBS=$save_LIBS + + + else LIBZSTD_CFLAGS=$pkg_cv_LIBZSTD_CFLAGS LIBZSTD_LIBS=$pkg_cv_LIBZSTD_LIBS @@ -23362,6 +24042,33 @@ fi +ac_fn_check_decl "$LINENO" "MAXLOGNAME" "ac_cv_have_decl_MAXLOGNAME" "#include <sys/param.h> +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_MAXLOGNAME" = xyes +then : + +printf "%s\n" "#define HAVE_MAXLOGNAME 1" >>confdefs.h + +fi + +ac_fn_check_decl "$LINENO" "UT_NAMESIZE" "ac_cv_have_decl_UT_NAMESIZE" "#include <utmp.h> +" "$ac_c_undeclared_builtin_options" "CFLAGS" +if test "x$ac_cv_have_decl_UT_NAMESIZE" = xyes +then : + ac_have_decl=1 +else case e in #( + e) ac_have_decl=0 ;; +esac +fi +printf "%s\n" "#define HAVE_DECL_UT_NAMESIZE $ac_have_decl" >>confdefs.h +if test $ac_have_decl = 1 +then : + +printf "%s\n" "#define HAVE_UT_NAMESIZE 1" >>confdefs.h + +fi + + # check for openpty, login_tty, and forkpty @@ -25469,8 +26176,8 @@ main (void) { unsigned int fpcr; - __asm__ __volatile__ ("fmove.l %%fpcr,%0" : "=g" (fpcr)); - __asm__ __volatile__ ("fmove.l %0,%%fpcr" : : "g" (fpcr)); + __asm__ __volatile__ ("fmove.l %%fpcr,%0" : "=dm" (fpcr)); + __asm__ __volatile__ ("fmove.l %0,%%fpcr" : : "dm" (fpcr)); ; return 0; @@ -27676,110 +28383,6 @@ printf "%s\n" "#define HAVE_STAT_TV_NSEC2 1" >>confdefs.h fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether year with century should be normalized for strftime" >&5 -printf %s "checking whether year with century should be normalized for strftime... " >&6; } -if test ${ac_cv_normalize_century+y} -then : - printf %s "(cached) " >&6 -else case e in #( - e) -if test "$cross_compiling" = yes -then : - ac_cv_normalize_century=yes -else case e in #( - e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -#include <time.h> -#include <string.h> - -int main(void) -{ - char year[5]; - struct tm date = { - .tm_year = -1801, - .tm_mon = 0, - .tm_mday = 1 - }; - if (strftime(year, sizeof(year), "%Y", &date) && !strcmp(year, "0099")) { - return 1; - } - return 0; -} - -_ACEOF -if ac_fn_c_try_run "$LINENO" -then : - ac_cv_normalize_century=yes -else case e in #( - e) ac_cv_normalize_century=no ;; -esac -fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext ;; -esac -fi - ;; -esac -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_normalize_century" >&5 -printf "%s\n" "$ac_cv_normalize_century" >&6; } -if test "$ac_cv_normalize_century" = yes -then - -printf "%s\n" "#define Py_NORMALIZE_CENTURY 1" >>confdefs.h - -fi - -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking whether C99-compatible strftime specifiers are supported" >&5 -printf %s "checking whether C99-compatible strftime specifiers are supported... " >&6; } -if test ${ac_cv_strftime_c99_support+y} -then : - printf %s "(cached) " >&6 -else case e in #( - e) -if test "$cross_compiling" = yes -then : - ac_cv_strftime_c99_support= -else case e in #( - e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -#include <time.h> -#include <string.h> - -int main(void) -{ - char full_date[11]; - struct tm date = { - .tm_year = 0, - .tm_mon = 0, - .tm_mday = 1 - }; - if (strftime(full_date, sizeof(full_date), "%F", &date) && !strcmp(full_date, "1900-01-01")) { - return 0; - } - return 1; -} - -_ACEOF -if ac_fn_c_try_run "$LINENO" -then : - ac_cv_strftime_c99_support=yes -else case e in #( - e) as_fn_error $? "Python requires C99-compatible strftime specifiers" "$LINENO" 5 ;; -esac -fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext ;; -esac -fi - ;; -esac -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_strftime_c99_support" >&5 -printf "%s\n" "$ac_cv_strftime_c99_support" >&6; } - have_curses=no have_panel=no @@ -28468,6 +29071,8 @@ fi + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function is_pad" >&5 printf %s "checking for curses function is_pad... " >&6; } if test ${ac_cv_lib_curses_is_pad+y} @@ -28856,8 +29461,128 @@ int main (void) { - #ifndef wchgat - void *x=wchgat + #ifndef wchgat + void *x=wchgat + #endif + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_lib_curses_wchgat=yes +else case e in #( + e) ac_cv_lib_curses_wchgat=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_wchgat" >&5 +printf "%s\n" "$ac_cv_lib_curses_wchgat" >&6; } + if test "x$ac_cv_lib_curses_wchgat" = xyes +then : + +printf "%s\n" "#define HAVE_CURSES_WCHGAT 1" >>confdefs.h + +fi + + + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function filter" >&5 +printf %s "checking for curses function filter... " >&6; } +if test ${ac_cv_lib_curses_filter+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include <ncursesw/ncurses.h> +#elif defined(HAVE_NCURSESW_CURSES_H) +# include <ncursesw/curses.h> +#elif defined(HAVE_NCURSES_NCURSES_H) +# include <ncurses/ncurses.h> +#elif defined(HAVE_NCURSES_CURSES_H) +# include <ncurses/curses.h> +#elif defined(HAVE_NCURSES_H) +# include <ncurses.h> +#elif defined(HAVE_CURSES_H) +# include <curses.h> +#endif + +int +main (void) +{ + + #ifndef filter + void *x=filter + #endif + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_lib_curses_filter=yes +else case e in #( + e) ac_cv_lib_curses_filter=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_filter" >&5 +printf "%s\n" "$ac_cv_lib_curses_filter" >&6; } + if test "x$ac_cv_lib_curses_filter" = xyes +then : + +printf "%s\n" "#define HAVE_CURSES_FILTER 1" >>confdefs.h + +fi + + + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function has_key" >&5 +printf %s "checking for curses function has_key... " >&6; } +if test ${ac_cv_lib_curses_has_key+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include <ncursesw/ncurses.h> +#elif defined(HAVE_NCURSESW_CURSES_H) +# include <ncursesw/curses.h> +#elif defined(HAVE_NCURSES_NCURSES_H) +# include <ncurses/ncurses.h> +#elif defined(HAVE_NCURSES_CURSES_H) +# include <ncurses/curses.h> +#elif defined(HAVE_NCURSES_H) +# include <ncurses.h> +#elif defined(HAVE_CURSES_H) +# include <curses.h> +#endif + +int +main (void) +{ + + #ifndef has_key + void *x=has_key #endif ; @@ -28866,21 +29591,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_wchgat=yes + ac_cv_lib_curses_has_key=yes else case e in #( - e) ac_cv_lib_curses_wchgat=no ;; + e) ac_cv_lib_curses_has_key=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_wchgat" >&5 -printf "%s\n" "$ac_cv_lib_curses_wchgat" >&6; } - if test "x$ac_cv_lib_curses_wchgat" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_has_key" >&5 +printf "%s\n" "$ac_cv_lib_curses_has_key" >&6; } + if test "x$ac_cv_lib_curses_has_key" = xyes then : -printf "%s\n" "#define HAVE_CURSES_WCHGAT 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_HAS_KEY 1" >>confdefs.h fi @@ -28888,9 +29613,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function filter" >&5 -printf %s "checking for curses function filter... " >&6; } -if test ${ac_cv_lib_curses_filter+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function typeahead" >&5 +printf %s "checking for curses function typeahead... " >&6; } +if test ${ac_cv_lib_curses_typeahead+y} then : printf %s "(cached) " >&6 else case e in #( @@ -28916,8 +29641,8 @@ int main (void) { - #ifndef filter - void *x=filter + #ifndef typeahead + void *x=typeahead #endif ; @@ -28926,21 +29651,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_filter=yes + ac_cv_lib_curses_typeahead=yes else case e in #( - e) ac_cv_lib_curses_filter=no ;; + e) ac_cv_lib_curses_typeahead=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_filter" >&5 -printf "%s\n" "$ac_cv_lib_curses_filter" >&6; } - if test "x$ac_cv_lib_curses_filter" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_typeahead" >&5 +printf "%s\n" "$ac_cv_lib_curses_typeahead" >&6; } + if test "x$ac_cv_lib_curses_typeahead" = xyes then : -printf "%s\n" "#define HAVE_CURSES_FILTER 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_TYPEAHEAD 1" >>confdefs.h fi @@ -28948,9 +29673,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function has_key" >&5 -printf %s "checking for curses function has_key... " >&6; } -if test ${ac_cv_lib_curses_has_key+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function use_env" >&5 +printf %s "checking for curses function use_env... " >&6; } +if test ${ac_cv_lib_curses_use_env+y} then : printf %s "(cached) " >&6 else case e in #( @@ -28976,8 +29701,8 @@ int main (void) { - #ifndef has_key - void *x=has_key + #ifndef use_env + void *x=use_env #endif ; @@ -28986,21 +29711,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_has_key=yes + ac_cv_lib_curses_use_env=yes else case e in #( - e) ac_cv_lib_curses_has_key=no ;; + e) ac_cv_lib_curses_use_env=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_has_key" >&5 -printf "%s\n" "$ac_cv_lib_curses_has_key" >&6; } - if test "x$ac_cv_lib_curses_has_key" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_use_env" >&5 +printf "%s\n" "$ac_cv_lib_curses_use_env" >&6; } + if test "x$ac_cv_lib_curses_use_env" = xyes then : -printf "%s\n" "#define HAVE_CURSES_HAS_KEY 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_USE_ENV 1" >>confdefs.h fi @@ -29008,9 +29733,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function typeahead" >&5 -printf %s "checking for curses function typeahead... " >&6; } -if test ${ac_cv_lib_curses_typeahead+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function set_escdelay" >&5 +printf %s "checking for curses function set_escdelay... " >&6; } +if test ${ac_cv_lib_curses_set_escdelay+y} then : printf %s "(cached) " >&6 else case e in #( @@ -29036,8 +29761,8 @@ int main (void) { - #ifndef typeahead - void *x=typeahead + #ifndef set_escdelay + void *x=set_escdelay #endif ; @@ -29046,21 +29771,21 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_typeahead=yes + ac_cv_lib_curses_set_escdelay=yes else case e in #( - e) ac_cv_lib_curses_typeahead=no ;; + e) ac_cv_lib_curses_set_escdelay=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_typeahead" >&5 -printf "%s\n" "$ac_cv_lib_curses_typeahead" >&6; } - if test "x$ac_cv_lib_curses_typeahead" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_set_escdelay" >&5 +printf "%s\n" "$ac_cv_lib_curses_set_escdelay" >&6; } + if test "x$ac_cv_lib_curses_set_escdelay" = xyes then : -printf "%s\n" "#define HAVE_CURSES_TYPEAHEAD 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_SET_ESCDELAY 1" >>confdefs.h fi @@ -29068,9 +29793,9 @@ fi - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function use_env" >&5 -printf %s "checking for curses function use_env... " >&6; } -if test ${ac_cv_lib_curses_use_env+y} + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses function set_tabsize" >&5 +printf %s "checking for curses function set_tabsize... " >&6; } +if test ${ac_cv_lib_curses_set_tabsize+y} then : printf %s "(cached) " >&6 else case e in #( @@ -29096,8 +29821,8 @@ int main (void) { - #ifndef use_env - void *x=use_env + #ifndef set_tabsize + void *x=set_tabsize #endif ; @@ -29106,26 +29831,193 @@ main (void) _ACEOF if ac_fn_c_try_compile "$LINENO" then : - ac_cv_lib_curses_use_env=yes + ac_cv_lib_curses_set_tabsize=yes else case e in #( - e) ac_cv_lib_curses_use_env=no ;; + e) ac_cv_lib_curses_set_tabsize=no ;; esac fi rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; esac fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_use_env" >&5 -printf "%s\n" "$ac_cv_lib_curses_use_env" >&6; } - if test "x$ac_cv_lib_curses_use_env" = xyes +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_set_tabsize" >&5 +printf "%s\n" "$ac_cv_lib_curses_set_tabsize" >&6; } + if test "x$ac_cv_lib_curses_set_tabsize" = xyes then : -printf "%s\n" "#define HAVE_CURSES_USE_ENV 1" >>confdefs.h +printf "%s\n" "#define HAVE_CURSES_SET_TABSIZE 1" >>confdefs.h + +fi + + + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses variable ESCDELAY" >&5 +printf %s "checking for curses variable ESCDELAY... " >&6; } +if test ${ac_cv_lib_curses_ESCDELAY+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include <ncursesw/ncurses.h> +#elif defined(HAVE_NCURSESW_CURSES_H) +# include <ncursesw/curses.h> +#elif defined(HAVE_NCURSES_NCURSES_H) +# include <ncurses/ncurses.h> +#elif defined(HAVE_NCURSES_CURSES_H) +# include <ncurses/curses.h> +#elif defined(HAVE_NCURSES_H) +# include <ncurses.h> +#elif defined(HAVE_CURSES_H) +# include <curses.h> +#endif + +int +main (void) +{ + int x = ESCDELAY; (void)x; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_lib_curses_ESCDELAY=yes +else case e in #( + e) ac_cv_lib_curses_ESCDELAY=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_ESCDELAY" >&5 +printf "%s\n" "$ac_cv_lib_curses_ESCDELAY" >&6; } + if test "x$ac_cv_lib_curses_ESCDELAY" = xyes +then : +printf "%s\n" "#define HAVE_CURSES_ESCDELAY 1" >>confdefs.h + +fi + + + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for curses variable TABSIZE" >&5 +printf %s "checking for curses variable TABSIZE... " >&6; } +if test ${ac_cv_lib_curses_TABSIZE+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include <ncursesw/ncurses.h> +#elif defined(HAVE_NCURSESW_CURSES_H) +# include <ncursesw/curses.h> +#elif defined(HAVE_NCURSES_NCURSES_H) +# include <ncurses/ncurses.h> +#elif defined(HAVE_NCURSES_CURSES_H) +# include <ncurses/curses.h> +#elif defined(HAVE_NCURSES_H) +# include <ncurses.h> +#elif defined(HAVE_CURSES_H) +# include <curses.h> +#endif + +int +main (void) +{ + + int x = TABSIZE; (void)x; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_lib_curses_TABSIZE=yes +else case e in #( + e) ac_cv_lib_curses_TABSIZE=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_TABSIZE" >&5 +printf "%s\n" "$ac_cv_lib_curses_TABSIZE" >&6; } + if test "x$ac_cv_lib_curses_TABSIZE" = xyes +then : + +printf "%s\n" "#define HAVE_CURSES_TABSIZE 1" >>confdefs.h + +fi + + + + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ncurses-style curses function getmouse" >&5 +printf %s "checking for ncurses-style curses function getmouse... " >&6; } +if test ${ac_cv_lib_curses_getmouse+y} +then : + printf %s "(cached) " >&6 +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#define NCURSES_OPAQUE 0 +#if defined(HAVE_NCURSESW_NCURSES_H) +# include <ncursesw/ncurses.h> +#elif defined(HAVE_NCURSESW_CURSES_H) +# include <ncursesw/curses.h> +#elif defined(HAVE_NCURSES_NCURSES_H) +# include <ncurses/ncurses.h> +#elif defined(HAVE_NCURSES_CURSES_H) +# include <ncurses/curses.h> +#elif defined(HAVE_NCURSES_H) +# include <ncurses.h> +#elif defined(HAVE_CURSES_H) +# include <curses.h> +#endif + +int +main (void) +{ +MEVENT event; (void)getmouse(&event); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO" +then : + ac_cv_lib_curses_getmouse=yes +else case e in #( + e) ac_cv_lib_curses_getmouse=no ;; +esac +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_curses_getmouse" >&5 +printf "%s\n" "$ac_cv_lib_curses_getmouse" >&6; } +if test "x$ac_cv_lib_curses_getmouse" = xyes +then : + +printf "%s\n" "#define HAVE_CURSES_GETMOUSE 1" >>confdefs.h + +fi CPPFLAGS=$ac_save_cppflags fi @@ -29426,9 +30318,6 @@ printf "%s\n" "#define Py_REMOTE_DEBUG 1" >>confdefs.h { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 printf "%s\n" "yes" >&6; } else - -printf "%s\n" "#define Py_REMOTE_DEBUG 0" >>confdefs.h - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 printf "%s\n" "no" >&6; } fi @@ -29575,6 +30464,52 @@ printf "%s\n" "#define HAVE_GLIBC_MEMMOVE_BUG 1" >>confdefs.h fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking if we need to manually block large inlining in ceval.c" >&5 +printf %s "checking if we need to manually block large inlining in ceval.c... " >&6; } +if test "$cross_compiling" = yes +then : + block_huge_inlining_in_ceval=undefined +else case e in #( + e) cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int main(void) { +// See gh-148284: Clang 22 seems to have interactions with inlining +// and the stackref buffer which cause 40 kB of stack usage on x86-64 +// in buggy versions of _PyEval_EvalFrameDefault() in computed goto +// interpreter. The normal usage seen is normally 1-2 kB. +#if defined(__clang__) && (__clang_major__ == 22) + return 1; +#else + return 0; +#endif +} + +_ACEOF +if ac_fn_c_try_run "$LINENO" +then : + block_huge_inlining_in_ceval=no +else case e in #( + e) block_huge_inlining_in_ceval=yes ;; +esac +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext ;; +esac +fi + +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $block_huge_inlining_in_ceval" >&5 +printf "%s\n" "$block_huge_inlining_in_ceval" >&6; } + +if test "$block_huge_inlining_in_ceval" = yes && test "$ac_cv_computed_gotos" = yes; then + # gh-148284: Suppress inlining of functions whose stack size exceeds + # 512 bytes. This number should be tuned to follow the C stack + # consumption in _PyEval_EvalFrameDefault() on computed goto + # interpreter. + CFLAGS_CEVAL="$CFLAGS_CEVAL -finline-max-stacksize=512" +fi + + if test "$ac_cv_gcc_asm_for_x87" = yes; then # Some versions of gcc miscompile inline asm: # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=46491 @@ -30689,9 +31624,7 @@ case $ac_sys_system in #( - py_cv_module_fcntl=n/a py_cv_module_readline=n/a - py_cv_module_termios=n/a py_cv_module_=n/a ;; #( @@ -32076,6 +33009,14 @@ LIBHACL_CFLAGS="${LIBHACL_FLAG_I} ${LIBHACL_FLAG_D} \$(PY_STDMODULE_CFLAGS) \$(C LIBHACL_LDFLAGS= # for now, no specific linker flags are needed +if test "$UNIVERSAL_ARCHS" = "universal2" -o \ + \( "$build_cpu" = "aarch64" -a "$build_vendor" = "apple" \) +then + use_hacl_universal2_impl=yes +else + use_hacl_universal2_impl=no +fi + # The SIMD files use aligned_alloc, which is not available on older versions of # Android. # The *mmintrin.h headers are x86-family-specific, so can't be used on WASI. @@ -32121,7 +33062,7 @@ then : LIBHACL_SIMD128_FLAGS="-msse -msse2 -msse3 -msse4.1 -msse4.2" -printf "%s\n" "#define HACL_CAN_COMPILE_SIMD128 1" >>confdefs.h +printf "%s\n" "#define _Py_HACL_CAN_COMPILE_VEC128 1" >>confdefs.h # macOS universal2 builds *support* the -msse etc flags because they're @@ -32129,7 +33070,7 @@ printf "%s\n" "#define HACL_CAN_COMPILE_SIMD128 1" >>confdefs.h # isn't great, so it's disabled on ARM64. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for HACL* SIMD128 implementation" >&5 printf %s "checking for HACL* SIMD128 implementation... " >&6; } - if test "$UNIVERSAL_ARCHS" == "universal2"; then + if test "$use_hacl_universal2_impl" = "yes"; then LIBHACL_BLAKE2_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: universal2" >&5 printf "%s\n" "universal2" >&6; } @@ -32197,7 +33138,7 @@ then : LIBHACL_SIMD256_FLAGS="-mavx2" -printf "%s\n" "#define HACL_CAN_COMPILE_SIMD256 1" >>confdefs.h +printf "%s\n" "#define _Py_HACL_CAN_COMPILE_VEC256 1" >>confdefs.h # macOS universal2 builds *support* the -mavx2 compiler flag because it's @@ -32206,7 +33147,7 @@ printf "%s\n" "#define HACL_CAN_COMPILE_SIMD256 1" >>confdefs.h # wrapped implementation if we're building for universal2. { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for HACL* SIMD256 implementation" >&5 printf %s "checking for HACL* SIMD256 implementation... " >&6; } - if test "$UNIVERSAL_ARCHS" == "universal2"; then + if test "$use_hacl_universal2_impl" = "yes"; then LIBHACL_BLAKE2_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o" { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: universal2" >&5 printf "%s\n" "universal2" >&6; } @@ -32231,7 +33172,7 @@ fi { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for HACL* library linking type" >&5 printf %s "checking for HACL* library linking type... " >&6; } -if test "$ac_sys_system" = "WASI"; then +if test "$ac_sys_system" = "WASI" || test "$MODULE_BUILDTYPE" = "static"; then LIBHACL_LDEPS_LIBTYPE=STATIC { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: static" >&5 printf "%s\n" "static" >&6; } @@ -33651,8 +34592,8 @@ fi if test "x$py_cv_module__ctypes_test" = xyes then : - as_fn_append MODULE_BLOCK "MODULE__CTYPES_TEST_CFLAGS=$LIBFFI_CFLAGS$as_nl" - as_fn_append MODULE_BLOCK "MODULE__CTYPES_TEST_LDFLAGS=$LIBFFI_LIBS $LIBM$as_nl" + + as_fn_append MODULE_BLOCK "MODULE__CTYPES_TEST_LDFLAGS=$LIBM$as_nl" fi if test "$py_cv_module__ctypes_test" = yes; then @@ -34790,7 +35731,7 @@ do "Mac/PythonLauncher/Makefile") CONFIG_FILES="$CONFIG_FILES Mac/PythonLauncher/Makefile" ;; "Mac/Resources/framework/Info.plist") CONFIG_FILES="$CONFIG_FILES Mac/Resources/framework/Info.plist" ;; "Mac/Resources/app/Info.plist") CONFIG_FILES="$CONFIG_FILES Mac/Resources/app/Info.plist" ;; - "iOS/Resources/Info.plist") CONFIG_FILES="$CONFIG_FILES iOS/Resources/Info.plist" ;; + "Apple/iOS/Resources/Info.plist") CONFIG_FILES="$CONFIG_FILES Apple/iOS/Resources/Info.plist" ;; "Makefile.pre") CONFIG_FILES="$CONFIG_FILES Makefile.pre" ;; "Misc/python.pc") CONFIG_FILES="$CONFIG_FILES Misc/python.pc" ;; "Misc/python-embed.pc") CONFIG_FILES="$CONFIG_FILES Misc/python-embed.pc" ;; diff --git a/configure.ac b/configure.ac index a7b2f62579b0e96..92b0a3c89625595 100644 --- a/configure.ac +++ b/configure.ac @@ -205,7 +205,7 @@ AC_SUBST([FREEZE_MODULE_DEPS]) AC_SUBST([PYTHON_FOR_BUILD_DEPS]) AC_CHECK_PROGS([PYTHON_FOR_REGEN], - [python$PACKAGE_VERSION python3.13 python3.12 python3.11 python3.10 python3 python], + [python$PACKAGE_VERSION python3.15 python3.14 python3.13 python3.12 python3.11 python3.10 python3 python], [python3]) AC_SUBST([PYTHON_FOR_REGEN]) @@ -559,7 +559,7 @@ AC_ARG_ENABLE([framework], yes) case $ac_sys_system in Darwin) enableval=/Library/Frameworks ;; - iOS) enableval=iOS/Frameworks/\$\(MULTIARCH\) ;; + iOS) enableval=Apple/iOS/Frameworks/\$\(MULTIARCH\) ;; *) AC_MSG_ERROR([Unknown platform for framework build]) esac esac @@ -666,9 +666,9 @@ AC_ARG_ENABLE([framework], prefix=$PYTHONFRAMEWORKPREFIX PYTHONFRAMEWORKINSTALLNAMEPREFIX="@rpath/$PYTHONFRAMEWORKDIR" - RESSRCDIR=iOS/Resources + RESSRCDIR=Apple/iOS/Resources - AC_CONFIG_FILES([iOS/Resources/Info.plist]) + AC_CONFIG_FILES([Apple/iOS/Resources/Info.plist]) ;; *) AC_MSG_ERROR([Unknown platform for framework build]) @@ -1122,10 +1122,10 @@ preset_cxx="$CXX" if test -z "$CXX" then case "$ac_cv_cc_name" in - gcc) AC_PATH_TOOL([CXX], [g++], [g++], [notfound]) ;; - cc) AC_PATH_TOOL([CXX], [c++], [c++], [notfound]) ;; - clang) AC_PATH_TOOL([CXX], [clang++], [clang++], [notfound]) ;; - icc) AC_PATH_TOOL([CXX], [icpc], [icpc], [notfound]) ;; + gcc) AC_PATH_TOOL([CXX], [g++], [notfound]) ;; + cc) AC_PATH_TOOL([CXX], [c++], [notfound]) ;; + clang) AC_PATH_TOOL([CXX], [clang++], [notfound]) ;; + icc) AC_PATH_TOOL([CXX], [icpc], [notfound]) ;; esac if test "$CXX" = "notfound" then @@ -1202,33 +1202,42 @@ if test x$MULTIARCH != x; then fi AC_SUBST([MULTIARCH_CPPFLAGS]) +# Guess C stack direction +AS_CASE([$host], + [hppa*], [_Py_STACK_GROWS_DOWN=0], + [_Py_STACK_GROWS_DOWN=1]) +AC_DEFINE_UNQUOTED([_Py_STACK_GROWS_DOWN], [$_Py_STACK_GROWS_DOWN], + [Define to 1 if the machine stack grows down (default); 0 if it grows up.]) +AC_SUBST([_Py_STACK_GROWS_DOWN]) + dnl Support tiers according to https://peps.python.org/pep-0011/ dnl dnl NOTE: Windows support tiers are defined in PC/pyconfig.h. dnl AC_MSG_CHECKING([for PEP 11 support tier]) AS_CASE([$host/$ac_cv_cc_name], - [x86_64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=1], dnl Linux on AMD64, any vendor, glibc, gcc - [x86_64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on Intel, any version - [aarch64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on M1, any version - [i686-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 32bit Windows on Intel, MSVC - [x86_64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 64bit Windows on AMD64, MSVC - - [aarch64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux ARM64, glibc, gcc+clang - [aarch64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], - [powerpc64le-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux on PPC64 little endian, glibc, gcc - [wasm32-unknown-wasip1/clang], [PY_SUPPORT_TIER=2], dnl WebAssembly System Interface preview1, clang - [x86_64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], dnl Linux on AMD64, any vendor, glibc, clang - - [aarch64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=3], dnl Windows ARM64, MSVC - [armv7l-*-linux-gnueabihf/gcc], [PY_SUPPORT_TIER=3], dnl ARMv7 LE with hardware floats, any vendor, glibc, gcc - [powerpc64le-*-linux-gnu/clang], [PY_SUPPORT_TIER=3], dnl Linux on PPC64 little endian, glibc, clang - [s390x-*-linux-gnu/gcc], [PY_SUPPORT_TIER=3], dnl Linux on 64bit s390x (big endian), glibc, gcc - [x86_64-*-freebsd*/clang], [PY_SUPPORT_TIER=3], dnl FreeBSD on AMD64 - [aarch64-apple-ios*-simulator/clang], [PY_SUPPORT_TIER=3], dnl iOS Simulator on arm64 - [aarch64-apple-ios*/clang], [PY_SUPPORT_TIER=3], dnl iOS on ARM64 - [aarch64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on ARM64 - [x86_64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on AMD64 + [x86_64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=1], dnl Linux on AMD64, any vendor, glibc, gcc + [x86_64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on Intel, any version + [aarch64-apple-darwin*/clang], [PY_SUPPORT_TIER=1], dnl macOS on M1, any version + [i686-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 32bit Windows on Intel, MSVC + [x86_64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=1], dnl 64bit Windows on AMD64, MSVC + + [aarch64-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux ARM64, glibc, gcc+clang + [aarch64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], + [powerpc64le-*-linux-gnu/gcc], [PY_SUPPORT_TIER=2], dnl Linux on PPC64 little endian, glibc, gcc + [wasm32-unknown-wasip1/clang], [PY_SUPPORT_TIER=2], dnl WebAssembly System Interface preview1, clang + [x86_64-*-linux-gnu/clang], [PY_SUPPORT_TIER=2], dnl Linux on AMD64, any vendor, glibc, clang + + [aarch64-pc-windows-msvc/msvc], [PY_SUPPORT_TIER=3], dnl Windows ARM64, MSVC + [armv7l-*-linux-gnueabihf/gcc], [PY_SUPPORT_TIER=3], dnl ARMv7 LE with hardware floats, any vendor, glibc, gcc + [powerpc64le-*-linux-gnu/clang], [PY_SUPPORT_TIER=3], dnl Linux on PPC64 little endian, glibc, clang + [s390x-*-linux-gnu/gcc], [PY_SUPPORT_TIER=3], dnl Linux on 64bit s390x (big endian), glibc, gcc + [x86_64-*-freebsd*/clang], [PY_SUPPORT_TIER=3], dnl FreeBSD on AMD64 + [aarch64-apple-ios*-simulator/clang], [PY_SUPPORT_TIER=3], dnl iOS Simulator on arm64 + [aarch64-apple-ios*/clang], [PY_SUPPORT_TIER=3], dnl iOS on ARM64 + [aarch64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on ARM64 + [x86_64-*-linux-android/clang], [PY_SUPPORT_TIER=3], dnl Android on AMD64 + [wasm32-*-emscripten/emcc], [PY_SUPPORT_TIER=3], dnl Emscripten [PY_SUPPORT_TIER=0] ) @@ -1716,7 +1725,7 @@ ABI_THREAD="" # --disable-gil AC_MSG_CHECKING([for --disable-gil]) AC_ARG_ENABLE([gil], - [AS_HELP_STRING([--disable-gil], [enable experimental support for running without the GIL (default is no)])], + [AS_HELP_STRING([--disable-gil], [enable support for running without the GIL (default is no)])], [AS_VAR_IF([enable_gil], [yes], [disable_gil=no], [disable_gil=yes])], [disable_gil=no] ) AC_MSG_RESULT([$disable_gil]) @@ -2063,7 +2072,32 @@ case "$ac_cv_cc_name" in fi ;; gcc) + # Check for 32-bit x86 ISA + AC_CACHE_CHECK([for i686], [ac_cv_i686], [ + AC_COMPILE_IFELSE([ + AC_LANG_PROGRAM([ + #ifdef __i386__ + # error "i386" + #endif + ], []) + ],[ac_cv_i686=no],[ac_cv_i686=yes]) + ]) + PGO_PROF_GEN_FLAG="-fprofile-generate" + + # Use -fprofile-update=atomic to fix a random GCC internal error on PGO + # build (gh-145801) caused by corruption of profile data (.gcda files). + # + # gh-148535: On i686, using -fprofile-update=atomic makes the PGO build + # way slower (up to 47x slower). So far, the GCC internal error on PGO + # build was not seen on i686, so don't use this flag on i686. + AS_VAR_IF([ac_cv_i686], [no], [ + AX_CHECK_COMPILE_FLAG( + [-fprofile-update=atomic], + [PGO_PROF_GEN_FLAG="$PGO_PROF_GEN_FLAG -fprofile-update=atomic"], + []) + ]) + PGO_PROF_USE_FLAG="-fprofile-use -fprofile-correction" LLVM_PROF_MERGER="true" LLVM_PROF_FILE="" @@ -2334,9 +2368,11 @@ AS_CASE([$ac_sys_system], dnl Include file system support AS_VAR_APPEND([LINKFORSHARED], [" -sFORCE_FILESYSTEM -lidbfs.js -lnodefs.js -lproxyfs.js -lworkerfs.js"]) - AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV"]) - AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,__PyEM_EMSCRIPTEN_COUNT_ARGS_OFFSET,_PyGILState_GetThisThreadState,__Py_DumpTraceback"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_RUNTIME_METHODS=FS,callMain,ENV,HEAPU32,TTY,ERRNO_CODES"]) + AS_VAR_APPEND([LINKFORSHARED], [" -sEXPORTED_FUNCTIONS=_main,_Py_Version,__PyRuntime,_PyGILState_GetThisThreadState,__Py_DumpTraceback,__PyEM_EMSCRIPTEN_TRAMPOLINE_OFFSET"]) AS_VAR_APPEND([LINKFORSHARED], [" -sSTACK_SIZE=5MB"]) + dnl Avoid bugs in JS fallback string decoding path + AS_VAR_APPEND([LINKFORSHARED], [" -sTEXTDECODER=2"]) AS_VAR_IF([enable_wasm_dynamic_linking], [yes], [ AS_VAR_APPEND([LINKFORSHARED], [" -sMAIN_MODULE"]) @@ -2389,6 +2425,57 @@ AS_CASE([$ac_sys_system], ] ) +dnl On Linux, check the thread stack size. musl (ex: Alpine Linux) uses +dnl a default thread stack size of 128 kB, whereas the glibc uses 8 MiB. +dnl Python needs at least 1 MiB. +if test "$ac_sys_system" = "Linux" -a "$cross_compiling" = no; then + AC_CACHE_CHECK([for thread stack size], [ac_cv_thread_stack_size], [ + cat > conftest.c <<EOF +#include <pthread.h> + +int main() +{ + pthread_attr_t attrs; + size_t size; + + int rc = pthread_attr_init(&attrs); + if (rc != 0) { + return 2; + } + + rc = pthread_attr_getstacksize(&attrs, &size); + if (rc != 0) { + return 2; + } + + if (size < 1024 * 1024) { + return 1; + } + return 0; +} +EOF + + ac_cv_thread_stack_size=unknown + if $CC -pthread $CFLAGS conftest.c -o conftest &>/dev/null; then + ./conftest &>/dev/null + exitcode=$? + if test $exitcode -eq 1; then + ac_cv_thread_stack_size=1048576 + elif test $exitcode -eq 0; then + ac_cv_thread_stack_size="default" + fi + fi + rm -f conftest.c conftest + ]) + + if test "$ac_cv_thread_stack_size" != "default" -a "$ac_cv_thread_stack_size" != "unknown"; then + LDFLAGS="$LDFLAGS -Wl,-z,stack-size=$ac_cv_thread_stack_size" + # Stack size used by Python/ceval.c to set Py_C_STACK_SIZE + AC_DEFINE_UNQUOTED([_Py_LINKER_THREAD_STACK_SIZE], [$ac_cv_thread_stack_size], + [Thread stack size set by the linker (in bytes).]) + fi +fi + AS_CASE([$enable_wasm_dynamic_linking], [yes], [ac_cv_func_dlopen=yes], [no], [ac_cv_func_dlopen=no], @@ -2776,7 +2863,7 @@ AS_VAR_IF([jit_flags], [], [AS_VAR_APPEND([CFLAGS_NODIST], [" $jit_flags"]) AS_VAR_SET([REGEN_JIT_COMMAND], - ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host}"]) + ["\$(PYTHON_FOR_REGEN) \$(srcdir)/Tools/jit/build.py ${ARCH_TRIPLES:-$host} --output-dir . --pyconfig-dir . --cflags=\"$CFLAGS_JIT\""]) AS_VAR_SET([JIT_STENCILS_H], ["jit_stencils.h"]) AS_VAR_IF([Py_DEBUG], [true], @@ -2992,7 +3079,8 @@ AC_CHECK_HEADERS([ \ sys/endian.h sys/epoll.h sys/event.h sys/eventfd.h sys/file.h sys/ioctl.h sys/kern_control.h \ sys/loadavg.h sys/lock.h sys/memfd.h sys/mkdev.h sys/mman.h sys/modem.h sys/param.h sys/pidfd.h sys/poll.h \ sys/random.h sys/resource.h sys/select.h sys/sendfile.h sys/socket.h sys/soundcard.h sys/stat.h \ - sys/statvfs.h sys/sys_domain.h sys/syscall.h sys/sysmacros.h sys/termio.h sys/time.h sys/times.h sys/timerfd.h \ + sys/statvfs.h sys/sys_domain.h sys/syscall.h sys/sysctl.h \ + sys/sysmacros.h sys/termio.h sys/time.h sys/times.h sys/timerfd.h \ sys/types.h sys/uio.h sys/un.h sys/utsname.h sys/wait.h sys/xattr.h sysexits.h syslog.h \ termios.h util.h utime.h utmp.h \ ]) @@ -3172,10 +3260,6 @@ if test "$ac_cv_sizeof_off_t" -gt "$ac_cv_sizeof_long" -a \ else have_largefile_support="no" fi -dnl LFS does not work with Emscripten 3.1 -AS_CASE([$ac_sys_system], - [Emscripten], [have_largefile_support="no"] -) AS_VAR_IF([have_largefile_support], [yes], [ AC_DEFINE([HAVE_LARGEFILE_SUPPORT], [1], [Defined to enable large file support when an off_t is bigger than a long @@ -3740,15 +3824,17 @@ dnl check for uuid dependencies AH_TEMPLATE([HAVE_UUID_H], [Define to 1 if you have the <uuid.h> header file.]) AH_TEMPLATE([HAVE_UUID_UUID_H], [Define to 1 if you have the <uuid/uuid.h> header file.]) AH_TEMPLATE([HAVE_UUID_GENERATE_TIME_SAFE], [Define if uuid_generate_time_safe() exists.]) +AH_TEMPLATE([HAVE_UUID_GENERATE_TIME_SAFE_STABLE_MAC], [Define if uuid_generate_time_safe() is able to deduce a MAC address.]) have_uuid=missing dnl AIX provides support for RFC4122 (uuid) in libc.a starting with AIX 6.1 dnl (anno 2007). FreeBSD and OpenBSD provides support in libc as well. dnl Little-endian FreeBSD, OpenBSD and NetBSD needs encoding into an octet dnl stream in big-endian byte-order -AC_CHECK_HEADERS([uuid.h], - [AC_CHECK_FUNCS([uuid_create uuid_enc_be], - [have_uuid=yes +AC_CHECK_HEADERS([uuid.h], [ + AC_CHECK_FUNCS([uuid_create uuid_enc_be], [ + have_uuid=yes + ac_cv_have_uuid_h=yes LIBUUID_CFLAGS=${LIBUUID_CFLAGS-""} LIBUUID_LIBS=${LIBUUID_LIBS-""} ]) @@ -3758,19 +3844,29 @@ AS_VAR_IF([have_uuid], [missing], [ PKG_CHECK_MODULES( [LIBUUID], [uuid >= 2.20], [dnl linux-util's libuuid has uuid_generate_time_safe() since v2.20 (2011) - dnl and provides <uuid.h>. + dnl and provides <uuid.h> assuming specific include paths are given have_uuid=yes - AC_DEFINE([HAVE_UUID_H], [1]) - AC_DEFINE([HAVE_UUID_GENERATE_TIME_SAFE], [1]) + ac_cv_have_uuid_generate_time_safe=yes + # The uuid.h file to include may be <uuid.h> *or* <uuid/uuid.h>. + # Since pkg-config --cflags uuid may return -I/usr/include/uuid, + # it's possible to write '#include <uuid.h>' in _uuidmodule.c, + # assuming that the compiler flags are properly updated. + # + # Ideally, we should have defined HAVE_UUID_H if and only if + # #include <uuid.h> can be written, *without* assuming extra + # include path. + ac_cv_have_uuid_h=yes ], [ WITH_SAVE_ENV([ CPPFLAGS="$CPPFLAGS $LIBUUID_CFLAGS" LIBS="$LIBS $LIBUUID_LIBS" AC_CHECK_HEADERS([uuid/uuid.h], [ + ac_cv_have_uuid_uuid_h=yes PY_CHECK_LIB([uuid], [uuid_generate_time], [have_uuid=yes]) - PY_CHECK_LIB([uuid], [uuid_generate_time_safe], - [have_uuid=yes - AC_DEFINE([HAVE_UUID_GENERATE_TIME_SAFE], [1]) ]) ]) + PY_CHECK_LIB([uuid], [uuid_generate_time_safe], [ + have_uuid=yes + ac_cv_have_uuid_generate_time_safe=yes + ])]) AS_VAR_IF([have_uuid], [yes], [ LIBUUID_CFLAGS=${LIBUUID_CFLAGS-""} LIBUUID_LIBS=${LIBUUID_LIBS-"-luuid"} @@ -3785,21 +3881,91 @@ AS_VAR_IF([have_uuid], [missing], [ AC_CHECK_HEADERS([uuid/uuid.h], [ AC_CHECK_FUNC([uuid_generate_time], [ have_uuid=yes + ac_cv_have_uuid_uuid_h=yes LIBUUID_CFLAGS=${LIBUUID_CFLAGS-""} LIBUUID_LIBS=${LIBUUID_LIBS-""} ]) ]) ]) -# gh-124228: While the libuuid library is available on NetBSD, it supports only UUID version 4. +AS_VAR_IF([ac_cv_have_uuid_h], [yes], [AC_DEFINE([HAVE_UUID_H], [1])]) +AS_VAR_IF([ac_cv_have_uuid_uuid_h], [yes], [AC_DEFINE([HAVE_UUID_UUID_H], [1])]) +AS_VAR_IF([ac_cv_have_uuid_generate_time_safe], [yes], [ + AC_DEFINE([HAVE_UUID_GENERATE_TIME_SAFE], [1]) +]) + +# gh-124228: While the libuuid library is available on NetBSD and OpenBSD, +# it supports only UUID version 4. # This restriction inhibits the proper generation of time-based UUIDs. -if test "$ac_sys_system" = "NetBSD"; then +if test "$ac_sys_system" = "NetBSD" || test "$ac_sys_system" = "OpenBSD"; then have_uuid=missing AC_DEFINE([HAVE_UUID_H], [0]) fi AS_VAR_IF([have_uuid], [missing], [have_uuid=no]) +# gh-132710: The UUID node is fetched by using libuuid when possible +# and cached. While the node is constant within the same process, +# different interpreters may have different values as libuuid may +# randomize the node value if the latter cannot be deduced. +# +# Consumers may define HAVE_UUID_GENERATE_TIME_SAFE_STABLE_MAC +# to indicate that libuuid is unstable and should not be relied +# upon to deduce the MAC address. +AC_DEFUN([PY_EXTRACT_UUID_GENERATE_TIME_SAFE_MAC], [WITH_SAVE_ENV([ + # Be sure to add the extra include path if we used pkg-config + # as HAVE_UUID_H may be set even though <uuid.h> is only reachable + # by adding extra -I flags. + # + # If the following script does not compile, we simply assume that + # libuuid is missing. + CFLAGS="$CFLAGS $LIBUUID_CFLAGS" + LIBS="$LIBS $LIBUUID_LIBS" + AC_RUN_IFELSE([AC_LANG_SOURCE([[ + #include <inttypes.h> // PRIu64 + #include <stdint.h> // uint64_t + #include <stdio.h> // fopen(), fclose() + + #ifdef HAVE_UUID_H + #include <uuid.h> + #else + #include <uuid/uuid.h> + #endif + + #define ERR 1 + int main(void) { + uuid_t uuid; // unsigned char[16] + (void)uuid_generate_time_safe(uuid); + uint64_t node = 0; + for (size_t i = 0; i < 6; i++) { + node |= (uint64_t)uuid[15 - i] << (8 * i); + } + FILE *fp = fopen("conftest.out", "w"); + if (fp == NULL) { + return ERR; + } + int rc = fprintf(fp, "%" PRIu64 "\n", node) >= 0; + rc |= fclose(fp); + return rc == 0 ? 0 : ERR; + }]])], [ + AS_VAR_SET([$1], [`cat conftest.out`]) + ], [], [] + )])]) + +if test "$have_uuid" = "yes" -a "$HAVE_UUID_GENERATE_TIME_SAFE" = "1" +then + AC_MSG_CHECKING([if uuid_generate_time_safe() node value is stable]) + PY_EXTRACT_UUID_GENERATE_TIME_SAFE_MAC([py_cv_uuid_node1]) + PY_EXTRACT_UUID_GENERATE_TIME_SAFE_MAC([py_cv_uuid_node2]) + if test -n "$py_cv_uuid_node1" -a "$py_cv_uuid_node1" = "$py_cv_uuid_node2" + then + AC_DEFINE([HAVE_UUID_GENERATE_TIME_SAFE_STABLE_MAC], [1]) + AC_MSG_RESULT([stable]) + else + AC_MSG_RESULT([unstable]) + fi +fi + # 'Real Time' functions on Solaris # posix4 on Solaris 2.6 # pthread (first!) on Linux @@ -4086,7 +4252,7 @@ int main(void) [ac_cv_ffi_complex_double_supported=no]) ])]) if test "$ac_cv_ffi_complex_double_supported" = "yes"; then - AC_DEFINE([Py_FFI_SUPPORT_C_COMPLEX], [1], + AC_DEFINE([_Py_FFI_SUPPORT_C_COMPLEX], [1], [Defined if _Complex C type can be used with libffi.]) fi @@ -4134,13 +4300,13 @@ AS_VAR_IF([with_system_libmpdec], [yes], [have_mpdec=no]) ])], [AC_MSG_WARN([m4_normalize([ - the bundled copy of libmpdecimal is scheduled for removal in Python 3.15; + the bundled copy of libmpdec is scheduled for removal in Python 3.16; consider using a system installed mpdecimal library.])])]) AS_IF([test "$with_system_libmpdec" = "yes" && test "$have_mpdec" = "no"], [AC_MSG_WARN([m4_normalize([ - no system libmpdecimal found; falling back to bundled libmpdecimal - (deprecated and scheduled for removal in Python 3.15)])]) + no system libmpdec found; falling back to bundled libmpdec + (deprecated and scheduled for removal in Python 3.16)])]) USE_BUNDLED_LIBMPDEC()]) # Disable forced inlining in debug builds, see GH-94847 @@ -4379,8 +4545,8 @@ WITH_SAVE_ENV([ # error "Tk older than 8.5.12 not supported" #endif ], [ - void *x1 = Tcl_Init; - void *x2 = Tk_Init; + Tcl_Init(NULL); + Tk_Init(NULL); ]) ], [ have_tcltk=yes @@ -5033,16 +5199,33 @@ then # linked into the binary. Correspondingly, dtrace(1) is missing the ELF # generation flag '-G'. We check for presence of this flag, rather than # hardcoding support by OS, in the interest of robustness. + # + # NetBSD DTrace requires the -x nolibs flag to avoid system library conflicts + # and uses header generation for testing instead of object generation. AC_CACHE_CHECK([whether DTrace probes require linking], - [ac_cv_dtrace_link], [dnl + [ac_cv_dtrace_link], [ ac_cv_dtrace_link=no echo 'BEGIN{}' > conftest.d - "$DTRACE" $DFLAGS -G -s conftest.d -o conftest.o > /dev/null 2>&1 && \ + case $host in + *netbsd*) + DTRACE_TEST_FLAGS="-x nolibs -h" + ;; + *) + DTRACE_TEST_FLAGS="-G" + ;; + esac + "$DTRACE" $DFLAGS $DTRACE_TEST_FLAGS -s conftest.d -o conftest.o > /dev/null 2>&1 && \ ac_cv_dtrace_link=yes ]) if test "$ac_cv_dtrace_link" = "yes"; then DTRACE_OBJS="Python/pydtrace.o" fi + # Set NetBSD-specific DTrace flags in DFLAGS + case $host in + *netbsd*) + DFLAGS="$DFLAGS -x nolibs" + ;; + esac fi dnl Platform-specific C and header files. @@ -5051,7 +5234,7 @@ PLATFORM_OBJS= AS_CASE([$ac_sys_system], [Emscripten], [ - AS_VAR_APPEND([PLATFORM_OBJS], [' Python/emscripten_signal.o Python/emscripten_trampoline.o']) + AS_VAR_APPEND([PLATFORM_OBJS], [' Python/emscripten_signal.o Python/emscripten_trampoline.o Python/emscripten_trampoline_wasm.o Python/emscripten_syscalls.o']) AS_VAR_APPEND([PLATFORM_HEADERS], [' $(srcdir)/Include/internal/pycore_emscripten_signal.h $(srcdir)/Include/internal/pycore_emscripten_trampoline.h']) ], ) @@ -5134,21 +5317,21 @@ fi # checks for library functions AC_CHECK_FUNCS([ \ accept4 alarm bind_textdomain_codeset chmod chown clock closefrom close_range confstr \ - copy_file_range ctermid dladdr dup dup3 execv explicit_bzero explicit_memset \ + copy_file_range ctermid dladdr dup execv explicit_bzero explicit_memset \ faccessat fchmod fchmodat fchown fchownat fdopendir fdwalk fexecve \ fork fork1 fpathconf fstatat ftime ftruncate futimens futimes futimesat \ gai_strerror getegid geteuid getgid getgrent getgrgid getgrgid_r \ - getgrnam_r getgrouplist gethostname getitimer getloadavg getlogin \ + getgrnam_r getgrouplist gethostname getitimer getloadavg getlogin getlogin_r \ getpeername getpgid getpid getppid getpriority _getpty \ getpwent getpwnam_r getpwuid getpwuid_r getresgid getresuid getrusage getsid getspent \ getspnam getuid getwd grantpt if_nameindex initgroups kill killpg lchown linkat \ lockf lstat lutimes madvise mbrtowc memrchr mkdirat mkfifo mkfifoat \ mknod mknodat mktime mmap mremap nice openat opendir pathconf pause pipe \ - pipe2 plock poll posix_fadvise posix_fallocate posix_openpt posix_spawn posix_spawnp \ + plock poll posix_fadvise posix_fallocate posix_openpt posix_spawn posix_spawnp \ posix_spawn_file_actions_addclosefrom_np \ pread preadv preadv2 process_vm_readv \ pthread_cond_timedwait_relative_np pthread_condattr_setclock pthread_init \ - pthread_kill pthread_get_name_np pthread_getname_np pthread_set_name_np + pthread_kill pthread_get_name_np pthread_getname_np pthread_set_name_np \ pthread_setname_np pthread_getattr_np \ ptsname ptsname_r pwrite pwritev pwritev2 readlink readlinkat readv realpath renameat \ rtpSpawn sched_get_priority_max sched_rr_get_interval sched_setaffinity \ @@ -5158,7 +5341,7 @@ AC_CHECK_FUNCS([ \ setresuid setreuid setsid setuid setvbuf shutdown sigaction sigaltstack \ sigfillset siginterrupt sigpending sigrelse sigtimedwait sigwait \ sigwaitinfo snprintf splice strftime strlcpy strsignal symlinkat sync \ - sysconf tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \ + sysconf sysctlbyname tcgetpgrp tcsetpgrp tempnam timegm times tmpfile \ tmpnam tmpnam_r truncate ttyname_r umask uname unlinkat unlockpt utimensat utimes vfork \ wait wait3 wait4 waitid waitpid wcscoll wcsftime wcsxfrm wmemcmp writev \ ]) @@ -5175,7 +5358,7 @@ fi # header definition prevents usage - autoconf doesn't use the headers), or # raise an error if used at runtime. Force these symbols off. if test "$ac_sys_system" != "iOS" ; then - AC_CHECK_FUNCS([getentropy getgroups system]) + AC_CHECK_FUNCS([dup3 getentropy getgroups pipe2 system]) fi AC_CHECK_DECL([dirfd], @@ -5387,7 +5570,33 @@ PKG_CHECK_MODULES([LIBLZMA], [liblzma], [have_liblzma=yes], [ ]) dnl zstd 1.4.5 stabilised ZDICT_finalizeDictionary -PKG_CHECK_MODULES([LIBZSTD], [libzstd >= 1.4.5], [have_libzstd=yes], [have_libzstd=no]) +PKG_CHECK_MODULES([LIBZSTD], [libzstd >= 1.4.5], [have_libzstd=yes], [ + WITH_SAVE_ENV([ + CPPFLAGS="$CPPFLAGS $LIBZSTD_CFLAGS" + CFLAGS="$CFLAGS $LIBZSTD_CFLAGS" + LIBS="$LIBS $LIBZSTD_LIBS" + AC_SEARCH_LIBS([ZDICT_finalizeDictionary], [zstd], [ + AC_MSG_CHECKING([ZSTD_VERSION_NUMBER >= 1.4.5]) + AC_COMPILE_IFELSE([ + AC_LANG_PROGRAM([@%:@include "zstd.h"], [ + #if ZSTD_VERSION_NUMBER < 10405 + # error "zstd version is too old" + #endif + ]) + ], [ + AC_MSG_RESULT([yes]) + AC_CHECK_HEADERS([zstd.h zdict.h], [have_libzstd=yes], [have_libzstd=no]) + ], [ + AC_MSG_RESULT([no]) + have_libzstd=no + ]) + ], [have_libzstd=no]) + AS_VAR_IF([have_libzstd], [yes], [ + LIBZSTD_CFLAGS=${LIBZSTD_CFLAGS-""} + LIBZSTD_LIBS=${LIBZSTD_LIBS-"-lzstd"} + ]) + ]) +]) dnl PY_CHECK_NETDB_FUNC(FUNCTION) AC_DEFUN([PY_CHECK_NETDB_FUNC], [PY_CHECK_FUNC([$1], [@%:@include <netdb.h>])]) @@ -5431,6 +5640,18 @@ PY_CHECK_FUNC([setgroups], [ #endif ]) +AC_CHECK_DECL([MAXLOGNAME], + [AC_DEFINE([HAVE_MAXLOGNAME], [1], + [Define if you have the 'MAXLOGNAME' constant.])], + [], + [@%:@include <sys/param.h>]) + +AC_CHECK_DECLS([UT_NAMESIZE], + [AC_DEFINE([HAVE_UT_NAMESIZE], [1], + [Define if you have the 'HAVE_UT_NAMESIZE' constant.])], + [], + [@%:@include <utmp.h>]) + # check for openpty, login_tty, and forkpty AC_CHECK_FUNCS([openpty], [], @@ -5956,8 +6177,8 @@ AS_VAR_IF([ac_cv_gcc_asm_for_x87], [yes], [ AC_CACHE_CHECK([whether we can use gcc inline assembler to get and set mc68881 fpcr], [ac_cv_gcc_asm_for_mc68881], [ AC_LINK_IFELSE( [AC_LANG_PROGRAM([[]], [[ unsigned int fpcr; - __asm__ __volatile__ ("fmove.l %%fpcr,%0" : "=g" (fpcr)); - __asm__ __volatile__ ("fmove.l %0,%%fpcr" : : "g" (fpcr)); + __asm__ __volatile__ ("fmove.l %%fpcr,%0" : "=dm" (fpcr)); + __asm__ __volatile__ ("fmove.l %0,%%fpcr" : : "dm" (fpcr)); ]])],[ac_cv_gcc_asm_for_mc68881=yes],[ac_cv_gcc_asm_for_mc68881=no]) ]) AS_VAR_IF([ac_cv_gcc_asm_for_mc68881], [yes], [ @@ -6674,57 +6895,6 @@ then [Define if you have struct stat.st_mtimensec]) fi -AC_CACHE_CHECK([whether year with century should be normalized for strftime], [ac_cv_normalize_century], [ -AC_RUN_IFELSE([AC_LANG_SOURCE([[ -#include <time.h> -#include <string.h> - -int main(void) -{ - char year[5]; - struct tm date = { - .tm_year = -1801, - .tm_mon = 0, - .tm_mday = 1 - }; - if (strftime(year, sizeof(year), "%Y", &date) && !strcmp(year, "0099")) { - return 1; - } - return 0; -} -]])], -[ac_cv_normalize_century=yes], -[ac_cv_normalize_century=no], -[ac_cv_normalize_century=yes])]) -if test "$ac_cv_normalize_century" = yes -then - AC_DEFINE([Py_NORMALIZE_CENTURY], [1], - [Define if year with century should be normalized for strftime.]) -fi - -AC_CACHE_CHECK([whether C99-compatible strftime specifiers are supported], [ac_cv_strftime_c99_support], [ -AC_RUN_IFELSE([AC_LANG_SOURCE([[ -#include <time.h> -#include <string.h> - -int main(void) -{ - char full_date[11]; - struct tm date = { - .tm_year = 0, - .tm_mon = 0, - .tm_mday = 1 - }; - if (strftime(full_date, sizeof(full_date), "%F", &date) && !strcmp(full_date, "1900-01-01")) { - return 0; - } - return 1; -} -]])], -[ac_cv_strftime_c99_support=yes], -[AC_MSG_ERROR([Python requires C99-compatible strftime specifiers])], -[ac_cv_strftime_c99_support=])]) - dnl check for ncursesw/ncurses and panelw/panel dnl NOTE: old curses is not detected. dnl have_curses=[no, yes] @@ -6878,6 +7048,30 @@ AC_DEFUN([PY_CHECK_CURSES_FUNC], AS_VAR_POPDEF([py_define]) ]) +dnl PY_CHECK_CURSES_VAR(VARIABLE) +dnl Like PY_CHECK_CURSES_FUNC, but for an integer variable (or macro), such as +dnl ESCDELAY, which a function probe cannot detect. +AC_DEFUN([PY_CHECK_CURSES_VAR], +[ AS_VAR_PUSHDEF([py_var], [ac_cv_lib_curses_$1]) + AS_VAR_PUSHDEF([py_define], [HAVE_CURSES_]m4_toupper($1)) + AC_CACHE_CHECK( + [for curses variable $1], + [py_var], + [AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM(_CURSES_INCLUDES, [ + int x = $1; (void)x; + ])], + [AS_VAR_SET([py_var], [yes])], + [AS_VAR_SET([py_var], [no])])] + ) + AS_VAR_IF( + [py_var], + [yes], + [AC_DEFINE([py_define], [1], [Define if you have the '$1' variable.])]) + AS_VAR_POPDEF([py_var]) + AS_VAR_POPDEF([py_define]) +]) + PY_CHECK_CURSES_FUNC([is_pad]) PY_CHECK_CURSES_FUNC([is_term_resized]) PY_CHECK_CURSES_FUNC([resize_term]) @@ -6889,6 +7083,22 @@ PY_CHECK_CURSES_FUNC([filter]) PY_CHECK_CURSES_FUNC([has_key]) PY_CHECK_CURSES_FUNC([typeahead]) PY_CHECK_CURSES_FUNC([use_env]) +PY_CHECK_CURSES_FUNC([set_escdelay]) +PY_CHECK_CURSES_FUNC([set_tabsize]) +PY_CHECK_CURSES_VAR([ESCDELAY]) +PY_CHECK_CURSES_VAR([TABSIZE]) + +dnl Probe for the X/Open getmouse(MEVENT *) signature specifically: PDCurses +dnl declares an incompatible getmouse(void) unless built for the ncurses mouse API. +AC_CACHE_CHECK([for ncurses-style curses function getmouse], + [ac_cv_lib_curses_getmouse], + [AC_COMPILE_IFELSE( + [AC_LANG_PROGRAM(_CURSES_INCLUDES, [MEVENT event; (void)getmouse(&event);])], + [ac_cv_lib_curses_getmouse=yes], + [ac_cv_lib_curses_getmouse=no])]) +AS_VAR_IF([ac_cv_lib_curses_getmouse], [yes], + [AC_DEFINE([HAVE_CURSES_GETMOUSE], [1], + [Define if you have the 'getmouse' function with the X/Open signature.])]) CPPFLAGS=$ac_save_cppflags ])dnl have_curses != no ])dnl save env @@ -7053,8 +7263,6 @@ if test "$with_remote_debug" = yes; then [Define if you want to enable remote debugging support.]) AC_MSG_RESULT([yes]) else - AC_DEFINE([Py_REMOTE_DEBUG], [0], - [Define if you want to enable remote debugging support.]) AC_MSG_RESULT([no]) fi @@ -7151,6 +7359,34 @@ if test "$have_glibc_memmove_bug" = yes; then for memmove and bcopy.]) fi +AC_MSG_CHECKING([if we need to manually block large inlining in ceval.c]) +AC_RUN_IFELSE([AC_LANG_SOURCE([[ +int main(void) { +// See gh-148284: Clang 22 seems to have interactions with inlining +// and the stackref buffer which cause 40 kB of stack usage on x86-64 +// in buggy versions of _PyEval_EvalFrameDefault() in computed goto +// interpreter. The normal usage seen is normally 1-2 kB. +#if defined(__clang__) && (__clang_major__ == 22) + return 1; +#else + return 0; +#endif +} +]])], +[block_huge_inlining_in_ceval=no], +[block_huge_inlining_in_ceval=yes], +[block_huge_inlining_in_ceval=undefined]) +AC_MSG_RESULT([$block_huge_inlining_in_ceval]) + +if test "$block_huge_inlining_in_ceval" = yes && test "$ac_cv_computed_gotos" = yes; then + # gh-148284: Suppress inlining of functions whose stack size exceeds + # 512 bytes. This number should be tuned to follow the C stack + # consumption in _PyEval_EvalFrameDefault() on computed goto + # interpreter. + CFLAGS_CEVAL="$CFLAGS_CEVAL -finline-max-stacksize=512" +fi +AC_SUBST([CFLAGS_CEVAL]) + if test "$ac_cv_gcc_asm_for_x87" = yes; then # Some versions of gcc miscompile inline asm: # http://gcc.gnu.org/bugzilla/show_bug.cgi?id=46491 @@ -7662,9 +7898,7 @@ AS_CASE([$ac_sys_system], ) dnl fcntl, readline, and termios are not particularly useful in browsers. PY_STDLIB_MOD_SET_NA( - [fcntl], [readline], - [termios], ) ], [WASI], [ @@ -7894,6 +8128,15 @@ AC_SUBST([LIBHACL_CFLAGS]) LIBHACL_LDFLAGS= # for now, no specific linker flags are needed AC_SUBST([LIBHACL_LDFLAGS]) +dnl Check if universal2 HACL* implementation should be used. +if test "$UNIVERSAL_ARCHS" = "universal2" -o \ + \( "$build_cpu" = "aarch64" -a "$build_vendor" = "apple" \) +then + use_hacl_universal2_impl=yes +else + use_hacl_universal2_impl=no +fi + # The SIMD files use aligned_alloc, which is not available on older versions of # Android. # The *mmintrin.h headers are x86-family-specific, so can't be used on WASI. @@ -7904,13 +8147,14 @@ then AX_CHECK_COMPILE_FLAG([-msse -msse2 -msse3 -msse4.1 -msse4.2],[ [LIBHACL_SIMD128_FLAGS="-msse -msse2 -msse3 -msse4.1 -msse4.2"] - AC_DEFINE([HACL_CAN_COMPILE_SIMD128], [1], [HACL* library can compile SIMD128 implementations]) + AC_DEFINE([_Py_HACL_CAN_COMPILE_VEC128], [1], [ + HACL* library can compile SIMD128 implementations]) # macOS universal2 builds *support* the -msse etc flags because they're # available on x86_64. However, performance of the HACL SIMD128 implementation # isn't great, so it's disabled on ARM64. AC_MSG_CHECKING([for HACL* SIMD128 implementation]) - if test "$UNIVERSAL_ARCHS" == "universal2"; then + if test "$use_hacl_universal2_impl" = "yes"; then [LIBHACL_BLAKE2_SIMD128_OBJS="Modules/_hacl/Hacl_Hash_Blake2s_Simd128_universal2.o"] AC_MSG_RESULT([universal2]) else @@ -7935,14 +8179,15 @@ if test "$ac_sys_system" != "Linux-android" -a "$ac_sys_system" != "WASI" || \ then AX_CHECK_COMPILE_FLAG([-mavx2],[ [LIBHACL_SIMD256_FLAGS="-mavx2"] - AC_DEFINE([HACL_CAN_COMPILE_SIMD256], [1], [HACL* library can compile SIMD256 implementations]) + AC_DEFINE([_Py_HACL_CAN_COMPILE_VEC256], [1], [ + HACL* library can compile SIMD256 implementations]) # macOS universal2 builds *support* the -mavx2 compiler flag because it's # available on x86_64; but the HACL SIMD256 build then fails because the # implementation requires symbols that aren't available on ARM64. Use a # wrapped implementation if we're building for universal2. AC_MSG_CHECKING([for HACL* SIMD256 implementation]) - if test "$UNIVERSAL_ARCHS" == "universal2"; then + if test "$use_hacl_universal2_impl" = "yes"; then [LIBHACL_BLAKE2_SIMD256_OBJS="Modules/_hacl/Hacl_Hash_Blake2b_Simd256_universal2.o"] AC_MSG_RESULT([universal2]) else @@ -7959,7 +8204,7 @@ AC_SUBST([LIBHACL_BLAKE2_SIMD256_OBJS]) # HACL*-based cryptographic primitives AC_MSG_CHECKING([for HACL* library linking type]) -if test "$ac_sys_system" = "WASI"; then +if test "$ac_sys_system" = "WASI" || test "$MODULE_BUILDTYPE" = "static"; then LIBHACL_LDEPS_LIBTYPE=STATIC AC_MSG_RESULT([static]) else @@ -8065,9 +8310,11 @@ PY_STDLIB_MOD([_testmultiphase], [test "$TEST_MODULES" = yes], [test "$ac_cv_fun PY_STDLIB_MOD([_testsinglephase], [test "$TEST_MODULES" = yes], [test "$ac_cv_func_dlopen" = yes]) PY_STDLIB_MOD([xxsubtype], [test "$TEST_MODULES" = yes]) PY_STDLIB_MOD([_xxtestfuzz], [test "$TEST_MODULES" = yes]) +dnl Check have_libffi so _ctypes_test is only built if _ctypes is built. +dnl _ctypes_test doesn't use libffi directly. PY_STDLIB_MOD([_ctypes_test], [test "$TEST_MODULES" = yes], [test "$have_libffi" = yes -a "$ac_cv_func_dlopen" = yes], - [$LIBFFI_CFLAGS], [$LIBFFI_LIBS $LIBM]) + [], [$LIBM]) dnl Limited API template modules. dnl Emscripten does not support shared libraries yet. diff --git a/iOS/README.rst b/iOS/README.rst deleted file mode 100644 index 13b885144932e47..000000000000000 --- a/iOS/README.rst +++ /dev/null @@ -1,375 +0,0 @@ -==================== -Python on iOS README -==================== - -:Authors: - Russell Keith-Magee (2023-11) - -This document provides a quick overview of some iOS specific features in the -Python distribution. - -These instructions are only needed if you're planning to compile Python for iOS -yourself. Most users should *not* need to do this. If you're looking to -experiment with writing an iOS app in Python, tools such as `BeeWare's Briefcase -<https://briefcase.readthedocs.io>`__ and `Kivy's Buildozer -<https://buildozer.readthedocs.io>`__ will provide a much more approachable -user experience. - -Compilers for building on iOS -============================= - -Building for iOS requires the use of Apple's Xcode tooling. It is strongly -recommended that you use the most recent stable release of Xcode. This will -require the use of the most (or second-most) recently released macOS version, -as Apple does not maintain Xcode for older macOS versions. The Xcode Command -Line Tools are not sufficient for iOS development; you need a *full* Xcode -install. - -If you want to run your code on the iOS simulator, you'll also need to install -an iOS Simulator Platform. You should be prompted to select an iOS Simulator -Platform when you first run Xcode. Alternatively, you can add an iOS Simulator -Platform by selecting an open the Platforms tab of the Xcode Settings panel. - -iOS specific arguments to configure -=================================== - -* ``--enable-framework[=DIR]`` - - This argument specifies the location where the Python.framework will be - installed. If ``DIR`` is not specified, the framework will be installed into - a subdirectory of the ``iOS/Frameworks`` folder. - - This argument *must* be provided when configuring iOS builds. iOS does not - support non-framework builds. - -* ``--with-framework-name=NAME`` - - Specify the name for the Python framework; defaults to ``Python``. - - .. admonition:: Use this option with care! - - Unless you know what you're doing, changing the name of the Python - framework on iOS is not advised. If you use this option, you won't be able - to run the ``make testios`` target without making significant manual - alterations, and you won't be able to use any binary packages unless you - compile them yourself using your own framework name. - -Building Python on iOS -====================== - -ABIs and Architectures ----------------------- - -iOS apps can be deployed on physical devices, and on the iOS simulator. Although -the API used on these devices is identical, the ABI is different - you need to -link against different libraries for an iOS device build (``iphoneos``) or an -iOS simulator build (``iphonesimulator``). - -Apple uses the ``XCframework`` format to allow specifying a single dependency -that supports multiple ABIs. An ``XCframework`` is a wrapper around multiple -ABI-specific frameworks that share a common API. - -iOS can also support different CPU architectures within each ABI. At present, -there is only a single supported architecture on physical devices - ARM64. -However, the *simulator* supports 2 architectures - ARM64 (for running on Apple -Silicon machines), and x86_64 (for running on older Intel-based machines). - -To support multiple CPU architectures on a single platform, Apple uses a "fat -binary" format - a single physical file that contains support for multiple -architectures. It is possible to compile and use a "thin" single architecture -version of a binary for testing purposes; however, the "thin" binary will not be -portable to machines using other architectures. - -Building a single-architecture framework ----------------------------------------- - -The Python build system will create a ``Python.framework`` that supports a -*single* ABI with a *single* architecture. Unlike macOS, iOS does not allow a -framework to contain non-library content, so the iOS build will produce a -``bin`` and ``lib`` folder in the same output folder as ``Python.framework``. -The ``lib`` folder will be needed at runtime to support the Python library. - -If you want to use Python in a real iOS project, you need to produce multiple -``Python.framework`` builds, one for each ABI and architecture. iOS builds of -Python *must* be constructed as framework builds. To support this, you must -provide the ``--enable-framework`` flag when configuring the build. The build -also requires the use of cross-compilation. The minimal commands for building -Python for the ARM64 iOS simulator will look something like:: - - $ export PATH="$(pwd)/iOS/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin" - $ ./configure \ - --enable-framework \ - --host=arm64-apple-ios-simulator \ - --build=arm64-apple-darwin \ - --with-build-python=/path/to/python.exe - $ make - $ make install - -In this invocation: - -* ``iOS/Resources/bin`` has been added to the path, providing some shims for the - compilers and linkers needed by the build. Xcode requires the use of ``xcrun`` - to invoke compiler tooling. However, if ``xcrun`` is pre-evaluated and the - result passed to ``configure``, these results can embed user- and - version-specific paths into the sysconfig data, which limits the portability - of the compiled Python. Alternatively, if ``xcrun`` is used *as* the compiler, - it requires that compiler variables like ``CC`` include spaces, which can - cause significant problems with many C configuration systems which assume that - ``CC`` will be a single executable. - - To work around this problem, the ``iOS/Resources/bin`` folder contains some - wrapper scripts that present as simple compilers and linkers, but wrap - underlying calls to ``xcrun``. This allows configure to use a ``CC`` - definition without spaces, and without user- or version-specific paths, while - retaining the ability to adapt to the local Xcode install. These scripts are - included in the ``bin`` directory of an iOS install. - - These scripts will, by default, use the currently active Xcode installation. - If you want to use a different Xcode installation, you can use - ``xcode-select`` to set a new default Xcode globally, or you can use the - ``DEVELOPER_DIR`` environment variable to specify an Xcode install. The - scripts will use the default ``iphoneos``/``iphonesimulator`` SDK version for - the select Xcode install; if you want to use a different SDK, you can set the - ``IOS_SDK_VERSION`` environment variable. (e.g, setting - ``IOS_SDK_VERSION=17.1`` would cause the scripts to use the ``iphoneos17.1`` - and ``iphonesimulator17.1`` SDKs, regardless of the Xcode default.) - - The path has also been cleared of any user customizations. A common source of - bugs is for tools like Homebrew to accidentally leak macOS binaries into an iOS - build. Resetting the path to a known "bare bones" value is the easiest way to - avoid these problems. - -* ``--host`` is the architecture and ABI that you want to build, in GNU compiler - triple format. This will be one of: - - - ``arm64-apple-ios`` for ARM64 iOS devices. - - ``arm64-apple-ios-simulator`` for the iOS simulator running on Apple - Silicon devices. - - ``x86_64-apple-ios-simulator`` for the iOS simulator running on Intel - devices. - -* ``--build`` is the GNU compiler triple for the machine that will be running - the compiler. This is one of: - - - ``arm64-apple-darwin`` for Apple Silicon devices. - - ``x86_64-apple-darwin`` for Intel devices. - -* ``/path/to/python.exe`` is the path to a Python binary on the machine that - will be running the compiler. This is needed because the Python compilation - process involves running some Python code. On a normal desktop build of - Python, you can compile a python interpreter and then use that interpreter to - run Python code. However, the binaries produced for iOS won't run on macOS, so - you need to provide an external Python interpreter. This interpreter must be - the same version as the Python that is being compiled. To be completely safe, - this should be the *exact* same commit hash. However, the longer a Python - release has been stable, the more likely it is that this constraint can be - relaxed - the same micro version will often be sufficient. - -* The ``install`` target for iOS builds is slightly different to other - platforms. On most platforms, ``make install`` will install the build into - the final runtime location. This won't be the case for iOS, as the final - runtime location will be on a physical device. - - However, you still need to run the ``install`` target for iOS builds, as it - performs some final framework assembly steps. The location specified with - ``--enable-framework`` will be the location where ``make install`` will - assemble the complete iOS framework. This completed framework can then - be copied and relocated as required. - -For a full CPython build, you also need to specify the paths to iOS builds of -the binary libraries that CPython depends on (XZ, BZip2, LibFFI and OpenSSL). -This can be done by defining the ``LIBLZMA_CFLAGS``, ``LIBLZMA_LIBS``, -``BZIP2_CFLAGS``, ``BZIP2_LIBS``, ``LIBFFI_CFLAGS``, and ``LIBFFI_LIBS`` -environment variables, and the ``--with-openssl`` configure option. Versions of -these libraries pre-compiled for iOS can be found in `this repository -<https://github.com/beeware/cpython-apple-source-deps/releases>`__. LibFFI is -especially important, as many parts of the standard library (including the -``platform``, ``sysconfig`` and ``webbrowser`` modules) require the use of the -``ctypes`` module at runtime. - -By default, Python will be compiled with an iOS deployment target (i.e., the -minimum supported iOS version) of 13.0. To specify a different deployment -target, provide the version number as part of the ``--host`` argument - for -example, ``--host=arm64-apple-ios15.4-simulator`` would compile an ARM64 -simulator build with a deployment target of 15.4. - -Merge thin frameworks into fat frameworks ------------------------------------------ - -Once you've built a ``Python.framework`` for each ABI and and architecture, you -must produce a "fat" framework for each ABI that contains all the architectures -for that ABI. - -The ``iphoneos`` build only needs to support a single architecture, so it can be -used without modification. - -If you only want to support a single simulator architecture, (e.g., only support -ARM64 simulators), you can use a single architecture ``Python.framework`` build. -However, if you want to create ``Python.xcframework`` that supports *all* -architectures, you'll need to merge the ``iphonesimulator`` builds for ARM64 and -x86_64 into a single "fat" framework. - -The "fat" framework can be constructed by performing a directory merge of the -content of the two "thin" ``Python.framework`` directories, plus the ``bin`` and -``lib`` folders for each thin framework. When performing this merge: - -* The pure Python standard library content is identical for each architecture, - except for a handful of platform-specific files (such as the ``sysconfig`` - module). Ensure that the "fat" framework has the union of all standard library - files. - -* Any binary files in the standard library, plus the main - ``libPython3.X.dylib``, can be merged using the ``lipo`` tool, provide by - Xcode:: - - $ lipo -create -output module.dylib path/to/x86_64/module.dylib path/to/arm64/module.dylib - -* The header files will be identical on both architectures, except for - ``pyconfig.h``. Copy all the headers from one platform (say, arm64), rename - ``pyconfig.h`` to ``pyconfig-arm64.h``, and copy the ``pyconfig.h`` for the - other architecture into the merged header folder as ``pyconfig-x86_64.h``. - Then copy the ``iOS/Resources/pyconfig.h`` file from the CPython sources into - the merged headers folder. This will allow the two Python architectures to - share a common ``pyconfig.h`` header file. - -At this point, you should have 2 Python.framework folders - one for ``iphoneos``, -and one for ``iphonesimulator`` that is a merge of x86+64 and ARM64 content. - -Merge frameworks into an XCframework ------------------------------------- - -Now that we have 2 (potentially fat) ABI-specific frameworks, we can merge those -frameworks into a single ``XCframework``. - -The initial skeleton of an ``XCframework`` is built using:: - - xcodebuild -create-xcframework -output Python.xcframework -framework path/to/iphoneos/Python.framework -framework path/to/iphonesimulator/Python.framework - -Then, copy the ``bin`` and ``lib`` folders into the architecture-specific slices of -the XCframework:: - - cp path/to/iphoneos/bin Python.xcframework/ios-arm64 - cp path/to/iphoneos/lib Python.xcframework/ios-arm64 - - cp path/to/iphonesimulator/bin Python.xcframework/ios-arm64_x86_64-simulator - cp path/to/iphonesimulator/lib Python.xcframework/ios-arm64_x86_64-simulator - -Note that the name of the architecture-specific slice for the simulator will -depend on the CPU architecture(s) that you build. - -You now have a Python.xcframework that can be used in a project. - -Testing Python on iOS -===================== - -The ``iOS/testbed`` folder that contains an Xcode project that is able to run -the iOS test suite. This project converts the Python test suite into a single -test case in Xcode's XCTest framework. The single XCTest passes if the test -suite passes. - -To run the test suite, configure a Python build for an iOS simulator (i.e., -``--host=arm64-apple-ios-simulator`` or ``--host=x86_64-apple-ios-simulator`` -), specifying a framework build (i.e. ``--enable-framework``). Ensure that your -``PATH`` has been configured to include the ``iOS/Resources/bin`` folder and -exclude any non-iOS tools, then run:: - - $ make all - $ make install - $ make testios - -This will: - -* Build an iOS framework for your chosen architecture; -* Finalize the single-platform framework; -* Make a clean copy of the testbed project; -* Install the Python iOS framework into the copy of the testbed project; and -* Run the test suite on an "iPhone SE (3rd generation)" simulator. - -On success, the test suite will exit and report successful completion of the -test suite. On a 2022 M1 MacBook Pro, the test suite takes approximately 15 -minutes to run; a couple of extra minutes is required to compile the testbed -project, and then boot and prepare the iOS simulator. - -Debugging test failures ------------------------ - -Running ``make test`` generates a standalone version of the ``iOS/testbed`` -project, and runs the full test suite. It does this using ``iOS/testbed`` -itself - the folder is an executable module that can be used to create and run -a clone of the testbed project. - -You can generate your own standalone testbed instance by running:: - - $ python iOS/testbed clone --framework iOS/Frameworks/arm64-iphonesimulator my-testbed - -This invocation assumes that ``iOS/Frameworks/arm64-iphonesimulator`` is the -path to the iOS simulator framework for your platform (ARM64 in this case); -``my-testbed`` is the name of the folder for the new testbed clone. - -You can then use the ``my-testbed`` folder to run the Python test suite, -passing in any command line arguments you may require. For example, if you're -trying to diagnose a failure in the ``os`` module, you might run:: - - $ python my-testbed run -- test -W test_os - -This is the equivalent of running ``python -m test -W test_os`` on a desktop -Python build. Any arguments after the ``--`` will be passed to testbed as if -they were arguments to ``python -m`` on a desktop machine. - -You can also open the testbed project in Xcode by running:: - - $ open my-testbed/iOSTestbed.xcodeproj - -This will allow you to use the full Xcode suite of tools for debugging. - -Testing on an iOS device -^^^^^^^^^^^^^^^^^^^^^^^^ - -To test on an iOS device, the app needs to be signed with known developer -credentials. To obtain these credentials, you must have an iOS Developer -account, and your Xcode install will need to be logged into your account (see -the Accounts tab of the Preferences dialog). - -Once the project is open, and you're signed into your Apple Developer account, -select the root node of the project tree (labeled "iOSTestbed"), then the -"Signing & Capabilities" tab in the details page. Select a development team -(this will likely be your own name), and plug in a physical device to your -macOS machine with a USB cable. You should then be able to select your physical -device from the list of targets in the pulldown in the Xcode titlebar. - -Running specific tests -^^^^^^^^^^^^^^^^^^^^^^ - -As the test suite is being executed on an iOS simulator, it is not possible to -pass in command line arguments to configure test suite operation. To work -around this limitation, the arguments that would normally be passed as command -line arguments are configured as part of the ``iOSTestbed-Info.plist`` file -that is used to configure the iOS testbed app. In this file, the ``TestArgs`` -key is an array containing the arguments that would be passed to ``python -m`` -on the command line (including ``test`` in position 0, the name of the test -module to be executed). - -Disabling automated breakpoints -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -By default, Xcode will inserts an automatic breakpoint whenever a signal is -raised. The Python test suite raises many of these signals as part of normal -operation; unless you are trying to diagnose an issue with signals, the -automatic breakpoints can be inconvenient. However, they can be disabled by -creating a symbolic breakpoint that is triggered at the start of the test run. - -Select "Debug > Breakpoints > Create Symbolic Breakpoint" from the Xcode menu, and -populate the new brewpoint with the following details: - -* **Name**: IgnoreSignals -* **Symbol**: UIApplicationMain -* **Action**: Add debugger commands for: - - ``process handle SIGINT -n true -p true -s false`` - - ``process handle SIGUSR1 -n true -p true -s false`` - - ``process handle SIGUSR2 -n true -p true -s false`` - - ``process handle SIGXFSZ -n true -p true -s false`` -* Check the "Automatically continue after evaluating" box. - -All other details can be left blank. When the process executes the -``UIApplicationMain`` entry point, the breakpoint will trigger, run the debugger -commands to disable the automatic breakpoints, and automatically resume. diff --git a/iOS/Resources/bin/arm64-apple-ios-simulator-strip b/iOS/Resources/bin/arm64-apple-ios-simulator-strip new file mode 100755 index 000000000000000..fd59d309b73a202 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-simulator-strip @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} strip -arch arm64 "$@" diff --git a/iOS/Resources/bin/arm64-apple-ios-strip b/iOS/Resources/bin/arm64-apple-ios-strip new file mode 100755 index 000000000000000..75e823a3d02d615 --- /dev/null +++ b/iOS/Resources/bin/arm64-apple-ios-strip @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphoneos${IOS_SDK_VERSION} strip -arch arm64 "$@" diff --git a/iOS/Resources/bin/x86_64-apple-ios-simulator-strip b/iOS/Resources/bin/x86_64-apple-ios-simulator-strip new file mode 100755 index 000000000000000..c5cfb28929195aa --- /dev/null +++ b/iOS/Resources/bin/x86_64-apple-ios-simulator-strip @@ -0,0 +1,2 @@ +#!/bin/sh +xcrun --sdk iphonesimulator${IOS_SDK_VERSION} strip -arch x86_64 "$@" diff --git a/iOS/testbed/__main__.py b/iOS/testbed/__main__.py deleted file mode 100644 index c05497ede3aa615..000000000000000 --- a/iOS/testbed/__main__.py +++ /dev/null @@ -1,548 +0,0 @@ -import argparse -import asyncio -import fcntl -import json -import os -import plistlib -import re -import shutil -import subprocess -import sys -import tempfile -from contextlib import asynccontextmanager -from datetime import datetime -from pathlib import Path - - -DECODE_ARGS = ("UTF-8", "backslashreplace") - -# The system log prefixes each line: -# 2025-01-17 16:14:29.090 Df iOSTestbed[23987:1fd393b4] (Python) ... -# 2025-01-17 16:14:29.090 E iOSTestbed[23987:1fd393b4] (Python) ... - -LOG_PREFIX_REGEX = re.compile( - r"^\d{4}-\d{2}-\d{2}" # YYYY-MM-DD - r"\s+\d+:\d{2}:\d{2}\.\d+" # HH:MM:SS.sss - r"\s+\w+" # Df/E - r"\s+iOSTestbed\[\d+:\w+\]" # Process/thread ID - r"\s+\(Python\)\s" # Logger name -) - - -# Work around a bug involving sys.exit and TaskGroups -# (https://github.com/python/cpython/issues/101515). -def exit(*args): - raise MySystemExit(*args) - - -class MySystemExit(Exception): - pass - - -class SimulatorLock: - # An fcntl-based filesystem lock that can be used to ensure that - def __init__(self, timeout): - self.filename = Path(tempfile.gettempdir()) / "python-ios-testbed" - self.timeout = timeout - - self.fd = None - - async def acquire(self): - # Ensure the lockfile exists - self.filename.touch(exist_ok=True) - - # Try `timeout` times to acquire the lock file, with a 1 second pause - # between each attempt. Report status every 10 seconds. - for i in range(0, self.timeout): - try: - fd = os.open(self.filename, os.O_RDWR | os.O_TRUNC, 0o644) - fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError: - os.close(fd) - if i % 10 == 0: - print("... waiting", flush=True) - await asyncio.sleep(1) - else: - self.fd = fd - return - - # If we reach the end of the loop, we've exceeded the allowed number of - # attempts. - raise ValueError("Unable to obtain lock on iOS simulator creation") - - def release(self): - # If a lock is held, release it. - if self.fd is not None: - # Release the lock. - fcntl.flock(self.fd, fcntl.LOCK_UN) - os.close(self.fd) - self.fd = None - - -# All subprocesses are executed through this context manager so that no matter -# what happens, they can always be cancelled from another task, and they will -# always be cleaned up on exit. -@asynccontextmanager -async def async_process(*args, **kwargs): - process = await asyncio.create_subprocess_exec(*args, **kwargs) - try: - yield process - finally: - if process.returncode is None: - # Allow a reasonably long time for Xcode to clean itself up, - # because we don't want stale emulators left behind. - timeout = 10 - process.terminate() - try: - await asyncio.wait_for(process.wait(), timeout) - except TimeoutError: - print( - f"Command {args} did not terminate after {timeout} seconds " - f" - sending SIGKILL" - ) - process.kill() - - # Even after killing the process we must still wait for it, - # otherwise we'll get the warning "Exception ignored in __del__". - await asyncio.wait_for(process.wait(), timeout=1) - - -async def async_check_output(*args, **kwargs): - async with async_process( - *args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs - ) as process: - stdout, stderr = await process.communicate() - if process.returncode == 0: - return stdout.decode(*DECODE_ARGS) - else: - raise subprocess.CalledProcessError( - process.returncode, - args, - stdout.decode(*DECODE_ARGS), - stderr.decode(*DECODE_ARGS), - ) - - -# Select a simulator device to use. -async def select_simulator_device(): - # List the testing simulators, in JSON format - raw_json = await async_check_output( - "xcrun", "simctl", "--set", "testing", "list", "-j" - ) - json_data = json.loads(raw_json) - - # Any device will do; we'll look for "SE" devices - but the name isn't - # consistent over time. Older Xcode versions will use "iPhone SE (Nth - # generation)"; As of 2025, they've started using "iPhone 16e". - # - # When Xcode is updated after a new release, new devices will be available - # and old ones will be dropped from the set available on the latest iOS - # version. Select the one with the highest minimum runtime version - this - # is an indicator of the "newest" released device, which should always be - # supported on the "most recent" iOS version. - se_simulators = sorted( - (devicetype["minRuntimeVersion"], devicetype["name"]) - for devicetype in json_data["devicetypes"] - if devicetype["productFamily"] == "iPhone" - and ( - ("iPhone " in devicetype["name"] and devicetype["name"].endswith("e")) - or "iPhone SE " in devicetype["name"] - ) - ) - - return se_simulators[-1][1] - - -# Return a list of UDIDs associated with booted simulators -async def list_devices(): - try: - # List the testing simulators, in JSON format - raw_json = await async_check_output( - "xcrun", "simctl", "--set", "testing", "list", "-j" - ) - json_data = json.loads(raw_json) - - # Filter out the booted iOS simulators - return [ - simulator["udid"] - for runtime, simulators in json_data["devices"].items() - for simulator in simulators - if runtime.split(".")[-1].startswith("iOS") and simulator["state"] == "Booted" - ] - except subprocess.CalledProcessError as e: - # If there's no ~/Library/Developer/XCTestDevices folder (which is the - # case on fresh installs, and in some CI environments), `simctl list` - # returns error code 1, rather than an empty list. Handle that case, - # but raise all other errors. - if e.returncode == 1: - return [] - else: - raise - - -async def find_device(initial_devices, lock): - while True: - new_devices = set(await list_devices()).difference(initial_devices) - if len(new_devices) == 0: - await asyncio.sleep(1) - elif len(new_devices) == 1: - udid = new_devices.pop() - print(f"{datetime.now():%Y-%m-%d %H:%M:%S}: New test simulator detected") - print(f"UDID: {udid}", flush=True) - lock.release() - return udid - else: - exit(f"Found more than one new device: {new_devices}") - - -async def log_stream_task(initial_devices, lock): - # Wait up to 5 minutes for the build to complete and the simulator to boot. - udid = await asyncio.wait_for(find_device(initial_devices, lock), 5 * 60) - - # Stream the iOS device's logs, filtering out messages that come from the - # XCTest test suite (catching NSLog messages from the test method), or - # Python itself (catching stdout/stderr content routed to the system log - # with config->use_system_logger). - args = [ - "xcrun", - "simctl", - "--set", - "testing", - "spawn", - udid, - "log", - "stream", - "--style", - "compact", - "--predicate", - ( - 'senderImagePath ENDSWITH "/iOSTestbedTests.xctest/iOSTestbedTests"' - ' OR senderImagePath ENDSWITH "/Python.framework/Python"' - ), - ] - - async with async_process( - *args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) as process: - suppress_dupes = False - while line := (await process.stdout.readline()).decode(*DECODE_ARGS): - # Strip the prefix from each log line - line = LOG_PREFIX_REGEX.sub("", line) - # The iOS log streamer can sometimes lag; when it does, it outputs - # a warning about messages being dropped... often multiple times. - # Only print the first of these duplicated warnings. - if line.startswith("=== Messages dropped "): - if not suppress_dupes: - suppress_dupes = True - sys.stdout.write(line) - else: - suppress_dupes = False - sys.stdout.write(line) - sys.stdout.flush() - - -async def xcode_test(location, simulator, verbose): - # Run the test suite on the named simulator - print("Starting xcodebuild...", flush=True) - args = [ - "xcodebuild", - "test", - "-project", - str(location / "iOSTestbed.xcodeproj"), - "-scheme", - "iOSTestbed", - "-destination", - f"platform=iOS Simulator,name={simulator}", - "-resultBundlePath", - str(location / f"{datetime.now():%Y%m%d-%H%M%S}.xcresult"), - "-derivedDataPath", - str(location / "DerivedData"), - ] - if not verbose: - args += ["-quiet"] - - async with async_process( - *args, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - ) as process: - while line := (await process.stdout.readline()).decode(*DECODE_ARGS): - sys.stdout.write(line) - sys.stdout.flush() - - status = await asyncio.wait_for(process.wait(), timeout=1) - exit(status) - - -def clone_testbed( - source: Path, - target: Path, - framework: Path, - apps: list[Path], -) -> None: - if target.exists(): - print(f"{target} already exists; aborting without creating project.") - sys.exit(10) - - if framework is None: - if not ( - source / "Python.xcframework/ios-arm64_x86_64-simulator/bin" - ).is_dir(): - print( - f"The testbed being cloned ({source}) does not contain " - f"a simulator framework. Re-run with --framework" - ) - sys.exit(11) - else: - if not framework.is_dir(): - print(f"{framework} does not exist.") - sys.exit(12) - elif not ( - framework.suffix == ".xcframework" - or (framework / "Python.framework").is_dir() - ): - print( - f"{framework} is not an XCframework, " - f"or a simulator slice of a framework build." - ) - sys.exit(13) - - print("Cloning testbed project:") - print(f" Cloning {source}...", end="", flush=True) - shutil.copytree(source, target, symlinks=True) - print(" done") - - xc_framework_path = target / "Python.xcframework" - sim_framework_path = xc_framework_path / "ios-arm64_x86_64-simulator" - if framework is not None: - if framework.suffix == ".xcframework": - print(" Installing XCFramework...", end="", flush=True) - if xc_framework_path.is_dir(): - shutil.rmtree(xc_framework_path) - else: - xc_framework_path.unlink(missing_ok=True) - xc_framework_path.symlink_to( - framework.relative_to(xc_framework_path.parent, walk_up=True) - ) - print(" done") - else: - print(" Installing simulator framework...", end="", flush=True) - if sim_framework_path.is_dir(): - shutil.rmtree(sim_framework_path) - else: - sim_framework_path.unlink(missing_ok=True) - sim_framework_path.symlink_to( - framework.relative_to(sim_framework_path.parent, walk_up=True) - ) - print(" done") - else: - if ( - xc_framework_path.is_symlink() - and not xc_framework_path.readlink().is_absolute() - ): - # XCFramework is a relative symlink. Rewrite the symlink relative - # to the new location. - print(" Rewriting symlink to XCframework...", end="", flush=True) - orig_xc_framework_path = ( - source - / xc_framework_path.readlink() - ).resolve() - xc_framework_path.unlink() - xc_framework_path.symlink_to( - orig_xc_framework_path.relative_to( - xc_framework_path.parent, walk_up=True - ) - ) - print(" done") - elif ( - sim_framework_path.is_symlink() - and not sim_framework_path.readlink().is_absolute() - ): - print(" Rewriting symlink to simulator framework...", end="", flush=True) - # Simulator framework is a relative symlink. Rewrite the symlink - # relative to the new location. - orig_sim_framework_path = ( - source - / "Python.XCframework" - / sim_framework_path.readlink() - ).resolve() - sim_framework_path.unlink() - sim_framework_path.symlink_to( - orig_sim_framework_path.relative_to( - sim_framework_path.parent, walk_up=True - ) - ) - print(" done") - else: - print(" Using pre-existing iOS framework.") - - for app_src in apps: - print(f" Installing app {app_src.name!r}...", end="", flush=True) - app_target = target / f"iOSTestbed/app/{app_src.name}" - if app_target.is_dir(): - shutil.rmtree(app_target) - shutil.copytree(app_src, app_target) - print(" done") - - print(f"Successfully cloned testbed: {target.resolve()}") - - -def update_plist(testbed_path, args): - # Add the test runner arguments to the testbed's Info.plist file. - info_plist = testbed_path / "iOSTestbed" / "iOSTestbed-Info.plist" - with info_plist.open("rb") as f: - info = plistlib.load(f) - - info["TestArgs"] = args - - with info_plist.open("wb") as f: - plistlib.dump(info, f) - - -async def run_testbed(simulator: str | None, args: list[str], verbose: bool=False): - location = Path(__file__).parent - print("Updating plist...", end="", flush=True) - update_plist(location, args) - print(" done.", flush=True) - - if simulator is None: - simulator = await select_simulator_device() - print(f"Running test on {simulator}", flush=True) - - # We need to get an exclusive lock on simulator creation, to avoid issues - # with multiple simulators starting and being unable to tell which - # simulator is due to which testbed instance. See - # https://github.com/python/cpython/issues/130294 for details. Wait up to - # 10 minutes for a simulator to boot. - print("Obtaining lock on simulator creation...", flush=True) - simulator_lock = SimulatorLock(timeout=10*60) - await simulator_lock.acquire() - print("Simulator lock acquired.", flush=True) - - # Get the list of devices that are booted at the start of the test run. - # The simulator started by the test suite will be detected as the new - # entry that appears on the device list. - initial_devices = await list_devices() - - try: - async with asyncio.TaskGroup() as tg: - tg.create_task(log_stream_task(initial_devices, simulator_lock)) - tg.create_task(xcode_test(location, simulator=simulator, verbose=verbose)) - except* MySystemExit as e: - raise SystemExit(*e.exceptions[0].args) from None - except* subprocess.CalledProcessError as e: - # Extract it from the ExceptionGroup so it can be handled by `main`. - raise e.exceptions[0] - finally: - simulator_lock.release() - - -def main(): - parser = argparse.ArgumentParser( - description=( - "Manages the process of testing a Python project in the iOS simulator." - ), - ) - - subcommands = parser.add_subparsers(dest="subcommand") - - clone = subcommands.add_parser( - "clone", - description=( - "Clone the testbed project, copying in an iOS Python framework and" - "any specified application code." - ), - help="Clone a testbed project to a new location.", - ) - clone.add_argument( - "--framework", - help=( - "The location of the XCFramework (or simulator-only slice of an " - "XCFramework) to use when running the testbed" - ), - ) - clone.add_argument( - "--app", - dest="apps", - action="append", - default=[], - help="The location of any code to include in the testbed project", - ) - clone.add_argument( - "location", - help="The path where the testbed will be cloned.", - ) - - run = subcommands.add_parser( - "run", - usage="%(prog)s [-h] [--simulator SIMULATOR] -- <test arg> [<test arg> ...]", - description=( - "Run a testbed project. The arguments provided after `--` will be " - "passed to the running iOS process as if they were arguments to " - "`python -m`." - ), - help="Run a testbed project", - ) - run.add_argument( - "--simulator", - help=( - "The name of the simulator to use (eg: 'iPhone 16e'). Defaults to ", - "the most recently released 'entry level' iPhone device." - ) - ) - run.add_argument( - "-v", "--verbose", - action="store_true", - help="Enable verbose output", - ) - - try: - pos = sys.argv.index("--") - testbed_args = sys.argv[1:pos] - test_args = sys.argv[pos + 1 :] - except ValueError: - testbed_args = sys.argv[1:] - test_args = [] - - context = parser.parse_args(testbed_args) - - if context.subcommand == "clone": - clone_testbed( - source=Path(__file__).parent.resolve(), - target=Path(context.location).resolve(), - framework=Path(context.framework).resolve() if context.framework else None, - apps=[Path(app) for app in context.apps], - ) - elif context.subcommand == "run": - if test_args: - if not ( - Path(__file__).parent / "Python.xcframework/ios-arm64_x86_64-simulator/bin" - ).is_dir(): - print( - f"Testbed does not contain a compiled iOS framework. Use " - f"`python {sys.argv[0]} clone ...` to create a runnable " - f"clone of this testbed." - ) - sys.exit(20) - - asyncio.run( - run_testbed( - simulator=context.simulator, - verbose=context.verbose, - args=test_args, - ) - ) - else: - print(f"Must specify test arguments (e.g., {sys.argv[0]} run -- test)") - print() - parser.print_help(sys.stderr) - sys.exit(21) - else: - parser.print_help(sys.stderr) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/iOS/testbed/iOSTestbed/dylib-Info-template.plist b/iOS/testbed/iOSTestbed/dylib-Info-template.plist deleted file mode 100644 index f652e272f71c88a..000000000000000 --- a/iOS/testbed/iOSTestbed/dylib-Info-template.plist +++ /dev/null @@ -1,26 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> -<plist version="1.0"> -<dict> - <key>CFBundleDevelopmentRegion</key> - <string>en</string> - <key>CFBundleExecutable</key> - <string></string> - <key>CFBundleIdentifier</key> - <string></string> - <key>CFBundleInfoDictionaryVersion</key> - <string>6.0</string> - <key>CFBundlePackageType</key> - <string>APPL</string> - <key>CFBundleShortVersionString</key> - <string>1.0</string> - <key>CFBundleSupportedPlatforms</key> - <array> - <string>iPhoneOS</string> - </array> - <key>MinimumOSVersion</key> - <string>12.0</string> - <key>CFBundleVersion</key> - <string>1</string> -</dict> -</plist> diff --git a/pyconfig.h.in b/pyconfig.h.in index 7586ad3f2667054..a7e18d42c2a604d 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -50,12 +50,6 @@ /* Define if getpgrp() must be called as getpgrp(0). */ #undef GETPGRP_HAVE_ARG -/* HACL* library can compile SIMD128 implementations */ -#undef HACL_CAN_COMPILE_SIMD128 - -/* HACL* library can compile SIMD256 implementations */ -#undef HACL_CAN_COMPILE_SIMD256 - /* Define if you have the 'accept' function. */ #undef HAVE_ACCEPT @@ -192,9 +186,15 @@ /* Define if you have the 'ctermid_r' function. */ #undef HAVE_CTERMID_R +/* Define if you have the 'ESCDELAY' variable. */ +#undef HAVE_CURSES_ESCDELAY + /* Define if you have the 'filter' function. */ #undef HAVE_CURSES_FILTER +/* Define if you have the 'getmouse' function with the X/Open signature. */ +#undef HAVE_CURSES_GETMOUSE + /* Define to 1 if you have the <curses.h> header file. */ #undef HAVE_CURSES_H @@ -216,9 +216,18 @@ /* Define if you have the 'resize_term' function. */ #undef HAVE_CURSES_RESIZE_TERM +/* Define if you have the 'set_escdelay' function. */ +#undef HAVE_CURSES_SET_ESCDELAY + +/* Define if you have the 'set_tabsize' function. */ +#undef HAVE_CURSES_SET_TABSIZE + /* Define if you have the 'syncok' function. */ #undef HAVE_CURSES_SYNCOK +/* Define if you have the 'TABSIZE' variable. */ +#undef HAVE_CURSES_TABSIZE + /* Define if you have the 'typeahead' function. */ #undef HAVE_CURSES_TYPEAHEAD @@ -267,6 +276,10 @@ */ #undef HAVE_DECL_TZNAME +/* Define to 1 if you have the declaration of 'UT_NAMESIZE', and to 0 if you + don't. */ +#undef HAVE_DECL_UT_NAMESIZE + /* Define to 1 if you have the device macros. */ #undef HAVE_DEVICE_MACROS @@ -539,6 +552,9 @@ /* Define to 1 if you have the 'getlogin' function. */ #undef HAVE_GETLOGIN +/* Define to 1 if you have the 'getlogin_r' function. */ +#undef HAVE_GETLOGIN_R + /* Define to 1 if you have the 'getnameinfo' function. */ #undef HAVE_GETNAMEINFO @@ -807,6 +823,9 @@ /* Define this if you have the makedev macro. */ #undef HAVE_MAKEDEV +/* Define if you have the 'MAXLOGNAME' constant. */ +#undef HAVE_MAXLOGNAME + /* Define to 1 if you have the 'mbrtowc' function. */ #undef HAVE_MBRTOWC @@ -1350,6 +1369,9 @@ /* Define to 1 if you have the 'sysconf' function. */ #undef HAVE_SYSCONF +/* Define to 1 if you have the 'sysctlbyname' function. */ +#undef HAVE_SYSCTLBYNAME + /* Define to 1 if you have the <sysexits.h> header file. */ #undef HAVE_SYSEXITS_H @@ -1454,6 +1476,9 @@ /* Define to 1 if you have the <sys/syscall.h> header file. */ #undef HAVE_SYS_SYSCALL_H +/* Define to 1 if you have the <sys/sysctl.h> header file. */ +#undef HAVE_SYS_SYSCTL_H + /* Define to 1 if you have the <sys/sysmacros.h> header file. */ #undef HAVE_SYS_SYSMACROS_H @@ -1575,6 +1600,9 @@ /* Define to 1 if you have the <utmp.h> header file. */ #undef HAVE_UTMP_H +/* Define if you have the 'HAVE_UT_NAMESIZE' constant. */ +#undef HAVE_UT_NAMESIZE + /* Define to 1 if you have the 'uuid_create' function. */ #undef HAVE_UUID_CREATE @@ -1584,6 +1612,9 @@ /* Define if uuid_generate_time_safe() exists. */ #undef HAVE_UUID_GENERATE_TIME_SAFE +/* Define if uuid_generate_time_safe() is able to deduce a MAC address. */ +#undef HAVE_UUID_GENERATE_TIME_SAFE_STABLE_MAC + /* Define to 1 if you have the <uuid.h> header file. */ #undef HAVE_UUID_H @@ -1630,12 +1661,18 @@ /* Define to 1 if you have the 'writev' function. */ #undef HAVE_WRITEV +/* Define to 1 if you have the <zdict.h> header file. */ +#undef HAVE_ZDICT_H + /* Define if the zlib library has inflateCopy */ #undef HAVE_ZLIB_COPY /* Define to 1 if you have the <zlib.h> header file. */ #undef HAVE_ZLIB_H +/* Define to 1 if you have the <zstd.h> header file. */ +#undef HAVE_ZSTD_H + /* Define to 1 if you have the '_getpty' function. */ #undef HAVE__GETPTY @@ -1714,9 +1751,6 @@ /* Defined if Python is built as a shared library. */ #undef Py_ENABLE_SHARED -/* Defined if _Complex C type can be used with libffi. */ -#undef Py_FFI_SUPPORT_C_COMPLEX - /* Define if you want to disable the GIL */ #undef Py_GIL_DISABLED @@ -1724,9 +1758,6 @@ SipHash13: 3, externally defined: 0 */ #undef Py_HASH_ALGORITHM -/* Define if year with century should be normalized for strftime. */ -#undef Py_NORMALIZE_CENTURY - /* Define if you want to enable remote debugging support. */ #undef Py_REMOTE_DEBUG @@ -2004,6 +2035,21 @@ /* Maximum length in bytes of a thread name */ #undef _PYTHREAD_NAME_MAXLEN +/* Defined if _Complex C type can be used with libffi. */ +#undef _Py_FFI_SUPPORT_C_COMPLEX + +/* HACL* library can compile SIMD128 implementations */ +#undef _Py_HACL_CAN_COMPILE_VEC128 + +/* HACL* library can compile SIMD256 implementations */ +#undef _Py_HACL_CAN_COMPILE_VEC256 + +/* Thread stack size set by the linker (in bytes). */ +#undef _Py_LINKER_THREAD_STACK_SIZE + +/* Define to 1 if the machine stack grows down (default); 0 if it grows up. */ +#undef _Py_STACK_GROWS_DOWN + /* Define to force use of thread-safe errno, h_errno, and other functions */ #undef _REENTRANT